diff --git a/.gitignore b/.gitignore
index 02146ef0ac5..e3d191dcbcc 100644
--- a/.gitignore
+++ b/.gitignore
@@ -44,7 +44,7 @@ autoconf/autom4te.cache
 #==============================================================================#
 # External projects that are tracked independently.
 projects/*
-!projects/CMakeLists.txt
+!projects/*.*
 !projects/Makefile
 # Clang, which is tracked independently.
 tools/clang
diff --git a/CMakeLists.txt b/CMakeLists.txt
index 7f9f40ffc84..b26e2692578 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -26,33 +26,29 @@ else()
   set(cmake_3_2_USES_TERMINAL USES_TERMINAL)
 endif()
 
-project(LLVM)
-
-# @LOCALMOD-BEGIN
-# Enable ccache for builds, if cmake is invoked with -DSYSTEM_HAS_CCACHE=ON.
-# Set CCACHE_CPP2 envvar, to avoid an error due to a strange ccache/clang++
-# interaction.  Specifically, errors about "argument unused during compilation".
-option(SYSTEM_HAS_CCACHE "Use ccache for building" OFF)
-if(SYSTEM_HAS_CCACHE)
-  set_property(GLOBAL PROPERTY RULE_LAUNCH_COMPILE ccache)
-  set(ENV(CCACHE_CPP2) "yes")
-  add_compile_options(-Qunused-arguments)
-endif()
-# @LOCALMOD-END
+project(LLVM C CXX ASM)
 
 # The following only works with the Ninja generator in CMake >= 3.0.
 set(LLVM_PARALLEL_COMPILE_JOBS "" CACHE STRING
   "Define the maximum number of concurrent compilation jobs.")
 if(LLVM_PARALLEL_COMPILE_JOBS)
-  set_property(GLOBAL APPEND PROPERTY JOB_POOLS compile_job_pool=${LLVM_PARALLEL_COMPILE_JOBS})
-  set(CMAKE_JOB_POOL_COMPILE compile_job_pool)
+  if(CMAKE_VERSION VERSION_LESS 3.0 OR NOT CMAKE_MAKE_PROGRAM MATCHES "ninja$")
+    message(WARNING "Job pooling is only available with Ninja generators and CMake 3.0 and later.")
+  else()
+    set_property(GLOBAL APPEND PROPERTY JOB_POOLS compile_job_pool=${LLVM_PARALLEL_COMPILE_JOBS})
+    set(CMAKE_JOB_POOL_COMPILE compile_job_pool)
+  endif()
 endif()
 
 set(LLVM_PARALLEL_LINK_JOBS "" CACHE STRING
   "Define the maximum number of concurrent link jobs.")
 if(LLVM_PARALLEL_LINK_JOBS)
-  set_property(GLOBAL APPEND PROPERTY JOB_POOLS link_job_pool=${LLVM_PARALLEL_LINK_JOBS})
-  set(CMAKE_JOB_POOL_LINK link_job_pool)
+  if(CMAKE_VERSION VERSION_LESS 3.0 OR NOT CMAKE_MAKE_PROGRAM MATCHES "ninja$")
+    message(WARNING "Job pooling is only available with Ninja generators and CMake 3.0 and later.")
+  else()
+    set_property(GLOBAL APPEND PROPERTY JOB_POOLS link_job_pool=${LLVM_PARALLEL_LINK_JOBS})
+    set(CMAKE_JOB_POOL_LINK link_job_pool)
+  endif()
 endif()
 
 # Add path for custom modules
@@ -63,7 +59,7 @@ set(CMAKE_MODULE_PATH
   )
 
 set(LLVM_VERSION_MAJOR 3)
-set(LLVM_VERSION_MINOR 7)
+set(LLVM_VERSION_MINOR 8)
 set(LLVM_VERSION_PATCH 0)
 set(LLVM_VERSION_SUFFIX svn)
 
@@ -105,6 +101,7 @@ set(CPACK_PACKAGE_VERSION_MINOR ${LLVM_VERSION_MINOR})
 set(CPACK_PACKAGE_VERSION_PATCH ${LLVM_VERSION_PATCH})
 set(CPACK_PACKAGE_VERSION ${PACKAGE_VERSION})
 set(CPACK_RESOURCE_FILE_LICENSE "${CMAKE_CURRENT_SOURCE_DIR}/LICENSE.TXT")
+set(CPACK_NSIS_COMPRESSOR "/SOLID lzma \r\n SetCompressorDictSize 32")
 if(WIN32 AND NOT UNIX)
   set(CPACK_PACKAGE_INSTALL_REGISTRY_KEY "LLVM")
   set(CPACK_PACKAGE_ICON "${CMAKE_CURRENT_SOURCE_DIR}\\\\cmake\\\\nsis_logo.bmp")
@@ -179,7 +176,9 @@ set(LLVM_INCLUDE_DIR ${CMAKE_CURRENT_BINARY_DIR}/include)
 
 set(LLVM_ALL_TARGETS
   AArch64
+  AMDGPU
   ARM
+  BPF
   CppBackend
   Hexagon
   JSBackend # @LOCALMOD
@@ -187,7 +186,6 @@ set(LLVM_ALL_TARGETS
   MSP430
   NVPTX
   PowerPC
-  R600
   Sparc
   SystemZ
   X86
@@ -234,13 +232,6 @@ option(LLVM_ENABLE_THREADS "Use threads if available." ON)
 
 option(LLVM_ENABLE_ZLIB "Use zlib for compression/decompression if available." ON)
 
-set(SUBZERO_TARGETS_TO_BUILD "none"
-    CACHE STRING "Semicolon-separated list of targets to build, or \"none\".")
-
-if( SUBZERO_TARGETS_TO_BUILD STREQUAL "none" )
-  set( SUBZERO_TARGETS_TO_BUILD "" )
-endif()
-
 if( LLVM_TARGETS_TO_BUILD STREQUAL "all" )
   set( LLVM_TARGETS_TO_BUILD ${LLVM_ALL_TARGETS} )
 endif()
@@ -486,11 +477,6 @@ foreach(t ${LLVM_TARGETS_TO_BUILD})
   endif()
 endforeach(t)
 
-set(SUBZERO_ENUM_TARGETS "")
-foreach(t ${SUBZERO_TARGETS_TO_BUILD})
-  set(SUBZERO_ENUM_TARGETS "${SUBZERO_ENUM_TARGETS}SUBZERO_TARGET(${t})\n")
-endforeach(t)
-
 # Produce the target definition files, which provide a way for clients to easily
 # include various classes of targets.
 configure_file(
@@ -505,10 +491,6 @@ configure_file(
   ${LLVM_MAIN_INCLUDE_DIR}/llvm/Config/Disassemblers.def.in
   ${LLVM_INCLUDE_DIR}/llvm/Config/Disassemblers.def
   )
-configure_file(
-  ${LLVM_MAIN_INCLUDE_DIR}/llvm/Config/SZTargets.def.in
-  ${LLVM_INCLUDE_DIR}/llvm/Config/SZTargets.def
-  )
 configure_file(
   ${LLVM_MAIN_INCLUDE_DIR}/llvm/Config/Targets.def.in
   ${LLVM_INCLUDE_DIR}/llvm/Config/Targets.def
@@ -549,7 +531,7 @@ endif()
 # check its symbols. This is wasteful (the check was done when foo.so
 # was created) and can fail since it is not the dynamic linker and
 # doesn't know how to handle search paths correctly.
-if (UNIX AND NOT APPLE)
+if (UNIX AND NOT APPLE AND NOT ${CMAKE_SYSTEM_NAME} MATCHES "SunOS")
   set(CMAKE_EXE_LINKER_FLAGS
       "${CMAKE_EXE_LINKER_FLAGS} -Wl,-allow-shlib-undefined")
 endif()
diff --git a/CODE_OWNERS.TXT b/CODE_OWNERS.TXT
index 4e8ff888929..a15f2919cc7 100644
--- a/CODE_OWNERS.TXT
+++ b/CODE_OWNERS.TXT
@@ -65,6 +65,10 @@ N: Hal Finkel
 E: hfinkel@anl.gov
 D: BBVectorize, the loop reroller, alias analysis and the PowerPC target
 
+N: Dan Gohman
+E: sunfish@mozilla.com
+D: WebAssembly Backend (lib/Target/WebAssembly/*)
+
 N: Renato Golin
 E: renato.golin@linaro.org
 D: ARM Linux support
@@ -89,6 +93,10 @@ N: Lang Hames
 E: lhames@gmail.com
 D: MCJIT, RuntimeDyld and JIT event listeners
 
+N: David Majnemer
+E: david.majnemer@gmail.com
+D: IR Constant Folder
+
 N: Galina Kistanova
 E: gkistanova@gmail.com
 D: LLVM Buildbot
@@ -182,3 +190,7 @@ D: libLTO, IR Linker
 N: Peter Zotov
 E: whitequark@whitequark.org
 D: OCaml bindings
+
+N: Andrey Churbanov
+E: andrey.churbanov@intel.com
+D: OpenMP runtime library
diff --git a/CREDITS.TXT b/CREDITS.TXT
index 40d67f4f2dc..da1fb010e35 100644
--- a/CREDITS.TXT
+++ b/CREDITS.TXT
@@ -152,8 +152,9 @@ E: foldr@codedgers.com
 D: Author of llvmc2
 
 N: Dan Gohman
-E: dan433584@gmail.com
+E: sunfish@mozilla.com
 D: Miscellaneous bug fixes
+D: WebAssembly Backend
 
 N: David Goodwin
 E: david@goodwinz.net
diff --git a/Makefile b/Makefile
index e9559af0e67..2b9468af205 100644
--- a/Makefile
+++ b/Makefile
@@ -43,11 +43,6 @@ EXTRA_DIST := test unittests llvm.spec include win32 Xcode
 
 include $(LEVEL)/Makefile.config
 
-ifeq ($(PNACL_BROWSER_TRANSLATOR),1)
-  DIRS := $(filter-out tools/llvm-shlib runtime docs unittests, $(DIRS))
-  OPTIONAL_DIRS :=
-endif
-
 ifneq ($(ENABLE_SHARED),1)
   DIRS := $(filter-out tools/llvm-shlib, $(DIRS))
 endif
@@ -111,15 +106,12 @@ all:: cross-compile-build-tools
 clean::
 	$(Verb) rm -rf BuildTools
 
-# @LOCALMOD-START Pass BUILD_CFLAGS et al through for host libcxx;
-# Only required to build on Ubuntu Precise
 cross-compile-build-tools:
 	$(Verb) if [ ! -f BuildTools/Makefile ]; then \
           $(MKDIR) BuildTools; \
 	  cd BuildTools ; \
-	  CFLAGS="$(BUILD_CFLAGS)" ; \
-	  CXXFLAGS="$(BUILD_CXXFLAGS)" ; \
-	  LDFLAGS="$(BUILD_LDFLAGS)" ; \
+	  unset CFLAGS ; \
+	  unset CXXFLAGS ; \
 	  AR=$(BUILD_AR) ; \
 	  AS=$(BUILD_AS) ; \
 	  LD=$(BUILD_LD) ; \
@@ -138,7 +130,6 @@ cross-compile-build-tools:
 	fi; \
 	($(MAKE) -C BuildTools \
 	  BUILD_DIRS_ONLY=1 \
-	  PNACL_BROWSER_TRANSLATOR=0 \
 	  UNIVERSAL= \
 	  UNIVERSAL_SDK_PATH= \
 	  SDKROOT= \
@@ -151,14 +142,10 @@ cross-compile-build-tools:
 	  DISABLE_ASSERTIONS=$(DISABLE_ASSERTIONS) \
 	  ENABLE_EXPENSIVE_CHECKS=$(ENABLE_EXPENSIVE_CHECKS) \
 	  ENABLE_LIBCPP=$(ENABLE_LIBCPP) \
-	  CC=$(BUILD_CC) \
-	  CXX=$(BUILD_CXX) \
-	  CFLAGS="$(BUILD_CFLAGS)" \
-	  CXXFLAGS="$(BUILD_CXXFLAGS)" \
-	  LDFLAGS="$(BUILD_LDFLAGS)" \
+	  CFLAGS= \
+	  CXXFLAGS= \
 	) || exit 1;
 endif
-# @LOCALMOD-END
 
 # Include the main makefile machinery.
 include $(LLVM_SRC_ROOT)/Makefile.rules
@@ -214,7 +201,6 @@ FilesToConfig := \
   llvm.spec \
   include/llvm/Config/config.h \
   include/llvm/Config/llvm-config.h \
-  include/llvm/Config/SZTargets.def \
   include/llvm/Config/Targets.def \
   include/llvm/Config/AsmPrinters.def \
   include/llvm/Config/AsmParsers.def \
diff --git a/Makefile.config.in b/Makefile.config.in
index f4354d5628f..3258714ac59 100644
--- a/Makefile.config.in
+++ b/Makefile.config.in
@@ -58,15 +58,16 @@ LLVM_OBJ_ROOT   := $(call realpath, @abs_top_builddir@)
 PROJ_SRC_ROOT   := $(LLVM_SRC_ROOT)
 PROJ_SRC_DIR    := $(LLVM_SRC_ROOT)$(patsubst $(PROJ_OBJ_ROOT)%,%,$(PROJ_OBJ_DIR))
 
+# See: http://lists.cs.uiuc.edu/pipermail/llvm-commits/Week-of-Mon-20150323/268067.html
+ifeq ($(LLVM_SRC_ROOT), $(LLVM_OBJ_ROOT))
+  $(error In-source builds are not allowed. Please configure from a separate build directory!)
+endif
+
 ifneq ($(CLANG_SRC_ROOT),)
   CLANG_SRC_ROOT:= $(call realpath, $(CLANG_SRC_ROOT))
   PROJ_SRC_DIR  := $(patsubst $(LLVM_SRC_ROOT)/tools/clang%,$(CLANG_SRC_ROOT)%,$(PROJ_SRC_DIR))
 endif
 
-ifneq ($(SUBZERO_SRC_ROOT),)
-  PROJ_SRC_DIR  := $(patsubst $(LLVM_SRC_ROOT)/tools/subzero%,$(SUBZERO_SRC_ROOT)%,$(PROJ_SRC_DIR))
-endif
-
 prefix          := @prefix@
 PROJ_prefix     := $(prefix)
 program_prefix  := @program_prefix@
diff --git a/Makefile.rules b/Makefile.rules
index 2a01833cc80..24cac3b3765 100644
--- a/Makefile.rules
+++ b/Makefile.rules
@@ -322,9 +322,9 @@ endif
 ifeq ($(ENABLE_CXX1Y),1)
   CXX.Flags += -std=c++1y
 else
-  ifeq ($(HOST_OS), $(filter $(HOST_OS), Cygwin MingW NativeClient))
-    # MinGW, Cygwin, and NativeClient+newlib are a bit stricter and
-    # lack things like 'strdup', 'stricmp', etc in c++11 mode.
+  ifeq ($(HOST_OS), $(filter $(HOST_OS), Cygwin MingW))
+    # MinGW and Cygwin are a bit stricter and lack things like
+    # 'strdup', 'stricmp', etc in c++11 mode.
     CXX.Flags += -std=gnu++11
  else
     CXX.Flags += -std=c++11
@@ -671,11 +671,6 @@ ifneq ($(HOST_OS), $(filter $(HOST_OS), Cygwin MingW))
   endif
 endif
 
-#@LOCALMOD-BEGIN
-ifeq ($(PNACL_BROWSER_TRANSLATOR),1)
-  CPP.Flags += -DPNACL_BROWSER_TRANSLATOR
-endif
-# @LOCALMOD-END
 
 #----------------------------------------------------------
 # Options To Invoke Tools
diff --git a/OWNERS b/OWNERS
deleted file mode 100644
index ac015a5aadb..00000000000
--- a/OWNERS
+++ /dev/null
@@ -1,8 +0,0 @@
-dschuff@chromium.org
-eliben@chromium.org
-jfb@chromium.org
-jvoung@chromium.org
-kschimpf@chromium.org
-mseaborn@chromium.org
-sehr@chromium.org
-stichnot@chromium.org
diff --git a/PRESUBMIT.py b/PRESUBMIT.py
deleted file mode 100644
index d81168ea093..00000000000
--- a/PRESUBMIT.py
+++ /dev/null
@@ -1,66 +0,0 @@
-# Copyright (c) 2012 The Native Client Authors. All rights reserved.
-# Use of this source code is governed by a BSD-style license that can be
-# found in the LICENSE file.
-
-# Documentation on PRESUBMIT.py can be found at:
-# http://www.chromium.org/developers/how-tos/depottools/presubmit-scripts
-
-EXCLUDE_PROJECT_CHECKS_DIRS = [ '.' ]
-
-import subprocess
-def CheckGitBranch():
-  p = subprocess.Popen("git branch -vv", shell=True,
-                       stdout=subprocess.PIPE)
-  output, _ = p.communicate()
-
-  lines = output.split('\n')
-  for line in lines:
-    # output format for checked-out branch should be
-    # * branchname hash [TrackedBranchName ...
-    toks = line.split()
-    if '*' not in toks[0]:
-      continue
-    if not 'origin/master' in toks[3]:
-      warning = 'Warning: your current branch:\n' + line
-      warning += '\nis not tracking origin/master. git cl push may silently '
-      warning += 'fail to push your change. To fix this, do\n'
-      warning += 'git branch -u origin/master'
-      return warning
-    return None
-  print 'Warning: presubmit check could not determine local git branch'
-  return None
-
-def _CommonChecks(input_api, output_api):
-  """Checks for both upload and commit."""
-  results = []
-  results.extend(input_api.canned_checks.PanProjectChecks(
-      input_api, output_api, project_name='Native Client',
-      excluded_paths=tuple(EXCLUDE_PROJECT_CHECKS_DIRS)))
-  branch_warning = CheckGitBranch()
-  if branch_warning:
-    results.append(output_api.PresubmitPromptWarning(branch_warning))
-  return results
-
-def CheckChangeOnUpload(input_api, output_api):
-  """Verifies all changes in all files.
-  Args:
-    input_api: the limited set of input modules allowed in presubmit.
-    output_api: the limited set of output modules allowed in presubmit.
-  """
-  report = []
-  report.extend(_CommonChecks(input_api, output_api))
-  return report
-
-def CheckChangeOnCommit(input_api, output_api):
-  """Verifies all changes in all files and verifies that the
-  tree is open and can accept a commit.
-  Args:
-    input_api: the limited set of input modules allowed in presubmit.
-    output_api: the limited set of output modules allowed in presubmit.
-  """
-  report = []
-  report.extend(CheckChangeOnUpload(input_api, output_api))
-  return report
-
-def GetPreferredTrySlaves(project, change):
-  return []
diff --git a/autoconf/config.guess b/autoconf/config.guess
index 73d671bc2c6..b209a1a06c6 100755
--- a/autoconf/config.guess
+++ b/autoconf/config.guess
@@ -810,6 +810,9 @@ EOF
     *:MINGW*:*)
 	echo ${UNAME_MACHINE}-pc-mingw32
 	exit ;;
+    *:MSYS*:*)
+	echo ${UNAME_MACHINE}-pc-msys
+	exit ;;
     i*:windows32*:*)
 	# uname -m includes "-pc" on this system.
 	echo ${UNAME_MACHINE}-mingw32
diff --git a/autoconf/config.sub b/autoconf/config.sub
index 4ef86e33fb3..2583c901235 100755
--- a/autoconf/config.sub
+++ b/autoconf/config.sub
@@ -239,10 +239,6 @@ case $os in
 		basic_machine=m68k-atari
 		os=-mint
 		;;
-        -nacl*)
-                basic_machine=i686-pc
-                os=-nacl
-                ;;
 esac
 
 # Decode aliases for certain CPU-COMPANY combinations.
@@ -352,14 +348,6 @@ case $basic_machine in
 	i*86 | x86_64)
 	  basic_machine=$basic_machine-pc
 	  ;;
-        nacl64*)
-          basic_machine=x86_64-pc
-          os=-nacl
-          ;;
-        nacl*)
-          basic_machine=i686-pc
-          os=-nacl
-          ;;
 	# Object if more than one company name word.
 	*-*-*)
 		echo Invalid configuration \`$1\': machine \`$basic_machine\' not recognized 1>&2
@@ -1378,9 +1366,6 @@ case $os in
 			;;
 		esac
 		;;
-        -nacl*)
-                os=-nacl
-                ;;
 	-nto-qnx*)
 		;;
 	-nto*)
diff --git a/autoconf/configure.ac b/autoconf/configure.ac
index fb5c4cc6d68..2968ec79a0b 100644
--- a/autoconf/configure.ac
+++ b/autoconf/configure.ac
@@ -32,10 +32,10 @@ dnl===-----------------------------------------------------------------------===
 dnl Initialize autoconf and define the package name, version number and
 dnl address for reporting bugs.
 
-AC_INIT([LLVM],[3.7.0svn],[http://llvm.org/bugs/])
+AC_INIT([LLVM],[3.8.0svn],[http://llvm.org/bugs/])
 
 LLVM_VERSION_MAJOR=3
-LLVM_VERSION_MINOR=7
+LLVM_VERSION_MINOR=8
 LLVM_VERSION_PATCH=0
 LLVM_VERSION_SUFFIX=svn
 
@@ -73,6 +73,11 @@ if test ${srcdir} != "." ; then
   fi
 fi
 
+dnl Quit if it is an in-source build
+if test ${srcdir} == "." ; then
+  AC_MSG_ERROR([In-source builds are not allowed. Please configure from a separate build directory!])
+fi
+
 dnl Default to empty (i.e. assigning the null string to) CFLAGS and CXXFLAGS,
 dnl instead of the autoconf default (for example, '-g -O2' for CC=gcc).
 : ${CFLAGS=}
@@ -105,7 +110,7 @@ if test "$CXX" = "clang++" ; then
 ],
 [
   AC_MSG_RESULT([no])
-  AC_MSG_ERROR([Selected compiler could not find or parse C++ standard library headers.  Rerun with CC=c-compiler CXX=c++-compiler ./configure ...])
+  AC_MSG_ERROR([Selected compiler could not find or parse C++ standard library headers.  Rerun with CC=c-compiler CXX=c++-compiler LLVM_SRC_DIR/configure ...])
 ])
   AC_LANG_POP([C++])
 fi
@@ -322,11 +327,6 @@ AC_CACHE_CHECK([type of operating system we're going to host on],
     llvm_cv_no_link_all_option="-Wl,--no-whole-archive"
     llvm_cv_os_type="Freestanding"
     llvm_cv_platform_type="Unix" ;;
-  *-*-nacl*)
-    llvm_cv_link_all_option="-Wl,--whole-archive"
-    llvm_cv_no_link_all_option="-Wl,--no-whole-archive"
-    llvm_cv_os_type="NativeClient"
-    llvm_cv_platform_type="Unix" ;;
   *)
     llvm_cv_link_all_option=""
     llvm_cv_no_link_all_option=""
@@ -437,6 +437,7 @@ AC_CACHE_CHECK([target architecture],[llvm_cv_target_arch],
   hexagon-*)              llvm_cv_target_arch="Hexagon" ;;
   nvptx-*)                llvm_cv_target_arch="NVPTX" ;;
   s390x-*)                llvm_cv_target_arch="SystemZ" ;;
+  wasm*-*)                llvm_cv_target_arch="WebAssembly" ;;
   *)                      llvm_cv_target_arch="Unknown" ;;
 esac])
 
@@ -472,6 +473,7 @@ case $host in
   msp430-*)               host_arch="MSP430" ;;
   hexagon-*)              host_arch="Hexagon" ;;
   s390x-*)                host_arch="SystemZ" ;;
+  wasm*-*)                host_arch="WebAssembly" ;;
   *)                      host_arch="Unknown" ;;
 esac
 
@@ -804,6 +806,7 @@ else
     Hexagon)     AC_SUBST(TARGET_HAS_JIT,0) ;;
     NVPTX)       AC_SUBST(TARGET_HAS_JIT,0) ;;
     SystemZ)     AC_SUBST(TARGET_HAS_JIT,1) ;;
+    WebAssembly) AC_SUBST(TARGET_HAS_JIT,0) ;;
     *)           AC_SUBST(TARGET_HAS_JIT,0) ;;
   esac
 fi
@@ -1089,7 +1092,7 @@ if test "$llvm_cv_enable_crash_overrides" = "yes" ; then
 fi
 
 dnl List all possible targets
-ALL_TARGETS="X86 Sparc PowerPC ARM AArch64 Mips XCore MSP430 CppBackend JSBackend NVPTX Hexagon SystemZ R600"
+ALL_TARGETS="X86 Sparc PowerPC ARM AArch64 Mips XCore MSP430 CppBackend NVPTX Hexagon SystemZ AMDGPU BPF JSBackend"
 AC_SUBST(ALL_TARGETS,$ALL_TARGETS)
 
 dnl Allow specific targets to be specified for building (or not)
@@ -1097,7 +1100,7 @@ TARGETS_TO_BUILD=""
 AC_ARG_ENABLE([targets],AS_HELP_STRING([--enable-targets],
     [Build specific host targets: all or target1,target2,... Valid targets are:
      host, x86, x86_64, sparc, powerpc, arm64, arm, aarch64, mips, hexagon,
-     xcore, msp430, nvptx, systemz, r600, js, and cpp (default=all)]),,
+     xcore, msp430, nvptx, systemz, r600, bpf, wasm, js, and cpp (default=all)]),,
     enableval=all)
 if test "$enableval" = host-only ; then
   enableval=host
@@ -1113,6 +1116,7 @@ case "$enableval" in
         aarch64)  TARGETS_TO_BUILD="AArch64 $TARGETS_TO_BUILD" ;;
         arm64)    TARGETS_TO_BUILD="AArch64 $TARGETS_TO_BUILD" ;;
         arm)      TARGETS_TO_BUILD="ARM $TARGETS_TO_BUILD" ;;
+        bpf)      TARGETS_TO_BUILD="BPF $TARGETS_TO_BUILD" ;;
         mips)     TARGETS_TO_BUILD="Mips $TARGETS_TO_BUILD" ;;
         mipsel)   TARGETS_TO_BUILD="Mips $TARGETS_TO_BUILD" ;;
         mips64)   TARGETS_TO_BUILD="Mips $TARGETS_TO_BUILD" ;;
@@ -1124,7 +1128,9 @@ case "$enableval" in
         hexagon)  TARGETS_TO_BUILD="Hexagon $TARGETS_TO_BUILD" ;;
         nvptx)    TARGETS_TO_BUILD="NVPTX $TARGETS_TO_BUILD" ;;
         systemz)  TARGETS_TO_BUILD="SystemZ $TARGETS_TO_BUILD" ;;
-        r600)     TARGETS_TO_BUILD="R600 $TARGETS_TO_BUILD" ;;
+        amdgpu)   TARGETS_TO_BUILD="AMDGPU $TARGETS_TO_BUILD" ;;
+        r600)     TARGETS_TO_BUILD="AMDGPU $TARGETS_TO_BUILD" ;;
+        wasm)     TARGETS_TO_BUILD="WebAssembly $TARGETS_TO_BUILD" ;;
         host) case "$llvm_cv_target_arch" in
             x86)         TARGETS_TO_BUILD="X86 $TARGETS_TO_BUILD" ;;
             x86_64)      TARGETS_TO_BUILD="X86 $TARGETS_TO_BUILD" ;;
@@ -1138,6 +1144,7 @@ case "$enableval" in
             Hexagon)     TARGETS_TO_BUILD="Hexagon $TARGETS_TO_BUILD" ;;
             NVPTX)       TARGETS_TO_BUILD="NVPTX $TARGETS_TO_BUILD" ;;
             SystemZ)     TARGETS_TO_BUILD="SystemZ $TARGETS_TO_BUILD" ;;
+            WebAssembly) TARGETS_TO_BUILD="WebAssembly $TARGETS_TO_BUILD" ;;
             *)       AC_MSG_ERROR([Can not set target to build]) ;;
           esac ;;
         *) AC_MSG_ERROR([Unrecognized target $a_target]) ;;
@@ -1216,34 +1223,6 @@ AC_SUBST(LLVM_ENUM_ASM_PRINTERS)
 AC_SUBST(LLVM_ENUM_ASM_PARSERS)
 AC_SUBST(LLVM_ENUM_DISASSEMBLERS)
 
-
-dnl Allow specific Subzero targets to be specified for building (or not)
-SUBZERO_TARGETS_TO_BUILD=""
-AC_ARG_ENABLE([subzero_targets],AS_HELP_STRING([--enable-subzero-targets],
-    [Build specific subzero targets: target1,target2,...(default=none)]),,
-    enableval=none)
-case "$enableval" in
-  none) ;;
-  *)for a_target in `echo $enableval|sed -e 's/,/ /g' ` ; do
-      case "$a_target" in
-        X8632)  SUBZERO_TARGETS_TO_BUILD="X8632 $SUBZERO_TARGETS_TO_BUILD" ;;
-        X8664)  SUBZERO_TARGETS_TO_BUILD="X8664 $SUBZERO_TARGETS_TO_BUILD" ;;
-        ARM32)  SUBZERO_TARGETS_TO_BUILD="ARM32 $SUBZERO_TARGETS_TO_BUILD" ;;
-        ARM64)  SUBZERO_TARGETS_TO_BUILD="ARM64 $SUBZERO_TARGETS_TO_BUILD" ;;
-        MIPS32) SUBZERO_TARGETS_TO_BUILD="MIPS32 $SUBZERO_TARGETS_TO_BUILD" ;;
-        *) AC_MSG_ERROR([Unrecognized subzero target $a_target]) ;;
-      esac
-  done
-  ;;
-esac
-
-dnl Build the SUBZERO_ENUM_TARGETS macro for SZTargets.def.
-SUBZERO_ENUM_TARGETS=""
-for target_to_build in $SUBZERO_TARGETS_TO_BUILD; do
-  SUBZERO_ENUM_TARGETS="SUBZERO_TARGET($target_to_build) $SUBZERO_ENUM_TARGETS"
-done
-AC_SUBST(SUBZERO_ENUM_TARGETS)
-
 dnl Override the option to use for optimized builds.
 AC_ARG_WITH(optimize-option,
   AS_HELP_STRING([--with-optimize-option],
@@ -1358,6 +1337,13 @@ AC_ARG_WITH(default-sysroot,
 AC_DEFINE_UNQUOTED(DEFAULT_SYSROOT,"$withval",
                    [Default <path> to all compiler invocations for --sysroot=<path>.])
 
+AC_ARG_WITH(clang-default-openmp-runtime,
+  AS_HELP_STRING([--with-clang-default-openmp-runtime],
+    [The default OpenMP runtime for Clang.]),,
+    withval="libgomp")
+AC_DEFINE_UNQUOTED(CLANG_DEFAULT_OPENMP_RUNTIME,"$withval",
+                   [Default OpenMP runtime used by -fopenmp.])
+
 dnl Allow linking of LLVM with GPLv3 binutils code.
 AC_ARG_WITH(binutils-include,
   AS_HELP_STRING([--with-binutils-include],
@@ -1643,7 +1629,7 @@ dnl===-----------------------------------------------------------------------===
 
 AC_CHECK_LIB(m,sin)
 if test "$llvm_cv_os_type" = "MingW" ; then
-  AC_CHECK_LIB(imagehlp, main)
+  AC_CHECK_LIB(ole32, main)
   AC_CHECK_LIB(psapi, main)
   AC_CHECK_LIB(shell32, main)
 fi
@@ -2184,10 +2170,12 @@ dnl contains the same list of files as AC_CONFIG_HEADERS below. This ensures the
 dnl files can be updated automatically when their *.in sources change.
 AC_CONFIG_HEADERS([include/llvm/Config/config.h include/llvm/Config/llvm-config.h])
 AH_TOP([#ifndef CONFIG_H
-#define CONFIG_H])
+#define CONFIG_H
+
+/* Exported configuration */
+#include "llvm/Config/llvm-config.h"])
 AH_BOTTOM([#endif])
 
-AC_CONFIG_FILES([include/llvm/Config/SZTargets.def])
 AC_CONFIG_FILES([include/llvm/Config/Targets.def])
 AC_CONFIG_FILES([include/llvm/Config/AsmPrinters.def])
 AC_CONFIG_FILES([include/llvm/Config/AsmParsers.def])
diff --git a/bindings/go/llvm/DIBuilderBindings.cpp b/bindings/go/llvm/DIBuilderBindings.cpp
index cfae6053bfe..627c09131aa 100644
--- a/bindings/go/llvm/DIBuilderBindings.cpp
+++ b/bindings/go/llvm/DIBuilderBindings.cpp
@@ -40,16 +40,14 @@ LLVMMetadataRef LLVMDIBuilderCreateCompileUnit(LLVMDIBuilderRef Dref,
                                                int Optimized, const char *Flags,
                                                unsigned RuntimeVersion) {
   DIBuilder *D = unwrap(Dref);
-  DICompileUnit CU = D->createCompileUnit(Lang, File, Dir, Producer, Optimized,
-                                          Flags, RuntimeVersion);
-  return wrap(CU);
+  return wrap(D->createCompileUnit(Lang, File, Dir, Producer, Optimized, Flags,
+                                   RuntimeVersion));
 }
 
 LLVMMetadataRef LLVMDIBuilderCreateFile(LLVMDIBuilderRef Dref, const char *File,
                                         const char *Dir) {
   DIBuilder *D = unwrap(Dref);
-  DIFile F = D->createFile(File, Dir);
-  return wrap(F);
+  return wrap(D->createFile(File, Dir));
 }
 
 LLVMMetadataRef LLVMDIBuilderCreateLexicalBlock(LLVMDIBuilderRef Dref,
@@ -58,8 +56,8 @@ LLVMMetadataRef LLVMDIBuilderCreateLexicalBlock(LLVMDIBuilderRef Dref,
                                                 unsigned Line,
                                                 unsigned Column) {
   DIBuilder *D = unwrap(Dref);
-  auto *LB = D->createLexicalBlock(unwrap<MDLocalScope>(Scope),
-                                   unwrap<MDFile>(File), Line, Column);
+  auto *LB = D->createLexicalBlock(unwrap<DILocalScope>(Scope),
+                                   unwrap<DIFile>(File), Line, Column);
   return wrap(LB);
 }
 
@@ -68,9 +66,8 @@ LLVMMetadataRef LLVMDIBuilderCreateLexicalBlockFile(LLVMDIBuilderRef Dref,
                                                     LLVMMetadataRef File,
                                                     unsigned Discriminator) {
   DIBuilder *D = unwrap(Dref);
-  DILexicalBlockFile LBF = D->createLexicalBlockFile(
-      unwrap<MDLocalScope>(Scope), unwrap<MDFile>(File), Discriminator);
-  return wrap(LBF);
+  return wrap(D->createLexicalBlockFile(unwrap<DILocalScope>(Scope),
+                                        unwrap<DIFile>(File), Discriminator));
 }
 
 LLVMMetadataRef LLVMDIBuilderCreateFunction(
@@ -79,23 +76,26 @@ LLVMMetadataRef LLVMDIBuilderCreateFunction(
     LLVMMetadataRef CompositeType, int IsLocalToUnit, int IsDefinition,
     unsigned ScopeLine, unsigned Flags, int IsOptimized, LLVMValueRef Func) {
   DIBuilder *D = unwrap(Dref);
-  DISubprogram SP = D->createFunction(
-      unwrap<MDScope>(Scope), Name, LinkageName,
-      File ? unwrap<MDFile>(File) : nullptr, Line,
-      unwrap<MDSubroutineType>(CompositeType), IsLocalToUnit, IsDefinition,
-      ScopeLine, Flags, IsOptimized, unwrap<Function>(Func));
-  return wrap(SP);
+  return wrap(D->createFunction(unwrap<DIScope>(Scope), Name, LinkageName,
+                                File ? unwrap<DIFile>(File) : nullptr, Line,
+                                unwrap<DISubroutineType>(CompositeType),
+                                IsLocalToUnit, IsDefinition, ScopeLine, Flags,
+                                IsOptimized, unwrap<Function>(Func)));
 }
 
 LLVMMetadataRef LLVMDIBuilderCreateLocalVariable(
-    LLVMDIBuilderRef Dref, unsigned Tag, LLVMMetadataRef Scope,
+    LLVMDIBuilderRef Dref, unsigned, LLVMMetadataRef Scope,
     const char *Name, LLVMMetadataRef File, unsigned Line, LLVMMetadataRef Ty,
     int AlwaysPreserve, unsigned Flags, unsigned ArgNo) {
   DIBuilder *D = unwrap(Dref);
-  DIVariable V = D->createLocalVariable(
-      Tag, unwrap<MDScope>(Scope), Name, unwrap<MDFile>(File), Line,
-      unwrap<MDType>(Ty), AlwaysPreserve, Flags, ArgNo);
-  return wrap(V);
+  // FIXME: Update the Go bindings to match the DIBuilder API.
+  if (ArgNo)
+    return wrap(D->createParameterVariable(
+        unwrap<DIScope>(Scope), Name, ArgNo, unwrap<DIFile>(File), Line,
+        unwrap<DIType>(Ty), AlwaysPreserve, Flags));
+  return wrap(D->createAutoVariable(unwrap<DIScope>(Scope), Name,
+                                    unwrap<DIFile>(File), Line,
+                                    unwrap<DIType>(Ty), AlwaysPreserve, Flags));
 }
 
 LLVMMetadataRef LLVMDIBuilderCreateBasicType(LLVMDIBuilderRef Dref,
@@ -104,8 +104,7 @@ LLVMMetadataRef LLVMDIBuilderCreateBasicType(LLVMDIBuilderRef Dref,
                                              uint64_t AlignInBits,
                                              unsigned Encoding) {
   DIBuilder *D = unwrap(Dref);
-  DIBasicType T = D->createBasicType(Name, SizeInBits, AlignInBits, Encoding);
-  return wrap(T);
+  return wrap(D->createBasicType(Name, SizeInBits, AlignInBits, Encoding));
 }
 
 LLVMMetadataRef LLVMDIBuilderCreatePointerType(LLVMDIBuilderRef Dref,
@@ -114,19 +113,17 @@ LLVMMetadataRef LLVMDIBuilderCreatePointerType(LLVMDIBuilderRef Dref,
                                                uint64_t AlignInBits,
                                                const char *Name) {
   DIBuilder *D = unwrap(Dref);
-  DIDerivedType T = D->createPointerType(unwrap<MDType>(PointeeType),
-                                         SizeInBits, AlignInBits, Name);
-  return wrap(T);
+  return wrap(D->createPointerType(unwrap<DIType>(PointeeType), SizeInBits,
+                                   AlignInBits, Name));
 }
 
 LLVMMetadataRef
 LLVMDIBuilderCreateSubroutineType(LLVMDIBuilderRef Dref, LLVMMetadataRef File,
                                   LLVMMetadataRef ParameterTypes) {
   DIBuilder *D = unwrap(Dref);
-  DICompositeType CT =
-      D->createSubroutineType(File ? unwrap<MDFile>(File) : nullptr,
-                              DITypeArray(unwrap<MDTuple>(ParameterTypes)));
-  return wrap(CT);
+  return wrap(
+      D->createSubroutineType(File ? unwrap<DIFile>(File) : nullptr,
+                              DITypeRefArray(unwrap<MDTuple>(ParameterTypes))));
 }
 
 LLVMMetadataRef LLVMDIBuilderCreateStructType(
@@ -135,12 +132,11 @@ LLVMMetadataRef LLVMDIBuilderCreateStructType(
     uint64_t AlignInBits, unsigned Flags, LLVMMetadataRef DerivedFrom,
     LLVMMetadataRef ElementTypes) {
   DIBuilder *D = unwrap(Dref);
-  DICompositeType CT = D->createStructType(
-      unwrap<MDScope>(Scope), Name, File ? unwrap<MDFile>(File) : nullptr, Line,
+  return wrap(D->createStructType(
+      unwrap<DIScope>(Scope), Name, File ? unwrap<DIFile>(File) : nullptr, Line,
       SizeInBits, AlignInBits, Flags,
-      DerivedFrom ? unwrap<MDType>(DerivedFrom) : nullptr,
-      ElementTypes ? DIArray(unwrap<MDTuple>(ElementTypes)) : nullptr);
-  return wrap(CT);
+      DerivedFrom ? unwrap<DIType>(DerivedFrom) : nullptr,
+      ElementTypes ? DINodeArray(unwrap<MDTuple>(ElementTypes)) : nullptr));
 }
 
 LLVMMetadataRef LLVMDIBuilderCreateReplaceableCompositeType(
@@ -149,10 +145,9 @@ LLVMMetadataRef LLVMDIBuilderCreateReplaceableCompositeType(
     unsigned RuntimeLang, uint64_t SizeInBits, uint64_t AlignInBits,
     unsigned Flags) {
   DIBuilder *D = unwrap(Dref);
-  DICompositeType CT = D->createReplaceableCompositeType(
-      Tag, Name, unwrap<MDScope>(Scope), File ? unwrap<MDFile>(File) : nullptr,
-      Line, RuntimeLang, SizeInBits, AlignInBits, Flags);
-  return wrap(CT);
+  return wrap(D->createReplaceableCompositeType(
+      Tag, Name, unwrap<DIScope>(Scope), File ? unwrap<DIFile>(File) : nullptr,
+      Line, RuntimeLang, SizeInBits, AlignInBits, Flags));
 }
 
 LLVMMetadataRef
@@ -162,10 +157,9 @@ LLVMDIBuilderCreateMemberType(LLVMDIBuilderRef Dref, LLVMMetadataRef Scope,
                               uint64_t AlignInBits, uint64_t OffsetInBits,
                               unsigned Flags, LLVMMetadataRef Ty) {
   DIBuilder *D = unwrap(Dref);
-  DIDerivedType DT = D->createMemberType(
-      unwrap<MDScope>(Scope), Name, File ? unwrap<MDFile>(File) : nullptr, Line,
-      SizeInBits, AlignInBits, OffsetInBits, Flags, unwrap<MDType>(Ty));
-  return wrap(DT);
+  return wrap(D->createMemberType(
+      unwrap<DIScope>(Scope), Name, File ? unwrap<DIFile>(File) : nullptr, Line,
+      SizeInBits, AlignInBits, OffsetInBits, Flags, unwrap<DIType>(Ty)));
 }
 
 LLVMMetadataRef LLVMDIBuilderCreateArrayType(LLVMDIBuilderRef Dref,
@@ -174,10 +168,9 @@ LLVMMetadataRef LLVMDIBuilderCreateArrayType(LLVMDIBuilderRef Dref,
                                              LLVMMetadataRef ElementType,
                                              LLVMMetadataRef Subscripts) {
   DIBuilder *D = unwrap(Dref);
-  DICompositeType CT =
-      D->createArrayType(SizeInBits, AlignInBits, unwrap<MDType>(ElementType),
-                         DIArray(unwrap<MDTuple>(Subscripts)));
-  return wrap(CT);
+  return wrap(D->createArrayType(SizeInBits, AlignInBits,
+                                 unwrap<DIType>(ElementType),
+                                 DINodeArray(unwrap<MDTuple>(Subscripts))));
 }
 
 LLVMMetadataRef LLVMDIBuilderCreateTypedef(LLVMDIBuilderRef Dref,
@@ -185,17 +178,15 @@ LLVMMetadataRef LLVMDIBuilderCreateTypedef(LLVMDIBuilderRef Dref,
                                            LLVMMetadataRef File, unsigned Line,
                                            LLVMMetadataRef Context) {
   DIBuilder *D = unwrap(Dref);
-  DIDerivedType DT = D->createTypedef(
-      unwrap<MDType>(Ty), Name, File ? unwrap<MDFile>(File) : nullptr, Line,
-      Context ? unwrap<MDScope>(Context) : nullptr);
-  return wrap(DT);
+  return wrap(D->createTypedef(unwrap<DIType>(Ty), Name,
+                               File ? unwrap<DIFile>(File) : nullptr, Line,
+                               Context ? unwrap<DIScope>(Context) : nullptr));
 }
 
 LLVMMetadataRef LLVMDIBuilderGetOrCreateSubrange(LLVMDIBuilderRef Dref,
                                                  int64_t Lo, int64_t Count) {
   DIBuilder *D = unwrap(Dref);
-  DISubrange S = D->getOrCreateSubrange(Lo, Count);
-  return wrap(S);
+  return wrap(D->getOrCreateSubrange(Lo, Count));
 }
 
 LLVMMetadataRef LLVMDIBuilderGetOrCreateArray(LLVMDIBuilderRef Dref,
@@ -204,7 +195,7 @@ LLVMMetadataRef LLVMDIBuilderGetOrCreateArray(LLVMDIBuilderRef Dref,
   DIBuilder *D = unwrap(Dref);
   Metadata **DataValue = unwrap(Data);
   ArrayRef<Metadata *> Elements(DataValue, Length);
-  DIArray A = D->getOrCreateArray(Elements);
+  DINodeArray A = D->getOrCreateArray(Elements);
   return wrap(A.get());
 }
 
@@ -214,15 +205,14 @@ LLVMMetadataRef LLVMDIBuilderGetOrCreateTypeArray(LLVMDIBuilderRef Dref,
   DIBuilder *D = unwrap(Dref);
   Metadata **DataValue = unwrap(Data);
   ArrayRef<Metadata *> Elements(DataValue, Length);
-  DITypeArray A = D->getOrCreateTypeArray(Elements);
+  DITypeRefArray A = D->getOrCreateTypeArray(Elements);
   return wrap(A.get());
 }
 
 LLVMMetadataRef LLVMDIBuilderCreateExpression(LLVMDIBuilderRef Dref,
                                               int64_t *Addr, size_t Length) {
   DIBuilder *D = unwrap(Dref);
-  DIExpression Expr = D->createExpression(ArrayRef<int64_t>(Addr, Length));
-  return wrap(Expr);
+  return wrap(D->createExpression(ArrayRef<int64_t>(Addr, Length)));
 }
 
 LLVMValueRef LLVMDIBuilderInsertDeclareAtEnd(LLVMDIBuilderRef Dref,
@@ -236,8 +226,8 @@ LLVMValueRef LLVMDIBuilderInsertDeclareAtEnd(LLVMDIBuilderRef Dref,
 
   DIBuilder *D = unwrap(Dref);
   Instruction *Instr = D->insertDeclare(
-      unwrap(Storage), unwrap<MDLocalVariable>(VarInfo),
-      unwrap<MDExpression>(Expr), /* DebugLoc */ nullptr, unwrap(Block));
+      unwrap(Storage), unwrap<DILocalVariable>(VarInfo),
+      unwrap<DIExpression>(Expr), /* DebugLoc */ nullptr, unwrap(Block));
   return wrap(Instr);
 }
 
@@ -252,7 +242,7 @@ LLVMValueRef LLVMDIBuilderInsertValueAtEnd(LLVMDIBuilderRef Dref,
 
   DIBuilder *D = unwrap(Dref);
   Instruction *Instr = D->insertDbgValueIntrinsic(
-      unwrap(Val), Offset, unwrap<MDLocalVariable>(VarInfo),
-      unwrap<MDExpression>(Expr), /* DebugLoc */ nullptr, unwrap(Block));
+      unwrap(Val), Offset, unwrap<DILocalVariable>(VarInfo),
+      unwrap<DIExpression>(Expr), /* DebugLoc */ nullptr, unwrap(Block));
   return wrap(Instr);
 }
diff --git a/bindings/go/llvm/ir.go b/bindings/go/llvm/ir.go
index e5916a1bacf..8fcd6ec4d74 100644
--- a/bindings/go/llvm/ir.go
+++ b/bindings/go/llvm/ir.go
@@ -160,6 +160,8 @@ const (
 	InAllocaAttribute           Attribute = 1 << 43
 	NonNullAttribute            Attribute = 1 << 44
 	JumpTableAttribute          Attribute = 1 << 45
+	ConvergentAttribute         Attribute = 1 << 46
+	SafeStackAttribute          Attribute = 1 << 47
 )
 
 //-------------------------------------------------------------------------
@@ -1052,6 +1054,9 @@ func (v Value) AddTargetDependentFunctionAttr(attr, value string) {
 	defer C.free(unsafe.Pointer(cvalue))
 	C.LLVMAddTargetDependentFunctionAttr(v.C, cattr, cvalue)
 }
+func (v Value) SetPersonality(p Value) {
+	C.LLVMSetPersonalityFn(v.C, p.C)
+}
 
 // Operations on parameters
 func (v Value) ParamsCount() int { return int(C.LLVMCountParams(v.C)) }
@@ -1204,7 +1209,7 @@ func (b Builder) Dispose() { C.LLVMDisposeBuilder(b.C) }
 func (b Builder) SetCurrentDebugLocation(line, col uint, scope, inlinedAt Metadata) {
 	C.LLVMSetCurrentDebugLocation2(b.C, C.unsigned(line), C.unsigned(col), scope.C, inlinedAt.C)
 }
-func (b Builder) SetInstDebugLocation(v Value)    { C.LLVMSetInstDebugLocation(b.C, v.C) }
+func (b Builder) SetInstDebugLocation(v Value) { C.LLVMSetInstDebugLocation(b.C, v.C) }
 func (b Builder) InsertDeclare(module Module, storage Value, md Value) Value {
 	f := module.NamedFunction("llvm.dbg.declare")
 	if f.IsNil() {
@@ -1723,10 +1728,10 @@ func (b Builder) CreatePtrDiff(lhs, rhs Value, name string) (v Value) {
 	return
 }
 
-func (b Builder) CreateLandingPad(t Type, personality Value, nclauses int, name string) (l Value) {
+func (b Builder) CreateLandingPad(t Type, nclauses int, name string) (l Value) {
 	cname := C.CString(name)
 	defer C.free(unsafe.Pointer(cname))
-	l.C = C.LLVMBuildLandingPad(b.C, t.C, personality.C, C.unsigned(nclauses), cname)
+	l.C = C.LLVMBuildLandingPad(b.C, t.C, nil, C.unsigned(nclauses), cname)
 	return l
 }
 
diff --git a/bindings/go/llvm/ir_test.go b/bindings/go/llvm/ir_test.go
index 981c94aa63e..edeeab935db 100644
--- a/bindings/go/llvm/ir_test.go
+++ b/bindings/go/llvm/ir_test.go
@@ -53,6 +53,7 @@ func TestAttributes(t *testing.T) {
 		{AlwaysInlineAttribute, "alwaysinline"},
 		{BuiltinAttribute, "builtin"},
 		{ByValAttribute, "byval"},
+		{ConvergentAttribute, "convergent"},
 		{InAllocaAttribute, "inalloca"},
 		{InlineHintAttribute, "inlinehint"},
 		{InRegAttribute, "inreg"},
@@ -78,6 +79,7 @@ func TestAttributes(t *testing.T) {
 		{ReturnedAttribute, "returned"},
 		{ReturnsTwiceAttribute, "returns_twice"},
 		{SExtAttribute, "signext"},
+		{SafeStackAttribute, "safestack"},
 		{StackProtectAttribute, "ssp"},
 		{StackProtectReqAttribute, "sspreq"},
 		{StackProtectStrongAttribute, "sspstrong"},
diff --git a/bindings/ocaml/Makefile.ocaml b/bindings/ocaml/Makefile.ocaml
index 1f65a7b8f90..22b96a298ef 100644
--- a/bindings/ocaml/Makefile.ocaml
+++ b/bindings/ocaml/Makefile.ocaml
@@ -277,6 +277,8 @@ uninstall-local:: uninstall-deplibs
 
 build-deplibs: $(OutputLibs)
 
+$(OcamlDir)/%.so: $(LibDir)/%.so
+	$(Verb) ln -sf $< $@
 $(OcamlDir)/%.a: $(LibDir)/%.a
 	$(Verb) ln -sf $< $@
 
diff --git a/bindings/ocaml/executionengine/llvm_executionengine.ml b/bindings/ocaml/executionengine/llvm_executionengine.ml
index 34031bed603..3f37e0c9d3d 100644
--- a/bindings/ocaml/executionengine/llvm_executionengine.ml
+++ b/bindings/ocaml/executionengine/llvm_executionengine.ml
@@ -43,11 +43,11 @@ external run_static_dtors : llexecutionengine -> unit
   = "llvm_ee_run_static_dtors"
 external data_layout : llexecutionengine -> Llvm_target.DataLayout.t
   = "llvm_ee_get_data_layout"
-external add_global_mapping_ : Llvm.llvalue -> int64 -> llexecutionengine -> unit
+external add_global_mapping_ : Llvm.llvalue -> nativeint -> llexecutionengine -> unit
   = "llvm_ee_add_global_mapping"
-external get_global_value_address_ : string -> llexecutionengine -> int64
+external get_global_value_address_ : string -> llexecutionengine -> nativeint
   = "llvm_ee_get_global_value_address"
-external get_function_address_ : string -> llexecutionengine -> int64
+external get_function_address_ : string -> llexecutionengine -> nativeint
   = "llvm_ee_get_function_address"
 
 let add_global_mapping llval ptr ee =
@@ -55,14 +55,14 @@ let add_global_mapping llval ptr ee =
 
 let get_global_value_address name typ ee =
   let vptr = get_global_value_address_ name ee in
-  if Int64.to_int vptr <> 0 then
+  if Nativeint.to_int vptr <> 0 then
     let open Ctypes in !@ (coerce (ptr void) (ptr typ) (ptr_of_raw_address vptr))
   else
     raise (Error ("Value " ^ name ^ " not found"))
 
 let get_function_address name typ ee =
   let fptr = get_function_address_ name ee in
-  if Int64.to_int fptr <> 0 then
+  if Nativeint.to_int fptr <> 0 then
     let open Ctypes in coerce (ptr void) typ (ptr_of_raw_address fptr)
   else
     raise (Error ("Function " ^ name ^ " not found"))
diff --git a/bindings/python/llvm/object.py b/bindings/python/llvm/object.py
index 4e912ed5da9..b427113e9ce 100644
--- a/bindings/python/llvm/object.py
+++ b/bindings/python/llvm/object.py
@@ -371,14 +371,6 @@ def __init__(self, ptr):
 
         self.expired = False
 
-    @CachedProperty
-    def address(self):
-        """The address of this relocation, in long bytes."""
-        if self.expired:
-            raise Exception('Relocation instance has expired.')
-
-        return lib.LLVMGetRelocationAddress(self)
-
     @CachedProperty
     def offset(self):
         """The offset of this relocation, in long bytes."""
@@ -498,9 +490,6 @@ def register_library(library):
     library.LLVMGetSymbolSize.argtypes = [Symbol]
     library.LLVMGetSymbolSize.restype = c_uint64
 
-    library.LLVMGetRelocationAddress.argtypes = [c_object_p]
-    library.LLVMGetRelocationAddress.restype = c_uint64
-
     library.LLVMGetRelocationOffset.argtypes = [c_object_p]
     library.LLVMGetRelocationOffset.restype = c_uint64
 
diff --git a/cmake/config-ix.cmake b/cmake/config-ix.cmake
index c57f9006a84..74cd76e5167 100755
--- a/cmake/config-ix.cmake
+++ b/cmake/config-ix.cmake
@@ -358,6 +358,10 @@ elseif (LLVM_NATIVE_ARCH MATCHES "hexagon")
   set(LLVM_NATIVE_ARCH Hexagon)
 elseif (LLVM_NATIVE_ARCH MATCHES "s390x")
   set(LLVM_NATIVE_ARCH SystemZ)
+elseif (LLVM_NATIVE_ARCH MATCHES "wasm32")
+  set(LLVM_NATIVE_ARCH WebAssembly)
+elseif (LLVM_NATIVE_ARCH MATCHES "wasm64")
+  set(LLVM_NATIVE_ARCH WebAssembly)
 else ()
   message(FATAL_ERROR "Unknown architecture ${LLVM_NATIVE_ARCH}")
 endif ()
@@ -393,12 +397,10 @@ else ()
 endif ()
 
 if( MINGW )
-  set(HAVE_LIBIMAGEHLP 1)
   set(HAVE_LIBPSAPI 1)
   set(HAVE_LIBSHELL32 1)
   # TODO: Check existence of libraries.
   #   include(CheckLibraryExists)
-  #   CHECK_LIBRARY_EXISTS(imagehlp ??? . HAVE_LIBIMAGEHLP)
 endif( MINGW )
 
 if (NOT HAVE_STRTOLL)
@@ -489,7 +491,7 @@ if (LLVM_ENABLE_DOXYGEN)
 
     option(LLVM_DOXYGEN_EXTERNAL_SEARCH "Enable doxygen external search." OFF)
     if (LLVM_DOXYGEN_EXTERNAL_SEARCH)
-      set(LLVM_DOXYGEN_SEARCHENGINE_URL "" CACHE STRING "URL to use for external searhc.")
+      set(LLVM_DOXYGEN_SEARCHENGINE_URL "" CACHE STRING "URL to use for external search.")
       set(LLVM_DOXYGEN_SEARCH_MAPPINGS "" CACHE STRING "Doxygen Search Mappings")
     endif()
   endif()
@@ -526,11 +528,9 @@ else()
   endif()
 endif()
 
-find_program(GOLD_EXECUTABLE NAMES ld.gold ld DOC "The gold linker")
-if(GOLD_EXECUTABLE)
-	set(LLVM_BINUTILS_INCDIR "" CACHE PATH
-		"PATH to binutils/include containing plugin-api.h for gold plugin.")
-endif()
+find_program(GOLD_EXECUTABLE NAMES ${LLVM_DEFAULT_TARGET_TRIPLE}-ld.gold ld.gold ${LLVM_DEFAULT_TARGET_TRIPLE}-ld ld DOC "The gold linker")
+set(LLVM_BINUTILS_INCDIR "" CACHE PATH
+	"PATH to binutils/include containing plugin-api.h for gold plugin.")
 
 if(APPLE)
   find_program(LD64_EXECUTABLE NAMES ld DOC "The ld64 linker")
@@ -548,13 +548,13 @@ else()
     if( OCAML_VERSION VERSION_LESS "4.00.0" )
       message(STATUS "OCaml bindings disabled, need OCaml >=4.00.0.")
     else()
-      find_ocamlfind_package(ctypes VERSION 0.3 OPTIONAL)
+      find_ocamlfind_package(ctypes VERSION 0.4 OPTIONAL)
       if( HAVE_OCAML_CTYPES )
         message(STATUS "OCaml bindings enabled.")
         find_ocamlfind_package(oUnit VERSION 2 OPTIONAL)
         set(LLVM_BINDINGS "${LLVM_BINDINGS} ocaml")
       else()
-        message(STATUS "OCaml bindings disabled, need ctypes >=0.3.")
+        message(STATUS "OCaml bindings disabled, need ctypes >=0.4.")
       endif()
     endif()
   endif()
diff --git a/cmake/modules/AddLLVM.cmake b/cmake/modules/AddLLVM.cmake
index ba6e2c508ea..34441173fac 100644
--- a/cmake/modules/AddLLVM.cmake
+++ b/cmake/modules/AddLLVM.cmake
@@ -1,4 +1,3 @@
-include(LLVMParseArguments)
 include(LLVMProcessSources)
 include(LLVM-Config)
 
@@ -84,34 +83,24 @@ function(add_llvm_symbol_exports target_name export_file)
       DEPENDS ${export_file}
       VERBATIM
       COMMENT "Creating export file for ${target_name}")
-    set_property(TARGET ${target_name} APPEND_STRING PROPERTY
-                 LINK_FLAGS "  -Wl,--version-script,${CMAKE_CURRENT_BINARY_DIR}/${native_export_file}")
+    if (${CMAKE_SYSTEM_NAME} MATCHES "SunOS")
+      set_property(TARGET ${target_name} APPEND_STRING PROPERTY
+                   LINK_FLAGS "  -Wl,-M,${CMAKE_CURRENT_BINARY_DIR}/${native_export_file}")
+    else()
+      set_property(TARGET ${target_name} APPEND_STRING PROPERTY
+                   LINK_FLAGS "  -Wl,--version-script,${CMAKE_CURRENT_BINARY_DIR}/${native_export_file}")
+    endif()
   else()
     set(native_export_file "${target_name}.def")
 
-    set(CAT "cat")
-    set(export_file_nativeslashes ${export_file})
-    if(WIN32 AND NOT CYGWIN)
-      set(CAT "type")
-      # Convert ${export_file} to native format (backslashes) for "type"
-      # Does not use file(TO_NATIVE_PATH) as it doesn't create a native
-      # path but a build-system specific format (see CMake bug
-      # http://public.kitware.com/Bug/print_bug_page.php?bug_id=5939 )
-      string(REPLACE / \\ export_file_nativeslashes ${export_file})
-    endif()
-
     add_custom_command(OUTPUT ${native_export_file}
-      COMMAND ${CMAKE_COMMAND} -E echo "EXPORTS" > ${native_export_file}
-      COMMAND ${CAT} ${export_file_nativeslashes} >> ${native_export_file}
+      COMMAND ${PYTHON_EXECUTABLE} -c "import sys;print(''.join(['EXPORTS\\n']+sys.stdin.readlines(),))"
+        < ${export_file} > ${native_export_file}
       DEPENDS ${export_file}
       VERBATIM
       COMMENT "Creating export file for ${target_name}")
     set(export_file_linker_flag "${CMAKE_CURRENT_BINARY_DIR}/${native_export_file}")
     if(MSVC)
-      # set(export_file_linker_flag "/DEF:${export_file_linker_flag}")
-      # XXX Emscripten: Fix build when build directory has spaces in it.
-      # See bug https://github.com/kripken/emscripten/issues/3382 and
-      # https://llvm.org/bugs/show_bug.cgi?id=23313 .
       set(export_file_linker_flag "/DEF:\"${export_file_linker_flag}\"")
     endif()
     set_property(TARGET ${target_name} APPEND_STRING PROPERTY
@@ -168,7 +157,7 @@ function(add_link_opts target_name)
 
     # Pass -O3 to the linker. This enabled different optimizations on different
     # linkers.
-    if(NOT (${CMAKE_SYSTEM_NAME} MATCHES "Darwin" OR WIN32))
+    if(NOT (${CMAKE_SYSTEM_NAME} MATCHES "Darwin|SunOS" OR WIN32))
       set_property(TARGET ${target_name} APPEND_STRING PROPERTY
                    LINK_FLAGS " -Wl,-O3")
     endif()
@@ -186,6 +175,9 @@ function(add_link_opts target_name)
         # ld64's implementation of -dead_strip breaks tools that use plugins.
         set_property(TARGET ${target_name} APPEND_STRING PROPERTY
                      LINK_FLAGS " -Wl,-dead_strip")
+      elseif(${CMAKE_SYSTEM_NAME} MATCHES "SunOS")
+        set_property(TARGET ${target_name} APPEND_STRING PROPERTY
+                     LINK_FLAGS " -Wl,-z -Wl,discard-unused=sections")
       elseif(NOT WIN32 AND NOT LLVM_LINKER_IS_GOLD)
         # Object files are compiled with -ffunction-data-sections.
         # Versions of bfd ld < 2.23.1 have a bug in --gc-sections that breaks
@@ -232,6 +224,78 @@ function(set_output_directory target bindir libdir)
   endif()
 endfunction()
 
+# If on Windows and building with MSVC, add the resource script containing the
+# VERSIONINFO data to the project.  This embeds version resource information
+# into the output .exe or .dll.
+# TODO: Enable for MinGW Windows builds too.
+#
+function(add_windows_version_resource_file OUT_VAR)
+  set(sources ${ARGN})
+  if (MSVC)
+    set(resource_file ${LLVM_SOURCE_DIR}/resources/windows_version_resource.rc)
+    if(EXISTS ${resource_file})
+      set(sources ${sources} ${resource_file})
+      source_group("Resource Files" ${resource_file})
+      set(windows_resource_file ${resource_file} PARENT_SCOPE)
+    endif()
+  endif(MSVC)
+
+  set(${OUT_VAR} ${sources} PARENT_SCOPE)
+endfunction(add_windows_version_resource_file)
+
+# set_windows_version_resource_properties(name resource_file...
+#   VERSION_MAJOR int
+#     Optional major version number (defaults to LLVM_VERSION_MAJOR)
+#   VERSION_MINOR int
+#     Optional minor version number (defaults to LLVM_VERSION_MINOR)
+#   VERSION_PATCHLEVEL int
+#     Optional patchlevel version number (defaults to LLVM_VERSION_PATCH)
+#   VERSION_STRING
+#     Optional version string (defaults to PACKAGE_VERSION)
+#   PRODUCT_NAME
+#     Optional product name string (defaults to "LLVM")
+#   )
+function(set_windows_version_resource_properties name resource_file)
+  cmake_parse_arguments(ARG
+    ""
+    "VERSION_MAJOR;VERSION_MINOR;VERSION_PATCHLEVEL;VERSION_STRING;PRODUCT_NAME"
+    ""
+    ${ARGN})
+
+  if (NOT DEFINED ARG_VERSION_MAJOR)
+    set(ARG_VERSION_MAJOR ${LLVM_VERSION_MAJOR})
+  endif()
+
+  if (NOT DEFINED ARG_VERSION_MINOR)
+    set(ARG_VERSION_MINOR ${LLVM_VERSION_MINOR})
+  endif()
+
+  if (NOT DEFINED ARG_VERSION_PATCHLEVEL)
+    set(ARG_VERSION_PATCHLEVEL ${LLVM_VERSION_PATCH})
+  endif()
+
+  if (NOT DEFINED ARG_VERSION_STRING)
+    set(ARG_VERSION_STRING ${PACKAGE_VERSION})
+  endif()
+
+  if (NOT DEFINED ARG_PRODUCT_NAME)
+    set(ARG_PRODUCT_NAME "LLVM")
+  endif()
+
+  set_property(SOURCE ${resource_file}
+               PROPERTY COMPILE_FLAGS /nologo)
+  set_property(SOURCE ${resource_file}
+               PROPERTY COMPILE_DEFINITIONS
+               "RC_VERSION_FIELD_1=${ARG_VERSION_MAJOR}"
+               "RC_VERSION_FIELD_2=${ARG_VERSION_MINOR}"
+               "RC_VERSION_FIELD_3=${ARG_VERSION_PATCHLEVEL}"
+               "RC_VERSION_FIELD_4=0"
+               "RC_FILE_VERSION=\"${ARG_VERSION_STRING}\""
+               "RC_INTERNAL_NAME=\"${name}\""
+               "RC_PRODUCT_NAME=\"${ARG_PRODUCT_NAME}\""
+               "RC_PRODUCT_VERSION=\"${ARG_VERSION_STRING}\"")
+endfunction(set_windows_version_resource_properties)
+
 # llvm_add_library(name sources...
 #   SHARED;STATIC
 #     STATIC by default w/o BUILD_SHARED_LIBS.
@@ -320,10 +384,17 @@ function(llvm_add_library name)
   if(ARG_MODULE)
     add_library(${name} MODULE ${ALL_FILES})
   elseif(ARG_SHARED)
+    add_windows_version_resource_file(ALL_FILES ${ALL_FILES})
     add_library(${name} SHARED ${ALL_FILES})
   else()
     add_library(${name} STATIC ${ALL_FILES})
   endif()
+
+  if(DEFINED windows_resource_file)
+    set_windows_version_resource_properties(${name} ${windows_resource_file})
+    set(windows_resource_file ${windows_resource_file} PARENT_SCOPE)
+  endif()
+
   set_output_directory(${name} ${LLVM_RUNTIME_OUTPUT_INTDIR} ${LLVM_LIBRARY_OUTPUT_INTDIR})
   llvm_update_compile_flags(${name})
   add_link_opts( ${name} )
@@ -421,11 +492,17 @@ macro(add_llvm_library name)
   else()
     llvm_add_library(${name} ${ARGN})
   endif()
-  set_property( GLOBAL APPEND PROPERTY LLVM_LIBS ${name} )
+  # The gtest libraries should not be installed or exported as a target
+  if ("${name}" STREQUAL gtest OR "${name}" STREQUAL gtest_main)
+    set(_is_gtest TRUE)
+  else()
+    set(_is_gtest FALSE)
+    set_property( GLOBAL APPEND PROPERTY LLVM_LIBS ${name} )
+  endif()
 
   if( EXCLUDE_FROM_ALL )
     set_target_properties( ${name} PROPERTIES EXCLUDE_FROM_ALL ON)
-  else()
+  elseif(NOT _is_gtest)
     if (NOT LLVM_INSTALL_TOOLCHAIN_ONLY OR ${name} STREQUAL "LTO")
       if(ARG_SHARED OR BUILD_SHARED_LIBS)
         if(WIN32 OR CYGWIN)
@@ -486,11 +563,18 @@ endmacro(add_llvm_loadable_module name)
 
 macro(add_llvm_executable name)
   llvm_process_sources( ALL_FILES ${ARGN} )
+  add_windows_version_resource_file(ALL_FILES ${ALL_FILES})
+
   if( EXCLUDE_FROM_ALL )
     add_executable(${name} EXCLUDE_FROM_ALL ${ALL_FILES})
   else()
     add_executable(${name} ${ALL_FILES})
   endif()
+
+  if(DEFINED windows_resource_file)
+    set_windows_version_resource_properties(${name} ${windows_resource_file})
+  endif()
+
   llvm_update_compile_flags(${name})
   add_link_opts( ${name} )
 
@@ -592,6 +676,13 @@ macro(add_llvm_target target_name)
   set( CURRENT_LLVM_TARGET LLVM${target_name} )
 endmacro(add_llvm_target)
 
+function(canonicalize_tool_name name output)
+  string(REPLACE "${CMAKE_CURRENT_SOURCE_DIR}/" "" nameStrip ${name})
+  string(REPLACE "-" "_" nameUNDERSCORE ${nameStrip})
+  string(TOUPPER ${nameUNDERSCORE} nameUPPER)
+  set(${output} "${nameUPPER}" PARENT_SCOPE)
+endfunction(canonicalize_tool_name)
+
 # Add external project that may want to be built as part of llvm such as Clang,
 # lld, and Polly. This adds two options. One for the source directory of the
 # project, which defaults to ${CMAKE_CURRENT_SOURCE_DIR}/${name}. Another to
@@ -602,38 +693,72 @@ macro(add_llvm_external_project name)
   if("${add_llvm_external_dir}" STREQUAL "")
     set(add_llvm_external_dir ${name})
   endif()
-  list(APPEND LLVM_IMPLICIT_PROJECT_IGNORE "${CMAKE_CURRENT_SOURCE_DIR}/${add_llvm_external_dir}")
-  string(REPLACE "-" "_" nameUNDERSCORE ${name})
-  string(TOUPPER ${nameUNDERSCORE} nameUPPER)
-  set(LLVM_EXTERNAL_${nameUPPER}_SOURCE_DIR "${CMAKE_CURRENT_SOURCE_DIR}/${add_llvm_external_dir}"
-      CACHE PATH "Path to ${name} source directory")
-  if (NOT ${LLVM_EXTERNAL_${nameUPPER}_SOURCE_DIR} STREQUAL ""
-      AND EXISTS ${LLVM_EXTERNAL_${nameUPPER}_SOURCE_DIR}/CMakeLists.txt)
-    option(LLVM_EXTERNAL_${nameUPPER}_BUILD
-           "Whether to build ${name} as part of LLVM" ON)
-    if (LLVM_EXTERNAL_${nameUPPER}_BUILD)
-      add_subdirectory(${LLVM_EXTERNAL_${nameUPPER}_SOURCE_DIR} ${add_llvm_external_dir})
+  canonicalize_tool_name(${name} nameUPPER)
+  if(NOT DEFINED LLVM_TOOL_${nameUPPER}_BUILD)
+    option(LLVM_TOOL_${nameUPPER}_BUILD
+           "Whether to build ${name} as part of LLVM" On)
+  endif()
+  if (LLVM_TOOL_${nameUPPER}_BUILD)
+    if(EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/${add_llvm_external_dir}/CMakeLists.txt)
+        add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/${add_llvm_external_dir} ${add_llvm_external_dir})
+        set(LLVM_TOOL_${nameUPPER}_BUILD Off)
+    elseif(LLVM_EXTERNAL_${nameUPPER}_SOURCE_DIR)
+      set(LLVM_EXTERNAL_${nameUPPER}_SOURCE_DIR
+        "${LLVM_EXTERNAL_${nameUPPER}_SOURCE_DIR}"
+        CACHE PATH "Path to ${name} source directory")
+      mark_as_advanced(LLVM_EXTERNAL_${nameUPPER}_SOURCE_DIR)
+      if(IS_DIRECTORY "${LLVM_EXTERNAL_${nameUPPER}_SOURCE_DIR} ${add_llvm_external_dir}") # XXX EMSCRIPTEN
+        add_subdirectory(${LLVM_EXTERNAL_${nameUPPER}_SOURCE_DIR} ${add_llvm_external_dir})
+      else()
+        message("    (ignoring non-existing external_project |${LLVM_EXTERNAL_${nameUPPER}_SOURCE_DIR} ${add_llvm_external_dir}|)")
+      endif()
+      set(LLVM_TOOL_${nameUPPER}_BUILD Off)
     endif()
   endif()
 endmacro(add_llvm_external_project)
 
 macro(add_llvm_tool_subdirectory name)
-  list(APPEND LLVM_IMPLICIT_PROJECT_IGNORE "${CMAKE_CURRENT_SOURCE_DIR}/${name}")
-  add_subdirectory(${name})
+  add_llvm_external_project(${name})
 endmacro(add_llvm_tool_subdirectory)
 
-macro(ignore_llvm_tool_subdirectory name)
-  list(APPEND LLVM_IMPLICIT_PROJECT_IGNORE "${CMAKE_CURRENT_SOURCE_DIR}/${name}")
-endmacro(ignore_llvm_tool_subdirectory)
+function(get_project_name_from_src_var var output)
+  string(REGEX MATCH "LLVM_EXTERNAL_(.*)_SOURCE_DIR"
+         MACHED_TOOL "${var}")
+  if(MACHED_TOOL)
+    set(${output} ${CMAKE_MATCH_1} PARENT_SCOPE)
+  else()
+    set(${output} PARENT_SCOPE)
+  endif()
+endfunction()
+
+function(create_llvm_tool_options)
+  file(GLOB sub-dirs "${CMAKE_CURRENT_SOURCE_DIR}/*")
+  foreach(dir ${sub-dirs})
+    if(IS_DIRECTORY "${dir}" AND EXISTS "${dir}/CMakeLists.txt")
+      canonicalize_tool_name(${dir} name)
+      option(LLVM_TOOL_${name}_BUILD
+           "Whether to build ${name} as part of LLVM" On)
+      mark_as_advanced(LLVM_TOOL_${name}_BUILD)
+    endif()
+  endforeach()
+  get_cmake_property(variableNames VARIABLES)
+  foreach (variableName ${variableNames})
+    get_project_name_from_src_var(${variableName} projectName)
+    if(projectName)
+      option(LLVM_TOOL_${projectName}_BUILD
+           "Whether to build ${name} as part of LLVM" On)
+      mark_as_advanced(LLVM_TOOL_${name}_BUILD)
+    endif()
+  endforeach()
+endfunction(create_llvm_tool_options)
 
-function(add_llvm_implicit_external_projects)
+function(add_llvm_implicit_projects)
   set(list_of_implicit_subdirs "")
   file(GLOB sub-dirs "${CMAKE_CURRENT_SOURCE_DIR}/*")
   foreach(dir ${sub-dirs})
-    if(IS_DIRECTORY "${dir}")
-      list(FIND LLVM_IMPLICIT_PROJECT_IGNORE "${dir}" tool_subdir_ignore)
-      if( tool_subdir_ignore EQUAL -1
-          AND EXISTS "${dir}/CMakeLists.txt")
+    if(IS_DIRECTORY "${dir}" AND EXISTS "${dir}/CMakeLists.txt")
+      canonicalize_tool_name(${dir} name)
+      if (LLVM_TOOL_${name}_BUILD)
         get_filename_component(fn "${dir}" NAME)
         list(APPEND list_of_implicit_subdirs "${fn}")
       endif()
@@ -643,7 +768,7 @@ function(add_llvm_implicit_external_projects)
   foreach(external_proj ${list_of_implicit_subdirs})
     add_llvm_external_project("${external_proj}")
   endforeach()
-endfunction(add_llvm_implicit_external_projects)
+endfunction(add_llvm_implicit_projects)
 
 # Generic support for adding a unittest.
 function(add_unittest test_suite test_name)
@@ -694,7 +819,7 @@ function(llvm_add_go_executable binary pkgpath)
     endforeach(d)
     set(ldflags "${CMAKE_EXE_LINKER_FLAGS}")
     add_custom_command(OUTPUT ${binpath}
-      COMMAND ${CMAKE_BINARY_DIR}/bin/llvm-go "cc=${cc}" "cxx=${cxx}" "cppflags=${cppflags}" "ldflags=${ldflags}"
+      COMMAND ${CMAKE_BINARY_DIR}/bin/llvm-go "go=${GO_EXECUTABLE}" "cc=${cc}" "cxx=${cxx}" "cppflags=${cppflags}" "ldflags=${ldflags}"
               ${ARG_GOFLAGS} build -o ${binpath} ${pkgpath}
       DEPENDS llvm-config ${CMAKE_BINARY_DIR}/bin/llvm-go${CMAKE_EXECUTABLE_SUFFIX}
               ${llvmlibs} ${ARG_DEPENDS}
@@ -765,7 +890,7 @@ endfunction()
 # A raw function to create a lit target. This is used to implement the testuite
 # management functions.
 function(add_lit_target target comment)
-  parse_arguments(ARG "PARAMS;DEPENDS;ARGS" "" ${ARGN})
+  cmake_parse_arguments(ARG "" "" "PARAMS;DEPENDS;ARGS" ${ARGN})
   set(LIT_ARGS "${ARG_ARGS} ${LLVM_LIT_ARGS}")
   separate_arguments(LIT_ARGS)
   if (NOT CMAKE_CFG_INTDIR STREQUAL ".")
@@ -780,9 +905,9 @@ function(add_lit_target target comment)
   foreach(param ${ARG_PARAMS})
     list(APPEND LIT_COMMAND --param ${param})
   endforeach()
-  if (ARG_DEFAULT_ARGS)
+  if (ARG_UNPARSED_ARGUMENTS)
     add_custom_target(${target}
-      COMMAND ${LIT_COMMAND} ${ARG_DEFAULT_ARGS}
+      COMMAND ${LIT_COMMAND} ${ARG_UNPARSED_ARGUMENTS}
       COMMENT "${comment}"
       ${cmake_3_2_USES_TERMINAL}
       )
@@ -801,12 +926,12 @@ endfunction()
 
 # A function to add a set of lit test suites to be driven through 'check-*' targets.
 function(add_lit_testsuite target comment)
-  parse_arguments(ARG "PARAMS;DEPENDS;ARGS" "" ${ARGN})
+  cmake_parse_arguments(ARG "" "" "PARAMS;DEPENDS;ARGS" ${ARGN})
 
   # EXCLUDE_FROM_ALL excludes the test ${target} out of check-all.
   if(NOT EXCLUDE_FROM_ALL)
     # Register the testsuites, params and depends for the global check rule.
-    set_property(GLOBAL APPEND PROPERTY LLVM_LIT_TESTSUITES ${ARG_DEFAULT_ARGS})
+    set_property(GLOBAL APPEND PROPERTY LLVM_LIT_TESTSUITES ${ARG_UNPARSED_ARGUMENTS})
     set_property(GLOBAL APPEND PROPERTY LLVM_LIT_PARAMS ${ARG_PARAMS})
     set_property(GLOBAL APPEND PROPERTY LLVM_LIT_DEPENDS ${ARG_DEPENDS})
     set_property(GLOBAL APPEND PROPERTY LLVM_LIT_EXTRA_ARGS ${ARG_ARGS})
@@ -814,7 +939,7 @@ function(add_lit_testsuite target comment)
 
   # Produce a specific suffixed check rule.
   add_lit_target(${target} ${comment}
-    ${ARG_DEFAULT_ARGS}
+    ${ARG_UNPARSED_ARGUMENTS}
     PARAMS ${ARG_PARAMS}
     DEPENDS ${ARG_DEPENDS}
     ARGS ${ARG_ARGS}
@@ -823,7 +948,7 @@ endfunction()
 
 function(add_lit_testsuites project directory)
   if (NOT CMAKE_CONFIGURATION_TYPES)
-    parse_arguments(ARG "PARAMS;DEPENDS;ARGS" "" ${ARGN})
+    cmake_parse_arguments(ARG "" "" "PARAMS;DEPENDS;ARGS" ${ARGN})
     file(GLOB_RECURSE litCfg ${directory}/lit*.cfg)
     set(lit_suites)
     foreach(f ${litCfg})
diff --git a/cmake/modules/AddOCaml.cmake b/cmake/modules/AddOCaml.cmake
index c58ac9cb734..8b33332d07b 100644
--- a/cmake/modules/AddOCaml.cmake
+++ b/cmake/modules/AddOCaml.cmake
@@ -149,7 +149,7 @@ function(add_ocaml_library name)
             "-I" "${LLVM_LIBRARY_DIR}/ocaml/"
             "-dump" "${bin}/${name}.odoc"
             ${ocaml_pkgs} ${ocaml_inputs}
-    DEPENDS ${ocaml_inputs}
+    DEPENDS ${ocaml_inputs} ${ocaml_outputs}
     COMMENT "Building OCaml documentation for ${name}"
     VERBATIM)
 
diff --git a/cmake/modules/CheckAtomic.cmake b/cmake/modules/CheckAtomic.cmake
index a03788ec9f9..551de6ade84 100644
--- a/cmake/modules/CheckAtomic.cmake
+++ b/cmake/modules/CheckAtomic.cmake
@@ -2,15 +2,45 @@
 
 INCLUDE(CheckCXXSourceCompiles)
 
-check_function_exists(__atomic_fetch_add_4 HAVE___ATOMIC_FETCH_ADD_4)
-if( NOT HAVE___ATOMIC_FETCH_ADD_4 )
-  check_library_exists(atomic __atomic_fetch_add_4 "" HAVE_LIBATOMIC)
-  set(HAVE_LIBATOMIC False)
-  if( HAVE_LIBATOMIC )
-    list(APPEND CMAKE_REQUIRED_LIBRARIES "atomic")
+# Sometimes linking against libatomic is required for atomic ops, if
+# the platform doesn't support lock-free atomics.
+
+function(check_working_cxx_atomics varname)
+  set(OLD_CMAKE_REQUIRED_FLAGS ${CMAKE_REQUIRED_FLAGS})
+  set(CMAKE_REQUIRED_FLAGS "-std=c++11")
+  CHECK_CXX_SOURCE_COMPILES("
+#include <atomic>
+std::atomic<int> x;
+int main() {
+  return x;
+}
+" ${varname})
+  set(CMAKE_REQUIRED_FLAGS ${OLD_CMAKE_REQUIRED_FLAGS})
+endfunction(check_working_cxx_atomics)
+
+# This isn't necessary on MSVC, so avoid command-line switch annoyance
+# by only running on GCC-like hosts.
+if (LLVM_COMPILER_IS_GCC_COMPATIBLE)
+  # First check if atomics work without the library.
+  check_working_cxx_atomics(HAVE_CXX_ATOMICS_WITHOUT_LIB)
+  # If not, check if the library exists, and atomics work with it.
+  if(NOT HAVE_CXX_ATOMICS_WITHOUT_LIB)
+    check_library_exists(atomic __atomic_fetch_add_4 "" HAVE_LIBATOMIC)
+    if( HAVE_LIBATOMIC )
+      list(APPEND CMAKE_REQUIRED_LIBRARIES "atomic")
+      check_working_cxx_atomics(HAVE_CXX_ATOMICS_WITH_LIB)
+      if (NOT HAVE_CXX_ATOMICS_WITH_LIB)
+	message(FATAL_ERROR "Host compiler must support std::atomic!")
+      endif()
+    else()
+      message(FATAL_ERROR "Host compiler appears to require libatomic, but cannot find it.")
+    endif()
   endif()
 endif()
 
+## TODO: This define is only used for the legacy atomic operations in
+## llvm's Atomic.h, which should be replaced.  Other code simply
+## assumes C++11 <atomic> works.
 CHECK_CXX_SOURCE_COMPILES("
 #ifdef _MSC_VER
 #include <Intrin.h> /* Workaround for PR19898. */
diff --git a/cmake/modules/HandleLLVMOptions.cmake b/cmake/modules/HandleLLVMOptions.cmake
index 27147dcfd29..bee77c14b5b 100644
--- a/cmake/modules/HandleLLVMOptions.cmake
+++ b/cmake/modules/HandleLLVMOptions.cmake
@@ -88,6 +88,9 @@ elseif( uppercase_LLVM_ABI_BREAKING_CHECKS STREQUAL "FORCE_ON" )
   set( LLVM_ENABLE_ABI_BREAKING_CHECKS 1 )
 elseif( uppercase_LLVM_ABI_BREAKING_CHECKS STREQUAL "FORCE_OFF" )
   # We don't need to do anything special to turn off ABI breaking checks.
+elseif( NOT DEFINED LLVM_ABI_BREAKING_CHECKS )
+  # Treat LLVM_ABI_BREAKING_CHECKS like "FORCE_OFF" when it has not been
+  # defined.
 else()
   message(FATAL_ERROR "Unknown value for LLVM_ABI_BREAKING_CHECKS: \"${LLVM_ABI_BREAKING_CHECKS}\"!")
 endif()
@@ -128,7 +131,7 @@ endif()
 
 # Pass -Wl,-z,defs. This makes sure all symbols are defined. Otherwise a DSO
 # build might work on ELF but fail on MachO/COFF.
-if(NOT (${CMAKE_SYSTEM_NAME} MATCHES "Darwin" OR WIN32 OR
+if(NOT (${CMAKE_SYSTEM_NAME} MATCHES "Darwin" OR WIN32 OR CYGWIN OR
         ${CMAKE_SYSTEM_NAME} MATCHES "FreeBSD") AND
    NOT LLVM_USE_SANITIZER)
   set(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} -Wl,-z,defs")
@@ -163,6 +166,7 @@ function(add_flag_or_print_warning flag name)
     message(STATUS "Building with ${flag}")
     set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${flag}" PARENT_SCOPE)
     set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${flag}" PARENT_SCOPE)
+    set(CMAKE_ASM_FLAGS "${CMAKE_ASM_FLAGS} ${flag}" PARENT_SCOPE)
   else()
     message(WARNING "${flag} is not supported.")
   endif()
@@ -304,7 +308,17 @@ if( MSVC )
     -wd4611 # Suppress 'interaction between '_setjmp' and C++ object destruction is non-portable'
     -wd4805 # Suppress 'unsafe mix of type <type> and type <type> in operation'
     -wd4204 # Suppress 'nonstandard extension used : non-constant aggregate initializer'
-    
+    -wd4577 # Suppress 'noexcept used with no exception handling mode specified; termination on exception is not guaranteed'
+    -wd4091 # Suppress 'typedef: ignored on left of '' when no variable is declared'
+
+	# Ideally, we'd like this warning to be enabled, but MSVC 2013 doesn't
+	# support the 'aligned' attribute in the way that clang sources requires (for
+	# any code that uses the LLVM_ALIGNAS macro), so this is must be disabled to
+	# avoid unwanted alignment warnings.
+	# When we switch to requiring a version of MSVC that supports the 'alignas'
+	# specifier (MSVC 2015?) this warning can be re-enabled.
+    -wd4324 # Suppress 'structure was padded due to __declspec(align())'
+	    
     # Promoted warnings.
     -w14062 # Promote 'enumerator in switch of enum is not handled' to level 1 warning.
 
@@ -334,6 +348,11 @@ if( MSVC )
     append("${flag}" CMAKE_C_FLAGS CMAKE_CXX_FLAGS)
   endforeach(flag)
 
+  # Disable sized deallocation if the flag is supported. MSVC fails to compile
+  # the operator new overload in User otherwise.
+  check_c_compiler_flag("/WX /Zc:sizedDealloc-" SUPPORTS_SIZED_DEALLOC)
+  append_if(SUPPORTS_SIZED_DEALLOC "/Zc:sizedDealloc-" CMAKE_C_FLAGS CMAKE_CXX_FLAGS)
+
 elseif( LLVM_COMPILER_IS_GCC_COMPATIBLE )
   if (LLVM_ENABLE_WARNINGS)
     append("-Wall -W -Wno-unused-parameter -Wwrite-strings" CMAKE_C_FLAGS CMAKE_CXX_FLAGS)
@@ -451,7 +470,7 @@ if(LLVM_USE_SANITIZER)
       endif()
     elseif (LLVM_USE_SANITIZER STREQUAL "Undefined")
       append_common_sanitizer_flags()
-      append("-fsanitize=undefined -fno-sanitize=vptr,function -fno-sanitize-recover"
+      append("-fsanitize=undefined -fno-sanitize=vptr,function -fno-sanitize-recover=all"
               CMAKE_C_FLAGS CMAKE_CXX_FLAGS)
     elseif (LLVM_USE_SANITIZER STREQUAL "Thread")
       append_common_sanitizer_flags()
@@ -459,7 +478,7 @@ if(LLVM_USE_SANITIZER)
     elseif (LLVM_USE_SANITIZER STREQUAL "Address;Undefined" OR
             LLVM_USE_SANITIZER STREQUAL "Undefined;Address")
       append_common_sanitizer_flags()
-      append("-fsanitize=address,undefined -fno-sanitize=vptr,function -fno-sanitize-recover"
+      append("-fsanitize=address,undefined -fno-sanitize=vptr,function -fno-sanitize-recover=all"
               CMAKE_C_FLAGS CMAKE_CXX_FLAGS)
     else()
       message(WARNING "Unsupported value of LLVM_USE_SANITIZER: ${LLVM_USE_SANITIZER}")
@@ -468,7 +487,7 @@ if(LLVM_USE_SANITIZER)
     message(WARNING "LLVM_USE_SANITIZER is not supported on this platform.")
   endif()
   if (LLVM_USE_SANITIZE_COVERAGE)
-    append("-fsanitize-coverage=4 -mllvm -sanitizer-coverage-8bit-counters=1" CMAKE_C_FLAGS CMAKE_CXX_FLAGS)
+    append("-fsanitize-coverage=edge,indirect-calls,8bit-counters,trace-cmp" CMAKE_C_FLAGS CMAKE_CXX_FLAGS)
   endif()
 endif()
 
diff --git a/cmake/modules/LLVM-Config.cmake b/cmake/modules/LLVM-Config.cmake
index bc0527f1db4..22ac7145191 100644
--- a/cmake/modules/LLVM-Config.cmake
+++ b/cmake/modules/LLVM-Config.cmake
@@ -132,6 +132,41 @@ function(llvm_map_components_to_libnames out_libs)
       # already processed
     elseif( c STREQUAL "all" )
       list(APPEND expanded_components ${LLVM_AVAILABLE_LIBS})
+    elseif( c STREQUAL "AllTargetsAsmPrinters" )
+      # Link all the asm printers from all the targets
+      foreach(t ${LLVM_TARGETS_TO_BUILD})
+        if( TARGET LLVM${t}AsmPrinter )
+          list(APPEND expanded_components "LLVM${t}AsmPrinter")
+        endif()
+      endforeach(t)
+    elseif( c STREQUAL "AllTargetsAsmParsers" )
+      # Link all the asm parsers from all the targets
+      foreach(t ${LLVM_TARGETS_TO_BUILD})
+        if( TARGET LLVM${t}AsmParser )
+          list(APPEND expanded_components "LLVM${t}AsmParser")
+        endif()
+      endforeach(t)
+    elseif( c STREQUAL "AllTargetsDescs" )
+      # Link all the descs from all the targets
+      foreach(t ${LLVM_TARGETS_TO_BUILD})
+        if( TARGET LLVM${t}Desc )
+          list(APPEND expanded_components "LLVM${t}Desc")
+        endif()
+      endforeach(t)
+    elseif( c STREQUAL "AllTargetsDisassemblers" )
+      # Link all the disassemblers from all the targets
+      foreach(t ${LLVM_TARGETS_TO_BUILD})
+        if( TARGET LLVM${t}Disassembler )
+          list(APPEND expanded_components "LLVM${t}Disassembler")
+        endif()
+      endforeach(t)
+    elseif( c STREQUAL "AllTargetsInfos" )
+      # Link all the infos from all the targets
+      foreach(t ${LLVM_TARGETS_TO_BUILD})
+        if( TARGET LLVM${t}Info )
+          list(APPEND expanded_components "LLVM${t}Info")
+        endif()
+      endforeach(t)
     else( NOT idx LESS 0 )
       # Canonize the component name:
       string(TOUPPER "${c}" capitalized)
diff --git a/cmake/modules/LLVMParseArguments.cmake b/cmake/modules/LLVMParseArguments.cmake
deleted file mode 100644
index ce19be114b3..00000000000
--- a/cmake/modules/LLVMParseArguments.cmake
+++ /dev/null
@@ -1,80 +0,0 @@
-# Copied from http://www.itk.org/Wiki/CMakeMacroParseArguments under
-# http://creativecommons.org/licenses/by/2.5/.
-#
-# The PARSE_ARGUMENTS macro will take the arguments of another macro and define
-# several variables. The first argument to PARSE_ARGUMENTS is a prefix to put on
-# all variables it creates. The second argument is a list of names, and the
-# third argument is a list of options. Both of these lists should be quoted. The
-# rest of PARSE_ARGUMENTS are arguments from another macro to be parsed.
-#
-# PARSE_ARGUMENTS(prefix arg_names options arg1 arg2...)
-#
-# For each item in options, PARSE_ARGUMENTS will create a variable with that
-# name, prefixed with prefix_. So, for example, if prefix is MY_MACRO and
-# options is OPTION1;OPTION2, then PARSE_ARGUMENTS will create the variables
-# MY_MACRO_OPTION1 and MY_MACRO_OPTION2. These variables will be set to true if
-# the option exists in the command line or false otherwise.
-#
-#For each item in arg_names, PARSE_ARGUMENTS will create a variable with that
-#name, prefixed with prefix_. Each variable will be filled with the arguments
-#that occur after the given arg_name is encountered up to the next arg_name or
-#the end of the arguments. All options are removed from these
-#lists. PARSE_ARGUMENTS also creates a prefix_DEFAULT_ARGS variable containing
-#the list of all arguments up to the first arg_name encountered.
-#
-#Here is a simple, albeit impractical, example of using PARSE_ARGUMENTS that
-#demonstrates its behavior.
-#
-# SET(arguments
-#     hello OPTION3 world
-#     LIST3 foo bar
-#     OPTION2
-#     LIST1 fuz baz
-#     )
-#
-# PARSE_ARGUMENTS(ARG "LIST1;LIST2;LIST3" "OPTION1;OPTION2;OPTION3" ${arguments})
-#
-# PARSE_ARGUMENTS creates 7 variables and sets them as follows:
-#   ARG_DEFAULT_ARGS: hello;world
-#   ARG_LIST1: fuz;baz
-#   ARG_LIST2:
-#   ARG_LIST3: foo;bar
-#   ARG_OPTION1: FALSE
-#   ARG_OPTION2: TRUE
-#   ARG_OPTION3: TRUE
-#
-# If you don't have any options, use an empty string in its place.
-#   PARSE_ARGUMENTS(ARG "LIST1;LIST2;LIST3" "" ${arguments})
-# Likewise if you have no lists.
-#   PARSE_ARGUMENTS(ARG "" "OPTION1;OPTION2;OPTION3" ${arguments})
-
-MACRO(PARSE_ARGUMENTS prefix arg_names option_names)
-  SET(DEFAULT_ARGS)
-  FOREACH(arg_name ${arg_names})
-    SET(${prefix}_${arg_name})
-  ENDFOREACH(arg_name)
-  FOREACH(option ${option_names})
-    SET(${prefix}_${option} FALSE)
-  ENDFOREACH(option)
-
-  SET(current_arg_name DEFAULT_ARGS)
-  SET(current_arg_list)
-  FOREACH(arg ${ARGN})
-    SET(larg_names ${arg_names})
-    LIST(FIND larg_names "${arg}" is_arg_name)
-    IF (is_arg_name GREATER -1)
-      SET(${prefix}_${current_arg_name} ${current_arg_list})
-      SET(current_arg_name ${arg})
-      SET(current_arg_list)
-    ELSE (is_arg_name GREATER -1)
-      SET(loption_names ${option_names})
-      LIST(FIND loption_names "${arg}" is_option)
-      IF (is_option GREATER -1)
-        SET(${prefix}_${arg} TRUE)
-      ELSE (is_option GREATER -1)
-        SET(current_arg_list ${current_arg_list} ${arg})
-      ENDIF (is_option GREATER -1)
-    ENDIF (is_arg_name GREATER -1)
-  ENDFOREACH(arg)
-  SET(${prefix}_${current_arg_name} ${current_arg_list})
-ENDMACRO(PARSE_ARGUMENTS)
diff --git a/cmake/modules/Makefile b/cmake/modules/Makefile
index 97ee7d3df4b..f644c45dcd0 100644
--- a/cmake/modules/Makefile
+++ b/cmake/modules/Makefile
@@ -47,6 +47,12 @@ ifeq ($(LLVM_LIBS_TO_EXPORT),Error)
 $(error llvm-config --libs failed)
 endif
 
+# Strip out gtest and gtest_main from LLVM_LIBS_TO_EXPORT, these are not
+# installed and won't be available from the install tree.
+# FIXME: If we used llvm-config from the install tree this wouldn't be
+# necessary.
+LLVM_LIBS_TO_EXPORT := $(filter-out gtest gtest_main,$(LLVM_LIBS_TO_EXPORT))
+
 ifndef LLVM_LIBS_TO_EXPORT
 $(error LLVM_LIBS_TO_EXPORT cannot be empty)
 endif
@@ -54,11 +60,27 @@ endif
 
 OBJMODS := LLVMConfig.cmake LLVMConfigVersion.cmake LLVMExports.cmake
 
+LLVM_CONFIG_CODE := \
+\# Compute the CMake directory from the LLVMConfig.cmake file location.\n\
+get_filename_component(_LLVM_CMAKE_DIR "$${CMAKE_CURRENT_LIST_FILE}" PATH)\n\
+\# Compute the installation prefix from the LLVMConfig.cmake file location.\n\
+get_filename_component(LLVM_INSTALL_PREFIX "$${CMAKE_CURRENT_LIST_FILE}" PATH)\n
+
+# Compute number of levels (typically 3 - ``share/llvm/cmake/``) to PROJ_prefix
+# from PROJ_cmake, then emit the appropriate number of calls to
+# get_filename_components(). Note this assumes there are no spaces in the
+# cmake_path_suffix variable.
+cmake_path_suffix := $(subst $(PROJ_prefix),,$(subst $(DESTDIR),,$(PROJ_cmake)))
+cmake_path_dirs := $(subst /, ,$(cmake_path_suffix))
+LLVM_CONFIG_CODE += $(foreach __not_used,$(cmake_path_dirs),get_filename_component(LLVM_INSTALL_PREFIX "$${LLVM_INSTALL_PREFIX}" PATH)\n)
+
+LLVM_CONFIG_CODE += set(_LLVM_LIBRARY_DIR "$${LLVM_INSTALL_PREFIX}\/lib")
+
 $(PROJ_OBJ_DIR)/LLVMConfig.cmake: LLVMConfig.cmake.in Makefile $(LLVMBuildCMakeFrag)
 	$(Echo) 'Generating LLVM CMake package config file'
 	$(Verb) ( \
 	 cat $< | sed \
-	  -e 's/@LLVM_CONFIG_CODE@/set(LLVM_INSTALL_PREFIX "'"$(subst /,\/,$(PROJ_prefix))"'")/' \
+	  -e 's/@LLVM_CONFIG_CODE@/$(LLVM_CONFIG_CODE)/' \
 	  -e 's/@LLVM_VERSION_MAJOR@/'"$(LLVM_VERSION_MAJOR)"'/' \
 	  -e 's/@LLVM_VERSION_MINOR@/'"$(LLVM_VERSION_MINOR)"'/' \
 	  -e 's/@LLVM_VERSION_PATCH@/'"$(LLVM_VERSION_PATCH)"'/' \
@@ -81,17 +103,20 @@ $(PROJ_OBJ_DIR)/LLVMConfig.cmake: LLVMConfig.cmake.in Makefile $(LLVMBuildCMakeF
 	  -e 's/@LLVM_ON_UNIX@/'"$(LLVM_ON_UNIX)"'/' \
 	  -e 's/@LLVM_ON_WIN32@/'"$(LLVM_ON_WIN32)"'/' \
 	  -e 's/@LLVM_LIBDIR_SUFFIX@//' \
-	  -e 's/@LLVM_CONFIG_INCLUDE_DIRS@/'"$(subst /,\/,$(PROJ_includedir))"'/' \
-	  -e 's/@LLVM_CONFIG_LIBRARY_DIRS@/'"$(subst /,\/,$(PROJ_libdir))"'/' \
-	  -e 's/@LLVM_CONFIG_CMAKE_DIR@/'"$(subst /,\/,$(PROJ_cmake))"'/' \
-	  -e 's/@LLVM_CONFIG_TOOLS_BINARY_DIR@/'"$(subst /,\/,$(PROJ_bindir))"'/' \
+	  -e 's#@LLVM_CONFIG_INCLUDE_DIRS@#$${LLVM_INSTALL_PREFIX}/include#' \
+	  -e 's#@LLVM_CONFIG_LIBRARY_DIRS@#$${_LLVM_LIBRARY_DIR}#' \
+	  -e 's#@LLVM_CONFIG_CMAKE_DIR@#$${_LLVM_CMAKE_DIR}#' \
+	  -e 's#@LLVM_CONFIG_TOOLS_BINARY_DIR@#$${LLVM_INSTALL_PREFIX}/bin#' \
 	  -e 's/@LLVM_CONFIG_EXPORTS_FILE@/$${LLVM_CMAKE_DIR}\/LLVMExports.cmake/' \
 	  -e 's/@all_llvm_lib_deps@//' \
 	 && \
-	 grep '^set_property.*LLVMBUILD_LIB_DEPS_' "$(LLVMBuildCMakeFrag)" \
+	 ( grep '^set_property.*LLVMBUILD_LIB_DEPS_' "$(LLVMBuildCMakeFrag)" | \
+	   grep -v LLVMBUILD_LIB_DEPS_gtest ) && \
+	 echo 'unset(_LLVM_CMAKE_DIR)' && \
+	 echo 'unset(_LLVM_LIBRARY_DIR)' \
 	) > $@
 
-$(PROJ_OBJ_DIR)/LLVMConfigVersion.cmake: LLVMConfigVersion.cmake.in
+$(PROJ_OBJ_DIR)/LLVMConfigVersion.cmake: LLVMConfigVersion.cmake.in Makefile
 	$(Echo) 'Generating LLVM CMake package version file'
 	$(Verb) cat $< | sed \
 	  -e 's/@PACKAGE_VERSION@/'"$(LLVMVersion)"'/' \
@@ -100,13 +125,13 @@ $(PROJ_OBJ_DIR)/LLVMConfigVersion.cmake: LLVMConfigVersion.cmake.in
 	  -e 's/@LLVM_VERSION_PATCH@/'"$(LLVM_VERSION_PATCH)"'/' \
 	  > $@
 
-$(PROJ_OBJ_DIR)/LLVMExports.cmake: $(LLVMBuildCMakeExportsFrag)
+$(PROJ_OBJ_DIR)/LLVMExports.cmake: $(LLVMBuildCMakeExportsFrag) Makefile
 	$(Echo) 'Generating LLVM CMake target exports file'
 	$(Verb) ( \
 	  echo '# LLVM CMake target exports.  Do not include directly.' && \
 	  for lib in $(LLVM_LIBS_TO_EXPORT); do \
 	    echo 'add_library('"$$lib"' STATIC IMPORTED)' && \
-	    echo 'set_property(TARGET '"$$lib"' PROPERTY IMPORTED_LOCATION "'"$(PROJ_libdir)/lib$$lib.a"'")' ; \
+	    echo 'set_property(TARGET '"$$lib"' PROPERTY IMPORTED_LOCATION "$${_LLVM_LIBRARY_DIR}/'lib$$lib.a'")' ; \
 	  done && \
 	  cat "$(LLVMBuildCMakeExportsFrag)" && \
 	  echo 'set_property(TARGET LLVMSupport APPEND PROPERTY IMPORTED_LINK_INTERFACE_LIBRARIES '"$(subst -l,,$(LIBS))"')' \
diff --git a/cmake/modules/TableGen.cmake b/cmake/modules/TableGen.cmake
index bb0d844605f..85d720e91fd 100644
--- a/cmake/modules/TableGen.cmake
+++ b/cmake/modules/TableGen.cmake
@@ -32,7 +32,7 @@ function(tablegen project ofn)
     # The file in LLVM_TARGET_DEFINITIONS may be not in the current
     # directory and local_tds may not contain it, so we must
     # explicitly list it here:
-    DEPENDS ${${project}_TABLEGEN_EXE} ${local_tds} ${global_tds}
+    DEPENDS ${${project}_TABLEGEN_TARGET} ${local_tds} ${global_tds}
     ${LLVM_TARGET_DEFINITIONS_ABSOLUTE}
     COMMENT "Building ${ofn}..."
     )
@@ -90,10 +90,15 @@ macro(add_tablegen target project)
 
   # Effective tblgen executable to be used:
   set(${project}_TABLEGEN_EXE ${${project}_TABLEGEN} PARENT_SCOPE)
+  set(${project}_TABLEGEN_TARGET ${${project}_TABLEGEN} PARENT_SCOPE)
 
   if(LLVM_USE_HOST_TOOLS)
     if( ${${project}_TABLEGEN} STREQUAL "${target}" )
-      set(${project}_TABLEGEN_EXE "${LLVM_NATIVE_BUILD}/bin/${target}")
+      if (NOT CMAKE_CONFIGURATION_TYPES)
+        set(${project}_TABLEGEN_EXE "${LLVM_NATIVE_BUILD}/bin/${target}")
+      else()
+        set(${project}_TABLEGEN_EXE "${LLVM_NATIVE_BUILD}/Release/bin/${target}")
+      endif()
       set(${project}_TABLEGEN_EXE ${${project}_TABLEGEN_EXE} PARENT_SCOPE)
 
       add_custom_command(OUTPUT ${${project}_TABLEGEN_EXE}
@@ -101,8 +106,8 @@ macro(add_tablegen target project)
         DEPENDS CONFIGURE_LLVM_NATIVE ${target}
         WORKING_DIRECTORY ${LLVM_NATIVE_BUILD}
         COMMENT "Building native TableGen...")
-      add_custom_target(${project}NativeTableGen DEPENDS ${${project}_TABLEGEN_EXE})
-      add_dependencies(${project}NativeTableGen CONFIGURE_LLVM_NATIVE)
+      add_custom_target(${project}-tablegen-host DEPENDS ${${project}_TABLEGEN_EXE})
+      set(${project}_TABLEGEN_TARGET ${project}-tablegen-host PARENT_SCOPE)
     endif()
   endif()
 
diff --git a/codereview.settings b/codereview.settings
deleted file mode 100644
index 1c4f6bdeec6..00000000000
--- a/codereview.settings
+++ /dev/null
@@ -1,8 +0,0 @@
-# This file is used by gcl to get repository specific information.
-CODE_REVIEW_SERVER: codereview.chromium.org
-CC_LIST: native-client-reviews@googlegroups.com
-VIEW_VC: https://chromium.googlesource.com/native_client/pnacl-llvm/+/
-STATUS: http://nativeclient-status.appspot.com/status
-TRY_ON_UPLOAD: False
-TRYSERVER_PROJECT: nacl
-TRYSERVER_SVN_URL: svn://svn.chromium.org/chrome-try/try-nacl
diff --git a/configure b/configure
index 20c36aa1b2d..6b6d0cdd3e4 100755
--- a/configure
+++ b/configure
@@ -1,6 +1,6 @@
 #! /bin/sh
 # Guess values for system-dependent variables and create Makefiles.
-# Generated by GNU Autoconf 2.60 for LLVM 3.7.0svn.
+# Generated by GNU Autoconf 2.60 for LLVM 3.8.0svn.
 #
 # Report bugs to <http://llvm.org/bugs/>.
 #
@@ -561,8 +561,8 @@ SHELL=${CONFIG_SHELL-/bin/sh}
 # Identity of this package.
 PACKAGE_NAME='LLVM'
 PACKAGE_TARNAME='llvm'
-PACKAGE_VERSION='3.7.0svn'
-PACKAGE_STRING='LLVM 3.7.0svn'
+PACKAGE_VERSION='3.8.0svn'
+PACKAGE_STRING='LLVM 3.8.0svn'
 PACKAGE_BUGREPORT='http://llvm.org/bugs/'
 
 ac_unique_file="lib/IR/Module.cpp"
@@ -737,7 +737,6 @@ LLVM_ENUM_TARGETS
 LLVM_ENUM_ASM_PRINTERS
 LLVM_ENUM_ASM_PARSERS
 LLVM_ENUM_DISASSEMBLERS
-SUBZERO_ENUM_TARGETS
 OPTIMIZE_OPTION
 EXTRA_OPTIONS
 EXTRA_LD_OPTIONS
@@ -1334,7 +1333,7 @@ if test "$ac_init_help" = "long"; then
   # Omit some internal or obsolete options to make the list less imposing.
   # This message is too long to be a string in the A/UX 3.1 sh.
   cat <<_ACEOF
-\`configure' configures LLVM 3.7.0svn to adapt to many kinds of systems.
+\`configure' configures LLVM 3.8.0svn to adapt to many kinds of systems.
 
 Usage: $0 [OPTION]... [VAR=VALUE]...
 
@@ -1400,7 +1399,7 @@ fi
 
 if test -n "$ac_init_help"; then
   case $ac_init_help in
-     short | recursive ) echo "Configuration of LLVM 3.7.0svn:";;
+     short | recursive ) echo "Configuration of LLVM 3.8.0svn:";;
    esac
   cat <<\_ACEOF
 
@@ -1463,14 +1462,11 @@ Optional Features:
   --enable-targets        Build specific host targets: all or
                           target1,target2,... Valid targets are: host, x86,
                           x86_64, sparc, powerpc, arm64, arm, aarch64, mips,
-                          hexagon, xcore, msp430, nvptx, systemz, r600, js,
-                          and cpp (default=all)
+                          hexagon, xcore, msp430, nvptx, systemz, r600, bpf,
+                          wasm, js, and cpp (default=all)
   --enable-experimental-targets
                           Build experimental host targets: disable or
                           target1,target2,... (default=disable)
-  --enable-subzero-targets
-                          Build specific subzero targets:
-                          target1,target2,...(default=none)
   --enable-bindings       Build specific language bindings:
                           all,auto,none,{binding-name} (default=auto)
   --enable-terminfo       Query the terminfo database if available (default is
@@ -1499,6 +1495,8 @@ Optional Packages:
                           search for headers
   --with-gcc-toolchain    Directory where gcc is installed.
   --with-default-sysroot  Add --sysroot=<path> to all compiler invocations.
+  --with-clang-default-openmp-runtime
+                          The default OpenMP runtime for Clang.
   --with-binutils-include Specify path to binutils/include/ containing
                           plugin-api.h file for gold plugin.
   --with-bug-report-url   Specify the URL where bug reports should be
@@ -1585,7 +1583,7 @@ fi
 test -n "$ac_init_help" && exit $ac_status
 if $ac_init_version; then
   cat <<\_ACEOF
-LLVM configure 3.7.0svn
+LLVM configure 3.8.0svn
 generated by GNU Autoconf 2.60
 
 Copyright (C) 1992, 1993, 1994, 1995, 1996, 1998, 1999, 2000, 2001,
@@ -1601,7 +1599,7 @@ cat >config.log <<_ACEOF
 This file contains any messages produced by compilers while
 running configure, to aid debugging if configure makes a mistake.
 
-It was created by LLVM $as_me 3.7.0svn, which was
+It was created by LLVM $as_me 3.8.0svn, which was
 generated by GNU Autoconf 2.60.  Invocation command line was
 
   $ $0 $@
@@ -1956,7 +1954,7 @@ ac_compiler_gnu=$ac_cv_c_compiler_gnu
 
 
 LLVM_VERSION_MAJOR=3
-LLVM_VERSION_MINOR=7
+LLVM_VERSION_MINOR=8
 LLVM_VERSION_PATCH=0
 LLVM_VERSION_SUFFIX=svn
 
@@ -2034,6 +2032,12 @@ echo "$as_me: error: Already configured in ${srcdir}" >&2;}
   fi
 fi
 
+if test ${srcdir} == "." ; then
+  { { echo "$as_me:$LINENO: error: In-source builds are not allowed. Please configure from a separate build directory!" >&5
+echo "$as_me: error: In-source builds are not allowed. Please configure from a separate build directory!" >&2;}
+   { (exit 1); exit 1; }; }
+fi
+
 : ${CFLAGS=}
 : ${CXXFLAGS=}
 
@@ -3609,8 +3613,8 @@ sed 's/^/| /' conftest.$ac_ext >&5
 
   { echo "$as_me:$LINENO: result: no" >&5
 echo "${ECHO_T}no" >&6; }
-  { { echo "$as_me:$LINENO: error: Selected compiler could not find or parse C++ standard library headers.  Rerun with CC=c-compiler CXX=c++-compiler ./configure ..." >&5
-echo "$as_me: error: Selected compiler could not find or parse C++ standard library headers.  Rerun with CC=c-compiler CXX=c++-compiler ./configure ..." >&2;}
+  { { echo "$as_me:$LINENO: error: Selected compiler could not find or parse C++ standard library headers.  Rerun with CC=c-compiler CXX=c++-compiler LLVM_SRC_DIR/configure ..." >&5
+echo "$as_me: error: Selected compiler could not find or parse C++ standard library headers.  Rerun with CC=c-compiler CXX=c++-compiler LLVM_SRC_DIR/configure ..." >&2;}
    { (exit 1); exit 1; }; }
 
 fi
@@ -4056,11 +4060,6 @@ else
     llvm_cv_no_link_all_option="-Wl,--no-whole-archive"
     llvm_cv_os_type="Freestanding"
     llvm_cv_platform_type="Unix" ;;
-  *-*-nacl*)
-    llvm_cv_link_all_option="-Wl,--whole-archive"
-    llvm_cv_no_link_all_option="-Wl,--no-whole-archive"
-    llvm_cv_os_type="NativeClient"
-    llvm_cv_platform_type="Unix" ;;
   *)
     llvm_cv_link_all_option=""
     llvm_cv_no_link_all_option=""
@@ -4193,6 +4192,7 @@ else
   hexagon-*)              llvm_cv_target_arch="Hexagon" ;;
   nvptx-*)                llvm_cv_target_arch="NVPTX" ;;
   s390x-*)                llvm_cv_target_arch="SystemZ" ;;
+  wasm*-*)                llvm_cv_target_arch="WebAssembly" ;;
   *)                      llvm_cv_target_arch="Unknown" ;;
 esac
 fi
@@ -4229,6 +4229,7 @@ case $host in
   msp430-*)               host_arch="MSP430" ;;
   hexagon-*)              host_arch="Hexagon" ;;
   s390x-*)                host_arch="SystemZ" ;;
+  wasm*-*)                host_arch="WebAssembly" ;;
   *)                      host_arch="Unknown" ;;
 esac
 
@@ -5155,6 +5156,8 @@ else
     NVPTX)       TARGET_HAS_JIT=0
  ;;
     SystemZ)     TARGET_HAS_JIT=1
+ ;;
+    WebAssembly) TARGET_HAS_JIT=0
  ;;
     *)           TARGET_HAS_JIT=0
  ;;
@@ -5614,7 +5617,7 @@ _ACEOF
 
 fi
 
-ALL_TARGETS="X86 Sparc PowerPC ARM AArch64 Mips XCore MSP430 CppBackend JSBackend NVPTX Hexagon SystemZ R600"
+ALL_TARGETS="X86 Sparc PowerPC ARM AArch64 Mips XCore MSP430 CppBackend NVPTX Hexagon SystemZ AMDGPU BPF JSBackend"
 ALL_TARGETS=$ALL_TARGETS
 
 
@@ -5640,6 +5643,7 @@ case "$enableval" in
         aarch64)  TARGETS_TO_BUILD="AArch64 $TARGETS_TO_BUILD" ;;
         arm64)    TARGETS_TO_BUILD="AArch64 $TARGETS_TO_BUILD" ;;
         arm)      TARGETS_TO_BUILD="ARM $TARGETS_TO_BUILD" ;;
+        bpf)      TARGETS_TO_BUILD="BPF $TARGETS_TO_BUILD" ;;
         mips)     TARGETS_TO_BUILD="Mips $TARGETS_TO_BUILD" ;;
         mipsel)   TARGETS_TO_BUILD="Mips $TARGETS_TO_BUILD" ;;
         mips64)   TARGETS_TO_BUILD="Mips $TARGETS_TO_BUILD" ;;
@@ -5651,7 +5655,9 @@ case "$enableval" in
         hexagon)  TARGETS_TO_BUILD="Hexagon $TARGETS_TO_BUILD" ;;
         nvptx)    TARGETS_TO_BUILD="NVPTX $TARGETS_TO_BUILD" ;;
         systemz)  TARGETS_TO_BUILD="SystemZ $TARGETS_TO_BUILD" ;;
-        r600)     TARGETS_TO_BUILD="R600 $TARGETS_TO_BUILD" ;;
+        amdgpu)   TARGETS_TO_BUILD="AMDGPU $TARGETS_TO_BUILD" ;;
+        r600)     TARGETS_TO_BUILD="AMDGPU $TARGETS_TO_BUILD" ;;
+        wasm)     TARGETS_TO_BUILD="WebAssembly $TARGETS_TO_BUILD" ;;
         host) case "$llvm_cv_target_arch" in
             x86)         TARGETS_TO_BUILD="X86 $TARGETS_TO_BUILD" ;;
             x86_64)      TARGETS_TO_BUILD="X86 $TARGETS_TO_BUILD" ;;
@@ -5665,6 +5671,7 @@ case "$enableval" in
             Hexagon)     TARGETS_TO_BUILD="Hexagon $TARGETS_TO_BUILD" ;;
             NVPTX)       TARGETS_TO_BUILD="NVPTX $TARGETS_TO_BUILD" ;;
             SystemZ)     TARGETS_TO_BUILD="SystemZ $TARGETS_TO_BUILD" ;;
+            WebAssembly) TARGETS_TO_BUILD="WebAssembly $TARGETS_TO_BUILD" ;;
             *)       { { echo "$as_me:$LINENO: error: Can not set target to build" >&5
 echo "$as_me: error: Can not set target to build" >&2;}
    { (exit 1); exit 1; }; } ;;
@@ -5769,38 +5776,6 @@ done
 
 
 
-SUBZERO_TARGETS_TO_BUILD=""
-# Check whether --enable-subzero_targets was given.
-if test "${enable_subzero_targets+set}" = set; then
-  enableval=$enable_subzero_targets;
-else
-  enableval=none
-fi
-
-case "$enableval" in
-  none) ;;
-  *)for a_target in `echo $enableval|sed -e 's/,/ /g' ` ; do
-      case "$a_target" in
-        X8632)  SUBZERO_TARGETS_TO_BUILD="X8632 $SUBZERO_TARGETS_TO_BUILD" ;;
-        X8664)  SUBZERO_TARGETS_TO_BUILD="X8664 $SUBZERO_TARGETS_TO_BUILD" ;;
-        ARM32)  SUBZERO_TARGETS_TO_BUILD="ARM32 $SUBZERO_TARGETS_TO_BUILD" ;;
-        ARM64)  SUBZERO_TARGETS_TO_BUILD="ARM64 $SUBZERO_TARGETS_TO_BUILD" ;;
-        MIPS32) SUBZERO_TARGETS_TO_BUILD="MIPS32 $SUBZERO_TARGETS_TO_BUILD" ;;
-        *) { { echo "$as_me:$LINENO: error: Unrecognized subzero target $a_target" >&5
-echo "$as_me: error: Unrecognized subzero target $a_target" >&2;}
-   { (exit 1); exit 1; }; } ;;
-      esac
-  done
-  ;;
-esac
-
-SUBZERO_ENUM_TARGETS=""
-for target_to_build in $SUBZERO_TARGETS_TO_BUILD; do
-  SUBZERO_ENUM_TARGETS="SUBZERO_TARGET($target_to_build) $SUBZERO_ENUM_TARGETS"
-done
-
-
-
 # Check whether --with-optimize-option was given.
 if test "${with_optimize_option+set}" = set; then
   withval=$with_optimize_option;
@@ -5969,6 +5944,20 @@ _ACEOF
 
 
 
+# Check whether --with-clang-default-openmp-runtime was given.
+if test "${with_clang_default_openmp_runtime+set}" = set; then
+  withval=$with_clang_default_openmp_runtime;
+else
+  withval="libgomp"
+fi
+
+
+cat >>confdefs.h <<_ACEOF
+#define CLANG_DEFAULT_OPENMP_RUNTIME "$withval"
+_ACEOF
+
+
+
 # Check whether --with-binutils-include was given.
 if test "${with_binutils_include+set}" = set; then
   withval=$with_binutils_include;
@@ -8736,6 +8725,87 @@ _ACEOF
 fi
 
 
+{ echo "$as_me:$LINENO: checking for main in -lole32" >&5
+echo $ECHO_N "checking for main in -lole32... $ECHO_C" >&6; }
+if test "${ac_cv_lib_ole32_main+set}" = set; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+  ac_check_lib_save_LIBS=$LIBS
+LIBS="-lole32  $LIBS"
+cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+
+
+int
+main ()
+{
+return main ();
+  ;
+  return 0;
+}
+_ACEOF
+rm -f conftest.$ac_objext conftest$ac_exeext
+if { (ac_try="$ac_link"
+case "(($ac_try" in
+  *\"* | *\`* | *\\*) ac_try_echo=\$ac_try;;
+  *) ac_try_echo=$ac_try;;
+esac
+eval "echo \"\$as_me:$LINENO: $ac_try_echo\"") >&5
+  (eval "$ac_link") 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } &&
+	 { ac_try='test -z "$ac_c_werror_flag" || test ! -s conftest.err'
+  { (case "(($ac_try" in
+  *\"* | *\`* | *\\*) ac_try_echo=\$ac_try;;
+  *) ac_try_echo=$ac_try;;
+esac
+eval "echo \"\$as_me:$LINENO: $ac_try_echo\"") >&5
+  (eval "$ac_try") 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; } &&
+	 { ac_try='test -s conftest$ac_exeext'
+  { (case "(($ac_try" in
+  *\"* | *\`* | *\\*) ac_try_echo=\$ac_try;;
+  *) ac_try_echo=$ac_try;;
+esac
+eval "echo \"\$as_me:$LINENO: $ac_try_echo\"") >&5
+  (eval "$ac_try") 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+  ac_cv_lib_ole32_main=yes
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+	ac_cv_lib_ole32_main=no
+fi
+
+rm -f core conftest.err conftest.$ac_objext \
+      conftest$ac_exeext conftest.$ac_ext
+LIBS=$ac_check_lib_save_LIBS
+fi
+{ echo "$as_me:$LINENO: result: $ac_cv_lib_ole32_main" >&5
+echo "${ECHO_T}$ac_cv_lib_ole32_main" >&6; }
+if test $ac_cv_lib_ole32_main = yes; then
+  cat >>confdefs.h <<_ACEOF
+#define HAVE_LIBOLE32 1
+_ACEOF
+
+  LIBS="-lole32 $LIBS"
+
+fi
+
+
 { echo "$as_me:$LINENO: checking for main in -lpsapi" >&5
 echo $ECHO_N "checking for main in -lpsapi... $ECHO_C" >&6; }
 if test "${ac_cv_lib_psapi_main+set}" = set; then
@@ -18074,8 +18144,6 @@ ac_config_headers="$ac_config_headers include/llvm/Config/config.h include/llvm/
 
 
 
-ac_config_files="$ac_config_files include/llvm/Config/SZTargets.def"
-
 ac_config_files="$ac_config_files include/llvm/Config/Targets.def"
 
 ac_config_files="$ac_config_files include/llvm/Config/AsmPrinters.def"
@@ -18543,7 +18611,7 @@ exec 6>&1
 # report actual input values of CONFIG_FILES etc. instead of their
 # values after options handling.
 ac_log="
-This file was extended by LLVM $as_me 3.7.0svn, which was
+This file was extended by LLVM $as_me 3.8.0svn, which was
 generated by GNU Autoconf 2.60.  Invocation command line was
 
   CONFIG_FILES    = $CONFIG_FILES
@@ -18596,7 +18664,7 @@ Report bugs to <bug-autoconf@gnu.org>."
 _ACEOF
 cat >>$CONFIG_STATUS <<_ACEOF
 ac_cs_version="\\
-LLVM config.status 3.7.0svn
+LLVM config.status 3.8.0svn
 configured by $0, generated by GNU Autoconf 2.60,
   with options \\"`echo "$ac_configure_args" | sed 's/^ //; s/[\\""\`\$]/\\\\&/g'`\\"
 
@@ -18711,7 +18779,6 @@ do
   case $ac_config_target in
     "include/llvm/Config/config.h") CONFIG_HEADERS="$CONFIG_HEADERS include/llvm/Config/config.h" ;;
     "include/llvm/Config/llvm-config.h") CONFIG_HEADERS="$CONFIG_HEADERS include/llvm/Config/llvm-config.h" ;;
-    "include/llvm/Config/SZTargets.def") CONFIG_FILES="$CONFIG_FILES include/llvm/Config/SZTargets.def" ;;
     "include/llvm/Config/Targets.def") CONFIG_FILES="$CONFIG_FILES include/llvm/Config/Targets.def" ;;
     "include/llvm/Config/AsmPrinters.def") CONFIG_FILES="$CONFIG_FILES include/llvm/Config/AsmPrinters.def" ;;
     "include/llvm/Config/AsmParsers.def") CONFIG_FILES="$CONFIG_FILES include/llvm/Config/AsmParsers.def" ;;
@@ -18974,7 +19041,6 @@ LLVM_ENUM_TARGETS!$LLVM_ENUM_TARGETS$ac_delim
 LLVM_ENUM_ASM_PRINTERS!$LLVM_ENUM_ASM_PRINTERS$ac_delim
 LLVM_ENUM_ASM_PARSERS!$LLVM_ENUM_ASM_PARSERS$ac_delim
 LLVM_ENUM_DISASSEMBLERS!$LLVM_ENUM_DISASSEMBLERS$ac_delim
-SUBZERO_ENUM_TARGETS!$SUBZERO_ENUM_TARGETS$ac_delim
 OPTIMIZE_OPTION!$OPTIMIZE_OPTION$ac_delim
 EXTRA_OPTIONS!$EXTRA_OPTIONS$ac_delim
 EXTRA_LD_OPTIONS!$EXTRA_LD_OPTIONS$ac_delim
@@ -19033,6 +19099,7 @@ HAVE_LIBZ!$HAVE_LIBZ$ac_delim
 HUGE_VAL_SANITY!$HUGE_VAL_SANITY$ac_delim
 MMAP_FILE!$MMAP_FILE$ac_delim
 SHLIBEXT!$SHLIBEXT$ac_delim
+LLVM_PREFIX!$LLVM_PREFIX$ac_delim
 _ACEOF
 
   if test `sed -n "s/.*$ac_delim\$/X/p" conf$$subs.sed | grep -c X` = 97; then
@@ -19074,7 +19141,6 @@ _ACEOF
 ac_delim='%!_!# '
 for ac_last_try in false false false false false :; do
   cat >conf$$subs.sed <<_ACEOF
-LLVM_PREFIX!$LLVM_PREFIX$ac_delim
 LLVM_BINDIR!$LLVM_BINDIR$ac_delim
 LLVM_DATADIR!$LLVM_DATADIR$ac_delim
 LLVM_DOCSDIR!$LLVM_DOCSDIR$ac_delim
@@ -19095,7 +19161,7 @@ LIBOBJS!$LIBOBJS$ac_delim
 LTLIBOBJS!$LTLIBOBJS$ac_delim
 _ACEOF
 
-  if test `sed -n "s/.*$ac_delim\$/X/p" conf$$subs.sed | grep -c X` = 19; then
+  if test `sed -n "s/.*$ac_delim\$/X/p" conf$$subs.sed | grep -c X` = 18; then
     break
   elif $ac_last_try; then
     { { echo "$as_me:$LINENO: error: could not make $CONFIG_STATUS" >&5
diff --git a/docs/AMDGPUUsage.rst b/docs/AMDGPUUsage.rst
new file mode 100644
index 00000000000..97d6662a2ed
--- /dev/null
+++ b/docs/AMDGPUUsage.rst
@@ -0,0 +1,177 @@
+==============================
+User Guide for AMDGPU Back-end
+==============================
+
+Introduction
+============
+
+The AMDGPU back-end provides ISA code generation for AMD GPUs, starting with
+the R600 family up until the current Volcanic Islands (GCN Gen 3).
+
+
+Assembler
+=========
+
+The assembler is currently considered experimental.
+
+For syntax examples look in test/MC/AMDGPU.
+
+Below some of the currently supported features (modulo bugs).  These
+all apply to the Southern Islands ISA, Sea Islands and Volcanic Islands
+are also supported but may be missing some instructions and have more bugs:
+
+DS Instructions
+---------------
+All DS instructions are supported.
+
+FLAT Instructions
+------------------
+These instructions are only present in the Sea Islands and Volcanic Islands
+instruction set.  All FLAT instructions are supported for these architectures
+
+MUBUF Instructions
+------------------
+All non-atomic MUBUF instructions are supported.
+
+SMRD Instructions
+-----------------
+Only the s_load_dword* SMRD instructions are supported.
+
+SOP1 Instructions
+-----------------
+All SOP1 instructions are supported.
+
+SOP2 Instructions
+-----------------
+All SOP2 instructions are supported.
+
+SOPC Instructions
+-----------------
+All SOPC instructions are supported.
+
+SOPP Instructions
+-----------------
+
+Unless otherwise mentioned, all SOPP instructions that have one or more
+operands accept integer operands only.  No verification is performed
+on the operands, so it is up to the programmer to be familiar with the
+range or acceptable values.
+
+s_waitcnt
+^^^^^^^^^
+
+s_waitcnt accepts named arguments to specify which memory counter(s) to
+wait for.
+
+.. code-block:: nasm
+
+   // Wait for all counters to be 0
+   s_waitcnt 0
+
+   // Equivalent to s_waitcnt 0.  Counter names can also be delimited by
+   // '&' or ','.
+   s_waitcnt vmcnt(0) expcnt(0) lgkcmt(0)
+
+   // Wait for vmcnt counter to be 1.
+   s_waitcnt vmcnt(1)
+
+VOP1, VOP2, VOP3, VOPC Instructions
+-----------------------------------
+
+All 32-bit and 64-bit encodings should work.
+
+The assembler will automatically detect which encoding size to use for
+VOP1, VOP2, and VOPC instructions based on the operands.  If you want to force
+a specific encoding size, you can add an _e32 (for 32-bit encoding) or
+_e64 (for 64-bit encoding) suffix to the instruction.  Most, but not all
+instructions support an explicit suffix.  These are all valid assembly
+strings:
+
+.. code-block:: nasm
+
+   v_mul_i32_i24 v1, v2, v3
+   v_mul_i32_i24_e32 v1, v2, v3
+   v_mul_i32_i24_e64 v1, v2, v3
+
+Assembler Directives
+--------------------
+
+.hsa_code_object_version major, minor
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+*major* and *minor* are integers that specify the version of the HSA code
+object that will be generated by the assembler.  This value will be stored
+in an entry of the .note section.
+
+.hsa_code_object_isa [major, minor, stepping, vendor, arch]
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+*major*, *minor*, and *stepping* are all integers that describe the instruction
+set architecture (ISA) version of the assembly program.
+
+*vendor* and *arch* are quoted strings.  *vendor* should always be equal to
+"AMD" and *arch* should always be equal to "AMDGPU".
+
+If no arguments are specified, then the assembler will derive the ISA version,
+*vendor*, and *arch* from the value of the -mcpu option that is passed to the
+assembler.
+
+ISA version, *vendor*, and *arch* will all be stored in a single entry of the
+.note section.
+
+.amd_kernel_code_t
+^^^^^^^^^^^^^^^^^^
+
+This directive marks the beginning of a list of key / value pairs that are used
+to specify the amd_kernel_code_t object that will be emitted by the assembler.
+The list must be terminated by the *.end_amd_kernel_code_t* directive.  For
+any amd_kernel_code_t values that are unspecified a default value will be
+used.  The default value for all keys is 0, with the following exceptions:
+
+- *kernel_code_version_major* defaults to 1.
+- *machine_kind* defaults to 1.
+- *machine_version_major*, *machine_version_minor*, and
+  *machine_version_stepping* are derived from the value of the -mcpu option
+  that is passed to the assembler.
+- *kernel_code_entry_byte_offset* defaults to 256.
+- *wavefront_size* defaults to 6.
+- *kernarg_segment_alignment*, *group_segment_alignment*, and
+  *private_segment_alignment* default to 4.  Note that alignments are specified
+  as a power of two, so a value of **n** means an alignment of 2^ **n**.
+
+The *.amd_kernel_code_t* directive must be placed immediately after the
+function label and before any instructions.
+
+For a full list of amd_kernel_code_t keys, see the examples in
+test/CodeGen/AMDGPU/hsa.s.  For an explanation of the meanings of the different
+keys, see the comments in lib/Target/AMDGPU/AmdKernelCodeT.h
+
+Here is an example of a minimal amd_kernel_code_t specification:
+
+.. code-block:: nasm
+
+   .hsa_code_object_version 1,0
+   .hsa_code_object_isa
+
+   .text
+
+   hello_world:
+
+      .amd_kernel_code_t
+         enable_sgpr_kernarg_segment_ptr = 1
+         is_ptr64 = 1
+         compute_pgm_rsrc1_vgprs = 0
+         compute_pgm_rsrc1_sgprs = 0
+         compute_pgm_rsrc2_user_sgpr = 2
+         kernarg_segment_byte_size = 8
+         wavefront_sgpr_count = 2
+         workitem_vgpr_count = 3
+     .end_amd_kernel_code_t
+
+     s_load_dwordx2 s[0:1], s[0:1] 0x0
+     v_mov_b32 v0, 3.14159
+     s_waitcnt lgkmcnt(0)
+     v_mov_b32 v1, s0
+     v_mov_b32 v2, s1
+     flat_store_dword v0, v[1:2]
+     s_endpgm
diff --git a/docs/AliasAnalysis.rst b/docs/AliasAnalysis.rst
index 1cbaee703f3..f62cc3fe4d3 100644
--- a/docs/AliasAnalysis.rst
+++ b/docs/AliasAnalysis.rst
@@ -286,8 +286,8 @@ Mod/Ref result, simply return whatever the superclass computes.  For example:
 
 .. code-block:: c++
 
-  AliasAnalysis::AliasResult alias(const Value *V1, unsigned V1Size,
-                                   const Value *V2, unsigned V2Size) {
+  AliasResult alias(const Value *V1, unsigned V1Size,
+                    const Value *V2, unsigned V2Size) {
     if (...)
       return NoAlias;
     ...
diff --git a/docs/BitCodeFormat.rst b/docs/BitCodeFormat.rst
index 4b398a4e956..25ea421ed08 100644
--- a/docs/BitCodeFormat.rst
+++ b/docs/BitCodeFormat.rst
@@ -741,7 +741,7 @@ global variable. The operand fields are:
 MODULE_CODE_FUNCTION Record
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
-``[FUNCTION, type, callingconv, isproto, linkage, paramattr, alignment, section, visibility, gc, prologuedata, dllstorageclass, comdat, prefixdata]``
+``[FUNCTION, type, callingconv, isproto, linkage, paramattr, alignment, section, visibility, gc, prologuedata, dllstorageclass, comdat, prefixdata, personalityfn]``
 
 The ``FUNCTION`` record (code 8) marks the declaration or definition of a
 function. The operand fields are:
@@ -795,6 +795,8 @@ function. The operand fields are:
 * *prefixdata*: If non-zero, the value index of the prefix data for this function,
   plus 1.
 
+* *personalityfn*: If non-zero, the value index of the personality function for this function,
+  plus 1.
 
 MODULE_CODE_ALIAS Record
 ^^^^^^^^^^^^^^^^^^^^^^^^
diff --git a/docs/BranchWeightMetadata.rst b/docs/BranchWeightMetadata.rst
index aff7923d186..2ebc6c32416 100644
--- a/docs/BranchWeightMetadata.rst
+++ b/docs/BranchWeightMetadata.rst
@@ -115,3 +115,26 @@ CFG Modifications
 Branch Weight Metatada is not proof against CFG changes. If terminator operands'
 are changed some action should be taken. In other case some misoptimizations may
 occur due to incorrent branch prediction information.
+
+Function Entry Counts
+=====================
+
+To allow comparing different functions durint inter-procedural analysis and
+optimization, ``MD_prof`` nodes can also be assigned to a function definition.
+The first operand is a string indicating the name of the associated counter.
+
+Currently, one counter is supported: "function_entry_count". This is a 64-bit
+counter that indicates the number of times that this function was invoked (in
+the case of instrumentation-based profiles). In the case of sampling-based
+profiles, this counter is an approximation of how many times the function was
+invoked.
+
+For example, in the code below, the instrumentation for function foo()
+indicates that it was called 2,590 times at runtime.
+
+.. code-block:: llvm
+
+  define i32 @foo() !prof !1 {
+    ret i32 0
+  }
+  !1 = !{!"function_entry_count", i64 2590}
diff --git a/docs/BuildingLLVMWithAutotools.rst b/docs/BuildingLLVMWithAutotools.rst
index 4c04f1e1865..6f9a1341055 100644
--- a/docs/BuildingLLVMWithAutotools.rst
+++ b/docs/BuildingLLVMWithAutotools.rst
@@ -121,7 +121,7 @@ To configure LLVM, follow these steps:
 
    .. code-block:: console
 
-     % SRC_ROOT/configure --prefix=/install/path [other options]
+     % $LLVM_SRC_DIR/configure --prefix=/install/path [other options]
 
 Compiling the LLVM Suite Source Code
 ------------------------------------
@@ -187,7 +187,7 @@ source code:
 ``gmake install``
 
   Installs LLVM header files, libraries, tools, and documentation in a hierarchy
-  under ``$PREFIX``, specified with ``./configure --prefix=[dir]``, which
+  under ``$PREFIX``, specified with ``$LLVM_SRC_DIR/configure --prefix=[dir]``, which
   defaults to ``/usr/local``.
 
 ``gmake -C runtime install-bytecode``
@@ -240,7 +240,7 @@ There are some HTML documents that have not yet been converted to the new
 system (which uses the easy-to-read and easy-to-write
 `reStructuredText <http://sphinx-doc.org/rest.html>`_ plaintext markup
 language).
-The generated documentation is built in the ``SRC_ROOT/docs`` directory using
+The generated documentation is built in the ``$LLVM_SRC_DIR/docs`` directory using
 a special makefile.
 For instructions on how to install Sphinx, see
 `Sphinx Introduction for LLVM Developers
@@ -250,7 +250,7 @@ HTML documentation by doing the following:
 
 .. code-block:: console
 
-  $ cd SRC_ROOT/docs
+  $ cd $LLVM_SRC_DIR/docs
   $ make -f Makefile.sphinx
 
 This creates a ``_build/html`` sub-directory with all of the HTML files, not
@@ -296,7 +296,7 @@ This is accomplished in the typical autoconf manner:
 
   .. code-block:: console
 
-    % SRC_ROOT/configure
+    % $LLVM_SRC_DIR/configure
 
 The LLVM build will place files underneath *OBJ_ROOT* in directories named after
 the build type:
diff --git a/docs/CMake.rst b/docs/CMake.rst
index b9e473fab2b..b6dd83850ff 100644
--- a/docs/CMake.rst
+++ b/docs/CMake.rst
@@ -327,8 +327,8 @@ LLVM-specific variables
 
 **LLVM_USE_SANITIZER**:STRING
   Define the sanitizer used to build LLVM binaries and tests. Possible values
-  are ``Address``, ``Memory``, ``MemoryWithOrigins`` and ``Undefined``.
-  Defaults to empty string.
+  are ``Address``, ``Memory``, ``MemoryWithOrigins``, ``Undefined``, ``Thread``,
+  and ``Address;Undefined``. Defaults to empty string.
 
 **LLVM_PARALLEL_COMPILE_JOBS**:STRING
   Define the maximum number of concurrent compilation jobs.
diff --git a/docs/CMakeLists.txt b/docs/CMakeLists.txt
index da27627f07e..e02fcd9a4d6 100644
--- a/docs/CMakeLists.txt
+++ b/docs/CMakeLists.txt
@@ -138,7 +138,9 @@ if( NOT uses_ocaml LESS 0 )
     COMMAND ${CMAKE_COMMAND} -E remove_directory ${CMAKE_CURRENT_BINARY_DIR}/ocamldoc/html
     COMMAND ${CMAKE_COMMAND} -E make_directory ${CMAKE_CURRENT_BINARY_DIR}/ocamldoc/html
     COMMAND ${OCAMLFIND} ocamldoc -d ${CMAKE_CURRENT_BINARY_DIR}/ocamldoc/html
-                                  -sort -colorize-code -html ${odoc_files})
+                                  -sort -colorize-code -html ${odoc_files}
+    COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_CURRENT_SOURCE_DIR}/_ocamldoc/style.css
+                                     ${CMAKE_CURRENT_BINARY_DIR}/ocamldoc/html)
 
   add_dependencies(ocaml_doc ${doc_targets})
 
diff --git a/docs/CodeGenerator.rst b/docs/CodeGenerator.rst
index 75d40db958c..516031dd7bc 100644
--- a/docs/CodeGenerator.rst
+++ b/docs/CodeGenerator.rst
@@ -749,7 +749,7 @@ The SelectionDAG is a Directed-Acyclic-Graph whose nodes are instances of the
 ``SDNode`` class.  The primary payload of the ``SDNode`` is its operation code
 (Opcode) that indicates what operation the node performs and the operands to the
 operation.  The various operation node types are described at the top of the
-``include/llvm/CodeGen/SelectionDAGNodes.h`` file.
+``include/llvm/CodeGen/ISDOpcodes.h`` file.
 
 Although most operations define a single value, each node in the graph may
 define multiple values.  For example, a combined div/rem operation will define
@@ -829,7 +829,7 @@ One great way to visualize what is going on here is to take advantage of a few
 LLC command line options.  The following options pop up a window displaying the
 SelectionDAG at specific times (if you only get errors printed to the console
 while using this, you probably `need to configure your
-system <ProgrammersManual.html#ViewGraph>`_ to add support for it).
+system <ProgrammersManual.html#viewing-graphs-while-debugging-code>`_ to add support for it).
 
 * ``-view-dag-combine1-dags`` displays the DAG after being built, before the
   first optimization pass.
diff --git a/docs/CodingStandards.rst b/docs/CodingStandards.rst
index 992939259ab..498d76b04d8 100644
--- a/docs/CodingStandards.rst
+++ b/docs/CodingStandards.rst
@@ -131,6 +131,12 @@ unlikely to be supported by our host compilers.
     cannot synthesize them.
 * Initializer lists: N2627_
 * Delegating constructors: N1986_
+* Default member initializers (non-static data member initializers): N2756_
+
+  * Only use these for scalar members that would otherwise be left
+    uninitialized. Non-scalar members generally have appropriate default
+    constructors, and MSVC 2013 has problems when braced initializer lists are
+    involved.
 
 .. _N2118: http://www.open-std.org/jtc1/sc22/wg21/docs/papers/2006/n2118.html
 .. _N2439: http://www.open-std.org/jtc1/sc22/wg21/docs/papers/2007/n2439.htm
@@ -156,7 +162,7 @@ unlikely to be supported by our host compilers.
 .. _N2346: http://www.open-std.org/jtc1/sc22/wg21/docs/papers/2007/n2346.htm
 .. _N2627: http://www.open-std.org/jtc1/sc22/wg21/docs/papers/2008/n2672.htm
 .. _N1986: http://www.open-std.org/jtc1/sc22/wg21/docs/papers/2006/n1986.pdf
-.. _MSVC-compatible RTTI: http://llvm.org/PR18951
+.. _N2756: http://www.open-std.org/jtc1/sc22/wg21/docs/papers/2008/n2756.htm
 
 The supported features in the C++11 standard libraries are less well tracked,
 but also much greater. Most of the standard libraries implement most of C++11's
@@ -241,8 +247,8 @@ tree.  The standard header looks like this:
   //===----------------------------------------------------------------------===//
   ///
   /// \file
-  /// \brief This file contains the declaration of the Instruction class, which is
-  /// the base class for all of the VM instructions.
+  /// This file contains the declaration of the Instruction class, which is the
+  /// base class for all of the VM instructions.
   ///
   //===----------------------------------------------------------------------===//
 
@@ -262,10 +268,10 @@ file is released under.  This makes it perfectly clear what terms the source
 code can be distributed under and should not be modified in any way.
 
 The main body is a ``doxygen`` comment (identified by the ``///`` comment
-marker instead of the usual ``//``) describing the purpose of the file.  It
-should have a ``\brief`` command that describes the file in one or two
-sentences.  Any additional information should be separated by a blank line.  If
-an algorithm is being implemented or something tricky is going on, a reference
+marker instead of the usual ``//``) describing the purpose of the file.  The
+first sentence or a passage beginning with ``\brief`` is used as an abstract.
+Any additional information should be separated by a blank line.  If an
+algorithm is being implemented or something tricky is going on, a reference
 to the paper where it is published should be included, as well as any notes or
 *gotchas* in the code to watch out for.
 
@@ -314,10 +320,11 @@ Doxygen Use in Documentation Comments
 Use the ``\file`` command to turn the standard file header into a file-level
 comment.
 
-Include descriptive ``\brief`` paragraphs for all public interfaces (public
-classes, member and non-member functions).  Explain API use and purpose in
-``\brief`` paragraphs, don't just restate the information that can be inferred
-from the API name.  Put detailed discussion into separate paragraphs.
+Include descriptive paragraphs for all public interfaces (public classes,
+member and non-member functions).  Don't just restate the information that can
+be inferred from the API name.  The first sentence or a paragraph beginning
+with ``\brief`` is used as an abstract. Put detailed discussion into separate
+paragraphs.
 
 To refer to parameter names inside a paragraph, use the ``\p name`` command.
 Don't use the ``\arg name`` command since it starts a new paragraph that
@@ -337,8 +344,8 @@ A minimal documentation comment:
 
 .. code-block:: c++
 
-  /// \brief Does foo and bar.
-  void fooBar(bool Baz);
+  /// Sets the xyzzy property to \p Baz.
+  void setXyzzy(bool Baz);
 
 A documentation comment that uses all Doxygen features in a preferred way:
 
@@ -395,10 +402,10 @@ Correct:
 
   // In Something.h:
 
-  /// \brief An abstraction for some complicated thing.
+  /// An abstraction for some complicated thing.
   class Something {
   public:
-    /// \brief Does foo and bar.
+    /// Does foo and bar.
     void fooBar();
   };
 
diff --git a/docs/CommandGuide/FileCheck.rst b/docs/CommandGuide/FileCheck.rst
index f48bf474519..03c88297677 100644
--- a/docs/CommandGuide/FileCheck.rst
+++ b/docs/CommandGuide/FileCheck.rst
@@ -200,9 +200,9 @@ For example, the following works like you'd expect:
 
 .. code-block:: llvm
 
-   !0 = !MDLocation(line: 5, scope: !1, inlinedAt: !2)
+   !0 = !DILocation(line: 5, scope: !1, inlinedAt: !2)
 
-   ; CHECK:       !MDLocation(line: 5,
+   ; CHECK:       !DILocation(line: 5,
    ; CHECK-NOT:               column:
    ; CHECK-SAME:              scope: ![[SCOPE:[0-9]+]]
 
diff --git a/docs/CommandGuide/index.rst b/docs/CommandGuide/index.rst
index ed18cd048aa..46db57f1c84 100644
--- a/docs/CommandGuide/index.rst
+++ b/docs/CommandGuide/index.rst
@@ -21,6 +21,7 @@ Basic Commands
    lli
    llvm-link
    llvm-ar
+   llvm-lib
    llvm-nm
    llvm-config
    llvm-diff
diff --git a/docs/CommandGuide/lit.rst b/docs/CommandGuide/lit.rst
index 9c63848a76d..e820eef2faf 100644
--- a/docs/CommandGuide/lit.rst
+++ b/docs/CommandGuide/lit.rst
@@ -56,7 +56,7 @@ GENERAL OPTIONS
  Search for :file:`{NAME}.cfg` and :file:`{NAME}.site.cfg` when searching for
  test suites, instead of :file:`lit.cfg` and :file:`lit.site.cfg`.
 
-.. option:: --param NAME, --param NAME=VALUE
+.. option:: -D NAME, -D NAME=VALUE, --param NAME, --param NAME=VALUE
 
  Add a user defined parameter ``NAME`` with the given ``VALUE`` (or the empty
  string if not given).  The meaning and use of these parameters is test suite
@@ -161,7 +161,7 @@ ADDITIONAL OPTIONS
 
 .. option:: --show-tests
 
- List all of the the discovered tests and exit.
+ List all of the discovered tests and exit.
 
 EXIT STATUS
 -----------
diff --git a/docs/CommandGuide/lli.rst b/docs/CommandGuide/lli.rst
index 502fbd60935..9da13ee47e0 100644
--- a/docs/CommandGuide/lli.rst
+++ b/docs/CommandGuide/lli.rst
@@ -1,172 +1,127 @@
 lli - directly execute programs from LLVM bitcode
 =================================================
 
-
 SYNOPSIS
 --------
 
-
-**lli** [*options*] [*filename*] [*program args*]
-
+:program:`lli` [*options*] [*filename*] [*program args*]
 
 DESCRIPTION
 -----------
 
+:program:`lli` directly executes programs in LLVM bitcode format.  It takes a program
+in LLVM bitcode format and executes it using a just-in-time compiler or an
+interpreter.
 
-**lli** directly executes programs in LLVM bitcode format.  It takes a program
-in LLVM bitcode format and executes it using a just-in-time compiler, if one is
-available for the current architecture, or an interpreter.  **lli** takes all of
-the same code generator options as llc|llc, but they are only effective when
-**lli** is using the just-in-time compiler.
+:program:`lli` is *not* an emulator. It will not execute IR of different architectures
+and it can only interpret (or JIT-compile) for the host architecture.
 
-If *filename* is not specified, then **lli** reads the LLVM bitcode for the
+The JIT compiler takes the same arguments as other tools, like :program:`llc`,
+but they don't necessarily work for the interpreter.
+
+If `filename` is not specified, then :program:`lli` reads the LLVM bitcode for the
 program from standard input.
 
 The optional *args* specified on the command line are passed to the program as
 arguments.
 
-
 GENERAL OPTIONS
 ---------------
 
-
-
-**-fake-argv0**\ =\ *executable*
+.. option:: -fake-argv0=executable
 
  Override the ``argv[0]`` value passed into the executing program.
 
-
-
-**-force-interpreter**\ =\ *{false,true}*
+.. option:: -force-interpreter={false,true}
 
  If set to true, use the interpreter even if a just-in-time compiler is available
  for this architecture. Defaults to false.
 
-
-
-**-help**
+.. option:: -help
 
  Print a summary of command line options.
 
+.. option:: -load=pluginfilename
 
-
-**-load**\ =\ *pluginfilename*
-
- Causes **lli** to load the plugin (shared object) named *pluginfilename* and use
+ Causes :program:`lli` to load the plugin (shared object) named *pluginfilename* and use
  it for optimization.
 
-
-
-**-stats**
+.. option:: -stats
 
  Print statistics from the code-generation passes. This is only meaningful for
  the just-in-time compiler, at present.
 
-
-
-**-time-passes**
+.. option:: -time-passes
 
  Record the amount of time needed for each code-generation pass and print it to
  standard error.
 
+.. option:: -version
 
-
-**-version**
-
- Print out the version of **lli** and exit without doing anything else.
-
-
-
+ Print out the version of :program:`lli` and exit without doing anything else.
 
 TARGET OPTIONS
 --------------
 
-
-
-**-mtriple**\ =\ *target triple*
+.. option:: -mtriple=target triple
 
  Override the target triple specified in the input bitcode file with the
  specified string.  This may result in a crash if you pick an
  architecture which is not compatible with the current system.
 
-
-
-**-march**\ =\ *arch*
+.. option:: -march=arch
 
  Specify the architecture for which to generate assembly, overriding the target
  encoded in the bitcode file.  See the output of **llc -help** for a list of
  valid architectures.  By default this is inferred from the target triple or
  autodetected to the current architecture.
 
-
-
-**-mcpu**\ =\ *cpuname*
+.. option:: -mcpu=cpuname
 
  Specify a specific chip in the current architecture to generate code for.
  By default this is inferred from the target triple and autodetected to
  the current architecture.  For a list of available CPUs, use:
  **llvm-as < /dev/null | llc -march=xyz -mcpu=help**
 
-
-
-**-mattr**\ =\ *a1,+a2,-a3,...*
+.. option:: -mattr=a1,+a2,-a3,...
 
  Override or control specific attributes of the target, such as whether SIMD
  operations are enabled or not.  The default set of attributes is set by the
  current CPU.  For a list of available attributes, use:
  **llvm-as < /dev/null | llc -march=xyz -mattr=help**
 
-
-
-
 FLOATING POINT OPTIONS
 ----------------------
 
-
-
-**-disable-excess-fp-precision**
+.. option:: -disable-excess-fp-precision
 
  Disable optimizations that may increase floating point precision.
 
-
-
-**-enable-no-infs-fp-math**
+.. option:: -enable-no-infs-fp-math
 
  Enable optimizations that assume no Inf values.
 
-
-
-**-enable-no-nans-fp-math**
+.. option:: -enable-no-nans-fp-math
 
  Enable optimizations that assume no NAN values.
 
+.. option:: -enable-unsafe-fp-math
 
-
-**-enable-unsafe-fp-math**
-
- Causes **lli** to enable optimizations that may decrease floating point
+ Causes :program:`lli` to enable optimizations that may decrease floating point
  precision.
 
+.. option:: -soft-float
 
-
-**-soft-float**
-
- Causes **lli** to generate software floating point library calls instead of
+ Causes :program:`lli` to generate software floating point library calls instead of
  equivalent hardware instructions.
 
-
-
-
 CODE GENERATION OPTIONS
 -----------------------
 
-
-
-**-code-model**\ =\ *model*
+.. option:: -code-model=model
 
  Choose the code model from:
 
-
  .. code-block:: perl
 
       default: Target default code model
@@ -175,42 +130,30 @@ CODE GENERATION OPTIONS
       medium: Medium code model
       large: Large code model
 
-
-
-
-**-disable-post-RA-scheduler**
+.. option:: -disable-post-RA-scheduler
 
  Disable scheduling after register allocation.
 
-
-
-**-disable-spill-fusing**
+.. option:: -disable-spill-fusing
 
  Disable fusing of spill code into instructions.
 
-
-
-**-jit-enable-eh**
+.. option:: -jit-enable-eh
 
  Exception handling should be enabled in the just-in-time compiler.
 
-
-
-**-join-liveintervals**
+.. option:: -join-liveintervals
 
  Coalesce copies (default=true).
 
+.. option:: -nozero-initialized-in-bss
 
+  Don't place zero-initialized symbols into the BSS section.
 
-**-nozero-initialized-in-bss** Don't place zero-initialized symbols into the BSS section.
-
-
-
-**-pre-RA-sched**\ =\ *scheduler*
+.. option:: -pre-RA-sched=scheduler
 
  Instruction schedulers available (before register allocation):
 
-
  .. code-block:: perl
 
       =default: Best scheduler for the target
@@ -221,74 +164,51 @@ CODE GENERATION OPTIONS
       =list-tdrr: Top-down register reduction list scheduling
       =list-td: Top-down list scheduler -print-machineinstrs - Print generated machine code
 
-
-
-
-**-regalloc**\ =\ *allocator*
+.. option:: -regalloc=allocator
 
  Register allocator to use (default=linearscan)
 
-
  .. code-block:: perl
 
       =bigblock: Big-block register allocator
       =linearscan: linear scan register allocator =local -   local register allocator
       =simple: simple register allocator
 
-
-
-
-**-relocation-model**\ =\ *model*
+.. option:: -relocation-model=model
 
  Choose relocation model from:
 
-
  .. code-block:: perl
 
       =default: Target default relocation model
       =static: Non-relocatable code =pic -   Fully relocatable, position independent code
       =dynamic-no-pic: Relocatable external references, non-relocatable code
 
-
-
-
-**-spiller**
+.. option:: -spiller
 
  Spiller to use (default=local)
 
-
  .. code-block:: perl
 
       =simple: simple spiller
       =local: local spiller
 
-
-
-
-**-x86-asm-syntax**\ =\ *syntax*
+.. option:: -x86-asm-syntax=syntax
 
  Choose style of code to emit from X86 backend:
 
-
  .. code-block:: perl
 
       =att: Emit AT&T-style assembly
       =intel: Emit Intel-style assembly
 
-
-
-
-
 EXIT STATUS
 -----------
 
-
-If **lli** fails to load the program, it will exit with an exit code of 1.
+If :program:`lli` fails to load the program, it will exit with an exit code of 1.
 Otherwise, it will return the exit code of the program it executes.
 
-
 SEE ALSO
 --------
 
-
-llc|llc
+:program:`llc`
diff --git a/docs/CommandGuide/llvm-cov.rst b/docs/CommandGuide/llvm-cov.rst
index dff337572e4..d0e78a9a1d1 100644
--- a/docs/CommandGuide/llvm-cov.rst
+++ b/docs/CommandGuide/llvm-cov.rst
@@ -175,7 +175,7 @@ The :program:`llvm-cov show` command shows line by line coverage of a binary
 *BIN* using the profile data *PROFILE*. It can optionally be filtered to only
 show the coverage for the files listed in *SOURCES*.
 
-To use :program:`llvm-cov show`, you need a program that's compiled with
+To use :program:`llvm-cov show`, you need a program that is compiled with
 instrumentation to emit profile and coverage data. To build such a program with
 ``clang`` use the ``-fprofile-instr-generate`` and ``-fcoverage-mapping``
 flags. If linking with the ``clang`` driver, pass ``-fprofile-instr-generate``
@@ -223,10 +223,10 @@ OPTIONS
 
 .. option:: -arch=<name>
 
- If the covered binary is a universal binary, select the architecture to use
- when looking up the coverage map. Errors out if the supplied architecture is
- not found in the universal binary, or if used on a non-universal binary of
- a different architecture.
+ If the covered binary is a universal binary, select the architecture to use.
+ It is an error to specify an architecture that is not included in the
+ universal binary or to use an architecture that does not match a
+ non-universal binary.
 
 .. option:: -name=<NAME>
 
@@ -291,7 +291,7 @@ OPTIONS
 
 .. option:: -arch=<name>
 
- If the covered binary is a universal binary, select the architecture to use
- when looking up the coverage map. Errors out if the supplied architecture is
- not found in the universal binary, or if used on a non-universal binary of
- a different architecture.
+ If the covered binary is a universal binary, select the architecture to use.
+ It is an error to specify an architecture that is not included in the
+ universal binary or to use an architecture that does not match a
+ non-universal binary.
diff --git a/docs/CommandGuide/llvm-dwarfdump.rst b/docs/CommandGuide/llvm-dwarfdump.rst
index afaa0be3819..30c18adb771 100644
--- a/docs/CommandGuide/llvm-dwarfdump.rst
+++ b/docs/CommandGuide/llvm-dwarfdump.rst
@@ -26,5 +26,5 @@ OPTIONS
 EXIT STATUS
 -----------
 
-:program:`llvm-dwarfdump` returns 0. Other exit codes imply internal
-program error.
+:program:`llvm-dwarfdump` returns 0 if the input files were parsed and dumped
+successfully. Otherwise, it returns 1.
diff --git a/docs/CommandGuide/llvm-lib.rst b/docs/CommandGuide/llvm-lib.rst
new file mode 100644
index 00000000000..ecd0a7db7e3
--- /dev/null
+++ b/docs/CommandGuide/llvm-lib.rst
@@ -0,0 +1,31 @@
+llvm-lib - LLVM lib.exe compatible library tool
+===============================================
+
+
+SYNOPSIS
+--------
+
+
+**llvm-lib** [/libpath:<path>] [/out:<output>] [/llvmlibthin]
+[/ignore] [/machine] [/nologo] [files...]
+
+
+DESCRIPTION
+-----------
+
+
+The **llvm-lib** command is intended to be a ``lib.exe`` compatible
+tool. See https://msdn.microsoft.com/en-us/library/7ykb2k5f for the
+general description.
+
+**llvm-lib** has the following extensions:
+
+* Bitcode files in symbol tables.
+  **llvm-lib** includes symbols from both bitcode files and regular
+  object files in the symbol table.
+
+* Creating thin archives.
+  The /llvmlibthin option causes **llvm-lib** to create thin archive
+  that contain only the symbol table and the header for the various
+  members. These files are much smaller, but are not compatible with
+  link.exe (lld can handle them).
diff --git a/docs/CommandGuide/llvm-profdata.rst b/docs/CommandGuide/llvm-profdata.rst
index 45f70731d69..7053b7fa710 100644
--- a/docs/CommandGuide/llvm-profdata.rst
+++ b/docs/CommandGuide/llvm-profdata.rst
@@ -49,6 +49,28 @@ OPTIONS
  Specify the output file name.  *Output* cannot be ``-`` as the resulting
  indexed profile data can't be written to standard output.
 
+.. option:: -instr (default)
+
+ Specify that the input profile is an instrumentation-based profile.
+
+.. option:: -sample
+
+ Specify that the input profile is a sample-based profile. When using
+ sample-based profiles, the format of the generated file can be generated
+ in one of three ways:
+
+ .. option:: -binary (default)
+
+ Emit the profile using a binary encoding.
+
+ .. option:: -text
+
+ Emit the profile in text mode.
+
+ .. option:: -gcc
+
+ Emit the profile using GCC's gcov format (Not yet supported).
+
 .. program:: llvm-profdata show
 
 .. _profdata-show:
@@ -95,6 +117,14 @@ OPTIONS
  Specify the output file name.  If *output* is ``-`` or it isn't specified,
  then the output is sent to standard output.
 
+.. option:: -instr (default)
+
+ Specify that the input profile is an instrumentation-based profile.
+
+.. option:: -sample
+
+ Specify that the input profile is a sample-based profile.
+
 EXIT STATUS
 -----------
 
diff --git a/docs/CommandLine.rst b/docs/CommandLine.rst
index 1d85215f2af..556c302501e 100644
--- a/docs/CommandLine.rst
+++ b/docs/CommandLine.rst
@@ -1737,6 +1737,7 @@ exported by the ``lib/VMCore/PassManager.cpp`` file.
 .. _dynamically loaded options:
 
 Dynamically adding command line options
+---------------------------------------
 
 .. todo::
 
diff --git a/docs/CompilerWriterInfo.rst b/docs/CompilerWriterInfo.rst
index 2dfdc9b142d..900ba24e230 100644
--- a/docs/CompilerWriterInfo.rst
+++ b/docs/CompilerWriterInfo.rst
@@ -68,8 +68,8 @@ Other documents, collections, notes
 * `PowerPC64 alignment of long doubles (from GCC) <http://gcc.gnu.org/ml/gcc-patches/2003-09/msg00997.html>`_
 * `Long branch stubs for powerpc64-linux (from binutils) <http://sources.redhat.com/ml/binutils/2002-04/msg00573.html>`_
 
-R600
-----
+AMDGPU
+------
 
 * `AMD R6xx shader ISA <http://developer.amd.com/wordpress/media/2012/10/R600_Instruction_Set_Architecture.pdf>`_
 * `AMD R7xx shader ISA <http://developer.amd.com/wordpress/media/2012/10/R700-Family_Instruction_Set_Architecture.pdf>`_
diff --git a/docs/CoverageMappingFormat.rst b/docs/CoverageMappingFormat.rst
index 8fcffb838a3..9ac476c88b3 100644
--- a/docs/CoverageMappingFormat.rst
+++ b/docs/CoverageMappingFormat.rst
@@ -405,7 +405,7 @@ LEB128 is an unsigned interger value that is encoded using DWARF's LEB128
 encoding, optimizing for the case where values are small
 (1 byte for values less than 128).
 
-.. _strings:
+.. _Strings:
 
 Strings
 ^^^^^^^
diff --git a/docs/DeveloperPolicy.rst b/docs/DeveloperPolicy.rst
index a07b5fab217..4ee43fcdb9e 100644
--- a/docs/DeveloperPolicy.rst
+++ b/docs/DeveloperPolicy.rst
@@ -324,6 +324,11 @@ Below are some guidelines about the format of the message itself:
 * Text formatting and spelling should follow the same rules as documentation
   and in-code comments, ex. capitalization, full stop, etc.
 
+* If the commit is a bug fix on top of another recently committed patch, or a
+  revert or reapply of a patch, include the svn revision number of the prior
+  related commit. This could be as simple as "Revert rNNNN because it caused
+  PR#".
+
 For minor violations of these recommendations, the community normally favors
 reminding the contributor of this policy over reverting. Minor corrections and
 omissions can be handled by sending a reply to the commits mailing list.
@@ -619,5 +624,5 @@ patent-related trouble with their changes (including from third parties).  If
 you or your employer own the rights to a patent and would like to contribute
 code to LLVM that relies on it, we require that the copyright owner sign an
 agreement that allows any other user of LLVM to freely use your patent.  Please
-contact the `oversight group <mailto:llvm-oversight@cs.uiuc.edu>`_ for more
+contact the `LLVM Foundation Board of Directors <mailto:board@llvm.org>`_ for more
 details.
diff --git a/docs/ExceptionHandling.rst b/docs/ExceptionHandling.rst
index 72ed78a3c99..55ffdb45efe 100644
--- a/docs/ExceptionHandling.rst
+++ b/docs/ExceptionHandling.rst
@@ -339,11 +339,11 @@ original context before code generation.
 
 Catch handlers are called with a pointer to the handler itself as the first
 argument and a pointer to the parent function's stack frame as the second
-argument.  The catch handler uses the `llvm.recoverframe
-<LangRef.html#llvm-frameallocate-and-llvm-framerecover-intrinsics>`_ to get a
+argument.  The catch handler uses the `llvm.localrecover
+<LangRef.html#llvm-localescape-and-llvm-localrecover-intrinsics>`_ to get a
 pointer to a frame allocation block that is created in the parent frame using
-the `llvm.allocateframe 
-<LangRef.html#llvm-frameallocate-and-llvm-framerecover-intrinsics>`_ intrinsic.
+the `llvm.localescape
+<LangRef.html#llvm-localescape-and-llvm-localrecover-intrinsics>`_ intrinsic.
 The ``WinEHPrepare`` pass will have created a structure definition for the
 contents of this block.  The first two members of the structure will always be
 (1) a 32-bit integer that the runtime uses to track the exception state of the
@@ -520,12 +520,12 @@ action.
 A code of ``i32 1`` indicates a catch action, which expects three additional
 arguments. Different EH schemes give different meanings to the three arguments,
 but the first argument indicates whether the catch should fire, the second is
-the frameescape index of the exception object, and the third is the code to run
+the localescape index of the exception object, and the third is the code to run
 to catch the exception.
 
 For Windows C++ exception handling, the first argument for a catch handler is a
 pointer to the RTTI type descriptor for the object to catch. The second
-argument is an index into the argument list of the ``llvm.frameescape`` call in
+argument is an index into the argument list of the ``llvm.localescape`` call in
 the main function. The exception object will be copied into the provided stack
 object. If the exception object is not required, this argument should be -1.
 The third argument is a pointer to a function implementing the catch.  This
diff --git a/docs/FAQ.rst b/docs/FAQ.rst
index 8db94c6250c..0559a1ff215 100644
--- a/docs/FAQ.rst
+++ b/docs/FAQ.rst
@@ -102,7 +102,7 @@ grabbing the wrong linker/assembler/etc, there are two ways to fix it:
 
 .. code-block:: console
 
-   % PATH=[the path without the bad program] ./configure ...
+   % PATH=[the path without the bad program] $LLVM_SRC_DIR/configure ...
 
 This is still somewhat inconvenient, but it allows ``configure`` to do its
 work without having to adjust your ``PATH`` permanently.
diff --git a/docs/FaultMaps.rst b/docs/FaultMaps.rst
new file mode 100644
index 00000000000..4ecdd86d769
--- /dev/null
+++ b/docs/FaultMaps.rst
@@ -0,0 +1,127 @@
+==============================
+FaultMaps and implicit checks
+==============================
+
+.. contents::
+   :local:
+   :depth: 2
+
+Motivation
+==========
+
+Code generated by managed language runtimes tend to have checks that
+are required for safety but never fail in practice.  In such cases, it
+is profitable to make the non-failing case cheaper even if it makes
+the failing case significantly more expensive.  This asymmetry can be
+exploited by folding such safety checks into operations that can be
+made to fault reliably if the check would have failed, and recovering
+from such a fault by using a signal handler.
+
+For example, Java requires null checks on objects before they are read
+from or written to.  If the object is ``null`` then a
+``NullPointerException`` has to be thrown, interrupting normal
+execution.  In practice, however, dereferencing a ``null`` pointer is
+extremely rare in well-behaved Java programs, and typically the null
+check can be folded into a nearby memory operation that operates on
+the same memory location.
+
+The Fault Map Section
+=====================
+
+Information about implicit checks generated by LLVM are put in a
+special "fault map" section.  On Darwin this section is named
+``__llvm_faultmaps``.
+
+The format of this section is
+
+.. code-block:: none
+
+  Header {
+    uint8  : Fault Map Version (current version is 1)
+    uint8  : Reserved (expected to be 0)
+    uint16 : Reserved (expected to be 0)
+  }
+  uint32 : NumFunctions
+  FunctionInfo[NumFunctions] {
+    uint64 : FunctionAddress
+    uint32 : NumFaultingPCs
+    uint32 : Reserved (expected to be 0)
+    FunctionFaultInfo[NumFaultingPCs] {
+      uint32  : FaultKind = FaultMaps::FaultingLoad (only legal value currently)
+      uint32  : FaultingPCOffset
+      uint32  : HandlerPCOffset
+    }
+  }
+
+
+The ``ImplicitNullChecks`` pass
+===============================
+
+The ``ImplicitNullChecks`` pass transforms explicit control flow for
+checking if a pointer is ``null``, like:
+
+.. code-block:: llvm
+
+    %ptr = call i32* @get_ptr()
+    %ptr_is_null = icmp i32* %ptr, null
+    br i1 %ptr_is_null, label %is_null, label %not_null, !make.implicit !0
+  
+  not_null:
+    %t = load i32, i32* %ptr
+    br label %do_something_with_t
+    
+  is_null:
+    call void @HFC()
+    unreachable
+  
+  !0 = !{}
+
+to control flow implicit in the instruction loading or storing through
+the pointer being null checked:
+
+.. code-block:: llvm
+
+    %ptr = call i32* @get_ptr()
+    %t = load i32, i32* %ptr  ;; handler-pc = label %is_null
+    br label %do_something_with_t
+    
+  is_null:
+    call void @HFC()
+    unreachable
+
+This transform happens at the ``MachineInstr`` level, not the LLVM IR
+level (so the above example is only representative, not literal).  The
+``ImplicitNullChecks`` pass runs during codegen, if
+``-enable-implicit-null-checks`` is passed to ``llc``.
+
+The ``ImplicitNullChecks`` pass adds entries to the
+``__llvm_faultmaps`` section described above as needed.
+
+``make.implicit`` metadata
+--------------------------
+
+Making null checks implicit is an aggressive optimization, and it can
+be a net performance pessimization if too many memory operations end
+up faulting because of it.  A language runtime typically needs to
+ensure that only a negligible number of implicit null checks actually
+fault once the application has reached a steady state.  A standard way
+of doing this is by healing failed implicit null checks into explicit
+null checks via code patching or recompilation.  It follows that there
+are two requirements an explicit null check needs to satisfy for it to
+be profitable to convert it to an implicit null check:
+
+  1. The case where the pointer is actually null (i.e. the "failing"
+     case) is extremely rare.
+
+  2. The failing path heals the implicit null check into an explicit
+     null check so that the application does not repeatedly page
+     fault.
+
+The frontend is expected to mark branches that satisfy (1) and (2)
+using a ``!make.implicit`` metadata node (the actual content of the
+metadata node is ignored).  Only branches that are marked with
+``!make.implicit`` metadata are considered as candidates for
+conversion into implicit null checks.
+
+(Note that while we could deal with (1) using profiling data, dealing
+with (2) requires some information not present in branch profiles.)
diff --git a/docs/Frontend/PerformanceTips.rst b/docs/Frontend/PerformanceTips.rst
index d8c04651f0a..22b3fe45984 100644
--- a/docs/Frontend/PerformanceTips.rst
+++ b/docs/Frontend/PerformanceTips.rst
@@ -55,8 +55,7 @@ Other things to consider
 
 #. Add nsw/nuw flags as appropriate.  Reasoning about overflow is 
    generally hard for an optimizer so providing these facts from the frontend 
-   can be very impactful.  For languages which need overflow semantics, 
-   consider using the :ref:`overflow intrinsics <int_overflow>`.
+   can be very impactful.  
 
 #. Use fast-math flags on floating point operations if legal.  If you don't 
    need strict IEEE floating point semantics, there are a number of additional 
@@ -142,6 +141,29 @@ Other things to consider
    perform badly with confronted with such structures.  The only exception to 
    this guidance is that a unified return block with high in-degree is fine.
 
+#. When checking a value against a constant, emit the check using a consistent
+   comparison type.  The GVN pass *will* optimize redundant equalities even if
+   the type of comparison is inverted, but GVN only runs late in the pipeline.
+   As a result, you may miss the opportunity to run other important 
+   optimizations.  Improvements to EarlyCSE to remove this issue are tracked in 
+   Bug 23333.
+
+#. Avoid using arithmetic intrinsics unless you are *required* by your source 
+   language specification to emit a particular code sequence.  The optimizer 
+   is quite good at reasoning about general control flow and arithmetic, it is
+   not anywhere near as strong at reasoning about the various intrinsics.  If 
+   profitable for code generation purposes, the optimizer will likely form the 
+   intrinsics itself late in the optimization pipeline.  It is *very* rarely 
+   profitable to emit these directly in the language frontend.  This item
+   explicitly includes the use of the :ref:`overflow intrinsics <int_overflow>`.
+
+#. Avoid using the :ref:`assume intrinsic <int_assume>` until you've 
+   established that a) there's no other way to express the given fact and b) 
+   that fact is critical for optimization purposes.  Assumes are a great 
+   prototyping mechanism, but they can have negative effects on both compile 
+   time and optimization effectiveness.  The former is fixable with enough 
+   effort, but the later is fairly fundamental to their designed purpose.
+
 p.s. If you want to help improve this document, patches expanding any of the 
 above items into standalone sections of their own with a more complete 
 discussion would be very welcome.  
diff --git a/docs/GarbageCollection.rst b/docs/GarbageCollection.rst
index a1557fcdb09..56b4b9f8f95 100644
--- a/docs/GarbageCollection.rst
+++ b/docs/GarbageCollection.rst
@@ -494,6 +494,7 @@ a ``gcroot`` strategy.
 As there names imply, the binary format produced is intended to model that 
 used by the Erlang and OCaml compilers respectively.  
 
+.. _statepoint_example_gc:
 
 The Statepoint Example GC
 -------------------------
@@ -517,6 +518,28 @@ The stack map format generated by this GC strategy can be found in the
 <statepoint-stackmap-format>`. This format is intended to be the standard 
 format supported by LLVM going forward.
 
+The CoreCLR GC
+-------------------------
+
+.. code-block:: c++
+
+  F.setGC("coreclr");
+
+This GC leverages the ``gc.statepoint`` mechanism to support the 
+`CoreCLR <https://github.com/dotnet/coreclr>`__ runtime.
+
+Support for this GC strategy is a work in progress. This strategy will 
+differ from 
+:ref:`statepoint-example GC<statepoint_example_gc>` strategy in 
+certain aspects like:
+
+* Base-pointers of interior pointers are not explicitly 
+  tracked and reported.
+
+* A different format is used for encoding stack maps.
+
+* Safe-point polls are only needed before loop-back edges
+  and before tail-calls (not needed at function-entry).
 
 Custom GC Strategies
 ====================
diff --git a/docs/GettingStarted.rst b/docs/GettingStarted.rst
index 18b3c1d87cc..75f0e60c41f 100644
--- a/docs/GettingStarted.rst
+++ b/docs/GettingStarted.rst
@@ -326,7 +326,11 @@ Easy steps for installing GCC 4.8.2:
 
 .. code-block:: console
 
-  % wget ftp://ftp.gnu.org/gnu/gcc/gcc-4.8.2/gcc-4.8.2.tar.bz2
+  % wget https://ftp.gnu.org/gnu/gcc/gcc-4.8.2/gcc-4.8.2.tar.bz2
+  % wget https://ftp.gnu.org/gnu/gcc/gcc-4.8.2/gcc-4.8.2.tar.bz2.sig
+  % wget https://ftp.gnu.org/gnu/gnu-keyring.gpg
+  % signature_invalid=`gpg --verify --no-default-keyring --keyring ./gnu-keyring.gpg gcc-4.8.2.tar.bz2.sig`
+  % if [ $signature_invalid ]; then echo "Invalid signature" ; exit 1 ; fi
   % tar -xvjf gcc-4.8.2.tar.bz2
   % cd gcc-4.8.2
   % ./contrib/download_prerequisites
@@ -711,7 +715,7 @@ used by people developing LLVM.
 |                         | as ``LLVM_ALL_TARGETS``, and can be set to include |
 |                         | out-of-tree targets. The default value includes:   |
 |                         | ``AArch64, ARM, CppBackend, Hexagon,               |
-|                         | Mips, MSP430, NVPTX, PowerPC, R600, Sparc,         |
+|                         | Mips, MSP430, NVPTX, PowerPC, AMDGPU, Sparc,       |
 |                         | SystemZ, X86, XCore``.                             |
 +-------------------------+----------------------------------------------------+
 | LLVM_ENABLE_DOXYGEN     | Build doxygen-based documentation from the source  |
diff --git a/docs/HowToBuildOnARM.rst b/docs/HowToBuildOnARM.rst
index f2edaef76cd..6579d36a72a 100644
--- a/docs/HowToBuildOnARM.rst
+++ b/docs/HowToBuildOnARM.rst
@@ -40,7 +40,7 @@ on the ARMv6 and ARMv7 architectures and may be inapplicable to older chips.
 
    .. code-block:: bash
 
-     ./configure --build=armv7l-unknown-linux-gnueabihf \
+     ../$LLVM_SRC_DIR/configure --build=armv7l-unknown-linux-gnueabihf \
      --host=armv7l-unknown-linux-gnueabihf \
      --target=armv7l-unknown-linux-gnueabihf --with-cpu=cortex-a9 \
      --with-float=hard --with-abi=aapcs-vfp --with-fpu=neon \
diff --git a/docs/HowToReleaseLLVM.rst b/docs/HowToReleaseLLVM.rst
index 26e9f3b2ee8..33c547e97a8 100644
--- a/docs/HowToReleaseLLVM.rst
+++ b/docs/HowToReleaseLLVM.rst
@@ -136,51 +136,24 @@ Regenerate the configure scripts for both ``llvm`` and the ``test-suite``.
 In addition, the version numbers of all the Bugzilla components must be updated
 for the next release.
 
-Build the LLVM Release Candidates
-^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+Tagging the LLVM Release Candidates
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
-Create release candidates for ``llvm``, ``clang``, ``dragonegg``, and the LLVM
-``test-suite`` by tagging the branch with the respective release candidate
-number.  For instance, to create **Release Candidate 1** you would issue the
-following commands:
+Tag release candidates using the tag.sh script in utils/release.
 
 ::
 
-  $ svn mkdir https://llvm.org/svn/llvm-project/llvm/tags/RELEASE_XYZ
-  $ svn copy https://llvm.org/svn/llvm-project/llvm/branches/release_XY \
-             https://llvm.org/svn/llvm-project/llvm/tags/RELEASE_XYZ/rc1
-
-  $ svn mkdir https://llvm.org/svn/llvm-project/cfe/tags/RELEASE_XYZ
-  $ svn copy https://llvm.org/svn/llvm-project/cfe/branches/release_XY \
-             https://llvm.org/svn/llvm-project/cfe/tags/RELEASE_XYZ/rc1
-
-  $ svn mkdir https://llvm.org/svn/llvm-project/dragonegg/tags/RELEASE_XYZ
-  $ svn copy https://llvm.org/svn/llvm-project/dragonegg/branches/release_XY \
-             https://llvm.org/svn/llvm-project/dragonegg/tags/RELEASE_XYZ/rc1
-
-  $ svn mkdir https://llvm.org/svn/llvm-project/test-suite/tags/RELEASE_XYZ
-  $ svn copy https://llvm.org/svn/llvm-project/test-suite/branches/release_XY \
-             https://llvm.org/svn/llvm-project/test-suite/tags/RELEASE_XYZ/rc1
-
-Similarly, **Release Candidate 2** would be named ``RC2`` and so on.  This keeps
-a permanent copy of the release candidate around for people to export and build
-as they wish.  The final released sources will be tagged in the ``RELEASE_XYZ``
-directory as ``Final`` (c.f. :ref:`tag`).
+  $ ./tag.sh -release X.Y.Z -rc $RC
 
 The Release Manager may supply pre-packaged source tarballs for users.  This can
-be done with the following commands:
+be done with the export.sh script in utils/release.
 
 ::
 
-  $ svn export https://llvm.org/svn/llvm-project/llvm/tags/RELEASE_XYZ/rc1 llvm-X.Yrc1
-  $ svn export https://llvm.org/svn/llvm-project/cfe/tags/RELEASE_XYZ/rc1 clang-X.Yrc1
-  $ svn export https://llvm.org/svn/llvm-project/dragonegg/tags/RELEASE_XYZ/rc1 dragonegg-X.Yrc1
-  $ svn export https://llvm.org/svn/llvm-project/test-suite/tags/RELEASE_XYZ/rc1 llvm-test-X.Yrc1
+  $ ./export.sh -release X.Y.Z -rc $RC
 
-  $ tar -cvf - llvm-X.Yrc1        | gzip > llvm-X.Yrc1.src.tar.gz
-  $ tar -cvf - clang-X.Yrc1       | gzip > clang-X.Yrc1.src.tar.gz
-  $ tar -cvf - dragonegg-X.Yrc1   | gzip > dragonegg-X.Yrc1.src.tar.gz
-  $ tar -cvf - llvm-test-X.Yrc1   | gzip > llvm-test-X.Yrc1.src.tar.gz
+This will generate source tarballs for each LLVM project being validated, which
+can be uploaded to the website for further testing.
 
 Building the Release
 --------------------
@@ -384,21 +357,11 @@ mainline into the release branch.
 Tag the LLVM Final Release
 ^^^^^^^^^^^^^^^^^^^^^^^^^^
 
-Tag the final release sources using the following procedure:
+Tag the final release sources using the tag.sh script in utils/release.
 
 ::
 
-  $ svn copy https://llvm.org/svn/llvm-project/llvm/branches/release_XY \
-             https://llvm.org/svn/llvm-project/llvm/tags/RELEASE_XYZ/Final
-
-  $ svn copy https://llvm.org/svn/llvm-project/cfe/branches/release_XY \
-             https://llvm.org/svn/llvm-project/cfe/tags/RELEASE_XYZ/Final
-
-  $ svn copy https://llvm.org/svn/llvm-project/dragonegg/branches/release_XY \
-             https://llvm.org/svn/llvm-project/dragonegg/tags/RELEASE_XYZ/Final
-
-  $ svn copy https://llvm.org/svn/llvm-project/test-suite/branches/release_XY \
-             https://llvm.org/svn/llvm-project/test-suite/tags/RELEASE_XYZ/Final
+  $ ./tag.sh -release X.Y.Z -final
 
 Update the LLVM Demo Page
 -------------------------
diff --git a/docs/LangRef.rst b/docs/LangRef.rst
index a5a886952b7..2030c475b09 100644
--- a/docs/LangRef.rst
+++ b/docs/LangRef.rst
@@ -494,6 +494,9 @@ model is not supported, or if a better choice of model can be made.
 A model can also be specified in a alias, but then it only governs how
 the alias is accessed. It will not have any effect in the aliasee.
 
+For platforms without linker support of ELF TLS model, the -femulated-tls
+flag can be used to generate GCC compatible emulated TLS code.
+
 .. _namedtypes:
 
 Structure Types
@@ -635,8 +638,9 @@ attributes <paramattrs>`), optional :ref:`function attributes <fnattrs>`,
 an optional section, an optional alignment,
 an optional :ref:`comdat <langref_comdats>`,
 an optional :ref:`garbage collector name <gc>`, an optional :ref:`prefix <prefixdata>`,
-an optional :ref:`prologue <prologuedata>`, an opening
-curly brace, a list of basic blocks, and a closing curly brace.
+an optional :ref:`prologue <prologuedata>`,
+an optional :ref:`personality <personalityfn>`,
+an opening curly brace, a list of basic blocks, and a closing curly brace.
 
 LLVM function declarations consist of the "``declare``" keyword, an
 optional :ref:`linkage type <linkage>`, an optional :ref:`visibility
@@ -683,7 +687,8 @@ Syntax::
            [cconv] [ret attrs]
            <ResultType> @<FunctionName> ([argument list])
            [unnamed_addr] [fn Attrs] [section "name"] [comdat [($name)]]
-           [align N] [gc] [prefix Constant] [prologue Constant] { ... }
+           [align N] [gc] [prefix Constant] [prologue Constant]
+           [personality Constant] { ... }
 
 The argument list is a comma seperated sequence of arguments where each
 argument is of the following form
@@ -821,9 +826,11 @@ with the same name.  This is necessary because both globals belong to different
 COMDAT groups and COMDATs, at the object file level, are represented by
 sections.
 
-Note that certain IR constructs like global variables and functions may create
-COMDATs in the object file in addition to any which are specified using COMDAT
-IR.  This arises, for example, when a global variable has linkonce_odr linkage.
+Note that certain IR constructs like global variables and functions may
+create COMDATs in the object file in addition to any which are specified using
+COMDAT IR.  This arises when the code generator is configured to emit globals
+in individual sections (e.g. when `-data-sections` or `-function-sections`
+is supplied to `llc`).
 
 .. _namedmetadatastructure:
 
@@ -834,6 +841,11 @@ Named metadata is a collection of metadata. :ref:`Metadata
 nodes <metadata>` (but not metadata strings) are the only valid
 operands for a named metadata.
 
+#. Named metadata are represented as a string of characters with the
+   metadata prefix. The rules for metadata names are the same as for
+   identifiers, but quoted names are not allowed. ``"\xx"`` type escapes
+   are still valid, which allows any character to be part of a name.
+
 Syntax::
 
     ; Some unnamed metadata nodes, which are referenced by the named metadata.
@@ -1125,6 +1137,14 @@ A function may have prologue data but no body.  This has similar semantics
 to the ``available_externally`` linkage in that the data may be used by the
 optimizers but will not be emitted in the object file.
 
+.. _personalityfn:
+
+Personality Function
+--------------------
+
+The ``personality`` attribute permits functions to specify what function
+to use for exception handling.
+
 .. _attrgrp:
 
 Attribute Groups
@@ -1196,6 +1216,13 @@ example:
     computing edge weights, basic blocks post-dominated by a cold
     function call are also considered to be cold; and, thus, given low
     weight.
+``convergent``
+    This attribute indicates that the callee is dependent on a convergent
+    thread execution pattern under certain parallel execution models.
+    Transformations that are execution model agnostic may only move or
+    tranform this call if the final location is control equivalent to its
+    original position in the program, where control equivalence is defined as
+    A dominates B and B post-dominates A, or vice versa.
 ``inlinehint``
     This attribute indicates that the source code contained a hint that
     inlining this function is desirable (such as the "inline" keyword in
@@ -1302,11 +1329,28 @@ example:
     On an argument, this attribute indicates that the function does not write
     through this pointer argument, even though it may write to the memory that
     the pointer points to.
+``argmemonly``
+    This attribute indicates that the only memory accesses inside function are
+    loads and stores from objects pointed to by its pointer-typed arguments,
+    with arbitrary offsets. Or in other words, all memory operations in the
+    function can refer to memory only using pointers based on its function
+    arguments.
+    Note that ``argmemonly`` can be used together with ``readonly`` attribute
+    in order to specify that function reads only from its arguments.
 ``returns_twice``
     This attribute indicates that this function can return twice. The C
     ``setjmp`` is an example of such a function. The compiler disables
     some optimizations (like tail calls) in the caller of these
     functions.
+``safestack``
+    This attribute indicates that
+    `SafeStack <http://clang.llvm.org/docs/SafeStack.html>`_
+    protection is enabled for this function.
+
+    If a function that has a ``safestack`` attribute is inlined into a
+    function that doesn't have a ``safestack`` attribute or which has an
+    ``ssp``, ``sspstrong`` or ``sspreq`` attribute, then the resulting
+    function will have a ``safestack`` attribute.
 ``sanitize_address``
     This attribute indicates that AddressSanitizer checks
     (dynamic address safety analysis) are enabled for this function.
@@ -1413,8 +1457,8 @@ The strings can contain any character by escaping non-printable
 characters. The escape sequence used is simply "\\xx" where "xx" is the
 two digit hex code for the number.
 
-The inline asm code is simply printed to the machine code .s file when
-assembly code is generated.
+Note that the assembly string *must* be parseable by LLVM's integrated assembler
+(unless it is disabled), even when emitting a ``.s`` file.
 
 .. _langref_datalayout:
 
@@ -1804,8 +1848,8 @@ Fast-Math Flags
 
 LLVM IR floating-point binary ops (:ref:`fadd <i_fadd>`,
 :ref:`fsub <i_fsub>`, :ref:`fmul <i_fmul>`, :ref:`fdiv <i_fdiv>`,
-:ref:`frem <i_frem>`) have the following flags that can be set to enable
-otherwise unsafe floating point operations
+:ref:`frem <i_frem>`, :ref:`fcmp <i_fcmp>`) have the following flags that can
+be set to enable otherwise unsafe floating point operations
 
 ``nnan``
    No NaNs - Allow optimizations to assume the arguments and result are not
@@ -2767,13 +2811,36 @@ Inline Assembler Expressions
 ----------------------------
 
 LLVM supports inline assembler expressions (as opposed to :ref:`Module-Level
-Inline Assembly <moduleasm>`) through the use of a special value. This
-value represents the inline assembler as a string (containing the
-instructions to emit), a list of operand constraints (stored as a
-string), a flag that indicates whether or not the inline asm expression
-has side effects, and a flag indicating whether the function containing
-the asm needs to align its stack conservatively. An example inline
-assembler expression is:
+Inline Assembly <moduleasm>`) through the use of a special value. This value
+represents the inline assembler as a template string (containing the
+instructions to emit), a list of operand constraints (stored as a string), a
+flag that indicates whether or not the inline asm expression has side effects,
+and a flag indicating whether the function containing the asm needs to align its
+stack conservatively.
+
+The template string supports argument substitution of the operands using "``$``"
+followed by a number, to indicate substitution of the given register/memory
+location, as specified by the constraint string. "``${NUM:MODIFIER}``" may also
+be used, where ``MODIFIER`` is a target-specific annotation for how to print the
+operand (See :ref:`inline-asm-modifiers`).
+
+A literal "``$``" may be included by using "``$$``" in the template. To include
+other special characters into the output, the usual "``\XX``" escapes may be
+used, just as in other strings. Note that after template substitution, the
+resulting assembly string is parsed by LLVM's integrated assembler unless it is
+disabled -- even when emitting a ``.s`` file -- and thus must contain assembly
+syntax known to LLVM.
+
+LLVM's support for inline asm is modeled closely on the requirements of Clang's
+GCC-compatible inline-asm support. Thus, the feature-set and the constraint and
+modifier codes listed here are similar or identical to those in GCC's inline asm
+support. However, to be clear, the syntax of the template and constraint strings
+described here is *not* the same as the syntax accepted by GCC and Clang, and,
+while most constraint letters are passed through as-is by Clang, some get
+translated to other codes when converting from the C source to the LLVM
+assembly.
+
+An example inline assembler expression is:
 
 .. code-block:: llvm
 
@@ -2819,6 +2886,596 @@ If multiple keywords appear the '``sideeffect``' keyword must come
 first, the '``alignstack``' keyword second and the '``inteldialect``'
 keyword last.
 
+Inline Asm Constraint String
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+The constraint list is a comma-separated string, each element containing one or
+more constraint codes.
+
+For each element in the constraint list an appropriate register or memory
+operand will be chosen, and it will be made available to assembly template
+string expansion as ``$0`` for the first constraint in the list, ``$1`` for the
+second, etc.
+
+There are three different types of constraints, which are distinguished by a
+prefix symbol in front of the constraint code: Output, Input, and Clobber. The
+constraints must always be given in that order: outputs first, then inputs, then
+clobbers. They cannot be intermingled.
+
+There are also three different categories of constraint codes:
+
+- Register constraint. This is either a register class, or a fixed physical
+  register. This kind of constraint will allocate a register, and if necessary,
+  bitcast the argument or result to the appropriate type.
+- Memory constraint. This kind of constraint is for use with an instruction
+  taking a memory operand. Different constraints allow for different addressing
+  modes used by the target.
+- Immediate value constraint. This kind of constraint is for an integer or other
+  immediate value which can be rendered directly into an instruction. The
+  various target-specific constraints allow the selection of a value in the
+  proper range for the instruction you wish to use it with.
+
+Output constraints
+""""""""""""""""""
+
+Output constraints are specified by an "``=``" prefix (e.g. "``=r``"). This
+indicates that the assembly will write to this operand, and the operand will
+then be made available as a return value of the ``asm`` expression. Output
+constraints do not consume an argument from the call instruction. (Except, see
+below about indirect outputs).
+
+Normally, it is expected that no output locations are written to by the assembly
+expression until *all* of the inputs have been read. As such, LLVM may assign
+the same register to an output and an input. If this is not safe (e.g. if the
+assembly contains two instructions, where the first writes to one output, and
+the second reads an input and writes to a second output), then the "``&``"
+modifier must be used (e.g. "``=&r``") to specify that the output is an
+"early-clobber" output. Marking an ouput as "early-clobber" ensures that LLVM
+will not use the same register for any inputs (other than an input tied to this
+output).
+
+Input constraints
+"""""""""""""""""
+
+Input constraints do not have a prefix -- just the constraint codes. Each input
+constraint will consume one argument from the call instruction. It is not
+permitted for the asm to write to any input register or memory location (unless
+that input is tied to an output). Note also that multiple inputs may all be
+assigned to the same register, if LLVM can determine that they necessarily all
+contain the same value.
+
+Instead of providing a Constraint Code, input constraints may also "tie"
+themselves to an output constraint, by providing an integer as the constraint
+string. Tied inputs still consume an argument from the call instruction, and
+take up a position in the asm template numbering as is usual -- they will simply
+be constrained to always use the same register as the output they've been tied
+to. For example, a constraint string of "``=r,0``" says to assign a register for
+output, and use that register as an input as well (it being the 0'th
+constraint).
+
+It is permitted to tie an input to an "early-clobber" output. In that case, no
+*other* input may share the same register as the input tied to the early-clobber
+(even when the other input has the same value).
+
+You may only tie an input to an output which has a register constraint, not a
+memory constraint. Only a single input may be tied to an output.
+
+There is also an "interesting" feature which deserves a bit of explanation: if a
+register class constraint allocates a register which is too small for the value
+type operand provided as input, the input value will be split into multiple
+registers, and all of them passed to the inline asm.
+
+However, this feature is often not as useful as you might think.
+
+Firstly, the registers are *not* guaranteed to be consecutive. So, on those
+architectures that have instructions which operate on multiple consecutive
+instructions, this is not an appropriate way to support them. (e.g. the 32-bit
+SparcV8 has a 64-bit load, which instruction takes a single 32-bit register. The
+hardware then loads into both the named register, and the next register. This
+feature of inline asm would not be useful to support that.)
+
+A few of the targets provide a template string modifier allowing explicit access
+to the second register of a two-register operand (e.g. MIPS ``L``, ``M``, and
+``D``). On such an architecture, you can actually access the second allocated
+register (yet, still, not any subsequent ones). But, in that case, you're still
+probably better off simply splitting the value into two separate operands, for
+clarity. (e.g. see the description of the ``A`` constraint on X86, which,
+despite existing only for use with this feature, is not really a good idea to
+use)
+
+Indirect inputs and outputs
+"""""""""""""""""""""""""""
+
+Indirect output or input constraints can be specified by the "``*``" modifier
+(which goes after the "``=``" in case of an output). This indicates that the asm
+will write to or read from the contents of an *address* provided as an input
+argument. (Note that in this way, indirect outputs act more like an *input* than
+an output: just like an input, they consume an argument of the call expression,
+rather than producing a return value. An indirect output constraint is an
+"output" only in that the asm is expected to write to the contents of the input
+memory location, instead of just read from it).
+
+This is most typically used for memory constraint, e.g. "``=*m``", to pass the
+address of a variable as a value.
+
+It is also possible to use an indirect *register* constraint, but only on output
+(e.g. "``=*r``"). This will cause LLVM to allocate a register for an output
+value normally, and then, separately emit a store to the address provided as
+input, after the provided inline asm. (It's not clear what value this
+functionality provides, compared to writing the store explicitly after the asm
+statement, and it can only produce worse code, since it bypasses many
+optimization passes. I would recommend not using it.)
+
+
+Clobber constraints
+"""""""""""""""""""
+
+A clobber constraint is indicated by a "``~``" prefix. A clobber does not
+consume an input operand, nor generate an output. Clobbers cannot use any of the
+general constraint code letters -- they may use only explicit register
+constraints, e.g. "``~{eax}``". The one exception is that a clobber string of
+"``~{memory}``" indicates that the assembly writes to arbitrary undeclared
+memory locations -- not only the memory pointed to by a declared indirect
+output.
+
+
+Constraint Codes
+""""""""""""""""
+After a potential prefix comes constraint code, or codes.
+
+A Constraint Code is either a single letter (e.g. "``r``"), a "``^``" character
+followed by two letters (e.g. "``^wc``"), or "``{``" register-name "``}``"
+(e.g. "``{eax}``").
+
+The one and two letter constraint codes are typically chosen to be the same as
+GCC's constraint codes.
+
+A single constraint may include one or more than constraint code in it, leaving
+it up to LLVM to choose which one to use. This is included mainly for
+compatibility with the translation of GCC inline asm coming from clang.
+
+There are two ways to specify alternatives, and either or both may be used in an
+inline asm constraint list:
+
+1) Append the codes to each other, making a constraint code set. E.g. "``im``"
+   or "``{eax}m``". This means "choose any of the options in the set". The
+   choice of constraint is made independently for each constraint in the
+   constraint list.
+
+2) Use "``|``" between constraint code sets, creating alternatives. Every
+   constraint in the constraint list must have the same number of alternative
+   sets. With this syntax, the same alternative in *all* of the items in the
+   constraint list will be chosen together.
+
+Putting those together, you might have a two operand constraint string like
+``"rm|r,ri|rm"``. This indicates that if operand 0 is ``r`` or ``m``, then
+operand 1 may be one of ``r`` or ``i``. If operand 0 is ``r``, then operand 1
+may be one of ``r`` or ``m``. But, operand 0 and 1 cannot both be of type m.
+
+However, the use of either of the alternatives features is *NOT* recommended, as
+LLVM is not able to make an intelligent choice about which one to use. (At the
+point it currently needs to choose, not enough information is available to do so
+in a smart way.) Thus, it simply tries to make a choice that's most likely to
+compile, not one that will be optimal performance. (e.g., given "``rm``", it'll
+always choose to use memory, not registers). And, if given multiple registers,
+or multiple register classes, it will simply choose the first one. (In fact, it
+doesn't currently even ensure explicitly specified physical registers are
+unique, so specifying multiple physical registers as alternatives, like
+``{r11}{r12},{r11}{r12}``, will assign r11 to both operands, not at all what was
+intended.)
+
+Supported Constraint Code List
+""""""""""""""""""""""""""""""
+
+The constraint codes are, in general, expected to behave the same way they do in
+GCC. LLVM's support is often implemented on an 'as-needed' basis, to support C
+inline asm code which was supported by GCC. A mismatch in behavior between LLVM
+and GCC likely indicates a bug in LLVM.
+
+Some constraint codes are typically supported by all targets:
+
+- ``r``: A register in the target's general purpose register class.
+- ``m``: A memory address operand. It is target-specific what addressing modes
+  are supported, typical examples are register, or register + register offset,
+  or register + immediate offset (of some target-specific size).
+- ``i``: An integer constant (of target-specific width). Allows either a simple
+  immediate, or a relocatable value.
+- ``n``: An integer constant -- *not* including relocatable values.
+- ``s``: An integer constant, but allowing *only* relocatable values.
+- ``X``: Allows an operand of any kind, no constraint whatsoever. Typically
+  useful to pass a label for an asm branch or call.
+
+  .. FIXME: but that surely isn't actually okay to jump out of an asm
+     block without telling llvm about the control transfer???)
+
+- ``{register-name}``: Requires exactly the named physical register.
+
+Other constraints are target-specific:
+
+AArch64:
+
+- ``z``: An immediate integer 0. Outputs ``WZR`` or ``XZR``, as appropriate.
+- ``I``: An immediate integer valid for an ``ADD`` or ``SUB`` instruction,
+  i.e. 0 to 4095 with optional shift by 12.
+- ``J``: An immediate integer that, when negated, is valid for an ``ADD`` or
+  ``SUB`` instruction, i.e. -1 to -4095 with optional left shift by 12.
+- ``K``: An immediate integer that is valid for the 'bitmask immediate 32' of a
+  logical instruction like ``AND``, ``EOR``, or ``ORR`` with a 32-bit register.
+- ``L``: An immediate integer that is valid for the 'bitmask immediate 64' of a
+  logical instruction like ``AND``, ``EOR``, or ``ORR`` with a 64-bit register.
+- ``M``: An immediate integer for use with the ``MOV`` assembly alias on a
+  32-bit register. This is a superset of ``K``: in addition to the bitmask
+  immediate, also allows immediate integers which can be loaded with a single
+  ``MOVZ`` or ``MOVL`` instruction.
+- ``N``: An immediate integer for use with the ``MOV`` assembly alias on a
+  64-bit register. This is a superset of ``L``.
+- ``Q``: Memory address operand must be in a single register (no
+  offsets). (However, LLVM currently does this for the ``m`` constraint as
+  well.)
+- ``r``: A 32 or 64-bit integer register (W* or X*).
+- ``w``: A 32, 64, or 128-bit floating-point/SIMD register.
+- ``x``: A lower 128-bit floating-point/SIMD register (``V0`` to ``V15``).
+
+AMDGPU:
+
+- ``r``: A 32 or 64-bit integer register.
+- ``[0-9]v``: The 32-bit VGPR register, number 0-9.
+- ``[0-9]s``: The 32-bit SGPR register, number 0-9.
+
+
+All ARM modes:
+
+- ``Q``, ``Um``, ``Un``, ``Uq``, ``Us``, ``Ut``, ``Uv``, ``Uy``: Memory address
+  operand. Treated the same as operand ``m``, at the moment.
+
+ARM and ARM's Thumb2 mode:
+
+- ``j``: An immediate integer between 0 and 65535 (valid for ``MOVW``)
+- ``I``: An immediate integer valid for a data-processing instruction.
+- ``J``: An immediate integer between -4095 and 4095.
+- ``K``: An immediate integer whose bitwise inverse is valid for a
+  data-processing instruction. (Can be used with template modifier "``B``" to
+  print the inverted value).
+- ``L``: An immediate integer whose negation is valid for a data-processing
+  instruction. (Can be used with template modifier "``n``" to print the negated
+  value).
+- ``M``: A power of two or a integer between 0 and 32.
+- ``N``: Invalid immediate constraint.
+- ``O``: Invalid immediate constraint.
+- ``r``: A general-purpose 32-bit integer register (``r0-r15``).
+- ``l``: In Thumb2 mode, low 32-bit GPR registers (``r0-r7``). In ARM mode, same
+  as ``r``.
+- ``h``: In Thumb2 mode, a high 32-bit GPR register (``r8-r15``). In ARM mode,
+  invalid.
+- ``w``: A 32, 64, or 128-bit floating-point/SIMD register: ``s0-s31``,
+  ``d0-d31``, or ``q0-q15``.
+- ``x``: A 32, 64, or 128-bit floating-point/SIMD register: ``s0-s15``,
+  ``d0-d7``, or ``q0-q3``.
+- ``t``: A floating-point/SIMD register, only supports 32-bit values:
+  ``s0-s31``.
+
+ARM's Thumb1 mode:
+
+- ``I``: An immediate integer between 0 and 255.
+- ``J``: An immediate integer between -255 and -1.
+- ``K``: An immediate integer between 0 and 255, with optional left-shift by
+  some amount.
+- ``L``: An immediate integer between -7 and 7.
+- ``M``: An immediate integer which is a multiple of 4 between 0 and 1020.
+- ``N``: An immediate integer between 0 and 31.
+- ``O``: An immediate integer which is a multiple of 4 between -508 and 508.
+- ``r``: A low 32-bit GPR register (``r0-r7``).
+- ``l``: A low 32-bit GPR register (``r0-r7``).
+- ``h``: A high GPR register (``r0-r7``).
+- ``w``: A 32, 64, or 128-bit floating-point/SIMD register: ``s0-s31``,
+  ``d0-d31``, or ``q0-q15``.
+- ``x``: A 32, 64, or 128-bit floating-point/SIMD register: ``s0-s15``,
+  ``d0-d7``, or ``q0-q3``.
+- ``t``: A floating-point/SIMD register, only supports 32-bit values:
+  ``s0-s31``.
+
+
+Hexagon:
+
+- ``o``, ``v``: A memory address operand, treated the same as constraint ``m``,
+  at the moment.
+- ``r``: A 32 or 64-bit register.
+
+MSP430:
+
+- ``r``: An 8 or 16-bit register.
+
+MIPS:
+
+- ``I``: An immediate signed 16-bit integer.
+- ``J``: An immediate integer zero.
+- ``K``: An immediate unsigned 16-bit integer.
+- ``L``: An immediate 32-bit integer, where the lower 16 bits are 0.
+- ``N``: An immediate integer between -65535 and -1.
+- ``O``: An immediate signed 15-bit integer.
+- ``P``: An immediate integer between 1 and 65535.
+- ``m``: A memory address operand. In MIPS-SE mode, allows a base address
+  register plus 16-bit immediate offset. In MIPS mode, just a base register.
+- ``R``: A memory address operand. In MIPS-SE mode, allows a base address
+  register plus a 9-bit signed offset. In MIPS mode, the same as constraint
+  ``m``.
+- ``ZC``: A memory address operand, suitable for use in a ``pref``, ``ll``, or
+  ``sc`` instruction on the given subtarget (details vary).
+- ``r``, ``d``,  ``y``: A 32 or 64-bit GPR register.
+- ``f``: A 32 or 64-bit FPU register (``F0-F31``), or a 128-bit MSA register
+  (``W0-W31``). In the case of MSA registers, it is recommended to use the ``w``
+  argument modifier for compatibility with GCC.
+- ``c``: A 32-bit or 64-bit GPR register suitable for indirect jump (always
+  ``25``).
+- ``l``: The ``lo`` register, 32 or 64-bit.
+- ``x``: Invalid.
+
+NVPTX:
+
+- ``b``: A 1-bit integer register.
+- ``c`` or ``h``: A 16-bit integer register.
+- ``r``: A 32-bit integer register.
+- ``l`` or ``N``: A 64-bit integer register.
+- ``f``: A 32-bit float register.
+- ``d``: A 64-bit float register.
+
+
+PowerPC:
+
+- ``I``: An immediate signed 16-bit integer.
+- ``J``: An immediate unsigned 16-bit integer, shifted left 16 bits.
+- ``K``: An immediate unsigned 16-bit integer.
+- ``L``: An immediate signed 16-bit integer, shifted left 16 bits.
+- ``M``: An immediate integer greater than 31.
+- ``N``: An immediate integer that is an exact power of 2.
+- ``O``: The immediate integer constant 0.
+- ``P``: An immediate integer constant whose negation is a signed 16-bit
+  constant.
+- ``es``, ``o``, ``Q``, ``Z``, ``Zy``: A memory address operand, currently
+  treated the same as ``m``.
+- ``r``: A 32 or 64-bit integer register.
+- ``b``: A 32 or 64-bit integer register, excluding ``R0`` (that is:
+  ``R1-R31``).
+- ``f``: A 32 or 64-bit float register (``F0-F31``), or when QPX is enabled, a
+  128 or 256-bit QPX register (``Q0-Q31``; aliases the ``F`` registers).
+- ``v``: For ``4 x f32`` or ``4 x f64`` types, when QPX is enabled, a
+  128 or 256-bit QPX register (``Q0-Q31``), otherwise a 128-bit
+  altivec vector register (``V0-V31``).
+
+  .. FIXME: is this a bug that v accepts QPX registers? I think this
+     is supposed to only use the altivec vector registers?
+
+- ``y``: Condition register (``CR0-CR7``).
+- ``wc``: An individual CR bit in a CR register.
+- ``wa``, ``wd``, ``wf``: Any 128-bit VSX vector register, from the full VSX
+  register set (overlapping both the floating-point and vector register files).
+- ``ws``: A 32 or 64-bit floating point register, from the full VSX register
+  set.
+
+Sparc:
+
+- ``I``: An immediate 13-bit signed integer.
+- ``r``: A 32-bit integer register.
+
+SystemZ:
+
+- ``I``: An immediate unsigned 8-bit integer.
+- ``J``: An immediate unsigned 12-bit integer.
+- ``K``: An immediate signed 16-bit integer.
+- ``L``: An immediate signed 20-bit integer.
+- ``M``: An immediate integer 0x7fffffff.
+- ``Q``, ``R``, ``S``, ``T``: A memory address operand, treated the same as
+  ``m``, at the moment.
+- ``r`` or ``d``: A 32, 64, or 128-bit integer register.
+- ``a``: A 32, 64, or 128-bit integer address register (excludes R0, which in an
+  address context evaluates as zero).
+- ``h``: A 32-bit value in the high part of a 64bit data register
+  (LLVM-specific)
+- ``f``: A 32, 64, or 128-bit floating point register.
+
+X86:
+
+- ``I``: An immediate integer between 0 and 31.
+- ``J``: An immediate integer between 0 and 64.
+- ``K``: An immediate signed 8-bit integer.
+- ``L``: An immediate integer, 0xff or 0xffff or (in 64-bit mode only)
+  0xffffffff.
+- ``M``: An immediate integer between 0 and 3.
+- ``N``: An immediate unsigned 8-bit integer.
+- ``O``: An immediate integer between 0 and 127.
+- ``e``: An immediate 32-bit signed integer.
+- ``Z``: An immediate 32-bit unsigned integer.
+- ``o``, ``v``: Treated the same as ``m``, at the moment.
+- ``q``: An 8, 16, 32, or 64-bit register which can be accessed as an 8-bit
+  ``l`` integer register. On X86-32, this is the ``a``, ``b``, ``c``, and ``d``
+  registers, and on X86-64, it is all of the integer registers.
+- ``Q``: An 8, 16, 32, or 64-bit register which can be accessed as an 8-bit
+  ``h`` integer register. This is the ``a``, ``b``, ``c``, and ``d`` registers.
+- ``r`` or ``l``: An 8, 16, 32, or 64-bit integer register.
+- ``R``: An 8, 16, 32, or 64-bit "legacy" integer register -- one which has
+  existed since i386, and can be accessed without the REX prefix.
+- ``f``: A 32, 64, or 80-bit '387 FPU stack pseudo-register.
+- ``y``: A 64-bit MMX register, if MMX is enabled.
+- ``x``: If SSE is enabled: a 32 or 64-bit scalar operand, or 128-bit vector
+  operand in a SSE register. If AVX is also enabled, can also be a 256-bit
+  vector operand in an AVX register. If AVX-512 is also enabled, can also be a
+  512-bit vector operand in an AVX512 register, Otherwise, an error.
+- ``Y``: The same as ``x``, if *SSE2* is enabled, otherwise an error.
+- ``A``: Special case: allocates EAX first, then EDX, for a single operand (in
+  32-bit mode, a 64-bit integer operand will get split into two registers). It
+  is not recommended to use this constraint, as in 64-bit mode, the 64-bit
+  operand will get allocated only to RAX -- if two 32-bit operands are needed,
+  you're better off splitting it yourself, before passing it to the asm
+  statement.
+
+XCore:
+
+- ``r``: A 32-bit integer register.
+
+
+.. _inline-asm-modifiers:
+
+Asm template argument modifiers
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+In the asm template string, modifiers can be used on the operand reference, like
+"``${0:n}``".
+
+The modifiers are, in general, expected to behave the same way they do in
+GCC. LLVM's support is often implemented on an 'as-needed' basis, to support C
+inline asm code which was supported by GCC. A mismatch in behavior between LLVM
+and GCC likely indicates a bug in LLVM.
+
+Target-independent:
+
+- ``c``: Print an immediate integer constant  unadorned, without
+  the target-specific immediate punctuation (e.g. no ``$`` prefix).
+- ``n``: Negate and print immediate integer constant unadorned, without the
+  target-specific immediate punctuation (e.g. no ``$`` prefix).
+- ``l``: Print as an unadorned label, without the target-specific label
+  punctuation (e.g. no ``$`` prefix).
+
+AArch64:
+
+- ``w``: Print a GPR register with a ``w*`` name instead of ``x*`` name. E.g.,
+  instead of ``x30``, print ``w30``.
+- ``x``: Print a GPR register with a ``x*`` name. (this is the default, anyhow).
+- ``b``, ``h``, ``s``, ``d``, ``q``: Print a floating-point/SIMD register with a
+  ``b*``, ``h*``, ``s*``, ``d*``, or ``q*`` name, rather than the default of
+  ``v*``.
+
+AMDGPU:
+
+- ``r``: No effect.
+
+ARM:
+
+- ``a``: Print an operand as an address (with ``[`` and ``]`` surrounding a
+  register).
+- ``P``: No effect.
+- ``q``: No effect.
+- ``y``: Print a VFP single-precision register as an indexed double (e.g. print
+  as ``d4[1]`` instead of ``s9``)
+- ``B``: Bitwise invert and print an immediate integer constant without ``#``
+  prefix.
+- ``L``: Print the low 16-bits of an immediate integer constant.
+- ``M``: Print as a register set suitable for ldm/stm. Also prints *all*
+  register operands subsequent to the specified one (!), so use carefully.
+- ``Q``: Print the low-order register of a register-pair, or the low-order
+  register of a two-register operand.
+- ``R``: Print the high-order register of a register-pair, or the high-order
+  register of a two-register operand.
+- ``H``: Print the second register of a register-pair. (On a big-endian system,
+  ``H`` is equivalent to ``Q``, and on little-endian system, ``H`` is equivalent
+  to ``R``.)
+
+  .. FIXME: H doesn't currently support printing the second register
+     of a two-register operand.
+
+- ``e``: Print the low doubleword register of a NEON quad register.
+- ``f``: Print the high doubleword register of a NEON quad register.
+- ``m``: Print the base register of a memory operand without the ``[`` and ``]``
+  adornment.
+
+Hexagon:
+
+- ``L``: Print the second register of a two-register operand. Requires that it
+  has been allocated consecutively to the first.
+
+  .. FIXME: why is it restricted to consecutive ones? And there's
+     nothing that ensures that happens, is there?
+
+- ``I``: Print the letter 'i' if the operand is an integer constant, otherwise
+  nothing. Used to print 'addi' vs 'add' instructions.
+
+MSP430:
+
+No additional modifiers.
+
+MIPS:
+
+- ``X``: Print an immediate integer as hexadecimal
+- ``x``: Print the low 16 bits of an immediate integer as hexadecimal.
+- ``d``: Print an immediate integer as decimal.
+- ``m``: Subtract one and print an immediate integer as decimal.
+- ``z``: Print $0 if an immediate zero, otherwise print normally.
+- ``L``: Print the low-order register of a two-register operand, or prints the
+  address of the low-order word of a double-word memory operand.
+
+  .. FIXME: L seems to be missing memory operand support.
+
+- ``M``: Print the high-order register of a two-register operand, or prints the
+  address of the high-order word of a double-word memory operand.
+
+  .. FIXME: M seems to be missing memory operand support.
+
+- ``D``: Print the second register of a two-register operand, or prints the
+  second word of a double-word memory operand. (On a big-endian system, ``D`` is
+  equivalent to ``L``, and on little-endian system, ``D`` is equivalent to
+  ``M``.)
+- ``w``: No effect. Provided for compatibility with GCC which requires this
+  modifier in order to print MSA registers (``W0-W31``) with the ``f``
+  constraint.
+
+NVPTX:
+
+- ``r``: No effect.
+
+PowerPC:
+
+- ``L``: Print the second register of a two-register operand. Requires that it
+  has been allocated consecutively to the first.
+
+  .. FIXME: why is it restricted to consecutive ones? And there's
+     nothing that ensures that happens, is there?
+
+- ``I``: Print the letter 'i' if the operand is an integer constant, otherwise
+  nothing. Used to print 'addi' vs 'add' instructions.
+- ``y``: For a memory operand, prints formatter for a two-register X-form
+  instruction. (Currently always prints ``r0,OPERAND``).
+- ``U``: Prints 'u' if the memory operand is an update form, and nothing
+  otherwise. (NOTE: LLVM does not support update form, so this will currently
+  always print nothing)
+- ``X``: Prints 'x' if the memory operand is an indexed form. (NOTE: LLVM does
+  not support indexed form, so this will currently always print nothing)
+
+Sparc:
+
+- ``r``: No effect.
+
+SystemZ:
+
+SystemZ implements only ``n``, and does *not* support any of the other
+target-independent modifiers.
+
+X86:
+
+- ``c``: Print an unadorned integer or symbol name. (The latter is
+  target-specific behavior for this typically target-independent modifier).
+- ``A``: Print a register name with a '``*``' before it.
+- ``b``: Print an 8-bit register name (e.g. ``al``); do nothing on a memory
+  operand.
+- ``h``: Print the upper 8-bit register name (e.g. ``ah``); do nothing on a
+  memory operand.
+- ``w``: Print the 16-bit register name (e.g. ``ax``); do nothing on a memory
+  operand.
+- ``k``: Print the 32-bit register name (e.g. ``eax``); do nothing on a memory
+  operand.
+- ``q``: Print the 64-bit register name (e.g. ``rax``), if 64-bit registers are
+  available, otherwise the 32-bit register name; do nothing on a memory operand.
+- ``n``: Negate and print an unadorned integer, or, for operands other than an
+  immediate integer (e.g. a relocatable symbol expression), print a '-' before
+  the operand. (The behavior for relocatable symbol expressions is a
+  target-specific behavior for this typically target-independent modifier)
+- ``H``: Print a memory reference with additional offset +8.
+- ``P``: Print a memory reference or operand for use as the argument of a call
+  instruction. (E.g. omit ``(rip)``, even though it's PC-relative.)
+
+XCore:
+
+No additional modifiers.
+
+
 Inline Asm Metadata
 ^^^^^^^^^^^^^^^^^^^
 
@@ -2922,12 +3579,12 @@ order.
 These aren't inherently debug info centric, but currently all the specialized
 metadata nodes are related to debug info.
 
-.. _MDCompileUnit:
+.. _DICompileUnit:
 
-MDCompileUnit
+DICompileUnit
 """""""""""""
 
-``MDCompileUnit`` nodes represent a compile unit.  The ``enums:``,
+``DICompileUnit`` nodes represent a compile unit.  The ``enums:``,
 ``retainedTypes:``, ``subprograms:``, ``globals:`` and ``imports:`` fields are
 tuples containing the debug info to be emitted along with the compile unit,
 regardless of code optimizations (some nodes are only emitted if there are
@@ -2935,7 +3592,7 @@ references to them from instructions).
 
 .. code-block:: llvm
 
-    !0 = !MDCompileUnit(language: DW_LANG_C99, file: !1, producer: "clang",
+    !0 = !DICompileUnit(language: DW_LANG_C99, file: !1, producer: "clang",
                         isOptimized: true, flags: "-O2", runtimeVersion: 2,
                         splitDebugFilename: "abc.debug", emissionKind: 1,
                         enums: !2, retainedTypes: !3, subprograms: !4,
@@ -2947,33 +3604,33 @@ These descriptors are collected by a named metadata ``!llvm.dbg.cu``.  They
 keep track of subprograms, global variables, type information, and imported
 entities (declarations and namespaces).
 
-.. _MDFile:
+.. _DIFile:
 
-MDFile
+DIFile
 """"""
 
-``MDFile`` nodes represent files.  The ``filename:`` can include slashes.
+``DIFile`` nodes represent files.  The ``filename:`` can include slashes.
 
 .. code-block:: llvm
 
-    !0 = !MDFile(filename: "path/to/file", directory: "/path/to/dir")
+    !0 = !DIFile(filename: "path/to/file", directory: "/path/to/dir")
 
 Files are sometimes used in ``scope:`` fields, and are the only valid target
 for ``file:`` fields.
 
-.. _MDLocation:
+.. _DIBasicType:
 
-MDBasicType
+DIBasicType
 """""""""""
 
-``MDBasicType`` nodes represent primitive types, such as ``int``, ``bool`` and
+``DIBasicType`` nodes represent primitive types, such as ``int``, ``bool`` and
 ``float``.  ``tag:`` defaults to ``DW_TAG_base_type``.
 
 .. code-block:: llvm
 
-    !0 = !MDBasicType(name: "unsigned char", size: 8, align: 8,
+    !0 = !DIBasicType(name: "unsigned char", size: 8, align: 8,
                       encoding: DW_ATE_unsigned_char)
-    !1 = !MDBasicType(tag: DW_TAG_unspecified_type, name: "decltype(nullptr)")
+    !1 = !DIBasicType(tag: DW_TAG_unspecified_type, name: "decltype(nullptr)")
 
 The ``encoding:`` describes the details of the type.  Usually it's one of the
 following:
@@ -2988,12 +3645,12 @@ following:
   DW_ATE_unsigned      = 7
   DW_ATE_unsigned_char = 8
 
-.. _MDSubroutineType:
+.. _DISubroutineType:
 
-MDSubroutineType
+DISubroutineType
 """"""""""""""""
 
-``MDSubroutineType`` nodes represent subroutine types.  Their ``types:`` field
+``DISubroutineType`` nodes represent subroutine types.  Their ``types:`` field
 refers to a tuple; the first operand is the return type, while the rest are the
 types of the formal arguments in order.  If the first operand is ``null``, that
 represents a function with no return value (such as ``void foo() {}`` in C++).
@@ -3002,21 +3659,21 @@ represents a function with no return value (such as ``void foo() {}`` in C++).
 
     !0 = !BasicType(name: "int", size: 32, align: 32, DW_ATE_signed)
     !1 = !BasicType(name: "char", size: 8, align: 8, DW_ATE_signed_char)
-    !2 = !MDSubroutineType(types: !{null, !0, !1}) ; void (int, char)
+    !2 = !DISubroutineType(types: !{null, !0, !1}) ; void (int, char)
 
-.. _MDDerivedType:
+.. _DIDerivedType:
 
-MDDerivedType
+DIDerivedType
 """""""""""""
 
-``MDDerivedType`` nodes represent types derived from other types, such as
+``DIDerivedType`` nodes represent types derived from other types, such as
 qualified types.
 
 .. code-block:: llvm
 
-    !0 = !MDBasicType(name: "unsigned char", size: 8, align: 8,
+    !0 = !DIBasicType(name: "unsigned char", size: 8, align: 8,
                       encoding: DW_ATE_unsigned_char)
-    !1 = !MDDerivedType(tag: DW_TAG_pointer_type, baseType: !0, size: 32,
+    !1 = !DIDerivedType(tag: DW_TAG_pointer_type, baseType: !0, size: 32,
                         align: 32)
 
 The following ``tag:`` values are valid:
@@ -3034,7 +3691,7 @@ The following ``tag:`` values are valid:
   DW_TAG_restrict_type      = 55
 
 ``DW_TAG_member`` is used to define a member of a :ref:`composite type
-<MDCompositeType>` or :ref:`subprogram <MDSubprogram>`.  The type of the member
+<DICompositeType>` or :ref:`subprogram <DISubprogram>`.  The type of the member
 is the ``baseType:``.  The ``offset:`` is the member's bit offset.
 ``DW_TAG_formal_parameter`` is used to define a member which is a formal
 argument of a subprogram.
@@ -3047,12 +3704,12 @@ argument of a subprogram.
 
 Note that the ``void *`` type is expressed as a type derived from NULL.
 
-.. _MDCompositeType:
+.. _DICompositeType:
 
-MDCompositeType
+DICompositeType
 """""""""""""""
 
-``MDCompositeType`` nodes represent types composed of other types, like
+``DICompositeType`` nodes represent types composed of other types, like
 structures and unions.  ``elements:`` points to a tuple of the composed types.
 
 If the source language supports ODR, the ``identifier:`` field gives the unique
@@ -3062,10 +3719,10 @@ can refer to composite types indirectly via a :ref:`metadata string
 
 .. code-block:: llvm
 
-    !0 = !MDEnumerator(name: "SixKind", value: 7)
-    !1 = !MDEnumerator(name: "SevenKind", value: 7)
-    !2 = !MDEnumerator(name: "NegEightKind", value: -8)
-    !3 = !MDCompositeType(tag: DW_TAG_enumeration_type, name: "Enum", file: !12,
+    !0 = !DIEnumerator(name: "SixKind", value: 7)
+    !1 = !DIEnumerator(name: "SevenKind", value: 7)
+    !2 = !DIEnumerator(name: "NegEightKind", value: -8)
+    !3 = !DICompositeType(tag: DW_TAG_enumeration_type, name: "Enum", file: !12,
                           line: 2, size: 32, align: 32, identifier: "_M4Enum",
                           elements: !{!0, !1, !2})
 
@@ -3083,108 +3740,108 @@ The following ``tag:`` values are valid:
 
 
 For ``DW_TAG_array_type``, the ``elements:`` should be :ref:`subrange
-descriptors <MDSubrange>`, each representing the range of subscripts at that
+descriptors <DISubrange>`, each representing the range of subscripts at that
 level of indexing.  The ``DIFlagVector`` flag to ``flags:`` indicates that an
 array type is a native packed vector.
 
 For ``DW_TAG_enumeration_type``, the ``elements:`` should be :ref:`enumerator
-descriptors <MDEnumerator>`, each representing the definition of an enumeration
+descriptors <DIEnumerator>`, each representing the definition of an enumeration
 value for the set.  All enumeration type descriptors are collected in the
-``enums:`` field of the :ref:`compile unit <MDCompileUnit>`.
+``enums:`` field of the :ref:`compile unit <DICompileUnit>`.
 
 For ``DW_TAG_structure_type``, ``DW_TAG_class_type``, and
 ``DW_TAG_union_type``, the ``elements:`` should be :ref:`derived types
-<MDDerivedType>` with ``tag: DW_TAG_member`` or ``tag: DW_TAG_inheritance``.
+<DIDerivedType>` with ``tag: DW_TAG_member`` or ``tag: DW_TAG_inheritance``.
 
-.. _MDSubrange:
+.. _DISubrange:
 
-MDSubrange
+DISubrange
 """"""""""
 
-``MDSubrange`` nodes are the elements for ``DW_TAG_array_type`` variants of
-:ref:`MDCompositeType`.  ``count: -1`` indicates an empty array.
+``DISubrange`` nodes are the elements for ``DW_TAG_array_type`` variants of
+:ref:`DICompositeType`.  ``count: -1`` indicates an empty array.
 
 .. code-block:: llvm
 
-    !0 = !MDSubrange(count: 5, lowerBound: 0) ; array counting from 0
-    !1 = !MDSubrange(count: 5, lowerBound: 1) ; array counting from 1
-    !2 = !MDSubrange(count: -1) ; empty array.
+    !0 = !DISubrange(count: 5, lowerBound: 0) ; array counting from 0
+    !1 = !DISubrange(count: 5, lowerBound: 1) ; array counting from 1
+    !2 = !DISubrange(count: -1) ; empty array.
 
-.. _MDEnumerator:
+.. _DIEnumerator:
 
-MDEnumerator
+DIEnumerator
 """"""""""""
 
-``MDEnumerator`` nodes are the elements for ``DW_TAG_enumeration_type``
-variants of :ref:`MDCompositeType`.
+``DIEnumerator`` nodes are the elements for ``DW_TAG_enumeration_type``
+variants of :ref:`DICompositeType`.
 
 .. code-block:: llvm
 
-    !0 = !MDEnumerator(name: "SixKind", value: 7)
-    !1 = !MDEnumerator(name: "SevenKind", value: 7)
-    !2 = !MDEnumerator(name: "NegEightKind", value: -8)
+    !0 = !DIEnumerator(name: "SixKind", value: 7)
+    !1 = !DIEnumerator(name: "SevenKind", value: 7)
+    !2 = !DIEnumerator(name: "NegEightKind", value: -8)
 
-MDTemplateTypeParameter
+DITemplateTypeParameter
 """""""""""""""""""""""
 
-``MDTemplateTypeParameter`` nodes represent type parameters to generic source
-language constructs.  They are used (optionally) in :ref:`MDCompositeType` and
-:ref:`MDSubprogram` ``templateParams:`` fields.
+``DITemplateTypeParameter`` nodes represent type parameters to generic source
+language constructs.  They are used (optionally) in :ref:`DICompositeType` and
+:ref:`DISubprogram` ``templateParams:`` fields.
 
 .. code-block:: llvm
 
-    !0 = !MDTemplateTypeParameter(name: "Ty", type: !1)
+    !0 = !DITemplateTypeParameter(name: "Ty", type: !1)
 
-MDTemplateValueParameter
+DITemplateValueParameter
 """"""""""""""""""""""""
 
-``MDTemplateValueParameter`` nodes represent value parameters to generic source
+``DITemplateValueParameter`` nodes represent value parameters to generic source
 language constructs.  ``tag:`` defaults to ``DW_TAG_template_value_parameter``,
 but if specified can also be set to ``DW_TAG_GNU_template_template_param`` or
 ``DW_TAG_GNU_template_param_pack``.  They are used (optionally) in
-:ref:`MDCompositeType` and :ref:`MDSubprogram` ``templateParams:`` fields.
+:ref:`DICompositeType` and :ref:`DISubprogram` ``templateParams:`` fields.
 
 .. code-block:: llvm
 
-    !0 = !MDTemplateValueParameter(name: "Ty", type: !1, value: i32 7)
+    !0 = !DITemplateValueParameter(name: "Ty", type: !1, value: i32 7)
 
-MDNamespace
+DINamespace
 """""""""""
 
-``MDNamespace`` nodes represent namespaces in the source language.
+``DINamespace`` nodes represent namespaces in the source language.
 
 .. code-block:: llvm
 
-    !0 = !MDNamespace(name: "myawesomeproject", scope: !1, file: !2, line: 7)
+    !0 = !DINamespace(name: "myawesomeproject", scope: !1, file: !2, line: 7)
 
-MDGlobalVariable
+DIGlobalVariable
 """"""""""""""""
 
-``MDGlobalVariable`` nodes represent global variables in the source language.
+``DIGlobalVariable`` nodes represent global variables in the source language.
 
 .. code-block:: llvm
 
-    !0 = !MDGlobalVariable(name: "foo", linkageName: "foo", scope: !1,
+    !0 = !DIGlobalVariable(name: "foo", linkageName: "foo", scope: !1,
                            file: !2, line: 7, type: !3, isLocal: true,
                            isDefinition: false, variable: i32* @foo,
                            declaration: !4)
 
 All global variables should be referenced by the `globals:` field of a
-:ref:`compile unit <MDCompileUnit>`.
+:ref:`compile unit <DICompileUnit>`.
 
-.. _MDSubprogram:
+.. _DISubprogram:
 
-MDSubprogram
+DISubprogram
 """"""""""""
 
-``MDSubprogram`` nodes represent functions from the source language.  The
-``variables:`` field points at :ref:`variables <MDLocalVariable>` that must be
+``DISubprogram`` nodes represent functions from the source language.  The
+``variables:`` field points at :ref:`variables <DILocalVariable>` that must be
 retained, even if their IR counterparts are optimized out of the IR.  The
-``type:`` field must point at an :ref:`MDSubroutineType`.
+``type:`` field must point at an :ref:`DISubroutineType`.
 
 .. code-block:: llvm
 
-    !0 = !MDSubprogram(name: "foo", linkageName: "_Zfoov", scope: !1,
+    !0 = !DISubprogram(name: "foo", linkageName: "_Zfoov", scope: !1,
                        file: !2, line: 7, type: !3, isLocal: true,
                        isDefinition: false, scopeLine: 8, containingType: !4,
                        virtuality: DW_VIRTUALITY_pure_virtual, virtualIndex: 10,
@@ -3192,76 +3849,78 @@ retained, even if their IR counterparts are optimized out of the IR.  The
                        function: void ()* @_Z3foov,
                        templateParams: !5, declaration: !6, variables: !7)
 
-.. _MDLexicalBlock:
+.. _DILexicalBlock:
 
-MDLexicalBlock
+DILexicalBlock
 """"""""""""""
 
-``MDLexicalBlock`` nodes describe nested blocks within a :ref:`subprogram
-<MDSubprogram>`.  The line number and column numbers are used to dinstinguish
+``DILexicalBlock`` nodes describe nested blocks within a :ref:`subprogram
+<DISubprogram>`.  The line number and column numbers are used to dinstinguish
 two lexical blocks at same depth.  They are valid targets for ``scope:``
 fields.
 
 .. code-block:: llvm
 
-    !0 = distinct !MDLexicalBlock(scope: !1, file: !2, line: 7, column: 35)
+    !0 = distinct !DILexicalBlock(scope: !1, file: !2, line: 7, column: 35)
 
 Usually lexical blocks are ``distinct`` to prevent node merging based on
 operands.
 
-.. _MDLexicalBlockFile:
+.. _DILexicalBlockFile:
 
-MDLexicalBlockFile
+DILexicalBlockFile
 """"""""""""""""""
 
-``MDLexicalBlockFile`` nodes are used to discriminate between sections of a
-:ref:`lexical block <MDLexicalBlock>`.  The ``file:`` field can be changed to
+``DILexicalBlockFile`` nodes are used to discriminate between sections of a
+:ref:`lexical block <DILexicalBlock>`.  The ``file:`` field can be changed to
 indicate textual inclusion, or the ``discriminator:`` field can be used to
 discriminate between control flow within a single block in the source language.
 
 .. code-block:: llvm
 
-    !0 = !MDLexicalBlock(scope: !3, file: !4, line: 7, column: 35)
-    !1 = !MDLexicalBlockFile(scope: !0, file: !4, discriminator: 0)
-    !2 = !MDLexicalBlockFile(scope: !0, file: !4, discriminator: 1)
+    !0 = !DILexicalBlock(scope: !3, file: !4, line: 7, column: 35)
+    !1 = !DILexicalBlockFile(scope: !0, file: !4, discriminator: 0)
+    !2 = !DILexicalBlockFile(scope: !0, file: !4, discriminator: 1)
+
+.. _DILocation:
 
-MDLocation
+DILocation
 """"""""""
 
-``MDLocation`` nodes represent source debug locations.  The ``scope:`` field is
-mandatory, and points at an :ref:`MDLexicalBlockFile`, an
-:ref:`MDLexicalBlock`, or an :ref:`MDSubprogram`.
+``DILocation`` nodes represent source debug locations.  The ``scope:`` field is
+mandatory, and points at an :ref:`DILexicalBlockFile`, an
+:ref:`DILexicalBlock`, or an :ref:`DISubprogram`.
 
 .. code-block:: llvm
 
-    !0 = !MDLocation(line: 2900, column: 42, scope: !1, inlinedAt: !2)
+    !0 = !DILocation(line: 2900, column: 42, scope: !1, inlinedAt: !2)
 
-.. _MDLocalVariable:
+.. _DILocalVariable:
 
-MDLocalVariable
+DILocalVariable
 """""""""""""""
 
-``MDLocalVariable`` nodes represent local variables in the source language.
+``DILocalVariable`` nodes represent local variables in the source language.
 Instead of ``DW_TAG_variable``, they use LLVM-specific fake tags to
 discriminate between local variables (``DW_TAG_auto_variable``) and subprogram
 arguments (``DW_TAG_arg_variable``).  In the latter case, the ``arg:`` field
 specifies the argument position, and this variable will be included in the
-``variables:`` field of its :ref:`MDSubprogram`.
+``variables:`` field of its :ref:`DISubprogram`.
 
 .. code-block:: llvm
 
-    !0 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "this", arg: 0,
+    !0 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "this", arg: 1,
                           scope: !3, file: !2, line: 7, type: !3,
                           flags: DIFlagArtificial)
-    !1 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "x", arg: 1,
+    !1 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "x", arg: 2,
                           scope: !4, file: !2, line: 7, type: !3)
-    !1 = !MDLocalVariable(tag: DW_TAG_auto_variable, name: "y",
+    !2 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "y",
                           scope: !5, file: !2, line: 7, type: !3)
 
-MDExpression
+DIExpression
 """"""""""""
 
-``MDExpression`` nodes represent DWARF expression sequences.  They are used in
+``DIExpression`` nodes represent DWARF expression sequences.  They are used in
 :ref:`debug intrinsics<dbg_intrinsics>` (such as ``llvm.dbg.declare``) to
 describe how the referenced LLVM variable relates to the source language
 variable.
@@ -3275,30 +3934,30 @@ The current supported vocabulary is limited:
 
 .. code-block:: llvm
 
-    !0 = !MDExpression(DW_OP_deref)
-    !1 = !MDExpression(DW_OP_plus, 3)
-    !2 = !MDExpression(DW_OP_bit_piece, 3, 7)
-    !3 = !MDExpression(DW_OP_deref, DW_OP_plus, 3, DW_OP_bit_piece, 3, 7)
+    !0 = !DIExpression(DW_OP_deref)
+    !1 = !DIExpression(DW_OP_plus, 3)
+    !2 = !DIExpression(DW_OP_bit_piece, 3, 7)
+    !3 = !DIExpression(DW_OP_deref, DW_OP_plus, 3, DW_OP_bit_piece, 3, 7)
 
-MDObjCProperty
+DIObjCProperty
 """"""""""""""
 
-``MDObjCProperty`` nodes represent Objective-C property nodes.
+``DIObjCProperty`` nodes represent Objective-C property nodes.
 
 .. code-block:: llvm
 
-    !3 = !MDObjCProperty(name: "foo", file: !1, line: 7, setter: "setFoo",
+    !3 = !DIObjCProperty(name: "foo", file: !1, line: 7, setter: "setFoo",
                          getter: "getFoo", attributes: 7, type: !2)
 
-MDImportedEntity
+DIImportedEntity
 """"""""""""""""
 
-``MDImportedEntity`` nodes represent entities (such as modules) imported into a
+``DIImportedEntity`` nodes represent entities (such as modules) imported into a
 compile unit.
 
 .. code-block:: llvm
 
-   !2 = !MDImportedEntity(tag: DW_TAG_imported_module, name: "foo", scope: !0,
+   !2 = !DIImportedEntity(tag: DW_TAG_imported_module, name: "foo", scope: !0,
                           entity: !1, line: 7)
 
 '``tbaa``' Metadata
@@ -3423,7 +4082,7 @@ For example,
     %2 = load float, float* %c, align 4, !alias.scope !5
     store float %2, float* %arrayidx.i2, align 4, !noalias !6
 
-    ; These two instructions don't alias (for domain !0, the set of scopes in
+    ; These two instructions may alias (for domain !0, the set of scopes in
     ; the !noalias list is not a superset of, or equal to, the scopes in the
     ; !alias.scope list):
     %2 = load float, float* %c, align 4, !alias.scope !6
@@ -3607,7 +4266,7 @@ will be partially unrolled.
 '``llvm.loop.unroll.disable``' Metadata
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
-This metadata either disables loop unrolling. The metadata has a single operand
+This metadata disables loop unrolling. The metadata has a single operand
 which is the string ``llvm.loop.unroll.disable``.  For example:
 
 .. code-block:: llvm
@@ -3617,7 +4276,7 @@ which is the string ``llvm.loop.unroll.disable``.  For example:
 '``llvm.loop.unroll.runtime.disable``' Metadata
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
-This metadata either disables runtime loop unrolling. The metadata has a single
+This metadata disables runtime loop unrolling. The metadata has a single
 operand which is the string ``llvm.loop.unroll.runtime.disable``.  For example:
 
 .. code-block:: llvm
@@ -3627,8 +4286,8 @@ operand which is the string ``llvm.loop.unroll.runtime.disable``.  For example:
 '``llvm.loop.unroll.full``' Metadata
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
-This metadata either suggests that the loop should be unrolled fully. The
-metadata has a single operand which is the string ``llvm.loop.unroll.disable``.
+This metadata suggests that the loop should be unrolled fully. The
+metadata has a single operand which is the string ``llvm.loop.unroll.full``.
 For example:
 
 .. code-block:: llvm
@@ -4082,7 +4741,12 @@ control flow, not values (the one exception being the
 The terminator instructions are: ':ref:`ret <i_ret>`',
 ':ref:`br <i_br>`', ':ref:`switch <i_switch>`',
 ':ref:`indirectbr <i_indirectbr>`', ':ref:`invoke <i_invoke>`',
-':ref:`resume <i_resume>`', and ':ref:`unreachable <i_unreachable>`'.
+':ref:`resume <i_resume>`', ':ref:`catchpad <i_catchpad>`',
+':ref:`catchendpad <i_catchendpad>`',
+':ref:`catchret <i_catchret>`',
+':ref:`cleanupret <i_cleanupret>`',
+':ref:`terminatepad <i_terminatepad>`',
+and ':ref:`unreachable <i_unreachable>`'.
 
 .. _i_ret:
 
@@ -4436,6 +5100,301 @@ Example:
 
       resume { i8*, i32 } %exn
 
+.. _i_catchpad:
+
+'``catchpad``' Instruction
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+Syntax:
+"""""""
+
+::
+
+      <resultval> = catchpad <resultty> [<args>*]
+          to label <normal label> unwind label <exception label>
+
+Overview:
+"""""""""
+
+The '``catchpad``' instruction is used by `LLVM's exception handling
+system <ExceptionHandling.html#overview>`_ to specify that a basic block
+is a catch block --- one where a personality routine attempts to transfer
+control to catch an exception.
+The ``args`` correspond to whatever information the personality
+routine requires to know if this is an appropriate place to catch the
+exception.  Control is tranfered to the ``exception`` label if the
+``catchpad`` is not an appropriate handler for the in-flight exception.
+The ``normal`` label should contain the code found in the ``catch``
+portion of a ``try``/``catch`` sequence. It defines values supplied by
+the :ref:`personality function <personalityfn>` upon re-entry to the
+function. The ``resultval`` has the type ``resultty``.
+
+Arguments:
+""""""""""
+
+The instruction takes a list of arbitrary values which are interpreted
+by the :ref:`personality function <personalityfn>`.
+
+The ``catchpad`` must be provided a ``normal`` label to transfer control
+to if the ``catchpad`` matches the exception and an ``exception``
+label to transfer control to if it doesn't.
+
+Semantics:
+""""""""""
+
+The '``catchpad``' instruction defines the values which are set by the
+:ref:`personality function <personalityfn>` upon re-entry to the function, and
+therefore the "result type" of the ``catchpad`` instruction. As with
+calling conventions, how the personality function results are
+represented in LLVM IR is target specific.
+
+When the call stack is being unwound due to an exception being thrown,
+the exception is compared against the ``args``. If it doesn't match,
+then control is transfered to the ``exception`` basic block.
+
+The ``catchpad`` instruction has several restrictions:
+
+-  A catch block is a basic block which is the unwind destination of
+   an exceptional instruction.
+-  A catch block must have a '``catchpad``' instruction as its
+   first non-PHI instruction.
+-  A catch block's ``exception`` edge must refer to a catch block or a
+   catch-end block.
+-  There can be only one '``catchpad``' instruction within the
+   catch block.
+-  A basic block that is not a catch block may not include a
+   '``catchpad``' instruction.
+-  It is undefined behavior for control to transfer from a ``catchpad`` to a
+   ``cleanupret`` without first executing a ``catchret`` and a subsequent
+   ``cleanuppad``.
+-  It is undefined behavior for control to transfer from a ``catchpad`` to a
+   ``ret`` without first executing a ``catchret``.
+
+Example:
+""""""""
+
+.. code-block:: llvm
+
+      ;; A catch block which can catch an integer.
+      %res = catchpad { i8*, i32 } [i8** @_ZTIi]
+        to label %int.handler unwind label %terminate
+
+.. _i_catchendpad:
+
+'``catchendpad``' Instruction
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+Syntax:
+"""""""
+
+::
+
+      catchendpad unwind label <nextaction>
+      catchendpad unwind to caller
+
+Overview:
+"""""""""
+
+The '``catchendpad``' instruction is used by `LLVM's exception handling
+system <ExceptionHandling.html#overview>`_ to communicate to the
+:ref:`personality function <personalityfn>` which invokes are associated
+with a chain of :ref:`catchpad <i_catchpad>` instructions.
+
+The ``nextaction`` label indicates where control should transfer to if
+none of the ``catchpad`` instructions are suitable for catching the
+in-flight exception.
+
+If a ``nextaction`` label is not present, the instruction unwinds out of
+its parent function.  The
+:ref:`personality function <personalityfn>` will continue processing
+exception handling actions in the caller.
+
+Arguments:
+""""""""""
+
+The instruction optionally takes a label, ``nextaction``, indicating
+where control should transfer to if none of the preceding
+``catchpad`` instructions are suitable for the in-flight exception.
+
+Semantics:
+""""""""""
+
+When the call stack is being unwound due to an exception being thrown
+and none of the constituent ``catchpad`` instructions match, then
+control is transfered to ``nextaction`` if it is present.  If it is not
+present, control is transfered to the caller.
+
+The ``catchendpad`` instruction has several restrictions:
+
+-  A catch-end block is a basic block which is the unwind destination of
+   an exceptional instruction.
+-  A catch-end block must have a '``catchendpad``' instruction as its
+   first non-PHI instruction.
+-  There can be only one '``catchendpad``' instruction within the
+   catch block.
+-  A basic block that is not a catch-end block may not include a
+   '``catchendpad``' instruction.
+-  Exactly one catch block may unwind to a ``catchendpad``.
+-  The unwind target of invokes between a ``catchpad`` and a
+   corresponding ``catchret`` must be its ``catchendpad``.
+
+Example:
+""""""""
+
+.. code-block:: llvm
+
+      catchendpad unwind label %terminate
+      catchendpad unwind to caller
+
+.. _i_catchret:
+
+'``catchret``' Instruction
+^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+Syntax:
+"""""""
+
+::
+
+      catchret label <normal>
+
+Overview:
+"""""""""
+
+The '``catchret``' instruction is a terminator instruction that has a
+single successor.
+
+
+Arguments:
+""""""""""
+
+The '``catchret``' instruction requires one argument which specifies
+where control will transfer to next.
+
+Semantics:
+""""""""""
+
+The '``catchret``' instruction ends the existing (in-flight) exception
+whose unwinding was interrupted with a
+:ref:`catchpad <i_catchpad>` instruction.
+The :ref:`personality function <personalityfn>` gets a chance to execute
+arbitrary code to, for example, run a C++ destructor.
+Control then transfers to ``normal``.
+
+Example:
+""""""""
+
+.. code-block:: llvm
+
+      catchret label %continue
+
+.. _i_cleanupret:
+
+'``cleanupret``' Instruction
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+Syntax:
+"""""""
+
+::
+
+      cleanupret <type> <value> unwind label <continue>
+      cleanupret <type> <value> unwind to caller
+
+Overview:
+"""""""""
+
+The '``cleanupret``' instruction is a terminator instruction that has
+an optional successor.
+
+
+Arguments:
+""""""""""
+
+The '``cleanupret``' instruction requires one argument, which must have the
+same type as the result of any '``cleanuppad``' instruction in the same
+function.  It also has an optional successor,  ``continue``.
+
+Semantics:
+""""""""""
+
+The '``cleanupret``' instruction indicates to the
+:ref:`personality function <personalityfn>` that one
+:ref:`cleanuppad <i_cleanuppad>` it transferred control to has ended.
+It transfers control to ``continue`` or unwinds out of the function.
+
+Example:
+""""""""
+
+.. code-block:: llvm
+
+      cleanupret void unwind to caller
+      cleanupret { i8*, i32 } %exn unwind label %continue
+
+.. _i_terminatepad:
+
+'``terminatepad``' Instruction
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+Syntax:
+"""""""
+
+::
+
+      terminatepad [<args>*] unwind label <exception label>
+      terminatepad [<args>*] unwind to caller
+
+Overview:
+"""""""""
+
+The '``terminatepad``' instruction is used by `LLVM's exception handling
+system <ExceptionHandling.html#overview>`_ to specify that a basic block
+is a terminate block --- one where a personality routine may decide to
+terminate the program.
+The ``args`` correspond to whatever information the personality
+routine requires to know if this is an appropriate place to terminate the
+program.  Control is transferred to the ``exception`` label if the
+personality routine decides not to terminate the program for the
+in-flight exception.
+
+Arguments:
+""""""""""
+
+The instruction takes a list of arbitrary values which are interpreted
+by the :ref:`personality function <personalityfn>`.
+
+The ``terminatepad`` may be given an ``exception`` label to
+transfer control to if the in-flight exception matches the ``args``.
+
+Semantics:
+""""""""""
+
+When the call stack is being unwound due to an exception being thrown,
+the exception is compared against the ``args``. If it matches,
+then control is transfered to the ``exception`` basic block.  Otherwise,
+the program is terminated via personality-specific means.  Typically,
+the first argument to ``terminatepad`` specifies what function the
+personality should defer to in order to terminate the program.
+
+The ``terminatepad`` instruction has several restrictions:
+
+-  A terminate block is a basic block which is the unwind destination of
+   an exceptional instruction.
+-  A terminate block must have a '``terminatepad``' instruction as its
+   first non-PHI instruction.
+-  There can be only one '``terminatepad``' instruction within the
+   terminate block.
+-  A basic block that is not a terminate block may not include a
+   '``terminatepad``' instruction.
+
+Example:
+""""""""
+
+.. code-block:: llvm
+
+      ;; A terminate block which only permits integers.
+      terminatepad [i8** @_ZTIi] unwind label %continue
+
 .. _i_unreachable:
 
 '``unreachable``' Instruction
@@ -5060,7 +6019,7 @@ Semantics:
 
 The value produced is ``op1`` \* 2\ :sup:`op2` mod 2\ :sup:`n`,
 where ``n`` is the width of the result. If ``op2`` is (statically or
-dynamically) negative or equal to or larger than the number of bits in
+dynamically) equal to or larger than the number of bits in
 ``op1``, the result is undefined. If the arguments are vectors, each
 vector element of ``op1`` is shifted by the corresponding shift amount
 in ``op2``.
@@ -5656,7 +6615,7 @@ Syntax:
 
 ::
 
-      <result> = load [volatile] <ty>, <ty>* <pointer>[, align <alignment>][, !nontemporal !<index>][, !invariant.load !<index>][, !nonnull !<index>]
+      <result> = load [volatile] <ty>, <ty>* <pointer>[, align <alignment>][, !nontemporal !<index>][, !invariant.load !<index>][, !nonnull !<index>][, !dereferenceable !<index>][, !dereferenceable_or_null !<index>]
       <result> = load atomic [volatile] <ty>* <pointer> [singlethread] <ordering>, align <alignment>
       !<index> = !{ i32 1 }
 
@@ -5721,6 +6680,25 @@ never be null.  This is analogous to the ''nonnull'' attribute
 on parameters and return values.  This metadata can only be applied
 to loads of a pointer type.
 
+The optional ``!dereferenceable`` metadata must reference a single
+metadata name ``<index>`` corresponding to a metadata node with one ``i64``
+entry. The existence of the ``!dereferenceable`` metadata on the instruction 
+tells the optimizer that the value loaded is known to be dereferenceable.
+The number of bytes known to be dereferenceable is specified by the integer 
+value in the metadata node. This is analogous to the ''dereferenceable'' 
+attribute on parameters and return values.  This metadata can only be applied 
+to loads of a pointer type.
+
+The optional ``!dereferenceable_or_null`` metadata must reference a single
+metadata name ``<index>`` corresponding to a metadata node with one ``i64``
+entry. The existence of the ``!dereferenceable_or_null`` metadata on the 
+instruction tells the optimizer that the value loaded is known to be either
+dereferenceable or null.
+The number of bytes known to be dereferenceable is specified by the integer 
+value in the metadata node. This is analogous to the ''dereferenceable_or_null'' 
+attribute on parameters and return values.  This metadata can only be applied 
+to loads of a pointer type.
+
 Semantics:
 """"""""""
 
@@ -6054,7 +7032,8 @@ Overview:
 
 The '``getelementptr``' instruction is used to get the address of a
 subelement of an :ref:`aggregate <t_aggregate>` data structure. It performs
-address calculation only and does not access memory.
+address calculation only and does not access memory. The instruction can also
+be used to calculate a vector of such addresses.
 
 Arguments:
 """"""""""
@@ -6180,12 +7159,61 @@ Example:
         ; yields i32*:iptr
         %iptr = getelementptr [10 x i32], [10 x i32]* @arr, i16 0, i16 0
 
-In cases where the pointer argument is a vector of pointers, each index
-must be a vector with the same number of elements. For example:
+Vector of pointers:
+"""""""""""""""""""
+
+The ``getelementptr`` returns a vector of pointers, instead of a single address,
+when one or more of its arguments is a vector. In such cases, all vector
+arguments should have the same number of elements, and every scalar argument
+will be effectively broadcast into a vector during address calculation.
+
+.. code-block:: llvm
+
+     ; All arguments are vectors:
+     ;   A[i] = ptrs[i] + offsets[i]*sizeof(i8)
+     %A = getelementptr i8, <4 x i8*> %ptrs, <4 x i64> %offsets
+     
+     ; Add the same scalar offset to each pointer of a vector:
+     ;   A[i] = ptrs[i] + offset*sizeof(i8)
+     %A = getelementptr i8, <4 x i8*> %ptrs, i64 %offset
+     
+     ; Add distinct offsets to the same pointer:
+     ;   A[i] = ptr + offsets[i]*sizeof(i8)
+     %A = getelementptr i8, i8* %ptr, <4 x i64> %offsets
+     
+     ; In all cases described above the type of the result is <4 x i8*>
+
+The two following instructions are equivalent:
+
+.. code-block:: llvm
+
+     getelementptr  %struct.ST, <4 x %struct.ST*> %s, <4 x i64> %ind1,
+       <4 x i32> <i32 2, i32 2, i32 2, i32 2>,
+       <4 x i32> <i32 1, i32 1, i32 1, i32 1>,
+       <4 x i32> %ind4,
+       <4 x i64> <i64 13, i64 13, i64 13, i64 13>
+     
+     getelementptr  %struct.ST, <4 x %struct.ST*> %s, <4 x i64> %ind1,
+       i32 2, i32 1, <4 x i32> %ind4, i64 13
+
+Let's look at the C code, where the vector version of ``getelementptr``
+makes sense:
+
+.. code-block:: c
+
+    // Let's assume that we vectorize the following loop:
+    double *A, B; int *C;
+    for (int i = 0; i < size; ++i) {
+      A[i] = B[C[i]];
+    }
 
 .. code-block:: llvm
 
-     %A = getelementptr i8, <4 x i8*> %ptrs, <4 x i64> %offsets,
+    ; get pointers for 8 elements from array B
+    %ptrs = getelementptr double, double* %B, <8 x i32> %C
+    ; load 8 elements from array B into A
+    %A = call <8 x double> @llvm.masked.gather.v8f64(<8 x double*> %ptrs,
+         i32 8, <8 x i1> %mask, <8 x double> %passthru)
 
 Conversion Operations
 ---------------------
@@ -6859,7 +7887,7 @@ Syntax:
 
 ::
 
-      <result> = fcmp <cond> <ty> <op1>, <op2>     ; yields i1 or <N x i1>:result
+      <result> = fcmp [fast-math flags]* <cond> <ty> <op1>, <op2>     ; yields i1 or <N x i1>:result
 
 Overview:
 """""""""
@@ -6942,6 +7970,15 @@ always yields an :ref:`i1 <t_integer>` result, as follows:
 #. ``uno``: yields ``true`` if either operand is a QNAN.
 #. ``true``: always yields ``true``, regardless of operands.
 
+The ``fcmp`` instruction can also optionally take any number of
+:ref:`fast-math flags <fastmath>`, which are optimization hints to enable
+otherwise unsafe floating point optimizations.
+
+Any set of fast-math flags are legal on an ``fcmp`` instruction, but the
+only flags that have any effect on its semantics are those that allow
+assumptions to be made about the values of input arguments; namely
+``nnan``, ``ninf``, and ``nsz``. See :ref:`fastmath` for more information.
+
 Example:
 """"""""
 
@@ -7241,8 +8278,8 @@ Syntax:
 
 ::
 
-      <resultval> = landingpad <resultty> personality <type> <pers_fn> <clause>+
-      <resultval> = landingpad <resultty> personality <type> <pers_fn> cleanup <clause>*
+      <resultval> = landingpad <resultty> <clause>+
+      <resultval> = landingpad <resultty> cleanup <clause>*
 
       <clause> := catch <type> <value>
       <clause> := filter <array constant type> <array constant>
@@ -7254,14 +8291,13 @@ The '``landingpad``' instruction is used by `LLVM's exception handling
 system <ExceptionHandling.html#overview>`_ to specify that a basic block
 is a landing pad --- one where the exception lands, and corresponds to the
 code found in the ``catch`` portion of a ``try``/``catch`` sequence. It
-defines values supplied by the personality function (``pers_fn``) upon
+defines values supplied by the :ref:`personality function <personalityfn>` upon
 re-entry to the function. The ``resultval`` has the type ``resultty``.
 
 Arguments:
 """"""""""
 
-This instruction takes a ``pers_fn`` value. This is the personality
-function associated with the unwinding mechanism. The optional
+The optional
 ``cleanup`` flag indicates that the landing pad block is a cleanup.
 
 A ``clause`` begins with the clause type --- ``catch`` or ``filter`` --- and
@@ -7276,7 +8312,7 @@ Semantics:
 """"""""""
 
 The '``landingpad``' instruction defines the values which are set by the
-personality function (``pers_fn``) upon re-entry to the function, and
+:ref:`personality function <personalityfn>` upon re-entry to the function, and
 therefore the "result type" of the ``landingpad`` instruction. As with
 calling conventions, how the personality function results are
 represented in LLVM IR is target specific.
@@ -7299,8 +8335,6 @@ The ``landingpad`` instruction has several restrictions:
    pad block.
 -  A basic block that is not a landing pad block may not include a
    '``landingpad``' instruction.
--  All '``landingpad``' instructions in a function must have the same
-   personality function.
 
 Example:
 """"""""
@@ -7308,16 +8342,82 @@ Example:
 .. code-block:: llvm
 
       ;; A landing pad which can catch an integer.
-      %res = landingpad { i8*, i32 } personality i32 (...)* @__gxx_personality_v0
+      %res = landingpad { i8*, i32 }
                catch i8** @_ZTIi
       ;; A landing pad that is a cleanup.
-      %res = landingpad { i8*, i32 } personality i32 (...)* @__gxx_personality_v0
+      %res = landingpad { i8*, i32 }
                cleanup
       ;; A landing pad which can catch an integer and can only throw a double.
-      %res = landingpad { i8*, i32 } personality i32 (...)* @__gxx_personality_v0
+      %res = landingpad { i8*, i32 }
                catch i8** @_ZTIi
                filter [1 x i8**] [@_ZTId]
 
+.. _i_cleanuppad:
+
+'``cleanuppad``' Instruction
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+Syntax:
+"""""""
+
+::
+
+      <resultval> = cleanuppad <resultty> [<args>*]
+
+Overview:
+"""""""""
+
+The '``cleanuppad``' instruction is used by `LLVM's exception handling
+system <ExceptionHandling.html#overview>`_ to specify that a basic block
+is a cleanup block --- one where a personality routine attempts to
+transfer control to run cleanup actions.
+The ``args`` correspond to whatever additional
+information the :ref:`personality function <personalityfn>` requires to
+execute the cleanup.
+The ``resultval`` has the type ``resultty``.
+
+Arguments:
+""""""""""
+
+The instruction takes a list of arbitrary values which are interpreted
+by the :ref:`personality function <personalityfn>`.
+
+Semantics:
+""""""""""
+
+The '``cleanuppad``' instruction defines the values which are set by the
+:ref:`personality function <personalityfn>` upon re-entry to the function, and
+therefore the "result type" of the ``cleanuppad`` instruction. As with
+calling conventions, how the personality function results are
+represented in LLVM IR is target specific.
+
+When the call stack is being unwound due to an exception being thrown,
+the :ref:`personality function <personalityfn>` transfers control to the
+``cleanuppad`` with the aid of the personality-specific arguments.
+
+The ``cleanuppad`` instruction has several restrictions:
+
+-  A cleanup block is a basic block which is the unwind destination of
+   an exceptional instruction.
+-  A cleanup block must have a '``cleanuppad``' instruction as its
+   first non-PHI instruction.
+-  There can be only one '``cleanuppad``' instruction within the
+   cleanup block.
+-  A basic block that is not a cleanup block may not include a
+   '``cleanuppad``' instruction.
+-  It is undefined behavior for control to transfer from a ``cleanuppad`` to a
+   ``catchret`` without first executing a ``cleanupret`` and a subsequent
+   ``catchpad``.
+-  It is undefined behavior for control to transfer from a ``cleanuppad`` to a
+   ``ret`` without first executing a ``cleanupret``.
+
+Example:
+""""""""
+
+.. code-block:: llvm
+
+      %res = cleanuppad { i8*, i32 } [label %nextaction]
+
 .. _intrinsics:
 
 Intrinsic Functions
@@ -7729,7 +8829,7 @@ Note that calling this intrinsic does not prevent function inlining or
 other aggressive transformations, so the value returned may not be that
 of the obvious source-language caller.
 
-'``llvm.frameescape``' and '``llvm.framerecover``' Intrinsics
+'``llvm.localescape``' and '``llvm.localrecover``' Intrinsics
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
 Syntax:
@@ -7737,49 +8837,47 @@ Syntax:
 
 ::
 
-      declare void @llvm.frameescape(...)
-      declare i8* @llvm.framerecover(i8* %func, i8* %fp, i32 %idx)
+      declare void @llvm.localescape(...)
+      declare i8* @llvm.localrecover(i8* %func, i8* %fp, i32 %idx)
 
 Overview:
 """""""""
 
-The '``llvm.frameescape``' intrinsic escapes offsets of a collection of static
-allocas, and the '``llvm.framerecover``' intrinsic applies those offsets to a
+The '``llvm.localescape``' intrinsic escapes offsets of a collection of static
+allocas, and the '``llvm.localrecover``' intrinsic applies those offsets to a
 live frame pointer to recover the address of the allocation. The offset is
-computed during frame layout of the caller of ``llvm.frameescape``.
+computed during frame layout of the caller of ``llvm.localescape``.
 
 Arguments:
 """"""""""
 
-All arguments to '``llvm.frameescape``' must be pointers to static allocas or
-casts of static allocas. Each function can only call '``llvm.frameescape``'
+All arguments to '``llvm.localescape``' must be pointers to static allocas or
+casts of static allocas. Each function can only call '``llvm.localescape``'
 once, and it can only do so from the entry block.
 
-The ``func`` argument to '``llvm.framerecover``' must be a constant
+The ``func`` argument to '``llvm.localrecover``' must be a constant
 bitcasted pointer to a function defined in the current module. The code
 generator cannot determine the frame allocation offset of functions defined in
 other modules.
 
-The ``fp`` argument to '``llvm.framerecover``' must be a frame
-pointer of a call frame that is currently live. The return value of
-'``llvm.frameaddress``' is one way to produce such a value, but most platforms
-also expose the frame pointer through stack unwinding mechanisms.
+The ``fp`` argument to '``llvm.localrecover``' must be a frame pointer of a
+call frame that is currently live. The return value of '``llvm.localaddress``'
+is one way to produce such a value, but various runtimes also expose a suitable
+pointer in platform-specific ways.
 
-The ``idx`` argument to '``llvm.framerecover``' indicates which alloca passed to
-'``llvm.frameescape``' to recover. It is zero-indexed.
+The ``idx`` argument to '``llvm.localrecover``' indicates which alloca passed to
+'``llvm.localescape``' to recover. It is zero-indexed.
 
 Semantics:
 """"""""""
 
-These intrinsics allow a group of functions to access one stack memory
-allocation in an ancestor stack frame. The memory returned from
-'``llvm.frameallocate``' may be allocated prior to stack realignment, so the
-memory is only aligned to the ABI-required stack alignment.  Each function may
-only call '``llvm.frameallocate``' one or zero times from the function entry
-block.  The frame allocation intrinsic inhibits inlining, as any frame
-allocations in the inlined function frame are likely to be at a different
-offset from the one used by '``llvm.framerecover``' called with the
-uninlined function.
+These intrinsics allow a group of functions to share access to a set of local
+stack allocations of a one parent function. The parent function may call the
+'``llvm.localescape``' intrinsic once from the function entry block, and the
+child functions can use '``llvm.localrecover``' to access the escaped allocas.
+The '``llvm.localescape``' intrinsic blocks inlining, as inlining changes where
+the escaped allocas are allocated, which would break attempts to use
+'``llvm.localrecover``'.
 
 .. _int_read_register:
 .. _int_write_register:
@@ -9481,6 +10579,75 @@ Examples:
 Specialised Arithmetic Intrinsics
 ---------------------------------
 
+'``llvm.canonicalize.*``' Intrinsic
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+Syntax:
+"""""""
+
+::
+
+      declare float @llvm.canonicalize.f32(float %a)
+      declare double @llvm.canonicalize.f64(double %b)
+
+Overview:
+"""""""""
+
+The '``llvm.canonicalize.*``' intrinsic returns the platform specific canonical
+encoding of a floating point number.  This canonicalization is useful for
+implementing certain numeric primitives such as frexp. The canonical encoding is
+defined by IEEE-754-2008 to be:
+
+::
+
+      2.1.8 canonical encoding: The preferred encoding of a floating-point
+      representation in a format.  Applied to declets, significands of finite
+      numbers, infinities, and NaNs, especially in decimal formats.
+
+This operation can also be considered equivalent to the IEEE-754-2008
+conversion of a floating-point value to the same format.  NaNs are handled
+according to section 6.2.
+
+Examples of non-canonical encodings:
+
+- x87 pseudo denormals, pseudo NaNs, pseudo Infinity, Unnormals.  These are
+  converted to a canonical representation per hardware-specific protocol.
+- Many normal decimal floating point numbers have non-canonical alternative
+  encodings.
+- Some machines, like GPUs or ARMv7 NEON, do not support subnormal values.
+  These are treated as non-canonical encodings of zero and with be flushed to
+  a zero of the same sign by this operation.
+
+Note that per IEEE-754-2008 6.2, systems that support signaling NaNs with
+default exception handling must signal an invalid exception, and produce a
+quiet NaN result.
+
+This function should always be implementable as multiplication by 1.0, provided
+that the compiler does not constant fold the operation.  Likewise, division by
+1.0 and ``llvm.minnum(x, x)`` are possible implementations.  Addition with
+-0.0 is also sufficient provided that the rounding mode is not -Infinity.
+
+``@llvm.canonicalize`` must preserve the equality relation.  That is:
+
+- ``(@llvm.canonicalize(x) == x)`` is equivalent to ``(x == x)``
+- ``(@llvm.canonicalize(x) == @llvm.canonicalize(y))`` is equivalent to
+  to ``(x == y)``
+
+Additionally, the sign of zero must be conserved:
+``@llvm.canonicalize(-0.0) = -0.0`` and ``@llvm.canonicalize(+0.0) = +0.0``
+
+The payload bits of a NaN must be conserved, with two exceptions.
+First, environments which use only a single canonical representation of NaN
+must perform said canonicalization.  Second, SNaNs must be quieted per the
+usual methods.
+
+The canonicalization operation may be optimized away if:
+
+- The input is known to be canonical.  For example, it was produced by a
+  floating-point operation that is required by the standard to be canonical.
+- The result is consumed only by (or fused with) other floating-point
+  operations.  That is, the bits of the floating point value are not examined.
+
 '``llvm.fmuladd.*``' Intrinsic
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
@@ -9530,6 +10697,65 @@ Examples:
 
       %r2 = call float @llvm.fmuladd.f32(float %a, float %b, float %c) ; yields float:r2 = (a * b) + c
 
+
+'``llvm.uabsdiff.*``' and '``llvm.sabsdiff.*``' Intrinsics
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+Syntax:
+"""""""
+This is an overloaded intrinsic. The loaded data is a vector of any integer bit width.
+
+.. code-block:: llvm
+
+      declare <4 x integer> @llvm.uabsdiff.v4i32(<4 x integer> %a, <4 x integer> %b)
+
+
+Overview:
+"""""""""
+
+The ``llvm.uabsdiff`` intrinsic returns a vector result of the absolute difference of the two operands,
+treating them both as unsigned integers.
+
+The ``llvm.sabsdiff`` intrinsic returns  a vector result of the absolute difference of the two operands,
+treating them both as signed integers.
+
+.. note::
+
+    These intrinsics are primarily used during the code generation stage of compilation.
+    They are generated by compiler passes such as the Loop and SLP vectorizers.it is not
+    recommended for users to create them manually.
+
+Arguments:
+""""""""""
+
+Both intrinsics take two integer of the same bitwidth.
+
+Semantics:
+""""""""""
+
+The expression::
+
+    call <4 x i32> @llvm.uabsdiff.v4i32(<4 x i32> %a, <4 x i32> %b)
+
+is equivalent to::
+
+    %sub = sub <4 x i32> %a, %b
+    %ispos = icmp ugt <4 x i32> %sub, <i32 -1, i32 -1, i32 -1, i32 -1>
+    %neg = sub <4 x i32> zeroinitializer, %sub
+    %1 = select <4 x i1> %ispos, <4 x i32> %sub, <4 x i32> %neg
+
+Similarly the expression::
+
+    call <4 x i32> @llvm.sabsdiff.v4i32(<4 x i32> %a, <4 x i32> %b)
+
+is equivalent to::
+
+    %sub = sub nsw <4 x i32> %a, %b
+    %ispos = icmp sgt <4 x i32> %sub, <i32 -1, i32 -1, i32 -1, i32 -1>
+    %neg = sub nsw <4 x i32> zeroinitializer, %sub
+    %1 = select <4 x i1> %ispos, <4 x i32> %sub, <4 x i32> %neg
+
+
 Half Precision Floating Point Intrinsics
 ----------------------------------------
 
@@ -9754,6 +10980,8 @@ intrinsic returns the executable address corresponding to ``tramp``
 after performing the required machine specific adjustments. The pointer
 returned can then be :ref:`bitcast and executed <int_trampoline>`.
 
+.. _int_mload_mstore:
+
 Masked Vector Load and Store Intrinsics
 ---------------------------------------
 
@@ -9776,13 +11004,13 @@ This is an overloaded intrinsic. The loaded data is a vector of any integer or f
 Overview:
 """""""""
 
-Reads a vector from memory according to the provided mask. The mask holds a bit for each vector lane, and is used to prevent memory accesses to the masked-off lanes. The masked-off lanes in the result vector are taken from the corresponding lanes in the passthru operand.
+Reads a vector from memory according to the provided mask. The mask holds a bit for each vector lane, and is used to prevent memory accesses to the masked-off lanes. The masked-off lanes in the result vector are taken from the corresponding lanes of the '``passthru``' operand.
 
 
 Arguments:
 """"""""""
 
-The first operand is the base pointer for the load. The second operand is the alignment of the source location. It must be a constant integer value. The third operand, mask, is a vector of boolean 'i1' values with the same number of elements as the return type. The fourth is a pass-through value that is used to fill the masked-off lanes of the result. The return type, underlying type of the base pointer and the type of passthru operand are the same vector types.
+The first operand is the base pointer for the load. The second operand is the alignment of the source location. It must be a constant integer value. The third operand, mask, is a vector of boolean values with the same number of elements as the return type. The fourth is a pass-through value that is used to fill the masked-off lanes of the result. The return type, underlying type of the base pointer and the type of the '``passthru``' operand are the same vector types.
 
 
 Semantics:
@@ -9841,6 +11069,115 @@ The result of this operation is equivalent to a load-modify-store sequence. Howe
        store <16 x float> %res, <16 x float>* %ptr, align 4
 
 
+Masked Vector Gather and Scatter Intrinsics
+-------------------------------------------
+
+LLVM provides intrinsics for vector gather and scatter operations. They are similar to :ref:`Masked Vector Load and Store <int_mload_mstore>`, except they are designed for arbitrary memory accesses, rather than sequential memory accesses. Gather and scatter also employ a mask operand, which holds one bit per vector element, switching the associated vector lane on or off. The memory addresses corresponding to the "off" lanes are not accessed. When all bits are off, no memory is accessed.
+
+.. _int_mgather:
+
+'``llvm.masked.gather.*``' Intrinsics
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+Syntax:
+"""""""
+This is an overloaded intrinsic. The loaded data are multiple scalar values of any integer or floating point data type gathered together into one vector.
+
+::
+
+      declare <16 x float> @llvm.masked.gather.v16f32 (<16 x float*> <ptrs>, i32 <alignment>, <16 x i1> <mask>, <16 x float> <passthru>)
+      declare <2 x double> @llvm.masked.gather.v2f64  (<2 x double*> <ptrs>, i32 <alignment>, <2 x i1>  <mask>, <2 x double> <passthru>)
+
+Overview:
+"""""""""
+
+Reads scalar values from arbitrary memory locations and gathers them into one vector. The memory locations are provided in the vector of pointers '``ptrs``'. The memory is accessed according to the provided mask. The mask holds a bit for each vector lane, and is used to prevent memory accesses to the masked-off lanes. The masked-off lanes in the result vector are taken from the corresponding lanes of the '``passthru``' operand.
+
+
+Arguments:
+""""""""""
+
+The first operand is a vector of pointers which holds all memory addresses to read. The second operand is an alignment of the source addresses. It must be a constant integer value. The third operand, mask, is a vector of boolean values with the same number of elements as the return type. The fourth is a pass-through value that is used to fill the masked-off lanes of the result. The return type, underlying type of the vector of pointers and the type of the '``passthru``' operand are the same vector types.
+
+
+Semantics:
+""""""""""
+
+The '``llvm.masked.gather``' intrinsic is designed for conditional reading of multiple scalar values from arbitrary memory locations in a single IR operation. It is useful for targets that support vector masked gathers and allows vectorizing basic blocks with data and control divergence. Other targets may support this intrinsic differently, for example by lowering it into a sequence of scalar load operations.
+The semantics of this operation are equivalent to a sequence of conditional scalar loads with subsequent gathering all loaded values into a single vector. The mask restricts memory access to certain lanes and facilitates vectorization of predicated basic blocks.
+
+
+::
+
+       %res = call <4 x double> @llvm.masked.gather.v4f64 (<4 x double*> %ptrs, i32 8, <4 x i1>%mask, <4 x double> <true, true, true, true>)
+
+       ;; The gather with all-true mask is equivalent to the following instruction sequence
+       %ptr0 = extractelement <4 x double*> %ptrs, i32 0
+       %ptr1 = extractelement <4 x double*> %ptrs, i32 1
+       %ptr2 = extractelement <4 x double*> %ptrs, i32 2
+       %ptr3 = extractelement <4 x double*> %ptrs, i32 3
+
+       %val0 = load double, double* %ptr0, align 8
+       %val1 = load double, double* %ptr1, align 8
+       %val2 = load double, double* %ptr2, align 8
+       %val3 = load double, double* %ptr3, align 8
+
+       %vec0    = insertelement <4 x double>undef, %val0, 0
+       %vec01   = insertelement <4 x double>%vec0, %val1, 1
+       %vec012  = insertelement <4 x double>%vec01, %val2, 2
+       %vec0123 = insertelement <4 x double>%vec012, %val3, 3
+
+.. _int_mscatter:
+
+'``llvm.masked.scatter.*``' Intrinsics
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+Syntax:
+"""""""
+This is an overloaded intrinsic. The data stored in memory is a vector of any integer or floating point data type. Each vector element is stored in an arbitrary memory addresses. Scatter with overlapping addresses is guaranteed to be ordered from least-significant to most-significant element.
+
+::
+
+       declare void @llvm.masked.scatter.v8i32 (<8 x i32>  <value>, <8 x i32*>  <ptrs>, i32 <alignment>,  <8 x i1>  <mask>)
+       declare void @llvm.masked.scatter.v16f32(<16 x i32> <value>, <16 x i32*> <ptrs>, i32 <alignment>,  <16 x i1> <mask>)
+
+Overview:
+"""""""""
+
+Writes each element from the value vector to the corresponding memory address. The memory addresses are represented as a vector of pointers. Writing is done according to the provided mask. The mask holds a bit for each vector lane, and is used to prevent memory accesses to the masked-off lanes.
+
+Arguments:
+""""""""""
+
+The first operand is a vector value to be written to memory. The second operand is a vector of pointers, pointing to where the value elements should be stored. It has the same underlying type as the value operand. The third operand is an alignment of the destination addresses. The fourth operand, mask, is a vector of boolean values. The types of the mask and the value operand must have the same number of vector elements.
+
+
+Semantics:
+""""""""""
+
+The '``llvm.masked.scatter``' intrinsics is designed for writing selected vector elements to arbitrary memory addresses in a single IR operation. The operation may be conditional, when not all bits in the mask are switched on. It is useful for targets that support vector masked scatter and allows vectorizing basic blocks with data and control divergency. Other targets may support this intrinsic differently, for example by lowering it into a sequence of branches that guard scalar store operations.
+
+::
+
+       ;; This instruction unconditionaly stores data vector in multiple addresses
+       call @llvm.masked.scatter.v8i32 (<8 x i32> %value, <8 x i32*> %ptrs, i32 4,  <8 x i1>  <true, true, .. true>)
+
+       ;; It is equivalent to a list of scalar stores
+       %val0 = extractelement <8 x i32> %value, i32 0
+       %val1 = extractelement <8 x i32> %value, i32 1
+       ..
+       %val7 = extractelement <8 x i32> %value, i32 7
+       %ptr0 = extractelement <8 x i32*> %ptrs, i32 0
+       %ptr1 = extractelement <8 x i32*> %ptrs, i32 1
+       ..
+       %ptr7 = extractelement <8 x i32*> %ptrs, i32 7
+       ;; Note: the order of the following stores is important when they overlap:
+       store i32 %val0, i32* %ptr0, align 4
+       store i32 %val1, i32* %ptr1, align 4
+       ..
+       store i32 %val7, i32* %ptr7, align 4
+
+
 Memory Use Markers
 ------------------
 
@@ -10285,6 +11622,8 @@ Semantics:
 
 This intrinsic is lowered to the ``val``.
 
+.. _int_assume:
+
 '``llvm.assume``' Intrinsic
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
diff --git a/docs/Lexicon.rst b/docs/Lexicon.rst
index 9a599da1285..912dee2cf07 100644
--- a/docs/Lexicon.rst
+++ b/docs/Lexicon.rst
@@ -46,6 +46,15 @@ B
 C
 -
 
+**CFI**
+    Call Frame Information. Used in DWARF debug info and in C++ unwind info
+    to show how the function prolog lays out the stack frame.
+
+**CIE**
+    Common Information Entry.  A kind of CFI used to reduce the size of FDEs.
+    The compiler creates a CIE which contains the information common across all
+    the FDEs.  Each FDE then points to its CIE.
+
 **CSE**
     Common Subexpression Elimination. An optimization that removes common
     subexpression compuation. For example ``(a+b)*(a+b)`` has two subexpressions
@@ -82,6 +91,10 @@ F
 **FCA**
     First Class Aggregate
 
+**FDE**
+    Frame Description Entry. A kind of CFI used to describe the stack frame of
+    one function.
+
 G
 -
 
@@ -118,9 +131,21 @@ L
 **LCSSA**
     Loop-Closed Static Single Assignment Form
 
+**LGTM**
+    "Looks Good To Me". In a review thread, this indicates that the
+    reviewer thinks that the patch is okay to commit.
+
 **LICM**
     Loop Invariant Code Motion
 
+**LSDA**
+    Language Specific Data Area.  C++ "zero cost" unwinding is built on top a
+    generic unwinding mechanism.  As the unwinder walks each frame, it calls
+    a "personality" function to do language specific analysis.  Each function's
+    FDE points to an optional LSDA which is passed to the personality function.
+    For C++, the LSDA contain info about the type and location of catch
+    statements in that function.
+
 **Load-VN**
     Load Value Numbering
 
diff --git a/docs/LibFuzzer.rst b/docs/LibFuzzer.rst
index a31f83d3950..1ac75a40698 100644
--- a/docs/LibFuzzer.rst
+++ b/docs/LibFuzzer.rst
@@ -14,26 +14,29 @@ This library is intended primarily for in-process coverage-guided fuzz testing
 * Build the Fuzzer library as a static archive (or just a set of .o files).
   Note that the Fuzzer contains the main() function.
   Preferably do *not* use sanitizers while building the Fuzzer.
-* Build the library you are going to test with -fsanitize-coverage=[234]
+* Build the library you are going to test with
+  `-fsanitize-coverage={bb,edge}[,indirect-calls,8bit-counters]`
   and one of the sanitizers. We recommend to build the library in several
   different modes (e.g. asan, msan, lsan, ubsan, etc) and even using different
   optimizations options (e.g. -O0, -O1, -O2) to diversify testing.
 * Build a test driver using the same options as the library.
   The test driver is a C/C++ file containing interesting calls to the library
-  inside a single function  ``extern "C" void TestOneInput(const uint8_t *Data, size_t Size);``
+  inside a single function  ``extern "C" void LLVMFuzzerTestOneInput(const uint8_t *Data, size_t Size);``
 * Link the Fuzzer, the library and the driver together into an executable
   using the same sanitizer options as for the library.
 * Collect the initial corpus of inputs for the
   fuzzer (a directory with test inputs, one file per input).
   The better your inputs are the faster you will find something interesting.
   Also try to keep your inputs small, otherwise the Fuzzer will run too slow.
+  By default, the Fuzzer limits the size of every input to 64 bytes
+  (use ``-max_len=N`` to override).
 * Run the fuzzer with the test corpus. As new interesting test cases are
   discovered they will be added to the corpus. If a bug is discovered by
   the sanitizer (asan, etc) it will be reported as usual and the reproducer
   will be written to disk.
   Each Fuzzer process is single-threaded (unless the library starts its own
-  threads). You can run the Fuzzer on the same corpus in multiple processes.
-  in parallel. For run-time options run the Fuzzer binary with '-help=1'.
+  threads). You can run the Fuzzer on the same corpus in multiple processes
+  in parallel.
 
 
 The Fuzzer is similar in concept to AFL_,
@@ -47,6 +50,27 @@ and is used to fuzz various parts of LLVM,
 but the Fuzzer itself does not (and should not) depend on any
 part of LLVM and can be used for other projects w/o requiring the rest of LLVM.
 
+Flags
+=====
+The most important flags are::
+
+  seed                               	0	Random seed. If 0, seed is generated.
+  runs                               	-1	Number of individual test runs (-1 for infinite runs).
+  max_len                            	64	Maximum length of the test input.
+  cross_over                         	1	If 1, cross over inputs.
+  mutate_depth                       	5	Apply this number of consecutive mutations to each input.
+  timeout                            	1200	Timeout in seconds (if positive). If one unit runs more than this number of seconds the process will abort.
+  help                               	0	Print help.
+  save_minimized_corpus              	0	If 1, the minimized corpus is saved into the first input directory
+  jobs                               	0	Number of jobs to run. If jobs >= 1 we spawn this number of jobs in separate worker processes with stdout/stderr redirected to fuzz-JOB.log.
+  workers                            	0	Number of simultaneous worker processes to run the jobs. If zero, "min(jobs,NumberOfCpuCores()/2)" is used.
+  tokens                             	0	Use the file with tokens (one token per line) to fuzz a token based input language.
+  apply_tokens                       	0	Read the given input file, substitute bytes  with tokens and write the result to stdout.
+  sync_command                       	0	Execute an external command "<sync_command> <test_corpus>" to synchronize the test corpus.
+  sync_timeout                       	600	Minimum timeout between syncs.
+
+For the full list of flags run the fuzzer binary with ``-help=1``.
+
 Usage examples
 ==============
 
@@ -56,7 +80,7 @@ Toy example
 A simple function that does something interesting if it receives the input "HI!"::
 
   cat << EOF >> test_fuzzer.cc
-  extern "C" void TestOneInput(const unsigned char *data, unsigned long size) {
+  extern "C" void LLVMFuzzerTestOneInput(const unsigned char *data, unsigned long size) {
     if (size > 0 && data[0] == 'H')
       if (size > 1 && data[1] == 'I')
          if (size > 2 && data[2] == '!')
@@ -68,7 +92,7 @@ A simple function that does something interesting if it receives the input "HI!"
   # Build lib/Fuzzer files.
   clang -c -g -O2 -std=c++11 Fuzzer/*.cpp -IFuzzer
   # Build test_fuzzer.cc with asan and link against lib/Fuzzer.
-  clang++ -fsanitize=address -fsanitize-coverage=3 test_fuzzer.cc Fuzzer*.o
+  clang++ -fsanitize=address -fsanitize-coverage=edge test_fuzzer.cc Fuzzer*.o
   # Run the fuzzer with no corpus.
   ./a.out
 
@@ -79,7 +103,7 @@ PCRE2
 
 Here we show how to use lib/Fuzzer on something real, yet simple: pcre2_::
 
-  COV_FLAGS=" -fsanitize-coverage=4 -mllvm -sanitizer-coverage-8bit-counters=1"
+  COV_FLAGS=" -fsanitize-coverage=edge,indirect-calls,8bit-counters"
   # Get PCRE2
   svn co svn://vcs.exim.org/pcre2/code/trunk pcre
   # Get lib/Fuzzer. Assuming that you already have fresh clang in PATH.
@@ -88,11 +112,11 @@ Here we show how to use lib/Fuzzer on something real, yet simple: pcre2_::
   (cd pcre; ./autogen.sh; CC="clang -fsanitize=address $COV_FLAGS" ./configure --prefix=`pwd`/../inst && make -j && make install)
   # Build lib/Fuzzer files.
   clang -c -g -O2 -std=c++11 Fuzzer/*.cpp -IFuzzer
-  # Build the the actual function that does something interesting with PCRE2.
+  # Build the actual function that does something interesting with PCRE2.
   cat << EOF > pcre_fuzzer.cc
   #include <string.h>
   #include "pcre2posix.h"
-  extern "C" void TestOneInput(const unsigned char *data, size_t size) {
+  extern "C" void LLVMFuzzerTestOneInput(const unsigned char *data, size_t size) {
     if (size < 1) return;
     char *str = new char[size+1];
     memcpy(str, data, size);
@@ -151,17 +175,21 @@ Now, interrupt the fuzzer and run it again the same way. You will see::
 This time you were running the fuzzer with a non-empty input corpus (564 items).
 As the first step, the fuzzer minimized the set to produce 344 interesting items (the ``INITED`` line)
 
+It is quite convenient to store test corpuses in git.
+As an example, here is a git repository with test inputs for the above PCRE2 fuzzer::
+
+  git clone https://github.com/kcc/fuzzing-with-sanitizers.git
+  ./pcre_fuzzer ./fuzzing-with-sanitizers/pcre2/C1/
+
 You may run ``N`` independent fuzzer jobs in parallel on ``M`` CPUs::
 
   N=100; M=4; ./pcre_fuzzer ./CORPUS -jobs=$N -workers=$M
 
-This is useful when you already have an exhaustive test corpus.
-If you've just started fuzzing with no good corpus running independent
-jobs will create a corpus with too many duplicates.
-One way to avoid this and still use all of your CPUs is to use the flag ``-exit_on_first=1``
-which will cause the fuzzer to exit on the first new synthesised input::
+By default (``-reload=1``) the fuzzer processes will periodically scan the CORPUS directory
+and reload any new tests. This way the test inputs found by one process will be picked up
+by all others.
 
-  N=100; M=4; ./pcre_fuzzer ./CORPUS -jobs=$N -workers=$M -exit_on_first=1
+If ``-workers=$M`` is not supplied, ``min($N,NumberOfCpuCore/2)`` will be used.
 
 Heartbleed
 ----------
@@ -172,7 +200,7 @@ to find Heartbleed with LibFuzzer::
 
   wget https://www.openssl.org/source/openssl-1.0.1f.tar.gz
   tar xf openssl-1.0.1f.tar.gz
-  COV_FLAGS="-fsanitize-coverage=4" # -mllvm -sanitizer-coverage-8bit-counters=1"
+  COV_FLAGS="-fsanitize-coverage=edge,indirect-calls" # -fsanitize-coverage=8bit-counters
   (cd openssl-1.0.1f/ && ./config &&
     make -j 32 CC="clang -g -fsanitize=address $COV_FLAGS")
   # Get and build LibFuzzer
@@ -196,7 +224,7 @@ to find Heartbleed with LibFuzzer::
     assert (SSL_CTX_use_PrivateKey_file(sctx, "server.key", SSL_FILETYPE_PEM));
     return 0;
   }
-  extern "C" void TestOneInput(unsigned char *Data, size_t Size) {
+  extern "C" void LLVMFuzzerTestOneInput(unsigned char *Data, size_t Size) {
     static int unused = Init();
     SSL *server = SSL_new(sctx);
     BIO *sinbio = BIO_new(BIO_s_mem());
@@ -259,7 +287,7 @@ Periodically restart both fuzzers so that they can use each other's findings.
 How good is my fuzzer?
 ----------------------
 
-Once you implement your target function ``TestOneInput`` and fuzz it to death,
+Once you implement your target function ``LLVMFuzzerTestOneInput`` and fuzz it to death,
 you will want to know whether the function or the corpus can be improved further.
 One easy to use metric is, of course, code coverage.
 You can get the coverage for your corpus like this::
@@ -271,6 +299,12 @@ and dump covered PCs to disk before exiting.
 Then you can subtract the set of covered PCs from the set of all instrumented PCs in the binary,
 see SanitizerCoverage_ for details.
 
+User-supplied mutators
+----------------------
+
+LibFuzzer allows to use custom (user-supplied) mutators,
+see FuzzerInterface.h_
+
 Fuzzing components of LLVM
 ==========================
 
@@ -299,6 +333,24 @@ Clang can also be fuzzed with Tokens_ using ``-tokens=$LLVM/lib/Fuzzer/cxx_fuzze
 
 Tracking bug: https://llvm.org/bugs/show_bug.cgi?id=23057
 
+Buildbot
+--------
+
+We have a buildbot that runs the above fuzzers for LLVM components
+24/7/365 at http://lab.llvm.org:8011/builders/sanitizer-x86_64-linux-fuzzer .
+
+Pre-fuzzed test inputs in git
+-----------------------------
+
+The buildbot occumulates large test corpuses over time.
+The corpuses are stored in git on github and can be used like this::
+
+  git clone https://github.com/kcc/fuzzing-with-sanitizers.git
+  bin/clang-format-fuzzer fuzzing-with-sanitizers/llvm/clang-format/C1
+  bin/clang-fuzzer        fuzzing-with-sanitizers/llvm/clang/C1/
+  bin/clang-fuzzer        fuzzing-with-sanitizers/llvm/clang/TOK1  -tokens=$LLVM/llvm/lib/Fuzzer/cxx_fuzzer_tokens.txt
+
+
 FAQ
 =========================
 
@@ -359,6 +411,8 @@ Examples: regular expression matchers, text or binary format parsers.
 
 .. _AFL: http://lcamtuf.coredump.cx/afl/
 
-.. _SanitizerCoverage: https://code.google.com/p/address-sanitizer/wiki/AsanCoverage
+.. _SanitizerCoverage: http://clang.llvm.org/docs/SanitizerCoverage.html
 
 .. _Heartbleed: http://en.wikipedia.org/wiki/Heartbleed
+
+.. _FuzzerInterface.h: https://github.com/llvm-mirror/llvm/blob/master/lib/Fuzzer/FuzzerInterface.h
diff --git a/docs/NVPTXUsage.rst b/docs/NVPTXUsage.rst
index e1c401df877..fc697ca0046 100644
--- a/docs/NVPTXUsage.rst
+++ b/docs/NVPTXUsage.rst
@@ -168,10 +168,10 @@ These are overloaded intrinsics.  You can use these on any pointer types.
 
 .. code-block:: llvm
 
-    declare i8* @llvm.nvvm.ptr.gen.to.global.p1i8.p0i8(i8 addrspace(1)*)
-    declare i8* @llvm.nvvm.ptr.gen.to.shared.p3i8.p0i8(i8 addrspace(3)*)
-    declare i8* @llvm.nvvm.ptr.gen.to.constant.p4i8.p0i8(i8 addrspace(4)*)
-    declare i8* @llvm.nvvm.ptr.gen.to.local.p5i8.p0i8(i8 addrspace(5)*)
+    declare i8 addrspace(1)* @llvm.nvvm.ptr.gen.to.global.p1i8.p0i8(i8*)
+    declare i8 addrspace(3)* @llvm.nvvm.ptr.gen.to.shared.p3i8.p0i8(i8*)
+    declare i8 addrspace(4)* @llvm.nvvm.ptr.gen.to.constant.p4i8.p0i8(i8*)
+    declare i8 addrspace(5)* @llvm.nvvm.ptr.gen.to.local.p5i8.p0i8(i8*)
 
 Overview:
 """""""""
diff --git a/docs/Phabricator.rst b/docs/Phabricator.rst
index b0f62e02d19..69bf3bfd103 100644
--- a/docs/Phabricator.rst
+++ b/docs/Phabricator.rst
@@ -60,26 +60,28 @@ to upload your patch):
 To upload a new patch:
 
 * Click *Differential*.
-* Click *Create Diff*.
-* Paste the text diff or upload the patch file.
-  Note that TODO
+* Click *+ Create Diff*.
+* Paste the text diff or browse to the patch file. Click *Create Diff*.
+* Leave the Repository field blank.
 * Leave the drop down on *Create a new Revision...* and click *Continue*.
 * Enter a descriptive title and summary.  The title and summary are usually
   in the form of a :ref:`commit message <commit messages>`.
 * Add reviewers and mailing
   lists that you want to be included in the review. If your patch is
-  for LLVM, add llvm-commits as a subscriber; if your patch is for Clang,
+  for LLVM, add llvm-commits as a Subscriber; if your patch is for Clang,
   add cfe-commits.
+* Leave the Repository and Project fields blank.
 * Click *Save*.
 
 To submit an updated patch:
 
 * Click *Differential*.
-* Click *Create Diff*.
-* Paste the updated diff.
+* Click *+ Create Diff*.
+* Paste the updated diff or browse to the updated patch file. Click *Create Diff*.
 * Select the review you want to from the *Attach To* dropdown and click
   *Continue*.
-* Click *Save*.
+* Leave the Repository and Project fields blank.
+* Add comments about the changes in the new diff. Click *Save*.
 
 Reviewing code with Phabricator
 -------------------------------
@@ -146,10 +148,20 @@ Submit button to finish closing the review.
 Status
 ------
 
-Please let us know whether you like it and what could be improved!
+Please let us know whether you like it and what could be improved! We're still
+working on setting up a bug tracker, but you can email klimek-at-google-dot-com
+and chandlerc-at-gmail-dot-com and CC the llvmdev mailing list with questions
+until then. We also could use help implementing improvements. This sadly is
+really painful and hard because the Phabricator codebase is in PHP and not as
+testable as you might like. However, we've put exactly what we're deploying up
+on an `llvm-reviews GitHub project`_ where folks can hack on it and post pull
+requests. We're looking into what the right long-term hosting for this is, but
+note that it is a derivative of an existing open source project, and so not
+trivially a good fit for an official LLVM project.
 
 .. _LLVM's Phabricator: http://reviews.llvm.org
 .. _`http://reviews.llvm.org`: http://reviews.llvm.org
 .. _Code Repository Browser: http://reviews.llvm.org/diffusion/
-.. _Arcanist Quick Start: http://www.phabricator.com/docs/phabricator/article/Arcanist_Quick_Start.html
-.. _Arcanist User Guide: http://www.phabricator.com/docs/phabricator/article/Arcanist_User_Guide.html
+.. _Arcanist Quick Start: https://secure.phabricator.com/book/phabricator/article/arcanist_quick_start/
+.. _Arcanist User Guide: https://secure.phabricator.com/book/phabricator/article/arcanist/
+.. _llvm-reviews GitHub project: https://github.com/r4nt/llvm-reviews/
diff --git a/docs/ProgrammersManual.rst b/docs/ProgrammersManual.rst
index 6a4c22a692d..08cc61a187b 100644
--- a/docs/ProgrammersManual.rst
+++ b/docs/ProgrammersManual.rst
@@ -1105,10 +1105,10 @@ If you have a set-like data structure that is usually small and whose elements
 are reasonably small, a ``SmallSet<Type, N>`` is a good choice.  This set has
 space for N elements in place (thus, if the set is dynamically smaller than N,
 no malloc traffic is required) and accesses them with a simple linear search.
-When the set grows beyond 'N' elements, it allocates a more expensive
+When the set grows beyond N elements, it allocates a more expensive
 representation that guarantees efficient access (for most types, it falls back
-to std::set, but for pointers it uses something far better, :ref:`SmallPtrSet
-<dss_smallptrset>`.
+to :ref:`std::set <dss_set>`, but for pointers it uses something far better,
+:ref:`SmallPtrSet <dss_smallptrset>`.
 
 The magic of this class is that it handles small sets extremely efficiently, but
 gracefully handles extremely large sets without loss of efficiency.  The
@@ -1120,16 +1120,31 @@ and erasing, but does not support iteration.
 llvm/ADT/SmallPtrSet.h
 ^^^^^^^^^^^^^^^^^^^^^^
 
-SmallPtrSet has all the advantages of ``SmallSet`` (and a ``SmallSet`` of
+``SmallPtrSet`` has all the advantages of ``SmallSet`` (and a ``SmallSet`` of
 pointers is transparently implemented with a ``SmallPtrSet``), but also supports
-iterators.  If more than 'N' insertions are performed, a single quadratically
+iterators.  If more than N insertions are performed, a single quadratically
 probed hash table is allocated and grows as needed, providing extremely
 efficient access (constant time insertion/deleting/queries with low constant
 factors) and is very stingy with malloc traffic.
 
-Note that, unlike ``std::set``, the iterators of ``SmallPtrSet`` are invalidated
-whenever an insertion occurs.  Also, the values visited by the iterators are not
-visited in sorted order.
+Note that, unlike :ref:`std::set <dss_set>`, the iterators of ``SmallPtrSet``
+are invalidated whenever an insertion occurs.  Also, the values visited by the
+iterators are not visited in sorted order.
+
+.. _dss_stringset:
+
+llvm/ADT/StringSet.h
+^^^^^^^^^^^^^^^^^^^^
+
+``StringSet`` is a thin wrapper around :ref:`StringMap\<char\> <dss_stringmap>`,
+and it allows efficient storage and retrieval of unique strings.
+
+Functionally analogous to ``SmallSet<StringRef>``, ``StringSet`` also suports
+iteration. (The iterator dereferences to a ``StringMapEntry<char>``, so you
+need to call ``i->getKey()`` to access the item of the StringSet.)  On the
+other hand, ``StringSet`` doesn't support range-insertion and
+copy-construction, which :ref:`SmallSet <dss_smallset>` and :ref:`SmallPtrSet
+<dss_smallptrset>` do support.
 
 .. _dss_denseset:
 
@@ -1297,8 +1312,9 @@ never use hash_set and unordered_set because they are generally very expensive
 (each insertion requires a malloc) and very non-portable.
 
 std::multiset is useful if you're not interested in elimination of duplicates,
-but has all the drawbacks of std::set.  A sorted vector (where you don't delete
-duplicate entries) or some other approach is almost always better.
+but has all the drawbacks of :ref:`std::set <dss_set>`.  A sorted vector
+(where you don't delete duplicate entries) or some other approach is almost
+always better.
 
 .. _ds_map:
 
@@ -1852,7 +1868,7 @@ Iterating over predecessors & successors of blocks
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
 Iterating over the predecessors and successors of a block is quite easy with the
-routines defined in ``"llvm/Support/CFG.h"``.  Just use code like this to
+routines defined in ``"llvm/IR/CFG.h"``.  Just use code like this to
 iterate over all predecessors of BB:
 
 .. code-block:: c++
diff --git a/docs/R600Usage.rst b/docs/R600Usage.rst
deleted file mode 100644
index 093cdd762b4..00000000000
--- a/docs/R600Usage.rst
+++ /dev/null
@@ -1,89 +0,0 @@
-============================
-User Guide for R600 Back-end
-============================
-
-Introduction
-============
-
-The R600 back-end provides ISA code generation for AMD GPUs, starting with
-the R600 family up until the current Volcanic Islands (GCN Gen 3).
-
-
-Assembler
-=========
-
-The assembler is currently considered experimental.
-
-For syntax examples look in test/MC/R600.
-
-Below some of the currently supported features (modulo bugs).  These
-all apply to the Southern Islands ISA, Sea Islands and Volcanic Islands
-are also supported but may be missing some instructions and have more bugs:
-
-DS Instructions
----------------
-All DS instructions are supported.
-
-MUBUF Instructions
-------------------
-All non-atomic MUBUF instructions are supported.
-
-SMRD Instructions
------------------
-Only the s_load_dword* SMRD instructions are supported.
-
-SOP1 Instructions
------------------
-All SOP1 instructions are supported.
-
-SOP2 Instructions
------------------
-All SOP2 instructions are supported.
-
-SOPC Instructions
------------------
-All SOPC instructions are supported.
-
-SOPP Instructions
------------------
-
-Unless otherwise mentioned, all SOPP instructions that have one or more
-operands accept integer operands only.  No verification is performed
-on the operands, so it is up to the programmer to be familiar with the
-range or acceptable values.
-
-s_waitcnt
-^^^^^^^^^
-
-s_waitcnt accepts named arguments to specify which memory counter(s) to
-wait for.
-
-.. code-block:: nasm
-
-   // Wait for all counters to be 0
-   s_waitcnt 0
-
-   // Equivalent to s_waitcnt 0.  Counter names can also be delimited by
-   // '&' or ','.
-   s_waitcnt vmcnt(0) expcnt(0) lgkcmt(0)
-
-   // Wait for vmcnt counter to be 1.
-   s_waitcnt vmcnt(1)
-
-VOP1, VOP2, VOP3, VOPC Instructions
------------------------------------
-
-All 32-bit and 64-bit encodings should work.
-
-The assembler will automatically detect which encoding size to use for
-VOP1, VOP2, and VOPC instructions based on the operands.  If you want to force
-a specific encoding size, you can add an _e32 (for 32-bit encoding) or
-_e64 (for 64-bit encoding) suffix to the instruction.  Most, but not all
-instructions support an explicit suffix.  These are all valid assembly
-strings:
-
-.. code-block:: nasm
-
-   v_mul_i32_i24 v1, v2, v3
-   v_mul_i32_i24_e32 v1, v2, v3
-   v_mul_i32_i24_e64 v1, v2, v3
diff --git a/docs/ReleaseNotes.rst b/docs/ReleaseNotes.rst
index c0d2ea18981..17c5c069d5f 100644
--- a/docs/ReleaseNotes.rst
+++ b/docs/ReleaseNotes.rst
@@ -1,12 +1,12 @@
 ======================
-LLVM 3.7 Release Notes
+LLVM 3.8 Release Notes
 ======================
 
 .. contents::
     :local:
 
 .. warning::
-   These are in-progress notes for the upcoming LLVM 3.7 release.  You may
+   These are in-progress notes for the upcoming LLVM 3.8 release.  You may
    prefer the `LLVM 3.6 Release Notes <http://llvm.org/releases/3.6.0/docs
    /ReleaseNotes.html>`_.
 
@@ -15,7 +15,7 @@ Introduction
 ============
 
 This document contains the release notes for the LLVM Compiler Infrastructure,
-release 3.7.  Here we describe the status of LLVM, including major improvements
+release 3.8.  Here we describe the status of LLVM, including major improvements
 from the previous release, improvements in various subprojects of LLVM, and
 some of the current users of the code.  All LLVM releases may be downloaded
 from the `LLVM releases web site <http://llvm.org/releases/>`_.
@@ -41,18 +41,6 @@ Non-comprehensive list of changes in this release
    functionality, or simply have a lot to talk about), see the `NOTE` below
    for adding a new subsection.
 
-* The minimum required Visual Studio version for building LLVM is now 2013
-  Update 4.
-
-* A new documentation page, :doc:`Frontend/PerformanceTips`, contains a
-  collection of tips for frontend authors on how to generate IR which LLVM is
-  able to effectively optimize.
-
-* The DataLayout is no longer optional. All the IR level optimizations expects
-  it to be present and the API has been changed to use a reference instead of
-  a pointer to make it explicit. The Module owns the datalayout and it has to
-  match the one attached to the TargetMachine for generating code.
-
 * ... next change ...
 
 .. NOTE
@@ -89,12 +77,12 @@ Changes to the OCaml bindings
  During this release ...
 
 
-External Open Source Projects Using LLVM 3.7
+External Open Source Projects Using LLVM 3.8
 ============================================
 
 An exciting aspect of LLVM is that it is used as an enabling technology for
 a lot of other language and tools projects. This section lists some of the
-projects that have already been updated to work with LLVM 3.7.
+projects that have already been updated to work with LLVM 3.8.
 
 * A project
 
diff --git a/docs/SourceLevelDebugging.rst b/docs/SourceLevelDebugging.rst
index 55e12d3efa0..99186f58188 100644
--- a/docs/SourceLevelDebugging.rst
+++ b/docs/SourceLevelDebugging.rst
@@ -153,8 +153,8 @@ debugger to interpret the information.
 To provide basic functionality, the LLVM debugger does have to make some
 assumptions about the source-level language being debugged, though it keeps
 these to a minimum.  The only common features that the LLVM debugger assumes
-exist are `source files <LangRef.html#MDFile>`_, and `program objects
-<LangRef.html#MDGlobalVariable>`_.  These abstract objects are used by a
+exist are `source files <LangRef.html#difile>`_, and `program objects
+<LangRef.html#diglobalvariable>`_.  These abstract objects are used by a
 debugger to form stack traces, show information about local variables, etc.
 
 This section of the documentation first describes the representation aspects
@@ -177,27 +177,27 @@ provide debug information at various points in generated code.
 
 .. code-block:: llvm
 
-  void %llvm.dbg.declare(metadata, metadata, metadata)
+  void @llvm.dbg.declare(metadata, metadata, metadata)
 
 This intrinsic provides information about a local element (e.g., variable).
 The first argument is metadata holding the alloca for the variable.  The second
-argument is a `local variable <LangRef.html#MDLocalVariable>`_ containing a
+argument is a `local variable <LangRef.html#dilocalvariable>`_ containing a
 description of the variable.  The third argument is a `complex expression
-<LangRef.html#MDExpression>`_.
+<LangRef.html#diexpression>`_.
 
 ``llvm.dbg.value``
 ^^^^^^^^^^^^^^^^^^
 
 .. code-block:: llvm
 
-  void %llvm.dbg.value(metadata, i64, metadata, metadata)
+  void @llvm.dbg.value(metadata, i64, metadata, metadata)
 
 This intrinsic provides information when a user source variable is set to a new
 value.  The first argument is the new value (wrapped as metadata).  The second
 argument is the offset in the user source variable where the new value is
 written.  The third argument is a `local variable
-<LangRef.html#MDLocalVariable>`_ containing a description of the variable.  The
-third argument is a `complex expression <LangRef.html#MDExpression>`_.
+<LangRef.html#dilocalvariable>`_ containing a description of the variable.  The
+third argument is a `complex expression <LangRef.html#diexpression>`_.
 
 Object lifetimes and scoping
 ============================
@@ -259,31 +259,31 @@ Compiled to LLVM, this function would be represented like this:
   !llvm.module.flags = !{!7, !8, !9}
   !llvm.ident = !{!10}
 
-  !0 = !MDCompileUnit(language: DW_LANG_C99, file: !1, producer: "clang version 3.7.0 (trunk 231150) (llvm/trunk 231154)", isOptimized: false, runtimeVersion: 0, emissionKind: 1, enums: !2, retainedTypes: !2, subprograms: !3, globals: !2, imports: !2)
-  !1 = !MDFile(filename: "/dev/stdin", directory: "/Users/dexonsmith/data/llvm/debug-info")
+  !0 = !DICompileUnit(language: DW_LANG_C99, file: !1, producer: "clang version 3.7.0 (trunk 231150) (llvm/trunk 231154)", isOptimized: false, runtimeVersion: 0, emissionKind: 1, enums: !2, retainedTypes: !2, subprograms: !3, globals: !2, imports: !2)
+  !1 = !DIFile(filename: "/dev/stdin", directory: "/Users/dexonsmith/data/llvm/debug-info")
   !2 = !{}
   !3 = !{!4}
-  !4 = !MDSubprogram(name: "foo", scope: !1, file: !1, line: 1, type: !5, isLocal: false, isDefinition: true, scopeLine: 1, isOptimized: false, function: void ()* @foo, variables: !2)
-  !5 = !MDSubroutineType(types: !6)
+  !4 = !DISubprogram(name: "foo", scope: !1, file: !1, line: 1, type: !5, isLocal: false, isDefinition: true, scopeLine: 1, isOptimized: false, function: void ()* @foo, variables: !2)
+  !5 = !DISubroutineType(types: !6)
   !6 = !{null}
   !7 = !{i32 2, !"Dwarf Version", i32 2}
   !8 = !{i32 2, !"Debug Info Version", i32 3}
   !9 = !{i32 1, !"PIC Level", i32 2}
   !10 = !{!"clang version 3.7.0 (trunk 231150) (llvm/trunk 231154)"}
-  !11 = !MDLocalVariable(tag: DW_TAG_auto_variable, name: "X", scope: !4, file: !1, line: 2, type: !12)
-  !12 = !MDBasicType(name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
-  !13 = !MDExpression()
-  !14 = !MDLocation(line: 2, column: 9, scope: !4)
-  !15 = !MDLocalVariable(tag: DW_TAG_auto_variable, name: "Y", scope: !4, file: !1, line: 3, type: !12)
-  !16 = !MDLocation(line: 3, column: 9, scope: !4)
-  !17 = !MDLocalVariable(tag: DW_TAG_auto_variable, name: "Z", scope: !18, file: !1, line: 5, type: !12)
-  !18 = distinct !MDLexicalBlock(scope: !4, file: !1, line: 4, column: 5)
-  !19 = !MDLocation(line: 5, column: 11, scope: !18)
-  !20 = !MDLocation(line: 6, column: 11, scope: !18)
-  !21 = !MDLocation(line: 6, column: 9, scope: !18)
-  !22 = !MDLocation(line: 8, column: 9, scope: !4)
-  !23 = !MDLocation(line: 8, column: 7, scope: !4)
-  !24 = !MDLocation(line: 9, column: 3, scope: !4)
+  !11 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "X", scope: !4, file: !1, line: 2, type: !12)
+  !12 = !DIBasicType(name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
+  !13 = !DIExpression()
+  !14 = !DILocation(line: 2, column: 9, scope: !4)
+  !15 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "Y", scope: !4, file: !1, line: 3, type: !12)
+  !16 = !DILocation(line: 3, column: 9, scope: !4)
+  !17 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "Z", scope: !18, file: !1, line: 5, type: !12)
+  !18 = distinct !DILexicalBlock(scope: !4, file: !1, line: 4, column: 5)
+  !19 = !DILocation(line: 5, column: 11, scope: !18)
+  !20 = !DILocation(line: 6, column: 11, scope: !18)
+  !21 = !DILocation(line: 6, column: 9, scope: !18)
+  !22 = !DILocation(line: 8, column: 9, scope: !4)
+  !23 = !DILocation(line: 8, column: 7, scope: !4)
+  !24 = !DILocation(line: 9, column: 3, scope: !4)
 
 
 This example illustrates a few important details about LLVM debugging
@@ -303,15 +303,15 @@ scope information for the variable ``X``.
 
 .. code-block:: llvm
 
-  !14 = !MDLocation(line: 2, column: 9, scope: !4)
-  !4 = !MDSubprogram(name: "foo", scope: !1, file: !1, line: 1, type: !5,
+  !14 = !DILocation(line: 2, column: 9, scope: !4)
+  !4 = !DISubprogram(name: "foo", scope: !1, file: !1, line: 1, type: !5,
                      isLocal: false, isDefinition: true, scopeLine: 1,
                      isOptimized: false, function: void ()* @foo,
                      variables: !2)
 
 Here ``!14`` is metadata providing `location information
-<LangRef.html#MDLocation>`_.  In this example, scope is encoded by ``!4``, a
-`subprogram descriptor <LangRef.html#MDSubprogram>`_.  This way the location
+<LangRef.html#dilocation>`_.  In this example, scope is encoded by ``!4``, a
+`subprogram descriptor <LangRef.html#disubprogram>`_.  This way the location
 information attached to the intrinsics indicates that the variable ``X`` is
 declared at line number 2 at a function level scope in function ``foo``.
 
@@ -328,8 +328,8 @@ scope information for the variable ``Z``.
 
 .. code-block:: llvm
 
-  !18 = distinct !MDLexicalBlock(scope: !4, file: !1, line: 4, column: 5)
-  !19 = !MDLocation(line: 5, column: 11, scope: !18)
+  !18 = distinct !DILexicalBlock(scope: !4, file: !1, line: 4, column: 5)
+  !19 = !DILocation(line: 5, column: 11, scope: !18)
 
 Here ``!19`` indicates that ``Z`` is declared at line number 5 and column
 number 0 inside of lexical scope ``!18``.  The lexical scope itself resides
@@ -406,7 +406,7 @@ a C/C++ front-end would generate the following descriptors:
   !llvm.module.flags = !{!6, !7}
 
   ;; Define the compile unit.
-  !0 = !MDCompileUnit(language: DW_LANG_C99, file: !1,
+  !0 = !DICompileUnit(language: DW_LANG_C99, file: !1,
                       producer:
                       "clang version 3.7.0 (trunk 231150) (llvm/trunk 231154)",
                       isOptimized: false, runtimeVersion: 0, emissionKind: 1,
@@ -416,7 +416,7 @@ a C/C++ front-end would generate the following descriptors:
   ;;
   ;; Define the file
   ;;
-  !1 = !MDFile(filename: "/dev/stdin",
+  !1 = !DIFile(filename: "/dev/stdin",
                directory: "/Users/dexonsmith/data/llvm/debug-info")
 
   ;; An empty array.
@@ -428,14 +428,14 @@ a C/C++ front-end would generate the following descriptors:
   ;;
   ;; Define the global variable itself.
   ;;
-  !4 = !MDGlobalVariable(name: "MyGlobal", scope: !0, file: !1, line: 1,
+  !4 = !DIGlobalVariable(name: "MyGlobal", scope: !0, file: !1, line: 1,
                          type: !5, isLocal: false, isDefinition: true,
                          variable: i32* @MyGlobal)
 
   ;;
   ;; Define the type
   ;;
-  !5 = !MDBasicType(name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
+  !5 = !DIBasicType(name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
 
   ;; Dwarf version to output.
   !6 = !{i32 2, !"Dwarf Version", i32 2}
@@ -461,7 +461,7 @@ a C/C++ front-end would generate the following descriptors:
   ;;
   ;; Define the anchor for subprograms.
   ;;
-  !4 = !MDSubprogram(name: "main", scope: !1, file: !1, line: 1, type: !5,
+  !4 = !DISubprogram(name: "main", scope: !1, file: !1, line: 1, type: !5,
                      isLocal: false, isDefinition: true, scopeLine: 1,
                      flags: DIFlagPrototyped, isOptimized: false,
                      function: i32 (i32, i8**)* @main, variables: !2)
diff --git a/docs/StackMaps.rst b/docs/StackMaps.rst
index 43c60c9e785..5bdae38b699 100644
--- a/docs/StackMaps.rst
+++ b/docs/StackMaps.rst
@@ -221,12 +221,10 @@ lowered according to the calling convention specified at the
 intrinsic's callsite. Variants of the intrinsic with non-void return
 type also return a value according to calling convention.
 
-On PowerPC, note that ``<target>`` must be the actual intended target of
-the indirect call. Specifically, even when compiling for the ELF V1 ABI,
-``<target>`` is not the function-descriptor address normally used as the C/C++
-function-pointer representation. As a result, the call target must be local
-because no adjustment or restoration of the TOC pointer (in register r2) will
-be performed.
+On PowerPC, note that ``<target>`` must be the ABI function pointer for the
+intended target of the indirect call. Specifically, when compiling for the
+ELF V1 ABI, ``<target>`` is the function-descriptor address normally used as
+the C/C++ function-pointer representation.
 
 Requesting zero patch point arguments is valid. In this case, all
 variable operands are handled just like
@@ -501,3 +499,13 @@ the same requirement imposed by the llvm.gcroot intrinsic.) LLVM
 transformations must not substitute the alloca with any intervening
 value. This can be verified by the runtime simply by checking that the
 stack map's location is a Direct location type.
+
+
+Supported Architectures
+=======================
+
+Support for StackMap generation and the related intrinsics requires 
+some code for each backend.  Today, only a subset of LLVM's backends 
+are supported.  The currently supported architectures are X86_64, 
+PowerPC, and Aarch64.
+
diff --git a/docs/Statepoints.rst b/docs/Statepoints.rst
index 9741c93cc81..2b01374a9ec 100644
--- a/docs/Statepoints.rst
+++ b/docs/Statepoints.rst
@@ -142,8 +142,8 @@ resulting relocation sequence is:
 
   define i8 addrspace(1)* @test1(i8 addrspace(1)* %obj) 
          gc "statepoint-example" {
-    %0 = call i32 (void ()*, i32, i32, ...)* @llvm.experimental.gc.statepoint.p0f_isVoidf(void ()* @foo, i32 0, i32 0, i32 5, i32 0, i32 -1, i32 0, i32 0, i32 0, i8 addrspace(1)* %obj)
-    %obj.relocated = call coldcc i8 addrspace(1)* @llvm.experimental.gc.relocate.p1i8(i32 %0, i32 9, i32 9)
+    %0 = call i32 (i64, i32, void ()*, i32, i32, ...)* @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 0, i32 0, void ()* @foo, i32 0, i32 0, i32 0, i32 0, i8 addrspace(1)* %obj)
+    %obj.relocated = call coldcc i8 addrspace(1)* @llvm.experimental.gc.relocate.p1i8(i32 %0, i32 7, i32 7)
     ret i8 addrspace(1)* %obj.relocated
   }
 
@@ -207,7 +207,82 @@ This example was taken from the tests for the :ref:`RewriteStatepointsForGC` uti
   opt -rewrite-statepoints-for-gc test/Transforms/RewriteStatepointsForGC/basics.ll -S | llc -debug-only=stackmaps
 
 
+GC Transitions
+^^^^^^^^^^^^^^^^^^
+
+As a practical consideration, many garbage-collected systems allow code that is
+collector-aware ("managed code") to call code that is not collector-aware
+("unmanaged code"). It is common that such calls must also be safepoints, since
+it is desirable to allow the collector to run during the execution of
+unmanaged code. Futhermore, it is common that coordinating the transition from
+managed to unmanaged code requires extra code generation at the call site to
+inform the collector of the transition. In order to support these needs, a
+statepoint may be marked as a GC transition, and data that is necessary to
+perform the transition (if any) may be provided as additional arguments to the
+statepoint.
+
+  Note that although in many cases statepoints may be inferred to be GC
+  transitions based on the function symbols involved (e.g. a call from a
+  function with GC strategy "foo" to a function with GC strategy "bar"),
+  indirect calls that are also GC transitions must also be supported. This
+  requirement is the driving force behing the decision to require that GC
+  transitions are explicitly marked.
+
+Let's revisit the sample given above, this time treating the call to ``@foo``
+as a GC transition. Depending on our target, the transition code may need to
+access some extra state in order to inform the collector of the transition.
+Let's assume a hypothetical GC--somewhat unimaginatively named "hypothetical-gc"
+--that requires that a TLS variable must be written to before and after a call
+to unmanaged code. The resulting relocation sequence is:
 
+.. code-block:: llvm
+
+  @flag = thread_local global i32 0, align 4
+
+  define i8 addrspace(1)* @test1(i8 addrspace(1) *%obj)
+         gc "hypothetical-gc" {
+
+    %0 = call i32 (i64, i32, void ()*, i32, i32, ...)* @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 0, i32 0, void ()* @foo, i32 0, i32 1, i32* @Flag, i32 0, i8 addrspace(1)* %obj)
+    %obj.relocated = call coldcc i8 addrspace(1)* @llvm.experimental.gc.relocate.p1i8(i32 %0, i32 7, i32 7)
+    ret i8 addrspace(1)* %obj.relocated
+  }
+
+During lowering, this will result in a instruction selection DAG that looks
+something like:
+
+::
+
+  CALLSEQ_START
+  ...
+  GC_TRANSITION_START (lowered i32 *@Flag), SRCVALUE i32* Flag
+  STATEPOINT
+  GC_TRANSITION_END (lowered i32 *@Flag), SRCVALUE i32 *Flag
+  ...
+  CALLSEQ_END
+
+In order to generate the necessary transition code, the backend for each target
+supported by "hypothetical-gc" must be modified to lower ``GC_TRANSITION_START``
+and ``GC_TRANSITION_END`` nodes appropriately when the "hypothetical-gc"
+strategy is in use for a particular function. Assuming that such lowering has
+been added for X86, the generated assembly would be:
+
+.. code-block:: gas
+
+	  .globl	test1
+	  .align	16, 0x90
+	  pushq	%rax
+	  movl $1, %fs:Flag@TPOFF
+	  callq	foo
+	  movl $0, %fs:Flag@TPOFF
+  .Ltmp1:
+	  movq	(%rsp), %rax  # This load is redundant (oops!)
+	  popq	%rdx
+	  retq
+
+Note that the design as presented above is not fully implemented: in particular,
+strategy-specific lowering is not present, and all GC transitions are emitted as
+as single no-op before and after the call instruction. These no-ops are often
+removed by the backend during dead machine instruction elimination.
 
 
 Intrinsics
@@ -222,9 +297,11 @@ Syntax:
 ::
 
       declare i32
-        @llvm.experimental.gc.statepoint(func_type <target>, 
-                       i64 <#call args>. i64 <unused>, 
+        @llvm.experimental.gc.statepoint(i64 <id>, i32 <num patch bytes>,
+                       func_type <target>, 
+                       i64 <#call args>, i64 <flags>,
                        ... (call parameters),
+                       i64 <# transition args>, ... (transition parameters),
                        i64 <# deopt args>, ... (deopt parameters),
                        ... (gc parameters))
 
@@ -237,6 +314,28 @@ runtime.
 Operands:
 """""""""
 
+The 'id' operand is a constant integer that is reported as the ID
+field in the generated stackmap.  LLVM does not interpret this
+parameter in any way and its meaning is up to the statepoint user to
+decide.  Note that LLVM is free to duplicate code containing
+statepoint calls, and this may transform IR that had a unique 'id' per
+lexical call to statepoint to IR that does not.
+
+If 'num patch bytes' is non-zero then the call instruction
+corresponding to the statepoint is not emitted and LLVM emits 'num
+patch bytes' bytes of nops in its place.  LLVM will emit code to
+prepare the function arguments and retrieve the function return value
+in accordance to the calling convention; the former before the nop
+sequence and the latter after the nop sequence.  It is expected that
+the user will patch over the 'num patch bytes' bytes of nops with a
+calling sequence specific to their runtime before executing the
+generated machine code.  There are no guarantees with respect to the
+alignment of the nop sequence.  Unlike :doc:`StackMaps` statepoints do
+not have a concept of shadow bytes.  Note that semantically the
+statepoint still represents a call or invoke to 'target', and the nop
+sequence after patching is expected to represent an operation
+equivalent to a call or invoke to 'target'.
+
 The 'target' operand is the function actually being called.  The
 target can be specified as either a symbolic LLVM function, or as an
 arbitrary Value of appropriate function type.  Note that the function
@@ -247,8 +346,19 @@ The '#call args' operand is the number of arguments to the actual
 call.  It must exactly match the number of arguments passed in the
 'call parameters' variable length section.
 
-The 'unused' operand is unused and likely to be removed.  Please do
-not use.
+The 'flags' operand is used to specify extra information about the
+statepoint. This is currently only used to mark certain statepoints
+as GC transitions. This operand is a 64-bit integer with the following
+layout, where bit 0 is the least significant bit:
+
+  +-------+---------------------------------------------------+
+  | Bit # | Usage                                             |
+  +=======+===================================================+
+  |     0 | Set if the statepoint is a GC transition, cleared |
+  |       | otherwise.                                        |
+  +-------+---------------------------------------------------+
+  |  1-63 | Reserved for future use; must be cleared.         |
+  +-------+---------------------------------------------------+
 
 The 'call parameters' arguments are simply the arguments which need to
 be passed to the call target.  They will be lowered according to the
@@ -257,6 +367,14 @@ instruction.  The number of arguments must exactly match what is
 specified in '# call args'.  The types must match the signature of
 'target'.
 
+The 'transition parameters' arguments contain an arbitrary list of
+Values which need to be passed to GC transition code. They will be
+lowered and passed as operands to the appropriate GC_TRANSITION nodes
+in the selection DAG. It is assumed that these arguments must be
+available before and after (but not necessarily during) the execution
+of the callee. The '# transition args' field indicates how many operands
+are to be interpreted as 'transition parameters'.
+
 The 'deopt parameters' arguments contain an arbitrary list of Values
 which is meaningful to the runtime.  The runtime may read any of these
 values, but is assumed not to modify them.  If the garbage collector
@@ -388,6 +506,12 @@ the runtime or collector are provided via the :ref:`Stack Map format
 
 Each statepoint generates the following Locations:
 
+* Constant which describes the calling convention of the call target. This
+  constant is a valid :ref:`calling convention identifier <callingconv>` for
+  the version of LLVM used to generate the stackmap. No additional compatibility
+  guarantees are made for this constant over what LLVM provides elsewhere w.r.t.
+  these identifiers.
+* Constant which describes the flags passed to the statepoint intrinsic
 * Constant which describes number of following deopt *Locations* (not
   operands)
 * Variable number of Locations, one for each deopt parameter listed in
@@ -405,9 +529,6 @@ Note that the Locations used in each section may describe the same
 physical location.  e.g. A stack slot may appear as a deopt location,
 a gc base pointer, and a gc derived pointer.
 
-The ID field of the 'StkMapRecord' for a statepoint is meaningless and
-it's value is explicitly unspecified.
-
 The LiveOut section of the StkMapRecord will be empty for a statepoint
 record.
 
@@ -471,7 +592,7 @@ As an example, given this code:
 
   define i8 addrspace(1)* @test1(i8 addrspace(1)* %obj) 
          gc "statepoint-example" {
-    call i32 (void ()*, i32, i32, ...)* @llvm.experimental.gc.statepoint.p0f_isVoidf(void ()* @foo, i32 0, i32 0, i32 5, i32 0, i32 -1, i32 0, i32 0, i32 0)
+    call i32 (i64, i32, void ()*, i32, i32, ...)* @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 2882400000, i32 0, void ()* @foo, i32 0, i32 0, i32 0, i32 5, i32 0, i32 -1, i32 0, i32 0, i32 0)
     ret i8 addrspace(1)* %obj
   }
 
@@ -481,8 +602,8 @@ The pass would produce this IR:
 
   define i8 addrspace(1)* @test1(i8 addrspace(1)* %obj) 
          gc "statepoint-example" {
-    %0 = call i32 (void ()*, i32, i32, ...)* @llvm.experimental.gc.statepoint.p0f_isVoidf(void ()* @foo, i32 0, i32 0, i32 5, i32 0, i32 -1, i32 0, i32 0, i32 0, i8 addrspace(1)* %obj)
-    %obj.relocated = call coldcc i8 addrspace(1)* @llvm.experimental.gc.relocate.p1i8(i32 %0, i32 9, i32 9)
+    %0 = call i32 (i64, i32, void ()*, i32, i32, ...)* @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 2882400000, i32 0, void ()* @foo, i32 0, i32 0, i32 0, i32 5, i32 0, i32 -1, i32 0, i32 0, i32 0, i8 addrspace(1)* %obj)
+    %obj.relocated = call coldcc i8 addrspace(1)* @llvm.experimental.gc.relocate.p1i8(i32 %0, i32 12, i32 12)
     ret i8 addrspace(1)* %obj.relocated
   }
 
@@ -535,8 +656,8 @@ This pass would produce the following IR:
 .. code-block:: llvm
 
   define void @test() gc "statepoint-example" {
-    %safepoint_token = call i32 (void ()*, i32, i32, ...)* @llvm.experimental.gc.statepoint.p0f_isVoidf(void ()* @do_safepoint, i32 0, i32 0, i32 0)
-    %safepoint_token1 = call i32 (void ()*, i32, i32, ...)* @llvm.experimental.gc.statepoint.p0f_isVoidf(void ()* @foo, i32 0, i32 0, i32 0)
+    %safepoint_token = call i32 (i64, i32, void ()*, i32, i32, ...)* @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 2882400000, i32 0, void ()* @do_safepoint, i32 0, i32 0, i32 0, i32 0)
+    %safepoint_token1 = call i32 (i64, i32, void ()*, i32, i32, ...)* @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 2882400000, i32 0, void ()* @foo, i32 0, i32 0, i32 0, i32 0)
     ret void
   }
 
@@ -558,6 +679,21 @@ of this function is inserted at each poll site desired.  While calls or invokes
 inside this method are transformed to a ``gc.statepoints``, recursive poll 
 insertion is not performed.
 
+By default PlaceSafepoints passes in ``0xABCDEF00`` as the statepoint
+ID and ``0`` as the number of patchable bytes to the newly constructed
+``gc.statepoint``.  These values can be configured on a per-callsite
+basis using the attributes ``"statepoint-id"`` and
+``"statepoint-num-patch-bytes"``.  If a call site is marked with a
+``"statepoint-id"`` function attribute and its value is a positive
+integer (represented as a string), then that value is used as the ID
+of the newly constructed ``gc.statepoint``.  If a call site is marked
+with a ``"statepoint-num-patch-bytes"`` function attribute and its
+value is a positive integer, then that value is used as the 'num patch
+bytes' parameter of the newly constructed ``gc.statepoint``.  The
+``"statepoint-id"`` and ``"statepoint-num-patch-bytes"`` attributes
+are not propagated to the ``gc.statepoint`` call or invoke if they
+could be successfully parsed.
+
 If you are scheduling the RewriteStatepointsForGC pass late in the pass order,
 you should probably schedule this pass immediately before it.  The exception 
 would be if you need to preserve abstract frame information (e.g. for
@@ -565,6 +701,12 @@ deoptimization or introspection) at safepoints.  In that case, ask on the
 llvmdev mailing list for suggestions.
 
 
+Supported Architectures
+=======================
+
+Support for statepoint generation requires some code for each backend.
+Today, only X86_64 is supported.  
+
 Bugs and Enhancements
 =====================
 
diff --git a/docs/TableGen/LangIntro.rst b/docs/TableGen/LangIntro.rst
index 85c74a5b460..4d4551e8a6e 100644
--- a/docs/TableGen/LangIntro.rst
+++ b/docs/TableGen/LangIntro.rst
@@ -190,7 +190,7 @@ supported include:
     for 'a' in 'c.'  This operation is analogous to $(subst) in GNU make.
 
 ``!foreach(a, b, c)``
-    For each member 'b' of dag or list 'a' apply operator 'c.'  'b' is a dummy
+    For each member of dag or list 'b' apply operator 'c.'  'a' is a dummy
     variable that should be declared as a member variable of an instantiated
     class.  This operation is analogous to $(foreach) in GNU make.
 
diff --git a/docs/TestingGuide.rst b/docs/TestingGuide.rst
index 3463156495a..adf5f3d4cfb 100644
--- a/docs/TestingGuide.rst
+++ b/docs/TestingGuide.rst
@@ -473,6 +473,25 @@ RUN lines:
 To add more substituations, look at ``test/lit.cfg`` or ``lit.local.cfg``.
 
 
+Options
+-------
+
+The llvm lit configuration allows to customize some things with user options:
+
+``llc``, ``opt``, ...
+    Substitute the respective llvm tool name with a custom command line. This
+    allows to specify custom paths and default arguments for these tools.
+    Example:
+
+    % llvm-lit "-Dllc=llc -verify-machineinstrs"
+
+``run_long_tests``
+    Enable the execution of long running tests.
+
+``llvm_site_config``
+    Load the specified lit configuration instead of the default one.
+
+
 Other Features
 --------------
 
diff --git a/docs/YamlIO.rst b/docs/YamlIO.rst
index 3cc683b6b82..f0baeb4c69d 100644
--- a/docs/YamlIO.rst
+++ b/docs/YamlIO.rst
@@ -456,10 +456,10 @@ looks like:
 
     template <>
     struct ScalarTraits<MyCustomType> {
-      static void output(const T &value, llvm::raw_ostream &out) {
+      static void output(const T &value, void*, llvm::raw_ostream &out) {
         out << value;  // do custom formatting here
       }
-      static StringRef input(StringRef scalar, T &value) {
+      static StringRef input(StringRef scalar, void*, T &value) {
         // do custom parsing here.  Return the empty string on success,
         // or an error message on failure.
         return StringRef();
@@ -467,6 +467,56 @@ looks like:
       // Determine if this scalar needs quotes.
       static bool mustQuote(StringRef) { return true; }
     };
+
+Block Scalars
+-------------
+
+YAML block scalars are string literals that are represented in YAML using the
+literal block notation, just like the example shown below:
+
+.. code-block:: yaml
+
+    text: |
+      First line
+      Second line
+
+The YAML I/O library provides support for translating between YAML block scalars
+and specific C++ types by allowing you to specialize BlockScalarTraits<> on
+your data type. The library doesn't provide any built-in support for block
+scalar I/O for types like std::string and llvm::StringRef as they are already
+supported by YAML I/O and use the ordinary scalar notation by default.
+
+BlockScalarTraits specializations are very similar to the
+ScalarTraits specialization - YAML I/O will provide the native type and your
+specialization must create a temporary llvm::StringRef when writing, and
+it will also provide an llvm::StringRef that has the value of that block scalar
+and your specialization must convert that to your native data type when reading.
+An example of a custom type with an appropriate specialization of
+BlockScalarTraits is shown below:
+
+.. code-block:: c++
+
+    using llvm::yaml::BlockScalarTraits;
+    using llvm::yaml::IO;
+
+    struct MyStringType {
+      std::string Str;
+    };
+
+    template <>
+    struct BlockScalarTraits<MyStringType> {
+      static void output(const MyStringType &Value, void *Ctxt,
+                         llvm::raw_ostream &OS) {
+        OS << Value.Str;
+      }
+
+      static StringRef input(StringRef Scalar, void *Ctxt,
+                             MyStringType &Value) {
+        Value.Str = Scalar.str();
+        return StringRef();
+      }
+    };
+
     
 
 Mappings
@@ -723,6 +773,33 @@ because it is a programming error to have invalid struct values.
       }
     };
 
+Flow Mapping
+------------
+A YAML "flow mapping" is a mapping that uses the inline notation
+(e.g { x: 1, y: 0 } ) when written to YAML. To specify that a type should be
+written in YAML using flow mapping, your MappingTraits specialization should
+add "static const bool flow = true;". For instance:
+
+.. code-block:: c++
+
+    using llvm::yaml::MappingTraits;
+    using llvm::yaml::IO;
+
+    struct Stuff {
+      ...
+    };
+
+    template <>
+    struct MappingTraits<Stuff> {
+      static void mapping(IO &io, Stuff &stuff) {
+        ...
+      }
+
+      static const bool flow = true;
+    }
+
+Flow mappings are subject to line wrapping according to the Output object
+configuration.
 
 Sequence
 ========
@@ -770,6 +847,8 @@ With the above, if you used MyList as the data type in your native data
 structures, then when converted to YAML, a flow sequence of integers 
 will be used (e.g. [ 10, -3, 4 ]).
 
+Flow sequences are subject to line wrapping according to the Output object
+configuration.
 
 Utility Macros
 --------------
@@ -833,14 +912,14 @@ Output
 
 The llvm::yaml::Output class is used to generate a YAML document from your 
 in-memory data structures, using traits defined on your data types.  
-To instantiate an Output object you need an llvm::raw_ostream, and optionally 
-a context pointer:
+To instantiate an Output object you need an llvm::raw_ostream, an optional 
+context pointer and an optional wrapping column:
 
 .. code-block:: c++
 
       class Output : public IO {
       public:
-        Output(llvm::raw_ostream &, void *context=NULL);
+        Output(llvm::raw_ostream &, void *context = NULL, int WrapColumn = 70);
     
 Once you have an Output object, you can use the C++ stream operator on it
 to write your native data as YAML. One thing to recall is that a YAML file
@@ -849,6 +928,10 @@ streaming as YAML is a mapping, scalar, or sequence, then Output assumes you
 are generating one document and wraps the mapping output 
 with  "``---``" and trailing "``...``".  
 
+The WrapColumn parameter will cause the flow mappings and sequences to
+line-wrap when they go over the supplied column. Pass 0 to completely
+suppress the wrapping.
+
 .. code-block:: c++
    
     using llvm::yaml::Output;
diff --git a/docs/_ocamldoc/style.css b/docs/_ocamldoc/style.css
new file mode 100644
index 00000000000..00595d7f2f2
--- /dev/null
+++ b/docs/_ocamldoc/style.css
@@ -0,0 +1,97 @@
+/* A style for ocamldoc. Daniel C. Buenzli */
+
+/* Reset a few things. */
+html,body,div,span,applet,object,iframe,h1,h2,h3,h4,h5,h6,p,blockquote,pre,
+a,abbr,acronym,address,big,cite,code,del,dfn,em,font,img,ins,kbd,q,s,samp,
+small,strike,strong,sub,sup,tt,var,b,u,i,center,dl,dt,dd,ol,ul,li,fieldset,
+form,label,legend,table,caption,tbody,tfoot,thead,tr,th,td
+{ margin: 0; padding: 0; border: 0 none; outline: 0; font-size: 100%;
+  font-weight: inherit; font-style:inherit; font-family:inherit;
+  line-height: inherit; vertical-align: baseline; text-align:inherit;
+  color:inherit; background: transparent; }
+
+table { border-collapse: collapse; border-spacing: 0; }
+
+/* Basic page layout */
+
+body { font: normal 10pt/1.375em helvetica, arial, sans-serif; text-align:left;
+       margin: 1.375em 10%; min-width: 40ex; max-width: 72ex;
+       color: black; background: transparent /* url(line-height-22.gif) */; }
+
+b { font-weight: bold }
+em { font-style: italic }
+
+tt, code, pre { font-family: WorkAroundWebKitAndMozilla, monospace;
+                font-size: 1em; }
+pre code { font-size : inherit; }
+.codepre { margin-bottom:1.375em /* after code example we introduce space. */ }
+
+.superscript,.subscript
+{ font-size : 0.813em; line-height:0; margin-left:0.4ex;}
+.superscript { vertical-align: super; }
+.subscript { vertical-align: sub; }
+
+/* ocamldoc markup workaround hacks */
+
+
+
+hr, hr + br, div + br, center + br, span + br, ul + br, ol + br, pre + br
+{ display: none } /* annoying */
+
+div.info + br { display:block}
+
+.codepre br + br { display: none }
+h1 + pre { margin-bottom:1.375em} /* Toplevel module description */
+
+/* Sections and document divisions */
+
+/* .navbar { margin-bottom: -1.375em } */
+h1 { font-weight: bold; font-size: 1.5em; /* margin-top:1.833em; */
+     margin-top:0.917em; padding-top:0.875em;
+     border-top-style:solid; border-width:1px; border-color:#AAA; }
+h2 { font-weight: bold; font-size: 1.313em; margin-top: 1.048em }
+h3 { font-weight: bold; font-size: 1.125em; margin-top: 1.222em }
+h3 { font-weight: bold; font-size: 1em; margin-top: 1.375em}
+h4 { font-style: italic;  }
+
+/* Used by OCaml's own library documentation. */
+ h6 { font-weight: bold; font-size: 1.125em; margin-top: 1.222em }
+ .h7 { font-weight: bold; font-size: 1em; margin-top: 1.375em }
+
+p { margin-top: 1.375em }
+pre { margin-top: 1.375em }
+.info { margin: 0.458em 0em -0.458em 2em;}/* Description of types values etc. */
+td .info { margin:0; padding:0; margin-left: 2em;} /* Description in indexes */
+
+ul, ol { margin-top:0.688em; padding-bottom:0.687em;
+   list-style-position:outside}
+ul + p, ol + p { margin-top: 0em }
+ul { list-style-type: square }
+
+
+/* h2 + ul, h3 + ul, p + ul { } */
+ul > li { margin-left: 1.375em; }
+ol > li { margin-left: 1.7em; }
+/* Links */
+
+a, a:link, a:visited, a:active, a:hover { color : #00B; text-decoration: none }
+a:hover { text-decoration : underline }
+*:target {background-color: #FFFF99;} /* anchor highlight */
+
+/* Code */
+
+.keyword { font-weight: bold; }
+.comment { color : red }
+.constructor { color : green }
+.string { color : brown }
+.warning { color : red ; font-weight : bold }
+
+/* Functors */
+
+.paramstable { border-style : hidden ; padding-bottom:1.375em}
+.paramstable code { margin-left: 1ex; margin-right: 1ex }
+.sig_block {margin-left: 1em}
+
+/* Images */
+
+img { margin-top: 1.375em }
diff --git a/docs/conf.py b/docs/conf.py
index 18972824dbb..e1e6c45a56f 100644
--- a/docs/conf.py
+++ b/docs/conf.py
@@ -47,9 +47,9 @@
 # built documents.
 #
 # The short X.Y version.
-version = '3.7'
+version = '3.8'
 # The full version, including alpha/beta/rc tags.
-release = '3.7'
+release = '3.8'
 
 # The language for content autogenerated by Sphinx. Refer to documentation
 # for a list of supported languages.
diff --git a/docs/doxygen.cfg.in b/docs/doxygen.cfg.in
index 114c9d11c06..d8c4051e563 100644
--- a/docs/doxygen.cfg.in
+++ b/docs/doxygen.cfg.in
@@ -169,7 +169,7 @@ SHORT_NAMES            = NO
 # description.)
 # The default value is: NO.
 
-JAVADOC_AUTOBRIEF      = NO
+JAVADOC_AUTOBRIEF      = YES
 
 # If the QT_AUTOBRIEF tag is set to YES then doxygen will interpret the first
 # line (until the first dot) of a Qt-style comment as the brief description. If
@@ -177,7 +177,7 @@ JAVADOC_AUTOBRIEF      = NO
 # requiring an explicit \brief command for a brief description.)
 # The default value is: NO.
 
-QT_AUTOBRIEF           = NO
+QT_AUTOBRIEF           = YES
 
 # The MULTILINE_CPP_IS_BRIEF tag can be set to YES to make doxygen treat a
 # multi-line C++ special comment block (i.e. a block of //! or /// comments) as
diff --git a/docs/index.rst b/docs/index.rst
index 2cc5b8bf095..5c04a4e3fc9 100644
--- a/docs/index.rst
+++ b/docs/index.rst
@@ -252,7 +252,7 @@ For API clients and LLVM developers.
    WritingAnLLVMPass
    HowToUseAttributes
    NVPTXUsage
-   R600Usage
+   AMDGPUUsage
    StackMaps
    InAlloca
    BigEndianNEON
@@ -260,6 +260,7 @@ For API clients and LLVM developers.
    Statepoints
    MergeFunctions
    BitSets
+   FaultMaps
 
 :doc:`WritingAnLLVMPass`
    Information on how to write LLVM transformations and analyses.
@@ -338,8 +339,8 @@ For API clients and LLVM developers.
 :doc:`NVPTXUsage`
    This document describes using the NVPTX back-end to compile GPU kernels.
 
-:doc:`R600Usage`
-   This document describes how to use the R600 back-end.
+:doc:`AMDGPUUsage`
+   This document describes how to use the AMDGPU back-end.
 
 :doc:`StackMaps`
   LLVM support for mapping instruction addresses to the location of
@@ -362,6 +363,9 @@ For API clients and LLVM developers.
 :doc:`InAlloca`
   Description of the ``inalloca`` argument attribute.
 
+:doc:`FaultMaps`
+  LLVM support for folding control flow into faulting machine instructions.
+
 Development Process Documentation
 =================================
 
diff --git a/docs/tutorial/LangImpl8.rst b/docs/tutorial/LangImpl8.rst
index 4473035462c..24c4d171f85 100644
--- a/docs/tutorial/LangImpl8.rst
+++ b/docs/tutorial/LangImpl8.rst
@@ -187,13 +187,13 @@ expressions:
   static DIBuilder *DBuilder;
 
   struct DebugInfo {
-    DICompileUnit TheCU;
-    DIType DblTy;
+    DICompileUnit *TheCU;
+    DIType *DblTy;
 
-    DIType getDoubleTy();
+    DIType *getDoubleTy();
   } KSDbgInfo;
 
-  DIType DebugInfo::getDoubleTy() {
+  DIType *DebugInfo::getDoubleTy() {
     if (DblTy.isValid())
       return DblTy;
 
@@ -245,26 +245,26 @@ So the context:
 
 .. code-block:: c++
 
-  DIFile Unit = DBuilder->createFile(KSDbgInfo.TheCU.getFilename(),
-                                     KSDbgInfo.TheCU.getDirectory());
+  DIFile *Unit = DBuilder->createFile(KSDbgInfo.TheCU.getFilename(),
+                                      KSDbgInfo.TheCU.getDirectory());
 
-giving us a DIFile and asking the ``Compile Unit`` we created above for the
+giving us an DIFile and asking the ``Compile Unit`` we created above for the
 directory and filename where we are currently. Then, for now, we use some
 source locations of 0 (since our AST doesn't currently have source location
 information) and construct our function definition:
 
 .. code-block:: c++
 
-  DIDescriptor FContext(Unit);
+  DIScope *FContext = Unit;
   unsigned LineNo = 0;
   unsigned ScopeLine = 0;
-  DISubprogram SP = DBuilder->createFunction(
+  DISubprogram *SP = DBuilder->createFunction(
       FContext, Name, StringRef(), Unit, LineNo,
       CreateFunctionType(Args.size(), Unit), false /* internal linkage */,
-      true /* definition */, ScopeLine, DIDescriptor::FlagPrototyped, false, F);
+      true /* definition */, ScopeLine, DINode::FlagPrototyped, false, F);
 
-and we now have a DISubprogram that contains a reference to all of our metadata
-for the function.
+and we now have an DISubprogram that contains a reference to all of our
+metadata for the function.
 
 Source Locations
 ================
@@ -332,11 +332,11 @@ by constructing another small function:
   void DebugInfo::emitLocation(ExprAST *AST) {
     DIScope *Scope;
     if (LexicalBlocks.empty())
-      Scope = &TheCU;
+      Scope = TheCU;
     else
       Scope = LexicalBlocks.back();
     Builder.SetCurrentDebugLocation(
-        DebugLoc::get(AST->getLine(), AST->getCol(), DIScope(*Scope)));
+        DebugLoc::get(AST->getLine(), AST->getCol(), Scope));
   }
 
 that both tells the main ``IRBuilder`` where we are, but also what scope
@@ -348,10 +348,10 @@ of scopes:
 .. code-block:: c++
 
    std::vector<DIScope *> LexicalBlocks;
-   std::map<const PrototypeAST *, DIScope> FnScopeMap;
+   std::map<const PrototypeAST *, DIScope *> FnScopeMap;
 
-and keep a map of each function to the scope that it represents (a DISubprogram
-is also a DIScope).
+and keep a map of each function to the scope that it represents (an
+DISubprogram is also an DIScope).
 
 Then we make sure to:
 
@@ -393,15 +393,14 @@ argument allocas in ``PrototypeAST::CreateArgumentAllocas``.
 .. code-block:: c++
 
   DIScope *Scope = KSDbgInfo.LexicalBlocks.back();
-  DIFile Unit = DBuilder->createFile(KSDbgInfo.TheCU.getFilename(),
-                                     KSDbgInfo.TheCU.getDirectory());
-  DIVariable D = DBuilder->createLocalVariable(dwarf::DW_TAG_arg_variable,
-                                               *Scope, Args[Idx], Unit, Line,
-                                               KSDbgInfo.getDoubleTy(), Idx);
+  DIFile *Unit = DBuilder->createFile(KSDbgInfo.TheCU.getFilename(),
+                                      KSDbgInfo.TheCU.getDirectory());
+  DILocalVariable D = DBuilder->createParameterVariable(
+      Scope, Args[Idx], Idx + 1, Unit, Line, KSDbgInfo.getDoubleTy(), true);
 
   Instruction *Call = DBuilder->insertDeclare(
       Alloca, D, DBuilder->createExpression(), Builder.GetInsertBlock());
-  Call->setDebugLoc(DebugLoc::get(Line, 0, *Scope));
+  Call->setDebugLoc(DebugLoc::get(Line, 0, Scope));
 
 Here we're doing a few things. First, we're grabbing our current scope
 for the variable so we can say what range of code our variable is valid
diff --git a/docs/yaml2obj.rst b/docs/yaml2obj.rst
index 1812e58914a..d18ce02a336 100644
--- a/docs/yaml2obj.rst
+++ b/docs/yaml2obj.rst
@@ -65,6 +65,7 @@ Here's a simplified Kwalify_ schema with an extension to allow alternate types.
                                  , IMAGE_FILE_MACHINE_AMD64
                                  , IMAGE_FILE_MACHINE_ARM
                                  , IMAGE_FILE_MACHINE_ARMNT
+                                 , IMAGE_FILE_MACHINE_ARM64
                                  , IMAGE_FILE_MACHINE_EBC
                                  , IMAGE_FILE_MACHINE_I386
                                  , IMAGE_FILE_MACHINE_IA64
diff --git a/emscripten-version.txt b/emscripten-version.txt
index 8b9623a5e15..73135129478 100644
--- a/emscripten-version.txt
+++ b/emscripten-version.txt
@@ -1,2 +1,2 @@
-1.35.1
+1.35.2
 
diff --git a/examples/BrainF/BrainF.cpp b/examples/BrainF/BrainF.cpp
index 81c48b9ef8a..8026adc8d07 100644
--- a/examples/BrainF/BrainF.cpp
+++ b/examples/BrainF/BrainF.cpp
@@ -201,7 +201,8 @@ void BrainF::readloop(PHINode *phi, BasicBlock *oldbb, BasicBlock *testbb,
       case SYM_READ:
         {
           //%tape.%d = call i32 @getchar()
-          CallInst *getchar_call = builder->CreateCall(getchar_func, tapereg);
+          CallInst *getchar_call =
+              builder->CreateCall(getchar_func, {}, tapereg);
           getchar_call->setTailCall(false);
           Value *tape_0 = getchar_call;
 
diff --git a/examples/ExceptionDemo/ExceptionDemo.cpp b/examples/ExceptionDemo/ExceptionDemo.cpp
index fed42b7c2bc..81337c4823b 100644
--- a/examples/ExceptionDemo/ExceptionDemo.cpp
+++ b/examples/ExceptionDemo/ExceptionDemo.cpp
@@ -1293,10 +1293,11 @@ static llvm::Function *createCatchWrappedInvokeFunction(
   // (_Unwind_Exception instance). This member tells us whether or not
   // the exception is foreign.
   llvm::Value *unwindExceptionClass =
-    builder.CreateLoad(builder.CreateStructGEP(
-             builder.CreatePointerCast(unwindException,
-                                       ourUnwindExceptionType->getPointerTo()),
-                                               0));
+      builder.CreateLoad(builder.CreateStructGEP(
+          ourUnwindExceptionType,
+          builder.CreatePointerCast(unwindException,
+                                    ourUnwindExceptionType->getPointerTo()),
+          0));
 
   // Branch to the externalExceptionBlock if the exception is foreign or
   // to a catch router if not. Either way the finally block will be run.
@@ -1336,10 +1337,10 @@ static llvm::Function *createCatchWrappedInvokeFunction(
   //
   // Note: Index is not relative to pointer but instead to structure
   //       unlike a true getelementptr (GEP) instruction
-  typeInfoThrown = builder.CreateStructGEP(typeInfoThrown, 0);
+  typeInfoThrown = builder.CreateStructGEP(ourExceptionType, typeInfoThrown, 0);
 
   llvm::Value *typeInfoThrownType =
-  builder.CreateStructGEP(typeInfoThrown, 0);
+      builder.CreateStructGEP(builder.getInt8PtrTy(), typeInfoThrown, 0);
 
   generateIntegerPrint(context,
                        module,
diff --git a/examples/Kaleidoscope/Chapter4/toy.cpp b/examples/Kaleidoscope/Chapter4/toy.cpp
index ad091e4496b..db59e265d2e 100644
--- a/examples/Kaleidoscope/Chapter4/toy.cpp
+++ b/examples/Kaleidoscope/Chapter4/toy.cpp
@@ -560,7 +560,7 @@ void *MCJITHelper::getPointerToFunction(Function *F) {
 
     // Set up the optimizer pipeline.  Start with registering info about how the
     // target lays out data structures.
-    OpenModule->setDataLayout(*NewEngine->getDataLayout());
+    OpenModule->setDataLayout(NewEngine->getDataLayout());
     // Provide basic AliasAnalysis support for GVN.
     FPM->add(createBasicAliasAnalysisPass());
     // Promote allocas to registers.
diff --git a/examples/Kaleidoscope/Chapter5/toy.cpp b/examples/Kaleidoscope/Chapter5/toy.cpp
index db990489573..08d0b77e708 100644
--- a/examples/Kaleidoscope/Chapter5/toy.cpp
+++ b/examples/Kaleidoscope/Chapter5/toy.cpp
@@ -913,7 +913,7 @@ int main() {
 
   // Set up the optimizer pipeline.  Start with registering info about how the
   // target lays out data structures.
-  TheModule->setDataLayout(*TheExecutionEngine->getDataLayout());
+  TheModule->setDataLayout(TheExecutionEngine->getDataLayout());
   // Provide basic AliasAnalysis support for GVN.
   OurFPM.add(createBasicAliasAnalysisPass());
   // Do simple "peephole" optimizations and bit-twiddling optzns.
diff --git a/examples/Kaleidoscope/Chapter6/toy.cpp b/examples/Kaleidoscope/Chapter6/toy.cpp
index e978a3ea368..109c43b9382 100644
--- a/examples/Kaleidoscope/Chapter6/toy.cpp
+++ b/examples/Kaleidoscope/Chapter6/toy.cpp
@@ -1034,7 +1034,7 @@ int main() {
 
   // Set up the optimizer pipeline.  Start with registering info about how the
   // target lays out data structures.
-  TheModule->setDataLayout(*TheExecutionEngine->getDataLayout());
+  TheModule->setDataLayout(TheExecutionEngine->getDataLayout());
   // Provide basic AliasAnalysis support for GVN.
   OurFPM.add(createBasicAliasAnalysisPass());
   // Do simple "peephole" optimizations and bit-twiddling optzns.
diff --git a/examples/Kaleidoscope/Chapter7/CMakeLists.txt b/examples/Kaleidoscope/Chapter7/CMakeLists.txt
index 23aba6510b8..8725e4761f7 100644
--- a/examples/Kaleidoscope/Chapter7/CMakeLists.txt
+++ b/examples/Kaleidoscope/Chapter7/CMakeLists.txt
@@ -11,10 +11,6 @@ set(LLVM_LINK_COMPONENTS
   native
   )
 
-set(LLVM_REQUIRES_RTTI 1)
-
-set(LLVM_BUILD_EXAMPLES OFF)
-
 add_kaleidoscope_chapter(Kaleidoscope-Ch7
   toy.cpp
   )
diff --git a/examples/Kaleidoscope/Chapter7/Makefile b/examples/Kaleidoscope/Chapter7/Makefile
index 7abeb3e5b67..c672c0a36c6 100644
--- a/examples/Kaleidoscope/Chapter7/Makefile
+++ b/examples/Kaleidoscope/Chapter7/Makefile
@@ -9,7 +9,6 @@
 LEVEL = ../../..
 TOOLNAME = Kaleidoscope-Ch7
 EXAMPLE_TOOL = 1
-REQUIRES_RTTI := 1
 
 LINK_COMPONENTS := core mcjit native
 
diff --git a/examples/Kaleidoscope/Chapter7/toy.cpp b/examples/Kaleidoscope/Chapter7/toy.cpp
index 53ea51c2b9c..0217fac10ec 100644
--- a/examples/Kaleidoscope/Chapter7/toy.cpp
+++ b/examples/Kaleidoscope/Chapter7/toy.cpp
@@ -713,7 +713,10 @@ Value *BinaryExprAST::Codegen() {
   // Special case '=' because we don't want to emit the LHS as an expression.
   if (Op == '=') {
     // Assignment requires the LHS to be an identifier.
-    VariableExprAST *LHSE = dynamic_cast<VariableExprAST *>(LHS);
+    // This assume we're building without RTTI because LLVM builds that way by
+    // default.  If you build LLVM with RTTI this can be changed to a
+    // dynamic_cast for automatic error checking.
+    VariableExprAST *LHSE = static_cast<VariableExprAST *>(LHS);
     if (!LHSE)
       return ErrorV("destination of '=' must be a variable");
     // Codegen the RHS.
@@ -1208,7 +1211,7 @@ int main() {
 
   // Set up the optimizer pipeline.  Start with registering info about how the
   // target lays out data structures.
-  TheModule->setDataLayout(*TheExecutionEngine->getDataLayout());
+  TheModule->setDataLayout(TheExecutionEngine->getDataLayout());
   // Provide basic AliasAnalysis support for GVN.
   OurFPM.add(createBasicAliasAnalysisPass());
   // Promote allocas to registers.
diff --git a/examples/Kaleidoscope/Chapter8/CMakeLists.txt b/examples/Kaleidoscope/Chapter8/CMakeLists.txt
index 90c79c0b9ae..f94ed743618 100644
--- a/examples/Kaleidoscope/Chapter8/CMakeLists.txt
+++ b/examples/Kaleidoscope/Chapter8/CMakeLists.txt
@@ -7,10 +7,6 @@ set(LLVM_LINK_COMPONENTS
   native
   )
 
-set(LLVM_REQUIRES_RTTI 1)
-
-set(LLVM_BUILD_EXAMPLES OFF)
-
 add_kaleidoscope_chapter(Kaleidoscope-Ch8
   toy.cpp
   )
diff --git a/examples/Kaleidoscope/Chapter8/Makefile b/examples/Kaleidoscope/Chapter8/Makefile
index 8e4d4221178..25f048c39b0 100644
--- a/examples/Kaleidoscope/Chapter8/Makefile
+++ b/examples/Kaleidoscope/Chapter8/Makefile
@@ -9,7 +9,6 @@
 LEVEL = ../../..
 TOOLNAME = Kaleidoscope-Ch8
 EXAMPLE_TOOL = 1
-REQUIRES_RTTI := 1
 
 LINK_COMPONENTS := core mcjit native
 
diff --git a/examples/Kaleidoscope/Chapter8/toy.cpp b/examples/Kaleidoscope/Chapter8/toy.cpp
index c7e096cc484..f525ac7a1b0 100644
--- a/examples/Kaleidoscope/Chapter8/toy.cpp
+++ b/examples/Kaleidoscope/Chapter8/toy.cpp
@@ -93,13 +93,13 @@ class ExprAST;
 }
 static IRBuilder<> Builder(getGlobalContext());
 struct DebugInfo {
-  DICompileUnit TheCU;
-  DIType DblTy;
+  DICompileUnit *TheCU;
+  DIType *DblTy;
   std::vector<DIScope *> LexicalBlocks;
-  std::map<const PrototypeAST *, DIScope> FnScopeMap;
+  std::map<const PrototypeAST *, DIScope *> FnScopeMap;
 
   void emitLocation(ExprAST *AST);
-  DIType getDoubleTy();
+  DIType *getDoubleTy();
 } KSDbgInfo;
 
 static std::string IdentifierStr; // Filled in if tok_identifier
@@ -816,7 +816,7 @@ static PrototypeAST *ParseExtern() {
 
 static DIBuilder *DBuilder;
 
-DIType DebugInfo::getDoubleTy() {
+DIType *DebugInfo::getDoubleTy() {
   if (DblTy)
     return DblTy;
 
@@ -827,18 +827,18 @@ DIType DebugInfo::getDoubleTy() {
 void DebugInfo::emitLocation(ExprAST *AST) {
   if (!AST)
     return Builder.SetCurrentDebugLocation(DebugLoc());
-  MDScope *Scope;
+  DIScope *Scope;
   if (LexicalBlocks.empty())
     Scope = TheCU;
   else
-    Scope = *LexicalBlocks.back();
+    Scope = LexicalBlocks.back();
   Builder.SetCurrentDebugLocation(
       DebugLoc::get(AST->getLine(), AST->getCol(), Scope));
 }
 
-static MDSubroutineType *CreateFunctionType(unsigned NumArgs, DIFile Unit) {
+static DISubroutineType *CreateFunctionType(unsigned NumArgs, DIFile *Unit) {
   SmallVector<Metadata *, 8> EltTys;
-  DIType DblTy = KSDbgInfo.getDoubleTy();
+  DIType *DblTy = KSDbgInfo.getDoubleTy();
 
   // Add the result type.
   EltTys.push_back(DblTy);
@@ -846,8 +846,8 @@ static MDSubroutineType *CreateFunctionType(unsigned NumArgs, DIFile Unit) {
   for (unsigned i = 0, e = NumArgs; i != e; ++i)
     EltTys.push_back(DblTy);
 
-  DITypeArray EltTypeArray = DBuilder->getOrCreateTypeArray(EltTys);
-  return DBuilder->createSubroutineType(Unit, EltTypeArray);
+  return DBuilder->createSubroutineType(Unit,
+                                        DBuilder->getOrCreateTypeArray(EltTys));
 }
 
 //===----------------------------------------------------------------------===//
@@ -908,7 +908,10 @@ Value *BinaryExprAST::Codegen() {
   // Special case '=' because we don't want to emit the LHS as an expression.
   if (Op == '=') {
     // Assignment requires the LHS to be an identifier.
-    VariableExprAST *LHSE = dynamic_cast<VariableExprAST *>(LHS);
+    // This assume we're building without RTTI because LLVM builds that way by
+    // default.  If you build LLVM with RTTI this can be changed to a
+    // dynamic_cast for automatic error checking.
+    VariableExprAST *LHSE = static_cast<VariableExprAST *>(LHS);
     if (!LHSE)
       return ErrorV("destination of '=' must be a variable");
     // Codegen the RHS.
@@ -1224,15 +1227,15 @@ Function *PrototypeAST::Codegen() {
     AI->setName(Args[Idx]);
 
   // Create a subprogram DIE for this function.
-  DIFile Unit = DBuilder->createFile(KSDbgInfo.TheCU->getFilename(),
-                                     KSDbgInfo.TheCU->getDirectory());
-  MDScope *FContext = Unit;
+  DIFile *Unit = DBuilder->createFile(KSDbgInfo.TheCU->getFilename(),
+                                      KSDbgInfo.TheCU->getDirectory());
+  DIScope *FContext = Unit;
   unsigned LineNo = Line;
   unsigned ScopeLine = Line;
-  DISubprogram SP = DBuilder->createFunction(
+  DISubprogram *SP = DBuilder->createFunction(
       FContext, Name, StringRef(), Unit, LineNo,
       CreateFunctionType(Args.size(), Unit), false /* internal linkage */,
-      true /* definition */, ScopeLine, DebugNode::FlagPrototyped, false, F);
+      true /* definition */, ScopeLine, DINode::FlagPrototyped, false, F);
 
   KSDbgInfo.FnScopeMap[this] = SP;
   return F;
@@ -1248,14 +1251,13 @@ void PrototypeAST::CreateArgumentAllocas(Function *F) {
 
     // Create a debug descriptor for the variable.
     DIScope *Scope = KSDbgInfo.LexicalBlocks.back();
-    DIFile Unit = DBuilder->createFile(KSDbgInfo.TheCU->getFilename(),
-                                       KSDbgInfo.TheCU->getDirectory());
-    DIVariable D = DBuilder->createLocalVariable(dwarf::DW_TAG_arg_variable,
-                                                 *Scope, Args[Idx], Unit, Line,
-                                                 KSDbgInfo.getDoubleTy(), Idx);
+    DIFile *Unit = DBuilder->createFile(KSDbgInfo.TheCU->getFilename(),
+                                        KSDbgInfo.TheCU->getDirectory());
+    DILocalVariable *D = DBuilder->createParameterVariable(
+        Scope, Args[Idx], Idx + 1, Unit, Line, KSDbgInfo.getDoubleTy(), true);
 
     DBuilder->insertDeclare(Alloca, D, DBuilder->createExpression(),
-                            DebugLoc::get(Line, 0, *Scope),
+                            DebugLoc::get(Line, 0, Scope),
                             Builder.GetInsertBlock());
 
     // Store the initial value into the alloca.
@@ -1274,7 +1276,7 @@ Function *FunctionAST::Codegen() {
     return 0;
 
   // Push the current scope.
-  KSDbgInfo.LexicalBlocks.push_back(&KSDbgInfo.FnScopeMap[Proto]);
+  KSDbgInfo.LexicalBlocks.push_back(KSDbgInfo.FnScopeMap[Proto]);
 
   // Unset the location for the prologue emission (leading instructions with no
   // location in a function are considered part of the prologue and the debugger
@@ -1459,7 +1461,7 @@ int main() {
 
   // Set up the optimizer pipeline.  Start with registering info about how the
   // target lays out data structures.
-  TheModule->setDataLayout(*TheExecutionEngine->getDataLayout());
+  TheModule->setDataLayout(TheExecutionEngine->getDataLayout());
 #if 0
   // Provide basic AliasAnalysis support for GVN.
   OurFPM.add(createBasicAliasAnalysisPass());
diff --git a/examples/Kaleidoscope/MCJIT/cached/toy-jit.cpp b/examples/Kaleidoscope/MCJIT/cached/toy-jit.cpp
index 7f5ed137cec..77b7f001095 100644
--- a/examples/Kaleidoscope/MCJIT/cached/toy-jit.cpp
+++ b/examples/Kaleidoscope/MCJIT/cached/toy-jit.cpp
@@ -672,7 +672,7 @@ Value *BinaryExprAST::Codegen() {
     // For now, I'm building without RTTI because LLVM builds that way by
     // default and so we need to build that way to use the command line supprt.
     // If you build LLVM with RTTI this can be changed back to a dynamic_cast.
-    VariableExprAST *LHSE = reinterpret_cast<VariableExprAST*>(LHS);
+    VariableExprAST *LHSE = static_cast<VariableExprAST*>(LHS);
     if (!LHSE)
       return ErrorV("destination of '=' must be a variable");
     // Codegen the RHS.
diff --git a/examples/Kaleidoscope/MCJIT/cached/toy.cpp b/examples/Kaleidoscope/MCJIT/cached/toy.cpp
index f598ca41f0f..cc12abcc431 100644
--- a/examples/Kaleidoscope/MCJIT/cached/toy.cpp
+++ b/examples/Kaleidoscope/MCJIT/cached/toy.cpp
@@ -1039,7 +1039,7 @@ Value *BinaryExprAST::Codegen() {
   // Special case '=' because we don't want to emit the LHS as an expression.
   if (Op == '=') {
     // Assignment requires the LHS to be an identifier.
-    VariableExprAST *LHSE = reinterpret_cast<VariableExprAST*>(LHS);
+    VariableExprAST *LHSE = static_cast<VariableExprAST*>(LHS);
     if (!LHSE)
       return ErrorV("destination of '=' must be a variable");
     // Codegen the RHS.
diff --git a/examples/Kaleidoscope/MCJIT/complete/toy.cpp b/examples/Kaleidoscope/MCJIT/complete/toy.cpp
index 5294bb71ee1..c78ec35fa0b 100644
--- a/examples/Kaleidoscope/MCJIT/complete/toy.cpp
+++ b/examples/Kaleidoscope/MCJIT/complete/toy.cpp
@@ -1113,7 +1113,7 @@ Value *BinaryExprAST::Codegen() {
     // This assume we're building without RTTI because LLVM builds that way by
     // default.  If you build LLVM with RTTI this can be changed to a
     // dynamic_cast for automatic error checking.
-    VariableExprAST *LHSE = reinterpret_cast<VariableExprAST*>(LHS);
+    VariableExprAST *LHSE = static_cast<VariableExprAST*>(LHS);
     if (!LHSE)
       return ErrorV("destination of '=' must be a variable");
     // Codegen the RHS.
diff --git a/examples/Kaleidoscope/MCJIT/initial/toy.cpp b/examples/Kaleidoscope/MCJIT/initial/toy.cpp
index dd35358753d..9455946087d 100644
--- a/examples/Kaleidoscope/MCJIT/initial/toy.cpp
+++ b/examples/Kaleidoscope/MCJIT/initial/toy.cpp
@@ -897,7 +897,7 @@ Value *BinaryExprAST::Codegen() {
   // Special case '=' because we don't want to emit the LHS as an expression.
   if (Op == '=') {
     // Assignment requires the LHS to be an identifier.
-    VariableExprAST *LHSE = reinterpret_cast<VariableExprAST*>(LHS);
+    VariableExprAST *LHSE = static_cast<VariableExprAST*>(LHS);
     if (!LHSE)
       return ErrorV("destination of '=' must be a variable");
     // Codegen the RHS.
diff --git a/examples/Kaleidoscope/MCJIT/lazy/toy.cpp b/examples/Kaleidoscope/MCJIT/lazy/toy.cpp
index fe7fb619697..14d758cfa79 100644
--- a/examples/Kaleidoscope/MCJIT/lazy/toy.cpp
+++ b/examples/Kaleidoscope/MCJIT/lazy/toy.cpp
@@ -937,7 +937,7 @@ Value *BinaryExprAST::Codegen() {
   // Special case '=' because we don't want to emit the LHS as an expression.
   if (Op == '=') {
     // Assignment requires the LHS to be an identifier.
-    VariableExprAST *LHSE = reinterpret_cast<VariableExprAST*>(LHS);
+    VariableExprAST *LHSE = static_cast<VariableExprAST*>(LHS);
     if (!LHSE)
       return ErrorV("destination of '=' must be a variable");
     // Codegen the RHS.
diff --git a/examples/Kaleidoscope/Orc/fully_lazy/toy.cpp b/examples/Kaleidoscope/Orc/fully_lazy/toy.cpp
index b5822921b40..b68f807e3e7 100644
--- a/examples/Kaleidoscope/Orc/fully_lazy/toy.cpp
+++ b/examples/Kaleidoscope/Orc/fully_lazy/toy.cpp
@@ -717,7 +717,7 @@ class IRGenContext {
       M(new Module(GenerateUniqueName("jit_module_"),
                    Session.getLLVMContext())),
       Builder(Session.getLLVMContext()) {
-    M->setDataLayout(*Session.getTarget().getDataLayout());
+    M->setDataLayout(Session.getTarget().createDataLayout());
   }
 
   SessionContext& getSession() { return Session; }
@@ -1168,7 +1168,6 @@ class KaleidoscopeJIT {
 
   KaleidoscopeJIT(SessionContext &Session)
     : Session(Session),
-      Mang(Session.getTarget().getDataLayout()),
       CompileLayer(ObjectLayer, SimpleCompiler(Session.getTarget())),
       LazyEmitLayer(CompileLayer),
       CompileCallbacks(LazyEmitLayer, CCMgrMemMgr, Session.getLLVMContext(),
@@ -1179,7 +1178,8 @@ class KaleidoscopeJIT {
     std::string MangledName;
     {
       raw_string_ostream MangledNameStream(MangledName);
-      Mang.getNameWithPrefix(MangledNameStream, Name);
+      Mangler::getNameWithPrefix(MangledNameStream, Name,
+                                 Session.getTarget().createDataLayout());
     }
     return MangledName;
   }
@@ -1214,11 +1214,11 @@ class KaleidoscopeJIT {
   void removeModule(ModuleHandleT H) { LazyEmitLayer.removeModuleSet(H); }
 
   JITSymbol findSymbol(const std::string &Name) {
-    return LazyEmitLayer.findSymbol(Name, true);
+    return LazyEmitLayer.findSymbol(Name, false);
   }
 
   JITSymbol findSymbolIn(ModuleHandleT H, const std::string &Name) {
-    return LazyEmitLayer.findSymbolIn(H, Name, true);
+    return LazyEmitLayer.findSymbolIn(H, Name, false);
   }
 
   JITSymbol findUnmangledSymbol(const std::string &Name) {
@@ -1276,7 +1276,7 @@ class KaleidoscopeJIT {
     makeStub(*F, *FunctionBodyPointer);
 
     // Step 4) Add the module containing the stub to the JIT.
-    auto H = addModule(C.takeM());
+    auto StubH = addModule(C.takeM());
 
     // Step 5) Set the compile and update actions.
     //
@@ -1289,18 +1289,23 @@ class KaleidoscopeJIT {
     //   The update action will update FunctionBodyPointer to point at the newly
     // compiled function.
     std::shared_ptr<FunctionAST> Fn = std::move(FnAST);
-    CallbackInfo.setCompileAction([this, Fn]() {
+    CallbackInfo.setCompileAction([this, Fn, BodyPtrName, StubH]() {
       auto H = addModule(IRGen(Session, *Fn));
-      return findUnmangledSymbolIn(H, Fn->Proto->Name).getAddress();
+      auto BodySym = findUnmangledSymbolIn(H, Fn->Proto->Name);
+      auto BodyPtrSym = findUnmangledSymbolIn(StubH, BodyPtrName);
+      assert(BodySym && "Missing function body.");
+      assert(BodyPtrSym && "Missing function pointer.");
+      auto BodyAddr = BodySym.getAddress();
+      auto BodyPtr = reinterpret_cast<void*>(
+                       static_cast<uintptr_t>(BodyPtrSym.getAddress()));
+      memcpy(BodyPtr, &BodyAddr, sizeof(uintptr_t));
+      return BodyAddr;
     });
-    CallbackInfo.setUpdateAction(
-        getLocalFPUpdater(LazyEmitLayer, H, mangle(BodyPtrName)));
 
-    return H;
+    return StubH;
   }
 
   SessionContext &Session;
-  Mangler Mang;
   SectionMemoryManager CCMgrMemMgr;
   ObjLayerT ObjectLayer;
   CompileLayerT CompileLayer;
diff --git a/examples/Kaleidoscope/Orc/initial/toy.cpp b/examples/Kaleidoscope/Orc/initial/toy.cpp
index bf43f2952c7..91290157ace 100644
--- a/examples/Kaleidoscope/Orc/initial/toy.cpp
+++ b/examples/Kaleidoscope/Orc/initial/toy.cpp
@@ -716,7 +716,7 @@ class IRGenContext {
       M(new Module(GenerateUniqueName("jit_module_"),
                    Session.getLLVMContext())),
       Builder(Session.getLLVMContext()) {
-    M->setDataLayout(*Session.getTarget().getDataLayout());
+    M->setDataLayout(Session.getTarget().createDataLayout());
   }
 
   SessionContext& getSession() { return Session; }
@@ -1160,14 +1160,14 @@ class KaleidoscopeJIT {
   typedef CompileLayerT::ModuleSetHandleT ModuleHandleT;
 
   KaleidoscopeJIT(SessionContext &Session)
-    : Mang(Session.getTarget().getDataLayout()),
-      CompileLayer(ObjectLayer, SimpleCompiler(Session.getTarget())) {}
+      : DL(Session.getTarget().createDataLayout()),
+        CompileLayer(ObjectLayer, SimpleCompiler(Session.getTarget())) {}
 
   std::string mangle(const std::string &Name) {
     std::string MangledName;
     {
       raw_string_ostream MangledNameStream(MangledName);
-      Mang.getNameWithPrefix(MangledNameStream, Name);
+      Mangler::getNameWithPrefix(MangledNameStream, Name, DL);
     }
     return MangledName;
   }
@@ -1201,8 +1201,7 @@ class KaleidoscopeJIT {
   }
 
 private:
-
-  Mangler Mang;
+  const DataLayout DL;
   ObjLayerT ObjectLayer;
   CompileLayerT CompileLayer;
 };
diff --git a/examples/Kaleidoscope/Orc/lazy_codegen/toy.cpp b/examples/Kaleidoscope/Orc/lazy_codegen/toy.cpp
index 1369ba6f5ee..17f9dbb6dfa 100644
--- a/examples/Kaleidoscope/Orc/lazy_codegen/toy.cpp
+++ b/examples/Kaleidoscope/Orc/lazy_codegen/toy.cpp
@@ -716,7 +716,7 @@ class IRGenContext {
       M(new Module(GenerateUniqueName("jit_module_"),
                    Session.getLLVMContext())),
       Builder(Session.getLLVMContext()) {
-    M->setDataLayout(*Session.getTarget().getDataLayout());
+    M->setDataLayout(Session.getTarget().createDataLayout());
   }
 
   SessionContext& getSession() { return Session; }
@@ -1162,15 +1162,15 @@ class KaleidoscopeJIT {
   typedef LazyEmitLayerT::ModuleSetHandleT ModuleHandleT;
 
   KaleidoscopeJIT(SessionContext &Session)
-    : Mang(Session.getTarget().getDataLayout()),
-      CompileLayer(ObjectLayer, SimpleCompiler(Session.getTarget())),
-      LazyEmitLayer(CompileLayer) {}
+      : DL(Session.getTarget().createDataLayout()),
+        CompileLayer(ObjectLayer, SimpleCompiler(Session.getTarget())),
+        LazyEmitLayer(CompileLayer) {}
 
   std::string mangle(const std::string &Name) {
     std::string MangledName;
     {
       raw_string_ostream MangledNameStream(MangledName);
-      Mang.getNameWithPrefix(MangledNameStream, Name);
+      Mangler::getNameWithPrefix(MangledNameStream, Name, DL);
     }
     return MangledName;
   }
@@ -1204,8 +1204,7 @@ class KaleidoscopeJIT {
   }
 
 private:
-
-  Mangler Mang;
+  const DataLayout DL;
   ObjLayerT ObjectLayer;
   CompileLayerT CompileLayer;
   LazyEmitLayerT LazyEmitLayer;
diff --git a/examples/Kaleidoscope/Orc/lazy_irgen/toy.cpp b/examples/Kaleidoscope/Orc/lazy_irgen/toy.cpp
index c489a450d79..7373c99d543 100644
--- a/examples/Kaleidoscope/Orc/lazy_irgen/toy.cpp
+++ b/examples/Kaleidoscope/Orc/lazy_irgen/toy.cpp
@@ -716,7 +716,7 @@ class IRGenContext {
       M(new Module(GenerateUniqueName("jit_module_"),
                    Session.getLLVMContext())),
       Builder(Session.getLLVMContext()) {
-    M->setDataLayout(*Session.getTarget().getDataLayout());
+    M->setDataLayout(Session.getTarget().createDataLayout());
   }
 
   SessionContext& getSession() { return Session; }
@@ -1162,7 +1162,6 @@ class KaleidoscopeJIT {
 
   KaleidoscopeJIT(SessionContext &Session)
     : Session(Session),
-      Mang(Session.getTarget().getDataLayout()),
       CompileLayer(ObjectLayer, SimpleCompiler(Session.getTarget())),
       LazyEmitLayer(CompileLayer) {}
 
@@ -1170,7 +1169,8 @@ class KaleidoscopeJIT {
     std::string MangledName;
     {
       raw_string_ostream MangledNameStream(MangledName);
-      Mang.getNameWithPrefix(MangledNameStream, Name);
+      Mangler::getNameWithPrefix(MangledNameStream, Name,
+                                 Session.getTarget().createDataLayout());
     }
     return MangledName;
   }
@@ -1236,7 +1236,6 @@ class KaleidoscopeJIT {
   }
 
   SessionContext &Session;
-  Mangler Mang;
   ObjLayerT ObjectLayer;
   CompileLayerT CompileLayer;
   LazyEmitLayerT LazyEmitLayer;
diff --git a/include/llvm-c/Core.h b/include/llvm-c/Core.h
index effbd15c10d..764dbfac342 100644
--- a/include/llvm-c/Core.h
+++ b/include/llvm-c/Core.h
@@ -161,15 +161,15 @@ typedef enum {
     /* FIXME: These attributes are currently not included in the C API as
        a temporary measure until the API/ABI impact to the C API is understood
        and the path forward agreed upon.
-    LLVMAddressSafety = 1ULL << 32,
-    LLVMStackProtectStrongAttribute = 1ULL<<33,
-    LLVMCold = 1ULL << 34,
-    LLVMOptimizeNone = 1ULL << 35,
-    LLVMInAllocaAttribute = 1ULL << 36,
-    LLVMNonNullAttribute = 1ULL << 37,
-    LLVMJumpTableAttribute = 1ULL << 38,
-    LLVMDereferenceableAttribute = 1ULL << 39,
-    LLVMDereferenceableOrNullAttribute = 1ULL << 40,
+    LLVMSanitizeAddressAttribute = 1ULL << 32,
+    LLVMStackProtectStrongAttribute = 1ULL<<35,
+    LLVMColdAttribute = 1ULL << 40,
+    LLVMOptimizeNoneAttribute = 1ULL << 42,
+    LLVMInAllocaAttribute = 1ULL << 43,
+    LLVMNonNullAttribute = 1ULL << 44,
+    LLVMJumpTableAttribute = 1ULL << 45,
+    LLVMConvergentAttribute = 1ULL << 46,
+    LLVMSafeStackAttribute = 1ULL << 47,
     */
 } LLVMAttribute;
 
@@ -248,7 +248,13 @@ typedef enum {
 
   /* Exception Handling Operators */
   LLVMResume         = 58,
-  LLVMLandingPad     = 59
+  LLVMLandingPad     = 59,
+  LLVMCleanupRet     = 61,
+  LLVMCatchRet       = 62,
+  LLVMCatchPad     = 63,
+  LLVMTerminatePad = 64,
+  LLVMCleanupPad   = 65,
+  LLVMCatchEndPad  = 66
 
 } LLVMOpcode;
 
@@ -997,6 +1003,13 @@ unsigned LLVMCountStructElementTypes(LLVMTypeRef StructTy);
  */
 void LLVMGetStructElementTypes(LLVMTypeRef StructTy, LLVMTypeRef *Dest);
 
+/**
+ * Get the type of the element at a given index in the structure.
+ *
+ * @see llvm::StructType::getTypeAtIndex()
+ */
+LLVMTypeRef LLVMStructGetTypeAtIndex(LLVMTypeRef StructTy, unsigned i);
+
 /**
  * Determine whether a structure is packed.
  *
@@ -1196,6 +1209,7 @@ LLVMTypeRef LLVMX86MMXType(void);
       macro(InsertElementInst)              \
       macro(InsertValueInst)                \
       macro(LandingPadInst)                 \
+      macro(CleanupPadInst)               \
       macro(PHINode)                        \
       macro(SelectInst)                     \
       macro(ShuffleVectorInst)              \
@@ -1208,6 +1222,11 @@ LLVMTypeRef LLVMX86MMXType(void);
         macro(SwitchInst)                   \
         macro(UnreachableInst)              \
         macro(ResumeInst)                   \
+        macro(CleanupReturnInst)            \
+        macro(CatchReturnInst)              \
+        macro(CatchPadInst)               \
+        macro(TerminatePadInst)           \
+        macro(CatchEndPadInst)            \
       macro(UnaryInstruction)               \
         macro(AllocaInst)                   \
         macro(CastInst)                     \
@@ -1880,6 +1899,20 @@ LLVMValueRef LLVMAddAlias(LLVMModuleRef M, LLVMTypeRef Ty, LLVMValueRef Aliasee,
  */
 void LLVMDeleteFunction(LLVMValueRef Fn);
 
+/**
+ * Obtain the personality function attached to the function.
+ *
+ * @see llvm::Function::getPersonalityFn()
+ */
+LLVMValueRef LLVMGetPersonalityFn(LLVMValueRef Fn);
+
+/**
+ * Set the personality function attached to the function.
+ *
+ * @see llvm::Function::setPersonalityFn()
+ */
+void LLVMSetPersonalityFn(LLVMValueRef Fn, LLVMValueRef PersonalityFn);
+
 /**
  * Obtain the ID number from a function instance.
  *
@@ -1929,7 +1962,7 @@ void LLVMSetGC(LLVMValueRef Fn, const char *Name);
 void LLVMAddFunctionAttr(LLVMValueRef Fn, LLVMAttribute PA);
 
 /**
- * Add a target-dependent attribute to a fuction
+ * Add a target-dependent attribute to a function
  * @see llvm::AttrBuilder::addAttribute()
  */
 void LLVMAddTargetDependentFunctionAttr(LLVMValueRef Fn, const char *A,
diff --git a/include/llvm-c/Object.h b/include/llvm-c/Object.h
index 447fcea7bc2..9cab5c426c4 100644
--- a/include/llvm-c/Object.h
+++ b/include/llvm-c/Object.h
@@ -81,7 +81,6 @@ uint64_t LLVMGetSymbolAddress(LLVMSymbolIteratorRef SI);
 uint64_t LLVMGetSymbolSize(LLVMSymbolIteratorRef SI);
 
 // RelocationRef accessors
-uint64_t LLVMGetRelocationAddress(LLVMRelocationIteratorRef RI);
 uint64_t LLVMGetRelocationOffset(LLVMRelocationIteratorRef RI);
 LLVMSymbolIteratorRef LLVMGetRelocationSymbol(LLVMRelocationIteratorRef RI);
 uint64_t LLVMGetRelocationType(LLVMRelocationIteratorRef RI);
diff --git a/include/llvm-c/Support.h b/include/llvm-c/Support.h
index a9216d0364a..eca3b7a4203 100644
--- a/include/llvm-c/Support.h
+++ b/include/llvm-c/Support.h
@@ -58,6 +58,24 @@ LLVMBool LLVMLoadLibraryPermanently(const char* Filename);
 void LLVMParseCommandLineOptions(int argc, const char *const *argv,
                                  const char *Overview);
 
+/**
+ * This function will search through all previously loaded dynamic
+ * libraries for the symbol \p symbolName. If it is found, the address of
+ * that symbol is returned. If not, null is returned.
+ *
+ * @see sys::DynamicLibrary::SearchForAddressOfSymbol()
+ */
+void *LLVMSearchForAddressOfSymbol(const char *symbolName);
+
+/**
+ * This functions permanently adds the symbol \p symbolName with the
+ * value \p symbolValue.  These symbols are searched before any
+ * libraries.
+ *
+ * @see sys::DynamicLibrary::AddSymbol()
+ */
+void LLVMAddSymbol(const char *symbolName, void *symbolValue);
+
 #ifdef __cplusplus
 }
 #endif
diff --git a/include/llvm-c/lto.h b/include/llvm-c/lto.h
index c6acdaddca5..cb3a6916045 100644
--- a/include/llvm-c/lto.h
+++ b/include/llvm-c/lto.h
@@ -40,7 +40,7 @@ typedef bool lto_bool_t;
  * @{
  */
 
-#define LTO_API_VERSION 13
+#define LTO_API_VERSION 17
 
 /**
  * \since prior to LTO_API_VERSION=3
@@ -62,7 +62,9 @@ typedef enum {
     LTO_SYMBOL_SCOPE_HIDDEN                = 0x00001000,
     LTO_SYMBOL_SCOPE_PROTECTED             = 0x00002000,
     LTO_SYMBOL_SCOPE_DEFAULT               = 0x00001800,
-    LTO_SYMBOL_SCOPE_DEFAULT_CAN_BE_HIDDEN = 0x00002800
+    LTO_SYMBOL_SCOPE_DEFAULT_CAN_BE_HIDDEN = 0x00002800,
+    LTO_SYMBOL_COMDAT                      = 0x00004000,
+    LTO_SYMBOL_ALIAS                       = 0x00008000
 } lto_symbol_attributes;
 
 /**
@@ -171,7 +173,7 @@ lto_module_create_from_memory(const void* mem, size_t length);
  * Loads an object file from memory with an extra path argument.
  * Returns NULL on error (check lto_get_error_message() for details).
  *
- * \since prior to LTO_API_VERSION=9
+ * \since LTO_API_VERSION=9
  */
 extern lto_module_t
 lto_module_create_from_memory_with_path(const void* mem, size_t length,
@@ -279,39 +281,15 @@ lto_module_get_symbol_attribute(lto_module_t mod, unsigned int index);
 
 
 /**
- * Returns the number of dependent libraries in the object module.
+ * Returns the module's linker options.
  *
- * \since LTO_API_VERSION=8
- */
-extern unsigned int
-lto_module_get_num_deplibs(lto_module_t mod);
-
-
-/**
- * Returns the ith dependent library in the module.
- *
- * \since LTO_API_VERSION=8
- */
-extern const char*
-lto_module_get_deplib(lto_module_t mod, unsigned int index);
-
-
-/**
- * Returns the number of linker options in the object module.
- *
- * \since LTO_API_VERSION=8
- */
-extern unsigned int
-lto_module_get_num_linkeropts(lto_module_t mod);
-
-
-/**
- * Returns the ith linker option in the module.
+ * The linker options may consist of multiple flags. It is the linker's
+ * responsibility to split the flags using a platform-specific mechanism.
  *
- * \since LTO_API_VERSION=8
+ * \since LTO_API_VERSION=16
  */
 extern const char*
-lto_module_get_linkeropt(lto_module_t mod, unsigned int index);
+lto_module_get_linkeropts(lto_module_t mod);
 
 
 /**
@@ -401,7 +379,7 @@ lto_codegen_add_module(lto_code_gen_t cg, lto_module_t mod);
  *
  * \c cg and \c mod must both be in the same context.
  *
- * \since prior to LTO_API_VERSION=13
+ * \since LTO_API_VERSION=13
  */
 extern void
 lto_codegen_set_module(lto_code_gen_t cg, lto_module_t mod);
@@ -548,6 +526,28 @@ lto_codegen_debug_options(lto_code_gen_t cg, const char *);
 extern void
 lto_initialize_disassembler(void);
 
+/**
+ * Sets if we should run internalize pass during optimization and code
+ * generation.
+ *
+ * \since LTO_API_VERSION=14
+ */
+extern void
+lto_codegen_set_should_internalize(lto_code_gen_t cg,
+                                   lto_bool_t ShouldInternalize);
+
+/**
+ * \brief Set whether to embed uselists in bitcode.
+ *
+ * Sets whether \a lto_codegen_write_merged_modules() should embed uselists in
+ * output bitcode.  This should be turned on for all -save-temps output.
+ *
+ * \since LTO_API_VERSION=15
+ */
+extern void
+lto_codegen_set_should_embed_uselists(lto_code_gen_t cg,
+                                      lto_bool_t ShouldEmbedUselists);
+
 #ifdef __cplusplus
 }
 #endif
diff --git a/include/llvm/ADT/APFloat.h b/include/llvm/ADT/APFloat.h
index 53b53c5dfa5..76615affb25 100644
--- a/include/llvm/ADT/APFloat.h
+++ b/include/llvm/ADT/APFloat.h
@@ -276,6 +276,10 @@ class APFloat {
   /// \param isIEEE   - If 128 bit number, select between PPC and IEEE
   static APFloat getAllOnesValue(unsigned BitWidth, bool isIEEE = false);
 
+  /// Returns the size of the floating point number (in bits) in the given
+  /// semantics.
+  static unsigned getSizeInBits(const fltSemantics &Sem);
+
   /// @}
 
   /// Used to insert APFloat objects, or objects that contain APFloat objects,
@@ -343,7 +347,7 @@ class APFloat {
   /// copied from some other APFloat.
   static APFloat copySign(APFloat Value, const APFloat &Sign) {
     Value.copySign(Sign);
-    return std::move(Value);
+    return Value;
   }
 
   /// @}
diff --git a/include/llvm/ADT/APInt.h b/include/llvm/ADT/APInt.h
index 36d8159d21b..5013f295f5c 100644
--- a/include/llvm/ADT/APInt.h
+++ b/include/llvm/ADT/APInt.h
@@ -129,7 +129,7 @@ class APInt {
 
   /// \brief Clear unused high order bits
   ///
-  /// This method is used internally to clear the to "N" bits in the high order
+  /// This method is used internally to clear the top "N" bits in the high order
   /// word that are not used by the APInt. This is needed after the most
   /// significant word is assigned a value to ensure that those bits are
   /// zero'd out.
@@ -351,8 +351,7 @@ class APInt {
   /// This checks to see if the value of this APInt is the maximum signed
   /// value for the APInt's bit width.
   bool isMaxSignedValue() const {
-    return BitWidth == 1 ? VAL == 0
-                         : !isNegative() && countPopulation() == BitWidth - 1;
+    return !isNegative() && countPopulation() == BitWidth - 1;
   }
 
   /// \brief Determine if this is the smallest unsigned value.
@@ -366,7 +365,7 @@ class APInt {
   /// This checks to see if the value of this APInt is the minimum signed
   /// value for the APInt's bit width.
   bool isMinSignedValue() const {
-    return BitWidth == 1 ? VAL == 1 : isNegative() && isPowerOf2();
+    return isNegative() && isPowerOf2();
   }
 
   /// \brief Check if this APInt has an N-bits unsigned integer value.
@@ -796,7 +795,7 @@ class APInt {
 
   /// \brief Bitwise OR function.
   ///
-  /// Performs a bitwise or on *this and RHS. This is implemented bny simply
+  /// Performs a bitwise or on *this and RHS. This is implemented by simply
   /// calling operator|.
   ///
   /// \returns An APInt value representing the bitwise OR of *this and RHS.
@@ -1039,7 +1038,9 @@ class APInt {
   /// the validity of the less-than relationship.
   ///
   /// \returns true if *this < RHS when considered unsigned.
-  bool ult(uint64_t RHS) const { return ult(APInt(getBitWidth(), RHS)); }
+  bool ult(uint64_t RHS) const {
+    return getActiveBits() > 64 ? false : getZExtValue() < RHS;
+  }
 
   /// \brief Signed less than comparison
   ///
@@ -1055,7 +1056,9 @@ class APInt {
   /// the validity of the less-than relationship.
   ///
   /// \returns true if *this < RHS when considered signed.
-  bool slt(uint64_t RHS) const { return slt(APInt(getBitWidth(), RHS)); }
+  bool slt(int64_t RHS) const {
+    return getMinSignedBits() > 64 ? isNegative() : getSExtValue() < RHS;
+  }
 
   /// \brief Unsigned less or equal comparison
   ///
@@ -1071,7 +1074,7 @@ class APInt {
   /// the validity of the less-or-equal relationship.
   ///
   /// \returns true if *this <= RHS when considered unsigned.
-  bool ule(uint64_t RHS) const { return ule(APInt(getBitWidth(), RHS)); }
+  bool ule(uint64_t RHS) const { return !ugt(RHS); }
 
   /// \brief Signed less or equal comparison
   ///
@@ -1087,7 +1090,7 @@ class APInt {
   /// validity of the less-or-equal relationship.
   ///
   /// \returns true if *this <= RHS when considered signed.
-  bool sle(uint64_t RHS) const { return sle(APInt(getBitWidth(), RHS)); }
+  bool sle(uint64_t RHS) const { return !sgt(RHS); }
 
   /// \brief Unsigned greather than comparison
   ///
@@ -1103,7 +1106,9 @@ class APInt {
   /// the validity of the greater-than relationship.
   ///
   /// \returns true if *this > RHS when considered unsigned.
-  bool ugt(uint64_t RHS) const { return ugt(APInt(getBitWidth(), RHS)); }
+  bool ugt(uint64_t RHS) const {
+    return getActiveBits() > 64 ? true : getZExtValue() > RHS;
+  }
 
   /// \brief Signed greather than comparison
   ///
@@ -1119,7 +1124,9 @@ class APInt {
   /// the validity of the greater-than relationship.
   ///
   /// \returns true if *this > RHS when considered signed.
-  bool sgt(uint64_t RHS) const { return sgt(APInt(getBitWidth(), RHS)); }
+  bool sgt(int64_t RHS) const {
+    return getMinSignedBits() > 64 ? !isNegative() : getSExtValue() > RHS;
+  }
 
   /// \brief Unsigned greater or equal comparison
   ///
@@ -1135,7 +1142,7 @@ class APInt {
   /// the validity of the greater-or-equal relationship.
   ///
   /// \returns true if *this >= RHS when considered unsigned.
-  bool uge(uint64_t RHS) const { return uge(APInt(getBitWidth(), RHS)); }
+  bool uge(uint64_t RHS) const { return !ult(RHS); }
 
   /// \brief Signed greather or equal comparison
   ///
@@ -1151,7 +1158,7 @@ class APInt {
   /// the validity of the greater-or-equal relationship.
   ///
   /// \returns true if *this >= RHS when considered signed.
-  bool sge(uint64_t RHS) const { return sge(APInt(getBitWidth(), RHS)); }
+  bool sge(int64_t RHS) const { return !slt(RHS); }
 
   /// This operation tests if there are any pairs of corresponding bits
   /// between this APInt and RHS that are both set.
diff --git a/include/llvm/ADT/APSInt.h b/include/llvm/ADT/APSInt.h
index 91ccda22f2f..a187515f859 100644
--- a/include/llvm/ADT/APSInt.h
+++ b/include/llvm/ADT/APSInt.h
@@ -33,6 +33,15 @@ class APSInt : public APInt {
   explicit APSInt(APInt I, bool isUnsigned = true)
    : APInt(std::move(I)), IsUnsigned(isUnsigned) {}
 
+  /// Construct an APSInt from a string representation.
+  ///
+  /// This constructor interprets the string \p Str using the radix of 10.
+  /// The interpretation stops at the end of the string. The bit width of the
+  /// constructed APSInt is determined automatically.
+  ///
+  /// \param Str the string to be interpreted.
+  explicit APSInt(StringRef Str);
+
   APSInt &operator=(APInt RHS) {
     // Retain our current sign.
     APInt::operator=(std::move(RHS));
diff --git a/include/llvm/ADT/ArrayRef.h b/include/llvm/ADT/ArrayRef.h
index f7b055e3a06..f6cc4f4e7c3 100644
--- a/include/llvm/ADT/ArrayRef.h
+++ b/include/llvm/ADT/ArrayRef.h
@@ -85,7 +85,7 @@ namespace llvm {
 
     /// Construct an ArrayRef from a std::initializer_list.
     /*implicit*/ ArrayRef(const std::initializer_list<T> &Vec)
-    : Data(Vec.begin() == Vec.end() ? (T*)0 : Vec.begin()),
+    : Data(Vec.begin() == Vec.end() ? (T*)nullptr : Vec.begin()),
       Length(Vec.size()) {}
 
     /// Construct an ArrayRef<const T*> from ArrayRef<T*>. This uses SFINAE to
@@ -96,6 +96,25 @@ namespace llvm {
                  std::is_convertible<U *const *, T const *>::value>::type* = 0)
       : Data(A.data()), Length(A.size()) {}
 
+    /// Construct an ArrayRef<const T*> from a SmallVector<T*>. This is
+    /// templated in order to avoid instantiating SmallVectorTemplateCommon<T>
+    /// whenever we copy-construct an ArrayRef.
+    template<typename U, typename DummyT>
+    /*implicit*/ ArrayRef(const SmallVectorTemplateCommon<U*, DummyT> &Vec,
+                          typename std::enable_if<
+                              std::is_convertible<U *const *,
+                                                  T const *>::value>::type* = 0)
+      : Data(Vec.data()), Length(Vec.size()) {
+    }
+
+    /// Construct an ArrayRef<const T*> from std::vector<T*>. This uses SFINAE
+    /// to ensure that only vectors of pointers can be converted.
+    template<typename U, typename A>
+    ArrayRef(const std::vector<U *, A> &Vec,
+             typename std::enable_if<
+                 std::is_convertible<U *const *, T const *>::value>::type* = 0)
+      : Data(Vec.data()), Length(Vec.size()) {}
+
     /// @}
     /// @name Simple Operations
     /// @{
@@ -267,6 +286,11 @@ namespace llvm {
       return MutableArrayRef<T>(data()+N, M);
     }
 
+    MutableArrayRef<T> drop_back(unsigned N) const {
+      assert(this->size() >= N && "Dropping more elements than exist");
+      return slice(0, this->size() - N);
+    }
+
     /// @}
     /// @name Operator Overloads
     /// @{
diff --git a/include/llvm/ADT/DenseMap.h b/include/llvm/ADT/DenseMap.h
index 9c2365af6eb..27f73157a29 100644
--- a/include/llvm/ADT/DenseMap.h
+++ b/include/llvm/ADT/DenseMap.h
@@ -272,10 +272,6 @@ class DenseMapBase : public DebugEpochBase {
         P->getSecond().~ValueT();
       P->getFirst().~KeyT();
     }
-
-#ifndef NDEBUG
-    memset((void*)getBuckets(), 0x5a, sizeof(BucketT)*getNumBuckets());
-#endif
   }
 
   void initEmpty() {
@@ -312,12 +308,6 @@ class DenseMapBase : public DebugEpochBase {
       }
       B->getFirst().~KeyT();
     }
-
-#ifndef NDEBUG
-    if (OldBucketsBegin != OldBucketsEnd)
-      memset((void*)OldBucketsBegin, 0x5a,
-             sizeof(BucketT) * (OldBucketsEnd - OldBucketsBegin));
-#endif
   }
 
   template <typename OtherBaseT>
diff --git a/include/llvm/ADT/DenseMapInfo.h b/include/llvm/ADT/DenseMapInfo.h
index 6f17a647b63..b0a05307207 100644
--- a/include/llvm/ADT/DenseMapInfo.h
+++ b/include/llvm/ADT/DenseMapInfo.h
@@ -14,6 +14,8 @@
 #ifndef LLVM_ADT_DENSEMAPINFO_H
 #define LLVM_ADT_DENSEMAPINFO_H
 
+#include "llvm/ADT/Hashing.h"
+#include "llvm/ADT/StringRef.h"
 #include "llvm/Support/PointerLikeTypeTraits.h"
 #include "llvm/Support/type_traits.h"
 
@@ -163,6 +165,31 @@ struct DenseMapInfo<std::pair<T, U> > {
   }
 };
 
+// Provide DenseMapInfo for StringRefs.
+template <> struct DenseMapInfo<StringRef> {
+  static inline StringRef getEmptyKey() {
+    return StringRef(reinterpret_cast<const char *>(~static_cast<uintptr_t>(0)),
+                     0);
+  }
+  static inline StringRef getTombstoneKey() {
+    return StringRef(reinterpret_cast<const char *>(~static_cast<uintptr_t>(1)),
+                     0);
+  }
+  static unsigned getHashValue(StringRef Val) {
+    assert(Val.data() != getEmptyKey().data() && "Cannot hash the empty key!");
+    assert(Val.data() != getTombstoneKey().data() &&
+           "Cannot hash the tombstone key!");
+    return (unsigned)(hash_value(Val));
+  }
+  static bool isEqual(StringRef LHS, StringRef RHS) {
+    if (RHS.data() == getEmptyKey().data())
+      return LHS.data() == getEmptyKey().data();
+    if (RHS.data() == getTombstoneKey().data())
+      return LHS.data() == getTombstoneKey().data();
+    return LHS == RHS;
+  }
+};
+
 } // end namespace llvm
 
 #endif
diff --git a/include/llvm/ADT/DepthFirstIterator.h b/include/llvm/ADT/DepthFirstIterator.h
index e9668c45d10..d79b9acacfa 100644
--- a/include/llvm/ADT/DepthFirstIterator.h
+++ b/include/llvm/ADT/DepthFirstIterator.h
@@ -159,8 +159,10 @@ class df_iterator : public std::iterator<std::forward_iterator_tag,
     return *this;
   }
 
-  // skips all children of the current node and traverses to next node
-  //
+  /// \brief Skips all children of the current node and traverses to next node
+  ///
+  /// Note: This function takes care of incrementing the iterator. If you
+  /// always increment and call this function, you risk walking off the end.
   df_iterator &skipChildren() {
     VisitStack.pop_back();
     if (!VisitStack.empty())
diff --git a/include/llvm/ADT/Hashing.h b/include/llvm/ADT/Hashing.h
index 77e6d77b1b8..de56f91eddb 100644
--- a/include/llvm/ADT/Hashing.h
+++ b/include/llvm/ADT/Hashing.h
@@ -53,6 +53,7 @@
 #include <cassert>
 #include <cstring>
 #include <iterator>
+#include <string>
 #include <utility>
 
 namespace llvm {
diff --git a/include/llvm/ADT/STLExtras.h b/include/llvm/ADT/STLExtras.h
index 921bd820d97..fd4412c4166 100644
--- a/include/llvm/ADT/STLExtras.h
+++ b/include/llvm/ADT/STLExtras.h
@@ -18,6 +18,7 @@
 #define LLVM_ADT_STLEXTRAS_H
 
 #include "llvm/Support/Compiler.h"
+#include <algorithm> // for std::all_of
 #include <cassert>
 #include <cstddef> // for std::size_t
 #include <cstdlib> // for qsort
@@ -195,6 +196,41 @@ inline mapped_iterator<ItTy, FuncTy> map_iterator(const ItTy &I, FuncTy F) {
   return mapped_iterator<ItTy, FuncTy>(I, F);
 }
 
+/// \brief Metafunction to determine if type T has a member called rbegin().
+template <typename T> struct has_rbegin {
+  template <typename U> static char(&f(const U &, decltype(&U::rbegin)))[1];
+  static char(&f(...))[2];
+  const static bool value = sizeof(f(std::declval<T>(), nullptr)) == 1;
+};
+
+// Returns an iterator_range over the given container which iterates in reverse.
+// Note that the container must have rbegin()/rend() methods for this to work.
+template <typename ContainerTy>
+auto reverse(ContainerTy &&C,
+             typename std::enable_if<has_rbegin<ContainerTy>::value>::type * =
+                 nullptr) -> decltype(make_range(C.rbegin(), C.rend())) {
+  return make_range(C.rbegin(), C.rend());
+}
+
+// Returns a std::reverse_iterator wrapped around the given iterator.
+template <typename IteratorTy>
+std::reverse_iterator<IteratorTy> make_reverse_iterator(IteratorTy It) {
+  return std::reverse_iterator<IteratorTy>(It);
+}
+
+// Returns an iterator_range over the given container which iterates in reverse.
+// Note that the container must have begin()/end() methods which return
+// bidirectional iterators for this to work.
+template <typename ContainerTy>
+auto reverse(
+    ContainerTy &&C,
+    typename std::enable_if<!has_rbegin<ContainerTy>::value>::type * = nullptr)
+    -> decltype(make_range(llvm::make_reverse_iterator(std::end(C)),
+                           llvm::make_reverse_iterator(std::begin(C)))) {
+  return make_range(llvm::make_reverse_iterator(std::end(C)),
+                    llvm::make_reverse_iterator(std::begin(C)));
+}
+
 //===----------------------------------------------------------------------===//
 //     Extra additions to <utility>
 //===----------------------------------------------------------------------===//
@@ -327,6 +363,14 @@ void DeleteContainerSeconds(Container &C) {
   C.clear();
 }
 
+/// Provide wrappers to std::all_of which take ranges instead of having to pass
+/// being/end explicitly.
+template<typename R, class UnaryPredicate>
+bool all_of(R &&Range, UnaryPredicate &&P) {
+  return std::all_of(Range.begin(), Range.end(),
+                     std::forward<UnaryPredicate>(P));
+}
+
 //===----------------------------------------------------------------------===//
 //     Extra additions to <memory>
 //===----------------------------------------------------------------------===//
diff --git a/include/llvm/ADT/SetVector.h b/include/llvm/ADT/SetVector.h
index a7fd408c854..2125b274f53 100644
--- a/include/llvm/ADT/SetVector.h
+++ b/include/llvm/ADT/SetVector.h
@@ -44,6 +44,8 @@ class SetVector {
   typedef Vector vector_type;
   typedef typename vector_type::const_iterator iterator;
   typedef typename vector_type::const_iterator const_iterator;
+  typedef typename vector_type::const_reverse_iterator reverse_iterator;
+  typedef typename vector_type::const_reverse_iterator const_reverse_iterator;
   typedef typename vector_type::size_type size_type;
 
   /// \brief Construct an empty SetVector
@@ -85,6 +87,26 @@ class SetVector {
     return vector_.end();
   }
 
+  /// \brief Get an reverse_iterator to the end of the SetVector.
+  reverse_iterator rbegin() {
+    return vector_.rbegin();
+  }
+
+  /// \brief Get a const_reverse_iterator to the end of the SetVector.
+  const_reverse_iterator rbegin() const {
+    return vector_.rbegin();
+  }
+
+  /// \brief Get a reverse_iterator to the beginning of the SetVector.
+  reverse_iterator rend() {
+    return vector_.rend();
+  }
+
+  /// \brief Get a const_reverse_iterator to the beginning of the SetVector.
+  const_reverse_iterator rend() const {
+    return vector_.rend();
+  }
+
   /// \brief Return the last element of the SetVector.
   const T &back() const {
     assert(!empty() && "Cannot call back() on empty SetVector!");
diff --git a/include/llvm/ADT/SparseBitVector.h b/include/llvm/ADT/SparseBitVector.h
index 20cbe2cddfc..bf7e1544c81 100644
--- a/include/llvm/ADT/SparseBitVector.h
+++ b/include/llvm/ADT/SparseBitVector.h
@@ -453,6 +453,9 @@ class SparseBitVector {
 
   // Assignment
   SparseBitVector& operator=(const SparseBitVector& RHS) {
+    if (this == &RHS)
+      return *this;
+
     Elements.clear();
 
     ElementListConstIter ElementIter = RHS.Elements.begin();
@@ -559,6 +562,9 @@ class SparseBitVector {
 
   // Union our bitmap with the RHS and return true if we changed.
   bool operator|=(const SparseBitVector &RHS) {
+    if (this == &RHS)
+      return false;
+
     bool changed = false;
     ElementListIter Iter1 = Elements.begin();
     ElementListConstIter Iter2 = RHS.Elements.begin();
@@ -587,6 +593,9 @@ class SparseBitVector {
 
   // Intersect our bitmap with the RHS and return true if ours changed.
   bool operator&=(const SparseBitVector &RHS) {
+    if (this == &RHS)
+      return false;
+
     bool changed = false;
     ElementListIter Iter1 = Elements.begin();
     ElementListConstIter Iter2 = RHS.Elements.begin();
@@ -619,9 +628,13 @@ class SparseBitVector {
         ElementListIter IterTmp = Iter1;
         ++Iter1;
         Elements.erase(IterTmp);
+        changed = true;
       }
     }
-    Elements.erase(Iter1, Elements.end());
+    if (Iter1 != Elements.end()) {
+      Elements.erase(Iter1, Elements.end());
+      changed = true;
+    }
     CurrElementIter = Elements.begin();
     return changed;
   }
@@ -629,6 +642,14 @@ class SparseBitVector {
   // Intersect our bitmap with the complement of the RHS and return true
   // if ours changed.
   bool intersectWithComplement(const SparseBitVector &RHS) {
+    if (this == &RHS) {
+      if (!empty()) {
+        clear();
+        return true;
+      }
+      return false;
+    }
+
     bool changed = false;
     ElementListIter Iter1 = Elements.begin();
     ElementListConstIter Iter2 = RHS.Elements.begin();
@@ -675,6 +696,15 @@ class SparseBitVector {
   void intersectWithComplement(const SparseBitVector<ElementSize> &RHS1,
                                const SparseBitVector<ElementSize> &RHS2)
   {
+    if (this == &RHS1) {
+      intersectWithComplement(RHS2);
+      return;
+    } else if (this == &RHS2) {
+      SparseBitVector RHS2Copy(RHS2);
+      intersectWithComplement(RHS1, RHS2Copy);
+      return;
+    }
+
     Elements.clear();
     CurrElementIter = Elements.begin();
     ElementListConstIter Iter1 = RHS1.Elements.begin();
diff --git a/include/llvm/ADT/Triple.h b/include/llvm/ADT/Triple.h
index 7621796fb50..fff287e4910 100644
--- a/include/llvm/ADT/Triple.h
+++ b/include/llvm/ADT/Triple.h
@@ -50,7 +50,8 @@ class Triple {
     armeb,      // ARM (big endian): armeb
     aarch64,    // AArch64 (little endian): aarch64
     aarch64_be, // AArch64 (big endian): aarch64_be
-    bpf,        // eBPF or extended BPF or 64-bit BPF (little endian)
+    bpfel,      // eBPF or extended BPF or 64-bit BPF (little endian)
+    bpfeb,      // eBPF or extended BPF or 64-bit BPF (big endian)
     hexagon,    // Hexagon: hexagon
     mips,       // MIPS: mips, mipsallegrex
     mipsel,     // MIPSEL: mipsel, mipsallegrexel
@@ -64,6 +65,7 @@ class Triple {
     amdgcn,     // AMDGCN: AMD GCN GPUs
     sparc,      // Sparc: sparc
     sparcv9,    // Sparcv9: Sparcv9
+    sparcel,    // Sparc: (endianness = little). NB: 'Sparcle' is a CPU variant
     systemz,    // SystemZ: s390x
     tce,        // TCE (http://tce.cs.tut.fi/): tce
     thumb,      // Thumb (little endian): thumb, thumbv.*
@@ -82,7 +84,11 @@ class Triple {
     hsail64,    // AMD HSAIL with 64-bit pointers
     spir,       // SPIR: standard portable IR for OpenCL 32-bit version
     spir64,     // SPIR: standard portable IR for OpenCL 64-bit version
-    kalimba     // Kalimba: generic kalimba
+    kalimba,    // Kalimba: generic kalimba
+    shave,      // SHAVE: Movidius vector VLIW processors
+    wasm32,     // WebAssembly with 32-bit pointers
+    wasm64,     // WebAssembly with 64-bit pointers
+    LastArchType = wasm64
   };
   enum SubArchType {
     NoSubArch,
@@ -118,7 +124,8 @@ class Triple {
     ImaginationTechnologies,
     MipsTechnologies,
     NVIDIA,
-    CSR
+    CSR,
+    LastVendorType = CSR
   };
   enum OSType {
     UnknownOS,
@@ -147,7 +154,8 @@ class Triple {
     CUDA,       // NVIDIA CUDA
     NVCL,       // NVIDIA OpenCL
     AMDHSA,     // AMD HSA Runtime
-    PS4
+    PS4,
+    LastOSType = PS4
   };
   enum EnvironmentType {
     UnknownEnvironment,
@@ -164,6 +172,8 @@ class Triple {
     MSVC,
     Itanium,
     Cygnus,
+    AMDOpenCL,
+    LastEnvironmentType = AMDOpenCL
   };
   enum ObjectFormatType {
     UnknownObjectFormat,
@@ -240,21 +250,9 @@ class Triple {
   /// getVendor - Get the parsed vendor type of this triple.
   VendorType getVendor() const { return Vendor; }
 
-  // @LOCALMOD-BEGIN -- hardcode NaCl for NaCl builds, to help
-  // prune OS-specific code that is litered all over and not
-  // cleanly separated.
-#if defined(__native_client__)
-  OSType getOS() const { return NaCl; }
-  ObjectFormatType getObjectFormat() const { return ELF; }
-#else
   /// getOS - Get the parsed operating system type of this triple.
   OSType getOS() const { return OS; }
 
-  /// getFormat - Get the object format for this triple.
-  ObjectFormatType getObjectFormat() const { return ObjectFormat; }
-#endif
-  // @LOCALMOD-END
-
   /// hasEnvironment - Does this triple have the optional environment
   /// (fourth) component?
   bool hasEnvironment() const {
@@ -264,6 +262,18 @@ class Triple {
   /// getEnvironment - Get the parsed environment type of this triple.
   EnvironmentType getEnvironment() const { return Environment; }
 
+  /// \brief Parse the version number from the OS name component of the
+  /// triple, if present.
+  ///
+  /// For example, "fooos1.2.3" would return (1, 2, 3).
+  ///
+  /// If an entry is not defined, it will be returned as 0.
+  void getEnvironmentVersion(unsigned &Major, unsigned &Minor,
+                             unsigned &Micro) const;
+
+  /// getFormat - Get the object format for this triple.
+  ObjectFormatType getObjectFormat() const { return ObjectFormat; }
+
   /// getOSVersion - Parse the version number from the OS name component of the
   /// triple, if present.
   ///
@@ -382,62 +392,6 @@ class Triple {
     return isOSVersionLT(Minor + 4, Micro, 0);
   }
 
-  // @LOCALMOD-BEGIN: Hardcode OS predicates to help prune code
-  // that is OS-specific, but not cleanly separated.
-  // Perhaps this would be cleaner if Triple.h was partly Table-gen'ed.
-#if defined(__native_client__)
-  bool isMacOSX() const { return false; }
-  bool isiOS() const { return false; }
-  bool isOSDarwin() const { return false; }
-  bool isOSNetBSD() const { return false; }
-  bool isOSOpenBSD() const { return false; }
-  bool isOSFreeBSD() const { return false; }
-  bool isOSDragonFly() const { return false; }
-  bool isOSSolaris() const { return false; }
-  bool isOSBitrig() const { return false; }
-  bool isWindowsMSVCEnvironment() const { return false; }
-  bool isKnownWindowsMSVCEnvironment() const { return false; }
-  bool isWindowsItaniumEnvironment() const { return false; }
-  bool isWindowsCygwinEnvironment() const { return false; }
-  bool isWindowsGNUEnvironment() const { return false; }
-  bool isOSCygMing() const { return false; }
-  bool isOSMSVCRT() const { return false; }
-  bool isOSWindows() const { return false; }
-  bool isOSNaCl() const { return true; }
-  bool isOSEmscripten() const { return false; }
-  bool isOSLinux() const { return false; }
-  bool isOSBinFormatELF() const { return true; }
-  bool isOSBinFormatCOFF() const { return false; }
-  bool isOSBinFormatMachO() const { return false; }
-  bool isPS4CPU() const { return false; }
-  bool isPS4() const { return false; }
-#elif defined(__EMSCRIPTEN__)
-  bool isMacOSX() const { return false; }
-  bool isiOS() const { return false; }
-  bool isOSDarwin() const { return false; }
-  bool isOSNetBSD() const { return false; }
-  bool isOSOpenBSD() const { return false; }
-  bool isOSFreeBSD() const { return false; }
-  bool isOSDragonFly() const { return false; }
-  bool isOSSolaris() const { return false; }
-  bool isOSBitrig() const { return false; }
-  bool isWindowsMSVCEnvironment() const { return false; }
-  bool isKnownWindowsMSVCEnvironment() const { return false; }
-  bool isWindowsItaniumEnvironment() const { return false; }
-  bool isWindowsCygwinEnvironment() const { return false; }
-  bool isWindowsGNUEnvironment() const { return false; }
-  bool isOSCygMing() const { return false; }
-  bool isOSMSVCRT() const { return false; }
-  bool isOSWindows() const { return false; }
-  bool isOSNaCl() const { return false; }
-  bool isOSEmscripten() const { return true; }
-  bool isOSLinux() const { return false; }
-  bool isOSBinFormatELF() const { return true; }
-  bool isOSBinFormatCOFF() const { return false; }
-  bool isOSBinFormatMachO() const { return false; }
-  bool isPS4CPU() const { return false; }
-  bool isPS4() const { return false; }
-#else
   /// isMacOSX - Is this a Mac OS X triple. For legacy reasons, we support both
   /// "darwin" and "osx" as OS X triples.
   bool isMacOSX() const {
@@ -558,8 +512,6 @@ class Triple {
     return getVendor() == Triple::SCEI &&
            getOS() == Triple::PS4;
   }
-#endif
-  // @LOCALMOD-END
 
   /// @}
   /// @name Mutators
@@ -627,6 +579,22 @@ class Triple {
   ///          architecture if no such variant can be found.
   llvm::Triple get64BitArchVariant() const;
 
+  /// Form a triple with a big endian variant of the current architecture.
+  ///
+  /// This can be used to move across "families" of architectures where useful.
+  ///
+  /// \returns A new triple with a big endian architecture or an unknown
+  ///          architecture if no such variant can be found.
+  llvm::Triple getBigEndianArchVariant() const;
+
+  /// Form a triple with a little endian variant of the current architecture.
+  ///
+  /// This can be used to move across "families" of architectures where useful.
+  ///
+  /// \returns A new triple with a little endian architecture or an unknown
+  ///          architecture if no such variant can be found.
+  llvm::Triple getLittleEndianArchVariant() const;
+
   /// Get the (LLVM) name of the minimum ARM CPU for the arch we are targeting.
   ///
   /// \param Arch the architecture name (e.g., "armv7s"). If it is an empty
diff --git a/include/llvm/ADT/Twine.h b/include/llvm/ADT/Twine.h
index fbe5b655ae7..db0bf4b68de 100644
--- a/include/llvm/ADT/Twine.h
+++ b/include/llvm/ADT/Twine.h
@@ -10,7 +10,7 @@
 #ifndef LLVM_ADT_TWINE_H
 #define LLVM_ADT_TWINE_H
 
-#include "llvm/ADT/SmallVector.h"
+#include "llvm/ADT/SmallVector.h"
 #include "llvm/ADT/StringRef.h"
 #include "llvm/Support/DataTypes.h"
 #include "llvm/Support/ErrorHandling.h"
@@ -137,7 +137,7 @@ namespace llvm {
       const char *cString;
       const std::string *stdString;
       const StringRef *stringRef;
-      const SmallVectorImpl<char> *smallString;
+      const SmallVectorImpl<char> *smallString;
       char character;
       unsigned int decUI;
       int decI;
@@ -410,7 +410,7 @@ namespace llvm {
       case CStringKind:
       case StdStringKind:
       case StringRefKind:
-      case SmallStringKind:
+      case SmallStringKind:
         return true;
       default:
         return false;
diff --git a/include/llvm/ADT/edit_distance.h b/include/llvm/ADT/edit_distance.h
index 9ee1edc54e0..06a01b18a9f 100644
--- a/include/llvm/ADT/edit_distance.h
+++ b/include/llvm/ADT/edit_distance.h
@@ -50,50 +50,51 @@ unsigned ComputeEditDistance(ArrayRef<T> FromArray, ArrayRef<T> ToArray,
   //   http://en.wikipedia.org/wiki/Levenshtein_distance
   //
   // Although the algorithm is typically described using an m x n
-  // array, only two rows are used at a time, so this implemenation
-  // just keeps two separate vectors for those two rows.
+  // array, only one row plus one element are used at a time, so this
+  // implementation just keeps one vector for the row.  To update one entry,
+  // only the entries to the left, top, and top-left are needed.  The left
+  // entry is in Row[x-1], the top entry is what's in Row[x] from the last
+  // iteration, and the top-left entry is stored in Previous.
   typename ArrayRef<T>::size_type m = FromArray.size();
   typename ArrayRef<T>::size_type n = ToArray.size();
 
   const unsigned SmallBufferSize = 64;
   unsigned SmallBuffer[SmallBufferSize];
   std::unique_ptr<unsigned[]> Allocated;
-  unsigned *Previous = SmallBuffer;
-  if (2*(n + 1) > SmallBufferSize) {
-    Previous = new unsigned [2*(n+1)];
-    Allocated.reset(Previous);
+  unsigned *Row = SmallBuffer;
+  if (n + 1 > SmallBufferSize) {
+    Row = new unsigned[n + 1];
+    Allocated.reset(Row);
   }
-  unsigned *Current = Previous + (n + 1);
 
-  for (unsigned i = 0; i <= n; ++i)
-    Previous[i] = i;
+  for (unsigned i = 1; i <= n; ++i)
+    Row[i] = i;
 
   for (typename ArrayRef<T>::size_type y = 1; y <= m; ++y) {
-    Current[0] = y;
-    unsigned BestThisRow = Current[0];
+    Row[0] = y;
+    unsigned BestThisRow = Row[0];
 
+    unsigned Previous = y - 1;
     for (typename ArrayRef<T>::size_type x = 1; x <= n; ++x) {
+      int OldRow = Row[x];
       if (AllowReplacements) {
-        Current[x] = std::min(
-            Previous[x-1] + (FromArray[y-1] == ToArray[x-1] ? 0u : 1u),
-            std::min(Current[x-1], Previous[x])+1);
+        Row[x] = std::min(
+            Previous + (FromArray[y-1] == ToArray[x-1] ? 0u : 1u),
+            std::min(Row[x-1], Row[x])+1);
       }
       else {
-        if (FromArray[y-1] == ToArray[x-1]) Current[x] = Previous[x-1];
-        else Current[x] = std::min(Current[x-1], Previous[x]) + 1;
+        if (FromArray[y-1] == ToArray[x-1]) Row[x] = Previous;
+        else Row[x] = std::min(Row[x-1], Row[x]) + 1;
       }
-      BestThisRow = std::min(BestThisRow, Current[x]);
+      Previous = OldRow;
+      BestThisRow = std::min(BestThisRow, Row[x]);
     }
 
     if (MaxEditDistance && BestThisRow > MaxEditDistance)
       return MaxEditDistance + 1;
-
-    unsigned *tmp = Current;
-    Current = Previous;
-    Previous = tmp;
   }
 
-  unsigned Result = Previous[n];
+  unsigned Result = Row[n];
   return Result;
 }
 
diff --git a/include/llvm/ADT/iterator.h b/include/llvm/ADT/iterator.h
index 54a288df017..c3079289270 100644
--- a/include/llvm/ADT/iterator.h
+++ b/include/llvm/ADT/iterator.h
@@ -162,6 +162,8 @@ class iterator_adaptor_base
           int>::type = 0)
       : I(std::forward<U &&>(u)) {}
 
+  const WrappedIteratorT &wrapped() const { return I; }
+
 public:
   typedef DifferenceTypeT difference_type;
 
diff --git a/include/llvm/ADT/iterator_range.h b/include/llvm/ADT/iterator_range.h
index 523a86f02e0..3dd679bd9b7 100644
--- a/include/llvm/ADT/iterator_range.h
+++ b/include/llvm/ADT/iterator_range.h
@@ -20,6 +20,7 @@
 #define LLVM_ADT_ITERATOR_RANGE_H
 
 #include <utility>
+#include <iterator>
 
 namespace llvm {
 
@@ -32,6 +33,12 @@ class iterator_range {
   IteratorT begin_iterator, end_iterator;
 
 public:
+  //TODO: Add SFINAE to test that the Container's iterators match the range's
+  //      iterators.
+  template <typename Container>
+  iterator_range(Container &&c)
+  //TODO: Consider ADL/non-member begin/end calls.
+      : begin_iterator(c.begin()), end_iterator(c.end()) {}
   iterator_range(IteratorT begin_iterator, IteratorT end_iterator)
       : begin_iterator(std::move(begin_iterator)),
         end_iterator(std::move(end_iterator)) {}
@@ -51,6 +58,11 @@ template <class T> iterator_range<T> make_range(T x, T y) {
 template <typename T> iterator_range<T> make_range(std::pair<T, T> p) {
   return iterator_range<T>(std::move(p.first), std::move(p.second));
 }
+
+template<typename T>
+iterator_range<decltype(begin(std::declval<T>()))> drop_begin(T &&t, int n) {
+  return make_range(std::next(begin(t), n), end(t));
+}
 }
 
 #endif
diff --git a/include/llvm/Analysis/AliasAnalysis.h b/include/llvm/Analysis/AliasAnalysis.h
index 6999bd1a029..e76f7771d4d 100644
--- a/include/llvm/Analysis/AliasAnalysis.h
+++ b/include/llvm/Analysis/AliasAnalysis.h
@@ -16,11 +16,12 @@
 // which automatically provides functionality for the entire suite of client
 // APIs.
 //
-// This API identifies memory regions with the Location class. The pointer
+// This API identifies memory regions with the MemoryLocation class. The pointer
 // component specifies the base memory address of the region. The Size specifies
-// the maximum size (in address units) of the memory region, or UnknownSize if
-// the size is not known. The TBAA tag identifies the "type" of the memory
-// reference; see the TypeBasedAliasAnalysis class for details.
+// the maximum size (in address units) of the memory region, or
+// MemoryLocation::UnknownSize if the size is not known. The TBAA tag
+// identifies the "type" of the memory reference; see the
+// TypeBasedAliasAnalysis class for details.
 //
 // Some non-obvious details include:
 //  - Pointers that point to two completely different objects in memory never
@@ -40,6 +41,7 @@
 #include "llvm/ADT/DenseMap.h"
 #include "llvm/IR/CallSite.h"
 #include "llvm/IR/Metadata.h"
+#include "llvm/Analysis/MemoryLocation.h"
 
 namespace llvm {
 
@@ -53,6 +55,106 @@ class AnalysisUsage;
 class MemTransferInst;
 class MemIntrinsic;
 class DominatorTree;
+class OrderedBasicBlock;
+
+/// The possible results of an alias query.
+///
+/// These results are always computed between two MemoryLocation objects as
+/// a query to some alias analysis.
+///
+/// Note that these are unscoped enumerations because we would like to support
+/// implicitly testing a result for the existence of any possible aliasing with
+/// a conversion to bool, but an "enum class" doesn't support this. The
+/// canonical names from the literature are suffixed and unique anyways, and so
+/// they serve as global constants in LLVM for these results.
+///
+/// See docs/AliasAnalysis.html for more information on the specific meanings
+/// of these values.
+enum AliasResult {
+  /// The two locations do not alias at all.
+  ///
+  /// This value is arranged to convert to false, while all other values
+  /// convert to true. This allows a boolean context to convert the result to
+  /// a binary flag indicating whether there is the possibility of aliasing.
+  NoAlias = 0,
+  /// The two locations may or may not alias. This is the least precise result.
+  MayAlias,
+  /// The two locations alias, but only due to a partial overlap.
+  PartialAlias,
+  /// The two locations precisely alias each other.
+  MustAlias,
+};
+
+/// Flags indicating whether a memory access modifies or references memory.
+///
+/// This is no access at all, a modification, a reference, or both
+/// a modification and a reference. These are specifically structured such that
+/// they form a two bit matrix and bit-tests for 'mod' or 'ref' work with any
+/// of the possible values.
+enum ModRefInfo {
+  /// The access neither references nor modifies the value stored in memory.
+  MRI_NoModRef = 0,
+  /// The access references the value stored in memory.
+  MRI_Ref = 1,
+  /// The access modifies the value stored in memory.
+  MRI_Mod = 2,
+  /// The access both references and modifies the value stored in memory.
+  MRI_ModRef = MRI_Ref | MRI_Mod
+};
+
+/// The locations at which a function might access memory.
+///
+/// These are primarily used in conjunction with the \c AccessKind bits to
+/// describe both the nature of access and the locations of access for a
+/// function call.
+enum FunctionModRefLocation {
+  /// Base case is no access to memory.
+  FMRL_Nowhere = 0,
+  /// Access to memory via argument pointers.
+  FMRL_ArgumentPointees = 4,
+  /// Access to any memory.
+  FMRL_Anywhere = 8 | FMRL_ArgumentPointees
+};
+
+/// Summary of how a function affects memory in the program.
+///
+/// Loads from constant globals are not considered memory accesses for this
+/// interface. Also, functions may freely modify stack space local to their
+/// invocation without having to report it through these interfaces.
+enum FunctionModRefBehavior {
+  /// This function does not perform any non-local loads or stores to memory.
+  ///
+  /// This property corresponds to the GCC 'const' attribute.
+  /// This property corresponds to the LLVM IR 'readnone' attribute.
+  /// This property corresponds to the IntrNoMem LLVM intrinsic flag.
+  FMRB_DoesNotAccessMemory = FMRL_Nowhere | MRI_NoModRef,
+
+  /// The only memory references in this function (if it has any) are
+  /// non-volatile loads from objects pointed to by its pointer-typed
+  /// arguments, with arbitrary offsets.
+  ///
+  /// This property corresponds to the IntrReadArgMem LLVM intrinsic flag.
+  FMRB_OnlyReadsArgumentPointees = FMRL_ArgumentPointees | MRI_Ref,
+
+  /// The only memory references in this function (if it has any) are
+  /// non-volatile loads and stores from objects pointed to by its
+  /// pointer-typed arguments, with arbitrary offsets.
+  ///
+  /// This property corresponds to the IntrReadWriteArgMem LLVM intrinsic flag.
+  FMRB_OnlyAccessesArgumentPointees = FMRL_ArgumentPointees | MRI_ModRef,
+
+  /// This function does not perform any non-local stores or volatile loads,
+  /// but may read from any memory location.
+  ///
+  /// This property corresponds to the GCC 'pure' attribute.
+  /// This property corresponds to the LLVM IR 'readonly' attribute.
+  /// This property corresponds to the IntrReadMem LLVM intrinsic flag.
+  FMRB_OnlyReadsMemory = FMRL_Anywhere | MRI_Ref,
+
+  /// This indicates that the function could not be classified into one of the
+  /// behaviors above.
+  FMRB_UnknownModRefBehavior = FMRL_Anywhere | MRI_ModRef
+};
 
 class AliasAnalysis {
 protected:
@@ -79,11 +181,6 @@ class AliasAnalysis {
   AliasAnalysis() : DL(nullptr), TLI(nullptr), AA(nullptr) {}
   virtual ~AliasAnalysis();  // We want to be subclassed
 
-  /// UnknownSize - This is a special value which can be used with the
-  /// size arguments in alias queries to indicate that the caller does not
-  /// know the sizes of the potential memory references.
-  static uint64_t const UnknownSize = ~UINT64_C(0);
-
   /// getTargetLibraryInfo - Return a pointer to the current TargetLibraryInfo
   /// object, or null if no TargetLibraryInfo object is available.
   ///
@@ -95,529 +192,360 @@ class AliasAnalysis {
   uint64_t getTypeStoreSize(Type *Ty);
 
   //===--------------------------------------------------------------------===//
-  /// Alias Queries...
-  ///
-
-  /// Location - A description of a memory location.
-  struct Location {
-    /// Ptr - The address of the start of the location.
-    const Value *Ptr;
-    /// Size - The maximum size of the location, in address-units, or
-    /// UnknownSize if the size is not known.  Note that an unknown size does
-    /// not mean the pointer aliases the entire virtual address space, because
-    /// there are restrictions on stepping out of one object and into another.
-    /// See http://llvm.org/docs/LangRef.html#pointeraliasing
-    uint64_t Size;
-    /// AATags - The metadata nodes which describes the aliasing of the
-    /// location (each member is null if that kind of information is
-    /// unavailable)..
-    AAMDNodes AATags;
-
-    explicit Location(const Value *P = nullptr, uint64_t S = UnknownSize,
-                      const AAMDNodes &N = AAMDNodes())
-      : Ptr(P), Size(S), AATags(N) {}
-
-    Location getWithNewPtr(const Value *NewPtr) const {
-      Location Copy(*this);
-      Copy.Ptr = NewPtr;
-      return Copy;
-    }
-
-    Location getWithNewSize(uint64_t NewSize) const {
-      Location Copy(*this);
-      Copy.Size = NewSize;
-      return Copy;
-    }
+  /// \name Alias Queries
+  /// @{
 
-    Location getWithoutAATags() const {
-      Location Copy(*this);
-      Copy.AATags = AAMDNodes();
-      return Copy;
-    }
-  };
-
-  /// getLocation - Fill in Loc with information about the memory reference by
-  /// the given instruction.
-  Location getLocation(const LoadInst *LI);
-  Location getLocation(const StoreInst *SI);
-  Location getLocation(const VAArgInst *VI);
-  Location getLocation(const AtomicCmpXchgInst *CXI);
-  Location getLocation(const AtomicRMWInst *RMWI);
-  static Location getLocationForSource(const MemTransferInst *MTI);
-  static Location getLocationForDest(const MemIntrinsic *MI);
-  Location getLocation(const Instruction *Inst) {
-    if (auto *I = dyn_cast<LoadInst>(Inst))
-      return getLocation(I);
-    else if (auto *I = dyn_cast<StoreInst>(Inst))
-      return getLocation(I);
-    else if (auto *I = dyn_cast<VAArgInst>(Inst))
-      return getLocation(I);
-    else if (auto *I = dyn_cast<AtomicCmpXchgInst>(Inst))
-      return getLocation(I);
-    else if (auto *I = dyn_cast<AtomicRMWInst>(Inst))
-      return getLocation(I);
-    llvm_unreachable("unsupported memory instruction");
-  }
-
-  /// Alias analysis result - Either we know for sure that it does not alias, we
-  /// know for sure it must alias, or we don't know anything: The two pointers
-  /// _might_ alias.  This enum is designed so you can do things like:
-  ///     if (AA.alias(P1, P2)) { ... }
-  /// to check to see if two pointers might alias.
-  ///
-  /// See docs/AliasAnalysis.html for more information on the specific meanings
-  /// of these values.
-  ///
-  enum AliasResult {
-    NoAlias = 0,        ///< No dependencies.
-    MayAlias,           ///< Anything goes.
-    PartialAlias,       ///< Pointers differ, but pointees overlap.
-    MustAlias           ///< Pointers are equal.
-  };
-
-  /// alias - The main low level interface to the alias analysis implementation.
+  /// The main low level interface to the alias analysis implementation.
   /// Returns an AliasResult indicating whether the two pointers are aliased to
-  /// each other.  This is the interface that must be implemented by specific
+  /// each other. This is the interface that must be implemented by specific
   /// alias analysis implementations.
-  virtual AliasResult alias(const Location &LocA, const Location &LocB);
+  virtual AliasResult alias(const MemoryLocation &LocA,
+                            const MemoryLocation &LocB);
 
-  /// alias - A convenience wrapper.
-  AliasResult alias(const Value *V1, uint64_t V1Size,
-                    const Value *V2, uint64_t V2Size) {
-    return alias(Location(V1, V1Size), Location(V2, V2Size));
+  /// A convenience wrapper around the primary \c alias interface.
+  AliasResult alias(const Value *V1, uint64_t V1Size, const Value *V2,
+                    uint64_t V2Size) {
+    return alias(MemoryLocation(V1, V1Size), MemoryLocation(V2, V2Size));
   }
 
-  /// alias - A convenience wrapper.
+  /// A convenience wrapper around the primary \c alias interface.
   AliasResult alias(const Value *V1, const Value *V2) {
-    return alias(V1, UnknownSize, V2, UnknownSize);
+    return alias(V1, MemoryLocation::UnknownSize, V2,
+                 MemoryLocation::UnknownSize);
   }
 
-  /// isNoAlias - A trivial helper function to check to see if the specified
-  /// pointers are no-alias.
-  bool isNoAlias(const Location &LocA, const Location &LocB) {
+  /// A trivial helper function to check to see if the specified pointers are
+  /// no-alias.
+  bool isNoAlias(const MemoryLocation &LocA, const MemoryLocation &LocB) {
     return alias(LocA, LocB) == NoAlias;
   }
 
-  /// isNoAlias - A convenience wrapper.
-  bool isNoAlias(const Value *V1, uint64_t V1Size,
-                 const Value *V2, uint64_t V2Size) {
-    return isNoAlias(Location(V1, V1Size), Location(V2, V2Size));
+  /// A convenience wrapper around the \c isNoAlias helper interface.
+  bool isNoAlias(const Value *V1, uint64_t V1Size, const Value *V2,
+                 uint64_t V2Size) {
+    return isNoAlias(MemoryLocation(V1, V1Size), MemoryLocation(V2, V2Size));
   }
-  
-  /// isNoAlias - A convenience wrapper.
+
+  /// A convenience wrapper around the \c isNoAlias helper interface.
   bool isNoAlias(const Value *V1, const Value *V2) {
-    return isNoAlias(Location(V1), Location(V2));
+    return isNoAlias(MemoryLocation(V1), MemoryLocation(V2));
   }
-  
-  /// isMustAlias - A convenience wrapper.
-  bool isMustAlias(const Location &LocA, const Location &LocB) {
+
+  /// A trivial helper function to check to see if the specified pointers are
+  /// must-alias.
+  bool isMustAlias(const MemoryLocation &LocA, const MemoryLocation &LocB) {
     return alias(LocA, LocB) == MustAlias;
   }
 
-  /// isMustAlias - A convenience wrapper.
+  /// A convenience wrapper around the \c isMustAlias helper interface.
   bool isMustAlias(const Value *V1, const Value *V2) {
     return alias(V1, 1, V2, 1) == MustAlias;
   }
-  
-  /// pointsToConstantMemory - If the specified memory location is
-  /// known to be constant, return true. If OrLocal is true and the
-  /// specified memory location is known to be "local" (derived from
-  /// an alloca), return true. Otherwise return false.
-  virtual bool pointsToConstantMemory(const Location &Loc,
+
+  /// Checks whether the given location points to constant memory, or if
+  /// \p OrLocal is true whether it points to a local alloca.
+  virtual bool pointsToConstantMemory(const MemoryLocation &Loc,
                                       bool OrLocal = false);
 
-  /// pointsToConstantMemory - A convenient wrapper.
+  /// A convenience wrapper around the primary \c pointsToConstantMemory
+  /// interface.
   bool pointsToConstantMemory(const Value *P, bool OrLocal = false) {
-    return pointsToConstantMemory(Location(P), OrLocal);
+    return pointsToConstantMemory(MemoryLocation(P), OrLocal);
   }
 
+  /// @}
   //===--------------------------------------------------------------------===//
-  /// Simple mod/ref information...
-  ///
+  /// \name Simple mod/ref information
+  /// @{
+
+  /// Get the ModRef info associated with a pointer argument of a callsite. The
+  /// result's bits are set to indicate the allowed aliasing ModRef kinds. Note
+  /// that these bits do not necessarily account for the overall behavior of
+  /// the function, but rather only provide additional per-argument
+  /// information.
+  virtual ModRefInfo getArgModRefInfo(ImmutableCallSite CS, unsigned ArgIdx);
 
-  /// ModRefResult - Represent the result of a mod/ref query.  Mod and Ref are
-  /// bits which may be or'd together.
+  /// Return the behavior of the given call site.
+  virtual FunctionModRefBehavior getModRefBehavior(ImmutableCallSite CS);
+
+  /// Return the behavior when calling the given function.
+  virtual FunctionModRefBehavior getModRefBehavior(const Function *F);
+
+  /// Checks if the specified call is known to never read or write memory.
   ///
-  enum ModRefResult { NoModRef = 0, Ref = 1, Mod = 2, ModRef = 3 };
-
-  /// These values define additional bits used to define the
-  /// ModRefBehavior values.
-  enum { Nowhere = 0, ArgumentPointees = 4, Anywhere = 8 | ArgumentPointees };
-
-  /// ModRefBehavior - Summary of how a function affects memory in the program.
-  /// Loads from constant globals are not considered memory accesses for this
-  /// interface.  Also, functions may freely modify stack space local to their
-  /// invocation without having to report it through these interfaces.
-  enum ModRefBehavior {
-    /// DoesNotAccessMemory - This function does not perform any non-local loads
-    /// or stores to memory.
-    ///
-    /// This property corresponds to the GCC 'const' attribute.
-    /// This property corresponds to the LLVM IR 'readnone' attribute.
-    /// This property corresponds to the IntrNoMem LLVM intrinsic flag.
-    DoesNotAccessMemory = Nowhere | NoModRef,
-
-    /// OnlyReadsArgumentPointees - The only memory references in this function
-    /// (if it has any) are non-volatile loads from objects pointed to by its
-    /// pointer-typed arguments, with arbitrary offsets.
-    ///
-    /// This property corresponds to the IntrReadArgMem LLVM intrinsic flag.
-    OnlyReadsArgumentPointees = ArgumentPointees | Ref,
-
-    /// OnlyAccessesArgumentPointees - The only memory references in this
-    /// function (if it has any) are non-volatile loads and stores from objects
-    /// pointed to by its pointer-typed arguments, with arbitrary offsets.
-    ///
-    /// This property corresponds to the IntrReadWriteArgMem LLVM intrinsic flag.
-    OnlyAccessesArgumentPointees = ArgumentPointees | ModRef,
-
-    /// OnlyReadsMemory - This function does not perform any non-local stores or
-    /// volatile loads, but may read from any memory location.
-    ///
-    /// This property corresponds to the GCC 'pure' attribute.
-    /// This property corresponds to the LLVM IR 'readonly' attribute.
-    /// This property corresponds to the IntrReadMem LLVM intrinsic flag.
-    OnlyReadsMemory = Anywhere | Ref,
-
-    /// UnknownModRefBehavior - This indicates that the function could not be
-    /// classified into one of the behaviors above.
-    UnknownModRefBehavior = Anywhere | ModRef
-  };
-
-  /// Get the location associated with a pointer argument of a callsite.
-  /// The mask bits are set to indicate the allowed aliasing ModRef kinds.
-  /// Note that these mask bits do not necessarily account for the overall
-  /// behavior of the function, but rather only provide additional
-  /// per-argument information.
-  virtual Location getArgLocation(ImmutableCallSite CS, unsigned ArgIdx,
-                                  ModRefResult &Mask);
-
-  /// getModRefBehavior - Return the behavior when calling the given call site.
-  virtual ModRefBehavior getModRefBehavior(ImmutableCallSite CS);
-
-  /// getModRefBehavior - Return the behavior when calling the given function.
-  /// For use when the call site is not known.
-  virtual ModRefBehavior getModRefBehavior(const Function *F);
-
-  /// doesNotAccessMemory - If the specified call is known to never read or
-  /// write memory, return true.  If the call only reads from known-constant
-  /// memory, it is also legal to return true.  Calls that unwind the stack
-  /// are legal for this predicate.
+  /// Note that if the call only reads from known-constant memory, it is also
+  /// legal to return true. Also, calls that unwind the stack are legal for
+  /// this predicate.
   ///
   /// Many optimizations (such as CSE and LICM) can be performed on such calls
   /// without worrying about aliasing properties, and many calls have this
   /// property (e.g. calls to 'sin' and 'cos').
   ///
   /// This property corresponds to the GCC 'const' attribute.
-  ///
   bool doesNotAccessMemory(ImmutableCallSite CS) {
-    return getModRefBehavior(CS) == DoesNotAccessMemory;
+    return getModRefBehavior(CS) == FMRB_DoesNotAccessMemory;
   }
 
-  /// doesNotAccessMemory - If the specified function is known to never read or
-  /// write memory, return true.  For use when the call site is not known.
+  /// Checks if the specified function is known to never read or write memory.
   ///
+  /// Note that if the function only reads from known-constant memory, it is
+  /// also legal to return true. Also, function that unwind the stack are legal
+  /// for this predicate.
+  ///
+  /// Many optimizations (such as CSE and LICM) can be performed on such calls
+  /// to such functions without worrying about aliasing properties, and many
+  /// functions have this property (e.g. 'sin' and 'cos').
+  ///
+  /// This property corresponds to the GCC 'const' attribute.
   bool doesNotAccessMemory(const Function *F) {
-    return getModRefBehavior(F) == DoesNotAccessMemory;
+    return getModRefBehavior(F) == FMRB_DoesNotAccessMemory;
   }
 
-  /// onlyReadsMemory - If the specified call is known to only read from
-  /// non-volatile memory (or not access memory at all), return true.  Calls
-  /// that unwind the stack are legal for this predicate.
+  /// Checks if the specified call is known to only read from non-volatile
+  /// memory (or not access memory at all).
+  ///
+  /// Calls that unwind the stack are legal for this predicate.
   ///
   /// This property allows many common optimizations to be performed in the
   /// absence of interfering store instructions, such as CSE of strlen calls.
   ///
   /// This property corresponds to the GCC 'pure' attribute.
-  ///
   bool onlyReadsMemory(ImmutableCallSite CS) {
     return onlyReadsMemory(getModRefBehavior(CS));
   }
 
-  /// onlyReadsMemory - If the specified function is known to only read from
-  /// non-volatile memory (or not access memory at all), return true.  For use
-  /// when the call site is not known.
-  ///
-  bool onlyReadsMemory(const Function *F) {
-    return onlyReadsMemory(getModRefBehavior(F));
-  }
-
-  /// onlyReadsMemory - Return true if functions with the specified behavior are
-  /// known to only read from non-volatile memory (or not access memory at all).
+  /// Checks if the specified function is known to only read from non-volatile
+  /// memory (or not access memory at all).
   ///
-  static bool onlyReadsMemory(ModRefBehavior MRB) {
-    return !(MRB & Mod);
-  }
-
-  /// onlyAccessesArgPointees - Return true if functions with the specified
-  /// behavior are known to read and write at most from objects pointed to by
-  /// their pointer-typed arguments (with arbitrary offsets).
+  /// Functions that unwind the stack are legal for this predicate.
   ///
-  static bool onlyAccessesArgPointees(ModRefBehavior MRB) {
-    return !(MRB & Anywhere & ~ArgumentPointees);
-  }
-
-  /// doesAccessArgPointees - Return true if functions with the specified
-  /// behavior are known to potentially read or write from objects pointed
-  /// to be their pointer-typed arguments (with arbitrary offsets).
+  /// This property allows many common optimizations to be performed in the
+  /// absence of interfering store instructions, such as CSE of strlen calls.
   ///
-  static bool doesAccessArgPointees(ModRefBehavior MRB) {
-    return (MRB & ModRef) && (MRB & ArgumentPointees);
+  /// This property corresponds to the GCC 'pure' attribute.
+  bool onlyReadsMemory(const Function *F) {
+    return onlyReadsMemory(getModRefBehavior(F));
   }
 
-  /// getModRefInfo - Return information about whether or not an
-  /// instruction may read or write memory (without regard to a
-  /// specific location)
-  ModRefResult getModRefInfo(const Instruction *I) {
-    if (auto CS = ImmutableCallSite(I)) {
-      auto MRB = getModRefBehavior(CS);
-      if (MRB & ModRef)
-        return ModRef;
-      else if (MRB & Ref)
-        return Ref;
-      else if (MRB & Mod)
-        return Mod;
-      return NoModRef;
-    }
-
-    return getModRefInfo(I, Location());
+  /// Checks if functions with the specified behavior are known to only read
+  /// from non-volatile memory (or not access memory at all).
+  static bool onlyReadsMemory(FunctionModRefBehavior MRB) {
+    return !(MRB & MRI_Mod);
   }
 
-  /// getModRefInfo - Return information about whether or not an instruction may
-  /// read or write the specified memory location.  An instruction
-  /// that doesn't read or write memory may be trivially LICM'd for example.
-  ModRefResult getModRefInfo(const Instruction *I,
-                             const Location &Loc) {
-    switch (I->getOpcode()) {
-    case Instruction::VAArg:  return getModRefInfo((const VAArgInst*)I, Loc);
-    case Instruction::Load:   return getModRefInfo((const LoadInst*)I,  Loc);
-    case Instruction::Store:  return getModRefInfo((const StoreInst*)I, Loc);
-    case Instruction::Fence:  return getModRefInfo((const FenceInst*)I, Loc);
-    case Instruction::AtomicCmpXchg:
-      return getModRefInfo((const AtomicCmpXchgInst*)I, Loc);
-    case Instruction::AtomicRMW:
-      return getModRefInfo((const AtomicRMWInst*)I, Loc);
-    case Instruction::Call:   return getModRefInfo((const CallInst*)I,  Loc);
-    case Instruction::Invoke: return getModRefInfo((const InvokeInst*)I,Loc);
-    default:                  return NoModRef;
-    }
+  /// Checks if functions with the specified behavior are known to read and
+  /// write at most from objects pointed to by their pointer-typed arguments
+  /// (with arbitrary offsets).
+  static bool onlyAccessesArgPointees(FunctionModRefBehavior MRB) {
+    return !(MRB & FMRL_Anywhere & ~FMRL_ArgumentPointees);
   }
 
-  /// getModRefInfo - A convenience wrapper.
-  ModRefResult getModRefInfo(const Instruction *I,
-                             const Value *P, uint64_t Size) {
-    return getModRefInfo(I, Location(P, Size));
+  /// Checks if functions with the specified behavior are known to potentially
+  /// read or write from objects pointed to be their pointer-typed arguments
+  /// (with arbitrary offsets).
+  static bool doesAccessArgPointees(FunctionModRefBehavior MRB) {
+    return (MRB & MRI_ModRef) && (MRB & FMRL_ArgumentPointees);
   }
 
   /// getModRefInfo (for call sites) - Return information about whether
   /// a particular call site modifies or reads the specified memory location.
-  virtual ModRefResult getModRefInfo(ImmutableCallSite CS,
-                                     const Location &Loc);
+  virtual ModRefInfo getModRefInfo(ImmutableCallSite CS,
+                                   const MemoryLocation &Loc);
 
   /// getModRefInfo (for call sites) - A convenience wrapper.
-  ModRefResult getModRefInfo(ImmutableCallSite CS,
-                             const Value *P, uint64_t Size) {
-    return getModRefInfo(CS, Location(P, Size));
+  ModRefInfo getModRefInfo(ImmutableCallSite CS, const Value *P,
+                           uint64_t Size) {
+    return getModRefInfo(CS, MemoryLocation(P, Size));
   }
 
   /// getModRefInfo (for calls) - Return information about whether
   /// a particular call modifies or reads the specified memory location.
-  ModRefResult getModRefInfo(const CallInst *C, const Location &Loc) {
+  ModRefInfo getModRefInfo(const CallInst *C, const MemoryLocation &Loc) {
     return getModRefInfo(ImmutableCallSite(C), Loc);
   }
 
   /// getModRefInfo (for calls) - A convenience wrapper.
-  ModRefResult getModRefInfo(const CallInst *C, const Value *P, uint64_t Size) {
-    return getModRefInfo(C, Location(P, Size));
+  ModRefInfo getModRefInfo(const CallInst *C, const Value *P, uint64_t Size) {
+    return getModRefInfo(C, MemoryLocation(P, Size));
   }
 
   /// getModRefInfo (for invokes) - Return information about whether
   /// a particular invoke modifies or reads the specified memory location.
-  ModRefResult getModRefInfo(const InvokeInst *I,
-                             const Location &Loc) {
+  ModRefInfo getModRefInfo(const InvokeInst *I, const MemoryLocation &Loc) {
     return getModRefInfo(ImmutableCallSite(I), Loc);
   }
 
   /// getModRefInfo (for invokes) - A convenience wrapper.
-  ModRefResult getModRefInfo(const InvokeInst *I,
-                             const Value *P, uint64_t Size) {
-    return getModRefInfo(I, Location(P, Size));
+  ModRefInfo getModRefInfo(const InvokeInst *I, const Value *P, uint64_t Size) {
+    return getModRefInfo(I, MemoryLocation(P, Size));
   }
 
   /// getModRefInfo (for loads) - Return information about whether
   /// a particular load modifies or reads the specified memory location.
-  ModRefResult getModRefInfo(const LoadInst *L, const Location &Loc);
+  ModRefInfo getModRefInfo(const LoadInst *L, const MemoryLocation &Loc);
 
   /// getModRefInfo (for loads) - A convenience wrapper.
-  ModRefResult getModRefInfo(const LoadInst *L, const Value *P, uint64_t Size) {
-    return getModRefInfo(L, Location(P, Size));
+  ModRefInfo getModRefInfo(const LoadInst *L, const Value *P, uint64_t Size) {
+    return getModRefInfo(L, MemoryLocation(P, Size));
   }
 
   /// getModRefInfo (for stores) - Return information about whether
   /// a particular store modifies or reads the specified memory location.
-  ModRefResult getModRefInfo(const StoreInst *S, const Location &Loc);
+  ModRefInfo getModRefInfo(const StoreInst *S, const MemoryLocation &Loc);
 
   /// getModRefInfo (for stores) - A convenience wrapper.
-  ModRefResult getModRefInfo(const StoreInst *S, const Value *P, uint64_t Size){
-    return getModRefInfo(S, Location(P, Size));
+  ModRefInfo getModRefInfo(const StoreInst *S, const Value *P, uint64_t Size) {
+    return getModRefInfo(S, MemoryLocation(P, Size));
   }
 
   /// getModRefInfo (for fences) - Return information about whether
   /// a particular store modifies or reads the specified memory location.
-  ModRefResult getModRefInfo(const FenceInst *S, const Location &Loc) {
+  ModRefInfo getModRefInfo(const FenceInst *S, const MemoryLocation &Loc) {
     // Conservatively correct.  (We could possibly be a bit smarter if
     // Loc is a alloca that doesn't escape.)
-    return ModRef;
+    return MRI_ModRef;
   }
 
   /// getModRefInfo (for fences) - A convenience wrapper.
-  ModRefResult getModRefInfo(const FenceInst *S, const Value *P, uint64_t Size){
-    return getModRefInfo(S, Location(P, Size));
+  ModRefInfo getModRefInfo(const FenceInst *S, const Value *P, uint64_t Size) {
+    return getModRefInfo(S, MemoryLocation(P, Size));
   }
 
   /// getModRefInfo (for cmpxchges) - Return information about whether
   /// a particular cmpxchg modifies or reads the specified memory location.
-  ModRefResult getModRefInfo(const AtomicCmpXchgInst *CX, const Location &Loc);
+  ModRefInfo getModRefInfo(const AtomicCmpXchgInst *CX,
+                           const MemoryLocation &Loc);
 
   /// getModRefInfo (for cmpxchges) - A convenience wrapper.
-  ModRefResult getModRefInfo(const AtomicCmpXchgInst *CX,
-                             const Value *P, unsigned Size) {
-    return getModRefInfo(CX, Location(P, Size));
+  ModRefInfo getModRefInfo(const AtomicCmpXchgInst *CX, const Value *P,
+                           unsigned Size) {
+    return getModRefInfo(CX, MemoryLocation(P, Size));
   }
 
   /// getModRefInfo (for atomicrmws) - Return information about whether
   /// a particular atomicrmw modifies or reads the specified memory location.
-  ModRefResult getModRefInfo(const AtomicRMWInst *RMW, const Location &Loc);
+  ModRefInfo getModRefInfo(const AtomicRMWInst *RMW, const MemoryLocation &Loc);
 
   /// getModRefInfo (for atomicrmws) - A convenience wrapper.
-  ModRefResult getModRefInfo(const AtomicRMWInst *RMW,
-                             const Value *P, unsigned Size) {
-    return getModRefInfo(RMW, Location(P, Size));
+  ModRefInfo getModRefInfo(const AtomicRMWInst *RMW, const Value *P,
+                           unsigned Size) {
+    return getModRefInfo(RMW, MemoryLocation(P, Size));
   }
 
   /// getModRefInfo (for va_args) - Return information about whether
   /// a particular va_arg modifies or reads the specified memory location.
-  ModRefResult getModRefInfo(const VAArgInst* I, const Location &Loc);
+  ModRefInfo getModRefInfo(const VAArgInst *I, const MemoryLocation &Loc);
 
   /// getModRefInfo (for va_args) - A convenience wrapper.
-  ModRefResult getModRefInfo(const VAArgInst* I, const Value* P, uint64_t Size){
-    return getModRefInfo(I, Location(P, Size));
+  ModRefInfo getModRefInfo(const VAArgInst *I, const Value *P, uint64_t Size) {
+    return getModRefInfo(I, MemoryLocation(P, Size));
   }
-  /// getModRefInfo - Return information about whether a call and an instruction
-  /// may refer to the same memory locations.
-  ModRefResult getModRefInfo(Instruction *I,
-                             ImmutableCallSite Call);
-
-  /// getModRefInfo - Return information about whether two call sites may refer
-  /// to the same set of memory locations.  See 
-  ///   http://llvm.org/docs/AliasAnalysis.html#ModRefInfo
-  /// for details.
-  virtual ModRefResult getModRefInfo(ImmutableCallSite CS1,
-                                     ImmutableCallSite CS2);
 
-  /// callCapturesBefore - Return information about whether a particular call 
-  /// site modifies or reads the specified memory location.
-  ModRefResult callCapturesBefore(const Instruction *I,
-                                  const AliasAnalysis::Location &MemLoc,
-                                  DominatorTree *DT);
+  /// Check whether or not an instruction may read or write memory (without
+  /// regard to a specific location).
+  ///
+  /// For function calls, this delegates to the alias-analysis specific
+  /// call-site mod-ref behavior queries. Otherwise it delegates to the generic
+  /// mod ref information query without a location.
+  ModRefInfo getModRefInfo(const Instruction *I) {
+    if (auto CS = ImmutableCallSite(I)) {
+      auto MRB = getModRefBehavior(CS);
+      if (MRB & MRI_ModRef)
+        return MRI_ModRef;
+      else if (MRB & MRI_Ref)
+        return MRI_Ref;
+      else if (MRB & MRI_Mod)
+        return MRI_Mod;
+      return MRI_NoModRef;
+    }
 
-  /// callCapturesBefore - A convenience wrapper.
-  ModRefResult callCapturesBefore(const Instruction *I, const Value *P,
-                                  uint64_t Size, DominatorTree *DT) {
-    return callCapturesBefore(I, Location(P, Size), DT);
+    return getModRefInfo(I, MemoryLocation());
   }
 
-  //===--------------------------------------------------------------------===//
-  /// Higher level methods for querying mod/ref information.
+  /// Check whether or not an instruction may read or write the specified
+  /// memory location.
   ///
+  /// An instruction that doesn't read or write memory may be trivially LICM'd
+  /// for example.
+  ///
+  /// This primarily delegates to specific helpers above.
+  ModRefInfo getModRefInfo(const Instruction *I, const MemoryLocation &Loc) {
+    switch (I->getOpcode()) {
+    case Instruction::VAArg:  return getModRefInfo((const VAArgInst*)I, Loc);
+    case Instruction::Load:   return getModRefInfo((const LoadInst*)I,  Loc);
+    case Instruction::Store:  return getModRefInfo((const StoreInst*)I, Loc);
+    case Instruction::Fence:  return getModRefInfo((const FenceInst*)I, Loc);
+    case Instruction::AtomicCmpXchg:
+      return getModRefInfo((const AtomicCmpXchgInst*)I, Loc);
+    case Instruction::AtomicRMW:
+      return getModRefInfo((const AtomicRMWInst*)I, Loc);
+    case Instruction::Call:   return getModRefInfo((const CallInst*)I,  Loc);
+    case Instruction::Invoke: return getModRefInfo((const InvokeInst*)I,Loc);
+    default:
+      return MRI_NoModRef;
+    }
+  }
 
-  /// canBasicBlockModify - Return true if it is possible for execution of the
-  /// specified basic block to modify the location Loc.
-  bool canBasicBlockModify(const BasicBlock &BB, const Location &Loc);
-
-  /// canBasicBlockModify - A convenience wrapper.
-  bool canBasicBlockModify(const BasicBlock &BB, const Value *P, uint64_t Size){
-    return canBasicBlockModify(BB, Location(P, Size));
+  /// A convenience wrapper for constructing the memory location.
+  ModRefInfo getModRefInfo(const Instruction *I, const Value *P,
+                           uint64_t Size) {
+    return getModRefInfo(I, MemoryLocation(P, Size));
   }
 
-  /// canInstructionRangeModRef - Return true if it is possible for the
-  /// execution of the specified instructions to mod\ref (according to the
-  /// mode) the location Loc. The instructions to consider are all
-  /// of the instructions in the range of [I1,I2] INCLUSIVE.
-  /// I1 and I2 must be in the same basic block.
-  bool canInstructionRangeModRef(const Instruction &I1,
-                                const Instruction &I2, const Location &Loc,
-                                const ModRefResult Mode);
+  /// Return information about whether a call and an instruction may refer to
+  /// the same memory locations.
+  ModRefInfo getModRefInfo(Instruction *I, ImmutableCallSite Call);
+
+  /// Return information about whether two call sites may refer to the same set
+  /// of memory locations. See the AA documentation for details:
+  ///   http://llvm.org/docs/AliasAnalysis.html#ModRefInfo
+  virtual ModRefInfo getModRefInfo(ImmutableCallSite CS1,
+                                   ImmutableCallSite CS2);
 
-  /// canInstructionRangeModRef - A convenience wrapper.
-  bool canInstructionRangeModRef(const Instruction &I1,
-                                 const Instruction &I2, const Value *Ptr,
-                                 uint64_t Size, const ModRefResult Mode) {
-    return canInstructionRangeModRef(I1, I2, Location(Ptr, Size), Mode);
+  /// \brief Return information about whether a particular call site modifies
+  /// or reads the specified memory location \p MemLoc before instruction \p I
+  /// in a BasicBlock. A ordered basic block \p OBB can be used to speed up
+  /// instruction ordering queries inside the BasicBlock containing \p I.
+  ModRefInfo callCapturesBefore(const Instruction *I,
+                                const MemoryLocation &MemLoc, DominatorTree *DT,
+                                OrderedBasicBlock *OBB = nullptr);
+
+  /// \brief A convenience wrapper to synthesize a memory location.
+  ModRefInfo callCapturesBefore(const Instruction *I, const Value *P,
+                                uint64_t Size, DominatorTree *DT,
+                                OrderedBasicBlock *OBB = nullptr) {
+    return callCapturesBefore(I, MemoryLocation(P, Size), DT, OBB);
   }
 
+  /// @}
   //===--------------------------------------------------------------------===//
-  /// Methods that clients should call when they transform the program to allow
-  /// alias analyses to update their internal data structures.  Note that these
-  /// methods may be called on any instruction, regardless of whether or not
-  /// they have pointer-analysis implications.
-  ///
+  /// \name Higher level methods for querying mod/ref information.
+  /// @{
 
-  /// deleteValue - This method should be called whenever an LLVM Value is
-  /// deleted from the program, for example when an instruction is found to be
-  /// redundant and is eliminated.
-  ///
-  virtual void deleteValue(Value *V);
+  /// Check if it is possible for execution of the specified basic block to
+  /// modify the location Loc.
+  bool canBasicBlockModify(const BasicBlock &BB, const MemoryLocation &Loc);
 
-  /// copyValue - This method should be used whenever a preexisting value in the
-  /// program is copied or cloned, introducing a new value.  Note that analysis
-  /// implementations should tolerate clients that use this method to introduce
-  /// the same value multiple times: if the analysis already knows about a
-  /// value, it should ignore the request.
-  ///
-  virtual void copyValue(Value *From, Value *To);
-
-  /// addEscapingUse - This method should be used whenever an escaping use is
-  /// added to a pointer value.  Analysis implementations may either return
-  /// conservative responses for that value in the future, or may recompute
-  /// some or all internal state to continue providing precise responses.
-  ///
-  /// Escaping uses are considered by anything _except_ the following:
-  ///  - GEPs or bitcasts of the pointer
-  ///  - Loads through the pointer
-  ///  - Stores through (but not of) the pointer
-  virtual void addEscapingUse(Use &U);
-
-  /// replaceWithNewValue - This method is the obvious combination of the two
-  /// above, and it provided as a helper to simplify client code.
-  ///
-  void replaceWithNewValue(Value *Old, Value *New) {
-    copyValue(Old, New);
-    deleteValue(Old);
+  /// A convenience wrapper synthesizing a memory location.
+  bool canBasicBlockModify(const BasicBlock &BB, const Value *P,
+                           uint64_t Size) {
+    return canBasicBlockModify(BB, MemoryLocation(P, Size));
   }
-};
 
-// Specialize DenseMapInfo for Location.
-template<>
-struct DenseMapInfo<AliasAnalysis::Location> {
-  static inline AliasAnalysis::Location getEmptyKey() {
-    return AliasAnalysis::Location(DenseMapInfo<const Value *>::getEmptyKey(),
-                                   0);
-  }
-  static inline AliasAnalysis::Location getTombstoneKey() {
-    return AliasAnalysis::Location(
-        DenseMapInfo<const Value *>::getTombstoneKey(), 0);
-  }
-  static unsigned getHashValue(const AliasAnalysis::Location &Val) {
-    return DenseMapInfo<const Value *>::getHashValue(Val.Ptr) ^
-           DenseMapInfo<uint64_t>::getHashValue(Val.Size) ^
-           DenseMapInfo<AAMDNodes>::getHashValue(Val.AATags);
-  }
-  static bool isEqual(const AliasAnalysis::Location &LHS,
-                      const AliasAnalysis::Location &RHS) {
-    return LHS.Ptr == RHS.Ptr &&
-           LHS.Size == RHS.Size &&
-           LHS.AATags == RHS.AATags;
+  /// Check if it is possible for the execution of the specified instructions
+  /// to mod\ref (according to the mode) the location Loc.
+  ///
+  /// The instructions to consider are all of the instructions in the range of
+  /// [I1,I2] INCLUSIVE. I1 and I2 must be in the same basic block.
+  bool canInstructionRangeModRef(const Instruction &I1, const Instruction &I2,
+                                 const MemoryLocation &Loc,
+                                 const ModRefInfo Mode);
+
+  /// A convenience wrapper synthesizing a memory location.
+  bool canInstructionRangeModRef(const Instruction &I1, const Instruction &I2,
+                                 const Value *Ptr, uint64_t Size,
+                                 const ModRefInfo Mode) {
+    return canInstructionRangeModRef(I1, I2, MemoryLocation(Ptr, Size), Mode);
   }
 };
 
diff --git a/include/llvm/Analysis/AliasSetTracker.h b/include/llvm/Analysis/AliasSetTracker.h
index fc45903042f..881699d0922 100644
--- a/include/llvm/Analysis/AliasSetTracker.h
+++ b/include/llvm/Analysis/AliasSetTracker.h
@@ -117,24 +117,30 @@ class AliasSet : public ilist_node<AliasSet> {
   // AliasSets forwarding to it.
   unsigned RefCount : 28;
 
-  /// AccessType - Keep track of whether this alias set merely refers to the
-  /// locations of memory, whether it modifies the memory, or whether it does
-  /// both.  The lattice goes from "NoModRef" to either Refs or Mods, then to
-  /// ModRef as necessary.
+  /// The kinds of access this alias set models.
   ///
-  enum AccessType {
-    NoModRef = 0, Refs = 1,         // Ref = bit 1
-    Mods     = 2, ModRef = 3        // Mod = bit 2
+  /// We keep track of whether this alias set merely refers to the locations of
+  /// memory (and not any particular access), whether it modifies or references
+  /// the memory, or whether it does both. The lattice goes from "NoAccess" to
+  /// either RefAccess or ModAccess, then to ModRefAccess as necessary.
+  enum AccessLattice {
+    NoAccess = 0,
+    RefAccess = 1,
+    ModAccess = 2,
+    ModRefAccess = RefAccess | ModAccess
   };
-  unsigned AccessTy : 2;
+  unsigned Access : 2;
 
-  /// AliasType - Keep track the relationships between the pointers in the set.
-  /// Lattice goes from MustAlias to MayAlias.
+  /// The kind of alias relationship between pointers of the set.
   ///
-  enum AliasType {
-    MustAlias = 0, MayAlias = 1
+  /// These represent conservatively correct alias results between any members
+  /// of the set. We represent these independently of the values of alias
+  /// results in order to pack it into a single bit. Lattice goes from
+  /// MustAlias to MayAlias.
+  enum AliasLattice {
+    SetMustAlias = 0, SetMayAlias = 1
   };
-  unsigned AliasTy : 1;
+  unsigned Alias : 1;
 
   // Volatile - True if this alias set contains volatile loads or stores.
   bool Volatile : 1;
@@ -153,10 +159,10 @@ class AliasSet : public ilist_node<AliasSet> {
   
 public:
   /// Accessors...
-  bool isRef() const { return AccessTy & Refs; }
-  bool isMod() const { return AccessTy & Mods; }
-  bool isMustAlias() const { return AliasTy == MustAlias; }
-  bool isMayAlias()  const { return AliasTy == MayAlias; }
+  bool isRef() const { return Access & RefAccess; }
+  bool isMod() const { return Access & ModAccess; }
+  bool isMustAlias() const { return Alias == SetMustAlias; }
+  bool isMayAlias()  const { return Alias == SetMayAlias; }
 
   // isVolatile - Return true if this alias set contains volatile loads or
   // stores.
@@ -218,7 +224,7 @@ class AliasSet : public ilist_node<AliasSet> {
   friend struct ilist_sentinel_traits<AliasSet>;
   AliasSet()
     : PtrList(nullptr), PtrListEnd(&PtrList), Forward(nullptr), RefCount(0),
-      AccessTy(NoModRef), AliasTy(MustAlias), Volatile(false) {
+      Access(NoAccess), Alias(SetMustAlias), Volatile(false) {
   }
 
   AliasSet(const AliasSet &AS) = delete;
@@ -268,7 +274,7 @@ class AliasSet : public ilist_node<AliasSet> {
   ///
   bool aliasesPointer(const Value *Ptr, uint64_t Size, const AAMDNodes &AAInfo,
                       AliasAnalysis &AA) const;
-  bool aliasesUnknownInst(Instruction *Inst, AliasAnalysis &AA) const;
+  bool aliasesUnknownInst(const Instruction *Inst, AliasAnalysis &AA) const;
 };
 
 inline raw_ostream& operator<<(raw_ostream &OS, const AliasSet &AS) {
@@ -358,7 +364,7 @@ class AliasSetTracker {
 
   /// getAliasSetForPointerIfExists - Return the alias set containing the
   /// location specified if one exists, otherwise return null.
-  AliasSet *getAliasSetForPointerIfExists(Value *P, uint64_t Size,
+  AliasSet *getAliasSetForPointerIfExists(const Value *P, uint64_t Size,
                                           const AAMDNodes &AAInfo) {
     return findAliasSetForPointer(P, Size, AAInfo);
   }
@@ -366,11 +372,12 @@ class AliasSetTracker {
   /// containsPointer - Return true if the specified location is represented by
   /// this alias set, false otherwise.  This does not modify the AST object or
   /// alias sets.
-  bool containsPointer(Value *P, uint64_t Size, const AAMDNodes &AAInfo) const;
+  bool containsPointer(const Value *P, uint64_t Size,
+                       const AAMDNodes &AAInfo) const;
 
   /// Return true if the specified instruction "may" (or must) alias one of the
   /// members in any of the sets.
-  bool containsUnknown(Instruction *I) const;
+  bool containsUnknown(const Instruction *I) const;
 
   /// getAliasAnalysis - Return the underlying alias analysis object used by
   /// this tracker.
@@ -418,11 +425,11 @@ class AliasSetTracker {
   }
 
   AliasSet &addPointer(Value *P, uint64_t Size, const AAMDNodes &AAInfo,
-                       AliasSet::AccessType E,
+                       AliasSet::AccessLattice E,
                        bool &NewSet) {
     NewSet = false;
     AliasSet &AS = getAliasSetForPointer(P, Size, AAInfo, &NewSet);
-    AS.AccessTy |= E;
+    AS.Access |= E;
     return AS;
   }
   AliasSet *findAliasSetForPointer(const Value *Ptr, uint64_t Size,
diff --git a/include/llvm/Analysis/AssumptionCache.h b/include/llvm/Analysis/AssumptionCache.h
index 1f00b691b30..9de8ed5d424 100644
--- a/include/llvm/Analysis/AssumptionCache.h
+++ b/include/llvm/Analysis/AssumptionCache.h
@@ -66,7 +66,7 @@ class AssumptionCache {
 
   /// \brief Add an @llvm.assume intrinsic to this function's cache.
   ///
-  /// The call passed in must be an instruction within this fuction and must
+  /// The call passed in must be an instruction within this function and must
   /// not already be in the cache.
   void registerAssumption(CallInst *CI);
 
@@ -79,7 +79,7 @@ class AssumptionCache {
   }
 
   /// \brief Access the list of assumption handles currently tracked for this
-  /// fuction.
+  /// function.
   ///
   /// Note that these produce weak handles that may be null. The caller must
   /// handle that case.
diff --git a/include/llvm/Analysis/BlockFrequencyInfo.h b/include/llvm/Analysis/BlockFrequencyInfo.h
index f27c32df928..2b6b16a3fb9 100644
--- a/include/llvm/Analysis/BlockFrequencyInfo.h
+++ b/include/llvm/Analysis/BlockFrequencyInfo.h
@@ -21,26 +21,20 @@
 namespace llvm {
 
 class BranchProbabilityInfo;
+class LoopInfo;
 template <class BlockT> class BlockFrequencyInfoImpl;
 
 /// BlockFrequencyInfo pass uses BlockFrequencyInfoImpl implementation to
 /// estimate IR basic block frequencies.
-class BlockFrequencyInfo : public FunctionPass {
+class BlockFrequencyInfo {
   typedef BlockFrequencyInfoImpl<BasicBlock> ImplType;
   std::unique_ptr<ImplType> BFI;
 
 public:
-  static char ID;
-
   BlockFrequencyInfo();
+  BlockFrequencyInfo(const Function &F, const BranchProbabilityInfo &BPI,
+                     const LoopInfo &LI);
 
-  ~BlockFrequencyInfo() override;
-
-  void getAnalysisUsage(AnalysisUsage &AU) const override;
-
-  bool runOnFunction(Function &F) override;
-  void releaseMemory() override;
-  void print(raw_ostream &O, const Module *M) const override;
   const Function *getFunction() const;
   void view() const;
 
@@ -51,6 +45,10 @@ class BlockFrequencyInfo : public FunctionPass {
   /// floating points.
   BlockFrequency getBlockFreq(const BasicBlock *BB) const;
 
+  /// calculate - compute block frequency info for the given function.
+  void calculate(const Function &F, const BranchProbabilityInfo &BPI,
+                 const LoopInfo &LI);
+
   // Print the block frequency Freq to OS using the current functions entry
   // frequency to convert freq into a relative decimal form.
   raw_ostream &printBlockFreq(raw_ostream &OS, const BlockFrequency Freq) const;
@@ -60,7 +58,28 @@ class BlockFrequencyInfo : public FunctionPass {
   raw_ostream &printBlockFreq(raw_ostream &OS, const BasicBlock *BB) const;
 
   uint64_t getEntryFreq() const;
+  void releaseMemory();
+  void print(raw_ostream &OS) const;
+};
+
+/// \brief Legacy analysis pass which computes \c BlockFrequencyInfo.
+class BlockFrequencyInfoWrapperPass : public FunctionPass {
+  BlockFrequencyInfo BFI;
 
+public:
+  static char ID;
+
+  BlockFrequencyInfoWrapperPass();
+  ~BlockFrequencyInfoWrapperPass() override;
+
+  BlockFrequencyInfo &getBFI() { return BFI; }
+  const BlockFrequencyInfo &getBFI() const { return BFI; }
+
+  void getAnalysisUsage(AnalysisUsage &AU) const override;
+
+  bool runOnFunction(Function &F) override;
+  void releaseMemory() override;
+  void print(raw_ostream &OS, const Module *M) const override;
 };
 
 }
diff --git a/include/llvm/Analysis/BlockFrequencyInfoImpl.h b/include/llvm/Analysis/BlockFrequencyInfoImpl.h
index 9acc863a738..6c101a63e12 100644
--- a/include/llvm/Analysis/BlockFrequencyInfoImpl.h
+++ b/include/llvm/Analysis/BlockFrequencyInfoImpl.h
@@ -191,28 +191,31 @@ class BlockFrequencyInfoImplBase {
 
   /// \brief Data about a loop.
   ///
-  /// Contains the data necessary to represent represent a loop as a
-  /// pseudo-node once it's packaged.
+  /// Contains the data necessary to represent a loop as a pseudo-node once it's
+  /// packaged.
   struct LoopData {
     typedef SmallVector<std::pair<BlockNode, BlockMass>, 4> ExitMap;
     typedef SmallVector<BlockNode, 4> NodeList;
-    LoopData *Parent;       ///< The parent loop.
-    bool IsPackaged;        ///< Whether this has been packaged.
-    uint32_t NumHeaders;    ///< Number of headers.
-    ExitMap Exits;          ///< Successor edges (and weights).
-    NodeList Nodes;         ///< Header and the members of the loop.
-    BlockMass BackedgeMass; ///< Mass returned to loop header.
+    typedef SmallVector<BlockMass, 1> HeaderMassList;
+    LoopData *Parent;            ///< The parent loop.
+    bool IsPackaged;             ///< Whether this has been packaged.
+    uint32_t NumHeaders;         ///< Number of headers.
+    ExitMap Exits;               ///< Successor edges (and weights).
+    NodeList Nodes;              ///< Header and the members of the loop.
+    HeaderMassList BackedgeMass; ///< Mass returned to each loop header.
     BlockMass Mass;
     Scaled64 Scale;
 
     LoopData(LoopData *Parent, const BlockNode &Header)
-        : Parent(Parent), IsPackaged(false), NumHeaders(1), Nodes(1, Header) {}
+        : Parent(Parent), IsPackaged(false), NumHeaders(1), Nodes(1, Header),
+          BackedgeMass(1) {}
     template <class It1, class It2>
     LoopData(LoopData *Parent, It1 FirstHeader, It1 LastHeader, It2 FirstOther,
              It2 LastOther)
         : Parent(Parent), IsPackaged(false), Nodes(FirstHeader, LastHeader) {
       NumHeaders = Nodes.size();
       Nodes.insert(Nodes.end(), FirstOther, LastOther);
+      BackedgeMass.resize(NumHeaders);
     }
     bool isHeader(const BlockNode &Node) const {
       if (isIrreducible())
@@ -223,6 +226,14 @@ class BlockFrequencyInfoImplBase {
     BlockNode getHeader() const { return Nodes[0]; }
     bool isIrreducible() const { return NumHeaders > 1; }
 
+    HeaderMassList::difference_type getHeaderIndex(const BlockNode &B) {
+      assert(isHeader(B) && "this is only valid on loop header blocks");
+      if (isIrreducible())
+        return std::lower_bound(Nodes.begin(), Nodes.begin() + NumHeaders, B) -
+               Nodes.begin();
+      return 0;
+    }
+
     NodeList::const_iterator members_begin() const {
       return Nodes.begin() + NumHeaders;
     }
@@ -431,6 +442,16 @@ class BlockFrequencyInfoImplBase {
   /// \brief Compute the loop scale for a loop.
   void computeLoopScale(LoopData &Loop);
 
+  /// Adjust the mass of all headers in an irreducible loop.
+  ///
+  /// Initially, irreducible loops are assumed to distribute their mass
+  /// equally among its headers. This can lead to wrong frequency estimates
+  /// since some headers may be executed more frequently than others.
+  ///
+  /// This adjusts header mass distribution so it matches the weights of
+  /// the backedges going into each of the loop headers.
+  void adjustLoopHeaderMass(LoopData &Loop);
+
   /// \brief Package up a loop.
   void packageLoop(LoopData &Loop);
 
@@ -695,6 +716,17 @@ void IrreducibleGraph::addEdges(const BlockNode &Node,
 ///         - Distribute the mass accordingly, dithering to minimize mass loss,
 ///           as described in \a distributeMass().
 ///
+///     In the case of irreducible loops, instead of a single loop header,
+///     there will be several. The computation of backedge masses is similar
+///     but instead of having a single backedge mass, there will be one
+///     backedge per loop header. In these cases, each backedge will carry
+///     a mass proportional to the edge weights along the corresponding
+///     path.
+///
+///     At the end of propagation, the full mass assigned to the loop will be
+///     distributed among the loop headers proportionally according to the
+///     mass flowing through their backedges.
+///
 ///     Finally, calculate the loop scale from the accumulated backedge mass.
 ///
 ///  3. Distribute mass in the function (\a computeMassInFunction()).
@@ -735,11 +767,6 @@ void IrreducibleGraph::addEdges(const BlockNode &Node,
 ///         as sub-loops, rather than arbitrarily shoving the problematic
 ///         blocks into the headers of the main irreducible SCC.
 ///
-///       - Backedge frequencies are assumed to be evenly split between the
-///         headers of a given irreducible SCC.  Instead, we could track the
-///         backedge mass separately for each header, and adjust their relative
-///         frequencies.
-///
 ///       - Entry frequencies are assumed to be evenly split between the
 ///         headers of a given irreducible SCC, which is the only option if we
 ///         need to compute mass in the SCC before its parent loop.  Instead,
@@ -846,7 +873,7 @@ template <class BT> class BlockFrequencyInfoImpl : BlockFrequencyInfoImplBase {
   ///
   /// \pre \a computeMassInLoop() has been called for each subloop of \c
   /// OuterLoop.
-  /// \pre \c Insert points at the the last loop successfully processed by \a
+  /// \pre \c Insert points at the last loop successfully processed by \a
   /// computeMassInLoop().
   /// \pre \c OuterLoop has irreducible SCCs.
   void computeIrreducibleMass(LoopData *OuterLoop,
@@ -878,8 +905,8 @@ template <class BT> class BlockFrequencyInfoImpl : BlockFrequencyInfoImplBase {
 public:
   const FunctionT *getFunction() const { return F; }
 
-  void doFunction(const FunctionT *F, const BranchProbabilityInfoT *BPI,
-                  const LoopInfoT *LI);
+  void calculate(const FunctionT &F, const BranchProbabilityInfoT &BPI,
+                 const LoopInfoT &LI);
   BlockFrequencyInfoImpl() : BPI(nullptr), LI(nullptr), F(nullptr) {}
 
   using BlockFrequencyInfoImplBase::getEntryFreq;
@@ -911,13 +938,13 @@ template <class BT> class BlockFrequencyInfoImpl : BlockFrequencyInfoImplBase {
 };
 
 template <class BT>
-void BlockFrequencyInfoImpl<BT>::doFunction(const FunctionT *F,
-                                            const BranchProbabilityInfoT *BPI,
-                                            const LoopInfoT *LI) {
+void BlockFrequencyInfoImpl<BT>::calculate(const FunctionT &F,
+                                           const BranchProbabilityInfoT &BPI,
+                                           const LoopInfoT &LI) {
   // Save the parameters.
-  this->BPI = BPI;
-  this->LI = LI;
-  this->F = F;
+  this->BPI = &BPI;
+  this->LI = &LI;
+  this->F = &F;
 
   // Clean up left-over data structures.
   BlockFrequencyInfoImplBase::clear();
@@ -925,12 +952,12 @@ void BlockFrequencyInfoImpl<BT>::doFunction(const FunctionT *F,
   Nodes.clear();
 
   // Initialize.
-  DEBUG(dbgs() << "\nblock-frequency: " << F->getName() << "\n================="
-               << std::string(F->getName().size(), '=') << "\n");
+  DEBUG(dbgs() << "\nblock-frequency: " << F.getName() << "\n================="
+               << std::string(F.getName().size(), '=') << "\n");
   initializeRPOT();
   initializeLoops();
 
-  // Visit loops in post-order to find thelocal mass distribution, and then do
+  // Visit loops in post-order to find the local mass distribution, and then do
   // the full function.
   computeMassInLoops();
   computeMassInFunction();
@@ -1042,6 +1069,8 @@ bool BlockFrequencyInfoImpl<BT>::computeMassInLoop(LoopData &Loop) {
     for (const BlockNode &M : Loop.Nodes)
       if (!propagateMassToSuccessors(&Loop, M))
         llvm_unreachable("unhandled irreducible control flow");
+
+    adjustLoopHeaderMass(Loop);
   } else {
     Working[Loop.getHeader().Index].getMass() = BlockMass::getFull();
     if (!propagateMassToSuccessors(&Loop, Loop.getHeader()))
diff --git a/include/llvm/Analysis/BranchProbabilityInfo.h b/include/llvm/Analysis/BranchProbabilityInfo.h
index 89eef68d843..a6e721a388e 100644
--- a/include/llvm/Analysis/BranchProbabilityInfo.h
+++ b/include/llvm/Analysis/BranchProbabilityInfo.h
@@ -25,9 +25,9 @@ namespace llvm {
 class LoopInfo;
 class raw_ostream;
 
-/// \brief Analysis pass providing branch probability information.
+/// \brief Analysis providing branch probability information.
 ///
-/// This is a function analysis pass which provides information on the relative
+/// This is a function analysis which provides information on the relative
 /// probabilities of each "edge" in the function's CFG where such an edge is
 /// defined by a pair (PredBlock and an index in the successors). The
 /// probability of an edge from one block is always relative to the
@@ -37,17 +37,14 @@ class raw_ostream;
 /// identify an edge, since we can have multiple edges from Src to Dst.
 /// As an example, we can have a switch which jumps to Dst with value 0 and
 /// value 10.
-class BranchProbabilityInfo : public FunctionPass {
+class BranchProbabilityInfo {
 public:
-  static char ID;
+  BranchProbabilityInfo() {}
+  BranchProbabilityInfo(Function &F, const LoopInfo &LI) { calculate(F, LI); }
 
-  BranchProbabilityInfo() : FunctionPass(ID) {
-    initializeBranchProbabilityInfoPass(*PassRegistry::getPassRegistry());
-  }
+  void releaseMemory();
 
-  void getAnalysisUsage(AnalysisUsage &AU) const override;
-  bool runOnFunction(Function &F) override;
-  void print(raw_ostream &OS, const Module *M = nullptr) const override;
+  void print(raw_ostream &OS) const;
 
   /// \brief Get an edge's probability, relative to other out-edges of the Src.
   ///
@@ -115,6 +112,8 @@ class BranchProbabilityInfo : public FunctionPass {
     return IsLikely ? (1u << 20) - 1 : 1;
   }
 
+  void calculate(Function &F, const LoopInfo& LI);
+
 private:
   // Since we allow duplicate edges from one basic block to another, we use
   // a pair (PredBlock and an index in the successors) to specify an edge.
@@ -130,9 +129,6 @@ class BranchProbabilityInfo : public FunctionPass {
 
   DenseMap<Edge, uint32_t> Weights;
 
-  /// \brief Handle to the LoopInfo analysis.
-  LoopInfo *LI;
-
   /// \brief Track the last function we run over for printing.
   Function *LastF;
 
@@ -149,12 +145,33 @@ class BranchProbabilityInfo : public FunctionPass {
   bool calcMetadataWeights(BasicBlock *BB);
   bool calcColdCallHeuristics(BasicBlock *BB);
   bool calcPointerHeuristics(BasicBlock *BB);
-  bool calcLoopBranchHeuristics(BasicBlock *BB);
+  bool calcLoopBranchHeuristics(BasicBlock *BB, const LoopInfo &LI);
   bool calcZeroHeuristics(BasicBlock *BB);
   bool calcFloatingPointHeuristics(BasicBlock *BB);
   bool calcInvokeHeuristics(BasicBlock *BB);
 };
 
+/// \brief Legacy analysis pass which computes \c BranchProbabilityInfo.
+class BranchProbabilityInfoWrapperPass : public FunctionPass {
+  BranchProbabilityInfo BPI;
+
+public:
+  static char ID;
+
+  BranchProbabilityInfoWrapperPass() : FunctionPass(ID) {
+    initializeBranchProbabilityInfoWrapperPassPass(
+        *PassRegistry::getPassRegistry());
+  }
+
+  BranchProbabilityInfo &getBPI() { return BPI; }
+  const BranchProbabilityInfo &getBPI() const { return BPI; }
+
+  void getAnalysisUsage(AnalysisUsage &AU) const override;
+  bool runOnFunction(Function &F) override;
+  void releaseMemory() override;
+  void print(raw_ostream &OS, const Module *M = nullptr) const override;
+};
+
 }
 
 #endif
diff --git a/include/llvm/Analysis/CFG.h b/include/llvm/Analysis/CFG.h
index 7f92eda8cb2..7c4df780198 100644
--- a/include/llvm/Analysis/CFG.h
+++ b/include/llvm/Analysis/CFG.h
@@ -78,6 +78,17 @@ bool isPotentiallyReachable(const BasicBlock *From, const BasicBlock *To,
                             const DominatorTree *DT = nullptr,
                             const LoopInfo *LI = nullptr);
 
+/// \brief Determine whether there is at least one path from a block in
+/// 'Worklist' to 'StopBB', returning true if uncertain.
+///
+/// Determine whether there is a path from at least one block in Worklist to
+/// StopBB within a single function. Returns false only if we can prove that
+/// once any block in 'Worklist' has been reached then 'StopBB' can not be
+/// executed. Conservatively returns true.
+bool isPotentiallyReachableFromMany(SmallVectorImpl<BasicBlock *> &Worklist,
+                                    BasicBlock *StopBB,
+                                    const DominatorTree *DT = nullptr,
+                                    const LoopInfo *LI = nullptr);
 } // End llvm namespace
 
 #endif
diff --git a/include/llvm/Analysis/CGSCCPassManager.h b/include/llvm/Analysis/CGSCCPassManager.h
index 0d4fe932481..6a406cd2440 100644
--- a/include/llvm/Analysis/CGSCCPassManager.h
+++ b/include/llvm/Analysis/CGSCCPassManager.h
@@ -263,8 +263,7 @@ template <typename CGSCCPassT> class ModuleToPostOrderCGSCCPassAdaptor {
 template <typename CGSCCPassT>
 ModuleToPostOrderCGSCCPassAdaptor<CGSCCPassT>
 createModuleToPostOrderCGSCCPassAdaptor(CGSCCPassT Pass) {
-  return std::move(
-      ModuleToPostOrderCGSCCPassAdaptor<CGSCCPassT>(std::move(Pass)));
+  return ModuleToPostOrderCGSCCPassAdaptor<CGSCCPassT>(std::move(Pass));
 }
 
 /// \brief A CGSCC analysis which acts as a proxy for a function analysis
@@ -484,7 +483,7 @@ template <typename FunctionPassT> class CGSCCToFunctionPassAdaptor {
 template <typename FunctionPassT>
 CGSCCToFunctionPassAdaptor<FunctionPassT>
 createCGSCCToFunctionPassAdaptor(FunctionPassT Pass) {
-  return std::move(CGSCCToFunctionPassAdaptor<FunctionPassT>(std::move(Pass)));
+  return CGSCCToFunctionPassAdaptor<FunctionPassT>(std::move(Pass));
 }
 }
 
diff --git a/include/llvm/Analysis/CallGraph.h b/include/llvm/Analysis/CallGraph.h
index 14b88220202..662ae0e6363 100644
--- a/include/llvm/Analysis/CallGraph.h
+++ b/include/llvm/Analysis/CallGraph.h
@@ -56,6 +56,7 @@
 #include "llvm/ADT/STLExtras.h"
 #include "llvm/IR/CallSite.h"
 #include "llvm/IR/Function.h"
+#include "llvm/IR/Intrinsics.h"
 #include "llvm/IR/ValueHandle.h"
 #include "llvm/Pass.h"
 #include <map>
@@ -229,8 +230,9 @@ class CallGraphNode {
   /// \brief Adds a function to the list of functions called by this one.
   void addCalledFunction(CallSite CS, CallGraphNode *M) {
     assert(!CS.getInstruction() || !CS.getCalledFunction() ||
-           !CS.getCalledFunction()->isIntrinsic());
-    CalledFunctions.push_back(std::make_pair(CS.getInstruction(), M));
+           !CS.getCalledFunction()->isIntrinsic() ||
+           !Intrinsic::isLeaf(CS.getCalledFunction()->getIntrinsicID()));
+    CalledFunctions.emplace_back(CS.getInstruction(), M);
     M->AddRef();
   }
 
diff --git a/include/llvm/Analysis/CaptureTracking.h b/include/llvm/Analysis/CaptureTracking.h
index 8b7c7a90f7c..8d2c095d858 100644
--- a/include/llvm/Analysis/CaptureTracking.h
+++ b/include/llvm/Analysis/CaptureTracking.h
@@ -20,6 +20,7 @@ namespace llvm {
   class Use;
   class Instruction;
   class DominatorTree;
+  class OrderedBasicBlock;
 
   /// PointerMayBeCaptured - Return true if this pointer value may be captured
   /// by the enclosing function (which is required to exist).  This routine can
@@ -41,10 +42,12 @@ namespace llvm {
   /// it or not.  The boolean StoreCaptures specified whether storing the value
   /// (or part of it) into memory anywhere automatically counts as capturing it
   /// or not. Captures by the provided instruction are considered if the
-  /// final parameter is true.
+  /// final parameter is true. An ordered basic block in \p OBB could be used
+  /// to speed up capture-tracker queries.
   bool PointerMayBeCapturedBefore(const Value *V, bool ReturnCaptures,
                                   bool StoreCaptures, const Instruction *I,
-                                  DominatorTree *DT, bool IncludeI = false);
+                                  DominatorTree *DT, bool IncludeI = false,
+                                  OrderedBasicBlock *OBB = nullptr);
 
   /// This callback is used in conjunction with PointerMayBeCaptured. In
   /// addition to the interface here, you'll need to provide your own getters
diff --git a/include/llvm/Analysis/ConstantFolding.h b/include/llvm/Analysis/ConstantFolding.h
index 541a2109af6..e8185b3b630 100644
--- a/include/llvm/Analysis/ConstantFolding.h
+++ b/include/llvm/Analysis/ConstantFolding.h
@@ -72,6 +72,17 @@ namespace llvm {
 Constant *ConstantFoldInsertValueInstruction(Constant *Agg, Constant *Val,
                                              ArrayRef<unsigned> Idxs);
 
+/// \brief Attempt to constant fold an extractvalue instruction with the
+/// specified operands and indices.  The constant result is returned if
+/// successful; if not, null is returned.
+Constant *ConstantFoldExtractValueInstruction(Constant *Agg,
+                                              ArrayRef<unsigned> Idxs);
+
+/// \brief Attempt to constant fold an extractelement instruction with the
+/// specified operands and indices.  The constant result is returned if
+/// successful; if not, null is returned.
+Constant *ConstantFoldExtractElementInstruction(Constant *Val, Constant *Idx);
+
 /// ConstantFoldLoadFromConstPtr - Return the value that a load from C would
 /// produce if it is constant and determinable.  If this is not determinable,
 /// return null.
diff --git a/include/llvm/Analysis/DependenceAnalysis.h b/include/llvm/Analysis/DependenceAnalysis.h
index 0b3b2ea4281..a08ce574ea5 100644
--- a/include/llvm/Analysis/DependenceAnalysis.h
+++ b/include/llvm/Analysis/DependenceAnalysis.h
@@ -41,6 +41,7 @@
 #define LLVM_ANALYSIS_DEPENDENCEANALYSIS_H
 
 #include "llvm/ADT/SmallBitVector.h"
+#include "llvm/ADT/ArrayRef.h"
 #include "llvm/IR/Instructions.h"
 #include "llvm/Pass.h"
 
@@ -520,11 +521,11 @@ namespace llvm {
     /// in LoopNest.
     bool isLoopInvariant(const SCEV *Expression, const Loop *LoopNest) const;
 
-    /// Makes sure both subscripts (i.e. Pair->Src and Pair->Dst) share the same
-    /// integer type by sign-extending one of them when necessary.
+    /// Makes sure all subscript pairs share the same integer type by 
+    /// sign-extending as necessary.
     /// Sign-extending a subscript is safe because getelementptr assumes the
-    /// array subscripts are signed.
-    void unifySubscriptType(Subscript *Pair);
+    /// array subscripts are signed. 
+    void unifySubscriptType(ArrayRef<Subscript *> Pairs);
 
     /// removeMatchingExtensions - Examines a subscript pair.
     /// If the source and destination are identically sign (or zero)
diff --git a/include/llvm/Analysis/DominanceFrontier.h b/include/llvm/Analysis/DominanceFrontier.h
index 996700efdb6..fb730054a8e 100644
--- a/include/llvm/Analysis/DominanceFrontier.h
+++ b/include/llvm/Analysis/DominanceFrontier.h
@@ -202,8 +202,8 @@ class DominanceFrontier : public FunctionPass {
   void dump() const;
 };
 
-EXTERN_TEMPLATE_INSTANTIATION(class DominanceFrontierBase<BasicBlock>);
-EXTERN_TEMPLATE_INSTANTIATION(class ForwardDominanceFrontierBase<BasicBlock>);
+extern template class DominanceFrontierBase<BasicBlock>;
+extern template class ForwardDominanceFrontierBase<BasicBlock>;
 
 } // End llvm namespace
 
diff --git a/include/llvm/Analysis/DominanceFrontierImpl.h b/include/llvm/Analysis/DominanceFrontierImpl.h
index 278d2981516..629ae380904 100644
--- a/include/llvm/Analysis/DominanceFrontierImpl.h
+++ b/include/llvm/Analysis/DominanceFrontierImpl.h
@@ -19,7 +19,9 @@
 #define LLVM_ANALYSIS_DOMINANCEFRONTIERIMPL_H
 
 #include "llvm/ADT/SmallPtrSet.h"
+#include "llvm/Analysis/DominanceFrontier.h"
 #include "llvm/Support/Debug.h"
+#include "llvm/Support/GenericDomTree.h"
 
 namespace llvm {
 
diff --git a/include/llvm/Analysis/IVUsers.h b/include/llvm/Analysis/IVUsers.h
index ae9c1f5bd9a..00dbcbdd780 100644
--- a/include/llvm/Analysis/IVUsers.h
+++ b/include/llvm/Analysis/IVUsers.h
@@ -21,6 +21,7 @@
 
 namespace llvm {
 
+class AssumptionCache;
 class DominatorTree;
 class Instruction;
 class Value;
@@ -119,15 +120,19 @@ template<> struct ilist_traits<IVStrideUse>
 class IVUsers : public LoopPass {
   friend class IVStrideUse;
   Loop *L;
+  AssumptionCache *AC;
   LoopInfo *LI;
   DominatorTree *DT;
   ScalarEvolution *SE;
-  SmallPtrSet<Instruction*,16> Processed;
+  SmallPtrSet<Instruction*, 16> Processed;
 
   /// IVUses - A list of all tracked IV uses of induction variable expressions
   /// we are interested in.
   ilist<IVStrideUse> IVUses;
 
+  // Ephemeral values used by @llvm.assume in this function.
+  SmallPtrSet<const Value *, 32> EphValues;
+
   void getAnalysisUsage(AnalysisUsage &AU) const override;
 
   bool runOnLoop(Loop *L, LPPassManager &LPM) override;
diff --git a/include/llvm/Analysis/InstructionSimplify.h b/include/llvm/Analysis/InstructionSimplify.h
index 706bd8000d3..d44c5ff4078 100644
--- a/include/llvm/Analysis/InstructionSimplify.h
+++ b/include/llvm/Analysis/InstructionSimplify.h
@@ -212,7 +212,7 @@ namespace llvm {
   /// SimplifyFCmpInst - Given operands for an FCmpInst, see if we can
   /// fold the result.  If not, this returns null.
   Value *SimplifyFCmpInst(unsigned Predicate, Value *LHS, Value *RHS,
-                          const DataLayout &DL,
+                          FastMathFlags FMF, const DataLayout &DL,
                           const TargetLibraryInfo *TLI = nullptr,
                           const DominatorTree *DT = nullptr,
                           AssumptionCache *AC = nullptr,
@@ -244,6 +244,24 @@ namespace llvm {
                                  AssumptionCache *AC = nullptr,
                                  const Instruction *CxtI = nullptr);
 
+  /// \brief Given operands for an ExtractValueInst, see if we can fold the
+  /// result.  If not, this returns null.
+  Value *SimplifyExtractValueInst(Value *Agg, ArrayRef<unsigned> Idxs,
+                                  const DataLayout &DL,
+                                  const TargetLibraryInfo *TLI = nullptr,
+                                  const DominatorTree *DT = nullptr,
+                                  AssumptionCache *AC = nullptr,
+                                  const Instruction *CxtI = nullptr);
+
+  /// \brief Given operands for an ExtractElementInst, see if we can fold the
+  /// result.  If not, this returns null.
+  Value *SimplifyExtractElementInst(Value *Vec, Value *Idx,
+                                    const DataLayout &DL,
+                                    const TargetLibraryInfo *TLI = nullptr,
+                                    const DominatorTree *DT = nullptr,
+                                    AssumptionCache *AC = nullptr,
+                                    const Instruction *CxtI = nullptr);
+
   /// SimplifyTruncInst - Given operands for an TruncInst, see if we can fold
   /// the result.  If not, this returns null.
   Value *SimplifyTruncInst(Value *Op, Type *Ty, const DataLayout &DL,
diff --git a/include/llvm/Analysis/IteratedDominanceFrontier.h b/include/llvm/Analysis/IteratedDominanceFrontier.h
new file mode 100644
index 00000000000..5a339f10f50
--- /dev/null
+++ b/include/llvm/Analysis/IteratedDominanceFrontier.h
@@ -0,0 +1,96 @@
+//===- IteratedDominanceFrontier.h - Calculate IDF --------------*- C++ -*-===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+/// \brief Compute iterated dominance frontiers using a linear time algorithm.
+///
+/// The algorithm used here is based on:
+///
+///   Sreedhar and Gao. A linear time algorithm for placing phi-nodes.
+///   In Proceedings of the 22nd ACM SIGPLAN-SIGACT Symposium on Principles of
+///   Programming Languages
+///   POPL '95. ACM, New York, NY, 62-73.
+///
+/// It has been modified to not explicitly use the DJ graph data structure and
+/// to directly compute pruned SSA using per-variable liveness information.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_ANALYSIS_IDF_H
+#define LLVM_ANALYSIS_IDF_H
+
+#include "llvm/ADT/ArrayRef.h"
+#include "llvm/ADT/DenseMap.h"
+#include "llvm/ADT/SmallPtrSet.h"
+#include "llvm/ADT/SmallVector.h"
+
+namespace llvm {
+
+class BasicBlock;
+template <class T> class DomTreeNodeBase;
+typedef DomTreeNodeBase<BasicBlock> DomTreeNode;
+class DominatorTree;
+
+/// \brief Determine the iterated dominance frontier, given a set of defining
+/// blocks, and optionally, a set of live-in blocks.
+///
+/// In turn, the results can be used to place phi nodes.
+///
+/// This algorithm is a linear time computation of Iterated Dominance Frontiers,
+/// pruned using the live-in set.
+/// By default, liveness is not used to prune the IDF computation.
+class IDFCalculator {
+
+public:
+  IDFCalculator(DominatorTree &DT) : DT(DT), useLiveIn(false) {}
+
+  /// \brief Give the IDF calculator the set of blocks in which the value is
+  /// defined.  This is equivalent to the set of starting blocks it should be
+  /// calculating the IDF for (though later gets pruned based on liveness).
+  ///
+  /// Note: This set *must* live for the entire lifetime of the IDF calculator.
+  void setDefiningBlocks(const SmallPtrSetImpl<BasicBlock *> &Blocks) {
+    DefBlocks = &Blocks;
+  }
+
+  /// \brief Give the IDF calculator the set of blocks in which the value is
+  /// live on entry to the block.   This is used to prune the IDF calculation to
+  /// not include blocks where any phi insertion would be dead.
+  ///
+  /// Note: This set *must* live for the entire lifetime of the IDF calculator.
+
+  void setLiveInBlocks(const SmallPtrSetImpl<BasicBlock *> &Blocks) {
+    LiveInBlocks = &Blocks;
+    useLiveIn = true;
+  }
+
+  /// \brief Reset the live-in block set to be empty, and tell the IDF
+  /// calculator to not use liveness anymore.
+  void resetLiveInBlocks() {
+    LiveInBlocks = nullptr;
+    useLiveIn = false;
+  }
+
+  /// \brief Calculate iterated dominance frontiers
+  ///
+  /// This uses the linear-time phi algorithm based on DJ-graphs mentioned in
+  /// the file-level comment.  It performs DF->IDF pruning using the live-in
+  /// set, to avoid computing the IDF for blocks where an inserted PHI node
+  /// would be dead.
+  void calculate(SmallVectorImpl<BasicBlock *> &IDFBlocks);
+
+private:
+  DominatorTree &DT;
+  bool useLiveIn;
+  DenseMap<DomTreeNode *, unsigned> DomLevels;
+  const SmallPtrSetImpl<BasicBlock *> *LiveInBlocks;
+  const SmallPtrSetImpl<BasicBlock *> *DefBlocks;
+  SmallVector<BasicBlock *, 32> PHIBlocks;
+};
+}
+#endif
diff --git a/include/llvm/Analysis/JumpInstrTableInfo.h b/include/llvm/Analysis/JumpInstrTableInfo.h
deleted file mode 100644
index b6dad478cdf..00000000000
--- a/include/llvm/Analysis/JumpInstrTableInfo.h
+++ /dev/null
@@ -1,71 +0,0 @@
-//===-- JumpInstrTableInfo.h: Info for Jump-Instruction Tables --*- C++ -*-===//
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
-//
-//===----------------------------------------------------------------------===//
-///
-/// \file
-/// \brief Information about jump-instruction tables that have been created by
-/// JumpInstrTables pass.
-///
-//===----------------------------------------------------------------------===//
-
-#ifndef LLVM_ANALYSIS_JUMPINSTRTABLEINFO_H
-#define LLVM_ANALYSIS_JUMPINSTRTABLEINFO_H
-
-#include "llvm/ADT/DenseMap.h"
-#include "llvm/Pass.h"
-#include <vector>
-
-namespace llvm {
-class Function;
-class FunctionType;
-
-/// This class stores information about jump-instruction tables created by the
-/// JumpInstrTables pass (in lib/CodeGen/JumpInstrTables.cpp). Each table is a
-/// map from a function type to a vector of pairs. The first element of each
-/// pair is the function that has the jumptable annotation. The second element
-/// is a function that was declared by JumpInstrTables and used to replace all
-/// address-taking sites for the original function.
-///
-/// The information in this pass is used in AsmPrinter
-/// (lib/CodeGen/AsmPrinter/AsmPrinter.cpp) to generate the required assembly
-/// for the jump-instruction tables.
-class JumpInstrTableInfo : public ImmutablePass {
-public:
-  static char ID;
-
-  /// The default byte alignment for jump tables is 16, which is large but
-  /// usually safe.
-  JumpInstrTableInfo(uint64_t ByteAlign = 16);
-  ~JumpInstrTableInfo() override;
-  const char *getPassName() const override {
-    return "Jump-Instruction Table Info";
-  }
-
-  typedef std::pair<Function *, Function *> JumpPair;
-  typedef DenseMap<FunctionType *, std::vector<JumpPair> > JumpTables;
-
-  /// Inserts an entry in a table, adding the table if it doesn't exist.
-  void insertEntry(FunctionType *TableFunTy, Function *Target, Function *Jump);
-
-  /// Gets the tables.
-  const JumpTables &getTables() const { return Tables; }
-
-  /// Gets the alignment in bytes of a jumptable entry.
-  uint64_t entryByteAlignment() const { return ByteAlignment; }
-private:
-  JumpTables Tables;
-
-  /// A power-of-two alignment of a jumptable entry.
-  uint64_t ByteAlignment;
-};
-
-/// Creates a JumpInstrTableInfo pass with the given bound on entry size. This
-/// bound specifies the maximum number of bytes needed to represent an
-/// unconditional jump or a trap instruction in the back end currently in use.
-ModulePass *createJumpInstrTableInfoPass(unsigned Bound);
-}
-
-#endif /* LLVM_ANALYSIS_JUMPINSTRTABLEINFO_H */
diff --git a/include/llvm/Analysis/LazyCallGraph.h b/include/llvm/Analysis/LazyCallGraph.h
index b0b9068de34..7cbc40f768e 100644
--- a/include/llvm/Analysis/LazyCallGraph.h
+++ b/include/llvm/Analysis/LazyCallGraph.h
@@ -190,7 +190,7 @@ class LazyCallGraph {
 
     Function &getFunction() const {
       return F;
-    };
+    }
 
     iterator begin() const {
       return iterator(*G, Callees.begin(), Callees.end());
diff --git a/include/llvm/Analysis/LazyValueInfo.h b/include/llvm/Analysis/LazyValueInfo.h
index 1051cff5efb..42002062dca 100644
--- a/include/llvm/Analysis/LazyValueInfo.h
+++ b/include/llvm/Analysis/LazyValueInfo.h
@@ -25,7 +25,7 @@ namespace llvm {
   class Instruction;
   class TargetLibraryInfo;
   class Value;
-  
+
 /// This pass computes, caches, and vends lazy value constraint information.
 class LazyValueInfo : public FunctionPass {
   AssumptionCache *AC;
@@ -45,23 +45,22 @@ class LazyValueInfo : public FunctionPass {
   enum Tristate {
     Unknown = -1, False = 0, True = 1
   };
-  
-  
+
   // Public query interface.
-  
+
   /// Determine whether the specified value comparison with a constant is known
   /// to be true or false on the specified CFG edge.
   /// Pred is a CmpInst predicate.
   Tristate getPredicateOnEdge(unsigned Pred, Value *V, Constant *C,
                               BasicBlock *FromBB, BasicBlock *ToBB,
                               Instruction *CxtI = nullptr);
-  
+
   /// Determine whether the specified value comparison with a constant is known
   /// to be true or false at the specified instruction
   /// (from an assume intrinsic). Pred is a CmpInst predicate.
   Tristate getPredicateAt(unsigned Pred, Value *V, Constant *C,
                           Instruction *CxtI);
- 
+
   /// Determine whether the specified value is known to be a
   /// constant at the end of the specified block.  Return null if not.
   Constant *getConstant(Value *V, BasicBlock *BB, Instruction *CxtI = nullptr);
@@ -70,14 +69,14 @@ class LazyValueInfo : public FunctionPass {
   /// constant on the specified edge.  Return null if not.
   Constant *getConstantOnEdge(Value *V, BasicBlock *FromBB, BasicBlock *ToBB,
                               Instruction *CxtI = nullptr);
-  
+
   /// Inform the analysis cache that we have threaded an edge from
   /// PredBB to OldSucc to be from PredBB to NewSucc instead.
   void threadEdge(BasicBlock *PredBB, BasicBlock *OldSucc, BasicBlock *NewSucc);
-  
+
   /// Inform the analysis cache that we have erased a block.
   void eraseBlock(BasicBlock *BB);
-  
+
   // Implementation boilerplate.
 
   void getAnalysisUsage(AnalysisUsage &AU) const override;
diff --git a/include/llvm/Analysis/LibCallAliasAnalysis.h b/include/llvm/Analysis/LibCallAliasAnalysis.h
index df95e0e6fdc..12b10a988eb 100644
--- a/include/llvm/Analysis/LibCallAliasAnalysis.h
+++ b/include/llvm/Analysis/LibCallAliasAnalysis.h
@@ -38,11 +38,11 @@ namespace llvm {
     }
     ~LibCallAliasAnalysis() override;
 
-    ModRefResult getModRefInfo(ImmutableCallSite CS,
-                               const Location &Loc) override;
- 
-    ModRefResult getModRefInfo(ImmutableCallSite CS1,
-                               ImmutableCallSite CS2) override {
+    ModRefInfo getModRefInfo(ImmutableCallSite CS,
+                             const MemoryLocation &Loc) override;
+
+    ModRefInfo getModRefInfo(ImmutableCallSite CS1,
+                             ImmutableCallSite CS2) override {
       // TODO: Could compare two direct calls against each other if we cared to.
       return AliasAnalysis::getModRefInfo(CS1, CS2);
     }
@@ -62,9 +62,9 @@ namespace llvm {
     }
     
   private:
-    ModRefResult AnalyzeLibCallDetails(const LibCallFunctionInfo *FI,
-                                       ImmutableCallSite CS,
-                                       const Location &Loc);
+    ModRefInfo AnalyzeLibCallDetails(const LibCallFunctionInfo *FI,
+                                     ImmutableCallSite CS,
+                                     const MemoryLocation &Loc);
   };
 }  // End of llvm namespace
 
diff --git a/include/llvm/Analysis/LibCallSemantics.h b/include/llvm/Analysis/LibCallSemantics.h
index 34831b2849d..09411b9b860 100644
--- a/include/llvm/Analysis/LibCallSemantics.h
+++ b/include/llvm/Analysis/LibCallSemantics.h
@@ -48,8 +48,7 @@ class InvokeInst;
     enum LocResult {
       Yes, No, Unknown
     };
-    LocResult (*isLocation)(ImmutableCallSite CS,
-                            const AliasAnalysis::Location &Loc);
+    LocResult (*isLocation)(ImmutableCallSite CS, const MemoryLocation &Loc);
   };
   
   /// LibCallFunctionInfo - Each record in the array of FunctionInfo structs
@@ -72,15 +71,15 @@ class InvokeInst;
     /// any specific context knowledge.  For example, if the function is known
     /// to be readonly, this would be set to 'ref'.  If known to be readnone,
     /// this is set to NoModRef.
-    AliasAnalysis::ModRefResult UniversalBehavior;
-    
+    ModRefInfo UniversalBehavior;
+
     /// LocationMRInfo - This pair captures info about whether a specific
     /// location is modified or referenced by a libcall.
     struct LocationMRInfo {
       /// LocationID - ID # of the accessed location or ~0U for array end.
       unsigned LocationID;
       /// MRInfo - Mod/Ref info for this location.
-      AliasAnalysis::ModRefResult MRInfo;
+      ModRefInfo MRInfo;
     };
     
     /// DetailsType - Indicate the sense of the LocationDetails array.  This
@@ -207,7 +206,19 @@ class InvokeInst;
     llvm_unreachable("invalid enum");
   }
 
-  bool canSimplifyInvokeNoUnwind(const InvokeInst *II);
+  /// \brief Return true if this personality may be safely removed if there
+  /// are no invoke instructions remaining in the current function.
+  inline bool isNoOpWithoutInvoke(EHPersonality Pers) {
+    switch (Pers) {
+    case EHPersonality::Unknown:
+      return false;
+    // All known personalities currently have this behavior
+    default: return true;
+    }
+    llvm_unreachable("invalid enum");
+  }
+
+  bool canSimplifyInvokeNoUnwind(const Function *F);
 
 } // end namespace llvm
 
diff --git a/include/llvm/Analysis/LoopAccessAnalysis.h b/include/llvm/Analysis/LoopAccessAnalysis.h
index a4393bba232..7971bbbbd19 100644
--- a/include/llvm/Analysis/LoopAccessAnalysis.h
+++ b/include/llvm/Analysis/LoopAccessAnalysis.h
@@ -225,6 +225,8 @@ class MemoryDepChecker {
     return RecordInterestingDependences ? &InterestingDependences : nullptr;
   }
 
+  void clearInterestingDependences() { InterestingDependences.clear(); }
+
   /// \brief The vector of memory access instructions.  The indices are used as
   /// instruction identifiers in the Dependence class.
   const SmallVectorImpl<Instruction *> &getMemoryInstructions() const {
@@ -290,6 +292,163 @@ class MemoryDepChecker {
   bool couldPreventStoreLoadForward(unsigned Distance, unsigned TypeByteSize);
 };
 
+/// \brief Holds information about the memory runtime legality checks to verify
+/// that a group of pointers do not overlap.
+class RuntimePointerChecking {
+public:
+  struct PointerInfo {
+    /// Holds the pointer value that we need to check.
+    TrackingVH<Value> PointerValue;
+    /// Holds the pointer value at the beginning of the loop.
+    const SCEV *Start;
+    /// Holds the pointer value at the end of the loop.
+    const SCEV *End;
+    /// Holds the information if this pointer is used for writing to memory.
+    bool IsWritePtr;
+    /// Holds the id of the set of pointers that could be dependent because of a
+    /// shared underlying object.
+    unsigned DependencySetId;
+    /// Holds the id of the disjoint alias set to which this pointer belongs.
+    unsigned AliasSetId;
+    /// SCEV for the access.
+    const SCEV *Expr;
+
+    PointerInfo(Value *PointerValue, const SCEV *Start, const SCEV *End,
+                bool IsWritePtr, unsigned DependencySetId, unsigned AliasSetId,
+                const SCEV *Expr)
+        : PointerValue(PointerValue), Start(Start), End(End),
+          IsWritePtr(IsWritePtr), DependencySetId(DependencySetId),
+          AliasSetId(AliasSetId), Expr(Expr) {}
+  };
+
+  RuntimePointerChecking(ScalarEvolution *SE) : Need(false), SE(SE) {}
+
+  /// Reset the state of the pointer runtime information.
+  void reset() {
+    Need = false;
+    Pointers.clear();
+  }
+
+  /// Insert a pointer and calculate the start and end SCEVs.
+  void insert(Loop *Lp, Value *Ptr, bool WritePtr, unsigned DepSetId,
+              unsigned ASId, const ValueToValueMap &Strides);
+
+  /// \brief No run-time memory checking is necessary.
+  bool empty() const { return Pointers.empty(); }
+
+  /// A grouping of pointers. A single memcheck is required between
+  /// two groups.
+  struct CheckingPtrGroup {
+    /// \brief Create a new pointer checking group containing a single
+    /// pointer, with index \p Index in RtCheck.
+    CheckingPtrGroup(unsigned Index, RuntimePointerChecking &RtCheck)
+        : RtCheck(RtCheck), High(RtCheck.Pointers[Index].End),
+          Low(RtCheck.Pointers[Index].Start) {
+      Members.push_back(Index);
+    }
+
+    /// \brief Tries to add the pointer recorded in RtCheck at index
+    /// \p Index to this pointer checking group. We can only add a pointer
+    /// to a checking group if we will still be able to get
+    /// the upper and lower bounds of the check. Returns true in case
+    /// of success, false otherwise.
+    bool addPointer(unsigned Index);
+
+    /// Constitutes the context of this pointer checking group. For each
+    /// pointer that is a member of this group we will retain the index
+    /// at which it appears in RtCheck.
+    RuntimePointerChecking &RtCheck;
+    /// The SCEV expression which represents the upper bound of all the
+    /// pointers in this group.
+    const SCEV *High;
+    /// The SCEV expression which represents the lower bound of all the
+    /// pointers in this group.
+    const SCEV *Low;
+    /// Indices of all the pointers that constitute this grouping.
+    SmallVector<unsigned, 2> Members;
+  };
+
+  /// \brief A memcheck which made up of a pair of grouped pointers.
+  ///
+  /// These *have* to be const for now, since checks are generated from
+  /// CheckingPtrGroups in LAI::addRuntimeCheck which is a const member
+  /// function.  FIXME: once check-generation is moved inside this class (after
+  /// the PtrPartition hack is removed), we could drop const.
+  typedef std::pair<const CheckingPtrGroup *, const CheckingPtrGroup *>
+      PointerCheck;
+
+  /// \brief Groups pointers such that a single memcheck is required
+  /// between two different groups. This will clear the CheckingGroups vector
+  /// and re-compute it. We will only group dependecies if \p UseDependencies
+  /// is true, otherwise we will create a separate group for each pointer.
+  void groupChecks(MemoryDepChecker::DepCandidates &DepCands,
+                   bool UseDependencies);
+
+  /// Generate the checks and return them.
+  ///
+  /// \p PtrToPartition contains the partition number for pointers.  If passed,
+  /// omit checks between pointers belonging to the same partition.  Partition
+  /// number -1 means that the pointer is used in multiple partitions.  In this
+  /// case we can't safely omit the check.
+  SmallVector<PointerCheck, 4>
+  generateChecks(const SmallVectorImpl<int> *PtrPartition = nullptr) const;
+
+  /// \brief Decide if we need to add a check between two groups of pointers,
+  /// according to needsChecking.
+  bool needsChecking(const CheckingPtrGroup &M, const CheckingPtrGroup &N,
+                     const SmallVectorImpl<int> *PtrPartition) const;
+
+  /// \brief Return true if any pointer requires run-time checking according
+  /// to needsChecking.
+  bool needsAnyChecking(const SmallVectorImpl<int> *PtrPartition) const;
+
+  /// \brief Returns the number of run-time checks required according to
+  /// needsChecking.
+  unsigned getNumberOfChecks(const SmallVectorImpl<int> *PtrPartition) const;
+
+  /// \brief Print the list run-time memory checks necessary.
+  ///
+  /// If \p PtrPartition is set, it contains the partition number for
+  /// pointers (-1 if the pointer belongs to multiple partitions).  In this
+  /// case omit checks between pointers belonging to the same partition.
+  void print(raw_ostream &OS, unsigned Depth = 0,
+             const SmallVectorImpl<int> *PtrPartition = nullptr) const;
+
+  /// Print \p Checks.
+  void printChecks(raw_ostream &OS, const SmallVectorImpl<PointerCheck> &Checks,
+                   unsigned Depth = 0) const;
+
+  /// This flag indicates if we need to add the runtime check.
+  bool Need;
+
+  /// Information about the pointers that may require checking.
+  SmallVector<PointerInfo, 2> Pointers;
+
+  /// Holds a partitioning of pointers into "check groups".
+  SmallVector<CheckingPtrGroup, 2> CheckingGroups;
+
+  /// \brief Check if pointers are in the same partition
+  ///
+  /// \p PtrToPartition contains the partition number for pointers (-1 if the
+  /// pointer belongs to multiple partitions).
+  static bool
+  arePointersInSamePartition(const SmallVectorImpl<int> &PtrToPartition,
+                             unsigned PtrIdx1, unsigned PtrIdx2);
+
+  /// \brief Decide whether we need to issue a run-time check for pointer at
+  /// index \p I and \p J to prove their independence.
+  ///
+  /// If \p PtrPartition is set, it contains the partition number for
+  /// pointers (-1 if the pointer belongs to multiple partitions).  In this
+  /// case omit checks between pointers belonging to the same partition.
+  bool needsChecking(unsigned I, unsigned J,
+                     const SmallVectorImpl<int> *PtrPartition = nullptr) const;
+
+private:
+  /// Holds a pointer to the ScalarEvolution analysis.
+  ScalarEvolution *SE;
+};
+
 /// \brief Drive the analysis of memory accesses in the loop
 ///
 /// This class is responsible for analyzing the memory accesses of a loop.  It
@@ -306,83 +465,25 @@ class MemoryDepChecker {
 /// RuntimePointerCheck class.
 class LoopAccessInfo {
 public:
-  /// This struct holds information about the memory runtime legality check that
-  /// a group of pointers do not overlap.
-  struct RuntimePointerCheck {
-    RuntimePointerCheck() : Need(false) {}
-
-    /// Reset the state of the pointer runtime information.
-    void reset() {
-      Need = false;
-      Pointers.clear();
-      Starts.clear();
-      Ends.clear();
-      IsWritePtr.clear();
-      DependencySetId.clear();
-      AliasSetId.clear();
-    }
-
-    /// Insert a pointer and calculate the start and end SCEVs.
-    void insert(ScalarEvolution *SE, Loop *Lp, Value *Ptr, bool WritePtr,
-                unsigned DepSetId, unsigned ASId,
-                const ValueToValueMap &Strides);
-
-    /// \brief No run-time memory checking is necessary.
-    bool empty() const { return Pointers.empty(); }
-
-    /// \brief Decide whether we need to issue a run-time check for pointer at
-    /// index \p I and \p J to prove their independence.
-    ///
-    /// If \p PtrPartition is set, it contains the partition number for
-    /// pointers (-1 if the pointer belongs to multiple partitions).  In this
-    /// case omit checks between pointers belonging to the same partition.
-    bool needsChecking(unsigned I, unsigned J,
-                       const SmallVectorImpl<int> *PtrPartition) const;
-
-    /// \brief Return true if any pointer requires run-time checking according
-    /// to needsChecking.
-    bool needsAnyChecking(const SmallVectorImpl<int> *PtrPartition) const;
-
-    /// \brief Print the list run-time memory checks necessary.
-    ///
-    /// If \p PtrPartition is set, it contains the partition number for
-    /// pointers (-1 if the pointer belongs to multiple partitions).  In this
-    /// case omit checks between pointers belonging to the same partition.
-    void print(raw_ostream &OS, unsigned Depth = 0,
-               const SmallVectorImpl<int> *PtrPartition = nullptr) const;
-
-    /// This flag indicates if we need to add the runtime check.
-    bool Need;
-    /// Holds the pointers that we need to check.
-    SmallVector<TrackingVH<Value>, 2> Pointers;
-    /// Holds the pointer value at the beginning of the loop.
-    SmallVector<const SCEV*, 2> Starts;
-    /// Holds the pointer value at the end of the loop.
-    SmallVector<const SCEV*, 2> Ends;
-    /// Holds the information if this pointer is used for writing to memory.
-    SmallVector<bool, 2> IsWritePtr;
-    /// Holds the id of the set of pointers that could be dependent because of a
-    /// shared underlying object.
-    SmallVector<unsigned, 2> DependencySetId;
-    /// Holds the id of the disjoint alias set to which this pointer belongs.
-    SmallVector<unsigned, 2> AliasSetId;
-  };
-
   LoopAccessInfo(Loop *L, ScalarEvolution *SE, const DataLayout &DL,
                  const TargetLibraryInfo *TLI, AliasAnalysis *AA,
-                 DominatorTree *DT, const ValueToValueMap &Strides);
+                 DominatorTree *DT, LoopInfo *LI,
+                 const ValueToValueMap &Strides);
 
   /// Return true we can analyze the memory accesses in the loop and there are
   /// no memory dependence cycles.
   bool canVectorizeMemory() const { return CanVecMem; }
 
-  const RuntimePointerCheck *getRuntimePointerCheck() const {
-    return &PtrRtCheck;
+  const RuntimePointerChecking *getRuntimePointerChecking() const {
+    return &PtrRtChecking;
   }
 
   /// \brief Number of memchecks required to prove independence of otherwise
   /// may-alias pointers.
-  unsigned getNumRuntimePointerChecks() const { return NumComparisons; }
+  unsigned getNumRuntimePointerChecks(
+    const SmallVectorImpl<int> *PtrPartition = nullptr) const {
+    return PtrRtChecking.getNumberOfChecks(PtrPartition);
+  }
 
   /// Return true if the block BB needs to be predicated in order for the loop
   /// to be vectorized.
@@ -409,6 +510,16 @@ class LoopAccessInfo {
   addRuntimeCheck(Instruction *Loc,
                   const SmallVectorImpl<int> *PtrPartition = nullptr) const;
 
+  /// \brief Generete the instructions for the checks in \p PointerChecks.
+  ///
+  /// Returns a pair of instructions where the first element is the first
+  /// instruction generated in possibly a sequence of instructions and the
+  /// second value is the final comparator value or NULL if no check is needed.
+  std::pair<Instruction *, Instruction *>
+  addRuntimeCheck(Instruction *Loc,
+                  const SmallVectorImpl<RuntimePointerChecking::PointerCheck>
+                      &PointerChecks) const;
+
   /// \brief The diagnostics report generated for the analysis.  E.g. why we
   /// couldn't analyze the loop.
   const Optional<LoopAccessReport> &getReport() const { return Report; }
@@ -451,22 +562,19 @@ class LoopAccessInfo {
 
   /// We need to check that all of the pointers in this list are disjoint
   /// at runtime.
-  RuntimePointerCheck PtrRtCheck;
+  RuntimePointerChecking PtrRtChecking;
 
   /// \brief the Memory Dependence Checker which can determine the
   /// loop-independent and loop-carried dependences between memory accesses.
   MemoryDepChecker DepChecker;
 
-  /// \brief Number of memchecks required to prove independence of otherwise
-  /// may-alias pointers
-  unsigned NumComparisons;
-
   Loop *TheLoop;
   ScalarEvolution *SE;
   const DataLayout &DL;
   const TargetLibraryInfo *TLI;
   AliasAnalysis *AA;
   DominatorTree *DT;
+  LoopInfo *LI;
 
   unsigned NumLoads;
   unsigned NumStores;
@@ -497,6 +605,11 @@ const SCEV *replaceSymbolicStrideSCEV(ScalarEvolution *SE,
                                       const ValueToValueMap &PtrToStride,
                                       Value *Ptr, Value *OrigPtr = nullptr);
 
+/// \brief Check the stride of the pointer and ensure that it does not wrap in
+/// the address space.
+int isStridedPtr(ScalarEvolution *SE, Value *Ptr, const Loop *Lp,
+                 const ValueToValueMap &StridesMap);
+
 /// \brief This analysis provides dependence information for the memory accesses
 /// of a loop.
 ///
@@ -541,6 +654,7 @@ class LoopAccessAnalysis : public FunctionPass {
   const TargetLibraryInfo *TLI;
   AliasAnalysis *AA;
   DominatorTree *DT;
+  LoopInfo *LI;
 };
 } // End llvm namespace
 
diff --git a/include/llvm/Analysis/LoopInfo.h b/include/llvm/Analysis/LoopInfo.h
index f3d85e68404..841c002a5c6 100644
--- a/include/llvm/Analysis/LoopInfo.h
+++ b/include/llvm/Analysis/LoopInfo.h
@@ -47,13 +47,6 @@ namespace llvm {
 template <typename IRUnitT> class AnalysisManager;
 class PreservedAnalyses;
 
-template<typename T>
-inline void RemoveFromVector(std::vector<T*> &V, T *N) {
-  typename std::vector<T*>::iterator I = std::find(V.begin(), V.end(), N);
-  assert(I != V.end() && "N is not in this list!");
-  V.erase(I);
-}
-
 class DominatorTree;
 class LoopInfo;
 class Loop;
@@ -324,7 +317,10 @@ class LoopBase {
   /// current loop, updating the Blocks as appropriate.  This does not update
   /// the mapping in the LoopInfo class.
   void removeBlockFromLoop(BlockT *BB) {
-    RemoveFromVector(Blocks, BB);
+    auto I = std::find(Blocks.begin(), Blocks.end(), BB);
+    assert(I != Blocks.end() && "N is not in this list!");
+    Blocks.erase(I);
+
     DenseBlockSet.erase(BB);
   }
 
@@ -351,9 +347,7 @@ raw_ostream& operator<<(raw_ostream &OS, const LoopBase<BlockT, LoopT> &Loop) {
 }
 
 // Implementation in LoopInfoImpl.h
-#ifdef __GNUC__
-__extension__ extern template class LoopBase<BasicBlock, Loop>;
-#endif
+extern template class LoopBase<BasicBlock, Loop>;
 
 class Loop : public LoopBase<BasicBlock, Loop> {
 public:
@@ -361,11 +355,11 @@ class Loop : public LoopBase<BasicBlock, Loop> {
 
   /// isLoopInvariant - Return true if the specified value is loop invariant
   ///
-  bool isLoopInvariant(Value *V) const;
+  bool isLoopInvariant(const Value *V) const;
 
   /// hasLoopInvariantOperands - Return true if all the operands of the
   /// specified instruction are loop invariant.
-  bool hasLoopInvariantOperands(Instruction *I) const;
+  bool hasLoopInvariantOperands(const Instruction *I) const;
 
   /// makeLoopInvariant - If the given value is an instruction inside of the
   /// loop and it can be hoisted, do so to make it trivially loop-invariant.
@@ -493,7 +487,7 @@ class Loop : public LoopBase<BasicBlock, Loop> {
 template<class BlockT, class LoopT>
 class LoopInfoBase {
   // BBMap - Mapping of basic blocks to the inner most loop they occur in
-  DenseMap<BlockT *, LoopT *> BBMap;
+  DenseMap<const BlockT *, LoopT *> BBMap;
   std::vector<LoopT *> TopLevelLoops;
   friend class LoopBase<BlockT, LoopT>;
   friend class LoopInfo;
@@ -543,9 +537,7 @@ class LoopInfoBase {
   /// getLoopFor - Return the inner most loop that BB lives in.  If a basic
   /// block is in no loop (for example the entry node), null is returned.
   ///
-  LoopT *getLoopFor(const BlockT *BB) const {
-    return BBMap.lookup(const_cast<BlockT*>(BB));
-  }
+  LoopT *getLoopFor(const BlockT *BB) const { return BBMap.lookup(BB); }
 
   /// operator[] - same as getLoopFor...
   ///
@@ -562,7 +554,7 @@ class LoopInfoBase {
   }
 
   // isLoopHeader - True if the block is a loop header node
-  bool isLoopHeader(BlockT *BB) const {
+  bool isLoopHeader(const BlockT *BB) const {
     const LoopT *L = getLoopFor(BB);
     return L && L->getHeader() == BB;
   }
@@ -630,7 +622,7 @@ class LoopInfoBase {
   }
 
   /// Create the loop forest using a stable algorithm.
-  void Analyze(DominatorTreeBase<BlockT> &DomTree);
+  void analyze(const DominatorTreeBase<BlockT> &DomTree);
 
   // Debugging
   void print(raw_ostream &OS) const;
@@ -639,9 +631,7 @@ class LoopInfoBase {
 };
 
 // Implementation in LoopInfoImpl.h
-#ifdef __GNUC__
-__extension__ extern template class LoopInfoBase<BasicBlock, Loop>;
-#endif
+extern template class LoopInfoBase<BasicBlock, Loop>;
 
 class LoopInfo : public LoopInfoBase<BasicBlock, Loop> {
   typedef LoopInfoBase<BasicBlock, Loop> BaseT;
@@ -652,6 +642,7 @@ class LoopInfo : public LoopInfoBase<BasicBlock, Loop> {
   LoopInfo(const LoopInfo &) = delete;
 public:
   LoopInfo() {}
+  explicit LoopInfo(const DominatorTreeBase<BasicBlock> &DomTree);
 
   LoopInfo(LoopInfo &&Arg) : BaseT(std::move(static_cast<BaseT &>(Arg))) {}
   LoopInfo &operator=(LoopInfo &&RHS) {
@@ -729,12 +720,6 @@ class LoopAnalysis {
   /// \brief Provide a name for the analysis for debugging and logging.
   static StringRef name() { return "LoopAnalysis"; }
 
-  LoopAnalysis() {}
-  LoopAnalysis(const LoopAnalysis &Arg) {}
-  LoopAnalysis(LoopAnalysis &&Arg) {}
-  LoopAnalysis &operator=(const LoopAnalysis &RHS) { return *this; }
-  LoopAnalysis &operator=(LoopAnalysis &&RHS) { return *this; }
-
   LoopInfo run(Function &F, AnalysisManager<Function> *AM);
 };
 
diff --git a/include/llvm/Analysis/LoopInfoImpl.h b/include/llvm/Analysis/LoopInfoImpl.h
index 0490bb1d761..dcd9a0f4cbc 100644
--- a/include/llvm/Analysis/LoopInfoImpl.h
+++ b/include/llvm/Analysis/LoopInfoImpl.h
@@ -345,7 +345,7 @@ void LoopBase<BlockT, LoopT>::print(raw_ostream &OS, unsigned Depth) const {
 template<class BlockT, class LoopT>
 static void discoverAndMapSubloop(LoopT *L, ArrayRef<BlockT*> Backedges,
                                   LoopInfoBase<BlockT, LoopT> *LI,
-                                  DominatorTreeBase<BlockT> &DomTree) {
+                                  const DominatorTreeBase<BlockT> &DomTree) {
   typedef GraphTraits<Inverse<BlockT*> > InvBlockTraits;
 
   unsigned NumBlocks = 0;
@@ -468,10 +468,10 @@ void PopulateLoopsDFS<BlockT, LoopT>::insertIntoLoop(BlockT *Block) {
 /// insertions per block.
 template<class BlockT, class LoopT>
 void LoopInfoBase<BlockT, LoopT>::
-Analyze(DominatorTreeBase<BlockT> &DomTree) {
+analyze(const DominatorTreeBase<BlockT> &DomTree) {
 
   // Postorder traversal of the dominator tree.
-  DomTreeNodeBase<BlockT>* DomRoot = DomTree.getRootNode();
+  const DomTreeNodeBase<BlockT> *DomRoot = DomTree.getRootNode();
   for (auto DomNode : post_order(DomRoot)) {
 
     BlockT *Header = DomNode->getBlock();
@@ -527,7 +527,7 @@ void LoopInfoBase<BlockT, LoopT>::verify() const {
   // Verify that blocks are mapped to valid loops.
 #ifndef NDEBUG
   for (auto &Entry : BBMap) {
-    BlockT *BB = Entry.first;
+    const BlockT *BB = Entry.first;
     LoopT *L = Entry.second;
     assert(Loops.count(L) && "orphaned loop");
     assert(L->contains(BB) && "orphaned block");
diff --git a/include/llvm/Analysis/MemoryDependenceAnalysis.h b/include/llvm/Analysis/MemoryDependenceAnalysis.h
index c8453e9ea34..511898071c2 100644
--- a/include/llvm/Analysis/MemoryDependenceAnalysis.h
+++ b/include/llvm/Analysis/MemoryDependenceAnalysis.h
@@ -19,6 +19,7 @@
 #include "llvm/ADT/SmallPtrSet.h"
 #include "llvm/Analysis/AliasAnalysis.h"
 #include "llvm/IR/BasicBlock.h"
+#include "llvm/IR/PredIteratorCache.h"
 #include "llvm/IR/ValueHandle.h"
 #include "llvm/Pass.h"
 
@@ -286,7 +287,7 @@ namespace llvm {
       /// conflicting tags.
       AAMDNodes AATags;
 
-      NonLocalPointerInfo() : Size(AliasAnalysis::UnknownSize) {}
+      NonLocalPointerInfo() : Size(MemoryLocation::UnknownSize) {}
     };
 
     /// CachedNonLocalPointerInfo - This map stores the cached results of doing
@@ -325,7 +326,7 @@ namespace llvm {
     AliasAnalysis *AA;
     DominatorTree *DT;
     AssumptionCache *AC;
-    std::unique_ptr<PredIteratorCache> PredCache;
+    PredIteratorCache PredCache;
 
   public:
     MemoryDependenceAnalysis();
@@ -402,13 +403,12 @@ namespace llvm {
     ///
     /// Note that this is an uncached query, and thus may be inefficient.
     ///
-    MemDepResult getPointerDependencyFrom(const AliasAnalysis::Location &Loc,
+    MemDepResult getPointerDependencyFrom(const MemoryLocation &Loc,
                                           bool isLoad,
                                           BasicBlock::iterator ScanIt,
                                           BasicBlock *BB,
                                           Instruction *QueryInst = nullptr);
 
-
     /// getLoadLoadClobberFullWidthSize - This is a little bit of analysis that
     /// looks at a memory location for a load (specified by MemLocBase, Offs,
     /// and Size) and compares it against a load.  If the specified load could
@@ -427,15 +427,14 @@ namespace llvm {
                                            BasicBlock *BB);
     bool getNonLocalPointerDepFromBB(Instruction *QueryInst,
                                      const PHITransAddr &Pointer,
-                                     const AliasAnalysis::Location &Loc,
-                                     bool isLoad, BasicBlock *BB,
+                                     const MemoryLocation &Loc, bool isLoad,
+                                     BasicBlock *BB,
                                      SmallVectorImpl<NonLocalDepResult> &Result,
-                                     DenseMap<BasicBlock*, Value*> &Visited,
+                                     DenseMap<BasicBlock *, Value *> &Visited,
                                      bool SkipFirstBlock = false);
     MemDepResult GetNonLocalInfoForBlock(Instruction *QueryInst,
-                                         const AliasAnalysis::Location &Loc,
-                                         bool isLoad, BasicBlock *BB,
-                                         NonLocalDepInfo *Cache,
+                                         const MemoryLocation &Loc, bool isLoad,
+                                         BasicBlock *BB, NonLocalDepInfo *Cache,
                                          unsigned NumSortedEntries);
 
     void RemoveCachedNonLocalPointerDependencies(ValueIsLoadPair P);
diff --git a/include/llvm/Analysis/MemoryLocation.h b/include/llvm/Analysis/MemoryLocation.h
new file mode 100644
index 00000000000..426b49a3ecd
--- /dev/null
+++ b/include/llvm/Analysis/MemoryLocation.h
@@ -0,0 +1,142 @@
+//===- MemoryLocation.h - Memory location descriptions ----------*- C++ -*-===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+/// \file
+/// This file provides utility analysis objects describing memory locations.
+/// These are used both by the Alias Analysis infrastructure and more
+/// specialized memory analysis layers.
+///
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_ANALYSIS_MEMORYLOCATION_H
+#define LLVM_ANALYSIS_MEMORYLOCATION_H
+
+#include "llvm/ADT/DenseMap.h"
+#include "llvm/IR/CallSite.h"
+#include "llvm/IR/Metadata.h"
+
+namespace llvm {
+
+class LoadInst;
+class StoreInst;
+class MemTransferInst;
+class MemIntrinsic;
+class TargetLibraryInfo;
+
+/// Representation for a specific memory location.
+///
+/// This abstraction can be used to represent a specific location in memory.
+/// The goal of the location is to represent enough information to describe
+/// abstract aliasing, modification, and reference behaviors of whatever
+/// value(s) are stored in memory at the particular location.
+///
+/// The primary user of this interface is LLVM's Alias Analysis, but other
+/// memory analyses such as MemoryDependence can use it as well.
+class MemoryLocation {
+public:
+  /// UnknownSize - This is a special value which can be used with the
+  /// size arguments in alias queries to indicate that the caller does not
+  /// know the sizes of the potential memory references.
+  enum : uint64_t { UnknownSize = ~UINT64_C(0) };
+
+  /// The address of the start of the location.
+  const Value *Ptr;
+
+  /// The maximum size of the location, in address-units, or
+  /// UnknownSize if the size is not known.
+  ///
+  /// Note that an unknown size does not mean the pointer aliases the entire
+  /// virtual address space, because there are restrictions on stepping out of
+  /// one object and into another. See
+  /// http://llvm.org/docs/LangRef.html#pointeraliasing
+  uint64_t Size;
+
+  /// The metadata nodes which describes the aliasing of the location (each
+  /// member is null if that kind of information is unavailable).
+  AAMDNodes AATags;
+
+  /// Return a location with information about the memory reference by the given
+  /// instruction.
+  static MemoryLocation get(const LoadInst *LI);
+  static MemoryLocation get(const StoreInst *SI);
+  static MemoryLocation get(const VAArgInst *VI);
+  static MemoryLocation get(const AtomicCmpXchgInst *CXI);
+  static MemoryLocation get(const AtomicRMWInst *RMWI);
+  static MemoryLocation get(const Instruction *Inst) {
+    if (auto *I = dyn_cast<LoadInst>(Inst))
+      return get(I);
+    else if (auto *I = dyn_cast<StoreInst>(Inst))
+      return get(I);
+    else if (auto *I = dyn_cast<VAArgInst>(Inst))
+      return get(I);
+    else if (auto *I = dyn_cast<AtomicCmpXchgInst>(Inst))
+      return get(I);
+    else if (auto *I = dyn_cast<AtomicRMWInst>(Inst))
+      return get(I);
+    llvm_unreachable("unsupported memory instruction");
+  }
+
+  /// Return a location representing the source of a memory transfer.
+  static MemoryLocation getForSource(const MemTransferInst *MTI);
+
+  /// Return a location representing the destination of a memory set or
+  /// transfer.
+  static MemoryLocation getForDest(const MemIntrinsic *MI);
+
+  /// Return a location representing a particular argument of a call.
+  static MemoryLocation getForArgument(ImmutableCallSite CS, unsigned ArgIdx,
+                                       const TargetLibraryInfo &TLI);
+
+  explicit MemoryLocation(const Value *Ptr = nullptr,
+                          uint64_t Size = UnknownSize,
+                          const AAMDNodes &AATags = AAMDNodes())
+      : Ptr(Ptr), Size(Size), AATags(AATags) {}
+
+  MemoryLocation getWithNewPtr(const Value *NewPtr) const {
+    MemoryLocation Copy(*this);
+    Copy.Ptr = NewPtr;
+    return Copy;
+  }
+
+  MemoryLocation getWithNewSize(uint64_t NewSize) const {
+    MemoryLocation Copy(*this);
+    Copy.Size = NewSize;
+    return Copy;
+  }
+
+  MemoryLocation getWithoutAATags() const {
+    MemoryLocation Copy(*this);
+    Copy.AATags = AAMDNodes();
+    return Copy;
+  }
+
+  bool operator==(const MemoryLocation &Other) const {
+    return Ptr == Other.Ptr && Size == Other.Size && AATags == Other.AATags;
+  }
+};
+
+// Specialize DenseMapInfo for MemoryLocation.
+template <> struct DenseMapInfo<MemoryLocation> {
+  static inline MemoryLocation getEmptyKey() {
+    return MemoryLocation(DenseMapInfo<const Value *>::getEmptyKey(), 0);
+  }
+  static inline MemoryLocation getTombstoneKey() {
+    return MemoryLocation(DenseMapInfo<const Value *>::getTombstoneKey(), 0);
+  }
+  static unsigned getHashValue(const MemoryLocation &Val) {
+    return DenseMapInfo<const Value *>::getHashValue(Val.Ptr) ^
+           DenseMapInfo<uint64_t>::getHashValue(Val.Size) ^
+           DenseMapInfo<AAMDNodes>::getHashValue(Val.AATags);
+  }
+  static bool isEqual(const MemoryLocation &LHS, const MemoryLocation &RHS) {
+    return LHS == RHS;
+  }
+};
+}
+
+#endif
diff --git a/include/llvm/Analysis/NaCl/PNaClABIProps.h b/include/llvm/Analysis/NaCl/PNaClABIProps.h
deleted file mode 100644
index dfb3d0d512b..00000000000
--- a/include/llvm/Analysis/NaCl/PNaClABIProps.h
+++ /dev/null
@@ -1,80 +0,0 @@
-//===- PNaClABIProps.h - Verify PNaCl ABI Properties ----------------------===//
-//
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
-//
-//===----------------------------------------------------------------------===//
-//
-// Verify PNaCl ABI properties.
-//
-//
-//===----------------------------------------------------------------------===//
-
-#ifndef LLVM_ANALYSIS_NACL_PNACLABIPROPS_H
-#define LLVM_ANALYSIS_NACL_PNACLABIPROPS_H
-
-#include "llvm/ADT/APInt.h"
-#include "llvm/IR/CallingConv.h"
-#include "llvm/IR/GlobalValue.h"
-#include "llvm/IR/Type.h"
-
-namespace llvm {
-
-class NamedMDNode;
-class DataLayout;
-
-// Checks properties needed to verify IR constructs. Unlike
-// PNaClABIVerifyFunctions and PNaClABIVerifyModule, this class is
-// pass-free, and checks individual elements within IR.
-class PNaClABIProps {
-  PNaClABIProps(const PNaClABIProps&) = delete;
-  void operator=(const PNaClABIProps&) = delete;
-public:
-  // Returns true if metadata kind MDKind is allowed.
-  static bool isWhitelistedMetadata(unsigned MDKind);
-  // Returns true if metadata is allowed.
-  static bool isWhitelistedMetadata(const NamedMDNode *MD);
-  // Returns true if integer constant Idx is in [0..NumElements).
-  static bool isVectorIndexSafe(const APInt &Idx,
-                                unsigned NumElements) {
-    return Idx.ult(NumElements);
-  }
-  // Returns true if Alignment is allowed for type Ty, assuming DL.
-  static bool isAllowedAlignment(const DataLayout *DL, uint64_t Alignment,
-                                 const Type *Ty);
-
-  // Returns true if alloca type Ty is correct.
-  static bool isAllocaAllocatedType(const Type *Ty) {
-    return Ty->isIntegerTy(8);
-  }
-  // Returns true if the type associated with the size field in an alloca
-  // instruction is valid.
-  static bool isAllocaSizeType(const Type *Ty) {
-    return Ty->isIntegerTy(32);
-  }
-  // Returns string describing expected type for the size field in an
-  // alloca instruction.
-  static const char *ExpectedAllocaSizeType() {
-    return "alloca array size is not i32";
-  }
-  // Returns the name for the given calling convention.
-  static const char *CallingConvName(CallingConv::ID CallingConv);
-  // Returns true if CallingConv is valid.
-  static bool isValidCallingConv(CallingConv::ID CallingConv) {
-    return CallingConv == CallingConv::C;
-  }
-  // Returns the name for linkage type LT.
-  static const char *LinkageName(GlobalValue::LinkageTypes LT);
-  // Returns true if Linkage is valid.
-  static bool isValidGlobalLinkage(GlobalValue::LinkageTypes Linkage);
-  // Returns kind of global value name, based on IsFunction.
-  static const char *GVTypeName(bool IsFunction) {
-    return IsFunction ? "Function" : "Variable";
-  }
-};
-
-}
-
-#endif  // LLVM_ANALYSIS_NACL_PNACLABIPROPS_H
diff --git a/include/llvm/Analysis/NaCl/PNaClABITypeChecker.h b/include/llvm/Analysis/NaCl/PNaClABITypeChecker.h
deleted file mode 100644
index 7d0a4643127..00000000000
--- a/include/llvm/Analysis/NaCl/PNaClABITypeChecker.h
+++ /dev/null
@@ -1,80 +0,0 @@
-//===- PNaClABITypeChecker.h - Verify PNaCl ABI rules -----------*- C++ -*-===//
-//
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
-//
-//===----------------------------------------------------------------------===//
-//
-// Common type-checking code for module and function-level passes
-//
-//
-//===----------------------------------------------------------------------===//
-
-#ifndef LLVM_ANALYSIS_NACL_PNACLABITYPECHECKER_H
-#define LLVM_ANALYSIS_NACL_PNACLABITYPECHECKER_H
-
-#include "llvm/ADT/DenseSet.h"
-#include "llvm/IR/Type.h"
-#include "llvm/Support/raw_ostream.h"
-
-namespace llvm {
-class FunctionType;
-
-class PNaClABITypeChecker {
-  PNaClABITypeChecker(const PNaClABITypeChecker&) = delete;
-  void operator=(const PNaClABITypeChecker&) = delete;
-public:
-  // Returns true if Ty is a valid argument or return value type for PNaCl.
-  static bool isValidParamType(const Type *Ty);
-
-  // Returns true if Ty is a valid function type for PNaCl.
-  static bool isValidFunctionType(const FunctionType *FTy);
-
-  // Returns true if Ty is a valid non-derived type for PNaCl.
-  static bool isValidScalarType(const Type *Ty);
-
-  // Returns true if Ty is a valid vector type for PNaCl.
-  static bool isValidVectorType(const Type *Ty);
-
-  // Returns true if type Ty can be used in (integer) arithmetic operations.
-  static bool isValidIntArithmeticType(const Type *Ty);
-
-  // Returns true if type Ty can be used to define the test condition of
-  // a switch instruction.
-  static bool isValidSwitchConditionType(const Type *Ty) {
-    return PNaClABITypeChecker::isValidIntArithmeticType(Ty);
-  }
-  // Returns error message showing what was expected when given the
-  // switch condition type Ty. Assumes isValidSwitchConditionType(Ty)
-  // returned false.
-  static const char *ExpectedSwitchConditionType(const Type *Ty) {
-    if (!Ty->isIntegerTy())
-      return "switch not on integer type";
-    if (Ty->isIntegerTy(1))
-      return "switch on i1 not allowed";
-    return "switch disallowed for integer type";
-  }
-
-  // There's no built-in way to get the name of a type, so use a
-  // string ostream to print it.
-  static std::string getTypeName(const Type *T) {
-    std::string TypeName;
-    raw_string_ostream N(TypeName);
-    T->print(N);
-    return N.str();
-  }
-
-  // Returns true if T1 is equivalent to T2, converting to i32 if
-  // a pointer type.
-  static bool IsPointerEquivType(Type *T1, Type *T2) {
-    if (T1->isPointerTy()) return T2->isIntegerTy(32);
-    if (T2->isPointerTy()) return T1->isIntegerTy(32);
-    return T1 == T2;
-  }
-
-};
-} // namespace llvm
-
-#endif // LLVM_ANALYSIS_NACL_PNACLABITYPECHECKER_H
diff --git a/include/llvm/Analysis/NaCl/PNaClABIVerifyFunctions.h b/include/llvm/Analysis/NaCl/PNaClABIVerifyFunctions.h
deleted file mode 100644
index f456d2ffd47..00000000000
--- a/include/llvm/Analysis/NaCl/PNaClABIVerifyFunctions.h
+++ /dev/null
@@ -1,67 +0,0 @@
-//===- PNaClABIVerifyFunctions.h - Verify PNaCl ABI rules -----------------===//
-//
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
-//
-//===----------------------------------------------------------------------===//
-//
-// Verify function-level PNaCl ABI requirements.
-//
-//
-//===----------------------------------------------------------------------===//
-
-#ifndef LLVM_ANALYSIS_NACL_PNACLABIVERIFYFUNCTIONS_H
-#define LLVM_ANALYSIS_NACL_PNACLABIVERIFYFUNCTIONS_H
-
-#include "llvm/Analysis/NaCl/PNaClABIProps.h"
-
-#include "llvm/Analysis/NaCl.h"
-#include "llvm/IR/DataLayout.h"
-#include "llvm/IR/Module.h"
-#include "llvm/IR/NaClAtomicIntrinsics.h"
-#include "llvm/Pass.h"
-
-namespace llvm {
-
-// Checks that examine anything in the function body should be in
-// FunctionPasses to make them streaming-friendly.
-class PNaClABIVerifyFunctions : public FunctionPass {
-  PNaClABIVerifyFunctions(const PNaClABIVerifyFunctions&) = delete;
-  void operator=(const PNaClABIVerifyFunctions&) = delete;
- public:
-  static char ID;
-  PNaClABIVerifyFunctions() :
-      FunctionPass(ID),
-      Reporter(new PNaClABIErrorReporter),
-      ReporterIsOwned(true) {
-    initializePNaClABIVerifyFunctionsPass(*PassRegistry::getPassRegistry());
-  }
-  explicit PNaClABIVerifyFunctions(PNaClABIErrorReporter *Reporter_) :
-      FunctionPass(ID),
-      Reporter(Reporter_),
-      ReporterIsOwned(false) {
-    initializePNaClABIVerifyFunctionsPass(*PassRegistry::getPassRegistry());
-  }
-  virtual ~PNaClABIVerifyFunctions();
-  virtual bool doInitialization(Module &M) {
-    AtomicIntrinsics.reset(new NaCl::AtomicIntrinsics(M.getContext()));
-    return false;
-  }
-  virtual void getAnalysisUsage(AnalysisUsage &Info) const {
-    Info.setPreservesAll();
-  }
-  bool runOnFunction(Function &F);
-  virtual void print(raw_ostream &O, const Module *M) const;
-
-private:
-  const char *checkInstruction(const DataLayout *DL, const Instruction *Inst);
-  PNaClABIErrorReporter *Reporter;
-  bool ReporterIsOwned;
-  std::unique_ptr<NaCl::AtomicIntrinsics> AtomicIntrinsics;
-};
-
-}
-
-#endif  // LLVM_ANALYSIS_NACL_PNACLABIVERIFYFUNCTIONS_H
diff --git a/include/llvm/Analysis/NaCl/PNaClABIVerifyModule.h b/include/llvm/Analysis/NaCl/PNaClABIVerifyModule.h
deleted file mode 100644
index e43fccb9dbd..00000000000
--- a/include/llvm/Analysis/NaCl/PNaClABIVerifyModule.h
+++ /dev/null
@@ -1,76 +0,0 @@
-//===- PNaClABIVerifyModule.h - Verify PNaCl ABI rules ----------*- C++ -*-===//
-//
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
-//
-//===----------------------------------------------------------------------===//
-//
-// Verify module-level PNaCl ABI requirements (specifically those that do not
-// require looking at the function bodies).
-//
-//
-//===----------------------------------------------------------------------===//
-
-#ifndef LLVM_ANALYSIS_NACL_PNACLABIVERIFYMODULE_H
-#define LLVM_ANALYSIS_NACL_PNACLABIVERIFYMODULE_H
-
-#include "llvm/ADT/StringMap.h"
-#include "llvm/Analysis/NaCl.h"
-#include "llvm/IR/Module.h"
-#include "llvm/Pass.h"
-
-namespace llvm {
-
-class PNaClAllowedIntrinsics;
-
-// This pass should not touch function bodies, to stay streaming-friendly
-class PNaClABIVerifyModule : public ModulePass {
-  PNaClABIVerifyModule(const PNaClABIVerifyModule&) = delete;
-  void operator=(const PNaClABIVerifyModule&) = delete;
- public:
-  static char ID;
-  PNaClABIVerifyModule() :
-      ModulePass(ID),
-      Reporter(new PNaClABIErrorReporter),
-      ReporterIsOwned(true),
-      StreamingMode(false),
-      SeenEntryPoint(false) {
-    initializePNaClABIVerifyModulePass(*PassRegistry::getPassRegistry());
-  }
-  PNaClABIVerifyModule(PNaClABIErrorReporter *Reporter_,
-                       bool StreamingMode) :
-      ModulePass(ID),
-      Reporter(Reporter_),
-      ReporterIsOwned(false),
-      StreamingMode(StreamingMode),
-      SeenEntryPoint(false) {
-    initializePNaClABIVerifyModulePass(*PassRegistry::getPassRegistry());
-  }
-  virtual ~PNaClABIVerifyModule();
-  bool runOnModule(Module &M);
-  virtual void print(raw_ostream &O, const Module *M) const;
-
-  // Checks validity of function declaration F with given name Name.
-  // (see PNaClABIVerifyFunctions.h for handling function bodies).
-  void checkFunction(const Function *F, const StringRef &Name,
-                     PNaClAllowedIntrinsics &Intrinsics);
-  // Checks validity of global variable declaration GV.
-  void checkGlobalVariable(const GlobalVariable *GV) {
-    return checkGlobalValue(GV);
-  }
- private:
-  void checkGlobalValue(const GlobalValue *GV);
-  /// Checks whether \p GV is an allowed external symbol in stable bitcode.
-  void checkExternalSymbol(const GlobalValue *GV);
-
-  void checkGlobalIsFlattened(const GlobalVariable *GV);
-  PNaClABIErrorReporter *Reporter;
-  bool ReporterIsOwned;
-  bool StreamingMode;
-  bool SeenEntryPoint;
-};
-
-}
-#endif  // LLVM_ANALYSIS_NACL_PNACLABIVERIFYMODULE_H
diff --git a/include/llvm/Analysis/NaCl/PNaClAllowedIntrinsics.h b/include/llvm/Analysis/NaCl/PNaClAllowedIntrinsics.h
deleted file mode 100644
index f9b7e669b5f..00000000000
--- a/include/llvm/Analysis/NaCl/PNaClAllowedIntrinsics.h
+++ /dev/null
@@ -1,73 +0,0 @@
-//===- PNaClAllowedIntrinsics.h - Set of allowed intrinsics -----*- C++ -*-===//
-//
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
-//
-//===----------------------------------------------------------------------===//
-//
-// Declares class that holds set of allowed PNaCl intrinsics.
-//
-//===----------------------------------------------------------------------===//
-
-#ifndef LLVM_ANALYSIS_NACL_PNACLALLOWEDINTRINSICS_H
-#define LLVM_ANALYSIS_NACL_PNACLALLOWEDINTRINSICS_H
-
-#include "llvm/ADT/StringMap.h"
-#include "llvm/IR/Intrinsics.h"
-
-namespace llvm {
-
-class LLVMContext;
-class Function;
-class FunctionType;
-
-// Holds the set of allowed instrinsics.
-class PNaClAllowedIntrinsics {
-  PNaClAllowedIntrinsics(const PNaClAllowedIntrinsics&) = delete;
-  void operator=(const PNaClAllowedIntrinsics&) = delete;
-public:
-  PNaClAllowedIntrinsics(LLVMContext *Context);
-
-  // Checks if there is an allowable PNaCl intrinsic function with the
-  // given name and type signature.
-  bool isAllowed(const std::string &FcnName, const FunctionType *FcnType) {
-    return isIntrinsicName(FcnName) && FcnType == getIntrinsicType(FcnName);
-  }
-  // Checks if Func is an allowed PNaCl intrinsic function.  Note:
-  // This function also allows debugging intrinsics if
-  // PNaClABIAllowDebugMetadata is true.
-  bool isAllowed(const Function *Func);
-
-  // Returns the type signature for the Name'd intrinsic, if entered
-  // via a call to AddIntrinsic. Returns 0 otherwise (implying we
-  // don't know the expected type signature).
-  FunctionType *getIntrinsicType(const std::string &Name) {
-    return isIntrinsicName(Name) ? TypeMap[Name] : 0;
-  }
-
-  static bool isAllowedDebugInfoIntrinsic(unsigned IntrinsicID);
-
-private:
-  LLVMContext *Context;
-  // Maps from an allowed intrinsic's name to its type.
-  StringMap<FunctionType *> TypeMap;
-
-  // Tys is an array of type parameters for the intrinsic.  This
-  // defaults to an empty array.
-  void addIntrinsic(Intrinsic::ID ID,
-                    ArrayRef<Type *> Tys = ArrayRef<Type*>());
-
-  // Returns true if a valid PNaCl intrinsic name.
-  bool isIntrinsicName(const std::string &Name) {
-    return TypeMap.count(Name) == 1;
-  }
-
-  // Returns true if intrinsic ID is allowed as a PNaCl intrinsic.
-  bool isAllowedIntrinsicID(unsigned ID);
-};
-
-}
-
-#endif // LLVM_ANALYSIS_NACL_PNACLALLOWEDINTRINSICS_H
diff --git a/include/llvm/Analysis/OrderedBasicBlock.h b/include/llvm/Analysis/OrderedBasicBlock.h
new file mode 100644
index 00000000000..5aa813eb483
--- /dev/null
+++ b/include/llvm/Analysis/OrderedBasicBlock.h
@@ -0,0 +1,66 @@
+//===- llvm/Analysis/OrderedBasicBlock.h --------------------- -*- C++ -*-===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This file defines the OrderedBasicBlock class. OrderedBasicBlock maintains
+// an interface where clients can query if one instruction comes before another
+// in a BasicBlock. Since BasicBlock currently lacks a reliable way to query
+// relative position between instructions one can use OrderedBasicBlock to do
+// such queries. OrderedBasicBlock is lazily built on a source BasicBlock and
+// maintains an internal Instruction -> Position map. A OrderedBasicBlock
+// instance should be discarded whenever the source BasicBlock changes.
+//
+// It's currently used by the CaptureTracker in order to find relative
+// positions of a pair of instructions inside a BasicBlock.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_ANALYSIS_ORDEREDBASICBLOCK_H
+#define LLVM_ANALYSIS_ORDEREDBASICBLOCK_H
+
+#include "llvm/ADT/DenseMap.h"
+#include "llvm/IR/BasicBlock.h"
+
+namespace llvm {
+
+class Instruction;
+class BasicBlock;
+
+class OrderedBasicBlock {
+private:
+  /// \brief Map a instruction to its position in a BasicBlock.
+  SmallDenseMap<const Instruction *, unsigned, 32> NumberedInsts;
+
+  /// \brief Keep track of last instruction inserted into \p NumberedInsts.
+  /// It speeds up queries for uncached instructions by providing a start point
+  /// for new queries in OrderedBasicBlock::comesBefore.
+  BasicBlock::const_iterator LastInstFound;
+
+  /// \brief The position/number to tag the next instruction to be found.
+  unsigned NextInstPos;
+
+  /// \brief The source BasicBlock to map.
+  const BasicBlock *BB;
+
+  /// \brief Given no cached results, find if \p A comes before \p B in \p BB.
+  /// Cache and number out instruction while walking \p BB.
+  bool comesBefore(const Instruction *A, const Instruction *B);
+
+public:
+  OrderedBasicBlock(const BasicBlock *BasicB);
+
+  /// \brief Find out whether \p A dominates \p B, meaning whether \p A
+  /// comes before \p B in \p BB. This is a simplification that considers
+  /// cached instruction positions and ignores other basic blocks, being
+  /// only relevant to compare relative instructions positions inside \p BB.
+  bool dominates(const Instruction *A, const Instruction *B);
+};
+
+} // End llvm namespace
+
+#endif
diff --git a/include/llvm/Analysis/PHITransAddr.h b/include/llvm/Analysis/PHITransAddr.h
index 84bb9d8008b..cbdbb88f740 100644
--- a/include/llvm/Analysis/PHITransAddr.h
+++ b/include/llvm/Analysis/PHITransAddr.h
@@ -75,12 +75,12 @@ class PHITransAddr {
   bool IsPotentiallyPHITranslatable() const;
   
   /// PHITranslateValue - PHI translate the current address up the CFG from
-  /// CurBB to Pred, updating our state to reflect any needed changes.  If the
-  /// dominator tree DT is non-null, the translated value must dominate
+  /// CurBB to Pred, updating our state to reflect any needed changes.  If
+  /// 'MustDominate' is true, the translated value must dominate
   /// PredBB.  This returns true on failure and sets Addr to null.
   bool PHITranslateValue(BasicBlock *CurBB, BasicBlock *PredBB,
-                         const DominatorTree *DT);
-  
+                         const DominatorTree *DT, bool MustDominate);
+
   /// PHITranslateWithInsertion - PHI translate this value into the specified
   /// predecessor block, inserting a computation of the value if it is
   /// unavailable.
diff --git a/include/llvm/Analysis/RegionInfo.h b/include/llvm/Analysis/RegionInfo.h
index 7ceb086ee0a..8560f1f6716 100644
--- a/include/llvm/Analysis/RegionInfo.h
+++ b/include/llvm/Analysis/RegionInfo.h
@@ -902,9 +902,9 @@ inline raw_ostream &operator<<(raw_ostream &OS,
     return OS << Node.template getNodeAs<BlockT>()->getName();
 }
 
-EXTERN_TEMPLATE_INSTANTIATION(class RegionBase<RegionTraits<Function>>);
-EXTERN_TEMPLATE_INSTANTIATION(class RegionNodeBase<RegionTraits<Function>>);
-EXTERN_TEMPLATE_INSTANTIATION(class RegionInfoBase<RegionTraits<Function>>);
+extern template class RegionBase<RegionTraits<Function>>;
+extern template class RegionNodeBase<RegionTraits<Function>>;
+extern template class RegionInfoBase<RegionTraits<Function>>;
 
 } // End llvm namespace
 #endif
diff --git a/include/llvm/Analysis/ScalarEvolution.h b/include/llvm/Analysis/ScalarEvolution.h
index 1240b85d176..c45fdb27191 100644
--- a/include/llvm/Analysis/ScalarEvolution.h
+++ b/include/llvm/Analysis/ScalarEvolution.h
@@ -48,6 +48,7 @@ namespace llvm {
   class LoopInfo;
   class Operator;
   class SCEVUnknown;
+  class SCEVAddRecExpr;
   class SCEV;
   template<> struct FoldingSetTrait<SCEV>;
 
@@ -565,19 +566,43 @@ namespace llvm {
     /// forgetMemoizedResults - Drop memoized information computed for S.
     void forgetMemoizedResults(const SCEV *S);
 
+    /// Return an existing SCEV for V if there is one, otherwise return nullptr.
+    const SCEV *getExistingSCEV(Value *V);
+
     /// Return false iff given SCEV contains a SCEVUnknown with NULL value-
     /// pointer.
     bool checkValidity(const SCEV *S) const;
 
-    // Return true if `ExtendOpTy`({`Start`,+,`Step`}) can be proved to be equal
-    // to {`ExtendOpTy`(`Start`),+,`ExtendOpTy`(`Step`)}.  This is equivalent to
-    // proving no signed (resp. unsigned) wrap in {`Start`,+,`Step`} if
-    // `ExtendOpTy` is `SCEVSignExtendExpr` (resp. `SCEVZeroExtendExpr`).
-    //
+    /// Return true if `ExtendOpTy`({`Start`,+,`Step`}) can be proved to be
+    /// equal to {`ExtendOpTy`(`Start`),+,`ExtendOpTy`(`Step`)}.  This is
+    /// equivalent to proving no signed (resp. unsigned) wrap in
+    /// {`Start`,+,`Step`} if `ExtendOpTy` is `SCEVSignExtendExpr`
+    /// (resp. `SCEVZeroExtendExpr`).
+    ///
     template<typename ExtendOpTy>
     bool proveNoWrapByVaryingStart(const SCEV *Start, const SCEV *Step,
                                    const Loop *L);
 
+    bool isMonotonicPredicateImpl(const SCEVAddRecExpr *LHS,
+                                  ICmpInst::Predicate Pred, bool &Increasing);
+
+    /// Return true if, for all loop invariant X, the predicate "LHS `Pred` X"
+    /// is monotonically increasing or decreasing.  In the former case set
+    /// `Increasing` to true and in the latter case set `Increasing` to false.
+    ///
+    /// A predicate is said to be monotonically increasing if may go from being
+    /// false to being true as the loop iterates, but never the other way
+    /// around.  A predicate is said to be monotonically decreasing if may go
+    /// from being true to being false as the loop iterates, but never the other
+    /// way around.
+    bool isMonotonicPredicate(const SCEVAddRecExpr *LHS,
+                              ICmpInst::Predicate Pred, bool &Increasing);
+
+    // Return SCEV no-wrap flags that can be proven based on reasoning
+    // about how poison produced from no-wrap flags on this value
+    // (e.g. a nuw add) would trigger undefined behavior on overflow.
+    SCEV::NoWrapFlags getNoWrapFlagsFromUB(const Value *V);
+
   public:
     static char ID; // Pass identification, replacement for typeid
     ScalarEvolution();
@@ -657,6 +682,15 @@ namespace llvm {
       SmallVector<const SCEV *, 4> NewOp(Operands.begin(), Operands.end());
       return getAddRecExpr(NewOp, L, Flags);
     }
+    /// \brief Returns an expression for a GEP
+    ///
+    /// \p PointeeType The type used as the basis for the pointer arithmetics
+    /// \p BaseExpr The expression for the pointer operand.
+    /// \p IndexExprs The expressions for the indices.
+    /// \p InBounds Whether the GEP is in bounds.
+    const SCEV *getGEPExpr(Type *PointeeType, const SCEV *BaseExpr,
+                           const SmallVectorImpl<const SCEV *> &IndexExprs,
+                           bool InBounds = false);
     const SCEV *getSMaxExpr(const SCEV *LHS, const SCEV *RHS);
     const SCEV *getSMaxExpr(SmallVectorImpl<const SCEV *> &Operands);
     const SCEV *getUMaxExpr(const SCEV *LHS, const SCEV *RHS);
@@ -890,6 +924,16 @@ namespace llvm {
     bool isKnownPredicate(ICmpInst::Predicate Pred,
                           const SCEV *LHS, const SCEV *RHS);
 
+    /// Return true if the result of the predicate LHS `Pred` RHS is loop
+    /// invariant with respect to L.  Set InvariantPred, InvariantLHS and
+    /// InvariantLHS so that InvariantLHS `InvariantPred` InvariantRHS is the
+    /// loop invariant form of LHS `Pred` RHS.
+    bool isLoopInvariantPredicate(ICmpInst::Predicate Pred, const SCEV *LHS,
+                                  const SCEV *RHS, const Loop *L,
+                                  ICmpInst::Predicate &InvariantPred,
+                                  const SCEV *&InvariantLHS,
+                                  const SCEV *&InvariantRHS);
+
     /// SimplifyICmpOperands - Simplify LHS and RHS in a comparison with
     /// predicate Pred. Return true iff any changes were made. If the
     /// operands are provably equal or unequal, LHS and RHS are set to
@@ -945,6 +989,86 @@ namespace llvm {
     void print(raw_ostream &OS, const Module* = nullptr) const override;
     void verifyAnalysis() const override;
 
+    /// Collect parametric terms occurring in step expressions.
+    void collectParametricTerms(const SCEV *Expr,
+                                SmallVectorImpl<const SCEV *> &Terms);
+
+
+
+    /// Return in Subscripts the access functions for each dimension in Sizes.
+    void computeAccessFunctions(const SCEV *Expr,
+                                SmallVectorImpl<const SCEV *> &Subscripts,
+                                SmallVectorImpl<const SCEV *> &Sizes);
+
+    /// Split this SCEVAddRecExpr into two vectors of SCEVs representing the
+    /// subscripts and sizes of an array access.
+    ///
+    /// The delinearization is a 3 step process: the first two steps compute the
+    /// sizes of each subscript and the third step computes the access functions
+    /// for the delinearized array:
+    ///
+    /// 1. Find the terms in the step functions
+    /// 2. Compute the array size
+    /// 3. Compute the access function: divide the SCEV by the array size
+    ///    starting with the innermost dimensions found in step 2. The Quotient
+    ///    is the SCEV to be divided in the next step of the recursion. The
+    ///    Remainder is the subscript of the innermost dimension. Loop over all
+    ///    array dimensions computed in step 2.
+    ///
+    /// To compute a uniform array size for several memory accesses to the same
+    /// object, one can collect in step 1 all the step terms for all the memory
+    /// accesses, and compute in step 2 a unique array shape. This guarantees
+    /// that the array shape will be the same across all memory accesses.
+    ///
+    /// FIXME: We could derive the result of steps 1 and 2 from a description of
+    /// the array shape given in metadata.
+    ///
+    /// Example:
+    ///
+    /// A[][n][m]
+    ///
+    /// for i
+    ///   for j
+    ///     for k
+    ///       A[j+k][2i][5i] =
+    ///
+    /// The initial SCEV:
+    ///
+    /// A[{{{0,+,2*m+5}_i, +, n*m}_j, +, n*m}_k]
+    ///
+    /// 1. Find the different terms in the step functions:
+    /// -> [2*m, 5, n*m, n*m]
+    ///
+    /// 2. Compute the array size: sort and unique them
+    /// -> [n*m, 2*m, 5]
+    /// find the GCD of all the terms = 1
+    /// divide by the GCD and erase constant terms
+    /// -> [n*m, 2*m]
+    /// GCD = m
+    /// divide by GCD -> [n, 2]
+    /// remove constant terms
+    /// -> [n]
+    /// size of the array is A[unknown][n][m]
+    ///
+    /// 3. Compute the access function
+    /// a. Divide {{{0,+,2*m+5}_i, +, n*m}_j, +, n*m}_k by the innermost size m
+    /// Quotient: {{{0,+,2}_i, +, n}_j, +, n}_k
+    /// Remainder: {{{0,+,5}_i, +, 0}_j, +, 0}_k
+    /// The remainder is the subscript of the innermost array dimension: [5i].
+    ///
+    /// b. Divide Quotient: {{{0,+,2}_i, +, n}_j, +, n}_k by next outer size n
+    /// Quotient: {{{0,+,0}_i, +, 1}_j, +, 1}_k
+    /// Remainder: {{{0,+,2}_i, +, 0}_j, +, 0}_k
+    /// The Remainder is the subscript of the next array dimension: [2i].
+    ///
+    /// The subscript of the outermost dimension is the Quotient: [j+k].
+    ///
+    /// Overall, we have: A[][n][m], and the access function: A[j+k][2i][5i].
+    void delinearize(const SCEV *Expr,
+                     SmallVectorImpl<const SCEV *> &Subscripts,
+                     SmallVectorImpl<const SCEV *> &Sizes,
+                     const SCEV *ElementSize);
+
   private:
     /// Compute the backedge taken count knowing the interval difference, the
     /// stride and presence of the equality in the comparison.
diff --git a/include/llvm/Analysis/ScalarEvolutionExpressions.h b/include/llvm/Analysis/ScalarEvolutionExpressions.h
index ff82db19b9e..da24de281d4 100644
--- a/include/llvm/Analysis/ScalarEvolutionExpressions.h
+++ b/include/llvm/Analysis/ScalarEvolutionExpressions.h
@@ -356,84 +356,6 @@ namespace llvm {
     static inline bool classof(const SCEV *S) {
       return S->getSCEVType() == scAddRecExpr;
     }
-
-    /// Collect parametric terms occurring in step expressions.
-    void collectParametricTerms(ScalarEvolution &SE,
-                                SmallVectorImpl<const SCEV *> &Terms) const;
-
-    /// Return in Subscripts the access functions for each dimension in Sizes.
-    void computeAccessFunctions(ScalarEvolution &SE,
-                                SmallVectorImpl<const SCEV *> &Subscripts,
-                                SmallVectorImpl<const SCEV *> &Sizes) const;
-
-    /// Split this SCEVAddRecExpr into two vectors of SCEVs representing the
-    /// subscripts and sizes of an array access.
-    ///
-    /// The delinearization is a 3 step process: the first two steps compute the
-    /// sizes of each subscript and the third step computes the access functions
-    /// for the delinearized array:
-    ///
-    /// 1. Find the terms in the step functions
-    /// 2. Compute the array size
-    /// 3. Compute the access function: divide the SCEV by the array size
-    ///    starting with the innermost dimensions found in step 2. The Quotient
-    ///    is the SCEV to be divided in the next step of the recursion. The
-    ///    Remainder is the subscript of the innermost dimension. Loop over all
-    ///    array dimensions computed in step 2.
-    ///
-    /// To compute a uniform array size for several memory accesses to the same
-    /// object, one can collect in step 1 all the step terms for all the memory
-    /// accesses, and compute in step 2 a unique array shape. This guarantees
-    /// that the array shape will be the same across all memory accesses.
-    ///
-    /// FIXME: We could derive the result of steps 1 and 2 from a description of
-    /// the array shape given in metadata.
-    ///
-    /// Example:
-    ///
-    /// A[][n][m]
-    ///
-    /// for i
-    ///   for j
-    ///     for k
-    ///       A[j+k][2i][5i] =
-    ///
-    /// The initial SCEV:
-    ///
-    /// A[{{{0,+,2*m+5}_i, +, n*m}_j, +, n*m}_k]
-    ///
-    /// 1. Find the different terms in the step functions:
-    /// -> [2*m, 5, n*m, n*m]
-    ///
-    /// 2. Compute the array size: sort and unique them
-    /// -> [n*m, 2*m, 5]
-    /// find the GCD of all the terms = 1
-    /// divide by the GCD and erase constant terms
-    /// -> [n*m, 2*m]
-    /// GCD = m
-    /// divide by GCD -> [n, 2]
-    /// remove constant terms
-    /// -> [n]
-    /// size of the array is A[unknown][n][m]
-    ///
-    /// 3. Compute the access function
-    /// a. Divide {{{0,+,2*m+5}_i, +, n*m}_j, +, n*m}_k by the innermost size m
-    /// Quotient: {{{0,+,2}_i, +, n}_j, +, n}_k
-    /// Remainder: {{{0,+,5}_i, +, 0}_j, +, 0}_k
-    /// The remainder is the subscript of the innermost array dimension: [5i].
-    ///
-    /// b. Divide Quotient: {{{0,+,2}_i, +, n}_j, +, n}_k by next outer size n
-    /// Quotient: {{{0,+,0}_i, +, 1}_j, +, 1}_k
-    /// Remainder: {{{0,+,2}_i, +, 0}_j, +, 0}_k
-    /// The Remainder is the subscript of the next array dimension: [2i].
-    ///
-    /// The subscript of the outermost dimension is the Quotient: [j+k].
-    ///
-    /// Overall, we have: A[][n][m], and the access function: A[j+k][2i][5i].
-    void delinearize(ScalarEvolution &SE,
-                     SmallVectorImpl<const SCEV *> &Subscripts,
-                     SmallVectorImpl<const SCEV *> &Sizes,
-                     const SCEV *ElementSize) const;
   };
 
   //===--------------------------------------------------------------------===//
diff --git a/include/llvm/Analysis/TargetLibraryInfo.def b/include/llvm/Analysis/TargetLibraryInfo.def
index 0837fbba6a2..1c1fdfef980 100644
--- a/include/llvm/Analysis/TargetLibraryInfo.def
+++ b/include/llvm/Analysis/TargetLibraryInfo.def
@@ -470,8 +470,7 @@ TLI_DEFINE_STRING_INTERNAL("fscanf")
 TLI_DEFINE_ENUM_INTERNAL(fseek)
 TLI_DEFINE_STRING_INTERNAL("fseek")
 /// int fseeko(FILE *stream, off_t offset, int whence);
-// LOCALMOD ftello->Ftello
-TLI_DEFINE_ENUM_INTERNAL(Fseeko)
+TLI_DEFINE_ENUM_INTERNAL(fseeko)
 TLI_DEFINE_STRING_INTERNAL("fseeko")
 /// int fseeko64(FILE *stream, off64_t offset, int whence)
 TLI_DEFINE_ENUM_INTERNAL(fseeko64)
@@ -495,8 +494,7 @@ TLI_DEFINE_STRING_INTERNAL("fstatvfs64")
 TLI_DEFINE_ENUM_INTERNAL(ftell)
 TLI_DEFINE_STRING_INTERNAL("ftell")
 /// off_t ftello(FILE *stream);
-// LOCALMOD ftello->Ftello
-TLI_DEFINE_ENUM_INTERNAL(Ftello)
+TLI_DEFINE_ENUM_INTERNAL(ftello)
 TLI_DEFINE_STRING_INTERNAL("ftello")
 /// off64_t ftello64(FILE *stream)
 TLI_DEFINE_ENUM_INTERNAL(ftello64)
diff --git a/include/llvm/Analysis/TargetLibraryInfo.h b/include/llvm/Analysis/TargetLibraryInfo.h
index e0a1ee37827..98e450c867d 100644
--- a/include/llvm/Analysis/TargetLibraryInfo.h
+++ b/include/llvm/Analysis/TargetLibraryInfo.h
@@ -201,13 +201,13 @@ class TargetLibraryInfo {
   }
   bool isFunctionVectorizable(StringRef F, unsigned VF) const {
     return Impl->isFunctionVectorizable(F, VF);
-  };
+  }
   bool isFunctionVectorizable(StringRef F) const {
     return Impl->isFunctionVectorizable(F);
-  };
+  }
   StringRef getVectorizedFunction(StringRef F, unsigned VF) const {
     return Impl->getVectorizedFunction(F, VF);
-  };
+  }
 
   /// \brief Tests if the function is both available and a candidate for
   /// optimized code generation.
diff --git a/include/llvm/Analysis/TargetTransformInfo.h b/include/llvm/Analysis/TargetTransformInfo.h
index f4195fbb072..fd3cfe72921 100644
--- a/include/llvm/Analysis/TargetTransformInfo.h
+++ b/include/llvm/Analysis/TargetTransformInfo.h
@@ -69,7 +69,7 @@ class TargetTransformInfo {
   ///
   /// The TTI implementation will reflect the information in the DataLayout
   /// provided if non-null.
-  explicit TargetTransformInfo(const DataLayout *DL);
+  explicit TargetTransformInfo(const DataLayout &DL);
 
   // Provide move semantics.
   TargetTransformInfo(TargetTransformInfo &&Arg);
@@ -136,7 +136,8 @@ class TargetTransformInfo {
   /// The contract for this function is the same as \c getOperationCost except
   /// that it supports an interface that provides extra information specific to
   /// the GEP operation.
-  unsigned getGEPCost(const Value *Ptr, ArrayRef<const Value *> Operands) const;
+  unsigned getGEPCost(Type *PointeeType, const Value *Ptr,
+                      ArrayRef<const Value *> Operands) const;
 
   /// \brief Estimate the cost of a function call when lowered.
   ///
@@ -221,19 +222,21 @@ class TargetTransformInfo {
 
   /// Parameters that control the generic loop unrolling transformation.
   struct UnrollingPreferences {
-    /// The cost threshold for the unrolled loop, compared to
-    /// CodeMetrics.NumInsts aggregated over all basic blocks in the loop body.
-    /// The unrolling factor is set such that the unrolled loop body does not
-    /// exceed this cost. Set this to UINT_MAX to disable the loop body cost
+    /// The cost threshold for the unrolled loop. Should be relative to the
+    /// getUserCost values returned by this API, and the expectation is that
+    /// the unrolled loop's instructions when run through that interface should
+    /// not exceed this cost. However, this is only an estimate. Also, specific
+    /// loops may be unrolled even with a cost above this threshold if deemed
+    /// profitable. Set this to UINT_MAX to disable the loop body cost
     /// restriction.
     unsigned Threshold;
-    /// If complete unrolling could help other optimizations (e.g. InstSimplify)
-    /// to remove N% of instructions, then we can go beyond unroll threshold.
-    /// This value set the minimal percent for allowing that.
-    unsigned MinPercentOfOptimized;
-    /// The absolute cost threshold. We won't go beyond this even if complete
-    /// unrolling could result in optimizing out 90% of instructions.
-    unsigned AbsoluteThreshold;
+    /// If complete unrolling will reduce the cost of the loop below its
+    /// expected dynamic cost while rolled by this percentage, apply a discount
+    /// (below) to its unrolled cost.
+    unsigned PercentDynamicCostSavedThreshold;
+    /// The discount applied to the unrolled cost when the *dynamic* cost
+    /// savings of unrolling exceed the \c PercentDynamicCostSavedThreshold.
+    unsigned DynamicCostSavingsDiscount;
     /// The cost threshold for the unrolled loop when optimizing for size (set
     /// to UINT_MAX to disable).
     unsigned OptSizeThreshold;
@@ -303,7 +306,8 @@ class TargetTransformInfo {
   /// mode is legal for a load/store of any legal type.
   /// TODO: Handle pre/postinc as well.
   bool isLegalAddressingMode(Type *Ty, GlobalValue *BaseGV, int64_t BaseOffset,
-                             bool HasBaseReg, int64_t Scale) const;
+                             bool HasBaseReg, int64_t Scale,
+                             unsigned AddrSpace = 0) const;
 
   /// \brief Return true if the target works with masked instruction
   /// AVX2 allows masks for consecutive load and store for i32 and i64 elements.
@@ -319,13 +323,19 @@ class TargetTransformInfo {
   /// If the AM is not supported, it returns a negative value.
   /// TODO: Handle pre/postinc as well.
   int getScalingFactorCost(Type *Ty, GlobalValue *BaseGV, int64_t BaseOffset,
-                           bool HasBaseReg, int64_t Scale) const;
+                           bool HasBaseReg, int64_t Scale,
+                           unsigned AddrSpace = 0) const;
 
   /// \brief Return true if it's free to truncate a value of type Ty1 to type
   /// Ty2. e.g. On x86 it's free to truncate a i32 value in register EAX to i16
   /// by referencing its sub-register AX.
   bool isTruncateFree(Type *Ty1, Type *Ty2) const;
 
+  /// \brief Return true if it's free to zero extend a value of type Ty1 to type
+  /// Ty2. e.g. on x86-64, all instructions that define 32-bit values implicit
+  /// zero-extend the result out to 64 bits.
+  bool isZExtFree(Type *Ty1, Type *Ty2) const;
+
   /// \brief Return true if it is profitable to hoist instruction in the
   /// then/else to before if.
   bool isProfitableToHoist(Instruction *I) const;
@@ -403,7 +413,7 @@ class TargetTransformInfo {
   /// \return The maximum interleave factor that any transform should try to
   /// perform for this target. This number depends on the level of parallelism
   /// and the number of execution units in the CPU.
-  unsigned getMaxInterleaveFactor() const;
+  unsigned getMaxInterleaveFactor(unsigned VF) const;
 
   /// \return The expected cost of arithmetic ops, such as mul, xor, fsub, etc.
   unsigned
@@ -444,6 +454,20 @@ class TargetTransformInfo {
   unsigned getMaskedMemoryOpCost(unsigned Opcode, Type *Src, unsigned Alignment,
                                  unsigned AddressSpace) const;
 
+  /// \return The cost of the interleaved memory operation.
+  /// \p Opcode is the memory operation code
+  /// \p VecTy is the vector type of the interleaved access.
+  /// \p Factor is the interleave factor
+  /// \p Indices is the indices for interleaved load members (as interleaved
+  ///    load allows gaps)
+  /// \p Alignment is the alignment of the memory operation
+  /// \p AddressSpace is address space of the pointer.
+  unsigned getInterleavedMemoryOpCost(unsigned Opcode, Type *VecTy,
+                                      unsigned Factor,
+                                      ArrayRef<unsigned> Indices,
+                                      unsigned Alignment,
+                                      unsigned AddressSpace) const;
+
   /// \brief Calculate the cost of performing a vector reduction.
   ///
   /// This is the cost of reducing the vector value of type \p Ty to a scalar
@@ -501,6 +525,11 @@ class TargetTransformInfo {
   Value *getOrCreateResultFromMemIntrinsic(IntrinsicInst *Inst,
                                            Type *ExpectedType) const;
 
+  /// \returns True if the two functions have compatible attributes for inlining
+  /// purposes.
+  bool areInlineCompatible(const Function *Caller,
+                           const Function *Callee) const;
+
   /// @}
 
 private:
@@ -518,9 +547,9 @@ class TargetTransformInfo {
 class TargetTransformInfo::Concept {
 public:
   virtual ~Concept() = 0;
-
+  virtual const DataLayout &getDataLayout() const = 0;
   virtual unsigned getOperationCost(unsigned Opcode, Type *Ty, Type *OpTy) = 0;
-  virtual unsigned getGEPCost(const Value *Ptr,
+  virtual unsigned getGEPCost(Type *PointeeType, const Value *Ptr,
                               ArrayRef<const Value *> Operands) = 0;
   virtual unsigned getCallCost(FunctionType *FTy, int NumArgs) = 0;
   virtual unsigned getCallCost(const Function *F, int NumArgs) = 0;
@@ -539,13 +568,15 @@ class TargetTransformInfo::Concept {
   virtual bool isLegalICmpImmediate(int64_t Imm) = 0;
   virtual bool isLegalAddressingMode(Type *Ty, GlobalValue *BaseGV,
                                      int64_t BaseOffset, bool HasBaseReg,
-                                     int64_t Scale) = 0;
+                                     int64_t Scale,
+                                     unsigned AddrSpace) = 0;
   virtual bool isLegalMaskedStore(Type *DataType, int Consecutive) = 0;
   virtual bool isLegalMaskedLoad(Type *DataType, int Consecutive) = 0;
   virtual int getScalingFactorCost(Type *Ty, GlobalValue *BaseGV,
                                    int64_t BaseOffset, bool HasBaseReg,
-                                   int64_t Scale) = 0;
+                                   int64_t Scale, unsigned AddrSpace) = 0;
   virtual bool isTruncateFree(Type *Ty1, Type *Ty2) = 0;
+  virtual bool isZExtFree(Type *Ty1, Type *Ty2) = 0;
   virtual bool isProfitableToHoist(Instruction *I) = 0;
   virtual bool isTypeLegal(Type *Ty) = 0;
   virtual unsigned getJumpBufAlignment() = 0;
@@ -562,7 +593,7 @@ class TargetTransformInfo::Concept {
                                  const APInt &Imm, Type *Ty) = 0;
   virtual unsigned getNumberOfRegisters(bool Vector) = 0;
   virtual unsigned getRegisterBitWidth(bool Vector) = 0;
-  virtual unsigned getMaxInterleaveFactor() = 0;
+  virtual unsigned getMaxInterleaveFactor(unsigned VF) = 0;
   virtual unsigned
   getArithmeticInstrCost(unsigned Opcode, Type *Ty, OperandValueKind Opd1Info,
                          OperandValueKind Opd2Info,
@@ -582,6 +613,11 @@ class TargetTransformInfo::Concept {
   virtual unsigned getMaskedMemoryOpCost(unsigned Opcode, Type *Src,
                                          unsigned Alignment,
                                          unsigned AddressSpace) = 0;
+  virtual unsigned getInterleavedMemoryOpCost(unsigned Opcode, Type *VecTy,
+                                              unsigned Factor,
+                                              ArrayRef<unsigned> Indices,
+                                              unsigned Alignment,
+                                              unsigned AddressSpace) = 0;
   virtual unsigned getReductionCost(unsigned Opcode, Type *Ty,
                                     bool IsPairwiseForm) = 0;
   virtual unsigned getIntrinsicInstrCost(Intrinsic::ID ID, Type *RetTy,
@@ -595,6 +631,8 @@ class TargetTransformInfo::Concept {
                                   MemIntrinsicInfo &Info) = 0;
   virtual Value *getOrCreateResultFromMemIntrinsic(IntrinsicInst *Inst,
                                                    Type *ExpectedType) = 0;
+  virtual bool areInlineCompatible(const Function *Caller,
+                                   const Function *Callee) const = 0;
 };
 
 template <typename T>
@@ -605,12 +643,16 @@ class TargetTransformInfo::Model final : public TargetTransformInfo::Concept {
   Model(T Impl) : Impl(std::move(Impl)) {}
   ~Model() override {}
 
+  const DataLayout &getDataLayout() const override {
+    return Impl.getDataLayout();
+  }
+
   unsigned getOperationCost(unsigned Opcode, Type *Ty, Type *OpTy) override {
     return Impl.getOperationCost(Opcode, Ty, OpTy);
   }
-  unsigned getGEPCost(const Value *Ptr,
+  unsigned getGEPCost(Type *PointeeType, const Value *Ptr,
                       ArrayRef<const Value *> Operands) override {
-    return Impl.getGEPCost(Ptr, Operands);
+    return Impl.getGEPCost(PointeeType, Ptr, Operands);
   }
   unsigned getCallCost(FunctionType *FTy, int NumArgs) override {
     return Impl.getCallCost(FTy, NumArgs);
@@ -648,9 +690,10 @@ class TargetTransformInfo::Model final : public TargetTransformInfo::Concept {
     return Impl.isLegalICmpImmediate(Imm);
   }
   bool isLegalAddressingMode(Type *Ty, GlobalValue *BaseGV, int64_t BaseOffset,
-                             bool HasBaseReg, int64_t Scale) override {
+                             bool HasBaseReg, int64_t Scale,
+                             unsigned AddrSpace) override {
     return Impl.isLegalAddressingMode(Ty, BaseGV, BaseOffset, HasBaseReg,
-                                      Scale);
+                                      Scale, AddrSpace);
   }
   bool isLegalMaskedStore(Type *DataType, int Consecutive) override {
     return Impl.isLegalMaskedStore(DataType, Consecutive);
@@ -659,12 +702,17 @@ class TargetTransformInfo::Model final : public TargetTransformInfo::Concept {
     return Impl.isLegalMaskedLoad(DataType, Consecutive);
   }
   int getScalingFactorCost(Type *Ty, GlobalValue *BaseGV, int64_t BaseOffset,
-                           bool HasBaseReg, int64_t Scale) override {
-    return Impl.getScalingFactorCost(Ty, BaseGV, BaseOffset, HasBaseReg, Scale);
+                           bool HasBaseReg, int64_t Scale,
+                           unsigned AddrSpace) override {
+    return Impl.getScalingFactorCost(Ty, BaseGV, BaseOffset, HasBaseReg,
+                                     Scale, AddrSpace);
   }
   bool isTruncateFree(Type *Ty1, Type *Ty2) override {
     return Impl.isTruncateFree(Ty1, Ty2);
   }
+  bool isZExtFree(Type *Ty1, Type *Ty2) override {
+    return Impl.isZExtFree(Ty1, Ty2);
+  }
   bool isProfitableToHoist(Instruction *I) override {
     return Impl.isProfitableToHoist(I);
   }
@@ -703,8 +751,8 @@ class TargetTransformInfo::Model final : public TargetTransformInfo::Concept {
   unsigned getRegisterBitWidth(bool Vector) override {
     return Impl.getRegisterBitWidth(Vector);
   }
-  unsigned getMaxInterleaveFactor() override {
-    return Impl.getMaxInterleaveFactor();
+  unsigned getMaxInterleaveFactor(unsigned VF) override {
+    return Impl.getMaxInterleaveFactor(VF);
   }
   unsigned
   getArithmeticInstrCost(unsigned Opcode, Type *Ty, OperandValueKind Opd1Info,
@@ -740,6 +788,14 @@ class TargetTransformInfo::Model final : public TargetTransformInfo::Concept {
                                  unsigned AddressSpace) override {
     return Impl.getMaskedMemoryOpCost(Opcode, Src, Alignment, AddressSpace);
   }
+  unsigned getInterleavedMemoryOpCost(unsigned Opcode, Type *VecTy,
+                                      unsigned Factor,
+                                      ArrayRef<unsigned> Indices,
+                                      unsigned Alignment,
+                                      unsigned AddressSpace) override {
+    return Impl.getInterleavedMemoryOpCost(Opcode, VecTy, Factor, Indices,
+                                           Alignment, AddressSpace);
+  }
   unsigned getReductionCost(unsigned Opcode, Type *Ty,
                             bool IsPairwiseForm) override {
     return Impl.getReductionCost(Opcode, Ty, IsPairwiseForm);
@@ -769,6 +825,10 @@ class TargetTransformInfo::Model final : public TargetTransformInfo::Concept {
                                            Type *ExpectedType) override {
     return Impl.getOrCreateResultFromMemIntrinsic(Inst, ExpectedType);
   }
+  bool areInlineCompatible(const Function *Caller,
+                           const Function *Callee) const override {
+    return Impl.areInlineCompatible(Caller, Callee);
+  }
 };
 
 template <typename T>
diff --git a/include/llvm/Analysis/TargetTransformInfoImpl.h b/include/llvm/Analysis/TargetTransformInfoImpl.h
index b00de7723ef..9805f6bc663 100644
--- a/include/llvm/Analysis/TargetTransformInfoImpl.h
+++ b/include/llvm/Analysis/TargetTransformInfoImpl.h
@@ -19,6 +19,7 @@
 #include "llvm/IR/CallSite.h"
 #include "llvm/IR/DataLayout.h"
 #include "llvm/IR/Function.h"
+#include "llvm/IR/GetElementPtrTypeIterator.h"
 #include "llvm/IR/Operator.h"
 #include "llvm/IR/Type.h"
 
@@ -30,26 +31,17 @@ class TargetTransformInfoImplBase {
 protected:
   typedef TargetTransformInfo TTI;
 
-  const DataLayout *DL;
+  const DataLayout &DL;
 
-  explicit TargetTransformInfoImplBase(const DataLayout *DL)
-      : DL(DL) {}
+  explicit TargetTransformInfoImplBase(const DataLayout &DL) : DL(DL) {}
 
 public:
   // Provide value semantics. MSVC requires that we spell all of these out.
   TargetTransformInfoImplBase(const TargetTransformInfoImplBase &Arg)
       : DL(Arg.DL) {}
-  TargetTransformInfoImplBase(TargetTransformInfoImplBase &&Arg)
-      : DL(std::move(Arg.DL)) {}
-  TargetTransformInfoImplBase &
-  operator=(const TargetTransformInfoImplBase &RHS) {
-    DL = RHS.DL;
-    return *this;
-  }
-  TargetTransformInfoImplBase &operator=(TargetTransformInfoImplBase &&RHS) {
-    DL = std::move(RHS.DL);
-    return *this;
-  }
+  TargetTransformInfoImplBase(TargetTransformInfoImplBase &&Arg) : DL(Arg.DL) {}
+
+  const DataLayout &getDataLayout() const { return DL; }
 
   unsigned getOperationCost(unsigned Opcode, Type *Ty, Type *OpTy) {
     switch (Opcode) {
@@ -70,28 +62,22 @@ class TargetTransformInfoImplBase {
       return TTI::TCC_Basic;
 
     case Instruction::IntToPtr: {
-      if (!DL)
-        return TTI::TCC_Basic;
-
       // An inttoptr cast is free so long as the input is a legal integer type
       // which doesn't contain values outside the range of a pointer.
       unsigned OpSize = OpTy->getScalarSizeInBits();
-      if (DL->isLegalInteger(OpSize) &&
-          OpSize <= DL->getPointerTypeSizeInBits(Ty))
+      if (DL.isLegalInteger(OpSize) &&
+          OpSize <= DL.getPointerTypeSizeInBits(Ty))
         return TTI::TCC_Free;
 
       // Otherwise it's not a no-op.
       return TTI::TCC_Basic;
     }
     case Instruction::PtrToInt: {
-      if (!DL)
-        return TTI::TCC_Basic;
-
       // A ptrtoint cast is free so long as the result is large enough to store
       // the pointer, and a legal integer type.
       unsigned DestSize = Ty->getScalarSizeInBits();
-      if (DL->isLegalInteger(DestSize) &&
-          DestSize >= DL->getPointerTypeSizeInBits(OpTy))
+      if (DL.isLegalInteger(DestSize) &&
+          DestSize >= DL.getPointerTypeSizeInBits(OpTy))
         return TTI::TCC_Free;
 
       // Otherwise it's not a no-op.
@@ -100,14 +86,15 @@ class TargetTransformInfoImplBase {
     case Instruction::Trunc:
       // trunc to a native type is free (assuming the target has compare and
       // shift-right of the same width).
-      if (DL && DL->isLegalInteger(DL->getTypeSizeInBits(Ty)))
+      if (DL.isLegalInteger(DL.getTypeSizeInBits(Ty)))
         return TTI::TCC_Free;
 
       return TTI::TCC_Basic;
     }
   }
 
-  unsigned getGEPCost(const Value *Ptr, ArrayRef<const Value *> Operands) {
+  unsigned getGEPCost(Type *PointeeType, const Value *Ptr,
+                      ArrayRef<const Value *> Operands) {
     // In the basic model, we just assume that all-constant GEPs will be folded
     // into their uses via addressing modes.
     for (unsigned Idx = 0, Size = Operands.size(); Idx != Size; ++Idx)
@@ -207,10 +194,11 @@ class TargetTransformInfoImplBase {
   bool isLegalICmpImmediate(int64_t Imm) { return false; }
 
   bool isLegalAddressingMode(Type *Ty, GlobalValue *BaseGV, int64_t BaseOffset,
-                             bool HasBaseReg, int64_t Scale) {
-    // Guess that reg+reg addressing is allowed. This heuristic is taken from
-    // the implementation of LSR.
-    return !BaseGV && BaseOffset == 0 && Scale <= 1;
+                             bool HasBaseReg, int64_t Scale,
+                             unsigned AddrSpace) {
+    // Guess that only reg and reg+reg addressing is allowed. This heuristic is
+    // taken from the implementation of LSR.
+    return !BaseGV && BaseOffset == 0 && (Scale == 0 || Scale == 1);
   }
 
   bool isLegalMaskedStore(Type *DataType, int Consecutive) { return false; }
@@ -218,15 +206,18 @@ class TargetTransformInfoImplBase {
   bool isLegalMaskedLoad(Type *DataType, int Consecutive) { return false; }
 
   int getScalingFactorCost(Type *Ty, GlobalValue *BaseGV, int64_t BaseOffset,
-                           bool HasBaseReg, int64_t Scale) {
+                           bool HasBaseReg, int64_t Scale, unsigned AddrSpace) {
     // Guess that all legal addressing mode are free.
-    if (isLegalAddressingMode(Ty, BaseGV, BaseOffset, HasBaseReg, Scale))
+    if (isLegalAddressingMode(Ty, BaseGV, BaseOffset, HasBaseReg,
+                              Scale, AddrSpace))
       return 0;
     return -1;
   }
 
   bool isTruncateFree(Type *Ty1, Type *Ty2) { return false; }
 
+  bool isZExtFree(Type *Ty1, Type *Ty2) { return false; }
+
   bool isProfitableToHoist(Instruction *I) { return true; }
 
   bool isTypeLegal(Type *Ty) { return false; }
@@ -263,7 +254,7 @@ class TargetTransformInfoImplBase {
 
   unsigned getRegisterBitWidth(bool Vector) { return 32; }
 
-  unsigned getMaxInterleaveFactor() { return 1; }
+  unsigned getMaxInterleaveFactor(unsigned VF) { return 1; }
 
   unsigned getArithmeticInstrCost(unsigned Opcode, Type *Ty,
                                   TTI::OperandValueKind Opd1Info,
@@ -300,6 +291,14 @@ class TargetTransformInfoImplBase {
     return 1;
   }
 
+  unsigned getInterleavedMemoryOpCost(unsigned Opcode, Type *VecTy,
+                                      unsigned Factor,
+                                      ArrayRef<unsigned> Indices,
+                                      unsigned Alignment,
+                                      unsigned AddressSpace) {
+    return 1;
+  }
+
   unsigned getIntrinsicInstrCost(Intrinsic::ID ID, Type *RetTy,
                                  ArrayRef<Type *> Tys) {
     return 1;
@@ -325,6 +324,14 @@ class TargetTransformInfoImplBase {
                                            Type *ExpectedType) {
     return nullptr;
   }
+
+  bool areInlineCompatible(const Function *Caller,
+                           const Function *Callee) const {
+    return (Caller->getFnAttribute("target-cpu") ==
+            Callee->getFnAttribute("target-cpu")) &&
+           (Caller->getFnAttribute("target-features") ==
+            Callee->getFnAttribute("target-features"));
+  }
 };
 
 /// \brief CRTP base class for use as a mix-in that aids implementing
@@ -335,8 +342,7 @@ class TargetTransformInfoImplCRTPBase : public TargetTransformInfoImplBase {
   typedef TargetTransformInfoImplBase BaseT;
 
 protected:
-  explicit TargetTransformInfoImplCRTPBase(const DataLayout *DL)
-      : BaseT(DL) {}
+  explicit TargetTransformInfoImplCRTPBase(const DataLayout &DL) : BaseT(DL) {}
 
 public:
   // Provide value semantics. MSVC requires that we spell all of these out.
@@ -344,16 +350,6 @@ class TargetTransformInfoImplCRTPBase : public TargetTransformInfoImplBase {
       : BaseT(static_cast<const BaseT &>(Arg)) {}
   TargetTransformInfoImplCRTPBase(TargetTransformInfoImplCRTPBase &&Arg)
       : BaseT(std::move(static_cast<BaseT &>(Arg))) {}
-  TargetTransformInfoImplCRTPBase &
-  operator=(const TargetTransformInfoImplCRTPBase &RHS) {
-    BaseT::operator=(static_cast<const BaseT &>(RHS));
-    return *this;
-  }
-  TargetTransformInfoImplCRTPBase &
-  operator=(TargetTransformInfoImplCRTPBase &&RHS) {
-    BaseT::operator=(std::move(static_cast<BaseT &>(RHS)));
-    return *this;
-  }
 
   using BaseT::getCallCost;
 
@@ -365,7 +361,7 @@ class TargetTransformInfoImplCRTPBase : public TargetTransformInfoImplBase {
       // function.
       NumArgs = F->arg_size();
 
-    if (Intrinsic::ID IID = (Intrinsic::ID)F->getIntrinsicID()) {
+    if (Intrinsic::ID IID = F->getIntrinsicID()) {
       FunctionType *FTy = F->getFunctionType();
       SmallVector<Type *, 8> ParamTys(FTy->param_begin(), FTy->param_end());
       return static_cast<T *>(this)
@@ -386,6 +382,54 @@ class TargetTransformInfoImplCRTPBase : public TargetTransformInfoImplBase {
     return static_cast<T *>(this)->getCallCost(F, Arguments.size());
   }
 
+  using BaseT::getGEPCost;
+
+  unsigned getGEPCost(Type *PointeeType, const Value *Ptr,
+                      ArrayRef<const Value *> Operands) {
+    const GlobalValue *BaseGV = nullptr;
+    if (Ptr != nullptr) {
+      // TODO: will remove this when pointers have an opaque type.
+      assert(Ptr->getType()->getScalarType()->getPointerElementType() ==
+                 PointeeType &&
+             "explicit pointee type doesn't match operand's pointee type");
+      BaseGV = dyn_cast<GlobalValue>(Ptr->stripPointerCasts());
+    }
+    bool HasBaseReg = (BaseGV == nullptr);
+    int64_t BaseOffset = 0;
+    int64_t Scale = 0;
+
+    // Assumes the address space is 0 when Ptr is nullptr.
+    unsigned AS =
+        (Ptr == nullptr ? 0 : Ptr->getType()->getPointerAddressSpace());
+    auto GTI = gep_type_begin(PointerType::get(PointeeType, AS), Operands);
+    for (auto I = Operands.begin(); I != Operands.end(); ++I, ++GTI) {
+      if (isa<SequentialType>(*GTI)) {
+        int64_t ElementSize = DL.getTypeAllocSize(GTI.getIndexedType());
+        if (const ConstantInt *ConstIdx = dyn_cast<ConstantInt>(*I)) {
+          BaseOffset += ConstIdx->getSExtValue() * ElementSize;
+        } else {
+          // Needs scale register.
+          if (Scale != 0) {
+            // No addressing mode takes two scale registers.
+            return TTI::TCC_Basic;
+          }
+          Scale = ElementSize;
+        }
+      } else {
+        StructType *STy = cast<StructType>(*GTI);
+        uint64_t Field = cast<ConstantInt>(*I)->getZExtValue();
+        BaseOffset += DL.getStructLayout(STy)->getElementOffset(Field);
+      }
+    }
+
+    if (static_cast<T *>(this)->isLegalAddressingMode(
+            PointerType::get(*GTI, AS), const_cast<GlobalValue *>(BaseGV),
+            BaseOffset, HasBaseReg, Scale, AS)) {
+      return TTI::TCC_Free;
+    }
+    return TTI::TCC_Basic;
+  }
+
   using BaseT::getIntrinsicCost;
 
   unsigned getIntrinsicCost(Intrinsic::ID IID, Type *RetTy,
@@ -405,9 +449,9 @@ class TargetTransformInfoImplCRTPBase : public TargetTransformInfoImplBase {
       return TTI::TCC_Free; // Model all PHI nodes as free.
 
     if (const GEPOperator *GEP = dyn_cast<GEPOperator>(U)) {
-      SmallVector<const Value *, 4> Indices(GEP->idx_begin(), GEP->idx_end());
-      return static_cast<T *>(this)
-          ->getGEPCost(GEP->getPointerOperand(), Indices);
+      SmallVector<Value *, 4> Indices(GEP->idx_begin(), GEP->idx_end());
+      return static_cast<T *>(this)->getGEPCost(
+          GEP->getSourceElementType(), GEP->getPointerOperand(), Indices);
     }
 
     if (auto CS = ImmutableCallSite(U)) {
diff --git a/include/llvm/Analysis/ValueTracking.h b/include/llvm/Analysis/ValueTracking.h
index 8955b7c7bd7..258723dc4db 100644
--- a/include/llvm/Analysis/ValueTracking.h
+++ b/include/llvm/Analysis/ValueTracking.h
@@ -16,6 +16,7 @@
 #define LLVM_ANALYSIS_VALUETRACKING_H
 
 #include "llvm/ADT/ArrayRef.h"
+#include "llvm/IR/Instruction.h"
 #include "llvm/Support/DataTypes.h"
 
 namespace llvm {
@@ -28,6 +29,8 @@ namespace llvm {
   class AssumptionCache;
   class DominatorTree;
   class TargetLibraryInfo;
+  class LoopInfo;
+  class Loop;
 
   /// Determine which bits of V are known to be either zero or one and return
   /// them in the KnownZero/KnownOne bit sets.
@@ -46,6 +49,11 @@ namespace llvm {
   /// \p KnownZero the set of bits that are known to be zero
   void computeKnownBitsFromRangeMetadata(const MDNode &Ranges,
                                          APInt &KnownZero);
+  /// Return true if LHS and RHS have no common bits set.
+  bool haveNoCommonBitsSet(Value *LHS, Value *RHS, const DataLayout &DL,
+                           AssumptionCache *AC = nullptr,
+                           const Instruction *CxtI = nullptr,
+                           const DominatorTree *DT = nullptr);
 
   /// ComputeSignBit - Determine whether the sign bit is known to be zero or
   /// one.  Convenience wrapper around computeKnownBits.
@@ -59,7 +67,7 @@ namespace llvm {
   /// exactly one bit set when defined. For vectors return true if every
   /// element is known to be a power of two when defined.  Supports values with
   /// integer or pointer type and vectors of integers.  If 'OrZero' is set then
-  /// returns true if the given value is either a power of two or zero.
+  /// return true if the given value is either a power of two or zero.
   bool isKnownToBeAPowerOfTwo(Value *V, const DataLayout &DL,
                               bool OrZero = false, unsigned Depth = 0,
                               AssumptionCache *AC = nullptr,
@@ -176,16 +184,51 @@ namespace llvm {
     return GetUnderlyingObject(const_cast<Value *>(V), DL, MaxLookup);
   }
 
-  /// GetUnderlyingObjects - This method is similar to GetUnderlyingObject
-  /// except that it can look through phi and select instructions and return
-  /// multiple objects.
+  /// \brief This method is similar to GetUnderlyingObject except that it can
+  /// look through phi and select instructions and return multiple objects.
+  ///
+  /// If LoopInfo is passed, loop phis are further analyzed.  If a pointer
+  /// accesses different objects in each iteration, we don't look through the
+  /// phi node. E.g. consider this loop nest:
+  ///
+  ///   int **A;
+  ///   for (i)
+  ///     for (j) {
+  ///        A[i][j] = A[i-1][j] * B[j]
+  ///     }
+  ///
+  /// This is transformed by Load-PRE to stash away A[i] for the next iteration
+  /// of the outer loop:
+  ///
+  ///   Curr = A[0];          // Prev_0
+  ///   for (i: 1..N) {
+  ///     Prev = Curr;        // Prev = PHI (Prev_0, Curr)
+  ///     Curr = A[i];
+  ///     for (j: 0..N) {
+  ///        Curr[j] = Prev[j] * B[j]
+  ///     }
+  ///   }
+  ///
+  /// Since A[i] and A[i-1] are independent pointers, getUnderlyingObjects
+  /// should not assume that Curr and Prev share the same underlying object thus
+  /// it shouldn't look through the phi above.
   void GetUnderlyingObjects(Value *V, SmallVectorImpl<Value *> &Objects,
-                            const DataLayout &DL, unsigned MaxLookup = 6);
+                            const DataLayout &DL, LoopInfo *LI = nullptr,
+                            unsigned MaxLookup = 6);
 
   /// onlyUsedByLifetimeMarkers - Return true if the only users of this pointer
   /// are lifetime markers.
   bool onlyUsedByLifetimeMarkers(const Value *V);
 
+  /// isDereferenceablePointer - Return true if this is always a dereferenceable
+  /// pointer. If the context instruction is specified perform context-sensitive
+  /// analysis and return true if the pointer is dereferenceable at the
+  /// specified instruction.
+  bool isDereferenceablePointer(const Value *V, const DataLayout &DL,
+                                const Instruction *CtxI = nullptr,
+                                const DominatorTree *DT = nullptr,
+                                const TargetLibraryInfo *TLI = nullptr);
+  
   /// isSafeToSpeculativelyExecute - Return true if the instruction does not
   /// have any effects besides calculating the result and does not have
   /// undefined behavior.
@@ -199,18 +242,36 @@ namespace llvm {
   /// memory leak. It also returns false for instructions related to control
   /// flow, specifically terminators and PHI nodes.
   ///
-  /// This method only looks at the instruction itself and its operands, so if
-  /// this method returns true, it is safe to move the instruction as long as
-  /// the correct dominance relationships for the operands and users hold.
-  /// However, this method can return true for instructions that read memory;
+  /// If the CtxI is specified this method performs context-sensitive analysis
+  /// and returns true if it is safe to execute the instruction immediately
+  /// before the CtxI.
+  ///
+  /// If the CtxI is NOT specified this method only looks at the instruction
+  /// itself and its operands, so if this method returns true, it is safe to
+  /// move the instruction as long as the correct dominance relationships for
+  /// the operands and users hold.
+  ///
+  /// This method can return true for instructions that read memory;
   /// for such instructions, moving them may change the resulting value.
-  bool isSafeToSpeculativelyExecute(const Value *V);
+  bool isSafeToSpeculativelyExecute(const Value *V,
+                                    const Instruction *CtxI = nullptr,
+                                    const DominatorTree *DT = nullptr,
+                                    const TargetLibraryInfo *TLI = nullptr);
 
   /// isKnownNonNull - Return true if this pointer couldn't possibly be null by
   /// its definition.  This returns true for allocas, non-extern-weak globals
   /// and byval arguments.
   bool isKnownNonNull(const Value *V, const TargetLibraryInfo *TLI = nullptr);
 
+  /// isKnownNonNullAt - Return true if this pointer couldn't possibly be null.
+  /// If the context instruction is specified perform context-sensitive analysis
+  /// and return true if the pointer couldn't possibly be null at the specified
+  /// instruction.
+  bool isKnownNonNullAt(const Value *V,
+                        const Instruction *CtxI = nullptr,
+                        const DominatorTree *DT  = nullptr,
+                        const TargetLibraryInfo *TLI = nullptr);
+
   /// Return true if it is valid to use the assumptions provided by an
   /// assume intrinsic, I, at the point in the control-flow identified by the
   /// context instruction, CxtI.
@@ -228,6 +289,81 @@ namespace llvm {
                                                AssumptionCache *AC,
                                                const Instruction *CxtI,
                                                const DominatorTree *DT);
+
+  /// Return true if this function can prove that the instruction I will
+  /// always transfer execution to one of its successors (including the next
+  /// instruction that follows within a basic block). E.g. this is not
+  /// guaranteed for function calls that could loop infinitely.
+  ///
+  /// In other words, this function returns false for instructions that may
+  /// transfer execution or fail to transfer execution in a way that is not
+  /// captured in the CFG nor in the sequence of instructions within a basic
+  /// block.
+  ///
+  /// Undefined behavior is assumed not to happen, so e.g. division is
+  /// guaranteed to transfer execution to the following instruction even
+  /// though division by zero might cause undefined behavior.
+  bool isGuaranteedToTransferExecutionToSuccessor(const Instruction *I);
+
+  /// Return true if this function can prove that the instruction I
+  /// is executed for every iteration of the loop L.
+  ///
+  /// Note that this currently only considers the loop header.
+  bool isGuaranteedToExecuteForEveryIteration(const Instruction *I,
+                                              const Loop *L);
+
+  /// Return true if this function can prove that I is guaranteed to yield
+  /// full-poison (all bits poison) if at least one of its operands are
+  /// full-poison (all bits poison).
+  ///
+  /// The exact rules for how poison propagates through instructions have
+  /// not been settled as of 2015-07-10, so this function is conservative
+  /// and only considers poison to be propagated in uncontroversial
+  /// cases. There is no attempt to track values that may be only partially
+  /// poison.
+  bool propagatesFullPoison(const Instruction *I);
+
+  /// Return either nullptr or an operand of I such that I will trigger
+  /// undefined behavior if I is executed and that operand has a full-poison
+  /// value (all bits poison).
+  const Value *getGuaranteedNonFullPoisonOp(const Instruction *I);
+
+  /// Return true if this function can prove that if PoisonI is executed
+  /// and yields a full-poison value (all bits poison), then that will
+  /// trigger undefined behavior.
+  ///
+  /// Note that this currently only considers the basic block that is
+  /// the parent of I.
+  bool isKnownNotFullPoison(const Instruction *PoisonI);
+
+  /// \brief Specific patterns of select instructions we can match.
+  enum SelectPatternFlavor {
+    SPF_UNKNOWN = 0,
+    SPF_SMIN,                   // Signed minimum
+    SPF_UMIN,                   // Unsigned minimum
+    SPF_SMAX,                   // Signed maximum
+    SPF_UMAX,                   // Unsigned maximum
+    SPF_ABS,                    // Absolute value
+    SPF_NABS                    // Negated absolute value
+  };
+  /// Pattern match integer [SU]MIN, [SU]MAX and ABS idioms, returning the kind
+  /// and providing the out parameter results if we successfully match.
+  ///
+  /// If CastOp is not nullptr, also match MIN/MAX idioms where the type does
+  /// not match that of the original select. If this is the case, the cast
+  /// operation (one of Trunc,SExt,Zext) that must be done to transform the
+  /// type of LHS and RHS into the type of V is returned in CastOp.
+  ///
+  /// For example:
+  ///   %1 = icmp slt i32 %a, i32 4
+  ///   %2 = sext i32 %a to i64
+  ///   %3 = select i1 %1, i64 %2, i64 4
+  ///
+  /// -> LHS = %a, RHS = i32 4, *CastOp = Instruction::SExt
+  ///
+  SelectPatternFlavor matchSelectPattern(Value *V, Value *&LHS, Value *&RHS,
+                                         Instruction::CastOps *CastOp = nullptr);
+
 } // end namespace llvm
 
 #endif
diff --git a/include/llvm/Analysis/VectorUtils.h b/include/llvm/Analysis/VectorUtils.h
new file mode 100644
index 00000000000..d8e9ca42e62
--- /dev/null
+++ b/include/llvm/Analysis/VectorUtils.h
@@ -0,0 +1,84 @@
+//===- llvm/Transforms/Utils/VectorUtils.h - Vector utilities -*- C++ -*-=====//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This file defines some vectorizer utilities.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_TRANSFORMS_UTILS_VECTORUTILS_H
+#define LLVM_TRANSFORMS_UTILS_VECTORUTILS_H
+
+#include "llvm/Analysis/TargetLibraryInfo.h"
+#include "llvm/IR/IntrinsicInst.h"
+#include "llvm/IR/Intrinsics.h"
+
+namespace llvm {
+
+class GetElementPtrInst;
+class Loop;
+class ScalarEvolution;
+class Type;
+class Value;
+
+/// \brief Identify if the intrinsic is trivially vectorizable.
+/// This method returns true if the intrinsic's argument types are all
+/// scalars for the scalar form of the intrinsic and all vectors for
+/// the vector form of the intrinsic.
+bool isTriviallyVectorizable(Intrinsic::ID ID);
+
+/// \brief Identifies if the intrinsic has a scalar operand. It checks for
+/// ctlz,cttz and powi special intrinsics whose argument is scalar.
+bool hasVectorInstrinsicScalarOpd(Intrinsic::ID ID, unsigned ScalarOpdIdx);
+
+/// \brief Identify if call has a unary float signature
+/// It returns input intrinsic ID if call has a single argument,
+/// argument type and call instruction type should be floating
+/// point type and call should only reads memory.
+/// else return not_intrinsic.
+Intrinsic::ID checkUnaryFloatSignature(const CallInst &I,
+                                       Intrinsic::ID ValidIntrinsicID);
+
+/// \brief Identify if call has a binary float signature
+/// It returns input intrinsic ID if call has two arguments,
+/// arguments type and call instruction type should be floating
+/// point type and call should only reads memory.
+/// else return not_intrinsic.
+Intrinsic::ID checkBinaryFloatSignature(const CallInst &I,
+                                        Intrinsic::ID ValidIntrinsicID);
+
+/// \brief Returns intrinsic ID for call.
+/// For the input call instruction it finds mapping intrinsic and returns
+/// its intrinsic ID, in case it does not found it return not_intrinsic.
+Intrinsic::ID getIntrinsicIDForCall(CallInst *CI, const TargetLibraryInfo *TLI);
+
+/// \brief Find the operand of the GEP that should be checked for consecutive
+/// stores. This ignores trailing indices that have no effect on the final
+/// pointer.
+unsigned getGEPInductionOperand(const GetElementPtrInst *Gep);
+
+/// \brief If the argument is a GEP, then returns the operand identified by 
+/// getGEPInductionOperand. However, if there is some other non-loop-invariant 
+/// operand, it returns that instead.
+Value *stripGetElementPtr(Value *Ptr, ScalarEvolution *SE, Loop *Lp);
+
+/// \brief If a value has only one user that is a CastInst, return it.
+Value *getUniqueCastUse(Value *Ptr, Loop *Lp, Type *Ty);
+
+/// \brief Get the stride of a pointer access in a loop. Looks for symbolic
+/// strides "a[i*stride]". Returns the symbolic stride, or null otherwise.
+Value *getStrideFromPointer(Value *Ptr, ScalarEvolution *SE, Loop *Lp);
+
+/// \brief Given a vector and an element number, see if the scalar value is
+/// already around as a register, for example if it were inserted then extracted
+/// from the vector.
+Value *findScalarElement(Value *V, unsigned EltNo);
+
+} // llvm namespace
+
+#endif
diff --git a/include/llvm/AsmParser/Parser.h b/include/llvm/AsmParser/Parser.h
index 7ef78d73da1..9fe33603bc6 100644
--- a/include/llvm/AsmParser/Parser.h
+++ b/include/llvm/AsmParser/Parser.h
@@ -18,54 +18,75 @@
 
 namespace llvm {
 
+class Constant;
+class LLVMContext;
 class Module;
+struct SlotMapping;
 class SMDiagnostic;
-class LLVMContext;
 
 /// This function is the main interface to the LLVM Assembly Parser. It parses
 /// an ASCII file that (presumably) contains LLVM Assembly code. It returns a
 /// Module (intermediate representation) with the corresponding features. Note
 /// that this does not verify that the generated Module is valid, so you should
 /// run the verifier after parsing the file to check that it is okay.
-/// @brief Parse LLVM Assembly from a file
-/// @param Filename The name of the file to parse
-/// @param Error Error result info.
-/// @param Context Context in which to allocate globals info.
+/// \brief Parse LLVM Assembly from a file
+/// \param Filename The name of the file to parse
+/// \param Error Error result info.
+/// \param Context Context in which to allocate globals info.
+/// \param Slots The optional slot mapping that will be initialized during
+///              parsing.
 std::unique_ptr<Module> parseAssemblyFile(StringRef Filename,
                                           SMDiagnostic &Error,
-                                          LLVMContext &Context);
+                                          LLVMContext &Context,
+                                          SlotMapping *Slots = nullptr);
 
 /// The function is a secondary interface to the LLVM Assembly Parser. It parses
 /// an ASCII string that (presumably) contains LLVM Assembly code. It returns a
 /// Module (intermediate representation) with the corresponding features. Note
 /// that this does not verify that the generated Module is valid, so you should
 /// run the verifier after parsing the file to check that it is okay.
-/// @brief Parse LLVM Assembly from a string
-/// @param AsmString The string containing assembly
-/// @param Error Error result info.
-/// @param Context Context in which to allocate globals info.
+/// \brief Parse LLVM Assembly from a string
+/// \param AsmString The string containing assembly
+/// \param Error Error result info.
+/// \param Context Context in which to allocate globals info.
+/// \param Slots The optional slot mapping that will be initialized during
+///              parsing.
 std::unique_ptr<Module> parseAssemblyString(StringRef AsmString,
                                             SMDiagnostic &Error,
-                                            LLVMContext &Context);
+                                            LLVMContext &Context,
+                                            SlotMapping *Slots = nullptr);
 
 /// parseAssemblyFile and parseAssemblyString are wrappers around this function.
-/// @brief Parse LLVM Assembly from a MemoryBuffer.
-/// @param F The MemoryBuffer containing assembly
-/// @param Err Error result info.
-/// @param Context Context in which to allocate globals info.
+/// \brief Parse LLVM Assembly from a MemoryBuffer.
+/// \param F The MemoryBuffer containing assembly
+/// \param Err Error result info.
+/// \param Slots The optional slot mapping that will be initialized during
+///              parsing.
 std::unique_ptr<Module> parseAssembly(MemoryBufferRef F, SMDiagnostic &Err,
-                                      LLVMContext &Context);
+                                      LLVMContext &Context,
+                                      SlotMapping *Slots = nullptr);
 
 /// This function is the low-level interface to the LLVM Assembly Parser.
 /// This is kept as an independent function instead of being inlined into
 /// parseAssembly for the convenience of interactive users that want to add
 /// recently parsed bits to an existing module.
 ///
-/// @param F The MemoryBuffer containing assembly
-/// @param M The module to add data to.
-/// @param Err Error result info.
-/// @return true on error.
-bool parseAssemblyInto(MemoryBufferRef F, Module &M, SMDiagnostic &Err);
+/// \param F The MemoryBuffer containing assembly
+/// \param M The module to add data to.
+/// \param Err Error result info.
+/// \param Slots The optional slot mapping that will be initialized during
+///              parsing.
+/// \return true on error.
+bool parseAssemblyInto(MemoryBufferRef F, Module &M, SMDiagnostic &Err,
+                       SlotMapping *Slots = nullptr);
+
+/// Parse a type and a constant value in the given string.
+///
+/// The constant value can be any LLVM constant, including a constant
+/// expression.
+///
+/// \return null on error.
+Constant *parseConstantValue(StringRef Asm, SMDiagnostic &Err, const Module &M);
 
 } // End llvm namespace
 
diff --git a/include/llvm/AsmParser/SlotMapping.h b/include/llvm/AsmParser/SlotMapping.h
new file mode 100644
index 00000000000..c5f61d25c3a
--- /dev/null
+++ b/include/llvm/AsmParser/SlotMapping.h
@@ -0,0 +1,34 @@
+//===-- SlotMapping.h - Slot number mapping for unnamed values --*- C++ -*-===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This file contains the declaration of the SlotMapping struct.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_ASMPARSER_SLOTMAPPING_H
+#define LLVM_ASMPARSER_SLOTMAPPING_H
+
+#include "llvm/IR/TrackingMDRef.h"
+#include <map>
+#include <vector>
+
+namespace llvm {
+
+class GlobalValue;
+
+/// This struct contains the mapping from the slot numbers to unnamed metadata
+/// nodes and global values.
+struct SlotMapping {
+  std::vector<GlobalValue *> GlobalValues;
+  std::map<unsigned, TrackingMDNodeRef> MetadataNodes;
+};
+
+} // end namespace llvm
+
+#endif
diff --git a/include/llvm/Bitcode/BitstreamReader.h b/include/llvm/Bitcode/BitstreamReader.h
index 18f6b9e011e..4c040a7f3e2 100644
--- a/include/llvm/Bitcode/BitstreamReader.h
+++ b/include/llvm/Bitcode/BitstreamReader.h
@@ -113,7 +113,7 @@ class BitstreamReader {
       return *const_cast<BlockInfo*>(BI);
 
     // Otherwise, add a new record.
-    BlockInfoRecords.push_back(BlockInfo());
+    BlockInfoRecords.emplace_back();
     BlockInfoRecords.back().BlockID = BlockID;
     return BlockInfoRecords.back();
   }
@@ -198,6 +198,8 @@ class BitstreamCursor {
 
 
 public:
+  static const size_t MaxChunkSize = sizeof(word_t) * 8;
+
   BitstreamCursor() { init(nullptr); }
 
   explicit BitstreamCursor(BitstreamReader &R) { init(&R); }
@@ -335,7 +337,7 @@ class BitstreamCursor {
   }
 
   word_t Read(unsigned NumBits) {
-    static const unsigned BitsInWord = sizeof(word_t) * 8;
+    static const unsigned BitsInWord = MaxChunkSize;
 
     assert(NumBits && NumBits <= BitsInWord &&
            "Cannot return zero or more than BitsInWord bits!");
diff --git a/include/llvm/Bitcode/BitstreamWriter.h b/include/llvm/Bitcode/BitstreamWriter.h
index 9e2c2fa4a15..9f23023a141 100644
--- a/include/llvm/Bitcode/BitstreamWriter.h
+++ b/include/llvm/Bitcode/BitstreamWriter.h
@@ -18,6 +18,7 @@
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/ADT/StringRef.h"
 #include "llvm/Bitcode/BitCodes.h"
+#include "llvm/Support/Endian.h"
 #include <vector>
 
 namespace llvm {
@@ -63,10 +64,7 @@ class BitstreamWriter {
   // BackpatchWord - Backpatch a 32-bit word in the output with the specified
   // value.
   void BackpatchWord(unsigned ByteNo, unsigned NewWord) {
-    Out[ByteNo++] = (unsigned char)(NewWord >>  0);
-    Out[ByteNo++] = (unsigned char)(NewWord >>  8);
-    Out[ByteNo++] = (unsigned char)(NewWord >> 16);
-    Out[ByteNo  ] = (unsigned char)(NewWord >> 24);
+    support::endian::write32le(&Out[ByteNo], NewWord);
   }
 
   void WriteByte(unsigned char Value) {
@@ -74,12 +72,9 @@ class BitstreamWriter {
   }
 
   void WriteWord(unsigned Value) {
-    unsigned char Bytes[4] = {
-      (unsigned char)(Value >>  0),
-      (unsigned char)(Value >>  8),
-      (unsigned char)(Value >> 16),
-      (unsigned char)(Value >> 24) };
-    Out.append(&Bytes[0], &Bytes[4]);
+    Value = support::endian::byte_swap<uint32_t, support::little>(Value);
+    Out.append(reinterpret_cast<const char *>(&Value),
+               reinterpret_cast<const char *>(&Value + 1));
   }
 
   unsigned GetBufferOffset() const {
@@ -215,7 +210,7 @@ class BitstreamWriter {
 
     // Push the outer block's abbrev set onto the stack, start out with an
     // empty abbrev set.
-    BlockScope.push_back(Block(OldCodeSize, BlockSizeWordIndex));
+    BlockScope.emplace_back(OldCodeSize, BlockSizeWordIndex);
     BlockScope.back().PrevAbbrevs.swap(CurAbbrevs);
 
     // If there is a blockinfo for this BlockID, add all the predefined abbrevs
@@ -503,7 +498,7 @@ class BitstreamWriter {
       return *BI;
 
     // Otherwise, add a new record.
-    BlockInfoRecords.push_back(BlockInfo());
+    BlockInfoRecords.emplace_back();
     BlockInfoRecords.back().BlockID = BlockID;
     return BlockInfoRecords.back();
   }
diff --git a/include/llvm/Bitcode/LLVMBitCodes.h b/include/llvm/Bitcode/LLVMBitCodes.h
index e450db0eba6..6a9f32e1aad 100644
--- a/include/llvm/Bitcode/LLVMBitCodes.h
+++ b/include/llvm/Bitcode/LLVMBitCodes.h
@@ -167,6 +167,7 @@ namespace bitc {
     METADATA_EXPRESSION    = 29,  // [distinct, n x element]
     METADATA_OBJC_PROPERTY = 30,  // [distinct, name, file, line, ...]
     METADATA_IMPORTED_ENTITY=31,  // [distinct, tag, scope, entity, line, name]
+    METADATA_MODULE=32,           // [distinct, scope, name, ...]
   };
 
   // The constants block (CONSTANTS_BLOCK_ID) describes emission for each
@@ -318,7 +319,7 @@ namespace bitc {
     // This store code encodes the pointer type, rather than the value type
     // this is so information only available in the pointer type (e.g. address
     // spaces) is retained.
-    FUNC_CODE_INST_STORE       = 24, // STORE:      [ptrty,ptr,val, align, vol]
+    FUNC_CODE_INST_STORE_OLD   = 24, // STORE:      [ptrty,ptr,val, align, vol]
     // 25 is unused.
     FUNC_CODE_INST_EXTRACTVAL  = 26, // EXTRACTVAL: [n x operands]
     FUNC_CODE_INST_INSERTVAL   = 27, // INSERTVAL:  [n x operands]
@@ -336,18 +337,29 @@ namespace bitc {
 
     FUNC_CODE_DEBUG_LOC        = 35, // DEBUG_LOC:  [Line,Col,ScopeVal, IAVal]
     FUNC_CODE_INST_FENCE       = 36, // FENCE: [ordering, synchscope]
-    FUNC_CODE_INST_CMPXCHG     = 37, // CMPXCHG: [ptrty,ptr,cmp,new, align, vol,
+    FUNC_CODE_INST_CMPXCHG_OLD = 37, // CMPXCHG: [ptrty,ptr,cmp,new, align, vol,
                                      //           ordering, synchscope]
     FUNC_CODE_INST_ATOMICRMW   = 38, // ATOMICRMW: [ptrty,ptr,val, operation,
                                      //             align, vol,
                                      //             ordering, synchscope]
     FUNC_CODE_INST_RESUME      = 39, // RESUME:     [opval]
-    FUNC_CODE_INST_LANDINGPAD  = 40, // LANDINGPAD: [ty,val,val,num,id0,val0...]
+    FUNC_CODE_INST_LANDINGPAD_OLD  = 40, // LANDINGPAD: [ty,val,val,num,id0,val0...]
     FUNC_CODE_INST_LOADATOMIC  = 41, // LOAD: [opty, op, align, vol,
                                      //        ordering, synchscope]
-    FUNC_CODE_INST_STOREATOMIC = 42, // STORE: [ptrty,ptr,val, align, vol
+    FUNC_CODE_INST_STOREATOMIC_OLD = 42, // STORE: [ptrty,ptr,val, align, vol
                                      //         ordering, synchscope]
     FUNC_CODE_INST_GEP         = 43, // GEP:  [inbounds, n x operands]
+    FUNC_CODE_INST_STORE       = 44, // STORE: [ptrty,ptr,valty,val, align, vol]
+    FUNC_CODE_INST_STOREATOMIC = 45, // STORE: [ptrty,ptr,val, align, vol
+    FUNC_CODE_INST_CMPXCHG     = 46, // CMPXCHG: [ptrty,ptr,valty,cmp,new, align,
+                                     //           vol,ordering,synchscope]
+    FUNC_CODE_INST_LANDINGPAD  = 47, // LANDINGPAD: [ty,val,num,id0,val0...]
+    FUNC_CODE_INST_CLEANUPRET  = 48, // CLEANUPRET: [] or [val] or [bb#] or [val,bb#]
+    FUNC_CODE_INST_CATCHRET    = 49, // CATCHRET: [bb#]
+    FUNC_CODE_INST_CATCHPAD  = 50, // CATCHPAD: [ty,val,val,num,args...]
+    FUNC_CODE_INST_TERMINATEPAD = 51, // TERMINATEPAD: [bb#,num,args...]
+    FUNC_CODE_INST_CLEANUPPAD = 52, // CLEANUPPAD: [num,args...]
+    FUNC_CODE_INST_CATCHENDPAD = 53, // CATCHENDPAD: [] or [bb#]
   };
 
   enum UseListCodes {
@@ -398,7 +410,12 @@ namespace bitc {
     ATTR_KIND_NON_NULL = 39,
     ATTR_KIND_JUMP_TABLE = 40,
     ATTR_KIND_DEREFERENCEABLE = 41,
-    ATTR_KIND_DEREFERENCEABLE_OR_NULL = 42
+    ATTR_KIND_DEREFERENCEABLE_OR_NULL = 42,
+    ATTR_KIND_CONVERGENT = 43,
+    ATTR_KIND_SAFESTACK = 44,
+    ATTR_KIND_ARGMEMONLY = 45,
+    ATTR_KIND_SWIFT_SELF = 46,
+    ATTR_KIND_SWIFT_ERROR = 47
   };
 
   enum ComdatSelectionKindCodes {
diff --git a/include/llvm/Bitcode/NaCl/AbbrevTrieNode.h b/include/llvm/Bitcode/NaCl/AbbrevTrieNode.h
deleted file mode 100644
index e3a6ee4ba88..00000000000
--- a/include/llvm/Bitcode/NaCl/AbbrevTrieNode.h
+++ /dev/null
@@ -1,135 +0,0 @@
-//===- AbbrevTrieNode.h - Abbreviation lookup tries         ----*- C++ -*-===//
-//
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
-//
-//===----------------------------------------------------------------------===//
-//
-// This header file defines class AbbrevTrieNode that implement abbreviation
-// lookup tries. These tries reduce the set of abbreviations that need to
-// be tested for best fit to a pnacl bitcode record, by sorting abbreviations
-// on literal constants that may appear in the abbreviations. By doing this,
-// we can reduce hundreds of possible abbreviations down to a small number
-// of possibly applicable abbreviations.
-//
-// The tries separate abbreviations based on constant size, and constants
-// that appear in the abbreviations. The trie is used to capture constants
-// that appear at any index, and use these constants to decide if a trie
-// node applies to the record.
-//
-//===----------------------------------------------------------------------===//
-
-#ifndef LLVM_BITCODE_NACL_ABBREV_TRIE_NODE_H
-#define LLVM_BITCODE_NACL_ABBREV_TRIE_NODE_H
-
-#include "llvm/Bitcode/NaCl/NaClBitcodeParser.h"
-#include <map>
-#include <set>
-
-namespace llvm {
-
-// Models the association of an abbreviation index with the
-// corresponding abbreviation associated with that index.
-typedef std::pair<size_t, NaClBitCodeAbbrev*> AbbrevIndexPair;
-
-// Models a trie of abbreviation matches that can be used to reduce
-// the number of applicable abbreviations. This is done by moving
-// abbreviations that require literals to appear in them, to be in
-// successor nodes. Abbreviations without literals are stored
-// in this node.
-class AbbrevTrieNode {
-  // For faster lookup, we could model the successor map as
-  // std::map<std::pair<size_t, uint64_t>, AbbrevTrieNode*>.  However,
-  // we split up the pair into a nested map. This allows us to reduce
-  // the size of the domain of the map considerably, as well as
-  // avoiding much value (i.e.  std::pair<size_t, uint64_t>)
-  // copying. These modifications result in considerable better time
-  // performance.
-  //
-  // We also do this representation because the trie is sparse,
-  // with respect to where constants can appear. Hence, we don't
-  // built a possible successor for all possible indicies, but only
-  // for those that can contain constants in some abbreviation.
-  typedef std::map<uint64_t, AbbrevTrieNode*> SuccessorValueMap;
-  typedef std::map<size_t, SuccessorValueMap*> SuccessorMap;
-
-public:
-  // Models the list of successor labels defined for a node.
-  typedef std::vector< std::pair<size_t, uint64_t> > SuccessorLabels;
-
-  // Creates an entry node into the trie.
-  AbbrevTrieNode() {}
-
-  ~AbbrevTrieNode();
-
-  // Print out the trie to the given stream, indenting the given
-  // amount.  If LocalOnly is true, no successor information is
-  // printed.
-  void Print(raw_ostream &Stream,
-             const std::string &Indent,
-             bool LocalOnly=false) const;
-
-  // Adds matching constants, defined in Abbreviation, to the trie.
-  // Returns true if any nodes were added to the trie to add the given
-  // abbreviation. Note: This method only creates nodes, Abbreviations
-  // must be aded in a separate pass using method Insert.  Note: If
-  // you call NaClBuildAbbrevLookupMap, it will construct the
-  // (complete) abbrevation trie, calling Add and Insert in the
-  // appropriate order.
-  bool Add(NaClBitCodeAbbrev *Abbrev) {
-    return Add(Abbrev, 0, 0);
-  }
-
-  // Inserts the given abbreviation (pair) in all trie nodes that
-  // might match the given abbreviation. Should not be called until
-  // all trie nodes are built using Add.
-  void Insert(AbbrevIndexPair &AbbrevPair);
-
-  // Returns the successor trie node matching the given Index/Value pair.
-  AbbrevTrieNode* GetSuccessor(size_t Index, uint64_t Value) const;
-
-  // Collects the set of successor (edge) labels defined for the node.
-  void GetSuccessorLabels(SuccessorLabels &labels) const;
-
-  // Returns a trie node (in the trie) that defines all possible
-  // abbreviations that may apply to the given record.
-  const AbbrevTrieNode *MatchRecord(const NaClBitcodeRecordData &Record) const;
-
-  // Returns the abbreviations associated with the node.
-  const std::set<AbbrevIndexPair> &GetAbbreviations() const {
-    return Abbreviations;
-  }
-
-private:
-  // Adds matching constants, defined in Abbreviation, to the trie.
-  // Returns true if any nodes were added to the trie to add the given
-  // abbreviation. Index is the positition (within the abbreviation)
-  // where search should begin for the next available
-  // literal. SkipIndex is the smallest skipped index used to find the
-  // next available literal.
-  bool Add(NaClBitCodeAbbrev *Abbrev,
-           size_t Index,
-           size_t SkipIndex);
-
-  // The set of possible successor trie nodes defined for this node.
-  SuccessorMap Successors;
-
-  // The set of abbreviations that apply if one can't match a pnacl
-  // bitcode record against any of the successors.
-  std::set<AbbrevIndexPair> Abbreviations;
-};
-
-// A map from record sizes, to the corresponding trie one should use
-// to find abbreviations for records of that size.
-typedef std::map<size_t, AbbrevTrieNode*> AbbrevLookupSizeMap;
-
-// Builds abbreviation lookup trie for abbreviations
-void NaClBuildAbbrevLookupMap(AbbrevLookupSizeMap &LookupMap,
-                              const SmallVectorImpl<NaClBitCodeAbbrev*> &Abbrevs,
-                              size_t InitialIndex = 0);
-
-}
-
-#endif
diff --git a/include/llvm/Bitcode/NaCl/NaClAnalyzerBlockDist.h b/include/llvm/Bitcode/NaCl/NaClAnalyzerBlockDist.h
deleted file mode 100644
index 9845fe6a6ae..00000000000
--- a/include/llvm/Bitcode/NaCl/NaClAnalyzerBlockDist.h
+++ /dev/null
@@ -1,100 +0,0 @@
-//===-- NaClAnalyzerBlockDist.h --------------------------------------------===//
-//      Defines distribution maps used to collect block and record
-//      distributions for tools pnacl-bcanalyzer.
-//
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
-//
-//===----------------------------------------------------------------------===//
-//
-// Defines a block distribution, with nested subblock and record code
-// distributions, to be collected during analysis.
-
-#ifndef LLVM_BITCODE_NACL_NACLANALYZERBLOCKDIST_H
-#define LLVM_BITCODE_NACL_NACLANALYZERBLOCKDIST_H
-
-#include "llvm/Bitcode/NaCl/NaClBitcodeBlockDist.h"
-#include "llvm/Bitcode/NaCl/NaClBitcodeCodeDist.h"
-#include "llvm/Bitcode/NaCl/NaClBitcodeSubblockDist.h"
-
-namespace llvm {
-
-/// Holds block distribution, and nested subblock and record code distributions,
-/// to be collected during analysis.
-class NaClAnalyzerBlockDistElement : public NaClBitcodeBlockDistElement {
-  NaClAnalyzerBlockDistElement(const NaClAnalyzerBlockDistElement&) = delete;
-  void operator=(const NaClAnalyzerBlockDistElement&) = delete;
-
-public:
-  static bool classof(const NaClBitcodeDistElement *Element) {
-    return Element->getKind() >= RDE_NaClAnalBlockDist &&
-        Element->getKind() < RDE_NaClAnalBlockDistLast;
-  }
-
-  explicit NaClAnalyzerBlockDistElement(unsigned BlockID=0,
-                                        bool OrderBlocksByID=false)
-      : NaClBitcodeBlockDistElement(RDE_NaClAnalBlockDist),
-        BlockID(BlockID),
-        RecordDist(BlockID),
-        OrderBlocksByID(OrderBlocksByID) {
-    NestedDists.push_back(&SubblockDist);
-    NestedDists.push_back(&RecordDist);
-  }
-
-  virtual ~NaClAnalyzerBlockDistElement();
-
-  virtual NaClBitcodeDistElement*
-  CreateElement(NaClBitcodeDistValue Value) const;
-
-  virtual double GetImportance(NaClBitcodeDistValue Value) const;
-
-  virtual const SmallVectorImpl<NaClBitcodeDist*> *
-  GetNestedDistributions() const;
-
-  NaClBitcodeDist &GetSubblockDist() {
-    return SubblockDist;
-  }
-
-  NaClBitcodeCodeDist &GetRecordDist() {
-    return RecordDist;
-  }
-
-private:
-  // The block ID of the distribution.
-  unsigned BlockID;
-
-  // Subblocks that appear in this block.
-  NaClBitcodeSubblockDist SubblockDist;
-
-  // Records that appear in this block.
-  NaClBitcodeCodeDist RecordDist;
-
-  // Nested blocks used by GetNestedDistributions.
-  SmallVector<NaClBitcodeDist*, 2> NestedDists;
-
-  // If true, order (top-level) blocks by block ID instead of file
-  // size.
-  bool OrderBlocksByID;
-};
-
-/// Holds block distribution, and nested subblock and record code distributions,
-/// to be collected during analysis.
-class NaClAnalyzerBlockDist : public NaClBitcodeBlockDist {
-  NaClAnalyzerBlockDist(const NaClAnalyzerBlockDist&) = delete;
-  void operator=(const NaClAnalyzerBlockDist&) = delete;
-public:
-  NaClAnalyzerBlockDist(NaClAnalyzerBlockDistElement &Sentinel)
-      : NaClBitcodeBlockDist(&Sentinel)
-  {}
-
-  virtual ~NaClAnalyzerBlockDist() {}
-
-  virtual void AddRecord(const NaClBitcodeRecord &Record);
-
-  virtual void AddBlock(const NaClBitcodeBlock &Block);
-};
-
-}
-#endif
diff --git a/include/llvm/Bitcode/NaCl/NaClBitCodes.h b/include/llvm/Bitcode/NaCl/NaClBitCodes.h
deleted file mode 100644
index 44b61a1c4bb..00000000000
--- a/include/llvm/Bitcode/NaCl/NaClBitCodes.h
+++ /dev/null
@@ -1,407 +0,0 @@
-//===- NaClBitCodes.h - Enum values for the bitcode format ------*- C++ -*-===//
-//
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
-//
-//===----------------------------------------------------------------------===//
-//
-// This header Bitcode enum values.
-//
-// The enum values defined in this file should be considered permanent.  If
-// new features are added, they should have values added at the end of the
-// respective lists.
-//
-//===----------------------------------------------------------------------===//
-
-#ifndef LLVM_BITCODE_NACL_NACLBITCODES_H
-#define LLVM_BITCODE_NACL_NACLBITCODES_H
-
-#include "llvm/ADT/SmallVector.h"
-#include "llvm/Support/DataTypes.h"
-#include "llvm/Support/ErrorHandling.h"
-#include "llvm/Support/MathExtras.h"
-#include <cassert>
-
-namespace llvm {
-class raw_ostream;
-
-namespace naclbitc {
-  enum StandardWidths {
-    BlockIDWidth   = 8,  // We use VBR-8 for block IDs.
-    CodeLenWidth   = 4,  // Codelen are VBR-4.
-    BlockSizeWidth = 32, // BlockSize up to 2^32 32-bit words = 16GB per block.
-    MaxAbbrevWidth = 32  // Maximum value allowed for Fixed and VBR.
-  };
-
-  // The standard abbrev namespace always has a way to exit a block, enter a
-  // nested block, define abbrevs, and define an unabbreviated record.
-  enum FixedAbbrevIDs {
-    END_BLOCK = 0,  // Must be zero to guarantee termination for broken bitcode.
-    ENTER_SUBBLOCK = 1,
-
-    /// DEFINE_ABBREV - Defines an abbrev for the current block.  It consists
-    /// of a vbr5 for # operand infos.  Each operand info is emitted with a
-    /// single bit to indicate if it is a literal encoding.  If so, the value is
-    /// emitted with a vbr8.  If not, the encoding is emitted as 3 bits followed
-    /// by the info value as a vbr5 if needed.
-    DEFINE_ABBREV = 2,
-
-    // UNABBREV_RECORDs are emitted with a vbr6 for the record code, followed by
-    // a vbr6 for the # operands, followed by vbr6's for each operand.
-    UNABBREV_RECORD = 3,
-
-    // This is not a code, this is a marker for the first abbrev assignment.
-    // In addition, we assume up to two additional enumerated constants are
-    // added for each extension. These constants are:
-    //
-    //   PREFIX_MAX_FIXED_ABBREV
-    //   PREFIX_MAX_ABBREV
-    //
-    // PREFIX_MAX_ABBREV defines the maximal enumeration value used for
-    // the code selector of a block. If Both PREFIX_MAX_FIXED_ABBREV
-    // and PREFIX_MAX_ABBREV is defined, then PREFIX_MAX_FIXED_ABBREV
-    // defines the last code selector of the block that must be read using
-    // a single read (i.e. a FIXED read, or the first chunk of a VBR read.
-    FIRST_APPLICATION_ABBREV = 4,
-    // Defines default values for code length, if no additional selectors
-    // are added.
-    DEFAULT_MAX_ABBREV = FIRST_APPLICATION_ABBREV-1
-  };
-
-  /// StandardBlockIDs - All bitcode files can optionally include a BLOCKINFO
-  /// block, which contains metadata about other blocks in the file.
-  enum StandardBlockIDs {
-    /// BLOCKINFO_BLOCK is used to define metadata about blocks, for example,
-    /// standard abbrevs that should be available to all blocks of a specified
-    /// ID.
-    BLOCKINFO_BLOCK_ID = 0,
-
-    // Block IDs 1-7 are reserved for future expansion.
-    FIRST_APPLICATION_BLOCKID = 8
-  };
-
-  /// BlockInfoCodes - The blockinfo block contains metadata about user-defined
-  /// blocks.
-  enum BlockInfoCodes {
-    // DEFINE_ABBREV has magic semantics here, applying to the current SETBID'd
-    // block, instead of the BlockInfo block.
-
-    BLOCKINFO_CODE_SETBID        = 1, // SETBID: [blockid#]
-                                      // The following two codes were removed
-                                      // because the PNaCl reader could read
-                                      // them, but couldn't be generated by
-                                      // the writer.
-    BLOCKINFO_CODE_BLOCKNAME     = 2, // Not used in PNaCl.
-    BLOCKINFO_CODE_SETRECORDNAME = 3  // Not used in PNaCl.
-  };
-
-} // End naclbitc namespace
-
-/// NaClBitCodeAbbrevOp - This describes one or more operands in an abbreviation.
-/// This is actually a union of two different things:
-///   1. It could be a literal integer value ("the operand is always 17").
-///   2. It could be an encoding specification ("this operand encoded like so").
-///
-class NaClBitCodeAbbrevOp {
-public:
-  enum Encoding {
-    Literal = 0, // Value is literal value.
-    Fixed = 1,   // A fixed width field, Val specifies number of bits.
-    VBR   = 2,   // A VBR field where Val specifies the width of each chunk.
-    Array = 3,   // A sequence of fields, next field species elt encoding.
-    Char6 = 4,   // A 6-bit fixed field which maps to [a-zA-Z0-9._].
-    Encoding_MAX = Char6
-  };
-
-  explicit NaClBitCodeAbbrevOp(uint64_t V) :  Enc(Literal), Val(V) {}
-  explicit NaClBitCodeAbbrevOp(Encoding E, uint64_t Data = 0);
-
-  Encoding getEncoding() const { return Enc; }
-
-  static bool isValidEncoding(uint64_t Enc) { return Enc <= Encoding_MAX; }
-
-  uint64_t getValue() const { return Val; }
-
-  bool hasValue() const {
-    return hasValue(Enc);
-  }
-  static bool hasValue(Encoding E) {
-    return E <= Encoding_MAX && HasValueArray[E];
-  }
-
-  bool isValid() const { return isValid(Enc, Val); }
-  static bool isValid(Encoding E, uint64_t Val);
-  static bool isValid(Encoding E) { return isValid(E, 0); }
-
-  bool isLiteral() const { return Enc == Literal; }
-
-  bool isArrayOp() const { return Enc == Array; }
-
-  /// Returns the number of arguments expected by this abbrevation operator.
-  unsigned NumArguments() const {
-    if (isArrayOp())
-      return 1;
-    else
-      return 0;
-  }
-
-  // Returns the name of the encoding
-  static const char *getEncodingName(Encoding E) {
-    if (E > Encoding_MAX)
-      return "???";
-    return EncodingNameArray[E];
-  }
-
-  /// Prints out the abbreviation operator to the given stream.
-  void Print(raw_ostream &Stream) const;
-
-  /// isChar6 - Return true if this character is legal in the Char6 encoding.
-  static bool isChar6(char C) {
-    if (C >= 'a' && C <= 'z') return true;
-    if (C >= 'A' && C <= 'Z') return true;
-    if (C >= '0' && C <= '9') return true;
-    if (C == '.' || C == '_') return true;
-    return false;
-  }
-  static unsigned EncodeChar6(char C) {
-    if (C >= 'a' && C <= 'z') return C-'a';
-    if (C >= 'A' && C <= 'Z') return C-'A'+26;
-    if (C >= '0' && C <= '9') return C-'0'+26+26;
-    if (C == '.')             return 62;
-    if (C == '_')             return 63;
-    llvm_unreachable("Not a value Char6 character!");
-  }
-
-  static char DecodeChar6(unsigned V) {
-    assert((V & ~63) == 0 && "Not a Char6 encoded character!");
-    if (V < 26)       return V+'a';
-    if (V < 26+26)    return V-26+'A';
-    if (V < 26+26+10) return V-26-26+'0';
-    if (V == 62)      return '.';
-    if (V == 63)      return '_';
-    llvm_unreachable("Not a value Char6 character!");
-  }
-
-  /// \brief Compares this to Op. Returns <0 if this is less than Op,
-  /// Returns 0 if they are equal, and >0 if this is greater than Op.
-  int Compare(const NaClBitCodeAbbrevOp &Op) const {
-    // Compare encoding values.
-    int EncodingDiff = static_cast<int>(Enc) - static_cast<int>(Op.Enc);
-    if (EncodingDiff != 0) return EncodingDiff;
-
-    // Encodings don't differ, so now base on data associated with the
-    // encoding.
-    return ValCompare(Op);
-  }
-
-private:
-  Encoding Enc;           // The encoding to use.
-  uint64_t Val;           // Data associated with encoding (if any).
-
-  int ValCompare(const NaClBitCodeAbbrevOp &Op) const {
-    if (Val < Op.Val)
-      return -1;
-    else if (Val > Op.Val)
-      return 1;
-    else
-      return 0;
-  }
-  static const bool HasValueArray[];
-  static const char *EncodingNameArray[];
-};
-
-template <> struct isPodLike<NaClBitCodeAbbrevOp> {
-  static const bool value=true;
-};
-
-static inline bool operator<(const NaClBitCodeAbbrevOp &Op1,
-                             const NaClBitCodeAbbrevOp &Op2) {
-  return Op1.Compare(Op2) < 0;
-}
-
-static inline bool operator<=(const NaClBitCodeAbbrevOp &Op1,
-                              const NaClBitCodeAbbrevOp &Op2) {
-  return Op1.Compare(Op2) <= 0;
-}
-
-static inline bool operator==(const NaClBitCodeAbbrevOp &Op1,
-                              const NaClBitCodeAbbrevOp &Op2) {
-  return Op1.Compare(Op2) == 0;
-}
-
-static inline bool operator!=(const NaClBitCodeAbbrevOp &Op1,
-                              const NaClBitCodeAbbrevOp &Op2) {
-  return Op1.Compare(Op2) != 0;
-}
-
-static inline bool operator>=(const NaClBitCodeAbbrevOp &Op1,
-                              const NaClBitCodeAbbrevOp &Op2) {
-  return Op1.Compare(Op2) >= 0;
-}
-
-static inline bool operator>(const NaClBitCodeAbbrevOp &Op1,
-                             const NaClBitCodeAbbrevOp &Op2) {
-  return Op1.Compare(Op2) > 0;
-}
-
-/// NaClBitCodeAbbrev - This class represents an abbreviation record.  An
-/// abbreviation allows a complex record that has redundancy to be stored in a
-/// specialized format instead of the fully-general, fully-vbr, format.
-class NaClBitCodeAbbrev {
-  SmallVector<NaClBitCodeAbbrevOp, 8> OperandList;
-  unsigned char RefCount; // Number of things using this.
-  ~NaClBitCodeAbbrev() {}
-public:
-  NaClBitCodeAbbrev() : RefCount(1) {}
-
-  void addRef() { ++RefCount; }
-  void dropRef() { if (--RefCount == 0) delete this; }
-
-  unsigned getNumOperandInfos() const {
-    return static_cast<unsigned>(OperandList.size());
-  }
-  const NaClBitCodeAbbrevOp &getOperandInfo(unsigned N) const {
-    return OperandList[N];
-  }
-
-  void Add(const NaClBitCodeAbbrevOp &OpInfo) {
-    OperandList.push_back(OpInfo);
-  }
-
-  // Returns a simplified version of the abbreviation. Used
-  // to recognize equivalent abbrevations.
-  NaClBitCodeAbbrev *Simplify() const;
-
-  // Returns true if the abbreviation is valid wrt to the bitcode reader.
-  bool isValid() const;
-
-  int Compare(const NaClBitCodeAbbrev &Abbrev) const {
-    // First order based on number of operands.
-    size_t OperandListSize = OperandList.size();
-    size_t AbbrevOperandListSize = Abbrev.OperandList.size();
-    if (OperandListSize < AbbrevOperandListSize)
-      return -1;
-    else if (OperandListSize > AbbrevOperandListSize)
-      return 1;
-
-    // Same number of operands, so compare element by element.
-    for (size_t I = 0; I < OperandListSize; ++I) {
-      if (int Diff = OperandList[I].Compare(Abbrev.OperandList[I]))
-        return Diff;
-    }
-    return 0;
-  }
-
-  // Returns true if all records matching the abbreviation must be
-  // of fixed length.
-  bool IsFixedSize() const {
-    unsigned Size = getNumOperandInfos();
-    if (Size < 2) return true;
-    return !OperandList[Size-2].isArrayOp();
-  }
-
-  // Returns the smallest record size that will match this
-  // abbreviation.
-  size_t GetMinRecordSize() const {
-    size_t Min = getNumOperandInfos();
-    if (!IsFixedSize()) Min -= 2;
-    return Min;
-  }
-
-  void Print(raw_ostream &Stream, bool AddNewline=true) const;
-
-  NaClBitCodeAbbrev *Copy() const {
-    NaClBitCodeAbbrev *AbbrevCopy = new NaClBitCodeAbbrev();
-    for (unsigned I = 0, IEnd = getNumOperandInfos();
-         I != IEnd; ++I) {
-      AbbrevCopy->Add(NaClBitCodeAbbrevOp(getOperandInfo(I)));
-    }
-    return AbbrevCopy;
-  }
-};
-
-static inline bool operator<(const NaClBitCodeAbbrev &A1,
-                             const NaClBitCodeAbbrev &A2) {
-  return A1.Compare(A2) < 0;
-}
-
-static inline bool operator<=(const NaClBitCodeAbbrev &A1,
-                              const NaClBitCodeAbbrev &A2) {
-  return A1.Compare(A2) <= 0;
-}
-static inline bool operator==(const NaClBitCodeAbbrev &A1,
-                              const NaClBitCodeAbbrev &A2) {
-  return A1.Compare(A2) == 0;
-}
-
-static inline bool operator!=(const NaClBitCodeAbbrev &A1,
-                              const NaClBitCodeAbbrev &A2) {
-  return A1.Compare(A2) != 0;
-}
-static inline bool operator>=(const NaClBitCodeAbbrev &A1,
-                              const NaClBitCodeAbbrev &A2) {
-  return A1.Compare(A2) >= 0;
-}
-
-static inline bool operator>(const NaClBitCodeAbbrev &A1,
-                             const NaClBitCodeAbbrev &A2) {
-  return A1.Compare(A2) > 0;
-}
-
-/// \brief Returns number of bits needed to encode
-/// value for dense FIXED encoding.
-inline unsigned NaClBitsNeededForValue(unsigned Value) {
-  // Note: Need to handle case where Value=0xFFFFFFFF as special case,
-  // since we can't add 1 to it.
-  if (Value >= 0x80000000) return 32;
-  return Log2_32_Ceil(Value+1);
-}
-
-/// \brief Encode a signed value by moving the sign to the LSB for dense
-/// VBR encoding.
-inline uint64_t NaClEncodeSignRotatedValue(int64_t V) {
-  return (V >= 0) ? (V << 1) : ((-V << 1) | 1);
-}
-
-/// \brief Decode a signed value stored with the sign bit in
-/// the LSB for dense VBR encoding.
-inline uint64_t NaClDecodeSignRotatedValue(uint64_t V) {
-  if ((V & 1) == 0)
-    return V >> 1;
-  if (V != 1)
-    return -(V >> 1);
-  // There is no such thing as -0 with integers.  "-0" really means MININT.
-  return 1ULL << 63;
-}
-
-/// \brief This class determines whether a FIXED or VBR
-/// abbreviation should be used for the selector, and the number of bits
-/// needed to capture such selectors.
-class NaClBitcodeSelectorAbbrev {
-
-public:
-  // If true, use a FIXED abbreviation. Otherwise, use a VBR abbreviation.
-  bool IsFixed;
-  // Number of bits needed for selector.
-  unsigned NumBits;
-
-  // Creates a selector range for the given values.
-  NaClBitcodeSelectorAbbrev(bool IF, unsigned NB)
-      : IsFixed(IF), NumBits(NB) {}
-
-  // Creates a selector range when no abbreviations are defined.
-  NaClBitcodeSelectorAbbrev()
-      : IsFixed(true),
-        NumBits(NaClBitsNeededForValue(naclbitc::DEFAULT_MAX_ABBREV)) {}
-
-  // Creates a selector range to handle fixed abbrevations up to
-  // the specified value.
-  explicit NaClBitcodeSelectorAbbrev(unsigned MaxAbbrev)
-      : IsFixed(true),
-        NumBits(NaClBitsNeededForValue(MaxAbbrev)) {}
-};
-} // End llvm namespace
-
-#endif
diff --git a/include/llvm/Bitcode/NaCl/NaClBitcodeAbbrevDist.h b/include/llvm/Bitcode/NaCl/NaClBitcodeAbbrevDist.h
deleted file mode 100644
index e9bbc61bab7..00000000000
--- a/include/llvm/Bitcode/NaCl/NaClBitcodeAbbrevDist.h
+++ /dev/null
@@ -1,118 +0,0 @@
-//===-- NaClBitcodeAbbrevDist.h ---------------------------------------------===//
-//      Defines distribution maps for abbreviations associated with
-//      bitcode records.
-//
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
-//
-//===----------------------------------------------------------------------===//
-//
-// Defines distribution maps for tracking abbreviations associated
-// with bitcode records.
-
-#ifndef LLVM_BITCODE_NACL_NACLBITCODEABBREVDIST_H
-#define LLVM_BITCODE_NACL_NACLBITCODEABBREVDIST_H
-
-#include "llvm/Bitcode/NaCl/NaClBitcodeDist.h"
-#include "llvm/Bitcode/NaCl/NaClCompressCodeDist.h"
-
-namespace llvm {
-
-/// Collects the number of instances associated with a given abbreviation
-/// index of a bitcode record. Note: Uses naclbitc::UNABBREV_RECORD index
-/// to denote bitcode records that did not use an abbreviation.
-class NaClBitcodeAbbrevDistElement : public NaClBitcodeDistElement {
-  NaClBitcodeAbbrevDistElement(const NaClBitcodeAbbrevDistElement&) = delete;
-  void operator=(const NaClBitcodeAbbrevDistElement&) = delete;
-
-public:
-  static bool classof(const NaClBitcodeDistElement *Element) {
-    return Element->getKind() >= RDE_AbbrevDist &&
-        Element->getKind() < RDE_AbbrevDistLast;
-  }
-
-  explicit NaClBitcodeAbbrevDistElement(unsigned BlockID=0)
-      : NaClBitcodeDistElement(RDE_AbbrevDist),
-        CodeDist(BlockID, &NaClCompressCodeDistElement::Sentinel) {
-    NestedDists.push_back(&CodeDist);
-  }
-
-  // Sentinel to create instances of this.
-  static NaClBitcodeAbbrevDistElement Sentinel;
-
-  virtual ~NaClBitcodeAbbrevDistElement();
-
-  virtual NaClBitcodeDistElement *CreateElement(
-      NaClBitcodeDistValue Value) const;
-
-  virtual void GetValueList(const NaClBitcodeRecord &Record,
-                            ValueListType &ValueList) const;
-
-  virtual void AddRecord(const NaClBitcodeRecord &Record);
-
-  virtual const char *GetTitle() const;
-
-  virtual const char *GetValueHeader() const;
-
-  virtual void PrintRowValue(raw_ostream &Stream,
-                             NaClBitcodeDistValue Value,
-                             const NaClBitcodeDist *Distribution) const;
-
-  virtual const SmallVectorImpl<NaClBitcodeDist*> *
-  GetNestedDistributions() const;
-
-  NaClBitcodeDist &GetCodeDist() {
-    return CodeDist;
-  }
-
-private:
-  // Nested blocks used by GetNestedDistributions.
-  SmallVector<NaClBitcodeDist*, 1> NestedDists;
-
-  /// The records associated with the given abbreviation.
-  NaClBitcodeCodeDist CodeDist;
-};
-
-/// Separates record codes based on abbreviations. This is done so
-/// that when we add abbreviations, they will be refinements of
-/// existing abbreviations. This should guarantee that we don't lose
-/// separation defined by previous iterations on calls to
-/// pnacl-bccompress.
-class NaClBitcodeAbbrevDist : public NaClBitcodeDist {
-public:
-  static bool classof(const NaClBitcodeDist *Dist) {
-    return Dist->getKind() >= RD_AbbrevDist &&
-        Dist->getKind() < RD_AbbrevDistLast;
-  }
-
-  explicit NaClBitcodeAbbrevDist(
-      unsigned BlockID,
-      NaClBitcodeAbbrevDistElement *Sentinel
-      = &NaClBitcodeAbbrevDistElement::Sentinel,
-      NaClBitcodeDistKind Kind=RD_AbbrevDist)
-      : NaClBitcodeDist(RecordStorage, Sentinel, Kind),
-        BlockID(BlockID) {
-  }
-
-  virtual ~NaClBitcodeAbbrevDist();
-
-  // Returns the block id associated with the abbreviations in this
-  // distribution map.
-  unsigned GetBlockID() const {
-    return BlockID;
-  }
-
-  virtual NaClBitcodeDistElement*
-  CreateElement(NaClBitcodeDistValue Value) const;
-
-private:
-  // The block id associated with the abbreviations in this
-  // distribution map.
-  unsigned BlockID;
-};
-
-}
-
-#endif
diff --git a/include/llvm/Bitcode/NaCl/NaClBitcodeAnalyzer.h b/include/llvm/Bitcode/NaCl/NaClBitcodeAnalyzer.h
deleted file mode 100644
index d93f7aa4071..00000000000
--- a/include/llvm/Bitcode/NaCl/NaClBitcodeAnalyzer.h
+++ /dev/null
@@ -1,66 +0,0 @@
-//===-- NaClBitcodeAnalyzer.h - Bitcode Analyzer --------------------------===//
-//
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
-//
-// Analytical information about a bitcode file. Intended as an aid to developers
-// of bitcode reading and writing software. It produces a summary of the bitcode
-// file that shows various statistics about the contents of the file. By default
-// this information is detailed and contains information about individual
-// bitcode blocks and the functions in the module. The tool is also able to
-// print a bitcode file in a straight forward text format that shows the
-// containment and relationships of the information in the bitcode file (-dump
-// option).
-//
-//===----------------------------------------------------------------------===//
-
-#ifndef NACL_BITCODE_ANALYZER_H
-#define NACL_BITCODE_ANALYZER_H
-
-#include <memory>
-
-namespace llvm {
-
-class MemoryBuffer;
-class StringRef;
-class raw_ostream;
-
-// Analysis options. See the command-line documentation in pnacl-bcanalyzer
-// for a description.
-struct AnalysisDumpOptions {
-  AnalysisDumpOptions()
-      : DumpRecords(false), DumpDetails(false), OpsPerLine(0)
-  {}
-
-  // When true, dump the records. When false, print out distribution
-  // statistics.
-  bool DumpRecords;
-
-  // When true, print out abbreviations, abbreviation ID's, and
-  // other (non-record specific) details when dumping records.
-  bool DumpDetails;
-
-  // The number of record operands to be dumped per text line.
-  unsigned OpsPerLine;
-
-  // When true, prints block statistics based on block ID rather than
-  // size. When false, prints block statistics base on percentage of
-  // file.
-  bool OrderBlocksByID;
-};
-
-/// Run analysis on the given file. Output goes to OS.
-int AnalyzeBitcodeInFile(const StringRef &InputFilename, raw_ostream &OS,
-                         const AnalysisDumpOptions &DumpOptions);
-
-/// Run analysis on a memory buffer with bitcode.  Output goes to
-/// OS. The buffer is owned by the caller.
-int AnalyzeBitcodeInBuffer(const std::unique_ptr<MemoryBuffer> &Buf,
-                           raw_ostream &OS,
-                           const AnalysisDumpOptions &DumpOptions);
-
-} // namespace llvm
-
-#endif
diff --git a/include/llvm/Bitcode/NaCl/NaClBitcodeBitsAndAbbrevsDist.h b/include/llvm/Bitcode/NaCl/NaClBitcodeBitsAndAbbrevsDist.h
deleted file mode 100644
index 13666b9351c..00000000000
--- a/include/llvm/Bitcode/NaCl/NaClBitcodeBitsAndAbbrevsDist.h
+++ /dev/null
@@ -1,69 +0,0 @@
-//===- NaClBitcodeBitsAndAbbrevsDist.h --------------------*- C++ -*-===//
-//     Maps distributions of values with corresponding number of bits,
-//     and percentage of abbreviations used in PNaCl bitcode records.
-//
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
-//
-//===----------------------------------------------------------------------===//
-//
-// Creates a distribution map of values and the
-// correspdonding bits and abbreviations in PNaCl bitcode records.
-
-#ifndef LLVM_BITCODE_NACL_NACLBITCODEBITSANDABBREVSDIST_H
-#define LLVM_BITCODE_NACL_NACLBITCODEBITSANDABBREVSDIST_H
-
-#include "llvm/Bitcode/NaCl/NaClBitcodeBitsDist.h"
-
-namespace llvm {
-
-/// Defines the element type of a PNaCl bitcode distribution map when
-/// we want to count both the number of instances, and the number of
-/// bits used by each record. Also tracks the number to times an
-/// abbreviation was used to parse the corresponding record.
-class NaClBitcodeBitsAndAbbrevsDistElement : public NaClBitcodeBitsDistElement {
-  NaClBitcodeBitsAndAbbrevsDistElement(
-      const NaClBitcodeBitsAndAbbrevsDistElement&) = delete;
-  void operator=(const NaClBitcodeBitsAndAbbrevsDistElement&) = delete;
-
-public:
-  static bool classof(const NaClBitcodeDistElement *Dist) {
-    return Dist->getKind() >= RDE_BitsAndAbbrevsDist
-        && Dist->getKind() < RDE_BitsAndAbbrevsDistLast;
-  }
-
-  // Create an element with no instances.
-  explicit NaClBitcodeBitsAndAbbrevsDistElement(
-      NaClBitcodeDistElementKind Kind=RDE_BitsAndAbbrevsDist)
-      : NaClBitcodeBitsDistElement(Kind),
-        NumAbbrevs(0)
-  {}
-
-  virtual ~NaClBitcodeBitsAndAbbrevsDistElement();
-
-  virtual void AddRecord(const NaClBitcodeRecord &Record);
-
-  // Note: No AddBlock method override because abbrevations only
-  // apply to records.
-
-  // Returns the number of times an abbreviation was used to represent
-  // the value.
-  unsigned GetNumAbbrevs() const {
-    return NumAbbrevs;
-  }
-
-  virtual void PrintStatsHeader(raw_ostream &Stream) const;
-
-  virtual void PrintRowStats(raw_ostream &Stream,
-                             const NaClBitcodeDist *Distribution) const;
-
-private:
-  // Number of times an abbreviation is used for the value.
-  unsigned NumAbbrevs;
-};
-
-}
-
-#endif
diff --git a/include/llvm/Bitcode/NaCl/NaClBitcodeBitsDist.h b/include/llvm/Bitcode/NaCl/NaClBitcodeBitsDist.h
deleted file mode 100644
index 3067bebff11..00000000000
--- a/include/llvm/Bitcode/NaCl/NaClBitcodeBitsDist.h
+++ /dev/null
@@ -1,68 +0,0 @@
-//===- NaClBitcodeBitsDist.h ------------------------------------*- C++ -*-===//
-//     Maps distributions of values and corresponding number of
-//     bits in PNaCl bitcode records.
-//
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
-//
-//===----------------------------------------------------------------------===//
-//
-// Creates a (nestable) distribution map of values, and the correspdonding
-// bits, in PNaCl bitcode records. These distributions are built directly
-// on top of the NaClBitcodeDistElement class.
-
-#ifndef LLVM_BITCODE_NACL_NACLBITCODEBITSDIST_H
-#define LLVM_BITCODE_NACL_NACLBITCODEBITSDIST_H
-
-#include "llvm/Bitcode/NaCl/NaClBitcodeDist.h"
-
-namespace llvm {
-
-/// Defines the element type of a PNaCl bitcode distribution map when
-/// we want to count both the number of instances, and the number of
-/// bits used by each record. Also tracks the number to times an
-/// abbreviation was used to parse the corresponding record.
-class NaClBitcodeBitsDistElement : public NaClBitcodeDistElement {
-  NaClBitcodeBitsDistElement(const NaClBitcodeBitsDistElement&) = delete;
-  void operator=(const NaClBitcodeBitsDistElement&) = delete;
-
-public:
-  static bool classof(const NaClBitcodeDistElement *Dist) {
-    return Dist->getKind() >= RDE_BitsDist
-        && Dist->getKind() < RDE_BitsDistLast;
-  }
-
-  // Create an element with no instances.
-  explicit NaClBitcodeBitsDistElement(
-      NaClBitcodeDistElementKind Kind=RDE_BitsDist)
-      : NaClBitcodeDistElement(Kind),
-        TotalBits(0)
-  {}
-
-  virtual ~NaClBitcodeBitsDistElement();
-
-  virtual void AddRecord(const NaClBitcodeRecord &Record);
-
-  virtual void AddBlock(const NaClBitcodeBlock &Block);
-
-  // Returns the total number of bits used to represent all instances
-  // of this value.
-  uint64_t GetTotalBits() const {
-    return TotalBits;
-  }
-
-  virtual void PrintStatsHeader(raw_ostream &Stream) const;
-
-  virtual void PrintRowStats(raw_ostream &Stream,
-                             const NaClBitcodeDist *Distribution) const;
-
-private:
-  // Number of bits used to represent all instances of the value.
-  uint64_t TotalBits;
-};
-
-}
-
-#endif
diff --git a/include/llvm/Bitcode/NaCl/NaClBitcodeBlockDist.h b/include/llvm/Bitcode/NaCl/NaClBitcodeBlockDist.h
deleted file mode 100644
index e74a40564ca..00000000000
--- a/include/llvm/Bitcode/NaCl/NaClBitcodeBlockDist.h
+++ /dev/null
@@ -1,95 +0,0 @@
-//===-- NaClBitcodeBlockDist.h -----------------------------------------===//
-//      Defines distribution maps for blocks within PNaCl bitcode.
-//
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
-//
-//===----------------------------------------------------------------------===//
-//
-// Defines a distribution map for blocks which tracks the number of bits
-// in each block, as well as the percentage of the file each bitcode block
-// ID holds.
-
-
-#ifndef LLVM_BITCODE_NACL_NACLBITCODEBLOCKDIST_H
-#define LLVM_BITCODE_NACL_NACLBITCODEBLOCKDIST_H
-
-#include "llvm/Bitcode/NaCl/NaClBitcodeBitsDist.h"
-
-namespace llvm {
-
-class NaClBitcodeBlockDistElement : public NaClBitcodeBitsDistElement {
-  NaClBitcodeBlockDistElement(const NaClBitcodeBlockDistElement&) = delete;
-  void operator=(const NaClBitcodeBlockDistElement&);
-
-public:
-  static bool classof(const NaClBitcodeDistElement *Element) {
-    return Element->getKind() >= RDE_BlockDist &&
-        Element->getKind() < RDE_BlockDistLast;
-  }
-
-  // Top-level constructor to create instances of this class.
-  explicit NaClBitcodeBlockDistElement(
-      NaClBitcodeDistElementKind Kind=RDE_BlockDist)
-      : NaClBitcodeBitsDistElement(Kind) {}
-
-  virtual ~NaClBitcodeBlockDistElement();
-
-  virtual NaClBitcodeDistElement *CreateElement(
-      NaClBitcodeDistValue Value) const;
-
-  // Sorts by %file, rather than number of instances.
-  virtual double GetImportance(NaClBitcodeDistValue value) const;
-
-  virtual const char *GetTitle() const;
-
-  virtual const char *GetValueHeader() const;
-
-  /// Prints out header for row of statistics associated with instances
-  /// of this distribution element.
-  virtual void PrintStatsHeader(raw_ostream &Stream) const;
-
-  /// Prints out statistics for the row with the given value.
-  virtual void PrintRowStats(raw_ostream &Stream,
-                             const NaClBitcodeDist *Distribution) const;
-
-  virtual void PrintRowValue(raw_ostream &Stream,
-                             NaClBitcodeDistValue Value,
-                             const NaClBitcodeDist *Distribution) const;
-};
-
-class NaClBitcodeBlockDist : public NaClBitcodeDist {
-  NaClBitcodeBlockDist(const NaClBitcodeBlockDist&) = delete;
-  void operator=(const NaClBitcodeBlockDist&) = delete;
-
-public:
-  static bool classof(const NaClBitcodeDist *Dist) {
-    return Dist->getKind() >= RD_BlockDist &&
-        Dist->getKind() < RD_BlockDistLast;
-  }
-
-  static NaClBitcodeBlockDistElement DefaultSentinal;
-
-  explicit NaClBitcodeBlockDist(
-      const NaClBitcodeBlockDistElement *Sentinal=&DefaultSentinal)
-      : NaClBitcodeDist(BlockStorage, Sentinal, RD_BlockDist)
-  {}
-
-  virtual ~NaClBitcodeBlockDist();
-
-  // Returns the total number of bits in all blocks in the distribution.
-  uint64_t GetTotalBits() const;
-
-  // Returns the printable name associated with the given BlockID.
-  //
-  // Note: If the name is not known, an "UnknownBlock" name is
-  // generated and returned.
-  static std::string GetName(unsigned BlockID);
-
-};
-
-}
-
-#endif
diff --git a/include/llvm/Bitcode/NaCl/NaClBitcodeCodeDist.h b/include/llvm/Bitcode/NaCl/NaClBitcodeCodeDist.h
deleted file mode 100644
index 6f794116482..00000000000
--- a/include/llvm/Bitcode/NaCl/NaClBitcodeCodeDist.h
+++ /dev/null
@@ -1,111 +0,0 @@
-//===-- NaClBitcodeCodeDist.h ---------------------------------------------===//
-//      Defines distribution maps for various values in bitcode records.
-//
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
-//
-//===----------------------------------------------------------------------===//
-//
-// Defines simple (non-nested) distribution maps for record codes
-// appearing in bitcode records (instances of class NaClBitcodeRecord
-// in NaClBitcodeParser.h).
-
-#ifndef LLVM_BITCODE_NACL_NACLBITCODECODEDIST_H
-#define LLVM_BITCODE_NACL_NACLBITCODECODEDIST_H
-
-#include "llvm/Bitcode/NaCl/NaClBitcodeBitsAndAbbrevsDist.h"
-
-namespace llvm {
-
-// Collects the distribution of record codes/number of bits used for a
-// particular blockID and Code ID.
-class NaClBitcodeCodeDistElement
-    : public NaClBitcodeBitsAndAbbrevsDistElement {
-  NaClBitcodeCodeDistElement(const NaClBitcodeCodeDistElement&)
-      = delete;
-  void operator=(const NaClBitcodeCodeDistElement&)
-      = delete;
-
-public:
-  static bool classof(const NaClBitcodeDistElement *Element) {
-    return Element->getKind() >= RDE_CodeDist
-        && Element->getKind() < RDE_CodeDistLast;
-  }
-
-  explicit NaClBitcodeCodeDistElement(
-      NaClBitcodeDistElementKind Kind=RDE_CodeDist)
-      : NaClBitcodeBitsAndAbbrevsDistElement(Kind)
-  {}
-
-  virtual ~NaClBitcodeCodeDistElement();
-
-  virtual NaClBitcodeDistElement *CreateElement(
-      NaClBitcodeDistValue Value) const;
-
-  virtual void GetValueList(const NaClBitcodeRecord &Record,
-                            ValueListType &ValueList) const;
-
-  virtual const char *GetTitle() const;
-
-  virtual const char *GetValueHeader() const;
-
-  virtual void PrintRowValue(raw_ostream &Stream,
-                             NaClBitcodeDistValue Value,
-                             const NaClBitcodeDist *Distribution) const;
-};
-
-// Collects the distribution of record codes/number of bits used for a
-// particular blockID. Assumes distribution elements are instances of
-// NaClBitcodeCodeDistElement.
-class NaClBitcodeCodeDist : public NaClBitcodeDist {
-  NaClBitcodeCodeDist(const NaClBitcodeCodeDist&) = delete;
-  void operator=(const NaClBitcodeCodeDist&) = delete;
-
-public:
-  static bool classof(const NaClBitcodeDist *Dist) {
-    return Dist->getKind() >= RD_CodeDist
-        && Dist->getKind() < RD_CodeDistLast;
-  }
-
-protected:
-  NaClBitcodeCodeDist(NaClBitcodeDistElement *Sentinal,
-                      unsigned BlockID,
-                      NaClBitcodeDistKind Kind=RD_CodeDist)
-      : NaClBitcodeDist(RecordStorage, Sentinal, Kind), BlockID(BlockID)
-  {}
-
-  static NaClBitcodeCodeDistElement DefaultSentinel;
-
-public:
-  explicit NaClBitcodeCodeDist(
-      unsigned BlockID,
-      NaClBitcodeCodeDistElement *Sentinel = &DefaultSentinel,
-      NaClBitcodeDistKind Kind=RD_CodeDist)
-      : NaClBitcodeDist(RecordStorage, Sentinel, Kind),
-        BlockID(BlockID)
-  {}
-
-  virtual ~NaClBitcodeCodeDist();
-
-  unsigned GetBlockID() const {
-    return BlockID;
-  }
-
-  // Returns the printable name for record code CodeID in blocks
-  // associated with BlockID.
-  //
-  // Note: If the name is not known, an "UnknownCode" name is
-  // generated and return.
-  static std::string GetCodeName(unsigned CodeID, unsigned BlockID);
-
-private:
-  // The blockID associated with the record code distribution.
-  // Used so that we can look up the print name for each record code.
-  unsigned BlockID;
-};
-
-}
-
-#endif
diff --git a/include/llvm/Bitcode/NaCl/NaClBitcodeDecoders.h b/include/llvm/Bitcode/NaCl/NaClBitcodeDecoders.h
deleted file mode 100644
index a3250e0c221..00000000000
--- a/include/llvm/Bitcode/NaCl/NaClBitcodeDecoders.h
+++ /dev/null
@@ -1,74 +0,0 @@
-//===- NaClBitcodeDecoders.h -------------------------------------*- C++ -*-===//
-//     Functions used to decode values in PNaCl bitcode files.
-//
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
-//
-//===----------------------------------------------------------------------===//
-//
-// This header file provides a public API for value decoders defined in
-// the PNaCl bitcode reader.
-//
-//===----------------------------------------------------------------------===//
-
-#ifndef LLVM_BITCODE_NACL_NACLBITCODEDECODERS_H
-#define LLVM_BITCODE_NACL_NACLBITCODEDECODERS_H
-
-#include "llvm/IR/CallingConv.h"
-#include "llvm/IR/GlobalValue.h"
-#include "llvm/IR/InstrTypes.h"
-#include "llvm/IR/Instruction.h"
-#include "llvm/Bitcode/NaCl/NaClLLVMBitCodes.h"
-
-namespace llvm {
-namespace naclbitc {
-
-/// Converts the NaCl (bitcode file) cast opcode to the corresponding
-/// LLVM cast opcode. Returns true if the conversion
-/// succeeds. Otherwise sets LLVMOpcode to Instruction::BitCast and
-/// returns false.
-bool DecodeCastOpcode(uint64_t NaClOpcode,
-                      Instruction::CastOps &LLVMOpcode);
-
-/// Converts the NaCl (bitcode file) linkage type to the corresponding
-/// LLVM linkage type. Returns true if the conversion
-/// succeeds. Otherwise sets LLVMLinkage to
-/// GlobalValue::InternalLinkage and returns false.
-bool DecodeLinkage(uint64_t NaClLinkage,
-                   GlobalValue::LinkageTypes &LLVMLinkage);
-
-/// Converts the NaCl (bitcode file) binary opcode to the
-/// corresponding LLVM binary opcode, assuming that the operator
-/// operates on OpType.  Returns true if the conversion
-/// succeeds. Otherwise sets LLVMOpcode to Instruction::Add and
-/// returns false.
-bool DecodeBinaryOpcode(uint64_t NaClOpcode, Type *OpType,
-                        Instruction::BinaryOps &LLVMOpcode);
-
-/// Converts the NaCl (bitcode file) calling convention value to the
-/// corresponding LLVM calling conventions. Returns true if the
-/// conversion succeeds. Otherwise sets LLVMCallingConv to
-/// CallingConv::C and returns false.
-bool DecodeCallingConv(uint64_t NaClCallingConv,
-                       CallingConv::ID &LLVMCallingConv);
-
-/// Converts the NaCl (bitcode file) float comparison predicate to the
-/// corresponding LLVM float comparison predicate. Returns true if the
-/// conversion succeeds. Otherwise sets LLVMPredicate to
-/// CmpInst::FCMP_FALSE and returns false.
-bool DecodeFcmpPredicate(uint64_t NaClPredicate,
-                         CmpInst::Predicate &LLVMPredicate);
-
-/// Converts the NaCl (bitcode file) integer comparison predicate to
-/// the corresponding LLVM integer comparison predicate. Returns true
-/// if the conversion succeeds. Otherwise sets LLVMPredicate to
-/// CmpInst::ICMP_EQ and returns false.
-bool DecodeIcmpPredicate(uint64_t NaClPredicate,
-                         CmpInst::Predicate &LLVMPredicate);
-
-}
-}
-
-#endif
diff --git a/include/llvm/Bitcode/NaCl/NaClBitcodeDefs.h b/include/llvm/Bitcode/NaCl/NaClBitcodeDefs.h
deleted file mode 100644
index bb1792ff0c1..00000000000
--- a/include/llvm/Bitcode/NaCl/NaClBitcodeDefs.h
+++ /dev/null
@@ -1,53 +0,0 @@
-//===- NaClBitcodeDefs.h ----------------------------------------*- C++ -*-===//
-//
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
-//
-//===----------------------------------------------------------------------===//
-//
-// Defines some common types/constants used by bitcode readers and
-// writers. It is intended to make clear assumptions made in
-// representing bitcode files.
-//
-//===----------------------------------------------------------------------===//
-
-#ifndef LLVM_BITCODE_NACL_NACLBITCODEDEFS_H
-#define LLVM_BITCODE_NACL_NACLBITCODEDEFS_H
-
-namespace llvm {
-
-namespace naclbitc {
-
-// Special record codes used to model codes for predefined records.
-// They are very large so that they do not conflict with existing
-// record codes for user-defined blocks.
-enum SpecialBlockCodes {
-  BLK_CODE_ENTER = 65535,
-  BLK_CODE_EXIT  = 65534,
-  BLK_CODE_DEFINE_ABBREV = 65533,
-  BLK_CODE_HEADER = 65532
-};
-
-} // end of namespace naclbitc
-
-/// Defines type for value indicies in bitcode. Like a size_t, but
-/// fixed across platforms.
-typedef uint32_t NaClBcIndexSize_t;
-
-/// Signed version of NaClBcIndexSize_t. Used to define relative indices.
-typedef int32_t NaClRelBcIndexSize_t;
-
-/// Defines maximum allowed bitcode index in bitcode files.
-static const size_t NaClBcIndexSize_t_Max =
-    std::numeric_limits<NaClBcIndexSize_t>::max();
-
-/// Defines the maximum number of initializers allowed, based on ILP32.
-static const size_t MaxNaClGlobalVarInits =
-    std::numeric_limits<uint32_t>::max();
-
-} // end of namespace llvm
-
-
-#endif // LLVM_BITCODE_NACL_NACLBITCODEDEFS_H
diff --git a/include/llvm/Bitcode/NaCl/NaClBitcodeDist.h b/include/llvm/Bitcode/NaCl/NaClBitcodeDist.h
deleted file mode 100644
index 60ecabf4965..00000000000
--- a/include/llvm/Bitcode/NaCl/NaClBitcodeDist.h
+++ /dev/null
@@ -1,507 +0,0 @@
-//===- NaClBitcodeDist.h ---------------------------------------*- C++ -*-===//
-//     Maps distributions of values in PNaCl bitcode files.
-//
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
-//
-//===----------------------------------------------------------------------===//
-//
-// Creates a (nestable) distribution map of values in PNaCl bitcode.
-// The domain of these maps is the set of record values being
-// tracked. The range is the information associated with each block
-// and/or record value, including the number of instances of that value. The
-// distribution map is nested if the range element contains another
-// distribution map.
-//
-// The goal of distribution maps is to build a (histogram)
-// distribution of values in bitcode records and blocks, of a PNaCl
-// bitcode file. From appropriately built distribution maps, one can
-// infer possible new abbreviations that can be used in the PNaCl
-// bitcode file.  Hence, one of the primary goals of distribution maps
-// is to support tools pnacl-bcanalyzer and pnacl-bccompress.
-//
-// Distribution maps are constructed either from NaClBitcodeBlock's or
-// NaClBitcodeRecord's (in NaClBitcodeParser.h), but not both. It is
-// assumed that only blocks, or records (but not both) are added to a
-// distribution map. To add data to the distribution map, one calls
-// AddRecord and/or AddBlock. If the distribution map contains record
-// values, you must call AddRecord for each record to be put into the
-// distribution map. If the distribution map contains block values (i.e.
-// block ID's), you must call AddBlock for each block to be put into
-// the distribution map.
-//
-// While it may counterintuitive, one can call both AddRecord and
-// AddBlock, for each corresponding record and block processed. The
-// reason for this is that an internal flag StorageKind is kept within
-// distribution maps.  If the flag value doesn't correspond to the
-// type of called add method, no update is done. This behaviour is
-// done so that nested distribution maps can be updated via blind
-// calls in NaClBitcodeAnalyzer.cpp.
-//
-// Via inheritance, and overriding the (virtual) AddRecord
-// method for a distribution map, we can redirect the add to look up
-// the distribution element associated with the block of the record,
-// and then update the corresponding record distribution map. In general,
-// it only makes sense (for nested distribution maps) to be able to
-// redirect record additions. Redirecting blocks within a record (since
-// a record is only associated with one block) does not make sense. Hence,
-// we have not made AddBlock virtual.
-//
-// When updating a block distribution map, the domain value is the
-// BlockID of the corresponding block being added.
-//
-// On the other hand, values associated with record distribution maps
-// are many possible values (the code, the abbreviation, the values
-// etc). To make the API uniform, record distribution maps are updated
-// using NaClBitcodeRecords (in NaClBitcodeParser.h). The values from
-// the record are defined by the extract method GetValueList, and
-// added via method AddRecord.
-//
-// Distribution maps are implemented using two classes:
-//
-//  NaClBitcodeDist
-//     A generic distribution map.
-//
-//  NaClBitcodeDistElement
-//     The implementation of elements in the range of the distribution
-//     map.
-//
-// The code has been written to put the (virtual) logic of
-// distribution maps into derived classes of NaClBitcodeDistElement
-// whenever possible. This is intentional, in that it keeps all
-// knowledge of how to handle/print elements in one class. However,
-// because some distributions have external data that is needed by all
-// elements, the virtual methods of class NaClBitcodeDist can be
-// overridden, and not delegate to NaClBitcodeDistElement.
-//
-// To do this, an NaClBitcodeDist requires a "sentinel" (derived)
-// instance of NaClBitcodeDistElement. This sentinel is used to define
-// behaviour needed by distribution maps.
-//
-// By having control passed to the (derived) instances of
-// NaClBitcodeDistElement, it also makes handling nested distributions
-// relatively easy. One just extends the methods AddRecord and/or
-// AddBlock to also update the corresponding nested distribution.
-//
-// The exception to this rule is printing, since header information is
-// dependent on properties of any possible nested distribution maps
-// (for example, we copy column headers after each nested distribution
-// map so that it is easier to read the output). To fix this, we let
-// virtual method NaClBitcodeDistElement::GetNestedDistributions
-// return the array of nested distribution pointers for the nested
-// distributions of that map. The order in the array is the order the
-// nested distributions will be printed. Typically, this is
-// implemented as a field of the distribution element, and is
-// initialized to contain the pointers of all nested distributions in
-// the element. This field can then be returned from method
-// GetNestedDistributions.
-//
-// Distribution maps are sortable (via method GetDistribution). The
-// purpose of sorting is to find interesting elements. This is done by
-// sorting the values in the domain of the distribution map, based on
-// the GetImportance method of the range element.
-//
-// Method GetImportance defines how (potentially) interesting the
-// value is in the distribution. "Interesting" is based on the notion
-// of how likely will the value show a case where adding an
-// abbreviation will shrink the size of the corresponding bitcode
-// file. For most distributions, the number of instances associated
-// with the value is the best measure.
-//
-// However, for cases where multiple domain entries are created for
-// the same NaClBitcodeRecord (i.e. method GetValueList defines more
-// than one value), things are not so simple.
-
-// For example, for some distributions (such as value index distributions)
-// the numbers of instances isn't sufficient. In such cases, you may
-// have to look at nested distributions to find important cases.
-//
-// In the case of value index distributions, when the size of the
-// records is the same, all value indices have the same number of
-// instances.  In this case, "interesting" may be measured in terms of
-// the (nested) distribution of the values that can appear at that
-// index, and how often each value appears.
-//
-// The larger the importance, the more interesting the value is
-// considered, and sorting is based on moving interesting values to
-// the front of the sorted list.
-//
-// When printing distribution maps, the values are sorted based on
-// the importance. By default, importance is based on the number of
-// times the value appears in records, putting the most used values
-// at the top of the printed list.
-//
-// Since sorting is expensive, the sorted distribution is built once
-// and cached.  This cache is flushed whenever the distribution map is
-// updated, so that a new sorted distribuition will be generated.
-//
-// Printing of distribution maps are stylized, so that virtuals can
-// easily fill in the necessary data.
-//
-// For concrete instances of NaClBitcodeDistElement, the following
-// virtual method must be defined:
-//
-//    CreateElement
-//       Creates a new instance of the distribution element, to
-//       be put into the corresponding distribution map when a new
-//       value is added to the distribution map.
-//
-// In addition, if the distribution element is based on record values,
-// the virtual method GetValueList must be defined, to extract values
-// out of the bitcode record.
-
-#ifndef LLVM_BITCODE_NACL_NACLBITCODEDIST_H
-#define LLVM_BITCODE_NACL_NACLBITCODEDIST_H
-
-#include "llvm/Bitcode/NaCl/NaClBitcodeParser.h"
-#include "llvm/Support/Casting.h"
-#include "llvm/Support/Format.h"
-#include "llvm/Support/raw_ostream.h"
-
-#include <map>
-#include <vector>
-
-namespace llvm {
-
-/// The domain type of PNaCl bitcode record distribution maps.
-typedef uint64_t NaClBitcodeDistValue;
-
-/// Base class of the range type of PNaCl bitcode record distribution
-/// maps.
-class NaClBitcodeDistElement;
-
-/// Type defining the list of values extracted from the corresponding
-/// bitcode record. Typically, the list size is one. However, there
-/// are cases where a record defines more than one value (i.e. value
-/// indices). Hence, this type defines the more generic API for
-/// values.
-typedef std::vector<NaClBitcodeDistValue> ValueListType;
-
-typedef ValueListType::const_iterator ValueListIterator;
-
-/// Defines a PNaCl bitcode record distribution map. The distribution
-/// map is a map from a (record) value, to the corresponding data
-/// associated with that value. Assumes distributions elements are
-/// instances of NaClBitcodeDistElement.
-class NaClBitcodeDist {
-  NaClBitcodeDist(const NaClBitcodeDist&) = delete;
-  void operator=(const NaClBitcodeDist&) = delete;
-  friend class NaClBitcodeDistElement;
-
-public:
-  /// Define kinds for isa, dyn_cast, etc. support (see
-  /// llvm/Support/Casting.h). Only defined for concrete classes.
-  enum NaClBitcodeDistKind {
-    RD_Dist,                 // class NaClBitcodeDist.
-      RD_BlockDist,          // class NaClBitcodeBlockDist.
-      RD_BlockDistLast,
-      RD_CodeDist,           // class NaClBitcodeCodeDist.
-      RD_CodeDistLast,
-      RD_AbbrevDist,         // class NaClBitcodeAbbrevDist.
-      RD_AbbrevDistLast,
-      RD_SubblockDist,       // class NaClBlockSubblockDist.
-      RD_SubblockDistLast,
-      RD_ValueDist,          // class NaClBitcodeValueDist.
-      RD_ValueDistLast,
-    RD_DistLast
-  };
-
-  NaClBitcodeDistKind getKind() const { return Kind; }
-
-private:
-  const NaClBitcodeDistKind Kind;
-
-  static bool classof(const NaClBitcodeDist *Dist) {
-    return Dist->getKind() >= RD_Dist && Dist->getKind() < RD_DistLast;
-  }
-
-public:
-  /// Type defining the mapping used to define the distribution.
-  typedef std::map<NaClBitcodeDistValue, NaClBitcodeDistElement*> MappedElement;
-
-  typedef MappedElement::const_iterator const_iterator;
-
-  /// Type defining a pair of values used to sort the
-  /// distribution. The first element is defined by method
-  /// GetImportance, and the second is the distribution value
-  /// associated with that importance.
-  typedef std::pair<double, NaClBitcodeDistValue> DistPair;
-
-  /// Type defining the sorted list of (domain) values in the
-  /// corresponding distribution map.
-  typedef std::vector<DistPair> Distribution;
-
-  /// Defines whether blocks or records are stored in the distribution map.
-  /// Used to decide if AddRecord/AddBlock methods should fire.
-  enum StorageSelector {
-    BlockStorage,
-    RecordStorage
-  };
-
-  NaClBitcodeDist(StorageSelector StorageKind,
-                  const NaClBitcodeDistElement *Sentinel,
-                  NaClBitcodeDistKind Kind=RD_Dist)
-      : Kind(Kind), StorageKind(StorageKind), Sentinel(Sentinel),
-        TableMap(), CachedDistribution(0), Total(0) {
-  }
-
-  virtual ~NaClBitcodeDist();
-
-  /// Number of elements in the distribution map.
-  size_t size() const {
-    return TableMap.size();
-  }
-
-  /// Iterator at beginning of distribution map.
-  const_iterator begin() const {
-    return TableMap.begin();
-  }
-
-  /// Iterator at end of distribution map.
-  const_iterator end() const {
-    return TableMap.end();
-  }
-
-  /// Returns true if the distribution map is empty.
-  bool empty() const {
-    return TableMap.empty();
-  }
-
-  /// Returns the element associated with the given distribution
-  /// value.  Creates the element if needed.
-  inline NaClBitcodeDistElement *GetElement(NaClBitcodeDistValue Value);
-
-  /// Returns the element associated with the given distribution
-  /// value.
-  NaClBitcodeDistElement *at(NaClBitcodeDistValue Value) const {
-    return TableMap.at(Value);
-  }
-
-  // Creates a new instance of this element for the given value. Used
-  // by class NaClBitcodeDist to create instances. Default method
-  // simply dispatches to the CreateElement method of the sentinel.
-  virtual NaClBitcodeDistElement *CreateElement(
-      NaClBitcodeDistValue Value) const;
-
-  /// Interrogates the block record, and returns the corresponding
-  /// values that are being tracked by the distribution map.  Default
-  /// method simply dispatches to the GetValueList of the sentinel.
-  virtual void GetValueList(const NaClBitcodeRecord &Record,
-                            ValueListType &ValueList) const;
-
-  /// Returns the total number of instances held in the distribution
-  /// map.
-  unsigned GetTotal() const {
-    return Total;
-  }
-
-  /// Adds the value(s) in the given bitcode record to the
-  /// distribution map.  The value(s) based on method GetValueList.
-  /// Note: Default requires that GetStorageKind() == RecordStorage.
-  /// Override if you want special handling for nested distributions
-  /// in a block distribution map.
-  virtual void AddRecord(const NaClBitcodeRecord &Record);
-
-  /// Adds the BlockID of the given bitcode block to the distribution
-  /// map, if applicable (based on the value of method UseBlockID).
-  /// Note: Requires that GetStorageKind() == BlockStorage.
-  virtual void AddBlock(const NaClBitcodeBlock &Block);
-
-  /// Builds the distribution associated with the distribution map.
-  /// Warning: The distribution is cached, and hence, only valid while
-  /// it's contents is not changed.
-  const Distribution *GetDistribution() const {
-    if (CachedDistribution == 0) Sort();
-    return CachedDistribution;
-  }
-
-  /// Prints out the contents of the distribution map to Stream.
-  void Print(raw_ostream &Stream, const std::string &Indent) const;
-
-  void Print(raw_ostream &Stream) const {
-    std::string Indent;
-    Print(Stream, Indent);
-  }
-
-protected:
-  /// If the distribution is cached, remove it. Should be called
-  /// whenever the distribution map is changed.
-  void RemoveCachedDistribution() const {
-    if (CachedDistribution) {
-      delete CachedDistribution;
-      CachedDistribution = 0;
-    }
-  }
-
-  /// Sorts the distribution, based on the importance of each element.
-  void Sort() const;
-
-private:
-  // Defines whether values in distribution map are from blocks or records.
-  const StorageSelector StorageKind;
-  // Sentinel element used to do generic operations for distribution.
-  const NaClBitcodeDistElement *Sentinel;
-  // Map from the distribution value to the corresponding distribution
-  // element.
-  MappedElement TableMap;
-  // Pointer to the cached distribution.
-  mutable Distribution *CachedDistribution;
-  // The total number of instances in the map.
-  unsigned Total;
-};
-
-/// Defines the element type of a PNaCl bitcode distribution map.
-/// This is the base class for all element types used in
-/// NaClBitcodeDist.  By default, only the number of instances
-/// of the corresponding distribution values is recorded.
-class NaClBitcodeDistElement {
-  NaClBitcodeDistElement(const NaClBitcodeDistElement &)
-      = delete;
-  void operator=(const NaClBitcodeDistElement &)
-      = delete;
-
-public:
-  /// Define kinds for isa, dyn_cast, etc. support. Only defined
-  /// for concrete classes.
-  enum NaClBitcodeDistElementKind {
-    RDE_Dist,                    // class NaClBitcodeDistElement.
-      RDE_AbbrevDist,            // class NaClBitcodeAbbrevDistElement.
-      RDE_AbbrevDistLast,
-      RDE_BitsDist,              // class NaClBitcodeBitsDistElement.
-        RDE_BitsAndAbbrevsDist,  // class NaClBitcodeBitsAndAbbrevsDistElement.
-          RDE_CodeDist,          // class NaClBitcodeCodeDistElement.
-            RDE_CompressCodeDist, // class NaClCompressCodeDistElement.
-            RDE_CompressCodeDistLast,
-          RDE_CodeDistLast,
-        RDE_BitsAndAbbrevsDistLast,
-      RDE_BitsDistLast,
-      RDE_BlockDist,             // class NaClBitcodeBlockDistElement.
-        RDE_NaClAnalBlockDist,   // class NaClAnalyzerBlockDistElement.
-        RDE_NaClAnalBlockDistLast,
-        RDE_PNaClCompressBlockDist, // class NaClCompressBlockDistElement.
-        RDE_PNaClCompressBlockDistLast,
-      RDE_BlockDistLast,
-      RDE_SizeDist,              // class NaClBitcodeSizeDistElement.
-      RDE_SizeDistLast,
-      RDE_SubblockDist,          // class NaClBitcodeSubblockDistElement
-      RDE_SubblockDistLast,
-      RDE_ValueDist,             // class NaClBitcodeValueDistElement.
-      RDE_ValueDistLast,
-      RDE_ValueIndexDist,        // class NaClBitcodeValueIndexDistElement.
-      RDE_ValueIndexDistLast,
-    RDE_DistLast
-  };
-
-  NaClBitcodeDistElementKind getKind() const { return Kind; }
-
-  static bool classof(const NaClBitcodeDistElement *Element) {
-    return Element->getKind() >= RDE_Dist && Element->getKind() < RDE_DistLast;
-  }
-
-private:
-  const NaClBitcodeDistElementKind Kind;
-
-public:
-
-  // Constructor to use in derived classes.
-  NaClBitcodeDistElement(
-      NaClBitcodeDistElementKind Kind=RDE_Dist)
-      : Kind(Kind), NumInstances(0)
-  {}
-
-  virtual ~NaClBitcodeDistElement();
-
-  // Adds an instance of the given record to this element.
-  virtual void AddRecord(const NaClBitcodeRecord &Record);
-
-  // Adds an instance of the given block to this element.
-  virtual void AddBlock(const NaClBitcodeBlock &Block);
-
-  // Returns the number of instances associated with this element.
-  unsigned GetNumInstances() const {
-    return NumInstances;
-  }
-
-  // Creates a new instance of this element for the given value. Used
-  // by class NaClBitcodeDist to create instances.
-  virtual NaClBitcodeDistElement *CreateElement(
-      NaClBitcodeDistValue Value) const = 0;
-
-  /// Interrogates the block record, and returns the corresponding
-  /// values that are being tracked by the distribution map. Must be
-  /// defined in derived classes.
-  virtual void GetValueList(const NaClBitcodeRecord &Record,
-                            ValueListType &ValueList) const;
-
-  // Returns the importance of this element. In many cases, it will be
-  // the number of instances associated with it. However, it need not
-  // be correlated to the number of instance. Used to sort the
-  // distribution map, where values with larger importance appear
-  // first. Value is the domain value associated with the element in
-  // the distribution map.
-  virtual double GetImportance(NaClBitcodeDistValue Value) const;
-
-  /// Returns the title to use when printing the title associated
-  /// with instances of this distribution element.
-  virtual const char *GetTitle() const;
-
-  /// Prints out the title of the distribution map associated with
-  /// instances of this distribution element.
-  virtual void PrintTitle(raw_ostream &Stream,
-                          const NaClBitcodeDist *Distribution) const;
-
-  /// Returns the header to use when printing the value associated
-  /// with instances of this distribution element.
-  virtual const char *GetValueHeader() const;
-
-  /// Prints out header for row of statistics associated with instances
-  /// of this distribution element.
-  virtual void PrintStatsHeader(raw_ostream &Stream) const;
-
-  /// Prints out the header to the printed distribution map associated
-  /// with instances of this distribution element.
-  void PrintHeader(raw_ostream &Stream) const;
-
-  /// Prints out statistics for the row with the given value.
-  virtual void PrintRowStats(raw_ostream &Stream,
-                             const NaClBitcodeDist *Distribution) const;
-
-  /// Prints out Value (in a row) to Stream.
-  virtual void PrintRowValue(raw_ostream &Stream,
-                             NaClBitcodeDistValue Value,
-                             const NaClBitcodeDist *Distribution) const;
-
-  /// Prints out a row in the printed distribution map.
-  virtual void PrintRow(raw_ostream &Stream,
-                        NaClBitcodeDistValue Value,
-                        const NaClBitcodeDist *Distribution) const;
-
-  /// Returns a pointer to the list of nested distributions that
-  /// should be printed when this element is printed. Return 0 if no
-  /// nested distributions should be printed.
-  virtual const SmallVectorImpl<NaClBitcodeDist*> *
-  GetNestedDistributions() const;
-
-  /// Prints out any nested distributions, if defined for the element.
-  /// Returns true if a nested distribution was printed.
-  bool PrintNestedDistIfApplicable(
-      raw_ostream &Stream, const std::string &Indent) const;
-
-private:
-  // The number of instances associated with this element.
-  unsigned NumInstances;
-};
-
-inline NaClBitcodeDistElement *NaClBitcodeDist::
-GetElement(NaClBitcodeDistValue Value) {
-  if (TableMap.find(Value) == TableMap.end()) {
-    TableMap[Value] = CreateElement(Value);
-  }
-  return TableMap[Value];
-}
-
-}
-
-#endif
diff --git a/include/llvm/Bitcode/NaCl/NaClBitcodeHeader.h b/include/llvm/Bitcode/NaCl/NaClBitcodeHeader.h
deleted file mode 100644
index 50ca5aaed86..00000000000
--- a/include/llvm/Bitcode/NaCl/NaClBitcodeHeader.h
+++ /dev/null
@@ -1,263 +0,0 @@
-//===-- llvm/Bitcode/NaCl/NaClBitcodeHeader.h - ----------------*- C++ -*-===//
-//      NaCl Bitcode header reader.
-//
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
-//
-//===----------------------------------------------------------------------===//
-//
-// This header defines interfaces to read and write NaCl bitcode wire format
-// file headers.
-//
-//===----------------------------------------------------------------------===//
-
-#ifndef LLVM_BITCODE_NACL_NACLBITCODEHEADER_H
-#define LLVM_BITCODE_NACL_NACLBITCODEHEADER_H
-
-#include "llvm/ADT/StringRef.h"
-#include "llvm/Support/Compiler.h"
-#include "llvm/Support/DataTypes.h"
-#include <string>
-#include <vector>
-
-namespace llvm {
-class MemoryObject;
-
-// Class representing a variable-size metadata field in the bitcode header.
-// Also contains the list of known (typed) Tag IDs.
-//
-// The serialized format has 2 fixed subfields (ID:type and data length) and the
-// variable-length data subfield
-class NaClBitcodeHeaderField {
-  NaClBitcodeHeaderField(const NaClBitcodeHeaderField &) = delete;
-  void operator=(const NaClBitcodeHeaderField &) = delete;
-
-public:
-  // Defines the ID associated with the value. Valid values are in
-  // {0x0, ..., 0xFFF}
-  typedef enum {
-    kInvalid = 0,             // KUnknownType.
-    kPNaClVersion = 1,        // kUint32Type.
-    kAlignBitcodeRecords = 2, // kFlagType.
-    kTag_MAX = kAlignBitcodeRecords
-  } Tag;
-  // Defines the type of value.
-  typedef enum {
-    kBufferType, // Buffer of form uint8_t[len].
-    kUInt32Type,
-    kFlagType,
-    kUnknownType,
-    kFieldType_MAX = kUnknownType
-  } FieldType;
-  // Defines the number of bytes in a (32-bit) word.
-  static const int WordSize = 4;
-
-  // Defines the encoding of the fixed fields {i.e. ID:type and data length).
-  typedef uint16_t FixedSubfield;
-
-  // Create an invalid header field.
-  NaClBitcodeHeaderField();
-
-  // Creates a header field where MyID is a flag.
-  NaClBitcodeHeaderField(Tag MyID);
-
-  // Create a header field with an uint32_t value.
-  NaClBitcodeHeaderField(Tag MyID, uint32_t value);
-
-  // Create a header field for the given data.
-  NaClBitcodeHeaderField(Tag MyID, size_t MyLen, uint8_t *MyData);
-
-  virtual ~NaClBitcodeHeaderField() {
-    if (Data)
-      delete[] Data;
-  }
-
-  /// \brief Number of bytes used to represent header field.
-  size_t GetTotalSize() const {
-    // Round up to 4 byte alignment
-    return (kTagLenSize + Len + (WordSize - 1)) & ~(WordSize - 1);
-  }
-
-  /// \brief Write field into Buf[BufLen].
-  bool Write(uint8_t *Buf, size_t BufLen) const;
-
-  /// \brief Read field from Buf[BufLen].
-  bool Read(const uint8_t *Buf, size_t BufLen);
-
-  /// \brief Returns string describing ID of field.
-  static const char *IDName(Tag ID);
-  const char *IDName() const {
-    return IDName(ID);
-  }
-
-  /// \brief Returns string describing type of field.
-  static const char *TypeName(FieldType FType);
-  const char *TypeName() const {
-    return TypeName(FType);
-  }
-
-  /// \brief Returns string describing field.
-  std::string Contents() const;
-
-  /// \brief Get the data size from a serialized field to allow allocation.
-  static size_t GetDataSizeFromSerialized(const uint8_t *Buf) {
-    FixedSubfield Length;
-    ReadFixedSubfield(&Length, Buf + sizeof(FixedSubfield));
-    return Length;
-  }
-
-  /// \brief Return the ID of the field.
-  Tag GetID() const { return ID; }
-
-  FieldType GetType() const { return FType; }
-
-  /// \brief Return the length of the data (in bytes).
-  size_t GetLen() const { return Len; }
-
-  /// \brief Return the data. Data is array getData()[getLen()].
-  const uint8_t *GetData() const { return Data; }
-
-  /// \brief Returns the uint32_t value stored. Requires that
-  /// getType() == kUint32Type
-  uint32_t GetUInt32Value() const;
-
-private:
-  // Convert ID:Type into a fixed subfield
-  FixedSubfield EncodeTypedID() const { return (ID << 4) | FType; }
-  // Extract out ID and Type from a fixed subfield.
-  void DecodeTypedID(FixedSubfield Subfield, Tag &ID, FieldType &FType) {
-    FixedSubfield PossibleID = Subfield >> 4;
-    ID = (PossibleID > kTag_MAX ? kInvalid : static_cast<Tag>(PossibleID));
-    FixedSubfield PossibleFType = Subfield & 0xF;
-    FType = (PossibleFType > kFieldType_MAX
-             ? kUnknownType : static_cast<FieldType>(PossibleFType));
-  }
-  // Combined size of the fixed subfields
-  const static size_t kTagLenSize = 2 * sizeof(FixedSubfield);
-  static void WriteFixedSubfield(FixedSubfield Value, uint8_t *Buf) {
-    Buf[0] = Value & 0xFF;
-    Buf[1] = (Value >> 8) & 0xFF;
-  }
-  static void ReadFixedSubfield(FixedSubfield *Value, const uint8_t *Buf) {
-    *Value = Buf[0] | Buf[1] << 8;
-  }
-  Tag ID;
-  FieldType FType;
-  size_t Len;
-  uint8_t *Data;
-};
-
-/// \brief Class holding parsed header fields in PNaCl bitcode file.
-class NaClBitcodeHeader {
-  NaClBitcodeHeader(const NaClBitcodeHeader &) = delete;
-  void operator=(const NaClBitcodeHeader &) = delete;
-
-  // The set of parsed header fields. The header takes ownership of
-  // all fields in this vector.
-  std::vector<NaClBitcodeHeaderField *> Fields;
-  // The number of bytes in the PNaCl header.
-  size_t HeaderSize;
-  // String defining why it is unsupported (if unsupported).
-  std::string UnsupportedMessage;
-  // Flag defining if header is supported.
-  bool IsSupportedFlag;
-  // Flag defining if the corresponding bitcode file is readable.
-  bool IsReadableFlag;
-  // Defines the PNaCl version defined by the header file.
-  uint32_t PNaClVersion;
-  // Byte align bitcode records when nonzero.
-  bool AlignBitcodeRecords = false;
-
-public:
-  static const int WordSize = NaClBitcodeHeaderField::WordSize;
-
-  NaClBitcodeHeader();
-  ~NaClBitcodeHeader();
-
-  /// \brief Installs the fields of the header, defining if the header
-  /// is readable and supported. Sets UnsupportedMessage on failure.
-  void InstallFields();
-
-  /// \brief Adds a field to the list of fields in a header. Takes ownership
-  /// of fields added.
-  void push_back(NaClBitcodeHeaderField *Field) {
-    Fields.push_back(Field);
-  }
-
-  /// \brief Read the PNaCl bitcode header, The format of the header is:
-  ///
-  ///    1) 'PEXE' - The four character sequence defining the magic number.
-  ///    2) uint_16 num_fields - The number of NaClBitcodeHeaderField's.
-  ///    3) uint_16 num_bytes - The number of bytes to hold fields in
-  ///                           the header.
-  ///    4) NaClBitcodeHeaderField f1 - The first bitcode header field.
-  ///    ...
-  ///    2 + num_fields) NaClBitcodeHeaderField fn - The last bitcode header
-  /// field.
-  ///
-  /// Returns false if able to read (all of) the bitcode header.
-  bool Read(const unsigned char *BufPtr, const unsigned char *BufEnd);
-
-  // \brief Read the PNaCl bitcode header, recording the fields found
-  // in the header. Returns false if able to read (all of) the bitcode header.
-  bool Read(MemoryObject *Bytes);
-
-  // \brief Returns the number of bytes read to consume the header.
-  size_t getHeaderSize() { return HeaderSize; }
-
-  /// \brief Returns string describing why the header describes
-  /// an unsupported PNaCl Bitcode file.
-  const std::string &Unsupported() const { return UnsupportedMessage; }
-
-  /// \brief Returns true if supported. That is, it can be run in the
-  /// browser.
-  bool IsSupported() const { return IsSupportedFlag; }
-
-  /// \brief Returns true if the bitcode file should be readable. Note
-  /// that just because it is readable, it doesn't necessarily mean that
-  /// it is supported.
-  bool IsReadable() const { return IsReadableFlag; }
-
-  /// \brief Returns number of fields defined.
-  size_t NumberFields() const { return Fields.size(); }
-
-  /// \brief Returns a pointer to the field with the given ID
-  /// (0 if no such field).
-  NaClBitcodeHeaderField *GetTaggedField(NaClBitcodeHeaderField::Tag ID) const;
-
-  /// \brief Returns a pointer to the Nth field in the header
-  /// (0 if no such field).
-  NaClBitcodeHeaderField *GetField(size_t index) const;
-
-  /// \brief Returns the PNaClVersion, as defined by the header.
-  uint32_t GetPNaClVersion() const { return PNaClVersion; }
-
-  /// \brief Returns if one should byte align bitcode records.
-  bool getAlignBitcodeRecords() const { return AlignBitcodeRecords; }
-
-private:
-  // Reads and verifies the first 8 bytes of the header, consisting
-  // of the magic number 'PEXE', and the value defining the number
-  // of fields and number of bytes used to hold fields.
-  // Returns false if successful, sets UnsupportedMessage otherwise.
-  bool ReadPrefix(const unsigned char *BufPtr, const unsigned char *BufEnd,
-                  unsigned &NumFields, unsigned &NumBytes);
-
-  // Reads and verifies the fields in the header.
-  // Returns false if successful, sets UnsupportedMessage otherwise.
-  bool ReadFields(const unsigned char *BufPtr, const unsigned char *BufEnd,
-                  unsigned NumFields, unsigned NumBytes);
-
-  // Sets the Unsupported error message and returns true.
-  bool UnsupportedError(StringRef Message) {
-    UnsupportedMessage = Message.str();
-    return true;
-  }
-
-};
-
-} // namespace llvm
-
-#endif
diff --git a/include/llvm/Bitcode/NaCl/NaClBitcodeMunge.h b/include/llvm/Bitcode/NaCl/NaClBitcodeMunge.h
deleted file mode 100644
index 058af99bb6c..00000000000
--- a/include/llvm/Bitcode/NaCl/NaClBitcodeMunge.h
+++ /dev/null
@@ -1,303 +0,0 @@
-//===- NaClBitcodeMunge.h - Bitcode Munger ----------------------*- C++ -*-===//
-//
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
-//
-//===----------------------------------------------------------------------===//
-//
-// Test harness for generating a PNaCl bitcode memory buffer from
-// an array, and parse/objdump/compress the resulting contents.
-//
-// Generates a bitcode memory buffer from an array containing 1 or
-// more PNaCl records. Used to test errors in PNaCl bitcode.
-//
-// Bitcode records are modeled using arrays using the format
-// specified in NaClBitcodeMungeUtils.h.
-//
-// Note: Since the header record doesn't have any abbreviation indices
-// associated with it, one can use any value. The value will simply be
-// ignored.
-//
-// In addition to specifying the sequence of records, one can also
-// define a sequence of edits to be applied to the original sequence
-// of records. This allows the same record sequence to be used in
-// multiple tests. Again, see NaClBitcodeMungeUtils.h for the
-// format of editing arrays.
-//
-// Generally, you can generate any legal/illegal record
-// sequence. However, abbreviations are intimately tied to the
-// internals of the bitstream writer and can't contain illegal
-// data. Whenever class NaClBitcodeMunger is unable to accept illegal
-// data, a corresponding "Fatal" error is generated and execution
-// is terminated.
-//
-// ===---------------------------------------------------------------------===//
-
-#ifndef LLVM_BITCODE_NACL_NACLBITCODEMUNGE_H
-#define LLVM_BITCODE_NACL_NACLBITCODEMUNGE_H
-
-#include "llvm/ADT/SmallVector.h"
-#include "llvm/Bitcode/NaCl/NaClBitcodeMungeUtils.h"
-#include "llvm/Support/MemoryBuffer.h"
-#include "llvm/Support/raw_ostream.h"
-
-#include <string>
-
-namespace llvm {
-
-class NaClBitCodeAbbrev;
-
-/// Base class to run tests on munged bitcode files.
-class NaClBitcodeMunger {
-public:
-  /// Creates a bitcode munger, based on the given array of values.
-  NaClBitcodeMunger(const uint64_t Records[], size_t RecordsSize,
-                    uint64_t RecordTerminator)
-      : MungedBitcode(Records, RecordsSize, RecordTerminator),
-        RecordTerminator(RecordTerminator),
-        DumpResults("Error: No previous dump results!\n"),
-        DumpStream(nullptr), FoundErrors(false), RunAsDeathTest(false) {}
-
-  /// Returns true if running as death test.
-  bool getRunAsDeathTest() const {
-    return RunAsDeathTest;
-  }
-
-  /// Sets death test flag. When true, output will be redirected to
-  /// the errs() (rather than buffered) so that the test can be
-  /// debugged.
-  void setRunAsDeathTest(bool NewValue) {
-    RunAsDeathTest = NewValue;
-  }
-
-  /// Creates MungedInput and DumpStream for running tests, based on
-  /// given Munges. Returns true if able to set up test.
-  bool setupTest(const uint64_t Munges[], size_t MungesSize, bool AddHeader);
-
-  // TODO(kschimpf): The following function is deprecated and only
-  // provided until subzero is updated to use the new API that no
-  // longer uses test names.
-  bool setupTest(const char *, const uint64_t Munges[], size_t MungesSize,
-                 bool AddHeader) {
-    return setupTest(Munges, MungesSize, AddHeader);
-  }
-
-  /// Cleans up state after a test. Returns true if no errors found.
-  bool cleanupTest();
-
-  /// Returns the resulting string generated by the corresponding test.
-  const std::string &getTestResults() const {
-    return DumpResults;
-  }
-
-  /// Returns the lines containing the given Substring, from the
-  /// string getTestResults().
-  std::string getLinesWithSubstring(const std::string &Substring) const {
-    return getLinesWithTextMatch(Substring, false);
-  }
-
-  /// Returns the lines starting with the given Prefix, from the string
-  /// getTestResults().
-  std::string getLinesWithPrefix(const std::string &Prefix) const {
-    return getLinesWithTextMatch(Prefix, true);
-  }
-
-  /// When NewValue, use error recovery when writing bitcode during
-  /// next test.
-  void setTryToRecoverOnWrite(bool NewValue) {
-    WriteFlags.setTryToRecover(NewValue);
-  }
-
-  /// When NewValue, write bad abbreviation index into bitcode when
-  /// writing during next test.
-  void setWriteBadAbbrevIndex(bool NewValue) {
-    WriteFlags.setWriteBadAbbrevIndex(NewValue);
-  }
-
-  /// Get access to munged bitcodes.
-  NaClMungedBitcode &getMungedBitcode() {
-    return MungedBitcode;
-  }
-
-  /// Apply given munges to the munged bitcode.
-  void munge(const uint64_t Munges[], size_t MungesSize) {
-    MungedBitcode.munge(Munges, MungesSize, RecordTerminator);
-  }
-
-protected:
-  // The bitcode records being munged.
-  NaClMungedBitcode MungedBitcode;
-  // The value used as record terminator.
-  uint64_t RecordTerminator;
-  // The results buffer of the last dump.
-  std::string DumpResults;
-  // The memory buffer containing the munged input.
-  std::unique_ptr<MemoryBuffer> MungedInput;
-  // The stream containing errors and the objdump of the generated bitcode file.
-  raw_ostream *DumpStream;
-  // True if any errors were reported.
-  bool FoundErrors;
-  // The buffer for the contents of the munged input.
-  SmallVector<char, 1024> MungedInputBuffer;
-  /// The write flags to use when writing bitcode.
-  NaClMungedBitcode::WriteFlags WriteFlags;
-  // Flag to redirect dump stream if running death test.
-  bool RunAsDeathTest;
-
-  // Records that an error occurred, and returns stream to print error
-  // message to.
-  raw_ostream &Error() {
-    FoundErrors = true;
-    return getDumpStream() << "Error: ";
-  }
-
-  // Returns the lines containing the given Substring, from the string
-  // getTestResults(). If MustBePrefix, then Substring must match at
-  // the beginning of the line.
-  std::string getLinesWithTextMatch(const std::string &Substring,
-                                    bool MustBePrefix = false) const;
-
-  // Returns the log stream to use. When running death tests, redirect output
-  // to the error stream (rather than buffering in DumpStream), so that
-  // the output can be seen in gtest death tests.
-  raw_ostream &getDumpStream() const {
-    return RunAsDeathTest ? errs() : *DumpStream;
-  }
-};
-
-/// Class to run tests writing munged bitcode.
-class NaClWriteMunger : public NaClBitcodeMunger {
-public:
-  NaClWriteMunger(const uint64_t Records[], size_t RecordsSize,
-                  uint64_t RecordTerminator)
-      : NaClBitcodeMunger(Records, RecordsSize, RecordTerminator) {}
-
-  /// Writes munged bitcode and puts error messages into DumpResults.
-  /// Returns true if successful.
-  bool runTest(const uint64_t Munges[], size_t MungesSize);
-
-  /// Same as above, but without any edits.
-  bool runTest() {
-    uint64_t NoMunges[] = {0};
-    return runTest(NoMunges, 0);
-  }
-};
-
-/// Class to run tests for function llvm::NaClObjDump.
-class NaClObjDumpMunger : public NaClBitcodeMunger {
-public:
-
-  /// Creates a bitcode munger, based on the given array of values.
-  NaClObjDumpMunger(const uint64_t Records[], size_t RecordsSize,
-                    uint64_t RecordTerminator)
-      : NaClBitcodeMunger(Records, RecordsSize, RecordTerminator) {}
-
-  /// Runs function NaClObjDump on the sequence of records associated
-  /// with the instance. The memory buffer containing the bitsequence
-  /// associated with the record is automatically generated, and
-  /// passed to NaClObjDump. If AddHeader is true, test assumes that the
-  /// sequence of records doesn't contain a header record, and the
-  /// test should add one. Arguments NoRecords and NoAssembly are
-  /// passed to NaClObjDump. Returns true if test succeeds without
-  /// errors.
-  bool runTestWithFlags(bool AddHeader, bool NoRecords, bool NoAssembly) {
-    uint64_t NoMunges[] = {0};
-    return runTestWithFlags(NoMunges, 0, AddHeader, NoRecords, NoAssembly);
-  }
-
-  /// Same as above except it runs function NaClObjDump with flags
-  /// NoRecords and NoAssembly set to false, and AddHeader set to true.
-  bool runTest() {
-    return runTestWithFlags(true, false, false);
-  }
-
-  // TODO(kschimpf): The following function is deprecated and only
-  // provided until subzero is updated to use the new API that no
-  // longer uses test names.
-  bool runTest(const char *) {
-    return runTest();
-  }
-
-  /// Same as above, but only print out assembly and errors.
-  bool runTestForAssembly() {
-    return runTestWithFlags(true, true, false);
-  }
-  /// Same as above, but only generate error messages.
-  bool runTestForErrors() {
-    return runTestWithFlags(true, true, true);
-  }
-
-  /// Runs function llvm::NaClObjDump on the sequence of records
-  /// associated with the instance. Array Munges contains the sequence
-  /// of edits to apply to the sequence of records when generating the
-  /// bitsequence in a memory buffer. This generated bitsequence is
-  /// then passed to NaClObjDump.  TestName is the name associated
-  /// with the memory buffer.  Arguments NoRecords and NoAssembly are
-  /// passed to NaClObjDump. Returns true if test succeeds without
-  /// errors.
-  bool runTestWithFlags(const uint64_t Munges[], size_t MungesSize,
-                        bool AddHeader, bool NoRecords, bool NoAssembly);
-
-  /// Same as above except it runs function NaClObjDump with flags
-  /// NoRecords and NoAssembly set to false, and AddHeader set to
-  /// true.
-  bool runTest(const uint64_t Munges[], size_t MungesSize) {
-    return runTestWithFlags(Munges, MungesSize, true, false, false);
-  }
-
-  bool runTestForAssembly(const uint64_t Munges[], size_t MungesSize) {
-    return runTestWithFlags(Munges, MungesSize, true, true, false);
-  }
-
-  // TODO(kschimpf): The following function is deprecated and only
-  // provided until subzero is updated to use the new API that no
-  // longer uses test names.
-  bool runTestForAssembly(const char *, const uint64_t Munges[],
-                          size_t MungesSize) {
-    return runTestForAssembly(Munges, MungesSize);
-  }
-
-
-  bool runTestForErrors(const uint64_t Munges[], size_t MungesSize) {
-    return runTestWithFlags(Munges, MungesSize, true, true, true);
-  }
-};
-
-// Class to run tests for function NaClParseBitcodeFile.
-class NaClParseBitcodeMunger : public NaClBitcodeMunger {
-public:
-  NaClParseBitcodeMunger(const uint64_t Records[], size_t RecordsSize,
-                         uint64_t RecordTerminator)
-      : NaClBitcodeMunger(Records, RecordsSize, RecordTerminator) {}
-
-  /// Runs function llvm::NaClParseBitcodeFile, and puts error messages
-  /// into DumpResults. Returns true if parse is successful.
-  bool runTest(const uint64_t Munges[], size_t MungesSize, bool VerboseErrors);
-
-  // Same as above, but without any edits.
-  bool runTest(bool VerboseErrors) {
-    uint64_t NoMunges[] = {0};
-    return runTest(NoMunges, 0, VerboseErrors);
-  }
-};
-
-// Class to run tests for NaClBitcodeCompressor.compress().
-class NaClCompressMunger : public NaClBitcodeMunger {
-public:
-  NaClCompressMunger(const uint64_t Records[], size_t RecordsSize,
-                     uint64_t RecordTerminator)
-      : NaClBitcodeMunger(Records, RecordsSize, RecordTerminator) {}
-
-  bool runTest(const uint64_t Munges[], size_t MungesSize);
-
-  bool runTest() {
-    uint64_t NoMunges[] = {0};
-    return runTest(NoMunges, 0);
-  }
-};
-
-} // end namespace llvm.
-
-#endif // LLVM_BITCODE_NACL_NACLBITCODEMUNGE_H
diff --git a/include/llvm/Bitcode/NaCl/NaClBitcodeMungeUtils.h b/include/llvm/Bitcode/NaCl/NaClBitcodeMungeUtils.h
deleted file mode 100644
index 0b749a87f38..00000000000
--- a/include/llvm/Bitcode/NaCl/NaClBitcodeMungeUtils.h
+++ /dev/null
@@ -1,492 +0,0 @@
-//===- NaClBitcodeMungeUtils.h - Munge bitcode records --------*- C++ -*-===//
-//
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
-//
-//===----------------------------------------------------------------------===//
-//
-// This file defines utility class NaClMungedBitcode to edit a base
-// sequence of PNaCl bitcode records. It is intended to be used for
-// both unit testing, and for fuzzing bitcode files.
-//
-// The editing actions are defined in terms of the base sequence of
-// bitcode records, and do not actually modify that base sequence. As
-// such, all edits are defined relative to a record index in the base
-// sequence of bitcode records. Using record indices into the base
-// sequence means that succeeding edits need not modify record indices
-// based on the effects of preceding edits. Rather, the record index
-// remains constant across all editing actions.
-//
-// There are two steps in creating munged bitcode. The first step
-// defines the initial (base) sequence of bitcode records to be
-// edited. The second is to apply editing actions to the base sequence
-// of records.
-//
-// Defining the initial sequence of bitcode records is done by
-// using NaClMungedBitcode's constructor.
-//
-// There are four kinds of editing actions:
-//
-// 1) Add a record before a (base) record index.
-// 2) Add a record after a (base) record index.
-// 3) Remove a record at a (base) record index.
-// 4) Replace a record at a (base) record index.
-//
-// Each of the editing actions are defined by the methods addBefore(),
-// addAfter(), remove(), and replace(). The edited sequence of records
-// is defined by an iterator, and is accessible using methods begin()
-// and end().
-//
-// If multiple records are added before/after a record index, the
-// order of the added records correspond to the order they were added.
-//
-// For unit testing, simple array interfaces are provided. The first
-// interface defines the initial base sequence of bitcode records. The
-// second interface, method munge(), defines a sequence of editing
-// actions to apply to the base sequence of records.
-//
-// These arrays are defined using type uint64_t. A bitcode record is
-// defined as a sequence of values of the form:
-//
-//   AbbrevIndex, RecordCode, Value1, ..., ValueN , Terminator
-//
-// Terminator is a (user-specified) designated constant used to mark
-// the end of bitcode records. A record can contain zero or more
-// values.
-//
-// An editing action is defined as one of the following sequences of
-// values:
-//
-//   RecordIndex, AddBefore, AbbrevIndex, RecordCode, Value, ..., Terminator
-//   RecordIndex, AddAfter, AbbrevIndex, RecordCode, Value, ..., Terminator
-//   RecordIndex, Remove
-//   RecordIndex, Replace, AbbrevIndex, RecordCode, Value, ..., Terminator.
-//
-// RecordIndex defines where the action applies, relative to the base
-// sequence of records. EditAction defines the editing action to
-// apply. The remaining fields define an optional record associated
-// with the editing action.
-//
-// ===---------------------------------------------------------------------===//
-
-#ifndef LLVM_BITCODE_NACL_NACLBITCODEMUNGEUTILS_H
-#define LLVM_BITCODE_NACL_NACLBITCODEMUNGEUTILS_H
-
-#include "llvm/Bitcode/NaCl/NaClBitcodeParser.h"
-#include "llvm/Support/MemoryBuffer.h"
-
-#include <list>
-#include <map>
-
-namespace llvm {
-
-class NaClBitcodeAbbrevRecord; // bitcode record.
-class NaClMungedBitcodeIter;   // iterator over edited bitcode records.
-
-/// \brief Defines a list of bitcode records.
-typedef std::vector<std::unique_ptr<NaClBitcodeAbbrevRecord>>
-    NaClBitcodeRecordList;
-
-// TODO(kschimpf): Modify following two methods to return error_code.
-
-/// \brief Extracts out the records in Records, and puts them into RecordList.
-///
-/// \brief RecordList[in/out] Record list to read into.
-/// \brief Records Array containing data defining records.
-/// \brief RecordsSize The size of array Records.
-/// \brief RecordTerminator The value used to terminate records.
-void readNaClBitcodeRecordList(NaClBitcodeRecordList &RecordList,
-                               const uint64_t Records[],
-                               size_t RecordsSize,
-                               uint64_t RecordTerminator);
-
-/// Read in the list of records from binary bitcode from a memory buffer.
-void readNaClBitcodeRecordList(NaClBitcodeRecordList &RecordList,
-                               std::unique_ptr<MemoryBuffer> InputBuffer);
-
-/// Read in the list of records from textual bitcode from a memory buffer.
-std::error_code readNaClTextBcRecordList(
-    NaClBitcodeRecordList &RecordList,
-    std::unique_ptr<MemoryBuffer> InputBuffer);
-
-/// Read textual bitcode records from Filename, and fill Buffer with
-/// corresponding bitcode. Return error_code describing success of
-/// read.  Verbose (if not nullptr) is used to generate more human
-/// readable error messages than the text in the returned error
-/// message.
-std::error_code readNaClRecordTextAndBuildBitcode(
-    StringRef Filename, SmallVectorImpl<char> &Buffer,
-    raw_ostream *Verbose = nullptr);
-
-/// Write out RecordList (as text) to Buffer. Returns true when
-/// successful. Error message are written to ErrStream.
-bool writeNaClBitcodeRecordList(NaClBitcodeRecordList &RecordList,
-                                SmallVectorImpl<char> &Buffer,
-                                raw_ostream &ErrStream);
-
-/// \brief An edited (i.e. munged) list of bitcode records. Edits are
-/// always relative to the initial list of records.
-class NaClMungedBitcode {
-  friend class NaClMungedBitcodeIter;
-  NaClMungedBitcode() = delete;
-  NaClMungedBitcode(const NaClMungedBitcode &) = delete;
-  NaClMungedBitcode &operator=(const NaClMungedBitcode &) = delete;
-
-public:
-  /// \brief Iterator over edited records.
-  typedef NaClMungedBitcodeIter iterator;
-
-  /// \brief Read in initial list of records from bitcode in a memory buffer.
-  explicit NaClMungedBitcode(std::unique_ptr<MemoryBuffer> InputBuffer,
-                             bool ReadAsText=false)
-      : BaseRecords(new NaClBitcodeRecordList()) {
-    if (ReadAsText)
-      readNaClTextBcRecordList(*BaseRecords, std::move(InputBuffer));
-    else
-      readNaClBitcodeRecordList(*BaseRecords, std::move(InputBuffer));
-  }
-
-  /// \brief Initialize the list of records to be edited.
-  explicit NaClMungedBitcode(std::unique_ptr<NaClBitcodeRecordList> BaseRecords)
-      : BaseRecords(std::move(BaseRecords)) {}
-
-  /// \brief Initialize the list of records to be edited using
-  /// array specification.
-  ///
-  /// \brief Records Array containing data defining records.
-  /// \brief RecordsSize The size of array Records.
-  /// \brief RecordTerminator The value used to terminate records.
-  NaClMungedBitcode(const uint64_t Records[], size_t RecordsSize,
-                    uint64_t RecordTerminator);
-
-  ~NaClMungedBitcode();
-
-  /// \brief Iterator pointing to first record in edited records.
-  NaClMungedBitcodeIter begin() const;
-
-  /// \brief Iterator pointing after last record in edited records.
-  NaClMungedBitcodeIter end() const;
-
-  /// \brief Insert Record immediately before the record at RecordIndex.
-  ///
-  /// \param RecordIndex The index (within BaseRecords) to add before.
-  /// \param Record The record to add.
-  void addBefore(size_t RecordIndex, NaClBitcodeAbbrevRecord &Record);
-
-  /// \brief Insert Record after the record at RecordIndex (and after
-  /// any previously added after records for RecordIndex).
-  ///
-  /// \param RecordIndex The index (within BaseRecords) to add after.
-  /// \param Record The record to add.
-  void addAfter(size_t RecordIndex, NaClBitcodeAbbrevRecord &Record);
-
-  /// \brief Remove the record at RecordIndex. Note that because
-  /// the record index is based on the base records, and those records
-  /// are not modified, this editing action effectively undoes all
-  /// previous remove/replace editing actions for this index.
-  void remove(size_t RecordIndex);
-
-  /// \brief Replace the record at RecordIndex with Record. Note that
-  /// because the record index is based on the base records, and those
-  /// records are not modified, this editing action effectively undoes
-  /// all previous remove/replace editing actions for this index.
-  void replace(size_t RecordIndex, NaClBitcodeAbbrevRecord &Record);
-
-  /// \brief Print out the resulting edited list of records.
-  void print(raw_ostream &Out) const;
-
-  /// Defines set of possible write flags.
-  struct WriteFlags {
-    /// True if error recovery should be applied.
-    bool getTryToRecover() const { return TryToRecover; }
-
-    /// Define that error recovery should be applied when writing.
-    void setTryToRecover(bool NewValue) {
-      TryToRecover = NewValue;
-      assert(!(TryToRecover && WriteBadAbbrevIndex));
-    }
-
-    /// True if a bad abbreviation index should be written (rather than
-    /// trying error recovery) so that bitcode readers can be tested for
-    /// this condition.
-    bool getWriteBadAbbrevIndex() const { return WriteBadAbbrevIndex; }
-
-    /// Define that the first bad abbreviation index should be written,
-    /// and corresponding minimal context added so that the bitcode can
-    /// be used to test reading the erroneous written bitcode.
-    void setWriteBadAbbrevIndex(bool NewValue) {
-      WriteBadAbbrevIndex = NewValue;
-      assert(!(TryToRecover && WriteBadAbbrevIndex));
-    }
-
-    /// Get the stream to print errors while writing bitcode.
-    raw_ostream &getErrStream() const {
-      return ErrStream ? *ErrStream : errs();
-    }
-
-    /// Set the stream to print errors to.
-    void setErrStream(raw_ostream &NewValue) {
-      ErrStream = &NewValue;
-    }
-
-    void reset() {
-      TryToRecover = false;
-      WriteBadAbbrevIndex = false;
-      ErrStream = nullptr;
-    }
-
-  private:
-    bool TryToRecover = false;
-    bool WriteBadAbbrevIndex = false;
-    raw_ostream *ErrStream = nullptr;
-  };
-
-  /// Defines the results associated with writing bitcode.
-  struct WriteResults {
-    /// Number of errors generated.
-    size_t NumErrors = 0;
-    /// Number of repairs (via error recovery) that were applied.
-    size_t NumRepairs = 0;
-    /// True if a bad abbreviation index were written.
-    bool WroteBadAbbrevIndex = false;
-  };
-
-  /// \brief Write out the edited list of bitcode records using
-  /// the given buffer.
-  ///
-  /// \param Buffer The buffer to write into.
-  /// \param AddHeader Add header block when true.
-  /// \param Flags Write flags to use.
-  ///
-  /// \return Returns the results of the write.
-  WriteResults writeMaybeRepair(
-      SmallVectorImpl<char> &Buffer, bool AddHeader,
-      const WriteFlags &Flags) const;
-
-  bool write(SmallVectorImpl<char> &Buffer, bool AddHeader,
-             const WriteFlags &Flags) const {
-    WriteResults Results = writeMaybeRepair(Buffer, AddHeader, Flags);
-    return Results.NumErrors == 0
-        || (Flags.getTryToRecover() && Results.NumErrors == Results.NumRepairs);
-  }
-
-  bool write(SmallVectorImpl<char> &Buffer, bool AddHeader) const {
-    WriteFlags Flags;
-    return write(Buffer, AddHeader, Flags);
-  }
-
-  /// \brief The types of editing actions that can be applied.
-  enum EditAction {
-    AddBefore, // Insert new record before base record at index.
-    AddAfter,  // Insert new record after base record at index.
-    Remove,    // Remove record at index.
-    Replace    // Replace base record at index with new record.
-  };
-
-  /// \brief Apply a set of edits defined in the given array.
-  ///
-  /// Actions are a sequence of values, followed by a (common)
-  /// terminator value. Valid action sequences are:
-  ///
-  /// RecordIndex AddBefore Abbrev Code Values Terminator
-  ///
-  /// RecordIndex AddAfter Abbrev Code Values Terminator
-  ///
-  /// RecordIndex Remove
-  ///
-  /// RecordIndex Replace Abbrev Code Values Terminator
-  ///
-  /// \param Munges The array containing the edits to apply.
-  /// \param MungesSize The size of Munges.
-  /// \param Terminator The value used to terminate records in editing actions.
-  void munge(const uint64_t Munges[], size_t MungesSize, uint64_t Terminator);
-
-  /// \brief Removes all editing actions and resets back to the original
-  /// set of base records.
-  void removeEdits();
-
-  /// Returns the unedited list of bitcode records.
-  const NaClBitcodeRecordList &getBaseRecords() const {
-    return *BaseRecords;
-  }
-
-private:
-  typedef std::list<NaClBitcodeAbbrevRecord *> RecordListType;
-  typedef std::map<size_t, RecordListType *> InsertionsMapType;
-  typedef std::map<size_t, NaClBitcodeAbbrevRecord *> ReplaceMapType;
-
-  /// \brief The list of base records that will be edited.
-  std::unique_ptr<NaClBitcodeRecordList> BaseRecords;
-  // Holds map from record index to list of records added before
-  // the corresponding record in the list of base records.
-  InsertionsMapType BeforeInsertionsMap;
-  // Holds map from record index to the list of records added after
-  // the corresponding record in the list of base records.
-  InsertionsMapType AfterInsertionsMap;
-  // Holds map from record index to the record that replaces the
-  // corresponding record in the list of base records. Note: If the
-  // range is the nullptr, it corresponds to a remove instead of a
-  // replace.
-  ReplaceMapType ReplaceMap;
-
-  // Returns the list of records associated with Index in Map.
-  RecordListType &at(InsertionsMapType &Map, size_t Index) {
-    InsertionsMapType::iterator Pos = Map.find(Index);
-    if (Pos == Map.end()) {
-      RecordListType *List = new RecordListType();
-      Map.emplace(Index, List);
-      return *List;
-    }
-    return *Pos->second;
-  }
-
-  // Creates a (heap allocated) copy of the given record.
-  NaClBitcodeAbbrevRecord *copy(const NaClBitcodeAbbrevRecord &Record);
-
-  // Delete nested objects within insertions map.
-  void destroyInsertionsMap(NaClMungedBitcode::InsertionsMapType &Map);
-};
-
-/// \brief Defines a bitcode record with its associated abbreviation index.
-class NaClBitcodeAbbrevRecord : public NaClBitcodeRecordData {
-  NaClBitcodeAbbrevRecord &operator=(NaClBitcodeAbbrevRecord &) = delete;
-
-public:
-  /// \brief The abbreviation associated with the bitcode record.
-  unsigned Abbrev;
-
-  /// \brief Creates a bitcode record.
-  ///
-  /// \param Abbrev Abbreviation index associated with record.
-  /// \param Code The selector code of the record.
-  /// \param Values The values associated with the selector code.
-  NaClBitcodeAbbrevRecord(unsigned Abbrev, unsigned Code,
-                          const NaClRecordVector &Values)
-      : NaClBitcodeRecordData(Code, Values), Abbrev(Abbrev) {}
-
-  /// \brief Creates a copy of the given abbreviated bitcode record.
-  explicit NaClBitcodeAbbrevRecord(const NaClBitcodeAbbrevRecord &Rcd)
-      : NaClBitcodeRecordData(Rcd), Abbrev(Rcd.Abbrev) {}
-
-  /// \brief Creates a default abbreviated bitcode record.
-  NaClBitcodeAbbrevRecord() : Abbrev(naclbitc::UNABBREV_RECORD) {}
-
-  ~NaClBitcodeAbbrevRecord() {}
-
-  /// \brief Replaces the contents of the abbreviated bitcode record with
-  /// the corresponding contents in the array.
-  ///
-  /// \param Values The array to get values from.
-  /// \param ValuesSize The length of the Values array.
-  /// \param Terminator The value defining the end of the record in Values.
-  /// \param Index The index of the first value to be used in Values.
-  void read(const uint64_t Values[], size_t ValuesSize,
-            const uint64_t Terminator, size_t &Index);
-
-  void print(raw_ostream &out) const;
-};
-
-inline raw_ostream &operator<<(raw_ostream &Out,
-                               const NaClBitcodeAbbrevRecord &Record) {
-  Record.print(Out);
-  return Out;
-}
-
-/// \brief Defines an iterator to walk over elements of an edited
-/// record list.
-class NaClMungedBitcodeIter {
-
-public:
-  /// \brief Returns an iterator pointing to the first record in the
-  /// edited list of records.
-  static NaClMungedBitcodeIter begin(const NaClMungedBitcode &MungedBitcode) {
-    return NaClMungedBitcodeIter(MungedBitcode, 0);
-  }
-
-  /// \brief Returns an iterator pointing past the last record in the
-  /// edited list of records.
-  static NaClMungedBitcodeIter end(const NaClMungedBitcode &MungedBitcode) {
-    return NaClMungedBitcodeIter(MungedBitcode,
-                                 MungedBitcode.BaseRecords->size());
-  }
-
-  bool operator==(const NaClMungedBitcodeIter &Iter) const;
-
-  bool operator!=(const NaClMungedBitcodeIter &Iter) const {
-    return !operator==(Iter);
-  }
-
-  /// \brief Advances the iterator over one record in the list of
-  /// edited records.
-  NaClMungedBitcodeIter &operator++();
-
-  /// \brief Returns the bitcode record the iterator is before.
-  NaClBitcodeAbbrevRecord &operator*();
-
-private:
-  /// \brief Defines position of the iterator, relative to the corresponding
-  /// record index in the base list of records.
-  enum MungedPosition {
-    /// Processing the list of records inserted before the record
-    /// index of the base list of records.
-    InBeforeInsertions,
-    /// Processing the record at the given record index of the base
-    /// list of records.
-    AtIndex,
-    /// Processing the list of records inserted after the record index
-    /// of the base list of records.
-    InAfterInsertions,
-  };
-
-  // The edited list of records to iterate over.
-  const NaClMungedBitcode *MungedBitcode;
-  // The corresponding index, wrt to the edited list of records, that
-  // is being processed.
-  size_t Index;
-  // The position of the iterator wrt to Index.
-  MungedPosition Position;
-  // An iterator defining the current position within a
-  // BeforeInsertions or AfterInsertions list. When the iterator is
-  // not in the corresponding position (i.e. InBeforeInsertions or
-  // InAfterInsertions) this iterator is undefined.
-  NaClMungedBitcode::RecordListType::const_iterator InsertionsIter;
-  // An iterator defining the end position of the corresponding list
-  // of records defined by InsertionsIter. Only defined when
-  // InsertionsIter is defined.
-  NaClMungedBitcode::RecordListType::const_iterator InsertionsIterEnd;
-  // Dummy list to initialize InsertionsIter if no such list exists.
-  NaClMungedBitcode::RecordListType EmptyList;
-
-  // \brief Defines an iterator at the beginning of the
-  // BeforeInsertions list associated with the given Index in
-  // MungedBitcode.
-  NaClMungedBitcodeIter(const NaClMungedBitcode &MungedBitcode, size_t Index)
-      : MungedBitcode(&MungedBitcode), Index(Index),
-        Position(InBeforeInsertions), InsertionsIter(), InsertionsIterEnd() {
-    placeAt(MungedBitcode.BeforeInsertionsMap, 0);
-    updatePosition();
-  }
-
-  // \brief Places the corresponding insertions iterator based on the
-  // list of records defined at Index for the given insertions Map.
-  void placeAt(const NaClMungedBitcode::InsertionsMapType &Map, size_t Index) {
-    NaClMungedBitcode::InsertionsMapType::const_iterator Pos = Map.find(Index);
-    if (Pos == Map.end()) {
-      InsertionsIter = EmptyList.end();
-      InsertionsIterEnd = EmptyList.end();
-    } else {
-      InsertionsIter = Pos->second->begin();
-      InsertionsIterEnd = Pos->second->end();
-    }
-  }
-
-  // \brief Moves the iterator to the position of the next edited
-  // record.
-  void updatePosition();
-};
-
-} // end namespace llvm.
-
-#endif // LLVM_BITCODE_NACL_NACLBITCODEMUNGE_H
diff --git a/include/llvm/Bitcode/NaCl/NaClBitcodeParser.h b/include/llvm/Bitcode/NaCl/NaClBitcodeParser.h
deleted file mode 100644
index 5efd7df19d0..00000000000
--- a/include/llvm/Bitcode/NaCl/NaClBitcodeParser.h
+++ /dev/null
@@ -1,649 +0,0 @@
-//===- NaClBitcodeParser.h -----------------------------------*- C++ -*-===//
-//     Low-level bitcode driver to parse PNaCl bitcode files.
-//
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
-//
-//===----------------------------------------------------------------------===//
-//
-// Parses and processes low-level PNaCl bitcode files. Defines class
-// NaClBitcodeParser.
-//
-// The concepts of PNaCl bitcode files are basically the same as for
-// LLVM bitcode files (see http://llvm.org/docs/BitCodeFormat.html for
-// details).
-//
-// The bitstream format is an abstract encoding of structured data,
-// very similar to XML in some ways. Like XML, bitstream files contain
-// tags, and nested structures, and you can parse the file without
-// having to understand the tags. Unlike XML, the bitstream format is
-// a binary encoding, and provides a mechanism for the file to
-// self-describe "abbreviations".  Abbreviations are effectively size
-// optimizations for the content.
-//
-// The bitcode file is conceptually a sequence of "blocks", defining
-// the content. Blocks contain a sequence of records and
-// blocks. Nested content is defined using nested blocks.  A (data)
-// "record" is a tag, and a vector of (unsigned integer) values.
-//
-// Blocks are identified using Block IDs. Each kind of block has a
-// unique block "ID". Records have two elements:
-//
-//   a) A "code" identifying what type of record it is.
-//   b) A vector of "values" defining the contents of the record.
-//
-// The bitstream "reader" (defined in NaClBitstreamReader.h) defines
-// the implementation that converts the low-level bit file into
-// records and blocks. The bit stream is processed by moving a
-// "cursor" over the sequence of bits.
-//
-// The bitstream reader assumes that each block/record is read in by
-// first reading the "entry". The entry defines whether it corresponds
-// to one of the following:
-//
-//    a) At the beginning of a (possibly nested) block
-//    b) At the end of the current block.
-//    c) The input defines an abberviation.
-//    d) The input defines a record.
-//
-// An entry contains two values, a "kind" and an "ID". The kind
-// defines which of the four cases above occurs. The ID provides
-// identifying information on how to further process the input. For
-// case (a), the ID is the identifier associated with the the block
-// being processed. For case (b) and (c) the ID is ignored. For case
-// (d) the ID identifies the abbreviation that should be used to parse
-// the values.
-//
-// The class NaClBitcodeParser defines a bitcode parser that extracts
-// the blocks and records, which are then processed using virtual
-// callbacks. In general, you will want to implement derived classes
-// for each type of block, so that the corresponding data is processed
-// appropriately.
-//
-// The class NaClBitcodeParser parses a bitcode block, and defines a
-// set of callbacks for that block, including:
-//
-//    a) EnterBlock: What to do once we have entered the block.
-//    b) ProcessRecord: What to do with each parsed record.
-//    c) ParseBlock: Parse the (nested) block with the given ID.
-//    d) ExitBlock: What to do once we have finished processing the block.
-//
-// Note that a separate instance of NaClBitcodeParser (or a
-// corresponding derived class) is created for each nested block. Each
-// instance is responsible for only parsing a single block. Method
-// ParseBlock creates new instances to parse nested blocks. Method
-// GetEnclosingParser() can be used to refer to the parser associated
-// with the enclosing block.
-//
-// Currently, the default processing of abbreviations is handled by
-// the PNaCl bitstream reader, rather than by the parser.
-//
-// If you need to process abbreviations processed by the PNaCl
-// bitstream reader, you must explicitly define a
-// NaClBitcodeParserListener to listen (within the bitstream reader),
-// and make appropriate call backs to the NaClBitcodeParser.
-// The listener is glued to parsers using method SetListener.
-//
-// TODO(kschimpf): Define an intermediate derived class of
-// NaClBitcodeParser that defines callbacks based on the actual
-// structure of PNaCl bitcode files.  That is, it has callbacks for
-// each of the types of blocks (i.e. module, types, global variables,
-// function, symbol tables etc). This derivied class can then be used
-// as the base class for the bitcode reader.
-// ===----------------------------------------------------------------------===//
-
-#ifndef LLVM_BITCODE_NACL_NACLBITCODEPARSER_H
-#define LLVM_BITCODE_NACL_NACLBITCODEPARSER_H
-
-#include "llvm/Bitcode/NaCl/NaClBitstreamReader.h"
-#include "llvm/Bitcode/NaCl/NaClBitcodeDefs.h"
-#include "llvm/Support/raw_ostream.h"
-#include <vector>
-
-namespace llvm {
-
-class NaClBitcodeRecord;
-class NaClBitcodeParser;
-class NaClBitcodeParserListener;
-
-// Defines the base class for data extracted from the input bitstream
-// (i.e blocks and records).
-class NaClBitcodeData {
-  void operator=(const NaClBitcodeData&) = delete;
-
-public:
-  /// Create data element to be read from input cursor.
-  explicit NaClBitcodeData(NaClBitstreamCursor &Cursor)
-      : Cursor(Cursor), StartBit(Cursor.GetCurrentBitNo())
-  {}
-
-  /// Create copy of the given data element.
-  explicit NaClBitcodeData(const NaClBitcodeData &Data)
-      : Cursor(Data.Cursor), StartBit(Data.StartBit)
-  {}
-
-  /// Returns the bitstream reader being used.
-  NaClBitstreamReader &GetReader() const {
-    return *Cursor.getBitStreamReader();
-  }
-
-  /// Returns the cursor position within the bitstream.
-  NaClBitstreamCursor &GetCursor() const {
-    return Cursor;
-  }
-
-  /// Returns the number of bits defined by the data.
-  uint64_t GetNumBits() const {
-    return GetCursor().GetCurrentBitNo() - StartBit;
-  }
-
-  /// Returns the first bit of the stream data.
-  uint64_t GetStartBit() const {
-    return StartBit;
-  }
-
-protected:
-  /// Change the start bit for the data to the new value.
-  void SetStartBit(uint64_t NewValue) {
-    StartBit = NewValue;
-  }
-
-private:
-  // The bitstream cursor defining location within the bitcode file.
-  NaClBitstreamCursor &Cursor;
-
-  // Start bit for the record.
-  uint64_t StartBit;
-};
-
-/// Models the block defined by a (begin) block record, through the
-/// (end) block record.
-class NaClBitcodeBlock : public NaClBitcodeData {
-  NaClBitcodeBlock(const NaClBitcodeBlock &) = delete;
-  void operator=(const NaClBitcodeBlock &) = delete;
-
-public:
-  /// Given the found (begin) block record for block BlockID, create
-  /// the corresponding data associated with that block.
-  NaClBitcodeBlock(unsigned BlockID, const NaClBitcodeRecord &Record);
-
-  /// Create block data for block BlockID, using the input cursor.
-  NaClBitcodeBlock(unsigned BlockID, NaClBitstreamCursor &Cursor)
-      : NaClBitcodeData(Cursor),
-        BlockID(BlockID),
-        EnclosingBlock(0)
-  {
-    LocalStartBit = GetStartBit();
-  }
-
-  /// Print the contents out to the given stream.
-  void Print(raw_ostream& os) const;
-
-  /// Returns pointer to the enclosing block.
-  const NaClBitcodeBlock *GetEnclosingBlock() const {
-    return EnclosingBlock;
-  }
-
-  /// Returns the block ID of the block.
-  unsigned GetBlockID() const {
-    return BlockID;
-  }
-
-  /// Returns the number of bits in the block associated with the
-  /// bitcode parser parsing this block, excluding nested blocks.
-  unsigned GetLocalNumBits() const {
-    return GetCursor().GetCurrentBitNo() - LocalStartBit;
-  }
-
-protected:
-  // The block ID associated with this record.
-  unsigned BlockID;
-  // The enclosing block, if defined.
-  const NaClBitcodeBlock *EnclosingBlock;
-  // Start bit for the block, updated to skip nested blocks.
-  uint64_t LocalStartBit;
-
-  // Note: We friend class NaClBitcodeParser, so that it can
-  // update field LocalStartBit.
-  friend class NaClBitcodeParser;
-};
-
-typedef NaClBitcodeRecordVector NaClRecordVector;
-
-class NaClBitcodeRecordData {
-  NaClBitcodeRecordData &operator=(const NaClBitcodeRecordData &) = delete;
-public:
-  NaClBitcodeRecordData(unsigned Code, const NaClRecordVector &Values)
-      : Code(Code), Values(Values) {}
-  explicit NaClBitcodeRecordData(const NaClBitcodeRecordData &Record)
-      : Code(Record.Code), Values(Record.Values) {}
-  NaClBitcodeRecordData() : Code(0) {}
-  // The selector code associated with the record.
-  unsigned Code;
-  // The sequence of values defining the parsed record.
-  NaClRecordVector Values;
-
-  void Print(raw_ostream &strm) const;
-};
-
-inline raw_ostream &operator<<(raw_ostream &Strm,
-                               const NaClBitcodeRecordData &Data) {
-  Data.Print(Strm);
-  return Strm;
-}
-
-/// Simple container class to convert the values of the corresponding
-/// read record to a simpler form, only containing values.
-struct NaClBitcodeValues {
-public:
-  NaClBitcodeValues(const NaClBitcodeRecordData &Record)
-      : Record(Record) {}
-
-  size_t size() const {
-    return Record.Values.size()+1;
-  }
-
-  uint64_t operator[](size_t index) const {
-    return index == 0 ? Record.Code : Record.Values[index-1];
-  }
-
-private:
-  const NaClBitcodeRecordData &Record;
-};
-
-/// Defines the data associated with reading a block record in the
-/// PNaCl bitcode stream.
-class NaClBitcodeRecord : public NaClBitcodeData {
-public:
-  /// Type for vector of values representing a record.
-  typedef NaClRecordVector RecordVector;
-
-  /// Creates a bitcode record, starting at the position defined
-  /// by cursor.
-  explicit NaClBitcodeRecord(const NaClBitcodeBlock &Block)
-      : NaClBitcodeData(Block.GetCursor()),
-        Block(Block)
-  {}
-
-  /// Print the contents out to the given stream.
-  void Print(raw_ostream& os) const;
-
-  /// The block the record appears in.
-  const NaClBitcodeBlock &GetBlock() const {
-    return Block;
-  }
-
-  /// Returns the block ID associated with the record.
-  unsigned GetBlockID() const {
-    return Block.GetBlockID();
-  }
-
-  /// Returns the kind of entry read from the input stream.
-  unsigned GetEntryKind() const {
-    return Entry.Kind;
-  }
-
-  /// Returns the code value (i.e. selector) associated with the
-  /// record.
-  unsigned GetCode() const {
-    return Data.Code;
-  }
-
-  /// Returns the EntryID (e.g. abbreviation if !=
-  /// naclbitc::UNABBREV_RECORD) associated with the record. Note:
-  /// for block-enter, block-exit, and define-abbreviation, EntryID is
-  /// not the corresponding abbreviation.
-  unsigned GetEntryID() const {
-    return Entry.ID;
-  }
-
-  /// Returns the (value) record associated with the read record.
-  const RecordVector &GetValues() const {
-    return Data.Values;
-  }
-
-  /// Allows lower level access to data representing record.
-  const NaClBitcodeRecordData &GetRecordData() const {
-    return Data;
-  }
-
-  /// Returns true if the record was read using an abbreviation.
-  bool UsedAnAbbreviation() const {
-    return GetEntryKind() == NaClBitstreamEntry::Record &&
-        GetEntryID() != naclbitc::UNABBREV_RECORD;
-  }
-
-  /// Returns the abbrevation index used to read the record.
-  /// Returns naclbitc::UNABBREV_RECORD if not applicable.
-  unsigned GetAbbreviationIndex() const {
-    return UsedAnAbbreviation()
-        ? GetEntryID() : static_cast<unsigned>(naclbitc::UNABBREV_RECORD);
-  }
-
-  /// Destructively change the abbreviation ID to the given value.
-  void SetAbbreviationIndex(unsigned Index) {
-    Entry.ID = Index;
-  }
-
-protected:
-  // The block associated with the record.
-  const NaClBitcodeBlock &Block;
-  // The data of the record.
-  NaClBitcodeRecordData Data;
-  // The entry (i.e. value(s) preceding the record that define what
-  // value comes next).
-  NaClBitstreamEntry Entry;
-
-private:
-  // Allows class NaClBitcodeParser to read values into the
-  // record, thereby hiding the details of how to read values.
-  friend class NaClBitcodeParser;
-  friend class NaClBitcodeParserListener;
-
-  /// Read bitstream entry. Defines what construct appears next in the
-  /// bitstream.
-  void ReadEntry() {
-    SetStartBit(GetCursor().GetCurrentBitNo());
-    Entry = GetCursor().
-        advance(NaClBitstreamCursor::AF_DontAutoprocessAbbrevs, 0);
-  }
-
-  /// Reads in a record's values, if the entry defines a record (Must
-  /// be called after ReadEntry).
-  void ReadValues() {
-    Data.Values.clear();
-    Data.Code = GetCursor().readRecord(Entry.ID, Data.Values);
-  }
-
-  NaClBitcodeRecord(const NaClBitcodeRecord &Rcd) = delete;
-  void operator=(const NaClBitcodeRecord &Rcd) = delete;
-};
-
-inline raw_ostream &operator<<(raw_ostream &Strm,
-                               const NaClBitcodeRecord &Record) {
-  Record.Print(Strm);
-  return Strm;
-}
-
-/// Defines a listener to handle abbreviations within a bitcode file.
-/// In particular, abbreviations and the BlockInfo block are made more
-/// explicit, and then sent to the parser through virtuals
-/// ProcessAbbreviation and SetBID.
-class NaClBitcodeParserListener : public NaClAbbrevListener {
-  friend class NaClBitcodeParser;
-public:
-  // Constructs a listener for the given parser.  Note: All nested
-  // parsers automatically inherit this listener.
-  NaClBitcodeParserListener(NaClBitcodeParser *Parser)
-      : Parser(Parser), GlobalBlockID(naclbitc::BLOCKINFO_BLOCK_ID) {
-  }
-
-  virtual ~NaClBitcodeParserListener() {}
-
-private:
-  virtual void BeginBlockInfoBlock(unsigned NumWords);
-
-  virtual void SetBID();
-
-  virtual void EndBlockInfoBlock();
-
-  virtual void ProcessAbbreviation(NaClBitCodeAbbrev *Abbrev,
-                                   bool IsLocal);
-
-  /// The block parser currently being listened to.
-  NaClBitcodeParser *Parser;
-
-  /// The block ID to use if a global abbreviation. Note: This field is
-  /// updated by calls to method SetBID.
-  unsigned GlobalBlockID;
-};
-
-/// Parses a block in the PNaCl bitcode stream.
-class NaClBitcodeParser {
-  // Allow listener privledges, so that it can update/call the parser
-  // using a clean API.
-  friend class NaClBitcodeParserListener;
-
-  // Implements an error handler for errors in the bitstream reader.
-  // Redirects bitstream reader errors to corresponding parrser error
-  // reporting function.
-  class ErrorHandler : public NaClBitstreamCursor::ErrorHandler {
-    NaClBitcodeParser *Parser;
-  public:
-    ErrorHandler(NaClBitcodeParser *Parser,
-                 NaClBitstreamCursor &Cursor):
-        NaClBitstreamCursor::ErrorHandler(Cursor), Parser(Parser) {}
-    LLVM_ATTRIBUTE_NORETURN
-    void Fatal(const std::string &ErrorMessage) const final {
-      Parser->FatalAt(getCurrentBitNo(), ErrorMessage);
-      llvm_unreachable("GCC treats noreturn virtual functions as returning");
-    }
-    ~ErrorHandler() override {}
-  };
-
-public:
-  // Creates a parser to parse the the block at the given cursor in
-  // the PNaCl bitcode stream. This instance is a "dummy" instance
-  // that starts the parser.
-  explicit NaClBitcodeParser(NaClBitstreamCursor &Cursor)
-      : EnclosingParser(0),
-        Block(ILLEGAL_BLOCK_ID, Cursor),
-        Record(Block),
-        Listener(0),
-        ErrStream(&errs()) {
-    std::unique_ptr<NaClBitstreamCursor::ErrorHandler>
-        ErrHandler(new ErrorHandler(this, Cursor));
-    Cursor.setErrorHandler(ErrHandler);
-  }
-
-  virtual ~NaClBitcodeParser();
-
-  /// Reads the (top-level) block associated with the given block
-  /// record at the stream cursor. Returns true if unable to parse.
-  /// Can be called multiple times to parse multiple blocks.
-  bool Parse();
-
-  // Called once the bitstream reader has entered the corresponding
-  // subblock.  Argument NumWords is set to the number of words in the
-  // corresponding subblock.
-  virtual void EnterBlock(unsigned /*NumWords*/) {}
-
-  // Called when the corresponding EndBlock of the block being parsed
-  // is found.
-  virtual void ExitBlock() {}
-
-  // Called after each record (within the block) is read (into field Record).
-  virtual void ProcessRecord() {}
-
-  // Called if a SetBID record is encountered in the BlockInfo block,
-  // and the parser has a listener.
-  virtual void SetBID() {}
-
-  // Called to process an abbreviation if the parser has a listener.
-  virtual void ProcessAbbreviation(unsigned /*BlockID*/,
-                                   NaClBitCodeAbbrev * /*Abbrev*/,
-                                   bool /*IsLocal*/) {}
-
-  // Creates an instance of the NaClBitcodeParser to use to parse the
-  // block with the given block ID, and then call's method
-  // ParseThisBlock() to parse the corresponding block. Note:
-  // Each derived class should define it's own version of this
-  // method, following the pattern below.
-  virtual bool ParseBlock(unsigned BlockID) {
-    // Default implementation just builds a parser that does nothing.
-    NaClBitcodeParser Parser(BlockID, this);
-    return Parser.ParseThisBlock();
-  }
-
-  // Changes the stream to print errors to, and returns the old error stream.
-  // There are two use cases:
-  // 1) To change (from the default errs()) inside the constructor of the
-  //    derived class. In this context, it will be used for all error
-  //    messages for the derived class.
-  // 2) Temporarily modify it for a single error message.
-  raw_ostream &setErrStream(raw_ostream &Stream) {
-    raw_ostream &OldErrStream = *ErrStream;
-    ErrStream = &Stream;
-    return OldErrStream;
-  }
-
-  // Called when an error occurs. BitPosition is the bit position the
-  // error was found, and Message is the error to report. Always
-  // returns true (the error return value of Parse). Level is
-  // the severity of the error.
-  virtual bool ErrorAt(naclbitc::ErrorLevel Level, uint64_t BitPosition,
-                       const std::string &Message);
-
-  bool ErrorAt(uint64_t BitPosition, const std::string &Message) {
-    return ErrorAt(naclbitc::Error, BitPosition, Message);
-  }
-
-  // Called when an error occurs. Message is the error to
-  // report. Always returns true (the error return value of Parse).
-  bool Error(const std::string &Message) {
-    return ErrorAt(Record.GetStartBit(), Message);
-  }
-
-  // Called when a fatal error occurs. BitPosition is the bit position
-  // the error was found, and Message is the error to report. Does not
-  // return.
-  LLVM_ATTRIBUTE_NORETURN
-  void FatalAt(uint64_t BitPosition, const std::string &Message) {
-    ErrorAt(naclbitc::Fatal, BitPosition, Message);
-    llvm_unreachable("Fatal errors should not return");
-  }
-
-  // Called when a fatal error occurs. Message is the error to
-  // report. Does not return.
-  LLVM_ATTRIBUTE_NORETURN
-  void Fatal(const std::string &Message) {
-    FatalAt(Record.GetStartBit(), Message);
-    llvm_unreachable("GCC treats noreturn virtual functions as returning");
-  }
-
-  // Generates fatal generic error message.
-  LLVM_ATTRIBUTE_NORETURN
-  void Fatal() {
-    Fatal("Fatal error occurred!");
-  }
-
-  // Returns the number of bits in this block, including nested blocks.
-  unsigned GetBlockNumBits() const {
-    return Block.GetNumBits();
-  }
-
-  // Returns the number of bits in this block, excluding nested blocks.
-  unsigned GetBlockLocalNumBits() const {
-    return Block.GetLocalNumBits();
-  }
-
-  /// Returns the block ID associated with the Parser.
-  unsigned GetBlockID() const {
-    return Block.GetBlockID();
-  }
-
-  NaClBitcodeBlock &GetBlock() {
-    return Block;
-  }
-
-  /// Returns the enclosing parser of this block.
-  NaClBitcodeParser *GetEnclosingParser() const {
-    // Note: The top-level parser instance is a dummy instance
-    // and is not considered an enclosing parser.
-    return EnclosingParser->EnclosingParser ? EnclosingParser : 0;
-  }
-
-  // Parses the block using the parser defined by
-  // ParseBlock(unsigned).  Returns true if unable to parse the
-  // block. Note: Should only be called by virtual ParseBlock(unsigned).
-  bool ParseThisBlock() {
-    bool Results;
-    if (Listener) {
-      NaClBitcodeParser *CallingParser = Listener->Parser;
-      Listener->Parser = this;
-      Results = ParseThisBlockInternal();
-      Listener->Parser = CallingParser;
-    } else {
-      Results = ParseThisBlockInternal();
-    }
-    return Results;
-  }
-
-  /// Skips the current block, assuming the parser is at the beginning
-  /// of the block. That is, Record.GetEntryKind() equals
-  /// NaClBitstreamEntry::SubBlock. Returns false if
-  /// successful. Otherwise returns 1.
-  bool SkipBlock() {
-    if (Record.GetEntryKind() != NaClBitstreamEntry::SubBlock)
-      return Error("SkipBlock on non-block record");
-    return Record.GetCursor().SkipBlock();
-  }
-
-protected:
-  // The containing parser.
-  NaClBitcodeParser *EnclosingParser;
-
-  // The block the parser is associated with.
-  NaClBitcodeBlock Block;
-
-  // The current record (within the block) being processed.
-  NaClBitcodeRecord Record;
-
-  // The listener (if any) to use.
-  NaClBitcodeParserListener *Listener;
-
-  // The error stream to use if non-null (uses errs() if null).
-  raw_ostream *ErrStream;
-
-  // Creates a block parser to parse the block associated with the
-  // bitcode entry that defines the beginning of a block. This
-  // instance actually parses the corresponding block.
-  NaClBitcodeParser(unsigned BlockID, NaClBitcodeParser *EnclosingParser)
-      : EnclosingParser(EnclosingParser),
-        Block(BlockID, EnclosingParser->Record),
-        Record(Block),
-        Listener(EnclosingParser->Listener),
-        ErrStream(EnclosingParser->ErrStream)
-  {}
-
-  /// Defines the listener for this block, and all enclosing blocks,
-  /// to be the given listener. Should be set in the constructor.
-  void SetListener(NaClBitcodeParserListener* UseListener) {
-    Listener = UseListener;
-  }
-
-private:
-  // Special constant identifying the top-level instance.
-  static const unsigned ILLEGAL_BLOCK_ID = UINT_MAX;
-
-  // Parses the block. Returns true if unable to parse the
-  // block. Note: Should only be called by virtual ParseThisBlock.
-  bool ParseThisBlockInternal() {
-    bool Results;
-    if (GetBlockID() == naclbitc::BLOCKINFO_BLOCK_ID) {
-      Results = ParseBlockInfoInternal();
-    } else {
-      Results = ParseBlockInternal();
-      ExitBlock();
-    }
-    return Results;
-  }
-
-  // Parses a BlockInfo block, where processing is handled through
-  // a listener in the bitstream reader.
-  bool ParseBlockInfoInternal();
-
-  // Parses the non-BlockInfo block. Returns true if unable to parse the
-  // block.
-  bool ParseBlockInternal();
-
-  void operator=(const NaClBitcodeParser &Parser) = delete;
-  NaClBitcodeParser(const NaClBitcodeParser &Parser) = delete;
-
-};
-
-}  // namespace llvm
-
-#endif
diff --git a/include/llvm/Bitcode/NaCl/NaClBitcodeSizeDist.h b/include/llvm/Bitcode/NaCl/NaClBitcodeSizeDist.h
deleted file mode 100644
index 0155d94b65f..00000000000
--- a/include/llvm/Bitcode/NaCl/NaClBitcodeSizeDist.h
+++ /dev/null
@@ -1,78 +0,0 @@
-//===-- NaClBitcodeSizeDist.h ---------------------------------------------===//
-//      Defines distribution maps for bitcode record sizes (arity).
-//
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
-//
-//===----------------------------------------------------------------------===//
-//
-// Defines distribution maps for tracking bitcode record sizes
-// (arity).
-
-#ifndef LLVM_BITCODE_NACL_NACLBITCODESIZEDIST_H
-#define LLVM_BITCODE_NACL_NACLBITCODESIZEDIST_H
-
-#include "llvm/Bitcode/NaCl/NaClBitcodeDist.h"
-#include "llvm/Bitcode/NaCl/NaClBitcodeValueDist.h"
-
-namespace llvm {
-
-/// Collects the number of bitcode record instances with the same number
-/// of elements in the vector of values, with nested value distribution maps.
-class NaClBitcodeSizeDistElement : public NaClBitcodeDistElement {
-  NaClBitcodeSizeDistElement(const NaClBitcodeSizeDistElement&) = delete;
-  void operator=(const NaClBitcodeSizeDistElement&) = delete;
-
-public:
-  static bool classof(const NaClBitcodeDistElement *Element) {
-    return Element->getKind() >= RDE_SizeDist &&
-        Element->getKind() < RDE_SizeDistLast;
-  }
-
-  NaClBitcodeSizeDistElement()
-      : NaClBitcodeDistElement(RDE_SizeDist),
-        ValueIndexDist(NaClBitcodeDist::RecordStorage,
-                       &NaClBitcodeValueIndexDistElement::Sentinel) {
-    NestedDists.push_back(&ValueIndexDist);
-  }
-
-  static NaClBitcodeSizeDistElement Sentinel;
-
-  virtual ~NaClBitcodeSizeDistElement();
-
-  virtual NaClBitcodeDistElement *CreateElement(
-      NaClBitcodeDistValue Value) const;
-
-  virtual void GetValueList(const NaClBitcodeRecord &Record,
-                            ValueListType &ValueList) const;
-
-  virtual void AddRecord(const NaClBitcodeRecord &Record);
-
-  virtual const SmallVectorImpl<NaClBitcodeDist*> *
-  GetNestedDistributions() const;
-
-  virtual const char *GetTitle() const;
-
-  virtual const char *GetValueHeader() const;
-
-  virtual void PrintRowValue(raw_ostream &Stream,
-                             NaClBitcodeDistValue Value,
-                             const NaClBitcodeDist *Distribution) const;
-
-  NaClBitcodeDist &GetValueIndexDist() {
-    return ValueIndexDist;
-  }
-
-private:
-  // Nested blocks used by GetNestedDistributions.
-  SmallVector<NaClBitcodeDist*, 1> NestedDists;
-
-  // The value distributions associated with records of the given size.
-  NaClBitcodeDist ValueIndexDist;
-};
-
-}
-
-#endif
diff --git a/include/llvm/Bitcode/NaCl/NaClBitcodeSubblockDist.h b/include/llvm/Bitcode/NaCl/NaClBitcodeSubblockDist.h
deleted file mode 100644
index 7729edffe42..00000000000
--- a/include/llvm/Bitcode/NaCl/NaClBitcodeSubblockDist.h
+++ /dev/null
@@ -1,76 +0,0 @@
-//===-- NaClBitcodeSubblockDist.h -----------------------------------------===//
-//      Defines distribution maps for subblock values within an
-//      (externally specified) block.
-//
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
-//
-//===----------------------------------------------------------------------===//
-//
-// Defines the notion subblock distribution map. Shows what subblocks appear
-// within a given block (defined externally to the distribution map).
-
-#ifndef LLVM_BITCODE_NACL_NACLBITCODESUBBLOCKDIST_H
-#define LLVM_BITCODE_NACL_NACLBITCODESUBBLOCKDIST_H
-
-#include "llvm/Bitcode/NaCl/NaClBitcodeDist.h"
-
-namespace llvm {
-
-/// Collects the distribution of subblocks within an (externally defined)
-/// block.
-class NaClBitcodeSubblockDistElement : public NaClBitcodeDistElement {
-  NaClBitcodeSubblockDistElement(const NaClBitcodeSubblockDistElement &)
-  = delete;
-  void operator=(const NaClBitcodeSubblockDistElement&);
-
-public:
-  static bool classof(const NaClBitcodeDistElement *Element) {
-    return Element->getKind() >= RDE_SubblockDist &&
-        Element->getKind() < RDE_SubblockDistLast;
-  }
-
-  NaClBitcodeSubblockDistElement()
-      : NaClBitcodeDistElement(RDE_SubblockDist) {}
-
-  virtual ~NaClBitcodeSubblockDistElement();
-
-  virtual NaClBitcodeDistElement *CreateElement(
-      NaClBitcodeDistValue Value) const;
-
-  virtual const char *GetTitle() const;
-
-  virtual const char *GetValueHeader() const;
-
-  virtual void PrintRowValue(raw_ostream &Stream,
-                             NaClBitcodeDistValue Value,
-                             const NaClBitcodeDist *Distribution) const;
-};
-
-/// Collects the distribution of subblocks within an (externally
-/// defined) block. Assumes distribution elements are instances of
-/// NaClBitcodeSubblockDistElement.
-class NaClBitcodeSubblockDist : public NaClBitcodeDist {
-  NaClBitcodeSubblockDist(const NaClBitcodeSubblockDist&) = delete;
-  void operator=(const NaClBitcodeSubblockDist&) = delete;
-
-public:
-  static bool classof(const NaClBitcodeDist *Dist) {
-    return Dist->getKind() >= RD_SubblockDist &&
-        Dist->getKind() < RD_SubblockDistLast;
-  }
-
-  static NaClBitcodeSubblockDistElement DefaultSentinal;
-
-  NaClBitcodeSubblockDist()
-      : NaClBitcodeDist(BlockStorage, &DefaultSentinal, RD_SubblockDist)
-  {}
-
-  virtual ~NaClBitcodeSubblockDist();
-};
-
-}
-
-#endif
diff --git a/include/llvm/Bitcode/NaCl/NaClBitcodeValueDist.h b/include/llvm/Bitcode/NaCl/NaClBitcodeValueDist.h
deleted file mode 100644
index c992627f5e6..00000000000
--- a/include/llvm/Bitcode/NaCl/NaClBitcodeValueDist.h
+++ /dev/null
@@ -1,182 +0,0 @@
-//===-- NaClBitcodeValueDist.h ---------------------------------------------===//
-//      Defines distribution maps to separate out values at each index
-//      in a bitcode record.
-//
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
-//
-//===----------------------------------------------------------------------===//
-
-#ifndef LLVM_BITCODE_NACL_NACLBITCODEVALUEDIST_H
-#define LLVM_BITCODE_NACL_NACLBITCODEVALUEDIST_H
-
-#include "llvm/Bitcode/NaCl/NaClBitcodeDist.h"
-
-namespace llvm {
-
-/// Defines the value index cutoff where we no longer track values associated
-/// with specific value indices of bitcode records.
-static const unsigned NaClValueIndexCutoff = 6;
-
-/// Defines an (inclusive) range of values in a bitcode record,
-/// defined by the given pair of values. Note that values are stored
-/// as ranges. For small values, each value is in a separate range, so
-/// that potential constants for abbreviations can be found. For
-/// larger values, values are coalesced together into multiple element
-/// ranges, since we don't look for constants and are only interested
-/// in the overall distribution of values.
-typedef std::pair<NaClBitcodeDistValue,
-                  NaClBitcodeDistValue> NaClValueRangeType;
-
-/// Models a range index. Ranges are encoded as a consecutive sequence
-/// of indices, starting at zero. The actual ranges chosen to
-/// represent bitcode records are internal, and is defined by function
-/// GetValueRange index below.
-typedef NaClBitcodeDistValue NaClValueRangeIndexType;
-
-/// Converts a bitcode record value to the corresponding range index that
-/// contains the value.
-NaClValueRangeIndexType GetNaClValueRangeIndex(NaClBitcodeDistValue Value);
-
-/// Converts a range index into the corresponding range of values.
-NaClValueRangeType GetNaClValueRange(NaClValueRangeIndexType RangeIndex);
-
-/// Defines the distribution of range indices.
-class NaClBitcodeValueDistElement : public NaClBitcodeDistElement {
-  NaClBitcodeValueDistElement(const NaClBitcodeValueDistElement&) = delete;
-  void operator=(const NaClBitcodeValueDistElement&) = delete;
-
-public:
-  static bool classof(const NaClBitcodeDistElement *Element) {
-    return Element->getKind() >= RDE_ValueDist &&
-        Element->getKind() < RDE_ValueDistLast;
-  }
-
-  NaClBitcodeValueDistElement()
-      : NaClBitcodeDistElement(RDE_ValueDist)
-  {}
-
-  static NaClBitcodeValueDistElement Sentinel;
-
-  virtual ~NaClBitcodeValueDistElement();
-
-  virtual NaClBitcodeDistElement *CreateElement(
-      NaClBitcodeDistValue Value) const;
-
-  /// Returns the number of instances, normalized over the
-  /// range of values, using a uniform distribution.
-  virtual double GetImportance(NaClBitcodeDistValue Value) const;
-
-  virtual const char *GetTitle() const;
-
-  virtual const char *GetValueHeader() const;
-
-  virtual void PrintRowValue(raw_ostream &Stream,
-                             NaClBitcodeDistValue Value,
-                             const NaClBitcodeDist *Distribution) const;
-};
-
-/// Defines the distribution of values for a set of value indices for
-/// bitcode records.
-class NaClBitcodeValueDist : public NaClBitcodeDist {
-  NaClBitcodeValueDist(const NaClBitcodeValueDist&) = delete;
-  void operator=(const NaClBitcodeValueDist&) = delete;
-
-public:
-  static bool classof(const NaClBitcodeDist *Dist) {
-    return Dist->getKind() >= RD_ValueDist &&
-        Dist->getKind() < RD_ValueDistLast;
-  }
-
-  /// Builds a value distribution for the given set of value indices.
-  /// If AllRemainingIndices is false, only value Index is considered.
-  /// Otherwise, builds a value distribution for all values stored in
-  /// record value indices >= Index.
-  explicit NaClBitcodeValueDist(unsigned Index,
-                                bool AllRemainingIndices=false);
-
-  virtual ~NaClBitcodeValueDist();
-
-  unsigned GetIndex() const {
-    return Index;
-  }
-
-  bool HoldsAllRemainingIndices() const {
-    return AllRemainingIndices;
-  }
-
-  virtual void GetValueList(const NaClBitcodeRecord &Record,
-                            ValueListType &ValueList) const;
-
-private:
-  // The range index being tracked.
-  unsigned Index;
-
-  // If true, then tracks all indices >= Index. Otherwise only Index.
-  bool AllRemainingIndices;
-};
-
-/// Defines the value distribution for each value index in corresponding
-/// bitcode records. This is a helper class used to separate each element
-/// in the bitcode record, so that we can determine the proper abbreviation
-/// for each element.
-class NaClBitcodeValueIndexDistElement : public NaClBitcodeDistElement {
-  NaClBitcodeValueIndexDistElement(
-      const NaClBitcodeValueIndexDistElement&) = delete;
-  void operator=(
-      const NaClBitcodeValueIndexDistElement&) = delete;
-
-public:
-  static bool classof(const NaClBitcodeDistElement *Element) {
-    return Element->getKind() <= RDE_ValueIndexDist &&
-        Element->getKind() < RDE_ValueIndexDistLast;
-  }
-
-  explicit NaClBitcodeValueIndexDistElement(unsigned Index=0)
-      : NaClBitcodeDistElement(RDE_ValueIndexDist),
-        ValueDist(Index) {
-    NestedDists.push_back(&ValueDist);
-  }
-
-  static NaClBitcodeValueIndexDistElement Sentinel;
-
-  virtual ~NaClBitcodeValueIndexDistElement();
-
-  virtual NaClBitcodeDistElement *CreateElement(
-      NaClBitcodeDistValue Value) const;
-
-  virtual void GetValueList(const NaClBitcodeRecord &Record,
-                            ValueListType &ValueList) const;
-
-  virtual double GetImportance(NaClBitcodeDistValue Value) const;
-
-  virtual void AddRecord(const NaClBitcodeRecord &Record);
-
-  virtual const char *GetTitle() const;
-
-  virtual const char *GetValueHeader() const;
-
-  virtual void PrintRowValue(raw_ostream &Stream,
-                             NaClBitcodeDistValue Value,
-                             const NaClBitcodeDist *Distribution) const;
-
-  virtual const SmallVectorImpl<NaClBitcodeDist*> *
-  GetNestedDistributions() const;
-
-  NaClBitcodeValueDist &GetValueDist() {
-    return ValueDist;
-  }
-
-private:
-  // Nested blocks used by GetNestedDistributions.
-  SmallVector<NaClBitcodeDist*, 1> NestedDists;
-
-  // The value distribution associated with the given index.
-  NaClBitcodeValueDist ValueDist;
-};
-
-}
-
-#endif
diff --git a/include/llvm/Bitcode/NaCl/NaClBitcodeWriterPass.h b/include/llvm/Bitcode/NaCl/NaClBitcodeWriterPass.h
deleted file mode 100644
index 6a7cf71a9c7..00000000000
--- a/include/llvm/Bitcode/NaCl/NaClBitcodeWriterPass.h
+++ /dev/null
@@ -1,51 +0,0 @@
-//===-- NaClBitcodeWriterPass.h - Bitcode writing pass ----------*- C++ -*-===//
-//
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
-//
-//===----------------------------------------------------------------------===//
-/// \file
-///
-/// This file provides a bitcode writing pass.
-///
-//===----------------------------------------------------------------------===//
-
-#ifndef LLVM_BITCODE_NACL_BITCODE_WRITER_PASS_H
-#define LLVM_BITCODE_NACL_BITCODE_WRITER_PASS_H
-
-#include "llvm/ADT/StringRef.h"
-
-namespace llvm {
-class Module;
-class ModulePass;
-class raw_ostream;
-class PreservedAnalyses;
-
-/// \brief Create and return a pass that writes the module to the specified
-/// ostream. Note that this pass is designed for use with the legacy pass
-/// manager.
-ModulePass *createNaClBitcodeWriterPass(raw_ostream &Str);
-
-/// \brief Pass for writing a module of IR out to a bitcode file.
-///
-/// Note that this is intended for use with the new pass manager. To construct
-/// a pass for the legacy pass manager, use the function above.
-class NaClBitcodeWriterPass {
-  raw_ostream &OS;
-
-public:
-  /// \brief Construct a bitcode writer pass around a particular output stream.
-  explicit NaClBitcodeWriterPass(raw_ostream &OS) : OS(OS) {}
-
-  /// \brief Run the bitcode writer pass, and output the module to the selected
-  /// output stream.
-  PreservedAnalyses run(Module *M);
-
-  static StringRef name() { return "NaClBitcodeWriterPass"; }
-};
-
-}
-
-#endif
diff --git a/include/llvm/Bitcode/NaCl/NaClBitstreamReader.h b/include/llvm/Bitcode/NaCl/NaClBitstreamReader.h
deleted file mode 100644
index 05cd91eb2aa..00000000000
--- a/include/llvm/Bitcode/NaCl/NaClBitstreamReader.h
+++ /dev/null
@@ -1,688 +0,0 @@
-//===- NaClBitstreamReader.h -----------------------------------*- C++ -*-===//
-//     Low-level bitstream reader interface
-//
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
-//
-//===----------------------------------------------------------------------===//
-//
-// This header defines the BitstreamReader class.  This class can be used to
-// read an arbitrary bitstream, regardless of its contents.
-//
-//===----------------------------------------------------------------------===//
-
-#ifndef LLVM_BITCODE_NACL_NACLBITSTREAMREADER_H
-#define LLVM_BITCODE_NACL_NACLBITSTREAMREADER_H
-
-#include "llvm/ADT/SmallVector.h"
-#include "llvm/Bitcode/NaCl/NaClBitcodeHeader.h"
-#include "llvm/Bitcode/NaCl/NaClLLVMBitCodes.h"
-#include "llvm/Support/Endian.h"
-#include "llvm/Support/StreamingMemoryObject.h"
-#include <climits>
-#include <vector>
-
-namespace llvm {
-
-class Deserializer;
-
-namespace naclbitc {
-
-/// Returns the Bit as a Byte:BitInByte string.
-std::string getBitAddress(uint64_t Bit);
-
-/// Severity levels for reporting errors.
-enum ErrorLevel {
-  Warning,
-  Error,
-  Fatal
-};
-
-// Basic printing routine to generate the beginning of an error
-// message. BitPosition is the bit position the error was found.
-// Level is the severity of the error.
-raw_ostream &ErrorAt(raw_ostream &Out, ErrorLevel Level,
-                     uint64_t BitPosition);
-
-} // End namespace naclbitc.
-
-/// This class is used to read from a NaCl bitcode wire format stream,
-/// maintaining information that is global to decoding the entire file.
-/// While a file is being read, multiple cursors can be independently
-/// advanced or skipped around within the file.  These are represented by
-/// the NaClBitstreamCursor class.
-class NaClBitstreamReader {
-public:
-  /// This contains information emitted to BLOCKINFO_BLOCK blocks. These
-  /// describe abbreviations that all blocks of the specified ID inherit.
-  struct BlockInfo {
-    unsigned BlockID;
-    std::vector<NaClBitCodeAbbrev*> Abbrevs;
-  };
-private:
-  friend class NaClBitstreamCursor;
-
-  std::unique_ptr<MemoryObject> BitcodeBytes;
-
-  std::vector<BlockInfo> BlockInfoRecords;
-
-  /// \brief Holds the offset of the first byte after the header.
-  size_t InitialAddress;
-
-  // True if filler should be added to byte align records.
-  bool AlignBitcodeRecords = false;
-  NaClBitstreamReader(const NaClBitstreamReader&) = delete;
-  void operator=(const NaClBitstreamReader&) = delete;
-
-
-  void initFromHeader(NaClBitcodeHeader &Header) {
-    InitialAddress = Header.getHeaderSize();
-    AlignBitcodeRecords = Header.getAlignBitcodeRecords();
-  }
-
-public:
-  /// Read stream from sequence of bytes [Start .. End) after parsing
-  /// the given bitcode header.
-  NaClBitstreamReader(const unsigned char *Start, const unsigned char *End,
-                      NaClBitcodeHeader &Header)
-      : BitcodeBytes(getNonStreamedMemoryObject(Start, End)) {
-    initFromHeader(Header);
-  }
-
-  /// Read stream from Bytes, after parsing the given bitcode header.
-  NaClBitstreamReader(MemoryObject *Bytes, NaClBitcodeHeader &Header)
-      : BitcodeBytes(Bytes) {
-    initFromHeader(Header);
-  }
-
-  /// Read stream from bytes, starting at the given initial address.
-  /// Provides simple API for unit testing.
-  NaClBitstreamReader(MemoryObject *Bytes, size_t InitialAddress)
-      : BitcodeBytes(Bytes), InitialAddress(InitialAddress) {
-  }
-
-  // Returns the memory object that is being read.
-  MemoryObject &getBitcodeBytes() { return *BitcodeBytes; }
-
-  ~NaClBitstreamReader() {
-    // Free the BlockInfoRecords.
-    while (!BlockInfoRecords.empty()) {
-      BlockInfo &Info = BlockInfoRecords.back();
-      // Free blockinfo abbrev info.
-      for (unsigned i = 0, e = static_cast<unsigned>(Info.Abbrevs.size());
-           i != e; ++i)
-        Info.Abbrevs[i]->dropRef();
-      BlockInfoRecords.pop_back();
-    }
-  }
-
-  /// \brief Returns the initial address (after the header) of the input stream.
-  size_t getInitialAddress() const {
-    return InitialAddress;
-  }
-
-  //===--------------------------------------------------------------------===//
-  // Block Manipulation
-  //===--------------------------------------------------------------------===//
-
-  /// Return true if we've already read and processed the block info block for
-  /// this Bitstream. We only process it for the first cursor that walks over
-  /// it.
-  bool hasBlockInfoRecords() const { return !BlockInfoRecords.empty(); }
-
-  /// If there is block info for the specified ID, return it, otherwise return
-  /// null.
-  const BlockInfo *getBlockInfo(unsigned BlockID) const {
-    // Common case, the most recent entry matches BlockID.
-    if (!BlockInfoRecords.empty() && BlockInfoRecords.back().BlockID == BlockID)
-      return &BlockInfoRecords.back();
-
-    for (unsigned i = 0, e = static_cast<unsigned>(BlockInfoRecords.size());
-         i != e; ++i)
-      if (BlockInfoRecords[i].BlockID == BlockID)
-        return &BlockInfoRecords[i];
-    return nullptr;
-  }
-
-  BlockInfo &getOrCreateBlockInfo(unsigned BlockID) {
-    if (const BlockInfo *BI = getBlockInfo(BlockID))
-      return *const_cast<BlockInfo*>(BI);
-
-    // Otherwise, add a new record.
-    BlockInfoRecords.push_back(BlockInfo());
-    BlockInfoRecords.back().BlockID = BlockID;
-    return BlockInfoRecords.back();
-  }
-};
-
-/// When advancing through a bitstream cursor, each advance can discover a few
-/// different kinds of entries:
-struct NaClBitstreamEntry {
-  enum {
-    Error,    // Malformed bitcode was found.
-    EndBlock, // We've reached the end of the current block, (or the end of the
-              // file, which is treated like a series of EndBlock records.
-    SubBlock, // This is the start of a new subblock of a specific ID.
-    Record    // This is a record with a specific AbbrevID.
-  } Kind;
-
-  unsigned ID;
-
-  static NaClBitstreamEntry getError() {
-    NaClBitstreamEntry E; E.Kind = Error; return E;
-  }
-  static NaClBitstreamEntry getEndBlock() {
-    NaClBitstreamEntry E; E.Kind = EndBlock; return E;
-  }
-  static NaClBitstreamEntry getSubBlock(unsigned ID) {
-    NaClBitstreamEntry E; E.Kind = SubBlock; E.ID = ID; return E;
-  }
-  static NaClBitstreamEntry getRecord(unsigned AbbrevID) {
-    NaClBitstreamEntry E; E.Kind = Record; E.ID = AbbrevID; return E;
-  }
-};
-
-/// Models default view of a bitcode record.
-typedef SmallVector<uint64_t, 8> NaClBitcodeRecordVector;
-
-/// Class NaClAbbrevListener is used to allow instances of class
-/// NaClBitcodeParser to listen to record details when processing
-/// abbreviations. The major reason for using a listener is that the
-/// NaCl bitcode reader would require a major rewrite (including the
-/// introduction of more overhead) if we were to lift abbreviations up
-/// to the bitcode reader. That is, not only would we have to lift the
-/// block processing up into the readers (i.e. many blocks in
-/// NaClBitcodeReader and NaClBitcodeParser), but add many new API's
-/// to allow the readers to update internals of the bit stream reader
-/// appropriately.
-class NaClAbbrevListener {
-  NaClAbbrevListener(const NaClAbbrevListener&) = delete;
-  void operator=(const NaClAbbrevListener&) = delete;
-public:
-  NaClAbbrevListener() {}
-  virtual ~NaClAbbrevListener() {}
-
-  /// Called to process the read abbreviation.
-  virtual void ProcessAbbreviation(NaClBitCodeAbbrev *Abbrev,
-                                   bool IsLocal) = 0;
-
-  /// Called after entering block. NumWords is the number of words
-  /// in the block.
-  virtual void BeginBlockInfoBlock(unsigned NumWords) = 0;
-
-  /// Called if a naclbitc::BLOCKINFO_CODE_SETBID record is found in
-  /// NaClBitstreamCursor::ReadBlockInfoBlock.
-  virtual void SetBID() = 0;
-
-  /// Called just before an EndBlock record is processed by
-  /// NaClBitstreamCursor::ReadBlockInfoBlock
-  virtual void EndBlockInfoBlock() = 0;
-
-  /// The values of the bitcode record associated with the called
-  /// virtual function.
-  NaClBitcodeRecordVector Values;
-
-  /// Start bit for current record being processed in
-  /// NaClBitstreamCursor::ReadBlockInfoBlock.
-  uint64_t StartBit;
-};
-
-/// This represents a position within a bitcode file. There may be multiple
-/// independent cursors reading within one bitstream, each maintaining their
-/// own local state.
-///
-/// Unlike iterators, NaClBitstreamCursors are heavy-weight objects
-/// that should not be passed by value.
-class NaClBitstreamCursor {
-public:
-  /// This class handles errors in the bitstream reader. Redirects
-  /// fatal error messages to virtual method Fatal.
-  class ErrorHandler {
-    ErrorHandler(const ErrorHandler &) = delete;
-    ErrorHandler &operator=(const ErrorHandler &) = delete;
-  public:
-    explicit ErrorHandler(NaClBitstreamCursor &Cursor) : Cursor(Cursor) {}
-    LLVM_ATTRIBUTE_NORETURN
-    virtual void Fatal(const std::string &ErrorMessage) const;
-    virtual ~ErrorHandler() {}
-    uint64_t getCurrentBitNo() const {
-      return Cursor.GetCurrentBitNo();
-    }
-  private:
-    NaClBitstreamCursor &Cursor;
-  };
-
-private:
-  friend class Deserializer;
-  NaClBitstreamReader *BitStream;
-  size_t NextChar;
-  // The current error handler for the bitstream reader.
-  std::unique_ptr<ErrorHandler> ErrHandler;
-
-  // The size of the bitcode. 0 if we don't know it yet.
-  size_t Size;
-
-  /// This is the current data we have pulled from the stream but have not
-  /// returned to the client. This is specifically and intentionally defined to
-  /// follow the word size of the host machine for efficiency. We use word_t in
-  /// places that are aware of this to make it perfectly explicit what is going
-  /// on.
-  typedef size_t word_t;
-  word_t CurWord;
-
-  /// This is the number of bits in CurWord that are valid. This
-  /// is always from [0...bits_of(word_t)-1] inclusive.
-  unsigned BitsInCurWord;
-
-  /// This is the declared size of code values used for the current
-  /// block, in bits.
-  NaClBitcodeSelectorAbbrev CurCodeSize;
-
-  /// Abbrevs installed in this block.
-  std::vector<NaClBitCodeAbbrev*> CurAbbrevs;
-
-  struct Block {
-    NaClBitcodeSelectorAbbrev PrevCodeSize;
-    std::vector<NaClBitCodeAbbrev*> PrevAbbrevs;
-    Block() : PrevCodeSize() {}
-    explicit Block(const NaClBitcodeSelectorAbbrev& PCS)
-        : PrevCodeSize(PCS) {}
-  };
-
-  /// This tracks the codesize of parent blocks.
-  SmallVector<Block, 8> BlockScope;
-
-  NaClBitstreamCursor(const NaClBitstreamCursor &) = delete;
-  NaClBitstreamCursor &operator=(const NaClBitstreamCursor &) = delete;
-
-public:
-  NaClBitstreamCursor() : ErrHandler(new ErrorHandler(*this)) {
-    init(nullptr);
-  }
-
-  explicit NaClBitstreamCursor(NaClBitstreamReader &R)
-      : ErrHandler(new ErrorHandler(*this)) { init(&R); }
-
-  void init(NaClBitstreamReader *R) {
-    freeState();
-    BitStream = R;
-    NextChar = (BitStream == nullptr) ? 0 : BitStream->getInitialAddress();
-    Size = 0;
-    BitsInCurWord = 0;
-  }
-
-  ~NaClBitstreamCursor() {
-    freeState();
-  }
-
-  void freeState();
-
-  // Replaces the current bitstream error handler with the new
-  // handler. Takes ownership of the new handler and deletes it when
-  // it is no longer needed.
-  void setErrorHandler(std::unique_ptr<ErrorHandler> &NewHandler) {
-    ErrHandler = std::move(NewHandler);
-  }
-
-  bool canSkipToPos(size_t pos) const {
-    // pos can be skipped to if it is a valid address or one byte past the end.
-    return pos == 0 || BitStream->getBitcodeBytes().isValidAddress(
-        static_cast<uint64_t>(pos - 1));
-  }
-
-  bool AtEndOfStream() {
-    if (BitsInCurWord != 0)
-      return false;
-    if (Size != 0)
-      return Size == NextChar;
-    fillCurWord();
-    return BitsInCurWord == 0;
-  }
-
-  /// Return the number of bits used to encode an abbrev #.
-  unsigned getAbbrevIDWidth() const { return CurCodeSize.NumBits; }
-
-  /// Return the bit # of the bit we are reading.
-  uint64_t GetCurrentBitNo() const {
-    return NextChar*CHAR_BIT - BitsInCurWord;
-  }
-
-  NaClBitstreamReader *getBitStreamReader() {
-    return BitStream;
-  }
-  const NaClBitstreamReader *getBitStreamReader() const {
-    return BitStream;
-  }
-
-  /// Returns the current bit address (string) of the bit cursor.
-  std::string getCurrentBitAddress() const {
-    return naclbitc::getBitAddress(GetCurrentBitNo());
-  }
-
-  /// Flags that modify the behavior of advance().
-  enum {
-    /// If this flag is used, the advance() method does not automatically pop
-    /// the block scope when the end of a block is reached.
-    AF_DontPopBlockAtEnd = 1,
-
-    /// If this flag is used, abbrev entries are returned just like normal
-    /// records.
-    AF_DontAutoprocessAbbrevs = 2
-  };
-
-  /// Advance the current bitstream, returning the next entry in the stream.
-  /// Use the given abbreviation listener (if provided).
-  NaClBitstreamEntry advance(unsigned Flags, NaClAbbrevListener *Listener) {
-    while (1) {
-      unsigned Code = ReadCode();
-      if (Code == naclbitc::END_BLOCK) {
-        // Pop the end of the block unless Flags tells us not to.
-        if (!(Flags & AF_DontPopBlockAtEnd) && ReadBlockEnd())
-          return NaClBitstreamEntry::getError();
-        return NaClBitstreamEntry::getEndBlock();
-      }
-
-      if (Code == naclbitc::ENTER_SUBBLOCK)
-        return NaClBitstreamEntry::getSubBlock(ReadSubBlockID());
-
-      if (Code == naclbitc::DEFINE_ABBREV &&
-          !(Flags & AF_DontAutoprocessAbbrevs)) {
-        // We read and accumulate abbrev's, the client can't do anything with
-        // them anyway.
-        ReadAbbrevRecord(true, Listener);
-        continue;
-      }
-
-      return NaClBitstreamEntry::getRecord(Code);
-    }
-  }
-
-  /// This is a convenience function for clients that don't expect any
-  /// subblocks. This just skips over them automatically.
-  NaClBitstreamEntry advanceSkippingSubblocks(unsigned Flags = 0) {
-    while (1) {
-      // If we found a normal entry, return it.
-      NaClBitstreamEntry Entry = advance(Flags, 0);
-      if (Entry.Kind != NaClBitstreamEntry::SubBlock)
-        return Entry;
-
-      // If we found a sub-block, just skip over it and check the next entry.
-      if (SkipBlock())
-        return NaClBitstreamEntry::getError();
-    }
-  }
-
-  /// Reset the stream to the specified bit number.
-  void JumpToBit(uint64_t BitNo) {
-    uintptr_t ByteNo = uintptr_t(BitNo/8) & ~(sizeof(word_t)-1);
-    unsigned WordBitNo = unsigned(BitNo & (sizeof(word_t)*8-1));
-    if (!canSkipToPos(ByteNo))
-      reportInvalidJumpToBit(BitNo);
-
-    // Move the cursor to the right word.
-    NextChar = ByteNo;
-    BitsInCurWord = 0;
-
-    // Skip over any bits that are already consumed.
-    if (WordBitNo)
-      Read(WordBitNo);
-  }
-
-  void fillCurWord() {
-    assert(Size == 0 || NextChar < (unsigned)Size);
-
-    // Read the next word from the stream.
-    uint8_t Array[sizeof(word_t)] = {0};
-
-    uint64_t BytesRead =
-        BitStream->getBitcodeBytes().readBytes(Array, sizeof(Array), NextChar);
-
-    // If we run out of data, stop at the end of the stream.
-    if (BytesRead == 0) {
-      Size = NextChar;
-      return;
-    }
-
-    CurWord =
-        support::endian::read<word_t, support::little, support::unaligned>(
-            Array);
-    NextChar += BytesRead;
-    BitsInCurWord = BytesRead * 8;
-  }
-
-  word_t Read(unsigned NumBits) {
-    static const unsigned BitsInWord = sizeof(word_t) * 8;
-
-    assert(NumBits && NumBits <= BitsInWord &&
-           "Cannot return zero or more than BitsInWord bits!");
-
-    static const unsigned Mask = sizeof(word_t) > 4 ? 0x3f : 0x1f;
-
-    // If the field is fully contained by CurWord, return it quickly.
-    if (BitsInCurWord >= NumBits) {
-      word_t R = CurWord & (~word_t(0) >> (BitsInWord - NumBits));
-
-      // Use a mask to avoid undefined behavior.
-      CurWord >>= (NumBits & Mask);
-
-      BitsInCurWord -= NumBits;
-      return R;
-    }
-
-    word_t R = BitsInCurWord ? CurWord : 0;
-    unsigned BitsLeft = NumBits - BitsInCurWord;
-
-    fillCurWord();
-
-    // If we run out of data, stop at the end of the stream.
-    if (BitsLeft > BitsInCurWord)
-      return 0;
-
-    word_t R2 = CurWord & (~word_t(0) >> (BitsInWord - BitsLeft));
-
-    // Use a mask to avoid undefined behavior.
-    CurWord >>= (BitsLeft & Mask);
-
-    BitsInCurWord -= BitsLeft;
-
-    R |= R2 << (NumBits - BitsLeft);
-
-    return R;
-  }
-
-  uint32_t ReadVBR(unsigned NumBits) {
-    uint32_t Piece = Read(NumBits);
-    if ((Piece & (1U << (NumBits-1))) == 0)
-      return Piece;
-
-    uint32_t Result = 0;
-    unsigned NextBit = 0;
-    while (1) {
-      Result |= (Piece & ((1U << (NumBits-1))-1)) << NextBit;
-
-      if ((Piece & (1U << (NumBits-1))) == 0)
-        return Result;
-
-      NextBit += NumBits-1;
-      Piece = Read(NumBits);
-    }
-  }
-
-  // Read a VBR that may have a value up to 64-bits in size. The chunk size of
-  // the VBR must still be <= 32 bits though.
-  uint64_t ReadVBR64(unsigned NumBits) {
-    uint32_t Piece = Read(NumBits);
-    if ((Piece & (1U << (NumBits-1))) == 0)
-      return uint64_t(Piece);
-
-    uint64_t Result = 0;
-    unsigned NextBit = 0;
-    while (1) {
-      Result |= uint64_t(Piece & ((1U << (NumBits-1))-1)) << NextBit;
-
-      if ((Piece & (1U << (NumBits-1))) == 0)
-        return Result;
-
-      NextBit += NumBits-1;
-      Piece = Read(NumBits);
-    }
-  }
-
-private:
-  void SkipToByteBoundary() {
-    unsigned BitsToSkip = BitsInCurWord % CHAR_BIT;
-    if (BitsToSkip) {
-      CurWord >>= BitsToSkip;
-      BitsInCurWord -= BitsToSkip;
-    }
-  }
-
-  void SkipToByteBoundaryIfAligned() {
-    if (BitStream->AlignBitcodeRecords)
-      SkipToByteBoundary();
-  }
-
-  void SkipToFourByteBoundary() {
-    // If word_t is 64-bits and if we've read less than 32 bits, just dump
-    // the bits we have up to the next 32-bit boundary.
-    if (sizeof(word_t) > 4 &&
-        BitsInCurWord >= 32) {
-      CurWord >>= BitsInCurWord-32;
-      BitsInCurWord = 32;
-      return;
-    }
-
-    BitsInCurWord = 0;
-  }
-public:
-
-  unsigned ReadCode() {
-    return CurCodeSize.IsFixed
-        ? Read(CurCodeSize.NumBits)
-        : ReadVBR(CurCodeSize.NumBits);
-  }
-
-  // Block header:
-  //    [ENTER_SUBBLOCK, blockid, newcodelen, <align4bytes>, blocklen]
-
-  /// Having read the ENTER_SUBBLOCK code, read the BlockID for the block.
-  unsigned ReadSubBlockID() {
-    return ReadVBR(naclbitc::BlockIDWidth);
-  }
-
-  /// Having read the ENTER_SUBBLOCK abbrevid and a BlockID, skip over the body
-  /// of this block. If the block record is malformed, return true.
-  bool SkipBlock() {
-    // Read and ignore the codelen value.  Since we are skipping this block, we
-    // don't care what code widths are used inside of it.
-    ReadVBR(naclbitc::CodeLenWidth);
-    SkipToFourByteBoundary();
-    unsigned NumFourBytes = Read(naclbitc::BlockSizeWidth);
-
-    // Check that the block wasn't partially defined, and that the offset isn't
-    // bogus.
-    size_t SkipTo = GetCurrentBitNo() + NumFourBytes*4*8;
-    if (AtEndOfStream() || !canSkipToPos(SkipTo/8))
-      return true;
-
-    JumpToBit(SkipTo);
-    return false;
-  }
-
-  /// Having read the ENTER_SUBBLOCK abbrevid, enter the block, and return true
-  /// if the block has an error.
-  bool EnterSubBlock(unsigned BlockID, unsigned *NumWordsP = nullptr);
-
-  bool ReadBlockEnd() {
-    if (BlockScope.empty()) return true;
-
-    // Block tail:
-    //    [END_BLOCK, <align4bytes>]
-    SkipToFourByteBoundary();
-
-    popBlockScope();
-    return false;
-  }
-
-private:
-
-  void popBlockScope() {
-    CurCodeSize = BlockScope.back().PrevCodeSize;
-
-    // Delete abbrevs from popped scope.
-    for (unsigned i = 0, e = static_cast<unsigned>(CurAbbrevs.size());
-         i != e; ++i)
-      CurAbbrevs[i]->dropRef();
-
-    BlockScope.back().PrevAbbrevs.swap(CurAbbrevs);
-    BlockScope.pop_back();
-  }
-
-  //===--------------------------------------------------------------------===//
-  // Record Processing
-  //===--------------------------------------------------------------------===//
-
-private:
-  // Returns abbreviation encoding associated with Value.
-  NaClBitCodeAbbrevOp::Encoding getEncoding(uint64_t Value);
-
-  void skipAbbreviatedField(const NaClBitCodeAbbrevOp &Op);
-
-  // Reads the next Value using the abbreviation Op. Returns true only
-  // if Op is an array (and sets Value to the number of elements in the
-  // array).
-  inline bool readRecordAbbrevField(const NaClBitCodeAbbrevOp &Op,
-                                    uint64_t &Value);
-
-  // Reads and returns the next value using the abbreviation Op,
-  // assuming Op appears after an array abbreviation.
-  inline uint64_t readArrayAbbreviatedField(const NaClBitCodeAbbrevOp &Op);
-
-  // Reads the array abbreviation Op, NumArrayElements times, putting
-  // the read values in Vals.
-  inline void readArrayAbbrev(const NaClBitCodeAbbrevOp &Op,
-                              unsigned NumArrayElements,
-                              SmallVectorImpl<uint64_t> &Vals);
-
-  // Reports that that abbreviation Index is not valid.
-  void reportInvalidAbbrevNumber(unsigned Index) const;
-
-  // Reports that jumping to Bit is not valid.
-  void reportInvalidJumpToBit(uint64_t Bit) const;
-
-public:
-
-  /// Return the abbreviation for the specified AbbrevId.
-  const NaClBitCodeAbbrev *getAbbrev(unsigned AbbrevID) const {
-    unsigned AbbrevNo = AbbrevID-naclbitc::FIRST_APPLICATION_ABBREV;
-    if (AbbrevNo >= CurAbbrevs.size())
-      reportInvalidAbbrevNumber(AbbrevID);
-    return CurAbbrevs[AbbrevNo];
-  }
-
-  /// Read the current record and discard it.
-  void skipRecord(unsigned AbbrevID);
-  
-  unsigned readRecord(unsigned AbbrevID, SmallVectorImpl<uint64_t> &Vals);
-
-  //===--------------------------------------------------------------------===//
-  // Abbrev Processing
-  //===--------------------------------------------------------------------===//
-  // IsLocal indicates where the abbreviation occurs. If it is in the
-  // BlockInfo block, IsLocal is false. In all other cases, IsLocal is
-  // true.
-  void ReadAbbrevRecord(bool IsLocal,
-                        NaClAbbrevListener *Listener);
-
-  // Skips over an abbreviation record. Duplicates code of ReadAbbrevRecord,
-  // except that no abbreviation is built.
-  void SkipAbbrevRecord();
-  
-  bool ReadBlockInfoBlock(NaClAbbrevListener *Listener);
-};
-
-} // End llvm namespace
-
-#endif
diff --git a/include/llvm/Bitcode/NaCl/NaClBitstreamWriter.h b/include/llvm/Bitcode/NaCl/NaClBitstreamWriter.h
deleted file mode 100644
index 39a5009c865..00000000000
--- a/include/llvm/Bitcode/NaCl/NaClBitstreamWriter.h
+++ /dev/null
@@ -1,553 +0,0 @@
-//===- NaClBitstreamWriter.h - NaCl bitstream writer ------------*- C++ -*-===//
-//
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
-//
-//===----------------------------------------------------------------------===//
-//
-// This header defines the BitstreamWriter class.  This class can be used to
-// write an arbitrary bitstream, regardless of its contents.
-//
-//===----------------------------------------------------------------------===//
-
-#ifndef LLVM_BITCODE_NACL_NACLBITSTREAMWRITER_H
-#define LLVM_BITCODE_NACL_NACLBITSTREAMWRITER_H
-
-#include "llvm/ADT/SmallVector.h"
-#include "llvm/ADT/StringRef.h"
-#include "llvm/Bitcode/NaCl/NaClBitCodes.h"
-#include "llvm/Bitcode/NaCl/NaClBitcodeHeader.h"
-#include <vector>
-
-namespace llvm {
-
-class NaClBitstreamWriter {
-  SmallVectorImpl<char> &Out;
-
-  /// CurBit - Always between 0 and 31 inclusive, specifies the next bit to use.
-  unsigned CurBit;
-
-  /// CurValue - The current value.  Only bits < CurBit are valid.
-  uint32_t CurValue;
-
-  /// CurCodeSize - This is the declared size of code values used for the
-  /// current block, in bits.
-  NaClBitcodeSelectorAbbrev CurCodeSize;
-
-  /// BlockInfoCurBID - When emitting a BLOCKINFO_BLOCK, this is the currently
-  /// selected BLOCK ID.
-  unsigned BlockInfoCurBID;
-
-  /// CurAbbrevs - Abbrevs installed at in this block.
-  std::vector<NaClBitCodeAbbrev*> CurAbbrevs;
-
-  struct Block {
-    const NaClBitcodeSelectorAbbrev PrevCodeSize;
-    const unsigned StartSizeWord;
-    std::vector<NaClBitCodeAbbrev*> PrevAbbrevs;
-    const unsigned AbbreviationIndexLimit;
-    Block(const NaClBitcodeSelectorAbbrev& PCS, unsigned SSW,
-          unsigned AbbreviationIndexLimit)
-        : PrevCodeSize(PCS), StartSizeWord(SSW),
-          AbbreviationIndexLimit(AbbreviationIndexLimit) {}
-  };
-
-  /// BlockScope - This tracks the current blocks that we have entered.
-  std::vector<Block> BlockScope;
-
-  /// BlockInfo - This contains information emitted to BLOCKINFO_BLOCK blocks.
-  /// These describe abbreviations that all blocks of the specified ID inherit.
-  struct BlockInfo {
-    unsigned BlockID;
-    std::vector<NaClBitCodeAbbrev*> Abbrevs;
-  };
-  std::vector<BlockInfo> BlockInfoRecords;
-
-  // True if filler should be added to byte align records.
-  bool AlignBitcodeRecords = false;
-
-  /// AbbrevValues - Wrapper class that allows the bitstream writer to
-  /// prefix a code to the set of values, associated with a record to
-  /// emit, without having to destructively change the contents of
-  /// values.
-  template<typename uintty>
-  struct AbbrevValues {
-    AbbrevValues(uintty Code, const SmallVectorImpl<uintty> &Values)
-        : Code(Code), Values(Values) {}
-
-    size_t size() const {
-      return Values.size() + 1;
-    }
-
-    uintty operator[](size_t Index) const {
-      return Index == 0 ? Code : Values[Index-1];
-    }
-
-  private:
-    // The code to use (if not DONT_USE_CODE).
-    uintty Code;
-    const SmallVectorImpl<uintty> &Values;
-  };
-
-public:
-  // BackpatchWord - Backpatch a 32-bit word in the output with the specified
-  // value.
-  void BackpatchWord(unsigned ByteNo, unsigned NewWord) {
-    Out[ByteNo++] = (unsigned char)(NewWord >>  0);
-    Out[ByteNo++] = (unsigned char)(NewWord >>  8);
-    Out[ByteNo++] = (unsigned char)(NewWord >> 16);
-    Out[ByteNo  ] = (unsigned char)(NewWord >> 24);
-  }
-
-private:
-  void WriteByte(unsigned char Value) {
-    Out.push_back(Value);
-  }
-
-  void WriteWord(unsigned Value) {
-    unsigned char Bytes[4] = {
-      (unsigned char)(Value >>  0),
-      (unsigned char)(Value >>  8),
-      (unsigned char)(Value >> 16),
-      (unsigned char)(Value >> 24) };
-    Out.append(&Bytes[0], &Bytes[4]);
-  }
-
-  unsigned GetBufferOffset() const {
-    return Out.size();
-  }
-
-  unsigned GetWordIndex() const {
-    unsigned Offset = GetBufferOffset();
-    assert((Offset & 3) == 0 && "Not 32-bit aligned");
-    return Offset / 4;
-  }
-
-public:
-  explicit NaClBitstreamWriter(SmallVectorImpl<char> &O)
-      : Out(O), CurBit(0), CurValue(0), CurCodeSize() {}
-
-  ~NaClBitstreamWriter() {
-    assert(CurBit == 0 && "Unflushed data remaining");
-    assert(BlockScope.empty() && CurAbbrevs.empty() && "Block imbalance");
-
-    // Free the BlockInfoRecords.
-    while (!BlockInfoRecords.empty()) {
-      BlockInfo &Info = BlockInfoRecords.back();
-      // Free blockinfo abbrev info.
-      for (unsigned i = 0, e = static_cast<unsigned>(Info.Abbrevs.size());
-           i != e; ++i)
-        Info.Abbrevs[i]->dropRef();
-      BlockInfoRecords.pop_back();
-    }
-  }
-
-  void initFromHeader(const NaClBitcodeHeader &Header) {
-    AlignBitcodeRecords = Header.getAlignBitcodeRecords();
-  }
-
-  /// \brief Retrieve the current position in the stream, in bits.
-  uint64_t GetCurrentBitNo() const { return GetBufferOffset() * 8 + CurBit; }
-
-  /// \brief Returns the maximum abbreviation index allowed for the
-  /// current block.
-  size_t getMaxCurAbbrevIndex() const {
-    return CurAbbrevs.size() + naclbitc::DEFAULT_MAX_ABBREV;
-  }
-
-  //===--------------------------------------------------------------------===//
-  // Basic Primitives for emitting bits to the stream.
-  //===--------------------------------------------------------------------===//
-
-  // Max Number of bits that can be written using Emit.
-  static const unsigned MaxEmitNumBits = 32;
-
-  void Emit(uint32_t Val, unsigned NumBits) {
-    assert(NumBits && NumBits <= MaxEmitNumBits && "Invalid value size!");
-    assert((Val &
-            ~(~0U >> (MaxEmitNumBits-NumBits))) == 0 && "High bits set!");
-    CurValue |= Val << CurBit;
-    if (CurBit + NumBits < MaxEmitNumBits) {
-      CurBit += NumBits;
-      return;
-    }
-
-    // Add the current word.
-    WriteWord(CurValue);
-
-    if (CurBit)
-      CurValue = Val >> (MaxEmitNumBits-CurBit);
-    else
-      CurValue = 0;
-    CurBit = (CurBit+NumBits) & (MaxEmitNumBits-1);
-  }
-
-  void Emit64(uint64_t Val, unsigned NumBits) {
-    while (NumBits > MaxEmitNumBits) {
-      Emit((uint32_t)Val, MaxEmitNumBits);
-      Val >>= MaxEmitNumBits;
-      NumBits -= MaxEmitNumBits;
-    }
-    Emit((uint32_t)Val, NumBits);
-  }
-
-  void flushToByte() {
-    unsigned BitsToFlush = (32 - CurBit) % CHAR_BIT;
-    if (BitsToFlush)
-      Emit(0, BitsToFlush);
-  }
-
-  void flushToByteIfAligned() {
-    if (AlignBitcodeRecords)
-      flushToByte();
-  }
-
-  void FlushToWord() {
-    if (CurBit) {
-      WriteWord(CurValue);
-      CurBit = 0;
-      CurValue = 0;
-    }
-  }
-
-  void EmitVBR(uint32_t Val, unsigned NumBits) {
-    assert(NumBits <= 32 && "Too many bits to emit!");
-    assert(NumBits > 1 && "Too few bits to emit!");
-    uint32_t Threshold = 1U << (NumBits-1);
-
-    // Emit the bits with VBR encoding, NumBits-1 bits at a time.
-    while (Val >= Threshold) {
-      Emit((Val & ((1 << (NumBits-1))-1)) | (1 << (NumBits-1)), NumBits);
-      Val >>= NumBits-1;
-    }
-
-    Emit(Val, NumBits);
-  }
-
-  void EmitVBR64(uint64_t Val, unsigned NumBits) {
-    assert(NumBits <= 32 && "Too many bits to emit!");
-    assert(NumBits > 1 && "Too few bits to emit!");
-    if ((uint32_t)Val == Val)
-      return EmitVBR((uint32_t)Val, NumBits);
-
-    uint32_t Threshold = 1U << (NumBits-1);
-
-    // Emit the bits with VBR encoding, NumBits-1 bits at a time.
-    while (Val >= Threshold) {
-      Emit(((uint32_t)Val & ((1 << (NumBits-1))-1)) |
-           (1 << (NumBits-1)), NumBits);
-      Val >>= NumBits-1;
-    }
-
-    Emit((uint32_t)Val, NumBits);
-  }
-
-  /// EmitCode - Emit the specified code.
-  void EmitCode(unsigned Val) {
-    if (CurCodeSize.IsFixed)
-      Emit(Val, CurCodeSize.NumBits);
-    else
-      EmitVBR(Val, CurCodeSize.NumBits);
-  }
-
-  //===--------------------------------------------------------------------===//
-  // Block Manipulation
-  //===--------------------------------------------------------------------===//
-
-  /// getBlockInfo - If there is block info for the specified ID, return it,
-  /// otherwise return null.
-  BlockInfo *getBlockInfo(unsigned BlockID) {
-    // Common case, the most recent entry matches BlockID.
-    if (!BlockInfoRecords.empty() && BlockInfoRecords.back().BlockID == BlockID)
-      return &BlockInfoRecords.back();
-
-    for (unsigned i = 0, e = static_cast<unsigned>(BlockInfoRecords.size());
-         i != e; ++i)
-      if (BlockInfoRecords[i].BlockID == BlockID)
-        return &BlockInfoRecords[i];
-    return 0;
-  }
-
-private:
-  // Enter block using CodeLen bits to read the size of the code
-  // selector associated with the block.
-  void EnterSubblock(unsigned BlockID,
-                     const NaClBitcodeSelectorAbbrev& CodeLen,
-                     BlockInfo *Info) {
-    // Block header:
-    //    [ENTER_SUBBLOCK, blockid, newcodelen, <align4bytes>, blocklen]
-    EmitCode(naclbitc::ENTER_SUBBLOCK);
-    EmitVBR(BlockID, naclbitc::BlockIDWidth);
-    assert(CodeLen.IsFixed && "Block codelens must be fixed");
-    EmitVBR(CodeLen.NumBits, naclbitc::CodeLenWidth);
-    FlushToWord();
-
-    unsigned BlockSizeWordIndex = GetWordIndex();
-    NaClBitcodeSelectorAbbrev OldCodeSize(CurCodeSize);
-
-    // Emit a placeholder, which will be replaced when the block is popped.
-    Emit(0, naclbitc::BlockSizeWidth);
-
-    CurCodeSize = CodeLen;
-
-    // Push the outer block's abbrev set onto the stack, start out with an
-    // empty abbrev set.
-    BlockScope.push_back(Block(OldCodeSize, BlockSizeWordIndex,
-                               1 << CodeLen.NumBits));
-    BlockScope.back().PrevAbbrevs.swap(CurAbbrevs);
-
-    // If there is a blockinfo for this BlockID, add all the predefined abbrevs
-    // to the abbrev list.
-    if (Info) {
-      for (unsigned i = 0, e = static_cast<unsigned>(Info->Abbrevs.size());
-           i != e; ++i) {
-        CurAbbrevs.push_back(Info->Abbrevs[i]);
-        Info->Abbrevs[i]->addRef();
-      }
-    }
-  }
-
-public:
-  /// \brief Enter block using CodeLen bits to read the size of the code
-  /// selector associated with the block.
-  void EnterSubblock(unsigned BlockID,
-                     const NaClBitcodeSelectorAbbrev& CodeLen) {
-    EnterSubblock(BlockID, CodeLen, getBlockInfo(BlockID));
-  }
-
-  /// \brief Enter block, using a code length based on the number of
-  /// (global) BlockInfo entries defined for the block. Note: This
-  /// should be used only if the block doesn't define any local abbreviations.
-  void EnterSubblock(unsigned BlockID) {
-    BlockInfo *Info = getBlockInfo(BlockID);
-    size_t NumAbbrevs = Info ? Info->Abbrevs.size() : 0;
-    NaClBitcodeSelectorAbbrev DefaultCodeLen(
-        naclbitc::DEFAULT_MAX_ABBREV+NumAbbrevs);
-    EnterSubblock(BlockID, DefaultCodeLen, Info);
-  }
-
-  /// \brief Enter block with the given number of abbreviations.
-  void EnterSubblock(unsigned BlockID, unsigned NumAbbrev) {
-    NaClBitcodeSelectorAbbrev CodeLenAbbrev(NumAbbrev);
-    EnterSubblock(BlockID, CodeLenAbbrev);
-  }
-
-  void ExitBlock() {
-    assert(!BlockScope.empty() && "Block scope imbalance!");
-
-    // Delete all abbrevs.
-    for (unsigned i = 0, e = static_cast<unsigned>(CurAbbrevs.size());
-         i != e; ++i)
-      CurAbbrevs[i]->dropRef();
-
-    const Block &B = BlockScope.back();
-
-    // Block tail:
-    //    [END_BLOCK, <align4bytes>]
-    EmitCode(naclbitc::END_BLOCK);
-    FlushToWord();
-
-    // Compute the size of the block, in words, not counting the size field.
-    unsigned SizeInWords = GetWordIndex() - B.StartSizeWord - 1;
-    unsigned ByteNo = B.StartSizeWord*4;
-
-    // Update the block size field in the header of this sub-block.
-    BackpatchWord(ByteNo, SizeInWords);
-
-    // Restore the inner block's code size and abbrev table.
-    CurCodeSize = B.PrevCodeSize;
-    BlockScope.back().PrevAbbrevs.swap(CurAbbrevs);
-    BlockScope.pop_back();
-  }
-
-  //===--------------------------------------------------------------------===//
-  // Record Emission
-  //===--------------------------------------------------------------------===//
-
-private:
-  /// EmitAbbreviatedField - Emit a single scalar field value with the specified
-  /// encoding.
-  template<typename uintty>
-  void EmitAbbreviatedField(const NaClBitCodeAbbrevOp &Op, uintty V) {
-    switch (Op.getEncoding()) {
-    case NaClBitCodeAbbrevOp::Literal:
-      // This is a no-op, since the abbrev specifies the literal to use.
-      assert(V == Op.getValue() && "Invalid abbrev for record!");
-      break;
-    case NaClBitCodeAbbrevOp::Fixed:
-      if (Op.getValue())
-        Emit((unsigned)V, (unsigned)Op.getValue());
-      break;
-    case NaClBitCodeAbbrevOp::VBR:
-      if (Op.getValue())
-        EmitVBR64(V, (unsigned)Op.getValue());
-      break;
-    case NaClBitCodeAbbrevOp::Array:
-      report_fatal_error("Not to be used with array abbreviation op!");
-    case NaClBitCodeAbbrevOp::Char6:
-      Emit(NaClBitCodeAbbrevOp::EncodeChar6((char)V), 6);
-      break;
-    }
-  }
-
-  /// EmitRecordWithAbbrevImpl - This is the core implementation of the record
-  /// emission code.
-  template<typename uintty>
-  void EmitRecordWithAbbrevImpl(unsigned Abbrev,
-                                const AbbrevValues<uintty> &Vals) {
-    const NaClBitCodeAbbrev *Abbv = getAbbreviation(Abbrev);
-    assert(Abbv && "Abbreviation index is invalid");
-
-    EmitCode(Abbrev);
-
-    unsigned RecordIdx = 0;
-    for (unsigned i = 0, e = static_cast<unsigned>(Abbv->getNumOperandInfos());
-         i != e; ++i) {
-      const NaClBitCodeAbbrevOp &Op = Abbv->getOperandInfo(i);
-      if (Op.getEncoding() == NaClBitCodeAbbrevOp::Array) {
-        // Array case.
-        assert(i+2 == e && "array op not second to last?");
-        const NaClBitCodeAbbrevOp &EltEnc = Abbv->getOperandInfo(++i);
-
-        // Emit a vbr6 to indicate the number of elements present.
-        EmitVBR(static_cast<uint32_t>(Vals.size()-RecordIdx), 6);
-
-        // Emit each field.
-        for (unsigned e = Vals.size(); RecordIdx != e; ++RecordIdx)
-          EmitAbbreviatedField(EltEnc, Vals[RecordIdx]);
-      } else {
-        assert(RecordIdx < Vals.size() && "Invalid abbrev/record");
-        EmitAbbreviatedField(Op, Vals[RecordIdx]);
-        ++RecordIdx;
-      }
-    }
-    assert(RecordIdx == Vals.size() && "Not all record operands emitted!");
-  }
-
-public:
-
-  /// Returns a pointer to the abbreviation currently associated with
-  /// the abbreviation index. Returns nullptr if no such abbreviation.
-  const NaClBitCodeAbbrev *getAbbreviation(unsigned Index) const {
-    if (Index < naclbitc::FIRST_APPLICATION_ABBREV)
-      return nullptr;
-    if (Index >= BlockScope.back().AbbreviationIndexLimit)
-      return nullptr;
-    unsigned AbbrevNo = Index - naclbitc::FIRST_APPLICATION_ABBREV;
-    if (AbbrevNo >= CurAbbrevs.size())
-      return nullptr;
-    return CurAbbrevs[AbbrevNo];
-  }
-
-  /// EmitRecord - Emit the specified record to the stream, using an abbrev if
-  /// we have one to compress the output.
-  template<typename uintty>
-  void EmitRecord(unsigned Code, const SmallVectorImpl<uintty> &Vals,
-                  unsigned Abbrev = 0) {
-    if (!Abbrev) {
-      // If we don't have an abbrev to use, emit this in its fully unabbreviated
-      // form.
-      EmitCode(naclbitc::UNABBREV_RECORD);
-      EmitVBR(Code, 6);
-      EmitVBR(static_cast<uint32_t>(Vals.size()), 6);
-      for (unsigned i = 0, e = static_cast<unsigned>(Vals.size()); i != e; ++i)
-        EmitVBR64(Vals[i], 6);
-      flushToByteIfAligned();
-      return;
-    }
-
-    // combine code and values, and then emit.
-    AbbrevValues<uintty> AbbrevVals(Code, Vals);
-    EmitRecordWithAbbrevImpl(Abbrev, AbbrevVals);
-    flushToByteIfAligned();
-  }
-
-  //===--------------------------------------------------------------------===//
-  // Abbrev Emission
-  //===--------------------------------------------------------------------===//
-
-private:
-  // Emit the abbreviation as a DEFINE_ABBREV record.
-  void EncodeAbbrev(NaClBitCodeAbbrev *Abbv) {
-    EmitCode(naclbitc::DEFINE_ABBREV);
-    EmitVBR(Abbv->getNumOperandInfos(), 5);
-    for (unsigned i = 0, e = static_cast<unsigned>(Abbv->getNumOperandInfos());
-         i != e; ++i) {
-      const NaClBitCodeAbbrevOp &Op = Abbv->getOperandInfo(i);
-      bool IsLiteral = Op.isLiteral();
-      Emit(IsLiteral, 1);
-      if (IsLiteral) {
-        EmitVBR64(Op.getValue(), 8);
-      } else {
-        Emit(Op.getEncoding(), 3);
-        if (Op.hasValue())
-          EmitVBR64(Op.getValue(), 5);
-      }
-    }
-    flushToByteIfAligned();
-  }
-public:
-
-  /// EmitAbbrev - This emits an abbreviation to the stream.  Note that this
-  /// method takes ownership of the specified abbrev.
-  unsigned EmitAbbrev(NaClBitCodeAbbrev *Abbv) {
-    assert(Abbv->isValid() && "Can't emit invalid abbreviation!");
-    // Emit the abbreviation as a record.
-    EncodeAbbrev(Abbv);
-    CurAbbrevs.push_back(Abbv);
-    return static_cast<unsigned>(CurAbbrevs.size())-1 +
-      naclbitc::FIRST_APPLICATION_ABBREV;
-  }
-
-  //===--------------------------------------------------------------------===//
-  // BlockInfo Block Emission
-  //===--------------------------------------------------------------------===//
-
-  /// EnterBlockInfoBlock - Start emitting the BLOCKINFO_BLOCK.
-  void EnterBlockInfoBlock() {
-    EnterSubblock(naclbitc::BLOCKINFO_BLOCK_ID);
-    BlockInfoCurBID = ~0U;
-  }
-private:
-  /// SwitchToBlockID - If we aren't already talking about the specified block
-  /// ID, emit a BLOCKINFO_CODE_SETBID record.
-  void SwitchToBlockID(unsigned BlockID) {
-    if (BlockInfoCurBID == BlockID) return;
-    SmallVector<unsigned, 2> V;
-    V.push_back(BlockID);
-    EmitRecord(naclbitc::BLOCKINFO_CODE_SETBID, V);
-    BlockInfoCurBID = BlockID;
-  }
-
-  BlockInfo &getOrCreateBlockInfo(unsigned BlockID) {
-    if (BlockInfo *BI = getBlockInfo(BlockID))
-      return *BI;
-
-    // Otherwise, add a new record.
-    BlockInfoRecords.push_back(BlockInfo());
-    BlockInfoRecords.back().BlockID = BlockID;
-    return BlockInfoRecords.back();
-  }
-
-public:
-
-  /// EmitBlockInfoAbbrev - Emit a DEFINE_ABBREV record for the specified
-  /// BlockID.
-  unsigned EmitBlockInfoAbbrev(unsigned BlockID, NaClBitCodeAbbrev *Abbv) {
-    SwitchToBlockID(BlockID);
-    EncodeAbbrev(Abbv);
-
-    // Add the abbrev to the specified block record.
-    BlockInfo &Info = getOrCreateBlockInfo(BlockID);
-    Info.Abbrevs.push_back(Abbv);
-
-    return Info.Abbrevs.size()-1+naclbitc::FIRST_APPLICATION_ABBREV;
-  }
-};
-
-
-} // End llvm namespace
-
-#endif
diff --git a/include/llvm/Bitcode/NaCl/NaClCompress.h b/include/llvm/Bitcode/NaCl/NaClCompress.h
deleted file mode 100644
index 41bfe7c2af3..00000000000
--- a/include/llvm/Bitcode/NaCl/NaClCompress.h
+++ /dev/null
@@ -1,61 +0,0 @@
-//===-- NaClCompress.h - Bitcode (abbrev) compression ---------------------===//
-//
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
-//
-//===----------------------------------------------------------------------===//
-//
-// Defines API's to read in a PNaCl bitcode buffer, and write it out to
-// a corresponding ostream.
-//
-//===----------------------------------------------------------------------===//
-
-#ifndef LLVM_BITCODE_NACL_NACLCOMPRESS_H
-#define LLVM_BITCODE_NACL_NACLCOMPRESS_H
-
-#include "llvm/Support/MemoryBuffer.h"
-#include "llvm/Support/raw_ostream.h"
-
-namespace llvm {
-
-/// Analyzes and (optionally) compresses a bitcode file.
-class NaClBitcodeCompressor {
-public:
-  /// The flags to use when compressing.
-  struct CompressFlags {
-    CompressFlags()
-        : TraceGeneratedAbbreviations(false),
-          ShowValueDistributions(false),
-          ShowAbbrevLookupTries(false),
-          ShowAbbreviationFrequencies(false),
-          RemoveAbbreviations(false) {}
-    bool TraceGeneratedAbbreviations;
-    bool ShowValueDistributions;
-    bool ShowAbbrevLookupTries;
-    bool ShowAbbreviationFrequencies;
-    bool RemoveAbbreviations;
-  };
-  CompressFlags Flags;
-
-  NaClBitcodeCompressor() {}
-
-  /// Analyzes bitcode in MemBuf for applicable abbreviations for
-  /// compression. Output is the stream to write data associated with
-  /// "show" flags (ShowValueDistributions, ShowAbbrevLookupTries, //
-  /// and ShowAbbreviationFrequencies).
-  bool analyze(MemoryBuffer *MemBuf, raw_ostream &Output = outs());
-
-  /// Reads bitcode in MemBuf, finds new applicable abbreviations for
-  /// compression, and then rewrites out the bitcode out using these
-  /// new abbreviations to BitcodeOutput. Returns true iff successful.
-  /// ShowOutput captures analysis output generated by "show" flags
-  /// (see method analyze).
-  bool compress(MemoryBuffer *MemBuf, raw_ostream &BitcodeOutput,
-                raw_ostream &ShowOutput = errs());
-};
-
-} // namespace llvm
-
-#endif // LLVM_BITCODE_NACL_NACLCOMPRESS_H
diff --git a/include/llvm/Bitcode/NaCl/NaClCompressBlockDist.h b/include/llvm/Bitcode/NaCl/NaClCompressBlockDist.h
deleted file mode 100644
index 66c3bdc99b9..00000000000
--- a/include/llvm/Bitcode/NaCl/NaClCompressBlockDist.h
+++ /dev/null
@@ -1,62 +0,0 @@
-//===-- NaClCompressBlockDist.h -------------------------------------------===//
-//      Defines distribution maps used to collect block and record
-//      distributions for tool pnacl-bccompress.
-//
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
-//
-//===----------------------------------------------------------------------===//
-
-#ifndef LLVM_BITCODE_NACL_NACLCOMPRESSBLOCKDIST_H
-#define LLVM_BITCODE_NACL_NACLCOMPRESSBLOCKDIST_H
-
-
-#include "llvm/Bitcode/NaCl/NaClBitcodeAbbrevDist.h"
-#include "llvm/Bitcode/NaCl/NaClBitcodeBlockDist.h"
-
-namespace llvm {
-
-/// Nests record distributions within the block they appear in,
-/// in a block distribution. The record distributions are refined
-/// by separating record codes that use the same abbreviation.
-class NaClCompressBlockDistElement : public NaClBitcodeBlockDistElement {
-public:
-  static bool classof(const NaClBitcodeDistElement *Element) {
-    return Element->getKind() >= RDE_PNaClCompressBlockDist
-        && Element->getKind() < RDE_PNaClCompressBlockDistLast;
-  }
-
-  explicit NaClCompressBlockDistElement(unsigned BlockID=0)
-      : NaClBitcodeBlockDistElement(RDE_PNaClCompressBlockDist),
-        AbbrevDist(BlockID) {
-    NestedDists.push_back(&AbbrevDist);
-  }
-
-  virtual ~NaClCompressBlockDistElement();
-
-  virtual NaClBitcodeDistElement*
-  CreateElement(NaClBitcodeDistValue Value) const;
-
-  virtual const SmallVectorImpl<NaClBitcodeDist*> *
-  GetNestedDistributions() const;
-
-  NaClBitcodeDist &GetAbbrevDist() {
-    return AbbrevDist;
-  }
-
-  // Sentinel for generating elements of this type.
-  static NaClCompressBlockDistElement Sentinel;
-
-private:
-  // Nested blocks used by GetNestedDistributions.
-  SmallVector<NaClBitcodeDist*, 1> NestedDists;
-
-  // The abbreviations/records associated with the corresponding block.
-  NaClBitcodeAbbrevDist AbbrevDist;
-};
-
-}
-
-#endif
diff --git a/include/llvm/Bitcode/NaCl/NaClCompressCodeDist.h b/include/llvm/Bitcode/NaCl/NaClCompressCodeDist.h
deleted file mode 100644
index 71b1775482a..00000000000
--- a/include/llvm/Bitcode/NaCl/NaClCompressCodeDist.h
+++ /dev/null
@@ -1,63 +0,0 @@
-//===-- NaClCompressCodeDist.h -------------------------------------------===//
-//      Defines distribution maps for record codes for pnacl-bccompress.
-//
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
-//
-//===----------------------------------------------------------------------===//
-
-#ifndef LLVM_BITCODE_NACL_NACLCOMPRESSCODEDIST_H
-#define LLVM_BITCODE_NACL_NACLCOMPRESSCODEDIST_H
-
-#include "llvm/Bitcode/NaCl/NaClBitcodeCodeDist.h"
-#include "llvm/Bitcode/NaCl/NaClBitcodeSizeDist.h"
-
-namespace llvm {
-
-/// Defines a record code distribution, with nested distributions separating
-/// the code distributions based on size and values.
-class NaClCompressCodeDistElement : public NaClBitcodeCodeDistElement {
-public:
-  static bool classof(const NaClBitcodeDistElement *Element) {
-    return Element->getKind() >= RDE_CompressCodeDist &&
-        Element->getKind() < RDE_CompressCodeDistLast;
-  }
-
-  NaClCompressCodeDistElement()
-      : NaClBitcodeCodeDistElement(RDE_CompressCodeDist),
-        SizeDist(NaClBitcodeDist::RecordStorage,
-                 &NaClBitcodeSizeDistElement::Sentinel) {
-    NestedDists.push_back(&SizeDist);
-  }
-
-  virtual ~NaClCompressCodeDistElement();
-
-  virtual NaClBitcodeDistElement *CreateElement(
-      NaClBitcodeDistValue Value) const;
-
-  virtual void AddRecord(const NaClBitcodeRecord &Record);
-
-  virtual const SmallVectorImpl<NaClBitcodeDist*> *
-  GetNestedDistributions() const;
-
-  NaClBitcodeDist &GetSizeDist() {
-    return SizeDist;
-  }
-
-  /// The sentinel used to generate instances of this in
-  /// a record code distribution map.
-  static NaClCompressCodeDistElement Sentinel;
-
-private:
-  // Nested blocks used by GetNestedDistributions.
-  SmallVector<NaClBitcodeDist*, 1> NestedDists;
-
-  /// The distribution of values, based on size.
-  NaClBitcodeDist SizeDist;
-};
-
-}
-
-#endif
diff --git a/include/llvm/Bitcode/NaCl/NaClFuzz.h b/include/llvm/Bitcode/NaCl/NaClFuzz.h
deleted file mode 100644
index 1d6524dbfd9..00000000000
--- a/include/llvm/Bitcode/NaCl/NaClFuzz.h
+++ /dev/null
@@ -1,94 +0,0 @@
-//===- NaClFuzz.h - Fuzz PNaCl bitcode records ------------------*- C++ -*-===//
-//
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
-//
-//===----------------------------------------------------------------------===//
-//
-// This file defines a basic fuzzer for a list of PNaCl bitcode records.
-//
-// *** WARNING *** The implementation of the fuzzer uses a random
-// number generator.  As a result, this code is not thread safe.
-//
-//===----------------------------------------------------------------------===//
-
-#ifndef LLVM_BITCODE_NACL_NACLFUZZ_H
-#define LLVM_BITCODE_NACL_NACLFUZZ_H
-
-#include "llvm/Bitcode/NaCl/NaClBitcodeMungeUtils.h"
-#include "llvm/Bitcode/NaCl/NaClRandNumGen.h"
-
-namespace naclfuzz {
-
-using namespace llvm;
-
-/// \brief Fuzzes a list of editable bitcode records.
-class RecordFuzzer {
-  RecordFuzzer(const RecordFuzzer&) = delete;
-  void operator=(const RecordFuzzer&) = delete;
-public:
-  typedef NaClMungedBitcode::iterator iterator;
-
-  /// \brief The set of possible fuzzing actions.
-  enum EditAction {
-    /// \brief Inserts a new record into the list of bitcode records.
-    InsertRecord,
-    /// \brief Mutate contents of an existing bitcode record.
-    MutateRecord,
-    /// \brief Removes an existing record from the list of bitcode
-    /// records.
-    RemoveRecord,
-    /// \brief Replaces an existing record with a new bitcode record.
-    ReplaceRecord,
-    /// \brief Swaps two records in the bitcode record list.
-    SwapRecord
-  };
-
-  virtual ~RecordFuzzer();
-
-  /// \brief Generates a random mutation of the bitcode, using the
-  /// provided random number generator. Percentage (a value between 0
-  /// and 1 defined by Count/Base) is used to define the number of
-  /// fuzzing actions applied to the bitcode.  Returns true if fuzzing
-  /// succeeded.
-  ///
-  /// May be called an arbitrary number of times. Results are left in
-  /// the munged bitcode records passed into static method
-  /// createSimpleRecordFuzzer.
-  virtual bool fuzz(unsigned Count, unsigned Base=100) = 0;
-
-  /// \brief Shows how many times each record was edited in the
-  /// corresponding (input) bitcode, over all calls to fuzz.
-  virtual void showRecordDistribution(raw_ostream &Out) const = 0;
-
-  /// \brief Shows how many times each type of edit action was applied
-  /// to the corresponding bitcode, over all calls to fuzz.
-  virtual void showEditDistribution(raw_ostream &Out) const = 0;
-
-  // Creates an instance of a fuzzer for the given bitcode.
-  static RecordFuzzer
-  *createSimpleRecordFuzzer(NaClMungedBitcode &Bitcode,
-                            RandomNumberGenerator &RandGenerator);
-
-  /// Returns printable name for the edit action.
-  static const char *actionName(EditAction Action);
-
-protected:
-  RecordFuzzer(NaClMungedBitcode &Bitcode, RandomNumberGenerator &Generator);
-
-  // Holds the bitcode being munged.
-  NaClMungedBitcode &Bitcode;
-
-  // Hold the random number generator.
-  RandomNumberGenerator &Generator;
-
-  // Erases the last fuzzing result from the munged bitcode records
-  // in Bitcode.
-  virtual void clear();
-};
-
-} // end of namespace naclfuzz
-
-#endif // LLVM_BITCODE_NACL_NACLFUZZ_H
diff --git a/include/llvm/Bitcode/NaCl/NaClLLVMBitCodes.h b/include/llvm/Bitcode/NaCl/NaClLLVMBitCodes.h
deleted file mode 100644
index adc554c7897..00000000000
--- a/include/llvm/Bitcode/NaCl/NaClLLVMBitCodes.h
+++ /dev/null
@@ -1,374 +0,0 @@
-//===- NaClLLVMBitCodes.h ---------------------------------------*- C++ -*-===//
-//     Enum values for the NaCl bitcode wire format
-//
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
-//
-//===----------------------------------------------------------------------===//
-//
-// This header defines Bitcode enum values for NaCl bitcode wire format.
-//
-// The enum values defined in this file should be considered permanent.  If
-// new features are added, they should have values added at the end of the
-// respective lists.
-//
-// Note: PNaCl version 1 is no longer supported, and has been removed from
-// comments.
-//
-//===----------------------------------------------------------------------===//
-
-#ifndef LLVM_BITCODE_NACL_NACLLLVMBITCODES_H
-#define LLVM_BITCODE_NACL_NACLLLVMBITCODES_H
-
-#include "llvm/Bitcode/NaCl/NaClBitCodes.h"
-
-namespace llvm {
-namespace naclbitc {
-  // The only top-level block type defined is for a module.
-  enum NaClBlockIDs {
-    // Blocks
-    MODULE_BLOCK_ID          = FIRST_APPLICATION_BLOCKID,
-
-    // Module sub-block id's.
-    PARAMATTR_BLOCK_ID,        // Not used in PNaCl.
-    PARAMATTR_GROUP_BLOCK_ID,  // Not used in PNaCl.
-
-    CONSTANTS_BLOCK_ID,
-    FUNCTION_BLOCK_ID,
-
-    UNUSED_ID1,
-
-    VALUE_SYMTAB_BLOCK_ID,
-    METADATA_BLOCK_ID,         // Not used in PNaCl.
-    METADATA_ATTACHMENT_ID,    // Not used in PNaCl.
-
-    TYPE_BLOCK_ID_NEW,
-
-    USELIST_BLOCK_ID,          // Not used in PNaCl.
-    GLOBALVAR_BLOCK_ID
-  };
-
-
-  /// MODULE blocks have a number of optional fields and subblocks.
-  enum NaClModuleCodes {
-    MODULE_CODE_VERSION     = 1,    // VERSION:     [version#]
-    MODULE_CODE_TRIPLE      = 2,    // Not used in PNaCl
-    MODULE_CODE_DATALAYOUT  = 3,    // Not used in PNaCl
-    MODULE_CODE_ASM         = 4,    // Not used in PNaCl
-    MODULE_CODE_SECTIONNAME = 5,    // Not used in PNaCl
-    MODULE_CODE_DEPLIB      = 6,    // Not used in PNaCl
-    MODULE_CODE_GLOBALVAR   = 7,    // Not used in PNaCl
-    // FUNCTION:  [type, callingconv, isproto, linkage]
-    MODULE_CODE_FUNCTION    = 8,
-    MODULE_CODE_ALIAS       = 9,    // Not used in PNaCl
-    MODULE_CODE_PURGEVALS   = 10,   // Not used in PNaCl
-    MODULE_CODE_GCNAME      = 11    // Not used in PNaCl
-  };
-
-  /// PARAMATTR blocks have code for defining a parameter attribute set.
-  enum NaClAttributeCodes {
-    // FIXME: Remove `PARAMATTR_CODE_ENTRY_OLD' in 4.0
-    PARAMATTR_CODE_ENTRY_OLD  = 1, // ENTRY: [paramidx0, attr0,
-                                   //         paramidx1, attr1...]
-    PARAMATTR_CODE_ENTRY      = 2, // ENTRY: [paramidx0, attrgrp0,
-                                   //         paramidx1, attrgrp1, ...]
-    PARAMATTR_GRP_CODE_ENTRY  = 3  // ENTRY: [id, attr0, att1, ...]
-  };
-
-  /// TYPE blocks have codes for each type primitive they use.
-  enum NaClTypeCodes {
-    TYPE_CODE_NUMENTRY =  1,    // NUMENTRY: [numentries]
-
-    // Type Codes
-    TYPE_CODE_VOID     =  2,    // VOID
-    TYPE_CODE_FLOAT    =  3,    // FLOAT
-    TYPE_CODE_DOUBLE   =  4,    // DOUBLE
-    // TODO(mseaborn): Remove LABEL when we drop support for v1 of the
-    // PNaCl bitcode format.  The writer no longer generates it.
-    TYPE_CODE_LABEL    =  5,    // LABEL
-    TYPE_CODE_OPAQUE   =  6,    // Not used in PNaCl.
-    TYPE_CODE_INTEGER  =  7,    // INTEGER: [width]
-    TYPE_CODE_POINTER  =  8,    // POINTER: [pointee type]
-
-    TYPE_CODE_FUNCTION_OLD = 9, // Not used in PNaCl.
-
-    TYPE_CODE_HALF     =  10,   // Not used in PNaCl.
-
-    TYPE_CODE_ARRAY    = 11,    // Not used in PNaCl.
-    TYPE_CODE_VECTOR   = 12,    // VECTOR: [numelts, eltty]
-
-    // These are not with the other floating point types because they're
-    // a late addition, and putting them in the right place breaks
-    // binary compatibility.
-    TYPE_CODE_X86_FP80 = 13,    // Not used in PNaCl.
-    TYPE_CODE_FP128    = 14,    // Not used in PNaCl.
-    TYPE_CODE_PPC_FP128= 15,    // Not used in PNaCl.
-
-    TYPE_CODE_METADATA = 16,    // Not used in PNaCl.
-
-    TYPE_CODE_X86_MMX = 17,     // Not used in PNaCl.
-
-    TYPE_CODE_STRUCT_ANON = 18, // Not used in PNaCl.
-    TYPE_CODE_STRUCT_NAME = 19, // Not used in PNaCl.
-    TYPE_CODE_STRUCT_NAMED = 20,// Not used in PNaCl.
-
-    TYPE_CODE_FUNCTION = 21     // FUNCTION: [vararg, retty, paramty x N]
-  };
-
-  // The type symbol table only has one code (TST_ENTRY_CODE).
-  enum NaClTypeSymtabCodes {
-    TST_CODE_ENTRY = 1     // TST_ENTRY: [typeid, namechar x N]
-  };
-
-  // The value symbol table only has one code (VST_ENTRY_CODE).
-  enum NaClValueSymtabCodes {
-    VST_CODE_ENTRY   = 1,  // VST_ENTRY: [valid, namechar x N]
-    VST_CODE_BBENTRY = 2   // VST_BBENTRY: [bbid, namechar x N]
-  };
-
-  // Not used in PNaCl.
-  enum NaClMetadataCodes {
-    METADATA_STRING        = 1,   // MDSTRING:      [values]
-    // 2 is unused.
-    // 3 is unused.
-    METADATA_NAME          = 4,   // STRING:        [values]
-    // 5 is unused.
-    METADATA_KIND          = 6,   // [n x [id, name]]
-    // 7 is unused.
-    METADATA_NODE          = 8,   // NODE:          [n x (type num, value num)]
-    METADATA_FN_NODE       = 9,   // FN_NODE:       [n x (type num, value num)]
-    METADATA_NAMED_NODE    = 10,  // NAMED_NODE:    [n x mdnodes]
-    METADATA_ATTACHMENT    = 11   // [m x [value, [n x [id, mdnode]]]
-  };
-
-  // The constants block (CONSTANTS_BLOCK_ID) describes emission for each
-  // constant and maintains an implicit current type value.
-  enum NaClConstantsCodes {
-    CST_CODE_SETTYPE       =  1,  // SETTYPE:       [typeid]
-    CST_CODE_NULL          =  2,  // Not used in PNaCl.
-    CST_CODE_UNDEF         =  3,  // UNDEF
-    CST_CODE_INTEGER       =  4,  // INTEGER:       [intval]
-    CST_CODE_WIDE_INTEGER  =  5,  // Not used in PNaCl.
-    CST_CODE_FLOAT         =  6,  // FLOAT:         [fpval]
-    CST_CODE_AGGREGATE     =  7,  // Not used in PNaCl.
-    CST_CODE_STRING        =  8,  // Not used in PNaCl.
-    CST_CODE_CSTRING       =  9,  // Not used in PNaCl.
-    CST_CODE_CE_BINOP      = 10,  // Not used in PNaCl.
-    CST_CODE_CE_CAST       = 11,  // Not used in PNaCl.
-    CST_CODE_CE_GEP        = 12,  // Not used in PNaCl.
-    CST_CODE_CE_SELECT     = 13,  // Not used in PNaCl.
-    CST_CODE_CE_EXTRACTELT = 14,  // Not used in PNaCl.
-    CST_CODE_CE_INSERTELT  = 15,  // Not used in PNaCl.
-    CST_CODE_CE_SHUFFLEVEC = 16,  // Not used in PNaCl.
-    CST_CODE_CE_CMP        = 17,  // Not used in PNaCl.
-    CST_CODE_INLINEASM_OLD = 18,  // No longer used.
-    CST_CODE_CE_SHUFVEC_EX = 19,  // Not used in PNaCl.
-    CST_CODE_CE_INBOUNDS_GEP = 20,// Not used in PNaCl.
-    CST_CODE_BLOCKADDRESS  = 21,  // Not used in PNaCl.
-    CST_CODE_DATA          = 22,  // Not used in PNaCl.
-    CST_CODE_INLINEASM     = 23   // Not used in PNaCl.
-  };
-
-  /// GlobalVarOpcodes - These are values used in the bitcode files to
-  /// encode records defining global variables.
-  ///
-  /// The structure of global variables can be summarized as follows:
-  ///
-  /// The global variable block begins with a GLOBALVAR_COUNT, defining
-  /// the number of global variables in the bitcode file. After that,
-  /// each global variable is defined.
-  ///
-  /// Global variables are defined by a GLOBALVAR_VAR record, followed
-  /// by 1 or more records defining its initial value. Simple
-  /// variables have a single initializer. Structured variables are
-  /// defined by an initial GLOBALVAR_COMPOUND record defining the
-  /// number of fields in the structure, followed by an initializer
-  /// for each of its fields. In this context, a field is either data,
-  /// or a relocation.  A data field is defined by a
-  /// GLOBALVAR_ZEROFILL or GLOBALVAR_DATA record.  A relocation field
-  /// is defined by a GLOBALVAR_RELOC record.
-  enum NaClGlobalVarOpcodes {
-    GLOBALVAR_VAR        = 0,     // VAR: [align, isconst]
-    GLOBALVAR_COMPOUND   = 1,     // COMPOUND: [size]
-    GLOBALVAR_ZEROFILL   = 2,     // ZEROFILL: [size]
-    GLOBALVAR_DATA       = 3,     // DATA: [b0, b1, ...]
-    GLOBALVAR_RELOC      = 4,     // RELOC: [val, [addend]]
-    GLOBALVAR_COUNT      = 5      // COUNT: [n]
-  };
-
-  /// CastOpcodes - These are values used in the bitcode files to encode which
-  /// cast a CST_CODE_CE_CAST or a XXX refers to.  The values of these enums
-  /// have no fixed relation to the LLVM IR enum values.  Changing these will
-  /// break compatibility with old files.
-  enum NaClCastOpcodes {
-    CAST_TRUNC    =  0,
-    CAST_ZEXT     =  1,
-    CAST_SEXT     =  2,
-    CAST_FPTOUI   =  3,
-    CAST_FPTOSI   =  4,
-    CAST_UITOFP   =  5,
-    CAST_SITOFP   =  6,
-    CAST_FPTRUNC  =  7,
-    CAST_FPEXT    =  8,
-    // 9 was CAST_PTRTOINT; not used in PNaCl.
-    // 10 was CAST_INTTOPTR; not used in PNaCl.
-    CAST_BITCAST  = 11
-  };
-
-  /// BinaryOpcodes - These are values used in the bitcode files to encode which
-  /// binop a CST_CODE_CE_BINOP or a XXX refers to.  The values of these enums
-  /// have no fixed relation to the LLVM IR enum values.  Changing these will
-  /// break compatibility with old files.
-  enum NaClBinaryOpcodes {
-    BINOP_ADD  =  0,
-    BINOP_SUB  =  1,
-    BINOP_MUL  =  2,
-    BINOP_UDIV =  3,
-    BINOP_SDIV =  4,    // overloaded for FP
-    BINOP_UREM =  5,
-    BINOP_SREM =  6,    // overloaded for FP
-    BINOP_SHL  =  7,
-    BINOP_LSHR =  8,
-    BINOP_ASHR =  9,
-    BINOP_AND  = 10,
-    BINOP_OR   = 11,
-    BINOP_XOR  = 12
-  };
-
-  /// OverflowingBinaryOperatorOptionalFlags - Flags for serializing
-  /// OverflowingBinaryOperator's SubclassOptionalData contents.
-  /// Note: This enum is no longer used in PNaCl, because these
-  /// flags can't exist in files that meet the PNaCl ABI.
-  enum NaClOverflowingBinaryOperatorOptionalFlags {
-    OBO_NO_UNSIGNED_WRAP = 0,
-    OBO_NO_SIGNED_WRAP = 1
-  };
-
-  /// PossiblyExactOperatorOptionalFlags - Flags for serializing
-  /// PossiblyExactOperator's SubclassOptionalData contents.
-  /// Note: This enum is no longer used in PNaCl, because these
-  /// flags can't exist in files that meet the PNaCl ABI.
-  enum NaClPossiblyExactOperatorOptionalFlags {
-    PEO_EXACT = 0
-  };
-
-  /// \brief Flags for serializing floating point binary operators's
-  /// SubclassOptionalData contents.
-  /// Note: This enum is no longer used in PNaCl, because these
-  /// flags shouldn't exist in files that meet the PNaCl ABI, unless
-  /// they are old. In the latter case, they are ignored by the reader.
-  enum NaClFloatingPointBinaryOperatorOptionalFlags {
-    FPO_UNSAFE_ALGEBRA = 0,
-    FPO_NO_NANS = 1,
-    FPO_NO_INFS = 2,
-    FPO_NO_SIGNED_ZEROS = 3,
-    FPO_ALLOW_RECIPROCAL = 4
-  };
-
-  /// Encoded function calling conventions.
-  enum NaClCallingConventions {
-    C_CallingConv = 0
-  };
-
-  /// Encoded comparison predicates.
-  enum NaClComparisonPredicates {
-    // Opcode              U L G E    Intuitive operation
-    FCMP_FALSE =  0,  ///< 0 0 0 0    Always false (always folded)
-    FCMP_OEQ   =  1,  ///< 0 0 0 1    True if ordered and equal
-    FCMP_OGT   =  2,  ///< 0 0 1 0    True if ordered and greater than
-    FCMP_OGE   =  3,  ///< 0 0 1 1    True if ordered and greater than or equal
-    FCMP_OLT   =  4,  ///< 0 1 0 0    True if ordered and less than
-    FCMP_OLE   =  5,  ///< 0 1 0 1    True if ordered and less than or equal
-    FCMP_ONE   =  6,  ///< 0 1 1 0    True if ordered and operands are unequal
-    FCMP_ORD   =  7,  ///< 0 1 1 1    True if ordered (no nans)
-    FCMP_UNO   =  8,  ///< 1 0 0 0    True if unordered: isnan(X) | isnan(Y)
-    FCMP_UEQ   =  9,  ///< 1 0 0 1    True if unordered or equal
-    FCMP_UGT   = 10,  ///< 1 0 1 0    True if unordered or greater than
-    FCMP_UGE   = 11,  ///< 1 0 1 1    True if unordered, greater than, or equal
-    FCMP_ULT   = 12,  ///< 1 1 0 0    True if unordered or less than
-    FCMP_ULE   = 13,  ///< 1 1 0 1    True if unordered, less than, or equal
-    FCMP_UNE   = 14,  ///< 1 1 1 0    True if unordered or not equal
-    FCMP_TRUE  = 15,  ///< 1 1 1 1    Always true (always folded)
-    ICMP_EQ    = 32,  ///< equal
-    ICMP_NE    = 33,  ///< not equal
-    ICMP_UGT   = 34,  ///< unsigned greater than
-    ICMP_UGE   = 35,  ///< unsigned greater or equal
-    ICMP_ULT   = 36,  ///< unsigned less than
-    ICMP_ULE   = 37,  ///< unsigned less or equal
-    ICMP_SGT   = 38,  ///< signed greater than
-    ICMP_SGE   = 39,  ///< signed greater or equal
-    ICMP_SLT   = 40,  ///< signed less than
-    ICMP_SLE   = 41   ///< signed less or equal
-  };
-
-  enum NaClLinkageTypes {
-    LINKAGE_EXTERNAL = 0,
-    LINKAGE_INTERNAL = 3
-  };
-
-  // The function body block (FUNCTION_BLOCK_ID) describes function bodies.  It
-  // can contain a constant block (CONSTANTS_BLOCK_ID).
-  enum NaClFunctionCodes {
-    FUNC_CODE_DECLAREBLOCKS    =  1, // DECLAREBLOCKS: [n]
-
-    FUNC_CODE_INST_BINOP       =  2, // BINOP:      [opval, opval, opcode]
-                                     // Note: because old PNaCl bitcode files
-                                     // may contain flags (which we now ignore),
-                                     // the reader must also support:
-                                     // BINOP: [opval, opval, opcode, flags]
-    FUNC_CODE_INST_CAST        =  3, // CAST:       [opval, destty, castopc]
-    FUNC_CODE_INST_GEP         =  4, // Not used in PNaCl.
-    FUNC_CODE_INST_SELECT      =  5, // Not used in PNaCl. Replaced by VSELECT.
-    FUNC_CODE_INST_EXTRACTELT  =  6, // EXTRACTELT: [opval, opval]
-    FUNC_CODE_INST_INSERTELT   =  7, // INSERTELT:  [opval, opval, opval]
-    FUNC_CODE_INST_SHUFFLEVEC  =  8, // Not used in PNaCl.
-    FUNC_CODE_INST_CMP         =  9, // Not used in PNaCl. Replaced by CMP2.
-    FUNC_CODE_INST_RET         = 10, // RET:        [opval<optional>]
-    FUNC_CODE_INST_BR          = 11, // BR:         [bb#, bb#, cond] or [bb#]
-    FUNC_CODE_INST_SWITCH      = 12, // SWITCH:     [opty, op0, op1, ...]
-    FUNC_CODE_INST_INVOKE      = 13, // Not used in PNaCl.
-    // 14 is unused.
-    FUNC_CODE_INST_UNREACHABLE = 15, // UNREACHABLE
-
-    FUNC_CODE_INST_PHI         = 16, // PHI:        [ty, val0,bb0, ...]
-    // 17 is unused.
-    // 18 is unused.
-    FUNC_CODE_INST_ALLOCA      = 19, // ALLOCA:     [op, align]
-    FUNC_CODE_INST_LOAD        = 20, // LOAD: [op, align, ty]
-    // 21 is unused.
-    // 22 is unused.
-    FUNC_CODE_INST_VAARG       = 23, // Not used in PNaCl.
-    FUNC_CODE_INST_STORE       = 24, // STORE: [ptr, val, align]
-    // 25 is unused.
-    FUNC_CODE_INST_EXTRACTVAL  = 26, // Not used in PNaCl.
-    FUNC_CODE_INST_INSERTVAL   = 27, // Not used in PNaCl.
-    // fcmp/icmp returning Int1TY or vector of Int1Ty. Same as CMP, exists to
-    // support legacy vicmp/vfcmp instructions.
-    FUNC_CODE_INST_CMP2        = 28, // CMP2:       [opval, opval, pred]
-    // new select on i1 or [N x i1]
-    FUNC_CODE_INST_VSELECT     = 29, // VSELECT:    [opval, opval, pred]
-    FUNC_CODE_INST_INBOUNDS_GEP= 30, // Not used in PNaCl.
-    FUNC_CODE_INST_INDIRECTBR  = 31, // Not used in PNaCl.
-    // 32 is unused.
-    FUNC_CODE_DEBUG_LOC_AGAIN  = 33, // Not used in PNaCl.
-
-    FUNC_CODE_INST_CALL        = 34, // CALL: [cc, fnid, args...]
-                                     // See FUNC_CODE_INST_CALL_INDIRECT below.
-    FUNC_CODE_DEBUG_LOC        = 35, // Not used in PNaCl.
-    FUNC_CODE_INST_FENCE       = 36, // Not used in PNaCl.
-    FUNC_CODE_INST_CMPXCHG     = 37, // Not used in PNaCl.
-    FUNC_CODE_INST_ATOMICRMW   = 38, // Not used in PNaCl.
-    FUNC_CODE_INST_RESUME      = 39, // Not used in PNaCl.
-    FUNC_CODE_INST_LANDINGPAD  = 40, // Not used in PNaCl.
-    FUNC_CODE_INST_LOADATOMIC  = 41, // Not used in PNaCl.
-    FUNC_CODE_INST_STOREATOMIC = 42, // Not used in PNaCl.
-    FUNC_CODE_INST_FORWARDTYPEREF = 43, // TYPE: [opval, ty]
-    // CALL_INDIRECT: [cc, fnid, returnty, args...]
-    FUNC_CODE_INST_CALL_INDIRECT = 44
-  };
-} // End naclbitc namespace
-} // End llvm namespace
-
-#endif
diff --git a/include/llvm/Bitcode/NaCl/NaClObjDumpStream.h b/include/llvm/Bitcode/NaCl/NaClObjDumpStream.h
deleted file mode 100644
index fd119ca6651..00000000000
--- a/include/llvm/Bitcode/NaCl/NaClObjDumpStream.h
+++ /dev/null
@@ -1,967 +0,0 @@
-//===- NaClObjDumpStream.h --------------------------------------*- C++ -*-===//
-//     Models an objdump stream (bitcode records/assembly code).
-//
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
-//
-//===----------------------------------------------------------------------===//
-
-#ifndef LLVM_BITCODE_NACL_NACLOBJDUMPSTREAM_H
-#define LLVM_BITCODE_NACL_NACLOBJDUMPSTREAM_H
-
-#include "llvm/ADT/STLExtras.h"
-#include "llvm/Bitcode/NaCl/NaClBitcodeParser.h"
-#include "llvm/Support/raw_ostream.h"
-#include <vector>
-#include <algorithm>
-
-namespace llvm {
-namespace naclbitc {
-
-// The default string assumed for a tab.
-static const char *DefaultTab = "        ";
-
-/// Class that implements text indenting for pretty printing text.
-class TextIndenter {
-public:
-  /// Creates a text indenter that indents using the given tab.
-  TextIndenter(const char* Tab = DefaultTab)
-    : Indent(""),
-      Tab(Tab),
-      TabSize(strlen(Tab)),
-      NumTabs(0) {
-    Values.push_back(Indent);
-  }
-
-  virtual ~TextIndenter() {}
-
-  /// Returns the current indentation to use.
-  const std::string &GetIndent() const {
-    return Indent;
-  }
-
-  /// Returns the indent with the given number of tabs.
-  const std::string &GetIndent(unsigned Count) {
-    if (Count >= Values.size()) {
-      // Indents not yet generated, fill in cache to size needed.
-      std::string Results;
-      if (Values.size() > 0) Results = Values.back();
-      for (size_t i = Values.size(); i <= Count; ++i) {
-        Results += Tab;
-        Values.push_back(Results);
-      }
-    }
-    return Values[Count];
-  }
-
-  /// Increments the current indentation by one tab.
-  void Inc() {
-    ++NumTabs;
-    if (NumTabs < Values.size()) {
-      Indent = Values[NumTabs];
-    } else {
-      Indent += Tab;
-      Values.push_back(Indent);
-    }
-  }
-
-  /// Decrements the current indentation by one tab.
-  void Dec() {
-    // Be sure not to underflow!
-    if (NumTabs) {
-      --NumTabs;
-      Indent = Values[NumTabs];
-    }
-  }
-
-  /// Returns the current number of tabs in the current indentation.
-  unsigned GetNumTabs() const {
-    return NumTabs;
-  }
-
-  const char *GetTab() const {
-    return Tab;
-  }
-
-  size_t GetTabSize() const {
-    return TabSize;
-  }
-
-private:
-  // The current indentation to use.
-  std::string Indent;
-  // The set of (previously computed) identations, based on the number
-  // of tabs.
-  std::vector<std::string> Values;
-  // The text defining a tab.
-  const char *Tab;
-  // The size of the tab.
-  size_t TabSize;
-  // The number of tabs currently being used.
-  unsigned NumTabs;
-};
-
-class TextFormatter;
-
-/// This template class maintains a simply pool of directives for
-/// a text formatter. Assumes that all elements in pool are associated
-/// with the same formatter.
-template<class Directive>
-class DirectiveMemoryPool {
-public:
-  DirectiveMemoryPool() {}
-
-  ~DirectiveMemoryPool() {
-    DeleteContainerPointers(FreeList);
-  }
-
-  Directive *Allocate(TextFormatter *Fmtr) {
-    if (FreeList.empty()) return new Directive(Fmtr);
-    Directive *Element = FreeList.back();
-    assert(&Element->GetFormatter() == Fmtr
-           && "Directive memory pool formatter mismatch");
-    FreeList.pop_back();
-    return Element;
-  }
-
-  void Free(Directive *Dir) {
-    FreeList.push_back(Dir);
-  }
-
-private:
-  std::vector<Directive*> FreeList;
-};
-
-/// This class defines a simple formatter for a stream that consists
-/// of a sequence of instructions. In general, this class assumes that
-/// instructions are a single line. In addition, some instructions
-/// define a block around a sequence of instructions. Each block of
-/// instructions is indented to clarify the block structure over that
-/// set of instructions.
-///
-/// To handle line indentation of blocks, this class inherits class
-/// TextIndenter.  The TextIndenter controls how blocks of lines are
-/// indented. At the beginning of a block, the user should call method
-/// Inc.  It should then write out the sequence of instructions to be
-/// indented.  Then, after the last indented instruction of the block
-/// is printed, the user should call method Dec. Nested blocks are
-/// handled by nesting methods Inc and Dec appropriately.
-///
-/// This class mainly focuses on the tools needed to format an
-/// instruction, given a specified viewing width. The issue is that
-/// while instructions should be on a single line, some instructions
-/// are too wide to fit into the viewing width. Hence, we need a way
-/// to deal with line overflow.
-///
-/// The way this class handles the line overflow problem is to
-/// basically force the user to break up the output into a sequence of
-/// tokens. This is done using two streams. The text stream is used to
-/// buffer tokens.  The base stream is the stream to write tokens to
-/// once they have been identified and positioned.  The goal of the
-/// formatter is to decide where the instruction text should be cut
-/// (i.e. between tokens), so that the instruction does not overflow
-/// the viewing width. It also handles the automatic insertion of line
-/// indentation into the base stream when needed.
-///
-/// To make it easy to print tokens to the text stream, we use text
-/// directives.  Whenever a text directive is written to the text
-/// stream, it is assumed that all text written to the text stream
-/// (since the last directive) is an (indivisible) token. The first
-/// thing directives do is move the token from the text stream (if
-/// present) to the base stream.
-///
-/// In addition to defining tokens, text directives can also be
-/// defined to query the formatter state and apply an appropriate
-/// action. An example of this is the space text directive. It only
-/// adds a space if the formatter isn't at the beginning of a new line
-/// (since a newline can act as whitespace).
-///
-/// The text formatter also has a notion of clustering
-/// tokens. Clustering forces a consecutive sequence of tokens to
-/// be treated as a single token, when deciding where to wrap long
-/// lines. In particular, clustered tokens will not be broken up
-/// unless there is no other way to print them, because the cluster is
-/// larger than what can fit in a line.
-///
-/// Clustering is implemented using two passes. In the first pass, the
-/// sequence of tokens/directives are collected to find the clustered
-/// text.  They are also applied to collect any text internal to the
-/// directives. Actions that can change (intraline) indenting are
-/// turned off.
-///
-/// Once the first pass is done, the second pass starts. It begins
-/// with a check to see if the clustered text can fit on the current
-/// line, and adds a newline if necessary. Then it replays the
-/// directives to put the tokens of the cluster into the base stream.
-/// The replay also changes (intraline) indenting as necessary.
-///
-/// Clusters can be nested. In such cases, they are stripped one layer
-/// per pass. Nested clusters are replayed after line wrapping of
-/// outer clusters have been resolved.
-class TextFormatter : public TextIndenter {
-public:
-
-  /// Creates a text formatter to print instructions onto the given
-  /// Base Stream. The viewing width is defined by LineWidth. The
-  /// given tab is the assumed value for tab characters and line
-  /// indents.
-  explicit TextFormatter(raw_ostream &BaseStream,
-                         unsigned LineWidth = 80,
-                         const char *Tab = DefaultTab);
-
-  ~TextFormatter() override;
-
-  /// Returns the user-level text stream of the formatter that tokens
-  /// should be written to.
-  raw_ostream &Tokens() {
-    return TextStream;
-  }
-
-  /// Changes the line width to the given value.
-  void SetLineWidth(unsigned NewLineWidth) {
-    LineWidth = NewLineWidth;
-    if (MinLineWidth > LineWidth) MinLineWidth = LineWidth;
-  }
-
-  // Changes the (default) line-wrap, continuation indent.
-  void SetContinuationIndent(const std::string &Indent) {
-    ContinuationIndent = Indent;
-  }
-
-  /// Base class for all directives. The basic functionality is that it
-  /// has a reference to the text formatter, and is applied by calling
-  /// method apply. This method apply is encorporated into the
-  /// stream operator<<, so that they can be used as part of the
-  /// streamed output.
-  ///
-  /// Method apply extracts any tokens in the text stream.  Moves them
-  /// into the base stream. Finally, it calls virtual method MyApply
-  /// to do the actions of the directive.
-  class Directive {
-    Directive(const Directive&) = delete;
-    void operator=(const Directive&) = delete;
-  public:
-    /// Creates a directive for the given stream.
-    explicit Directive(TextFormatter *Formatter)
-        : Formatter(Formatter) {}
-
-    virtual ~Directive() {}
-
-    /// Returns the formatter associated with the directive.
-    TextFormatter &GetFormatter() const {
-      return *Formatter;
-    }
-
-    /// Does action of directive.
-    void Apply() const {
-      // Start by writing token if Tokens() buffer (if non-empty).
-      Formatter->WriteToken();
-      // Apply the directive.
-      MyApply(false);
-      // Save the directive for replay if we are clustering.
-      MaybeSaveForReplay();
-    }
-
-   protected:
-    // The formatter associated with the directive.
-    TextFormatter *Formatter;
-
-    // Like Apply, but called instead of Apply whey doing a replay.
-    void Reapply() const;
-
-    // Does directive specific action. Replay is true only if the
-    // directive was in a cluster, and it is being called to replay
-    // the directive a second time.
-    virtual void MyApply(bool Replay) const = 0;
-
-    // Adds the directive to the clustered directives if appropriate
-    // (i.e. inside a cluster).
-    virtual void MaybeSaveForReplay() const {
-      if (IsClustering()) AppendForReplay(this);
-    }
-
-    // ***********************************************************
-    // Note: The following have been added so that derived classes
-    // have public access to protected text formatter methods.
-    // (Otherwise, you get a compiler error that they are protected
-    //  in derived classes).
-    // ***********************************************************
-
-    bool IsClustering() const {
-      return Formatter->IsClustering();
-    }
-
-    unsigned GetClusteringLevel() const {
-      return Formatter->GetClusteringLevel();
-    }
-
-    void AppendForReplay(const Directive *D) const {
-      Formatter->AppendForReplay(D);
-    }
-
-    raw_ostream &Tokens() const {
-      return Formatter->Tokens();
-    }
-
-    void WriteToken(const std::string &Token) const {
-      Formatter->WriteToken(Token);
-    }
-
-    void WriteEndline() const {
-      Formatter->WriteEndline();
-    }
-
-    void PopIndent() const {
-      Formatter->PopIndent();
-    }
-
-    void PushIndent() const {
-      Formatter->PushIndent();
-    }
-
-    bool AddLineWrapIfNeeded(unsigned TextSize) const {
-      return Formatter->AddLineWrapIfNeeded(TextSize);
-    }
-
-    void StartClustering() const {
-      Formatter->StartClustering();
-    }
-
-    void FinishClustering() const {
-      Formatter->FinishClustering();
-    }
-  };
-
-protected:
-  // The base stream to send formatted text to.
-  raw_ostream &BaseStream;
-  // Buffer that holds token text.
-  std::string TextBuffer;
-  // The user-level stream for (token) text and directives.
-  raw_string_ostream TextStream;
-  // The buffered token waiting to be flushed to the base stream.
-  // std::string BufferedToken;
-  // The expected line width the formatter should try and match.
-  unsigned LineWidth;
-  // The current position (i.e. column of previous character on line)
-  // associated with the current line in the base stream.
-  unsigned LinePosition;
-  // The stack of (intraline) indents added by PushIndent.
-  std::vector<unsigned> IntralineIndents;
-  // The current intraline indent to use.
-  unsigned CurrentIndent;
-  // The minimum line width. Used to limit indents so that there
-  // will always be at least this amount of space on the line.
-  unsigned MinLineWidth;
-  // True if no characters have moved to the base stream for the
-  // instruction being printed.  Note: This is different than
-  // LinePosition == 0. The latter can be true, when
-  // AtInstructionBeginning is false, if the current line is a
-  // continuation line caused by line overflow.
-  bool AtInstructionBeginning;
-  // The indent string to use for indenting each line of the instruction.
-  std::string LineIndent;
-  // The indent to add on continuation (i.e. overflow) lines for an
-  // instruction.
-  std::string ContinuationIndent;
-  // Counts how nested we are within clustering directives.
-  unsigned ClusteringLevel;
-  // Contains the number of columns needed to hold the generated text,
-  // while clustering. Computed so that we can test if it fits on the
-  // current line.
-  unsigned ClusteredTextSize;
-  // Contains the sequence of directives (including tokens) during
-  // clustering, so that it can be replayed after we determine if a
-  // new line needs to be added.
-  std::vector<const Directive*> ClusteredDirectives;
-
-  // Returns if we are currently inside a cluster.
-  bool IsClustering() const {
-    return ClusteringLevel > 0;
-  }
-
-  // Returns the clustering level of the formatter.
-  unsigned GetClusteringLevel() const {
-    return ClusteringLevel;
-  }
-
-  // Appends the given directive to the list of directives to
-  // replay, if clustering.
-  void AppendForReplay(const Directive *D) {
-    ClusteredDirectives.push_back(D);
-  }
-
-  /// Writes out the token in the text stream. If necessary, moves to
-  // a new line first.
-  void WriteToken() {
-    WriteToken(GetToken());
-  }
-
-  /// Writes out the given token. If necessary, moves to a new line first.
-  void WriteToken(const std::string &Token) {
-    if (Token.empty()) return;
-    AddLineWrapIfNeeded(Token.size());
-    Write(Token);
-  }
-
-  // Extracts the text that has been added to the text stream, and
-  // returns it.
-  std::string GetToken();
-
-  /// Writes out a non-newline character to the base stream.
-  void Write(char ch);
-
-  /// Writes out a string.
-  void Write(const std::string &Text);
-
-  /// Starts a new cluster of tokens.
-  /// Note: We do not allow nested clusterings.
-  void StartClustering();
-
-  /// Ends the existing cluster of tokens.
-  void FinishClustering();
-
-  /// Adds a newline that ends the current instruction being printed.
-  void WriteEndline();
-
-  /// Called just before the first character on a line is printed, to
-  /// add indentation text for the line.
-  virtual void WriteLineIndents();
-
-  /// Analyzes the formatter state to determine if a token of the
-  /// given TextSize can fit on the current line. If it can't fit,
-  /// it wraps the input line. Returns true only if line wrap was
-  /// added by this method.
-  bool AddLineWrapIfNeeded(unsigned TextSize) {
-    if (IsClustering()) {
-      // Don't consider line wrapping until all text is clustered. That
-      // way we have full information on what we should do.
-      return false;
-    } else {
-      // If the text fits on the current line, there is nothing to do.
-      // Otherwise, we should add a newline, unless we are already
-      // at the new line. If we are already at the newline, we can't
-      // split up the token, so just allow line overflow.
-      if (LinePosition + TextSize <= LineWidth
-          || LinePosition == 0)
-        return false;
-      // If reached, it must not fit, so add a line wrap.
-      Write('\n');
-      return true;
-    }
-  }
-
-  /// Pops the current (intraline) indentation and restores it to the
-  /// previous indentation. Used to restore the indentation of
-  /// parenthesized subexpressions, where this should be called on the
-  /// close parenthesis.
-  void PopIndent() {
-    // Be pragmatic. If there is underflow, assume that it was due to
-    // the fact that the caller of this text formatter used a close
-    // directive for a pair of matching parenthesis that crossed
-    // multiple (i.e. instruction) lines.
-    if (IsClustering() || IntralineIndents.empty()) return;
-    CurrentIndent = IntralineIndents.back();
-    IntralineIndents.pop_back();
-  }
-
-  /// Pushes the current (intraline) indentation to match the current
-  /// position within the line. Used to indent parenthesized
-  /// subexpressions, where this should be called on the open
-  /// parenthesis.
-  void PushIndent() {
-    if (IsClustering()) return;
-    IntralineIndents.push_back(CurrentIndent);
-    CurrentIndent = FixIndentValue(LinePosition);
-  }
-
-  /// Fixes the given position, to the appropriate value for setting
-  /// CurrentIndent. That is, it makes sure there is always MinLineWidth
-  /// printable characters on a line.
-  unsigned FixIndentValue(unsigned Position) {
-    return std::min(Position, LineWidth - MinLineWidth);
-  }
-
-private:
-  /// Directive that generates temporary instances to hold tokens from
-  /// the Tokens() stream. Generated when GetToken() is called. Used
-  /// during clustering to regenerate the corresponding extracted
-  /// tokens during a replay.
-  class GetTokenDirective : public Directive {
-    friend class DirectiveMemoryPool<GetTokenDirective>;
-    GetTokenDirective(TextFormatter *Formatter)
-        : Directive(Formatter) {}
-
-  public:
-    ~GetTokenDirective() override {}
-
-    /// Allocates an instance of a GetTokenDirective.
-    /// Note: Will be reclaimed when MyApply is called.
-    static Directive *Allocate(TextFormatter *Formatter,
-                               const std::string &Text);
-
-  protected:
-    void MyApply(bool Replay) const override {
-      WriteToken(Text);
-      if (!IsClustering())
-        Formatter->GetTokenFreeList.Free(const_cast<GetTokenDirective*>(this));
-    }
-
-  private:
-    // The token text.
-    std::string Text;
-  };
-
-  // The set of freed GetTokenDirectives, that can be reused.
-  DirectiveMemoryPool<GetTokenDirective> GetTokenFreeList;
-};
-
-inline raw_ostream &operator<<(raw_ostream &Stream,
-                               const TextFormatter::Directive &Directive) {
-  // Be sure that the directive is defined for the given stream,
-  // before applying the directive, since directives may have
-  // different meanings on different text streams.
-  assert(&Stream == &Directive.GetFormatter().Tokens());
-  Directive.Apply();
-  return Stream;
-}
-
-/// Defines a directive that only tokenizes the text in the Tokens()
-/// stream.
-class TokenizeTextDirective : public TextFormatter::Directive {
-public:
-  explicit TokenizeTextDirective(TextFormatter *Formatter)
-      : TextFormatter::Directive(Formatter) {}
-
-  ~TokenizeTextDirective() override {}
-
-protected:
-  void MyApply(bool Replay) const override {}
-};
-
-/// Defines a token which doesn't need whitespace on either side of
-/// the token to be a valid token (such as punctuation).
-class TokenTextDirective : public TextFormatter::Directive {
-public:
-  TokenTextDirective(TextFormatter *Formatter, const std::string &Str)
-      : TextFormatter::Directive(Formatter), Text(Str) {
-  }
-
-  ~TokenTextDirective() override {}
-
-protected:
-  void MyApply(bool Replay) const override {
-    WriteToken(Text);
-  }
-
-private:
-  std::string Text;
-};
-
-/// Writes out the current token. Then adds a space if the base
-/// stream is not at the beginning of a continuation line.
-class SpaceTextDirective : public TextFormatter::Directive {
-public:
-  SpaceTextDirective(TextFormatter *Formatter, const std::string &Space)
-      : TextFormatter::Directive(Formatter), Space(Space) {}
-
-  explicit SpaceTextDirective(TextFormatter *Formatter)
-      : TextFormatter::Directive(Formatter), Space(" ") {}
-
-  ~SpaceTextDirective() {}
-
-protected:
-  void MyApply(bool Replay) const override {
-    if (!AddLineWrapIfNeeded(Space.size()))
-      WriteToken(Space);
-  }
-private:
-  std::string Space;
-};
-
-/// Adds a newline that ends the current instruction being printed.
-class EndlineTextDirective : public TextFormatter::Directive {
-public:
-  explicit EndlineTextDirective(TextFormatter *Formatter)
-      : TextFormatter::Directive(Formatter) {}
-
-  ~EndlineTextDirective() override {}
-
-protected:
-  void MyApply(bool Replay) const override {
-    WriteEndline();
-  }
-};
-
-/// Inserts a token and then pushes the current indent to the position
-/// after the token. Used to model a open parenthesis.
-class OpenTextDirective : public TokenTextDirective {
-public:
-
-  // Creates an open using the given indent.
-  OpenTextDirective(TextFormatter *Formatter, const std::string &Text)
-      : TokenTextDirective(Formatter, Text) {}
-
-  ~OpenTextDirective() override {}
-
-protected:
-  void MyApply(bool Replay) const override {
-    TokenTextDirective::MyApply(Replay);
-    PushIndent();
-  }
-};
-
-/// Inserts a token and then pops current indent.  Used to model a
-/// close parenthesis.
-class CloseTextDirective : public TokenTextDirective {
-public:
-  CloseTextDirective(TextFormatter *Formatter, const std::string &Text)
-      : TokenTextDirective(Formatter, Text) {}
-
-  ~CloseTextDirective() override {}
-
-protected:
-  void MyApply(bool Replay) const override {
-    TokenTextDirective::MyApply(Replay);
-    PopIndent();
-  }
-};
-
-/// Defines the beginning of a token cluster, which should be put on the
-/// same line if possible.
-class StartClusteringDirective : public TextFormatter::Directive {
-public:
-  explicit StartClusteringDirective(TextFormatter *Formatter)
-      : TextFormatter::Directive(Formatter) {}
-
-  ~StartClusteringDirective() override {}
-
-protected:
-  void MyApply(bool Replay) const override {
-    StartClustering();
-  }
-
-  void MaybeSaveForReplay() const override {
-    if (GetClusteringLevel() > 1) AppendForReplay(this);
-  }
-};
-
-class FinishClusteringDirective : public TextFormatter::Directive {
-public:
-  explicit FinishClusteringDirective(TextFormatter *Formatter)
-      : TextFormatter::Directive(Formatter) {}
-
-  ~FinishClusteringDirective() override {}
-
-protected:
-  void MyApply(bool Replay) const override {
-    FinishClustering();
-  }
-};
-
-class ObjDumpStream;
-
-/// Models that an abbreviation index is not specified when dumping a
-/// bitcode record.
-static int32_t ABBREV_INDEX_NOT_SPECIFIED = -1;
-
-/// The formatter used for dumping records in ObjDumpStream.
-class RecordTextFormatter : public TextFormatter {
-  RecordTextFormatter(const RecordTextFormatter&) = delete;
-  RecordTextFormatter &operator=(const RecordTextFormatter&) = delete;
-public:
-  /// The address write width used to print a bit address, when
-  /// printing records.
-  static const unsigned AddressWriteWidth = 10;
-
-  explicit RecordTextFormatter(ObjDumpStream *ObjDump);
-
-  ~RecordTextFormatter() override {}
-
-  /// Writes out the given record of values as an instruction.
-  void WriteValues(uint64_t Bit,
-                   const llvm::NaClBitcodeValues &Values,
-                   int32_t AbbrevIndex = ABBREV_INDEX_NOT_SPECIFIED);
-
-  /// Returns text corresponding to an empty label column.
-  std::string GetEmptyLabelColumn();
-
-protected:
-  void WriteLineIndents() override;
-
-private:
-  // The address label associated with the current line.
-  std::string Label;
-  // The open brace '<' for a record.
-  OpenTextDirective OpenBrace;
-  // The close brace '>' for a record.
-  CloseTextDirective CloseBrace;
-  // A comma between elements in a record.
-  TokenTextDirective Comma;
-  // A space inside a record.
-  SpaceTextDirective Space;
-  // End the line.
-  EndlineTextDirective Endline;
-  // Start clustering tokens.
-  StartClusteringDirective StartCluster;
-  // End clustering tokens.
-  FinishClusteringDirective FinishCluster;
-
-  // Generates an address label with padding to match AddressWriteWidth;
-  std::string getBitAddress(uint64_t Bit);
-};
-
-/// Implements a stream to print out bitcode records, assembly code,
-/// comments, and errors. The general format is to print records and
-/// assembly side by side. Comments and errors (associated with the
-/// record and/or assembly code) follow each printed record.
-///
-/// Alignment of records, assembly, comments, and errors is done by
-/// buffering assembly, comments, and errors until a write or flush
-/// occurs.  Then, the various streams are stitched together to
-/// produce the corresponding output text. There are two buffers: one
-/// holds the assembly, and the other holds comments and errors.
-///
-/// There are exactly two methods that can cause text to be added to
-/// the objdump stream. Method Flush just flushes out the assembly and
-/// comments/errors buffer without printing a record. If there is no
-/// buffered assembly/comments/errors, nothing is done. Method Write
-/// prints the given record, and also flushes out the assembly and
-/// comments/errors buffer. Hence, in general, comments and errors
-/// follow the record/assembly. However, if you want them to appear
-/// before, use method Flush.
-///
-/// The constructor is defined with two flags: DumpRecords and
-/// DumpAssembly.  Comments and errors are flushed on every write,
-/// independent of these flags.  Records are printed out only if
-/// DumpRecords is true. Assembly is flushed only if DumpAssembly is
-/// true.
-///
-/// To buffer assembly, call method Assembly to get a (string) stream
-/// to buffer the assembly code. To buffer comments, call method
-/// Comments() to get a (string) stream to buffer the comments.
-///
-/// To buffer an error, call method Error. This method will increment
-/// the error count, and return the comments stream after writing
-/// "Error(byte:bit): ".
-///
-/// If a single line of text is buffered into the assembly stream, no
-/// new line is needed. The corresponding call to Write will
-/// automatically insert the newline for you, if you did not add
-/// it. If multiple lines are to be buffered into the assembly stream,
-/// each line must be separated with a newline character.  It is
-/// always safe to end all assembly lines with a newline character.
-///
-/// Also note that this class takes care of formatting records to fit
-/// into a calculated record width (based on value set to
-/// RecordWidth).  On the other hand, we assume that the assembly is
-/// formatted by the caller (i.e. owner of this object).
-class ObjDumpStream {
-  ObjDumpStream(const ObjDumpStream&) = delete;
-  void operator=(const ObjDumpStream&) = delete;
-public:
-  /// The default number of error messages that will be printed before
-  /// execution is stopped due to too many errors.
-  static unsigned DefaultMaxErrors;
-
-  /// The default value for the column that separates records and
-  /// assembly, when DumpRecords and DumpAssembly is true.
-  static unsigned ComboObjDumpSeparatorColumn;
-
-  /// The default value for line width when DumpRecords is true,
-  /// and DumpAssembly is false. This value is typically larger
-  /// than ComboObjDumpSeparatorColumn, since the entire line
-  /// can be used to print records.
-  static unsigned RecordObjectDumpLength;
-
-  /// Creates an objdump stream which will dump records, assembly,
-  /// comments and errors into a single (user proved Stream).  When
-  /// DumpRecords is true, the contents of records will be
-  /// dumped. When DumpAssembly is true, the corresponding assembly
-  /// will be printed. When both are true, the records and assembly
-  /// will be printed side by side. When both are false, only comments
-  /// and errors will be printed.
-  ObjDumpStream(raw_ostream &Stream, bool DumpRecords, bool DumpAssembly);
-
-  ~ObjDumpStream() { Flush(); }
-
-  /// Returns stream to buffer assembly that will be printed during the
-  /// next write call.
-  raw_ostream &Assembly() {
-    return AssemblyStream;
-  }
-
-  /// Returns stream to buffer records that will be printed during the
-  /// next write call.
-  raw_ostream &Records() {
-    return RecordStream;
-  }
-
-  /// Returns stream to buffer messages that will be printed during the
-  // next write call.
-  raw_ostream &Comments() {
-    return MessageStream;
-  }
-
-  /// Prints "Warning(Bit/8:Bit%8): " onto the comments stream, and
-  /// then returns the comments stream. In general, warnings will be
-  /// printed after the next record, unless a call to Flush is made.
-  raw_ostream &Warning() {
-    return WarningAt(LastKnownBit);
-  }
-
-  /// Prints "Warning(Bit/8:Bit%8): " onto the comments stream, and
-  /// then returns the comments stream. In general, warnings will be
-  /// printed after the next record, unless a call to Flush is made.
-  raw_ostream &WarningAt(uint64_t Bit) {
-    LastKnownBit = Bit;
-    return naclbitc::ErrorAt(Comments(), naclbitc::Warning, Bit);
-  }
-
-  /// Prints "Error(Bit/8:Bit%8): " onto the comments stream using the
-  /// last know bit position of the input. Then, it records that an
-  /// error has occurred and returns the comments stream. In general
-  /// errors will be printed after the next record, unless a call to
-  /// Flush is made.
-  raw_ostream &Error() {
-    return ErrorAt(LastKnownBit);
-  }
-
-  /// Prints "Error(Bit/8:Bit%8): " onto the comments stream at the
-  /// given Bit position. Then, it records that an error has occurred
-  /// and returns the comments stream. In general errors will be
-  /// printed after the next record, unless a call to Flush is made.
-  raw_ostream &ErrorAt(uint64_t Bit) {
-    return ErrorAt(naclbitc::Error, Bit);
-  }
-
-  /// Prints "Level(Bit/8:Bit%8): " onto the comments stream at the
-  /// given bit position and severity Level.  Then, it records that an
-  /// error has occurred and then returns the comments stream. In
-  /// general errors will be printed after the next record, unless a
-  /// call to Flush is made.
-  raw_ostream &ErrorAt(naclbitc::ErrorLevel Level, uint64_t Bit);
-
-  /// Dumps a record (at the given bit), along with all buffered assembly,
-  /// comments, and errors, into the objdump stream.
-  void Write(uint64_t Bit,
-             const llvm::NaClBitcodeRecordData &Record,
-             int32_t AbbrevIndex = ABBREV_INDEX_NOT_SPECIFIED) {
-    LastKnownBit = Bit;
-    NaClBitcodeValues Values(Record);
-    RecordFormatter.WriteValues(Bit, Values, AbbrevIndex);
-    Flush();
-  }
-
-  /// Dumps the buffered assembly, comments, and errors, without any
-  /// corresponding record, into the objdump stream.
-  void Flush();
-
-  /// Flushes out the last record and/or error, and then stops
-  /// the executable. Should be called immediately after generating
-  /// the error message using ErrorAt(naclbitc::Fatal,...).
-  void FlushThenQuit();
-
-  /// Increments the record indent by one.
-  void IncRecordIndent() {
-    RecordFormatter.Inc();
-  }
-
-  /// Decrements the record indent by one.
-  void DecRecordIndent() {
-    RecordFormatter.Dec();
-  }
-
-  /// Returns the record indenter being used by the objdump stream, for
-  /// the purposes of querying state.
-  const TextIndenter &GetRecordIndenter() {
-    return RecordFormatter;
-  }
-
-  /// Returns the number of errors reported to the dump stream.
-  unsigned GetNumErrors() const {
-    return NumErrors;
-  }
-
-  // Returns true if dumping record.
-  bool GetDumpRecords() const {
-    return DumpRecords;
-  }
-
-  // Returns true if dumping assembly.
-  bool GetDumpAssembly() const {
-    return DumpAssembly;
-  }
-
-  /// Changes the maximum number of errors allowed.
-  void SetMaxErrors(unsigned NewMax) {
-    MaxErrors = NewMax;
-  }
-
-  /// Changes the width allowed for records (from the default).
-  void SetRecordWidth(unsigned Width) {
-    RecordWidth = Width;
-  }
-
-  unsigned GetRecordWidth() const {
-    return RecordWidth;
-  }
-
-  /// Changes the column separator character to the given value.
-  void SetColumnSeparator(char Separator) {
-    ColumnSeparator = Separator;
-  }
-
-  /// Changes the internal state, to assume one is processing a record
-  /// at the given bit.
-  void SetRecordBitAddress(uint64_t Bit) {
-    LastKnownBit = Bit;
-  }
-
-private:
-  // The stream to dump to.
-  raw_ostream &Stream;
-  // True if records should be dumped to the dump stream.
-  bool DumpRecords;
-  // True if assembly text should be dumped to the dump stream.
-  bool DumpAssembly;
-  // The number of errors reported.
-  unsigned NumErrors;
-  // The maximum number of errors before quitting.
-  unsigned MaxErrors;
-  // The number of columns available to print bitcode records.
-  unsigned RecordWidth;
-  // The buffer for assembly to be printed during the next write.
-  std::string AssemblyBuffer;
-  // The stream to buffer assembly into the assembly buffer.
-  raw_string_ostream AssemblyStream;
-  // The buffer for comments and errors.
-  std::string MessageBuffer;
-  // The stream to buffer comments and errors into the message.
-  raw_string_ostream MessageStream;
-  // The character used to separate records from assembly.
-  char ColumnSeparator;
-  // The last known bit passed to the objdump object. Used as default
-  // for automatically generated errors.
-  uint64_t LastKnownBit;
-  /// The buffer for records to be printed to during the next write.
-  std::string RecordBuffer;
-  /// The stream to buffer recordds into the record buffer.
-  raw_string_ostream RecordStream;
-  /// The text formatter for generating records.
-  RecordTextFormatter RecordFormatter;
-
-  // Resets assembly and buffers.
-  void ResetBuffers() {
-    RecordBuffer.clear();
-    AssemblyBuffer.clear();
-    MessageBuffer.clear();
-  }
-};
-
-}
-}
-
-#endif
diff --git a/include/llvm/Bitcode/NaCl/NaClRandNumGen.h b/include/llvm/Bitcode/NaCl/NaClRandNumGen.h
deleted file mode 100644
index 620eadc4aa9..00000000000
--- a/include/llvm/Bitcode/NaCl/NaClRandNumGen.h
+++ /dev/null
@@ -1,65 +0,0 @@
-//===- NaClRandNumGen.h - random number generator ---------------*- C++ -*-===//
-//
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
-//
-//===----------------------------------------------------------------------===//
-//
-// This file defines a random number generator API for 64-bit unsigned
-// values, and a corresponding default implementation.
-//
-// *** WARNING *** One should assume that random number generators are not
-// thread safe.
-//
-//===----------------------------------------------------------------------===//
-
-#ifndef LLVM_BITCODE_NACL_NACLRANDNUMGEN_H
-#define LLVM_BITCODE_NACL_NACLRANDNUMGEN_H
-
-#include "llvm/ADT/StringRef.h"
-#include <random>
-#include <stdint.h>
-
-namespace naclfuzz {
-
-/// Defines API for a random number generator to use with fuzzing.
-class RandomNumberGenerator {
-  RandomNumberGenerator(const RandomNumberGenerator&) = delete;
-  void operator=(const RandomNumberGenerator&) = delete;
-public:
-  virtual ~RandomNumberGenerator();
-  /// Returns a random number.
-  virtual uint64_t operator()() = 0;
-  // Returns a random value in [0..Limit)
-  uint64_t chooseInRange(uint64_t Limit) {
-    return (*this)() % Limit;
-  }
-protected:
-  RandomNumberGenerator() {}
-};
-
-/// Defines a random number generator based on C++ generator std::mt19937_64.
-class DefaultRandomNumberGenerator : public RandomNumberGenerator {
-  DefaultRandomNumberGenerator(const DefaultRandomNumberGenerator&) = delete;
-  void operator=(const DefaultRandomNumberGenerator&) = delete;
-public:
-  DefaultRandomNumberGenerator(llvm::StringRef Seed);
-  uint64_t operator()() final;
-  ~DefaultRandomNumberGenerator() final {}
-  // Resets random number seed by salting the seed of constructor with Salt.
-  void saltSeed(uint64_t Salt);
-private:
-  // 64-bit Mersenne Twister by Matsumoto and Nishimura, 2000
-  // http://en.cppreference.com/w/cpp/numeric/random/mersenne_twister_engine
-  // This RNG is deterministically portable across C++11
-  // implementations.
-  std::mt19937_64 Generator;
-  // Seed for the random number generator.
-  std::string Seed;
-};
-
-} // end of namespace naclfuzz
-
-#endif // LLVM_BITCODE_NACL_NACLRANDNUMGEN_H
diff --git a/include/llvm/Bitcode/NaCl/NaClReaderWriter.h b/include/llvm/Bitcode/NaCl/NaClReaderWriter.h
deleted file mode 100644
index 6ac74722b3e..00000000000
--- a/include/llvm/Bitcode/NaCl/NaClReaderWriter.h
+++ /dev/null
@@ -1,142 +0,0 @@
-//===-- llvm/Bitcode/NaCl/NaClReaderWriter.h - ------------------*- C++ -*-===//
-//      NaCl Bitcode reader/writer.
-//
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
-//
-//===----------------------------------------------------------------------===//
-//
-// This header defines interfaces to read and write NaCl bitcode wire format
-// files.
-//
-//===----------------------------------------------------------------------===//
-
-#ifndef LLVM_BITCODE_NACL_NACLREADERWRITER_H
-#define LLVM_BITCODE_NACL_NACLREADERWRITER_H
-
-#include "llvm/Support/CommandLine.h"
-#include "llvm/Support/ErrorOr.h"
-#include "llvm/Support/MemoryBuffer.h"
-
-#include <string>
-
-namespace llvm {
-  class LLVMContext;
-  class Module;
-  class NaClBitcodeHeader;
-  class NaClBitstreamWriter;
-  class StreamingMemoryObject;
-  class raw_ostream;
-
-  /// Defines the data layout used for PNaCl bitcode files. We set the
-  /// data layout of the module in the bitcode readers rather than in
-  /// pnacl-llc so that 'opt' will also use the correct data layout if
-  /// it is run on a pexe.
-  extern const char *PNaClDataLayout;
-
-  /// Allows (function) local symbol tables (unsupported) in PNaCl bitcode
-  /// files.
-  extern cl::opt<bool> PNaClAllowLocalSymbolTables;
-
-  /// \brief Defines the integer bit size used to model pointers in PNaCl.
-  static const unsigned PNaClIntPtrTypeBitSize = 32;
-
-  /// Read the header of the specified bitcode buffer and prepare for lazy
-  /// deserialization of function bodies.  If successful, this takes ownership
-  /// of 'Buffer' (extending its lifetime).  On error, this returns an error code
-  /// and deletes Buffer.
-  ///
-  /// When Verbose is non-null, more descriptive error messages are also
-  /// written to Verbose.
-  ///
-  /// The AcceptSupportedOnly argument is used to decide which PNaCl versions
-  /// of the PNaCl bitcode to accept. There are three forms:
-  ///    1) Readable and supported.
-  ///    2) Readable and unsupported. Allows testing of code before becoming
-  ///       supported, as well as running experiments on the bitcode format.
-  ///    3) Unreadable.
-  /// When AcceptSupportedOnly is true, only form 1 is allowed. When
-  /// AcceptSupportedOnly is false, forms 1 and 2 are allowed.
-  ErrorOr<Module *> getNaClLazyBitcodeModule(
-      std::unique_ptr<MemoryBuffer> &&Buffer, LLVMContext &Context,
-      raw_ostream *Verbose = nullptr, bool AcceptSupportedOnly = true);
-
-  /// Read the header of the specified stream and prepare for lazy
-  /// deserialization and streaming of function bodies. On error,
-  /// this returns null, and fills in *ErrMsg with an error description
-  /// if ErrMsg is non-null.
-  ///
-  /// See getNaClLazyBitcodeModule for an explanation of arguments
-  /// Verbose, AcceptSupportedOnly.
-  /// TODO(kschimpf): Refactor this and getStreamedBitcodeModule to use
-  /// ErrorOr<Module *> API so that all methods have the same interface.
-  Module *getNaClStreamedBitcodeModule(const std::string &name,
-                                       StreamingMemoryObject *streamer,
-                                       LLVMContext &Context,
-                                       raw_ostream *Verbose = nullptr,
-                                       std::string *ErrMsg = nullptr,
-                                       bool AcceptSupportedOnly = true);
-
-  /// Read the bitcode file from a buffer, returning the module.
-  ///
-  /// See getNaClLazyBitcodeModule for an explanation of arguments
-  /// Verbose, AcceptSupportedOnly.
-  ErrorOr<Module *> NaClParseBitcodeFile(MemoryBufferRef Buffer,
-                                         LLVMContext &Context,
-                                         raw_ostream *Verbose = nullptr,
-                                         bool AcceptSupportedOnly = true);
-
-  /// Read the textual bitcode records in Filename, returning the module.
-  /// Note: If Filename is "-", stdin will be read.
-  ///
-  /// See getNaClLazyBitcodeModule for an explanation of argument Verbose.
-  ErrorOr<Module *> parseNaClBitcodeText(const std::string &Filename,
-                                         LLVMContext &Context,
-                                         raw_ostream *Verbose = nullptr);
-
-  /// Write the specified module to the specified raw output stream, using
-  /// PNaCl wire format.  For streams where it matters, the given stream
-  /// should be in "binary" mode.
-  ///
-  /// The AcceptSupportedOnly argument is used to decide which PNaCl versions
-  /// of the PNaCl bitcode to generate. There are two forms:
-  ///    1) Writable and supported.
-  ///    2) Writable and unsupported. Allows testing of code before becoming
-  ///       supported, as well as running experiments on the bitcode format.
-  /// When AcceptSupportedOnly is true, only form 1 is allowed. When
-  /// AcceptSupportedOnly is false, forms 1 and 2 are allowed.
-  void NaClWriteBitcodeToFile(const Module *M, raw_ostream &Out,
-                              bool AcceptSupportedOnly = true);
-
-  /// isNaClBitcode - Return true if the given bytes are the magic bytes for
-  /// PNaCl bitcode wire format.
-  ///
-  inline bool isNaClBitcode(const unsigned char *BufPtr,
-                        const unsigned char *BufEnd) {
-    return BufPtr+4 <= BufEnd &&
-        BufPtr[0] == 'P' &&
-        BufPtr[1] == 'E' &&
-        BufPtr[2] == 'X' &&
-        BufPtr[3] == 'E';
-  }
-
-  /// NaClWriteHeader - Generate a default header (using the version
-  /// number defined by kPNaClVersion) and write to the corresponding
-  /// bitcode stream.
-  void NaClWriteHeader(NaClBitstreamWriter &Stream, bool AcceptSupportedOnly);
-
-  // NaClWriteHeader - Write the contents of the bitcode header to the
-  // corresponding bitcode stream.
-  void NaClWriteHeader(const NaClBitcodeHeader &Header,
-                       NaClBitstreamWriter &Stream);
-
-  /// NaClObjDump - Read PNaCl bitcode file from input, and print a
-  /// textual representation of its contents. NoRecords and NoAssembly
-  /// define what should not be included in the dump.
-  bool NaClObjDump(MemoryBufferRef Input, raw_ostream &output,
-                   bool NoRecords, bool NoAssembly);
-
-} // end llvm namespace
-#endif
diff --git a/include/llvm/Bitcode/ReaderWriter.h b/include/llvm/Bitcode/ReaderWriter.h
index e73552992bf..6f92e2e1b3c 100644
--- a/include/llvm/Bitcode/ReaderWriter.h
+++ b/include/llvm/Bitcode/ReaderWriter.h
@@ -15,6 +15,7 @@
 #define LLVM_BITCODE_READERWRITER_H
 
 #include "llvm/IR/DiagnosticInfo.h"
+#include "llvm/Support/Endian.h"
 #include "llvm/Support/ErrorOr.h"
 #include "llvm/Support/MemoryBuffer.h"
 #include <memory>
@@ -22,17 +23,17 @@
 
 namespace llvm {
   class BitstreamWriter;
+  class DataStreamer;
   class LLVMContext;
   class Module;
   class ModulePass;
-  class StreamingMemoryObject; // @LOCALMOD
   class raw_ostream;
 
   /// Read the header of the specified bitcode buffer and prepare for lazy
   /// deserialization of function bodies. If ShouldLazyLoadMetadata is true,
   /// lazily load metadata as well. If successful, this moves Buffer. On
   /// error, this *does not* move Buffer.
-  ErrorOr<Module *>
+  ErrorOr<std::unique_ptr<Module>>
   getLazyBitcodeModule(std::unique_ptr<MemoryBuffer> &&Buffer,
                        LLVMContext &Context,
                        DiagnosticHandlerFunction DiagnosticHandler = nullptr,
@@ -41,7 +42,8 @@ namespace llvm {
   /// Read the header of the specified stream and prepare for lazy
   /// deserialization and streaming of function bodies.
   ErrorOr<std::unique_ptr<Module>> getStreamedBitcodeModule(
-      StringRef Name, StreamingMemoryObject *Streamer, LLVMContext &Context, //@LOCALMOD
+      StringRef Name, std::unique_ptr<DataStreamer> Streamer,
+      LLVMContext &Context,
       DiagnosticHandlerFunction DiagnosticHandler = nullptr);
 
   /// Read the header of the specified bitcode buffer and extract just the
@@ -52,7 +54,7 @@ namespace llvm {
                          DiagnosticHandlerFunction DiagnosticHandler = nullptr);
 
   /// Read the specified bitcode file, returning the module.
-  ErrorOr<Module *>
+  ErrorOr<std::unique_ptr<Module>>
   parseBitcodeFile(MemoryBufferRef Buffer, LLVMContext &Context,
                    DiagnosticHandlerFunction DiagnosticHandler = nullptr);
 
@@ -132,14 +134,8 @@ namespace llvm {
     // Must contain the header!
     if (BufEnd-BufPtr < KnownHeaderSize) return true;
 
-    unsigned Offset = ( BufPtr[OffsetField  ]        |
-                       (BufPtr[OffsetField+1] << 8)  |
-                       (BufPtr[OffsetField+2] << 16) |
-                       (BufPtr[OffsetField+3] << 24));
-    unsigned Size   = ( BufPtr[SizeField    ]        |
-                       (BufPtr[SizeField  +1] << 8)  |
-                       (BufPtr[SizeField  +2] << 16) |
-                       (BufPtr[SizeField  +3] << 24));
+    unsigned Offset = support::endian::read32le(&BufPtr[OffsetField]);
+    unsigned Size = support::endian::read32le(&BufPtr[SizeField]);
 
     // Verify that Offset+Size fits in the file.
     if (VerifyBufferSize && Offset+Size > unsigned(BufEnd-BufPtr))
@@ -150,7 +146,7 @@ namespace llvm {
   }
 
   const std::error_category &BitcodeErrorCategory();
-  enum class BitcodeError { InvalidBitcodeSignature, CorruptedBitcode };
+  enum class BitcodeError { InvalidBitcodeSignature = 1, CorruptedBitcode };
   inline std::error_code make_error_code(BitcodeError E) {
     return std::error_code(static_cast<int>(E), BitcodeErrorCategory());
   }
@@ -163,7 +159,7 @@ namespace llvm {
     BitcodeDiagnosticInfo(std::error_code EC, DiagnosticSeverity Severity,
                           const Twine &Msg);
     void print(DiagnosticPrinter &DP) const override;
-    std::error_code getError() const { return EC; };
+    std::error_code getError() const { return EC; }
 
     static bool classof(const DiagnosticInfo *DI) {
       return DI->getKind() == DK_Bitcode;
diff --git a/include/llvm/CodeGen/Analysis.h b/include/llvm/CodeGen/Analysis.h
index c4b94ede4f5..82d1e8ada17 100644
--- a/include/llvm/CodeGen/Analysis.h
+++ b/include/llvm/CodeGen/Analysis.h
@@ -64,7 +64,7 @@ inline unsigned ComputeLinearIndex(Type *Ty,
 /// If Offsets is non-null, it points to a vector to be filled in
 /// with the in-memory offsets of each of the individual values.
 ///
-void ComputeValueVTs(const TargetLowering &TLI, Type *Ty,
+void ComputeValueVTs(const TargetLowering &TLI, const DataLayout &DL, Type *Ty,
                      SmallVectorImpl<EVT> &ValueVTs,
                      SmallVectorImpl<uint64_t> *Offsets = nullptr,
                      uint64_t StartingOffset = 0);
diff --git a/include/llvm/CodeGen/AsmPrinter.h b/include/llvm/CodeGen/AsmPrinter.h
index 3eddfa4ba57..6a81bb8543f 100644
--- a/include/llvm/CodeGen/AsmPrinter.h
+++ b/include/llvm/CodeGen/AsmPrinter.h
@@ -19,6 +19,7 @@
 #include "llvm/ADT/MapVector.h"
 #include "llvm/ADT/Twine.h"
 #include "llvm/CodeGen/MachineFunctionPass.h"
+#include "llvm/CodeGen/DwarfStringPoolEntry.h"
 #include "llvm/IR/InlineAsm.h"
 #include "llvm/Support/DataTypes.h"
 #include "llvm/Support/ErrorHandling.h"
@@ -53,6 +54,7 @@ class MCSection;
 class MCStreamer;
 class MCSubtargetInfo;
 class MCSymbol;
+class MCTargetOptions;
 class MDNode;
 class DwarfDebug;
 class Mangler;
@@ -78,7 +80,7 @@ class AsmPrinter : public MachineFunctionPass {
   /// This is the MCStreamer object for the file we are generating. This
   /// contains the transient state for the current translation unit that we are
   /// generating (such as the current section etc).
-  MCStreamer &OutStreamer;
+  std::unique_ptr<MCStreamer> OutStreamer;
 
   /// The current machine function.
   const MachineFunction *MF;
@@ -163,6 +165,9 @@ class AsmPrinter : public MachineFunctionPass {
   /// Return information about data layout.
   const DataLayout &getDataLayout() const;
 
+  /// Return the pointer size from the TargetMachine
+  unsigned getPointerSize() const;
+
   /// Return information about subtarget.
   const MCSubtargetInfo &getSubtargetInfo() const;
 
@@ -233,6 +238,11 @@ class AsmPrinter : public MachineFunctionPass {
   ///
   void EmitJumpTableInfo();
 
+  /// Emit the control variable for an emulated TLS variable.
+  virtual void EmitEmulatedTLSControlVariable(const GlobalVariable *GV,
+                                              MCSymbol *EmittedSym,
+                                              bool AllZeroInitValue);
+
   /// Emit the specified global variable to the .s file.
   virtual void EmitGlobalVariable(const GlobalVariable *GV);
 
@@ -252,7 +262,7 @@ class AsmPrinter : public MachineFunctionPass {
   const MCExpr *lowerConstant(const Constant *CV);
 
   /// \brief Print a general LLVM constant to the .s file.
-  void EmitGlobalConstant(const Constant *CV);
+  void EmitGlobalConstant(const DataLayout &DL, const Constant *CV);
 
   /// \brief Unnamed constant global variables solely contaning a pointer to
   /// another globals variable act like a global variable "proxy", or GOT
@@ -276,18 +286,6 @@ class AsmPrinter : public MachineFunctionPass {
   // Targets can, or in the case of EmitInstruction, must implement these to
   // customize output.
 
-  // @LOCALMOD-START
-  /// UseReadOnlyJumpTables - true if JumpTableInfo must be in rodata.
-  virtual bool UseReadOnlyJumpTables() const { return false; }
-  /// GetTargetBasicBlockAlign - the target alignment for basic blocks.
-  virtual unsigned GetTargetBasicBlockAlign() const { return 0; }
-  /// GetTargetLabelAlign - Get optional alignment for TargetOpcode
-  /// labels E.g., EH_LABEL.
-  virtual unsigned GetTargetLabelAlign(const MachineInstr *MI) const {
-    return 0;
-  }
-  // @LOCALMOD-END
-
   /// This virtual method can be overridden by targets that want to emit
   /// something at the start of their file.
   virtual void EmitStartOfAsmFile(Module &) {}
@@ -302,12 +300,7 @@ class AsmPrinter : public MachineFunctionPass {
 
   /// Targets can override this to emit stuff after the last basic block in the
   /// function.
-  virtual void EmitFunctionBodyEnd() {
-    // @LOCALMOD-START
-    unsigned NextFunctionAlignment = GetTargetBasicBlockAlign();
-    if (NextFunctionAlignment) EmitAlignment(NextFunctionAlignment);
-    // @LOCALMOD-END
-  }
+  virtual void EmitFunctionBodyEnd() {}
 
   /// Targets can override this to emit stuff at the start of a basic block.
   /// By default, this method prints the label for the specified
@@ -332,7 +325,9 @@ class AsmPrinter : public MachineFunctionPass {
 
   /// Targets can override this to change how global constants that are part of
   /// a C++ static/global constructor list are emitted.
-  virtual void EmitXXStructor(const Constant *CV) { EmitGlobalConstant(CV); }
+  virtual void EmitXXStructor(const DataLayout &DL, const Constant *CV) {
+    EmitGlobalConstant(DL, CV);
+  }
 
   /// Return true if the basic block has exactly one predecessor and the control
   /// transfer mechanism between the predecessor and this block is a
@@ -433,10 +428,18 @@ class AsmPrinter : public MachineFunctionPass {
   /// Emit reference to a ttype global with a specified encoding.
   void EmitTTypeReference(const GlobalValue *GV, unsigned Encoding) const;
 
-  /// Emit the 4-byte offset of Label from the start of its section.  This can
-  /// be done with a special directive if the target supports it (e.g. cygwin)
-  /// or by emitting it as an offset from a label at the start of the section.
-  void emitSectionOffset(const MCSymbol *Label) const;
+  /// Emit a reference to a symbol for use in dwarf. Different object formats
+  /// represent this in different ways. Some use a relocation others encode
+  /// the label offset in its section.
+  void emitDwarfSymbolReference(const MCSymbol *Label,
+                                bool ForceOffset = false) const;
+
+  /// Emit the 4-byte offset of a string from the start of its section.
+  ///
+  /// When possible, emit a DwarfStringPool section offset without any
+  /// relocations, and without using the symbol.  Otherwise, defers to \a
+  /// emitDwarfSymbolReference().
+  void emitDwarfStringOffset(DwarfStringPoolEntryRef S) const;
 
   /// Get the value for DW_AT_APPLE_isa. Zero if no isa encoding specified.
   virtual unsigned getISAEncoding() { return 0; }
@@ -510,11 +513,12 @@ class AsmPrinter : public MachineFunctionPass {
   mutable unsigned Counter;
 
   /// This method emits the header for the current function.
-  void EmitFunctionHeader();
+  virtual void EmitFunctionHeader();
 
   /// Emit a blob of inline asm to the output streamer.
   void
   EmitInlineAsm(StringRef Str, const MCSubtargetInfo &STI,
+                const MCTargetOptions &MCOptions,
                 const MDNode *LocMDNode = nullptr,
                 InlineAsm::AsmDialect AsmDialect = InlineAsm::AD_ATT) const;
 
@@ -538,7 +542,8 @@ class AsmPrinter : public MachineFunctionPass {
   void EmitLLVMUsedList(const ConstantArray *InitList);
   /// Emit llvm.ident metadata in an '.ident' directive.
   void EmitModuleIdents(Module &M);
-  void EmitXXStructorList(const Constant *List, bool isCtor);
+  void EmitXXStructorList(const DataLayout &DL, const Constant *List,
+                          bool isCtor);
   GCMetadataPrinter *GetOrCreateGCPrinter(GCStrategy &C);
 };
 }
diff --git a/include/llvm/CodeGen/BasicTTIImpl.h b/include/llvm/CodeGen/BasicTTIImpl.h
index c5efef32d07..796a78c16ac 100644
--- a/include/llvm/CodeGen/BasicTTIImpl.h
+++ b/include/llvm/CodeGen/BasicTTIImpl.h
@@ -91,8 +91,10 @@ class BasicTTIImplBase : public TargetTransformInfoImplCRTPBase<T> {
   }
 
 protected:
-  explicit BasicTTIImplBase(const TargetMachine *TM)
-      : BaseT(TM->getDataLayout()) {}
+  explicit BasicTTIImplBase(const TargetMachine *TM, const DataLayout &DL)
+      : BaseT(DL) {}
+
+  using TargetTransformInfoImplBase::DL;
 
 public:
   // Provide value semantics. MSVC requires that we spell all of these out.
@@ -100,14 +102,6 @@ class BasicTTIImplBase : public TargetTransformInfoImplCRTPBase<T> {
       : BaseT(static_cast<const BaseT &>(Arg)) {}
   BasicTTIImplBase(BasicTTIImplBase &&Arg)
       : BaseT(std::move(static_cast<BaseT &>(Arg))) {}
-  BasicTTIImplBase &operator=(const BasicTTIImplBase &RHS) {
-    BaseT::operator=(static_cast<const BaseT &>(RHS));
-    return *this;
-  }
-  BasicTTIImplBase &operator=(BasicTTIImplBase &&RHS) {
-    BaseT::operator=(std::move(static_cast<BaseT &>(RHS)));
-    return *this;
-  }
 
   /// \name Scalar TTI Implementations
   /// @{
@@ -125,35 +119,40 @@ class BasicTTIImplBase : public TargetTransformInfoImplCRTPBase<T> {
   }
 
   bool isLegalAddressingMode(Type *Ty, GlobalValue *BaseGV, int64_t BaseOffset,
-                             bool HasBaseReg, int64_t Scale) {
+                             bool HasBaseReg, int64_t Scale,
+                             unsigned AddrSpace) {
     TargetLoweringBase::AddrMode AM;
     AM.BaseGV = BaseGV;
     AM.BaseOffs = BaseOffset;
     AM.HasBaseReg = HasBaseReg;
     AM.Scale = Scale;
-    return getTLI()->isLegalAddressingMode(AM, Ty);
+    return getTLI()->isLegalAddressingMode(DL, AM, Ty, AddrSpace);
   }
 
   int getScalingFactorCost(Type *Ty, GlobalValue *BaseGV, int64_t BaseOffset,
-                           bool HasBaseReg, int64_t Scale) {
+                           bool HasBaseReg, int64_t Scale, unsigned AddrSpace) {
     TargetLoweringBase::AddrMode AM;
     AM.BaseGV = BaseGV;
     AM.BaseOffs = BaseOffset;
     AM.HasBaseReg = HasBaseReg;
     AM.Scale = Scale;
-    return getTLI()->getScalingFactorCost(AM, Ty);
+    return getTLI()->getScalingFactorCost(DL, AM, Ty, AddrSpace);
   }
 
   bool isTruncateFree(Type *Ty1, Type *Ty2) {
     return getTLI()->isTruncateFree(Ty1, Ty2);
   }
 
+  bool isZExtFree(Type *Ty1, Type *Ty2) const {
+    return getTLI()->isZExtFree(Ty1, Ty2);
+  }
+
   bool isProfitableToHoist(Instruction *I) {
     return getTLI()->isProfitableToHoist(I);
   }
 
   bool isTypeLegal(Type *Ty) {
-    EVT VT = getTLI()->getValueType(Ty);
+    EVT VT = getTLI()->getValueType(DL, Ty);
     return getTLI()->isTypeLegal(VT);
   }
 
@@ -191,7 +190,7 @@ class BasicTTIImplBase : public TargetTransformInfoImplCRTPBase<T> {
 
   bool haveFastSqrt(Type *Ty) {
     const TargetLoweringBase *TLI = getTLI();
-    EVT VT = TLI->getValueType(Ty);
+    EVT VT = TLI->getValueType(DL, Ty);
     return TLI->isTypeLegal(VT) &&
            TLI->isOperationLegalOrCustom(ISD::FSQRT, VT);
   }
@@ -281,11 +280,11 @@ class BasicTTIImplBase : public TargetTransformInfoImplCRTPBase<T> {
   /// \name Vector TTI Implementations
   /// @{
 
-  unsigned getNumberOfRegisters(bool Vector) { return 1; }
+  unsigned getNumberOfRegisters(bool Vector) { return Vector ? 0 : 1; }
 
   unsigned getRegisterBitWidth(bool Vector) { return 32; }
 
-  unsigned getMaxInterleaveFactor() { return 1; }
+  unsigned getMaxInterleaveFactor(unsigned VF) { return 1; }
 
   unsigned getArithmeticInstrCost(
       unsigned Opcode, Type *Ty,
@@ -298,7 +297,7 @@ class BasicTTIImplBase : public TargetTransformInfoImplCRTPBase<T> {
     int ISD = TLI->InstructionOpcodeToISD(Opcode);
     assert(ISD && "Invalid opcode");
 
-    std::pair<unsigned, MVT> LT = TLI->getTypeLegalizationCost(Ty);
+    std::pair<unsigned, MVT> LT = TLI->getTypeLegalizationCost(DL, Ty);
 
     bool IsFloat = Ty->getScalarType()->isFloatingPointTy();
     // Assume that floating point arithmetic operations cost twice as much as
@@ -348,9 +347,8 @@ class BasicTTIImplBase : public TargetTransformInfoImplCRTPBase<T> {
     const TargetLoweringBase *TLI = getTLI();
     int ISD = TLI->InstructionOpcodeToISD(Opcode);
     assert(ISD && "Invalid opcode");
-
-    std::pair<unsigned, MVT> SrcLT = TLI->getTypeLegalizationCost(Src);
-    std::pair<unsigned, MVT> DstLT = TLI->getTypeLegalizationCost(Dst);
+    std::pair<unsigned, MVT> SrcLT = TLI->getTypeLegalizationCost(DL, Src);
+    std::pair<unsigned, MVT> DstLT = TLI->getTypeLegalizationCost(DL, Dst);
 
     // Check for NOOP conversions.
     if (SrcLT.first == DstLT.first &&
@@ -454,8 +452,7 @@ class BasicTTIImplBase : public TargetTransformInfoImplCRTPBase<T> {
       if (CondTy->isVectorTy())
         ISD = ISD::VSELECT;
     }
-
-    std::pair<unsigned, MVT> LT = TLI->getTypeLegalizationCost(ValTy);
+    std::pair<unsigned, MVT> LT = TLI->getTypeLegalizationCost(DL, ValTy);
 
     if (!(ValTy->isVectorTy() && !LT.second.isVector()) &&
         !TLI->isOperationExpand(ISD, LT.second)) {
@@ -484,7 +481,7 @@ class BasicTTIImplBase : public TargetTransformInfoImplCRTPBase<T> {
 
   unsigned getVectorInstrCost(unsigned Opcode, Type *Val, unsigned Index) {
     std::pair<unsigned, MVT> LT =
-        getTLI()->getTypeLegalizationCost(Val->getScalarType());
+        getTLI()->getTypeLegalizationCost(DL, Val->getScalarType());
 
     return LT.first;
   }
@@ -492,7 +489,7 @@ class BasicTTIImplBase : public TargetTransformInfoImplCRTPBase<T> {
   unsigned getMemoryOpCost(unsigned Opcode, Type *Src, unsigned Alignment,
                            unsigned AddressSpace) {
     assert(!Src->isVoidTy() && "Invalid type");
-    std::pair<unsigned, MVT> LT = getTLI()->getTypeLegalizationCost(Src);
+    std::pair<unsigned, MVT> LT = getTLI()->getTypeLegalizationCost(DL, Src);
 
     // Assuming that all loads of legal types cost 1.
     unsigned Cost = LT.first;
@@ -503,7 +500,7 @@ class BasicTTIImplBase : public TargetTransformInfoImplCRTPBase<T> {
       // itself. Unless the corresponding extending load or truncating store is
       // legal, then this will scalarize.
       TargetLowering::LegalizeAction LA = TargetLowering::Expand;
-      EVT MemVT = getTLI()->getValueType(Src, true);
+      EVT MemVT = getTLI()->getValueType(DL, Src, true);
       if (MemVT.isSimple() && MemVT != MVT::Other) {
         if (Opcode == Instruction::Store)
           LA = getTLI()->getTruncStoreAction(LT.second, MemVT.getSimpleVT());
@@ -522,6 +519,73 @@ class BasicTTIImplBase : public TargetTransformInfoImplCRTPBase<T> {
     return Cost;
   }
 
+  unsigned getInterleavedMemoryOpCost(unsigned Opcode, Type *VecTy,
+                                      unsigned Factor,
+                                      ArrayRef<unsigned> Indices,
+                                      unsigned Alignment,
+                                      unsigned AddressSpace) {
+    VectorType *VT = dyn_cast<VectorType>(VecTy);
+    assert(VT && "Expect a vector type for interleaved memory op");
+
+    unsigned NumElts = VT->getNumElements();
+    assert(Factor > 1 && NumElts % Factor == 0 && "Invalid interleave factor");
+
+    unsigned NumSubElts = NumElts / Factor;
+    VectorType *SubVT = VectorType::get(VT->getElementType(), NumSubElts);
+
+    // Firstly, the cost of load/store operation.
+    unsigned Cost = getMemoryOpCost(Opcode, VecTy, Alignment, AddressSpace);
+
+    // Then plus the cost of interleave operation.
+    if (Opcode == Instruction::Load) {
+      // The interleave cost is similar to extract sub vectors' elements
+      // from the wide vector, and insert them into sub vectors.
+      //
+      // E.g. An interleaved load of factor 2 (with one member of index 0):
+      //      %vec = load <8 x i32>, <8 x i32>* %ptr
+      //      %v0 = shuffle %vec, undef, <0, 2, 4, 6>         ; Index 0
+      // The cost is estimated as extract elements at 0, 2, 4, 6 from the
+      // <8 x i32> vector and insert them into a <4 x i32> vector.
+
+      assert(Indices.size() <= Factor &&
+             "Interleaved memory op has too many members");
+      for (unsigned Index : Indices) {
+        assert(Index < Factor && "Invalid index for interleaved memory op");
+
+        // Extract elements from loaded vector for each sub vector.
+        for (unsigned i = 0; i < NumSubElts; i++)
+          Cost += getVectorInstrCost(Instruction::ExtractElement, VT,
+                                     Index + i * Factor);
+      }
+
+      unsigned InsSubCost = 0;
+      for (unsigned i = 0; i < NumSubElts; i++)
+        InsSubCost += getVectorInstrCost(Instruction::InsertElement, SubVT, i);
+
+      Cost += Indices.size() * InsSubCost;
+    } else {
+      // The interleave cost is extract all elements from sub vectors, and
+      // insert them into the wide vector.
+      //
+      // E.g. An interleaved store of factor 2:
+      //      %v0_v1 = shuffle %v0, %v1, <0, 4, 1, 5, 2, 6, 3, 7>
+      //      store <8 x i32> %interleaved.vec, <8 x i32>* %ptr
+      // The cost is estimated as extract all elements from both <4 x i32>
+      // vectors and insert into the <8 x i32> vector.
+
+      unsigned ExtSubCost = 0;
+      for (unsigned i = 0; i < NumSubElts; i++)
+        ExtSubCost += getVectorInstrCost(Instruction::ExtractElement, SubVT, i);
+
+      Cost += Factor * ExtSubCost;
+
+      for (unsigned i = 0; i < NumElts; i++)
+        Cost += getVectorInstrCost(Instruction::InsertElement, VT, i);
+    }
+
+    return Cost;
+  }
+
   unsigned getIntrinsicInstrCost(Intrinsic::ID IID, Type *RetTy,
                                  ArrayRef<Type *> Tys) {
     unsigned ISD = 0;
@@ -632,7 +696,7 @@ class BasicTTIImplBase : public TargetTransformInfoImplCRTPBase<T> {
     }
 
     const TargetLoweringBase *TLI = getTLI();
-    std::pair<unsigned, MVT> LT = TLI->getTypeLegalizationCost(RetTy);
+    std::pair<unsigned, MVT> LT = TLI->getTypeLegalizationCost(DL, RetTy);
 
     if (TLI->isOperationLegalOrPromote(ISD, LT.second)) {
       // The operation is legal. Assume it costs 1.
@@ -703,7 +767,7 @@ class BasicTTIImplBase : public TargetTransformInfoImplCRTPBase<T> {
   }
 
   unsigned getNumberOfParts(Type *Tp) {
-    std::pair<unsigned, MVT> LT = getTLI()->getTypeLegalizationCost(Tp);
+    std::pair<unsigned, MVT> LT = getTLI()->getTypeLegalizationCost(DL, Tp);
     return LT.first;
   }
 
@@ -748,18 +812,6 @@ class BasicTTIImpl : public BasicTTIImplBase<BasicTTIImpl> {
   BasicTTIImpl(BasicTTIImpl &&Arg)
       : BaseT(std::move(static_cast<BaseT &>(Arg))), ST(std::move(Arg.ST)),
         TLI(std::move(Arg.TLI)) {}
-  BasicTTIImpl &operator=(const BasicTTIImpl &RHS) {
-    BaseT::operator=(static_cast<const BaseT &>(RHS));
-    ST = RHS.ST;
-    TLI = RHS.TLI;
-    return *this;
-  }
-  BasicTTIImpl &operator=(BasicTTIImpl &&RHS) {
-    BaseT::operator=(std::move(static_cast<BaseT &>(RHS)));
-    ST = std::move(RHS.ST);
-    TLI = std::move(RHS.TLI);
-    return *this;
-  }
 };
 
 }
diff --git a/include/llvm/CodeGen/CallingConvLower.h b/include/llvm/CodeGen/CallingConvLower.h
index e2b744f08eb..1fd4eeb46b3 100644
--- a/include/llvm/CodeGen/CallingConvLower.h
+++ b/include/llvm/CodeGen/CallingConvLower.h
@@ -19,6 +19,7 @@
 #include "llvm/CodeGen/MachineFrameInfo.h"
 #include "llvm/CodeGen/MachineFunction.h"
 #include "llvm/IR/CallingConv.h"
+#include "llvm/MC/MCRegisterInfo.h"
 #include "llvm/Target/TargetCallingConv.h"
 
 namespace llvm {
diff --git a/include/llvm/CodeGen/CommandFlags.h b/include/llvm/CodeGen/CommandFlags.h
index 9f86429cbf1..8ac7555dfb0 100644
--- a/include/llvm/CodeGen/CommandFlags.h
+++ b/include/llvm/CodeGen/CommandFlags.h
@@ -16,11 +16,18 @@
 #ifndef LLVM_CODEGEN_COMMANDFLAGS_H
 #define LLVM_CODEGEN_COMMANDFLAGS_H
 
+#include "llvm/ADT/StringExtras.h"
+#include "llvm/IR/Instructions.h"
+#include "llvm/IR/Intrinsics.h"
+#include "llvm/IR/Module.h"
 #include "llvm/MC/MCTargetOptionsCommandFlags.h"
+#include "llvm//MC/SubtargetFeature.h"
 #include "llvm/Support/CodeGen.h"
 #include "llvm/Support/CommandLine.h"
+#include "llvm/Support/Host.h"
 #include "llvm/Target/TargetMachine.h"
 #include "llvm/Target/TargetOptions.h"
+#include "llvm/Target/TargetRecip.h"
 #include <string>
 using namespace llvm;
 
@@ -123,11 +130,6 @@ EnableHonorSignDependentRoundingFPMath("enable-sign-dependent-rounding-fp-math",
       cl::desc("Force codegen to assume rounding mode can change dynamically"),
       cl::init(false));
 
-cl::opt<bool>
-GenerateSoftFloatCalls("soft-float",
-                    cl::desc("Generate software floating point library calls"),
-                    cl::init(false));
-
 cl::opt<llvm::FloatABI::ABIType>
 FloatABIForCalls("float-abi",
                  cl::desc("Choose float ABI type"),
@@ -151,9 +153,15 @@ FuseFPOps("fp-contract",
               clEnumValN(FPOpFusion::Standard, "on",
                          "Only fuse 'blessed' FP ops."),
               clEnumValN(FPOpFusion::Strict, "off",
-                         "Only fuse FP ops when the result won't be effected."),
+                         "Only fuse FP ops when the result won't be affected."),
               clEnumValEnd));
 
+cl::list<std::string>
+ReciprocalOps("recip",
+  cl::CommaSeparated,
+  cl::desc("Choose reciprocal operation types and parameters."),
+  cl::value_desc("all,none,default,divf,!vec-sqrtd,vec-divd:0,sqrt:9..."));
+
 cl::opt<bool>
 DontPlaceZerosInBSS("nozero-initialized-in-bss",
               cl::desc("Don't place zero-initialized symbols into bss section"),
@@ -198,6 +206,10 @@ cl::opt<std::string> StartAfter("start-after",
                           cl::value_desc("pass-name"),
                           cl::init(""));
 
+cl::opt<std::string>
+    RunPass("run-pass", cl::desc("Run compiler only for one specific pass"),
+            cl::value_desc("pass-name"), cl::init(""));
+
 cl::opt<bool> DataSections("data-sections",
                            cl::desc("Emit data into separate sections"),
                            cl::init(false));
@@ -207,6 +219,10 @@ FunctionSections("function-sections",
                  cl::desc("Emit functions into separate sections"),
                  cl::init(false));
 
+cl::opt<bool> EmulatedTLS("emulated-tls",
+                          cl::desc("Use emulated TLS model"),
+                          cl::init(false));
+
 cl::opt<bool> UniqueSectionNames("unique-section-names",
                                  cl::desc("Give unique names to every section"),
                                  cl::init(true));
@@ -226,80 +242,108 @@ JTableType("jump-table-type",
                          "Create one table per unique function type."),
               clEnumValEnd));
 
-cl::opt<bool>
-FCFI("fcfi",
-     cl::desc("Apply forward-edge control-flow integrity"),
-     cl::init(false));
-
-cl::opt<llvm::CFIntegrity>
-CFIType("cfi-type",
-        cl::desc("Choose the type of Control-Flow Integrity check to add"),
-        cl::init(CFIntegrity::Sub),
-        cl::values(
-            clEnumValN(CFIntegrity::Sub, "sub",
-                       "Subtract the pointer from the table base, then mask."),
-            clEnumValN(CFIntegrity::Ror, "ror",
-                       "Use rotate to check the offset from a table base."),
-            clEnumValN(CFIntegrity::Add, "add",
-                       "Mask out the high bits and add to an aligned base."),
-            clEnumValEnd));
-
-cl::opt<bool>
-CFIEnforcing("cfi-enforcing",
-             cl::desc("Enforce CFI or pass the violation to a function."),
-             cl::init(false));
-
-// Note that this option is linked to the cfi-enforcing option above: if
-// cfi-enforcing is set, then the cfi-func-name option is entirely ignored. If
-// cfi-enforcing is false and no cfi-func-name is set, then a default function
-// will be generated that ignores all CFI violations. The expected signature for
-// functions called with CFI violations is
-//
-// void (i8*, i8*)
-//
-// The first pointer is a C string containing the name of the function in which
-// the violation occurs, and the second pointer is the pointer that violated
-// CFI.
-cl::opt<std::string>
-CFIFuncName("cfi-func-name", cl::desc("The name of the CFI function to call"),
-            cl::init(""));
-
 // Common utility function tightly tied to the options listed here. Initializes
 // a TargetOptions object with CodeGen flags and returns it.
 static inline TargetOptions InitTargetOptionsFromCodeGenFlags() {
   TargetOptions Options;
   Options.LessPreciseFPMADOption = EnableFPMAD;
-  Options.NoFramePointerElim = DisableFPElim;
   Options.AllowFPOpFusion = FuseFPOps;
+  Options.Reciprocals = TargetRecip(ReciprocalOps);
   Options.UnsafeFPMath = EnableUnsafeFPMath;
   Options.NoInfsFPMath = EnableNoInfsFPMath;
   Options.NoNaNsFPMath = EnableNoNaNsFPMath;
   Options.HonorSignDependentRoundingFPMathOption =
       EnableHonorSignDependentRoundingFPMath;
-  Options.UseSoftFloat = GenerateSoftFloatCalls;
   if (FloatABIForCalls != FloatABI::Default)
     Options.FloatABIType = FloatABIForCalls;
   Options.NoZerosInBSS = DontPlaceZerosInBSS;
   Options.GuaranteedTailCallOpt = EnableGuaranteedTailCallOpt;
-  Options.DisableTailCalls = DisableTailCalls;
   Options.StackAlignmentOverride = OverrideStackAlignment;
-  Options.TrapFuncName = TrapFuncName;
   Options.PositionIndependentExecutable = EnablePIE;
   Options.UseInitArray = !UseCtors;
   Options.DataSections = DataSections;
   Options.FunctionSections = FunctionSections;
   Options.UniqueSectionNames = UniqueSectionNames;
+  Options.EmulatedTLS = EmulatedTLS;
 
   Options.MCOptions = InitMCTargetOptionsFromFlags();
   Options.JTType = JTableType;
-  Options.FCFI = FCFI;
-  Options.CFIType = CFIType;
-  Options.CFIEnforcing = CFIEnforcing;
-  Options.CFIFuncName = CFIFuncName;
 
   Options.ThreadModel = TMModel;
 
   return Options;
 }
 
+static inline std::string getCPUStr() {
+  // If user asked for the 'native' CPU, autodetect here. If autodection fails,
+  // this will set the CPU to an empty string which tells the target to
+  // pick a basic default.
+  if (MCPU == "native")
+    return sys::getHostCPUName();
+
+  return MCPU;
+}
+
+static inline std::string getFeaturesStr() {
+  SubtargetFeatures Features;
+
+  // If user asked for the 'native' CPU, we need to autodetect features.
+  // This is necessary for x86 where the CPU might not support all the
+  // features the autodetected CPU name lists in the target. For example,
+  // not all Sandybridge processors support AVX.
+  if (MCPU == "native") {
+    StringMap<bool> HostFeatures;
+    if (sys::getHostCPUFeatures(HostFeatures))
+      for (auto &F : HostFeatures)
+        Features.AddFeature(F.first(), F.second);
+  }
+
+  for (unsigned i = 0; i != MAttrs.size(); ++i)
+    Features.AddFeature(MAttrs[i]);
+
+  return Features.getString();
+}
+
+/// \brief Set function attributes of functions in Module M based on CPU,
+/// Features, and command line flags.
+static inline void setFunctionAttributes(StringRef CPU, StringRef Features,
+                                         Module &M) {
+  for (auto &F : M) {
+    auto &Ctx = F.getContext();
+    AttributeSet Attrs = F.getAttributes(), NewAttrs;
+
+    if (!CPU.empty())
+      NewAttrs = NewAttrs.addAttribute(Ctx, AttributeSet::FunctionIndex,
+                                       "target-cpu", CPU);
+
+    if (!Features.empty())
+      NewAttrs = NewAttrs.addAttribute(Ctx, AttributeSet::FunctionIndex,
+                                       "target-features", Features);
+
+    if (DisableFPElim.getNumOccurrences() > 0)
+      NewAttrs = NewAttrs.addAttribute(Ctx, AttributeSet::FunctionIndex,
+                                       "no-frame-pointer-elim",
+                                       DisableFPElim ? "true" : "false");
+
+    if (DisableTailCalls.getNumOccurrences() > 0)
+      NewAttrs = NewAttrs.addAttribute(Ctx, AttributeSet::FunctionIndex,
+                                       "disable-tail-calls",
+                                       toStringRef(DisableTailCalls));
+
+    if (TrapFuncName.getNumOccurrences() > 0)
+      for (auto &B : F)
+        for (auto &I : B)
+          if (auto *Call = dyn_cast<CallInst>(&I))
+            if (const auto *F = Call->getCalledFunction())
+              if (F->getIntrinsicID() == Intrinsic::debugtrap ||
+                  F->getIntrinsicID() == Intrinsic::trap)
+                Call->addAttribute(llvm::AttributeSet::FunctionIndex,
+                                   "trap-func-name", TrapFuncName);
+
+    // Let NewAttrs override Attrs.
+    NewAttrs = Attrs.addAttributes(Ctx, AttributeSet::FunctionIndex, NewAttrs);
+    F.setAttributes(NewAttrs);
+  }
+}
+
 #endif
diff --git a/include/llvm/CodeGen/DFAPacketizer.h b/include/llvm/CodeGen/DFAPacketizer.h
index f9cdc2a469f..c44a7e0b673 100644
--- a/include/llvm/CodeGen/DFAPacketizer.h
+++ b/include/llvm/CodeGen/DFAPacketizer.h
@@ -91,7 +91,7 @@ class DFAPacketizer {
 // API call is made to prune the dependence.
 class VLIWPacketizerList {
 protected:
-  const MachineFunction &MF;
+  MachineFunction &MF;
   const TargetInstrInfo *TII;
 
   // The VLIW Scheduler.
diff --git a/include/llvm/CodeGen/DIE.h b/include/llvm/CodeGen/DIE.h
index 7803898a8d4..f07712a676d 100644
--- a/include/llvm/CodeGen/DIE.h
+++ b/include/llvm/CodeGen/DIE.h
@@ -15,7 +15,10 @@
 #define LLVM_LIB_CODEGEN_ASMPRINTER_DIE_H
 
 #include "llvm/ADT/FoldingSet.h"
+#include "llvm/ADT/PointerIntPair.h"
+#include "llvm/ADT/STLExtras.h"
 #include "llvm/ADT/SmallVector.h"
+#include "llvm/CodeGen/DwarfStringPoolEntry.h"
 #include "llvm/Support/Dwarf.h"
 #include <vector>
 
@@ -103,153 +106,14 @@ class DIEAbbrev : public FoldingSetNode {
 #endif
 };
 
-//===--------------------------------------------------------------------===//
-/// DIE - A structured debug information entry.  Has an abbreviation which
-/// describes its organization.
-class DIEValue;
-
-class DIE {
-protected:
-  /// Offset - Offset in debug info section.
-  ///
-  unsigned Offset;
-
-  /// Size - Size of instance + children.
-  ///
-  unsigned Size;
-
-  /// Abbrev - Buffer for constructing abbreviation.
-  ///
-  DIEAbbrev Abbrev;
-
-  /// Children DIEs.
-  ///
-  // This can't be a vector<DIE> because pointer validity is requirent for the
-  // Parent pointer and DIEEntry.
-  // It can't be a list<DIE> because some clients need pointer validity before
-  // the object has been added to any child list
-  // (eg: DwarfUnit::constructVariableDIE). These aren't insurmountable, but may
-  // be more convoluted than beneficial.
-  std::vector<std::unique_ptr<DIE>> Children;
-
-  DIE *Parent;
-
-  /// Attribute values.
-  ///
-  SmallVector<DIEValue *, 12> Values;
-
-protected:
-  DIE()
-      : Offset(0), Size(0), Abbrev((dwarf::Tag)0, dwarf::DW_CHILDREN_no),
-        Parent(nullptr) {}
-
-public:
-  explicit DIE(dwarf::Tag Tag)
-      : Offset(0), Size(0), Abbrev((dwarf::Tag)Tag, dwarf::DW_CHILDREN_no),
-        Parent(nullptr) {}
-
-  // Accessors.
-  DIEAbbrev &getAbbrev() { return Abbrev; }
-  const DIEAbbrev &getAbbrev() const { return Abbrev; }
-  unsigned getAbbrevNumber() const { return Abbrev.getNumber(); }
-  dwarf::Tag getTag() const { return Abbrev.getTag(); }
-  unsigned getOffset() const { return Offset; }
-  unsigned getSize() const { return Size; }
-  const std::vector<std::unique_ptr<DIE>> &getChildren() const {
-    return Children;
-  }
-  const SmallVectorImpl<DIEValue *> &getValues() const { return Values; }
-  DIE *getParent() const { return Parent; }
-  /// Climb up the parent chain to get the compile or type unit DIE this DIE
-  /// belongs to.
-  const DIE *getUnit() const;
-  /// Similar to getUnit, returns null when DIE is not added to an
-  /// owner yet.
-  const DIE *getUnitOrNull() const;
-  void setOffset(unsigned O) { Offset = O; }
-  void setSize(unsigned S) { Size = S; }
-
-  /// addValue - Add a value and attributes to a DIE.
-  ///
-  void addValue(dwarf::Attribute Attribute, dwarf::Form Form, DIEValue *Value) {
-    Abbrev.AddAttribute(Attribute, Form);
-    Values.push_back(Value);
-  }
-
-  /// addChild - Add a child to the DIE.
-  ///
-  void addChild(std::unique_ptr<DIE> Child) {
-    assert(!Child->getParent());
-    Abbrev.setChildrenFlag(dwarf::DW_CHILDREN_yes);
-    Child->Parent = this;
-    Children.push_back(std::move(Child));
-  }
-
-  /// findAttribute - Find a value in the DIE with the attribute given,
-  /// returns NULL if no such attribute exists.
-  DIEValue *findAttribute(dwarf::Attribute Attribute) const;
-
-#ifndef NDEBUG
-  void print(raw_ostream &O, unsigned IndentCount = 0) const;
-  void dump();
-#endif
-};
-
-//===--------------------------------------------------------------------===//
-/// DIEValue - A debug information entry value. Some of these roughly correlate
-/// to DWARF attribute classes.
-///
-class DIEValue {
-  virtual void anchor();
-
-public:
-  enum Type {
-    isInteger,
-    isString,
-    isExpr,
-    isLabel,
-    isDelta,
-    isEntry,
-    isTypeSignature,
-    isBlock,
-    isLoc,
-    isLocList,
-  };
-
-protected:
-  /// Ty - Type of data stored in the value.
-  ///
-  Type Ty;
-
-  explicit DIEValue(Type T) : Ty(T) {}
-  virtual ~DIEValue() {}
-
-public:
-  // Accessors
-  Type getType() const { return Ty; }
-
-  /// EmitValue - Emit value via the Dwarf writer.
-  ///
-  virtual void EmitValue(const AsmPrinter *AP, dwarf::Form Form) const = 0;
-
-  /// SizeOf - Return the size of a value in bytes.
-  ///
-  virtual unsigned SizeOf(const AsmPrinter *AP, dwarf::Form Form) const = 0;
-
-#ifndef NDEBUG
-  virtual void print(raw_ostream &O) const = 0;
-  void dump() const;
-#endif
-};
-
 //===--------------------------------------------------------------------===//
 /// DIEInteger - An integer value DIE.
 ///
-class DIEInteger : public DIEValue {
+class DIEInteger {
   uint64_t Integer;
 
 public:
-  explicit DIEInteger(uint64_t I) : DIEValue(isInteger), Integer(I) {}
+  explicit DIEInteger(uint64_t I) : Integer(I) {}
 
   /// BestForm - Choose the best form for integer.
   ///
@@ -273,137 +137,94 @@ class DIEInteger : public DIEValue {
     return dwarf::DW_FORM_data8;
   }
 
-  /// EmitValue - Emit integer of appropriate size.
-  ///
-  void EmitValue(const AsmPrinter *AP, dwarf::Form Form) const override;
-
   uint64_t getValue() const { return Integer; }
   void setValue(uint64_t Val) { Integer = Val; }
 
-  /// SizeOf - Determine size of integer value in bytes.
-  ///
-  unsigned SizeOf(const AsmPrinter *AP, dwarf::Form Form) const override;
-
-  // Implement isa/cast/dyncast.
-  static bool classof(const DIEValue *I) { return I->getType() == isInteger; }
+  void EmitValue(const AsmPrinter *AP, dwarf::Form Form) const;
+  unsigned SizeOf(const AsmPrinter *AP, dwarf::Form Form) const;
 
 #ifndef NDEBUG
-  void print(raw_ostream &O) const override;
+  void print(raw_ostream &O) const;
 #endif
 };
 
 //===--------------------------------------------------------------------===//
 /// DIEExpr - An expression DIE.
 //
-class DIEExpr : public DIEValue {
+class DIEExpr {
   const MCExpr *Expr;
 
 public:
-  explicit DIEExpr(const MCExpr *E) : DIEValue(isExpr), Expr(E) {}
-
-  /// EmitValue - Emit expression value.
-  ///
-  void EmitValue(const AsmPrinter *AP, dwarf::Form Form) const override;
+  explicit DIEExpr(const MCExpr *E) : Expr(E) {}
 
   /// getValue - Get MCExpr.
   ///
   const MCExpr *getValue() const { return Expr; }
 
-  /// SizeOf - Determine size of expression value in bytes.
-  ///
-  unsigned SizeOf(const AsmPrinter *AP, dwarf::Form Form) const override;
-
-  // Implement isa/cast/dyncast.
-  static bool classof(const DIEValue *E) { return E->getType() == isExpr; }
+  void EmitValue(const AsmPrinter *AP, dwarf::Form Form) const;
+  unsigned SizeOf(const AsmPrinter *AP, dwarf::Form Form) const;
 
 #ifndef NDEBUG
-  void print(raw_ostream &O) const override;
+  void print(raw_ostream &O) const;
 #endif
 };
 
 //===--------------------------------------------------------------------===//
 /// DIELabel - A label DIE.
 //
-class DIELabel : public DIEValue {
+class DIELabel {
   const MCSymbol *Label;
 
 public:
-  explicit DIELabel(const MCSymbol *L) : DIEValue(isLabel), Label(L) {}
-
-  /// EmitValue - Emit label value.
-  ///
-  void EmitValue(const AsmPrinter *AP, dwarf::Form Form) const override;
+  explicit DIELabel(const MCSymbol *L) : Label(L) {}
 
   /// getValue - Get MCSymbol.
   ///
   const MCSymbol *getValue() const { return Label; }
 
-  /// SizeOf - Determine size of label value in bytes.
-  ///
-  unsigned SizeOf(const AsmPrinter *AP, dwarf::Form Form) const override;
-
-  // Implement isa/cast/dyncast.
-  static bool classof(const DIEValue *L) { return L->getType() == isLabel; }
+  void EmitValue(const AsmPrinter *AP, dwarf::Form Form) const;
+  unsigned SizeOf(const AsmPrinter *AP, dwarf::Form Form) const;
 
 #ifndef NDEBUG
-  void print(raw_ostream &O) const override;
+  void print(raw_ostream &O) const;
 #endif
 };
 
 //===--------------------------------------------------------------------===//
 /// DIEDelta - A simple label difference DIE.
 ///
-class DIEDelta : public DIEValue {
+class DIEDelta {
   const MCSymbol *LabelHi;
   const MCSymbol *LabelLo;
 
 public:
-  DIEDelta(const MCSymbol *Hi, const MCSymbol *Lo)
-      : DIEValue(isDelta), LabelHi(Hi), LabelLo(Lo) {}
-
-  /// EmitValue - Emit delta value.
-  ///
-  void EmitValue(const AsmPrinter *AP, dwarf::Form Form) const override;
-
-  /// SizeOf - Determine size of delta value in bytes.
-  ///
-  unsigned SizeOf(const AsmPrinter *AP, dwarf::Form Form) const override;
+  DIEDelta(const MCSymbol *Hi, const MCSymbol *Lo) : LabelHi(Hi), LabelLo(Lo) {}
 
-  // Implement isa/cast/dyncast.
-  static bool classof(const DIEValue *D) { return D->getType() == isDelta; }
+  void EmitValue(const AsmPrinter *AP, dwarf::Form Form) const;
+  unsigned SizeOf(const AsmPrinter *AP, dwarf::Form Form) const;
 
 #ifndef NDEBUG
-  void print(raw_ostream &O) const override;
+  void print(raw_ostream &O) const;
 #endif
 };
 
 //===--------------------------------------------------------------------===//
 /// DIEString - A container for string values.
 ///
-class DIEString : public DIEValue {
-  const DIEValue *Access;
-  StringRef Str;
+class DIEString {
+  DwarfStringPoolEntryRef S;
 
 public:
-  DIEString(const DIEValue *Acc, StringRef S)
-      : DIEValue(isString), Access(Acc), Str(S) {}
+  DIEString(DwarfStringPoolEntryRef S) : S(S) {}
 
   /// getString - Grab the string out of the object.
-  StringRef getString() const { return Str; }
+  StringRef getString() const { return S.getString(); }
 
-  /// EmitValue - Emit delta value.
-  ///
-  void EmitValue(const AsmPrinter *AP, dwarf::Form Form) const override;
-
-  /// SizeOf - Determine size of delta value in bytes.
-  ///
-  unsigned SizeOf(const AsmPrinter *AP, dwarf::Form Form) const override;
-
-  // Implement isa/cast/dyncast.
-  static bool classof(const DIEValue *D) { return D->getType() == isString; }
+  void EmitValue(const AsmPrinter *AP, dwarf::Form Form) const;
+  unsigned SizeOf(const AsmPrinter *AP, dwarf::Form Form) const;
 
 #ifndef NDEBUG
-  void print(raw_ostream &O) const override;
+  void print(raw_ostream &O) const;
 #endif
 };
 
@@ -411,72 +232,524 @@ class DIEString : public DIEValue {
 /// DIEEntry - A pointer to another debug information entry.  An instance of
 /// this class can also be used as a proxy for a debug information entry not
 /// yet defined (ie. types.)
-class DIEEntry : public DIEValue {
-  DIE &Entry;
+class DIE;
+class DIEEntry {
+  DIE *Entry;
+
+  DIEEntry() = delete;
 
 public:
-  explicit DIEEntry(DIE &E) : DIEValue(isEntry), Entry(E) {
-  }
+  explicit DIEEntry(DIE &E) : Entry(&E) {}
 
-  DIE &getEntry() const { return Entry; }
+  DIE &getEntry() const { return *Entry; }
 
-  /// EmitValue - Emit debug information entry offset.
-  ///
-  void EmitValue(const AsmPrinter *AP, dwarf::Form Form) const override;
+  /// Returns size of a ref_addr entry.
+  static unsigned getRefAddrSize(const AsmPrinter *AP);
 
-  /// SizeOf - Determine size of debug information entry in bytes.
-  ///
-   unsigned SizeOf(const AsmPrinter *AP, dwarf::Form Form) const override {
+  void EmitValue(const AsmPrinter *AP, dwarf::Form Form) const;
+  unsigned SizeOf(const AsmPrinter *AP, dwarf::Form Form) const {
     return Form == dwarf::DW_FORM_ref_addr ? getRefAddrSize(AP)
                                            : sizeof(int32_t);
   }
 
-  /// Returns size of a ref_addr entry.
-  static unsigned getRefAddrSize(const AsmPrinter *AP);
-
-  // Implement isa/cast/dyncast.
-  static bool classof(const DIEValue *E) { return E->getType() == isEntry; }
-
 #ifndef NDEBUG
-  void print(raw_ostream &O) const override;
+  void print(raw_ostream &O) const;
 #endif
 };
 
 //===--------------------------------------------------------------------===//
 /// \brief A signature reference to a type unit.
-class DIETypeSignature : public DIEValue {
-  const DwarfTypeUnit &Unit;
+class DIETypeSignature {
+  const DwarfTypeUnit *Unit;
 
-public:
-  explicit DIETypeSignature(const DwarfTypeUnit &Unit)
-      : DIEValue(isTypeSignature), Unit(Unit) {}
+  DIETypeSignature() = delete;
 
-  /// \brief Emit type unit signature.
-  void EmitValue(const AsmPrinter *Asm, dwarf::Form Form) const override;
+public:
+  explicit DIETypeSignature(const DwarfTypeUnit &Unit) : Unit(&Unit) {}
 
-  /// Returns size of a ref_sig8 entry.
-  unsigned SizeOf(const AsmPrinter *AP, dwarf::Form Form) const override {
+  void EmitValue(const AsmPrinter *AP, dwarf::Form Form) const;
+  unsigned SizeOf(const AsmPrinter *AP, dwarf::Form Form) const {
     assert(Form == dwarf::DW_FORM_ref_sig8);
     return 8;
   }
 
-  // \brief Implement isa/cast/dyncast.
-  static bool classof(const DIEValue *E) {
-    return E->getType() == isTypeSignature;
+#ifndef NDEBUG
+  void print(raw_ostream &O) const;
+#endif
+};
+
+//===--------------------------------------------------------------------===//
+/// DIELocList - Represents a pointer to a location list in the debug_loc
+/// section.
+//
+class DIELocList {
+  // Index into the .debug_loc vector.
+  size_t Index;
+
+public:
+  DIELocList(size_t I) : Index(I) {}
+
+  /// getValue - Grab the current index out.
+  size_t getValue() const { return Index; }
+
+  void EmitValue(const AsmPrinter *AP, dwarf::Form Form) const;
+  unsigned SizeOf(const AsmPrinter *AP, dwarf::Form Form) const;
+
+#ifndef NDEBUG
+  void print(raw_ostream &O) const;
+#endif
+};
+
+//===--------------------------------------------------------------------===//
+/// DIEValue - A debug information entry value. Some of these roughly correlate
+/// to DWARF attribute classes.
+///
+class DIEBlock;
+class DIELoc;
+class DIEValue {
+public:
+  enum Type {
+    isNone,
+#define HANDLE_DIEVALUE(T) is##T,
+#include "llvm/CodeGen/DIEValue.def"
+  };
+
+private:
+  /// Ty - Type of data stored in the value.
+  ///
+  Type Ty = isNone;
+  dwarf::Attribute Attribute = (dwarf::Attribute)0;
+  dwarf::Form Form = (dwarf::Form)0;
+
+  /// Storage for the value.
+  ///
+  /// All values that aren't standard layout (or are larger than 8 bytes)
+  /// should be stored by reference instead of by value.
+  typedef AlignedCharArrayUnion<DIEInteger, DIEString, DIEExpr, DIELabel,
+                                DIEDelta *, DIEEntry, DIETypeSignature,
+                                DIEBlock *, DIELoc *, DIELocList> ValTy;
+  static_assert(sizeof(ValTy) <= sizeof(uint64_t) ||
+                    sizeof(ValTy) <= sizeof(void *),
+                "Expected all large types to be stored via pointer");
+
+  /// Underlying stored value.
+  ValTy Val;
+
+  template <class T> void construct(T V) {
+    static_assert(std::is_standard_layout<T>::value ||
+                      std::is_pointer<T>::value,
+                  "Expected standard layout or pointer");
+    new (reinterpret_cast<void *>(Val.buffer)) T(V);
+  }
+
+  template <class T> T *get() { return reinterpret_cast<T *>(Val.buffer); }
+  template <class T> const T *get() const {
+    return reinterpret_cast<const T *>(Val.buffer);
+  }
+  template <class T> void destruct() { get<T>()->~T(); }
+
+  /// Destroy the underlying value.
+  ///
+  /// This should get optimized down to a no-op.  We could skip it if we could
+  /// add a static assert on \a std::is_trivially_copyable(), but we currently
+  /// support versions of GCC that don't understand that.
+  void destroyVal() {
+    switch (Ty) {
+    case isNone:
+      return;
+#define HANDLE_DIEVALUE_SMALL(T)                                               \
+  case is##T:                                                                  \
+    destruct<DIE##T>();
+    return;
+#define HANDLE_DIEVALUE_LARGE(T)                                               \
+  case is##T:                                                                  \
+    destruct<const DIE##T *>();
+    return;
+#include "llvm/CodeGen/DIEValue.def"
+    }
+  }
+
+  /// Copy the underlying value.
+  ///
+  /// This should get optimized down to a simple copy.  We need to actually
+  /// construct the value, rather than calling memcpy, to satisfy strict
+  /// aliasing rules.
+  void copyVal(const DIEValue &X) {
+    switch (Ty) {
+    case isNone:
+      return;
+#define HANDLE_DIEVALUE_SMALL(T)                                               \
+  case is##T:                                                                  \
+    construct<DIE##T>(*X.get<DIE##T>());                                       \
+    return;
+#define HANDLE_DIEVALUE_LARGE(T)                                               \
+  case is##T:                                                                  \
+    construct<const DIE##T *>(*X.get<const DIE##T *>());                       \
+    return;
+#include "llvm/CodeGen/DIEValue.def"
+    }
+  }
+
+public:
+  DIEValue() = default;
+  DIEValue(const DIEValue &X) : Ty(X.Ty), Attribute(X.Attribute), Form(X.Form) {
+    copyVal(X);
+  }
+  DIEValue &operator=(const DIEValue &X) {
+    destroyVal();
+    Ty = X.Ty;
+    Attribute = X.Attribute;
+    Form = X.Form;
+    copyVal(X);
+    return *this;
+  }
+  ~DIEValue() { destroyVal(); }
+
+#define HANDLE_DIEVALUE_SMALL(T)                                               \
+  DIEValue(dwarf::Attribute Attribute, dwarf::Form Form, const DIE##T &V)      \
+      : Ty(is##T), Attribute(Attribute), Form(Form) {                          \
+    construct<DIE##T>(V);                                                      \
   }
+#define HANDLE_DIEVALUE_LARGE(T)                                               \
+  DIEValue(dwarf::Attribute Attribute, dwarf::Form Form, const DIE##T *V)      \
+      : Ty(is##T), Attribute(Attribute), Form(Form) {                          \
+    assert(V && "Expected valid value");                                       \
+    construct<const DIE##T *>(V);                                              \
+  }
+#include "llvm/CodeGen/DIEValue.def"
+
+  // Accessors
+  Type getType() const { return Ty; }
+  dwarf::Attribute getAttribute() const { return Attribute; }
+  dwarf::Form getForm() const { return Form; }
+  explicit operator bool() const { return Ty; }
+
+#define HANDLE_DIEVALUE_SMALL(T)                                               \
+  const DIE##T &getDIE##T() const {                                            \
+    assert(getType() == is##T && "Expected " #T);                              \
+    return *get<DIE##T>();                                                     \
+  }
+#define HANDLE_DIEVALUE_LARGE(T)                                               \
+  const DIE##T &getDIE##T() const {                                            \
+    assert(getType() == is##T && "Expected " #T);                              \
+    return **get<const DIE##T *>();                                            \
+  }
+#include "llvm/CodeGen/DIEValue.def"
+
+  /// EmitValue - Emit value via the Dwarf writer.
+  ///
+  void EmitValue(const AsmPrinter *AP) const;
+
+  /// SizeOf - Return the size of a value in bytes.
+  ///
+  unsigned SizeOf(const AsmPrinter *AP) const;
+
 #ifndef NDEBUG
-  void print(raw_ostream &O) const override;
+  void print(raw_ostream &O) const;
   void dump() const;
 #endif
 };
 
+struct IntrusiveBackListNode {
+  PointerIntPair<IntrusiveBackListNode *, 1> Next;
+  IntrusiveBackListNode() : Next(this, true) {}
+
+  IntrusiveBackListNode *getNext() const {
+    return Next.getInt() ? nullptr : Next.getPointer();
+  }
+};
+
+struct IntrusiveBackListBase {
+  typedef IntrusiveBackListNode Node;
+  Node *Last = nullptr;
+
+  bool empty() const { return !Last; }
+  void push_back(Node &N) {
+    assert(N.Next.getPointer() == &N && "Expected unlinked node");
+    assert(N.Next.getInt() == true && "Expected unlinked node");
+
+    if (Last) {
+      N.Next = Last->Next;
+      Last->Next.setPointerAndInt(&N, false);
+    }
+    Last = &N;
+  }
+};
+
+template <class T> class IntrusiveBackList : IntrusiveBackListBase {
+public:
+  using IntrusiveBackListBase::empty;
+  void push_back(T &N) { IntrusiveBackListBase::push_back(N); }
+  T &back() { return *static_cast<T *>(Last); }
+  const T &back() const { return *static_cast<T *>(Last); }
+
+  class const_iterator;
+  class iterator
+      : public iterator_facade_base<iterator, std::forward_iterator_tag, T> {
+    friend class const_iterator;
+    Node *N = nullptr;
+
+  public:
+    iterator() = default;
+    explicit iterator(T *N) : N(N) {}
+
+    iterator &operator++() {
+      N = N->getNext();
+      return *this;
+    }
+
+    explicit operator bool() const { return N; }
+    T &operator*() const { return *static_cast<T *>(N); }
+
+    bool operator==(const iterator &X) const { return N == X.N; }
+    bool operator!=(const iterator &X) const { return N != X.N; }
+  };
+
+  class const_iterator
+      : public iterator_facade_base<const_iterator, std::forward_iterator_tag,
+                                    const T> {
+    const Node *N = nullptr;
+
+  public:
+    const_iterator() = default;
+    // Placate MSVC by explicitly scoping 'iterator'.
+    const_iterator(typename IntrusiveBackList<T>::iterator X) : N(X.N) {}
+    explicit const_iterator(const T *N) : N(N) {}
+
+    const_iterator &operator++() {
+      N = N->getNext();
+      return *this;
+    }
+
+    explicit operator bool() const { return N; }
+    const T &operator*() const { return *static_cast<const T *>(N); }
+
+    bool operator==(const const_iterator &X) const { return N == X.N; }
+    bool operator!=(const const_iterator &X) const { return N != X.N; }
+  };
+
+  iterator begin() {
+    return Last ? iterator(static_cast<T *>(Last->Next.getPointer())) : end();
+  }
+  const_iterator begin() const {
+    return const_cast<IntrusiveBackList *>(this)->begin();
+  }
+  iterator end() { return iterator(); }
+  const_iterator end() const { return const_iterator(); }
+
+  static iterator toIterator(T &N) { return iterator(&N); }
+  static const_iterator toIterator(const T &N) { return const_iterator(&N); }
+};
+
+/// A list of DIE values.
+///
+/// This is a singly-linked list, but instead of reversing the order of
+/// insertion, we keep a pointer to the back of the list so we can push in
+/// order.
+///
+/// There are two main reasons to choose a linked list over a customized
+/// vector-like data structure.
+///
+///  1. For teardown efficiency, we want DIEs to be BumpPtrAllocated.  Using a
+///     linked list here makes this way easier to accomplish.
+///  2. Carrying an extra pointer per \a DIEValue isn't expensive.  45% of DIEs
+///     have 2 or fewer values, and 90% have 5 or fewer.  A vector would be
+///     over-allocated by 50% on average anyway, the same cost as the
+///     linked-list node.
+class DIEValueList {
+  struct Node : IntrusiveBackListNode {
+    DIEValue V;
+    explicit Node(DIEValue V) : V(V) {}
+  };
+
+  typedef IntrusiveBackList<Node> ListTy;
+  ListTy List;
+
+public:
+  bool empty() const { return List.empty(); }
+
+  class const_iterator;
+  class iterator
+      : public iterator_adaptor_base<iterator, ListTy::iterator,
+                                     std::forward_iterator_tag, DIEValue> {
+    friend class const_iterator;
+    typedef iterator_adaptor_base<iterator, ListTy::iterator,
+                                  std::forward_iterator_tag,
+                                  DIEValue> iterator_adaptor;
+
+  public:
+    iterator() = default;
+    explicit iterator(ListTy::iterator X) : iterator_adaptor(X) {}
+
+    explicit operator bool() const { return bool(wrapped()); }
+    DIEValue &operator*() const { return wrapped()->V; }
+  };
+
+  class const_iterator
+      : public iterator_adaptor_base<const_iterator, ListTy::const_iterator,
+                                     std::forward_iterator_tag,
+                                     const DIEValue> {
+    typedef iterator_adaptor_base<const_iterator, ListTy::const_iterator,
+                                  std::forward_iterator_tag,
+                                  const DIEValue> iterator_adaptor;
+
+  public:
+    const_iterator() = default;
+    const_iterator(DIEValueList::iterator X) : iterator_adaptor(X.wrapped()) {}
+    explicit const_iterator(ListTy::const_iterator X) : iterator_adaptor(X) {}
+
+    explicit operator bool() const { return bool(wrapped()); }
+    const DIEValue &operator*() const { return wrapped()->V; }
+  };
+
+  iterator insert(BumpPtrAllocator &Alloc, DIEValue V) {
+    List.push_back(*new (Alloc) Node(V));
+    return iterator(ListTy::toIterator(List.back()));
+  }
+  template <class... Ts>
+  iterator emplace(BumpPtrAllocator &Alloc, Ts &&... Args) {
+    return insert(Alloc, DIEValue(std::forward<Ts>(Args)...));
+  }
+
+  iterator begin() { return iterator(List.begin()); }
+  iterator end() { return iterator(List.end()); }
+  const_iterator begin() const { return const_iterator(List.begin()); }
+  const_iterator end() const { return const_iterator(List.end()); }
+};
+
+//===--------------------------------------------------------------------===//
+/// DIE - A structured debug information entry.  Has an abbreviation which
+/// describes its organization.
+class DIE : IntrusiveBackListNode {
+  friend class IntrusiveBackList<DIE>;
+
+protected:
+  /// Offset - Offset in debug info section.
+  ///
+  unsigned Offset;
+
+  /// Size - Size of instance + children.
+  ///
+  unsigned Size;
+
+  unsigned AbbrevNumber = ~0u;
+
+  /// Tag - Dwarf tag code.
+  ///
+  dwarf::Tag Tag = (dwarf::Tag)0;
+
+  /// Children DIEs.
+  IntrusiveBackList<DIE> Children;
+
+  DIE *Parent = nullptr;
+
+  /// Attribute values.
+  ///
+  DIEValueList Values;
+
+protected:
+  DIE() : Offset(0), Size(0) {}
+
+private:
+  explicit DIE(dwarf::Tag Tag) : Offset(0), Size(0), Tag(Tag) {}
+
+public:
+  static DIE *get(BumpPtrAllocator &Alloc, dwarf::Tag Tag) {
+    return new (Alloc) DIE(Tag);
+  }
+
+  // Accessors.
+  unsigned getAbbrevNumber() const { return AbbrevNumber; }
+  dwarf::Tag getTag() const { return Tag; }
+  unsigned getOffset() const { return Offset; }
+  unsigned getSize() const { return Size; }
+  bool hasChildren() const { return !Children.empty(); }
+
+  typedef IntrusiveBackList<DIE>::iterator child_iterator;
+  typedef IntrusiveBackList<DIE>::const_iterator const_child_iterator;
+  typedef iterator_range<child_iterator> child_range;
+  typedef iterator_range<const_child_iterator> const_child_range;
+
+  child_range children() {
+    return llvm::make_range(Children.begin(), Children.end());
+  }
+  const_child_range children() const {
+    return llvm::make_range(Children.begin(), Children.end());
+  }
+
+  typedef DIEValueList::iterator value_iterator;
+  typedef iterator_range<value_iterator> value_range;
+
+  value_range values() {
+    return llvm::make_range(Values.begin(), Values.end());
+  }
+
+  typedef DIEValueList::const_iterator const_value_iterator;
+  typedef iterator_range<const_value_iterator> const_value_range;
+
+  const_value_range values() const {
+    return llvm::make_range(Values.begin(), Values.end());
+  }
+
+  DIE *getParent() const { return Parent; }
+
+  /// Generate the abbreviation for this DIE.
+  ///
+  /// Calculate the abbreviation for this, which should be uniqued and
+  /// eventually used to call \a setAbbrevNumber().
+  DIEAbbrev generateAbbrev() const;
+
+  /// Set the abbreviation number for this DIE.
+  void setAbbrevNumber(unsigned I) { AbbrevNumber = I; }
+
+  /// Climb up the parent chain to get the compile or type unit DIE this DIE
+  /// belongs to.
+  const DIE *getUnit() const;
+  /// Similar to getUnit, returns null when DIE is not added to an
+  /// owner yet.
+  const DIE *getUnitOrNull() const;
+  void setOffset(unsigned O) { Offset = O; }
+  void setSize(unsigned S) { Size = S; }
+
+  /// addValue - Add a value and attributes to a DIE.
+  ///
+  value_iterator addValue(BumpPtrAllocator &Alloc, DIEValue Value) {
+    return Values.insert(Alloc, Value);
+  }
+  template <class T>
+  value_iterator addValue(BumpPtrAllocator &Alloc, dwarf::Attribute Attribute,
+                          dwarf::Form Form, T &&Value) {
+    return Values.emplace(Alloc, Attribute, Form, std::forward<T>(Value));
+  }
+
+  /// Add a child to the DIE.
+  DIE &addChild(DIE *Child) {
+    assert(!Child->getParent() && "Child should be orphaned");
+    Child->Parent = this;
+    Children.push_back(*Child);
+    return Children.back();
+  }
+
+  /// Find a value in the DIE with the attribute given.
+  ///
+  /// Returns a default-constructed DIEValue (where \a DIEValue::getType()
+  /// gives \a DIEValue::isNone) if no such attribute exists.
+  DIEValue findAttribute(dwarf::Attribute Attribute) const;
+
+#ifndef NDEBUG
+  void print(raw_ostream &O, unsigned IndentCount = 0) const;
+  void dump();
+#endif
+};
+
 //===--------------------------------------------------------------------===//
 /// DIELoc - Represents an expression location.
 //
-class DIELoc : public DIEValue, public DIE {
+class DIELoc : public DIE {
   mutable unsigned Size; // Size in bytes excluding size header.
+
 public:
-  DIELoc() : DIEValue(isLoc), Size(0) {}
+  DIELoc() : Size(0) {}
 
   /// ComputeSize - Calculate the size of the location expression.
   ///
@@ -497,29 +770,22 @@ class DIELoc : public DIEValue, public DIE {
     return dwarf::DW_FORM_block;
   }
 
-  /// EmitValue - Emit location data.
-  ///
-  void EmitValue(const AsmPrinter *AP, dwarf::Form Form) const override;
-
-  /// SizeOf - Determine size of location data in bytes.
-  ///
-  unsigned SizeOf(const AsmPrinter *AP, dwarf::Form Form) const override;
-
-  // Implement isa/cast/dyncast.
-  static bool classof(const DIEValue *E) { return E->getType() == isLoc; }
+  void EmitValue(const AsmPrinter *AP, dwarf::Form Form) const;
+  unsigned SizeOf(const AsmPrinter *AP, dwarf::Form Form) const;
 
 #ifndef NDEBUG
-  void print(raw_ostream &O) const override;
+  void print(raw_ostream &O) const;
 #endif
 };
 
 //===--------------------------------------------------------------------===//
 /// DIEBlock - Represents a block of values.
 //
-class DIEBlock : public DIEValue, public DIE {
+class DIEBlock : public DIE {
   mutable unsigned Size; // Size in bytes excluding size header.
+
 public:
-  DIEBlock() : DIEValue(isBlock), Size(0) {}
+  DIEBlock() : Size(0) {}
 
   /// ComputeSize - Calculate the size of the location expression.
   ///
@@ -537,49 +803,11 @@ class DIEBlock : public DIEValue, public DIE {
     return dwarf::DW_FORM_block;
   }
 
-  /// EmitValue - Emit location data.
-  ///
-  void EmitValue(const AsmPrinter *AP, dwarf::Form Form) const override;
-
-  /// SizeOf - Determine size of location data in bytes.
-  ///
-  unsigned SizeOf(const AsmPrinter *AP, dwarf::Form Form) const override;
-
-  // Implement isa/cast/dyncast.
-  static bool classof(const DIEValue *E) { return E->getType() == isBlock; }
-
-#ifndef NDEBUG
-  void print(raw_ostream &O) const override;
-#endif
-};
-
-//===--------------------------------------------------------------------===//
-/// DIELocList - Represents a pointer to a location list in the debug_loc
-/// section.
-//
-class DIELocList : public DIEValue {
-  // Index into the .debug_loc vector.
-  size_t Index;
-
-public:
-  DIELocList(size_t I) : DIEValue(isLocList), Index(I) {}
-
-  /// getValue - Grab the current index out.
-  size_t getValue() const { return Index; }
-
-  /// EmitValue - Emit location data.
-  ///
-  void EmitValue(const AsmPrinter *AP, dwarf::Form Form) const override;
-
-  /// SizeOf - Determine size of location data in bytes.
-  ///
-  unsigned SizeOf(const AsmPrinter *AP, dwarf::Form Form) const override;
-
-  // Implement isa/cast/dyncast.
-  static bool classof(const DIEValue *E) { return E->getType() == isLocList; }
+  void EmitValue(const AsmPrinter *AP, dwarf::Form Form) const;
+  unsigned SizeOf(const AsmPrinter *AP, dwarf::Form Form) const;
 
 #ifndef NDEBUG
-  void print(raw_ostream &O) const override;
+  void print(raw_ostream &O) const;
 #endif
 };
 
diff --git a/include/llvm/CodeGen/DIEValue.def b/include/llvm/CodeGen/DIEValue.def
new file mode 100644
index 00000000000..2cfae7b608c
--- /dev/null
+++ b/include/llvm/CodeGen/DIEValue.def
@@ -0,0 +1,47 @@
+//===- llvm/CodeGen/DIEValue.def - DIEValue types ---------------*- C++ -*-===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// Macros for running through all types of DIEValue.
+//
+//===----------------------------------------------------------------------===//
+
+#if !(defined HANDLE_DIEVALUE || defined HANDLE_DIEVALUE_SMALL ||              \
+      defined HANDLE_DIEVALUE_LARGE)
+#error "Missing macro definition of HANDLE_DIEVALUE"
+#endif
+
+// Handler for all values.
+#ifndef HANDLE_DIEVALUE
+#define HANDLE_DIEVALUE(T)
+#endif
+
+// Handler for small values.
+#ifndef HANDLE_DIEVALUE_SMALL
+#define HANDLE_DIEVALUE_SMALL(T) HANDLE_DIEVALUE(T)
+#endif
+
+// Handler for large values.
+#ifndef HANDLE_DIEVALUE_LARGE
+#define HANDLE_DIEVALUE_LARGE(T) HANDLE_DIEVALUE(T)
+#endif
+
+HANDLE_DIEVALUE_SMALL(Integer)
+HANDLE_DIEVALUE_SMALL(String)
+HANDLE_DIEVALUE_SMALL(Expr)
+HANDLE_DIEVALUE_SMALL(Label)
+HANDLE_DIEVALUE_LARGE(Delta)
+HANDLE_DIEVALUE_SMALL(Entry)
+HANDLE_DIEVALUE_SMALL(TypeSignature)
+HANDLE_DIEVALUE_LARGE(Block)
+HANDLE_DIEVALUE_LARGE(Loc)
+HANDLE_DIEVALUE_SMALL(LocList)
+
+#undef HANDLE_DIEVALUE
+#undef HANDLE_DIEVALUE_SMALL
+#undef HANDLE_DIEVALUE_LARGE
diff --git a/include/llvm/CodeGen/DwarfStringPoolEntry.h b/include/llvm/CodeGen/DwarfStringPoolEntry.h
new file mode 100644
index 00000000000..fc2b5ddd2d2
--- /dev/null
+++ b/include/llvm/CodeGen/DwarfStringPoolEntry.h
@@ -0,0 +1,51 @@
+//===- llvm/CodeGen/DwarfStringPoolEntry.h - String pool entry --*- C++ -*-===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_CODEGEN_DWARFSTRINGPOOLENTRY_H
+#define LLVM_CODEGEN_DWARFSTRINGPOOLENTRY_H
+
+#include "llvm/ADT/StringMap.h"
+
+namespace llvm {
+
+class MCSymbol;
+
+/// Data for a string pool entry.
+struct DwarfStringPoolEntry {
+  MCSymbol *Symbol;
+  unsigned Offset;
+  unsigned Index;
+};
+
+/// String pool entry reference.
+struct DwarfStringPoolEntryRef {
+  const StringMapEntry<DwarfStringPoolEntry> *I = nullptr;
+
+public:
+  DwarfStringPoolEntryRef() = default;
+  explicit DwarfStringPoolEntryRef(
+      const StringMapEntry<DwarfStringPoolEntry> &I)
+      : I(&I) {}
+
+  explicit operator bool() const { return I; }
+  MCSymbol *getSymbol() const {
+    assert(I->second.Symbol && "No symbol available!");
+    return I->second.Symbol;
+  }
+  unsigned getOffset() const { return I->second.Offset; }
+  unsigned getIndex() const { return I->second.Index; }
+  StringRef getString() const { return I->first(); }
+
+  bool operator==(const DwarfStringPoolEntryRef &X) const { return I == X.I; }
+  bool operator!=(const DwarfStringPoolEntryRef &X) const { return I != X.I; }
+};
+
+} // end namespace llvm
+
+#endif
diff --git a/include/llvm/CodeGen/FastISel.h b/include/llvm/CodeGen/FastISel.h
index 1dca2ce1ab2..f04a7cd6966 100644
--- a/include/llvm/CodeGen/FastISel.h
+++ b/include/llvm/CodeGen/FastISel.h
@@ -69,7 +69,7 @@ class FastISel {
     unsigned NumFixedArgs;
     CallingConv::ID CallConv;
     const Value *Callee;
-    const char *SymName;
+    MCSymbol *Symbol;
     ArgListTy Args;
     ImmutableCallSite *CS;
     MachineInstr *Call;
@@ -88,7 +88,7 @@ class FastISel {
         : RetTy(nullptr), RetSExt(false), RetZExt(false), IsVarArg(false),
           IsInReg(false), DoesNotReturn(false), IsReturnValueUsed(true),
           IsTailCall(false), NumFixedArgs(-1), CallConv(CallingConv::C),
-          Callee(nullptr), SymName(nullptr), CS(nullptr), Call(nullptr),
+          Callee(nullptr), Symbol(nullptr), CS(nullptr), Call(nullptr),
           ResultReg(0), NumResultRegs(0), IsPatchPoint(false) {}
 
     CallLoweringInfo &setCallee(Type *ResultTy, FunctionType *FuncTy,
@@ -114,12 +114,12 @@ class FastISel {
     }
 
     CallLoweringInfo &setCallee(Type *ResultTy, FunctionType *FuncTy,
-                                const char *Target, ArgListTy &&ArgsList,
+                                MCSymbol *Target, ArgListTy &&ArgsList,
                                 ImmutableCallSite &Call,
                                 unsigned FixedArgs = ~0U) {
       RetTy = ResultTy;
       Callee = Call.getCalledValue();
-      SymName = Target;
+      Symbol = Target;
 
       IsInReg = Call.paramHasAttr(0, Attribute::InReg);
       DoesNotReturn = Call.doesNotReturn();
@@ -148,11 +148,16 @@ class FastISel {
       return *this;
     }
 
-    CallLoweringInfo &setCallee(CallingConv::ID CC, Type *ResultTy,
+    CallLoweringInfo &setCallee(const DataLayout &DL, MCContext &Ctx,
+                                CallingConv::ID CC, Type *ResultTy,
                                 const char *Target, ArgListTy &&ArgsList,
+                                unsigned FixedArgs = ~0U);
+
+    CallLoweringInfo &setCallee(CallingConv::ID CC, Type *ResultTy,
+                                MCSymbol *Target, ArgListTy &&ArgsList,
                                 unsigned FixedArgs = ~0U) {
       RetTy = ResultTy;
-      SymName = Target;
+      Symbol = Target;
       CallConv = CC;
       Args = std::move(ArgsList);
       NumFixedArgs = (FixedArgs == ~0U) ? Args.size() : FixedArgs;
@@ -504,7 +509,9 @@ class FastISel {
 
   CmpInst::Predicate optimizeCmpPredicate(const CmpInst *CI) const;
 
-  bool lowerCallTo(const CallInst *CI, const char *SymName, unsigned NumArgs);
+  bool lowerCallTo(const CallInst *CI, MCSymbol *Symbol, unsigned NumArgs);
+  bool lowerCallTo(const CallInst *CI, const char *SymbolName,
+                   unsigned NumArgs);
   bool lowerCallTo(CallLoweringInfo &CLI);
 
   bool isCommutativeIntrinsic(IntrinsicInst const *II) {
diff --git a/include/llvm/CodeGen/FaultMaps.h b/include/llvm/CodeGen/FaultMaps.h
new file mode 100644
index 00000000000..f4b64632214
--- /dev/null
+++ b/include/llvm/CodeGen/FaultMaps.h
@@ -0,0 +1,220 @@
+//===------------------- FaultMaps.h - The "FaultMaps" section --*- C++ -*-===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_CODEGEN_FAULTMAPS_H
+#define LLVM_CODEGEN_FAULTMAPS_H
+
+#include "llvm/ADT/DenseMap.h"
+#include "llvm/MC/MCSymbol.h"
+#include "llvm/Support/Endian.h"
+#include "llvm/Support/Format.h"
+
+#include <vector>
+#include <map>
+
+namespace llvm {
+
+class AsmPrinter;
+class MCExpr;
+class MCSymbol;
+class MCStreamer;
+
+class FaultMaps {
+public:
+  enum FaultKind { FaultingLoad = 1, FaultKindMax };
+
+  static const char *faultTypeToString(FaultKind);
+
+  explicit FaultMaps(AsmPrinter &AP);
+
+  void recordFaultingOp(FaultKind FaultTy, const MCSymbol *HandlerLabel);
+  void serializeToFaultMapSection();
+
+private:
+  static const char *WFMP;
+
+  struct FaultInfo {
+    FaultKind Kind;
+    const MCExpr *FaultingOffsetExpr;
+    const MCExpr *HandlerOffsetExpr;
+
+    FaultInfo()
+        : Kind(FaultKindMax), FaultingOffsetExpr(nullptr),
+          HandlerOffsetExpr(nullptr) {}
+
+    explicit FaultInfo(FaultMaps::FaultKind Kind, const MCExpr *FaultingOffset,
+                       const MCExpr *HandlerOffset)
+        : Kind(Kind), FaultingOffsetExpr(FaultingOffset),
+          HandlerOffsetExpr(HandlerOffset) {}
+  };
+
+  typedef std::vector<FaultInfo> FunctionFaultInfos;
+
+  // We'd like to keep a stable iteration order for FunctionInfos to help
+  // FileCheck based testing.
+  struct MCSymbolComparator {
+    bool operator()(const MCSymbol *LHS, const MCSymbol *RHS) const {
+      return LHS->getName() < RHS->getName();
+    }
+  };
+
+  std::map<const MCSymbol *, FunctionFaultInfos, MCSymbolComparator>
+      FunctionInfos;
+  AsmPrinter &AP;
+
+  void emitFunctionInfo(const MCSymbol *FnLabel, const FunctionFaultInfos &FFI);
+};
+
+/// A parser for the __llvm_faultmaps section generated by the FaultMaps class
+/// above.  This parser is version locked with with the __llvm_faultmaps section
+/// generated by the version of LLVM that includes it.  No guarantees are made
+/// with respect to forward or backward compatibility.
+class FaultMapParser {
+  typedef uint8_t FaultMapVersionType;
+  static const size_t FaultMapVersionOffset = 0;
+
+  typedef uint8_t Reserved0Type;
+  static const size_t Reserved0Offset =
+      FaultMapVersionOffset + sizeof(FaultMapVersionType);
+
+  typedef uint16_t Reserved1Type;
+  static const size_t Reserved1Offset = Reserved0Offset + sizeof(Reserved0Type);
+
+  typedef uint32_t NumFunctionsType;
+  static const size_t NumFunctionsOffset =
+      Reserved1Offset + sizeof(Reserved1Type);
+
+  static const size_t FunctionInfosOffset =
+      NumFunctionsOffset + sizeof(NumFunctionsType);
+
+  const uint8_t *P;
+  const uint8_t *E;
+
+  template <typename T> static T read(const uint8_t *P, const uint8_t *E) {
+    assert(P + sizeof(T) <= E && "out of bounds read!");
+    return support::endian::read<T, support::little, 1>(P);
+  }
+
+public:
+  class FunctionFaultInfoAccessor {
+    typedef uint32_t FaultKindType;
+    static const size_t FaultKindOffset = 0;
+
+    typedef uint32_t FaultingPCOffsetType;
+    static const size_t FaultingPCOffsetOffset =
+        FaultKindOffset + sizeof(FaultKindType);
+
+    typedef uint32_t HandlerPCOffsetType;
+    static const size_t HandlerPCOffsetOffset =
+        FaultingPCOffsetOffset + sizeof(FaultingPCOffsetType);
+
+    const uint8_t *P;
+    const uint8_t *E;
+
+  public:
+    static const size_t Size =
+        HandlerPCOffsetOffset + sizeof(HandlerPCOffsetType);
+
+    explicit FunctionFaultInfoAccessor(const uint8_t *P, const uint8_t *E)
+        : P(P), E(E) {}
+
+    FaultKindType getFaultKind() const {
+      return read<FaultKindType>(P + FaultKindOffset, E);
+    }
+
+    FaultingPCOffsetType getFaultingPCOffset() const {
+      return read<FaultingPCOffsetType>(P + FaultingPCOffsetOffset, E);
+    }
+
+    HandlerPCOffsetType getHandlerPCOffset() const {
+      return read<HandlerPCOffsetType>(P + HandlerPCOffsetOffset, E);
+    }
+  };
+
+  class FunctionInfoAccessor {
+    typedef uint64_t FunctionAddrType;
+    static const size_t FunctionAddrOffset = 0;
+
+    typedef uint32_t NumFaultingPCsType;
+    static const size_t NumFaultingPCsOffset =
+        FunctionAddrOffset + sizeof(FunctionAddrType);
+
+    typedef uint32_t ReservedType;
+    static const size_t ReservedOffset =
+        NumFaultingPCsOffset + sizeof(NumFaultingPCsType);
+
+    static const size_t FunctionFaultInfosOffset =
+        ReservedOffset + sizeof(ReservedType);
+
+    static const size_t FunctionInfoHeaderSize = FunctionFaultInfosOffset;
+
+    const uint8_t *P;
+    const uint8_t *E;
+
+  public:
+    FunctionInfoAccessor() : P(nullptr), E(nullptr) {}
+
+    explicit FunctionInfoAccessor(const uint8_t *P, const uint8_t *E)
+        : P(P), E(E) {}
+
+    FunctionAddrType getFunctionAddr() const {
+      return read<FunctionAddrType>(P + FunctionAddrOffset, E);
+    }
+
+    NumFaultingPCsType getNumFaultingPCs() const {
+      return read<NumFaultingPCsType>(P + NumFaultingPCsOffset, E);
+    }
+
+    FunctionFaultInfoAccessor getFunctionFaultInfoAt(uint32_t Index) const {
+      assert(Index < getNumFaultingPCs() && "index out of bounds!");
+      const uint8_t *Begin = P + FunctionFaultInfosOffset +
+                             FunctionFaultInfoAccessor::Size * Index;
+      return FunctionFaultInfoAccessor(Begin, E);
+    }
+
+    FunctionInfoAccessor getNextFunctionInfo() const {
+      size_t MySize = FunctionInfoHeaderSize +
+                      getNumFaultingPCs() * FunctionFaultInfoAccessor::Size;
+
+      const uint8_t *Begin = P + MySize;
+      assert(Begin < E && "out of bounds!");
+      return FunctionInfoAccessor(Begin, E);
+    }
+  };
+
+  explicit FaultMapParser(const uint8_t *Begin, const uint8_t *End)
+      : P(Begin), E(End) {}
+
+  FaultMapVersionType getFaultMapVersion() const {
+    auto Version = read<FaultMapVersionType>(P + FaultMapVersionOffset, E);
+    assert(Version == 1 && "only version 1 supported!");
+    return Version;
+  }
+
+  NumFunctionsType getNumFunctions() const {
+    return read<NumFunctionsType>(P + NumFunctionsOffset, E);
+  }
+
+  FunctionInfoAccessor getFirstFunctionInfo() const {
+    const uint8_t *Begin = P + FunctionInfosOffset;
+    return FunctionInfoAccessor(Begin, E);
+  }
+};
+
+raw_ostream &
+operator<<(raw_ostream &OS, const FaultMapParser::FunctionFaultInfoAccessor &);
+
+raw_ostream &operator<<(raw_ostream &OS,
+                        const FaultMapParser::FunctionInfoAccessor &);
+
+raw_ostream &operator<<(raw_ostream &OS, const FaultMapParser &);
+
+} // namespace llvm
+
+#endif
diff --git a/include/llvm/CodeGen/FunctionLoweringInfo.h b/include/llvm/CodeGen/FunctionLoweringInfo.h
index e12866e04df..82c762ed850 100644
--- a/include/llvm/CodeGen/FunctionLoweringInfo.h
+++ b/include/llvm/CodeGen/FunctionLoweringInfo.h
@@ -18,6 +18,7 @@
 #include "llvm/ADT/APInt.h"
 #include "llvm/ADT/DenseMap.h"
 #include "llvm/ADT/IndexedMap.h"
+#include "llvm/ADT/Optional.h"
 #include "llvm/ADT/SmallPtrSet.h"
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/CodeGen/ISDOpcodes.h"
@@ -73,6 +74,16 @@ class FunctionLoweringInfo {
   /// cross-basic-block values.
   DenseMap<const Value*, unsigned> ValueMap;
 
+  // Keep track of frame indices allocated for statepoints as they could be used
+  // across basic block boundaries.
+  // Key of the map is statepoint instruction, value is a map from spilled
+  // llvm Value to the optional stack stack slot index.
+  // If optional is unspecified it means that we have visited this value
+  // but didn't spill it.
+  typedef DenseMap<const Value*, Optional<int>> StatepointSpilledValueMapTy;
+  DenseMap<const Instruction*, StatepointSpilledValueMapTy>
+    StatepointRelocatedValues;
+
   /// StaticAllocaMap - Keep track of frame indices for fixed sized allocas in
   /// the entry block.  This allows the allocas to be efficiently referenced
   /// anywhere in the function.
@@ -221,6 +232,8 @@ class FunctionLoweringInfo {
   int getArgumentFrameIndex(const Argument *A);
 
 private:
+  void addSEHHandlersForLPads(ArrayRef<const LandingPadInst *> LPads);
+
   /// LiveOutRegInfo - Information about live out vregs.
   IndexedMap<LiveOutInfo, VirtReg2IndexFunctor> LiveOutRegInfo;
 };
diff --git a/include/llvm/CodeGen/GCMetadata.h b/include/llvm/CodeGen/GCMetadata.h
index 357b2d8a7ca..e883bd196ea 100644
--- a/include/llvm/CodeGen/GCMetadata.h
+++ b/include/llvm/CodeGen/GCMetadata.h
@@ -121,7 +121,7 @@ class GCFunctionInfo {
   /// label just prior to the safe point (if the code generator is using
   /// MachineModuleInfo).
   void addSafePoint(GC::PointKind Kind, MCSymbol *Label, DebugLoc DL) {
-    SafePoints.push_back(GCPoint(Kind, Label, DL));
+    SafePoints.emplace_back(Kind, Label, DL);
   }
 
   /// getFrameSize/setFrameSize - Records the function's frame size.
diff --git a/include/llvm/CodeGen/GCs.h b/include/llvm/CodeGen/GCs.h
index 5bae41ea030..5207f801c84 100644
--- a/include/llvm/CodeGen/GCs.h
+++ b/include/llvm/CodeGen/GCs.h
@@ -21,6 +21,9 @@ class GCMetadataPrinter;
 /// FIXME: Collector instances are not useful on their own. These no longer
 ///        serve any purpose except to link in the plugins.
 
+/// Creates a CoreCLR-compatible garbage collector.
+void linkCoreCLRGC();
+
 /// Creates an ocaml-compatible garbage collector.
 void linkOcamlGC();
 
diff --git a/include/llvm/CodeGen/ISDOpcodes.h b/include/llvm/CodeGen/ISDOpcodes.h
index 9b439df0abb..49c002bf5f4 100644
--- a/include/llvm/CodeGen/ISDOpcodes.h
+++ b/include/llvm/CodeGen/ISDOpcodes.h
@@ -72,10 +72,13 @@ namespace ISD {
     /// the parent's frame or return address, and so on.
     FRAMEADDR, RETURNADDR,
 
-    /// FRAME_ALLOC_RECOVER - Represents the llvm.framerecover
-    /// intrinsic. Materializes the offset from the frame pointer of another
-    /// function to the result of llvm.frameallocate.
-    FRAME_ALLOC_RECOVER,
+    /// LOCAL_RECOVER - Represents the llvm.localrecover intrinsic.
+    /// Materializes the offset from the local object pointer of another
+    /// function to a particular local object passed to llvm.localescape. The
+    /// operand is the MCSymbol label used to represent this offset, since
+    /// typically the offset is not known until after code generation of the
+    /// parent.
+    LOCAL_RECOVER,
 
     /// READ_REGISTER, WRITE_REGISTER - This node represents llvm.register on
     /// the DAG, which implements the named register global variables extension.
@@ -105,6 +108,10 @@ namespace ISD {
     /// and returns an outchain.
     EH_SJLJ_LONGJMP,
 
+    /// OUTCHAIN = EH_SJLJ_SETUP_DISPATCH(INCHAIN)
+    /// The target initializes the dispatch table here.
+    EH_SJLJ_SETUP_DISPATCH,
+
     /// TargetConstant* - Like Constant*, but the DAG does not do any folding,
     /// simplification, or lowering of the constant. They are used for constants
     /// which are known to fit in the immediate fields of their users, or for
@@ -124,6 +131,8 @@ namespace ISD {
     TargetExternalSymbol,
     TargetBlockAddress,
 
+    MCSymbol,
+
     /// TargetIndex - Like a constant pool entry, but with completely
     /// target-dependent semantics. Holds target flags, a 32-bit index, and a
     /// 64-bit index. Targets can use this however they like.
@@ -239,8 +248,8 @@ namespace ISD {
     FMAD,
 
     /// FCOPYSIGN(X, Y) - Return the value of X with the sign of Y.  NOTE: This
-    /// DAG node does not require that X and Y have the same type, just that the
-    /// are both floating point.  X and the result must have the same type.
+    /// DAG node does not require that X and Y have the same type, just that
+    /// they are both floating point.  X and the result must have the same type.
     /// FCOPYSIGN(f32, f64) is allowed.
     FCOPYSIGN,
 
@@ -308,6 +317,10 @@ namespace ISD {
     /// part.
     MULHU, MULHS,
 
+    /// [US]{MIN/MAX} - Binary minimum or maximum or signed or unsigned
+    /// integers.
+    SMIN, SMAX, UMIN, UMAX,
+
     /// Bitwise operators - logical and, logical or, logical xor.
     AND, OR, XOR,
 
@@ -325,6 +338,10 @@ namespace ISD {
     /// Byte Swap and Counting operators.
     BSWAP, CTTZ, CTLZ, CTPOP,
 
+    /// [SU]ABSDIFF - Signed/Unsigned absolute difference of two input integer
+    /// vector. These nodes are generated from llvm.*absdiff* intrinsics.
+    SABSDIFF, UABSDIFF,
+
     /// Bit counting operators with an undefined result for zero inputs.
     CTTZ_ZERO_UNDEF, CTLZ_ZERO_UNDEF,
 
@@ -687,19 +704,28 @@ namespace ISD {
     ATOMIC_LOAD_UMIN,
     ATOMIC_LOAD_UMAX,
 
-    // Masked load and store
+    // Masked load and store - consecutive vector load and store operations
+    // with additional mask operand that prevents memory accesses to the
+    // masked-off lanes.
     MLOAD, MSTORE,
 
+    // Masked gather and scatter - load and store operations for a vector of
+    // random addresses with additional mask operand that prevents memory
+    // accesses to the masked-off lanes.
+    MGATHER, MSCATTER,
+
     /// This corresponds to the llvm.lifetime.* intrinsics. The first operand
     /// is the chain and the second operand is the alloca pointer.
     LIFETIME_START, LIFETIME_END,
 
-    // @LOCALMOD-BEGIN
-    // NACL_* - Native Client instrinsics.
-    // NACL_READ_TP is a fast built-in version of NaCl's tls_get() IRT
-    // interface.
-    NACL_READ_TP,
-    // @LOCALMOD-END
+    /// GC_TRANSITION_START/GC_TRANSITION_END - These operators mark the
+    /// beginning and end of GC transition  sequence, and carry arbitrary
+    /// information that target might need for lowering.  The first operand is
+    /// a chain, the rest are specified by the target and not touched by the DAG
+    /// optimizers. GC_TRANSITION_START..GC_TRANSITION_END pairs may not be
+    /// nested.
+    GC_TRANSITION_START,
+    GC_TRANSITION_END,
 
     /// BUILTIN_OP_END - This must be the last enum value in this list.
     /// The target-specific pre-isel opcode values start here.
@@ -710,7 +736,7 @@ namespace ISD {
   /// which do not reference a specific memory location should be less than
   /// this value. Those that do must not be less than this value, and can
   /// be used with SelectionDAG::getMemIntrinsicNode.
-  static const int FIRST_TARGET_MEMORY_OPCODE = BUILTIN_OP_END+200;
+  static const int FIRST_TARGET_MEMORY_OPCODE = BUILTIN_OP_END+300;
 
   //===--------------------------------------------------------------------===//
   /// MemIndexedMode enum - This enum defines the load / store indexed
diff --git a/include/llvm/CodeGen/LexicalScopes.h b/include/llvm/CodeGen/LexicalScopes.h
index aa217d5519e..7d7e48af2a0 100644
--- a/include/llvm/CodeGen/LexicalScopes.h
+++ b/include/llvm/CodeGen/LexicalScopes.h
@@ -45,7 +45,7 @@ typedef std::pair<const MachineInstr *, const MachineInstr *> InsnRange;
 class LexicalScope {
 
 public:
-  LexicalScope(LexicalScope *P, const MDLocalScope *D, const MDLocation *I,
+  LexicalScope(LexicalScope *P, const DILocalScope *D, const DILocation *I,
                bool A)
       : Parent(P), Desc(D), InlinedAtLocation(I), AbstractScope(A),
         LastInsn(nullptr), FirstInsn(nullptr), DFSIn(0), DFSOut(0) {
@@ -58,8 +58,8 @@ class LexicalScope {
   // Accessors.
   LexicalScope *getParent() const { return Parent; }
   const MDNode *getDesc() const { return Desc; }
-  const MDLocation *getInlinedAt() const { return InlinedAtLocation; }
-  const MDLocalScope *getScopeNode() const { return Desc; }
+  const DILocation *getInlinedAt() const { return InlinedAtLocation; }
+  const DILocalScope *getScopeNode() const { return Desc; }
   bool isAbstractScope() const { return AbstractScope; }
   SmallVectorImpl<LexicalScope *> &getChildren() { return Children; }
   SmallVectorImpl<InsnRange> &getRanges() { return Ranges; }
@@ -119,8 +119,8 @@ class LexicalScope {
 
 private:
   LexicalScope *Parent;                        // Parent to this scope.
-  const MDLocalScope *Desc;                    // Debug info descriptor.
-  const MDLocation *InlinedAtLocation;         // Location at which this
+  const DILocalScope *Desc;                    // Debug info descriptor.
+  const DILocation *InlinedAtLocation;         // Location at which this
                                                // scope is inlined.
   bool AbstractScope;                          // Abstract Scope
   SmallVector<LexicalScope *, 4> Children;     // Scopes defined in scope.
@@ -159,16 +159,16 @@ class LexicalScopes {
   /// getMachineBasicBlocks - Populate given set using machine basic blocks
   /// which have machine instructions that belong to lexical scope identified by
   /// DebugLoc.
-  void getMachineBasicBlocks(const MDLocation *DL,
+  void getMachineBasicBlocks(const DILocation *DL,
                              SmallPtrSetImpl<const MachineBasicBlock *> &MBBs);
 
   /// dominates - Return true if DebugLoc's lexical scope dominates at least one
   /// machine instruction's lexical scope in a given machine basic block.
-  bool dominates(const MDLocation *DL, MachineBasicBlock *MBB);
+  bool dominates(const DILocation *DL, MachineBasicBlock *MBB);
 
   /// findLexicalScope - Find lexical scope, either regular or inlined, for the
   /// given DebugLoc. Return NULL if not found.
-  LexicalScope *findLexicalScope(const MDLocation *DL);
+  LexicalScope *findLexicalScope(const DILocation *DL);
 
   /// getAbstractScopesList - Return a reference to list of abstract scopes.
   ArrayRef<LexicalScope *> getAbstractScopesList() const {
@@ -176,19 +176,19 @@ class LexicalScopes {
   }
 
   /// findAbstractScope - Find an abstract scope or return null.
-  LexicalScope *findAbstractScope(const MDLocalScope *N) {
+  LexicalScope *findAbstractScope(const DILocalScope *N) {
     auto I = AbstractScopeMap.find(N);
     return I != AbstractScopeMap.end() ? &I->second : nullptr;
   }
 
   /// findInlinedScope - Find an inlined scope for the given scope/inlined-at.
-  LexicalScope *findInlinedScope(const MDLocalScope *N, const MDLocation *IA) {
+  LexicalScope *findInlinedScope(const DILocalScope *N, const DILocation *IA) {
     auto I = InlinedLexicalScopeMap.find(std::make_pair(N, IA));
     return I != InlinedLexicalScopeMap.end() ? &I->second : nullptr;
   }
 
   /// findLexicalScope - Find regular lexical scope or return null.
-  LexicalScope *findLexicalScope(const MDLocalScope *N) {
+  LexicalScope *findLexicalScope(const DILocalScope *N) {
     auto I = LexicalScopeMap.find(N);
     return I != LexicalScopeMap.end() ? &I->second : nullptr;
   }
@@ -197,24 +197,24 @@ class LexicalScopes {
   void dump();
 
   /// getOrCreateAbstractScope - Find or create an abstract lexical scope.
-  LexicalScope *getOrCreateAbstractScope(const MDLocalScope *Scope);
+  LexicalScope *getOrCreateAbstractScope(const DILocalScope *Scope);
 
 private:
   /// getOrCreateLexicalScope - Find lexical scope for the given Scope/IA. If
   /// not available then create new lexical scope.
-  LexicalScope *getOrCreateLexicalScope(const MDLocalScope *Scope,
-                                        const MDLocation *IA = nullptr);
-  LexicalScope *getOrCreateLexicalScope(const MDLocation *DL) {
+  LexicalScope *getOrCreateLexicalScope(const DILocalScope *Scope,
+                                        const DILocation *IA = nullptr);
+  LexicalScope *getOrCreateLexicalScope(const DILocation *DL) {
     return DL ? getOrCreateLexicalScope(DL->getScope(), DL->getInlinedAt())
               : nullptr;
   }
 
   /// getOrCreateRegularScope - Find or create a regular lexical scope.
-  LexicalScope *getOrCreateRegularScope(const MDLocalScope *Scope);
+  LexicalScope *getOrCreateRegularScope(const DILocalScope *Scope);
 
   /// getOrCreateInlinedScope - Find or create an inlined lexical scope.
-  LexicalScope *getOrCreateInlinedScope(const MDLocalScope *Scope,
-                                        const MDLocation *InlinedAt);
+  LexicalScope *getOrCreateInlinedScope(const DILocalScope *Scope,
+                                        const DILocation *InlinedAt);
 
   /// extractLexicalScopes - Extract instruction ranges for each lexical scopes
   /// for the given machine function.
@@ -230,18 +230,18 @@ class LexicalScopes {
 
   /// LexicalScopeMap - Tracks the scopes in the current function.
   // Use an unordered_map to ensure value pointer validity over insertion.
-  std::unordered_map<const MDLocalScope *, LexicalScope> LexicalScopeMap;
+  std::unordered_map<const DILocalScope *, LexicalScope> LexicalScopeMap;
 
   /// InlinedLexicalScopeMap - Tracks inlined function scopes in current
   /// function.
-  std::unordered_map<std::pair<const MDLocalScope *, const MDLocation *>,
+  std::unordered_map<std::pair<const DILocalScope *, const DILocation *>,
                      LexicalScope,
-                     pair_hash<const MDLocalScope *, const MDLocation *>>
+                     pair_hash<const DILocalScope *, const DILocation *>>
       InlinedLexicalScopeMap;
 
   /// AbstractScopeMap - These scopes are  not included LexicalScopeMap.
   // Use an unordered_map to ensure value pointer validity over insertion.
-  std::unordered_map<const MDLocalScope *, LexicalScope> AbstractScopeMap;
+  std::unordered_map<const DILocalScope *, LexicalScope> AbstractScopeMap;
 
   /// AbstractScopesList - Tracks abstract scopes constructed while processing
   /// a function.
diff --git a/include/llvm/CodeGen/LinkAllCodegenComponents.h b/include/llvm/CodeGen/LinkAllCodegenComponents.h
index 4c07a68fb60..fee131e4a3c 100644
--- a/include/llvm/CodeGen/LinkAllCodegenComponents.h
+++ b/include/llvm/CodeGen/LinkAllCodegenComponents.h
@@ -34,11 +34,9 @@ namespace {
       (void) llvm::createFastRegisterAllocator();
       (void) llvm::createBasicRegisterAllocator();
       (void) llvm::createGreedyRegisterAllocator();
-#if !defined(__native_client__)
-      // Not needed by sandboxed translator.
       (void) llvm::createDefaultPBQPRegisterAllocator();
-#endif
 
+      llvm::linkCoreCLRGC();
       llvm::linkOcamlGC();
       llvm::linkErlangGC();
       llvm::linkShadowStackGC();
diff --git a/include/llvm/CodeGen/LiveIntervalAnalysis.h b/include/llvm/CodeGen/LiveIntervalAnalysis.h
index 9673f80e085..47ba1ff0454 100644
--- a/include/llvm/CodeGen/LiveIntervalAnalysis.h
+++ b/include/llvm/CodeGen/LiveIntervalAnalysis.h
@@ -147,13 +147,12 @@ extern cl::opt<bool> UseSegmentSetForPhysRegs;
     LiveInterval::Segment addSegmentToEndOfBlock(unsigned reg,
                                                  MachineInstr* startInst);
 
-    /// shrinkToUses - After removing some uses of a register, shrink its live
-    /// range to just the remaining uses. This method does not compute reaching
-    /// defs for new uses, and it doesn't remove dead defs.
-    /// Dead PHIDef values are marked as unused.
-    /// New dead machine instructions are added to the dead vector.
-    /// Return true if the interval may have been separated into multiple
-    /// connected components.
+    /// After removing some uses of a register, shrink its live range to just
+    /// the remaining uses. This method does not compute reaching defs for new
+    /// uses, and it doesn't remove dead defs.
+    /// Dead PHIDef values are marked as unused. New dead machine instructions
+    /// are added to the dead vector. Returns true if the interval may have been
+    /// separated into multiple connected components.
     bool shrinkToUses(LiveInterval *li,
                       SmallVectorImpl<MachineInstr*> *dead = nullptr);
 
@@ -161,6 +160,8 @@ extern cl::opt<bool> UseSegmentSetForPhysRegs;
     /// shrinkToUses(LiveInterval *li, SmallVectorImpl<MachineInstr*> *dead)
     /// that works on a subregister live range and only looks at uses matching
     /// the lane mask of the subregister range.
+    /// This may leave the subrange empty which needs to be cleaned up with
+    /// LiveInterval::removeEmptySubranges() afterwards.
     void shrinkToUses(LiveInterval::SubRange &SR, unsigned Reg);
 
     /// extendToIndices - Extend the live range of LI to reach all points in
diff --git a/include/llvm/CodeGen/LiveIntervalUnion.h b/include/llvm/CodeGen/LiveIntervalUnion.h
index 967f0cbac71..f0f1637dc92 100644
--- a/include/llvm/CodeGen/LiveIntervalUnion.h
+++ b/include/llvm/CodeGen/LiveIntervalUnion.h
@@ -203,6 +203,11 @@ class LiveIntervalUnion {
       assert(idx <  Size && "idx out of bounds");
       return LIUs[idx];
     }
+
+    const LiveIntervalUnion& operator[](unsigned Idx) const {
+      assert(Idx < Size && "Idx out of bounds");
+      return LIUs[Idx];
+    }
   };
 };
 
diff --git a/include/llvm/CodeGen/LivePhysRegs.h b/include/llvm/CodeGen/LivePhysRegs.h
index f44d627d717..6475e7b4af3 100644
--- a/include/llvm/CodeGen/LivePhysRegs.h
+++ b/include/llvm/CodeGen/LivePhysRegs.h
@@ -93,7 +93,8 @@ class LivePhysRegs {
   }
 
   /// \brief Removes physical registers clobbered by the regmask operand @p MO.
-  void removeRegsInMask(const MachineOperand &MO);
+  void removeRegsInMask(const MachineOperand &MO,
+        SmallVectorImpl<std::pair<unsigned, const MachineOperand*>> *Clobbers);
 
   /// \brief Returns true if register @p Reg is contained in the set. This also
   /// works if only the super register of @p Reg has been defined, because we
@@ -109,21 +110,21 @@ class LivePhysRegs {
   /// instruction(bundle): Remove killed-uses, add defs. This is the not
   /// recommended way, because it depends on accurate kill flags. If possible
   /// use stepBackwards() instead of this function.
-  void stepForward(const MachineInstr &MI);
-
-  /// \brief Adds all live-in registers of basic block @p MBB.
-  void addLiveIns(const MachineBasicBlock *MBB) {
-    for (MachineBasicBlock::livein_iterator LI = MBB->livein_begin(),
-         LE = MBB->livein_end(); LI != LE; ++LI)
-      addReg(*LI);
-  }
-
-  /// \brief Adds all live-out registers of basic block @p MBB.
-  void addLiveOuts(const MachineBasicBlock *MBB) {
-    for (MachineBasicBlock::const_succ_iterator SI = MBB->succ_begin(),
-         SE = MBB->succ_end(); SI != SE; ++SI)
-      addLiveIns(*SI);
-  }
+  /// The clobbers set will be the list of registers either defined or clobbered
+  /// by a regmask.  The operand will identify whether this is a regmask or
+  /// register operand.
+  void stepForward(const MachineInstr &MI,
+        SmallVectorImpl<std::pair<unsigned, const MachineOperand*>> &Clobbers);
+
+  /// \brief Adds all live-in registers of basic block @p MBB; After prologue/
+  /// epilogue insertion \p AddPristines should be set to true to insert the
+  /// pristine registers.
+  void addLiveIns(const MachineBasicBlock *MBB, bool AddPristines = false);
+
+  /// \brief Adds all live-out registers of basic block @p MBB; After prologue/
+  /// epilogue insertion \p AddPristines should be set to true to insert the
+  /// pristine registers.
+  void addLiveOuts(const MachineBasicBlock *MBB, bool AddPristines = false);
 
   typedef SparseSet<unsigned>::const_iterator const_iterator;
   const_iterator begin() const { return LiveRegs.begin(); }
diff --git a/include/llvm/CodeGen/LiveRangeEdit.h b/include/llvm/CodeGen/LiveRangeEdit.h
index de855f2fe7a..c97c636abbb 100644
--- a/include/llvm/CodeGen/LiveRangeEdit.h
+++ b/include/llvm/CodeGen/LiveRangeEdit.h
@@ -102,6 +102,10 @@ class LiveRangeEdit : private MachineRegisterInfo::Delegate {
   /// registers are created.
   void MRI_NoteNewVirtualRegister(unsigned VReg) override;
 
+  /// \brief Check if MachineOperand \p MO is a last use/kill either in the
+  /// main live range of \p LI or in one of the matching subregister ranges.
+  bool useIsKill(const LiveInterval &LI, const MachineOperand &MO) const;
+
 public:
   /// Create a LiveRangeEdit for breaking down parent into smaller pieces.
   /// @param parent The register being spilled or split.
diff --git a/include/llvm/CodeGen/LiveRegMatrix.h b/include/llvm/CodeGen/LiveRegMatrix.h
index 878b4d9836f..e169058ca56 100644
--- a/include/llvm/CodeGen/LiveRegMatrix.h
+++ b/include/llvm/CodeGen/LiveRegMatrix.h
@@ -32,13 +32,11 @@ namespace llvm {
 
 class LiveInterval;
 class LiveIntervalAnalysis;
-class MachineRegisterInfo;
 class TargetRegisterInfo;
 class VirtRegMap;
 
 class LiveRegMatrix : public MachineFunctionPass {
   const TargetRegisterInfo *TRI;
-  MachineRegisterInfo *MRI;
   LiveIntervals *LIS;
   VirtRegMap *VRM;
 
@@ -114,6 +112,9 @@ class LiveRegMatrix : public MachineFunctionPass {
   /// the assignment and updates VirtRegMap accordingly.
   void unassign(LiveInterval &VirtReg);
 
+  /// Returns true if the given \p PhysReg has any live intervals assigned.
+  bool isPhysRegUsed(unsigned PhysReg) const;
+
   //===--------------------------------------------------------------------===//
   // Low-level interface.
   //===--------------------------------------------------------------------===//
diff --git a/include/llvm/CodeGen/MIRParser/MIRParser.h b/include/llvm/CodeGen/MIRParser/MIRParser.h
new file mode 100644
index 00000000000..67b756d5e88
--- /dev/null
+++ b/include/llvm/CodeGen/MIRParser/MIRParser.h
@@ -0,0 +1,81 @@
+//===- MIRParser.h - MIR serialization format parser ----------------------===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This MIR serialization library is currently a work in progress. It can't
+// serialize machine functions at this time.
+//
+// This file declares the functions that parse the MIR serialization format
+// files.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_CODEGEN_MIRPARSER_MIRPARSER_H
+#define LLVM_CODEGEN_MIRPARSER_MIRPARSER_H
+
+#include "llvm/ADT/StringRef.h"
+#include "llvm/CodeGen/MachineFunctionInitializer.h"
+#include "llvm/IR/Module.h"
+#include "llvm/Support/MemoryBuffer.h"
+#include <memory>
+
+namespace llvm {
+
+class MIRParserImpl;
+class SMDiagnostic;
+
+/// This class initializes machine functions by applying the state loaded from
+/// a MIR file.
+class MIRParser : public MachineFunctionInitializer {
+  std::unique_ptr<MIRParserImpl> Impl;
+
+public:
+  MIRParser(std::unique_ptr<MIRParserImpl> Impl);
+  MIRParser(const MIRParser &) = delete;
+  ~MIRParser();
+
+  /// Parse the optional LLVM IR module that's embedded in the MIR file.
+  ///
+  /// A new, empty module is created if the LLVM IR isn't present.
+  /// Returns null if a parsing error occurred.
+  std::unique_ptr<Module> parseLLVMModule();
+
+  /// Initialize the machine function to the state that's described in the MIR
+  /// file.
+  ///
+  /// Return true if error occurred.
+  bool initializeMachineFunction(MachineFunction &MF) override;
+};
+
+/// This function is the main interface to the MIR serialization format parser.
+///
+/// It reads in a MIR file and returns a MIR parser that can parse the embedded
+/// LLVM IR module and initialize the machine functions by parsing the machine
+/// function's state.
+///
+/// \param Filename - The name of the file to parse.
+/// \param Error - Error result info.
+/// \param Context - Context which will be used for the parsed LLVM IR module.
+std::unique_ptr<MIRParser> createMIRParserFromFile(StringRef Filename,
+                                                   SMDiagnostic &Error,
+                                                   LLVMContext &Context);
+
+/// This function is another interface to the MIR serialization format parser.
+///
+/// It returns a MIR parser that works with the given memory buffer and that can
+/// parse the embedded LLVM IR module and initialize the machine functions by
+/// parsing the machine function's state.
+///
+/// \param Contents - The MemoryBuffer containing the machine level IR.
+/// \param Context - Context which will be used for the parsed LLVM IR module.
+std::unique_ptr<MIRParser>
+createMIRParser(std::unique_ptr<MemoryBuffer> Contents, LLVMContext &Context);
+
+} // end namespace llvm
+
+#endif
diff --git a/include/llvm/CodeGen/MIRYamlMapping.h b/include/llvm/CodeGen/MIRYamlMapping.h
new file mode 100644
index 00000000000..777819f3eae
--- /dev/null
+++ b/include/llvm/CodeGen/MIRYamlMapping.h
@@ -0,0 +1,434 @@
+//===- MIRYAMLMapping.h - Describes the mapping between MIR and YAML ------===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// The MIR serialization library is currently a work in progress. It can't
+// serialize machine functions at this time.
+//
+// This file implements the mapping between various MIR data structures and
+// their corresponding YAML representation.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIB_CODEGEN_MIRYAMLMAPPING_H
+#define LLVM_LIB_CODEGEN_MIRYAMLMAPPING_H
+
+#include "llvm/ADT/StringRef.h"
+#include "llvm/CodeGen/MachineJumpTableInfo.h"
+#include "llvm/Support/YAMLTraits.h"
+#include <vector>
+
+namespace llvm {
+namespace yaml {
+
+/// A wrapper around std::string which contains a source range that's being
+/// set during parsing.
+struct StringValue {
+  std::string Value;
+  SMRange SourceRange;
+
+  StringValue() {}
+  StringValue(std::string Value) : Value(std::move(Value)) {}
+
+  bool operator==(const StringValue &Other) const {
+    return Value == Other.Value;
+  }
+};
+
+template <> struct ScalarTraits<StringValue> {
+  static void output(const StringValue &S, void *, llvm::raw_ostream &OS) {
+    OS << S.Value;
+  }
+
+  static StringRef input(StringRef Scalar, void *Ctx, StringValue &S) {
+    S.Value = Scalar.str();
+    if (const auto *Node =
+            reinterpret_cast<yaml::Input *>(Ctx)->getCurrentNode())
+      S.SourceRange = Node->getSourceRange();
+    return "";
+  }
+
+  static bool mustQuote(StringRef Scalar) { return needsQuotes(Scalar); }
+};
+
+struct FlowStringValue : StringValue {
+  FlowStringValue() {}
+  FlowStringValue(std::string Value) : StringValue(Value) {}
+};
+
+template <> struct ScalarTraits<FlowStringValue> {
+  static void output(const FlowStringValue &S, void *, llvm::raw_ostream &OS) {
+    return ScalarTraits<StringValue>::output(S, nullptr, OS);
+  }
+
+  static StringRef input(StringRef Scalar, void *Ctx, FlowStringValue &S) {
+    return ScalarTraits<StringValue>::input(Scalar, Ctx, S);
+  }
+
+  static bool mustQuote(StringRef Scalar) { return needsQuotes(Scalar); }
+};
+
+/// A wrapper around unsigned which contains a source range that's being set
+/// during parsing.
+struct UnsignedValue {
+  unsigned Value;
+  SMRange SourceRange;
+
+  UnsignedValue() : Value(0) {}
+  UnsignedValue(unsigned Value) : Value(Value) {}
+
+  bool operator==(const UnsignedValue &Other) const {
+    return Value == Other.Value;
+  }
+};
+
+template <> struct ScalarTraits<UnsignedValue> {
+  static void output(const UnsignedValue &Value, void *Ctx, raw_ostream &OS) {
+    return ScalarTraits<unsigned>::output(Value.Value, Ctx, OS);
+  }
+
+  static StringRef input(StringRef Scalar, void *Ctx, UnsignedValue &Value) {
+    if (const auto *Node =
+            reinterpret_cast<yaml::Input *>(Ctx)->getCurrentNode())
+      Value.SourceRange = Node->getSourceRange();
+    return ScalarTraits<unsigned>::input(Scalar, Ctx, Value.Value);
+  }
+
+  static bool mustQuote(StringRef Scalar) {
+    return ScalarTraits<unsigned>::mustQuote(Scalar);
+  }
+};
+
+template <> struct ScalarEnumerationTraits<MachineJumpTableInfo::JTEntryKind> {
+  static void enumeration(yaml::IO &IO,
+                          MachineJumpTableInfo::JTEntryKind &EntryKind) {
+    IO.enumCase(EntryKind, "block-address",
+                MachineJumpTableInfo::EK_BlockAddress);
+    IO.enumCase(EntryKind, "gp-rel64-block-address",
+                MachineJumpTableInfo::EK_GPRel64BlockAddress);
+    IO.enumCase(EntryKind, "gp-rel32-block-address",
+                MachineJumpTableInfo::EK_GPRel32BlockAddress);
+    IO.enumCase(EntryKind, "label-difference32",
+                MachineJumpTableInfo::EK_LabelDifference32);
+    IO.enumCase(EntryKind, "inline", MachineJumpTableInfo::EK_Inline);
+    IO.enumCase(EntryKind, "custom32", MachineJumpTableInfo::EK_Custom32);
+  }
+};
+
+} // end namespace yaml
+} // end namespace llvm
+
+LLVM_YAML_IS_SEQUENCE_VECTOR(llvm::yaml::StringValue)
+LLVM_YAML_IS_FLOW_SEQUENCE_VECTOR(llvm::yaml::FlowStringValue)
+LLVM_YAML_IS_FLOW_SEQUENCE_VECTOR(llvm::yaml::UnsignedValue)
+
+namespace llvm {
+namespace yaml {
+
+struct VirtualRegisterDefinition {
+  UnsignedValue ID;
+  StringValue Class;
+  StringValue PreferredRegister;
+  // TODO: Serialize the target specific register hints.
+};
+
+template <> struct MappingTraits<VirtualRegisterDefinition> {
+  static void mapping(IO &YamlIO, VirtualRegisterDefinition &Reg) {
+    YamlIO.mapRequired("id", Reg.ID);
+    YamlIO.mapRequired("class", Reg.Class);
+    YamlIO.mapOptional("preferred-register", Reg.PreferredRegister,
+                       StringValue()); // Don't print out when it's empty.
+  }
+
+  static const bool flow = true;
+};
+
+struct MachineFunctionLiveIn {
+  StringValue Register;
+  StringValue VirtualRegister;
+};
+
+template <> struct MappingTraits<MachineFunctionLiveIn> {
+  static void mapping(IO &YamlIO, MachineFunctionLiveIn &LiveIn) {
+    YamlIO.mapRequired("reg", LiveIn.Register);
+    YamlIO.mapOptional(
+        "virtual-reg", LiveIn.VirtualRegister,
+        StringValue()); // Don't print the virtual register when it's empty.
+  }
+
+  static const bool flow = true;
+};
+
+struct MachineBasicBlock {
+  unsigned ID;
+  StringValue Name;
+  StringValue IRBlock;
+  unsigned Alignment = 0;
+  bool IsLandingPad = false;
+  bool AddressTaken = false;
+  std::vector<FlowStringValue> Successors;
+  std::vector<UnsignedValue> SuccessorWeights;
+  std::vector<FlowStringValue> LiveIns;
+  std::vector<StringValue> Instructions;
+};
+
+template <> struct MappingTraits<MachineBasicBlock> {
+  static void mapping(IO &YamlIO, MachineBasicBlock &MBB) {
+    YamlIO.mapRequired("id", MBB.ID);
+    YamlIO.mapOptional("name", MBB.Name,
+                       StringValue()); // Don't print out an empty name.
+    YamlIO.mapOptional("ir-block", MBB.IRBlock,
+                       StringValue()); // Don't print out an empty BB reference.
+    YamlIO.mapOptional("alignment", MBB.Alignment);
+    YamlIO.mapOptional("isLandingPad", MBB.IsLandingPad);
+    YamlIO.mapOptional("addressTaken", MBB.AddressTaken);
+    YamlIO.mapOptional("successors", MBB.Successors);
+    YamlIO.mapOptional("weights", MBB.SuccessorWeights);
+    YamlIO.mapOptional("liveins", MBB.LiveIns);
+    YamlIO.mapOptional("instructions", MBB.Instructions);
+  }
+};
+
+/// Serializable representation of stack object from the MachineFrameInfo class.
+///
+/// The flags 'isImmutable' and 'isAliased' aren't serialized, as they are
+/// determined by the object's type and frame information flags.
+/// Dead stack objects aren't serialized.
+///
+/// TODO: Determine isPreallocated flag by mapping between objects and local
+/// objects (Serialize local objects).
+struct MachineStackObject {
+  enum ObjectType { DefaultType, SpillSlot, VariableSized };
+  unsigned ID;
+  StringValue Name;
+  // TODO: Serialize unnamed LLVM alloca reference.
+  ObjectType Type = DefaultType;
+  int64_t Offset = 0;
+  uint64_t Size = 0;
+  unsigned Alignment = 0;
+  StringValue CalleeSavedRegister;
+};
+
+template <> struct ScalarEnumerationTraits<MachineStackObject::ObjectType> {
+  static void enumeration(yaml::IO &IO, MachineStackObject::ObjectType &Type) {
+    IO.enumCase(Type, "default", MachineStackObject::DefaultType);
+    IO.enumCase(Type, "spill-slot", MachineStackObject::SpillSlot);
+    IO.enumCase(Type, "variable-sized", MachineStackObject::VariableSized);
+  }
+};
+
+template <> struct MappingTraits<MachineStackObject> {
+  static void mapping(yaml::IO &YamlIO, MachineStackObject &Object) {
+    YamlIO.mapRequired("id", Object.ID);
+    YamlIO.mapOptional("name", Object.Name,
+                       StringValue()); // Don't print out an empty name.
+    YamlIO.mapOptional(
+        "type", Object.Type,
+        MachineStackObject::DefaultType); // Don't print the default type.
+    YamlIO.mapOptional("offset", Object.Offset);
+    if (Object.Type != MachineStackObject::VariableSized)
+      YamlIO.mapRequired("size", Object.Size);
+    YamlIO.mapOptional("alignment", Object.Alignment);
+    YamlIO.mapOptional("callee-saved-register", Object.CalleeSavedRegister,
+                       StringValue()); // Don't print it out when it's empty.
+  }
+
+  static const bool flow = true;
+};
+
+/// Serializable representation of the fixed stack object from the
+/// MachineFrameInfo class.
+struct FixedMachineStackObject {
+  enum ObjectType { DefaultType, SpillSlot };
+  unsigned ID;
+  ObjectType Type = DefaultType;
+  int64_t Offset = 0;
+  uint64_t Size = 0;
+  unsigned Alignment = 0;
+  bool IsImmutable = false;
+  bool IsAliased = false;
+  StringValue CalleeSavedRegister;
+};
+
+template <>
+struct ScalarEnumerationTraits<FixedMachineStackObject::ObjectType> {
+  static void enumeration(yaml::IO &IO,
+                          FixedMachineStackObject::ObjectType &Type) {
+    IO.enumCase(Type, "default", FixedMachineStackObject::DefaultType);
+    IO.enumCase(Type, "spill-slot", FixedMachineStackObject::SpillSlot);
+  }
+};
+
+template <> struct MappingTraits<FixedMachineStackObject> {
+  static void mapping(yaml::IO &YamlIO, FixedMachineStackObject &Object) {
+    YamlIO.mapRequired("id", Object.ID);
+    YamlIO.mapOptional(
+        "type", Object.Type,
+        FixedMachineStackObject::DefaultType); // Don't print the default type.
+    YamlIO.mapOptional("offset", Object.Offset);
+    YamlIO.mapOptional("size", Object.Size);
+    YamlIO.mapOptional("alignment", Object.Alignment);
+    if (Object.Type != FixedMachineStackObject::SpillSlot) {
+      YamlIO.mapOptional("isImmutable", Object.IsImmutable);
+      YamlIO.mapOptional("isAliased", Object.IsAliased);
+    }
+    YamlIO.mapOptional("callee-saved-register", Object.CalleeSavedRegister,
+                       StringValue()); // Don't print it out when it's empty.
+  }
+
+  static const bool flow = true;
+};
+
+struct MachineConstantPoolValue {
+  UnsignedValue ID;
+  StringValue Value;
+  unsigned Alignment = 0;
+};
+
+template <> struct MappingTraits<MachineConstantPoolValue> {
+  static void mapping(IO &YamlIO, MachineConstantPoolValue &Constant) {
+    YamlIO.mapRequired("id", Constant.ID);
+    YamlIO.mapOptional("value", Constant.Value);
+    YamlIO.mapOptional("alignment", Constant.Alignment);
+  }
+};
+
+struct MachineJumpTable {
+  struct Entry {
+    unsigned ID;
+    std::vector<FlowStringValue> Blocks;
+  };
+
+  MachineJumpTableInfo::JTEntryKind Kind = MachineJumpTableInfo::EK_Custom32;
+  std::vector<Entry> Entries;
+};
+
+template <> struct MappingTraits<MachineJumpTable::Entry> {
+  static void mapping(IO &YamlIO, MachineJumpTable::Entry &Entry) {
+    YamlIO.mapRequired("id", Entry.ID);
+    YamlIO.mapOptional("blocks", Entry.Blocks);
+  }
+};
+
+} // end namespace yaml
+} // end namespace llvm
+
+LLVM_YAML_IS_SEQUENCE_VECTOR(llvm::yaml::MachineFunctionLiveIn)
+LLVM_YAML_IS_SEQUENCE_VECTOR(llvm::yaml::VirtualRegisterDefinition)
+LLVM_YAML_IS_SEQUENCE_VECTOR(llvm::yaml::MachineBasicBlock)
+LLVM_YAML_IS_SEQUENCE_VECTOR(llvm::yaml::MachineStackObject)
+LLVM_YAML_IS_SEQUENCE_VECTOR(llvm::yaml::FixedMachineStackObject)
+LLVM_YAML_IS_SEQUENCE_VECTOR(llvm::yaml::MachineConstantPoolValue)
+LLVM_YAML_IS_SEQUENCE_VECTOR(llvm::yaml::MachineJumpTable::Entry)
+
+namespace llvm {
+namespace yaml {
+
+template <> struct MappingTraits<MachineJumpTable> {
+  static void mapping(IO &YamlIO, MachineJumpTable &JT) {
+    YamlIO.mapRequired("kind", JT.Kind);
+    YamlIO.mapOptional("entries", JT.Entries);
+  }
+};
+
+/// Serializable representation of MachineFrameInfo.
+///
+/// Doesn't serialize attributes like 'StackAlignment', 'IsStackRealignable' and
+/// 'RealignOption' as they are determined by the target and LLVM function
+/// attributes.
+/// It also doesn't serialize attributes like 'NumFixedObject' and
+/// 'HasVarSizedObjects' as they are determined by the frame objects themselves.
+struct MachineFrameInfo {
+  bool IsFrameAddressTaken = false;
+  bool IsReturnAddressTaken = false;
+  bool HasStackMap = false;
+  bool HasPatchPoint = false;
+  uint64_t StackSize = 0;
+  int OffsetAdjustment = 0;
+  unsigned MaxAlignment = 0;
+  bool AdjustsStack = false;
+  bool HasCalls = false;
+  // TODO: Serialize StackProtectorIdx and FunctionContextIdx
+  unsigned MaxCallFrameSize = 0;
+  // TODO: Serialize local frame objects.
+  bool HasOpaqueSPAdjustment = false;
+  bool HasVAStart = false;
+  bool HasMustTailInVarArgFunc = false;
+  StringValue SavePoint;
+  StringValue RestorePoint;
+};
+
+template <> struct MappingTraits<MachineFrameInfo> {
+  static void mapping(IO &YamlIO, MachineFrameInfo &MFI) {
+    YamlIO.mapOptional("isFrameAddressTaken", MFI.IsFrameAddressTaken);
+    YamlIO.mapOptional("isReturnAddressTaken", MFI.IsReturnAddressTaken);
+    YamlIO.mapOptional("hasStackMap", MFI.HasStackMap);
+    YamlIO.mapOptional("hasPatchPoint", MFI.HasPatchPoint);
+    YamlIO.mapOptional("stackSize", MFI.StackSize);
+    YamlIO.mapOptional("offsetAdjustment", MFI.OffsetAdjustment);
+    YamlIO.mapOptional("maxAlignment", MFI.MaxAlignment);
+    YamlIO.mapOptional("adjustsStack", MFI.AdjustsStack);
+    YamlIO.mapOptional("hasCalls", MFI.HasCalls);
+    YamlIO.mapOptional("maxCallFrameSize", MFI.MaxCallFrameSize);
+    YamlIO.mapOptional("hasOpaqueSPAdjustment", MFI.HasOpaqueSPAdjustment);
+    YamlIO.mapOptional("hasVAStart", MFI.HasVAStart);
+    YamlIO.mapOptional("hasMustTailInVarArgFunc", MFI.HasMustTailInVarArgFunc);
+    YamlIO.mapOptional("savePoint", MFI.SavePoint,
+                       StringValue()); // Don't print it out when it's empty.
+    YamlIO.mapOptional("restorePoint", MFI.RestorePoint,
+                       StringValue()); // Don't print it out when it's empty.
+  }
+};
+
+struct MachineFunction {
+  StringRef Name;
+  unsigned Alignment = 0;
+  bool ExposesReturnsTwice = false;
+  bool HasInlineAsm = false;
+  // Register information
+  bool IsSSA = false;
+  bool TracksRegLiveness = false;
+  bool TracksSubRegLiveness = false;
+  std::vector<VirtualRegisterDefinition> VirtualRegisters;
+  std::vector<MachineFunctionLiveIn> LiveIns;
+  // TODO: Serialize the various register masks.
+  // Frame information
+  MachineFrameInfo FrameInfo;
+  std::vector<FixedMachineStackObject> FixedStackObjects;
+  std::vector<MachineStackObject> StackObjects;
+  std::vector<MachineConstantPoolValue> Constants; /// Constant pool.
+  MachineJumpTable JumpTableInfo;
+
+  std::vector<MachineBasicBlock> BasicBlocks;
+};
+
+template <> struct MappingTraits<MachineFunction> {
+  static void mapping(IO &YamlIO, MachineFunction &MF) {
+    YamlIO.mapRequired("name", MF.Name);
+    YamlIO.mapOptional("alignment", MF.Alignment);
+    YamlIO.mapOptional("exposesReturnsTwice", MF.ExposesReturnsTwice);
+    YamlIO.mapOptional("hasInlineAsm", MF.HasInlineAsm);
+    YamlIO.mapOptional("isSSA", MF.IsSSA);
+    YamlIO.mapOptional("tracksRegLiveness", MF.TracksRegLiveness);
+    YamlIO.mapOptional("tracksSubRegLiveness", MF.TracksSubRegLiveness);
+    YamlIO.mapOptional("registers", MF.VirtualRegisters);
+    YamlIO.mapOptional("liveins", MF.LiveIns);
+    YamlIO.mapOptional("frameInfo", MF.FrameInfo);
+    YamlIO.mapOptional("fixedStack", MF.FixedStackObjects);
+    YamlIO.mapOptional("stack", MF.StackObjects);
+    YamlIO.mapOptional("constants", MF.Constants);
+    if (!YamlIO.outputting() || !MF.JumpTableInfo.Entries.empty())
+      YamlIO.mapOptional("jumpTable", MF.JumpTableInfo);
+    YamlIO.mapOptional("body", MF.BasicBlocks);
+  }
+};
+
+} // end namespace yaml
+} // end namespace llvm
+
+#endif
diff --git a/include/llvm/CodeGen/MachineBasicBlock.h b/include/llvm/CodeGen/MachineBasicBlock.h
index 1440b967aea..aba64ff9a1a 100644
--- a/include/llvm/CodeGen/MachineBasicBlock.h
+++ b/include/llvm/CodeGen/MachineBasicBlock.h
@@ -25,6 +25,7 @@ class Pass;
 class BasicBlock;
 class MachineFunction;
 class MCSymbol;
+class MIRPrinter;
 class SlotIndexes;
 class StringRef;
 class raw_ostream;
@@ -315,9 +316,18 @@ class MachineBasicBlock : public ilist_node<MachineBasicBlock> {
 
   // LiveIn management methods.
 
-  /// addLiveIn - Add the specified register as a live in.  Note that it
-  /// is an error to add the same register to the same set more than once.
-  void addLiveIn(unsigned Reg)  { LiveIns.push_back(Reg); }
+  /// Adds the specified register as a live in. Note that it is an error to add
+  /// the same register to the same set more than once unless the intention is
+  /// to call sortUniqueLiveIns after all registers are added.
+  void addLiveIn(unsigned Reg) { LiveIns.push_back(Reg); }
+
+  /// Sorts and uniques the LiveIns vector. It can be significantly faster to do
+  /// this than repeatedly calling isLiveIn before calling addLiveIn for every
+  /// LiveIn insertion.
+  void sortUniqueLiveIns() {
+    std::sort(LiveIns.begin(), LiveIns.end());
+    LiveIns.erase(std::unique(LiveIns.begin(), LiveIns.end()), LiveIns.end());
+  }
 
   /// Add PhysReg as live in to this block, and ensure that there is a copy of
   /// PhysReg to a virtual register of class RC. Return the virtual register
@@ -415,6 +425,9 @@ class MachineBasicBlock : public ilist_node<MachineBasicBlock> {
   /// which refer to fromMBB to refer to this.
   void transferSuccessorsAndUpdatePHIs(MachineBasicBlock *fromMBB);
 
+  /// Return true if any of the successors have weights attached to them.
+  bool hasSuccessorWeights() const { return !Weights.empty(); }
+
   /// isPredecessor - Return true if the specified MBB is a predecessor of this
   /// block.
   bool isPredecessor(const MachineBasicBlock *MBB) const;
@@ -452,16 +465,27 @@ class MachineBasicBlock : public ilist_node<MachineBasicBlock> {
   /// instruction of this basic block. If a terminator does not exist,
   /// it returns end()
   iterator getFirstTerminator();
-  const_iterator getFirstTerminator() const;
+  const_iterator getFirstTerminator() const {
+    return const_cast<MachineBasicBlock *>(this)->getFirstTerminator();
+  }
 
   /// getFirstInstrTerminator - Same getFirstTerminator but it ignores bundles
   /// and return an instr_iterator instead.
   instr_iterator getFirstInstrTerminator();
 
+  /// getFirstNonDebugInstr - returns an iterator to the first non-debug
+  /// instruction in the basic block, or end()
+  iterator getFirstNonDebugInstr();
+  const_iterator getFirstNonDebugInstr() const {
+    return const_cast<MachineBasicBlock *>(this)->getFirstNonDebugInstr();
+  }
+
   /// getLastNonDebugInstr - returns an iterator to the last non-debug
   /// instruction in the basic block, or end()
   iterator getLastNonDebugInstr();
-  const_iterator getLastNonDebugInstr() const;
+  const_iterator getLastNonDebugInstr() const {
+    return const_cast<MachineBasicBlock *>(this)->getLastNonDebugInstr();
+  }
 
   /// SplitCriticalEdge - Split the critical edge from this block to the
   /// given successor block, and return the newly created block, or null
@@ -624,21 +648,24 @@ class MachineBasicBlock : public ilist_node<MachineBasicBlock> {
                          ///< neighborhood.
   };
 
-  /// computeRegisterLiveness - Return whether (physical) register \c Reg
-  /// has been <def>ined and not <kill>ed as of just before \c MI.
+  /// Return whether (physical) register \p Reg has been <def>ined and not
+  /// <kill>ed as of just before \p Before.
   ///
-  /// Search is localised to a neighborhood of
-  /// \c Neighborhood instructions before (searching for defs or kills) and
-  /// Neighborhood instructions after (searching just for defs) MI.
+  /// Search is localised to a neighborhood of \p Neighborhood instructions
+  /// before (searching for defs or kills) and \p Neighborhood instructions
+  /// after (searching just for defs) \p Before.
   ///
-  /// \c Reg must be a physical register.
+  /// \p Reg must be a physical register.
   LivenessQueryResult computeRegisterLiveness(const TargetRegisterInfo *TRI,
-                                              unsigned Reg, MachineInstr *MI,
-                                              unsigned Neighborhood=10);
+                                              unsigned Reg,
+                                              const_iterator Before,
+                                              unsigned Neighborhood=10) const;
 
   // Debugging methods.
   void dump() const;
   void print(raw_ostream &OS, SlotIndexes* = nullptr) const;
+  void print(raw_ostream &OS, ModuleSlotTracker &MST,
+             SlotIndexes * = nullptr) const;
 
   // Printing method used by LoopInfo.
   void printAsOperand(raw_ostream &OS, bool PrintType = true) const;
@@ -662,6 +689,7 @@ class MachineBasicBlock : public ilist_node<MachineBasicBlock> {
   const_weight_iterator getWeightIterator(const_succ_iterator I) const;
 
   friend class MachineBranchProbabilityInfo;
+  friend class MIRPrinter;
 
   /// getSuccWeight - Return weight of the edge from this block to MBB. This
   /// method should NOT be called directly, but by using getEdgeWeight method
diff --git a/include/llvm/CodeGen/MachineConstantPool.h b/include/llvm/CodeGen/MachineConstantPool.h
index 20fb181b5c5..628400322f6 100644
--- a/include/llvm/CodeGen/MachineConstantPool.h
+++ b/include/llvm/CodeGen/MachineConstantPool.h
@@ -58,17 +58,6 @@ class MachineConstantPoolValue {
 
   virtual void addSelectionDAGCSEId(FoldingSetNodeID &ID) = 0;
 
-  // @LOCALMOD-START
-  /// getJumpTableIndex - Check if this is a reference to a jump table.
-  /// If so, return a pointer to the jump table index value that is stored
-  /// in the constant pool, else return 0.
-  /// The default behavior is to indicate that the value is not a jump table
-  /// index. This is used by BranchFolder::runOnMachineFunction() and only in
-  /// conjunction with ARM targets
-  /// TODO: this should be cleaned up as it does tripple duty: tester, setter, getter
-  virtual unsigned *getJumpTableIndex() { return 0; }
-  // @LOCALMOD-END
-
   /// print - Implement operator<<
   virtual void print(raw_ostream &O) const = 0;
 };
@@ -146,17 +135,18 @@ class MachineConstantPoolEntry {
 /// address of the function constant pool values.
 /// @brief The machine constant pool.
 class MachineConstantPool {
-  const TargetMachine &TM;      ///< The target machine.
   unsigned PoolAlignment;       ///< The alignment for the pool.
   std::vector<MachineConstantPoolEntry> Constants; ///< The pool of constants.
   /// MachineConstantPoolValues that use an existing MachineConstantPoolEntry.
   DenseSet<MachineConstantPoolValue*> MachineCPVsSharingEntries;
+  const DataLayout &DL;
+
+  const DataLayout &getDataLayout() const { return DL; }
 
-  const DataLayout *getDataLayout() const;
 public:
   /// @brief The only constructor.
-  explicit MachineConstantPool(const TargetMachine &TM)
-    : TM(TM), PoolAlignment(1) {}
+  explicit MachineConstantPool(const DataLayout &DL)
+      : PoolAlignment(1), DL(DL) {}
   ~MachineConstantPool();
     
   /// getConstantPoolAlignment - Return the alignment required by
diff --git a/include/llvm/CodeGen/MachineDominators.h b/include/llvm/CodeGen/MachineDominators.h
index 4428fa618fb..0ac8a5d0e2a 100644
--- a/include/llvm/CodeGen/MachineDominators.h
+++ b/include/llvm/CodeGen/MachineDominators.h
@@ -29,8 +29,8 @@ inline void DominatorTreeBase<MachineBasicBlock>::addRoot(MachineBasicBlock* MBB
   this->Roots.push_back(MBB);
 }
 
-EXTERN_TEMPLATE_INSTANTIATION(class DomTreeNodeBase<MachineBasicBlock>);
-EXTERN_TEMPLATE_INSTANTIATION(class DominatorTreeBase<MachineBasicBlock>);
+extern template class DomTreeNodeBase<MachineBasicBlock>;
+extern template class DominatorTreeBase<MachineBasicBlock>;
 
 typedef DomTreeNodeBase<MachineBasicBlock> MachineDomTreeNode;
 
@@ -263,6 +263,21 @@ template <> struct GraphTraits<MachineDomTreeNode *> {
   }
 };
 
+template <> struct GraphTraits<const MachineDomTreeNode *> {
+  typedef const MachineDomTreeNode NodeType;
+  typedef NodeType::const_iterator ChildIteratorType;
+
+  static NodeType *getEntryNode(NodeType *N) {
+    return N;
+  }
+  static inline ChildIteratorType child_begin(NodeType* N) {
+    return N->begin();
+  }
+  static inline ChildIteratorType child_end(NodeType* N) {
+    return N->end();
+  }
+};
+
 template <> struct GraphTraits<MachineDominatorTree*>
   : public GraphTraits<MachineDomTreeNode *> {
   static NodeType *getEntryNode(MachineDominatorTree *DT) {
diff --git a/include/llvm/CodeGen/MachineFrameInfo.h b/include/llvm/CodeGen/MachineFrameInfo.h
index 1e7fee666fc..943ea5d6728 100644
--- a/include/llvm/CodeGen/MachineFrameInfo.h
+++ b/include/llvm/CodeGen/MachineFrameInfo.h
@@ -79,9 +79,9 @@ class CalleeSavedInfo {
 /// @brief Abstract Stack Frame Information
 class MachineFrameInfo {
 
-  // StackObject - Represent a single object allocated on the stack.
+  // Represent a single object allocated on the stack.
   struct StackObject {
-    // SPOffset - The offset of this object from the stack pointer on entry to
+    // The offset of this object from the stack pointer on entry to
     // the function.  This field has no meaning for a variable sized element.
     int64_t SPOffset;
 
@@ -89,23 +89,23 @@ class MachineFrameInfo {
     // ~0ULL means a dead object.
     uint64_t Size;
 
-    // Alignment - The required alignment of this stack slot.
+    // The required alignment of this stack slot.
     unsigned Alignment;
 
-    // isImmutable - If true, the value of the stack object is set before
+    // If true, the value of the stack object is set before
     // entering the function and is not modified inside the function. By
     // default, fixed objects are immutable unless marked otherwise.
     bool isImmutable;
 
-    // isSpillSlot - If true the stack object is used as spill slot. It
+    // If true the stack object is used as spill slot. It
     // cannot alias any other memory objects.
     bool isSpillSlot;
 
-    /// Alloca - If this stack object is originated from an Alloca instruction
+    /// If this stack object is originated from an Alloca instruction
     /// this value saves the original IR allocation. Can be NULL.
     const AllocaInst *Alloca;
 
-    // PreAllocated - If true, the object was mapped into the local frame
+    // If true, the object was mapped into the local frame
     // block and doesn't need additional handling for allocation beyond that.
     bool PreAllocated;
 
@@ -121,51 +121,47 @@ class MachineFrameInfo {
         isSpillSlot(isSS), Alloca(Val), PreAllocated(false), isAliased(A) {}
   };
 
-  /// StackAlignment - The alignment of the stack.
+  /// The alignment of the stack.
   unsigned StackAlignment;
 
-  /// StackRealignable - Can the stack be realigned.
+  /// Can the stack be realigned.
   bool StackRealignable;
 
-  /// Objects - The list of stack objects allocated...
-  ///
+  /// The list of stack objects allocated.
   std::vector<StackObject> Objects;
 
-  /// NumFixedObjects - This contains the number of fixed objects contained on
+  /// This contains the number of fixed objects contained on
   /// the stack.  Because fixed objects are stored at a negative index in the
   /// Objects list, this is also the index to the 0th object in the list.
-  ///
   unsigned NumFixedObjects;
 
-  /// HasVarSizedObjects - This boolean keeps track of whether any variable
+  /// This boolean keeps track of whether any variable
   /// sized objects have been allocated yet.
-  ///
   bool HasVarSizedObjects;
 
-  /// FrameAddressTaken - This boolean keeps track of whether there is a call
+  /// This boolean keeps track of whether there is a call
   /// to builtin \@llvm.frameaddress.
   bool FrameAddressTaken;
 
-  /// ReturnAddressTaken - This boolean keeps track of whether there is a call
+  /// This boolean keeps track of whether there is a call
   /// to builtin \@llvm.returnaddress.
   bool ReturnAddressTaken;
 
-  /// HasStackMap - This boolean keeps track of whether there is a call
+  /// This boolean keeps track of whether there is a call
   /// to builtin \@llvm.experimental.stackmap.
   bool HasStackMap;
 
-  /// HasPatchPoint - This boolean keeps track of whether there is a call
+  /// This boolean keeps track of whether there is a call
   /// to builtin \@llvm.experimental.patchpoint.
   bool HasPatchPoint;
 
-  /// StackSize - The prolog/epilog code inserter calculates the final stack
+  /// The prolog/epilog code inserter calculates the final stack
   /// offsets for all of the fixed size objects, updating the Objects list
   /// above.  It then updates StackSize to contain the number of bytes that need
   /// to be allocated on entry to the function.
-  ///
   uint64_t StackSize;
 
-  /// OffsetAdjustment - The amount that a frame offset needs to be adjusted to
+  /// The amount that a frame offset needs to be adjusted to
   /// have the actual offset from the stack/frame pointer.  The exact usage of
   /// this is target-dependent, but it is typically used to adjust between
   /// SP-relative and FP-relative offsets.  E.G., if objects are accessed via
@@ -176,52 +172,49 @@ class MachineFrameInfo {
   /// corresponding adjustments are performed directly.
   int OffsetAdjustment;
 
-  /// MaxAlignment - The prolog/epilog code inserter may process objects
-  /// that require greater alignment than the default alignment the target
-  /// provides. To handle this, MaxAlignment is set to the maximum alignment
+  /// The prolog/epilog code inserter may process objects that require greater
+  /// alignment than the default alignment the target provides.
+  /// To handle this, MaxAlignment is set to the maximum alignment
   /// needed by the objects on the current frame.  If this is greater than the
   /// native alignment maintained by the compiler, dynamic alignment code will
   /// be needed.
   ///
   unsigned MaxAlignment;
 
-  /// AdjustsStack - Set to true if this function adjusts the stack -- e.g.,
+  /// Set to true if this function adjusts the stack -- e.g.,
   /// when calling another function. This is only valid during and after
   /// prolog/epilog code insertion.
   bool AdjustsStack;
 
-  /// HasCalls - Set to true if this function has any function calls.
+  /// Set to true if this function has any function calls.
   bool HasCalls;
 
-  /// StackProtectorIdx - The frame index for the stack protector.
+  /// The frame index for the stack protector.
   int StackProtectorIdx;
 
-  /// FunctionContextIdx - The frame index for the function context. Used for
-  /// SjLj exceptions.
+  /// The frame index for the function context. Used for SjLj exceptions.
   int FunctionContextIdx;
 
-  /// MaxCallFrameSize - This contains the size of the largest call frame if the
-  /// target uses frame setup/destroy pseudo instructions (as defined in the
-  /// TargetFrameInfo class).  This information is important for frame pointer
-  /// elimination.  If is only valid during and after prolog/epilog code
-  /// insertion.
-  ///
+  /// This contains the size of the largest call frame if the target uses frame
+  /// setup/destroy pseudo instructions (as defined in the TargetFrameInfo
+  /// class).  This information is important for frame pointer elimination.
+  /// If is only valid during and after prolog/epilog code insertion.
   unsigned MaxCallFrameSize;
 
-  /// CSInfo - The prolog/epilog code inserter fills in this vector with each
+  /// The prolog/epilog code inserter fills in this vector with each
   /// callee saved register saved in the frame.  Beyond its use by the prolog/
   /// epilog code inserter, this data used for debug info and exception
   /// handling.
   std::vector<CalleeSavedInfo> CSInfo;
 
-  /// CSIValid - Has CSInfo been set yet?
+  /// Has CSInfo been set yet?
   bool CSIValid;
 
-  /// LocalFrameObjects - References to frame indices which are mapped
+  /// References to frame indices which are mapped
   /// into the local frame allocation block. <FrameIdx, LocalOffset>
   SmallVector<std::pair<int, int64_t>, 32> LocalFrameObjects;
 
-  /// LocalFrameSize - Size of the pre-allocated local frame block.
+  /// Size of the pre-allocated local frame block.
   int64_t LocalFrameSize;
 
   /// Required alignment of the local object blob, which is the strictest
@@ -236,9 +229,9 @@ class MachineFrameInfo {
   /// Whether the "realign-stack" option is on.
   bool RealignOption;
 
-  /// True if the function includes inline assembly that adjusts the stack
-  /// pointer.
-  bool HasInlineAsmWithSPAdjust;
+  /// True if the function dynamically adjusts the stack pointer through some
+  /// opaque mechanism like inline assembly or Win32 EH.
+  bool HasOpaqueSPAdjustment;
 
   /// True if the function contains a call to the llvm.vastart intrinsic.
   bool HasVAStart;
@@ -246,6 +239,16 @@ class MachineFrameInfo {
   /// True if this is a varargs function that contains a musttail call.
   bool HasMustTailInVarArgFunc;
 
+  /// True if this function contains a tail call. If so immutable objects like
+  /// function arguments are no longer so. A tail call *can* override fixed
+  /// stack objects like arguments so we can't treat them as immutable.
+  bool HasTailCall;
+
+  /// Not null, if shrink-wrapping found a better place for the prologue.
+  MachineBasicBlock *Save;
+  /// Not null, if shrink-wrapping found a better place for the epilogue.
+  MachineBasicBlock *Restore;
+
 public:
   explicit MachineFrameInfo(unsigned StackAlign, bool isStackRealign,
                             bool RealignOpt)
@@ -266,107 +269,101 @@ class MachineFrameInfo {
     LocalFrameSize = 0;
     LocalFrameMaxAlign = 0;
     UseLocalStackAllocationBlock = false;
-    HasInlineAsmWithSPAdjust = false;
+    HasOpaqueSPAdjustment = false;
     HasVAStart = false;
     HasMustTailInVarArgFunc = false;
+    Save = nullptr;
+    Restore = nullptr;
+    HasTailCall = false;
   }
 
-  /// hasStackObjects - Return true if there are any stack objects in this
-  /// function.
-  ///
+  /// Return true if there are any stack objects in this function.
   bool hasStackObjects() const { return !Objects.empty(); }
 
-  /// hasVarSizedObjects - This method may be called any time after instruction
+  /// This method may be called any time after instruction
   /// selection is complete to determine if the stack frame for this function
   /// contains any variable sized objects.
-  ///
   bool hasVarSizedObjects() const { return HasVarSizedObjects; }
 
-  /// getStackProtectorIndex/setStackProtectorIndex - Return the index for the
-  /// stack protector object.
-  ///
+  /// Return the index for the stack protector object.
   int getStackProtectorIndex() const { return StackProtectorIdx; }
   void setStackProtectorIndex(int I) { StackProtectorIdx = I; }
 
-  /// getFunctionContextIndex/setFunctionContextIndex - Return the index for the
-  /// function context object. This object is used for SjLj exceptions.
+  /// Return the index for the function context object.
+  /// This object is used for SjLj exceptions.
   int getFunctionContextIndex() const { return FunctionContextIdx; }
   void setFunctionContextIndex(int I) { FunctionContextIdx = I; }
 
-  /// isFrameAddressTaken - This method may be called any time after instruction
+  /// This method may be called any time after instruction
   /// selection is complete to determine if there is a call to
   /// \@llvm.frameaddress in this function.
   bool isFrameAddressTaken() const { return FrameAddressTaken; }
   void setFrameAddressIsTaken(bool T) { FrameAddressTaken = T; }
 
-  /// isReturnAddressTaken - This method may be called any time after
+  /// This method may be called any time after
   /// instruction selection is complete to determine if there is a call to
   /// \@llvm.returnaddress in this function.
   bool isReturnAddressTaken() const { return ReturnAddressTaken; }
   void setReturnAddressIsTaken(bool s) { ReturnAddressTaken = s; }
 
-  /// hasStackMap - This method may be called any time after instruction
+  /// This method may be called any time after instruction
   /// selection is complete to determine if there is a call to builtin
   /// \@llvm.experimental.stackmap.
   bool hasStackMap() const { return HasStackMap; }
   void setHasStackMap(bool s = true) { HasStackMap = s; }
 
-  /// hasPatchPoint - This method may be called any time after instruction
+  /// This method may be called any time after instruction
   /// selection is complete to determine if there is a call to builtin
   /// \@llvm.experimental.patchpoint.
   bool hasPatchPoint() const { return HasPatchPoint; }
   void setHasPatchPoint(bool s = true) { HasPatchPoint = s; }
 
-  /// getObjectIndexBegin - Return the minimum frame object index.
-  ///
+  /// Return the minimum frame object index.
   int getObjectIndexBegin() const { return -NumFixedObjects; }
 
-  /// getObjectIndexEnd - Return one past the maximum frame object index.
-  ///
+  /// Return one past the maximum frame object index.
   int getObjectIndexEnd() const { return (int)Objects.size()-NumFixedObjects; }
 
-  /// getNumFixedObjects - Return the number of fixed objects.
+  /// Return the number of fixed objects.
   unsigned getNumFixedObjects() const { return NumFixedObjects; }
 
-  /// getNumObjects - Return the number of objects.
-  ///
+  /// Return the number of objects.
   unsigned getNumObjects() const { return Objects.size(); }
 
-  /// mapLocalFrameObject - Map a frame index into the local object block
+  /// Map a frame index into the local object block
   void mapLocalFrameObject(int ObjectIndex, int64_t Offset) {
     LocalFrameObjects.push_back(std::pair<int, int64_t>(ObjectIndex, Offset));
     Objects[ObjectIndex + NumFixedObjects].PreAllocated = true;
   }
 
-  /// getLocalFrameObjectMap - Get the local offset mapping for a for an object
-  std::pair<int, int64_t> getLocalFrameObjectMap(int i) {
+  /// Get the local offset mapping for a for an object.
+  std::pair<int, int64_t> getLocalFrameObjectMap(int i) const {
     assert (i >= 0 && (unsigned)i < LocalFrameObjects.size() &&
             "Invalid local object reference!");
     return LocalFrameObjects[i];
   }
 
-  /// getLocalFrameObjectCount - Return the number of objects allocated into
-  /// the local object block.
-  int64_t getLocalFrameObjectCount() { return LocalFrameObjects.size(); }
+  /// Return the number of objects allocated into the local object block.
+  int64_t getLocalFrameObjectCount() const { return LocalFrameObjects.size(); }
 
-  /// setLocalFrameSize - Set the size of the local object blob.
+  /// Set the size of the local object blob.
   void setLocalFrameSize(int64_t sz) { LocalFrameSize = sz; }
 
-  /// getLocalFrameSize - Get the size of the local object blob.
+  /// Get the size of the local object blob.
   int64_t getLocalFrameSize() const { return LocalFrameSize; }
 
-  /// setLocalFrameMaxAlign - Required alignment of the local object blob,
+  /// Required alignment of the local object blob,
   /// which is the strictest alignment of any object in it.
   void setLocalFrameMaxAlign(unsigned Align) { LocalFrameMaxAlign = Align; }
 
-  /// getLocalFrameMaxAlign - Return the required alignment of the local
-  /// object blob.
+  /// Return the required alignment of the local object blob.
   unsigned getLocalFrameMaxAlign() const { return LocalFrameMaxAlign; }
 
-  /// getUseLocalStackAllocationBlock - Get whether the local allocation blob
-  /// should be allocated together or let PEI allocate the locals in it
-  /// directly.
-  bool getUseLocalStackAllocationBlock() {return UseLocalStackAllocationBlock;}
+  /// Get whether the local allocation blob should be allocated together or
+  /// let PEI allocate the locals in it directly.
+  bool getUseLocalStackAllocationBlock() const {
+    return UseLocalStackAllocationBlock;
+  }
 
   /// setUseLocalStackAllocationBlock - Set whether the local allocation blob
   /// should be allocated together or let PEI allocate the locals in it
@@ -375,30 +372,28 @@ class MachineFrameInfo {
     UseLocalStackAllocationBlock = v;
   }
 
-  /// isObjectPreAllocated - Return true if the object was pre-allocated into
-  /// the local block.
+  /// Return true if the object was pre-allocated into the local block.
   bool isObjectPreAllocated(int ObjectIdx) const {
     assert(unsigned(ObjectIdx+NumFixedObjects) < Objects.size() &&
            "Invalid Object Idx!");
     return Objects[ObjectIdx+NumFixedObjects].PreAllocated;
   }
 
-  /// getObjectSize - Return the size of the specified object.
-  ///
+  /// Return the size of the specified object.
   int64_t getObjectSize(int ObjectIdx) const {
     assert(unsigned(ObjectIdx+NumFixedObjects) < Objects.size() &&
            "Invalid Object Idx!");
     return Objects[ObjectIdx+NumFixedObjects].Size;
   }
 
-  /// setObjectSize - Change the size of the specified stack object.
+  /// Change the size of the specified stack object.
   void setObjectSize(int ObjectIdx, int64_t Size) {
     assert(unsigned(ObjectIdx+NumFixedObjects) < Objects.size() &&
            "Invalid Object Idx!");
     Objects[ObjectIdx+NumFixedObjects].Size = Size;
   }
 
-  /// getObjectAlignment - Return the alignment of the specified stack object.
+  /// Return the alignment of the specified stack object.
   unsigned getObjectAlignment(int ObjectIdx) const {
     assert(unsigned(ObjectIdx+NumFixedObjects) < Objects.size() &&
            "Invalid Object Idx!");
@@ -413,7 +408,7 @@ class MachineFrameInfo {
     ensureMaxAlignment(Align);
   }
 
-  /// getObjectAllocation - Return the underlying Alloca of the specified
+  /// Return the underlying Alloca of the specified
   /// stack object if it exists. Returns 0 if none exists.
   const AllocaInst* getObjectAllocation(int ObjectIdx) const {
     assert(unsigned(ObjectIdx+NumFixedObjects) < Objects.size() &&
@@ -421,9 +416,8 @@ class MachineFrameInfo {
     return Objects[ObjectIdx+NumFixedObjects].Alloca;
   }
 
-  /// getObjectOffset - Return the assigned stack offset of the specified object
+  /// Return the assigned stack offset of the specified object
   /// from the incoming stack pointer.
-  ///
   int64_t getObjectOffset(int ObjectIdx) const {
     assert(unsigned(ObjectIdx+NumFixedObjects) < Objects.size() &&
            "Invalid Object Idx!");
@@ -432,9 +426,8 @@ class MachineFrameInfo {
     return Objects[ObjectIdx+NumFixedObjects].SPOffset;
   }
 
-  /// setObjectOffset - Set the stack frame offset of the specified object.  The
+  /// Set the stack frame offset of the specified object. The
   /// offset is relative to the stack pointer on entry to the function.
-  ///
   void setObjectOffset(int ObjectIdx, int64_t SPOffset) {
     assert(unsigned(ObjectIdx+NumFixedObjects) < Objects.size() &&
            "Invalid Object Idx!");
@@ -443,50 +436,43 @@ class MachineFrameInfo {
     Objects[ObjectIdx+NumFixedObjects].SPOffset = SPOffset;
   }
 
-  /// getStackSize - Return the number of bytes that must be allocated to hold
+  /// Return the number of bytes that must be allocated to hold
   /// all of the fixed size frame objects.  This is only valid after
   /// Prolog/Epilog code insertion has finalized the stack frame layout.
-  ///
   uint64_t getStackSize() const { return StackSize; }
 
-  /// setStackSize - Set the size of the stack...
-  ///
+  /// Set the size of the stack.
   void setStackSize(uint64_t Size) { StackSize = Size; }
 
   /// Estimate and return the size of the stack frame.
   unsigned estimateStackSize(const MachineFunction &MF) const;
 
-  /// getOffsetAdjustment - Return the correction for frame offsets.
-  ///
+  /// Return the correction for frame offsets.
   int getOffsetAdjustment() const { return OffsetAdjustment; }
 
-  /// setOffsetAdjustment - Set the correction for frame offsets.
-  ///
+  /// Set the correction for frame offsets.
   void setOffsetAdjustment(int Adj) { OffsetAdjustment = Adj; }
 
-  /// getMaxAlignment - Return the alignment in bytes that this function must be
-  /// aligned to, which is greater than the default stack alignment provided by
-  /// the target.
-  ///
+  /// Return the alignment in bytes that this function must be aligned to,
+  /// which is greater than the default stack alignment provided by the target.
   unsigned getMaxAlignment() const { return MaxAlignment; }
 
-  /// ensureMaxAlignment - Make sure the function is at least Align bytes
-  /// aligned.
+  /// Make sure the function is at least Align bytes aligned.
   void ensureMaxAlignment(unsigned Align);
 
-  /// AdjustsStack - Return true if this function adjusts the stack -- e.g.,
+  /// Return true if this function adjusts the stack -- e.g.,
   /// when calling another function. This is only valid during and after
   /// prolog/epilog code insertion.
   bool adjustsStack() const { return AdjustsStack; }
   void setAdjustsStack(bool V) { AdjustsStack = V; }
 
-  /// hasCalls - Return true if the current function has any function calls.
+  /// Return true if the current function has any function calls.
   bool hasCalls() const { return HasCalls; }
   void setHasCalls(bool V) { HasCalls = V; }
 
-  /// Returns true if the function contains any stack-adjusting inline assembly.
-  bool hasInlineAsmWithSPAdjust() const { return HasInlineAsmWithSPAdjust; }
-  void setHasInlineAsmWithSPAdjust(bool B) { HasInlineAsmWithSPAdjust = B; }
+  /// Returns true if the function contains opaque dynamic stack adjustments.
+  bool hasOpaqueSPAdjustment() const { return HasOpaqueSPAdjustment; }
+  void setHasOpaqueSPAdjustment(bool B) { HasOpaqueSPAdjustment = B; }
 
   /// Returns true if the function calls the llvm.va_start intrinsic.
   bool hasVAStart() const { return HasVAStart; }
@@ -496,7 +482,11 @@ class MachineFrameInfo {
   bool hasMustTailInVarArgFunc() const { return HasMustTailInVarArgFunc; }
   void setHasMustTailInVarArgFunc(bool B) { HasMustTailInVarArgFunc = B; }
 
-  /// getMaxCallFrameSize - Return the maximum size of a call frame that must be
+  /// Returns true if the function contains a tail call.
+  bool hasTailCall() const { return HasTailCall; }
+  void setHasTailCall() { HasTailCall = true; }
+
+  /// Return the maximum size of a call frame that must be
   /// allocated for an outgoing function call.  This is only available if
   /// CallFrameSetup/Destroy pseudo instructions are used by the target, and
   /// then only during or after prolog/epilog code insertion.
@@ -504,25 +494,23 @@ class MachineFrameInfo {
   unsigned getMaxCallFrameSize() const { return MaxCallFrameSize; }
   void setMaxCallFrameSize(unsigned S) { MaxCallFrameSize = S; }
 
-  /// CreateFixedObject - Create a new object at a fixed location on the stack.
+  /// Create a new object at a fixed location on the stack.
   /// All fixed objects should be created before other objects are created for
   /// efficiency. By default, fixed objects are not pointed to by LLVM IR
   /// values. This returns an index with a negative value.
-  ///
   int CreateFixedObject(uint64_t Size, int64_t SPOffset, bool Immutable,
                         bool isAliased = false);
 
-  /// CreateFixedSpillStackObject - Create a spill slot at a fixed location
-  /// on the stack.  Returns an index with a negative value.
+  /// Create a spill slot at a fixed location on the stack.
+  /// Returns an index with a negative value.
   int CreateFixedSpillStackObject(uint64_t Size, int64_t SPOffset);
 
-  /// isFixedObjectIndex - Returns true if the specified index corresponds to a
-  /// fixed stack object.
+  /// Returns true if the specified index corresponds to a fixed stack object.
   bool isFixedObjectIndex(int ObjectIdx) const {
     return ObjectIdx < 0 && (ObjectIdx >= -(int)NumFixedObjects);
   }
 
-  /// isAliasedObjectIndex - Returns true if the specified index corresponds
+  /// Returns true if the specified index corresponds
   /// to an object that might be pointed to by an LLVM IR value.
   bool isAliasedObjectIndex(int ObjectIdx) const {
     assert(unsigned(ObjectIdx+NumFixedObjects) < Objects.size() &&
@@ -533,84 +521,89 @@ class MachineFrameInfo {
   /// isImmutableObjectIndex - Returns true if the specified index corresponds
   /// to an immutable object.
   bool isImmutableObjectIndex(int ObjectIdx) const {
+    // Tail calling functions can clobber their function arguments.
+    if (HasTailCall)
+      return false;
     assert(unsigned(ObjectIdx+NumFixedObjects) < Objects.size() &&
            "Invalid Object Idx!");
     return Objects[ObjectIdx+NumFixedObjects].isImmutable;
   }
 
-  /// isSpillSlotObjectIndex - Returns true if the specified index corresponds
-  /// to a spill slot..
+  /// Returns true if the specified index corresponds to a spill slot.
   bool isSpillSlotObjectIndex(int ObjectIdx) const {
     assert(unsigned(ObjectIdx+NumFixedObjects) < Objects.size() &&
            "Invalid Object Idx!");
     return Objects[ObjectIdx+NumFixedObjects].isSpillSlot;
   }
 
-  /// isDeadObjectIndex - Returns true if the specified index corresponds to
-  /// a dead object.
+  /// Returns true if the specified index corresponds to a dead object.
   bool isDeadObjectIndex(int ObjectIdx) const {
     assert(unsigned(ObjectIdx+NumFixedObjects) < Objects.size() &&
            "Invalid Object Idx!");
     return Objects[ObjectIdx+NumFixedObjects].Size == ~0ULL;
   }
 
-  /// CreateStackObject - Create a new statically sized stack object, returning
+  /// Returns true if the specified index corresponds to a variable sized
+  /// object.
+  bool isVariableSizedObjectIndex(int ObjectIdx) const {
+    assert(unsigned(ObjectIdx + NumFixedObjects) < Objects.size() &&
+           "Invalid Object Idx!");
+    return Objects[ObjectIdx + NumFixedObjects].Size == 0;
+  }
+
+  /// Create a new statically sized stack object, returning
   /// a nonnegative identifier to represent it.
-  ///
   int CreateStackObject(uint64_t Size, unsigned Alignment, bool isSS,
                         const AllocaInst *Alloca = nullptr);
 
-  /// CreateSpillStackObject - Create a new statically sized stack object that
-  /// represents a spill slot, returning a nonnegative identifier to represent
-  /// it.
-  ///
+  /// Create a new statically sized stack object that represents a spill slot,
+  /// returning a nonnegative identifier to represent it.
   int CreateSpillStackObject(uint64_t Size, unsigned Alignment);
 
-  /// RemoveStackObject - Remove or mark dead a statically sized stack object.
-  ///
+  /// Remove or mark dead a statically sized stack object.
   void RemoveStackObject(int ObjectIdx) {
     // Mark it dead.
     Objects[ObjectIdx+NumFixedObjects].Size = ~0ULL;
   }
 
-  /// CreateVariableSizedObject - Notify the MachineFrameInfo object that a
-  /// variable sized object has been created.  This must be created whenever a
-  /// variable sized object is created, whether or not the index returned is
-  /// actually used.
-  ///
+  /// Notify the MachineFrameInfo object that a variable sized object has been
+  /// created.  This must be created whenever a variable sized object is
+  /// created, whether or not the index returned is actually used.
   int CreateVariableSizedObject(unsigned Alignment, const AllocaInst *Alloca);
 
-  /// getCalleeSavedInfo - Returns a reference to call saved info vector for the
-  /// current function.
+  /// Returns a reference to call saved info vector for the current function.
   const std::vector<CalleeSavedInfo> &getCalleeSavedInfo() const {
     return CSInfo;
   }
 
-  /// setCalleeSavedInfo - Used by prolog/epilog inserter to set the function's
-  /// callee saved information.
+  /// Used by prolog/epilog inserter to set the function's callee saved
+  /// information.
   void setCalleeSavedInfo(const std::vector<CalleeSavedInfo> &CSI) {
     CSInfo = CSI;
   }
 
-  /// isCalleeSavedInfoValid - Has the callee saved info been calculated yet?
+  /// Has the callee saved info been calculated yet?
   bool isCalleeSavedInfoValid() const { return CSIValid; }
 
   void setCalleeSavedInfoValid(bool v) { CSIValid = v; }
 
-  /// getPristineRegs - Return a set of physical registers that are pristine on
-  /// entry to the MBB.
+  MachineBasicBlock *getSavePoint() const { return Save; }
+  void setSavePoint(MachineBasicBlock *NewSave) { Save = NewSave; }
+  MachineBasicBlock *getRestorePoint() const { return Restore; }
+  void setRestorePoint(MachineBasicBlock *NewRestore) { Restore = NewRestore; }
+
+  /// Return a set of physical registers that are pristine.
   ///
   /// Pristine registers hold a value that is useless to the current function,
-  /// but that must be preserved - they are callee saved registers that have not
-  /// been saved yet.
+  /// but that must be preserved - they are callee saved registers that are not
+  /// saved.
   ///
   /// Before the PrologueEpilogueInserter has placed the CSR spill code, this
   /// method always returns an empty set.
-  BitVector getPristineRegs(const MachineBasicBlock *MBB) const;
+  BitVector getPristineRegs(const MachineFunction &MF) const;
 
-  /// print - Used by the MachineFunction printer to print information about
-  /// stack objects. Implemented in MachineFunction.cpp
-  ///
+  /// Used by the MachineFunction printer to print information about
+  /// stack objects. Implemented in MachineFunction.cpp.
   void print(const MachineFunction &MF, raw_ostream &OS) const;
 
   /// dump - Print the function to stderr.
diff --git a/include/llvm/CodeGen/MachineFunction.h b/include/llvm/CodeGen/MachineFunction.h
index 94610cabf56..6297defb1c5 100644
--- a/include/llvm/CodeGen/MachineFunction.h
+++ b/include/llvm/CodeGen/MachineFunction.h
@@ -155,6 +155,9 @@ class MachineFunction {
   MachineModuleInfo &getMMI() const { return MMI; }
   MCContext &getContext() const { return Ctx; }
 
+  /// Return the DataLayout attached to the Module associated to this MF.
+  const DataLayout &getDataLayout() const;
+
   /// getFunction - Return the LLVM function that this machine code represents
   ///
   const Function *getFunction() const { return Fn; }
@@ -472,6 +475,9 @@ class MachineFunction {
     extractStoreMemRefs(MachineInstr::mmo_iterator Begin,
                         MachineInstr::mmo_iterator End);
 
+  /// Allocate a string and populate it with the given external symbol name.
+  const char *createExternalSymbolName(StringRef Name);
+
   //===--------------------------------------------------------------------===//
   // Label Manipulation.
   //
diff --git a/include/llvm/CodeGen/MachineFunctionAnalysis.h b/include/llvm/CodeGen/MachineFunctionAnalysis.h
index 023eeb1b4d0..4c0f5e63ea1 100644
--- a/include/llvm/CodeGen/MachineFunctionAnalysis.h
+++ b/include/llvm/CodeGen/MachineFunctionAnalysis.h
@@ -19,6 +19,7 @@
 namespace llvm {
 
 class MachineFunction;
+class MachineFunctionInitializer;
 class TargetMachine;
 
 /// MachineFunctionAnalysis - This class is a Pass that manages a
@@ -28,9 +29,12 @@ struct MachineFunctionAnalysis : public FunctionPass {
   const TargetMachine &TM;
   MachineFunction *MF;
   unsigned NextFnNum;
+  MachineFunctionInitializer *MFInitializer;
+
 public:
   static char ID;
-  explicit MachineFunctionAnalysis(const TargetMachine &tm);
+  explicit MachineFunctionAnalysis(const TargetMachine &tm,
+                                   MachineFunctionInitializer *MFInitializer);
   ~MachineFunctionAnalysis() override;
 
   MachineFunction &getMF() const { return *MF; }
diff --git a/include/llvm/CodeGen/MachineFunctionInitializer.h b/include/llvm/CodeGen/MachineFunctionInitializer.h
new file mode 100644
index 00000000000..ff4c29cc014
--- /dev/null
+++ b/include/llvm/CodeGen/MachineFunctionInitializer.h
@@ -0,0 +1,38 @@
+//===- MachineFunctionInitalizer.h - machine function initializer ---------===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This file declares an interface that allows custom machine function
+// initialization.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_CODEGEN_MACHINEFUNCTIONINITIALIZER_H
+#define LLVM_CODEGEN_MACHINEFUNCTIONINITIALIZER_H
+
+namespace llvm {
+
+class MachineFunction;
+
+/// This interface provides a way to initialize machine functions after they are
+/// created by the machine function analysis pass.
+class MachineFunctionInitializer {
+  virtual void anchor();
+
+public:
+  virtual ~MachineFunctionInitializer() {}
+
+  /// Initialize the machine function.
+  ///
+  /// Return true if error occurred.
+  virtual bool initializeMachineFunction(MachineFunction &MF) = 0;
+};
+
+} // end namespace llvm
+
+#endif
diff --git a/include/llvm/CodeGen/MachineInstr.h b/include/llvm/CodeGen/MachineInstr.h
index 9097150ab62..be0b629a697 100644
--- a/include/llvm/CodeGen/MachineInstr.h
+++ b/include/llvm/CodeGen/MachineInstr.h
@@ -42,7 +42,7 @@ class MachineFunction;
 class MachineMemOperand;
 
 //===----------------------------------------------------------------------===//
-/// MachineInstr - Representation of each machine instruction.
+/// Representation of each machine instruction.
 ///
 /// This class isn't a POD type, but it must have a trivial destructor. When a
 /// MachineFunction is deleted, all the contained MachineInstrs are deallocated
@@ -103,12 +103,12 @@ class MachineInstr : public ilist_node<MachineInstr> {
   friend struct ilist_traits<MachineBasicBlock>;
   void setParent(MachineBasicBlock *P) { Parent = P; }
 
-  /// MachineInstr ctor - This constructor creates a copy of the given
+  /// This constructor creates a copy of the given
   /// MachineInstr in the given MachineFunction.
   MachineInstr(MachineFunction &, const MachineInstr &);
 
-  /// MachineInstr ctor - This constructor create a MachineInstr and add the
-  /// implicit operands.  It reserves space for number of operands specified by
+  /// This constructor create a MachineInstr and add the implicit operands.
+  /// It reserves space for number of operands specified by
   /// MCInstrDesc.  An explicit DebugLoc is supplied.
   MachineInstr(MachineFunction &, const MCInstrDesc &MCID, DebugLoc dl,
                bool NoImp = false);
@@ -120,43 +120,38 @@ class MachineInstr : public ilist_node<MachineInstr> {
   const MachineBasicBlock* getParent() const { return Parent; }
   MachineBasicBlock* getParent() { return Parent; }
 
-  /// getAsmPrinterFlags - Return the asm printer flags bitvector.
-  ///
+  /// Return the asm printer flags bitvector.
   uint8_t getAsmPrinterFlags() const { return AsmPrinterFlags; }
 
-  /// clearAsmPrinterFlags - clear the AsmPrinter bitvector
-  ///
+  /// Clear the AsmPrinter bitvector.
   void clearAsmPrinterFlags() { AsmPrinterFlags = 0; }
 
-  /// getAsmPrinterFlag - Return whether an AsmPrinter flag is set.
-  ///
+  /// Return whether an AsmPrinter flag is set.
   bool getAsmPrinterFlag(CommentFlag Flag) const {
     return AsmPrinterFlags & Flag;
   }
 
-  /// setAsmPrinterFlag - Set a flag for the AsmPrinter.
-  ///
+  /// Set a flag for the AsmPrinter.
   void setAsmPrinterFlag(CommentFlag Flag) {
     AsmPrinterFlags |= (uint8_t)Flag;
   }
 
-  /// clearAsmPrinterFlag - clear specific AsmPrinter flags
-  ///
+  /// Clear specific AsmPrinter flags.
   void clearAsmPrinterFlag(CommentFlag Flag) {
     AsmPrinterFlags &= ~Flag;
   }
 
-  /// getFlags - Return the MI flags bitvector.
+  /// Return the MI flags bitvector.
   uint8_t getFlags() const {
     return Flags;
   }
 
-  /// getFlag - Return whether an MI flag is set.
+  /// Return whether an MI flag is set.
   bool getFlag(MIFlag Flag) const {
     return Flags & Flag;
   }
 
-  /// setFlag - Set a MI flag.
+  /// Set a MI flag.
   void setFlag(MIFlag Flag) {
     Flags |= (uint8_t)Flag;
   }
@@ -172,8 +167,7 @@ class MachineInstr : public ilist_node<MachineInstr> {
     Flags &= ~((uint8_t)Flag);
   }
 
-  /// isInsideBundle - Return true if MI is in a bundle (but not the first MI
-  /// in a bundle).
+  /// Return true if MI is in a bundle (but not the first MI in a bundle).
   ///
   /// A bundle looks like this before it's finalized:
   ///   ----------------
@@ -212,7 +206,7 @@ class MachineInstr : public ilist_node<MachineInstr> {
     return getFlag(BundledPred);
   }
 
-  /// isBundled - Return true if this instruction part of a bundle. This is true
+  /// Return true if this instruction part of a bundle. This is true
   /// if either itself or its following instruction is marked "InsideBundle".
   bool isBundled() const {
     return isBundledWithPred() || isBundledWithSucc();
@@ -240,26 +234,25 @@ class MachineInstr : public ilist_node<MachineInstr> {
   /// Break bundle below this instruction.
   void unbundleFromSucc();
 
-  /// getDebugLoc - Returns the debug location id of this MachineInstr.
-  ///
+  /// Returns the debug location id of this MachineInstr.
   const DebugLoc &getDebugLoc() const { return debugLoc; }
 
-  /// \brief Return the debug variable referenced by
+  /// Return the debug variable referenced by
   /// this DBG_VALUE instruction.
-  DIVariable getDebugVariable() const {
+  const DILocalVariable *getDebugVariable() const {
     assert(isDebugValue() && "not a DBG_VALUE");
-    return cast<MDLocalVariable>(getOperand(2).getMetadata());
+    return cast<DILocalVariable>(getOperand(2).getMetadata());
   }
 
-  /// \brief Return the complex address expression referenced by
+  /// Return the complex address expression referenced by
   /// this DBG_VALUE instruction.
-  DIExpression getDebugExpression() const {
+  const DIExpression *getDebugExpression() const {
     assert(isDebugValue() && "not a DBG_VALUE");
-    return cast<MDExpression>(getOperand(3).getMetadata());
+    return cast<DIExpression>(getOperand(3).getMetadata());
   }
 
-  /// emitError - Emit an error referring to the source location of this
-  /// instruction. This should only be used for inline assembly that is somehow
+  /// Emit an error referring to the source location of this instruction.
+  /// This should only be used for inline assembly that is somehow
   /// impossible to compile. Other errors should have been handled much
   /// earlier.
   ///
@@ -267,13 +260,11 @@ class MachineInstr : public ilist_node<MachineInstr> {
   ///
   void emitError(StringRef Msg) const;
 
-  /// getDesc - Returns the target instruction descriptor of this
-  /// MachineInstr.
+  /// Returns the target instruction descriptor of this MachineInstr.
   const MCInstrDesc &getDesc() const { return *MCID; }
 
-  /// getOpcode - Returns the opcode of this MachineInstr.
-  ///
-  int getOpcode() const { return MCID->Opcode; }
+  /// Returns the opcode of this MachineInstr.
+  unsigned getOpcode() const { return MCID->Opcode; }
 
   /// Access to explicit operands of the instruction.
   ///
@@ -288,8 +279,7 @@ class MachineInstr : public ilist_node<MachineInstr> {
     return Operands[i];
   }
 
-  /// getNumExplicitOperands - Returns the number of non-implicit operands.
-  ///
+  /// Returns the number of non-implicit operands.
   unsigned getNumExplicitOperands() const;
 
   /// iterator/begin/end - Iterate over all operands of a machine instruction.
@@ -324,23 +314,34 @@ class MachineInstr : public ilist_node<MachineInstr> {
     return iterator_range<const_mop_iterator>(explicit_operands().end(),
                                               operands_end());
   }
+  /// Returns a range over all explicit operands that are register definitions.
+  /// Implicit definition are not included!
   iterator_range<mop_iterator> defs() {
     return iterator_range<mop_iterator>(
         operands_begin(), operands_begin() + getDesc().getNumDefs());
   }
+  /// \copydoc defs()
   iterator_range<const_mop_iterator> defs() const {
     return iterator_range<const_mop_iterator>(
         operands_begin(), operands_begin() + getDesc().getNumDefs());
   }
+  /// Returns a range that includes all operands that are register uses.
+  /// This may include unrelated operands which are not register uses.
   iterator_range<mop_iterator> uses() {
     return iterator_range<mop_iterator>(
         operands_begin() + getDesc().getNumDefs(), operands_end());
   }
+  /// \copydoc uses()
   iterator_range<const_mop_iterator> uses() const {
     return iterator_range<const_mop_iterator>(
         operands_begin() + getDesc().getNumDefs(), operands_end());
   }
 
+  /// Returns the number of the operand iterator \p I points to.
+  unsigned getOperandNo(const_mop_iterator I) const {
+    return I - operands_begin();
+  }
+
   /// Access to memory operands of the instruction
   mmo_iterator memoperands_begin() const { return MemRefs; }
   mmo_iterator memoperands_end() const { return MemRefs + NumMemRefs; }
@@ -353,8 +354,7 @@ class MachineInstr : public ilist_node<MachineInstr> {
     return iterator_range<mmo_iterator>(memoperands_begin(), memoperands_end());
   }
 
-  /// hasOneMemOperand - Return true if this instruction has exactly one
-  /// MachineMemOperand.
+  /// Return true if this instruction has exactly one MachineMemOperand.
   bool hasOneMemOperand() const {
     return NumMemRefs == 1;
   }
@@ -368,7 +368,7 @@ class MachineInstr : public ilist_node<MachineInstr> {
     AllInBundle      // Return true if all instructions in bundle have property
   };
 
-  /// hasProperty - Return true if the instruction (or in the case of a bundle,
+  /// Return true if the instruction (or in the case of a bundle,
   /// the instructions inside the bundle) has the specified property.
   /// The first argument is the property being queried.
   /// The second argument indicates whether the query should look inside
@@ -382,23 +382,22 @@ class MachineInstr : public ilist_node<MachineInstr> {
     return hasPropertyInBundle(1 << MCFlag, Type);
   }
 
-  /// isVariadic - Return true if this instruction can have a variable number of
-  /// operands.  In this case, the variable operands will be after the normal
+  /// Return true if this instruction can have a variable number of operands.
+  /// In this case, the variable operands will be after the normal
   /// operands but before the implicit definitions and uses (if any are
   /// present).
   bool isVariadic(QueryType Type = IgnoreBundle) const {
     return hasProperty(MCID::Variadic, Type);
   }
 
-  /// hasOptionalDef - Set if this instruction has an optional definition, e.g.
+  /// Set if this instruction has an optional definition, e.g.
   /// ARM instructions which can set condition code if 's' bit is set.
   bool hasOptionalDef(QueryType Type = IgnoreBundle) const {
     return hasProperty(MCID::HasOptionalDef, Type);
   }
 
-  /// isPseudo - Return true if this is a pseudo instruction that doesn't
+  /// Return true if this is a pseudo instruction that doesn't
   /// correspond to a real machine instruction.
-  ///
   bool isPseudo(QueryType Type = IgnoreBundle) const {
     return hasProperty(MCID::Pseudo, Type);
   }
@@ -411,16 +410,15 @@ class MachineInstr : public ilist_node<MachineInstr> {
     return hasProperty(MCID::Call, Type);
   }
 
-  /// isBarrier - Returns true if the specified instruction stops control flow
+  /// Returns true if the specified instruction stops control flow
   /// from executing the instruction immediately following it.  Examples include
   /// unconditional branches and return instructions.
   bool isBarrier(QueryType Type = AnyInBundle) const {
     return hasProperty(MCID::Barrier, Type);
   }
 
-  /// isTerminator - Returns true if this instruction part of the terminator for
-  /// a basic block.  Typically this is things like return and branch
-  /// instructions.
+  /// Returns true if this instruction part of the terminator for a basic block.
+  /// Typically this is things like return and branch instructions.
   ///
   /// Various passes use this to insert code into the bottom of a basic block,
   /// but before control flow occurs.
@@ -428,21 +426,21 @@ class MachineInstr : public ilist_node<MachineInstr> {
     return hasProperty(MCID::Terminator, Type);
   }
 
-  /// isBranch - Returns true if this is a conditional, unconditional, or
-  /// indirect branch.  Predicates below can be used to discriminate between
+  /// Returns true if this is a conditional, unconditional, or indirect branch.
+  /// Predicates below can be used to discriminate between
   /// these cases, and the TargetInstrInfo::AnalyzeBranch method can be used to
   /// get more information.
   bool isBranch(QueryType Type = AnyInBundle) const {
     return hasProperty(MCID::Branch, Type);
   }
 
-  /// isIndirectBranch - Return true if this is an indirect branch, such as a
+  /// Return true if this is an indirect branch, such as a
   /// branch through a register.
   bool isIndirectBranch(QueryType Type = AnyInBundle) const {
     return hasProperty(MCID::IndirectBranch, Type);
   }
 
-  /// isConditionalBranch - Return true if this is a branch which may fall
+  /// Return true if this is a branch which may fall
   /// through to the next instruction or may transfer control flow to some other
   /// block.  The TargetInstrInfo::AnalyzeBranch method can be used to get more
   /// information about this branch.
@@ -450,7 +448,7 @@ class MachineInstr : public ilist_node<MachineInstr> {
     return isBranch(Type) & !isBarrier(Type) & !isIndirectBranch(Type);
   }
 
-  /// isUnconditionalBranch - Return true if this is a branch which always
+  /// Return true if this is a branch which always
   /// transfers control flow to some other block.  The
   /// TargetInstrInfo::AnalyzeBranch method can be used to get more information
   /// about this branch.
@@ -468,43 +466,48 @@ class MachineInstr : public ilist_node<MachineInstr> {
     return hasProperty(MCID::Predicable, Type);
   }
 
-  /// isCompare - Return true if this instruction is a comparison.
+  /// Return true if this instruction is a comparison.
   bool isCompare(QueryType Type = IgnoreBundle) const {
     return hasProperty(MCID::Compare, Type);
   }
 
-  /// isMoveImmediate - Return true if this instruction is a move immediate
+  /// Return true if this instruction is a move immediate
   /// (including conditional moves) instruction.
   bool isMoveImmediate(QueryType Type = IgnoreBundle) const {
     return hasProperty(MCID::MoveImm, Type);
   }
 
-  /// isBitcast - Return true if this instruction is a bitcast instruction.
-  ///
+  /// Return true if this instruction is a bitcast instruction.
   bool isBitcast(QueryType Type = IgnoreBundle) const {
     return hasProperty(MCID::Bitcast, Type);
   }
 
-  /// isSelect - Return true if this instruction is a select instruction.
-  ///
+  /// Return true if this instruction is a select instruction.
   bool isSelect(QueryType Type = IgnoreBundle) const {
     return hasProperty(MCID::Select, Type);
   }
 
-  /// isNotDuplicable - Return true if this instruction cannot be safely
-  /// duplicated.  For example, if the instruction has a unique labels attached
+  /// Return true if this instruction cannot be safely duplicated.
+  /// For example, if the instruction has a unique labels attached
   /// to it, duplicating it would cause multiple definition errors.
   bool isNotDuplicable(QueryType Type = AnyInBundle) const {
     return hasProperty(MCID::NotDuplicable, Type);
   }
 
-  /// hasDelaySlot - Returns true if the specified instruction has a delay slot
+  /// Return true if this instruction is convergent.
+  /// Convergent instructions can only be moved to locations that are
+  /// control-equivalent to their initial position.
+  bool isConvergent(QueryType Type = AnyInBundle) const {
+    return hasProperty(MCID::Convergent, Type);
+  }
+
+  /// Returns true if the specified instruction has a delay slot
   /// which must be filled by the code generator.
   bool hasDelaySlot(QueryType Type = AnyInBundle) const {
     return hasProperty(MCID::DelaySlot, Type);
   }
 
-  /// canFoldAsLoad - Return true for instructions that can be folded as
+  /// Return true for instructions that can be folded as
   /// memory operands in other instructions. The most common use for this
   /// is instructions that are simple loads from memory that don't modify
   /// the loaded value in any way, but it can also be used for instructions
@@ -563,7 +566,7 @@ class MachineInstr : public ilist_node<MachineInstr> {
   // Side Effect Analysis
   //===--------------------------------------------------------------------===//
 
-  /// mayLoad - Return true if this instruction could possibly read memory.
+  /// Return true if this instruction could possibly read memory.
   /// Instructions with this flag set are not necessarily simple load
   /// instructions, they may load a value and modify it, for example.
   bool mayLoad(QueryType Type = AnyInBundle) const {
@@ -575,8 +578,7 @@ class MachineInstr : public ilist_node<MachineInstr> {
     return hasProperty(MCID::MayLoad, Type);
   }
 
-
-  /// mayStore - Return true if this instruction could possibly modify memory.
+  /// Return true if this instruction could possibly modify memory.
   /// Instructions with this flag set are not necessarily simple store
   /// instructions, they may store a modified value based on their operands, or
   /// may not actually modify anything, for example.
@@ -589,11 +591,16 @@ class MachineInstr : public ilist_node<MachineInstr> {
     return hasProperty(MCID::MayStore, Type);
   }
 
+  /// Return true if this instruction could possibly read or modify memory.
+  bool mayLoadOrStore(QueryType Type = AnyInBundle) const {
+    return mayLoad(Type) || mayStore(Type);
+  }
+
   //===--------------------------------------------------------------------===//
   // Flags that indicate whether an instruction can be modified by a method.
   //===--------------------------------------------------------------------===//
 
-  /// isCommutable - Return true if this may be a 2- or 3-address
+  /// Return true if this may be a 2- or 3-address
   /// instruction (of the form "X = op Y, Z, ..."), which produces the same
   /// result if Y and Z are exchanged.  If this flag is set, then the
   /// TargetInstrInfo::commuteInstruction method may be used to hack on the
@@ -607,7 +614,7 @@ class MachineInstr : public ilist_node<MachineInstr> {
     return hasProperty(MCID::Commutable, Type);
   }
 
-  /// isConvertibleTo3Addr - Return true if this is a 2-address instruction
+  /// Return true if this is a 2-address instruction
   /// which can be changed into a 3-address instruction if needed.  Doing this
   /// transformation can be profitable in the register allocator, because it
   /// means that the instruction can use a 2-address form if possible, but
@@ -625,7 +632,7 @@ class MachineInstr : public ilist_node<MachineInstr> {
     return hasProperty(MCID::ConvertibleTo3Addr, Type);
   }
 
-  /// usesCustomInsertionHook - Return true if this instruction requires
+  /// Return true if this instruction requires
   /// custom insertion support when the DAG scheduler is inserting it into a
   /// machine basic block.  If this is true for the instruction, it basically
   /// means that it is a pseudo instruction used at SelectionDAG time that is
@@ -637,7 +644,7 @@ class MachineInstr : public ilist_node<MachineInstr> {
     return hasProperty(MCID::UsesCustomInserter, Type);
   }
 
-  /// hasPostISelHook - Return true if this instruction requires *adjustment*
+  /// Return true if this instruction requires *adjustment*
   /// after instruction selection by calling a target hook. For example, this
   /// can be used to fill in ARM 's' optional operand depending on whether
   /// the conditional flag register is used.
@@ -645,8 +652,8 @@ class MachineInstr : public ilist_node<MachineInstr> {
     return hasProperty(MCID::HasPostISelHook, Type);
   }
 
-  /// isRematerializable - Returns true if this instruction is a candidate for
-  /// remat.  This flag is deprecated, please don't use it anymore.  If this
+  /// Returns true if this instruction is a candidate for remat.
+  /// This flag is deprecated, please don't use it anymore.  If this
   /// flag is set, the isReallyTriviallyReMaterializable() method is called to
   /// verify the instruction is really rematable.
   bool isRematerializable(QueryType Type = AllInBundle) const {
@@ -655,9 +662,9 @@ class MachineInstr : public ilist_node<MachineInstr> {
     return hasProperty(MCID::Rematerializable, Type);
   }
 
-  /// isAsCheapAsAMove - Returns true if this instruction has the same cost (or
-  /// less) than a move instruction. This is useful during certain types of
-  /// optimizations (e.g., remat during two-address conversion or machine licm)
+  /// Returns true if this instruction has the same cost (or less) than a move
+  /// instruction. This is useful during certain types of optimizations
+  /// (e.g., remat during two-address conversion or machine licm)
   /// where we would like to remat or hoist the instruction, but not if it costs
   /// more than moving the instruction into the appropriate register. Note, we
   /// are not marking copies from and to the same register class with this flag.
@@ -666,7 +673,7 @@ class MachineInstr : public ilist_node<MachineInstr> {
     return hasProperty(MCID::CheapAsAMove, Type);
   }
 
-  /// hasExtraSrcRegAllocReq - Returns true if this instruction source operands
+  /// Returns true if this instruction source operands
   /// have special register allocation requirements that are not captured by the
   /// operand register classes. e.g. ARM::STRD's two source registers must be an
   /// even / odd pair, ARM::STM registers have to be in ascending order.
@@ -676,7 +683,7 @@ class MachineInstr : public ilist_node<MachineInstr> {
     return hasProperty(MCID::ExtraSrcRegAllocReq, Type);
   }
 
-  /// hasExtraDefRegAllocReq - Returns true if this instruction def operands
+  /// Returns true if this instruction def operands
   /// have special register allocation requirements that are not captured by the
   /// operand register classes. e.g. ARM::LDRD's two def registers must be an
   /// even / odd pair, ARM::LDM registers have to be in ascending order.
@@ -694,7 +701,7 @@ class MachineInstr : public ilist_node<MachineInstr> {
     IgnoreVRegDefs  // Ignore virtual register definitions
   };
 
-  /// isIdenticalTo - Return true if this instruction is identical to (same
+  /// Return true if this instruction is identical to (same
   /// opcode and same operands as) the specified instruction.
   bool isIdenticalTo(const MachineInstr *Other,
                      MICheckType Check = CheckDefs) const;
@@ -735,8 +742,7 @@ class MachineInstr : public ilist_node<MachineInstr> {
   bool isEHLabel() const { return getOpcode() == TargetOpcode::EH_LABEL; }
   bool isGCLabel() const { return getOpcode() == TargetOpcode::GC_LABEL; }
 
-  /// isLabel - Returns true if the MachineInstr represents a label.
-  ///
+  /// Returns true if the MachineInstr represents a label.
   bool isLabel() const { return isEHLabel() || isGCLabel(); }
   bool isCFIInstruction() const {
     return getOpcode() == TargetOpcode::CFI_INSTRUCTION;
@@ -785,19 +791,19 @@ class MachineInstr : public ilist_node<MachineInstr> {
     return getOpcode() == TargetOpcode::EXTRACT_SUBREG;
   }
 
-  /// isCopyLike - Return true if the instruction behaves like a copy.
+  /// Return true if the instruction behaves like a copy.
   /// This does not include native copy instructions.
   bool isCopyLike() const {
     return isCopy() || isSubregToReg();
   }
 
-  /// isIdentityCopy - Return true is the instruction is an identity copy.
+  /// Return true is the instruction is an identity copy.
   bool isIdentityCopy() const {
     return isCopy() && getOperand(0).getReg() == getOperand(1).getReg() &&
       getOperand(0).getSubReg() == getOperand(1).getSubReg();
   }
 
-  /// isTransient - Return true if this is a transient instruction that is
+  /// Return true if this is a transient instruction that is
   /// either very likely to be eliminated during register allocation (such as
   /// copy-like instructions), or if this instruction doesn't have an
   /// execution-time cost.
@@ -828,8 +834,8 @@ class MachineInstr : public ilist_node<MachineInstr> {
   /// skips, 0 for unbundled instructions.
   unsigned getBundleSize() const;
 
-  /// readsRegister - Return true if the MachineInstr reads the specified
-  /// register. If TargetRegisterInfo is passed, then it also checks if there
+  /// Return true if the MachineInstr reads the specified register.
+  /// If TargetRegisterInfo is passed, then it also checks if there
   /// is a read of a super-register.
   /// This does not count partial redefines of virtual registers as reads:
   ///   %reg1024:6 = OP.
@@ -838,30 +844,29 @@ class MachineInstr : public ilist_node<MachineInstr> {
     return findRegisterUseOperandIdx(Reg, false, TRI) != -1;
   }
 
-  /// readsVirtualRegister - Return true if the MachineInstr reads the specified
-  /// virtual register. Take into account that a partial define is a
+  /// Return true if the MachineInstr reads the specified virtual register.
+  /// Take into account that a partial define is a
   /// read-modify-write operation.
   bool readsVirtualRegister(unsigned Reg) const {
     return readsWritesVirtualRegister(Reg).first;
   }
 
-  /// readsWritesVirtualRegister - Return a pair of bools (reads, writes)
-  /// indicating if this instruction reads or writes Reg. This also considers
-  /// partial defines.
+  /// Return a pair of bools (reads, writes) indicating if this instruction
+  /// reads or writes Reg. This also considers partial defines.
   /// If Ops is not null, all operand indices for Reg are added.
   std::pair<bool,bool> readsWritesVirtualRegister(unsigned Reg,
                                 SmallVectorImpl<unsigned> *Ops = nullptr) const;
 
-  /// killsRegister - Return true if the MachineInstr kills the specified
-  /// register. If TargetRegisterInfo is passed, then it also checks if there is
+  /// Return true if the MachineInstr kills the specified register.
+  /// If TargetRegisterInfo is passed, then it also checks if there is
   /// a kill of a super-register.
   bool killsRegister(unsigned Reg,
                      const TargetRegisterInfo *TRI = nullptr) const {
     return findRegisterUseOperandIdx(Reg, true, TRI) != -1;
   }
 
-  /// definesRegister - Return true if the MachineInstr fully defines the
-  /// specified register. If TargetRegisterInfo is passed, then it also checks
+  /// Return true if the MachineInstr fully defines the specified register.
+  /// If TargetRegisterInfo is passed, then it also checks
   /// if there is a def of a super-register.
   /// NOTE: It's ignoring subreg indices on virtual registers.
   bool definesRegister(unsigned Reg,
@@ -869,28 +874,28 @@ class MachineInstr : public ilist_node<MachineInstr> {
     return findRegisterDefOperandIdx(Reg, false, false, TRI) != -1;
   }
 
-  /// modifiesRegister - Return true if the MachineInstr modifies (fully define
-  /// or partially define) the specified register.
+  /// Return true if the MachineInstr modifies (fully define or partially
+  /// define) the specified register.
   /// NOTE: It's ignoring subreg indices on virtual registers.
   bool modifiesRegister(unsigned Reg, const TargetRegisterInfo *TRI) const {
     return findRegisterDefOperandIdx(Reg, false, true, TRI) != -1;
   }
 
-  /// registerDefIsDead - Returns true if the register is dead in this machine
-  /// instruction. If TargetRegisterInfo is passed, then it also checks
+  /// Returns true if the register is dead in this machine instruction.
+  /// If TargetRegisterInfo is passed, then it also checks
   /// if there is a dead def of a super-register.
   bool registerDefIsDead(unsigned Reg,
                          const TargetRegisterInfo *TRI = nullptr) const {
     return findRegisterDefOperandIdx(Reg, true, false, TRI) != -1;
   }
 
-  /// findRegisterUseOperandIdx() - Returns the operand index that is a use of
-  /// the specific register or -1 if it is not found. It further tightens
-  /// the search criteria to a use that kills the register if isKill is true.
+  /// Returns the operand index that is a use of the specific register or -1
+  /// if it is not found. It further tightens the search criteria to a use
+  /// that kills the register if isKill is true.
   int findRegisterUseOperandIdx(unsigned Reg, bool isKill = false,
                                 const TargetRegisterInfo *TRI = nullptr) const;
 
-  /// findRegisterUseOperand - Wrapper for findRegisterUseOperandIdx, it returns
+  /// Wrapper for findRegisterUseOperandIdx, it returns
   /// a pointer to the MachineOperand rather than an index.
   MachineOperand *findRegisterUseOperand(unsigned Reg, bool isKill = false,
                                       const TargetRegisterInfo *TRI = nullptr) {
@@ -898,17 +903,17 @@ class MachineInstr : public ilist_node<MachineInstr> {
     return (Idx == -1) ? nullptr : &getOperand(Idx);
   }
 
-  /// findRegisterDefOperandIdx() - Returns the operand index that is a def of
-  /// the specified register or -1 if it is not found. If isDead is true, defs
-  /// that are not dead are skipped. If Overlap is true, then it also looks for
-  /// defs that merely overlap the specified register. If TargetRegisterInfo is
-  /// non-null, then it also checks if there is a def of a super-register.
+  /// Returns the operand index that is a def of the specified register or
+  /// -1 if it is not found. If isDead is true, defs that are not dead are
+  /// skipped. If Overlap is true, then it also looks for defs that merely
+  /// overlap the specified register. If TargetRegisterInfo is non-null,
+  /// then it also checks if there is a def of a super-register.
   /// This may also return a register mask operand when Overlap is true.
   int findRegisterDefOperandIdx(unsigned Reg,
                                 bool isDead = false, bool Overlap = false,
                                 const TargetRegisterInfo *TRI = nullptr) const;
 
-  /// findRegisterDefOperand - Wrapper for findRegisterDefOperandIdx, it returns
+  /// Wrapper for findRegisterDefOperandIdx, it returns
   /// a pointer to the MachineOperand rather than an index.
   MachineOperand *findRegisterDefOperand(unsigned Reg, bool isDead = false,
                                       const TargetRegisterInfo *TRI = nullptr) {
@@ -916,12 +921,12 @@ class MachineInstr : public ilist_node<MachineInstr> {
     return (Idx == -1) ? nullptr : &getOperand(Idx);
   }
 
-  /// findFirstPredOperandIdx() - Find the index of the first operand in the
+  /// Find the index of the first operand in the
   /// operand list that is used to represent the predicate. It returns -1 if
   /// none is found.
   int findFirstPredOperandIdx() const;
 
-  /// findInlineAsmFlagIdx() - Find the index of the flag word operand that
+  /// Find the index of the flag word operand that
   /// corresponds to operand OpIdx on an inline asm instruction.  Returns -1 if
   /// getOperand(OpIdx) does not belong to an inline asm operand group.
   ///
@@ -933,11 +938,11 @@ class MachineInstr : public ilist_node<MachineInstr> {
   ///
   int findInlineAsmFlagIdx(unsigned OpIdx, unsigned *GroupNo = nullptr) const;
 
-  /// getRegClassConstraint - Compute the static register class constraint for
-  /// operand OpIdx.  For normal instructions, this is derived from the
-  /// MCInstrDesc.  For inline assembly it is derived from the flag words.
+  /// Compute the static register class constraint for operand OpIdx.
+  /// For normal instructions, this is derived from the MCInstrDesc.
+  /// For inline assembly it is derived from the flag words.
   ///
-  /// Returns NULL if the static register classs constraint cannot be
+  /// Returns NULL if the static register class constraint cannot be
   /// determined.
   ///
   const TargetRegisterClass*
@@ -949,10 +954,10 @@ class MachineInstr : public ilist_node<MachineInstr> {
   /// the given \p CurRC.
   /// If \p ExploreBundle is set and MI is part of a bundle, all the
   /// instructions inside the bundle will be taken into account. In other words,
-  /// this method accumulates all the constrains of the operand of this MI and
+  /// this method accumulates all the constraints of the operand of this MI and
   /// the related bundle if MI is a bundle or inside a bundle.
   ///
-  /// Returns the register class that statisfies both \p CurRC and the
+  /// Returns the register class that satisfies both \p CurRC and the
   /// constraints set by MI. Returns NULL if such a register class does not
   /// exist.
   ///
@@ -965,7 +970,7 @@ class MachineInstr : public ilist_node<MachineInstr> {
   /// \brief Applies the constraints (def/use) implied by the \p OpIdx operand
   /// to the given \p CurRC.
   ///
-  /// Returns the register class that statisfies both \p CurRC and the
+  /// Returns the register class that satisfies both \p CurRC and the
   /// constraints set by \p OpIdx MI. Returns NULL if such a register class
   /// does not exist.
   ///
@@ -976,20 +981,20 @@ class MachineInstr : public ilist_node<MachineInstr> {
                               const TargetInstrInfo *TII,
                               const TargetRegisterInfo *TRI) const;
 
-  /// tieOperands - Add a tie between the register operands at DefIdx and
-  /// UseIdx. The tie will cause the register allocator to ensure that the two
+  /// Add a tie between the register operands at DefIdx and UseIdx.
+  /// The tie will cause the register allocator to ensure that the two
   /// operands are assigned the same physical register.
   ///
   /// Tied operands are managed automatically for explicit operands in the
   /// MCInstrDesc. This method is for exceptional cases like inline asm.
   void tieOperands(unsigned DefIdx, unsigned UseIdx);
 
-  /// findTiedOperandIdx - Given the index of a tied register operand, find the
+  /// Given the index of a tied register operand, find the
   /// operand it is tied to. Defs are tied to uses and vice versa. Returns the
   /// index of the tied operand which must exist.
   unsigned findTiedOperandIdx(unsigned OpIdx) const;
 
-  /// isRegTiedToUseOperand - Given the index of a register def operand,
+  /// Given the index of a register def operand,
   /// check if the register def is tied to a source operand, due to either
   /// two-address elimination or inline assembly constraints. Returns the
   /// first tied use operand index by reference if UseOpIdx is not null.
@@ -1003,9 +1008,9 @@ class MachineInstr : public ilist_node<MachineInstr> {
     return true;
   }
 
-  /// isRegTiedToDefOperand - Return true if the use operand of the specified
-  /// index is tied to a def operand. It also returns the def operand index by
-  /// reference if DefOpIdx is not null.
+  /// Return true if the use operand of the specified index is tied to a def
+  /// operand. It also returns the def operand index by reference if DefOpIdx
+  /// is not null.
   bool isRegTiedToDefOperand(unsigned UseOpIdx,
                              unsigned *DefOpIdx = nullptr) const {
     const MachineOperand &MO = getOperand(UseOpIdx);
@@ -1016,16 +1021,15 @@ class MachineInstr : public ilist_node<MachineInstr> {
     return true;
   }
 
-  /// clearKillInfo - Clears kill flags on all operands.
-  ///
+  /// Clears kill flags on all operands.
   void clearKillInfo();
 
-  /// substituteRegister - Replace all occurrences of FromReg with ToReg:SubIdx,
+  /// Replace all occurrences of FromReg with ToReg:SubIdx,
   /// properly composing subreg indices where necessary.
   void substituteRegister(unsigned FromReg, unsigned ToReg, unsigned SubIdx,
                           const TargetRegisterInfo &RegInfo);
 
-  /// addRegisterKilled - We have determined MI kills a register. Look for the
+  /// We have determined MI kills a register. Look for the
   /// operand that uses it and mark it as IsKill. If AddIfNotFound is true,
   /// add a implicit operand if it's not found. Returns true if the operand
   /// exists / is added.
@@ -1033,11 +1037,11 @@ class MachineInstr : public ilist_node<MachineInstr> {
                          const TargetRegisterInfo *RegInfo,
                          bool AddIfNotFound = false);
 
-  /// clearRegisterKills - Clear all kill flags affecting Reg.  If RegInfo is
+  /// Clear all kill flags affecting Reg.  If RegInfo is
   /// provided, this includes super-register kills.
   void clearRegisterKills(unsigned Reg, const TargetRegisterInfo *RegInfo);
 
-  /// addRegisterDead - We have determined MI defined a register without a use.
+  /// We have determined MI defined a register without a use.
   /// Look for the operand that defines it and mark it as IsDead. If
   /// AddIfNotFound is true, add a implicit operand if it's not found. Returns
   /// true if the operand exists / is added.
@@ -1052,12 +1056,12 @@ class MachineInstr : public ilist_node<MachineInstr> {
   /// otherwise undefined super register.
   void addRegisterDefReadUndef(unsigned Reg);
 
-  /// addRegisterDefined - We have determined MI defines a register. Make sure
-  /// there is an operand defining Reg.
+  /// We have determined MI defines a register. Make sure there is an operand
+  /// defining Reg.
   void addRegisterDefined(unsigned Reg,
                           const TargetRegisterInfo *RegInfo = nullptr);
 
-  /// setPhysRegsDeadExcept - Mark every physreg used by this instruction as
+  /// Mark every physreg used by this instruction as
   /// dead except those in the UsedRegs list.
   ///
   /// On instructions with register mask operands, also add implicit-def
@@ -1065,32 +1069,30 @@ class MachineInstr : public ilist_node<MachineInstr> {
   void setPhysRegsDeadExcept(ArrayRef<unsigned> UsedRegs,
                              const TargetRegisterInfo &TRI);
 
-  /// isSafeToMove - Return true if it is safe to move this instruction. If
+  /// Return true if it is safe to move this instruction. If
   /// SawStore is set to true, it means that there is a store (or call) between
   /// the instruction's location and its intended destination.
-  bool isSafeToMove(const TargetInstrInfo *TII, AliasAnalysis *AA,
-                    bool &SawStore) const;
+  bool isSafeToMove(AliasAnalysis *AA, bool &SawStore) const;
 
-  /// hasOrderedMemoryRef - Return true if this instruction may have an ordered
+  /// Return true if this instruction may have an ordered
   /// or volatile memory reference, or if the information describing the memory
   /// reference is not available. Return false if it is known to have no
   /// ordered or volatile memory references.
   bool hasOrderedMemoryRef() const;
 
-  /// isInvariantLoad - Return true if this instruction is loading from a
+  /// Return true if this instruction is loading from a
   /// location whose value is invariant across the function.  For example,
   /// loading a value from the constant pool or from the argument area of
   /// a function if it does not change.  This should only return true of *all*
   /// loads the instruction does are invariant (if it does multiple loads).
   bool isInvariantLoad(AliasAnalysis *AA) const;
 
-  /// isConstantValuePHI - If the specified instruction is a PHI that always
-  /// merges together the same virtual register, return the register, otherwise
-  /// return 0.
+  /// If the specified instruction is a PHI that always merges together the
+  /// same virtual register, return the register, otherwise return 0.
   unsigned isConstantValuePHI() const;
 
-  /// hasUnmodeledSideEffects - Return true if this instruction has side
-  /// effects that are not modeled by mayLoad / mayStore, etc.
+  /// Return true if this instruction has side effects that are not modeled
+  /// by mayLoad / mayStore, etc.
   /// For all instructions, the property is encoded in MCInstrDesc::Flags
   /// (see MCInstrDesc::hasUnmodeledSideEffects(). The only exception is
   /// INLINEASM instruction, in which case the side effect property is encoded
@@ -1098,11 +1100,10 @@ class MachineInstr : public ilist_node<MachineInstr> {
   ///
   bool hasUnmodeledSideEffects() const;
 
-  /// allDefsAreDead - Return true if all the defs of this instruction are dead.
-  ///
+  /// Return true if all the defs of this instruction are dead.
   bool allDefsAreDead() const;
 
-  /// copyImplicitOps - Copy implicit register operands from specified
+  /// Copy implicit register operands from specified
   /// instruction to this instruction.
   void copyImplicitOps(MachineFunction &MF, const MachineInstr *MI);
 
@@ -1110,6 +1111,8 @@ class MachineInstr : public ilist_node<MachineInstr> {
   // Debugging support
   //
   void print(raw_ostream &OS, bool SkipOpers = false) const;
+  void print(raw_ostream &OS, ModuleSlotTracker &MST,
+             bool SkipOpers = false) const;
   void dump() const;
 
   //===--------------------------------------------------------------------===//
@@ -1134,50 +1137,41 @@ class MachineInstr : public ilist_node<MachineInstr> {
   /// preferred.
   void addOperand(const MachineOperand &Op);
 
-  /// setDesc - Replace the instruction descriptor (thus opcode) of
+  /// Replace the instruction descriptor (thus opcode) of
   /// the current instruction with a new one.
-  ///
   void setDesc(const MCInstrDesc &tid) { MCID = &tid; }
 
-  /// setDebugLoc - Replace current source information with new such.
+  /// Replace current source information with new such.
   /// Avoid using this, the constructor argument is preferable.
-  ///
   void setDebugLoc(DebugLoc dl) {
     debugLoc = std::move(dl);
     assert(debugLoc.hasTrivialDestructor() && "Expected trivial destructor");
   }
 
-  /// RemoveOperand - Erase an operand  from an instruction, leaving it with one
+  /// Erase an operand  from an instruction, leaving it with one
   /// fewer operand than it started with.
-  ///
   void RemoveOperand(unsigned i);
 
-  /// addMemOperand - Add a MachineMemOperand to the machine instruction.
+  /// Add a MachineMemOperand to the machine instruction.
   /// This function should be used only occasionally. The setMemRefs function
   /// is the primary method for setting up a MachineInstr's MemRefs list.
   void addMemOperand(MachineFunction &MF, MachineMemOperand *MO);
 
-  /// setMemRefs - Assign this MachineInstr's memory reference descriptor
-  /// list. This does not transfer ownership.
+  /// Assign this MachineInstr's memory reference descriptor list.
+  /// This does not transfer ownership.
   void setMemRefs(mmo_iterator NewMemRefs, mmo_iterator NewMemRefsEnd) {
     MemRefs = NewMemRefs;
     NumMemRefs = uint8_t(NewMemRefsEnd - NewMemRefs);
     assert(NumMemRefs == NewMemRefsEnd - NewMemRefs && "Too many memrefs");
   }
 
-  /// clearMemRefs - Clear this MachineInstr's memory reference descriptor list.
+  /// Clear this MachineInstr's memory reference descriptor list.
   void clearMemRefs() {
     MemRefs = nullptr;
     NumMemRefs = 0;
   }
 
-private:
-  /// getRegInfo - If this instruction is embedded into a MachineFunction,
-  /// return the MachineRegisterInfo object for the current function, otherwise
-  /// return null.
-  MachineRegisterInfo *getRegInfo();
-
-  /// untieRegOperand - Break any tie involving OpIdx.
+  /// Break any tie involving OpIdx.
   void untieRegOperand(unsigned OpIdx) {
     MachineOperand &MO = getOperand(OpIdx);
     if (MO.isReg() && MO.isTied()) {
@@ -1186,22 +1180,27 @@ class MachineInstr : public ilist_node<MachineInstr> {
     }
   }
 
-  /// addImplicitDefUseOperands - Add all implicit def and use operands to
-  /// this instruction.
+
+private:
+  /// If this instruction is embedded into a MachineFunction, return the
+  /// MachineRegisterInfo object for the current function, otherwise
+  /// return null.
+  MachineRegisterInfo *getRegInfo();
+
+  /// Add all implicit def and use operands to this instruction.
   void addImplicitDefUseOperands(MachineFunction &MF);
 
-  /// RemoveRegOperandsFromUseLists - Unlink all of the register operands in
-  /// this instruction from their respective use lists.  This requires that the
-  /// operands already be on their use lists.
+  /// Unlink all of the register operands in this instruction from their
+  /// respective use lists.  This requires that the operands already be on their
+  /// use lists.
   void RemoveRegOperandsFromUseLists(MachineRegisterInfo&);
 
-  /// AddRegOperandsToUseLists - Add all of the register operands in
-  /// this instruction from their respective use lists.  This requires that the
-  /// operands not be on their use lists yet.
+  /// Add all of the register operands in this instruction from their
+  /// respective use lists.  This requires that the operands not be on their
+  /// use lists yet.
   void AddRegOperandsToUseLists(MachineRegisterInfo&);
 
-  /// hasPropertyInBundle - Slow path for hasProperty when we're dealing with a
-  /// bundle.
+  /// Slow path for hasProperty when we're dealing with a bundle.
   bool hasPropertyInBundle(unsigned Mask, QueryType Type) const;
 
   /// \brief Implements the logic of getRegClassConstraintEffectForVReg for the
@@ -1212,8 +1211,8 @@ class MachineInstr : public ilist_node<MachineInstr> {
       const TargetInstrInfo *TII, const TargetRegisterInfo *TRI) const;
 };
 
-/// MachineInstrExpressionTrait - Special DenseMapInfo traits to compare
-/// MachineInstr* by *value* of the instruction rather than by pointer value.
+/// Special DenseMapInfo traits to compare MachineInstr* by *value* of the
+/// instruction rather than by pointer value.
 /// The hashing and equality testing functions ignore definitions so this is
 /// useful for CSE, etc.
 struct MachineInstrExpressionTrait : DenseMapInfo<MachineInstr*> {
diff --git a/include/llvm/CodeGen/MachineInstrBuilder.h b/include/llvm/CodeGen/MachineInstrBuilder.h
index 15000aa1774..e5e106e71cf 100644
--- a/include/llvm/CodeGen/MachineInstrBuilder.h
+++ b/include/llvm/CodeGen/MachineInstrBuilder.h
@@ -49,11 +49,10 @@ class MachineInstrBuilder {
   MachineInstrBuilder() : MF(nullptr), MI(nullptr) {}
 
   /// Create a MachineInstrBuilder for manipulating an existing instruction.
-  /// F must be the machine function  that was used to allocate I.
+  /// F must be the machine function that was used to allocate I.
   MachineInstrBuilder(MachineFunction &F, MachineInstr *I) : MF(&F), MI(I) {}
 
   /// Allow automatic conversion to the machine instruction we are working on.
-  ///
   operator MachineInstr*() const { return MI; }
   MachineInstr *operator->() const { return MI; }
   operator MachineBasicBlock::iterator() const { return MI; }
@@ -62,11 +61,9 @@ class MachineInstrBuilder {
   /// explicitly.
   MachineInstr *getInstr() const { return MI; }
 
-  /// addReg - Add a new virtual register operand...
-  ///
-  const
-  MachineInstrBuilder &addReg(unsigned RegNo, unsigned flags = 0,
-                              unsigned SubReg = 0) const {
+  /// Add a new virtual register operand.
+  const MachineInstrBuilder &addReg(unsigned RegNo, unsigned flags = 0,
+                                    unsigned SubReg = 0) const {
     assert((flags & 0x1) == 0 &&
            "Passing in 'true' to addReg is forbidden! Use enums instead.");
     MI->addOperand(*MF, MachineOperand::CreateReg(RegNo,
@@ -82,8 +79,7 @@ class MachineInstrBuilder {
     return *this;
   }
 
-  /// addImm - Add a new immediate operand.
-  ///
+  /// Add a new immediate operand.
   const MachineInstrBuilder &addImm(int64_t Val) const {
     MI->addOperand(*MF, MachineOperand::CreateImm(Val));
     return *this;
@@ -176,7 +172,7 @@ class MachineInstrBuilder {
     MI->addOperand(*MF, MachineOperand::CreateMetadata(MD));
     assert((MI->isDebugValue() ? static_cast<bool>(MI->getDebugVariable())
                                : true) &&
-           "first MDNode argument of a DBG_VALUE not a DIVariable");
+           "first MDNode argument of a DBG_VALUE not a variable");
     return *this;
   }
 
@@ -185,8 +181,9 @@ class MachineInstrBuilder {
     return *this;
   }
 
-  const MachineInstrBuilder &addSym(MCSymbol *Sym) const {
-    MI->addOperand(*MF, MachineOperand::CreateMCSymbol(Sym));
+  const MachineInstrBuilder &addSym(MCSymbol *Sym,
+                                    unsigned char TargetFlags = 0) const {
+    MI->addOperand(*MF, MachineOperand::CreateMCSymbol(Sym, TargetFlags));
     return *this;
   }
 
@@ -229,18 +226,15 @@ class MachineInstrBuilder {
   }
 };
 
-/// BuildMI - Builder interface.  Specify how to create the initial instruction
-/// itself.
-///
+/// Builder interface. Specify how to create the initial instruction itself.
 inline MachineInstrBuilder BuildMI(MachineFunction &MF,
                                    DebugLoc DL,
                                    const MCInstrDesc &MCID) {
   return MachineInstrBuilder(MF, MF.CreateMachineInstr(MCID, DL));
 }
 
-/// BuildMI - This version of the builder sets up the first operand as a
+/// This version of the builder sets up the first operand as a
 /// destination virtual register.
-///
 inline MachineInstrBuilder BuildMI(MachineFunction &MF,
                                    DebugLoc DL,
                                    const MCInstrDesc &MCID,
@@ -249,10 +243,9 @@ inline MachineInstrBuilder BuildMI(MachineFunction &MF,
            .addReg(DestReg, RegState::Define);
 }
 
-/// BuildMI - This version of the builder inserts the newly-built
-/// instruction before the given position in the given MachineBasicBlock, and
-/// sets up the first operand as a destination virtual register.
-///
+/// This version of the builder inserts the newly-built instruction before
+/// the given position in the given MachineBasicBlock, and sets up the first
+/// operand as a destination virtual register.
 inline MachineInstrBuilder BuildMI(MachineBasicBlock &BB,
                                    MachineBasicBlock::iterator I,
                                    DebugLoc DL,
@@ -289,10 +282,9 @@ inline MachineInstrBuilder BuildMI(MachineBasicBlock &BB,
   return BuildMI(BB, MII, DL, MCID, DestReg);
 }
 
-/// BuildMI - This version of the builder inserts the newly-built
-/// instruction before the given position in the given MachineBasicBlock, and
-/// does NOT take a destination register.
-///
+/// This version of the builder inserts the newly-built instruction before the
+/// given position in the given MachineBasicBlock, and does NOT take a
+/// destination register.
 inline MachineInstrBuilder BuildMI(MachineBasicBlock &BB,
                                    MachineBasicBlock::iterator I,
                                    DebugLoc DL,
@@ -326,35 +318,17 @@ inline MachineInstrBuilder BuildMI(MachineBasicBlock &BB,
   return BuildMI(BB, MII, DL, MCID);
 }
 
-// @LOCALMOD-BEGIN
-/// BuildMI - This version of the builder inserts the newly-built
-/// instruction before the given position in the given MachineBasicBlock,
-/// does NOT take a destination register, and does not add implicit operands.
-///
-inline MachineInstrBuilder BuildMI_NoImp(MachineBasicBlock &BB,
-                                         MachineBasicBlock::iterator I,
-                                         DebugLoc DL,
-                                         const MCInstrDesc &MCID) {
-  MachineInstr *MI = BB.getParent()->CreateMachineInstr(MCID, DL, true);
-  BB.insert(I, MI);
-  return MachineInstrBuilder(*BB.getParent(), MI);
-}
-// @LOCALMOD-END
-
-/// BuildMI - This version of the builder inserts the newly-built
-/// instruction at the end of the given MachineBasicBlock, and does NOT take a
-/// destination register.
-///
+/// This version of the builder inserts the newly-built instruction at the end
+/// of the given MachineBasicBlock, and does NOT take a destination register.
 inline MachineInstrBuilder BuildMI(MachineBasicBlock *BB,
                                    DebugLoc DL,
                                    const MCInstrDesc &MCID) {
   return BuildMI(*BB, BB->end(), DL, MCID);
 }
 
-/// BuildMI - This version of the builder inserts the newly-built
-/// instruction at the end of the given MachineBasicBlock, and sets up the first
-/// operand as a destination virtual register.
-///
+/// This version of the builder inserts the newly-built instruction at the
+/// end of the given MachineBasicBlock, and sets up the first operand as a
+/// destination virtual register.
 inline MachineInstrBuilder BuildMI(MachineBasicBlock *BB,
                                    DebugLoc DL,
                                    const MCInstrDesc &MCID,
@@ -362,18 +336,17 @@ inline MachineInstrBuilder BuildMI(MachineBasicBlock *BB,
   return BuildMI(*BB, BB->end(), DL, MCID, DestReg);
 }
 
-/// BuildMI - This version of the builder builds a DBG_VALUE intrinsic
+/// This version of the builder builds a DBG_VALUE intrinsic
 /// for either a value in a register or a register-indirect+offset
 /// address.  The convention is that a DBG_VALUE is indirect iff the
 /// second operand is an immediate.
-///
 inline MachineInstrBuilder BuildMI(MachineFunction &MF, DebugLoc DL,
                                    const MCInstrDesc &MCID, bool IsIndirect,
                                    unsigned Reg, unsigned Offset,
                                    const MDNode *Variable, const MDNode *Expr) {
-  assert(isa<MDLocalVariable>(Variable) && "not a DIVariable");
-  assert(cast<MDExpression>(Expr)->isValid() && "not a DIExpression");
-  assert(cast<MDLocalVariable>(Variable)->isValidLocationForIntrinsic(DL) &&
+  assert(isa<DILocalVariable>(Variable) && "not a variable");
+  assert(cast<DIExpression>(Expr)->isValid() && "not an expression");
+  assert(cast<DILocalVariable>(Variable)->isValidLocationForIntrinsic(DL) &&
          "Expected inlined-at fields to agree");
   if (IsIndirect)
     return BuildMI(MF, DL, MCID)
@@ -391,17 +364,16 @@ inline MachineInstrBuilder BuildMI(MachineFunction &MF, DebugLoc DL,
   }
 }
 
-/// BuildMI - This version of the builder builds a DBG_VALUE intrinsic
+/// This version of the builder builds a DBG_VALUE intrinsic
 /// for either a value in a register or a register-indirect+offset
 /// address and inserts it at position I.
-///
 inline MachineInstrBuilder BuildMI(MachineBasicBlock &BB,
                                    MachineBasicBlock::iterator I, DebugLoc DL,
                                    const MCInstrDesc &MCID, bool IsIndirect,
                                    unsigned Reg, unsigned Offset,
                                    const MDNode *Variable, const MDNode *Expr) {
-  assert(isa<MDLocalVariable>(Variable) && "not a DIVariable");
-  assert(cast<MDExpression>(Expr)->isValid() && "not a DIExpression");
+  assert(isa<DILocalVariable>(Variable) && "not a variable");
+  assert(cast<DIExpression>(Expr)->isValid() && "not an expression");
   MachineFunction &MF = *BB.getParent();
   MachineInstr *MI =
       BuildMI(MF, DL, MCID, IsIndirect, Reg, Offset, Variable, Expr);
diff --git a/include/llvm/CodeGen/MachineLoopInfo.h b/include/llvm/CodeGen/MachineLoopInfo.h
index f7bcf45d93c..4868b7363f8 100644
--- a/include/llvm/CodeGen/MachineLoopInfo.h
+++ b/include/llvm/CodeGen/MachineLoopInfo.h
@@ -37,10 +37,8 @@
 namespace llvm {
 
 // Implementation in LoopInfoImpl.h
-#ifdef __GNUC__
 class MachineLoop;
-__extension__ extern template class LoopBase<MachineBasicBlock, MachineLoop>;
-#endif
+extern template class LoopBase<MachineBasicBlock, MachineLoop>;
 
 class MachineLoop : public LoopBase<MachineBasicBlock, MachineLoop> {
 public:
@@ -65,10 +63,7 @@ class MachineLoop : public LoopBase<MachineBasicBlock, MachineLoop> {
 };
 
 // Implementation in LoopInfoImpl.h
-#ifdef __GNUC__
-__extension__ extern template
-class LoopInfoBase<MachineBasicBlock, MachineLoop>;
-#endif
+extern template class LoopInfoBase<MachineBasicBlock, MachineLoop>;
 
 class MachineLoopInfo : public MachineFunctionPass {
   LoopInfoBase<MachineBasicBlock, MachineLoop> LI;
@@ -114,7 +109,7 @@ class MachineLoopInfo : public MachineFunctionPass {
   }
 
   // isLoopHeader - True if the block is a loop header node
-  inline bool isLoopHeader(MachineBasicBlock *BB) const {
+  inline bool isLoopHeader(const MachineBasicBlock *BB) const {
     return LI.isLoopHeader(BB);
   }
 
diff --git a/include/llvm/CodeGen/MachineMemOperand.h b/include/llvm/CodeGen/MachineMemOperand.h
index a16c294a074..a73b92f9a25 100644
--- a/include/llvm/CodeGen/MachineMemOperand.h
+++ b/include/llvm/CodeGen/MachineMemOperand.h
@@ -27,6 +27,7 @@ namespace llvm {
 class FoldingSetNodeID;
 class MDNode;
 class raw_ostream;
+class ModuleSlotTracker;
 
 /// MachinePointerInfo - This class contains a discriminated union of
 /// information about pointers in memory operands, relating them back to LLVM IR
@@ -200,6 +201,12 @@ class MachineMemOperand {
   ///
   void Profile(FoldingSetNodeID &ID) const;
 
+  /// Support for operator<<.
+  /// @{
+  void print(raw_ostream &OS) const;
+  void print(raw_ostream &OS, ModuleSlotTracker &MST) const;
+  /// @}
+
   friend bool operator==(const MachineMemOperand &LHS,
                          const MachineMemOperand &RHS) {
     return LHS.getValue() == RHS.getValue() &&
@@ -219,7 +226,10 @@ class MachineMemOperand {
   }
 };
 
-raw_ostream &operator<<(raw_ostream &OS, const MachineMemOperand &MRO);
+inline raw_ostream &operator<<(raw_ostream &OS, const MachineMemOperand &MRO) {
+  MRO.print(OS);
+  return OS;
+}
 
 } // End llvm namespace
 
diff --git a/include/llvm/CodeGen/MachineModuleInfo.h b/include/llvm/CodeGen/MachineModuleInfo.h
index 3965b1dea70..4cdfe2463c9 100644
--- a/include/llvm/CodeGen/MachineModuleInfo.h
+++ b/include/llvm/CodeGen/MachineModuleInfo.h
@@ -51,6 +51,7 @@ namespace llvm {
 // Forward declarations.
 class Constant;
 class GlobalVariable;
+class BlockAddress;
 class MDNode;
 class MMIAddrLabelMap;
 class MachineBasicBlock;
@@ -60,6 +61,14 @@ class PointerType;
 class StructType;
 struct WinEHFuncInfo;
 
+struct SEHHandler {
+  // Filter or finally function. Null indicates a catch-all.
+  const Function *FilterOrFinally;
+
+  // Address of block to recover at. Null for a finally handler.
+  const BlockAddress *RecoverBA;
+};
+
 //===----------------------------------------------------------------------===//
 /// LandingPadInfo - This structure is used to retain landing pad info for
 /// the current function.
@@ -68,7 +77,7 @@ struct LandingPadInfo {
   MachineBasicBlock *LandingPadBlock;      // Landing pad block.
   SmallVector<MCSymbol *, 1> BeginLabels;  // Labels prior to invoke.
   SmallVector<MCSymbol *, 1> EndLabels;    // Labels after invoke.
-  SmallVector<MCSymbol *, 1> ClauseLabels; // Labels for each clause.
+  SmallVector<SEHHandler, 1> SEHHandlers;  // SEH handlers active at this lpad.
   MCSymbol *LandingPadLabel;               // Label at beginning of landing pad.
   const Function *Personality;             // Personality function.
   std::vector<int> TypeIds;               // List of type ids (filters negative).
@@ -148,11 +157,6 @@ class MachineModuleInfo : public ImmutablePass {
   /// emit common EH frames.
   std::vector<const Function *> Personalities;
 
-  /// UsedFunctions - The functions in the @llvm.used list in a more easily
-  /// searchable format.  This does not include the functions in
-  /// llvm.compiler.used.
-  SmallPtrSet<const Function *, 32> UsedFunctions;
-
   /// AddrLabelSymbols - This map keeps track of which symbol is being used for
   /// the specified basic block's address of label.
   MMIAddrLabelMap *AddrLabelSymbols;
@@ -184,13 +188,13 @@ class MachineModuleInfo : public ImmutablePass {
   static char ID; // Pass identification, replacement for typeid
 
   struct VariableDbgInfo {
-    const MDLocalVariable *Var;
-    const MDExpression *Expr;
+    const DILocalVariable *Var;
+    const DIExpression *Expr;
     unsigned Slot;
-    const MDLocation *Loc;
+    const DILocation *Loc;
 
-    VariableDbgInfo(const MDLocalVariable *Var, const MDExpression *Expr,
-                    unsigned Slot, const MDLocation *Loc)
+    VariableDbgInfo(const DILocalVariable *Var, const DIExpression *Expr,
+                    unsigned Slot, const DILocation *Loc)
         : Var(Var), Expr(Expr), Slot(Slot), Loc(Loc) {}
   };
   typedef SmallVector<VariableDbgInfo, 4> VariableDbgInfoMapTy;
@@ -237,10 +241,6 @@ class MachineModuleInfo : public ImmutablePass {
     return const_cast<MachineModuleInfo*>(this)->getObjFileInfo<Ty>();
   }
 
-  /// AnalyzeModule - Scan the module for global debug information.
-  ///
-  void AnalyzeModule(const Module &M);
-
   /// hasDebugInfo - Returns true if valid debug info is present.
   ///
   bool hasDebugInfo() const { return DbgInfoAvailable; }
@@ -284,12 +284,14 @@ class MachineModuleInfo : public ImmutablePass {
   /// getAddrLabelSymbol - Return the symbol to be used for the specified basic
   /// block when its address is taken.  This cannot be its normal LBB label
   /// because the block may be accessed outside its containing function.
-  MCSymbol *getAddrLabelSymbol(const BasicBlock *BB);
+  MCSymbol *getAddrLabelSymbol(const BasicBlock *BB) {
+    return getAddrLabelSymbolToEmit(BB).front();
+  }
 
   /// getAddrLabelSymbolToEmit - Return the symbol to be used for the specified
   /// basic block when its address is taken.  If other blocks were RAUW'd to
   /// this one, we may have to emit them as well, return the whole set.
-  std::vector<MCSymbol*> getAddrLabelSymbolToEmit(const BasicBlock *BB);
+  ArrayRef<MCSymbol *> getAddrLabelSymbolToEmit(const BasicBlock *BB);
 
   /// takeDeletedSymbolsForFunction - If the specified function has had any
   /// references to address-taken blocks generated, but the block got deleted,
@@ -318,6 +320,7 @@ class MachineModuleInfo : public ImmutablePass {
   /// information.
   void addPersonality(MachineBasicBlock *LandingPad,
                       const Function *Personality);
+  void addPersonality(const Function *Personality);
 
   void addWinEHState(MachineBasicBlock *LandingPad, int State);
 
@@ -330,13 +333,6 @@ class MachineModuleInfo : public ImmutablePass {
     return Personalities;
   }
 
-  /// isUsedFunction - Return true if the functions in the llvm.used list.  This
-  /// does not return true for things in llvm.compiler.used unless they are also
-  /// in llvm.used.
-  bool isUsedFunction(const Function *F) const {
-    return UsedFunctions.count(F);
-  }
-
   /// addCatchTypeInfo - Provide the catch typeinfo for a landing pad.
   ///
   void addCatchTypeInfo(MachineBasicBlock *LandingPad,
@@ -351,10 +347,11 @@ class MachineModuleInfo : public ImmutablePass {
   ///
   void addCleanup(MachineBasicBlock *LandingPad);
 
-  /// Add a clause for a landing pad. Returns a new label for the clause. This
-  /// is used by EH schemes that have more than one landing pad. In this case,
-  /// each clause gets its own basic block.
-  MCSymbol *addClauseForLandingPad(MachineBasicBlock *LandingPad);
+  void addSEHCatchHandler(MachineBasicBlock *LandingPad, const Function *Filter,
+                          const BlockAddress *RecoverLabel);
+
+  void addSEHCleanupHandler(MachineBasicBlock *LandingPad,
+                            const Function *Cleanup);
 
   /// getTypeIDFor - Return the type id for the specified typeinfo.  This is
   /// function wide.
@@ -438,8 +435,8 @@ class MachineModuleInfo : public ImmutablePass {
 
   /// setVariableDbgInfo - Collect information used to emit debugging
   /// information of a variable.
-  void setVariableDbgInfo(const MDLocalVariable *Var, const MDExpression *Expr,
-                          unsigned Slot, const MDLocation *Loc) {
+  void setVariableDbgInfo(const DILocalVariable *Var, const DIExpression *Expr,
+                          unsigned Slot, const DILocation *Loc) {
     VariableDbgInfos.emplace_back(Var, Expr, Slot, Loc);
   }
 
diff --git a/include/llvm/CodeGen/MachineOperand.h b/include/llvm/CodeGen/MachineOperand.h
index 04d5361ed5d..c43e47c36d0 100644
--- a/include/llvm/CodeGen/MachineOperand.h
+++ b/include/llvm/CodeGen/MachineOperand.h
@@ -27,6 +27,7 @@ class MachineBasicBlock;
 class MachineInstr;
 class MachineRegisterInfo;
 class MDNode;
+class ModuleSlotTracker;
 class TargetMachine;
 class TargetRegisterInfo;
 class hash_code;
@@ -65,7 +66,7 @@ class MachineOperand {
 private:
   /// OpKind - Specify what kind of operand this is.  This discriminates the
   /// union.
-  MachineOperandType OpKind;
+  MachineOperandType OpKind : 8;
 
   /// Subregister number for MO_Register.  A value of 0 indicates the
   /// MO_Register has no subReg.
@@ -218,6 +219,8 @@ class MachineOperand {
   void clearParent() { ParentMI = nullptr; }
 
   void print(raw_ostream &os, const TargetRegisterInfo *TRI = nullptr) const;
+  void print(raw_ostream &os, ModuleSlotTracker &MST,
+             const TargetRegisterInfo *TRI = nullptr) const;
 
   //===--------------------------------------------------------------------===//
   // Accessors that tell you what kind of MachineOperand you're looking at.
@@ -450,11 +453,12 @@ class MachineOperand {
     return Contents.CFIIndex;
   }
 
-  /// getOffset - Return the offset from the symbol in this operand. This always
-  /// returns 0 for ExternalSymbol operands.
+  /// Return the offset from the symbol in this operand. This always returns 0
+  /// for ExternalSymbol operands.
   int64_t getOffset() const {
-    assert((isGlobal() || isSymbol() || isCPI() || isTargetIndex() ||
-            isBlockAddress()) && "Wrong MachineOperand accessor");
+    assert((isGlobal() || isSymbol() || isMCSymbol() || isCPI() ||
+            isTargetIndex() || isBlockAddress()) &&
+           "Wrong MachineOperand accessor");
     return int64_t(uint64_t(Contents.OffsetedInfo.OffsetHi) << 32) |
            SmallContents.OffsetLo;
   }
@@ -512,8 +516,9 @@ class MachineOperand {
   }
 
   void setOffset(int64_t Offset) {
-    assert((isGlobal() || isSymbol() || isCPI() || isTargetIndex() ||
-            isBlockAddress()) && "Wrong MachineOperand accessor");
+    assert((isGlobal() || isSymbol() || isMCSymbol() || isCPI() ||
+            isTargetIndex() || isBlockAddress()) &&
+           "Wrong MachineOperand accessor");
     SmallContents.OffsetLo = unsigned(Offset);
     Contents.OffsetedInfo.OffsetHi = int(Offset >> 32);
   }
@@ -554,6 +559,12 @@ class MachineOperand {
   /// already, the setFPImm method should be used.
   void ChangeToFPImmediate(const ConstantFP *FPImm);
 
+  /// ChangeToES - Replace this operand with a new external symbol operand.
+  void ChangeToES(const char *SymName, unsigned char TargetFlags = 0);
+
+  /// ChangeToMCSymbol - Replace this operand with a new MC symbol operand.
+  void ChangeToMCSymbol(MCSymbol *Sym);
+
   /// ChangeToRegister - Replace this operand with a new register operand of
   /// the specified value.  If an operand is known to be an register already,
   /// the setReg method should be used.
@@ -697,9 +708,12 @@ class MachineOperand {
     return Op;
   }
 
-  static MachineOperand CreateMCSymbol(MCSymbol *Sym) {
+  static MachineOperand CreateMCSymbol(MCSymbol *Sym,
+                                       unsigned char TargetFlags = 0) {
     MachineOperand Op(MachineOperand::MO_MCSymbol);
     Op.Contents.Sym = Sym;
+    Op.setOffset(0);
+    Op.setTargetFlags(TargetFlags);
     return Op;
   }
 
diff --git a/include/llvm/CodeGen/MachineRegionInfo.h b/include/llvm/CodeGen/MachineRegionInfo.h
index cf49c297c28..df9823f741d 100644
--- a/include/llvm/CodeGen/MachineRegionInfo.h
+++ b/include/llvm/CodeGen/MachineRegionInfo.h
@@ -172,10 +172,9 @@ template <> struct GraphTraits<MachineRegionInfoPass*>
   }
 };
 
-EXTERN_TEMPLATE_INSTANTIATION(class RegionBase<RegionTraits<MachineFunction>>);
-EXTERN_TEMPLATE_INSTANTIATION(class RegionNodeBase<RegionTraits<MachineFunction>>);
-EXTERN_TEMPLATE_INSTANTIATION(class RegionInfoBase<RegionTraits<MachineFunction>>);
-
+extern template class RegionBase<RegionTraits<MachineFunction>>;
+extern template class RegionNodeBase<RegionTraits<MachineFunction>>;
+extern template class RegionInfoBase<RegionTraits<MachineFunction>>;
 }
 
 #endif
diff --git a/include/llvm/CodeGen/MachineRegisterInfo.h b/include/llvm/CodeGen/MachineRegisterInfo.h
index e5b837aeea2..2f823b44801 100644
--- a/include/llvm/CodeGen/MachineRegisterInfo.h
+++ b/include/llvm/CodeGen/MachineRegisterInfo.h
@@ -95,20 +95,8 @@ class MachineRegisterInfo {
     return MO->Contents.Reg.Next;
   }
 
-  /// UsedRegUnits - This is a bit vector that is computed and set by the
-  /// register allocator, and must be kept up to date by passes that run after
-  /// register allocation (though most don't modify this).  This is used
-  /// so that the code generator knows which callee save registers to save and
-  /// for other target specific uses.
-  /// This vector has bits set for register units that are modified in the
-  /// current function. It doesn't include registers clobbered by function
-  /// calls with register mask operands.
-  BitVector UsedRegUnits;
-
   /// UsedPhysRegMask - Additional used physregs including aliases.
   /// This bit vector represents all the registers clobbered by function calls.
-  /// It can model things that UsedRegUnits can't, such as function calls that
-  /// clobber ymm7 but preserve the low half in xmm7.
   BitVector UsedPhysRegMask;
 
   /// ReservedRegs - This is a bit vector of reserved registers.  The target
@@ -626,6 +614,12 @@ class MachineRegisterInfo {
     RegAllocHints[VReg].second = PrefReg;
   }
 
+  /// Specify the preferred register allocation hint for the specified virtual
+  /// register.
+  void setSimpleHint(unsigned VReg, unsigned PrefReg) {
+    setRegAllocationHint(VReg, /*Type=*/0, PrefReg);
+  }
+
   /// getRegAllocationHint - Return the register allocation hint for the
   /// specified virtual register.
   std::pair<unsigned, unsigned>
@@ -647,40 +641,11 @@ class MachineRegisterInfo {
   /// deleted during LiveDebugVariables analysis.
   void markUsesInDebugValueAsUndef(unsigned Reg) const;
 
-  //===--------------------------------------------------------------------===//
-  // Physical Register Use Info
-  //===--------------------------------------------------------------------===//
-
-  /// isPhysRegUsed - Return true if the specified register is used in this
-  /// function. Also check for clobbered aliases and registers clobbered by
-  /// function calls with register mask operands.
-  ///
-  /// This only works after register allocation. It is primarily used by
-  /// PrologEpilogInserter to determine which callee-saved registers need
-  /// spilling.
-  bool isPhysRegUsed(unsigned Reg) const {
-    if (UsedPhysRegMask.test(Reg))
-      return true;
-    for (MCRegUnitIterator Units(Reg, getTargetRegisterInfo());
-         Units.isValid(); ++Units)
-      if (UsedRegUnits.test(*Units))
-        return true;
-    return false;
-  }
-
-  /// Mark the specified register unit as used in this function.
-  /// This should only be called during and after register allocation.
-  void setRegUnitUsed(unsigned RegUnit) {
-    UsedRegUnits.set(RegUnit);
-  }
-
-  /// setPhysRegUsed - Mark the specified register used in this function.
-  /// This should only be called during and after register allocation.
-  void setPhysRegUsed(unsigned Reg) {
-    for (MCRegUnitIterator Units(Reg, getTargetRegisterInfo());
-         Units.isValid(); ++Units)
-      UsedRegUnits.set(*Units);
-  }
+  /// Return true if the specified register is modified in this function.
+  /// This checks that no defining machine operands exist for the register or
+  /// any of its aliases. Definitions found on functions marked noreturn are
+  /// ignored.
+  bool isPhysRegModified(unsigned PhysReg) const;
 
   /// addPhysRegsUsedFromRegMask - Mark any registers not in RegMask as used.
   /// This corresponds to the bit mask attached to register mask operands.
@@ -688,16 +653,6 @@ class MachineRegisterInfo {
     UsedPhysRegMask.setBitsNotInMask(RegMask);
   }
 
-  /// setPhysRegUnused - Mark the specified register unused in this function.
-  /// This should only be called during and after register allocation.
-  void setPhysRegUnused(unsigned Reg) {
-    UsedPhysRegMask.reset(Reg);
-    for (MCRegUnitIterator Units(Reg, getTargetRegisterInfo());
-         Units.isValid(); ++Units)
-      UsedRegUnits.reset(*Units);
-  }
-
-
   //===--------------------------------------------------------------------===//
   // Reserved Register Info
   //===--------------------------------------------------------------------===//
diff --git a/include/llvm/CodeGen/MachineScheduler.h b/include/llvm/CodeGen/MachineScheduler.h
index e80e14e5ccf..dee2b7107a4 100644
--- a/include/llvm/CodeGen/MachineScheduler.h
+++ b/include/llvm/CodeGen/MachineScheduler.h
@@ -915,7 +915,7 @@ class PostGenericScheduler : public GenericSchedulerBase {
                   MachineBasicBlock::iterator End,
                   unsigned NumRegionInstrs) override {
     /* no configurable policy */
-  };
+  }
 
   /// PostRA scheduling does not track pressure.
   bool shouldTrackPressure() const override { return false; }
diff --git a/include/llvm/CodeGen/MachineValueType.h b/include/llvm/CodeGen/MachineValueType.h
index 3eb4d0b85de..a728df35467 100644
--- a/include/llvm/CodeGen/MachineValueType.h
+++ b/include/llvm/CodeGen/MachineValueType.h
@@ -26,7 +26,7 @@ namespace llvm {
   /// MVT - Machine Value Type. Every type that is supported natively by some
   /// processor targeted by LLVM occurs here. This means that any legal value
   /// type can be represented by an MVT.
-  class MVT {
+class MVT {
   public:
     enum SimpleValueType {
       // INVALID_SIMPLE_VALUE_TYPE - Simple value types less than zero are
@@ -86,22 +86,23 @@ namespace llvm {
       v4i64          =  39,   //  4 x i64
       v8i64          =  40,   //  8 x i64
       v16i64         =  41,   // 16 x i64
-
+      v1i128         =  42,   //  1 x i128
+      
       FIRST_INTEGER_VECTOR_VALUETYPE = v2i1,
-      LAST_INTEGER_VECTOR_VALUETYPE = v16i64,
-
-      v2f16          =  42,   //  2 x f16
-      v4f16          =  43,   //  4 x f16
-      v8f16          =  44,   //  8 x f16
-      v1f32          =  45,   //  1 x f32
-      v2f32          =  46,   //  2 x f32
-      v4f32          =  47,   //  4 x f32
-      v8f32          =  48,   //  8 x f32
-      v16f32         =  49,   // 16 x f32
-      v1f64          =  50,   //  1 x f64
-      v2f64          =  51,   //  2 x f64
-      v4f64          =  52,   //  4 x f64
-      v8f64          =  53,   //  8 x f64
+      LAST_INTEGER_VECTOR_VALUETYPE = v1i128,
+
+      v2f16          =  43,   //  2 x f16
+      v4f16          =  44,   //  4 x f16
+      v8f16          =  45,   //  8 x f16
+      v1f32          =  46,   //  1 x f32
+      v2f32          =  47,   //  2 x f32
+      v4f32          =  48,   //  4 x f32
+      v8f32          =  49,   //  8 x f32
+      v16f32         =  50,   // 16 x f32
+      v1f64          =  51,   //  1 x f64
+      v2f64          =  52,   //  2 x f64
+      v4f64          =  53,   //  4 x f64
+      v8f64          =  54,   //  8 x f64
 
       FIRST_FP_VECTOR_VALUETYPE = v2f16,
       LAST_FP_VECTOR_VALUETYPE = v8f64,
@@ -109,18 +110,18 @@ namespace llvm {
       FIRST_VECTOR_VALUETYPE = v2i1,
       LAST_VECTOR_VALUETYPE  = v8f64,
 
-      x86mmx         =  54,   // This is an X86 MMX value
+      x86mmx         =  55,   // This is an X86 MMX value
 
-      Glue           =  55,   // This glues nodes together during pre-RA sched
+      Glue           =  56,   // This glues nodes together during pre-RA sched
 
-      isVoid         =  56,   // This has no value
+      isVoid         =  57,   // This has no value
 
-      Untyped        =  57,   // This value takes a register, but has
+      Untyped        =  58,   // This value takes a register, but has
                               // unspecified type.  The register class
                               // will be determined by the opcode.
 
       FIRST_VALUETYPE = 0,    // This is always the beginning of the list.
-      LAST_VALUETYPE =  58,   // This always remains at the end of the list.
+      LAST_VALUETYPE =  59,   // This always remains at the end of the list.
 
       // This is the current maximum for LAST_VALUETYPE.
       // MVT::MAX_ALLOWED_VALUETYPE is used for asserts and to size bit vectors
@@ -222,10 +223,10 @@ namespace llvm {
 
     /// is128BitVector - Return true if this is a 128-bit vector type.
     bool is128BitVector() const {
-      return (SimpleTy == MVT::v16i8 || SimpleTy == MVT::v8i16 ||
-              SimpleTy == MVT::v4i32 || SimpleTy == MVT::v2i64 ||
-              SimpleTy == MVT::v8f16 || SimpleTy == MVT::v4f32 ||
-              SimpleTy == MVT::v2f64);
+      return (SimpleTy == MVT::v16i8  || SimpleTy == MVT::v8i16 ||
+              SimpleTy == MVT::v4i32  || SimpleTy == MVT::v2i64 ||
+              SimpleTy == MVT::v1i128 || SimpleTy == MVT::v8f16 ||
+              SimpleTy == MVT::v4f32  || SimpleTy == MVT::v2f64);
     }
 
     /// is256BitVector - Return true if this is a 256-bit vector type.
@@ -310,6 +311,7 @@ namespace llvm {
       case v4i64:
       case v8i64:
       case v16i64: return i64;
+      case v1i128: return i128;
       case v2f16:
       case v4f16:
       case v8f16: return f16;
@@ -368,6 +370,7 @@ namespace llvm {
       case v1i16:
       case v1i32:
       case v1i64:
+      case v1i128:
       case v1f32:
       case v1f64: return 1;
       }
@@ -427,6 +430,7 @@ namespace llvm {
       case v8i16:
       case v4i32:
       case v2i64:
+      case v1i128:
       case v8f16:
       case v4f32:
       case v2f64: return 128;
@@ -562,6 +566,9 @@ namespace llvm {
         if (NumElements == 8)  return MVT::v8i64;
         if (NumElements == 16) return MVT::v16i64;
         break;
+      case MVT::i128:
+        if (NumElements == 1)  return MVT::v1i128;
+        break;
       case MVT::f16:
         if (NumElements == 2)  return MVT::v2f16;
         if (NumElements == 4)  return MVT::v4f16;
diff --git a/include/llvm/CodeGen/PBQP/Graph.h b/include/llvm/CodeGen/PBQP/Graph.h
index efb723cab39..f73383ed100 100644
--- a/include/llvm/CodeGen/PBQP/Graph.h
+++ b/include/llvm/CodeGen/PBQP/Graph.h
@@ -21,6 +21,7 @@
 #include <list>
 #include <map>
 #include <set>
+#include <vector>
 
 namespace llvm {
 namespace PBQP {
diff --git a/include/llvm/CodeGen/Passes.h b/include/llvm/CodeGen/Passes.h
index 2505c04a720..5d829217447 100644
--- a/include/llvm/CodeGen/Passes.h
+++ b/include/llvm/CodeGen/Passes.h
@@ -17,11 +17,11 @@
 
 #include "llvm/Pass.h"
 #include "llvm/Target/TargetMachine.h"
+#include <functional>
 #include <string>
 
 namespace llvm {
 
-class FunctionPass;
 class MachineFunctionPass;
 class PassConfigImpl;
 class PassInfo;
@@ -101,7 +101,7 @@ class TargetPassConfig : public ImmutablePass {
 
 private:
   PassManagerBase *PM;
-  AnalysisID StartAfter;
+  AnalysisID StartBefore, StartAfter;
   AnalysisID StopAfter;
   bool Started;
   bool Stopped;
@@ -120,6 +120,9 @@ class TargetPassConfig : public ImmutablePass {
   /// Default setting for -enable-tail-merge on this target.
   bool EnableTailMerge;
 
+  /// Default setting for -enable-shrink-wrap on this target.
+  bool EnableShrinkWrap;
+
 public:
   TargetPassConfig(TargetMachine *tm, PassManagerBase &pm);
   // Dummy constructor.
@@ -139,16 +142,24 @@ class TargetPassConfig : public ImmutablePass {
 
   CodeGenOpt::Level getOptLevel() const { return TM->getOptLevel(); }
 
-  /// setStartStopPasses - Set the StartAfter and StopAfter passes to allow
-  /// running only a portion of the normal code-gen pass sequence.  If the
-  /// Start pass ID is zero, then compilation will begin at the normal point;
-  /// otherwise, clear the Started flag to indicate that passes should not be
-  /// added until the starting pass is seen.  If the Stop pass ID is zero,
-  /// then compilation will continue to the end.
-  void setStartStopPasses(AnalysisID Start, AnalysisID Stop) {
-    StartAfter = Start;
-    StopAfter = Stop;
-    Started = (StartAfter == nullptr);
+  /// Set the StartAfter, StartBefore and StopAfter passes to allow running only
+  /// a portion of the normal code-gen pass sequence.
+  ///
+  /// If the StartAfter and StartBefore pass ID is zero, then compilation will
+  /// begin at the normal point; otherwise, clear the Started flag to indicate
+  /// that passes should not be added until the starting pass is seen.  If the
+  /// Stop pass ID is zero, then compilation will continue to the end.
+  ///
+  /// This function expects that at least one of the StartAfter or the
+  /// StartBefore pass IDs is null.
+  void setStartStopPasses(AnalysisID StartBefore, AnalysisID StartAfter,
+                          AnalysisID StopAfter) {
+    if (StartAfter)
+      assert(!StartBefore && "Start after and start before passes are given");
+    this->StartBefore = StartBefore;
+    this->StartAfter = StartAfter;
+    this->StopAfter = StopAfter;
+    Started = (StartAfter == nullptr) && (StartBefore == nullptr);
   }
 
   void setDisableVerify(bool Disable) { setOpt(DisableVerify, Disable); }
@@ -179,6 +190,9 @@ class TargetPassConfig : public ImmutablePass {
   /// Return true if the optimized regalloc pipeline is enabled.
   bool getOptimizeRegAlloc() const;
 
+  /// Return true if shrink wrapping is enabled.
+  bool getEnableShrinkWrap() const;
+
   /// Return true if the default global register allocator is in use and
   /// has not be overriden on the command line with '-regalloc=...'
   bool usingDefaultRegAlloc() const;
@@ -368,6 +382,10 @@ namespace llvm {
   createMachineFunctionPrinterPass(raw_ostream &OS,
                                    const std::string &Banner ="");
 
+  /// MIRPrinting pass - this pass prints out the LLVM IR into the given stream
+  /// using the MIR serialization format.
+  MachineFunctionPass *createPrintMIRPass(raw_ostream &OS);
+
   /// createCodeGenPreparePass - Transform the code to expose more pattern
   /// matching during instruction selection.
   FunctionPass *createCodeGenPreparePass(const TargetMachine *TM = nullptr);
@@ -426,6 +444,10 @@ namespace llvm {
   /// basic blocks.
   extern char &SpillPlacementID;
 
+  /// ShrinkWrap pass. Look for the best place to insert save and restore
+  // instruction and update the MachineFunctionInfo with that information.
+  extern char &ShrinkWrapID;
+
   /// VirtRegRewriter pass. Rewrite virtual registers to physical registers as
   /// assigned in VirtRegMap.
   extern char &VirtRegRewriterID;
@@ -478,6 +500,10 @@ namespace llvm {
   /// MachineFunctionPrinterPass - This pass prints out MachineInstr's.
   extern char &MachineFunctionPrinterPassID;
 
+  /// MIRPrintingPass - this pass prints out the LLVM IR using the MIR
+  /// serialization format.
+  extern char &MIRPrintingPassID;
+
   /// TailDuplicate - Duplicate blocks with unconditional branches
   /// into tails of their predecessors.
   extern char &TailDuplicateID;
@@ -501,6 +527,8 @@ namespace llvm {
   /// IfConverter - This pass performs machine code if conversion.
   extern char &IfConverterID;
 
+  FunctionPass *createIfConverter(std::function<bool(const Function &)> Ftor);
+
   /// MachineBlockPlacement - This pass places basic blocks based on branch
   /// probabilities.
   extern char &MachineBlockPlacementID;
@@ -532,6 +560,10 @@ namespace llvm {
   /// MachineCSE - This pass performs global CSE on machine instructions.
   extern char &MachineCSEID;
 
+  /// ImplicitNullChecks - This pass folds null pointer checks into nearby
+  /// memory operations.
+  extern char &ImplicitNullChecksID;
+
   /// MachineLICM - This pass performs LICM on machine instructions.
   extern char &MachineLICMID;
 
@@ -573,7 +605,7 @@ namespace llvm {
   /// createSjLjEHPreparePass - This pass adapts exception handling code to use
   /// the GCC-style builtin setjmp/longjmp (sjlj) to handling EH control flow.
   ///
-  FunctionPass *createSjLjEHPreparePass(const TargetMachine *TM);
+  FunctionPass *createSjLjEHPreparePass();
 
   /// LocalStackSlotAllocation - This pass assigns local frame indices to stack
   /// slots relative to one another and allocates base registers to access them
@@ -595,6 +627,9 @@ namespace llvm {
   /// UnpackMachineBundles - This pass unpack machine instruction bundles.
   extern char &UnpackMachineBundlesID;
 
+  FunctionPass *
+  createUnpackMachineBundles(std::function<bool(const Function &)> Ftor);
+
   /// FinalizeMachineBundles - This pass finalize machine instruction
   /// bundles (created earlier, e.g. during pre-RA scheduling).
   extern char &FinalizeMachineBundlesID;
@@ -610,6 +645,11 @@ namespace llvm {
   /// createForwardControlFlowIntegrityPass - This pass adds control-flow
   /// integrity.
   ModulePass *createForwardControlFlowIntegrityPass();
+
+  /// InterleavedAccess Pass - This pass identifies and matches interleaved
+  /// memory accesses to target specific intrinsics.
+  ///
+  FunctionPass *createInterleavedAccessPass(const TargetMachine *TM);
 } // End llvm namespace
 
 /// Target machine pass initializer for passes with dependencies. Use with
diff --git a/include/llvm/CodeGen/PseudoSourceValue.h b/include/llvm/CodeGen/PseudoSourceValue.h
index cc3e25aeb89..a518b623325 100644
--- a/include/llvm/CodeGen/PseudoSourceValue.h
+++ b/include/llvm/CodeGen/PseudoSourceValue.h
@@ -29,8 +29,7 @@ namespace llvm {
   /// space), or constant pool.
   class PseudoSourceValue {
   private:
-    friend raw_ostream &llvm::operator<<(raw_ostream &OS,
-                                         const MachineMemOperand &MMO);
+    friend class MachineMemOperand; // For printCustom().
 
     /// printCustom - Implement printing for PseudoSourceValue. This is called
     /// from Value::print or Value's operator<<.
diff --git a/include/llvm/CodeGen/RegisterPressure.h b/include/llvm/CodeGen/RegisterPressure.h
index fcb6feed68c..9d8843d1d74 100644
--- a/include/llvm/CodeGen/RegisterPressure.h
+++ b/include/llvm/CodeGen/RegisterPressure.h
@@ -135,6 +135,8 @@ class PressureDiff {
 
   void addPressureChange(unsigned RegUnit, bool IsDec,
                          const MachineRegisterInfo *MRI);
+
+  LLVM_DUMP_METHOD void dump(const TargetRegisterInfo &TRI) const;
 };
 
 /// Array of PressureDiffs.
diff --git a/include/llvm/CodeGen/RegisterScavenging.h b/include/llvm/CodeGen/RegisterScavenging.h
index 474861e45df..df3fd34e0af 100644
--- a/include/llvm/CodeGen/RegisterScavenging.h
+++ b/include/llvm/CodeGen/RegisterScavenging.h
@@ -9,7 +9,7 @@
 //
 // This file declares the machine register scavenger class. It can provide
 // information such as unused register at any point in a machine basic block.
-// It also provides a mechanism to make registers availbale by evicting them
+// It also provides a mechanism to make registers available by evicting them
 // to spill slots.
 //
 //===----------------------------------------------------------------------===//
@@ -36,8 +36,7 @@ class RegScavenger {
   MachineBasicBlock::iterator MBBI;
   unsigned NumRegUnits;
 
-  /// Tracking - True if RegScavenger is currently tracking the liveness of 
-  /// registers.
+  /// True if RegScavenger is currently tracking the liveness of registers.
   bool Tracking;
 
   /// Information on scavenged registers (held in a spill slot).
@@ -58,8 +57,8 @@ class RegScavenger {
   /// A vector of information on scavenged registers.
   SmallVector<ScavengedInfo, 2> Scavenged;
 
-  /// RegUnitsAvailable - The current state of each reg unit immediatelly
-  /// before MBBI. One bit per register unit. If bit is not set it means any
+  /// The current state of each reg unit immediately before MBBI.
+  /// One bit per register unit. If bit is not set it means any
   /// register containing that register unit is currently being used.
   BitVector RegUnitsAvailable;
 
@@ -72,18 +71,17 @@ class RegScavenger {
   RegScavenger()
     : MBB(nullptr), NumRegUnits(0), Tracking(false) {}
 
-  /// enterBasicBlock - Start tracking liveness from the begin of the specific
-  /// basic block.
+  /// Start tracking liveness from the begin of the specific basic block.
   void enterBasicBlock(MachineBasicBlock *mbb);
 
-  /// initRegState - allow resetting register state info for multiple
+  /// Allow resetting register state info for multiple
   /// passes over/within the same function.
   void initRegState();
 
-  /// forward - Move the internal MBB iterator and update register states.
+  /// Move the internal MBB iterator and update register states.
   void forward();
 
-  /// forward - Move the internal MBB iterator and update register states until
+  /// Move the internal MBB iterator and update register states until
   /// it has processed the specific iterator.
   void forward(MachineBasicBlock::iterator I) {
     if (!Tracking && MBB->begin() != I) forward();
@@ -99,7 +97,7 @@ class RegScavenger {
     while (MBBI != I) unprocess();
   }
 
-  /// skipTo - Move the internal MBB iterator but do not update register states.
+  /// Move the internal MBB iterator but do not update register states.
   void skipTo(MachineBasicBlock::iterator I) {
     if (I == MachineBasicBlock::iterator(nullptr))
       Tracking = false;
@@ -110,14 +108,13 @@ class RegScavenger {
     return MBBI;
   }
   
-  /// isRegUsed - return if a specific register is currently used.
+  /// Return if a specific register is currently used.
   bool isRegUsed(unsigned Reg, bool includeReserved = true) const;
 
-  /// getRegsAvailable - Return all available registers in the register class
-  /// in Mask.
+  /// Return all available registers in the register class in Mask.
   BitVector getRegsAvailable(const TargetRegisterClass *RC);
 
-  /// FindUnusedReg - Find a unused register of the specified register class.
+  /// Find an unused register of the specified register class.
   /// Return 0 if none is found.
   unsigned FindUnusedReg(const TargetRegisterClass *RegClass) const;
 
@@ -144,7 +141,7 @@ class RegScavenger {
         A.push_back(I->FrameIndex);
   }
 
-  /// scavengeRegister - Make a register of the specific register class
+  /// Make a register of the specific register class
   /// available and do the appropriate bookkeeping. SPAdj is the stack
   /// adjustment due to call frame, it's passed along to eliminateFrameIndex().
   /// Returns the scavenged register.
@@ -154,11 +151,10 @@ class RegScavenger {
     return scavengeRegister(RegClass, MBBI, SPAdj);
   }
 
-  /// setRegUsed - Tell the scavenger a register is used.
-  ///
+  /// Tell the scavenger a register is used.
   void setRegUsed(unsigned Reg);
 private:
-  /// isReserved - Returns true if a register is reserved. It is never "unused".
+  /// Returns true if a register is reserved. It is never "unused".
   bool isReserved(unsigned Reg) const { return MRI->isReserved(Reg); }
 
   /// setUsed / setUnused - Mark the state of one or a number of register units.
@@ -177,9 +173,8 @@ class RegScavenger {
   /// Add all Reg Units that Reg contains to BV.
   void addRegUnits(BitVector &BV, unsigned Reg);
   
-  /// findSurvivorReg - Return the candidate register that is unused for the
-  /// longest after StartMI. UseMI is set to the instruction where the search
-  /// stopped.
+  /// Return the candidate register that is unused for the longest after
+  /// StartMI. UseMI is set to the instruction where the search stopped.
   ///
   /// No more than InstrLimit instructions are inspected.
   unsigned findSurvivorReg(MachineBasicBlock::iterator StartMI,
diff --git a/include/llvm/CodeGen/ScheduleDAG.h b/include/llvm/CodeGen/ScheduleDAG.h
index 83913141656..539756ef502 100644
--- a/include/llvm/CodeGen/ScheduleDAG.h
+++ b/include/llvm/CodeGen/ScheduleDAG.h
@@ -374,7 +374,7 @@ namespace llvm {
     /// correspond to schedulable entities (e.g. instructions) and do not have a
     /// valid ID. Consequently, always check for boundary nodes before accessing
     /// an assoicative data structure keyed on node ID.
-    bool isBoundaryNode() const { return NodeNum == BoundaryID; };
+    bool isBoundaryNode() const { return NodeNum == BoundaryID; }
 
     /// setNode - Assign the representative SDNode for this SUnit.
     /// This may be used during pre-regalloc scheduling.
diff --git a/include/llvm/CodeGen/ScheduleDAGInstrs.h b/include/llvm/CodeGen/ScheduleDAGInstrs.h
index 1196783e820..b56d5ec8ce6 100644
--- a/include/llvm/CodeGen/ScheduleDAGInstrs.h
+++ b/include/llvm/CodeGen/ScheduleDAGInstrs.h
@@ -260,7 +260,7 @@ namespace llvm {
 #ifndef NDEBUG
     const SUnit *Addr = SUnits.empty() ? nullptr : &SUnits[0];
 #endif
-    SUnits.push_back(SUnit(MI, (unsigned)SUnits.size()));
+    SUnits.emplace_back(MI, (unsigned)SUnits.size());
     assert((Addr == nullptr || Addr == &SUnits[0]) &&
            "SUnits std::vector reallocated on the fly!");
     SUnits.back().OrigNode = &SUnits.back();
diff --git a/include/llvm/CodeGen/SchedulerRegistry.h b/include/llvm/CodeGen/SchedulerRegistry.h
index 51ac7f28527..a7a6227664d 100644
--- a/include/llvm/CodeGen/SchedulerRegistry.h
+++ b/include/llvm/CodeGen/SchedulerRegistry.h
@@ -52,12 +52,6 @@ class RegisterScheduler : public MachinePassRegistryNode {
   static RegisterScheduler *getList() {
     return (RegisterScheduler *)Registry.getList();
   }
-  static FunctionPassCtor getDefault() {
-    return (FunctionPassCtor)Registry.getDefault();
-  }
-  static void setDefault(FunctionPassCtor C) {
-    Registry.setDefault((MachinePassCtor)C);
-  }
   static void setListener(MachinePassRegistryListener *L) {
     Registry.setListener(L);
   }
diff --git a/include/llvm/CodeGen/SelectionDAG.h b/include/llvm/CodeGen/SelectionDAG.h
index 582febda085..1ee92380e69 100644
--- a/include/llvm/CodeGen/SelectionDAG.h
+++ b/include/llvm/CodeGen/SelectionDAG.h
@@ -112,6 +112,7 @@ template<> struct ilist_traits<SDNode> : public ilist_default_traits<SDNode> {
 /// motion, and debug info for them is potentially useful even if the parameter
 /// is unused.  Right now only byval parameters are handled separately.
 class SDDbgInfo {
+  BumpPtrAllocator Alloc;
   SmallVector<SDDbgValue*, 32> DbgValues;
   SmallVector<SDDbgValue*, 32> ByvalParmDbgValues;
   typedef DenseMap<const SDNode*, SmallVector<SDDbgValue*, 2> > DbgValMapType;
@@ -138,8 +139,11 @@ class SDDbgInfo {
     DbgValMap.clear();
     DbgValues.clear();
     ByvalParmDbgValues.clear();
+    Alloc.Reset();
   }
 
+  BumpPtrAllocator &getAlloc() { return Alloc; }
+
   bool empty() const {
     return DbgValues.empty() && ByvalParmDbgValues.empty();
   }
@@ -277,6 +281,7 @@ class SelectionDAG {
   void clear();
 
   MachineFunction &getMachineFunction() const { return *MF; }
+  const DataLayout &getDataLayout() const { return MF->getDataLayout(); }
   const TargetMachine &getTarget() const { return TM; }
   const TargetSubtargetInfo &getSubtarget() const { return MF->getSubtarget(); }
   const TargetLowering &getTargetLoweringInfo() const { return *TLI; }
@@ -318,6 +323,14 @@ class SelectionDAG {
     return AllNodes.size();
   }
 
+  iterator_range<allnodes_iterator> allnodes() {
+    return iterator_range<allnodes_iterator>(allnodes_begin(), allnodes_end());
+  }
+  iterator_range<allnodes_const_iterator> allnodes() const {
+    return iterator_range<allnodes_const_iterator>(allnodes_begin(),
+                                                   allnodes_end());
+  }
+
   /// Return the root tag of the SelectionDAG.
   const SDValue &getRoot() const { return Root; }
 
@@ -413,36 +426,40 @@ class SelectionDAG {
   //===--------------------------------------------------------------------===//
   // Node creation methods.
   //
-  SDValue getConstant(uint64_t Val, EVT VT, bool isTarget = false,
+  SDValue getConstant(uint64_t Val, SDLoc DL, EVT VT, bool isTarget = false,
                       bool isOpaque = false);
-  SDValue getConstant(const APInt &Val, EVT VT, bool isTarget = false,
+  SDValue getConstant(const APInt &Val, SDLoc DL, EVT VT, bool isTarget = false,
                       bool isOpaque = false);
-  SDValue getConstant(const ConstantInt &Val, EVT VT, bool isTarget = false,
-                      bool isOpaque = false);
-  SDValue getIntPtrConstant(uint64_t Val, bool isTarget = false);
-  SDValue getTargetConstant(uint64_t Val, EVT VT, bool isOpaque = false) {
-    return getConstant(Val, VT, true, isOpaque);
+  SDValue getConstant(const ConstantInt &Val, SDLoc DL, EVT VT,
+                      bool isTarget = false, bool isOpaque = false);
+  SDValue getIntPtrConstant(uint64_t Val, SDLoc DL, bool isTarget = false);
+  SDValue getTargetConstant(uint64_t Val, SDLoc DL, EVT VT,
+                            bool isOpaque = false) {
+    return getConstant(Val, DL, VT, true, isOpaque);
   }
-  SDValue getTargetConstant(const APInt &Val, EVT VT, bool isOpaque = false) {
-    return getConstant(Val, VT, true, isOpaque);
+  SDValue getTargetConstant(const APInt &Val, SDLoc DL, EVT VT,
+                            bool isOpaque = false) {
+    return getConstant(Val, DL, VT, true, isOpaque);
   }
-  SDValue getTargetConstant(const ConstantInt &Val, EVT VT,
+  SDValue getTargetConstant(const ConstantInt &Val, SDLoc DL, EVT VT,
                             bool isOpaque = false) {
-    return getConstant(Val, VT, true, isOpaque);
+    return getConstant(Val, DL, VT, true, isOpaque);
   }
   // The forms below that take a double should only be used for simple
   // constants that can be exactly represented in VT.  No checks are made.
-  SDValue getConstantFP(double Val, EVT VT, bool isTarget = false);
-  SDValue getConstantFP(const APFloat& Val, EVT VT, bool isTarget = false);
-  SDValue getConstantFP(const ConstantFP &CF, EVT VT, bool isTarget = false);
-  SDValue getTargetConstantFP(double Val, EVT VT) {
-    return getConstantFP(Val, VT, true);
+  SDValue getConstantFP(double Val, SDLoc DL, EVT VT, bool isTarget = false);
+  SDValue getConstantFP(const APFloat& Val, SDLoc DL, EVT VT,
+                        bool isTarget = false);
+  SDValue getConstantFP(const ConstantFP &CF, SDLoc DL, EVT VT,
+                        bool isTarget = false);
+  SDValue getTargetConstantFP(double Val, SDLoc DL, EVT VT) {
+    return getConstantFP(Val, DL, VT, true);
   }
-  SDValue getTargetConstantFP(const APFloat& Val, EVT VT) {
-    return getConstantFP(Val, VT, true);
+  SDValue getTargetConstantFP(const APFloat& Val, SDLoc DL, EVT VT) {
+    return getConstantFP(Val, DL, VT, true);
   }
-  SDValue getTargetConstantFP(const ConstantFP &Val, EVT VT) {
-    return getConstantFP(Val, VT, true);
+  SDValue getTargetConstantFP(const ConstantFP &Val, SDLoc DL, EVT VT) {
+    return getConstantFP(Val, DL, VT, true);
   }
   SDValue getGlobalAddress(const GlobalValue *GV, SDLoc DL, EVT VT,
                            int64_t offset = 0, bool isTargetGA = false,
@@ -487,6 +504,8 @@ class SelectionDAG {
   SDValue getExternalSymbol(const char *Sym, SDLoc dl, EVT VT);
   SDValue getTargetExternalSymbol(const char *Sym, EVT VT,
                                   unsigned char TargetFlags = 0);
+  SDValue getMCSymbol(MCSymbol *Sym, EVT VT);
+
   SDValue getValueType(EVT);
   SDValue getRegister(unsigned Reg, EVT VT);
   SDValue getRegisterMask(const uint32_t *RegMask);
@@ -648,35 +667,39 @@ class SelectionDAG {
 
   /// Gets or creates the specified node.
   ///
+  SDValue getNode(unsigned Opcode, SDLoc DL, EVT VT,
+                  ArrayRef<SDUse> Ops);
+  SDValue getNode(unsigned Opcode, SDLoc DL, EVT VT,
+                  ArrayRef<SDValue> Ops);
+  SDValue getNode(unsigned Opcode, SDLoc DL, ArrayRef<EVT> ResultTys,
+                  ArrayRef<SDValue> Ops);
+  SDValue getNode(unsigned Opcode, SDLoc DL, SDVTList VTs,
+                  ArrayRef<SDValue> Ops);
+
+  // Specialize based on number of operands.
   SDValue getNode(unsigned Opcode, SDLoc DL, EVT VT);
   SDValue getNode(unsigned Opcode, SDLoc DL, EVT VT, SDValue N);
   SDValue getNode(unsigned Opcode, SDLoc DL, EVT VT, SDValue N1, SDValue N2,
-                  bool nuw = false, bool nsw = false, bool exact = false);
+                  const SDNodeFlags *Flags = nullptr);
   SDValue getNode(unsigned Opcode, SDLoc DL, EVT VT, SDValue N1, SDValue N2,
                   SDValue N3);
   SDValue getNode(unsigned Opcode, SDLoc DL, EVT VT, SDValue N1, SDValue N2,
                   SDValue N3, SDValue N4);
   SDValue getNode(unsigned Opcode, SDLoc DL, EVT VT, SDValue N1, SDValue N2,
                   SDValue N3, SDValue N4, SDValue N5);
-  SDValue getNode(unsigned Opcode, SDLoc DL, EVT VT, ArrayRef<SDUse> Ops);
-  SDValue getNode(unsigned Opcode, SDLoc DL, EVT VT,
-                  ArrayRef<SDValue> Ops);
-  SDValue getNode(unsigned Opcode, SDLoc DL,
-                  ArrayRef<EVT> ResultTys,
-                  ArrayRef<SDValue> Ops);
-  SDValue getNode(unsigned Opcode, SDLoc DL, SDVTList VTs,
-                  ArrayRef<SDValue> Ops);
+  
+  // Specialize again based on number of operands for nodes with a VTList
+  // rather than a single VT.
   SDValue getNode(unsigned Opcode, SDLoc DL, SDVTList VTs);
   SDValue getNode(unsigned Opcode, SDLoc DL, SDVTList VTs, SDValue N);
-  SDValue getNode(unsigned Opcode, SDLoc DL, SDVTList VTs,
-                  SDValue N1, SDValue N2);
-  SDValue getNode(unsigned Opcode, SDLoc DL, SDVTList VTs,
-                  SDValue N1, SDValue N2, SDValue N3);
-  SDValue getNode(unsigned Opcode, SDLoc DL, SDVTList VTs,
-                  SDValue N1, SDValue N2, SDValue N3, SDValue N4);
-  SDValue getNode(unsigned Opcode, SDLoc DL, SDVTList VTs,
-                  SDValue N1, SDValue N2, SDValue N3, SDValue N4,
-                  SDValue N5);
+  SDValue getNode(unsigned Opcode, SDLoc DL, SDVTList VTs, SDValue N1,
+                  SDValue N2);
+  SDValue getNode(unsigned Opcode, SDLoc DL, SDVTList VTs, SDValue N1,
+                  SDValue N2, SDValue N3);
+  SDValue getNode(unsigned Opcode, SDLoc DL, SDVTList VTs, SDValue N1,
+                  SDValue N2, SDValue N3, SDValue N4);
+  SDValue getNode(unsigned Opcode, SDLoc DL, SDVTList VTs, SDValue N1,
+                  SDValue N2, SDValue N3, SDValue N4, SDValue N5);
 
   /// Compute a TokenFactor to force all the incoming stack arguments to be
   /// loaded from the stack. This is used in tail call lowering to protect
@@ -856,12 +879,20 @@ class SelectionDAG {
   SDValue getMaskedStore(SDValue Chain, SDLoc dl, SDValue Val,
                          SDValue Ptr, SDValue Mask, EVT MemVT,
                          MachineMemOperand *MMO, bool IsTrunc);
+  SDValue getMaskedGather(SDVTList VTs, EVT VT, SDLoc dl,
+                          ArrayRef<SDValue> Ops, MachineMemOperand *MMO);
+  SDValue getMaskedScatter(SDVTList VTs, EVT VT, SDLoc dl,
+                           ArrayRef<SDValue> Ops, MachineMemOperand *MMO);
   /// Construct a node to track a Value* through the backend.
   SDValue getSrcValue(const Value *v);
 
   /// Return an MDNodeSDNode which holds an MDNode.
   SDValue getMDNode(const MDNode *MD);
 
+  /// Return a bitcast using the SDLoc of the value operand, and casting to the
+  /// provided type. Use getNode to set a custom SDLoc.
+  SDValue getBitcast(EVT VT, SDValue V);
+
   /// Return an AddrSpaceCastSDNode.
   SDValue getAddrSpaceCast(SDLoc dl, EVT VT, SDValue Ptr,
                            unsigned SrcAS, unsigned DestAS);
@@ -970,8 +1001,7 @@ class SelectionDAG {
 
   /// Get the specified node if it's already available, or else return NULL.
   SDNode *getNodeIfExists(unsigned Opcode, SDVTList VTs, ArrayRef<SDValue> Ops,
-                          bool nuw = false, bool nsw = false,
-                          bool exact = false);
+                          const SDNodeFlags *Flags = nullptr);
 
   /// Creates a SDDbgValue node.
   SDDbgValue *getDbgValue(MDNode *Var, MDNode *Expr, SDNode *N, unsigned R,
@@ -1113,9 +1143,13 @@ class SelectionDAG {
   /// either of the specified value types.
   SDValue CreateStackTemporary(EVT VT1, EVT VT2);
 
-  SDValue FoldConstantArithmetic(unsigned Opcode, EVT VT,
+  SDValue FoldConstantArithmetic(unsigned Opcode, SDLoc DL, EVT VT,
                                  SDNode *Cst1, SDNode *Cst2);
 
+  SDValue FoldConstantArithmetic(unsigned Opcode, SDLoc DL, EVT VT,
+                                 const ConstantSDNode *Cst1,
+                                 const ConstantSDNode *Cst2);
+
   /// Constant fold a setcc to true or false.
   SDValue FoldSetCC(EVT VT, SDValue N1,
                     SDValue N2, ISD::CondCode Cond, SDLoc dl);
@@ -1229,8 +1263,20 @@ class SelectionDAG {
   void allnodes_clear();
 
   BinarySDNode *GetBinarySDNode(unsigned Opcode, SDLoc DL, SDVTList VTs,
-                                SDValue N1, SDValue N2, bool nuw, bool nsw,
-                                bool exact);
+                                SDValue N1, SDValue N2,
+                                const SDNodeFlags *Flags = nullptr);
+
+  /// Look up the node specified by ID in CSEMap.  If it exists, return it.  If
+  /// not, return the insertion token that will make insertion faster.  This
+  /// overload is for nodes other than Constant or ConstantFP, use the other one
+  /// for those.
+  SDNode *FindNodeOrInsertPos(const FoldingSetNodeID &ID, void *&InsertPos);
+
+  /// Look up the node specified by ID in CSEMap.  If it exists, return it.  If
+  /// not, return the insertion token that will make insertion faster.  Performs
+  /// additional processing for constant nodes.
+  SDNode *FindNodeOrInsertPos(const FoldingSetNodeID &ID, DebugLoc DL,
+                              void *&InsertPos);
 
   /// List of non-single value types.
   FoldingSet<SDVTListNode> VTListMap;
@@ -1243,6 +1289,7 @@ class SelectionDAG {
   StringMap<SDNode*> ExternalSymbols;
 
   std::map<std::pair<std::string, unsigned char>,SDNode*> TargetExternalSymbols;
+  DenseMap<MCSymbol *, SDNode *> MCSymbols;
 };
 
 template <> struct GraphTraits<SelectionDAG*> : public GraphTraits<SDNode*> {
diff --git a/include/llvm/CodeGen/SelectionDAGISel.h b/include/llvm/CodeGen/SelectionDAGISel.h
index a8743754f40..a011e4c338c 100644
--- a/include/llvm/CodeGen/SelectionDAGISel.h
+++ b/include/llvm/CodeGen/SelectionDAGISel.h
@@ -199,7 +199,7 @@ class SelectionDAGISel : public MachineFunctionPass {
 
   /// SelectInlineAsmMemoryOperands - Calls to this are automatically generated
   /// by tblgen.  Others should not call it.
-  void SelectInlineAsmMemoryOperands(std::vector<SDValue> &Ops);
+  void SelectInlineAsmMemoryOperands(std::vector<SDValue> &Ops, SDLoc DL);
 
 
 public:
@@ -260,7 +260,10 @@ class SelectionDAGISel : public MachineFunctionPass {
   SDNode *MorphNode(SDNode *Node, unsigned TargetOpc, SDVTList VTs,
                     ArrayRef<SDValue> Ops, unsigned EmitNodeInfo);
 
-  void PrepareEHLandingPad();
+  /// Prepares the landing pad to take incoming values or do other EH
+  /// personality specific tasks. Returns true if the block should be
+  /// instruction selected, false if no code should be emitted for it.
+  bool PrepareEHLandingPad();
 
   /// \brief Perform instruction selection on all basic blocks in the function.
   void SelectAllBasicBlocks(const Function &Fn);
diff --git a/include/llvm/CodeGen/SelectionDAGNodes.h b/include/llvm/CodeGen/SelectionDAGNodes.h
index 66f060c6b5a..4821d1aae9e 100644
--- a/include/llvm/CodeGen/SelectionDAGNodes.h
+++ b/include/llvm/CodeGen/SelectionDAGNodes.h
@@ -140,7 +140,7 @@ class SDValue {
   }
 
   // Return true if this node is an operand of N.
-  bool isOperandOf(SDNode *N) const;
+  bool isOperandOf(const SDNode *N) const;
 
   /// Return the ValueType of the referenced return value.
   inline EVT getValueType() const;
@@ -357,9 +357,6 @@ class SDNode : public FoldingSetNode, public ilist_node<SDNode> {
   /// The number of entries in the Operand/Value list.
   unsigned short NumOperands, NumValues;
 
-  /// Source line information.
-  DebugLoc debugLoc;
-
   // The ordering of the SDNodes. It roughly corresponds to the ordering of the
   // original LLVM instructions.
   // This is used for turning off scheduling, because we'll forgo
@@ -367,6 +364,9 @@ class SDNode : public FoldingSetNode, public ilist_node<SDNode> {
   // this ordering.
   unsigned IROrder;
 
+  /// Source line information.
+  DebugLoc debugLoc;
+
   /// Return a pointer to the specified value type.
   static const EVT *getValueTypeList(EVT VT);
 
@@ -532,10 +532,10 @@ class SDNode : public FoldingSetNode, public ilist_node<SDNode> {
   bool hasAnyUseOfValue(unsigned Value) const;
 
   /// Return true if this node is the only use of N.
-  bool isOnlyUserOf(SDNode *N) const;
+  bool isOnlyUserOf(const SDNode *N) const;
 
   /// Return true if this node is an operand of N.
-  bool isOperandOf(SDNode *N) const;
+  bool isOperandOf(const SDNode *N) const;
 
   /// Return true if this node is a predecessor of N.
   /// NOTE: Implemented on top of hasPredecessor and every bit as
@@ -579,6 +579,23 @@ class SDNode : public FoldingSetNode, public ilist_node<SDNode> {
   op_iterator op_end() const { return OperandList+NumOperands; }
   ArrayRef<SDUse> ops() const { return makeArrayRef(op_begin(), op_end()); }
 
+  /// Iterator for directly iterating over the operand SDValue's.
+  struct value_op_iterator
+      : iterator_adaptor_base<value_op_iterator, op_iterator,
+                              std::random_access_iterator_tag, SDValue,
+                              ptrdiff_t, value_op_iterator *,
+                              value_op_iterator *> {
+    explicit value_op_iterator(SDUse *U = nullptr)
+      : iterator_adaptor_base(U) {}
+
+    const SDValue &operator*() const { return I->get(); }
+  };
+
+  iterator_range<value_op_iterator> op_values() const {
+    return iterator_range<value_op_iterator>(value_op_iterator(op_begin()),
+                                             value_op_iterator(op_end()));
+  }
+
   SDVTList getVTList() const {
     SDVTList X = { ValueList, NumValues };
     return X;
@@ -715,7 +732,7 @@ class SDNode : public FoldingSetNode, public ilist_node<SDNode> {
         SubclassData(0), NodeId(-1),
         OperandList(Ops.size() ? new SDUse[Ops.size()] : nullptr),
         ValueList(VTs.VTs), UseList(nullptr), NumOperands(Ops.size()),
-        NumValues(VTs.NumVTs), debugLoc(std::move(dl)), IROrder(Order) {
+        NumValues(VTs.NumVTs), IROrder(Order), debugLoc(std::move(dl)) {
     assert(debugLoc.hasTrivialDestructor() && "Expected trivial destructor");
     assert(NumOperands == Ops.size() &&
            "NumOperands wasn't wide enough for its operands!");
@@ -735,7 +752,7 @@ class SDNode : public FoldingSetNode, public ilist_node<SDNode> {
       : NodeType(Opc), OperandsNeedDelete(false), HasDebugValue(false),
         SubclassData(0), NodeId(-1), OperandList(nullptr), ValueList(VTs.VTs),
         UseList(nullptr), NumOperands(0), NumValues(VTs.NumVTs),
-        debugLoc(std::move(dl)), IROrder(Order) {
+        IROrder(Order), debugLoc(std::move(dl)) {
     assert(debugLoc.hasTrivialDestructor() && "Expected trivial destructor");
     assert(NumValues == VTs.NumVTs &&
            "NumValues wasn't wide enough for its operands!");
@@ -926,6 +943,62 @@ inline void SDUse::setNode(SDNode *N) {
   if (N) N->addUse(*this);
 }
 
+/// These are IR-level optimization flags that may be propagated to SDNodes.
+/// TODO: This data structure should be shared by the IR optimizer and the
+/// the backend.
+struct SDNodeFlags {
+private:
+  bool NoUnsignedWrap : 1;
+  bool NoSignedWrap : 1;
+  bool Exact : 1;
+  bool UnsafeAlgebra : 1;
+  bool NoNaNs : 1;
+  bool NoInfs : 1;
+  bool NoSignedZeros : 1;
+  bool AllowReciprocal : 1;
+
+public:
+  /// Default constructor turns off all optimization flags.
+  SDNodeFlags() {
+    NoUnsignedWrap = false;
+    NoSignedWrap = false;
+    Exact = false;
+    UnsafeAlgebra = false;
+    NoNaNs = false;
+    NoInfs = false;
+    NoSignedZeros = false;
+    AllowReciprocal = false;
+  }
+
+  // These are mutators for each flag.
+  void setNoUnsignedWrap(bool b) { NoUnsignedWrap = b; }
+  void setNoSignedWrap(bool b) { NoSignedWrap = b; }
+  void setExact(bool b) { Exact = b; }
+  void setUnsafeAlgebra(bool b) { UnsafeAlgebra = b; }
+  void setNoNaNs(bool b) { NoNaNs = b; }
+  void setNoInfs(bool b) { NoInfs = b; }
+  void setNoSignedZeros(bool b) { NoSignedZeros = b; }
+  void setAllowReciprocal(bool b) { AllowReciprocal = b; }
+
+  // These are accessors for each flag.
+  bool hasNoUnsignedWrap() const { return NoUnsignedWrap; }
+  bool hasNoSignedWrap() const { return NoSignedWrap; }
+  bool hasExact() const { return Exact; }
+  bool hasUnsafeAlgebra() const { return UnsafeAlgebra; }
+  bool hasNoNaNs() const { return NoNaNs; }
+  bool hasNoInfs() const { return NoInfs; }
+  bool hasNoSignedZeros() const { return NoSignedZeros; }
+  bool hasAllowReciprocal() const { return AllowReciprocal; }
+
+  /// Return a raw encoding of the flags.
+  /// This function should only be used to add data to the NodeID value.
+  unsigned getRawFlags() const {
+    return (NoUnsignedWrap << 0) | (NoSignedWrap << 1) | (Exact << 2) |
+           (UnsafeAlgebra << 3) | (NoNaNs << 4) | (NoInfs << 5) |
+           (NoSignedZeros << 6) | (AllowReciprocal << 7);
+  }
+};
+
 /// This class is used for single-operand SDNodes.  This is solely
 /// to allow co-allocation of node operands with the node itself.
 class UnarySDNode : public SDNode {
@@ -961,6 +1034,11 @@ static bool isBinOpWithFlags(unsigned Opcode) {
   case ISD::ADD:
   case ISD::SUB:
   case ISD::SHL:
+  case ISD::FADD:
+  case ISD::FDIV:
+  case ISD::FMUL:
+  case ISD::FREM:
+  case ISD::FSUB:
     return true;
   default:
     return false;
@@ -970,27 +1048,11 @@ static bool isBinOpWithFlags(unsigned Opcode) {
 /// This class is an extension of BinarySDNode
 /// used from those opcodes that have associated extra flags.
 class BinaryWithFlagsSDNode : public BinarySDNode {
-  enum { NUW = (1 << 0), NSW = (1 << 1), EXACT = (1 << 2) };
-
 public:
+  SDNodeFlags Flags;
   BinaryWithFlagsSDNode(unsigned Opc, unsigned Order, DebugLoc dl, SDVTList VTs,
-                        SDValue X, SDValue Y)
-      : BinarySDNode(Opc, Order, dl, VTs, X, Y) {}
-  /// Return the SubclassData value, which contains an encoding of the flags.
-  /// This function should be used to add subclass data to the NodeID value.
-  unsigned getRawSubclassData() const { return SubclassData; }
-  void setHasNoUnsignedWrap(bool b) {
-    SubclassData = (SubclassData & ~NUW) | (b ? NUW : 0);
-  }
-  void setHasNoSignedWrap(bool b) {
-    SubclassData = (SubclassData & ~NSW) | (b ? NSW : 0);
-  }
-  void setIsExact(bool b) {
-    SubclassData = (SubclassData & ~EXACT) | (b ? EXACT : 0);
-  }
-  bool hasNoUnsignedWrap() const { return SubclassData & NUW; }
-  bool hasNoSignedWrap() const { return SubclassData & NSW; }
-  bool isExact() const { return SubclassData & EXACT; }
+                        SDValue X, SDValue Y, const SDNodeFlags &NodeFlags)
+      : BinarySDNode(Opc, Order, dl, VTs, X, Y), Flags(NodeFlags) {}
   static bool classof(const SDNode *N) {
     return isBinOpWithFlags(N->getOpcode());
   }
@@ -1041,7 +1103,7 @@ class AddrSpaceCastSDNode : public UnarySDNode {
   }
 };
 
-/// Abstact virtual class for operations for memory operations
+/// This is an abstract virtual class for memory operations.
 class MemSDNode : public SDNode {
 private:
   // VT of in-memory value.
@@ -1151,12 +1213,14 @@ class MemSDNode : public SDNode {
            N->getOpcode() == ISD::ATOMIC_STORE        ||
            N->getOpcode() == ISD::MLOAD               ||
            N->getOpcode() == ISD::MSTORE              ||
+           N->getOpcode() == ISD::MGATHER             ||
+           N->getOpcode() == ISD::MSCATTER            ||
            N->isMemIntrinsic()                        ||
            N->isTargetMemoryOpcode();
   }
 };
 
-/// A SDNode reprenting atomic operations.
+/// This is an SDNode representing atomic operations.
 class AtomicSDNode : public MemSDNode {
   SDUse Ops[4];
 
@@ -1359,9 +1423,10 @@ class ShuffleVectorSDNode : public SDNode {
 class ConstantSDNode : public SDNode {
   const ConstantInt *Value;
   friend class SelectionDAG;
-  ConstantSDNode(bool isTarget, bool isOpaque, const ConstantInt *val, EVT VT)
+  ConstantSDNode(bool isTarget, bool isOpaque, const ConstantInt *val,
+                 DebugLoc DL, EVT VT)
     : SDNode(isTarget ? ISD::TargetConstant : ISD::Constant,
-             0, DebugLoc(), getSDVTList(VT)), Value(val) {
+             0, DL, getSDVTList(VT)), Value(val) {
     SubclassData |= (uint16_t)isOpaque;
   }
 public:
@@ -1386,9 +1451,9 @@ class ConstantSDNode : public SDNode {
 class ConstantFPSDNode : public SDNode {
   const ConstantFP *Value;
   friend class SelectionDAG;
-  ConstantFPSDNode(bool isTarget, const ConstantFP *val, EVT VT)
+  ConstantFPSDNode(bool isTarget, const ConstantFP *val, DebugLoc DL, EVT VT)
     : SDNode(isTarget ? ISD::TargetConstantFP : ISD::ConstantFP,
-             0, DebugLoc(), getSDVTList(VT)), Value(val) {
+             0, DL, getSDVTList(VT)), Value(val) {
   }
 public:
 
@@ -1762,6 +1827,21 @@ class ExternalSymbolSDNode : public SDNode {
   }
 };
 
+class MCSymbolSDNode : public SDNode {
+  MCSymbol *Symbol;
+
+  friend class SelectionDAG;
+  MCSymbolSDNode(MCSymbol *Symbol, EVT VT)
+      : SDNode(ISD::MCSymbol, 0, DebugLoc(), getSDVTList(VT)), Symbol(Symbol) {}
+
+public:
+  MCSymbol *getMCSymbol() const { return Symbol; }
+
+  static bool classof(const SDNode *N) {
+    return N->getOpcode() == ISD::MCSymbol;
+  }
+};
+
 class CondCodeSDNode : public SDNode {
   ISD::CondCode Condition;
   friend class SelectionDAG;
@@ -1987,6 +2067,82 @@ class MaskedStoreSDNode : public MaskedLoadStoreSDNode {
   }
 };
 
+/// This is a base class used to represent
+/// MGATHER and MSCATTER nodes
+///
+class MaskedGatherScatterSDNode : public MemSDNode {
+  // Operands
+  SDUse Ops[5];
+public:
+  friend class SelectionDAG;
+  MaskedGatherScatterSDNode(ISD::NodeType NodeTy, unsigned Order, DebugLoc dl,
+                            ArrayRef<SDValue> Operands, SDVTList VTs, EVT MemVT,
+                            MachineMemOperand *MMO)
+    : MemSDNode(NodeTy, Order, dl, VTs, MemVT, MMO) {
+    assert(Operands.size() == 5 && "Incompatible number of operands");
+    InitOperands(Ops, Operands.data(), Operands.size());
+  }
+
+  // In the both nodes address is Op1, mask is Op2:
+  // MaskedGatherSDNode  (Chain, src0, mask, base, index), src0 is a passthru value
+  // MaskedScatterSDNode (Chain, value, mask, base, index)
+  // Mask is a vector of i1 elements
+  const SDValue &getBasePtr() const { return getOperand(3); }
+  const SDValue &getIndex()   const { return getOperand(4); }
+  const SDValue &getMask()    const { return getOperand(2); }
+  const SDValue &getValue()   const { return getOperand(1); }
+
+  static bool classof(const SDNode *N) {
+    return N->getOpcode() == ISD::MGATHER ||
+           N->getOpcode() == ISD::MSCATTER;
+  }
+};
+
+/// This class is used to represent an MGATHER node
+///
+class MaskedGatherSDNode : public MaskedGatherScatterSDNode {
+public:
+  friend class SelectionDAG;
+  MaskedGatherSDNode(unsigned Order, DebugLoc dl, ArrayRef<SDValue> Operands, 
+                     SDVTList VTs, EVT MemVT, MachineMemOperand *MMO)
+    : MaskedGatherScatterSDNode(ISD::MGATHER, Order, dl, Operands, VTs, MemVT,
+                                MMO) {
+    assert(getValue().getValueType() == getValueType(0) &&
+           "Incompatible type of the PathThru value in MaskedGatherSDNode");
+    assert(getMask().getValueType().getVectorNumElements() == 
+           getValueType(0).getVectorNumElements() && 
+           "Vector width mismatch between mask and data");
+    assert(getMask().getValueType().getScalarType() == MVT::i1 && 
+           "Vector width mismatch between mask and data");
+  }
+
+  static bool classof(const SDNode *N) {
+    return N->getOpcode() == ISD::MGATHER;
+  }
+};
+
+/// This class is used to represent an MSCATTER node
+///
+class MaskedScatterSDNode : public MaskedGatherScatterSDNode {
+
+public:
+  friend class SelectionDAG;
+  MaskedScatterSDNode(unsigned Order, DebugLoc dl,ArrayRef<SDValue> Operands,
+                      SDVTList VTs, EVT MemVT, MachineMemOperand *MMO)
+    : MaskedGatherScatterSDNode(ISD::MSCATTER, Order, dl, Operands, VTs, MemVT,
+                                MMO) {
+    assert(getMask().getValueType().getVectorNumElements() == 
+           getValue().getValueType().getVectorNumElements() && 
+           "Vector width mismatch between mask and data");
+    assert(getMask().getValueType().getScalarType() == MVT::i1 && 
+           "Vector width mismatch between mask and data");
+  }
+
+  static bool classof(const SDNode *N) {
+    return N->getOpcode() == ISD::MSCATTER;
+  }
+};
+
 /// An SDNode that represents everything that will be needed
 /// to construct a MachineInstr. These nodes are created during the
 /// instruction selection proper phase.
@@ -2078,7 +2234,7 @@ template <> struct GraphTraits<SDNode*> {
 };
 
 /// The largest SDNode class.
-typedef AtomicSDNode LargestSDNode;
+typedef MaskedGatherScatterSDNode LargestSDNode;
 
 /// The SDNode class with the greatest alignment requirement.
 typedef GlobalAddressSDNode MostAlignedSDNode;
diff --git a/include/llvm/CodeGen/StackMaps.h b/include/llvm/CodeGen/StackMaps.h
index caec994bc37..fdc1a9143ed 100644
--- a/include/llvm/CodeGen/StackMaps.h
+++ b/include/llvm/CodeGen/StackMaps.h
@@ -1,5 +1,4 @@
 //===------------------- StackMaps.h - StackMaps ----------------*- C++ -*-===//
-
 //
 //                     The LLVM Compiler Infrastructure
 //
@@ -42,10 +41,12 @@ class PatchPointOpers {
 public:
   /// Enumerate the meta operands.
   enum { IDPos, NBytesPos, TargetPos, NArgPos, CCPos, MetaEnd };
+
 private:
   const MachineInstr *MI;
   bool HasDef;
   bool IsAnyReg;
+
 public:
   explicit PatchPointOpers(const MachineInstr *MI);
 
@@ -66,8 +67,8 @@ class PatchPointOpers {
   /// Get the operand index of the variable list of non-argument operands.
   /// These hold the "live state".
   unsigned getVarIdx() const {
-    return getMetaIdx() + MetaEnd
-      + MI->getOperand(getMetaIdx(NArgPos)).getImm();
+    return getMetaIdx() + MetaEnd +
+           MI->getOperand(getMetaIdx(NArgPos)).getImm();
   }
 
   /// Get the index at which stack map locations will be recorded.
@@ -85,38 +86,36 @@ class PatchPointOpers {
 /// MI-level Statepoint operands
 ///
 /// Statepoint operands take the form:
-///   <num call arguments>, <call target>, [call arguments],
-///   <StackMaps::ConstantOp>, <flags>,
+///   <id>, <num patch bytes >, <num call arguments>, <call target>,
+///   [call arguments], <StackMaps::ConstantOp>, <calling convention>,
+///   <StackMaps::ConstantOp>, <statepoint flags>,
 ///   <StackMaps::ConstantOp>, <num other args>, [other args],
 ///   [gc values]
 class StatepointOpers {
 private:
-  enum {
-    NCallArgsPos = 0,
-    CallTargetPos = 1
-  };
+  // These values are aboolute offsets into the operands of the statepoint
+  // instruction.
+  enum { IDPos, NBytesPos, NCallArgsPos, CallTargetPos, MetaEnd };
+
+  // These values are relative offests from the start of the statepoint meta
+  // arguments (i.e. the end of the call arguments).
+  enum { CCOffset = 1, FlagsOffset = 3, NumVMSArgsOffset = 5 };
 
 public:
-  explicit StatepointOpers(const MachineInstr *MI):
-    MI(MI) { }
+  explicit StatepointOpers(const MachineInstr *MI) : MI(MI) {}
 
   /// Get starting index of non call related arguments
-  /// (statepoint flags, vm state and gc state).
+  /// (calling convention, statepoint flags, vm state and gc state).
   unsigned getVarIdx() const {
-    return MI->getOperand(NCallArgsPos).getImm() + 2;
+    return MI->getOperand(NCallArgsPos).getImm() + MetaEnd;
   }
 
-  /// Returns the index of the operand containing the number of non-gc non-call
-  /// arguments. 
-  unsigned getNumVMSArgsIdx() const {
-    return getVarIdx() + 3;
-  }
+  /// Return the ID for the given statepoint.
+  uint64_t getID() const { return MI->getOperand(IDPos).getImm(); }
 
-  /// Returns the number of non-gc non-call arguments attached to the
-  /// statepoint.  Note that this is the number of arguments, not the number of
-  /// operands required to represent those arguments.
-  unsigned getNumVMSArgs() const {
-    return MI->getOperand(getNumVMSArgsIdx()).getImm();
+  /// Return the number of patchable bytes the given statepoint should emit.
+  uint32_t getNumPatchBytes() const {
+    return MI->getOperand(NBytesPos).getImm();
   }
 
   /// Returns the target of the underlying call.
@@ -131,31 +130,32 @@ class StatepointOpers {
 class StackMaps {
 public:
   struct Location {
-    enum LocationType { Unprocessed, Register, Direct, Indirect, Constant,
-                        ConstantIndex };
-    LocationType LocType;
+    enum LocationType {
+      Unprocessed,
+      Register,
+      Direct,
+      Indirect,
+      Constant,
+      ConstantIndex
+    };
+    LocationType Type;
     unsigned Size;
     unsigned Reg;
     int64_t Offset;
-    Location() : LocType(Unprocessed), Size(0), Reg(0), Offset(0) {}
-    Location(LocationType LocType, unsigned Size, unsigned Reg, int64_t Offset)
-      : LocType(LocType), Size(Size), Reg(Reg), Offset(Offset) {}
+    Location() : Type(Unprocessed), Size(0), Reg(0), Offset(0) {}
+    Location(LocationType Type, unsigned Size, unsigned Reg, int64_t Offset)
+        : Type(Type), Size(Size), Reg(Reg), Offset(Offset) {}
   };
 
   struct LiveOutReg {
     unsigned short Reg;
-    unsigned short RegNo;
+    unsigned short DwarfRegNum;
     unsigned short Size;
 
-    LiveOutReg() : Reg(0), RegNo(0), Size(0) {}
-    LiveOutReg(unsigned short Reg, unsigned short RegNo, unsigned short Size)
-      : Reg(Reg), RegNo(RegNo), Size(Size) {}
-
-    void MarkInvalid() { Reg = 0; }
-
-    // Only sort by the dwarf register number.
-    bool operator< (const LiveOutReg &LO) const { return RegNo < LO.RegNo; }
-    static bool IsInvalid(const LiveOutReg &LO) { return LO.Reg == 0; }
+    LiveOutReg() : Reg(0), DwarfRegNum(0), Size(0) {}
+    LiveOutReg(unsigned short Reg, unsigned short DwarfRegNum,
+               unsigned short Size)
+        : Reg(Reg), DwarfRegNum(DwarfRegNum), Size(Size) {}
   };
 
   // OpTypes are used to encode information about the following logical
@@ -202,8 +202,8 @@ class StackMaps {
     CallsiteInfo() : CSOffsetExpr(nullptr), ID(0) {}
     CallsiteInfo(const MCExpr *CSOffsetExpr, uint64_t ID,
                  LocationVec &&Locations, LiveOutVec &&LiveOuts)
-      : CSOffsetExpr(CSOffsetExpr), ID(ID), Locations(std::move(Locations)),
-        LiveOuts(std::move(LiveOuts)) {}
+        : CSOffsetExpr(CSOffsetExpr), ID(ID), Locations(std::move(Locations)),
+          LiveOuts(std::move(LiveOuts)) {}
   };
 
   typedef std::vector<CallsiteInfo> CallsiteInfoList;
@@ -215,8 +215,8 @@ class StackMaps {
 
   MachineInstr::const_mop_iterator
   parseOperand(MachineInstr::const_mop_iterator MOI,
-               MachineInstr::const_mop_iterator MOE,
-               LocationVec &Locs, LiveOutVec &LiveOuts) const;
+               MachineInstr::const_mop_iterator MOE, LocationVec &Locs,
+               LiveOutVec &LiveOuts) const;
 
   /// \brief Create a live-out register record for the given register @p Reg.
   LiveOutReg createLiveOutReg(unsigned Reg,
@@ -251,7 +251,6 @@ class StackMaps {
   void print(raw_ostream &OS);
   void debug() { print(dbgs()); }
 };
-
 }
 
 #endif
diff --git a/include/llvm/CodeGen/TargetLoweringObjectFileImpl.h b/include/llvm/CodeGen/TargetLoweringObjectFileImpl.h
index a868cbd020f..9a3abc4dc1f 100644
--- a/include/llvm/CodeGen/TargetLoweringObjectFileImpl.h
+++ b/include/llvm/CodeGen/TargetLoweringObjectFileImpl.h
@@ -41,25 +41,24 @@ class TargetLoweringObjectFileELF : public TargetLoweringObjectFile {
 
   ~TargetLoweringObjectFileELF() override {}
 
-  void emitPersonalityValue(MCStreamer &Streamer, const TargetMachine &TM,
+  void emitPersonalityValue(MCStreamer &Streamer, const DataLayout &TM,
                             const MCSymbol *Sym) const override;
 
   /// Given a constant with the SectionKind, return a section that it should be
   /// placed in.
-  const MCSection *getSectionForConstant(SectionKind Kind,
-                                         const Constant *C) const override;
+  MCSection *getSectionForConstant(const DataLayout &DL, SectionKind Kind,
+                                   const Constant *C) const override;
 
-  const MCSection *getExplicitSectionGlobal(const GlobalValue *GV,
-                                        SectionKind Kind, Mangler &Mang,
-                                        const TargetMachine &TM) const override;
+  MCSection *getExplicitSectionGlobal(const GlobalValue *GV, SectionKind Kind,
+                                      Mangler &Mang,
+                                      const TargetMachine &TM) const override;
 
-  const MCSection *SelectSectionForGlobal(const GlobalValue *GV,
-                                        SectionKind Kind, Mangler &Mang,
-                                        const TargetMachine &TM) const override;
+  MCSection *SelectSectionForGlobal(const GlobalValue *GV, SectionKind Kind,
+                                    Mangler &Mang,
+                                    const TargetMachine &TM) const override;
 
-  const MCSection *
-  getSectionForJumpTable(const Function &F, Mangler &Mang,
-                         const TargetMachine &TM) const override;
+  MCSection *getSectionForJumpTable(const Function &F, Mangler &Mang,
+                                    const TargetMachine &TM) const override;
 
   bool shouldPutJumpTableInFunctionSection(bool UsesLabelDifference,
                                            const Function &F) const override;
@@ -78,10 +77,10 @@ class TargetLoweringObjectFileELF : public TargetLoweringObjectFile {
                                     MachineModuleInfo *MMI) const override;
 
   void InitializeELF(bool UseInitArray_);
-  const MCSection *getStaticCtorSection(unsigned Priority,
-                                        const MCSymbol *KeySym) const override;
-  const MCSection *getStaticDtorSection(unsigned Priority,
-                                        const MCSymbol *KeySym) const override;
+  MCSection *getStaticCtorSection(unsigned Priority,
+                                  const MCSymbol *KeySym) const override;
+  MCSection *getStaticDtorSection(unsigned Priority,
+                                  const MCSymbol *KeySym) const override;
 };
 
 
@@ -91,27 +90,21 @@ class TargetLoweringObjectFileMachO : public TargetLoweringObjectFile {
   ~TargetLoweringObjectFileMachO() override {}
   TargetLoweringObjectFileMachO();
 
-  /// Extract the dependent library name from a linker option string. Returns
-  /// StringRef() if the option does not specify a library.
-  StringRef getDepLibFromLinkerOpt(StringRef LinkerOption) const override;
-
   /// Emit the module flags that specify the garbage collection information.
   void emitModuleFlags(MCStreamer &Streamer,
                        ArrayRef<Module::ModuleFlagEntry> ModuleFlags,
                        Mangler &Mang, const TargetMachine &TM) const override;
 
-  const MCSection *
-    SelectSectionForGlobal(const GlobalValue *GV,
-                           SectionKind Kind, Mangler &Mang,
-                           const TargetMachine &TM) const override;
+  MCSection *SelectSectionForGlobal(const GlobalValue *GV, SectionKind Kind,
+                                    Mangler &Mang,
+                                    const TargetMachine &TM) const override;
 
-  const MCSection *
-    getExplicitSectionGlobal(const GlobalValue *GV,
-                             SectionKind Kind, Mangler &Mang,
-                             const TargetMachine &TM) const override;
+  MCSection *getExplicitSectionGlobal(const GlobalValue *GV, SectionKind Kind,
+                                      Mangler &Mang,
+                                      const TargetMachine &TM) const override;
 
-  const MCSection *getSectionForConstant(SectionKind Kind,
-                                         const Constant *C) const override;
+  MCSection *getSectionForConstant(const DataLayout &DL, SectionKind Kind,
+                                   const Constant *C) const override;
 
   /// The mach-o version of this method defaults to returning a stub reference.
   const MCExpr *
@@ -138,27 +131,20 @@ class TargetLoweringObjectFileCOFF : public TargetLoweringObjectFile {
 public:
   ~TargetLoweringObjectFileCOFF() override {}
 
-  const MCSection *
-    getExplicitSectionGlobal(const GlobalValue *GV,
-                             SectionKind Kind, Mangler &Mang,
-                             const TargetMachine &TM) const override;
+  MCSection *getExplicitSectionGlobal(const GlobalValue *GV, SectionKind Kind,
+                                      Mangler &Mang,
+                                      const TargetMachine &TM) const override;
 
-  const MCSection *
-    SelectSectionForGlobal(const GlobalValue *GV,
-                           SectionKind Kind, Mangler &Mang,
-                           const TargetMachine &TM) const override;
+  MCSection *SelectSectionForGlobal(const GlobalValue *GV, SectionKind Kind,
+                                    Mangler &Mang,
+                                    const TargetMachine &TM) const override;
 
   void getNameWithPrefix(SmallVectorImpl<char> &OutName, const GlobalValue *GV,
                          bool CannotUsePrivateLabel, Mangler &Mang,
                          const TargetMachine &TM) const override;
 
-  const MCSection *
-  getSectionForJumpTable(const Function &F, Mangler &Mang,
-                         const TargetMachine &TM) const override;
-
-  /// Extract the dependent library name from a linker option string. Returns
-  /// StringRef() if the option does not specify a library.
-  StringRef getDepLibFromLinkerOpt(StringRef LinkerOption) const override;
+  MCSection *getSectionForJumpTable(const Function &F, Mangler &Mang,
+                                    const TargetMachine &TM) const override;
 
   /// Emit Obj-C garbage collection and linker options. Only linker option
   /// emission is implemented for COFF.
@@ -166,10 +152,13 @@ class TargetLoweringObjectFileCOFF : public TargetLoweringObjectFile {
                        ArrayRef<Module::ModuleFlagEntry> ModuleFlags,
                        Mangler &Mang, const TargetMachine &TM) const override;
 
-  const MCSection *getStaticCtorSection(unsigned Priority,
-                                        const MCSymbol *KeySym) const override;
-  const MCSection *getStaticDtorSection(unsigned Priority,
-                                        const MCSymbol *KeySym) const override;
+  MCSection *getStaticCtorSection(unsigned Priority,
+                                  const MCSymbol *KeySym) const override;
+  MCSection *getStaticDtorSection(unsigned Priority,
+                                  const MCSymbol *KeySym) const override;
+
+  void emitLinkerFlagsForGlobal(raw_ostream &OS, const GlobalValue *GV,
+                                const Mangler &Mang) const override;
 };
 
 } // end namespace llvm
diff --git a/include/llvm/CodeGen/TargetSchedule.h b/include/llvm/CodeGen/TargetSchedule.h
index b6136665b96..81054aba066 100644
--- a/include/llvm/CodeGen/TargetSchedule.h
+++ b/include/llvm/CodeGen/TargetSchedule.h
@@ -40,6 +40,9 @@ class TargetSchedModel {
   SmallVector<unsigned, 16> ResourceFactors;
   unsigned MicroOpFactor; // Multiply to normalize microops to resource units.
   unsigned ResourceLCM;   // Resource units per cycle. Latency normalization factor.
+
+  unsigned computeInstrLatency(const MCSchedClassDesc &SCDesc) const;
+
 public:
   TargetSchedModel(): SchedModel(MCSchedModel::GetDefaultSchedModel()), STI(nullptr), TII(nullptr) {}
 
@@ -78,6 +81,12 @@ class TargetSchedModel {
     return nullptr;
   }
 
+  /// \brief Return true if this machine model includes an instruction-level
+  /// scheduling model or cycle-to-cycle itinerary data.
+  bool hasInstrSchedModelOrItineraries() const {
+    return hasInstrSchedModel() || hasInstrItineraries();
+  }
+
   /// \brief Identify the processor corresponding to the current subtarget.
   unsigned getProcessorID() const { return SchedModel.getProcessorID(); }
 
diff --git a/include/llvm/CodeGen/ValueTypes.td b/include/llvm/CodeGen/ValueTypes.td
index 756262f6dad..2b30f14f902 100644
--- a/include/llvm/CodeGen/ValueTypes.td
+++ b/include/llvm/CodeGen/ValueTypes.td
@@ -62,25 +62,26 @@ def v2i64  : ValueType<128, 38>;   //  2 x i64 vector value
 def v4i64  : ValueType<256, 39>;   //  4 x i64 vector value
 def v8i64  : ValueType<512, 40>;   //  8 x i64 vector value
 def v16i64 : ValueType<1024,41>;   // 16 x i64 vector value
+def v1i128 : ValueType<128, 42>;   //  1 x i128 vector value
 
-def v2f16  : ValueType<32 , 42>;   //  2 x f16 vector value
-def v4f16  : ValueType<64 , 43>;   //  4 x f16 vector value
-def v8f16  : ValueType<128, 44>;   //  8 x f16 vector value
-def v1f32  : ValueType<32 , 45>;   //  1 x f32 vector value
-def v2f32  : ValueType<64 , 46>;   //  2 x f32 vector value
-def v4f32  : ValueType<128, 47>;   //  4 x f32 vector value
-def v8f32  : ValueType<256, 48>;   //  8 x f32 vector value
-def v16f32 : ValueType<512, 49>;   // 16 x f32 vector value
-def v1f64  : ValueType<64, 50>;    //  1 x f64 vector value
-def v2f64  : ValueType<128, 51>;   //  2 x f64 vector value
-def v4f64  : ValueType<256, 52>;   //  4 x f64 vector value
-def v8f64  : ValueType<512, 53>;   //  8 x f64 vector value
+def v2f16  : ValueType<32 , 43>;   //  2 x f16 vector value
+def v4f16  : ValueType<64 , 44>;   //  4 x f16 vector value
+def v8f16  : ValueType<128, 45>;   //  8 x f16 vector value
+def v1f32  : ValueType<32 , 46>;   //  1 x f32 vector value
+def v2f32  : ValueType<64 , 47>;   //  2 x f32 vector value
+def v4f32  : ValueType<128, 48>;   //  4 x f32 vector value
+def v8f32  : ValueType<256, 49>;   //  8 x f32 vector value
+def v16f32 : ValueType<512, 50>;   // 16 x f32 vector value
+def v1f64  : ValueType<64, 51>;    //  1 x f64 vector value
+def v2f64  : ValueType<128, 52>;   //  2 x f64 vector value
+def v4f64  : ValueType<256, 53>;   //  4 x f64 vector value
+def v8f64  : ValueType<512, 54>;   //  8 x f64 vector value
 
 
-def x86mmx : ValueType<64 , 54>;   // X86 MMX value
-def FlagVT : ValueType<0  , 55>;   // Pre-RA sched glue
-def isVoid : ValueType<0  , 56>;   // Produces no value
-def untyped: ValueType<8  , 57>;   // Produces an untyped value
+def x86mmx : ValueType<64 , 55>;   // X86 MMX value
+def FlagVT : ValueType<0  , 56>;   // Pre-RA sched glue
+def isVoid : ValueType<0  , 57>;   // Produces no value
+def untyped: ValueType<8  , 58>;   // Produces an untyped value
 def MetadataVT: ValueType<0, 250>; // Metadata
 
 // Pseudo valuetype mapped to the current pointer size to any address space.
diff --git a/include/llvm/CodeGen/WinEHFuncInfo.h b/include/llvm/CodeGen/WinEHFuncInfo.h
index 5fc2b126db8..75638a058a3 100644
--- a/include/llvm/CodeGen/WinEHFuncInfo.h
+++ b/include/llvm/CodeGen/WinEHFuncInfo.h
@@ -22,7 +22,8 @@ namespace llvm {
 class BasicBlock;
 class Constant;
 class Function;
-class GlobalValue;
+class GlobalVariable;
+class InvokeInst;
 class IntrinsicInst;
 class LandingPadInst;
 class MCSymbol;
@@ -90,7 +91,7 @@ class CatchHandler : public ActionHandler {
   // When the parseEHActions function is called to populate a vector of
   // instances of this class, the ExceptionObjectVar field will be nullptr
   // and the ExceptionObjectIndex will be the index of the exception object in
-  // the parent function's frameescape block.
+  // the parent function's localescape block.
   const Value *ExceptionObjectVar;
   int ExceptionObjectIndex;
   TinyPtrVector<BasicBlock *> ReturnTargets;
@@ -107,8 +108,7 @@ class CleanupHandler : public ActionHandler {
 };
 
 void parseEHActions(const IntrinsicInst *II,
-  SmallVectorImpl<ActionHandler *> &Actions);
-
+                    SmallVectorImpl<std::unique_ptr<ActionHandler>> &Actions);
 
 // The following structs respresent the .xdata for functions using C++
 // exceptions on Windows.
@@ -132,22 +132,34 @@ struct WinEHTryBlockMapEntry {
 };
 
 struct WinEHFuncInfo {
+  DenseMap<const Function *, const LandingPadInst *> RootLPad;
+  DenseMap<const Function *, const InvokeInst *> LastInvoke;
+  DenseMap<const Function *, int> HandlerEnclosedState;
+  DenseMap<const Function *, bool> LastInvokeVisited;
   DenseMap<const LandingPadInst *, int> LandingPadStateMap;
   DenseMap<const Function *, int> CatchHandlerParentFrameObjIdx;
   DenseMap<const Function *, int> CatchHandlerParentFrameObjOffset;
   DenseMap<const Function *, int> CatchHandlerMaxState;
+  DenseMap<const Function *, int> HandlerBaseState;
   SmallVector<WinEHUnwindMapEntry, 4> UnwindMap;
   SmallVector<WinEHTryBlockMapEntry, 4> TryBlockMap;
   SmallVector<std::pair<MCSymbol *, int>, 4> IPToStateList;
-  int UnwindHelpFrameIdx;
-  int UnwindHelpFrameOffset;
+  int UnwindHelpFrameIdx = INT_MAX;
+  int UnwindHelpFrameOffset = -1;
+  unsigned NumIPToStateFuncsVisited = 0;
 
-  unsigned NumIPToStateFuncsVisited;
+  /// localescape index of the 32-bit EH registration node. Set by
+  /// WinEHStatePass and used indirectly by SEH filter functions of the parent.
+  int EHRegNodeEscapeIndex = INT_MAX;
 
-  WinEHFuncInfo()
-      : UnwindHelpFrameIdx(INT_MAX), UnwindHelpFrameOffset(-1),
-        NumIPToStateFuncsVisited(0) {}
+  WinEHFuncInfo() {}
 };
 
+/// Analyze the IR in ParentFn and it's handlers to build WinEHFuncInfo, which
+/// describes the state numbers and tables used by __CxxFrameHandler3. This
+/// analysis assumes that WinEHPrepare has already been run.
+void calculateWinCXXEHStateNumbers(const Function *ParentFn,
+                                   WinEHFuncInfo &FuncInfo);
+
 }
 #endif // LLVM_CODEGEN_WINEHFUNCINFO_H
diff --git a/include/llvm/Config/SZTargets.def.in b/include/llvm/Config/SZTargets.def.in
deleted file mode 100644
index 72d7d4532f3..00000000000
--- a/include/llvm/Config/SZTargets.def.in
+++ /dev/null
@@ -1,28 +0,0 @@
-/*===- llvm/Config/SZTargets.def - Subzero Target Architectures -*- C++ -*-===*\
-|*                                                                            *|
-|*                     The LLVM Compiler Infrastructure                       *|
-|*                                                                            *|
-|* This file is distributed under the University of Illinois Open Source      *|
-|* License. See LICENSE.TXT for details.                                      *|
-|*                                                                            *|
-|*===----------------------------------------------------------------------===*|
-|*                                                                            *|
-|* This file enumerates all of the target architectures supported by          *|
-|* this build of Subzero. Clients of this file should define the              *|
-|* SUBZERO_TARGET macro to be a function-like macro with a single             *|
-|* parameter (the name of the target); including this file will then          *|
-|* enumerate all of the targets.                                              *|
-|*                                                                            *|
-|* The set of targets supported by LLVM is generated at configuration         *|
-|* time, at which point this header is generated. Do not modify this          *|
-|* header directly.                                                           *|
-|*                                                                            *|
-\*===----------------------------------------------------------------------===*/
-
-#ifndef SUBZERO_TARGET
-#  error Please define the macro SUBZERO_TARGET(TargetName)
-#endif
-
-@SUBZERO_ENUM_TARGETS@
-
-#undef SUBZERO_TARGET
diff --git a/include/llvm/Config/config.h.cmake b/include/llvm/Config/config.h.cmake
index bd50d6ee1d6..b9fd4504ad7 100644
--- a/include/llvm/Config/config.h.cmake
+++ b/include/llvm/Config/config.h.cmake
@@ -125,12 +125,12 @@
 /* Define if you have the libdl library or equivalent. */
 #cmakedefine HAVE_LIBDL ${HAVE_LIBDL}
 
-/* Define to 1 if you have the `imagehlp' library (-limagehlp). */
-#cmakedefine HAVE_LIBIMAGEHLP ${HAVE_LIBIMAGEHLP}
-
 /* Define to 1 if you have the `m' library (-lm). */
 #undef HAVE_LIBM
 
+/* Define to 1 if you have the `ole32' library (-lole32). */
+#undef HAVE_LIBOLE32
+
 /* Define to 1 if you have the `psapi' library (-lpsapi). */
 #cmakedefine HAVE_LIBPSAPI ${HAVE_LIBPSAPI}
 
@@ -411,12 +411,103 @@
 /* Have host's ___chkstk_ms */
 #cmakedefine HAVE____CHKSTK_MS ${HAVE____CHKSTK_MS}
 
-/* Define if we link Polly to the tools */
-#cmakedefine LINK_POLLY_INTO_TOOLS
+/* Linker version detected at compile time. */
+#undef HOST_LINK_VERSION
+
+/* Installation directory for binary executables */
+#cmakedefine LLVM_BINDIR "${LLVM_BINDIR}"
+
+/* Time at which LLVM was configured */
+#cmakedefine LLVM_CONFIGTIME "${LLVM_CONFIGTIME}"
+
+/* Installation directory for data files */
+#cmakedefine LLVM_DATADIR "${LLVM_DATADIR}"
+
+/* Target triple LLVM will generate code for by default */
+#cmakedefine LLVM_DEFAULT_TARGET_TRIPLE "${LLVM_DEFAULT_TARGET_TRIPLE}"
+
+/* Installation directory for documentation */
+#cmakedefine LLVM_DOCSDIR "${LLVM_DOCSDIR}"
+
+/* Define if LLVM is built with asserts and checks that change the layout of
+   client-visible data structures.  */
+#cmakedefine LLVM_ENABLE_ABI_BREAKING_CHECKS
+
+/* Define if threads enabled */
+#cmakedefine01 LLVM_ENABLE_THREADS
 
 /* Define if zlib compression is available */
 #cmakedefine01 LLVM_ENABLE_ZLIB
 
+/* Installation directory for config files */
+#cmakedefine LLVM_ETCDIR "${LLVM_ETCDIR}"
+
+/* Has gcc/MSVC atomic intrinsics */
+#cmakedefine01 LLVM_HAS_ATOMICS
+
+/* Host triple LLVM will be executed on */
+#cmakedefine LLVM_HOST_TRIPLE "${LLVM_HOST_TRIPLE}"
+
+/* Installation directory for include files */
+#cmakedefine LLVM_INCLUDEDIR "${LLVM_INCLUDEDIR}"
+
+/* Installation directory for .info files */
+#cmakedefine LLVM_INFODIR "${LLVM_INFODIR}"
+
+/* Installation directory for man pages */
+#cmakedefine LLVM_MANDIR "${LLVM_MANDIR}"
+
+/* LLVM architecture name for the native architecture, if available */
+#cmakedefine LLVM_NATIVE_ARCH ${LLVM_NATIVE_ARCH}
+
+/* LLVM name for the native AsmParser init function, if available */
+#cmakedefine LLVM_NATIVE_ASMPARSER LLVMInitialize${LLVM_NATIVE_ARCH}AsmParser
+
+/* LLVM name for the native AsmPrinter init function, if available */
+#cmakedefine LLVM_NATIVE_ASMPRINTER LLVMInitialize${LLVM_NATIVE_ARCH}AsmPrinter
+
+/* LLVM name for the native Disassembler init function, if available */
+#cmakedefine LLVM_NATIVE_DISASSEMBLER LLVMInitialize${LLVM_NATIVE_ARCH}Disassembler
+
+/* LLVM name for the native Target init function, if available */
+#cmakedefine LLVM_NATIVE_TARGET LLVMInitialize${LLVM_NATIVE_ARCH}Target
+
+/* LLVM name for the native TargetInfo init function, if available */
+#cmakedefine LLVM_NATIVE_TARGETINFO LLVMInitialize${LLVM_NATIVE_ARCH}TargetInfo
+
+/* LLVM name for the native target MC init function, if available */
+#cmakedefine LLVM_NATIVE_TARGETMC LLVMInitialize${LLVM_NATIVE_ARCH}TargetMC
+
+/* Define if this is Unixish platform */
+#cmakedefine LLVM_ON_UNIX ${LLVM_ON_UNIX}
+
+/* Define if this is Win32ish platform */
+#cmakedefine LLVM_ON_WIN32 ${LLVM_ON_WIN32}
+
+/* Installation prefix directory */
+#cmakedefine LLVM_PREFIX "${LLVM_PREFIX}"
+
+/* Define if we have the Intel JIT API runtime support library */
+#cmakedefine LLVM_USE_INTEL_JITEVENTS 1
+
+/* Define if we have the oprofile JIT-support library */
+#cmakedefine LLVM_USE_OPROFILE 1
+
+/* Major version of the LLVM API */
+#define LLVM_VERSION_MAJOR ${LLVM_VERSION_MAJOR}
+
+/* Minor version of the LLVM API */
+#define LLVM_VERSION_MINOR ${LLVM_VERSION_MINOR}
+
+/* Patch version of the LLVM API */
+#define LLVM_VERSION_PATCH ${LLVM_VERSION_PATCH}
+
+/* LLVM version string */
+#define LLVM_VERSION_STRING "${PACKAGE_VERSION}"
+
+/* Define if we link Polly to the tools */
+#cmakedefine LINK_POLLY_INTO_TOOLS
+
 /* Define if the OS needs help to load dependent libraries for dlopen(). */
 #cmakedefine LTDL_DLOPEN_DEPLIBS ${LTDL_DLOPEN_DEPLIBS}
 
diff --git a/include/llvm/Config/config.h.in b/include/llvm/Config/config.h.in
index 67d7c8485a4..09706499ea3 100644
--- a/include/llvm/Config/config.h.in
+++ b/include/llvm/Config/config.h.in
@@ -3,9 +3,15 @@
 #ifndef CONFIG_H
 #define CONFIG_H
 
+/* Exported configuration */
+#include "llvm/Config/llvm-config.h"
+
 /* Bug report URL. */
 #undef BUG_REPORT_URL
 
+/* Default OpenMP runtime used by -fopenmp. */
+#undef CLANG_DEFAULT_OPENMP_RUNTIME
+
 /* Define if we have libxml2 */
 #undef CLANG_HAVE_LIBXML
 
@@ -131,12 +137,12 @@
 /* Define if libedit is available on this platform. */
 #undef HAVE_LIBEDIT
 
-/* Define to 1 if you have the `imagehlp' library (-limagehlp). */
-#undef HAVE_LIBIMAGEHLP
-
 /* Define to 1 if you have the `m' library (-lm). */
 #undef HAVE_LIBM
 
+/* Define to 1 if you have the `ole32' library (-lole32). */
+#undef HAVE_LIBOLE32
+
 /* Define to 1 if you have the `psapi' library (-lpsapi). */
 #undef HAVE_LIBPSAPI
 
diff --git a/include/llvm/Config/llvm-config.h.cmake b/include/llvm/Config/llvm-config.h.cmake
index bb7dc067ea0..e0f30678c0a 100644
--- a/include/llvm/Config/llvm-config.h.cmake
+++ b/include/llvm/Config/llvm-config.h.cmake
@@ -29,6 +29,10 @@
 /* Installation directory for documentation */
 #cmakedefine LLVM_DOCSDIR "${LLVM_DOCSDIR}"
 
+/* Define if LLVM is built with asserts and checks that change the layout of
+   client-visible data structures.  */
+#cmakedefine LLVM_ENABLE_ABI_BREAKING_CHECKS
+
 /* Define if threads enabled */
 #cmakedefine01 LLVM_ENABLE_THREADS
 
@@ -101,8 +105,4 @@
 /* Define if we link Polly to the tools */
 #cmakedefine LINK_POLLY_INTO_TOOLS
 
-/* Define if LLVM is built with asserts and checks that change the layout of
-   client-visible data structures.  */
-#cmakedefine LLVM_ENABLE_ABI_BREAKING_CHECKS
-
 #endif
diff --git a/include/llvm/Config/llvm-config.h.in b/include/llvm/Config/llvm-config.h.in
index 03e482ac175..2dd5d0af7b5 100644
--- a/include/llvm/Config/llvm-config.h.in
+++ b/include/llvm/Config/llvm-config.h.in
@@ -29,6 +29,9 @@
 /* Installation directory for documentation */
 #undef LLVM_DOCSDIR
 
+/* Define to enable checks that alter the LLVM C++ ABI */
+#undef LLVM_ENABLE_ABI_BREAKING_CHECKS
+
 /* Define if threads enabled */
 #undef LLVM_ENABLE_THREADS
 
@@ -98,7 +101,4 @@
 /* LLVM version string */
 #undef LLVM_VERSION_STRING
 
-/* Define to enable checks that alter the LLVM C++ ABI */
-#undef LLVM_ENABLE_ABI_BREAKING_CHECKS
-
 #endif
diff --git a/include/llvm/DebugInfo/DIContext.h b/include/llvm/DebugInfo/DIContext.h
new file mode 100644
index 00000000000..bdfd6683f80
--- /dev/null
+++ b/include/llvm/DebugInfo/DIContext.h
@@ -0,0 +1,178 @@
+//===-- DIContext.h ---------------------------------------------*- C++ -*-===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This file defines DIContext, an abstract data structure that holds
+// debug information data.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_DEBUGINFO_DICONTEXT_H
+#define LLVM_DEBUGINFO_DICONTEXT_H
+
+#include "llvm/ADT/DenseMap.h"
+#include "llvm/ADT/SmallVector.h"
+#include "llvm/Object/ObjectFile.h"
+#include "llvm/Object/RelocVisitor.h"
+#include "llvm/Support/Casting.h"
+#include "llvm/Support/DataTypes.h"
+#include <string>
+
+namespace llvm {
+
+class raw_ostream;
+
+/// DILineInfo - a format-neutral container for source line information.
+struct DILineInfo {
+  std::string FileName;
+  std::string FunctionName;
+  uint32_t Line;
+  uint32_t Column;
+
+  DILineInfo()
+      : FileName("<invalid>"), FunctionName("<invalid>"), Line(0), Column(0) {}
+
+  bool operator==(const DILineInfo &RHS) const {
+    return Line == RHS.Line && Column == RHS.Column &&
+           FileName == RHS.FileName && FunctionName == RHS.FunctionName;
+  }
+  bool operator!=(const DILineInfo &RHS) const {
+    return !(*this == RHS);
+  }
+};
+
+typedef SmallVector<std::pair<uint64_t, DILineInfo>, 16> DILineInfoTable;
+
+/// DIInliningInfo - a format-neutral container for inlined code description.
+class DIInliningInfo {
+  SmallVector<DILineInfo, 4> Frames;
+ public:
+  DIInliningInfo() {}
+  DILineInfo getFrame(unsigned Index) const {
+    assert(Index < Frames.size());
+    return Frames[Index];
+  }
+  uint32_t getNumberOfFrames() const {
+    return Frames.size();
+  }
+  void addFrame(const DILineInfo &Frame) {
+    Frames.push_back(Frame);
+  }
+};
+
+/// A DINameKind is passed to name search methods to specify a
+/// preference regarding the type of name resolution the caller wants.
+enum class DINameKind { None, ShortName, LinkageName };
+
+/// DILineInfoSpecifier - controls which fields of DILineInfo container
+/// should be filled with data.
+struct DILineInfoSpecifier {
+  enum class FileLineInfoKind { None, Default, AbsoluteFilePath };
+  typedef DINameKind FunctionNameKind;
+
+  FileLineInfoKind FLIKind;
+  FunctionNameKind FNKind;
+
+  DILineInfoSpecifier(FileLineInfoKind FLIKind = FileLineInfoKind::Default,
+                      FunctionNameKind FNKind = FunctionNameKind::None)
+      : FLIKind(FLIKind), FNKind(FNKind) {}
+};
+
+/// Selects which debug sections get dumped.
+enum DIDumpType {
+  DIDT_Null,
+  DIDT_All,
+  DIDT_Abbrev,
+  DIDT_AbbrevDwo,
+  DIDT_Aranges,
+  DIDT_Frames,
+  DIDT_Info,
+  DIDT_InfoDwo,
+  DIDT_Types,
+  DIDT_TypesDwo,
+  DIDT_Line,
+  DIDT_LineDwo,
+  DIDT_Loc,
+  DIDT_LocDwo,
+  DIDT_Ranges,
+  DIDT_Pubnames,
+  DIDT_Pubtypes,
+  DIDT_GnuPubnames,
+  DIDT_GnuPubtypes,
+  DIDT_Str,
+  DIDT_StrDwo,
+  DIDT_StrOffsetsDwo,
+  DIDT_AppleNames,
+  DIDT_AppleTypes,
+  DIDT_AppleNamespaces,
+  DIDT_AppleObjC
+};
+
+class DIContext {
+public:
+  enum DIContextKind {
+    CK_DWARF,
+    CK_PDB
+  };
+  DIContextKind getKind() const { return Kind; }
+
+  DIContext(DIContextKind K) : Kind(K) {}
+  virtual ~DIContext() {}
+
+  virtual void dump(raw_ostream &OS, DIDumpType DumpType = DIDT_All) = 0;
+
+  virtual DILineInfo getLineInfoForAddress(uint64_t Address,
+      DILineInfoSpecifier Specifier = DILineInfoSpecifier()) = 0;
+  virtual DILineInfoTable getLineInfoForAddressRange(uint64_t Address,
+      uint64_t Size, DILineInfoSpecifier Specifier = DILineInfoSpecifier()) = 0;
+  virtual DIInliningInfo getInliningInfoForAddress(uint64_t Address,
+      DILineInfoSpecifier Specifier = DILineInfoSpecifier()) = 0;
+private:
+  const DIContextKind Kind;
+};
+
+/// An inferface for inquiring the load address of a loaded object file
+/// to be used by the DIContext implementations when applying relocations
+/// on the fly.
+class LoadedObjectInfo {
+public:
+  virtual ~LoadedObjectInfo() = default;
+
+  /// Obtain the Load Address of a section by SectionRef.
+  ///
+  /// Calculate the address of the given section.
+  /// The section need not be present in the local address space. The addresses
+  /// need to be consistent with the addresses used to query the DIContext and
+  /// the output of this function should be deterministic, i.e. repeated calls with
+  /// the same Sec should give the same address.
+  virtual uint64_t getSectionLoadAddress(const object::SectionRef &Sec) const = 0;
+
+  /// If conveniently available, return the content of the given Section.
+  ///
+  /// When the section is available in the local address space, in relocated (loaded)
+  /// form, e.g. because it was relocated by a JIT for execution, this function
+  /// should provide the contents of said section in `Data`. If the loaded section
+  /// is not available, or the cost of retrieving it would be prohibitive, this
+  /// function should return false. In that case, relocations will be read from the
+  /// local (unrelocated) object file and applied on the fly. Note that this method
+  /// is used purely for optimzation purposes in the common case of JITting in the
+  /// local address space, so returning false should always be correct.
+  virtual bool getLoadedSectionContents(const object::SectionRef &Sec,
+                                        StringRef &Data) const {
+    return false;
+  }
+
+  /// Obtain a copy of this LoadedObjectInfo.
+  ///
+  /// The caller is responsible for deallocation once the copy is no longer required.
+  virtual std::unique_ptr<LoadedObjectInfo> clone() const = 0;
+};
+
+}
+
+#endif
diff --git a/include/llvm/DebugInfo/DWARF/DIContext.h b/include/llvm/DebugInfo/DWARF/DIContext.h
deleted file mode 100644
index 622aa699c43..00000000000
--- a/include/llvm/DebugInfo/DWARF/DIContext.h
+++ /dev/null
@@ -1,150 +0,0 @@
-//===-- DIContext.h ---------------------------------------------*- C++ -*-===//
-//
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
-//
-//===----------------------------------------------------------------------===//
-//
-// This file defines DIContext, an abstract data structure that holds
-// debug information data.
-//
-//===----------------------------------------------------------------------===//
-
-#ifndef LLVM_DEBUGINFO_DICONTEXT_H
-#define LLVM_DEBUGINFO_DICONTEXT_H
-
-#include "llvm/ADT/DenseMap.h"
-#include "llvm/ADT/SmallVector.h"
-#include "llvm/Object/ObjectFile.h"
-#include "llvm/Object/RelocVisitor.h"
-#include "llvm/Support/Casting.h"
-#include "llvm/Support/DataTypes.h"
-#include <string>
-
-namespace llvm {
-
-class raw_ostream;
-
-/// DILineInfo - a format-neutral container for source line information.
-struct DILineInfo {
-  std::string FileName;
-  std::string FunctionName;
-  uint32_t Line;
-  uint32_t Column;
-
-  DILineInfo()
-      : FileName("<invalid>"), FunctionName("<invalid>"), Line(0), Column(0) {}
-
-  bool operator==(const DILineInfo &RHS) const {
-    return Line == RHS.Line && Column == RHS.Column &&
-           FileName == RHS.FileName && FunctionName == RHS.FunctionName;
-  }
-  bool operator!=(const DILineInfo &RHS) const {
-    return !(*this == RHS);
-  }
-};
-
-typedef SmallVector<std::pair<uint64_t, DILineInfo>, 16> DILineInfoTable;
-
-/// DIInliningInfo - a format-neutral container for inlined code description.
-class DIInliningInfo {
-  SmallVector<DILineInfo, 4> Frames;
- public:
-  DIInliningInfo() {}
-  DILineInfo getFrame(unsigned Index) const {
-    assert(Index < Frames.size());
-    return Frames[Index];
-  }
-  uint32_t getNumberOfFrames() const {
-    return Frames.size();
-  }
-  void addFrame(const DILineInfo &Frame) {
-    Frames.push_back(Frame);
-  }
-};
-
-/// A DINameKind is passed to name search methods to specify a
-/// preference regarding the type of name resolution the caller wants.
-enum class DINameKind { None, ShortName, LinkageName };
-
-/// DILineInfoSpecifier - controls which fields of DILineInfo container
-/// should be filled with data.
-struct DILineInfoSpecifier {
-  enum class FileLineInfoKind { None, Default, AbsoluteFilePath };
-  typedef DINameKind FunctionNameKind;
-
-  FileLineInfoKind FLIKind;
-  FunctionNameKind FNKind;
-
-  DILineInfoSpecifier(FileLineInfoKind FLIKind = FileLineInfoKind::Default,
-                      FunctionNameKind FNKind = FunctionNameKind::None)
-      : FLIKind(FLIKind), FNKind(FNKind) {}
-};
-
-/// Selects which debug sections get dumped.
-enum DIDumpType {
-  DIDT_Null,
-  DIDT_All,
-  DIDT_Abbrev,
-  DIDT_AbbrevDwo,
-  DIDT_Aranges,
-  DIDT_Frames,
-  DIDT_Info,
-  DIDT_InfoDwo,
-  DIDT_Types,
-  DIDT_TypesDwo,
-  DIDT_Line,
-  DIDT_LineDwo,
-  DIDT_Loc,
-  DIDT_LocDwo,
-  DIDT_Ranges,
-  DIDT_Pubnames,
-  DIDT_Pubtypes,
-  DIDT_GnuPubnames,
-  DIDT_GnuPubtypes,
-  DIDT_Str,
-  DIDT_StrDwo,
-  DIDT_StrOffsetsDwo,
-  DIDT_AppleNames,
-  DIDT_AppleTypes,
-  DIDT_AppleNamespaces,
-  DIDT_AppleObjC
-};
-
-// In place of applying the relocations to the data we've read from disk we use
-// a separate mapping table to the side and checking that at locations in the
-// dwarf where we expect relocated values. This adds a bit of complexity to the
-// dwarf parsing/extraction at the benefit of not allocating memory for the
-// entire size of the debug info sections.
-typedef DenseMap<uint64_t, std::pair<uint8_t, int64_t> > RelocAddrMap;
-
-class DIContext {
-public:
-  enum DIContextKind {
-    CK_DWARF
-  };
-  DIContextKind getKind() const { return Kind; }
-
-  DIContext(DIContextKind K) : Kind(K) {}
-  virtual ~DIContext();
-
-  /// getDWARFContext - get a context for binary DWARF data.
-  static DIContext *getDWARFContext(const object::ObjectFile &Obj);
-
-  virtual void dump(raw_ostream &OS, DIDumpType DumpType = DIDT_All) = 0;
-
-  virtual DILineInfo getLineInfoForAddress(uint64_t Address,
-      DILineInfoSpecifier Specifier = DILineInfoSpecifier()) = 0;
-  virtual DILineInfoTable getLineInfoForAddressRange(uint64_t Address,
-      uint64_t Size, DILineInfoSpecifier Specifier = DILineInfoSpecifier()) = 0;
-  virtual DIInliningInfo getInliningInfoForAddress(uint64_t Address,
-      DILineInfoSpecifier Specifier = DILineInfoSpecifier()) = 0;
-private:
-  const DIContextKind Kind;
-};
-
-}
-
-#endif
diff --git a/include/llvm/DebugInfo/DWARF/DWARFContext.h b/include/llvm/DebugInfo/DWARF/DWARFContext.h
index 677242b1216..423c0d32f1b 100644
--- a/include/llvm/DebugInfo/DWARF/DWARFContext.h
+++ b/include/llvm/DebugInfo/DWARF/DWARFContext.h
@@ -12,7 +12,7 @@
 
 #include "llvm/ADT/MapVector.h"
 #include "llvm/ADT/SmallVector.h"
-#include "llvm/DebugInfo/DWARF/DIContext.h"
+#include "llvm/DebugInfo/DIContext.h"
 #include "llvm/DebugInfo/DWARF/DWARFCompileUnit.h"
 #include "llvm/DebugInfo/DWARF/DWARFDebugAranges.h"
 #include "llvm/DebugInfo/DWARF/DWARFDebugFrame.h"
@@ -25,6 +25,13 @@
 
 namespace llvm {
 
+// In place of applying the relocations to the data we've read from disk we use
+// a separate mapping table to the side and checking that at locations in the
+// dwarf where we expect relocated values. This adds a bit of complexity to the
+// dwarf parsing/extraction at the benefit of not allocating memory for the
+// entire size of the debug info sections.
+typedef DenseMap<uint64_t, std::pair<uint8_t, int64_t> > RelocAddrMap;
+
 /// DWARFContext
 /// This data structure is the top level entity that deals with dwarf debug
 /// information parsing. The actual data is supplied through pure virtual
@@ -248,7 +255,8 @@ class DWARFContextInMemory : public DWARFContext {
   SmallVector<SmallString<32>, 4> UncompressedSections;
 
 public:
-  DWARFContextInMemory(const object::ObjectFile &Obj);
+  DWARFContextInMemory(const object::ObjectFile &Obj,
+    const LoadedObjectInfo *L = nullptr);
   bool isLittleEndian() const override { return IsLittleEndian; }
   uint8_t getAddressSize() const override { return AddressSize; }
   const DWARFSection &getInfoSection() override { return InfoSection; }
diff --git a/include/llvm/DebugInfo/DWARF/DWARFDebugInfoEntry.h b/include/llvm/DebugInfo/DWARF/DWARFDebugInfoEntry.h
index 1080327adad..f29d5fe9ecd 100644
--- a/include/llvm/DebugInfo/DWARF/DWARFDebugInfoEntry.h
+++ b/include/llvm/DebugInfo/DWARF/DWARFDebugInfoEntry.h
@@ -11,7 +11,7 @@
 #define LLVM_LIB_DEBUGINFO_DWARFDEBUGINFOENTRY_H
 
 #include "llvm/ADT/SmallVector.h"
-#include "llvm/DebugInfo/DWARF/DIContext.h"
+#include "llvm/DebugInfo/DIContext.h"
 #include "llvm/DebugInfo/DWARF/DWARFAbbreviationDeclaration.h"
 #include "llvm/DebugInfo/DWARF/DWARFDebugRangeList.h"
 #include "llvm/Support/DataTypes.h"
diff --git a/include/llvm/DebugInfo/DWARF/DWARFDebugLine.h b/include/llvm/DebugInfo/DWARF/DWARFDebugLine.h
index 0c564c49f62..93e7c790ccf 100644
--- a/include/llvm/DebugInfo/DWARF/DWARFDebugLine.h
+++ b/include/llvm/DebugInfo/DWARF/DWARFDebugLine.h
@@ -10,7 +10,7 @@
 #ifndef LLVM_LIB_DEBUGINFO_DWARFDEBUGLINE_H
 #define LLVM_LIB_DEBUGINFO_DWARFDEBUGLINE_H
 
-#include "llvm/DebugInfo/DWARF/DIContext.h"
+#include "llvm/DebugInfo/DIContext.h"
 #include "llvm/DebugInfo/DWARF/DWARFRelocMap.h"
 #include "llvm/Support/DataExtractor.h"
 #include <map>
@@ -38,12 +38,12 @@ class DWARFDebugLine {
 
     // The size in bytes of the statement information for this compilation unit
     // (not including the total_length field itself).
-    uint32_t TotalLength;
+    uint64_t TotalLength;
     // Version identifier for the statement information format.
     uint16_t Version;
     // The number of bytes following the prologue_length field to the beginning
     // of the first byte of the statement program itself.
-    uint32_t PrologueLength;
+    uint64_t PrologueLength;
     // The size in bytes of the smallest target machine instruction. Statement
     // program opcodes that alter the address register first multiply their
     // operands by this value.
@@ -63,14 +63,22 @@ class DWARFDebugLine {
     std::vector<const char*> IncludeDirectories;
     std::vector<FileNameEntry> FileNames;
 
+    bool IsDWARF64;
+    uint32_t sizeofTotalLength() const {
+      return IsDWARF64 ? 12 : 4;
+    }
+    uint32_t sizeofPrologueLength() const {
+      return IsDWARF64 ? 8 : 4;
+    }
+
     // Length of the prologue in bytes.
     uint32_t getLength() const {
-      return PrologueLength + sizeof(TotalLength) + sizeof(Version) +
-             sizeof(PrologueLength);
+      return PrologueLength + sizeofTotalLength() + sizeof(Version) +
+             sizeofPrologueLength();
     }
     // Length of the line table data in bytes (not including the prologue).
     uint32_t getStatementTableLength() const {
-      return TotalLength + sizeof(TotalLength) - getLength();
+      return TotalLength + sizeofTotalLength() - getLength();
     }
     int32_t getMaxLineIncrementForSpecialOpcode() const {
       return LineBase + (int8_t)LineRange - 1;
@@ -163,6 +171,9 @@ class DWARFDebugLine {
   struct LineTable {
     LineTable();
 
+    // Represents an invalid row
+    const uint32_t UnknownRowIndex = UINT32_MAX;
+
     void appendRow(const DWARFDebugLine::Row &R) {
       Rows.push_back(R);
     }
@@ -171,7 +182,7 @@ class DWARFDebugLine {
     }
 
     // Returns the index of the row with file/line info for a given address,
-    // or -1 if there is no such row.
+    // or UnknownRowIndex if there is no such row.
     uint32_t lookupAddress(uint64_t address) const;
 
     bool lookupAddressRange(uint64_t address, uint64_t size,
@@ -203,6 +214,10 @@ class DWARFDebugLine {
     typedef SequenceVector::const_iterator SequenceIter;
     RowVector Rows;
     SequenceVector Sequences;
+
+  private:
+    uint32_t findRowInSeq(const DWARFDebugLine::Sequence &seq,
+                          uint64_t address) const;
   };
 
   const LineTable *getLineTable(uint32_t offset) const;
diff --git a/include/llvm/DebugInfo/DWARF/DWARFFormValue.h b/include/llvm/DebugInfo/DWARF/DWARFFormValue.h
index 02f468189d2..7ddcc0d81d5 100644
--- a/include/llvm/DebugInfo/DWARF/DWARFFormValue.h
+++ b/include/llvm/DebugInfo/DWARF/DWARFFormValue.h
@@ -87,6 +87,8 @@ class DWARFFormValue {
 
   static ArrayRef<uint8_t> getFixedFormSizes(uint8_t AddrSize,
                                              uint16_t Version);
+private:
+  void dumpString(raw_ostream &OS, const DWARFUnit *U) const;
 };
 
 }
diff --git a/include/llvm/DebugInfo/DWARF/DWARFSection.h b/include/llvm/DebugInfo/DWARF/DWARFSection.h
index f52004cac8e..3e27b529e97 100644
--- a/include/llvm/DebugInfo/DWARF/DWARFSection.h
+++ b/include/llvm/DebugInfo/DWARF/DWARFSection.h
@@ -11,6 +11,7 @@
 #define LLVM_LIB_DEBUGINFO_DWARFSECTION_H
 
 #include "llvm/DebugInfo/DWARF/DWARFRelocMap.h"
+#include "llvm/ADT/StringRef.h"
 
 namespace llvm {
 
diff --git a/include/llvm/DebugInfo/DWARF/DWARFUnit.h b/include/llvm/DebugInfo/DWARF/DWARFUnit.h
index 2b377a9ad2f..5604b93f220 100644
--- a/include/llvm/DebugInfo/DWARF/DWARFUnit.h
+++ b/include/llvm/DebugInfo/DWARF/DWARFUnit.h
@@ -195,9 +195,8 @@ class DWARFUnit {
     BaseAddr = base_addr;
   }
 
-  const DWARFDebugInfoEntryMinimal *
-  getCompileUnitDIE(bool extract_cu_die_only = true) {
-    extractDIEsIfNeeded(extract_cu_die_only);
+  const DWARFDebugInfoEntryMinimal *getUnitDIE(bool ExtractUnitDIEOnly = true) {
+    extractDIEsIfNeeded(ExtractUnitDIEOnly);
     return DieArray.empty() ? nullptr : &DieArray[0];
   }
 
@@ -226,8 +225,7 @@ class DWARFUnit {
   /// It is illegal to call this method with a DIE that hasn't be
   /// created by this unit. In other word, it's illegal to call this
   /// method on a DIE that isn't accessible by following
-  /// children/sibling links starting from this unit's
-  /// getCompileUnitDIE().
+  /// children/sibling links starting from this unit's getUnitDIE().
   uint32_t getDIEIndex(const DWARFDebugInfoEntryMinimal *DIE) {
     assert(!DieArray.empty() && DIE >= &DieArray[0] &&
            DIE < &DieArray[0] + DieArray.size());
diff --git a/include/llvm/DebugInfo/PDB/DIA/DIASession.h b/include/llvm/DebugInfo/PDB/DIA/DIASession.h
index 6c72c3e6fc6..9a8600fb85e 100644
--- a/include/llvm/DebugInfo/PDB/DIA/DIASession.h
+++ b/include/llvm/DebugInfo/PDB/DIA/DIASession.h
@@ -21,12 +21,20 @@ class DIASession : public IPDBSession {
 
   static PDB_ErrorCode createFromPdb(StringRef Path,
                                      std::unique_ptr<IPDBSession> &Session);
+  static PDB_ErrorCode createFromExe(StringRef Path,
+                                     std::unique_ptr<IPDBSession> &Session);
 
   uint64_t getLoadAddress() const override;
   void setLoadAddress(uint64_t Address) override;
   std::unique_ptr<PDBSymbolExe> getGlobalScope() const override;
   std::unique_ptr<PDBSymbol> getSymbolById(uint32_t SymbolId) const override;
 
+  std::unique_ptr<PDBSymbol>
+  findSymbolByAddress(uint64_t Address, PDB_SymType Type) const override;
+
+  std::unique_ptr<IPDBEnumLineNumbers>
+  findLineNumbersByAddress(uint64_t Address, uint32_t Length) const override;
+
   std::unique_ptr<IPDBEnumSourceFiles> getAllSourceFiles() const override;
   std::unique_ptr<IPDBEnumSourceFiles> getSourceFilesForCompiland(
       const PDBSymbolCompiland &Compiland) const override;
diff --git a/include/llvm/DebugInfo/PDB/IPDBSession.h b/include/llvm/DebugInfo/PDB/IPDBSession.h
index 60d6f62068e..a130a38a653 100644
--- a/include/llvm/DebugInfo/PDB/IPDBSession.h
+++ b/include/llvm/DebugInfo/PDB/IPDBSession.h
@@ -43,6 +43,11 @@ class IPDBSession {
     return std::unique_ptr<T>(ConcreteSymbol);
   }
 
+  virtual std::unique_ptr<PDBSymbol>
+  findSymbolByAddress(uint64_t Address, PDB_SymType Type) const = 0;
+  virtual std::unique_ptr<IPDBEnumLineNumbers>
+  findLineNumbersByAddress(uint64_t Address, uint32_t Length) const = 0;
+
   virtual std::unique_ptr<IPDBEnumSourceFiles> getAllSourceFiles() const = 0;
   virtual std::unique_ptr<IPDBEnumSourceFiles>
   getSourceFilesForCompiland(const PDBSymbolCompiland &Compiland) const = 0;
diff --git a/include/llvm/DebugInfo/PDB/PDB.h b/include/llvm/DebugInfo/PDB/PDB.h
index 67878e9ccf6..5df3be85e38 100644
--- a/include/llvm/DebugInfo/PDB/PDB.h
+++ b/include/llvm/DebugInfo/PDB/PDB.h
@@ -16,8 +16,11 @@
 namespace llvm {
 class StringRef;
 
-PDB_ErrorCode createPDBReader(PDB_ReaderType Type, StringRef Path,
-                              std::unique_ptr<IPDBSession> &Session);
+PDB_ErrorCode loadDataForPDB(PDB_ReaderType Type, StringRef Path,
+                             std::unique_ptr<IPDBSession> &Session);
+
+PDB_ErrorCode loadDataForEXE(PDB_ReaderType Type, StringRef Path,
+                             std::unique_ptr<IPDBSession> &Session);
 }
 
 #endif
diff --git a/include/llvm/DebugInfo/PDB/PDBContext.h b/include/llvm/DebugInfo/PDB/PDBContext.h
new file mode 100644
index 00000000000..2bb97463f90
--- /dev/null
+++ b/include/llvm/DebugInfo/PDB/PDBContext.h
@@ -0,0 +1,60 @@
+//===-- PDBContext.h --------------------------------------------*- C++ -*-===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===/
+
+#ifndef LLVM_DEBUGINFO_PDB_PDBCONTEXT_H
+#define LLVM_DEBUGINFO_PDB_PDBCONTEXT_H
+
+#include "llvm/DebugInfo/DIContext.h"
+#include "llvm/DebugInfo/PDB/IPDBSession.h"
+
+namespace llvm {
+
+namespace object {
+class COFFObjectFile;
+}
+
+/// PDBContext
+/// This data structure is the top level entity that deals with PDB debug
+/// information parsing.  This data structure exists only when there is a
+/// need for a transparent interface to different debug information formats
+/// (e.g. PDB and DWARF).  More control and power over the debug information
+/// access can be had by using the PDB interfaces directly.
+class PDBContext : public DIContext {
+
+  PDBContext(PDBContext &) = delete;
+  PDBContext &operator=(PDBContext &) = delete;
+
+public:
+  PDBContext(const object::COFFObjectFile &Object,
+             std::unique_ptr<IPDBSession> PDBSession,
+             bool RelativeAddress);
+
+  static bool classof(const DIContext *DICtx) {
+    return DICtx->getKind() == CK_PDB;
+  }
+
+  void dump(raw_ostream &OS, DIDumpType DumpType = DIDT_All) override;
+
+  DILineInfo getLineInfoForAddress(
+      uint64_t Address,
+      DILineInfoSpecifier Specifier = DILineInfoSpecifier()) override;
+  DILineInfoTable getLineInfoForAddressRange(
+      uint64_t Address, uint64_t Size,
+      DILineInfoSpecifier Specifier = DILineInfoSpecifier()) override;
+  DIInliningInfo getInliningInfoForAddress(
+      uint64_t Address,
+      DILineInfoSpecifier Specifier = DILineInfoSpecifier()) override;
+
+private:
+  std::string getFunctionName(uint64_t Address, DINameKind NameKind) const;
+  std::unique_ptr<IPDBSession> Session;
+};
+}
+
+#endif
diff --git a/include/llvm/DebugInfo/PDB/PDBTypes.h b/include/llvm/DebugInfo/PDB/PDBTypes.h
index 686e08f81a1..2d19e792d3d 100644
--- a/include/llvm/DebugInfo/PDB/PDBTypes.h
+++ b/include/llvm/DebugInfo/PDB/PDBTypes.h
@@ -21,6 +21,7 @@ class PDBSymbol;
 
 class IPDBDataStream;
 template <class T> class IPDBEnumChildren;
+class IPDBLineNumber;
 class IPDBRawSymbol;
 class IPDBSession;
 class IPDBSourceFile;
@@ -28,6 +29,7 @@ class IPDBSourceFile;
 typedef IPDBEnumChildren<PDBSymbol> IPDBEnumSymbols;
 typedef IPDBEnumChildren<IPDBSourceFile> IPDBEnumSourceFiles;
 typedef IPDBEnumChildren<IPDBDataStream> IPDBEnumDataStreams;
+typedef IPDBEnumChildren<IPDBLineNumber> IPDBEnumLineNumbers;
 
 class PDBSymbolExe;
 class PDBSymbolCompiland;
@@ -426,7 +428,8 @@ enum class PDB_ErrorCode {
   InvalidParameter,
   AlreadyLoaded,
   UnknownError,
-  NoMemory
+  NoMemory,
+  DebugInfoMismatch
 };
 
 struct VersionInfo {
diff --git a/include/llvm/ExecutionEngine/ExecutionEngine.h b/include/llvm/ExecutionEngine/ExecutionEngine.h
index 4b2add8bf5d..a7302602dcd 100644
--- a/include/llvm/ExecutionEngine/ExecutionEngine.h
+++ b/include/llvm/ExecutionEngine/ExecutionEngine.h
@@ -31,6 +31,7 @@
 #include <map>
 #include <string>
 #include <vector>
+#include <functional>
 
 namespace llvm {
 
@@ -89,6 +90,8 @@ class ExecutionEngineState {
   uint64_t RemoveMapping(StringRef Name);
 };
 
+using FunctionCreator = std::function<void *(const std::string &)>;
+
 /// \brief Abstract interface for implementation execution of LLVM modules,
 /// designed to support both interpreter and just-in-time (JIT) compiler
 /// implementations.
@@ -101,7 +104,12 @@ class ExecutionEngine {
   ExecutionEngineState EEState;
 
   /// The target data for the platform for which execution is being performed.
-  const DataLayout *DL;
+  ///
+  /// Note: the DataLayout is LLVMContext specific because it has an
+  /// internal cache based on type pointers. It makes unsafe to reuse the
+  /// ExecutionEngine across context, we don't enforce this rule but undefined
+  /// behavior can occurs if the user tries to do it.
+  const DataLayout DL;
 
   /// Whether lazy JIT compilation is enabled.
   bool CompilingLazily;
@@ -123,8 +131,6 @@ class ExecutionEngine {
   /// optimize for the case where there is only one module.
   SmallVector<std::unique_ptr<Module>, 1> Modules;
 
-  void setDataLayout(const DataLayout *Val) { DL = Val; }
-
   /// getMemoryforGV - Allocate memory for a global variable.
   virtual char *getMemoryForGV(const GlobalVariable *GV);
 
@@ -147,7 +153,7 @@ class ExecutionEngine {
   /// LazyFunctionCreator - If an unknown function is needed, this function
   /// pointer is invoked to create it.  If this returns null, the JIT will
   /// abort.
-  void *(*LazyFunctionCreator)(const std::string &);
+  FunctionCreator LazyFunctionCreator;
 
   /// getMangledName - Get mangled name.
   std::string getMangledName(const GlobalValue *GV);
@@ -191,21 +197,26 @@ class ExecutionEngine {
 
   //===--------------------------------------------------------------------===//
 
-  const DataLayout *getDataLayout() const { return DL; }
+  const DataLayout &getDataLayout() const { return DL; }
 
   /// removeModule - Remove a Module from the list of modules.  Returns true if
   /// M is found.
   virtual bool removeModule(Module *M);
 
-  /// FindFunctionNamed - Search all of the active modules to find the one that
+  /// FindFunctionNamed - Search all of the active modules to find the function that
   /// defines FnName.  This is very slow operation and shouldn't be used for
   /// general code.
   virtual Function *FindFunctionNamed(const char *FnName);
 
+  /// FindGlobalVariableNamed - Search all of the active modules to find the global variable
+  /// that defines Name.  This is very slow operation and shouldn't be used for
+  /// general code.
+  virtual GlobalVariable *FindGlobalVariableNamed(const char *Name, bool AllowInternal = false);
+
   /// runFunction - Execute the specified function with the specified arguments,
   /// and return the result.
   virtual GenericValue runFunction(Function *F,
-                                const std::vector<GenericValue> &ArgValues) = 0;
+                                   ArrayRef<GenericValue> ArgValues) = 0;
 
   /// getPointerToNamedFunction - This method returns the address of the
   /// specified function by using the dlsym function call.  As such it is only
@@ -465,12 +476,13 @@ class ExecutionEngine {
   /// InstallLazyFunctionCreator - If an unknown function is needed, the
   /// specified function pointer is invoked to create it.  If it returns null,
   /// the JIT will abort.
-  void InstallLazyFunctionCreator(void* (*P)(const std::string &)) {
-    LazyFunctionCreator = P;
+  void InstallLazyFunctionCreator(FunctionCreator C) {
+    LazyFunctionCreator = C;
   }
 
 protected:
-  ExecutionEngine() {}
+  ExecutionEngine(const DataLayout DL) : DL(std::move(DL)){}
+  explicit ExecutionEngine(DataLayout DL, std::unique_ptr<Module> M);
   explicit ExecutionEngine(std::unique_ptr<Module> M);
 
   void emitGlobals();
@@ -480,6 +492,9 @@ class ExecutionEngine {
   GenericValue getConstantValue(const Constant *C);
   void LoadValueFromMemory(GenericValue &Result, GenericValue *Ptr,
                            Type *Ty);
+
+private:
+  void Init(std::unique_ptr<Module> M);
 };
 
 namespace EngineKind {
diff --git a/include/llvm/ExecutionEngine/Orc/CloneSubModule.h b/include/llvm/ExecutionEngine/Orc/CloneSubModule.h
deleted file mode 100644
index 1bd3955e879..00000000000
--- a/include/llvm/ExecutionEngine/Orc/CloneSubModule.h
+++ /dev/null
@@ -1,60 +0,0 @@
-//===-- CloneSubModule.h - Utilities for extracting sub-modules -*- C++ -*-===//
-//
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
-//
-//===----------------------------------------------------------------------===//
-//
-// Contains utilities for extracting sub-modules. Useful for breaking up modules
-// for lazy jitting.
-//
-//===----------------------------------------------------------------------===//
-
-#ifndef LLVM_EXECUTIONENGINE_ORC_CLONESUBMODULE_H
-#define LLVM_EXECUTIONENGINE_ORC_CLONESUBMODULE_H
-
-#include "llvm/ADT/DenseSet.h"
-#include "llvm/Transforms/Utils/ValueMapper.h"
-#include <functional>
-
-namespace llvm {
-
-class Function;
-class GlobalVariable;
-class Module;
-
-namespace orc {
-
-/// @brief Functor type for describing how CloneSubModule should mutate a
-///        GlobalVariable.
-typedef std::function<void(GlobalVariable &, const GlobalVariable &,
-                           ValueToValueMapTy &)> HandleGlobalVariableFtor;
-
-/// @brief Functor type for describing how CloneSubModule should mutate a
-///        Function.
-typedef std::function<void(Function &, const Function &, ValueToValueMapTy &)>
-    HandleFunctionFtor;
-
-/// @brief Copies the initializer from Orig to New.
-///
-///   Type is suitable for implicit conversion to a HandleGlobalVariableFtor.
-void copyGVInitializer(GlobalVariable &New, const GlobalVariable &Orig,
-                       ValueToValueMapTy &VMap);
-
-/// @brief Copies the body of Orig to New.
-///
-///   Type is suitable for implicit conversion to a HandleFunctionFtor.
-void copyFunctionBody(Function &New, const Function &Orig,
-                      ValueToValueMapTy &VMap);
-
-/// @brief Clone a subset of the module Src into Dst.
-void CloneSubModule(Module &Dst, const Module &Src,
-                    HandleGlobalVariableFtor HandleGlobalVariable,
-                    HandleFunctionFtor HandleFunction, bool KeepInlineAsm);
-
-} // End namespace orc.
-} // End namespace llvm.
-
-#endif // LLVM_EXECUTIONENGINE_ORC_CLONESUBMODULE_H
diff --git a/include/llvm/ExecutionEngine/Orc/CompileOnDemandLayer.h b/include/llvm/ExecutionEngine/Orc/CompileOnDemandLayer.h
index 30f7f1cd5f5..9694b80d192 100644
--- a/include/llvm/ExecutionEngine/Orc/CompileOnDemandLayer.h
+++ b/include/llvm/ExecutionEngine/Orc/CompileOnDemandLayer.h
@@ -17,158 +17,92 @@
 
 #include "IndirectionUtils.h"
 #include "LambdaResolver.h"
+#include "LogicalDylib.h"
 #include "llvm/ADT/STLExtras.h"
 #include "llvm/ExecutionEngine/SectionMemoryManager.h"
+#include "llvm/Transforms/Utils/Cloning.h"
 #include <list>
+#include <set>
+
+#include "llvm/Support/Debug.h"
 
 namespace llvm {
 namespace orc {
 
 /// @brief Compile-on-demand layer.
 ///
-///   Modules added to this layer have their calls indirected, and are then
-/// broken up into a set of single-function modules, each of which is added
-/// to the layer below in a singleton set. The lower layer can be any layer that
-/// accepts IR module sets.
-///
-/// It is expected that this layer will frequently be used on top of a
-/// LazyEmittingLayer. The combination of the two ensures that each function is
-/// compiled only when it is first called.
-template <typename BaseLayerT, typename CompileCallbackMgrT>
+///   When a module is added to this layer a stub is created for each of its
+/// function definitions. The stubs and other global values are immediately
+/// added to the layer below. When a stub is called it triggers the extraction
+/// of the function body from the original module. The extracted body is then
+/// compiled and executed.
+template <typename BaseLayerT, typename CompileCallbackMgrT,
+          typename PartitioningFtor =
+            std::function<std::set<Function*>(Function&)>>
 class CompileOnDemandLayer {
 private:
-  /// @brief Lookup helper that provides compatibility with the classic
-  ///        static-compilation symbol resolution process.
-  ///
-  ///   The CompileOnDemand (COD) layer splits modules up into multiple
-  /// sub-modules, each held in its own llvm::Module instance, in order to
-  /// support lazy compilation. When a module that contains private symbols is
-  /// broken up symbol linkage changes may be required to enable access to
-  /// "private" data that now resides in a different llvm::Module instance. To
-  /// retain expected symbol resolution behavior for clients of the COD layer,
-  /// the CODScopedLookup class uses a two-tiered lookup system to resolve
-  /// symbols. Lookup first scans sibling modules that were split from the same
-  /// original module (logical-module scoped lookup), then scans all other
-  /// modules that have been added to the lookup scope (logical-dylib scoped
-  /// lookup).
-  class CODScopedLookup {
-  private:
-    typedef typename BaseLayerT::ModuleSetHandleT BaseLayerModuleSetHandleT;
-    typedef std::vector<BaseLayerModuleSetHandleT> SiblingHandlesList;
-    typedef std::list<SiblingHandlesList> PseudoDylibModuleSetHandlesList;
 
+  // Utility class for MapValue. Only materializes declarations for global
+  // variables.
+  class GlobalDeclMaterializer : public ValueMaterializer {
   public:
-    /// @brief Handle for a logical module.
-    typedef typename PseudoDylibModuleSetHandlesList::iterator LMHandle;
-
-    /// @brief Construct a scoped lookup.
-    CODScopedLookup(BaseLayerT &BaseLayer) : BaseLayer(BaseLayer) {}
-
-    virtual ~CODScopedLookup() {}
-
-    /// @brief Start a new context for a single logical module.
-    LMHandle createLogicalModule() {
-      Handles.push_back(SiblingHandlesList());
-      return std::prev(Handles.end());
-    }
-
-    /// @brief Add a concrete Module's handle to the given logical Module's
-    ///        lookup scope.
-    void addToLogicalModule(LMHandle LMH, BaseLayerModuleSetHandleT H) {
-      LMH->push_back(H);
-    }
-
-    /// @brief Remove a logical Module from the CODScopedLookup entirely.
-    void removeLogicalModule(LMHandle LMH) { Handles.erase(LMH); }
-
-    /// @brief Look up a symbol in this context.
-    JITSymbol findSymbol(LMHandle LMH, const std::string &Name) {
-      if (auto Symbol = findSymbolIn(LMH, Name))
-        return Symbol;
-
-      for (auto I = Handles.begin(), E = Handles.end(); I != E; ++I)
-        if (I != LMH)
-          if (auto Symbol = findSymbolIn(I, Name))
-            return Symbol;
-
-      return nullptr;
-    }
-
-    /// @brief Find an external symbol (via the user supplied SymbolResolver).
-    virtual RuntimeDyld::SymbolInfo
-    externalLookup(const std::string &Name) const = 0;
-
-  private:
-
-    JITSymbol findSymbolIn(LMHandle LMH, const std::string &Name) {
-      for (auto H : *LMH)
-        if (auto Symbol = BaseLayer.findSymbolIn(H, Name, false))
-          return Symbol;
+    typedef std::set<const Function*> StubSet;
+
+    GlobalDeclMaterializer(Module &Dst, const StubSet *StubsToClone = nullptr)
+        : Dst(Dst), StubsToClone(StubsToClone) {}
+
+    Value* materializeValueFor(Value *V) final {
+      if (auto *GV = dyn_cast<GlobalVariable>(V))
+        return cloneGlobalVariableDecl(Dst, *GV);
+      else if (auto *F = dyn_cast<Function>(V)) {
+        auto *ClonedF = cloneFunctionDecl(Dst, *F);
+        if (StubsToClone && StubsToClone->count(F)) {
+          GlobalVariable *FnBodyPtr =
+            createImplPointer(*ClonedF->getType(), *ClonedF->getParent(),
+                              ClonedF->getName() + "$orc_addr", nullptr);
+          makeStub(*ClonedF, *FnBodyPtr);
+          ClonedF->setLinkage(GlobalValue::AvailableExternallyLinkage);
+          ClonedF->addFnAttr(Attribute::AlwaysInline);
+        }
+        return ClonedF;
+      }
+      // Else.
       return nullptr;
     }
-
-    BaseLayerT &BaseLayer;
-    PseudoDylibModuleSetHandlesList Handles;
-  };
-
-  template <typename ResolverPtrT>
-  class CODScopedLookupImpl : public CODScopedLookup {
-  public:
-    CODScopedLookupImpl(BaseLayerT &BaseLayer, ResolverPtrT Resolver)
-      : CODScopedLookup(BaseLayer), Resolver(std::move(Resolver)) {}
-
-    RuntimeDyld::SymbolInfo
-    externalLookup(const std::string &Name) const override {
-      return Resolver->findSymbol(Name);
-    }
-
   private:
-    ResolverPtrT Resolver;
+    Module &Dst;
+    const StubSet *StubsToClone;
   };
 
-  template <typename ResolverPtrT>
-  static std::shared_ptr<CODScopedLookup>
-  createCODScopedLookup(BaseLayerT &BaseLayer,
-                        ResolverPtrT Resolver) {
-    typedef CODScopedLookupImpl<ResolverPtrT> Impl;
-    return std::make_shared<Impl>(BaseLayer, std::move(Resolver));
-  }
-
   typedef typename BaseLayerT::ModuleSetHandleT BaseLayerModuleSetHandleT;
-  typedef std::vector<BaseLayerModuleSetHandleT> BaseLayerModuleSetHandleListT;
-
-  struct ModuleSetInfo {
-    // Symbol lookup - just one for the whole module set.
-    std::shared_ptr<CODScopedLookup> Lookup;
 
-    // Logical module handles.
-    std::vector<typename CODScopedLookup::LMHandle> LMHandles;
-
-    // List of vectors of module set handles:
-    // One vector per logical module - each vector holds the handles for the
-    // exploded modules for that logical module in the base layer.
-    BaseLayerModuleSetHandleListT BaseLayerModuleSetHandles;
-
-    ModuleSetInfo(std::shared_ptr<CODScopedLookup> Lookup)
-        : Lookup(std::move(Lookup)) {}
+  struct LogicalModuleResources {
+    std::shared_ptr<Module> SourceModule;
+    std::set<const Function*> StubsToClone;
+  };
 
-    void releaseResources(BaseLayerT &BaseLayer) {
-      for (auto LMH : LMHandles)
-        Lookup->removeLogicalModule(LMH);
-      for (auto H : BaseLayerModuleSetHandles)
-        BaseLayer.removeModuleSet(H);
-    }
+  struct LogicalDylibResources {
+    typedef std::function<RuntimeDyld::SymbolInfo(const std::string&)>
+      SymbolResolverFtor;
+    SymbolResolverFtor ExternalSymbolResolver;
+    PartitioningFtor Partitioner;
   };
 
-  typedef std::list<ModuleSetInfo> ModuleSetInfoListT;
+  typedef LogicalDylib<BaseLayerT, LogicalModuleResources,
+                       LogicalDylibResources> CODLogicalDylib;
+
+  typedef typename CODLogicalDylib::LogicalModuleHandle LogicalModuleHandle;
+  typedef std::list<CODLogicalDylib> LogicalDylibList;
 
 public:
   /// @brief Handle to a set of loaded modules.
-  typedef typename ModuleSetInfoListT::iterator ModuleSetHandleT;
+  typedef typename LogicalDylibList::iterator ModuleSetHandleT;
 
   /// @brief Construct a compile-on-demand layer instance.
-  CompileOnDemandLayer(BaseLayerT &BaseLayer, CompileCallbackMgrT &CallbackMgr)
-      : BaseLayer(BaseLayer), CompileCallbackMgr(CallbackMgr) {}
+  CompileOnDemandLayer(BaseLayerT &BaseLayer, CompileCallbackMgrT &CallbackMgr,
+                       bool CloneStubsIntoPartitions)
+      : BaseLayer(BaseLayer), CompileCallbackMgr(CallbackMgr),
+        CloneStubsIntoPartitions(CloneStubsIntoPartitions) {}
 
   /// @brief Add a module to the compile-on-demand layer.
   template <typename ModuleSetT, typename MemoryManagerPtrT,
@@ -180,19 +114,27 @@ class CompileOnDemandLayer {
     assert(MemMgr == nullptr &&
            "User supplied memory managers not supported with COD yet.");
 
-    // Create a lookup context and ModuleSetInfo for this module set.
-    // For the purposes of symbol resolution the set Ms will be treated as if
-    // the modules it contained had been linked together as a dylib.
-    auto DylibLookup = createCODScopedLookup(BaseLayer, std::move(Resolver));
-    ModuleSetHandleT H =
-        ModuleSetInfos.insert(ModuleSetInfos.end(), ModuleSetInfo(DylibLookup));
-    ModuleSetInfo &MSI = ModuleSetInfos.back();
+    LogicalDylibs.push_back(CODLogicalDylib(BaseLayer));
+    auto &LDResources = LogicalDylibs.back().getDylibResources();
+
+    LDResources.ExternalSymbolResolver =
+      [Resolver](const std::string &Name) {
+        return Resolver->findSymbol(Name);
+      };
+
+    LDResources.Partitioner =
+      [](Function &F) {
+        std::set<Function*> Partition;
+        Partition.insert(&F);
+        return Partition;
+      };
 
     // Process each of the modules in this module set.
     for (auto &M : Ms)
-      partitionAndAdd(*M, MSI);
+      addLogicalModule(LogicalDylibs.back(),
+                       std::shared_ptr<Module>(std::move(M)));
 
-    return H;
+    return std::prev(LogicalDylibs.end());
   }
 
   /// @brief Remove the module represented by the given handle.
@@ -200,8 +142,7 @@ class CompileOnDemandLayer {
   ///   This will remove all modules in the layers below that were derived from
   /// the module represented by H.
   void removeModuleSet(ModuleSetHandleT H) {
-    H->releaseResources(BaseLayer);
-    ModuleSetInfos.erase(H);
+    LogicalDylibs.erase(H);
   }
 
   /// @brief Search for the given named symbol.
@@ -216,164 +157,194 @@ class CompileOnDemandLayer {
   ///        below this one.
   JITSymbol findSymbolIn(ModuleSetHandleT H, const std::string &Name,
                          bool ExportedSymbolsOnly) {
-
-    for (auto &BH : H->BaseLayerModuleSetHandles) {
-      if (auto Symbol = BaseLayer.findSymbolIn(BH, Name, ExportedSymbolsOnly))
-        return Symbol;
-    }
-    return nullptr;
+    return H->findSymbol(Name, ExportedSymbolsOnly);
   }
 
 private:
 
-  void partitionAndAdd(Module &M, ModuleSetInfo &MSI) {
-    const char *AddrSuffix = "$orc_addr";
-    const char *BodySuffix = "$orc_body";
-
-    // We're going to break M up into a bunch of sub-modules, but we want
-    // internal linkage symbols to still resolve sensibly. CODScopedLookup
-    // provides the "logical module" concept to make this work, so create a
-    // new logical module for M.
-    auto DylibLookup = MSI.Lookup;
-    auto LogicalModule = DylibLookup->createLogicalModule();
-    MSI.LMHandles.push_back(LogicalModule);
-
-    // Partition M into a "globals and stubs" module, a "common symbols" module,
-    // and a list of single-function modules.
-    auto PartitionedModule = fullyPartition(M);
-    auto StubsModule = std::move(PartitionedModule.GlobalVars);
-    auto CommonsModule = std::move(PartitionedModule.Commons);
-    auto FunctionModules = std::move(PartitionedModule.Functions);
-
-    // Emit the commons stright away.
-    auto CommonHandle = addModule(std::move(CommonsModule), MSI, LogicalModule);
-    BaseLayer.emitAndFinalize(CommonHandle);
-
-    // Map of definition names to callback-info data structures. We'll use
-    // this to build the compile actions for the stubs below.
-    typedef std::map<std::string,
-                     typename CompileCallbackMgrT::CompileCallbackInfo>
-      StubInfoMap;
-    StubInfoMap StubInfos;
-
-    // Now we need to take each of the extracted Modules and add them to
-    // base layer. Each Module will be added individually to make sure they
-    // can be compiled separately, and each will get its own lookaside
-    // memory manager that will resolve within this logical module first.
-    for (auto &SubM : FunctionModules) {
-
-      // Keep track of the stubs we create for this module so that we can set
-      // their compile actions.
-      std::vector<typename StubInfoMap::iterator> NewStubInfos;
-
-      // Search for function definitions and insert stubs into the stubs
-      // module.
-      for (auto &F : *SubM) {
-        if (F.isDeclaration())
-          continue;
-
-        std::string Name = F.getName();
-        Function *Proto = StubsModule->getFunction(Name);
-        assert(Proto && "Failed to clone function decl into stubs module.");
-        auto CallbackInfo =
-          CompileCallbackMgr.getCompileCallback(Proto->getContext());
-        GlobalVariable *FunctionBodyPointer =
-          createImplPointer(*Proto->getType(), *Proto->getParent(),
-                            Name + AddrSuffix,
-                            createIRTypedAddress(*Proto->getFunctionType(),
-                                                 CallbackInfo.getAddress()));
-        makeStub(*Proto, *FunctionBodyPointer);
-
-        F.setName(Name + BodySuffix);
-        F.setVisibility(GlobalValue::HiddenVisibility);
-
-        auto KV = std::make_pair(std::move(Name), std::move(CallbackInfo));
-        NewStubInfos.push_back(StubInfos.insert(StubInfos.begin(), KV));
-      }
-
-      auto H = addModule(std::move(SubM), MSI, LogicalModule);
-
-      // Set the compile actions for this module:
-      for (auto &KVPair : NewStubInfos) {
-        std::string BodyName = Mangle(KVPair->first + BodySuffix,
-                                      M.getDataLayout());
-        auto &CCInfo = KVPair->second;
-        CCInfo.setCompileAction(
-          [=](){
-            return BaseLayer.findSymbolIn(H, BodyName, false).getAddress();
-          });
-      }
-
+  void addLogicalModule(CODLogicalDylib &LD, std::shared_ptr<Module> SrcM) {
+
+    // Bump the linkage and rename any anonymous/privote members in SrcM to
+    // ensure that everything will resolve properly after we partition SrcM.
+    makeAllSymbolsExternallyAccessible(*SrcM);
+
+    // Create a logical module handle for SrcM within the logical dylib.
+    auto LMH = LD.createLogicalModule();
+    auto &LMResources =  LD.getLogicalModuleResources(LMH);
+    LMResources.SourceModule = SrcM;
+
+    // Create the GVs-and-stubs module.
+    auto GVsAndStubsM = llvm::make_unique<Module>(
+                          (SrcM->getName() + ".globals_and_stubs").str(),
+                          SrcM->getContext());
+    GVsAndStubsM->setDataLayout(SrcM->getDataLayout());
+    ValueToValueMapTy VMap;
+
+    // Process module and create stubs.
+    // We create the stubs before copying the global variables as we know the
+    // stubs won't refer to any globals (they only refer to their implementation
+    // pointer) so there's no ordering/value-mapping issues.
+    for (auto &F : *SrcM) {
+
+      // Skip declarations.
+      if (F.isDeclaration())
+        continue;
+
+      // Record all functions defined by this module.
+      if (CloneStubsIntoPartitions)
+        LMResources.StubsToClone.insert(&F);
+
+      // For each definition: create a callback, a stub, and a function body
+      // pointer. Initialize the function body pointer to point at the callback,
+      // and set the callback to compile the function body.
+      auto CCInfo = CompileCallbackMgr.getCompileCallback(SrcM->getContext());
+      Function *StubF = cloneFunctionDecl(*GVsAndStubsM, F, &VMap);
+      GlobalVariable *FnBodyPtr =
+        createImplPointer(*StubF->getType(), *StubF->getParent(),
+                          StubF->getName() + "$orc_addr",
+                          createIRTypedAddress(*StubF->getFunctionType(),
+                                               CCInfo.getAddress()));
+      makeStub(*StubF, *FnBodyPtr);
+      CCInfo.setCompileAction(
+        [this, &LD, LMH, &F]() {
+          return this->extractAndCompile(LD, LMH, F);
+        });
     }
 
-    // Ok - we've processed all the partitioned modules. Now add the
-    // stubs/globals module and set the update actions.
-    auto StubsH =
-      addModule(std::move(StubsModule), MSI, LogicalModule);
-
-    for (auto &KVPair : StubInfos) {
-      std::string AddrName = Mangle(KVPair.first + AddrSuffix,
-                                    M.getDataLayout());
-      auto &CCInfo = KVPair.second;
-      CCInfo.setUpdateAction(
-        getLocalFPUpdater(BaseLayer, StubsH, AddrName));
-    }
-  }
-
-  // Add the given Module to the base layer using a memory manager that will
-  // perform the appropriate scoped lookup (i.e. will look first with in the
-  // module from which it was extracted, then into the set to which that module
-  // belonged, and finally externally).
-  BaseLayerModuleSetHandleT addModule(
-                               std::unique_ptr<Module> M,
-                               ModuleSetInfo &MSI,
-                               typename CODScopedLookup::LMHandle LogicalModule) {
-
-    // Add this module to the JIT with a memory manager that uses the
-    // DylibLookup to resolve symbols.
-    std::vector<std::unique_ptr<Module>> MSet;
-    MSet.push_back(std::move(M));
-
-    auto DylibLookup = MSI.Lookup;
-    auto Resolver =
-      createLambdaResolver(
-        [=](const std::string &Name) {
-          if (auto Symbol = DylibLookup->findSymbol(LogicalModule, Name))
-            return RuntimeDyld::SymbolInfo(Symbol.getAddress(),
-                                           Symbol.getFlags());
-          return DylibLookup->externalLookup(Name);
+    // Now clone the global variable declarations.
+    GlobalDeclMaterializer GDMat(*GVsAndStubsM);
+    for (auto &GV : SrcM->globals())
+      if (!GV.isDeclaration())
+        cloneGlobalVariableDecl(*GVsAndStubsM, GV, &VMap);
+
+    // Then clone the initializers.
+    for (auto &GV : SrcM->globals())
+      if (!GV.isDeclaration())
+        moveGlobalVariableInitializer(GV, VMap, &GDMat);
+
+    // Build a resolver for the stubs module and add it to the base layer.
+    auto GVsAndStubsResolver = createLambdaResolver(
+        [&LD](const std::string &Name) {
+          return LD.getDylibResources().ExternalSymbolResolver(Name);
         },
-        [=](const std::string &Name) -> RuntimeDyld::SymbolInfo {
-          if (auto Symbol = DylibLookup->findSymbol(LogicalModule, Name))
-            return RuntimeDyld::SymbolInfo(Symbol.getAddress(),
-                                           Symbol.getFlags());
-          return nullptr;
+        [](const std::string &Name) {
+          return RuntimeDyld::SymbolInfo(nullptr);
         });
 
-    BaseLayerModuleSetHandleT H =
-      BaseLayer.addModuleSet(std::move(MSet),
-                             make_unique<SectionMemoryManager>(),
-                             std::move(Resolver));
-    // Add this module to the logical module lookup.
-    DylibLookup->addToLogicalModule(LogicalModule, H);
-    MSI.BaseLayerModuleSetHandles.push_back(H);
-
-    return H;
+    std::vector<std::unique_ptr<Module>> GVsAndStubsMSet;
+    GVsAndStubsMSet.push_back(std::move(GVsAndStubsM));
+    auto GVsAndStubsH =
+      BaseLayer.addModuleSet(std::move(GVsAndStubsMSet),
+                             llvm::make_unique<SectionMemoryManager>(),
+                             std::move(GVsAndStubsResolver));
+    LD.addToLogicalModule(LMH, GVsAndStubsH);
   }
 
   static std::string Mangle(StringRef Name, const DataLayout &DL) {
-    Mangler M(&DL);
     std::string MangledName;
     {
       raw_string_ostream MangledNameStream(MangledName);
-      M.getNameWithPrefix(MangledNameStream, Name);
+      Mangler::getNameWithPrefix(MangledNameStream, Name, DL);
     }
     return MangledName;
   }
 
+  TargetAddress extractAndCompile(CODLogicalDylib &LD,
+                                  LogicalModuleHandle LMH,
+                                  Function &F) {
+    Module &SrcM = *LD.getLogicalModuleResources(LMH).SourceModule;
+
+    // If F is a declaration we must already have compiled it.
+    if (F.isDeclaration())
+      return 0;
+
+    // Grab the name of the function being called here.
+    std::string CalledFnName = Mangle(F.getName(), SrcM.getDataLayout());
+
+    auto Partition = LD.getDylibResources().Partitioner(F);
+    auto PartitionH = emitPartition(LD, LMH, Partition);
+
+    TargetAddress CalledAddr = 0;
+    for (auto *SubF : Partition) {
+      std::string FName = SubF->getName();
+      auto FnBodySym =
+        BaseLayer.findSymbolIn(PartitionH, Mangle(FName, SrcM.getDataLayout()),
+                               false);
+      auto FnPtrSym =
+        BaseLayer.findSymbolIn(*LD.moduleHandlesBegin(LMH),
+                               Mangle(FName + "$orc_addr",
+                                      SrcM.getDataLayout()),
+                               false);
+      assert(FnBodySym && "Couldn't find function body.");
+      assert(FnPtrSym && "Couldn't find function body pointer.");
+
+      TargetAddress FnBodyAddr = FnBodySym.getAddress();
+      void *FnPtrAddr = reinterpret_cast<void*>(
+          static_cast<uintptr_t>(FnPtrSym.getAddress()));
+
+      // If this is the function we're calling record the address so we can
+      // return it from this function.
+      if (SubF == &F)
+        CalledAddr = FnBodyAddr;
+
+      memcpy(FnPtrAddr, &FnBodyAddr, sizeof(uintptr_t));
+    }
+
+    return CalledAddr;
+  }
+
+  template <typename PartitionT>
+  BaseLayerModuleSetHandleT emitPartition(CODLogicalDylib &LD,
+                                          LogicalModuleHandle LMH,
+                                          const PartitionT &Partition) {
+    auto &LMResources = LD.getLogicalModuleResources(LMH);
+    Module &SrcM = *LMResources.SourceModule;
+
+    // Create the module.
+    std::string NewName = SrcM.getName();
+    for (auto *F : Partition) {
+      NewName += ".";
+      NewName += F->getName();
+    }
+
+    auto M = llvm::make_unique<Module>(NewName, SrcM.getContext());
+    M->setDataLayout(SrcM.getDataLayout());
+    ValueToValueMapTy VMap;
+    GlobalDeclMaterializer GDM(*M, &LMResources.StubsToClone);
+
+    // Create decls in the new module.
+    for (auto *F : Partition)
+      cloneFunctionDecl(*M, *F, &VMap);
+
+    // Move the function bodies.
+    for (auto *F : Partition)
+      moveFunctionBody(*F, VMap, &GDM);
+
+    // Create memory manager and symbol resolver.
+    auto MemMgr = llvm::make_unique<SectionMemoryManager>();
+    auto Resolver = createLambdaResolver(
+        [this, &LD, LMH](const std::string &Name) {
+          if (auto Symbol = LD.findSymbolInternally(LMH, Name))
+            return RuntimeDyld::SymbolInfo(Symbol.getAddress(),
+                                           Symbol.getFlags());
+          return LD.getDylibResources().ExternalSymbolResolver(Name);
+        },
+        [this, &LD, LMH](const std::string &Name) {
+          if (auto Symbol = LD.findSymbolInternally(LMH, Name))
+            return RuntimeDyld::SymbolInfo(Symbol.getAddress(),
+                                           Symbol.getFlags());
+          return RuntimeDyld::SymbolInfo(nullptr);
+        });
+    std::vector<std::unique_ptr<Module>> PartMSet;
+    PartMSet.push_back(std::move(M));
+    return BaseLayer.addModuleSet(std::move(PartMSet), std::move(MemMgr),
+                                  std::move(Resolver));
+  }
+
   BaseLayerT &BaseLayer;
   CompileCallbackMgrT &CompileCallbackMgr;
-  ModuleSetInfoListT ModuleSetInfos;
+  LogicalDylibList LogicalDylibs;
+  bool CloneStubsIntoPartitions;
 };
 
 } // End namespace orc.
diff --git a/include/llvm/ExecutionEngine/Orc/IndirectionUtils.h b/include/llvm/ExecutionEngine/Orc/IndirectionUtils.h
index 7b4f611393f..4b7fc5e84b9 100644
--- a/include/llvm/ExecutionEngine/Orc/IndirectionUtils.h
+++ b/include/llvm/ExecutionEngine/Orc/IndirectionUtils.h
@@ -15,11 +15,13 @@
 #define LLVM_EXECUTIONENGINE_ORC_INDIRECTIONUTILS_H
 
 #include "JITSymbol.h"
+#include "LambdaResolver.h"
 #include "llvm/ADT/DenseSet.h"
 #include "llvm/ExecutionEngine/RuntimeDyld.h"
 #include "llvm/IR/IRBuilder.h"
 #include "llvm/IR/Mangler.h"
 #include "llvm/IR/Module.h"
+#include "llvm/Transforms/Utils/ValueMapper.h"
 #include <sstream>
 
 namespace llvm {
@@ -31,28 +33,22 @@ class JITCompileCallbackManagerBase {
 public:
 
   typedef std::function<TargetAddress()> CompileFtor;
-  typedef std::function<void(TargetAddress)> UpdateFtor;
 
   /// @brief Handle to a newly created compile callback. Can be used to get an
   ///        IR constant representing the address of the trampoline, and to set
-  ///        the compile and update actions for the callback.
+  ///        the compile action for the callback.
   class CompileCallbackInfo {
   public:
-    CompileCallbackInfo(TargetAddress Addr, CompileFtor &Compile,
-                        UpdateFtor &Update)
-      : Addr(Addr), Compile(Compile), Update(Update) {}
+    CompileCallbackInfo(TargetAddress Addr, CompileFtor &Compile)
+      : Addr(Addr), Compile(Compile) {}
 
     TargetAddress getAddress() const { return Addr; }
     void setCompileAction(CompileFtor Compile) {
       this->Compile = std::move(Compile);
     }
-    void setUpdateAction(UpdateFtor Update) {
-      this->Update = std::move(Update);
-    }
   private:
     TargetAddress Addr;
     CompileFtor &Compile;
-    UpdateFtor &Update;
   };
 
   /// @brief Construct a JITCompileCallbackManagerBase.
@@ -70,8 +66,8 @@ class JITCompileCallbackManagerBase {
 
   /// @brief Execute the callback for the given trampoline id. Called by the JIT
   ///        to compile functions on demand.
-  TargetAddress executeCompileCallback(TargetAddress TrampolineID) {
-    TrampolineMapT::iterator I = ActiveTrampolines.find(TrampolineID);
+  TargetAddress executeCompileCallback(TargetAddress TrampolineAddr) {
+    auto I = ActiveTrampolines.find(TrampolineAddr);
     // FIXME: Also raise an error in the Orc error-handler when we finally have
     //        one.
     if (I == ActiveTrampolines.end())
@@ -83,31 +79,43 @@ class JITCompileCallbackManagerBase {
     // Moving the trampoline ID back to the available list first means there's at
     // least one available trampoline if the compile action triggers a request for
     // a new one.
-    AvailableTrampolines.push_back(I->first);
-    auto CallbackHandler = std::move(I->second);
+    auto Compile = std::move(I->second);
     ActiveTrampolines.erase(I);
+    AvailableTrampolines.push_back(TrampolineAddr);
 
-    if (auto Addr = CallbackHandler.Compile()) {
-      CallbackHandler.Update(Addr);
+    if (auto Addr = Compile())
       return Addr;
-    }
+
     return ErrorHandlerAddress;
   }
 
-  /// @brief Get/create a compile callback with the given signature.
+  /// @brief Reserve a compile callback.
   virtual CompileCallbackInfo getCompileCallback(LLVMContext &Context) = 0;
 
-protected:
+  /// @brief Get a CompileCallbackInfo for an existing callback.
+  CompileCallbackInfo getCompileCallbackInfo(TargetAddress TrampolineAddr) {
+    auto I = ActiveTrampolines.find(TrampolineAddr);
+    assert(I != ActiveTrampolines.end() && "Not an active trampoline.");
+    return CompileCallbackInfo(I->first, I->second);
+  }
 
-  struct CallbackHandler {
-    CompileFtor Compile;
-    UpdateFtor Update;
-  };
+  /// @brief Release a compile callback.
+  ///
+  ///   Note: Callbacks are auto-released after they execute. This method should
+  /// only be called to manually release a callback that is not going to
+  /// execute.
+  void releaseCompileCallback(TargetAddress TrampolineAddr) {
+    auto I = ActiveTrampolines.find(TrampolineAddr);
+    assert(I != ActiveTrampolines.end() && "Not an active trampoline.");
+    ActiveTrampolines.erase(I);
+    AvailableTrampolines.push_back(TrampolineAddr);
+  }
 
+protected:
   TargetAddress ErrorHandlerAddress;
   unsigned NumTrampolinesPerBlock;
 
-  typedef std::map<TargetAddress, CallbackHandler> TrampolineMapT;
+  typedef std::map<TargetAddress, CompileFtor> TrampolineMapT;
   TrampolineMapT ActiveTrampolines;
   std::vector<TargetAddress> AvailableTrampolines;
 };
@@ -139,11 +147,8 @@ class JITCompileCallbackManager : public JITCompileCallbackManagerBase {
   /// @brief Get/create a compile callback with the given signature.
   CompileCallbackInfo getCompileCallback(LLVMContext &Context) final {
     TargetAddress TrampolineAddr = getAvailableTrampolineAddr(Context);
-    auto &CallbackHandler =
-      this->ActiveTrampolines[TrampolineAddr];
-
-    return CompileCallbackInfo(TrampolineAddr, CallbackHandler.Compile,
-                               CallbackHandler.Update);
+    auto &Compile = this->ActiveTrampolines[TrampolineAddr];
+    return CompileCallbackInfo(TrampolineAddr, Compile);
   }
 
 private:
@@ -159,9 +164,16 @@ class JITCompileCallbackManager : public JITCompileCallbackManagerBase {
     std::unique_ptr<Module> M(new Module("resolver_block_module",
                                          Context));
     TargetT::insertResolverBlock(*M, *this);
+    auto NonResolver =
+      createLambdaResolver(
+          [](const std::string &Name) -> RuntimeDyld::SymbolInfo {
+            llvm_unreachable("External symbols in resolver block?");
+          },
+          [](const std::string &Name) -> RuntimeDyld::SymbolInfo {
+            llvm_unreachable("Dylib symbols in resolver block?");
+          });
     auto H = JIT.addModuleSet(SingletonSet(std::move(M)), &MemMgr,
-                              static_cast<RuntimeDyld::SymbolResolver*>(
-                                  nullptr));
+                              std::move(NonResolver));
     JIT.emitAndFinalize(H);
     auto ResolverBlockSymbol =
       JIT.findSymbolIn(H, TargetT::ResolverBlockName, false);
@@ -186,9 +198,16 @@ class JITCompileCallbackManager : public JITCompileCallbackManagerBase {
       TargetT::insertCompileCallbackTrampolines(*M, ResolverBlockAddr,
                                                 this->NumTrampolinesPerBlock,
                                                 this->ActiveTrampolines.size());
+    auto NonResolver =
+      createLambdaResolver(
+          [](const std::string &Name) -> RuntimeDyld::SymbolInfo {
+            llvm_unreachable("External symbols in trampoline block?");
+          },
+          [](const std::string &Name) -> RuntimeDyld::SymbolInfo {
+            llvm_unreachable("Dylib symbols in trampoline block?");
+          });
     auto H = JIT.addModuleSet(SingletonSet(std::move(M)), &MemMgr,
-                              static_cast<RuntimeDyld::SymbolResolver*>(
-                                  nullptr));
+                              std::move(NonResolver));
     JIT.emitAndFinalize(H);
     for (unsigned I = 0; I < this->NumTrampolinesPerBlock; ++I) {
       std::string Name = GetLabelName(I);
@@ -203,22 +222,6 @@ class JITCompileCallbackManager : public JITCompileCallbackManagerBase {
   TargetAddress ResolverBlockAddr;
 };
 
-/// @brief Get an update functor that updates the value of a named function
-///        pointer.
-template <typename JITLayerT>
-JITCompileCallbackManagerBase::UpdateFtor
-getLocalFPUpdater(JITLayerT &JIT, typename JITLayerT::ModuleSetHandleT H,
-                  std::string Name) {
-    // FIXME: Move-capture Name once we can use C++14.
-    return [=,&JIT](TargetAddress Addr) {
-      auto FPSym = JIT.findSymbolIn(H, Name, true);
-      assert(FPSym && "Cannot find function pointer to update.");
-      void *FPAddr = reinterpret_cast<void*>(
-                       static_cast<uintptr_t>(FPSym.getAddress()));
-      memcpy(FPAddr, &Addr, sizeof(uintptr_t));
-    };
-  }
-
 /// @brief Build a function pointer of FunctionType with the given constant
 ///        address.
 ///
@@ -235,27 +238,56 @@ GlobalVariable* createImplPointer(PointerType &PT, Module &M,
 ///        indirect call using the given function pointer.
 void makeStub(Function &F, GlobalVariable &ImplPointer);
 
-typedef std::map<Module*, DenseSet<const GlobalValue*>> ModulePartitionMap;
-
-/// @brief Extract subsections of a Module into the given Module according to
-///        the given ModulePartitionMap.
-void partition(Module &M, const ModulePartitionMap &PMap);
-
-/// @brief Struct for trivial "complete" partitioning of a module.
-class FullyPartitionedModule {
-public:
-  std::unique_ptr<Module> GlobalVars;
-  std::unique_ptr<Module> Commons;
-  std::vector<std::unique_ptr<Module>> Functions;
-
-  FullyPartitionedModule() = default;
-  FullyPartitionedModule(FullyPartitionedModule &&S)
-      : GlobalVars(std::move(S.GlobalVars)), Commons(std::move(S.Commons)),
-        Functions(std::move(S.Functions)) {}
-};
+/// @brief Raise linkage types and rename as necessary to ensure that all
+///        symbols are accessible for other modules.
+///
+///   This should be called before partitioning a module to ensure that the
+/// partitions retain access to each other's symbols.
+void makeAllSymbolsExternallyAccessible(Module &M);
 
-/// @brief Extract every function in M into a separate module.
-FullyPartitionedModule fullyPartition(Module &M);
+/// @brief Clone a function declaration into a new module.
+///
+///   This function can be used as the first step towards creating a callback
+/// stub (see makeStub), or moving a function body (see moveFunctionBody).
+///
+///   If the VMap argument is non-null, a mapping will be added between F and
+/// the new declaration, and between each of F's arguments and the new
+/// declaration's arguments. This map can then be passed in to moveFunction to
+/// move the function body if required. Note: When moving functions between
+/// modules with these utilities, all decls should be cloned (and added to a
+/// single VMap) before any bodies are moved. This will ensure that references
+/// between functions all refer to the versions in the new module.
+Function* cloneFunctionDecl(Module &Dst, const Function &F,
+                            ValueToValueMapTy *VMap = nullptr);
+
+/// @brief Move the body of function 'F' to a cloned function declaration in a
+///        different module (See related cloneFunctionDecl).
+///
+///   If the target function declaration is not supplied via the NewF parameter
+/// then it will be looked up via the VMap.
+///
+///   This will delete the body of function 'F' from its original parent module,
+/// but leave its declaration.
+void moveFunctionBody(Function &OrigF, ValueToValueMapTy &VMap,
+                      ValueMaterializer *Materializer = nullptr,
+                      Function *NewF = nullptr);
+
+/// @brief Clone a global variable declaration into a new module.
+GlobalVariable* cloneGlobalVariableDecl(Module &Dst, const GlobalVariable &GV,
+                                        ValueToValueMapTy *VMap = nullptr);
+
+/// @brief Move global variable GV from its parent module to cloned global
+///        declaration in a different module.
+///
+///   If the target global declaration is not supplied via the NewGV parameter
+/// then it will be looked up via the VMap.
+///
+///   This will delete the initializer of GV from its original parent module,
+/// but leave its declaration.
+void moveGlobalVariableInitializer(GlobalVariable &OrigGV,
+                                   ValueToValueMapTy &VMap,
+                                   ValueMaterializer *Materializer = nullptr,
+                                   GlobalVariable *NewGV = nullptr);
 
 } // End namespace orc.
 } // End namespace llvm.
diff --git a/include/llvm/ExecutionEngine/Orc/LazyEmittingLayer.h b/include/llvm/ExecutionEngine/Orc/LazyEmittingLayer.h
index 71c83f7e05f..93ba02b3870 100644
--- a/include/llvm/ExecutionEngine/Orc/LazyEmittingLayer.h
+++ b/include/llvm/ExecutionEngine/Orc/LazyEmittingLayer.h
@@ -193,7 +193,7 @@ template <typename BaseLayerT> class LazyEmittingLayer {
       auto Symbols = llvm::make_unique<StringMap<const GlobalValue*>>();
 
       for (const auto &M : Ms) {
-        Mangler Mang(&M->getDataLayout());
+        Mangler Mang;
 
         for (const auto &V : M->globals())
           if (auto GV = addGlobalValue(*Symbols, V, Mang, SearchName,
diff --git a/include/llvm/ExecutionEngine/Orc/LogicalDylib.h b/include/llvm/ExecutionEngine/Orc/LogicalDylib.h
new file mode 100644
index 00000000000..28700ef347d
--- /dev/null
+++ b/include/llvm/ExecutionEngine/Orc/LogicalDylib.h
@@ -0,0 +1,115 @@
+//===--- LogicalDylib.h - Simulates dylib-style symbol lookup ---*- C++ -*-===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// Simulates symbol resolution inside a dylib.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_EXECUTIONENGINE_ORC_LOGICALDYLIB_H
+#define LLVM_EXECUTIONENGINE_ORC_LOGICALDYLIB_H
+
+namespace llvm {
+namespace orc {
+
+template <typename BaseLayerT,
+          typename LogicalModuleResources,
+          typename LogicalDylibResources>
+class LogicalDylib {
+public:
+  typedef typename BaseLayerT::ModuleSetHandleT BaseLayerModuleSetHandleT;
+private:
+
+  typedef std::vector<BaseLayerModuleSetHandleT> BaseLayerHandleList;
+
+  struct LogicalModule {
+    LogicalModuleResources Resources;
+    BaseLayerHandleList BaseLayerHandles;
+  };
+  typedef std::vector<LogicalModule> LogicalModuleList;
+
+public:
+
+  typedef typename BaseLayerHandleList::iterator BaseLayerHandleIterator;
+  typedef typename LogicalModuleList::iterator LogicalModuleHandle;
+
+  LogicalDylib(BaseLayerT &BaseLayer) : BaseLayer(BaseLayer) {}
+
+  ~LogicalDylib() {
+    for (auto &LM : LogicalModules)
+      for (auto BLH : LM.BaseLayerHandles)
+        BaseLayer.removeModuleSet(BLH);
+  }
+
+  LogicalModuleHandle createLogicalModule() {
+    LogicalModules.push_back(LogicalModule());
+    return std::prev(LogicalModules.end());
+  }
+
+  void addToLogicalModule(LogicalModuleHandle LMH,
+                          BaseLayerModuleSetHandleT BaseLayerHandle) {
+    LMH->BaseLayerHandles.push_back(BaseLayerHandle);
+  }
+
+  LogicalModuleResources& getLogicalModuleResources(LogicalModuleHandle LMH) {
+    return LMH->Resources;
+  }
+
+  BaseLayerHandleIterator moduleHandlesBegin(LogicalModuleHandle LMH) {
+    return LMH->BaseLayerHandles.begin();
+  }
+
+  BaseLayerHandleIterator moduleHandlesEnd(LogicalModuleHandle LMH) {
+    return LMH->BaseLayerHandles.end();
+  }
+
+  JITSymbol findSymbolInLogicalModule(LogicalModuleHandle LMH,
+                                      const std::string &Name) {
+    for (auto BLH : LMH->BaseLayerHandles)
+      if (auto Symbol = BaseLayer.findSymbolIn(BLH, Name, false))
+        return Symbol;
+    return nullptr;
+  }
+
+  JITSymbol findSymbolInternally(LogicalModuleHandle LMH,
+                                 const std::string &Name) {
+    if (auto Symbol = findSymbolInLogicalModule(LMH, Name))
+      return Symbol;
+
+    for (auto LMI = LogicalModules.begin(), LME = LogicalModules.end();
+           LMI != LME; ++LMI) {
+      if (LMI != LMH)
+        if (auto Symbol = findSymbolInLogicalModule(LMI, Name))
+          return Symbol;
+    }
+
+    return nullptr;
+  }
+
+  JITSymbol findSymbol(const std::string &Name, bool ExportedSymbolsOnly) {
+    for (auto &LM : LogicalModules)
+      for (auto BLH : LM.BaseLayerHandles)
+        if (auto Symbol =
+            BaseLayer.findSymbolIn(BLH, Name, ExportedSymbolsOnly))
+          return Symbol;
+    return nullptr;
+  }
+
+  LogicalDylibResources& getDylibResources() { return DylibResources; }
+
+protected:
+  BaseLayerT BaseLayer;
+  LogicalModuleList LogicalModules;
+  LogicalDylibResources DylibResources;
+
+};
+
+} // End namespace orc.
+} // End namespace llvm.
+
+#endif // LLVM_EXECUTIONENGINE_ORC_LOGICALDYLIB_H
diff --git a/include/llvm/ExecutionEngine/Orc/NullResolver.h b/include/llvm/ExecutionEngine/Orc/NullResolver.h
new file mode 100644
index 00000000000..1560c6d86e0
--- /dev/null
+++ b/include/llvm/ExecutionEngine/Orc/NullResolver.h
@@ -0,0 +1,36 @@
+//===------ NullResolver.h - Reject symbol lookup requests ------*- C++ -*-===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+//   Defines a RuntimeDyld::SymbolResolver subclass that rejects all symbol
+// resolution requests, for clients that have no cross-object fixups.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_EXECUTIONENGINE_ORC_NULLRESOLVER_H
+#define LLVM_EXECUTIONENGINE_ORC_NULLRESOLVER_H
+
+#include "llvm/ExecutionEngine/RuntimeDyld.h"
+
+namespace llvm {
+namespace orc {
+
+/// SymbolResolver impliementation that rejects all resolution requests.
+/// Useful for clients that have no cross-object fixups.
+class NullResolver : public RuntimeDyld::SymbolResolver {
+public:
+  RuntimeDyld::SymbolInfo findSymbol(const std::string &Name) final;
+
+  RuntimeDyld::SymbolInfo
+  findSymbolInLogicalDylib(const std::string &Name) final;
+};
+
+} // End namespace orc.
+} // End namespace llvm.
+
+#endif // LLVM_EXECUTIONENGINE_ORC_NULLRESOLVER_H
diff --git a/include/llvm/ExecutionEngine/Orc/ObjectTransformLayer.h b/include/llvm/ExecutionEngine/Orc/ObjectTransformLayer.h
new file mode 100644
index 00000000000..7af66208547
--- /dev/null
+++ b/include/llvm/ExecutionEngine/Orc/ObjectTransformLayer.h
@@ -0,0 +1,112 @@
+//===- ObjectTransformLayer.h - Run all objects through functor -*- C++ -*-===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// Run all objects passed in through a user supplied functor.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_EXECUTIONENGINE_ORC_OBJECTTRANSFORMLAYER_H
+#define LLVM_EXECUTIONENGINE_ORC_OBJECTTRANSFORMLAYER_H
+
+#include "JITSymbol.h"
+
+namespace llvm {
+namespace orc {
+
+/// @brief Object mutating layer.
+///
+///   This layer accepts sets of ObjectFiles (via addObjectSet). It
+/// immediately applies the user supplied functor to each object, then adds
+/// the set of transformed objects to the layer below.
+template <typename BaseLayerT, typename TransformFtor>
+class ObjectTransformLayer {
+public:
+  /// @brief Handle to a set of added objects.
+  typedef typename BaseLayerT::ObjSetHandleT ObjSetHandleT;
+
+  /// @brief Construct an ObjectTransformLayer with the given BaseLayer
+  ObjectTransformLayer(BaseLayerT &BaseLayer,
+                       TransformFtor Transform = TransformFtor())
+      : BaseLayer(BaseLayer), Transform(std::move(Transform)) {}
+
+  /// @brief Apply the transform functor to each object in the object set, then
+  ///        add the resulting set of objects to the base layer, along with the
+  ///        memory manager and symbol resolver.
+  ///
+  /// @return A handle for the added objects.
+  template <typename ObjSetT, typename MemoryManagerPtrT,
+            typename SymbolResolverPtrT>
+  ObjSetHandleT addObjectSet(ObjSetT &Objects, MemoryManagerPtrT MemMgr,
+                             SymbolResolverPtrT Resolver) {
+
+    for (auto I = Objects.begin(), E = Objects.end(); I != E; ++I)
+      *I = Transform(std::move(*I));
+
+    return BaseLayer.addObjectSet(Objects, std::move(MemMgr),
+                                  std::move(Resolver));
+  }
+
+  /// @brief Remove the object set associated with the handle H.
+  void removeObjectSet(ObjSetHandleT H) { BaseLayer.removeObjectSet(H); }
+
+  /// @brief Search for the given named symbol.
+  /// @param Name The name of the symbol to search for.
+  /// @param ExportedSymbolsOnly If true, search only for exported symbols.
+  /// @return A handle for the given named symbol, if it exists.
+  JITSymbol findSymbol(const std::string &Name, bool ExportedSymbolsOnly) {
+    return BaseLayer.findSymbol(Name, ExportedSymbolsOnly);
+  }
+
+  /// @brief Get the address of the given symbol in the context of the set of
+  ///        objects represented by the handle H. This call is forwarded to the
+  ///        base layer's implementation.
+  /// @param H The handle for the object set to search in.
+  /// @param Name The name of the symbol to search for.
+  /// @param ExportedSymbolsOnly If true, search only for exported symbols.
+  /// @return A handle for the given named symbol, if it is found in the
+  ///         given object set.
+  JITSymbol findSymbolIn(ObjSetHandleT H, const std::string &Name,
+                         bool ExportedSymbolsOnly) {
+    return BaseLayer.findSymbolIn(H, Name, ExportedSymbolsOnly);
+  }
+
+  /// @brief Immediately emit and finalize the object set represented by the
+  ///        given handle.
+  /// @param H Handle for object set to emit/finalize.
+  void emitAndFinalize(ObjSetHandleT H) { BaseLayer.emitAndFinalize(H); }
+
+  /// @brief Map section addresses for the objects associated with the handle H.
+  void mapSectionAddress(ObjSetHandleT H, const void *LocalAddress,
+                         TargetAddress TargetAddr) {
+    BaseLayer.mapSectionAddress(H, LocalAddress, TargetAddr);
+  }
+
+  // Ownership hack.
+  // FIXME: Remove this as soon as RuntimeDyldELF can apply relocations without
+  //        referencing the original object.
+  template <typename OwningMBSet>
+  void takeOwnershipOfBuffers(ObjSetHandleT H, OwningMBSet MBs) {
+    BaseLayer.takeOwnershipOfBuffers(H, std::move(MBs));
+  }
+
+  /// @brief Access the transform functor directly.
+  TransformFtor &getTransform() { return Transform; }
+
+  /// @brief Access the mumate functor directly.
+  const TransformFtor &getTransform() const { return Transform; }
+
+private:
+  BaseLayerT &BaseLayer;
+  TransformFtor Transform;
+};
+
+} // End namespace orc.
+} // End namespace llvm.
+
+#endif // LLVM_EXECUTIONENGINE_ORC_OBJECTTRANSFORMLAYER_H
diff --git a/include/llvm/ExecutionEngine/RuntimeDyld.h b/include/llvm/ExecutionEngine/RuntimeDyld.h
index 5723f058f72..17c5fbd5649 100644
--- a/include/llvm/ExecutionEngine/RuntimeDyld.h
+++ b/include/llvm/ExecutionEngine/RuntimeDyld.h
@@ -15,8 +15,12 @@
 #define LLVM_EXECUTIONENGINE_RUNTIMEDYLD_H
 
 #include "JITSymbolFlags.h"
+#include "llvm/ADT/STLExtras.h"
 #include "llvm/ADT/StringRef.h"
+#include "llvm/Object/ObjectFile.h"
 #include "llvm/Support/Memory.h"
+#include "llvm/DebugInfo/DIContext.h"
+#include <map>
 #include <memory>
 
 namespace llvm {
@@ -54,31 +58,39 @@ class RuntimeDyld {
   };
 
   /// \brief Information about the loaded object.
-  class LoadedObjectInfo {
+  class LoadedObjectInfo : public llvm::LoadedObjectInfo {
     friend class RuntimeDyldImpl;
   public:
-    LoadedObjectInfo(RuntimeDyldImpl &RTDyld, unsigned BeginIdx,
-                     unsigned EndIdx)
-      : RTDyld(RTDyld), BeginIdx(BeginIdx), EndIdx(EndIdx) { }
+    typedef std::map<object::SectionRef, unsigned> ObjSectionToIDMap;
 
-    virtual ~LoadedObjectInfo() {}
+    LoadedObjectInfo(RuntimeDyldImpl &RTDyld, ObjSectionToIDMap ObjSecToIDMap)
+      : RTDyld(RTDyld), ObjSecToIDMap(ObjSecToIDMap) { }
 
     virtual object::OwningBinary<object::ObjectFile>
     getObjectForDebug(const object::ObjectFile &Obj) const = 0;
 
-    uint64_t getSectionLoadAddress(StringRef Name) const;
+    uint64_t getSectionLoadAddress(const object::SectionRef &Sec) const;
 
   protected:
     virtual void anchor();
 
     RuntimeDyldImpl &RTDyld;
-    unsigned BeginIdx, EndIdx;
+    ObjSectionToIDMap ObjSecToIDMap;
+  };
+
+  template <typename Derived> struct LoadedObjectInfoHelper : LoadedObjectInfo {
+    LoadedObjectInfoHelper(RuntimeDyldImpl &RTDyld,
+                           LoadedObjectInfo::ObjSectionToIDMap ObjSecToIDMap)
+        : LoadedObjectInfo(RTDyld, std::move(ObjSecToIDMap)) {}
+    std::unique_ptr<llvm::LoadedObjectInfo> clone() const override {
+      return llvm::make_unique<Derived>(static_cast<const Derived &>(*this));
+    }
   };
 
   /// \brief Memory Management.
   class MemoryManager {
   public:
-    virtual ~MemoryManager() {};
+    virtual ~MemoryManager() {}
 
     /// Allocate a memory block of (at least) the given size suitable for
     /// executable code. The SectionID is a unique identifier assigned by the
@@ -140,10 +152,14 @@ class RuntimeDyld {
   /// \brief Symbol resolution.
   class SymbolResolver {
   public:
-    virtual ~SymbolResolver() {};
+    virtual ~SymbolResolver() {}
 
     /// This method returns the address of the specified function or variable.
     /// It is used to resolve symbols during module linking.
+    ///
+    /// If the returned symbol's address is equal to ~0ULL then RuntimeDyld will
+    /// skip all relocations for that symbol, and the client will be responsible
+    /// for handling them manually.
     virtual SymbolInfo findSymbol(const std::string &Name) = 0;
 
     /// This method returns the address of the specified symbol if it exists
diff --git a/include/llvm/IR/Argument.h b/include/llvm/IR/Argument.h
index dd76a90aa5e..fc04fe71cbf 100644
--- a/include/llvm/IR/Argument.h
+++ b/include/llvm/IR/Argument.h
@@ -64,6 +64,11 @@ class Argument : public Value, public ilist_node<Argument> {
   /// containing function, return the number of bytes known to be
   /// dereferenceable. Otherwise, zero is returned.
   uint64_t getDereferenceableBytes() const;
+  
+  /// \brief If this argument has the dereferenceable_or_null attribute on 
+  /// it in its containing function, return the number of bytes known to be
+  /// dereferenceable. Otherwise, zero is returned.
+  uint64_t getDereferenceableOrNullBytes() const;
 
   /// \brief Return true if this argument has the byval attribute on it in its
   /// containing function.
diff --git a/include/llvm/IR/Attributes.h b/include/llvm/IR/Attributes.h
index d0d1b5e57a9..4d6d7da1fa5 100644
--- a/include/llvm/IR/Attributes.h
+++ b/include/llvm/IR/Attributes.h
@@ -73,6 +73,7 @@ class Attribute {
     ByVal,                 ///< Pass structure by value
     InAlloca,              ///< Pass structure in an alloca
     Cold,                  ///< Marks function as being in a cold path.
+    Convergent,            ///< Can only be moved to control-equivalent blocks
     InlineHint,            ///< Source said inlining was desirable
     InReg,                 ///< Force argument to be passed in register
     JumpTable,             ///< Build jump-instruction tables and replace refs.
@@ -97,6 +98,8 @@ class Attribute {
     OptimizeNone,          ///< Function must not be optimized.
     ReadNone,              ///< Function does not access memory
     ReadOnly,              ///< Function only reads from memory
+    ArgMemOnly,            ///< Funciton can access memory only using pointers
+                           ///< based on its arguments.
     Returned,              ///< Return value is always equal to this argument
     ReturnsTwice,          ///< Function can return twice
     SExt,                  ///< Sign extended before/after call
@@ -107,6 +110,7 @@ class Attribute {
     StackProtect,          ///< Stack protection.
     StackProtectReq,       ///< Stack protection required.
     StackProtectStrong,    ///< Strong Stack protection.
+    SafeStack,             ///< Safe Stack protection.
     StructRet,             ///< Hidden pointer to structure to return
     SanitizeAddress,       ///< AddressSanitizer is on.
     SanitizeThread,        ///< ThreadSanitizer is on.
@@ -260,42 +264,48 @@ class AttributeSet {
                           ArrayRef<Attribute::AttrKind> Kind);
   static AttributeSet get(LLVMContext &C, unsigned Index, const AttrBuilder &B);
 
-  /// \brief Add an attribute to the attribute set at the given index. Since
+  /// \brief Add an attribute to the attribute set at the given index. Because
   /// attribute sets are immutable, this returns a new set.
   AttributeSet addAttribute(LLVMContext &C, unsigned Index,
                             Attribute::AttrKind Attr) const;
 
-  /// \brief Add an attribute to the attribute set at the given index. Since
+  /// \brief Add an attribute to the attribute set at the given index. Because
   /// attribute sets are immutable, this returns a new set.
   AttributeSet addAttribute(LLVMContext &C, unsigned Index,
                             StringRef Kind) const;
   AttributeSet addAttribute(LLVMContext &C, unsigned Index,
                             StringRef Kind, StringRef Value) const;
 
-  /// \brief Add attributes to the attribute set at the given index. Since
+  /// \brief Add attributes to the attribute set at the given index. Because
   /// attribute sets are immutable, this returns a new set.
   AttributeSet addAttributes(LLVMContext &C, unsigned Index,
                              AttributeSet Attrs) const;
 
   /// \brief Remove the specified attribute at the specified index from this
-  /// attribute list. Since attribute lists are immutable, this returns the new
-  /// list.
+  /// attribute list. Because attribute lists are immutable, this returns the
+  /// new list.
   AttributeSet removeAttribute(LLVMContext &C, unsigned Index, 
                                Attribute::AttrKind Attr) const;
 
   /// \brief Remove the specified attributes at the specified index from this
-  /// attribute list. Since attribute lists are immutable, this returns the new
-  /// list.
+  /// attribute list. Because attribute lists are immutable, this returns the
+  /// new list.
   AttributeSet removeAttributes(LLVMContext &C, unsigned Index, 
                                 AttributeSet Attrs) const;
 
+  /// \brief Remove the specified attributes at the specified index from this
+  /// attribute list. Because attribute lists are immutable, this returns the
+  /// new list.
+  AttributeSet removeAttributes(LLVMContext &C, unsigned Index,
+                                const AttrBuilder &Attrs) const;
+
   /// \brief Add the dereferenceable attribute to the attribute set at the given
-  /// index. Since attribute sets are immutable, this returns a new set.
+  /// index. Because attribute sets are immutable, this returns a new set.
   AttributeSet addDereferenceableAttr(LLVMContext &C, unsigned Index,
                                       uint64_t Bytes) const;
 
   /// \brief Add the dereferenceable_or_null attribute to the attribute set at
-  /// the given index. Since attribute sets are immutable, this returns a new
+  /// the given index. Because attribute sets are immutable, this returns a new
   /// set.
   AttributeSet addDereferenceableOrNullAttr(LLVMContext &C, unsigned Index,
                                             uint64_t Bytes) const;
@@ -430,13 +440,17 @@ class AttrBuilder {
   uint64_t DerefBytes;
   uint64_t DerefOrNullBytes;
 public:
-  AttrBuilder() : Attrs(0), Alignment(0), StackAlignment(0), DerefBytes(0) {}
+  AttrBuilder()
+      : Attrs(0), Alignment(0), StackAlignment(0), DerefBytes(0),
+        DerefOrNullBytes(0) {}
   explicit AttrBuilder(uint64_t Val)
-    : Attrs(0), Alignment(0), StackAlignment(0), DerefBytes(0) {
+      : Attrs(0), Alignment(0), StackAlignment(0), DerefBytes(0),
+        DerefOrNullBytes(0) {
     addRawValue(Val);
   }
   AttrBuilder(const Attribute &A)
-    : Attrs(0), Alignment(0), StackAlignment(0), DerefBytes(0) {
+      : Attrs(0), Alignment(0), StackAlignment(0), DerefBytes(0),
+        DerefOrNullBytes(0) {
     addAttribute(A);
   }
   AttrBuilder(AttributeSet AS, unsigned Idx);
@@ -464,6 +478,13 @@ class AttrBuilder {
   /// \brief Add the attributes from the builder.
   AttrBuilder &merge(const AttrBuilder &B);
 
+  /// \brief Remove the attributes from the builder.
+  AttrBuilder &remove(const AttrBuilder &B);
+
+  /// \brief Return true if the builder has any attribute that's in the
+  /// specified builder.
+  bool overlaps(const AttrBuilder &B) const;
+
   /// \brief Return true if the builder has the specified attribute.
   bool contains(Attribute::AttrKind A) const {
     assert((unsigned)A < Attribute::EndAttrKinds && "Attribute out of range!");
@@ -552,7 +573,7 @@ class AttrBuilder {
 namespace AttributeFuncs {
 
 /// \brief Which attributes cannot be applied to a type.
-AttributeSet typeIncompatible(Type *Ty, uint64_t Index);
+AttrBuilder typeIncompatible(const Type *Ty);
 
 } // end AttributeFuncs namespace
 
diff --git a/include/llvm/IR/BasicBlock.h b/include/llvm/IR/BasicBlock.h
index a71946eda6e..08e30bdc941 100644
--- a/include/llvm/IR/BasicBlock.h
+++ b/include/llvm/IR/BasicBlock.h
@@ -116,6 +116,7 @@ class BasicBlock : public Value, // Basic blocks are data objects also
   ///
   /// Note: this is undefined behavior if the block does not have a parent.
   const Module *getModule() const;
+  Module *getModule();
 
   /// \brief Returns the terminator instruction if the block is well formed or
   /// null if the block is not well formed.
@@ -206,9 +207,19 @@ class BasicBlock : public Value, // Basic blocks are data objects also
     return const_cast<BasicBlock*>(this)->getUniquePredecessor();
   }
 
-  /// Return the successor of this block if it has a unique successor.
-  /// Otherwise return a null pointer.  This method is analogous to
-  /// getUniquePredeccessor above.
+  /// \brief Return the successor of this block if it has a single successor.
+  /// Otherwise return a null pointer.
+  ///
+  /// This method is analogous to getSinglePredecessor above.
+  BasicBlock *getSingleSuccessor();
+  const BasicBlock *getSingleSuccessor() const {
+    return const_cast<BasicBlock*>(this)->getSingleSuccessor();
+  }
+
+  /// \brief Return the successor of this block if it has a unique successor.
+  /// Otherwise return a null pointer.
+  ///
+  /// This method is analogous to getUniquePredecessor above.
   BasicBlock *getUniqueSuccessor();
   const BasicBlock *getUniqueSuccessor() const {
     return const_cast<BasicBlock*>(this)->getUniqueSuccessor();
@@ -272,6 +283,8 @@ class BasicBlock : public Value, // Basic blocks are data objects also
   /// should be called while the predecessor still refers to this block.
   void removePredecessor(BasicBlock *Pred, bool DontDeleteUselessPHIs = false);
 
+  bool canSplitPredecessors() const;
+
   /// \brief Split the basic block into two basic blocks at the specified
   /// instruction.
   ///
@@ -298,6 +311,9 @@ class BasicBlock : public Value, // Basic blocks are data objects also
   /// basic block \p New instead of to it.
   void replaceSuccessorsPhiUsesWith(BasicBlock *New);
 
+  /// \brief Return true if this basic block is an exception handling block.
+  bool isEHPad() const { return getFirstNonPHI()->isEHPad(); }
+
   /// \brief Return true if this basic block is a landing pad.
   ///
   /// Being a ``landing pad'' means that the basic block is the destination of
diff --git a/include/llvm/IR/CallSite.h b/include/llvm/IR/CallSite.h
index ebace3395ac..2841781e8a9 100644
--- a/include/llvm/IR/CallSite.h
+++ b/include/llvm/IR/CallSite.h
@@ -27,6 +27,7 @@
 #define LLVM_IR_CALLSITE_H
 
 #include "llvm/ADT/PointerIntPair.h"
+#include "llvm/ADT/iterator_range.h"
 #include "llvm/IR/Attributes.h"
 #include "llvm/IR/CallingConv.h"
 #include "llvm/IR/Instructions.h"
@@ -37,6 +38,7 @@ class CallInst;
 class InvokeInst;
 
 template <typename FunTy = const Function,
+          typename BBTy = const BasicBlock,
           typename ValTy = const Value,
           typename UserTy = const User,
           typename InstrTy = const Instruction,
@@ -81,6 +83,9 @@ class CallSiteBase {
   InstrTy *operator->() const { return I.getPointer(); }
   explicit operator bool() const { return I.getPointer(); }
 
+  /// Get the basic block containing the call site
+  BBTy* getParent() const { return getInstruction()->getParent(); }
+
   /// getCalledValue - Return the pointer to function that is being called.
   ///
   ValTy *getCalledValue() const {
@@ -150,6 +155,9 @@ class CallSiteBase {
   }
 
   IterTy arg_end() const { return (*this)->op_end() - getArgumentEndOffset(); }
+  iterator_range<IterTy> args() const {
+    return iterator_range<IterTy>(arg_begin(), arg_end());
+  }
   bool arg_empty() const { return arg_end() == arg_begin(); }
   unsigned arg_size() const { return unsigned(arg_end() - arg_begin()); }
 
@@ -185,6 +193,20 @@ class CallSiteBase {
   else                                   \
     cast<InvokeInst>(II)->METHOD
 
+  unsigned getNumArgOperands() const {
+    CALLSITE_DELEGATE_GETTER(getNumArgOperands());
+  }
+
+  ValTy *getArgOperand(unsigned i) const { 
+    CALLSITE_DELEGATE_GETTER(getArgOperand(i));
+  }
+
+  bool isInlineAsm() const { 
+    if (isCall())
+      return cast<CallInst>(getInstruction())->isInlineAsm();
+    return false;
+  }
+
   /// getCallingConv/setCallingConv - get or set the calling convention of the
   /// call.
   CallingConv::ID getCallingConv() const {
@@ -194,6 +216,14 @@ class CallSiteBase {
     CALLSITE_DELEGATE_SETTER(setCallingConv(CC));
   }
 
+  FunctionType *getFunctionType() const {
+    CALLSITE_DELEGATE_GETTER(getFunctionType());
+  }
+
+  void mutateFunctionType(FunctionType *Ty) const {
+    CALLSITE_DELEGATE_SETTER(mutateFunctionType(Ty));
+  }
+
   /// getAttributes/setAttributes - get or set the parameter attributes of
   /// the call.
   const AttributeSet &getAttributes() const {
@@ -223,7 +253,13 @@ class CallSiteBase {
   uint64_t getDereferenceableBytes(uint16_t i) const {
     CALLSITE_DELEGATE_GETTER(getDereferenceableBytes(i));
   }
-
+  
+  /// @brief Extract the number of dereferenceable_or_null bytes for a call or
+  /// parameter (0=unknown).
+  uint64_t getDereferenceableOrNullBytes(uint16_t i) const {
+    CALLSITE_DELEGATE_GETTER(getDereferenceableOrNullBytes(i));
+  }
+  
   /// \brief Return true if the call should not be treated as a call to a
   /// builtin.
   bool isNoBuiltin() const {
@@ -254,6 +290,15 @@ class CallSiteBase {
     CALLSITE_DELEGATE_SETTER(setOnlyReadsMemory());
   }
 
+  /// @brief Determine if the call can access memmory only using pointers based
+  /// on its arguments.
+  bool onlyAccessesArgMemory() const {
+    CALLSITE_DELEGATE_GETTER(onlyAccessesArgMemory());
+  }
+  void setOnlyAccessesArgMemory() {
+    CALLSITE_DELEGATE_SETTER(setOnlyAccessesArgMemory());
+  }
+
   /// @brief Determine if the call cannot return.
   bool doesNotReturn() const {
     CALLSITE_DELEGATE_GETTER(doesNotReturn());
@@ -348,8 +393,9 @@ class CallSiteBase {
   }
 };
 
-class CallSite : public CallSiteBase<Function, Value, User, Instruction,
-                                     CallInst, InvokeInst, User::op_iterator> {
+class CallSite : public CallSiteBase<Function, BasicBlock, Value, User,
+                                     Instruction, CallInst, InvokeInst,
+                                     User::op_iterator> {
 public:
   CallSite() {}
   CallSite(CallSiteBase B) : CallSiteBase(B) {}
diff --git a/include/llvm/IR/CallingConv.h b/include/llvm/IR/CallingConv.h
index 9872e6ec794..f7a45afc9ca 100644
--- a/include/llvm/IR/CallingConv.h
+++ b/include/llvm/IR/CallingConv.h
@@ -69,6 +69,9 @@ namespace CallingConv {
     // (almost) all registers.
     PreserveAll = 15,
 
+    // Swift - Calling convention for Swift.
+    Swift = 16,
+
     // Target - This is the start of the target-specific calling conventions,
     // e.g. fastcall and thiscall on X86.
     FirstTargetCC = 64,
diff --git a/include/llvm/IR/Constant.h b/include/llvm/IR/Constant.h
index 75499e0a4db..019b4343a13 100644
--- a/include/llvm/IR/Constant.h
+++ b/include/llvm/IR/Constant.h
@@ -47,9 +47,6 @@ class Constant : public User {
   Constant(Type *ty, ValueTy vty, Use *Ops, unsigned NumOps)
     : User(ty, vty, Ops, NumOps) {}
 
-  void destroyConstantImpl();
-  void replaceUsesOfWithOnConstantImpl(Constant *Replacement);
-
 public:
   /// isNullValue - Return true if this is the value that would be returned by
   /// getNullValue.
@@ -126,14 +123,14 @@ class Constant : public User {
   /// vector of constant integers, all equal, and the common value is returned.
   const APInt &getUniqueInteger() const;
 
-  /// destroyConstant - Called if some element of this constant is no longer
-  /// valid.  At this point only other constants may be on the use_list for this
+  /// Called if some element of this constant is no longer valid.
+  /// At this point only other constants may be on the use_list for this
   /// constant.  Any constants on our Use list must also be destroy'd.  The
   /// implementation must be sure to remove the constant from the list of
-  /// available cached constants.  Implementations should call
-  /// destroyConstantImpl as the last thing they do, to destroy all users and
-  /// delete this.
-  virtual void destroyConstant() { llvm_unreachable("Not reached!"); }
+  /// available cached constants.  Implementations should implement
+  /// destroyConstantImpl to remove constants from any pools/maps they are
+  /// contained it.
+  void destroyConstant();
 
   //// Methods for support type inquiry through isa, cast, and dyn_cast:
   static inline bool classof(const Value *V) {
@@ -141,8 +138,8 @@ class Constant : public User {
            V->getValueID() <= ConstantLastVal;
   }
 
-  /// replaceUsesOfWithOnConstant - This method is a special form of
-  /// User::replaceUsesOfWith (which does not work on constants) that does work
+  /// This method is a special form of User::replaceUsesOfWith
+  /// (which does not work on constants) that does work
   /// on constants.  Basically this method goes through the trouble of building
   /// a new constant that is equivalent to the current one, with all uses of
   /// From replaced with uses of To.  After this construction is completed, all
@@ -151,15 +148,7 @@ class Constant : public User {
   /// use Value::replaceAllUsesWith, which automatically dispatches to this
   /// method as needed.
   ///
-  virtual void replaceUsesOfWithOnConstant(Value *, Value *, Use *) {
-    // Provide a default implementation for constants (like integers) that
-    // cannot use any other values.  This cannot be called at runtime, but needs
-    // to be here to avoid link errors.
-    assert(getNumOperands() == 0 && "replaceUsesOfWithOnConstant must be "
-           "implemented for all constants that have operands!");
-    llvm_unreachable("Constants that do not have operands cannot be using "
-                     "'From'!");
-  }
+  void handleOperandChange(Value *, Value *, Use *);
 
   static Constant *getNullValue(Type* Ty);
 
diff --git a/include/llvm/IR/Constants.h b/include/llvm/IR/Constants.h
index 70437e66cbf..0c7a84fc8bf 100644
--- a/include/llvm/IR/Constants.h
+++ b/include/llvm/IR/Constants.h
@@ -12,7 +12,7 @@
 /// which represent the different flavors of constant values that live in LLVM.
 /// Note that Constants are immutable (once created they never change) and are
 /// fully shared by structural equivalence.  This means that two structurally
-/// equivalent constants will always have the same address.  Constant's are
+/// equivalent constants will always have the same address.  Constants are
 /// created on demand as needed and never deleted: thus clients don't have to
 /// worry about the lifetime of the objects.
 //
@@ -50,6 +50,11 @@ class ConstantInt : public Constant {
   ConstantInt(const ConstantInt &) = delete;
   ConstantInt(IntegerType *Ty, const APInt& V);
   APInt Val;
+
+  friend class Constant;
+  void destroyConstantImpl();
+  Value *handleOperandChangeImpl(Value *From, Value *To, Use *U);
+
 protected:
   // allocate space for exactly zero operands
   void *operator new(size_t s) {
@@ -231,6 +236,11 @@ class ConstantFP : public Constant {
   void *operator new(size_t, unsigned) = delete;
   ConstantFP(const ConstantFP &) = delete;
   friend class LLVMContextImpl;
+
+  friend class Constant;
+  void destroyConstantImpl();
+  Value *handleOperandChangeImpl(Value *From, Value *To, Use *U);
+
 protected:
   ConstantFP(Type *Ty, const APFloat& V);
 protected:
@@ -251,6 +261,7 @@ class ConstantFP : public Constant {
   static Constant *get(Type* Ty, double V);
   static Constant *get(Type* Ty, StringRef Str);
   static ConstantFP *get(LLVMContext &Context, const APFloat &V);
+  static Constant *getNaN(Type *Ty, bool Negative = false, unsigned type = 0);
   static Constant *getNegativeZero(Type *Ty);
   static Constant *getInfinity(Type *Ty, bool Negative = false);
 
@@ -296,6 +307,11 @@ class ConstantFP : public Constant {
 class ConstantAggregateZero : public Constant {
   void *operator new(size_t, unsigned) = delete;
   ConstantAggregateZero(const ConstantAggregateZero &) = delete;
+
+  friend class Constant;
+  void destroyConstantImpl();
+  Value *handleOperandChangeImpl(Value *From, Value *To, Use *U);
+
 protected:
   explicit ConstantAggregateZero(Type *ty)
     : Constant(ty, ConstantAggregateZeroVal, nullptr, 0) {}
@@ -307,8 +323,6 @@ class ConstantAggregateZero : public Constant {
 public:
   static ConstantAggregateZero *get(Type *Ty);
 
-  void destroyConstant() override;
-
   /// getSequentialElement - If this CAZ has array or vector type, return a zero
   /// with the right element type.
   Constant *getSequentialElement() const;
@@ -342,6 +356,11 @@ class ConstantAggregateZero : public Constant {
 class ConstantArray : public Constant {
   friend struct ConstantAggrKeyType<ConstantArray>;
   ConstantArray(const ConstantArray &) = delete;
+
+  friend class Constant;
+  void destroyConstantImpl();
+  Value *handleOperandChangeImpl(Value *From, Value *To, Use *U);
+
 protected:
   ConstantArray(ArrayType *T, ArrayRef<Constant *> Val);
 public:
@@ -362,9 +381,6 @@ class ConstantArray : public Constant {
     return cast<ArrayType>(Value::getType());
   }
 
-  void destroyConstant() override;
-  void replaceUsesOfWithOnConstant(Value *From, Value *To, Use *U) override;
-
   /// Methods for support type inquiry through isa, cast, and dyn_cast:
   static bool classof(const Value *V) {
     return V->getValueID() == ConstantArrayVal;
@@ -384,6 +400,11 @@ DEFINE_TRANSPARENT_OPERAND_ACCESSORS(ConstantArray, Constant)
 class ConstantStruct : public Constant {
   friend struct ConstantAggrKeyType<ConstantStruct>;
   ConstantStruct(const ConstantStruct &) = delete;
+
+  friend class Constant;
+  void destroyConstantImpl();
+  Value *handleOperandChangeImpl(Value *From, Value *To, Use *U);
+
 protected:
   ConstantStruct(StructType *T, ArrayRef<Constant *> Val);
 public:
@@ -420,9 +441,6 @@ class ConstantStruct : public Constant {
     return cast<StructType>(Value::getType());
   }
 
-  void destroyConstant() override;
-  void replaceUsesOfWithOnConstant(Value *From, Value *To, Use *U) override;
-
   /// Methods for support type inquiry through isa, cast, and dyn_cast:
   static bool classof(const Value *V) {
     return V->getValueID() == ConstantStructVal;
@@ -443,6 +461,11 @@ DEFINE_TRANSPARENT_OPERAND_ACCESSORS(ConstantStruct, Constant)
 class ConstantVector : public Constant {
   friend struct ConstantAggrKeyType<ConstantVector>;
   ConstantVector(const ConstantVector &) = delete;
+
+  friend class Constant;
+  void destroyConstantImpl();
+  Value *handleOperandChangeImpl(Value *From, Value *To, Use *U);
+
 protected:
   ConstantVector(VectorType *T, ArrayRef<Constant *> Val);
 public:
@@ -471,9 +494,6 @@ class ConstantVector : public Constant {
   /// elements have the same value, return that value. Otherwise return NULL.
   Constant *getSplatValue() const;
 
-  void destroyConstant() override;
-  void replaceUsesOfWithOnConstant(Value *From, Value *To, Use *U) override;
-
   /// Methods for support type inquiry through isa, cast, and dyn_cast:
   static bool classof(const Value *V) {
     return V->getValueID() == ConstantVectorVal;
@@ -493,6 +513,11 @@ DEFINE_TRANSPARENT_OPERAND_ACCESSORS(ConstantVector, Constant)
 class ConstantPointerNull : public Constant {
   void *operator new(size_t, unsigned) = delete;
   ConstantPointerNull(const ConstantPointerNull &) = delete;
+
+  friend class Constant;
+  void destroyConstantImpl();
+  Value *handleOperandChangeImpl(Value *From, Value *To, Use *U);
+
 protected:
   explicit ConstantPointerNull(PointerType *T)
     : Constant(T,
@@ -507,8 +532,6 @@ class ConstantPointerNull : public Constant {
   /// get() - Static factory methods - Return objects of the specified value
   static ConstantPointerNull *get(PointerType *T);
 
-  void destroyConstant() override;
-
   /// getType - Specialize the getType() method to always return an PointerType,
   /// which reduces the amount of casting needed in parts of the compiler.
   ///
@@ -544,6 +567,11 @@ class ConstantDataSequential : public Constant {
   ConstantDataSequential *Next;
   void *operator new(size_t, unsigned) = delete;
   ConstantDataSequential(const ConstantDataSequential &) = delete;
+
+  friend class Constant;
+  void destroyConstantImpl();
+  Value *handleOperandChangeImpl(Value *From, Value *To, Use *U);
+
 protected:
   explicit ConstantDataSequential(Type *ty, ValueTy VT, const char *Data)
     : Constant(ty, VT, nullptr, 0), DataElements(Data), Next(nullptr) {}
@@ -634,8 +662,6 @@ class ConstantDataSequential : public Constant {
   /// host endianness of the data elements.
   StringRef getRawDataValues() const;
 
-  void destroyConstant() override;
-
   /// Methods for support type inquiry through isa, cast, and dyn_cast:
   ///
   static bool classof(const Value *V) {
@@ -777,6 +803,11 @@ class BlockAddress : public Constant {
   void *operator new(size_t, unsigned) = delete;
   void *operator new(size_t s) { return User::operator new(s, 2); }
   BlockAddress(Function *F, BasicBlock *BB);
+
+  friend class Constant;
+  void destroyConstantImpl();
+  Value *handleOperandChangeImpl(Value *From, Value *To, Use *U);
+
 public:
   /// get - Return a BlockAddress for the specified function and basic block.
   static BlockAddress *get(Function *F, BasicBlock *BB);
@@ -797,9 +828,6 @@ class BlockAddress : public Constant {
   Function *getFunction() const { return (Function*)Op<0>().get(); }
   BasicBlock *getBasicBlock() const { return (BasicBlock*)Op<1>().get(); }
 
-  void destroyConstant() override;
-  void replaceUsesOfWithOnConstant(Value *From, Value *To, Use *U) override;
-
   /// Methods for support type inquiry through isa, cast, and dyn_cast:
   static inline bool classof(const Value *V) {
     return V->getValueID() == BlockAddressVal;
@@ -824,6 +852,10 @@ DEFINE_TRANSPARENT_OPERAND_ACCESSORS(BlockAddress, Value)
 class ConstantExpr : public Constant {
   friend struct ConstantExprKeyType;
 
+  friend class Constant;
+  void destroyConstantImpl();
+  Value *handleOperandChangeImpl(Value *From, Value *To, Use *U);
+
 protected:
   ConstantExpr(Type *ty, unsigned Opcode, Use *Ops, unsigned NumOps)
     : Constant(ty, ConstantExprVal, Ops, NumOps) {
@@ -1054,7 +1086,7 @@ class ConstantExpr : public Constant {
                            bool OnlyIfReduced = false);
 
   /// Getelementptr form.  Value* is only accepted for convenience;
-  /// all elements must be Constant's.
+  /// all elements must be Constants.
   ///
   /// \param OnlyIfReducedTy see \a getWithOperands() docs.
   static Constant *getGetElementPtr(Type *Ty, Constant *C,
@@ -1145,8 +1177,9 @@ class ConstantExpr : public Constant {
   Constant *getWithOperands(ArrayRef<Constant *> Ops, Type *Ty,
                             bool OnlyIfReduced = false) const;
 
-  /// getAsInstruction - Returns an Instruction which implements the same operation
-  /// as this ConstantExpr. The instruction is not linked to any basic block.
+  /// getAsInstruction - Returns an Instruction which implements the same
+  /// operation as this ConstantExpr. The instruction is not linked to any basic
+  /// block.
   ///
   /// A better approach to this could be to have a constructor for Instruction
   /// which would take a ConstantExpr parameter, but that would have spread
@@ -1154,9 +1187,6 @@ class ConstantExpr : public Constant {
   /// would make it harder to remove ConstantExprs altogether.
   Instruction *getAsInstruction();
 
-  void destroyConstant() override;
-  void replaceUsesOfWithOnConstant(Value *From, Value *To, Use *U) override;
-
   /// Methods for support type inquiry through isa, cast, and dyn_cast:
   static inline bool classof(const Value *V) {
     return V->getValueID() == ConstantExprVal;
@@ -1190,6 +1220,11 @@ DEFINE_TRANSPARENT_OPERAND_ACCESSORS(ConstantExpr, Constant)
 class UndefValue : public Constant {
   void *operator new(size_t, unsigned) = delete;
   UndefValue(const UndefValue &) = delete;
+
+  friend class Constant;
+  void destroyConstantImpl();
+  Value *handleOperandChangeImpl(Value *From, Value *To, Use *U);
+
 protected:
   explicit UndefValue(Type *T) : Constant(T, UndefValueVal, nullptr, 0) {}
 protected:
@@ -1222,8 +1257,6 @@ class UndefValue : public Constant {
   /// \brief Return the number of elements in the array, vector, or struct.
   unsigned getNumElements() const;
 
-  void destroyConstant() override;
-
   /// Methods for support type inquiry through isa, cast, and dyn_cast:
   static bool classof(const Value *V) {
     return V->getValueID() == UndefValueVal;
diff --git a/include/llvm/IR/DIBuilder.h b/include/llvm/IR/DIBuilder.h
index db876549ca8..8aca266ce77 100644
--- a/include/llvm/IR/DIBuilder.h
+++ b/include/llvm/IR/DIBuilder.h
@@ -36,14 +36,9 @@ namespace llvm {
     Module &M;
     LLVMContext &VMContext;
 
-    TempMDTuple TempEnumTypes;
-    TempMDTuple TempRetainTypes;
-    TempMDTuple TempSubprograms;
-    TempMDTuple TempGVs;
-    TempMDTuple TempImportedModules;
-
-    Function *DeclareFn;     // llvm.dbg.declare
-    Function *ValueFn;       // llvm.dbg.value
+    DICompileUnit *CUNode;   ///< The one compile unit created by this DIBuiler.
+    Function *DeclareFn;     ///< llvm.dbg.declare
+    Function *ValueFn;       ///< llvm.dbg.value
 
     SmallVector<Metadata *, 4> AllEnumTypes;
     /// Track the RetainTypes, since they can be updated later on.
@@ -52,7 +47,7 @@ namespace llvm {
     SmallVector<Metadata *, 4> AllGVs;
     SmallVector<TrackingMDNodeRef, 4> AllImportedModules;
 
-    /// \brief Track nodes that may be unresolved.
+    /// Track nodes that may be unresolved.
     SmallVector<TrackingMDNodeRef, 4> UnresolvedNodes;
     bool AllowUnresolvedNodes;
 
@@ -62,612 +57,654 @@ namespace llvm {
     DIBuilder(const DIBuilder &) = delete;
     void operator=(const DIBuilder &) = delete;
 
-    /// \brief Create a temporary.
+    /// Create a temporary.
     ///
     /// Create an \a temporary node and track it in \a UnresolvedNodes.
     void trackIfUnresolved(MDNode *N);
 
   public:
-    /// \brief Construct a builder for a module.
+    /// Construct a builder for a module.
     ///
     /// If \c AllowUnresolved, collect unresolved nodes attached to the module
     /// in order to resolve cycles during \a finalize().
     explicit DIBuilder(Module &M, bool AllowUnresolved = true);
     enum DebugEmissionKind { FullDebug=1, LineTablesOnly };
 
-    /// finalize - Construct any deferred debug info descriptors.
+    /// Construct any deferred debug info descriptors.
     void finalize();
 
-    /// createCompileUnit - A CompileUnit provides an anchor for all debugging
+    /// A CompileUnit provides an anchor for all debugging
     /// information generated during this instance of compilation.
-    /// @param Lang     Source programming language, eg. dwarf::DW_LANG_C99
-    /// @param File     File name
-    /// @param Dir      Directory
-    /// @param Producer Identify the producer of debugging information and code.
-    ///                 Usually this is a compiler version string.
-    /// @param isOptimized A boolean flag which indicates whether optimization
-    ///                    is ON or not.
-    /// @param Flags    This string lists command line options. This string is
-    ///                 directly embedded in debug info output which may be used
-    ///                 by a tool analyzing generated debugging information.
-    /// @param RV       This indicates runtime version for languages like
-    ///                 Objective-C.
-    /// @param SplitName The name of the file that we'll split debug info out
-    ///                  into.
-    /// @param Kind     The kind of debug information to generate.
-    /// @param EmitDebugInfo   A boolean flag which indicates whether debug
-    ///                        information should be written to the final
-    ///                        output or not. When this is false, debug
-    ///                        information annotations will be present in
-    ///                        the IL but they are not written to the final
-    ///                        assembly or object file. This supports tracking
-    ///                        source location information in the back end
-    ///                        without actually changing the output (e.g.,
-    ///                        when using optimization remarks).
-    MDCompileUnit *createCompileUnit(unsigned Lang, StringRef File,
-                                     StringRef Dir, StringRef Producer,
-                                     bool isOptimized, StringRef Flags,
-                                     unsigned RV, StringRef SplitName = "",
-                                     DebugEmissionKind Kind = FullDebug,
-                                     bool EmitDebugInfo = true);
-
-    /// createFile - Create a file descriptor to hold debugging information
+    /// \param Lang          Source programming language, eg. dwarf::DW_LANG_C99
+    /// \param File          File name
+    /// \param Dir           Directory
+    /// \param Producer      Identify the producer of debugging information
+    ///                      and code.  Usually this is a compiler
+    ///                      version string.
+    /// \param isOptimized   A boolean flag which indicates whether optimization
+    ///                      is enabled or not.
+    /// \param Flags         This string lists command line options. This
+    ///                      string is directly embedded in debug info
+    ///                      output which may be used by a tool
+    ///                      analyzing generated debugging information.
+    /// \param RV            This indicates runtime version for languages like
+    ///                      Objective-C.
+    /// \param SplitName     The name of the file that we'll split debug info
+    ///                      out into.
+    /// \param Kind          The kind of debug information to generate.
+    /// \param DWOId         The DWOId if this is a split skeleton compile unit.
+    /// \param EmitDebugInfo A boolean flag which indicates whether
+    ///                      debug information should be written to
+    ///                      the final output or not. When this is
+    ///                      false, debug information annotations will
+    ///                      be present in the IL but they are not
+    ///                      written to the final assembly or object
+    ///                      file. This supports tracking source
+    ///                      location information in the back end
+    ///                      without actually changing the output
+    ///                      (e.g., when using optimization remarks).
+    DICompileUnit *
+    createCompileUnit(unsigned Lang, StringRef File, StringRef Dir,
+                      StringRef Producer, bool isOptimized, StringRef Flags,
+                      unsigned RV, StringRef SplitName = StringRef(),
+                      DebugEmissionKind Kind = FullDebug, uint64_t DWOId = 0,
+                      bool EmitDebugInfo = true);
+
+    /// Create a file descriptor to hold debugging information
     /// for a file.
-    MDFile *createFile(StringRef Filename, StringRef Directory);
+    DIFile *createFile(StringRef Filename, StringRef Directory);
 
-    /// createEnumerator - Create a single enumerator value.
-    MDEnumerator *createEnumerator(StringRef Name, int64_t Val);
+    /// Create a single enumerator value.
+    DIEnumerator *createEnumerator(StringRef Name, int64_t Val);
 
-    /// \brief Create a DWARF unspecified type.
-    MDBasicType *createUnspecifiedType(StringRef Name);
+    /// Create a DWARF unspecified type.
+    DIBasicType *createUnspecifiedType(StringRef Name);
 
-    /// \brief Create C++11 nullptr type.
-    MDBasicType *createNullPtrType();
+    /// Create C++11 nullptr type.
+    DIBasicType *createNullPtrType();
 
-    /// createBasicType - Create debugging information entry for a basic
+    /// Create debugging information entry for a basic
     /// type.
-    /// @param Name        Type name.
-    /// @param SizeInBits  Size of the type.
-    /// @param AlignInBits Type alignment.
-    /// @param Encoding    DWARF encoding code, e.g. dwarf::DW_ATE_float.
-    MDBasicType *createBasicType(StringRef Name, uint64_t SizeInBits,
+    /// \param Name        Type name.
+    /// \param SizeInBits  Size of the type.
+    /// \param AlignInBits Type alignment.
+    /// \param Encoding    DWARF encoding code, e.g. dwarf::DW_ATE_float.
+    DIBasicType *createBasicType(StringRef Name, uint64_t SizeInBits,
                                  uint64_t AlignInBits, unsigned Encoding);
 
-    /// createQualifiedType - Create debugging information entry for a qualified
+    /// Create debugging information entry for a qualified
     /// type, e.g. 'const int'.
-    /// @param Tag         Tag identifing type, e.g. dwarf::TAG_volatile_type
-    /// @param FromTy      Base Type.
-    MDDerivedType *createQualifiedType(unsigned Tag, MDType *FromTy);
-
-    /// createPointerType - Create debugging information entry for a pointer.
-    /// @param PointeeTy   Type pointed by this pointer.
-    /// @param SizeInBits  Size.
-    /// @param AlignInBits Alignment. (optional)
-    /// @param Name        Pointer type name. (optional)
-    MDDerivedType *createPointerType(MDType *PointeeTy, uint64_t SizeInBits,
+    /// \param Tag         Tag identifing type, e.g. dwarf::TAG_volatile_type
+    /// \param FromTy      Base Type.
+    DIDerivedType *createQualifiedType(unsigned Tag, DIType *FromTy);
+
+    /// Create debugging information entry for a pointer.
+    /// \param PointeeTy   Type pointed by this pointer.
+    /// \param SizeInBits  Size.
+    /// \param AlignInBits Alignment. (optional)
+    /// \param Name        Pointer type name. (optional)
+    DIDerivedType *createPointerType(DIType *PointeeTy, uint64_t SizeInBits,
                                      uint64_t AlignInBits = 0,
                                      StringRef Name = "");
 
-    /// \brief Create debugging information entry for a pointer to member.
-    /// @param PointeeTy Type pointed to by this pointer.
-    /// @param SizeInBits  Size.
-    /// @param AlignInBits Alignment. (optional)
-    /// @param Class Type for which this pointer points to members of.
-    MDDerivedType *createMemberPointerType(MDType *PointeeTy, MDType *Class,
+    /// Create debugging information entry for a pointer to member.
+    /// \param PointeeTy Type pointed to by this pointer.
+    /// \param SizeInBits  Size.
+    /// \param AlignInBits Alignment. (optional)
+    /// \param Class Type for which this pointer points to members of.
+    DIDerivedType *createMemberPointerType(DIType *PointeeTy, DIType *Class,
                                            uint64_t SizeInBits,
                                            uint64_t AlignInBits = 0);
 
-    /// createReferenceType - Create debugging information entry for a c++
+    /// Create debugging information entry for a c++
     /// style reference or rvalue reference type.
-    MDDerivedType *createReferenceType(unsigned Tag, MDType *RTy);
+    DIDerivedType *createReferenceType(unsigned Tag, DIType *RTy);
 
-    /// createTypedef - Create debugging information entry for a typedef.
-    /// @param Ty          Original type.
-    /// @param Name        Typedef name.
-    /// @param File        File where this type is defined.
-    /// @param LineNo      Line number.
-    /// @param Context     The surrounding context for the typedef.
-    MDDerivedType *createTypedef(MDType *Ty, StringRef Name, MDFile *File,
-                                 unsigned LineNo, MDScope *Context);
+    /// Create debugging information entry for a typedef.
+    /// \param Ty          Original type.
+    /// \param Name        Typedef name.
+    /// \param File        File where this type is defined.
+    /// \param LineNo      Line number.
+    /// \param Context     The surrounding context for the typedef.
+    DIDerivedType *createTypedef(DIType *Ty, StringRef Name, DIFile *File,
+                                 unsigned LineNo, DIScope *Context);
 
-    /// createFriend - Create debugging information entry for a 'friend'.
-    MDDerivedType *createFriend(MDType *Ty, MDType *FriendTy);
+    /// Create debugging information entry for a 'friend'.
+    DIDerivedType *createFriend(DIType *Ty, DIType *FriendTy);
 
-    /// createInheritance - Create debugging information entry to establish
+    /// Create debugging information entry to establish
     /// inheritance relationship between two types.
-    /// @param Ty           Original type.
-    /// @param BaseTy       Base type. Ty is inherits from base.
-    /// @param BaseOffset   Base offset.
-    /// @param Flags        Flags to describe inheritance attribute,
+    /// \param Ty           Original type.
+    /// \param BaseTy       Base type. Ty is inherits from base.
+    /// \param BaseOffset   Base offset.
+    /// \param Flags        Flags to describe inheritance attribute,
     ///                     e.g. private
-    MDDerivedType *createInheritance(MDType *Ty, MDType *BaseTy,
+    DIDerivedType *createInheritance(DIType *Ty, DIType *BaseTy,
                                      uint64_t BaseOffset, unsigned Flags);
 
-    /// createMemberType - Create debugging information entry for a member.
-    /// @param Scope        Member scope.
-    /// @param Name         Member name.
-    /// @param File         File where this member is defined.
-    /// @param LineNo       Line number.
-    /// @param SizeInBits   Member size.
-    /// @param AlignInBits  Member alignment.
-    /// @param OffsetInBits Member offset.
-    /// @param Flags        Flags to encode member attribute, e.g. private
-    /// @param Ty           Parent type.
-    MDDerivedType *createMemberType(MDScope *Scope, StringRef Name,
-                                    MDFile *File, unsigned LineNo,
+    /// Create debugging information entry for a member.
+    /// \param Scope        Member scope.
+    /// \param Name         Member name.
+    /// \param File         File where this member is defined.
+    /// \param LineNo       Line number.
+    /// \param SizeInBits   Member size.
+    /// \param AlignInBits  Member alignment.
+    /// \param OffsetInBits Member offset.
+    /// \param Flags        Flags to encode member attribute, e.g. private
+    /// \param Ty           Parent type.
+    DIDerivedType *createMemberType(DIScope *Scope, StringRef Name,
+                                    DIFile *File, unsigned LineNo,
                                     uint64_t SizeInBits, uint64_t AlignInBits,
                                     uint64_t OffsetInBits, unsigned Flags,
-                                    MDType *Ty);
+                                    DIType *Ty);
 
-    /// createStaticMemberType - Create debugging information entry for a
+    /// Create debugging information entry for a
     /// C++ static data member.
-    /// @param Scope      Member scope.
-    /// @param Name       Member name.
-    /// @param File       File where this member is declared.
-    /// @param LineNo     Line number.
-    /// @param Ty         Type of the static member.
-    /// @param Flags      Flags to encode member attribute, e.g. private.
-    /// @param Val        Const initializer of the member.
-    MDDerivedType *createStaticMemberType(MDScope *Scope, StringRef Name,
-                                          MDFile *File, unsigned LineNo,
-                                          MDType *Ty, unsigned Flags,
+    /// \param Scope      Member scope.
+    /// \param Name       Member name.
+    /// \param File       File where this member is declared.
+    /// \param LineNo     Line number.
+    /// \param Ty         Type of the static member.
+    /// \param Flags      Flags to encode member attribute, e.g. private.
+    /// \param Val        Const initializer of the member.
+    DIDerivedType *createStaticMemberType(DIScope *Scope, StringRef Name,
+                                          DIFile *File, unsigned LineNo,
+                                          DIType *Ty, unsigned Flags,
                                           llvm::Constant *Val);
 
-    /// createObjCIVar - Create debugging information entry for Objective-C
+    /// Create debugging information entry for Objective-C
     /// instance variable.
-    /// @param Name         Member name.
-    /// @param File         File where this member is defined.
-    /// @param LineNo       Line number.
-    /// @param SizeInBits   Member size.
-    /// @param AlignInBits  Member alignment.
-    /// @param OffsetInBits Member offset.
-    /// @param Flags        Flags to encode member attribute, e.g. private
-    /// @param Ty           Parent type.
-    /// @param PropertyNode Property associated with this ivar.
-    MDDerivedType *createObjCIVar(StringRef Name, MDFile *File, unsigned LineNo,
+    /// \param Name         Member name.
+    /// \param File         File where this member is defined.
+    /// \param LineNo       Line number.
+    /// \param SizeInBits   Member size.
+    /// \param AlignInBits  Member alignment.
+    /// \param OffsetInBits Member offset.
+    /// \param Flags        Flags to encode member attribute, e.g. private
+    /// \param Ty           Parent type.
+    /// \param PropertyNode Property associated with this ivar.
+    DIDerivedType *createObjCIVar(StringRef Name, DIFile *File, unsigned LineNo,
                                   uint64_t SizeInBits, uint64_t AlignInBits,
                                   uint64_t OffsetInBits, unsigned Flags,
-                                  MDType *Ty, MDNode *PropertyNode);
+                                  DIType *Ty, MDNode *PropertyNode);
 
-    /// createObjCProperty - Create debugging information entry for Objective-C
+    /// Create debugging information entry for Objective-C
     /// property.
-    /// @param Name         Property name.
-    /// @param File         File where this property is defined.
-    /// @param LineNumber   Line number.
-    /// @param GetterName   Name of the Objective C property getter selector.
-    /// @param SetterName   Name of the Objective C property setter selector.
-    /// @param PropertyAttributes Objective C property attributes.
-    /// @param Ty           Type.
-    MDObjCProperty *createObjCProperty(StringRef Name, MDFile *File,
+    /// \param Name         Property name.
+    /// \param File         File where this property is defined.
+    /// \param LineNumber   Line number.
+    /// \param GetterName   Name of the Objective C property getter selector.
+    /// \param SetterName   Name of the Objective C property setter selector.
+    /// \param PropertyAttributes Objective C property attributes.
+    /// \param Ty           Type.
+    DIObjCProperty *createObjCProperty(StringRef Name, DIFile *File,
                                        unsigned LineNumber,
                                        StringRef GetterName,
                                        StringRef SetterName,
-                                       unsigned PropertyAttributes, MDType *Ty);
-
-    /// createClassType - Create debugging information entry for a class.
-    /// @param Scope        Scope in which this class is defined.
-    /// @param Name         class name.
-    /// @param File         File where this member is defined.
-    /// @param LineNumber   Line number.
-    /// @param SizeInBits   Member size.
-    /// @param AlignInBits  Member alignment.
-    /// @param OffsetInBits Member offset.
-    /// @param Flags        Flags to encode member attribute, e.g. private
-    /// @param Elements     class members.
-    /// @param VTableHolder Debug info of the base class that contains vtable
+                                       unsigned PropertyAttributes, DIType *Ty);
+
+    /// Create debugging information entry for a class.
+    /// \param Scope        Scope in which this class is defined.
+    /// \param Name         class name.
+    /// \param File         File where this member is defined.
+    /// \param LineNumber   Line number.
+    /// \param SizeInBits   Member size.
+    /// \param AlignInBits  Member alignment.
+    /// \param OffsetInBits Member offset.
+    /// \param Flags        Flags to encode member attribute, e.g. private
+    /// \param Elements     class members.
+    /// \param VTableHolder Debug info of the base class that contains vtable
     ///                     for this type. This is used in
     ///                     DW_AT_containing_type. See DWARF documentation
     ///                     for more info.
-    /// @param TemplateParms Template type parameters.
-    /// @param UniqueIdentifier A unique identifier for the class.
-    MDCompositeType *createClassType(MDScope *Scope, StringRef Name,
-                                     MDFile *File, unsigned LineNumber,
+    /// \param TemplateParms Template type parameters.
+    /// \param UniqueIdentifier A unique identifier for the class.
+    DICompositeType *createClassType(DIScope *Scope, StringRef Name,
+                                     DIFile *File, unsigned LineNumber,
                                      uint64_t SizeInBits, uint64_t AlignInBits,
                                      uint64_t OffsetInBits, unsigned Flags,
-                                     MDType *DerivedFrom, DIArray Elements,
-                                     MDType *VTableHolder = nullptr,
+                                     DIType *DerivedFrom, DINodeArray Elements,
+                                     DIType *VTableHolder = nullptr,
                                      MDNode *TemplateParms = nullptr,
                                      StringRef UniqueIdentifier = "");
 
-    /// createStructType - Create debugging information entry for a struct.
-    /// @param Scope        Scope in which this struct is defined.
-    /// @param Name         Struct name.
-    /// @param File         File where this member is defined.
-    /// @param LineNumber   Line number.
-    /// @param SizeInBits   Member size.
-    /// @param AlignInBits  Member alignment.
-    /// @param Flags        Flags to encode member attribute, e.g. private
-    /// @param Elements     Struct elements.
-    /// @param RunTimeLang  Optional parameter, Objective-C runtime version.
-    /// @param UniqueIdentifier A unique identifier for the struct.
-    MDCompositeType *createStructType(
-        MDScope *Scope, StringRef Name, MDFile *File, unsigned LineNumber,
+    /// Create debugging information entry for a struct.
+    /// \param Scope        Scope in which this struct is defined.
+    /// \param Name         Struct name.
+    /// \param File         File where this member is defined.
+    /// \param LineNumber   Line number.
+    /// \param SizeInBits   Member size.
+    /// \param AlignInBits  Member alignment.
+    /// \param Flags        Flags to encode member attribute, e.g. private
+    /// \param Elements     Struct elements.
+    /// \param RunTimeLang  Optional parameter, Objective-C runtime version.
+    /// \param UniqueIdentifier A unique identifier for the struct.
+    DICompositeType *createStructType(
+        DIScope *Scope, StringRef Name, DIFile *File, unsigned LineNumber,
         uint64_t SizeInBits, uint64_t AlignInBits, unsigned Flags,
-        MDType *DerivedFrom, DIArray Elements, unsigned RunTimeLang = 0,
-        MDType *VTableHolder = nullptr, StringRef UniqueIdentifier = "");
-
-    /// createUnionType - Create debugging information entry for an union.
-    /// @param Scope        Scope in which this union is defined.
-    /// @param Name         Union name.
-    /// @param File         File where this member is defined.
-    /// @param LineNumber   Line number.
-    /// @param SizeInBits   Member size.
-    /// @param AlignInBits  Member alignment.
-    /// @param Flags        Flags to encode member attribute, e.g. private
-    /// @param Elements     Union elements.
-    /// @param RunTimeLang  Optional parameter, Objective-C runtime version.
-    /// @param UniqueIdentifier A unique identifier for the union.
-    MDCompositeType *createUnionType(MDScope *Scope, StringRef Name,
-                                     MDFile *File, unsigned LineNumber,
+        DIType *DerivedFrom, DINodeArray Elements, unsigned RunTimeLang = 0,
+        DIType *VTableHolder = nullptr, StringRef UniqueIdentifier = "");
+
+    /// Create debugging information entry for an union.
+    /// \param Scope        Scope in which this union is defined.
+    /// \param Name         Union name.
+    /// \param File         File where this member is defined.
+    /// \param LineNumber   Line number.
+    /// \param SizeInBits   Member size.
+    /// \param AlignInBits  Member alignment.
+    /// \param Flags        Flags to encode member attribute, e.g. private
+    /// \param Elements     Union elements.
+    /// \param RunTimeLang  Optional parameter, Objective-C runtime version.
+    /// \param UniqueIdentifier A unique identifier for the union.
+    DICompositeType *createUnionType(DIScope *Scope, StringRef Name,
+                                     DIFile *File, unsigned LineNumber,
                                      uint64_t SizeInBits, uint64_t AlignInBits,
-                                     unsigned Flags, DIArray Elements,
+                                     unsigned Flags, DINodeArray Elements,
                                      unsigned RunTimeLang = 0,
                                      StringRef UniqueIdentifier = "");
 
-    /// createTemplateTypeParameter - Create debugging information for template
+    /// Create debugging information for template
     /// type parameter.
-    /// @param Scope        Scope in which this type is defined.
-    /// @param Name         Type parameter name.
-    /// @param Ty           Parameter type.
-    MDTemplateTypeParameter *
-    createTemplateTypeParameter(MDScope *Scope, StringRef Name, MDType *Ty);
+    /// \param Scope        Scope in which this type is defined.
+    /// \param Name         Type parameter name.
+    /// \param Ty           Parameter type.
+    DITemplateTypeParameter *
+    createTemplateTypeParameter(DIScope *Scope, StringRef Name, DIType *Ty);
 
-    /// createTemplateValueParameter - Create debugging information for template
+    /// Create debugging information for template
     /// value parameter.
-    /// @param Scope        Scope in which this type is defined.
-    /// @param Name         Value parameter name.
-    /// @param Ty           Parameter type.
-    /// @param Val          Constant parameter value.
-    MDTemplateValueParameter *createTemplateValueParameter(MDScope *Scope,
+    /// \param Scope        Scope in which this type is defined.
+    /// \param Name         Value parameter name.
+    /// \param Ty           Parameter type.
+    /// \param Val          Constant parameter value.
+    DITemplateValueParameter *createTemplateValueParameter(DIScope *Scope,
                                                            StringRef Name,
-                                                           MDType *Ty,
+                                                           DIType *Ty,
                                                            Constant *Val);
 
-    /// \brief Create debugging information for a template template parameter.
-    /// @param Scope        Scope in which this type is defined.
-    /// @param Name         Value parameter name.
-    /// @param Ty           Parameter type.
-    /// @param Val          The fully qualified name of the template.
-    MDTemplateValueParameter *createTemplateTemplateParameter(MDScope *Scope,
+    /// Create debugging information for a template template parameter.
+    /// \param Scope        Scope in which this type is defined.
+    /// \param Name         Value parameter name.
+    /// \param Ty           Parameter type.
+    /// \param Val          The fully qualified name of the template.
+    DITemplateValueParameter *createTemplateTemplateParameter(DIScope *Scope,
                                                               StringRef Name,
-                                                              MDType *Ty,
+                                                              DIType *Ty,
                                                               StringRef Val);
 
-    /// \brief Create debugging information for a template parameter pack.
-    /// @param Scope        Scope in which this type is defined.
-    /// @param Name         Value parameter name.
-    /// @param Ty           Parameter type.
-    /// @param Val          An array of types in the pack.
-    MDTemplateValueParameter *createTemplateParameterPack(MDScope *Scope,
+    /// Create debugging information for a template parameter pack.
+    /// \param Scope        Scope in which this type is defined.
+    /// \param Name         Value parameter name.
+    /// \param Ty           Parameter type.
+    /// \param Val          An array of types in the pack.
+    DITemplateValueParameter *createTemplateParameterPack(DIScope *Scope,
                                                           StringRef Name,
-                                                          MDType *Ty,
-                                                          DIArray Val);
-
-    /// createArrayType - Create debugging information entry for an array.
-    /// @param Size         Array size.
-    /// @param AlignInBits  Alignment.
-    /// @param Ty           Element type.
-    /// @param Subscripts   Subscripts.
-    MDCompositeType *createArrayType(uint64_t Size, uint64_t AlignInBits,
-                                     MDType *Ty, DIArray Subscripts);
-
-    /// createVectorType - Create debugging information entry for a vector type.
-    /// @param Size         Array size.
-    /// @param AlignInBits  Alignment.
-    /// @param Ty           Element type.
-    /// @param Subscripts   Subscripts.
-    MDCompositeType *createVectorType(uint64_t Size, uint64_t AlignInBits,
-                                      MDType *Ty, DIArray Subscripts);
-
-    /// createEnumerationType - Create debugging information entry for an
+                                                          DIType *Ty,
+                                                          DINodeArray Val);
+
+    /// Create debugging information entry for an array.
+    /// \param Size         Array size.
+    /// \param AlignInBits  Alignment.
+    /// \param Ty           Element type.
+    /// \param Subscripts   Subscripts.
+    DICompositeType *createArrayType(uint64_t Size, uint64_t AlignInBits,
+                                     DIType *Ty, DINodeArray Subscripts);
+
+    /// Create debugging information entry for a vector type.
+    /// \param Size         Array size.
+    /// \param AlignInBits  Alignment.
+    /// \param Ty           Element type.
+    /// \param Subscripts   Subscripts.
+    DICompositeType *createVectorType(uint64_t Size, uint64_t AlignInBits,
+                                      DIType *Ty, DINodeArray Subscripts);
+
+    /// Create debugging information entry for an
     /// enumeration.
-    /// @param Scope          Scope in which this enumeration is defined.
-    /// @param Name           Union name.
-    /// @param File           File where this member is defined.
-    /// @param LineNumber     Line number.
-    /// @param SizeInBits     Member size.
-    /// @param AlignInBits    Member alignment.
-    /// @param Elements       Enumeration elements.
-    /// @param UnderlyingType Underlying type of a C++11/ObjC fixed enum.
-    /// @param UniqueIdentifier A unique identifier for the enum.
-    MDCompositeType *createEnumerationType(
-        MDScope *Scope, StringRef Name, MDFile *File, unsigned LineNumber,
-        uint64_t SizeInBits, uint64_t AlignInBits, DIArray Elements,
-        MDType *UnderlyingType, StringRef UniqueIdentifier = "");
-
-    /// createSubroutineType - Create subroutine type.
-    /// @param File            File in which this subroutine is defined.
-    /// @param ParameterTypes  An array of subroutine parameter types. This
+    /// \param Scope          Scope in which this enumeration is defined.
+    /// \param Name           Union name.
+    /// \param File           File where this member is defined.
+    /// \param LineNumber     Line number.
+    /// \param SizeInBits     Member size.
+    /// \param AlignInBits    Member alignment.
+    /// \param Elements       Enumeration elements.
+    /// \param UnderlyingType Underlying type of a C++11/ObjC fixed enum.
+    /// \param UniqueIdentifier A unique identifier for the enum.
+    DICompositeType *createEnumerationType(
+        DIScope *Scope, StringRef Name, DIFile *File, unsigned LineNumber,
+        uint64_t SizeInBits, uint64_t AlignInBits, DINodeArray Elements,
+        DIType *UnderlyingType, StringRef UniqueIdentifier = "");
+
+    /// Create subroutine type.
+    /// \param File            File in which this subroutine is defined.
+    /// \param ParameterTypes  An array of subroutine parameter types. This
     ///                        includes return type at 0th index.
-    /// @param Flags           E.g.: LValueReference.
+    /// \param Flags           E.g.: LValueReference.
     ///                        These flags are used to emit dwarf attributes.
-    MDSubroutineType *createSubroutineType(MDFile *File,
-                                           DITypeArray ParameterTypes,
+    DISubroutineType *createSubroutineType(DIFile *File,
+                                           DITypeRefArray ParameterTypes,
                                            unsigned Flags = 0);
 
-    /// createArtificialType - Create a new MDType* with "artificial" flag set.
-    MDType *createArtificialType(MDType *Ty);
+    /// Create an external type reference.
+    /// \param Tag              Dwarf TAG.
+    /// \param File             File in which the type is defined.
+    /// \param UniqueIdentifier A unique identifier for the type.
+    DICompositeType *createExternalTypeRef(unsigned Tag, DIFile *File,
+                                           StringRef UniqueIdentifier);
+
+    /// Create a new DIType* with "artificial" flag set.
+    DIType *createArtificialType(DIType *Ty);
 
-    /// createObjectPointerType - Create a new MDType* with the "object pointer"
+    /// Create a new DIType* with the "object pointer"
     /// flag set.
-    MDType *createObjectPointerType(MDType *Ty);
+    DIType *createObjectPointerType(DIType *Ty);
 
-    /// \brief Create a permanent forward-declared type.
-    MDCompositeType *createForwardDecl(unsigned Tag, StringRef Name,
-                                       MDScope *Scope, MDFile *F, unsigned Line,
+    /// Create a permanent forward-declared type.
+    DICompositeType *createForwardDecl(unsigned Tag, StringRef Name,
+                                       DIScope *Scope, DIFile *F, unsigned Line,
                                        unsigned RuntimeLang = 0,
                                        uint64_t SizeInBits = 0,
                                        uint64_t AlignInBits = 0,
                                        StringRef UniqueIdentifier = "");
 
-    /// \brief Create a temporary forward-declared type.
-    MDCompositeType *createReplaceableCompositeType(
-        unsigned Tag, StringRef Name, MDScope *Scope, MDFile *F, unsigned Line,
+    /// Create a temporary forward-declared type.
+    DICompositeType *createReplaceableCompositeType(
+        unsigned Tag, StringRef Name, DIScope *Scope, DIFile *F, unsigned Line,
         unsigned RuntimeLang = 0, uint64_t SizeInBits = 0,
-        uint64_t AlignInBits = 0, unsigned Flags = DebugNode::FlagFwdDecl,
+        uint64_t AlignInBits = 0, unsigned Flags = DINode::FlagFwdDecl,
         StringRef UniqueIdentifier = "");
 
-    /// retainType - Retain MDType* in a module even if it is not referenced
+    /// Retain DIType* in a module even if it is not referenced
     /// through debug info anchors.
-    void retainType(MDType *T);
+    void retainType(DIType *T);
 
-    /// createUnspecifiedParameter - Create unspecified parameter type
+    /// Create unspecified parameter type
     /// for a subroutine type.
-    MDBasicType *createUnspecifiedParameter();
+    DIBasicType *createUnspecifiedParameter();
 
-    /// getOrCreateArray - Get a DIArray, create one if required.
-    DIArray getOrCreateArray(ArrayRef<Metadata *> Elements);
+    /// Get a DINodeArray, create one if required.
+    DINodeArray getOrCreateArray(ArrayRef<Metadata *> Elements);
 
-    /// getOrCreateTypeArray - Get a DITypeArray, create one if required.
-    DITypeArray getOrCreateTypeArray(ArrayRef<Metadata *> Elements);
+    /// Get a DITypeRefArray, create one if required.
+    DITypeRefArray getOrCreateTypeArray(ArrayRef<Metadata *> Elements);
 
-    /// getOrCreateSubrange - Create a descriptor for a value range.  This
+    /// Create a descriptor for a value range.  This
     /// implicitly uniques the values returned.
-    MDSubrange *getOrCreateSubrange(int64_t Lo, int64_t Count);
+    DISubrange *getOrCreateSubrange(int64_t Lo, int64_t Count);
 
-    /// createGlobalVariable - Create a new descriptor for the specified
+    /// Create a new descriptor for the specified
     /// variable.
-    /// @param Context     Variable scope.
-    /// @param Name        Name of the variable.
-    /// @param LinkageName Mangled  name of the variable.
-    /// @param File        File where this variable is defined.
-    /// @param LineNo      Line number.
-    /// @param Ty          Variable Type.
-    /// @param isLocalToUnit Boolean flag indicate whether this variable is
+    /// \param Context     Variable scope.
+    /// \param Name        Name of the variable.
+    /// \param LinkageName Mangled  name of the variable.
+    /// \param File        File where this variable is defined.
+    /// \param LineNo      Line number.
+    /// \param Ty          Variable Type.
+    /// \param isLocalToUnit Boolean flag indicate whether this variable is
     ///                      externally visible or not.
-    /// @param Val         llvm::Value of the variable.
-    /// @param Decl        Reference to the corresponding declaration.
-    MDGlobalVariable *createGlobalVariable(MDScope *Context, StringRef Name,
-                                           StringRef LinkageName, MDFile *File,
-                                           unsigned LineNo, MDType *Ty,
+    /// \param Val         llvm::Value of the variable.
+    /// \param Decl        Reference to the corresponding declaration.
+    DIGlobalVariable *createGlobalVariable(DIScope *Context, StringRef Name,
+                                           StringRef LinkageName, DIFile *File,
+                                           unsigned LineNo, DIType *Ty,
                                            bool isLocalToUnit,
                                            llvm::Constant *Val,
                                            MDNode *Decl = nullptr);
 
-    /// createTempGlobalVariableFwdDecl - Identical to createGlobalVariable
+    /// Identical to createGlobalVariable
     /// except that the resulting DbgNode is temporary and meant to be RAUWed.
-    MDGlobalVariable *createTempGlobalVariableFwdDecl(
-        MDScope *Context, StringRef Name, StringRef LinkageName, MDFile *File,
-        unsigned LineNo, MDType *Ty, bool isLocalToUnit, llvm::Constant *Val,
+    DIGlobalVariable *createTempGlobalVariableFwdDecl(
+        DIScope *Context, StringRef Name, StringRef LinkageName, DIFile *File,
+        unsigned LineNo, DIType *Ty, bool isLocalToUnit, llvm::Constant *Val,
         MDNode *Decl = nullptr);
 
-    /// createLocalVariable - Create a new descriptor for the specified
-    /// local variable.
-    /// @param Tag         Dwarf TAG. Usually DW_TAG_auto_variable or
-    ///                    DW_TAG_arg_variable.
-    /// @param Scope       Variable scope.
-    /// @param Name        Variable name.
-    /// @param File        File where this variable is defined.
-    /// @param LineNo      Line number.
-    /// @param Ty          Variable Type
-    /// @param AlwaysPreserve Boolean. Set to true if debug info for this
-    ///                       variable should be preserved in optimized build.
-    /// @param Flags       Flags, e.g. artificial variable.
-    /// @param ArgNo       If this variable is an argument then this argument's
-    ///                    number. 1 indicates 1st argument.
-    MDLocalVariable *createLocalVariable(unsigned Tag, MDScope *Scope,
-                                         StringRef Name, MDFile *File,
-                                         unsigned LineNo, MDType *Ty,
+    /// Create a new descriptor for an auto variable.  This is a local variable
+    /// that is not a subprogram parameter.
+    ///
+    /// \c Scope must be a \a DILocalScope, and thus its scope chain eventually
+    /// leads to a \a DISubprogram.
+    ///
+    /// If \c AlwaysPreserve, this variable will be referenced from its
+    /// containing subprogram, and will survive some optimizations.
+    DILocalVariable *createAutoVariable(DIScope *Scope, StringRef Name,
+                                         DIFile *File, unsigned LineNo,
+                                         DIType *Ty,
                                          bool AlwaysPreserve = false,
-                                         unsigned Flags = 0,
-                                         unsigned ArgNo = 0);
+                                         unsigned Flags = 0);
 
-    /// createExpression - Create a new descriptor for the specified
+    /// Create a new descriptor for a parameter variable.
+    ///
+    /// \c Scope must be a \a DILocalScope, and thus its scope chain eventually
+    /// leads to a \a DISubprogram.
+    ///
+    /// \c ArgNo is the index (starting from \c 1) of this variable in the
+    /// subprogram parameters.  \c ArgNo should not conflict with other
+    /// parameters of the same subprogram.
+    ///
+    /// If \c AlwaysPreserve, this variable will be referenced from its
+    /// containing subprogram, and will survive some optimizations.
+    DILocalVariable *createParameterVariable(DIScope *Scope, StringRef Name,
+                                             unsigned ArgNo, DIFile *File,
+                                             unsigned LineNo, DIType *Ty,
+                                             bool AlwaysPreserve = false,
+                                             unsigned Flags = 0);
+
+    /// Create a new descriptor for the specified
     /// variable which has a complex address expression for its address.
-    /// @param Addr        An array of complex address operations.
-    MDExpression *createExpression(ArrayRef<uint64_t> Addr = None);
-    MDExpression *createExpression(ArrayRef<int64_t> Addr);
+    /// \param Addr        An array of complex address operations.
+    DIExpression *createExpression(ArrayRef<uint64_t> Addr = None);
+    DIExpression *createExpression(ArrayRef<int64_t> Addr);
 
-    /// createBitPieceExpression - Create a descriptor to describe one part
+    /// Create a descriptor to describe one part
     /// of aggregate variable that is fragmented across multiple Values.
     ///
-    /// @param OffsetInBits Offset of the piece in bits.
-    /// @param SizeInBits   Size of the piece in bits.
-    MDExpression *createBitPieceExpression(unsigned OffsetInBits,
+    /// \param OffsetInBits Offset of the piece in bits.
+    /// \param SizeInBits   Size of the piece in bits.
+    DIExpression *createBitPieceExpression(unsigned OffsetInBits,
                                            unsigned SizeInBits);
 
-    /// createFunction - Create a new descriptor for the specified subprogram.
-    /// See comments in MDSubprogram* for descriptions of these fields.
-    /// @param Scope         Function scope.
-    /// @param Name          Function name.
-    /// @param LinkageName   Mangled function name.
-    /// @param File          File where this variable is defined.
-    /// @param LineNo        Line number.
-    /// @param Ty            Function type.
-    /// @param isLocalToUnit True if this function is not externally visible.
-    /// @param isDefinition  True if this is a function definition.
-    /// @param ScopeLine     Set to the beginning of the scope this starts
-    /// @param Flags         e.g. is this function prototyped or not.
+    /// Create a new descriptor for the specified subprogram.
+    /// See comments in DISubprogram* for descriptions of these fields.
+    /// \param Scope         Function scope.
+    /// \param Name          Function name.
+    /// \param LinkageName   Mangled function name.
+    /// \param File          File where this variable is defined.
+    /// \param LineNo        Line number.
+    /// \param Ty            Function type.
+    /// \param isLocalToUnit True if this function is not externally visible.
+    /// \param isDefinition  True if this is a function definition.
+    /// \param ScopeLine     Set to the beginning of the scope this starts
+    /// \param Flags         e.g. is this function prototyped or not.
     ///                      These flags are used to emit dwarf attributes.
-    /// @param isOptimized   True if optimization is ON.
-    /// @param Fn            llvm::Function pointer.
-    /// @param TParam        Function template parameters.
-    MDSubprogram *
-    createFunction(MDScope *Scope, StringRef Name, StringRef LinkageName,
-                   MDFile *File, unsigned LineNo, MDSubroutineType *Ty,
+    /// \param isOptimized   True if optimization is ON.
+    /// \param Fn            llvm::Function pointer.
+    /// \param TParam        Function template parameters.
+    DISubprogram *
+    createFunction(DIScope *Scope, StringRef Name, StringRef LinkageName,
+                   DIFile *File, unsigned LineNo, DISubroutineType *Ty,
                    bool isLocalToUnit, bool isDefinition, unsigned ScopeLine,
                    unsigned Flags = 0, bool isOptimized = false,
                    Function *Fn = nullptr, MDNode *TParam = nullptr,
                    MDNode *Decl = nullptr);
 
-    /// createTempFunctionFwdDecl - Identical to createFunction,
+    /// Identical to createFunction,
     /// except that the resulting DbgNode is meant to be RAUWed.
-    MDSubprogram *createTempFunctionFwdDecl(
-        MDScope *Scope, StringRef Name, StringRef LinkageName, MDFile *File,
-        unsigned LineNo, MDSubroutineType *Ty, bool isLocalToUnit,
+    DISubprogram *createTempFunctionFwdDecl(
+        DIScope *Scope, StringRef Name, StringRef LinkageName, DIFile *File,
+        unsigned LineNo, DISubroutineType *Ty, bool isLocalToUnit,
         bool isDefinition, unsigned ScopeLine, unsigned Flags = 0,
         bool isOptimized = false, Function *Fn = nullptr,
         MDNode *TParam = nullptr, MDNode *Decl = nullptr);
 
     /// FIXME: this is added for dragonegg. Once we update dragonegg
     /// to call resolve function, this will be removed.
-    MDSubprogram *
+    DISubprogram *
     createFunction(DIScopeRef Scope, StringRef Name, StringRef LinkageName,
-                   MDFile *File, unsigned LineNo, MDSubroutineType *Ty,
+                   DIFile *File, unsigned LineNo, DISubroutineType *Ty,
                    bool isLocalToUnit, bool isDefinition, unsigned ScopeLine,
                    unsigned Flags = 0, bool isOptimized = false,
                    Function *Fn = nullptr, MDNode *TParam = nullptr,
                    MDNode *Decl = nullptr);
 
-    /// createMethod - Create a new descriptor for the specified C++ method.
-    /// See comments in MDSubprogram* for descriptions of these fields.
-    /// @param Scope         Function scope.
-    /// @param Name          Function name.
-    /// @param LinkageName   Mangled function name.
-    /// @param File          File where this variable is defined.
-    /// @param LineNo        Line number.
-    /// @param Ty            Function type.
-    /// @param isLocalToUnit True if this function is not externally visible..
-    /// @param isDefinition  True if this is a function definition.
-    /// @param Virtuality    Attributes describing virtualness. e.g. pure
+    /// Create a new descriptor for the specified C++ method.
+    /// See comments in \a DISubprogram* for descriptions of these fields.
+    /// \param Scope         Function scope.
+    /// \param Name          Function name.
+    /// \param LinkageName   Mangled function name.
+    /// \param File          File where this variable is defined.
+    /// \param LineNo        Line number.
+    /// \param Ty            Function type.
+    /// \param isLocalToUnit True if this function is not externally visible..
+    /// \param isDefinition  True if this is a function definition.
+    /// \param Virtuality    Attributes describing virtualness. e.g. pure
     ///                      virtual function.
-    /// @param VTableIndex   Index no of this method in virtual table.
-    /// @param VTableHolder  Type that holds vtable.
-    /// @param Flags         e.g. is this function prototyped or not.
+    /// \param VTableIndex   Index no of this method in virtual table.
+    /// \param VTableHolder  Type that holds vtable.
+    /// \param Flags         e.g. is this function prototyped or not.
     ///                      This flags are used to emit dwarf attributes.
-    /// @param isOptimized   True if optimization is ON.
-    /// @param Fn            llvm::Function pointer.
-    /// @param TParam        Function template parameters.
-    MDSubprogram *
-    createMethod(MDScope *Scope, StringRef Name, StringRef LinkageName,
-                 MDFile *File, unsigned LineNo, MDSubroutineType *Ty,
+    /// \param isOptimized   True if optimization is ON.
+    /// \param Fn            llvm::Function pointer.
+    /// \param TParam        Function template parameters.
+    DISubprogram *
+    createMethod(DIScope *Scope, StringRef Name, StringRef LinkageName,
+                 DIFile *File, unsigned LineNo, DISubroutineType *Ty,
                  bool isLocalToUnit, bool isDefinition, unsigned Virtuality = 0,
-                 unsigned VTableIndex = 0, MDType *VTableHolder = nullptr,
+                 unsigned VTableIndex = 0, DIType *VTableHolder = nullptr,
                  unsigned Flags = 0, bool isOptimized = false,
                  Function *Fn = nullptr, MDNode *TParam = nullptr);
 
-    /// createNameSpace - This creates new descriptor for a namespace
-    /// with the specified parent scope.
-    /// @param Scope       Namespace scope
-    /// @param Name        Name of this namespace
-    /// @param File        Source file
-    /// @param LineNo      Line number
-    MDNamespace *createNameSpace(MDScope *Scope, StringRef Name, MDFile *File,
+    /// This creates new descriptor for a namespace with the specified
+    /// parent scope.
+    /// \param Scope       Namespace scope
+    /// \param Name        Name of this namespace
+    /// \param File        Source file
+    /// \param LineNo      Line number
+    DINamespace *createNameSpace(DIScope *Scope, StringRef Name, DIFile *File,
                                  unsigned LineNo);
 
-    /// createLexicalBlockFile - This creates a descriptor for a lexical
-    /// block with a new file attached. This merely extends the existing
+    /// This creates new descriptor for a module with the specified
+    /// parent scope.
+    /// \param Scope       Parent scope
+    /// \param Name        Name of this module
+    /// \param ConfigurationMacros
+    ///                    A space-separated shell-quoted list of -D macro
+    ///                    definitions as they would appear on a command line.
+    /// \param IncludePath The path to the module map file.
+    /// \param ISysRoot    The clang system root (value of -isysroot).
+    DIModule *createModule(DIScope *Scope, StringRef Name,
+                           StringRef ConfigurationMacros,
+                           StringRef IncludePath,
+                           StringRef ISysRoot);
+
+    /// This creates a descriptor for a lexical block with a new file
+    /// attached. This merely extends the existing
     /// lexical block as it crosses a file.
-    /// @param Scope       Lexical block.
-    /// @param File        Source file.
-    /// @param Discriminator DWARF path discriminator value.
-    MDLexicalBlockFile *createLexicalBlockFile(MDScope *Scope, MDFile *File,
+    /// \param Scope       Lexical block.
+    /// \param File        Source file.
+    /// \param Discriminator DWARF path discriminator value.
+    DILexicalBlockFile *createLexicalBlockFile(DIScope *Scope, DIFile *File,
                                                unsigned Discriminator = 0);
 
-    /// createLexicalBlock - This creates a descriptor for a lexical block
-    /// with the specified parent context.
-    /// @param Scope         Parent lexical scope.
-    /// @param File          Source file.
-    /// @param Line          Line number.
-    /// @param Col           Column number.
-    MDLexicalBlock *createLexicalBlock(MDScope *Scope, MDFile *File,
+    /// This creates a descriptor for a lexical block with the
+    /// specified parent context.
+    /// \param Scope         Parent lexical scope.
+    /// \param File          Source file.
+    /// \param Line          Line number.
+    /// \param Col           Column number.
+    DILexicalBlock *createLexicalBlock(DIScope *Scope, DIFile *File,
                                        unsigned Line, unsigned Col);
 
-    /// \brief Create a descriptor for an imported module.
-    /// @param Context The scope this module is imported into
-    /// @param NS The namespace being imported here
-    /// @param Line Line number
-    MDImportedEntity *createImportedModule(MDScope *Context, MDNamespace *NS,
+    /// Create a descriptor for an imported module.
+    /// \param Context The scope this module is imported into
+    /// \param NS The namespace being imported here
+    /// \param Line Line number
+    DIImportedEntity *createImportedModule(DIScope *Context, DINamespace *NS,
                                            unsigned Line);
 
-    /// \brief Create a descriptor for an imported module.
-    /// @param Context The scope this module is imported into
-    /// @param NS An aliased namespace
-    /// @param Line Line number
-    MDImportedEntity *createImportedModule(MDScope *Context,
-                                           MDImportedEntity *NS, unsigned Line);
+    /// Create a descriptor for an imported module.
+    /// \param Context The scope this module is imported into
+    /// \param NS An aliased namespace
+    /// \param Line Line number
+    DIImportedEntity *createImportedModule(DIScope *Context,
+                                           DIImportedEntity *NS, unsigned Line);
+
+    /// Create a descriptor for an imported module.
+    /// \param Context The scope this module is imported into
+    /// \param M The module being imported here
+    /// \param Line Line number
+    DIImportedEntity *createImportedModule(DIScope *Context, DIModule *M,
+                                           unsigned Line);
 
-    /// \brief Create a descriptor for an imported function.
-    /// @param Context The scope this module is imported into
-    /// @param Decl The declaration (or definition) of a function, type, or
+    /// Create a descriptor for an imported function.
+    /// \param Context The scope this module is imported into
+    /// \param Decl The declaration (or definition) of a function, type, or
     ///             variable
-    /// @param Line Line number
-    MDImportedEntity *createImportedDeclaration(MDScope *Context,
-                                                DebugNode *Decl, unsigned Line,
+    /// \param Line Line number
+    DIImportedEntity *createImportedDeclaration(DIScope *Context, DINode *Decl,
+                                                unsigned Line,
                                                 StringRef Name = "");
 
-    /// insertDeclare - Insert a new llvm.dbg.declare intrinsic call.
-    /// @param Storage     llvm::Value of the variable
-    /// @param VarInfo     Variable's debug info descriptor.
-    /// @param Expr         A complex location expression.
-    /// @param DL           Debug info location.
-    /// @param InsertAtEnd Location for the new intrinsic.
-    Instruction *insertDeclare(llvm::Value *Storage, MDLocalVariable *VarInfo,
-                               MDExpression *Expr, const MDLocation *DL,
+    /// Insert a new llvm.dbg.declare intrinsic call.
+    /// \param Storage     llvm::Value of the variable
+    /// \param VarInfo     Variable's debug info descriptor.
+    /// \param Expr        A complex location expression.
+    /// \param DL          Debug info location.
+    /// \param InsertAtEnd Location for the new intrinsic.
+    Instruction *insertDeclare(llvm::Value *Storage, DILocalVariable *VarInfo,
+                               DIExpression *Expr, const DILocation *DL,
                                BasicBlock *InsertAtEnd);
 
-    /// insertDeclare - Insert a new llvm.dbg.declare intrinsic call.
-    /// @param Storage      llvm::Value of the variable
-    /// @param VarInfo      Variable's debug info descriptor.
-    /// @param Expr         A complex location expression.
-    /// @param DL           Debug info location.
-    /// @param InsertBefore Location for the new intrinsic.
-    Instruction *insertDeclare(llvm::Value *Storage, MDLocalVariable *VarInfo,
-                               MDExpression *Expr, const MDLocation *DL,
+    /// Insert a new llvm.dbg.declare intrinsic call.
+    /// \param Storage      llvm::Value of the variable
+    /// \param VarInfo      Variable's debug info descriptor.
+    /// \param Expr         A complex location expression.
+    /// \param DL           Debug info location.
+    /// \param InsertBefore Location for the new intrinsic.
+    Instruction *insertDeclare(llvm::Value *Storage, DILocalVariable *VarInfo,
+                               DIExpression *Expr, const DILocation *DL,
                                Instruction *InsertBefore);
 
-    /// insertDbgValueIntrinsic - Insert a new llvm.dbg.value intrinsic call.
-    /// @param Val          llvm::Value of the variable
-    /// @param Offset       Offset
-    /// @param VarInfo      Variable's debug info descriptor.
-    /// @param Expr         A complex location expression.
-    /// @param DL           Debug info location.
-    /// @param InsertAtEnd Location for the new intrinsic.
+    /// Insert a new llvm.dbg.value intrinsic call.
+    /// \param Val          llvm::Value of the variable
+    /// \param Offset       Offset
+    /// \param VarInfo      Variable's debug info descriptor.
+    /// \param Expr         A complex location expression.
+    /// \param DL           Debug info location.
+    /// \param InsertAtEnd Location for the new intrinsic.
     Instruction *insertDbgValueIntrinsic(llvm::Value *Val, uint64_t Offset,
-                                         MDLocalVariable *VarInfo,
-                                         MDExpression *Expr,
-                                         const MDLocation *DL,
+                                         DILocalVariable *VarInfo,
+                                         DIExpression *Expr,
+                                         const DILocation *DL,
                                          BasicBlock *InsertAtEnd);
 
-    /// insertDbgValueIntrinsic - Insert a new llvm.dbg.value intrinsic call.
-    /// @param Val          llvm::Value of the variable
-    /// @param Offset       Offset
-    /// @param VarInfo      Variable's debug info descriptor.
-    /// @param Expr         A complex location expression.
-    /// @param DL           Debug info location.
-    /// @param InsertBefore Location for the new intrinsic.
+    /// Insert a new llvm.dbg.value intrinsic call.
+    /// \param Val          llvm::Value of the variable
+    /// \param Offset       Offset
+    /// \param VarInfo      Variable's debug info descriptor.
+    /// \param Expr         A complex location expression.
+    /// \param DL           Debug info location.
+    /// \param InsertBefore Location for the new intrinsic.
     Instruction *insertDbgValueIntrinsic(llvm::Value *Val, uint64_t Offset,
-                                         MDLocalVariable *VarInfo,
-                                         MDExpression *Expr,
-                                         const MDLocation *DL,
+                                         DILocalVariable *VarInfo,
+                                         DIExpression *Expr,
+                                         const DILocation *DL,
                                          Instruction *InsertBefore);
 
-    /// \brief Replace the vtable holder in the given composite type.
+    /// Replace the vtable holder in the given composite type.
     ///
     /// If this creates a self reference, it may orphan some unresolved cycles
     /// in the operands of \c T, so \a DIBuilder needs to track that.
-    void replaceVTableHolder(MDCompositeType *&T,
-                             MDCompositeType *VTableHolder);
+    void replaceVTableHolder(DICompositeType *&T,
+                             DICompositeType *VTableHolder);
 
-    /// \brief Replace arrays on a composite type.
+    /// Replace arrays on a composite type.
     ///
     /// If \c T is resolved, but the arrays aren't -- which can happen if \c T
     /// has a self-reference -- \a DIBuilder needs to track the array to
     /// resolve cycles.
-    void replaceArrays(MDCompositeType *&T, DIArray Elements,
-                       DIArray TParems = DIArray());
+    void replaceArrays(DICompositeType *&T, DINodeArray Elements,
+                       DINodeArray TParems = DINodeArray());
 
-    /// \brief Replace a temporary node.
+    /// Replace a temporary node.
     ///
     /// Call \a MDNode::replaceAllUsesWith() on \c N, replacing it with \c
     /// Replacement.
diff --git a/include/llvm/IR/DataLayout.h b/include/llvm/IR/DataLayout.h
index 3e1f9744f9e..892d6c9936c 100644
--- a/include/llvm/IR/DataLayout.h
+++ b/include/llvm/IR/DataLayout.h
@@ -222,7 +222,9 @@ class DataLayout {
   /// This representation is in the same format accepted by the string
   /// constructor above. This should not be used to compare two DataLayout as
   /// different string can represent the same layout.
-  std::string getStringRepresentation() const { return StringRepresentation; }
+  const std::string &getStringRepresentation() const {
+    return StringRepresentation;
+  }
 
   /// \brief Test if the DataLayout was constructed from an empty string.
   bool isDefault() const { return StringRepresentation.empty(); }
diff --git a/include/llvm/IR/DebugInfo.h b/include/llvm/IR/DebugInfo.h
index 18b0c72b008..a06174827ba 100644
--- a/include/llvm/IR/DebugInfo.h
+++ b/include/llvm/IR/DebugInfo.h
@@ -29,405 +29,20 @@
 #include <iterator>
 
 namespace llvm {
-class BasicBlock;
-class Constant;
-class Function;
-class GlobalVariable;
 class Module;
-class Type;
-class Value;
 class DbgDeclareInst;
 class DbgValueInst;
-class Instruction;
-class Metadata;
-class MDNode;
-class MDString;
-class NamedMDNode;
-class LLVMContext;
-class raw_ostream;
-
-class DIFile;
-class DISubprogram;
-class DILexicalBlock;
-class DILexicalBlockFile;
-class DIVariable;
-class DIType;
-class DIScope;
-class DIObjCProperty;
 
 /// \brief Maps from type identifier to the actual MDNode.
-typedef DenseMap<const MDString *, MDNode *> DITypeIdentifierMap;
-
-class DIDescriptor {
-  MDNode *N;
-
-public:
-  DIDescriptor(const MDNode *N = nullptr) : N(const_cast<MDNode *>(N)) {}
-
-  operator MDNode *() const { return N; }
-  MDNode *operator->() const { return N; }
-  MDNode &operator*() const { return *N; }
-};
-
-#define DECLARE_SIMPLIFY_DESCRIPTOR(DESC)                                      \
-  class DESC;                                                                  \
-  template <> struct simplify_type<const DESC>;                                \
-  template <> struct simplify_type<DESC>;
-DECLARE_SIMPLIFY_DESCRIPTOR(DIDescriptor)
-DECLARE_SIMPLIFY_DESCRIPTOR(DISubrange)
-DECLARE_SIMPLIFY_DESCRIPTOR(DIEnumerator)
-DECLARE_SIMPLIFY_DESCRIPTOR(DIScope)
-DECLARE_SIMPLIFY_DESCRIPTOR(DIType)
-DECLARE_SIMPLIFY_DESCRIPTOR(DIBasicType)
-DECLARE_SIMPLIFY_DESCRIPTOR(DIDerivedType)
-DECLARE_SIMPLIFY_DESCRIPTOR(DICompositeType)
-DECLARE_SIMPLIFY_DESCRIPTOR(DISubroutineType)
-DECLARE_SIMPLIFY_DESCRIPTOR(DIFile)
-DECLARE_SIMPLIFY_DESCRIPTOR(DICompileUnit)
-DECLARE_SIMPLIFY_DESCRIPTOR(DISubprogram)
-DECLARE_SIMPLIFY_DESCRIPTOR(DILexicalBlock)
-DECLARE_SIMPLIFY_DESCRIPTOR(DILexicalBlockFile)
-DECLARE_SIMPLIFY_DESCRIPTOR(DINameSpace)
-DECLARE_SIMPLIFY_DESCRIPTOR(DITemplateTypeParameter)
-DECLARE_SIMPLIFY_DESCRIPTOR(DITemplateValueParameter)
-DECLARE_SIMPLIFY_DESCRIPTOR(DIGlobalVariable)
-DECLARE_SIMPLIFY_DESCRIPTOR(DIVariable)
-DECLARE_SIMPLIFY_DESCRIPTOR(DIExpression)
-DECLARE_SIMPLIFY_DESCRIPTOR(DILocation)
-DECLARE_SIMPLIFY_DESCRIPTOR(DIObjCProperty)
-DECLARE_SIMPLIFY_DESCRIPTOR(DIImportedEntity)
-#undef DECLARE_SIMPLIFY_DESCRIPTOR
-
-typedef DebugNodeArray DIArray;
-typedef MDTypeRefArray DITypeArray;
-typedef DebugNodeRef DIDescriptorRef;
-typedef MDScopeRef DIScopeRef;
-typedef MDTypeRef DITypeRef;
-
-class DISubrange {
-  MDSubrange *N;
-
-public:
-  DISubrange(const MDSubrange *N = nullptr) : N(const_cast<MDSubrange *>(N)) {}
-
-  operator MDSubrange *() const { return N; }
-  MDSubrange *operator->() const { return N; }
-  MDSubrange &operator*() const { return *N; }
-};
-
-class DIEnumerator {
-  MDEnumerator *N;
-
-public:
-  DIEnumerator(const MDEnumerator *N = nullptr)
-      : N(const_cast<MDEnumerator *>(N)) {}
-
-  operator MDEnumerator *() const { return N; }
-  MDEnumerator *operator->() const { return N; }
-  MDEnumerator &operator*() const { return *N; }
-};
-
-class DIScope {
-  MDScope *N;
-
-public:
-  DIScope(const MDScope *N = nullptr) : N(const_cast<MDScope *>(N)) {}
-
-  operator DIDescriptor() const { return N; }
-  operator MDScope *() const { return N; }
-  MDScope *operator->() const { return N; }
-  MDScope &operator*() const { return *N; }
-};
-
-class DIType {
-  MDType *N;
-
-public:
-  DIType(const MDType *N = nullptr) : N(const_cast<MDType *>(N)) {}
-
-  operator DIDescriptor() const { return N; }
-  operator DIScope() const { return N; }
-  operator MDType *() const { return N; }
-  MDType *operator->() const { return N; }
-  MDType &operator*() const { return *N; }
-};
-
-class DIBasicType {
-  MDBasicType *N;
-
-public:
-  DIBasicType(const MDBasicType *N = nullptr)
-      : N(const_cast<MDBasicType *>(N)) {}
-
-  operator DIDescriptor() const { return N; }
-  operator DIType() const { return N; }
-  operator MDBasicType *() const { return N; }
-  MDBasicType *operator->() const { return N; }
-  MDBasicType &operator*() const { return *N; }
-};
-
-class DIDerivedType {
-  MDDerivedTypeBase *N;
-
-public:
-  DIDerivedType(const MDDerivedTypeBase *N = nullptr)
-      : N(const_cast<MDDerivedTypeBase *>(N)) {}
-
-  operator DIDescriptor() const { return N; }
-  operator DIType() const { return N; }
-  operator MDDerivedTypeBase *() const { return N; }
-  MDDerivedTypeBase *operator->() const { return N; }
-  MDDerivedTypeBase &operator*() const { return *N; }
-};
-
-class DICompositeType {
-  MDCompositeTypeBase *N;
-
-public:
-  DICompositeType(const MDCompositeTypeBase *N = nullptr)
-      : N(const_cast<MDCompositeTypeBase *>(N)) {}
-
-  operator DIDescriptor() const { return N; }
-  operator DIType() const { return N; }
-  operator MDCompositeTypeBase *() const { return N; }
-  MDCompositeTypeBase *operator->() const { return N; }
-  MDCompositeTypeBase &operator*() const { return *N; }
-};
-
-class DISubroutineType {
-  MDSubroutineType *N;
-
-public:
-  DISubroutineType(const MDSubroutineType *N = nullptr)
-      : N(const_cast<MDSubroutineType *>(N)) {}
-
-  operator DIDescriptor() const { return N; }
-  operator DIType() const { return N; }
-  operator DICompositeType() const { return N; }
-  operator MDSubroutineType *() const { return N; }
-  MDSubroutineType *operator->() const { return N; }
-  MDSubroutineType &operator*() const { return *N; }
-};
-
-class DIFile {
-  MDFile *N;
-
-public:
-  DIFile(const MDFile *N = nullptr) : N(const_cast<MDFile *>(N)) {}
-
-  operator DIDescriptor() const { return N; }
-  operator DIScope() const { return N; }
-  operator MDFile *() const { return N; }
-  MDFile *operator->() const { return N; }
-  MDFile &operator*() const { return *N; }
-};
-
-class DICompileUnit {
-  MDCompileUnit *N;
-
-public:
-  DICompileUnit(const MDCompileUnit *N = nullptr)
-      : N(const_cast<MDCompileUnit *>(N)) {}
-
-  operator DIDescriptor() const { return N; }
-  operator DIScope() const { return N; }
-  operator MDCompileUnit *() const { return N; }
-  MDCompileUnit *operator->() const { return N; }
-  MDCompileUnit &operator*() const { return *N; }
-};
-
-class DISubprogram {
-  MDSubprogram *N;
-
-public:
-  DISubprogram(const MDSubprogram *N = nullptr)
-      : N(const_cast<MDSubprogram *>(N)) {}
-
-  operator DIDescriptor() const { return N; }
-  operator DIScope() const { return N; }
-  operator MDSubprogram *() const { return N; }
-  MDSubprogram *operator->() const { return N; }
-  MDSubprogram &operator*() const { return *N; }
-};
-
-class DILexicalBlock {
-  MDLexicalBlockBase *N;
-
-public:
-  DILexicalBlock(const MDLexicalBlockBase *N = nullptr)
-      : N(const_cast<MDLexicalBlockBase *>(N)) {}
-
-  operator DIDescriptor() const { return N; }
-  operator MDLexicalBlockBase *() const { return N; }
-  MDLexicalBlockBase *operator->() const { return N; }
-  MDLexicalBlockBase &operator*() const { return *N; }
-};
-
-class DILexicalBlockFile {
-  MDLexicalBlockFile *N;
-
-public:
-  DILexicalBlockFile(const MDLexicalBlockFile *N = nullptr)
-      : N(const_cast<MDLexicalBlockFile *>(N)) {}
-
-  operator DIDescriptor() const { return N; }
-  operator MDLexicalBlockFile *() const { return N; }
-  MDLexicalBlockFile *operator->() const { return N; }
-  MDLexicalBlockFile &operator*() const { return *N; }
-};
-
-class DINameSpace {
-  MDNamespace *N;
-
-public:
-  DINameSpace(const MDNamespace *N = nullptr)
-      : N(const_cast<MDNamespace *>(N)) {}
-
-  operator DIDescriptor() const { return N; }
-  operator DIScope() const { return N; }
-  operator MDNamespace *() const { return N; }
-  MDNamespace *operator->() const { return N; }
-  MDNamespace &operator*() const { return *N; }
-};
-
-class DITemplateTypeParameter {
-  MDTemplateTypeParameter *N;
-
-public:
-  DITemplateTypeParameter(const MDTemplateTypeParameter *N = nullptr)
-      : N(const_cast<MDTemplateTypeParameter *>(N)) {}
-
-  operator MDTemplateTypeParameter *() const { return N; }
-  MDTemplateTypeParameter *operator->() const { return N; }
-  MDTemplateTypeParameter &operator*() const { return *N; }
-};
-
-class DITemplateValueParameter {
-  MDTemplateValueParameter *N;
-
-public:
-  DITemplateValueParameter(const MDTemplateValueParameter *N = nullptr)
-      : N(const_cast<MDTemplateValueParameter *>(N)) {}
-
-  operator MDTemplateValueParameter *() const { return N; }
-  MDTemplateValueParameter *operator->() const { return N; }
-  MDTemplateValueParameter &operator*() const { return *N; }
-};
-
-class DIGlobalVariable {
-  MDGlobalVariable *N;
-
-public:
-  DIGlobalVariable(const MDGlobalVariable *N = nullptr)
-      : N(const_cast<MDGlobalVariable *>(N)) {}
-
-  operator DIDescriptor() const { return N; }
-  operator MDGlobalVariable *() const { return N; }
-  MDGlobalVariable *operator->() const { return N; }
-  MDGlobalVariable &operator*() const { return *N; }
-};
-
-class DIVariable {
-  MDLocalVariable *N;
-
-public:
-  DIVariable(const MDLocalVariable *N = nullptr)
-      : N(const_cast<MDLocalVariable *>(N)) {}
-
-  operator MDLocalVariable *() const { return N; }
-  MDLocalVariable *operator->() const { return N; }
-  MDLocalVariable &operator*() const { return *N; }
-};
-
-class DIExpression {
-  MDExpression *N;
-
-public:
-  DIExpression(const MDExpression *N = nullptr)
-      : N(const_cast<MDExpression *>(N)) {}
-
-  operator MDExpression *() const { return N; }
-  MDExpression *operator->() const { return N; }
-  MDExpression &operator*() const { return *N; }
-};
-
-class DILocation {
-  MDLocation *N;
-
-public:
-  DILocation(const MDLocation *N = nullptr) : N(const_cast<MDLocation *>(N)) {}
-
-  operator MDLocation *() const { return N; }
-  MDLocation *operator->() const { return N; }
-  MDLocation &operator*() const { return *N; }
-};
-
-class DIObjCProperty {
-  MDObjCProperty *N;
-
-public:
-  DIObjCProperty(const MDObjCProperty *N = nullptr)
-      : N(const_cast<MDObjCProperty *>(N)) {}
-
-  operator MDObjCProperty *() const { return N; }
-  MDObjCProperty *operator->() const { return N; }
-  MDObjCProperty &operator*() const { return *N; }
-};
-
-class DIImportedEntity {
-  MDImportedEntity *N;
-
-public:
-  DIImportedEntity(const MDImportedEntity *N = nullptr)
-      : N(const_cast<MDImportedEntity *>(N)) {}
-
-  operator DIDescriptor() const { return N; }
-  operator MDImportedEntity *() const { return N; }
-  MDImportedEntity *operator->() const { return N; }
-  MDImportedEntity &operator*() const { return *N; }
-};
-
-#define SIMPLIFY_DESCRIPTOR(DESC)                                              \
-  template <> struct simplify_type<const DESC> {                               \
-    typedef Metadata *SimpleType;                                              \
-    static SimpleType getSimplifiedValue(const DESC &DI) { return DI; }        \
-  };                                                                           \
-  template <> struct simplify_type<DESC> : simplify_type<const DESC> {};
-SIMPLIFY_DESCRIPTOR(DIDescriptor)
-SIMPLIFY_DESCRIPTOR(DISubrange)
-SIMPLIFY_DESCRIPTOR(DIEnumerator)
-SIMPLIFY_DESCRIPTOR(DIScope)
-SIMPLIFY_DESCRIPTOR(DIType)
-SIMPLIFY_DESCRIPTOR(DIBasicType)
-SIMPLIFY_DESCRIPTOR(DIDerivedType)
-SIMPLIFY_DESCRIPTOR(DICompositeType)
-SIMPLIFY_DESCRIPTOR(DISubroutineType)
-SIMPLIFY_DESCRIPTOR(DIFile)
-SIMPLIFY_DESCRIPTOR(DICompileUnit)
-SIMPLIFY_DESCRIPTOR(DISubprogram)
-SIMPLIFY_DESCRIPTOR(DILexicalBlock)
-SIMPLIFY_DESCRIPTOR(DILexicalBlockFile)
-SIMPLIFY_DESCRIPTOR(DINameSpace)
-SIMPLIFY_DESCRIPTOR(DITemplateTypeParameter)
-SIMPLIFY_DESCRIPTOR(DITemplateValueParameter)
-SIMPLIFY_DESCRIPTOR(DIGlobalVariable)
-SIMPLIFY_DESCRIPTOR(DIVariable)
-SIMPLIFY_DESCRIPTOR(DIExpression)
-SIMPLIFY_DESCRIPTOR(DILocation)
-SIMPLIFY_DESCRIPTOR(DIObjCProperty)
-SIMPLIFY_DESCRIPTOR(DIImportedEntity)
-#undef SIMPLIFY_DESCRIPTOR
+typedef DenseMap<const MDString *, DIType *> DITypeIdentifierMap;
 
 /// \brief Find subprogram that is enclosing this scope.
-DISubprogram getDISubprogram(const MDNode *Scope);
+DISubprogram *getDISubprogram(const MDNode *Scope);
 
 /// \brief Find debug info for a given function.
-/// \returns a valid DISubprogram, if found. Otherwise, it returns an empty
-/// DISubprogram.
-DISubprogram getDISubprogram(const Function *F);
-
-/// \brief Find underlying composite type.
-DICompositeType getDICompositeType(DIType T);
+///
+/// \returns a valid subprogram, if found. Otherwise, return \c nullptr.
+DISubprogram *getDISubprogram(const Function *F);
 
 /// \brief Generate map by visiting all retained types.
 DITypeIdentifierMap generateDITypeIdentifierMap(const NamedMDNode *CU_Nodes);
@@ -462,8 +77,8 @@ class DebugInfoFinder {
   void processDeclare(const Module &M, const DbgDeclareInst *DDI);
   /// \brief Process DbgValueInst.
   void processValue(const Module &M, const DbgValueInst *DVI);
-  /// \brief Process DILocation.
-  void processLocation(const Module &M, DILocation Loc);
+  /// \brief Process debug info location.
+  void processLocation(const Module &M, const DILocation *Loc);
 
   /// \brief Clear all lists.
   void reset();
@@ -471,22 +86,23 @@ class DebugInfoFinder {
 private:
   void InitializeTypeMap(const Module &M);
 
-  void processType(DIType DT);
-  void processSubprogram(DISubprogram SP);
-  void processScope(DIScope Scope);
-  bool addCompileUnit(DICompileUnit CU);
-  bool addGlobalVariable(DIGlobalVariable DIG);
-  bool addSubprogram(DISubprogram SP);
-  bool addType(DIType DT);
-  bool addScope(DIScope Scope);
+  void processType(DIType *DT);
+  void processSubprogram(DISubprogram *SP);
+  void processScope(DIScope *Scope);
+  bool addCompileUnit(DICompileUnit *CU);
+  bool addGlobalVariable(DIGlobalVariable *DIG);
+  bool addSubprogram(DISubprogram *SP);
+  bool addType(DIType *DT);
+  bool addScope(DIScope *Scope);
 
 public:
-  typedef SmallVectorImpl<DICompileUnit>::const_iterator compile_unit_iterator;
-  typedef SmallVectorImpl<DISubprogram>::const_iterator subprogram_iterator;
-  typedef SmallVectorImpl<DIGlobalVariable>::const_iterator
+  typedef SmallVectorImpl<DICompileUnit *>::const_iterator
+      compile_unit_iterator;
+  typedef SmallVectorImpl<DISubprogram *>::const_iterator subprogram_iterator;
+  typedef SmallVectorImpl<DIGlobalVariable *>::const_iterator
       global_variable_iterator;
-  typedef SmallVectorImpl<DIType>::const_iterator type_iterator;
-  typedef SmallVectorImpl<DIScope>::const_iterator scope_iterator;
+  typedef SmallVectorImpl<DIType *>::const_iterator type_iterator;
+  typedef SmallVectorImpl<DIScope *>::const_iterator scope_iterator;
 
   iterator_range<compile_unit_iterator> compile_units() const {
     return iterator_range<compile_unit_iterator>(CUs.begin(), CUs.end());
@@ -515,19 +131,19 @@ class DebugInfoFinder {
   unsigned scope_count() const { return Scopes.size(); }
 
 private:
-  SmallVector<DICompileUnit, 8> CUs;
-  SmallVector<DISubprogram, 8> SPs;
-  SmallVector<DIGlobalVariable, 8> GVs;
-  SmallVector<DIType, 8> TYs;
-  SmallVector<DIScope, 8> Scopes;
-  SmallPtrSet<MDNode *, 64> NodesSeen;
+  SmallVector<DICompileUnit *, 8> CUs;
+  SmallVector<DISubprogram *, 8> SPs;
+  SmallVector<DIGlobalVariable *, 8> GVs;
+  SmallVector<DIType *, 8> TYs;
+  SmallVector<DIScope *, 8> Scopes;
+  SmallPtrSet<const MDNode *, 64> NodesSeen;
   DITypeIdentifierMap TypeIdentifierMap;
 
   /// \brief Specify if TypeIdentifierMap is initialized.
   bool TypeMapInitialized;
 };
 
-DenseMap<const Function *, DISubprogram> makeSubprogramMap(const Module &M);
+DenseMap<const Function *, DISubprogram *> makeSubprogramMap(const Module &M);
 
 } // end namespace llvm
 
diff --git a/include/llvm/IR/DebugInfoFlags.def b/include/llvm/IR/DebugInfoFlags.def
index d5de8683fd3..9756c12264b 100644
--- a/include/llvm/IR/DebugInfoFlags.def
+++ b/include/llvm/IR/DebugInfoFlags.def
@@ -32,5 +32,6 @@ HANDLE_DI_FLAG((1 << 11), Vector)
 HANDLE_DI_FLAG((1 << 12), StaticMember)
 HANDLE_DI_FLAG((1 << 13), LValueReference)
 HANDLE_DI_FLAG((1 << 14), RValueReference)
+HANDLE_DI_FLAG((1 << 15), ExternalTypeRef)
 
 #undef HANDLE_DI_FLAG
diff --git a/include/llvm/IR/DebugInfoMetadata.h b/include/llvm/IR/DebugInfoMetadata.h
index 62373c4b09f..1f247f59b65 100644
--- a/include/llvm/IR/DebugInfoMetadata.h
+++ b/include/llvm/IR/DebugInfoMetadata.h
@@ -41,39 +41,39 @@
 
 namespace llvm {
 
-/// \brief Pointer union between a subclass of DebugNode and MDString.
+/// \brief Pointer union between a subclass of DINode and MDString.
 ///
-/// \a MDCompositeType can be referenced via an \a MDString unique identifier.
+/// \a DICompositeType can be referenced via an \a MDString unique identifier.
 /// This class allows some type safety in the face of that, requiring either a
 /// node of a particular type or an \a MDString.
-template <class T> class TypedDebugNodeRef {
+template <class T> class TypedDINodeRef {
   const Metadata *MD = nullptr;
 
 public:
-  TypedDebugNodeRef() = default;
-  TypedDebugNodeRef(std::nullptr_t) {}
+  TypedDINodeRef() = default;
+  TypedDINodeRef(std::nullptr_t) {}
 
   /// \brief Construct from a raw pointer.
-  explicit TypedDebugNodeRef(const Metadata *MD) : MD(MD) {
+  explicit TypedDINodeRef(const Metadata *MD) : MD(MD) {
     assert((!MD || isa<MDString>(MD) || isa<T>(MD)) && "Expected valid ref");
   }
 
   template <class U>
-  TypedDebugNodeRef(
-      const TypedDebugNodeRef<U> &X,
+  TypedDINodeRef(
+      const TypedDINodeRef<U> &X,
       typename std::enable_if<std::is_convertible<U *, T *>::value>::type * =
           nullptr)
       : MD(X) {}
 
   operator Metadata *() const { return const_cast<Metadata *>(MD); }
 
-  bool operator==(const TypedDebugNodeRef<T> &X) const { return MD == X.MD; };
-  bool operator!=(const TypedDebugNodeRef<T> &X) const { return MD != X.MD; };
+  bool operator==(const TypedDINodeRef<T> &X) const { return MD == X.MD; }
+  bool operator!=(const TypedDINodeRef<T> &X) const { return MD != X.MD; }
 
   /// \brief Create a reference.
   ///
   /// Get a reference to \c N, using an \a MDString reference if available.
-  static TypedDebugNodeRef get(const T *N);
+  static TypedDINodeRef get(const T *N);
 
   template <class MapTy> T *resolve(const MapTy &Map) const {
     if (!MD)
@@ -89,15 +89,15 @@ template <class T> class TypedDebugNodeRef {
   }
 };
 
-typedef TypedDebugNodeRef<DebugNode> DebugNodeRef;
-typedef TypedDebugNodeRef<MDScope> MDScopeRef;
-typedef TypedDebugNodeRef<MDType> MDTypeRef;
+typedef TypedDINodeRef<DINode> DINodeRef;
+typedef TypedDINodeRef<DIScope> DIScopeRef;
+typedef TypedDINodeRef<DIType> DITypeRef;
 
-class MDTypeRefArray {
+class DITypeRefArray {
   const MDTuple *N = nullptr;
 
 public:
-  MDTypeRefArray(const MDTuple *N) : N(N) {}
+  DITypeRefArray(const MDTuple *N) : N(N) {}
 
   explicit operator bool() const { return get(); }
   explicit operator MDTuple *() const { return get(); }
@@ -108,16 +108,16 @@ class MDTypeRefArray {
 
   // FIXME: Fix callers and remove condition on N.
   unsigned size() const { return N ? N->getNumOperands() : 0u; }
-  MDTypeRef operator[](unsigned I) const { return MDTypeRef(N->getOperand(I)); }
+  DITypeRef operator[](unsigned I) const { return DITypeRef(N->getOperand(I)); }
 
-  class iterator : std::iterator<std::input_iterator_tag, MDTypeRef,
-                                 std::ptrdiff_t, void, MDTypeRef> {
+  class iterator : std::iterator<std::input_iterator_tag, DITypeRef,
+                                 std::ptrdiff_t, void, DITypeRef> {
     MDNode::op_iterator I = nullptr;
 
   public:
     iterator() = default;
     explicit iterator(MDNode::op_iterator I) : I(I) {}
-    MDTypeRef operator*() const { return MDTypeRef(*I); }
+    DITypeRef operator*() const { return DITypeRef(*I); }
     iterator &operator++() {
       ++I;
       return *this;
@@ -139,20 +139,20 @@ class MDTypeRefArray {
 /// \brief Tagged DWARF-like metadata node.
 ///
 /// A metadata node with a DWARF tag (i.e., a constant named \c DW_TAG_*,
-/// defined in llvm/Support/Dwarf.h).  Called \a DebugNode because it's
+/// defined in llvm/Support/Dwarf.h).  Called \a DINode because it's
 /// potentially used for non-DWARF output.
-class DebugNode : public MDNode {
+class DINode : public MDNode {
   friend class LLVMContextImpl;
   friend class MDNode;
 
 protected:
-  DebugNode(LLVMContext &C, unsigned ID, StorageType Storage, unsigned Tag,
-            ArrayRef<Metadata *> Ops1, ArrayRef<Metadata *> Ops2 = None)
+  DINode(LLVMContext &C, unsigned ID, StorageType Storage, unsigned Tag,
+         ArrayRef<Metadata *> Ops1, ArrayRef<Metadata *> Ops2 = None)
       : MDNode(C, ID, Storage, Ops1, Ops2) {
     assert(Tag < 1u << 16);
     SubclassData16 = Tag;
   }
-  ~DebugNode() = default;
+  ~DINode() = default;
 
   template <class Ty> Ty *getOperandAs(unsigned I) const {
     return cast_or_null<Ty>(getOperand(I));
@@ -193,47 +193,47 @@ class DebugNode : public MDNode {
   static unsigned splitFlags(unsigned Flags,
                              SmallVectorImpl<unsigned> &SplitFlags);
 
-  DebugNodeRef getRef() const { return DebugNodeRef::get(this); }
+  DINodeRef getRef() const { return DINodeRef::get(this); }
 
   static bool classof(const Metadata *MD) {
     switch (MD->getMetadataID()) {
     default:
       return false;
-    case GenericDebugNodeKind:
-    case MDSubrangeKind:
-    case MDEnumeratorKind:
-    case MDBasicTypeKind:
-    case MDDerivedTypeKind:
-    case MDCompositeTypeKind:
-    case MDSubroutineTypeKind:
-    case MDFileKind:
-    case MDCompileUnitKind:
-    case MDSubprogramKind:
-    case MDLexicalBlockKind:
-    case MDLexicalBlockFileKind:
-    case MDNamespaceKind:
-    case MDTemplateTypeParameterKind:
-    case MDTemplateValueParameterKind:
-    case MDGlobalVariableKind:
-    case MDLocalVariableKind:
-    case MDObjCPropertyKind:
-    case MDImportedEntityKind:
+    case GenericDINodeKind:
+    case DISubrangeKind:
+    case DIEnumeratorKind:
+    case DIBasicTypeKind:
+    case DIDerivedTypeKind:
+    case DICompositeTypeKind:
+    case DISubroutineTypeKind:
+    case DIFileKind:
+    case DICompileUnitKind:
+    case DISubprogramKind:
+    case DILexicalBlockKind:
+    case DILexicalBlockFileKind:
+    case DINamespaceKind:
+    case DITemplateTypeParameterKind:
+    case DITemplateValueParameterKind:
+    case DIGlobalVariableKind:
+    case DILocalVariableKind:
+    case DIObjCPropertyKind:
+    case DIImportedEntityKind:
+    case DIModuleKind:
       return true;
     }
   }
 };
 
-template <class T>
-struct simplify_type<const TypedDebugNodeRef<T>> {
+template <class T> struct simplify_type<const TypedDINodeRef<T>> {
   typedef Metadata *SimpleType;
-  static SimpleType getSimplifiedValue(const TypedDebugNodeRef<T> &MD) {
+  static SimpleType getSimplifiedValue(const TypedDINodeRef<T> &MD) {
     return MD;
   }
 };
 
 template <class T>
-struct simplify_type<TypedDebugNodeRef<T>>
-    : simplify_type<const TypedDebugNodeRef<T>> {};
+struct simplify_type<TypedDINodeRef<T>>
+    : simplify_type<const TypedDINodeRef<T>> {};
 
 /// \brief Generic tagged DWARF-like metadata node.
 ///
@@ -241,37 +241,33 @@ struct simplify_type<TypedDebugNodeRef<T>>
 /// (possibly empty) null-separated \a MDString header that contains arbitrary
 /// fields.  The remaining operands are \a dwarf_operands(), and are pointers
 /// to other metadata.
-class GenericDebugNode : public DebugNode {
+class GenericDINode : public DINode {
   friend class LLVMContextImpl;
   friend class MDNode;
 
-  GenericDebugNode(LLVMContext &C, StorageType Storage, unsigned Hash,
-                   unsigned Tag, ArrayRef<Metadata *> Ops1,
-                   ArrayRef<Metadata *> Ops2)
-      : DebugNode(C, GenericDebugNodeKind, Storage, Tag, Ops1, Ops2) {
+  GenericDINode(LLVMContext &C, StorageType Storage, unsigned Hash,
+                unsigned Tag, ArrayRef<Metadata *> Ops1,
+                ArrayRef<Metadata *> Ops2)
+      : DINode(C, GenericDINodeKind, Storage, Tag, Ops1, Ops2) {
     setHash(Hash);
   }
-  ~GenericDebugNode() { dropAllReferences(); }
+  ~GenericDINode() { dropAllReferences(); }
 
   void setHash(unsigned Hash) { SubclassData32 = Hash; }
   void recalculateHash();
 
-  static GenericDebugNode *getImpl(LLVMContext &Context, unsigned Tag,
-                                   StringRef Header,
-                                   ArrayRef<Metadata *> DwarfOps,
-                                   StorageType Storage,
-                                   bool ShouldCreate = true) {
+  static GenericDINode *getImpl(LLVMContext &Context, unsigned Tag,
+                                StringRef Header, ArrayRef<Metadata *> DwarfOps,
+                                StorageType Storage, bool ShouldCreate = true) {
     return getImpl(Context, Tag, getCanonicalMDString(Context, Header),
                    DwarfOps, Storage, ShouldCreate);
   }
 
-  static GenericDebugNode *getImpl(LLVMContext &Context, unsigned Tag,
-                                   MDString *Header,
-                                   ArrayRef<Metadata *> DwarfOps,
-                                   StorageType Storage,
-                                   bool ShouldCreate = true);
+  static GenericDINode *getImpl(LLVMContext &Context, unsigned Tag,
+                                MDString *Header, ArrayRef<Metadata *> DwarfOps,
+                                StorageType Storage, bool ShouldCreate = true);
 
-  TempGenericDebugNode cloneImpl() const {
+  TempGenericDINode cloneImpl() const {
     return getTemporary(
         getContext(), getTag(), getHeader(),
         SmallVector<Metadata *, 4>(dwarf_op_begin(), dwarf_op_end()));
@@ -280,15 +276,15 @@ class GenericDebugNode : public DebugNode {
 public:
   unsigned getHash() const { return SubclassData32; }
 
-  DEFINE_MDNODE_GET(GenericDebugNode, (unsigned Tag, StringRef Header,
-                                       ArrayRef<Metadata *> DwarfOps),
+  DEFINE_MDNODE_GET(GenericDINode, (unsigned Tag, StringRef Header,
+                                    ArrayRef<Metadata *> DwarfOps),
                     (Tag, Header, DwarfOps))
-  DEFINE_MDNODE_GET(GenericDebugNode, (unsigned Tag, MDString *Header,
-                                       ArrayRef<Metadata *> DwarfOps),
+  DEFINE_MDNODE_GET(GenericDINode, (unsigned Tag, MDString *Header,
+                                    ArrayRef<Metadata *> DwarfOps),
                     (Tag, Header, DwarfOps))
 
   /// \brief Return a (temporary) clone of this.
-  TempGenericDebugNode clone() const { return cloneImpl(); }
+  TempGenericDINode clone() const { return cloneImpl(); }
 
   unsigned getTag() const { return SubclassData16; }
   StringRef getHeader() const { return getStringOperand(0); }
@@ -308,7 +304,7 @@ class GenericDebugNode : public DebugNode {
   }
 
   static bool classof(const Metadata *MD) {
-    return MD->getMetadataID() == GenericDebugNodeKind;
+    return MD->getMetadataID() == GenericDINodeKind;
   }
 };
 
@@ -316,39 +312,38 @@ class GenericDebugNode : public DebugNode {
 ///
 /// TODO: Merge into node for DW_TAG_array_type, which should have a custom
 /// type.
-class MDSubrange : public DebugNode {
+class DISubrange : public DINode {
   friend class LLVMContextImpl;
   friend class MDNode;
 
   int64_t Count;
   int64_t LowerBound;
 
-  MDSubrange(LLVMContext &C, StorageType Storage, int64_t Count,
+  DISubrange(LLVMContext &C, StorageType Storage, int64_t Count,
              int64_t LowerBound)
-      : DebugNode(C, MDSubrangeKind, Storage, dwarf::DW_TAG_subrange_type,
-                  None),
+      : DINode(C, DISubrangeKind, Storage, dwarf::DW_TAG_subrange_type, None),
         Count(Count), LowerBound(LowerBound) {}
-  ~MDSubrange() = default;
+  ~DISubrange() = default;
 
-  static MDSubrange *getImpl(LLVMContext &Context, int64_t Count,
+  static DISubrange *getImpl(LLVMContext &Context, int64_t Count,
                              int64_t LowerBound, StorageType Storage,
                              bool ShouldCreate = true);
 
-  TempMDSubrange cloneImpl() const {
+  TempDISubrange cloneImpl() const {
     return getTemporary(getContext(), getCount(), getLowerBound());
   }
 
 public:
-  DEFINE_MDNODE_GET(MDSubrange, (int64_t Count, int64_t LowerBound = 0),
+  DEFINE_MDNODE_GET(DISubrange, (int64_t Count, int64_t LowerBound = 0),
                     (Count, LowerBound))
 
-  TempMDSubrange clone() const { return cloneImpl(); }
+  TempDISubrange clone() const { return cloneImpl(); }
 
   int64_t getLowerBound() const { return LowerBound; }
   int64_t getCount() const { return Count; }
 
   static bool classof(const Metadata *MD) {
-    return MD->getMetadataID() == MDSubrangeKind;
+    return MD->getMetadataID() == DISubrangeKind;
   }
 };
 
@@ -356,39 +351,39 @@ class MDSubrange : public DebugNode {
 ///
 /// TODO: Add a pointer to the context (DW_TAG_enumeration_type) once that no
 /// longer creates a type cycle.
-class MDEnumerator : public DebugNode {
+class DIEnumerator : public DINode {
   friend class LLVMContextImpl;
   friend class MDNode;
 
   int64_t Value;
 
-  MDEnumerator(LLVMContext &C, StorageType Storage, int64_t Value,
+  DIEnumerator(LLVMContext &C, StorageType Storage, int64_t Value,
                ArrayRef<Metadata *> Ops)
-      : DebugNode(C, MDEnumeratorKind, Storage, dwarf::DW_TAG_enumerator, Ops),
+      : DINode(C, DIEnumeratorKind, Storage, dwarf::DW_TAG_enumerator, Ops),
         Value(Value) {}
-  ~MDEnumerator() = default;
+  ~DIEnumerator() = default;
 
-  static MDEnumerator *getImpl(LLVMContext &Context, int64_t Value,
+  static DIEnumerator *getImpl(LLVMContext &Context, int64_t Value,
                                StringRef Name, StorageType Storage,
                                bool ShouldCreate = true) {
     return getImpl(Context, Value, getCanonicalMDString(Context, Name), Storage,
                    ShouldCreate);
   }
-  static MDEnumerator *getImpl(LLVMContext &Context, int64_t Value,
+  static DIEnumerator *getImpl(LLVMContext &Context, int64_t Value,
                                MDString *Name, StorageType Storage,
                                bool ShouldCreate = true);
 
-  TempMDEnumerator cloneImpl() const {
+  TempDIEnumerator cloneImpl() const {
     return getTemporary(getContext(), getValue(), getName());
   }
 
 public:
-  DEFINE_MDNODE_GET(MDEnumerator, (int64_t Value, StringRef Name),
+  DEFINE_MDNODE_GET(DIEnumerator, (int64_t Value, StringRef Name),
                     (Value, Name))
-  DEFINE_MDNODE_GET(MDEnumerator, (int64_t Value, MDString *Name),
+  DEFINE_MDNODE_GET(DIEnumerator, (int64_t Value, MDString *Name),
                     (Value, Name))
 
-  TempMDEnumerator clone() const { return cloneImpl(); }
+  TempDIEnumerator clone() const { return cloneImpl(); }
 
   int64_t getValue() const { return Value; }
   StringRef getName() const { return getStringOperand(0); }
@@ -396,7 +391,7 @@ class MDEnumerator : public DebugNode {
   MDString *getRawName() const { return getOperandAs<MDString>(0); }
 
   static bool classof(const Metadata *MD) {
-    return MD->getMetadataID() == MDEnumeratorKind;
+    return MD->getMetadataID() == DIEnumeratorKind;
   }
 };
 
@@ -406,49 +401,50 @@ class MDEnumerator : public DebugNode {
 /// contexts).
 ///
 /// TODO: Separate the concepts of declaration contexts and lexical scopes.
-class MDScope : public DebugNode {
+class DIScope : public DINode {
 protected:
-  MDScope(LLVMContext &C, unsigned ID, StorageType Storage, unsigned Tag,
+  DIScope(LLVMContext &C, unsigned ID, StorageType Storage, unsigned Tag,
           ArrayRef<Metadata *> Ops)
-      : DebugNode(C, ID, Storage, Tag, Ops) {}
-  ~MDScope() = default;
+      : DINode(C, ID, Storage, Tag, Ops) {}
+  ~DIScope() = default;
 
 public:
-  MDFile *getFile() const { return cast_or_null<MDFile>(getRawFile()); }
+  DIFile *getFile() const { return cast_or_null<DIFile>(getRawFile()); }
 
   inline StringRef getFilename() const;
   inline StringRef getDirectory() const;
 
   StringRef getName() const;
-  MDScopeRef getScope() const;
+  DIScopeRef getScope() const;
 
   /// \brief Return the raw underlying file.
   ///
-  /// An \a MDFile is an \a MDScope, but it doesn't point at a separate file
-  /// (it\em is the file).  If \c this is an \a MDFile, we need to return \c
+  /// An \a DIFile is an \a DIScope, but it doesn't point at a separate file
+  /// (it\em is the file).  If \c this is an \a DIFile, we need to return \c
   /// this.  Otherwise, return the first operand, which is where all other
   /// subclasses store their file pointer.
   Metadata *getRawFile() const {
-    return isa<MDFile>(this) ? const_cast<MDScope *>(this)
+    return isa<DIFile>(this) ? const_cast<DIScope *>(this)
                              : static_cast<Metadata *>(getOperand(0));
   }
 
-  MDScopeRef getRef() const { return MDScopeRef::get(this); }
+  DIScopeRef getRef() const { return DIScopeRef::get(this); }
 
   static bool classof(const Metadata *MD) {
     switch (MD->getMetadataID()) {
     default:
       return false;
-    case MDBasicTypeKind:
-    case MDDerivedTypeKind:
-    case MDCompositeTypeKind:
-    case MDSubroutineTypeKind:
-    case MDFileKind:
-    case MDCompileUnitKind:
-    case MDSubprogramKind:
-    case MDLexicalBlockKind:
-    case MDLexicalBlockFileKind:
-    case MDNamespaceKind:
+    case DIBasicTypeKind:
+    case DIDerivedTypeKind:
+    case DICompositeTypeKind:
+    case DISubroutineTypeKind:
+    case DIFileKind:
+    case DICompileUnitKind:
+    case DISubprogramKind:
+    case DILexicalBlockKind:
+    case DILexicalBlockFileKind:
+    case DINamespaceKind:
+    case DIModuleKind:
       return true;
     }
   }
@@ -458,36 +454,36 @@ class MDScope : public DebugNode {
 ///
 /// TODO: Merge with directory/file node (including users).
 /// TODO: Canonicalize paths on creation.
-class MDFile : public MDScope {
+class DIFile : public DIScope {
   friend class LLVMContextImpl;
   friend class MDNode;
 
-  MDFile(LLVMContext &C, StorageType Storage, ArrayRef<Metadata *> Ops)
-      : MDScope(C, MDFileKind, Storage, dwarf::DW_TAG_file_type, Ops) {}
-  ~MDFile() = default;
+  DIFile(LLVMContext &C, StorageType Storage, ArrayRef<Metadata *> Ops)
+      : DIScope(C, DIFileKind, Storage, dwarf::DW_TAG_file_type, Ops) {}
+  ~DIFile() = default;
 
-  static MDFile *getImpl(LLVMContext &Context, StringRef Filename,
+  static DIFile *getImpl(LLVMContext &Context, StringRef Filename,
                          StringRef Directory, StorageType Storage,
                          bool ShouldCreate = true) {
     return getImpl(Context, getCanonicalMDString(Context, Filename),
                    getCanonicalMDString(Context, Directory), Storage,
                    ShouldCreate);
   }
-  static MDFile *getImpl(LLVMContext &Context, MDString *Filename,
+  static DIFile *getImpl(LLVMContext &Context, MDString *Filename,
                          MDString *Directory, StorageType Storage,
                          bool ShouldCreate = true);
 
-  TempMDFile cloneImpl() const {
+  TempDIFile cloneImpl() const {
     return getTemporary(getContext(), getFilename(), getDirectory());
   }
 
 public:
-  DEFINE_MDNODE_GET(MDFile, (StringRef Filename, StringRef Directory),
+  DEFINE_MDNODE_GET(DIFile, (StringRef Filename, StringRef Directory),
                     (Filename, Directory))
-  DEFINE_MDNODE_GET(MDFile, (MDString * Filename, MDString *Directory),
+  DEFINE_MDNODE_GET(DIFile, (MDString * Filename, MDString *Directory),
                     (Filename, Directory))
 
-  TempMDFile clone() const { return cloneImpl(); }
+  TempDIFile clone() const { return cloneImpl(); }
 
   StringRef getFilename() const { return getStringOperand(0); }
   StringRef getDirectory() const { return getStringOperand(1); }
@@ -496,17 +492,17 @@ class MDFile : public MDScope {
   MDString *getRawDirectory() const { return getOperandAs<MDString>(1); }
 
   static bool classof(const Metadata *MD) {
-    return MD->getMetadataID() == MDFileKind;
+    return MD->getMetadataID() == DIFileKind;
   }
 };
 
-StringRef MDScope::getFilename() const {
+StringRef DIScope::getFilename() const {
   if (auto *F = getFile())
     return F->getFilename();
   return "";
 }
 
-StringRef MDScope::getDirectory() const {
+StringRef DIScope::getDirectory() const {
   if (auto *F = getFile())
     return F->getDirectory();
   return "";
@@ -517,7 +513,7 @@ StringRef MDScope::getDirectory() const {
 /// TODO: Remove the hardcoded name and context, since many types don't use
 /// them.
 /// TODO: Split up flags.
-class MDType : public MDScope {
+class DIType : public DIScope {
   unsigned Line;
   unsigned Flags;
   uint64_t SizeInBits;
@@ -525,17 +521,17 @@ class MDType : public MDScope {
   uint64_t OffsetInBits;
 
 protected:
-  MDType(LLVMContext &C, unsigned ID, StorageType Storage, unsigned Tag,
+  DIType(LLVMContext &C, unsigned ID, StorageType Storage, unsigned Tag,
          unsigned Line, uint64_t SizeInBits, uint64_t AlignInBits,
          uint64_t OffsetInBits, unsigned Flags, ArrayRef<Metadata *> Ops)
-      : MDScope(C, ID, Storage, Tag, Ops), Line(Line), Flags(Flags),
+      : DIScope(C, ID, Storage, Tag, Ops), Line(Line), Flags(Flags),
         SizeInBits(SizeInBits), AlignInBits(AlignInBits),
         OffsetInBits(OffsetInBits) {}
-  ~MDType() = default;
+  ~DIType() = default;
 
 public:
-  TempMDType clone() const {
-    return TempMDType(cast<MDType>(MDNode::clone().release()));
+  TempDIType clone() const {
+    return TempDIType(cast<DIType>(MDNode::clone().release()));
   }
 
   unsigned getLine() const { return Line; }
@@ -544,7 +540,7 @@ class MDType : public MDScope {
   uint64_t getOffsetInBits() const { return OffsetInBits; }
   unsigned getFlags() const { return Flags; }
 
-  MDScopeRef getScope() const { return MDScopeRef(getRawScope()); }
+  DIScopeRef getScope() const { return DIScopeRef(getRawScope()); }
   StringRef getName() const { return getStringOperand(2); }
 
 
@@ -578,17 +574,18 @@ class MDType : public MDScope {
   bool isStaticMember() const { return getFlags() & FlagStaticMember; }
   bool isLValueReference() const { return getFlags() & FlagLValueReference; }
   bool isRValueReference() const { return getFlags() & FlagRValueReference; }
+  bool isExternalTypeRef() const { return getFlags() & FlagExternalTypeRef; }
 
-  MDTypeRef getRef() const { return MDTypeRef::get(this); }
+  DITypeRef getRef() const { return DITypeRef::get(this); }
 
   static bool classof(const Metadata *MD) {
     switch (MD->getMetadataID()) {
     default:
       return false;
-    case MDBasicTypeKind:
-    case MDDerivedTypeKind:
-    case MDCompositeTypeKind:
-    case MDSubroutineTypeKind:
+    case DIBasicTypeKind:
+    case DIDerivedTypeKind:
+    case DICompositeTypeKind:
+    case DISubroutineTypeKind:
       return true;
     }
   }
@@ -598,79 +595,55 @@ class MDType : public MDScope {
 ///
 /// TODO: Split out DW_TAG_unspecified_type.
 /// TODO: Drop unused accessors.
-class MDBasicType : public MDType {
+class DIBasicType : public DIType {
   friend class LLVMContextImpl;
   friend class MDNode;
 
   unsigned Encoding;
 
-  MDBasicType(LLVMContext &C, StorageType Storage, unsigned Tag,
+  DIBasicType(LLVMContext &C, StorageType Storage, unsigned Tag,
               uint64_t SizeInBits, uint64_t AlignInBits, unsigned Encoding,
               ArrayRef<Metadata *> Ops)
-      : MDType(C, MDBasicTypeKind, Storage, Tag, 0, SizeInBits, AlignInBits, 0,
+      : DIType(C, DIBasicTypeKind, Storage, Tag, 0, SizeInBits, AlignInBits, 0,
                0, Ops),
         Encoding(Encoding) {}
-  ~MDBasicType() = default;
+  ~DIBasicType() = default;
 
-  static MDBasicType *getImpl(LLVMContext &Context, unsigned Tag,
+  static DIBasicType *getImpl(LLVMContext &Context, unsigned Tag,
                               StringRef Name, uint64_t SizeInBits,
                               uint64_t AlignInBits, unsigned Encoding,
                               StorageType Storage, bool ShouldCreate = true) {
     return getImpl(Context, Tag, getCanonicalMDString(Context, Name),
                    SizeInBits, AlignInBits, Encoding, Storage, ShouldCreate);
   }
-  static MDBasicType *getImpl(LLVMContext &Context, unsigned Tag,
+  static DIBasicType *getImpl(LLVMContext &Context, unsigned Tag,
                               MDString *Name, uint64_t SizeInBits,
                               uint64_t AlignInBits, unsigned Encoding,
                               StorageType Storage, bool ShouldCreate = true);
 
-  TempMDBasicType cloneImpl() const {
+  TempDIBasicType cloneImpl() const {
     return getTemporary(getContext(), getTag(), getName(), getSizeInBits(),
                         getAlignInBits(), getEncoding());
   }
 
 public:
-  DEFINE_MDNODE_GET(MDBasicType, (unsigned Tag, StringRef Name),
+  DEFINE_MDNODE_GET(DIBasicType, (unsigned Tag, StringRef Name),
                     (Tag, Name, 0, 0, 0))
-  DEFINE_MDNODE_GET(MDBasicType,
+  DEFINE_MDNODE_GET(DIBasicType,
                     (unsigned Tag, StringRef Name, uint64_t SizeInBits,
                      uint64_t AlignInBits, unsigned Encoding),
                     (Tag, Name, SizeInBits, AlignInBits, Encoding))
-  DEFINE_MDNODE_GET(MDBasicType,
+  DEFINE_MDNODE_GET(DIBasicType,
                     (unsigned Tag, MDString *Name, uint64_t SizeInBits,
                      uint64_t AlignInBits, unsigned Encoding),
                     (Tag, Name, SizeInBits, AlignInBits, Encoding))
 
-  TempMDBasicType clone() const { return cloneImpl(); }
+  TempDIBasicType clone() const { return cloneImpl(); }
 
   unsigned getEncoding() const { return Encoding; }
 
   static bool classof(const Metadata *MD) {
-    return MD->getMetadataID() == MDBasicTypeKind;
-  }
-};
-
-/// \brief Base class for MDDerivedType and MDCompositeType.
-///
-/// TODO: Delete; they're not really related.
-class MDDerivedTypeBase : public MDType {
-protected:
-  MDDerivedTypeBase(LLVMContext &C, unsigned ID, StorageType Storage,
-                    unsigned Tag, unsigned Line, uint64_t SizeInBits,
-                    uint64_t AlignInBits, uint64_t OffsetInBits, unsigned Flags,
-                    ArrayRef<Metadata *> Ops)
-      : MDType(C, ID, Storage, Tag, Line, SizeInBits, AlignInBits, OffsetInBits,
-               Flags, Ops) {}
-  ~MDDerivedTypeBase() = default;
-
-public:
-  MDTypeRef getBaseType() const { return MDTypeRef(getRawBaseType()); }
-  Metadata *getRawBaseType() const { return getOperand(3); }
-
-  static bool classof(const Metadata *MD) {
-    return MD->getMetadataID() == MDDerivedTypeKind ||
-           MD->getMetadataID() == MDCompositeTypeKind ||
-           MD->getMetadataID() == MDSubroutineTypeKind;
+    return MD->getMetadataID() == DIBasicTypeKind;
   }
 };
 
@@ -680,20 +653,20 @@ class MDDerivedTypeBase : public MDType {
 /// class members.
 ///
 /// TODO: Split out members (inheritance, fields, methods, etc.).
-class MDDerivedType : public MDDerivedTypeBase {
+class DIDerivedType : public DIType {
   friend class LLVMContextImpl;
   friend class MDNode;
 
-  MDDerivedType(LLVMContext &C, StorageType Storage, unsigned Tag,
+  DIDerivedType(LLVMContext &C, StorageType Storage, unsigned Tag,
                 unsigned Line, uint64_t SizeInBits, uint64_t AlignInBits,
                 uint64_t OffsetInBits, unsigned Flags, ArrayRef<Metadata *> Ops)
-      : MDDerivedTypeBase(C, MDDerivedTypeKind, Storage, Tag, Line, SizeInBits,
-                          AlignInBits, OffsetInBits, Flags, Ops) {}
-  ~MDDerivedType() = default;
+      : DIType(C, DIDerivedTypeKind, Storage, Tag, Line, SizeInBits,
+               AlignInBits, OffsetInBits, Flags, Ops) {}
+  ~DIDerivedType() = default;
 
-  static MDDerivedType *getImpl(LLVMContext &Context, unsigned Tag,
-                                StringRef Name, MDFile *File, unsigned Line,
-                                MDScopeRef Scope, MDTypeRef BaseType,
+  static DIDerivedType *getImpl(LLVMContext &Context, unsigned Tag,
+                                StringRef Name, DIFile *File, unsigned Line,
+                                DIScopeRef Scope, DITypeRef BaseType,
                                 uint64_t SizeInBits, uint64_t AlignInBits,
                                 uint64_t OffsetInBits, unsigned Flags,
                                 Metadata *ExtraData, StorageType Storage,
@@ -702,7 +675,7 @@ class MDDerivedType : public MDDerivedTypeBase {
                    Line, Scope, BaseType, SizeInBits, AlignInBits, OffsetInBits,
                    Flags, ExtraData, Storage, ShouldCreate);
   }
-  static MDDerivedType *getImpl(LLVMContext &Context, unsigned Tag,
+  static DIDerivedType *getImpl(LLVMContext &Context, unsigned Tag,
                                 MDString *Name, Metadata *File, unsigned Line,
                                 Metadata *Scope, Metadata *BaseType,
                                 uint64_t SizeInBits, uint64_t AlignInBits,
@@ -710,7 +683,7 @@ class MDDerivedType : public MDDerivedTypeBase {
                                 Metadata *ExtraData, StorageType Storage,
                                 bool ShouldCreate = true);
 
-  TempMDDerivedType cloneImpl() const {
+  TempDIDerivedType cloneImpl() const {
     return getTemporary(getContext(), getTag(), getName(), getFile(), getLine(),
                         getScope(), getBaseType(), getSizeInBits(),
                         getAlignInBits(), getOffsetInBits(), getFlags(),
@@ -718,7 +691,7 @@ class MDDerivedType : public MDDerivedTypeBase {
   }
 
 public:
-  DEFINE_MDNODE_GET(MDDerivedType,
+  DEFINE_MDNODE_GET(DIDerivedType,
                     (unsigned Tag, MDString *Name, Metadata *File,
                      unsigned Line, Metadata *Scope, Metadata *BaseType,
                      uint64_t SizeInBits, uint64_t AlignInBits,
@@ -726,15 +699,19 @@ class MDDerivedType : public MDDerivedTypeBase {
                      Metadata *ExtraData = nullptr),
                     (Tag, Name, File, Line, Scope, BaseType, SizeInBits,
                      AlignInBits, OffsetInBits, Flags, ExtraData))
-  DEFINE_MDNODE_GET(MDDerivedType,
-                    (unsigned Tag, StringRef Name, MDFile *File, unsigned Line,
-                     MDScopeRef Scope, MDTypeRef BaseType, uint64_t SizeInBits,
+  DEFINE_MDNODE_GET(DIDerivedType,
+                    (unsigned Tag, StringRef Name, DIFile *File, unsigned Line,
+                     DIScopeRef Scope, DITypeRef BaseType, uint64_t SizeInBits,
                      uint64_t AlignInBits, uint64_t OffsetInBits,
                      unsigned Flags, Metadata *ExtraData = nullptr),
                     (Tag, Name, File, Line, Scope, BaseType, SizeInBits,
                      AlignInBits, OffsetInBits, Flags, ExtraData))
 
-  TempMDDerivedType clone() const { return cloneImpl(); }
+  TempDIDerivedType clone() const { return cloneImpl(); }
+
+  //// Get the base type this is derived from.
+  DITypeRef getBaseType() const { return DITypeRef(getRawBaseType()); }
+  Metadata *getRawBaseType() const { return getOperand(3); }
 
   /// \brief Get extra data associated with this derived type.
   ///
@@ -748,12 +725,12 @@ class MDDerivedType : public MDDerivedTypeBase {
 
   /// \brief Get casted version of extra data.
   /// @{
-  MDTypeRef getClassType() const {
+  DITypeRef getClassType() const {
     assert(getTag() == dwarf::DW_TAG_ptr_to_member_type);
-    return MDTypeRef(getExtraData());
+    return DITypeRef(getExtraData());
   }
-  MDObjCProperty *getObjCProperty() const {
-    return dyn_cast_or_null<MDObjCProperty>(getExtraData());
+  DIObjCProperty *getObjCProperty() const {
+    return dyn_cast_or_null<DIObjCProperty>(getExtraData());
   }
   Constant *getConstant() const {
     assert(getTag() == dwarf::DW_TAG_member && isStaticMember());
@@ -764,74 +741,7 @@ class MDDerivedType : public MDDerivedTypeBase {
   /// @}
 
   static bool classof(const Metadata *MD) {
-    return MD->getMetadataID() == MDDerivedTypeKind;
-  }
-};
-
-/// \brief Base class for MDCompositeType and MDSubroutineType.
-///
-/// TODO: Delete; they're not really related.
-class MDCompositeTypeBase : public MDDerivedTypeBase {
-  unsigned RuntimeLang;
-
-protected:
-  MDCompositeTypeBase(LLVMContext &C, unsigned ID, StorageType Storage,
-                      unsigned Tag, unsigned Line, unsigned RuntimeLang,
-                      uint64_t SizeInBits, uint64_t AlignInBits,
-                      uint64_t OffsetInBits, unsigned Flags,
-                      ArrayRef<Metadata *> Ops)
-      : MDDerivedTypeBase(C, ID, Storage, Tag, Line, SizeInBits, AlignInBits,
-                          OffsetInBits, Flags, Ops),
-        RuntimeLang(RuntimeLang) {}
-  ~MDCompositeTypeBase() = default;
-
-public:
-  /// \brief Get the elements of the composite type.
-  ///
-  /// \note Calling this is only valid for \a MDCompositeType.  This assertion
-  /// can be removed once \a MDSubroutineType has been separated from
-  /// "composite types".
-  DebugNodeArray getElements() const {
-    assert(!isa<MDSubroutineType>(this) && "no elements for DISubroutineType");
-    return cast_or_null<MDTuple>(getRawElements());
-  }
-  MDTypeRef getVTableHolder() const { return MDTypeRef(getRawVTableHolder()); }
-  MDTemplateParameterArray getTemplateParams() const {
-    return cast_or_null<MDTuple>(getRawTemplateParams());
-  }
-  StringRef getIdentifier() const { return getStringOperand(7); }
-  unsigned getRuntimeLang() const { return RuntimeLang; }
-
-  Metadata *getRawElements() const { return getOperand(4); }
-  Metadata *getRawVTableHolder() const { return getOperand(5); }
-  Metadata *getRawTemplateParams() const { return getOperand(6); }
-  MDString *getRawIdentifier() const { return getOperandAs<MDString>(7); }
-
-  /// \brief Replace operands.
-  ///
-  /// If this \a isUniqued() and not \a isResolved(), on a uniquing collision
-  /// this will be RAUW'ed and deleted.  Use a \a TrackingMDRef to keep track
-  /// of its movement if necessary.
-  /// @{
-  void replaceElements(DebugNodeArray Elements) {
-#ifndef NDEBUG
-    for (DebugNode *Op : getElements())
-      assert(std::find(Elements->op_begin(), Elements->op_end(), Op) &&
-             "Lost a member during member list replacement");
-#endif
-    replaceOperandWith(4, Elements.get());
-  }
-  void replaceVTableHolder(MDTypeRef VTableHolder) {
-    replaceOperandWith(5, VTableHolder);
-  }
-  void replaceTemplateParams(MDTemplateParameterArray TemplateParams) {
-    replaceOperandWith(6, TemplateParams.get());
-  }
-  /// @}
-
-  static bool classof(const Metadata *MD) {
-    return MD->getMetadataID() == MDCompositeTypeKind ||
-           MD->getMetadataID() == MDSubroutineTypeKind;
+    return MD->getMetadataID() == DIDerivedTypeKind;
   }
 };
 
@@ -839,25 +749,27 @@ class MDCompositeTypeBase : public MDDerivedTypeBase {
 ///
 /// TODO: Detach from DerivedTypeBase (split out MDEnumType?).
 /// TODO: Create a custom, unrelated node for DW_TAG_array_type.
-class MDCompositeType : public MDCompositeTypeBase {
+class DICompositeType : public DIType {
   friend class LLVMContextImpl;
   friend class MDNode;
 
-  MDCompositeType(LLVMContext &C, StorageType Storage, unsigned Tag,
+  unsigned RuntimeLang;
+
+  DICompositeType(LLVMContext &C, StorageType Storage, unsigned Tag,
                   unsigned Line, unsigned RuntimeLang, uint64_t SizeInBits,
                   uint64_t AlignInBits, uint64_t OffsetInBits, unsigned Flags,
                   ArrayRef<Metadata *> Ops)
-      : MDCompositeTypeBase(C, MDCompositeTypeKind, Storage, Tag, Line,
-                            RuntimeLang, SizeInBits, AlignInBits, OffsetInBits,
-                            Flags, Ops) {}
-  ~MDCompositeType() = default;
+      : DIType(C, DICompositeTypeKind, Storage, Tag, Line, SizeInBits,
+               AlignInBits, OffsetInBits, Flags, Ops),
+        RuntimeLang(RuntimeLang) {}
+  ~DICompositeType() = default;
 
-  static MDCompositeType *
+  static DICompositeType *
   getImpl(LLVMContext &Context, unsigned Tag, StringRef Name, Metadata *File,
-          unsigned Line, MDScopeRef Scope, MDTypeRef BaseType,
+          unsigned Line, DIScopeRef Scope, DITypeRef BaseType,
           uint64_t SizeInBits, uint64_t AlignInBits, uint64_t OffsetInBits,
-          uint64_t Flags, DebugNodeArray Elements, unsigned RuntimeLang,
-          MDTypeRef VTableHolder, MDTemplateParameterArray TemplateParams,
+          uint64_t Flags, DINodeArray Elements, unsigned RuntimeLang,
+          DITypeRef VTableHolder, DITemplateParameterArray TemplateParams,
           StringRef Identifier, StorageType Storage, bool ShouldCreate = true) {
     return getImpl(
         Context, Tag, getCanonicalMDString(Context, Name), File, Line, Scope,
@@ -865,7 +777,7 @@ class MDCompositeType : public MDCompositeTypeBase {
         RuntimeLang, VTableHolder, TemplateParams.get(),
         getCanonicalMDString(Context, Identifier), Storage, ShouldCreate);
   }
-  static MDCompositeType *
+  static DICompositeType *
   getImpl(LLVMContext &Context, unsigned Tag, MDString *Name, Metadata *File,
           unsigned Line, Metadata *Scope, Metadata *BaseType,
           uint64_t SizeInBits, uint64_t AlignInBits, uint64_t OffsetInBits,
@@ -873,7 +785,7 @@ class MDCompositeType : public MDCompositeTypeBase {
           Metadata *VTableHolder, Metadata *TemplateParams,
           MDString *Identifier, StorageType Storage, bool ShouldCreate = true);
 
-  TempMDCompositeType cloneImpl() const {
+  TempDICompositeType cloneImpl() const {
     return getTemporary(getContext(), getTag(), getName(), getFile(), getLine(),
                         getScope(), getBaseType(), getSizeInBits(),
                         getAlignInBits(), getOffsetInBits(), getFlags(),
@@ -882,18 +794,18 @@ class MDCompositeType : public MDCompositeTypeBase {
   }
 
 public:
-  DEFINE_MDNODE_GET(MDCompositeType,
-                    (unsigned Tag, StringRef Name, MDFile *File, unsigned Line,
-                     MDScopeRef Scope, MDTypeRef BaseType, uint64_t SizeInBits,
+  DEFINE_MDNODE_GET(DICompositeType,
+                    (unsigned Tag, StringRef Name, DIFile *File, unsigned Line,
+                     DIScopeRef Scope, DITypeRef BaseType, uint64_t SizeInBits,
                      uint64_t AlignInBits, uint64_t OffsetInBits,
-                     unsigned Flags, DebugNodeArray Elements,
-                     unsigned RuntimeLang, MDTypeRef VTableHolder,
-                     MDTemplateParameterArray TemplateParams = nullptr,
+                     unsigned Flags, DINodeArray Elements, unsigned RuntimeLang,
+                     DITypeRef VTableHolder,
+                     DITemplateParameterArray TemplateParams = nullptr,
                      StringRef Identifier = ""),
                     (Tag, Name, File, Line, Scope, BaseType, SizeInBits,
                      AlignInBits, OffsetInBits, Flags, Elements, RuntimeLang,
                      VTableHolder, TemplateParams, Identifier))
-  DEFINE_MDNODE_GET(MDCompositeType,
+  DEFINE_MDNODE_GET(DICompositeType,
                     (unsigned Tag, MDString *Name, Metadata *File,
                      unsigned Line, Metadata *Scope, Metadata *BaseType,
                      uint64_t SizeInBits, uint64_t AlignInBits,
@@ -905,71 +817,108 @@ class MDCompositeType : public MDCompositeTypeBase {
                      AlignInBits, OffsetInBits, Flags, Elements, RuntimeLang,
                      VTableHolder, TemplateParams, Identifier))
 
-  TempMDCompositeType clone() const { return cloneImpl(); }
+  TempDICompositeType clone() const { return cloneImpl(); }
+
+  DITypeRef getBaseType() const { return DITypeRef(getRawBaseType()); }
+  DINodeArray getElements() const {
+    return cast_or_null<MDTuple>(getRawElements());
+  }
+  DITypeRef getVTableHolder() const { return DITypeRef(getRawVTableHolder()); }
+  DITemplateParameterArray getTemplateParams() const {
+    return cast_or_null<MDTuple>(getRawTemplateParams());
+  }
+  StringRef getIdentifier() const { return getStringOperand(7); }
+  unsigned getRuntimeLang() const { return RuntimeLang; }
+
+  Metadata *getRawBaseType() const { return getOperand(3); }
+  Metadata *getRawElements() const { return getOperand(4); }
+  Metadata *getRawVTableHolder() const { return getOperand(5); }
+  Metadata *getRawTemplateParams() const { return getOperand(6); }
+  MDString *getRawIdentifier() const { return getOperandAs<MDString>(7); }
+
+  /// \brief Replace operands.
+  ///
+  /// If this \a isUniqued() and not \a isResolved(), on a uniquing collision
+  /// this will be RAUW'ed and deleted.  Use a \a TrackingMDRef to keep track
+  /// of its movement if necessary.
+  /// @{
+  void replaceElements(DINodeArray Elements) {
+#ifndef NDEBUG
+    for (DINode *Op : getElements())
+      assert(std::find(Elements->op_begin(), Elements->op_end(), Op) &&
+             "Lost a member during member list replacement");
+#endif
+    replaceOperandWith(4, Elements.get());
+  }
+  void replaceVTableHolder(DITypeRef VTableHolder) {
+    replaceOperandWith(5, VTableHolder);
+  }
+  void replaceTemplateParams(DITemplateParameterArray TemplateParams) {
+    replaceOperandWith(6, TemplateParams.get());
+  }
+  /// @}
 
   static bool classof(const Metadata *MD) {
-    return MD->getMetadataID() == MDCompositeTypeKind;
+    return MD->getMetadataID() == DICompositeTypeKind;
   }
 };
 
-template <class T> TypedDebugNodeRef<T> TypedDebugNodeRef<T>::get(const T *N) {
+template <class T> TypedDINodeRef<T> TypedDINodeRef<T>::get(const T *N) {
   if (N)
-    if (auto *Composite = dyn_cast<MDCompositeType>(N))
+    if (auto *Composite = dyn_cast<DICompositeType>(N))
       if (auto *S = Composite->getRawIdentifier())
-        return TypedDebugNodeRef<T>(S);
-  return TypedDebugNodeRef<T>(N);
+        return TypedDINodeRef<T>(S);
+  return TypedDINodeRef<T>(N);
 }
 
 /// \brief Type array for a subprogram.
 ///
-/// TODO: Detach from CompositeType, and fold the array of types in directly
-/// as operands.
-class MDSubroutineType : public MDCompositeTypeBase {
+/// TODO: Fold the array of types in directly as operands.
+class DISubroutineType : public DIType {
   friend class LLVMContextImpl;
   friend class MDNode;
 
-  MDSubroutineType(LLVMContext &C, StorageType Storage, unsigned Flags,
+  DISubroutineType(LLVMContext &C, StorageType Storage, unsigned Flags,
                    ArrayRef<Metadata *> Ops)
-      : MDCompositeTypeBase(C, MDSubroutineTypeKind, Storage,
-                            dwarf::DW_TAG_subroutine_type, 0, 0, 0, 0, 0, Flags,
-                            Ops) {}
-  ~MDSubroutineType() = default;
+      : DIType(C, DISubroutineTypeKind, Storage, dwarf::DW_TAG_subroutine_type,
+               0, 0, 0, 0, Flags, Ops) {}
+  ~DISubroutineType() = default;
 
-  static MDSubroutineType *getImpl(LLVMContext &Context, unsigned Flags,
-                                   MDTypeRefArray TypeArray,
+  static DISubroutineType *getImpl(LLVMContext &Context, unsigned Flags,
+                                   DITypeRefArray TypeArray,
                                    StorageType Storage,
                                    bool ShouldCreate = true) {
     return getImpl(Context, Flags, TypeArray.get(), Storage, ShouldCreate);
   }
-  static MDSubroutineType *getImpl(LLVMContext &Context, unsigned Flags,
+  static DISubroutineType *getImpl(LLVMContext &Context, unsigned Flags,
                                    Metadata *TypeArray, StorageType Storage,
                                    bool ShouldCreate = true);
 
-  TempMDSubroutineType cloneImpl() const {
+  TempDISubroutineType cloneImpl() const {
     return getTemporary(getContext(), getFlags(), getTypeArray());
   }
 
 public:
-  DEFINE_MDNODE_GET(MDSubroutineType,
-                    (unsigned Flags, MDTypeRefArray TypeArray),
+  DEFINE_MDNODE_GET(DISubroutineType,
+                    (unsigned Flags, DITypeRefArray TypeArray),
                     (Flags, TypeArray))
-  DEFINE_MDNODE_GET(MDSubroutineType, (unsigned Flags, Metadata *TypeArray),
+  DEFINE_MDNODE_GET(DISubroutineType, (unsigned Flags, Metadata *TypeArray),
                     (Flags, TypeArray))
 
-  TempMDSubroutineType clone() const { return cloneImpl(); }
+  TempDISubroutineType clone() const { return cloneImpl(); }
 
-  MDTypeRefArray getTypeArray() const {
+  DITypeRefArray getTypeArray() const {
     return cast_or_null<MDTuple>(getRawTypeArray());
   }
-  Metadata *getRawTypeArray() const { return getRawElements(); }
+  Metadata *getRawTypeArray() const { return getOperand(3); }
 
   static bool classof(const Metadata *MD) {
-    return MD->getMetadataID() == MDSubroutineTypeKind;
+    return MD->getMetadataID() == DISubroutineTypeKind;
   }
 };
 
 /// \brief Compile unit.
-class MDCompileUnit : public MDScope {
+class DICompileUnit : public DIScope {
   friend class LLVMContextImpl;
   friend class MDNode;
 
@@ -977,74 +926,76 @@ class MDCompileUnit : public MDScope {
   bool IsOptimized;
   unsigned RuntimeVersion;
   unsigned EmissionKind;
+  uint64_t DWOId;
 
-  MDCompileUnit(LLVMContext &C, StorageType Storage, unsigned SourceLanguage,
+  DICompileUnit(LLVMContext &C, StorageType Storage, unsigned SourceLanguage,
                 bool IsOptimized, unsigned RuntimeVersion,
-                unsigned EmissionKind, ArrayRef<Metadata *> Ops)
-      : MDScope(C, MDCompileUnitKind, Storage, dwarf::DW_TAG_compile_unit, Ops),
+                unsigned EmissionKind, uint64_t DWOId, ArrayRef<Metadata *> Ops)
+      : DIScope(C, DICompileUnitKind, Storage, dwarf::DW_TAG_compile_unit, Ops),
         SourceLanguage(SourceLanguage), IsOptimized(IsOptimized),
-        RuntimeVersion(RuntimeVersion), EmissionKind(EmissionKind) {}
-  ~MDCompileUnit() = default;
+        RuntimeVersion(RuntimeVersion), EmissionKind(EmissionKind),
+        DWOId(DWOId) {}
+  ~DICompileUnit() = default;
 
-  static MDCompileUnit *
-  getImpl(LLVMContext &Context, unsigned SourceLanguage, MDFile *File,
+  static DICompileUnit *
+  getImpl(LLVMContext &Context, unsigned SourceLanguage, DIFile *File,
           StringRef Producer, bool IsOptimized, StringRef Flags,
           unsigned RuntimeVersion, StringRef SplitDebugFilename,
-          unsigned EmissionKind, MDCompositeTypeArray EnumTypes,
-          MDTypeArray RetainedTypes, MDSubprogramArray Subprograms,
-          MDGlobalVariableArray GlobalVariables,
-          MDImportedEntityArray ImportedEntities, StorageType Storage,
-          bool ShouldCreate = true) {
-    return getImpl(
-        Context, SourceLanguage, File, getCanonicalMDString(Context, Producer),
-        IsOptimized, getCanonicalMDString(Context, Flags), RuntimeVersion,
-        getCanonicalMDString(Context, SplitDebugFilename), EmissionKind,
-        EnumTypes.get(), RetainedTypes.get(), Subprograms.get(),
-        GlobalVariables.get(), ImportedEntities.get(), Storage, ShouldCreate);
-  }
-  static MDCompileUnit *
+          unsigned EmissionKind, DICompositeTypeArray EnumTypes,
+          DITypeArray RetainedTypes, DISubprogramArray Subprograms,
+          DIGlobalVariableArray GlobalVariables,
+          DIImportedEntityArray ImportedEntities, uint64_t DWOId,
+          StorageType Storage, bool ShouldCreate = true) {
+    return getImpl(Context, SourceLanguage, File,
+                   getCanonicalMDString(Context, Producer), IsOptimized,
+                   getCanonicalMDString(Context, Flags), RuntimeVersion,
+                   getCanonicalMDString(Context, SplitDebugFilename),
+                   EmissionKind, EnumTypes.get(), RetainedTypes.get(),
+                   Subprograms.get(), GlobalVariables.get(),
+                   ImportedEntities.get(), DWOId, Storage, ShouldCreate);
+  }
+  static DICompileUnit *
   getImpl(LLVMContext &Context, unsigned SourceLanguage, Metadata *File,
           MDString *Producer, bool IsOptimized, MDString *Flags,
           unsigned RuntimeVersion, MDString *SplitDebugFilename,
           unsigned EmissionKind, Metadata *EnumTypes, Metadata *RetainedTypes,
           Metadata *Subprograms, Metadata *GlobalVariables,
-          Metadata *ImportedEntities, StorageType Storage,
+          Metadata *ImportedEntities, uint64_t DWOId, StorageType Storage,
           bool ShouldCreate = true);
 
-  TempMDCompileUnit cloneImpl() const {
+  TempDICompileUnit cloneImpl() const {
     return getTemporary(
         getContext(), getSourceLanguage(), getFile(), getProducer(),
         isOptimized(), getFlags(), getRuntimeVersion(), getSplitDebugFilename(),
         getEmissionKind(), getEnumTypes(), getRetainedTypes(), getSubprograms(),
-        getGlobalVariables(), getImportedEntities());
+        getGlobalVariables(), getImportedEntities(), DWOId);
   }
 
 public:
-  DEFINE_MDNODE_GET(MDCompileUnit,
-                    (unsigned SourceLanguage, MDFile *File, StringRef Producer,
+  DEFINE_MDNODE_GET(DICompileUnit,
+                    (unsigned SourceLanguage, DIFile *File, StringRef Producer,
                      bool IsOptimized, StringRef Flags, unsigned RuntimeVersion,
                      StringRef SplitDebugFilename, unsigned EmissionKind,
-                     MDCompositeTypeArray EnumTypes, MDTypeArray RetainedTypes,
-                     MDSubprogramArray Subprograms,
-                     MDGlobalVariableArray GlobalVariables,
-                     MDImportedEntityArray ImportedEntities),
+                     DICompositeTypeArray EnumTypes, DITypeArray RetainedTypes,
+                     DISubprogramArray Subprograms,
+                     DIGlobalVariableArray GlobalVariables,
+                     DIImportedEntityArray ImportedEntities, uint64_t DWOId),
                     (SourceLanguage, File, Producer, IsOptimized, Flags,
                      RuntimeVersion, SplitDebugFilename, EmissionKind,
                      EnumTypes, RetainedTypes, Subprograms, GlobalVariables,
-                     ImportedEntities))
-  DEFINE_MDNODE_GET(MDCompileUnit,
-                    (unsigned SourceLanguage, Metadata *File,
-                     MDString *Producer, bool IsOptimized, MDString *Flags,
-                     unsigned RuntimeVersion, MDString *SplitDebugFilename,
-                     unsigned EmissionKind, Metadata *EnumTypes,
-                     Metadata *RetainedTypes, Metadata *Subprograms,
-                     Metadata *GlobalVariables, Metadata *ImportedEntities),
-                    (SourceLanguage, File, Producer, IsOptimized, Flags,
-                     RuntimeVersion, SplitDebugFilename, EmissionKind,
-                     EnumTypes, RetainedTypes, Subprograms, GlobalVariables,
-                     ImportedEntities))
-
-  TempMDCompileUnit clone() const { return cloneImpl(); }
+                     ImportedEntities, DWOId))
+  DEFINE_MDNODE_GET(
+      DICompileUnit,
+      (unsigned SourceLanguage, Metadata *File, MDString *Producer,
+       bool IsOptimized, MDString *Flags, unsigned RuntimeVersion,
+       MDString *SplitDebugFilename, unsigned EmissionKind, Metadata *EnumTypes,
+       Metadata *RetainedTypes, Metadata *Subprograms,
+       Metadata *GlobalVariables, Metadata *ImportedEntities, uint64_t DWOId),
+      (SourceLanguage, File, Producer, IsOptimized, Flags, RuntimeVersion,
+       SplitDebugFilename, EmissionKind, EnumTypes, RetainedTypes, Subprograms,
+       GlobalVariables, ImportedEntities, DWOId))
+
+  TempDICompileUnit clone() const { return cloneImpl(); }
 
   unsigned getSourceLanguage() const { return SourceLanguage; }
   bool isOptimized() const { return IsOptimized; }
@@ -1053,21 +1004,22 @@ class MDCompileUnit : public MDScope {
   StringRef getProducer() const { return getStringOperand(1); }
   StringRef getFlags() const { return getStringOperand(2); }
   StringRef getSplitDebugFilename() const { return getStringOperand(3); }
-  MDCompositeTypeArray getEnumTypes() const {
+  DICompositeTypeArray getEnumTypes() const {
     return cast_or_null<MDTuple>(getRawEnumTypes());
   }
-  MDTypeArray getRetainedTypes() const {
+  DITypeArray getRetainedTypes() const {
     return cast_or_null<MDTuple>(getRawRetainedTypes());
   }
-  MDSubprogramArray getSubprograms() const {
+  DISubprogramArray getSubprograms() const {
     return cast_or_null<MDTuple>(getRawSubprograms());
   }
-  MDGlobalVariableArray getGlobalVariables() const {
+  DIGlobalVariableArray getGlobalVariables() const {
     return cast_or_null<MDTuple>(getRawGlobalVariables());
   }
-  MDImportedEntityArray getImportedEntities() const {
+  DIImportedEntityArray getImportedEntities() const {
     return cast_or_null<MDTuple>(getRawImportedEntities());
   }
+  unsigned getDWOId() const { return DWOId; }
 
   MDString *getRawProducer() const { return getOperandAs<MDString>(1); }
   MDString *getRawFlags() const { return getOperandAs<MDString>(2); }
@@ -1084,71 +1036,80 @@ class MDCompileUnit : public MDScope {
   ///
   /// If this \a isUniqued() and not \a isResolved(), it will be RAUW'ed and
   /// deleted on a uniquing collision.  In practice, uniquing collisions on \a
-  /// MDCompileUnit should be fairly rare.
+  /// DICompileUnit should be fairly rare.
   /// @{
-  void replaceSubprograms(MDSubprogramArray N) {
+  void replaceEnumTypes(DICompositeTypeArray N) {
+    replaceOperandWith(4, N.get());
+  }
+  void replaceRetainedTypes(DITypeArray N) {
+    replaceOperandWith(5, N.get());
+  }
+  void replaceSubprograms(DISubprogramArray N) {
     replaceOperandWith(6, N.get());
   }
-  void replaceGlobalVariables(MDGlobalVariableArray N) {
+  void replaceGlobalVariables(DIGlobalVariableArray N) {
     replaceOperandWith(7, N.get());
   }
+  void replaceImportedEntities(DIImportedEntityArray N) {
+    replaceOperandWith(8, N.get());
+  }
   /// @}
 
   static bool classof(const Metadata *MD) {
-    return MD->getMetadataID() == MDCompileUnitKind;
+    return MD->getMetadataID() == DICompileUnitKind;
   }
 };
 
 /// \brief A scope for locals.
 ///
 /// A legal scope for lexical blocks, local variables, and debug info
-/// locations.  Subclasses are \a MDSubprogram, \a MDLexicalBlock, and \a
-/// MDLexicalBlockFile.
-class MDLocalScope : public MDScope {
+/// locations.  Subclasses are \a DISubprogram, \a DILexicalBlock, and \a
+/// DILexicalBlockFile.
+class DILocalScope : public DIScope {
 protected:
-  MDLocalScope(LLVMContext &C, unsigned ID, StorageType Storage, unsigned Tag,
+  DILocalScope(LLVMContext &C, unsigned ID, StorageType Storage, unsigned Tag,
                ArrayRef<Metadata *> Ops)
-      : MDScope(C, ID, Storage, Tag, Ops) {}
-  ~MDLocalScope() = default;
+      : DIScope(C, ID, Storage, Tag, Ops) {}
+  ~DILocalScope() = default;
 
 public:
   /// \brief Get the subprogram for this scope.
   ///
-  /// Return this if it's an \a MDSubprogram; otherwise, look up the scope
+  /// Return this if it's an \a DISubprogram; otherwise, look up the scope
   /// chain.
-  MDSubprogram *getSubprogram() const;
+  DISubprogram *getSubprogram() const;
 
   static bool classof(const Metadata *MD) {
-    return MD->getMetadataID() == MDSubprogramKind ||
-           MD->getMetadataID() == MDLexicalBlockKind ||
-           MD->getMetadataID() == MDLexicalBlockFileKind;
+    return MD->getMetadataID() == DISubprogramKind ||
+           MD->getMetadataID() == DILexicalBlockKind ||
+           MD->getMetadataID() == DILexicalBlockFileKind;
   }
 };
 
 /// \brief Debug location.
 ///
 /// A debug location in source code, used for debug info and otherwise.
-class MDLocation : public MDNode {
+class DILocation : public MDNode {
   friend class LLVMContextImpl;
   friend class MDNode;
 
-  MDLocation(LLVMContext &C, StorageType Storage, unsigned Line,
+  DILocation(LLVMContext &C, StorageType Storage, unsigned Line,
              unsigned Column, ArrayRef<Metadata *> MDs);
-  ~MDLocation() { dropAllReferences(); }
+  ~DILocation() { dropAllReferences(); }
 
-  static MDLocation *getImpl(LLVMContext &Context, unsigned Line,
+  static DILocation *getImpl(LLVMContext &Context, unsigned Line,
                              unsigned Column, Metadata *Scope,
                              Metadata *InlinedAt, StorageType Storage,
                              bool ShouldCreate = true);
-  static MDLocation *getImpl(LLVMContext &Context, unsigned Line,
-                             unsigned Column, MDLocalScope *Scope,
-                             MDLocation *InlinedAt, StorageType Storage,
+  static DILocation *getImpl(LLVMContext &Context, unsigned Line,
+                             unsigned Column, DILocalScope *Scope,
+                             DILocation *InlinedAt, StorageType Storage,
                              bool ShouldCreate = true) {
     return getImpl(Context, Line, Column, static_cast<Metadata *>(Scope),
                    static_cast<Metadata *>(InlinedAt), Storage, ShouldCreate);
   }
 
-  TempMDLocation cloneImpl() const {
+  TempDILocation cloneImpl() const {
     return getTemporary(getContext(), getLine(), getColumn(), getScope(),
                         getInlinedAt());
   }
@@ -1157,28 +1118,26 @@ class MDLocation : public MDNode {
   void replaceOperandWith(unsigned I, Metadata *New) = delete;
 
 public:
-  DEFINE_MDNODE_GET(MDLocation,
+  DEFINE_MDNODE_GET(DILocation,
                     (unsigned Line, unsigned Column, Metadata *Scope,
                      Metadata *InlinedAt = nullptr),
                     (Line, Column, Scope, InlinedAt))
-  DEFINE_MDNODE_GET(MDLocation,
-                    (unsigned Line, unsigned Column, MDLocalScope *Scope,
-                     MDLocation *InlinedAt = nullptr),
+  DEFINE_MDNODE_GET(DILocation,
+                    (unsigned Line, unsigned Column, DILocalScope *Scope,
+                     DILocation *InlinedAt = nullptr),
                     (Line, Column, Scope, InlinedAt))
 
   /// \brief Return a (temporary) clone of this.
-  TempMDLocation clone() const { return cloneImpl(); }
+  TempDILocation clone() const { return cloneImpl(); }
 
   unsigned getLine() const { return SubclassData32; }
   unsigned getColumn() const { return SubclassData16; }
-  MDLocalScope *getScope() const {
-    return cast<MDLocalScope>(getRawScope());
-  }
-  MDLocation *getInlinedAt() const {
-    return cast_or_null<MDLocation>(getRawInlinedAt());
+  DILocalScope *getScope() const { return cast<DILocalScope>(getRawScope()); }
+  DILocation *getInlinedAt() const {
+    return cast_or_null<DILocation>(getRawInlinedAt());
   }
 
-  MDFile *getFile() const { return getScope()->getFile(); }
+  DIFile *getFile() const { return getScope()->getFile(); }
   StringRef getFilename() const { return getScope()->getFilename(); }
   StringRef getDirectory() const { return getScope()->getDirectory(); }
 
@@ -1186,7 +1145,7 @@ class MDLocation : public MDNode {
   ///
   /// Walk through \a getInlinedAt() and return \a getScope() from the deepest
   /// location.
-  MDLocalScope *getInlinedAtScope() const {
+  DILocalScope *getInlinedAtScope() const {
     if (auto *IA = getInlinedAt())
       return IA->getInlinedAtScope();
     return getScope();
@@ -1205,7 +1164,7 @@ class MDLocation : public MDNode {
   /// FIXME: Add a check for getColumn().
   /// FIXME: Change the getFilename() check to getFile() (or add one for
   /// getDirectory()).
-  bool canDiscriminate(const MDLocation &RHS) const {
+  bool canDiscriminate(const DILocation &RHS) const {
     return getFilename() != RHS.getFilename() || getLine() != RHS.getLine();
   }
 
@@ -1232,7 +1191,7 @@ class MDLocation : public MDNode {
   }
 
   static bool classof(const Metadata *MD) {
-    return MD->getMetadataID() == MDLocationKind;
+    return MD->getMetadataID() == DILocationKind;
   }
 };
 
@@ -1240,7 +1199,7 @@ class MDLocation : public MDNode {
 ///
 /// TODO: Remove DisplayName.  It's always equal to Name.
 /// TODO: Split up flags.
-class MDSubprogram : public MDLocalScope {
+class DISubprogram : public DILocalScope {
   friend class LLVMContextImpl;
   friend class MDNode;
 
@@ -1253,25 +1212,25 @@ class MDSubprogram : public MDLocalScope {
   bool IsDefinition;
   bool IsOptimized;
 
-  MDSubprogram(LLVMContext &C, StorageType Storage, unsigned Line,
+  DISubprogram(LLVMContext &C, StorageType Storage, unsigned Line,
                unsigned ScopeLine, unsigned Virtuality, unsigned VirtualIndex,
                unsigned Flags, bool IsLocalToUnit, bool IsDefinition,
                bool IsOptimized, ArrayRef<Metadata *> Ops)
-      : MDLocalScope(C, MDSubprogramKind, Storage, dwarf::DW_TAG_subprogram,
+      : DILocalScope(C, DISubprogramKind, Storage, dwarf::DW_TAG_subprogram,
                      Ops),
         Line(Line), ScopeLine(ScopeLine), Virtuality(Virtuality),
         VirtualIndex(VirtualIndex), Flags(Flags), IsLocalToUnit(IsLocalToUnit),
         IsDefinition(IsDefinition), IsOptimized(IsOptimized) {}
-  ~MDSubprogram() = default;
+  ~DISubprogram() = default;
 
-  static MDSubprogram *
-  getImpl(LLVMContext &Context, MDScopeRef Scope, StringRef Name,
-          StringRef LinkageName, MDFile *File, unsigned Line,
-          MDSubroutineType *Type, bool IsLocalToUnit, bool IsDefinition,
-          unsigned ScopeLine, MDTypeRef ContainingType, unsigned Virtuality,
+  static DISubprogram *
+  getImpl(LLVMContext &Context, DIScopeRef Scope, StringRef Name,
+          StringRef LinkageName, DIFile *File, unsigned Line,
+          DISubroutineType *Type, bool IsLocalToUnit, bool IsDefinition,
+          unsigned ScopeLine, DITypeRef ContainingType, unsigned Virtuality,
           unsigned VirtualIndex, unsigned Flags, bool IsOptimized,
-          Constant *Function, MDTemplateParameterArray TemplateParams,
-          MDSubprogram *Declaration, MDLocalVariableArray Variables,
+          Constant *Function, DITemplateParameterArray TemplateParams,
+          DISubprogram *Declaration, DILocalVariableArray Variables,
           StorageType Storage, bool ShouldCreate = true) {
     return getImpl(Context, Scope, getCanonicalMDString(Context, Name),
                    getCanonicalMDString(Context, LinkageName), File, Line, Type,
@@ -1281,7 +1240,7 @@ class MDSubprogram : public MDLocalScope {
                    TemplateParams.get(), Declaration, Variables.get(), Storage,
                    ShouldCreate);
   }
-  static MDSubprogram *
+  static DISubprogram *
   getImpl(LLVMContext &Context, Metadata *Scope, MDString *Name,
           MDString *LinkageName, Metadata *File, unsigned Line, Metadata *Type,
           bool IsLocalToUnit, bool IsDefinition, unsigned ScopeLine,
@@ -1290,7 +1249,7 @@ class MDSubprogram : public MDLocalScope {
           Metadata *TemplateParams, Metadata *Declaration, Metadata *Variables,
           StorageType Storage, bool ShouldCreate = true);
 
-  TempMDSubprogram cloneImpl() const {
+  TempDISubprogram cloneImpl() const {
     return getTemporary(getContext(), getScope(), getName(), getLinkageName(),
                         getFile(), getLine(), getType(), isLocalToUnit(),
                         isDefinition(), getScopeLine(), getContainingType(),
@@ -1300,22 +1259,22 @@ class MDSubprogram : public MDLocalScope {
   }
 
 public:
-  DEFINE_MDNODE_GET(MDSubprogram,
-                    (MDScopeRef Scope, StringRef Name, StringRef LinkageName,
-                     MDFile *File, unsigned Line, MDSubroutineType *Type,
+  DEFINE_MDNODE_GET(DISubprogram,
+                    (DIScopeRef Scope, StringRef Name, StringRef LinkageName,
+                     DIFile *File, unsigned Line, DISubroutineType *Type,
                      bool IsLocalToUnit, bool IsDefinition, unsigned ScopeLine,
-                     MDTypeRef ContainingType, unsigned Virtuality,
+                     DITypeRef ContainingType, unsigned Virtuality,
                      unsigned VirtualIndex, unsigned Flags, bool IsOptimized,
                      Constant *Function = nullptr,
-                     MDTemplateParameterArray TemplateParams = nullptr,
-                     MDSubprogram *Declaration = nullptr,
-                     MDLocalVariableArray Variables = nullptr),
+                     DITemplateParameterArray TemplateParams = nullptr,
+                     DISubprogram *Declaration = nullptr,
+                     DILocalVariableArray Variables = nullptr),
                     (Scope, Name, LinkageName, File, Line, Type, IsLocalToUnit,
                      IsDefinition, ScopeLine, ContainingType, Virtuality,
                      VirtualIndex, Flags, IsOptimized, Function, TemplateParams,
                      Declaration, Variables))
   DEFINE_MDNODE_GET(
-      MDSubprogram,
+      DISubprogram,
       (Metadata * Scope, MDString *Name, MDString *LinkageName, Metadata *File,
        unsigned Line, Metadata *Type, bool IsLocalToUnit, bool IsDefinition,
        unsigned ScopeLine, Metadata *ContainingType, unsigned Virtuality,
@@ -1326,7 +1285,7 @@ class MDSubprogram : public MDLocalScope {
        ScopeLine, ContainingType, Virtuality, VirtualIndex, Flags, IsOptimized,
        Function, TemplateParams, Declaration, Variables))
 
-  TempMDSubprogram clone() const { return cloneImpl(); }
+  TempDISubprogram clone() const { return cloneImpl(); }
 
 public:
   unsigned getLine() const { return Line; }
@@ -1367,7 +1326,7 @@ class MDSubprogram : public MDLocalScope {
     return getFlags() & FlagRValueReference;
   }
 
-  MDScopeRef getScope() const { return MDScopeRef(getRawScope()); }
+  DIScopeRef getScope() const { return DIScopeRef(getRawScope()); }
 
   StringRef getName() const { return getStringOperand(2); }
   StringRef getDisplayName() const { return getStringOperand(3); }
@@ -1376,11 +1335,11 @@ class MDSubprogram : public MDLocalScope {
   MDString *getRawName() const { return getOperandAs<MDString>(2); }
   MDString *getRawLinkageName() const { return getOperandAs<MDString>(4); }
 
-  MDSubroutineType *getType() const {
-    return cast_or_null<MDSubroutineType>(getRawType());
+  DISubroutineType *getType() const {
+    return cast_or_null<DISubroutineType>(getRawType());
   }
-  MDTypeRef getContainingType() const {
-    return MDTypeRef(getRawContainingType());
+  DITypeRef getContainingType() const {
+    return DITypeRef(getRawContainingType());
   }
 
   Constant *getFunctionConstant() const {
@@ -1388,13 +1347,13 @@ class MDSubprogram : public MDLocalScope {
       return C->getValue();
     return nullptr;
   }
-  MDTemplateParameterArray getTemplateParams() const {
+  DITemplateParameterArray getTemplateParams() const {
     return cast_or_null<MDTuple>(getRawTemplateParams());
   }
-  MDSubprogram *getDeclaration() const {
-    return cast_or_null<MDSubprogram>(getRawDeclaration());
+  DISubprogram *getDeclaration() const {
+    return cast_or_null<DISubprogram>(getRawDeclaration());
   }
-  MDLocalVariableArray getVariables() const {
+  DILocalVariableArray getVariables() const {
     return cast_or_null<MDTuple>(getRawVariables());
   }
 
@@ -1430,50 +1389,50 @@ class MDSubprogram : public MDLocalScope {
   bool describes(const Function *F) const;
 
   static bool classof(const Metadata *MD) {
-    return MD->getMetadataID() == MDSubprogramKind;
+    return MD->getMetadataID() == DISubprogramKind;
   }
 };
 
-class MDLexicalBlockBase : public MDLocalScope {
+class DILexicalBlockBase : public DILocalScope {
 protected:
-  MDLexicalBlockBase(LLVMContext &C, unsigned ID, StorageType Storage,
+  DILexicalBlockBase(LLVMContext &C, unsigned ID, StorageType Storage,
                      ArrayRef<Metadata *> Ops)
-      : MDLocalScope(C, ID, Storage, dwarf::DW_TAG_lexical_block, Ops) {}
-  ~MDLexicalBlockBase() = default;
+      : DILocalScope(C, ID, Storage, dwarf::DW_TAG_lexical_block, Ops) {}
+  ~DILexicalBlockBase() = default;
 
 public:
-  MDLocalScope *getScope() const { return cast<MDLocalScope>(getRawScope()); }
+  DILocalScope *getScope() const { return cast<DILocalScope>(getRawScope()); }
 
   Metadata *getRawScope() const { return getOperand(1); }
 
   /// \brief Forwarding accessors to LexicalBlock.
   ///
-  /// TODO: Remove these and update code to use \a MDLexicalBlock directly.
+  /// TODO: Remove these and update code to use \a DILexicalBlock directly.
   /// @{
   inline unsigned getLine() const;
   inline unsigned getColumn() const;
   /// @}
   static bool classof(const Metadata *MD) {
-    return MD->getMetadataID() == MDLexicalBlockKind ||
-           MD->getMetadataID() == MDLexicalBlockFileKind;
+    return MD->getMetadataID() == DILexicalBlockKind ||
+           MD->getMetadataID() == DILexicalBlockFileKind;
   }
 };
 
-class MDLexicalBlock : public MDLexicalBlockBase {
+class DILexicalBlock : public DILexicalBlockBase {
   friend class LLVMContextImpl;
   friend class MDNode;
 
   unsigned Line;
   unsigned Column;
 
-  MDLexicalBlock(LLVMContext &C, StorageType Storage, unsigned Line,
+  DILexicalBlock(LLVMContext &C, StorageType Storage, unsigned Line,
                  unsigned Column, ArrayRef<Metadata *> Ops)
-      : MDLexicalBlockBase(C, MDLexicalBlockKind, Storage, Ops), Line(Line),
+      : DILexicalBlockBase(C, DILexicalBlockKind, Storage, Ops), Line(Line),
         Column(Column) {}
-  ~MDLexicalBlock() = default;
+  ~DILexicalBlock() = default;
 
-  static MDLexicalBlock *getImpl(LLVMContext &Context, MDLocalScope *Scope,
-                                 MDFile *File, unsigned Line, unsigned Column,
+  static DILexicalBlock *getImpl(LLVMContext &Context, DILocalScope *Scope,
+                                 DIFile *File, unsigned Line, unsigned Column,
                                  StorageType Storage,
                                  bool ShouldCreate = true) {
     return getImpl(Context, static_cast<Metadata *>(Scope),
@@ -1481,59 +1440,59 @@ class MDLexicalBlock : public MDLexicalBlockBase {
                    ShouldCreate);
   }
 
-  static MDLexicalBlock *getImpl(LLVMContext &Context, Metadata *Scope,
+  static DILexicalBlock *getImpl(LLVMContext &Context, Metadata *Scope,
                                  Metadata *File, unsigned Line, unsigned Column,
                                  StorageType Storage, bool ShouldCreate = true);
 
-  TempMDLexicalBlock cloneImpl() const {
+  TempDILexicalBlock cloneImpl() const {
     return getTemporary(getContext(), getScope(), getFile(), getLine(),
                         getColumn());
   }
 
 public:
-  DEFINE_MDNODE_GET(MDLexicalBlock, (MDLocalScope * Scope, MDFile *File,
+  DEFINE_MDNODE_GET(DILexicalBlock, (DILocalScope * Scope, DIFile *File,
                                      unsigned Line, unsigned Column),
                     (Scope, File, Line, Column))
-  DEFINE_MDNODE_GET(MDLexicalBlock, (Metadata * Scope, Metadata *File,
+  DEFINE_MDNODE_GET(DILexicalBlock, (Metadata * Scope, Metadata *File,
                                      unsigned Line, unsigned Column),
                     (Scope, File, Line, Column))
 
-  TempMDLexicalBlock clone() const { return cloneImpl(); }
+  TempDILexicalBlock clone() const { return cloneImpl(); }
 
   unsigned getLine() const { return Line; }
   unsigned getColumn() const { return Column; }
 
   static bool classof(const Metadata *MD) {
-    return MD->getMetadataID() == MDLexicalBlockKind;
+    return MD->getMetadataID() == DILexicalBlockKind;
   }
 };
 
-unsigned MDLexicalBlockBase::getLine() const {
-  if (auto *N = dyn_cast<MDLexicalBlock>(this))
+unsigned DILexicalBlockBase::getLine() const {
+  if (auto *N = dyn_cast<DILexicalBlock>(this))
     return N->getLine();
   return 0;
 }
 
-unsigned MDLexicalBlockBase::getColumn() const {
-  if (auto *N = dyn_cast<MDLexicalBlock>(this))
+unsigned DILexicalBlockBase::getColumn() const {
+  if (auto *N = dyn_cast<DILexicalBlock>(this))
     return N->getColumn();
   return 0;
 }
 
-class MDLexicalBlockFile : public MDLexicalBlockBase {
+class DILexicalBlockFile : public DILexicalBlockBase {
   friend class LLVMContextImpl;
   friend class MDNode;
 
   unsigned Discriminator;
 
-  MDLexicalBlockFile(LLVMContext &C, StorageType Storage,
+  DILexicalBlockFile(LLVMContext &C, StorageType Storage,
                      unsigned Discriminator, ArrayRef<Metadata *> Ops)
-      : MDLexicalBlockBase(C, MDLexicalBlockFileKind, Storage, Ops),
+      : DILexicalBlockBase(C, DILexicalBlockFileKind, Storage, Ops),
         Discriminator(Discriminator) {}
-  ~MDLexicalBlockFile() = default;
+  ~DILexicalBlockFile() = default;
 
-  static MDLexicalBlockFile *getImpl(LLVMContext &Context, MDLocalScope *Scope,
-                                     MDFile *File, unsigned Discriminator,
+  static DILexicalBlockFile *getImpl(LLVMContext &Context, DILocalScope *Scope,
+                                     DIFile *File, unsigned Discriminator,
                                      StorageType Storage,
                                      bool ShouldCreate = true) {
     return getImpl(Context, static_cast<Metadata *>(Scope),
@@ -1541,213 +1500,273 @@ class MDLexicalBlockFile : public MDLexicalBlockBase {
                    ShouldCreate);
   }
 
-  static MDLexicalBlockFile *getImpl(LLVMContext &Context, Metadata *Scope,
+  static DILexicalBlockFile *getImpl(LLVMContext &Context, Metadata *Scope,
                                      Metadata *File, unsigned Discriminator,
                                      StorageType Storage,
                                      bool ShouldCreate = true);
 
-  TempMDLexicalBlockFile cloneImpl() const {
+  TempDILexicalBlockFile cloneImpl() const {
     return getTemporary(getContext(), getScope(), getFile(),
                         getDiscriminator());
   }
 
 public:
-  DEFINE_MDNODE_GET(MDLexicalBlockFile, (MDLocalScope * Scope, MDFile *File,
+  DEFINE_MDNODE_GET(DILexicalBlockFile, (DILocalScope * Scope, DIFile *File,
                                          unsigned Discriminator),
                     (Scope, File, Discriminator))
-  DEFINE_MDNODE_GET(MDLexicalBlockFile,
+  DEFINE_MDNODE_GET(DILexicalBlockFile,
                     (Metadata * Scope, Metadata *File, unsigned Discriminator),
                     (Scope, File, Discriminator))
 
-  TempMDLexicalBlockFile clone() const { return cloneImpl(); }
+  TempDILexicalBlockFile clone() const { return cloneImpl(); }
 
-  // TODO: Remove these once they're gone from MDLexicalBlockBase.
+  // TODO: Remove these once they're gone from DILexicalBlockBase.
   unsigned getLine() const = delete;
   unsigned getColumn() const = delete;
 
   unsigned getDiscriminator() const { return Discriminator; }
 
   static bool classof(const Metadata *MD) {
-    return MD->getMetadataID() == MDLexicalBlockFileKind;
+    return MD->getMetadataID() == DILexicalBlockFileKind;
   }
 };
 
-unsigned MDLocation::getDiscriminator() const {
-  if (auto *F = dyn_cast<MDLexicalBlockFile>(getScope()))
+unsigned DILocation::getDiscriminator() const {
+  if (auto *F = dyn_cast<DILexicalBlockFile>(getScope()))
     return F->getDiscriminator();
   return 0;
 }
 
-class MDNamespace : public MDScope {
+class DINamespace : public DIScope {
   friend class LLVMContextImpl;
   friend class MDNode;
 
   unsigned Line;
 
-  MDNamespace(LLVMContext &Context, StorageType Storage, unsigned Line,
+  DINamespace(LLVMContext &Context, StorageType Storage, unsigned Line,
               ArrayRef<Metadata *> Ops)
-      : MDScope(Context, MDNamespaceKind, Storage, dwarf::DW_TAG_namespace,
+      : DIScope(Context, DINamespaceKind, Storage, dwarf::DW_TAG_namespace,
                 Ops),
         Line(Line) {}
-  ~MDNamespace() = default;
+  ~DINamespace() = default;
 
-  static MDNamespace *getImpl(LLVMContext &Context, MDScope *Scope,
-                              MDFile *File, StringRef Name, unsigned Line,
+  static DINamespace *getImpl(LLVMContext &Context, DIScope *Scope,
+                              DIFile *File, StringRef Name, unsigned Line,
                               StorageType Storage, bool ShouldCreate = true) {
     return getImpl(Context, Scope, File, getCanonicalMDString(Context, Name),
                    Line, Storage, ShouldCreate);
   }
-  static MDNamespace *getImpl(LLVMContext &Context, Metadata *Scope,
+  static DINamespace *getImpl(LLVMContext &Context, Metadata *Scope,
                               Metadata *File, MDString *Name, unsigned Line,
                               StorageType Storage, bool ShouldCreate = true);
 
-  TempMDNamespace cloneImpl() const {
+  TempDINamespace cloneImpl() const {
     return getTemporary(getContext(), getScope(), getFile(), getName(),
                         getLine());
   }
 
 public:
-  DEFINE_MDNODE_GET(MDNamespace, (MDScope * Scope, MDFile *File, StringRef Name,
+  DEFINE_MDNODE_GET(DINamespace, (DIScope * Scope, DIFile *File, StringRef Name,
                                   unsigned Line),
                     (Scope, File, Name, Line))
-  DEFINE_MDNODE_GET(MDNamespace, (Metadata * Scope, Metadata *File,
+  DEFINE_MDNODE_GET(DINamespace, (Metadata * Scope, Metadata *File,
                                   MDString *Name, unsigned Line),
                     (Scope, File, Name, Line))
 
-  TempMDNamespace clone() const { return cloneImpl(); }
+  TempDINamespace clone() const { return cloneImpl(); }
 
   unsigned getLine() const { return Line; }
-  MDScope *getScope() const { return cast_or_null<MDScope>(getRawScope()); }
+  DIScope *getScope() const { return cast_or_null<DIScope>(getRawScope()); }
   StringRef getName() const { return getStringOperand(2); }
 
   Metadata *getRawScope() const { return getOperand(1); }
   MDString *getRawName() const { return getOperandAs<MDString>(2); }
 
   static bool classof(const Metadata *MD) {
-    return MD->getMetadataID() == MDNamespaceKind;
+    return MD->getMetadataID() == DINamespaceKind;
+  }
+};
+
+/// \brief A (clang) module that has been imported by the compile unit.
+///
+class DIModule : public DIScope {
+  friend class LLVMContextImpl;
+  friend class MDNode;
+
+  DIModule(LLVMContext &Context, StorageType Storage, ArrayRef<Metadata *> Ops)
+      : DIScope(Context, DIModuleKind, Storage, dwarf::DW_TAG_module, Ops) {}
+  ~DIModule() {}
+
+  static DIModule *getImpl(LLVMContext &Context, DIScope *Scope,
+                           StringRef Name, StringRef ConfigurationMacros,
+                           StringRef IncludePath, StringRef ISysRoot,
+                           StorageType Storage, bool ShouldCreate = true) {
+    return getImpl(Context, Scope, getCanonicalMDString(Context, Name),
+                   getCanonicalMDString(Context, ConfigurationMacros),
+                   getCanonicalMDString(Context, IncludePath),
+                   getCanonicalMDString(Context, ISysRoot),
+                   Storage, ShouldCreate);
+  }
+  static DIModule *getImpl(LLVMContext &Context, Metadata *Scope,
+                           MDString *Name, MDString *ConfigurationMacros,
+                           MDString *IncludePath, MDString *ISysRoot,
+                           StorageType Storage, bool ShouldCreate = true);
+
+  TempDIModule cloneImpl() const {
+    return getTemporary(getContext(), getScope(), getName(),
+                        getConfigurationMacros(), getIncludePath(),
+                        getISysRoot());
+  }
+
+public:
+  DEFINE_MDNODE_GET(DIModule, (DIScope *Scope, StringRef Name,
+                               StringRef ConfigurationMacros, StringRef IncludePath,
+                               StringRef ISysRoot),
+                    (Scope, Name, ConfigurationMacros, IncludePath, ISysRoot))
+  DEFINE_MDNODE_GET(DIModule,
+                    (Metadata *Scope, MDString *Name, MDString *ConfigurationMacros,
+                     MDString *IncludePath, MDString *ISysRoot),
+                    (Scope, Name, ConfigurationMacros, IncludePath, ISysRoot))
+
+  TempDIModule clone() const { return cloneImpl(); }
+
+  DIScope *getScope() const { return cast_or_null<DIScope>(getRawScope()); }
+  StringRef getName() const { return getStringOperand(1); }
+  StringRef getConfigurationMacros() const { return getStringOperand(2); }
+  StringRef getIncludePath() const { return getStringOperand(3); }
+  StringRef getISysRoot() const { return getStringOperand(4); }
+
+  Metadata *getRawScope() const { return getOperand(0); }
+  MDString *getRawName() const { return getOperandAs<MDString>(1); }
+  MDString *getRawConfigurationMacros() const { return getOperandAs<MDString>(2); }
+  MDString *getRawIncludePath() const { return getOperandAs<MDString>(3); }
+  MDString *getRawISysRoot() const { return getOperandAs<MDString>(4); }
+
+  static bool classof(const Metadata *MD) {
+    return MD->getMetadataID() == DIModuleKind;
   }
 };
 
 /// \brief Base class for template parameters.
-class MDTemplateParameter : public DebugNode {
+class DITemplateParameter : public DINode {
 protected:
-  MDTemplateParameter(LLVMContext &Context, unsigned ID, StorageType Storage,
+  DITemplateParameter(LLVMContext &Context, unsigned ID, StorageType Storage,
                       unsigned Tag, ArrayRef<Metadata *> Ops)
-      : DebugNode(Context, ID, Storage, Tag, Ops) {}
-  ~MDTemplateParameter() = default;
+      : DINode(Context, ID, Storage, Tag, Ops) {}
+  ~DITemplateParameter() = default;
 
 public:
   StringRef getName() const { return getStringOperand(0); }
-  MDTypeRef getType() const { return MDTypeRef(getRawType()); }
+  DITypeRef getType() const { return DITypeRef(getRawType()); }
 
   MDString *getRawName() const { return getOperandAs<MDString>(0); }
   Metadata *getRawType() const { return getOperand(1); }
 
   static bool classof(const Metadata *MD) {
-    return MD->getMetadataID() == MDTemplateTypeParameterKind ||
-           MD->getMetadataID() == MDTemplateValueParameterKind;
+    return MD->getMetadataID() == DITemplateTypeParameterKind ||
+           MD->getMetadataID() == DITemplateValueParameterKind;
   }
 };
 
-class MDTemplateTypeParameter : public MDTemplateParameter {
+class DITemplateTypeParameter : public DITemplateParameter {
   friend class LLVMContextImpl;
   friend class MDNode;
 
-  MDTemplateTypeParameter(LLVMContext &Context, StorageType Storage,
+  DITemplateTypeParameter(LLVMContext &Context, StorageType Storage,
                           ArrayRef<Metadata *> Ops)
-      : MDTemplateParameter(Context, MDTemplateTypeParameterKind, Storage,
+      : DITemplateParameter(Context, DITemplateTypeParameterKind, Storage,
                             dwarf::DW_TAG_template_type_parameter, Ops) {}
-  ~MDTemplateTypeParameter() = default;
+  ~DITemplateTypeParameter() = default;
 
-  static MDTemplateTypeParameter *getImpl(LLVMContext &Context, StringRef Name,
-                                          MDTypeRef Type, StorageType Storage,
+  static DITemplateTypeParameter *getImpl(LLVMContext &Context, StringRef Name,
+                                          DITypeRef Type, StorageType Storage,
                                           bool ShouldCreate = true) {
     return getImpl(Context, getCanonicalMDString(Context, Name), Type, Storage,
                    ShouldCreate);
   }
-  static MDTemplateTypeParameter *getImpl(LLVMContext &Context, MDString *Name,
+  static DITemplateTypeParameter *getImpl(LLVMContext &Context, MDString *Name,
                                           Metadata *Type, StorageType Storage,
                                           bool ShouldCreate = true);
 
-  TempMDTemplateTypeParameter cloneImpl() const {
+  TempDITemplateTypeParameter cloneImpl() const {
     return getTemporary(getContext(), getName(), getType());
   }
 
 public:
-  DEFINE_MDNODE_GET(MDTemplateTypeParameter, (StringRef Name, MDTypeRef Type),
+  DEFINE_MDNODE_GET(DITemplateTypeParameter, (StringRef Name, DITypeRef Type),
                     (Name, Type))
-  DEFINE_MDNODE_GET(MDTemplateTypeParameter, (MDString * Name, Metadata *Type),
+  DEFINE_MDNODE_GET(DITemplateTypeParameter, (MDString * Name, Metadata *Type),
                     (Name, Type))
 
-  TempMDTemplateTypeParameter clone() const { return cloneImpl(); }
+  TempDITemplateTypeParameter clone() const { return cloneImpl(); }
 
   static bool classof(const Metadata *MD) {
-    return MD->getMetadataID() == MDTemplateTypeParameterKind;
+    return MD->getMetadataID() == DITemplateTypeParameterKind;
   }
 };
 
-class MDTemplateValueParameter : public MDTemplateParameter {
+class DITemplateValueParameter : public DITemplateParameter {
   friend class LLVMContextImpl;
   friend class MDNode;
 
-  MDTemplateValueParameter(LLVMContext &Context, StorageType Storage,
+  DITemplateValueParameter(LLVMContext &Context, StorageType Storage,
                            unsigned Tag, ArrayRef<Metadata *> Ops)
-      : MDTemplateParameter(Context, MDTemplateValueParameterKind, Storage, Tag,
+      : DITemplateParameter(Context, DITemplateValueParameterKind, Storage, Tag,
                             Ops) {}
-  ~MDTemplateValueParameter() = default;
+  ~DITemplateValueParameter() = default;
 
-  static MDTemplateValueParameter *getImpl(LLVMContext &Context, unsigned Tag,
-                                           StringRef Name, MDTypeRef Type,
+  static DITemplateValueParameter *getImpl(LLVMContext &Context, unsigned Tag,
+                                           StringRef Name, DITypeRef Type,
                                            Metadata *Value, StorageType Storage,
                                            bool ShouldCreate = true) {
     return getImpl(Context, Tag, getCanonicalMDString(Context, Name), Type,
                    Value, Storage, ShouldCreate);
   }
-  static MDTemplateValueParameter *getImpl(LLVMContext &Context, unsigned Tag,
+  static DITemplateValueParameter *getImpl(LLVMContext &Context, unsigned Tag,
                                            MDString *Name, Metadata *Type,
                                            Metadata *Value, StorageType Storage,
                                            bool ShouldCreate = true);
 
-  TempMDTemplateValueParameter cloneImpl() const {
+  TempDITemplateValueParameter cloneImpl() const {
     return getTemporary(getContext(), getTag(), getName(), getType(),
                         getValue());
   }
 
 public:
-  DEFINE_MDNODE_GET(MDTemplateValueParameter, (unsigned Tag, StringRef Name,
-                                               MDTypeRef Type, Metadata *Value),
+  DEFINE_MDNODE_GET(DITemplateValueParameter, (unsigned Tag, StringRef Name,
+                                               DITypeRef Type, Metadata *Value),
                     (Tag, Name, Type, Value))
-  DEFINE_MDNODE_GET(MDTemplateValueParameter, (unsigned Tag, MDString *Name,
+  DEFINE_MDNODE_GET(DITemplateValueParameter, (unsigned Tag, MDString *Name,
                                                Metadata *Type, Metadata *Value),
                     (Tag, Name, Type, Value))
 
-  TempMDTemplateValueParameter clone() const { return cloneImpl(); }
+  TempDITemplateValueParameter clone() const { return cloneImpl(); }
 
   Metadata *getValue() const { return getOperand(2); }
 
   static bool classof(const Metadata *MD) {
-    return MD->getMetadataID() == MDTemplateValueParameterKind;
+    return MD->getMetadataID() == DITemplateValueParameterKind;
   }
 };
 
 /// \brief Base class for variables.
 ///
 /// TODO: Hardcode to DW_TAG_variable.
-class MDVariable : public DebugNode {
+class DIVariable : public DINode {
   unsigned Line;
 
 protected:
-  MDVariable(LLVMContext &C, unsigned ID, StorageType Storage, unsigned Tag,
+  DIVariable(LLVMContext &C, unsigned ID, StorageType Storage, unsigned Tag,
              unsigned Line, ArrayRef<Metadata *> Ops)
-      : DebugNode(C, ID, Storage, Tag, Ops), Line(Line) {}
-  ~MDVariable() = default;
+      : DINode(C, ID, Storage, Tag, Ops), Line(Line) {}
+  ~DIVariable() = default;
 
 public:
   unsigned getLine() const { return Line; }
-  MDScope *getScope() const { return cast_or_null<MDScope>(getRawScope()); }
+  DIScope *getScope() const { return cast_or_null<DIScope>(getRawScope()); }
   StringRef getName() const { return getStringOperand(1); }
-  MDFile *getFile() const { return cast_or_null<MDFile>(getRawFile()); }
-  MDTypeRef getType() const { return MDTypeRef(getRawType()); }
+  DIFile *getFile() const { return cast_or_null<DIFile>(getRawFile()); }
+  DITypeRef getType() const { return DITypeRef(getRawType()); }
 
   StringRef getFilename() const {
     if (auto *F = getFile())
@@ -1766,34 +1785,34 @@ class MDVariable : public DebugNode {
   Metadata *getRawType() const { return getOperand(3); }
 
   static bool classof(const Metadata *MD) {
-    return MD->getMetadataID() == MDLocalVariableKind ||
-           MD->getMetadataID() == MDGlobalVariableKind;
+    return MD->getMetadataID() == DILocalVariableKind ||
+           MD->getMetadataID() == DIGlobalVariableKind;
   }
 };
 
 /// \brief Global variables.
 ///
 /// TODO: Remove DisplayName.  It's always equal to Name.
-class MDGlobalVariable : public MDVariable {
+class DIGlobalVariable : public DIVariable {
   friend class LLVMContextImpl;
   friend class MDNode;
 
   bool IsLocalToUnit;
   bool IsDefinition;
 
-  MDGlobalVariable(LLVMContext &C, StorageType Storage, unsigned Line,
+  DIGlobalVariable(LLVMContext &C, StorageType Storage, unsigned Line,
                    bool IsLocalToUnit, bool IsDefinition,
                    ArrayRef<Metadata *> Ops)
-      : MDVariable(C, MDGlobalVariableKind, Storage, dwarf::DW_TAG_variable,
+      : DIVariable(C, DIGlobalVariableKind, Storage, dwarf::DW_TAG_variable,
                    Line, Ops),
         IsLocalToUnit(IsLocalToUnit), IsDefinition(IsDefinition) {}
-  ~MDGlobalVariable() = default;
+  ~DIGlobalVariable() = default;
 
-  static MDGlobalVariable *
-  getImpl(LLVMContext &Context, MDScope *Scope, StringRef Name,
-          StringRef LinkageName, MDFile *File, unsigned Line, MDTypeRef Type,
+  static DIGlobalVariable *
+  getImpl(LLVMContext &Context, DIScope *Scope, StringRef Name,
+          StringRef LinkageName, DIFile *File, unsigned Line, DITypeRef Type,
           bool IsLocalToUnit, bool IsDefinition, Constant *Variable,
-          MDDerivedType *StaticDataMemberDeclaration, StorageType Storage,
+          DIDerivedType *StaticDataMemberDeclaration, StorageType Storage,
           bool ShouldCreate = true) {
     return getImpl(Context, Scope, getCanonicalMDString(Context, Name),
                    getCanonicalMDString(Context, LinkageName), File, Line, Type,
@@ -1801,14 +1820,14 @@ class MDGlobalVariable : public MDVariable {
                    Variable ? ConstantAsMetadata::get(Variable) : nullptr,
                    StaticDataMemberDeclaration, Storage, ShouldCreate);
   }
-  static MDGlobalVariable *
+  static DIGlobalVariable *
   getImpl(LLVMContext &Context, Metadata *Scope, MDString *Name,
           MDString *LinkageName, Metadata *File, unsigned Line, Metadata *Type,
           bool IsLocalToUnit, bool IsDefinition, Metadata *Variable,
           Metadata *StaticDataMemberDeclaration, StorageType Storage,
           bool ShouldCreate = true);
 
-  TempMDGlobalVariable cloneImpl() const {
+  TempDIGlobalVariable cloneImpl() const {
     return getTemporary(getContext(), getScope(), getName(), getLinkageName(),
                         getFile(), getLine(), getType(), isLocalToUnit(),
                         isDefinition(), getVariable(),
@@ -1816,14 +1835,14 @@ class MDGlobalVariable : public MDVariable {
   }
 
 public:
-  DEFINE_MDNODE_GET(MDGlobalVariable,
-                    (MDScope * Scope, StringRef Name, StringRef LinkageName,
-                     MDFile *File, unsigned Line, MDTypeRef Type,
+  DEFINE_MDNODE_GET(DIGlobalVariable,
+                    (DIScope * Scope, StringRef Name, StringRef LinkageName,
+                     DIFile *File, unsigned Line, DITypeRef Type,
                      bool IsLocalToUnit, bool IsDefinition, Constant *Variable,
-                     MDDerivedType *StaticDataMemberDeclaration),
+                     DIDerivedType *StaticDataMemberDeclaration),
                     (Scope, Name, LinkageName, File, Line, Type, IsLocalToUnit,
                      IsDefinition, Variable, StaticDataMemberDeclaration))
-  DEFINE_MDNODE_GET(MDGlobalVariable,
+  DEFINE_MDNODE_GET(DIGlobalVariable,
                     (Metadata * Scope, MDString *Name, MDString *LinkageName,
                      Metadata *File, unsigned Line, Metadata *Type,
                      bool IsLocalToUnit, bool IsDefinition, Metadata *Variable,
@@ -1831,7 +1850,7 @@ class MDGlobalVariable : public MDVariable {
                     (Scope, Name, LinkageName, File, Line, Type, IsLocalToUnit,
                      IsDefinition, Variable, StaticDataMemberDeclaration))
 
-  TempMDGlobalVariable clone() const { return cloneImpl(); }
+  TempDIGlobalVariable clone() const { return cloneImpl(); }
 
   bool isLocalToUnit() const { return IsLocalToUnit; }
   bool isDefinition() const { return IsDefinition; }
@@ -1842,8 +1861,8 @@ class MDGlobalVariable : public MDVariable {
       return dyn_cast<Constant>(C->getValue());
     return nullptr;
   }
-  MDDerivedType *getStaticDataMemberDeclaration() const {
-    return cast_or_null<MDDerivedType>(getRawStaticDataMemberDeclaration());
+  DIDerivedType *getStaticDataMemberDeclaration() const {
+    return cast_or_null<DIDerivedType>(getRawStaticDataMemberDeclaration());
   }
 
   MDString *getRawLinkageName() const { return getOperandAs<MDString>(5); }
@@ -1851,7 +1870,7 @@ class MDGlobalVariable : public MDVariable {
   Metadata *getRawStaticDataMemberDeclaration() const { return getOperand(7); }
 
   static bool classof(const Metadata *MD) {
-    return MD->getMetadataID() == MDGlobalVariableKind;
+    return MD->getMetadataID() == DIGlobalVariableKind;
   }
 };
 
@@ -1860,57 +1879,57 @@ class MDGlobalVariable : public MDVariable {
 /// TODO: Split between arguments and otherwise.
 /// TODO: Use \c DW_TAG_variable instead of fake tags.
 /// TODO: Split up flags.
-class MDLocalVariable : public MDVariable {
+class DILocalVariable : public DIVariable {
   friend class LLVMContextImpl;
   friend class MDNode;
 
   unsigned Arg;
   unsigned Flags;
 
-  MDLocalVariable(LLVMContext &C, StorageType Storage, unsigned Tag,
+  DILocalVariable(LLVMContext &C, StorageType Storage, unsigned Tag,
                   unsigned Line, unsigned Arg, unsigned Flags,
                   ArrayRef<Metadata *> Ops)
-      : MDVariable(C, MDLocalVariableKind, Storage, Tag, Line, Ops), Arg(Arg),
+      : DIVariable(C, DILocalVariableKind, Storage, Tag, Line, Ops), Arg(Arg),
         Flags(Flags) {}
-  ~MDLocalVariable() = default;
+  ~DILocalVariable() = default;
 
-  static MDLocalVariable *getImpl(LLVMContext &Context, unsigned Tag,
-                                  MDScope *Scope, StringRef Name, MDFile *File,
-                                  unsigned Line, MDTypeRef Type, unsigned Arg,
+  static DILocalVariable *getImpl(LLVMContext &Context, unsigned Tag,
+                                  DIScope *Scope, StringRef Name, DIFile *File,
+                                  unsigned Line, DITypeRef Type, unsigned Arg,
                                   unsigned Flags, StorageType Storage,
                                   bool ShouldCreate = true) {
     return getImpl(Context, Tag, Scope, getCanonicalMDString(Context, Name),
                    File, Line, Type, Arg, Flags, Storage, ShouldCreate);
   }
-  static MDLocalVariable *
+  static DILocalVariable *
   getImpl(LLVMContext &Context, unsigned Tag, Metadata *Scope, MDString *Name,
           Metadata *File, unsigned Line, Metadata *Type, unsigned Arg,
           unsigned Flags, StorageType Storage, bool ShouldCreate = true);
 
-  TempMDLocalVariable cloneImpl() const {
+  TempDILocalVariable cloneImpl() const {
     return getTemporary(getContext(), getTag(), getScope(), getName(),
                         getFile(), getLine(), getType(), getArg(), getFlags());
   }
 
 public:
-  DEFINE_MDNODE_GET(MDLocalVariable,
-                    (unsigned Tag, MDLocalScope *Scope, StringRef Name,
-                     MDFile *File, unsigned Line, MDTypeRef Type, unsigned Arg,
+  DEFINE_MDNODE_GET(DILocalVariable,
+                    (unsigned Tag, DILocalScope *Scope, StringRef Name,
+                     DIFile *File, unsigned Line, DITypeRef Type, unsigned Arg,
                      unsigned Flags),
                     (Tag, Scope, Name, File, Line, Type, Arg, Flags))
-  DEFINE_MDNODE_GET(MDLocalVariable,
+  DEFINE_MDNODE_GET(DILocalVariable,
                     (unsigned Tag, Metadata *Scope, MDString *Name,
                      Metadata *File, unsigned Line, Metadata *Type,
                      unsigned Arg, unsigned Flags),
                     (Tag, Scope, Name, File, Line, Type, Arg, Flags))
 
-  TempMDLocalVariable clone() const { return cloneImpl(); }
+  TempDILocalVariable clone() const { return cloneImpl(); }
 
   /// \brief Get the local scope for this variable.
   ///
   /// Variables must be defined in a local scope.
-  MDLocalScope *getScope() const {
-    return cast<MDLocalScope>(MDVariable::getScope());
+  DILocalScope *getScope() const {
+    return cast<DILocalScope>(DIVariable::getScope());
   }
 
   unsigned getArg() const { return Arg; }
@@ -1924,12 +1943,12 @@ class MDLocalVariable : public MDVariable {
   /// Check that \c DL exists, is in the same subprogram, and has the same
   /// inlined-at location as \c this.  (Otherwise, it's not a valid attachemnt
   /// to a \a DbgInfoIntrinsic.)
-  bool isValidLocationForIntrinsic(const MDLocation *DL) const {
+  bool isValidLocationForIntrinsic(const DILocation *DL) const {
     return DL && getScope()->getSubprogram() == DL->getScope()->getSubprogram();
   }
 
   static bool classof(const Metadata *MD) {
-    return MD->getMetadataID() == MDLocalVariableKind;
+    return MD->getMetadataID() == DILocalVariableKind;
   }
 };
 
@@ -1944,29 +1963,29 @@ class MDLocalVariable : public MDVariable {
 /// TODO: Co-allocate the expression elements.
 /// TODO: Separate from MDNode, or otherwise drop Distinct and Temporary
 /// storage types.
-class MDExpression : public MDNode {
+class DIExpression : public MDNode {
   friend class LLVMContextImpl;
   friend class MDNode;
 
   std::vector<uint64_t> Elements;
 
-  MDExpression(LLVMContext &C, StorageType Storage, ArrayRef<uint64_t> Elements)
-      : MDNode(C, MDExpressionKind, Storage, None),
+  DIExpression(LLVMContext &C, StorageType Storage, ArrayRef<uint64_t> Elements)
+      : MDNode(C, DIExpressionKind, Storage, None),
         Elements(Elements.begin(), Elements.end()) {}
-  ~MDExpression() = default;
+  ~DIExpression() = default;
 
-  static MDExpression *getImpl(LLVMContext &Context,
+  static DIExpression *getImpl(LLVMContext &Context,
                                ArrayRef<uint64_t> Elements, StorageType Storage,
                                bool ShouldCreate = true);
 
-  TempMDExpression cloneImpl() const {
+  TempDIExpression cloneImpl() const {
     return getTemporary(getContext(), getElements());
   }
 
 public:
-  DEFINE_MDNODE_GET(MDExpression, (ArrayRef<uint64_t> Elements), (Elements))
+  DEFINE_MDNODE_GET(DIExpression, (ArrayRef<uint64_t> Elements), (Elements))
 
-  TempMDExpression clone() const { return cloneImpl(); }
+  TempDIExpression clone() const { return cloneImpl(); }
 
   ArrayRef<uint64_t> getElements() const { return Elements; }
 
@@ -1991,7 +2010,7 @@ class MDExpression : public MDNode {
 
   /// \brief A lightweight wrapper around an expression operand.
   ///
-  /// TODO: Store arguments directly and change \a MDExpression to store a
+  /// TODO: Store arguments directly and change \a DIExpression to store a
   /// range of these.
   class ExprOperand {
     const uint64_t *Op;
@@ -2076,73 +2095,68 @@ class MDExpression : public MDNode {
   bool isValid() const;
 
   static bool classof(const Metadata *MD) {
-    return MD->getMetadataID() == MDExpressionKind;
+    return MD->getMetadataID() == DIExpressionKind;
   }
 };
 
-class MDObjCProperty : public DebugNode {
+class DIObjCProperty : public DINode {
   friend class LLVMContextImpl;
   friend class MDNode;
 
   unsigned Line;
   unsigned Attributes;
 
-  MDObjCProperty(LLVMContext &C, StorageType Storage, unsigned Line,
+  DIObjCProperty(LLVMContext &C, StorageType Storage, unsigned Line,
                  unsigned Attributes, ArrayRef<Metadata *> Ops)
-      : DebugNode(C, MDObjCPropertyKind, Storage, dwarf::DW_TAG_APPLE_property,
-                  Ops),
+      : DINode(C, DIObjCPropertyKind, Storage, dwarf::DW_TAG_APPLE_property,
+               Ops),
         Line(Line), Attributes(Attributes) {}
-  ~MDObjCProperty() = default;
+  ~DIObjCProperty() = default;
 
-  static MDObjCProperty *
-  getImpl(LLVMContext &Context, StringRef Name, MDFile *File, unsigned Line,
+  static DIObjCProperty *
+  getImpl(LLVMContext &Context, StringRef Name, DIFile *File, unsigned Line,
           StringRef GetterName, StringRef SetterName, unsigned Attributes,
-          MDType *Type, StorageType Storage, bool ShouldCreate = true) {
+          DITypeRef Type, StorageType Storage, bool ShouldCreate = true) {
     return getImpl(Context, getCanonicalMDString(Context, Name), File, Line,
                    getCanonicalMDString(Context, GetterName),
                    getCanonicalMDString(Context, SetterName), Attributes, Type,
                    Storage, ShouldCreate);
   }
-  static MDObjCProperty *getImpl(LLVMContext &Context, MDString *Name,
+  static DIObjCProperty *getImpl(LLVMContext &Context, MDString *Name,
                                  Metadata *File, unsigned Line,
                                  MDString *GetterName, MDString *SetterName,
                                  unsigned Attributes, Metadata *Type,
                                  StorageType Storage, bool ShouldCreate = true);
 
-  TempMDObjCProperty cloneImpl() const {
+  TempDIObjCProperty cloneImpl() const {
     return getTemporary(getContext(), getName(), getFile(), getLine(),
                         getGetterName(), getSetterName(), getAttributes(),
                         getType());
   }
 
 public:
-  DEFINE_MDNODE_GET(MDObjCProperty,
-                    (StringRef Name, MDFile *File, unsigned Line,
+  DEFINE_MDNODE_GET(DIObjCProperty,
+                    (StringRef Name, DIFile *File, unsigned Line,
                      StringRef GetterName, StringRef SetterName,
-                     unsigned Attributes, MDType *Type),
+                     unsigned Attributes, DITypeRef Type),
                     (Name, File, Line, GetterName, SetterName, Attributes,
                      Type))
-  DEFINE_MDNODE_GET(MDObjCProperty,
+  DEFINE_MDNODE_GET(DIObjCProperty,
                     (MDString * Name, Metadata *File, unsigned Line,
                      MDString *GetterName, MDString *SetterName,
                      unsigned Attributes, Metadata *Type),
                     (Name, File, Line, GetterName, SetterName, Attributes,
                      Type))
 
-  TempMDObjCProperty clone() const { return cloneImpl(); }
+  TempDIObjCProperty clone() const { return cloneImpl(); }
 
   unsigned getLine() const { return Line; }
   unsigned getAttributes() const { return Attributes; }
   StringRef getName() const { return getStringOperand(0); }
-  MDFile *getFile() const { return cast_or_null<MDFile>(getRawFile()); }
+  DIFile *getFile() const { return cast_or_null<DIFile>(getRawFile()); }
   StringRef getGetterName() const { return getStringOperand(2); }
   StringRef getSetterName() const { return getStringOperand(3); }
-
-  /// \brief Get the type.
-  ///
-  /// \note Objective-C doesn't have an ODR, so there is no benefit in storing
-  /// the type as a DITypeRef here.
-  MDType *getType() const { return cast_or_null<MDType>(getRawType()); }
+  DITypeRef getType() const { return DITypeRef(getRawType()); }
 
   StringRef getFilename() const {
     if (auto *F = getFile())
@@ -2162,56 +2176,56 @@ class MDObjCProperty : public DebugNode {
   Metadata *getRawType() const { return getOperand(4); }
 
   static bool classof(const Metadata *MD) {
-    return MD->getMetadataID() == MDObjCPropertyKind;
+    return MD->getMetadataID() == DIObjCPropertyKind;
   }
 };
 
 /// \brief An imported module (C++ using directive or similar).
-class MDImportedEntity : public DebugNode {
+class DIImportedEntity : public DINode {
   friend class LLVMContextImpl;
   friend class MDNode;
 
   unsigned Line;
 
-  MDImportedEntity(LLVMContext &C, StorageType Storage, unsigned Tag,
+  DIImportedEntity(LLVMContext &C, StorageType Storage, unsigned Tag,
                    unsigned Line, ArrayRef<Metadata *> Ops)
-      : DebugNode(C, MDImportedEntityKind, Storage, Tag, Ops), Line(Line) {}
-  ~MDImportedEntity() = default;
+      : DINode(C, DIImportedEntityKind, Storage, Tag, Ops), Line(Line) {}
+  ~DIImportedEntity() = default;
 
-  static MDImportedEntity *getImpl(LLVMContext &Context, unsigned Tag,
-                                   MDScope *Scope, DebugNodeRef Entity,
+  static DIImportedEntity *getImpl(LLVMContext &Context, unsigned Tag,
+                                   DIScope *Scope, DINodeRef Entity,
                                    unsigned Line, StringRef Name,
                                    StorageType Storage,
                                    bool ShouldCreate = true) {
     return getImpl(Context, Tag, Scope, Entity, Line,
                    getCanonicalMDString(Context, Name), Storage, ShouldCreate);
   }
-  static MDImportedEntity *getImpl(LLVMContext &Context, unsigned Tag,
+  static DIImportedEntity *getImpl(LLVMContext &Context, unsigned Tag,
                                    Metadata *Scope, Metadata *Entity,
                                    unsigned Line, MDString *Name,
                                    StorageType Storage,
                                    bool ShouldCreate = true);
 
-  TempMDImportedEntity cloneImpl() const {
+  TempDIImportedEntity cloneImpl() const {
     return getTemporary(getContext(), getTag(), getScope(), getEntity(),
                         getLine(), getName());
   }
 
 public:
-  DEFINE_MDNODE_GET(MDImportedEntity,
-                    (unsigned Tag, MDScope *Scope, DebugNodeRef Entity,
+  DEFINE_MDNODE_GET(DIImportedEntity,
+                    (unsigned Tag, DIScope *Scope, DINodeRef Entity,
                      unsigned Line, StringRef Name = ""),
                     (Tag, Scope, Entity, Line, Name))
-  DEFINE_MDNODE_GET(MDImportedEntity,
+  DEFINE_MDNODE_GET(DIImportedEntity,
                     (unsigned Tag, Metadata *Scope, Metadata *Entity,
                      unsigned Line, MDString *Name),
                     (Tag, Scope, Entity, Line, Name))
 
-  TempMDImportedEntity clone() const { return cloneImpl(); }
+  TempDIImportedEntity clone() const { return cloneImpl(); }
 
   unsigned getLine() const { return Line; }
-  MDScope *getScope() const { return cast_or_null<MDScope>(getRawScope()); }
-  DebugNodeRef getEntity() const { return DebugNodeRef(getRawEntity()); }
+  DIScope *getScope() const { return cast_or_null<DIScope>(getRawScope()); }
+  DINodeRef getEntity() const { return DINodeRef(getRawEntity()); }
   StringRef getName() const { return getStringOperand(2); }
 
   Metadata *getRawScope() const { return getOperand(0); }
@@ -2219,7 +2233,7 @@ class MDImportedEntity : public DebugNode {
   MDString *getRawName() const { return getOperandAs<MDString>(2); }
 
   static bool classof(const Metadata *MD) {
-    return MD->getMetadataID() == MDImportedEntityKind;
+    return MD->getMetadataID() == DIImportedEntityKind;
   }
 };
 
diff --git a/include/llvm/IR/DebugLoc.h b/include/llvm/IR/DebugLoc.h
index f88a7b156b2..8ea5875e1f8 100644
--- a/include/llvm/IR/DebugLoc.h
+++ b/include/llvm/IR/DebugLoc.h
@@ -22,14 +22,14 @@ namespace llvm {
 
   class LLVMContext;
   class raw_ostream;
-  class MDLocation;
+  class DILocation;
 
   /// \brief A debug info location.
   ///
-  /// This class is a wrapper around a tracking reference to an \a MDLocation
+  /// This class is a wrapper around a tracking reference to an \a DILocation
   /// pointer.
   ///
-  /// To avoid extra includes, \a DebugLoc doubles the \a MDLocation API with a
+  /// To avoid extra includes, \a DebugLoc doubles the \a DILocation API with a
   /// one based on relatively opaque \a MDNode pointers.
   class DebugLoc {
     TrackingMDNodeRef Loc;
@@ -47,31 +47,31 @@ namespace llvm {
       return *this;
     }
 
-    /// \brief Construct from an \a MDLocation.
-    DebugLoc(const MDLocation *L);
+    /// \brief Construct from an \a DILocation.
+    DebugLoc(const DILocation *L);
 
     /// \brief Construct from an \a MDNode.
     ///
-    /// Note: if \c N is not an \a MDLocation, a verifier check will fail, and
+    /// Note: if \c N is not an \a DILocation, a verifier check will fail, and
     /// accessors will crash.  However, construction from other nodes is
     /// supported in order to handle forward references when reading textual
     /// IR.
     explicit DebugLoc(const MDNode *N);
 
-    /// \brief Get the underlying \a MDLocation.
+    /// \brief Get the underlying \a DILocation.
     ///
-    /// \pre !*this or \c isa<MDLocation>(getAsMDNode()).
+    /// \pre !*this or \c isa<DILocation>(getAsMDNode()).
     /// @{
-    MDLocation *get() const;
-    operator MDLocation *() const { return get(); }
-    MDLocation *operator->() const { return get(); }
-    MDLocation &operator*() const { return *get(); }
+    DILocation *get() const;
+    operator DILocation *() const { return get(); }
+    DILocation *operator->() const { return get(); }
+    DILocation &operator*() const { return *get(); }
     /// @}
 
     /// \brief Check for null.
     ///
     /// Check for null in a way that is safe with broken debug info.  Unlike
-    /// the conversion to \c MDLocation, this doesn't require that \c Loc is of
+    /// the conversion to \c DILocation, this doesn't require that \c Loc is of
     /// the right type.  Important for cases like \a llvm::StripDebugInfo() and
     /// \a Instruction::hasMetadata().
     explicit operator bool() const { return Loc; }
@@ -82,18 +82,18 @@ namespace llvm {
     /// \brief Create a new DebugLoc.
     ///
     /// Create a new DebugLoc at the specified line/col and scope/inline.  This
-    /// forwards to \a MDLocation::get().
+    /// forwards to \a DILocation::get().
     ///
     /// If \c !Scope, returns a default-constructed \a DebugLoc.
     ///
-    /// FIXME: Remove this.  Users should use MDLocation::get().
+    /// FIXME: Remove this.  Users should use DILocation::get().
     static DebugLoc get(unsigned Line, unsigned Col, const MDNode *Scope,
                         const MDNode *InlinedAt = nullptr);
 
     unsigned getLine() const;
     unsigned getCol() const;
     MDNode *getScope() const;
-    MDLocation *getInlinedAt() const;
+    DILocation *getInlinedAt() const;
 
     /// \brief Get the fully inlined-at scope for a DebugLoc.
     ///
@@ -105,8 +105,8 @@ namespace llvm {
     /// Walk up the scope chain of given debug loc and find line number info
     /// for the function.
     ///
-    /// FIXME: Remove this.  Users should use MDLocation/MDLocalScope API to
-    /// find the subprogram, and then MDLocation::get().
+    /// FIXME: Remove this.  Users should use DILocation/DILocalScope API to
+    /// find the subprogram, and then DILocation::get().
     DebugLoc getFnDebugLoc() const;
 
     /// \brief Return \c this as a bar \a MDNode.
diff --git a/include/llvm/IR/DerivedTypes.h b/include/llvm/IR/DerivedTypes.h
index 76f806434f4..4a94499b4cf 100644
--- a/include/llvm/IR/DerivedTypes.h
+++ b/include/llvm/IR/DerivedTypes.h
@@ -140,7 +140,8 @@ class FunctionType : public Type {
     return T->getTypeID() == FunctionTyID;
   }
 };
-
+static_assert(AlignOf<FunctionType>::Alignment >= AlignOf<Type *>::Alignment,
+              "Alignment sufficient for objects appended to FunctionType");
 
 /// CompositeType - Common super class of ArrayType, StructType, PointerType
 /// and VectorType.
@@ -464,6 +465,9 @@ class PointerType : public SequentialType {
   /// element type.
   static bool isValidElementType(Type *ElemTy);
 
+  /// Return true if we can load or store from a pointer to this type.
+  static bool isLoadableOrStorableType(Type *ElemTy);
+
   /// @brief Return the address space of the Pointer type.
   inline unsigned getAddressSpace() const { return getSubclassData(); }
 
diff --git a/include/llvm/IR/DiagnosticInfo.h b/include/llvm/IR/DiagnosticInfo.h
index 6db5a40c577..f38313f82ea 100644
--- a/include/llvm/IR/DiagnosticInfo.h
+++ b/include/llvm/IR/DiagnosticInfo.h
@@ -32,6 +32,7 @@ class LLVMContextImpl;
 class Twine;
 class Value;
 class DebugLoc;
+class SMDiagnostic;
 
 /// \brief Defines the different supported severity of a diagnostic.
 enum DiagnosticSeverity {
@@ -56,6 +57,7 @@ enum DiagnosticKind {
   DK_OptimizationRemarkMissed,
   DK_OptimizationRemarkAnalysis,
   DK_OptimizationFailure,
+  DK_MIRParser,
   DK_FirstPluginKind
 };
 
@@ -386,6 +388,24 @@ class DiagnosticInfoOptimizationRemarkAnalysis
   bool isEnabled() const override;
 };
 
+/// Diagnostic information for machine IR parser.
+class DiagnosticInfoMIRParser : public DiagnosticInfo {
+  const SMDiagnostic &Diagnostic;
+
+public:
+  DiagnosticInfoMIRParser(DiagnosticSeverity Severity,
+                          const SMDiagnostic &Diagnostic)
+      : DiagnosticInfo(DK_MIRParser, Severity), Diagnostic(Diagnostic) {}
+
+  const SMDiagnostic &getDiagnostic() const { return Diagnostic; }
+
+  void print(DiagnosticPrinter &DP) const override;
+
+  static bool classof(const DiagnosticInfo *DI) {
+    return DI->getKind() == DK_MIRParser;
+  }
+};
+
 // Create wrappers for C Binding types (see CBindingWrapping.h).
 DEFINE_SIMPLE_CONVERSION_FUNCTIONS(DiagnosticInfo, LLVMDiagnosticInfoRef)
 
diff --git a/include/llvm/IR/DiagnosticPrinter.h b/include/llvm/IR/DiagnosticPrinter.h
index db5779a8a8a..1bcd73738b6 100644
--- a/include/llvm/IR/DiagnosticPrinter.h
+++ b/include/llvm/IR/DiagnosticPrinter.h
@@ -22,6 +22,7 @@ namespace llvm {
 // Forward declarations.
 class Module;
 class raw_ostream;
+class SMDiagnostic;
 class StringRef;
 class Twine;
 class Value;
@@ -51,6 +52,9 @@ class DiagnosticPrinter {
   // IR related types.
   virtual DiagnosticPrinter &operator<<(const Value &V) = 0;
   virtual DiagnosticPrinter &operator<<(const Module &M) = 0;
+
+  // Other types.
+  virtual DiagnosticPrinter &operator<<(const SMDiagnostic &Diag) = 0;
 };
 
 /// \brief Basic diagnostic printer that uses an underlying raw_ostream.
@@ -59,7 +63,7 @@ class DiagnosticPrinterRawOStream : public DiagnosticPrinter {
   raw_ostream &Stream;
 
 public:
-  DiagnosticPrinterRawOStream(raw_ostream &Stream) : Stream(Stream) {};
+  DiagnosticPrinterRawOStream(raw_ostream &Stream) : Stream(Stream) {}
 
   // Simple types.
   DiagnosticPrinter &operator<<(char C) override;
@@ -81,6 +85,9 @@ class DiagnosticPrinterRawOStream : public DiagnosticPrinter {
   // IR related types.
   DiagnosticPrinter &operator<<(const Value &V) override;
   DiagnosticPrinter &operator<<(const Module &M) override;
+
+  // Other types.
+  DiagnosticPrinter &operator<<(const SMDiagnostic &Diag) override;
 };
 } // End namespace llvm
 
diff --git a/include/llvm/IR/Dominators.h b/include/llvm/IR/Dominators.h
index c1f208e3d72..52c2a8e2fc4 100644
--- a/include/llvm/IR/Dominators.h
+++ b/include/llvm/IR/Dominators.h
@@ -36,18 +36,14 @@ namespace llvm {
 template <typename IRUnitT> class AnalysisManager;
 class PreservedAnalyses;
 
-EXTERN_TEMPLATE_INSTANTIATION(class DomTreeNodeBase<BasicBlock>);
-EXTERN_TEMPLATE_INSTANTIATION(class DominatorTreeBase<BasicBlock>);
-
-#define LLVM_COMMA ,
-EXTERN_TEMPLATE_INSTANTIATION(void Calculate<Function LLVM_COMMA BasicBlock *>(
-    DominatorTreeBase<GraphTraits<BasicBlock *>::NodeType> &DT LLVM_COMMA
-        Function &F));
-EXTERN_TEMPLATE_INSTANTIATION(
-    void Calculate<Function LLVM_COMMA Inverse<BasicBlock *> >(
-        DominatorTreeBase<GraphTraits<Inverse<BasicBlock *> >::NodeType> &DT
-            LLVM_COMMA Function &F));
-#undef LLVM_COMMA
+extern template class DomTreeNodeBase<BasicBlock>;
+extern template class DominatorTreeBase<BasicBlock>;
+
+extern template void Calculate<Function, BasicBlock *>(
+    DominatorTreeBase<GraphTraits<BasicBlock *>::NodeType> &DT, Function &F);
+extern template void Calculate<Function, Inverse<BasicBlock *>>(
+    DominatorTreeBase<GraphTraits<Inverse<BasicBlock *>>::NodeType> &DT,
+    Function &F);
 
 typedef DomTreeNodeBase<BasicBlock> DomTreeNode;
 
@@ -73,6 +69,9 @@ class DominatorTree : public DominatorTreeBase<BasicBlock> {
   typedef DominatorTreeBase<BasicBlock> Base;
 
   DominatorTree() : DominatorTreeBase<BasicBlock>(false) {}
+  explicit DominatorTree(Function &F) : DominatorTreeBase<BasicBlock>(false) {
+    recalculate(F);
+  }
 
   DominatorTree(DominatorTree &&Arg)
       : Base(std::move(static_cast<Base &>(Arg))) {}
@@ -151,6 +150,31 @@ template <> struct GraphTraits<DomTreeNode*> {
   }
 };
 
+template <> struct GraphTraits<const DomTreeNode *> {
+  typedef const DomTreeNode NodeType;
+  typedef NodeType::const_iterator ChildIteratorType;
+
+  static NodeType *getEntryNode(NodeType *N) {
+    return N;
+  }
+  static inline ChildIteratorType child_begin(NodeType *N) {
+    return N->begin();
+  }
+  static inline ChildIteratorType child_end(NodeType *N) {
+    return N->end();
+  }
+
+  typedef df_iterator<const DomTreeNode *> nodes_iterator;
+
+  static nodes_iterator nodes_begin(const DomTreeNode *N) {
+    return df_begin(getEntryNode(N));
+  }
+
+  static nodes_iterator nodes_end(const DomTreeNode *N) {
+    return df_end(getEntryNode(N));
+  }
+};
+
 template <> struct GraphTraits<DominatorTree*>
   : public GraphTraits<DomTreeNode*> {
   static NodeType *getEntryNode(DominatorTree *DT) {
diff --git a/include/llvm/IR/Function.h b/include/llvm/IR/Function.h
index a5eed9b68b0..ec9f4cad094 100644
--- a/include/llvm/IR/Function.h
+++ b/include/llvm/IR/Function.h
@@ -19,11 +19,13 @@
 #define LLVM_IR_FUNCTION_H
 
 #include "llvm/ADT/iterator_range.h"
+#include "llvm/ADT/Optional.h"
 #include "llvm/IR/Argument.h"
 #include "llvm/IR/Attributes.h"
 #include "llvm/IR/BasicBlock.h"
 #include "llvm/IR/CallingConv.h"
 #include "llvm/IR/GlobalObject.h"
+#include "llvm/IR/OperandTraits.h"
 #include "llvm/Support/Compiler.h"
 
 namespace llvm {
@@ -77,6 +79,17 @@ class Function : public GlobalObject, public ilist_node<Function> {
    * bit 3-6: CallingConvention
    */
 
+  /// Bits from GlobalObject::GlobalObjectSubclassData.
+  enum {
+    /// Whether this function is materializable.
+    IsMaterializableBit = 1 << 0,
+    HasMetadataHashEntryBit = 1 << 1
+  };
+  void setGlobalObjectBit(unsigned Mask, bool Value) {
+    setGlobalObjectSubClassData((~Mask & getGlobalObjectSubClassData()) |
+                                (Value ? Mask : 0u));
+  }
+
   friend class SymbolTableListTraits<Function, Module>;
 
   void setParent(Module *parent);
@@ -97,10 +110,6 @@ class Function : public GlobalObject, public ilist_node<Function> {
   Function(const Function&) = delete;
   void operator=(const Function&) = delete;
 
-  /// Do the actual lookup of an intrinsic ID when the query could not be
-  /// answered from the cache.
-  unsigned lookupIntrinsicID() const LLVM_READONLY;
-
   /// Function ctor - If the (optional) Module argument is specified, the
   /// function is automatically inserted into the end of the function list for
   /// the module.
@@ -111,16 +120,27 @@ class Function : public GlobalObject, public ilist_node<Function> {
 public:
   static Function *Create(FunctionType *Ty, LinkageTypes Linkage,
                           const Twine &N = "", Module *M = nullptr) {
-    return new(0) Function(Ty, Linkage, N, M);
+    return new(1) Function(Ty, Linkage, N, M);
   }
 
   ~Function() override;
 
+  /// \brief Provide fast operand accessors
+  DECLARE_TRANSPARENT_OPERAND_ACCESSORS(Value);
+
+  /// \brief Get the personality function associated with this function.
+  bool hasPersonalityFn() const { return getNumOperands() != 0; }
+  Constant *getPersonalityFn() const {
+    assert(hasPersonalityFn());
+    return cast<Constant>(Op<0>());
+  }
+  void setPersonalityFn(Constant *C);
+
   Type *getReturnType() const;           // Return the type of the ret val
   FunctionType *getFunctionType() const; // Return the FunctionType for me
 
-  /// getContext - Return a pointer to the LLVMContext associated with this
-  /// function, or NULL if this function is not bound to a context yet.
+  /// getContext - Return a reference to the LLVMContext associated with this
+  /// function.
   LLVMContext &getContext() const;
 
   /// isVarArg - Return true if this function takes a variable number of
@@ -135,13 +155,17 @@ class Function : public GlobalObject, public ilist_node<Function> {
   /// intrinsic, or if the pointer is null.  This value is always defined to be
   /// zero to allow easy checking for whether a function is intrinsic or not.
   /// The particular intrinsic functions which correspond to this value are
-  /// defined in llvm/Intrinsics.h.  Results are cached in the LLVM context,
-  /// subsequent requests for the same ID return results much faster from the
-  /// cache.
-  ///
-  unsigned getIntrinsicID() const LLVM_READONLY;
+  /// defined in llvm/Intrinsics.h.
+  Intrinsic::ID getIntrinsicID() const LLVM_READONLY { return IntID; }
   bool isIntrinsic() const { return getName().startswith("llvm."); }
 
+  /// \brief Recalculate the ID for this function if it is an Intrinsic defined
+  /// in llvm/Intrinsics.h.  Sets the intrinsic ID to Intrinsic::not_intrinsic
+  /// if the name of this function does not match an intrinsic in that header.
+  /// Note, this method does not need to be called directly, as it is called
+  /// from Value::setName() whenever the name of this function changes.
+  void recalculateIntrinsicID();
+
   /// getCallingConv()/setCallingConv(CC) - These method get and set the
   /// calling convention of this function.  The enum values for the known
   /// calling conventions are defined in CallingConv.h.
@@ -183,6 +207,12 @@ class Function : public GlobalObject, public ilist_node<Function> {
                                  AttributeSet::FunctionIndex, Kind, Value));
   }
 
+  /// Set the entry count for this function.
+  void setEntryCount(uint64_t Count);
+
+  /// Get the entry count for this function.
+  Optional<uint64_t> getEntryCount() const;
+
   /// @brief Return true if the function has the attribute.
   bool hasFnAttribute(Attribute::AttrKind Kind) const {
     return AttributeSets.hasAttribute(AttributeSet::FunctionIndex, Kind);
@@ -237,7 +267,13 @@ class Function : public GlobalObject, public ilist_node<Function> {
   uint64_t getDereferenceableBytes(unsigned i) const {
     return AttributeSets.getDereferenceableBytes(i);
   }
-
+  
+  /// @brief Extract the number of dereferenceable_or_null bytes for a call or
+  /// parameter (0=unknown).
+  uint64_t getDereferenceableOrNullBytes(unsigned i) const {
+    return AttributeSets.getDereferenceableOrNullBytes(i);
+  }
+  
   /// @brief Determine if the function does not access memory.
   bool doesNotAccessMemory() const {
     return AttributeSets.hasAttribute(AttributeSet::FunctionIndex,
@@ -257,6 +293,16 @@ class Function : public GlobalObject, public ilist_node<Function> {
     addFnAttr(Attribute::ReadOnly);
   }
 
+  /// @brief Determine if the call can access memmory only using pointers based
+  /// on its arguments.
+  bool onlyAccessesArgMemory() const {
+    return AttributeSets.hasAttribute(AttributeSet::FunctionIndex,
+                                      Attribute::ArgMemOnly);
+  }
+  void setOnlyAccessesArgMemory() {
+    addFnAttr(Attribute::ArgMemOnly);
+  }
+  
   /// @brief Determine if the function cannot return.
   bool doesNotReturn() const {
     return AttributeSets.hasAttribute(AttributeSet::FunctionIndex,
@@ -284,6 +330,16 @@ class Function : public GlobalObject, public ilist_node<Function> {
     addFnAttr(Attribute::NoDuplicate);
   }
 
+  /// @brief Determine if the call is convergent.
+  bool isConvergent() const {
+    return AttributeSets.hasAttribute(AttributeSet::FunctionIndex,
+                                      Attribute::Convergent);
+  }
+  void setConvergent() {
+    addFnAttr(Attribute::Convergent);
+  }
+
+
   /// @brief True if the ABI mandates (or the user requested) that this
   /// function be in a unwind table.
   bool hasUWTable() const {
@@ -511,12 +567,50 @@ class Function : public GlobalObject, public ilist_node<Function> {
   /// setjmp or other function that gcc recognizes as "returning twice".
   bool callsFunctionThatReturnsTwice() const;
 
+  /// \brief Check if this has any metadata.
+  bool hasMetadata() const { return hasMetadataHashEntry(); }
+
+  /// \brief Get the current metadata attachment, if any.
+  ///
+  /// Returns \c nullptr if such an attachment is missing.
+  /// @{
+  MDNode *getMetadata(unsigned KindID) const;
+  MDNode *getMetadata(StringRef Kind) const;
+  /// @}
+
+  /// \brief Set a particular kind of metadata attachment.
+  ///
+  /// Sets the given attachment to \c MD, erasing it if \c MD is \c nullptr or
+  /// replacing it if it already exists.
+  /// @{
+  void setMetadata(unsigned KindID, MDNode *MD);
+  void setMetadata(StringRef Kind, MDNode *MD);
+  /// @}
+
+  /// \brief Get all current metadata attachments.
+  void
+  getAllMetadata(SmallVectorImpl<std::pair<unsigned, MDNode *>> &MDs) const;
+
+  /// \brief Drop metadata not in the given list.
+  ///
+  /// Drop all metadata from \c this not included in \c KnownIDs.
+  void dropUnknownMetadata(ArrayRef<unsigned> KnownIDs);
+
 private:
   // Shadow Value::setValueSubclassData with a private forwarding method so that
   // subclasses cannot accidentally use it.
   void setValueSubclassData(unsigned short D) {
     Value::setValueSubclassData(D);
   }
+
+  bool hasMetadataHashEntry() const {
+    return getGlobalObjectSubClassData() & HasMetadataHashEntryBit;
+  }
+  void setHasMetadataHashEntry(bool HasEntry) {
+    setGlobalObjectBit(HasMetadataHashEntryBit, HasEntry);
+  }
+
+  void clearMetadata();
 };
 
 inline ValueSymbolTable *
@@ -529,6 +623,11 @@ ilist_traits<Argument>::getSymTab(Function *F) {
   return F ? &F->getValueSymbolTable() : nullptr;
 }
 
+template <>
+struct OperandTraits<Function> : public OptionalOperandTraits<Function> {};
+
+DEFINE_TRANSPARENT_OPERAND_ACCESSORS(Function, Value)
+
 } // End llvm namespace
 
 #endif
diff --git a/include/llvm/IR/GVMaterializer.h b/include/llvm/IR/GVMaterializer.h
index 779f2e01818..1d6c9157f0b 100644
--- a/include/llvm/IR/GVMaterializer.h
+++ b/include/llvm/IR/GVMaterializer.h
@@ -47,11 +47,11 @@ class GVMaterializer {
   /// lazily. If the Materializer doesn't support this capability, this method
   /// is a noop.
   ///
-  virtual void Dematerialize(GlobalValue *) {}
+  virtual void dematerialize(GlobalValue *) {}
 
   /// Make sure the entire Module has been completely read.
   ///
-  virtual std::error_code MaterializeModule(Module *M) = 0;
+  virtual std::error_code materializeModule(Module *M) = 0;
 
   virtual std::error_code materializeMetadata() = 0;
   virtual void setStripDebugInfo() = 0;
diff --git a/include/llvm/IR/GetElementPtrTypeIterator.h b/include/llvm/IR/GetElementPtrTypeIterator.h
index dcf8e64dbb2..7cb13fa33aa 100644
--- a/include/llvm/IR/GetElementPtrTypeIterator.h
+++ b/include/llvm/IR/GetElementPtrTypeIterator.h
@@ -16,7 +16,9 @@
 #define LLVM_IR_GETELEMENTPTRTYPEITERATOR_H
 
 #include "llvm/IR/DerivedTypes.h"
+#include "llvm/IR/Operator.h"
 #include "llvm/IR/User.h"
+#include "llvm/ADT/PointerIntPair.h"
 
 namespace llvm {
   template<typename ItTy = User::const_op_iterator>
@@ -26,19 +28,28 @@ namespace llvm {
                           Type *, ptrdiff_t> super;
 
     ItTy OpIt;
-    Type *CurTy;
+    PointerIntPair<Type *, 1> CurTy;
+    unsigned AddrSpace;
     generic_gep_type_iterator() {}
   public:
 
     static generic_gep_type_iterator begin(Type *Ty, ItTy It) {
       generic_gep_type_iterator I;
-      I.CurTy = Ty;
+      I.CurTy.setPointer(Ty);
+      I.OpIt = It;
+      return I;
+    }
+    static generic_gep_type_iterator begin(Type *Ty, unsigned AddrSpace,
+                                           ItTy It) {
+      generic_gep_type_iterator I;
+      I.CurTy.setPointer(Ty);
+      I.CurTy.setInt(true);
+      I.AddrSpace = AddrSpace;
       I.OpIt = It;
       return I;
     }
     static generic_gep_type_iterator end(ItTy It) {
       generic_gep_type_iterator I;
-      I.CurTy = nullptr;
       I.OpIt = It;
       return I;
     }
@@ -51,11 +62,15 @@ namespace llvm {
     }
 
     Type *operator*() const {
-      return CurTy;
+      if (CurTy.getInt())
+        return CurTy.getPointer()->getPointerTo(AddrSpace);
+      return CurTy.getPointer();
     }
 
     Type *getIndexedType() const {
-      CompositeType *CT = cast<CompositeType>(CurTy);
+      if (CurTy.getInt())
+        return CurTy.getPointer();
+      CompositeType *CT = cast<CompositeType>(CurTy.getPointer());
       return CT->getTypeAtIndex(getOperand());
     }
 
@@ -63,13 +78,16 @@ namespace llvm {
     // current type directly.
     Type *operator->() const { return operator*(); }
 
-    Value *getOperand() const { return *OpIt; }
+    Value *getOperand() const { return const_cast<Value *>(&**OpIt); }
 
     generic_gep_type_iterator& operator++() {   // Preincrement
-      if (CompositeType *CT = dyn_cast<CompositeType>(CurTy)) {
-        CurTy = CT->getTypeAtIndex(getOperand());
+      if (CurTy.getInt()) {
+        CurTy.setInt(false);
+      } else if (CompositeType *CT =
+                     dyn_cast<CompositeType>(CurTy.getPointer())) {
+        CurTy.setPointer(CT->getTypeAtIndex(getOperand()));
       } else {
-        CurTy = nullptr;
+        CurTy.setPointer(nullptr);
       }
       ++OpIt;
       return *this;
@@ -83,15 +101,23 @@ namespace llvm {
   typedef generic_gep_type_iterator<> gep_type_iterator;
 
   inline gep_type_iterator gep_type_begin(const User *GEP) {
-    return gep_type_iterator::begin
-      (GEP->getOperand(0)->getType()->getScalarType(), GEP->op_begin()+1);
+    auto *GEPOp = cast<GEPOperator>(GEP);
+    return gep_type_iterator::begin(
+        GEPOp->getSourceElementType(),
+        cast<PointerType>(GEPOp->getPointerOperandType()->getScalarType())
+            ->getAddressSpace(),
+        GEP->op_begin() + 1);
   }
   inline gep_type_iterator gep_type_end(const User *GEP) {
     return gep_type_iterator::end(GEP->op_end());
   }
   inline gep_type_iterator gep_type_begin(const User &GEP) {
-    return gep_type_iterator::begin
-      (GEP.getOperand(0)->getType()->getScalarType(), GEP.op_begin()+1);
+    auto &GEPOp = cast<GEPOperator>(GEP);
+    return gep_type_iterator::begin(
+        GEPOp.getSourceElementType(),
+        cast<PointerType>(GEPOp.getPointerOperandType()->getScalarType())
+            ->getAddressSpace(),
+        GEP.op_begin() + 1);
   }
   inline gep_type_iterator gep_type_end(const User &GEP) {
     return gep_type_iterator::end(GEP.op_end());
diff --git a/include/llvm/IR/GlobalAlias.h b/include/llvm/IR/GlobalAlias.h
index d0672c8aac0..ce73b7af8ca 100644
--- a/include/llvm/IR/GlobalAlias.h
+++ b/include/llvm/IR/GlobalAlias.h
@@ -33,8 +33,8 @@ class GlobalAlias : public GlobalValue, public ilist_node<GlobalAlias> {
 
   void setParent(Module *parent);
 
-  GlobalAlias(Type *Ty, unsigned AddressSpace, LinkageTypes Linkage,
-              const Twine &Name, Constant *Aliasee, Module *Parent);
+  GlobalAlias(PointerType *Ty, LinkageTypes Linkage, const Twine &Name,
+              Constant *Aliasee, Module *Parent);
 
 public:
   // allocate space for exactly one operand
@@ -44,19 +44,17 @@ class GlobalAlias : public GlobalValue, public ilist_node<GlobalAlias> {
 
   /// If a parent module is specified, the alias is automatically inserted into
   /// the end of the specified module's alias list.
-  static GlobalAlias *create(Type *Ty, unsigned AddressSpace,
-                             LinkageTypes Linkage, const Twine &Name,
-                             Constant *Aliasee, Module *Parent);
+  static GlobalAlias *create(PointerType *Ty, LinkageTypes Linkage,
+                             const Twine &Name, Constant *Aliasee,
+                             Module *Parent);
 
   // Without the Aliasee.
-  static GlobalAlias *create(Type *Ty, unsigned AddressSpace,
-                             LinkageTypes Linkage, const Twine &Name,
-                             Module *Parent);
+  static GlobalAlias *create(PointerType *Ty, LinkageTypes Linkage,
+                             const Twine &Name, Module *Parent);
 
   // The module is taken from the Aliasee.
-  static GlobalAlias *create(Type *Ty, unsigned AddressSpace,
-                             LinkageTypes Linkage, const Twine &Name,
-                             GlobalValue *Aliasee);
+  static GlobalAlias *create(PointerType *Ty, LinkageTypes Linkage,
+                             const Twine &Name, GlobalValue *Aliasee);
 
   // Type, Parent and AddressSpace taken from the Aliasee.
   static GlobalAlias *create(LinkageTypes Linkage, const Twine &Name,
diff --git a/include/llvm/IR/GlobalValue.h b/include/llvm/IR/GlobalValue.h
index aeaaef4bd8b..2961369a732 100644
--- a/include/llvm/IR/GlobalValue.h
+++ b/include/llvm/IR/GlobalValue.h
@@ -28,6 +28,10 @@ class Comdat;
 class PointerType;
 class Module;
 
+namespace Intrinsic {
+  enum ID : unsigned;
+}
+
 class GlobalValue : public Constant {
   GlobalValue(const GlobalValue &) = delete;
 public:
@@ -66,7 +70,7 @@ class GlobalValue : public Constant {
       : Constant(Ty, VTy, Ops, NumOps), Linkage(Linkage),
         Visibility(DefaultVisibility), UnnamedAddr(0),
         DllStorageClass(DefaultStorageClass),
-        ThreadLocal(NotThreadLocal), Parent(nullptr) {
+        ThreadLocal(NotThreadLocal), IntID((Intrinsic::ID)0U), Parent(nullptr) {
     setName(Name);
   }
 
@@ -79,18 +83,31 @@ class GlobalValue : public Constant {
 
   unsigned ThreadLocal : 3; // Is this symbol "Thread Local", if so, what is
                             // the desired model?
+  static const unsigned GlobalValueSubClassDataBits = 19;
 
 private:
   // Give subclasses access to what otherwise would be wasted padding.
   // (19 + 3 + 2 + 1 + 2 + 5) == 32.
-  unsigned SubClassData : 19;
+  unsigned SubClassData : GlobalValueSubClassDataBits;
+
+  friend class Constant;
+  void destroyConstantImpl();
+  Value *handleOperandChangeImpl(Value *From, Value *To, Use *U);
+
 protected:
-  static const unsigned GlobalValueSubClassDataBits = 19;
+  /// \brief The intrinsic ID for this subclass (which must be a Function).
+  ///
+  /// This member is defined by this class, but not used for anything.
+  /// Subclasses can use it to store their intrinsic ID, if they have one.
+  ///
+  /// This is stored here to save space in Function on 64-bit hosts.
+  Intrinsic::ID IntID;
+
   unsigned getGlobalValueSubClassData() const {
     return SubClassData;
   }
   void setGlobalValueSubClassData(unsigned V) {
-    assert(V < (1 << 19) && "It will not fit");
+    assert(V < (1 << GlobalValueSubClassDataBits) && "It will not fit");
     SubClassData = V;
   }
 
@@ -235,10 +252,9 @@ class GlobalValue : public Constant {
   /// mistake: when working at the IR level use mayBeOverridden instead as it
   /// knows about ODR semantics.
   static bool isWeakForLinker(LinkageTypes Linkage)  {
-    return Linkage == AvailableExternallyLinkage || Linkage == WeakAnyLinkage ||
-           Linkage == WeakODRLinkage || Linkage == LinkOnceAnyLinkage ||
-           Linkage == LinkOnceODRLinkage || Linkage == CommonLinkage ||
-           Linkage == ExternalWeakLinkage;
+    return Linkage == WeakAnyLinkage || Linkage == WeakODRLinkage ||
+           Linkage == LinkOnceAnyLinkage || Linkage == LinkOnceODRLinkage ||
+           Linkage == CommonLinkage || Linkage == ExternalWeakLinkage;
   }
 
   bool hasExternalLinkage() const { return isExternalLinkage(Linkage); }
@@ -317,13 +333,10 @@ class GlobalValue : public Constant {
   /// If this GlobalValue is read in, and if the GVMaterializer supports it,
   /// release the memory for the function, and set it up to be materialized
   /// lazily. If !isDematerializable(), this method is a noop.
-  void Dematerialize();
+  void dematerialize();
 
 /// @}
 
-  /// Override from Constant class.
-  void destroyConstant() override;
-
   /// Return true if the primary definition of this global value is outside of
   /// the current translation unit.
   bool isDeclaration() const;
@@ -335,6 +348,12 @@ class GlobalValue : public Constant {
     return isDeclaration();
   }
 
+  /// Returns true if this global's definition will be the one chosen by the
+  /// linker.
+  bool isStrongDefinitionForLinker() const {
+    return !(isDeclarationForLinker() || isWeakForLinker());
+  }
+
   /// This method unlinks 'this' from the containing module, but does not delete
   /// it.
   virtual void removeFromParent() = 0;
diff --git a/include/llvm/IR/GlobalVariable.h b/include/llvm/IR/GlobalVariable.h
index 9f57705dae7..a0159830ba3 100644
--- a/include/llvm/IR/GlobalVariable.h
+++ b/include/llvm/IR/GlobalVariable.h
@@ -45,7 +45,6 @@ class GlobalVariable : public GlobalObject, public ilist_node<GlobalVariable> {
                                                // can change from its initial
                                                // value before global
                                                // initializers are run?
-
 public:
   // allocate space for exactly one operand
   void *operator new(size_t s) {
@@ -67,7 +66,8 @@ class GlobalVariable : public GlobalObject, public ilist_node<GlobalVariable> {
                  bool isExternallyInitialized = false);
 
   ~GlobalVariable() override {
-    NumOperands = 1; // FIXME: needed by operator delete
+    // FIXME: needed by operator delete
+    setGlobalVariableNumOperands(1);
   }
 
   /// Provide fast operand accessors
@@ -165,10 +165,6 @@ class GlobalVariable : public GlobalObject, public ilist_node<GlobalVariable> {
   ///
   void eraseFromParent() override;
 
-  /// Override Constant's implementation of this method so we can
-  /// replace constant initializers.
-  void replaceUsesOfWithOnConstant(Value *From, Value *To, Use *U) override;
-
   // Methods for support type inquiry through isa, cast, and dyn_cast:
   static inline bool classof(const Value *V) {
     return V->getValueID() == Value::GlobalVariableVal;
diff --git a/include/llvm/IR/IRBuilder.h b/include/llvm/IR/IRBuilder.h
index 9c4ba078995..d02f41b7e27 100644
--- a/include/llvm/IR/IRBuilder.h
+++ b/include/llvm/IR/IRBuilder.h
@@ -21,6 +21,7 @@
 #include "llvm/IR/BasicBlock.h"
 #include "llvm/IR/ConstantFolder.h"
 #include "llvm/IR/DataLayout.h"
+#include "llvm/IR/Function.h"
 #include "llvm/IR/GlobalVariable.h"
 #include "llvm/IR/Instructions.h"
 #include "llvm/IR/LLVMContext.h"
@@ -100,19 +101,8 @@ class IRBuilderBase {
   void SetInsertPoint(BasicBlock *TheBB, BasicBlock::iterator IP) {
     BB = TheBB;
     InsertPt = IP;
-  }
-
-  /// \brief Find the nearest point that dominates this use, and specify that
-  /// created instructions should be inserted at this point.
-  void SetInsertPoint(Use &U) {
-    Instruction *UseInst = cast<Instruction>(U.getUser());
-    if (PHINode *Phi = dyn_cast<PHINode>(UseInst)) {
-      BasicBlock *PredBB = Phi->getIncomingBlock(U);
-      assert(U != PredBB->getTerminator() && "critical edge not split");
-      SetInsertPoint(PredBB, PredBB->getTerminator());
-      return;
-    }
-    SetInsertPoint(UseInst);
+    if (IP != TheBB->end())
+      SetCurrentDebugLocation(IP->getDebugLoc());
   }
 
   /// \brief Set location information used by debugging information.
@@ -244,7 +234,8 @@ class IRBuilderBase {
   /// filled in with the null terminated string value specified.  The new global
   /// variable will be marked mergable with any others of the same contents.  If
   /// Name is specified, it is the name of the global variable created.
-  GlobalVariable *CreateGlobalString(StringRef Str, const Twine &Name = "");
+  GlobalVariable *CreateGlobalString(StringRef Str, const Twine &Name = "",
+                                     unsigned AddressSpace = 0);
 
   /// \brief Get a constant value representing either true or false.
   ConstantInt *getInt1(bool V) {
@@ -321,6 +312,11 @@ class IRBuilderBase {
     return Type::getInt64Ty(Context);
   }
 
+  /// \brief Fetch the type representing a 128-bit integer.
+  IntegerType *getInt128Ty() {
+    return Type::getInt128Ty(Context);
+  }
+  
   /// \brief Fetch the type representing an N-bit integer.
   IntegerType *getIntNTy(unsigned N) {
     return Type::getIntNTy(Context, N);
@@ -442,19 +438,40 @@ class IRBuilderBase {
 
   /// \brief Create a call to the experimental.gc.statepoint intrinsic to
   /// start a new statepoint sequence.
-  CallInst *CreateGCStatepoint(Value *ActualCallee,
-                               ArrayRef<Value *> CallArgs,
-                               ArrayRef<Value *> DeoptArgs,
-                               ArrayRef<Value *> GCArgs,
-                               const Twine &Name = "");
+  CallInst *CreateGCStatepointCall(uint64_t ID, uint32_t NumPatchBytes,
+                                   Value *ActualCallee,
+                                   ArrayRef<Value *> CallArgs,
+                                   ArrayRef<Value *> DeoptArgs,
+                                   ArrayRef<Value *> GCArgs,
+                                   const Twine &Name = "");
+
+  // \brief Conveninence function for the common case when CallArgs are filled
+  // in using makeArrayRef(CS.arg_begin(), CS.arg_end()); Use needs to be
+  // .get()'ed to get the Value pointer.
+  CallInst *CreateGCStatepointCall(uint64_t ID, uint32_t NumPatchBytes,
+                                   Value *ActualCallee, ArrayRef<Use> CallArgs,
+                                   ArrayRef<Value *> DeoptArgs,
+                                   ArrayRef<Value *> GCArgs,
+                                   const Twine &Name = "");
+
+  /// brief Create an invoke to the experimental.gc.statepoint intrinsic to
+  /// start a new statepoint sequence.
+  InvokeInst *
+  CreateGCStatepointInvoke(uint64_t ID, uint32_t NumPatchBytes,
+                           Value *ActualInvokee, BasicBlock *NormalDest,
+                           BasicBlock *UnwindDest, ArrayRef<Value *> InvokeArgs,
+                           ArrayRef<Value *> DeoptArgs,
+                           ArrayRef<Value *> GCArgs, const Twine &Name = "");
 
   // Conveninence function for the common case when CallArgs are filled in using
-  // makeArrayRef(CS.arg_begin(), .arg_end()); Use needs to be .get()'ed to get
-  // the Value *.
-  CallInst *CreateGCStatepoint(Value *ActualCallee, ArrayRef<Use> CallArgs,
-                               ArrayRef<Value *> DeoptArgs,
-                               ArrayRef<Value *> GCArgs,
-                               const Twine &Name = "");
+  // makeArrayRef(CS.arg_begin(), CS.arg_end()); Use needs to be .get()'ed to
+  // get the Value *.
+  InvokeInst *
+  CreateGCStatepointInvoke(uint64_t ID, uint32_t NumPatchBytes,
+                           Value *ActualInvokee, BasicBlock *NormalDest,
+                           BasicBlock *UnwindDest, ArrayRef<Use> InvokeArgs,
+                           ArrayRef<Value *> DeoptArgs,
+                           ArrayRef<Value *> GCArgs, const Twine &Name = "");
 
   /// \brief Create a call to the experimental.gc.result intrinsic to extract
   /// the result from a call wrapped in a statepoint.
@@ -473,7 +490,7 @@ class IRBuilderBase {
 private:
   /// \brief Create a call to a masked intrinsic with given Id.
   /// Masked intrinsic has only one overloaded type - data type.
-  CallInst *CreateMaskedIntrinsic(unsigned Id, ArrayRef<Value *> Ops,
+  CallInst *CreateMaskedIntrinsic(Intrinsic::ID Id, ArrayRef<Value *> Ops,
                                   Type *DataTy, const Twine &Name = "");
 
   Value *getCastedInt8PtrValue(Value *Ptr);
@@ -522,13 +539,6 @@ class IRBuilder : public IRBuilderBase, public Inserter {
   explicit IRBuilder(Instruction *IP, MDNode *FPMathTag = nullptr)
     : IRBuilderBase(IP->getContext(), FPMathTag), Folder() {
     SetInsertPoint(IP);
-    SetCurrentDebugLocation(IP->getDebugLoc());
-  }
-
-  explicit IRBuilder(Use &U, MDNode *FPMathTag = nullptr)
-    : IRBuilderBase(U->getContext(), FPMathTag), Folder() {
-    SetInsertPoint(U);
-    SetCurrentDebugLocation(cast<Instruction>(U.getUser())->getDebugLoc());
   }
 
   IRBuilder(BasicBlock *TheBB, BasicBlock::iterator IP, const T& F,
@@ -662,6 +672,38 @@ class IRBuilder : public IRBuilderBase, public Inserter {
     return Insert(ResumeInst::Create(Exn));
   }
 
+  CleanupReturnInst *CreateCleanupRet(BasicBlock *UnwindBB = nullptr,
+                               Value *RetVal = nullptr) {
+    return Insert(CleanupReturnInst::Create(Context, RetVal, UnwindBB));
+  }
+
+  CatchPadInst *CreateCatchPad(Type *Ty, BasicBlock *NormalDest,
+                                   BasicBlock *UnwindDest,
+                                   ArrayRef<Value *> Args,
+                                   const Twine &Name = "") {
+    return Insert(CatchPadInst::Create(Ty, NormalDest, UnwindDest, Args),
+                  Name);
+  }
+
+  CatchEndPadInst *CreateCatchEndPad(BasicBlock *UnwindBB = nullptr) {
+    return Insert(CatchEndPadInst::Create(Context, UnwindBB));
+  }
+
+  TerminatePadInst *CreateTerminatePad(BasicBlock *UnwindBB = nullptr,
+                                           ArrayRef<Value *> Args = {},
+                                           const Twine &Name = "") {
+    return Insert(TerminatePadInst::Create(Context, UnwindBB, Args), Name);
+  }
+
+  CleanupPadInst *CreateCleanupPad(Type *Ty, ArrayRef<Value *> Args,
+                                       const Twine &Name = "") {
+    return Insert(CleanupPadInst::Create(Ty, Args), Name);
+  }
+
+  CatchReturnInst *CreateCatchRet(BasicBlock *BB) {
+    return Insert(CatchReturnInst::Create(BB));
+  }
+
   UnreachableInst *CreateUnreachable() {
     return Insert(new UnreachableInst(Context));
   }
@@ -966,6 +1008,9 @@ class IRBuilder : public IRBuilderBase, public Inserter {
   LoadInst *CreateLoad(Value *Ptr, const Twine &Name = "") {
     return Insert(new LoadInst(Ptr), Name);
   }
+  LoadInst *CreateLoad(Type *Ty, Value *Ptr, const Twine &Name = "") {
+    return Insert(new LoadInst(Ty, Ptr), Name);
+  }
   LoadInst *CreateLoad(Value *Ptr, bool isVolatile, const Twine &Name = "") {
     return Insert(new LoadInst(Ptr, nullptr, isVolatile), Name);
   }
@@ -1161,8 +1206,9 @@ class IRBuilder : public IRBuilderBase, public Inserter {
 
   /// \brief Same as CreateGlobalString, but return a pointer with "i8*" type
   /// instead of a pointer to array of i8.
-  Value *CreateGlobalStringPtr(StringRef Str, const Twine &Name = "") {
-    GlobalVariable *gv = CreateGlobalString(Str, Name);
+  Value *CreateGlobalStringPtr(StringRef Str, const Twine &Name = "",
+                               unsigned AddressSpace = 0) {
+    GlobalVariable *gv = CreateGlobalString(Str, Name, AddressSpace);
     Value *zero = ConstantInt::get(Type::getInt32Ty(Context), 0);
     Value *Args[] = { zero, zero };
     return CreateInBoundsGEP(gv->getValueType(), gv, Args, Name);
@@ -1368,47 +1414,61 @@ class IRBuilder : public IRBuilderBase, public Inserter {
     return CreateICmp(ICmpInst::ICMP_SLE, LHS, RHS, Name);
   }
 
-  Value *CreateFCmpOEQ(Value *LHS, Value *RHS, const Twine &Name = "") {
-    return CreateFCmp(FCmpInst::FCMP_OEQ, LHS, RHS, Name);
+  Value *CreateFCmpOEQ(Value *LHS, Value *RHS, const Twine &Name = "",
+                       MDNode *FPMathTag = nullptr) {
+    return CreateFCmp(FCmpInst::FCMP_OEQ, LHS, RHS, Name, FPMathTag);
   }
-  Value *CreateFCmpOGT(Value *LHS, Value *RHS, const Twine &Name = "") {
-    return CreateFCmp(FCmpInst::FCMP_OGT, LHS, RHS, Name);
+  Value *CreateFCmpOGT(Value *LHS, Value *RHS, const Twine &Name = "",
+                       MDNode *FPMathTag = nullptr) {
+    return CreateFCmp(FCmpInst::FCMP_OGT, LHS, RHS, Name, FPMathTag);
   }
-  Value *CreateFCmpOGE(Value *LHS, Value *RHS, const Twine &Name = "") {
-    return CreateFCmp(FCmpInst::FCMP_OGE, LHS, RHS, Name);
+  Value *CreateFCmpOGE(Value *LHS, Value *RHS, const Twine &Name = "",
+                       MDNode *FPMathTag = nullptr) {
+    return CreateFCmp(FCmpInst::FCMP_OGE, LHS, RHS, Name, FPMathTag);
   }
-  Value *CreateFCmpOLT(Value *LHS, Value *RHS, const Twine &Name = "") {
-    return CreateFCmp(FCmpInst::FCMP_OLT, LHS, RHS, Name);
+  Value *CreateFCmpOLT(Value *LHS, Value *RHS, const Twine &Name = "",
+                       MDNode *FPMathTag = nullptr) {
+    return CreateFCmp(FCmpInst::FCMP_OLT, LHS, RHS, Name, FPMathTag);
   }
-  Value *CreateFCmpOLE(Value *LHS, Value *RHS, const Twine &Name = "") {
-    return CreateFCmp(FCmpInst::FCMP_OLE, LHS, RHS, Name);
+  Value *CreateFCmpOLE(Value *LHS, Value *RHS, const Twine &Name = "",
+                       MDNode *FPMathTag = nullptr) {
+    return CreateFCmp(FCmpInst::FCMP_OLE, LHS, RHS, Name, FPMathTag);
   }
-  Value *CreateFCmpONE(Value *LHS, Value *RHS, const Twine &Name = "") {
-    return CreateFCmp(FCmpInst::FCMP_ONE, LHS, RHS, Name);
+  Value *CreateFCmpONE(Value *LHS, Value *RHS, const Twine &Name = "",
+                       MDNode *FPMathTag = nullptr) {
+    return CreateFCmp(FCmpInst::FCMP_ONE, LHS, RHS, Name, FPMathTag);
   }
-  Value *CreateFCmpORD(Value *LHS, Value *RHS, const Twine &Name = "") {
-    return CreateFCmp(FCmpInst::FCMP_ORD, LHS, RHS, Name);
+  Value *CreateFCmpORD(Value *LHS, Value *RHS, const Twine &Name = "",
+                       MDNode *FPMathTag = nullptr) {
+    return CreateFCmp(FCmpInst::FCMP_ORD, LHS, RHS, Name, FPMathTag);
   }
-  Value *CreateFCmpUNO(Value *LHS, Value *RHS, const Twine &Name = "") {
-    return CreateFCmp(FCmpInst::FCMP_UNO, LHS, RHS, Name);
+  Value *CreateFCmpUNO(Value *LHS, Value *RHS, const Twine &Name = "",
+                       MDNode *FPMathTag = nullptr) {
+    return CreateFCmp(FCmpInst::FCMP_UNO, LHS, RHS, Name, FPMathTag);
   }
-  Value *CreateFCmpUEQ(Value *LHS, Value *RHS, const Twine &Name = "") {
-    return CreateFCmp(FCmpInst::FCMP_UEQ, LHS, RHS, Name);
+  Value *CreateFCmpUEQ(Value *LHS, Value *RHS, const Twine &Name = "",
+                       MDNode *FPMathTag = nullptr) {
+    return CreateFCmp(FCmpInst::FCMP_UEQ, LHS, RHS, Name, FPMathTag);
   }
-  Value *CreateFCmpUGT(Value *LHS, Value *RHS, const Twine &Name = "") {
-    return CreateFCmp(FCmpInst::FCMP_UGT, LHS, RHS, Name);
+  Value *CreateFCmpUGT(Value *LHS, Value *RHS, const Twine &Name = "",
+                       MDNode *FPMathTag = nullptr) {
+    return CreateFCmp(FCmpInst::FCMP_UGT, LHS, RHS, Name, FPMathTag);
   }
-  Value *CreateFCmpUGE(Value *LHS, Value *RHS, const Twine &Name = "") {
-    return CreateFCmp(FCmpInst::FCMP_UGE, LHS, RHS, Name);
+  Value *CreateFCmpUGE(Value *LHS, Value *RHS, const Twine &Name = "",
+                       MDNode *FPMathTag = nullptr) {
+    return CreateFCmp(FCmpInst::FCMP_UGE, LHS, RHS, Name, FPMathTag);
   }
-  Value *CreateFCmpULT(Value *LHS, Value *RHS, const Twine &Name = "") {
-    return CreateFCmp(FCmpInst::FCMP_ULT, LHS, RHS, Name);
+  Value *CreateFCmpULT(Value *LHS, Value *RHS, const Twine &Name = "",
+                       MDNode *FPMathTag = nullptr) {
+    return CreateFCmp(FCmpInst::FCMP_ULT, LHS, RHS, Name, FPMathTag);
   }
-  Value *CreateFCmpULE(Value *LHS, Value *RHS, const Twine &Name = "") {
-    return CreateFCmp(FCmpInst::FCMP_ULE, LHS, RHS, Name);
+  Value *CreateFCmpULE(Value *LHS, Value *RHS, const Twine &Name = "",
+                       MDNode *FPMathTag = nullptr) {
+    return CreateFCmp(FCmpInst::FCMP_ULE, LHS, RHS, Name, FPMathTag);
   }
-  Value *CreateFCmpUNE(Value *LHS, Value *RHS, const Twine &Name = "") {
-    return CreateFCmp(FCmpInst::FCMP_UNE, LHS, RHS, Name);
+  Value *CreateFCmpUNE(Value *LHS, Value *RHS, const Twine &Name = "",
+                       MDNode *FPMathTag = nullptr) {
+    return CreateFCmp(FCmpInst::FCMP_UNE, LHS, RHS, Name, FPMathTag);
   }
 
   Value *CreateICmp(CmpInst::Predicate P, Value *LHS, Value *RHS,
@@ -1419,11 +1479,12 @@ class IRBuilder : public IRBuilderBase, public Inserter {
     return Insert(new ICmpInst(P, LHS, RHS), Name);
   }
   Value *CreateFCmp(CmpInst::Predicate P, Value *LHS, Value *RHS,
-                    const Twine &Name = "") {
+                    const Twine &Name = "", MDNode *FPMathTag = nullptr) {
     if (Constant *LC = dyn_cast<Constant>(LHS))
       if (Constant *RC = dyn_cast<Constant>(RHS))
         return Insert(Folder.CreateFCmp(P, LC, RC), Name);
-    return Insert(new FCmpInst(P, LHS, RHS), Name);
+    return Insert(AddFPMathAttributes(new FCmpInst(P, LHS, RHS),
+                                      FPMathTag, FMF), Name);
   }
 
   //===--------------------------------------------------------------------===//
@@ -1435,36 +1496,19 @@ class IRBuilder : public IRBuilderBase, public Inserter {
     return Insert(PHINode::Create(Ty, NumReservedValues), Name);
   }
 
-  CallInst *CreateCall(Value *Callee, const Twine &Name = "") {
-    return Insert(CallInst::Create(Callee), Name);
-  }
-  CallInst *CreateCall(Value *Callee, Value *Arg, const Twine &Name = "") {
-    return Insert(CallInst::Create(Callee, Arg), Name);
-  }
-  CallInst *CreateCall2(Value *Callee, Value *Arg1, Value *Arg2,
-                        const Twine &Name = "") {
-    Value *Args[] = { Arg1, Arg2 };
-    return Insert(CallInst::Create(Callee, Args), Name);
-  }
-  CallInst *CreateCall3(Value *Callee, Value *Arg1, Value *Arg2, Value *Arg3,
-                        const Twine &Name = "") {
-    Value *Args[] = { Arg1, Arg2, Arg3 };
-    return Insert(CallInst::Create(Callee, Args), Name);
-  }
-  CallInst *CreateCall4(Value *Callee, Value *Arg1, Value *Arg2, Value *Arg3,
-                        Value *Arg4, const Twine &Name = "") {
-    Value *Args[] = { Arg1, Arg2, Arg3, Arg4 };
+  CallInst *CreateCall(Value *Callee, ArrayRef<Value *> Args = None,
+                       const Twine &Name = "") {
     return Insert(CallInst::Create(Callee, Args), Name);
   }
-  CallInst *CreateCall5(Value *Callee, Value *Arg1, Value *Arg2, Value *Arg3,
-                        Value *Arg4, Value *Arg5, const Twine &Name = "") {
-    Value *Args[] = { Arg1, Arg2, Arg3, Arg4, Arg5 };
-    return Insert(CallInst::Create(Callee, Args), Name);
+
+  CallInst *CreateCall(llvm::FunctionType *FTy, Value *Callee,
+                       ArrayRef<Value *> Args, const Twine &Name = "") {
+    return Insert(CallInst::Create(FTy, Callee, Args), Name);
   }
 
-  CallInst *CreateCall(Value *Callee, ArrayRef<Value *> Args,
+  CallInst *CreateCall(Function *Callee, ArrayRef<Value *> Args,
                        const Twine &Name = "") {
-    return Insert(CallInst::Create(Callee, Args), Name);
+    return CreateCall(Callee->getFunctionType(), Callee, Args, Name);
   }
 
   Value *CreateSelect(Value *C, Value *True, Value *False,
@@ -1488,6 +1532,11 @@ class IRBuilder : public IRBuilderBase, public Inserter {
     return Insert(ExtractElementInst::Create(Vec, Idx), Name);
   }
 
+  Value *CreateExtractElement(Value *Vec, uint64_t Idx,
+                              const Twine &Name = "") {
+    return CreateExtractElement(Vec, getInt64(Idx), Name);
+  }
+
   Value *CreateInsertElement(Value *Vec, Value *NewElt, Value *Idx,
                              const Twine &Name = "") {
     if (Constant *VC = dyn_cast<Constant>(Vec))
@@ -1497,6 +1546,11 @@ class IRBuilder : public IRBuilderBase, public Inserter {
     return Insert(InsertElementInst::Create(Vec, NewElt, Idx), Name);
   }
 
+  Value *CreateInsertElement(Value *Vec, Value *NewElt, uint64_t Idx,
+                             const Twine &Name = "") {
+    return CreateInsertElement(Vec, NewElt, getInt64(Idx), Name);
+  }
+
   Value *CreateShuffleVector(Value *V1, Value *V2, Value *Mask,
                              const Twine &Name = "") {
     if (Constant *V1C = dyn_cast<Constant>(V1))
@@ -1533,9 +1587,9 @@ class IRBuilder : public IRBuilderBase, public Inserter {
     return Insert(InsertValueInst::Create(Agg, Val, Idxs), Name);
   }
 
-  LandingPadInst *CreateLandingPad(Type *Ty, Value *PersFn, unsigned NumClauses,
+  LandingPadInst *CreateLandingPad(Type *Ty, unsigned NumClauses,
                                    const Twine &Name = "") {
-    return Insert(LandingPadInst::Create(Ty, PersFn, NumClauses), Name);
+    return Insert(LandingPadInst::Create(Ty, NumClauses), Name);
   }
 
   //===--------------------------------------------------------------------===//
diff --git a/include/llvm/IR/IRPrintingPasses.h b/include/llvm/IR/IRPrintingPasses.h
index 5f1d56f7e83..88b18e826da 100644
--- a/include/llvm/IR/IRPrintingPasses.h
+++ b/include/llvm/IR/IRPrintingPasses.h
@@ -47,6 +47,12 @@ FunctionPass *createPrintFunctionPass(raw_ostream &OS,
 BasicBlockPass *createPrintBasicBlockPass(raw_ostream &OS,
                                           const std::string &Banner = "");
 
+/// Print out a name of an LLVM value without any prefixes.
+///
+/// The name is surrounded with ""'s and escaped if it has any special or
+/// non-printable characters in it.
+void printLLVMNameWithoutPrefix(raw_ostream &OS, StringRef Name);
+
 /// \brief Pass for printing a Module as LLVM's text IR assembly.
 ///
 /// Note: This pass is for use with the new pass manager. Use the create...Pass
diff --git a/include/llvm/IR/InlineAsm.h b/include/llvm/IR/InlineAsm.h
index 6decc210586..aa0eccf666d 100644
--- a/include/llvm/IR/InlineAsm.h
+++ b/include/llvm/IR/InlineAsm.h
@@ -44,11 +44,12 @@ class InlineAsm : public Value {
   void operator=(const InlineAsm&) = delete;
 
   std::string AsmString, Constraints;
+  FunctionType *FTy;
   bool HasSideEffects;
   bool IsAlignStack;
   AsmDialect Dialect;
 
-  InlineAsm(PointerType *Ty, const std::string &AsmString,
+  InlineAsm(FunctionType *Ty, const std::string &AsmString,
             const std::string &Constraints, bool hasSideEffects,
             bool isAlignStack, AsmDialect asmDialect);
   ~InlineAsm() override;
@@ -88,13 +89,6 @@ class InlineAsm : public Value {
   ///
   static bool Verify(FunctionType *Ty, StringRef Constraints);
 
-  // @LOCALMOD-START
-  /// isAsmMemory - Returns true if the Instruction corresponds to
-  /// ``asm("":::"memory")``, which is often used as a compiler barrier.
-  ///
-  bool isAsmMemory() const;
-  // @LOCALMOD-END
-
   // Constraint String Parsing 
   enum ConstraintPrefix {
     isInput,            // 'x'
@@ -255,6 +249,14 @@ class InlineAsm : public Value {
     Constraint_R,
     Constraint_S,
     Constraint_T,
+    Constraint_Um,
+    Constraint_Un,
+    Constraint_Uq,
+    Constraint_Us,
+    Constraint_Ut,
+    Constraint_Uv,
+    Constraint_Uy,
+    Constraint_X,
     Constraint_Z,
     Constraint_ZC,
     Constraint_Zy,
diff --git a/include/llvm/IR/InstVisitor.h b/include/llvm/IR/InstVisitor.h
index 1cdcd55448c..b6322044624 100644
--- a/include/llvm/IR/InstVisitor.h
+++ b/include/llvm/IR/InstVisitor.h
@@ -169,6 +169,11 @@ class InstVisitor {
   RetTy visitIndirectBrInst(IndirectBrInst &I)    { DELEGATE(TerminatorInst);}
   RetTy visitResumeInst(ResumeInst &I)            { DELEGATE(TerminatorInst);}
   RetTy visitUnreachableInst(UnreachableInst &I)  { DELEGATE(TerminatorInst);}
+  RetTy visitCleanupReturnInst(CleanupReturnInst &I) { DELEGATE(TerminatorInst);}
+  RetTy visitCatchReturnInst(CatchReturnInst &I)  { DELEGATE(TerminatorInst);}
+  RetTy visitCatchPadInst(CatchPadInst &I)    { DELEGATE(TerminatorInst);}
+  RetTy visitCatchEndPadInst(CatchEndPadInst &I) { DELEGATE(TerminatorInst); }
+  RetTy visitTerminatePadInst(TerminatePadInst &I) { DELEGATE(TerminatorInst);}
   RetTy visitICmpInst(ICmpInst &I)                { DELEGATE(CmpInst);}
   RetTy visitFCmpInst(FCmpInst &I)                { DELEGATE(CmpInst);}
   RetTy visitAllocaInst(AllocaInst &I)            { DELEGATE(UnaryInstruction);}
@@ -200,6 +205,7 @@ class InstVisitor {
   RetTy visitExtractValueInst(ExtractValueInst &I){ DELEGATE(UnaryInstruction);}
   RetTy visitInsertValueInst(InsertValueInst &I)  { DELEGATE(Instruction); }
   RetTy visitLandingPadInst(LandingPadInst &I)    { DELEGATE(Instruction); }
+  RetTy visitCleanupPadInst(CleanupPadInst &I) { DELEGATE(Instruction); }
 
   // Handle the special instrinsic instruction classes.
   RetTy visitDbgDeclareInst(DbgDeclareInst &I)    { DELEGATE(DbgInfoIntrinsic);}
@@ -259,7 +265,7 @@ class InstVisitor {
   // Special helper function to delegate to CallInst subclass visitors.
   RetTy delegateCallInst(CallInst &I) {
     if (const Function *F = I.getCalledFunction()) {
-      switch ((Intrinsic::ID)F->getIntrinsicID()) {
+      switch (F->getIntrinsicID()) {
       default:                     DELEGATE(IntrinsicInst);
       case Intrinsic::dbg_declare: DELEGATE(DbgDeclareInst);
       case Intrinsic::dbg_value:   DELEGATE(DbgValueInst);
diff --git a/include/llvm/IR/InstrTypes.h b/include/llvm/IR/InstrTypes.h
index d3cda6aca2a..ed12dd75b91 100644
--- a/include/llvm/IR/InstrTypes.h
+++ b/include/llvm/IR/InstrTypes.h
@@ -75,6 +75,22 @@ class TerminatorInst : public Instruction {
   static inline bool classof(const Value *V) {
     return isa<Instruction>(V) && classof(cast<Instruction>(V));
   }
+
+  // \brief Returns true if this terminator relates to exception handling.
+  bool isExceptional() const {
+    switch (getOpcode()) {
+    case Instruction::CatchPad:
+    case Instruction::CatchEndPad:
+    case Instruction::CatchRet:
+    case Instruction::CleanupRet:
+    case Instruction::Invoke:
+    case Instruction::Resume:
+    case Instruction::TerminatePad:
+      return true;
+    default:
+      return false;
+    }
+  }
 };
 
 
@@ -139,7 +155,11 @@ class BinaryOperator : public Instruction {
                  const Twine &Name, Instruction *InsertBefore);
   BinaryOperator(BinaryOps iType, Value *S1, Value *S2, Type *Ty,
                  const Twine &Name, BasicBlock *InsertAtEnd);
-  BinaryOperator *clone_impl() const override;
+
+  // Note: Instruction needs to be a friend here to call cloneImpl.
+  friend class Instruction;
+  BinaryOperator *cloneImpl() const;
+
 public:
   // allocate space for exactly two operands
   void *operator new(size_t s) {
@@ -655,13 +675,7 @@ class CastInst : public UnaryInstruction {
   /// Opcode op is valid or not.
   /// @returns true iff the proposed cast is valid.
   /// @brief Determine if a cast is valid without creating one.
-  // @LOCALMOD-BEGIN
-  static bool castIsValid(Instruction::CastOps op, Type *SrcTy, Type *DstTy);
-
-  static bool castIsValid(Instruction::CastOps op, Value *S, Type *DstTy) {
-    return castIsValid(op, S->getType(), DstTy);
-  }
-  // @LOCALMOD-END
+  static bool castIsValid(Instruction::CastOps op, Value *S, Type *DstTy);
 
   /// @brief Methods for support type inquiry through isa, cast, and dyn_cast:
   static inline bool classof(const Instruction *I) {
diff --git a/include/llvm/IR/Instruction.def b/include/llvm/IR/Instruction.def
index d46314cc761..9af666204da 100644
--- a/include/llvm/IR/Instruction.def
+++ b/include/llvm/IR/Instruction.def
@@ -94,86 +94,92 @@
 // instructions for it to be a well formed basic block.
 //
  FIRST_TERM_INST  ( 1)
-HANDLE_TERM_INST  ( 1, Ret        , ReturnInst)
-HANDLE_TERM_INST  ( 2, Br         , BranchInst)
-HANDLE_TERM_INST  ( 3, Switch     , SwitchInst)
-HANDLE_TERM_INST  ( 4, IndirectBr , IndirectBrInst)
-HANDLE_TERM_INST  ( 5, Invoke     , InvokeInst)
-HANDLE_TERM_INST  ( 6, Resume     , ResumeInst)
-HANDLE_TERM_INST  ( 7, Unreachable, UnreachableInst)
-  LAST_TERM_INST  ( 7)
+HANDLE_TERM_INST  ( 1, Ret           , ReturnInst)
+HANDLE_TERM_INST  ( 2, Br            , BranchInst)
+HANDLE_TERM_INST  ( 3, Switch        , SwitchInst)
+HANDLE_TERM_INST  ( 4, IndirectBr    , IndirectBrInst)
+HANDLE_TERM_INST  ( 5, Invoke        , InvokeInst)
+HANDLE_TERM_INST  ( 6, Resume        , ResumeInst)
+HANDLE_TERM_INST  ( 7, Unreachable   , UnreachableInst)
+HANDLE_TERM_INST  ( 8, CleanupRet    , CleanupReturnInst)
+HANDLE_TERM_INST  ( 9, CatchRet      , CatchReturnInst)
+HANDLE_TERM_INST  (10, CatchPad    , CatchPadInst)
+HANDLE_TERM_INST  (11, TerminatePad, TerminatePadInst)
+HANDLE_TERM_INST  (12, CatchEndPad , CatchEndPadInst)
+  LAST_TERM_INST  (12)
 
 // Standard binary operators...
- FIRST_BINARY_INST( 8)
-HANDLE_BINARY_INST( 8, Add  , BinaryOperator)
-HANDLE_BINARY_INST( 9, FAdd , BinaryOperator)
-HANDLE_BINARY_INST(10, Sub  , BinaryOperator)
-HANDLE_BINARY_INST(11, FSub , BinaryOperator)
-HANDLE_BINARY_INST(12, Mul  , BinaryOperator)
-HANDLE_BINARY_INST(13, FMul , BinaryOperator)
-HANDLE_BINARY_INST(14, UDiv , BinaryOperator)
-HANDLE_BINARY_INST(15, SDiv , BinaryOperator)
-HANDLE_BINARY_INST(16, FDiv , BinaryOperator)
-HANDLE_BINARY_INST(17, URem , BinaryOperator)
-HANDLE_BINARY_INST(18, SRem , BinaryOperator)
-HANDLE_BINARY_INST(19, FRem , BinaryOperator)
+ FIRST_BINARY_INST(13)
+HANDLE_BINARY_INST(13, Add  , BinaryOperator)
+HANDLE_BINARY_INST(14, FAdd , BinaryOperator)
+HANDLE_BINARY_INST(15, Sub  , BinaryOperator)
+HANDLE_BINARY_INST(16, FSub , BinaryOperator)
+HANDLE_BINARY_INST(17, Mul  , BinaryOperator)
+HANDLE_BINARY_INST(18, FMul , BinaryOperator)
+HANDLE_BINARY_INST(19, UDiv , BinaryOperator)
+HANDLE_BINARY_INST(20, SDiv , BinaryOperator)
+HANDLE_BINARY_INST(21, FDiv , BinaryOperator)
+HANDLE_BINARY_INST(22, URem , BinaryOperator)
+HANDLE_BINARY_INST(23, SRem , BinaryOperator)
+HANDLE_BINARY_INST(24, FRem , BinaryOperator)
 
 // Logical operators (integer operands)
-HANDLE_BINARY_INST(20, Shl  , BinaryOperator) // Shift left  (logical)
-HANDLE_BINARY_INST(21, LShr , BinaryOperator) // Shift right (logical)
-HANDLE_BINARY_INST(22, AShr , BinaryOperator) // Shift right (arithmetic)
-HANDLE_BINARY_INST(23, And  , BinaryOperator)
-HANDLE_BINARY_INST(24, Or   , BinaryOperator)
-HANDLE_BINARY_INST(25, Xor  , BinaryOperator)
-  LAST_BINARY_INST(25)
+HANDLE_BINARY_INST(25, Shl  , BinaryOperator) // Shift left  (logical)
+HANDLE_BINARY_INST(26, LShr , BinaryOperator) // Shift right (logical)
+HANDLE_BINARY_INST(27, AShr , BinaryOperator) // Shift right (arithmetic)
+HANDLE_BINARY_INST(28, And  , BinaryOperator)
+HANDLE_BINARY_INST(29, Or   , BinaryOperator)
+HANDLE_BINARY_INST(30, Xor  , BinaryOperator)
+  LAST_BINARY_INST(30)
 
 // Memory operators...
- FIRST_MEMORY_INST(26)
-HANDLE_MEMORY_INST(26, Alloca, AllocaInst)  // Stack management
-HANDLE_MEMORY_INST(27, Load  , LoadInst  )  // Memory manipulation instrs
-HANDLE_MEMORY_INST(28, Store , StoreInst )
-HANDLE_MEMORY_INST(29, GetElementPtr, GetElementPtrInst)
-HANDLE_MEMORY_INST(30, Fence , FenceInst )
-HANDLE_MEMORY_INST(31, AtomicCmpXchg , AtomicCmpXchgInst )
-HANDLE_MEMORY_INST(32, AtomicRMW , AtomicRMWInst )
-  LAST_MEMORY_INST(32)
+ FIRST_MEMORY_INST(31)
+HANDLE_MEMORY_INST(31, Alloca, AllocaInst)  // Stack management
+HANDLE_MEMORY_INST(32, Load  , LoadInst  )  // Memory manipulation instrs
+HANDLE_MEMORY_INST(33, Store , StoreInst )
+HANDLE_MEMORY_INST(34, GetElementPtr, GetElementPtrInst)
+HANDLE_MEMORY_INST(35, Fence , FenceInst )
+HANDLE_MEMORY_INST(36, AtomicCmpXchg , AtomicCmpXchgInst )
+HANDLE_MEMORY_INST(37, AtomicRMW , AtomicRMWInst )
+  LAST_MEMORY_INST(37)
 
 // Cast operators ...
 // NOTE: The order matters here because CastInst::isEliminableCastPair 
 // NOTE: (see Instructions.cpp) encodes a table based on this ordering.
- FIRST_CAST_INST(33)
-HANDLE_CAST_INST(33, Trunc   , TruncInst   )  // Truncate integers
-HANDLE_CAST_INST(34, ZExt    , ZExtInst    )  // Zero extend integers
-HANDLE_CAST_INST(35, SExt    , SExtInst    )  // Sign extend integers
-HANDLE_CAST_INST(36, FPToUI  , FPToUIInst  )  // floating point -> UInt
-HANDLE_CAST_INST(37, FPToSI  , FPToSIInst  )  // floating point -> SInt
-HANDLE_CAST_INST(38, UIToFP  , UIToFPInst  )  // UInt -> floating point
-HANDLE_CAST_INST(39, SIToFP  , SIToFPInst  )  // SInt -> floating point
-HANDLE_CAST_INST(40, FPTrunc , FPTruncInst )  // Truncate floating point
-HANDLE_CAST_INST(41, FPExt   , FPExtInst   )  // Extend floating point
-HANDLE_CAST_INST(42, PtrToInt, PtrToIntInst)  // Pointer -> Integer
-HANDLE_CAST_INST(43, IntToPtr, IntToPtrInst)  // Integer -> Pointer
-HANDLE_CAST_INST(44, BitCast , BitCastInst )  // Type cast
-HANDLE_CAST_INST(45, AddrSpaceCast, AddrSpaceCastInst)  // addrspace cast
-  LAST_CAST_INST(45)
+ FIRST_CAST_INST(38)
+HANDLE_CAST_INST(38, Trunc   , TruncInst   )  // Truncate integers
+HANDLE_CAST_INST(39, ZExt    , ZExtInst    )  // Zero extend integers
+HANDLE_CAST_INST(40, SExt    , SExtInst    )  // Sign extend integers
+HANDLE_CAST_INST(41, FPToUI  , FPToUIInst  )  // floating point -> UInt
+HANDLE_CAST_INST(42, FPToSI  , FPToSIInst  )  // floating point -> SInt
+HANDLE_CAST_INST(43, UIToFP  , UIToFPInst  )  // UInt -> floating point
+HANDLE_CAST_INST(44, SIToFP  , SIToFPInst  )  // SInt -> floating point
+HANDLE_CAST_INST(45, FPTrunc , FPTruncInst )  // Truncate floating point
+HANDLE_CAST_INST(46, FPExt   , FPExtInst   )  // Extend floating point
+HANDLE_CAST_INST(47, PtrToInt, PtrToIntInst)  // Pointer -> Integer
+HANDLE_CAST_INST(48, IntToPtr, IntToPtrInst)  // Integer -> Pointer
+HANDLE_CAST_INST(49, BitCast , BitCastInst )  // Type cast
+HANDLE_CAST_INST(50, AddrSpaceCast, AddrSpaceCastInst)  // addrspace cast
+  LAST_CAST_INST(50)
 
 // Other operators...
- FIRST_OTHER_INST(46)
-HANDLE_OTHER_INST(46, ICmp   , ICmpInst   )  // Integer comparison instruction
-HANDLE_OTHER_INST(47, FCmp   , FCmpInst   )  // Floating point comparison instr.
-HANDLE_OTHER_INST(48, PHI    , PHINode    )  // PHI node instruction
-HANDLE_OTHER_INST(49, Call   , CallInst   )  // Call a function
-HANDLE_OTHER_INST(50, Select , SelectInst )  // select instruction
-HANDLE_OTHER_INST(51, UserOp1, Instruction)  // May be used internally in a pass
-HANDLE_OTHER_INST(52, UserOp2, Instruction)  // Internal to passes only
-HANDLE_OTHER_INST(53, VAArg  , VAArgInst  )  // vaarg instruction
-HANDLE_OTHER_INST(54, ExtractElement, ExtractElementInst)// extract from vector
-HANDLE_OTHER_INST(55, InsertElement, InsertElementInst)  // insert into vector
-HANDLE_OTHER_INST(56, ShuffleVector, ShuffleVectorInst)  // shuffle two vectors.
-HANDLE_OTHER_INST(57, ExtractValue, ExtractValueInst)// extract from aggregate
-HANDLE_OTHER_INST(58, InsertValue, InsertValueInst)  // insert into aggregate
-HANDLE_OTHER_INST(59, LandingPad, LandingPadInst)  // Landing pad instruction.
-  LAST_OTHER_INST(59)
+ FIRST_OTHER_INST(51)
+HANDLE_OTHER_INST(51, ICmp   , ICmpInst   )  // Integer comparison instruction
+HANDLE_OTHER_INST(52, FCmp   , FCmpInst   )  // Floating point comparison instr.
+HANDLE_OTHER_INST(53, PHI    , PHINode    )  // PHI node instruction
+HANDLE_OTHER_INST(54, Call   , CallInst   )  // Call a function
+HANDLE_OTHER_INST(55, Select , SelectInst )  // select instruction
+HANDLE_OTHER_INST(56, UserOp1, Instruction)  // May be used internally in a pass
+HANDLE_OTHER_INST(57, UserOp2, Instruction)  // Internal to passes only
+HANDLE_OTHER_INST(58, VAArg  , VAArgInst  )  // vaarg instruction
+HANDLE_OTHER_INST(59, ExtractElement, ExtractElementInst)// extract from vector
+HANDLE_OTHER_INST(60, InsertElement, InsertElementInst)  // insert into vector
+HANDLE_OTHER_INST(61, ShuffleVector, ShuffleVectorInst)  // shuffle two vectors.
+HANDLE_OTHER_INST(62, ExtractValue, ExtractValueInst)// extract from aggregate
+HANDLE_OTHER_INST(63, InsertValue, InsertValueInst)  // insert into aggregate
+HANDLE_OTHER_INST(64, LandingPad, LandingPadInst)  // Landing pad instruction.
+HANDLE_OTHER_INST(65, CleanupPad, CleanupPadInst)
+  LAST_OTHER_INST(65)
 
 #undef  FIRST_TERM_INST
 #undef HANDLE_TERM_INST
diff --git a/include/llvm/IR/Instruction.h b/include/llvm/IR/Instruction.h
index 9dd16fd5a60..3c48efb4561 100644
--- a/include/llvm/IR/Instruction.h
+++ b/include/llvm/IR/Instruction.h
@@ -78,6 +78,7 @@ class Instruction : public User, public ilist_node<Instruction> {
   /// Note: this is undefined behavior if the instruction does not have a
   /// parent, or the parent basic block does not have a parent function.
   const Module *getModule() const;
+  Module *getModule();
 
   /// removeFromParent - This method unlinks 'this' from the containing basic
   /// block, but does not delete it.
@@ -90,12 +91,12 @@ class Instruction : public User, public ilist_node<Instruction> {
   /// \returns an iterator pointing to the element after the erased one
   iplist<Instruction>::iterator eraseFromParent();
 
-  /// insertBefore - Insert an unlinked instructions into a basic block
-  /// immediately before the specified instruction.
+  /// Insert an unlinked instruction into a basic block immediately before
+  /// the specified instruction.
   void insertBefore(Instruction *InsertPos);
 
-  /// insertAfter - Insert an unlinked instructions into a basic block
-  /// immediately after the specified instruction.
+  /// Insert an unlinked instruction into a basic block immediately after the
+  /// specified instruction.
   void insertAfter(Instruction *InsertPos);
 
   /// moveBefore - Unlink this instruction from its current basic block and
@@ -381,12 +382,26 @@ class Instruction : public User, public ilist_node<Instruction> {
   ///
   /// Note that this does not consider malloc and alloca to have side
   /// effects because the newly allocated memory is completely invisible to
-  /// instructions which don't used the returned value.  For cases where this
+  /// instructions which don't use the returned value.  For cases where this
   /// matters, isSafeToSpeculativelyExecute may be more appropriate.
   bool mayHaveSideEffects() const {
     return mayWriteToMemory() || mayThrow() || !mayReturn();
   }
 
+  /// \brief Return true if the instruction is a variety of EH-block.
+  bool isEHPad() const {
+    switch (getOpcode()) {
+    case Instruction::CatchPad:
+    case Instruction::CatchEndPad:
+    case Instruction::CleanupPad:
+    case Instruction::LandingPad:
+    case Instruction::TerminatePad:
+      return true;
+    default:
+      return false;
+    }
+  }
+
   /// clone() - Create a copy of 'this' instruction that is identical in all
   /// ways except the following:
   ///   * The instruction has no parent
@@ -508,8 +523,10 @@ class Instruction : public User, public ilist_node<Instruction> {
               Instruction *InsertBefore = nullptr);
   Instruction(Type *Ty, unsigned iType, Use *Ops, unsigned NumOps,
               BasicBlock *InsertAtEnd);
-  virtual Instruction *clone_impl() const = 0;
 
+private:
+  /// Create a copy of this instruction.
+  Instruction *cloneImpl() const;
 };
 
 inline Instruction *ilist_traits<Instruction>::createSentinel() const {
diff --git a/include/llvm/IR/Instructions.h b/include/llvm/IR/Instructions.h
index 1f2ca301d24..4330f2f87a7 100644
--- a/include/llvm/IR/Instructions.h
+++ b/include/llvm/IR/Instructions.h
@@ -22,6 +22,7 @@
 #include "llvm/IR/Attributes.h"
 #include "llvm/IR/CallingConv.h"
 #include "llvm/IR/DerivedTypes.h"
+#include "llvm/IR/Function.h"
 #include "llvm/IR/InstrTypes.h"
 #include "llvm/Support/ErrorHandling.h"
 #include <iterator>
@@ -73,8 +74,13 @@ return (Ord == Release ||
 /// AllocaInst - an instruction to allocate memory on the stack
 ///
 class AllocaInst : public UnaryInstruction {
+  Type *AllocatedType;
+
 protected:
-  AllocaInst *clone_impl() const override;
+  // Note: Instruction needs to be a friend here to call cloneImpl.
+  friend class Instruction;
+  AllocaInst *cloneImpl() const;
+
 public:
   explicit AllocaInst(Type *Ty, Value *ArraySize = nullptr,
                       const Twine &Name = "",
@@ -113,7 +119,10 @@ class AllocaInst : public UnaryInstruction {
   /// getAllocatedType - Return the type that is being allocated by the
   /// instruction.
   ///
-  Type *getAllocatedType() const;
+  Type *getAllocatedType() const { return AllocatedType; }
+  /// \brief for use only in special circumstances that need to generically
+  /// transform a whole instruction (eg: IR linking and vectorization).
+  void setAllocatedType(Type *Ty) { AllocatedType = Ty; }
 
   /// getAlignment - Return the alignment of the memory that is being allocated
   /// by the instruction.
@@ -168,15 +177,26 @@ class AllocaInst : public UnaryInstruction {
 class LoadInst : public UnaryInstruction {
   void AssertOK();
 protected:
-  LoadInst *clone_impl() const override;
+  // Note: Instruction needs to be a friend here to call cloneImpl.
+  friend class Instruction;
+  LoadInst *cloneImpl() const;
+
 public:
   LoadInst(Value *Ptr, const Twine &NameStr, Instruction *InsertBefore);
   LoadInst(Value *Ptr, const Twine &NameStr, BasicBlock *InsertAtEnd);
-  LoadInst(Value *Ptr, const Twine &NameStr, bool isVolatile = false,
+  LoadInst(Type *Ty, Value *Ptr, const Twine &NameStr, bool isVolatile = false,
            Instruction *InsertBefore = nullptr);
+  LoadInst(Value *Ptr, const Twine &NameStr, bool isVolatile = false,
+           Instruction *InsertBefore = nullptr)
+      : LoadInst(cast<PointerType>(Ptr->getType())->getElementType(), Ptr,
+                 NameStr, isVolatile, InsertBefore) {}
   LoadInst(Value *Ptr, const Twine &NameStr, bool isVolatile,
            BasicBlock *InsertAtEnd);
-  LoadInst(Value *Ptr, const Twine &NameStr, bool isVolatile,
+  LoadInst(Value *Ptr, const Twine &NameStr, bool isVolatile, unsigned Align,
+           Instruction *InsertBefore = nullptr)
+      : LoadInst(cast<PointerType>(Ptr->getType())->getElementType(), Ptr,
+                 NameStr, isVolatile, Align, InsertBefore) {}
+  LoadInst(Type *Ty, Value *Ptr, const Twine &NameStr, bool isVolatile,
            unsigned Align, Instruction *InsertBefore = nullptr);
   LoadInst(Value *Ptr, const Twine &NameStr, bool isVolatile,
            unsigned Align, BasicBlock *InsertAtEnd);
@@ -196,9 +216,13 @@ class LoadInst : public UnaryInstruction {
 
   LoadInst(Value *Ptr, const char *NameStr, Instruction *InsertBefore);
   LoadInst(Value *Ptr, const char *NameStr, BasicBlock *InsertAtEnd);
+  LoadInst(Type *Ty, Value *Ptr, const char *NameStr = nullptr,
+           bool isVolatile = false, Instruction *InsertBefore = nullptr);
   explicit LoadInst(Value *Ptr, const char *NameStr = nullptr,
                     bool isVolatile = false,
-                    Instruction *InsertBefore = nullptr);
+                    Instruction *InsertBefore = nullptr)
+      : LoadInst(cast<PointerType>(Ptr->getType())->getElementType(), Ptr,
+                 NameStr, isVolatile, InsertBefore) {}
   LoadInst(Value *Ptr, const char *NameStr, bool isVolatile,
            BasicBlock *InsertAtEnd);
 
@@ -293,7 +317,10 @@ class StoreInst : public Instruction {
   void *operator new(size_t, unsigned) = delete;
   void AssertOK();
 protected:
-  StoreInst *clone_impl() const override;
+  // Note: Instruction needs to be a friend here to call cloneImpl.
+  friend class Instruction;
+  StoreInst *cloneImpl() const;
+
 public:
   // allocate space for exactly two operands
   void *operator new(size_t s) {
@@ -419,7 +446,10 @@ class FenceInst : public Instruction {
   void *operator new(size_t, unsigned) = delete;
   void Init(AtomicOrdering Ordering, SynchronizationScope SynchScope);
 protected:
-  FenceInst *clone_impl() const override;
+  // Note: Instruction needs to be a friend here to call cloneImpl.
+  friend class Instruction;
+  FenceInst *cloneImpl() const;
+
 public:
   // allocate space for exactly zero operands
   void *operator new(size_t s) {
@@ -488,7 +518,10 @@ class AtomicCmpXchgInst : public Instruction {
             AtomicOrdering SuccessOrdering, AtomicOrdering FailureOrdering,
             SynchronizationScope SynchScope);
 protected:
-  AtomicCmpXchgInst *clone_impl() const override;
+  // Note: Instruction needs to be a friend here to call cloneImpl.
+  friend class Instruction;
+  AtomicCmpXchgInst *cloneImpl() const;
+
 public:
   // allocate space for exactly three operands
   void *operator new(size_t s) {
@@ -641,7 +674,10 @@ DEFINE_TRANSPARENT_OPERAND_ACCESSORS(AtomicCmpXchgInst, Value)
 class AtomicRMWInst : public Instruction {
   void *operator new(size_t, unsigned) = delete;
 protected:
-  AtomicRMWInst *clone_impl() const override;
+  // Note: Instruction needs to be a friend here to call cloneImpl.
+  friend class Instruction;
+  AtomicRMWInst *cloneImpl() const;
+
 public:
   /// This enumeration lists the possible modifications atomicrmw can make.  In
   /// the descriptions, 'p' is the pointer to the instruction's memory location,
@@ -792,6 +828,9 @@ inline Type *checkGEPType(Type *Ty) {
 /// access elements of arrays and structs
 ///
 class GetElementPtrInst : public Instruction {
+  Type *SourceElementType;
+  Type *ResultElementType;
+
   GetElementPtrInst(const GetElementPtrInst &GEPI);
   void init(Value *Ptr, ArrayRef<Value *> IdxList, const Twine &NameStr);
 
@@ -807,13 +846,23 @@ class GetElementPtrInst : public Instruction {
                            const Twine &NameStr, BasicBlock *InsertAtEnd);
 
 protected:
-  GetElementPtrInst *clone_impl() const override;
+  // Note: Instruction needs to be a friend here to call cloneImpl.
+  friend class Instruction;
+  GetElementPtrInst *cloneImpl() const;
+
 public:
   static GetElementPtrInst *Create(Type *PointeeType, Value *Ptr,
                                    ArrayRef<Value *> IdxList,
                                    const Twine &NameStr = "",
                                    Instruction *InsertBefore = nullptr) {
     unsigned Values = 1 + unsigned(IdxList.size());
+    if (!PointeeType)
+      PointeeType =
+          cast<PointerType>(Ptr->getType()->getScalarType())->getElementType();
+    else
+      assert(
+          PointeeType ==
+          cast<PointerType>(Ptr->getType()->getScalarType())->getElementType());
     return new (Values) GetElementPtrInst(PointeeType, Ptr, IdxList, Values,
                                           NameStr, InsertBefore);
   }
@@ -822,6 +871,13 @@ class GetElementPtrInst : public Instruction {
                                    const Twine &NameStr,
                                    BasicBlock *InsertAtEnd) {
     unsigned Values = 1 + unsigned(IdxList.size());
+    if (!PointeeType)
+      PointeeType =
+          cast<PointerType>(Ptr->getType()->getScalarType())->getElementType();
+    else
+      assert(
+          PointeeType ==
+          cast<PointerType>(Ptr->getType()->getScalarType())->getElementType());
     return new (Values) GetElementPtrInst(PointeeType, Ptr, IdxList, Values,
                                           NameStr, InsertAtEnd);
   }
@@ -867,12 +923,16 @@ class GetElementPtrInst : public Instruction {
     return cast<SequentialType>(Instruction::getType());
   }
 
-  Type *getSourceElementType() const {
-    return cast<SequentialType>(getPointerOperandType()->getScalarType())
-        ->getElementType();
-  }
+  Type *getSourceElementType() const { return SourceElementType; }
+
+  void setSourceElementType(Type *Ty) { SourceElementType = Ty; }
+  void setResultElementType(Type *Ty) { ResultElementType = Ty; }
 
-  Type *getResultElementType() const { return getType()->getElementType(); }
+  Type *getResultElementType() const {
+    assert(ResultElementType ==
+           cast<PointerType>(getType()->getScalarType())->getElementType());
+    return ResultElementType;
+  }
 
   /// \brief Returns the address space of this instruction's pointer type.
   unsigned getAddressSpace() const {
@@ -920,18 +980,24 @@ class GetElementPtrInst : public Instruction {
   /// GetGEPReturnType - Returns the pointer type returned by the GEP
   /// instruction, which may be a vector of pointers.
   static Type *getGEPReturnType(Value *Ptr, ArrayRef<Value *> IdxList) {
-    Type *PtrTy =
-        PointerType::get(checkGEPType(getIndexedType(
-                             cast<PointerType>(Ptr->getType()->getScalarType())
-                                 ->getElementType(),
-                             IdxList)),
-                         Ptr->getType()->getPointerAddressSpace());
+    return getGEPReturnType(
+        cast<PointerType>(Ptr->getType()->getScalarType())->getElementType(),
+        Ptr, IdxList);
+  }
+  static Type *getGEPReturnType(Type *ElTy, Value *Ptr,
+                                ArrayRef<Value *> IdxList) {
+    Type *PtrTy = PointerType::get(checkGEPType(getIndexedType(ElTy, IdxList)),
+                                   Ptr->getType()->getPointerAddressSpace());
     // Vector GEP
     if (Ptr->getType()->isVectorTy()) {
-      unsigned NumElem = cast<VectorType>(Ptr->getType())->getNumElements();
+      unsigned NumElem = Ptr->getType()->getVectorNumElements();
       return VectorType::get(PtrTy, NumElem);
     }
-
+    for (Value *Index : IdxList)
+      if (Index->getType()->isVectorTy()) {
+        unsigned NumElem = Index->getType()->getVectorNumElements();
+        return VectorType::get(PtrTy, NumElem);
+      }
     // Scalar GEP
     return PtrTy;
   }
@@ -989,21 +1055,27 @@ GetElementPtrInst::GetElementPtrInst(Type *PointeeType, Value *Ptr,
                                      ArrayRef<Value *> IdxList, unsigned Values,
                                      const Twine &NameStr,
                                      Instruction *InsertBefore)
-    : Instruction(getGEPReturnType(Ptr, IdxList), GetElementPtr,
+    : Instruction(getGEPReturnType(PointeeType, Ptr, IdxList), GetElementPtr,
                   OperandTraits<GetElementPtrInst>::op_end(this) - Values,
-                  Values, InsertBefore) {
+                  Values, InsertBefore),
+      SourceElementType(PointeeType),
+      ResultElementType(getIndexedType(PointeeType, IdxList)) {
+  assert(ResultElementType ==
+         cast<PointerType>(getType()->getScalarType())->getElementType());
   init(Ptr, IdxList, NameStr);
-  assert(!PointeeType || PointeeType == getSourceElementType());
 }
 GetElementPtrInst::GetElementPtrInst(Type *PointeeType, Value *Ptr,
                                      ArrayRef<Value *> IdxList, unsigned Values,
                                      const Twine &NameStr,
                                      BasicBlock *InsertAtEnd)
-    : Instruction(getGEPReturnType(Ptr, IdxList), GetElementPtr,
+    : Instruction(getGEPReturnType(PointeeType, Ptr, IdxList), GetElementPtr,
                   OperandTraits<GetElementPtrInst>::op_end(this) - Values,
-                  Values, InsertAtEnd) {
+                  Values, InsertAtEnd),
+      SourceElementType(PointeeType),
+      ResultElementType(getIndexedType(PointeeType, IdxList)) {
+  assert(ResultElementType ==
+         cast<PointerType>(getType()->getScalarType())->getElementType());
   init(Ptr, IdxList, NameStr);
-  assert(!PointeeType || PointeeType == getSourceElementType());
 }
 
 
@@ -1032,8 +1104,11 @@ class ICmpInst: public CmpInst {
   }
 
 protected:
+  // Note: Instruction needs to be a friend here to call cloneImpl.
+  friend class Instruction;
   /// \brief Clone an identical ICmpInst
-  ICmpInst *clone_impl() const override;
+  ICmpInst *cloneImpl() const;
+
 public:
   /// \brief Constructor with insert-before-instruction semantics.
   ICmpInst(
@@ -1164,8 +1239,11 @@ class ICmpInst: public CmpInst {
 /// \brief Represents a floating point comparison operator.
 class FCmpInst: public CmpInst {
 protected:
+  // Note: Instruction needs to be a friend here to call cloneImpl.
+  friend class Instruction;
   /// \brief Clone an identical FCmpInst
-  FCmpInst *clone_impl() const override;
+  FCmpInst *cloneImpl() const;
+
 public:
   /// \brief Constructor with insert-before-instruction semantics.
   FCmpInst(
@@ -1274,14 +1352,26 @@ class FCmpInst: public CmpInst {
 ///
 class CallInst : public Instruction {
   AttributeSet AttributeList; ///< parameter attributes for call
+  FunctionType *FTy;
   CallInst(const CallInst &CI);
-  void init(Value *Func, ArrayRef<Value *> Args, const Twine &NameStr);
+  void init(Value *Func, ArrayRef<Value *> Args, const Twine &NameStr) {
+    init(cast<FunctionType>(
+             cast<PointerType>(Func->getType())->getElementType()),
+         Func, Args, NameStr);
+  }
+  void init(FunctionType *FTy, Value *Func, ArrayRef<Value *> Args,
+            const Twine &NameStr);
   void init(Value *Func, const Twine &NameStr);
 
   /// Construct a CallInst given a range of arguments.
   /// \brief Construct a CallInst from a range of arguments
-  inline CallInst(Value *Func, ArrayRef<Value *> Args,
+  inline CallInst(FunctionType *Ty, Value *Func, ArrayRef<Value *> Args,
                   const Twine &NameStr, Instruction *InsertBefore);
+  inline CallInst(Value *Func, ArrayRef<Value *> Args, const Twine &NameStr,
+                  Instruction *InsertBefore)
+      : CallInst(cast<FunctionType>(
+                     cast<PointerType>(Func->getType())->getElementType()),
+                 Func, Args, NameStr, InsertBefore) {}
 
   /// Construct a CallInst given a range of arguments.
   /// \brief Construct a CallInst from a range of arguments
@@ -1292,14 +1382,24 @@ class CallInst : public Instruction {
                     Instruction *InsertBefore);
   CallInst(Value *F, const Twine &NameStr, BasicBlock *InsertAtEnd);
 protected:
-  CallInst *clone_impl() const override;
+  // Note: Instruction needs to be a friend here to call cloneImpl.
+  friend class Instruction;
+  CallInst *cloneImpl() const;
+
 public:
   static CallInst *Create(Value *Func,
                           ArrayRef<Value *> Args,
                           const Twine &NameStr = "",
                           Instruction *InsertBefore = nullptr) {
-    return new(unsigned(Args.size() + 1))
-      CallInst(Func, Args, NameStr, InsertBefore);
+    return Create(cast<FunctionType>(
+                      cast<PointerType>(Func->getType())->getElementType()),
+                  Func, Args, NameStr, InsertBefore);
+  }
+  static CallInst *Create(FunctionType *Ty, Value *Func, ArrayRef<Value *> Args,
+                          const Twine &NameStr = "",
+                          Instruction *InsertBefore = nullptr) {
+    return new (unsigned(Args.size() + 1))
+        CallInst(Ty, Func, Args, NameStr, InsertBefore);
   }
   static CallInst *Create(Value *Func,
                           ArrayRef<Value *> Args,
@@ -1337,8 +1437,11 @@ class CallInst : public Instruction {
 
   ~CallInst() override;
 
-  Type *getFunctionType() const {
-    return cast<PointerType>(getCalledValue()->getType())->getElementType();
+  FunctionType *getFunctionType() const { return FTy; }
+
+  void mutateFunctionType(FunctionType *FTy) {
+    mutateType(FTy->getReturnType());
+    this->FTy = FTy;
   }
 
   // Note that 'musttail' implies 'tail'.
@@ -1410,6 +1513,9 @@ class CallInst : public Instruction {
   /// addAttribute - adds the attribute to the list of attributes.
   void addAttribute(unsigned i, Attribute::AttrKind attr);
 
+  /// addAttribute - adds the attribute to the list of attributes.
+  void addAttribute(unsigned i, StringRef Kind, StringRef Value);
+
   /// removeAttribute - removes the attribute from the list of attributes.
   void removeAttribute(unsigned i, Attribute attr);
 
@@ -1427,6 +1533,11 @@ class CallInst : public Instruction {
     return hasFnAttrImpl(A);
   }
 
+  /// \brief Determine whether this call has the given attribute.
+  bool hasFnAttr(StringRef A) const {
+    return hasFnAttrImpl(A);
+  }
+
   /// \brief Determine whether the call or the callee has the given attributes.
   bool paramHasAttr(unsigned i, Attribute::AttrKind A) const;
 
@@ -1441,6 +1552,12 @@ class CallInst : public Instruction {
     return AttributeList.getDereferenceableBytes(i);
   }
 
+  /// \brief Extract the number of dereferenceable_or_null bytes for a call or
+  /// parameter (0=unknown).
+  uint64_t getDereferenceableOrNullBytes(unsigned i) const {
+    return AttributeList.getDereferenceableOrNullBytes(i);
+  }
+  
   /// \brief Return true if the call should not be treated as a call to a
   /// builtin.
   bool isNoBuiltin() const {
@@ -1478,6 +1595,15 @@ class CallInst : public Instruction {
     addAttribute(AttributeSet::FunctionIndex, Attribute::ReadOnly);
   }
 
+  /// @brief Determine if the call can access memmory only using pointers based
+  /// on its arguments.
+  bool onlyAccessesArgMemory() const {
+    return hasFnAttr(Attribute::ArgMemOnly);
+  }
+  void setOnlyAccessesArgMemory() {
+    addAttribute(AttributeSet::FunctionIndex, Attribute::ArgMemOnly);
+  }
+
   /// \brief Determine if the call cannot return.
   bool doesNotReturn() const { return hasFnAttr(Attribute::NoReturn); }
   void setDoesNotReturn() {
@@ -1522,6 +1648,14 @@ class CallInst : public Instruction {
 
   /// setCalledFunction - Set the function called.
   void setCalledFunction(Value* Fn) {
+    setCalledFunction(
+        cast<FunctionType>(cast<PointerType>(Fn->getType())->getElementType()),
+        Fn);
+  }
+  void setCalledFunction(FunctionType *FTy, Value *Fn) {
+    this->FTy = FTy;
+    assert(FTy == cast<FunctionType>(
+                      cast<PointerType>(Fn->getType())->getElementType()));
     Op<-1>() = Fn;
   }
 
@@ -1539,7 +1673,14 @@ class CallInst : public Instruction {
   }
 private:
 
-  bool hasFnAttrImpl(Attribute::AttrKind A) const;
+  template<typename AttrKind>
+  bool hasFnAttrImpl(AttrKind A) const {
+    if (AttributeList.hasAttribute(AttributeSet::FunctionIndex, A))
+      return true;
+    if (const Function *F = getCalledFunction())
+      return F->getAttributes().hasAttribute(AttributeSet::FunctionIndex, A);
+    return false;
+  }
 
   // Shadow Instruction::setInstructionSubclassData with a private forwarding
   // method so that subclasses cannot accidentally use it.
@@ -1562,14 +1703,12 @@ CallInst::CallInst(Value *Func, ArrayRef<Value *> Args,
   init(Func, Args, NameStr);
 }
 
-CallInst::CallInst(Value *Func, ArrayRef<Value *> Args,
+CallInst::CallInst(FunctionType *Ty, Value *Func, ArrayRef<Value *> Args,
                    const Twine &NameStr, Instruction *InsertBefore)
-  : Instruction(cast<FunctionType>(cast<PointerType>(Func->getType())
-                                   ->getElementType())->getReturnType(),
-                Instruction::Call,
-                OperandTraits<CallInst>::op_end(this) - (Args.size() + 1),
-                unsigned(Args.size() + 1), InsertBefore) {
-  init(Func, Args, NameStr);
+    : Instruction(Ty->getReturnType(), Instruction::Call,
+                  OperandTraits<CallInst>::op_end(this) - (Args.size() + 1),
+                  unsigned(Args.size() + 1), InsertBefore) {
+  init(Ty, Func, Args, NameStr);
 }
 
 
@@ -1607,7 +1746,10 @@ class SelectInst : public Instruction {
     setName(NameStr);
   }
 protected:
-  SelectInst *clone_impl() const override;
+  // Note: Instruction needs to be a friend here to call cloneImpl.
+  friend class Instruction;
+  SelectInst *cloneImpl() const;
+
 public:
   static SelectInst *Create(Value *C, Value *S1, Value *S2,
                             const Twine &NameStr = "",
@@ -1662,7 +1804,9 @@ DEFINE_TRANSPARENT_OPERAND_ACCESSORS(SelectInst, Value)
 ///
 class VAArgInst : public UnaryInstruction {
 protected:
-  VAArgInst *clone_impl() const override;
+  // Note: Instruction needs to be a friend here to call cloneImpl.
+  friend class Instruction;
+  VAArgInst *cloneImpl() const;
 
 public:
   VAArgInst(Value *List, Type *Ty, const Twine &NameStr = "",
@@ -1702,7 +1846,9 @@ class ExtractElementInst : public Instruction {
   ExtractElementInst(Value *Vec, Value *Idx, const Twine &NameStr,
                      BasicBlock *InsertAtEnd);
 protected:
-  ExtractElementInst *clone_impl() const override;
+  // Note: Instruction needs to be a friend here to call cloneImpl.
+  friend class Instruction;
+  ExtractElementInst *cloneImpl() const;
 
 public:
   static ExtractElementInst *Create(Value *Vec, Value *Idx,
@@ -1763,7 +1909,9 @@ class InsertElementInst : public Instruction {
   InsertElementInst(Value *Vec, Value *NewElt, Value *Idx,
                     const Twine &NameStr, BasicBlock *InsertAtEnd);
 protected:
-  InsertElementInst *clone_impl() const override;
+  // Note: Instruction needs to be a friend here to call cloneImpl.
+  friend class Instruction;
+  InsertElementInst *cloneImpl() const;
 
 public:
   static InsertElementInst *Create(Value *Vec, Value *NewElt, Value *Idx,
@@ -1816,7 +1964,9 @@ DEFINE_TRANSPARENT_OPERAND_ACCESSORS(InsertElementInst, Value)
 ///
 class ShuffleVectorInst : public Instruction {
 protected:
-  ShuffleVectorInst *clone_impl() const override;
+  // Note: Instruction needs to be a friend here to call cloneImpl.
+  friend class Instruction;
+  ShuffleVectorInst *cloneImpl() const;
 
 public:
   // allocate space for exactly three operands
@@ -1917,7 +2067,9 @@ class ExtractValueInst : public UnaryInstruction {
     return User::operator new(s, 1);
   }
 protected:
-  ExtractValueInst *clone_impl() const override;
+  // Note: Instruction needs to be a friend here to call cloneImpl.
+  friend class Instruction;
+  ExtractValueInst *cloneImpl() const;
 
 public:
   static ExtractValueInst *Create(Value *Agg,
@@ -2031,7 +2183,10 @@ class InsertValueInst : public Instruction {
   InsertValueInst(Value *Agg, Value *Val, unsigned Idx,
                   const Twine &NameStr, BasicBlock *InsertAtEnd);
 protected:
-  InsertValueInst *clone_impl() const override;
+  // Note: Instruction needs to be a friend here to call cloneImpl.
+  friend class Instruction;
+  InsertValueInst *cloneImpl() const;
+
 public:
   // allocate space for exactly two operands
   void *operator new(size_t s) {
@@ -2146,7 +2301,7 @@ class PHINode : public Instruction {
   PHINode(const PHINode &PN);
   // allocate space for exactly zero operands
   void *operator new(size_t s) {
-    return User::operator new(s, 0);
+    return User::operator new(s);
   }
   explicit PHINode(Type *Ty, unsigned NumReservedValues,
                    const Twine &NameStr = "",
@@ -2154,7 +2309,7 @@ class PHINode : public Instruction {
     : Instruction(Ty, Instruction::PHI, nullptr, 0, InsertBefore),
       ReservedSpace(NumReservedValues) {
     setName(NameStr);
-    OperandList = allocHungoffUses(ReservedSpace);
+    allocHungoffUses(ReservedSpace);
   }
 
   PHINode(Type *Ty, unsigned NumReservedValues, const Twine &NameStr,
@@ -2162,15 +2317,20 @@ class PHINode : public Instruction {
     : Instruction(Ty, Instruction::PHI, nullptr, 0, InsertAtEnd),
       ReservedSpace(NumReservedValues) {
     setName(NameStr);
-    OperandList = allocHungoffUses(ReservedSpace);
+    allocHungoffUses(ReservedSpace);
   }
 protected:
   // allocHungoffUses - this is more complicated than the generic
   // User::allocHungoffUses, because we have to allocate Uses for the incoming
   // values and pointers to the incoming blocks, all in one allocation.
-  Use *allocHungoffUses(unsigned) const;
+  void allocHungoffUses(unsigned N) {
+    User::allocHungoffUses(N, /* IsPhi */ true);
+  }
+
+  // Note: Instruction needs to be a friend here to call cloneImpl.
+  friend class Instruction;
+  PHINode *cloneImpl() const;
 
-  PHINode *clone_impl() const override;
 public:
   /// Constructors - NumReservedValues is a hint for the number of incoming
   /// edges that this phi node will have (use 0 if you really have no idea).
@@ -2183,7 +2343,6 @@ class PHINode : public Instruction {
                          const Twine &NameStr, BasicBlock *InsertAtEnd) {
     return new PHINode(Ty, NumReservedValues, NameStr, InsertAtEnd);
   }
-  ~PHINode() override;
 
   /// Provide fast operand accessors
   DECLARE_TRANSPARENT_OPERAND_ACCESSORS(Value);
@@ -2216,6 +2375,8 @@ class PHINode : public Instruction {
 
   op_range incoming_values() { return operands(); }
 
+  const_op_range incoming_values() const { return operands(); }
+
   /// getNumIncomingValues - Return the number of incoming edges
   ///
   unsigned getNumIncomingValues() const { return getNumOperands(); }
@@ -2267,12 +2428,12 @@ class PHINode : public Instruction {
     assert(BB && "PHI node got a null basic block!");
     assert(getType() == V->getType() &&
            "All operands to PHI node must be the same type as the PHI node!");
-    if (NumOperands == ReservedSpace)
+    if (getNumOperands() == ReservedSpace)
       growOperands();  // Get more space!
     // Initialize some new operands.
-    ++NumOperands;
-    setIncomingValue(NumOperands - 1, V);
-    setIncomingBlock(NumOperands - 1, BB);
+    setNumHungOffUseOperands(getNumOperands() + 1);
+    setIncomingValue(getNumOperands() - 1, V);
+    setIncomingBlock(getNumOperands() - 1, BB);
   }
 
   /// removeIncomingValue - Remove an incoming value.  This is useful if a
@@ -2351,38 +2512,33 @@ class LandingPadInst : public Instruction {
   void *operator new(size_t, unsigned) = delete;
   // Allocate space for exactly zero operands.
   void *operator new(size_t s) {
-    return User::operator new(s, 0);
+    return User::operator new(s);
   }
   void growOperands(unsigned Size);
-  void init(Value *PersFn, unsigned NumReservedValues, const Twine &NameStr);
+  void init(unsigned NumReservedValues, const Twine &NameStr);
+
+  explicit LandingPadInst(Type *RetTy, unsigned NumReservedValues,
+                          const Twine &NameStr, Instruction *InsertBefore);
+  explicit LandingPadInst(Type *RetTy, unsigned NumReservedValues,
+                          const Twine &NameStr, BasicBlock *InsertAtEnd);
 
-  explicit LandingPadInst(Type *RetTy, Value *PersonalityFn,
-                          unsigned NumReservedValues, const Twine &NameStr,
-                          Instruction *InsertBefore);
-  explicit LandingPadInst(Type *RetTy, Value *PersonalityFn,
-                          unsigned NumReservedValues, const Twine &NameStr,
-                          BasicBlock *InsertAtEnd);
 protected:
-  LandingPadInst *clone_impl() const override;
+  // Note: Instruction needs to be a friend here to call cloneImpl.
+  friend class Instruction;
+  LandingPadInst *cloneImpl() const;
+
 public:
   /// Constructors - NumReservedClauses is a hint for the number of incoming
   /// clauses that this landingpad will have (use 0 if you really have no idea).
-  static LandingPadInst *Create(Type *RetTy, Value *PersonalityFn,
-                                unsigned NumReservedClauses,
+  static LandingPadInst *Create(Type *RetTy, unsigned NumReservedClauses,
                                 const Twine &NameStr = "",
                                 Instruction *InsertBefore = nullptr);
-  static LandingPadInst *Create(Type *RetTy, Value *PersonalityFn,
-                                unsigned NumReservedClauses,
+  static LandingPadInst *Create(Type *RetTy, unsigned NumReservedClauses,
                                 const Twine &NameStr, BasicBlock *InsertAtEnd);
-  ~LandingPadInst() override;
 
   /// Provide fast operand accessors
   DECLARE_TRANSPARENT_OPERAND_ACCESSORS(Value);
 
-  /// getPersonalityFn - Get the personality function associated with this
-  /// landing pad.
-  Value *getPersonalityFn() const { return getOperand(0); }
-
   /// isCleanup - Return 'true' if this landingpad instruction is a
   /// cleanup. I.e., it should be run when unwinding even if its landing pad
   /// doesn't catch the exception.
@@ -2400,21 +2556,21 @@ class LandingPadInst : public Instruction {
   /// Get the value of the clause at index Idx. Use isCatch/isFilter to
   /// determine what type of clause this is.
   Constant *getClause(unsigned Idx) const {
-    return cast<Constant>(OperandList[Idx + 1]);
+    return cast<Constant>(getOperandList()[Idx]);
   }
 
   /// isCatch - Return 'true' if the clause and index Idx is a catch clause.
   bool isCatch(unsigned Idx) const {
-    return !isa<ArrayType>(OperandList[Idx + 1]->getType());
+    return !isa<ArrayType>(getOperandList()[Idx]->getType());
   }
 
   /// isFilter - Return 'true' if the clause and index Idx is a filter clause.
   bool isFilter(unsigned Idx) const {
-    return isa<ArrayType>(OperandList[Idx + 1]->getType());
+    return isa<ArrayType>(getOperandList()[Idx]->getType());
   }
 
   /// getNumClauses - Get the number of clauses for this landing pad.
-  unsigned getNumClauses() const { return getNumOperands() - 1; }
+  unsigned getNumClauses() const { return getNumOperands(); }
 
   /// reserveClauses - Grow the size of the operand list to accommodate the new
   /// number of clauses.
@@ -2430,7 +2586,7 @@ class LandingPadInst : public Instruction {
 };
 
 template <>
-struct OperandTraits<LandingPadInst> : public HungoffOperandTraits<2> {
+struct OperandTraits<LandingPadInst> : public HungoffOperandTraits<1> {
 };
 
 DEFINE_TRANSPARENT_OPERAND_ACCESSORS(LandingPadInst, Value)
@@ -2463,7 +2619,10 @@ class ReturnInst : public TerminatorInst {
   ReturnInst(LLVMContext &C, Value *retVal, BasicBlock *InsertAtEnd);
   explicit ReturnInst(LLVMContext &C, BasicBlock *InsertAtEnd);
 protected:
-  ReturnInst *clone_impl() const override;
+  // Note: Instruction needs to be a friend here to call cloneImpl.
+  friend class Instruction;
+  ReturnInst *cloneImpl() const;
+
 public:
   static ReturnInst* Create(LLVMContext &C, Value *retVal = nullptr,
                             Instruction *InsertBefore = nullptr) {
@@ -2535,7 +2694,10 @@ class BranchInst : public TerminatorInst {
   BranchInst(BasicBlock *IfTrue, BasicBlock *IfFalse, Value *Cond,
              BasicBlock *InsertAtEnd);
 protected:
-  BranchInst *clone_impl() const override;
+  // Note: Instruction needs to be a friend here to call cloneImpl.
+  friend class Instruction;
+  BranchInst *cloneImpl() const;
+
 public:
   static BranchInst *Create(BasicBlock *IfTrue,
                             Instruction *InsertBefore = nullptr) {
@@ -2626,7 +2788,7 @@ class SwitchInst : public TerminatorInst {
   void growOperands();
   // allocate space for exactly zero operands
   void *operator new(size_t s) {
-    return User::operator new(s, 0);
+    return User::operator new(s);
   }
   /// SwitchInst ctor - Create a new switch instruction, specifying a value to
   /// switch on and a default destination.  The number of additional cases can
@@ -2642,7 +2804,10 @@ class SwitchInst : public TerminatorInst {
   SwitchInst(Value *Value, BasicBlock *Default, unsigned NumCases,
              BasicBlock *InsertAtEnd);
 protected:
-  SwitchInst *clone_impl() const override;
+  // Note: Instruction needs to be a friend here to call cloneImpl.
+  friend class Instruction;
+  SwitchInst *cloneImpl() const;
+
 public:
 
   // -2
@@ -2773,8 +2938,6 @@ class SwitchInst : public TerminatorInst {
     return new SwitchInst(Value, Default, NumCases, InsertAtEnd);
   }
 
-  ~SwitchInst() override;
-
   /// Provide fast operand accessors
   DECLARE_TRANSPARENT_OPERAND_ACCESSORS(Value);
 
@@ -2935,7 +3098,7 @@ class IndirectBrInst : public TerminatorInst {
   void growOperands();
   // allocate space for exactly zero operands
   void *operator new(size_t s) {
-    return User::operator new(s, 0);
+    return User::operator new(s);
   }
   /// IndirectBrInst ctor - Create a new indirectbr instruction, specifying an
   /// Address to jump to.  The number of expected destinations can be specified
@@ -2949,7 +3112,10 @@ class IndirectBrInst : public TerminatorInst {
   /// autoinserts at the end of the specified BasicBlock.
   IndirectBrInst(Value *Address, unsigned NumDests, BasicBlock *InsertAtEnd);
 protected:
-  IndirectBrInst *clone_impl() const override;
+  // Note: Instruction needs to be a friend here to call cloneImpl.
+  friend class Instruction;
+  IndirectBrInst *cloneImpl() const;
+
 public:
   static IndirectBrInst *Create(Value *Address, unsigned NumDests,
                                 Instruction *InsertBefore = nullptr) {
@@ -2959,7 +3125,6 @@ class IndirectBrInst : public TerminatorInst {
                                 BasicBlock *InsertAtEnd) {
     return new IndirectBrInst(Address, NumDests, InsertAtEnd);
   }
-  ~IndirectBrInst() override;
 
   /// Provide fast operand accessors.
   DECLARE_TRANSPARENT_OPERAND_ACCESSORS(Value);
@@ -3023,17 +3188,33 @@ DEFINE_TRANSPARENT_OPERAND_ACCESSORS(IndirectBrInst, Value)
 ///
 class InvokeInst : public TerminatorInst {
   AttributeSet AttributeList;
+  FunctionType *FTy;
   InvokeInst(const InvokeInst &BI);
   void init(Value *Func, BasicBlock *IfNormal, BasicBlock *IfException,
-            ArrayRef<Value *> Args, const Twine &NameStr);
+            ArrayRef<Value *> Args, const Twine &NameStr) {
+    init(cast<FunctionType>(
+             cast<PointerType>(Func->getType())->getElementType()),
+         Func, IfNormal, IfException, Args, NameStr);
+  }
+  void init(FunctionType *FTy, Value *Func, BasicBlock *IfNormal,
+            BasicBlock *IfException, ArrayRef<Value *> Args,
+            const Twine &NameStr);
 
   /// Construct an InvokeInst given a range of arguments.
   ///
   /// \brief Construct an InvokeInst from a range of arguments
   inline InvokeInst(Value *Func, BasicBlock *IfNormal, BasicBlock *IfException,
                     ArrayRef<Value *> Args, unsigned Values,
-                    const Twine &NameStr, Instruction *InsertBefore);
-
+                    const Twine &NameStr, Instruction *InsertBefore)
+      : InvokeInst(cast<FunctionType>(
+                       cast<PointerType>(Func->getType())->getElementType()),
+                   Func, IfNormal, IfException, Args, Values, NameStr,
+                   InsertBefore) {}
+
+  inline InvokeInst(FunctionType *Ty, Value *Func, BasicBlock *IfNormal,
+                    BasicBlock *IfException, ArrayRef<Value *> Args,
+                    unsigned Values, const Twine &NameStr,
+                    Instruction *InsertBefore);
   /// Construct an InvokeInst given a range of arguments.
   ///
   /// \brief Construct an InvokeInst from a range of arguments
@@ -3041,15 +3222,26 @@ class InvokeInst : public TerminatorInst {
                     ArrayRef<Value *> Args, unsigned Values,
                     const Twine &NameStr, BasicBlock *InsertAtEnd);
 protected:
-  InvokeInst *clone_impl() const override;
+  // Note: Instruction needs to be a friend here to call cloneImpl.
+  friend class Instruction;
+  InvokeInst *cloneImpl() const;
+
 public:
   static InvokeInst *Create(Value *Func,
                             BasicBlock *IfNormal, BasicBlock *IfException,
                             ArrayRef<Value *> Args, const Twine &NameStr = "",
                             Instruction *InsertBefore = nullptr) {
+    return Create(cast<FunctionType>(
+                      cast<PointerType>(Func->getType())->getElementType()),
+                  Func, IfNormal, IfException, Args, NameStr, InsertBefore);
+  }
+  static InvokeInst *Create(FunctionType *Ty, Value *Func, BasicBlock *IfNormal,
+                            BasicBlock *IfException, ArrayRef<Value *> Args,
+                            const Twine &NameStr = "",
+                            Instruction *InsertBefore = nullptr) {
     unsigned Values = unsigned(Args.size()) + 3;
-    return new(Values) InvokeInst(Func, IfNormal, IfException, Args,
-                                  Values, NameStr, InsertBefore);
+    return new (Values) InvokeInst(Ty, Func, IfNormal, IfException, Args,
+                                   Values, NameStr, InsertBefore);
   }
   static InvokeInst *Create(Value *Func,
                             BasicBlock *IfNormal, BasicBlock *IfException,
@@ -3063,6 +3255,13 @@ class InvokeInst : public TerminatorInst {
   /// Provide fast operand accessors
   DECLARE_TRANSPARENT_OPERAND_ACCESSORS(Value);
 
+  FunctionType *getFunctionType() const { return FTy; }
+
+  void mutateFunctionType(FunctionType *FTy) {
+    mutateType(FTy->getReturnType());
+    this->FTy = FTy;
+  }
+
   /// getNumArgOperands - Return the number of invoke arguments.
   ///
   unsigned getNumArgOperands() const { return getNumOperands() - 3; }
@@ -3136,6 +3335,12 @@ class InvokeInst : public TerminatorInst {
   uint64_t getDereferenceableBytes(unsigned i) const {
     return AttributeList.getDereferenceableBytes(i);
   }
+  
+  /// \brief Extract the number of dereferenceable_or_null bytes for a call or
+  /// parameter (0=unknown).
+  uint64_t getDereferenceableOrNullBytes(unsigned i) const {
+    return AttributeList.getDereferenceableOrNullBytes(i);
+  }
 
   /// \brief Return true if the call should not be treated as a call to a
   /// builtin.
@@ -3168,6 +3373,15 @@ class InvokeInst : public TerminatorInst {
     addAttribute(AttributeSet::FunctionIndex, Attribute::ReadOnly);
   }
 
+  /// @brief Determine if the call access memmory only using it's pointer
+  /// arguments.
+  bool onlyAccessesArgMemory() const {
+    return hasFnAttr(Attribute::ArgMemOnly);
+  }
+  void setOnlyAccessesArgMemory() {
+    addAttribute(AttributeSet::FunctionIndex, Attribute::ArgMemOnly);
+  }
+
   /// \brief Determine if the call cannot return.
   bool doesNotReturn() const { return hasFnAttr(Attribute::NoReturn); }
   void setDoesNotReturn() {
@@ -3212,6 +3426,14 @@ class InvokeInst : public TerminatorInst {
 
   /// setCalledFunction - Set the function called.
   void setCalledFunction(Value* Fn) {
+    setCalledFunction(
+        cast<FunctionType>(cast<PointerType>(Fn->getType())->getElementType()),
+        Fn);
+  }
+  void setCalledFunction(FunctionType *FTy, Value *Fn) {
+    this->FTy = FTy;
+    assert(FTy == cast<FunctionType>(
+                      cast<PointerType>(Fn->getType())->getElementType()));
     Op<-3>() = Fn;
   }
 
@@ -3271,16 +3493,14 @@ template <>
 struct OperandTraits<InvokeInst> : public VariadicOperandTraits<InvokeInst, 3> {
 };
 
-InvokeInst::InvokeInst(Value *Func,
-                       BasicBlock *IfNormal, BasicBlock *IfException,
-                       ArrayRef<Value *> Args, unsigned Values,
-                       const Twine &NameStr, Instruction *InsertBefore)
-  : TerminatorInst(cast<FunctionType>(cast<PointerType>(Func->getType())
-                                      ->getElementType())->getReturnType(),
-                   Instruction::Invoke,
-                   OperandTraits<InvokeInst>::op_end(this) - Values,
-                   Values, InsertBefore) {
-  init(Func, IfNormal, IfException, Args, NameStr);
+InvokeInst::InvokeInst(FunctionType *Ty, Value *Func, BasicBlock *IfNormal,
+                       BasicBlock *IfException, ArrayRef<Value *> Args,
+                       unsigned Values, const Twine &NameStr,
+                       Instruction *InsertBefore)
+    : TerminatorInst(Ty->getReturnType(), Instruction::Invoke,
+                     OperandTraits<InvokeInst>::op_end(this) - Values, Values,
+                     InsertBefore) {
+  init(Ty, Func, IfNormal, IfException, Args, NameStr);
 }
 InvokeInst::InvokeInst(Value *Func,
                        BasicBlock *IfNormal, BasicBlock *IfException,
@@ -3309,7 +3529,10 @@ class ResumeInst : public TerminatorInst {
   explicit ResumeInst(Value *Exn, Instruction *InsertBefore=nullptr);
   ResumeInst(Value *Exn, BasicBlock *InsertAtEnd);
 protected:
-  ResumeInst *clone_impl() const override;
+  // Note: Instruction needs to be a friend here to call cloneImpl.
+  friend class Instruction;
+  ResumeInst *cloneImpl() const;
+
 public:
   static ResumeInst *Create(Value *Exn, Instruction *InsertBefore = nullptr) {
     return new(1) ResumeInst(Exn, InsertBefore);
@@ -3346,6 +3569,528 @@ struct OperandTraits<ResumeInst> :
 
 DEFINE_TRANSPARENT_OPERAND_ACCESSORS(ResumeInst, Value)
 
+//===----------------------------------------------------------------------===//
+//                               CleanupReturnInst Class
+//===----------------------------------------------------------------------===//
+
+class CleanupReturnInst : public TerminatorInst {
+  CleanupReturnInst(const CleanupReturnInst &RI);
+
+private:
+  void init(Value *RetVal, BasicBlock *UnwindBB);
+  CleanupReturnInst(LLVMContext &C, Value *RetVal, BasicBlock *UnwindBB,
+                    unsigned Values, Instruction *InsertBefore = nullptr);
+  CleanupReturnInst(LLVMContext &C, Value *RetVal, BasicBlock *UnwindBB,
+                    unsigned Values, BasicBlock *InsertAtEnd);
+
+  int getUnwindLabelOpIdx() const {
+    assert(hasUnwindDest());
+    return 0;
+  }
+
+  int getRetValOpIdx() const {
+    assert(hasReturnValue());
+    if (hasUnwindDest())
+      return 1;
+    return 0;
+  }
+
+protected:
+  // Note: Instruction needs to be a friend here to call cloneImpl.
+  friend class Instruction;
+  CleanupReturnInst *cloneImpl() const;
+
+public:
+  static CleanupReturnInst *Create(LLVMContext &C,
+                                   Value *RetVal = nullptr,
+                                   BasicBlock *UnwindBB = nullptr,
+                                   Instruction *InsertBefore = nullptr) {
+    unsigned Values = 0;
+    if (RetVal)
+      ++Values;
+    if (UnwindBB)
+      ++Values;
+    return new (Values)
+        CleanupReturnInst(C, RetVal, UnwindBB, Values, InsertBefore);
+  }
+  static CleanupReturnInst *Create(LLVMContext &C, Value *RetVal,
+                                   BasicBlock *UnwindBB,
+                                   BasicBlock *InsertAtEnd) {
+    unsigned Values = 0;
+    if (RetVal)
+      ++Values;
+    if (UnwindBB)
+      ++Values;
+    return new (Values)
+        CleanupReturnInst(C, RetVal, UnwindBB, Values, InsertAtEnd);
+  }
+
+  /// Provide fast operand accessors
+  DECLARE_TRANSPARENT_OPERAND_ACCESSORS(Value);
+
+  bool hasUnwindDest() const { return getSubclassDataFromInstruction() & 1; }
+  bool unwindsToCaller() const { return !hasUnwindDest(); }
+  bool hasReturnValue() const { return getSubclassDataFromInstruction() & 2; }
+
+  /// Convenience accessor. Returns null if there is no return value.
+  Value *getReturnValue() const {
+    if (!hasReturnValue())
+      return nullptr;
+    return getOperand(getRetValOpIdx());
+  }
+  void setReturnValue(Value *RetVal) {
+    assert(hasReturnValue());
+    setOperand(getRetValOpIdx(), RetVal);
+  }
+
+  unsigned getNumSuccessors() const { return hasUnwindDest() ? 1 : 0; }
+
+  BasicBlock *getUnwindDest() const;
+  void setUnwindDest(BasicBlock *NewDest);
+
+  // Methods for support type inquiry through isa, cast, and dyn_cast:
+  static inline bool classof(const Instruction *I) {
+    return (I->getOpcode() == Instruction::CleanupRet);
+  }
+  static inline bool classof(const Value *V) {
+    return isa<Instruction>(V) && classof(cast<Instruction>(V));
+  }
+
+private:
+  BasicBlock *getSuccessorV(unsigned Idx) const override;
+  unsigned getNumSuccessorsV() const override;
+  void setSuccessorV(unsigned Idx, BasicBlock *B) override;
+
+  // Shadow Instruction::setInstructionSubclassData with a private forwarding
+  // method so that subclasses cannot accidentally use it.
+  void setInstructionSubclassData(unsigned short D) {
+    Instruction::setInstructionSubclassData(D);
+  }
+};
+
+template <>
+struct OperandTraits<CleanupReturnInst>
+    : public VariadicOperandTraits<CleanupReturnInst> {};
+
+DEFINE_TRANSPARENT_OPERAND_ACCESSORS(CleanupReturnInst, Value)
+
+//===----------------------------------------------------------------------===//
+//                               CatchEndPadInst Class
+//===----------------------------------------------------------------------===//
+
+class CatchEndPadInst : public TerminatorInst {
+  CatchEndPadInst(const CatchEndPadInst &RI);
+
+private:
+  void init(BasicBlock *UnwindBB);
+  CatchEndPadInst(LLVMContext &C, BasicBlock *UnwindBB, unsigned Values,
+                    Instruction *InsertBefore = nullptr);
+  CatchEndPadInst(LLVMContext &C, BasicBlock *UnwindBB, unsigned Values,
+                    BasicBlock *InsertAtEnd);
+
+protected:
+  // Note: Instruction needs to be a friend here to call cloneImpl.
+  friend class Instruction;
+  CatchEndPadInst *cloneImpl() const;
+
+public:
+  static CatchEndPadInst *Create(LLVMContext &C,
+                                   BasicBlock *UnwindBB = nullptr,
+                                   Instruction *InsertBefore = nullptr) {
+    unsigned Values = UnwindBB ? 1 : 0;
+    return new (Values) CatchEndPadInst(C, UnwindBB, Values, InsertBefore);
+  }
+  static CatchEndPadInst *Create(LLVMContext &C, BasicBlock *UnwindBB,
+                                   BasicBlock *InsertAtEnd) {
+    unsigned Values = UnwindBB ? 1 : 0;
+    return new (Values) CatchEndPadInst(C, UnwindBB, Values, InsertAtEnd);
+  }
+
+  /// Provide fast operand accessors
+  DECLARE_TRANSPARENT_OPERAND_ACCESSORS(Value);
+
+  bool hasUnwindDest() const { return getSubclassDataFromInstruction() & 1; }
+  bool unwindsToCaller() const { return !hasUnwindDest(); }
+
+  /// Convenience accessor. Returns null if there is no return value.
+  unsigned getNumSuccessors() const { return hasUnwindDest() ? 1 : 0; }
+
+  BasicBlock *getUnwindDest() const {
+    return hasUnwindDest() ? cast<BasicBlock>(Op<-1>()) : nullptr;
+  }
+  void setUnwindDest(BasicBlock *NewDest) {
+    assert(NewDest);
+    Op<-1>() = NewDest;
+  }
+
+  // Methods for support type inquiry through isa, cast, and dyn_cast:
+  static inline bool classof(const Instruction *I) {
+    return (I->getOpcode() == Instruction::CatchEndPad);
+  }
+  static inline bool classof(const Value *V) {
+    return isa<Instruction>(V) && classof(cast<Instruction>(V));
+  }
+
+private:
+  BasicBlock *getSuccessorV(unsigned Idx) const override;
+  unsigned getNumSuccessorsV() const override;
+  void setSuccessorV(unsigned Idx, BasicBlock *B) override;
+
+private:
+  // Shadow Instruction::setInstructionSubclassData with a private forwarding
+  // method so that subclasses cannot accidentally use it.
+  void setInstructionSubclassData(unsigned short D) {
+    Instruction::setInstructionSubclassData(D);
+  }
+};
+
+template <>
+struct OperandTraits<CatchEndPadInst>
+    : public VariadicOperandTraits<CatchEndPadInst> {};
+
+DEFINE_TRANSPARENT_OPERAND_ACCESSORS(CatchEndPadInst, Value)
+
+//===----------------------------------------------------------------------===//
+//                           CatchPadInst Class
+//===----------------------------------------------------------------------===//
+
+class CatchPadInst : public TerminatorInst {
+private:
+  void init(BasicBlock *IfNormal, BasicBlock *IfException,
+            ArrayRef<Value *> Args, const Twine &NameStr);
+
+  CatchPadInst(const CatchPadInst &CPI);
+
+  explicit CatchPadInst(Type *RetTy, BasicBlock *IfNormal,
+                          BasicBlock *IfException, ArrayRef<Value *> Args,
+                          unsigned Values, const Twine &NameStr,
+                          Instruction *InsertBefore);
+  explicit CatchPadInst(Type *RetTy, BasicBlock *IfNormal,
+                          BasicBlock *IfException, ArrayRef<Value *> Args,
+                          unsigned Values, const Twine &NameStr,
+                          BasicBlock *InsertAtEnd);
+
+protected:
+  // Note: Instruction needs to be a friend here to call cloneImpl.
+  friend class Instruction;
+  CatchPadInst *cloneImpl() const;
+
+public:
+  static CatchPadInst *Create(Type *RetTy, BasicBlock *IfNormal,
+                                BasicBlock *IfException, ArrayRef<Value *> Args,
+                                const Twine &NameStr = "",
+                                Instruction *InsertBefore = nullptr) {
+    unsigned Values = unsigned(Args.size()) + 2;
+    return new (Values) CatchPadInst(RetTy, IfNormal, IfException, Args,
+                                       Values, NameStr, InsertBefore);
+  }
+  static CatchPadInst *Create(Type *RetTy, BasicBlock *IfNormal,
+                                BasicBlock *IfException, ArrayRef<Value *> Args,
+                                const Twine &NameStr, BasicBlock *InsertAtEnd) {
+    unsigned Values = unsigned(Args.size()) + 2;
+    return new (Values) CatchPadInst(RetTy, IfNormal, IfException, Args,
+                                       Values, NameStr, InsertAtEnd);
+  }
+
+  /// Provide fast operand accessors
+  DECLARE_TRANSPARENT_OPERAND_ACCESSORS(Value);
+
+  /// getNumArgOperands - Return the number of catchpad arguments.
+  ///
+  unsigned getNumArgOperands() const { return getNumOperands() - 2; }
+
+  /// getArgOperand/setArgOperand - Return/set the i-th catchpad argument.
+  ///
+  Value *getArgOperand(unsigned i) const { return getOperand(i); }
+  void setArgOperand(unsigned i, Value *v) { setOperand(i, v); }
+
+  /// arg_operands - iteration adapter for range-for loops.
+  iterator_range<op_iterator> arg_operands() {
+    return iterator_range<op_iterator>(op_begin(), op_end() - 2);
+  }
+
+  /// arg_operands - iteration adapter for range-for loops.
+  iterator_range<const_op_iterator> arg_operands() const {
+    return iterator_range<const_op_iterator>(op_begin(), op_end() - 2);
+  }
+
+  /// \brief Wrappers for getting the \c Use of a catchpad argument.
+  const Use &getArgOperandUse(unsigned i) const { return getOperandUse(i); }
+  Use &getArgOperandUse(unsigned i) { return getOperandUse(i); }
+
+  // get*Dest - Return the destination basic blocks...
+  BasicBlock *getNormalDest() const { return cast<BasicBlock>(Op<-2>()); }
+  BasicBlock *getUnwindDest() const { return cast<BasicBlock>(Op<-1>()); }
+  void setNormalDest(BasicBlock *B) { Op<-2>() = reinterpret_cast<Value *>(B); }
+  void setUnwindDest(BasicBlock *B) { Op<-1>() = reinterpret_cast<Value *>(B); }
+
+  BasicBlock *getSuccessor(unsigned i) const {
+    assert(i < 2 && "Successor # out of range for catchpad!");
+    return i == 0 ? getNormalDest() : getUnwindDest();
+  }
+
+  void setSuccessor(unsigned idx, BasicBlock *NewSucc) {
+    assert(idx < 2 && "Successor # out of range for catchpad!");
+    *(&Op<-2>() + idx) = reinterpret_cast<Value *>(NewSucc);
+  }
+
+  unsigned getNumSuccessors() const { return 2; }
+
+  // Methods for support type inquiry through isa, cast, and dyn_cast:
+  static inline bool classof(const Instruction *I) {
+    return I->getOpcode() == Instruction::CatchPad;
+  }
+  static inline bool classof(const Value *V) {
+    return isa<Instruction>(V) && classof(cast<Instruction>(V));
+  }
+
+private:
+  BasicBlock *getSuccessorV(unsigned idx) const override;
+  unsigned getNumSuccessorsV() const override;
+  void setSuccessorV(unsigned idx, BasicBlock *B) override;
+};
+
+template <>
+struct OperandTraits<CatchPadInst>
+    : public VariadicOperandTraits<CatchPadInst, /*MINARITY=*/2> {};
+
+DEFINE_TRANSPARENT_OPERAND_ACCESSORS(CatchPadInst, Value)
+
+//===----------------------------------------------------------------------===//
+//                           TerminatePadInst Class
+//===----------------------------------------------------------------------===//
+
+class TerminatePadInst : public TerminatorInst {
+private:
+  void init(BasicBlock *BB, ArrayRef<Value *> Args, const Twine &NameStr);
+
+  TerminatePadInst(const TerminatePadInst &TPI);
+
+  explicit TerminatePadInst(LLVMContext &C, BasicBlock *BB,
+                              ArrayRef<Value *> Args, unsigned Values,
+                              const Twine &NameStr, Instruction *InsertBefore);
+  explicit TerminatePadInst(LLVMContext &C, BasicBlock *BB,
+                              ArrayRef<Value *> Args, unsigned Values,
+                              const Twine &NameStr, BasicBlock *InsertAtEnd);
+
+protected:
+  // Note: Instruction needs to be a friend here to call cloneImpl.
+  friend class Instruction;
+  TerminatePadInst *cloneImpl() const;
+
+public:
+  static TerminatePadInst *Create(LLVMContext &C, BasicBlock *BB = nullptr,
+                                    ArrayRef<Value *> Args = None,
+                                    const Twine &NameStr = "",
+                                    Instruction *InsertBefore = nullptr) {
+    unsigned Values = unsigned(Args.size());
+    if (BB)
+      ++Values;
+    return new (Values)
+        TerminatePadInst(C, BB, Args, Values, NameStr, InsertBefore);
+  }
+  static TerminatePadInst *Create(LLVMContext &C, BasicBlock *BB,
+                                    ArrayRef<Value *> Args,
+                                    const Twine &NameStr,
+                                    BasicBlock *InsertAtEnd) {
+    unsigned Values = unsigned(Args.size());
+    if (BB)
+      ++Values;
+    return new (Values)
+        TerminatePadInst(C, BB, Args, Values, NameStr, InsertAtEnd);
+  }
+
+  /// Provide fast operand accessors
+  DECLARE_TRANSPARENT_OPERAND_ACCESSORS(Value);
+
+  bool hasUnwindDest() const { return getSubclassDataFromInstruction() & 1; }
+  bool unwindsToCaller() const { return !hasUnwindDest(); }
+
+  /// getNumArgOperands - Return the number of terminatepad arguments.
+  ///
+  unsigned getNumArgOperands() const {
+    unsigned NumOperands = getNumOperands();
+    if (hasUnwindDest())
+      return NumOperands - 1;
+    return NumOperands;
+  }
+
+  /// getArgOperand/setArgOperand - Return/set the i-th terminatepad argument.
+  ///
+  Value *getArgOperand(unsigned i) const { return getOperand(i); }
+  void setArgOperand(unsigned i, Value *v) { setOperand(i, v); }
+
+  const_op_iterator arg_end() const {
+    if (hasUnwindDest())
+      return op_end() - 1;
+    return op_end();
+  }
+
+  op_iterator arg_end() {
+    if (hasUnwindDest())
+      return op_end() - 1;
+    return op_end();
+  }
+
+  /// arg_operands - iteration adapter for range-for loops.
+  iterator_range<op_iterator> arg_operands() {
+    return iterator_range<op_iterator>(op_begin(), arg_end());
+  }
+
+  /// arg_operands - iteration adapter for range-for loops.
+  iterator_range<const_op_iterator> arg_operands() const {
+    return iterator_range<const_op_iterator>(op_begin(), arg_end());
+  }
+
+  /// \brief Wrappers for getting the \c Use of a terminatepad argument.
+  const Use &getArgOperandUse(unsigned i) const { return getOperandUse(i); }
+  Use &getArgOperandUse(unsigned i) { return getOperandUse(i); }
+
+  // get*Dest - Return the destination basic blocks...
+  BasicBlock *getUnwindDest() const {
+    if (!hasUnwindDest())
+      return nullptr;
+    return cast<BasicBlock>(Op<-1>());
+  }
+  void setUnwindDest(BasicBlock *B) {
+    assert(B && hasUnwindDest());
+    Op<-1>() = reinterpret_cast<Value *>(B);
+  }
+
+  unsigned getNumSuccessors() const { return hasUnwindDest() ? 1 : 0; }
+
+  // Methods for support type inquiry through isa, cast, and dyn_cast:
+  static inline bool classof(const Instruction *I) {
+    return I->getOpcode() == Instruction::TerminatePad;
+  }
+  static inline bool classof(const Value *V) {
+    return isa<Instruction>(V) && classof(cast<Instruction>(V));
+  }
+
+private:
+  BasicBlock *getSuccessorV(unsigned idx) const override;
+  unsigned getNumSuccessorsV() const override;
+  void setSuccessorV(unsigned idx, BasicBlock *B) override;
+
+  // Shadow Instruction::setInstructionSubclassData with a private forwarding
+  // method so that subclasses cannot accidentally use it.
+  void setInstructionSubclassData(unsigned short D) {
+    Instruction::setInstructionSubclassData(D);
+  }
+};
+
+template <>
+struct OperandTraits<TerminatePadInst>
+    : public VariadicOperandTraits<TerminatePadInst, /*MINARITY=*/1> {};
+
+DEFINE_TRANSPARENT_OPERAND_ACCESSORS(TerminatePadInst, Value)
+
+//===----------------------------------------------------------------------===//
+//                           CleanupPadInst Class
+//===----------------------------------------------------------------------===//
+
+class CleanupPadInst : public Instruction {
+private:
+  void init(ArrayRef<Value *> Args, const Twine &NameStr);
+
+  CleanupPadInst(const CleanupPadInst &CPI);
+
+  explicit CleanupPadInst(Type *RetTy, ArrayRef<Value *> Args,
+                            const Twine &NameStr, Instruction *InsertBefore);
+  explicit CleanupPadInst(Type *RetTy, ArrayRef<Value *> Args,
+                            const Twine &NameStr, BasicBlock *InsertAtEnd);
+
+protected:
+  // Note: Instruction needs to be a friend here to call cloneImpl.
+  friend class Instruction;
+  CleanupPadInst *cloneImpl() const;
+
+public:
+  static CleanupPadInst *Create(Type *RetTy, ArrayRef<Value *> Args,
+                                  const Twine &NameStr = "",
+                                  Instruction *InsertBefore = nullptr) {
+    return new (Args.size())
+        CleanupPadInst(RetTy, Args, NameStr, InsertBefore);
+  }
+  static CleanupPadInst *Create(Type *RetTy, ArrayRef<Value *> Args,
+                                  const Twine &NameStr,
+                                  BasicBlock *InsertAtEnd) {
+    return new (Args.size())
+        CleanupPadInst(RetTy, Args, NameStr, InsertAtEnd);
+  }
+
+  /// Provide fast operand accessors
+  DECLARE_TRANSPARENT_OPERAND_ACCESSORS(Value);
+
+  // Methods for support type inquiry through isa, cast, and dyn_cast:
+  static inline bool classof(const Instruction *I) {
+    return I->getOpcode() == Instruction::CleanupPad;
+  }
+  static inline bool classof(const Value *V) {
+    return isa<Instruction>(V) && classof(cast<Instruction>(V));
+  }
+};
+
+template <>
+struct OperandTraits<CleanupPadInst>
+    : public VariadicOperandTraits<CleanupPadInst, /*MINARITY=*/0> {};
+
+DEFINE_TRANSPARENT_OPERAND_ACCESSORS(CleanupPadInst, Value)
+
+//===----------------------------------------------------------------------===//
+//                               CatchReturnInst Class
+//===----------------------------------------------------------------------===//
+
+class CatchReturnInst : public TerminatorInst {
+  CatchReturnInst(const CatchReturnInst &RI);
+
+private:
+  void init(Value *RetVal, BasicBlock *UnwindBB);
+  CatchReturnInst(BasicBlock *BB, Instruction *InsertBefore = nullptr);
+  CatchReturnInst(BasicBlock *BB, BasicBlock *InsertAtEnd);
+
+protected:
+  // Note: Instruction needs to be a friend here to call cloneImpl.
+  friend class Instruction;
+  CatchReturnInst *cloneImpl() const;
+
+public:
+  static CatchReturnInst *Create(BasicBlock *BB,
+                                 Instruction *InsertBefore = nullptr) {
+    return new (1) CatchReturnInst(BB, InsertBefore);
+  }
+  static CatchReturnInst *Create(BasicBlock *BB, BasicBlock *InsertAtEnd) {
+    return new (1) CatchReturnInst(BB, InsertAtEnd);
+  }
+
+  /// Provide fast operand accessors
+  DECLARE_TRANSPARENT_OPERAND_ACCESSORS(Value);
+
+  /// Convenience accessors.
+  BasicBlock *getSuccessor() const { return cast<BasicBlock>(Op<0>()); }
+  void setSuccessor(BasicBlock *NewSucc) { Op<0>() = (Value *)NewSucc; }
+  unsigned getNumSuccessors() const { return 1; }
+
+  // Methods for support type inquiry through isa, cast, and dyn_cast:
+  static inline bool classof(const Instruction *I) {
+    return (I->getOpcode() == Instruction::CatchRet);
+  }
+  static inline bool classof(const Value *V) {
+    return isa<Instruction>(V) && classof(cast<Instruction>(V));
+  }
+
+private:
+  BasicBlock *getSuccessorV(unsigned Idx) const override;
+  unsigned getNumSuccessorsV() const override;
+  void setSuccessorV(unsigned Idx, BasicBlock *B) override;
+};
+
+template <>
+struct OperandTraits<CatchReturnInst>
+    : public FixedNumOperandTraits<CatchReturnInst, /*ARITY=*/1> {};
+
+DEFINE_TRANSPARENT_OPERAND_ACCESSORS(CatchReturnInst, Value)
+
 //===----------------------------------------------------------------------===//
 //                           UnreachableInst Class
 //===----------------------------------------------------------------------===//
@@ -3358,7 +4103,9 @@ DEFINE_TRANSPARENT_OPERAND_ACCESSORS(ResumeInst, Value)
 class UnreachableInst : public TerminatorInst {
   void *operator new(size_t, unsigned) = delete;
 protected:
-  UnreachableInst *clone_impl() const override;
+  // Note: Instruction needs to be a friend here to call cloneImpl.
+  friend class Instruction;
+  UnreachableInst *cloneImpl() const;
 
 public:
   // allocate space for exactly zero operands
@@ -3390,8 +4137,10 @@ class UnreachableInst : public TerminatorInst {
 /// \brief This class represents a truncation of integer types.
 class TruncInst : public CastInst {
 protected:
+  // Note: Instruction needs to be a friend here to call cloneImpl.
+  friend class Instruction;
   /// \brief Clone an identical TruncInst
-  TruncInst *clone_impl() const override;
+  TruncInst *cloneImpl() const;
 
 public:
   /// \brief Constructor with insert-before-instruction semantics
@@ -3426,8 +4175,10 @@ class TruncInst : public CastInst {
 /// \brief This class represents zero extension of integer types.
 class ZExtInst : public CastInst {
 protected:
+  // Note: Instruction needs to be a friend here to call cloneImpl.
+  friend class Instruction;
   /// \brief Clone an identical ZExtInst
-  ZExtInst *clone_impl() const override;
+  ZExtInst *cloneImpl() const;
 
 public:
   /// \brief Constructor with insert-before-instruction semantics
@@ -3462,8 +4213,10 @@ class ZExtInst : public CastInst {
 /// \brief This class represents a sign extension of integer types.
 class SExtInst : public CastInst {
 protected:
+  // Note: Instruction needs to be a friend here to call cloneImpl.
+  friend class Instruction;
   /// \brief Clone an identical SExtInst
-  SExtInst *clone_impl() const override;
+  SExtInst *cloneImpl() const;
 
 public:
   /// \brief Constructor with insert-before-instruction semantics
@@ -3498,8 +4251,10 @@ class SExtInst : public CastInst {
 /// \brief This class represents a truncation of floating point types.
 class FPTruncInst : public CastInst {
 protected:
+  // Note: Instruction needs to be a friend here to call cloneImpl.
+  friend class Instruction;
   /// \brief Clone an identical FPTruncInst
-  FPTruncInst *clone_impl() const override;
+  FPTruncInst *cloneImpl() const;
 
 public:
   /// \brief Constructor with insert-before-instruction semantics
@@ -3534,8 +4289,10 @@ class FPTruncInst : public CastInst {
 /// \brief This class represents an extension of floating point types.
 class FPExtInst : public CastInst {
 protected:
+  // Note: Instruction needs to be a friend here to call cloneImpl.
+  friend class Instruction;
   /// \brief Clone an identical FPExtInst
-  FPExtInst *clone_impl() const override;
+  FPExtInst *cloneImpl() const;
 
 public:
   /// \brief Constructor with insert-before-instruction semantics
@@ -3570,8 +4327,10 @@ class FPExtInst : public CastInst {
 /// \brief This class represents a cast unsigned integer to floating point.
 class UIToFPInst : public CastInst {
 protected:
+  // Note: Instruction needs to be a friend here to call cloneImpl.
+  friend class Instruction;
   /// \brief Clone an identical UIToFPInst
-  UIToFPInst *clone_impl() const override;
+  UIToFPInst *cloneImpl() const;
 
 public:
   /// \brief Constructor with insert-before-instruction semantics
@@ -3606,8 +4365,10 @@ class UIToFPInst : public CastInst {
 /// \brief This class represents a cast from signed integer to floating point.
 class SIToFPInst : public CastInst {
 protected:
+  // Note: Instruction needs to be a friend here to call cloneImpl.
+  friend class Instruction;
   /// \brief Clone an identical SIToFPInst
-  SIToFPInst *clone_impl() const override;
+  SIToFPInst *cloneImpl() const;
 
 public:
   /// \brief Constructor with insert-before-instruction semantics
@@ -3642,8 +4403,10 @@ class SIToFPInst : public CastInst {
 /// \brief This class represents a cast from floating point to unsigned integer
 class FPToUIInst  : public CastInst {
 protected:
+  // Note: Instruction needs to be a friend here to call cloneImpl.
+  friend class Instruction;
   /// \brief Clone an identical FPToUIInst
-  FPToUIInst *clone_impl() const override;
+  FPToUIInst *cloneImpl() const;
 
 public:
   /// \brief Constructor with insert-before-instruction semantics
@@ -3678,8 +4441,10 @@ class FPToUIInst  : public CastInst {
 /// \brief This class represents a cast from floating point to signed integer.
 class FPToSIInst  : public CastInst {
 protected:
+  // Note: Instruction needs to be a friend here to call cloneImpl.
+  friend class Instruction;
   /// \brief Clone an identical FPToSIInst
-  FPToSIInst *clone_impl() const override;
+  FPToSIInst *cloneImpl() const;
 
 public:
   /// \brief Constructor with insert-before-instruction semantics
@@ -3730,8 +4495,10 @@ class IntToPtrInst : public CastInst {
     BasicBlock *InsertAtEnd       ///< The block to insert the instruction into
   );
 
+  // Note: Instruction needs to be a friend here to call cloneImpl.
+  friend class Instruction;
   /// \brief Clone an identical IntToPtrInst
-  IntToPtrInst *clone_impl() const override;
+  IntToPtrInst *cloneImpl() const;
 
   /// \brief Returns the address space of this instruction's pointer type.
   unsigned getAddressSpace() const {
@@ -3754,8 +4521,10 @@ class IntToPtrInst : public CastInst {
 /// \brief This class represents a cast from a pointer to an integer
 class PtrToIntInst : public CastInst {
 protected:
+  // Note: Instruction needs to be a friend here to call cloneImpl.
+  friend class Instruction;
   /// \brief Clone an identical PtrToIntInst
-  PtrToIntInst *clone_impl() const override;
+  PtrToIntInst *cloneImpl() const;
 
 public:
   /// \brief Constructor with insert-before-instruction semantics
@@ -3802,8 +4571,10 @@ class PtrToIntInst : public CastInst {
 /// \brief This class represents a no-op cast from one type to another.
 class BitCastInst : public CastInst {
 protected:
+  // Note: Instruction needs to be a friend here to call cloneImpl.
+  friend class Instruction;
   /// \brief Clone an identical BitCastInst
-  BitCastInst *clone_impl() const override;
+  BitCastInst *cloneImpl() const;
 
 public:
   /// \brief Constructor with insert-before-instruction semantics
@@ -3839,8 +4610,10 @@ class BitCastInst : public CastInst {
 /// one address space to another.
 class AddrSpaceCastInst : public CastInst {
 protected:
+  // Note: Instruction needs to be a friend here to call cloneImpl.
+  friend class Instruction;
   /// \brief Clone an identical AddrSpaceCastInst
-  AddrSpaceCastInst *clone_impl() const override;
+  AddrSpaceCastInst *cloneImpl() const;
 
 public:
   /// \brief Constructor with insert-before-instruction semantics
diff --git a/include/llvm/IR/IntrinsicInst.h b/include/llvm/IR/IntrinsicInst.h
index bf0adc39347..2c8b6eb6f39 100644
--- a/include/llvm/IR/IntrinsicInst.h
+++ b/include/llvm/IR/IntrinsicInst.h
@@ -42,7 +42,7 @@ namespace llvm {
     /// getIntrinsicID - Return the intrinsic ID of this intrinsic.
     ///
     Intrinsic::ID getIntrinsicID() const {
-      return (Intrinsic::ID)getCalledFunction()->getIntrinsicID();
+      return getCalledFunction()->getIntrinsicID();
     }
 
     // Methods for support type inquiry through isa, cast, and dyn_cast:
@@ -82,11 +82,11 @@ namespace llvm {
   class DbgDeclareInst : public DbgInfoIntrinsic {
   public:
     Value *getAddress() const;
-    MDLocalVariable *getVariable() const {
-      return cast<MDLocalVariable>(getRawVariable());
+    DILocalVariable *getVariable() const {
+      return cast<DILocalVariable>(getRawVariable());
     }
-    MDExpression *getExpression() const {
-      return cast<MDExpression>(getRawExpression());
+    DIExpression *getExpression() const {
+      return cast<DIExpression>(getRawExpression());
     }
 
     Metadata *getRawVariable() const {
@@ -115,11 +115,11 @@ namespace llvm {
       return cast<ConstantInt>(
                           const_cast<Value*>(getArgOperand(1)))->getZExtValue();
     }
-    MDLocalVariable *getVariable() const {
-      return cast<MDLocalVariable>(getRawVariable());
+    DILocalVariable *getVariable() const {
+      return cast<DILocalVariable>(getRawVariable());
     }
-    MDExpression *getExpression() const {
-      return cast<MDExpression>(getRawExpression());
+    DIExpression *getExpression() const {
+      return cast<DIExpression>(getRawExpression());
     }
 
     Metadata *getRawVariable() const {
diff --git a/include/llvm/IR/Intrinsics.h b/include/llvm/IR/Intrinsics.h
index a66bd2cd7d9..43b8325107f 100644
--- a/include/llvm/IR/Intrinsics.h
+++ b/include/llvm/IR/Intrinsics.h
@@ -32,7 +32,7 @@ class AttributeSet;
 /// function known by LLVM. The enum values are returned by
 /// Function::getIntrinsicID().
 namespace Intrinsic {
-  enum ID {
+  enum ID : unsigned {
     not_intrinsic = 0,   // Must be zero
 
     // Get the intrinsic enums generated from Intrinsics.td
@@ -52,6 +52,11 @@ namespace Intrinsic {
   /// Returns true if the intrinsic can be overloaded.
   bool isOverloaded(ID id);
 
+  /// Returns true if the intrinsic is a leaf, i.e. it does not make any calls
+  /// itself.  Most intrinsics are leafs, the exceptions being the patchpoint
+  /// and statepoint intrinsics. These call (or invoke) their "target" argument.
+  bool isLeaf(ID id);
+
   /// Return the attributes for an intrinsic.
   AttributeSet getAttributes(LLVMContext &C, ID id);
 
diff --git a/include/llvm/IR/Intrinsics.td b/include/llvm/IR/Intrinsics.td
index 9ff923da3dc..34774ccdbe4 100644
--- a/include/llvm/IR/Intrinsics.td
+++ b/include/llvm/IR/Intrinsics.td
@@ -73,6 +73,11 @@ def IntrNoReturn : IntrinsicProperty;
 // Parallels the noduplicate attribute on LLVM IR functions.
 def IntrNoDuplicate : IntrinsicProperty;
 
+// IntrConvergent - Calls to this intrinsic are convergent and may only be
+// moved to control equivalent blocks.
+// Parallels the convergent attribute on LLVM IR functions.
+def IntrConvergent : IntrinsicProperty;
+
 //===----------------------------------------------------------------------===//
 // Types used by intrinsics.
 //===----------------------------------------------------------------------===//
@@ -181,6 +186,8 @@ def llvm_v4i64_ty      : LLVMType<v4i64>;    //  4 x i64
 def llvm_v8i64_ty      : LLVMType<v8i64>;    //  8 x i64
 def llvm_v16i64_ty     : LLVMType<v16i64>;   // 16 x i64
 
+def llvm_v1i128_ty     : LLVMType<v1i128>;   //  1 x i128
+
 def llvm_v2f16_ty      : LLVMType<v2f16>;    //  2 x half (__fp16)
 def llvm_v4f16_ty      : LLVMType<v4f16>;    //  4 x half (__fp16)
 def llvm_v8f16_ty      : LLVMType<v8f16>;    //  8 x half (__fp16)
@@ -261,15 +268,23 @@ def int_gcwrite : Intrinsic<[],
 //
 def int_returnaddress : Intrinsic<[llvm_ptr_ty], [llvm_i32_ty], [IntrNoMem]>;
 def int_frameaddress  : Intrinsic<[llvm_ptr_ty], [llvm_i32_ty], [IntrNoMem]>;
-def int_frameescape : Intrinsic<[], [llvm_vararg_ty]>;
-def int_framerecover : Intrinsic<[llvm_ptr_ty],
-                                 [llvm_ptr_ty, llvm_ptr_ty, llvm_i32_ty],
-                                 [IntrNoMem]>;
 def int_read_register  : Intrinsic<[llvm_anyint_ty], [llvm_metadata_ty],
-                                   [IntrNoMem], "llvm.read_register">;
+                                   [IntrReadMem], "llvm.read_register">;
 def int_write_register : Intrinsic<[], [llvm_metadata_ty, llvm_anyint_ty],
                                    [], "llvm.write_register">;
 
+// Gets the address of the local variable area. This is typically a copy of the
+// stack, frame, or base pointer depending on the type of prologue.
+def int_localaddress : Intrinsic<[llvm_ptr_ty], [], [IntrNoMem]>;
+
+// Escapes local variables to allow access from other functions.
+def int_localescape : Intrinsic<[], [llvm_vararg_ty]>;
+
+// Given a function and the localaddress of a parent frame, returns a pointer
+// to an escaped allocation indicated by the index.
+def int_localrecover : Intrinsic<[llvm_ptr_ty],
+                                 [llvm_ptr_ty, llvm_ptr_ty, llvm_i32_ty],
+                                 [IntrNoMem]>;
 // Note: we treat stacksave/stackrestore as writemem because we don't otherwise
 // model their dependencies on allocas.
 def int_stacksave     : Intrinsic<[llvm_ptr_ty]>,
@@ -355,6 +370,8 @@ let Properties = [IntrNoMem] in {
   def int_rint  : Intrinsic<[llvm_anyfloat_ty], [LLVMMatchType<0>]>;
   def int_nearbyint : Intrinsic<[llvm_anyfloat_ty], [LLVMMatchType<0>]>;
   def int_round : Intrinsic<[llvm_anyfloat_ty], [LLVMMatchType<0>]>;
+  def int_canonicalize : Intrinsic<[llvm_anyfloat_ty], [LLVMMatchType<0>],
+                                   [IntrNoMem]>;
 }
 
 // NOTE: these are internal interfaces.
@@ -421,6 +438,8 @@ def int_eh_endcatch : Intrinsic<[], []>;
 // Represents the list of actions to take when an exception is thrown.
 def int_eh_actions : Intrinsic<[llvm_ptr_ty], [llvm_vararg_ty], []>;
 
+def int_eh_exceptioncode : Intrinsic<[llvm_i32_ty], [], [IntrReadMem]>;
+
 // __builtin_unwind_init is an undocumented GCC intrinsic that causes all
 // callee-saved registers to be saved and restored (regardless of whether they
 // are used) in the calling function. It is used by libgcc_eh.
@@ -436,6 +455,7 @@ let Properties = [IntrNoMem] in {
 def int_eh_sjlj_functioncontext : Intrinsic<[], [llvm_ptr_ty]>;
 def int_eh_sjlj_setjmp          : Intrinsic<[llvm_i32_ty], [llvm_ptr_ty]>;
 def int_eh_sjlj_longjmp         : Intrinsic<[], [llvm_ptr_ty], [IntrNoReturn]>;
+def int_eh_sjlj_setup_dispatch  : Intrinsic<[], []>;
 
 //===---------------- Generic Variable Attribute Intrinsics----------------===//
 //
@@ -525,8 +545,10 @@ def int_experimental_patchpoint_i64 : Intrinsic<[llvm_i64_ty],
 // These are documented in docs/Statepoint.rst
 
 def int_experimental_gc_statepoint : Intrinsic<[llvm_i32_ty],
-                               [llvm_anyptr_ty, llvm_i32_ty,
-                                llvm_i32_ty, llvm_vararg_ty]>;
+                               [llvm_i64_ty, llvm_i32_ty,
+                                llvm_anyptr_ty, llvm_i32_ty,
+                                llvm_i32_ty, llvm_vararg_ty],
+                                [Throws]>;
 
 def int_experimental_gc_result   : Intrinsic<[llvm_any_ty], [llvm_i32_ty]>;
 def int_experimental_gc_relocate : Intrinsic<[llvm_anyptr_ty],
@@ -578,6 +600,7 @@ def int_convertus  : Intrinsic<[llvm_anyint_ty],
                                [llvm_anyint_ty, llvm_i32_ty, llvm_i32_ty]>;
 def int_convertuu  : Intrinsic<[llvm_anyint_ty],
                                [llvm_anyint_ty, llvm_i32_ty, llvm_i32_ty]>;
+
 // Clear cache intrinsic, default to ignore (ie. emit nothing)
 // maps to void __clear_cache() on supporting platforms
 def int_clear_cache : Intrinsic<[], [llvm_ptr_ty, llvm_ptr_ty],
@@ -627,6 +650,11 @@ def int_nacl_atomic_is_lock_free : Intrinsic<[llvm_i1_ty],
     [llvm_i32_ty, llvm_ptr_ty], [IntrNoMem]>,
     GCCBuiltin<"__nacl_atomic_is_lock_free">;
 // @LOCALMOD-END
+// Calculate the Absolute Differences of the two input vectors.
+def int_sabsdiff : Intrinsic<[llvm_anyvector_ty],
+                        [ LLVMMatchType<0>, LLVMMatchType<0> ], [IntrNoMem]>;
+def int_uabsdiff : Intrinsic<[llvm_anyvector_ty],
+                        [ LLVMMatchType<0>, LLVMMatchType<0> ], [IntrNoMem]>;
 
 //===-------------------------- Masked Intrinsics -------------------------===//
 //
@@ -668,6 +696,7 @@ include "llvm/IR/IntrinsicsXCore.td"
 include "llvm/IR/IntrinsicsHexagon.td"
 include "llvm/IR/IntrinsicsNVVM.td"
 include "llvm/IR/IntrinsicsMips.td"
-include "llvm/IR/IntrinsicsR600.td"
+include "llvm/IR/IntrinsicsAMDGPU.td"
 include "llvm/IR/IntrinsicsBPF.td"
 include "llvm/IR/IntrinsicsSystemZ.td"
+include "llvm/IR/IntrinsicsWebAssembly.td"
diff --git a/include/llvm/IR/IntrinsicsAArch64.td b/include/llvm/IR/IntrinsicsAArch64.td
index 7d69ed52171..578f259aae1 100644
--- a/include/llvm/IR/IntrinsicsAArch64.td
+++ b/include/llvm/IR/IntrinsicsAArch64.td
@@ -13,6 +13,9 @@
 
 let TargetPrefix = "aarch64" in {
 
+def int_aarch64_thread_pointer : GCCBuiltin<"__builtin_thread_pointer">,
+            Intrinsic<[llvm_ptr_ty], [], [IntrNoMem]>;
+
 def int_aarch64_ldxr : Intrinsic<[llvm_i64_ty], [llvm_anyptr_ty]>;
 def int_aarch64_ldaxr : Intrinsic<[llvm_i64_ty], [llvm_anyptr_ty]>;
 def int_aarch64_stxr : Intrinsic<[llvm_i32_ty], [llvm_i64_ty, llvm_anyptr_ty]>;
diff --git a/include/llvm/IR/IntrinsicsAMDGPU.td b/include/llvm/IR/IntrinsicsAMDGPU.td
new file mode 100644
index 00000000000..510e5ad2d9b
--- /dev/null
+++ b/include/llvm/IR/IntrinsicsAMDGPU.td
@@ -0,0 +1,85 @@
+//===- IntrinsicsAMDGPU.td - Defines AMDGPU intrinsics -----*- tablegen -*-===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This file defines all of the R600-specific intrinsics.
+//
+//===----------------------------------------------------------------------===//
+
+let TargetPrefix = "r600" in {
+
+class R600ReadPreloadRegisterIntrinsic<string name>
+  : Intrinsic<[llvm_i32_ty], [], [IntrNoMem]>,
+    GCCBuiltin<name>;
+
+multiclass R600ReadPreloadRegisterIntrinsic_xyz<string prefix> {
+  def _x : R600ReadPreloadRegisterIntrinsic<!strconcat(prefix, "_x")>;
+  def _y : R600ReadPreloadRegisterIntrinsic<!strconcat(prefix, "_y")>;
+  def _z : R600ReadPreloadRegisterIntrinsic<!strconcat(prefix, "_z")>;
+}
+
+defm int_r600_read_global_size : R600ReadPreloadRegisterIntrinsic_xyz <
+                                       "__builtin_r600_read_global_size">;
+defm int_r600_read_local_size : R600ReadPreloadRegisterIntrinsic_xyz <
+                                       "__builtin_r600_read_local_size">;
+defm int_r600_read_ngroups : R600ReadPreloadRegisterIntrinsic_xyz <
+                                       "__builtin_r600_read_ngroups">;
+defm int_r600_read_tgid : R600ReadPreloadRegisterIntrinsic_xyz <
+                                       "__builtin_r600_read_tgid">;
+defm int_r600_read_tidig : R600ReadPreloadRegisterIntrinsic_xyz <
+                                       "__builtin_r600_read_tidig">;
+} // End TargetPrefix = "r600"
+
+let TargetPrefix = "AMDGPU" in {
+
+class AMDGPUReadPreloadRegisterIntrinsic<string name>
+  : Intrinsic<[llvm_i32_ty], [], [IntrNoMem]>,
+    GCCBuiltin<name>;
+
+def int_AMDGPU_div_scale : GCCBuiltin<"__builtin_amdgpu_div_scale">,
+  // 1st parameter: Numerator
+  // 2nd parameter: Denominator
+  // 3rd parameter: Constant to select select between first and
+  //                second. (0 = first, 1 = second).
+  Intrinsic<[llvm_anyfloat_ty, llvm_i1_ty],
+            [LLVMMatchType<0>, LLVMMatchType<0>, llvm_i1_ty],
+            [IntrNoMem]>;
+
+def int_AMDGPU_div_fmas : GCCBuiltin<"__builtin_amdgpu_div_fmas">,
+  Intrinsic<[llvm_anyfloat_ty],
+            [LLVMMatchType<0>, LLVMMatchType<0>, LLVMMatchType<0>, llvm_i1_ty],
+            [IntrNoMem]>;
+
+def int_AMDGPU_div_fixup : GCCBuiltin<"__builtin_amdgpu_div_fixup">,
+  Intrinsic<[llvm_anyfloat_ty],
+            [LLVMMatchType<0>, LLVMMatchType<0>, LLVMMatchType<0>],
+            [IntrNoMem]>;
+
+def int_AMDGPU_trig_preop : GCCBuiltin<"__builtin_amdgpu_trig_preop">,
+  Intrinsic<[llvm_anyfloat_ty], [LLVMMatchType<0>, llvm_i32_ty],
+            [IntrNoMem]>;
+
+def int_AMDGPU_rcp : GCCBuiltin<"__builtin_amdgpu_rcp">,
+  Intrinsic<[llvm_anyfloat_ty], [LLVMMatchType<0>], [IntrNoMem]>;
+
+def int_AMDGPU_rsq : GCCBuiltin<"__builtin_amdgpu_rsq">,
+  Intrinsic<[llvm_anyfloat_ty], [LLVMMatchType<0>], [IntrNoMem]>;
+
+def int_AMDGPU_rsq_clamped : GCCBuiltin<"__builtin_amdgpu_rsq_clamped">,
+  Intrinsic<[llvm_anyfloat_ty], [LLVMMatchType<0>], [IntrNoMem]>;
+
+def int_AMDGPU_ldexp : GCCBuiltin<"__builtin_amdgpu_ldexp">,
+  Intrinsic<[llvm_anyfloat_ty], [LLVMMatchType<0>, llvm_i32_ty], [IntrNoMem]>;
+
+def int_AMDGPU_class : GCCBuiltin<"__builtin_amdgpu_class">,
+  Intrinsic<[llvm_i1_ty], [llvm_anyfloat_ty, llvm_i32_ty], [IntrNoMem]>;
+
+def int_AMDGPU_read_workdim : AMDGPUReadPreloadRegisterIntrinsic <
+                                       "__builtin_amdgpu_read_workdim">;
+
+} // End TargetPrefix = "AMDGPU"
diff --git a/include/llvm/IR/IntrinsicsARM.td b/include/llvm/IR/IntrinsicsARM.td
index ce758e25731..1dff8087859 100644
--- a/include/llvm/IR/IntrinsicsARM.td
+++ b/include/llvm/IR/IntrinsicsARM.td
@@ -83,11 +83,9 @@ def int_arm_vcvtru    : Intrinsic<[llvm_float_ty], [llvm_anyfloat_ty],
 
 // Move to coprocessor
 def int_arm_mcr : GCCBuiltin<"__builtin_arm_mcr">,
-                  MSBuiltin<"_MoveToCoprocessor">,
    Intrinsic<[], [llvm_i32_ty, llvm_i32_ty, llvm_i32_ty,
                   llvm_i32_ty, llvm_i32_ty, llvm_i32_ty], []>;
 def int_arm_mcr2 : GCCBuiltin<"__builtin_arm_mcr2">,
-                   MSBuiltin<"_MoveToCoprocessor2">,
    Intrinsic<[], [llvm_i32_ty, llvm_i32_ty, llvm_i32_ty,
                   llvm_i32_ty, llvm_i32_ty, llvm_i32_ty], []>;
 
diff --git a/include/llvm/IR/IntrinsicsPowerPC.td b/include/llvm/IR/IntrinsicsPowerPC.td
index 74c017234f5..eb8f1e6cd07 100644
--- a/include/llvm/IR/IntrinsicsPowerPC.td
+++ b/include/llvm/IR/IntrinsicsPowerPC.td
@@ -24,7 +24,8 @@ let TargetPrefix = "ppc" in {  // All intrinsics start with "llvm.ppc.".
   def int_ppc_dcbst : Intrinsic<[], [llvm_ptr_ty], []>;
   def int_ppc_dcbt  : Intrinsic<[], [llvm_ptr_ty],
     [IntrReadWriteArgMem, NoCapture<0>]>;
-  def int_ppc_dcbtst: Intrinsic<[], [llvm_ptr_ty], []>;
+  def int_ppc_dcbtst: Intrinsic<[], [llvm_ptr_ty],
+    [IntrReadWriteArgMem, NoCapture<0>]>;
   def int_ppc_dcbz  : Intrinsic<[], [llvm_ptr_ty], []>;
   def int_ppc_dcbzl : Intrinsic<[], [llvm_ptr_ty], []>;
 
@@ -120,6 +121,13 @@ class PowerPC_Vec_DDD_Intrinsic<string GCCIntSuffix>
                           [llvm_v2i64_ty], [llvm_v2i64_ty, llvm_v2i64_ty],
                           [IntrNoMem]>;
 
+/// PowerPC_Vec_QQQ_Intrinsic - A PowerPC intrinsic that takes two v1i128
+/// vectors and returns one. These intrinsics have no side effects.
+class PowerPC_Vec_QQQ_Intrinsic<string GCCIntSuffix>
+  : PowerPC_Vec_Intrinsic<GCCIntSuffix,
+                         [llvm_v1i128_ty], [llvm_v1i128_ty, llvm_v1i128_ty],
+                         [IntrNoMem]>;
+
 //===----------------------------------------------------------------------===//
 // PowerPC VSX Intrinsic Class Definitions.
 //
@@ -356,6 +364,7 @@ def int_ppc_altivec_vaddshs : PowerPC_Vec_HHH_Intrinsic<"vaddshs">;
 def int_ppc_altivec_vadduws : PowerPC_Vec_WWW_Intrinsic<"vadduws">;
 def int_ppc_altivec_vaddsws : PowerPC_Vec_WWW_Intrinsic<"vaddsws">;
 def int_ppc_altivec_vaddcuw : PowerPC_Vec_WWW_Intrinsic<"vaddcuw">;
+def int_ppc_altivec_vaddcuq : PowerPC_Vec_QQQ_Intrinsic<"vaddcuq">;
 
 // Saturating subs.
 def int_ppc_altivec_vsububs : PowerPC_Vec_BBB_Intrinsic<"vsububs">;
@@ -365,6 +374,7 @@ def int_ppc_altivec_vsubshs : PowerPC_Vec_HHH_Intrinsic<"vsubshs">;
 def int_ppc_altivec_vsubuws : PowerPC_Vec_WWW_Intrinsic<"vsubuws">;
 def int_ppc_altivec_vsubsws : PowerPC_Vec_WWW_Intrinsic<"vsubsws">;
 def int_ppc_altivec_vsubcuw : PowerPC_Vec_WWW_Intrinsic<"vsubcuw">;
+def int_ppc_altivec_vsubcuq : PowerPC_Vec_QQQ_Intrinsic<"vsubcuq">;
 
 let TargetPrefix = "ppc" in {  // All PPC intrinsics start with "llvm.ppc.".
   // Saturating multiply-adds.
@@ -479,6 +489,12 @@ let TargetPrefix = "ppc" in {  // All PPC intrinsics start with "llvm.ppc.".
   def int_ppc_altivec_vpkswus : GCCBuiltin<"__builtin_altivec_vpkswus">,
             Intrinsic<[llvm_v8i16_ty], [llvm_v4i32_ty, llvm_v4i32_ty],
                       [IntrNoMem]>;
+  def int_ppc_altivec_vpksdss : GCCBuiltin<"__builtin_altivec_vpksdss">,
+            Intrinsic<[llvm_v4i32_ty], [llvm_v2i64_ty, llvm_v2i64_ty],
+                      [IntrNoMem]>;
+  def int_ppc_altivec_vpksdus : GCCBuiltin<"__builtin_altivec_vpksdus">,
+            Intrinsic<[llvm_v4i32_ty], [llvm_v2i64_ty, llvm_v2i64_ty],
+                      [IntrNoMem]>;
   // vpkuhum is lowered to a shuffle.
   def int_ppc_altivec_vpkuhus : GCCBuiltin<"__builtin_altivec_vpkuhus">,
             Intrinsic<[llvm_v16i8_ty], [llvm_v8i16_ty, llvm_v8i16_ty],
@@ -487,6 +503,10 @@ let TargetPrefix = "ppc" in {  // All PPC intrinsics start with "llvm.ppc.".
   def int_ppc_altivec_vpkuwus : GCCBuiltin<"__builtin_altivec_vpkuwus">,
             Intrinsic<[llvm_v8i16_ty], [llvm_v4i32_ty, llvm_v4i32_ty],
                       [IntrNoMem]>;
+  // vpkudum is lowered to a shuffle.
+  def int_ppc_altivec_vpkudus : GCCBuiltin<"__builtin_altivec_vpkudus">,
+            Intrinsic<[llvm_v4i32_ty], [llvm_v2i64_ty, llvm_v2i64_ty],
+                      [IntrNoMem]>;
 
   // Unpacks.
   def int_ppc_altivec_vupkhpx : GCCBuiltin<"__builtin_altivec_vupkhpx">,
@@ -495,12 +515,16 @@ let TargetPrefix = "ppc" in {  // All PPC intrinsics start with "llvm.ppc.".
             Intrinsic<[llvm_v8i16_ty], [llvm_v16i8_ty], [IntrNoMem]>;
   def int_ppc_altivec_vupkhsh : GCCBuiltin<"__builtin_altivec_vupkhsh">,
             Intrinsic<[llvm_v4i32_ty], [llvm_v8i16_ty], [IntrNoMem]>;
+  def int_ppc_altivec_vupkhsw : GCCBuiltin<"__builtin_altivec_vupkhsw">,
+            Intrinsic<[llvm_v2i64_ty], [llvm_v4i32_ty], [IntrNoMem]>;
   def int_ppc_altivec_vupklpx : GCCBuiltin<"__builtin_altivec_vupklpx">,
             Intrinsic<[llvm_v4i32_ty], [llvm_v8i16_ty], [IntrNoMem]>;
   def int_ppc_altivec_vupklsb : GCCBuiltin<"__builtin_altivec_vupklsb">,
             Intrinsic<[llvm_v8i16_ty], [llvm_v16i8_ty], [IntrNoMem]>;
   def int_ppc_altivec_vupklsh : GCCBuiltin<"__builtin_altivec_vupklsh">,
             Intrinsic<[llvm_v4i32_ty], [llvm_v8i16_ty], [IntrNoMem]>;
+  def int_ppc_altivec_vupklsw : GCCBuiltin<"__builtin_altivec_vupklsw">,
+            Intrinsic<[llvm_v2i64_ty], [llvm_v4i32_ty], [IntrNoMem]>;
 
 
   // FP <-> integer conversion.
@@ -525,6 +549,26 @@ let TargetPrefix = "ppc" in {  // All PPC intrinsics start with "llvm.ppc.".
               Intrinsic<[llvm_v4f32_ty], [llvm_v4f32_ty], [IntrNoMem]>;
   def int_ppc_altivec_vrfiz : GCCBuiltin<"__builtin_altivec_vrfiz">,
               Intrinsic<[llvm_v4f32_ty], [llvm_v4f32_ty], [IntrNoMem]>;
+
+  // Add Extended Quadword
+  def int_ppc_altivec_vaddeuqm : GCCBuiltin<"__builtin_altivec_vaddeuqm">,
+              Intrinsic<[llvm_v1i128_ty], 
+                        [llvm_v1i128_ty, llvm_v1i128_ty, llvm_v1i128_ty],
+                        [IntrNoMem]>;
+  def int_ppc_altivec_vaddecuq : GCCBuiltin<"__builtin_altivec_vaddecuq">,
+              Intrinsic<[llvm_v1i128_ty], 
+                        [llvm_v1i128_ty, llvm_v1i128_ty, llvm_v1i128_ty],
+                        [IntrNoMem]>;
+
+  // Sub Extended Quadword
+  def int_ppc_altivec_vsubeuqm : GCCBuiltin<"__builtin_altivec_vsubeuqm">,
+              Intrinsic<[llvm_v1i128_ty], 
+                        [llvm_v1i128_ty, llvm_v1i128_ty, llvm_v1i128_ty],
+                        [IntrNoMem]>;
+  def int_ppc_altivec_vsubecuq : GCCBuiltin<"__builtin_altivec_vsubecuq">,
+              Intrinsic<[llvm_v1i128_ty], 
+                        [llvm_v1i128_ty, llvm_v1i128_ty, llvm_v1i128_ty],
+                        [IntrNoMem]>;
 }
 
 def int_ppc_altivec_vsl   : PowerPC_Vec_WWW_Intrinsic<"vsl">;
@@ -564,6 +608,11 @@ let TargetPrefix = "ppc" in {  // All PPC intrinsics start with "llvm.ppc.".
   def int_ppc_altivec_vsel : GCCBuiltin<"__builtin_altivec_vsel_4si">,
               Intrinsic<[llvm_v4i32_ty], [llvm_v4i32_ty, 
                          llvm_v4i32_ty, llvm_v4i32_ty], [IntrNoMem]>;
+  def int_ppc_altivec_vgbbd : GCCBuiltin<"__builtin_altivec_vgbbd">,
+              Intrinsic<[llvm_v16i8_ty], [llvm_v16i8_ty], [IntrNoMem]>;
+  def int_ppc_altivec_vbpermq : GCCBuiltin<"__builtin_altivec_vbpermq">,
+              Intrinsic<[llvm_v2i64_ty], [llvm_v16i8_ty, llvm_v16i8_ty],
+                        [IntrNoMem]>;
 }
 
 def int_ppc_altivec_vexptefp  : PowerPC_Vec_FF_Intrinsic<"vexptefp">;
@@ -638,6 +687,47 @@ def int_ppc_vsx_xsmindp : PowerPC_VSX_Sca_DDD_Intrinsic<"xsmindp">;
 // Vector divide.
 def int_ppc_vsx_xvdivdp : PowerPC_VSX_Vec_DDD_Intrinsic<"xvdivdp">;
 def int_ppc_vsx_xvdivsp : PowerPC_VSX_Vec_FFF_Intrinsic<"xvdivsp">;
+
+// Vector round-to-infinity (ceil)
+def int_ppc_vsx_xvrspip :
+      Intrinsic<[llvm_v4f32_ty], [llvm_v4f32_ty], [IntrNoMem]>;
+def int_ppc_vsx_xvrdpip :
+      Intrinsic<[llvm_v2f64_ty], [llvm_v2f64_ty], [IntrNoMem]>;
+
+// Vector reciprocal estimate
+def int_ppc_vsx_xvresp : GCCBuiltin<"__builtin_vsx_xvresp">,
+      Intrinsic<[llvm_v4f32_ty], [llvm_v4f32_ty], [IntrNoMem]>;
+def int_ppc_vsx_xvredp : GCCBuiltin<"__builtin_vsx_xvredp">,
+      Intrinsic<[llvm_v2f64_ty], [llvm_v2f64_ty], [IntrNoMem]>;
+
+// Vector rsqrte
+def int_ppc_vsx_xvrsqrtesp : GCCBuiltin<"__builtin_vsx_xvrsqrtesp">,
+      Intrinsic<[llvm_v4f32_ty], [llvm_v4f32_ty], [IntrNoMem]>;
+def int_ppc_vsx_xvrsqrtedp : GCCBuiltin<"__builtin_vsx_xvrsqrtedp">,
+      Intrinsic<[llvm_v2f64_ty], [llvm_v2f64_ty], [IntrNoMem]>;
+
+// Vector compare
+def int_ppc_vsx_xvcmpeqdp :
+      PowerPC_VSX_Intrinsic<"xvcmpeqdp", [llvm_v2i64_ty],
+                            [llvm_v2f64_ty, llvm_v2f64_ty], [IntrNoMem]>;
+def int_ppc_vsx_xvcmpeqsp :
+      PowerPC_VSX_Intrinsic<"xvcmpeqsp", [llvm_v4i32_ty],
+                            [llvm_v4f32_ty, llvm_v4f32_ty], [IntrNoMem]>;
+def int_ppc_vsx_xvcmpgedp :
+      PowerPC_VSX_Intrinsic<"xvcmpgedp", [llvm_v2i64_ty],
+                            [llvm_v2f64_ty, llvm_v2f64_ty], [IntrNoMem]>;
+def int_ppc_vsx_xvcmpgesp :
+      PowerPC_VSX_Intrinsic<"xvcmpgesp", [llvm_v4i32_ty],
+                            [llvm_v4f32_ty, llvm_v4f32_ty], [IntrNoMem]>;
+def int_ppc_vsx_xvcmpgtdp :
+      PowerPC_VSX_Intrinsic<"xvcmpgtdp", [llvm_v2i64_ty],
+                            [llvm_v2f64_ty, llvm_v2f64_ty], [IntrNoMem]>;
+def int_ppc_vsx_xvcmpgtsp :
+      PowerPC_VSX_Intrinsic<"xvcmpgtsp", [llvm_v4i32_ty],
+                            [llvm_v4f32_ty, llvm_v4f32_ty], [IntrNoMem]>;
+def int_ppc_vsx_xxleqv :
+      PowerPC_VSX_Intrinsic<"xxleqv", [llvm_v4i32_ty],
+                            [llvm_v4i32_ty, llvm_v4i32_ty], [IntrNoMem]>;
 }
 
 //===----------------------------------------------------------------------===//
diff --git a/include/llvm/IR/IntrinsicsR600.td b/include/llvm/IR/IntrinsicsR600.td
deleted file mode 100644
index 50556673822..00000000000
--- a/include/llvm/IR/IntrinsicsR600.td
+++ /dev/null
@@ -1,85 +0,0 @@
-//===- IntrinsicsR600.td - Defines R600 intrinsics ---------*- tablegen -*-===//
-//
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
-//
-//===----------------------------------------------------------------------===//
-//
-// This file defines all of the R600-specific intrinsics.
-//
-//===----------------------------------------------------------------------===//
-
-let TargetPrefix = "r600" in {
-
-class R600ReadPreloadRegisterIntrinsic<string name>
-  : Intrinsic<[llvm_i32_ty], [], [IntrNoMem]>,
-    GCCBuiltin<name>;
-
-multiclass R600ReadPreloadRegisterIntrinsic_xyz<string prefix> {
-  def _x : R600ReadPreloadRegisterIntrinsic<!strconcat(prefix, "_x")>;
-  def _y : R600ReadPreloadRegisterIntrinsic<!strconcat(prefix, "_y")>;
-  def _z : R600ReadPreloadRegisterIntrinsic<!strconcat(prefix, "_z")>;
-}
-
-defm int_r600_read_global_size : R600ReadPreloadRegisterIntrinsic_xyz <
-                                       "__builtin_r600_read_global_size">;
-defm int_r600_read_local_size : R600ReadPreloadRegisterIntrinsic_xyz <
-                                       "__builtin_r600_read_local_size">;
-defm int_r600_read_ngroups : R600ReadPreloadRegisterIntrinsic_xyz <
-                                       "__builtin_r600_read_ngroups">;
-defm int_r600_read_tgid : R600ReadPreloadRegisterIntrinsic_xyz <
-                                       "__builtin_r600_read_tgid">;
-defm int_r600_read_tidig : R600ReadPreloadRegisterIntrinsic_xyz <
-                                       "__builtin_r600_read_tidig">;
-} // End TargetPrefix = "r600"
-
-let TargetPrefix = "AMDGPU" in {
-
-class AMDGPUReadPreloadRegisterIntrinsic<string name>
-  : Intrinsic<[llvm_i32_ty], [], [IntrNoMem]>,
-    GCCBuiltin<name>;
-
-def int_AMDGPU_div_scale : GCCBuiltin<"__builtin_amdgpu_div_scale">,
-  // 1st parameter: Numerator
-  // 2nd parameter: Denominator
-  // 3rd parameter: Constant to select select between first and
-  //                second. (0 = first, 1 = second).
-  Intrinsic<[llvm_anyfloat_ty, llvm_i1_ty],
-            [LLVMMatchType<0>, LLVMMatchType<0>, llvm_i1_ty],
-            [IntrNoMem]>;
-
-def int_AMDGPU_div_fmas : GCCBuiltin<"__builtin_amdgpu_div_fmas">,
-  Intrinsic<[llvm_anyfloat_ty],
-            [LLVMMatchType<0>, LLVMMatchType<0>, LLVMMatchType<0>, llvm_i1_ty],
-            [IntrNoMem]>;
-
-def int_AMDGPU_div_fixup : GCCBuiltin<"__builtin_amdgpu_div_fixup">,
-  Intrinsic<[llvm_anyfloat_ty],
-            [LLVMMatchType<0>, LLVMMatchType<0>, LLVMMatchType<0>],
-            [IntrNoMem]>;
-
-def int_AMDGPU_trig_preop : GCCBuiltin<"__builtin_amdgpu_trig_preop">,
-  Intrinsic<[llvm_anyfloat_ty], [LLVMMatchType<0>, llvm_i32_ty],
-            [IntrNoMem]>;
-
-def int_AMDGPU_rcp : GCCBuiltin<"__builtin_amdgpu_rcp">,
-  Intrinsic<[llvm_anyfloat_ty], [LLVMMatchType<0>], [IntrNoMem]>;
-
-def int_AMDGPU_rsq : GCCBuiltin<"__builtin_amdgpu_rsq">,
-  Intrinsic<[llvm_anyfloat_ty], [LLVMMatchType<0>], [IntrNoMem]>;
-
-def int_AMDGPU_rsq_clamped : GCCBuiltin<"__builtin_amdgpu_rsq_clamped">,
-  Intrinsic<[llvm_anyfloat_ty], [LLVMMatchType<0>], [IntrNoMem]>;
-
-def int_AMDGPU_ldexp : GCCBuiltin<"__builtin_amdgpu_ldexp">,
-  Intrinsic<[llvm_anyfloat_ty], [LLVMMatchType<0>, llvm_i32_ty], [IntrNoMem]>;
-
-def int_AMDGPU_class : GCCBuiltin<"__builtin_amdgpu_class">,
-  Intrinsic<[llvm_i1_ty], [llvm_anyfloat_ty, llvm_i32_ty], [IntrNoMem]>;
-
-def int_AMDGPU_read_workdim : AMDGPUReadPreloadRegisterIntrinsic <
-                                       "__builtin_amdgpu_read_workdim">;
-
-} // End TargetPrefix = "AMDGPU"
diff --git a/include/llvm/IR/IntrinsicsSystemZ.td b/include/llvm/IR/IntrinsicsSystemZ.td
index 6883db3bf07..96e7ca52569 100644
--- a/include/llvm/IR/IntrinsicsSystemZ.td
+++ b/include/llvm/IR/IntrinsicsSystemZ.td
@@ -11,6 +11,185 @@
 //
 //===----------------------------------------------------------------------===//
 
+class SystemZUnaryConv<string name, LLVMType result, LLVMType arg>
+  : GCCBuiltin<"__builtin_s390_" ## name>,
+    Intrinsic<[result], [arg], [IntrNoMem]>;
+
+class SystemZUnary<string name, LLVMType type>
+  : SystemZUnaryConv<name, type, type>;
+
+class SystemZUnaryConvCC<LLVMType result, LLVMType arg>
+  : Intrinsic<[result, llvm_i32_ty], [arg], [IntrNoMem]>;
+
+class SystemZUnaryCC<LLVMType type>
+  : SystemZUnaryConvCC<type, type>;
+
+class SystemZBinaryConv<string name, LLVMType result, LLVMType arg>
+  : GCCBuiltin<"__builtin_s390_" ## name>,
+    Intrinsic<[result], [arg, arg], [IntrNoMem]>;
+
+class SystemZBinary<string name, LLVMType type>
+  : SystemZBinaryConv<name, type, type>;
+
+class SystemZBinaryInt<string name, LLVMType type>
+  : GCCBuiltin<"__builtin_s390_" ## name>,
+    Intrinsic<[type], [type, llvm_i32_ty], [IntrNoMem]>;
+
+class SystemZBinaryConvCC<LLVMType result, LLVMType arg>
+  : Intrinsic<[result, llvm_i32_ty], [arg, arg], [IntrNoMem]>;
+
+class SystemZBinaryConvIntCC<LLVMType result, LLVMType arg>
+  : Intrinsic<[result, llvm_i32_ty], [arg, llvm_i32_ty], [IntrNoMem]>;
+
+class SystemZBinaryCC<LLVMType type>
+  : SystemZBinaryConvCC<type, type>;
+
+class SystemZTernaryConv<string name, LLVMType result, LLVMType arg>
+  : GCCBuiltin<"__builtin_s390_" ## name>,
+    Intrinsic<[result], [arg, arg, result], [IntrNoMem]>;
+
+class SystemZTernary<string name, LLVMType type>
+  : SystemZTernaryConv<name, type, type>;
+
+class SystemZTernaryInt<string name, LLVMType type>
+  : GCCBuiltin<"__builtin_s390_" ## name>,
+    Intrinsic<[type], [type, type, llvm_i32_ty], [IntrNoMem]>;
+
+class SystemZTernaryIntCC<LLVMType type>
+  : Intrinsic<[type, llvm_i32_ty], [type, type, llvm_i32_ty], [IntrNoMem]>;
+
+class SystemZQuaternaryInt<string name, LLVMType type>
+  : GCCBuiltin<"__builtin_s390_" ## name>,
+    Intrinsic<[type], [type, type, type, llvm_i32_ty], [IntrNoMem]>;
+
+class SystemZQuaternaryIntCC<LLVMType type>
+  : Intrinsic<[type, llvm_i32_ty], [type, type, type, llvm_i32_ty],
+              [IntrNoMem]>;
+
+multiclass SystemZUnaryExtBHF<string name> {
+  def b : SystemZUnaryConv<name##"b", llvm_v8i16_ty, llvm_v16i8_ty>;
+  def h : SystemZUnaryConv<name##"h", llvm_v4i32_ty, llvm_v8i16_ty>;
+  def f : SystemZUnaryConv<name##"f", llvm_v2i64_ty, llvm_v4i32_ty>;
+}
+
+multiclass SystemZUnaryExtBHWF<string name> {
+  def b  : SystemZUnaryConv<name##"b",  llvm_v8i16_ty, llvm_v16i8_ty>;
+  def hw : SystemZUnaryConv<name##"hw", llvm_v4i32_ty, llvm_v8i16_ty>;
+  def f  : SystemZUnaryConv<name##"f",  llvm_v2i64_ty, llvm_v4i32_ty>;
+}
+
+multiclass SystemZUnaryBHF<string name> {
+  def b : SystemZUnary<name##"b", llvm_v16i8_ty>;
+  def h : SystemZUnary<name##"h", llvm_v8i16_ty>;
+  def f : SystemZUnary<name##"f", llvm_v4i32_ty>;
+}
+
+multiclass SystemZUnaryBHFG<string name> : SystemZUnaryBHF<name> {
+  def g : SystemZUnary<name##"g", llvm_v2i64_ty>;
+}
+
+multiclass SystemZUnaryCCBHF {
+  def bs : SystemZUnaryCC<llvm_v16i8_ty>;
+  def hs : SystemZUnaryCC<llvm_v8i16_ty>;
+  def fs : SystemZUnaryCC<llvm_v4i32_ty>;
+}
+
+multiclass SystemZBinaryTruncHFG<string name> {
+  def h : SystemZBinaryConv<name##"h", llvm_v16i8_ty, llvm_v8i16_ty>;
+  def f : SystemZBinaryConv<name##"f", llvm_v8i16_ty, llvm_v4i32_ty>;
+  def g : SystemZBinaryConv<name##"g", llvm_v4i32_ty, llvm_v2i64_ty>;
+}
+
+multiclass SystemZBinaryTruncCCHFG {
+  def hs : SystemZBinaryConvCC<llvm_v16i8_ty, llvm_v8i16_ty>;
+  def fs : SystemZBinaryConvCC<llvm_v8i16_ty, llvm_v4i32_ty>;
+  def gs : SystemZBinaryConvCC<llvm_v4i32_ty, llvm_v2i64_ty>;
+}
+
+multiclass SystemZBinaryExtBHF<string name> {
+  def b : SystemZBinaryConv<name##"b", llvm_v8i16_ty, llvm_v16i8_ty>;
+  def h : SystemZBinaryConv<name##"h", llvm_v4i32_ty, llvm_v8i16_ty>;
+  def f : SystemZBinaryConv<name##"f", llvm_v2i64_ty, llvm_v4i32_ty>;
+}
+
+multiclass SystemZBinaryExtBHFG<string name> : SystemZBinaryExtBHF<name> {
+  def g : SystemZBinaryConv<name##"g", llvm_v16i8_ty, llvm_v2i64_ty>;
+}
+
+multiclass SystemZBinaryBHF<string name> {
+  def b : SystemZBinary<name##"b", llvm_v16i8_ty>;
+  def h : SystemZBinary<name##"h", llvm_v8i16_ty>;
+  def f : SystemZBinary<name##"f", llvm_v4i32_ty>;
+}
+
+multiclass SystemZBinaryBHFG<string name> : SystemZBinaryBHF<name> {
+  def g : SystemZBinary<name##"g", llvm_v2i64_ty>;
+}
+
+multiclass SystemZBinaryIntBHFG<string name> {
+  def b : SystemZBinaryInt<name##"b", llvm_v16i8_ty>;
+  def h : SystemZBinaryInt<name##"h", llvm_v8i16_ty>;
+  def f : SystemZBinaryInt<name##"f", llvm_v4i32_ty>;
+  def g : SystemZBinaryInt<name##"g", llvm_v2i64_ty>;
+}
+
+multiclass SystemZBinaryCCBHF {
+  def bs : SystemZBinaryCC<llvm_v16i8_ty>;
+  def hs : SystemZBinaryCC<llvm_v8i16_ty>;
+  def fs : SystemZBinaryCC<llvm_v4i32_ty>;
+}
+
+multiclass SystemZCompareBHFG<string name> {
+  def bs : SystemZBinaryCC<llvm_v16i8_ty>;
+  def hs : SystemZBinaryCC<llvm_v8i16_ty>;
+  def fs : SystemZBinaryCC<llvm_v4i32_ty>;
+  def gs : SystemZBinaryCC<llvm_v2i64_ty>;
+}
+
+multiclass SystemZTernaryExtBHF<string name> {
+  def b : SystemZTernaryConv<name##"b", llvm_v8i16_ty, llvm_v16i8_ty>;
+  def h : SystemZTernaryConv<name##"h", llvm_v4i32_ty, llvm_v8i16_ty>;
+  def f : SystemZTernaryConv<name##"f", llvm_v2i64_ty, llvm_v4i32_ty>;
+}
+
+multiclass SystemZTernaryExtBHFG<string name> : SystemZTernaryExtBHF<name> {
+  def g : SystemZTernaryConv<name##"g", llvm_v16i8_ty, llvm_v2i64_ty>;
+}
+
+multiclass SystemZTernaryBHF<string name> {
+  def b : SystemZTernary<name##"b", llvm_v16i8_ty>;
+  def h : SystemZTernary<name##"h", llvm_v8i16_ty>;
+  def f : SystemZTernary<name##"f", llvm_v4i32_ty>;
+}
+
+multiclass SystemZTernaryIntBHF<string name> {
+  def b : SystemZTernaryInt<name##"b", llvm_v16i8_ty>;
+  def h : SystemZTernaryInt<name##"h", llvm_v8i16_ty>;
+  def f : SystemZTernaryInt<name##"f", llvm_v4i32_ty>;
+}
+
+multiclass SystemZTernaryIntCCBHF {
+  def bs : SystemZTernaryIntCC<llvm_v16i8_ty>;
+  def hs : SystemZTernaryIntCC<llvm_v8i16_ty>;
+  def fs : SystemZTernaryIntCC<llvm_v4i32_ty>;
+}
+
+multiclass SystemZQuaternaryIntBHF<string name> {
+  def b : SystemZQuaternaryInt<name##"b", llvm_v16i8_ty>;
+  def h : SystemZQuaternaryInt<name##"h", llvm_v8i16_ty>;
+  def f : SystemZQuaternaryInt<name##"f", llvm_v4i32_ty>;
+}
+
+multiclass SystemZQuaternaryIntBHFG<string name> : SystemZQuaternaryIntBHF<name> {
+  def g : SystemZQuaternaryInt<name##"g", llvm_v2i64_ty>;
+}
+
+multiclass SystemZQuaternaryIntCCBHF {
+  def bs : SystemZQuaternaryIntCC<llvm_v16i8_ty>;
+  def hs : SystemZQuaternaryIntCC<llvm_v8i16_ty>;
+  def fs : SystemZQuaternaryIntCC<llvm_v4i32_ty>;
+}
+
 //===----------------------------------------------------------------------===//
 //
 // Transactional-execution intrinsics
@@ -44,3 +223,154 @@ let TargetPrefix = "s390" in {
                               Intrinsic<[], [llvm_i32_ty]>;
 }
 
+//===----------------------------------------------------------------------===//
+//
+// Vector intrinsics
+//
+//===----------------------------------------------------------------------===//
+
+let TargetPrefix = "s390" in {
+  def int_s390_lcbb : GCCBuiltin<"__builtin_s390_lcbb">,
+                      Intrinsic<[llvm_i32_ty], [llvm_ptr_ty, llvm_i32_ty],
+                                [IntrNoMem]>;
+
+  def int_s390_vlbb : GCCBuiltin<"__builtin_s390_vlbb">,
+                      Intrinsic<[llvm_v16i8_ty], [llvm_ptr_ty, llvm_i32_ty],
+                                [IntrReadArgMem]>;
+
+  def int_s390_vll : GCCBuiltin<"__builtin_s390_vll">,
+                     Intrinsic<[llvm_v16i8_ty], [llvm_i32_ty, llvm_ptr_ty],
+                               [IntrReadArgMem]>;
+
+  def int_s390_vpdi : GCCBuiltin<"__builtin_s390_vpdi">,
+                      Intrinsic<[llvm_v2i64_ty],
+                                [llvm_v2i64_ty, llvm_v2i64_ty, llvm_i32_ty],
+                                [IntrNoMem]>;
+
+  def int_s390_vperm : GCCBuiltin<"__builtin_s390_vperm">,
+                       Intrinsic<[llvm_v16i8_ty],
+                                 [llvm_v16i8_ty, llvm_v16i8_ty, llvm_v16i8_ty],
+                                 [IntrNoMem]>;
+
+  defm int_s390_vpks : SystemZBinaryTruncHFG<"vpks">;
+  defm int_s390_vpks : SystemZBinaryTruncCCHFG;
+
+  defm int_s390_vpkls : SystemZBinaryTruncHFG<"vpkls">;
+  defm int_s390_vpkls : SystemZBinaryTruncCCHFG;
+
+  def int_s390_vstl : GCCBuiltin<"__builtin_s390_vstl">,
+                      Intrinsic<[], [llvm_v16i8_ty, llvm_i32_ty, llvm_ptr_ty],
+                                // In fact write-only but there's no property
+                                // for that.
+                                [IntrReadWriteArgMem]>;
+
+  defm int_s390_vupl  : SystemZUnaryExtBHWF<"vupl">;
+  defm int_s390_vupll : SystemZUnaryExtBHF<"vupll">;
+
+  defm int_s390_vuph  : SystemZUnaryExtBHF<"vuph">;
+  defm int_s390_vuplh : SystemZUnaryExtBHF<"vuplh">;
+
+  defm int_s390_vacc : SystemZBinaryBHFG<"vacc">;
+
+  def int_s390_vaq    : SystemZBinary<"vaq",     llvm_v16i8_ty>;
+  def int_s390_vacq   : SystemZTernary<"vacq",   llvm_v16i8_ty>;
+  def int_s390_vaccq  : SystemZBinary<"vaccq",   llvm_v16i8_ty>;
+  def int_s390_vacccq : SystemZTernary<"vacccq", llvm_v16i8_ty>;
+
+  defm int_s390_vavg  : SystemZBinaryBHFG<"vavg">;
+  defm int_s390_vavgl : SystemZBinaryBHFG<"vavgl">;
+
+  def int_s390_vcksm : SystemZBinary<"vcksm", llvm_v4i32_ty>;
+
+  defm int_s390_vgfm  : SystemZBinaryExtBHFG<"vgfm">;
+  defm int_s390_vgfma : SystemZTernaryExtBHFG<"vgfma">;
+
+  defm int_s390_vmah  : SystemZTernaryBHF<"vmah">;
+  defm int_s390_vmalh : SystemZTernaryBHF<"vmalh">;
+  defm int_s390_vmae  : SystemZTernaryExtBHF<"vmae">;
+  defm int_s390_vmale : SystemZTernaryExtBHF<"vmale">;
+  defm int_s390_vmao  : SystemZTernaryExtBHF<"vmao">;
+  defm int_s390_vmalo : SystemZTernaryExtBHF<"vmalo">;
+
+  defm int_s390_vmh  : SystemZBinaryBHF<"vmh">;
+  defm int_s390_vmlh : SystemZBinaryBHF<"vmlh">;
+  defm int_s390_vme  : SystemZBinaryExtBHF<"vme">;
+  defm int_s390_vmle : SystemZBinaryExtBHF<"vmle">;
+  defm int_s390_vmo  : SystemZBinaryExtBHF<"vmo">;
+  defm int_s390_vmlo : SystemZBinaryExtBHF<"vmlo">;
+
+  defm int_s390_verllv : SystemZBinaryBHFG<"verllv">;
+  defm int_s390_verll  : SystemZBinaryIntBHFG<"verll">;
+  defm int_s390_verim  : SystemZQuaternaryIntBHFG<"verim">;
+
+  def int_s390_vsl   : SystemZBinary<"vsl",   llvm_v16i8_ty>;
+  def int_s390_vslb  : SystemZBinary<"vslb",  llvm_v16i8_ty>;
+  def int_s390_vsra  : SystemZBinary<"vsra",  llvm_v16i8_ty>;
+  def int_s390_vsrab : SystemZBinary<"vsrab", llvm_v16i8_ty>;
+  def int_s390_vsrl  : SystemZBinary<"vsrl",  llvm_v16i8_ty>;
+  def int_s390_vsrlb : SystemZBinary<"vsrlb", llvm_v16i8_ty>;
+
+  def int_s390_vsldb : GCCBuiltin<"__builtin_s390_vsldb">,
+                       Intrinsic<[llvm_v16i8_ty],
+                                 [llvm_v16i8_ty, llvm_v16i8_ty, llvm_i32_ty],
+                                 [IntrNoMem]>;
+
+  defm int_s390_vscbi : SystemZBinaryBHFG<"vscbi">;
+
+  def int_s390_vsq     : SystemZBinary<"vsq",      llvm_v16i8_ty>;
+  def int_s390_vsbiq   : SystemZTernary<"vsbiq",   llvm_v16i8_ty>;
+  def int_s390_vscbiq  : SystemZBinary<"vscbiq",   llvm_v16i8_ty>;
+  def int_s390_vsbcbiq : SystemZTernary<"vsbcbiq", llvm_v16i8_ty>;
+
+  def int_s390_vsumb : SystemZBinaryConv<"vsumb", llvm_v4i32_ty, llvm_v16i8_ty>;
+  def int_s390_vsumh : SystemZBinaryConv<"vsumh", llvm_v4i32_ty, llvm_v8i16_ty>;
+
+  def int_s390_vsumgh : SystemZBinaryConv<"vsumgh", llvm_v2i64_ty,
+                                          llvm_v8i16_ty>;
+  def int_s390_vsumgf : SystemZBinaryConv<"vsumgf", llvm_v2i64_ty,
+                                          llvm_v4i32_ty>;
+
+  def int_s390_vsumqf : SystemZBinaryConv<"vsumqf", llvm_v16i8_ty,
+                                          llvm_v4i32_ty>;
+  def int_s390_vsumqg : SystemZBinaryConv<"vsumqg", llvm_v16i8_ty,
+                                          llvm_v2i64_ty>;
+
+  def int_s390_vtm : SystemZBinaryConv<"vtm", llvm_i32_ty, llvm_v16i8_ty>;
+
+  defm int_s390_vceq : SystemZCompareBHFG<"vceq">;
+  defm int_s390_vch  : SystemZCompareBHFG<"vch">;
+  defm int_s390_vchl : SystemZCompareBHFG<"vchl">;
+
+  defm int_s390_vfae  : SystemZTernaryIntBHF<"vfae">;
+  defm int_s390_vfae  : SystemZTernaryIntCCBHF;
+  defm int_s390_vfaez : SystemZTernaryIntBHF<"vfaez">;
+  defm int_s390_vfaez : SystemZTernaryIntCCBHF;
+
+  defm int_s390_vfee  : SystemZBinaryBHF<"vfee">;
+  defm int_s390_vfee  : SystemZBinaryCCBHF;
+  defm int_s390_vfeez : SystemZBinaryBHF<"vfeez">;
+  defm int_s390_vfeez : SystemZBinaryCCBHF;
+
+  defm int_s390_vfene  : SystemZBinaryBHF<"vfene">;
+  defm int_s390_vfene  : SystemZBinaryCCBHF;
+  defm int_s390_vfenez : SystemZBinaryBHF<"vfenez">;
+  defm int_s390_vfenez : SystemZBinaryCCBHF;
+
+  defm int_s390_vistr : SystemZUnaryBHF<"vistr">;
+  defm int_s390_vistr : SystemZUnaryCCBHF;
+
+  defm int_s390_vstrc  : SystemZQuaternaryIntBHF<"vstrc">;
+  defm int_s390_vstrc  : SystemZQuaternaryIntCCBHF;
+  defm int_s390_vstrcz : SystemZQuaternaryIntBHF<"vstrcz">;
+  defm int_s390_vstrcz : SystemZQuaternaryIntCCBHF;
+
+  def int_s390_vfcedbs  : SystemZBinaryConvCC<llvm_v2i64_ty, llvm_v2f64_ty>;
+  def int_s390_vfchdbs  : SystemZBinaryConvCC<llvm_v2i64_ty, llvm_v2f64_ty>;
+  def int_s390_vfchedbs : SystemZBinaryConvCC<llvm_v2i64_ty, llvm_v2f64_ty>;
+
+  def int_s390_vftcidb : SystemZBinaryConvIntCC<llvm_v2i64_ty, llvm_v2f64_ty>;
+
+  def int_s390_vfidb : Intrinsic<[llvm_v2f64_ty],
+                                 [llvm_v2f64_ty, llvm_i32_ty, llvm_i32_ty],
+                                 [IntrNoMem]>;
+}
diff --git a/include/llvm/IR/IntrinsicsWebAssembly.td b/include/llvm/IR/IntrinsicsWebAssembly.td
new file mode 100644
index 00000000000..3ccde474238
--- /dev/null
+++ b/include/llvm/IR/IntrinsicsWebAssembly.td
@@ -0,0 +1,16 @@
+//===- IntrinsicsWebAssembly.td - Defines wasm intrinsics --*- tablegen -*-===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+///
+/// \file
+/// \brief This file defines all of the WebAssembly-specific intrinsics.
+///
+//===----------------------------------------------------------------------===//
+
+let TargetPrefix = "wasm" in {  // All intrinsics start with "llvm.wasm.".
+}
diff --git a/include/llvm/IR/IntrinsicsX86.td b/include/llvm/IR/IntrinsicsX86.td
index 49231d814f3..0d60aefed15 100644
--- a/include/llvm/IR/IntrinsicsX86.td
+++ b/include/llvm/IR/IntrinsicsX86.td
@@ -17,6 +17,23 @@ let TargetPrefix = "x86" in {  // All intrinsics start with "llvm.x86.".
   def int_x86_int : Intrinsic<[], [llvm_i8_ty]>;
 }
 
+//===----------------------------------------------------------------------===//
+// SEH intrinsics for Windows
+let TargetPrefix = "x86" in {
+  def int_x86_seh_lsda : Intrinsic<[llvm_ptr_ty], [llvm_ptr_ty], [IntrNoMem]>;
+
+  // Restores the frame, base, and stack pointers as necessary after recovering
+  // from an exception. Any block resuming control flow in the parent function
+  // should call this before accessing any stack memory.
+  def int_x86_seh_restoreframe : Intrinsic<[], [], []>;
+
+  // Given a pointer to the end of an EH registration object, returns the true
+  // parent frame address that can be used with llvm.localrecover.
+  def int_x86_seh_recoverfp : Intrinsic<[llvm_ptr_ty],
+                                        [llvm_ptr_ty, llvm_ptr_ty],
+                                        [IntrNoMem]>;
+}
+
 //===----------------------------------------------------------------------===//
 // Read Time Stamp Counter.
 let TargetPrefix = "x86" in {
@@ -1123,28 +1140,292 @@ let TargetPrefix = "x86" in {  // All intrinsics start with "llvm.x86.".
         Intrinsic<[llvm_v8i32_ty], [llvm_v8i32_ty,
                   llvm_v8i32_ty, llvm_i8_ty], [IntrNoMem]>;
 
-  def int_x86_avx512_mask_vpermt_d_512:
+  def int_x86_avx512_mask_vpermi2var_d_128 : 
+       GCCBuiltin<"__builtin_ia32_vpermi2vard128_mask">,
+        Intrinsic<[llvm_v4i32_ty],
+        [llvm_v4i32_ty, llvm_v4i32_ty, llvm_v4i32_ty, llvm_i8_ty],
+        [IntrNoMem]>;
+
+  def int_x86_avx512_mask_vpermi2var_d_256 : 
+        GCCBuiltin<"__builtin_ia32_vpermi2vard256_mask">,
+          Intrinsic<[llvm_v8i32_ty],
+          [llvm_v8i32_ty, llvm_v8i32_ty, llvm_v8i32_ty, llvm_i8_ty],
+          [IntrNoMem]>;
+
+  def int_x86_avx512_mask_vpermi2var_d_512 : 
+        GCCBuiltin<"__builtin_ia32_vpermi2vard512_mask">,
+          Intrinsic<[llvm_v16i32_ty],
+          [llvm_v16i32_ty, llvm_v16i32_ty, llvm_v16i32_ty, llvm_i16_ty],
+          [IntrNoMem]>;
+
+  def int_x86_avx512_mask_vpermi2var_hi_128 : 
+        GCCBuiltin<"__builtin_ia32_vpermi2varhi128_mask">,
+          Intrinsic<[llvm_v8i16_ty],
+          [llvm_v8i16_ty, llvm_v8i16_ty, llvm_v8i16_ty, llvm_i8_ty],
+          [IntrNoMem]>;
+
+  def int_x86_avx512_mask_vpermi2var_hi_256 : 
+        GCCBuiltin<"__builtin_ia32_vpermi2varhi256_mask">,
+          Intrinsic<[llvm_v16i16_ty],
+          [llvm_v16i16_ty, llvm_v16i16_ty, llvm_v16i16_ty, llvm_i16_ty],
+          [IntrNoMem]>;
+
+  def int_x86_avx512_mask_vpermi2var_hi_512 : 
+        GCCBuiltin<"__builtin_ia32_vpermi2varhi512_mask">,
+          Intrinsic<[llvm_v32i16_ty],
+          [llvm_v32i16_ty, llvm_v32i16_ty, llvm_v32i16_ty, llvm_i32_ty],
+          [IntrNoMem]>;
+
+  def int_x86_avx512_mask_vpermi2var_pd_128 : 
+        GCCBuiltin<"__builtin_ia32_vpermi2varpd128_mask">,
+          Intrinsic<[llvm_v2f64_ty],
+          [llvm_v2f64_ty, llvm_v2i64_ty, llvm_v2f64_ty, llvm_i8_ty],
+          [IntrNoMem]>;
+
+  def int_x86_avx512_mask_vpermi2var_pd_256 : 
+        GCCBuiltin<"__builtin_ia32_vpermi2varpd256_mask">,
+          Intrinsic<[llvm_v4f64_ty],
+          [llvm_v4f64_ty, llvm_v4i64_ty, llvm_v4f64_ty, llvm_i8_ty],
+          [IntrNoMem]>;
+
+  def int_x86_avx512_mask_vpermi2var_pd_512 : 
+        GCCBuiltin<"__builtin_ia32_vpermi2varpd512_mask">,
+          Intrinsic<[llvm_v8f64_ty],
+          [llvm_v8f64_ty, llvm_v8i64_ty, llvm_v8f64_ty, llvm_i8_ty],
+          [IntrNoMem]>;
+
+  def int_x86_avx512_mask_vpermi2var_ps_128 : 
+        GCCBuiltin<"__builtin_ia32_vpermi2varps128_mask">,
+          Intrinsic<[llvm_v4f32_ty],
+          [llvm_v4f32_ty, llvm_v4i32_ty, llvm_v4f32_ty, llvm_i8_ty],
+          [IntrNoMem]>;
+
+  def int_x86_avx512_mask_vpermi2var_ps_256 : 
+        GCCBuiltin<"__builtin_ia32_vpermi2varps256_mask">,
+          Intrinsic<[llvm_v8f32_ty],
+          [llvm_v8f32_ty, llvm_v8i32_ty, llvm_v8f32_ty, llvm_i8_ty],
+          [IntrNoMem]>;
+
+  def int_x86_avx512_mask_vpermi2var_ps_512 : 
+        GCCBuiltin<"__builtin_ia32_vpermi2varps512_mask">,
+          Intrinsic<[llvm_v16f32_ty],
+          [llvm_v16f32_ty, llvm_v16i32_ty, llvm_v16f32_ty, llvm_i16_ty],
+          [IntrNoMem]>;
+
+  def int_x86_avx512_mask_vpermi2var_q_128 : 
+        GCCBuiltin<"__builtin_ia32_vpermi2varq128_mask">,
+          Intrinsic<[llvm_v2i64_ty],
+          [llvm_v2i64_ty, llvm_v2i64_ty, llvm_v2i64_ty, llvm_i8_ty],
+          [IntrNoMem]>;
+
+  def int_x86_avx512_mask_vpermi2var_q_256 : 
+        GCCBuiltin<"__builtin_ia32_vpermi2varq256_mask">,
+          Intrinsic<[llvm_v4i64_ty],
+          [llvm_v4i64_ty, llvm_v4i64_ty, llvm_v4i64_ty, llvm_i8_ty],
+          [IntrNoMem]>;
+
+  def int_x86_avx512_mask_vpermi2var_q_512 : 
+        GCCBuiltin<"__builtin_ia32_vpermi2varq512_mask">,
+          Intrinsic<[llvm_v8i64_ty],
+          [llvm_v8i64_ty, llvm_v8i64_ty, llvm_v8i64_ty, llvm_i8_ty],
+          [IntrNoMem]>;
+
+  def int_x86_avx512_mask_vpermt2var_d_512:
         GCCBuiltin<"__builtin_ia32_vpermt2vard512_mask">,
         Intrinsic<[llvm_v16i32_ty], [llvm_v16i32_ty,
                   llvm_v16i32_ty, llvm_v16i32_ty, llvm_i16_ty], [IntrNoMem]>;
 
-  def int_x86_avx512_mask_vpermt_q_512:
+  def int_x86_avx512_mask_vpermt2var_q_512:
         GCCBuiltin<"__builtin_ia32_vpermt2varq512_mask">,
         Intrinsic<[llvm_v8i64_ty], [llvm_v8i64_ty,
                   llvm_v8i64_ty, llvm_v8i64_ty, llvm_i8_ty], [IntrNoMem]>;
 
-  def int_x86_avx512_mask_vpermt_ps_512:
+  def int_x86_avx512_mask_vpermt2var_ps_512:
         GCCBuiltin<"__builtin_ia32_vpermt2varps512_mask">,
         Intrinsic<[llvm_v16f32_ty], [llvm_v16i32_ty,
                   llvm_v16f32_ty, llvm_v16f32_ty, llvm_i16_ty], [IntrNoMem]>;
 
-  def int_x86_avx512_mask_vpermt_pd_512:
+  def int_x86_avx512_mask_vpermt2var_pd_512:
         GCCBuiltin<"__builtin_ia32_vpermt2varpd512_mask">,
         Intrinsic<[llvm_v8f64_ty], [llvm_v8i64_ty,
                   llvm_v8f64_ty, llvm_v8f64_ty, llvm_i8_ty], [IntrNoMem]>;
 
+  def int_x86_avx512_mask_vpermt2var_d_128 : 
+        GCCBuiltin<"__builtin_ia32_vpermt2vard128_mask">,
+          Intrinsic<[llvm_v4i32_ty],
+          [llvm_v4i32_ty, llvm_v4i32_ty, llvm_v4i32_ty, llvm_i8_ty],
+          [IntrNoMem]>;
+
+  def int_x86_avx512_maskz_vpermt2var_d_128 : 
+        GCCBuiltin<"__builtin_ia32_vpermt2vard128_maskz">,
+          Intrinsic<[llvm_v4i32_ty],
+          [llvm_v4i32_ty, llvm_v4i32_ty, llvm_v4i32_ty, llvm_i8_ty],
+          [IntrNoMem]>;
+
+  def int_x86_avx512_mask_vpermt2var_d_256 : 
+        GCCBuiltin<"__builtin_ia32_vpermt2vard256_mask">,
+          Intrinsic<[llvm_v8i32_ty],
+          [llvm_v8i32_ty, llvm_v8i32_ty, llvm_v8i32_ty, llvm_i8_ty],
+          [IntrNoMem]>;
+
+  def int_x86_avx512_maskz_vpermt2var_d_256 : 
+        GCCBuiltin<"__builtin_ia32_vpermt2vard256_maskz">,
+          Intrinsic<[llvm_v8i32_ty],
+          [llvm_v8i32_ty, llvm_v8i32_ty, llvm_v8i32_ty, llvm_i8_ty],
+          [IntrNoMem]>;
+
+  def int_x86_avx512_maskz_vpermt2var_d_512 : 
+        GCCBuiltin<"__builtin_ia32_vpermt2vard512_maskz">,
+          Intrinsic<[llvm_v16i32_ty],
+          [llvm_v16i32_ty, llvm_v16i32_ty, llvm_v16i32_ty, llvm_i16_ty],
+          [IntrNoMem]>;
+
+  def int_x86_avx512_mask_vpermt2var_hi_128 : 
+        GCCBuiltin<"__builtin_ia32_vpermt2varhi128_mask">,
+          Intrinsic<[llvm_v8i16_ty],
+          [llvm_v8i16_ty, llvm_v8i16_ty, llvm_v8i16_ty, llvm_i8_ty],
+          [IntrNoMem]>;
+
+  def int_x86_avx512_maskz_vpermt2var_hi_128 : 
+        GCCBuiltin<"__builtin_ia32_vpermt2varhi128_maskz">,
+          Intrinsic<[llvm_v8i16_ty],
+          [llvm_v8i16_ty, llvm_v8i16_ty, llvm_v8i16_ty, llvm_i8_ty],
+          [IntrNoMem]>;
+
+  def int_x86_avx512_mask_vpermt2var_hi_256 : 
+        GCCBuiltin<"__builtin_ia32_vpermt2varhi256_mask">,
+          Intrinsic<[llvm_v16i16_ty],
+          [llvm_v16i16_ty, llvm_v16i16_ty, llvm_v16i16_ty, llvm_i16_ty],
+          [IntrNoMem]>;
+
+  def int_x86_avx512_maskz_vpermt2var_hi_256 : 
+        GCCBuiltin<"__builtin_ia32_vpermt2varhi256_maskz">,
+          Intrinsic<[llvm_v16i16_ty],
+          [llvm_v16i16_ty, llvm_v16i16_ty, llvm_v16i16_ty, llvm_i16_ty],
+          [IntrNoMem]>;
+
+  def int_x86_avx512_mask_vpermt2var_hi_512 : 
+        GCCBuiltin<"__builtin_ia32_vpermt2varhi512_mask">,
+          Intrinsic<[llvm_v32i16_ty],
+          [llvm_v32i16_ty, llvm_v32i16_ty, llvm_v32i16_ty, llvm_i32_ty],
+          [IntrNoMem]>;
+
+  def int_x86_avx512_maskz_vpermt2var_hi_512 : 
+        GCCBuiltin<"__builtin_ia32_vpermt2varhi512_maskz">,
+          Intrinsic<[llvm_v32i16_ty],
+          [llvm_v32i16_ty, llvm_v32i16_ty, llvm_v32i16_ty, llvm_i32_ty],
+          [IntrNoMem]>;
+
+  def int_x86_avx512_mask_vpermt2var_pd_128 : 
+        GCCBuiltin<"__builtin_ia32_vpermt2varpd128_mask">,
+          Intrinsic<[llvm_v2f64_ty],
+          [llvm_v2i64_ty, llvm_v2f64_ty, llvm_v2f64_ty, llvm_i8_ty],
+          [IntrNoMem]>;
+
+  def int_x86_avx512_maskz_vpermt2var_pd_128 : 
+        GCCBuiltin<"__builtin_ia32_vpermt2varpd128_maskz">,
+          Intrinsic<[llvm_v2f64_ty],
+          [llvm_v2i64_ty, llvm_v2f64_ty, llvm_v2f64_ty, llvm_i8_ty],
+          [IntrNoMem]>;
+
+  def int_x86_avx512_mask_vpermt2var_pd_256 : 
+        GCCBuiltin<"__builtin_ia32_vpermt2varpd256_mask">,
+          Intrinsic<[llvm_v4f64_ty],
+          [llvm_v4i64_ty, llvm_v4f64_ty, llvm_v4f64_ty, llvm_i8_ty],
+          [IntrNoMem]>;
+
+  def int_x86_avx512_maskz_vpermt2var_pd_256 : 
+        GCCBuiltin<"__builtin_ia32_vpermt2varpd256_maskz">,
+          Intrinsic<[llvm_v4f64_ty],
+          [llvm_v4i64_ty, llvm_v4f64_ty, llvm_v4f64_ty, llvm_i8_ty],
+          [IntrNoMem]>;
+
+  def int_x86_avx512_maskz_vpermt2var_pd_512 : 
+        GCCBuiltin<"__builtin_ia32_vpermt2varpd512_maskz">,
+          Intrinsic<[llvm_v8f64_ty],
+          [llvm_v8i64_ty, llvm_v8f64_ty, llvm_v8f64_ty, llvm_i8_ty],
+          [IntrNoMem]>;
+
+  def int_x86_avx512_mask_vpermt2var_ps_128 : 
+        GCCBuiltin<"__builtin_ia32_vpermt2varps128_mask">,
+          Intrinsic<[llvm_v4f32_ty],
+          [llvm_v4i32_ty, llvm_v4f32_ty, llvm_v4f32_ty, llvm_i8_ty],
+          [IntrNoMem]>;
+
+  def int_x86_avx512_maskz_vpermt2var_ps_128 : 
+        GCCBuiltin<"__builtin_ia32_vpermt2varps128_maskz">,
+          Intrinsic<[llvm_v4f32_ty],
+          [llvm_v4i32_ty, llvm_v4f32_ty, llvm_v4f32_ty, llvm_i8_ty],
+          [IntrNoMem]>;
+
+  def int_x86_avx512_mask_vpermt2var_ps_256 : 
+        GCCBuiltin<"__builtin_ia32_vpermt2varps256_mask">,
+          Intrinsic<[llvm_v8f32_ty],
+          [llvm_v8i32_ty, llvm_v8f32_ty, llvm_v8f32_ty, llvm_i8_ty],
+          [IntrNoMem]>;
+
+  def int_x86_avx512_maskz_vpermt2var_ps_256 : 
+        GCCBuiltin<"__builtin_ia32_vpermt2varps256_maskz">,
+          Intrinsic<[llvm_v8f32_ty],
+          [llvm_v8i32_ty, llvm_v8f32_ty, llvm_v8f32_ty, llvm_i8_ty],
+          [IntrNoMem]>;
+
+  def int_x86_avx512_maskz_vpermt2var_ps_512 : 
+        GCCBuiltin<"__builtin_ia32_vpermt2varps512_maskz">,
+          Intrinsic<[llvm_v16f32_ty],
+          [llvm_v16i32_ty, llvm_v16f32_ty, llvm_v16f32_ty, llvm_i16_ty],
+          [IntrNoMem]>;
+
+  def int_x86_avx512_mask_vpermt2var_q_128 : 
+        GCCBuiltin<"__builtin_ia32_vpermt2varq128_mask">,
+          Intrinsic<[llvm_v2i64_ty],
+          [llvm_v2i64_ty, llvm_v2i64_ty, llvm_v2i64_ty, llvm_i8_ty],
+          [IntrNoMem]>;
+
+  def int_x86_avx512_maskz_vpermt2var_q_128 : 
+        GCCBuiltin<"__builtin_ia32_vpermt2varq128_maskz">,
+          Intrinsic<[llvm_v2i64_ty],
+          [llvm_v2i64_ty, llvm_v2i64_ty, llvm_v2i64_ty, llvm_i8_ty],
+          [IntrNoMem]>;
+
+  def int_x86_avx512_mask_vpermt2var_q_256 : 
+        GCCBuiltin<"__builtin_ia32_vpermt2varq256_mask">,
+          Intrinsic<[llvm_v4i64_ty],
+          [llvm_v4i64_ty, llvm_v4i64_ty, llvm_v4i64_ty, llvm_i8_ty],
+          [IntrNoMem]>;
+
+  def int_x86_avx512_maskz_vpermt2var_q_256 : 
+        GCCBuiltin<"__builtin_ia32_vpermt2varq256_maskz">,
+          Intrinsic<[llvm_v4i64_ty],
+          [llvm_v4i64_ty, llvm_v4i64_ty, llvm_v4i64_ty, llvm_i8_ty],
+          [IntrNoMem]>;
+
+  def int_x86_avx512_maskz_vpermt2var_q_512 : 
+        GCCBuiltin<"__builtin_ia32_vpermt2varq512_maskz">,
+          Intrinsic<[llvm_v8i64_ty],
+          [llvm_v8i64_ty, llvm_v8i64_ty, llvm_v8i64_ty, llvm_i8_ty],
+          [IntrNoMem]>;
+
+  def int_x86_avx512_mask_pshuf_b_128 : 
+        GCCBuiltin<"__builtin_ia32_pshufb128_mask">,
+          Intrinsic<[llvm_v16i8_ty],
+          [llvm_v16i8_ty, llvm_v16i8_ty, llvm_v16i8_ty,  llvm_i16_ty],
+          [IntrNoMem]>;
+
+  def int_x86_avx512_mask_pshuf_b_256 : 
+        GCCBuiltin<"__builtin_ia32_pshufb256_mask">,
+          Intrinsic<[llvm_v32i8_ty],
+          [llvm_v32i8_ty, llvm_v32i8_ty, llvm_v32i8_ty,  llvm_i32_ty],
+          [IntrNoMem]>;
+
+  def int_x86_avx512_mask_pshuf_b_512 : 
+        GCCBuiltin<"__builtin_ia32_pshufb512_mask">,
+          Intrinsic<[llvm_v64i8_ty],
+          [llvm_v64i8_ty, llvm_v64i8_ty, llvm_v64i8_ty,  llvm_i64_ty],
+          [IntrNoMem]>;
 }
 
+
 // Vector blend
 let TargetPrefix = "x86" in {  // All intrinsics start with "llvm.x86.".
   def int_x86_avx_blendv_pd_256 : GCCBuiltin<"__builtin_ia32_blendvpd256">,
@@ -1448,30 +1729,150 @@ let TargetPrefix = "x86" in {  // All intrinsics start with "llvm.x86.".
   def int_x86_avx2_pmins_d : GCCBuiltin<"__builtin_ia32_pminsd256">,
               Intrinsic<[llvm_v8i32_ty], [llvm_v8i32_ty,
                          llvm_v8i32_ty], [IntrNoMem, Commutative]>;
+  def int_x86_avx512_mask_pmaxs_b_128 : GCCBuiltin<"__builtin_ia32_pmaxsb128_mask">,
+              Intrinsic<[llvm_v16i8_ty], [llvm_v16i8_ty, llvm_v16i8_ty, 
+                         llvm_v16i8_ty, llvm_i16_ty], [IntrNoMem]>;
+  def int_x86_avx512_mask_pmaxs_b_256 : GCCBuiltin<"__builtin_ia32_pmaxsb256_mask">,
+              Intrinsic<[llvm_v32i8_ty], [llvm_v32i8_ty, llvm_v32i8_ty, 
+                         llvm_v32i8_ty,  llvm_i32_ty], [IntrNoMem]>;
+  def int_x86_avx512_mask_pmaxs_b_512 : GCCBuiltin<"__builtin_ia32_pmaxsb512_mask">,
+              Intrinsic<[llvm_v64i8_ty], [llvm_v64i8_ty, llvm_v64i8_ty, 
+                         llvm_v64i8_ty,  llvm_i64_ty], [IntrNoMem]>;
+  def int_x86_avx512_mask_pmaxu_b_128 : GCCBuiltin<"__builtin_ia32_pmaxub128_mask">,
+              Intrinsic<[llvm_v16i8_ty], [llvm_v16i8_ty, llvm_v16i8_ty, 
+                         llvm_v16i8_ty, llvm_i16_ty], [IntrNoMem]>;
+  def int_x86_avx512_mask_pmaxu_b_256 : GCCBuiltin<"__builtin_ia32_pmaxub256_mask">,
+              Intrinsic<[llvm_v32i8_ty], [llvm_v32i8_ty, llvm_v32i8_ty, 
+                         llvm_v32i8_ty,  llvm_i32_ty], [IntrNoMem]>;
+  def int_x86_avx512_mask_pmaxu_b_512 : GCCBuiltin<"__builtin_ia32_pmaxub512_mask">,
+              Intrinsic<[llvm_v64i8_ty], [llvm_v64i8_ty, llvm_v64i8_ty, 
+                         llvm_v64i8_ty,  llvm_i64_ty], [IntrNoMem]>;
+  def int_x86_avx512_mask_pmaxs_w_128 : GCCBuiltin<"__builtin_ia32_pmaxsw128_mask">,
+              Intrinsic<[llvm_v8i16_ty], [llvm_v8i16_ty, llvm_v8i16_ty, 
+                         llvm_v8i16_ty, llvm_i8_ty], [IntrNoMem]>;
+  def int_x86_avx512_mask_pmaxs_w_256 : GCCBuiltin<"__builtin_ia32_pmaxsw256_mask">,
+              Intrinsic<[llvm_v16i16_ty], [llvm_v16i16_ty, llvm_v16i16_ty, 
+                         llvm_v16i16_ty, llvm_i16_ty], [IntrNoMem]>;
+  def int_x86_avx512_mask_pmaxs_w_512 : GCCBuiltin<"__builtin_ia32_pmaxsw512_mask">,
+              Intrinsic<[llvm_v32i16_ty], [llvm_v32i16_ty, llvm_v32i16_ty, 
+                   llvm_v32i16_ty,  llvm_i32_ty],[IntrNoMem]>;
+  def int_x86_avx512_mask_pmaxu_w_128 : GCCBuiltin<"__builtin_ia32_pmaxuw128_mask">,
+              Intrinsic<[llvm_v8i16_ty], [llvm_v8i16_ty, llvm_v8i16_ty, 
+                         llvm_v8i16_ty, llvm_i8_ty], [IntrNoMem]>;
+  def int_x86_avx512_mask_pmaxu_w_256 : GCCBuiltin<"__builtin_ia32_pmaxuw256_mask">,
+              Intrinsic<[llvm_v16i16_ty], [llvm_v16i16_ty, llvm_v16i16_ty, 
+                         llvm_v16i16_ty, llvm_i16_ty], [IntrNoMem]>;
+  def int_x86_avx512_mask_pmaxu_w_512 : GCCBuiltin<"__builtin_ia32_pmaxuw512_mask">,
+              Intrinsic<[llvm_v32i16_ty], [llvm_v32i16_ty, llvm_v32i16_ty,
+                         llvm_v32i16_ty,  llvm_i32_ty],[IntrNoMem]>;
+  def int_x86_avx512_mask_pmins_b_128 : GCCBuiltin<"__builtin_ia32_pminsb128_mask">,
+              Intrinsic<[llvm_v16i8_ty], [llvm_v16i8_ty, llvm_v16i8_ty, 
+                         llvm_v16i8_ty,llvm_i16_ty], [IntrNoMem]>;
+  def int_x86_avx512_mask_pmins_b_256 : GCCBuiltin<"__builtin_ia32_pminsb256_mask">,
+              Intrinsic<[llvm_v32i8_ty], [llvm_v32i8_ty, llvm_v32i8_ty, 
+                         llvm_v32i8_ty, llvm_i32_ty], [IntrNoMem]>;
+  def int_x86_avx512_mask_pmins_b_512 : GCCBuiltin<"__builtin_ia32_pminsb512_mask">,
+              Intrinsic<[llvm_v64i8_ty], [llvm_v64i8_ty, llvm_v64i8_ty,
+                         llvm_v64i8_ty, llvm_i64_ty], [IntrNoMem]>;
+  def int_x86_avx512_mask_pminu_b_128 : GCCBuiltin<"__builtin_ia32_pminub128_mask">,
+              Intrinsic<[llvm_v16i8_ty], [llvm_v16i8_ty, llvm_v16i8_ty,
+                         llvm_v16i8_ty, llvm_i16_ty], [IntrNoMem]>;
+  def int_x86_avx512_mask_pminu_b_256 : GCCBuiltin<"__builtin_ia32_pminub256_mask">,
+              Intrinsic<[llvm_v32i8_ty], [llvm_v32i8_ty, llvm_v32i8_ty, 
+                         llvm_v32i8_ty, llvm_i32_ty], [IntrNoMem]>;
+  def int_x86_avx512_mask_pminu_b_512 : GCCBuiltin<"__builtin_ia32_pminub512_mask">,
+              Intrinsic<[llvm_v64i8_ty], [llvm_v64i8_ty, llvm_v64i8_ty, 
+                         llvm_v64i8_ty,  llvm_i64_ty], [IntrNoMem]>;
+  def int_x86_avx512_mask_pmins_w_128 : GCCBuiltin<"__builtin_ia32_pminsw128_mask">,
+              Intrinsic<[llvm_v8i16_ty], [llvm_v8i16_ty, llvm_v8i16_ty, 
+                         llvm_v8i16_ty, llvm_i8_ty], [IntrNoMem]>;
+  def int_x86_avx512_mask_pmins_w_256 : GCCBuiltin<"__builtin_ia32_pminsw256_mask">,
+              Intrinsic<[llvm_v16i16_ty], [llvm_v16i16_ty, llvm_v16i16_ty, 
+                         llvm_v16i16_ty, llvm_i16_ty], [IntrNoMem]>;
+  def int_x86_avx512_mask_pmins_w_512 : GCCBuiltin<"__builtin_ia32_pminsw512_mask">,
+              Intrinsic<[llvm_v32i16_ty], [llvm_v32i16_ty, llvm_v32i16_ty,
+                         llvm_v32i16_ty,  llvm_i32_ty],[IntrNoMem]>;
+  def int_x86_avx512_mask_pminu_w_128 : GCCBuiltin<"__builtin_ia32_pminuw128_mask">,
+              Intrinsic<[llvm_v8i16_ty], [llvm_v8i16_ty, llvm_v8i16_ty, 
+                         llvm_v8i16_ty, llvm_i8_ty], [IntrNoMem]>;
+  def int_x86_avx512_mask_pminu_w_256 : GCCBuiltin<"__builtin_ia32_pminuw256_mask">,
+              Intrinsic<[llvm_v16i16_ty], [llvm_v16i16_ty, llvm_v16i16_ty, 
+                         llvm_v16i16_ty, llvm_i16_ty], [IntrNoMem]>;
+  def int_x86_avx512_mask_pminu_w_512 : GCCBuiltin<"__builtin_ia32_pminuw512_mask">,
+              Intrinsic<[llvm_v32i16_ty], [llvm_v32i16_ty, llvm_v32i16_ty, 
+                         llvm_v32i16_ty,  llvm_i32_ty], [IntrNoMem]>;
   def int_x86_avx512_mask_pmaxu_d_512 : GCCBuiltin<"__builtin_ia32_pmaxud512_mask">,
               Intrinsic<[llvm_v16i32_ty], [llvm_v16i32_ty, llvm_v16i32_ty,
                          llvm_v16i32_ty, llvm_i16_ty], [IntrNoMem]>;
+  def int_x86_avx512_mask_pmaxu_d_256 : GCCBuiltin<"__builtin_ia32_pmaxud256_mask">,
+              Intrinsic<[llvm_v8i32_ty], [llvm_v8i32_ty, llvm_v8i32_ty,
+                         llvm_v8i32_ty, llvm_i8_ty], [IntrNoMem]>;
+  def int_x86_avx512_mask_pmaxu_d_128 : GCCBuiltin<"__builtin_ia32_pmaxud128_mask">,
+              Intrinsic<[llvm_v4i32_ty], [llvm_v4i32_ty, llvm_v4i32_ty,
+                         llvm_v4i32_ty, llvm_i8_ty], [IntrNoMem]>;
   def int_x86_avx512_mask_pmaxs_d_512 : GCCBuiltin<"__builtin_ia32_pmaxsd512_mask">,
               Intrinsic<[llvm_v16i32_ty], [llvm_v16i32_ty, llvm_v16i32_ty,
                          llvm_v16i32_ty, llvm_i16_ty], [IntrNoMem]>;
+  def int_x86_avx512_mask_pmaxs_d_256 : GCCBuiltin<"__builtin_ia32_pmaxsd256_mask">,
+              Intrinsic<[llvm_v8i32_ty], [llvm_v8i32_ty, llvm_v8i32_ty,
+                         llvm_v8i32_ty, llvm_i8_ty], [IntrNoMem]>;
+  def int_x86_avx512_mask_pmaxs_d_128 : GCCBuiltin<"__builtin_ia32_pmaxsd128_mask">,
+              Intrinsic<[llvm_v4i32_ty], [llvm_v4i32_ty, llvm_v4i32_ty,
+                         llvm_v4i32_ty, llvm_i8_ty], [IntrNoMem]>;
   def int_x86_avx512_mask_pmaxu_q_512 : GCCBuiltin<"__builtin_ia32_pmaxuq512_mask">,
               Intrinsic<[llvm_v8i64_ty], [llvm_v8i64_ty, llvm_v8i64_ty,
                          llvm_v8i64_ty, llvm_i8_ty], [IntrNoMem]>;
+  def int_x86_avx512_mask_pmaxu_q_256 : GCCBuiltin<"__builtin_ia32_pmaxuq256_mask">,
+              Intrinsic<[llvm_v4i64_ty], [llvm_v4i64_ty, llvm_v4i64_ty,
+                         llvm_v4i64_ty, llvm_i8_ty], [IntrNoMem]>;
+  def int_x86_avx512_mask_pmaxu_q_128 : GCCBuiltin<"__builtin_ia32_pmaxuq128_mask">,
+              Intrinsic<[llvm_v2i64_ty], [llvm_v2i64_ty, llvm_v2i64_ty,
+                         llvm_v2i64_ty, llvm_i8_ty], [IntrNoMem]>;
   def int_x86_avx512_mask_pmaxs_q_512 : GCCBuiltin<"__builtin_ia32_pmaxsq512_mask">,
               Intrinsic<[llvm_v8i64_ty], [llvm_v8i64_ty, llvm_v8i64_ty,
                          llvm_v8i64_ty, llvm_i8_ty], [IntrNoMem]>;
+  def int_x86_avx512_mask_pmaxs_q_256 : GCCBuiltin<"__builtin_ia32_pmaxsq256_mask">,
+              Intrinsic<[llvm_v4i64_ty], [llvm_v4i64_ty, llvm_v4i64_ty,
+                         llvm_v4i64_ty, llvm_i8_ty], [IntrNoMem]>;
+  def int_x86_avx512_mask_pmaxs_q_128 : GCCBuiltin<"__builtin_ia32_pmaxsq128_mask">,
+              Intrinsic<[llvm_v2i64_ty], [llvm_v2i64_ty, llvm_v2i64_ty,
+                         llvm_v2i64_ty, llvm_i8_ty], [IntrNoMem]>;
   def int_x86_avx512_mask_pminu_d_512 : GCCBuiltin<"__builtin_ia32_pminud512_mask">,
               Intrinsic<[llvm_v16i32_ty], [llvm_v16i32_ty, llvm_v16i32_ty,
                          llvm_v16i32_ty, llvm_i16_ty], [IntrNoMem]>;
+  def int_x86_avx512_mask_pminu_d_256 : GCCBuiltin<"__builtin_ia32_pminud256_mask">,
+              Intrinsic<[llvm_v8i32_ty], [llvm_v8i32_ty, llvm_v8i32_ty,
+                         llvm_v8i32_ty, llvm_i8_ty], [IntrNoMem]>;
+  def int_x86_avx512_mask_pminu_d_128 : GCCBuiltin<"__builtin_ia32_pminud128_mask">,
+              Intrinsic<[llvm_v4i32_ty], [llvm_v4i32_ty, llvm_v4i32_ty,
+                         llvm_v4i32_ty, llvm_i8_ty], [IntrNoMem]>;
   def int_x86_avx512_mask_pmins_d_512 : GCCBuiltin<"__builtin_ia32_pminsd512_mask">,
               Intrinsic<[llvm_v16i32_ty], [llvm_v16i32_ty, llvm_v16i32_ty,
                          llvm_v16i32_ty, llvm_i16_ty], [IntrNoMem]>;
+  def int_x86_avx512_mask_pmins_d_256 : GCCBuiltin<"__builtin_ia32_pminsd256_mask">,
+              Intrinsic<[llvm_v8i32_ty], [llvm_v8i32_ty, llvm_v8i32_ty,
+                         llvm_v8i32_ty, llvm_i8_ty], [IntrNoMem]>;
+  def int_x86_avx512_mask_pmins_d_128 : GCCBuiltin<"__builtin_ia32_pminsd128_mask">,
+              Intrinsic<[llvm_v4i32_ty], [llvm_v4i32_ty, llvm_v4i32_ty,
+                         llvm_v4i32_ty, llvm_i8_ty], [IntrNoMem]>;
   def int_x86_avx512_mask_pminu_q_512 : GCCBuiltin<"__builtin_ia32_pminuq512_mask">,
               Intrinsic<[llvm_v8i64_ty], [llvm_v8i64_ty, llvm_v8i64_ty,
                          llvm_v8i64_ty, llvm_i8_ty], [IntrNoMem]>;
+  def int_x86_avx512_mask_pminu_q_256 : GCCBuiltin<"__builtin_ia32_pminuq256_mask">,
+              Intrinsic<[llvm_v4i64_ty], [llvm_v4i64_ty, llvm_v4i64_ty,
+                         llvm_v4i64_ty, llvm_i8_ty], [IntrNoMem]>;
+  def int_x86_avx512_mask_pminu_q_128 : GCCBuiltin<"__builtin_ia32_pminuq128_mask">,
+              Intrinsic<[llvm_v2i64_ty], [llvm_v2i64_ty, llvm_v2i64_ty,
+                         llvm_v2i64_ty, llvm_i8_ty], [IntrNoMem]>;
   def int_x86_avx512_mask_pmins_q_512 : GCCBuiltin<"__builtin_ia32_pminsq512_mask">,
               Intrinsic<[llvm_v8i64_ty], [llvm_v8i64_ty, llvm_v8i64_ty,
                          llvm_v8i64_ty, llvm_i8_ty], [IntrNoMem]>;
+  def int_x86_avx512_mask_pmins_q_256 : GCCBuiltin<"__builtin_ia32_pminsq256_mask">,
+              Intrinsic<[llvm_v4i64_ty], [llvm_v4i64_ty, llvm_v4i64_ty,
+                         llvm_v4i64_ty, llvm_i8_ty], [IntrNoMem]>;
+  def int_x86_avx512_mask_pmins_q_128 : GCCBuiltin<"__builtin_ia32_pminsq128_mask">,
+              Intrinsic<[llvm_v2i64_ty], [llvm_v2i64_ty, llvm_v2i64_ty,
+                         llvm_v2i64_ty, llvm_i8_ty], [IntrNoMem]>;
 }
 
 // Integer shift ops.
@@ -1589,12 +1990,78 @@ let TargetPrefix = "x86" in {  // All intrinsics start with "llvm.x86.".
               Intrinsic<[llvm_v16i16_ty], [llvm_v16i16_ty], [IntrNoMem]>;
   def int_x86_avx2_pabs_d : GCCBuiltin<"__builtin_ia32_pabsd256">,
               Intrinsic<[llvm_v8i32_ty], [llvm_v8i32_ty], [IntrNoMem]>;
-  def int_x86_avx512_mask_pabs_d_512 : GCCBuiltin<"__builtin_ia32_pabsd512_mask">,
-              Intrinsic<[llvm_v16i32_ty], [llvm_v16i32_ty, llvm_v16i32_ty,
-                                           llvm_i16_ty], [IntrNoMem]>;
-  def int_x86_avx512_mask_pabs_q_512 : GCCBuiltin<"__builtin_ia32_pabsq512_mask">,
-              Intrinsic<[llvm_v8i64_ty], [llvm_v8i64_ty, llvm_v8i64_ty,
-                                          llvm_i8_ty], [IntrNoMem]>;
+
+  def int_x86_avx512_mask_pabs_b_128 : 
+       GCCBuiltin<"__builtin_ia32_pabsb128_mask">,
+        Intrinsic<[llvm_v16i8_ty],
+        [llvm_v16i8_ty, llvm_v16i8_ty,  llvm_i16_ty],
+        [IntrNoMem]>;
+
+  def int_x86_avx512_mask_pabs_b_256 : 
+        GCCBuiltin<"__builtin_ia32_pabsb256_mask">,
+          Intrinsic<[llvm_v32i8_ty],
+          [llvm_v32i8_ty, llvm_v32i8_ty,  llvm_i32_ty],
+          [IntrNoMem]>;
+
+  def int_x86_avx512_mask_pabs_b_512 : 
+        GCCBuiltin<"__builtin_ia32_pabsb512_mask">,
+          Intrinsic<[llvm_v64i8_ty],
+          [llvm_v64i8_ty, llvm_v64i8_ty,  llvm_i64_ty],
+          [IntrNoMem]>;
+
+  def int_x86_avx512_mask_pabs_d_128 : 
+        GCCBuiltin<"__builtin_ia32_pabsd128_mask">,
+          Intrinsic<[llvm_v4i32_ty],
+          [llvm_v4i32_ty, llvm_v4i32_ty,  llvm_i8_ty],
+          [IntrNoMem]>;
+
+  def int_x86_avx512_mask_pabs_d_256 : 
+        GCCBuiltin<"__builtin_ia32_pabsd256_mask">,
+          Intrinsic<[llvm_v8i32_ty],
+          [llvm_v8i32_ty, llvm_v8i32_ty,  llvm_i8_ty],
+          [IntrNoMem]>;
+
+  def int_x86_avx512_mask_pabs_d_512 : 
+        GCCBuiltin<"__builtin_ia32_pabsd512_mask">,
+          Intrinsic<[llvm_v16i32_ty],
+          [llvm_v16i32_ty, llvm_v16i32_ty,  llvm_i16_ty],
+          [IntrNoMem]>;
+
+  def int_x86_avx512_mask_pabs_q_128 : 
+        GCCBuiltin<"__builtin_ia32_pabsq128_mask">,
+          Intrinsic<[llvm_v2i64_ty],
+          [llvm_v2i64_ty, llvm_v2i64_ty,  llvm_i8_ty],
+          [IntrNoMem]>;
+
+  def int_x86_avx512_mask_pabs_q_256 : 
+        GCCBuiltin<"__builtin_ia32_pabsq256_mask">,
+          Intrinsic<[llvm_v4i64_ty],
+          [llvm_v4i64_ty, llvm_v4i64_ty,  llvm_i8_ty],
+          [IntrNoMem]>;
+
+  def int_x86_avx512_mask_pabs_q_512 : 
+        GCCBuiltin<"__builtin_ia32_pabsq512_mask">,
+          Intrinsic<[llvm_v8i64_ty],
+          [llvm_v8i64_ty, llvm_v8i64_ty,  llvm_i8_ty],
+          [IntrNoMem]>;
+
+  def int_x86_avx512_mask_pabs_w_128 : 
+        GCCBuiltin<"__builtin_ia32_pabsw128_mask">,
+          Intrinsic<[llvm_v8i16_ty],
+          [llvm_v8i16_ty, llvm_v8i16_ty,  llvm_i8_ty],
+          [IntrNoMem]>;
+
+  def int_x86_avx512_mask_pabs_w_256 : 
+        GCCBuiltin<"__builtin_ia32_pabsw256_mask">,
+          Intrinsic<[llvm_v16i16_ty],
+          [llvm_v16i16_ty, llvm_v16i16_ty,  llvm_i16_ty],
+          [IntrNoMem]>;
+
+  def int_x86_avx512_mask_pabs_w_512 : 
+        GCCBuiltin<"__builtin_ia32_pabsw512_mask">,
+          Intrinsic<[llvm_v32i16_ty],
+          [llvm_v32i16_ty, llvm_v32i16_ty,  llvm_i32_ty],
+          [IntrNoMem]>;
 }
 
 // Horizontal arithmetic ops
@@ -1640,6 +2107,15 @@ let TargetPrefix = "x86" in {  // All intrinsics start with "llvm.x86.".
   def int_x86_avx2_pmul_hr_sw : GCCBuiltin<"__builtin_ia32_pmulhrsw256">,
               Intrinsic<[llvm_v16i16_ty], [llvm_v16i16_ty,
                          llvm_v16i16_ty], [IntrNoMem, Commutative]>;
+  def int_x86_avx512_mask_pmul_hr_sw_128 : GCCBuiltin<"__builtin_ia32_pmulhrsw128_mask">,
+              Intrinsic<[llvm_v8i16_ty], [llvm_v8i16_ty, llvm_v8i16_ty, 
+                     llvm_v8i16_ty, llvm_i8_ty], [IntrNoMem]>;
+  def int_x86_avx512_mask_pmul_hr_sw_256 : GCCBuiltin<"__builtin_ia32_pmulhrsw256_mask">,
+              Intrinsic<[llvm_v16i16_ty], [llvm_v16i16_ty, llvm_v16i16_ty, 
+                     llvm_v16i16_ty, llvm_i16_ty], [IntrNoMem]>;
+  def int_x86_avx512_mask_pmul_hr_sw_512 : GCCBuiltin<"__builtin_ia32_pmulhrsw512_mask">,
+              Intrinsic<[llvm_v32i16_ty], [llvm_v32i16_ty, llvm_v32i16_ty, 
+                     llvm_v32i16_ty, llvm_i32_ty], [IntrNoMem]>;
 }
 
 // Vector sign and zero extend
@@ -1991,36 +2467,7 @@ let TargetPrefix = "x86" in {  // All intrinsics start with "llvm.x86.".
               Intrinsic<[llvm_v4f64_ty],
                         [llvm_v4f64_ty, llvm_v4f64_ty, llvm_v4f64_ty],
                         [IntrNoMem]>;
-  def int_x86_fma_mask_vfmadd_ps_512 : GCCBuiltin<"__builtin_ia32_vfmaddps512_mask">,
-              Intrinsic<[llvm_v16f32_ty],
-                        [llvm_v16f32_ty, llvm_v16f32_ty, llvm_v16f32_ty,
-                         llvm_i16_ty, llvm_i32_ty],
-                        [IntrNoMem]>;
-  def int_x86_fma_mask_vfmadd_ps_256 : GCCBuiltin<"__builtin_ia32_vfmaddps256_mask">,
-              Intrinsic<[llvm_v8f32_ty],
-                        [llvm_v8f32_ty, llvm_v8f32_ty, llvm_v8f32_ty,
-                         llvm_i8_ty],
-                        [IntrNoMem]>;
-  def int_x86_fma_mask_vfmadd_ps_128 : GCCBuiltin<"__builtin_ia32_vfmaddps128_mask">,
-              Intrinsic<[llvm_v4f32_ty],
-                        [llvm_v4f32_ty, llvm_v4f32_ty, llvm_v4f32_ty,
-                         llvm_i8_ty],
-                        [IntrNoMem]>;
-  def int_x86_fma_mask_vfmadd_pd_512 : GCCBuiltin<"__builtin_ia32_vfmaddpd512_mask">,
-              Intrinsic<[llvm_v8f64_ty],
-                        [llvm_v8f64_ty, llvm_v8f64_ty, llvm_v8f64_ty,
-                         llvm_i8_ty, llvm_i32_ty],
-                        [IntrNoMem]>;
-  def int_x86_fma_mask_vfmadd_pd_256 : GCCBuiltin<"__builtin_ia32_vfmaddpd256_mask">,
-              Intrinsic<[llvm_v4f64_ty],
-                        [llvm_v4f64_ty, llvm_v4f64_ty, llvm_v4f64_ty,
-                         llvm_i8_ty],
-                        [IntrNoMem]>;
-  def int_x86_fma_mask_vfmadd_pd_128 : GCCBuiltin<"__builtin_ia32_vfmaddpd128_mask">,
-              Intrinsic<[llvm_v2f64_ty],
-                        [llvm_v2f64_ty, llvm_v2f64_ty, llvm_v2f64_ty,
-                         llvm_i8_ty],
-                        [IntrNoMem]>;
+
   def int_x86_fma_vfmsub_ss : GCCBuiltin<"__builtin_ia32_vfmsubss">,
               Intrinsic<[llvm_v4f32_ty],
                         [llvm_v4f32_ty, llvm_v4f32_ty, llvm_v4f32_ty],
@@ -2045,36 +2492,6 @@ let TargetPrefix = "x86" in {  // All intrinsics start with "llvm.x86.".
               Intrinsic<[llvm_v4f64_ty],
                         [llvm_v4f64_ty, llvm_v4f64_ty, llvm_v4f64_ty],
                         [IntrNoMem]>;
-  def int_x86_fma_mask_vfmsub_ps_512 : GCCBuiltin<"__builtin_ia32_vfmsubps512_mask">,
-              Intrinsic<[llvm_v16f32_ty],
-                        [llvm_v16f32_ty, llvm_v16f32_ty, llvm_v16f32_ty,
-                         llvm_i16_ty, llvm_i32_ty],
-                        [IntrNoMem]>;
-  def int_x86_fma_mask_vfmsub_ps_256 : GCCBuiltin<"__builtin_ia32_vfmsubps256_mask">,
-              Intrinsic<[llvm_v8f32_ty],
-                        [llvm_v8f32_ty, llvm_v8f32_ty, llvm_v8f32_ty,
-                         llvm_i8_ty],
-                        [IntrNoMem]>;
-  def int_x86_fma_mask_vfmsub_ps_128 : GCCBuiltin<"__builtin_ia32_vfmsubps128_mask">,
-              Intrinsic<[llvm_v4f32_ty],
-                        [llvm_v4f32_ty, llvm_v4f32_ty, llvm_v4f32_ty,
-                         llvm_i8_ty],
-                        [IntrNoMem]>;
-  def int_x86_fma_mask_vfmsub_pd_512 : GCCBuiltin<"__builtin_ia32_vfmsubpd512_mask">,
-              Intrinsic<[llvm_v8f64_ty],
-                        [llvm_v8f64_ty, llvm_v8f64_ty, llvm_v8f64_ty,
-                         llvm_i8_ty, llvm_i32_ty],
-                        [IntrNoMem]>;
-  def int_x86_fma_mask_vfmsub_pd_256 : GCCBuiltin<"__builtin_ia32_vfmsubpd256_mask">,
-              Intrinsic<[llvm_v4f64_ty],
-                        [llvm_v4f64_ty, llvm_v4f64_ty, llvm_v4f64_ty,
-                         llvm_i8_ty],
-                        [IntrNoMem]>;
-  def int_x86_fma_mask_vfmsub_pd_128 : GCCBuiltin<"__builtin_ia32_vfmsubpd128_mask">,
-              Intrinsic<[llvm_v2f64_ty],
-                        [llvm_v2f64_ty, llvm_v2f64_ty, llvm_v2f64_ty,
-                         llvm_i8_ty],
-                        [IntrNoMem]>;
   def int_x86_fma_vfnmadd_ss : GCCBuiltin<"__builtin_ia32_vfnmaddss">,
               Intrinsic<[llvm_v4f32_ty],
                         [llvm_v4f32_ty, llvm_v4f32_ty, llvm_v4f32_ty],
@@ -2099,36 +2516,6 @@ let TargetPrefix = "x86" in {  // All intrinsics start with "llvm.x86.".
               Intrinsic<[llvm_v4f64_ty],
                         [llvm_v4f64_ty, llvm_v4f64_ty, llvm_v4f64_ty],
                         [IntrNoMem]>;
-  def int_x86_fma_mask_vfnmadd_ps_512 : GCCBuiltin<"__builtin_ia32_vfnmaddps512_mask">,
-              Intrinsic<[llvm_v16f32_ty],
-                        [llvm_v16f32_ty, llvm_v16f32_ty, llvm_v16f32_ty,
-                         llvm_i16_ty, llvm_i32_ty],
-                        [IntrNoMem]>;
-  def int_x86_fma_mask_vfnmadd_ps_256 : GCCBuiltin<"__builtin_ia32_vfnmaddps256_mask">,
-              Intrinsic<[llvm_v8f32_ty],
-                        [llvm_v8f32_ty, llvm_v8f32_ty, llvm_v8f32_ty,
-                         llvm_i8_ty],
-                        [IntrNoMem]>;
-  def int_x86_fma_mask_vfnmadd_ps_128 : GCCBuiltin<"__builtin_ia32_vfnmaddps128_mask">,
-              Intrinsic<[llvm_v4f32_ty],
-                        [llvm_v4f32_ty, llvm_v4f32_ty, llvm_v4f32_ty,
-                         llvm_i8_ty],
-                        [IntrNoMem]>;
-  def int_x86_fma_mask_vfnmadd_pd_512 : GCCBuiltin<"__builtin_ia32_vfnmaddpd512_mask">,
-              Intrinsic<[llvm_v8f64_ty],
-                        [llvm_v8f64_ty, llvm_v8f64_ty, llvm_v8f64_ty,
-                         llvm_i8_ty, llvm_i32_ty],
-                        [IntrNoMem]>;
-  def int_x86_fma_mask_vfnmadd_pd_256 : GCCBuiltin<"__builtin_ia32_vfnmaddpd256_mask">,
-              Intrinsic<[llvm_v4f64_ty],
-                        [llvm_v4f64_ty, llvm_v4f64_ty, llvm_v4f64_ty,
-                         llvm_i8_ty],
-                        [IntrNoMem]>;
-  def int_x86_fma_mask_vfnmadd_pd_128 : GCCBuiltin<"__builtin_ia32_vfnmaddpd128_mask">,
-              Intrinsic<[llvm_v2f64_ty],
-                        [llvm_v2f64_ty, llvm_v2f64_ty, llvm_v2f64_ty,
-                         llvm_i8_ty],
-                        [IntrNoMem]>;
   def int_x86_fma_vfnmsub_ss : GCCBuiltin<"__builtin_ia32_vfnmsubss">,
               Intrinsic<[llvm_v4f32_ty],
                         [llvm_v4f32_ty, llvm_v4f32_ty, llvm_v4f32_ty],
@@ -2153,36 +2540,6 @@ let TargetPrefix = "x86" in {  // All intrinsics start with "llvm.x86.".
               Intrinsic<[llvm_v4f64_ty],
                         [llvm_v4f64_ty, llvm_v4f64_ty, llvm_v4f64_ty],
                         [IntrNoMem]>;
-  def int_x86_fma_mask_vfnmsub_ps_512 : GCCBuiltin<"__builtin_ia32_vfnmsubps512_mask">,
-              Intrinsic<[llvm_v16f32_ty],
-                        [llvm_v16f32_ty, llvm_v16f32_ty, llvm_v16f32_ty,
-                         llvm_i16_ty, llvm_i32_ty],
-                        [IntrNoMem]>;
-  def int_x86_fma_mask_vfnmsub_ps_256 : GCCBuiltin<"__builtin_ia32_vfnmsubps256_mask">,
-              Intrinsic<[llvm_v8f32_ty],
-                        [llvm_v8f32_ty, llvm_v8f32_ty, llvm_v8f32_ty,
-                         llvm_i8_ty],
-                        [IntrNoMem]>;
-  def int_x86_fma_mask_vfnmsub_ps_128 : GCCBuiltin<"__builtin_ia32_vfnmsubps128_mask">,
-              Intrinsic<[llvm_v4f32_ty],
-                        [llvm_v4f32_ty, llvm_v4f32_ty, llvm_v4f32_ty,
-                         llvm_i8_ty],
-                        [IntrNoMem]>;
-  def int_x86_fma_mask_vfnmsub_pd_512 : GCCBuiltin<"__builtin_ia32_vfnmsubpd512_mask">,
-              Intrinsic<[llvm_v8f64_ty],
-                        [llvm_v8f64_ty, llvm_v8f64_ty, llvm_v8f64_ty,
-                         llvm_i8_ty, llvm_i32_ty],
-                        [IntrNoMem]>;
-  def int_x86_fma_mask_vfnmsub_pd_256 : GCCBuiltin<"__builtin_ia32_vfnmsubpd256_mask">,
-              Intrinsic<[llvm_v4f64_ty],
-                        [llvm_v4f64_ty, llvm_v4f64_ty, llvm_v4f64_ty,
-                         llvm_i8_ty],
-                        [IntrNoMem]>;
-  def int_x86_fma_mask_vfnmsub_pd_128 : GCCBuiltin<"__builtin_ia32_vfnmsubpd128_mask">,
-              Intrinsic<[llvm_v2f64_ty],
-                        [llvm_v2f64_ty, llvm_v2f64_ty, llvm_v2f64_ty,
-                         llvm_i8_ty],
-                        [IntrNoMem]>;
   def int_x86_fma_vfmaddsub_ps : GCCBuiltin<"__builtin_ia32_vfmaddsubps">,
               Intrinsic<[llvm_v4f32_ty],
                         [llvm_v4f32_ty, llvm_v4f32_ty, llvm_v4f32_ty],
@@ -2201,36 +2558,6 @@ let TargetPrefix = "x86" in {  // All intrinsics start with "llvm.x86.".
               Intrinsic<[llvm_v4f64_ty],
                         [llvm_v4f64_ty, llvm_v4f64_ty, llvm_v4f64_ty],
                         [IntrNoMem]>;
-  def int_x86_fma_mask_vfmaddsub_ps_512 : GCCBuiltin<"__builtin_ia32_vfmaddsubps512_mask">,
-              Intrinsic<[llvm_v16f32_ty],
-                        [llvm_v16f32_ty, llvm_v16f32_ty, llvm_v16f32_ty,
-                         llvm_i16_ty, llvm_i32_ty],
-                        [IntrNoMem]>;
-  def int_x86_fma_mask_vfmaddsub_ps_256 : GCCBuiltin<"__builtin_ia32_vfmaddsubps256_mask">,
-              Intrinsic<[llvm_v8f32_ty],
-                        [llvm_v8f32_ty, llvm_v8f32_ty, llvm_v8f32_ty,
-                         llvm_i8_ty],
-                        [IntrNoMem]>;
-  def int_x86_fma_mask_vfmaddsub_ps_128 : GCCBuiltin<"__builtin_ia32_vfmaddsubps128_mask">,
-              Intrinsic<[llvm_v4f32_ty],
-                        [llvm_v4f32_ty, llvm_v4f32_ty, llvm_v4f32_ty,
-                         llvm_i8_ty],
-                        [IntrNoMem]>;
-  def int_x86_fma_mask_vfmaddsub_pd_512 : GCCBuiltin<"__builtin_ia32_vfmaddsubpd512_mask">,
-              Intrinsic<[llvm_v8f64_ty],
-                        [llvm_v8f64_ty, llvm_v8f64_ty, llvm_v8f64_ty,
-                         llvm_i8_ty, llvm_i32_ty],
-                        [IntrNoMem]>;
-  def int_x86_fma_mask_vfmaddsub_pd_256 : GCCBuiltin<"__builtin_ia32_vfmaddsubpd256_mask">,
-              Intrinsic<[llvm_v4f64_ty],
-                        [llvm_v4f64_ty, llvm_v4f64_ty, llvm_v4f64_ty,
-                         llvm_i8_ty],
-                        [IntrNoMem]>;
-  def int_x86_fma_mask_vfmaddsub_pd_128 : GCCBuiltin<"__builtin_ia32_vfmaddsubpd128_mask">,
-              Intrinsic<[llvm_v2f64_ty],
-                        [llvm_v2f64_ty, llvm_v2f64_ty, llvm_v2f64_ty,
-                         llvm_i8_ty],
-                        [IntrNoMem]>;
   def int_x86_fma_vfmsubadd_ps : GCCBuiltin<"__builtin_ia32_vfmsubaddps">,
               Intrinsic<[llvm_v4f32_ty],
                         [llvm_v4f32_ty, llvm_v4f32_ty, llvm_v4f32_ty],
@@ -2249,36 +2576,403 @@ let TargetPrefix = "x86" in {  // All intrinsics start with "llvm.x86.".
               Intrinsic<[llvm_v4f64_ty],
                         [llvm_v4f64_ty, llvm_v4f64_ty, llvm_v4f64_ty],
                         [IntrNoMem]>;
-  def int_x86_fma_mask_vfmsubadd_ps_512 : GCCBuiltin<"__builtin_ia32_vfmsubaddps512_mask">,
-              Intrinsic<[llvm_v16f32_ty],
-                        [llvm_v16f32_ty, llvm_v16f32_ty, llvm_v16f32_ty,
-                         llvm_i16_ty, llvm_i32_ty],
-                        [IntrNoMem]>;
-  def int_x86_fma_mask_vfmsubadd_ps_256 : GCCBuiltin<"__builtin_ia32_vfmsubaddps256_mask">,
-              Intrinsic<[llvm_v8f32_ty],
-                        [llvm_v8f32_ty, llvm_v8f32_ty, llvm_v8f32_ty,
-                         llvm_i8_ty],
-                        [IntrNoMem]>;
-  def int_x86_fma_mask_vfmsubadd_ps_128 : GCCBuiltin<"__builtin_ia32_vfmsubaddps128_mask">,
-              Intrinsic<[llvm_v4f32_ty],
-                        [llvm_v4f32_ty, llvm_v4f32_ty, llvm_v4f32_ty,
-                         llvm_i8_ty],
-                        [IntrNoMem]>;
-  def int_x86_fma_mask_vfmsubadd_pd_512 : GCCBuiltin<"__builtin_ia32_vfmsubaddpd512_mask">,
-              Intrinsic<[llvm_v8f64_ty],
-                        [llvm_v8f64_ty, llvm_v8f64_ty, llvm_v8f64_ty,
-                         llvm_i8_ty, llvm_i32_ty],
-                        [IntrNoMem]>;
-  def int_x86_fma_mask_vfmsubadd_pd_256 : GCCBuiltin<"__builtin_ia32_vfmsubaddpd256_mask">,
-              Intrinsic<[llvm_v4f64_ty],
-                        [llvm_v4f64_ty, llvm_v4f64_ty, llvm_v4f64_ty,
-                         llvm_i8_ty],
-                        [IntrNoMem]>;
-  def int_x86_fma_mask_vfmsubadd_pd_128 : GCCBuiltin<"__builtin_ia32_vfmsubaddpd128_mask">,
-              Intrinsic<[llvm_v2f64_ty],
-                        [llvm_v2f64_ty, llvm_v2f64_ty, llvm_v2f64_ty,
-                         llvm_i8_ty],
-                        [IntrNoMem]>;
+
+  def int_x86_avx512_mask_vfmadd_pd_128 :
+         GCCBuiltin<"__builtin_ia32_vfmaddpd128_mask">,
+          Intrinsic<[llvm_v2f64_ty],
+          [llvm_v2f64_ty, llvm_v2f64_ty, llvm_v2f64_ty,  llvm_i8_ty],
+          [IntrNoMem]>;
+
+  def int_x86_avx512_mask3_vfmadd_pd_128 :
+         GCCBuiltin<"__builtin_ia32_vfmaddpd128_mask3">,
+          Intrinsic<[llvm_v2f64_ty],
+          [llvm_v2f64_ty, llvm_v2f64_ty, llvm_v2f64_ty,  llvm_i8_ty],
+          [IntrNoMem]>;
+
+  def int_x86_avx512_maskz_vfmadd_pd_128 :
+         GCCBuiltin<"__builtin_ia32_vfmaddpd128_maskz">,
+          Intrinsic<[llvm_v2f64_ty],
+          [llvm_v2f64_ty, llvm_v2f64_ty, llvm_v2f64_ty,  llvm_i8_ty],
+          [IntrNoMem]>;
+
+  def int_x86_avx512_mask_vfmadd_pd_256 :
+         GCCBuiltin<"__builtin_ia32_vfmaddpd256_mask">,
+          Intrinsic<[llvm_v4f64_ty],
+          [llvm_v4f64_ty, llvm_v4f64_ty, llvm_v4f64_ty,  llvm_i8_ty],
+          [IntrNoMem]>;
+
+  def int_x86_avx512_mask3_vfmadd_pd_256 :
+         GCCBuiltin<"__builtin_ia32_vfmaddpd256_mask3">,
+          Intrinsic<[llvm_v4f64_ty],
+          [llvm_v4f64_ty, llvm_v4f64_ty, llvm_v4f64_ty,  llvm_i8_ty],
+          [IntrNoMem]>;
+
+  def int_x86_avx512_maskz_vfmadd_pd_256 :
+         GCCBuiltin<"__builtin_ia32_vfmaddpd256_maskz">,
+          Intrinsic<[llvm_v4f64_ty],
+          [llvm_v4f64_ty, llvm_v4f64_ty, llvm_v4f64_ty,  llvm_i8_ty],
+          [IntrNoMem]>;
+
+  def int_x86_avx512_mask_vfmadd_pd_512 :
+         GCCBuiltin<"__builtin_ia32_vfmaddpd512_mask">,
+          Intrinsic<[llvm_v8f64_ty],
+          [llvm_v8f64_ty, llvm_v8f64_ty, llvm_v8f64_ty,  llvm_i8_ty,
+          llvm_i32_ty], [IntrNoMem]>;
+
+  def int_x86_avx512_mask3_vfmadd_pd_512 :
+         GCCBuiltin<"__builtin_ia32_vfmaddpd512_mask3">,
+          Intrinsic<[llvm_v8f64_ty],
+          [llvm_v8f64_ty, llvm_v8f64_ty, llvm_v8f64_ty,  llvm_i8_ty,
+          llvm_i32_ty], [IntrNoMem]>;
+
+  def int_x86_avx512_maskz_vfmadd_pd_512 :
+         GCCBuiltin<"__builtin_ia32_vfmaddpd512_maskz">,
+          Intrinsic<[llvm_v8f64_ty],
+          [llvm_v8f64_ty, llvm_v8f64_ty, llvm_v8f64_ty,  llvm_i8_ty,
+          llvm_i32_ty], [IntrNoMem]>;
+
+  def int_x86_avx512_mask_vfmadd_ps_128 :
+         GCCBuiltin<"__builtin_ia32_vfmaddps128_mask">,
+          Intrinsic<[llvm_v4f32_ty],
+          [llvm_v4f32_ty, llvm_v4f32_ty, llvm_v4f32_ty,  llvm_i8_ty],
+          [IntrNoMem]>;
+
+  def int_x86_avx512_mask3_vfmadd_ps_128 :
+         GCCBuiltin<"__builtin_ia32_vfmaddps128_mask3">,
+          Intrinsic<[llvm_v4f32_ty],
+          [llvm_v4f32_ty, llvm_v4f32_ty, llvm_v4f32_ty,  llvm_i8_ty],
+          [IntrNoMem]>;
+
+  def int_x86_avx512_maskz_vfmadd_ps_128 :
+         GCCBuiltin<"__builtin_ia32_vfmaddps128_maskz">,
+          Intrinsic<[llvm_v4f32_ty],
+          [llvm_v4f32_ty, llvm_v4f32_ty, llvm_v4f32_ty,  llvm_i8_ty],
+          [IntrNoMem]>;
+
+  def int_x86_avx512_mask_vfmadd_ps_256 :
+         GCCBuiltin<"__builtin_ia32_vfmaddps256_mask">,
+          Intrinsic<[llvm_v8f32_ty],
+          [llvm_v8f32_ty, llvm_v8f32_ty, llvm_v8f32_ty,  llvm_i8_ty],
+          [IntrNoMem]>;
+
+  def int_x86_avx512_mask3_vfmadd_ps_256 :
+         GCCBuiltin<"__builtin_ia32_vfmaddps256_mask3">,
+          Intrinsic<[llvm_v8f32_ty],
+          [llvm_v8f32_ty, llvm_v8f32_ty, llvm_v8f32_ty,  llvm_i8_ty],
+          [IntrNoMem]>;
+
+  def int_x86_avx512_maskz_vfmadd_ps_256 :
+         GCCBuiltin<"__builtin_ia32_vfmaddps256_maskz">,
+          Intrinsic<[llvm_v8f32_ty],
+          [llvm_v8f32_ty, llvm_v8f32_ty, llvm_v8f32_ty,  llvm_i8_ty],
+          [IntrNoMem]>;
+
+  def int_x86_avx512_mask_vfmadd_ps_512 :
+         GCCBuiltin<"__builtin_ia32_vfmaddps512_mask">,
+          Intrinsic<[llvm_v16f32_ty],
+          [llvm_v16f32_ty, llvm_v16f32_ty, llvm_v16f32_ty,  llvm_i16_ty,
+          llvm_i32_ty], [IntrNoMem]>;
+
+  def int_x86_avx512_mask3_vfmadd_ps_512 :
+         GCCBuiltin<"__builtin_ia32_vfmaddps512_mask3">,
+          Intrinsic<[llvm_v16f32_ty],
+          [llvm_v16f32_ty, llvm_v16f32_ty, llvm_v16f32_ty,  llvm_i16_ty,
+          llvm_i32_ty], [IntrNoMem]>;
+
+  def int_x86_avx512_maskz_vfmadd_ps_512 :
+         GCCBuiltin<"__builtin_ia32_vfmaddps512_maskz">,
+          Intrinsic<[llvm_v16f32_ty],
+          [llvm_v16f32_ty, llvm_v16f32_ty, llvm_v16f32_ty,  llvm_i16_ty,
+          llvm_i32_ty], [IntrNoMem]>;
+
+  def int_x86_avx512_mask_vfmaddsub_pd_128 :
+         GCCBuiltin<"__builtin_ia32_vfmaddsubpd128_mask">,
+          Intrinsic<[llvm_v2f64_ty],
+          [llvm_v2f64_ty, llvm_v2f64_ty, llvm_v2f64_ty,  llvm_i8_ty],
+          [IntrNoMem]>;
+
+  def int_x86_avx512_mask3_vfmaddsub_pd_128 :
+         GCCBuiltin<"__builtin_ia32_vfmaddsubpd128_mask3">,
+          Intrinsic<[llvm_v2f64_ty],
+          [llvm_v2f64_ty, llvm_v2f64_ty, llvm_v2f64_ty,  llvm_i8_ty],
+          [IntrNoMem]>;
+
+  def int_x86_avx512_maskz_vfmaddsub_pd_128 :
+         GCCBuiltin<"__builtin_ia32_vfmaddsubpd128_maskz">,
+          Intrinsic<[llvm_v2f64_ty],
+          [llvm_v2f64_ty, llvm_v2f64_ty, llvm_v2f64_ty,  llvm_i8_ty],
+          [IntrNoMem]>;
+
+  def int_x86_avx512_mask_vfmaddsub_pd_256 :
+         GCCBuiltin<"__builtin_ia32_vfmaddsubpd256_mask">,
+          Intrinsic<[llvm_v4f64_ty],
+          [llvm_v4f64_ty, llvm_v4f64_ty, llvm_v4f64_ty,  llvm_i8_ty],
+          [IntrNoMem]>;
+
+  def int_x86_avx512_mask3_vfmaddsub_pd_256 :
+         GCCBuiltin<"__builtin_ia32_vfmaddsubpd256_mask3">,
+          Intrinsic<[llvm_v4f64_ty],
+          [llvm_v4f64_ty, llvm_v4f64_ty, llvm_v4f64_ty,  llvm_i8_ty],
+          [IntrNoMem]>;
+
+  def int_x86_avx512_maskz_vfmaddsub_pd_256 :
+         GCCBuiltin<"__builtin_ia32_vfmaddsubpd256_maskz">,
+          Intrinsic<[llvm_v4f64_ty],
+          [llvm_v4f64_ty, llvm_v4f64_ty, llvm_v4f64_ty,  llvm_i8_ty],
+          [IntrNoMem]>;
+
+  def int_x86_avx512_mask_vfmaddsub_pd_512 :
+         GCCBuiltin<"__builtin_ia32_vfmaddsubpd512_mask">,
+          Intrinsic<[llvm_v8f64_ty],
+          [llvm_v8f64_ty, llvm_v8f64_ty, llvm_v8f64_ty,  llvm_i8_ty,
+          llvm_i32_ty], [IntrNoMem]>;
+
+  def int_x86_avx512_mask3_vfmaddsub_pd_512 :
+         GCCBuiltin<"__builtin_ia32_vfmaddsubpd512_mask3">,
+          Intrinsic<[llvm_v8f64_ty],
+          [llvm_v8f64_ty, llvm_v8f64_ty, llvm_v8f64_ty,  llvm_i8_ty,
+          llvm_i32_ty], [IntrNoMem]>;
+
+  def int_x86_avx512_maskz_vfmaddsub_pd_512 :
+         GCCBuiltin<"__builtin_ia32_vfmaddsubpd512_maskz">,
+          Intrinsic<[llvm_v8f64_ty],
+          [llvm_v8f64_ty, llvm_v8f64_ty, llvm_v8f64_ty,  llvm_i8_ty,
+          llvm_i32_ty], [IntrNoMem]>;
+
+  def int_x86_avx512_mask_vfmaddsub_ps_128 :
+         GCCBuiltin<"__builtin_ia32_vfmaddsubps128_mask">,
+          Intrinsic<[llvm_v4f32_ty],
+          [llvm_v4f32_ty, llvm_v4f32_ty, llvm_v4f32_ty,  llvm_i8_ty],
+          [IntrNoMem]>;
+
+  def int_x86_avx512_mask3_vfmaddsub_ps_128 :
+         GCCBuiltin<"__builtin_ia32_vfmaddsubps128_mask3">,
+          Intrinsic<[llvm_v4f32_ty],
+          [llvm_v4f32_ty, llvm_v4f32_ty, llvm_v4f32_ty,  llvm_i8_ty],
+          [IntrNoMem]>;
+
+  def int_x86_avx512_maskz_vfmaddsub_ps_128 :
+         GCCBuiltin<"__builtin_ia32_vfmaddsubps128_maskz">,
+          Intrinsic<[llvm_v4f32_ty],
+          [llvm_v4f32_ty, llvm_v4f32_ty, llvm_v4f32_ty,  llvm_i8_ty],
+          [IntrNoMem]>;
+
+  def int_x86_avx512_mask_vfmaddsub_ps_256 :
+         GCCBuiltin<"__builtin_ia32_vfmaddsubps256_mask">,
+          Intrinsic<[llvm_v8f32_ty],
+          [llvm_v8f32_ty, llvm_v8f32_ty, llvm_v8f32_ty,  llvm_i8_ty],
+          [IntrNoMem]>;
+
+  def int_x86_avx512_mask3_vfmaddsub_ps_256 :
+         GCCBuiltin<"__builtin_ia32_vfmaddsubps256_mask3">,
+          Intrinsic<[llvm_v8f32_ty],
+          [llvm_v8f32_ty, llvm_v8f32_ty, llvm_v8f32_ty,  llvm_i8_ty],
+          [IntrNoMem]>;
+
+  def int_x86_avx512_maskz_vfmaddsub_ps_256 :
+         GCCBuiltin<"__builtin_ia32_vfmaddsubps256_maskz">,
+          Intrinsic<[llvm_v8f32_ty],
+          [llvm_v8f32_ty, llvm_v8f32_ty, llvm_v8f32_ty,  llvm_i8_ty],
+          [IntrNoMem]>;
+
+  def int_x86_avx512_mask_vfmaddsub_ps_512 :
+         GCCBuiltin<"__builtin_ia32_vfmaddsubps512_mask">,
+          Intrinsic<[llvm_v16f32_ty],
+          [llvm_v16f32_ty, llvm_v16f32_ty, llvm_v16f32_ty,  llvm_i16_ty,
+          llvm_i32_ty], [IntrNoMem]>;
+
+  def int_x86_avx512_mask3_vfmaddsub_ps_512 :
+         GCCBuiltin<"__builtin_ia32_vfmaddsubps512_mask3">,
+          Intrinsic<[llvm_v16f32_ty],
+          [llvm_v16f32_ty, llvm_v16f32_ty, llvm_v16f32_ty,  llvm_i16_ty,
+          llvm_i32_ty], [IntrNoMem]>;
+
+  def int_x86_avx512_maskz_vfmaddsub_ps_512 :
+         GCCBuiltin<"__builtin_ia32_vfmaddsubps512_maskz">,
+          Intrinsic<[llvm_v16f32_ty],
+          [llvm_v16f32_ty, llvm_v16f32_ty, llvm_v16f32_ty,  llvm_i16_ty,
+          llvm_i32_ty], [IntrNoMem]>;
+
+  def int_x86_avx512_mask3_vfmsub_pd_128 :
+         GCCBuiltin<"__builtin_ia32_vfmsubpd128_mask3">,
+          Intrinsic<[llvm_v2f64_ty],
+          [llvm_v2f64_ty, llvm_v2f64_ty, llvm_v2f64_ty,  llvm_i8_ty],
+          [IntrNoMem]>;
+
+  def int_x86_avx512_mask3_vfmsub_pd_256 :
+         GCCBuiltin<"__builtin_ia32_vfmsubpd256_mask3">,
+          Intrinsic<[llvm_v4f64_ty],
+          [llvm_v4f64_ty, llvm_v4f64_ty, llvm_v4f64_ty,  llvm_i8_ty],
+          [IntrNoMem]>;
+
+  def int_x86_avx512_mask3_vfmsub_pd_512 :
+         GCCBuiltin<"__builtin_ia32_vfmsubpd512_mask3">,
+          Intrinsic<[llvm_v8f64_ty],
+          [llvm_v8f64_ty, llvm_v8f64_ty, llvm_v8f64_ty,  llvm_i8_ty,
+          llvm_i32_ty], [IntrNoMem]>;
+
+  def int_x86_avx512_mask3_vfmsub_ps_128 :
+         GCCBuiltin<"__builtin_ia32_vfmsubps128_mask3">,
+          Intrinsic<[llvm_v4f32_ty],
+          [llvm_v4f32_ty, llvm_v4f32_ty, llvm_v4f32_ty,  llvm_i8_ty],
+          [IntrNoMem]>;
+
+  def int_x86_avx512_mask3_vfmsub_ps_256 :
+         GCCBuiltin<"__builtin_ia32_vfmsubps256_mask3">,
+          Intrinsic<[llvm_v8f32_ty],
+          [llvm_v8f32_ty, llvm_v8f32_ty, llvm_v8f32_ty,  llvm_i8_ty],
+          [IntrNoMem]>;
+
+  def int_x86_avx512_mask3_vfmsub_ps_512 :
+         GCCBuiltin<"__builtin_ia32_vfmsubps512_mask3">,
+          Intrinsic<[llvm_v16f32_ty],
+          [llvm_v16f32_ty, llvm_v16f32_ty, llvm_v16f32_ty,  llvm_i16_ty,
+          llvm_i32_ty], [IntrNoMem]>;
+
+  def int_x86_avx512_mask3_vfmsubadd_pd_128 :
+         GCCBuiltin<"__builtin_ia32_vfmsubaddpd128_mask3">,
+          Intrinsic<[llvm_v2f64_ty],
+          [llvm_v2f64_ty, llvm_v2f64_ty, llvm_v2f64_ty,  llvm_i8_ty],
+          [IntrNoMem]>;
+
+  def int_x86_avx512_mask3_vfmsubadd_pd_256 :
+         GCCBuiltin<"__builtin_ia32_vfmsubaddpd256_mask3">,
+          Intrinsic<[llvm_v4f64_ty],
+          [llvm_v4f64_ty, llvm_v4f64_ty, llvm_v4f64_ty,  llvm_i8_ty],
+          [IntrNoMem]>;
+
+  def int_x86_avx512_mask3_vfmsubadd_pd_512 :
+         GCCBuiltin<"__builtin_ia32_vfmsubaddpd512_mask3">,
+          Intrinsic<[llvm_v8f64_ty],
+          [llvm_v8f64_ty, llvm_v8f64_ty, llvm_v8f64_ty,  llvm_i8_ty,
+          llvm_i32_ty], [IntrNoMem]>;
+
+  def int_x86_avx512_mask3_vfmsubadd_ps_128 :
+         GCCBuiltin<"__builtin_ia32_vfmsubaddps128_mask3">,
+          Intrinsic<[llvm_v4f32_ty],
+          [llvm_v4f32_ty, llvm_v4f32_ty, llvm_v4f32_ty,  llvm_i8_ty],
+          [IntrNoMem]>;
+
+  def int_x86_avx512_mask3_vfmsubadd_ps_256 :
+         GCCBuiltin<"__builtin_ia32_vfmsubaddps256_mask3">,
+          Intrinsic<[llvm_v8f32_ty],
+          [llvm_v8f32_ty, llvm_v8f32_ty, llvm_v8f32_ty,  llvm_i8_ty],
+          [IntrNoMem]>;
+
+  def int_x86_avx512_mask3_vfmsubadd_ps_512 :
+         GCCBuiltin<"__builtin_ia32_vfmsubaddps512_mask3">,
+          Intrinsic<[llvm_v16f32_ty],
+          [llvm_v16f32_ty, llvm_v16f32_ty, llvm_v16f32_ty,  llvm_i16_ty,
+          llvm_i32_ty], [IntrNoMem]>;
+
+  def int_x86_avx512_mask_vfnmadd_pd_128 :
+         GCCBuiltin<"__builtin_ia32_vfnmaddpd128_mask">,
+          Intrinsic<[llvm_v2f64_ty],
+          [llvm_v2f64_ty, llvm_v2f64_ty, llvm_v2f64_ty,  llvm_i8_ty],
+          [IntrNoMem]>;
+
+  def int_x86_avx512_mask_vfnmadd_pd_256 :
+         GCCBuiltin<"__builtin_ia32_vfnmaddpd256_mask">,
+          Intrinsic<[llvm_v4f64_ty],
+          [llvm_v4f64_ty, llvm_v4f64_ty, llvm_v4f64_ty,  llvm_i8_ty],
+          [IntrNoMem]>;
+
+  def int_x86_avx512_mask_vfnmadd_pd_512 :
+         GCCBuiltin<"__builtin_ia32_vfnmaddpd512_mask">,
+          Intrinsic<[llvm_v8f64_ty],
+          [llvm_v8f64_ty, llvm_v8f64_ty, llvm_v8f64_ty,  llvm_i8_ty,
+          llvm_i32_ty], [IntrNoMem]>;
+
+  def int_x86_avx512_mask_vfnmadd_ps_128 :
+         GCCBuiltin<"__builtin_ia32_vfnmaddps128_mask">,
+          Intrinsic<[llvm_v4f32_ty],
+          [llvm_v4f32_ty, llvm_v4f32_ty, llvm_v4f32_ty,  llvm_i8_ty],
+          [IntrNoMem]>;
+
+  def int_x86_avx512_mask_vfnmadd_ps_256 :
+         GCCBuiltin<"__builtin_ia32_vfnmaddps256_mask">,
+          Intrinsic<[llvm_v8f32_ty],
+          [llvm_v8f32_ty, llvm_v8f32_ty, llvm_v8f32_ty,  llvm_i8_ty],
+          [IntrNoMem]>;
+
+  def int_x86_avx512_mask_vfnmadd_ps_512 :
+         GCCBuiltin<"__builtin_ia32_vfnmaddps512_mask">,
+          Intrinsic<[llvm_v16f32_ty],
+          [llvm_v16f32_ty, llvm_v16f32_ty, llvm_v16f32_ty,  llvm_i16_ty,
+          llvm_i32_ty], [IntrNoMem]>;
+
+  def int_x86_avx512_mask_vfnmsub_pd_128 :
+         GCCBuiltin<"__builtin_ia32_vfnmsubpd128_mask">,
+          Intrinsic<[llvm_v2f64_ty],
+          [llvm_v2f64_ty, llvm_v2f64_ty, llvm_v2f64_ty,  llvm_i8_ty],
+          [IntrNoMem]>;
+
+  def int_x86_avx512_mask3_vfnmsub_pd_128 :
+         GCCBuiltin<"__builtin_ia32_vfnmsubpd128_mask3">,
+          Intrinsic<[llvm_v2f64_ty],
+          [llvm_v2f64_ty, llvm_v2f64_ty, llvm_v2f64_ty,  llvm_i8_ty],
+          [IntrNoMem]>;
+
+  def int_x86_avx512_mask_vfnmsub_pd_256 :
+         GCCBuiltin<"__builtin_ia32_vfnmsubpd256_mask">,
+          Intrinsic<[llvm_v4f64_ty],
+          [llvm_v4f64_ty, llvm_v4f64_ty, llvm_v4f64_ty,  llvm_i8_ty],
+          [IntrNoMem]>;
+
+  def int_x86_avx512_mask3_vfnmsub_pd_256 :
+         GCCBuiltin<"__builtin_ia32_vfnmsubpd256_mask3">,
+          Intrinsic<[llvm_v4f64_ty],
+          [llvm_v4f64_ty, llvm_v4f64_ty, llvm_v4f64_ty,  llvm_i8_ty],
+          [IntrNoMem]>;
+
+  def int_x86_avx512_mask_vfnmsub_pd_512 :
+         GCCBuiltin<"__builtin_ia32_vfnmsubpd512_mask">,
+          Intrinsic<[llvm_v8f64_ty],
+          [llvm_v8f64_ty, llvm_v8f64_ty, llvm_v8f64_ty,  llvm_i8_ty,
+          llvm_i32_ty], [IntrNoMem]>;
+
+  def int_x86_avx512_mask3_vfnmsub_pd_512 :
+         GCCBuiltin<"__builtin_ia32_vfnmsubpd512_mask3">,
+          Intrinsic<[llvm_v8f64_ty],
+          [llvm_v8f64_ty, llvm_v8f64_ty, llvm_v8f64_ty,  llvm_i8_ty,
+          llvm_i32_ty], [IntrNoMem]>;
+
+  def int_x86_avx512_mask_vfnmsub_ps_128 :
+         GCCBuiltin<"__builtin_ia32_vfnmsubps128_mask">,
+          Intrinsic<[llvm_v4f32_ty],
+          [llvm_v4f32_ty, llvm_v4f32_ty, llvm_v4f32_ty,  llvm_i8_ty],
+          [IntrNoMem]>;
+
+  def int_x86_avx512_mask3_vfnmsub_ps_128 :
+         GCCBuiltin<"__builtin_ia32_vfnmsubps128_mask3">,
+          Intrinsic<[llvm_v4f32_ty],
+          [llvm_v4f32_ty, llvm_v4f32_ty, llvm_v4f32_ty,  llvm_i8_ty],
+          [IntrNoMem]>;
+
+  def int_x86_avx512_mask_vfnmsub_ps_256 :
+         GCCBuiltin<"__builtin_ia32_vfnmsubps256_mask">,
+          Intrinsic<[llvm_v8f32_ty],
+          [llvm_v8f32_ty, llvm_v8f32_ty, llvm_v8f32_ty,  llvm_i8_ty],
+          [IntrNoMem]>;
+
+  def int_x86_avx512_mask3_vfnmsub_ps_256 :
+         GCCBuiltin<"__builtin_ia32_vfnmsubps256_mask3">,
+          Intrinsic<[llvm_v8f32_ty],
+          [llvm_v8f32_ty, llvm_v8f32_ty, llvm_v8f32_ty,  llvm_i8_ty],
+          [IntrNoMem]>;
+
+  def int_x86_avx512_mask_vfnmsub_ps_512 :
+         GCCBuiltin<"__builtin_ia32_vfnmsubps512_mask">,
+          Intrinsic<[llvm_v16f32_ty],
+          [llvm_v16f32_ty, llvm_v16f32_ty, llvm_v16f32_ty,  llvm_i16_ty,
+          llvm_i32_ty], [IntrNoMem]>;
+
+  def int_x86_avx512_mask3_vfnmsub_ps_512 :
+         GCCBuiltin<"__builtin_ia32_vfnmsubps512_mask3">,
+          Intrinsic<[llvm_v16f32_ty],
+          [llvm_v16f32_ty, llvm_v16f32_ty, llvm_v16f32_ty,  llvm_i16_ty,
+          llvm_i32_ty], [IntrNoMem]>;
+
 }
 
 //===----------------------------------------------------------------------===//
@@ -2837,6 +3531,19 @@ let TargetPrefix = "x86" in {  // All intrinsics start with "llvm.x86.".
               Intrinsic<[], [llvm_i64_ty]>;
 }
 
+//===----------------------------------------------------------------------===//
+// FXSR
+let TargetPrefix = "x86" in {  // All intrinsics start with "llvm.x86.".
+  def int_x86_fxrstor : GCCBuiltin<"__builtin_ia32_fxrstor">,
+              Intrinsic<[], [llvm_ptr_ty], []>;
+  def int_x86_fxrstor64 : GCCBuiltin<"__builtin_ia32_fxrstor64">,
+              Intrinsic<[], [llvm_ptr_ty], []>;
+  def int_x86_fxsave : GCCBuiltin<"__builtin_ia32_fxsave">,
+              Intrinsic<[], [llvm_ptr_ty], []>;
+  def int_x86_fxsave64 : GCCBuiltin<"__builtin_ia32_fxsave64">,
+              Intrinsic<[], [llvm_ptr_ty], []>;
+}
+
 //===----------------------------------------------------------------------===//
 // Half float conversion
 
@@ -2968,12 +3675,12 @@ let TargetPrefix = "x86" in {  // All intrinsics start with "llvm.x86.".
               Intrinsic<[llvm_i32_ty], [llvm_v4f32_ty], [IntrNoMem]>;
   def int_x86_avx512_cvttss2usi64 : GCCBuiltin<"__builtin_ia32_cvttss2usi64">,
               Intrinsic<[llvm_i64_ty], [llvm_v4f32_ty], [IntrNoMem]>;
-  def int_x86_avx512_cvtusi2ss : GCCBuiltin<"__builtin_ia32_cvtusi2ss">,
+  def int_x86_avx512_cvtusi2ss : GCCBuiltin<"__builtin_ia32_cvtusi2ss32">,
               Intrinsic<[llvm_v4f32_ty], [llvm_v4f32_ty,
-                         llvm_i32_ty], [IntrNoMem]>;
-  def int_x86_avx512_cvtusi642ss : GCCBuiltin<"__builtin_ia32_cvtusi642ss">,
+                         llvm_i32_ty, llvm_i32_ty], [IntrNoMem]>;
+  def int_x86_avx512_cvtusi642ss : GCCBuiltin<"__builtin_ia32_cvtusi2ss64">,
               Intrinsic<[llvm_v4f32_ty], [llvm_v4f32_ty,
-                         llvm_i64_ty], [IntrNoMem]>;
+                         llvm_i64_ty, llvm_i32_ty], [IntrNoMem]>;
 
   def int_x86_avx512_cvtsd2usi : GCCBuiltin<"__builtin_ia32_cvtsd2usi">,
               Intrinsic<[llvm_i32_ty], [llvm_v2f64_ty], [IntrNoMem]>;
@@ -2983,61 +3690,810 @@ let TargetPrefix = "x86" in {  // All intrinsics start with "llvm.x86.".
               Intrinsic<[llvm_i32_ty], [llvm_v2f64_ty], [IntrNoMem]>;
   def int_x86_avx512_cvttsd2usi64 : GCCBuiltin<"__builtin_ia32_cvttsd2usi64">,
               Intrinsic<[llvm_i64_ty], [llvm_v2f64_ty], [IntrNoMem]>;
-  def int_x86_avx512_cvtusi2sd : GCCBuiltin<"__builtin_ia32_cvtusi2sd">,
+  def int_x86_avx512_cvtusi2sd : GCCBuiltin<"__builtin_ia32_cvtusi2sd32">,
               Intrinsic<[llvm_v2f64_ty], [llvm_v2f64_ty,
                          llvm_i32_ty], [IntrNoMem]>;
-  def int_x86_avx512_cvtusi642sd : GCCBuiltin<"__builtin_ia32_cvtusi642sd">,
+  def int_x86_avx512_cvtusi642sd : GCCBuiltin<"__builtin_ia32_cvtusi2sd64">,
               Intrinsic<[llvm_v2f64_ty], [llvm_v2f64_ty,
-                         llvm_i64_ty], [IntrNoMem]>;
+                         llvm_i64_ty, llvm_i32_ty], [IntrNoMem]>;
+
+  def int_x86_avx512_cvtsi2ss32 : GCCBuiltin<"__builtin_ia32_cvtsi2ss32">,
+                Intrinsic<[llvm_v4f32_ty], [llvm_v4f32_ty,
+                           llvm_i32_ty, llvm_i32_ty], [IntrNoMem]>;
+  def int_x86_avx512_cvtsi2ss64 : GCCBuiltin<"__builtin_ia32_cvtsi2ss64">,
+                Intrinsic<[llvm_v4f32_ty], [llvm_v4f32_ty,
+                           llvm_i64_ty, llvm_i32_ty], [IntrNoMem]>;
+  def int_x86_avx512_cvtsi2sd32 : GCCBuiltin<"__builtin_ia32_cvtsi2sd32">,
+                Intrinsic<[llvm_v2f64_ty], [llvm_v2f64_ty,
+                           llvm_i32_ty, llvm_i32_ty], [IntrNoMem]>;
+  def int_x86_avx512_cvtsi2sd64 : GCCBuiltin<"__builtin_ia32_cvtsi2sd64">,
+                Intrinsic<[llvm_v2f64_ty], [llvm_v2f64_ty,
+                           llvm_i64_ty, llvm_i32_ty], [IntrNoMem]>;
+}
+
+// Pack ops.
+let TargetPrefix = "x86" in {  // All intrinsics start with "llvm.x86.".
+  def int_x86_avx512_mask_packsswb_128 : GCCBuiltin<"__builtin_ia32_packsswb128_mask">,
+              Intrinsic<[llvm_v16i8_ty], [llvm_v8i16_ty, llvm_v8i16_ty,
+                         llvm_v16i8_ty, llvm_i16_ty], [IntrNoMem]>;
+  def int_x86_avx512_mask_packsswb_256 : GCCBuiltin<"__builtin_ia32_packsswb256_mask">,
+              Intrinsic<[llvm_v32i8_ty], [llvm_v16i16_ty,llvm_v16i16_ty,
+                         llvm_v32i8_ty, llvm_i32_ty], [IntrNoMem]>;
+  def int_x86_avx512_mask_packsswb_512 : GCCBuiltin<"__builtin_ia32_packsswb512_mask">,
+              Intrinsic<[llvm_v64i8_ty], [llvm_v32i16_ty,llvm_v32i16_ty, 
+                         llvm_v64i8_ty, llvm_i64_ty], [IntrNoMem]>;
+  def int_x86_avx512_mask_packssdw_128 : GCCBuiltin<"__builtin_ia32_packssdw128_mask">,
+              Intrinsic<[llvm_v8i16_ty], [llvm_v4i32_ty, llvm_v4i32_ty,
+                         llvm_v8i16_ty, llvm_i8_ty], [IntrNoMem]>;
+  def int_x86_avx512_mask_packssdw_256 : GCCBuiltin<"__builtin_ia32_packssdw256_mask">,
+              Intrinsic<[llvm_v16i16_ty], [llvm_v8i32_ty, llvm_v8i32_ty,
+                         llvm_v16i16_ty, llvm_i16_ty], [IntrNoMem]>;
+  def int_x86_avx512_mask_packssdw_512 : GCCBuiltin<"__builtin_ia32_packssdw512_mask">,
+              Intrinsic<[llvm_v32i16_ty], [llvm_v16i32_ty, llvm_v16i32_ty,
+                         llvm_v32i16_ty, llvm_i32_ty], [IntrNoMem]>;
+  def int_x86_avx512_mask_packuswb_128 : GCCBuiltin<"__builtin_ia32_packuswb128_mask">,
+              Intrinsic<[llvm_v16i8_ty], [llvm_v8i16_ty, llvm_v8i16_ty,
+                         llvm_v16i8_ty, llvm_i16_ty], [IntrNoMem]>;
+  def int_x86_avx512_mask_packuswb_256 : GCCBuiltin<"__builtin_ia32_packuswb256_mask">,
+              Intrinsic<[llvm_v32i8_ty], [llvm_v16i16_ty,llvm_v16i16_ty,
+                         llvm_v32i8_ty, llvm_i32_ty], [IntrNoMem]>;
+  def int_x86_avx512_mask_packuswb_512 : GCCBuiltin<"__builtin_ia32_packuswb512_mask">,
+              Intrinsic<[llvm_v64i8_ty], [llvm_v32i16_ty,llvm_v32i16_ty, 
+                         llvm_v64i8_ty, llvm_i64_ty], [IntrNoMem]>;
+  def int_x86_avx512_mask_packusdw_128 : GCCBuiltin<"__builtin_ia32_packusdw128_mask">,
+              Intrinsic<[llvm_v8i16_ty], [llvm_v4i32_ty, llvm_v4i32_ty,
+                         llvm_v8i16_ty, llvm_i8_ty], [IntrNoMem]>;
+  def int_x86_avx512_mask_packusdw_256 : GCCBuiltin<"__builtin_ia32_packusdw256_mask">,
+              Intrinsic<[llvm_v16i16_ty], [llvm_v8i32_ty, llvm_v8i32_ty,
+                         llvm_v16i16_ty, llvm_i16_ty], [IntrNoMem]>;
+  def int_x86_avx512_mask_packusdw_512 : GCCBuiltin<"__builtin_ia32_packusdw512_mask">,
+              Intrinsic<[llvm_v32i16_ty], [llvm_v16i32_ty, llvm_v16i32_ty,
+                         llvm_v32i16_ty, llvm_i32_ty], [IntrNoMem]>;
+}
+
+// Unpack ops.
+let TargetPrefix = "x86" in {  // All intrinsics start with "llvm.x86.".
+  def int_x86_avx512_mask_unpckh_pd_128 :
+         GCCBuiltin<"__builtin_ia32_unpckhpd128_mask">,
+          Intrinsic<[llvm_v2f64_ty],
+          [llvm_v2f64_ty, llvm_v2f64_ty, llvm_v2f64_ty,  llvm_i8_ty],
+          [IntrNoMem]>;
+
+  def int_x86_avx512_mask_unpckh_pd_256 :
+         GCCBuiltin<"__builtin_ia32_unpckhpd256_mask">,
+          Intrinsic<[llvm_v4f64_ty],
+          [llvm_v4f64_ty, llvm_v4f64_ty, llvm_v4f64_ty,  llvm_i8_ty],
+          [IntrNoMem]>;
+
+  def int_x86_avx512_mask_unpckh_pd_512 :
+         GCCBuiltin<"__builtin_ia32_unpckhpd512_mask">,
+          Intrinsic<[llvm_v8f64_ty],
+          [llvm_v8f64_ty, llvm_v8f64_ty, llvm_v8f64_ty,  llvm_i8_ty],
+          [IntrNoMem]>;
+
+  def int_x86_avx512_mask_unpckh_ps_128 :
+         GCCBuiltin<"__builtin_ia32_unpckhps128_mask">,
+          Intrinsic<[llvm_v4f32_ty],
+          [llvm_v4f32_ty, llvm_v4f32_ty, llvm_v4f32_ty,  llvm_i8_ty],
+          [IntrNoMem]>;
+
+  def int_x86_avx512_mask_unpckh_ps_256 :
+         GCCBuiltin<"__builtin_ia32_unpckhps256_mask">,
+          Intrinsic<[llvm_v8f32_ty],
+          [llvm_v8f32_ty, llvm_v8f32_ty, llvm_v8f32_ty,  llvm_i8_ty],
+          [IntrNoMem]>;
+
+  def int_x86_avx512_mask_unpckh_ps_512 :
+         GCCBuiltin<"__builtin_ia32_unpckhps512_mask">,
+          Intrinsic<[llvm_v16f32_ty],
+          [llvm_v16f32_ty, llvm_v16f32_ty, llvm_v16f32_ty,  llvm_i16_ty],
+          [IntrNoMem]>;
+
+  def int_x86_avx512_mask_unpckl_pd_128 :
+         GCCBuiltin<"__builtin_ia32_unpcklpd128_mask">,
+          Intrinsic<[llvm_v2f64_ty],
+          [llvm_v2f64_ty, llvm_v2f64_ty, llvm_v2f64_ty,  llvm_i8_ty],
+          [IntrNoMem]>;
+
+  def int_x86_avx512_mask_unpckl_pd_256 :
+         GCCBuiltin<"__builtin_ia32_unpcklpd256_mask">,
+          Intrinsic<[llvm_v4f64_ty],
+          [llvm_v4f64_ty, llvm_v4f64_ty, llvm_v4f64_ty,  llvm_i8_ty],
+          [IntrNoMem]>;
+
+  def int_x86_avx512_mask_unpckl_pd_512 :
+         GCCBuiltin<"__builtin_ia32_unpcklpd512_mask">,
+          Intrinsic<[llvm_v8f64_ty],
+          [llvm_v8f64_ty, llvm_v8f64_ty, llvm_v8f64_ty,  llvm_i8_ty],
+          [IntrNoMem]>;
+
+  def int_x86_avx512_mask_unpckl_ps_128 :
+         GCCBuiltin<"__builtin_ia32_unpcklps128_mask">,
+          Intrinsic<[llvm_v4f32_ty],
+          [llvm_v4f32_ty, llvm_v4f32_ty, llvm_v4f32_ty,  llvm_i8_ty],
+          [IntrNoMem]>;
+
+  def int_x86_avx512_mask_unpckl_ps_256 :
+         GCCBuiltin<"__builtin_ia32_unpcklps256_mask">,
+          Intrinsic<[llvm_v8f32_ty],
+          [llvm_v8f32_ty, llvm_v8f32_ty, llvm_v8f32_ty,  llvm_i8_ty],
+          [IntrNoMem]>;
+
+  def int_x86_avx512_mask_unpckl_ps_512 :
+         GCCBuiltin<"__builtin_ia32_unpcklps512_mask">,
+          Intrinsic<[llvm_v16f32_ty],
+          [llvm_v16f32_ty, llvm_v16f32_ty, llvm_v16f32_ty,  llvm_i16_ty],
+          [IntrNoMem]>;
+
+  def int_x86_avx512_mask_punpckhb_w_128 :
+         GCCBuiltin<"__builtin_ia32_punpckhbw128_mask">,
+          Intrinsic<[llvm_v16i8_ty],
+          [llvm_v16i8_ty, llvm_v16i8_ty, llvm_v16i8_ty,  llvm_i16_ty],
+          [IntrNoMem]>;
+
+  def int_x86_avx512_mask_punpckhb_w_256 :
+         GCCBuiltin<"__builtin_ia32_punpckhbw256_mask">,
+          Intrinsic<[llvm_v32i8_ty],
+          [llvm_v32i8_ty, llvm_v32i8_ty, llvm_v32i8_ty,  llvm_i32_ty],
+          [IntrNoMem]>;
+
+  def int_x86_avx512_mask_punpckhb_w_512 :
+         GCCBuiltin<"__builtin_ia32_punpckhbw512_mask">,
+          Intrinsic<[llvm_v64i8_ty],
+          [llvm_v64i8_ty, llvm_v64i8_ty, llvm_v64i8_ty,  llvm_i64_ty],
+          [IntrNoMem]>;
+
+  def int_x86_avx512_mask_punpckhd_q_128 :
+         GCCBuiltin<"__builtin_ia32_punpckhdq128_mask">,
+          Intrinsic<[llvm_v4i32_ty],
+          [llvm_v4i32_ty, llvm_v4i32_ty, llvm_v4i32_ty,  llvm_i8_ty],
+          [IntrNoMem]>;
+
+  def int_x86_avx512_mask_punpckhd_q_256 :
+         GCCBuiltin<"__builtin_ia32_punpckhdq256_mask">,
+          Intrinsic<[llvm_v8i32_ty],
+          [llvm_v8i32_ty, llvm_v8i32_ty, llvm_v8i32_ty,  llvm_i8_ty],
+          [IntrNoMem]>;
+
+  def int_x86_avx512_mask_punpckhd_q_512 :
+         GCCBuiltin<"__builtin_ia32_punpckhdq512_mask">,
+          Intrinsic<[llvm_v16i32_ty],
+          [llvm_v16i32_ty, llvm_v16i32_ty, llvm_v16i32_ty,  llvm_i16_ty],
+          [IntrNoMem]>;
+
+  def int_x86_avx512_mask_punpckhqd_q_128 :
+         GCCBuiltin<"__builtin_ia32_punpckhqdq128_mask">,
+          Intrinsic<[llvm_v2i64_ty],
+          [llvm_v2i64_ty, llvm_v2i64_ty, llvm_v2i64_ty,  llvm_i8_ty],
+          [IntrNoMem]>;
+
+  def int_x86_avx512_mask_punpckhqd_q_256 :
+         GCCBuiltin<"__builtin_ia32_punpckhqdq256_mask">,
+          Intrinsic<[llvm_v4i64_ty],
+          [llvm_v4i64_ty, llvm_v4i64_ty, llvm_v4i64_ty,  llvm_i8_ty],
+          [IntrNoMem]>;
+
+  def int_x86_avx512_mask_punpckhqd_q_512 :
+         GCCBuiltin<"__builtin_ia32_punpckhqdq512_mask">,
+          Intrinsic<[llvm_v8i64_ty],
+          [llvm_v8i64_ty, llvm_v8i64_ty, llvm_v8i64_ty,  llvm_i8_ty],
+          [IntrNoMem]>;
+
+  def int_x86_avx512_mask_punpckhw_d_128 :
+         GCCBuiltin<"__builtin_ia32_punpckhwd128_mask">,
+          Intrinsic<[llvm_v8i16_ty],
+          [llvm_v8i16_ty, llvm_v8i16_ty, llvm_v8i16_ty,  llvm_i8_ty],
+          [IntrNoMem]>;
+
+  def int_x86_avx512_mask_punpckhw_d_256 :
+         GCCBuiltin<"__builtin_ia32_punpckhwd256_mask">,
+          Intrinsic<[llvm_v16i16_ty],
+          [llvm_v16i16_ty, llvm_v16i16_ty, llvm_v16i16_ty,  llvm_i16_ty],
+          [IntrNoMem]>;
+
+  def int_x86_avx512_mask_punpckhw_d_512 :
+         GCCBuiltin<"__builtin_ia32_punpckhwd512_mask">,
+          Intrinsic<[llvm_v32i16_ty],
+          [llvm_v32i16_ty, llvm_v32i16_ty, llvm_v32i16_ty,  llvm_i32_ty],
+          [IntrNoMem]>;
+
+  def int_x86_avx512_mask_punpcklb_w_128 :
+         GCCBuiltin<"__builtin_ia32_punpcklbw128_mask">,
+          Intrinsic<[llvm_v16i8_ty],
+          [llvm_v16i8_ty, llvm_v16i8_ty, llvm_v16i8_ty,  llvm_i16_ty],
+          [IntrNoMem]>;
+
+  def int_x86_avx512_mask_punpcklb_w_256 :
+         GCCBuiltin<"__builtin_ia32_punpcklbw256_mask">,
+          Intrinsic<[llvm_v32i8_ty],
+          [llvm_v32i8_ty, llvm_v32i8_ty, llvm_v32i8_ty,  llvm_i32_ty],
+          [IntrNoMem]>;
+
+  def int_x86_avx512_mask_punpcklb_w_512 :
+         GCCBuiltin<"__builtin_ia32_punpcklbw512_mask">,
+          Intrinsic<[llvm_v64i8_ty],
+          [llvm_v64i8_ty, llvm_v64i8_ty, llvm_v64i8_ty,  llvm_i64_ty],
+          [IntrNoMem]>;
+
+  def int_x86_avx512_mask_punpckld_q_128 :
+         GCCBuiltin<"__builtin_ia32_punpckldq128_mask">,
+          Intrinsic<[llvm_v4i32_ty],
+          [llvm_v4i32_ty, llvm_v4i32_ty, llvm_v4i32_ty,  llvm_i8_ty],
+          [IntrNoMem]>;
+
+  def int_x86_avx512_mask_punpckld_q_256 :
+         GCCBuiltin<"__builtin_ia32_punpckldq256_mask">,
+          Intrinsic<[llvm_v8i32_ty],
+          [llvm_v8i32_ty, llvm_v8i32_ty, llvm_v8i32_ty,  llvm_i8_ty],
+          [IntrNoMem]>;
+
+  def int_x86_avx512_mask_punpckld_q_512 :
+         GCCBuiltin<"__builtin_ia32_punpckldq512_mask">,
+          Intrinsic<[llvm_v16i32_ty],
+          [llvm_v16i32_ty, llvm_v16i32_ty, llvm_v16i32_ty,  llvm_i16_ty],
+          [IntrNoMem]>;
+
+  def int_x86_avx512_mask_punpcklqd_q_128 :
+         GCCBuiltin<"__builtin_ia32_punpcklqdq128_mask">,
+          Intrinsic<[llvm_v2i64_ty],
+          [llvm_v2i64_ty, llvm_v2i64_ty, llvm_v2i64_ty,  llvm_i8_ty],
+          [IntrNoMem]>;
+
+  def int_x86_avx512_mask_punpcklqd_q_256 :
+         GCCBuiltin<"__builtin_ia32_punpcklqdq256_mask">,
+          Intrinsic<[llvm_v4i64_ty],
+          [llvm_v4i64_ty, llvm_v4i64_ty, llvm_v4i64_ty,  llvm_i8_ty],
+          [IntrNoMem]>;
+
+  def int_x86_avx512_mask_punpcklqd_q_512 :
+         GCCBuiltin<"__builtin_ia32_punpcklqdq512_mask">,
+          Intrinsic<[llvm_v8i64_ty],
+          [llvm_v8i64_ty, llvm_v8i64_ty, llvm_v8i64_ty,  llvm_i8_ty],
+          [IntrNoMem]>;
+
+  def int_x86_avx512_mask_punpcklw_d_128 :
+         GCCBuiltin<"__builtin_ia32_punpcklwd128_mask">,
+          Intrinsic<[llvm_v8i16_ty],
+          [llvm_v8i16_ty, llvm_v8i16_ty, llvm_v8i16_ty,  llvm_i8_ty],
+          [IntrNoMem]>;
+
+  def int_x86_avx512_mask_punpcklw_d_256 :
+         GCCBuiltin<"__builtin_ia32_punpcklwd256_mask">,
+          Intrinsic<[llvm_v16i16_ty],
+          [llvm_v16i16_ty, llvm_v16i16_ty, llvm_v16i16_ty,  llvm_i16_ty],
+          [IntrNoMem]>;
+
+  def int_x86_avx512_mask_punpcklw_d_512 :
+         GCCBuiltin<"__builtin_ia32_punpcklwd512_mask">,
+          Intrinsic<[llvm_v32i16_ty],
+          [llvm_v32i16_ty, llvm_v32i16_ty, llvm_v32i16_ty,  llvm_i32_ty],
+          [IntrNoMem]>;
 }
 
 // Vector convert
 let TargetPrefix = "x86" in {  // All intrinsics start with "llvm.x86.".
-  def int_x86_avx512_mask_cvttps2dq_512: GCCBuiltin<"__builtin_ia32_cvttps2dq512_mask">,
-        Intrinsic<[llvm_v16i32_ty], [llvm_v16f32_ty, llvm_v16i32_ty,
-                                     llvm_i16_ty, llvm_i32_ty], [IntrNoMem]>;
-  def int_x86_avx512_mask_cvttps2udq_512: GCCBuiltin<"__builtin_ia32_cvttps2udq512_mask">,
-        Intrinsic<[llvm_v16i32_ty], [llvm_v16f32_ty, llvm_v16i32_ty,
-                                     llvm_i16_ty, llvm_i32_ty], [IntrNoMem]>;
-  def int_x86_avx512_mask_cvttpd2dq_512: GCCBuiltin<"__builtin_ia32_cvttpd2dq512_mask">,
-        Intrinsic<[llvm_v8i32_ty], [llvm_v8f64_ty, llvm_v8i32_ty,
-                                    llvm_i8_ty, llvm_i32_ty], [IntrNoMem]>;
-  def int_x86_avx512_mask_cvttpd2udq_512: GCCBuiltin<"__builtin_ia32_cvttpd2udq512_mask">,
-        Intrinsic<[llvm_v8i32_ty], [llvm_v8f64_ty, llvm_v8i32_ty,
-                                    llvm_i8_ty, llvm_i32_ty], [IntrNoMem]>;
-  def int_x86_avx512_mask_rndscale_ps_512: GCCBuiltin<"__builtin_ia32_rndscaleps_mask">,
+  def int_x86_avx512_mask_cvtdq2pd_128 : 
+        GCCBuiltin<"__builtin_ia32_cvtdq2pd128_mask">,
+          Intrinsic<[llvm_v2f64_ty],
+          [llvm_v4i32_ty, llvm_v2f64_ty,  llvm_i8_ty],
+          [IntrNoMem]>;
+
+  def int_x86_avx512_mask_cvtdq2pd_256 : 
+        GCCBuiltin<"__builtin_ia32_cvtdq2pd256_mask">,
+          Intrinsic<[llvm_v4f64_ty],
+          [llvm_v4i32_ty, llvm_v4f64_ty,  llvm_i8_ty],
+          [IntrNoMem]>;
+
+  def int_x86_avx512_mask_cvtdq2pd_512 : 
+        GCCBuiltin<"__builtin_ia32_cvtdq2pd512_mask">,
+          Intrinsic<[llvm_v8f64_ty],
+          [llvm_v8i32_ty, llvm_v8f64_ty,  llvm_i8_ty],
+          [IntrNoMem]>;
+
+  def int_x86_avx512_mask_cvtdq2ps_128 : 
+        GCCBuiltin<"__builtin_ia32_cvtdq2ps128_mask">,
+          Intrinsic<[llvm_v4f32_ty],
+          [llvm_v4i32_ty, llvm_v4f32_ty,  llvm_i8_ty],
+          [IntrNoMem]>;
+
+  def int_x86_avx512_mask_cvtdq2ps_256 : 
+        GCCBuiltin<"__builtin_ia32_cvtdq2ps256_mask">,
+          Intrinsic<[llvm_v8f32_ty],
+          [llvm_v8i32_ty, llvm_v8f32_ty,  llvm_i8_ty],
+          [IntrNoMem]>;
+
+  def int_x86_avx512_mask_cvtdq2ps_512 : 
+        GCCBuiltin<"__builtin_ia32_cvtdq2ps512_mask">,
+          Intrinsic<[llvm_v16f32_ty],
+          [llvm_v16i32_ty, llvm_v16f32_ty,  llvm_i16_ty,  llvm_i32_ty],
+          [IntrNoMem]>;
+
+  def int_x86_avx512_mask_cvtpd2dq_128 : 
+        GCCBuiltin<"__builtin_ia32_cvtpd2dq128_mask">,
+          Intrinsic<[llvm_v4i32_ty],
+          [llvm_v2f64_ty, llvm_v4i32_ty,  llvm_i8_ty],
+          [IntrNoMem]>;
+
+  def int_x86_avx512_mask_cvtpd2dq_256 : 
+        GCCBuiltin<"__builtin_ia32_cvtpd2dq256_mask">,
+          Intrinsic<[llvm_v4i32_ty],
+          [llvm_v4f64_ty, llvm_v4i32_ty,  llvm_i8_ty],
+          [IntrNoMem]>;
+
+  def int_x86_avx512_mask_cvtpd2dq_512 : 
+        GCCBuiltin<"__builtin_ia32_cvtpd2dq512_mask">,
+          Intrinsic<[llvm_v8i32_ty],
+          [llvm_v8f64_ty, llvm_v8i32_ty,  llvm_i8_ty,  llvm_i32_ty],
+          [IntrNoMem]>;
+
+  def int_x86_avx512_mask_cvtpd2ps_256 : 
+        GCCBuiltin<"__builtin_ia32_cvtpd2ps256_mask">,
+          Intrinsic<[llvm_v4f32_ty],
+          [llvm_v4f64_ty, llvm_v4f32_ty,  llvm_i8_ty],
+          [IntrNoMem]>;
+
+  def int_x86_avx512_mask_cvtpd2ps_512 : 
+        GCCBuiltin<"__builtin_ia32_cvtpd2ps512_mask">,
+          Intrinsic<[llvm_v8f32_ty],
+          [llvm_v8f64_ty, llvm_v8f32_ty,  llvm_i8_ty,  llvm_i32_ty],
+          [IntrNoMem]>;
+
+  def int_x86_avx512_mask_cvtpd2ps : 
+        GCCBuiltin<"__builtin_ia32_cvtpd2ps_mask">,
+          Intrinsic<[llvm_v4f32_ty],
+          [llvm_v2f64_ty, llvm_v4f32_ty,  llvm_i8_ty],
+          [IntrNoMem]>;
+
+  def int_x86_avx512_mask_cvtpd2qq_128 : 
+        GCCBuiltin<"__builtin_ia32_cvtpd2qq128_mask">,
+          Intrinsic<[llvm_v2i64_ty],
+          [llvm_v2f64_ty, llvm_v2i64_ty,  llvm_i8_ty],
+          [IntrNoMem]>;
+
+  def int_x86_avx512_mask_cvtpd2qq_256 : 
+        GCCBuiltin<"__builtin_ia32_cvtpd2qq256_mask">,
+          Intrinsic<[llvm_v4i64_ty],
+          [llvm_v4f64_ty, llvm_v4i64_ty,  llvm_i8_ty],
+          [IntrNoMem]>;
+
+  def int_x86_avx512_mask_cvtpd2qq_512 : 
+        GCCBuiltin<"__builtin_ia32_cvtpd2qq512_mask">,
+          Intrinsic<[llvm_v8i64_ty],
+          [llvm_v8f64_ty, llvm_v8i64_ty,  llvm_i8_ty,  llvm_i32_ty],
+          [IntrNoMem]>;
+
+  def int_x86_avx512_mask_cvtpd2udq_128 : 
+        GCCBuiltin<"__builtin_ia32_cvtpd2udq128_mask">,
+          Intrinsic<[llvm_v4i32_ty],
+          [llvm_v2f64_ty, llvm_v4i32_ty,  llvm_i8_ty],
+          [IntrNoMem]>;
+
+  def int_x86_avx512_mask_cvtpd2udq_256 : 
+        GCCBuiltin<"__builtin_ia32_cvtpd2udq256_mask">,
+          Intrinsic<[llvm_v4i32_ty],
+          [llvm_v4f64_ty, llvm_v4i32_ty,  llvm_i8_ty],
+          [IntrNoMem]>;
+
+  def int_x86_avx512_mask_cvtpd2udq_512 : 
+        GCCBuiltin<"__builtin_ia32_cvtpd2udq512_mask">,
+          Intrinsic<[llvm_v8i32_ty],
+          [llvm_v8f64_ty, llvm_v8i32_ty,  llvm_i8_ty,  llvm_i32_ty],
+          [IntrNoMem]>;
+
+  def int_x86_avx512_mask_cvtpd2uqq_128 : 
+        GCCBuiltin<"__builtin_ia32_cvtpd2uqq128_mask">,
+          Intrinsic<[llvm_v2i64_ty],
+          [llvm_v2f64_ty, llvm_v2i64_ty,  llvm_i8_ty],
+          [IntrNoMem]>;
+
+  def int_x86_avx512_mask_cvtpd2uqq_256 : 
+        GCCBuiltin<"__builtin_ia32_cvtpd2uqq256_mask">,
+          Intrinsic<[llvm_v4i64_ty],
+          [llvm_v4f64_ty, llvm_v4i64_ty,  llvm_i8_ty],
+          [IntrNoMem]>;
+
+  def int_x86_avx512_mask_cvtpd2uqq_512 : 
+        GCCBuiltin<"__builtin_ia32_cvtpd2uqq512_mask">,
+          Intrinsic<[llvm_v8i64_ty],
+          [llvm_v8f64_ty, llvm_v8i64_ty,  llvm_i8_ty,  llvm_i32_ty],
+          [IntrNoMem]>;
+
+  def int_x86_avx512_mask_cvtps2dq_128 : 
+        GCCBuiltin<"__builtin_ia32_cvtps2dq128_mask">,
+          Intrinsic<[llvm_v4i32_ty],
+          [llvm_v4f32_ty, llvm_v4i32_ty,  llvm_i8_ty],
+          [IntrNoMem]>;
+
+  def int_x86_avx512_mask_cvtps2dq_256 : 
+        GCCBuiltin<"__builtin_ia32_cvtps2dq256_mask">,
+          Intrinsic<[llvm_v8i32_ty],
+          [llvm_v8f32_ty, llvm_v8i32_ty,  llvm_i8_ty],
+          [IntrNoMem]>;
+
+  def int_x86_avx512_mask_cvtps2dq_512 : 
+        GCCBuiltin<"__builtin_ia32_cvtps2dq512_mask">,
+          Intrinsic<[llvm_v16i32_ty],
+          [llvm_v16f32_ty, llvm_v16i32_ty,  llvm_i16_ty,  llvm_i32_ty],
+          [IntrNoMem]>;
+
+  def int_x86_avx512_mask_cvtps2pd_128 : 
+        GCCBuiltin<"__builtin_ia32_cvtps2pd128_mask">,
+          Intrinsic<[llvm_v2f64_ty],
+          [llvm_v4f32_ty, llvm_v2f64_ty,  llvm_i8_ty],
+          [IntrNoMem]>;
+
+  def int_x86_avx512_mask_cvtps2pd_256 : 
+        GCCBuiltin<"__builtin_ia32_cvtps2pd256_mask">,
+          Intrinsic<[llvm_v4f64_ty],
+          [llvm_v4f32_ty, llvm_v4f64_ty,  llvm_i8_ty],
+          [IntrNoMem]>;
+
+  def int_x86_avx512_mask_cvtps2pd_512 : 
+        GCCBuiltin<"__builtin_ia32_cvtps2pd512_mask">,
+          Intrinsic<[llvm_v8f64_ty],
+          [llvm_v8f32_ty, llvm_v8f64_ty,  llvm_i8_ty,  llvm_i32_ty],
+          [IntrNoMem]>;
+
+  def int_x86_avx512_mask_cvtps2qq_128 : 
+        GCCBuiltin<"__builtin_ia32_cvtps2qq128_mask">,
+          Intrinsic<[llvm_v2i64_ty],
+          [llvm_v4f32_ty, llvm_v2i64_ty,  llvm_i8_ty],
+          [IntrNoMem]>;
+
+  def int_x86_avx512_mask_cvtps2qq_256 : 
+        GCCBuiltin<"__builtin_ia32_cvtps2qq256_mask">,
+          Intrinsic<[llvm_v4i64_ty],
+          [llvm_v4f32_ty, llvm_v4i64_ty,  llvm_i8_ty],
+          [IntrNoMem]>;
+
+  def int_x86_avx512_mask_cvtps2qq_512 : 
+        GCCBuiltin<"__builtin_ia32_cvtps2qq512_mask">,
+          Intrinsic<[llvm_v8i64_ty],
+          [llvm_v8f32_ty, llvm_v8i64_ty,  llvm_i8_ty,  llvm_i32_ty],
+          [IntrNoMem]>;
+
+  def int_x86_avx512_mask_cvtps2udq_128 : 
+        GCCBuiltin<"__builtin_ia32_cvtps2udq128_mask">,
+          Intrinsic<[llvm_v4i32_ty],
+          [llvm_v4f32_ty, llvm_v4i32_ty,  llvm_i8_ty],
+          [IntrNoMem]>;
+
+  def int_x86_avx512_mask_cvtps2udq_256 : 
+        GCCBuiltin<"__builtin_ia32_cvtps2udq256_mask">,
+          Intrinsic<[llvm_v8i32_ty],
+          [llvm_v8f32_ty, llvm_v8i32_ty,  llvm_i8_ty],
+          [IntrNoMem]>;
+
+  def int_x86_avx512_mask_cvtps2udq_512 : 
+        GCCBuiltin<"__builtin_ia32_cvtps2udq512_mask">,
+          Intrinsic<[llvm_v16i32_ty],
+          [llvm_v16f32_ty, llvm_v16i32_ty,  llvm_i16_ty,  llvm_i32_ty],
+          [IntrNoMem]>;
+
+  def int_x86_avx512_mask_cvtps2uqq_128 : 
+        GCCBuiltin<"__builtin_ia32_cvtps2uqq128_mask">,
+          Intrinsic<[llvm_v2i64_ty],
+          [llvm_v4f32_ty, llvm_v2i64_ty,  llvm_i8_ty],
+          [IntrNoMem]>;
+
+  def int_x86_avx512_mask_cvtps2uqq_256 : 
+        GCCBuiltin<"__builtin_ia32_cvtps2uqq256_mask">,
+          Intrinsic<[llvm_v4i64_ty],
+          [llvm_v4f32_ty, llvm_v4i64_ty,  llvm_i8_ty],
+          [IntrNoMem]>;
+
+  def int_x86_avx512_mask_cvtps2uqq_512 : 
+        GCCBuiltin<"__builtin_ia32_cvtps2uqq512_mask">,
+          Intrinsic<[llvm_v8i64_ty],
+          [llvm_v8f32_ty, llvm_v8i64_ty,  llvm_i8_ty,  llvm_i32_ty],
+          [IntrNoMem]>;
+
+  def int_x86_avx512_mask_cvtqq2pd_128 : 
+        GCCBuiltin<"__builtin_ia32_cvtqq2pd128_mask">,
+          Intrinsic<[llvm_v2f64_ty],
+          [llvm_v2i64_ty, llvm_v2f64_ty,  llvm_i8_ty],
+          [IntrNoMem]>;
+
+  def int_x86_avx512_mask_cvtqq2pd_256 : 
+        GCCBuiltin<"__builtin_ia32_cvtqq2pd256_mask">,
+          Intrinsic<[llvm_v4f64_ty],
+          [llvm_v4i64_ty, llvm_v4f64_ty,  llvm_i8_ty],
+          [IntrNoMem]>;
+
+  def int_x86_avx512_mask_cvtqq2pd_512 : 
+        GCCBuiltin<"__builtin_ia32_cvtqq2pd512_mask">,
+          Intrinsic<[llvm_v8f64_ty],
+          [llvm_v8i64_ty, llvm_v8f64_ty,  llvm_i8_ty,  llvm_i32_ty],
+          [IntrNoMem]>;
+
+  def int_x86_avx512_mask_cvtqq2ps_128 : 
+        GCCBuiltin<"__builtin_ia32_cvtqq2ps128_mask">,
+          Intrinsic<[llvm_v4f32_ty],
+          [llvm_v2i64_ty, llvm_v4f32_ty,  llvm_i8_ty],
+          [IntrNoMem]>;
+
+  def int_x86_avx512_mask_cvtqq2ps_256 : 
+        GCCBuiltin<"__builtin_ia32_cvtqq2ps256_mask">,
+          Intrinsic<[llvm_v4f32_ty],
+          [llvm_v4i64_ty, llvm_v4f32_ty,  llvm_i8_ty],
+          [IntrNoMem]>;
+
+  def int_x86_avx512_mask_cvtqq2ps_512 : 
+        GCCBuiltin<"__builtin_ia32_cvtqq2ps512_mask">,
+          Intrinsic<[llvm_v8f32_ty],
+          [llvm_v8i64_ty, llvm_v8f32_ty,  llvm_i8_ty,  llvm_i32_ty],
+          [IntrNoMem]>;
+
+  def int_x86_avx512_mask_cvttpd2dq_128 : 
+        GCCBuiltin<"__builtin_ia32_cvttpd2dq128_mask">,
+          Intrinsic<[llvm_v4i32_ty],
+          [llvm_v2f64_ty, llvm_v4i32_ty,  llvm_i8_ty],
+          [IntrNoMem]>;
+
+  def int_x86_avx512_mask_cvttpd2dq_256 : 
+        GCCBuiltin<"__builtin_ia32_cvttpd2dq256_mask">,
+          Intrinsic<[llvm_v4i32_ty],
+          [llvm_v4f64_ty, llvm_v4i32_ty,  llvm_i8_ty],
+          [IntrNoMem]>;
+
+  def int_x86_avx512_mask_cvttpd2dq_512 : 
+        GCCBuiltin<"__builtin_ia32_cvttpd2dq512_mask">,
+          Intrinsic<[llvm_v8i32_ty],
+          [llvm_v8f64_ty, llvm_v8i32_ty,  llvm_i8_ty,  llvm_i32_ty],
+          [IntrNoMem]>;
+
+  def int_x86_avx512_mask_cvttpd2qq_128 : 
+        GCCBuiltin<"__builtin_ia32_cvttpd2qq128_mask">,
+          Intrinsic<[llvm_v2i64_ty],
+          [llvm_v2f64_ty, llvm_v2i64_ty,  llvm_i8_ty],
+          [IntrNoMem]>;
+
+  def int_x86_avx512_mask_cvttpd2qq_256 : 
+        GCCBuiltin<"__builtin_ia32_cvttpd2qq256_mask">,
+          Intrinsic<[llvm_v4i64_ty],
+          [llvm_v4f64_ty, llvm_v4i64_ty,  llvm_i8_ty],
+          [IntrNoMem]>;
+
+  def int_x86_avx512_mask_cvttpd2qq_512 : 
+        GCCBuiltin<"__builtin_ia32_cvttpd2qq512_mask">,
+          Intrinsic<[llvm_v8i64_ty],
+          [llvm_v8f64_ty, llvm_v8i64_ty,  llvm_i8_ty,  llvm_i32_ty],
+          [IntrNoMem]>;
+
+  def int_x86_avx512_mask_cvttpd2udq_128 : 
+        GCCBuiltin<"__builtin_ia32_cvttpd2udq128_mask">,
+          Intrinsic<[llvm_v4i32_ty],
+          [llvm_v2f64_ty, llvm_v4i32_ty,  llvm_i8_ty],
+          [IntrNoMem]>;
+
+  def int_x86_avx512_mask_cvttpd2udq_256 : 
+        GCCBuiltin<"__builtin_ia32_cvttpd2udq256_mask">,
+          Intrinsic<[llvm_v4i32_ty],
+          [llvm_v4f64_ty, llvm_v4i32_ty,  llvm_i8_ty],
+          [IntrNoMem]>;
+
+  def int_x86_avx512_mask_cvttpd2udq_512 : 
+        GCCBuiltin<"__builtin_ia32_cvttpd2udq512_mask">,
+          Intrinsic<[llvm_v8i32_ty],
+          [llvm_v8f64_ty, llvm_v8i32_ty,  llvm_i8_ty,  llvm_i32_ty],
+          [IntrNoMem]>;
+
+  def int_x86_avx512_mask_cvttpd2uqq_128 : 
+        GCCBuiltin<"__builtin_ia32_cvttpd2uqq128_mask">,
+          Intrinsic<[llvm_v2i64_ty],
+          [llvm_v2f64_ty, llvm_v2i64_ty,  llvm_i8_ty],
+          [IntrNoMem]>;
+
+  def int_x86_avx512_mask_cvttpd2uqq_256 : 
+        GCCBuiltin<"__builtin_ia32_cvttpd2uqq256_mask">,
+          Intrinsic<[llvm_v4i64_ty],
+          [llvm_v4f64_ty, llvm_v4i64_ty,  llvm_i8_ty],
+          [IntrNoMem]>;
+
+  def int_x86_avx512_mask_cvttpd2uqq_512 : 
+        GCCBuiltin<"__builtin_ia32_cvttpd2uqq512_mask">,
+          Intrinsic<[llvm_v8i64_ty],
+          [llvm_v8f64_ty, llvm_v8i64_ty,  llvm_i8_ty,  llvm_i32_ty],
+          [IntrNoMem]>;
+
+  def int_x86_avx512_mask_cvttps2dq_128 : 
+        GCCBuiltin<"__builtin_ia32_cvttps2dq128_mask">,
+          Intrinsic<[llvm_v4i32_ty],
+          [llvm_v4f32_ty, llvm_v4i32_ty,  llvm_i8_ty],
+          [IntrNoMem]>;
+
+  def int_x86_avx512_mask_cvttps2dq_256 : 
+        GCCBuiltin<"__builtin_ia32_cvttps2dq256_mask">,
+          Intrinsic<[llvm_v8i32_ty],
+          [llvm_v8f32_ty, llvm_v8i32_ty,  llvm_i8_ty],
+          [IntrNoMem]>;
+
+  def int_x86_avx512_mask_cvttps2dq_512 : 
+        GCCBuiltin<"__builtin_ia32_cvttps2dq512_mask">,
+          Intrinsic<[llvm_v16i32_ty],
+          [llvm_v16f32_ty, llvm_v16i32_ty,  llvm_i16_ty,  llvm_i32_ty],
+          [IntrNoMem]>;
+
+  def int_x86_avx512_mask_cvttps2qq_128 : 
+        GCCBuiltin<"__builtin_ia32_cvttps2qq128_mask">,
+          Intrinsic<[llvm_v2i64_ty],
+          [llvm_v4f32_ty, llvm_v2i64_ty,  llvm_i8_ty],
+          [IntrNoMem]>;
+
+  def int_x86_avx512_mask_cvttps2qq_256 : 
+        GCCBuiltin<"__builtin_ia32_cvttps2qq256_mask">,
+          Intrinsic<[llvm_v4i64_ty],
+          [llvm_v4f32_ty, llvm_v4i64_ty,  llvm_i8_ty],
+          [IntrNoMem]>;
+
+  def int_x86_avx512_mask_cvttps2qq_512 : 
+        GCCBuiltin<"__builtin_ia32_cvttps2qq512_mask">,
+          Intrinsic<[llvm_v8i64_ty],
+          [llvm_v8f32_ty, llvm_v8i64_ty,  llvm_i8_ty,  llvm_i32_ty],
+          [IntrNoMem]>;
+
+  def int_x86_avx512_mask_cvttps2udq_128 : 
+        GCCBuiltin<"__builtin_ia32_cvttps2udq128_mask">,
+          Intrinsic<[llvm_v4i32_ty],
+          [llvm_v4f32_ty, llvm_v4i32_ty,  llvm_i8_ty],
+          [IntrNoMem]>;
+
+  def int_x86_avx512_mask_cvttps2udq_256 : 
+        GCCBuiltin<"__builtin_ia32_cvttps2udq256_mask">,
+          Intrinsic<[llvm_v8i32_ty],
+          [llvm_v8f32_ty, llvm_v8i32_ty,  llvm_i8_ty],
+          [IntrNoMem]>;
+
+  def int_x86_avx512_mask_cvttps2udq_512 : 
+        GCCBuiltin<"__builtin_ia32_cvttps2udq512_mask">,
+          Intrinsic<[llvm_v16i32_ty],
+          [llvm_v16f32_ty, llvm_v16i32_ty,  llvm_i16_ty,  llvm_i32_ty],
+          [IntrNoMem]>;
+
+  def int_x86_avx512_mask_cvttps2uqq_128 : 
+        GCCBuiltin<"__builtin_ia32_cvttps2uqq128_mask">,
+          Intrinsic<[llvm_v2i64_ty],
+          [llvm_v4f32_ty, llvm_v2i64_ty,  llvm_i8_ty],
+          [IntrNoMem]>;
+
+  def int_x86_avx512_mask_cvttps2uqq_256 : 
+        GCCBuiltin<"__builtin_ia32_cvttps2uqq256_mask">,
+          Intrinsic<[llvm_v4i64_ty],
+          [llvm_v4f32_ty, llvm_v4i64_ty,  llvm_i8_ty],
+          [IntrNoMem]>;
+
+  def int_x86_avx512_mask_cvttps2uqq_512 : 
+        GCCBuiltin<"__builtin_ia32_cvttps2uqq512_mask">,
+          Intrinsic<[llvm_v8i64_ty],
+          [llvm_v8f32_ty, llvm_v8i64_ty,  llvm_i8_ty,  llvm_i32_ty],
+          [IntrNoMem]>;
+
+  def int_x86_avx512_mask_cvtudq2pd_128 : 
+        GCCBuiltin<"__builtin_ia32_cvtudq2pd128_mask">,
+          Intrinsic<[llvm_v2f64_ty],
+          [llvm_v4i32_ty, llvm_v2f64_ty,  llvm_i8_ty],
+          [IntrNoMem]>;
+
+  def int_x86_avx512_mask_cvtudq2pd_256 : 
+        GCCBuiltin<"__builtin_ia32_cvtudq2pd256_mask">,
+          Intrinsic<[llvm_v4f64_ty],
+          [llvm_v4i32_ty, llvm_v4f64_ty,  llvm_i8_ty],
+          [IntrNoMem]>;
+
+  def int_x86_avx512_mask_cvtudq2pd_512 : 
+        GCCBuiltin<"__builtin_ia32_cvtudq2pd512_mask">,
+          Intrinsic<[llvm_v8f64_ty],
+          [llvm_v8i32_ty, llvm_v8f64_ty,  llvm_i8_ty],
+          [IntrNoMem]>;
+
+  def int_x86_avx512_mask_cvtudq2ps_128 : 
+        GCCBuiltin<"__builtin_ia32_cvtudq2ps128_mask">,
+          Intrinsic<[llvm_v4f32_ty],
+          [llvm_v4i32_ty, llvm_v4f32_ty,  llvm_i8_ty],
+          [IntrNoMem]>;
+
+  def int_x86_avx512_mask_cvtudq2ps_256 : 
+        GCCBuiltin<"__builtin_ia32_cvtudq2ps256_mask">,
+          Intrinsic<[llvm_v8f32_ty],
+          [llvm_v8i32_ty, llvm_v8f32_ty,  llvm_i8_ty],
+          [IntrNoMem]>;
+
+  def int_x86_avx512_mask_cvtudq2ps_512 : 
+        GCCBuiltin<"__builtin_ia32_cvtudq2ps512_mask">,
+          Intrinsic<[llvm_v16f32_ty],
+          [llvm_v16i32_ty, llvm_v16f32_ty,  llvm_i16_ty,  llvm_i32_ty],
+          [IntrNoMem]>;
+
+  def int_x86_avx512_mask_cvtuqq2pd_128 : 
+        GCCBuiltin<"__builtin_ia32_cvtuqq2pd128_mask">,
+          Intrinsic<[llvm_v2f64_ty],
+          [llvm_v2i64_ty, llvm_v2f64_ty,  llvm_i8_ty],
+          [IntrNoMem]>;
+
+  def int_x86_avx512_mask_cvtuqq2pd_256 : 
+        GCCBuiltin<"__builtin_ia32_cvtuqq2pd256_mask">,
+          Intrinsic<[llvm_v4f64_ty],
+          [llvm_v4i64_ty, llvm_v4f64_ty,  llvm_i8_ty],
+          [IntrNoMem]>;
+
+  def int_x86_avx512_mask_cvtuqq2pd_512 : 
+        GCCBuiltin<"__builtin_ia32_cvtuqq2pd512_mask">,
+          Intrinsic<[llvm_v8f64_ty],
+          [llvm_v8i64_ty, llvm_v8f64_ty,  llvm_i8_ty,  llvm_i32_ty],
+          [IntrNoMem]>;
+
+  def int_x86_avx512_mask_cvtuqq2ps_128 : 
+        GCCBuiltin<"__builtin_ia32_cvtuqq2ps128_mask">,
+          Intrinsic<[llvm_v4f32_ty],
+          [llvm_v2i64_ty, llvm_v4f32_ty,  llvm_i8_ty],
+          [IntrNoMem]>;
+
+  def int_x86_avx512_mask_cvtuqq2ps_256 : 
+        GCCBuiltin<"__builtin_ia32_cvtuqq2ps256_mask">,
+          Intrinsic<[llvm_v4f32_ty],
+          [llvm_v4i64_ty, llvm_v4f32_ty,  llvm_i8_ty],
+          [IntrNoMem]>;
+
+  def int_x86_avx512_mask_cvtuqq2ps_512 : 
+        GCCBuiltin<"__builtin_ia32_cvtuqq2ps512_mask">,
+          Intrinsic<[llvm_v8f32_ty],
+          [llvm_v8i64_ty, llvm_v8f32_ty,  llvm_i8_ty,  llvm_i32_ty],
+          [IntrNoMem]>;
+
+  def int_x86_avx512_mask_rndscale_pd_128 : GCCBuiltin<"__builtin_ia32_rndscalepd_128_mask">,
+        Intrinsic<[llvm_v2f64_ty], [llvm_v2f64_ty, llvm_i32_ty,
+                                     llvm_v2f64_ty,  llvm_i8_ty], [IntrNoMem]>;
+  def int_x86_avx512_mask_rndscale_pd_256 : GCCBuiltin<"__builtin_ia32_rndscalepd_256_mask">,
+        Intrinsic<[llvm_v4f64_ty], [llvm_v4f64_ty, llvm_i32_ty, 
+                                     llvm_v4f64_ty,  llvm_i8_ty], [IntrNoMem]>;
+  def int_x86_avx512_mask_rndscale_pd_512 : GCCBuiltin<"__builtin_ia32_rndscalepd_mask">,
+        Intrinsic<[llvm_v8f64_ty], [llvm_v8f64_ty, llvm_i32_ty, llvm_v8f64_ty,
+                                     llvm_i8_ty, llvm_i32_ty], [IntrNoMem]>;
+  def int_x86_avx512_mask_rndscale_ps_128 : GCCBuiltin<"__builtin_ia32_rndscaleps_128_mask">,
+        Intrinsic<[llvm_v4f32_ty], [llvm_v4f32_ty, llvm_i32_ty,
+                                     llvm_v4f32_ty,  llvm_i8_ty], [IntrNoMem]>;
+  def int_x86_avx512_mask_rndscale_ps_256 : GCCBuiltin<"__builtin_ia32_rndscaleps_256_mask">,
+        Intrinsic<[llvm_v8f32_ty], [llvm_v8f32_ty, llvm_i32_ty, 
+                                     llvm_v8f32_ty,  llvm_i8_ty], [IntrNoMem]>;
+  def int_x86_avx512_mask_rndscale_ps_512 : GCCBuiltin<"__builtin_ia32_rndscaleps_mask">,
         Intrinsic<[llvm_v16f32_ty], [llvm_v16f32_ty, llvm_i32_ty, llvm_v16f32_ty,
                                      llvm_i16_ty, llvm_i32_ty], [IntrNoMem]>;
-  def int_x86_avx512_mask_rndscale_pd_512: GCCBuiltin<"__builtin_ia32_rndscalepd_mask">,
+  def int_x86_avx512_mask_reduce_pd_128 : GCCBuiltin<"__builtin_ia32_reducepd128_mask">,
+        Intrinsic<[llvm_v2f64_ty], [llvm_v2f64_ty, llvm_i32_ty, 
+                                     llvm_v2f64_ty,  llvm_i8_ty], [IntrNoMem]>;
+  def int_x86_avx512_mask_reduce_pd_256 : GCCBuiltin<"__builtin_ia32_reducepd256_mask">,
+        Intrinsic<[llvm_v4f64_ty], [llvm_v4f64_ty, llvm_i32_ty,
+                                     llvm_v4f64_ty,  llvm_i8_ty], [IntrNoMem]>;
+  def int_x86_avx512_mask_reduce_pd_512 : GCCBuiltin<"__builtin_ia32_reducepd512_mask">,
         Intrinsic<[llvm_v8f64_ty], [llvm_v8f64_ty, llvm_i32_ty, llvm_v8f64_ty,
                                      llvm_i8_ty, llvm_i32_ty], [IntrNoMem]>;
-  def int_x86_avx512_mask_cvtps2dq_512: GCCBuiltin<"__builtin_ia32_cvtps2dq512_mask">,
-        Intrinsic<[llvm_v16i32_ty], [llvm_v16f32_ty, llvm_v16i32_ty,
-                                     llvm_i16_ty, llvm_i32_ty], [IntrNoMem]>;
-  def int_x86_avx512_mask_cvtpd2dq_512: GCCBuiltin<"__builtin_ia32_cvtpd2dq512_mask">,
-        Intrinsic<[llvm_v8i32_ty], [llvm_v8f64_ty, llvm_v8i32_ty,
-                                    llvm_i8_ty, llvm_i32_ty], [IntrNoMem]>;
-  def int_x86_avx512_mask_cvtps2udq_512: GCCBuiltin<"__builtin_ia32_cvtps2udq512_mask">,
-        Intrinsic<[llvm_v16i32_ty], [llvm_v16f32_ty, llvm_v16i32_ty,
-                                     llvm_i16_ty, llvm_i32_ty], [IntrNoMem]>;
-  def int_x86_avx512_mask_cvtpd2udq_512: GCCBuiltin<"__builtin_ia32_cvtpd2udq512_mask">,
-        Intrinsic<[llvm_v8i32_ty], [llvm_v8f64_ty, llvm_v8i32_ty,
-                                    llvm_i8_ty, llvm_i32_ty], [IntrNoMem]>;
-  def int_x86_avx512_mask_cvtdq2ps_512 : GCCBuiltin<"__builtin_ia32_cvtdq2ps512_mask">,
-        Intrinsic<[llvm_v16f32_ty], [llvm_v16i32_ty, llvm_v16f32_ty,
-                                     llvm_i16_ty, llvm_i32_ty], [IntrNoMem]>;
-  def int_x86_avx512_mask_cvtdq2pd_512 : GCCBuiltin<"__builtin_ia32_cvtdq2pd512_mask">,
-        Intrinsic<[llvm_v8f64_ty], [llvm_v8i32_ty, llvm_v8f64_ty,
-                                    llvm_i8_ty], [IntrNoMem]>;
-  def int_x86_avx512_mask_cvtudq2ps_512 : GCCBuiltin<"__builtin_ia32_cvtudq2ps512_mask">,
-        Intrinsic<[llvm_v16f32_ty], [llvm_v16i32_ty, llvm_v16f32_ty,
+  def int_x86_avx512_mask_reduce_ps_128 : GCCBuiltin<"__builtin_ia32_reduceps128_mask">,
+        Intrinsic<[llvm_v4f32_ty], [llvm_v4f32_ty, llvm_i32_ty, 
+                                     llvm_v4f32_ty,  llvm_i8_ty], [IntrNoMem]>;
+  def int_x86_avx512_mask_reduce_ps_256 : GCCBuiltin<"__builtin_ia32_reduceps256_mask">,
+        Intrinsic<[llvm_v8f32_ty], [llvm_v8f32_ty, llvm_i32_ty,
+                                     llvm_v8f32_ty,  llvm_i8_ty], [IntrNoMem]>;
+  def int_x86_avx512_mask_reduce_ps_512 : GCCBuiltin<"__builtin_ia32_reduceps512_mask">,
+        Intrinsic<[llvm_v16f32_ty], [llvm_v16f32_ty, llvm_i32_ty, llvm_v16f32_ty,
                                      llvm_i16_ty, llvm_i32_ty], [IntrNoMem]>;
-  def int_x86_avx512_mask_cvtudq2pd_512 : GCCBuiltin<"__builtin_ia32_cvtudq2pd512_mask">,
-        Intrinsic<[llvm_v8f64_ty], [llvm_v8i32_ty, llvm_v8f64_ty,
-                                    llvm_i8_ty], [IntrNoMem]>;
-  def int_x86_avx512_mask_cvtpd2ps_512 : GCCBuiltin<"__builtin_ia32_cvtpd2ps512_mask">,
-        Intrinsic<[llvm_v8f32_ty], [llvm_v8f64_ty, llvm_v8f32_ty,
-                                    llvm_i8_ty, llvm_i32_ty], [IntrNoMem]>;
+def int_x86_avx512_mask_range_pd_128 : GCCBuiltin<"__builtin_ia32_rangepd128_mask">,
+        Intrinsic<[llvm_v2f64_ty], [llvm_v2f64_ty, llvm_v2f64_ty, llvm_i32_ty, 
+                                    llvm_v2f64_ty,  llvm_i8_ty], [IntrNoMem]>;
+def int_x86_avx512_mask_range_pd_256 : GCCBuiltin<"__builtin_ia32_rangepd256_mask">,
+        Intrinsic<[llvm_v4f64_ty], [llvm_v4f64_ty, llvm_v4f64_ty, llvm_i32_ty,
+                                    llvm_v4f64_ty,  llvm_i8_ty], [IntrNoMem]>;
+def int_x86_avx512_mask_range_pd_512 : GCCBuiltin<"__builtin_ia32_rangepd512_mask">,
+        Intrinsic<[llvm_v8f64_ty], [llvm_v8f64_ty, llvm_v8f64_ty, llvm_i32_ty,
+                                    llvm_v8f64_ty,  llvm_i8_ty,  llvm_i32_ty], [IntrNoMem]>;
+def int_x86_avx512_mask_range_ps_128 : GCCBuiltin<"__builtin_ia32_rangeps128_mask">,
+        Intrinsic<[llvm_v4f32_ty], [llvm_v4f32_ty, llvm_v4f32_ty, llvm_i32_ty,
+                                    llvm_v4f32_ty,  llvm_i8_ty], [IntrNoMem]>;
+def int_x86_avx512_mask_range_ps_256 : GCCBuiltin<"__builtin_ia32_rangeps256_mask">,
+        Intrinsic<[llvm_v8f32_ty], [llvm_v8f32_ty, llvm_v8f32_ty, llvm_i32_ty, 
+                                    llvm_v8f32_ty,  llvm_i8_ty], [IntrNoMem]>;
+def int_x86_avx512_mask_range_ps_512 : GCCBuiltin<"__builtin_ia32_rangeps512_mask">,
+        Intrinsic<[llvm_v16f32_ty], [llvm_v16f32_ty, llvm_v16f32_ty, llvm_i32_ty,
+                                     llvm_v16f32_ty,  llvm_i16_ty,  llvm_i32_ty], [IntrNoMem]>;
 }
 
 // Vector load with broadcast
@@ -3089,21 +4545,75 @@ let TargetPrefix = "x86" in {  // All intrinsics start with "llvm.x86.".
 }
 //Bitwise Ops
 let TargetPrefix = "x86" in {  // All intrinsics start with "llvm.x86.".
+  def int_x86_avx512_mask_pand_d_128 : GCCBuiltin<"__builtin_ia32_pandd128_mask">,
+          Intrinsic<[llvm_v4i32_ty], [llvm_v4i32_ty, llvm_v4i32_ty,
+                     llvm_v4i32_ty, llvm_i8_ty], [IntrNoMem]>;
+  def int_x86_avx512_mask_pand_d_256 : GCCBuiltin<"__builtin_ia32_pandd256_mask">,
+          Intrinsic<[llvm_v8i32_ty], [llvm_v8i32_ty, llvm_v8i32_ty,
+                     llvm_v8i32_ty, llvm_i8_ty], [IntrNoMem]>;
   def int_x86_avx512_mask_pand_d_512 : GCCBuiltin<"__builtin_ia32_pandd512_mask">,
           Intrinsic<[llvm_v16i32_ty], [llvm_v16i32_ty, llvm_v16i32_ty,
                      llvm_v16i32_ty, llvm_i16_ty], [IntrNoMem]>;
+  def int_x86_avx512_mask_pand_q_128 : GCCBuiltin<"__builtin_ia32_pandq128_mask">,
+          Intrinsic<[llvm_v2i64_ty], [llvm_v2i64_ty, llvm_v2i64_ty,
+                     llvm_v2i64_ty, llvm_i8_ty], [IntrNoMem]>;
+  def int_x86_avx512_mask_pand_q_256 : GCCBuiltin<"__builtin_ia32_pandq256_mask">,
+          Intrinsic<[llvm_v4i64_ty], [llvm_v4i64_ty, llvm_v4i64_ty,
+                     llvm_v4i64_ty, llvm_i8_ty], [IntrNoMem]>;
   def int_x86_avx512_mask_pand_q_512 : GCCBuiltin<"__builtin_ia32_pandq512_mask">,
           Intrinsic<[llvm_v8i64_ty], [llvm_v8i64_ty, llvm_v8i64_ty,
                      llvm_v8i64_ty, llvm_i8_ty], [IntrNoMem]>;
+  def int_x86_avx512_mask_pandn_d_128 : GCCBuiltin<"__builtin_ia32_pandnd128_mask">,
+          Intrinsic<[llvm_v4i32_ty], [llvm_v4i32_ty, llvm_v4i32_ty,
+                     llvm_v4i32_ty, llvm_i8_ty], [IntrNoMem]>;
+  def int_x86_avx512_mask_pandn_d_256 : GCCBuiltin<"__builtin_ia32_pandnd256_mask">,
+          Intrinsic<[llvm_v8i32_ty], [llvm_v8i32_ty, llvm_v8i32_ty,
+                     llvm_v8i32_ty, llvm_i8_ty], [IntrNoMem]>;
+  def int_x86_avx512_mask_pandn_d_512 : GCCBuiltin<"__builtin_ia32_pandnd512_mask">,
+          Intrinsic<[llvm_v16i32_ty], [llvm_v16i32_ty, llvm_v16i32_ty,
+                     llvm_v16i32_ty, llvm_i16_ty], [IntrNoMem]>;
+  def int_x86_avx512_mask_pandn_q_128 : GCCBuiltin<"__builtin_ia32_pandnq128_mask">,
+          Intrinsic<[llvm_v2i64_ty], [llvm_v2i64_ty, llvm_v2i64_ty,
+                     llvm_v2i64_ty, llvm_i8_ty], [IntrNoMem]>;
+  def int_x86_avx512_mask_pandn_q_256 : GCCBuiltin<"__builtin_ia32_pandnq256_mask">,
+          Intrinsic<[llvm_v4i64_ty], [llvm_v4i64_ty, llvm_v4i64_ty,
+                     llvm_v4i64_ty, llvm_i8_ty], [IntrNoMem]>;
+  def int_x86_avx512_mask_pandn_q_512 : GCCBuiltin<"__builtin_ia32_pandnq512_mask">,
+          Intrinsic<[llvm_v8i64_ty], [llvm_v8i64_ty, llvm_v8i64_ty,
+                     llvm_v8i64_ty, llvm_i8_ty], [IntrNoMem]>;
+  def int_x86_avx512_mask_por_d_128 : GCCBuiltin<"__builtin_ia32_pord128_mask">,
+          Intrinsic<[llvm_v4i32_ty], [llvm_v4i32_ty, llvm_v4i32_ty,
+                     llvm_v4i32_ty, llvm_i8_ty], [IntrNoMem]>;
+  def int_x86_avx512_mask_por_d_256 : GCCBuiltin<"__builtin_ia32_pord256_mask">,
+          Intrinsic<[llvm_v8i32_ty], [llvm_v8i32_ty, llvm_v8i32_ty,
+                     llvm_v8i32_ty, llvm_i8_ty], [IntrNoMem]>;
   def int_x86_avx512_mask_por_d_512 : GCCBuiltin<"__builtin_ia32_pord512_mask">,
           Intrinsic<[llvm_v16i32_ty], [llvm_v16i32_ty, llvm_v16i32_ty,
                      llvm_v16i32_ty, llvm_i16_ty], [IntrNoMem]>;
+  def int_x86_avx512_mask_por_q_128 : GCCBuiltin<"__builtin_ia32_porq128_mask">,
+          Intrinsic<[llvm_v2i64_ty], [llvm_v2i64_ty, llvm_v2i64_ty,
+                     llvm_v2i64_ty, llvm_i8_ty], [IntrNoMem]>;
+  def int_x86_avx512_mask_por_q_256 : GCCBuiltin<"__builtin_ia32_porq256_mask">,
+          Intrinsic<[llvm_v4i64_ty], [llvm_v4i64_ty, llvm_v4i64_ty,
+                     llvm_v4i64_ty, llvm_i8_ty], [IntrNoMem]>;
   def int_x86_avx512_mask_por_q_512 : GCCBuiltin<"__builtin_ia32_porq512_mask">,
           Intrinsic<[llvm_v8i64_ty], [llvm_v8i64_ty, llvm_v8i64_ty,
                      llvm_v8i64_ty, llvm_i8_ty], [IntrNoMem]>;
+  def int_x86_avx512_mask_pxor_d_128 : GCCBuiltin<"__builtin_ia32_pxord128_mask">,
+          Intrinsic<[llvm_v4i32_ty], [llvm_v4i32_ty, llvm_v4i32_ty,
+                     llvm_v4i32_ty, llvm_i8_ty], [IntrNoMem]>;
+  def int_x86_avx512_mask_pxor_d_256 : GCCBuiltin<"__builtin_ia32_pxord256_mask">,
+          Intrinsic<[llvm_v8i32_ty], [llvm_v8i32_ty, llvm_v8i32_ty,
+                     llvm_v8i32_ty, llvm_i8_ty], [IntrNoMem]>;
   def int_x86_avx512_mask_pxor_d_512 : GCCBuiltin<"__builtin_ia32_pxord512_mask">,
           Intrinsic<[llvm_v16i32_ty], [llvm_v16i32_ty, llvm_v16i32_ty,
                      llvm_v16i32_ty, llvm_i16_ty], [IntrNoMem]>;
+  def int_x86_avx512_mask_pxor_q_128 : GCCBuiltin<"__builtin_ia32_pxorq128_mask">,
+          Intrinsic<[llvm_v2i64_ty], [llvm_v2i64_ty, llvm_v2i64_ty,
+                     llvm_v2i64_ty, llvm_i8_ty], [IntrNoMem]>;
+  def int_x86_avx512_mask_pxor_q_256 : GCCBuiltin<"__builtin_ia32_pxorq256_mask">,
+          Intrinsic<[llvm_v4i64_ty], [llvm_v4i64_ty, llvm_v4i64_ty,
+                     llvm_v4i64_ty, llvm_i8_ty], [IntrNoMem]>;
   def int_x86_avx512_mask_pxor_q_512 : GCCBuiltin<"__builtin_ia32_pxorq512_mask">,
           Intrinsic<[llvm_v8i64_ty], [llvm_v8i64_ty, llvm_v8i64_ty,
                      llvm_v8i64_ty, llvm_i8_ty], [IntrNoMem]>;
@@ -3111,43 +4621,152 @@ let TargetPrefix = "x86" in {  // All intrinsics start with "llvm.x86.".
 // Arithmetic ops
 let TargetPrefix = "x86" in {  // All intrinsics start with "llvm.x86.".
 
+  def int_x86_avx512_mask_add_ps_128 : GCCBuiltin<"__builtin_ia32_addps128_mask">,
+          Intrinsic<[llvm_v4f32_ty], [llvm_v4f32_ty, llvm_v4f32_ty,
+                     llvm_v4f32_ty, llvm_i8_ty], [IntrNoMem]>;
+  def int_x86_avx512_mask_add_ps_256 : GCCBuiltin<"__builtin_ia32_addps256_mask">,
+          Intrinsic<[llvm_v8f32_ty], [llvm_v8f32_ty, llvm_v8f32_ty,
+                     llvm_v8f32_ty, llvm_i8_ty], [IntrNoMem]>;
   def int_x86_avx512_mask_add_ps_512 : GCCBuiltin<"__builtin_ia32_addps512_mask">,
           Intrinsic<[llvm_v16f32_ty], [llvm_v16f32_ty, llvm_v16f32_ty,
                      llvm_v16f32_ty, llvm_i16_ty, llvm_i32_ty], [IntrNoMem]>;
+  def int_x86_avx512_mask_add_pd_128 : GCCBuiltin<"__builtin_ia32_addpd128_mask">,
+          Intrinsic<[llvm_v2f64_ty], [llvm_v2f64_ty, llvm_v2f64_ty,
+                     llvm_v2f64_ty, llvm_i8_ty], [IntrNoMem]>;
+  def int_x86_avx512_mask_add_pd_256 : GCCBuiltin<"__builtin_ia32_addpd256_mask">,
+          Intrinsic<[llvm_v4f64_ty], [llvm_v4f64_ty, llvm_v4f64_ty,
+                     llvm_v4f64_ty, llvm_i8_ty], [IntrNoMem]>;
   def int_x86_avx512_mask_add_pd_512 : GCCBuiltin<"__builtin_ia32_addpd512_mask">,
           Intrinsic<[llvm_v8f64_ty], [llvm_v8f64_ty, llvm_v8f64_ty,
                      llvm_v8f64_ty, llvm_i8_ty, llvm_i32_ty], [IntrNoMem]>;
+  def int_x86_avx512_mask_sub_ps_128 : GCCBuiltin<"__builtin_ia32_subps128_mask">,
+          Intrinsic<[llvm_v4f32_ty], [llvm_v4f32_ty, llvm_v4f32_ty,
+                     llvm_v4f32_ty, llvm_i8_ty], [IntrNoMem]>;
+  def int_x86_avx512_mask_sub_ps_256 : GCCBuiltin<"__builtin_ia32_subps256_mask">,
+          Intrinsic<[llvm_v8f32_ty], [llvm_v8f32_ty, llvm_v8f32_ty,
+                     llvm_v8f32_ty, llvm_i8_ty], [IntrNoMem]>;
   def int_x86_avx512_mask_sub_ps_512 : GCCBuiltin<"__builtin_ia32_subps512_mask">,
           Intrinsic<[llvm_v16f32_ty], [llvm_v16f32_ty, llvm_v16f32_ty,
                      llvm_v16f32_ty, llvm_i16_ty, llvm_i32_ty], [IntrNoMem]>;
+  def int_x86_avx512_mask_sub_pd_128 : GCCBuiltin<"__builtin_ia32_subpd128_mask">,
+          Intrinsic<[llvm_v2f64_ty], [llvm_v2f64_ty, llvm_v2f64_ty,
+                     llvm_v2f64_ty, llvm_i8_ty], [IntrNoMem]>;
+  def int_x86_avx512_mask_sub_pd_256 : GCCBuiltin<"__builtin_ia32_subpd256_mask">,
+          Intrinsic<[llvm_v4f64_ty], [llvm_v4f64_ty, llvm_v4f64_ty,
+                     llvm_v4f64_ty, llvm_i8_ty], [IntrNoMem]>;
   def int_x86_avx512_mask_sub_pd_512 : GCCBuiltin<"__builtin_ia32_subpd512_mask">,
           Intrinsic<[llvm_v8f64_ty], [llvm_v8f64_ty, llvm_v8f64_ty,
                      llvm_v8f64_ty, llvm_i8_ty, llvm_i32_ty], [IntrNoMem]>;
+  def int_x86_avx512_mask_mul_ps_128 : GCCBuiltin<"__builtin_ia32_mulps_mask">,
+          Intrinsic<[llvm_v4f32_ty], [llvm_v4f32_ty, llvm_v4f32_ty,
+                     llvm_v4f32_ty, llvm_i8_ty], [IntrNoMem]>;
+  def int_x86_avx512_mask_mul_ps_256 : GCCBuiltin<"__builtin_ia32_mulps256_mask">,
+          Intrinsic<[llvm_v8f32_ty], [llvm_v8f32_ty, llvm_v8f32_ty,
+                     llvm_v8f32_ty, llvm_i8_ty], [IntrNoMem]>;
   def int_x86_avx512_mask_mul_ps_512 : GCCBuiltin<"__builtin_ia32_mulps512_mask">,
           Intrinsic<[llvm_v16f32_ty], [llvm_v16f32_ty, llvm_v16f32_ty,
                      llvm_v16f32_ty, llvm_i16_ty, llvm_i32_ty], [IntrNoMem]>;
+  def int_x86_avx512_mask_mul_pd_128 : GCCBuiltin<"__builtin_ia32_mulpd_mask">,
+          Intrinsic<[llvm_v2f64_ty], [llvm_v2f64_ty, llvm_v2f64_ty,
+                     llvm_v2f64_ty, llvm_i8_ty], [IntrNoMem]>;
+  def int_x86_avx512_mask_mul_pd_256 : GCCBuiltin<"__builtin_ia32_mulpd256_mask">,
+          Intrinsic<[llvm_v4f64_ty], [llvm_v4f64_ty, llvm_v4f64_ty,
+                     llvm_v4f64_ty, llvm_i8_ty], [IntrNoMem]>;
   def int_x86_avx512_mask_mul_pd_512 : GCCBuiltin<"__builtin_ia32_mulpd512_mask">,
           Intrinsic<[llvm_v8f64_ty], [llvm_v8f64_ty, llvm_v8f64_ty,
                      llvm_v8f64_ty, llvm_i8_ty, llvm_i32_ty], [IntrNoMem]>;
+  def int_x86_avx512_mask_div_ps_128 : GCCBuiltin<"__builtin_ia32_divps_mask">,
+          Intrinsic<[llvm_v4f32_ty], [llvm_v4f32_ty, llvm_v4f32_ty,
+                     llvm_v4f32_ty, llvm_i8_ty], [IntrNoMem]>;
+  def int_x86_avx512_mask_div_ps_256 : GCCBuiltin<"__builtin_ia32_divps256_mask">,
+          Intrinsic<[llvm_v8f32_ty], [llvm_v8f32_ty, llvm_v8f32_ty,
+                     llvm_v8f32_ty, llvm_i8_ty], [IntrNoMem]>;
   def int_x86_avx512_mask_div_ps_512 : GCCBuiltin<"__builtin_ia32_divps512_mask">,
           Intrinsic<[llvm_v16f32_ty], [llvm_v16f32_ty, llvm_v16f32_ty,
                      llvm_v16f32_ty, llvm_i16_ty, llvm_i32_ty], [IntrNoMem]>;
+  def int_x86_avx512_mask_div_pd_128 : GCCBuiltin<"__builtin_ia32_divpd_mask">,
+          Intrinsic<[llvm_v2f64_ty], [llvm_v2f64_ty, llvm_v2f64_ty,
+                     llvm_v2f64_ty, llvm_i8_ty], [IntrNoMem]>;
+  def int_x86_avx512_mask_div_pd_256 : GCCBuiltin<"__builtin_ia32_divpd256_mask">,
+          Intrinsic<[llvm_v4f64_ty], [llvm_v4f64_ty, llvm_v4f64_ty,
+                     llvm_v4f64_ty, llvm_i8_ty], [IntrNoMem]>;
   def int_x86_avx512_mask_div_pd_512 : GCCBuiltin<"__builtin_ia32_divpd512_mask">,
           Intrinsic<[llvm_v8f64_ty], [llvm_v8f64_ty, llvm_v8f64_ty,
                      llvm_v8f64_ty, llvm_i8_ty, llvm_i32_ty], [IntrNoMem]>;
+  def int_x86_avx512_mask_max_ps_128 : GCCBuiltin<"__builtin_ia32_maxps_mask">,
+          Intrinsic<[llvm_v4f32_ty], [llvm_v4f32_ty, llvm_v4f32_ty,
+                     llvm_v4f32_ty, llvm_i8_ty], [IntrNoMem]>;
+  def int_x86_avx512_mask_max_ps_256 : GCCBuiltin<"__builtin_ia32_maxps256_mask">,
+          Intrinsic<[llvm_v8f32_ty], [llvm_v8f32_ty, llvm_v8f32_ty,
+                     llvm_v8f32_ty, llvm_i8_ty], [IntrNoMem]>;
   def int_x86_avx512_mask_max_ps_512 : GCCBuiltin<"__builtin_ia32_maxps512_mask">,
           Intrinsic<[llvm_v16f32_ty], [llvm_v16f32_ty, llvm_v16f32_ty,
                      llvm_v16f32_ty, llvm_i16_ty, llvm_i32_ty], [IntrNoMem]>;
+  def int_x86_avx512_mask_max_pd_128 : GCCBuiltin<"__builtin_ia32_maxpd_mask">,
+          Intrinsic<[llvm_v2f64_ty], [llvm_v2f64_ty, llvm_v2f64_ty,
+                     llvm_v2f64_ty, llvm_i8_ty], [IntrNoMem]>;
+  def int_x86_avx512_mask_max_pd_256 : GCCBuiltin<"__builtin_ia32_maxpd256_mask">,
+          Intrinsic<[llvm_v4f64_ty], [llvm_v4f64_ty, llvm_v4f64_ty,
+                     llvm_v4f64_ty, llvm_i8_ty], [IntrNoMem]>;
   def int_x86_avx512_mask_max_pd_512 : GCCBuiltin<"__builtin_ia32_maxpd512_mask">,
           Intrinsic<[llvm_v8f64_ty], [llvm_v8f64_ty, llvm_v8f64_ty,
                      llvm_v8f64_ty, llvm_i8_ty, llvm_i32_ty], [IntrNoMem]>;
+  def int_x86_avx512_mask_min_ps_128 : GCCBuiltin<"__builtin_ia32_minps_mask">,
+          Intrinsic<[llvm_v4f32_ty], [llvm_v4f32_ty, llvm_v4f32_ty,
+                     llvm_v4f32_ty, llvm_i8_ty], [IntrNoMem]>;
+  def int_x86_avx512_mask_min_ps_256 : GCCBuiltin<"__builtin_ia32_minps256_mask">,
+          Intrinsic<[llvm_v8f32_ty], [llvm_v8f32_ty, llvm_v8f32_ty,
+                     llvm_v8f32_ty, llvm_i8_ty], [IntrNoMem]>;
   def int_x86_avx512_mask_min_ps_512 : GCCBuiltin<"__builtin_ia32_minps512_mask">,
           Intrinsic<[llvm_v16f32_ty], [llvm_v16f32_ty, llvm_v16f32_ty,
                      llvm_v16f32_ty, llvm_i16_ty, llvm_i32_ty], [IntrNoMem]>;
+  def int_x86_avx512_mask_min_pd_128 : GCCBuiltin<"__builtin_ia32_minpd_mask">,
+          Intrinsic<[llvm_v2f64_ty], [llvm_v2f64_ty, llvm_v2f64_ty,
+                     llvm_v2f64_ty, llvm_i8_ty], [IntrNoMem]>;
+  def int_x86_avx512_mask_min_pd_256 : GCCBuiltin<"__builtin_ia32_minpd256_mask">,
+          Intrinsic<[llvm_v4f64_ty], [llvm_v4f64_ty, llvm_v4f64_ty,
+                     llvm_v4f64_ty, llvm_i8_ty], [IntrNoMem]>;
   def int_x86_avx512_mask_min_pd_512 : GCCBuiltin<"__builtin_ia32_minpd512_mask">,
           Intrinsic<[llvm_v8f64_ty], [llvm_v8f64_ty, llvm_v8f64_ty,
                      llvm_v8f64_ty, llvm_i8_ty, llvm_i32_ty], [IntrNoMem]>;
 
+  def int_x86_avx512_mask_add_ss_round : GCCBuiltin<"__builtin_ia32_addss_mask">,
+          Intrinsic<[llvm_v4f32_ty], [llvm_v4f32_ty, llvm_v4f32_ty,
+                     llvm_v4f32_ty, llvm_i8_ty, llvm_i32_ty], [IntrNoMem]>;
+  def int_x86_avx512_mask_div_ss_round : GCCBuiltin<"__builtin_ia32_divss_mask">,
+          Intrinsic<[llvm_v4f32_ty], [llvm_v4f32_ty, llvm_v4f32_ty,
+                     llvm_v4f32_ty, llvm_i8_ty, llvm_i32_ty], [IntrNoMem]>;
+  def int_x86_avx512_mask_mul_ss_round : GCCBuiltin<"__builtin_ia32_mulss_mask">,
+          Intrinsic<[llvm_v4f32_ty], [llvm_v4f32_ty, llvm_v4f32_ty,
+                     llvm_v4f32_ty, llvm_i8_ty, llvm_i32_ty], [IntrNoMem]>;
+  def int_x86_avx512_mask_sub_ss_round : GCCBuiltin<"__builtin_ia32_subss_mask">,
+          Intrinsic<[llvm_v4f32_ty], [llvm_v4f32_ty, llvm_v4f32_ty,
+                     llvm_v4f32_ty, llvm_i8_ty, llvm_i32_ty], [IntrNoMem]>;
+  def int_x86_avx512_mask_max_ss_round : GCCBuiltin<"__builtin_ia32_maxss_mask">,
+          Intrinsic<[llvm_v4f32_ty], [llvm_v4f32_ty, llvm_v4f32_ty,
+                     llvm_v4f32_ty, llvm_i8_ty, llvm_i32_ty], [IntrNoMem]>;
+  def int_x86_avx512_mask_min_ss_round : GCCBuiltin<"__builtin_ia32_minss_mask">,
+          Intrinsic<[llvm_v4f32_ty], [llvm_v4f32_ty, llvm_v4f32_ty,
+                     llvm_v4f32_ty, llvm_i8_ty, llvm_i32_ty], [IntrNoMem]>;
+  def int_x86_avx512_mask_add_sd_round : GCCBuiltin<"__builtin_ia32_addsd_mask">,
+          Intrinsic<[llvm_v2f64_ty], [llvm_v2f64_ty, llvm_v2f64_ty,
+                     llvm_v2f64_ty, llvm_i8_ty, llvm_i32_ty], [IntrNoMem]>;
+  def int_x86_avx512_mask_div_sd_round : GCCBuiltin<"__builtin_ia32_divsd_mask">,
+          Intrinsic<[llvm_v2f64_ty], [llvm_v2f64_ty, llvm_v2f64_ty,
+                     llvm_v2f64_ty, llvm_i8_ty, llvm_i32_ty], [IntrNoMem]>;
+  def int_x86_avx512_mask_mul_sd_round : GCCBuiltin<"__builtin_ia32_mulsd_mask">,
+          Intrinsic<[llvm_v2f64_ty], [llvm_v2f64_ty, llvm_v2f64_ty,
+                     llvm_v2f64_ty, llvm_i8_ty, llvm_i32_ty], [IntrNoMem]>;
+  def int_x86_avx512_mask_sub_sd_round : GCCBuiltin<"__builtin_ia32_subsd_mask">,
+          Intrinsic<[llvm_v2f64_ty], [llvm_v2f64_ty, llvm_v2f64_ty,
+                     llvm_v2f64_ty, llvm_i8_ty, llvm_i32_ty], [IntrNoMem]>;
+  def int_x86_avx512_mask_max_sd_round : GCCBuiltin<"__builtin_ia32_maxsd_mask">,
+          Intrinsic<[llvm_v2f64_ty], [llvm_v2f64_ty, llvm_v2f64_ty,
+                     llvm_v2f64_ty, llvm_i8_ty, llvm_i32_ty], [IntrNoMem]>;
+  def int_x86_avx512_mask_min_sd_round : GCCBuiltin<"__builtin_ia32_minsd_mask">,
+          Intrinsic<[llvm_v2f64_ty], [llvm_v2f64_ty, llvm_v2f64_ty,
+                     llvm_v2f64_ty, llvm_i8_ty, llvm_i32_ty], [IntrNoMem]>;
+
   def int_x86_avx512_mask_rndscale_ss : GCCBuiltin<"__builtin_ia32_rndscaless_mask">,
           Intrinsic<[llvm_v4f32_ty], [llvm_v4f32_ty, llvm_v4f32_ty, llvm_v4f32_ty,
                                      llvm_i8_ty, llvm_i32_ty, llvm_i32_ty],
@@ -3156,6 +4775,47 @@ let TargetPrefix = "x86" in {  // All intrinsics start with "llvm.x86.".
           Intrinsic<[llvm_v2f64_ty], [llvm_v2f64_ty, llvm_v2f64_ty, llvm_v2f64_ty,
                                       llvm_i8_ty, llvm_i32_ty, llvm_i32_ty],
                                      [IntrNoMem]>;
+  def int_x86_avx512_mask_range_ss : GCCBuiltin<"__builtin_ia32_rangess_mask">,
+          Intrinsic<[llvm_v4f32_ty], [llvm_v4f32_ty, llvm_v4f32_ty, llvm_v4f32_ty,
+                                     llvm_i8_ty, llvm_i32_ty, llvm_i32_ty],
+                                     [IntrNoMem]>;
+  def int_x86_avx512_mask_range_sd : GCCBuiltin<"__builtin_ia32_rangesd_mask">,
+          Intrinsic<[llvm_v2f64_ty], [llvm_v2f64_ty, llvm_v2f64_ty, llvm_v2f64_ty,
+                                      llvm_i8_ty, llvm_i32_ty, llvm_i32_ty],
+                                     [IntrNoMem]>;
+  def int_x86_avx512_mask_reduce_ss : GCCBuiltin<"__builtin_ia32_reducess_mask">,
+          Intrinsic<[llvm_v4f32_ty], [llvm_v4f32_ty, llvm_v4f32_ty, llvm_v4f32_ty,
+                                     llvm_i8_ty, llvm_i32_ty, llvm_i32_ty],
+                                     [IntrNoMem]>;
+  def int_x86_avx512_mask_reduce_sd : GCCBuiltin<"__builtin_ia32_reducesd_mask">,
+          Intrinsic<[llvm_v2f64_ty], [llvm_v2f64_ty, llvm_v2f64_ty, llvm_v2f64_ty,
+                                      llvm_i8_ty, llvm_i32_ty, llvm_i32_ty],
+                                     [IntrNoMem]>;
+  def int_x86_avx512_mask_scalef_sd : GCCBuiltin<"__builtin_ia32_scalefsd_round">,
+          Intrinsic<[llvm_v2f64_ty], [llvm_v2f64_ty, llvm_v2f64_ty, 
+                                      llvm_v2f64_ty, llvm_i8_ty, llvm_i32_ty], [IntrNoMem]>;
+  def int_x86_avx512_mask_scalef_ss : GCCBuiltin<"__builtin_ia32_scalefss_round">,
+          Intrinsic<[llvm_v4f32_ty], [llvm_v4f32_ty, llvm_v4f32_ty, 
+                                      llvm_v4f32_ty, llvm_i8_ty, llvm_i32_ty], [IntrNoMem]>;
+  def int_x86_avx512_mask_scalef_pd_128 : GCCBuiltin<"__builtin_ia32_scalefpd128_mask">,
+          Intrinsic<[llvm_v2f64_ty], [llvm_v2f64_ty, llvm_v2f64_ty, 
+                    llvm_v2f64_ty, llvm_i8_ty], [IntrNoMem]>;
+  def int_x86_avx512_mask_scalef_pd_256 : GCCBuiltin<"__builtin_ia32_scalefpd256_mask">,
+          Intrinsic<[llvm_v4f64_ty], [llvm_v4f64_ty, llvm_v4f64_ty, 
+                    llvm_v4f64_ty, llvm_i8_ty],[IntrNoMem]>;
+  def int_x86_avx512_mask_scalef_pd_512 : GCCBuiltin<"__builtin_ia32_scalefpd512_mask">,
+          Intrinsic<[llvm_v8f64_ty], [llvm_v8f64_ty, llvm_v8f64_ty, 
+                    llvm_v8f64_ty, llvm_i8_ty, llvm_i32_ty], [IntrNoMem]>;
+  def int_x86_avx512_mask_scalef_ps_128 : GCCBuiltin<"__builtin_ia32_scalefps128_mask">,
+          Intrinsic<[llvm_v4f32_ty], [llvm_v4f32_ty, llvm_v4f32_ty, 
+                    llvm_v4f32_ty, llvm_i8_ty], [IntrNoMem]>;
+  def int_x86_avx512_mask_scalef_ps_256 : GCCBuiltin<"__builtin_ia32_scalefps256_mask">,
+          Intrinsic<[llvm_v8f32_ty], [llvm_v8f32_ty, llvm_v8f32_ty, 
+                    llvm_v8f32_ty, llvm_i8_ty], [IntrNoMem]>;
+  def int_x86_avx512_mask_scalef_ps_512 : GCCBuiltin<"__builtin_ia32_scalefps512_mask">,
+          Intrinsic<[llvm_v16f32_ty], [llvm_v16f32_ty, llvm_v16f32_ty, 
+                    llvm_v16f32_ty, llvm_i16_ty, llvm_i32_ty], [IntrNoMem]>;
+
   def int_x86_avx512_sqrt_ss        : GCCBuiltin<"__builtin_ia32_sqrtrndss">,
               Intrinsic<[llvm_v4f32_ty], [llvm_v4f32_ty, llvm_v4f32_ty],
                         [IntrNoMem]>;
@@ -3163,13 +4823,50 @@ let TargetPrefix = "x86" in {  // All intrinsics start with "llvm.x86.".
               Intrinsic<[llvm_v2f64_ty], [llvm_v2f64_ty, llvm_v2f64_ty],
                         [IntrNoMem]>;
 
-  def int_x86_avx512_sqrt_pd_512 : GCCBuiltin<"__builtin_ia32_sqrtpd512_mask">,
+  def int_x86_avx512_mask_sqrt_pd_128 : GCCBuiltin<"__builtin_ia32_sqrtpd128_mask">,
+        Intrinsic<[llvm_v2f64_ty], [llvm_v2f64_ty, llvm_v2f64_ty,
+                                    llvm_i8_ty], [IntrNoMem]>;
+  def int_x86_avx512_mask_sqrt_pd_256 : GCCBuiltin<"__builtin_ia32_sqrtpd256_mask">,
+        Intrinsic<[llvm_v4f64_ty], [llvm_v4f64_ty, llvm_v4f64_ty,
+                                    llvm_i8_ty], [IntrNoMem]>;
+  def int_x86_avx512_mask_sqrt_pd_512 : GCCBuiltin<"__builtin_ia32_sqrtpd512_mask">,
+        Intrinsic<[llvm_v8f64_ty], [llvm_v8f64_ty, llvm_v8f64_ty,
+                                    llvm_i8_ty, llvm_i32_ty], [IntrNoMem]>;
+  def int_x86_avx512_mask_sqrt_ps_128 : GCCBuiltin<"__builtin_ia32_sqrtps128_mask">,
+        Intrinsic<[llvm_v4f32_ty], [llvm_v4f32_ty, llvm_v4f32_ty,
+                                     llvm_i8_ty], [IntrNoMem]>;
+  def int_x86_avx512_mask_sqrt_ps_256 : GCCBuiltin<"__builtin_ia32_sqrtps256_mask">,
+        Intrinsic<[llvm_v8f32_ty], [llvm_v8f32_ty, llvm_v8f32_ty,
+                                     llvm_i8_ty], [IntrNoMem]>;
+  def int_x86_avx512_mask_sqrt_ps_512 : GCCBuiltin<"__builtin_ia32_sqrtps512_mask">,
+        Intrinsic<[llvm_v16f32_ty], [llvm_v16f32_ty, llvm_v16f32_ty,
+                                     llvm_i16_ty, llvm_i32_ty], [IntrNoMem]>;
+  def int_x86_avx512_mask_getexp_pd_128 : GCCBuiltin<"__builtin_ia32_getexppd128_mask">,
+        Intrinsic<[llvm_v2f64_ty], [llvm_v2f64_ty, llvm_v2f64_ty,
+                                    llvm_i8_ty], [IntrNoMem]>;
+  def int_x86_avx512_mask_getexp_pd_256 : GCCBuiltin<"__builtin_ia32_getexppd256_mask">,
+        Intrinsic<[llvm_v4f64_ty], [llvm_v4f64_ty, llvm_v4f64_ty,
+                                    llvm_i8_ty], [IntrNoMem]>;
+  def int_x86_avx512_mask_getexp_pd_512 : GCCBuiltin<"__builtin_ia32_getexppd512_mask">,
         Intrinsic<[llvm_v8f64_ty], [llvm_v8f64_ty, llvm_v8f64_ty,
                                     llvm_i8_ty, llvm_i32_ty], [IntrNoMem]>;
-  def int_x86_avx512_sqrt_ps_512 : GCCBuiltin<"__builtin_ia32_sqrtps512_mask">,
+  def int_x86_avx512_mask_getexp_ps_128 : GCCBuiltin<"__builtin_ia32_getexpps128_mask">,
+        Intrinsic<[llvm_v4f32_ty], [llvm_v4f32_ty, llvm_v4f32_ty,
+                                     llvm_i8_ty], [IntrNoMem]>;
+  def int_x86_avx512_mask_getexp_ps_256 : GCCBuiltin<"__builtin_ia32_getexpps256_mask">,
+        Intrinsic<[llvm_v8f32_ty], [llvm_v8f32_ty, llvm_v8f32_ty,
+                                     llvm_i8_ty], [IntrNoMem]>;
+  def int_x86_avx512_mask_getexp_ps_512 : GCCBuiltin<"__builtin_ia32_getexpps512_mask">,
         Intrinsic<[llvm_v16f32_ty], [llvm_v16f32_ty, llvm_v16f32_ty,
                                      llvm_i16_ty, llvm_i32_ty], [IntrNoMem]>;
 
+  def int_x86_avx512_mask_getexp_ss : GCCBuiltin<"__builtin_ia32_getexpss_mask">,
+        Intrinsic<[llvm_v4f32_ty], [llvm_v4f32_ty, llvm_v4f32_ty, llvm_v4f32_ty,
+                                    llvm_i8_ty, llvm_i32_ty], [IntrNoMem]>;
+  def int_x86_avx512_mask_getexp_sd : GCCBuiltin<"__builtin_ia32_getexpsd_mask">,
+        Intrinsic<[llvm_v2f64_ty], [llvm_v2f64_ty, llvm_v2f64_ty, llvm_v2f64_ty,
+                                    llvm_i8_ty, llvm_i32_ty], [IntrNoMem]>;
+
   def int_x86_avx512_rsqrt14_ss : GCCBuiltin<"__builtin_ia32_rsqrt14ss_mask">,
         Intrinsic<[llvm_v4f32_ty], [llvm_v4f32_ty, llvm_v4f32_ty, llvm_v4f32_ty,
                                     llvm_i8_ty], [IntrNoMem]>;
@@ -3235,28 +4932,341 @@ let TargetPrefix = "x86" in {  // All intrinsics start with "llvm.x86.".
                                         llvm_v2f64_ty, llvm_i8_ty, llvm_i32_ty],
                       [IntrNoMem]>;
 }
-
+// FP logical ops
+let TargetPrefix = "x86" in {
+  def int_x86_avx512_mask_and_pd_128 : GCCBuiltin<"__builtin_ia32_andpd128_mask">,
+            Intrinsic<[llvm_v2f64_ty], [llvm_v2f64_ty, llvm_v2f64_ty,
+                      llvm_v2f64_ty, llvm_i8_ty], [IntrNoMem]>;
+  def int_x86_avx512_mask_and_pd_256 : GCCBuiltin<"__builtin_ia32_andpd256_mask">,
+            Intrinsic<[llvm_v4f64_ty], [llvm_v4f64_ty, llvm_v4f64_ty,
+                      llvm_v4f64_ty, llvm_i8_ty], [IntrNoMem]>;
+  def int_x86_avx512_mask_and_pd_512 : GCCBuiltin<"__builtin_ia32_andpd512_mask">,
+          Intrinsic<[llvm_v8f64_ty], [llvm_v8f64_ty, llvm_v8f64_ty,
+                    llvm_v8f64_ty, llvm_i8_ty], [IntrNoMem]>;
+  def int_x86_avx512_mask_and_ps_128 : GCCBuiltin<"__builtin_ia32_andps128_mask">,
+            Intrinsic<[llvm_v4f32_ty], [llvm_v4f32_ty, llvm_v4f32_ty,
+                      llvm_v4f32_ty, llvm_i8_ty], [IntrNoMem]>;
+  def int_x86_avx512_mask_and_ps_256 : GCCBuiltin<"__builtin_ia32_andps256_mask">,
+            Intrinsic<[llvm_v8f32_ty], [llvm_v8f32_ty, llvm_v8f32_ty,
+                      llvm_v8f32_ty, llvm_i8_ty], [IntrNoMem]>;
+  def int_x86_avx512_mask_and_ps_512 : GCCBuiltin<"__builtin_ia32_andps512_mask">,
+          Intrinsic<[llvm_v16f32_ty], [llvm_v16f32_ty, llvm_v16f32_ty,
+                    llvm_v16f32_ty, llvm_i16_ty], [IntrNoMem]>;
+  def int_x86_avx512_mask_andn_pd_128 : GCCBuiltin<"__builtin_ia32_andnpd128_mask">,
+            Intrinsic<[llvm_v2f64_ty], [llvm_v2f64_ty, llvm_v2f64_ty,
+                      llvm_v2f64_ty, llvm_i8_ty], [IntrNoMem]>;
+  def int_x86_avx512_mask_andn_pd_256 : GCCBuiltin<"__builtin_ia32_andnpd256_mask">,
+            Intrinsic<[llvm_v4f64_ty], [llvm_v4f64_ty, llvm_v4f64_ty,
+                      llvm_v4f64_ty, llvm_i8_ty], [IntrNoMem]>;
+  def int_x86_avx512_mask_andn_pd_512 : GCCBuiltin<"__builtin_ia32_andnpd512_mask">,
+          Intrinsic<[llvm_v8f64_ty], [llvm_v8f64_ty, llvm_v8f64_ty,
+                    llvm_v8f64_ty, llvm_i8_ty], [IntrNoMem]>;
+  def int_x86_avx512_mask_andn_ps_128 : GCCBuiltin<"__builtin_ia32_andnps128_mask">,
+            Intrinsic<[llvm_v4f32_ty], [llvm_v4f32_ty, llvm_v4f32_ty,
+                      llvm_v4f32_ty, llvm_i8_ty], [IntrNoMem]>;
+  def int_x86_avx512_mask_andn_ps_256 : GCCBuiltin<"__builtin_ia32_andnps256_mask">,
+            Intrinsic<[llvm_v8f32_ty], [llvm_v8f32_ty, llvm_v8f32_ty,
+                      llvm_v8f32_ty, llvm_i8_ty], [IntrNoMem]>;
+  def int_x86_avx512_mask_andn_ps_512 : GCCBuiltin<"__builtin_ia32_andnps512_mask">,
+          Intrinsic<[llvm_v16f32_ty], [llvm_v16f32_ty, llvm_v16f32_ty,
+                    llvm_v16f32_ty, llvm_i16_ty], [IntrNoMem]>;
+  def int_x86_avx512_mask_or_pd_128 : GCCBuiltin<"__builtin_ia32_orpd128_mask">,
+            Intrinsic<[llvm_v2f64_ty], [llvm_v2f64_ty, llvm_v2f64_ty,
+                      llvm_v2f64_ty, llvm_i8_ty], [IntrNoMem]>;
+  def int_x86_avx512_mask_or_pd_256 : GCCBuiltin<"__builtin_ia32_orpd256_mask">,
+            Intrinsic<[llvm_v4f64_ty], [llvm_v4f64_ty, llvm_v4f64_ty,
+                      llvm_v4f64_ty, llvm_i8_ty], [IntrNoMem]>;
+  def int_x86_avx512_mask_or_pd_512 : GCCBuiltin<"__builtin_ia32_orpd512_mask">,
+          Intrinsic<[llvm_v8f64_ty], [llvm_v8f64_ty, llvm_v8f64_ty,
+                    llvm_v8f64_ty, llvm_i8_ty], [IntrNoMem]>;
+  def int_x86_avx512_mask_or_ps_128 : GCCBuiltin<"__builtin_ia32_orps128_mask">,
+            Intrinsic<[llvm_v4f32_ty], [llvm_v4f32_ty, llvm_v4f32_ty,
+                      llvm_v4f32_ty, llvm_i8_ty], [IntrNoMem]>;
+  def int_x86_avx512_mask_or_ps_256 : GCCBuiltin<"__builtin_ia32_orps256_mask">,
+            Intrinsic<[llvm_v8f32_ty], [llvm_v8f32_ty, llvm_v8f32_ty,
+                      llvm_v8f32_ty, llvm_i8_ty], [IntrNoMem]>;
+  def int_x86_avx512_mask_or_ps_512 : GCCBuiltin<"__builtin_ia32_orps512_mask">,
+          Intrinsic<[llvm_v16f32_ty], [llvm_v16f32_ty, llvm_v16f32_ty,
+                    llvm_v16f32_ty, llvm_i16_ty], [IntrNoMem]>;
+  
+  def int_x86_avx512_mask_xor_pd_128 : GCCBuiltin<"__builtin_ia32_xorpd128_mask">,
+            Intrinsic<[llvm_v2f64_ty], [llvm_v2f64_ty, llvm_v2f64_ty,
+                      llvm_v2f64_ty, llvm_i8_ty], [IntrNoMem]>;
+  def int_x86_avx512_mask_xor_pd_256 : GCCBuiltin<"__builtin_ia32_xorpd256_mask">,
+            Intrinsic<[llvm_v4f64_ty], [llvm_v4f64_ty, llvm_v4f64_ty,
+                      llvm_v4f64_ty, llvm_i8_ty], [IntrNoMem]>;
+  def int_x86_avx512_mask_xor_pd_512 : GCCBuiltin<"__builtin_ia32_xorpd512_mask">,
+          Intrinsic<[llvm_v8f64_ty], [llvm_v8f64_ty, llvm_v8f64_ty,
+                    llvm_v8f64_ty, llvm_i8_ty], [IntrNoMem]>;
+  def int_x86_avx512_mask_xor_ps_128 : GCCBuiltin<"__builtin_ia32_xorps128_mask">,
+            Intrinsic<[llvm_v4f32_ty], [llvm_v4f32_ty, llvm_v4f32_ty,
+                      llvm_v4f32_ty, llvm_i8_ty], [IntrNoMem]>;
+  def int_x86_avx512_mask_xor_ps_256 : GCCBuiltin<"__builtin_ia32_xorps256_mask">,
+            Intrinsic<[llvm_v8f32_ty], [llvm_v8f32_ty, llvm_v8f32_ty,
+                      llvm_v8f32_ty, llvm_i8_ty], [IntrNoMem]>;
+  def int_x86_avx512_mask_xor_ps_512 : GCCBuiltin<"__builtin_ia32_xorps512_mask">,
+          Intrinsic<[llvm_v16f32_ty], [llvm_v16f32_ty, llvm_v16f32_ty,
+                    llvm_v16f32_ty, llvm_i16_ty], [IntrNoMem]>;      
+}
 // Integer arithmetic ops
 let TargetPrefix = "x86" in {
+  def int_x86_avx512_mask_padd_b_128 : GCCBuiltin<"__builtin_ia32_paddb128_mask">,
+          Intrinsic<[llvm_v16i8_ty], [llvm_v16i8_ty, llvm_v16i8_ty,
+                     llvm_v16i8_ty, llvm_i16_ty], [IntrNoMem]>;
+  def int_x86_avx512_mask_padd_b_256 : GCCBuiltin<"__builtin_ia32_paddb256_mask">,
+          Intrinsic<[llvm_v32i8_ty], [llvm_v32i8_ty, llvm_v32i8_ty,
+                     llvm_v32i8_ty, llvm_i32_ty], [IntrNoMem]>;
+  def int_x86_avx512_mask_padd_b_512 : GCCBuiltin<"__builtin_ia32_paddb512_mask">,
+          Intrinsic<[llvm_v64i8_ty], [llvm_v64i8_ty, llvm_v64i8_ty,
+                     llvm_v64i8_ty, llvm_i64_ty], [IntrNoMem]>;
+  def int_x86_avx512_mask_padd_w_128 : GCCBuiltin<"__builtin_ia32_paddw128_mask">,
+          Intrinsic<[llvm_v8i16_ty], [llvm_v8i16_ty, llvm_v8i16_ty,
+                     llvm_v8i16_ty, llvm_i8_ty], [IntrNoMem]>;
+  def int_x86_avx512_mask_padd_w_256 : GCCBuiltin<"__builtin_ia32_paddw256_mask">,
+          Intrinsic<[llvm_v16i16_ty], [llvm_v16i16_ty, llvm_v16i16_ty,
+                     llvm_v16i16_ty, llvm_i16_ty], [IntrNoMem]>;
+  def int_x86_avx512_mask_padd_w_512 : GCCBuiltin<"__builtin_ia32_paddw512_mask">,
+          Intrinsic<[llvm_v32i16_ty], [llvm_v32i16_ty, llvm_v32i16_ty,
+                     llvm_v32i16_ty, llvm_i32_ty], [IntrNoMem]>;
+  def int_x86_avx512_mask_padds_b_128 : GCCBuiltin<"__builtin_ia32_paddsb128_mask">,
+          Intrinsic<[llvm_v16i8_ty], [llvm_v16i8_ty, llvm_v16i8_ty,
+                     llvm_v16i8_ty, llvm_i16_ty], [IntrNoMem]>;
+  def int_x86_avx512_mask_padds_b_256 : GCCBuiltin<"__builtin_ia32_paddsb256_mask">,
+          Intrinsic<[llvm_v32i8_ty], [llvm_v32i8_ty, llvm_v32i8_ty,
+                     llvm_v32i8_ty, llvm_i32_ty], [IntrNoMem]>;
+  def int_x86_avx512_mask_padds_b_512 : GCCBuiltin<"__builtin_ia32_paddsb512_mask">,
+          Intrinsic<[llvm_v64i8_ty], [llvm_v64i8_ty, llvm_v64i8_ty,
+                     llvm_v64i8_ty, llvm_i64_ty], [IntrNoMem]>;
+  def int_x86_avx512_mask_padds_w_128 : GCCBuiltin<"__builtin_ia32_paddsw128_mask">,
+          Intrinsic<[llvm_v8i16_ty], [llvm_v8i16_ty, llvm_v8i16_ty,
+                     llvm_v8i16_ty, llvm_i8_ty], [IntrNoMem]>;
+  def int_x86_avx512_mask_padds_w_256 : GCCBuiltin<"__builtin_ia32_paddsw256_mask">,
+          Intrinsic<[llvm_v16i16_ty], [llvm_v16i16_ty, llvm_v16i16_ty,
+                     llvm_v16i16_ty, llvm_i16_ty], [IntrNoMem]>;
+  def int_x86_avx512_mask_padds_w_512 : GCCBuiltin<"__builtin_ia32_paddsw512_mask">,
+          Intrinsic<[llvm_v32i16_ty], [llvm_v32i16_ty, llvm_v32i16_ty,
+                     llvm_v32i16_ty, llvm_i32_ty], [IntrNoMem]>;
+  def int_x86_avx512_mask_paddus_b_128 : GCCBuiltin<"__builtin_ia32_paddusb128_mask">,
+          Intrinsic<[llvm_v16i8_ty], [llvm_v16i8_ty, llvm_v16i8_ty,
+                     llvm_v16i8_ty, llvm_i16_ty], [IntrNoMem]>;
+  def int_x86_avx512_mask_paddus_b_256 : GCCBuiltin<"__builtin_ia32_paddusb256_mask">,
+          Intrinsic<[llvm_v32i8_ty], [llvm_v32i8_ty, llvm_v32i8_ty,
+                     llvm_v32i8_ty, llvm_i32_ty], [IntrNoMem]>;
+  def int_x86_avx512_mask_paddus_b_512 : GCCBuiltin<"__builtin_ia32_paddusb512_mask">,
+          Intrinsic<[llvm_v64i8_ty], [llvm_v64i8_ty, llvm_v64i8_ty,
+                     llvm_v64i8_ty, llvm_i64_ty], [IntrNoMem]>;
+  def int_x86_avx512_mask_paddus_w_128 : GCCBuiltin<"__builtin_ia32_paddusw128_mask">,
+          Intrinsic<[llvm_v8i16_ty], [llvm_v8i16_ty, llvm_v8i16_ty,
+                     llvm_v8i16_ty, llvm_i8_ty], [IntrNoMem]>;
+  def int_x86_avx512_mask_paddus_w_256 : GCCBuiltin<"__builtin_ia32_paddusw256_mask">,
+          Intrinsic<[llvm_v16i16_ty], [llvm_v16i16_ty, llvm_v16i16_ty,
+                     llvm_v16i16_ty, llvm_i16_ty], [IntrNoMem]>;
+  def int_x86_avx512_mask_paddus_w_512 : GCCBuiltin<"__builtin_ia32_paddusw512_mask">,
+          Intrinsic<[llvm_v32i16_ty], [llvm_v32i16_ty, llvm_v32i16_ty,
+                     llvm_v32i16_ty, llvm_i32_ty], [IntrNoMem]>;
+  def int_x86_avx512_mask_padd_d_128 : GCCBuiltin<"__builtin_ia32_paddd128_mask">,
+          Intrinsic<[llvm_v4i32_ty], [llvm_v4i32_ty, llvm_v4i32_ty,
+                     llvm_v4i32_ty, llvm_i8_ty], [IntrNoMem]>;
+  def int_x86_avx512_mask_padd_d_256 : GCCBuiltin<"__builtin_ia32_paddd256_mask">,
+          Intrinsic<[llvm_v8i32_ty], [llvm_v8i32_ty, llvm_v8i32_ty,
+                     llvm_v8i32_ty, llvm_i8_ty], [IntrNoMem]>;
   def int_x86_avx512_mask_padd_d_512 : GCCBuiltin<"__builtin_ia32_paddd512_mask">,
           Intrinsic<[llvm_v16i32_ty], [llvm_v16i32_ty, llvm_v16i32_ty,
                      llvm_v16i32_ty, llvm_i16_ty], [IntrNoMem]>;
+  def int_x86_avx512_mask_padd_q_128 : GCCBuiltin<"__builtin_ia32_paddq128_mask">,
+          Intrinsic<[llvm_v2i64_ty], [llvm_v2i64_ty, llvm_v2i64_ty,
+                     llvm_v2i64_ty, llvm_i8_ty], [IntrNoMem]>;
+  def int_x86_avx512_mask_padd_q_256 : GCCBuiltin<"__builtin_ia32_paddq256_mask">,
+          Intrinsic<[llvm_v4i64_ty], [llvm_v4i64_ty, llvm_v4i64_ty,
+                     llvm_v4i64_ty, llvm_i8_ty], [IntrNoMem]>;
   def int_x86_avx512_mask_padd_q_512 : GCCBuiltin<"__builtin_ia32_paddq512_mask">,
           Intrinsic<[llvm_v8i64_ty], [llvm_v8i64_ty, llvm_v8i64_ty,
                      llvm_v8i64_ty, llvm_i8_ty], [IntrNoMem]>;
+  def int_x86_avx512_mask_psub_b_128 : GCCBuiltin<"__builtin_ia32_psubb128_mask">,
+          Intrinsic<[llvm_v16i8_ty], [llvm_v16i8_ty, llvm_v16i8_ty,
+                     llvm_v16i8_ty, llvm_i16_ty], [IntrNoMem]>;
+  def int_x86_avx512_mask_psub_b_256 : GCCBuiltin<"__builtin_ia32_psubb256_mask">,
+          Intrinsic<[llvm_v32i8_ty], [llvm_v32i8_ty, llvm_v32i8_ty,
+                     llvm_v32i8_ty, llvm_i32_ty], [IntrNoMem]>;
+  def int_x86_avx512_mask_psub_b_512 : GCCBuiltin<"__builtin_ia32_psubb512_mask">,
+          Intrinsic<[llvm_v64i8_ty], [llvm_v64i8_ty, llvm_v64i8_ty,
+                     llvm_v64i8_ty, llvm_i64_ty], [IntrNoMem]>;
+  def int_x86_avx512_mask_psub_w_128 : GCCBuiltin<"__builtin_ia32_psubw128_mask">,
+          Intrinsic<[llvm_v8i16_ty], [llvm_v8i16_ty, llvm_v8i16_ty,
+                     llvm_v8i16_ty, llvm_i8_ty], [IntrNoMem]>;
+  def int_x86_avx512_mask_psub_w_256 : GCCBuiltin<"__builtin_ia32_psubw256_mask">,
+          Intrinsic<[llvm_v16i16_ty], [llvm_v16i16_ty, llvm_v16i16_ty,
+                     llvm_v16i16_ty, llvm_i16_ty], [IntrNoMem]>;
+  def int_x86_avx512_mask_psub_w_512 : GCCBuiltin<"__builtin_ia32_psubw512_mask">,
+          Intrinsic<[llvm_v32i16_ty], [llvm_v32i16_ty, llvm_v32i16_ty,
+                     llvm_v32i16_ty, llvm_i32_ty], [IntrNoMem]>;
+  def int_x86_avx512_mask_psubs_b_128 : GCCBuiltin<"__builtin_ia32_psubsb128_mask">,
+          Intrinsic<[llvm_v16i8_ty], [llvm_v16i8_ty, llvm_v16i8_ty,
+                     llvm_v16i8_ty, llvm_i16_ty], [IntrNoMem]>;
+  def int_x86_avx512_mask_psubs_b_256 : GCCBuiltin<"__builtin_ia32_psubsb256_mask">,
+          Intrinsic<[llvm_v32i8_ty], [llvm_v32i8_ty, llvm_v32i8_ty,
+                     llvm_v32i8_ty, llvm_i32_ty], [IntrNoMem]>;
+  def int_x86_avx512_mask_psubs_b_512 : GCCBuiltin<"__builtin_ia32_psubsb512_mask">,
+          Intrinsic<[llvm_v64i8_ty], [llvm_v64i8_ty, llvm_v64i8_ty,
+                     llvm_v64i8_ty, llvm_i64_ty], [IntrNoMem]>;
+  def int_x86_avx512_mask_psubs_w_128 : GCCBuiltin<"__builtin_ia32_psubsw128_mask">,
+          Intrinsic<[llvm_v8i16_ty], [llvm_v8i16_ty, llvm_v8i16_ty,
+                     llvm_v8i16_ty, llvm_i8_ty], [IntrNoMem]>;
+  def int_x86_avx512_mask_psubs_w_256 : GCCBuiltin<"__builtin_ia32_psubsw256_mask">,
+          Intrinsic<[llvm_v16i16_ty], [llvm_v16i16_ty, llvm_v16i16_ty,
+                     llvm_v16i16_ty, llvm_i16_ty], [IntrNoMem]>;
+  def int_x86_avx512_mask_psubs_w_512 : GCCBuiltin<"__builtin_ia32_psubsw512_mask">,
+          Intrinsic<[llvm_v32i16_ty], [llvm_v32i16_ty, llvm_v32i16_ty,
+                     llvm_v32i16_ty, llvm_i32_ty], [IntrNoMem]>;
+  def int_x86_avx512_mask_psubus_b_128 : GCCBuiltin<"__builtin_ia32_psubusb128_mask">,
+          Intrinsic<[llvm_v16i8_ty], [llvm_v16i8_ty, llvm_v16i8_ty,
+                     llvm_v16i8_ty, llvm_i16_ty], [IntrNoMem]>;
+  def int_x86_avx512_mask_psubus_b_256 : GCCBuiltin<"__builtin_ia32_psubusb256_mask">,
+          Intrinsic<[llvm_v32i8_ty], [llvm_v32i8_ty, llvm_v32i8_ty,
+                     llvm_v32i8_ty, llvm_i32_ty], [IntrNoMem]>;
+  def int_x86_avx512_mask_psubus_b_512 : GCCBuiltin<"__builtin_ia32_psubusb512_mask">,
+          Intrinsic<[llvm_v64i8_ty], [llvm_v64i8_ty, llvm_v64i8_ty,
+                     llvm_v64i8_ty, llvm_i64_ty], [IntrNoMem]>;
+  def int_x86_avx512_mask_psubus_w_128 : GCCBuiltin<"__builtin_ia32_psubusw128_mask">,
+          Intrinsic<[llvm_v8i16_ty], [llvm_v8i16_ty, llvm_v8i16_ty,
+                     llvm_v8i16_ty, llvm_i8_ty], [IntrNoMem]>;
+  def int_x86_avx512_mask_psubus_w_256 : GCCBuiltin<"__builtin_ia32_psubusw256_mask">,
+          Intrinsic<[llvm_v16i16_ty], [llvm_v16i16_ty, llvm_v16i16_ty,
+                     llvm_v16i16_ty, llvm_i16_ty], [IntrNoMem]>;
+  def int_x86_avx512_mask_psubus_w_512 : GCCBuiltin<"__builtin_ia32_psubusw512_mask">,
+          Intrinsic<[llvm_v32i16_ty], [llvm_v32i16_ty, llvm_v32i16_ty,
+                     llvm_v32i16_ty, llvm_i32_ty], [IntrNoMem]>;
+  def int_x86_avx512_mask_psub_d_128 : GCCBuiltin<"__builtin_ia32_psubd128_mask">,
+          Intrinsic<[llvm_v4i32_ty], [llvm_v4i32_ty, llvm_v4i32_ty,
+                     llvm_v4i32_ty, llvm_i8_ty], [IntrNoMem]>;
+  def int_x86_avx512_mask_psub_d_256 : GCCBuiltin<"__builtin_ia32_psubd256_mask">,
+          Intrinsic<[llvm_v8i32_ty], [llvm_v8i32_ty, llvm_v8i32_ty,
+                     llvm_v8i32_ty, llvm_i8_ty], [IntrNoMem]>;
   def int_x86_avx512_mask_psub_d_512 : GCCBuiltin<"__builtin_ia32_psubd512_mask">,
           Intrinsic<[llvm_v16i32_ty], [llvm_v16i32_ty, llvm_v16i32_ty,
                      llvm_v16i32_ty, llvm_i16_ty], [IntrNoMem]>;
+  def int_x86_avx512_mask_psub_q_128 : GCCBuiltin<"__builtin_ia32_psubq128_mask">,
+          Intrinsic<[llvm_v2i64_ty], [llvm_v2i64_ty, llvm_v2i64_ty,
+                     llvm_v2i64_ty, llvm_i8_ty], [IntrNoMem]>;
+  def int_x86_avx512_mask_psub_q_256 : GCCBuiltin<"__builtin_ia32_psubq256_mask">,
+          Intrinsic<[llvm_v4i64_ty], [llvm_v4i64_ty, llvm_v4i64_ty,
+                     llvm_v4i64_ty, llvm_i8_ty], [IntrNoMem]>;
   def int_x86_avx512_mask_psub_q_512 : GCCBuiltin<"__builtin_ia32_psubq512_mask">,
           Intrinsic<[llvm_v8i64_ty], [llvm_v8i64_ty, llvm_v8i64_ty,
                      llvm_v8i64_ty, llvm_i8_ty], [IntrNoMem]>;
+  def int_x86_avx512_mask_pmulu_dq_128 : GCCBuiltin<"__builtin_ia32_pmuludq128_mask">,
+              Intrinsic<[llvm_v2i64_ty], [llvm_v4i32_ty, llvm_v4i32_ty,
+                         llvm_v2i64_ty, llvm_i8_ty], [IntrNoMem]>;
+  def int_x86_avx512_mask_pmul_dq_128 : GCCBuiltin<"__builtin_ia32_pmuldq128_mask">,
+              Intrinsic<[llvm_v2i64_ty], [llvm_v4i32_ty, llvm_v4i32_ty,
+                         llvm_v2i64_ty, llvm_i8_ty], [IntrNoMem]>;
+  def int_x86_avx512_mask_pmulu_dq_256 : GCCBuiltin<"__builtin_ia32_pmuludq256_mask">,
+              Intrinsic<[llvm_v4i64_ty], [llvm_v8i32_ty, llvm_v8i32_ty,
+                         llvm_v4i64_ty, llvm_i8_ty], [IntrNoMem]>;
+  def int_x86_avx512_mask_pmul_dq_256 : GCCBuiltin<"__builtin_ia32_pmuldq256_mask">,
+              Intrinsic<[llvm_v4i64_ty], [llvm_v8i32_ty, llvm_v8i32_ty,
+                         llvm_v4i64_ty, llvm_i8_ty], [IntrNoMem]>;
   def int_x86_avx512_mask_pmulu_dq_512 : GCCBuiltin<"__builtin_ia32_pmuludq512_mask">,
               Intrinsic<[llvm_v8i64_ty], [llvm_v16i32_ty, llvm_v16i32_ty,
                          llvm_v8i64_ty, llvm_i8_ty], [IntrNoMem]>;
   def int_x86_avx512_mask_pmul_dq_512 : GCCBuiltin<"__builtin_ia32_pmuldq512_mask">,
               Intrinsic<[llvm_v8i64_ty], [llvm_v16i32_ty, llvm_v16i32_ty,
                          llvm_v8i64_ty, llvm_i8_ty], [IntrNoMem]>;
+  def int_x86_avx512_mask_pmull_w_128 : GCCBuiltin<"__builtin_ia32_pmullw128_mask">,
+              Intrinsic<[llvm_v8i16_ty], [llvm_v8i16_ty, llvm_v8i16_ty,
+                     llvm_v8i16_ty, llvm_i8_ty], [IntrNoMem]>;
+  def int_x86_avx512_mask_pmull_w_256 : GCCBuiltin<"__builtin_ia32_pmullw256_mask">,
+              Intrinsic<[llvm_v16i16_ty], [llvm_v16i16_ty, llvm_v16i16_ty,
+                     llvm_v16i16_ty, llvm_i16_ty], [IntrNoMem]>;
+  def int_x86_avx512_mask_pmull_w_512 : GCCBuiltin<"__builtin_ia32_pmullw512_mask">,
+              Intrinsic<[llvm_v32i16_ty], [llvm_v32i16_ty, llvm_v32i16_ty,
+                     llvm_v32i16_ty, llvm_i32_ty], [IntrNoMem]>;
+  def int_x86_avx512_mask_pmull_d_128 : GCCBuiltin<"__builtin_ia32_pmulld128_mask">,
+              Intrinsic<[llvm_v4i32_ty], [llvm_v4i32_ty, llvm_v4i32_ty,
+                     llvm_v4i32_ty, llvm_i8_ty], [IntrNoMem]>;
+  def int_x86_avx512_mask_pmull_d_256 : GCCBuiltin<"__builtin_ia32_pmulld256_mask">,
+              Intrinsic<[llvm_v8i32_ty], [llvm_v8i32_ty, llvm_v8i32_ty,
+                     llvm_v8i32_ty, llvm_i8_ty], [IntrNoMem]>;
+  def int_x86_avx512_mask_pmull_d_512 : GCCBuiltin<"__builtin_ia32_pmulld512_mask">,
+              Intrinsic<[llvm_v16i32_ty], [llvm_v16i32_ty, llvm_v16i32_ty,
+                     llvm_v16i32_ty, llvm_i16_ty], [IntrNoMem]>;
+  def int_x86_avx512_mask_pmull_q_128 : GCCBuiltin<"__builtin_ia32_pmullq128_mask">,
+              Intrinsic<[llvm_v2i64_ty], [llvm_v2i64_ty, llvm_v2i64_ty,
+                     llvm_v2i64_ty, llvm_i8_ty], [IntrNoMem]>;
+  def int_x86_avx512_mask_pmull_q_256 : GCCBuiltin<"__builtin_ia32_pmullq256_mask">,
+              Intrinsic<[llvm_v4i64_ty], [llvm_v4i64_ty, llvm_v4i64_ty,
+                     llvm_v4i64_ty, llvm_i8_ty], [IntrNoMem]>;
+  def int_x86_avx512_mask_pmull_q_512 : GCCBuiltin<"__builtin_ia32_pmullq512_mask">,
+              Intrinsic<[llvm_v8i64_ty], [llvm_v8i64_ty, llvm_v8i64_ty,
+                     llvm_v8i64_ty, llvm_i8_ty], [IntrNoMem]>;
+  def int_x86_avx512_mask_pmulhu_w_512 : GCCBuiltin<"__builtin_ia32_pmulhuw512_mask">,
+              Intrinsic<[llvm_v32i16_ty], [llvm_v32i16_ty, llvm_v32i16_ty,
+                     llvm_v32i16_ty, llvm_i32_ty], [IntrNoMem]>;
+  def int_x86_avx512_mask_pmulh_w_512 : GCCBuiltin<"__builtin_ia32_pmulhw512_mask">,
+              Intrinsic<[llvm_v32i16_ty], [llvm_v32i16_ty, llvm_v32i16_ty,
+                     llvm_v32i16_ty, llvm_i32_ty], [IntrNoMem]>;
+  def int_x86_avx512_mask_pmulhu_w_128 : GCCBuiltin<"__builtin_ia32_pmulhuw128_mask">,
+              Intrinsic<[llvm_v8i16_ty], [llvm_v8i16_ty, llvm_v8i16_ty, 
+                     llvm_v8i16_ty, llvm_i8_ty], [IntrNoMem]>;
+  def int_x86_avx512_mask_pmulhu_w_256 : GCCBuiltin<"__builtin_ia32_pmulhuw256_mask">,
+              Intrinsic<[llvm_v16i16_ty], [llvm_v16i16_ty, llvm_v16i16_ty, 
+                     llvm_v16i16_ty, llvm_i16_ty], [IntrNoMem]>;
+  def int_x86_avx512_mask_pmulh_w_128 : GCCBuiltin<"__builtin_ia32_pmulhw128_mask">,
+              Intrinsic<[llvm_v8i16_ty], [llvm_v8i16_ty, llvm_v8i16_ty, 
+                     llvm_v8i16_ty, llvm_i8_ty], [IntrNoMem]>;
+  def int_x86_avx512_mask_pmulh_w_256 : GCCBuiltin<"__builtin_ia32_pmulhw256_mask">,
+              Intrinsic<[llvm_v16i16_ty], [llvm_v16i16_ty, llvm_v16i16_ty, 
+                     llvm_v16i16_ty, llvm_i16_ty], [IntrNoMem]>;
+  def int_x86_avx512_mask_pavg_b_512 : GCCBuiltin<"__builtin_ia32_pavgb512_mask">,
+          Intrinsic<[llvm_v64i8_ty], [llvm_v64i8_ty, llvm_v64i8_ty, 
+                    llvm_v64i8_ty, llvm_i64_ty], [IntrNoMem]>;
+  def int_x86_avx512_mask_pavg_w_512 : GCCBuiltin<"__builtin_ia32_pavgw512_mask">,
+          Intrinsic<[llvm_v32i16_ty], [llvm_v32i16_ty, llvm_v32i16_ty,  
+                    llvm_v32i16_ty, llvm_i32_ty], [IntrNoMem]>;
+  def int_x86_avx512_mask_pavg_b_128 : GCCBuiltin<"__builtin_ia32_pavgb128_mask">,
+          Intrinsic<[llvm_v16i8_ty], [llvm_v16i8_ty, llvm_v16i8_ty,  
+                    llvm_v16i8_ty, llvm_i16_ty], [IntrNoMem]>;
+  def int_x86_avx512_mask_pavg_b_256 : GCCBuiltin<"__builtin_ia32_pavgb256_mask">,
+          Intrinsic<[llvm_v32i8_ty], [llvm_v32i8_ty, llvm_v32i8_ty,  
+                    llvm_v32i8_ty, llvm_i32_ty], [IntrNoMem]>;
+  def int_x86_avx512_mask_pavg_w_128 : GCCBuiltin<"__builtin_ia32_pavgw128_mask">,
+          Intrinsic<[llvm_v8i16_ty], [llvm_v8i16_ty, llvm_v8i16_ty,  
+                    llvm_v8i16_ty, llvm_i8_ty], [IntrNoMem]>;
+  def int_x86_avx512_mask_pavg_w_256 : GCCBuiltin<"__builtin_ia32_pavgw256_mask">,
+          Intrinsic<[llvm_v16i16_ty], [llvm_v16i16_ty, llvm_v16i16_ty, 
+                    llvm_v16i16_ty, llvm_i16_ty], [IntrNoMem]>;
+  def int_x86_avx512_mask_pmaddw_d_128 :
+         GCCBuiltin<"__builtin_ia32_pmaddwd128_mask">,
+          Intrinsic<[llvm_v4i32_ty],
+          [llvm_v8i16_ty, llvm_v8i16_ty, llvm_v4i32_ty,  llvm_i8_ty],
+          [IntrNoMem]>;
+  def int_x86_avx512_mask_pmaddw_d_256 :
+         GCCBuiltin<"__builtin_ia32_pmaddwd256_mask">,
+          Intrinsic<[llvm_v8i32_ty],
+          [llvm_v16i16_ty, llvm_v16i16_ty, llvm_v8i32_ty,  llvm_i8_ty],
+          [IntrNoMem]>;
+  def int_x86_avx512_mask_pmaddw_d_512 :
+         GCCBuiltin<"__builtin_ia32_pmaddwd512_mask">,
+          Intrinsic<[llvm_v16i32_ty],
+          [llvm_v32i16_ty, llvm_v32i16_ty, llvm_v16i32_ty,  llvm_i16_ty],
+          [IntrNoMem]>;
+  def int_x86_avx512_mask_pmaddubs_w_128 :
+         GCCBuiltin<"__builtin_ia32_pmaddubsw128_mask">,
+          Intrinsic<[llvm_v8i16_ty],
+          [llvm_v16i8_ty, llvm_v16i8_ty, llvm_v8i16_ty,  llvm_i8_ty],
+          [IntrNoMem]>;
+  def int_x86_avx512_mask_pmaddubs_w_256 :
+         GCCBuiltin<"__builtin_ia32_pmaddubsw256_mask">,
+          Intrinsic<[llvm_v16i16_ty],
+          [llvm_v32i8_ty, llvm_v32i8_ty, llvm_v16i16_ty,  llvm_i16_ty],
+          [IntrNoMem]>;
+  def int_x86_avx512_mask_pmaddubs_w_512 :
+         GCCBuiltin<"__builtin_ia32_pmaddubsw512_mask">,
+          Intrinsic<[llvm_v32i16_ty],
+          [llvm_v64i8_ty, llvm_v64i8_ty, llvm_v32i16_ty,  llvm_i32_ty],
+          [IntrNoMem]>;
 }
+
 // Gather and Scatter ops
 let TargetPrefix = "x86" in {
   def int_x86_avx512_gather_dpd_512  : GCCBuiltin<"__builtin_ia32_gathersiv8df">,
@@ -3294,6 +5304,102 @@ let TargetPrefix = "x86" in {
                      llvm_v8i64_ty, llvm_i8_ty, llvm_i32_ty],
                     [IntrReadArgMem]>;
 
+  def int_x86_avx512_gather3div2_df : 
+        GCCBuiltin<"__builtin_ia32_gather3div2df">,
+          Intrinsic<[llvm_v2f64_ty],
+          [llvm_v2f64_ty, llvm_ptr_ty, llvm_v2i64_ty, llvm_i8_ty, llvm_i32_ty],
+          [IntrReadArgMem]>;
+
+  def int_x86_avx512_gather3div2_di : 
+        GCCBuiltin<"__builtin_ia32_gather3div2di">,
+          Intrinsic<[llvm_v4i32_ty],
+          [llvm_v2i64_ty, llvm_ptr_ty, llvm_v2i64_ty, llvm_i8_ty, llvm_i32_ty],
+          [IntrReadArgMem]>;
+
+  def int_x86_avx512_gather3div4_df : 
+        GCCBuiltin<"__builtin_ia32_gather3div4df">,
+          Intrinsic<[llvm_v4f64_ty],
+          [llvm_v4f64_ty, llvm_ptr_ty, llvm_v4i64_ty, llvm_i8_ty, llvm_i32_ty],
+          [IntrReadArgMem]>;
+
+  def int_x86_avx512_gather3div4_di : 
+        GCCBuiltin<"__builtin_ia32_gather3div4di">,
+          Intrinsic<[llvm_v8i32_ty],
+          [llvm_v4i64_ty, llvm_ptr_ty, llvm_v4i64_ty, llvm_i8_ty, llvm_i32_ty],
+          [IntrReadArgMem]>;
+
+  def int_x86_avx512_gather3div4_sf : 
+        GCCBuiltin<"__builtin_ia32_gather3div4sf">,
+          Intrinsic<[llvm_v4f32_ty],
+          [llvm_v4f32_ty, llvm_ptr_ty, llvm_v2i64_ty, llvm_i8_ty, llvm_i32_ty],
+          [IntrReadArgMem]>;
+
+  def int_x86_avx512_gather3div4_si : 
+        GCCBuiltin<"__builtin_ia32_gather3div4si">,
+          Intrinsic<[llvm_v4i32_ty],
+          [llvm_v4i32_ty, llvm_ptr_ty, llvm_v2i64_ty, llvm_i8_ty, llvm_i32_ty],
+          [IntrReadArgMem]>;
+
+  def int_x86_avx512_gather3div8_sf : 
+        GCCBuiltin<"__builtin_ia32_gather3div8sf">,
+          Intrinsic<[llvm_v4f32_ty],
+          [llvm_v4f32_ty, llvm_ptr_ty, llvm_v4i64_ty, llvm_i8_ty, llvm_i32_ty],
+          [IntrReadArgMem]>;
+
+  def int_x86_avx512_gather3div8_si : 
+        GCCBuiltin<"__builtin_ia32_gather3div8si">,
+          Intrinsic<[llvm_v4i32_ty],
+          [llvm_v4i32_ty, llvm_ptr_ty, llvm_v4i64_ty, llvm_i8_ty, llvm_i32_ty],
+          [IntrReadArgMem]>;
+
+  def int_x86_avx512_gather3siv2_df : 
+        GCCBuiltin<"__builtin_ia32_gather3siv2df">,
+          Intrinsic<[llvm_v2f64_ty],
+          [llvm_v2f64_ty, llvm_ptr_ty, llvm_v4i32_ty, llvm_i8_ty, llvm_i32_ty],
+          [IntrReadArgMem]>;
+
+  def int_x86_avx512_gather3siv2_di : 
+        GCCBuiltin<"__builtin_ia32_gather3siv2di">,
+          Intrinsic<[llvm_v4i32_ty],
+          [llvm_v2i64_ty, llvm_ptr_ty, llvm_v4i32_ty, llvm_i8_ty, llvm_i32_ty],
+          [IntrReadArgMem]>;
+
+  def int_x86_avx512_gather3siv4_df : 
+        GCCBuiltin<"__builtin_ia32_gather3siv4df">,
+          Intrinsic<[llvm_v4f64_ty],
+          [llvm_v4f64_ty, llvm_ptr_ty, llvm_v4i32_ty, llvm_i8_ty, llvm_i32_ty],
+          [IntrReadArgMem]>;
+
+  def int_x86_avx512_gather3siv4_di : 
+        GCCBuiltin<"__builtin_ia32_gather3siv4di">,
+          Intrinsic<[llvm_v8i32_ty],
+          [llvm_v4i64_ty, llvm_ptr_ty, llvm_v4i32_ty, llvm_i8_ty, llvm_i32_ty],
+          [IntrReadArgMem]>;
+
+  def int_x86_avx512_gather3siv4_sf : 
+        GCCBuiltin<"__builtin_ia32_gather3siv4sf">,
+          Intrinsic<[llvm_v4f32_ty],
+          [llvm_v4f32_ty, llvm_ptr_ty, llvm_v4i32_ty, llvm_i8_ty, llvm_i32_ty],
+          [IntrReadArgMem]>;
+
+  def int_x86_avx512_gather3siv4_si : 
+        GCCBuiltin<"__builtin_ia32_gather3siv4si">,
+          Intrinsic<[llvm_v4i32_ty],
+          [llvm_v4i32_ty, llvm_ptr_ty, llvm_v4i32_ty, llvm_i8_ty, llvm_i32_ty],
+          [IntrReadArgMem]>;
+
+  def int_x86_avx512_gather3siv8_sf : 
+        GCCBuiltin<"__builtin_ia32_gather3siv8sf">,
+          Intrinsic<[llvm_v8f32_ty],
+          [llvm_v8f32_ty, llvm_ptr_ty, llvm_v8i32_ty, llvm_i8_ty, llvm_i32_ty],
+          [IntrReadArgMem]>;
+
+  def int_x86_avx512_gather3siv8_si : 
+        GCCBuiltin<"__builtin_ia32_gather3siv8si">,
+          Intrinsic<[llvm_v8i32_ty],
+          [llvm_v8i32_ty, llvm_ptr_ty, llvm_v8i32_ty, llvm_i8_ty, llvm_i32_ty],
+          [IntrReadArgMem]>;
+
 // scatter
   def int_x86_avx512_scatter_dpd_512  : GCCBuiltin<"__builtin_ia32_scattersiv8df">,
           Intrinsic<[], [llvm_ptr_ty, llvm_i8_ty,
@@ -3330,6 +5436,102 @@ let TargetPrefix = "x86" in {
                          llvm_i32_ty],
                     [IntrReadWriteArgMem]>;
 
+  def int_x86_avx512_scatterdiv2_df : 
+       GCCBuiltin<"__builtin_ia32_scatterdiv2df">,
+        Intrinsic<[],
+        [llvm_ptr_ty, llvm_i8_ty, llvm_v2i64_ty, llvm_v2f64_ty, llvm_i32_ty],
+        [IntrReadWriteArgMem]>;
+
+  def int_x86_avx512_scatterdiv2_di : 
+        GCCBuiltin<"__builtin_ia32_scatterdiv2di">,
+          Intrinsic<[],
+          [llvm_ptr_ty, llvm_i8_ty, llvm_v2i64_ty, llvm_v2i64_ty, llvm_i32_ty],
+          [IntrReadWriteArgMem]>;
+
+  def int_x86_avx512_scatterdiv4_df : 
+        GCCBuiltin<"__builtin_ia32_scatterdiv4df">,
+          Intrinsic<[],
+          [llvm_ptr_ty, llvm_i8_ty, llvm_v4i64_ty, llvm_v4f64_ty, llvm_i32_ty],
+          [IntrReadWriteArgMem]>;
+
+  def int_x86_avx512_scatterdiv4_di : 
+        GCCBuiltin<"__builtin_ia32_scatterdiv4di">,
+          Intrinsic<[],
+          [llvm_ptr_ty, llvm_i8_ty, llvm_v4i64_ty, llvm_v4i64_ty, llvm_i32_ty],
+          [IntrReadWriteArgMem]>;
+
+  def int_x86_avx512_scatterdiv4_sf : 
+        GCCBuiltin<"__builtin_ia32_scatterdiv4sf">,
+          Intrinsic<[],
+          [llvm_ptr_ty, llvm_i8_ty, llvm_v2i64_ty, llvm_v4f32_ty, llvm_i32_ty],
+          [IntrReadWriteArgMem]>;
+
+  def int_x86_avx512_scatterdiv4_si : 
+        GCCBuiltin<"__builtin_ia32_scatterdiv4si">,
+          Intrinsic<[],
+          [llvm_ptr_ty, llvm_i8_ty, llvm_v2i64_ty, llvm_v4i32_ty, llvm_i32_ty],
+          [IntrReadWriteArgMem]>;
+
+  def int_x86_avx512_scatterdiv8_sf : 
+        GCCBuiltin<"__builtin_ia32_scatterdiv8sf">,
+          Intrinsic<[],
+          [llvm_ptr_ty, llvm_i8_ty, llvm_v4i64_ty, llvm_v4f32_ty, llvm_i32_ty],
+          [IntrReadWriteArgMem]>;
+
+  def int_x86_avx512_scatterdiv8_si : 
+        GCCBuiltin<"__builtin_ia32_scatterdiv8si">,
+          Intrinsic<[],
+          [llvm_ptr_ty, llvm_i8_ty, llvm_v4i64_ty, llvm_v4i32_ty, llvm_i32_ty],
+          [IntrReadWriteArgMem]>;
+
+  def int_x86_avx512_scattersiv2_df : 
+        GCCBuiltin<"__builtin_ia32_scattersiv2df">,
+          Intrinsic<[],
+          [llvm_ptr_ty, llvm_i8_ty, llvm_v4i32_ty, llvm_v2f64_ty, llvm_i32_ty],
+          [IntrReadWriteArgMem]>;
+
+  def int_x86_avx512_scattersiv2_di : 
+        GCCBuiltin<"__builtin_ia32_scattersiv2di">,
+          Intrinsic<[],
+          [llvm_ptr_ty, llvm_i8_ty, llvm_v4i32_ty, llvm_v2i64_ty, llvm_i32_ty],
+          [IntrReadWriteArgMem]>;
+
+  def int_x86_avx512_scattersiv4_df : 
+        GCCBuiltin<"__builtin_ia32_scattersiv4df">,
+          Intrinsic<[],
+          [llvm_ptr_ty, llvm_i8_ty, llvm_v4i32_ty, llvm_v4f64_ty, llvm_i32_ty],
+          [IntrReadWriteArgMem]>;
+
+  def int_x86_avx512_scattersiv4_di : 
+        GCCBuiltin<"__builtin_ia32_scattersiv4di">,
+          Intrinsic<[],
+          [llvm_ptr_ty, llvm_i8_ty, llvm_v4i32_ty, llvm_v4i64_ty, llvm_i32_ty],
+          [IntrReadWriteArgMem]>;
+
+  def int_x86_avx512_scattersiv4_sf : 
+        GCCBuiltin<"__builtin_ia32_scattersiv4sf">,
+          Intrinsic<[],
+          [llvm_ptr_ty, llvm_i8_ty, llvm_v4i32_ty, llvm_v4f32_ty, llvm_i32_ty],
+          [IntrReadWriteArgMem]>;
+
+  def int_x86_avx512_scattersiv4_si : 
+        GCCBuiltin<"__builtin_ia32_scattersiv4si">,
+          Intrinsic<[],
+          [llvm_ptr_ty, llvm_i8_ty, llvm_v4i32_ty, llvm_v4i32_ty, llvm_i32_ty],
+          [IntrReadWriteArgMem]>;
+
+  def int_x86_avx512_scattersiv8_sf : 
+        GCCBuiltin<"__builtin_ia32_scattersiv8sf">,
+          Intrinsic<[],
+          [llvm_ptr_ty, llvm_i8_ty, llvm_v8i32_ty, llvm_v8f32_ty, llvm_i32_ty],
+          [IntrReadWriteArgMem]>;
+
+  def int_x86_avx512_scattersiv8_si : 
+        GCCBuiltin<"__builtin_ia32_scattersiv8si">,
+          Intrinsic<[],
+          [llvm_ptr_ty, llvm_i8_ty, llvm_v8i32_ty, llvm_v8i32_ty, llvm_i32_ty],
+          [IntrReadWriteArgMem]>;
+
   // gather prefetch
   def int_x86_avx512_gatherpf_dpd_512  : GCCBuiltin<"__builtin_ia32_gatherpfdpd">,
           Intrinsic<[], [llvm_i8_ty, llvm_v8i32_ty, llvm_ptr_ty,
@@ -3504,29 +5706,29 @@ let TargetPrefix = "x86" in {
                   [IntrNoMem]>;
 
   def int_x86_avx512_mask_cmp_b_512: GCCBuiltin<"__builtin_ia32_cmpb512_mask">,
-        Intrinsic<[llvm_i64_ty], [llvm_v64i8_ty, llvm_v64i8_ty, llvm_i8_ty,
+        Intrinsic<[llvm_i64_ty], [llvm_v64i8_ty, llvm_v64i8_ty, llvm_i32_ty,
                   llvm_i64_ty], [IntrNoMem]>;
   def int_x86_avx512_mask_cmp_w_512: GCCBuiltin<"__builtin_ia32_cmpw512_mask">,
-        Intrinsic<[llvm_i32_ty], [llvm_v32i16_ty, llvm_v32i16_ty, llvm_i8_ty,
+        Intrinsic<[llvm_i32_ty], [llvm_v32i16_ty, llvm_v32i16_ty, llvm_i32_ty,
                   llvm_i32_ty], [IntrNoMem]>;
   def int_x86_avx512_mask_cmp_d_512: GCCBuiltin<"__builtin_ia32_cmpd512_mask">,
-        Intrinsic<[llvm_i16_ty], [llvm_v16i32_ty, llvm_v16i32_ty, llvm_i8_ty,
+        Intrinsic<[llvm_i16_ty], [llvm_v16i32_ty, llvm_v16i32_ty, llvm_i32_ty,
                   llvm_i16_ty], [IntrNoMem ]>;
   def int_x86_avx512_mask_cmp_q_512: GCCBuiltin<"__builtin_ia32_cmpq512_mask">,
-        Intrinsic<[llvm_i8_ty], [llvm_v8i64_ty, llvm_v8i64_ty, llvm_i8_ty,
+        Intrinsic<[llvm_i8_ty], [llvm_v8i64_ty, llvm_v8i64_ty, llvm_i32_ty,
                   llvm_i8_ty], [IntrNoMem]>;
 
   def int_x86_avx512_mask_ucmp_b_512: GCCBuiltin<"__builtin_ia32_ucmpb512_mask">,
-        Intrinsic<[llvm_i64_ty], [llvm_v64i8_ty, llvm_v64i8_ty, llvm_i8_ty,
+        Intrinsic<[llvm_i64_ty], [llvm_v64i8_ty, llvm_v64i8_ty, llvm_i32_ty,
                   llvm_i64_ty], [IntrNoMem]>;
   def int_x86_avx512_mask_ucmp_w_512: GCCBuiltin<"__builtin_ia32_ucmpw512_mask">,
-        Intrinsic<[llvm_i32_ty], [llvm_v32i16_ty, llvm_v32i16_ty, llvm_i8_ty,
+        Intrinsic<[llvm_i32_ty], [llvm_v32i16_ty, llvm_v32i16_ty, llvm_i32_ty,
                   llvm_i32_ty], [IntrNoMem]>;
   def int_x86_avx512_mask_ucmp_d_512: GCCBuiltin<"__builtin_ia32_ucmpd512_mask">,
-        Intrinsic<[llvm_i16_ty], [llvm_v16i32_ty, llvm_v16i32_ty, llvm_i8_ty,
+        Intrinsic<[llvm_i16_ty], [llvm_v16i32_ty, llvm_v16i32_ty, llvm_i32_ty,
                   llvm_i16_ty], [IntrNoMem]>;
   def int_x86_avx512_mask_ucmp_q_512: GCCBuiltin<"__builtin_ia32_ucmpq512_mask">,
-        Intrinsic<[llvm_i8_ty], [llvm_v8i64_ty, llvm_v8i64_ty, llvm_i8_ty,
+        Intrinsic<[llvm_i8_ty], [llvm_v8i64_ty, llvm_v8i64_ty, llvm_i32_ty,
                   llvm_i8_ty], [IntrNoMem]>;
 
   // 256-bit
@@ -3557,29 +5759,29 @@ let TargetPrefix = "x86" in {
                   [IntrNoMem]>;
 
   def int_x86_avx512_mask_cmp_b_256: GCCBuiltin<"__builtin_ia32_cmpb256_mask">,
-        Intrinsic<[llvm_i32_ty], [llvm_v32i8_ty, llvm_v32i8_ty, llvm_i8_ty,
+        Intrinsic<[llvm_i32_ty], [llvm_v32i8_ty, llvm_v32i8_ty, llvm_i32_ty,
                   llvm_i32_ty], [IntrNoMem]>;
   def int_x86_avx512_mask_cmp_w_256: GCCBuiltin<"__builtin_ia32_cmpw256_mask">,
-        Intrinsic<[llvm_i16_ty], [llvm_v16i16_ty, llvm_v16i16_ty, llvm_i8_ty,
+        Intrinsic<[llvm_i16_ty], [llvm_v16i16_ty, llvm_v16i16_ty, llvm_i32_ty,
                   llvm_i16_ty], [IntrNoMem]>;
   def int_x86_avx512_mask_cmp_d_256: GCCBuiltin<"__builtin_ia32_cmpd256_mask">,
-        Intrinsic<[llvm_i8_ty], [llvm_v8i32_ty, llvm_v8i32_ty, llvm_i8_ty,
+        Intrinsic<[llvm_i8_ty], [llvm_v8i32_ty, llvm_v8i32_ty, llvm_i32_ty,
                   llvm_i8_ty], [IntrNoMem]>;
   def int_x86_avx512_mask_cmp_q_256: GCCBuiltin<"__builtin_ia32_cmpq256_mask">,
-        Intrinsic<[llvm_i8_ty], [llvm_v4i64_ty, llvm_v4i64_ty, llvm_i8_ty,
+        Intrinsic<[llvm_i8_ty], [llvm_v4i64_ty, llvm_v4i64_ty, llvm_i32_ty,
                   llvm_i8_ty], [IntrNoMem]>;
 
   def int_x86_avx512_mask_ucmp_b_256: GCCBuiltin<"__builtin_ia32_ucmpb256_mask">,
-        Intrinsic<[llvm_i32_ty], [llvm_v32i8_ty, llvm_v32i8_ty, llvm_i8_ty,
+        Intrinsic<[llvm_i32_ty], [llvm_v32i8_ty, llvm_v32i8_ty, llvm_i32_ty,
                   llvm_i32_ty], [IntrNoMem]>;
   def int_x86_avx512_mask_ucmp_w_256: GCCBuiltin<"__builtin_ia32_ucmpw256_mask">,
-        Intrinsic<[llvm_i16_ty], [llvm_v16i16_ty, llvm_v16i16_ty, llvm_i8_ty,
+        Intrinsic<[llvm_i16_ty], [llvm_v16i16_ty, llvm_v16i16_ty, llvm_i32_ty,
                   llvm_i16_ty], [IntrNoMem]>;
   def int_x86_avx512_mask_ucmp_d_256: GCCBuiltin<"__builtin_ia32_ucmpd256_mask">,
-        Intrinsic<[llvm_i8_ty], [llvm_v8i32_ty, llvm_v8i32_ty, llvm_i8_ty,
+        Intrinsic<[llvm_i8_ty], [llvm_v8i32_ty, llvm_v8i32_ty, llvm_i32_ty,
                   llvm_i8_ty], [IntrNoMem]>;
   def int_x86_avx512_mask_ucmp_q_256: GCCBuiltin<"__builtin_ia32_ucmpq256_mask">,
-        Intrinsic<[llvm_i8_ty], [llvm_v4i64_ty, llvm_v4i64_ty, llvm_i8_ty,
+        Intrinsic<[llvm_i8_ty], [llvm_v4i64_ty, llvm_v4i64_ty, llvm_i32_ty,
                   llvm_i8_ty], [IntrNoMem]>;
 
   // 128-bit
@@ -3610,29 +5812,29 @@ let TargetPrefix = "x86" in {
                   [IntrNoMem]>;
 
   def int_x86_avx512_mask_cmp_b_128: GCCBuiltin<"__builtin_ia32_cmpb128_mask">,
-        Intrinsic<[llvm_i16_ty], [llvm_v16i8_ty, llvm_v16i8_ty, llvm_i8_ty,
+        Intrinsic<[llvm_i16_ty], [llvm_v16i8_ty, llvm_v16i8_ty, llvm_i32_ty,
                   llvm_i16_ty], [IntrNoMem]>;
   def int_x86_avx512_mask_cmp_w_128: GCCBuiltin<"__builtin_ia32_cmpw128_mask">,
-        Intrinsic<[llvm_i8_ty], [llvm_v8i16_ty, llvm_v8i16_ty, llvm_i8_ty,
+        Intrinsic<[llvm_i8_ty], [llvm_v8i16_ty, llvm_v8i16_ty, llvm_i32_ty,
                   llvm_i8_ty], [IntrNoMem]>;
   def int_x86_avx512_mask_cmp_d_128: GCCBuiltin<"__builtin_ia32_cmpd128_mask">,
-        Intrinsic<[llvm_i8_ty], [llvm_v4i32_ty, llvm_v4i32_ty, llvm_i8_ty,
+        Intrinsic<[llvm_i8_ty], [llvm_v4i32_ty, llvm_v4i32_ty, llvm_i32_ty,
                   llvm_i8_ty], [IntrNoMem]>;
   def int_x86_avx512_mask_cmp_q_128: GCCBuiltin<"__builtin_ia32_cmpq128_mask">,
-        Intrinsic<[llvm_i8_ty], [llvm_v2i64_ty, llvm_v2i64_ty, llvm_i8_ty,
+        Intrinsic<[llvm_i8_ty], [llvm_v2i64_ty, llvm_v2i64_ty, llvm_i32_ty,
                   llvm_i8_ty], [IntrNoMem]>;
 
   def int_x86_avx512_mask_ucmp_b_128: GCCBuiltin<"__builtin_ia32_ucmpb128_mask">,
-        Intrinsic<[llvm_i16_ty], [llvm_v16i8_ty, llvm_v16i8_ty, llvm_i8_ty,
+        Intrinsic<[llvm_i16_ty], [llvm_v16i8_ty, llvm_v16i8_ty, llvm_i32_ty,
                   llvm_i16_ty], [IntrNoMem]>;
   def int_x86_avx512_mask_ucmp_w_128: GCCBuiltin<"__builtin_ia32_ucmpw128_mask">,
-        Intrinsic<[llvm_i8_ty], [llvm_v8i16_ty, llvm_v8i16_ty, llvm_i8_ty,
+        Intrinsic<[llvm_i8_ty], [llvm_v8i16_ty, llvm_v8i16_ty, llvm_i32_ty,
                   llvm_i8_ty], [IntrNoMem]>;
   def int_x86_avx512_mask_ucmp_d_128: GCCBuiltin<"__builtin_ia32_ucmpd128_mask">,
-        Intrinsic<[llvm_i8_ty], [llvm_v4i32_ty, llvm_v4i32_ty, llvm_i8_ty,
+        Intrinsic<[llvm_i8_ty], [llvm_v4i32_ty, llvm_v4i32_ty, llvm_i32_ty,
                   llvm_i8_ty], [IntrNoMem]>;
   def int_x86_avx512_mask_ucmp_q_128: GCCBuiltin<"__builtin_ia32_ucmpq128_mask">,
-        Intrinsic<[llvm_i8_ty], [llvm_v2i64_ty, llvm_v2i64_ty, llvm_i8_ty,
+        Intrinsic<[llvm_i8_ty], [llvm_v2i64_ty, llvm_v2i64_ty, llvm_i32_ty,
                   llvm_i8_ty], [IntrNoMem]>;
 }
 
@@ -3840,15 +6042,579 @@ let TargetPrefix = "x86" in {
                    llvm_i8_ty], [IntrReadArgMem]>;
 
 }
+
+// truncate
+let TargetPrefix = "x86" in {
+  def int_x86_avx512_mask_pmov_qb_128 :
+          GCCBuiltin<"__builtin_ia32_pmovqb128_mask">,
+          Intrinsic<[llvm_v16i8_ty],
+                    [llvm_v2i64_ty, llvm_v16i8_ty, llvm_i8_ty],
+                    [IntrNoMem]>;
+  def int_x86_avx512_mask_pmov_qb_mem_128 :
+          GCCBuiltin<"__builtin_ia32_pmovqb128mem_mask">,
+          Intrinsic<[],
+                    [llvm_ptr_ty, llvm_v2i64_ty, llvm_i8_ty],
+                    [IntrReadWriteArgMem]>;
+  def int_x86_avx512_mask_pmovs_qb_128 :
+          GCCBuiltin<"__builtin_ia32_pmovsqb128_mask">,
+          Intrinsic<[llvm_v16i8_ty],
+                    [llvm_v2i64_ty, llvm_v16i8_ty, llvm_i8_ty],
+                    [IntrNoMem]>;
+  def int_x86_avx512_mask_pmovs_qb_mem_128 :
+          GCCBuiltin<"__builtin_ia32_pmovsqb128mem_mask">,
+          Intrinsic<[],
+                    [llvm_ptr_ty, llvm_v2i64_ty, llvm_i8_ty],
+                    [IntrReadWriteArgMem]>;
+  def int_x86_avx512_mask_pmovus_qb_128 :
+          GCCBuiltin<"__builtin_ia32_pmovusqb128_mask">,
+          Intrinsic<[llvm_v16i8_ty],
+                    [llvm_v2i64_ty, llvm_v16i8_ty, llvm_i8_ty],
+                    [IntrNoMem]>;
+  def int_x86_avx512_mask_pmovus_qb_mem_128 :
+          GCCBuiltin<"__builtin_ia32_pmovusqb128mem_mask">,
+          Intrinsic<[],
+                    [llvm_ptr_ty, llvm_v2i64_ty, llvm_i8_ty],
+                    [IntrReadWriteArgMem]>;
+  def int_x86_avx512_mask_pmov_qb_256 :
+          GCCBuiltin<"__builtin_ia32_pmovqb256_mask">,
+          Intrinsic<[llvm_v16i8_ty],
+                    [llvm_v4i64_ty, llvm_v16i8_ty, llvm_i8_ty],
+                    [IntrNoMem]>;
+  def int_x86_avx512_mask_pmov_qb_mem_256 :
+          GCCBuiltin<"__builtin_ia32_pmovqb256mem_mask">,
+          Intrinsic<[],
+                    [llvm_ptr_ty, llvm_v4i64_ty, llvm_i8_ty],
+                    [IntrReadWriteArgMem]>;
+  def int_x86_avx512_mask_pmovs_qb_256 :
+          GCCBuiltin<"__builtin_ia32_pmovsqb256_mask">,
+          Intrinsic<[llvm_v16i8_ty],
+                    [llvm_v4i64_ty, llvm_v16i8_ty, llvm_i8_ty],
+                    [IntrNoMem]>;
+  def int_x86_avx512_mask_pmovs_qb_mem_256 :
+          GCCBuiltin<"__builtin_ia32_pmovsqb256mem_mask">,
+          Intrinsic<[],
+                    [llvm_ptr_ty, llvm_v4i64_ty, llvm_i8_ty],
+                    [IntrReadWriteArgMem]>;
+  def int_x86_avx512_mask_pmovus_qb_256 :
+          GCCBuiltin<"__builtin_ia32_pmovusqb256_mask">,
+          Intrinsic<[llvm_v16i8_ty],
+                    [llvm_v4i64_ty, llvm_v16i8_ty, llvm_i8_ty],
+                    [IntrNoMem]>;
+  def int_x86_avx512_mask_pmovus_qb_mem_256 :
+          GCCBuiltin<"__builtin_ia32_pmovusqb256mem_mask">,
+          Intrinsic<[],
+                    [llvm_ptr_ty, llvm_v4i64_ty, llvm_i8_ty],
+                    [IntrReadWriteArgMem]>;
+  def int_x86_avx512_mask_pmov_qb_512 :
+          GCCBuiltin<"__builtin_ia32_pmovqb512_mask">,
+          Intrinsic<[llvm_v16i8_ty],
+                    [llvm_v8i64_ty, llvm_v16i8_ty, llvm_i8_ty],
+                    [IntrNoMem]>;
+  def int_x86_avx512_mask_pmov_qb_mem_512 :
+          GCCBuiltin<"__builtin_ia32_pmovqb512mem_mask">,
+          Intrinsic<[],
+                    [llvm_ptr_ty, llvm_v8i64_ty, llvm_i8_ty],
+                    [IntrReadWriteArgMem]>;
+  def int_x86_avx512_mask_pmovs_qb_512 :
+          GCCBuiltin<"__builtin_ia32_pmovsqb512_mask">,
+          Intrinsic<[llvm_v16i8_ty],
+                    [llvm_v8i64_ty, llvm_v16i8_ty, llvm_i8_ty],
+                    [IntrNoMem]>;
+  def int_x86_avx512_mask_pmovs_qb_mem_512 :
+          GCCBuiltin<"__builtin_ia32_pmovsqb512mem_mask">,
+          Intrinsic<[],
+                    [llvm_ptr_ty, llvm_v8i64_ty, llvm_i8_ty],
+                    [IntrReadWriteArgMem]>;
+  def int_x86_avx512_mask_pmovus_qb_512 :
+          GCCBuiltin<"__builtin_ia32_pmovusqb512_mask">,
+          Intrinsic<[llvm_v16i8_ty],
+                    [llvm_v8i64_ty, llvm_v16i8_ty, llvm_i8_ty],
+                    [IntrNoMem]>;
+  def int_x86_avx512_mask_pmovus_qb_mem_512 :
+          GCCBuiltin<"__builtin_ia32_pmovusqb512mem_mask">,
+          Intrinsic<[],
+                    [llvm_ptr_ty, llvm_v8i64_ty, llvm_i8_ty],
+                    [IntrReadWriteArgMem]>;
+  def int_x86_avx512_mask_pmov_qw_128 :
+          GCCBuiltin<"__builtin_ia32_pmovqw128_mask">,
+          Intrinsic<[llvm_v8i16_ty],
+                    [llvm_v2i64_ty, llvm_v8i16_ty, llvm_i8_ty],
+                    [IntrNoMem]>;
+  def int_x86_avx512_mask_pmov_qw_mem_128 :
+          GCCBuiltin<"__builtin_ia32_pmovqw128mem_mask">,
+          Intrinsic<[],
+                    [llvm_ptr_ty, llvm_v2i64_ty, llvm_i8_ty],
+                    [IntrReadWriteArgMem]>;
+  def int_x86_avx512_mask_pmovs_qw_128 :
+          GCCBuiltin<"__builtin_ia32_pmovsqw128_mask">,
+          Intrinsic<[llvm_v8i16_ty],
+                    [llvm_v2i64_ty, llvm_v8i16_ty, llvm_i8_ty],
+                    [IntrNoMem]>;
+  def int_x86_avx512_mask_pmovs_qw_mem_128 :
+          GCCBuiltin<"__builtin_ia32_pmovsqw128mem_mask">,
+          Intrinsic<[],
+                    [llvm_ptr_ty, llvm_v2i64_ty, llvm_i8_ty],
+                    [IntrReadWriteArgMem]>;
+  def int_x86_avx512_mask_pmovus_qw_128 :
+          GCCBuiltin<"__builtin_ia32_pmovusqw128_mask">,
+          Intrinsic<[llvm_v8i16_ty],
+                    [llvm_v2i64_ty, llvm_v8i16_ty, llvm_i8_ty],
+                    [IntrNoMem]>;
+  def int_x86_avx512_mask_pmovus_qw_mem_128 :
+          GCCBuiltin<"__builtin_ia32_pmovusqw128mem_mask">,
+          Intrinsic<[],
+                    [llvm_ptr_ty, llvm_v2i64_ty, llvm_i8_ty],
+                    [IntrReadWriteArgMem]>;
+  def int_x86_avx512_mask_pmov_qw_256 :
+          GCCBuiltin<"__builtin_ia32_pmovqw256_mask">,
+          Intrinsic<[llvm_v8i16_ty],
+                    [llvm_v4i64_ty, llvm_v8i16_ty, llvm_i8_ty],
+                    [IntrNoMem]>;
+  def int_x86_avx512_mask_pmov_qw_mem_256 :
+          GCCBuiltin<"__builtin_ia32_pmovqw256mem_mask">,
+          Intrinsic<[],
+                    [llvm_ptr_ty, llvm_v4i64_ty, llvm_i8_ty],
+                    [IntrReadWriteArgMem]>;
+  def int_x86_avx512_mask_pmovs_qw_256 :
+          GCCBuiltin<"__builtin_ia32_pmovsqw256_mask">,
+          Intrinsic<[llvm_v8i16_ty],
+                    [llvm_v4i64_ty, llvm_v8i16_ty, llvm_i8_ty],
+                    [IntrNoMem]>;
+  def int_x86_avx512_mask_pmovs_qw_mem_256 :
+          GCCBuiltin<"__builtin_ia32_pmovsqw256mem_mask">,
+          Intrinsic<[],
+                    [llvm_ptr_ty, llvm_v4i64_ty, llvm_i8_ty],
+                    [IntrReadWriteArgMem]>;
+  def int_x86_avx512_mask_pmovus_qw_256 :
+          GCCBuiltin<"__builtin_ia32_pmovusqw256_mask">,
+          Intrinsic<[llvm_v8i16_ty],
+                    [llvm_v4i64_ty, llvm_v8i16_ty, llvm_i8_ty],
+                    [IntrNoMem]>;
+  def int_x86_avx512_mask_pmovus_qw_mem_256 :
+          GCCBuiltin<"__builtin_ia32_pmovusqw256mem_mask">,
+          Intrinsic<[],
+                    [llvm_ptr_ty, llvm_v4i64_ty, llvm_i8_ty],
+                    [IntrReadWriteArgMem]>;
+  def int_x86_avx512_mask_pmov_qw_512 :
+          GCCBuiltin<"__builtin_ia32_pmovqw512_mask">,
+          Intrinsic<[llvm_v8i16_ty],
+                    [llvm_v8i64_ty, llvm_v8i16_ty, llvm_i8_ty],
+                    [IntrNoMem]>;
+  def int_x86_avx512_mask_pmov_qw_mem_512 :
+          GCCBuiltin<"__builtin_ia32_pmovqw512mem_mask">,
+          Intrinsic<[],
+                    [llvm_ptr_ty, llvm_v8i64_ty, llvm_i8_ty],
+                    [IntrReadWriteArgMem]>;
+  def int_x86_avx512_mask_pmovs_qw_512 :
+          GCCBuiltin<"__builtin_ia32_pmovsqw512_mask">,
+          Intrinsic<[llvm_v8i16_ty],
+                    [llvm_v8i64_ty, llvm_v8i16_ty, llvm_i8_ty],
+                    [IntrNoMem]>;
+  def int_x86_avx512_mask_pmovs_qw_mem_512 :
+          GCCBuiltin<"__builtin_ia32_pmovsqw512mem_mask">,
+          Intrinsic<[],
+                    [llvm_ptr_ty, llvm_v8i64_ty, llvm_i8_ty],
+                    [IntrReadWriteArgMem]>;
+  def int_x86_avx512_mask_pmovus_qw_512 :
+          GCCBuiltin<"__builtin_ia32_pmovusqw512_mask">,
+          Intrinsic<[llvm_v8i16_ty],
+                    [llvm_v8i64_ty, llvm_v8i16_ty, llvm_i8_ty],
+                    [IntrNoMem]>;
+  def int_x86_avx512_mask_pmovus_qw_mem_512 :
+          GCCBuiltin<"__builtin_ia32_pmovusqw512mem_mask">,
+          Intrinsic<[],
+                    [llvm_ptr_ty, llvm_v8i64_ty, llvm_i8_ty],
+                    [IntrReadWriteArgMem]>;
+  def int_x86_avx512_mask_pmov_qd_128 :
+          GCCBuiltin<"__builtin_ia32_pmovqd128_mask">,
+          Intrinsic<[llvm_v4i32_ty],
+                    [llvm_v2i64_ty, llvm_v4i32_ty, llvm_i8_ty],
+                    [IntrNoMem]>;
+  def int_x86_avx512_mask_pmov_qd_mem_128 :
+          GCCBuiltin<"__builtin_ia32_pmovqd128mem_mask">,
+          Intrinsic<[],
+                    [llvm_ptr_ty, llvm_v2i64_ty, llvm_i8_ty],
+                    [IntrReadWriteArgMem]>;
+  def int_x86_avx512_mask_pmovs_qd_128 :
+          GCCBuiltin<"__builtin_ia32_pmovsqd128_mask">,
+          Intrinsic<[llvm_v4i32_ty],
+                    [llvm_v2i64_ty, llvm_v4i32_ty, llvm_i8_ty],
+                    [IntrNoMem]>;
+  def int_x86_avx512_mask_pmovs_qd_mem_128 :
+          GCCBuiltin<"__builtin_ia32_pmovsqd128mem_mask">,
+          Intrinsic<[],
+                    [llvm_ptr_ty, llvm_v2i64_ty, llvm_i8_ty],
+                    [IntrReadWriteArgMem]>;
+  def int_x86_avx512_mask_pmovus_qd_128 :
+          GCCBuiltin<"__builtin_ia32_pmovusqd128_mask">,
+          Intrinsic<[llvm_v4i32_ty],
+                    [llvm_v2i64_ty, llvm_v4i32_ty, llvm_i8_ty],
+                    [IntrNoMem]>;
+  def int_x86_avx512_mask_pmovus_qd_mem_128 :
+          GCCBuiltin<"__builtin_ia32_pmovusqd128mem_mask">,
+          Intrinsic<[],
+                    [llvm_ptr_ty, llvm_v2i64_ty, llvm_i8_ty],
+                    [IntrReadWriteArgMem]>;
+  def int_x86_avx512_mask_pmov_qd_256 :
+          GCCBuiltin<"__builtin_ia32_pmovqd256_mask">,
+          Intrinsic<[llvm_v4i32_ty],
+                    [llvm_v4i64_ty, llvm_v4i32_ty, llvm_i8_ty],
+                    [IntrNoMem]>;
+  def int_x86_avx512_mask_pmov_qd_mem_256 :
+          GCCBuiltin<"__builtin_ia32_pmovqd256mem_mask">,
+          Intrinsic<[],
+                    [llvm_ptr_ty, llvm_v4i64_ty, llvm_i8_ty],
+                    [IntrReadWriteArgMem]>;
+  def int_x86_avx512_mask_pmovs_qd_256 :
+          GCCBuiltin<"__builtin_ia32_pmovsqd256_mask">,
+          Intrinsic<[llvm_v4i32_ty],
+                    [llvm_v4i64_ty, llvm_v4i32_ty, llvm_i8_ty],
+                    [IntrNoMem]>;
+  def int_x86_avx512_mask_pmovs_qd_mem_256 :
+          GCCBuiltin<"__builtin_ia32_pmovsqd256mem_mask">,
+          Intrinsic<[],
+                    [llvm_ptr_ty, llvm_v4i64_ty, llvm_i8_ty],
+                    [IntrReadWriteArgMem]>;
+  def int_x86_avx512_mask_pmovus_qd_256 :
+          GCCBuiltin<"__builtin_ia32_pmovusqd256_mask">,
+          Intrinsic<[llvm_v4i32_ty],
+                    [llvm_v4i64_ty, llvm_v4i32_ty, llvm_i8_ty],
+                    [IntrNoMem]>;
+  def int_x86_avx512_mask_pmovus_qd_mem_256 :
+          GCCBuiltin<"__builtin_ia32_pmovusqd256mem_mask">,
+          Intrinsic<[],
+                    [llvm_ptr_ty, llvm_v4i64_ty, llvm_i8_ty],
+                    [IntrReadWriteArgMem]>;
+  def int_x86_avx512_mask_pmov_qd_512 :
+          GCCBuiltin<"__builtin_ia32_pmovqd512_mask">,
+          Intrinsic<[llvm_v8i32_ty],
+                    [llvm_v8i64_ty, llvm_v8i32_ty, llvm_i8_ty],
+                    [IntrNoMem]>;
+  def int_x86_avx512_mask_pmov_qd_mem_512 :
+          GCCBuiltin<"__builtin_ia32_pmovqd512mem_mask">,
+          Intrinsic<[],
+                    [llvm_ptr_ty, llvm_v8i64_ty, llvm_i8_ty],
+                    [IntrReadWriteArgMem]>;
+  def int_x86_avx512_mask_pmovs_qd_512 :
+          GCCBuiltin<"__builtin_ia32_pmovsqd512_mask">,
+          Intrinsic<[llvm_v8i32_ty],
+                    [llvm_v8i64_ty, llvm_v8i32_ty, llvm_i8_ty],
+                    [IntrNoMem]>;
+  def int_x86_avx512_mask_pmovs_qd_mem_512 :
+          GCCBuiltin<"__builtin_ia32_pmovsqd512mem_mask">,
+          Intrinsic<[],
+                    [llvm_ptr_ty, llvm_v8i64_ty, llvm_i8_ty],
+                    [IntrReadWriteArgMem]>;
+  def int_x86_avx512_mask_pmovus_qd_512 :
+          GCCBuiltin<"__builtin_ia32_pmovusqd512_mask">,
+          Intrinsic<[llvm_v8i32_ty],
+                    [llvm_v8i64_ty, llvm_v8i32_ty, llvm_i8_ty],
+                    [IntrNoMem]>;
+  def int_x86_avx512_mask_pmovus_qd_mem_512 :
+          GCCBuiltin<"__builtin_ia32_pmovusqd512mem_mask">,
+          Intrinsic<[],
+                    [llvm_ptr_ty, llvm_v8i64_ty, llvm_i8_ty],
+                    [IntrReadWriteArgMem]>;
+  def int_x86_avx512_mask_pmov_db_128 :
+          GCCBuiltin<"__builtin_ia32_pmovdb128_mask">,
+          Intrinsic<[llvm_v16i8_ty],
+                    [llvm_v4i32_ty, llvm_v16i8_ty, llvm_i8_ty],
+                    [IntrNoMem]>;
+  def int_x86_avx512_mask_pmov_db_mem_128 :
+          GCCBuiltin<"__builtin_ia32_pmovdb128mem_mask">,
+          Intrinsic<[],
+                    [llvm_ptr_ty, llvm_v4i32_ty, llvm_i8_ty],
+                    [IntrReadWriteArgMem]>;
+  def int_x86_avx512_mask_pmovs_db_128 :
+          GCCBuiltin<"__builtin_ia32_pmovsdb128_mask">,
+          Intrinsic<[llvm_v16i8_ty],
+                    [llvm_v4i32_ty, llvm_v16i8_ty, llvm_i8_ty],
+                    [IntrNoMem]>;
+  def int_x86_avx512_mask_pmovs_db_mem_128 :
+          GCCBuiltin<"__builtin_ia32_pmovsdb128mem_mask">,
+          Intrinsic<[],
+                    [llvm_ptr_ty, llvm_v4i32_ty, llvm_i8_ty],
+                    [IntrReadWriteArgMem]>;
+  def int_x86_avx512_mask_pmovus_db_128 :
+          GCCBuiltin<"__builtin_ia32_pmovusdb128_mask">,
+          Intrinsic<[llvm_v16i8_ty],
+                    [llvm_v4i32_ty, llvm_v16i8_ty, llvm_i8_ty],
+                    [IntrNoMem]>;
+  def int_x86_avx512_mask_pmovus_db_mem_128 :
+          GCCBuiltin<"__builtin_ia32_pmovusdb128mem_mask">,
+          Intrinsic<[],
+                    [llvm_ptr_ty, llvm_v4i32_ty, llvm_i8_ty],
+                    [IntrReadWriteArgMem]>;
+  def int_x86_avx512_mask_pmov_db_256 :
+          GCCBuiltin<"__builtin_ia32_pmovdb256_mask">,
+          Intrinsic<[llvm_v16i8_ty],
+                    [llvm_v8i32_ty, llvm_v16i8_ty, llvm_i8_ty],
+                    [IntrNoMem]>;
+  def int_x86_avx512_mask_pmov_db_mem_256 :
+          GCCBuiltin<"__builtin_ia32_pmovdb256mem_mask">,
+          Intrinsic<[],
+                    [llvm_ptr_ty, llvm_v8i32_ty, llvm_i8_ty],
+                    [IntrReadWriteArgMem]>;
+  def int_x86_avx512_mask_pmovs_db_256 :
+          GCCBuiltin<"__builtin_ia32_pmovsdb256_mask">,
+          Intrinsic<[llvm_v16i8_ty],
+                    [llvm_v8i32_ty, llvm_v16i8_ty, llvm_i8_ty],
+                    [IntrNoMem]>;
+  def int_x86_avx512_mask_pmovs_db_mem_256 :
+          GCCBuiltin<"__builtin_ia32_pmovsdb256mem_mask">,
+          Intrinsic<[],
+                    [llvm_ptr_ty, llvm_v8i32_ty, llvm_i8_ty],
+                    [IntrReadWriteArgMem]>;
+  def int_x86_avx512_mask_pmovus_db_256 :
+          GCCBuiltin<"__builtin_ia32_pmovusdb256_mask">,
+          Intrinsic<[llvm_v16i8_ty],
+                    [llvm_v8i32_ty, llvm_v16i8_ty, llvm_i8_ty],
+                    [IntrNoMem]>;
+  def int_x86_avx512_mask_pmovus_db_mem_256 :
+          GCCBuiltin<"__builtin_ia32_pmovusdb256mem_mask">,
+          Intrinsic<[],
+                    [llvm_ptr_ty, llvm_v8i32_ty, llvm_i8_ty],
+                    [IntrReadWriteArgMem]>;
+  def int_x86_avx512_mask_pmov_db_512 :
+          GCCBuiltin<"__builtin_ia32_pmovdb512_mask">,
+          Intrinsic<[llvm_v16i8_ty],
+                    [llvm_v16i32_ty, llvm_v16i8_ty, llvm_i16_ty],
+                    [IntrNoMem]>;
+  def int_x86_avx512_mask_pmov_db_mem_512 :
+          GCCBuiltin<"__builtin_ia32_pmovdb512mem_mask">,
+          Intrinsic<[],
+                    [llvm_ptr_ty, llvm_v16i32_ty, llvm_i16_ty],
+                    [IntrReadWriteArgMem]>;
+  def int_x86_avx512_mask_pmovs_db_512 :
+          GCCBuiltin<"__builtin_ia32_pmovsdb512_mask">,
+          Intrinsic<[llvm_v16i8_ty],
+                    [llvm_v16i32_ty, llvm_v16i8_ty, llvm_i16_ty],
+                    [IntrNoMem]>;
+  def int_x86_avx512_mask_pmovs_db_mem_512 :
+          GCCBuiltin<"__builtin_ia32_pmovsdb512mem_mask">,
+          Intrinsic<[],
+                    [llvm_ptr_ty, llvm_v16i32_ty, llvm_i16_ty],
+                    [IntrReadWriteArgMem]>;
+  def int_x86_avx512_mask_pmovus_db_512 :
+          GCCBuiltin<"__builtin_ia32_pmovusdb512_mask">,
+          Intrinsic<[llvm_v16i8_ty],
+                    [llvm_v16i32_ty, llvm_v16i8_ty, llvm_i16_ty],
+                    [IntrNoMem]>;
+  def int_x86_avx512_mask_pmovus_db_mem_512 :
+          GCCBuiltin<"__builtin_ia32_pmovusdb512mem_mask">,
+          Intrinsic<[],
+                    [llvm_ptr_ty, llvm_v16i32_ty, llvm_i16_ty],
+                    [IntrReadWriteArgMem]>;
+  def int_x86_avx512_mask_pmov_dw_128 :
+          GCCBuiltin<"__builtin_ia32_pmovdw128_mask">,
+          Intrinsic<[llvm_v8i16_ty],
+                    [llvm_v4i32_ty, llvm_v8i16_ty, llvm_i8_ty],
+                    [IntrNoMem]>;
+  def int_x86_avx512_mask_pmov_dw_mem_128 :
+          GCCBuiltin<"__builtin_ia32_pmovdw128mem_mask">,
+          Intrinsic<[],
+                    [llvm_ptr_ty, llvm_v4i32_ty, llvm_i8_ty],
+                    [IntrReadWriteArgMem]>;
+  def int_x86_avx512_mask_pmovs_dw_128 :
+          GCCBuiltin<"__builtin_ia32_pmovsdw128_mask">,
+          Intrinsic<[llvm_v8i16_ty],
+                    [llvm_v4i32_ty, llvm_v8i16_ty, llvm_i8_ty],
+                    [IntrNoMem]>;
+  def int_x86_avx512_mask_pmovs_dw_mem_128 :
+          GCCBuiltin<"__builtin_ia32_pmovsdw128mem_mask">,
+          Intrinsic<[],
+                    [llvm_ptr_ty, llvm_v4i32_ty, llvm_i8_ty],
+                    [IntrReadWriteArgMem]>;
+  def int_x86_avx512_mask_pmovus_dw_128 :
+          GCCBuiltin<"__builtin_ia32_pmovusdw128_mask">,
+          Intrinsic<[llvm_v8i16_ty],
+                    [llvm_v4i32_ty, llvm_v8i16_ty, llvm_i8_ty],
+                    [IntrNoMem]>;
+  def int_x86_avx512_mask_pmovus_dw_mem_128 :
+          GCCBuiltin<"__builtin_ia32_pmovusdw128mem_mask">,
+          Intrinsic<[],
+                    [llvm_ptr_ty, llvm_v4i32_ty, llvm_i8_ty],
+                    [IntrReadWriteArgMem]>;
+  def int_x86_avx512_mask_pmov_dw_256 :
+          GCCBuiltin<"__builtin_ia32_pmovdw256_mask">,
+          Intrinsic<[llvm_v8i16_ty],
+                    [llvm_v8i32_ty, llvm_v8i16_ty, llvm_i8_ty],
+                    [IntrNoMem]>;
+  def int_x86_avx512_mask_pmov_dw_mem_256 :
+          GCCBuiltin<"__builtin_ia32_pmovdw256mem_mask">,
+          Intrinsic<[],
+                    [llvm_ptr_ty, llvm_v8i32_ty, llvm_i8_ty],
+                    [IntrReadWriteArgMem]>;
+  def int_x86_avx512_mask_pmovs_dw_256 :
+          GCCBuiltin<"__builtin_ia32_pmovsdw256_mask">,
+          Intrinsic<[llvm_v8i16_ty],
+                    [llvm_v8i32_ty, llvm_v8i16_ty, llvm_i8_ty],
+                    [IntrNoMem]>;
+  def int_x86_avx512_mask_pmovs_dw_mem_256 :
+          GCCBuiltin<"__builtin_ia32_pmovsdw256mem_mask">,
+          Intrinsic<[],
+                    [llvm_ptr_ty, llvm_v8i32_ty, llvm_i8_ty],
+                    [IntrReadWriteArgMem]>;
+  def int_x86_avx512_mask_pmovus_dw_256 :
+          GCCBuiltin<"__builtin_ia32_pmovusdw256_mask">,
+          Intrinsic<[llvm_v8i16_ty],
+                    [llvm_v8i32_ty, llvm_v8i16_ty, llvm_i8_ty],
+                    [IntrNoMem]>;
+  def int_x86_avx512_mask_pmovus_dw_mem_256 :
+          GCCBuiltin<"__builtin_ia32_pmovusdw256mem_mask">,
+          Intrinsic<[],
+                    [llvm_ptr_ty, llvm_v8i32_ty, llvm_i8_ty],
+                    [IntrReadWriteArgMem]>;
+  def int_x86_avx512_mask_pmov_dw_512 :
+          GCCBuiltin<"__builtin_ia32_pmovdw512_mask">,
+          Intrinsic<[llvm_v16i16_ty],
+                    [llvm_v16i32_ty, llvm_v16i16_ty, llvm_i16_ty],
+                    [IntrNoMem]>;
+  def int_x86_avx512_mask_pmov_dw_mem_512 :
+          GCCBuiltin<"__builtin_ia32_pmovdw512mem_mask">,
+          Intrinsic<[],
+                    [llvm_ptr_ty, llvm_v16i32_ty, llvm_i16_ty],
+                    [IntrReadWriteArgMem]>;
+  def int_x86_avx512_mask_pmovs_dw_512 :
+          GCCBuiltin<"__builtin_ia32_pmovsdw512_mask">,
+          Intrinsic<[llvm_v16i16_ty],
+                    [llvm_v16i32_ty, llvm_v16i16_ty, llvm_i16_ty],
+                    [IntrNoMem]>;
+  def int_x86_avx512_mask_pmovs_dw_mem_512 :
+          GCCBuiltin<"__builtin_ia32_pmovsdw512mem_mask">,
+          Intrinsic<[],
+                    [llvm_ptr_ty, llvm_v16i32_ty, llvm_i16_ty],
+                    [IntrReadWriteArgMem]>;
+  def int_x86_avx512_mask_pmovus_dw_512 :
+          GCCBuiltin<"__builtin_ia32_pmovusdw512_mask">,
+          Intrinsic<[llvm_v16i16_ty],
+                    [llvm_v16i32_ty, llvm_v16i16_ty, llvm_i16_ty],
+                    [IntrNoMem]>;
+  def int_x86_avx512_mask_pmovus_dw_mem_512 :
+          GCCBuiltin<"__builtin_ia32_pmovusdw512mem_mask">,
+          Intrinsic<[],
+                    [llvm_ptr_ty, llvm_v16i32_ty, llvm_i16_ty],
+                    [IntrReadWriteArgMem]>;
+  def int_x86_avx512_mask_pmov_wb_128 :
+          GCCBuiltin<"__builtin_ia32_pmovwb128_mask">,
+          Intrinsic<[llvm_v16i8_ty],
+                    [llvm_v8i16_ty, llvm_v16i8_ty, llvm_i8_ty],
+                    [IntrNoMem]>;
+  def int_x86_avx512_mask_pmov_wb_mem_128 :
+          GCCBuiltin<"__builtin_ia32_pmovwb128mem_mask">,
+          Intrinsic<[],
+                    [llvm_ptr_ty, llvm_v8i16_ty, llvm_i8_ty],
+                    [IntrReadWriteArgMem]>;
+  def int_x86_avx512_mask_pmovs_wb_128 :
+          GCCBuiltin<"__builtin_ia32_pmovswb128_mask">,
+          Intrinsic<[llvm_v16i8_ty],
+                    [llvm_v8i16_ty, llvm_v16i8_ty, llvm_i8_ty],
+                    [IntrNoMem]>;
+  def int_x86_avx512_mask_pmovs_wb_mem_128 :
+          GCCBuiltin<"__builtin_ia32_pmovswb128mem_mask">,
+          Intrinsic<[],
+                    [llvm_ptr_ty, llvm_v8i16_ty, llvm_i8_ty],
+                    [IntrReadWriteArgMem]>;
+  def int_x86_avx512_mask_pmovus_wb_128 :
+          GCCBuiltin<"__builtin_ia32_pmovuswb128_mask">,
+          Intrinsic<[llvm_v16i8_ty],
+                    [llvm_v8i16_ty, llvm_v16i8_ty, llvm_i8_ty],
+                    [IntrNoMem]>;
+  def int_x86_avx512_mask_pmovus_wb_mem_128 :
+          GCCBuiltin<"__builtin_ia32_pmovuswb128mem_mask">,
+          Intrinsic<[],
+                    [llvm_ptr_ty, llvm_v8i16_ty, llvm_i8_ty],
+                    [IntrReadWriteArgMem]>;
+  def int_x86_avx512_mask_pmov_wb_256 :
+          GCCBuiltin<"__builtin_ia32_pmovwb256_mask">,
+          Intrinsic<[llvm_v16i8_ty],
+                    [llvm_v16i16_ty, llvm_v16i8_ty, llvm_i16_ty],
+                    [IntrNoMem]>;
+  def int_x86_avx512_mask_pmov_wb_mem_256 :
+          GCCBuiltin<"__builtin_ia32_pmovwb256mem_mask">,
+          Intrinsic<[],
+                    [llvm_ptr_ty, llvm_v16i16_ty, llvm_i16_ty],
+                    [IntrReadWriteArgMem]>;
+  def int_x86_avx512_mask_pmovs_wb_256 :
+          GCCBuiltin<"__builtin_ia32_pmovswb256_mask">,
+          Intrinsic<[llvm_v16i8_ty],
+                    [llvm_v16i16_ty, llvm_v16i8_ty, llvm_i16_ty],
+                    [IntrNoMem]>;
+  def int_x86_avx512_mask_pmovs_wb_mem_256 :
+          GCCBuiltin<"__builtin_ia32_pmovswb256mem_mask">,
+          Intrinsic<[],
+                    [llvm_ptr_ty, llvm_v16i16_ty, llvm_i16_ty],
+                    [IntrReadWriteArgMem]>;
+  def int_x86_avx512_mask_pmovus_wb_256 :
+          GCCBuiltin<"__builtin_ia32_pmovuswb256_mask">,
+          Intrinsic<[llvm_v16i8_ty],
+                    [llvm_v16i16_ty, llvm_v16i8_ty, llvm_i16_ty],
+                    [IntrNoMem]>;
+  def int_x86_avx512_mask_pmovus_wb_mem_256 :
+          GCCBuiltin<"__builtin_ia32_pmovuswb256mem_mask">,
+          Intrinsic<[],
+                    [llvm_ptr_ty, llvm_v16i16_ty, llvm_i16_ty],
+                    [IntrReadWriteArgMem]>;
+  def int_x86_avx512_mask_pmov_wb_512 :
+          GCCBuiltin<"__builtin_ia32_pmovwb512_mask">,
+          Intrinsic<[llvm_v32i8_ty],
+                    [llvm_v32i16_ty, llvm_v32i8_ty, llvm_i32_ty],
+                    [IntrNoMem]>;
+  def int_x86_avx512_mask_pmov_wb_mem_512 :
+          GCCBuiltin<"__builtin_ia32_pmovwb512mem_mask">,
+          Intrinsic<[],
+                    [llvm_ptr_ty, llvm_v32i16_ty, llvm_i32_ty],
+                    [IntrReadWriteArgMem]>;
+  def int_x86_avx512_mask_pmovs_wb_512 :
+          GCCBuiltin<"__builtin_ia32_pmovswb512_mask">,
+          Intrinsic<[llvm_v32i8_ty],
+                    [llvm_v32i16_ty, llvm_v32i8_ty, llvm_i32_ty],
+                    [IntrNoMem]>;
+  def int_x86_avx512_mask_pmovs_wb_mem_512 :
+          GCCBuiltin<"__builtin_ia32_pmovswb512mem_mask">,
+          Intrinsic<[],
+                    [llvm_ptr_ty, llvm_v32i16_ty, llvm_i32_ty],
+                    [IntrReadWriteArgMem]>;
+  def int_x86_avx512_mask_pmovus_wb_512 :
+          GCCBuiltin<"__builtin_ia32_pmovuswb512_mask">,
+          Intrinsic<[llvm_v32i8_ty],
+                    [llvm_v32i16_ty, llvm_v32i8_ty, llvm_i32_ty],
+                    [IntrNoMem]>;
+  def int_x86_avx512_mask_pmovus_wb_mem_512 :
+          GCCBuiltin<"__builtin_ia32_pmovuswb512mem_mask">,
+          Intrinsic<[],
+                    [llvm_ptr_ty, llvm_v32i16_ty, llvm_i32_ty],
+                    [IntrReadWriteArgMem]>;
+}
 // Misc.
 let TargetPrefix = "x86" in {
-  def int_x86_avx512_mask_cmp_ps_512 : GCCBuiltin<"__builtin_ia32_cmpps512_mask">,
-            Intrinsic<[llvm_i16_ty], [llvm_v16f32_ty, llvm_v16f32_ty, llvm_i8_ty,
-                                      llvm_i16_ty, llvm_i32_ty], [IntrNoMem]>;
-  def int_x86_avx512_mask_cmp_pd_512 : GCCBuiltin<"__builtin_ia32_cmppd512_mask">,
-            Intrinsic<[llvm_i8_ty], [llvm_v8f64_ty, llvm_v8f64_ty, llvm_i8_ty,
-                                      llvm_i8_ty, llvm_i32_ty], [IntrNoMem]>;
-  def int_x86_avx512_movntdqa : GCCBuiltin<"__builtin_ia32_movntdqa512">,
+  def int_x86_avx512_mask_cmp_ps_512 :
+        GCCBuiltin<"__builtin_ia32_cmpps512_mask">,
+              Intrinsic<[llvm_i16_ty], [llvm_v16f32_ty, llvm_v16f32_ty,
+                         llvm_i32_ty, llvm_i16_ty, llvm_i32_ty], [IntrNoMem]>;
+  def int_x86_avx512_mask_cmp_pd_512 :
+        GCCBuiltin<"__builtin_ia32_cmppd512_mask">,
+              Intrinsic<[llvm_i8_ty], [llvm_v8f64_ty, llvm_v8f64_ty,
+                         llvm_i32_ty, llvm_i8_ty, llvm_i32_ty], [IntrNoMem]>;
+  def int_x86_avx512_mask_cmp_ps_256 :
+        GCCBuiltin<"__builtin_ia32_cmpps256_mask">,
+              Intrinsic<[llvm_i8_ty], [llvm_v8f32_ty, llvm_v8f32_ty,
+                         llvm_i32_ty, llvm_i8_ty], [IntrNoMem]>;
+  def int_x86_avx512_mask_cmp_pd_256 :
+        GCCBuiltin<"__builtin_ia32_cmppd256_mask">,
+              Intrinsic<[llvm_i8_ty], [llvm_v4f64_ty, llvm_v4f64_ty,
+                         llvm_i32_ty, llvm_i8_ty], [IntrNoMem]>;
+  def int_x86_avx512_mask_cmp_ps_128 :
+        GCCBuiltin<"__builtin_ia32_cmpps128_mask">,
+            Intrinsic<[llvm_i8_ty], [llvm_v4f32_ty, llvm_v4f32_ty,
+                       llvm_i32_ty, llvm_i8_ty], [IntrNoMem]>;
+  def int_x86_avx512_mask_cmp_pd_128 :
+        GCCBuiltin<"__builtin_ia32_cmppd128_mask">,
+            Intrinsic<[llvm_i8_ty], [llvm_v2f64_ty, llvm_v2f64_ty,
+                       llvm_i32_ty, llvm_i8_ty], [IntrNoMem]>;
+
+  def int_x86_avx512_movntdqa :
+        GCCBuiltin<"__builtin_ia32_movntdqa512">,
             Intrinsic<[llvm_v8i64_ty], [llvm_ptr_ty], [IntrReadMem]>;
 }
 
diff --git a/include/llvm/IR/LLVMContext.h b/include/llvm/IR/LLVMContext.h
index 8c04b08c4fa..e6c22090ab6 100644
--- a/include/llvm/IR/LLVMContext.h
+++ b/include/llvm/IR/LLVMContext.h
@@ -58,7 +58,9 @@ class LLVMContext {
     MD_noalias = 8, // "noalias",
     MD_nontemporal = 9, // "nontemporal"
     MD_mem_parallel_loop_access = 10, // "llvm.mem.parallel_loop_access"
-    MD_nonnull = 11 // "nonnull"
+    MD_nonnull = 11, // "nonnull"
+    MD_dereferenceable = 12, // "dereferenceable"
+    MD_dereferenceable_or_null = 13 // "dereferenceable_or_null"
   };
 
   /// getMDKindID - Return a unique non-zero ID for the specified metadata kind.
diff --git a/include/llvm/IR/MDBuilder.h b/include/llvm/IR/MDBuilder.h
index 51b8b02d8e6..ceb1c736e5c 100644
--- a/include/llvm/IR/MDBuilder.h
+++ b/include/llvm/IR/MDBuilder.h
@@ -60,6 +60,9 @@ class MDBuilder {
   /// \brief Return metadata containing a number of branch weights.
   MDNode *createBranchWeights(ArrayRef<uint32_t> Weights);
 
+  /// Return metadata containing the entry count for a function.
+  MDNode *createFunctionEntryCount(uint64_t Count);
+
   //===------------------------------------------------------------------===//
   // Range metadata.
   //===------------------------------------------------------------------===//
@@ -150,7 +153,7 @@ class MDBuilder {
   /// \brief Return metadata for a TBAA tag node with the given
   /// base type, access type and offset relative to the base type.
   MDNode *createTBAAStructTagNode(MDNode *BaseType, MDNode *AccessType,
-                                  uint64_t Offset);
+                                  uint64_t Offset, bool IsConstant = false);
 };
 
 } // end namespace llvm
diff --git a/include/llvm/IR/Mangler.h b/include/llvm/IR/Mangler.h
index 1e6b5b1dca0..b72b259097c 100644
--- a/include/llvm/IR/Mangler.h
+++ b/include/llvm/IR/Mangler.h
@@ -25,28 +25,15 @@ template <typename T> class SmallVectorImpl;
 class Twine;
 
 class Mangler {
-public:
-  enum ManglerPrefixTy {
-    Default,               ///< Emit default string before each symbol.
-    Private,               ///< Emit "private" prefix before each symbol.
-    LinkerPrivate          ///< Emit "linker private" prefix before each symbol.
-  };
-
-private:
-  const DataLayout *DL;
-
-  /// AnonGlobalIDs - We need to give global values the same name every time
-  /// they are mangled.  This keeps track of the number we give to anonymous
-  /// ones.
-  ///
+  /// We need to give global values the same name every time they are mangled.
+  /// This keeps track of the number we give to anonymous ones.
   mutable DenseMap<const GlobalValue*, unsigned> AnonGlobalIDs;
 
-  /// NextAnonGlobalID - This simple counter is used to unique value names.
-  ///
+  /// This simple counter is used to unique value names.
   mutable unsigned NextAnonGlobalID;
 
 public:
-  Mangler(const DataLayout *DL) : DL(DL), NextAnonGlobalID(1) {}
+  Mangler() : NextAnonGlobalID(1) {}
 
   /// Print the appropriate prefix and the specified global variable's name.
   /// If the global variable doesn't have a name, this fills in a unique name
@@ -58,10 +45,10 @@ class Mangler {
 
   /// Print the appropriate prefix and the specified name as the global variable
   /// name. GVName must not be empty.
-  void getNameWithPrefix(raw_ostream &OS, const Twine &GVName,
-                         ManglerPrefixTy PrefixTy = Mangler::Default) const;
-  void getNameWithPrefix(SmallVectorImpl<char> &OutName, const Twine &GVName,
-                         ManglerPrefixTy PrefixTy = Mangler::Default) const;
+  static void getNameWithPrefix(raw_ostream &OS, const Twine &GVName,
+                                const DataLayout &DL);
+  static void getNameWithPrefix(SmallVectorImpl<char> &OutName,
+                                const Twine &GVName, const DataLayout &DL);
 };
 
 } // End llvm namespace
diff --git a/include/llvm/IR/Metadata.def b/include/llvm/IR/Metadata.def
index 0eb29bcad14..c88c87e4e48 100644
--- a/include/llvm/IR/Metadata.def
+++ b/include/llvm/IR/Metadata.def
@@ -60,36 +60,35 @@ HANDLE_METADATA_LEAF(ConstantAsMetadata)
 HANDLE_METADATA_LEAF(LocalAsMetadata)
 HANDLE_MDNODE_BRANCH(MDNode)
 HANDLE_MDNODE_LEAF(MDTuple)
-HANDLE_SPECIALIZED_MDNODE_LEAF(MDLocation)
-HANDLE_SPECIALIZED_MDNODE_LEAF(MDExpression)
-HANDLE_SPECIALIZED_MDNODE_BRANCH(DebugNode)
-HANDLE_SPECIALIZED_MDNODE_LEAF(GenericDebugNode)
-HANDLE_SPECIALIZED_MDNODE_LEAF(MDSubrange)
-HANDLE_SPECIALIZED_MDNODE_LEAF(MDEnumerator)
-HANDLE_SPECIALIZED_MDNODE_BRANCH(MDScope)
-HANDLE_SPECIALIZED_MDNODE_BRANCH(MDType)
-HANDLE_SPECIALIZED_MDNODE_LEAF(MDBasicType)
-HANDLE_SPECIALIZED_MDNODE_BRANCH(MDDerivedTypeBase)
-HANDLE_SPECIALIZED_MDNODE_LEAF(MDDerivedType)
-HANDLE_SPECIALIZED_MDNODE_BRANCH(MDCompositeTypeBase)
-HANDLE_SPECIALIZED_MDNODE_LEAF(MDCompositeType)
-HANDLE_SPECIALIZED_MDNODE_LEAF(MDSubroutineType)
-HANDLE_SPECIALIZED_MDNODE_LEAF(MDFile)
-HANDLE_SPECIALIZED_MDNODE_LEAF(MDCompileUnit)
-HANDLE_SPECIALIZED_MDNODE_BRANCH(MDLocalScope)
-HANDLE_SPECIALIZED_MDNODE_LEAF(MDSubprogram)
-HANDLE_SPECIALIZED_MDNODE_BRANCH(MDLexicalBlockBase)
-HANDLE_SPECIALIZED_MDNODE_LEAF(MDLexicalBlock)
-HANDLE_SPECIALIZED_MDNODE_LEAF(MDLexicalBlockFile)
-HANDLE_SPECIALIZED_MDNODE_LEAF(MDNamespace)
-HANDLE_SPECIALIZED_MDNODE_BRANCH(MDTemplateParameter)
-HANDLE_SPECIALIZED_MDNODE_LEAF(MDTemplateTypeParameter)
-HANDLE_SPECIALIZED_MDNODE_LEAF(MDTemplateValueParameter)
-HANDLE_SPECIALIZED_MDNODE_BRANCH(MDVariable)
-HANDLE_SPECIALIZED_MDNODE_LEAF(MDGlobalVariable)
-HANDLE_SPECIALIZED_MDNODE_LEAF(MDLocalVariable)
-HANDLE_SPECIALIZED_MDNODE_LEAF(MDObjCProperty)
-HANDLE_SPECIALIZED_MDNODE_LEAF(MDImportedEntity)
+HANDLE_SPECIALIZED_MDNODE_LEAF(DILocation)
+HANDLE_SPECIALIZED_MDNODE_LEAF(DIExpression)
+HANDLE_SPECIALIZED_MDNODE_BRANCH(DINode)
+HANDLE_SPECIALIZED_MDNODE_LEAF(GenericDINode)
+HANDLE_SPECIALIZED_MDNODE_LEAF(DISubrange)
+HANDLE_SPECIALIZED_MDNODE_LEAF(DIEnumerator)
+HANDLE_SPECIALIZED_MDNODE_BRANCH(DIScope)
+HANDLE_SPECIALIZED_MDNODE_BRANCH(DIType)
+HANDLE_SPECIALIZED_MDNODE_LEAF(DIBasicType)
+HANDLE_SPECIALIZED_MDNODE_LEAF(DIDerivedType)
+HANDLE_SPECIALIZED_MDNODE_LEAF(DICompositeType)
+HANDLE_SPECIALIZED_MDNODE_LEAF(DISubroutineType)
+HANDLE_SPECIALIZED_MDNODE_LEAF(DIFile)
+HANDLE_SPECIALIZED_MDNODE_LEAF(DICompileUnit)
+HANDLE_SPECIALIZED_MDNODE_BRANCH(DILocalScope)
+HANDLE_SPECIALIZED_MDNODE_LEAF(DISubprogram)
+HANDLE_SPECIALIZED_MDNODE_BRANCH(DILexicalBlockBase)
+HANDLE_SPECIALIZED_MDNODE_LEAF(DILexicalBlock)
+HANDLE_SPECIALIZED_MDNODE_LEAF(DILexicalBlockFile)
+HANDLE_SPECIALIZED_MDNODE_LEAF(DINamespace)
+HANDLE_SPECIALIZED_MDNODE_LEAF(DIModule)
+HANDLE_SPECIALIZED_MDNODE_BRANCH(DITemplateParameter)
+HANDLE_SPECIALIZED_MDNODE_LEAF(DITemplateTypeParameter)
+HANDLE_SPECIALIZED_MDNODE_LEAF(DITemplateValueParameter)
+HANDLE_SPECIALIZED_MDNODE_BRANCH(DIVariable)
+HANDLE_SPECIALIZED_MDNODE_LEAF(DIGlobalVariable)
+HANDLE_SPECIALIZED_MDNODE_LEAF(DILocalVariable)
+HANDLE_SPECIALIZED_MDNODE_LEAF(DIObjCProperty)
+HANDLE_SPECIALIZED_MDNODE_LEAF(DIImportedEntity)
 
 #undef HANDLE_METADATA
 #undef HANDLE_METADATA_LEAF
diff --git a/include/llvm/IR/Metadata.h b/include/llvm/IR/Metadata.h
index 03e70fe2671..c639625bf16 100644
--- a/include/llvm/IR/Metadata.h
+++ b/include/llvm/IR/Metadata.h
@@ -27,8 +27,11 @@
 #include <type_traits>
 
 namespace llvm {
+
 class LLVMContext;
 class Module;
+class ModuleSlotTracker;
+
 template<typename ValueSubClass, typename ItemParentClass>
   class SymbolTableListTraits;
 
@@ -59,27 +62,28 @@ class Metadata {
 public:
   enum MetadataKind {
     MDTupleKind,
-    MDLocationKind,
-    GenericDebugNodeKind,
-    MDSubrangeKind,
-    MDEnumeratorKind,
-    MDBasicTypeKind,
-    MDDerivedTypeKind,
-    MDCompositeTypeKind,
-    MDSubroutineTypeKind,
-    MDFileKind,
-    MDCompileUnitKind,
-    MDSubprogramKind,
-    MDLexicalBlockKind,
-    MDLexicalBlockFileKind,
-    MDNamespaceKind,
-    MDTemplateTypeParameterKind,
-    MDTemplateValueParameterKind,
-    MDGlobalVariableKind,
-    MDLocalVariableKind,
-    MDExpressionKind,
-    MDObjCPropertyKind,
-    MDImportedEntityKind,
+    DILocationKind,
+    GenericDINodeKind,
+    DISubrangeKind,
+    DIEnumeratorKind,
+    DIBasicTypeKind,
+    DIDerivedTypeKind,
+    DICompositeTypeKind,
+    DISubroutineTypeKind,
+    DIFileKind,
+    DICompileUnitKind,
+    DISubprogramKind,
+    DILexicalBlockKind,
+    DILexicalBlockFileKind,
+    DINamespaceKind,
+    DIModuleKind,
+    DITemplateTypeParameterKind,
+    DITemplateValueParameterKind,
+    DIGlobalVariableKind,
+    DILocalVariableKind,
+    DIExpressionKind,
+    DIObjCPropertyKind,
+    DIImportedEntityKind,
     ConstantAsMetadataKind,
     LocalAsMetadataKind,
     MDStringKind
@@ -121,7 +125,11 @@ class Metadata {
   ///
   /// If \c M is provided, metadata nodes will be numbered canonically;
   /// otherwise, pointer addresses are substituted.
+  /// @{
   void print(raw_ostream &OS, const Module *M = nullptr) const;
+  void print(raw_ostream &OS, ModuleSlotTracker &MST,
+             const Module *M = nullptr) const;
+  /// @}
 
   /// \brief Print as operand.
   ///
@@ -129,7 +137,11 @@ class Metadata {
   ///
   /// If \c M is provided, metadata nodes will be numbered canonically;
   /// otherwise, pointer addresses are substituted.
+  /// @{
   void printAsOperand(raw_ostream &OS, const Module *M = nullptr) const;
+  void printAsOperand(raw_ostream &OS, ModuleSlotTracker &MST,
+                      const Module *M = nullptr) const;
+  /// @}
 };
 
 #define HANDLE_METADATA(CLASS) class CLASS;
diff --git a/include/llvm/IR/Module.h b/include/llvm/IR/Module.h
index dbaf322263d..1668b95c8bd 100644
--- a/include/llvm/IR/Module.h
+++ b/include/llvm/IR/Module.h
@@ -249,7 +249,7 @@ class Module {
 
   /// Get the data layout string for the module's target platform. This is
   /// equivalent to getDataLayout()->getStringRepresentation().
-  const std::string getDataLayoutStr() const {
+  const std::string &getDataLayoutStr() const {
     return DL.getStringRepresentation();
   }
 
@@ -492,7 +492,7 @@ class Module {
   /// If the GlobalValue is read in, and if the GVMaterializer supports it,
   /// release the memory for the function, and set it up to be materialized
   /// lazily. If !isDematerializable(), this method is a no-op.
-  void Dematerialize(GlobalValue *GV);
+  void dematerialize(GlobalValue *GV);
 
   /// Make sure all GlobalValues in this Module are fully read.
   std::error_code materializeAll();
@@ -512,28 +512,28 @@ class Module {
   const GlobalListType   &getGlobalList() const       { return GlobalList; }
   /// Get the Module's list of global variables.
   GlobalListType         &getGlobalList()             { return GlobalList; }
-  static iplist<GlobalVariable> Module::*getSublistAccess(GlobalVariable*) {
+  static GlobalListType Module::*getSublistAccess(GlobalVariable*) {
     return &Module::GlobalList;
   }
   /// Get the Module's list of functions (constant).
   const FunctionListType &getFunctionList() const     { return FunctionList; }
   /// Get the Module's list of functions.
   FunctionListType       &getFunctionList()           { return FunctionList; }
-  static iplist<Function> Module::*getSublistAccess(Function*) {
+  static FunctionListType Module::*getSublistAccess(Function*) {
     return &Module::FunctionList;
   }
   /// Get the Module's list of aliases (constant).
   const AliasListType    &getAliasList() const        { return AliasList; }
   /// Get the Module's list of aliases.
   AliasListType          &getAliasList()              { return AliasList; }
-  static iplist<GlobalAlias> Module::*getSublistAccess(GlobalAlias*) {
+  static AliasListType Module::*getSublistAccess(GlobalAlias*) {
     return &Module::AliasList;
   }
   /// Get the Module's list of named metadata (constant).
   const NamedMDListType  &getNamedMDList() const      { return NamedMDList; }
   /// Get the Module's list of named metadata.
   NamedMDListType        &getNamedMDList()            { return NamedMDList; }
-  static ilist<NamedMDNode> Module::*getSublistAccess(NamedMDNode*) {
+  static NamedMDListType Module::*getSublistAccess(NamedMDNode*) {
     return &Module::NamedMDList;
   }
   /// Get the symbol table of global variable and function identifiers
diff --git a/include/llvm/IR/ModuleSlotTracker.h b/include/llvm/IR/ModuleSlotTracker.h
new file mode 100644
index 00000000000..49730a66bdf
--- /dev/null
+++ b/include/llvm/IR/ModuleSlotTracker.h
@@ -0,0 +1,76 @@
+//===-- llvm/IR/ModuleSlotTracker.h -----------------------------*- C++ -*-===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_IR_MODULESLOTTRACKER_H
+#define LLVM_IR_MODULESLOTTRACKER_H
+
+#include <memory>
+
+namespace llvm {
+
+class Module;
+class Function;
+class SlotTracker;
+class Value;
+
+/// Manage lifetime of a slot tracker for printing IR.
+///
+/// Wrapper around the \a SlotTracker used internally by \a AsmWriter.  This
+/// class allows callers to share the cost of incorporating the metadata in a
+/// module or a function.
+///
+/// If the IR changes from underneath \a ModuleSlotTracker, strings like
+/// "<badref>" will be printed, or, worse, the wrong slots entirely.
+class ModuleSlotTracker {
+  /// Storage for a slot tracker.
+  std::unique_ptr<SlotTracker> MachineStorage;
+
+  const Module *M = nullptr;
+  const Function *F = nullptr;
+  SlotTracker *Machine = nullptr;
+
+public:
+  /// Wrap a preinitialized SlotTracker.
+  ModuleSlotTracker(SlotTracker &Machine, const Module *M,
+                    const Function *F = nullptr);
+
+  /// Construct a slot tracker from a module.
+  ///
+  /// If \a M is \c nullptr, uses a null slot tracker.  Otherwise, initializes
+  /// a slot tracker, and initializes all metadata slots.  \c
+  /// ShouldInitializeAllMetadata defaults to true because this is expected to
+  /// be shared between multiple callers, and otherwise MDNode references will
+  /// not match up.
+  explicit ModuleSlotTracker(const Module *M,
+                             bool ShouldInitializeAllMetadata = true);
+
+  /// Destructor to clean up storage.
+  ~ModuleSlotTracker();
+
+  SlotTracker *getMachine() const { return Machine; }
+  const Module *getModule() const { return M; }
+  const Function *getCurrentFunction() const { return F; }
+
+  /// Incorporate the given function.
+  ///
+  /// Purge the currently incorporated function and incorporate \c F.  If \c F
+  /// is currently incorporated, this is a no-op.
+  void incorporateFunction(const Function &F);
+
+  /// Return the slot number of the specified local value.
+  ///
+  /// A function that defines this value should be incorporated prior to calling
+  /// this method.
+  /// Return -1 if the value is not in the function's SlotTracker.
+  int getLocalSlot(const Value *V);
+};
+
+} // end namespace llvm
+
+#endif
diff --git a/include/llvm/IR/NaClAtomicIntrinsics.h b/include/llvm/IR/NaClAtomicIntrinsics.h
index a0091b84557..e820b9df504 100644
--- a/include/llvm/IR/NaClAtomicIntrinsics.h
+++ b/include/llvm/IR/NaClAtomicIntrinsics.h
@@ -40,7 +40,7 @@ class AtomicIntrinsics {
 
   struct AtomicIntrinsic {
     Type *OverloadedType;
-    Intrinsic::ID ID : 16;
+    Intrinsic::ID ID;
     uint8_t Overloaded : 1;
     uint8_t NumParams : 7;
     uint8_t ParamType[MaxAtomicIntrinsicsParameters];
diff --git a/include/llvm/IR/OperandTraits.h b/include/llvm/IR/OperandTraits.h
index 0e4b1950f27..e97a8009ccc 100644
--- a/include/llvm/IR/OperandTraits.h
+++ b/include/llvm/IR/OperandTraits.h
@@ -92,10 +92,10 @@ struct VariadicOperandTraits {
 template <unsigned MINARITY = 1>
 struct HungoffOperandTraits {
   static Use *op_begin(User* U) {
-    return U->OperandList;
+    return U->getOperandList();
   }
   static Use *op_end(User* U) {
-    return U->OperandList + U->getNumOperands();
+    return U->getOperandList() + U->getNumOperands();
   }
   static unsigned operands(const User *U) {
     return U->getNumOperands();
diff --git a/include/llvm/IR/Operator.h b/include/llvm/IR/Operator.h
index 8c3afdd86ae..372b254ab18 100644
--- a/include/llvm/IR/Operator.h
+++ b/include/llvm/IR/Operator.h
@@ -18,7 +18,6 @@
 #include "llvm/IR/Constants.h"
 #include "llvm/IR/DataLayout.h"
 #include "llvm/IR/DerivedTypes.h"
-#include "llvm/IR/GetElementPtrTypeIterator.h"
 #include "llvm/IR/Instruction.h"
 #include "llvm/IR/Type.h"
 
@@ -306,7 +305,8 @@ class FPMathOperator : public Operator {
   float getFPAccuracy() const;
 
   static inline bool classof(const Instruction *I) {
-    return I->getType()->isFPOrFPVectorTy();
+    return I->getType()->isFPOrFPVectorTy() ||
+      I->getOpcode() == Instruction::FCmp;
   }
   static inline bool classof(const Value *V) {
     return isa<Instruction>(V) && classof(cast<Instruction>(V));
@@ -400,10 +400,7 @@ class GEPOperator
     return getPointerOperand()->getType();
   }
 
-  Type *getSourceElementType() const {
-    return cast<SequentialType>(getPointerOperandType()->getScalarType())
-        ->getElementType();
-  }
+  Type *getSourceElementType() const;
 
   /// Method to return the address space of the pointer operand.
   unsigned getPointerAddressSpace() const {
@@ -450,36 +447,7 @@ class GEPOperator
   /// undefined (it is *not* preserved!). The APInt passed into this routine
   /// must be at exactly as wide as the IntPtr type for the address space of the
   /// base GEP pointer.
-  bool accumulateConstantOffset(const DataLayout &DL, APInt &Offset) const {
-    assert(Offset.getBitWidth() ==
-           DL.getPointerSizeInBits(getPointerAddressSpace()) &&
-           "The offset must have exactly as many bits as our pointer.");
-
-    for (gep_type_iterator GTI = gep_type_begin(this), GTE = gep_type_end(this);
-         GTI != GTE; ++GTI) {
-      ConstantInt *OpC = dyn_cast<ConstantInt>(GTI.getOperand());
-      if (!OpC)
-        return false;
-      if (OpC->isZero())
-        continue;
-
-      // Handle a struct index, which adds its field offset to the pointer.
-      if (StructType *STy = dyn_cast<StructType>(*GTI)) {
-        unsigned ElementIdx = OpC->getZExtValue();
-        const StructLayout *SL = DL.getStructLayout(STy);
-        Offset += APInt(Offset.getBitWidth(),
-                        SL->getElementOffset(ElementIdx));
-        continue;
-      }
-
-      // For array or vector indices, scale the index by the size of the type.
-      APInt Index = OpC->getValue().sextOrTrunc(Offset.getBitWidth());
-      Offset += Index * APInt(Offset.getBitWidth(),
-                              DL.getTypeAllocSize(GTI.getIndexedType()));
-    }
-    return true;
-  }
-
+  bool accumulateConstantOffset(const DataLayout &DL, APInt &Offset) const;
 };
 
 class PtrToIntOperator
diff --git a/include/llvm/IR/PassManager.h b/include/llvm/IR/PassManager.h
index 3c24e7231a1..4166babd63e 100644
--- a/include/llvm/IR/PassManager.h
+++ b/include/llvm/IR/PassManager.h
@@ -509,7 +509,7 @@ class AnalysisManager
   PreservedAnalyses invalidateImpl(IRUnitT &IR, PreservedAnalyses PA) {
     // Short circuit for a common case of all analyses being preserved.
     if (PA.areAllPreserved())
-      return std::move(PA);
+      return PA;
 
     if (DebugLogging)
       dbgs() << "Invalidating all non-preserved analyses for: "
@@ -549,7 +549,7 @@ class AnalysisManager
     if (ResultsList.empty())
       AnalysisResultLists.erase(&IR);
 
-    return std::move(PA);
+    return PA;
   }
 
   /// \brief List of function analysis pass IDs and associated concept pointers.
@@ -827,7 +827,7 @@ template <typename FunctionPassT> class ModuleToFunctionPassAdaptor {
 template <typename FunctionPassT>
 ModuleToFunctionPassAdaptor<FunctionPassT>
 createModuleToFunctionPassAdaptor(FunctionPassT Pass) {
-  return std::move(ModuleToFunctionPassAdaptor<FunctionPassT>(std::move(Pass)));
+  return ModuleToFunctionPassAdaptor<FunctionPassT>(std::move(Pass));
 }
 
 /// \brief A template utility pass to force an analysis result to be available.
diff --git a/include/llvm/IR/PredIteratorCache.h b/include/llvm/IR/PredIteratorCache.h
index 5e1be37805f..118310aed1d 100644
--- a/include/llvm/IR/PredIteratorCache.h
+++ b/include/llvm/IR/PredIteratorCache.h
@@ -14,6 +14,7 @@
 #ifndef LLVM_IR_PREDITERATORCACHE_H
 #define LLVM_IR_PREDITERATORCACHE_H
 
+#include "llvm/ADT/ArrayRef.h"
 #include "llvm/ADT/DenseMap.h"
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/IR/CFG.h"
@@ -21,50 +22,58 @@
 
 namespace llvm {
 
-  /// PredIteratorCache - This class is an extremely trivial cache for
-  /// predecessor iterator queries.  This is useful for code that repeatedly
-  /// wants the predecessor list for the same blocks.
-  class PredIteratorCache {
-    /// BlockToPredsMap - Pointer to null-terminated list.
-    DenseMap<BasicBlock*, BasicBlock**> BlockToPredsMap;
-    DenseMap<BasicBlock*, unsigned> BlockToPredCountMap;
+/// PredIteratorCache - This class is an extremely trivial cache for
+/// predecessor iterator queries.  This is useful for code that repeatedly
+/// wants the predecessor list for the same blocks.
+class PredIteratorCache {
+  /// BlockToPredsMap - Pointer to null-terminated list.
+  DenseMap<BasicBlock *, BasicBlock **> BlockToPredsMap;
+  DenseMap<BasicBlock *, unsigned> BlockToPredCountMap;
 
-    /// Memory - This is the space that holds cached preds.
-    BumpPtrAllocator Memory;
-  public:
+  /// Memory - This is the space that holds cached preds.
+  BumpPtrAllocator Memory;
 
-    /// GetPreds - Get a cached list for the null-terminated predecessor list of
-    /// the specified block.  This can be used in a loop like this:
-    ///   for (BasicBlock **PI = PredCache->GetPreds(BB); *PI; ++PI)
-    ///      use(*PI);
-    /// instead of:
-    /// for (pred_iterator PI = pred_begin(BB), E = pred_end(BB); PI != E; ++PI)
-    BasicBlock **GetPreds(BasicBlock *BB) {
-      BasicBlock **&Entry = BlockToPredsMap[BB];
-      if (Entry) return Entry;
+private:
+  /// GetPreds - Get a cached list for the null-terminated predecessor list of
+  /// the specified block.  This can be used in a loop like this:
+  ///   for (BasicBlock **PI = PredCache->GetPreds(BB); *PI; ++PI)
+  ///      use(*PI);
+  /// instead of:
+  /// for (pred_iterator PI = pred_begin(BB), E = pred_end(BB); PI != E; ++PI)
+  BasicBlock **GetPreds(BasicBlock *BB) {
+    BasicBlock **&Entry = BlockToPredsMap[BB];
+    if (Entry)
+      return Entry;
 
-      SmallVector<BasicBlock*, 32> PredCache(pred_begin(BB), pred_end(BB));
-      PredCache.push_back(nullptr); // null terminator.
-      
-      BlockToPredCountMap[BB] = PredCache.size()-1;
+    SmallVector<BasicBlock *, 32> PredCache(pred_begin(BB), pred_end(BB));
+    PredCache.push_back(nullptr); // null terminator.
 
-      Entry = Memory.Allocate<BasicBlock*>(PredCache.size());
-      std::copy(PredCache.begin(), PredCache.end(), Entry);
-      return Entry;
-    }
-    
-    unsigned GetNumPreds(BasicBlock *BB) {
-      GetPreds(BB);
-      return BlockToPredCountMap[BB];
-    }
+    BlockToPredCountMap[BB] = PredCache.size() - 1;
+
+    Entry = Memory.Allocate<BasicBlock *>(PredCache.size());
+    std::copy(PredCache.begin(), PredCache.end(), Entry);
+    return Entry;
+  }
+
+  unsigned GetNumPreds(BasicBlock *BB) {
+    GetPreds(BB);
+    return BlockToPredCountMap[BB];
+  }
+
+public:
+  size_t size(BasicBlock *BB) { return GetNumPreds(BB); }
+  ArrayRef<BasicBlock *> get(BasicBlock *BB) {
+    return makeArrayRef(GetPreds(BB), GetNumPreds(BB));
+  }
+
+  /// clear - Remove all information.
+  void clear() {
+    BlockToPredsMap.clear();
+    BlockToPredCountMap.clear();
+    Memory.Reset();
+  }
+};
 
-    /// clear - Remove all information.
-    void clear() {
-      BlockToPredsMap.clear();
-      BlockToPredCountMap.clear();
-      Memory.Reset();
-    }
-  };
 } // end namespace llvm
 
 #endif
diff --git a/include/llvm/IR/Statepoint.h b/include/llvm/IR/Statepoint.h
index 38720edabad..4ab1f8497ad 100644
--- a/include/llvm/IR/Statepoint.h
+++ b/include/llvm/IR/Statepoint.h
@@ -13,29 +13,40 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef __LLVM_IR_STATEPOINT_H
-#define __LLVM_IR_STATEPOINT_H
+#ifndef LLVM_IR_STATEPOINT_H
+#define LLVM_IR_STATEPOINT_H
 
 #include "llvm/ADT/iterator_range.h"
 #include "llvm/IR/BasicBlock.h"
 #include "llvm/IR/CallSite.h"
+#include "llvm/IR/Constants.h"
+#include "llvm/IR/Function.h"
 #include "llvm/IR/Instructions.h"
 #include "llvm/IR/Intrinsics.h"
 #include "llvm/Support/Compiler.h"
 
 namespace llvm {
+/// The statepoint intrinsic accepts a set of flags as its third argument.
+/// Valid values come out of this set.
+enum class StatepointFlags {
+  None = 0,
+  GCTransition = 1, ///< Indicates that this statepoint is a transition from
+                    ///< GC-aware code to code that is not GC-aware.
+
+  MaskAll = GCTransition ///< A bitmask that includes all valid flags.
+};
 
 class GCRelocateOperands;
 class ImmutableStatepoint;
 
 bool isStatepoint(const ImmutableCallSite &CS);
-bool isStatepoint(const Value *inst);
-bool isStatepoint(const Value &inst);
+bool isStatepoint(const Value *V);
+bool isStatepoint(const Value &V);
 
-bool isGCRelocate(const Value *inst);
+bool isGCRelocate(const Value *V);
 bool isGCRelocate(const ImmutableCallSite &CS);
 
-bool isGCResult(const Value *inst);
+bool isGCResult(const Value *V);
 bool isGCResult(const ImmutableCallSite &CS);
 
 /// Analogous to CallSiteBase, this provides most of the actual
@@ -44,90 +55,188 @@ bool isGCResult(const ImmutableCallSite &CS);
 /// concrete subtypes.  This is structured analogous to CallSite
 /// rather than the IntrinsicInst.h helpers since we want to support
 /// invokable statepoints in the near future.
-/// TODO: This does not currently allow the if(Statepoint S = ...)
-///   idiom used with CallSites.  Consider refactoring to support.
-template <typename InstructionTy, typename ValueTy, typename CallSiteTy>
+template <typename FunTy, typename InstructionTy, typename ValueTy,
+          typename CallSiteTy>
 class StatepointBase {
   CallSiteTy StatepointCS;
   void *operator new(size_t, unsigned) = delete;
   void *operator new(size_t s) = delete;
 
- protected:
-  explicit StatepointBase(InstructionTy *I) : StatepointCS(I) {
-    assert(isStatepoint(I));
+protected:
+  explicit StatepointBase(InstructionTy *I) {
+    if (isStatepoint(I)) {
+      StatepointCS = CallSiteTy(I);
+      assert(StatepointCS && "isStatepoint implies CallSite");
+    }
   }
-  explicit StatepointBase(CallSiteTy CS) : StatepointCS(CS) {
-    assert(isStatepoint(CS));
+  explicit StatepointBase(CallSiteTy CS) {
+    if (isStatepoint(CS))
+      StatepointCS = CS;
   }
 
- public:
+public:
   typedef typename CallSiteTy::arg_iterator arg_iterator;
 
+  enum {
+    IDPos = 0,
+    NumPatchBytesPos = 1,
+    CalledFunctionPos = 2,
+    NumCallArgsPos = 3,
+    FlagsPos = 4,
+    CallArgsBeginPos = 5,
+  };
+
+  explicit operator bool() const {
+    // We do not assign non-statepoint CallSites to StatepointCS.
+    return (bool)StatepointCS;
+  }
+
   /// Return the underlying CallSite.
-  CallSiteTy getCallSite() {
+  CallSiteTy getCallSite() const {
+    assert(*this && "check validity first!");
     return StatepointCS;
   }
 
+  uint64_t getFlags() const {
+    return cast<ConstantInt>(getCallSite().getArgument(FlagsPos))
+        ->getZExtValue();
+  }
+
+  /// Return the ID associated with this statepoint.
+  uint64_t getID() const {
+    const Value *IDVal = getCallSite().getArgument(IDPos);
+    return cast<ConstantInt>(IDVal)->getZExtValue();
+  }
+
+  /// Return the number of patchable bytes associated with this statepoint.
+  uint32_t getNumPatchBytes() const {
+    const Value *NumPatchBytesVal = getCallSite().getArgument(NumPatchBytesPos);
+    uint64_t NumPatchBytes =
+      cast<ConstantInt>(NumPatchBytesVal)->getZExtValue();
+    assert(isInt<32>(NumPatchBytes) && "should fit in 32 bits!");
+    return NumPatchBytes;
+  }
+
   /// Return the value actually being called or invoked.
-  ValueTy *actualCallee() {
-    return StatepointCS.getArgument(0);
+  ValueTy *getCalledValue() const {
+    return getCallSite().getArgument(CalledFunctionPos);
   }
-  /// Number of arguments to be passed to the actual callee.
-  int numCallArgs() {
-    return cast<ConstantInt>(StatepointCS.getArgument(1))->getZExtValue();
+
+  InstructionTy *getInstruction() const {
+    return getCallSite().getInstruction();
   }
-  /// Number of additional arguments excluding those intended
-  /// for garbage collection.
-  int numTotalVMSArgs() {
-    return cast<ConstantInt>(StatepointCS.getArgument(3 + numCallArgs()))->getZExtValue();
+
+  /// Return the function being called if this is a direct call, otherwise
+  /// return null (if it's an indirect call).
+  FunTy *getCalledFunction() const {
+    return dyn_cast<Function>(getCalledValue());
+  }
+
+  /// Return the caller function for this statepoint.
+  FunTy *getCaller() const { return getCallSite().getCaller(); }
+
+  /// Determine if the statepoint cannot unwind.
+  bool doesNotThrow() const {
+    Function *F = getCalledFunction();
+    return getCallSite().doesNotThrow() || (F ? F->doesNotThrow() : false);
   }
 
-  typename CallSiteTy::arg_iterator call_args_begin() {
-    // 3 = callTarget, #callArgs, flag
-    int Offset = 3;
-    assert(Offset <= (int)StatepointCS.arg_size());
-    return StatepointCS.arg_begin() + Offset;
+  /// Return the type of the value returned by the call underlying the
+  /// statepoint.
+  Type *getActualReturnType() const {
+    auto *FTy = cast<FunctionType>(
+        cast<PointerType>(getCalledValue()->getType())->getElementType());
+    return FTy->getReturnType();
   }
-  typename CallSiteTy::arg_iterator call_args_end() {
-    int Offset = 3 + numCallArgs();
-    assert(Offset <= (int)StatepointCS.arg_size());
-    return StatepointCS.arg_begin() + Offset;
+
+  /// Number of arguments to be passed to the actual callee.
+  int getNumCallArgs() const {
+    const Value *NumCallArgsVal = getCallSite().getArgument(NumCallArgsPos);
+    return cast<ConstantInt>(NumCallArgsVal)->getZExtValue();
+  }
+
+  size_t arg_size() const { return getNumCallArgs(); }
+  typename CallSiteTy::arg_iterator arg_begin() const {
+    assert(CallArgsBeginPos <= (int)getCallSite().arg_size());
+    return getCallSite().arg_begin() + CallArgsBeginPos;
+  }
+  typename CallSiteTy::arg_iterator arg_end() const {
+    auto I = arg_begin() + arg_size();
+    assert((getCallSite().arg_end() - I) >= 0);
+    return I;
+  }
+
+  ValueTy *getArgument(unsigned Index) {
+    assert(Index < arg_size() && "out of bounds!");
+    return *(arg_begin() + Index);
   }
 
   /// range adapter for call arguments
-  iterator_range<arg_iterator> call_args() {
-    return iterator_range<arg_iterator>(call_args_begin(), call_args_end());
+  iterator_range<arg_iterator> call_args() const {
+    return iterator_range<arg_iterator>(arg_begin(), arg_end());
   }
 
-  typename CallSiteTy::arg_iterator vm_state_begin() {
-    return call_args_end();
+  /// \brief Return true if the call or the callee has the given attribute.
+  bool paramHasAttr(unsigned i, Attribute::AttrKind A) const {
+    Function *F = getCalledFunction();
+    return getCallSite().paramHasAttr(i + CallArgsBeginPos, A) ||
+          (F ? F->getAttributes().hasAttribute(i, A) : false);
   }
-  typename CallSiteTy::arg_iterator vm_state_end() {
-    int Offset = 3 + numCallArgs() + 1 + numTotalVMSArgs();
-    assert(Offset <= (int)StatepointCS.arg_size());
-    return StatepointCS.arg_begin() + Offset;
+
+  /// Number of GC transition args.
+  int getNumTotalGCTransitionArgs() const {
+    const Value *NumGCTransitionArgs = *arg_end();
+    return cast<ConstantInt>(NumGCTransitionArgs)->getZExtValue();
+  }
+  typename CallSiteTy::arg_iterator gc_transition_args_begin() const {
+    auto I = arg_end() + 1;
+    assert((getCallSite().arg_end() - I) >= 0);
+    return I;
+  }
+  typename CallSiteTy::arg_iterator gc_transition_args_end() const {
+    auto I = gc_transition_args_begin() + getNumTotalGCTransitionArgs();
+    assert((getCallSite().arg_end() - I) >= 0);
+    return I;
   }
 
-  /// range adapter for vm state arguments
-  iterator_range<arg_iterator> vm_state_args() {
-    return iterator_range<arg_iterator>(vm_state_begin(), vm_state_end());
+  /// range adapter for GC transition arguments
+  iterator_range<arg_iterator> gc_transition_args() const {
+    return iterator_range<arg_iterator>(gc_transition_args_begin(),
+                                        gc_transition_args_end());
+  }
+
+  /// Number of additional arguments excluding those intended
+  /// for garbage collection.
+  int getNumTotalVMSArgs() const {
+    const Value *NumVMSArgs = *gc_transition_args_end();
+    return cast<ConstantInt>(NumVMSArgs)->getZExtValue();
   }
 
-  typename CallSiteTy::arg_iterator first_vm_state_stack_begin() {
-    // 6 = numTotalVMSArgs, 1st_objectID, 1st_bci,
-    //     1st_#stack, 1st_#local, 1st_#monitor
-    return vm_state_begin() + 6;
+  typename CallSiteTy::arg_iterator vm_state_begin() const {
+    auto I = gc_transition_args_end() + 1;
+    assert((getCallSite().arg_end() - I) >= 0);
+    return I;
+  }
+  typename CallSiteTy::arg_iterator vm_state_end() const {
+    auto I = vm_state_begin() + getNumTotalVMSArgs();
+    assert((getCallSite().arg_end() - I) >= 0);
+    return I;
   }
 
-  typename CallSiteTy::arg_iterator gc_args_begin() {
+  /// range adapter for vm state arguments
+  iterator_range<arg_iterator> vm_state_args() const {
+    return iterator_range<arg_iterator>(vm_state_begin(), vm_state_end());
+  }
+
+  typename CallSiteTy::arg_iterator gc_args_begin() const {
     return vm_state_end();
   }
-  typename CallSiteTy::arg_iterator gc_args_end() {
-    return StatepointCS.arg_end();
+  typename CallSiteTy::arg_iterator gc_args_end() const {
+    return getCallSite().arg_end();
   }
 
   /// range adapter for gc arguments
-  iterator_range<arg_iterator> gc_args() {
+  iterator_range<arg_iterator> gc_args() const {
     return iterator_range<arg_iterator>(gc_args_begin(), gc_args_end());
   }
 
@@ -135,19 +244,32 @@ class StatepointBase {
   /// May contain several relocations for the same base/derived pair.
   /// For example this could happen due to relocations on unwinding
   /// path of invoke.
-  std::vector<GCRelocateOperands> getRelocates(ImmutableStatepoint &IS);
+  std::vector<GCRelocateOperands> getRelocates() const;
+
+  /// Get the experimental_gc_result call tied to this statepoint.  Can be
+  /// nullptr if there isn't a gc_result tied to this statepoint.  Guaranteed to
+  /// be a CallInst if non-null.
+  InstructionTy *getGCResult() const {
+    for (auto *U : getInstruction()->users())
+      if (isGCResult(U))
+        return cast<CallInst>(U);
+
+    return nullptr;
+  }
 
 #ifndef NDEBUG
   /// Asserts if this statepoint is malformed.  Common cases for failure
   /// include incorrect length prefixes for variable length sections or
   /// illegal values for parameters.
   void verify() {
-    assert(numCallArgs() >= 0 &&
+    assert(getNumCallArgs() >= 0 &&
            "number of arguments to actually callee can't be negative");
 
     // The internal asserts in the iterator accessors do the rest.
-    (void)call_args_begin();
-    (void)call_args_end();
+    (void)arg_begin();
+    (void)arg_end();
+    (void)gc_transition_args_begin();
+    (void)gc_transition_args_end();
     (void)vm_state_begin();
     (void)vm_state_end();
     (void)gc_args_begin();
@@ -159,10 +281,10 @@ class StatepointBase {
 /// A specialization of it's base class for read only access
 /// to a gc.statepoint.
 class ImmutableStatepoint
-    : public StatepointBase<const Instruction, const Value,
+    : public StatepointBase<const Function, const Instruction, const Value,
                             ImmutableCallSite> {
-  typedef StatepointBase<const Instruction, const Value, ImmutableCallSite>
-      Base;
+  typedef StatepointBase<const Function, const Instruction, const Value,
+                         ImmutableCallSite> Base;
 
 public:
   explicit ImmutableStatepoint(const Instruction *I) : Base(I) {}
@@ -171,8 +293,9 @@ class ImmutableStatepoint
 
 /// A specialization of it's base class for read-write access
 /// to a gc.statepoint.
-class Statepoint : public StatepointBase<Instruction, Value, CallSite> {
-  typedef StatepointBase<Instruction, Value, CallSite> Base;
+class Statepoint
+    : public StatepointBase<Function, Instruction, Value, CallSite> {
+  typedef StatepointBase<Function, Instruction, Value, CallSite> Base;
 
 public:
   explicit Statepoint(Instruction *I) : Base(I) {}
@@ -185,114 +308,107 @@ class Statepoint : public StatepointBase<Instruction, Value, CallSite> {
 class GCRelocateOperands {
   ImmutableCallSite RelocateCS;
 
- public:
-  GCRelocateOperands(const User* U) : RelocateCS(U) {
-    assert(isGCRelocate(U));
-  }
+public:
+  GCRelocateOperands(const User *U) : RelocateCS(U) { assert(isGCRelocate(U)); }
   GCRelocateOperands(const Instruction *inst) : RelocateCS(inst) {
     assert(isGCRelocate(inst));
   }
-  GCRelocateOperands(CallSite CS) : RelocateCS(CS) {
-    assert(isGCRelocate(CS));
-  }
+  GCRelocateOperands(CallSite CS) : RelocateCS(CS) { assert(isGCRelocate(CS)); }
 
   /// Return true if this relocate is tied to the invoke statepoint.
   /// This includes relocates which are on the unwinding path.
   bool isTiedToInvoke() const {
     const Value *Token = RelocateCS.getArgument(0);
 
-    return isa<ExtractValueInst>(Token) ||
-      isa<InvokeInst>(Token);
+    return isa<ExtractValueInst>(Token) || isa<InvokeInst>(Token);
   }
 
   /// Get enclosed relocate intrinsic
-  ImmutableCallSite getUnderlyingCallSite() {
-    return RelocateCS;
-  }
+  ImmutableCallSite getUnderlyingCallSite() { return RelocateCS; }
 
   /// The statepoint with which this gc.relocate is associated.
-  const Instruction *statepoint() {
-    const Value *token = RelocateCS.getArgument(0);
+  const Instruction *getStatepoint() {
+    const Value *Token = RelocateCS.getArgument(0);
 
     // This takes care both of relocates for call statepoints and relocates
     // on normal path of invoke statepoint.
-    if (!isa<ExtractValueInst>(token)) {
-      return cast<Instruction>(token);
+    if (!isa<ExtractValueInst>(Token)) {
+      return cast<Instruction>(Token);
     }
 
     // This relocate is on exceptional path of an invoke statepoint
-    const BasicBlock *invokeBB =
-      cast<Instruction>(token)->getParent()->getUniquePredecessor();
+    const BasicBlock *InvokeBB =
+        cast<Instruction>(Token)->getParent()->getUniquePredecessor();
 
-    assert(invokeBB && "safepoints should have unique landingpads");
-    assert(invokeBB->getTerminator() && "safepoint block should be well formed");
-    assert(isStatepoint(invokeBB->getTerminator()));
+    assert(InvokeBB && "safepoints should have unique landingpads");
+    assert(InvokeBB->getTerminator() &&
+           "safepoint block should be well formed");
+    assert(isStatepoint(InvokeBB->getTerminator()));
 
-    return invokeBB->getTerminator();
+    return InvokeBB->getTerminator();
   }
+
   /// The index into the associate statepoint's argument list
   /// which contains the base pointer of the pointer whose
   /// relocation this gc.relocate describes.
-  unsigned basePtrIndex() {
+  unsigned getBasePtrIndex() {
     return cast<ConstantInt>(RelocateCS.getArgument(1))->getZExtValue();
   }
+
   /// The index into the associate statepoint's argument list which
   /// contains the pointer whose relocation this gc.relocate describes.
-  unsigned derivedPtrIndex() {
+  unsigned getDerivedPtrIndex() {
     return cast<ConstantInt>(RelocateCS.getArgument(2))->getZExtValue();
   }
-  Value *basePtr() {
-    ImmutableCallSite CS(statepoint());
-    return *(CS.arg_begin() + basePtrIndex());
+
+  Value *getBasePtr() {
+    ImmutableCallSite CS(getStatepoint());
+    return *(CS.arg_begin() + getBasePtrIndex());
   }
-  Value *derivedPtr() {
-    ImmutableCallSite CS(statepoint());
-    return *(CS.arg_begin() + derivedPtrIndex());
+
+  Value *getDerivedPtr() {
+    ImmutableCallSite CS(getStatepoint());
+    return *(CS.arg_begin() + getDerivedPtrIndex());
   }
 };
 
-template <typename InstructionTy, typename ValueTy, typename CallSiteTy>
+template <typename FunTy, typename InstructionTy, typename ValueTy,
+          typename CallSiteTy>
 std::vector<GCRelocateOperands>
-  StatepointBase<InstructionTy, ValueTy, CallSiteTy>::
-    getRelocates(ImmutableStatepoint &IS) {
+StatepointBase<FunTy, InstructionTy, ValueTy, CallSiteTy>::getRelocates()
+    const {
 
-  std::vector<GCRelocateOperands> res;
+  std::vector<GCRelocateOperands> Result;
 
-  ImmutableCallSite StatepointCS = IS.getCallSite();
+  CallSiteTy StatepointCS = getCallSite();
 
   // Search for relocated pointers.  Note that working backwards from the
   // gc_relocates ensures that we only get pairs which are actually relocated
   // and used after the statepoint.
-  for (const User *U : StatepointCS.getInstruction()->users()) {
-    if (isGCRelocate(U)) {
-      res.push_back(GCRelocateOperands(U));
-    }
-  }
+  for (const User *U : getInstruction()->users())
+    if (isGCRelocate(U))
+      Result.push_back(GCRelocateOperands(U));
 
-  if (!StatepointCS.isInvoke()) {
-    return res;
-  }
+  if (!StatepointCS.isInvoke())
+    return Result;
 
   // We need to scan thorough exceptional relocations if it is invoke statepoint
   LandingPadInst *LandingPad =
-    cast<InvokeInst>(StatepointCS.getInstruction())->getLandingPadInst();
+      cast<InvokeInst>(getInstruction())->getLandingPadInst();
 
   // Search for extract value from landingpad instruction to which
   // gc relocates will be attached
   for (const User *LandingPadUser : LandingPad->users()) {
-    if (!isa<ExtractValueInst>(LandingPadUser)) {
+    if (!isa<ExtractValueInst>(LandingPadUser))
       continue;
-    }
 
     // gc relocates should be attached to this extract value
-    for (const User *U : LandingPadUser->users()) {
-      if (isGCRelocate(U)) {
-        res.push_back(GCRelocateOperands(U));
-      }
-    }
+    for (const User *U : LandingPadUser->users())
+      if (isGCRelocate(U))
+        Result.push_back(GCRelocateOperands(U));
   }
-  return res;
+  return Result;
 }
-
 }
+
 #endif
diff --git a/include/llvm/IR/Type.h b/include/llvm/IR/Type.h
index 09b6388bd26..6ab0bd0631a 100644
--- a/include/llvm/IR/Type.h
+++ b/include/llvm/IR/Type.h
@@ -395,7 +395,8 @@ class Type {
   static IntegerType *getInt16Ty(LLVMContext &C);
   static IntegerType *getInt32Ty(LLVMContext &C);
   static IntegerType *getInt64Ty(LLVMContext &C);
-
+  static IntegerType *getInt128Ty(LLVMContext &C);
+  
   //===--------------------------------------------------------------------===//
   // Convenience methods for getting pointer types with one of the above builtin
   // types as pointee.
diff --git a/include/llvm/IR/TypeBuilder.h b/include/llvm/IR/TypeBuilder.h
index 5a29e1e1085..d2c6f00079d 100644
--- a/include/llvm/IR/TypeBuilder.h
+++ b/include/llvm/IR/TypeBuilder.h
@@ -34,6 +34,8 @@ namespace llvm {
 /// you'll need to specialize it.  For example, say you want to call a
 /// function defined externally as:
 ///
+/// \code{.cpp}
+///
 ///   struct MyType {
 ///     int32 a;
 ///     int32 *b;
@@ -41,12 +43,16 @@ namespace llvm {
 ///   };
 ///   int8 AFunction(struct MyType *value);
 ///
+/// \endcode
+///
 /// You'll want to use
 ///   Function::Create(TypeBuilder<types::i<8>(MyType*), true>::get(), ...)
 /// to declare the function, but when you first try this, your compiler will
 /// complain that TypeBuilder<MyType, true>::get() doesn't exist. To fix this,
 /// write:
 ///
+/// \code{.cpp}
+///
 ///   namespace llvm {
 ///   template<bool xcompile> class TypeBuilder<MyType, xcompile> {
 ///   public:
@@ -57,7 +63,7 @@ namespace llvm {
 ///         TypeBuilder<types::i<32>, xcompile>::get(Context),
 ///         TypeBuilder<types::i<32>*, xcompile>::get(Context),
 ///         TypeBuilder<types::i<8>*[], xcompile>::get(Context),
-///         NULL);
+///         nullptr);
 ///     }
 ///
 ///     // You may find this a convenient place to put some constants
@@ -71,6 +77,8 @@ namespace llvm {
 ///   }
 ///   }  // namespace llvm
 ///
+/// \endcode
+///
 /// TypeBuilder cannot handle recursive types or types you only know at runtime.
 /// If you try to give it a recursive type, it will deadlock, infinitely
 /// recurse, or do something similarly undesirable.
diff --git a/include/llvm/IR/User.h b/include/llvm/IR/User.h
index 455900566af..93614fab575 100644
--- a/include/llvm/IR/User.h
+++ b/include/llvm/IR/User.h
@@ -22,6 +22,7 @@
 #include "llvm/ADT/iterator.h"
 #include "llvm/ADT/iterator_range.h"
 #include "llvm/IR/Value.h"
+#include "llvm/Support/AlignOf.h"
 #include "llvm/Support/ErrorHandling.h"
 
 namespace llvm {
@@ -34,33 +35,45 @@ struct OperandTraits;
 
 class User : public Value {
   User(const User &) = delete;
-  void *operator new(size_t) = delete;
   template <unsigned>
   friend struct HungoffOperandTraits;
   virtual void anchor();
+
 protected:
-  /// \brief This is a pointer to the array of Uses for this User.
+  /// Allocate a User with an operand pointer co-allocated.
+  ///
+  /// This is used for subclasses which need to allocate a variable number
+  /// of operands, ie, 'hung off uses'.
+  void *operator new(size_t Size);
+
+  /// Allocate a User with the operands co-allocated.
   ///
-  /// For nodes of fixed arity (e.g. a binary operator) this array will live
-  /// prefixed to some derived class instance.  For nodes of resizable variable
-  /// arity (e.g. PHINodes, SwitchInst etc.), this memory will be dynamically
-  /// allocated and should be destroyed by the classes' virtual dtor.
-  Use *OperandList;
+  /// This is used for subclasses which have a fixed number of operands.
+  void *operator new(size_t Size, unsigned Us);
 
-  void *operator new(size_t s, unsigned Us);
   User(Type *ty, unsigned vty, Use *OpList, unsigned NumOps)
-      : Value(ty, vty), OperandList(OpList) {
-    NumOperands = NumOps;
-  }
-  Use *allocHungoffUses(unsigned) const;
-  void dropHungoffUses() {
-    Use::zap(OperandList, OperandList + NumOperands, true);
-    OperandList = nullptr;
-    // Reset NumOperands so User::operator delete() does the right thing.
-    NumOperands = 0;
+      : Value(ty, vty) {
+    assert(NumOps < (1u << NumUserOperandsBits) && "Too many operands");
+    NumUserOperands = NumOps;
+    // If we have hung off uses, then the operand list should initially be
+    // null.
+    assert((!HasHungOffUses || !getOperandList()) &&
+           "Error in initializing hung off uses for User");
   }
+
+  /// \brief Allocate the array of Uses, followed by a pointer
+  /// (with bottom bit set) to the User.
+  /// \param IsPhi identifies callers which are phi nodes and which need
+  /// N BasicBlock* allocated along with N
+  void allocHungoffUses(unsigned N, bool IsPhi = false);
+
+  /// \brief Grow the number of hung off uses.  Note that allocHungoffUses
+  /// should be called if there are no uses.
+  void growHungoffUses(unsigned N, bool IsPhi = false);
+
 public:
-  ~User() override { Use::zap(OperandList, OperandList + NumOperands); }
+  ~User() override {
+  }
   /// \brief Free memory allocated for User and Use objects.
   void operator delete(void *Usr);
   /// \brief Placement delete - required by std, but never called.
@@ -83,28 +96,81 @@ class User : public Value {
   template <int Idx> const Use &Op() const {
     return OpFrom<Idx>(this);
   }
+private:
+  Use *&getHungOffOperands() { return *(reinterpret_cast<Use **>(this) - 1); }
+
+  Use *getIntrusiveOperands() {
+    return reinterpret_cast<Use *>(this) - NumUserOperands;
+  }
+
+  void setOperandList(Use *NewList) {
+    assert(HasHungOffUses &&
+           "Setting operand list only required for hung off uses");
+    getHungOffOperands() = NewList;
+  }
 public:
+  Use *getOperandList() {
+    return HasHungOffUses ? getHungOffOperands() : getIntrusiveOperands();
+  }
+  const Use *getOperandList() const {
+    return const_cast<User *>(this)->getOperandList();
+  }
   Value *getOperand(unsigned i) const {
-    assert(i < NumOperands && "getOperand() out of range!");
-    return OperandList[i];
+    assert(i < NumUserOperands && "getOperand() out of range!");
+    return getOperandList()[i];
   }
   void setOperand(unsigned i, Value *Val) {
-    assert(i < NumOperands && "setOperand() out of range!");
+    assert(i < NumUserOperands && "setOperand() out of range!");
     assert((!isa<Constant>((const Value*)this) ||
             isa<GlobalValue>((const Value*)this)) &&
            "Cannot mutate a constant with setOperand!");
-    OperandList[i] = Val;
+    getOperandList()[i] = Val;
   }
   const Use &getOperandUse(unsigned i) const {
-    assert(i < NumOperands && "getOperandUse() out of range!");
-    return OperandList[i];
+    assert(i < NumUserOperands && "getOperandUse() out of range!");
+    return getOperandList()[i];
   }
   Use &getOperandUse(unsigned i) {
-    assert(i < NumOperands && "getOperandUse() out of range!");
-    return OperandList[i];
+    assert(i < NumUserOperands && "getOperandUse() out of range!");
+    return getOperandList()[i];
   }
 
-  unsigned getNumOperands() const { return NumOperands; }
+  unsigned getNumOperands() const { return NumUserOperands; }
+
+  /// Set the number of operands on a GlobalVariable.
+  ///
+  /// GlobalVariable always allocates space for a single operands, but
+  /// doesn't always use it.
+  ///
+  /// FIXME: As that the number of operands is used to find the start of
+  /// the allocated memory in operator delete, we need to always think we have
+  /// 1 operand before delete.
+  void setGlobalVariableNumOperands(unsigned NumOps) {
+    assert(NumOps <= 1 && "GlobalVariable can only have 0 or 1 operands");
+    NumUserOperands = NumOps;
+  }
+
+  /// Set the number of operands on a Function.
+  ///
+  /// Function always allocates space for a single operands, but
+  /// doesn't always use it.
+  ///
+  /// FIXME: As that the number of operands is used to find the start of
+  /// the allocated memory in operator delete, we need to always think we have
+  /// 1 operand before delete.
+  void setFunctionNumOperands(unsigned NumOps) {
+    assert(NumOps <= 1 && "Function can only have 0 or 1 operands");
+    NumUserOperands = NumOps;
+  }
+
+  /// \brief Subclasses with hung off uses need to manage the operand count
+  /// themselves.  In these instances, the operand count isn't used to find the
+  /// OperandList, so there's no issue in having the operand count change.
+  void setNumHungOffUseOperands(unsigned NumOps) {
+    assert(HasHungOffUses && "Must have hung off uses to use this method");
+    assert(NumOps < (1u << NumUserOperandsBits) && "Too many operands");
+    NumUserOperands = NumOps;
+  }
 
   // ---------------------------------------------------------------------------
   // Operand Iterator interface...
@@ -114,14 +180,18 @@ class User : public Value {
   typedef iterator_range<op_iterator> op_range;
   typedef iterator_range<const_op_iterator> const_op_range;
 
-  inline op_iterator       op_begin()       { return OperandList; }
-  inline const_op_iterator op_begin() const { return OperandList; }
-  inline op_iterator       op_end()         { return OperandList+NumOperands; }
-  inline const_op_iterator op_end()   const { return OperandList+NumOperands; }
-  inline op_range operands() {
+  op_iterator       op_begin()       { return getOperandList(); }
+  const_op_iterator op_begin() const { return getOperandList(); }
+  op_iterator       op_end()         {
+    return getOperandList() + NumUserOperands;
+  }
+  const_op_iterator op_end()   const {
+    return getOperandList() + NumUserOperands;
+  }
+  op_range operands() {
     return op_range(op_begin(), op_end());
   }
-  inline const_op_range operands() const {
+  const_op_range operands() const {
     return const_op_range(op_begin(), op_end());
   }
 
@@ -136,13 +206,13 @@ class User : public Value {
     Value *operator->() const { return operator*(); }
   };
 
-  inline value_op_iterator value_op_begin() {
+  value_op_iterator value_op_begin() {
     return value_op_iterator(op_begin());
   }
-  inline value_op_iterator value_op_end() {
+  value_op_iterator value_op_end() {
     return value_op_iterator(op_end());
   }
-  inline iterator_range<value_op_iterator> operand_values() {
+  iterator_range<value_op_iterator> operand_values() {
     return iterator_range<value_op_iterator>(value_op_begin(), value_op_end());
   }
 
@@ -170,6 +240,11 @@ class User : public Value {
     return isa<Instruction>(V) || isa<Constant>(V);
   }
 };
+// Either Use objects, or a Use pointer can be prepended to User.
+static_assert(AlignOf<Use>::Alignment >= AlignOf<User>::Alignment,
+              "Alignment is insufficient after objects prepended to User");
+static_assert(AlignOf<Use *>::Alignment >= AlignOf<User>::Alignment,
+              "Alignment is insufficient after objects prepended to User");
 
 template<> struct simplify_type<User::op_iterator> {
   typedef Value* SimpleType;
diff --git a/include/llvm/IR/Value.def b/include/llvm/IR/Value.def
new file mode 100644
index 00000000000..c2a0639603e
--- /dev/null
+++ b/include/llvm/IR/Value.def
@@ -0,0 +1,90 @@
+//===-------- llvm/IR/Value.def - File that describes Values ---v-*- C++ -*-===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This file contains descriptions of the various LLVM values.  This is
+// used as a central place for enumerating the different values.
+//
+//===----------------------------------------------------------------------===//
+
+// NOTE: NO INCLUDE GUARD DESIRED!
+
+// Provide definitions of macros so that users of this file do not have to
+// define everything to use it...
+//
+#if !(defined HANDLE_GLOBAL_VALUE || defined HANDLE_CONSTANT ||                \
+      defined HANDLE_INSTRUCTION || defined HANDLE_INLINE_ASM_VALUE ||         \
+      defined HANDLE_METADATA_VALUE || defined HANDLE_VALUE ||                 \
+      defined HANDLE_CONSTANT_MARKER)
+#error "Missing macro definition of HANDLE_VALUE*"
+#endif
+
+#ifndef HANDLE_GLOBAL_VALUE
+#define HANDLE_GLOBAL_VALUE(ValueName) HANDLE_CONSTANT(ValueName)
+#endif
+
+#ifndef HANDLE_CONSTANT
+#define HANDLE_CONSTANT(ValueName) HANDLE_VALUE(ValueName)
+#endif
+
+#ifndef HANDLE_INSTRUCTION
+#define HANDLE_INSTRUCTION(ValueName) HANDLE_VALUE(ValueName)
+#endif
+
+#ifndef HANDLE_INLINE_ASM_VALUE
+#define HANDLE_INLINE_ASM_VALUE(ValueName) HANDLE_VALUE(ValueName)
+#endif
+
+#ifndef HANDLE_METADATA_VALUE
+#define HANDLE_METADATA_VALUE(ValueName) HANDLE_VALUE(ValueName)
+#endif
+
+#ifndef HANDLE_VALUE
+#define HANDLE_VALUE(ValueName)
+#endif
+
+#ifndef HANDLE_CONSTANT_MARKER
+#define HANDLE_CONSTANT_MARKER(MarkerName, ValueName)
+#endif
+
+HANDLE_VALUE(Argument)
+HANDLE_VALUE(BasicBlock)
+
+HANDLE_GLOBAL_VALUE(Function)
+HANDLE_GLOBAL_VALUE(GlobalAlias)
+HANDLE_GLOBAL_VALUE(GlobalVariable)
+HANDLE_CONSTANT(UndefValue)
+HANDLE_CONSTANT(BlockAddress)
+HANDLE_CONSTANT(ConstantExpr)
+HANDLE_CONSTANT(ConstantAggregateZero)
+HANDLE_CONSTANT(ConstantDataArray)
+HANDLE_CONSTANT(ConstantDataVector)
+HANDLE_CONSTANT(ConstantInt)
+HANDLE_CONSTANT(ConstantFP)
+HANDLE_CONSTANT(ConstantArray)
+HANDLE_CONSTANT(ConstantStruct)
+HANDLE_CONSTANT(ConstantVector)
+HANDLE_CONSTANT(ConstantPointerNull)
+
+HANDLE_METADATA_VALUE(MetadataAsValue)
+HANDLE_INLINE_ASM_VALUE(InlineAsm)
+
+HANDLE_INSTRUCTION(Instruction)
+// Enum values starting at InstructionVal are used for Instructions;
+// don't add new values here!
+
+HANDLE_CONSTANT_MARKER(ConstantFirstVal, Function)
+HANDLE_CONSTANT_MARKER(ConstantLastVal, ConstantPointerNull)
+
+#undef HANDLE_GLOBAL_VALUE
+#undef HANDLE_CONSTANT
+#undef HANDLE_INSTRUCTION
+#undef HANDLE_METADATA_VALUE
+#undef HANDLE_INLINE_ASM_VALUE
+#undef HANDLE_VALUE
+#undef HANDLE_CONSTANT_MARKER
diff --git a/include/llvm/IR/Value.h b/include/llvm/IR/Value.h
index fb79a8abb0a..9d83cef76f7 100644
--- a/include/llvm/IR/Value.h
+++ b/include/llvm/IR/Value.h
@@ -38,6 +38,7 @@ class InlineAsm;
 class Instruction;
 class LLVMContext;
 class Module;
+class ModuleSlotTracker;
 class StringRef;
 class Twine;
 class Type;
@@ -69,9 +70,8 @@ class Value {
   Type *VTy;
   Use *UseList;
 
-  friend class ValueAsMetadata; // Allow access to NameAndIsUsedByMD.
+  friend class ValueAsMetadata; // Allow access to IsUsedByMD.
   friend class ValueHandleBase;
-  PointerIntPair<ValueName *, 1> NameAndIsUsedByMD;
 
   const unsigned char SubclassID;   // Subclass identifier (for isa/dyn_cast)
   unsigned char HasValueHandle : 1; // Has a ValueHandle pointing to this?
@@ -101,7 +101,15 @@ class Value {
   /// This is stored here to save space in User on 64-bit hosts.  Since most
   /// instances of Value have operands, 32-bit hosts aren't significantly
   /// affected.
-  unsigned NumOperands;
+  ///
+  /// Note, this should *NOT* be used directly by any class other than User.
+  /// User uses this value to find the Use list.
+  enum : unsigned { NumUserOperandsBits = 29 };
+  unsigned NumUserOperands : NumUserOperandsBits;
+
+  bool IsUsedByMD : 1;
+  bool HasName : 1;
+  bool HasHungOffUses : 1;
 
 private:
   template <typename UseT> // UseT == 'Use' or 'const Use'
@@ -192,7 +200,10 @@ class Value {
   void dump() const;
 
   /// \brief Implement operator<< on Value.
+  /// @{
   void print(raw_ostream &O) const;
+  void print(raw_ostream &O, ModuleSlotTracker &MST) const;
+  /// @}
 
   /// \brief Print the name of this Value out to the specified raw_ostream.
   ///
@@ -200,8 +211,12 @@ class Value {
   /// instruction that generated it. If you specify a Module for context, then
   /// even constanst get pretty-printed; for example, the type of a null
   /// pointer is printed symbolically.
+  /// @{
   void printAsOperand(raw_ostream &O, bool PrintType = true,
                       const Module *M = nullptr) const;
+  void printAsOperand(raw_ostream &O, bool PrintType,
+                      ModuleSlotTracker &MST) const;
+  /// @}
 
   /// \brief All values are typed, get the type of this value.
   Type *getType() const { return VTy; }
@@ -210,12 +225,13 @@ class Value {
   LLVMContext &getContext() const;
 
   // \brief All values can potentially be named.
-  bool hasName() const { return getValueName() != nullptr; }
-  ValueName *getValueName() const { return NameAndIsUsedByMD.getPointer(); }
-  void setValueName(ValueName *VN) { NameAndIsUsedByMD.setPointer(VN); }
+  bool hasName() const { return HasName; }
+  ValueName *getValueName() const;
+  void setValueName(ValueName *VN);
 
 private:
   void destroyValueName();
+  void setNameImpl(const Twine &Name);
 
 public:
   /// \brief Return a constant reference to the value's name.
@@ -325,32 +341,12 @@ class Value {
   /// Value classes SubclassID field. They are used for concrete type
   /// identification.
   enum ValueTy {
-    ArgumentVal,              // This is an instance of Argument
-    BasicBlockVal,            // This is an instance of BasicBlock
-    FunctionVal,              // This is an instance of Function
-    GlobalAliasVal,           // This is an instance of GlobalAlias
-    GlobalVariableVal,        // This is an instance of GlobalVariable
-    UndefValueVal,            // This is an instance of UndefValue
-    BlockAddressVal,          // This is an instance of BlockAddress
-    ConstantExprVal,          // This is an instance of ConstantExpr
-    ConstantAggregateZeroVal, // This is an instance of ConstantAggregateZero
-    ConstantDataArrayVal,     // This is an instance of ConstantDataArray
-    ConstantDataVectorVal,    // This is an instance of ConstantDataVector
-    ConstantIntVal,           // This is an instance of ConstantInt
-    ConstantFPVal,            // This is an instance of ConstantFP
-    ConstantArrayVal,         // This is an instance of ConstantArray
-    ConstantStructVal,        // This is an instance of ConstantStruct
-    ConstantVectorVal,        // This is an instance of ConstantVector
-    ConstantPointerNullVal,   // This is an instance of ConstantPointerNull
-    MetadataAsValueVal,       // This is an instance of MetadataAsValue
-    InlineAsmVal,             // This is an instance of InlineAsm
-    InstructionVal,           // This is an instance of Instruction
-    // Enum values starting at InstructionVal are used for Instructions;
-    // don't add new values here!
+#define HANDLE_VALUE(Name) Name##Val,
+#include "llvm/IR/Value.def"
 
     // Markers:
-    ConstantFirstVal = FunctionVal,
-    ConstantLastVal  = ConstantPointerNullVal
+#define HANDLE_CONSTANT_MARKER(Marker, Constant) Marker = Constant##Val,
+#include "llvm/IR/Value.def"
   };
 
   /// \brief Return an ID for the concrete type of this object.
@@ -393,7 +389,7 @@ class Value {
   bool hasValueHandle() const { return HasValueHandle; }
 
   /// \brief Return true if there is metadata referencing this value.
-  bool isUsedByMetadata() const { return NameAndIsUsedByMD.getInt(); }
+  bool isUsedByMetadata() const { return IsUsedByMD; }
 
   /// \brief Strip off pointer casts, all-zero GEPs, and aliases.
   ///
@@ -446,12 +442,6 @@ class Value {
     return const_cast<Value*>(this)->stripInBoundsOffsets();
   }
 
-  /// \brief Check if this is always a dereferenceable pointer.
-  ///
-  /// Test if this value is always a pointer to allocated and suitably aligned
-  /// memory for a simple load or store.
-  bool isDereferenceablePointer(const DataLayout &DL) const;
-
   /// \brief Translate PHI node to its predecessor from the given basic block.
   ///
   /// If this value is a PHI node with CurBB as its parent, return the value in
@@ -502,40 +492,38 @@ class Value {
   /// \note Completely ignores \a Use::Prev (doesn't read, doesn't update).
   template <class Compare>
   static Use *mergeUseLists(Use *L, Use *R, Compare Cmp) {
-    // XXX EMSCRIPTEN: Reimplemented to operate iteratively instead of recursively,
-    //                 since there exists use lists that have ~300k elems, which would
-    //                 blow up the stack.
-    if (!L && !R) return 0;
-    Use *Merged = 0;
-    Use *MergedLast = 0;
-    if (!L || (R && Cmp(*R, *L))) {
-      Merged = MergedLast = R;
-      R = R->Next;
-    } else {
-      Merged = MergedLast = L;
-      L = L->Next;
-    }
+    Use *Merged;
+    Use **Next = &Merged;
 
-    while (L && R) {
+    for (;;) {
+      if (!L) {
+        *Next = R;
+        break;
+      }
+      if (!R) {
+        *Next = L;
+        break;
+      }
       if (Cmp(*R, *L)) {
-        MergedLast->Next = R;
-        MergedLast = R;
+        *Next = R;
+        Next = &R->Next;
         R = R->Next;
       } else {
-        MergedLast->Next = L;
-        MergedLast = L;
+        *Next = L;
+        Next = &L->Next;
         L = L->Next;
       }
     }
-    if (L) {
-      MergedLast->Next = L;
-    }
-    if (R) {
-      MergedLast->Next = R;
-    }
+
     return Merged;
   }
 
+  /// \brief Tail-recursive helper for \a mergeUseLists().
+  ///
+  /// \param[out] Next the first element in the list.
+  template <class Compare>
+  static void mergeUseListsImpl(Use *L, Use *R, Use **Next, Compare Cmp);
+
 protected:
   unsigned short getSubclassDataFromValue() const { return SubclassData; }
   void setValueSubclassData(unsigned short D) { SubclassData = D; }
diff --git a/include/llvm/IR/ValueHandle.h b/include/llvm/IR/ValueHandle.h
index 355748e0597..53fa80a626a 100644
--- a/include/llvm/IR/ValueHandle.h
+++ b/include/llvm/IR/ValueHandle.h
@@ -159,11 +159,13 @@ class WeakVH : public ValueHandleBase {
 
 // Specialize simplify_type to allow WeakVH to participate in
 // dyn_cast, isa, etc.
-template<> struct simplify_type<WeakVH> {
-  typedef Value* SimpleType;
-  static SimpleType getSimplifiedValue(WeakVH &WVH) {
-    return WVH;
-  }
+template <> struct simplify_type<WeakVH> {
+  typedef Value *SimpleType;
+  static SimpleType getSimplifiedValue(WeakVH &WVH) { return WVH; }
+};
+template <> struct simplify_type<const WeakVH> {
+  typedef Value *SimpleType;
+  static SimpleType getSimplifiedValue(const WeakVH &WVH) { return WVH; }
 };
 
 /// \brief Value handle that asserts if the Value is deleted.
diff --git a/include/llvm/IRReader/IRReader.h b/include/llvm/IRReader/IRReader.h
index 1816e0a2e36..2d9ace0b62a 100644
--- a/include/llvm/IRReader/IRReader.h
+++ b/include/llvm/IRReader/IRReader.h
@@ -43,42 +43,6 @@ std::unique_ptr<Module> parseIR(MemoryBufferRef Buffer, SMDiagnostic &Err,
 /// for it.
 std::unique_ptr<Module> parseIRFile(StringRef Filename, SMDiagnostic &Err,
                                     LLVMContext &Context);
-
-// @LOCALMOD-BEGIN
-class raw_ostream;
-
-// \brief Define the expected format of the file.
-enum NaClFileFormat {
-  // LLVM IR source or bitcode file (as appropriate).
-  LLVMFormat,
-  // PNaCl bitcode file.
-  PNaClFormat,
-  // Autodetect if PNaCl or LLVM format.
-  AutodetectFileFormat
-};
-
-// \brief If the given MemoryBuffer holds a bitcode image, return a
-// Module for it.  Otherwise, attempt to parse it as LLVM Assembly and
-// return a Module for it. When Format=PNaClFormat and Verbose
-// is non-null, more descriptive error messages are also written to
-// Verbose.
-std::unique_ptr<Module> NaClParseIR(MemoryBufferRef Buffer,
-                                    NaClFileFormat Format,
-                                    SMDiagnostic &Err,
-                                    raw_ostream *Verbose,
-                                    LLVMContext &Context);
-
-/// \brief If the given file holds a Bitcode image, read the file.
-/// Otherwise, attempt to parse it as LLVM assembly and return a
-/// Module for it. When Format=PNaClFormat and Verbose
-// is non-null, more descriptive error messages are also written to
-// Verbose.
-std::unique_ptr<Module> NaClParseIRFile(StringRef Filename,
-                                        NaClFileFormat Format,
-                                        SMDiagnostic &Err,
-                                        raw_ostream *Verbose,
-                                        LLVMContext &Context);
-// @LOCALMOD-END
 }
 
 #endif
diff --git a/include/llvm/InitializePasses.h b/include/llvm/InitializePasses.h
index 49b2a04ffdb..8b784036fdb 100644
--- a/include/llvm/InitializePasses.h
+++ b/include/llvm/InitializePasses.h
@@ -79,10 +79,10 @@ void initializeBarrierNoopPass(PassRegistry&);
 void initializeBasicAliasAnalysisPass(PassRegistry&);
 void initializeCallGraphWrapperPassPass(PassRegistry &);
 void initializeBlockExtractorPassPass(PassRegistry&);
-void initializeBlockFrequencyInfoPass(PassRegistry&);
+void initializeBlockFrequencyInfoWrapperPassPass(PassRegistry&);
 void initializeBoundsCheckingPass(PassRegistry&);
 void initializeBranchFolderPassPass(PassRegistry&);
-void initializeBranchProbabilityInfoPass(PassRegistry&);
+void initializeBranchProbabilityInfoWrapperPassPass(PassRegistry&);
 void initializeBreakCriticalEdgesPass(PassRegistry&);
 void initializeCallGraphPrinterPass(PassRegistry&);
 void initializeCallGraphViewerPass(PassRegistry&);
@@ -130,6 +130,7 @@ void initializeSanitizerCoverageModulePass(PassRegistry&);
 void initializeDataFlowSanitizerPass(PassRegistry&);
 void initializeScalarizerPass(PassRegistry&);
 void initializeEarlyCSELegacyPassPass(PassRegistry &);
+void initializeEliminateAvailableExternallyPass(PassRegistry&);
 void initializeExpandISelPseudosPass(PassRegistry&);
 void initializeFunctionAttrsPass(PassRegistry&);
 void initializeGCMachineCodeAnalysisPass(PassRegistry&);
@@ -187,6 +188,7 @@ void initializeMachineBlockPlacementPass(PassRegistry&);
 void initializeMachineBlockPlacementStatsPass(PassRegistry&);
 void initializeMachineBranchProbabilityInfoPass(PassRegistry&);
 void initializeMachineCSEPass(PassRegistry&);
+void initializeImplicitNullChecksPass(PassRegistry&);
 void initializeMachineDominatorTreePass(PassRegistry&);
 void initializeMachineDominanceFrontierPass(PassRegistry&);
 void initializeMachinePostDominatorTreePass(PassRegistry&);
@@ -241,12 +243,14 @@ void initializeRegionOnlyViewerPass(PassRegistry&);
 void initializeRegionPrinterPass(PassRegistry&);
 void initializeRegionViewerPass(PassRegistry&);
 void initializeRewriteStatepointsForGCPass(PassRegistry&);
+void initializeSafeStackPass(PassRegistry&);
 void initializeSCCPPass(PassRegistry&);
 void initializeSROAPass(PassRegistry&);
 void initializeSROA_DTPass(PassRegistry&);
 void initializeSROA_SSAUpPass(PassRegistry&);
 void initializeScalarEvolutionAliasAnalysisPass(PassRegistry&);
 void initializeScalarEvolutionPass(PassRegistry&);
+void initializeShrinkWrapPass(PassRegistry &);
 void initializeSimpleInlinerPass(PassRegistry&);
 void initializeShadowStackGCLoweringPass(PassRegistry&);  
 void initializeRegisterCoalescerPass(PassRegistry&);
@@ -255,6 +259,7 @@ void initializeSinkingPass(PassRegistry&);
 void initializeSeparateConstOffsetFromGEPPass(PassRegistry &);
 void initializeSlotIndexesPass(PassRegistry&);
 void initializeSpillPlacementPass(PassRegistry&);
+void initializeSpeculativeExecutionPass(PassRegistry&);
 void initializeStackProtectorPass(PassRegistry&);
 void initializeStackColoringPass(PassRegistry&);
 void initializeStackSlotColoringPass(PassRegistry&);
@@ -267,7 +272,6 @@ void initializeStripSymbolsPass(PassRegistry&);
 void initializeTailCallElimPass(PassRegistry&);
 void initializeTailDuplicatePassPass(PassRegistry&);
 void initializeTargetPassConfigPass(PassRegistry&);
-void initializeDataLayoutPassPass(PassRegistry &);
 void initializeTargetTransformInfoWrapperPassPass(PassRegistry &);
 void initializeTargetLibraryInfoWrapperPassPass(PassRegistry &);
 void initializeAssumptionCacheTrackerPass(PassRegistry &);
@@ -288,6 +292,7 @@ void initializeLoopVectorizePass(PassRegistry&);
 void initializeSLPVectorizerPass(PassRegistry&);
 void initializeBBVectorizePass(PassRegistry&);
 void initializeMachineFunctionPrinterPassPass(PassRegistry&);
+void initializeMIRPrintingPassPass(PassRegistry&);
 void initializeStackMapLivenessPass(PassRegistry&);
 void initializeMachineCombinerPass(PassRegistry &);
 void initializeLoadCombinePass(PassRegistry&);
@@ -359,6 +364,8 @@ void initializeLowerEmSetjmpPass(PassRegistry&);
 void initializeNoExitRuntimePass(PassRegistry&);
 // Emscripten passes end.
 // @LOCALMOD-END
+void initializeLoopDistributePass(PassRegistry&);
+void initializeSjLjEHPreparePass(PassRegistry&);
 }
 
 #endif
diff --git a/include/llvm/LTO/LTOCodeGenerator.h b/include/llvm/LTO/LTOCodeGenerator.h
index c19a41c3af6..0c46fc048a4 100644
--- a/include/llvm/LTO/LTOCodeGenerator.h
+++ b/include/llvm/LTO/LTOCodeGenerator.h
@@ -79,7 +79,10 @@ struct LTOCodeGenerator {
   void setAttr(const char *mAttr) { MAttr = mAttr; }
   void setOptLevel(unsigned optLevel) { OptLevel = optLevel; }
 
-  void addMustPreserveSymbol(const char *sym) { MustPreserveSymbols[sym] = 1; }
+  void setShouldInternalize(bool Value) { ShouldInternalize = Value; }
+  void setShouldEmbedUselists(bool Value) { ShouldEmbedUselists = Value; }
+
+  void addMustPreserveSymbol(StringRef sym) { MustPreserveSymbols[sym] = 1; }
 
   // To pass options to the driver and optimization passes. These options are
   // not necessarily for debugging purpose (The function name is misleading).
@@ -96,10 +99,6 @@ struct LTOCodeGenerator {
   // Return true on success.
   bool writeMergedModules(const char *path, std::string &errMsg);
 
-  // @LOCALMOD-BEGIN
-  void wrapSymbol(const char *sym);
-  // @LOCALMOD-END
-
   // Compile the merged module into a *single* object file; the path to object
   // file is returned to the caller via argument "name". Return true on
   // success.
@@ -118,11 +117,10 @@ struct LTOCodeGenerator {
   // (linker), it brings the object to a buffer, and return the buffer to the
   // caller. This function should delete intermediate object file once its content
   // is brought to memory. Return NULL if the compilation was not successful.
-  const void *compile(size_t *length,
-                      bool disableInline,
-                      bool disableGVNLoadPRE,
-                      bool disableVectorization,
-                      std::string &errMsg);
+  std::unique_ptr<MemoryBuffer> compile(bool disableInline,
+                                        bool disableGVNLoadPRE,
+                                        bool disableVectorization,
+                                        std::string &errMsg);
 
   // Optimizes the merged module. Returns true on success.
   bool optimize(bool disableInline,
@@ -133,7 +131,7 @@ struct LTOCodeGenerator {
   // Compiles the merged optimized module into a single object file. It brings
   // the object to a buffer, and returns the buffer to the caller. Return NULL
   // if the compilation was not successful.
-  const void *compileOptimized(size_t *length, std::string &errMsg);
+  std::unique_ptr<MemoryBuffer> compileOptimized(std::string &errMsg);
 
   void setDiagnosticHandler(lto_diagnostic_handler_t, void *);
 
@@ -157,27 +155,27 @@ struct LTOCodeGenerator {
 
   typedef StringMap<uint8_t> StringSet;
 
-  void initialize();
   void destroyMergedModule();
   std::unique_ptr<LLVMContext> OwnedContext;
   LLVMContext &Context;
   Linker IRLinker;
-  TargetMachine *TargetMach;
-  bool EmitDwarfDebugInfo;
-  bool ScopeRestrictionsDone;
-  lto_codegen_model CodeModel;
+  TargetMachine *TargetMach = nullptr;
+  bool EmitDwarfDebugInfo = false;
+  bool ScopeRestrictionsDone = false;
+  lto_codegen_model CodeModel = LTO_CODEGEN_PIC_MODEL_DEFAULT;
   StringSet MustPreserveSymbols;
   StringSet AsmUndefinedRefs;
-  std::unique_ptr<MemoryBuffer> NativeObjectFile;
   std::vector<char *> CodegenOptions;
   std::string MCpu;
   std::string MAttr;
   std::string NativeObjectPath;
   TargetOptions Options;
-  unsigned OptLevel;
-  lto_diagnostic_handler_t DiagHandler;
-  void *DiagContext;
-  LTOModule *OwnedModule;
+  unsigned OptLevel = 2;
+  lto_diagnostic_handler_t DiagHandler = nullptr;
+  void *DiagContext = nullptr;
+  LTOModule *OwnedModule = nullptr;
+  bool ShouldInternalize = true;
+  bool ShouldEmbedUselists = false;
 };
 }
 #endif
diff --git a/include/llvm/LTO/LTOModule.h b/include/llvm/LTO/LTOModule.h
index 53c2b8e521b..c4e2be62739 100644
--- a/include/llvm/LTO/LTOModule.h
+++ b/include/llvm/LTO/LTOModule.h
@@ -47,12 +47,11 @@ struct LTOModule {
 
   std::unique_ptr<LLVMContext> OwnedContext;
 
+  std::string LinkerOpts;
+
   std::unique_ptr<object::IRObjectFile> IRFile;
   std::unique_ptr<TargetMachine> _target;
-  StringSet<>                             _linkeropt_strings;
-  std::vector<const char *>               _deplibs;
-  std::vector<const char *>               _linkeropts;
-  std::vector<NameAndAttributes>          _symbols;
+  std::vector<NameAndAttributes> _symbols;
 
   // _defines and _undefines only needed to disambiguate tentative definitions
   StringSet<>                             _defines;
@@ -143,28 +142,14 @@ struct LTOModule {
     return nullptr;
   }
 
-  /// Get the number of dependent libraries
-  uint32_t getDependentLibraryCount() {
-    return _deplibs.size();
-  }
-
-  /// Get the dependent library at the specified index.
-  const char *getDependentLibrary(uint32_t index) {
-    if (index < _deplibs.size())
-      return _deplibs[index];
+  const GlobalValue *getSymbolGV(uint32_t index) {
+    if (index < _symbols.size())
+      return _symbols[index].symbol;
     return nullptr;
   }
 
-  /// Get the number of linker options
-  uint32_t getLinkerOptCount() {
-    return _linkeropts.size();
-  }
-
-  /// Get the linker option at the specified index.
-  const char *getLinkerOpt(uint32_t index) {
-    if (index < _linkeropts.size())
-      return _linkeropts[index];
-    return nullptr;
+  const char *getLinkerOpts() {
+    return LinkerOpts.c_str();
   }
 
   const std::vector<const char*> &getAsmUndefinedRefs() {
diff --git a/include/llvm/LibDriver/LibDriver.h b/include/llvm/LibDriver/LibDriver.h
new file mode 100644
index 00000000000..aaaa7b7d21c
--- /dev/null
+++ b/include/llvm/LibDriver/LibDriver.h
@@ -0,0 +1,26 @@
+//===- llvm/LibDriver/LibDriver.h - lib.exe-compatible driver ---*- C++ -*-===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// Defines an interface to a lib.exe-compatible driver that also understands
+// bitcode files. Used by llvm-lib and lld-link2 /lib.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBDRIVER_LIBDRIVER_H
+#define LLVM_LIBDRIVER_LIBDRIVER_H
+
+#include "llvm/ADT/ArrayRef.h"
+
+namespace llvm {
+
+int libDriverMain(llvm::ArrayRef<const char*> ARgs);
+
+}
+
+#endif
diff --git a/include/llvm/LinkAllPasses.h b/include/llvm/LinkAllPasses.h
index bee39b19366..cea5530db3b 100644
--- a/include/llvm/LinkAllPasses.h
+++ b/include/llvm/LinkAllPasses.h
@@ -98,7 +98,7 @@ namespace {
       (void) llvm::createLICMPass();
       (void) llvm::createLazyValueInfoPass();
       (void) llvm::createLoopExtractorPass();
-      (void)llvm::createLoopInterchangePass();
+      (void) llvm::createLoopInterchangePass();
       (void) llvm::createLoopSimplifyPass();
       (void) llvm::createLoopStrengthReducePass();
       (void) llvm::createLoopRerollPass();
@@ -131,6 +131,7 @@ namespace {
       (void) llvm::createRegionPrinterPass();
       (void) llvm::createRegionViewerPass();
       (void) llvm::createSCCPPass();
+      (void) llvm::createSafeStackPass();
       (void) llvm::createScalarReplAggregatesPass();
       (void) llvm::createSingleLoopExtractorPass();
       (void) llvm::createStripSymbolsPass();
@@ -170,10 +171,12 @@ namespace {
       (void) llvm::createPartiallyInlineLibCallsPass();
       (void) llvm::createScalarizerPass();
       (void) llvm::createSeparateConstOffsetFromGEPPass();
+      (void) llvm::createSpeculativeExecutionPass();
       (void) llvm::createRewriteSymbolsPass();
       (void) llvm::createStraightLineStrengthReducePass();
       (void) llvm::createMemDerefPrinter();
       (void) llvm::createFloat2IntPass();
+      (void) llvm::createEliminateAvailableExternallyPass();
 
       (void)new llvm::IntervalPartition();
       (void)new llvm::ScalarEvolution();
diff --git a/include/llvm/Linker/Linker.h b/include/llvm/Linker/Linker.h
index 5ca815c325a..c43b90e9cd2 100644
--- a/include/llvm/Linker/Linker.h
+++ b/include/llvm/Linker/Linker.h
@@ -68,8 +68,10 @@ class Linker {
   void deleteModule();
 
   /// \brief Link \p Src into the composite. The source is destroyed.
+  /// Passing OverrideSymbols as true will have symbols from Src
+  /// shadow those in the Dest.
   /// Returns true on error.
-  bool linkInModule(Module *Src);
+  bool linkInModule(Module *Src, bool OverrideSymbols = false);
 
   /// \brief Set the composite to the passed-in module.
   void setModule(Module *Dst);
diff --git a/include/llvm/MC/ConstantPools.h b/include/llvm/MC/ConstantPools.h
index ee7022bf178..9aa4663ba0f 100644
--- a/include/llvm/MC/ConstantPools.h
+++ b/include/llvm/MC/ConstantPools.h
@@ -73,7 +73,7 @@ class AssemblerConstantPools {
   // sections in a stable order to ensure that we have print the
   // constant pools in a deterministic order when printing an assembly
   // file.
-  typedef MapVector<const MCSection *, ConstantPool> ConstantPoolMapTy;
+  typedef MapVector<MCSection *, ConstantPool> ConstantPoolMapTy;
   ConstantPoolMapTy ConstantPools;
 
 public:
@@ -83,8 +83,8 @@ class AssemblerConstantPools {
                          unsigned Size);
 
 private:
-  ConstantPool *getConstantPool(const MCSection *Section);
-  ConstantPool &getOrCreateConstantPool(const MCSection *Section);
+  ConstantPool *getConstantPool(MCSection *Section);
+  ConstantPool &getOrCreateConstantPool(MCSection *Section);
 };
 } // end namespace llvm
 
diff --git a/include/llvm/MC/MCAsmBackend.h b/include/llvm/MC/MCAsmBackend.h
index d3dd9cb356f..2bfad2d355b 100644
--- a/include/llvm/MC/MCAsmBackend.h
+++ b/include/llvm/MC/MCAsmBackend.h
@@ -27,7 +27,6 @@ class MCInst;
 class MCRelaxableFragment;
 class MCObjectWriter;
 class MCSection;
-class MCStreamer;
 class MCValue;
 class raw_ostream;
 
@@ -62,7 +61,7 @@ class MCAsmBackend {
   /// region directives will be ignored.
   bool hasDataInCodeSupport() const { return HasDataInCodeSupport; }
 
-  /// @name Target Fixup Interfaces
+  /// \name Target Fixup Interfaces
   /// @{
 
   /// Get the number of target specific fixup kinds.
@@ -88,7 +87,7 @@ class MCAsmBackend {
 
   /// @}
 
-  /// @name Target Relaxation Interfaces
+  /// \name Target Relaxation Interfaces
   /// @{
 
   /// Check whether the given instruction may need relaxation.
@@ -98,6 +97,12 @@ class MCAsmBackend {
 
   /// Target specific predicate for whether a given fixup requires the
   /// associated instruction to be relaxed.
+  virtual bool fixupNeedsRelaxationAdvanced(const MCFixup &Fixup, bool Resolved,
+                                            uint64_t Value,
+                                            const MCRelaxableFragment *DF,
+                                            const MCAsmLayout &Layout) const;
+
+  /// Simple predicate for targets where !Resolved implies requiring relaxation
   virtual bool fixupNeedsRelaxation(const MCFixup &Fixup, uint64_t Value,
                                     const MCRelaxableFragment *DF,
                                     const MCAsmLayout &Layout) const = 0;
@@ -131,16 +136,6 @@ class MCAsmBackend {
       generateCompactUnwindEncoding(ArrayRef<MCCFIInstruction>) const {
     return 0;
   }
-  
-  // @LOCALMOD-BEGIN
-  /// CustomExpandInst -
-  ///   If the MCInst instruction has a custom expansion, write it to the
-  /// MCStreamer 'Out'. This can be used to perform "last minute" rewrites of
-  /// MCInst instructions for emission.
-  virtual bool CustomExpandInst(const MCInst &Inst, MCStreamer &Out) {
-    return false;
-  }
-  // @LOCALMOD-END
 };
 
 } // End llvm namespace
diff --git a/include/llvm/MC/MCAsmInfo.h b/include/llvm/MC/MCAsmInfo.h
index 658d77cbddf..9bb0fa63c52 100644
--- a/include/llvm/MC/MCAsmInfo.h
+++ b/include/llvm/MC/MCAsmInfo.h
@@ -18,7 +18,6 @@
 
 #include "llvm/MC/MCDirectives.h"
 #include "llvm/MC/MCDwarf.h"
-#include "llvm/MC/MachineLocation.h"
 #include <cassert>
 #include <vector>
 
@@ -37,6 +36,7 @@ enum class EncodingType {
   ARM,     /// Windows NT (Windows on ARM)
   CE,      /// Windows CE ARM, PowerPC, SH3, SH4
   Itanium, /// Windows x64, Windows Itanium (IA-64)
+  X86,     /// Windows x86, uses no CFI, just EH tables
   MIPS = Alpha,
 };
 }
@@ -155,6 +155,10 @@ class MCAsmInfo {
   /// Defaults to false.
   bool AllowAtInName;
 
+  /// If this is true, symbol names with invalid characters will be printed in
+  /// quotes.
+  bool SupportsQuotedNames;
+
   /// This is true if data region markers should be printed as
   /// ".data_region/.end_data_region" directives. If false, use "$d/$a" labels
   /// instead.
@@ -228,7 +232,7 @@ class MCAsmInfo {
 
   /// True if the expression
   ///   .long f - g
-  /// uses an relocation but it can be supressed by writting
+  /// uses a relocation but it can be suppressed by writing
   ///   a = f - g
   ///   .long a
   bool SetDirectiveSuppressesReloc;
@@ -256,7 +260,7 @@ class MCAsmInfo {
   /// argument and how it is interpreted.  Defaults to NoAlignment.
   LCOMM::LCOMMType LCOMMDirectiveAlignmentType;
 
-  // True if the target allows .align directives on funtions. This is true for
+  // True if the target allows .align directives on functions. This is true for
   // most targets, so defaults to true.
   bool HasFunctionAlignment;
 
@@ -339,7 +343,7 @@ class MCAsmInfo {
 
   std::vector<MCCFIInstruction> InitialFrameState;
 
-  //===--- Integrated Assembler State ----------------------------------===//
+  //===--- Integrated Assembler Information ----------------------------===//
 
   /// Should we use the integrated assembler?
   /// The integrated assembler should be enabled by default (by the
@@ -351,6 +355,10 @@ class MCAsmInfo {
   /// Compress DWARF debug sections. Defaults to false.
   bool CompressDebugSections;
 
+  /// True if the integrated assembler should interpret 'a >> b' constant
+  /// expressions as logical rather than arithmetic.
+  bool UseLogicalShr;
+
 public:
   explicit MCAsmInfo();
   virtual ~MCAsmInfo();
@@ -384,7 +392,7 @@ class MCAsmInfo {
   /// Targets can implement this method to specify a section to switch to if the
   /// translation unit doesn't have any trampolines that require an executable
   /// stack.
-  virtual const MCSection *getNonexecutableStackSection(MCContext &Ctx) const {
+  virtual MCSection *getNonexecutableStackSection(MCContext &Ctx) const {
     return nullptr;
   }
 
@@ -402,6 +410,10 @@ class MCAsmInfo {
                                             unsigned Encoding,
                                             MCStreamer &Streamer) const;
 
+  /// Return true if the identifier \p Name does not need quotes to be
+  /// syntactically correct.
+  virtual bool isValidUnquotedName(StringRef Name) const;
+
   bool usesSunStyleELFSectionSwitchSyntax() const {
     return SunStyleELFSectionSwitchSyntax;
   }
@@ -452,6 +464,7 @@ class MCAsmInfo {
   const char *getCode64Directive() const { return Code64Directive; }
   unsigned getAssemblerDialect() const { return AssemblerDialect; }
   bool doesAllowAtInName() const { return AllowAtInName; }
+  bool supportsNameQuoting() const { return SupportsQuotedNames; }
   bool doesSupportDataRegionDirectives() const {
     return UseDataRegionDirectives;
   }
@@ -502,12 +515,13 @@ class MCAsmInfo {
   /// frame information to unwind.
   bool usesCFIForEH() const {
     return (ExceptionsType == ExceptionHandling::DwarfCFI ||
-            ExceptionsType == ExceptionHandling::ARM ||
-            ExceptionsType == ExceptionHandling::WinEH);
+            ExceptionsType == ExceptionHandling::ARM || usesWindowsCFI());
   }
 
   bool usesWindowsCFI() const {
-    return ExceptionsType == ExceptionHandling::WinEH;
+    return ExceptionsType == ExceptionHandling::WinEH &&
+           (WinEHEncodingType != WinEH::EncodingType::Invalid &&
+            WinEHEncodingType != WinEH::EncodingType::X86);
   }
 
   bool doesDwarfUseRelocationsAcrossSections() const {
@@ -538,6 +552,8 @@ class MCAsmInfo {
   void setCompressDebugSections(bool CompressDebugSections) {
     this->CompressDebugSections = CompressDebugSections;
   }
+
+  bool shouldUseLogicalShr() const { return UseLogicalShr; }
 };
 }
 
diff --git a/include/llvm/MC/MCAsmInfoELF.h b/include/llvm/MC/MCAsmInfoELF.h
index afd49336646..7125f5c7ad7 100644
--- a/include/llvm/MC/MCAsmInfoELF.h
+++ b/include/llvm/MC/MCAsmInfoELF.h
@@ -15,7 +15,7 @@
 namespace llvm {
 class MCAsmInfoELF : public MCAsmInfo {
   virtual void anchor();
-  const MCSection *getNonexecutableStackSection(MCContext &Ctx) const final;
+  MCSection *getNonexecutableStackSection(MCContext &Ctx) const final;
 
 protected:
   MCAsmInfoELF();
diff --git a/include/llvm/MC/MCAsmLayout.h b/include/llvm/MC/MCAsmLayout.h
index 8b012be46c9..1b20d5b804a 100644
--- a/include/llvm/MC/MCAsmLayout.h
+++ b/include/llvm/MC/MCAsmLayout.h
@@ -16,9 +16,8 @@
 namespace llvm {
 class MCAssembler;
 class MCFragment;
-class MCSectionData;
+class MCSection;
 class MCSymbol;
-class MCSymbolData;
 
 /// Encapsulates the layout of an assembly file at a particular point in time.
 ///
@@ -28,20 +27,15 @@ class MCSymbolData;
 /// efficiently compute the exact address of any symbol in the assembly file,
 /// even during the relaxation process.
 class MCAsmLayout {
-public:
-  typedef llvm::SmallVectorImpl<MCSectionData*>::const_iterator const_iterator;
-  typedef llvm::SmallVectorImpl<MCSectionData*>::iterator iterator;
-
-private:
   MCAssembler &Assembler;
 
   /// List of sections in layout order.
-  llvm::SmallVector<MCSectionData*, 16> SectionOrder;
+  llvm::SmallVector<MCSection *, 16> SectionOrder;
 
   /// The last fragment which was laid out, or 0 if nothing has been laid
   /// out. Fragments are always laid out in order, so all fragments with a
   /// lower ordinal will be valid.
-  mutable DenseMap<const MCSectionData*, MCFragment*> LastValidFragment;
+  mutable DenseMap<const MCSection *, MCFragment *> LastValidFragment;
 
   /// \brief Make sure that the layout for the given fragment is valid, lazily
   /// computing it if necessary.
@@ -66,43 +60,41 @@ class MCAsmLayout {
   /// been initialized.
   void layoutFragment(MCFragment *Fragment);
 
-  /// @name Section Access (in layout order)
+  /// \name Section Access (in layout order)
   /// @{
 
-  llvm::SmallVectorImpl<MCSectionData*> &getSectionOrder() {
-    return SectionOrder;
-  }
-  const llvm::SmallVectorImpl<MCSectionData*> &getSectionOrder() const {
+  llvm::SmallVectorImpl<MCSection *> &getSectionOrder() { return SectionOrder; }
+  const llvm::SmallVectorImpl<MCSection *> &getSectionOrder() const {
     return SectionOrder;
   }
 
   /// @}
-  /// @name Fragment Layout Data
+  /// \name Fragment Layout Data
   /// @{
 
   /// \brief Get the offset of the given fragment inside its containing section.
   uint64_t getFragmentOffset(const MCFragment *F) const;
 
   /// @}
-  /// @name Utility Functions
+  /// \name Utility Functions
   /// @{
 
   /// \brief Get the address space size of the given section, as it effects
   /// layout. This may differ from the size reported by \see getSectionSize() by
   /// not including section tail padding.
-  uint64_t getSectionAddressSize(const MCSectionData *SD) const;
+  uint64_t getSectionAddressSize(const MCSection *Sec) const;
 
   /// \brief Get the data size of the given section, as emitted to the object
   /// file. This may include additional padding, or be 0 for virtual sections.
-  uint64_t getSectionFileSize(const MCSectionData *SD) const;
+  uint64_t getSectionFileSize(const MCSection *Sec) const;
 
   /// \brief Get the offset of the given symbol, as computed in the current
   /// layout.
-  /// \result True on success.
-  bool getSymbolOffset(const MCSymbolData *SD, uint64_t &Val) const;
+  /// \return True on success.
+  bool getSymbolOffset(const MCSymbol &S, uint64_t &Val) const;
 
   /// \brief Variant that reports a fatal error if the offset is not computable.
-  uint64_t getSymbolOffset(const MCSymbolData *SD) const;
+  uint64_t getSymbolOffset(const MCSymbol &S) const;
 
   /// \brief If this symbol is equivalent to A + Constant, return A.
   const MCSymbol *getBaseSymbol(const MCSymbol &Symbol) const;
diff --git a/include/llvm/MC/MCAssembler.h b/include/llvm/MC/MCAssembler.h
index b8e9227d0f8..0642af837e7 100644
--- a/include/llvm/MC/MCAssembler.h
+++ b/include/llvm/MC/MCAssembler.h
@@ -12,15 +12,16 @@
 
 #include "llvm/ADT/DenseMap.h"
 #include "llvm/ADT/DenseSet.h"
-#include "llvm/ADT/PointerIntPair.h"
 #include "llvm/ADT/SmallPtrSet.h"
 #include "llvm/ADT/SmallString.h"
 #include "llvm/ADT/ilist.h"
 #include "llvm/ADT/ilist_node.h"
+#include "llvm/ADT/iterator.h"
 #include "llvm/MC/MCDirectives.h"
 #include "llvm/MC/MCFixup.h"
 #include "llvm/MC/MCInst.h"
 #include "llvm/MC/MCLinkerOptimizationHint.h"
+#include "llvm/MC/MCSection.h"
 #include "llvm/MC/MCSubtargetInfo.h"
 #include "llvm/Support/Casting.h"
 #include "llvm/Support/DataTypes.h"
@@ -37,21 +38,18 @@ class MCExpr;
 class MCFragment;
 class MCObjectWriter;
 class MCSection;
-class MCSectionData;
 class MCSubtargetInfo;
-class MCSymbol;
-class MCSymbolData;
 class MCValue;
 class MCAsmBackend;
 
 class MCFragment : public ilist_node<MCFragment> {
   friend class MCAsmLayout;
 
-  MCFragment(const MCFragment&) = delete;
-  void operator=(const MCFragment&) = delete;
+  MCFragment(const MCFragment &) = delete;
+  void operator=(const MCFragment &) = delete;
 
 public:
-  enum FragmentType {
+  enum FragmentType : uint8_t {
     FT_Align,
     FT_Data,
     FT_CompactEncodedInst,
@@ -60,20 +58,33 @@ class MCFragment : public ilist_node<MCFragment> {
     FT_Org,
     FT_Dwarf,
     FT_DwarfFrame,
-    FT_LEB
+    FT_LEB,
+    FT_SafeSEH
   };
 
 private:
   FragmentType Kind;
 
-  /// Parent - The data for the section this fragment is in.
-  MCSectionData *Parent;
+protected:
+  bool HasInstructions;
+
+private:
+  /// \brief Should this fragment be aligned to the end of a bundle?
+  bool AlignToBundleEnd;
+
+  uint8_t BundlePadding;
+
+  /// LayoutOrder - The layout order of this fragment.
+  unsigned LayoutOrder;
+
+  /// The data for the section this fragment is in.
+  MCSection *Parent;
 
   /// Atom - The atom this fragment is in, as represented by it's defining
   /// symbol.
-  MCSymbolData *Atom;
+  const MCSymbol *Atom;
 
-  /// @name Assembler Backend Data
+  /// \name Assembler Backend Data
   /// @{
   //
   // FIXME: This could all be kept private to the assembler implementation.
@@ -82,51 +93,55 @@ class MCFragment : public ilist_node<MCFragment> {
   /// initialized.
   uint64_t Offset;
 
-  /// LayoutOrder - The layout order of this fragment.
-  unsigned LayoutOrder;
-
   /// @}
 
 protected:
-  MCFragment(FragmentType Kind, MCSectionData *Parent = nullptr);
+  MCFragment(FragmentType Kind, bool HasInstructions,
+             uint8_t BundlePadding, MCSection *Parent = nullptr);
 
-public:
-  // Only for sentinel.
+  ~MCFragment();
+private:
+
+  // This is a friend so that the sentinal can be created.
+  friend struct ilist_sentinel_traits<MCFragment>;
   MCFragment();
-  virtual ~MCFragment();
+
+public:
+  /// Destroys the current fragment.
+  ///
+  /// This must be used instead of delete as MCFragment is non-virtual.
+  /// This method will dispatch to the appropriate subclass.
+  void destroy();
 
   FragmentType getKind() const { return Kind; }
 
-  MCSectionData *getParent() const { return Parent; }
-  void setParent(MCSectionData *Value) { Parent = Value; }
+  MCSection *getParent() const { return Parent; }
+  void setParent(MCSection *Value) { Parent = Value; }
 
-  MCSymbolData *getAtom() const { return Atom; }
-  void setAtom(MCSymbolData *Value) { Atom = Value; }
+  const MCSymbol *getAtom() const { return Atom; }
+  void setAtom(const MCSymbol *Value) { Atom = Value; }
 
   unsigned getLayoutOrder() const { return LayoutOrder; }
   void setLayoutOrder(unsigned Value) { LayoutOrder = Value; }
 
   /// \brief Does this fragment have instructions emitted into it? By default
   /// this is false, but specific fragment types may set it to true.
-  virtual bool hasInstructions() const { return false; }
+  bool hasInstructions() const { return HasInstructions; }
 
   /// \brief Should this fragment be placed at the end of an aligned bundle?
-  virtual bool alignToBundleEnd() const { return false; }
-  virtual void setAlignToBundleEnd(bool V) { }
+  bool alignToBundleEnd() const { return AlignToBundleEnd; }
+  void setAlignToBundleEnd(bool V) { AlignToBundleEnd = V; }
 
   /// \brief Get the padding size that must be inserted before this fragment.
   /// Used for bundling. By default, no padding is inserted.
   /// Note that padding size is restricted to 8 bits. This is an optimization
   /// to reduce the amount of space used for each fragment. In practice, larger
   /// padding should never be required.
-  virtual uint8_t getBundlePadding() const {
-    return 0;
-  }
+  uint8_t getBundlePadding() const { return BundlePadding; }
 
   /// \brief Set the padding size for this fragment. By default it's a no-op,
   /// and only some fragments have a meaningful implementation.
-  virtual void setBundlePadding(uint8_t N) {
-  }
+  void setBundlePadding(uint8_t N) { BundlePadding = N; }
 
   void dump();
 };
@@ -135,65 +150,72 @@ class MCFragment : public ilist_node<MCFragment> {
 /// data.
 ///
 class MCEncodedFragment : public MCFragment {
-  virtual void anchor();
+protected:
+  MCEncodedFragment(MCFragment::FragmentType FType, bool HasInstructions,
+                    MCSection *Sec)
+      : MCFragment(FType, HasInstructions, 0, Sec) {}
 
-  uint8_t BundlePadding;
 public:
-  MCEncodedFragment(MCFragment::FragmentType FType, MCSectionData *SD = nullptr)
-    : MCFragment(FType, SD), BundlePadding(0)
-  {
-  }
-  ~MCEncodedFragment() override;
-
-  virtual SmallVectorImpl<char> &getContents() = 0;
-  virtual const SmallVectorImpl<char> &getContents() const = 0;
-
-  uint8_t getBundlePadding() const override {
-    return BundlePadding;
-  }
-
-  void setBundlePadding(uint8_t N) override {
-    BundlePadding = N;
-  }
-
   static bool classof(const MCFragment *F) {
     MCFragment::FragmentType Kind = F->getKind();
     switch (Kind) {
-      default:
-        return false;
-      case MCFragment::FT_Relaxable:
-      case MCFragment::FT_CompactEncodedInst:
-      case MCFragment::FT_Data:
-        return true;
+    default:
+      return false;
+    case MCFragment::FT_Relaxable:
+    case MCFragment::FT_CompactEncodedInst:
+    case MCFragment::FT_Data:
+      return true;
     }
   }
 };
 
 /// Interface implemented by fragments that contain encoded instructions and/or
-/// data and also have fixups registered.
+/// data.
 ///
-class MCEncodedFragmentWithFixups : public MCEncodedFragment {
-  void anchor() override;
+template<unsigned ContentsSize>
+class MCEncodedFragmentWithContents : public MCEncodedFragment {
+  SmallVector<char, ContentsSize> Contents;
+
+protected:
+  MCEncodedFragmentWithContents(MCFragment::FragmentType FType,
+                                bool HasInstructions,
+                                MCSection *Sec)
+      : MCEncodedFragment(FType, HasInstructions, Sec) {}
 
 public:
-  MCEncodedFragmentWithFixups(MCFragment::FragmentType FType,
-                              MCSectionData *SD = nullptr)
-    : MCEncodedFragment(FType, SD)
-  {
-  }
+  SmallVectorImpl<char> &getContents() { return Contents; }
+  const SmallVectorImpl<char> &getContents() const { return Contents; }
+};
+
+/// Interface implemented by fragments that contain encoded instructions and/or
+/// data and also have fixups registered.
+///
+template<unsigned ContentsSize, unsigned FixupsSize>
+class MCEncodedFragmentWithFixups :
+  public MCEncodedFragmentWithContents<ContentsSize> {
+
+  /// Fixups - The list of fixups in this fragment.
+  SmallVector<MCFixup, FixupsSize> Fixups;
 
-  ~MCEncodedFragmentWithFixups() override;
+protected:
+  MCEncodedFragmentWithFixups(MCFragment::FragmentType FType,
+                              bool HasInstructions,
+                              MCSection *Sec)
+      : MCEncodedFragmentWithContents<ContentsSize>(FType, HasInstructions,
+                                                    Sec) {}
 
+public:
   typedef SmallVectorImpl<MCFixup>::const_iterator const_fixup_iterator;
   typedef SmallVectorImpl<MCFixup>::iterator fixup_iterator;
 
-  virtual SmallVectorImpl<MCFixup> &getFixups() = 0;
-  virtual const SmallVectorImpl<MCFixup> &getFixups() const = 0;
+  SmallVectorImpl<MCFixup> &getFixups() { return Fixups; }
+  const SmallVectorImpl<MCFixup> &getFixups() const { return Fixups; }
+
+  fixup_iterator fixup_begin() { return Fixups.begin(); }
+  const_fixup_iterator fixup_begin() const { return Fixups.begin(); }
 
-  virtual fixup_iterator fixup_begin() = 0;
-  virtual const_fixup_iterator fixup_begin() const  = 0;
-  virtual fixup_iterator fixup_end() = 0;
-  virtual const_fixup_iterator fixup_end() const = 0;
+  fixup_iterator fixup_end() { return Fixups.end(); }
+  const_fixup_iterator fixup_end() const { return Fixups.end(); }
 
   static bool classof(const MCFragment *F) {
     MCFragment::FragmentType Kind = F->getKind();
@@ -203,50 +225,12 @@ class MCEncodedFragmentWithFixups : public MCEncodedFragment {
 
 /// Fragment for data and encoded instructions.
 ///
-class MCDataFragment : public MCEncodedFragmentWithFixups {
-  void anchor() override;
-
-  /// \brief Does this fragment contain encoded instructions anywhere in it?
-  bool HasInstructions;
-
-  /// \brief Should this fragment be aligned to the end of a bundle?
-  bool AlignToBundleEnd;
-
-  SmallVector<char, 32> Contents;
-
-  /// Fixups - The list of fixups in this fragment.
-  SmallVector<MCFixup, 4> Fixups;
+class MCDataFragment : public MCEncodedFragmentWithFixups<32, 4> {
 public:
-  MCDataFragment(MCSectionData *SD = nullptr)
-    : MCEncodedFragmentWithFixups(FT_Data, SD),
-      HasInstructions(false), AlignToBundleEnd(false)
-  {
-  }
+  MCDataFragment(MCSection *Sec = nullptr)
+      : MCEncodedFragmentWithFixups<32, 4>(FT_Data, false, Sec) {}
 
-  SmallVectorImpl<char> &getContents() override { return Contents; }
-  const SmallVectorImpl<char> &getContents() const override {
-    return Contents;
-  }
-
-  SmallVectorImpl<MCFixup> &getFixups() override {
-    return Fixups;
-  }
-
-  const SmallVectorImpl<MCFixup> &getFixups() const override {
-    return Fixups;
-  }
-
-  bool hasInstructions() const override { return HasInstructions; }
-  virtual void setHasInstructions(bool V) { HasInstructions = V; }
-
-  bool alignToBundleEnd() const override { return AlignToBundleEnd; }
-  void setAlignToBundleEnd(bool V) override { AlignToBundleEnd = V; }
-
-  fixup_iterator fixup_begin() override { return Fixups.begin(); }
-  const_fixup_iterator fixup_begin() const override { return Fixups.begin(); }
-
-  fixup_iterator fixup_end() override {return Fixups.end();}
-  const_fixup_iterator fixup_end() const override {return Fixups.end();}
+  void setHasInstructions(bool V) { HasInstructions = V; }
 
   static bool classof(const MCFragment *F) {
     return F->getKind() == MCFragment::FT_Data;
@@ -258,29 +242,12 @@ class MCDataFragment : public MCEncodedFragmentWithFixups {
 /// it can be used instead of MCDataFragment and lead to lower memory
 /// consumption.
 ///
-class MCCompactEncodedInstFragment : public MCEncodedFragment {
-  void anchor() override;
-
-  /// \brief Should this fragment be aligned to the end of a bundle?
-  bool AlignToBundleEnd;
-
-  SmallVector<char, 4> Contents;
+class MCCompactEncodedInstFragment : public MCEncodedFragmentWithContents<4> {
 public:
-  MCCompactEncodedInstFragment(MCSectionData *SD = nullptr)
-    : MCEncodedFragment(FT_CompactEncodedInst, SD), AlignToBundleEnd(false)
-  {
+  MCCompactEncodedInstFragment(MCSection *Sec = nullptr)
+      : MCEncodedFragmentWithContents(FT_CompactEncodedInst, true, Sec) {
   }
 
-  bool hasInstructions() const override {
-    return true;
-  }
-
-  SmallVectorImpl<char> &getContents() override { return Contents; }
-  const SmallVectorImpl<char> &getContents() const override { return Contents; }
-
-  bool alignToBundleEnd() const override { return AlignToBundleEnd; }
-  void setAlignToBundleEnd(bool V) override { AlignToBundleEnd = V; }
-
   static bool classof(const MCFragment *F) {
     return F->getKind() == MCFragment::FT_CompactEncodedInst;
   }
@@ -289,8 +256,7 @@ class MCCompactEncodedInstFragment : public MCEncodedFragment {
 /// A relaxable fragment holds on to its MCInst, since it may need to be
 /// relaxed during the assembler layout and relaxation stage.
 ///
-class MCRelaxableFragment : public MCEncodedFragmentWithFixups {
-  void anchor() override;
+class MCRelaxableFragment : public MCEncodedFragmentWithFixups<8, 1> {
 
   /// Inst - The instruction this is a fragment for.
   MCInst Inst;
@@ -300,52 +266,32 @@ class MCRelaxableFragment : public MCEncodedFragmentWithFixups {
   /// in the assembler are not seen here.
   const MCSubtargetInfo STI;
 
-  /// Contents - Binary data for the currently encoded instruction.
-  SmallVector<char, 8> Contents;
-
-  /// Fixups - The list of fixups in this fragment.
-  SmallVector<MCFixup, 1> Fixups;
-
 public:
   MCRelaxableFragment(const MCInst &Inst, const MCSubtargetInfo &STI,
-                      MCSectionData *SD = nullptr)
-      : MCEncodedFragmentWithFixups(FT_Relaxable, SD), Inst(Inst), STI(STI) {}
-
-  SmallVectorImpl<char> &getContents() override { return Contents; }
-  const SmallVectorImpl<char> &getContents() const override { return Contents; }
+                      MCSection *Sec = nullptr)
+      : MCEncodedFragmentWithFixups(FT_Relaxable, true, Sec),
+        Inst(Inst), STI(STI) {}
 
   const MCInst &getInst() const { return Inst; }
-  void setInst(const MCInst& Value) { Inst = Value; }
+  void setInst(const MCInst &Value) { Inst = Value; }
 
   const MCSubtargetInfo &getSubtargetInfo() { return STI; }
 
-  SmallVectorImpl<MCFixup> &getFixups() override {
-    return Fixups;
-  }
-
-  const SmallVectorImpl<MCFixup> &getFixups() const override {
-    return Fixups;
-  }
-
-  bool hasInstructions() const override { return true; }
-
-  fixup_iterator fixup_begin() override { return Fixups.begin(); }
-  const_fixup_iterator fixup_begin() const override { return Fixups.begin(); }
-
-  fixup_iterator fixup_end() override {return Fixups.end();}
-  const_fixup_iterator fixup_end() const override {return Fixups.end();}
-
   static bool classof(const MCFragment *F) {
     return F->getKind() == MCFragment::FT_Relaxable;
   }
 };
 
 class MCAlignFragment : public MCFragment {
-  virtual void anchor();
 
   /// Alignment - The alignment to ensure, in bytes.
   unsigned Alignment;
 
+  /// EmitNops - Flag to indicate that (optimal) NOPs should be emitted instead
+  /// of using the provided value. The exact interpretation of this flag is
+  /// target dependent.
+  bool EmitNops : 1;
+
   /// Value - Value to use for filling padding bytes.
   int64_t Value;
 
@@ -356,18 +302,14 @@ class MCAlignFragment : public MCFragment {
   /// cannot be satisfied in this width then this fragment is ignored.
   unsigned MaxBytesToEmit;
 
-  /// EmitNops - Flag to indicate that (optimal) NOPs should be emitted instead
-  /// of using the provided value. The exact interpretation of this flag is
-  /// target dependent.
-  bool EmitNops : 1;
-
 public:
   MCAlignFragment(unsigned Alignment, int64_t Value, unsigned ValueSize,
-                  unsigned MaxBytesToEmit, MCSectionData *SD = nullptr)
-      : MCFragment(FT_Align, SD), Alignment(Alignment), Value(Value),
-        ValueSize(ValueSize), MaxBytesToEmit(MaxBytesToEmit), EmitNops(false) {}
+                  unsigned MaxBytesToEmit, MCSection *Sec = nullptr)
+      : MCFragment(FT_Align, false, 0, Sec), Alignment(Alignment),
+        EmitNops(false), Value(Value),
+        ValueSize(ValueSize), MaxBytesToEmit(MaxBytesToEmit) {}
 
-  /// @name Accessors
+  /// \name Accessors
   /// @{
 
   unsigned getAlignment() const { return Alignment; }
@@ -389,7 +331,6 @@ class MCAlignFragment : public MCFragment {
 };
 
 class MCFillFragment : public MCFragment {
-  virtual void anchor();
 
   /// Value - Value to use for filling bytes.
   int64_t Value;
@@ -403,14 +344,14 @@ class MCFillFragment : public MCFragment {
 
 public:
   MCFillFragment(int64_t Value, unsigned ValueSize, uint64_t Size,
-                 MCSectionData *SD = nullptr)
-      : MCFragment(FT_Fill, SD), Value(Value), ValueSize(ValueSize),
+                 MCSection *Sec = nullptr)
+      : MCFragment(FT_Fill, false, 0, Sec), Value(Value), ValueSize(ValueSize),
         Size(Size) {
     assert((!ValueSize || (Size % ValueSize) == 0) &&
            "Fill size must be a multiple of the value size!");
   }
 
-  /// @name Accessors
+  /// \name Accessors
   /// @{
 
   int64_t getValue() const { return Value; }
@@ -427,7 +368,6 @@ class MCFillFragment : public MCFragment {
 };
 
 class MCOrgFragment : public MCFragment {
-  virtual void anchor();
 
   /// Offset - The offset this fragment should start at.
   const MCExpr *Offset;
@@ -436,10 +376,10 @@ class MCOrgFragment : public MCFragment {
   int8_t Value;
 
 public:
-  MCOrgFragment(const MCExpr &Offset, int8_t Value, MCSectionData *SD = nullptr)
-      : MCFragment(FT_Org, SD), Offset(&Offset), Value(Value) {}
+  MCOrgFragment(const MCExpr &Offset, int8_t Value, MCSection *Sec = nullptr)
+      : MCFragment(FT_Org, false, 0, Sec), Offset(&Offset), Value(Value) {}
 
-  /// @name Accessors
+  /// \name Accessors
   /// @{
 
   const MCExpr &getOffset() const { return *Offset; }
@@ -454,7 +394,6 @@ class MCOrgFragment : public MCFragment {
 };
 
 class MCLEBFragment : public MCFragment {
-  virtual void anchor();
 
   /// Value - The value this fragment should contain.
   const MCExpr *Value;
@@ -463,13 +402,14 @@ class MCLEBFragment : public MCFragment {
   bool IsSigned;
 
   SmallString<8> Contents;
+
 public:
-  MCLEBFragment(const MCExpr &Value_, bool IsSigned_,
-                MCSectionData *SD = nullptr)
-    : MCFragment(FT_LEB, SD),
-      Value(&Value_), IsSigned(IsSigned_) { Contents.push_back(0); }
+  MCLEBFragment(const MCExpr &Value_, bool IsSigned_, MCSection *Sec = nullptr)
+      : MCFragment(FT_LEB, false, 0, Sec), Value(&Value_), IsSigned(IsSigned_) {
+    Contents.push_back(0);
+  }
 
-  /// @name Accessors
+  /// \name Accessors
   /// @{
 
   const MCExpr &getValue() const { return *Value; }
@@ -487,7 +427,6 @@ class MCLEBFragment : public MCFragment {
 };
 
 class MCDwarfLineAddrFragment : public MCFragment {
-  virtual void anchor();
 
   /// LineDelta - the value of the difference between the two line numbers
   /// between two .loc dwarf directives.
@@ -501,12 +440,13 @@ class MCDwarfLineAddrFragment : public MCFragment {
 
 public:
   MCDwarfLineAddrFragment(int64_t LineDelta, const MCExpr &AddrDelta,
-                          MCSectionData *SD = nullptr)
-      : MCFragment(FT_Dwarf, SD), LineDelta(LineDelta), AddrDelta(&AddrDelta) {
+                          MCSection *Sec = nullptr)
+      : MCFragment(FT_Dwarf, false, 0, Sec), LineDelta(LineDelta),
+        AddrDelta(&AddrDelta) {
     Contents.push_back(0);
   }
 
-  /// @name Accessors
+  /// \name Accessors
   /// @{
 
   int64_t getLineDelta() const { return LineDelta; }
@@ -524,7 +464,6 @@ class MCDwarfLineAddrFragment : public MCFragment {
 };
 
 class MCDwarfCallFrameFragment : public MCFragment {
-  virtual void anchor();
 
   /// AddrDelta - The expression for the difference of the two symbols that
   /// make up the address delta between two .cfi_* dwarf directives.
@@ -533,12 +472,12 @@ class MCDwarfCallFrameFragment : public MCFragment {
   SmallString<8> Contents;
 
 public:
-  MCDwarfCallFrameFragment(const MCExpr &AddrDelta, MCSectionData *SD = nullptr)
-      : MCFragment(FT_DwarfFrame, SD), AddrDelta(&AddrDelta) {
+  MCDwarfCallFrameFragment(const MCExpr &AddrDelta, MCSection *Sec = nullptr)
+      : MCFragment(FT_DwarfFrame, false, 0, Sec), AddrDelta(&AddrDelta) {
     Contents.push_back(0);
   }
 
-  /// @name Accessors
+  /// \name Accessors
   /// @{
 
   const MCExpr &getAddrDelta() const { return *AddrDelta; }
@@ -553,265 +492,30 @@ class MCDwarfCallFrameFragment : public MCFragment {
   }
 };
 
-// FIXME: Should this be a separate class, or just merged into MCSection? Since
-// we anticipate the fast path being through an MCAssembler, the only reason to
-// keep it out is for API abstraction.
-class MCSectionData : public ilist_node<MCSectionData> {
-  friend class MCAsmLayout;
-
-  MCSectionData(const MCSectionData&) = delete;
-  void operator=(const MCSectionData&) = delete;
+class MCSafeSEHFragment : public MCFragment {
+  const MCSymbol *Sym;
 
 public:
-  typedef iplist<MCFragment> FragmentListType;
-
-  typedef FragmentListType::const_iterator const_iterator;
-  typedef FragmentListType::iterator iterator;
-
-  typedef FragmentListType::const_reverse_iterator const_reverse_iterator;
-  typedef FragmentListType::reverse_iterator reverse_iterator;
-
-  /// \brief Express the state of bundle locked groups while emitting code.
-  enum BundleLockStateType {
-    NotBundleLocked,
-    BundleLocked,
-    BundleLockedAlignToEnd
-  };
-private:
-  FragmentListType Fragments;
-  const MCSection *Section;
-
-  /// Ordinal - The section index in the assemblers section list.
-  unsigned Ordinal;
-
-  /// LayoutOrder - The index of this section in the layout order.
-  unsigned LayoutOrder;
-
-  /// Alignment - The maximum alignment seen in this section.
-  unsigned Alignment;
-
-  /// \brief Keeping track of bundle-locked state.
-  BundleLockStateType BundleLockState;
+  MCSafeSEHFragment(const MCSymbol *Sym, MCSection *Sec = nullptr)
+      : MCFragment(FT_SafeSEH, false, 0, Sec), Sym(Sym) {}
 
-  /// \brief Current nesting depth of bundle_lock directives.
-  unsigned BundleLockNestingDepth;
-
-  /// \brief We've seen a bundle_lock directive but not its first instruction
-  /// yet.
-  bool BundleGroupBeforeFirstInst;
-
-  /// @name Assembler Backend Data
+  /// \name Accessors
   /// @{
-  //
-  // FIXME: This could all be kept private to the assembler implementation.
 
-  /// HasInstructions - Whether this section has had instructions emitted into
-  /// it.
-  unsigned HasInstructions : 1;
-
-  /// Mapping from subsection number to insertion point for subsection numbers
-  /// below that number.
-  SmallVector<std::pair<unsigned, MCFragment *>, 1> SubsectionFragmentMap;
+  const MCSymbol *getSymbol() { return Sym; }
+  const MCSymbol *getSymbol() const { return Sym; }
 
   /// @}
 
-public:
-  // Only for use as sentinel.
-  MCSectionData();
-  MCSectionData(const MCSection &Section, MCAssembler *A = nullptr);
-
-  const MCSection &getSection() const { return *Section; }
-
-  unsigned getAlignment() const { return Alignment; }
-  void setAlignment(unsigned Value) { Alignment = Value; }
-
-  bool hasInstructions() const { return HasInstructions; }
-  void setHasInstructions(bool Value) { HasInstructions = Value; }
-
-  unsigned getOrdinal() const { return Ordinal; }
-  void setOrdinal(unsigned Value) { Ordinal = Value; }
-
-  unsigned getLayoutOrder() const { return LayoutOrder; }
-  void setLayoutOrder(unsigned Value) { LayoutOrder = Value; }
-
-  /// @name Fragment Access
-  /// @{
-
-  const FragmentListType &getFragmentList() const { return Fragments; }
-  FragmentListType &getFragmentList() { return Fragments; }
-
-  iterator begin() { return Fragments.begin(); }
-  const_iterator begin() const { return Fragments.begin(); }
-
-  iterator end() { return Fragments.end(); }
-  const_iterator end() const { return Fragments.end(); }
-
-  reverse_iterator rbegin() { return Fragments.rbegin(); }
-  const_reverse_iterator rbegin() const { return Fragments.rbegin(); }
-
-  reverse_iterator rend() { return Fragments.rend(); }
-  const_reverse_iterator rend() const { return Fragments.rend(); }
-
-  size_t size() const { return Fragments.size(); }
-
-  bool empty() const { return Fragments.empty(); }
-
-  iterator getSubsectionInsertionPoint(unsigned Subsection);
-
-  bool isBundleLocked() const {
-    return BundleLockState != NotBundleLocked;
-  }
-
-  BundleLockStateType getBundleLockState() const {
-    return BundleLockState;
-  }
-
-  void setBundleLockState(BundleLockStateType NewState);
-
-  bool isBundleGroupBeforeFirstInst() const {
-    return BundleGroupBeforeFirstInst;
-  }
-
-  void setBundleGroupBeforeFirstInst(bool IsFirst) {
-    BundleGroupBeforeFirstInst = IsFirst;
-  }
-
-  void dump();
-
-  /// @}
-};
-
-// FIXME: Same concerns as with SectionData.
-class MCSymbolData : public ilist_node<MCSymbolData> {
-  const MCSymbol *Symbol;
-
-  /// Fragment - The fragment this symbol's value is relative to, if any. Also
-  /// stores if this symbol is visible outside this translation unit (bit 0) or
-  /// if it is private extern (bit 1).
-  PointerIntPair<MCFragment *, 2> Fragment;
-
-  union {
-    /// Offset - The offset to apply to the fragment address to form this
-    /// symbol's value.
-    uint64_t Offset;
-
-    /// CommonSize - The size of the symbol, if it is 'common'.
-    uint64_t CommonSize;
-  };
-
-  /// SymbolSize - An expression describing how to calculate the size of
-  /// a symbol. If a symbol has no size this field will be NULL.
-  const MCExpr *SymbolSize;
-
-  /// CommonAlign - The alignment of the symbol, if it is 'common', or -1.
-  //
-  // FIXME: Pack this in with other fields?
-  unsigned CommonAlign;
-
-  /// Flags - The Flags field is used by object file implementations to store
-  /// additional per symbol information which is not easily classified.
-  uint32_t Flags;
-
-  /// Index - Index field, for use by the object file implementation.
-  uint64_t Index;
-
-public:
-  // Only for use as sentinel.
-  MCSymbolData();
-  MCSymbolData(const MCSymbol &Symbol, MCFragment *Fragment, uint64_t Offset,
-               MCAssembler *A = nullptr);
-
-  /// @name Accessors
-  /// @{
-
-  const MCSymbol &getSymbol() const { return *Symbol; }
-
-  MCFragment *getFragment() const { return Fragment.getPointer(); }
-  void setFragment(MCFragment *Value) { Fragment.setPointer(Value); }
-
-  uint64_t getOffset() const {
-    assert(!isCommon());
-    return Offset;
-  }
-  void setOffset(uint64_t Value) {
-    assert(!isCommon());
-    Offset = Value;
-  }
-
-  /// @}
-  /// @name Symbol Attributes
-  /// @{
-
-  bool isExternal() const { return Fragment.getInt() & 1; }
-  void setExternal(bool Value) {
-    Fragment.setInt((Fragment.getInt() & ~1) | unsigned(Value));
-  }
-
-  bool isPrivateExtern() const { return Fragment.getInt() & 2; }
-  void setPrivateExtern(bool Value) {
-    Fragment.setInt((Fragment.getInt() & ~2) | (unsigned(Value) << 1));
-  }
-
-  /// isCommon - Is this a 'common' symbol.
-  bool isCommon() const { return CommonAlign != -1U; }
-
-  /// setCommon - Mark this symbol as being 'common'.
-  ///
-  /// \param Size - The size of the symbol.
-  /// \param Align - The alignment of the symbol.
-  void setCommon(uint64_t Size, unsigned Align) {
-    assert(getOffset() == 0);
-    CommonSize = Size;
-    CommonAlign = Align;
-  }
-
-  /// getCommonSize - Return the size of a 'common' symbol.
-  uint64_t getCommonSize() const {
-    assert(isCommon() && "Not a 'common' symbol!");
-    return CommonSize;
-  }
-
-  void setSize(const MCExpr *SS) {
-    SymbolSize = SS;
-  }
-
-  const MCExpr *getSize() const {
-    return SymbolSize;
-  }
-
-
-  /// getCommonAlignment - Return the alignment of a 'common' symbol.
-  unsigned getCommonAlignment() const {
-    assert(isCommon() && "Not a 'common' symbol!");
-    return CommonAlign;
-  }
-
-  /// getFlags - Get the (implementation defined) symbol flags.
-  uint32_t getFlags() const { return Flags; }
-
-  /// setFlags - Set the (implementation defined) symbol flags.
-  void setFlags(uint32_t Value) { Flags = Value; }
-
-  /// modifyFlags - Modify the flags via a mask
-  void modifyFlags(uint32_t Value, uint32_t Mask) {
-    Flags = (Flags & ~Mask) | Value;
+  static bool classof(const MCFragment *F) {
+    return F->getKind() == MCFragment::FT_SafeSEH;
   }
-
-  /// getIndex - Get the (implementation defined) index.
-  uint64_t getIndex() const { return Index; }
-
-  /// setIndex - Set the (implementation defined) index.
-  void setIndex(uint64_t Value) { Index = Value; }
-
-  /// @}
-
-  void dump() const;
 };
 
 // FIXME: This really doesn't belong here. See comments below.
 struct IndirectSymbolData {
   MCSymbol *Symbol;
-  MCSectionData *SectionData;
+  MCSection *Section;
 };
 
 // FIXME: Ditto this. Purely so the Streamer and the ObjectWriter can talk
@@ -828,27 +532,25 @@ class MCAssembler {
   friend class MCAsmLayout;
 
 public:
-  typedef iplist<MCSectionData> SectionDataListType;
-  typedef iplist<MCSymbolData> SymbolDataListType;
+  typedef std::vector<MCSection *> SectionListType;
+  typedef std::vector<const MCSymbol *> SymbolDataListType;
 
-  typedef SectionDataListType::const_iterator const_iterator;
-  typedef SectionDataListType::iterator iterator;
+  typedef pointee_iterator<SectionListType::const_iterator> const_iterator;
+  typedef pointee_iterator<SectionListType::iterator> iterator;
 
-  typedef SymbolDataListType::const_iterator const_symbol_iterator;
-  typedef SymbolDataListType::iterator symbol_iterator;
+  typedef pointee_iterator<SymbolDataListType::const_iterator>
+  const_symbol_iterator;
+  typedef pointee_iterator<SymbolDataListType::iterator> symbol_iterator;
 
   typedef iterator_range<symbol_iterator> symbol_range;
   typedef iterator_range<const_symbol_iterator> const_symbol_range;
 
-  typedef std::vector<std::string> FileNameVectorType;
-  typedef FileNameVectorType::const_iterator const_file_name_iterator;
-
   typedef std::vector<IndirectSymbolData>::const_iterator
-    const_indirect_symbol_iterator;
+      const_indirect_symbol_iterator;
   typedef std::vector<IndirectSymbolData>::iterator indirect_symbol_iterator;
 
   typedef std::vector<DataRegionData>::const_iterator
-    const_data_region_iterator;
+      const_data_region_iterator;
   typedef std::vector<DataRegionData>::iterator data_region_iterator;
 
   /// MachO specific deployment target version info.
@@ -860,9 +562,10 @@ class MCAssembler {
     unsigned Minor;
     unsigned Update;
   } VersionMinInfoType;
+
 private:
-  MCAssembler(const MCAssembler&) = delete;
-  void operator=(const MCAssembler&) = delete;
+  MCAssembler(const MCAssembler &) = delete;
+  void operator=(const MCAssembler &) = delete;
 
   MCContext &Context;
 
@@ -874,31 +577,19 @@ class MCAssembler {
 
   raw_ostream &OS;
 
-  iplist<MCSectionData> Sections;
-
-  iplist<MCSymbolData> Symbols;
+  SectionListType Sections;
 
-  DenseSet<const MCSymbol *> LocalsUsedInReloc;
-
-  /// The map of sections to their associated assembler backend data.
-  //
-  // FIXME: Avoid this indirection?
-  DenseMap<const MCSection*, MCSectionData*> SectionMap;
-
-  /// The map of symbols to their associated assembler backend data.
-  //
-  // FIXME: Avoid this indirection?
-  DenseMap<const MCSymbol*, MCSymbolData*> SymbolMap;
+  SymbolDataListType Symbols;
 
   std::vector<IndirectSymbolData> IndirectSymbols;
 
   std::vector<DataRegionData> DataRegions;
 
   /// The list of linker options to propagate into the object file.
-  std::vector<std::vector<std::string> > LinkerOptions;
+  std::vector<std::vector<std::string>> LinkerOptions;
 
   /// List of declared file names
-  FileNameVectorType FileNames;
+  std::vector<std::string> FileNames;
 
   /// The set of function symbols for which a .thumb_func directive has
   /// been seen.
@@ -907,7 +598,7 @@ class MCAssembler {
   // here. Maybe when the relocation stuff moves to target specific,
   // this can go with it? The streamer would need some target specific
   // refactoring too.
-  mutable SmallPtrSet<const MCSymbol*, 64> ThumbFuncs;
+  mutable SmallPtrSet<const MCSymbol *, 64> ThumbFuncs;
 
   /// \brief The bundle alignment size currently set in the assembler.
   ///
@@ -929,6 +620,7 @@ class MCAssembler {
   MCLOHContainer LOHContainer;
 
   VersionMinInfoType VersionMinInfo;
+
 private:
   /// Evaluate a fixup to a relocatable expression and the value which should be
   /// placed into the fixup.
@@ -943,9 +635,9 @@ class MCAssembler {
   /// \return Whether the fixup value was fully resolved. This is true if the
   /// \p Value result is fixed, otherwise the value may change due to
   /// relocation.
-  bool evaluateFixup(const MCAsmLayout &Layout,
-                     const MCFixup &Fixup, const MCFragment *DF,
-                     MCValue &Target, uint64_t &Value) const;
+  bool evaluateFixup(const MCAsmLayout &Layout, const MCFixup &Fixup,
+                     const MCFragment *DF, MCValue &Target,
+                     uint64_t &Value) const;
 
   /// Check whether a fixup can be satisfied, or whether it needs to be relaxed
   /// (increased in size, in order to hold its value correctly).
@@ -962,7 +654,7 @@ class MCAssembler {
 
   /// \brief Perform one layout iteration of the given section and return true
   /// if any offsets were adjusted.
-  bool layoutSectionOnce(MCAsmLayout &Layout, MCSectionData &SD);
+  bool layoutSectionOnce(MCAsmLayout &Layout, MCSection &Sec);
 
   bool relaxInstruction(MCAsmLayout &Layout, MCRelaxableFragment &IF);
 
@@ -979,9 +671,6 @@ class MCAssembler {
                                         MCFragment &F, const MCFixup &Fixup);
 
 public:
-  void addLocalUsedInReloc(const MCSymbol &Sym);
-  bool isLocalUsedInReloc(const MCSymbol &Sym) const;
-
   /// Compute the effective fragment size assuming it is laid out at the given
   /// \p SectionAddress and \p FragmentOffset.
   uint64_t computeFragmentSize(const MCAsmLayout &Layout,
@@ -989,7 +678,7 @@ class MCAssembler {
 
   /// Find the symbol which defines the atom containing the given symbol, or
   /// null if there is no such symbol.
-  const MCSymbolData *getAtom(const MCSymbolData *Symbol) const;
+  const MCSymbol *getAtom(const MCSymbol &S) const;
 
   /// Check whether a particular symbol is visible to the linker and is required
   /// in the symbol table, or whether it can be discarded by the assembler. This
@@ -998,7 +687,7 @@ class MCAssembler {
   bool isSymbolLinkerVisible(const MCSymbol &SD) const;
 
   /// Emit the section contents using the given object writer.
-  void writeSectionData(const MCSectionData *Section,
+  void writeSectionData(const MCSection *Section,
                         const MCAsmLayout &Layout) const;
 
   /// Check whether a given symbol has been flagged with .thumb_func.
@@ -1008,8 +697,8 @@ class MCAssembler {
   void setIsThumbFunc(const MCSymbol *Func) { ThumbFuncs.insert(Func); }
 
   /// ELF e_header flags
-  unsigned getELFHeaderEFlags() const {return ELFHeaderEFlags;}
-  void setELFHeaderEFlags(unsigned Flags) { ELFHeaderEFlags = Flags;}
+  unsigned getELFHeaderEFlags() const { return ELFHeaderEFlags; }
+  void setELFHeaderEFlags(unsigned Flags) { ELFHeaderEFlags = Flags; }
 
   /// MachO deployment target version information.
   const VersionMinInfoType &getVersionMinInfo() const { return VersionMinInfo; }
@@ -1053,36 +742,25 @@ class MCAssembler {
   void Finish();
 
   // FIXME: This does not belong here.
-  bool getSubsectionsViaSymbols() const {
-    return SubsectionsViaSymbols;
-  }
-  void setSubsectionsViaSymbols(bool Value) {
-    SubsectionsViaSymbols = Value;
-  }
+  bool getSubsectionsViaSymbols() const { return SubsectionsViaSymbols; }
+  void setSubsectionsViaSymbols(bool Value) { SubsectionsViaSymbols = Value; }
 
   bool getRelaxAll() const { return RelaxAll; }
   void setRelaxAll(bool Value) { RelaxAll = Value; }
 
-  bool isBundlingEnabled() const {
-    return BundleAlignSize != 0;
-  }
+  bool isBundlingEnabled() const { return BundleAlignSize != 0; }
 
-  unsigned getBundleAlignSize() const {
-    return BundleAlignSize;
-  }
+  unsigned getBundleAlignSize() const { return BundleAlignSize; }
 
   void setBundleAlignSize(unsigned Size) {
-    assert((Size == 0 || !(Size & (Size - 1))) && 
+    assert((Size == 0 || !(Size & (Size - 1))) &&
            "Expect a power-of-two bundle align size");
     BundleAlignSize = Size;
   }
 
-  /// @name Section List Access
+  /// \name Section List Access
   /// @{
 
-  const SectionDataListType &getSectionList() const { return Sections; }
-  SectionDataListType &getSectionList() { return Sections; }
-
   iterator begin() { return Sections.begin(); }
   const_iterator begin() const { return Sections.begin(); }
 
@@ -1092,12 +770,8 @@ class MCAssembler {
   size_t size() const { return Sections.size(); }
 
   /// @}
-  /// @name Symbol List Access
+  /// \name Symbol List Access
   /// @{
-
-  const SymbolDataListType &getSymbolList() const { return Symbols; }
-  SymbolDataListType &getSymbolList() { return Symbols; }
-
   symbol_iterator symbol_begin() { return Symbols.begin(); }
   const_symbol_iterator symbol_begin() const { return Symbols.begin(); }
 
@@ -1105,12 +779,14 @@ class MCAssembler {
   const_symbol_iterator symbol_end() const { return Symbols.end(); }
 
   symbol_range symbols() { return make_range(symbol_begin(), symbol_end()); }
-  const_symbol_range symbols() const { return make_range(symbol_begin(), symbol_end()); }
+  const_symbol_range symbols() const {
+    return make_range(symbol_begin(), symbol_end());
+  }
 
   size_t symbol_size() const { return Symbols.size(); }
 
   /// @}
-  /// @name Indirect Symbol List Access
+  /// \name Indirect Symbol List Access
   /// @{
 
   // FIXME: This is a total hack, this should not be here. Once things are
@@ -1137,34 +813,28 @@ class MCAssembler {
   size_t indirect_symbol_size() const { return IndirectSymbols.size(); }
 
   /// @}
-  /// @name Linker Option List Access
+  /// \name Linker Option List Access
   /// @{
 
-  std::vector<std::vector<std::string> > &getLinkerOptions() {
+  std::vector<std::vector<std::string>> &getLinkerOptions() {
     return LinkerOptions;
   }
 
   /// @}
-  /// @name Data Region List Access
+  /// \name Data Region List Access
   /// @{
 
   // FIXME: This is a total hack, this should not be here. Once things are
   // factored so that the streamer has direct access to the .o writer, it can
   // disappear.
-  std::vector<DataRegionData> &getDataRegions() {
-    return DataRegions;
-  }
+  std::vector<DataRegionData> &getDataRegions() { return DataRegions; }
 
-  data_region_iterator data_region_begin() {
-    return DataRegions.begin();
-  }
+  data_region_iterator data_region_begin() { return DataRegions.begin(); }
   const_data_region_iterator data_region_begin() const {
     return DataRegions.begin();
   }
 
-  data_region_iterator data_region_end() {
-    return DataRegions.end();
-  }
+  data_region_iterator data_region_end() { return DataRegions.end(); }
   const_data_region_iterator data_region_end() const {
     return DataRegions.end();
   }
@@ -1172,76 +842,35 @@ class MCAssembler {
   size_t data_region_size() const { return DataRegions.size(); }
 
   /// @}
-  /// @name Data Region List Access
+  /// \name Data Region List Access
   /// @{
 
   // FIXME: This is a total hack, this should not be here. Once things are
   // factored so that the streamer has direct access to the .o writer, it can
   // disappear.
-  MCLOHContainer & getLOHContainer() {
-    return LOHContainer;
-  }
-  const MCLOHContainer & getLOHContainer() const {
+  MCLOHContainer &getLOHContainer() { return LOHContainer; }
+  const MCLOHContainer &getLOHContainer() const {
     return const_cast<MCAssembler *>(this)->getLOHContainer();
   }
   /// @}
-  /// @name Backend Data Access
+  /// \name Backend Data Access
   /// @{
 
-  MCSectionData &getSectionData(const MCSection &Section) const {
-    MCSectionData *Entry = SectionMap.lookup(&Section);
-    assert(Entry && "Missing section data!");
-    return *Entry;
-  }
-
-  MCSectionData &getOrCreateSectionData(const MCSection &Section,
-                                        bool *Created = nullptr) {
-    MCSectionData *&Entry = SectionMap[&Section];
-
-    if (Created) *Created = !Entry;
-    if (!Entry)
-      Entry = new MCSectionData(Section, this);
-
-    return *Entry;
-  }
-
-  bool hasSymbolData(const MCSymbol &Symbol) const {
-    return SymbolMap.lookup(&Symbol) != nullptr;
-  }
-
-  MCSymbolData &getSymbolData(const MCSymbol &Symbol) {
-    return const_cast<MCSymbolData &>(
-        static_cast<const MCAssembler &>(*this).getSymbolData(Symbol));
-  }
-
-  const MCSymbolData &getSymbolData(const MCSymbol &Symbol) const {
-    MCSymbolData *Entry = SymbolMap.lookup(&Symbol);
-    assert(Entry && "Missing symbol data!");
-    return *Entry;
-  }
-
-  MCSymbolData &getOrCreateSymbolData(const MCSymbol &Symbol,
-                                      bool *Created = nullptr) {
-    MCSymbolData *&Entry = SymbolMap[&Symbol];
-
-    if (Created) *Created = !Entry;
-    if (!Entry)
-      Entry = new MCSymbolData(Symbol, nullptr, 0, this);
-
-    return *Entry;
+  bool registerSection(MCSection &Section) {
+    if (Section.isRegistered())
+      return false;
+    Sections.push_back(&Section);
+    Section.setIsRegistered(true);
+    return true;
   }
 
-  const_file_name_iterator file_names_begin() const {
-    return FileNames.begin();
-  }
+  void registerSymbol(const MCSymbol &Symbol, bool *Created = nullptr);
 
-  const_file_name_iterator file_names_end() const {
-    return FileNames.end();
-  }
+  ArrayRef<std::string> getFileNames() { return FileNames; }
 
   void addFileName(StringRef FileName) {
-    if (std::find(file_names_begin(), file_names_end(), FileName) ==
-        file_names_end())
+    if (std::find(FileNames.begin(), FileNames.end(), FileName) ==
+        FileNames.end())
       FileNames.push_back(FileName);
   }
 
@@ -1258,8 +887,7 @@ class MCAssembler {
 /// \brief Compute the amount of padding required before the fragment \p F to
 /// obey bundling restrictions, where \p FOffset is the fragment's offset in
 /// its section and \p FSize is the fragment's size.
-uint64_t computeBundlePadding(const MCAssembler &Assembler,
-                              const MCFragment *F,
+uint64_t computeBundlePadding(const MCAssembler &Assembler, const MCFragment *F,
                               uint64_t FOffset, uint64_t FSize);
 
 } // end namespace llvm
diff --git a/include/llvm/MC/MCCodeEmitter.h b/include/llvm/MC/MCCodeEmitter.h
index ddf0f82e39e..b6c19150c12 100644
--- a/include/llvm/MC/MCCodeEmitter.h
+++ b/include/llvm/MC/MCCodeEmitter.h
@@ -24,6 +24,7 @@ class MCCodeEmitter {
 private:
   MCCodeEmitter(const MCCodeEmitter &) = delete;
   void operator=(const MCCodeEmitter &) = delete;
+
 protected: // Can only create subclasses.
   MCCodeEmitter();
 
@@ -31,11 +32,11 @@ class MCCodeEmitter {
   virtual ~MCCodeEmitter();
 
   /// Lifetime management
-  virtual void reset() { }
+  virtual void reset() {}
 
   /// EncodeInstruction - Encode the given \p Inst to bytes on the output
   /// stream \p OS.
-  virtual void EncodeInstruction(const MCInst &Inst, raw_ostream &OS,
+  virtual void encodeInstruction(const MCInst &Inst, raw_ostream &OS,
                                  SmallVectorImpl<MCFixup> &Fixups,
                                  const MCSubtargetInfo &STI) const = 0;
 };
diff --git a/include/llvm/MC/MCCodeGenInfo.h b/include/llvm/MC/MCCodeGenInfo.h
index 84ce934d822..0a4744f1d0f 100644
--- a/include/llvm/MC/MCCodeGenInfo.h
+++ b/include/llvm/MC/MCCodeGenInfo.h
@@ -19,33 +19,33 @@
 
 namespace llvm {
 
-  class MCCodeGenInfo {
-    /// RelocationModel - Relocation model: static, pic, etc.
-    ///
-    Reloc::Model RelocationModel;
+class MCCodeGenInfo {
+  /// RelocationModel - Relocation model: static, pic, etc.
+  ///
+  Reloc::Model RelocationModel;
 
-    /// CMModel - Code model.
-    ///
-    CodeModel::Model CMModel;
+  /// CMModel - Code model.
+  ///
+  CodeModel::Model CMModel;
 
-    /// OptLevel - Optimization level.
-    ///
-    CodeGenOpt::Level OptLevel;
+  /// OptLevel - Optimization level.
+  ///
+  CodeGenOpt::Level OptLevel;
 
-  public:
-    void InitMCCodeGenInfo(Reloc::Model RM = Reloc::Default,
-                           CodeModel::Model CM = CodeModel::Default,
-                           CodeGenOpt::Level OL = CodeGenOpt::Default);
+public:
+  void initMCCodeGenInfo(Reloc::Model RM = Reloc::Default,
+                         CodeModel::Model CM = CodeModel::Default,
+                         CodeGenOpt::Level OL = CodeGenOpt::Default);
 
-    Reloc::Model getRelocationModel() const { return RelocationModel; }
+  Reloc::Model getRelocationModel() const { return RelocationModel; }
 
-    CodeModel::Model getCodeModel() const { return CMModel; }
+  CodeModel::Model getCodeModel() const { return CMModel; }
 
-    CodeGenOpt::Level getOptLevel() const { return OptLevel; }
+  CodeGenOpt::Level getOptLevel() const { return OptLevel; }
 
-    // Allow overriding OptLevel on a per-function basis.
-    void setOptLevel(CodeGenOpt::Level Level) { OptLevel = Level; }
-  };
+  // Allow overriding OptLevel on a per-function basis.
+  void setOptLevel(CodeGenOpt::Level Level) { OptLevel = Level; }
+};
 } // namespace llvm
 
 #endif
diff --git a/include/llvm/MC/MCContext.h b/include/llvm/MC/MCContext.h
index bf473ccc68e..41169e9a12a 100644
--- a/include/llvm/MC/MCContext.h
+++ b/include/llvm/MC/MCContext.h
@@ -30,6 +30,7 @@ namespace llvm {
   class MCExpr;
   class MCSection;
   class MCSymbol;
+  class MCSymbolELF;
   class MCLabel;
   struct MCDwarfFile;
   class MCDwarfLoc;
@@ -45,10 +46,12 @@ namespace llvm {
   /// sections that it creates.
   ///
   class MCContext {
-    MCContext(const MCContext&) = delete;
-    MCContext &operator=(const MCContext&) = delete;
+    MCContext(const MCContext &) = delete;
+    MCContext &operator=(const MCContext &) = delete;
+
   public:
-    typedef StringMap<MCSymbol*, BumpPtrAllocator&> SymbolTable;
+    typedef StringMap<MCSymbol *, BumpPtrAllocator &> SymbolTable;
+
   private:
     /// The SourceMgr for this object, if any.
     const SourceMgr *SrcMgr;
@@ -73,19 +76,19 @@ namespace llvm {
 
     /// ELF sections can have a corresponding symbol. This maps one to the
     /// other.
-    DenseMap<const MCSectionELF*, MCSymbol*> SectionSymbols;
+    DenseMap<const MCSectionELF *, MCSymbolELF *> SectionSymbols;
 
-    /// A maping from a local label number and an instance count to a symbol.
+    /// A mapping from a local label number and an instance count to a symbol.
     /// For example, in the assembly
     ///     1:
     ///     2:
     ///     1:
     /// We have three labels represented by the pairs (1, 0), (2, 0) and (1, 1)
-    DenseMap<std::pair<unsigned, unsigned>, MCSymbol*> LocalSymbols;
+    DenseMap<std::pair<unsigned, unsigned>, MCSymbol *> LocalSymbols;
 
     /// Keeps tracks of names that were used both for used declared and
     /// artificial symbols.
-    StringMap<bool, BumpPtrAllocator&> UsedNames;
+    StringMap<bool, BumpPtrAllocator &> UsedNames;
 
     /// The next ID to dole out to an unnamed assembler temporary symbol with
     /// a given prefix.
@@ -134,10 +137,8 @@ namespace llvm {
     /// assembly source files.
     unsigned GenDwarfFileNumber;
 
-    /// Symbols created for the start and end of each section, used for
-    /// generating the .debug_ranges and .debug_aranges sections.
-    MapVector<const MCSection *, std::pair<MCSymbol *, MCSymbol *> >
-    SectionStartEndSyms;
+    /// Sections for generating the .debug_ranges and .debug_aranges sections.
+    SetVector<MCSection *> SectionsForRanges;
 
     /// The information gathered from labels that will have dwarf label
     /// entries when generating dwarf assembly source files.
@@ -158,6 +159,7 @@ namespace llvm {
     /// differences between temporary and non-temporary labels (primarily on
     /// Darwin).
     bool AllowTemporaryLabels;
+    bool UseNamesOnTempLabels = true;
 
     /// The Compile Unit ID that we are currently processing.
     unsigned DwarfCompileUnitID;
@@ -196,15 +198,18 @@ namespace llvm {
       }
     };
 
-    StringMap<const MCSectionMachO*> MachOUniquingMap;
-    std::map<ELFSectionKey, const MCSectionELF *> ELFUniquingMap;
-    std::map<COFFSectionKey, const MCSectionCOFF *> COFFUniquingMap;
+    StringMap<MCSectionMachO *> MachOUniquingMap;
+    std::map<ELFSectionKey, MCSectionELF *> ELFUniquingMap;
+    std::map<COFFSectionKey, MCSectionCOFF *> COFFUniquingMap;
     StringMap<bool> ELFRelSecNames;
 
     /// Do automatic reset in destructor
     bool AutoReset;
 
-    MCSymbol *CreateSymbol(StringRef Name, bool AlwaysAddSuffix);
+    MCSymbol *createSymbolImpl(const StringMapEntry<bool> *Name,
+                               bool CanBeUnnamed);
+    MCSymbol *createSymbol(StringRef Name, bool AlwaysAddSuffix,
+                           bool IsTemporary);
 
     MCSymbol *getOrCreateDirectionalLocalSymbol(unsigned LocalLabelVal,
                                                 unsigned Instance);
@@ -224,8 +229,9 @@ namespace llvm {
     const MCObjectFileInfo *getObjectFileInfo() const { return MOFI; }
 
     void setAllowTemporaryLabels(bool Value) { AllowTemporaryLabels = Value; }
+    void setUseNamesOnTempLabels(bool Value) { UseNamesOnTempLabels = Value; }
 
-    /// @name Module Lifetime Management
+    /// \name Module Lifetime Management
     /// @{
 
     /// reset - return object to right after construction state to prepare
@@ -234,149 +240,145 @@ namespace llvm {
 
     /// @}
 
-    /// @name Symbol Management
+    /// \name Symbol Management
     /// @{
 
     /// Create and return a new linker temporary symbol with a unique but
     /// unspecified name.
-    MCSymbol *CreateLinkerPrivateTempSymbol();
+    MCSymbol *createLinkerPrivateTempSymbol();
 
     /// Create and return a new assembler temporary symbol with a unique but
     /// unspecified name.
-    MCSymbol *CreateTempSymbol();
+    MCSymbol *createTempSymbol(bool CanBeUnnamed = true);
 
-    MCSymbol *createTempSymbol(const Twine &Name, bool AlwaysAddSuffix);
+    MCSymbol *createTempSymbol(const Twine &Name, bool AlwaysAddSuffix,
+                               bool CanBeUnnamed = true);
 
     /// Create the definition of a directional local symbol for numbered label
     /// (used for "1:" definitions).
-    MCSymbol *CreateDirectionalLocalSymbol(unsigned LocalLabelVal);
+    MCSymbol *createDirectionalLocalSymbol(unsigned LocalLabelVal);
 
     /// Create and return a directional local symbol for numbered label (used
     /// for "1b" or 1f" references).
-    MCSymbol *GetDirectionalLocalSymbol(unsigned LocalLabelVal, bool Before);
+    MCSymbol *getDirectionalLocalSymbol(unsigned LocalLabelVal, bool Before);
 
-    /// Lookup the symbol inside with the specified @p Name.  If it exists,
+    /// Lookup the symbol inside with the specified \p Name.  If it exists,
     /// return it.  If not, create a forward reference and return it.
     ///
-    /// @param Name - The symbol name, which must be unique across all symbols.
-    MCSymbol *GetOrCreateSymbol(const Twine &Name);
+    /// \param Name - The symbol name, which must be unique across all symbols.
+    MCSymbol *getOrCreateSymbol(const Twine &Name);
 
-    MCSymbol *getOrCreateSectionSymbol(const MCSectionELF &Section);
+    MCSymbolELF *getOrCreateSectionSymbol(const MCSectionELF &Section);
 
     /// Gets a symbol that will be defined to the final stack offset of a local
     /// variable after codegen.
     ///
-    /// @param Idx - The index of a local variable passed to @llvm.frameescape.
+    /// \param Idx - The index of a local variable passed to @llvm.localescape.
     MCSymbol *getOrCreateFrameAllocSymbol(StringRef FuncName, unsigned Idx);
 
     MCSymbol *getOrCreateParentFrameOffsetSymbol(StringRef FuncName);
 
+    MCSymbol *getOrCreateLSDASymbol(StringRef FuncName);
+
     /// Get the symbol for \p Name, or null.
-    MCSymbol *LookupSymbol(const Twine &Name) const;
+    MCSymbol *lookupSymbol(const Twine &Name) const;
 
     /// getSymbols - Get a reference for the symbol table for clients that
     /// want to, for example, iterate over all symbols. 'const' because we
     /// still want any modifications to the table itself to use the MCContext
     /// APIs.
-    const SymbolTable &getSymbols() const {
-      return Symbols;
-    }
+    const SymbolTable &getSymbols() const { return Symbols; }
 
     /// @}
 
-    /// @name Section Management
+    /// \name Section Management
     /// @{
 
     /// Return the MCSection for the specified mach-o section.  This requires
     /// the operands to be valid.
-    const MCSectionMachO *getMachOSection(StringRef Segment, StringRef Section,
-                                          unsigned TypeAndAttributes,
-                                          unsigned Reserved2, SectionKind K,
-                                          const char *BeginSymName = nullptr);
-
-    const MCSectionMachO *getMachOSection(StringRef Segment, StringRef Section,
-                                          unsigned TypeAndAttributes,
-                                          SectionKind K,
-                                          const char *BeginSymName = nullptr) {
+    MCSectionMachO *getMachOSection(StringRef Segment, StringRef Section,
+                                    unsigned TypeAndAttributes,
+                                    unsigned Reserved2, SectionKind K,
+                                    const char *BeginSymName = nullptr);
+
+    MCSectionMachO *getMachOSection(StringRef Segment, StringRef Section,
+                                    unsigned TypeAndAttributes, SectionKind K,
+                                    const char *BeginSymName = nullptr) {
       return getMachOSection(Segment, Section, TypeAndAttributes, 0, K,
                              BeginSymName);
     }
 
-    const MCSectionELF *getELFSection(StringRef Section, unsigned Type,
-                                      unsigned Flags) {
+    MCSectionELF *getELFSection(StringRef Section, unsigned Type,
+                                unsigned Flags) {
       return getELFSection(Section, Type, Flags, nullptr);
     }
 
-    const MCSectionELF *getELFSection(StringRef Section, unsigned Type,
-                                      unsigned Flags,
-                                      const char *BeginSymName) {
+    MCSectionELF *getELFSection(StringRef Section, unsigned Type,
+                                unsigned Flags, const char *BeginSymName) {
       return getELFSection(Section, Type, Flags, 0, "", BeginSymName);
     }
 
-    const MCSectionELF *getELFSection(StringRef Section, unsigned Type,
-                                      unsigned Flags, unsigned EntrySize,
-                                      StringRef Group) {
+    MCSectionELF *getELFSection(StringRef Section, unsigned Type,
+                                unsigned Flags, unsigned EntrySize,
+                                StringRef Group) {
       return getELFSection(Section, Type, Flags, EntrySize, Group, nullptr);
     }
 
-    const MCSectionELF *getELFSection(StringRef Section, unsigned Type,
-                                      unsigned Flags, unsigned EntrySize,
-                                      StringRef Group,
-                                      const char *BeginSymName) {
+    MCSectionELF *getELFSection(StringRef Section, unsigned Type,
+                                unsigned Flags, unsigned EntrySize,
+                                StringRef Group, const char *BeginSymName) {
       return getELFSection(Section, Type, Flags, EntrySize, Group, ~0,
                            BeginSymName);
     }
 
-    const MCSectionELF *getELFSection(StringRef Section, unsigned Type,
-                                      unsigned Flags, unsigned EntrySize,
-                                      StringRef Group, unsigned UniqueID) {
+    MCSectionELF *getELFSection(StringRef Section, unsigned Type,
+                                unsigned Flags, unsigned EntrySize,
+                                StringRef Group, unsigned UniqueID) {
       return getELFSection(Section, Type, Flags, EntrySize, Group, UniqueID,
                            nullptr);
     }
 
-    const MCSectionELF *getELFSection(StringRef Section, unsigned Type,
-                                      unsigned Flags, unsigned EntrySize,
-                                      StringRef Group, unsigned UniqueID,
-                                      const char *BeginSymName);
+    MCSectionELF *getELFSection(StringRef Section, unsigned Type,
+                                unsigned Flags, unsigned EntrySize,
+                                StringRef Group, unsigned UniqueID,
+                                const char *BeginSymName);
 
-    const MCSectionELF *getELFSection(StringRef Section, unsigned Type,
+    MCSectionELF *getELFSection(StringRef Section, unsigned Type,
+                                unsigned Flags, unsigned EntrySize,
+                                const MCSymbolELF *Group, unsigned UniqueID,
+                                const char *BeginSymName,
+                                const MCSectionELF *Associated);
+
+    MCSectionELF *createELFRelSection(StringRef Name, unsigned Type,
                                       unsigned Flags, unsigned EntrySize,
-                                      const MCSymbol *Group, unsigned UniqueID,
-                                      const char *BeginSymName,
+                                      const MCSymbolELF *Group,
                                       const MCSectionELF *Associated);
 
-    const MCSectionELF *createELFRelSection(StringRef Name, unsigned Type,
-                                            unsigned Flags, unsigned EntrySize,
-                                            const MCSymbol *Group,
-                                            const MCSectionELF *Associated);
-
-    void renameELFSection(const MCSectionELF *Section, StringRef Name);
+    void renameELFSection(MCSectionELF *Section, StringRef Name);
 
-    const MCSectionELF *CreateELFGroupSection();
+    MCSectionELF *createELFGroupSection(const MCSymbolELF *Group);
 
-    const MCSectionCOFF *getCOFFSection(StringRef Section,
-                                        unsigned Characteristics,
-                                        SectionKind Kind,
-                                        StringRef COMDATSymName, int Selection,
-                                        const char *BeginSymName = nullptr);
+    MCSectionCOFF *getCOFFSection(StringRef Section, unsigned Characteristics,
+                                  SectionKind Kind, StringRef COMDATSymName,
+                                  int Selection,
+                                  const char *BeginSymName = nullptr);
 
-    const MCSectionCOFF *getCOFFSection(StringRef Section,
-                                        unsigned Characteristics,
-                                        SectionKind Kind,
-                                        const char *BeginSymName = nullptr);
+    MCSectionCOFF *getCOFFSection(StringRef Section, unsigned Characteristics,
+                                  SectionKind Kind,
+                                  const char *BeginSymName = nullptr);
 
-    const MCSectionCOFF *getCOFFSection(StringRef Section);
+    MCSectionCOFF *getCOFFSection(StringRef Section);
 
     /// Gets or creates a section equivalent to Sec that is associated with the
     /// section containing KeySym. For example, to create a debug info section
     /// associated with an inline function, pass the normal debug info section
     /// as Sec and the function symbol as KeySym.
-    const MCSectionCOFF *getAssociativeCOFFSection(const MCSectionCOFF *Sec,
-                                                   const MCSymbol *KeySym);
+    MCSectionCOFF *getAssociativeCOFFSection(MCSectionCOFF *Sec,
+                                             const MCSymbol *KeySym);
 
     /// @}
 
-    /// @name Dwarf Management
+    /// \name Dwarf Management
     /// @{
 
     /// \brief Get the compilation directory for DW_AT_comp_dir
@@ -399,7 +401,7 @@ namespace llvm {
     void setMainFileName(StringRef S) { MainFileName = S; }
 
     /// Creates an entry in the dwarf file and directory tables.
-    unsigned GetDwarfFile(StringRef Directory, StringRef FileName,
+    unsigned getDwarfFile(StringRef Directory, StringRef FileName,
                           unsigned FileNumber, unsigned CUID);
 
     bool isValidDwarfFileNumber(unsigned FileNumber, unsigned CUID = 0);
@@ -431,9 +433,7 @@ namespace llvm {
           return true;
       return false;
     }
-    unsigned getDwarfCompileUnitID() {
-      return DwarfCompileUnitID;
-    }
+    unsigned getDwarfCompileUnitID() { return DwarfCompileUnitID; }
     void setDwarfCompileUnitID(unsigned CUIndex) {
       DwarfCompileUnitID = CUIndex;
     }
@@ -456,7 +456,7 @@ namespace llvm {
       CurrentDwarfLoc.setDiscriminator(Discriminator);
       DwarfLocSeen = true;
     }
-    void ClearDwarfLocSeen() { DwarfLocSeen = false; }
+    void clearDwarfLocSeen() { DwarfLocSeen = false; }
 
     bool getDwarfLocSeen() { return DwarfLocSeen; }
     const MCDwarfLoc &getCurrentDwarfLoc() { return CurrentDwarfLoc; }
@@ -467,17 +467,13 @@ namespace llvm {
     void setGenDwarfFileNumber(unsigned FileNumber) {
       GenDwarfFileNumber = FileNumber;
     }
-    MapVector<const MCSection *, std::pair<MCSymbol *, MCSymbol *> > &
-    getGenDwarfSectionSyms() {
-      return SectionStartEndSyms;
+    const SetVector<MCSection *> &getGenDwarfSectionSyms() {
+      return SectionsForRanges;
     }
-    std::pair<MapVector<const MCSection *,
-                        std::pair<MCSymbol *, MCSymbol *> >::iterator,
-              bool>
-    addGenDwarfSection(const MCSection *Sec) {
-      return SectionStartEndSyms.insert(
-          std::make_pair(Sec, std::make_pair(nullptr, nullptr)));
+    bool addGenDwarfSection(MCSection *Sec) {
+      return SectionsForRanges.insert(Sec);
     }
+
     void finalizeDwarfSections(MCStreamer &MCOS);
     const std::vector<MCGenDwarfLabelEntry> &getMCGenDwarfLabelEntries() const {
       return MCGenDwarfLabelEntries;
@@ -500,56 +496,52 @@ namespace llvm {
     char *getSecureLogFile() { return SecureLogFile; }
     raw_ostream *getSecureLog() { return SecureLog; }
     bool getSecureLogUsed() { return SecureLogUsed; }
-    void setSecureLog(raw_ostream *Value) {
-      SecureLog = Value;
-    }
-    void setSecureLogUsed(bool Value) {
-      SecureLogUsed = Value;
-    }
+    void setSecureLog(raw_ostream *Value) { SecureLog = Value; }
+    void setSecureLogUsed(bool Value) { SecureLogUsed = Value; }
 
-    void *Allocate(unsigned Size, unsigned Align = 8) {
+    void *allocate(unsigned Size, unsigned Align = 8) {
       return Allocator.Allocate(Size, Align);
     }
-    void Deallocate(void *Ptr) {
-    }
+    void deallocate(void *Ptr) {}
 
     // Unrecoverable error has occurred. Display the best diagnostic we can
     // and bail via exit(1). For now, most MC backend errors are unrecoverable.
     // FIXME: We should really do something about that.
-    LLVM_ATTRIBUTE_NORETURN void FatalError(SMLoc L, const Twine &Msg) const;
+    LLVM_ATTRIBUTE_NORETURN void reportFatalError(SMLoc L,
+                                                  const Twine &Msg) const;
   };
 
 } // end namespace llvm
 
 // operator new and delete aren't allowed inside namespaces.
 // The throw specifications are mandated by the standard.
-/// @brief Placement new for using the MCContext's allocator.
+/// \brief Placement new for using the MCContext's allocator.
 ///
 /// This placement form of operator new uses the MCContext's allocator for
 /// obtaining memory. It is a non-throwing new, which means that it returns
 /// null on error. (If that is what the allocator does. The current does, so if
 /// this ever changes, this operator will have to be changed, too.)
 /// Usage looks like this (assuming there's an MCContext 'Context' in scope):
-/// @code
-/// // Default alignment (16)
+/// \code
+/// // Default alignment (8)
 /// IntegerLiteral *Ex = new (Context) IntegerLiteral(arguments);
 /// // Specific alignment
-/// IntegerLiteral *Ex2 = new (Context, 8) IntegerLiteral(arguments);
-/// @endcode
+/// IntegerLiteral *Ex2 = new (Context, 4) IntegerLiteral(arguments);
+/// \endcode
 /// Please note that you cannot use delete on the pointer; it must be
 /// deallocated using an explicit destructor call followed by
-/// @c Context.Deallocate(Ptr).
+/// \c Context.Deallocate(Ptr).
 ///
-/// @param Bytes The number of bytes to allocate. Calculated by the compiler.
-/// @param C The MCContext that provides the allocator.
-/// @param Alignment The alignment of the allocated memory (if the underlying
+/// \param Bytes The number of bytes to allocate. Calculated by the compiler.
+/// \param C The MCContext that provides the allocator.
+/// \param Alignment The alignment of the allocated memory (if the underlying
 ///                  allocator supports it).
-/// @return The allocated memory. Could be NULL.
+/// \return The allocated memory. Could be NULL.
 inline void *operator new(size_t Bytes, llvm::MCContext &C,
-                          size_t Alignment = 16) throw () {
-  return C.Allocate(Bytes, Alignment);
+                          size_t Alignment = 8) throw() {
+  return C.allocate(Bytes, Alignment);
 }
-/// @brief Placement delete companion to the new above.
+/// \brief Placement delete companion to the new above.
 ///
 /// This operator is just a companion to the new above. There is no way of
 /// invoking it directly; see the new operator for more details. This operator
@@ -557,41 +549,41 @@ inline void *operator new(size_t Bytes, llvm::MCContext &C,
 /// the MCContext throws in the object constructor.
 inline void operator delete(void *Ptr, llvm::MCContext &C, size_t)
               throw () {
-  C.Deallocate(Ptr);
+  C.deallocate(Ptr);
 }
 
 /// This placement form of operator new[] uses the MCContext's allocator for
 /// obtaining memory. It is a non-throwing new[], which means that it returns
 /// null on error.
 /// Usage looks like this (assuming there's an MCContext 'Context' in scope):
-/// @code
-/// // Default alignment (16)
+/// \code
+/// // Default alignment (8)
 /// char *data = new (Context) char[10];
 /// // Specific alignment
-/// char *data = new (Context, 8) char[10];
-/// @endcode
+/// char *data = new (Context, 4) char[10];
+/// \endcode
 /// Please note that you cannot use delete on the pointer; it must be
 /// deallocated using an explicit destructor call followed by
-/// @c Context.Deallocate(Ptr).
+/// \c Context.Deallocate(Ptr).
 ///
-/// @param Bytes The number of bytes to allocate. Calculated by the compiler.
-/// @param C The MCContext that provides the allocator.
-/// @param Alignment The alignment of the allocated memory (if the underlying
+/// \param Bytes The number of bytes to allocate. Calculated by the compiler.
+/// \param C The MCContext that provides the allocator.
+/// \param Alignment The alignment of the allocated memory (if the underlying
 ///                  allocator supports it).
-/// @return The allocated memory. Could be NULL.
+/// \return The allocated memory. Could be NULL.
 inline void *operator new[](size_t Bytes, llvm::MCContext& C,
-                            size_t Alignment = 16) throw () {
-  return C.Allocate(Bytes, Alignment);
+                            size_t Alignment = 8) throw() {
+  return C.allocate(Bytes, Alignment);
 }
 
-/// @brief Placement delete[] companion to the new[] above.
+/// \brief Placement delete[] companion to the new[] above.
 ///
 /// This operator is just a companion to the new[] above. There is no way of
 /// invoking it directly; see the new[] operator for more details. This operator
 /// is called implicitly by the compiler if a placement new[] expression using
 /// the MCContext throws in the object constructor.
 inline void operator delete[](void *Ptr, llvm::MCContext &C) throw () {
-  C.Deallocate(Ptr);
+  C.deallocate(Ptr);
 }
 
 #endif
diff --git a/include/llvm/MC/MCDisassembler.h b/include/llvm/MC/MCDisassembler.h
index d6b0a305b1d..57c40d660f6 100644
--- a/include/llvm/MC/MCDisassembler.h
+++ b/include/llvm/MC/MCDisassembler.h
@@ -11,7 +11,6 @@
 
 #include "llvm-c/Disassembler.h"
 #include "llvm/ADT/ArrayRef.h"
-#include "llvm/MC/MCRelocationInfo.h"
 #include "llvm/MC/MCSymbolizer.h"
 #include "llvm/Support/DataTypes.h"
 
@@ -61,16 +60,16 @@ class MCDisassembler {
 
   /// Returns the disassembly of a single instruction.
   ///
-  /// @param Instr    - An MCInst to populate with the contents of the
+  /// \param Instr    - An MCInst to populate with the contents of the
   ///                   instruction.
-  /// @param Size     - A value to populate with the size of the instruction, or
+  /// \param Size     - A value to populate with the size of the instruction, or
   ///                   the number of bytes consumed while attempting to decode
   ///                   an invalid instruction.
-  /// @param Address  - The address, in the memory space of region, of the first
+  /// \param Address  - The address, in the memory space of region, of the first
   ///                   byte of the instruction.
-  /// @param VStream  - The stream to print warnings and diagnostic messages on.
-  /// @param CStream  - The stream to print comments and annotations on.
-  /// @return         - MCDisassembler::Success if the instruction is valid,
+  /// \param VStream  - The stream to print warnings and diagnostic messages on.
+  /// \param CStream  - The stream to print comments and annotations on.
+  /// \return         - MCDisassembler::Success if the instruction is valid,
   ///                   MCDisassembler::SoftFail if the instruction was
   ///                                            disassemblable but invalid,
   ///                   MCDisassembler::Fail if the instruction was invalid.
diff --git a/include/llvm/MC/MCDwarf.h b/include/llvm/MC/MCDwarf.h
index c266acf2f09..1e72dfee4ad 100644
--- a/include/llvm/MC/MCDwarf.h
+++ b/include/llvm/MC/MCDwarf.h
@@ -37,35 +37,30 @@ class MCSymbol;
 class SourceMgr;
 class SMLoc;
 
-/// MCDwarfFile - Instances of this class represent the name of the dwarf
+/// \brief Instances of this class represent the name of the dwarf
 /// .file directive and its associated dwarf file number in the MC file,
-/// and MCDwarfFile's are created and unique'd by the MCContext class where
+/// and MCDwarfFile's are created and uniqued by the MCContext class where
 /// the file number for each is its index into the vector of DwarfFiles (note
 /// index 0 is not used and not a valid dwarf file number).
 struct MCDwarfFile {
-  // Name - the base name of the file without its directory path.
+  // \brief The base name of the file without its directory path.
   // The StringRef references memory allocated in the MCContext.
   std::string Name;
 
-  // DirIndex - the index into the list of directory names for this file name.
+  // \brief The index into the list of directory names for this file name.
   unsigned DirIndex;
 };
 
-/// MCDwarfLoc - Instances of this class represent the information from a
+/// \brief Instances of this class represent the information from a
 /// dwarf .loc directive.
 class MCDwarfLoc {
-  // FileNum - the file number.
-  unsigned FileNum;
-  // Line - the line number.
-  unsigned Line;
-  // Column - the column position.
-  unsigned Column;
+  uint32_t FileNum;
+  uint32_t Line;
+  uint16_t Column;
   // Flags (see #define's below)
-  unsigned Flags;
-  // Isa
-  unsigned Isa;
-  // Discriminator
-  unsigned Discriminator;
+  uint8_t Flags;
+  uint8_t Isa;
+  uint32_t Discriminator;
 
 // Flag that indicates the initial value of the is_stmt_start flag.
 #define DWARF2_LINE_DEFAULT_IS_STMT 1
@@ -87,46 +82,55 @@ class MCDwarfLoc {
   // for an MCDwarfLoc object.
 
 public:
-  /// getFileNum - Get the FileNum of this MCDwarfLoc.
+  /// \brief Get the FileNum of this MCDwarfLoc.
   unsigned getFileNum() const { return FileNum; }
 
-  /// getLine - Get the Line of this MCDwarfLoc.
+  /// \brief Get the Line of this MCDwarfLoc.
   unsigned getLine() const { return Line; }
 
-  /// getColumn - Get the Column of this MCDwarfLoc.
+  /// \brief Get the Column of this MCDwarfLoc.
   unsigned getColumn() const { return Column; }
 
-  /// getFlags - Get the Flags of this MCDwarfLoc.
+  /// \brief Get the Flags of this MCDwarfLoc.
   unsigned getFlags() const { return Flags; }
 
-  /// getIsa - Get the Isa of this MCDwarfLoc.
+  /// \brief Get the Isa of this MCDwarfLoc.
   unsigned getIsa() const { return Isa; }
 
-  /// getDiscriminator - Get the Discriminator of this MCDwarfLoc.
+  /// \brief Get the Discriminator of this MCDwarfLoc.
   unsigned getDiscriminator() const { return Discriminator; }
 
-  /// setFileNum - Set the FileNum of this MCDwarfLoc.
+  /// \brief Set the FileNum of this MCDwarfLoc.
   void setFileNum(unsigned fileNum) { FileNum = fileNum; }
 
-  /// setLine - Set the Line of this MCDwarfLoc.
+  /// \brief Set the Line of this MCDwarfLoc.
   void setLine(unsigned line) { Line = line; }
 
-  /// setColumn - Set the Column of this MCDwarfLoc.
-  void setColumn(unsigned column) { Column = column; }
+  /// \brief Set the Column of this MCDwarfLoc.
+  void setColumn(unsigned column) {
+    assert(column <= UINT16_MAX);
+    Column = column;
+  }
 
-  /// setFlags - Set the Flags of this MCDwarfLoc.
-  void setFlags(unsigned flags) { Flags = flags; }
+  /// \brief Set the Flags of this MCDwarfLoc.
+  void setFlags(unsigned flags) {
+    assert(flags <= UINT8_MAX);
+    Flags = flags;
+  }
 
-  /// setIsa - Set the Isa of this MCDwarfLoc.
-  void setIsa(unsigned isa) { Isa = isa; }
+  /// \brief Set the Isa of this MCDwarfLoc.
+  void setIsa(unsigned isa) {
+    assert(isa <= UINT8_MAX);
+    Isa = isa;
+  }
 
-  /// setDiscriminator - Set the Discriminator of this MCDwarfLoc.
+  /// \brief Set the Discriminator of this MCDwarfLoc.
   void setDiscriminator(unsigned discriminator) {
     Discriminator = discriminator;
   }
 };
 
-/// MCLineEntry - Instances of this class represent the line information for
+/// \brief Instances of this class represent the line information for
 /// the dwarf line table entries.  Which is created after a machine
 /// instruction is assembled and uses an address from a temporary label
 /// created at the current address in the current section and the info from
@@ -148,24 +152,24 @@ class MCLineEntry : public MCDwarfLoc {
   // This is called when an instruction is assembled into the specified
   // section and if there is information from the last .loc directive that
   // has yet to have a line entry made for it is made.
-  static void Make(MCObjectStreamer *MCOS, const MCSection *Section);
+  static void Make(MCObjectStreamer *MCOS, MCSection *Section);
 };
 
-/// MCLineSection - Instances of this class represent the line information
-/// for a compile unit where machine instructions have been assembled after seeing
-/// .loc directives.  This is the information used to build the dwarf line
+/// \brief Instances of this class represent the line information for a compile
+/// unit where machine instructions have been assembled after seeing .loc
+/// directives.  This is the information used to build the dwarf line
 /// table for a section.
 class MCLineSection {
 public:
-  // addLineEntry - adds an entry to this MCLineSection's line entries
-  void addLineEntry(const MCLineEntry &LineEntry, const MCSection *Sec) {
+  // \brief Add an entry to this MCLineSection's line entries.
+  void addLineEntry(const MCLineEntry &LineEntry, MCSection *Sec) {
     MCLineDivisions[Sec].push_back(LineEntry);
   }
 
   typedef std::vector<MCLineEntry> MCLineEntryCollection;
   typedef MCLineEntryCollection::iterator iterator;
   typedef MCLineEntryCollection::const_iterator const_iterator;
-  typedef MapVector<const MCSection *, MCLineEntryCollection> MCLineDivisionMap;
+  typedef MapVector<MCSection *, MCLineEntryCollection> MCLineDivisionMap;
 
 private:
   // A collection of MCLineEntry for each section.
diff --git a/include/llvm/MC/MCELF.h b/include/llvm/MC/MCELF.h
deleted file mode 100644
index 294a51bf7c7..00000000000
--- a/include/llvm/MC/MCELF.h
+++ /dev/null
@@ -1,37 +0,0 @@
-//===- lib/MC/MCELF.h - ELF MC --------------------------------------------===//
-//
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
-//
-//===----------------------------------------------------------------------===//
-//
-// This file contains some support functions used by the ELF Streamer and
-// ObjectWriter.
-//
-//===----------------------------------------------------------------------===//
-
-#ifndef LLVM_MC_MCELF_H
-#define LLVM_MC_MCELF_H
-
-#include "llvm/MC/MCExpr.h"
-
-namespace llvm {
-class MCSymbolData;
-
-class MCELF {
- public:
-  static void SetBinding(MCSymbolData &SD, unsigned Binding);
-  static unsigned GetBinding(const MCSymbolData &SD);
-  static void SetType(MCSymbolData &SD, unsigned Type);
-  static unsigned GetType(const MCSymbolData &SD);
-  static void SetVisibility(MCSymbolData &SD, unsigned Visibility);
-  static unsigned GetVisibility(const MCSymbolData &SD);
-  static void setOther(MCSymbolData &SD, unsigned Other);
-  static unsigned getOther(const MCSymbolData &SD);
-};
-
-}
-
-#endif
diff --git a/include/llvm/MC/MCELFObjectWriter.h b/include/llvm/MC/MCELFObjectWriter.h
index e9fef76c20f..01f694d3b75 100644
--- a/include/llvm/MC/MCELFObjectWriter.h
+++ b/include/llvm/MC/MCELFObjectWriter.h
@@ -20,19 +20,18 @@ class MCAssembler;
 class MCFixup;
 class MCFragment;
 class MCObjectWriter;
-class MCSectionData;
 class MCSymbol;
-class MCSymbolData;
+class MCSymbolELF;
 class MCValue;
 class raw_pwrite_stream;
 
 struct ELFRelocationEntry {
   uint64_t Offset; // Where is the relocation.
-  const MCSymbol *Symbol;       // The symbol to relocate with.
+  const MCSymbolELF *Symbol; // The symbol to relocate with.
   unsigned Type;   // The type of the relocation.
   uint64_t Addend; // The addend to use.
 
-  ELFRelocationEntry(uint64_t Offset, const MCSymbol *Symbol, unsigned Type,
+  ELFRelocationEntry(uint64_t Offset, const MCSymbolELF *Symbol, unsigned Type,
                      uint64_t Addend)
       : Offset(Offset), Symbol(Symbol), Type(Type), Addend(Addend) {}
 };
@@ -70,13 +69,13 @@ class MCELFObjectTargetWriter {
   virtual unsigned GetRelocType(const MCValue &Target, const MCFixup &Fixup,
                                 bool IsPCRel) const = 0;
 
-  virtual bool needsRelocateWithSymbol(const MCSymbolData &SD,
+  virtual bool needsRelocateWithSymbol(const MCSymbol &Sym,
                                        unsigned Type) const;
 
   virtual void sortRelocs(const MCAssembler &Asm,
                           std::vector<ELFRelocationEntry> &Relocs);
 
-  /// @name Accessors
+  /// \name Accessors
   /// @{
   uint8_t getOSABI() const { return OSABI; }
   uint16_t getEMachine() const { return EMachine; }
diff --git a/include/llvm/MC/MCELFStreamer.h b/include/llvm/MC/MCELFStreamer.h
index ace4ee3e542..a5b257f5958 100644
--- a/include/llvm/MC/MCELFStreamer.h
+++ b/include/llvm/MC/MCELFStreamer.h
@@ -23,8 +23,6 @@ class MCAssembler;
 class MCCodeEmitter;
 class MCExpr;
 class MCInst;
-class MCSymbol;
-class MCSymbolData;
 class raw_ostream;
 
 class MCELFStreamer : public MCObjectStreamer {
@@ -39,17 +37,15 @@ class MCELFStreamer : public MCObjectStreamer {
   void reset() override {
     SeenIdent = false;
     LocalCommons.clear();
-    BindingExplicitlySet.clear();
     BundleGroups.clear();
     MCObjectStreamer::reset();
   }
 
-  /// @name MCStreamer Interface
+  /// \name MCStreamer Interface
   /// @{
 
   void InitSections(bool NoExecStack) override;
-  void ChangeSection(const MCSection *Section,
-                     const MCExpr *Subsection) override;
+  void ChangeSection(MCSection *Section, const MCExpr *Subsection) override;
   void EmitLabel(MCSymbol *Symbol) override;
   void EmitAssemblerFlag(MCAssemblerFlag Flag) override;
   void EmitThumbFunc(MCSymbol *Func) override;
@@ -63,15 +59,15 @@ class MCELFStreamer : public MCObjectStreamer {
   void EmitCOFFSymbolType(int Type) override;
   void EndCOFFSymbolDef() override;
 
-  void EmitELFSize(MCSymbol *Symbol, const MCExpr *Value) override;
+  void emitELFSize(MCSymbolELF *Symbol, const MCExpr *Value) override;
 
   void EmitLocalCommonSymbol(MCSymbol *Symbol, uint64_t Size,
                              unsigned ByteAlignment) override;
 
-  void EmitZerofill(const MCSection *Section, MCSymbol *Symbol = nullptr,
+  void EmitZerofill(MCSection *Section, MCSymbol *Symbol = nullptr,
                     uint64_t Size = 0, unsigned ByteAlignment = 0) override;
-  void EmitTBSSSymbol(const MCSection *Section, MCSymbol *Symbol,
-                      uint64_t Size, unsigned ByteAlignment = 0) override;
+  void EmitTBSSSymbol(MCSection *Section, MCSymbol *Symbol, uint64_t Size,
+                      unsigned ByteAlignment = 0) override;
   void EmitValueImpl(const MCExpr *Value, unsigned Size,
                      const SMLoc &Loc = SMLoc()) override;
 
@@ -90,26 +86,25 @@ class MCELFStreamer : public MCObjectStreamer {
   void EmitBundleUnlock() override;
 
 private:
+  bool isBundleLocked() const;
   void EmitInstToFragment(const MCInst &Inst, const MCSubtargetInfo &) override;
   void EmitInstToData(const MCInst &Inst, const MCSubtargetInfo &) override;
 
   void fixSymbolsInTLSFixups(const MCExpr *expr);
 
   /// \brief Merge the content of the fragment \p EF into the fragment \p DF.
-  void mergeFragment(MCDataFragment *, MCEncodedFragmentWithFixups *);
+  void mergeFragment(MCDataFragment *, MCDataFragment *);
 
   bool SeenIdent;
 
   struct LocalCommon {
-    MCSymbolData *SD;
+    const MCSymbol *Symbol;
     uint64_t Size;
     unsigned ByteAlignment;
   };
 
   std::vector<LocalCommon> LocalCommons;
 
-  SmallPtrSet<MCSymbol *, 16> BindingExplicitlySet;
-
   /// BundleGroups - The stack of fragments holding the bundle-locked
   /// instructions.
   llvm::SmallVector<MCDataFragment *, 4> BundleGroups;
diff --git a/include/llvm/MC/MCELFSymbolFlags.h b/include/llvm/MC/MCELFSymbolFlags.h
deleted file mode 100644
index 297c44269a8..00000000000
--- a/include/llvm/MC/MCELFSymbolFlags.h
+++ /dev/null
@@ -1,56 +0,0 @@
-//===- MCELFSymbolFlags.h - ELF Symbol Flags ----------------*- C++ -*-===//
-//
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
-//
-//===----------------------------------------------------------------------===//
-//
-// This file declares the SymbolFlags used for the ELF target.
-//
-//===----------------------------------------------------------------------===//
-
-#ifndef LLVM_MC_MCELFSYMBOLFLAGS_H
-#define LLVM_MC_MCELFSYMBOLFLAGS_H
-
-#include "llvm/Support/ELF.h"
-
-// Because all the symbol flags need to be stored in the MCSymbolData
-// 'flags' variable we need to provide shift constants per flag type.
-
-namespace llvm {
-  enum {
-    ELF_STT_Shift   = 0,  // Shift value for STT_* flags.
-    ELF_STB_Shift   = 4,  // Shift value for STB_* flags.
-    ELF_STV_Shift   = 8,  // Shift value for STV_* flags.
-    ELF_STO_Shift   = 10  // Shift value for STO_* flags.
-  };
-
-  enum ELFSymbolFlags {
-    ELF_STB_Local     = (ELF::STB_LOCAL     << ELF_STB_Shift),
-      ELF_STB_Global    = (ELF::STB_GLOBAL    << ELF_STB_Shift),
-      ELF_STB_Weak      = (ELF::STB_WEAK      << ELF_STB_Shift),
-      ELF_STB_Loproc    = (ELF::STB_LOPROC    << ELF_STB_Shift),
-      ELF_STB_Hiproc    = (ELF::STB_HIPROC    << ELF_STB_Shift),
-
-      ELF_STT_Notype    = (ELF::STT_NOTYPE    << ELF_STT_Shift),
-      ELF_STT_Object    = (ELF::STT_OBJECT    << ELF_STT_Shift),
-      ELF_STT_Func      = (ELF::STT_FUNC      << ELF_STT_Shift),
-      ELF_STT_Section   = (ELF::STT_SECTION   << ELF_STT_Shift),
-      ELF_STT_File      = (ELF::STT_FILE      << ELF_STT_Shift),
-      ELF_STT_Common    = (ELF::STT_COMMON    << ELF_STT_Shift),
-      ELF_STT_Tls       = (ELF::STT_TLS       << ELF_STT_Shift),
-      ELF_STT_GnuIFunc  = (ELF::STT_GNU_IFUNC << ELF_STT_Shift),
-      ELF_STT_Loproc    = (ELF::STT_LOPROC    << ELF_STT_Shift),
-      ELF_STT_Hiproc    = (ELF::STT_HIPROC    << ELF_STT_Shift),
-
-      ELF_STV_Default   = (ELF::STV_DEFAULT   << ELF_STV_Shift),
-      ELF_STV_Internal  = (ELF::STV_INTERNAL  << ELF_STV_Shift),
-      ELF_STV_Hidden    = (ELF::STV_HIDDEN    << ELF_STV_Shift),
-      ELF_STV_Protected = (ELF::STV_PROTECTED << ELF_STV_Shift)
-  };
-
-} // end namespace llvm
-
-#endif
diff --git a/include/llvm/MC/MCExpr.h b/include/llvm/MC/MCExpr.h
index d5a68be3ec6..b3a607351a8 100644
--- a/include/llvm/MC/MCExpr.h
+++ b/include/llvm/MC/MCExpr.h
@@ -21,15 +21,14 @@ class MCAssembler;
 class MCContext;
 class MCFixup;
 class MCSection;
-class MCSectionData;
 class MCStreamer;
 class MCSymbol;
 class MCValue;
 class raw_ostream;
 class StringRef;
-typedef DenseMap<const MCSectionData*, uint64_t> SectionAddrMap;
+typedef DenseMap<const MCSection *, uint64_t> SectionAddrMap;
 
-/// MCExpr - Base class for the full range of assembler expressions which are
+/// \brief Base class for the full range of assembler expressions which are
 /// needed for parsing.
 class MCExpr {
 public:
@@ -47,7 +46,7 @@ class MCExpr {
   MCExpr(const MCExpr&) = delete;
   void operator=(const MCExpr&) = delete;
 
-  bool EvaluateAsAbsolute(int64_t &Res, const MCAssembler *Asm,
+  bool evaluateAsAbsolute(int64_t &Res, const MCAssembler *Asm,
                           const MCAsmLayout *Layout,
                           const SectionAddrMap *Addrs) const;
 
@@ -58,75 +57,75 @@ class MCExpr {
 protected:
   explicit MCExpr(ExprKind Kind) : Kind(Kind) {}
 
-  bool EvaluateAsRelocatableImpl(MCValue &Res, const MCAssembler *Asm,
+  bool evaluateAsRelocatableImpl(MCValue &Res, const MCAssembler *Asm,
                                  const MCAsmLayout *Layout,
                                  const MCFixup *Fixup,
                                  const SectionAddrMap *Addrs, bool InSet) const;
 
 public:
-  /// @name Accessors
+  /// \name Accessors
   /// @{
 
   ExprKind getKind() const { return Kind; }
 
   /// @}
-  /// @name Utility Methods
+  /// \name Utility Methods
   /// @{
 
-  void print(raw_ostream &OS) const;
+  void print(raw_ostream &OS, const MCAsmInfo *MAI) const;
   void dump() const;
 
   /// @}
-  /// @name Expression Evaluation
+  /// \name Expression Evaluation
   /// @{
 
-  /// EvaluateAsAbsolute - Try to evaluate the expression to an absolute value.
+  /// \brief Try to evaluate the expression to an absolute value.
   ///
-  /// @param Res - The absolute value, if evaluation succeeds.
-  /// @param Layout - The assembler layout object to use for evaluating symbol
+  /// \param Res - The absolute value, if evaluation succeeds.
+  /// \param Layout - The assembler layout object to use for evaluating symbol
   /// values. If not given, then only non-symbolic expressions will be
   /// evaluated.
-  /// @result - True on success.
-  bool EvaluateAsAbsolute(int64_t &Res, const MCAsmLayout &Layout,
+  /// \return - True on success.
+  bool evaluateAsAbsolute(int64_t &Res, const MCAsmLayout &Layout,
                           const SectionAddrMap &Addrs) const;
-  bool EvaluateAsAbsolute(int64_t &Res) const;
-  bool EvaluateAsAbsolute(int64_t &Res, const MCAssembler &Asm) const;
-  bool EvaluateAsAbsolute(int64_t &Res, const MCAsmLayout &Layout) const;
+  bool evaluateAsAbsolute(int64_t &Res) const;
+  bool evaluateAsAbsolute(int64_t &Res, const MCAssembler &Asm) const;
+  bool evaluateAsAbsolute(int64_t &Res, const MCAsmLayout &Layout) const;
 
   bool evaluateKnownAbsolute(int64_t &Res, const MCAsmLayout &Layout) const;
 
-  /// EvaluateAsRelocatable - Try to evaluate the expression to a relocatable
-  /// value, i.e. an expression of the fixed form (a - b + constant).
+  /// \brief Try to evaluate the expression to a relocatable value, i.e. an
+  /// expression of the fixed form (a - b + constant).
   ///
-  /// @param Res - The relocatable value, if evaluation succeeds.
-  /// @param Layout - The assembler layout object to use for evaluating values.
-  /// @param Fixup - The Fixup object if available.
-  /// @result - True on success.
-  bool EvaluateAsRelocatable(MCValue &Res, const MCAsmLayout *Layout,
+  /// \param Res - The relocatable value, if evaluation succeeds.
+  /// \param Layout - The assembler layout object to use for evaluating values.
+  /// \param Fixup - The Fixup object if available.
+  /// \return - True on success.
+  bool evaluateAsRelocatable(MCValue &Res, const MCAsmLayout *Layout,
                              const MCFixup *Fixup) const;
 
   /// \brief Try to evaluate the expression to the form (a - b + constant) where
   /// neither a nor b are variables.
   ///
-  /// This is a more aggressive variant of EvaluateAsRelocatable. The intended
+  /// This is a more aggressive variant of evaluateAsRelocatable. The intended
   /// use is for when relocations are not available, like the .size directive.
   bool evaluateAsValue(MCValue &Res, const MCAsmLayout &Layout) const;
 
-  /// FindAssociatedSection - Find the "associated section" for this expression,
-  /// which is currently defined as the absolute section for constants, or
+  /// \brief Find the "associated section" for this expression, which is
+  /// currently defined as the absolute section for constants, or
   /// otherwise the section associated with the first defined symbol in the
   /// expression.
-  const MCSection *FindAssociatedSection() const;
+  MCSection *findAssociatedSection() const;
 
   /// @}
 };
 
 inline raw_ostream &operator<<(raw_ostream &OS, const MCExpr &E) {
-  E.print(OS);
+  E.print(OS, nullptr);
   return OS;
 }
 
-//// MCConstantExpr - Represent a constant integer expression.
+//// \brief  Represent a constant integer expression.
 class MCConstantExpr : public MCExpr {
   int64_t Value;
 
@@ -134,13 +133,13 @@ class MCConstantExpr : public MCExpr {
       : MCExpr(MCExpr::Constant), Value(Value) {}
 
 public:
-  /// @name Construction
+  /// \name Construction
   /// @{
 
-  static const MCConstantExpr *Create(int64_t Value, MCContext &Ctx);
+  static const MCConstantExpr *create(int64_t Value, MCContext &Ctx);
 
   /// @}
-  /// @name Accessors
+  /// \name Accessors
   /// @{
 
   int64_t getValue() const { return Value; }
@@ -152,15 +151,14 @@ class MCConstantExpr : public MCExpr {
   }
 };
 
-/// MCSymbolRefExpr - Represent a reference to a symbol from inside an
-/// expression.
+/// \brief  Represent a reference to a symbol from inside an expression.
 ///
 /// A symbol reference in an expression may be a use of a label, a use of an
 /// assembler variable (defined constant), or constitute an implicit definition
 /// of the symbol as external.
 class MCSymbolRefExpr : public MCExpr {
 public:
-  enum VariantKind {
+  enum VariantKind : uint16_t {
     VK_None,
     VK_Invalid,
 
@@ -278,12 +276,25 @@ class MCSymbolRefExpr : public MCExpr {
     VK_Mips_PCREL_HI16,
     VK_Mips_PCREL_LO16,
 
-    VK_COFF_IMGREL32 // symbol@imgrel (image-relative)
+    VK_COFF_IMGREL32, // symbol@imgrel (image-relative)
+
+    VK_Hexagon_PCREL,
+    VK_Hexagon_LO16,
+    VK_Hexagon_HI16,
+    VK_Hexagon_GPREL,
+    VK_Hexagon_GD_GOT,
+    VK_Hexagon_LD_GOT,
+    VK_Hexagon_GD_PLT,
+    VK_Hexagon_LD_PLT,
+    VK_Hexagon_IE,
+    VK_Hexagon_IE_GOT,
+    VK_TPREL,
+    VK_DTPREL
   };
 
 private:
   /// The symbol reference modifier.
-  const unsigned Kind : 16;
+  const VariantKind Kind;
 
   /// Specifies how the variant kind should be printed.
   const unsigned UseParensForSymbolVariant : 1;
@@ -298,32 +309,32 @@ class MCSymbolRefExpr : public MCExpr {
                            const MCAsmInfo *MAI);
 
 public:
-  /// @name Construction
+  /// \name Construction
   /// @{
 
-  static const MCSymbolRefExpr *Create(const MCSymbol *Symbol, MCContext &Ctx) {
-    return MCSymbolRefExpr::Create(Symbol, VK_None, Ctx);
+  static const MCSymbolRefExpr *create(const MCSymbol *Symbol, MCContext &Ctx) {
+    return MCSymbolRefExpr::create(Symbol, VK_None, Ctx);
   }
 
-  static const MCSymbolRefExpr *Create(const MCSymbol *Symbol, VariantKind Kind,
+  static const MCSymbolRefExpr *create(const MCSymbol *Symbol, VariantKind Kind,
                                        MCContext &Ctx);
-  static const MCSymbolRefExpr *Create(StringRef Name, VariantKind Kind,
+  static const MCSymbolRefExpr *create(StringRef Name, VariantKind Kind,
                                        MCContext &Ctx);
 
   /// @}
-  /// @name Accessors
+  /// \name Accessors
   /// @{
 
   const MCSymbol &getSymbol() const { return *Symbol; }
 
-  VariantKind getKind() const { return static_cast<VariantKind>(Kind); }
+  VariantKind getKind() const { return Kind; }
 
   void printVariantKind(raw_ostream &OS) const;
 
   bool hasSubsectionsViaSymbols() const { return HasSubsectionsViaSymbols; }
 
   /// @}
-  /// @name Static Utility Functions
+  /// \name Static Utility Functions
   /// @{
 
   static StringRef getVariantKindName(VariantKind Kind);
@@ -337,7 +348,7 @@ class MCSymbolRefExpr : public MCExpr {
   }
 };
 
-/// MCUnaryExpr - Unary assembler expressions.
+/// \brief Unary assembler expressions.
 class MCUnaryExpr : public MCExpr {
 public:
   enum Opcode {
@@ -355,32 +366,32 @@ class MCUnaryExpr : public MCExpr {
       : MCExpr(MCExpr::Unary), Op(Op), Expr(Expr) {}
 
 public:
-  /// @name Construction
+  /// \name Construction
   /// @{
 
-  static const MCUnaryExpr *Create(Opcode Op, const MCExpr *Expr,
+  static const MCUnaryExpr *create(Opcode Op, const MCExpr *Expr,
                                    MCContext &Ctx);
-  static const MCUnaryExpr *CreateLNot(const MCExpr *Expr, MCContext &Ctx) {
-    return Create(LNot, Expr, Ctx);
+  static const MCUnaryExpr *createLNot(const MCExpr *Expr, MCContext &Ctx) {
+    return create(LNot, Expr, Ctx);
   }
-  static const MCUnaryExpr *CreateMinus(const MCExpr *Expr, MCContext &Ctx) {
-    return Create(Minus, Expr, Ctx);
+  static const MCUnaryExpr *createMinus(const MCExpr *Expr, MCContext &Ctx) {
+    return create(Minus, Expr, Ctx);
   }
-  static const MCUnaryExpr *CreateNot(const MCExpr *Expr, MCContext &Ctx) {
-    return Create(Not, Expr, Ctx);
+  static const MCUnaryExpr *createNot(const MCExpr *Expr, MCContext &Ctx) {
+    return create(Not, Expr, Ctx);
   }
-  static const MCUnaryExpr *CreatePlus(const MCExpr *Expr, MCContext &Ctx) {
-    return Create(Plus, Expr, Ctx);
+  static const MCUnaryExpr *createPlus(const MCExpr *Expr, MCContext &Ctx) {
+    return create(Plus, Expr, Ctx);
   }
 
   /// @}
-  /// @name Accessors
+  /// \name Accessors
   /// @{
 
-  /// getOpcode - Get the kind of this unary expression.
+  /// \brief Get the kind of this unary expression.
   Opcode getOpcode() const { return Op; }
 
-  /// getSubExpr - Get the child of this unary expression.
+  /// \brief Get the child of this unary expression.
   const MCExpr *getSubExpr() const { return Expr; }
 
   /// @}
@@ -390,7 +401,7 @@ class MCUnaryExpr : public MCExpr {
   }
 };
 
-/// MCBinaryExpr - Binary assembler expressions.
+/// \brief Binary assembler expressions.
 class MCBinaryExpr : public MCExpr {
 public:
   enum Opcode {
@@ -413,7 +424,8 @@ class MCBinaryExpr : public MCExpr {
     NE,   ///< Inequality comparison.
     Or,   ///< Bitwise or.
     Shl,  ///< Shift left.
-    Shr,  ///< Shift right (arithmetic or logical, depending on target)
+    AShr, ///< Arithmetic shift right.
+    LShr, ///< Logical shift right.
     Sub,  ///< Subtraction.
     Xor   ///< Bitwise exclusive or.
   };
@@ -426,95 +438,99 @@ class MCBinaryExpr : public MCExpr {
       : MCExpr(MCExpr::Binary), Op(Op), LHS(LHS), RHS(RHS) {}
 
 public:
-  /// @name Construction
+  /// \name Construction
   /// @{
 
-  static const MCBinaryExpr *Create(Opcode Op, const MCExpr *LHS,
+  static const MCBinaryExpr *create(Opcode Op, const MCExpr *LHS,
                                     const MCExpr *RHS, MCContext &Ctx);
-  static const MCBinaryExpr *CreateAdd(const MCExpr *LHS, const MCExpr *RHS,
+  static const MCBinaryExpr *createAdd(const MCExpr *LHS, const MCExpr *RHS,
                                        MCContext &Ctx) {
-    return Create(Add, LHS, RHS, Ctx);
+    return create(Add, LHS, RHS, Ctx);
   }
-  static const MCBinaryExpr *CreateAnd(const MCExpr *LHS, const MCExpr *RHS,
+  static const MCBinaryExpr *createAnd(const MCExpr *LHS, const MCExpr *RHS,
                                        MCContext &Ctx) {
-    return Create(And, LHS, RHS, Ctx);
+    return create(And, LHS, RHS, Ctx);
   }
-  static const MCBinaryExpr *CreateDiv(const MCExpr *LHS, const MCExpr *RHS,
+  static const MCBinaryExpr *createDiv(const MCExpr *LHS, const MCExpr *RHS,
                                        MCContext &Ctx) {
-    return Create(Div, LHS, RHS, Ctx);
+    return create(Div, LHS, RHS, Ctx);
   }
-  static const MCBinaryExpr *CreateEQ(const MCExpr *LHS, const MCExpr *RHS,
+  static const MCBinaryExpr *createEQ(const MCExpr *LHS, const MCExpr *RHS,
                                       MCContext &Ctx) {
-    return Create(EQ, LHS, RHS, Ctx);
+    return create(EQ, LHS, RHS, Ctx);
   }
-  static const MCBinaryExpr *CreateGT(const MCExpr *LHS, const MCExpr *RHS,
+  static const MCBinaryExpr *createGT(const MCExpr *LHS, const MCExpr *RHS,
                                       MCContext &Ctx) {
-    return Create(GT, LHS, RHS, Ctx);
+    return create(GT, LHS, RHS, Ctx);
   }
-  static const MCBinaryExpr *CreateGTE(const MCExpr *LHS, const MCExpr *RHS,
+  static const MCBinaryExpr *createGTE(const MCExpr *LHS, const MCExpr *RHS,
                                        MCContext &Ctx) {
-    return Create(GTE, LHS, RHS, Ctx);
+    return create(GTE, LHS, RHS, Ctx);
   }
-  static const MCBinaryExpr *CreateLAnd(const MCExpr *LHS, const MCExpr *RHS,
+  static const MCBinaryExpr *createLAnd(const MCExpr *LHS, const MCExpr *RHS,
                                         MCContext &Ctx) {
-    return Create(LAnd, LHS, RHS, Ctx);
+    return create(LAnd, LHS, RHS, Ctx);
   }
-  static const MCBinaryExpr *CreateLOr(const MCExpr *LHS, const MCExpr *RHS,
+  static const MCBinaryExpr *createLOr(const MCExpr *LHS, const MCExpr *RHS,
                                        MCContext &Ctx) {
-    return Create(LOr, LHS, RHS, Ctx);
+    return create(LOr, LHS, RHS, Ctx);
   }
-  static const MCBinaryExpr *CreateLT(const MCExpr *LHS, const MCExpr *RHS,
+  static const MCBinaryExpr *createLT(const MCExpr *LHS, const MCExpr *RHS,
                                       MCContext &Ctx) {
-    return Create(LT, LHS, RHS, Ctx);
+    return create(LT, LHS, RHS, Ctx);
   }
-  static const MCBinaryExpr *CreateLTE(const MCExpr *LHS, const MCExpr *RHS,
+  static const MCBinaryExpr *createLTE(const MCExpr *LHS, const MCExpr *RHS,
                                        MCContext &Ctx) {
-    return Create(LTE, LHS, RHS, Ctx);
+    return create(LTE, LHS, RHS, Ctx);
   }
-  static const MCBinaryExpr *CreateMod(const MCExpr *LHS, const MCExpr *RHS,
+  static const MCBinaryExpr *createMod(const MCExpr *LHS, const MCExpr *RHS,
                                        MCContext &Ctx) {
-    return Create(Mod, LHS, RHS, Ctx);
+    return create(Mod, LHS, RHS, Ctx);
   }
-  static const MCBinaryExpr *CreateMul(const MCExpr *LHS, const MCExpr *RHS,
+  static const MCBinaryExpr *createMul(const MCExpr *LHS, const MCExpr *RHS,
                                        MCContext &Ctx) {
-    return Create(Mul, LHS, RHS, Ctx);
+    return create(Mul, LHS, RHS, Ctx);
   }
-  static const MCBinaryExpr *CreateNE(const MCExpr *LHS, const MCExpr *RHS,
+  static const MCBinaryExpr *createNE(const MCExpr *LHS, const MCExpr *RHS,
                                       MCContext &Ctx) {
-    return Create(NE, LHS, RHS, Ctx);
+    return create(NE, LHS, RHS, Ctx);
   }
-  static const MCBinaryExpr *CreateOr(const MCExpr *LHS, const MCExpr *RHS,
+  static const MCBinaryExpr *createOr(const MCExpr *LHS, const MCExpr *RHS,
                                       MCContext &Ctx) {
-    return Create(Or, LHS, RHS, Ctx);
+    return create(Or, LHS, RHS, Ctx);
+  }
+  static const MCBinaryExpr *createShl(const MCExpr *LHS, const MCExpr *RHS,
+                                       MCContext &Ctx) {
+    return create(Shl, LHS, RHS, Ctx);
   }
-  static const MCBinaryExpr *CreateShl(const MCExpr *LHS, const MCExpr *RHS,
+  static const MCBinaryExpr *createAShr(const MCExpr *LHS, const MCExpr *RHS,
                                        MCContext &Ctx) {
-    return Create(Shl, LHS, RHS, Ctx);
+    return create(AShr, LHS, RHS, Ctx);
   }
-  static const MCBinaryExpr *CreateShr(const MCExpr *LHS, const MCExpr *RHS,
+  static const MCBinaryExpr *createLShr(const MCExpr *LHS, const MCExpr *RHS,
                                        MCContext &Ctx) {
-    return Create(Shr, LHS, RHS, Ctx);
+    return create(LShr, LHS, RHS, Ctx);
   }
-  static const MCBinaryExpr *CreateSub(const MCExpr *LHS, const MCExpr *RHS,
+  static const MCBinaryExpr *createSub(const MCExpr *LHS, const MCExpr *RHS,
                                        MCContext &Ctx) {
-    return Create(Sub, LHS, RHS, Ctx);
+    return create(Sub, LHS, RHS, Ctx);
   }
-  static const MCBinaryExpr *CreateXor(const MCExpr *LHS, const MCExpr *RHS,
+  static const MCBinaryExpr *createXor(const MCExpr *LHS, const MCExpr *RHS,
                                        MCContext &Ctx) {
-    return Create(Xor, LHS, RHS, Ctx);
+    return create(Xor, LHS, RHS, Ctx);
   }
 
   /// @}
-  /// @name Accessors
+  /// \name Accessors
   /// @{
 
-  /// getOpcode - Get the kind of this binary expression.
+  /// \brief Get the kind of this binary expression.
   Opcode getOpcode() const { return Op; }
 
-  /// getLHS - Get the left-hand side expression of the binary operator.
+  /// \brief Get the left-hand side expression of the binary operator.
   const MCExpr *getLHS() const { return LHS; }
 
-  /// getRHS - Get the right-hand side expression of the binary operator.
+  /// \brief Get the right-hand side expression of the binary operator.
   const MCExpr *getRHS() const { return RHS; }
 
   /// @}
@@ -524,8 +540,8 @@ class MCBinaryExpr : public MCExpr {
   }
 };
 
-/// MCTargetExpr - This is an extension point for target-specific MCExpr
-/// subclasses to implement.
+/// \brief This is an extension point for target-specific MCExpr subclasses to
+/// implement.
 ///
 /// NOTE: All subclasses are required to have trivial destructors because
 /// MCExprs are bump pointer allocated and not destructed.
@@ -535,13 +551,12 @@ class MCTargetExpr : public MCExpr {
   MCTargetExpr() : MCExpr(Target) {}
   virtual ~MCTargetExpr() {}
 public:
-
-  virtual void PrintImpl(raw_ostream &OS) const = 0;
-  virtual bool EvaluateAsRelocatableImpl(MCValue &Res,
+  virtual void printImpl(raw_ostream &OS, const MCAsmInfo *MAI) const = 0;
+  virtual bool evaluateAsRelocatableImpl(MCValue &Res,
                                          const MCAsmLayout *Layout,
                                          const MCFixup *Fixup) const = 0;
   virtual void visitUsedExpr(MCStreamer& Streamer) const = 0;
-  virtual const MCSection *FindAssociatedSection() const = 0;
+  virtual MCSection *findAssociatedSection() const = 0;
 
   virtual void fixELFSymbolsInTLSFixups(MCAssembler &) const = 0;
 
diff --git a/include/llvm/MC/MCFixedLenDisassembler.h b/include/llvm/MC/MCFixedLenDisassembler.h
index ad99943df2c..ad34d9494bb 100644
--- a/include/llvm/MC/MCFixedLenDisassembler.h
+++ b/include/llvm/MC/MCFixedLenDisassembler.h
@@ -22,6 +22,8 @@ enum DecoderOps {
                         //                uleb128 Val, uint16_t NumToSkip)
   OPC_CheckPredicate,   // OPC_CheckPredicate(uleb128 PIdx, uint16_t NumToSkip)
   OPC_Decode,           // OPC_Decode(uleb128 Opcode, uleb128 DIdx)
+  OPC_TryDecode,        // OPC_TryDecode(uleb128 Opcode, uleb128 DIdx,
+                        //               uint16_t NumToSkip)
   OPC_SoftFail,         // OPC_SoftFail(uleb128 PMask, uleb128 NMask)
   OPC_Fail              // OPC_Fail()
 };
diff --git a/include/llvm/MC/MCFixup.h b/include/llvm/MC/MCFixup.h
index 98a1419a193..8ab477c401a 100644
--- a/include/llvm/MC/MCFixup.h
+++ b/include/llvm/MC/MCFixup.h
@@ -19,7 +19,7 @@
 namespace llvm {
 class MCExpr;
 
-/// MCFixupKind - Extensible enumeration to represent the type of a fixup.
+/// \brief Extensible enumeration to represent the type of a fixup.
 enum MCFixupKind {
   FK_Data_1 = 0, ///< A one-byte fixup.
   FK_Data_2,     ///< A two-byte fixup.
@@ -45,7 +45,7 @@ enum MCFixupKind {
   MaxTargetFixupKind = (1 << 8)
 };
 
-/// MCFixup - Encode information on a single operation to perform on a byte
+/// \brief Encode information on a single operation to perform on a byte
 /// sequence (e.g., an encoded instruction) which requires assemble- or run-
 /// time patching.
 ///
@@ -75,7 +75,7 @@ class MCFixup {
   /// The source location which gave rise to the fixup, if any.
   SMLoc Loc;
 public:
-  static MCFixup Create(uint32_t Offset, const MCExpr *Value,
+  static MCFixup create(uint32_t Offset, const MCExpr *Value,
                         MCFixupKind Kind, SMLoc Loc = SMLoc()) {
     assert(unsigned(Kind) < MaxTargetFixupKind && "Kind out of range!");
     MCFixup FI;
@@ -93,8 +93,8 @@ class MCFixup {
 
   const MCExpr *getValue() const { return Value; }
 
-  /// getKindForSize - Return the generic fixup kind for a value with the given
-  /// size. It is an error to pass an unsupported size.
+  /// \brief Return the generic fixup kind for a value with the given size. It
+  /// is an error to pass an unsupported size.
   static MCFixupKind getKindForSize(unsigned Size, bool isPCRel) {
     switch (Size) {
     default: llvm_unreachable("Invalid generic fixup size!");
diff --git a/include/llvm/MC/MCFixupKindInfo.h b/include/llvm/MC/MCFixupKindInfo.h
index 6979ad5807d..58183bd778e 100644
--- a/include/llvm/MC/MCFixupKindInfo.h
+++ b/include/llvm/MC/MCFixupKindInfo.h
@@ -12,7 +12,7 @@
 
 namespace llvm {
 
-/// MCFixupKindInfo - Target independent information on a fixup kind.
+/// \brief Target independent information on a fixup kind.
 struct MCFixupKindInfo {
   enum FixupKindFlags {
     /// Is this fixup kind PCrelative? This is used by the assembler backend to
diff --git a/include/llvm/MC/MCInst.h b/include/llvm/MC/MCInst.h
index 963b329cc07..4688b5f2b6e 100644
--- a/include/llvm/MC/MCInst.h
+++ b/include/llvm/MC/MCInst.h
@@ -28,16 +28,16 @@ class MCInstPrinter;
 class MCExpr;
 class MCInst;
 
-/// MCOperand - Instances of this class represent operands of the MCInst class.
+/// \brief Instances of this class represent operands of the MCInst class.
 /// This is a simple discriminated union.
 class MCOperand {
   enum MachineOperandType : unsigned char {
-    kInvalid,                 ///< Uninitialized.
-    kRegister,                ///< Register operand.
-    kImmediate,               ///< Immediate operand.
-    kFPImmediate,             ///< Floating-point immediate operand.
-    kExpr,                    ///< Relocatable immediate operand.
-    kInst                     ///< Sub-instruction operand.
+    kInvalid,     ///< Uninitialized.
+    kRegister,    ///< Register operand.
+    kImmediate,   ///< Immediate operand.
+    kFPImmediate, ///< Floating-point immediate operand.
+    kExpr,        ///< Relocatable immediate operand.
+    kInst         ///< Sub-instruction operand.
   };
   MachineOperandType Kind;
 
@@ -48,13 +48,9 @@ class MCOperand {
     const MCExpr *ExprVal;
     const MCInst *InstVal;
   };
-public:
 
-  // @LOCALMOD-START
-  // Initialize ImmVal instead of FPImmVal, thanks to
-  // http://gcc.gnu.org/bugzilla/show_bug.cgi?id=58416
-  MCOperand() : Kind(kInvalid), ImmVal(0) {}
-  // @LOCALMOD-END
+public:
+  MCOperand() : Kind(kInvalid), FPImmVal(0.0) {}
 
   bool isValid() const { return Kind != kInvalid; }
   bool isReg() const { return Kind == kRegister; }
@@ -63,13 +59,13 @@ class MCOperand {
   bool isExpr() const { return Kind == kExpr; }
   bool isInst() const { return Kind == kInst; }
 
-  /// getReg - Returns the register number.
+  /// \brief Returns the register number.
   unsigned getReg() const {
     assert(isReg() && "This is not a register operand!");
     return RegVal;
   }
 
-  /// setReg - Set the register number.
+  /// \brief Set the register number.
   void setReg(unsigned Reg) {
     assert(isReg() && "This is not a register operand!");
     RegVal = Reg;
@@ -112,31 +108,31 @@ class MCOperand {
     InstVal = Val;
   }
 
-  static MCOperand CreateReg(unsigned Reg) {
+  static MCOperand createReg(unsigned Reg) {
     MCOperand Op;
     Op.Kind = kRegister;
     Op.RegVal = Reg;
     return Op;
   }
-  static MCOperand CreateImm(int64_t Val) {
+  static MCOperand createImm(int64_t Val) {
     MCOperand Op;
     Op.Kind = kImmediate;
     Op.ImmVal = Val;
     return Op;
   }
-  static MCOperand CreateFPImm(double Val) {
+  static MCOperand createFPImm(double Val) {
     MCOperand Op;
     Op.Kind = kFPImmediate;
     Op.FPImmVal = Val;
     return Op;
   }
-  static MCOperand CreateExpr(const MCExpr *Val) {
+  static MCOperand createExpr(const MCExpr *Val) {
     MCOperand Op;
     Op.Kind = kExpr;
     Op.ExprVal = Val;
     return Op;
   }
-  static MCOperand CreateInst(const MCInst *Val) {
+  static MCOperand createInst(const MCInst *Val) {
     MCOperand Op;
     Op.Kind = kInst;
     Op.InstVal = Val;
@@ -149,12 +145,13 @@ class MCOperand {
 
 template <> struct isPodLike<MCOperand> { static const bool value = true; };
 
-/// MCInst - Instances of this class represent a single low-level machine
+/// \brief Instances of this class represent a single low-level machine
 /// instruction.
 class MCInst {
   unsigned Opcode;
   SMLoc Loc;
   SmallVector<MCOperand, 8> Operands;
+
 public:
   MCInst() : Opcode(0) {}
 
@@ -168,18 +165,16 @@ class MCInst {
   MCOperand &getOperand(unsigned i) { return Operands[i]; }
   unsigned getNumOperands() const { return Operands.size(); }
 
-  void addOperand(const MCOperand &Op) {
-    Operands.push_back(Op);
-  }
-
-  void clear() { Operands.clear(); }
-  size_t size() const { return Operands.size(); }
+  void addOperand(const MCOperand &Op) { Operands.push_back(Op); }
 
   typedef SmallVectorImpl<MCOperand>::iterator iterator;
   typedef SmallVectorImpl<MCOperand>::const_iterator const_iterator;
+  void clear() { Operands.clear(); }
+  void erase(iterator I) { Operands.erase(I); }
+  size_t size() const { return Operands.size(); }
   iterator begin() { return Operands.begin(); }
   const_iterator begin() const { return Operands.begin(); }
-  iterator end()   { return Operands.end(); }
+  iterator end() { return Operands.end(); }
   const_iterator end() const { return Operands.end(); }
   iterator insert(iterator I, const MCOperand &Op) {
     return Operands.insert(I, Op);
diff --git a/include/llvm/MC/MCInstBuilder.h b/include/llvm/MC/MCInstBuilder.h
index c5acb26eeca..30609bdb8b2 100644
--- a/include/llvm/MC/MCInstBuilder.h
+++ b/include/llvm/MC/MCInstBuilder.h
@@ -30,31 +30,37 @@ class MCInstBuilder {
 
   /// \brief Add a new register operand.
   MCInstBuilder &addReg(unsigned Reg) {
-    Inst.addOperand(MCOperand::CreateReg(Reg));
+    Inst.addOperand(MCOperand::createReg(Reg));
     return *this;
   }
 
   /// \brief Add a new integer immediate operand.
   MCInstBuilder &addImm(int64_t Val) {
-    Inst.addOperand(MCOperand::CreateImm(Val));
+    Inst.addOperand(MCOperand::createImm(Val));
     return *this;
   }
 
   /// \brief Add a new floating point immediate operand.
   MCInstBuilder &addFPImm(double Val) {
-    Inst.addOperand(MCOperand::CreateFPImm(Val));
+    Inst.addOperand(MCOperand::createFPImm(Val));
     return *this;
   }
 
   /// \brief Add a new MCExpr operand.
   MCInstBuilder &addExpr(const MCExpr *Val) {
-    Inst.addOperand(MCOperand::CreateExpr(Val));
+    Inst.addOperand(MCOperand::createExpr(Val));
     return *this;
   }
 
   /// \brief Add a new MCInst operand.
   MCInstBuilder &addInst(const MCInst *Val) {
-    Inst.addOperand(MCOperand::CreateInst(Val));
+    Inst.addOperand(MCOperand::createInst(Val));
+    return *this;
+  }
+
+  /// \brief Add an operand.
+  MCInstBuilder &addOperand(const MCOperand &Op) {
+    Inst.addOperand(Op);
     return *this;
   }
 
diff --git a/include/llvm/MC/MCInstPrinter.h b/include/llvm/MC/MCInstPrinter.h
index 4181d001add..0eafd02c51c 100644
--- a/include/llvm/MC/MCInstPrinter.h
+++ b/include/llvm/MC/MCInstPrinter.h
@@ -10,6 +10,7 @@
 #ifndef LLVM_MC_MCINSTPRINTER_H
 #define LLVM_MC_MCINSTPRINTER_H
 
+#include "llvm/ADT/ArrayRef.h"
 #include "llvm/Support/DataTypes.h"
 #include "llvm/Support/Format.h"
 
@@ -22,20 +23,23 @@ class MCRegisterInfo;
 class MCSubtargetInfo;
 class StringRef;
 
+/// Convert `Bytes' to a hex string and output to `OS'
+void dumpBytes(ArrayRef<uint8_t> Bytes, raw_ostream &OS);
+
 namespace HexStyle {
-    enum Style {
-        C,          ///< 0xff
-        Asm         ///< 0ffh
-    };
+enum Style {
+  C,  ///< 0xff
+  Asm ///< 0ffh
+};
 }
 
-/// MCInstPrinter - This is an instance of a target assembly language printer
-/// that converts an MCInst to valid target assembly syntax.
+/// \brief This is an instance of a target assembly language printer that
+/// converts an MCInst to valid target assembly syntax.
 class MCInstPrinter {
 protected:
-  /// CommentStream - a stream that comments can be emitted to if desired.
-  /// Each comment must end with a newline.  This will be null if verbose
-  /// assembly emission is disable.
+  /// \brief A stream that comments can be emitted to if desired.  Each comment
+  /// must end with a newline.  This will be null if verbose assembly emission
+  /// is disable.
   raw_ostream *CommentStream;
   const MCAsmInfo &MAI;
   const MCInstrInfo &MII;
@@ -52,28 +56,27 @@ class MCInstPrinter {
 
   /// Utility function for printing annotations.
   void printAnnotation(raw_ostream &OS, StringRef Annot);
+
 public:
   MCInstPrinter(const MCAsmInfo &mai, const MCInstrInfo &mii,
                 const MCRegisterInfo &mri)
-    : CommentStream(nullptr), MAI(mai), MII(mii), MRI(mri),
-      UseMarkup(0), PrintImmHex(0),
-      PrintHexStyle(HexStyle::C) {}
+      : CommentStream(nullptr), MAI(mai), MII(mii), MRI(mri), UseMarkup(0),
+        PrintImmHex(0), PrintHexStyle(HexStyle::C) {}
 
   virtual ~MCInstPrinter();
 
-  /// setCommentStream - Specify a stream to emit comments to.
+  /// \brief Specify a stream to emit comments to.
   void setCommentStream(raw_ostream &OS) { CommentStream = &OS; }
 
-  /// printInst - Print the specified MCInst to the specified raw_ostream.
-  ///
-  virtual void printInst(const MCInst *MI, raw_ostream &OS,
-                         StringRef Annot, const MCSubtargetInfo &STI) = 0;
+  /// \brief Print the specified MCInst to the specified raw_ostream.
+  virtual void printInst(const MCInst *MI, raw_ostream &OS, StringRef Annot,
+                         const MCSubtargetInfo &STI) = 0;
 
-  /// getOpcodeName - Return the name of the specified opcode enum (e.g.
-  /// "MOV32ri") or empty if we can't resolve it.
+  /// \brief Return the name of the specified opcode enum (e.g. "MOV32ri") or
+  /// empty if we can't resolve it.
   StringRef getOpcodeName(unsigned Opcode) const;
 
-  /// printRegName - Print the assembler register name.
+  /// \brief Print the assembler register name.
   virtual void printRegName(raw_ostream &OS, unsigned RegNo) const;
 
   bool getUseMarkup() const { return UseMarkup; }
@@ -86,8 +89,8 @@ class MCInstPrinter {
   bool getPrintImmHex() const { return PrintImmHex; }
   void setPrintImmHex(bool Value) { PrintImmHex = Value; }
 
-  HexStyle::Style getPrintHexStyleHex() const { return PrintHexStyle; }
-  void setPrintImmHex(HexStyle::Style Value) { PrintHexStyle = Value; }
+  HexStyle::Style getPrintHexStyle() const { return PrintHexStyle; }
+  void setPrintHexStyle(HexStyle::Style Value) { PrintHexStyle = Value; }
 
   /// Utility function to print immediates in decimal or hex.
   format_object<int64_t> formatImm(int64_t Value) const {
diff --git a/include/llvm/MC/MCInstrAnalysis.h b/include/llvm/MC/MCInstrAnalysis.h
index e921f768ac4..8f5159e9e1c 100644
--- a/include/llvm/MC/MCInstrAnalysis.h
+++ b/include/llvm/MC/MCInstrAnalysis.h
@@ -59,8 +59,8 @@ class MCInstrAnalysis {
     return Info->get(Inst.getOpcode()).isTerminator();
   }
 
-  /// evaluateBranch - Given a branch instruction try to get the address the
-  /// branch targets. Return true on success, and the address in Target.
+  /// \brief Given a branch instruction try to get the address the branch
+  /// targets. Return true on success, and the address in Target.
   virtual bool
   evaluateBranch(const MCInst &Inst, uint64_t Addr, uint64_t Size,
                  uint64_t &Target) const;
diff --git a/include/llvm/MC/MCInstrDesc.h b/include/llvm/MC/MCInstrDesc.h
index 360989305d3..6a582e82d00 100644
--- a/include/llvm/MC/MCInstrDesc.h
+++ b/include/llvm/MC/MCInstrDesc.h
@@ -15,145 +15,147 @@
 #ifndef LLVM_MC_MCINSTRDESC_H
 #define LLVM_MC_MCINSTRDESC_H
 
-#include "llvm/MC/MCInst.h"
-#include "llvm/MC/MCRegisterInfo.h"
-#include "llvm/MC/MCSubtargetInfo.h"
 #include "llvm/Support/DataTypes.h"
+#include <string>
 
 namespace llvm {
+  class MCInst;
+  class MCRegisterInfo;
+  class MCSubtargetInfo;
+  class FeatureBitset;
 
 //===----------------------------------------------------------------------===//
 // Machine Operand Flags and Description
 //===----------------------------------------------------------------------===//
 
 namespace MCOI {
-  // Operand constraints
-  enum OperandConstraint {
-    TIED_TO = 0,    // Must be allocated the same register as.
-    EARLY_CLOBBER   // Operand is an early clobber register operand
-  };
-
-  /// OperandFlags - These are flags set on operands, but should be considered
-  /// private, all access should go through the MCOperandInfo accessors.
-  /// See the accessors for a description of what these are.
-  enum OperandFlags {
-    LookupPtrRegClass = 0,
-    Predicate,
-    OptionalDef
-  };
-
-  /// Operand Type - Operands are tagged with one of the values of this enum.
-  enum OperandType {
-    OPERAND_UNKNOWN      = 0,
-    OPERAND_IMMEDIATE    = 1,
-    OPERAND_REGISTER     = 2,
-    OPERAND_MEMORY       = 3,
-    OPERAND_PCREL        = 4,
-    OPERAND_FIRST_TARGET = 5
-  };
+// Operand constraints
+enum OperandConstraint {
+  TIED_TO = 0,  // Must be allocated the same register as.
+  EARLY_CLOBBER // Operand is an early clobber register operand
+};
+
+/// \brief These are flags set on operands, but should be considered
+/// private, all access should go through the MCOperandInfo accessors.
+/// See the accessors for a description of what these are.
+enum OperandFlags { LookupPtrRegClass = 0, Predicate, OptionalDef };
+
+/// \brief Operands are tagged with one of the values of this enum.
+enum OperandType {
+  OPERAND_UNKNOWN = 0,
+  OPERAND_IMMEDIATE = 1,
+  OPERAND_REGISTER = 2,
+  OPERAND_MEMORY = 3,
+  OPERAND_PCREL = 4,
+  OPERAND_FIRST_TARGET = 5
+};
 }
 
-/// MCOperandInfo - This holds information about one operand of a machine
-/// instruction, indicating the register class for register operands, etc.
-///
+/// \brief This holds information about one operand of a machine instruction,
+/// indicating the register class for register operands, etc.
 class MCOperandInfo {
 public:
-  /// RegClass - This specifies the register class enumeration of the operand
+  /// \brief This specifies the register class enumeration of the operand
   /// if the operand is a register.  If isLookupPtrRegClass is set, then this is
   /// an index that is passed to TargetRegisterInfo::getPointerRegClass(x) to
   /// get a dynamic register class.
   int16_t RegClass;
 
-  /// Flags - These are flags from the MCOI::OperandFlags enum.
+  /// \brief These are flags from the MCOI::OperandFlags enum.
   uint8_t Flags;
 
-  /// OperandType - Information about the type of the operand.
+  /// \brief Information about the type of the operand.
   uint8_t OperandType;
-
-  /// Lower 16 bits are used to specify which constraints are set. The higher 16
-  /// bits are used to specify the value of constraints (4 bits each).
+  /// \brief The lower 16 bits are used to specify which constraints are set.
+  /// The higher 16 bits are used to specify the value of constraints (4 bits
+  /// each).
   uint32_t Constraints;
-  /// Currently no other information.
 
-  /// isLookupPtrRegClass - Set if this operand is a pointer value and it
-  /// requires a callback to look up its register class.
-  bool isLookupPtrRegClass() const {return Flags&(1 <<MCOI::LookupPtrRegClass);}
+  /// \brief Set if this operand is a pointer value and it requires a callback
+  /// to look up its register class.
+  bool isLookupPtrRegClass() const {
+    return Flags & (1 << MCOI::LookupPtrRegClass);
+  }
 
-  /// isPredicate - Set if this is one of the operands that made up of
-  /// the predicate operand that controls an isPredicable() instruction.
+  /// \brief Set if this is one of the operands that made up of the predicate
+  /// operand that controls an isPredicable() instruction.
   bool isPredicate() const { return Flags & (1 << MCOI::Predicate); }
 
-  /// isOptionalDef - Set if this operand is a optional def.
-  ///
+  /// \brief Set if this operand is a optional def.
   bool isOptionalDef() const { return Flags & (1 << MCOI::OptionalDef); }
 };
 
-
 //===----------------------------------------------------------------------===//
 // Machine Instruction Flags and Description
 //===----------------------------------------------------------------------===//
 
-/// MCInstrDesc flags - These should be considered private to the
-/// implementation of the MCInstrDesc class.  Clients should use the predicate
-/// methods on MCInstrDesc, not use these directly.  These all correspond to
-/// bitfields in the MCInstrDesc::Flags field.
 namespace MCID {
-  enum {
-    Variadic = 0,
-    HasOptionalDef,
-    Pseudo,
-    Return,
-    Call,
-    Barrier,
-    Terminator,
-    Branch,
-    IndirectBranch,
-    Compare,
-    MoveImm,
-    Bitcast,
-    Select,
-    DelaySlot,
-    FoldableAsLoad,
-    MayLoad,
-    MayStore,
-    Predicable,
-    NotDuplicable,
-    UnmodeledSideEffects,
-    Commutable,
-    ConvertibleTo3Addr,
-    UsesCustomInserter,
-    HasPostISelHook,
-    Rematerializable,
-    CheapAsAMove,
-    ExtraSrcRegAllocReq,
-    ExtraDefRegAllocReq,
-    RegSequence,
-    ExtractSubreg,
-    InsertSubreg
-  };
+/// \brief These should be considered private to the implementation of the
+/// MCInstrDesc class.  Clients should use the predicate methods on MCInstrDesc,
+/// not use these directly.  These all correspond to bitfields in the
+/// MCInstrDesc::Flags field.
+enum Flag {
+  Variadic = 0,
+  HasOptionalDef,
+  Pseudo,
+  Return,
+  Call,
+  Barrier,
+  Terminator,
+  Branch,
+  IndirectBranch,
+  Compare,
+  MoveImm,
+  Bitcast,
+  Select,
+  DelaySlot,
+  FoldableAsLoad,
+  MayLoad,
+  MayStore,
+  Predicable,
+  NotDuplicable,
+  UnmodeledSideEffects,
+  Commutable,
+  ConvertibleTo3Addr,
+  UsesCustomInserter,
+  HasPostISelHook,
+  Rematerializable,
+  CheapAsAMove,
+  ExtraSrcRegAllocReq,
+  ExtraDefRegAllocReq,
+  RegSequence,
+  ExtractSubreg,
+  InsertSubreg,
+  Convergent
+};
 }
 
-/// MCInstrDesc - Describe properties that are true of each instruction in the
-/// target description file.  This captures information about side effects,
-/// register use and many other things.  There is one instance of this struct
-/// for each target instruction class, and the MachineInstr class points to
-/// this struct directly to describe itself.
+/// \brief Describe properties that are true of each instruction in the target
+/// description file.  This captures information about side effects, register
+/// use and many other things.  There is one instance of this struct for each
+/// target instruction class, and the MachineInstr class points to this struct
+/// directly to describe itself.
 class MCInstrDesc {
 public:
-  unsigned short  Opcode;        // The opcode number
-  unsigned short  NumOperands;   // Num of args (may be more if variable_ops)
-  unsigned short  NumDefs;       // Num of args that are definitions
-  unsigned short  SchedClass;    // enum identifying instr sched class
-  unsigned short  Size;          // Number of bytes in encoding.
-  unsigned        Flags;         // Flags identifying machine instr class
-  uint64_t        TSFlags;       // Target Specific Flag values
-  const uint16_t *ImplicitUses;  // Registers implicitly read by this instr
-  const uint16_t *ImplicitDefs;  // Registers implicitly defined by this instr
-  const MCOperandInfo *OpInfo;   // 'NumOperands' entries about operands
-  uint64_t DeprecatedFeatureMask;// Feature bits that this is deprecated on, if any
+  unsigned short Opcode;        // The opcode number
+  unsigned short NumOperands;   // Num of args (may be more if variable_ops)
+  unsigned char NumDefs;        // Num of args that are definitions
+  unsigned char Size;           // Number of bytes in encoding.
+  unsigned short SchedClass;    // enum identifying instr sched class
+  uint64_t Flags;               // Flags identifying machine instr class
+  uint64_t TSFlags;             // Target Specific Flag values
+  const uint16_t *ImplicitUses; // Registers implicitly read by this instr
+  const uint16_t *ImplicitDefs; // Registers implicitly defined by this instr
+  const MCOperandInfo *OpInfo;  // 'NumOperands' entries about operands
+  // Subtarget feature that this is deprecated on, if any
+  // -1 implies this is not deprecated by any single feature. It may still be 
+  // deprecated due to a "complex" reason, below.
+  int64_t DeprecatedFeature;
+
   // A complex method to determine is a certain is deprecated or not, and return
   // the reason for deprecation.
-  bool (*ComplexDeprecationInfo)(MCInst &, MCSubtargetInfo &, std::string &);
+  bool (*ComplexDeprecationInfo)(MCInst &, const MCSubtargetInfo &,
+                                 std::string &);
 
   /// \brief Returns the value of the specific constraint if
   /// it is set. Returns -1 if it is not set.
@@ -169,39 +171,24 @@ class MCInstrDesc {
 
   /// \brief Returns true if a certain instruction is deprecated and if so
   /// returns the reason in \p Info.
-  bool getDeprecatedInfo(MCInst &MI, MCSubtargetInfo &STI,
-                         std::string &Info) const {
-    if (ComplexDeprecationInfo)
-      return ComplexDeprecationInfo(MI, STI, Info);
-    if ((DeprecatedFeatureMask & STI.getFeatureBits()) != 0) {
-      // FIXME: it would be nice to include the subtarget feature here.
-      Info = "deprecated";
-      return true;
-    }
-    return false;
-  }
+  bool getDeprecatedInfo(MCInst &MI, const MCSubtargetInfo &STI,
+                         std::string &Info) const;
 
   /// \brief Return the opcode number for this descriptor.
-  unsigned getOpcode() const {
-    return Opcode;
-  }
+  unsigned getOpcode() const { return Opcode; }
 
   /// \brief Return the number of declared MachineOperands for this
   /// MachineInstruction.  Note that variadic (isVariadic() returns true)
   /// instructions may have additional operands at the end of the list, and note
   /// that the machine instruction may include implicit register def/uses as
   /// well.
-  unsigned getNumOperands() const {
-    return NumOperands;
-  }
+  unsigned getNumOperands() const { return NumOperands; }
 
   /// \brief Return the number of MachineOperands that are register
   /// definitions.  Register definitions always occur at the start of the
   /// machine operand list.  This is the number of "outs" in the .td file,
   /// and does not include implicit defs.
-  unsigned getNumDefs() const {
-    return NumDefs;
-  }
+  unsigned getNumDefs() const { return NumDefs; }
 
   /// \brief Return flags of this instruction.
   unsigned getFlags() const { return Flags; }
@@ -210,39 +197,26 @@ class MCInstrDesc {
   /// operands.  In this case, the variable operands will be after the normal
   /// operands but before the implicit definitions and uses (if any are
   /// present).
-  bool isVariadic() const {
-    return Flags & (1 << MCID::Variadic);
-  }
+  bool isVariadic() const { return Flags & (1 << MCID::Variadic); }
 
   /// \brief Set if this instruction has an optional definition, e.g.
   /// ARM instructions which can set condition code if 's' bit is set.
-  bool hasOptionalDef() const {
-    return Flags & (1 << MCID::HasOptionalDef);
-  }
+  bool hasOptionalDef() const { return Flags & (1 << MCID::HasOptionalDef); }
 
   /// \brief Return true if this is a pseudo instruction that doesn't
   /// correspond to a real machine instruction.
-  ///
-  bool isPseudo() const {
-    return Flags & (1 << MCID::Pseudo);
-  }
+  bool isPseudo() const { return Flags & (1 << MCID::Pseudo); }
 
   /// \brief Return true if the instruction is a return.
-  bool isReturn() const {
-    return Flags & (1 << MCID::Return);
-  }
+  bool isReturn() const { return Flags & (1 << MCID::Return); }
 
   /// \brief  Return true if the instruction is a call.
-  bool isCall() const {
-    return Flags & (1 << MCID::Call);
-  }
+  bool isCall() const { return Flags & (1 << MCID::Call); }
 
   /// \brief Returns true if the specified instruction stops control flow
   /// from executing the instruction immediately following it.  Examples include
   /// unconditional branches and return instructions.
-  bool isBarrier() const {
-    return Flags & (1 << MCID::Barrier);
-  }
+  bool isBarrier() const { return Flags & (1 << MCID::Barrier); }
 
   /// \brief Returns true if this instruction part of the terminator for
   /// a basic block.  Typically this is things like return and branch
@@ -250,23 +224,17 @@ class MCInstrDesc {
   ///
   /// Various passes use this to insert code into the bottom of a basic block,
   /// but before control flow occurs.
-  bool isTerminator() const {
-    return Flags & (1 << MCID::Terminator);
-  }
+  bool isTerminator() const { return Flags & (1 << MCID::Terminator); }
 
   /// \brief Returns true if this is a conditional, unconditional, or
   /// indirect branch.  Predicates below can be used to discriminate between
   /// these cases, and the TargetInstrInfo::AnalyzeBranch method can be used to
   /// get more information.
-  bool isBranch() const {
-    return Flags & (1 << MCID::Branch);
-  }
+  bool isBranch() const { return Flags & (1 << MCID::Branch); }
 
   /// \brief Return true if this is an indirect branch, such as a
   /// branch through a register.
-  bool isIndirectBranch() const {
-    return Flags & (1 << MCID::IndirectBranch);
-  }
+  bool isIndirectBranch() const { return Flags & (1 << MCID::IndirectBranch); }
 
   /// \brief Return true if this is a branch which may fall
   /// through to the next instruction or may transfer control flow to some other
@@ -287,79 +255,44 @@ class MCInstrDesc {
   /// \brief Return true if this is a branch or an instruction which directly
   /// writes to the program counter. Considered 'may' affect rather than
   /// 'does' affect as things like predication are not taken into account.
-  bool mayAffectControlFlow(const MCInst &MI, const MCRegisterInfo &RI) const {
-    if (isBranch() || isCall() || isReturn() || isIndirectBranch())
-      return true;
-    unsigned PC = RI.getProgramCounter();
-    if (PC == 0)
-      return false;
-    if (hasDefOfPhysReg(MI, PC, RI))
-      return true;
-    // A variadic instruction may define PC in the variable operand list.
-    // There's currently no indication of which entries in a variable
-    // list are defs and which are uses. While that's the case, this function
-    // needs to assume they're defs in order to be conservatively correct.
-    for (int i = NumOperands, e = MI.getNumOperands(); i != e; ++i) {
-      if (MI.getOperand(i).isReg() &&
-          RI.isSubRegisterEq(PC, MI.getOperand(i).getReg()))
-        return true;
-    }
-    return false;
-  }
+  bool mayAffectControlFlow(const MCInst &MI, const MCRegisterInfo &RI) const;
 
   /// \brief Return true if this instruction has a predicate operand
   /// that controls execution. It may be set to 'always', or may be set to other
   /// values. There are various methods in TargetInstrInfo that can be used to
   /// control and modify the predicate in this instruction.
-  bool isPredicable() const {
-    return Flags & (1 << MCID::Predicable);
-  }
+  bool isPredicable() const { return Flags & (1 << MCID::Predicable); }
 
   /// \brief Return true if this instruction is a comparison.
-  bool isCompare() const {
-    return Flags & (1 << MCID::Compare);
-  }
+  bool isCompare() const { return Flags & (1 << MCID::Compare); }
 
   /// \brief Return true if this instruction is a move immediate
   /// (including conditional moves) instruction.
-  bool isMoveImmediate() const {
-    return Flags & (1 << MCID::MoveImm);
-  }
+  bool isMoveImmediate() const { return Flags & (1 << MCID::MoveImm); }
 
   /// \brief Return true if this instruction is a bitcast instruction.
-  bool isBitcast() const {
-    return Flags & (1 << MCID::Bitcast);
-  }
+  bool isBitcast() const { return Flags & (1 << MCID::Bitcast); }
 
   /// \brief Return true if this is a select instruction.
-  bool isSelect() const {
-    return Flags & (1 << MCID::Select);
-  }
+  bool isSelect() const { return Flags & (1 << MCID::Select); }
 
   /// \brief Return true if this instruction cannot be safely
   /// duplicated.  For example, if the instruction has a unique labels attached
   /// to it, duplicating it would cause multiple definition errors.
-  bool isNotDuplicable() const {
-    return Flags & (1 << MCID::NotDuplicable);
-  }
+  bool isNotDuplicable() const { return Flags & (1 << MCID::NotDuplicable); }
 
-  /// hasDelaySlot - Returns true if the specified instruction has a delay slot
-  /// which must be filled by the code generator.
-  bool hasDelaySlot() const {
-    return Flags & (1 << MCID::DelaySlot);
-  }
+  /// \brief Returns true if the specified instruction has a delay slot which
+  /// must be filled by the code generator.
+  bool hasDelaySlot() const { return Flags & (1 << MCID::DelaySlot); }
 
-  /// canFoldAsLoad - Return true for instructions that can be folded as
-  /// memory operands in other instructions. The most common use for this
-  /// is instructions that are simple loads from memory that don't modify
-  /// the loaded value in any way, but it can also be used for instructions
-  /// that can be expressed as constant-pool loads, such as V_SETALLONES
-  /// on x86, to allow them to be folded when it is beneficial.
-  /// This should only be set on instructions that return a value in their
-  /// only virtual register definition.
-  bool canFoldAsLoad() const {
-    return Flags & (1 << MCID::FoldableAsLoad);
-  }
+  /// \brief Return true for instructions that can be folded as memory operands
+  /// in other instructions. The most common use for this is instructions that
+  /// are simple loads from memory that don't modify the loaded value in any
+  /// way, but it can also be used for instructions that can be expressed as
+  /// constant-pool loads, such as V_SETALLONES on x86, to allow them to be
+  /// folded when it is beneficial.  This should only be set on instructions
+  /// that return a value in their only virtual register definition.
+  bool canFoldAsLoad() const { return Flags & (1 << MCID::FoldableAsLoad); }
 
   /// \brief Return true if this instruction behaves
   /// the same way as the generic REG_SEQUENCE instructions.
@@ -398,9 +331,14 @@ class MCInstrDesc {
   /// Note that for the optimizers to be able to take advantage of
   /// this property, TargetInstrInfo::getInsertSubregLikeInputs has to be
   /// override accordingly.
-  bool isInsertSubregLike() const {
-    return Flags & (1 << MCID::InsertSubreg);
-  }
+  bool isInsertSubregLike() const { return Flags & (1 << MCID::InsertSubreg); }
+
+
+  /// \brief Return true if this instruction is convergent.
+  ///
+  /// Convergent instructions may only be moved to locations that are
+  /// control-equivalent to their original positions.
+  bool isConvergent() const { return Flags & (1 << MCID::Convergent); }
 
   //===--------------------------------------------------------------------===//
   // Side Effect Analysis
@@ -409,20 +347,15 @@ class MCInstrDesc {
   /// \brief Return true if this instruction could possibly read memory.
   /// Instructions with this flag set are not necessarily simple load
   /// instructions, they may load a value and modify it, for example.
-  bool mayLoad() const {
-    return Flags & (1 << MCID::MayLoad);
-  }
-
+  bool mayLoad() const { return Flags & (1 << MCID::MayLoad); }
 
   /// \brief Return true if this instruction could possibly modify memory.
   /// Instructions with this flag set are not necessarily simple store
   /// instructions, they may store a modified value based on their operands, or
   /// may not actually modify anything, for example.
-  bool mayStore() const {
-    return Flags & (1 << MCID::MayStore);
-  }
+  bool mayStore() const { return Flags & (1 << MCID::MayStore); }
 
-  /// hasUnmodeledSideEffects - Return true if this instruction has side
+  /// \brief Return true if this instruction has side
   /// effects that are not modeled by other flags.  This does not return true
   /// for instructions whose effects are captured by:
   ///
@@ -434,7 +367,6 @@ class MCInstrDesc {
   /// Examples of side effects would be modifying 'invisible' machine state like
   /// a control register, flushing a cache, modifying a register invisible to
   /// LLVM, etc.
-  ///
   bool hasUnmodeledSideEffects() const {
     return Flags & (1 << MCID::UnmodeledSideEffects);
   }
@@ -443,9 +375,9 @@ class MCInstrDesc {
   // Flags that indicate whether an instruction can be modified by a method.
   //===--------------------------------------------------------------------===//
 
-  /// isCommutable - Return true if this may be a 2- or 3-address
-  /// instruction (of the form "X = op Y, Z, ..."), which produces the same
-  /// result if Y and Z are exchanged.  If this flag is set, then the
+  /// \brief Return true if this may be a 2- or 3-address instruction (of the
+  /// form "X = op Y, Z, ..."), which produces the same result if Y and Z are
+  /// exchanged.  If this flag is set, then the
   /// TargetInstrInfo::commuteInstruction method may be used to hack on the
   /// instruction.
   ///
@@ -453,18 +385,16 @@ class MCInstrDesc {
   /// sometimes.  In these cases, the call to commuteInstruction will fail.
   /// Also note that some instructions require non-trivial modification to
   /// commute them.
-  bool isCommutable() const {
-    return Flags & (1 << MCID::Commutable);
-  }
-
-  /// isConvertibleTo3Addr - Return true if this is a 2-address instruction
-  /// which can be changed into a 3-address instruction if needed.  Doing this
-  /// transformation can be profitable in the register allocator, because it
-  /// means that the instruction can use a 2-address form if possible, but
-  /// degrade into a less efficient form if the source and dest register cannot
-  /// be assigned to the same register.  For example, this allows the x86
-  /// backend to turn a "shl reg, 3" instruction into an LEA instruction, which
-  /// is the same speed as the shift but has bigger code size.
+  bool isCommutable() const { return Flags & (1 << MCID::Commutable); }
+
+  /// \brief Return true if this is a 2-address instruction which can be changed
+  /// into a 3-address instruction if needed.  Doing this transformation can be
+  /// profitable in the register allocator, because it means that the
+  /// instruction can use a 2-address form if possible, but degrade into a less
+  /// efficient form if the source and dest register cannot be assigned to the
+  /// same register.  For example, this allows the x86 backend to turn a "shl
+  /// reg, 3" instruction into an LEA instruction, which is the same speed as
+  /// the shift but has bigger code size.
   ///
   /// If this returns true, then the target must implement the
   /// TargetInstrInfo::convertToThreeAddress method for this instruction, which
@@ -475,11 +405,11 @@ class MCInstrDesc {
     return Flags & (1 << MCID::ConvertibleTo3Addr);
   }
 
-  /// usesCustomInsertionHook - Return true if this instruction requires
-  /// custom insertion support when the DAG scheduler is inserting it into a
-  /// machine basic block.  If this is true for the instruction, it basically
-  /// means that it is a pseudo instruction used at SelectionDAG time that is
-  /// expanded out into magic code by the target when MachineInstrs are formed.
+  /// \brief Return true if this instruction requires custom insertion support
+  /// when the DAG scheduler is inserting it into a machine basic block.  If
+  /// this is true for the instruction, it basically means that it is a pseudo
+  /// instruction used at SelectionDAG time that is expanded out into magic code
+  /// by the target when MachineInstrs are formed.
   ///
   /// If this is true, the TargetLoweringInfo::InsertAtEndOfBasicBlock method
   /// is used to insert this into the MachineBasicBlock.
@@ -487,17 +417,14 @@ class MCInstrDesc {
     return Flags & (1 << MCID::UsesCustomInserter);
   }
 
-  /// hasPostISelHook - Return true if this instruction requires *adjustment*
-  /// after instruction selection by calling a target hook. For example, this
-  /// can be used to fill in ARM 's' optional operand depending on whether
-  /// the conditional flag register is used.
-  bool hasPostISelHook() const {
-    return Flags & (1 << MCID::HasPostISelHook);
-  }
+  /// \brief Return true if this instruction requires *adjustment* after
+  /// instruction selection by calling a target hook. For example, this can be
+  /// used to fill in ARM 's' optional operand depending on whether the
+  /// conditional flag register is used.
+  bool hasPostISelHook() const { return Flags & (1 << MCID::HasPostISelHook); }
 
-  /// isRematerializable - Returns true if this instruction is a candidate for
-  /// remat. This flag is only used in TargetInstrInfo method
-  /// isTriviallyRematerializable.
+  /// \brief Returns true if this instruction is a candidate for remat. This
+  /// flag is only used in TargetInstrInfo method isTriviallyRematerializable.
   ///
   /// If this flag is set, the isReallyTriviallyReMaterializable()
   /// or isReallyTriviallyReMaterializableGeneric methods are called to verify
@@ -506,80 +433,76 @@ class MCInstrDesc {
     return Flags & (1 << MCID::Rematerializable);
   }
 
-  /// isAsCheapAsAMove - Returns true if this instruction has the same cost (or
-  /// less) than a move instruction. This is useful during certain types of
-  /// optimizations (e.g., remat during two-address conversion or machine licm)
-  /// where we would like to remat or hoist the instruction, but not if it costs
-  /// more than moving the instruction into the appropriate register. Note, we
-  /// are not marking copies from and to the same register class with this flag.
+  /// \brief Returns true if this instruction has the same cost (or less) than a
+  /// move instruction. This is useful during certain types of optimizations
+  /// (e.g., remat during two-address conversion or machine licm) where we would
+  /// like to remat or hoist the instruction, but not if it costs more than
+  /// moving the instruction into the appropriate register. Note, we are not
+  /// marking copies from and to the same register class with this flag.
   ///
   /// This method could be called by interface TargetInstrInfo::isAsCheapAsAMove
   /// for different subtargets.
-  bool isAsCheapAsAMove() const {
-    return Flags & (1 << MCID::CheapAsAMove);
-  }
-
-  /// hasExtraSrcRegAllocReq - Returns true if this instruction source operands
-  /// have special register allocation requirements that are not captured by the
-  /// operand register classes. e.g. ARM::STRD's two source registers must be an
-  /// even / odd pair, ARM::STM registers have to be in ascending order.
-  /// Post-register allocation passes should not attempt to change allocations
-  /// for sources of instructions with this flag.
+  bool isAsCheapAsAMove() const { return Flags & (1 << MCID::CheapAsAMove); }
+
+  /// \brief Returns true if this instruction source operands have special
+  /// register allocation requirements that are not captured by the operand
+  /// register classes. e.g. ARM::STRD's two source registers must be an even /
+  /// odd pair, ARM::STM registers have to be in ascending order.  Post-register
+  /// allocation passes should not attempt to change allocations for sources of
+  /// instructions with this flag.
   bool hasExtraSrcRegAllocReq() const {
     return Flags & (1 << MCID::ExtraSrcRegAllocReq);
   }
 
-  /// hasExtraDefRegAllocReq - Returns true if this instruction def operands
-  /// have special register allocation requirements that are not captured by the
-  /// operand register classes. e.g. ARM::LDRD's two def registers must be an
-  /// even / odd pair, ARM::LDM registers have to be in ascending order.
-  /// Post-register allocation passes should not attempt to change allocations
-  /// for definitions of instructions with this flag.
+  /// \brief Returns true if this instruction def operands have special register
+  /// allocation requirements that are not captured by the operand register
+  /// classes. e.g. ARM::LDRD's two def registers must be an even / odd pair,
+  /// ARM::LDM registers have to be in ascending order.  Post-register
+  /// allocation passes should not attempt to change allocations for definitions
+  /// of instructions with this flag.
   bool hasExtraDefRegAllocReq() const {
     return Flags & (1 << MCID::ExtraDefRegAllocReq);
   }
 
-
-  /// getImplicitUses - Return a list of registers that are potentially
-  /// read by any instance of this machine instruction.  For example, on X86,
-  /// the "adc" instruction adds two register operands and adds the carry bit in
-  /// from the flags register.  In this case, the instruction is marked as
-  /// implicitly reading the flags.  Likewise, the variable shift instruction on
-  /// X86 is marked as implicitly reading the 'CL' register, which it always
-  /// does.
+  /// \brief Return a list of registers that are potentially read by any
+  /// instance of this machine instruction.  For example, on X86, the "adc"
+  /// instruction adds two register operands and adds the carry bit in from the
+  /// flags register.  In this case, the instruction is marked as implicitly
+  /// reading the flags.  Likewise, the variable shift instruction on X86 is
+  /// marked as implicitly reading the 'CL' register, which it always does.
   ///
   /// This method returns null if the instruction has no implicit uses.
-  const uint16_t *getImplicitUses() const {
-    return ImplicitUses;
-  }
+  const uint16_t *getImplicitUses() const { return ImplicitUses; }
 
   /// \brief Return the number of implicit uses this instruction has.
   unsigned getNumImplicitUses() const {
-    if (!ImplicitUses) return 0;
+    if (!ImplicitUses)
+      return 0;
     unsigned i = 0;
-    for (; ImplicitUses[i]; ++i) /*empty*/;
+    for (; ImplicitUses[i]; ++i) /*empty*/
+      ;
     return i;
   }
 
-  /// getImplicitDefs - Return a list of registers that are potentially
-  /// written by any instance of this machine instruction.  For example, on X86,
-  /// many instructions implicitly set the flags register.  In this case, they
-  /// are marked as setting the FLAGS.  Likewise, many instructions always
-  /// deposit their result in a physical register.  For example, the X86 divide
+  /// \brief Return a list of registers that are potentially written by any
+  /// instance of this machine instruction.  For example, on X86, many
+  /// instructions implicitly set the flags register.  In this case, they are
+  /// marked as setting the FLAGS.  Likewise, many instructions always deposit
+  /// their result in a physical register.  For example, the X86 divide
   /// instruction always deposits the quotient and remainder in the EAX/EDX
   /// registers.  For that instruction, this will return a list containing the
   /// EAX/EDX/EFLAGS registers.
   ///
   /// This method returns null if the instruction has no implicit defs.
-  const uint16_t *getImplicitDefs() const {
-    return ImplicitDefs;
-  }
+  const uint16_t *getImplicitDefs() const { return ImplicitDefs; }
 
   /// \brief Return the number of implicit defs this instruct has.
   unsigned getNumImplicitDefs() const {
-    if (!ImplicitDefs) return 0;
+    if (!ImplicitDefs)
+      return 0;
     unsigned i = 0;
-    for (; ImplicitDefs[i]; ++i) /*empty*/;
+    for (; ImplicitDefs[i]; ++i) /*empty*/
+      ;
     return i;
   }
 
@@ -588,45 +511,25 @@ class MCInstrDesc {
   bool hasImplicitUseOfPhysReg(unsigned Reg) const {
     if (const uint16_t *ImpUses = ImplicitUses)
       for (; *ImpUses; ++ImpUses)
-        if (*ImpUses == Reg) return true;
+        if (*ImpUses == Reg)
+          return true;
     return false;
   }
 
   /// \brief Return true if this instruction implicitly
   /// defines the specified physical register.
   bool hasImplicitDefOfPhysReg(unsigned Reg,
-                               const MCRegisterInfo *MRI = nullptr) const {
-    if (const uint16_t *ImpDefs = ImplicitDefs)
-      for (; *ImpDefs; ++ImpDefs)
-        if (*ImpDefs == Reg || (MRI && MRI->isSubRegister(Reg, *ImpDefs)))
-            return true;
-    return false;
-  }
-
-  /// \brief Return true if this instruction defines the specified physical
-  /// register, either explicitly or implicitly.
-  bool hasDefOfPhysReg(const MCInst &MI, unsigned Reg,
-                       const MCRegisterInfo &RI) const {
-    for (int i = 0, e = NumDefs; i != e; ++i)
-      if (MI.getOperand(i).isReg() &&
-          RI.isSubRegisterEq(Reg, MI.getOperand(i).getReg()))
-        return true;
-    return hasImplicitDefOfPhysReg(Reg, &RI);
-  }
+                               const MCRegisterInfo *MRI = nullptr) const;
 
   /// \brief Return the scheduling class for this instruction.  The
   /// scheduling class is an index into the InstrItineraryData table.  This
   /// returns zero if there is no known scheduling information for the
   /// instruction.
-  unsigned getSchedClass() const {
-    return SchedClass;
-  }
+  unsigned getSchedClass() const { return SchedClass; }
 
   /// \brief Return the number of bytes in the encoding of this instruction,
   /// or zero if the encoding size cannot be known from the opcode.
-  unsigned getSize() const {
-    return Size;
-  }
+  unsigned getSize() const { return Size; }
 
   /// \brief Find the index of the first operand in the
   /// operand list that is used to represent the predicate. It returns -1 if
@@ -639,6 +542,13 @@ class MCInstrDesc {
     }
     return -1;
   }
+
+private:
+
+  /// \brief Return true if this instruction defines the specified physical
+  /// register, either explicitly or implicitly.
+  bool hasDefOfPhysReg(const MCInst &MI, unsigned Reg,
+                       const MCRegisterInfo &RI) const;
 };
 
 } // end namespace llvm
diff --git a/include/llvm/MC/MCInstrInfo.h b/include/llvm/MC/MCInstrInfo.h
index 1d3a36ca7c7..70c86587b08 100644
--- a/include/llvm/MC/MCInstrInfo.h
+++ b/include/llvm/MC/MCInstrInfo.h
@@ -20,9 +20,7 @@
 namespace llvm {
 
 //---------------------------------------------------------------------------
-///
-/// MCInstrInfo - Interface to description of machine instruction set
-///
+/// \brief Interface to description of machine instruction set.
 class MCInstrInfo {
   const MCInstrDesc *Desc;          // Raw array to allow static init'n
   const unsigned *InstrNameIndices; // Array for name indices in InstrNameData
@@ -30,8 +28,8 @@ class MCInstrInfo {
   unsigned NumOpcodes;              // Number of entries in the desc array
 
 public:
-  /// InitMCInstrInfo - Initialize MCInstrInfo, called by TableGen
-  /// auto-generated routines. *DO NOT USE*.
+  /// \brief Initialize MCInstrInfo, called by TableGen auto-generated routines.
+  /// *DO NOT USE*.
   void InitMCInstrInfo(const MCInstrDesc *D, const unsigned *NI, const char *ND,
                        unsigned NO) {
     Desc = D;
@@ -42,15 +40,14 @@ class MCInstrInfo {
 
   unsigned getNumOpcodes() const { return NumOpcodes; }
 
-  /// get - Return the machine instruction descriptor that corresponds to the
+  /// \brief Return the machine instruction descriptor that corresponds to the
   /// specified instruction opcode.
-  ///
   const MCInstrDesc &get(unsigned Opcode) const {
     assert(Opcode < NumOpcodes && "Invalid opcode!");
     return Desc[Opcode];
   }
 
-  /// getName - Returns the name for the instructions with the given opcode.
+  /// \brief Returns the name for the instructions with the given opcode.
   const char *getName(unsigned Opcode) const {
     assert(Opcode < NumOpcodes && "Invalid opcode!");
     return &InstrNameData[InstrNameIndices[Opcode]];
diff --git a/include/llvm/MC/MCInstrItineraries.h b/include/llvm/MC/MCInstrItineraries.h
index 94d599f672a..161705de7c4 100644
--- a/include/llvm/MC/MCInstrItineraries.h
+++ b/include/llvm/MC/MCInstrItineraries.h
@@ -67,12 +67,12 @@ struct InstrStage {
   int NextCycles_;   ///< Number of machine cycles to next stage
   ReservationKinds Kind_; ///< Kind of the FU reservation
 
-  /// Returns the number of cycles the stage is occupied.
+  /// \brief Returns the number of cycles the stage is occupied.
   unsigned getCycles() const {
     return Cycles_;
   }
 
-  /// Returns the choice of FUs.
+  /// \brief Returns the choice of FUs.
   unsigned getUnits() const {
     return Units_;
   }
@@ -81,8 +81,8 @@ struct InstrStage {
     return Kind_;
   }
 
-  /// Returns the number of cycles from the start of
-  /// this stage to the start of the next stage in the itinerary
+  /// \brief Returns the number of cycles from the start of this stage to the
+  /// start of the next stage in the itinerary
   unsigned getNextCycles() const {
     return (NextCycles_ >= 0) ? (unsigned)NextCycles_ : Cycles_;
   }
@@ -115,7 +115,6 @@ class InstrItineraryData {
   const InstrItinerary *Itineraries;    ///< Array of itineraries selected
 
   /// Ctors.
-  ///
   InstrItineraryData() : SchedModel(MCSchedModel::GetDefaultSchedModel()),
                          Stages(nullptr), OperandCycles(nullptr),
                          Forwardings(nullptr), Itineraries(nullptr) {}
@@ -125,30 +124,30 @@ class InstrItineraryData {
     : SchedModel(SM), Stages(S), OperandCycles(OS), Forwardings(F),
       Itineraries(SchedModel.InstrItineraries) {}
 
-  /// Returns true if there are no itineraries.
+  /// \brief Returns true if there are no itineraries.
   bool isEmpty() const { return Itineraries == nullptr; }
 
-  /// Returns true if the index is for the end marker itinerary.
+  /// \brief Returns true if the index is for the end marker itinerary.
   bool isEndMarker(unsigned ItinClassIndx) const {
     return ((Itineraries[ItinClassIndx].FirstStage == ~0U) &&
             (Itineraries[ItinClassIndx].LastStage == ~0U));
   }
 
-  /// Return the first stage of the itinerary.
+  /// \brief Return the first stage of the itinerary.
   const InstrStage *beginStage(unsigned ItinClassIndx) const {
     unsigned StageIdx = Itineraries[ItinClassIndx].FirstStage;
     return Stages + StageIdx;
   }
 
-  /// Return the last+1 stage of the itinerary.
+  /// \brief Return the last+1 stage of the itinerary.
   const InstrStage *endStage(unsigned ItinClassIndx) const {
     unsigned StageIdx = Itineraries[ItinClassIndx].LastStage;
     return Stages + StageIdx;
   }
 
-  /// Return the total stage latency of the given class.
-  /// The latency is the maximum completion time for any stage in the itinerary.
-  /// If no stages exist, it defaults to one cycle.
+  /// \brief Return the total stage latency of the given class.  The latency is
+  /// the maximum completion time for any stage in the itinerary.  If no stages
+  /// exist, it defaults to one cycle.
   unsigned getStageLatency(unsigned ItinClassIndx) const {
     // If the target doesn't provide itinerary information, use a simple
     // non-zero default value for all instructions.
@@ -165,8 +164,8 @@ class InstrItineraryData {
     return Latency;
   }
 
-  /// Return the cycle for the given class and operand.
-  /// Return -1 if no cycle is specified for the operand.
+  /// \brief Return the cycle for the given class and operand.  Return -1 if no
+  /// cycle is specified for the operand.
   int getOperandCycle(unsigned ItinClassIndx, unsigned OperandIdx) const {
     if (isEmpty())
       return -1;
@@ -179,11 +178,11 @@ class InstrItineraryData {
     return (int)OperandCycles[FirstIdx + OperandIdx];
   }
 
-  /// Return true if there is a pipeline forwarding
-  /// between instructions of itinerary classes DefClass and UseClasses so that
-  /// value produced by an instruction of itinerary class DefClass, operand
-  /// index DefIdx can be bypassed when it's read by an instruction of
-  /// itinerary class UseClass, operand index UseIdx.
+  /// \brief Return true if there is a pipeline forwarding between instructions
+  /// of itinerary classes DefClass and UseClasses so that value produced by an
+  /// instruction of itinerary class DefClass, operand index DefIdx can be
+  /// bypassed when it's read by an instruction of itinerary class UseClass,
+  /// operand index UseIdx.
   bool hasPipelineForwarding(unsigned DefClass, unsigned DefIdx,
                              unsigned UseClass, unsigned UseIdx) const {
     unsigned FirstDefIdx = Itineraries[DefClass].FirstOperandCycle;
@@ -202,9 +201,9 @@ class InstrItineraryData {
       Forwardings[FirstUseIdx + UseIdx];
   }
 
-  /// Compute and return the use operand latency of a given
-  /// itinerary class and operand index if the value is produced by an
-  /// instruction of the specified itinerary class and def operand index.
+  /// \brief Compute and return the use operand latency of a given itinerary
+  /// class and operand index if the value is produced by an instruction of the
+  /// specified itinerary class and def operand index.
   int getOperandLatency(unsigned DefClass, unsigned DefIdx,
                         unsigned UseClass, unsigned UseIdx) const {
     if (isEmpty())
@@ -226,7 +225,7 @@ class InstrItineraryData {
     return UseCycle;
   }
 
-  /// Return the number of micro-ops that the given class decodes to.
+  /// \brief Return the number of micro-ops that the given class decodes to.
   /// Return -1 for classes that require dynamic lookup via TargetInstrInfo.
   int getNumMicroOps(unsigned ItinClassIndx) const {
     if (isEmpty())
diff --git a/include/llvm/MC/MCLabel.h b/include/llvm/MC/MCLabel.h
index a386deb554c..a12473fdad0 100644
--- a/include/llvm/MC/MCLabel.h
+++ b/include/llvm/MC/MCLabel.h
@@ -17,42 +17,41 @@
 #include "llvm/Support/Compiler.h"
 
 namespace llvm {
-  class MCContext;
-  class raw_ostream;
-
-  /// MCLabel - Instances of this class represent a label name in the MC file,
-  /// and MCLabel are created and unique'd by the MCContext class.  MCLabel
-  /// should only be constructed for valid instances in the object file.
-  class MCLabel {
-    // Instance - the instance number of this Directional Local Label
-    unsigned Instance;
-
-  private:  // MCContext creates and uniques these.
-    friend class MCContext;
-    MCLabel(unsigned instance)
-      : Instance(instance) {}
-
-    MCLabel(const MCLabel&) = delete;
-    void operator=(const MCLabel&) = delete;
-  public:
-    /// getInstance - Get the current instance of this Directional Local Label.
-    unsigned getInstance() const { return Instance; }
-
-    /// incInstance - Increment the current instance of this Directional Local
-    /// Label.
-    unsigned incInstance() { return ++Instance; }
-
-    /// print - Print the value to the stream \p OS.
-    void print(raw_ostream &OS) const;
-
-    /// dump - Print the value to stderr.
-    void dump() const;
-  };
-
-  inline raw_ostream &operator<<(raw_ostream &OS, const MCLabel &Label) {
-    Label.print(OS);
-    return OS;
-  }
+class MCContext;
+class raw_ostream;
+
+/// \brief Instances of this class represent a label name in the MC file,
+/// and MCLabel are created and uniqued by the MCContext class.  MCLabel
+/// should only be constructed for valid instances in the object file.
+class MCLabel {
+  // \brief The instance number of this Directional Local Label.
+  unsigned Instance;
+
+private: // MCContext creates and uniques these.
+  friend class MCContext;
+  MCLabel(unsigned instance) : Instance(instance) {}
+
+  MCLabel(const MCLabel &) = delete;
+  void operator=(const MCLabel &) = delete;
+
+public:
+  /// \brief Get the current instance of this Directional Local Label.
+  unsigned getInstance() const { return Instance; }
+
+  /// \brief Increment the current instance of this Directional Local Label.
+  unsigned incInstance() { return ++Instance; }
+
+  /// \brief Print the value to the stream \p OS.
+  void print(raw_ostream &OS) const;
+
+  /// \brief Print the value to stderr.
+  void dump() const;
+};
+
+inline raw_ostream &operator<<(raw_ostream &OS, const MCLabel &Label) {
+  Label.print(OS);
+  return OS;
+}
 } // end namespace llvm
 
 #endif
diff --git a/include/llvm/MC/MCLinkerOptimizationHint.h b/include/llvm/MC/MCLinkerOptimizationHint.h
index c96d5782bc4..a519c4b71b0 100644
--- a/include/llvm/MC/MCLinkerOptimizationHint.h
+++ b/include/llvm/MC/MCLinkerOptimizationHint.h
@@ -103,10 +103,10 @@ class MCLOHDirective {
   /// Arguments of this directive. Order matters.
   SmallVector<MCSymbol *, 3> Args;
 
-  /// Emit this directive in @p OutStream using the information available
-  /// in the given @p ObjWriter and @p Layout to get the address of the
+  /// Emit this directive in \p OutStream using the information available
+  /// in the given \p ObjWriter and \p Layout to get the address of the
   /// arguments within the object file.
-  void Emit_impl(raw_ostream &OutStream, const MachObjectWriter &ObjWriter,
+  void emit_impl(raw_ostream &OutStream, const MachObjectWriter &ObjWriter,
                  const MCAsmLayout &Layout) const;
 
 public:
@@ -123,13 +123,13 @@ class MCLOHDirective {
 
   /// Emit this directive as:
   /// <kind, numArgs, addr1, ..., addrN>
-  void Emit(MachObjectWriter &ObjWriter, const MCAsmLayout &Layout) const {
+  void emit(MachObjectWriter &ObjWriter, const MCAsmLayout &Layout) const {
     raw_ostream &OutStream = ObjWriter.getStream();
-    Emit_impl(OutStream, ObjWriter, Layout);
+    emit_impl(OutStream, ObjWriter, Layout);
   }
 
-  /// Get the size in bytes of this directive if emitted in @p ObjWriter with
-  /// the given @p Layout.
+  /// Get the size in bytes of this directive if emitted in \p ObjWriter with
+  /// the given \p Layout.
   uint64_t getEmitSize(const MachObjectWriter &ObjWriter,
                        const MCAsmLayout &Layout) const {
     class raw_counting_ostream : public raw_ostream {
@@ -145,7 +145,7 @@ class MCLOHDirective {
     };
 
     raw_counting_ostream OutStream;
-    Emit_impl(OutStream, ObjWriter, Layout);
+    emit_impl(OutStream, ObjWriter, Layout);
     return OutStream.tell();
   }
 };
@@ -160,15 +160,15 @@ class MCLOHContainer {
 public:
   typedef SmallVectorImpl<MCLOHDirective> LOHDirectives;
 
-  MCLOHContainer() : EmitSize(0) {};
+  MCLOHContainer() : EmitSize(0) {}
 
   /// Const accessor to the directives.
   const LOHDirectives &getDirectives() const {
     return Directives;
   }
 
-  /// Add the directive of the given kind @p Kind with the given arguments
-  /// @p Args to the container.
+  /// Add the directive of the given kind \p Kind with the given arguments
+  /// \p Args to the container.
   void addDirective(MCLOHType Kind, const MCLOHDirective::LOHArgs &Args) {
     Directives.push_back(MCLOHDirective(Kind, Args));
   }
@@ -184,10 +184,10 @@ class MCLOHContainer {
   }
 
   /// Emit all Linker Optimization Hint in one big table.
-  /// Each line of the table is emitted by LOHDirective::Emit.
-  void Emit(MachObjectWriter &ObjWriter, const MCAsmLayout &Layout) const {
+  /// Each line of the table is emitted by LOHDirective::emit.
+  void emit(MachObjectWriter &ObjWriter, const MCAsmLayout &Layout) const {
     for (const MCLOHDirective &D : Directives)
-      D.Emit(ObjWriter, Layout);
+      D.emit(ObjWriter, Layout);
   }
 
   void reset() {
diff --git a/include/llvm/MC/MCMachOSymbolFlags.h b/include/llvm/MC/MCMachOSymbolFlags.h
deleted file mode 100644
index 71f01fab206..00000000000
--- a/include/llvm/MC/MCMachOSymbolFlags.h
+++ /dev/null
@@ -1,46 +0,0 @@
-//===- MCMachOSymbolFlags.h - MachO Symbol Flags ----------------*- C++ -*-===//
-//
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
-//
-//===----------------------------------------------------------------------===//
-//
-// This file declares the SymbolFlags used for the MachO target.
-//
-//===----------------------------------------------------------------------===//
-
-#ifndef LLVM_MC_MCMACHOSYMBOLFLAGS_H
-#define LLVM_MC_MCMACHOSYMBOLFLAGS_H
-
-// These flags are mostly used in MCMachOStreamer.cpp but also needed in
-// MachObjectWriter.cpp to test for Weak Definitions of symbols to emit
-// the correct relocation information.
-
-namespace llvm {
-  /// MachOSymbolFlags - We store the value for the 'desc' symbol field in the
-  /// lowest 16 bits of the implementation defined flags.
-  enum MachOSymbolFlags { // See <mach-o/nlist.h>.
-    SF_DescFlagsMask                        = 0xFFFF,
-
-    // Reference type flags.
-    SF_ReferenceTypeMask                    = 0x0007,
-    SF_ReferenceTypeUndefinedNonLazy        = 0x0000,
-    SF_ReferenceTypeUndefinedLazy           = 0x0001,
-    SF_ReferenceTypeDefined                 = 0x0002,
-    SF_ReferenceTypePrivateDefined          = 0x0003,
-    SF_ReferenceTypePrivateUndefinedNonLazy = 0x0004,
-    SF_ReferenceTypePrivateUndefinedLazy    = 0x0005,
-
-    // Other 'desc' flags.
-    SF_ThumbFunc                            = 0x0008,
-    SF_NoDeadStrip                          = 0x0020,
-    SF_WeakReference                        = 0x0040,
-    SF_WeakDefinition                       = 0x0080,
-    SF_SymbolResolver                       = 0x0100
-  };
-
-} // end namespace llvm
-
-#endif
diff --git a/include/llvm/MC/MCMachObjectWriter.h b/include/llvm/MC/MCMachObjectWriter.h
index bfe5d3614c0..175d73e72c1 100644
--- a/include/llvm/MC/MCMachObjectWriter.h
+++ b/include/llvm/MC/MCMachObjectWriter.h
@@ -21,22 +21,17 @@
 
 namespace llvm {
 
-class MCSectionData;
 class MachObjectWriter;
 
 class MCMachObjectTargetWriter {
   const unsigned Is64Bit : 1;
   const uint32_t CPUType;
   const uint32_t CPUSubtype;
-  // FIXME: Remove this, we should just always use it once we no longer care
-  // about Darwin 'as' compatibility.
-  const unsigned UseAggressiveSymbolFolding : 1;
   unsigned LocalDifference_RIT;
 
 protected:
   MCMachObjectTargetWriter(bool Is64Bit_, uint32_t CPUType_,
-                           uint32_t CPUSubtype_,
-                           bool UseAggressiveSymbolFolding_ = false);
+                           uint32_t CPUSubtype_);
 
   void setLocalDifferenceRelocationType(unsigned Type) {
     LocalDifference_RIT = Type;
@@ -45,18 +40,17 @@ class MCMachObjectTargetWriter {
 public:
   virtual ~MCMachObjectTargetWriter();
 
-  /// @name Lifetime Management
+  /// \name Lifetime Management
   /// @{
 
-  virtual void reset() {};
+  virtual void reset() {}
 
   /// @}
 
-  /// @name Accessors
+  /// \name Accessors
   /// @{
 
   bool is64Bit() const { return Is64Bit; }
-  bool useAggressiveSymbolFolding() const { return UseAggressiveSymbolFolding; }
   uint32_t getCPUType() const { return CPUType; }
   uint32_t getCPUSubtype() const { return CPUSubtype; }
   unsigned getLocalDifferenceRelocationType() const {
@@ -65,10 +59,10 @@ class MCMachObjectTargetWriter {
 
   /// @}
 
-  /// @name API
+  /// \name API
   /// @{
 
-  virtual void RecordRelocation(MachObjectWriter *Writer, MCAssembler &Asm,
+  virtual void recordRelocation(MachObjectWriter *Writer, MCAssembler &Asm,
                                 const MCAsmLayout &Layout,
                                 const MCFragment *Fragment,
                                 const MCFixup &Fixup, MCValue Target,
@@ -78,10 +72,9 @@ class MCMachObjectTargetWriter {
 };
 
 class MachObjectWriter : public MCObjectWriter {
-  /// MachSymbolData - Helper struct for containing some precomputed information
-  /// on symbols.
+  /// Helper struct for containing some precomputed information on symbols.
   struct MachSymbolData {
-    MCSymbolData *SymbolData;
+    const MCSymbol *Symbol;
     uint64_t StringIndex;
     uint8_t SectionIndex;
 
@@ -92,21 +85,23 @@ class MachObjectWriter : public MCObjectWriter {
   /// The target specific Mach-O writer instance.
   std::unique_ptr<MCMachObjectTargetWriter> TargetObjectWriter;
 
-  /// @name Relocation Data
+  /// \name Relocation Data
   /// @{
 
   struct RelAndSymbol {
-    const MCSymbolData *Sym;
+    const MCSymbol *Sym;
     MachO::any_relocation_info MRE;
-    RelAndSymbol(const MCSymbolData *Sym, const MachO::any_relocation_info &MRE)
+    RelAndSymbol(const MCSymbol *Sym, const MachO::any_relocation_info &MRE)
         : Sym(Sym), MRE(MRE) {}
   };
 
-  llvm::DenseMap<const MCSectionData *, std::vector<RelAndSymbol>> Relocations;
-  llvm::DenseMap<const MCSectionData*, unsigned> IndirectSymBase;
+  llvm::DenseMap<const MCSection *, std::vector<RelAndSymbol>> Relocations;
+  llvm::DenseMap<const MCSection *, unsigned> IndirectSymBase;
+
+  SectionAddrMap SectionAddress;
 
   /// @}
-  /// @name Symbol Table Data
+  /// \name Symbol Table Data
   /// @{
 
   StringTableBuilder StringTable;
@@ -123,39 +118,37 @@ class MachObjectWriter : public MCObjectWriter {
                    bool IsLittleEndian)
       : MCObjectWriter(OS, IsLittleEndian), TargetObjectWriter(MOTW) {}
 
-  /// @name Lifetime management Methods
+  const MCSymbol &findAliasedSymbol(const MCSymbol &Sym) const;
+
+  /// \name Lifetime management Methods
   /// @{
 
   void reset() override;
 
   /// @}
 
-  /// @name Utility Methods
+  /// \name Utility Methods
   /// @{
 
   bool isFixupKindPCRel(const MCAssembler &Asm, unsigned Kind);
 
-  SectionAddrMap SectionAddress;
-
   SectionAddrMap &getSectionAddressMap() { return SectionAddress; }
 
-  uint64_t getSectionAddress(const MCSectionData* SD) const {
-    return SectionAddress.lookup(SD);
+  uint64_t getSectionAddress(const MCSection *Sec) const {
+    return SectionAddress.lookup(Sec);
   }
-  uint64_t getSymbolAddress(const MCSymbolData* SD,
-                            const MCAsmLayout &Layout) const;
+  uint64_t getSymbolAddress(const MCSymbol &S, const MCAsmLayout &Layout) const;
 
   uint64_t getFragmentAddress(const MCFragment *Fragment,
                               const MCAsmLayout &Layout) const;
 
-  uint64_t getPaddingSize(const MCSectionData *SD,
-                          const MCAsmLayout &Layout) const;
+  uint64_t getPaddingSize(const MCSection *SD, const MCAsmLayout &Layout) const;
 
-  bool doesSymbolRequireExternRelocation(const MCSymbolData *SD);
+  bool doesSymbolRequireExternRelocation(const MCSymbol &S);
 
   /// @}
 
-  /// @name Target Writer Proxy Accessors
+  /// \name Target Writer Proxy Accessors
   /// @{
 
   bool is64Bit() const { return TargetObjectWriter->is64Bit(); }
@@ -166,41 +159,37 @@ class MachObjectWriter : public MCObjectWriter {
 
   /// @}
 
-  void WriteHeader(unsigned NumLoadCommands, unsigned LoadCommandsSize,
+  void writeHeader(unsigned NumLoadCommands, unsigned LoadCommandsSize,
                    bool SubsectionsViaSymbols);
 
-  /// WriteSegmentLoadCommand - Write a segment load command.
+  /// Write a segment load command.
   ///
   /// \param NumSections The number of sections in this segment.
   /// \param SectionDataSize The total size of the sections.
-  void WriteSegmentLoadCommand(unsigned NumSections,
-                               uint64_t VMSize,
+  void writeSegmentLoadCommand(unsigned NumSections, uint64_t VMSize,
                                uint64_t SectionDataStartOffset,
                                uint64_t SectionDataSize);
 
-  void WriteSection(const MCAssembler &Asm, const MCAsmLayout &Layout,
-                    const MCSectionData &SD, uint64_t FileOffset,
+  void writeSection(const MCAssembler &Asm, const MCAsmLayout &Layout,
+                    const MCSection &Sec, uint64_t FileOffset,
                     uint64_t RelocationsStart, unsigned NumRelocations);
 
-  void WriteSymtabLoadCommand(uint32_t SymbolOffset, uint32_t NumSymbols,
+  void writeSymtabLoadCommand(uint32_t SymbolOffset, uint32_t NumSymbols,
                               uint32_t StringTableOffset,
                               uint32_t StringTableSize);
 
-  void WriteDysymtabLoadCommand(uint32_t FirstLocalSymbol,
-                                uint32_t NumLocalSymbols,
-                                uint32_t FirstExternalSymbol,
-                                uint32_t NumExternalSymbols,
-                                uint32_t FirstUndefinedSymbol,
-                                uint32_t NumUndefinedSymbols,
-                                uint32_t IndirectSymbolOffset,
-                                uint32_t NumIndirectSymbols);
+  void writeDysymtabLoadCommand(
+      uint32_t FirstLocalSymbol, uint32_t NumLocalSymbols,
+      uint32_t FirstExternalSymbol, uint32_t NumExternalSymbols,
+      uint32_t FirstUndefinedSymbol, uint32_t NumUndefinedSymbols,
+      uint32_t IndirectSymbolOffset, uint32_t NumIndirectSymbols);
 
-  void WriteNlist(MachSymbolData &MSD, const MCAsmLayout &Layout);
+  void writeNlist(MachSymbolData &MSD, const MCAsmLayout &Layout);
 
-  void WriteLinkeditLoadCommand(uint32_t Type, uint32_t DataOffset,
+  void writeLinkeditLoadCommand(uint32_t Type, uint32_t DataOffset,
                                 uint32_t DataSize);
 
-  void WriteLinkerOptionsLoadCommand(const std::vector<std::string> &Options);
+  void writeLinkerOptionsLoadCommand(const std::vector<std::string> &Options);
 
   // FIXME: We really need to improve the relocation validation. Basically, we
   // want to implement a separate computation which evaluates the relocation
@@ -221,35 +210,31 @@ class MachObjectWriter : public MCObjectWriter {
   // to a symbol it should be passed as \p RelSymbol so that it can be updated
   // afterwards. If the relocation doesn't refer to a symbol, nullptr should be
   // used.
-  void addRelocation(const MCSymbolData *RelSymbol, const MCSectionData *SD,
+  void addRelocation(const MCSymbol *RelSymbol, const MCSection *Sec,
                      MachO::any_relocation_info &MRE) {
     RelAndSymbol P(RelSymbol, MRE);
-    Relocations[SD].push_back(P);
+    Relocations[Sec].push_back(P);
   }
 
-  void RecordScatteredRelocation(const MCAssembler &Asm,
+  void recordScatteredRelocation(const MCAssembler &Asm,
                                  const MCAsmLayout &Layout,
                                  const MCFragment *Fragment,
                                  const MCFixup &Fixup, MCValue Target,
-                                 unsigned Log2Size,
-                                 uint64_t &FixedValue);
+                                 unsigned Log2Size, uint64_t &FixedValue);
 
-  void RecordTLVPRelocation(const MCAssembler &Asm,
-                            const MCAsmLayout &Layout,
-                            const MCFragment *Fragment,
-                            const MCFixup &Fixup, MCValue Target,
-                            uint64_t &FixedValue);
+  void recordTLVPRelocation(const MCAssembler &Asm, const MCAsmLayout &Layout,
+                            const MCFragment *Fragment, const MCFixup &Fixup,
+                            MCValue Target, uint64_t &FixedValue);
 
-  void RecordRelocation(MCAssembler &Asm, const MCAsmLayout &Layout,
+  void recordRelocation(MCAssembler &Asm, const MCAsmLayout &Layout,
                         const MCFragment *Fragment, const MCFixup &Fixup,
                         MCValue Target, bool &IsPCRel,
                         uint64_t &FixedValue) override;
 
-  void BindIndirectSymbols(MCAssembler &Asm);
+  void bindIndirectSymbols(MCAssembler &Asm);
 
-  /// ComputeSymbolTable - Compute the symbol table data
-  ///
-  void ComputeSymbolTable(MCAssembler &Asm,
+  /// Compute the symbol table data.
+  void computeSymbolTable(MCAssembler &Asm,
                           std::vector<MachSymbolData> &LocalSymbolData,
                           std::vector<MachSymbolData> &ExternalSymbolData,
                           std::vector<MachSymbolData> &UndefinedSymbolData);
@@ -257,21 +242,18 @@ class MachObjectWriter : public MCObjectWriter {
   void computeSectionAddresses(const MCAssembler &Asm,
                                const MCAsmLayout &Layout);
 
-  void ExecutePostLayoutBinding(MCAssembler &Asm,
+  void executePostLayoutBinding(MCAssembler &Asm,
                                 const MCAsmLayout &Layout) override;
 
-  bool IsSymbolRefDifferenceFullyResolvedImpl(const MCAssembler &Asm,
-                                              const MCSymbolData &DataA,
-                                              const MCSymbolData *DataB,
-                                              const MCFragment &FB,
-                                              bool InSet,
+  bool isSymbolRefDifferenceFullyResolvedImpl(const MCAssembler &Asm,
+                                              const MCSymbol &SymA,
+                                              const MCFragment &FB, bool InSet,
                                               bool IsPCRel) const override;
 
-  void WriteObject(MCAssembler &Asm, const MCAsmLayout &Layout) override;
+  void writeObject(MCAssembler &Asm, const MCAsmLayout &Layout) override;
 };
 
-
-/// \brief Construct a new Mach-O writer instance.
+/// Construct a new Mach-O writer instance.
 ///
 /// This routine takes ownership of the target writer subclass.
 ///
diff --git a/include/llvm/MC/MCNaCl.h b/include/llvm/MC/MCNaCl.h
deleted file mode 100644
index d7617da6ca1..00000000000
--- a/include/llvm/MC/MCNaCl.h
+++ /dev/null
@@ -1,22 +0,0 @@
-//===- MCNaCl.h - NaCl-specific code for MC  --------------------*- C++ -*-===//
-//
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
-//
-//===----------------------------------------------------------------------===//
-
-#include "llvm/Support/CommandLine.h"
-
-namespace llvm {
-
-class MCContext;
-class MCStreamer;
-class Triple;
-/// Initialize target-specific bundle alignment and emit target-specific NaCl
-/// ELF note sections.
-void initializeNaClMCStreamer(MCStreamer &Streamer, MCContext &Ctx,
-                              const Triple &TheTriple);
-
-}
diff --git a/include/llvm/MC/MCNaClExpander.h b/include/llvm/MC/MCNaClExpander.h
deleted file mode 100644
index 80d3d2f76ea..00000000000
--- a/include/llvm/MC/MCNaClExpander.h
+++ /dev/null
@@ -1,59 +0,0 @@
-//===- MCNaClExpander.h -----------------------------------------*- C++ -*-===//
-//
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
-//
-//===----------------------------------------------------------------------===//
-//
-// This file declares the MCNaClExpander class. This is an abstract
-// class that encapsulates the expansion logic for MCInsts, and holds
-// state such as available scratch registers.
-//
-//===----------------------------------------------------------------------===//
-
-#ifndef LLVM_MC_MCNACLEXPANDER_H
-#define LLVM_MC_MCNACLEXPANDER_H
-
-#include "llvm/ADT/SmallVector.h"
-#include "llvm/MC/MCContext.h"
-#include "llvm/MC/MCInstrInfo.h"
-#include "llvm/MC/MCRegisterInfo.h"
-
-namespace llvm {
-class MCInst;
-class MCSubtargetInfo;
-class MCStreamer;
-class SourceMgr;
-
-class MCNaClExpander {
-private:
-  SmallVector<unsigned, 2> ScratchRegs;
-  const SourceMgr *SrcMgr;
-
-protected:
-  std::unique_ptr<MCInstrInfo> InstInfo;
-  std::unique_ptr<MCRegisterInfo> RegInfo;
-
-public:
-  MCNaClExpander(const MCContext &Ctx, std::unique_ptr<MCRegisterInfo> &&RI,
-                 std::unique_ptr<MCInstrInfo> &&II)
-      : InstInfo(std::move(II)), RegInfo(std::move(RI)) {
-    SrcMgr = Ctx.getSourceManager();
-  }
-
-  void Error(const MCInst &Inst, const char msg[]);
-
-  void pushScratchReg(unsigned Reg);
-  unsigned popScratchReg();
-  unsigned getScratchReg(int index);
-  unsigned numScratchRegs();
-
-  virtual ~MCNaClExpander() = default;
-  virtual bool expandInst(const MCInst &Inst, MCStreamer &Out,
-                          const MCSubtargetInfo &STI) = 0;
-};
-
-}
-#endif
diff --git a/include/llvm/MC/MCObjectFileInfo.h b/include/llvm/MC/MCObjectFileInfo.h
index 025a76a27af..b9cea06161d 100644
--- a/include/llvm/MC/MCObjectFileInfo.h
+++ b/include/llvm/MC/MCObjectFileInfo.h
@@ -18,28 +18,24 @@
 #include "llvm/Support/CodeGen.h"
 
 namespace llvm {
-  class MCContext;
-  class MCSection;
-  class StringRef;
+class MCContext;
+class MCSection;
 
 class MCObjectFileInfo {
 protected:
-  /// CommDirectiveSupportsAlignment - True if .comm supports alignment.  This
-  /// is a hack for as long as we support 10.4 Tiger, whose assembler doesn't
-  /// support alignment on comm.
+  /// True if .comm supports alignment.  This is a hack for as long as we
+  /// support 10.4 Tiger, whose assembler doesn't support alignment on comm.
   bool CommDirectiveSupportsAlignment;
 
-  /// SupportsWeakEmptyEHFrame - True if target object file supports a
-  /// weak_definition of constant 0 for an omitted EH frame.
+  /// True if target object file supports a weak_definition of constant 0 for an
+  /// omitted EH frame.
   bool SupportsWeakOmittedEHFrame;
 
-  /// SupportsCompactUnwindWithoutEHFrame - True if the target object file
-  /// supports emitting a compact unwind section without an associated EH frame
-  /// section.
+  /// True if the target object file supports emitting a compact unwind section
+  /// without an associated EH frame section.
   bool SupportsCompactUnwindWithoutEHFrame;
 
-  /// PersonalityEncoding, LSDAEncoding, TTypeEncoding - Some encoding values
-  /// for EH.
+  /// Some encoding values for EH.
   unsigned PersonalityEncoding;
   unsigned LSDAEncoding;
   unsigned FDECFIEncoding;
@@ -49,151 +45,154 @@ class MCObjectFileInfo {
   unsigned EHSectionType;
   unsigned EHSectionFlags;
 
-  /// CompactUnwindDwarfEHFrameOnly - Compact unwind encoding indicating that we
-  /// should emit only an EH frame.
+  /// Compact unwind encoding indicating that we should emit only an EH frame.
   unsigned CompactUnwindDwarfEHFrameOnly;
 
-  /// TextSection - Section directive for standard text.
-  ///
-  const MCSection *TextSection;
+  /// Section directive for standard text.
+  MCSection *TextSection;
 
-  /// DataSection - Section directive for standard data.
-  ///
-  const MCSection *DataSection;
+  /// Section directive for standard data.
+  MCSection *DataSection;
 
-  /// BSSSection - Section that is default initialized to zero.
-  const MCSection *BSSSection;
+  /// Section that is default initialized to zero.
+  MCSection *BSSSection;
 
-  /// ReadOnlySection - Section that is readonly and can contain arbitrary
-  /// initialized data.  Targets are not required to have a readonly section.
-  /// If they don't, various bits of code will fall back to using the data
-  /// section for constants.
-  const MCSection *ReadOnlySection;
+  /// Section that is readonly and can contain arbitrary initialized data.
+  /// Targets are not required to have a readonly section. If they don't,
+  /// various bits of code will fall back to using the data section for
+  /// constants.
+  MCSection *ReadOnlySection;
 
-  /// StaticCtorSection - This section contains the static constructor pointer
-  /// list.
-  const MCSection *StaticCtorSection;
+  /// This section contains the static constructor pointer list.
+  MCSection *StaticCtorSection;
 
-  /// StaticDtorSection - This section contains the static destructor pointer
-  /// list.
-  const MCSection *StaticDtorSection;
+  /// This section contains the static destructor pointer list.
+  MCSection *StaticDtorSection;
 
-  /// LSDASection - If exception handling is supported by the target, this is
-  /// the section the Language Specific Data Area information is emitted to.
-  const MCSection *LSDASection;
+  /// If exception handling is supported by the target, this is the section the
+  /// Language Specific Data Area information is emitted to.
+  MCSection *LSDASection;
 
-  /// CompactUnwindSection - If exception handling is supported by the target
-  /// and the target can support a compact representation of the CIE and FDE,
-  /// this is the section to emit them into.
-  const MCSection *CompactUnwindSection;
+  /// If exception handling is supported by the target and the target can
+  /// support a compact representation of the CIE and FDE, this is the section
+  /// to emit them into.
+  MCSection *CompactUnwindSection;
 
   // Dwarf sections for debug info.  If a target supports debug info, these must
   // be set.
-  const MCSection *DwarfAbbrevSection;
-  const MCSection *DwarfInfoSection;
-  const MCSection *DwarfLineSection;
-  const MCSection *DwarfFrameSection;
-  const MCSection *DwarfPubTypesSection;
+  MCSection *DwarfAbbrevSection;
+  MCSection *DwarfInfoSection;
+  MCSection *DwarfLineSection;
+  MCSection *DwarfFrameSection;
+  MCSection *DwarfPubTypesSection;
   const MCSection *DwarfDebugInlineSection;
-  const MCSection *DwarfStrSection;
-  const MCSection *DwarfLocSection;
-  const MCSection *DwarfARangesSection;
-  const MCSection *DwarfRangesSection;
+  MCSection *DwarfStrSection;
+  MCSection *DwarfLocSection;
+  MCSection *DwarfARangesSection;
+  MCSection *DwarfRangesSection;
   // The pubnames section is no longer generated by default.  The generation
   // can be enabled by a compiler flag.
-  const MCSection *DwarfPubNamesSection;
+  MCSection *DwarfPubNamesSection;
 
-  // DWARF5 Experimental Debug Info Sections
+  /// DWARF5 Experimental Debug Info Sections
   /// DwarfAccelNamesSection, DwarfAccelObjCSection,
   /// DwarfAccelNamespaceSection, DwarfAccelTypesSection -
   /// If we use the DWARF accelerated hash tables then we want to emit these
   /// sections.
-  const MCSection *DwarfAccelNamesSection;
-  const MCSection *DwarfAccelObjCSection;
-  const MCSection *DwarfAccelNamespaceSection;
-  const MCSection *DwarfAccelTypesSection;
-
-  /// These are used for the Fission separate debug information files.
-  const MCSection *DwarfInfoDWOSection;
-  const MCSection *DwarfTypesDWOSection;
-  const MCSection *DwarfAbbrevDWOSection;
-  const MCSection *DwarfStrDWOSection;
-  const MCSection *DwarfLineDWOSection;
-  const MCSection *DwarfLocDWOSection;
-  const MCSection *DwarfStrOffDWOSection;
-  const MCSection *DwarfAddrSection;
-
-  /// Sections for newer gnu pubnames and pubtypes.
-  const MCSection *DwarfGnuPubNamesSection;
-  const MCSection *DwarfGnuPubTypesSection;
-
-  const MCSection *COFFDebugSymbolsSection;
-
-  // Extra TLS Variable Data section.  If the target needs to put additional
-  // information for a TLS variable, it'll go here.
-  const MCSection *TLSExtraDataSection;
-
-  /// TLSDataSection - Section directive for Thread Local data.
-  /// ELF, MachO and COFF.
-  const MCSection *TLSDataSection;        // Defaults to ".tdata".
-
-  /// TLSBSSSection - Section directive for Thread Local uninitialized data.
-  /// Null if this target doesn't support a BSS section.
-  /// ELF and MachO only.
-  const MCSection *TLSBSSSection;         // Defaults to ".tbss".
+  MCSection *DwarfAccelNamesSection;
+  MCSection *DwarfAccelObjCSection;
+  MCSection *DwarfAccelNamespaceSection;
+  MCSection *DwarfAccelTypesSection;
+
+  // These are used for the Fission separate debug information files.
+  MCSection *DwarfInfoDWOSection;
+  MCSection *DwarfTypesDWOSection;
+  MCSection *DwarfAbbrevDWOSection;
+  MCSection *DwarfStrDWOSection;
+  MCSection *DwarfLineDWOSection;
+  MCSection *DwarfLocDWOSection;
+  MCSection *DwarfStrOffDWOSection;
+  MCSection *DwarfAddrSection;
+
+  /// Section for newer gnu pubnames.
+  MCSection *DwarfGnuPubNamesSection;
+  /// Section for newer gnu pubtypes.
+  MCSection *DwarfGnuPubTypesSection;
+
+  MCSection *COFFDebugSymbolsSection;
+
+  /// Extra TLS Variable Data section.
+  ///
+  /// If the target needs to put additional information for a TLS variable,
+  /// it'll go here.
+  MCSection *TLSExtraDataSection;
+
+  /// Section directive for Thread Local data. ELF, MachO and COFF.
+  MCSection *TLSDataSection; // Defaults to ".tdata".
+
+  /// Section directive for Thread Local uninitialized data.
+  ///
+  /// Null if this target doesn't support a BSS section. ELF and MachO only.
+  MCSection *TLSBSSSection; // Defaults to ".tbss".
 
   /// StackMap section.
-  const MCSection *StackMapSection;
+  MCSection *StackMapSection;
 
-  /// EHFrameSection - EH frame section. It is initialized on demand so it
-  /// can be overwritten (with uniquing).
-  const MCSection *EHFrameSection;
+  /// FaultMap section.
+  MCSection *FaultMapSection;
 
-  /// ELF specific sections.
+  /// EH frame section.
   ///
-  const MCSection *DataRelSection;
+  /// It is initialized on demand so it can be overwritten (with uniquing).
+  MCSection *EHFrameSection;
+
+  // ELF specific sections.
+  MCSection *DataRelSection;
   const MCSection *DataRelLocalSection;
-  const MCSection *DataRelROSection;
-  const MCSection *DataRelROLocalSection;
-  const MCSection *MergeableConst4Section;
-  const MCSection *MergeableConst8Section;
-  const MCSection *MergeableConst16Section;
+  MCSection *DataRelROSection;
+  MCSection *DataRelROLocalSection;
+  MCSection *MergeableConst4Section;
+  MCSection *MergeableConst8Section;
+  MCSection *MergeableConst16Section;
 
-  /// MachO specific sections.
-  ///
+  // MachO specific sections.
 
-  /// TLSTLVSection - Section for thread local structure information.
-  /// Contains the source code name of the variable, visibility and a pointer
-  /// to the initial value (.tdata or .tbss).
-  const MCSection *TLSTLVSection;         // Defaults to ".tlv".
-
-  /// TLSThreadInitSection - Section for thread local data initialization
-  /// functions.
-  const MCSection *TLSThreadInitSection;  // Defaults to ".thread_init_func".
-
-  const MCSection *CStringSection;
-  const MCSection *UStringSection;
-  const MCSection *TextCoalSection;
-  const MCSection *ConstTextCoalSection;
-  const MCSection *ConstDataSection;
-  const MCSection *DataCoalSection;
-  const MCSection *DataCommonSection;
-  const MCSection *DataBSSSection;
-  const MCSection *FourByteConstantSection;
-  const MCSection *EightByteConstantSection;
-  const MCSection *SixteenByteConstantSection;
-  const MCSection *LazySymbolPointerSection;
-  const MCSection *NonLazySymbolPointerSection;
+  /// Section for thread local structure information.
+  ///
+  /// Contains the source code name of the variable, visibility and a pointer to
+  /// the initial value (.tdata or .tbss).
+  MCSection *TLSTLVSection; // Defaults to ".tlv".
+
+  /// Section for thread local data initialization functions.
+  const MCSection *TLSThreadInitSection; // Defaults to ".thread_init_func".
+
+  MCSection *CStringSection;
+  MCSection *UStringSection;
+  MCSection *TextCoalSection;
+  MCSection *ConstTextCoalSection;
+  MCSection *ConstDataSection;
+  MCSection *DataCoalSection;
+  MCSection *DataCommonSection;
+  MCSection *DataBSSSection;
+  MCSection *FourByteConstantSection;
+  MCSection *EightByteConstantSection;
+  MCSection *SixteenByteConstantSection;
+  MCSection *LazySymbolPointerSection;
+  MCSection *NonLazySymbolPointerSection;
 
   /// COFF specific sections.
-  ///
-  const MCSection *DrectveSection;
-  const MCSection *PDataSection;
-  const MCSection *XDataSection;
+  MCSection *DrectveSection;
+  MCSection *PDataSection;
+  MCSection *XDataSection;
+  MCSection *SXDataSection;
 
 public:
-  void InitMCObjectFileInfo(StringRef TT, Reloc::Model RM, CodeModel::Model CM,
-                            MCContext &ctx);
+  void InitMCObjectFileInfo(const Triple &TT, Reloc::Model RM,
+                            CodeModel::Model CM, MCContext &ctx);
+  LLVM_ATTRIBUTE_DEPRECATED(
+      void InitMCObjectFileInfo(StringRef TT, Reloc::Model RM,
+                                CodeModel::Model CM, MCContext &ctx),
+      "StringRef GNU Triple argument replaced by a llvm::Triple object");
 
   bool getSupportsWeakOmittedEHFrame() const {
     return SupportsWeakOmittedEHFrame;
@@ -214,94 +213,71 @@ class MCObjectFileInfo {
     return CompactUnwindDwarfEHFrameOnly;
   }
 
-  const MCSection *getTextSection() const { return TextSection; }
-  const MCSection *getDataSection() const { return DataSection; }
-  const MCSection *getBSSSection() const { return BSSSection; }
-  const MCSection *getLSDASection() const { return LSDASection; }
-  const MCSection *getCompactUnwindSection() const{
-    return CompactUnwindSection;
-  }
-  const MCSection *getDwarfAbbrevSection() const { return DwarfAbbrevSection; }
-  const MCSection *getDwarfInfoSection() const { return DwarfInfoSection; }
-  const MCSection *getDwarfLineSection() const { return DwarfLineSection; }
-  const MCSection *getDwarfFrameSection() const { return DwarfFrameSection; }
-  const MCSection *getDwarfPubNamesSection() const{return DwarfPubNamesSection;}
-  const MCSection *getDwarfPubTypesSection() const{return DwarfPubTypesSection;}
-  const MCSection *getDwarfGnuPubNamesSection() const {
+  MCSection *getTextSection() const { return TextSection; }
+  MCSection *getDataSection() const { return DataSection; }
+  MCSection *getBSSSection() const { return BSSSection; }
+  MCSection *getReadOnlySection() const { return ReadOnlySection; }
+  MCSection *getLSDASection() const { return LSDASection; }
+  MCSection *getCompactUnwindSection() const { return CompactUnwindSection; }
+  MCSection *getDwarfAbbrevSection() const { return DwarfAbbrevSection; }
+  MCSection *getDwarfInfoSection() const { return DwarfInfoSection; }
+  MCSection *getDwarfLineSection() const { return DwarfLineSection; }
+  MCSection *getDwarfFrameSection() const { return DwarfFrameSection; }
+  MCSection *getDwarfPubNamesSection() const { return DwarfPubNamesSection; }
+  MCSection *getDwarfPubTypesSection() const { return DwarfPubTypesSection; }
+  MCSection *getDwarfGnuPubNamesSection() const {
     return DwarfGnuPubNamesSection;
   }
-  const MCSection *getDwarfGnuPubTypesSection() const {
+  MCSection *getDwarfGnuPubTypesSection() const {
     return DwarfGnuPubTypesSection;
   }
   const MCSection *getDwarfDebugInlineSection() const {
     return DwarfDebugInlineSection;
   }
-  const MCSection *getDwarfStrSection() const { return DwarfStrSection; }
-  const MCSection *getDwarfLocSection() const { return DwarfLocSection; }
-  const MCSection *getDwarfARangesSection() const { return DwarfARangesSection;}
-  const MCSection *getDwarfRangesSection() const { return DwarfRangesSection; }
+  MCSection *getDwarfStrSection() const { return DwarfStrSection; }
+  MCSection *getDwarfLocSection() const { return DwarfLocSection; }
+  MCSection *getDwarfARangesSection() const { return DwarfARangesSection; }
+  MCSection *getDwarfRangesSection() const { return DwarfRangesSection; }
 
   // DWARF5 Experimental Debug Info Sections
-  const MCSection *getDwarfAccelNamesSection() const {
+  MCSection *getDwarfAccelNamesSection() const {
     return DwarfAccelNamesSection;
   }
-  const MCSection *getDwarfAccelObjCSection() const {
-    return DwarfAccelObjCSection;
-  }
-  const MCSection *getDwarfAccelNamespaceSection() const {
+  MCSection *getDwarfAccelObjCSection() const { return DwarfAccelObjCSection; }
+  MCSection *getDwarfAccelNamespaceSection() const {
     return DwarfAccelNamespaceSection;
   }
-  const MCSection *getDwarfAccelTypesSection() const {
+  MCSection *getDwarfAccelTypesSection() const {
     return DwarfAccelTypesSection;
   }
-  const MCSection *getDwarfInfoDWOSection() const {
-    return DwarfInfoDWOSection;
-  }
-  const MCSection *getDwarfTypesSection(uint64_t Hash) const;
-  const MCSection *getDwarfTypesDWOSection() const {
-    return DwarfTypesDWOSection;
-  }
-  const MCSection *getDwarfAbbrevDWOSection() const {
-    return DwarfAbbrevDWOSection;
-  }
-  const MCSection *getDwarfStrDWOSection() const {
-    return DwarfStrDWOSection;
-  }
-  const MCSection *getDwarfLineDWOSection() const {
-    return DwarfLineDWOSection;
-  }
-  const MCSection *getDwarfLocDWOSection() const {
-    return DwarfLocDWOSection;
-  }
-  const MCSection *getDwarfStrOffDWOSection() const {
-    return DwarfStrOffDWOSection;
-  }
-  const MCSection *getDwarfAddrSection() const {
-    return DwarfAddrSection;
-  }
-
-  const MCSection *getCOFFDebugSymbolsSection() const {
+  MCSection *getDwarfInfoDWOSection() const { return DwarfInfoDWOSection; }
+  MCSection *getDwarfTypesSection(uint64_t Hash) const;
+  MCSection *getDwarfTypesDWOSection() const { return DwarfTypesDWOSection; }
+  MCSection *getDwarfAbbrevDWOSection() const { return DwarfAbbrevDWOSection; }
+  MCSection *getDwarfStrDWOSection() const { return DwarfStrDWOSection; }
+  MCSection *getDwarfLineDWOSection() const { return DwarfLineDWOSection; }
+  MCSection *getDwarfLocDWOSection() const { return DwarfLocDWOSection; }
+  MCSection *getDwarfStrOffDWOSection() const { return DwarfStrOffDWOSection; }
+  MCSection *getDwarfAddrSection() const { return DwarfAddrSection; }
+
+  MCSection *getCOFFDebugSymbolsSection() const {
     return COFFDebugSymbolsSection;
   }
 
-  const MCSection *getTLSExtraDataSection() const {
-    return TLSExtraDataSection;
-  }
+  MCSection *getTLSExtraDataSection() const { return TLSExtraDataSection; }
   const MCSection *getTLSDataSection() const { return TLSDataSection; }
-  const MCSection *getTLSBSSSection() const { return TLSBSSSection; }
+  MCSection *getTLSBSSSection() const { return TLSBSSSection; }
 
-  const MCSection *getStackMapSection() const { return StackMapSection; }
+  MCSection *getStackMapSection() const { return StackMapSection; }
+  MCSection *getFaultMapSection() const { return FaultMapSection; }
 
-  /// ELF specific sections.
-  ///
-  const MCSection *getDataRelSection() const { return DataRelSection; }
+  // ELF specific sections.
+  MCSection *getDataRelSection() const { return DataRelSection; }
   const MCSection *getDataRelLocalSection() const {
     return DataRelLocalSection;
   }
-  const MCSection *getDataRelROSection() const { return DataRelROSection; }
-  const MCSection *getDataRelROLocalSection() const {
-    return DataRelROLocalSection;
-  }
+  MCSection *getDataRelROSection() const { return DataRelROSection; }
+  MCSection *getDataRelROLocalSection() const { return DataRelROLocalSection; }
   const MCSection *getMergeableConst4Section() const {
     return MergeableConst4Section;
   }
@@ -312,22 +288,21 @@ class MCObjectFileInfo {
     return MergeableConst16Section;
   }
 
-  /// MachO specific sections.
-  ///
+  // MachO specific sections.
   const MCSection *getTLSTLVSection() const { return TLSTLVSection; }
   const MCSection *getTLSThreadInitSection() const {
     return TLSThreadInitSection;
   }
   const MCSection *getCStringSection() const { return CStringSection; }
   const MCSection *getUStringSection() const { return UStringSection; }
-  const MCSection *getTextCoalSection() const { return TextCoalSection; }
+  MCSection *getTextCoalSection() const { return TextCoalSection; }
   const MCSection *getConstTextCoalSection() const {
     return ConstTextCoalSection;
   }
   const MCSection *getConstDataSection() const { return ConstDataSection; }
   const MCSection *getDataCoalSection() const { return DataCoalSection; }
   const MCSection *getDataCommonSection() const { return DataCommonSection; }
-  const MCSection *getDataBSSSection() const { return DataBSSSection; }
+  MCSection *getDataBSSSection() const { return DataBSSSection; }
   const MCSection *getFourByteConstantSection() const {
     return FourByteConstantSection;
   }
@@ -337,33 +312,29 @@ class MCObjectFileInfo {
   const MCSection *getSixteenByteConstantSection() const {
     return SixteenByteConstantSection;
   }
-  const MCSection *getLazySymbolPointerSection() const {
+  MCSection *getLazySymbolPointerSection() const {
     return LazySymbolPointerSection;
   }
-  const MCSection *getNonLazySymbolPointerSection() const {
+  MCSection *getNonLazySymbolPointerSection() const {
     return NonLazySymbolPointerSection;
   }
 
-  /// COFF specific sections.
-  ///
-  const MCSection *getDrectveSection() const { return DrectveSection; }
-  const MCSection *getPDataSection() const { return PDataSection; }
-  const MCSection *getXDataSection() const { return XDataSection; }
+  // COFF specific sections.
+  MCSection *getDrectveSection() const { return DrectveSection; }
+  MCSection *getPDataSection() const { return PDataSection; }
+  MCSection *getXDataSection() const { return XDataSection; }
+  MCSection *getSXDataSection() const { return SXDataSection; }
 
-  const MCSection *getEHFrameSection() {
+  MCSection *getEHFrameSection() {
     if (!EHFrameSection)
       InitEHFrameSection();
     return EHFrameSection;
   }
 
   enum Environment { IsMachO, IsELF, IsCOFF };
-  Environment getObjectFileType() const {
-    return Env;
-  }
+  Environment getObjectFileType() const { return Env; }
 
-  Reloc::Model getRelocM() const {
-    return RelocM;
-  }
+  Reloc::Model getRelocM() const { return RelocM; }
 
 private:
   Environment Env;
@@ -372,12 +343,11 @@ class MCObjectFileInfo {
   MCContext *Ctx;
   Triple TT;
 
-  void InitMachOMCObjectFileInfo(Triple T);
-  void InitELFMCObjectFileInfo(Triple T);
-  void InitCOFFMCObjectFileInfo(Triple T);
+  void initMachOMCObjectFileInfo(Triple T);
+  void initELFMCObjectFileInfo(Triple T);
+  void initCOFFMCObjectFileInfo(Triple T);
 
-  /// InitEHFrameSection - Initialize EHFrameSection on demand.
-  ///
+  /// Initialize EHFrameSection on demand.
   void InitEHFrameSection();
 
 public:
diff --git a/include/llvm/MC/MCObjectStreamer.h b/include/llvm/MC/MCObjectStreamer.h
index 6666a503ea8..ce1fc80f2cf 100644
--- a/include/llvm/MC/MCObjectStreamer.h
+++ b/include/llvm/MC/MCObjectStreamer.h
@@ -12,12 +12,12 @@
 
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/MC/MCAssembler.h"
+#include "llvm/MC/MCSection.h"
 #include "llvm/MC/MCStreamer.h"
 
 namespace llvm {
 class MCAssembler;
 class MCCodeEmitter;
-class MCSectionData;
 class MCSubtargetInfo;
 class MCExpr;
 class MCFragment;
@@ -35,11 +35,10 @@ class raw_pwrite_stream;
 /// implementation.
 class MCObjectStreamer : public MCStreamer {
   MCAssembler *Assembler;
-  MCSectionData *CurSectionData;
-  MCSectionData::iterator CurInsertionPoint;
+  MCSection::iterator CurInsertionPoint;
   bool EmitEHFrame;
   bool EmitDebugFrame;
-  SmallVector<MCSymbolData *, 2> PendingLabels;
+  SmallVector<MCSymbol *, 2> PendingLabels;
 
   virtual void EmitInstToData(const MCInst &Inst, const MCSubtargetInfo&) = 0;
   void EmitCFIStartProcImpl(MCDwarfFrameInfo &Frame) override;
@@ -57,30 +56,24 @@ class MCObjectStreamer : public MCStreamer {
   /// Object streamers require the integrated assembler.
   bool isIntegratedAssemblerRequired() const override { return true; }
 
-  MCSymbolData &getOrCreateSymbolData(const MCSymbol *Symbol) {
-    return getAssembler().getOrCreateSymbolData(*Symbol);
-  }
   void EmitFrames(MCAsmBackend *MAB);
   void EmitCFISections(bool EH, bool Debug) override;
 
 protected:
-  MCSectionData *getCurrentSectionData() const {
-    return CurSectionData;
-  }
-
   MCFragment *getCurrentFragment() const;
 
   void insert(MCFragment *F) {
     flushPendingLabels(F);
-    CurSectionData->getFragmentList().insert(CurInsertionPoint, F);
-    F->setParent(CurSectionData);
+    MCSection *CurSection = getCurrentSectionOnly();
+    CurSection->getFragmentList().insert(CurInsertionPoint, F);
+    F->setParent(CurSection);
   }
 
   /// Get a data fragment to write into, creating a new one if the current
   /// fragment is not a data fragment.
   MCDataFragment *getOrCreateDataFragment();
 
-  bool changeSectionImpl(const MCSection *Section, const MCExpr *Subsection);
+  bool changeSectionImpl(MCSection *Section, const MCExpr *Subsection);
 
   /// If any labels have been emitted but not assigned fragments, ensure that
   /// they get assigned, either to F if possible or to a new data fragment.
@@ -93,7 +86,7 @@ class MCObjectStreamer : public MCStreamer {
 
   MCAssembler &getAssembler() { return *Assembler; }
 
-  /// @name MCStreamer Interface
+  /// \name MCStreamer Interface
   /// @{
 
   void EmitLabel(MCSymbol *Symbol) override;
@@ -103,8 +96,7 @@ class MCObjectStreamer : public MCStreamer {
   void EmitULEB128Value(const MCExpr *Value) override;
   void EmitSLEB128Value(const MCExpr *Value) override;
   void EmitWeakReference(MCSymbol *Alias, const MCSymbol *Symbol) override;
-  void ChangeSection(const MCSection *Section,
-                     const MCExpr *Subsection) override;
+  void ChangeSection(MCSection *Section, const MCExpr *Subsection) override;
   void EmitInstruction(const MCInst &Inst, const MCSubtargetInfo& STI) override;
 
   /// \brief Emit an instruction to a special fragment, because this instruction
@@ -136,9 +128,17 @@ class MCObjectStreamer : public MCStreamer {
   void EmitZeros(uint64_t NumBytes) override;
   void FinishImpl() override;
 
-  bool mayHaveInstructions() const override {
-    return getCurrentSectionData()->hasInstructions();
-  }
+  /// Emit the absolute difference between two symbols if possible.
+  ///
+  /// Emit the absolute difference between \c Hi and \c Lo, as long as we can
+  /// compute it.  Currently, that requires that both symbols are in the same
+  /// data fragment.  Otherwise, do nothing and return \c false.
+  ///
+  /// \pre Offset of \c Hi is greater than the offset \c Lo.
+  void emitAbsoluteSymbolDiff(const MCSymbol *Hi, const MCSymbol *Lo,
+                              unsigned Size) override;
+
+  bool mayHaveInstructions(MCSection &Sec) const override;
 };
 
 } // end namespace llvm
diff --git a/include/llvm/MC/MCObjectWriter.h b/include/llvm/MC/MCObjectWriter.h
index f8e2821c4dc..2211673efc3 100644
--- a/include/llvm/MC/MCObjectWriter.h
+++ b/include/llvm/MC/MCObjectWriter.h
@@ -22,18 +22,17 @@ class MCAsmLayout;
 class MCAssembler;
 class MCFixup;
 class MCFragment;
-class MCSymbolData;
 class MCSymbolRefExpr;
 class MCValue;
 
-/// MCObjectWriter - Defines the object file and target independent interfaces
-/// used by the assembler backend to write native file format object files.
+/// Defines the object file and target independent interfaces used by the
+/// assembler backend to write native file format object files.
 ///
 /// The object writer contains a few callbacks used by the assembler to allow
 /// the object writer to modify the assembler data structures at appropriate
 /// points. Once assembly is complete, the object writer is given the
 /// MCAssembler instance, which contains all the symbol and section data which
-/// should be emitted as part of WriteObject().
+/// should be emitted as part of writeObject().
 ///
 /// The object writer also contains a number of helper methods for writing
 /// binary data to the output stream.
@@ -54,119 +53,115 @@ class MCObjectWriter {
   virtual ~MCObjectWriter();
 
   /// lifetime management
-  virtual void reset() { }
+  virtual void reset() {}
 
   bool isLittleEndian() const { return IsLittleEndian; }
 
   raw_ostream &getStream() { return OS; }
 
-  /// @name High-Level API
+  /// \name High-Level API
   /// @{
 
-  /// \brief Perform any late binding of symbols (for example, to assign symbol
+  /// Perform any late binding of symbols (for example, to assign symbol
   /// indices for use when generating relocations).
   ///
   /// This routine is called by the assembler after layout and relaxation is
   /// complete.
-  virtual void ExecutePostLayoutBinding(MCAssembler &Asm,
+  virtual void executePostLayoutBinding(MCAssembler &Asm,
                                         const MCAsmLayout &Layout) = 0;
 
-  /// \brief Record a relocation entry.
+  /// Record a relocation entry.
   ///
   /// This routine is called by the assembler after layout and relaxation, and
   /// post layout binding. The implementation is responsible for storing
   /// information about the relocation so that it can be emitted during
-  /// WriteObject().
-  virtual void RecordRelocation(MCAssembler &Asm, const MCAsmLayout &Layout,
+  /// writeObject().
+  virtual void recordRelocation(MCAssembler &Asm, const MCAsmLayout &Layout,
                                 const MCFragment *Fragment,
                                 const MCFixup &Fixup, MCValue Target,
                                 bool &IsPCRel, uint64_t &FixedValue) = 0;
 
-  /// \brief Check whether the difference (A - B) between two symbol
-  /// references is fully resolved.
+  /// Check whether the difference (A - B) between two symbol references is
+  /// fully resolved.
   ///
   /// Clients are not required to answer precisely and may conservatively return
   /// false, even when a difference is fully resolved.
-  bool IsSymbolRefDifferenceFullyResolved(const MCAssembler &Asm,
+  bool isSymbolRefDifferenceFullyResolved(const MCAssembler &Asm,
                                           const MCSymbolRefExpr *A,
                                           const MCSymbolRefExpr *B,
                                           bool InSet) const;
 
-  virtual bool IsSymbolRefDifferenceFullyResolvedImpl(const MCAssembler &Asm,
-                                                      const MCSymbolData &DataA,
-                                                      const MCSymbolData *DataB,
+  virtual bool isSymbolRefDifferenceFullyResolvedImpl(const MCAssembler &Asm,
+                                                      const MCSymbol &SymA,
                                                       const MCFragment &FB,
                                                       bool InSet,
                                                       bool IsPCRel) const;
 
-  /// \brief True if this symbol (which is a variable) is weak. This is not
+  /// True if this symbol (which is a variable) is weak. This is not
   /// just STB_WEAK, but more generally whether or not we can evaluate
   /// past it.
-  virtual bool isWeak(const MCSymbolData &SD) const;
+  virtual bool isWeak(const MCSymbol &Sym) const;
 
-  /// \brief Write the object file.
+  /// Write the object file.
   ///
   /// This routine is called by the assembler after layout and relaxation is
   /// complete, fixups have been evaluated and applied, and relocations
   /// generated.
-  virtual void WriteObject(MCAssembler &Asm,
-                           const MCAsmLayout &Layout) = 0;
+  virtual void writeObject(MCAssembler &Asm, const MCAsmLayout &Layout) = 0;
 
   /// @}
-  /// @name Binary Output
+  /// \name Binary Output
   /// @{
 
-  void Write8(uint8_t Value) {
-    OS << char(Value);
-  }
+  void write8(uint8_t Value) { OS << char(Value); }
 
-  void WriteLE16(uint16_t Value) {
+  void writeLE16(uint16_t Value) {
     support::endian::Writer<support::little>(OS).write(Value);
   }
 
-  void WriteLE32(uint32_t Value) {
+  void writeLE32(uint32_t Value) {
     support::endian::Writer<support::little>(OS).write(Value);
   }
 
-  void WriteLE64(uint64_t Value) {
+  void writeLE64(uint64_t Value) {
     support::endian::Writer<support::little>(OS).write(Value);
   }
 
-  void WriteBE16(uint16_t Value) {
+  void writeBE16(uint16_t Value) {
     support::endian::Writer<support::big>(OS).write(Value);
   }
 
-  void WriteBE32(uint32_t Value) {
+  void writeBE32(uint32_t Value) {
     support::endian::Writer<support::big>(OS).write(Value);
   }
 
-  void WriteBE64(uint64_t Value) {
+  void writeBE64(uint64_t Value) {
     support::endian::Writer<support::big>(OS).write(Value);
   }
 
-  void Write16(uint16_t Value) {
+  void write16(uint16_t Value) {
     if (IsLittleEndian)
-      WriteLE16(Value);
+      writeLE16(Value);
     else
-      WriteBE16(Value);
+      writeBE16(Value);
   }
 
-  void Write32(uint32_t Value) {
+  void write32(uint32_t Value) {
     if (IsLittleEndian)
-      WriteLE32(Value);
+      writeLE32(Value);
     else
-      WriteBE32(Value);
+      writeBE32(Value);
   }
 
-  void Write64(uint64_t Value) {
+  void write64(uint64_t Value) {
     if (IsLittleEndian)
-      WriteLE64(Value);
+      writeLE64(Value);
     else
-      WriteBE64(Value);
+      writeBE64(Value);
   }
 
   void WriteZeros(unsigned N) {
-    const char Zeros[16] = { 0 };
+    const char Zeros[16] = {0};
 
     for (unsigned i = 0, e = N / 16; i != e; ++i)
       OS << StringRef(Zeros, 16);
@@ -174,22 +169,23 @@ class MCObjectWriter {
     OS << StringRef(Zeros, N % 16);
   }
 
-  void WriteBytes(const SmallVectorImpl<char> &ByteVec, unsigned ZeroFillSize = 0) {
-    WriteBytes(StringRef(ByteVec.data(), ByteVec.size()), ZeroFillSize);
+  void writeBytes(const SmallVectorImpl<char> &ByteVec,
+                  unsigned ZeroFillSize = 0) {
+    writeBytes(StringRef(ByteVec.data(), ByteVec.size()), ZeroFillSize);
   }
 
-  void WriteBytes(StringRef Str, unsigned ZeroFillSize = 0) {
+  void writeBytes(StringRef Str, unsigned ZeroFillSize = 0) {
     // TODO: this version may need to go away once all fragment contents are
     // converted to SmallVector<char, N>
-    assert((ZeroFillSize == 0 || Str.size () <= ZeroFillSize) &&
-      "data size greater than fill size, unexpected large write will occur");
+    assert(
+        (ZeroFillSize == 0 || Str.size() <= ZeroFillSize) &&
+        "data size greater than fill size, unexpected large write will occur");
     OS << Str;
     if (ZeroFillSize)
       WriteZeros(ZeroFillSize - Str.size());
   }
 
   /// @}
-
 };
 
 } // End llvm namespace
diff --git a/include/llvm/MC/MCParser/MCAsmParser.h b/include/llvm/MC/MCParser/MCAsmParser.h
index 83ffbb5044d..ac8706d9950 100644
--- a/include/llvm/MC/MCParser/MCAsmParser.h
+++ b/include/llvm/MC/MCParser/MCAsmParser.h
@@ -45,7 +45,7 @@ class InlineAsmIdentifierInfo {
   }
 };
 
-/// Generic Sema callback for assembly parser.
+/// \brief Generic Sema callback for assembly parser.
 class MCAsmParserSemaCallback {
 public:
   virtual ~MCAsmParserSemaCallback();
@@ -59,8 +59,8 @@ class MCAsmParserSemaCallback {
                                     unsigned &Offset) = 0;
 };
 
-/// Generic assembler parser interface, for use by target specific assembly
-/// parsers.
+/// \brief Generic assembler parser interface, for use by target specific
+/// assembly parsers.
 class MCAsmParser {
 public:
   typedef bool (*DirectiveHandler)(MCAsmParserExtension*, StringRef, SMLoc);
@@ -84,6 +84,8 @@ class MCAsmParser {
   virtual void addDirectiveHandler(StringRef Directive,
                                    ExtensionDirectiveHandler Handler) = 0;
 
+  virtual void addAliasForDirective(StringRef Directive, StringRef Alias) = 0;
+
   virtual SourceMgr &getSourceManager() = 0;
 
   virtual MCAsmLexer &getLexer() = 0;
@@ -93,7 +95,7 @@ class MCAsmParser {
 
   virtual MCContext &getContext() = 0;
 
-  /// Return the output streamer for the assembler.
+  /// \brief Return the output streamer for the assembler.
   virtual MCStreamer &getStreamer() = 0;
 
   MCTargetAsmParser &getTargetParser() const { return *TargetParser; }
@@ -105,13 +107,13 @@ class MCAsmParser {
   bool getShowParsedOperands() const { return ShowParsedOperands; }
   void setShowParsedOperands(bool Value) { ShowParsedOperands = Value; }
 
-  /// Run the parser on the input source buffer.
+  /// \brief Run the parser on the input source buffer.
   virtual bool Run(bool NoInitialTextSection, bool NoFinalize = false) = 0;
 
   virtual void setParsingInlineAsm(bool V) = 0;
   virtual bool isParsingInlineAsm() = 0;
 
-  /// Parse ms-style inline assembly.
+  /// \brief Parse MS-style inline assembly.
   virtual bool parseMSInlineAsm(
       void *AsmLoc, std::string &AsmString, unsigned &NumOutputs,
       unsigned &NumInputs, SmallVectorImpl<std::pair<void *, bool>> &OpDecls,
@@ -119,35 +121,35 @@ class MCAsmParser {
       SmallVectorImpl<std::string> &Clobbers, const MCInstrInfo *MII,
       const MCInstPrinter *IP, MCAsmParserSemaCallback &SI) = 0;
 
-  /// Emit a note at the location \p L, with the message \p Msg.
+  /// \brief Emit a note at the location \p L, with the message \p Msg.
   virtual void Note(SMLoc L, const Twine &Msg,
                     ArrayRef<SMRange> Ranges = None) = 0;
 
-  /// Emit a warning at the location \p L, with the message \p Msg.
+  /// \brief Emit a warning at the location \p L, with the message \p Msg.
   ///
   /// \return The return value is true, if warnings are fatal.
   virtual bool Warning(SMLoc L, const Twine &Msg,
                        ArrayRef<SMRange> Ranges = None) = 0;
 
-  /// Emit an error at the location \p L, with the message \p Msg.
+  /// \brief Emit an error at the location \p L, with the message \p Msg.
   ///
   /// \return The return value is always true, as an idiomatic convenience to
   /// clients.
   virtual bool Error(SMLoc L, const Twine &Msg,
                      ArrayRef<SMRange> Ranges = None) = 0;
 
-  /// Get the next AsmToken in the stream, possibly handling file inclusion
-  /// first.
+  /// \brief Get the next AsmToken in the stream, possibly handling file
+  /// inclusion first.
   virtual const AsmToken &Lex() = 0;
 
-  /// Get the current AsmToken from the stream.
+  /// \brief Get the current AsmToken from the stream.
   const AsmToken &getTok() const;
 
   /// \brief Report an error at the current lexer location.
   bool TokError(const Twine &Msg, ArrayRef<SMRange> Ranges = None);
 
-  /// Parse an identifier or string (as a quoted identifier) and set \p Res to
-  /// the identifier contents.
+  /// \brief Parse an identifier or string (as a quoted identifier) and set \p
+  /// Res to the identifier contents.
   virtual bool parseIdentifier(StringRef &Res) = 0;
 
   /// \brief Parse up to the end of statement and return the contents from the
@@ -155,51 +157,62 @@ class MCAsmParser {
   /// will be either the EndOfStatement or EOF.
   virtual StringRef parseStringToEndOfStatement() = 0;
 
-  /// Parse the current token as a string which may include escaped characters
-  /// and return the string contents.
+  /// \brief Parse the current token as a string which may include escaped
+  /// characters and return the string contents.
   virtual bool parseEscapedString(std::string &Data) = 0;
 
-  /// Skip to the end of the current statement, for error recovery.
+  /// \brief Skip to the end of the current statement, for error recovery.
   virtual void eatToEndOfStatement() = 0;
 
-  /// Parse an arbitrary expression.
+  /// \brief Parse an arbitrary expression.
   ///
-  /// @param Res - The value of the expression. The result is undefined
+  /// \param Res - The value of the expression. The result is undefined
   /// on error.
-  /// @result - False on success.
+  /// \return - False on success.
   virtual bool parseExpression(const MCExpr *&Res, SMLoc &EndLoc) = 0;
   bool parseExpression(const MCExpr *&Res);
 
-  /// Parse a primary expression.
+  /// \brief Parse a primary expression.
   ///
-  /// @param Res - The value of the expression. The result is undefined
+  /// \param Res - The value of the expression. The result is undefined
   /// on error.
-  /// @result - False on success.
+  /// \return - False on success.
   virtual bool parsePrimaryExpr(const MCExpr *&Res, SMLoc &EndLoc) = 0;
 
-  /// Parse an arbitrary expression, assuming that an initial '(' has already
-  /// been consumed.
+  /// \brief Parse an arbitrary expression, assuming that an initial '(' has
+  /// already been consumed.
   ///
-  /// @param Res - The value of the expression. The result is undefined
+  /// \param Res - The value of the expression. The result is undefined
   /// on error.
-  /// @result - False on success.
+  /// \return - False on success.
   virtual bool parseParenExpression(const MCExpr *&Res, SMLoc &EndLoc) = 0;
 
-  /// Parse an expression which must evaluate to an absolute value.
+  /// \brief Parse an expression which must evaluate to an absolute value.
   ///
-  /// @param Res - The value of the absolute expression. The result is undefined
+  /// \param Res - The value of the absolute expression. The result is undefined
   /// on error.
-  /// @result - False on success.
+  /// \return - False on success.
   virtual bool parseAbsoluteExpression(int64_t &Res) = 0;
 
-  /// Ensure that we have a valid section set in the streamer. Otherwise, report
-  /// an error and switch to .text.
+  /// \brief Ensure that we have a valid section set in the streamer. Otherwise,
+  /// report an error and switch to .text.
   virtual void checkForValidSection() = 0;
+
+  /// \brief Parse an arbitrary expression of a specified parenthesis depth,
+  /// assuming that the initial '(' characters have already been consumed.
+  ///
+  /// \param ParenDepth - Specifies how many trailing expressions outside the
+  /// current parentheses we have to parse.
+  /// \param Res - The value of the expression. The result is undefined
+  /// on error.
+  /// \return - False on success.
+  virtual bool parseParenExprOfDepth(unsigned ParenDepth, const MCExpr *&Res,
+                                     SMLoc &EndLoc) = 0;
 };
 
 /// \brief Create an MCAsmParser instance.
-MCAsmParser *createMCAsmParser(SourceMgr &, MCContext &,
-                               MCStreamer &, const MCAsmInfo &);
+MCAsmParser *createMCAsmParser(SourceMgr &, MCContext &, MCStreamer &,
+                               const MCAsmInfo &);
 
 } // End llvm namespace
 
diff --git a/include/llvm/MC/MCParser/MCAsmParserExtension.h b/include/llvm/MC/MCParser/MCAsmParserExtension.h
index 53515d78d3c..077fd21e073 100644
--- a/include/llvm/MC/MCParser/MCAsmParserExtension.h
+++ b/include/llvm/MC/MCParser/MCAsmParserExtension.h
@@ -48,7 +48,7 @@ class MCAsmParserExtension {
   /// parsing routines.
   virtual void Initialize(MCAsmParser &Parser);
 
-  /// @name MCAsmParser Proxy Interfaces
+  /// \name MCAsmParser Proxy Interfaces
   /// @{
 
   MCContext &getContext() { return getParser().getContext(); }
diff --git a/include/llvm/MC/MCParser/MCAsmParserUtils.h b/include/llvm/MC/MCParser/MCAsmParserUtils.h
new file mode 100644
index 00000000000..9834fe96307
--- /dev/null
+++ b/include/llvm/MC/MCParser/MCAsmParserUtils.h
@@ -0,0 +1,33 @@
+//===------ llvm/MC/MCAsmParserUtils.h - Asm Parser Utilities ---*- C++ -*-===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_MC_MCPARSER_MCASMPARSERUTILS_H
+#define LLVM_MC_MCPARSER_MCASMPARSERUTILS_H
+
+namespace llvm {
+
+class MCAsmParser;
+class MCExpr;
+class MCSymbol;
+class StringRef;
+
+namespace MCParserUtils {
+
+/// Parse a value expression and return whether it can be assigned to a symbol
+/// with the given name.
+///
+/// On success, returns false and sets the Symbol and Value output parameters.
+bool parseAssignmentExpression(StringRef Name, bool allow_redef,
+                               MCAsmParser &Parser, MCSymbol *&Symbol,
+                               const MCExpr *&Value);
+
+} // namespace MCParserUtils
+} // namespace llvm
+
+#endif
diff --git a/include/llvm/MC/MCParser/MCParsedAsmOperand.h b/include/llvm/MC/MCParser/MCParsedAsmOperand.h
index e8740aa10aa..a25108a0eff 100644
--- a/include/llvm/MC/MCParser/MCParsedAsmOperand.h
+++ b/include/llvm/MC/MCParser/MCParsedAsmOperand.h
@@ -10,8 +10,11 @@
 #ifndef LLVM_MC_MCPARSER_MCPARSEDASMOPERAND_H
 #define LLVM_MC_MCPARSER_MCPARSEDASMOPERAND_H
 
+#include <string>
+#include "llvm/ADT/StringRef.h"
+#include "llvm/Support/SMLoc.h"
+
 namespace llvm {
-class SMLoc;
 class raw_ostream;
 
 /// MCParsedAsmOperand - This abstract class represents a source-level assembly
diff --git a/include/llvm/MC/MCSchedule.h b/include/llvm/MC/MCSchedule.h
index 1adfedd2638..a8b20570103 100644
--- a/include/llvm/MC/MCSchedule.h
+++ b/include/llvm/MC/MCSchedule.h
@@ -206,6 +206,9 @@ struct MCSchedModel {
   /// scheduling class (itinerary class or SchedRW list).
   bool isComplete() const { return CompleteModel; }
 
+  /// Return true if machine supports out of order execution.
+  bool isOutOfOrder() const { return MicroOpBufferSize > 1; }
+
   unsigned getNumProcResourceKinds() const {
     return NumProcResourceKinds;
   }
@@ -224,25 +227,9 @@ struct MCSchedModel {
     return &SchedClassTable[SchedClassIdx];
   }
 
-  // /\brief Returns a default initialized model. Used for unknown processors.
-  static MCSchedModel GetDefaultSchedModel() {
-    MCSchedModel Ret = { DefaultIssueWidth,
-                         DefaultMicroOpBufferSize,
-                         DefaultLoopMicroOpBufferSize,
-                         DefaultLoadLatency,
-                         DefaultHighLatency,
-                         DefaultMispredictPenalty,
-                         false,
-                         true,
-                         0,
-                         nullptr,
-                         nullptr,
-                         0,
-                         0,
-                         nullptr
-                       };
-    return Ret;
-  }
+  /// Returns the default initialized model.
+  static const MCSchedModel &GetDefaultSchedModel() { return Default; }
+  static const MCSchedModel Default;
 };
 
 } // End llvm namespace
diff --git a/include/llvm/MC/MCSection.h b/include/llvm/MC/MCSection.h
index ab8968ef2ba..2d0d4dfc591 100644
--- a/include/llvm/MC/MCSection.h
+++ b/include/llvm/MC/MCSection.h
@@ -14,33 +14,92 @@
 #ifndef LLVM_MC_MCSECTION_H
 #define LLVM_MC_MCSECTION_H
 
+#include "llvm/ADT/SmallVector.h"
 #include "llvm/ADT/StringRef.h"
+#include "llvm/ADT/ilist.h"
+#include "llvm/ADT/ilist_node.h"
 #include "llvm/MC/SectionKind.h"
 #include "llvm/Support/Compiler.h"
 
 namespace llvm {
+class MCAssembler;
 class MCAsmInfo;
 class MCContext;
 class MCExpr;
+class MCFragment;
+class MCSection;
 class MCSymbol;
 class raw_ostream;
 
+template<>
+struct ilist_node_traits<MCFragment> {
+  MCFragment *createNode(const MCFragment &V);
+  static void deleteNode(MCFragment *V);
+
+  void addNodeToList(MCFragment *) {}
+  void removeNodeFromList(MCFragment *) {}
+  void transferNodesFromList(ilist_node_traits &    /*SrcTraits*/,
+                             ilist_iterator<MCFragment> /*first*/,
+                             ilist_iterator<MCFragment> /*last*/) {}
+};
+
 /// Instances of this class represent a uniqued identifier for a section in the
 /// current translation unit.  The MCContext class uniques and creates these.
 class MCSection {
 public:
   enum SectionVariant { SV_COFF = 0, SV_ELF, SV_MachO };
 
+  /// \brief Express the state of bundle locked groups while emitting code.
+  enum BundleLockStateType {
+    NotBundleLocked,
+    BundleLocked,
+    BundleLockedAlignToEnd
+  };
+
+  typedef iplist<MCFragment> FragmentListType;
+
+  typedef FragmentListType::const_iterator const_iterator;
+  typedef FragmentListType::iterator iterator;
+
+  typedef FragmentListType::const_reverse_iterator const_reverse_iterator;
+  typedef FragmentListType::reverse_iterator reverse_iterator;
+
 private:
   MCSection(const MCSection &) = delete;
   void operator=(const MCSection &) = delete;
 
   MCSymbol *Begin;
-  mutable MCSymbol *End;
+  MCSymbol *End = nullptr;
+  /// The alignment requirement of this section.
+  unsigned Alignment = 1;
+  /// The section index in the assemblers section list.
+  unsigned Ordinal = 0;
+  /// The index of this section in the layout order.
+  unsigned LayoutOrder;
+
+  /// \brief Keeping track of bundle-locked state.
+  BundleLockStateType BundleLockState = NotBundleLocked;
+
+  /// \brief Current nesting depth of bundle_lock directives.
+  unsigned BundleLockNestingDepth = 0;
+
+  /// \brief We've seen a bundle_lock directive but not its first instruction
+  /// yet.
+  unsigned BundleGroupBeforeFirstInst : 1;
+
+  /// Whether this section has had instructions emitted into it.
+  unsigned HasInstructions : 1;
+
+  unsigned IsRegistered : 1;
+
+  FragmentListType Fragments;
+
+  /// Mapping from subsection number to insertion point for subsection numbers
+  /// below that number.
+  SmallVector<std::pair<unsigned, MCFragment *>, 1> SubsectionFragmentMap;
 
 protected:
-  MCSection(SectionVariant V, SectionKind K, MCSymbol *Begin)
-      : Begin(Begin), End(nullptr), Variant(V), Kind(K) {}
+  MCSection(SectionVariant V, SectionKind K, MCSymbol *Begin);
   SectionVariant Variant;
   SectionKind Kind;
 
@@ -51,10 +110,72 @@ class MCSection {
 
   SectionVariant getVariant() const { return Variant; }
 
-  MCSymbol *getBeginSymbol() const { return Begin; }
-  MCSymbol *getEndSymbol(MCContext &Ctx) const;
+  MCSymbol *getBeginSymbol() { return Begin; }
+  const MCSymbol *getBeginSymbol() const {
+    return const_cast<MCSection *>(this)->getBeginSymbol();
+  }
+  void setBeginSymbol(MCSymbol *Sym) {
+    assert(!Begin);
+    Begin = Sym;
+  }
+  MCSymbol *getEndSymbol(MCContext &Ctx);
   bool hasEnded() const;
 
+  unsigned getAlignment() const { return Alignment; }
+  void setAlignment(unsigned Value) { Alignment = Value; }
+
+  unsigned getOrdinal() const { return Ordinal; }
+  void setOrdinal(unsigned Value) { Ordinal = Value; }
+
+  unsigned getLayoutOrder() const { return LayoutOrder; }
+  void setLayoutOrder(unsigned Value) { LayoutOrder = Value; }
+
+  BundleLockStateType getBundleLockState() const { return BundleLockState; }
+  void setBundleLockState(BundleLockStateType NewState);
+  bool isBundleLocked() const { return BundleLockState != NotBundleLocked; }
+
+  bool isBundleGroupBeforeFirstInst() const {
+    return BundleGroupBeforeFirstInst;
+  }
+  void setBundleGroupBeforeFirstInst(bool IsFirst) {
+    BundleGroupBeforeFirstInst = IsFirst;
+  }
+
+  bool hasInstructions() const { return HasInstructions; }
+  void setHasInstructions(bool Value) { HasInstructions = Value; }
+
+  bool isRegistered() const { return IsRegistered; }
+  void setIsRegistered(bool Value) { IsRegistered = Value; }
+
+  MCSection::FragmentListType &getFragmentList() { return Fragments; }
+  const MCSection::FragmentListType &getFragmentList() const {
+    return const_cast<MCSection *>(this)->getFragmentList();
+  }
+
+  MCSection::iterator begin();
+  MCSection::const_iterator begin() const {
+    return const_cast<MCSection *>(this)->begin();
+  }
+
+  MCSection::iterator end();
+  MCSection::const_iterator end() const {
+    return const_cast<MCSection *>(this)->end();
+  }
+
+  MCSection::reverse_iterator rbegin();
+  MCSection::const_reverse_iterator rbegin() const {
+    return const_cast<MCSection *>(this)->rbegin();
+  }
+
+  MCSection::reverse_iterator rend();
+  MCSection::const_reverse_iterator rend() const {
+    return const_cast<MCSection *>(this)->rend();
+  }
+
+  MCSection::iterator getSubsectionInsertionPoint(unsigned Subsection);
+
+  void dump();
+
   virtual void PrintSwitchToSection(const MCAsmInfo &MAI, raw_ostream &OS,
                                     const MCExpr *Subsection) const = 0;
 
diff --git a/include/llvm/MC/MCSectionELF.h b/include/llvm/MC/MCSectionELF.h
index 9efe1022f29..f6730371fe1 100644
--- a/include/llvm/MC/MCSectionELF.h
+++ b/include/llvm/MC/MCSectionELF.h
@@ -16,7 +16,7 @@
 
 #include "llvm/ADT/Twine.h"
 #include "llvm/MC/MCSection.h"
-#include "llvm/MC/MCSymbol.h"
+#include "llvm/MC/MCSymbolELF.h"
 #include "llvm/Support/Debug.h"
 #include "llvm/Support/ELF.h"
 #include "llvm/Support/raw_ostream.h"
@@ -46,7 +46,7 @@ class MCSectionELF : public MCSection {
   /// section does not contain fixed-sized entries 'EntrySize' will be 0.
   unsigned EntrySize;
 
-  const MCSymbol *Group;
+  const MCSymbolELF *Group;
 
   /// Depending on the type of the section this is sh_link or sh_info.
   const MCSectionELF *Associated;
@@ -54,11 +54,14 @@ class MCSectionELF : public MCSection {
 private:
   friend class MCContext;
   MCSectionELF(StringRef Section, unsigned type, unsigned flags, SectionKind K,
-               unsigned entrySize, const MCSymbol *group, unsigned UniqueID,
+               unsigned entrySize, const MCSymbolELF *group, unsigned UniqueID,
                MCSymbol *Begin, const MCSectionELF *Associated)
       : MCSection(SV_ELF, K, Begin), SectionName(Section), Type(type),
         Flags(flags), UniqueID(UniqueID), EntrySize(entrySize), Group(group),
-        Associated(Associated) {}
+        Associated(Associated) {
+    if (Group)
+      Group->setIsSignature();
+  }
   ~MCSectionELF() override;
 
   void setSectionName(StringRef Name) { SectionName = Name; }
@@ -73,7 +76,7 @@ class MCSectionELF : public MCSection {
   unsigned getType() const { return Type; }
   unsigned getFlags() const { return Flags; }
   unsigned getEntrySize() const { return EntrySize; }
-  const MCSymbol *getGroup() const { return Group; }
+  const MCSymbolELF *getGroup() const { return Group; }
 
   void PrintSwitchToSection(const MCAsmInfo &MAI, raw_ostream &OS,
                             const MCExpr *Subsection) const override;
diff --git a/include/llvm/MC/MCStreamer.h b/include/llvm/MC/MCStreamer.h
index 12cee2a6bfb..63e30bc3d54 100644
--- a/include/llvm/MC/MCStreamer.h
+++ b/include/llvm/MC/MCStreamer.h
@@ -16,13 +16,12 @@
 
 #include "llvm/ADT/ArrayRef.h"
 #include "llvm/ADT/SmallVector.h"
-#include "llvm/MC/MCAssembler.h"
 #include "llvm/MC/MCDirectives.h"
 #include "llvm/MC/MCDwarf.h"
 #include "llvm/MC/MCLinkerOptimizationHint.h"
 #include "llvm/MC/MCWinEH.h"
-#include "llvm/MC/MCNaClExpander.h" // @LOCALMOD
 #include "llvm/Support/DataTypes.h"
+#include "llvm/Support/SMLoc.h"
 #include <string>
 
 namespace llvm {
@@ -35,6 +34,7 @@ class MCInstPrinter;
 class MCSection;
 class MCStreamer;
 class MCSymbol;
+class MCSymbolELF;
 class MCSymbolRefExpr;
 class MCSubtargetInfo;
 class StringRef;
@@ -43,7 +43,7 @@ class raw_ostream;
 class formatted_raw_ostream;
 class AssemblerConstantPools;
 
-typedef std::pair<const MCSection *, const MCExpr *> MCSectionSubPair;
+typedef std::pair<MCSection *, const MCExpr *> MCSectionSubPair;
 
 /// Target specific streamer interface. This is used so that targets can
 /// implement support for target specific assembly directives.
@@ -78,37 +78,17 @@ class MCTargetStreamer {
   MCTargetStreamer(MCStreamer &S);
   virtual ~MCTargetStreamer();
 
-  const MCStreamer &getStreamer() { return Streamer; }
+  MCStreamer &getStreamer() { return Streamer; }
 
   // Allow a target to add behavior to the EmitLabel of MCStreamer.
   virtual void emitLabel(MCSymbol *Symbol);
   // Allow a target to add behavior to the emitAssignment of MCStreamer.
   virtual void emitAssignment(MCSymbol *Symbol, const MCExpr *Value);
 
-  virtual void finish();
-};
-
-class AArch64TargetStreamer : public MCTargetStreamer {
-public:
-  AArch64TargetStreamer(MCStreamer &S);
-  ~AArch64TargetStreamer() override;
-
-  void finish() override;
-
-  /// Callback used to implement the ldr= pseudo.
-  /// Add a new entry to the constant pool for the current section and return an
-  /// MCExpr that can be used to refer to the constant pool location.
-  const MCExpr *addConstantPoolEntry(const MCExpr *, unsigned Size);
-
-  /// Callback used to implemnt the .ltorg directive.
-  /// Emit contents of constant pool for the current section.
-  void emitCurrentConstantPool();
-
-  /// Callback used to implement the .inst directive.
-  virtual void emitInst(uint32_t Inst);
+  virtual void prettyPrintAsm(MCInstPrinter &InstPrinter, raw_ostream &OS,
+                              const MCInst &Inst, const MCSubtargetInfo &STI);
 
-private:
-  std::unique_ptr<AssemblerConstantPools> ConstantPools;
+  virtual void finish();
 };
 
 // FIXME: declared here because it is used from
@@ -164,11 +144,12 @@ class ARMTargetStreamer : public MCTargetStreamer {
   std::unique_ptr<AssemblerConstantPools> ConstantPools;
 };
 
-/// MCStreamer - Streaming machine code generation interface.  This interface
-/// is intended to provide a programatic interface that is very similar to the
-/// level that an assembler .s file provides.  It has callbacks to emit bytes,
-/// handle directives, etc.  The implementation of this interface retains
-/// state to know what the current section is etc.
+/// \brief Streaming machine code generation interface.
+///
+/// This interface is intended to provide a programatic interface that is very
+/// similar to the level that an assembler .s file provides.  It has callbacks
+/// to emit bytes, handle directives, etc.  The implementation of this interface
+/// retains state to know what the current section is etc.
 ///
 /// There are multiple implementations of this interface: one for writing out
 /// a .s file, and implementations that write out .o files of various formats.
@@ -190,12 +171,12 @@ class MCStreamer {
   WinEH::FrameInfo *CurrentWinFrameInfo;
   void EnsureValidWinFrameInfo();
 
-  // SymbolOrdering - Tracks an index to represent the order
-  // a symbol was emitted in. Zero means we did not emit that symbol.
+  /// \brief Tracks an index to represent the order a symbol was emitted in.
+  /// Zero means we did not emit that symbol.
   DenseMap<const MCSymbol *, unsigned> SymbolOrdering;
 
-  /// SectionStack - This is stack of current and previous section
-  /// values saved by PushSection.
+  /// \brief This is stack of current and previous section values saved by
+  /// PushSection.
   SmallVector<std::pair<MCSectionSubPair, MCSectionSubPair>, 4> SectionStack;
 
 protected:
@@ -211,7 +192,6 @@ class MCStreamer {
   virtual void EmitWindowsUnwindTables();
 
   virtual void EmitRawTextImpl(StringRef String);
-  std::unique_ptr<MCNaClExpander> NaClExpander; //@LOCALMOD
 
 public:
   virtual ~MCStreamer();
@@ -223,11 +203,6 @@ class MCStreamer {
     TargetStreamer.reset(TS);
   }
 
-  // @LOCALMOD-START
-  void setNaClExpander(MCNaClExpander *Exp) { NaClExpander.reset(Exp); }
-
-  MCNaClExpander *getNaClExpander() { return NaClExpander.get(); }
-  // @LOCALMOD-END
   /// State management
   ///
   virtual void reset();
@@ -250,22 +225,24 @@ class MCStreamer {
 
   void generateCompactUnwindEncodings(MCAsmBackend *MAB);
 
-  /// @name Assembly File Formatting.
+  /// \name Assembly File Formatting.
   /// @{
 
-  /// isVerboseAsm - Return true if this streamer supports verbose assembly
-  /// and if it is enabled.
+  /// \brief Return true if this streamer supports verbose assembly and if it is
+  /// enabled.
   virtual bool isVerboseAsm() const { return false; }
 
-  /// hasRawTextSupport - Return true if this asm streamer supports emitting
-  /// unformatted text to the .s file with EmitRawText.
+  /// \brief Return true if this asm streamer supports emitting unformatted text
+  /// to the .s file with EmitRawText.
   virtual bool hasRawTextSupport() const { return false; }
 
-  /// Is the integrated assembler required for this streamer to function
+  /// \brief Is the integrated assembler required for this streamer to function
   /// correctly?
   virtual bool isIntegratedAssemblerRequired() const { return false; }
 
-  /// AddComment - Add a comment that can be emitted to the generated .s
+  /// \brief Add a textual command.
+  ///
+  /// Typically for comments that can be emitted to the generated .s
   /// file if applicable as a QoI issue to make the output of the compiler
   /// more readable.  This only affects the MCAsmStreamer, and only when
   /// verbose assembly output is enabled.
@@ -274,14 +251,14 @@ class MCStreamer {
   /// prefix as appropriate.  The added comment should not end with a \n.
   virtual void AddComment(const Twine &T) {}
 
-  /// GetCommentOS - Return a raw_ostream that comments can be written to.
-  /// Unlike AddComment, you are required to terminate comments with \n if you
-  /// use this method.
+  /// \brief Return a raw_ostream that comments can be written to. Unlike
+  /// AddComment, you are required to terminate comments with \n if you use this
+  /// method.
   virtual raw_ostream &GetCommentOS();
 
-  /// Print T and prefix it with the comment string (normally #) and optionally
-  /// a tab. This prints the comment immediately, not at the end of the
-  /// current line. It is basically a safe version of EmitRawText: since it
+  /// \brief Print T and prefix it with the comment string (normally #) and
+  /// optionally a tab. This prints the comment immediately, not at the end of
+  /// the current line. It is basically a safe version of EmitRawText: since it
   /// only prints comments, the object streamer ignores it instead of asserting.
   virtual void emitRawComment(const Twine &T, bool TabPrefix = true);
 
@@ -290,56 +267,58 @@ class MCStreamer {
 
   /// @}
 
-  /// @name Symbol & Section Management
+  /// \name Symbol & Section Management
   /// @{
 
-  /// getCurrentSection - Return the current section that the streamer is
-  /// emitting code to.
+  /// \brief Return the current section that the streamer is emitting code to.
   MCSectionSubPair getCurrentSection() const {
     if (!SectionStack.empty())
       return SectionStack.back().first;
     return MCSectionSubPair();
   }
+  MCSection *getCurrentSectionOnly() const { return getCurrentSection().first; }
 
-  /// getPreviousSection - Return the previous section that the streamer is
-  /// emitting code to.
+  /// \brief Return the previous section that the streamer is emitting code to.
   MCSectionSubPair getPreviousSection() const {
     if (!SectionStack.empty())
       return SectionStack.back().second;
     return MCSectionSubPair();
   }
 
-  /// GetSymbolOrder - Returns an index to represent the order
-  /// a symbol was emitted in. (zero if we did not emit that symbol)
+  /// \brief Returns an index to represent the order a symbol was emitted in.
+  /// (zero if we did not emit that symbol)
   unsigned GetSymbolOrder(const MCSymbol *Sym) const {
     return SymbolOrdering.lookup(Sym);
   }
 
-  /// ChangeSection - Update streamer for a new active section.
+  /// \brief Update streamer for a new active section.
   ///
   /// This is called by PopSection and SwitchSection, if the current
   /// section changes.
-  virtual void ChangeSection(const MCSection *, const MCExpr *);
+  virtual void ChangeSection(MCSection *, const MCExpr *);
 
-  /// pushSection - Save the current and previous section on the
-  /// section stack.
+  /// \brief Save the current and previous section on the section stack.
   void PushSection() {
     SectionStack.push_back(
         std::make_pair(getCurrentSection(), getPreviousSection()));
   }
 
-  /// popSection - Restore the current and previous section from
-  /// the section stack.  Calls ChangeSection as needed.
+  /// \brief Restore the current and previous section from the section stack.
+  /// Calls ChangeSection as needed.
   ///
   /// Returns false if the stack was empty.
   bool PopSection() {
     if (SectionStack.size() <= 1)
       return false;
-    MCSectionSubPair oldSection = SectionStack.pop_back_val().first;
-    MCSectionSubPair curSection = SectionStack.back().first;
-
-    if (oldSection != curSection)
-      ChangeSection(curSection.first, curSection.second);
+    auto I = SectionStack.end();
+    --I;
+    MCSectionSubPair OldSection = I->first;
+    --I;
+    MCSectionSubPair NewSection = I->first;
+
+    if (OldSection != NewSection)
+      ChangeSection(NewSection.first, NewSection.second);
+    SectionStack.pop_back();
     return true;
   }
 
@@ -351,17 +330,17 @@ class MCStreamer {
     return true;
   }
 
-  /// Set the current section where code is being emitted to @p Section.  This
+  /// Set the current section where code is being emitted to \p Section.  This
   /// is required to update CurSection.
   ///
   /// This corresponds to assembler directives like .section, .text, etc.
-  virtual void SwitchSection(const MCSection *Section,
+  virtual void SwitchSection(MCSection *Section,
                              const MCExpr *Subsection = nullptr);
 
-  /// SwitchSectionNoChange - Set the current section where code is being
-  /// emitted to @p Section.  This is required to update CurSection. This
-  /// version does not call ChangeSection.
-  void SwitchSectionNoChange(const MCSection *Section,
+  /// \brief Set the current section where code is being emitted to \p Section.
+  /// This is required to update CurSection. This version does not call
+  /// ChangeSection.
+  void SwitchSectionNoChange(MCSection *Section,
                              const MCExpr *Subsection = nullptr) {
     assert(Section && "Cannot switch to a null section!");
     MCSectionSubPair curSection = SectionStack.back().first;
@@ -370,23 +349,23 @@ class MCStreamer {
       SectionStack.back().first = MCSectionSubPair(Section, Subsection);
   }
 
-  /// Create the default sections and set the initial one.
+  /// \brief Create the default sections and set the initial one.
   virtual void InitSections(bool NoExecStack);
 
-  MCSymbol *endSection(const MCSection *Section);
+  MCSymbol *endSection(MCSection *Section);
 
-  /// AssignSection - Sets the symbol's section.
+  /// \brief Sets the symbol's section.
   ///
   /// Each emitted symbol will be tracked in the ordering table,
   /// so we can sort on them later.
-  void AssignSection(MCSymbol *Symbol, const MCSection *Section);
+  void AssignSection(MCSymbol *Symbol, MCSection *Section);
 
-  /// EmitLabel - Emit a label for @p Symbol into the current section.
+  /// \brief Emit a label for \p Symbol into the current section.
   ///
   /// This corresponds to an assembler statement such as:
   ///   foo:
   ///
-  /// @param Symbol - The symbol to emit. A given symbol should only be
+  /// \param Symbol - The symbol to emit. A given symbol should only be
   /// emitted as a label once, and symbols emitted as a label should never be
   /// used in an assignment.
   // FIXME: These emission are non-const because we mutate the symbol to
@@ -395,25 +374,25 @@ class MCStreamer {
 
   virtual void EmitEHSymAttributes(const MCSymbol *Symbol, MCSymbol *EHSymbol);
 
-  /// EmitAssemblerFlag - Note in the output the specified @p Flag.
+  /// \brief Note in the output the specified \p Flag.
   virtual void EmitAssemblerFlag(MCAssemblerFlag Flag);
 
-  /// EmitLinkerOptions - Emit the given list @p Options of strings as linker
+  /// \brief Emit the given list \p Options of strings as linker
   /// options into the output.
   virtual void EmitLinkerOptions(ArrayRef<std::string> Kind) {}
 
-  /// EmitDataRegion - Note in the output the specified region @p Kind.
+  /// \brief Note in the output the specified region \p Kind.
   virtual void EmitDataRegion(MCDataRegionType Kind) {}
 
-  /// EmitVersionMin - Specify the MachO minimum deployment target version.
+  /// \brief Specify the MachO minimum deployment target version.
   virtual void EmitVersionMin(MCVersionMinType, unsigned Major, unsigned Minor,
                               unsigned Update) {}
 
-  /// EmitThumbFunc - Note in the output that the specified @p Func is
-  /// a Thumb mode function (ARM target only).
+  /// \brief Note in the output that the specified \p Func is a Thumb mode
+  /// function (ARM target only).
   virtual void EmitThumbFunc(MCSymbol *Func);
 
-  /// EmitAssignment - Emit an assignment of @p Value to @p Symbol.
+  /// \brief Emit an assignment of \p Value to \p Symbol.
   ///
   /// This corresponds to an assembler statement such as:
   ///  symbol = value
@@ -422,133 +401,133 @@ class MCStreamer {
   /// value in the current context. For the assembly streamer, this prints the
   /// binding into the .s file.
   ///
-  /// @param Symbol - The symbol being assigned to.
-  /// @param Value - The value for the symbol.
+  /// \param Symbol - The symbol being assigned to.
+  /// \param Value - The value for the symbol.
   virtual void EmitAssignment(MCSymbol *Symbol, const MCExpr *Value);
 
-  /// EmitWeakReference - Emit an weak reference from @p Alias to @p Symbol.
+  /// \brief Emit an weak reference from \p Alias to \p Symbol.
   ///
   /// This corresponds to an assembler statement such as:
   ///  .weakref alias, symbol
   ///
-  /// @param Alias - The alias that is being created.
-  /// @param Symbol - The symbol being aliased.
+  /// \param Alias - The alias that is being created.
+  /// \param Symbol - The symbol being aliased.
   virtual void EmitWeakReference(MCSymbol *Alias, const MCSymbol *Symbol);
 
-  /// EmitSymbolAttribute - Add the given @p Attribute to @p Symbol.
+  /// \brief Add the given \p Attribute to \p Symbol.
   virtual bool EmitSymbolAttribute(MCSymbol *Symbol,
                                    MCSymbolAttr Attribute) = 0;
 
-  /// EmitSymbolDesc - Set the @p DescValue for the @p Symbol.
+  /// \brief Set the \p DescValue for the \p Symbol.
   ///
-  /// @param Symbol - The symbol to have its n_desc field set.
-  /// @param DescValue - The value to set into the n_desc field.
+  /// \param Symbol - The symbol to have its n_desc field set.
+  /// \param DescValue - The value to set into the n_desc field.
   virtual void EmitSymbolDesc(MCSymbol *Symbol, unsigned DescValue);
 
-  /// BeginCOFFSymbolDef - Start emitting COFF symbol definition
+  /// \brief Start emitting COFF symbol definition
   ///
-  /// @param Symbol - The symbol to have its External & Type fields set.
+  /// \param Symbol - The symbol to have its External & Type fields set.
   virtual void BeginCOFFSymbolDef(const MCSymbol *Symbol);
 
-  /// EmitCOFFSymbolStorageClass - Emit the storage class of the symbol.
+  /// \brief Emit the storage class of the symbol.
   ///
-  /// @param StorageClass - The storage class the symbol should have.
+  /// \param StorageClass - The storage class the symbol should have.
   virtual void EmitCOFFSymbolStorageClass(int StorageClass);
 
-  /// EmitCOFFSymbolType - Emit the type of the symbol.
+  /// \brief Emit the type of the symbol.
   ///
-  /// @param Type - A COFF type identifier (see COFF::SymbolType in X86COFF.h)
+  /// \param Type - A COFF type identifier (see COFF::SymbolType in X86COFF.h)
   virtual void EmitCOFFSymbolType(int Type);
 
-  /// EndCOFFSymbolDef - Marks the end of the symbol definition.
+  /// \brief Marks the end of the symbol definition.
   virtual void EndCOFFSymbolDef();
 
-  /// EmitCOFFSectionIndex - Emits a COFF section index.
+  virtual void EmitCOFFSafeSEH(MCSymbol const *Symbol);
+
+  /// \brief Emits a COFF section index.
   ///
-  /// @param Symbol - Symbol the section number relocation should point to.
+  /// \param Symbol - Symbol the section number relocation should point to.
   virtual void EmitCOFFSectionIndex(MCSymbol const *Symbol);
 
-  /// EmitCOFFSecRel32 - Emits a COFF section relative relocation.
+  /// \brief Emits a COFF section relative relocation.
   ///
-  /// @param Symbol - Symbol the section relative relocation should point to.
+  /// \param Symbol - Symbol the section relative relocation should point to.
   virtual void EmitCOFFSecRel32(MCSymbol const *Symbol);
 
-  /// EmitELFSize - Emit an ELF .size directive.
+  /// \brief Emit an ELF .size directive.
   ///
   /// This corresponds to an assembler statement such as:
   ///  .size symbol, expression
-  ///
-  virtual void EmitELFSize(MCSymbol *Symbol, const MCExpr *Value);
+  virtual void emitELFSize(MCSymbolELF *Symbol, const MCExpr *Value);
 
   /// \brief Emit a Linker Optimization Hint (LOH) directive.
   /// \param Args - Arguments of the LOH.
   virtual void EmitLOHDirective(MCLOHType Kind, const MCLOHArgs &Args) {}
 
-  /// EmitCommonSymbol - Emit a common symbol.
+  /// \brief Emit a common symbol.
   ///
-  /// @param Symbol - The common symbol to emit.
-  /// @param Size - The size of the common symbol.
-  /// @param ByteAlignment - The alignment of the symbol if
+  /// \param Symbol - The common symbol to emit.
+  /// \param Size - The size of the common symbol.
+  /// \param ByteAlignment - The alignment of the symbol if
   /// non-zero. This must be a power of 2.
   virtual void EmitCommonSymbol(MCSymbol *Symbol, uint64_t Size,
                                 unsigned ByteAlignment) = 0;
 
-  /// EmitLocalCommonSymbol - Emit a local common (.lcomm) symbol.
+  /// \brief Emit a local common (.lcomm) symbol.
   ///
-  /// @param Symbol - The common symbol to emit.
-  /// @param Size - The size of the common symbol.
-  /// @param ByteAlignment - The alignment of the common symbol in bytes.
+  /// \param Symbol - The common symbol to emit.
+  /// \param Size - The size of the common symbol.
+  /// \param ByteAlignment - The alignment of the common symbol in bytes.
   virtual void EmitLocalCommonSymbol(MCSymbol *Symbol, uint64_t Size,
                                      unsigned ByteAlignment);
 
-  /// EmitZerofill - Emit the zerofill section and an optional symbol.
+  /// \brief Emit the zerofill section and an optional symbol.
   ///
-  /// @param Section - The zerofill section to create and or to put the symbol
-  /// @param Symbol - The zerofill symbol to emit, if non-NULL.
-  /// @param Size - The size of the zerofill symbol.
-  /// @param ByteAlignment - The alignment of the zerofill symbol if
+  /// \param Section - The zerofill section to create and or to put the symbol
+  /// \param Symbol - The zerofill symbol to emit, if non-NULL.
+  /// \param Size - The size of the zerofill symbol.
+  /// \param ByteAlignment - The alignment of the zerofill symbol if
   /// non-zero. This must be a power of 2 on some targets.
-  virtual void EmitZerofill(const MCSection *Section,
-                            MCSymbol *Symbol = nullptr, uint64_t Size = 0,
-                            unsigned ByteAlignment = 0) = 0;
+  virtual void EmitZerofill(MCSection *Section, MCSymbol *Symbol = nullptr,
+                            uint64_t Size = 0, unsigned ByteAlignment = 0) = 0;
 
-  /// EmitTBSSSymbol - Emit a thread local bss (.tbss) symbol.
+  /// \brief Emit a thread local bss (.tbss) symbol.
   ///
-  /// @param Section - The thread local common section.
-  /// @param Symbol - The thread local common symbol to emit.
-  /// @param Size - The size of the symbol.
-  /// @param ByteAlignment - The alignment of the thread local common symbol
+  /// \param Section - The thread local common section.
+  /// \param Symbol - The thread local common symbol to emit.
+  /// \param Size - The size of the symbol.
+  /// \param ByteAlignment - The alignment of the thread local common symbol
   /// if non-zero.  This must be a power of 2 on some targets.
-  virtual void EmitTBSSSymbol(const MCSection *Section, MCSymbol *Symbol,
+  virtual void EmitTBSSSymbol(MCSection *Section, MCSymbol *Symbol,
                               uint64_t Size, unsigned ByteAlignment = 0);
 
   /// @}
-  /// @name Generating Data
+  /// \name Generating Data
   /// @{
 
-  /// EmitBytes - Emit the bytes in \p Data into the output.
+  /// \brief Emit the bytes in \p Data into the output.
   ///
   /// This is used to implement assembler directives such as .byte, .ascii,
   /// etc.
   virtual void EmitBytes(StringRef Data);
 
-  /// EmitValue - Emit the expression @p Value into the output as a native
-  /// integer of the given @p Size bytes.
+  /// \brief Emit the expression \p Value into the output as a native
+  /// integer of the given \p Size bytes.
   ///
   /// This is used to implement assembler directives such as .word, .quad,
   /// etc.
   ///
-  /// @param Value - The value to emit.
-  /// @param Size - The size of the integer (in bytes) to emit. This must
+  /// \param Value - The value to emit.
+  /// \param Size - The size of the integer (in bytes) to emit. This must
   /// match a native machine width.
-  /// @param Loc - The location of the expression for error reporting.
+  /// \param Loc - The location of the expression for error reporting.
   virtual void EmitValueImpl(const MCExpr *Value, unsigned Size,
                              const SMLoc &Loc = SMLoc());
 
   void EmitValue(const MCExpr *Value, unsigned Size,
                  const SMLoc &Loc = SMLoc());
 
-  /// EmitIntValue - Special case of EmitValue that avoids the client having
+  /// \brief Special case of EmitValue that avoids the client having
   /// to pass in a MCExpr for constant integers.
   virtual void EmitIntValue(uint64_t Value, unsigned Size);
 
@@ -556,112 +535,116 @@ class MCStreamer {
 
   virtual void EmitSLEB128Value(const MCExpr *Value);
 
-  /// EmitULEB128Value - Special case of EmitULEB128Value that avoids the
-  /// client having to pass in a MCExpr for constant integers.
+  /// \brief Special case of EmitULEB128Value that avoids the client having to
+  /// pass in a MCExpr for constant integers.
   void EmitULEB128IntValue(uint64_t Value, unsigned Padding = 0);
 
-  /// EmitSLEB128Value - Special case of EmitSLEB128Value that avoids the
-  /// client having to pass in a MCExpr for constant integers.
+  /// \brief Special case of EmitSLEB128Value that avoids the client having to
+  /// pass in a MCExpr for constant integers.
   void EmitSLEB128IntValue(int64_t Value);
 
-  /// EmitSymbolValue - Special case of EmitValue that avoids the client
-  /// having to pass in a MCExpr for MCSymbols.
+  /// \brief Special case of EmitValue that avoids the client having to pass in
+  /// a MCExpr for MCSymbols.
   void EmitSymbolValue(const MCSymbol *Sym, unsigned Size,
                        bool IsSectionRelative = false);
 
-  /// EmitGPRel64Value - Emit the expression @p Value into the output as a
-  /// gprel64 (64-bit GP relative) value.
+  /// \brief Emit the expression \p Value into the output as a gprel64 (64-bit
+  /// GP relative) value.
   ///
   /// This is used to implement assembler directives such as .gpdword on
   /// targets that support them.
   virtual void EmitGPRel64Value(const MCExpr *Value);
 
-  /// EmitGPRel32Value - Emit the expression @p Value into the output as a
-  /// gprel32 (32-bit GP relative) value.
+  /// \brief Emit the expression \p Value into the output as a gprel32 (32-bit
+  /// GP relative) value.
   ///
   /// This is used to implement assembler directives such as .gprel32 on
   /// targets that support them.
   virtual void EmitGPRel32Value(const MCExpr *Value);
 
-  /// EmitFill - Emit NumBytes bytes worth of the value specified by
-  /// FillValue.  This implements directives such as '.space'.
+  /// \brief Emit NumBytes bytes worth of the value specified by FillValue.
+  /// This implements directives such as '.space'.
   virtual void EmitFill(uint64_t NumBytes, uint8_t FillValue);
 
   /// \brief Emit NumBytes worth of zeros.
   /// This function properly handles data in virtual sections.
   virtual void EmitZeros(uint64_t NumBytes);
 
-  /// EmitValueToAlignment - Emit some number of copies of @p Value until
-  /// the byte alignment @p ByteAlignment is reached.
+  /// \brief Emit some number of copies of \p Value until the byte alignment \p
+  /// ByteAlignment is reached.
   ///
   /// If the number of bytes need to emit for the alignment is not a multiple
-  /// of @p ValueSize, then the contents of the emitted fill bytes is
+  /// of \p ValueSize, then the contents of the emitted fill bytes is
   /// undefined.
   ///
   /// This used to implement the .align assembler directive.
   ///
-  /// @param ByteAlignment - The alignment to reach. This must be a power of
+  /// \param ByteAlignment - The alignment to reach. This must be a power of
   /// two on some targets.
-  /// @param Value - The value to use when filling bytes.
-  /// @param ValueSize - The size of the integer (in bytes) to emit for
-  /// @p Value. This must match a native machine width.
-  /// @param MaxBytesToEmit - The maximum numbers of bytes to emit, or 0. If
+  /// \param Value - The value to use when filling bytes.
+  /// \param ValueSize - The size of the integer (in bytes) to emit for
+  /// \p Value. This must match a native machine width.
+  /// \param MaxBytesToEmit - The maximum numbers of bytes to emit, or 0. If
   /// the alignment cannot be reached in this many bytes, no bytes are
   /// emitted.
   virtual void EmitValueToAlignment(unsigned ByteAlignment, int64_t Value = 0,
                                     unsigned ValueSize = 1,
                                     unsigned MaxBytesToEmit = 0);
 
-  /// EmitCodeAlignment - Emit nops until the byte alignment @p ByteAlignment
-  /// is reached.
+  /// \brief Emit nops until the byte alignment \p ByteAlignment is reached.
   ///
   /// This used to align code where the alignment bytes may be executed.  This
   /// can emit different bytes for different sizes to optimize execution.
   ///
-  /// @param ByteAlignment - The alignment to reach. This must be a power of
+  /// \param ByteAlignment - The alignment to reach. This must be a power of
   /// two on some targets.
-  /// @param MaxBytesToEmit - The maximum numbers of bytes to emit, or 0. If
+  /// \param MaxBytesToEmit - The maximum numbers of bytes to emit, or 0. If
   /// the alignment cannot be reached in this many bytes, no bytes are
   /// emitted.
   virtual void EmitCodeAlignment(unsigned ByteAlignment,
                                  unsigned MaxBytesToEmit = 0);
 
-  /// EmitValueToOffset - Emit some number of copies of @p Value until the
-  /// byte offset @p Offset is reached.
+  /// \brief Emit some number of copies of \p Value until the byte offset \p
+  /// Offset is reached.
   ///
   /// This is used to implement assembler directives such as .org.
   ///
-  /// @param Offset - The offset to reach. This may be an expression, but the
+  /// \param Offset - The offset to reach. This may be an expression, but the
   /// expression must be associated with the current section.
-  /// @param Value - The value to use when filling bytes.
-  /// @return false on success, true if the offset was invalid.
+  /// \param Value - The value to use when filling bytes.
+  /// \return false on success, true if the offset was invalid.
   virtual bool EmitValueToOffset(const MCExpr *Offset,
                                  unsigned char Value = 0);
 
   /// @}
 
-  /// EmitFileDirective - Switch to a new logical file.  This is used to
-  /// implement the '.file "foo.c"' assembler directive.
+  /// \brief Switch to a new logical file.  This is used to implement the '.file
+  /// "foo.c"' assembler directive.
   virtual void EmitFileDirective(StringRef Filename);
 
-  /// Emit the "identifiers" directive.  This implements the
+  /// \brief Emit the "identifiers" directive.  This implements the
   /// '.ident "version foo"' assembler directive.
   virtual void EmitIdent(StringRef IdentString) {}
 
-  /// EmitDwarfFileDirective - Associate a filename with a specified logical
-  /// file number.  This implements the DWARF2 '.file 4 "foo.c"' assembler
-  /// directive.
+  /// \brief Associate a filename with a specified logical file number.  This
+  /// implements the DWARF2 '.file 4 "foo.c"' assembler directive.
   virtual unsigned EmitDwarfFileDirective(unsigned FileNo, StringRef Directory,
                                           StringRef Filename,
                                           unsigned CUID = 0);
 
-  /// EmitDwarfLocDirective - This implements the DWARF2
-  // '.loc fileno lineno ...' assembler directive.
+  /// \brief This implements the DWARF2 '.loc fileno lineno ...' assembler
+  /// directive.
   virtual void EmitDwarfLocDirective(unsigned FileNo, unsigned Line,
                                      unsigned Column, unsigned Flags,
                                      unsigned Isa, unsigned Discriminator,
                                      StringRef FileName);
 
+  /// Emit the absolute difference between two symbols.
+  ///
+  /// \pre Offset of \c Hi is greater than the offset \c Lo.
+  virtual void emitAbsoluteSymbolDiff(const MCSymbol *Hi, const MCSymbol *Lo,
+                                      unsigned Size);
+
   virtual MCSymbol *getDwarfLineTableSymbol(unsigned CUID);
   virtual void EmitCFISections(bool EH, bool Debug);
   void EmitCFIStartProc(bool IsSimple);
@@ -699,8 +682,9 @@ class MCStreamer {
   virtual void EmitWinEHHandler(const MCSymbol *Sym, bool Unwind, bool Except);
   virtual void EmitWinEHHandlerData();
 
-  /// EmitInstruction - Emit the given @p Instruction into the current
-  /// section.
+  virtual void EmitSyntaxDirective();
+
+  /// \brief Emit the given \p Instruction into the current section.
   virtual void EmitInstruction(const MCInst &Inst, const MCSubtargetInfo &STI);
 
   /// \brief Set the bundle alignment mode from now on in the section.
@@ -717,20 +701,20 @@ class MCStreamer {
   /// \brief Ends a bundle-locked group.
   virtual void EmitBundleUnlock();
 
-  /// EmitRawText - If this file is backed by a assembly streamer, this dumps
-  /// the specified string in the output .s file.  This capability is
-  /// indicated by the hasRawTextSupport() predicate.  By default this aborts.
+  /// \brief If this file is backed by a assembly streamer, this dumps the
+  /// specified string in the output .s file.  This capability is indicated by
+  /// the hasRawTextSupport() predicate.  By default this aborts.
   void EmitRawText(const Twine &String);
 
-  /// Flush - Causes any cached state to be written out.
+  /// \brief Causes any cached state to be written out.
   virtual void Flush() {}
 
-  /// FinishImpl - Streamer specific finalization.
+  /// \brief Streamer specific finalization.
   virtual void FinishImpl();
-  /// Finish - Finish emission of machine code.
+  /// \brief Finish emission of machine code.
   void Finish();
 
-  virtual bool mayHaveInstructions() const { return true; }
+  virtual bool mayHaveInstructions(MCSection &Sec) const { return true; }
 };
 
 /// Create a dummy machine code streamer, which does nothing. This is useful for
diff --git a/include/llvm/MC/MCSubtargetInfo.h b/include/llvm/MC/MCSubtargetInfo.h
index 3984a1fb21b..d5ad4eebf9e 100644
--- a/include/llvm/MC/MCSubtargetInfo.h
+++ b/include/llvm/MC/MCSubtargetInfo.h
@@ -27,7 +27,7 @@ class StringRef;
 /// MCSubtargetInfo - Generic base class for all target subtargets.
 ///
 class MCSubtargetInfo {
-  std::string TargetTriple;            // Target triple
+  Triple TargetTriple;                        // Target triple
   std::string CPU; // CPU being targeted.
   ArrayRef<SubtargetFeatureKV> ProcFeatures;  // Processor feature list
   ArrayRef<SubtargetFeatureKV> ProcDesc;  // Processor descriptions
@@ -37,28 +37,29 @@ class MCSubtargetInfo {
   const MCWriteProcResEntry *WriteProcResTable;
   const MCWriteLatencyEntry *WriteLatencyTable;
   const MCReadAdvanceEntry *ReadAdvanceTable;
-  MCSchedModel CPUSchedModel;
+  const MCSchedModel *CPUSchedModel;
 
   const InstrStage *Stages;            // Instruction itinerary stages
   const unsigned *OperandCycles;       // Itinerary operand cycles
   const unsigned *ForwardingPaths;     // Forwarding paths
-  uint64_t FeatureBits;                // Feature bits for current CPU + FS
+  FeatureBitset FeatureBits;           // Feature bits for current CPU + FS
+
+  MCSubtargetInfo() = delete;
+  MCSubtargetInfo &operator=(MCSubtargetInfo &&) = delete;
+  MCSubtargetInfo &operator=(const MCSubtargetInfo &) = delete;
 
 public:
-  void InitMCSubtargetInfo(StringRef TT, StringRef CPU, StringRef FS,
-                           ArrayRef<SubtargetFeatureKV> PF,
-                           ArrayRef<SubtargetFeatureKV> PD,
-                           const SubtargetInfoKV *ProcSched,
-                           const MCWriteProcResEntry *WPR,
-                           const MCWriteLatencyEntry *WL,
-                           const MCReadAdvanceEntry *RA,
-                           const InstrStage *IS,
-                           const unsigned *OC, const unsigned *FP);
+  MCSubtargetInfo(const MCSubtargetInfo &) = default;
+  MCSubtargetInfo(const Triple &TT, StringRef CPU, StringRef FS,
+                  ArrayRef<SubtargetFeatureKV> PF,
+                  ArrayRef<SubtargetFeatureKV> PD,
+                  const SubtargetInfoKV *ProcSched,
+                  const MCWriteProcResEntry *WPR, const MCWriteLatencyEntry *WL,
+                  const MCReadAdvanceEntry *RA, const InstrStage *IS,
+                  const unsigned *OC, const unsigned *FP);
 
   /// getTargetTriple - Return the target triple string.
-  StringRef getTargetTriple() const {
-    return TargetTriple;
-  }
+  const Triple &getTargetTriple() const { return TargetTriple; }
 
   /// getCPU - Return the CPU string.
   StringRef getCPU() const {
@@ -67,36 +68,49 @@ class MCSubtargetInfo {
 
   /// getFeatureBits - Return the feature bits.
   ///
-  uint64_t getFeatureBits() const {
+  const FeatureBitset& getFeatureBits() const {
     return FeatureBits;
   }
 
   /// setFeatureBits - Set the feature bits.
   ///
-  void setFeatureBits(uint64_t FeatureBits_) { FeatureBits = FeatureBits_; }
+  void setFeatureBits(const FeatureBitset &FeatureBits_) {
+    FeatureBits = FeatureBits_;
+  }
 
-  /// InitMCProcessorInfo - Set or change the CPU (optionally supplemented with
-  /// feature string). Recompute feature bits and scheduling model.
+protected:
+  /// Initialize the scheduling model and feature bits.
+  ///
+  /// FIXME: Find a way to stick this in the constructor, since it should only
+  /// be called during initialization.
   void InitMCProcessorInfo(StringRef CPU, StringRef FS);
 
-  /// InitCPUSchedModel - Recompute scheduling model based on CPU.
-  void InitCPUSchedModel(StringRef CPU);
+public:
+  /// Set the features to the default for the given CPU.
+  void setDefaultFeatures(StringRef CPU);
 
   /// ToggleFeature - Toggle a feature and returns the re-computed feature
   /// bits. This version does not change the implied bits.
-  uint64_t ToggleFeature(uint64_t FB);
+  FeatureBitset ToggleFeature(uint64_t FB);
 
   /// ToggleFeature - Toggle a feature and returns the re-computed feature
-  /// bits. This version will also change all implied bits.
-  uint64_t ToggleFeature(StringRef FS);
+  /// bits. This version does not change the implied bits.
+  FeatureBitset ToggleFeature(const FeatureBitset& FB);
+
+  /// ToggleFeature - Toggle a set of features and returns the re-computed
+  /// feature bits. This version will also change all implied bits.
+  FeatureBitset ToggleFeature(StringRef FS);
+
+  /// Apply a feature flag and return the re-computed feature bits, including
+  /// all feature bits implied by the flag.
+  FeatureBitset ApplyFeatureFlag(StringRef FS);
 
   /// getSchedModelForCPU - Get the machine model of a CPU.
   ///
-  MCSchedModel getSchedModelForCPU(StringRef CPU) const;
+  const MCSchedModel &getSchedModelForCPU(StringRef CPU) const;
 
-  /// getSchedModel - Get the machine model for this subtarget's CPU.
-  ///
-  const MCSchedModel &getSchedModel() const { return CPUSchedModel; }
+  /// Get the machine model for this subtarget's CPU.
+  const MCSchedModel &getSchedModel() const { return *CPUSchedModel; }
 
   /// Return an iterator at the first process resource consumed by the given
   /// scheduling class.
@@ -144,7 +158,7 @@ class MCSubtargetInfo {
   void initInstrItins(InstrItineraryData &InstrItins) const;
 
   /// Check whether the CPU string is valid.
-  bool isCPUStringValid(StringRef CPU) {
+  bool isCPUStringValid(StringRef CPU) const {
     auto Found = std::find_if(ProcDesc.begin(), ProcDesc.end(),
                               [=](const SubtargetFeatureKV &KV) {
                                 return CPU == KV.Key; 
diff --git a/include/llvm/MC/MCSymbol.h b/include/llvm/MC/MCSymbol.h
index e4bdfda55d6..b2910dfccd6 100644
--- a/include/llvm/MC/MCSymbol.h
+++ b/include/llvm/MC/MCSymbol.h
@@ -14,176 +14,411 @@
 #ifndef LLVM_MC_MCSYMBOL_H
 #define LLVM_MC_MCSYMBOL_H
 
-#include "llvm/ADT/StringRef.h"
-#include "llvm/MC/MCExpr.h"
+#include "llvm/ADT/PointerIntPair.h"
+#include "llvm/ADT/PointerUnion.h"
+#include "llvm/ADT/StringMap.h"
+#include "llvm/MC/MCAssembler.h"
 #include "llvm/Support/Compiler.h"
 
 namespace llvm {
-  class MCExpr;
-  class MCSection;
-  class MCContext;
-  class raw_ostream;
-
-  /// MCSymbol - Instances of this class represent a symbol name in the MC file,
-  /// and MCSymbols are created and unique'd by the MCContext class.  MCSymbols
-  /// should only be constructed with valid names for the object file.
+class MCAsmInfo;
+class MCExpr;
+class MCSymbol;
+class MCFragment;
+class MCSection;
+class MCContext;
+class raw_ostream;
+
+/// MCSymbol - Instances of this class represent a symbol name in the MC file,
+/// and MCSymbols are created and uniqued by the MCContext class.  MCSymbols
+/// should only be constructed with valid names for the object file.
+///
+/// If the symbol is defined/emitted into the current translation unit, the
+/// Section member is set to indicate what section it lives in.  Otherwise, if
+/// it is a reference to an external entity, it has a null section.
+class MCSymbol {
+protected:
+  /// The kind of the symbol.  If it is any value other than unset then this
+  /// class is actually one of the appropriate subclasses of MCSymbol.
+  enum SymbolKind {
+    SymbolKindUnset,
+    SymbolKindCOFF,
+    SymbolKindELF,
+    SymbolKindMachO,
+  };
+
+  /// A symbol can contain an Offset, or Value, or be Common, but never more
+  /// than one of these.
+  enum Contents : uint8_t {
+    SymContentsUnset,
+    SymContentsOffset,
+    SymContentsVariable,
+    SymContentsCommon,
+  };
+
+  // Special sentinal value for the absolute pseudo section.
+  //
+  // FIXME: Use a PointerInt wrapper for this?
+  static MCSection *AbsolutePseudoSection;
+
+  /// If a symbol has a Fragment, the section is implied, so we only need
+  /// one pointer.
+  /// FIXME: We might be able to simplify this by having the asm streamer create
+  /// dummy fragments.
+  /// If this is a section, then it gives the symbol is defined in. This is null
+  /// for undefined symbols, and the special AbsolutePseudoSection value for
+  /// absolute symbols. If this is a variable symbol, this caches the variable
+  /// value's section.
+  ///
+  /// If this is a fragment, then it gives the fragment this symbol's value is
+  /// relative to, if any.
+  ///
+  /// For the 'HasName' integer, this is true if this symbol is named.
+  /// A named symbol will have a pointer to the name allocated in the bytes
+  /// immediately prior to the MCSymbol.
+  mutable PointerIntPair<PointerUnion<MCSection *, MCFragment *>, 1>
+      SectionOrFragmentAndHasName;
+
+  /// IsTemporary - True if this is an assembler temporary label, which
+  /// typically does not survive in the .o file's symbol table.  Usually
+  /// "Lfoo" or ".foo".
+  unsigned IsTemporary : 1;
+
+  /// \brief True if this symbol can be redefined.
+  unsigned IsRedefinable : 1;
+
+  /// IsUsed - True if this symbol has been used.
+  mutable unsigned IsUsed : 1;
+
+  mutable bool IsRegistered : 1;
+
+  /// This symbol is visible outside this translation unit.
+  mutable unsigned IsExternal : 1;
+
+  /// This symbol is private extern.
+  mutable unsigned IsPrivateExtern : 1;
+
+  /// LLVM RTTI discriminator. This is actually a SymbolKind enumerator, but is
+  /// unsigned to avoid sign extension and achieve better bitpacking with MSVC.
+  unsigned Kind : 2;
+
+  /// True if we have created a relocation that uses this symbol.
+  mutable unsigned IsUsedInReloc : 1;
+
+  /// This is actually a Contents enumerator, but is unsigned to avoid sign
+  /// extension and achieve better bitpacking with MSVC.
+  unsigned SymbolContents : 2;
+
+  /// The alignment of the symbol, if it is 'common', or -1.
   ///
-  /// If the symbol is defined/emitted into the current translation unit, the
-  /// Section member is set to indicate what section it lives in.  Otherwise, if
-  /// it is a reference to an external entity, it has a null section.
-  class MCSymbol {
-    // Special sentinal value for the absolute pseudo section.
-    //
-    // FIXME: Use a PointerInt wrapper for this?
-    static const MCSection *AbsolutePseudoSection;
-
-    /// Name - The name of the symbol.  The referred-to string data is actually
-    /// held by the StringMap that lives in MCContext.
-    StringRef Name;
-
-    /// Section - The section the symbol is defined in. This is null for
-    /// undefined symbols, and the special AbsolutePseudoSection value for
-    /// absolute symbols. If this is a variable symbol, this caches the
-    /// variable value's section.
-    mutable const MCSection *Section;
-
-    /// Value - If non-null, the value for a variable symbol.
+  /// The alignment is stored as log2(align) + 1.  This allows all values from
+  /// 0 to 2^31 to be stored which is every power of 2 representable by an
+  /// unsigned.
+  enum : unsigned { NumCommonAlignmentBits = 5 };
+  unsigned CommonAlignLog2 : NumCommonAlignmentBits;
+
+  /// The Flags field is used by object file implementations to store
+  /// additional per symbol information which is not easily classified.
+  enum : unsigned { NumFlagsBits = 16 };
+  mutable uint32_t Flags : NumFlagsBits;
+
+  /// Index field, for use by the object file implementation.
+  mutable uint32_t Index = 0;
+
+  union {
+    /// The offset to apply to the fragment address to form this symbol's value.
+    uint64_t Offset;
+
+    /// The size of the symbol, if it is 'common'.
+    uint64_t CommonSize;
+
+    /// If non-null, the value for a variable symbol.
     const MCExpr *Value;
+  };
 
-    /// IsTemporary - True if this is an assembler temporary label, which
-    /// typically does not survive in the .o file's symbol table.  Usually
-    /// "Lfoo" or ".foo".
-    unsigned IsTemporary : 1;
-
-    /// \brief True if this symbol can be redefined.
-    unsigned IsRedefinable : 1;
-
-    /// IsUsed - True if this symbol has been used.
-    mutable unsigned IsUsed : 1;
-
-  private:  // MCContext creates and uniques these.
-    friend class MCExpr;
-    friend class MCContext;
-    MCSymbol(StringRef name, bool isTemporary)
-      : Name(name), Section(nullptr), Value(nullptr),
-        IsTemporary(isTemporary), IsRedefinable(false), IsUsed(false) {}
-
-    MCSymbol(const MCSymbol&) = delete;
-    void operator=(const MCSymbol&) = delete;
-    const MCSection *getSectionPtr() const {
-      if (Section || !Value)
-        return Section;
-      return Section = Value->FindAssociatedSection();
-    }
+protected: // MCContext creates and uniques these.
+  friend class MCExpr;
+  friend class MCContext;
+
+  /// \brief The name for a symbol.
+  /// MCSymbol contains a uint64_t so is probably aligned to 8.  On a 32-bit
+  /// system, the name is a pointer so isn't going to satisfy the 8 byte
+  /// alignment of uint64_t.  Account for that here.
+  typedef union {
+    const StringMapEntry<bool> *NameEntry;
+    uint64_t AlignmentPadding;
+  } NameEntryStorageTy;
+
+  MCSymbol(SymbolKind Kind, const StringMapEntry<bool> *Name, bool isTemporary)
+      : IsTemporary(isTemporary), IsRedefinable(false), IsUsed(false),
+        IsRegistered(false), IsExternal(false), IsPrivateExtern(false),
+        Kind(Kind), IsUsedInReloc(false), SymbolContents(SymContentsUnset),
+        CommonAlignLog2(0), Flags(0) {
+    Offset = 0;
+    SectionOrFragmentAndHasName.setInt(!!Name);
+    if (Name)
+      getNameEntryPtr() = Name;
+  }
+
+  // Provide custom new/delete as we will only allocate space for a name
+  // if we need one.
+  void *operator new(size_t s, const StringMapEntry<bool> *Name,
+                     MCContext &Ctx);
 
-  public:
-    /// getName - Get the symbol name.
-    StringRef getName() const { return Name; }
+private:
 
-    /// @name Accessors
-    /// @{
+  void operator delete(void *);
+  /// \brief Placement delete - required by std, but never called.
+  void operator delete(void*, unsigned) {
+    llvm_unreachable("Constructor throws?");
+  }
+  /// \brief Placement delete - required by std, but never called.
+  void operator delete(void*, unsigned, bool) {
+    llvm_unreachable("Constructor throws?");
+  }
 
-    /// isTemporary - Check if this is an assembler temporary symbol.
-    bool isTemporary() const { return IsTemporary; }
+  MCSymbol(const MCSymbol &) = delete;
+  void operator=(const MCSymbol &) = delete;
+  MCSection *getSectionPtr() const {
+    if (MCFragment *F = getFragment())
+      return F->getParent();
+    const auto &SectionOrFragment = SectionOrFragmentAndHasName.getPointer();
+    assert(!SectionOrFragment.is<MCFragment *>() && "Section or null expected");
+    MCSection *Section = SectionOrFragment.dyn_cast<MCSection *>();
+    if (Section || !isVariable())
+      return Section;
+    return Section = getVariableValue()->findAssociatedSection();
+  }
 
-    /// isUsed - Check if this is used.
-    bool isUsed() const { return IsUsed; }
-    void setUsed(bool Value) const { IsUsed = Value; }
+  /// \brief Get a reference to the name field.  Requires that we have a name
+  const StringMapEntry<bool> *&getNameEntryPtr() {
+    assert(SectionOrFragmentAndHasName.getInt() && "Name is required");
+    NameEntryStorageTy *Name = reinterpret_cast<NameEntryStorageTy *>(this);
+    return (*(Name - 1)).NameEntry;
+  }
+  const StringMapEntry<bool> *&getNameEntryPtr() const {
+    return const_cast<MCSymbol*>(this)->getNameEntryPtr();
+  }
+
+public:
+  /// getName - Get the symbol name.
+  StringRef getName() const {
+    if (!SectionOrFragmentAndHasName.getInt())
+      return StringRef();
+
+    return getNameEntryPtr()->first();
+  }
 
-    /// \brief Check if this symbol is redefinable.
-    bool isRedefinable() const { return IsRedefinable; }
-    /// \brief Mark this symbol as redefinable.
-    void setRedefinable(bool Value) { IsRedefinable = Value; }
-    /// \brief Prepare this symbol to be redefined.
-    void redefineIfPossible() {
-      if (IsRedefinable) {
+  bool isRegistered() const { return IsRegistered; }
+  void setIsRegistered(bool Value) const { IsRegistered = Value; }
+
+  void setUsedInReloc() const { IsUsedInReloc = true; }
+  bool isUsedInReloc() const { return IsUsedInReloc; }
+
+  /// \name Accessors
+  /// @{
+
+  /// isTemporary - Check if this is an assembler temporary symbol.
+  bool isTemporary() const { return IsTemporary; }
+
+  /// isUsed - Check if this is used.
+  bool isUsed() const { return IsUsed; }
+  void setUsed(bool Value) const { IsUsed = Value; }
+
+  /// \brief Check if this symbol is redefinable.
+  bool isRedefinable() const { return IsRedefinable; }
+  /// \brief Mark this symbol as redefinable.
+  void setRedefinable(bool Value) { IsRedefinable = Value; }
+  /// \brief Prepare this symbol to be redefined.
+  void redefineIfPossible() {
+    if (IsRedefinable) {
+      if (SymbolContents == SymContentsVariable) {
         Value = nullptr;
-        Section = nullptr;
-        IsRedefinable = false;
+        SymbolContents = SymContentsUnset;
       }
+      setUndefined();
+      IsRedefinable = false;
     }
+  }
 
-    /// @}
-    /// @name Associated Sections
-    /// @{
+  /// @}
+  /// \name Associated Sections
+  /// @{
 
-    /// isDefined - Check if this symbol is defined (i.e., it has an address).
-    ///
-    /// Defined symbols are either absolute or in some section.
-    bool isDefined() const {
-      return getSectionPtr() != nullptr;
-    }
+  /// isDefined - Check if this symbol is defined (i.e., it has an address).
+  ///
+  /// Defined symbols are either absolute or in some section.
+  bool isDefined() const { return getSectionPtr() != nullptr; }
 
-    /// isInSection - Check if this symbol is defined in some section (i.e., it
-    /// is defined but not absolute).
-    bool isInSection() const {
-      return isDefined() && !isAbsolute();
-    }
+  /// isInSection - Check if this symbol is defined in some section (i.e., it
+  /// is defined but not absolute).
+  bool isInSection() const { return isDefined() && !isAbsolute(); }
 
-    /// isUndefined - Check if this symbol undefined (i.e., implicitly defined).
-    bool isUndefined() const {
-      return !isDefined();
-    }
+  /// isUndefined - Check if this symbol undefined (i.e., implicitly defined).
+  bool isUndefined() const { return !isDefined(); }
 
-    /// isAbsolute - Check if this is an absolute symbol.
-    bool isAbsolute() const {
-      return getSectionPtr() == AbsolutePseudoSection;
-    }
+  /// isAbsolute - Check if this is an absolute symbol.
+  bool isAbsolute() const { return getSectionPtr() == AbsolutePseudoSection; }
 
-    /// getSection - Get the section associated with a defined, non-absolute
-    /// symbol.
-    const MCSection &getSection() const {
-      assert(isInSection() && "Invalid accessor!");
-      return *getSectionPtr();
-    }
+  /// Get the section associated with a defined, non-absolute symbol.
+  MCSection &getSection() const {
+    assert(isInSection() && "Invalid accessor!");
+    return *getSectionPtr();
+  }
 
-    /// setSection - Mark the symbol as defined in the section \p S.
-    void setSection(const MCSection &S) {
-      assert(!isVariable() && "Cannot set section of variable");
-      Section = &S;
-    }
+  /// Mark the symbol as defined in the section \p S.
+  void setSection(MCSection &S) {
+    assert(!isVariable() && "Cannot set section of variable");
+    assert(!SectionOrFragmentAndHasName.getPointer().is<MCFragment *>() &&
+           "Section or null expected");
+    SectionOrFragmentAndHasName.setPointer(&S);
+  }
 
-    /// setUndefined - Mark the symbol as undefined.
-    void setUndefined() {
-      Section = nullptr;
-    }
+  /// Mark the symbol as undefined.
+  void setUndefined() {
+    SectionOrFragmentAndHasName.setPointer(
+        PointerUnion<MCSection *, MCFragment *>());
+  }
 
-    /// @}
-    /// @name Variable Symbols
-    /// @{
+  bool isELF() const { return Kind == SymbolKindELF; }
 
-    /// isVariable - Check if this is a variable symbol.
-    bool isVariable() const {
-      return Value != nullptr;
-    }
+  bool isCOFF() const { return Kind == SymbolKindCOFF; }
 
-    /// getVariableValue() - Get the value for variable symbols.
-    const MCExpr *getVariableValue() const {
-      assert(isVariable() && "Invalid accessor!");
-      IsUsed = true;
-      return Value;
-    }
+  bool isMachO() const { return Kind == SymbolKindMachO; }
 
-    // AliasedSymbol() - If this is an alias (a = b), return the symbol
-    // we ultimately point to. For a non-alias, this just returns the symbol
-    // itself.
-    const MCSymbol &AliasedSymbol() const;
+  /// @}
+  /// \name Variable Symbols
+  /// @{
 
-    void setVariableValue(const MCExpr *Value);
+  /// isVariable - Check if this is a variable symbol.
+  bool isVariable() const {
+    return SymbolContents == SymContentsVariable;
+  }
 
-    /// @}
+  /// getVariableValue() - Get the value for variable symbols.
+  const MCExpr *getVariableValue() const {
+    assert(isVariable() && "Invalid accessor!");
+    IsUsed = true;
+    return Value;
+  }
 
-    /// print - Print the value to the stream \p OS.
-    void print(raw_ostream &OS) const;
+  void setVariableValue(const MCExpr *Value);
 
-    /// dump - Print the value to stderr.
-    void dump() const;
-  };
+  /// @}
+
+  /// Get the (implementation defined) index.
+  uint32_t getIndex() const {
+    return Index;
+  }
+
+  /// Set the (implementation defined) index.
+  void setIndex(uint32_t Value) const {
+    Index = Value;
+  }
+
+  uint64_t getOffset() const {
+    assert((SymbolContents == SymContentsUnset ||
+            SymbolContents == SymContentsOffset) &&
+           "Cannot get offset for a common/variable symbol");
+    return Offset;
+  }
+  void setOffset(uint64_t Value) {
+    assert((SymbolContents == SymContentsUnset ||
+            SymbolContents == SymContentsOffset) &&
+           "Cannot set offset for a common/variable symbol");
+    Offset = Value;
+    SymbolContents = SymContentsOffset;
+  }
+
+  /// Return the size of a 'common' symbol.
+  uint64_t getCommonSize() const {
+    assert(isCommon() && "Not a 'common' symbol!");
+    return CommonSize;
+  }
 
-  inline raw_ostream &operator<<(raw_ostream &OS, const MCSymbol &Sym) {
-    Sym.print(OS);
-    return OS;
+  /// Mark this symbol as being 'common'.
+  ///
+  /// \param Size - The size of the symbol.
+  /// \param Align - The alignment of the symbol.
+  void setCommon(uint64_t Size, unsigned Align) {
+    assert(getOffset() == 0);
+    CommonSize = Size;
+    SymbolContents = SymContentsCommon;
+
+    assert((!Align || isPowerOf2_32(Align)) &&
+           "Alignment must be a power of 2");
+    unsigned Log2Align = Log2_32(Align) + 1;
+    assert(Log2Align < (1U << NumCommonAlignmentBits) &&
+           "Out of range alignment");
+    CommonAlignLog2 = Log2Align;
+  }
+
+  ///  Return the alignment of a 'common' symbol.
+  unsigned getCommonAlignment() const {
+    assert(isCommon() && "Not a 'common' symbol!");
+    return CommonAlignLog2 ? (1U << (CommonAlignLog2 - 1)) : 0;
+  }
+
+  /// Declare this symbol as being 'common'.
+  ///
+  /// \param Size - The size of the symbol.
+  /// \param Align - The alignment of the symbol.
+  /// \return True if symbol was already declared as a different type
+  bool declareCommon(uint64_t Size, unsigned Align) {
+    assert(isCommon() || getOffset() == 0);
+    if(isCommon()) {
+      if(CommonSize != Size || getCommonAlignment() != Align)
+       return true;
+    } else
+      setCommon(Size, Align);
+    return false;
+  }
+
+  /// Is this a 'common' symbol.
+  bool isCommon() const {
+    return SymbolContents == SymContentsCommon;
+  }
+
+  MCFragment *getFragment() const {
+    return SectionOrFragmentAndHasName.getPointer().dyn_cast<MCFragment *>();
+  }
+  void setFragment(MCFragment *Value) const {
+    SectionOrFragmentAndHasName.setPointer(Value);
   }
+
+  bool isExternal() const { return IsExternal; }
+  void setExternal(bool Value) const { IsExternal = Value; }
+
+  bool isPrivateExtern() const { return IsPrivateExtern; }
+  void setPrivateExtern(bool Value) { IsPrivateExtern = Value; }
+
+  /// print - Print the value to the stream \p OS.
+  void print(raw_ostream &OS, const MCAsmInfo *MAI) const;
+
+  /// dump - Print the value to stderr.
+  void dump() const;
+
+protected:
+  /// Get the (implementation defined) symbol flags.
+  uint32_t getFlags() const { return Flags; }
+
+  /// Set the (implementation defined) symbol flags.
+  void setFlags(uint32_t Value) const {
+    assert(Value < (1U << NumFlagsBits) && "Out of range flags");
+    Flags = Value;
+  }
+
+  /// Modify the flags via a mask
+  void modifyFlags(uint32_t Value, uint32_t Mask) const {
+    assert(Value < (1U << NumFlagsBits) && "Out of range flags");
+    Flags = (Flags & ~Mask) | Value;
+  }
+};
+
+inline raw_ostream &operator<<(raw_ostream &OS, const MCSymbol &Sym) {
+  Sym.print(OS, nullptr);
+  return OS;
+}
 } // end namespace llvm
 
 #endif
diff --git a/include/llvm/MC/MCSymbolCOFF.h b/include/llvm/MC/MCSymbolCOFF.h
new file mode 100644
index 00000000000..2172c67981c
--- /dev/null
+++ b/include/llvm/MC/MCSymbolCOFF.h
@@ -0,0 +1,64 @@
+//===- MCSymbolCOFF.h -  ----------------------------------------*- C++ -*-===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+#ifndef LLVM_MC_MCSYMBOLCOFF_H
+#define LLVM_MC_MCSYMBOLCOFF_H
+
+#include "llvm/MC/MCSymbol.h"
+
+namespace llvm {
+class MCSymbolCOFF : public MCSymbol {
+
+  /// This corresponds to the e_type field of the COFF symbol.
+  mutable uint16_t Type;
+
+  enum SymbolFlags : uint16_t {
+    SF_ClassMask = 0x00FF,
+    SF_ClassShift = 0,
+
+    SF_WeakExternal = 0x0100,
+    SF_SafeSEH = 0x0200,
+  };
+
+public:
+  MCSymbolCOFF(const StringMapEntry<bool> *Name, bool isTemporary)
+      : MCSymbol(SymbolKindCOFF, Name, isTemporary), Type(0) {}
+
+  uint16_t getType() const {
+    return Type;
+  }
+  void setType(uint16_t Ty) const {
+    Type = Ty;
+  }
+
+  uint16_t getClass() const {
+    return (getFlags() & SF_ClassMask) >> SF_ClassShift;
+  }
+  void setClass(uint16_t StorageClass) const {
+    modifyFlags(StorageClass << SF_ClassShift, SF_ClassMask);
+  }
+
+  bool isWeakExternal() const {
+    return getFlags() & SF_WeakExternal;
+  }
+  void setIsWeakExternal() const {
+    modifyFlags(SF_WeakExternal, SF_WeakExternal);
+  }
+
+  bool isSafeSEH() const {
+    return getFlags() & SF_SafeSEH;
+  }
+  void setIsSafeSEH() const {
+    modifyFlags(SF_SafeSEH, SF_SafeSEH);
+  }
+
+  static bool classof(const MCSymbol *S) { return S->isCOFF(); }
+};
+}
+
+#endif
diff --git a/include/llvm/MC/MCSymbolELF.h b/include/llvm/MC/MCSymbolELF.h
new file mode 100644
index 00000000000..bbcd22e8e7d
--- /dev/null
+++ b/include/llvm/MC/MCSymbolELF.h
@@ -0,0 +1,54 @@
+//===- MCSymbolELF.h -  -----------------------------------------*- C++ -*-===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+#ifndef LLVM_MC_MCSYMBOLELF_H
+#define LLVM_MC_MCSYMBOLELF_H
+
+#include "llvm/MC/MCSymbol.h"
+
+namespace llvm {
+class MCSymbolELF : public MCSymbol {
+  /// An expression describing how to calculate the size of a symbol. If a
+  /// symbol has no size this field will be NULL.
+  const MCExpr *SymbolSize = nullptr;
+
+public:
+  MCSymbolELF(const StringMapEntry<bool> *Name, bool isTemporary)
+      : MCSymbol(SymbolKindELF, Name, isTemporary) {}
+  void setSize(const MCExpr *SS) { SymbolSize = SS; }
+
+  const MCExpr *getSize() const { return SymbolSize; }
+
+  void setVisibility(unsigned Visibility);
+  unsigned getVisibility() const;
+
+  void setOther(unsigned Other);
+  unsigned getOther() const;
+
+  void setType(unsigned Type) const;
+  unsigned getType() const;
+
+  void setBinding(unsigned Binding) const;
+  unsigned getBinding() const;
+
+  bool isBindingSet() const;
+
+  void setIsWeakrefUsedInReloc() const;
+  bool isWeakrefUsedInReloc() const;
+
+  void setIsSignature() const;
+  bool isSignature() const;
+
+  static bool classof(const MCSymbol *S) { return S->isELF(); }
+
+private:
+  void setIsBindingSet() const;
+};
+}
+
+#endif
diff --git a/include/llvm/MC/MCSymbolMachO.h b/include/llvm/MC/MCSymbolMachO.h
new file mode 100644
index 00000000000..5b0321fe9f7
--- /dev/null
+++ b/include/llvm/MC/MCSymbolMachO.h
@@ -0,0 +1,123 @@
+//===- MCSymbolMachO.h -  ---------------------------------------*- C++ -*-===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+#ifndef LLVM_MC_MCSYMBOLMACHO_H
+#define LLVM_MC_MCSYMBOLMACHO_H
+
+#include "llvm/MC/MCSymbol.h"
+
+namespace llvm {
+class MCSymbolMachO : public MCSymbol {
+  /// \brief We store the value for the 'desc' symbol field in the
+  /// lowest 16 bits of the implementation defined flags.
+  enum MachOSymbolFlags : uint16_t { // See <mach-o/nlist.h>.
+    SF_DescFlagsMask                        = 0xFFFF,
+
+    // Reference type flags.
+    SF_ReferenceTypeMask                    = 0x0007,
+    SF_ReferenceTypeUndefinedNonLazy        = 0x0000,
+    SF_ReferenceTypeUndefinedLazy           = 0x0001,
+    SF_ReferenceTypeDefined                 = 0x0002,
+    SF_ReferenceTypePrivateDefined          = 0x0003,
+    SF_ReferenceTypePrivateUndefinedNonLazy = 0x0004,
+    SF_ReferenceTypePrivateUndefinedLazy    = 0x0005,
+
+    // Other 'desc' flags.
+    SF_ThumbFunc                            = 0x0008,
+    SF_NoDeadStrip                          = 0x0020,
+    SF_WeakReference                        = 0x0040,
+    SF_WeakDefinition                       = 0x0080,
+    SF_SymbolResolver                       = 0x0100,
+
+    // Common alignment
+    SF_CommonAlignmentMask                  = 0xF0FF,
+    SF_CommonAlignmentShift                 = 8
+  };
+
+public:
+  MCSymbolMachO(const StringMapEntry<bool> *Name, bool isTemporary)
+      : MCSymbol(SymbolKindMachO, Name, isTemporary) {}
+
+  // Reference type methods.
+
+  void clearReferenceType() const {
+    modifyFlags(0, SF_ReferenceTypeMask);
+  }
+
+  void setReferenceTypeUndefinedLazy(bool Value) const {
+    modifyFlags(Value ? SF_ReferenceTypeUndefinedLazy : 0,
+                SF_ReferenceTypeUndefinedLazy);
+  }
+
+  // Other 'desc' methods.
+
+  void setThumbFunc() const {
+    modifyFlags(SF_ThumbFunc, SF_ThumbFunc);
+  }
+
+  bool isNoDeadStrip() const {
+    return getFlags() & SF_NoDeadStrip;
+  }
+  void setNoDeadStrip() const {
+    modifyFlags(SF_NoDeadStrip, SF_NoDeadStrip);
+  }
+
+  bool isWeakReference() const {
+    return getFlags() & SF_WeakReference;
+  }
+  void setWeakReference() const {
+    modifyFlags(SF_WeakReference, SF_WeakReference);
+  }
+
+  bool isWeakDefinition() const {
+    return getFlags() & SF_WeakDefinition;
+  }
+  void setWeakDefinition() const {
+    modifyFlags(SF_WeakDefinition, SF_WeakDefinition);
+  }
+
+  bool isSymbolResolver() const {
+    return getFlags() & SF_SymbolResolver;
+  }
+  void setSymbolResolver() const {
+    modifyFlags(SF_SymbolResolver, SF_SymbolResolver);
+  }
+
+  void setDesc(unsigned Value) const {
+    assert(Value == (Value & SF_DescFlagsMask) &&
+           "Invalid .desc value!");
+    setFlags(Value & SF_DescFlagsMask);
+  }
+
+  /// \brief Get the encoded value of the flags as they will be emitted in to
+  /// the MachO binary
+  uint16_t getEncodedFlags() const {
+    uint16_t Flags = getFlags();
+
+    // Common alignment is packed into the 'desc' bits.
+    if (isCommon()) {
+      if (unsigned Align = getCommonAlignment()) {
+        unsigned Log2Size = Log2_32(Align);
+        assert((1U << Log2Size) == Align && "Invalid 'common' alignment!");
+        if (Log2Size > 15)
+          report_fatal_error("invalid 'common' alignment '" +
+                             Twine(Align) + "' for '" + getName() + "'",
+                             false);
+        Flags = (Flags & SF_CommonAlignmentMask) |
+                (Log2Size << SF_CommonAlignmentShift);
+      }
+    }
+
+    return Flags;
+  }
+
+  static bool classof(const MCSymbol *S) { return S->isMachO(); }
+};
+}
+
+#endif
diff --git a/include/llvm/MC/MCSymbolizer.h b/include/llvm/MC/MCSymbolizer.h
index 20e7b81e733..2ef17673f09 100644
--- a/include/llvm/MC/MCSymbolizer.h
+++ b/include/llvm/MC/MCSymbolizer.h
@@ -59,14 +59,14 @@ class MCSymbolizer {
   /// represent this immediate in a more understandable way, for instance as a
   /// symbol or an offset from a symbol. Relocations can also be used to enrich
   /// the symbolic expression.
-  /// @param Inst      - The MCInst where to insert the symbolic operand.
-  /// @param cStream   - Stream to print comments and annotations on.
-  /// @param Value     - Operand value, pc-adjusted by the caller if necessary.
-  /// @param Address   - Load address of the instruction.
-  /// @param IsBranch  - Is the instruction a branch?
-  /// @param Offset    - Byte offset of the operand inside the inst.
-  /// @param InstSize  - Size of the instruction in bytes.
-  /// @return Whether a symbolic operand was added.
+  /// \param Inst      - The MCInst where to insert the symbolic operand.
+  /// \param cStream   - Stream to print comments and annotations on.
+  /// \param Value     - Operand value, pc-adjusted by the caller if necessary.
+  /// \param Address   - Load address of the instruction.
+  /// \param IsBranch  - Is the instruction a branch?
+  /// \param Offset    - Byte offset of the operand inside the inst.
+  /// \param InstSize  - Size of the instruction in bytes.
+  /// \return Whether a symbolic operand was added.
   virtual bool tryAddingSymbolicOperand(MCInst &Inst, raw_ostream &cStream,
                                         int64_t Value, uint64_t Address,
                                         bool IsBranch, uint64_t Offset,
diff --git a/include/llvm/MC/MCTargetAsmParser.h b/include/llvm/MC/MCTargetAsmParser.h
index 36db3914f01..2b857ca4b2d 100644
--- a/include/llvm/MC/MCTargetAsmParser.h
+++ b/include/llvm/MC/MCTargetAsmParser.h
@@ -92,7 +92,7 @@ class MCTargetAsmParser : public MCAsmParserExtension {
   MCTargetAsmParser(const MCTargetAsmParser &) = delete;
   void operator=(const MCTargetAsmParser &) = delete;
 protected: // Can only create subclasses.
-  MCTargetAsmParser();
+  MCTargetAsmParser(MCTargetOptions const &);
 
   /// AvailableFeatures - The current set of available features.
   uint64_t AvailableFeatures;
@@ -198,7 +198,7 @@ class MCTargetAsmParser : public MCAsmParserExtension {
     return nullptr;
   }
 
-  virtual void onLabelParsed(MCSymbol *Symbol) { };
+  virtual void onLabelParsed(MCSymbol *Symbol) { }
 };
 
 } // End llvm namespace
diff --git a/include/llvm/MC/MCTargetOptions.h b/include/llvm/MC/MCTargetOptions.h
index ce28a196e97..00a74de9d06 100644
--- a/include/llvm/MC/MCTargetOptions.h
+++ b/include/llvm/MC/MCTargetOptions.h
@@ -29,6 +29,7 @@ class MCTargetOptions {
   bool MCRelaxAll : 1;
   bool MCNoExecStack : 1;
   bool MCFatalWarnings : 1;
+  bool MCNoWarn : 1;
   bool MCSaveTempLabels : 1;
   bool MCUseDwarfDirectory : 1;
   bool ShowMCEncoding : 1;
@@ -49,13 +50,14 @@ inline bool operator==(const MCTargetOptions &LHS, const MCTargetOptions &RHS) {
           ARE_EQUAL(MCRelaxAll) &&
           ARE_EQUAL(MCNoExecStack) &&
           ARE_EQUAL(MCFatalWarnings) &&
+          ARE_EQUAL(MCNoWarn) &&
           ARE_EQUAL(MCSaveTempLabels) &&
           ARE_EQUAL(MCUseDwarfDirectory) &&
           ARE_EQUAL(ShowMCEncoding) &&
           ARE_EQUAL(ShowMCInst) &&
           ARE_EQUAL(AsmVerbose) &&
           ARE_EQUAL(DwarfVersion) &&
-	  ARE_EQUAL(ABIName));
+          ARE_EQUAL(ABIName));
 #undef ARE_EQUAL
 }
 
diff --git a/include/llvm/MC/MCTargetOptionsCommandFlags.h b/include/llvm/MC/MCTargetOptionsCommandFlags.h
index af23a92e6e9..1240e135ea0 100644
--- a/include/llvm/MC/MCTargetOptionsCommandFlags.h
+++ b/include/llvm/MC/MCTargetOptionsCommandFlags.h
@@ -40,6 +40,12 @@ cl::opt<bool> ShowMCInst("asm-show-inst",
                          cl::desc("Emit internal instruction representation to "
                                   "assembly file"));
 
+cl::opt<bool> FatalWarnings("fatal-warnings",
+                            cl::desc("Treat warnings as errors"));
+
+cl::opt<bool> NoWarn("no-warn", cl::desc("Suppress all warnings"));
+cl::alias NoWarnW("W", cl::desc("Alias for --no-warn"), cl::aliasopt(NoWarn));
+
 cl::opt<std::string>
 ABIName("target-abi", cl::Hidden,
         cl::desc("The name of the ABI to be targeted from the backend."),
@@ -53,6 +59,8 @@ static inline MCTargetOptions InitMCTargetOptionsFromFlags() {
   Options.DwarfVersion = DwarfVersion;
   Options.ShowMCInst = ShowMCInst;
   Options.ABIName = ABIName;
+  Options.MCFatalWarnings = FatalWarnings;
+  Options.MCNoWarn = NoWarn;
   return Options;
 }
 
diff --git a/include/llvm/MC/MCValue.h b/include/llvm/MC/MCValue.h
index 175fed090b3..6bdf43685f2 100644
--- a/include/llvm/MC/MCValue.h
+++ b/include/llvm/MC/MCValue.h
@@ -23,10 +23,11 @@ namespace llvm {
 class MCAsmInfo;
 class raw_ostream;
 
-/// MCValue - This represents an "assembler immediate".  In its most
-/// general form, this can hold ":Kind:(SymbolA - SymbolB + imm64)".
-/// Not all targets supports relocations of this general form, but we
-/// need to represent this anyway.
+/// \brief This represents an "assembler immediate".
+///
+///  In its most general form, this can hold ":Kind:(SymbolA - SymbolB +
+///  imm64)".  Not all targets supports relocations of this general form, but we
+///  need to represent this anyway.
 ///
 /// In general both SymbolA and SymbolB will also have a modifier
 /// analogous to the top-level Kind. Current targets are not expected
@@ -51,13 +52,13 @@ class MCValue {
   const MCSymbolRefExpr *getSymB() const { return SymB; }
   uint32_t getRefKind() const { return RefKind; }
 
-  /// isAbsolute - Is this an absolute (as opposed to relocatable) value.
+  /// \brief Is this an absolute (as opposed to relocatable) value.
   bool isAbsolute() const { return !SymA && !SymB; }
 
-  /// print - Print the value to the stream \p OS.
+  /// \brief Print the value to the stream \p OS.
   void print(raw_ostream &OS) const;
 
-  /// dump - Print the value to stderr.
+  /// \brief Print the value to stderr.
   void dump() const;
 
   MCSymbolRefExpr::VariantKind getAccessVariant() const;
diff --git a/include/llvm/MC/MCWinCOFFStreamer.h b/include/llvm/MC/MCWinCOFFStreamer.h
index b3fee810b6b..6fbc754f112 100644
--- a/include/llvm/MC/MCWinCOFFStreamer.h
+++ b/include/llvm/MC/MCWinCOFFStreamer.h
@@ -50,16 +50,16 @@ class MCWinCOFFStreamer : public MCObjectStreamer {
   void EmitCOFFSymbolStorageClass(int StorageClass) override;
   void EmitCOFFSymbolType(int Type) override;
   void EndCOFFSymbolDef() override;
+  void EmitCOFFSafeSEH(MCSymbol const *Symbol) override;
   void EmitCOFFSectionIndex(MCSymbol const *Symbol) override;
   void EmitCOFFSecRel32(MCSymbol const *Symbol) override;
-  void EmitELFSize(MCSymbol *Symbol, const MCExpr *Value) override;
   void EmitCommonSymbol(MCSymbol *Symbol, uint64_t Size,
                         unsigned ByteAlignment) override;
   void EmitLocalCommonSymbol(MCSymbol *Symbol, uint64_t Size,
                              unsigned ByteAlignment) override;
-  void EmitZerofill(const MCSection *Section, MCSymbol *Symbol, uint64_t Size,
+  void EmitZerofill(MCSection *Section, MCSymbol *Symbol, uint64_t Size,
                     unsigned ByteAlignment) override;
-  void EmitTBSSSymbol(const MCSection *Section, MCSymbol *Symbol, uint64_t Size,
+  void EmitTBSSSymbol(MCSection *Section, MCSymbol *Symbol, uint64_t Size,
                       unsigned ByteAlignment) override;
   void EmitFileDirective(StringRef Filename) override;
   void EmitIdent(StringRef IdentString) override;
diff --git a/include/llvm/MC/MCWinEH.h b/include/llvm/MC/MCWinEH.h
index 05b58c753b6..723d7a397c4 100644
--- a/include/llvm/MC/MCWinEH.h
+++ b/include/llvm/MC/MCWinEH.h
@@ -65,10 +65,10 @@ struct FrameInfo {
 
 class UnwindEmitter {
 public:
-  static const MCSection *getPDataSection(const MCSymbol *Function,
-                                          MCContext &Context);
-  static const MCSection *getXDataSection(const MCSymbol *Function,
-                                          MCContext &Context);
+  static MCSection *getPDataSection(const MCSymbol *Function,
+                                    MCContext &Context);
+  static MCSection *getXDataSection(const MCSymbol *Function,
+                                    MCContext &Context);
 
   virtual ~UnwindEmitter() { }
 
diff --git a/include/llvm/MC/SubtargetFeature.h b/include/llvm/MC/SubtargetFeature.h
index 6f195d7be99..8edd798f153 100644
--- a/include/llvm/MC/SubtargetFeature.h
+++ b/include/llvm/MC/SubtargetFeature.h
@@ -21,11 +21,29 @@
 #include "llvm/ADT/ArrayRef.h"
 #include "llvm/ADT/Triple.h"
 #include "llvm/Support/DataTypes.h"
+#include <bitset>
 
 namespace llvm {
   class raw_ostream;
   class StringRef;
 
+// A container class for subtarget features.
+// This is convenient because std::bitset does not have a constructor
+// with an initializer list of set bits.
+const unsigned MAX_SUBTARGET_FEATURES = 96;
+class FeatureBitset : public std::bitset<MAX_SUBTARGET_FEATURES> {
+public:
+  // Cannot inherit constructors because it's not supported by VC++..
+  FeatureBitset() : bitset() {}
+
+  FeatureBitset(const bitset<MAX_SUBTARGET_FEATURES>& B) : bitset(B) {}
+
+  FeatureBitset(std::initializer_list<unsigned> Init) : bitset() {
+    for (auto I = Init.begin() , E = Init.end(); I != E; ++I)
+      set(*I);
+  }
+};
+
 //===----------------------------------------------------------------------===//
 ///
 /// SubtargetFeatureKV - Used to provide key value pairs for feature and
@@ -34,8 +52,8 @@ namespace llvm {
 struct SubtargetFeatureKV {
   const char *Key;                      // K-V key string
   const char *Desc;                     // Help descriptor
-  uint64_t Value;                       // K-V integer value
-  uint64_t Implies;                     // K-V bit mask
+  FeatureBitset Value;                  // K-V integer value
+  FeatureBitset Implies;                // K-V bit mask
 
   // Compare routine for std::lower_bound
   bool operator<(StringRef S) const {
@@ -82,11 +100,15 @@ class SubtargetFeatures {
 
   /// ToggleFeature - Toggle a feature and returns the newly updated feature
   /// bits.
-  uint64_t ToggleFeature(uint64_t Bits, StringRef String,
+  FeatureBitset ToggleFeature(FeatureBitset Bits, StringRef String,
                          ArrayRef<SubtargetFeatureKV> FeatureTable);
 
+  /// Apply the feature flag and return the newly updated feature bits.
+  FeatureBitset ApplyFeatureFlag(FeatureBitset Bits, StringRef Feature,
+                                 ArrayRef<SubtargetFeatureKV> FeatureTable);
+
   /// Get feature bits of a CPU.
-  uint64_t getFeatureBits(StringRef CPU,
+  FeatureBitset getFeatureBits(StringRef CPU,
                           ArrayRef<SubtargetFeatureKV> CPUTable,
                           ArrayRef<SubtargetFeatureKV> FeatureTable);
 
diff --git a/include/llvm/Object/Archive.h b/include/llvm/Object/Archive.h
index d40dc9db9f5..d7f46551827 100644
--- a/include/llvm/Object/Archive.h
+++ b/include/llvm/Object/Archive.h
@@ -62,6 +62,8 @@ class Archive : public Binary {
       return reinterpret_cast<const ArchiveMemberHeader *>(Data.data());
     }
 
+    bool isThinMember() const;
+
   public:
     Child(const Archive *Parent, const char *Start);
 
@@ -74,6 +76,7 @@ class Archive : public Binary {
       return Data.begin() < other.Data.begin();
     }
 
+    const Archive *getParent() const { return Parent; }
     Child getNext() const;
 
     ErrorOr<StringRef> getName() const;
@@ -94,9 +97,7 @@ class Archive : public Binary {
     /// \return the size in the archive header for this member.
     uint64_t getRawSize() const;
 
-    StringRef getBuffer() const {
-      return StringRef(Data.data() + StartOfFile, getSize());
-    }
+    ErrorOr<StringRef> getBuffer() const;
     uint64_t getChildOffset() const;
 
     ErrorOr<MemoryBufferRef> getMemoryBufferRef() const;
@@ -183,6 +184,7 @@ class Archive : public Binary {
   };
 
   Kind kind() const { return (Kind)Format; }
+  bool isThin() const { return IsThin; }
 
   child_iterator child_begin(bool SkipInternal = true) const;
   child_iterator child_end() const;
@@ -207,6 +209,12 @@ class Archive : public Binary {
 
   bool hasSymbolTable() const;
   child_iterator getSymbolTableChild() const { return SymbolTable; }
+  StringRef getSymbolTable() const {
+    // We know that the symbol table is not an external file,
+    // so we just assert there is no error.
+    return *SymbolTable->getBuffer();
+  }
+  uint32_t getNumberOfSymbols() const;
 
 private:
   child_iterator SymbolTable;
@@ -214,6 +222,7 @@ class Archive : public Binary {
   child_iterator FirstRegular;
   unsigned Format : 2;
   unsigned IsThin : 1;
+  mutable std::vector<std::unique_ptr<MemoryBuffer>> ThinBuffers;
 };
 
 }
diff --git a/include/llvm/Object/ArchiveWriter.h b/include/llvm/Object/ArchiveWriter.h
new file mode 100644
index 00000000000..fc4669de1ba
--- /dev/null
+++ b/include/llvm/Object/ArchiveWriter.h
@@ -0,0 +1,48 @@
+//===- ArchiveWriter.h - ar archive file format writer ----------*- C++ -*-===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// Declares the writeArchive function for writing an archive file.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_OBJECT_ARCHIVEWRITER_H
+#define LLVM_OBJECT_ARCHIVEWRITER_H
+
+#include "llvm/ADT/StringRef.h"
+#include "llvm/Object/Archive.h"
+#include "llvm/Support/FileSystem.h"
+
+namespace llvm {
+
+class NewArchiveIterator {
+  bool IsNewMember;
+  StringRef Name;
+
+  object::Archive::child_iterator OldI;
+
+public:
+  NewArchiveIterator(object::Archive::child_iterator I, StringRef Name);
+  NewArchiveIterator(StringRef FileName);
+  bool isNewMember() const;
+  StringRef getName() const;
+
+  object::Archive::child_iterator getOld() const;
+
+  StringRef getNew() const;
+  llvm::ErrorOr<int> getFD(sys::fs::file_status &NewStatus) const;
+  const sys::fs::file_status &getStatus() const;
+};
+
+std::pair<StringRef, std::error_code>
+writeArchive(StringRef ArcName, std::vector<NewArchiveIterator> &NewMembers,
+             bool WriteSymtab, object::Archive::Kind Kind, bool Deterministic,
+             bool Thin);
+}
+
+#endif
diff --git a/include/llvm/Object/COFF.h b/include/llvm/Object/COFF.h
index 522bf68c6cb..025a9dbc6bc 100644
--- a/include/llvm/Object/COFF.h
+++ b/include/llvm/Object/COFF.h
@@ -249,6 +249,15 @@ struct coff_symbol {
 typedef coff_symbol<support::ulittle16_t> coff_symbol16;
 typedef coff_symbol<support::ulittle32_t> coff_symbol32;
 
+// Contains only common parts of coff_symbol16 and coff_symbol32.
+struct coff_symbol_generic {
+  union {
+    char ShortName[COFF::NameSize];
+    StringTableOffset Offset;
+  } Name;
+  support::ulittle32_t Value;
+};
+
 class COFFSymbolRef {
 public:
   COFFSymbolRef(const coff_symbol16 *CS) : CS16(CS), CS32(nullptr) {}
@@ -259,6 +268,12 @@ class COFFSymbolRef {
     return CS16 ? static_cast<const void *>(CS16) : CS32;
   }
 
+  const coff_symbol_generic *getGeneric() const {
+    if (CS16)
+      return reinterpret_cast<const coff_symbol_generic *>(CS16);
+    return reinterpret_cast<const coff_symbol_generic *>(CS32);
+  }
+
   friend bool operator<(COFFSymbolRef A, COFFSymbolRef B) {
     return A.getRawPtr() < B.getRawPtr();
   }
@@ -314,6 +329,10 @@ class COFFSymbolRef {
     return (getType() & 0xF0) >> COFF::SCT_COMPLEX_TYPE_SHIFT;
   }
 
+  bool isAbsolute() const {
+    return getSectionNumber() == -1;
+  }
+
   bool isExternal() const {
     return getStorageClass() == COFF::IMAGE_SYM_CLASS_EXTERNAL;
   }
@@ -350,6 +369,10 @@ class COFFSymbolRef {
     return getStorageClass() == COFF::IMAGE_SYM_CLASS_FILE;
   }
 
+  bool isSection() const {
+    return getStorageClass() == COFF::IMAGE_SYM_CLASS_SECTION;
+  }
+
   bool isSectionDefinition() const {
     // C++/CLI creates external ABS symbols for non-const appdomain globals.
     // These are also followed by an auxiliary section definition.
@@ -441,6 +464,27 @@ struct coff_aux_clr_token {
   support::ulittle32_t SymbolTableIndex;
 };
 
+struct coff_import_header {
+  support::ulittle16_t Sig1;
+  support::ulittle16_t Sig2;
+  support::ulittle16_t Version;
+  support::ulittle16_t Machine;
+  support::ulittle32_t TimeDateStamp;
+  support::ulittle32_t SizeOfData;
+  support::ulittle16_t OrdinalHint;
+  support::ulittle16_t TypeInfo;
+  int getType() const { return TypeInfo & 0x3; }
+  int getNameType() const { return (TypeInfo >> 2) & 0x7; }
+};
+
+struct coff_import_directory_table_entry {
+  support::ulittle32_t ImportLookupTableRVA;
+  support::ulittle32_t TimeDateStamp;
+  support::ulittle32_t ForwarderChain;
+  support::ulittle32_t NameRVA;
+  support::ulittle32_t ImportAddressTableRVA;
+};
+
 struct coff_load_configuration32 {
   support::ulittle32_t Characteristics;
   support::ulittle32_t TimeDateStamp;
@@ -464,6 +508,29 @@ struct coff_load_configuration32 {
   support::ulittle32_t SEHandlerCount;
 };
 
+struct coff_load_configuration64 {
+  support::ulittle32_t Characteristics;
+  support::ulittle32_t TimeDateStamp;
+  support::ulittle16_t MajorVersion;
+  support::ulittle16_t MinorVersion;
+  support::ulittle32_t GlobalFlagsClear;
+  support::ulittle32_t GlobalFlagsSet;
+  support::ulittle32_t CriticalSectionDefaultTimeout;
+  support::ulittle32_t DeCommitFreeBlockThreshold;
+  support::ulittle32_t DeCommitTotalFreeThreshold;
+  support::ulittle32_t LockPrefixTable;
+  support::ulittle32_t MaximumAllocationSize;
+  support::ulittle32_t VirtualMemoryThreshold;
+  support::ulittle32_t ProcessAffinityMask;
+  support::ulittle32_t ProcessHeapFlags;
+  support::ulittle16_t CSDVersion;
+  support::ulittle16_t Reserved;
+  support::ulittle32_t EditList;
+  support::ulittle64_t SecurityCookie;
+  support::ulittle64_t SEHandlerTable;
+  support::ulittle64_t SEHandlerCount;
+};
+
 struct coff_runtime_function_x64 {
   support::ulittle32_t BeginAddress;
   support::ulittle32_t EndAddress;
@@ -580,14 +647,12 @@ class COFFObjectFile : public ObjectFile {
   }
 protected:
   void moveSymbolNext(DataRefImpl &Symb) const override;
-  std::error_code getSymbolName(DataRefImpl Symb,
-                                StringRef &Res) const override;
-  std::error_code getSymbolAddress(DataRefImpl Symb,
-                                   uint64_t &Res) const override;
-  std::error_code getSymbolSize(DataRefImpl Symb, uint64_t &Res) const override;
+  ErrorOr<StringRef> getSymbolName(DataRefImpl Symb) const override;
+  ErrorOr<uint64_t> getSymbolAddress(DataRefImpl Symb) const override;
+  uint64_t getSymbolValueImpl(DataRefImpl Symb) const override;
+  uint64_t getCommonSymbolSizeImpl(DataRefImpl Symb) const override;
   uint32_t getSymbolFlags(DataRefImpl Symb) const override;
-  std::error_code getSymbolType(DataRefImpl Symb,
-                                SymbolRef::Type &Res) const override;
+  SymbolRef::Type getSymbolType(DataRefImpl Symb) const override;
   std::error_code getSymbolSection(DataRefImpl Symb,
                                    section_iterator &Res) const override;
   void moveSectionNext(DataRefImpl &Sec) const override;
@@ -602,24 +667,15 @@ class COFFObjectFile : public ObjectFile {
   bool isSectionData(DataRefImpl Sec) const override;
   bool isSectionBSS(DataRefImpl Sec) const override;
   bool isSectionVirtual(DataRefImpl Sec) const override;
-  bool sectionContainsSymbol(DataRefImpl Sec, DataRefImpl Symb) const override;
   relocation_iterator section_rel_begin(DataRefImpl Sec) const override;
   relocation_iterator section_rel_end(DataRefImpl Sec) const override;
 
   void moveRelocationNext(DataRefImpl &Rel) const override;
-  std::error_code getRelocationAddress(DataRefImpl Rel,
-                                       uint64_t &Res) const override;
-  std::error_code getRelocationOffset(DataRefImpl Rel,
-                                      uint64_t &Res) const override;
+  uint64_t getRelocationOffset(DataRefImpl Rel) const override;
   symbol_iterator getRelocationSymbol(DataRefImpl Rel) const override;
-  std::error_code getRelocationType(DataRefImpl Rel,
-                                    uint64_t &Res) const override;
-  std::error_code
-  getRelocationTypeName(DataRefImpl Rel,
-                        SmallVectorImpl<char> &Result) const override;
-  std::error_code
-  getRelocationValueString(DataRefImpl Rel,
-                           SmallVectorImpl<char> &Result) const override;
+  uint64_t getRelocationType(DataRefImpl Rel) const override;
+  void getRelocationTypeName(DataRefImpl Rel,
+                             SmallVectorImpl<char> &Result) const override;
 
 public:
   COFFObjectFile(MemoryBufferRef Object, std::error_code &EC);
@@ -632,6 +688,8 @@ class COFFObjectFile : public ObjectFile {
   COFFSymbolRef getCOFFSymbol(const DataRefImpl &Ref) const;
   COFFSymbolRef getCOFFSymbol(const SymbolRef &Symbol) const;
   const coff_relocation *getCOFFRelocation(const RelocationRef &Reloc) const;
+  unsigned getSectionID(SectionRef Sec) const;
+  unsigned getSymbolSectionID(SymbolRef Sym) const;
 
   uint8_t getBytesInAddress() const override;
   StringRef getFileFormatName() const override;
@@ -669,7 +727,7 @@ class COFFObjectFile : public ObjectFile {
       return object_error::parse_failed;
 
     Res = reinterpret_cast<coff_symbol_type *>(getSymbolTable()) + Index;
-    return object_error::success;
+    return std::error_code();
   }
   ErrorOr<COFFSymbolRef> getSymbol(uint32_t index) const {
     if (SymbolTable16) {
@@ -692,9 +750,11 @@ class COFFObjectFile : public ObjectFile {
     if (std::error_code EC = s.getError())
       return EC;
     Res = reinterpret_cast<const T *>(s->getRawPtr());
-    return object_error::success;
+    return std::error_code();
   }
   std::error_code getSymbolName(COFFSymbolRef Symbol, StringRef &Res) const;
+  std::error_code getSymbolName(const coff_symbol_generic *Symbol,
+                                StringRef &Res) const;
 
   ArrayRef<uint8_t> getSymbolAuxData(COFFSymbolRef Symbol) const;
 
@@ -706,6 +766,9 @@ class COFFObjectFile : public ObjectFile {
     llvm_unreachable("null symbol table pointer!");
   }
 
+  iterator_range<const coff_relocation *>
+  getRelocations(const coff_section *Sec) const;
+
   std::error_code getSectionName(const coff_section *Sec, StringRef &Res) const;
   uint64_t getSectionSize(const coff_section *Sec) const;
   std::error_code getSectionContents(const coff_section *Sec,
diff --git a/include/llvm/Object/ELF.h b/include/llvm/Object/ELF.h
index ddabf59f309..4518ffae661 100644
--- a/include/llvm/Object/ELF.h
+++ b/include/llvm/Object/ELF.h
@@ -55,78 +55,6 @@ class ELFFile {
   typedef typename std::conditional<ELFT::Is64Bits,
                                     uint64_t, uint32_t>::type uintX_t;
 
-  /// \brief Iterate over constant sized entities.
-  template <class EntT>
-  class ELFEntityIterator {
-  public:
-    typedef ptrdiff_t difference_type;
-    typedef EntT value_type;
-    typedef std::forward_iterator_tag iterator_category;
-    typedef value_type &reference;
-    typedef value_type *pointer;
-
-    /// \brief Default construct iterator.
-    ELFEntityIterator() : EntitySize(0), Current(nullptr) {}
-    ELFEntityIterator(uintX_t EntSize, const char *Start)
-        : EntitySize(EntSize), Current(Start) {}
-
-    reference operator *() {
-      assert(Current && "Attempted to dereference an invalid iterator!");
-      return *reinterpret_cast<pointer>(Current);
-    }
-
-    pointer operator ->() {
-      assert(Current && "Attempted to dereference an invalid iterator!");
-      return reinterpret_cast<pointer>(Current);
-    }
-
-    bool operator ==(const ELFEntityIterator &Other) {
-      return Current == Other.Current;
-    }
-
-    bool operator !=(const ELFEntityIterator &Other) {
-      return !(*this == Other);
-    }
-
-    ELFEntityIterator &operator ++() {
-      assert(Current && "Attempted to increment an invalid iterator!");
-      Current += EntitySize;
-      return *this;
-    }
-
-    ELFEntityIterator &operator+(difference_type n) {
-      assert(Current && "Attempted to increment an invalid iterator!");
-      Current += (n * EntitySize);
-      return *this;
-    }
-
-    ELFEntityIterator &operator-(difference_type n) {
-      assert(Current && "Attempted to subtract an invalid iterator!");
-      Current -= (n * EntitySize);
-      return *this;
-    }
-
-    ELFEntityIterator operator ++(int) {
-      ELFEntityIterator Tmp = *this;
-      ++*this;
-      return Tmp;
-    }
-
-    difference_type operator -(const ELFEntityIterator &Other) const {
-      assert(EntitySize == Other.EntitySize &&
-             "Subtracting iterators of different EntitySize!");
-      return (Current - Other.Current) / EntitySize;
-    }
-
-    const char *get() const { return Current; }
-
-    uintX_t getEntSize() const { return EntitySize; }
-
-  private:
-    uintX_t EntitySize;
-    const char *Current;
-  };
-
   typedef Elf_Ehdr_Impl<ELFT> Elf_Ehdr;
   typedef Elf_Shdr_Impl<ELFT> Elf_Shdr;
   typedef Elf_Sym_Impl<ELFT> Elf_Sym;
@@ -139,12 +67,9 @@ class ELFFile {
   typedef Elf_Verneed_Impl<ELFT> Elf_Verneed;
   typedef Elf_Vernaux_Impl<ELFT> Elf_Vernaux;
   typedef Elf_Versym_Impl<ELFT> Elf_Versym;
-  typedef ELFEntityIterator<const Elf_Dyn> Elf_Dyn_Iter;
-  typedef iterator_range<Elf_Dyn_Iter> Elf_Dyn_Range;
-  typedef ELFEntityIterator<const Elf_Rela> Elf_Rela_Iter;
-  typedef ELFEntityIterator<const Elf_Rel> Elf_Rel_Iter;
-  typedef ELFEntityIterator<const Elf_Shdr> Elf_Shdr_Iter;
-  typedef iterator_range<Elf_Shdr_Iter> Elf_Shdr_Range;
+  typedef Elf_Hash_Impl<ELFT> Elf_Hash;
+  typedef iterator_range<const Elf_Dyn *> Elf_Dyn_Range;
+  typedef iterator_range<const Elf_Shdr *> Elf_Shdr_Range;
 
   /// \brief Archive files are 2 byte aligned, so we need this for
   ///     PointerIntPair to work.
@@ -158,74 +83,11 @@ class ELFFile {
     enum { NumLowBitsAvailable = 1 };
   };
 
-  class Elf_Sym_Iter {
-  public:
-    typedef ptrdiff_t difference_type;
-    typedef const Elf_Sym value_type;
-    typedef std::random_access_iterator_tag iterator_category;
-    typedef value_type &reference;
-    typedef value_type *pointer;
-
-    /// \brief Default construct iterator.
-    Elf_Sym_Iter() : EntitySize(0), Current(0, false) {}
-    Elf_Sym_Iter(uintX_t EntSize, const char *Start, bool IsDynamic)
-        : EntitySize(EntSize), Current(Start, IsDynamic) {}
-
-    reference operator*() {
-      assert(Current.getPointer() &&
-             "Attempted to dereference an invalid iterator!");
-      return *reinterpret_cast<pointer>(Current.getPointer());
-    }
-
-    pointer operator->() {
-      assert(Current.getPointer() &&
-             "Attempted to dereference an invalid iterator!");
-      return reinterpret_cast<pointer>(Current.getPointer());
-    }
-
-    bool operator==(const Elf_Sym_Iter &Other) {
-      return Current == Other.Current;
-    }
-
-    bool operator!=(const Elf_Sym_Iter &Other) { return !(*this == Other); }
-
-    Elf_Sym_Iter &operator++() {
-      assert(Current.getPointer() &&
-             "Attempted to increment an invalid iterator!");
-      Current.setPointer(Current.getPointer() + EntitySize);
-      return *this;
-    }
-
-    Elf_Sym_Iter operator++(int) {
-      Elf_Sym_Iter Tmp = *this;
-      ++*this;
-      return Tmp;
-    }
-
-    Elf_Sym_Iter operator+(difference_type Dist) {
-      assert(Current.getPointer() &&
-             "Attempted to increment an invalid iterator!");
-      Current.setPointer(Current.getPointer() + EntitySize * Dist);
-      return *this;
-    }
-
-    difference_type operator-(const Elf_Sym_Iter &Other) const {
-      assert(EntitySize == Other.EntitySize &&
-             "Subtracting iterators of different EntitySize!");
-      return (Current.getPointer() - Other.Current.getPointer()) / EntitySize;
-    }
-
-    const char *get() const { return Current.getPointer(); }
-
-    bool isDynamic() const { return Current.getInt(); }
+  typedef iterator_range<const Elf_Sym *> Elf_Sym_Range;
 
-    uintX_t getEntSize() const { return EntitySize; }
-
-  private:
-    uintX_t EntitySize;
-    PointerIntPair<const char *, 1, bool,
-                   ArchivePointerTypeTraits<const char> > Current;
-  };
+  const uint8_t *base() const {
+    return reinterpret_cast<const uint8_t *>(Buf.data());
+  }
 
 private:
   typedef SmallVector<const Elf_Shdr *, 2> Sections_t;
@@ -233,42 +95,19 @@ class ELFFile {
 
   StringRef Buf;
 
-  const uint8_t *base() const {
-    return reinterpret_cast<const uint8_t *>(Buf.data());
-  }
-
   const Elf_Ehdr *Header;
-  const Elf_Shdr *SectionHeaderTable;
-  const Elf_Shdr *dot_shstrtab_sec; // Section header string table.
-  const Elf_Shdr *dot_strtab_sec;   // Symbol header string table.
-  const Elf_Shdr *dot_symtab_sec;   // Symbol table section.
+  const Elf_Shdr *SectionHeaderTable = nullptr;
+  StringRef DotShstrtab;                    // Section header string table.
+  StringRef DotStrtab;                      // Symbol header string table.
+  const Elf_Shdr *dot_symtab_sec = nullptr; // Symbol table section.
+  const Elf_Shdr *DotDynSymSec = nullptr;   // Dynamic symbol table section.
 
-  const Elf_Shdr *SymbolTableSectionHeaderIndex;
+  const Elf_Shdr *SymbolTableSectionHeaderIndex = nullptr;
   DenseMap<const Elf_Sym *, ELF::Elf64_Word> ExtendedSymbolTable;
 
-  const Elf_Shdr *dot_gnu_version_sec;   // .gnu.version
-  const Elf_Shdr *dot_gnu_version_r_sec; // .gnu.version_r
-  const Elf_Shdr *dot_gnu_version_d_sec; // .gnu.version_d
-
-  /// \brief Represents a region described by entries in the .dynamic table.
-  struct DynRegionInfo {
-    DynRegionInfo() : Addr(nullptr), Size(0), EntSize(0) {}
-    /// \brief Address in current address space.
-    const void *Addr;
-    /// \brief Size in bytes of the region.
-    uintX_t Size;
-    /// \brief Size of each entity in the region.
-    uintX_t EntSize;
-  };
-
-  DynRegionInfo DynamicRegion;
-  DynRegionInfo DynHashRegion;
-  DynRegionInfo DynStrRegion;
-  DynRegionInfo DynSymRegion;
-
-  // Pointer to SONAME entry in dynamic string table
-  // This is set the first time getLoadName is called.
-  mutable const char *dt_soname;
+  const Elf_Shdr *dot_gnu_version_sec = nullptr;   // .gnu.version
+  const Elf_Shdr *dot_gnu_version_r_sec = nullptr; // .gnu.version_r
+  const Elf_Shdr *dot_gnu_version_d_sec = nullptr; // .gnu.version_d
 
   // Records for each version index the corresponding Verdef or Vernaux entry.
   // This is filled the first time LoadVersionMap() is called.
@@ -301,11 +140,14 @@ class ELFFile {
   const T        *getEntry(uint32_t Section, uint32_t Entry) const;
   template <typename T>
   const T *getEntry(const Elf_Shdr *Section, uint32_t Entry) const;
-  const char     *getString(uint32_t section, uint32_t offset) const;
-  const char     *getString(const Elf_Shdr *section, uint32_t offset) const;
-  const char *getDynamicString(uintX_t Offset) const;
-  ErrorOr<StringRef> getSymbolVersion(const Elf_Shdr *section,
-                                      const Elf_Sym *Symb,
+
+  const Elf_Shdr *getDotSymtabSec() const { return dot_symtab_sec; }
+  const Elf_Shdr *getDotDynSymSec() const { return DotDynSymSec; }
+
+  ErrorOr<StringRef> getStringTable(const Elf_Shdr *Section) const;
+  ErrorOr<StringRef> getStringTableForSymtab(const Elf_Shdr &Section) const;
+
+  ErrorOr<StringRef> getSymbolVersion(StringRef StrTab, const Elf_Sym *Symb,
                                       bool &IsDefault) const;
   void VerifyStrTab(const Elf_Shdr *sh) const;
 
@@ -318,7 +160,7 @@ class ELFFile {
   std::pair<const Elf_Shdr *, const Elf_Sym *>
   getRelocationSymbol(const Elf_Shdr *RelSec, const RelT *Rel) const;
 
-  ELFFile(StringRef Object, std::error_code &ec);
+  ELFFile(StringRef Object, std::error_code &EC);
 
   bool isMipsELF64() const {
     return Header->e_machine == ELF::EM_MIPS &&
@@ -331,104 +173,114 @@ class ELFFile {
       Header->getDataEncoding() == ELF::ELFDATA2LSB;
   }
 
-  Elf_Shdr_Iter begin_sections() const;
-  Elf_Shdr_Iter end_sections() const;
+  const Elf_Shdr *section_begin() const;
+  const Elf_Shdr *section_end() const;
   Elf_Shdr_Range sections() const {
-    return make_range(begin_sections(), end_sections());
+    return make_range(section_begin(), section_end());
+  }
+
+  const Elf_Sym *symbol_begin(const Elf_Shdr *Sec) const {
+    if (!Sec)
+      return nullptr;
+    if (Sec->sh_entsize != sizeof(Elf_Sym))
+      report_fatal_error("Invalid symbol size");
+    return reinterpret_cast<const Elf_Sym *>(base() + Sec->sh_offset);
+  }
+  const Elf_Sym *symbol_end(const Elf_Shdr *Sec) const {
+    if (!Sec)
+      return nullptr;
+    uint64_t Size = Sec->sh_size;
+    if (Size % sizeof(Elf_Sym))
+      report_fatal_error("Invalid symbol table size");
+    return symbol_begin(Sec) + Size / sizeof(Elf_Sym);
+  }
+  Elf_Sym_Range symbols(const Elf_Shdr *Sec) const {
+    return make_range(symbol_begin(Sec), symbol_end(Sec));
   }
 
-  Elf_Sym_Iter begin_symbols() const;
-  Elf_Sym_Iter end_symbols() const;
+  const Elf_Sym *symbol_begin() const { return symbol_begin(dot_symtab_sec); }
+  const Elf_Sym *symbol_end() const { return symbol_end(dot_symtab_sec); }
+  Elf_Sym_Range symbols() const { return symbols(dot_symtab_sec); }
 
-  Elf_Dyn_Iter begin_dynamic_table() const;
-  /// \param NULLEnd use one past the first DT_NULL entry as the end instead of
-  /// the section size.
-  Elf_Dyn_Iter end_dynamic_table(bool NULLEnd = false) const;
-  Elf_Dyn_Range dynamic_table(bool NULLEnd = false) const {
-    return make_range(begin_dynamic_table(), end_dynamic_table(NULLEnd));
+  const Elf_Sym *dynamic_symbol_begin() const {
+    return symbol_begin(DotDynSymSec);
   }
+  const Elf_Sym *dynamic_symbol_end() const { return symbol_end(DotDynSymSec); }
+  Elf_Sym_Range dynamic_symbols() const { return symbols(DotDynSymSec); }
+
+  typedef iterator_range<const Elf_Rela *> Elf_Rela_Range;
 
-  Elf_Sym_Iter begin_dynamic_symbols() const {
-    if (DynSymRegion.Addr)
-      return Elf_Sym_Iter(DynSymRegion.EntSize, (const char *)DynSymRegion.Addr,
-                          true);
-    return Elf_Sym_Iter(0, nullptr, true);
+  const Elf_Rela *rela_begin(const Elf_Shdr *sec) const {
+    if (sec->sh_entsize != sizeof(Elf_Rela))
+      report_fatal_error("Invalid relocation entry size");
+    return reinterpret_cast<const Elf_Rela *>(base() + sec->sh_offset);
   }
 
-  Elf_Sym_Iter end_dynamic_symbols() const {
-    if (DynSymRegion.Addr)
-      return Elf_Sym_Iter(DynSymRegion.EntSize,
-                          (const char *)DynSymRegion.Addr + DynSymRegion.Size,
-                          true);
-    return Elf_Sym_Iter(0, nullptr, true);
+  const Elf_Rela *rela_end(const Elf_Shdr *sec) const {
+    uint64_t Size = sec->sh_size;
+    if (Size % sizeof(Elf_Rela))
+      report_fatal_error("Invalid relocation table size");
+    return rela_begin(sec) + Size / sizeof(Elf_Rela);
   }
 
-  Elf_Rela_Iter begin_rela(const Elf_Shdr *sec) const {
-    return Elf_Rela_Iter(sec->sh_entsize,
-                         (const char *)(base() + sec->sh_offset));
+  Elf_Rela_Range relas(const Elf_Shdr *Sec) const {
+    return make_range(rela_begin(Sec), rela_end(Sec));
   }
 
-  Elf_Rela_Iter end_rela(const Elf_Shdr *sec) const {
-    return Elf_Rela_Iter(
-        sec->sh_entsize,
-        (const char *)(base() + sec->sh_offset + sec->sh_size));
+  const Elf_Rel *rel_begin(const Elf_Shdr *sec) const {
+    if (sec->sh_entsize != sizeof(Elf_Rel))
+      report_fatal_error("Invalid relocation entry size");
+    return reinterpret_cast<const Elf_Rel *>(base() + sec->sh_offset);
   }
 
-  Elf_Rel_Iter begin_rel(const Elf_Shdr *sec) const {
-    return Elf_Rel_Iter(sec->sh_entsize,
-                        (const char *)(base() + sec->sh_offset));
+  const Elf_Rel *rel_end(const Elf_Shdr *sec) const {
+    uint64_t Size = sec->sh_size;
+    if (Size % sizeof(Elf_Rel))
+      report_fatal_error("Invalid relocation table size");
+    return rel_begin(sec) + Size / sizeof(Elf_Rel);
   }
 
-  Elf_Rel_Iter end_rel(const Elf_Shdr *sec) const {
-    return Elf_Rel_Iter(sec->sh_entsize,
-                        (const char *)(base() + sec->sh_offset + sec->sh_size));
+  typedef iterator_range<const Elf_Rel *> Elf_Rel_Range;
+  Elf_Rel_Range rels(const Elf_Shdr *Sec) const {
+    return make_range(rel_begin(Sec), rel_end(Sec));
   }
 
   /// \brief Iterate over program header table.
-  typedef ELFEntityIterator<const Elf_Phdr> Elf_Phdr_Iter;
+  const Elf_Phdr *program_header_begin() const {
+    if (Header->e_phnum && Header->e_phentsize != sizeof(Elf_Phdr))
+      report_fatal_error("Invalid program header size");
+    return reinterpret_cast<const Elf_Phdr *>(base() + Header->e_phoff);
+  }
 
-  Elf_Phdr_Iter begin_program_headers() const {
-    return Elf_Phdr_Iter(Header->e_phentsize,
-                         (const char*)base() + Header->e_phoff);
+  const Elf_Phdr *program_header_end() const {
+    return program_header_begin() + Header->e_phnum;
   }
 
-  Elf_Phdr_Iter end_program_headers() const {
-    return Elf_Phdr_Iter(Header->e_phentsize,
-                         (const char*)base() +
-                           Header->e_phoff +
-                           (Header->e_phnum * Header->e_phentsize));
+  typedef iterator_range<const Elf_Phdr *> Elf_Phdr_Range;
+
+  const Elf_Phdr_Range program_headers() const {
+    return make_range(program_header_begin(), program_header_end());
   }
 
   uint64_t getNumSections() const;
   uintX_t getStringTableIndex() const;
-  ELF::Elf64_Word getSymbolTableIndex(const Elf_Sym *symb) const;
+  ELF::Elf64_Word getExtendedSymbolTableIndex(const Elf_Sym *symb) const;
   const Elf_Ehdr *getHeader() const { return Header; }
-  const Elf_Shdr *getSection(const Elf_Sym *symb) const;
-  const Elf_Shdr *getSection(uint32_t Index) const;
-  const Elf_Sym *getSymbol(uint32_t index) const;
-
-  ErrorOr<StringRef> getSymbolName(Elf_Sym_Iter Sym) const;
-
-  /// \brief Get the name of \p Symb.
-  /// \param SymTab The symbol table section \p Symb is contained in.
-  /// \param Symb The symbol to get the name of.
-  ///
-  /// \p SymTab is used to lookup the string table to use to get the symbol's
-  /// name.
-  ErrorOr<StringRef> getSymbolName(const Elf_Shdr *SymTab,
-                                   const Elf_Sym *Symb) const;
+  ErrorOr<const Elf_Shdr *> getSection(const Elf_Sym *symb) const;
+  ErrorOr<const Elf_Shdr *> getSection(uint32_t Index) const;
+
+  const Elf_Sym *getSymbol(const Elf_Shdr *Sec, uint32_t Index) const {
+    return &*(symbol_begin(Sec) + Index);
+  }
+
   ErrorOr<StringRef> getSectionName(const Elf_Shdr *Section) const;
-  uint64_t getSymbolIndex(const Elf_Sym *sym) const;
   ErrorOr<ArrayRef<uint8_t> > getSectionContents(const Elf_Shdr *Sec) const;
-  StringRef getLoadName() const;
 };
 
-// Use an alignment of 2 for the typedefs since that is the worst case for
-// ELF files in archives.
-typedef ELFFile<ELFType<support::little, 2, false> > ELF32LEFile;
-typedef ELFFile<ELFType<support::little, 2, true> > ELF64LEFile;
-typedef ELFFile<ELFType<support::big, 2, false> > ELF32BEFile;
-typedef ELFFile<ELFType<support::big, 2, true> > ELF64BEFile;
+typedef ELFFile<ELFType<support::little, false>> ELF32LEFile;
+typedef ELFFile<ELFType<support::little, true>> ELF64LEFile;
+typedef ELFFile<ELFType<support::big, false>> ELF32BEFile;
+typedef ELFFile<ELFType<support::big, true>> ELF64BEFile;
 
 // Iterate through the version definitions, and place each Elf_Verdef
 // in the VersionMap according to its index.
@@ -492,7 +344,7 @@ void ELFFile<ELFT>::LoadVersionNeeds(const Elf_Shdr *sec) const {
 template <class ELFT>
 void ELFFile<ELFT>::LoadVersionMap() const {
   // If there is no dynamic symtab or version table, there is nothing to do.
-  if (!DynSymRegion.Addr || !dot_gnu_version_sec)
+  if (!DotDynSymSec || !dot_gnu_version_sec)
     return;
 
   // Has the VersionMap already been loaded?
@@ -512,28 +364,23 @@ void ELFFile<ELFT>::LoadVersionMap() const {
 }
 
 template <class ELFT>
-ELF::Elf64_Word ELFFile<ELFT>::getSymbolTableIndex(const Elf_Sym *symb) const {
-  if (symb->st_shndx == ELF::SHN_XINDEX)
-    return ExtendedSymbolTable.lookup(symb);
-  return symb->st_shndx;
+ELF::Elf64_Word
+ELFFile<ELFT>::getExtendedSymbolTableIndex(const Elf_Sym *symb) const {
+  assert(symb->st_shndx == ELF::SHN_XINDEX);
+  return ExtendedSymbolTable.lookup(symb);
 }
 
 template <class ELFT>
-const typename ELFFile<ELFT>::Elf_Shdr *
+ErrorOr<const typename ELFFile<ELFT>::Elf_Shdr *>
 ELFFile<ELFT>::getSection(const Elf_Sym *symb) const {
-  if (symb->st_shndx == ELF::SHN_XINDEX)
+  uint32_t Index = symb->st_shndx;
+  if (Index == ELF::SHN_XINDEX)
     return getSection(ExtendedSymbolTable.lookup(symb));
-  if (symb->st_shndx >= ELF::SHN_LORESERVE)
+  if (Index == ELF::SHN_UNDEF || Index >= ELF::SHN_LORESERVE)
     return nullptr;
   return getSection(symb->st_shndx);
 }
 
-template <class ELFT>
-const typename ELFFile<ELFT>::Elf_Sym *
-ELFFile<ELFT>::getSymbol(uint32_t Index) const {
-  return &*(begin_symbols() + Index);
-}
-
 template <class ELFT>
 ErrorOr<ArrayRef<uint8_t> >
 ELFFile<ELFT>::getSectionContents(const Elf_Shdr *Sec) const {
@@ -586,20 +433,14 @@ std::pair<const typename ELFFile<ELFT>::Elf_Shdr *,
 ELFFile<ELFT>::getRelocationSymbol(const Elf_Shdr *Sec, const RelT *Rel) const {
   if (!Sec->sh_link)
     return std::make_pair(nullptr, nullptr);
-  const Elf_Shdr *SymTable = getSection(Sec->sh_link);
+  ErrorOr<const Elf_Shdr *> SymTableOrErr = getSection(Sec->sh_link);
+  if (std::error_code EC = SymTableOrErr.getError())
+    report_fatal_error(EC.message());
+  const Elf_Shdr *SymTable = *SymTableOrErr;
   return std::make_pair(
       SymTable, getEntry<Elf_Sym>(SymTable, Rel->getSymbol(isMips64EL())));
 }
 
-// Verify that the last byte in the string table in a null.
-template <class ELFT>
-void ELFFile<ELFT>::VerifyStrTab(const Elf_Shdr *sh) const {
-  const char *strtab = (const char *)base() + sh->sh_offset;
-  if (strtab[sh->sh_size - 1] != 0)
-    // FIXME: Proper error handling.
-    report_fatal_error("String table must end with a null terminator!");
-}
-
 template <class ELFT>
 uint64_t ELFFile<ELFT>::getNumSections() const {
   assert(Header && "Header not initialized!");
@@ -622,17 +463,15 @@ typename ELFFile<ELFT>::uintX_t ELFFile<ELFT>::getStringTableIndex() const {
 }
 
 template <class ELFT>
-ELFFile<ELFT>::ELFFile(StringRef Object, std::error_code &ec)
-    : Buf(Object), SectionHeaderTable(nullptr), dot_shstrtab_sec(nullptr),
-      dot_strtab_sec(nullptr), dot_symtab_sec(nullptr),
-      SymbolTableSectionHeaderIndex(nullptr), dot_gnu_version_sec(nullptr),
-      dot_gnu_version_r_sec(nullptr), dot_gnu_version_d_sec(nullptr),
-      dt_soname(nullptr) {
+ELFFile<ELFT>::ELFFile(StringRef Object, std::error_code &EC)
+    : Buf(Object) {
   const uint64_t FileSize = Buf.size();
 
-  if (sizeof(Elf_Ehdr) > FileSize)
-    // FIXME: Proper error handling.
-    report_fatal_error("File too short!");
+  if (sizeof(Elf_Ehdr) > FileSize) {
+    // File too short!
+    EC = object_error::parse_failed;
+    return;
+  }
 
   Header = reinterpret_cast<const Elf_Ehdr *>(base());
 
@@ -641,203 +480,136 @@ ELFFile<ELFT>::ELFFile(StringRef Object, std::error_code &ec)
 
   const uint64_t SectionTableOffset = Header->e_shoff;
 
-  if (SectionTableOffset + sizeof(Elf_Shdr) > FileSize)
-    // FIXME: Proper error handling.
-    report_fatal_error("Section header table goes past end of file!");
+  if (SectionTableOffset + sizeof(Elf_Shdr) > FileSize) {
+    // Section header table goes past end of file!
+    EC = object_error::parse_failed;
+    return;
+  }
 
   // The getNumSections() call below depends on SectionHeaderTable being set.
   SectionHeaderTable =
     reinterpret_cast<const Elf_Shdr *>(base() + SectionTableOffset);
   const uint64_t SectionTableSize = getNumSections() * Header->e_shentsize;
 
-  if (SectionTableOffset + SectionTableSize > FileSize)
-    // FIXME: Proper error handling.
-    report_fatal_error("Section table goes past end of file!");
+  if (SectionTableOffset + SectionTableSize > FileSize) {
+    // Section table goes past end of file!
+    EC = object_error::parse_failed;
+    return;
+  }
 
   // Scan sections for special sections.
 
   for (const Elf_Shdr &Sec : sections()) {
     switch (Sec.sh_type) {
     case ELF::SHT_SYMTAB_SHNDX:
-      if (SymbolTableSectionHeaderIndex)
-        // FIXME: Proper error handling.
-        report_fatal_error("More than one .symtab_shndx!");
+      if (SymbolTableSectionHeaderIndex) {
+        // More than one .symtab_shndx!
+        EC = object_error::parse_failed;
+        return;
+      }
       SymbolTableSectionHeaderIndex = &Sec;
       break;
-    case ELF::SHT_SYMTAB:
-      if (dot_symtab_sec)
-        // FIXME: Proper error handling.
-        report_fatal_error("More than one .symtab!");
+    case ELF::SHT_SYMTAB: {
+      if (dot_symtab_sec) {
+        // More than one .symtab!
+        EC = object_error::parse_failed;
+        return;
+      }
       dot_symtab_sec = &Sec;
-      dot_strtab_sec = getSection(Sec.sh_link);
-      break;
+      ErrorOr<StringRef> SymtabOrErr = getStringTableForSymtab(Sec);
+      if ((EC = SymtabOrErr.getError()))
+        return;
+      DotStrtab = *SymtabOrErr;
+    } break;
     case ELF::SHT_DYNSYM: {
-      if (DynSymRegion.Addr)
-        // FIXME: Proper error handling.
-        report_fatal_error("More than one .dynsym!");
-      DynSymRegion.Addr = base() + Sec.sh_offset;
-      DynSymRegion.Size = Sec.sh_size;
-      DynSymRegion.EntSize = Sec.sh_entsize;
-      const Elf_Shdr *DynStr = getSection(Sec.sh_link);
-      DynStrRegion.Addr = base() + DynStr->sh_offset;
-      DynStrRegion.Size = DynStr->sh_size;
-      DynStrRegion.EntSize = DynStr->sh_entsize;
+      if (DotDynSymSec) {
+        // More than one .dynsym!
+        EC = object_error::parse_failed;
+        return;
+      }
+      DotDynSymSec = &Sec;
       break;
     }
-    case ELF::SHT_DYNAMIC:
-      if (DynamicRegion.Addr)
-        // FIXME: Proper error handling.
-        report_fatal_error("More than one .dynamic!");
-      DynamicRegion.Addr = base() + Sec.sh_offset;
-      DynamicRegion.Size = Sec.sh_size;
-      DynamicRegion.EntSize = Sec.sh_entsize;
-      break;
     case ELF::SHT_GNU_versym:
-      if (dot_gnu_version_sec != nullptr)
-        // FIXME: Proper error handling.
-        report_fatal_error("More than one .gnu.version section!");
+      if (dot_gnu_version_sec != nullptr) {
+        // More than one .gnu.version section!
+        EC = object_error::parse_failed;
+        return;
+      }
       dot_gnu_version_sec = &Sec;
       break;
     case ELF::SHT_GNU_verdef:
-      if (dot_gnu_version_d_sec != nullptr)
-        // FIXME: Proper error handling.
-        report_fatal_error("More than one .gnu.version_d section!");
+      if (dot_gnu_version_d_sec != nullptr) {
+        // More than one .gnu.version_d section!
+        EC = object_error::parse_failed;
+        return;
+      }
       dot_gnu_version_d_sec = &Sec;
       break;
     case ELF::SHT_GNU_verneed:
-      if (dot_gnu_version_r_sec != nullptr)
-        // FIXME: Proper error handling.
-        report_fatal_error("More than one .gnu.version_r section!");
+      if (dot_gnu_version_r_sec != nullptr) {
+        // More than one .gnu.version_r section!
+        EC = object_error::parse_failed;
+        return;
+      }
       dot_gnu_version_r_sec = &Sec;
       break;
     }
   }
 
   // Get string table sections.
-  dot_shstrtab_sec = getSection(getStringTableIndex());
-  if (dot_shstrtab_sec) {
-    // Verify that the last byte in the string table in a null.
-    VerifyStrTab(dot_shstrtab_sec);
+  uintX_t StringTableIndex = getStringTableIndex();
+  if (StringTableIndex) {
+    ErrorOr<const Elf_Shdr *> StrTabSecOrErr =
+        getSection(getStringTableIndex());
+    if ((EC = StrTabSecOrErr.getError()))
+      return;
+
+    ErrorOr<StringRef> SymtabOrErr = getStringTable(*StrTabSecOrErr);
+    if ((EC = SymtabOrErr.getError()))
+      return;
+    DotShstrtab = *SymtabOrErr;
   }
 
   // Build symbol name side-mapping if there is one.
   if (SymbolTableSectionHeaderIndex) {
     const Elf_Word *ShndxTable = reinterpret_cast<const Elf_Word*>(base() +
                                       SymbolTableSectionHeaderIndex->sh_offset);
-    for (Elf_Sym_Iter SI = begin_symbols(), SE = end_symbols(); SI != SE;
-         ++SI) {
+    for (const Elf_Sym &S : symbols()) {
       if (*ShndxTable != ELF::SHN_UNDEF)
-        ExtendedSymbolTable[&*SI] = *ShndxTable;
+        ExtendedSymbolTable[&S] = *ShndxTable;
       ++ShndxTable;
     }
   }
 
-  // Scan program headers.
-  for (Elf_Phdr_Iter PhdrI = begin_program_headers(),
-                     PhdrE = end_program_headers();
-       PhdrI != PhdrE; ++PhdrI) {
-    if (PhdrI->p_type == ELF::PT_DYNAMIC) {
-      DynamicRegion.Addr = base() + PhdrI->p_offset;
-      DynamicRegion.Size = PhdrI->p_filesz;
-      DynamicRegion.EntSize = sizeof(Elf_Dyn);
-      break;
-    }
-  }
-
-  ec = std::error_code();
-}
-
-// Get the symbol table index in the symtab section given a symbol
-template <class ELFT>
-uint64_t ELFFile<ELFT>::getSymbolIndex(const Elf_Sym *Sym) const {
-  uintptr_t SymLoc = uintptr_t(Sym);
-  uintptr_t SymTabLoc = uintptr_t(base() + dot_symtab_sec->sh_offset);
-  assert(SymLoc > SymTabLoc && "Symbol not in symbol table!");
-  uint64_t SymOffset = SymLoc - SymTabLoc;
-  assert(SymOffset % dot_symtab_sec->sh_entsize == 0 &&
-         "Symbol not multiple of symbol size!");
-  return SymOffset / dot_symtab_sec->sh_entsize;
-}
-
-template <class ELFT>
-typename ELFFile<ELFT>::Elf_Shdr_Iter ELFFile<ELFT>::begin_sections() const {
-  return Elf_Shdr_Iter(Header->e_shentsize,
-                       (const char *)base() + Header->e_shoff);
+  EC = std::error_code();
 }
 
 template <class ELFT>
-typename ELFFile<ELFT>::Elf_Shdr_Iter ELFFile<ELFT>::end_sections() const {
-  return Elf_Shdr_Iter(Header->e_shentsize,
-                       (const char *)base() + Header->e_shoff +
-                           (getNumSections() * Header->e_shentsize));
+static bool compareAddr(uint64_t VAddr, const Elf_Phdr_Impl<ELFT> *Phdr) {
+  return VAddr < Phdr->p_vaddr;
 }
 
 template <class ELFT>
-typename ELFFile<ELFT>::Elf_Sym_Iter ELFFile<ELFT>::begin_symbols() const {
-  if (!dot_symtab_sec)
-    return Elf_Sym_Iter(0, nullptr, false);
-  return Elf_Sym_Iter(dot_symtab_sec->sh_entsize,
-                      (const char *)base() + dot_symtab_sec->sh_offset, false);
+const typename ELFFile<ELFT>::Elf_Shdr *ELFFile<ELFT>::section_begin() const {
+  if (Header->e_shentsize != sizeof(Elf_Shdr))
+    report_fatal_error(
+        "Invalid section header entry size (e_shentsize) in ELF header");
+  return reinterpret_cast<const Elf_Shdr *>(base() + Header->e_shoff);
 }
 
 template <class ELFT>
-typename ELFFile<ELFT>::Elf_Sym_Iter ELFFile<ELFT>::end_symbols() const {
-  if (!dot_symtab_sec)
-    return Elf_Sym_Iter(0, nullptr, false);
-  return Elf_Sym_Iter(dot_symtab_sec->sh_entsize,
-                      (const char *)base() + dot_symtab_sec->sh_offset +
-                          dot_symtab_sec->sh_size,
-                      false);
-}
-
-template <class ELFT>
-typename ELFFile<ELFT>::Elf_Dyn_Iter
-ELFFile<ELFT>::begin_dynamic_table() const {
-  if (DynamicRegion.Addr)
-    return Elf_Dyn_Iter(DynamicRegion.EntSize,
-                        (const char *)DynamicRegion.Addr);
-  return Elf_Dyn_Iter(0, nullptr);
-}
-
-template <class ELFT>
-typename ELFFile<ELFT>::Elf_Dyn_Iter
-ELFFile<ELFT>::end_dynamic_table(bool NULLEnd) const {
-  if (!DynamicRegion.Addr)
-    return Elf_Dyn_Iter(0, nullptr);
-  Elf_Dyn_Iter Ret(DynamicRegion.EntSize,
-                    (const char *)DynamicRegion.Addr + DynamicRegion.Size);
-
-  if (NULLEnd) {
-    Elf_Dyn_Iter Start = begin_dynamic_table();
-    while (Start != Ret && Start->getTag() != ELF::DT_NULL)
-      ++Start;
-
-    // Include the DT_NULL.
-    if (Start != Ret)
-      ++Start;
-    Ret = Start;
-  }
-  return Ret;
-}
-
-template <class ELFT>
-StringRef ELFFile<ELFT>::getLoadName() const {
-  if (!dt_soname) {
-    dt_soname = "";
-    // Find the DT_SONAME entry
-    for (const auto &Entry : dynamic_table())
-      if (Entry.getTag() == ELF::DT_SONAME) {
-        dt_soname = getDynamicString(Entry.getVal());
-        break;
-      }
-  }
-  return dt_soname;
+const typename ELFFile<ELFT>::Elf_Shdr *ELFFile<ELFT>::section_end() const {
+  return section_begin() + getNumSections();
 }
 
 template <class ELFT>
 template <typename T>
 const T *ELFFile<ELFT>::getEntry(uint32_t Section, uint32_t Entry) const {
-  return getEntry<T>(getSection(Section), Entry);
+  ErrorOr<const Elf_Shdr *> Sec = getSection(Section);
+  if (std::error_code EC = Sec.getError())
+    report_fatal_error(EC.message());
+  return getEntry<T>(*Sec, Entry);
 }
 
 template <class ELFT>
@@ -849,103 +621,58 @@ const T *ELFFile<ELFT>::getEntry(const Elf_Shdr *Section,
 }
 
 template <class ELFT>
-const typename ELFFile<ELFT>::Elf_Shdr *
-ELFFile<ELFT>::getSection(uint32_t index) const {
-  if (index == 0)
-    return nullptr;
-  if (!SectionHeaderTable || index >= getNumSections())
-    // FIXME: Proper error handling.
-    report_fatal_error("Invalid section index!");
+ErrorOr<const typename ELFFile<ELFT>::Elf_Shdr *>
+ELFFile<ELFT>::getSection(uint32_t Index) const {
+  assert(SectionHeaderTable && "SectionHeaderTable not initialized!");
+  if (Index >= getNumSections())
+    return object_error::invalid_section_index;
 
   return reinterpret_cast<const Elf_Shdr *>(
-         reinterpret_cast<const char *>(SectionHeaderTable)
-         + (index * Header->e_shentsize));
-}
-
-template <class ELFT>
-const char *ELFFile<ELFT>::getString(uint32_t section,
-                                     ELF::Elf32_Word offset) const {
-  return getString(getSection(section), offset);
+      reinterpret_cast<const char *>(SectionHeaderTable) +
+      (Index * Header->e_shentsize));
 }
 
 template <class ELFT>
-const char *ELFFile<ELFT>::getString(const Elf_Shdr *section,
-                                     ELF::Elf32_Word offset) const {
-  assert(section && section->sh_type == ELF::SHT_STRTAB && "Invalid section!");
-  if (offset >= section->sh_size)
-    // FIXME: Proper error handling.
-    report_fatal_error("Symbol name offset outside of string table!");
-  return (const char *)base() + section->sh_offset + offset;
-}
-
-template <class ELFT>
-const char *ELFFile<ELFT>::getDynamicString(uintX_t Offset) const {
-  if (!DynStrRegion.Addr || Offset >= DynStrRegion.Size)
-    return nullptr;
-  return (const char *)DynStrRegion.Addr + Offset;
-}
-
-template <class ELFT>
-ErrorOr<StringRef> ELFFile<ELFT>::getSymbolName(Elf_Sym_Iter Sym) const {
-  if (!Sym.isDynamic())
-    return getSymbolName(dot_symtab_sec, &*Sym);
-
-  if (!DynStrRegion.Addr || Sym->st_name >= DynStrRegion.Size)
+ErrorOr<StringRef>
+ELFFile<ELFT>::getStringTable(const Elf_Shdr *Section) const {
+  if (Section->sh_type != ELF::SHT_STRTAB)
+    return object_error::parse_failed;
+  uint64_t Offset = Section->sh_offset;
+  uint64_t Size = Section->sh_size;
+  if (Offset + Size > Buf.size())
     return object_error::parse_failed;
-  return StringRef(getDynamicString(Sym->st_name));
+  StringRef Data((const char *)base() + Section->sh_offset, Size);
+  if (Data[Size - 1] != '\0')
+    return object_error::string_table_non_null_end;
+  return Data;
 }
 
 template <class ELFT>
-ErrorOr<StringRef> ELFFile<ELFT>::getSymbolName(const Elf_Shdr *Section,
-                                                const Elf_Sym *Symb) const {
-  if (Symb->st_name == 0) {
-    const Elf_Shdr *ContainingSec = getSection(Symb);
-    if (ContainingSec)
-      return getSectionName(ContainingSec);
-  }
-
-  const Elf_Shdr *StrTab = getSection(Section->sh_link);
-  if (Symb->st_name >= StrTab->sh_size)
+ErrorOr<StringRef>
+ELFFile<ELFT>::getStringTableForSymtab(const Elf_Shdr &Sec) const {
+  if (Sec.sh_type != ELF::SHT_SYMTAB && Sec.sh_type != ELF::SHT_DYNSYM)
     return object_error::parse_failed;
-  return StringRef(getString(StrTab, Symb->st_name));
+  ErrorOr<const Elf_Shdr *> SectionOrErr = getSection(Sec.sh_link);
+  if (std::error_code EC = SectionOrErr.getError())
+    return EC;
+  return getStringTable(*SectionOrErr);
 }
 
 template <class ELFT>
 ErrorOr<StringRef>
 ELFFile<ELFT>::getSectionName(const Elf_Shdr *Section) const {
-  if (Section->sh_name >= dot_shstrtab_sec->sh_size)
+  uint32_t Offset = Section->sh_name;
+  if (Offset == 0)
+    return StringRef();
+  if (Offset >= DotShstrtab.size())
     return object_error::parse_failed;
-  return StringRef(getString(dot_shstrtab_sec, Section->sh_name));
+  return StringRef(DotShstrtab.data() + Offset);
 }
 
 template <class ELFT>
-ErrorOr<StringRef> ELFFile<ELFT>::getSymbolVersion(const Elf_Shdr *section,
+ErrorOr<StringRef> ELFFile<ELFT>::getSymbolVersion(StringRef StrTab,
                                                    const Elf_Sym *symb,
                                                    bool &IsDefault) const {
-  // Handle non-dynamic symbols.
-  if (section != DynSymRegion.Addr && section != nullptr) {
-    // Non-dynamic symbols can have versions in their names
-    // A name of the form 'foo@V1' indicates version 'V1', non-default.
-    // A name of the form 'foo@@V2' indicates version 'V2', default version.
-    ErrorOr<StringRef> SymName = getSymbolName(section, symb);
-    if (!SymName)
-      return SymName;
-    StringRef Name = *SymName;
-    size_t atpos = Name.find('@');
-    if (atpos == StringRef::npos) {
-      IsDefault = false;
-      return StringRef("");
-    }
-    ++atpos;
-    if (atpos < Name.size() && Name[atpos] == '@') {
-      IsDefault = true;
-      ++atpos;
-    } else {
-      IsDefault = false;
-    }
-    return Name.substr(atpos);
-  }
-
   // This is a dynamic symbol. Look in the GNU symbol version table.
   if (!dot_gnu_version_sec) {
     // No version table.
@@ -954,8 +681,10 @@ ErrorOr<StringRef> ELFFile<ELFT>::getSymbolVersion(const Elf_Shdr *section,
   }
 
   // Determine the position in the symbol table of this entry.
-  size_t entry_index = ((const char *)symb - (const char *)DynSymRegion.Addr) /
-                       DynSymRegion.EntSize;
+  size_t entry_index =
+      (reinterpret_cast<uintptr_t>(symb) - DotDynSymSec->sh_offset -
+       reinterpret_cast<uintptr_t>(base())) /
+      sizeof(Elf_Sym);
 
   // Get the corresponding version index entry
   const Elf_Versym *vs = getEntry<Elf_Versym>(dot_gnu_version_sec, entry_index);
@@ -990,9 +719,9 @@ ErrorOr<StringRef> ELFFile<ELFT>::getSymbolVersion(const Elf_Shdr *section,
     IsDefault = false;
   }
 
-  if (name_offset >= DynStrRegion.Size)
+  if (name_offset >= StrTab.size())
     return object_error::parse_failed;
-  return StringRef(getDynamicString(name_offset));
+  return StringRef(StrTab.data() + name_offset);
 }
 
 /// This function returns the hash value for a symbol in the .dynsym section
diff --git a/include/llvm/Object/ELFObjectFile.h b/include/llvm/Object/ELFObjectFile.h
index e7eba976fd4..2a870bdbca9 100644
--- a/include/llvm/Object/ELFObjectFile.h
+++ b/include/llvm/Object/ELFObjectFile.h
@@ -35,26 +35,148 @@
 namespace llvm {
 namespace object {
 
+class elf_symbol_iterator;
+class ELFSymbolRef;
+class ELFRelocationRef;
+
 class ELFObjectFileBase : public ObjectFile {
+  friend class ELFSymbolRef;
+  friend class ELFSectionRef;
+  friend class ELFRelocationRef;
+
 protected:
   ELFObjectFileBase(unsigned int Type, MemoryBufferRef Source);
 
+  virtual uint64_t getSymbolSize(DataRefImpl Symb) const = 0;
+  virtual uint8_t getSymbolOther(DataRefImpl Symb) const = 0;
+  virtual uint8_t getSymbolELFType(DataRefImpl Symb) const = 0;
+
+  virtual uint32_t getSectionType(DataRefImpl Sec) const = 0;
+  virtual uint64_t getSectionFlags(DataRefImpl Sec) const = 0;
+
+  virtual ErrorOr<int64_t> getRelocationAddend(DataRefImpl Rel) const = 0;
 public:
-  virtual std::error_code getRelocationAddend(DataRefImpl Rel,
-                                              int64_t &Res) const = 0;
-  virtual std::pair<symbol_iterator, symbol_iterator>
-  getELFDynamicSymbolIterators() const = 0;
 
-  virtual std::error_code getSymbolVersion(SymbolRef Symb, StringRef &Version,
-                                           bool &IsDefault) const = 0;
+  typedef iterator_range<elf_symbol_iterator> elf_symbol_iterator_range;
+  virtual elf_symbol_iterator_range getDynamicSymbolIterators() const = 0;
 
-  virtual uint64_t getSectionFlags(SectionRef Sec) const = 0;
-  virtual uint32_t getSectionType(SectionRef Sec) const = 0;
+  elf_symbol_iterator_range symbols() const;
 
   static inline bool classof(const Binary *v) { return v->isELF(); }
 };
 
+class ELFSectionRef : public SectionRef {
+public:
+  ELFSectionRef(const SectionRef &B) : SectionRef(B) {
+    assert(isa<ELFObjectFileBase>(SectionRef::getObject()));
+  }
+
+  const ELFObjectFileBase *getObject() const {
+    return cast<ELFObjectFileBase>(SectionRef::getObject());
+  }
+
+  uint32_t getType() const {
+    return getObject()->getSectionType(getRawDataRefImpl());
+  }
+
+  uint64_t getFlags() const {
+    return getObject()->getSectionFlags(getRawDataRefImpl());
+  }
+};
+
+class elf_section_iterator : public section_iterator {
+public:
+  elf_section_iterator(const section_iterator &B) : section_iterator(B) {
+    assert(isa<ELFObjectFileBase>(B->getObject()));
+  }
+
+  const ELFSectionRef *operator->() const {
+    return static_cast<const ELFSectionRef *>(section_iterator::operator->());
+  }
+
+  const ELFSectionRef &operator*() const {
+    return static_cast<const ELFSectionRef &>(section_iterator::operator*());
+  }
+};
+
+class ELFSymbolRef : public SymbolRef {
+public:
+  ELFSymbolRef(const SymbolRef &B) : SymbolRef(B) {
+    assert(isa<ELFObjectFileBase>(SymbolRef::getObject()));
+  }
+
+  const ELFObjectFileBase *getObject() const {
+    return cast<ELFObjectFileBase>(BasicSymbolRef::getObject());
+  }
+
+  uint64_t getSize() const {
+    return getObject()->getSymbolSize(getRawDataRefImpl());
+  }
+
+  uint8_t getOther() const {
+    return getObject()->getSymbolOther(getRawDataRefImpl());
+  }
+
+  uint8_t getELFType() const {
+    return getObject()->getSymbolELFType(getRawDataRefImpl());
+  }
+};
+
+class elf_symbol_iterator : public symbol_iterator {
+public:
+  elf_symbol_iterator(const basic_symbol_iterator &B)
+      : symbol_iterator(SymbolRef(B->getRawDataRefImpl(),
+                                  cast<ELFObjectFileBase>(B->getObject()))) {}
+
+  const ELFSymbolRef *operator->() const {
+    return static_cast<const ELFSymbolRef *>(symbol_iterator::operator->());
+  }
+
+  const ELFSymbolRef &operator*() const {
+    return static_cast<const ELFSymbolRef &>(symbol_iterator::operator*());
+  }
+};
+
+class ELFRelocationRef : public RelocationRef {
+public:
+  ELFRelocationRef(const RelocationRef &B) : RelocationRef(B) {
+    assert(isa<ELFObjectFileBase>(RelocationRef::getObject()));
+  }
+
+  const ELFObjectFileBase *getObject() const {
+    return cast<ELFObjectFileBase>(RelocationRef::getObject());
+  }
+
+  ErrorOr<int64_t> getAddend() const {
+    return getObject()->getRelocationAddend(getRawDataRefImpl());
+  }
+};
+
+class elf_relocation_iterator : public relocation_iterator {
+public:
+  elf_relocation_iterator(const relocation_iterator &B)
+      : relocation_iterator(RelocationRef(
+            B->getRawDataRefImpl(), cast<ELFObjectFileBase>(B->getObject()))) {}
+
+  const ELFRelocationRef *operator->() const {
+    return static_cast<const ELFRelocationRef *>(
+        relocation_iterator::operator->());
+  }
+
+  const ELFRelocationRef &operator*() const {
+    return static_cast<const ELFRelocationRef &>(
+        relocation_iterator::operator*());
+  }
+};
+
+inline ELFObjectFileBase::elf_symbol_iterator_range
+ELFObjectFileBase::symbols() const {
+  return elf_symbol_iterator_range(symbol_begin(), symbol_end());
+}
+
 template <class ELFT> class ELFObjectFile : public ELFObjectFileBase {
+  uint64_t getSymbolSize(DataRefImpl Sym) const override;
+
 public:
   LLVM_ELF_IMPORT_TYPES_ELFT(ELFT)
 
@@ -67,25 +189,20 @@ template <class ELFT> class ELFObjectFile : public ELFObjectFileBase {
   typedef typename ELFFile<ELFT>::Elf_Rela Elf_Rela;
   typedef typename ELFFile<ELFT>::Elf_Dyn Elf_Dyn;
 
-  typedef typename ELFFile<ELFT>::Elf_Sym_Iter Elf_Sym_Iter;
-  typedef typename ELFFile<ELFT>::Elf_Shdr_Iter Elf_Shdr_Iter;
-  typedef typename ELFFile<ELFT>::Elf_Dyn_Iter Elf_Dyn_Iter;
-
 protected:
   ELFFile<ELFT> EF;
 
   void moveSymbolNext(DataRefImpl &Symb) const override;
-  std::error_code getSymbolName(DataRefImpl Symb,
-                                StringRef &Res) const override;
-  std::error_code getSymbolAddress(DataRefImpl Symb,
-                                   uint64_t &Res) const override;
-  std::error_code getSymbolAlignment(DataRefImpl Symb,
-                                     uint32_t &Res) const override;
-  std::error_code getSymbolSize(DataRefImpl Symb, uint64_t &Res) const override;
+  ErrorOr<StringRef> getSymbolName(DataRefImpl Symb) const override;
+  ErrorOr<uint64_t> getSymbolAddress(DataRefImpl Symb) const override;
+  uint64_t getSymbolValueImpl(DataRefImpl Symb) const override;
+  uint32_t getSymbolAlignment(DataRefImpl Symb) const override;
+  uint64_t getCommonSymbolSizeImpl(DataRefImpl Symb) const override;
   uint32_t getSymbolFlags(DataRefImpl Symb) const override;
-  std::error_code getSymbolOther(DataRefImpl Symb, uint8_t &Res) const override;
-  std::error_code getSymbolType(DataRefImpl Symb,
-                                SymbolRef::Type &Res) const override;
+  uint8_t getSymbolOther(DataRefImpl Symb) const override;
+  uint8_t getSymbolELFType(DataRefImpl Symb) const override;
+  SymbolRef::Type getSymbolType(DataRefImpl Symb) const override;
+  section_iterator getSymbolSection(const Elf_Sym *Symb) const;
   std::error_code getSymbolSection(DataRefImpl Symb,
                                    section_iterator &Res) const override;
 
@@ -101,79 +218,62 @@ template <class ELFT> class ELFObjectFile : public ELFObjectFileBase {
   bool isSectionData(DataRefImpl Sec) const override;
   bool isSectionBSS(DataRefImpl Sec) const override;
   bool isSectionVirtual(DataRefImpl Sec) const override;
-  bool sectionContainsSymbol(DataRefImpl Sec, DataRefImpl Symb) const override;
   relocation_iterator section_rel_begin(DataRefImpl Sec) const override;
   relocation_iterator section_rel_end(DataRefImpl Sec) const override;
   section_iterator getRelocatedSection(DataRefImpl Sec) const override;
 
   void moveRelocationNext(DataRefImpl &Rel) const override;
-  std::error_code getRelocationAddress(DataRefImpl Rel,
-                                       uint64_t &Res) const override;
-  std::error_code getRelocationOffset(DataRefImpl Rel,
-                                      uint64_t &Res) const override;
+  uint64_t getRelocationOffset(DataRefImpl Rel) const override;
   symbol_iterator getRelocationSymbol(DataRefImpl Rel) const override;
-  std::error_code getRelocationType(DataRefImpl Rel,
-                                    uint64_t &Res) const override;
-  std::error_code
-  getRelocationTypeName(DataRefImpl Rel,
-                        SmallVectorImpl<char> &Result) const override;
-  std::error_code
-  getRelocationValueString(DataRefImpl Rel,
-                           SmallVectorImpl<char> &Result) const override;
-
-  uint64_t getROffset(DataRefImpl Rel) const;
+  uint64_t getRelocationType(DataRefImpl Rel) const override;
+  void getRelocationTypeName(DataRefImpl Rel,
+                             SmallVectorImpl<char> &Result) const override;
+
+  uint32_t getSectionType(DataRefImpl Sec) const override;
+  uint64_t getSectionFlags(DataRefImpl Sec) const override;
   StringRef getRelocationTypeName(uint32_t Type) const;
 
   /// \brief Get the relocation section that contains \a Rel.
   const Elf_Shdr *getRelSection(DataRefImpl Rel) const {
-    return EF.getSection(Rel.d.a);
+    return *EF.getSection(Rel.d.a);
   }
 
-  const Elf_Rel *getRel(DataRefImpl Rel) const;
-  const Elf_Rela *getRela(DataRefImpl Rela) const;
-
-  Elf_Sym_Iter toELFSymIter(DataRefImpl Symb) const {
-    bool IsDynamic = Symb.p & 1;
-    if (IsDynamic)
-      return Elf_Sym_Iter(
-          EF.begin_dynamic_symbols().getEntSize(),
-          reinterpret_cast<const char *>(Symb.p & ~uintptr_t(1)), IsDynamic);
-    return Elf_Sym_Iter(EF.begin_symbols().getEntSize(),
-                        reinterpret_cast<const char *>(Symb.p), IsDynamic);
+  const Elf_Sym *toELFSymIter(DataRefImpl Sym) const {
+    return EF.template getEntry<Elf_Sym>(Sym.d.a, Sym.d.b);
   }
 
-  DataRefImpl toDRI(Elf_Sym_Iter Symb) const {
+  DataRefImpl toDRI(const Elf_Shdr *SymTable, unsigned SymbolNum) const {
     DataRefImpl DRI;
-    DRI.p = reinterpret_cast<uintptr_t>(Symb.get()) |
-      static_cast<uintptr_t>(Symb.isDynamic());
-    return DRI;
-  }
+    if (!SymTable) {
+      DRI.d.a = 0;
+      DRI.d.b = 0;
+      return DRI;
+    }
+    assert(SymTable->sh_type == ELF::SHT_SYMTAB ||
+           SymTable->sh_type == ELF::SHT_DYNSYM);
 
-  Elf_Shdr_Iter toELFShdrIter(DataRefImpl Sec) const {
-    return Elf_Shdr_Iter(EF.getHeader()->e_shentsize,
-                         reinterpret_cast<const char *>(Sec.p));
-  }
+    uintptr_t SHT = reinterpret_cast<uintptr_t>(EF.section_begin());
+    unsigned SymTableIndex =
+        (reinterpret_cast<uintptr_t>(SymTable) - SHT) / sizeof(Elf_Shdr);
 
-  DataRefImpl toDRI(Elf_Shdr_Iter Sec) const {
-    DataRefImpl DRI;
-    DRI.p = reinterpret_cast<uintptr_t>(Sec.get());
+    DRI.d.a = SymTableIndex;
+    DRI.d.b = SymbolNum;
     return DRI;
   }
 
+  const Elf_Shdr *toELFShdrIter(DataRefImpl Sec) const {
+    return reinterpret_cast<const Elf_Shdr *>(Sec.p);
+  }
+
   DataRefImpl toDRI(const Elf_Shdr *Sec) const {
     DataRefImpl DRI;
     DRI.p = reinterpret_cast<uintptr_t>(Sec);
     return DRI;
   }
 
-  Elf_Dyn_Iter toELFDynIter(DataRefImpl Dyn) const {
-    return Elf_Dyn_Iter(EF.begin_dynamic_table().getEntSize(),
-                        reinterpret_cast<const char *>(Dyn.p));
-  }
-
-  DataRefImpl toDRI(Elf_Dyn_Iter Dyn) const {
+  DataRefImpl toDRI(const Elf_Dyn *Dyn) const {
     DataRefImpl DRI;
-    DRI.p = reinterpret_cast<uintptr_t>(Dyn.get());
+    DRI.p = reinterpret_cast<uintptr_t>(Dyn);
     return DRI;
   }
 
@@ -199,33 +299,29 @@ template <class ELFT> class ELFObjectFile : public ELFObjectFileBase {
 public:
   ELFObjectFile(MemoryBufferRef Object, std::error_code &EC);
 
+  const Elf_Rel *getRel(DataRefImpl Rel) const;
+  const Elf_Rela *getRela(DataRefImpl Rela) const;
+
   const Elf_Sym *getSymbol(DataRefImpl Symb) const;
 
   basic_symbol_iterator symbol_begin_impl() const override;
   basic_symbol_iterator symbol_end_impl() const override;
 
-  symbol_iterator dynamic_symbol_begin() const;
-  symbol_iterator dynamic_symbol_end() const;
+  elf_symbol_iterator dynamic_symbol_begin() const;
+  elf_symbol_iterator dynamic_symbol_end() const;
 
   section_iterator section_begin() const override;
   section_iterator section_end() const override;
 
-  std::error_code getRelocationAddend(DataRefImpl Rel,
-                                      int64_t &Res) const override;
-  std::error_code getSymbolVersion(SymbolRef Symb, StringRef &Version,
-                                   bool &IsDefault) const override;
-
-  uint64_t getSectionFlags(SectionRef Sec) const override;
-  uint32_t getSectionType(SectionRef Sec) const override;
+  ErrorOr<int64_t> getRelocationAddend(DataRefImpl Rel) const override;
 
   uint8_t getBytesInAddress() const override;
   StringRef getFileFormatName() const override;
   unsigned getArch() const override;
-  StringRef getLoadName() const;
 
   std::error_code getPlatformFlags(unsigned &Result) const override {
     Result = EF.getHeader()->e_flags;
-    return object_error::success;
+    return std::error_code();
   }
 
   const ELFFile<ELFT> *getELFFile() const { return &EF; }
@@ -236,153 +332,135 @@ template <class ELFT> class ELFObjectFile : public ELFObjectFileBase {
                                       ELFT::Is64Bits);
   }
 
-  std::pair<symbol_iterator, symbol_iterator>
-  getELFDynamicSymbolIterators() const override;
+  elf_symbol_iterator_range getDynamicSymbolIterators() const override;
 
   bool isRelocatableObject() const override;
 };
 
-// Use an alignment of 2 for the typedefs since that is the worst case for
-// ELF files in archives.
-typedef ELFObjectFile<ELFType<support::little, 2, false> > ELF32LEObjectFile;
-typedef ELFObjectFile<ELFType<support::little, 2, true> > ELF64LEObjectFile;
-typedef ELFObjectFile<ELFType<support::big, 2, false> > ELF32BEObjectFile;
-typedef ELFObjectFile<ELFType<support::big, 2, true> > ELF64BEObjectFile;
+typedef ELFObjectFile<ELFType<support::little, false>> ELF32LEObjectFile;
+typedef ELFObjectFile<ELFType<support::little, true>> ELF64LEObjectFile;
+typedef ELFObjectFile<ELFType<support::big, false>> ELF32BEObjectFile;
+typedef ELFObjectFile<ELFType<support::big, true>> ELF64BEObjectFile;
 
 template <class ELFT>
-void ELFObjectFile<ELFT>::moveSymbolNext(DataRefImpl &Symb) const {
-  Symb = toDRI(++toELFSymIter(Symb));
+void ELFObjectFile<ELFT>::moveSymbolNext(DataRefImpl &Sym) const {
+  ++Sym.d.b;
 }
 
 template <class ELFT>
-std::error_code ELFObjectFile<ELFT>::getSymbolName(DataRefImpl Symb,
-                                                   StringRef &Result) const {
-  ErrorOr<StringRef> Name = EF.getSymbolName(toELFSymIter(Symb));
-  if (!Name)
-    return Name.getError();
-  Result = *Name;
-  return object_error::success;
+ErrorOr<StringRef> ELFObjectFile<ELFT>::getSymbolName(DataRefImpl Sym) const {
+  const Elf_Sym *ESym = toELFSymIter(Sym);
+  const Elf_Shdr *SymTableSec = *EF.getSection(Sym.d.a);
+  const Elf_Shdr *StringTableSec = *EF.getSection(SymTableSec->sh_link);
+  StringRef SymTable = *EF.getStringTable(StringTableSec);
+  return ESym->getName(SymTable);
 }
 
 template <class ELFT>
-std::error_code ELFObjectFile<ELFT>::getSymbolVersion(SymbolRef SymRef,
-                                                      StringRef &Version,
-                                                      bool &IsDefault) const {
-  DataRefImpl Symb = SymRef.getRawDataRefImpl();
-  const Elf_Sym *symb = getSymbol(Symb);
-  ErrorOr<StringRef> Ver =
-      EF.getSymbolVersion(EF.getSection(Symb.d.b), symb, IsDefault);
-  if (!Ver)
-    return Ver.getError();
-  Version = *Ver;
-  return object_error::success;
+uint64_t ELFObjectFile<ELFT>::getSectionFlags(DataRefImpl Sec) const {
+  return toELFShdrIter(Sec)->sh_flags;
 }
 
 template <class ELFT>
-uint64_t ELFObjectFile<ELFT>::getSectionFlags(SectionRef Sec) const {
-  DataRefImpl DRI = Sec.getRawDataRefImpl();
-  return toELFShdrIter(DRI)->sh_flags;
+uint32_t ELFObjectFile<ELFT>::getSectionType(DataRefImpl Sec) const {
+  return toELFShdrIter(Sec)->sh_type;
 }
 
 template <class ELFT>
-uint32_t ELFObjectFile<ELFT>::getSectionType(SectionRef Sec) const {
-  DataRefImpl DRI = Sec.getRawDataRefImpl();
-  return toELFShdrIter(DRI)->sh_type;
+uint64_t ELFObjectFile<ELFT>::getSymbolValueImpl(DataRefImpl Symb) const {
+  const Elf_Sym *ESym = getSymbol(Symb);
+  uint64_t Ret = ESym->st_value;
+  if (ESym->st_shndx == ELF::SHN_ABS)
+    return Ret;
+
+  const Elf_Ehdr *Header = EF.getHeader();
+  // Clear the ARM/Thumb or microMIPS indicator flag.
+  if ((Header->e_machine == ELF::EM_ARM || Header->e_machine == ELF::EM_MIPS) &&
+      ESym->getType() == ELF::STT_FUNC)
+    Ret &= ~1;
+
+  return Ret;
 }
 
 template <class ELFT>
-std::error_code ELFObjectFile<ELFT>::getSymbolAddress(DataRefImpl Symb,
-                                                      uint64_t &Result) const {
+ErrorOr<uint64_t>
+ELFObjectFile<ELFT>::getSymbolAddress(DataRefImpl Symb) const {
+  uint64_t Result = getSymbolValue(Symb);
   const Elf_Sym *ESym = getSymbol(Symb);
-  switch (EF.getSymbolTableIndex(ESym)) {
+  switch (ESym->st_shndx) {
   case ELF::SHN_COMMON:
   case ELF::SHN_UNDEF:
-    Result = UnknownAddressOrSize;
-    return object_error::success;
   case ELF::SHN_ABS:
-    Result = ESym->st_value;
-    return object_error::success;
-  default:
-    break;
+    return Result;
   }
 
   const Elf_Ehdr *Header = EF.getHeader();
-  Result = ESym->st_value;
-
-  // Clear the ARM/Thumb or microMIPS indicator flag.
-  if ((Header->e_machine == ELF::EM_ARM || Header->e_machine == ELF::EM_MIPS) &&
-      ESym->getType() == ELF::STT_FUNC)
-    Result &= ~1;
 
   if (Header->e_type == ELF::ET_REL) {
-    const typename ELFFile<ELFT>::Elf_Shdr * Section = EF.getSection(ESym);
-    if (Section != nullptr)
+    ErrorOr<const Elf_Shdr *> SectionOrErr = EF.getSection(ESym);
+    if (std::error_code EC = SectionOrErr.getError())
+      return EC;
+    const Elf_Shdr *Section = *SectionOrErr;
+    if (Section)
       Result += Section->sh_addr;
   }
 
-  return object_error::success;
+  return Result;
 }
 
 template <class ELFT>
-std::error_code ELFObjectFile<ELFT>::getSymbolAlignment(DataRefImpl Symb,
-                                                        uint32_t &Res) const {
-  Elf_Sym_Iter Sym = toELFSymIter(Symb);
+uint32_t ELFObjectFile<ELFT>::getSymbolAlignment(DataRefImpl Symb) const {
+  const Elf_Sym *Sym = toELFSymIter(Symb);
   if (Sym->st_shndx == ELF::SHN_COMMON)
-    Res = Sym->st_value;
-  else
-    Res = 0;
-  return object_error::success;
+    return Sym->st_value;
+  return 0;
 }
 
 template <class ELFT>
-std::error_code ELFObjectFile<ELFT>::getSymbolSize(DataRefImpl Symb,
-                                                   uint64_t &Result) const {
-  Result = toELFSymIter(Symb)->st_size;
-  return object_error::success;
+uint64_t ELFObjectFile<ELFT>::getSymbolSize(DataRefImpl Sym) const {
+  return toELFSymIter(Sym)->st_size;
 }
 
 template <class ELFT>
-std::error_code ELFObjectFile<ELFT>::getSymbolOther(DataRefImpl Symb,
-                                                    uint8_t &Result) const {
-  Result = toELFSymIter(Symb)->st_other;
-  return object_error::success;
+uint64_t ELFObjectFile<ELFT>::getCommonSymbolSizeImpl(DataRefImpl Symb) const {
+  return toELFSymIter(Symb)->st_size;
 }
 
 template <class ELFT>
-std::error_code
-ELFObjectFile<ELFT>::getSymbolType(DataRefImpl Symb,
-                                   SymbolRef::Type &Result) const {
+uint8_t ELFObjectFile<ELFT>::getSymbolOther(DataRefImpl Symb) const {
+  return toELFSymIter(Symb)->st_other;
+}
+
+template <class ELFT>
+uint8_t ELFObjectFile<ELFT>::getSymbolELFType(DataRefImpl Symb) const {
+  return toELFSymIter(Symb)->getType();
+}
+
+template <class ELFT>
+SymbolRef::Type ELFObjectFile<ELFT>::getSymbolType(DataRefImpl Symb) const {
   const Elf_Sym *ESym = getSymbol(Symb);
 
   switch (ESym->getType()) {
   case ELF::STT_NOTYPE:
-    Result = SymbolRef::ST_Unknown;
-    break;
+    return SymbolRef::ST_Unknown;
   case ELF::STT_SECTION:
-    Result = SymbolRef::ST_Debug;
-    break;
+    return SymbolRef::ST_Debug;
   case ELF::STT_FILE:
-    Result = SymbolRef::ST_File;
-    break;
+    return SymbolRef::ST_File;
   case ELF::STT_FUNC:
-    Result = SymbolRef::ST_Function;
-    break;
+    return SymbolRef::ST_Function;
   case ELF::STT_OBJECT:
   case ELF::STT_COMMON:
   case ELF::STT_TLS:
-    Result = SymbolRef::ST_Data;
-    break;
+    return SymbolRef::ST_Data;
   default:
-    Result = SymbolRef::ST_Other;
-    break;
+    return SymbolRef::ST_Other;
   }
-  return object_error::success;
 }
 
 template <class ELFT>
-uint32_t ELFObjectFile<ELFT>::getSymbolFlags(DataRefImpl Symb) const {
-  Elf_Sym_Iter EIter = toELFSymIter(Symb);
-  const Elf_Sym *ESym = &*EIter;
+uint32_t ELFObjectFile<ELFT>::getSymbolFlags(DataRefImpl Sym) const {
+  const Elf_Sym *ESym = toELFSymIter(Sym);
 
   uint32_t Result = SymbolRef::SF_None;
 
@@ -396,41 +474,61 @@ uint32_t ELFObjectFile<ELFT>::getSymbolFlags(DataRefImpl Symb) const {
     Result |= SymbolRef::SF_Absolute;
 
   if (ESym->getType() == ELF::STT_FILE || ESym->getType() == ELF::STT_SECTION ||
-      EIter == EF.begin_symbols() || EIter == EF.begin_dynamic_symbols())
+      ESym == EF.symbol_begin() || ESym == EF.dynamic_symbol_begin())
     Result |= SymbolRef::SF_FormatSpecific;
 
-  if (EF.getSymbolTableIndex(ESym) == ELF::SHN_UNDEF)
+  if (EF.getHeader()->e_machine == ELF::EM_ARM) {
+    if (ErrorOr<StringRef> NameOrErr = getSymbolName(Sym)) {
+      StringRef Name = *NameOrErr;
+      if (Name.startswith("$d") || Name.startswith("$t") ||
+          Name.startswith("$a"))
+        Result |= SymbolRef::SF_FormatSpecific;
+    }
+  }
+
+  if (ESym->st_shndx == ELF::SHN_UNDEF)
     Result |= SymbolRef::SF_Undefined;
 
-  if (ESym->getType() == ELF::STT_COMMON ||
-      EF.getSymbolTableIndex(ESym) == ELF::SHN_COMMON)
+  if (ESym->getType() == ELF::STT_COMMON || ESym->st_shndx == ELF::SHN_COMMON)
     Result |= SymbolRef::SF_Common;
 
   if (isExportedToOtherDSO(ESym))
     Result |= SymbolRef::SF_Exported;
 
+  if (ESym->getVisibility() == ELF::STV_HIDDEN)
+    Result |= SymbolRef::SF_Hidden;
+
   return Result;
 }
 
+template <class ELFT>
+section_iterator
+ELFObjectFile<ELFT>::getSymbolSection(const Elf_Sym *ESym) const {
+  ErrorOr<const Elf_Shdr *> ESecOrErr = EF.getSection(ESym);
+  if (std::error_code EC = ESecOrErr.getError())
+    report_fatal_error(EC.message());
+
+  const Elf_Shdr *ESec = *ESecOrErr;
+  if (!ESec)
+    return section_end();
+
+  DataRefImpl Sec;
+  Sec.p = reinterpret_cast<intptr_t>(ESec);
+  return section_iterator(SectionRef(Sec, this));
+}
+
 template <class ELFT>
 std::error_code
 ELFObjectFile<ELFT>::getSymbolSection(DataRefImpl Symb,
                                       section_iterator &Res) const {
-  const Elf_Sym *ESym = getSymbol(Symb);
-  const Elf_Shdr *ESec = EF.getSection(ESym);
-  if (!ESec)
-    Res = section_end();
-  else {
-    DataRefImpl Sec;
-    Sec.p = reinterpret_cast<intptr_t>(ESec);
-    Res = section_iterator(SectionRef(Sec, this));
-  }
-  return object_error::success;
+  Res = getSymbolSection(getSymbol(Symb));
+  return std::error_code();
 }
 
 template <class ELFT>
 void ELFObjectFile<ELFT>::moveSectionNext(DataRefImpl &Sec) const {
-  Sec = toDRI(++toELFShdrIter(Sec));
+  const Elf_Shdr *ESec = toELFShdrIter(Sec);
+  Sec = toDRI(++ESec);
 }
 
 template <class ELFT>
@@ -440,7 +538,7 @@ std::error_code ELFObjectFile<ELFT>::getSectionName(DataRefImpl Sec,
   if (!Name)
     return Name.getError();
   Result = *Name;
-  return object_error::success;
+  return std::error_code();
 }
 
 template <class ELFT>
@@ -457,9 +555,9 @@ template <class ELFT>
 std::error_code
 ELFObjectFile<ELFT>::getSectionContents(DataRefImpl Sec,
                                         StringRef &Result) const {
-  Elf_Shdr_Iter EShdr = toELFShdrIter(Sec);
+  const Elf_Shdr *EShdr = toELFShdrIter(Sec);
   Result = StringRef((const char *)base() + EShdr->sh_offset, EShdr->sh_size);
-  return object_error::success;
+  return std::error_code();
 }
 
 template <class ELFT>
@@ -474,14 +572,14 @@ bool ELFObjectFile<ELFT>::isSectionText(DataRefImpl Sec) const {
 
 template <class ELFT>
 bool ELFObjectFile<ELFT>::isSectionData(DataRefImpl Sec) const {
-  Elf_Shdr_Iter EShdr = toELFShdrIter(Sec);
+  const Elf_Shdr *EShdr = toELFShdrIter(Sec);
   return EShdr->sh_flags & (ELF::SHF_ALLOC | ELF::SHF_WRITE) &&
          EShdr->sh_type == ELF::SHT_PROGBITS;
 }
 
 template <class ELFT>
 bool ELFObjectFile<ELFT>::isSectionBSS(DataRefImpl Sec) const {
-  Elf_Shdr_Iter EShdr = toELFShdrIter(Sec);
+  const Elf_Shdr *EShdr = toELFShdrIter(Sec);
   return EShdr->sh_flags & (ELF::SHF_ALLOC | ELF::SHF_WRITE) &&
          EShdr->sh_type == ELF::SHT_NOBITS;
 }
@@ -491,39 +589,41 @@ bool ELFObjectFile<ELFT>::isSectionVirtual(DataRefImpl Sec) const {
   return toELFShdrIter(Sec)->sh_type == ELF::SHT_NOBITS;
 }
 
-template <class ELFT>
-bool ELFObjectFile<ELFT>::sectionContainsSymbol(DataRefImpl Sec,
-                                                DataRefImpl Symb) const {
-  Elf_Sym_Iter ESym = toELFSymIter(Symb);
-
-  uintX_t Index = ESym->st_shndx;
-  bool Reserved = Index >= ELF::SHN_LORESERVE && Index <= ELF::SHN_HIRESERVE;
-
-  return !Reserved && (&*toELFShdrIter(Sec) == EF.getSection(ESym->st_shndx));
-}
-
 template <class ELFT>
 relocation_iterator
 ELFObjectFile<ELFT>::section_rel_begin(DataRefImpl Sec) const {
   DataRefImpl RelData;
-  uintptr_t SHT = reinterpret_cast<uintptr_t>(EF.begin_sections().get());
+  uintptr_t SHT = reinterpret_cast<uintptr_t>(EF.section_begin());
   RelData.d.a = (Sec.p - SHT) / EF.getHeader()->e_shentsize;
   RelData.d.b = 0;
+
+  const Elf_Shdr *S = reinterpret_cast<const Elf_Shdr *>(Sec.p);
+  if (S->sh_type != ELF::SHT_RELA && S->sh_type != ELF::SHT_REL)
+    return relocation_iterator(RelocationRef(RelData, this));
+
+  const Elf_Shdr *RelSec = getRelSection(RelData);
+  ErrorOr<const Elf_Shdr *> SymSecOrErr = EF.getSection(RelSec->sh_link);
+  if (std::error_code EC = SymSecOrErr.getError())
+    report_fatal_error(EC.message());
+  const Elf_Shdr *SymSec = *SymSecOrErr;
+  uint32_t SymSecType = SymSec->sh_type;
+  if (SymSecType != ELF::SHT_SYMTAB && SymSecType != ELF::SHT_DYNSYM)
+    report_fatal_error("Invalid symbol table section type!");
+  if (SymSecType == ELF::SHT_DYNSYM)
+    RelData.d.b = 1;
+
   return relocation_iterator(RelocationRef(RelData, this));
 }
 
 template <class ELFT>
 relocation_iterator
 ELFObjectFile<ELFT>::section_rel_end(DataRefImpl Sec) const {
-  DataRefImpl RelData;
-  uintptr_t SHT = reinterpret_cast<uintptr_t>(EF.begin_sections().get());
   const Elf_Shdr *S = reinterpret_cast<const Elf_Shdr *>(Sec.p);
-  RelData.d.a = (Sec.p - SHT) / EF.getHeader()->e_shentsize;
+  relocation_iterator Begin = section_rel_begin(Sec);
   if (S->sh_type != ELF::SHT_RELA && S->sh_type != ELF::SHT_REL)
-    RelData.d.b = 0;
-  else
-    RelData.d.b = S->sh_size / S->sh_entsize;
-
+    return Begin;
+  DataRefImpl RelData = Begin->getRawDataRefImpl();
+  RelData.d.b += (S->sh_size / S->sh_entsize) << 1;
   return relocation_iterator(RelocationRef(RelData, this));
 }
 
@@ -533,19 +633,21 @@ ELFObjectFile<ELFT>::getRelocatedSection(DataRefImpl Sec) const {
   if (EF.getHeader()->e_type != ELF::ET_REL)
     return section_end();
 
-  Elf_Shdr_Iter EShdr = toELFShdrIter(Sec);
+  const Elf_Shdr *EShdr = toELFShdrIter(Sec);
   uintX_t Type = EShdr->sh_type;
   if (Type != ELF::SHT_REL && Type != ELF::SHT_RELA)
     return section_end();
 
-  const Elf_Shdr *R = EF.getSection(EShdr->sh_info);
-  return section_iterator(SectionRef(toDRI(R), this));
+  ErrorOr<const Elf_Shdr *> R = EF.getSection(EShdr->sh_info);
+  if (std::error_code EC = R.getError())
+    report_fatal_error(EC.message());
+  return section_iterator(SectionRef(toDRI(*R), this));
 }
 
 // Relocations
 template <class ELFT>
 void ELFObjectFile<ELFT>::moveRelocationNext(DataRefImpl &Rel) const {
-  ++Rel.d.b;
+  Rel.d.b += 2;
 }
 
 template <class ELFT>
@@ -553,96 +655,40 @@ symbol_iterator
 ELFObjectFile<ELFT>::getRelocationSymbol(DataRefImpl Rel) const {
   uint32_t symbolIdx;
   const Elf_Shdr *sec = getRelSection(Rel);
-  switch (sec->sh_type) {
-  default:
-    report_fatal_error("Invalid section type in Rel!");
-  case ELF::SHT_REL: {
+  if (sec->sh_type == ELF::SHT_REL)
     symbolIdx = getRel(Rel)->getSymbol(EF.isMips64EL());
-    break;
-  }
-  case ELF::SHT_RELA: {
+  else
     symbolIdx = getRela(Rel)->getSymbol(EF.isMips64EL());
-    break;
-  }
-  }
   if (!symbolIdx)
     return symbol_end();
 
-  const Elf_Shdr *SymSec = EF.getSection(sec->sh_link);
-
+  bool IsDyn = Rel.d.b & 1;
   DataRefImpl SymbolData;
-  switch (SymSec->sh_type) {
-  default:
-    report_fatal_error("Invalid symbol table section type!");
-  case ELF::SHT_SYMTAB:
-    SymbolData = toDRI(EF.begin_symbols() + symbolIdx);
-    break;
-  case ELF::SHT_DYNSYM:
-    SymbolData = toDRI(EF.begin_dynamic_symbols() + symbolIdx);
-    break;
-  }
-
+  if (IsDyn)
+    SymbolData = toDRI(EF.getDotDynSymSec(), symbolIdx);
+  else
+    SymbolData = toDRI(EF.getDotSymtabSec(), symbolIdx);
   return symbol_iterator(SymbolRef(SymbolData, this));
 }
 
 template <class ELFT>
-std::error_code
-ELFObjectFile<ELFT>::getRelocationAddress(DataRefImpl Rel,
-                                          uint64_t &Result) const {
-  uint64_t ROffset = getROffset(Rel);
-  const Elf_Ehdr *Header = EF.getHeader();
-
-  if (Header->e_type == ELF::ET_REL) {
-    const Elf_Shdr *RelocationSec = getRelSection(Rel);
-    const Elf_Shdr *RelocatedSec = EF.getSection(RelocationSec->sh_info);
-    Result = ROffset + RelocatedSec->sh_addr;
-  } else {
-    Result = ROffset;
-  }
-
-  return object_error::success;
-}
-
-template <class ELFT>
-std::error_code
-ELFObjectFile<ELFT>::getRelocationOffset(DataRefImpl Rel,
-                                         uint64_t &Result) const {
+uint64_t ELFObjectFile<ELFT>::getRelocationOffset(DataRefImpl Rel) const {
   assert(EF.getHeader()->e_type == ELF::ET_REL &&
          "Only relocatable object files have relocation offsets");
-  Result = getROffset(Rel);
-  return object_error::success;
-}
-
-template <class ELFT>
-uint64_t ELFObjectFile<ELFT>::getROffset(DataRefImpl Rel) const {
   const Elf_Shdr *sec = getRelSection(Rel);
-  switch (sec->sh_type) {
-  default:
-    report_fatal_error("Invalid section type in Rel!");
-  case ELF::SHT_REL:
+  if (sec->sh_type == ELF::SHT_REL)
     return getRel(Rel)->r_offset;
-  case ELF::SHT_RELA:
-    return getRela(Rel)->r_offset;
-  }
+
+  return getRela(Rel)->r_offset;
 }
 
 template <class ELFT>
-std::error_code ELFObjectFile<ELFT>::getRelocationType(DataRefImpl Rel,
-                                                       uint64_t &Result) const {
+uint64_t ELFObjectFile<ELFT>::getRelocationType(DataRefImpl Rel) const {
   const Elf_Shdr *sec = getRelSection(Rel);
-  switch (sec->sh_type) {
-  default:
-    report_fatal_error("Invalid section type in Rel!");
-  case ELF::SHT_REL: {
-    Result = getRel(Rel)->getType(EF.isMips64EL());
-    break;
-  }
-  case ELF::SHT_RELA: {
-    Result = getRela(Rel)->getType(EF.isMips64EL());
-    break;
-  }
-  }
-  return object_error::success;
+  if (sec->sh_type == ELF::SHT_REL)
+    return getRel(Rel)->getType(EF.isMips64EL());
+  else
+    return getRela(Rel)->getType(EF.isMips64EL());
 }
 
 template <class ELFT>
@@ -651,125 +697,18 @@ StringRef ELFObjectFile<ELFT>::getRelocationTypeName(uint32_t Type) const {
 }
 
 template <class ELFT>
-std::error_code ELFObjectFile<ELFT>::getRelocationTypeName(
+void ELFObjectFile<ELFT>::getRelocationTypeName(
     DataRefImpl Rel, SmallVectorImpl<char> &Result) const {
-  const Elf_Shdr *sec = getRelSection(Rel);
-  uint32_t type;
-  switch (sec->sh_type) {
-  default:
-    return object_error::parse_failed;
-  case ELF::SHT_REL: {
-    type = getRel(Rel)->getType(EF.isMips64EL());
-    break;
-  }
-  case ELF::SHT_RELA: {
-    type = getRela(Rel)->getType(EF.isMips64EL());
-    break;
-  }
-  }
-
+  uint32_t type = getRelocationType(Rel);
   EF.getRelocationTypeName(type, Result);
-  return object_error::success;
-}
-
-template <class ELFT>
-std::error_code
-ELFObjectFile<ELFT>::getRelocationAddend(DataRefImpl Rel,
-                                         int64_t &Result) const {
-  const Elf_Shdr *sec = getRelSection(Rel);
-  switch (sec->sh_type) {
-  default:
-    report_fatal_error("Invalid section type in Rel!");
-  case ELF::SHT_REL: {
-    Result = 0;
-    return object_error::success;
-  }
-  case ELF::SHT_RELA: {
-    Result = getRela(Rel)->r_addend;
-    return object_error::success;
-  }
-  }
 }
 
 template <class ELFT>
-std::error_code ELFObjectFile<ELFT>::getRelocationValueString(
-    DataRefImpl Rel, SmallVectorImpl<char> &Result) const {
-  const Elf_Shdr *sec = getRelSection(Rel);
-  uint8_t type;
-  StringRef res;
-  int64_t addend = 0;
-  uint16_t symbol_index = 0;
-  switch (sec->sh_type) {
-  default:
+ErrorOr<int64_t>
+ELFObjectFile<ELFT>::getRelocationAddend(DataRefImpl Rel) const {
+  if (getRelSection(Rel)->sh_type != ELF::SHT_RELA)
     return object_error::parse_failed;
-  case ELF::SHT_REL: {
-    type = getRel(Rel)->getType(EF.isMips64EL());
-    symbol_index = getRel(Rel)->getSymbol(EF.isMips64EL());
-    // TODO: Read implicit addend from section data.
-    break;
-  }
-  case ELF::SHT_RELA: {
-    type = getRela(Rel)->getType(EF.isMips64EL());
-    symbol_index = getRela(Rel)->getSymbol(EF.isMips64EL());
-    addend = getRela(Rel)->r_addend;
-    break;
-  }
-  }
-  const Elf_Sym *symb =
-      EF.template getEntry<Elf_Sym>(sec->sh_link, symbol_index);
-  ErrorOr<StringRef> SymName =
-      EF.getSymbolName(EF.getSection(sec->sh_link), symb);
-  if (!SymName)
-    return SymName.getError();
-  switch (EF.getHeader()->e_machine) {
-  case ELF::EM_X86_64:
-    switch (type) {
-    case ELF::R_X86_64_PC8:
-    case ELF::R_X86_64_PC16:
-    case ELF::R_X86_64_PC32: {
-      std::string fmtbuf;
-      raw_string_ostream fmt(fmtbuf);
-      fmt << *SymName << (addend < 0 ? "" : "+") << addend << "-P";
-      fmt.flush();
-      Result.append(fmtbuf.begin(), fmtbuf.end());
-    } break;
-    case ELF::R_X86_64_8:
-    case ELF::R_X86_64_16:
-    case ELF::R_X86_64_32:
-    case ELF::R_X86_64_32S:
-    case ELF::R_X86_64_64: {
-      std::string fmtbuf;
-      raw_string_ostream fmt(fmtbuf);
-      fmt << *SymName << (addend < 0 ? "" : "+") << addend;
-      fmt.flush();
-      Result.append(fmtbuf.begin(), fmtbuf.end());
-    } break;
-    default:
-      res = "Unknown";
-    }
-    break;
-  case ELF::EM_AARCH64: {
-    std::string fmtbuf;
-    raw_string_ostream fmt(fmtbuf);
-    fmt << *SymName;
-    if (addend != 0)
-      fmt << (addend < 0 ? "" : "+") << addend;
-    fmt.flush();
-    Result.append(fmtbuf.begin(), fmtbuf.end());
-    break;
-  }
-  case ELF::EM_386:
-  case ELF::EM_ARM:
-  case ELF::EM_HEXAGON:
-  case ELF::EM_MIPS:
-    res = *SymName;
-    break;
-  default:
-    res = "Unknown";
-  }
-  if (Result.empty())
-    Result.append(res.begin(), res.end());
-  return object_error::success;
+  return (int64_t)getRela(Rel)->r_addend;
 }
 
 template <class ELFT>
@@ -781,13 +720,15 @@ ELFObjectFile<ELFT>::getSymbol(DataRefImpl Symb) const {
 template <class ELFT>
 const typename ELFObjectFile<ELFT>::Elf_Rel *
 ELFObjectFile<ELFT>::getRel(DataRefImpl Rel) const {
-  return EF.template getEntry<Elf_Rel>(Rel.d.a, Rel.d.b);
+  assert(getRelSection(Rel)->sh_type == ELF::SHT_REL);
+  return EF.template getEntry<Elf_Rel>(Rel.d.a, Rel.d.b >> 1);
 }
 
 template <class ELFT>
 const typename ELFObjectFile<ELFT>::Elf_Rela *
 ELFObjectFile<ELFT>::getRela(DataRefImpl Rela) const {
-  return EF.template getEntry<Elf_Rela>(Rela.d.a, Rela.d.b);
+  assert(getRelSection(Rela)->sh_type == ELF::SHT_RELA);
+  return EF.template getEntry<Elf_Rela>(Rela.d.a, Rela.d.b >> 1);
 }
 
 template <class ELFT>
@@ -801,45 +742,40 @@ ELFObjectFile<ELFT>::ELFObjectFile(MemoryBufferRef Object, std::error_code &EC)
 
 template <class ELFT>
 basic_symbol_iterator ELFObjectFile<ELFT>::symbol_begin_impl() const {
-  return basic_symbol_iterator(SymbolRef(toDRI(EF.begin_symbols()), this));
+  DataRefImpl Sym = toDRI(EF.getDotSymtabSec(), 0);
+  return basic_symbol_iterator(SymbolRef(Sym, this));
 }
 
 template <class ELFT>
 basic_symbol_iterator ELFObjectFile<ELFT>::symbol_end_impl() const {
-  return basic_symbol_iterator(SymbolRef(toDRI(EF.end_symbols()), this));
+  const Elf_Shdr *SymTab = EF.getDotSymtabSec();
+  if (!SymTab)
+    return symbol_begin_impl();
+  DataRefImpl Sym = toDRI(SymTab, SymTab->sh_size / sizeof(Elf_Sym));
+  return basic_symbol_iterator(SymbolRef(Sym, this));
 }
 
 template <class ELFT>
-symbol_iterator ELFObjectFile<ELFT>::dynamic_symbol_begin() const {
-  return symbol_iterator(SymbolRef(toDRI(EF.begin_dynamic_symbols()), this));
+elf_symbol_iterator ELFObjectFile<ELFT>::dynamic_symbol_begin() const {
+  DataRefImpl Sym = toDRI(EF.getDotDynSymSec(), 0);
+  return symbol_iterator(SymbolRef(Sym, this));
 }
 
 template <class ELFT>
-symbol_iterator ELFObjectFile<ELFT>::dynamic_symbol_end() const {
-  return symbol_iterator(SymbolRef(toDRI(EF.end_dynamic_symbols()), this));
+elf_symbol_iterator ELFObjectFile<ELFT>::dynamic_symbol_end() const {
+  const Elf_Shdr *SymTab = EF.getDotDynSymSec();
+  DataRefImpl Sym = toDRI(SymTab, SymTab->sh_size / sizeof(Elf_Sym));
+  return basic_symbol_iterator(SymbolRef(Sym, this));
 }
 
 template <class ELFT>
 section_iterator ELFObjectFile<ELFT>::section_begin() const {
-  return section_iterator(SectionRef(toDRI(EF.begin_sections()), this));
+  return section_iterator(SectionRef(toDRI(EF.section_begin()), this));
 }
 
 template <class ELFT>
 section_iterator ELFObjectFile<ELFT>::section_end() const {
-  return section_iterator(SectionRef(toDRI(EF.end_sections()), this));
-}
-
-template <class ELFT>
-StringRef ELFObjectFile<ELFT>::getLoadName() const {
-  Elf_Dyn_Iter DI = EF.begin_dynamic_table();
-  Elf_Dyn_Iter DE = EF.end_dynamic_table();
-
-  while (DI != DE && DI->getTag() != ELF::DT_SONAME)
-    ++DI;
-
-  if (DI != DE)
-    return EF.getDynamicString(DI->getVal());
-  return "";
+  return section_iterator(SectionRef(toDRI(EF.section_end()), this));
 }
 
 template <class ELFT>
@@ -928,7 +864,7 @@ unsigned ELFObjectFile<ELFT>::getArch() const {
 
   case ELF::EM_SPARC:
   case ELF::EM_SPARC32PLUS:
-    return Triple::sparc;
+    return IsLittleEndian ? Triple::sparcel : Triple::sparc;
   case ELF::EM_SPARCV9:
     return Triple::sparcv9;
 
@@ -938,34 +874,15 @@ unsigned ELFObjectFile<ELFT>::getArch() const {
 }
 
 template <class ELFT>
-std::pair<symbol_iterator, symbol_iterator>
-ELFObjectFile<ELFT>::getELFDynamicSymbolIterators() const {
-  return std::make_pair(dynamic_symbol_begin(), dynamic_symbol_end());
+ELFObjectFileBase::elf_symbol_iterator_range
+ELFObjectFile<ELFT>::getDynamicSymbolIterators() const {
+  return make_range(dynamic_symbol_begin(), dynamic_symbol_end());
 }
 
 template <class ELFT> bool ELFObjectFile<ELFT>::isRelocatableObject() const {
   return EF.getHeader()->e_type == ELF::ET_REL;
 }
 
-inline std::error_code getELFRelocationAddend(const RelocationRef R,
-                                              int64_t &Addend) {
-  const ObjectFile *Obj = R.getObjectFile();
-  DataRefImpl DRI = R.getRawDataRefImpl();
-  return cast<ELFObjectFileBase>(Obj)->getRelocationAddend(DRI, Addend);
-}
-
-inline std::pair<symbol_iterator, symbol_iterator>
-getELFDynamicSymbolIterators(const SymbolicFile *Obj) {
-  return cast<ELFObjectFileBase>(Obj)->getELFDynamicSymbolIterators();
-}
-
-inline std::error_code GetELFSymbolVersion(const ObjectFile *Obj,
-                                           const SymbolRef &Sym,
-                                           StringRef &Version,
-                                           bool &IsDefault) {
-  return cast<ELFObjectFileBase>(Obj)
-      ->getSymbolVersion(Sym, Version, IsDefault);
-}
 }
 }
 
diff --git a/include/llvm/Object/ELFTypes.h b/include/llvm/Object/ELFTypes.h
index 5a4e03e7bbe..27e987ba285 100644
--- a/include/llvm/Object/ELFTypes.h
+++ b/include/llvm/Object/ELFTypes.h
@@ -10,105 +10,91 @@
 #ifndef LLVM_OBJECT_ELFTYPES_H
 #define LLVM_OBJECT_ELFTYPES_H
 
-#include "llvm/Support/AlignOf.h"
+#include "llvm/ADT/ArrayRef.h"
+#include "llvm/Object/Error.h"
 #include "llvm/Support/DataTypes.h"
 #include "llvm/Support/ELF.h"
 #include "llvm/Support/Endian.h"
+#include "llvm/Support/ErrorOr.h"
 
 namespace llvm {
 namespace object {
 
 using support::endianness;
 
-template <endianness target_endianness, std::size_t max_alignment,
-          bool is64Bits>
-struct ELFType {
+template <endianness target_endianness, bool is64Bits> struct ELFType {
   static const endianness TargetEndianness = target_endianness;
-  static const std::size_t MaxAlignment = max_alignment;
   static const bool Is64Bits = is64Bits;
 };
 
-template <typename T, int max_align> struct MaximumAlignment {
-  enum { value = AlignOf<T>::Alignment > max_align ? max_align
-                                                   : AlignOf<T>::Alignment
-  };
-};
+typedef ELFType<support::little, false> ELF32LE;
+typedef ELFType<support::big, false> ELF32BE;
+typedef ELFType<support::little, true> ELF64LE;
+typedef ELFType<support::big, true> ELF64BE;
+
+// Use an alignment of 2 for the typedefs since that is the worst case for
+// ELF files in archives.
 
 // Templates to choose Elf_Addr and Elf_Off depending on is64Bits.
-template <endianness target_endianness, std::size_t max_alignment>
-struct ELFDataTypeTypedefHelperCommon {
+template <endianness target_endianness> struct ELFDataTypeTypedefHelperCommon {
   typedef support::detail::packed_endian_specific_integral<
-      uint16_t, target_endianness,
-      MaximumAlignment<uint16_t, max_alignment>::value> Elf_Half;
+      uint16_t, target_endianness, 2> Elf_Half;
   typedef support::detail::packed_endian_specific_integral<
-      uint32_t, target_endianness,
-      MaximumAlignment<uint32_t, max_alignment>::value> Elf_Word;
+      uint32_t, target_endianness, 2> Elf_Word;
   typedef support::detail::packed_endian_specific_integral<
-      int32_t, target_endianness,
-      MaximumAlignment<int32_t, max_alignment>::value> Elf_Sword;
+      int32_t, target_endianness, 2> Elf_Sword;
   typedef support::detail::packed_endian_specific_integral<
-      uint64_t, target_endianness,
-      MaximumAlignment<uint64_t, max_alignment>::value> Elf_Xword;
+      uint64_t, target_endianness, 2> Elf_Xword;
   typedef support::detail::packed_endian_specific_integral<
-      int64_t, target_endianness,
-      MaximumAlignment<int64_t, max_alignment>::value> Elf_Sxword;
+      int64_t, target_endianness, 2> Elf_Sxword;
 };
 
 template <class ELFT> struct ELFDataTypeTypedefHelper;
 
 /// ELF 32bit types.
-template <endianness TargetEndianness, std::size_t MaxAlign>
-struct ELFDataTypeTypedefHelper<ELFType<TargetEndianness, MaxAlign, false> >
-    : ELFDataTypeTypedefHelperCommon<TargetEndianness, MaxAlign> {
+template <endianness TargetEndianness>
+struct ELFDataTypeTypedefHelper<ELFType<TargetEndianness, false>>
+    : ELFDataTypeTypedefHelperCommon<TargetEndianness> {
   typedef uint32_t value_type;
   typedef support::detail::packed_endian_specific_integral<
-      value_type, TargetEndianness,
-      MaximumAlignment<value_type, MaxAlign>::value> Elf_Addr;
+      value_type, TargetEndianness, 2> Elf_Addr;
   typedef support::detail::packed_endian_specific_integral<
-      value_type, TargetEndianness,
-      MaximumAlignment<value_type, MaxAlign>::value> Elf_Off;
+      value_type, TargetEndianness, 2> Elf_Off;
 };
 
 /// ELF 64bit types.
-template <endianness TargetEndianness, std::size_t MaxAlign>
-struct ELFDataTypeTypedefHelper<ELFType<TargetEndianness, MaxAlign, true> >
-    : ELFDataTypeTypedefHelperCommon<TargetEndianness, MaxAlign> {
+template <endianness TargetEndianness>
+struct ELFDataTypeTypedefHelper<ELFType<TargetEndianness, true>>
+    : ELFDataTypeTypedefHelperCommon<TargetEndianness> {
   typedef uint64_t value_type;
   typedef support::detail::packed_endian_specific_integral<
-      value_type, TargetEndianness,
-      MaximumAlignment<value_type, MaxAlign>::value> Elf_Addr;
+      value_type, TargetEndianness, 2> Elf_Addr;
   typedef support::detail::packed_endian_specific_integral<
-      value_type, TargetEndianness,
-      MaximumAlignment<value_type, MaxAlign>::value> Elf_Off;
+      value_type, TargetEndianness, 2> Elf_Off;
 };
 
 // I really don't like doing this, but the alternative is copypasta.
-#define LLVM_ELF_IMPORT_TYPES(E, M, W)                                         \
-typedef typename ELFDataTypeTypedefHelper<ELFType<E, M, W> >::Elf_Addr         \
-    Elf_Addr;                                                                  \
-typedef typename ELFDataTypeTypedefHelper<ELFType<E, M, W> >::Elf_Off          \
-    Elf_Off;                                                                   \
-typedef typename ELFDataTypeTypedefHelper<ELFType<E, M, W> >::Elf_Half         \
-    Elf_Half;                                                                  \
-typedef typename ELFDataTypeTypedefHelper<ELFType<E, M, W> >::Elf_Word         \
-    Elf_Word;                                                                  \
-typedef typename ELFDataTypeTypedefHelper<ELFType<E, M, W> >::Elf_Sword        \
-    Elf_Sword;                                                                 \
-typedef typename ELFDataTypeTypedefHelper<ELFType<E, M, W> >::Elf_Xword        \
-    Elf_Xword;                                                                 \
-typedef typename ELFDataTypeTypedefHelper<ELFType<E, M, W> >::Elf_Sxword       \
-    Elf_Sxword;
+#define LLVM_ELF_IMPORT_TYPES(E, W)                                            \
+  typedef typename ELFDataTypeTypedefHelper<ELFType<E, W>>::Elf_Addr Elf_Addr; \
+  typedef typename ELFDataTypeTypedefHelper<ELFType<E, W>>::Elf_Off Elf_Off;   \
+  typedef typename ELFDataTypeTypedefHelper<ELFType<E, W>>::Elf_Half Elf_Half; \
+  typedef typename ELFDataTypeTypedefHelper<ELFType<E, W>>::Elf_Word Elf_Word; \
+  typedef                                                                      \
+      typename ELFDataTypeTypedefHelper<ELFType<E, W>>::Elf_Sword Elf_Sword;   \
+  typedef                                                                      \
+      typename ELFDataTypeTypedefHelper<ELFType<E, W>>::Elf_Xword Elf_Xword;   \
+  typedef                                                                      \
+      typename ELFDataTypeTypedefHelper<ELFType<E, W>>::Elf_Sxword Elf_Sxword;
 
 #define LLVM_ELF_IMPORT_TYPES_ELFT(ELFT)                                       \
-  LLVM_ELF_IMPORT_TYPES(ELFT::TargetEndianness, ELFT::MaxAlignment,            \
-                        ELFT::Is64Bits)
+  LLVM_ELF_IMPORT_TYPES(ELFT::TargetEndianness, ELFT::Is64Bits)
 
 // Section header.
 template <class ELFT> struct Elf_Shdr_Base;
 
-template <endianness TargetEndianness, std::size_t MaxAlign>
-struct Elf_Shdr_Base<ELFType<TargetEndianness, MaxAlign, false> > {
-  LLVM_ELF_IMPORT_TYPES(TargetEndianness, MaxAlign, false)
+template <endianness TargetEndianness>
+struct Elf_Shdr_Base<ELFType<TargetEndianness, false>> {
+  LLVM_ELF_IMPORT_TYPES(TargetEndianness, false)
   Elf_Word sh_name;      // Section name (index into string table)
   Elf_Word sh_type;      // Section type (SHT_*)
   Elf_Word sh_flags;     // Section flags (SHF_*)
@@ -121,9 +107,9 @@ struct Elf_Shdr_Base<ELFType<TargetEndianness, MaxAlign, false> > {
   Elf_Word sh_entsize;   // Size of records contained within the section
 };
 
-template <endianness TargetEndianness, std::size_t MaxAlign>
-struct Elf_Shdr_Base<ELFType<TargetEndianness, MaxAlign, true> > {
-  LLVM_ELF_IMPORT_TYPES(TargetEndianness, MaxAlign, true)
+template <endianness TargetEndianness>
+struct Elf_Shdr_Base<ELFType<TargetEndianness, true>> {
+  LLVM_ELF_IMPORT_TYPES(TargetEndianness, true)
   Elf_Word sh_name;       // Section name (index into string table)
   Elf_Word sh_type;       // Section type (SHT_*)
   Elf_Xword sh_flags;     // Section flags (SHF_*)
@@ -151,9 +137,9 @@ struct Elf_Shdr_Impl : Elf_Shdr_Base<ELFT> {
 
 template <class ELFT> struct Elf_Sym_Base;
 
-template <endianness TargetEndianness, std::size_t MaxAlign>
-struct Elf_Sym_Base<ELFType<TargetEndianness, MaxAlign, false> > {
-  LLVM_ELF_IMPORT_TYPES(TargetEndianness, MaxAlign, false)
+template <endianness TargetEndianness>
+struct Elf_Sym_Base<ELFType<TargetEndianness, false>> {
+  LLVM_ELF_IMPORT_TYPES(TargetEndianness, false)
   Elf_Word st_name;       // Symbol name (index into string table)
   Elf_Addr st_value;      // Value or address associated with the symbol
   Elf_Word st_size;       // Size of the symbol
@@ -162,9 +148,9 @@ struct Elf_Sym_Base<ELFType<TargetEndianness, MaxAlign, false> > {
   Elf_Half st_shndx;      // Which section (header table index) it's defined in
 };
 
-template <endianness TargetEndianness, std::size_t MaxAlign>
-struct Elf_Sym_Base<ELFType<TargetEndianness, MaxAlign, true> > {
-  LLVM_ELF_IMPORT_TYPES(TargetEndianness, MaxAlign, true)
+template <endianness TargetEndianness>
+struct Elf_Sym_Base<ELFType<TargetEndianness, true>> {
+  LLVM_ELF_IMPORT_TYPES(TargetEndianness, true)
   Elf_Word st_name;       // Symbol name (index into string table)
   unsigned char st_info;  // Symbol's type and binding attributes
   unsigned char st_other; // Must be zero; reserved
@@ -176,12 +162,15 @@ struct Elf_Sym_Base<ELFType<TargetEndianness, MaxAlign, true> > {
 template <class ELFT>
 struct Elf_Sym_Impl : Elf_Sym_Base<ELFT> {
   using Elf_Sym_Base<ELFT>::st_info;
+  using Elf_Sym_Base<ELFT>::st_shndx;
   using Elf_Sym_Base<ELFT>::st_other;
+  using Elf_Sym_Base<ELFT>::st_value;
 
   // These accessors and mutators correspond to the ELF32_ST_BIND,
   // ELF32_ST_TYPE, and ELF32_ST_INFO macros defined in the ELF specification:
   unsigned char getBinding() const { return st_info >> 4; }
   unsigned char getType() const { return st_info & 0x0f; }
+  uint64_t getValue() const { return st_value; }
   void setBinding(unsigned char b) { setBindingAndType(b, getType()); }
   void setType(unsigned char t) { setBindingAndType(getBinding(), t); }
   void setBindingAndType(unsigned char b, unsigned char t) {
@@ -198,8 +187,39 @@ struct Elf_Sym_Impl : Elf_Sym_Base<ELFT> {
     assert(v < 4 && "Invalid value for visibility");
     st_other = (st_other & ~0x3) | v;
   }
+
+  bool isAbsolute() const { return st_shndx == ELF::SHN_ABS; }
+  bool isCommon() const {
+    return getType() == ELF::STT_COMMON || st_shndx == ELF::SHN_COMMON;
+  }
+  bool isDefined() const { return !isUndefined(); }
+  bool isProcessorSpecific() const {
+    return st_shndx >= ELF::SHN_LOPROC && st_shndx <= ELF::SHN_HIPROC;
+  }
+  bool isOSSpecific() const {
+    return st_shndx >= ELF::SHN_LOOS && st_shndx <= ELF::SHN_HIOS;
+  }
+  bool isReserved() const {
+    // ELF::SHN_HIRESERVE is 0xffff so st_shndx <= ELF::SHN_HIRESERVE is always
+    // true and some compilers warn about it.
+    return st_shndx >= ELF::SHN_LORESERVE;
+  }
+  bool isUndefined() const { return st_shndx == ELF::SHN_UNDEF; }
+  bool isExternal() const {
+    return getBinding() != ELF::STB_LOCAL;
+  }
+
+  ErrorOr<StringRef> getName(StringRef StrTab) const;
 };
 
+template <class ELFT>
+ErrorOr<StringRef> Elf_Sym_Impl<ELFT>::getName(StringRef StrTab) const {
+  uint32_t Offset = this->st_name;
+  if (Offset >= StrTab.size())
+    return object_error::parse_failed;
+  return StringRef(StrTab.data() + Offset);
+}
+
 /// Elf_Versym: This is the structure of entries in the SHT_GNU_versym section
 /// (.gnu.version). This structure is identical for ELF32 and ELF64.
 template <class ELFT>
@@ -267,9 +287,9 @@ struct Elf_Vernaux_Impl {
 ///               table section (.dynamic) look like.
 template <class ELFT> struct Elf_Dyn_Base;
 
-template <endianness TargetEndianness, std::size_t MaxAlign>
-struct Elf_Dyn_Base<ELFType<TargetEndianness, MaxAlign, false> > {
-  LLVM_ELF_IMPORT_TYPES(TargetEndianness, MaxAlign, false)
+template <endianness TargetEndianness>
+struct Elf_Dyn_Base<ELFType<TargetEndianness, false>> {
+  LLVM_ELF_IMPORT_TYPES(TargetEndianness, false)
   Elf_Sword d_tag;
   union {
     Elf_Word d_val;
@@ -277,9 +297,9 @@ struct Elf_Dyn_Base<ELFType<TargetEndianness, MaxAlign, false> > {
   } d_un;
 };
 
-template <endianness TargetEndianness, std::size_t MaxAlign>
-struct Elf_Dyn_Base<ELFType<TargetEndianness, MaxAlign, true> > {
-  LLVM_ELF_IMPORT_TYPES(TargetEndianness, MaxAlign, true)
+template <endianness TargetEndianness>
+struct Elf_Dyn_Base<ELFType<TargetEndianness, true>> {
+  LLVM_ELF_IMPORT_TYPES(TargetEndianness, true)
   Elf_Sxword d_tag;
   union {
     Elf_Xword d_val;
@@ -294,15 +314,15 @@ struct Elf_Dyn_Impl : Elf_Dyn_Base<ELFT> {
   using Elf_Dyn_Base<ELFT>::d_un;
   int64_t getTag() const { return d_tag; }
   uint64_t getVal() const { return d_un.d_val; }
-  uint64_t getPtr() const { return d_un.ptr; }
+  uint64_t getPtr() const { return d_un.d_ptr; }
 };
 
 // Elf_Rel: Elf Relocation
-template <class ELFT, bool isRela> struct Elf_Rel_Base;
+template <class ELFT, bool isRela> struct Elf_Rel_Impl;
 
-template <endianness TargetEndianness, std::size_t MaxAlign>
-struct Elf_Rel_Base<ELFType<TargetEndianness, MaxAlign, false>, false> {
-  LLVM_ELF_IMPORT_TYPES(TargetEndianness, MaxAlign, false)
+template <endianness TargetEndianness>
+struct Elf_Rel_Impl<ELFType<TargetEndianness, false>, false> {
+  LLVM_ELF_IMPORT_TYPES(TargetEndianness, false)
   Elf_Addr r_offset; // Location (file byte offset, or program virtual addr)
   Elf_Word r_info;   // Symbol table index and type of relocation to apply
 
@@ -314,64 +334,46 @@ struct Elf_Rel_Base<ELFType<TargetEndianness, MaxAlign, false>, false> {
     assert(!IsMips64EL);
     r_info = R;
   }
-};
 
-template <endianness TargetEndianness, std::size_t MaxAlign>
-struct Elf_Rel_Base<ELFType<TargetEndianness, MaxAlign, true>, false> {
-  LLVM_ELF_IMPORT_TYPES(TargetEndianness, MaxAlign, true)
-  Elf_Addr r_offset; // Location (file byte offset, or program virtual addr)
-  Elf_Xword r_info;  // Symbol table index and type of relocation to apply
-
-  uint64_t getRInfo(bool isMips64EL) const {
-    uint64_t t = r_info;
-    if (!isMips64EL)
-      return t;
-    // Mips64 little endian has a "special" encoding of r_info. Instead of one
-    // 64 bit little endian number, it is a little endian 32 bit number followed
-    // by a 32 bit big endian number.
-    return (t << 32) | ((t >> 8) & 0xff000000) | ((t >> 24) & 0x00ff0000) |
-           ((t >> 40) & 0x0000ff00) | ((t >> 56) & 0x000000ff);
+  // These accessors and mutators correspond to the ELF32_R_SYM, ELF32_R_TYPE,
+  // and ELF32_R_INFO macros defined in the ELF specification:
+  uint32_t getSymbol(bool isMips64EL) const {
+    return this->getRInfo(isMips64EL) >> 8;
   }
-  void setRInfo(uint64_t R, bool IsMips64EL) {
-    if (IsMips64EL)
-      r_info = (R >> 32) | ((R & 0xff000000) << 8) | ((R & 0x00ff0000) << 24) |
-               ((R & 0x0000ff00) << 40) | ((R & 0x000000ff) << 56);
-    else
-      r_info = R;
+  unsigned char getType(bool isMips64EL) const {
+    return (unsigned char)(this->getRInfo(isMips64EL) & 0x0ff);
+  }
+  void setSymbol(uint32_t s, bool IsMips64EL) {
+    setSymbolAndType(s, getType(), IsMips64EL);
+  }
+  void setType(unsigned char t, bool IsMips64EL) {
+    setSymbolAndType(getSymbol(), t, IsMips64EL);
+  }
+  void setSymbolAndType(uint32_t s, unsigned char t, bool IsMips64EL) {
+    this->setRInfo((s << 8) + t, IsMips64EL);
   }
 };
 
-template <endianness TargetEndianness, std::size_t MaxAlign>
-struct Elf_Rel_Base<ELFType<TargetEndianness, MaxAlign, false>, true> {
-  LLVM_ELF_IMPORT_TYPES(TargetEndianness, MaxAlign, false)
-  Elf_Addr r_offset;  // Location (file byte offset, or program virtual addr)
-  Elf_Word r_info;    // Symbol table index and type of relocation to apply
+template <endianness TargetEndianness>
+struct Elf_Rel_Impl<ELFType<TargetEndianness, false>, true>
+    : public Elf_Rel_Impl<ELFType<TargetEndianness, false>, false> {
+  LLVM_ELF_IMPORT_TYPES(TargetEndianness, false)
   Elf_Sword r_addend; // Compute value for relocatable field by adding this
-
-  uint32_t getRInfo(bool isMips64EL) const {
-    assert(!isMips64EL);
-    return r_info;
-  }
-  void setRInfo(uint32_t R, bool IsMips64EL) {
-    assert(!IsMips64EL);
-    r_info = R;
-  }
 };
 
-template <endianness TargetEndianness, std::size_t MaxAlign>
-struct Elf_Rel_Base<ELFType<TargetEndianness, MaxAlign, true>, true> {
-  LLVM_ELF_IMPORT_TYPES(TargetEndianness, MaxAlign, true)
-  Elf_Addr r_offset;   // Location (file byte offset, or program virtual addr)
-  Elf_Xword r_info;    // Symbol table index and type of relocation to apply
-  Elf_Sxword r_addend; // Compute value for relocatable field by adding this.
+template <endianness TargetEndianness>
+struct Elf_Rel_Impl<ELFType<TargetEndianness, true>, false> {
+  LLVM_ELF_IMPORT_TYPES(TargetEndianness, true)
+  Elf_Addr r_offset; // Location (file byte offset, or program virtual addr)
+  Elf_Xword r_info;  // Symbol table index and type of relocation to apply
 
   uint64_t getRInfo(bool isMips64EL) const {
-    // Mips64 little endian has a "special" encoding of r_info. Instead of one
-    // 64 bit little endian number, it is a little endian 32 bit number followed
-    // by a 32 bit big endian number.
     uint64_t t = r_info;
     if (!isMips64EL)
       return t;
+    // Mips64 little endian has a "special" encoding of r_info. Instead of one
+    // 64 bit little endian number, it is a little endian 32 bit number followed
+    // by a 32 bit big endian number.
     return (t << 32) | ((t >> 8) & 0xff000000) | ((t >> 24) & 0x00ff0000) |
            ((t >> 40) & 0x0000ff00) | ((t >> 56) & 0x000000ff);
   }
@@ -382,15 +384,6 @@ struct Elf_Rel_Base<ELFType<TargetEndianness, MaxAlign, true>, true> {
     else
       r_info = R;
   }
-};
-
-template <class ELFT, bool isRela> struct Elf_Rel_Impl;
-
-template <endianness TargetEndianness, std::size_t MaxAlign, bool isRela>
-struct Elf_Rel_Impl<ELFType<TargetEndianness, MaxAlign, true>,
-                    isRela> : Elf_Rel_Base<
-    ELFType<TargetEndianness, MaxAlign, true>, isRela> {
-  LLVM_ELF_IMPORT_TYPES(TargetEndianness, MaxAlign, true)
 
   // These accessors and mutators correspond to the ELF64_R_SYM, ELF64_R_TYPE,
   // and ELF64_R_INFO macros defined in the ELF specification:
@@ -411,29 +404,11 @@ struct Elf_Rel_Impl<ELFType<TargetEndianness, MaxAlign, true>,
   }
 };
 
-template <endianness TargetEndianness, std::size_t MaxAlign, bool isRela>
-struct Elf_Rel_Impl<ELFType<TargetEndianness, MaxAlign, false>,
-                    isRela> : Elf_Rel_Base<
-    ELFType<TargetEndianness, MaxAlign, false>, isRela> {
-  LLVM_ELF_IMPORT_TYPES(TargetEndianness, MaxAlign, false)
-
-  // These accessors and mutators correspond to the ELF32_R_SYM, ELF32_R_TYPE,
-  // and ELF32_R_INFO macros defined in the ELF specification:
-  uint32_t getSymbol(bool isMips64EL) const {
-    return this->getRInfo(isMips64EL) >> 8;
-  }
-  unsigned char getType(bool isMips64EL) const {
-    return (unsigned char)(this->getRInfo(isMips64EL) & 0x0ff);
-  }
-  void setSymbol(uint32_t s, bool IsMips64EL) {
-    setSymbolAndType(s, getType(), IsMips64EL);
-  }
-  void setType(unsigned char t, bool IsMips64EL) {
-    setSymbolAndType(getSymbol(), t, IsMips64EL);
-  }
-  void setSymbolAndType(uint32_t s, unsigned char t, bool IsMips64EL) {
-    this->setRInfo((s << 8) + t, IsMips64EL);
-  }
+template <endianness TargetEndianness>
+struct Elf_Rel_Impl<ELFType<TargetEndianness, true>, true>
+    : public Elf_Rel_Impl<ELFType<TargetEndianness, true>, false> {
+  LLVM_ELF_IMPORT_TYPES(TargetEndianness, true)
+  Elf_Sxword r_addend; // Compute value for relocatable field by adding this.
 };
 
 template <class ELFT>
@@ -463,9 +438,9 @@ struct Elf_Ehdr_Impl {
 
 template <class ELFT> struct Elf_Phdr_Impl;
 
-template <endianness TargetEndianness, std::size_t MaxAlign>
-struct Elf_Phdr_Impl<ELFType<TargetEndianness, MaxAlign, false> > {
-  LLVM_ELF_IMPORT_TYPES(TargetEndianness, MaxAlign, false)
+template <endianness TargetEndianness>
+struct Elf_Phdr_Impl<ELFType<TargetEndianness, false>> {
+  LLVM_ELF_IMPORT_TYPES(TargetEndianness, false)
   Elf_Word p_type;   // Type of segment
   Elf_Off p_offset;  // FileOffset where segment is located, in bytes
   Elf_Addr p_vaddr;  // Virtual Address of beginning of segment
@@ -476,9 +451,9 @@ struct Elf_Phdr_Impl<ELFType<TargetEndianness, MaxAlign, false> > {
   Elf_Word p_align;  // Segment alignment constraint
 };
 
-template <endianness TargetEndianness, std::size_t MaxAlign>
-struct Elf_Phdr_Impl<ELFType<TargetEndianness, MaxAlign, true> > {
-  LLVM_ELF_IMPORT_TYPES(TargetEndianness, MaxAlign, true)
+template <endianness TargetEndianness>
+struct Elf_Phdr_Impl<ELFType<TargetEndianness, true>> {
+  LLVM_ELF_IMPORT_TYPES(TargetEndianness, true)
   Elf_Word p_type;    // Type of segment
   Elf_Word p_flags;   // Segment flags
   Elf_Off p_offset;   // FileOffset where segment is located, in bytes
@@ -489,6 +464,76 @@ struct Elf_Phdr_Impl<ELFType<TargetEndianness, MaxAlign, true> > {
   Elf_Xword p_align;  // Segment alignment constraint
 };
 
+// ELFT needed for endianess.
+template <class ELFT>
+struct Elf_Hash_Impl {
+  LLVM_ELF_IMPORT_TYPES_ELFT(ELFT)
+  Elf_Word nbucket;
+  Elf_Word nchain;
+
+  ArrayRef<Elf_Word> buckets() const {
+    return ArrayRef<Elf_Word>(&nbucket + 2, &nbucket + 2 + nbucket);
+  }
+
+  ArrayRef<Elf_Word> chains() const {
+    return ArrayRef<Elf_Word>(&nbucket + 2 + nbucket,
+                              &nbucket + 2 + nbucket + nchain);
+  }
+};
+
+// MIPS .reginfo section
+template <class ELFT>
+struct Elf_Mips_RegInfo;
+
+template <llvm::support::endianness TargetEndianness>
+struct Elf_Mips_RegInfo<ELFType<TargetEndianness, false>> {
+  LLVM_ELF_IMPORT_TYPES(TargetEndianness, false)
+  Elf_Word ri_gprmask;     // bit-mask of used general registers
+  Elf_Word ri_cprmask[4];  // bit-mask of used co-processor registers
+  Elf_Addr ri_gp_value;    // gp register value
+};
+
+template <llvm::support::endianness TargetEndianness>
+struct Elf_Mips_RegInfo<ELFType<TargetEndianness, true>> {
+  LLVM_ELF_IMPORT_TYPES(TargetEndianness, true)
+  Elf_Word ri_gprmask;     // bit-mask of used general registers
+  Elf_Word ri_pad;         // unused padding field
+  Elf_Word ri_cprmask[4];  // bit-mask of used co-processor registers
+  Elf_Addr ri_gp_value;    // gp register value
+};
+
+// .MIPS.options section
+template <class ELFT> struct Elf_Mips_Options {
+  LLVM_ELF_IMPORT_TYPES_ELFT(ELFT)
+  uint8_t kind;     // Determines interpretation of variable part of descriptor
+  uint8_t size;     // Byte size of descriptor, including this header
+  Elf_Half section; // Section header index of section affected,
+                    // or 0 for global options
+  Elf_Word info;    // Kind-specific information
+
+  const Elf_Mips_RegInfo<ELFT> &getRegInfo() const {
+    assert(kind == llvm::ELF::ODK_REGINFO);
+    return *reinterpret_cast<const Elf_Mips_RegInfo<ELFT> *>(
+               (const uint8_t *)this + sizeof(Elf_Mips_Options));
+  }
+};
+
+// .MIPS.abiflags section content
+template <class ELFT> struct Elf_Mips_ABIFlags {
+  LLVM_ELF_IMPORT_TYPES_ELFT(ELFT)
+  Elf_Half version;  // Version of the structure
+  uint8_t isa_level; // ISA level: 1-5, 32, and 64
+  uint8_t isa_rev;   // ISA revision (0 for MIPS I - MIPS V)
+  uint8_t gpr_size;  // General purpose registers size
+  uint8_t cpr1_size; // Co-processor 1 registers size
+  uint8_t cpr2_size; // Co-processor 2 registers size
+  uint8_t fp_abi;    // Floating-point ABI flag
+  Elf_Word isa_ext;  // Processor-specific extension
+  Elf_Word ases;     // ASEs flags
+  Elf_Word flags1;   // General flags
+  Elf_Word flags2;   // General flags
+};
+
 } // end namespace object.
 } // end namespace llvm.
 
diff --git a/include/llvm/Object/ELFYAML.h b/include/llvm/Object/ELFYAML.h
index 45cbdbd2da3..df0aa500c8a 100644
--- a/include/llvm/Object/ELFYAML.h
+++ b/include/llvm/Object/ELFYAML.h
@@ -48,6 +48,13 @@ LLVM_YAML_STRONG_TYPEDEF(uint8_t, ELF_STT)
 LLVM_YAML_STRONG_TYPEDEF(uint8_t, ELF_STV)
 LLVM_YAML_STRONG_TYPEDEF(uint8_t, ELF_STO)
 
+LLVM_YAML_STRONG_TYPEDEF(uint8_t, MIPS_AFL_REG)
+LLVM_YAML_STRONG_TYPEDEF(uint8_t, MIPS_ABI_FP)
+LLVM_YAML_STRONG_TYPEDEF(uint32_t, MIPS_AFL_EXT)
+LLVM_YAML_STRONG_TYPEDEF(uint32_t, MIPS_AFL_ASE)
+LLVM_YAML_STRONG_TYPEDEF(uint32_t, MIPS_AFL_FLAGS1)
+LLVM_YAML_STRONG_TYPEDEF(uint32_t, MIPS_ISA)
+
 // For now, hardcode 64 bits everywhere that 32 or 64 would be needed
 // since 64-bit can hold 32-bit values too.
 struct FileHeader {
@@ -78,7 +85,13 @@ struct SectionOrType {
 };
 
 struct Section {
-  enum class SectionKind { Group, RawContent, Relocation };
+  enum class SectionKind {
+    Group,
+    RawContent,
+    Relocation,
+    NoBits,
+    MipsABIFlags
+  };
   SectionKind Kind;
   StringRef Name;
   ELF_SHT Type;
@@ -99,6 +112,14 @@ struct RawContentSection : Section {
   }
 };
 
+struct NoBitsSection : Section {
+  llvm::yaml::Hex64 Size;
+  NoBitsSection() : Section(SectionKind::NoBits) {}
+  static bool classof(const Section *S) {
+    return S->Kind == SectionKind::NoBits;
+  }
+};
+
 struct Group : Section {
   // Members of a group contain a flag and a list of section indices
   // that are part of the group.
@@ -122,6 +143,26 @@ struct RelocationSection : Section {
     return S->Kind == SectionKind::Relocation;
   }
 };
+
+// Represents .MIPS.abiflags section
+struct MipsABIFlags : Section {
+  llvm::yaml::Hex16 Version;
+  MIPS_ISA ISALevel;
+  llvm::yaml::Hex8 ISARevision;
+  MIPS_AFL_REG GPRSize;
+  MIPS_AFL_REG CPR1Size;
+  MIPS_AFL_REG CPR2Size;
+  MIPS_ABI_FP FpABI;
+  MIPS_AFL_EXT ISAExtension;
+  MIPS_AFL_ASE ASEs;
+  MIPS_AFL_FLAGS1 Flags1;
+  llvm::yaml::Hex32 Flags2;
+  MipsABIFlags() : Section(SectionKind::MipsABIFlags) {}
+  static bool classof(const Section *S) {
+    return S->Kind == SectionKind::MipsABIFlags;
+  }
+};
+
 struct Object {
   FileHeader Header;
   std::vector<std::unique_ptr<Section>> Sections;
@@ -208,6 +249,36 @@ struct ScalarEnumerationTraits<ELFYAML::ELF_RSS> {
   static void enumeration(IO &IO, ELFYAML::ELF_RSS &Value);
 };
 
+template <>
+struct ScalarEnumerationTraits<ELFYAML::MIPS_AFL_REG> {
+  static void enumeration(IO &IO, ELFYAML::MIPS_AFL_REG &Value);
+};
+
+template <>
+struct ScalarEnumerationTraits<ELFYAML::MIPS_ABI_FP> {
+  static void enumeration(IO &IO, ELFYAML::MIPS_ABI_FP &Value);
+};
+
+template <>
+struct ScalarEnumerationTraits<ELFYAML::MIPS_AFL_EXT> {
+  static void enumeration(IO &IO, ELFYAML::MIPS_AFL_EXT &Value);
+};
+
+template <>
+struct ScalarEnumerationTraits<ELFYAML::MIPS_ISA> {
+  static void enumeration(IO &IO, ELFYAML::MIPS_ISA &Value);
+};
+
+template <>
+struct ScalarBitSetTraits<ELFYAML::MIPS_AFL_ASE> {
+  static void bitset(IO &IO, ELFYAML::MIPS_AFL_ASE &Value);
+};
+
+template <>
+struct ScalarBitSetTraits<ELFYAML::MIPS_AFL_FLAGS1> {
+  static void bitset(IO &IO, ELFYAML::MIPS_AFL_FLAGS1 &Value);
+};
+
 template <>
 struct MappingTraits<ELFYAML::FileHeader> {
   static void mapping(IO &IO, ELFYAML::FileHeader &FileHdr);
diff --git a/include/llvm/Object/Error.h b/include/llvm/Object/Error.h
index 90c2bd74b43..aa320bb51a4 100644
--- a/include/llvm/Object/Error.h
+++ b/include/llvm/Object/Error.h
@@ -22,12 +22,17 @@ namespace object {
 const std::error_category &object_category();
 
 enum class object_error {
-  success = 0,
-  arch_not_found,
+  // Error code 0 is absent. Use std::error_code() instead.
+  arch_not_found = 1,
   invalid_file_type,
   parse_failed,
   unexpected_eof,
+  string_table_non_null_end,
+  invalid_section_index,
   bitcode_section_not_found,
+  macho_small_load_command,
+  macho_load_segment_too_many_sections,
+  macho_load_segment_too_small,
 };
 
 inline std::error_code make_error_code(object_error e) {
diff --git a/include/llvm/Object/MachO.h b/include/llvm/Object/MachO.h
index a552aca91d7..489ecef5c99 100644
--- a/include/llvm/Object/MachO.h
+++ b/include/llvm/Object/MachO.h
@@ -190,28 +190,30 @@ class MachOObjectFile : public ObjectFile {
     const char *Ptr;      // Where in memory the load command is.
     MachO::load_command C; // The command itself.
   };
+  typedef SmallVector<LoadCommandInfo, 4> LoadCommandList;
+  typedef LoadCommandList::const_iterator load_command_iterator;
 
   MachOObjectFile(MemoryBufferRef Object, bool IsLittleEndian, bool Is64Bits,
                   std::error_code &EC);
 
   void moveSymbolNext(DataRefImpl &Symb) const override;
-  std::error_code getSymbolName(DataRefImpl Symb,
-                                StringRef &Res) const override;
+
+  uint64_t getNValue(DataRefImpl Sym) const;
+  ErrorOr<StringRef> getSymbolName(DataRefImpl Symb) const override;
 
   // MachO specific.
   std::error_code getIndirectName(DataRefImpl Symb, StringRef &Res) const;
   unsigned getSectionType(SectionRef Sec) const;
 
-  std::error_code getSymbolAddress(DataRefImpl Symb,
-                                   uint64_t &Res) const override;
-  std::error_code getSymbolAlignment(DataRefImpl Symb,
-                                     uint32_t &Res) const override;
-  std::error_code getSymbolSize(DataRefImpl Symb, uint64_t &Res) const override;
-  std::error_code getSymbolType(DataRefImpl Symb,
-                                SymbolRef::Type &Res) const override;
+  ErrorOr<uint64_t> getSymbolAddress(DataRefImpl Symb) const override;
+  uint32_t getSymbolAlignment(DataRefImpl Symb) const override;
+  uint64_t getCommonSymbolSizeImpl(DataRefImpl Symb) const override;
+  SymbolRef::Type getSymbolType(DataRefImpl Symb) const override;
   uint32_t getSymbolFlags(DataRefImpl Symb) const override;
   std::error_code getSymbolSection(DataRefImpl Symb,
                                    section_iterator &Res) const override;
+  unsigned getSymbolSectionID(SymbolRef Symb) const;
+  unsigned getSectionID(SectionRef Sec) const;
 
   void moveSectionNext(DataRefImpl &Sec) const override;
   std::error_code getSectionName(DataRefImpl Sec,
@@ -225,30 +227,23 @@ class MachOObjectFile : public ObjectFile {
   bool isSectionData(DataRefImpl Sec) const override;
   bool isSectionBSS(DataRefImpl Sec) const override;
   bool isSectionVirtual(DataRefImpl Sec) const override;
-  bool sectionContainsSymbol(DataRefImpl Sec, DataRefImpl Symb) const override;
   relocation_iterator section_rel_begin(DataRefImpl Sec) const override;
   relocation_iterator section_rel_end(DataRefImpl Sec) const override;
 
   void moveRelocationNext(DataRefImpl &Rel) const override;
-  std::error_code getRelocationAddress(DataRefImpl Rel,
-                                       uint64_t &Res) const override;
-  std::error_code getRelocationOffset(DataRefImpl Rel,
-                                      uint64_t &Res) const override;
+  uint64_t getRelocationOffset(DataRefImpl Rel) const override;
   symbol_iterator getRelocationSymbol(DataRefImpl Rel) const override;
-  std::error_code getRelocationType(DataRefImpl Rel,
-                                    uint64_t &Res) const override;
-  std::error_code
-  getRelocationTypeName(DataRefImpl Rel,
-                        SmallVectorImpl<char> &Result) const override;
-  std::error_code
-  getRelocationValueString(DataRefImpl Rel,
-                           SmallVectorImpl<char> &Result) const override;
-  std::error_code getRelocationHidden(DataRefImpl Rel,
-                                      bool &Result) const override;
+  section_iterator getRelocationSection(DataRefImpl Rel) const;
+  uint64_t getRelocationType(DataRefImpl Rel) const override;
+  void getRelocationTypeName(DataRefImpl Rel,
+                             SmallVectorImpl<char> &Result) const override;
+  uint8_t getRelocationLength(DataRefImpl Rel) const;
 
   // MachO specific.
   std::error_code getLibraryShortNameByIndex(unsigned Index, StringRef &) const;
 
+  section_iterator getRelocationRelocatedSection(relocation_iterator Rel) const;
+
   // TODO: Would be useful to have an iterator based version
   // of the load command interface too.
 
@@ -272,10 +267,14 @@ class MachOObjectFile : public ObjectFile {
 
   dice_iterator begin_dices() const;
   dice_iterator end_dices() const;
-  
+
+  load_command_iterator begin_load_commands() const;
+  load_command_iterator end_load_commands() const;
+  iterator_range<load_command_iterator> load_commands() const;
+
   /// For use iterating over all exported symbols.
   iterator_range<export_iterator> exports() const;
-  
+
   /// For use examining a trie not in a MachOObjectFile.
   static iterator_range<export_iterator> exports(ArrayRef<uint8_t> Trie);
 
@@ -326,11 +325,7 @@ class MachOObjectFile : public ObjectFile {
   unsigned getAnyRelocationPCRel(const MachO::any_relocation_info &RE) const;
   unsigned getAnyRelocationLength(const MachO::any_relocation_info &RE) const;
   unsigned getAnyRelocationType(const MachO::any_relocation_info &RE) const;
-  SectionRef getRelocationSection(const MachO::any_relocation_info &RE) const;
-
-  // Walk load commands.
-  LoadCommandInfo getFirstLoadCommandInfo() const;
-  LoadCommandInfo getNextLoadCommandInfo(const LoadCommandInfo &L) const;
+  SectionRef getAnyRelocationSection(const MachO::any_relocation_info &RE) const;
 
   // MachO specific structures.
   MachO::section getSection(DataRefImpl DRI) const;
@@ -385,8 +380,8 @@ class MachOObjectFile : public ObjectFile {
 
   MachO::any_relocation_info getRelocation(DataRefImpl Rel) const;
   MachO::data_in_code_entry getDice(DataRefImpl Rel) const;
-  MachO::mach_header getHeader() const;
-  MachO::mach_header_64 getHeader64() const;
+  const MachO::mach_header &getHeader() const;
+  const MachO::mach_header_64 &getHeader64() const;
   uint32_t
   getIndirectSymbolTableEntry(const MachO::dysymtab_command &DLC,
                               unsigned Index) const;
@@ -429,10 +424,17 @@ class MachOObjectFile : public ObjectFile {
   }
 
 private:
+  uint64_t getSymbolValueImpl(DataRefImpl Symb) const override;
+
+  union {
+    MachO::mach_header_64 Header64;
+    MachO::mach_header Header;
+  };
   typedef SmallVector<const char*, 1> SectionList;
   SectionList Sections;
   typedef SmallVector<const char*, 1> LibraryList;
   LibraryList Libraries;
+  LoadCommandList LoadCommands;
   typedef SmallVector<StringRef, 1> LibraryShortName;
   mutable LibraryShortName LibrariesShortNames;
   const char *SymtabLoadCmd;
@@ -471,7 +473,7 @@ inline std::error_code DiceRef::getOffset(uint32_t &Result) const {
     static_cast<const MachOObjectFile *>(OwningObject);
   MachO::data_in_code_entry Dice = MachOOF->getDice(DicePimpl);
   Result = Dice.offset;
-  return object_error::success;
+  return std::error_code();
 }
 
 inline std::error_code DiceRef::getLength(uint16_t &Result) const {
@@ -479,7 +481,7 @@ inline std::error_code DiceRef::getLength(uint16_t &Result) const {
     static_cast<const MachOObjectFile *>(OwningObject);
   MachO::data_in_code_entry Dice = MachOOF->getDice(DicePimpl);
   Result = Dice.length;
-  return object_error::success;
+  return std::error_code();
 }
 
 inline std::error_code DiceRef::getKind(uint16_t &Result) const {
@@ -487,7 +489,7 @@ inline std::error_code DiceRef::getKind(uint16_t &Result) const {
     static_cast<const MachOObjectFile *>(OwningObject);
   MachO::data_in_code_entry Dice = MachOOF->getDice(DicePimpl);
   Result = Dice.kind;
-  return object_error::success;
+  return std::error_code();
 }
 
 inline DataRefImpl DiceRef::getRawDataRefImpl() const {
diff --git a/include/llvm/Object/MachOUniversal.h b/include/llvm/Object/MachOUniversal.h
index 05119b29331..a11d381a700 100644
--- a/include/llvm/Object/MachOUniversal.h
+++ b/include/llvm/Object/MachOUniversal.h
@@ -109,7 +109,7 @@ class MachOUniversalBinary : public Binary {
   }
 
   ErrorOr<std::unique_ptr<MachOObjectFile>>
-  getObjectForArch(Triple::ArchType Arch) const;
+  getObjectForArch(StringRef ArchName) const;
 };
 
 }
diff --git a/include/llvm/Object/ObjectFile.h b/include/llvm/Object/ObjectFile.h
index b7e19b941c8..8dd52562621 100644
--- a/include/llvm/Object/ObjectFile.h
+++ b/include/llvm/Object/ObjectFile.h
@@ -32,9 +32,11 @@ class MachOObjectFile;
 
 class SymbolRef;
 class symbol_iterator;
+class SectionRef;
+typedef content_iterator<SectionRef> section_iterator;
 
-/// RelocationRef - This is a value type class that represents a single
-/// relocation in the list of relocations in the object file.
+/// This is a value type class that represents a single relocation in the list
+/// of relocations in the object file.
 class RelocationRef {
   DataRefImpl RelocationPimpl;
   const ObjectFile *OwningObject;
@@ -48,36 +50,22 @@ class RelocationRef {
 
   void moveNext();
 
-  std::error_code getAddress(uint64_t &Result) const;
-  std::error_code getOffset(uint64_t &Result) const;
+  uint64_t getOffset() const;
   symbol_iterator getSymbol() const;
-  std::error_code getType(uint64_t &Result) const;
-
-  /// @brief Indicates whether this relocation should hidden when listing
-  /// relocations, usually because it is the trailing part of a multipart
-  /// relocation that will be printed as part of the leading relocation.
-  std::error_code getHidden(bool &Result) const;
+  uint64_t getType() const;
 
   /// @brief Get a string that represents the type of this relocation.
   ///
   /// This is for display purposes only.
-  std::error_code getTypeName(SmallVectorImpl<char> &Result) const;
-
-  /// @brief Get a string that represents the calculation of the value of this
-  ///        relocation.
-  ///
-  /// This is for display purposes only.
-  std::error_code getValueString(SmallVectorImpl<char> &Result) const;
+  void getTypeName(SmallVectorImpl<char> &Result) const;
 
   DataRefImpl getRawDataRefImpl() const;
-  const ObjectFile *getObjectFile() const;
+  const ObjectFile *getObject() const;
 };
 typedef content_iterator<RelocationRef> relocation_iterator;
 
-/// SectionRef - This is a value type class that represents a single section in
-/// the list of sections in the object file.
-class SectionRef;
-typedef content_iterator<SectionRef> section_iterator;
+/// This is a value type class that represents a single section in the list of
+/// sections in the object file.
 class SectionRef {
   friend class SymbolRef;
   DataRefImpl SectionPimpl;
@@ -121,8 +109,8 @@ class SectionRef {
   const ObjectFile *getObject() const;
 };
 
-/// SymbolRef - This is a value type class that represents a single symbol in
-/// the list of symbols in the object file.
+/// This is a value type class that represents a single symbol in the list of
+/// symbols in the object file.
 class SymbolRef : public BasicSymbolRef {
   friend class SectionRef;
 
@@ -139,16 +127,23 @@ class SymbolRef : public BasicSymbolRef {
   };
 
   SymbolRef(DataRefImpl SymbolP, const ObjectFile *Owner);
+  SymbolRef(const BasicSymbolRef &B) : BasicSymbolRef(B) {
+    assert(isa<ObjectFile>(BasicSymbolRef::getObject()));
+  }
 
-  std::error_code getName(StringRef &Result) const;
+  ErrorOr<StringRef> getName() const;
   /// Returns the symbol virtual address (i.e. address at which it will be
   /// mapped).
-  std::error_code getAddress(uint64_t &Result) const;
+  ErrorOr<uint64_t> getAddress() const;
+
+  /// Return the value of the symbol depending on the object this can be an
+  /// offset or a virtual address.
+  uint64_t getValue() const;
+
   /// @brief Get the alignment of this symbol as the actual value (not log 2).
-  std::error_code getAlignment(uint32_t &Result) const;
-  std::error_code getSize(uint64_t &Result) const;
-  std::error_code getType(SymbolRef::Type &Result) const;
-  std::error_code getOther(uint8_t &Result) const;
+  uint32_t getAlignment() const;
+  uint64_t getCommonSize() const;
+  SymbolRef::Type getType() const;
 
   /// @brief Get section this symbol is defined in reference to. Result is
   /// end_sections() if it is undefined or is an absolute symbol.
@@ -175,9 +170,9 @@ class symbol_iterator : public basic_symbol_iterator {
   }
 };
 
-/// ObjectFile - This class is the base class for all object file types.
-/// Concrete instances of this object are created by createObjectFile, which
-/// figures out which type to create.
+/// This class is the base class for all object file types. Concrete instances
+/// of this object are created by createObjectFile, which figures out which type
+/// to create.
 class ObjectFile : public SymbolicFile {
   virtual void anchor();
   ObjectFile() = delete;
@@ -199,24 +194,16 @@ class ObjectFile : public SymbolicFile {
   // Implementations assume that the DataRefImpl is valid and has not been
   // modified externally. It's UB otherwise.
   friend class SymbolRef;
-  virtual std::error_code getSymbolName(DataRefImpl Symb,
-                                        StringRef &Res) const = 0;
+  virtual ErrorOr<StringRef> getSymbolName(DataRefImpl Symb) const = 0;
   std::error_code printSymbolName(raw_ostream &OS,
                                   DataRefImpl Symb) const override;
-  virtual std::error_code getSymbolAddress(DataRefImpl Symb,
-                                           uint64_t &Res) const = 0;
-  virtual std::error_code getSymbolAlignment(DataRefImpl Symb,
-                                             uint32_t &Res) const;
-  virtual std::error_code getSymbolSize(DataRefImpl Symb,
-                                        uint64_t &Res) const = 0;
-  virtual std::error_code getSymbolType(DataRefImpl Symb,
-                                        SymbolRef::Type &Res) const = 0;
+  virtual ErrorOr<uint64_t> getSymbolAddress(DataRefImpl Symb) const = 0;
+  virtual uint64_t getSymbolValueImpl(DataRefImpl Symb) const = 0;
+  virtual uint32_t getSymbolAlignment(DataRefImpl Symb) const;
+  virtual uint64_t getCommonSymbolSizeImpl(DataRefImpl Symb) const = 0;
+  virtual SymbolRef::Type getSymbolType(DataRefImpl Symb) const = 0;
   virtual std::error_code getSymbolSection(DataRefImpl Symb,
                                            section_iterator &Res) const = 0;
-  virtual std::error_code getSymbolOther(DataRefImpl Symb,
-                                         uint8_t &Res) const {
-    return object_error::invalid_file_type;
-  }
 
   // Same as above for SectionRef.
   friend class SectionRef;
@@ -233,8 +220,6 @@ class ObjectFile : public SymbolicFile {
   virtual bool isSectionBSS(DataRefImpl Sec) const = 0;
   // A section is 'virtual' if its contents aren't present in the object image.
   virtual bool isSectionVirtual(DataRefImpl Sec) const = 0;
-  virtual bool sectionContainsSymbol(DataRefImpl Sec,
-                                     DataRefImpl Symb) const = 0;
   virtual relocation_iterator section_rel_begin(DataRefImpl Sec) const = 0;
   virtual relocation_iterator section_rel_end(DataRefImpl Sec) const = 0;
   virtual section_iterator getRelocatedSection(DataRefImpl Sec) const;
@@ -242,26 +227,20 @@ class ObjectFile : public SymbolicFile {
   // Same as above for RelocationRef.
   friend class RelocationRef;
   virtual void moveRelocationNext(DataRefImpl &Rel) const = 0;
-  virtual std::error_code getRelocationAddress(DataRefImpl Rel,
-                                               uint64_t &Res) const = 0;
-  virtual std::error_code getRelocationOffset(DataRefImpl Rel,
-                                              uint64_t &Res) const = 0;
+  virtual uint64_t getRelocationOffset(DataRefImpl Rel) const = 0;
   virtual symbol_iterator getRelocationSymbol(DataRefImpl Rel) const = 0;
-  virtual std::error_code getRelocationType(DataRefImpl Rel,
-                                            uint64_t &Res) const = 0;
-  virtual std::error_code
-  getRelocationTypeName(DataRefImpl Rel,
-                        SmallVectorImpl<char> &Result) const = 0;
-  virtual std::error_code
-  getRelocationValueString(DataRefImpl Rel,
-                           SmallVectorImpl<char> &Result) const = 0;
-  virtual std::error_code getRelocationHidden(DataRefImpl Rel,
-                                              bool &Result) const {
-    Result = false;
-    return object_error::success;
-  }
+  virtual uint64_t getRelocationType(DataRefImpl Rel) const = 0;
+  virtual void getRelocationTypeName(DataRefImpl Rel,
+                                     SmallVectorImpl<char> &Result) const = 0;
+
+  uint64_t getSymbolValue(DataRefImpl Symb) const;
 
 public:
+  uint64_t getCommonSymbolSize(DataRefImpl Symb) const {
+    assert(getSymbolFlags(Symb) & SymbolRef::SF_Common);
+    return getCommonSymbolSizeImpl(Symb);
+  }
+
   typedef iterator_range<symbol_iterator> symbol_iterator_range;
   symbol_iterator_range symbols() const {
     return symbol_iterator_range(symbol_begin(), symbol_end());
@@ -324,32 +303,32 @@ class ObjectFile : public SymbolicFile {
 inline SymbolRef::SymbolRef(DataRefImpl SymbolP, const ObjectFile *Owner)
     : BasicSymbolRef(SymbolP, Owner) {}
 
-inline std::error_code SymbolRef::getName(StringRef &Result) const {
-  return getObject()->getSymbolName(getRawDataRefImpl(), Result);
+inline ErrorOr<StringRef> SymbolRef::getName() const {
+  return getObject()->getSymbolName(getRawDataRefImpl());
 }
 
-inline std::error_code SymbolRef::getAddress(uint64_t &Result) const {
-  return getObject()->getSymbolAddress(getRawDataRefImpl(), Result);
+inline ErrorOr<uint64_t> SymbolRef::getAddress() const {
+  return getObject()->getSymbolAddress(getRawDataRefImpl());
 }
 
-inline std::error_code SymbolRef::getAlignment(uint32_t &Result) const {
-  return getObject()->getSymbolAlignment(getRawDataRefImpl(), Result);
+inline uint64_t SymbolRef::getValue() const {
+  return getObject()->getSymbolValue(getRawDataRefImpl());
 }
 
-inline std::error_code SymbolRef::getSize(uint64_t &Result) const {
-  return getObject()->getSymbolSize(getRawDataRefImpl(), Result);
+inline uint32_t SymbolRef::getAlignment() const {
+  return getObject()->getSymbolAlignment(getRawDataRefImpl());
 }
 
-inline std::error_code SymbolRef::getSection(section_iterator &Result) const {
-  return getObject()->getSymbolSection(getRawDataRefImpl(), Result);
+inline uint64_t SymbolRef::getCommonSize() const {
+  return getObject()->getCommonSymbolSize(getRawDataRefImpl());
 }
 
-inline std::error_code SymbolRef::getType(SymbolRef::Type &Result) const {
-  return getObject()->getSymbolType(getRawDataRefImpl(), Result);
+inline std::error_code SymbolRef::getSection(section_iterator &Result) const {
+  return getObject()->getSymbolSection(getRawDataRefImpl(), Result);
 }
 
-inline std::error_code SymbolRef::getOther(uint8_t &Result) const {
-  return getObject()->getSymbolOther(getRawDataRefImpl(), Result);
+inline SymbolRef::Type SymbolRef::getType() const {
+  return getObject()->getSymbolType(getRawDataRefImpl());
 }
 
 inline const ObjectFile *SymbolRef::getObject() const {
@@ -416,11 +395,6 @@ inline bool SectionRef::isVirtual() const {
   return OwningObject->isSectionVirtual(SectionPimpl);
 }
 
-inline bool SectionRef::containsSymbol(SymbolRef S) const {
-  return OwningObject->sectionContainsSymbol(SectionPimpl,
-                                             S.getRawDataRefImpl());
-}
-
 inline relocation_iterator SectionRef::relocation_begin() const {
   return OwningObject->section_rel_begin(SectionPimpl);
 }
@@ -455,41 +429,27 @@ inline void RelocationRef::moveNext() {
   return OwningObject->moveRelocationNext(RelocationPimpl);
 }
 
-inline std::error_code RelocationRef::getAddress(uint64_t &Result) const {
-  return OwningObject->getRelocationAddress(RelocationPimpl, Result);
-}
-
-inline std::error_code RelocationRef::getOffset(uint64_t &Result) const {
-  return OwningObject->getRelocationOffset(RelocationPimpl, Result);
+inline uint64_t RelocationRef::getOffset() const {
+  return OwningObject->getRelocationOffset(RelocationPimpl);
 }
 
 inline symbol_iterator RelocationRef::getSymbol() const {
   return OwningObject->getRelocationSymbol(RelocationPimpl);
 }
 
-inline std::error_code RelocationRef::getType(uint64_t &Result) const {
-  return OwningObject->getRelocationType(RelocationPimpl, Result);
+inline uint64_t RelocationRef::getType() const {
+  return OwningObject->getRelocationType(RelocationPimpl);
 }
 
-inline std::error_code
-RelocationRef::getTypeName(SmallVectorImpl<char> &Result) const {
+inline void RelocationRef::getTypeName(SmallVectorImpl<char> &Result) const {
   return OwningObject->getRelocationTypeName(RelocationPimpl, Result);
 }
 
-inline std::error_code
-RelocationRef::getValueString(SmallVectorImpl<char> &Result) const {
-  return OwningObject->getRelocationValueString(RelocationPimpl, Result);
-}
-
-inline std::error_code RelocationRef::getHidden(bool &Result) const {
-  return OwningObject->getRelocationHidden(RelocationPimpl, Result);
-}
-
 inline DataRefImpl RelocationRef::getRawDataRefImpl() const {
   return RelocationPimpl;
 }
 
-inline const ObjectFile *RelocationRef::getObjectFile() const {
+inline const ObjectFile *RelocationRef::getObject() const {
   return OwningObject;
 }
 
diff --git a/include/llvm/Object/RelocVisitor.h b/include/llvm/Object/RelocVisitor.h
index 91eafd55ad7..d5e4258cb0a 100644
--- a/include/llvm/Object/RelocVisitor.h
+++ b/include/llvm/Object/RelocVisitor.h
@@ -19,9 +19,11 @@
 #include "llvm/ADT/StringRef.h"
 #include "llvm/Object/COFF.h"
 #include "llvm/Object/ELFObjectFile.h"
+#include "llvm/Object/MachO.h"
 #include "llvm/Object/ObjectFile.h"
 #include "llvm/Support/Debug.h"
 #include "llvm/Support/ELF.h"
+#include "llvm/Support/MachO.h"
 #include "llvm/Support/raw_ostream.h"
 
 namespace llvm {
@@ -52,6 +54,8 @@ class RelocVisitor {
       return visitELF(RelocType, R, Value);
     if (isa<COFFObjectFile>(ObjToVisit))
       return visitCOFF(RelocType, R, Value);
+    if (isa<MachOObjectFile>(ObjToVisit))
+      return visitMachO(RelocType, R, Value);
 
     HasError = true;
     return RelocToApply();
@@ -96,9 +100,9 @@ class RelocVisitor {
       case Triple::mips64:
         switch (RelocType) {
         case llvm::ELF::R_MIPS_32:
-          return visitELF_MIPS_32(R, Value);
+          return visitELF_MIPS64_32(R, Value);
         case llvm::ELF::R_MIPS_64:
-          return visitELF_MIPS_64(R, Value);
+          return visitELF_MIPS64_64(R, Value);
         default:
           HasError = true;
           return RelocToApply();
@@ -221,37 +225,32 @@ class RelocVisitor {
     return RelocToApply();
   }
 
-  int64_t getELFAddend32LE(RelocationRef R) {
-    const ELF32LEObjectFile *Obj = cast<ELF32LEObjectFile>(R.getObjectFile());
-    DataRefImpl DRI = R.getRawDataRefImpl();
-    int64_t Addend;
-    Obj->getRelocationAddend(DRI, Addend);
-    return Addend;
+  RelocToApply visitMachO(uint32_t RelocType, RelocationRef R, uint64_t Value) {
+    switch (ObjToVisit.getArch()) {
+    default: break;
+    case Triple::x86_64:
+      switch (RelocType) {
+        default: break;
+        case MachO::X86_64_RELOC_UNSIGNED:
+          return visitMACHO_X86_64_UNSIGNED(R, Value);
+      }
+    }
+    HasError = true;
+    return RelocToApply();
   }
 
-  int64_t getELFAddend64LE(RelocationRef R) {
-    const ELF64LEObjectFile *Obj = cast<ELF64LEObjectFile>(R.getObjectFile());
-    DataRefImpl DRI = R.getRawDataRefImpl();
-    int64_t Addend;
-    Obj->getRelocationAddend(DRI, Addend);
-    return Addend;
+  int64_t getELFAddend(RelocationRef R) {
+    ErrorOr<int64_t> AddendOrErr = ELFRelocationRef(R).getAddend();
+    if (std::error_code EC = AddendOrErr.getError())
+      report_fatal_error(EC.message());
+    return *AddendOrErr;
   }
 
-  int64_t getELFAddend32BE(RelocationRef R) {
-    const ELF32BEObjectFile *Obj = cast<ELF32BEObjectFile>(R.getObjectFile());
-    DataRefImpl DRI = R.getRawDataRefImpl();
-    int64_t Addend;
-    Obj->getRelocationAddend(DRI, Addend);
-    return Addend;
+  uint8_t getLengthMachO64(RelocationRef R) {
+    const MachOObjectFile *Obj = cast<MachOObjectFile>(R.getObject());
+    return Obj->getRelocationLength(R.getRawDataRefImpl());
   }
 
-  int64_t getELFAddend64BE(RelocationRef R) {
-    const ELF64BEObjectFile *Obj = cast<ELF64BEObjectFile>(R.getObjectFile());
-    DataRefImpl DRI = R.getRawDataRefImpl();
-    int64_t Addend;
-    Obj->getRelocationAddend(DRI, Addend);
-    return Addend;
-  }
   /// Operations
 
   /// 386-ELF
@@ -262,15 +261,12 @@ class RelocVisitor {
   // Ideally the Addend here will be the addend in the data for
   // the relocation. It's not actually the case for Rel relocations.
   RelocToApply visitELF_386_32(RelocationRef R, uint64_t Value) {
-    int64_t Addend = getELFAddend32LE(R);
-    return RelocToApply(Value + Addend, 4);
+    return RelocToApply(Value, 4);
   }
 
   RelocToApply visitELF_386_PC32(RelocationRef R, uint64_t Value) {
-    int64_t Addend = getELFAddend32LE(R);
-    uint64_t Address;
-    R.getOffset(Address);
-    return RelocToApply(Value + Addend - Address, 4);
+    uint64_t Address = R.getOffset();
+    return RelocToApply(Value - Address, 4);
   }
 
   /// X86-64 ELF
@@ -278,65 +274,65 @@ class RelocVisitor {
     return RelocToApply(0, 0);
   }
   RelocToApply visitELF_X86_64_64(RelocationRef R, uint64_t Value) {
-    int64_t Addend = getELFAddend64LE(R);
+    int64_t Addend = getELFAddend(R);
     return RelocToApply(Value + Addend, 8);
   }
   RelocToApply visitELF_X86_64_PC32(RelocationRef R, uint64_t Value) {
-    int64_t Addend = getELFAddend64LE(R);
-    uint64_t Address;
-    R.getOffset(Address);
+    int64_t Addend = getELFAddend(R);
+    uint64_t Address = R.getOffset();
     return RelocToApply(Value + Addend - Address, 4);
   }
   RelocToApply visitELF_X86_64_32(RelocationRef R, uint64_t Value) {
-    int64_t Addend = getELFAddend64LE(R);
+    int64_t Addend = getELFAddend(R);
     uint32_t Res = (Value + Addend) & 0xFFFFFFFF;
     return RelocToApply(Res, 4);
   }
   RelocToApply visitELF_X86_64_32S(RelocationRef R, uint64_t Value) {
-    int64_t Addend = getELFAddend64LE(R);
+    int64_t Addend = getELFAddend(R);
     int32_t Res = (Value + Addend) & 0xFFFFFFFF;
     return RelocToApply(Res, 4);
   }
 
   /// PPC64 ELF
   RelocToApply visitELF_PPC64_ADDR32(RelocationRef R, uint64_t Value) {
-    int64_t Addend;
-    getELFRelocationAddend(R, Addend);
+    int64_t Addend = getELFAddend(R);
     uint32_t Res = (Value + Addend) & 0xFFFFFFFF;
     return RelocToApply(Res, 4);
   }
   RelocToApply visitELF_PPC64_ADDR64(RelocationRef R, uint64_t Value) {
-    int64_t Addend;
-    getELFRelocationAddend(R, Addend);
+    int64_t Addend = getELFAddend(R);
     return RelocToApply(Value + Addend, 8);
   }
 
   /// PPC32 ELF
   RelocToApply visitELF_PPC_ADDR32(RelocationRef R, uint64_t Value) {
-    int64_t Addend = getELFAddend32BE(R);
+    int64_t Addend = getELFAddend(R);
     uint32_t Res = (Value + Addend) & 0xFFFFFFFF;
     return RelocToApply(Res, 4);
   }
 
   /// MIPS ELF
   RelocToApply visitELF_MIPS_32(RelocationRef R, uint64_t Value) {
-    int64_t Addend;
-    getELFRelocationAddend(R, Addend);
+    uint32_t Res = Value & 0xFFFFFFFF;
+    return RelocToApply(Res, 4);
+  }
+
+  /// MIPS64 ELF
+  RelocToApply visitELF_MIPS64_32(RelocationRef R, uint64_t Value) {
+    int64_t Addend = getELFAddend(R);
     uint32_t Res = (Value + Addend) & 0xFFFFFFFF;
     return RelocToApply(Res, 4);
   }
 
-  RelocToApply visitELF_MIPS_64(RelocationRef R, uint64_t Value) {
-    int64_t Addend;
-    getELFRelocationAddend(R, Addend);
+  RelocToApply visitELF_MIPS64_64(RelocationRef R, uint64_t Value) {
+    int64_t Addend = getELFAddend(R);
     uint64_t Res = (Value + Addend);
     return RelocToApply(Res, 8);
   }
 
   // AArch64 ELF
   RelocToApply visitELF_AARCH64_ABS32(RelocationRef R, uint64_t Value) {
-    int64_t Addend;
-    getELFRelocationAddend(R, Addend);
+    int64_t Addend = getELFAddend(R);
     int64_t Res =  Value + Addend;
 
     // Overflow check allows for both signed and unsigned interpretation.
@@ -347,14 +343,13 @@ class RelocVisitor {
   }
 
   RelocToApply visitELF_AARCH64_ABS64(RelocationRef R, uint64_t Value) {
-    int64_t Addend;
-    getELFRelocationAddend(R, Addend);
+    int64_t Addend = getELFAddend(R);
     return RelocToApply(Value + Addend, 8);
   }
 
   // SystemZ ELF
   RelocToApply visitELF_390_32(RelocationRef R, uint64_t Value) {
-    int64_t Addend = getELFAddend64BE(R);
+    int64_t Addend = getELFAddend(R);
     int64_t Res = Value + Addend;
 
     // Overflow check allows for both signed and unsigned interpretation.
@@ -365,29 +360,27 @@ class RelocVisitor {
   }
 
   RelocToApply visitELF_390_64(RelocationRef R, uint64_t Value) {
-    int64_t Addend = getELFAddend64BE(R);
+    int64_t Addend = getELFAddend(R);
     return RelocToApply(Value + Addend, 8);
   }
 
   RelocToApply visitELF_SPARC_32(RelocationRef R, uint32_t Value) {
-    int32_t Addend = getELFAddend32BE(R);
+    int32_t Addend = getELFAddend(R);
     return RelocToApply(Value + Addend, 4);
   }
 
   RelocToApply visitELF_SPARCV9_32(RelocationRef R, uint64_t Value) {
-    int32_t Addend = getELFAddend64BE(R);
+    int32_t Addend = getELFAddend(R);
     return RelocToApply(Value + Addend, 4);
   }
 
   RelocToApply visitELF_SPARCV9_64(RelocationRef R, uint64_t Value) {
-    int64_t Addend = getELFAddend64BE(R);
+    int64_t Addend = getELFAddend(R);
     return RelocToApply(Value + Addend, 8);
   }
 
   RelocToApply visitELF_ARM_ABS32(RelocationRef R, uint64_t Value) {
-    int64_t Addend;
-    getELFRelocationAddend(R, Addend);
-    int64_t Res = Value + Addend;
+    int64_t Res = Value;
 
     // Overflow check allows for both signed and unsigned interpretation.
     if (Res < INT32_MIN || Res > UINT32_MAX)
@@ -413,6 +406,13 @@ class RelocVisitor {
   RelocToApply visitCOFF_AMD64_ADDR64(RelocationRef R, uint64_t Value) {
     return RelocToApply(Value, /*Width=*/8);
   }
+
+  // X86_64 MachO
+  RelocToApply visitMACHO_X86_64_UNSIGNED(RelocationRef R, uint64_t Value) {
+    uint8_t Length = getLengthMachO64(R);
+    Length = 1<<Length;
+    return RelocToApply(Value, Length);
+  }
 };
 
 }
diff --git a/include/llvm/Object/StackMapParser.h b/include/llvm/Object/StackMapParser.h
new file mode 100644
index 00000000000..276eab6c294
--- /dev/null
+++ b/include/llvm/Object/StackMapParser.h
@@ -0,0 +1,442 @@
+//===-------- StackMapParser.h - StackMap Parsing Support -------*- C++ -*-===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_CODEGEN_STACKMAPPARSER_H
+#define LLVM_CODEGEN_STACKMAPPARSER_H
+
+#include "llvm/Support/Debug.h"
+#include "llvm/Support/Endian.h"
+#include <map>
+#include <vector>
+
+namespace llvm {
+
+template <support::endianness Endianness>
+class StackMapV1Parser {
+public:
+
+  template <typename AccessorT>
+  class AccessorIterator {
+  public:
+
+    AccessorIterator(AccessorT A) : A(A) {}
+    AccessorIterator& operator++() { A = A.next(); return *this; }
+    AccessorIterator operator++(int) {
+      auto tmp = *this;
+      ++*this;
+      return tmp;
+    }
+
+    bool operator==(const AccessorIterator &Other) {
+      return A.P == Other.A.P;
+    }
+
+    bool operator!=(const AccessorIterator &Other) { return !(*this == Other); }
+
+    AccessorT& operator*() { return A; }
+    AccessorT* operator->() { return &A; }
+
+  private:
+    AccessorT A;
+  };
+
+  /// Accessor for function records.
+  class FunctionAccessor {
+    friend class StackMapV1Parser;
+  public:
+
+    /// Get the function address.
+    uint64_t getFunctionAddress() const {
+      return read<uint64_t>(P);
+    }
+
+    /// Get the function's stack size.
+    uint32_t getStackSize() const {
+      return read<uint64_t>(P + sizeof(uint64_t));
+    }
+
+  private:
+    FunctionAccessor(const uint8_t *P) : P(P) {}
+
+    const static int FunctionAccessorSize = 2 * sizeof(uint64_t);
+
+    FunctionAccessor next() const {
+      return FunctionAccessor(P + FunctionAccessorSize);
+    }
+
+    const uint8_t *P;
+  };
+
+  /// Accessor for constants.
+  class ConstantAccessor {
+    friend class StackMapV1Parser;
+  public:
+
+    /// Return the value of this constant.
+    uint64_t getValue() const { return read<uint64_t>(P); }
+
+  private:
+
+    ConstantAccessor(const uint8_t *P) : P(P) {}
+
+    const static int ConstantAccessorSize = sizeof(uint64_t);
+
+    ConstantAccessor next() const {
+      return ConstantAccessor(P + ConstantAccessorSize);
+    }
+
+    const uint8_t *P;
+  };
+
+  // Forward-declare RecordAccessor so we can friend it below.
+  class RecordAccessor;
+
+  enum class LocationKind : uint8_t {
+    Register = 1, Direct = 2, Indirect = 3, Constant = 4, ConstantIndex = 5
+  };
+
+
+  /// Accessor for location records.
+  class LocationAccessor {
+    friend class StackMapV1Parser;
+    friend class RecordAccessor;
+  public:
+
+    /// Get the Kind for this location.
+    LocationKind getKind() const {
+      return LocationKind(P[KindOffset]);
+    }
+
+    /// Get the Dwarf register number for this location.
+    uint16_t getDwarfRegNum() const {
+      return read<uint16_t>(P + DwarfRegNumOffset);
+    }
+
+    /// Get the small-constant for this location. (Kind must be Constant).
+    uint32_t getSmallConstant() const {
+      assert(getKind() == LocationKind::Constant && "Not a small constant.");
+      return read<uint32_t>(P + SmallConstantOffset);
+    }
+
+    /// Get the constant-index for this location. (Kind must be ConstantIndex).
+    uint32_t getConstantIndex() const {
+      assert(getKind() == LocationKind::ConstantIndex &&
+             "Not a constant-index.");
+      return read<uint32_t>(P + SmallConstantOffset);
+    }
+
+    /// Get the offset for this location. (Kind must be Direct or Indirect).
+    int32_t getOffset() const {
+      assert((getKind() == LocationKind::Direct ||
+              getKind() == LocationKind::Indirect) &&
+             "Not direct or indirect.");
+      return read<int32_t>(P + SmallConstantOffset);
+    }
+
+  private:
+
+    LocationAccessor(const uint8_t *P) : P(P) {}
+
+    LocationAccessor next() const {
+      return LocationAccessor(P + LocationAccessorSize);
+    }
+
+    static const int KindOffset = 0;
+    static const int DwarfRegNumOffset = KindOffset + sizeof(uint16_t);
+    static const int SmallConstantOffset = DwarfRegNumOffset + sizeof(uint16_t);
+    static const int LocationAccessorSize = sizeof(uint64_t);
+
+    const uint8_t *P;
+  };
+
+  /// Accessor for stackmap live-out fields.
+  class LiveOutAccessor {
+    friend class StackMapV1Parser;
+    friend class RecordAccessor;
+  public:
+
+    /// Get the Dwarf register number for this live-out.
+    uint16_t getDwarfRegNum() const {
+      return read<uint16_t>(P + DwarfRegNumOffset);
+    }
+
+    /// Get the size in bytes of live [sub]register.
+    unsigned getSizeInBytes() const {
+      return read<uint8_t>(P + SizeOffset);
+    }
+
+  private:
+
+    LiveOutAccessor(const uint8_t *P) : P(P) {}
+
+    LiveOutAccessor next() const {
+      return LiveOutAccessor(P + LiveOutAccessorSize);
+    }
+
+    static const int DwarfRegNumOffset = 0;
+    static const int SizeOffset =
+      DwarfRegNumOffset + sizeof(uint16_t) + sizeof(uint8_t);
+    static const int LiveOutAccessorSize = sizeof(uint32_t);
+
+    const uint8_t *P;
+  };
+
+  /// Accessor for stackmap records.
+  class RecordAccessor {
+    friend class StackMapV1Parser;
+  public:
+
+    typedef AccessorIterator<LocationAccessor> location_iterator;
+    typedef AccessorIterator<LiveOutAccessor> liveout_iterator;
+
+    /// Get the patchpoint/stackmap ID for this record.
+    uint64_t getID() const {
+      return read<uint64_t>(P + PatchpointIDOffset);
+    }
+
+    /// Get the instruction offset (from the start of the containing function)
+    /// for this record.
+    uint32_t getInstructionOffset() const {
+      return read<uint32_t>(P + InstructionOffsetOffset);
+    }
+
+    /// Get the number of locations contained in this record.
+    uint16_t getNumLocations() const {
+      return read<uint16_t>(P + NumLocationsOffset);
+    }
+
+    /// Get the location with the given index.
+    LocationAccessor getLocation(unsigned LocationIndex) const {
+      unsigned LocationOffset =
+        LocationListOffset + LocationIndex * LocationSize;
+      return LocationAccessor(P + LocationOffset);
+    }
+
+    /// Begin iterator for locations.
+    location_iterator location_begin() const {
+      return location_iterator(getLocation(0));
+    }
+
+    /// End iterator for locations.
+    location_iterator location_end() const {
+      return location_iterator(getLocation(getNumLocations()));
+    }
+
+    /// Iterator range for locations.
+    iterator_range<location_iterator> locations() const {
+      return make_range(location_begin(), location_end());
+    }
+
+    /// Get the number of liveouts contained in this record.
+    uint16_t getNumLiveOuts() const {
+      return read<uint16_t>(P + getNumLiveOutsOffset());
+    }
+
+    /// Get the live-out with the given index.
+    LiveOutAccessor getLiveOut(unsigned LiveOutIndex) const {
+      unsigned LiveOutOffset =
+        getNumLiveOutsOffset() + sizeof(uint16_t) + LiveOutIndex * LiveOutSize;
+      return LiveOutAccessor(P + LiveOutOffset);
+    }
+
+    /// Begin iterator for live-outs.
+    liveout_iterator liveouts_begin() const {
+      return liveout_iterator(getLiveOut(0));
+    }
+
+
+    /// End iterator for live-outs.
+    liveout_iterator liveouts_end() const {
+      return liveout_iterator(getLiveOut(getNumLiveOuts()));
+    }
+
+    /// Iterator range for live-outs.
+    iterator_range<liveout_iterator> liveouts() const {
+      return make_range(liveouts_begin(), liveouts_end());
+    }
+
+  private:
+
+    RecordAccessor(const uint8_t *P) : P(P) {}
+
+    unsigned getNumLiveOutsOffset() const {
+      return LocationListOffset + LocationSize * getNumLocations() +
+             sizeof(uint16_t);
+    }
+
+    unsigned getSizeInBytes() const {
+      unsigned RecordSize =
+        getNumLiveOutsOffset() + sizeof(uint16_t) + getNumLiveOuts() * LiveOutSize;
+      return (RecordSize + 7) & ~0x7;
+    }
+
+    RecordAccessor next() const {
+      return RecordAccessor(P + getSizeInBytes());
+    }
+
+    static const unsigned PatchpointIDOffset = 0;
+    static const unsigned InstructionOffsetOffset =
+      PatchpointIDOffset + sizeof(uint64_t);
+    static const unsigned NumLocationsOffset =
+      InstructionOffsetOffset + sizeof(uint32_t) + sizeof(uint16_t);
+    static const unsigned LocationListOffset =
+      NumLocationsOffset + sizeof(uint16_t);
+    static const unsigned LocationSize = sizeof(uint64_t);
+    static const unsigned LiveOutSize = sizeof(uint32_t);
+
+    const uint8_t *P;
+  };
+
+  /// Construct a parser for a version-1 stackmap. StackMap data will be read
+  /// from the given array.
+  StackMapV1Parser(ArrayRef<uint8_t> StackMapSection)
+      : StackMapSection(StackMapSection) {
+    ConstantsListOffset = FunctionListOffset + getNumFunctions() * FunctionSize;
+
+    assert(StackMapSection[0] == 1 &&
+           "StackMapV1Parser can only parse version 1 stackmaps");
+
+    unsigned CurrentRecordOffset =
+      ConstantsListOffset + getNumConstants() * ConstantSize;
+
+    for (unsigned I = 0, E = getNumRecords(); I != E; ++I) {
+      StackMapRecordOffsets.push_back(CurrentRecordOffset);
+      CurrentRecordOffset +=
+        RecordAccessor(&StackMapSection[CurrentRecordOffset]).getSizeInBytes();
+    }
+  }
+
+  typedef AccessorIterator<FunctionAccessor> function_iterator;
+  typedef AccessorIterator<ConstantAccessor> constant_iterator;
+  typedef AccessorIterator<RecordAccessor> record_iterator;
+
+  /// Get the version number of this stackmap. (Always returns 1).
+  unsigned getVersion() const { return 1; }
+
+  /// Get the number of functions in the stack map.
+  uint32_t getNumFunctions() const {
+    return read<uint32_t>(&StackMapSection[NumFunctionsOffset]);
+  }
+
+  /// Get the number of large constants in the stack map.
+  uint32_t getNumConstants() const {
+    return read<uint32_t>(&StackMapSection[NumConstantsOffset]);
+  }
+
+  /// Get the number of stackmap records in the stackmap.
+  uint32_t getNumRecords() const {
+    return read<uint32_t>(&StackMapSection[NumRecordsOffset]);
+  }
+
+  /// Return an FunctionAccessor for the given function index.
+  FunctionAccessor getFunction(unsigned FunctionIndex) const {
+    return FunctionAccessor(StackMapSection.data() +
+                            getFunctionOffset(FunctionIndex));
+  }
+
+  /// Begin iterator for functions.
+  function_iterator functions_begin() const {
+    return function_iterator(getFunction(0));
+  }
+
+  /// End iterator for functions.
+  function_iterator functions_end() const {
+    return function_iterator(
+             FunctionAccessor(StackMapSection.data() +
+                              getFunctionOffset(getNumFunctions())));
+  }
+
+  /// Iterator range for functions.
+  iterator_range<function_iterator> functions() const {
+    return make_range(functions_begin(), functions_end());
+  }
+
+  /// Return the large constant at the given index.
+  ConstantAccessor getConstant(unsigned ConstantIndex) const {
+    return ConstantAccessor(StackMapSection.data() +
+                            getConstantOffset(ConstantIndex));
+  }
+
+  /// Begin iterator for constants.
+  constant_iterator constants_begin() const {
+    return constant_iterator(getConstant(0));
+  }
+
+  /// End iterator for constants.
+  constant_iterator constants_end() const {
+    return constant_iterator(
+             ConstantAccessor(StackMapSection.data() +
+                              getConstantOffset(getNumConstants())));
+  }
+
+  /// Iterator range for constants.
+  iterator_range<constant_iterator> constants() const {
+    return make_range(constants_begin(), constants_end());
+  }
+
+  /// Return a RecordAccessor for the given record index.
+  RecordAccessor getRecord(unsigned RecordIndex) const {
+    std::size_t RecordOffset = StackMapRecordOffsets[RecordIndex];
+    return RecordAccessor(StackMapSection.data() + RecordOffset);
+  }
+
+  /// Begin iterator for records.
+  record_iterator records_begin() const {
+    if (getNumRecords() == 0)
+      return record_iterator(RecordAccessor(nullptr));
+    return record_iterator(getRecord(0));
+  }
+
+  /// End iterator for records.
+  record_iterator records_end() const {
+    // Records need to be handled specially, since we cache the start addresses
+    // for them: We can't just compute the 1-past-the-end address, we have to
+    // look at the last record and use the 'next' method.
+    if (getNumRecords() == 0)
+      return record_iterator(RecordAccessor(nullptr));
+    return record_iterator(getRecord(getNumRecords() - 1).next());
+  }
+
+  /// Iterator range for records.
+  iterator_range<record_iterator> records() const {
+    return make_range(records_begin(), records_end());
+  }
+
+private:
+
+  template <typename T>
+  static T read(const uint8_t *P) {
+    return support::endian::read<T, Endianness, 1>(P);
+  }
+
+  static const unsigned HeaderOffset = 0;
+  static const unsigned NumFunctionsOffset = HeaderOffset + sizeof(uint32_t);
+  static const unsigned NumConstantsOffset = NumFunctionsOffset + sizeof(uint32_t);
+  static const unsigned NumRecordsOffset = NumConstantsOffset + sizeof(uint32_t);
+  static const unsigned FunctionListOffset = NumRecordsOffset + sizeof(uint32_t);
+
+  static const unsigned FunctionSize = 2 * sizeof(uint64_t);
+  static const unsigned ConstantSize = sizeof(uint64_t);
+
+  std::size_t getFunctionOffset(unsigned FunctionIndex) const {
+    return FunctionListOffset + FunctionIndex * FunctionSize;
+  }
+
+  std::size_t getConstantOffset(unsigned ConstantIndex) const {
+    return ConstantsListOffset + ConstantIndex * ConstantSize;
+  }
+
+  ArrayRef<uint8_t> StackMapSection;
+  unsigned ConstantsListOffset;
+  std::vector<unsigned> StackMapRecordOffsets;
+};
+
+}
+
+#endif
diff --git a/include/llvm/Object/SymbolSize.h b/include/llvm/Object/SymbolSize.h
new file mode 100644
index 00000000000..f2ce70f4208
--- /dev/null
+++ b/include/llvm/Object/SymbolSize.h
@@ -0,0 +1,23 @@
+//===- SymbolSize.h ---------------------------------------------*- C++ -*-===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_OBJECT_SYMBOLSIZE_H
+#define LLVM_OBJECT_SYMBOLSIZE_H
+
+#include "llvm/Object/ObjectFile.h"
+
+namespace llvm {
+namespace object {
+std::vector<std::pair<SymbolRef, uint64_t>>
+computeSymbolSizes(const ObjectFile &O);
+}
+} // namespace llvm
+
+#endif
diff --git a/include/llvm/Object/SymbolicFile.h b/include/llvm/Object/SymbolicFile.h
index 114b229b822..6b10ceb263a 100644
--- a/include/llvm/Object/SymbolicFile.h
+++ b/include/llvm/Object/SymbolicFile.h
@@ -15,6 +15,7 @@
 #define LLVM_OBJECT_SYMBOLICFILE_H
 
 #include "llvm/Object/Binary.h"
+#include "llvm/Support/Format.h"
 
 namespace llvm {
 namespace object {
@@ -29,6 +30,12 @@ union DataRefImpl {
   DataRefImpl() { std::memset(this, 0, sizeof(DataRefImpl)); }
 };
 
+template <typename OStream>
+OStream& operator<<(OStream &OS, const DataRefImpl &D) {
+  OS << "(" << format("0x%x8", D.p) << " (" << format("0x%x8", D.d.a) << ", " << format("0x%x8", D.d.b) << "))";
+  return OS;
+}
+
 inline bool operator==(const DataRefImpl &a, const DataRefImpl &b) {
   // Check bitwise identical. This is the only legal way to compare a union w/o
   // knowing which member is in use.
@@ -93,6 +100,7 @@ class BasicSymbolRef {
     SF_FormatSpecific = 1U << 7, // Specific to the object file format
                                  // (e.g. section symbols)
     SF_Thumb = 1U << 8,          // Thumb symbol in a 32-bit ARM binary
+    SF_Hidden = 1U << 9,         // Symbol has hidden visibility
   };
 
   BasicSymbolRef() : OwningObject(nullptr) { }
@@ -114,8 +122,6 @@ class BasicSymbolRef {
 
 typedef content_iterator<BasicSymbolRef> basic_symbol_iterator;
 
-const uint64_t UnknownAddressOrSize = ~0ULL;
-
 class SymbolicFile : public Binary {
 public:
   ~SymbolicFile() override;
diff --git a/include/llvm/Option/Arg.h b/include/llvm/Option/Arg.h
index 5f6941a53d0..e1b72b6267c 100644
--- a/include/llvm/Option/Arg.h
+++ b/include/llvm/Option/Arg.h
@@ -93,9 +93,8 @@ class Arg {
     return Values[N];
   }
 
-  SmallVectorImpl<const char*> &getValues() {
-    return Values;
-  }
+  SmallVectorImpl<const char *> &getValues() { return Values; }
+  const SmallVectorImpl<const char *> &getValues() const { return Values; }
 
   bool containsValue(StringRef Value) const {
     for (unsigned i = 0, e = getNumValues(); i != e; ++i)
diff --git a/include/llvm/Option/ArgList.h b/include/llvm/Option/ArgList.h
index 23b04513178..0a78e935afa 100644
--- a/include/llvm/Option/ArgList.h
+++ b/include/llvm/Option/ArgList.h
@@ -14,6 +14,7 @@
 #include "llvm/ADT/SmallString.h"
 #include "llvm/ADT/StringRef.h"
 #include "llvm/ADT/Twine.h"
+#include "llvm/Option/Arg.h"
 #include "llvm/Option/OptSpecifier.h"
 #include "llvm/Option/Option.h"
 #include <list>
@@ -23,7 +24,6 @@
 
 namespace llvm {
 namespace opt {
-class Arg;
 class ArgList;
 class Option;
 
@@ -92,10 +92,6 @@ class arg_iterator {
 /// check for the presence of Arg instances for a particular Option
 /// and to iterate over groups of arguments.
 class ArgList {
-private:
-  ArgList(const ArgList &) = delete;
-  void operator=(const ArgList &) = delete;
-
 public:
   typedef SmallVector<Arg*, 16> arglist_type;
   typedef arglist_type::iterator iterator;
@@ -108,12 +104,23 @@ class ArgList {
   arglist_type Args;
 
 protected:
-  // Default ctor provided explicitly as it is not provided implicitly due to
-  // the presence of the (deleted) copy ctor above.
-  ArgList() { }
-  // Virtual to provide a vtable anchor and because -Wnon-virtua-dtor warns, not
-  // because this type is ever actually destroyed polymorphically.
-  virtual ~ArgList();
+  // Make the default special members protected so they won't be used to slice
+  // derived objects, but can still be used by derived objects to implement
+  // their own special members.
+  ArgList() = default;
+  // Explicit move operations to ensure the container is cleared post-move
+  // otherwise it could lead to a double-delete in the case of moving of an
+  // InputArgList which deletes the contents of the container. If we could fix
+  // up the ownership here (delegate storage/ownership to the derived class so
+  // it can be a container of unique_ptr) this would be simpler.
+  ArgList(ArgList &&RHS) : Args(std::move(RHS.Args)) { RHS.Args.clear(); }
+  ArgList &operator=(ArgList &&RHS) {
+    Args = std::move(RHS.Args);
+    RHS.Args.clear();
+    return *this;
+  }
+  // Protect the dtor to ensure this type is never destroyed polymorphically.
+  ~ArgList() = default;
 
 public:
 
@@ -252,6 +259,9 @@ class ArgList {
   void AddLastArg(ArgStringList &Output, OptSpecifier Id0,
                   OptSpecifier Id1) const;
 
+  /// AddAllArgs - Render all arguments matching any of the given ids.
+  void AddAllArgs(ArgStringList &Output, ArrayRef<OptSpecifier> Ids) const;
+
   /// AddAllArgs - Render all arguments matching the given ids.
   void AddAllArgs(ArgStringList &Output, OptSpecifier Id0,
                   OptSpecifier Id1 = 0U, OptSpecifier Id2 = 0U) const;
@@ -299,7 +309,7 @@ class ArgList {
   /// @}
 };
 
-class InputArgList : public ArgList  {
+class InputArgList final : public ArgList {
 private:
   /// List of argument strings used by the contained Args.
   ///
@@ -318,9 +328,24 @@ class InputArgList : public ArgList  {
   /// The number of original input argument strings.
   unsigned NumInputArgStrings;
 
+  /// Release allocated arguments.
+  void releaseMemory();
+
 public:
   InputArgList(const char* const *ArgBegin, const char* const *ArgEnd);
-  ~InputArgList() override;
+  InputArgList(InputArgList &&RHS)
+      : ArgList(std::move(RHS)), ArgStrings(std::move(RHS.ArgStrings)),
+        SynthesizedStrings(std::move(RHS.SynthesizedStrings)),
+        NumInputArgStrings(RHS.NumInputArgStrings) {}
+  InputArgList &operator=(InputArgList &&RHS) {
+    releaseMemory();
+    ArgList::operator=(std::move(RHS));
+    ArgStrings = std::move(RHS.ArgStrings);
+    SynthesizedStrings = std::move(RHS.SynthesizedStrings);
+    NumInputArgStrings = RHS.NumInputArgStrings;
+    return *this;
+  }
+  ~InputArgList() { releaseMemory(); }
 
   const char *getArgString(unsigned Index) const override {
     return ArgStrings[Index];
@@ -346,7 +371,7 @@ class InputArgList : public ArgList  {
 
 /// DerivedArgList - An ordered collection of driver arguments,
 /// whose storage may be in another argument list.
-class DerivedArgList : public ArgList {
+class DerivedArgList final : public ArgList {
   const InputArgList &BaseArgs;
 
   /// The list of arguments we synthesized.
@@ -355,7 +380,6 @@ class DerivedArgList : public ArgList {
 public:
   /// Construct a new derived arg list from \p BaseArgs.
   DerivedArgList(const InputArgList &BaseArgs);
-  ~DerivedArgList() override;
 
   const char *getArgString(unsigned Index) const override {
     return BaseArgs.getArgString(Index);
diff --git a/include/llvm/Option/OptTable.h b/include/llvm/Option/OptTable.h
index a7ff46919cd..96f51cf3317 100644
--- a/include/llvm/Option/OptTable.h
+++ b/include/llvm/Option/OptTable.h
@@ -10,6 +10,7 @@
 #ifndef LLVM_OPTION_OPTTABLE_H
 #define LLVM_OPTION_OPTTABLE_H
 
+#include "llvm/ADT/ArrayRef.h"
 #include "llvm/ADT/StringSet.h"
 #include "llvm/Option/OptSpecifier.h"
 
@@ -141,8 +142,6 @@ class OptTable {
   /// The only error that can occur in this routine is if an argument is
   /// missing values; in this case \p MissingArgCount will be non-zero.
   ///
-  /// \param ArgBegin - The beginning of the argument vector.
-  /// \param ArgEnd - The end of the argument vector.
   /// \param MissingArgIndex - On error, the index of the option which could
   /// not be parsed.
   /// \param MissingArgCount - On error, the number of missing options.
@@ -152,12 +151,9 @@ class OptTable {
   /// is the default and means exclude nothing.
   /// \return An InputArgList; on error this will contain all the options
   /// which could be parsed.
-  InputArgList *ParseArgs(const char* const *ArgBegin,
-                          const char* const *ArgEnd,
-                          unsigned &MissingArgIndex,
-                          unsigned &MissingArgCount,
-                          unsigned FlagsToInclude = 0,
-                          unsigned FlagsToExclude = 0) const;
+  InputArgList ParseArgs(ArrayRef<const char *> Args, unsigned &MissingArgIndex,
+                         unsigned &MissingArgCount, unsigned FlagsToInclude = 0,
+                         unsigned FlagsToExclude = 0) const;
 
   /// \brief Render the help text for an option table.
   ///
diff --git a/include/llvm/PassAnalysisSupport.h b/include/llvm/PassAnalysisSupport.h
index 38adb2dbb69..0b318fc8fb0 100644
--- a/include/llvm/PassAnalysisSupport.h
+++ b/include/llvm/PassAnalysisSupport.h
@@ -27,28 +27,27 @@
 namespace llvm {
 
 //===----------------------------------------------------------------------===//
-// AnalysisUsage - Represent the analysis usage information of a pass.  This
-// tracks analyses that the pass REQUIRES (must be available when the pass
-// runs), REQUIRES TRANSITIVE (must be available throughout the lifetime of the
-// pass), and analyses that the pass PRESERVES (the pass does not invalidate the
-// results of these analyses).  This information is provided by a pass to the
-// Pass infrastructure through the getAnalysisUsage virtual function.
-//
+/// Represent the analysis usage information of a pass.  This tracks analyses
+/// that the pass REQUIRES (must be available when the pass runs), REQUIRES
+/// TRANSITIVE (must be available throughout the lifetime of the pass), and
+/// analyses that the pass PRESERVES (the pass does not invalidate the results
+/// of these analyses).  This information is provided by a pass to the Pass
+/// infrastructure through the getAnalysisUsage virtual function.
+///
 class AnalysisUsage {
 public:
   typedef SmallVector<AnalysisID, 32> VectorType;
 
 private:
-  // Sets of analyses required and preserved by a pass
+  /// Sets of analyses required and preserved by a pass
   VectorType Required, RequiredTransitive, Preserved;
   bool PreservesAll;
 
 public:
   AnalysisUsage() : PreservesAll(false) {}
 
-  // addRequired - Add the specified ID to the required set of the usage info
-  // for a pass.
-  //
+  ///@{
+  /// Add the specified ID to the required set of the usage info for a pass.
   AnalysisUsage &addRequiredID(const void *ID);
   AnalysisUsage &addRequiredID(char &ID);
   template<class PassClass>
@@ -61,10 +60,10 @@ class AnalysisUsage {
   AnalysisUsage &addRequiredTransitive() {
     return addRequiredTransitiveID(PassClass::ID);
   }
+  ///@}
 
-  // addPreserved - Add the specified ID to the set of analyses preserved by
-  // this pass
-  //
+  ///@{
+  /// Add the specified ID to the set of analyses preserved by this pass.
   AnalysisUsage &addPreservedID(const void *ID) {
     Preserved.push_back(ID);
     return *this;
@@ -73,29 +72,28 @@ class AnalysisUsage {
     Preserved.push_back(&ID);
     return *this;
   }
+  ///@}
 
-  // addPreserved - Add the specified Pass class to the set of analyses
-  // preserved by this pass.
-  //
+  /// Add the specified Pass class to the set of analyses preserved by this pass.
   template<class PassClass>
   AnalysisUsage &addPreserved() {
     Preserved.push_back(&PassClass::ID);
     return *this;
   }
 
-  // addPreserved - Add the Pass with the specified argument string to the set
-  // of analyses preserved by this pass. If no such Pass exists, do nothing.
-  // This can be useful when a pass is trivially preserved, but may not be
-  // linked in. Be careful about spelling!
-  //
+  /// Add the Pass with the specified argument string to the set of analyses
+  /// preserved by this pass. If no such Pass exists, do nothing. This can be
+  /// useful when a pass is trivially preserved, but may not be linked in. Be
+  /// careful about spelling!
   AnalysisUsage &addPreserved(StringRef Arg);
 
-  // setPreservesAll - Set by analyses that do not transform their input at all
+  /// Set by analyses that do not transform their input at all
   void setPreservesAll() { PreservesAll = true; }
+
+  /// Determine whether a pass said it does not transform its input at all
   bool getPreservesAll() const { return PreservesAll; }
 
-  /// setPreservesCFG - This function should be called by the pass, iff they do
-  /// not:
+  /// This function should be called by the pass, iff they do not:
   ///
   ///  1. Add or remove basic blocks from the function
   ///  2. Modify terminator instructions in any way.
@@ -113,10 +111,10 @@ class AnalysisUsage {
 };
 
 //===----------------------------------------------------------------------===//
-// AnalysisResolver - Simple interface used by Pass objects to pull all
-// analysis information out of pass manager that is responsible to manage
-// the pass.
-//
+/// AnalysisResolver - Simple interface used by Pass objects to pull all
+/// analysis information out of pass manager that is responsible to manage
+/// the pass.
+///
 class PMDataManager;
 class AnalysisResolver {
 private:
@@ -124,10 +122,10 @@ class AnalysisResolver {
 
 public:
   explicit AnalysisResolver(PMDataManager &P) : PM(P) { }
-  
+
   inline PMDataManager &getPMDataManager() { return PM; }
 
-  // Find pass that is implementing PI.
+  /// Find pass that is implementing PI.
   Pass *findImplPass(AnalysisID PI) {
     Pass *ResultPass = nullptr;
     for (unsigned i = 0; i < AnalysisImpls.size() ; ++i) {
@@ -139,7 +137,7 @@ class AnalysisResolver {
     return ResultPass;
   }
 
-  // Find pass that is implementing PI. Initialize pass for Function F.
+  /// Find pass that is implementing PI. Initialize pass for Function F.
   Pass *findImplPass(Pass *P, AnalysisID PI, Function &F);
 
   void addAnalysisImplsPair(AnalysisID PI, Pass *P) {
@@ -149,21 +147,20 @@ class AnalysisResolver {
     AnalysisImpls.push_back(pir);
   }
 
-  /// clearAnalysisImpls - Clear cache that is used to connect a pass to the
-  /// the analysis (PassInfo).
+  /// Clear cache that is used to connect a pass to the the analysis (PassInfo).
   void clearAnalysisImpls() {
     AnalysisImpls.clear();
   }
 
-  // getAnalysisIfAvailable - Return analysis result or null if it doesn't exist
+  /// Return analysis result or null if it doesn't exist.
   Pass *getAnalysisIfAvailable(AnalysisID ID, bool Direction) const;
 
 private:
-  // AnalysisImpls - This keeps track of which passes implements the interfaces
-  // that are required by the current pass (to implement getAnalysis()).
+  /// This keeps track of which passes implements the interfaces that are
+  /// required by the current pass (to implement getAnalysis()).
   std::vector<std::pair<AnalysisID, Pass*> > AnalysisImpls;
 
-  // PassManager that is used to resolve analysis info
+  /// PassManager that is used to resolve analysis info
   PMDataManager &PM;
 };
 
@@ -240,7 +237,7 @@ AnalysisType &Pass::getAnalysisID(AnalysisID PI, Function &F) {
   // vector.
   Pass *ResultPass = Resolver->findImplPass(this, PI, F);
   assert(ResultPass && "Unable to find requested analysis info");
-  
+
   // Because the AnalysisType may not be a subclass of pass (for
   // AnalysisGroups), we use getAdjustedAnalysisPointer here to potentially
   // adjust the return pointer (because the class may multiply inherit, once
diff --git a/include/llvm/ProfileData/CoverageMapping.h b/include/llvm/ProfileData/CoverageMapping.h
index e9d6ea5a492..3488e793d84 100644
--- a/include/llvm/ProfileData/CoverageMapping.h
+++ b/include/llvm/ProfileData/CoverageMapping.h
@@ -410,7 +410,7 @@ class CoverageMapping {
   /// \brief Load the coverage mapping from the given files.
   static ErrorOr<std::unique_ptr<CoverageMapping>>
   load(StringRef ObjectFilename, StringRef ProfileFilename,
-       Triple::ArchType Arch = Triple::ArchType::UnknownArch);
+       StringRef Arch = StringRef());
 
   /// \brief The number of functions that couldn't have their profiles mapped.
   ///
@@ -484,7 +484,26 @@ template<> struct DenseMapInfo<coverage::CounterExpression> {
   }
 };
 
+const std::error_category &coveragemap_category();
+
+enum class coveragemap_error {
+  success = 0,
+  eof,
+  no_data_found,
+  unsupported_version,
+  truncated,
+  malformed
+};
+
+inline std::error_code make_error_code(coveragemap_error E) {
+  return std::error_code(static_cast<int>(E), coveragemap_category());
+}
 
 } // end namespace llvm
 
+namespace std {
+template <>
+struct is_error_code_enum<llvm::coveragemap_error> : std::true_type {};
+}
+
 #endif // LLVM_PROFILEDATA_COVERAGEMAPPING_H_
diff --git a/include/llvm/ProfileData/CoverageMappingReader.h b/include/llvm/ProfileData/CoverageMappingReader.h
index bc68f0b47a2..38fb4680476 100644
--- a/include/llvm/ProfileData/CoverageMappingReader.h
+++ b/include/llvm/ProfileData/CoverageMappingReader.h
@@ -80,12 +80,6 @@ class RawCoverageReader {
 protected:
   StringRef Data;
 
-  /// \brief Return the error code.
-  std::error_code error(std::error_code EC) { return EC; }
-
-  /// \brief Clear the current error code and return a successful one.
-  std::error_code success() { return error(instrprof_error::success); }
-
   RawCoverageReader(StringRef Data) : Data(Data) {}
 
   std::error_code readULEB128(uint64_t &Result);
@@ -177,7 +171,7 @@ class BinaryCoverageReader : public CoverageMappingReader {
 public:
   static ErrorOr<std::unique_ptr<BinaryCoverageReader>>
   create(std::unique_ptr<MemoryBuffer> &ObjectBuffer,
-         Triple::ArchType Arch = Triple::ArchType::UnknownArch);
+         StringRef Arch);
 
   std::error_code readNextRecord(CoverageMappingRecord &Record) override;
 };
diff --git a/include/llvm/ProfileData/InstrProf.h b/include/llvm/ProfileData/InstrProf.h
index eafb76886c8..77055ba8726 100644
--- a/include/llvm/ProfileData/InstrProf.h
+++ b/include/llvm/ProfileData/InstrProf.h
@@ -16,7 +16,10 @@
 #ifndef LLVM_PROFILEDATA_INSTRPROF_H_
 #define LLVM_PROFILEDATA_INSTRPROF_H_
 
+#include "llvm/ADT/StringRef.h"
+#include <cstdint>
 #include <system_error>
+#include <vector>
 
 namespace llvm {
 const std::error_category &instrprof_category();
@@ -41,6 +44,16 @@ inline std::error_code make_error_code(instrprof_error E) {
   return std::error_code(static_cast<int>(E), instrprof_category());
 }
 
+/// Profiling information for a single function.
+struct InstrProfRecord {
+  InstrProfRecord() {}
+  InstrProfRecord(StringRef Name, uint64_t Hash, std::vector<uint64_t> Counts)
+      : Name(Name), Hash(Hash), Counts(std::move(Counts)) {}
+  StringRef Name;
+  uint64_t Hash;
+  std::vector<uint64_t> Counts;
+};
+
 } // end namespace llvm
 
 namespace std {
diff --git a/include/llvm/ProfileData/InstrProfReader.h b/include/llvm/ProfileData/InstrProfReader.h
index 63a6ac671f2..f937e7d08d5 100644
--- a/include/llvm/ProfileData/InstrProfReader.h
+++ b/include/llvm/ProfileData/InstrProfReader.h
@@ -29,16 +29,6 @@ namespace llvm {
 
 class InstrProfReader;
 
-/// Profiling information for a single function.
-struct InstrProfRecord {
-  InstrProfRecord() {}
-  InstrProfRecord(StringRef Name, uint64_t Hash, ArrayRef<uint64_t> Counts)
-      : Name(Name), Hash(Hash), Counts(Counts) {}
-  StringRef Name;
-  uint64_t Hash;
-  ArrayRef<uint64_t> Counts;
-};
-
 /// A file format agnostic iterator over profiling data.
 class InstrProfIterator : public std::iterator<std::input_iterator_tag,
                                                InstrProfRecord> {
@@ -114,8 +104,6 @@ class TextInstrProfReader : public InstrProfReader {
   std::unique_ptr<MemoryBuffer> DataBuffer;
   /// Iterator over the profile data.
   line_iterator Line;
-  /// The current set of counter values.
-  std::vector<uint64_t> Counts;
 
   TextInstrProfReader(const TextInstrProfReader &) = delete;
   TextInstrProfReader &operator=(const TextInstrProfReader &) = delete;
@@ -141,8 +129,6 @@ class RawInstrProfReader : public InstrProfReader {
 private:
   /// The profile data file contents.
   std::unique_ptr<MemoryBuffer> DataBuffer;
-  /// The current set of counter values.
-  std::vector<uint64_t> Counts;
   struct ProfileData {
     const uint32_t NameSize;
     const uint32_t NumCounters;
@@ -206,17 +192,16 @@ enum class HashT : uint32_t;
 /// Trait for lookups into the on-disk hash table for the binary instrprof
 /// format.
 class InstrProfLookupTrait {
-  std::vector<uint64_t> DataBuffer;
+  std::vector<InstrProfRecord> DataBuffer;
   IndexedInstrProf::HashT HashType;
+  unsigned FormatVersion;
+
 public:
-  InstrProfLookupTrait(IndexedInstrProf::HashT HashType) : HashType(HashType) {}
+  InstrProfLookupTrait(IndexedInstrProf::HashT HashType, unsigned FormatVersion)
+      : HashType(HashType), FormatVersion(FormatVersion) {}
+
+  typedef ArrayRef<InstrProfRecord> data_type;
 
-  struct data_type {
-    data_type(StringRef Name, ArrayRef<uint64_t> Data)
-        : Name(Name), Data(Data) {}
-    StringRef Name;
-    ArrayRef<uint64_t> Data;
-  };
   typedef StringRef internal_key_type;
   typedef StringRef external_key_type;
   typedef uint64_t hash_value_type;
@@ -239,22 +224,9 @@ class InstrProfLookupTrait {
     return StringRef((const char *)D, N);
   }
 
-  data_type ReadData(StringRef K, const unsigned char *D, offset_type N) {
-    DataBuffer.clear();
-    if (N % sizeof(uint64_t))
-      // The data is corrupt, don't try to read it.
-      return data_type("", DataBuffer);
-
-    using namespace support;
-    // We just treat the data as opaque here. It's simpler to handle in
-    // IndexedInstrProfReader.
-    unsigned NumEntries = N / sizeof(uint64_t);
-    DataBuffer.reserve(NumEntries);
-    for (unsigned I = 0; I < NumEntries; ++I)
-      DataBuffer.push_back(endian::readNext<uint64_t, little, unaligned>(D));
-    return data_type(K, DataBuffer);
-  }
+  data_type ReadData(StringRef K, const unsigned char *D, offset_type N);
 };
+
 typedef OnDiskIterableChainedHashTable<InstrProfLookupTrait>
     InstrProfReaderIndex;
 
@@ -267,8 +239,6 @@ class IndexedInstrProfReader : public InstrProfReader {
   std::unique_ptr<InstrProfReaderIndex> Index;
   /// Iterator over the profile data.
   InstrProfReaderIndex::data_iterator RecordIterator;
-  /// Offset into our current data set.
-  size_t CurrentOffset;
   /// The file format version of the profile data.
   uint64_t FormatVersion;
   /// The maximal execution count among all functions.
@@ -278,7 +248,7 @@ class IndexedInstrProfReader : public InstrProfReader {
   IndexedInstrProfReader &operator=(const IndexedInstrProfReader &) = delete;
 public:
   IndexedInstrProfReader(std::unique_ptr<MemoryBuffer> DataBuffer)
-      : DataBuffer(std::move(DataBuffer)), Index(nullptr), CurrentOffset(0) {}
+      : DataBuffer(std::move(DataBuffer)), Index(nullptr) {}
 
   /// Return true if the given buffer is in an indexed instrprof format.
   static bool hasFormat(const MemoryBuffer &DataBuffer);
diff --git a/include/llvm/ProfileData/SampleProf.h b/include/llvm/ProfileData/SampleProf.h
index df0a055c2a7..1b82e55aa77 100644
--- a/include/llvm/ProfileData/SampleProf.h
+++ b/include/llvm/ProfileData/SampleProf.h
@@ -59,7 +59,7 @@ static inline uint64_t SPMagic() {
 
 static inline uint64_t SPVersion() { return 100; }
 
-/// \brief Represents the relative location of an instruction.
+/// Represents the relative location of an instruction.
 ///
 /// Instruction locations are specified by the line offset from the
 /// beginning of the function (marked by the line where the function
@@ -100,7 +100,7 @@ template <> struct DenseMapInfo<sampleprof::LineLocation> {
 
 namespace sampleprof {
 
-/// \brief Representation of a single sample record.
+/// Representation of a single sample record.
 ///
 /// A sample record is represented by a positive integer value, which
 /// indicates how frequently was the associated line location executed.
@@ -116,7 +116,7 @@ class SampleRecord {
 
   SampleRecord() : NumSamples(0), CallTargets() {}
 
-  /// \brief Increment the number of samples for this record by \p S.
+  /// Increment the number of samples for this record by \p S.
   ///
   /// Sample counts accumulate using saturating arithmetic, to avoid wrapping
   /// around unsigned integers.
@@ -127,7 +127,7 @@ class SampleRecord {
       NumSamples = std::numeric_limits<unsigned>::max();
   }
 
-  /// \brief Add called function \p F with samples \p S.
+  /// Add called function \p F with samples \p S.
   ///
   /// Sample counts accumulate using saturating arithmetic, to avoid wrapping
   /// around unsigned integers.
@@ -139,13 +139,13 @@ class SampleRecord {
       TargetSamples = std::numeric_limits<unsigned>::max();
   }
 
-  /// \brief Return true if this sample record contains function calls.
+  /// Return true if this sample record contains function calls.
   bool hasCalls() const { return CallTargets.size() > 0; }
 
   unsigned getSamples() const { return NumSamples; }
   const CallTargetMap &getCallTargets() const { return CallTargets; }
 
-  /// \brief Merge the samples in \p Other into this record.
+  /// Merge the samples in \p Other into this record.
   void merge(const SampleRecord &Other) {
     addSamples(Other.getSamples());
     for (const auto &I : Other.getCallTargets())
@@ -159,7 +159,7 @@ class SampleRecord {
 
 typedef DenseMap<LineLocation, SampleRecord> BodySampleMap;
 
-/// \brief Representation of the samples collected for a function.
+/// Representation of the samples collected for a function.
 ///
 /// This data structure contains all the collected samples for the body
 /// of a function. Each sample corresponds to a LineLocation instance
@@ -187,13 +187,13 @@ class FunctionSamples {
                                                                          Num);
   }
 
-  /// \brief Return the sample record at the given location.
+  /// Return the sample record at the given location.
   /// Each location is specified by \p LineOffset and \p Discriminator.
   SampleRecord &sampleRecordAt(const LineLocation &Loc) {
     return BodySamples[Loc];
   }
 
-  /// \brief Return the number of samples collected at the given location.
+  /// Return the number of samples collected at the given location.
   /// Each location is specified by \p LineOffset and \p Discriminator.
   unsigned samplesAt(int LineOffset, unsigned Discriminator) {
     return sampleRecordAt(LineLocation(LineOffset, Discriminator)).getSamples();
@@ -201,17 +201,17 @@ class FunctionSamples {
 
   bool empty() const { return BodySamples.empty(); }
 
-  /// \brief Return the total number of samples collected inside the function.
+  /// Return the total number of samples collected inside the function.
   unsigned getTotalSamples() const { return TotalSamples; }
 
-  /// \brief Return the total number of samples collected at the head of the
+  /// Return the total number of samples collected at the head of the
   /// function.
   unsigned getHeadSamples() const { return TotalHeadSamples; }
 
-  /// \brief Return all the samples collected in the body of the function.
+  /// Return all the samples collected in the body of the function.
   const BodySampleMap &getBodySamples() const { return BodySamples; }
 
-  /// \brief Merge the samples in \p Other into this one.
+  /// Merge the samples in \p Other into this one.
   void merge(const FunctionSamples &Other) {
     addTotalSamples(Other.getTotalSamples());
     addHeadSamples(Other.getHeadSamples());
@@ -223,16 +223,18 @@ class FunctionSamples {
   }
 
 private:
-  /// \brief Total number of samples collected inside this function.
+  /// Total number of samples collected inside this function.
   ///
   /// Samples are cumulative, they include all the samples collected
   /// inside this function and all its inlined callees.
   unsigned TotalSamples;
 
-  /// \brief Total number of samples collected at the head of the function.
+  /// Total number of samples collected at the head of the function.
+  /// This is an approximation of the number of calls made to this function
+  /// at runtime.
   unsigned TotalHeadSamples;
 
-  /// \brief Map instruction locations to collected samples.
+  /// Map instruction locations to collected samples.
   ///
   /// Each entry in this map contains the number of samples
   /// collected at the corresponding line offset. All line locations
diff --git a/include/llvm/Support/AlignOf.h b/include/llvm/Support/AlignOf.h
index 061d5acf232..07da02d063c 100644
--- a/include/llvm/Support/AlignOf.h
+++ b/include/llvm/Support/AlignOf.h
@@ -22,6 +22,14 @@ namespace llvm {
 template <typename T>
 struct AlignmentCalcImpl {
   char x;
+#if defined(_MSC_VER)
+// Disables "structure was padded due to __declspec(align())" warnings that are
+// generated by any class using AlignOf<T> with a manually specified alignment.
+// Although the warning is disabled in the LLVM project we need this pragma
+// as AlignOf.h is a published support header that's available for use
+// out-of-tree, and we would like that to compile cleanly at /W4.
+#pragma warning(suppress : 4324)
+#endif
   T t;
 private:
   AlignmentCalcImpl() {} // Never instantiate.
@@ -36,9 +44,18 @@ struct AlignmentCalcImpl {
 ///  compile-time constant (e.g., for template instantiation).
 template <typename T>
 struct AlignOf {
+#ifndef _MSC_VER
+  // Avoid warnings from GCC like:
+  //   comparison between 'enum llvm::AlignOf<X>::<anonymous>' and 'enum
+  //   llvm::AlignOf<Y>::<anonymous>' [-Wenum-compare]
+  // by using constexpr instead of enum.
+  // (except on MSVC, since it doesn't support constexpr yet).
+  static constexpr unsigned Alignment =
+      static_cast<unsigned int>(sizeof(AlignmentCalcImpl<T>) - sizeof(T));
+#else
   enum { Alignment =
          static_cast<unsigned int>(sizeof(AlignmentCalcImpl<T>) - sizeof(T)) };
-
+#endif
   enum { Alignment_GreaterEqual_2Bytes = Alignment >= 2 ? 1 : 0 };
   enum { Alignment_GreaterEqual_4Bytes = Alignment >= 4 ? 1 : 0 };
   enum { Alignment_GreaterEqual_8Bytes = Alignment >= 8 ? 1 : 0 };
@@ -50,6 +67,10 @@ struct AlignOf {
   enum { Alignment_LessEqual_16Bytes = Alignment <= 16 ? 1 : 0 };
 };
 
+#ifndef _MSC_VER
+template <typename T> constexpr unsigned AlignOf<T>::Alignment;
+#endif
+
 /// alignOf - A templated function that returns the minimum alignment of
 ///  of a type.  This provides no extra functionality beyond the AlignOf
 ///  class besides some cosmetic cleanliness.  Example usage:
@@ -190,12 +211,12 @@ union SizerImpl {
 } // end namespace detail
 
 /// \brief This union template exposes a suitably aligned and sized character
-/// array member which can hold elements of any of up to four types.
+/// array member which can hold elements of any of up to ten types.
 ///
 /// These types may be arrays, structs, or any other types. The goal is to
 /// expose a char array buffer member which can be used as suitable storage for
-/// a placement new of any of these types. Support for more than seven types can
-/// be added at the cost of more boiler plate.
+/// a placement new of any of these types. Support for more than ten types can
+/// be added at the cost of more boilerplate.
 template <typename T1,
           typename T2 = char, typename T3 = char, typename T4 = char,
           typename T5 = char, typename T6 = char, typename T7 = char,
diff --git a/include/llvm/Support/Allocator.h b/include/llvm/Support/Allocator.h
index 94cf3e85f36..f9b5cf22f97 100644
--- a/include/llvm/Support/Allocator.h
+++ b/include/llvm/Support/Allocator.h
@@ -148,7 +148,7 @@ class BumpPtrAllocatorImpl
       : CurPtr(nullptr), End(nullptr), BytesAllocated(0),
         Allocator(std::forward<T &&>(Allocator)) {}
 
-  // Manually implement a move constructor as we must clear the old allocators
+  // Manually implement a move constructor as we must clear the old allocator's
   // slabs as a matter of correctness.
   BumpPtrAllocatorImpl(BumpPtrAllocatorImpl &&Old)
       : CurPtr(Old.CurPtr), End(Old.End), Slabs(std::move(Old.Slabs)),
@@ -187,6 +187,9 @@ class BumpPtrAllocatorImpl
   /// \brief Deallocate all but the current slab and reset the current pointer
   /// to the beginning of it, freeing all memory allocated so far.
   void Reset() {
+    DeallocateCustomSizedSlabs();
+    CustomSizedSlabs.clear();
+
     if (Slabs.empty())
       return;
 
@@ -195,11 +198,9 @@ class BumpPtrAllocatorImpl
     CurPtr = (char *)Slabs.front();
     End = CurPtr + SlabSize;
 
-    // Deallocate all but the first slab, and all custome sized slabs.
+    // Deallocate all but the first slab, and deallocate all custom-sized slabs.
     DeallocateSlabs(std::next(Slabs.begin()), Slabs.end());
     Slabs.erase(std::next(Slabs.begin()), Slabs.end());
-    DeallocateCustomSizedSlabs();
-    CustomSizedSlabs.clear();
   }
 
   /// \brief Allocate space at the specified alignment.
@@ -320,14 +321,6 @@ class BumpPtrAllocatorImpl
     for (; I != E; ++I) {
       size_t AllocatedSlabSize =
           computeSlabSize(std::distance(Slabs.begin(), I));
-#ifndef NDEBUG
-      // Poison the memory so stale pointers crash sooner.  Note we must
-      // preserve the Size and NextPtr fields at the beginning.
-      if (AllocatedSlabSize != 0) {
-        sys::Memory::setRangeWritable(*I, AllocatedSlabSize);
-        memset(*I, 0xCD, AllocatedSlabSize);
-      }
-#endif
       Allocator.Deallocate(*I, AllocatedSlabSize);
     }
   }
@@ -337,12 +330,6 @@ class BumpPtrAllocatorImpl
     for (auto &PtrAndSize : CustomSizedSlabs) {
       void *Ptr = PtrAndSize.first;
       size_t Size = PtrAndSize.second;
-#ifndef NDEBUG
-      // Poison the memory so stale pointers crash sooner.  Note we must
-      // preserve the Size and NextPtr fields at the beginning.
-      sys::Memory::setRangeWritable(Ptr, Size);
-      memset(Ptr, 0xCD, Size);
-#endif
       Allocator.Deallocate(Ptr, Size);
     }
   }
diff --git a/include/llvm/Support/BranchProbability.h b/include/llvm/Support/BranchProbability.h
index 9aab6ace6e3..a6429dd22a3 100644
--- a/include/llvm/Support/BranchProbability.h
+++ b/include/llvm/Support/BranchProbability.h
@@ -31,7 +31,7 @@ class BranchProbability {
 
 public:
   BranchProbability(uint32_t n, uint32_t d) : N(n), D(d) {
-    assert(d > 0 && "Denomiator cannot be 0!");
+    assert(d > 0 && "Denominator cannot be 0!");
     assert(n <= d && "Probability cannot be bigger than 1!");
   }
 
diff --git a/include/llvm/Support/COFF.h b/include/llvm/Support/COFF.h
index 7f54822d503..0162175efe3 100644
--- a/include/llvm/Support/COFF.h
+++ b/include/llvm/Support/COFF.h
@@ -88,6 +88,7 @@ namespace COFF {
     IMAGE_FILE_MACHINE_AMD64     = 0x8664,
     IMAGE_FILE_MACHINE_ARM       = 0x1C0,
     IMAGE_FILE_MACHINE_ARMNT     = 0x1C4,
+    IMAGE_FILE_MACHINE_ARM64     = 0xAA64,
     IMAGE_FILE_MACHINE_EBC       = 0xEBC,
     IMAGE_FILE_MACHINE_I386      = 0x14C,
     IMAGE_FILE_MACHINE_IA64      = 0x200,
@@ -155,16 +156,6 @@ namespace COFF {
     uint8_t  NumberOfAuxSymbols;
   };
 
-  enum SymbolFlags {
-    SF_TypeMask = 0x0000FFFF,
-    SF_TypeShift = 0,
-
-    SF_ClassMask = 0x00FF0000,
-    SF_ClassShift = 16,
-
-    SF_WeakExternal = 0x01000000
-  };
-
   enum SymbolSectionNumber : int32_t {
     IMAGE_SYM_DEBUG     = -2,
     IMAGE_SYM_ABSOLUTE  = -1,
@@ -257,6 +248,7 @@ namespace COFF {
   enum SectionCharacteristics : uint32_t {
     SC_Invalid = 0xffffffff,
 
+    IMAGE_SCN_TYPE_NOLOAD            = 0x00000002,
     IMAGE_SCN_TYPE_NO_PAD            = 0x00000008,
     IMAGE_SCN_CNT_CODE               = 0x00000020,
     IMAGE_SCN_CNT_INITIALIZED_DATA   = 0x00000040,
@@ -665,6 +657,7 @@ namespace COFF {
   };
 
   enum CodeViewIdentifiers {
+    DEBUG_LINE_TABLES_HAVE_COLUMN_RECORDS = 0x1,
     DEBUG_SECTION_MAGIC = 0x4,
     DEBUG_SYMBOL_SUBSECTION = 0xF1,
     DEBUG_LINE_TABLE_SUBSECTION = 0xF2,
diff --git a/include/llvm/Support/COM.h b/include/llvm/Support/COM.h
new file mode 100644
index 00000000000..a2d5a7a68ba
--- /dev/null
+++ b/include/llvm/Support/COM.h
@@ -0,0 +1,36 @@
+//===- llvm/Support/COM.h ---------------------------------------*- C++ -*-===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+/// \file
+///
+/// Provides a library for accessing COM functionality of the Host OS.
+///
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_SUPPORT_COM_H
+#define LLVM_SUPPORT_COM_H
+
+namespace llvm {
+namespace sys {
+
+enum class COMThreadingMode { SingleThreaded, MultiThreaded };
+
+class InitializeCOMRAII {
+public:
+  explicit InitializeCOMRAII(COMThreadingMode Threading,
+                             bool SpeedOverMemory = false);
+  ~InitializeCOMRAII();
+
+private:
+  InitializeCOMRAII(const InitializeCOMRAII &) = delete;
+  void operator=(const InitializeCOMRAII &) = delete;
+};
+}
+}
+
+#endif
diff --git a/include/llvm/Support/CommandLine.h b/include/llvm/Support/CommandLine.h
index bd1d1cb6dc9..379d06a6574 100644
--- a/include/llvm/Support/CommandLine.h
+++ b/include/llvm/Support/CommandLine.h
@@ -33,6 +33,9 @@
 
 namespace llvm {
 
+class BumpPtrStringSaver;
+class StringSaver;
+
 /// cl Namespace - This namespace contains all of the command line option
 /// processing machinery.  It is intentionally a short name to make qualified
 /// usage concise.
@@ -787,7 +790,7 @@ template <> class parser<bool> final : public basic_parser<bool> {
   void anchor() override;
 };
 
-EXTERN_TEMPLATE_INSTANTIATION(class basic_parser<bool>);
+extern template class basic_parser<bool>;
 
 //--------------------------------------------------
 // parser<boolOrDefault>
@@ -813,7 +816,7 @@ class parser<boolOrDefault> final : public basic_parser<boolOrDefault> {
   void anchor() override;
 };
 
-EXTERN_TEMPLATE_INSTANTIATION(class basic_parser<boolOrDefault>);
+extern template class basic_parser<boolOrDefault>;
 
 //--------------------------------------------------
 // parser<int>
@@ -835,7 +838,7 @@ template <> class parser<int> final : public basic_parser<int> {
   void anchor() override;
 };
 
-EXTERN_TEMPLATE_INSTANTIATION(class basic_parser<int>);
+extern template class basic_parser<int>;
 
 //--------------------------------------------------
 // parser<unsigned>
@@ -857,7 +860,7 @@ template <> class parser<unsigned> final : public basic_parser<unsigned> {
   void anchor() override;
 };
 
-EXTERN_TEMPLATE_INSTANTIATION(class basic_parser<unsigned>);
+extern template class basic_parser<unsigned>;
 
 //--------------------------------------------------
 // parser<unsigned long long>
@@ -882,7 +885,7 @@ class parser<unsigned long long> final
   void anchor() override;
 };
 
-EXTERN_TEMPLATE_INSTANTIATION(class basic_parser<unsigned long long>);
+extern template class basic_parser<unsigned long long>;
 
 //--------------------------------------------------
 // parser<double>
@@ -904,7 +907,7 @@ template <> class parser<double> final : public basic_parser<double> {
   void anchor() override;
 };
 
-EXTERN_TEMPLATE_INSTANTIATION(class basic_parser<double>);
+extern template class basic_parser<double>;
 
 //--------------------------------------------------
 // parser<float>
@@ -926,7 +929,7 @@ template <> class parser<float> final : public basic_parser<float> {
   void anchor() override;
 };
 
-EXTERN_TEMPLATE_INSTANTIATION(class basic_parser<float>);
+extern template class basic_parser<float>;
 
 //--------------------------------------------------
 // parser<std::string>
@@ -951,7 +954,7 @@ template <> class parser<std::string> final : public basic_parser<std::string> {
   void anchor() override;
 };
 
-EXTERN_TEMPLATE_INSTANTIATION(class basic_parser<std::string>);
+extern template class basic_parser<std::string>;
 
 //--------------------------------------------------
 // parser<char>
@@ -976,7 +979,7 @@ template <> class parser<char> final : public basic_parser<char> {
   void anchor() override;
 };
 
-EXTERN_TEMPLATE_INSTANTIATION(class basic_parser<char>);
+extern template class basic_parser<char>;
 
 //--------------------------------------------------
 // PrintOptionDiff
@@ -1251,11 +1254,11 @@ class opt : public Option,
   }
 };
 
-EXTERN_TEMPLATE_INSTANTIATION(class opt<unsigned>);
-EXTERN_TEMPLATE_INSTANTIATION(class opt<int>);
-EXTERN_TEMPLATE_INSTANTIATION(class opt<std::string>);
-EXTERN_TEMPLATE_INSTANTIATION(class opt<char>);
-EXTERN_TEMPLATE_INSTANTIATION(class opt<bool>);
+extern template class opt<unsigned>;
+extern template class opt<int>;
+extern template class opt<std::string>;
+extern template class opt<char>;
+extern template class opt<bool>;
 
 //===----------------------------------------------------------------------===//
 // list_storage class
@@ -1284,24 +1287,81 @@ template <class DataType, class StorageClass> class list_storage {
   }
 };
 
-// Define how to hold a class type object, such as a string.  Since we can
-// inherit from a class, we do so.  This makes us exactly compatible with the
-// object in all cases that it is used.
+// Define how to hold a class type object, such as a string.
+// Originally this code inherited from std::vector. In transitioning to a new
+// API for command line options we should change this. The new implementation
+// of this list_storage specialization implements the minimum subset of the
+// std::vector API required for all the current clients.
 //
-template <class DataType>
-class list_storage<DataType, bool> : public std::vector<DataType> {
+// FIXME: Reduce this API to a more narrow subset of std::vector
+//
+template <class DataType> class list_storage<DataType, bool> {
+  std::vector<DataType> Storage;
+
 public:
-  template <class T> void addValue(const T &V) {
-    std::vector<DataType>::push_back(V);
+  typedef typename std::vector<DataType>::iterator iterator;
+
+  iterator begin() { return Storage.begin(); }
+  iterator end() { return Storage.end(); }
+
+  typedef typename std::vector<DataType>::const_iterator const_iterator;
+  const_iterator begin() const { return Storage.begin(); }
+  const_iterator end() const { return Storage.end(); }
+
+  typedef typename std::vector<DataType>::size_type size_type;
+  size_type size() const { return Storage.size(); }
+
+  bool empty() const { return Storage.empty(); }
+
+  void push_back(const DataType &value) { Storage.push_back(value); }
+  void push_back(DataType &&value) { Storage.push_back(value); }
+
+  typedef typename std::vector<DataType>::reference reference;
+  typedef typename std::vector<DataType>::const_reference const_reference;
+  reference operator[](size_type pos) { return Storage[pos]; }
+  const_reference operator[](size_type pos) const { return Storage[pos]; }
+
+  iterator erase(const_iterator pos) { return Storage.erase(pos); }
+  iterator erase(const_iterator first, const_iterator last) {
+    return Storage.erase(first, last);
+  }
+
+  iterator erase(iterator pos) { return Storage.erase(pos); }
+  iterator erase(iterator first, iterator last) {
+    return Storage.erase(first, last);
+  }
+
+  iterator insert(const_iterator pos, const DataType &value) {
+    return Storage.insert(pos, value);
+  }
+  iterator insert(const_iterator pos, DataType &&value) {
+    return Storage.insert(pos, value);
   }
+
+  iterator insert(iterator pos, const DataType &value) {
+    return Storage.insert(pos, value);
+  }
+  iterator insert(iterator pos, DataType &&value) {
+    return Storage.insert(pos, value);
+  }
+
+  reference front() { return Storage.front(); }
+  const_reference front() const { return Storage.front(); }
+
+  operator std::vector<DataType>&() { return Storage; }
+  operator ArrayRef<DataType>() { return Storage; }
+  std::vector<DataType> *operator&() { return &Storage; }
+  const std::vector<DataType> *operator&() const { return &Storage; }
+
+  template <class T> void addValue(const T &V) { Storage.push_back(V); }
 };
 
 //===----------------------------------------------------------------------===//
 // list - A list of command line options.
 //
-template <class DataType, class Storage = bool,
+template <class DataType, class StorageClass = bool,
           class ParserClass = parser<DataType>>
-class list : public Option, public list_storage<DataType, Storage> {
+class list : public Option, public list_storage<DataType, StorageClass> {
   std::vector<unsigned> Positions;
   ParserClass Parser;
 
@@ -1319,7 +1379,7 @@ class list : public Option, public list_storage<DataType, Storage> {
         typename ParserClass::parser_data_type();
     if (Parser.parse(*this, ArgName, Arg, Val))
       return true; // Parse Error!
-    list_storage<DataType, Storage>::addValue(Val);
+    list_storage<DataType, StorageClass>::addValue(Val);
     setPosition(pos);
     Positions.push_back(pos);
     return false;
@@ -1619,16 +1679,6 @@ StringMap<Option *> &getRegisteredOptions();
 // Standalone command line processing utilities.
 //
 
-/// \brief Saves strings in the inheritor's stable storage and returns a stable
-/// raw character pointer.
-class StringSaver {
-  virtual void anchor();
-
-public:
-  virtual const char *SaveString(const char *Str) = 0;
-  virtual ~StringSaver(){}; // Pacify -Wnon-virtual-dtor.
-};
-
 /// \brief Tokenizes a command line that can contain escapes and quotes.
 //
 /// The quoting rules match those used by GCC and other tools that use
diff --git a/include/llvm/Support/Compiler.h b/include/llvm/Support/Compiler.h
index c81fbaff9db..141639839cc 100644
--- a/include/llvm/Support/Compiler.h
+++ b/include/llvm/Support/Compiler.h
@@ -174,19 +174,6 @@
 #define LLVM_UNLIKELY(EXPR) (EXPR)
 #endif
 
-// C++ doesn't support 'extern template' of template specializations.  GCC does,
-// but requires __extension__ before it.  In the header, use this:
-//   EXTERN_TEMPLATE_INSTANTIATION(class foo<bar>);
-// in the .cpp file, use this:
-//   TEMPLATE_INSTANTIATION(class foo<bar>);
-#ifdef __GNUC__
-#define EXTERN_TEMPLATE_INSTANTIATION(X) __extension__ extern template X
-#define TEMPLATE_INSTANTIATION(X) template X
-#else
-#define EXTERN_TEMPLATE_INSTANTIATION(X)
-#define TEMPLATE_INSTANTIATION(X)
-#endif
-
 /// LLVM_ATTRIBUTE_NOINLINE - On compilers where we have a directive to do so,
 /// mark a method "not for inlining".
 #if __has_attribute(noinline) || LLVM_GNUC_PREREQ(3, 4, 0)
@@ -350,19 +337,6 @@
 # define LLVM_ADDRESS_SANITIZER_BUILD 0
 #endif
 
-/// \macro LLVM_IS_UNALIGNED_ACCESS_FAST
-/// \brief Is unaligned memory access fast on the host machine.
-///
-/// Don't specialize on alignment for platforms where unaligned memory accesses
-/// generates the same code as aligned memory accesses for common types.
-#if defined(_M_AMD64) || defined(_M_IX86) || defined(__amd64) || \
-    defined(__amd64__) || defined(__x86_64) || defined(__x86_64__) || \
-    defined(_X86_) || defined(__i386) || defined(__i386__)
-# define LLVM_IS_UNALIGNED_ACCESS_FAST 1
-#else
-# define LLVM_IS_UNALIGNED_ACCESS_FAST 0
-#endif
-
 /// \brief Mark debug helper function definitions like dump() that should not be
 /// stripped from debug builds.
 // FIXME: Move this to a private config.h as it's not usable in public headers.
diff --git a/include/llvm/Support/DataStream.h b/include/llvm/Support/DataStream.h
index 8bc41336030..a544316f430 100644
--- a/include/llvm/Support/DataStream.h
+++ b/include/llvm/Support/DataStream.h
@@ -17,6 +17,7 @@
 #ifndef LLVM_SUPPORT_DATASTREAM_H
 #define LLVM_SUPPORT_DATASTREAM_H
 
+#include <memory>
 #include <string>
 
 namespace llvm {
@@ -30,9 +31,8 @@ class DataStreamer {
   virtual ~DataStreamer();
 };
 
-DataStreamer *getDataFileStreamer(const std::string &Filename,
-                                  std::string *Err);
-
+std::unique_ptr<DataStreamer> getDataFileStreamer(const std::string &Filename,
+                                                  std::string *Err);
 }
 
 #endif  // LLVM_SUPPORT_DATASTREAM_H_
diff --git a/include/llvm/Support/Dwarf.h b/include/llvm/Support/Dwarf.h
index c294a72a04b..17e9c1540a4 100644
--- a/include/llvm/Support/Dwarf.h
+++ b/include/llvm/Support/Dwarf.h
@@ -239,6 +239,11 @@ enum Attribute : uint16_t {
   DW_AT_GNU_pubnames = 0x2134,
   DW_AT_GNU_pubtypes = 0x2135,
 
+  // LLVM project extensions.
+  DW_AT_LLVM_include_path = 0x3e00,
+  DW_AT_LLVM_config_macros = 0x3e01,
+  DW_AT_LLVM_isysroot = 0x3e02,
+
   // Apple extensions.
   DW_AT_APPLE_optimized = 0x3fe1,
   DW_AT_APPLE_flags = 0x3fe2,
@@ -285,7 +290,11 @@ enum Form : uint16_t {
 
   // Extensions for Fission proposal
   DW_FORM_GNU_addr_index = 0x1f01,
-  DW_FORM_GNU_str_index = 0x1f02
+  DW_FORM_GNU_str_index = 0x1f02,
+
+  // Alternate debug sections proposal (output of "dwz" tool).
+  DW_FORM_GNU_ref_alt = 0x1f20,
+  DW_FORM_GNU_strp_alt = 0x1f21
 };
 
 enum LocationAtom {
diff --git a/include/llvm/Support/ELF.h b/include/llvm/Support/ELF.h
index 2bfad4f5ab5..94a4bfb2202 100644
--- a/include/llvm/Support/ELF.h
+++ b/include/llvm/Support/ELF.h
@@ -133,7 +133,7 @@ enum {
   EM_386           = 3, // Intel 386
   EM_68K           = 4, // Motorola 68000
   EM_88K           = 5, // Motorola 88000
-  EM_486           = 6, // Intel 486 (deprecated)
+  EM_IAMCU         = 6, // Intel MCU
   EM_860           = 7, // Intel 80860
   EM_MIPS          = 8, // MIPS R3000
   EM_S370          = 9, // IBM System/370
@@ -308,7 +308,8 @@ enum {
   EM_COGE          = 216, // Cognitive Smart Memory Processor
   EM_COOL          = 217, // iCelero CoolEngine
   EM_NORC          = 218, // Nanoradio Optimized RISC
-  EM_CSR_KALIMBA   = 219  // CSR Kalimba architecture family
+  EM_CSR_KALIMBA   = 219, // CSR Kalimba architecture family
+  EM_AMDGPU        = 224  // AMD GPU architecture
 };
 
 // Object file classes.
@@ -346,6 +347,7 @@ enum {
   ELFOSABI_FENIXOS = 16,      // FenixOS
   ELFOSABI_CLOUDABI = 17,     // Nuxi CloudABI
   ELFOSABI_C6000_ELFABI = 64, // Bare-metal TMS320C6000
+  ELFOSABI_AMDGPU_HSA = 64,   // AMD HSA runtime
   ELFOSABI_C6000_LINUX = 65,  // Linux TMS320C6000
   ELFOSABI_ARM = 97,          // ARM
   ELFOSABI_STANDALONE = 255   // Standalone (embedded) application
@@ -432,22 +434,50 @@ enum : unsigned {
   EF_MIPS_NOREORDER = 0x00000001, // Don't reorder instructions
   EF_MIPS_PIC       = 0x00000002, // Position independent code
   EF_MIPS_CPIC      = 0x00000004, // Call object with Position independent code
-  EF_MIPS_ABI2      = 0x00000020,
-  EF_MIPS_32BITMODE = 0x00000100,
+  EF_MIPS_ABI2      = 0x00000020, // File uses N32 ABI
+  EF_MIPS_32BITMODE = 0x00000100, // Code compiled for a 64-bit machine
+                                  // in 32-bit mode
+  EF_MIPS_FP64      = 0x00000200, // Code compiled for a 32-bit machine
+                                  // but uses 64-bit FP registers
   EF_MIPS_NAN2008   = 0x00000400, // Uses IEE 754-2008 NaN encoding
 
   // ABI flags
-  EF_MIPS_ABI_O32   = 0x00001000, // This file follows the first MIPS 32 bit ABI
+  EF_MIPS_ABI_O32    = 0x00001000, // This file follows the first MIPS 32 bit ABI
   EF_MIPS_ABI_O64    = 0x00002000, // O32 ABI extended for 64-bit architecture.
   EF_MIPS_ABI_EABI32 = 0x00003000, // EABI in 32 bit mode.
   EF_MIPS_ABI_EABI64 = 0x00004000, // EABI in 64 bit mode.
   EF_MIPS_ABI        = 0x0000f000, // Mask for selecting EF_MIPS_ABI_ variant.
 
-  //ARCH_ASE
+  // MIPS machine variant
+  EF_MIPS_MACH_3900    = 0x00810000, // Toshiba R3900
+  EF_MIPS_MACH_4010    = 0x00820000, // LSI R4010
+  EF_MIPS_MACH_4100    = 0x00830000, // NEC VR4100
+  EF_MIPS_MACH_4650    = 0x00850000, // MIPS R4650
+  EF_MIPS_MACH_4120    = 0x00870000, // NEC VR4120
+  EF_MIPS_MACH_4111    = 0x00880000, // NEC VR4111/VR4181
+  EF_MIPS_MACH_SB1     = 0x008a0000, // Broadcom SB-1
+  EF_MIPS_MACH_OCTEON  = 0x008b0000, // Cavium Networks Octeon
+  EF_MIPS_MACH_XLR     = 0x008c0000, // RMI Xlr
+  EF_MIPS_MACH_OCTEON2 = 0x008d0000, // Cavium Networks Octeon2
+  EF_MIPS_MACH_OCTEON3 = 0x008e0000, // Cavium Networks Octeon3
+  EF_MIPS_MACH_5400    = 0x00910000, // NEC VR5400
+  EF_MIPS_MACH_5900    = 0x00920000, // MIPS R5900
+  EF_MIPS_MACH_5500    = 0x00980000, // NEC VR5500
+  EF_MIPS_MACH_9000    = 0x00990000, // Unknown
+  EF_MIPS_MACH_LS2E    = 0x00a00000, // ST Microelectronics Loongson 2E
+  EF_MIPS_MACH_LS2F    = 0x00a10000, // ST Microelectronics Loongson 2F
+  EF_MIPS_MACH_LS3A    = 0x00a20000, // Loongson 3A
+  EF_MIPS_MACH         = 0x00ff0000, // EF_MIPS_MACH_xxx selection mask
+
+  // ARCH_ASE
   EF_MIPS_MICROMIPS = 0x02000000, // microMIPS
   EF_MIPS_ARCH_ASE_M16 =
                       0x04000000, // Has Mips-16 ISA extensions
-  //ARCH
+  EF_MIPS_ARCH_ASE_MDMX =
+                      0x08000000, // Has MDMX multimedia extensions
+  EF_MIPS_ARCH_ASE  = 0x0f000000, // Mask for EF_MIPS_ARCH_ASE_xxx flags
+
+  // ARCH
   EF_MIPS_ARCH_1    = 0x00000000, // MIPS1 instruction set
   EF_MIPS_ARCH_2    = 0x10000000, // MIPS2 instruction set
   EF_MIPS_ARCH_3    = 0x20000000, // MIPS3 instruction set
@@ -794,9 +824,9 @@ enum {
   STT_FILE    = 4,   // Local, absolute symbol that refers to a file
   STT_COMMON  = 5,   // An uninitialized common block
   STT_TLS     = 6,   // Thread local data object
-  STT_LOOS    = 7,   // Lowest operating system-specific symbol type
-  STT_HIOS    = 8,   // Highest operating system-specific symbol type
   STT_GNU_IFUNC = 10, // GNU indirect function
+  STT_LOOS    = 10,  // Lowest operating system-specific symbol type
+  STT_HIOS    = 12,  // Highest operating system-specific symbol type
   STT_LOPROC  = 13,  // Lowest processor-specific symbol type
   STT_HIPROC  = 15   // Highest processor-specific symbol type
 };
@@ -961,13 +991,6 @@ enum : unsigned {
   PF_MASKPROC = 0xf0000000 // Bits for processor-specific semantics.
 };
 
-// @LOCALMOD-BEGIN
-// Note segment descriptor types (for object files).
-enum {
-  NT_VERSION  = 1          // Note contains a version string.
-};
-// @LOCALMOD-END
-
 // Dynamic table entry for ELF32.
 struct Elf32_Dyn
 {
@@ -1147,7 +1170,16 @@ enum {
   DF_1_CONFALT    = 0x00002000, // Configuration alternative created.
   DF_1_ENDFILTEE  = 0x00004000, // Filtee terminates filters search.
   DF_1_DISPRELDNE = 0x00008000, // Disp reloc applied at build time.
-  DF_1_DISPRELPND = 0x00010000  // Disp reloc applied at run-time.
+  DF_1_DISPRELPND = 0x00010000, // Disp reloc applied at run-time.
+  DF_1_NODIRECT   = 0x00020000, // Object has no-direct binding.
+  DF_1_IGNMULDEF  = 0x00040000,
+  DF_1_NOKSYMS    = 0x00080000,
+  DF_1_NOHDR      = 0x00100000,
+  DF_1_EDITED     = 0x00200000, // Object is modified after built.
+  DF_1_NORELOC    = 0x00400000,
+  DF_1_SYMINTPOSE = 0x00800000, // Object has individual interposers.
+  DF_1_GLOBAUDIT  = 0x01000000, // Global auditing required.
+  DF_1_SINGLETON  = 0x02000000  // Singleton symbols are used.
 };
 
 // DT_MIPS_FLAGS values.
diff --git a/include/llvm/Support/ELFRelocs/Hexagon.def b/include/llvm/Support/ELFRelocs/Hexagon.def
index c9d35b8bd75..a698ecb89e1 100644
--- a/include/llvm/Support/ELFRelocs/Hexagon.def
+++ b/include/llvm/Support/ELFRelocs/Hexagon.def
@@ -90,3 +90,11 @@ ELF_RELOC(R_HEX_IE_GOT_11_X,         82)
 ELF_RELOC(R_HEX_TPREL_32_6_X,        83)
 ELF_RELOC(R_HEX_TPREL_16_X,          84)
 ELF_RELOC(R_HEX_TPREL_11_X,          85)
+ELF_RELOC(R_HEX_LD_PLT_B22_PCREL,    86)
+ELF_RELOC(R_HEX_LD_GOT_LO16,         87)
+ELF_RELOC(R_HEX_LD_GOT_HI16,         88)
+ELF_RELOC(R_HEX_LD_GOT_32,           89)
+ELF_RELOC(R_HEX_LD_GOT_16,           90)
+ELF_RELOC(R_HEX_LD_GOT_32_6_X,       91)
+ELF_RELOC(R_HEX_LD_GOT_16_X,         92)
+ELF_RELOC(R_HEX_LD_GOT_11_X,         93)
diff --git a/include/llvm/Support/ELFRelocs/Mips.def b/include/llvm/Support/ELFRelocs/Mips.def
index 0e6de3ba29c..77e7f8e8712 100644
--- a/include/llvm/Support/ELFRelocs/Mips.def
+++ b/include/llvm/Support/ELFRelocs/Mips.def
@@ -108,6 +108,10 @@ ELF_RELOC(R_MICROMIPS_TLS_TPREL_HI16,   169)
 ELF_RELOC(R_MICROMIPS_TLS_TPREL_LO16,   170)
 ELF_RELOC(R_MICROMIPS_GPREL7_S2,        172)
 ELF_RELOC(R_MICROMIPS_PC23_S2,          173)
+ELF_RELOC(R_MICROMIPS_PC21_S2,          174)
+ELF_RELOC(R_MICROMIPS_PC26_S2,          175)
+ELF_RELOC(R_MICROMIPS_PC18_S3,          176)
+ELF_RELOC(R_MICROMIPS_PC19_S2,          177)
 ELF_RELOC(R_MIPS_NUM,                218)
 ELF_RELOC(R_MIPS_PC32,               248)
 ELF_RELOC(R_MIPS_EH,                 249)
diff --git a/include/llvm/Support/ELFRelocs/Sparc.def b/include/llvm/Support/ELFRelocs/Sparc.def
index d6772ea675d..7e01a4a8a0a 100644
--- a/include/llvm/Support/ELFRelocs/Sparc.def
+++ b/include/llvm/Support/ELFRelocs/Sparc.def
@@ -83,7 +83,7 @@ ELF_RELOC(R_SPARC_TLS_DTPOFF64,   77)
 ELF_RELOC(R_SPARC_TLS_TPOFF32,    78)
 ELF_RELOC(R_SPARC_TLS_TPOFF64,    79)
 ELF_RELOC(R_SPARC_GOTDATA_HIX22,  80)
-ELF_RELOC(R_SPARC_GOTDATA_LOX22,  81)
+ELF_RELOC(R_SPARC_GOTDATA_LOX10,  81)
 ELF_RELOC(R_SPARC_GOTDATA_OP_HIX22,  82)
-ELF_RELOC(R_SPARC_GOTDATA_OP_LOX22,  83)
+ELF_RELOC(R_SPARC_GOTDATA_OP_LOX10,  83)
 ELF_RELOC(R_SPARC_GOTDATA_OP,     84)
diff --git a/include/llvm/Support/Endian.h b/include/llvm/Support/Endian.h
index e9fe22e5eda..fd59009e0d3 100644
--- a/include/llvm/Support/Endian.h
+++ b/include/llvm/Support/Endian.h
@@ -104,6 +104,16 @@ struct packed_endian_specific_integral {
     return *this;
   }
 
+  packed_endian_specific_integral &operator|=(value_type newValue) {
+    *this = *this | newValue;
+    return *this;
+  }
+
+  packed_endian_specific_integral &operator&=(value_type newValue) {
+    *this = *this & newValue;
+    return *this;
+  }
+
 private:
   AlignedCharArray<PickAlignment<value_type, alignment>::value,
                    sizeof(value_type)> Value;
diff --git a/include/llvm/Support/ErrorOr.h b/include/llvm/Support/ErrorOr.h
index 3577a12cec3..589404f9b4e 100644
--- a/include/llvm/Support/ErrorOr.h
+++ b/include/llvm/Support/ErrorOr.h
@@ -281,8 +281,8 @@ template <class T, class E>
 typename std::enable_if<std::is_error_code_enum<E>::value ||
                             std::is_error_condition_enum<E>::value,
                         bool>::type
-operator==(ErrorOr<T> &Err, E Code) {
-  return std::error_code(Err) == Code;
+operator==(const ErrorOr<T> &Err, E Code) {
+  return Err.getError() == Code;
 }
 } // end namespace llvm
 
diff --git a/include/llvm/Support/GraphWriter.h b/include/llvm/Support/GraphWriter.h
index 7d1c273c988..b1af3d7c263 100644
--- a/include/llvm/Support/GraphWriter.h
+++ b/include/llvm/Support/GraphWriter.h
@@ -353,7 +353,7 @@ void ViewGraph(const GraphType &G, const Twine &Name,
   if (Filename.empty())
     return;
 
-  DisplayGraph(Filename, true, Program);
+  DisplayGraph(Filename, false, Program);
 }
 
 } // End llvm namespace
diff --git a/include/llvm/Support/MathExtras.h b/include/llvm/Support/MathExtras.h
index 7edc2ac2e7a..2cf7e0e5d0b 100644
--- a/include/llvm/Support/MathExtras.h
+++ b/include/llvm/Support/MathExtras.h
@@ -24,6 +24,10 @@
 #include <intrin.h>
 #endif
 
+#ifdef __ANDROID_NDK__
+#include <android/api-level.h>
+#endif
+
 namespace llvm {
 /// \brief The behavior an operation has on an input of 0.
 enum ZeroBehavior {
@@ -449,6 +453,15 @@ inline unsigned countPopulation(T Value) {
   return detail::PopulationCounter<T, sizeof(T)>::count(Value);
 }
 
+/// Log2 - This function returns the log base 2 of the specified value
+inline double Log2(double Value) {
+#if defined(__ANDROID_API__) && __ANDROID_API__ < 18
+  return __builtin_log(Value) / __builtin_log(2.0);
+#else
+  return log2(Value);
+#endif
+}
+
 /// Log2_32 - This function returns the floor log base 2 of the specified value,
 /// -1 if the value is zero. (32 bit edition.)
 /// Ex. Log2_32(32) == 5, Log2_32(1) == 0, Log2_32(0) == -1, Log2_32(6) == 2
@@ -549,7 +562,7 @@ inline uint64_t MinAlign(uint64_t A, uint64_t B) {
 ///
 /// Alignment should be a power of two.  This method rounds up, so
 /// alignAddr(7, 4) == 8 and alignAddr(8, 4) == 8.
-inline uintptr_t alignAddr(void *Addr, size_t Alignment) {
+inline uintptr_t alignAddr(const void *Addr, size_t Alignment) {
   assert(Alignment && isPowerOf2_64((uint64_t)Alignment) &&
          "Alignment is not a power of two!");
 
@@ -560,7 +573,7 @@ inline uintptr_t alignAddr(void *Addr, size_t Alignment) {
 
 /// \brief Returns the necessary adjustment for aligning \c Ptr to \c Alignment
 /// bytes, rounding up.
-inline size_t alignmentAdjustment(void *Ptr, size_t Alignment) {
+inline size_t alignmentAdjustment(const void *Ptr, size_t Alignment) {
   return alignAddr(Ptr, Alignment) - (uintptr_t)Ptr;
 }
 
diff --git a/include/llvm/Support/MemoryBuffer.h b/include/llvm/Support/MemoryBuffer.h
index 35a7bdb004a..81616d8ba3a 100644
--- a/include/llvm/Support/MemoryBuffer.h
+++ b/include/llvm/Support/MemoryBuffer.h
@@ -124,7 +124,7 @@ class MemoryBuffer {
   static ErrorOr<std::unique_ptr<MemoryBuffer>>
   getFileOrSTDIN(const Twine &Filename, int64_t FileSize = -1);
 
-  /// Map a subrange of the the specified file as a MemoryBuffer.
+  /// Map a subrange of the specified file as a MemoryBuffer.
   static ErrorOr<std::unique_ptr<MemoryBuffer>>
   getFileSlice(const Twine &Filename, uint64_t MapSize, uint64_t Offset);
 
diff --git a/include/llvm/Support/MipsABIFlags.h b/include/llvm/Support/MipsABIFlags.h
new file mode 100644
index 00000000000..93f6b416ba8
--- /dev/null
+++ b/include/llvm/Support/MipsABIFlags.h
@@ -0,0 +1,102 @@
+//===--- MipsABIFlags.h - MIPS ABI flags ----------------------------------===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This file defines the constants for the ABI flags structure contained
+// in the .MIPS.abiflags section.
+//
+// https://dmz-portal.mips.com/wiki/MIPS_O32_ABI_-_FR0_and_FR1_Interlinking
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_SUPPORT_MIPSABIFLAGS_H
+#define LLVM_SUPPORT_MIPSABIFLAGS_H
+
+namespace llvm {
+namespace Mips {
+
+// Values for the xxx_size bytes of an ABI flags structure.
+enum AFL_REG {
+  AFL_REG_NONE = 0x00, // No registers
+  AFL_REG_32 = 0x01,   // 32-bit registers
+  AFL_REG_64 = 0x02,   // 64-bit registers
+  AFL_REG_128 = 0x03   // 128-bit registers
+};
+
+// Masks for the ases word of an ABI flags structure.
+enum AFL_ASE {
+  AFL_ASE_DSP = 0x00000001,       // DSP ASE
+  AFL_ASE_DSPR2 = 0x00000002,     // DSP R2 ASE
+  AFL_ASE_EVA = 0x00000004,       // Enhanced VA Scheme
+  AFL_ASE_MCU = 0x00000008,       // MCU (MicroController) ASE
+  AFL_ASE_MDMX = 0x00000010,      // MDMX ASE
+  AFL_ASE_MIPS3D = 0x00000020,    // MIPS-3D ASE
+  AFL_ASE_MT = 0x00000040,        // MT ASE
+  AFL_ASE_SMARTMIPS = 0x00000080, // SmartMIPS ASE
+  AFL_ASE_VIRT = 0x00000100,      // VZ ASE
+  AFL_ASE_MSA = 0x00000200,       // MSA ASE
+  AFL_ASE_MIPS16 = 0x00000400,    // MIPS16 ASE
+  AFL_ASE_MICROMIPS = 0x00000800, // MICROMIPS ASE
+  AFL_ASE_XPA = 0x00001000        // XPA ASE
+};
+
+// Values for the isa_ext word of an ABI flags structure.
+enum AFL_EXT {
+  AFL_EXT_NONE = 0,         // None
+  AFL_EXT_XLR = 1,          // RMI Xlr instruction
+  AFL_EXT_OCTEON2 = 2,      // Cavium Networks Octeon2
+  AFL_EXT_OCTEONP = 3,      // Cavium Networks OcteonP
+  AFL_EXT_LOONGSON_3A = 4,  // Loongson 3A
+  AFL_EXT_OCTEON = 5,       // Cavium Networks Octeon
+  AFL_EXT_5900 = 6,         // MIPS R5900 instruction
+  AFL_EXT_4650 = 7,         // MIPS R4650 instruction
+  AFL_EXT_4010 = 8,         // LSI R4010 instruction
+  AFL_EXT_4100 = 9,         // NEC VR4100 instruction
+  AFL_EXT_3900 = 10,        // Toshiba R3900 instruction
+  AFL_EXT_10000 = 11,       // MIPS R10000 instruction
+  AFL_EXT_SB1 = 12,         // Broadcom SB-1 instruction
+  AFL_EXT_4111 = 13,        // NEC VR4111/VR4181 instruction
+  AFL_EXT_4120 = 14,        // NEC VR4120 instruction
+  AFL_EXT_5400 = 15,        // NEC VR5400 instruction
+  AFL_EXT_5500 = 16,        // NEC VR5500 instruction
+  AFL_EXT_LOONGSON_2E = 17, // ST Microelectronics Loongson 2E
+  AFL_EXT_LOONGSON_2F = 18, // ST Microelectronics Loongson 2F
+  AFL_EXT_OCTEON3 = 19      // Cavium Networks Octeon3
+};
+
+// Values for the flags1 word of an ABI flags structure.
+enum AFL_FLAGS1 { AFL_FLAGS1_ODDSPREG = 1 };
+
+// MIPS object attribute tags
+enum {
+  Tag_GNU_MIPS_ABI_FP = 4,  // Floating-point ABI used by this object file
+  Tag_GNU_MIPS_ABI_MSA = 8, // MSA ABI used by this object file
+};
+
+// Values for the fp_abi word of an ABI flags structure
+// and for the Tag_GNU_MIPS_ABI_FP attribute tag.
+enum Val_GNU_MIPS_ABI_FP {
+  Val_GNU_MIPS_ABI_FP_ANY = 0,    // not tagged
+  Val_GNU_MIPS_ABI_FP_DOUBLE = 1, // hard float / -mdouble-float
+  Val_GNU_MIPS_ABI_FP_SINGLE = 2, // hard float / -msingle-float
+  Val_GNU_MIPS_ABI_FP_SOFT = 3,   // soft float
+  Val_GNU_MIPS_ABI_FP_OLD_64 = 4, // -mips32r2 -mfp64
+  Val_GNU_MIPS_ABI_FP_XX = 5,     // -mfpxx
+  Val_GNU_MIPS_ABI_FP_64 = 6,     // -mips32r2 -mfp64
+  Val_GNU_MIPS_ABI_FP_64A = 7     // -mips32r2 -mfp64 -mno-odd-spreg
+};
+
+// Values for the Tag_GNU_MIPS_ABI_MSA attribute tag.
+enum Val_GNU_MIPS_ABI_MSA {
+  Val_GNU_MIPS_ABI_MSA_ANY = 0, // not tagged
+  Val_GNU_MIPS_ABI_MSA_128 = 1  // 128-bit MSA
+};
+}
+}
+
+#endif
diff --git a/include/llvm/Support/Mutex.h b/include/llvm/Support/Mutex.h
index ae69634eff7..0f4e61af443 100644
--- a/include/llvm/Support/Mutex.h
+++ b/include/llvm/Support/Mutex.h
@@ -70,7 +70,9 @@ namespace llvm
     /// @name Platform Dependent Data
     /// @{
     private:
+#if defined(LLVM_ENABLE_THREADS) && LLVM_ENABLE_THREADS != 0
       void* data_; ///< We don't know what the data will be
+#endif
 
     /// @}
     /// @name Do Not Implement
diff --git a/include/llvm/Support/OnDiskHashTable.h b/include/llvm/Support/OnDiskHashTable.h
index 0f097f28728..08e277ad5ce 100644
--- a/include/llvm/Support/OnDiskHashTable.h
+++ b/include/llvm/Support/OnDiskHashTable.h
@@ -280,13 +280,19 @@ template <typename Info> class OnDiskChainedHashTable {
   };
 
   /// \brief Look up the stored data for a particular key.
-  iterator find(const external_key_type &EKey, Info *InfoPtr = 0) {
-    if (!InfoPtr)
-      InfoPtr = &InfoObj;
-
-    using namespace llvm::support;
+  iterator find(const external_key_type &EKey, Info *InfoPtr = nullptr) {
     const internal_key_type &IKey = InfoObj.GetInternalKey(EKey);
     hash_value_type KeyHash = InfoObj.ComputeHash(IKey);
+    return find_hashed(IKey, KeyHash, InfoPtr);
+  }
+
+  /// \brief Look up the stored data for a particular key with a known hash.
+  iterator find_hashed(const internal_key_type &IKey, hash_value_type KeyHash,
+                       Info *InfoPtr = nullptr) {
+    using namespace llvm::support;
+
+    if (!InfoPtr)
+      InfoPtr = &InfoObj;
 
     // Each bucket is just an offset into the hash table file.
     offset_type Idx = KeyHash & (NumBuckets - 1);
diff --git a/include/llvm/Support/Path.h b/include/llvm/Support/Path.h
index 8fae853e2cf..f7dd1fc1208 100644
--- a/include/llvm/Support/Path.h
+++ b/include/llvm/Support/Path.h
@@ -61,7 +61,6 @@ class const_iterator
   reference operator*() const { return Component; }
   pointer   operator->() const { return &Component; }
   const_iterator &operator++();    // preincrement
-  const_iterator &operator++(int); // postincrement
   bool operator==(const const_iterator &RHS) const;
   bool operator!=(const const_iterator &RHS) const { return !(*this == RHS); }
 
@@ -87,7 +86,6 @@ class reverse_iterator
   reference operator*() const { return Component; }
   pointer   operator->() const { return &Component; }
   reverse_iterator &operator++();    // preincrement
-  reverse_iterator &operator++(int); // postincrement
   bool operator==(const reverse_iterator &RHS) const;
   bool operator!=(const reverse_iterator &RHS) const { return !(*this == RHS); }
 };
diff --git a/include/llvm/Support/QueueStreamer.h b/include/llvm/Support/QueueStreamer.h
deleted file mode 100644
index 40f95f5a4b8..00000000000
--- a/include/llvm/Support/QueueStreamer.h
+++ /dev/null
@@ -1,99 +0,0 @@
-//===-- QueueStreamer.h - Stream data from external source ------*- C++ -*-===//
-//
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
-//
-//===----------------------------------------------------------------------===//
-//
-// Implements LLVM's interface for fetching data from a stream source
-// (DataStreamer). Typically, bytes are pushed by an external source.
-// Typically, the there is a handler thread that waits for push requests
-// and calls PutBytes, and a consumer thread that calls GetBytes. The bytes
-// are buffered until the consumer calls GetBytes to remove them.
-// The blocking behavior of GetBytes and PutBytes means that if the consumer
-// is faster than the producer, then the whole consumer pipeline can block
-// waiting for the producer. Similarly, if the consumer is slower, then PutBytes
-// will block, and the external source (producer) will know how far along
-// the consumer has advanced, modulo the amount in the bounded buffer of
-// the QueueStreamer.
-//
-//===----------------------------------------------------------------------===//
-
-
-#ifndef LLVM_SUPPORT_QUEUESTREAMER_H
-#define LLVM_SUPPORT_QUEUESTREAMER_H
-
-#include <condition_variable>
-#include <mutex>
-#include <vector>
-
-#include "llvm/Support/DataStream.h"
-
-namespace llvm {
-
-class QueueStreamer : public DataStreamer {
-  QueueStreamer(const QueueStreamer &) = delete;
-  QueueStreamer &operator=(const QueueStreamer &) = delete;
-
-  enum {
-    // Initial size of the queue's buffer.
-    BaseSize = 64 * 1024,
-    // Maximum size of the queue. Since PutBytes and GetBytes may block,
-    // the partial-copying behavior of GetBytes and PutBytes allows progress
-    // to be made and prevents deadlock even if the requested number of
-    // bytes from Put/Get are greater than the size limit.
-    // Keep the max size "small" so that the external source can approximate
-    // how far along the consumer has advanced (modulo this buffer amount).
-    MaxSize = 256 * 1024
-  };
-
-public:
-  QueueStreamer() : Done(false), Prod(0), Cons(0) {
-    Bytes.resize(BaseSize);
-  }
-
-  // Copy Len bytes from the QueueStreamer into buf. If there are less
-  // than Len bytes available, copy as many as there are and signal the
-  // thread that may be blocking on PutBytes, and block GetBytes to wait
-  // for the rest. If all bytes have been received (SetDone is called)
-  // and there are fewer than Len bytes available, copy all remaining bytes.
-  // Return the number of bytes copied.
-  size_t GetBytes(unsigned char *Buf, size_t Len) override;
-
-  // Copy Len bytes from Buf into the QueueStreamer. If there is not enough
-  // space in the queue, copy as many bytes as will fit, signal the thread
-  // that may be blocking on GetBytes, and block until there is enough space
-  // for the rest. Return the number of bytes copied.
-  size_t PutBytes(unsigned char *Buf, size_t Len);
-
-  // Called by the same thread that does PutBytes. Signals the end of the
-  // data stream and may unblock GetBytes.
-  void SetDone();
-
-private:
-  bool Done;
-  typedef std::mutex LockType;
-  LockType Mutex;
-  std::condition_variable Cond;
-
-  // Variables and functions to manage the circular queue
-  std::vector<unsigned char> Bytes;
-  size_t Prod; // Queue producer index
-  size_t Cons; // Queue consumer index
-
-  size_t queueSize() const {
-    return Prod >= Cons ? Prod - Cons : Bytes.size() - (Cons - Prod);
-  }
-  size_t capacityRemaining() const {
-    return (Prod >= Cons ? Bytes.size() - (Prod - Cons) : (Cons - Prod)) - 1;
-  }
-  void queueResize();
-  void queuePut(unsigned char *Buf, size_t Len);
-  void queueGet(unsigned char *Buf, size_t Len);
-};
-
-} // end of namespace llvm
-
-#endif  // LLVM_SUPPORT_QUEUESTREAMER_H
diff --git a/include/llvm/Support/RWMutex.h b/include/llvm/Support/RWMutex.h
index 9a5e4213dfa..4be93133776 100644
--- a/include/llvm/Support/RWMutex.h
+++ b/include/llvm/Support/RWMutex.h
@@ -70,7 +70,9 @@ namespace llvm
     /// @name Platform Dependent Data
     /// @{
     private:
+#if defined(LLVM_ENABLE_THREADS) && LLVM_ENABLE_THREADS != 0
       void* data_; ///< We don't know what the data will be
+#endif
 
     /// @}
     /// @name Do Not Implement
diff --git a/include/llvm/Support/Registry.h b/include/llvm/Support/Registry.h
index 1f81d075136..95c4e96f7f2 100644
--- a/include/llvm/Support/Registry.h
+++ b/include/llvm/Support/Registry.h
@@ -14,6 +14,7 @@
 #ifndef LLVM_SUPPORT_REGISTRY_H
 #define LLVM_SUPPORT_REGISTRY_H
 
+#include "llvm/ADT/iterator_range.h"
 #include "llvm/ADT/STLExtras.h"
 #include "llvm/Support/Compiler.h"
 #include <memory>
diff --git a/include/llvm/Support/ScaledNumber.h b/include/llvm/Support/ScaledNumber.h
index a1c4c80c356..0a4262b7eec 100644
--- a/include/llvm/Support/ScaledNumber.h
+++ b/include/llvm/Support/ScaledNumber.h
@@ -514,7 +514,7 @@ template <class DigitsT> class ScaledNumber : ScaledNumberBase {
       : Digits(Digits), Scale(Scale) {}
 
 private:
-  ScaledNumber(const std::pair<uint64_t, int16_t> &X)
+  ScaledNumber(const std::pair<DigitsT, int16_t> &X)
       : Digits(X.first), Scale(X.second) {}
 
 public:
@@ -670,14 +670,7 @@ template <class DigitsT> class ScaledNumber : ScaledNumberBase {
     return ScaledNumbers::compare(Digits, Scale, X.Digits, X.Scale);
   }
   int compareTo(uint64_t N) const {
-    ScaledNumber Scaled = get(N);
-    int Compare = compare(Scaled);
-    if (Width == 64 || Compare != 0)
-      return Compare;
-
-    // Check for precision loss.  We know *this == RoundTrip.
-    uint64_t RoundTrip = Scaled.template toInt<uint64_t>();
-    return N == RoundTrip ? 0 : RoundTrip < N ? -1 : 1;
+    return ScaledNumbers::compare<uint64_t>(Digits, Scale, N, 0);
   }
   int compareTo(int64_t N) const { return N < 0 ? 1 : compareTo(uint64_t(N)); }
 
@@ -732,10 +725,20 @@ SCALED_NUMBER_BOP(+, += )
 SCALED_NUMBER_BOP(-, -= )
 SCALED_NUMBER_BOP(*, *= )
 SCALED_NUMBER_BOP(/, /= )
-SCALED_NUMBER_BOP(<<, <<= )
-SCALED_NUMBER_BOP(>>, >>= )
 #undef SCALED_NUMBER_BOP
 
+template <class DigitsT>
+ScaledNumber<DigitsT> operator<<(const ScaledNumber<DigitsT> &L,
+                                 int16_t Shift) {
+  return ScaledNumber<DigitsT>(L) <<= Shift;
+}
+
+template <class DigitsT>
+ScaledNumber<DigitsT> operator>>(const ScaledNumber<DigitsT> &L,
+                                 int16_t Shift) {
+  return ScaledNumber<DigitsT>(L) >>= Shift;
+}
+
 template <class DigitsT>
 raw_ostream &operator<<(raw_ostream &OS, const ScaledNumber<DigitsT> &X) {
   return X.print(OS, 10);
diff --git a/include/llvm/Support/Signals.h b/include/llvm/Support/Signals.h
index 7e165d7f3a4..2a4d84bd891 100644
--- a/include/llvm/Support/Signals.h
+++ b/include/llvm/Support/Signals.h
@@ -47,6 +47,9 @@ namespace sys {
   /// \brief Print the stack trace using the given \c raw_ostream object.
   void PrintStackTrace(raw_ostream &OS);
 
+  // Run all registered signal handlers.
+  void RunSignalHandlers();
+
   /// AddSignalHandler - Add a function to be called when an abort/kill signal
   /// is delivered to the process.  The handler can have a cookie passed to it
   /// to identify what instance of the handler it is.
diff --git a/include/llvm/Support/SourceMgr.h b/include/llvm/Support/SourceMgr.h
index d492748e778..1f8b1a01865 100644
--- a/include/llvm/Support/SourceMgr.h
+++ b/include/llvm/Support/SourceMgr.h
@@ -276,8 +276,8 @@ class SMDiagnostic {
     return FixIts;
   }
 
-  void print(const char *ProgName, raw_ostream &S,
-             bool ShowColors = true) const;
+  void print(const char *ProgName, raw_ostream &S, bool ShowColors = true,
+             bool ShowKindLabel = true) const;
 };
 
 }  // end llvm namespace
diff --git a/include/llvm/Support/StreamingMemoryObject.h b/include/llvm/Support/StreamingMemoryObject.h
index e6eb1cc7b35..7cb6438d134 100644
--- a/include/llvm/Support/StreamingMemoryObject.h
+++ b/include/llvm/Support/StreamingMemoryObject.h
@@ -22,35 +22,19 @@ namespace llvm {
 /// Interface to data which is actually streamed from a DataStreamer. In
 /// addition to inherited members, it has the dropLeadingBytes and
 /// setKnownObjectSize methods which are not applicable to non-streamed objects.
-/// @LOCALMOD -- Made dropLeadingBytes/setKnownObjectSize virtual.
 class StreamingMemoryObject : public MemoryObject {
- public:
-  /// Drop s bytes from the front of the stream, pushing the positions of the
-  /// remaining bytes down by s. This is used to skip past the bitcode header,
-  /// since we don't know a priori if it's present, and we can't put bytes
-  /// back into the stream once we've read them.
-  virtual bool dropLeadingBytes(size_t s) = 0;
-
-  /// If the data object size is known in advance, many of the operations can
-  /// be made more efficient, so this method should be called before reading
-  /// starts (although it can be called anytime).
-  virtual void setKnownObjectSize(size_t size) = 0;
-};
-
-/// @LOCALMOD -- split out StreamingMemoryObjectImpl from StreamingMemoryObject.
-/// StreamingMemoryObjectImpl - an implementation of a StreamingMemoryObject.
-class StreamingMemoryObjectImpl : public StreamingMemoryObject {
 public:
-  StreamingMemoryObjectImpl(DataStreamer *streamer);
+  StreamingMemoryObject(std::unique_ptr<DataStreamer> Streamer);
   uint64_t getExtent() const override;
   uint64_t readBytes(uint8_t *Buf, uint64_t Size,
                      uint64_t Address) const override;
   const uint8_t *getPointer(uint64_t address, uint64_t size) const override {
-    // This could be fixed by ensuring the bytes are fetched and making a copy,
-    // requiring that the bitcode size be known, or otherwise ensuring that
-    // the memory doesn't go away/get reallocated, but it's
-    // not currently necessary. Users that need the pointer don't stream.
-    llvm_unreachable("getPointer in streaming memory objects not allowed");
+    // FIXME: This could be fixed by ensuring the bytes are fetched and
+    // making a copy, requiring that the bitcode size be known, or
+    // otherwise ensuring that the memory doesn't go away/get reallocated,
+    // but it's not currently necessary. Users that need the pointer (any
+    // that need Blobs) don't stream.
+    report_fatal_error("getPointer in streaming memory objects not allowed");
     return nullptr;
   }
   bool isValidAddress(uint64_t address) const override;
@@ -59,12 +43,12 @@ class StreamingMemoryObjectImpl : public StreamingMemoryObject {
   /// remaining bytes down by s. This is used to skip past the bitcode header,
   /// since we don't know a priori if it's present, and we can't put bytes
   /// back into the stream once we've read them.
-  bool dropLeadingBytes(size_t s) override;
+  bool dropLeadingBytes(size_t s);
 
   /// If the data object size is known in advance, many of the operations can
   /// be made more efficient, so this method should be called before reading
   /// starts (although it can be called anytime).
-  void setKnownObjectSize(size_t size) override;
+  void setKnownObjectSize(size_t size);
 
 private:
   const static uint32_t kChunkSize = 4096 * 4;
@@ -75,30 +59,31 @@ class StreamingMemoryObjectImpl : public StreamingMemoryObject {
   mutable size_t ObjectSize; // 0 if unknown, set if wrapper seen or EOF reached
   mutable bool EOFReached;
 
-  // Fetch enough bytes such that Pos can be read or EOF is reached
-  // (i.e. BytesRead > Pos). Return true if Pos can be read.
-  // Unlike most of the functions in BitcodeReader, returns true on success.
-  // Most of the requests will be small, but we fetch at kChunkSize bytes
-  // at a time to avoid making too many potentially expensive GetBytes calls
+  // Fetch enough bytes such that Pos can be read (i.e. BytesRead >
+  // Pos). Returns true if Pos can be read.  Unlike most of the
+  // functions in BitcodeReader, returns true on success.  Most of the
+  // requests will be small, but we fetch at kChunkSize bytes at a
+  // time to avoid making too many potentially expensive GetBytes
+  // calls.
   bool fetchToPos(size_t Pos) const {
-    if (EOFReached)
-      return Pos < ObjectSize;
     while (Pos >= BytesRead) {
+      if (EOFReached)
+        return false;
       Bytes.resize(BytesRead + BytesSkipped + kChunkSize);
       size_t bytes = Streamer->GetBytes(&Bytes[BytesRead + BytesSkipped],
                                         kChunkSize);
       BytesRead += bytes;
-      if (bytes != kChunkSize) { // reached EOF/ran out of bytes
-        ObjectSize = BytesRead;
+      if (bytes == 0) { // reached EOF/ran out of bytes
+        if (ObjectSize == 0)
+          ObjectSize = BytesRead;
         EOFReached = true;
-        break;
       }
     }
-    return Pos < BytesRead;
+    return !ObjectSize || Pos < ObjectSize;
   }
 
-  StreamingMemoryObjectImpl(const StreamingMemoryObjectImpl&) = delete;
-  void operator=(const StreamingMemoryObjectImpl&) = delete;
+  StreamingMemoryObject(const StreamingMemoryObject&) = delete;
+  void operator=(const StreamingMemoryObject&) = delete;
 };
 
 MemoryObject *getNonStreamedMemoryObject(
diff --git a/include/llvm/Support/StringSaver.h b/include/llvm/Support/StringSaver.h
new file mode 100644
index 00000000000..f3853ee9157
--- /dev/null
+++ b/include/llvm/Support/StringSaver.h
@@ -0,0 +1,42 @@
+//===- llvm/Support/StringSaver.h -------------------------------*- C++ -*-===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_SUPPORT_STRINGSAVER_H
+#define LLVM_SUPPORT_STRINGSAVER_H
+
+#include "llvm/ADT/StringRef.h"
+#include "llvm/ADT/Twine.h"
+#include "llvm/Support/Allocator.h"
+
+namespace llvm {
+
+/// \brief Saves strings in the inheritor's stable storage and returns a stable
+/// raw character pointer.
+class StringSaver {
+protected:
+  ~StringSaver() {}
+  virtual const char *saveImpl(StringRef S);
+
+public:
+  StringSaver(BumpPtrAllocator &Alloc) : Alloc(Alloc) {}
+  const char *save(const char *S) { return save(StringRef(S)); }
+  const char *save(StringRef S) { return saveImpl(S); }
+  const char *save(const Twine &S) { return save(StringRef(S.str())); }
+  const char *save(std::string &S) { return save(StringRef(S)); }
+
+private:
+  BumpPtrAllocator &Alloc;
+};
+
+class BumpPtrStringSaver final : public StringSaver {
+public:
+  BumpPtrStringSaver(BumpPtrAllocator &Alloc) : StringSaver(Alloc) {}
+};
+}
+#endif
diff --git a/include/llvm/Support/SwapByteOrder.h b/include/llvm/Support/SwapByteOrder.h
index 70564be890e..7761fa1d7e0 100644
--- a/include/llvm/Support/SwapByteOrder.h
+++ b/include/llvm/Support/SwapByteOrder.h
@@ -104,7 +104,7 @@ inline float getSwappedBytes(float C) {
   return out.f;
 }
 
-inline float getSwappedBytes(double C) {
+inline double getSwappedBytes(double C) {
   union {
     uint64_t i;
     double d;
diff --git a/include/llvm/Support/TargetParser.h b/include/llvm/Support/TargetParser.h
new file mode 100644
index 00000000000..dd37afad290
--- /dev/null
+++ b/include/llvm/Support/TargetParser.h
@@ -0,0 +1,212 @@
+//===-- TargetParser - Parser for target features ---------------*- C++ -*-===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This file implements a target parser to recognise hardware features such as
+// FPU/CPU/ARCH names as well as specific support such as HDIV, etc.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_SUPPORT_TARGETPARSER_H
+#define LLVM_SUPPORT_TARGETPARSER_H
+
+// FIXME: vector is used because that's what clang uses for subtarget feature
+// lists, but SmallVector would probably be better
+#include <vector>
+
+namespace llvm {
+  class StringRef;
+
+// Target specific information into their own namespaces. These should be
+// generated from TableGen because the information is already there, and there
+// is where new information about targets will be added.
+// FIXME: To TableGen this we need to make some table generated files available
+// even if the back-end is not compiled with LLVM, plus we need to create a new
+// back-end to TableGen to create these clean tables.
+namespace ARM {
+  // FPU names.
+  enum FPUKind {
+    FK_INVALID = 0,
+    FK_NONE,
+    FK_VFP,
+    FK_VFPV2,
+    FK_VFPV3,
+    FK_VFPV3_FP16,
+    FK_VFPV3_D16,
+    FK_VFPV3_D16_FP16,
+    FK_VFPV3XD,
+    FK_VFPV3XD_FP16,
+    FK_VFPV4,
+    FK_VFPV4_D16,
+    FK_FPV4_SP_D16,
+    FK_FPV5_D16,
+    FK_FPV5_SP_D16,
+    FK_FP_ARMV8,
+    FK_NEON,
+    FK_NEON_FP16,
+    FK_NEON_VFPV4,
+    FK_NEON_FP_ARMV8,
+    FK_CRYPTO_NEON_FP_ARMV8,
+    FK_SOFTVFP,
+    FK_LAST
+  };
+
+  // FPU Version
+  enum FPUVersion {
+    FV_NONE = 0,
+    FV_VFPV2,
+    FV_VFPV3,
+    FV_VFPV3_FP16,
+    FV_VFPV4,
+    FV_VFPV5
+  };
+
+  // An FPU name implies one of three levels of Neon support:
+  enum NeonSupportLevel {
+    NS_None = 0, ///< No Neon
+    NS_Neon,     ///< Neon
+    NS_Crypto    ///< Neon with Crypto
+  };
+
+  // An FPU name restricts the FPU in one of three ways:
+  enum FPURestriction {
+    FR_None = 0, ///< No restriction
+    FR_D16,      ///< Only 16 D registers
+    FR_SP_D16    ///< Only single-precision instructions, with 16 D registers
+  };
+
+  // Arch names.
+  enum ArchKind {
+    AK_INVALID = 0,
+    AK_ARMV2,
+    AK_ARMV2A,
+    AK_ARMV3,
+    AK_ARMV3M,
+    AK_ARMV4,
+    AK_ARMV4T,
+    AK_ARMV5T,
+    AK_ARMV5TE,
+    AK_ARMV5TEJ,
+    AK_ARMV6,
+    AK_ARMV6K,
+    AK_ARMV6T2,
+    AK_ARMV6Z,
+    AK_ARMV6ZK,
+    AK_ARMV6M,
+    AK_ARMV6SM,
+    AK_ARMV7A,
+    AK_ARMV7R,
+    AK_ARMV7M,
+    AK_ARMV7EM,
+    AK_ARMV8A,
+    AK_ARMV8_1A,
+    // Non-standard Arch names.
+    AK_IWMMXT,
+    AK_IWMMXT2,
+    AK_XSCALE,
+    AK_ARMV5,
+    AK_ARMV5E,
+    AK_ARMV6J,
+    AK_ARMV6HL,
+    AK_ARMV7,
+    AK_ARMV7L,
+    AK_ARMV7HL,
+    AK_ARMV7S,
+    AK_LAST
+  };
+
+  // Arch extension modifiers for CPUs.
+  enum ArchExtKind : unsigned {
+    AEK_INVALID  = 0x0,
+    AEK_NONE     = 0x1,
+    AEK_CRC      = 0x2,
+    AEK_CRYPTO   = 0x4,
+    AEK_FP       = 0x8,
+    AEK_HWDIV    = 0x10,
+    AEK_HWDIVARM = 0x20,
+    AEK_MP       = 0x40,
+    AEK_SIMD     = 0x80,
+    AEK_SEC      = 0x100,
+    AEK_VIRT     = 0x200,
+    // Unsupported extensions.
+    AEK_OS       = 0x8000000,
+    AEK_IWMMXT   = 0x10000000,
+    AEK_IWMMXT2  = 0x20000000,
+    AEK_MAVERICK = 0x40000000,
+    AEK_XSCALE   = 0x80000000,
+  };
+
+  // ISA kinds.
+  enum ISAKind {
+    IK_INVALID = 0,
+    IK_ARM,
+    IK_THUMB,
+    IK_AARCH64
+  };
+
+  // Endianness
+  // FIXME: BE8 vs. BE32?
+  enum EndianKind {
+    EK_INVALID = 0,
+    EK_LITTLE,
+    EK_BIG
+  };
+
+  // v6/v7/v8 Profile
+  enum ProfileKind {
+    PK_INVALID = 0,
+    PK_A,
+    PK_R,
+    PK_M
+  };
+} // namespace ARM
+
+// Target Parsers, one per architecture.
+class ARMTargetParser {
+  static StringRef getHWDivSynonym(StringRef HWDiv);
+  static StringRef getFPUSynonym(StringRef FPU);
+  static StringRef getArchSynonym(StringRef Arch);
+
+public:
+  static StringRef getCanonicalArchName(StringRef Arch);
+
+  // Information by ID
+  static const char * getFPUName(unsigned FPUKind);
+  static     unsigned getFPUVersion(unsigned FPUKind);
+  static     unsigned getFPUNeonSupportLevel(unsigned FPUKind);
+  static     unsigned getFPURestriction(unsigned FPUKind);
+  static     unsigned getDefaultFPU(StringRef CPU);
+  // FIXME: This should be moved to TargetTuple once it exists
+  static       bool   getFPUFeatures(unsigned FPUKind,
+                                     std::vector<const char*> &Features);
+  static       bool   getHWDivFeatures(unsigned HWDivKind,
+                                       std::vector<const char*> &Features);
+  static const char * getArchName(unsigned ArchKind);
+  static   unsigned   getArchAttr(unsigned ArchKind);
+  static const char * getCPUAttr(unsigned ArchKind);
+  static const char * getSubArch(unsigned ArchKind);
+  static const char * getArchExtName(unsigned ArchExtKind);
+  static const char * getHWDivName(unsigned HWDivKind);
+  static const char * getDefaultCPU(StringRef Arch);
+
+  // Parser
+  static unsigned parseHWDiv(StringRef HWDiv);
+  static unsigned parseFPU(StringRef FPU);
+  static unsigned parseArch(StringRef Arch);
+  static unsigned parseArchExt(StringRef ArchExt);
+  static unsigned parseCPUArch(StringRef CPU);
+  static unsigned parseArchISA(StringRef Arch);
+  static unsigned parseArchEndian(StringRef Arch);
+  static unsigned parseArchProfile(StringRef Arch);
+  static unsigned parseArchVersion(StringRef Arch);
+
+};
+
+} // namespace llvm
+
+#endif
diff --git a/include/llvm/Support/TargetRegistry.h b/include/llvm/Support/TargetRegistry.h
index 5d168de9a50..40bf6fb20c9 100644
--- a/include/llvm/Support/TargetRegistry.h
+++ b/include/llvm/Support/TargetRegistry.h
@@ -28,1216 +28,1158 @@
 #include <string>
 
 namespace llvm {
-  class AsmPrinter;
-  class MCAsmBackend;
-  class MCAsmInfo;
-  class MCAsmParser;
-  class MCCodeEmitter;
-  class MCCodeGenInfo;
-  class MCContext;
-  class MCDisassembler;
-  class MCInstrAnalysis;
-  class MCInstPrinter;
-  class MCInstrInfo;
-  class MCNaClExpander; // @LOCALMOD
-  class MCRegisterInfo;
-  class MCStreamer;
-  class MCSubtargetInfo;
-  class MCSymbolizer;
-  class MCRelocationInfo;
-  class MCTargetAsmParser;
-  class MCTargetOptions;
-  class MCTargetStreamer;
-  class TargetMachine;
-  class TargetOptions;
-  class raw_ostream;
-  class raw_pwrite_stream;
-  class formatted_raw_ostream;
-
-  MCStreamer *createNullStreamer(MCContext &Ctx);
-  MCStreamer *createAsmStreamer(MCContext &Ctx,
-                                std::unique_ptr<formatted_raw_ostream> OS,
-                                bool isVerboseAsm, bool useDwarfDirectory,
-                                MCInstPrinter *InstPrint, MCCodeEmitter *CE,
-                                MCAsmBackend *TAB, bool ShowInst);
-
-  /// Takes ownership of \p TAB and \p CE.
-  MCStreamer *createELFStreamer(MCContext &Ctx, MCAsmBackend &TAB,
+class AsmPrinter;
+class MCAsmBackend;
+class MCAsmInfo;
+class MCAsmParser;
+class MCCodeEmitter;
+class MCCodeGenInfo;
+class MCContext;
+class MCDisassembler;
+class MCInstrAnalysis;
+class MCInstPrinter;
+class MCInstrInfo;
+class MCRegisterInfo;
+class MCStreamer;
+class MCSubtargetInfo;
+class MCSymbolizer;
+class MCRelocationInfo;
+class MCTargetAsmParser;
+class MCTargetOptions;
+class MCTargetStreamer;
+class TargetMachine;
+class TargetOptions;
+class raw_ostream;
+class raw_pwrite_stream;
+class formatted_raw_ostream;
+
+MCStreamer *createNullStreamer(MCContext &Ctx);
+MCStreamer *createAsmStreamer(MCContext &Ctx,
+                              std::unique_ptr<formatted_raw_ostream> OS,
+                              bool isVerboseAsm, bool useDwarfDirectory,
+                              MCInstPrinter *InstPrint, MCCodeEmitter *CE,
+                              MCAsmBackend *TAB, bool ShowInst);
+
+/// Takes ownership of \p TAB and \p CE.
+MCStreamer *createELFStreamer(MCContext &Ctx, MCAsmBackend &TAB,
+                              raw_pwrite_stream &OS, MCCodeEmitter *CE,
+                              bool RelaxAll);
+MCStreamer *createMachOStreamer(MCContext &Ctx, MCAsmBackend &TAB,
                                 raw_pwrite_stream &OS, MCCodeEmitter *CE,
-                                bool RelaxAll);
-  MCStreamer *createMachOStreamer(MCContext &Ctx, MCAsmBackend &TAB,
-                                  raw_pwrite_stream &OS, MCCodeEmitter *CE,
-                                  bool RelaxAll, bool DWARFMustBeAtTheEnd,
-                                  bool LabelSections = false);
-
-  MCRelocationInfo *createMCRelocationInfo(StringRef TT, MCContext &Ctx);
-
-  MCSymbolizer *createMCSymbolizer(StringRef TT, LLVMOpInfoCallback GetOpInfo,
-                                   LLVMSymbolLookupCallback SymbolLookUp,
-                                   void *DisInfo, MCContext *Ctx,
-                                   std::unique_ptr<MCRelocationInfo> &&RelInfo);
-
-  /// Target - Wrapper for Target specific information.
-  ///
-  /// For registration purposes, this is a POD type so that targets can be
-  /// registered without the use of static constructors.
-  ///
-  /// Targets should implement a single global instance of this class (which
-  /// will be zero initialized), and pass that instance to the TargetRegistry as
-  /// part of their initialization.
-  class Target {
-  public:
-    friend struct TargetRegistry;
-
-    typedef bool (*ArchMatchFnTy)(Triple::ArchType Arch);
-
-    typedef MCAsmInfo *(*MCAsmInfoCtorFnTy)(const MCRegisterInfo &MRI,
-                                            StringRef TT);
-    typedef MCCodeGenInfo *(*MCCodeGenInfoCtorFnTy)(StringRef TT,
-                                                    Reloc::Model RM,
-                                                    CodeModel::Model CM,
-                                                    CodeGenOpt::Level OL);
-    typedef MCInstrInfo *(*MCInstrInfoCtorFnTy)(void);
-    typedef MCInstrAnalysis *(*MCInstrAnalysisCtorFnTy)(const MCInstrInfo*Info);
-    typedef MCRegisterInfo *(*MCRegInfoCtorFnTy)(StringRef TT);
-    typedef MCSubtargetInfo *(*MCSubtargetInfoCtorFnTy)(StringRef TT,
-                                                        StringRef CPU,
-                                                        StringRef Features);
-    typedef TargetMachine *(*TargetMachineCtorTy)(const Target &T,
-                                                  StringRef TT,
-                                                  StringRef CPU,
-                                                  StringRef Features,
-                                                  const TargetOptions &Options,
+                                bool RelaxAll, bool DWARFMustBeAtTheEnd,
+                                bool LabelSections = false);
+
+MCRelocationInfo *createMCRelocationInfo(const Triple &TT, MCContext &Ctx);
+
+MCSymbolizer *createMCSymbolizer(const Triple &TT, LLVMOpInfoCallback GetOpInfo,
+                                 LLVMSymbolLookupCallback SymbolLookUp,
+                                 void *DisInfo, MCContext *Ctx,
+                                 std::unique_ptr<MCRelocationInfo> &&RelInfo);
+
+/// Target - Wrapper for Target specific information.
+///
+/// For registration purposes, this is a POD type so that targets can be
+/// registered without the use of static constructors.
+///
+/// Targets should implement a single global instance of this class (which
+/// will be zero initialized), and pass that instance to the TargetRegistry as
+/// part of their initialization.
+class Target {
+public:
+  friend struct TargetRegistry;
+
+  typedef bool (*ArchMatchFnTy)(Triple::ArchType Arch);
+
+  typedef MCAsmInfo *(*MCAsmInfoCtorFnTy)(const MCRegisterInfo &MRI,
+                                          const Triple &TT);
+  typedef MCCodeGenInfo *(*MCCodeGenInfoCtorFnTy)(const Triple &TT,
                                                   Reloc::Model RM,
                                                   CodeModel::Model CM,
                                                   CodeGenOpt::Level OL);
-    // If it weren't for layering issues (this header is in llvm/Support, but
-    // depends on MC?) this should take the Streamer by value rather than rvalue
-    // reference.
-    typedef AsmPrinter *(*AsmPrinterCtorTy)(
-        TargetMachine &TM, std::unique_ptr<MCStreamer> &&Streamer);
-    typedef MCAsmBackend *(*MCAsmBackendCtorTy)(const Target &T,
-                                                const MCRegisterInfo &MRI,
-                                                StringRef TT,
-                                                StringRef CPU);
-    typedef MCTargetAsmParser *(*MCAsmParserCtorTy)(
-        MCSubtargetInfo &STI,
-        MCAsmParser &P,
-        const MCInstrInfo &MII,
-        const MCTargetOptions &Options);
-    typedef MCDisassembler *(*MCDisassemblerCtorTy)(const Target &T,
-                                                    const MCSubtargetInfo &STI,
-                                                    MCContext &Ctx);
-    typedef MCInstPrinter *(*MCInstPrinterCtorTy)(const Triple &T,
-                                                  unsigned SyntaxVariant,
-                                                  const MCAsmInfo &MAI,
-                                                  const MCInstrInfo &MII,
-                                                  const MCRegisterInfo &MRI);
-    typedef MCCodeEmitter *(*MCCodeEmitterCtorTy)(const MCInstrInfo &II,
-                                                  const MCRegisterInfo &MRI,
+  typedef MCInstrInfo *(*MCInstrInfoCtorFnTy)(void);
+  typedef MCInstrAnalysis *(*MCInstrAnalysisCtorFnTy)(const MCInstrInfo *Info);
+  typedef MCRegisterInfo *(*MCRegInfoCtorFnTy)(const Triple &TT);
+  typedef MCSubtargetInfo *(*MCSubtargetInfoCtorFnTy)(const Triple &TT,
+                                                      StringRef CPU,
+                                                      StringRef Features);
+  typedef TargetMachine *(*TargetMachineCtorTy)(
+      const Target &T, const Triple &TT, StringRef CPU, StringRef Features,
+      const TargetOptions &Options, Reloc::Model RM, CodeModel::Model CM,
+      CodeGenOpt::Level OL);
+  // If it weren't for layering issues (this header is in llvm/Support, but
+  // depends on MC?) this should take the Streamer by value rather than rvalue
+  // reference.
+  typedef AsmPrinter *(*AsmPrinterCtorTy)(
+      TargetMachine &TM, std::unique_ptr<MCStreamer> &&Streamer);
+  typedef MCAsmBackend *(*MCAsmBackendCtorTy)(const Target &T,
+                                              const MCRegisterInfo &MRI,
+                                              const Triple &TT, StringRef CPU);
+  typedef MCTargetAsmParser *(*MCAsmParserCtorTy)(
+      MCSubtargetInfo &STI, MCAsmParser &P, const MCInstrInfo &MII,
+      const MCTargetOptions &Options);
+  typedef MCDisassembler *(*MCDisassemblerCtorTy)(const Target &T,
+                                                  const MCSubtargetInfo &STI,
                                                   MCContext &Ctx);
-    typedef MCStreamer *(*ELFStreamerCtorTy)(const Triple &T, MCContext &Ctx,
-                                             MCAsmBackend &TAB,
+  typedef MCInstPrinter *(*MCInstPrinterCtorTy)(const Triple &T,
+                                                unsigned SyntaxVariant,
+                                                const MCAsmInfo &MAI,
+                                                const MCInstrInfo &MII,
+                                                const MCRegisterInfo &MRI);
+  typedef MCCodeEmitter *(*MCCodeEmitterCtorTy)(const MCInstrInfo &II,
+                                                const MCRegisterInfo &MRI,
+                                                MCContext &Ctx);
+  typedef MCStreamer *(*ELFStreamerCtorTy)(const Triple &T, MCContext &Ctx,
+                                           MCAsmBackend &TAB,
+                                           raw_pwrite_stream &OS,
+                                           MCCodeEmitter *Emitter,
+                                           bool RelaxAll);
+  typedef MCStreamer *(*MachOStreamerCtorTy)(MCContext &Ctx, MCAsmBackend &TAB,
                                              raw_pwrite_stream &OS,
                                              MCCodeEmitter *Emitter,
-                                             bool RelaxAll);
-    typedef MCStreamer *(*MachOStreamerCtorTy)(
-        MCContext &Ctx, MCAsmBackend &TAB, raw_pwrite_stream &OS,
-        MCCodeEmitter *Emitter, bool RelaxAll, bool DWARFMustBeAtTheEnd);
-    typedef MCStreamer *(*COFFStreamerCtorTy)(MCContext &Ctx, MCAsmBackend &TAB,
-                                              raw_pwrite_stream &OS,
-                                              MCCodeEmitter *Emitter,
-                                              bool RelaxAll);
-    typedef MCTargetStreamer *(*NullTargetStreamerCtorTy)(MCStreamer &S);
-    typedef MCTargetStreamer *(*AsmTargetStreamerCtorTy)(
-        MCStreamer &S, formatted_raw_ostream &OS, MCInstPrinter *InstPrint,
-        bool IsVerboseAsm);
-    typedef MCTargetStreamer *(*ObjectTargetStreamerCtorTy)(
-        MCStreamer &S, const MCSubtargetInfo &STI);
-    typedef MCRelocationInfo *(*MCRelocationInfoCtorTy)(StringRef TT,
-                                                        MCContext &Ctx);
-    typedef MCSymbolizer *(*MCSymbolizerCtorTy)(
-        StringRef TT, LLVMOpInfoCallback GetOpInfo,
-        LLVMSymbolLookupCallback SymbolLookUp, void *DisInfo, MCContext *Ctx,
-        std::unique_ptr<MCRelocationInfo> &&RelInfo);
-
-    // @LOCALMOD-START
-    typedef void (*MCNaClExpanderCtorTy)(
-        MCStreamer &S, std::unique_ptr<MCRegisterInfo> &&RegInfo,
-        std::unique_ptr<MCInstrInfo> &&InstInfo);
-    // @LOCALMOD-END
-
-  private:
-    /// Next - The next registered target in the linked list, maintained by the
-    /// TargetRegistry.
-    Target *Next;
-
-    /// The target function for checking if an architecture is supported.
-    ArchMatchFnTy ArchMatchFn;
-
-    /// Name - The target name.
-    const char *Name;
-
-    /// ShortDesc - A short description of the target.
-    const char *ShortDesc;
-
-    /// HasJIT - Whether this target supports the JIT.
-    bool HasJIT;
-
-    /// MCAsmInfoCtorFn - Constructor function for this target's MCAsmInfo, if
-    /// registered.
-    MCAsmInfoCtorFnTy MCAsmInfoCtorFn;
-
-    /// MCCodeGenInfoCtorFn - Constructor function for this target's
-    /// MCCodeGenInfo, if registered.
-    MCCodeGenInfoCtorFnTy MCCodeGenInfoCtorFn;
-
-    /// MCInstrInfoCtorFn - Constructor function for this target's MCInstrInfo,
-    /// if registered.
-    MCInstrInfoCtorFnTy MCInstrInfoCtorFn;
-
-    /// MCInstrAnalysisCtorFn - Constructor function for this target's
-    /// MCInstrAnalysis, if registered.
-    MCInstrAnalysisCtorFnTy MCInstrAnalysisCtorFn;
-
-    /// MCRegInfoCtorFn - Constructor function for this target's MCRegisterInfo,
-    /// if registered.
-    MCRegInfoCtorFnTy MCRegInfoCtorFn;
-
-    /// MCSubtargetInfoCtorFn - Constructor function for this target's
-    /// MCSubtargetInfo, if registered.
-    MCSubtargetInfoCtorFnTy MCSubtargetInfoCtorFn;
-
-    /// TargetMachineCtorFn - Construction function for this target's
-    /// TargetMachine, if registered.
-    TargetMachineCtorTy TargetMachineCtorFn;
-
-    /// MCAsmBackendCtorFn - Construction function for this target's
-    /// MCAsmBackend, if registered.
-    MCAsmBackendCtorTy MCAsmBackendCtorFn;
-
-    /// MCAsmParserCtorFn - Construction function for this target's
-    /// MCTargetAsmParser, if registered.
-    MCAsmParserCtorTy MCAsmParserCtorFn;
-
-    /// AsmPrinterCtorFn - Construction function for this target's AsmPrinter,
-    /// if registered.
-    AsmPrinterCtorTy AsmPrinterCtorFn;
-
-    /// MCDisassemblerCtorFn - Construction function for this target's
-    /// MCDisassembler, if registered.
-    MCDisassemblerCtorTy MCDisassemblerCtorFn;
-
-    /// MCInstPrinterCtorFn - Construction function for this target's
-    /// MCInstPrinter, if registered.
-    MCInstPrinterCtorTy MCInstPrinterCtorFn;
-
-    /// MCCodeEmitterCtorFn - Construction function for this target's
-    /// CodeEmitter, if registered.
-    MCCodeEmitterCtorTy MCCodeEmitterCtorFn;
-
-    // Construction functions for the various object formats, if registered.
-    COFFStreamerCtorTy COFFStreamerCtorFn;
-    MachOStreamerCtorTy MachOStreamerCtorFn;
-    ELFStreamerCtorTy ELFStreamerCtorFn;
-
-    /// Construction function for this target's null TargetStreamer, if
-    /// registered (default = nullptr).
-    NullTargetStreamerCtorTy NullTargetStreamerCtorFn;
-
-    /// Construction function for this target's asm TargetStreamer, if
-    /// registered (default = nullptr).
-    AsmTargetStreamerCtorTy AsmTargetStreamerCtorFn;
-
-    /// Construction function for this target's obj TargetStreamer, if
-    /// registered (default = nullptr).
-    ObjectTargetStreamerCtorTy ObjectTargetStreamerCtorFn;
-
-    /// MCRelocationInfoCtorFn - Construction function for this target's
-    /// MCRelocationInfo, if registered (default = llvm::createMCRelocationInfo)
-    MCRelocationInfoCtorTy MCRelocationInfoCtorFn;
+                                             bool RelaxAll,
+                                             bool DWARFMustBeAtTheEnd);
+  typedef MCStreamer *(*COFFStreamerCtorTy)(MCContext &Ctx, MCAsmBackend &TAB,
+                                            raw_pwrite_stream &OS,
+                                            MCCodeEmitter *Emitter,
+                                            bool RelaxAll);
+  typedef MCTargetStreamer *(*NullTargetStreamerCtorTy)(MCStreamer &S);
+  typedef MCTargetStreamer *(*AsmTargetStreamerCtorTy)(
+      MCStreamer &S, formatted_raw_ostream &OS, MCInstPrinter *InstPrint,
+      bool IsVerboseAsm);
+  typedef MCTargetStreamer *(*ObjectTargetStreamerCtorTy)(
+      MCStreamer &S, const MCSubtargetInfo &STI);
+  typedef MCRelocationInfo *(*MCRelocationInfoCtorTy)(const Triple &TT,
+                                                      MCContext &Ctx);
+  typedef MCSymbolizer *(*MCSymbolizerCtorTy)(
+      const Triple &TT, LLVMOpInfoCallback GetOpInfo,
+      LLVMSymbolLookupCallback SymbolLookUp, void *DisInfo, MCContext *Ctx,
+      std::unique_ptr<MCRelocationInfo> &&RelInfo);
+
+private:
+  /// Next - The next registered target in the linked list, maintained by the
+  /// TargetRegistry.
+  Target *Next;
+
+  /// The target function for checking if an architecture is supported.
+  ArchMatchFnTy ArchMatchFn;
+
+  /// Name - The target name.
+  const char *Name;
+
+  /// ShortDesc - A short description of the target.
+  const char *ShortDesc;
+
+  /// HasJIT - Whether this target supports the JIT.
+  bool HasJIT;
+
+  /// MCAsmInfoCtorFn - Constructor function for this target's MCAsmInfo, if
+  /// registered.
+  MCAsmInfoCtorFnTy MCAsmInfoCtorFn;
+
+  /// MCCodeGenInfoCtorFn - Constructor function for this target's
+  /// MCCodeGenInfo, if registered.
+  MCCodeGenInfoCtorFnTy MCCodeGenInfoCtorFn;
+
+  /// MCInstrInfoCtorFn - Constructor function for this target's MCInstrInfo,
+  /// if registered.
+  MCInstrInfoCtorFnTy MCInstrInfoCtorFn;
+
+  /// MCInstrAnalysisCtorFn - Constructor function for this target's
+  /// MCInstrAnalysis, if registered.
+  MCInstrAnalysisCtorFnTy MCInstrAnalysisCtorFn;
+
+  /// MCRegInfoCtorFn - Constructor function for this target's MCRegisterInfo,
+  /// if registered.
+  MCRegInfoCtorFnTy MCRegInfoCtorFn;
+
+  /// MCSubtargetInfoCtorFn - Constructor function for this target's
+  /// MCSubtargetInfo, if registered.
+  MCSubtargetInfoCtorFnTy MCSubtargetInfoCtorFn;
+
+  /// TargetMachineCtorFn - Construction function for this target's
+  /// TargetMachine, if registered.
+  TargetMachineCtorTy TargetMachineCtorFn;
+
+  /// MCAsmBackendCtorFn - Construction function for this target's
+  /// MCAsmBackend, if registered.
+  MCAsmBackendCtorTy MCAsmBackendCtorFn;
+
+  /// MCAsmParserCtorFn - Construction function for this target's
+  /// MCTargetAsmParser, if registered.
+  MCAsmParserCtorTy MCAsmParserCtorFn;
+
+  /// AsmPrinterCtorFn - Construction function for this target's AsmPrinter,
+  /// if registered.
+  AsmPrinterCtorTy AsmPrinterCtorFn;
+
+  /// MCDisassemblerCtorFn - Construction function for this target's
+  /// MCDisassembler, if registered.
+  MCDisassemblerCtorTy MCDisassemblerCtorFn;
+
+  /// MCInstPrinterCtorFn - Construction function for this target's
+  /// MCInstPrinter, if registered.
+  MCInstPrinterCtorTy MCInstPrinterCtorFn;
+
+  /// MCCodeEmitterCtorFn - Construction function for this target's
+  /// CodeEmitter, if registered.
+  MCCodeEmitterCtorTy MCCodeEmitterCtorFn;
+
+  // Construction functions for the various object formats, if registered.
+  COFFStreamerCtorTy COFFStreamerCtorFn;
+  MachOStreamerCtorTy MachOStreamerCtorFn;
+  ELFStreamerCtorTy ELFStreamerCtorFn;
+
+  /// Construction function for this target's null TargetStreamer, if
+  /// registered (default = nullptr).
+  NullTargetStreamerCtorTy NullTargetStreamerCtorFn;
+
+  /// Construction function for this target's asm TargetStreamer, if
+  /// registered (default = nullptr).
+  AsmTargetStreamerCtorTy AsmTargetStreamerCtorFn;
+
+  /// Construction function for this target's obj TargetStreamer, if
+  /// registered (default = nullptr).
+  ObjectTargetStreamerCtorTy ObjectTargetStreamerCtorFn;
+
+  /// MCRelocationInfoCtorFn - Construction function for this target's
+  /// MCRelocationInfo, if registered (default = llvm::createMCRelocationInfo)
+  MCRelocationInfoCtorTy MCRelocationInfoCtorFn;
 
-    /// MCSymbolizerCtorFn - Construction function for this target's
-    /// MCSymbolizer, if registered (default = llvm::createMCSymbolizer)
-    MCSymbolizerCtorTy MCSymbolizerCtorFn;
+  /// MCSymbolizerCtorFn - Construction function for this target's
+  /// MCSymbolizer, if registered (default = llvm::createMCSymbolizer)
+  MCSymbolizerCtorTy MCSymbolizerCtorFn;
 
-    // @LOCALMOD-START
-    // MCNaClExpanderCtorFn - Construction function for this target's
-    // MCNaClExpander, if registered.
-    MCNaClExpanderCtorTy MCNaClExpanderCtorFn;
-    // @LOCALMOD-END
+public:
+  Target()
+      : COFFStreamerCtorFn(nullptr), MachOStreamerCtorFn(nullptr),
+        ELFStreamerCtorFn(nullptr), NullTargetStreamerCtorFn(nullptr),
+        AsmTargetStreamerCtorFn(nullptr), ObjectTargetStreamerCtorFn(nullptr),
+        MCRelocationInfoCtorFn(nullptr), MCSymbolizerCtorFn(nullptr) {}
 
-  public:
-    Target()
-        : COFFStreamerCtorFn(nullptr), MachOStreamerCtorFn(nullptr),
-          ELFStreamerCtorFn(nullptr), NullTargetStreamerCtorFn(nullptr),
-          AsmTargetStreamerCtorFn(nullptr), ObjectTargetStreamerCtorFn(nullptr),
-          MCRelocationInfoCtorFn(nullptr), MCSymbolizerCtorFn(nullptr) {}
-
-    /// @name Target Information
-    /// @{
-
-    // getNext - Return the next registered target.
-    const Target *getNext() const { return Next; }
-
-    /// getName - Get the target name.
-    const char *getName() const { return Name; }
-
-    /// getShortDescription - Get a short description of the target.
-    const char *getShortDescription() const { return ShortDesc; }
-
-    /// @}
-    /// @name Feature Predicates
-    /// @{
-
-    /// hasJIT - Check if this targets supports the just-in-time compilation.
-    bool hasJIT() const { return HasJIT; }
-
-    /// hasTargetMachine - Check if this target supports code generation.
-    bool hasTargetMachine() const { return TargetMachineCtorFn != nullptr; }
-
-    /// hasMCAsmBackend - Check if this target supports .o generation.
-    bool hasMCAsmBackend() const { return MCAsmBackendCtorFn != nullptr; }
-
-    /// @}
-    /// @name Feature Constructors
-    /// @{
-
-    /// createMCAsmInfo - Create a MCAsmInfo implementation for the specified
-    /// target triple.
-    ///
-    /// \param Triple This argument is used to determine the target machine
-    /// feature set; it should always be provided. Generally this should be
-    /// either the target triple from the module, or the target triple of the
-    /// host if that does not exist.
-    MCAsmInfo *createMCAsmInfo(const MCRegisterInfo &MRI,
-                               StringRef Triple) const {
-      if (!MCAsmInfoCtorFn)
-        return nullptr;
-      return MCAsmInfoCtorFn(MRI, Triple);
-    }
+  /// @name Target Information
+  /// @{
 
-    /// createMCCodeGenInfo - Create a MCCodeGenInfo implementation.
-    ///
-    MCCodeGenInfo *createMCCodeGenInfo(StringRef Triple, Reloc::Model RM,
-                                       CodeModel::Model CM,
-                                       CodeGenOpt::Level OL) const {
-      if (!MCCodeGenInfoCtorFn)
-        return nullptr;
-      return MCCodeGenInfoCtorFn(Triple, RM, CM, OL);
-    }
+  // getNext - Return the next registered target.
+  const Target *getNext() const { return Next; }
 
-    /// createMCInstrInfo - Create a MCInstrInfo implementation.
-    ///
-    MCInstrInfo *createMCInstrInfo() const {
-      if (!MCInstrInfoCtorFn)
-        return nullptr;
-      return MCInstrInfoCtorFn();
-    }
+  /// getName - Get the target name.
+  const char *getName() const { return Name; }
 
-    /// createMCInstrAnalysis - Create a MCInstrAnalysis implementation.
-    ///
-    MCInstrAnalysis *createMCInstrAnalysis(const MCInstrInfo *Info) const {
-      if (!MCInstrAnalysisCtorFn)
-        return nullptr;
-      return MCInstrAnalysisCtorFn(Info);
-    }
+  /// getShortDescription - Get a short description of the target.
+  const char *getShortDescription() const { return ShortDesc; }
 
-    /// createMCRegInfo - Create a MCRegisterInfo implementation.
-    ///
-    MCRegisterInfo *createMCRegInfo(StringRef Triple) const {
-      if (!MCRegInfoCtorFn)
-        return nullptr;
-      return MCRegInfoCtorFn(Triple);
-    }
+  /// @}
+  /// @name Feature Predicates
+  /// @{
 
-    /// createMCSubtargetInfo - Create a MCSubtargetInfo implementation.
-    ///
-    /// \param Triple This argument is used to determine the target machine
-    /// feature set; it should always be provided. Generally this should be
-    /// either the target triple from the module, or the target triple of the
-    /// host if that does not exist.
-    /// \param CPU This specifies the name of the target CPU.
-    /// \param Features This specifies the string representation of the
-    /// additional target features.
-    MCSubtargetInfo *createMCSubtargetInfo(StringRef Triple, StringRef CPU,
-                                           StringRef Features) const {
-      if (!MCSubtargetInfoCtorFn)
-        return nullptr;
-      return MCSubtargetInfoCtorFn(Triple, CPU, Features);
-    }
+  /// hasJIT - Check if this targets supports the just-in-time compilation.
+  bool hasJIT() const { return HasJIT; }
 
-    /// createTargetMachine - Create a target specific machine implementation
-    /// for the specified \p Triple.
-    ///
-    /// \param Triple This argument is used to determine the target machine
-    /// feature set; it should always be provided. Generally this should be
-    /// either the target triple from the module, or the target triple of the
-    /// host if that does not exist.
-    TargetMachine *createTargetMachine(StringRef Triple, StringRef CPU,
-                             StringRef Features, const TargetOptions &Options,
-                             Reloc::Model RM = Reloc::Default,
-                             CodeModel::Model CM = CodeModel::Default,
-                             CodeGenOpt::Level OL = CodeGenOpt::Default) const {
-      if (!TargetMachineCtorFn)
-        return nullptr;
-      return TargetMachineCtorFn(*this, Triple, CPU, Features, Options,
-                                 RM, CM, OL);
-    }
+  /// hasTargetMachine - Check if this target supports code generation.
+  bool hasTargetMachine() const { return TargetMachineCtorFn != nullptr; }
 
-    /// createMCAsmBackend - Create a target specific assembly parser.
-    ///
-    /// \param Triple The target triple string.
-    MCAsmBackend *createMCAsmBackend(const MCRegisterInfo &MRI,
-                                     StringRef Triple, StringRef CPU) const {
-      if (!MCAsmBackendCtorFn)
-        return nullptr;
-      return MCAsmBackendCtorFn(*this, MRI, Triple, CPU);
-    }
+  /// hasMCAsmBackend - Check if this target supports .o generation.
+  bool hasMCAsmBackend() const { return MCAsmBackendCtorFn != nullptr; }
 
-    /// createMCAsmParser - Create a target specific assembly parser.
-    ///
-    /// \param Parser The target independent parser implementation to use for
-    /// parsing and lexing.
-    MCTargetAsmParser *createMCAsmParser(
-        MCSubtargetInfo &STI,
-        MCAsmParser &Parser,
-        const MCInstrInfo &MII,
-        const MCTargetOptions &Options) const {
-      if (!MCAsmParserCtorFn)
-        return nullptr;
-      return MCAsmParserCtorFn(STI, Parser, MII, Options);
-    }
+  /// @}
+  /// @name Feature Constructors
+  /// @{
 
-    /// createAsmPrinter - Create a target specific assembly printer pass.  This
-    /// takes ownership of the MCStreamer object.
-    AsmPrinter *createAsmPrinter(TargetMachine &TM,
-                                 std::unique_ptr<MCStreamer> &&Streamer) const {
-      if (!AsmPrinterCtorFn)
-        return nullptr;
-      return AsmPrinterCtorFn(TM, std::move(Streamer));
-    }
-
-    MCDisassembler *createMCDisassembler(const MCSubtargetInfo &STI,
-                                         MCContext &Ctx) const {
-      if (!MCDisassemblerCtorFn)
-        return nullptr;
-      return MCDisassemblerCtorFn(*this, STI, Ctx);
-    }
-
-    MCInstPrinter *createMCInstPrinter(const Triple &T, unsigned SyntaxVariant,
-                                       const MCAsmInfo &MAI,
-                                       const MCInstrInfo &MII,
-                                       const MCRegisterInfo &MRI) const {
-      if (!MCInstPrinterCtorFn)
-        return nullptr;
-      return MCInstPrinterCtorFn(T, SyntaxVariant, MAI, MII, MRI);
-    }
-
-
-    /// createMCCodeEmitter - Create a target specific code emitter.
-    MCCodeEmitter *createMCCodeEmitter(const MCInstrInfo &II,
-                                       const MCRegisterInfo &MRI,
-                                       MCContext &Ctx) const {
-      if (!MCCodeEmitterCtorFn)
-        return nullptr;
-      return MCCodeEmitterCtorFn(II, MRI, Ctx);
-    }
-
-    /// Create a target specific MCStreamer.
-    ///
-    /// \param T The target triple.
-    /// \param Ctx The target context.
-    /// \param TAB The target assembler backend object. Takes ownership.
-    /// \param OS The stream object.
-    /// \param Emitter The target independent assembler object.Takes ownership.
-    /// \param RelaxAll Relax all fixups?
-    MCStreamer *createMCObjectStreamer(const Triple &T, MCContext &Ctx,
-                                       MCAsmBackend &TAB, raw_pwrite_stream &OS,
-                                       MCCodeEmitter *Emitter,
-                                       const MCSubtargetInfo &STI,
-                                       bool RelaxAll,
-                                       bool DWARFMustBeAtTheEnd) const {
-      MCStreamer *S;
-      switch (T.getObjectFormat()) {
-      default:
-        llvm_unreachable("Unknown object format");
-      case Triple::COFF:
-        assert(T.isOSWindows() && "only Windows COFF is supported");
-        S = COFFStreamerCtorFn(Ctx, TAB, OS, Emitter, RelaxAll);
-        break;
-      case Triple::MachO:
-        if (MachOStreamerCtorFn)
-          S = MachOStreamerCtorFn(Ctx, TAB, OS, Emitter, RelaxAll,
-                                  DWARFMustBeAtTheEnd);
-        else
-          S = createMachOStreamer(Ctx, TAB, OS, Emitter, RelaxAll,
-                                  DWARFMustBeAtTheEnd);
-        break;
-      case Triple::ELF:
-        if (ELFStreamerCtorFn)
-          S = ELFStreamerCtorFn(T, Ctx, TAB, OS, Emitter, RelaxAll);
-        else
-          S = createELFStreamer(Ctx, TAB, OS, Emitter, RelaxAll);
-        break;
-      }
-      if (ObjectTargetStreamerCtorFn)
-        ObjectTargetStreamerCtorFn(*S, STI);
-      return S;
-    }
-
-    MCStreamer *createAsmStreamer(MCContext &Ctx,
-                                  std::unique_ptr<formatted_raw_ostream> OS,
-                                  bool IsVerboseAsm, bool UseDwarfDirectory,
-                                  MCInstPrinter *InstPrint, MCCodeEmitter *CE,
-                                  MCAsmBackend *TAB, bool ShowInst) const {
-      formatted_raw_ostream &OSRef = *OS;
-      MCStreamer *S = llvm::createAsmStreamer(Ctx, std::move(OS), IsVerboseAsm,
-                                              UseDwarfDirectory, InstPrint, CE,
-                                              TAB, ShowInst);
-      createAsmTargetStreamer(*S, OSRef, InstPrint, IsVerboseAsm);
-      return S;
-    }
-
-    // @LOCALMOD-START
-    void createMCNaClExpander(MCStreamer &S,
-                              std::unique_ptr<MCRegisterInfo> &&RegInfo,
-                              std::unique_ptr<MCInstrInfo> &&InstInfo) const {
-      if (MCNaClExpanderCtorFn)
-        MCNaClExpanderCtorFn(S, std::move(RegInfo), std::move(InstInfo));
-    }
-    // @LOCALMOD-END
-
-    MCTargetStreamer *createAsmTargetStreamer(MCStreamer &S,
-                                              formatted_raw_ostream &OS,
-                                              MCInstPrinter *InstPrint,
-                                              bool IsVerboseAsm) const {
-      if (AsmTargetStreamerCtorFn)
-        return AsmTargetStreamerCtorFn(S, OS, InstPrint, IsVerboseAsm);
+  /// createMCAsmInfo - Create a MCAsmInfo implementation for the specified
+  /// target triple.
+  ///
+  /// \param TheTriple This argument is used to determine the target machine
+  /// feature set; it should always be provided. Generally this should be
+  /// either the target triple from the module, or the target triple of the
+  /// host if that does not exist.
+  MCAsmInfo *createMCAsmInfo(const MCRegisterInfo &MRI,
+                             StringRef TheTriple) const {
+    if (!MCAsmInfoCtorFn)
       return nullptr;
-    }
-
-    MCStreamer *createNullStreamer(MCContext &Ctx) const {
-      MCStreamer *S = llvm::createNullStreamer(Ctx);
-      createNullTargetStreamer(*S);
-      return S;
-    }
+    return MCAsmInfoCtorFn(MRI, Triple(TheTriple));
+  }
 
-    MCTargetStreamer *createNullTargetStreamer(MCStreamer &S) const {
-      if (NullTargetStreamerCtorFn)
-        return NullTargetStreamerCtorFn(S);
+  /// createMCCodeGenInfo - Create a MCCodeGenInfo implementation.
+  ///
+  MCCodeGenInfo *createMCCodeGenInfo(StringRef TT, Reloc::Model RM,
+                                     CodeModel::Model CM,
+                                     CodeGenOpt::Level OL) const {
+    if (!MCCodeGenInfoCtorFn)
       return nullptr;
-    }
+    return MCCodeGenInfoCtorFn(Triple(TT), RM, CM, OL);
+  }
 
-    /// createMCRelocationInfo - Create a target specific MCRelocationInfo.
-    ///
-    /// \param TT The target triple.
-    /// \param Ctx The target context.
-    MCRelocationInfo *
-      createMCRelocationInfo(StringRef TT, MCContext &Ctx) const {
-      MCRelocationInfoCtorTy Fn = MCRelocationInfoCtorFn
-                                      ? MCRelocationInfoCtorFn
-                                      : llvm::createMCRelocationInfo;
-      return Fn(TT, Ctx);
-    }
-
-    /// createMCSymbolizer - Create a target specific MCSymbolizer.
-    ///
-    /// \param TT The target triple.
-    /// \param GetOpInfo The function to get the symbolic information for operands.
-    /// \param SymbolLookUp The function to lookup a symbol name.
-    /// \param DisInfo The pointer to the block of symbolic information for above call
-    /// back.
-    /// \param Ctx The target context.
-    /// \param RelInfo The relocation information for this target. Takes ownership.
-    MCSymbolizer *
-    createMCSymbolizer(StringRef TT, LLVMOpInfoCallback GetOpInfo,
-                       LLVMSymbolLookupCallback SymbolLookUp, void *DisInfo,
-                       MCContext *Ctx,
-                       std::unique_ptr<MCRelocationInfo> &&RelInfo) const {
-      MCSymbolizerCtorTy Fn =
-          MCSymbolizerCtorFn ? MCSymbolizerCtorFn : llvm::createMCSymbolizer;
-      return Fn(TT, GetOpInfo, SymbolLookUp, DisInfo, Ctx, std::move(RelInfo));
-    }
-
-    /// @}
-  };
-
-  /// TargetRegistry - Generic interface to target specific features.
-  struct TargetRegistry {
-    class iterator {
-      const Target *Current;
-      explicit iterator(Target *T) : Current(T) {}
-      friend struct TargetRegistry;
-    public:
-      iterator() : Current(nullptr) {}
-
-      bool operator==(const iterator &x) const {
-        return Current == x.Current;
-      }
-      bool operator!=(const iterator &x) const {
-        return !operator==(x);
-      }
-
-      // Iterator traversal: forward iteration only
-      iterator &operator++() {          // Preincrement
-        assert(Current && "Cannot increment end iterator!");
-        Current = Current->getNext();
-        return *this;
-      }
-      iterator operator++(int) {        // Postincrement
-        iterator tmp = *this;
-        ++*this;
-        return tmp;
-      }
-
-      const Target &operator*() const {
-        assert(Current && "Cannot dereference end iterator!");
-        return *Current;
-      }
-
-      const Target *operator->() const {
-        return &operator*();
-      }
-    };
-
-    /// printRegisteredTargetsForVersion - Print the registered targets
-    /// appropriately for inclusion in a tool's version output.
-    static void printRegisteredTargetsForVersion();
-
-    /// @name Registry Access
-    /// @{
-
-    static iterator begin();
-
-    static iterator end() { return iterator(); }
-
-    /// lookupTarget - Lookup a target based on a target triple.
-    ///
-    /// \param Triple - The triple to use for finding a target.
-    /// \param Error - On failure, an error string describing why no target was
-    /// found.
-    static const Target *lookupTarget(const std::string &Triple,
-                                      std::string &Error);
-
-    /// lookupTarget - Lookup a target based on an architecture name
-    /// and a target triple.  If the architecture name is non-empty,
-    /// then the lookup is done by architecture.  Otherwise, the target
-    /// triple is used.
-    ///
-    /// \param ArchName - The architecture to use for finding a target.
-    /// \param TheTriple - The triple to use for finding a target.  The
-    /// triple is updated with canonical architecture name if a lookup
-    /// by architecture is done.
-    /// \param Error - On failure, an error string describing why no target was
-    /// found.
-    static const Target *lookupTarget(const std::string &ArchName,
-                                      Triple &TheTriple,
-                                      std::string &Error);
-
-    /// @}
-    /// @name Target Registration
-    /// @{
-
-    /// RegisterTarget - Register the given target. Attempts to register a
-    /// target which has already been registered will be ignored.
-    ///
-    /// Clients are responsible for ensuring that registration doesn't occur
-    /// while another thread is attempting to access the registry. Typically
-    /// this is done by initializing all targets at program startup.
-    ///
-    /// @param T - The target being registered.
-    /// @param Name - The target name. This should be a static string.
-    /// @param ShortDesc - A short target description. This should be a static
-    /// string.
-    /// @param ArchMatchFn - The arch match checking function for this target.
-    /// @param HasJIT - Whether the target supports JIT code
-    /// generation.
-    static void RegisterTarget(Target &T,
-                               const char *Name,
-                               const char *ShortDesc,
-                               Target::ArchMatchFnTy ArchMatchFn,
-                               bool HasJIT = false);
-
-    /// RegisterMCAsmInfo - Register a MCAsmInfo implementation for the
-    /// given target.
-    ///
-    /// Clients are responsible for ensuring that registration doesn't occur
-    /// while another thread is attempting to access the registry. Typically
-    /// this is done by initializing all targets at program startup.
-    ///
-    /// @param T - The target being registered.
-    /// @param Fn - A function to construct a MCAsmInfo for the target.
-    static void RegisterMCAsmInfo(Target &T, Target::MCAsmInfoCtorFnTy Fn) {
-      T.MCAsmInfoCtorFn = Fn;
-    }
-
-    /// RegisterMCCodeGenInfo - Register a MCCodeGenInfo implementation for the
-    /// given target.
-    ///
-    /// Clients are responsible for ensuring that registration doesn't occur
-    /// while another thread is attempting to access the registry. Typically
-    /// this is done by initializing all targets at program startup.
-    ///
-    /// @param T - The target being registered.
-    /// @param Fn - A function to construct a MCCodeGenInfo for the target.
-    static void RegisterMCCodeGenInfo(Target &T,
-                                     Target::MCCodeGenInfoCtorFnTy Fn) {
-      T.MCCodeGenInfoCtorFn = Fn;
-    }
-
-    /// RegisterMCInstrInfo - Register a MCInstrInfo implementation for the
-    /// given target.
-    ///
-    /// Clients are responsible for ensuring that registration doesn't occur
-    /// while another thread is attempting to access the registry. Typically
-    /// this is done by initializing all targets at program startup.
-    ///
-    /// @param T - The target being registered.
-    /// @param Fn - A function to construct a MCInstrInfo for the target.
-    static void RegisterMCInstrInfo(Target &T, Target::MCInstrInfoCtorFnTy Fn) {
-      T.MCInstrInfoCtorFn = Fn;
-    }
-
-    /// RegisterMCInstrAnalysis - Register a MCInstrAnalysis implementation for
-    /// the given target.
-    static void RegisterMCInstrAnalysis(Target &T,
-                                        Target::MCInstrAnalysisCtorFnTy Fn) {
-      T.MCInstrAnalysisCtorFn = Fn;
-    }
-
-    /// RegisterMCRegInfo - Register a MCRegisterInfo implementation for the
-    /// given target.
-    ///
-    /// Clients are responsible for ensuring that registration doesn't occur
-    /// while another thread is attempting to access the registry. Typically
-    /// this is done by initializing all targets at program startup.
-    ///
-    /// @param T - The target being registered.
-    /// @param Fn - A function to construct a MCRegisterInfo for the target.
-    static void RegisterMCRegInfo(Target &T, Target::MCRegInfoCtorFnTy Fn) {
-      T.MCRegInfoCtorFn = Fn;
-    }
-
-    /// RegisterMCSubtargetInfo - Register a MCSubtargetInfo implementation for
-    /// the given target.
-    ///
-    /// Clients are responsible for ensuring that registration doesn't occur
-    /// while another thread is attempting to access the registry. Typically
-    /// this is done by initializing all targets at program startup.
-    ///
-    /// @param T - The target being registered.
-    /// @param Fn - A function to construct a MCSubtargetInfo for the target.
-    static void RegisterMCSubtargetInfo(Target &T,
-                                        Target::MCSubtargetInfoCtorFnTy Fn) {
-      T.MCSubtargetInfoCtorFn = Fn;
-    }
-
-    /// RegisterTargetMachine - Register a TargetMachine implementation for the
-    /// given target.
-    ///
-    /// Clients are responsible for ensuring that registration doesn't occur
-    /// while another thread is attempting to access the registry. Typically
-    /// this is done by initializing all targets at program startup.
-    ///
-    /// @param T - The target being registered.
-    /// @param Fn - A function to construct a TargetMachine for the target.
-    static void RegisterTargetMachine(Target &T,
-                                      Target::TargetMachineCtorTy Fn) {
-      T.TargetMachineCtorFn = Fn;
-    }
-
-    /// RegisterMCAsmBackend - Register a MCAsmBackend implementation for the
-    /// given target.
-    ///
-    /// Clients are responsible for ensuring that registration doesn't occur
-    /// while another thread is attempting to access the registry. Typically
-    /// this is done by initializing all targets at program startup.
-    ///
-    /// @param T - The target being registered.
-    /// @param Fn - A function to construct an AsmBackend for the target.
-    static void RegisterMCAsmBackend(Target &T, Target::MCAsmBackendCtorTy Fn) {
-      T.MCAsmBackendCtorFn = Fn;
-    }
+  /// createMCInstrInfo - Create a MCInstrInfo implementation.
+  ///
+  MCInstrInfo *createMCInstrInfo() const {
+    if (!MCInstrInfoCtorFn)
+      return nullptr;
+    return MCInstrInfoCtorFn();
+  }
 
-    /// RegisterMCAsmParser - Register a MCTargetAsmParser implementation for
-    /// the given target.
-    ///
-    /// Clients are responsible for ensuring that registration doesn't occur
-    /// while another thread is attempting to access the registry. Typically
-    /// this is done by initializing all targets at program startup.
-    ///
-    /// @param T - The target being registered.
-    /// @param Fn - A function to construct an MCTargetAsmParser for the target.
-    static void RegisterMCAsmParser(Target &T, Target::MCAsmParserCtorTy Fn) {
-      T.MCAsmParserCtorFn = Fn;
-    }
+  /// createMCInstrAnalysis - Create a MCInstrAnalysis implementation.
+  ///
+  MCInstrAnalysis *createMCInstrAnalysis(const MCInstrInfo *Info) const {
+    if (!MCInstrAnalysisCtorFn)
+      return nullptr;
+    return MCInstrAnalysisCtorFn(Info);
+  }
 
-    /// RegisterAsmPrinter - Register an AsmPrinter implementation for the given
-    /// target.
-    ///
-    /// Clients are responsible for ensuring that registration doesn't occur
-    /// while another thread is attempting to access the registry. Typically
-    /// this is done by initializing all targets at program startup.
-    ///
-    /// @param T - The target being registered.
-    /// @param Fn - A function to construct an AsmPrinter for the target.
-    static void RegisterAsmPrinter(Target &T, Target::AsmPrinterCtorTy Fn) {
-      T.AsmPrinterCtorFn = Fn;
-    }
+  /// createMCRegInfo - Create a MCRegisterInfo implementation.
+  ///
+  MCRegisterInfo *createMCRegInfo(StringRef TT) const {
+    if (!MCRegInfoCtorFn)
+      return nullptr;
+    return MCRegInfoCtorFn(Triple(TT));
+  }
 
-    /// RegisterMCDisassembler - Register a MCDisassembler implementation for
-    /// the given target.
-    ///
-    /// Clients are responsible for ensuring that registration doesn't occur
-    /// while another thread is attempting to access the registry. Typically
-    /// this is done by initializing all targets at program startup.
-    ///
-    /// @param T - The target being registered.
-    /// @param Fn - A function to construct an MCDisassembler for the target.
-    static void RegisterMCDisassembler(Target &T,
-                                       Target::MCDisassemblerCtorTy Fn) {
-      T.MCDisassemblerCtorFn = Fn;
-    }
+  /// createMCSubtargetInfo - Create a MCSubtargetInfo implementation.
+  ///
+  /// \param TheTriple This argument is used to determine the target machine
+  /// feature set; it should always be provided. Generally this should be
+  /// either the target triple from the module, or the target triple of the
+  /// host if that does not exist.
+  /// \param CPU This specifies the name of the target CPU.
+  /// \param Features This specifies the string representation of the
+  /// additional target features.
+  MCSubtargetInfo *createMCSubtargetInfo(StringRef TheTriple, StringRef CPU,
+                                         StringRef Features) const {
+    if (!MCSubtargetInfoCtorFn)
+      return nullptr;
+    return MCSubtargetInfoCtorFn(Triple(TheTriple), CPU, Features);
+  }
 
-    /// RegisterMCInstPrinter - Register a MCInstPrinter implementation for the
-    /// given target.
-    ///
-    /// Clients are responsible for ensuring that registration doesn't occur
-    /// while another thread is attempting to access the registry. Typically
-    /// this is done by initializing all targets at program startup.
-    ///
-    /// @param T - The target being registered.
-    /// @param Fn - A function to construct an MCInstPrinter for the target.
-    static void RegisterMCInstPrinter(Target &T,
-                                      Target::MCInstPrinterCtorTy Fn) {
-      // @LOCALMOD-BEGIN
-      // Prune out the .s printer for the sandboxed translator,
-      // by preventing an InstPrinter from being used at all.
-      #if !defined(__native_client__)
-      T.MCInstPrinterCtorFn = Fn;
-      #endif
-      // @LOCALMOD-END
-    }
+  /// createTargetMachine - Create a target specific machine implementation
+  /// for the specified \p Triple.
+  ///
+  /// \param TT This argument is used to determine the target machine
+  /// feature set; it should always be provided. Generally this should be
+  /// either the target triple from the module, or the target triple of the
+  /// host if that does not exist.
+  TargetMachine *
+  createTargetMachine(StringRef TT, StringRef CPU, StringRef Features,
+                      const TargetOptions &Options,
+                      Reloc::Model RM = Reloc::Default,
+                      CodeModel::Model CM = CodeModel::Default,
+                      CodeGenOpt::Level OL = CodeGenOpt::Default) const {
+    if (!TargetMachineCtorFn)
+      return nullptr;
+    return TargetMachineCtorFn(*this, Triple(TT), CPU, Features, Options, RM,
+                               CM, OL);
+  }
 
-    /// RegisterMCCodeEmitter - Register a MCCodeEmitter implementation for the
-    /// given target.
-    ///
-    /// Clients are responsible for ensuring that registration doesn't occur
-    /// while another thread is attempting to access the registry. Typically
-    /// this is done by initializing all targets at program startup.
-    ///
-    /// @param T - The target being registered.
-    /// @param Fn - A function to construct an MCCodeEmitter for the target.
-    static void RegisterMCCodeEmitter(Target &T,
-                                      Target::MCCodeEmitterCtorTy Fn) {
-      T.MCCodeEmitterCtorFn = Fn;
-    }
+  /// createMCAsmBackend - Create a target specific assembly parser.
+  ///
+  /// \param TheTriple The target triple string.
+  MCAsmBackend *createMCAsmBackend(const MCRegisterInfo &MRI,
+                                   StringRef TheTriple, StringRef CPU) const {
+    if (!MCAsmBackendCtorFn)
+      return nullptr;
+    return MCAsmBackendCtorFn(*this, MRI, Triple(TheTriple), CPU);
+  }
 
-    static void RegisterCOFFStreamer(Target &T, Target::COFFStreamerCtorTy Fn) {
-      T.COFFStreamerCtorFn = Fn;
-    }
+  /// createMCAsmParser - Create a target specific assembly parser.
+  ///
+  /// \param Parser The target independent parser implementation to use for
+  /// parsing and lexing.
+  MCTargetAsmParser *createMCAsmParser(MCSubtargetInfo &STI,
+                                       MCAsmParser &Parser,
+                                       const MCInstrInfo &MII,
+                                       const MCTargetOptions &Options) const {
+    if (!MCAsmParserCtorFn)
+      return nullptr;
+    return MCAsmParserCtorFn(STI, Parser, MII, Options);
+  }
+
+  /// createAsmPrinter - Create a target specific assembly printer pass.  This
+  /// takes ownership of the MCStreamer object.
+  AsmPrinter *createAsmPrinter(TargetMachine &TM,
+                               std::unique_ptr<MCStreamer> &&Streamer) const {
+    if (!AsmPrinterCtorFn)
+      return nullptr;
+    return AsmPrinterCtorFn(TM, std::move(Streamer));
+  }
 
-    static void RegisterMachOStreamer(Target &T,
-                                      Target::MachOStreamerCtorTy Fn) {
-      T.MachOStreamerCtorFn = Fn;
-    }
+  MCDisassembler *createMCDisassembler(const MCSubtargetInfo &STI,
+                                       MCContext &Ctx) const {
+    if (!MCDisassemblerCtorFn)
+      return nullptr;
+    return MCDisassemblerCtorFn(*this, STI, Ctx);
+  }
+
+  MCInstPrinter *createMCInstPrinter(const Triple &T, unsigned SyntaxVariant,
+                                     const MCAsmInfo &MAI,
+                                     const MCInstrInfo &MII,
+                                     const MCRegisterInfo &MRI) const {
+    if (!MCInstPrinterCtorFn)
+      return nullptr;
+    return MCInstPrinterCtorFn(T, SyntaxVariant, MAI, MII, MRI);
+  }
+
+  /// createMCCodeEmitter - Create a target specific code emitter.
+  MCCodeEmitter *createMCCodeEmitter(const MCInstrInfo &II,
+                                     const MCRegisterInfo &MRI,
+                                     MCContext &Ctx) const {
+    if (!MCCodeEmitterCtorFn)
+      return nullptr;
+    return MCCodeEmitterCtorFn(II, MRI, Ctx);
+  }
 
-    static void RegisterELFStreamer(Target &T, Target::ELFStreamerCtorTy Fn) {
-      T.ELFStreamerCtorFn = Fn;
-    }
+  /// Create a target specific MCStreamer.
+  ///
+  /// \param T The target triple.
+  /// \param Ctx The target context.
+  /// \param TAB The target assembler backend object. Takes ownership.
+  /// \param OS The stream object.
+  /// \param Emitter The target independent assembler object.Takes ownership.
+  /// \param RelaxAll Relax all fixups?
+  MCStreamer *createMCObjectStreamer(const Triple &T, MCContext &Ctx,
+                                     MCAsmBackend &TAB, raw_pwrite_stream &OS,
+                                     MCCodeEmitter *Emitter,
+                                     const MCSubtargetInfo &STI, bool RelaxAll,
+                                     bool DWARFMustBeAtTheEnd) const {
+    MCStreamer *S;
+    switch (T.getObjectFormat()) {
+    default:
+      llvm_unreachable("Unknown object format");
+    case Triple::COFF:
+      assert(T.isOSWindows() && "only Windows COFF is supported");
+      S = COFFStreamerCtorFn(Ctx, TAB, OS, Emitter, RelaxAll);
+      break;
+    case Triple::MachO:
+      if (MachOStreamerCtorFn)
+        S = MachOStreamerCtorFn(Ctx, TAB, OS, Emitter, RelaxAll,
+                                DWARFMustBeAtTheEnd);
+      else
+        S = createMachOStreamer(Ctx, TAB, OS, Emitter, RelaxAll,
+                                DWARFMustBeAtTheEnd);
+      break;
+    case Triple::ELF:
+      if (ELFStreamerCtorFn)
+        S = ELFStreamerCtorFn(T, Ctx, TAB, OS, Emitter, RelaxAll);
+      else
+        S = createELFStreamer(Ctx, TAB, OS, Emitter, RelaxAll);
+      break;
+    }
+    if (ObjectTargetStreamerCtorFn)
+      ObjectTargetStreamerCtorFn(*S, STI);
+    return S;
+  }
 
-    static void
-    RegisterNullTargetStreamer(Target &T, Target::NullTargetStreamerCtorTy Fn) {
-      T.NullTargetStreamerCtorFn = Fn;
-    }
+  MCStreamer *createAsmStreamer(MCContext &Ctx,
+                                std::unique_ptr<formatted_raw_ostream> OS,
+                                bool IsVerboseAsm, bool UseDwarfDirectory,
+                                MCInstPrinter *InstPrint, MCCodeEmitter *CE,
+                                MCAsmBackend *TAB, bool ShowInst) const {
+    formatted_raw_ostream &OSRef = *OS;
+    MCStreamer *S = llvm::createAsmStreamer(Ctx, std::move(OS), IsVerboseAsm,
+                                            UseDwarfDirectory, InstPrint, CE,
+                                            TAB, ShowInst);
+    createAsmTargetStreamer(*S, OSRef, InstPrint, IsVerboseAsm);
+    return S;
+  }
+
+  MCTargetStreamer *createAsmTargetStreamer(MCStreamer &S,
+                                            formatted_raw_ostream &OS,
+                                            MCInstPrinter *InstPrint,
+                                            bool IsVerboseAsm) const {
+    if (AsmTargetStreamerCtorFn)
+      return AsmTargetStreamerCtorFn(S, OS, InstPrint, IsVerboseAsm);
+    return nullptr;
+  }
+
+  MCStreamer *createNullStreamer(MCContext &Ctx) const {
+    MCStreamer *S = llvm::createNullStreamer(Ctx);
+    createNullTargetStreamer(*S);
+    return S;
+  }
+
+  MCTargetStreamer *createNullTargetStreamer(MCStreamer &S) const {
+    if (NullTargetStreamerCtorFn)
+      return NullTargetStreamerCtorFn(S);
+    return nullptr;
+  }
+
+  /// createMCRelocationInfo - Create a target specific MCRelocationInfo.
+  ///
+  /// \param TT The target triple.
+  /// \param Ctx The target context.
+  MCRelocationInfo *createMCRelocationInfo(StringRef TT, MCContext &Ctx) const {
+    MCRelocationInfoCtorTy Fn = MCRelocationInfoCtorFn
+                                    ? MCRelocationInfoCtorFn
+                                    : llvm::createMCRelocationInfo;
+    return Fn(Triple(TT), Ctx);
+  }
+
+  /// createMCSymbolizer - Create a target specific MCSymbolizer.
+  ///
+  /// \param TT The target triple.
+  /// \param GetOpInfo The function to get the symbolic information for
+  /// operands.
+  /// \param SymbolLookUp The function to lookup a symbol name.
+  /// \param DisInfo The pointer to the block of symbolic information for above
+  /// call
+  /// back.
+  /// \param Ctx The target context.
+  /// \param RelInfo The relocation information for this target. Takes
+  /// ownership.
+  MCSymbolizer *
+  createMCSymbolizer(StringRef TT, LLVMOpInfoCallback GetOpInfo,
+                     LLVMSymbolLookupCallback SymbolLookUp, void *DisInfo,
+                     MCContext *Ctx,
+                     std::unique_ptr<MCRelocationInfo> &&RelInfo) const {
+    MCSymbolizerCtorTy Fn =
+        MCSymbolizerCtorFn ? MCSymbolizerCtorFn : llvm::createMCSymbolizer;
+    return Fn(Triple(TT), GetOpInfo, SymbolLookUp, DisInfo, Ctx,
+              std::move(RelInfo));
+  }
+
+  /// @}
+};
+
+/// TargetRegistry - Generic interface to target specific features.
+struct TargetRegistry {
+  // FIXME: Make this a namespace, probably just move all the Register*
+  // functions into Target (currently they all just set members on the Target
+  // anyway, and Target friends this class so those functions can...
+  // function).
+  TargetRegistry() = delete;
+
+  class iterator
+      : public std::iterator<std::forward_iterator_tag, Target, ptrdiff_t> {
+    const Target *Current;
+    explicit iterator(Target *T) : Current(T) {}
+    friend struct TargetRegistry;
 
-    static void RegisterAsmTargetStreamer(Target &T,
-                                          Target::AsmTargetStreamerCtorTy Fn) {
-      T.AsmTargetStreamerCtorFn = Fn;
-    }
+  public:
+    iterator() : Current(nullptr) {}
 
-    static void
-    RegisterObjectTargetStreamer(Target &T,
-                                 Target::ObjectTargetStreamerCtorTy Fn) {
-      T.ObjectTargetStreamerCtorFn = Fn;
-    }
+    bool operator==(const iterator &x) const { return Current == x.Current; }
+    bool operator!=(const iterator &x) const { return !operator==(x); }
 
-    /// RegisterMCRelocationInfo - Register an MCRelocationInfo
-    /// implementation for the given target.
-    ///
-    /// Clients are responsible for ensuring that registration doesn't occur
-    /// while another thread is attempting to access the registry. Typically
-    /// this is done by initializing all targets at program startup.
-    ///
-    /// @param T - The target being registered.
-    /// @param Fn - A function to construct an MCRelocationInfo for the target.
-    static void RegisterMCRelocationInfo(Target &T,
-                                         Target::MCRelocationInfoCtorTy Fn) {
-      T.MCRelocationInfoCtorFn = Fn;
+    // Iterator traversal: forward iteration only
+    iterator &operator++() { // Preincrement
+      assert(Current && "Cannot increment end iterator!");
+      Current = Current->getNext();
+      return *this;
     }
-
-    /// RegisterMCSymbolizer - Register an MCSymbolizer
-    /// implementation for the given target.
-    ///
-    /// Clients are responsible for ensuring that registration doesn't occur
-    /// while another thread is attempting to access the registry. Typically
-    /// this is done by initializing all targets at program startup.
-    ///
-    /// @param T - The target being registered.
-    /// @param Fn - A function to construct an MCSymbolizer for the target.
-    static void RegisterMCSymbolizer(Target &T,
-                                     Target::MCSymbolizerCtorTy Fn) {
-      T.MCSymbolizerCtorFn = Fn;
+    iterator operator++(int) { // Postincrement
+      iterator tmp = *this;
+      ++*this;
+      return tmp;
     }
 
-    // @LOCALMOD-START
-    static void RegisterMCNaClExpander(Target &T,
-                                       Target::MCNaClExpanderCtorTy Fn) {
-      T.MCNaClExpanderCtorFn = Fn;
+    const Target &operator*() const {
+      assert(Current && "Cannot dereference end iterator!");
+      return *Current;
     }
-    // @LOCALMOD-END
 
-    /// @}
+    const Target *operator->() const { return &operator*(); }
   };
 
+  /// printRegisteredTargetsForVersion - Print the registered targets
+  /// appropriately for inclusion in a tool's version output.
+  static void printRegisteredTargetsForVersion();
+
+  /// @name Registry Access
+  /// @{
 
-  //===--------------------------------------------------------------------===//
+  static iterator_range<iterator> targets();
 
-  /// RegisterTarget - Helper template for registering a target, for use in the
-  /// target's initialization function. Usage:
+  /// lookupTarget - Lookup a target based on a target triple.
   ///
+  /// \param Triple - The triple to use for finding a target.
+  /// \param Error - On failure, an error string describing why no target was
+  /// found.
+  static const Target *lookupTarget(const std::string &Triple,
+                                    std::string &Error);
+
+  /// lookupTarget - Lookup a target based on an architecture name
+  /// and a target triple.  If the architecture name is non-empty,
+  /// then the lookup is done by architecture.  Otherwise, the target
+  /// triple is used.
   ///
-  /// Target TheFooTarget; // The global target instance.
+  /// \param ArchName - The architecture to use for finding a target.
+  /// \param TheTriple - The triple to use for finding a target.  The
+  /// triple is updated with canonical architecture name if a lookup
+  /// by architecture is done.
+  /// \param Error - On failure, an error string describing why no target was
+  /// found.
+  static const Target *lookupTarget(const std::string &ArchName,
+                                    Triple &TheTriple, std::string &Error);
+
+  /// @}
+  /// @name Target Registration
+  /// @{
+
+  /// RegisterTarget - Register the given target. Attempts to register a
+  /// target which has already been registered will be ignored.
   ///
-  /// extern "C" void LLVMInitializeFooTargetInfo() {
-  ///   RegisterTarget<Triple::foo> X(TheFooTarget, "foo", "Foo description");
-  /// }
-  template<Triple::ArchType TargetArchType = Triple::UnknownArch,
-           bool HasJIT = false>
-  struct RegisterTarget {
-    RegisterTarget(Target &T, const char *Name, const char *Desc) {
-      TargetRegistry::RegisterTarget(T, Name, Desc, &getArchMatch, HasJIT);
-    }
-
-    static bool getArchMatch(Triple::ArchType Arch) {
-      return Arch == TargetArchType;
-    }
-  };
-
-  /// RegisterMCAsmInfo - Helper template for registering a target assembly info
-  /// implementation.  This invokes the static "Create" method on the class to
-  /// actually do the construction.  Usage:
+  /// Clients are responsible for ensuring that registration doesn't occur
+  /// while another thread is attempting to access the registry. Typically
+  /// this is done by initializing all targets at program startup.
   ///
-  /// extern "C" void LLVMInitializeFooTarget() {
-  ///   extern Target TheFooTarget;
-  ///   RegisterMCAsmInfo<FooMCAsmInfo> X(TheFooTarget);
-  /// }
-  template<class MCAsmInfoImpl>
-  struct RegisterMCAsmInfo {
-    RegisterMCAsmInfo(Target &T) {
-      TargetRegistry::RegisterMCAsmInfo(T, &Allocator);
-    }
-  private:
-    static MCAsmInfo *Allocator(const MCRegisterInfo &/*MRI*/, StringRef TT) {
-      return new MCAsmInfoImpl(TT);
-    }
-
-  };
-
-  /// RegisterMCAsmInfoFn - Helper template for registering a target assembly info
-  /// implementation.  This invokes the specified function to do the
-  /// construction.  Usage:
+  /// @param T - The target being registered.
+  /// @param Name - The target name. This should be a static string.
+  /// @param ShortDesc - A short target description. This should be a static
+  /// string.
+  /// @param ArchMatchFn - The arch match checking function for this target.
+  /// @param HasJIT - Whether the target supports JIT code
+  /// generation.
+  static void RegisterTarget(Target &T, const char *Name, const char *ShortDesc,
+                             Target::ArchMatchFnTy ArchMatchFn,
+                             bool HasJIT = false);
+
+  /// RegisterMCAsmInfo - Register a MCAsmInfo implementation for the
+  /// given target.
   ///
-  /// extern "C" void LLVMInitializeFooTarget() {
-  ///   extern Target TheFooTarget;
-  ///   RegisterMCAsmInfoFn X(TheFooTarget, TheFunction);
-  /// }
-  struct RegisterMCAsmInfoFn {
-    RegisterMCAsmInfoFn(Target &T, Target::MCAsmInfoCtorFnTy Fn) {
-      TargetRegistry::RegisterMCAsmInfo(T, Fn);
-    }
-  };
-
-  /// RegisterMCCodeGenInfo - Helper template for registering a target codegen info
-  /// implementation.  This invokes the static "Create" method on the class
-  /// to actually do the construction.  Usage:
+  /// Clients are responsible for ensuring that registration doesn't occur
+  /// while another thread is attempting to access the registry. Typically
+  /// this is done by initializing all targets at program startup.
   ///
-  /// extern "C" void LLVMInitializeFooTarget() {
-  ///   extern Target TheFooTarget;
-  ///   RegisterMCCodeGenInfo<FooMCCodeGenInfo> X(TheFooTarget);
-  /// }
-  template<class MCCodeGenInfoImpl>
-  struct RegisterMCCodeGenInfo {
-    RegisterMCCodeGenInfo(Target &T) {
-      TargetRegistry::RegisterMCCodeGenInfo(T, &Allocator);
-    }
-  private:
-    static MCCodeGenInfo *Allocator(StringRef /*TT*/, Reloc::Model /*RM*/,
-                                    CodeModel::Model /*CM*/,
-                                    CodeGenOpt::Level /*OL*/) {
-      return new MCCodeGenInfoImpl();
-    }
-  };
-
-  /// RegisterMCCodeGenInfoFn - Helper template for registering a target codegen
-  /// info implementation.  This invokes the specified function to do the
-  /// construction.  Usage:
+  /// @param T - The target being registered.
+  /// @param Fn - A function to construct a MCAsmInfo for the target.
+  static void RegisterMCAsmInfo(Target &T, Target::MCAsmInfoCtorFnTy Fn) {
+    T.MCAsmInfoCtorFn = Fn;
+  }
+
+  /// RegisterMCCodeGenInfo - Register a MCCodeGenInfo implementation for the
+  /// given target.
   ///
-  /// extern "C" void LLVMInitializeFooTarget() {
-  ///   extern Target TheFooTarget;
-  ///   RegisterMCCodeGenInfoFn X(TheFooTarget, TheFunction);
-  /// }
-  struct RegisterMCCodeGenInfoFn {
-    RegisterMCCodeGenInfoFn(Target &T, Target::MCCodeGenInfoCtorFnTy Fn) {
-      TargetRegistry::RegisterMCCodeGenInfo(T, Fn);
-    }
-  };
-
-  /// RegisterMCInstrInfo - Helper template for registering a target instruction
-  /// info implementation.  This invokes the static "Create" method on the class
-  /// to actually do the construction.  Usage:
+  /// Clients are responsible for ensuring that registration doesn't occur
+  /// while another thread is attempting to access the registry. Typically
+  /// this is done by initializing all targets at program startup.
   ///
-  /// extern "C" void LLVMInitializeFooTarget() {
-  ///   extern Target TheFooTarget;
-  ///   RegisterMCInstrInfo<FooMCInstrInfo> X(TheFooTarget);
-  /// }
-  template<class MCInstrInfoImpl>
-  struct RegisterMCInstrInfo {
-    RegisterMCInstrInfo(Target &T) {
-      TargetRegistry::RegisterMCInstrInfo(T, &Allocator);
-    }
-  private:
-    static MCInstrInfo *Allocator() {
-      return new MCInstrInfoImpl();
-    }
-  };
-
-  /// RegisterMCInstrInfoFn - Helper template for registering a target
-  /// instruction info implementation.  This invokes the specified function to
-  /// do the construction.  Usage:
+  /// @param T - The target being registered.
+  /// @param Fn - A function to construct a MCCodeGenInfo for the target.
+  static void RegisterMCCodeGenInfo(Target &T,
+                                    Target::MCCodeGenInfoCtorFnTy Fn) {
+    T.MCCodeGenInfoCtorFn = Fn;
+  }
+
+  /// RegisterMCInstrInfo - Register a MCInstrInfo implementation for the
+  /// given target.
   ///
-  /// extern "C" void LLVMInitializeFooTarget() {
-  ///   extern Target TheFooTarget;
-  ///   RegisterMCInstrInfoFn X(TheFooTarget, TheFunction);
-  /// }
-  struct RegisterMCInstrInfoFn {
-    RegisterMCInstrInfoFn(Target &T, Target::MCInstrInfoCtorFnTy Fn) {
-      TargetRegistry::RegisterMCInstrInfo(T, Fn);
-    }
-  };
-
-  /// RegisterMCInstrAnalysis - Helper template for registering a target
-  /// instruction analyzer implementation.  This invokes the static "Create"
-  /// method on the class to actually do the construction.  Usage:
+  /// Clients are responsible for ensuring that registration doesn't occur
+  /// while another thread is attempting to access the registry. Typically
+  /// this is done by initializing all targets at program startup.
   ///
-  /// extern "C" void LLVMInitializeFooTarget() {
-  ///   extern Target TheFooTarget;
-  ///   RegisterMCInstrAnalysis<FooMCInstrAnalysis> X(TheFooTarget);
-  /// }
-  template<class MCInstrAnalysisImpl>
-  struct RegisterMCInstrAnalysis {
-    RegisterMCInstrAnalysis(Target &T) {
-      TargetRegistry::RegisterMCInstrAnalysis(T, &Allocator);
-    }
-  private:
-    static MCInstrAnalysis *Allocator(const MCInstrInfo *Info) {
-      return new MCInstrAnalysisImpl(Info);
-    }
-  };
-
-  /// RegisterMCInstrAnalysisFn - Helper template for registering a target
-  /// instruction analyzer implementation.  This invokes the specified function
-  /// to do the construction.  Usage:
+  /// @param T - The target being registered.
+  /// @param Fn - A function to construct a MCInstrInfo for the target.
+  static void RegisterMCInstrInfo(Target &T, Target::MCInstrInfoCtorFnTy Fn) {
+    T.MCInstrInfoCtorFn = Fn;
+  }
+
+  /// RegisterMCInstrAnalysis - Register a MCInstrAnalysis implementation for
+  /// the given target.
+  static void RegisterMCInstrAnalysis(Target &T,
+                                      Target::MCInstrAnalysisCtorFnTy Fn) {
+    T.MCInstrAnalysisCtorFn = Fn;
+  }
+
+  /// RegisterMCRegInfo - Register a MCRegisterInfo implementation for the
+  /// given target.
   ///
-  /// extern "C" void LLVMInitializeFooTarget() {
-  ///   extern Target TheFooTarget;
-  ///   RegisterMCInstrAnalysisFn X(TheFooTarget, TheFunction);
-  /// }
-  struct RegisterMCInstrAnalysisFn {
-    RegisterMCInstrAnalysisFn(Target &T, Target::MCInstrAnalysisCtorFnTy Fn) {
-      TargetRegistry::RegisterMCInstrAnalysis(T, Fn);
-    }
-  };
-
-  /// RegisterMCRegInfo - Helper template for registering a target register info
-  /// implementation.  This invokes the static "Create" method on the class to
-  /// actually do the construction.  Usage:
+  /// Clients are responsible for ensuring that registration doesn't occur
+  /// while another thread is attempting to access the registry. Typically
+  /// this is done by initializing all targets at program startup.
   ///
-  /// extern "C" void LLVMInitializeFooTarget() {
-  ///   extern Target TheFooTarget;
-  ///   RegisterMCRegInfo<FooMCRegInfo> X(TheFooTarget);
-  /// }
-  template<class MCRegisterInfoImpl>
-  struct RegisterMCRegInfo {
-    RegisterMCRegInfo(Target &T) {
-      TargetRegistry::RegisterMCRegInfo(T, &Allocator);
-    }
-  private:
-    static MCRegisterInfo *Allocator(StringRef /*TT*/) {
-      return new MCRegisterInfoImpl();
-    }
-  };
-
-  /// RegisterMCRegInfoFn - Helper template for registering a target register
-  /// info implementation.  This invokes the specified function to do the
-  /// construction.  Usage:
+  /// @param T - The target being registered.
+  /// @param Fn - A function to construct a MCRegisterInfo for the target.
+  static void RegisterMCRegInfo(Target &T, Target::MCRegInfoCtorFnTy Fn) {
+    T.MCRegInfoCtorFn = Fn;
+  }
+
+  /// RegisterMCSubtargetInfo - Register a MCSubtargetInfo implementation for
+  /// the given target.
   ///
-  /// extern "C" void LLVMInitializeFooTarget() {
-  ///   extern Target TheFooTarget;
-  ///   RegisterMCRegInfoFn X(TheFooTarget, TheFunction);
-  /// }
-  struct RegisterMCRegInfoFn {
-    RegisterMCRegInfoFn(Target &T, Target::MCRegInfoCtorFnTy Fn) {
-      TargetRegistry::RegisterMCRegInfo(T, Fn);
-    }
-  };
-
-  /// RegisterMCSubtargetInfo - Helper template for registering a target
-  /// subtarget info implementation.  This invokes the static "Create" method
-  /// on the class to actually do the construction.  Usage:
+  /// Clients are responsible for ensuring that registration doesn't occur
+  /// while another thread is attempting to access the registry. Typically
+  /// this is done by initializing all targets at program startup.
   ///
-  /// extern "C" void LLVMInitializeFooTarget() {
-  ///   extern Target TheFooTarget;
-  ///   RegisterMCSubtargetInfo<FooMCSubtargetInfo> X(TheFooTarget);
-  /// }
-  template<class MCSubtargetInfoImpl>
-  struct RegisterMCSubtargetInfo {
-    RegisterMCSubtargetInfo(Target &T) {
-      TargetRegistry::RegisterMCSubtargetInfo(T, &Allocator);
-    }
-  private:
-    static MCSubtargetInfo *Allocator(StringRef /*TT*/, StringRef /*CPU*/,
-                                      StringRef /*FS*/) {
-      return new MCSubtargetInfoImpl();
-    }
-  };
-
-  /// RegisterMCSubtargetInfoFn - Helper template for registering a target
-  /// subtarget info implementation.  This invokes the specified function to
-  /// do the construction.  Usage:
+  /// @param T - The target being registered.
+  /// @param Fn - A function to construct a MCSubtargetInfo for the target.
+  static void RegisterMCSubtargetInfo(Target &T,
+                                      Target::MCSubtargetInfoCtorFnTy Fn) {
+    T.MCSubtargetInfoCtorFn = Fn;
+  }
+
+  /// RegisterTargetMachine - Register a TargetMachine implementation for the
+  /// given target.
   ///
-  /// extern "C" void LLVMInitializeFooTarget() {
-  ///   extern Target TheFooTarget;
-  ///   RegisterMCSubtargetInfoFn X(TheFooTarget, TheFunction);
-  /// }
-  struct RegisterMCSubtargetInfoFn {
-    RegisterMCSubtargetInfoFn(Target &T, Target::MCSubtargetInfoCtorFnTy Fn) {
-      TargetRegistry::RegisterMCSubtargetInfo(T, Fn);
-    }
-  };
-
-  /// RegisterTargetMachine - Helper template for registering a target machine
-  /// implementation, for use in the target machine initialization
-  /// function. Usage:
+  /// Clients are responsible for ensuring that registration doesn't occur
+  /// while another thread is attempting to access the registry. Typically
+  /// this is done by initializing all targets at program startup.
   ///
-  /// extern "C" void LLVMInitializeFooTarget() {
-  ///   extern Target TheFooTarget;
-  ///   RegisterTargetMachine<FooTargetMachine> X(TheFooTarget);
-  /// }
-  template<class TargetMachineImpl>
-  struct RegisterTargetMachine {
-    RegisterTargetMachine(Target &T) {
-      TargetRegistry::RegisterTargetMachine(T, &Allocator);
-    }
-
-  private:
-    static TargetMachine *Allocator(const Target &T, StringRef TT,
-                                    StringRef CPU, StringRef FS,
-                                    const TargetOptions &Options,
-                                    Reloc::Model RM,
-                                    CodeModel::Model CM,
-                                    CodeGenOpt::Level OL) {
-      return new TargetMachineImpl(T, TT, CPU, FS, Options, RM, CM, OL);
-    }
-  };
-
-  /// RegisterMCAsmBackend - Helper template for registering a target specific
-  /// assembler backend. Usage:
+  /// @param T - The target being registered.
+  /// @param Fn - A function to construct a TargetMachine for the target.
+  static void RegisterTargetMachine(Target &T, Target::TargetMachineCtorTy Fn) {
+    T.TargetMachineCtorFn = Fn;
+  }
+
+  /// RegisterMCAsmBackend - Register a MCAsmBackend implementation for the
+  /// given target.
   ///
-  /// extern "C" void LLVMInitializeFooMCAsmBackend() {
-  ///   extern Target TheFooTarget;
-  ///   RegisterMCAsmBackend<FooAsmLexer> X(TheFooTarget);
-  /// }
-  template<class MCAsmBackendImpl>
-  struct RegisterMCAsmBackend {
-    RegisterMCAsmBackend(Target &T) {
-      TargetRegistry::RegisterMCAsmBackend(T, &Allocator);
-    }
-
-  private:
-    static MCAsmBackend *Allocator(const Target &T,
-                                   const MCRegisterInfo &MRI,
-                                   StringRef Triple, StringRef CPU) {
-      return new MCAsmBackendImpl(T, MRI, Triple, CPU);
-    }
-  };
-
-  /// RegisterMCAsmParser - Helper template for registering a target specific
-  /// assembly parser, for use in the target machine initialization
-  /// function. Usage:
+  /// Clients are responsible for ensuring that registration doesn't occur
+  /// while another thread is attempting to access the registry. Typically
+  /// this is done by initializing all targets at program startup.
   ///
-  /// extern "C" void LLVMInitializeFooMCAsmParser() {
-  ///   extern Target TheFooTarget;
-  ///   RegisterMCAsmParser<FooAsmParser> X(TheFooTarget);
-  /// }
-  template<class MCAsmParserImpl>
-  struct RegisterMCAsmParser {
-    RegisterMCAsmParser(Target &T) {
-      TargetRegistry::RegisterMCAsmParser(T, &Allocator);
-    }
-
-  private:
-    static MCTargetAsmParser *Allocator(MCSubtargetInfo &STI, MCAsmParser &P,
-                                        const MCInstrInfo &MII,
-                                        const MCTargetOptions &Options) {
-      return new MCAsmParserImpl(STI, P, MII, Options);
-    }
-  };
-
-  /// RegisterAsmPrinter - Helper template for registering a target specific
-  /// assembly printer, for use in the target machine initialization
-  /// function. Usage:
+  /// @param T - The target being registered.
+  /// @param Fn - A function to construct an AsmBackend for the target.
+  static void RegisterMCAsmBackend(Target &T, Target::MCAsmBackendCtorTy Fn) {
+    T.MCAsmBackendCtorFn = Fn;
+  }
+
+  /// RegisterMCAsmParser - Register a MCTargetAsmParser implementation for
+  /// the given target.
   ///
-  /// extern "C" void LLVMInitializeFooAsmPrinter() {
-  ///   extern Target TheFooTarget;
-  ///   RegisterAsmPrinter<FooAsmPrinter> X(TheFooTarget);
-  /// }
-  template<class AsmPrinterImpl>
-  struct RegisterAsmPrinter {
-    RegisterAsmPrinter(Target &T) {
-      TargetRegistry::RegisterAsmPrinter(T, &Allocator);
-    }
-
-  private:
-    static AsmPrinter *Allocator(TargetMachine &TM,
-                                 std::unique_ptr<MCStreamer> &&Streamer) {
-      return new AsmPrinterImpl(TM, std::move(Streamer));
-    }
-  };
-
-  /// RegisterMCCodeEmitter - Helper template for registering a target specific
-  /// machine code emitter, for use in the target initialization
-  /// function. Usage:
+  /// Clients are responsible for ensuring that registration doesn't occur
+  /// while another thread is attempting to access the registry. Typically
+  /// this is done by initializing all targets at program startup.
   ///
-  /// extern "C" void LLVMInitializeFooMCCodeEmitter() {
-  ///   extern Target TheFooTarget;
-  ///   RegisterMCCodeEmitter<FooCodeEmitter> X(TheFooTarget);
-  /// }
-  template<class MCCodeEmitterImpl>
-  struct RegisterMCCodeEmitter {
-    RegisterMCCodeEmitter(Target &T) {
-      TargetRegistry::RegisterMCCodeEmitter(T, &Allocator);
-    }
-
-  private:
-    static MCCodeEmitter *Allocator(const MCInstrInfo & /*II*/,
-                                    const MCRegisterInfo & /*MRI*/,
-                                    MCContext & /*Ctx*/) {
-      return new MCCodeEmitterImpl();
-    }
-  };
-
+  /// @param T - The target being registered.
+  /// @param Fn - A function to construct an MCTargetAsmParser for the target.
+  static void RegisterMCAsmParser(Target &T, Target::MCAsmParserCtorTy Fn) {
+    T.MCAsmParserCtorFn = Fn;
+  }
+
+  /// RegisterAsmPrinter - Register an AsmPrinter implementation for the given
+  /// target.
+  ///
+  /// Clients are responsible for ensuring that registration doesn't occur
+  /// while another thread is attempting to access the registry. Typically
+  /// this is done by initializing all targets at program startup.
+  ///
+  /// @param T - The target being registered.
+  /// @param Fn - A function to construct an AsmPrinter for the target.
+  static void RegisterAsmPrinter(Target &T, Target::AsmPrinterCtorTy Fn) {
+    T.AsmPrinterCtorFn = Fn;
+  }
+
+  /// RegisterMCDisassembler - Register a MCDisassembler implementation for
+  /// the given target.
+  ///
+  /// Clients are responsible for ensuring that registration doesn't occur
+  /// while another thread is attempting to access the registry. Typically
+  /// this is done by initializing all targets at program startup.
+  ///
+  /// @param T - The target being registered.
+  /// @param Fn - A function to construct an MCDisassembler for the target.
+  static void RegisterMCDisassembler(Target &T,
+                                     Target::MCDisassemblerCtorTy Fn) {
+    T.MCDisassemblerCtorFn = Fn;
+  }
+
+  /// RegisterMCInstPrinter - Register a MCInstPrinter implementation for the
+  /// given target.
+  ///
+  /// Clients are responsible for ensuring that registration doesn't occur
+  /// while another thread is attempting to access the registry. Typically
+  /// this is done by initializing all targets at program startup.
+  ///
+  /// @param T - The target being registered.
+  /// @param Fn - A function to construct an MCInstPrinter for the target.
+  static void RegisterMCInstPrinter(Target &T, Target::MCInstPrinterCtorTy Fn) {
+    T.MCInstPrinterCtorFn = Fn;
+  }
+
+  /// RegisterMCCodeEmitter - Register a MCCodeEmitter implementation for the
+  /// given target.
+  ///
+  /// Clients are responsible for ensuring that registration doesn't occur
+  /// while another thread is attempting to access the registry. Typically
+  /// this is done by initializing all targets at program startup.
+  ///
+  /// @param T - The target being registered.
+  /// @param Fn - A function to construct an MCCodeEmitter for the target.
+  static void RegisterMCCodeEmitter(Target &T, Target::MCCodeEmitterCtorTy Fn) {
+    T.MCCodeEmitterCtorFn = Fn;
+  }
+
+  static void RegisterCOFFStreamer(Target &T, Target::COFFStreamerCtorTy Fn) {
+    T.COFFStreamerCtorFn = Fn;
+  }
+
+  static void RegisterMachOStreamer(Target &T, Target::MachOStreamerCtorTy Fn) {
+    T.MachOStreamerCtorFn = Fn;
+  }
+
+  static void RegisterELFStreamer(Target &T, Target::ELFStreamerCtorTy Fn) {
+    T.ELFStreamerCtorFn = Fn;
+  }
+
+  static void RegisterNullTargetStreamer(Target &T,
+                                         Target::NullTargetStreamerCtorTy Fn) {
+    T.NullTargetStreamerCtorFn = Fn;
+  }
+
+  static void RegisterAsmTargetStreamer(Target &T,
+                                        Target::AsmTargetStreamerCtorTy Fn) {
+    T.AsmTargetStreamerCtorFn = Fn;
+  }
+
+  static void
+  RegisterObjectTargetStreamer(Target &T,
+                               Target::ObjectTargetStreamerCtorTy Fn) {
+    T.ObjectTargetStreamerCtorFn = Fn;
+  }
+
+  /// RegisterMCRelocationInfo - Register an MCRelocationInfo
+  /// implementation for the given target.
+  ///
+  /// Clients are responsible for ensuring that registration doesn't occur
+  /// while another thread is attempting to access the registry. Typically
+  /// this is done by initializing all targets at program startup.
+  ///
+  /// @param T - The target being registered.
+  /// @param Fn - A function to construct an MCRelocationInfo for the target.
+  static void RegisterMCRelocationInfo(Target &T,
+                                       Target::MCRelocationInfoCtorTy Fn) {
+    T.MCRelocationInfoCtorFn = Fn;
+  }
+
+  /// RegisterMCSymbolizer - Register an MCSymbolizer
+  /// implementation for the given target.
+  ///
+  /// Clients are responsible for ensuring that registration doesn't occur
+  /// while another thread is attempting to access the registry. Typically
+  /// this is done by initializing all targets at program startup.
+  ///
+  /// @param T - The target being registered.
+  /// @param Fn - A function to construct an MCSymbolizer for the target.
+  static void RegisterMCSymbolizer(Target &T, Target::MCSymbolizerCtorTy Fn) {
+    T.MCSymbolizerCtorFn = Fn;
+  }
+
+  /// @}
+};
+
+//===--------------------------------------------------------------------===//
+
+/// RegisterTarget - Helper template for registering a target, for use in the
+/// target's initialization function. Usage:
+///
+///
+/// Target TheFooTarget; // The global target instance.
+///
+/// extern "C" void LLVMInitializeFooTargetInfo() {
+///   RegisterTarget<Triple::foo> X(TheFooTarget, "foo", "Foo description");
+/// }
+template <Triple::ArchType TargetArchType = Triple::UnknownArch,
+          bool HasJIT = false>
+struct RegisterTarget {
+  RegisterTarget(Target &T, const char *Name, const char *Desc) {
+    TargetRegistry::RegisterTarget(T, Name, Desc, &getArchMatch, HasJIT);
+  }
+
+  static bool getArchMatch(Triple::ArchType Arch) {
+    return Arch == TargetArchType;
+  }
+};
+
+/// RegisterMCAsmInfo - Helper template for registering a target assembly info
+/// implementation.  This invokes the static "Create" method on the class to
+/// actually do the construction.  Usage:
+///
+/// extern "C" void LLVMInitializeFooTarget() {
+///   extern Target TheFooTarget;
+///   RegisterMCAsmInfo<FooMCAsmInfo> X(TheFooTarget);
+/// }
+template <class MCAsmInfoImpl> struct RegisterMCAsmInfo {
+  RegisterMCAsmInfo(Target &T) {
+    TargetRegistry::RegisterMCAsmInfo(T, &Allocator);
+  }
+
+private:
+  static MCAsmInfo *Allocator(const MCRegisterInfo & /*MRI*/,
+                              const Triple &TT) {
+    return new MCAsmInfoImpl(TT);
+  }
+};
+
+/// RegisterMCAsmInfoFn - Helper template for registering a target assembly info
+/// implementation.  This invokes the specified function to do the
+/// construction.  Usage:
+///
+/// extern "C" void LLVMInitializeFooTarget() {
+///   extern Target TheFooTarget;
+///   RegisterMCAsmInfoFn X(TheFooTarget, TheFunction);
+/// }
+struct RegisterMCAsmInfoFn {
+  RegisterMCAsmInfoFn(Target &T, Target::MCAsmInfoCtorFnTy Fn) {
+    TargetRegistry::RegisterMCAsmInfo(T, Fn);
+  }
+};
+
+/// RegisterMCCodeGenInfo - Helper template for registering a target codegen
+/// info
+/// implementation.  This invokes the static "Create" method on the class
+/// to actually do the construction.  Usage:
+///
+/// extern "C" void LLVMInitializeFooTarget() {
+///   extern Target TheFooTarget;
+///   RegisterMCCodeGenInfo<FooMCCodeGenInfo> X(TheFooTarget);
+/// }
+template <class MCCodeGenInfoImpl> struct RegisterMCCodeGenInfo {
+  RegisterMCCodeGenInfo(Target &T) {
+    TargetRegistry::RegisterMCCodeGenInfo(T, &Allocator);
+  }
+
+private:
+  static MCCodeGenInfo *Allocator(const Triple & /*TT*/, Reloc::Model /*RM*/,
+                                  CodeModel::Model /*CM*/,
+                                  CodeGenOpt::Level /*OL*/) {
+    return new MCCodeGenInfoImpl();
+  }
+};
+
+/// RegisterMCCodeGenInfoFn - Helper template for registering a target codegen
+/// info implementation.  This invokes the specified function to do the
+/// construction.  Usage:
+///
+/// extern "C" void LLVMInitializeFooTarget() {
+///   extern Target TheFooTarget;
+///   RegisterMCCodeGenInfoFn X(TheFooTarget, TheFunction);
+/// }
+struct RegisterMCCodeGenInfoFn {
+  RegisterMCCodeGenInfoFn(Target &T, Target::MCCodeGenInfoCtorFnTy Fn) {
+    TargetRegistry::RegisterMCCodeGenInfo(T, Fn);
+  }
+};
+
+/// RegisterMCInstrInfo - Helper template for registering a target instruction
+/// info implementation.  This invokes the static "Create" method on the class
+/// to actually do the construction.  Usage:
+///
+/// extern "C" void LLVMInitializeFooTarget() {
+///   extern Target TheFooTarget;
+///   RegisterMCInstrInfo<FooMCInstrInfo> X(TheFooTarget);
+/// }
+template <class MCInstrInfoImpl> struct RegisterMCInstrInfo {
+  RegisterMCInstrInfo(Target &T) {
+    TargetRegistry::RegisterMCInstrInfo(T, &Allocator);
+  }
+
+private:
+  static MCInstrInfo *Allocator() { return new MCInstrInfoImpl(); }
+};
+
+/// RegisterMCInstrInfoFn - Helper template for registering a target
+/// instruction info implementation.  This invokes the specified function to
+/// do the construction.  Usage:
+///
+/// extern "C" void LLVMInitializeFooTarget() {
+///   extern Target TheFooTarget;
+///   RegisterMCInstrInfoFn X(TheFooTarget, TheFunction);
+/// }
+struct RegisterMCInstrInfoFn {
+  RegisterMCInstrInfoFn(Target &T, Target::MCInstrInfoCtorFnTy Fn) {
+    TargetRegistry::RegisterMCInstrInfo(T, Fn);
+  }
+};
+
+/// RegisterMCInstrAnalysis - Helper template for registering a target
+/// instruction analyzer implementation.  This invokes the static "Create"
+/// method on the class to actually do the construction.  Usage:
+///
+/// extern "C" void LLVMInitializeFooTarget() {
+///   extern Target TheFooTarget;
+///   RegisterMCInstrAnalysis<FooMCInstrAnalysis> X(TheFooTarget);
+/// }
+template <class MCInstrAnalysisImpl> struct RegisterMCInstrAnalysis {
+  RegisterMCInstrAnalysis(Target &T) {
+    TargetRegistry::RegisterMCInstrAnalysis(T, &Allocator);
+  }
+
+private:
+  static MCInstrAnalysis *Allocator(const MCInstrInfo *Info) {
+    return new MCInstrAnalysisImpl(Info);
+  }
+};
+
+/// RegisterMCInstrAnalysisFn - Helper template for registering a target
+/// instruction analyzer implementation.  This invokes the specified function
+/// to do the construction.  Usage:
+///
+/// extern "C" void LLVMInitializeFooTarget() {
+///   extern Target TheFooTarget;
+///   RegisterMCInstrAnalysisFn X(TheFooTarget, TheFunction);
+/// }
+struct RegisterMCInstrAnalysisFn {
+  RegisterMCInstrAnalysisFn(Target &T, Target::MCInstrAnalysisCtorFnTy Fn) {
+    TargetRegistry::RegisterMCInstrAnalysis(T, Fn);
+  }
+};
+
+/// RegisterMCRegInfo - Helper template for registering a target register info
+/// implementation.  This invokes the static "Create" method on the class to
+/// actually do the construction.  Usage:
+///
+/// extern "C" void LLVMInitializeFooTarget() {
+///   extern Target TheFooTarget;
+///   RegisterMCRegInfo<FooMCRegInfo> X(TheFooTarget);
+/// }
+template <class MCRegisterInfoImpl> struct RegisterMCRegInfo {
+  RegisterMCRegInfo(Target &T) {
+    TargetRegistry::RegisterMCRegInfo(T, &Allocator);
+  }
+
+private:
+  static MCRegisterInfo *Allocator(const Triple & /*TT*/) {
+    return new MCRegisterInfoImpl();
+  }
+};
+
+/// RegisterMCRegInfoFn - Helper template for registering a target register
+/// info implementation.  This invokes the specified function to do the
+/// construction.  Usage:
+///
+/// extern "C" void LLVMInitializeFooTarget() {
+///   extern Target TheFooTarget;
+///   RegisterMCRegInfoFn X(TheFooTarget, TheFunction);
+/// }
+struct RegisterMCRegInfoFn {
+  RegisterMCRegInfoFn(Target &T, Target::MCRegInfoCtorFnTy Fn) {
+    TargetRegistry::RegisterMCRegInfo(T, Fn);
+  }
+};
+
+/// RegisterMCSubtargetInfo - Helper template for registering a target
+/// subtarget info implementation.  This invokes the static "Create" method
+/// on the class to actually do the construction.  Usage:
+///
+/// extern "C" void LLVMInitializeFooTarget() {
+///   extern Target TheFooTarget;
+///   RegisterMCSubtargetInfo<FooMCSubtargetInfo> X(TheFooTarget);
+/// }
+template <class MCSubtargetInfoImpl> struct RegisterMCSubtargetInfo {
+  RegisterMCSubtargetInfo(Target &T) {
+    TargetRegistry::RegisterMCSubtargetInfo(T, &Allocator);
+  }
+
+private:
+  static MCSubtargetInfo *Allocator(const Triple & /*TT*/, StringRef /*CPU*/,
+                                    StringRef /*FS*/) {
+    return new MCSubtargetInfoImpl();
+  }
+};
+
+/// RegisterMCSubtargetInfoFn - Helper template for registering a target
+/// subtarget info implementation.  This invokes the specified function to
+/// do the construction.  Usage:
+///
+/// extern "C" void LLVMInitializeFooTarget() {
+///   extern Target TheFooTarget;
+///   RegisterMCSubtargetInfoFn X(TheFooTarget, TheFunction);
+/// }
+struct RegisterMCSubtargetInfoFn {
+  RegisterMCSubtargetInfoFn(Target &T, Target::MCSubtargetInfoCtorFnTy Fn) {
+    TargetRegistry::RegisterMCSubtargetInfo(T, Fn);
+  }
+};
+
+/// RegisterTargetMachine - Helper template for registering a target machine
+/// implementation, for use in the target machine initialization
+/// function. Usage:
+///
+/// extern "C" void LLVMInitializeFooTarget() {
+///   extern Target TheFooTarget;
+///   RegisterTargetMachine<FooTargetMachine> X(TheFooTarget);
+/// }
+template <class TargetMachineImpl> struct RegisterTargetMachine {
+  RegisterTargetMachine(Target &T) {
+    TargetRegistry::RegisterTargetMachine(T, &Allocator);
+  }
+
+private:
+  static TargetMachine *Allocator(const Target &T, const Triple &TT,
+                                  StringRef CPU, StringRef FS,
+                                  const TargetOptions &Options, Reloc::Model RM,
+                                  CodeModel::Model CM, CodeGenOpt::Level OL) {
+    return new TargetMachineImpl(T, TT, CPU, FS, Options, RM, CM, OL);
+  }
+};
+
+/// RegisterMCAsmBackend - Helper template for registering a target specific
+/// assembler backend. Usage:
+///
+/// extern "C" void LLVMInitializeFooMCAsmBackend() {
+///   extern Target TheFooTarget;
+///   RegisterMCAsmBackend<FooAsmLexer> X(TheFooTarget);
+/// }
+template <class MCAsmBackendImpl> struct RegisterMCAsmBackend {
+  RegisterMCAsmBackend(Target &T) {
+    TargetRegistry::RegisterMCAsmBackend(T, &Allocator);
+  }
+
+private:
+  static MCAsmBackend *Allocator(const Target &T, const MCRegisterInfo &MRI,
+                                 const Triple &TheTriple, StringRef CPU) {
+    return new MCAsmBackendImpl(T, MRI, TheTriple, CPU);
+  }
+};
+
+/// RegisterMCAsmParser - Helper template for registering a target specific
+/// assembly parser, for use in the target machine initialization
+/// function. Usage:
+///
+/// extern "C" void LLVMInitializeFooMCAsmParser() {
+///   extern Target TheFooTarget;
+///   RegisterMCAsmParser<FooAsmParser> X(TheFooTarget);
+/// }
+template <class MCAsmParserImpl> struct RegisterMCAsmParser {
+  RegisterMCAsmParser(Target &T) {
+    TargetRegistry::RegisterMCAsmParser(T, &Allocator);
+  }
+
+private:
+  static MCTargetAsmParser *Allocator(MCSubtargetInfo &STI, MCAsmParser &P,
+                                      const MCInstrInfo &MII,
+                                      const MCTargetOptions &Options) {
+    return new MCAsmParserImpl(STI, P, MII, Options);
+  }
+};
+
+/// RegisterAsmPrinter - Helper template for registering a target specific
+/// assembly printer, for use in the target machine initialization
+/// function. Usage:
+///
+/// extern "C" void LLVMInitializeFooAsmPrinter() {
+///   extern Target TheFooTarget;
+///   RegisterAsmPrinter<FooAsmPrinter> X(TheFooTarget);
+/// }
+template <class AsmPrinterImpl> struct RegisterAsmPrinter {
+  RegisterAsmPrinter(Target &T) {
+    TargetRegistry::RegisterAsmPrinter(T, &Allocator);
+  }
+
+private:
+  static AsmPrinter *Allocator(TargetMachine &TM,
+                               std::unique_ptr<MCStreamer> &&Streamer) {
+    return new AsmPrinterImpl(TM, std::move(Streamer));
+  }
+};
+
+/// RegisterMCCodeEmitter - Helper template for registering a target specific
+/// machine code emitter, for use in the target initialization
+/// function. Usage:
+///
+/// extern "C" void LLVMInitializeFooMCCodeEmitter() {
+///   extern Target TheFooTarget;
+///   RegisterMCCodeEmitter<FooCodeEmitter> X(TheFooTarget);
+/// }
+template <class MCCodeEmitterImpl> struct RegisterMCCodeEmitter {
+  RegisterMCCodeEmitter(Target &T) {
+    TargetRegistry::RegisterMCCodeEmitter(T, &Allocator);
+  }
+
+private:
+  static MCCodeEmitter *Allocator(const MCInstrInfo & /*II*/,
+                                  const MCRegisterInfo & /*MRI*/,
+                                  MCContext & /*Ctx*/) {
+    return new MCCodeEmitterImpl();
+  }
+};
 }
 
 #endif
diff --git a/include/llvm/Support/YAMLParser.h b/include/llvm/Support/YAMLParser.h
index 3d423293b27..0fbb7d2e6c7 100644
--- a/include/llvm/Support/YAMLParser.h
+++ b/include/llvm/Support/YAMLParser.h
@@ -76,9 +76,9 @@ std::string escape(StringRef Input);
 class Stream {
 public:
   /// \brief This keeps a reference to the string referenced by \p Input.
-  Stream(StringRef Input, SourceMgr &);
+  Stream(StringRef Input, SourceMgr &, bool ShowColors = true);
 
-  Stream(MemoryBufferRef InputBuffer, SourceMgr &);
+  Stream(MemoryBufferRef InputBuffer, SourceMgr &, bool ShowColors = true);
   ~Stream();
 
   document_iterator begin();
@@ -107,6 +107,7 @@ class Node {
   enum NodeKind {
     NK_Null,
     NK_Scalar,
+    NK_BlockScalar,
     NK_KeyValue,
     NK_Mapping,
     NK_Sequence,
@@ -158,7 +159,7 @@ class Node {
 
   void operator delete(void *) throw() {}
 
-  virtual ~Node() {}
+  ~Node() = default;
 
 private:
   unsigned int TypeID;
@@ -171,7 +172,7 @@ class Node {
 ///
 /// Example:
 ///   !!null null
-class NullNode : public Node {
+class NullNode final : public Node {
   void anchor() override;
 
 public:
@@ -186,7 +187,7 @@ class NullNode : public Node {
 ///
 /// Example:
 ///   Adena
-class ScalarNode : public Node {
+class ScalarNode final : public Node {
   void anchor() override;
 
 public:
@@ -222,6 +223,36 @@ class ScalarNode : public Node {
                                  SmallVectorImpl<char> &Storage) const;
 };
 
+/// \brief A block scalar node is an opaque datum that can be presented as a
+///        series of zero or more Unicode scalar values.
+///
+/// Example:
+///   |
+///     Hello
+///     World
+class BlockScalarNode final : public Node {
+  void anchor() override;
+
+public:
+  BlockScalarNode(std::unique_ptr<Document> &D, StringRef Anchor, StringRef Tag,
+                  StringRef Value, StringRef RawVal)
+      : Node(NK_BlockScalar, D, Anchor, Tag), Value(Value) {
+    SMLoc Start = SMLoc::getFromPointer(RawVal.begin());
+    SMLoc End = SMLoc::getFromPointer(RawVal.end());
+    SourceRange = SMRange(Start, End);
+  }
+
+  /// \brief Gets the value of this node as a StringRef.
+  StringRef getValue() const { return Value; }
+
+  static inline bool classof(const Node *N) {
+    return N->getType() == NK_BlockScalar;
+  }
+
+private:
+  StringRef Value;
+};
+
 /// \brief A key and value pair. While not technically a Node under the YAML
 ///        representation graph, it is easier to treat them this way.
 ///
@@ -229,7 +260,7 @@ class ScalarNode : public Node {
 ///
 /// Example:
 ///   Section: .text
-class KeyValueNode : public Node {
+class KeyValueNode final : public Node {
   void anchor() override;
 
 public:
@@ -253,7 +284,8 @@ class KeyValueNode : public Node {
 
   void skip() override {
     getKey()->skip();
-    getValue()->skip();
+    if (Node *Val = getValue())
+      Val->skip();
   }
 
   static inline bool classof(const Node *N) {
@@ -339,7 +371,7 @@ template <class CollectionType> void skip(CollectionType &C) {
 /// Example:
 ///   Name: _main
 ///   Scope: Global
-class MappingNode : public Node {
+class MappingNode final : public Node {
   void anchor() override;
 
 public:
@@ -386,7 +418,7 @@ class MappingNode : public Node {
 /// Example:
 ///   - Hello
 ///   - World
-class SequenceNode : public Node {
+class SequenceNode final : public Node {
   void anchor() override;
 
 public:
@@ -439,7 +471,7 @@ class SequenceNode : public Node {
 ///
 /// Example:
 ///   *AnchorName
-class AliasNode : public Node {
+class AliasNode final : public Node {
   void anchor() override;
 
 public:
diff --git a/include/llvm/Support/YAMLTraits.h b/include/llvm/Support/YAMLTraits.h
index 672cf30994a..c04294a5e87 100644
--- a/include/llvm/Support/YAMLTraits.h
+++ b/include/llvm/Support/YAMLTraits.h
@@ -46,6 +46,10 @@ struct MappingTraits {
   // static void mapping(IO &io, T &fields);
   // Optionally may provide:
   // static StringRef validate(IO &io, T &fields);
+  //
+  // The optional flow flag will cause generated YAML to use a flow mapping
+  // (e.g. { a: 0, b: 1 }):
+  // static const bool flow = true;
 };
 
 
@@ -117,6 +121,35 @@ struct ScalarTraits {
 };
 
 
+/// This class should be specialized by type that requires custom conversion
+/// to/from a YAML literal block scalar. For example:
+///
+///    template <>
+///    struct BlockScalarTraits<MyType> {
+///      static void output(const MyType &Value, void*, llvm::raw_ostream &Out)
+///      {
+///        // stream out custom formatting
+///        Out << Val;
+///      }
+///      static StringRef input(StringRef Scalar, void*, MyType &Value) {
+///        // parse scalar and set `value`
+///        // return empty string on success, or error string
+///        return StringRef();
+///      }
+///    };
+template <typename T>
+struct BlockScalarTraits {
+  // Must provide:
+  //
+  // Function to write the value as a string:
+  // static void output(const T &Value, void *ctx, llvm::raw_ostream &Out);
+  //
+  // Function to convert a string to a value.  Returns the empty
+  // StringRef on success or an error string if string is malformed:
+  // static StringRef input(StringRef Scalar, void *ctxt, T &Value);
+};
+
+
 /// This class should be specialized by any type that needs to be converted
 /// to/from a YAML sequence.  For example:
 ///
@@ -220,6 +253,26 @@ struct has_ScalarTraits
 };
 
 
+// Test if BlockScalarTraits<T> is defined on type T.
+template <class T>
+struct has_BlockScalarTraits
+{
+  typedef StringRef (*Signature_input)(StringRef, void *, T &);
+  typedef void (*Signature_output)(const T &, void *, llvm::raw_ostream &);
+
+  template <typename U>
+  static char test(SameType<Signature_input, &U::input> *,
+                   SameType<Signature_output, &U::output> *);
+
+  template <typename U>
+  static double test(...);
+
+public:
+  static bool const value =
+      (sizeof(test<BlockScalarTraits<T>>(nullptr, nullptr)) == 1);
+};
+
+
 // Test if MappingTraits<T> is defined on type T.
 template <class T>
 struct has_MappingTraits
@@ -406,6 +459,7 @@ struct missingTraits : public std::integral_constant<bool,
                                          !has_ScalarEnumerationTraits<T>::value
                                       && !has_ScalarBitSetTraits<T>::value
                                       && !has_ScalarTraits<T>::value
+                                      && !has_BlockScalarTraits<T>::value
                                       && !has_MappingTraits<T>::value
                                       && !has_SequenceTraits<T>::value
                                       && !has_DocumentListTraits<T>::value >  {};
@@ -445,6 +499,9 @@ class IO {
   virtual bool preflightKey(const char*, bool, bool, bool &, void *&) = 0;
   virtual void postflightKey(void*) = 0;
 
+  virtual void beginFlowMapping() = 0;
+  virtual void endFlowMapping() = 0;
+
   virtual void beginEnumScalar() = 0;
   virtual bool matchEnumScalar(const char*, bool) = 0;
   virtual bool matchEnumFallback() = 0;
@@ -455,6 +512,7 @@ class IO {
   virtual void endBitSetScalar() = 0;
 
   virtual void scalarString(StringRef &, bool) = 0;
+  virtual void blockScalarString(StringRef &) = 0;
 
   virtual void setError(const Twine &) = 0;
 
@@ -639,11 +697,32 @@ yamlize(IO &io, T &Val, bool) {
   }
 }
 
+template <typename T>
+typename std::enable_if<has_BlockScalarTraits<T>::value, void>::type
+yamlize(IO &YamlIO, T &Val, bool) {
+  if (YamlIO.outputting()) {
+    std::string Storage;
+    llvm::raw_string_ostream Buffer(Storage);
+    BlockScalarTraits<T>::output(Val, YamlIO.getContext(), Buffer);
+    StringRef Str = Buffer.str();
+    YamlIO.blockScalarString(Str);
+  } else {
+    StringRef Str;
+    YamlIO.blockScalarString(Str);
+    StringRef Result =
+        BlockScalarTraits<T>::input(Str, YamlIO.getContext(), Val);
+    if (!Result.empty())
+      YamlIO.setError(llvm::Twine(Result));
+  }
+}
 
 template<typename T>
 typename std::enable_if<validatedMappingTraits<T>::value, void>::type
 yamlize(IO &io, T &Val, bool) {
-  io.beginMapping();
+  if (has_FlowTraits<MappingTraits<T>>::value)
+    io.beginFlowMapping();
+  else
+    io.beginMapping();
   if (io.outputting()) {
     StringRef Err = MappingTraits<T>::validate(io, Val);
     if (!Err.empty()) {
@@ -657,15 +736,24 @@ yamlize(IO &io, T &Val, bool) {
     if (!Err.empty())
       io.setError(Err);
   }
-  io.endMapping();
+  if (has_FlowTraits<MappingTraits<T>>::value)
+    io.endFlowMapping();
+  else
+    io.endMapping();
 }
 
 template<typename T>
 typename std::enable_if<unvalidatedMappingTraits<T>::value, void>::type
 yamlize(IO &io, T &Val, bool) {
-  io.beginMapping();
-  MappingTraits<T>::mapping(io, Val);
-  io.endMapping();
+  if (has_FlowTraits<MappingTraits<T>>::value) {
+    io.beginFlowMapping();
+    MappingTraits<T>::mapping(io, Val);
+    io.endFlowMapping();
+  } else {
+    io.beginMapping();
+    MappingTraits<T>::mapping(io, Val);
+    io.endMapping();
+  }
 }
 
 template<typename T>
@@ -900,6 +988,8 @@ class Input : public IO {
   void endMapping() override;
   bool preflightKey(const char *, bool, bool, bool &, void *&) override;
   void postflightKey(void *) override;
+  void beginFlowMapping() override;
+  void endFlowMapping() override;
   unsigned beginSequence() override;
   void endSequence() override;
   bool preflightElement(unsigned index, void *&) override;
@@ -916,6 +1006,7 @@ class Input : public IO {
   bool bitSetMatch(const char *, bool ) override;
   void endBitSetScalar() override;
   void scalarString(StringRef &, bool) override;
+  void blockScalarString(StringRef &) override;
   void setError(const Twine &message) override;
   bool canElideEmptySequence() override;
 
@@ -947,7 +1038,8 @@ class Input : public IO {
     StringRef value() const { return _value; }
 
     static inline bool classof(const HNode *n) {
-      return ScalarNode::classof(n->_node);
+      return ScalarNode::classof(n->_node) ||
+             BlockScalarNode::classof(n->_node);
     }
     static inline bool classof(const ScalarHNode *) { return true; }
   protected:
@@ -998,6 +1090,9 @@ class Input : public IO {
   bool setCurrentDocument();
   bool nextDocument();
 
+  /// Returns the current node that's being parsed by the YAML Parser.
+  const Node *getCurrentNode() const;
+
 private:
   llvm::SourceMgr                     SrcMgr; // must be before Strm
   std::unique_ptr<llvm::yaml::Stream> Strm;
@@ -1019,7 +1114,7 @@ class Input : public IO {
 ///
 class Output : public IO {
 public:
-  Output(llvm::raw_ostream &, void *Ctxt=nullptr);
+  Output(llvm::raw_ostream &, void *Ctxt = nullptr, int WrapColumn = 70);
   ~Output() override;
 
   bool outputting() override;
@@ -1028,6 +1123,8 @@ class Output : public IO {
   void endMapping() override;
   bool preflightKey(const char *key, bool, bool, bool &, void *&) override;
   void postflightKey(void *) override;
+  void beginFlowMapping() override;
+  void endFlowMapping() override;
   unsigned beginSequence() override;
   void endSequence() override;
   bool preflightElement(unsigned, void *&) override;
@@ -1044,6 +1141,7 @@ class Output : public IO {
   bool bitSetMatch(const char *, bool ) override;
   void endBitSetScalar() override;
   void scalarString(StringRef &, bool) override;
+  void blockScalarString(StringRef &) override;
   void setError(const Twine &message) override;
   bool canElideEmptySequence() override;
 public:
@@ -1060,13 +1158,23 @@ class Output : public IO {
   void newLineCheck();
   void outputNewLine();
   void paddedKey(StringRef key);
-
-  enum InState { inSeq, inFlowSeq, inMapFirstKey, inMapOtherKey };
+  void flowKey(StringRef Key);
+
+  enum InState {
+    inSeq,
+    inFlowSeq,
+    inMapFirstKey,
+    inMapOtherKey,
+    inFlowMapFirstKey,
+    inFlowMapOtherKey
+  };
 
   llvm::raw_ostream       &Out;
+  int                      WrapColumn;
   SmallVector<InState, 8>  StateStack;
   int                      Column;
   int                      ColumnAtFlowStart;
+  int                      ColumnAtMapFlowStart;
   bool                     NeedBitValueComma;
   bool                     NeedFlowSequenceComma;
   bool                     EnumerationMatchFound;
@@ -1176,6 +1284,16 @@ operator>>(Input &yin, T &docSeq) {
   return yin;
 }
 
+// Define non-member operator>> so that Input can stream in a block scalar.
+template <typename T>
+inline
+typename std::enable_if<has_BlockScalarTraits<T>::value, Input &>::type
+operator>>(Input &In, T &Val) {
+  if (In.setCurrentDocument())
+    yamlize(In, Val, true);
+  return In;
+}
+
 // Provide better error message about types missing a trait specialization
 template <typename T>
 inline
@@ -1231,6 +1349,20 @@ operator<<(Output &yout, T &seq) {
   return yout;
 }
 
+// Define non-member operator<< so that Output can stream out a block scalar.
+template <typename T>
+inline
+typename std::enable_if<has_BlockScalarTraits<T>::value, Output &>::type
+operator<<(Output &Out, T &Val) {
+  Out.beginDocuments();
+  if (Out.preflightDocument(0)) {
+    yamlize(Out, Val, true);
+    Out.postflightDocument();
+  }
+  Out.endDocuments();
+  return Out;
+}
+
 // Provide better error message about types missing a trait specialization
 template <typename T>
 inline
diff --git a/include/llvm/Support/raw_ostream.h b/include/llvm/Support/raw_ostream.h
index 338f0f4295c..28e512c8694 100644
--- a/include/llvm/Support/raw_ostream.h
+++ b/include/llvm/Support/raw_ostream.h
@@ -165,8 +165,10 @@ class raw_ostream {
     if (Size > (size_t)(OutBufEnd - OutBufCur))
       return write(Str.data(), Size);
 
-    memcpy(OutBufCur, Str.data(), Size);
-    OutBufCur += Size;
+    if (Size) {
+      memcpy(OutBufCur, Str.data(), Size);
+      OutBufCur += Size;
+    }
     return *this;
   }
 
@@ -317,10 +319,21 @@ class raw_ostream {
 /// pwrite operation. This is usefull for code that can mostly stream out data,
 /// but needs to patch in a header that needs to know the output size.
 class raw_pwrite_stream : public raw_ostream {
+  virtual void pwrite_impl(const char *Ptr, size_t Size, uint64_t Offset) = 0;
+
 public:
   explicit raw_pwrite_stream(bool Unbuffered = false)
       : raw_ostream(Unbuffered) {}
-  virtual void pwrite(const char *Ptr, size_t Size, uint64_t Offset) = 0;
+  void pwrite(const char *Ptr, size_t Size, uint64_t Offset) {
+#ifndef NDBEBUG
+    uint64_t Pos = tell();
+    // /dev/null always reports a pos of 0, so we cannot perform this check
+    // in that case.
+    if (Pos)
+      assert(Size + Offset <= Pos && "We don't support extending the stream");
+#endif
+    pwrite_impl(Ptr, Size, Offset);
+  }
 };
 
 //===----------------------------------------------------------------------===//
@@ -348,6 +361,8 @@ class raw_fd_ostream : public raw_pwrite_stream {
   /// See raw_ostream::write_impl.
   void write_impl(const char *Ptr, size_t Size) override;
 
+  void pwrite_impl(const char *Ptr, size_t Size, uint64_t Offset) override;
+
   /// Return the current position within the stream, not counting the bytes
   /// currently in the buffer.
   uint64_t current_pos() const override { return pos; }
@@ -388,8 +403,6 @@ class raw_fd_ostream : public raw_pwrite_stream {
   /// to the offset specified from the beginning of the file.
   uint64_t seek(uint64_t off);
 
-  void pwrite(const char *Ptr, size_t Size, uint64_t Offset) override;
-
   /// Set the stream to attempt to use atomic writes for individual output
   /// routines where possible.
   ///
@@ -478,6 +491,8 @@ class raw_svector_ostream : public raw_pwrite_stream {
   /// See raw_ostream::write_impl.
   void write_impl(const char *Ptr, size_t Size) override;
 
+  void pwrite_impl(const char *Ptr, size_t Size, uint64_t Offset) override;
+
   /// Return the current position within the stream, not counting the bytes
   /// currently in the buffer.
   uint64_t current_pos() const override;
@@ -495,7 +510,6 @@ class raw_svector_ostream : public raw_pwrite_stream {
   explicit raw_svector_ostream(SmallVectorImpl<char> &O);
   ~raw_svector_ostream() override;
 
-  void pwrite(const char *Ptr, size_t Size, uint64_t Offset) override;
 
   /// This is called when the SmallVector we're appending to is changed outside
   /// of the raw_svector_ostream's control.  It is only safe to do this if the
@@ -511,6 +525,7 @@ class raw_svector_ostream : public raw_pwrite_stream {
 class raw_null_ostream : public raw_pwrite_stream {
   /// See raw_ostream::write_impl.
   void write_impl(const char *Ptr, size_t size) override;
+  void pwrite_impl(const char *Ptr, size_t Size, uint64_t Offset) override;
 
   /// Return the current position within the stream, not counting the bytes
   /// currently in the buffer.
@@ -519,7 +534,6 @@ class raw_null_ostream : public raw_pwrite_stream {
 public:
   explicit raw_null_ostream() {}
   ~raw_null_ostream() override;
-  void pwrite(const char *Ptr, size_t Size, uint64_t Offset) override;
 };
 
 class buffer_ostream : public raw_svector_ostream {
diff --git a/include/llvm/TableGen/Record.h b/include/llvm/TableGen/Record.h
index e746e7d486f..b4642c99119 100644
--- a/include/llvm/TableGen/Record.h
+++ b/include/llvm/TableGen/Record.h
@@ -17,49 +17,20 @@
 
 #include "llvm/ADT/ArrayRef.h"
 #include "llvm/ADT/FoldingSet.h"
-#include "llvm/Support/Allocator.h"
+#include "llvm/ADT/PointerIntPair.h"
 #include "llvm/Support/Casting.h"
 #include "llvm/Support/DataTypes.h"
 #include "llvm/Support/ErrorHandling.h"
-#include "llvm/Support/SourceMgr.h"
+#include "llvm/Support/SMLoc.h"
 #include "llvm/Support/raw_ostream.h"
 #include <map>
 
 namespace llvm {
-class raw_ostream;
 
-// RecTy subclasses.
-class BitRecTy;
-class BitsRecTy;
-class IntRecTy;
-class StringRecTy;
 class ListRecTy;
-class DagRecTy;
-class RecordRecTy;
-
-// Init subclasses.
-class Init;
-class UnsetInit;
-class BitInit;
-class BitsInit;
-class IntInit;
-class StringInit;
-class ListInit;
-class UnOpInit;
-class BinOpInit;
-class TernOpInit;
-class DefInit;
-class DagInit;
-class TypedInit;
-class VarInit;
-class FieldInit;
-class VarBitInit;
-class VarListElementInit;
-
-// Other classes.
+struct MultiClass;
 class Record;
 class RecordVal;
-struct MultiClass;
 class RecordKeeper;
 
 //===----------------------------------------------------------------------===//
@@ -81,13 +52,12 @@ class RecTy {
 
 private:
   RecTyKind Kind;
-  ListRecTy *ListTy;
-  virtual void anchor();
+  std::unique_ptr<ListRecTy> ListTy;
 
 public:
   RecTyKind getRecTyKind() const { return Kind; }
 
-  RecTy(RecTyKind K) : Kind(K), ListTy(nullptr) {}
+  RecTy(RecTyKind K) : Kind(K) {}
   virtual ~RecTy() {}
 
   virtual std::string getAsString() const = 0;
@@ -96,40 +66,10 @@ class RecTy {
 
   /// typeIsConvertibleTo - Return true if all values of 'this' type can be
   /// converted to the specified type.
-  virtual bool typeIsConvertibleTo(const RecTy *RHS) const = 0;
+  virtual bool typeIsConvertibleTo(const RecTy *RHS) const;
 
   /// getListTy - Returns the type representing list<this>.
   ListRecTy *getListTy();
-
-public:   // These methods should only be called from subclasses of Init
-  virtual Init *convertValue( UnsetInit *UI) { return nullptr; }
-  virtual Init *convertValue(   BitInit *BI) { return nullptr; }
-  virtual Init *convertValue(  BitsInit *BI) { return nullptr; }
-  virtual Init *convertValue(   IntInit *II) { return nullptr; }
-  virtual Init *convertValue(StringInit *SI) { return nullptr; }
-  virtual Init *convertValue(  ListInit *LI) { return nullptr; }
-  virtual Init *convertValue( UnOpInit *UI) {
-    return convertValue((TypedInit*)UI);
-  }
-  virtual Init *convertValue( BinOpInit *UI) {
-    return convertValue((TypedInit*)UI);
-  }
-  virtual Init *convertValue( TernOpInit *UI) {
-    return convertValue((TypedInit*)UI);
-  }
-  virtual Init *convertValue(VarBitInit *VB) { return nullptr; }
-  virtual Init *convertValue(   DefInit *DI) { return nullptr; }
-  virtual Init *convertValue(   DagInit *DI) { return nullptr; }
-  virtual Init *convertValue( TypedInit *TI) { return nullptr; }
-  virtual Init *convertValue(   VarInit *VI) {
-    return convertValue((TypedInit*)VI);
-  }
-  virtual Init *convertValue( FieldInit *FI) {
-    return convertValue((TypedInit*)FI);
-  }
-
-public:
-  virtual bool baseClassOf(const RecTy*) const;
 };
 
 inline raw_ostream &operator<<(raw_ostream &OS, const RecTy &Ty) {
@@ -150,28 +90,9 @@ class BitRecTy : public RecTy {
 
   static BitRecTy *get() { return &Shared; }
 
-  Init *convertValue( UnsetInit *UI) override { return (Init*)UI; }
-  Init *convertValue(   BitInit *BI) override { return (Init*)BI; }
-  Init *convertValue(  BitsInit *BI) override;
-  Init *convertValue(   IntInit *II) override;
-  Init *convertValue(StringInit *SI) override { return nullptr; }
-  Init *convertValue(  ListInit *LI) override { return nullptr; }
-  Init *convertValue(VarBitInit *VB) override { return (Init*)VB; }
-  Init *convertValue(   DefInit *DI) override { return nullptr; }
-  Init *convertValue(   DagInit *DI) override { return nullptr; }
-  Init *convertValue( UnOpInit *UI) override { return RecTy::convertValue(UI);}
-  Init *convertValue( BinOpInit *UI) override { return RecTy::convertValue(UI);}
-  Init *convertValue( TernOpInit *UI) override {return RecTy::convertValue(UI);}
-  Init *convertValue( TypedInit *TI) override;
-  Init *convertValue(   VarInit *VI) override { return RecTy::convertValue(VI);}
-  Init *convertValue( FieldInit *FI) override { return RecTy::convertValue(FI);}
-
   std::string getAsString() const override { return "bit"; }
 
-  bool typeIsConvertibleTo(const RecTy *RHS) const override {
-    return RHS->baseClassOf(this);
-  }
-  bool baseClassOf(const RecTy*) const override;
+  bool typeIsConvertibleTo(const RecTy *RHS) const override;
 };
 
 /// BitsRecTy - 'bits<n>' - Represent a fixed number of bits
@@ -189,28 +110,9 @@ class BitsRecTy : public RecTy {
 
   unsigned getNumBits() const { return Size; }
 
-  Init *convertValue( UnsetInit *UI) override;
-  Init *convertValue(   BitInit *UI) override;
-  Init *convertValue(  BitsInit *BI) override;
-  Init *convertValue(   IntInit *II) override;
-  Init *convertValue(StringInit *SI) override { return nullptr; }
-  Init *convertValue(  ListInit *LI) override { return nullptr; }
-  Init *convertValue(VarBitInit *VB) override { return nullptr; }
-  Init *convertValue(   DefInit *DI) override { return nullptr; }
-  Init *convertValue(   DagInit *DI) override { return nullptr; }
-  Init *convertValue(  UnOpInit *UI) override { return RecTy::convertValue(UI);}
-  Init *convertValue( BinOpInit *UI) override { return RecTy::convertValue(UI);}
-  Init *convertValue(TernOpInit *UI) override { return RecTy::convertValue(UI);}
-  Init *convertValue( TypedInit *TI) override;
-  Init *convertValue(   VarInit *VI) override { return RecTy::convertValue(VI);}
-  Init *convertValue( FieldInit *FI) override { return RecTy::convertValue(FI);}
-
   std::string getAsString() const override;
 
-  bool typeIsConvertibleTo(const RecTy *RHS) const override {
-    return RHS->baseClassOf(this);
-  }
-  bool baseClassOf(const RecTy*) const override;
+  bool typeIsConvertibleTo(const RecTy *RHS) const override;
 };
 
 /// IntRecTy - 'int' - Represent an integer value of no particular size
@@ -226,29 +128,9 @@ class IntRecTy : public RecTy {
 
   static IntRecTy *get() { return &Shared; }
 
-  Init *convertValue( UnsetInit *UI) override { return (Init*)UI; }
-  Init *convertValue(   BitInit *BI) override;
-  Init *convertValue(  BitsInit *BI) override;
-  Init *convertValue(   IntInit *II) override { return (Init*)II; }
-  Init *convertValue(StringInit *SI) override { return nullptr; }
-  Init *convertValue(  ListInit *LI) override { return nullptr; }
-  Init *convertValue(VarBitInit *VB) override { return nullptr; }
-  Init *convertValue(   DefInit *DI) override { return nullptr; }
-  Init *convertValue(   DagInit *DI) override { return nullptr; }
-  Init *convertValue( UnOpInit *UI)  override { return RecTy::convertValue(UI);}
-  Init *convertValue( BinOpInit *UI) override { return RecTy::convertValue(UI);}
-  Init *convertValue( TernOpInit *UI) override {return RecTy::convertValue(UI);}
-  Init *convertValue( TypedInit *TI) override;
-  Init *convertValue(   VarInit *VI) override { return RecTy::convertValue(VI);}
-  Init *convertValue( FieldInit *FI) override { return RecTy::convertValue(FI);}
-
   std::string getAsString() const override { return "int"; }
 
-  bool typeIsConvertibleTo(const RecTy *RHS) const override {
-    return RHS->baseClassOf(this);
-  }
-
-  bool baseClassOf(const RecTy*) const override;
+  bool typeIsConvertibleTo(const RecTy *RHS) const override;
 };
 
 /// StringRecTy - 'string' - Represent an string value
@@ -264,28 +146,7 @@ class StringRecTy : public RecTy {
 
   static StringRecTy *get() { return &Shared; }
 
-  Init *convertValue( UnsetInit *UI) override { return (Init*)UI; }
-  Init *convertValue(   BitInit *BI) override { return nullptr; }
-  Init *convertValue(  BitsInit *BI) override { return nullptr; }
-  Init *convertValue(   IntInit *II) override { return nullptr; }
-  Init *convertValue(StringInit *SI) override { return (Init*)SI; }
-  Init *convertValue(  ListInit *LI) override { return nullptr; }
-  Init *convertValue( UnOpInit *BO) override;
-  Init *convertValue( BinOpInit *BO) override;
-  Init *convertValue( TernOpInit *BO) override {return RecTy::convertValue(BO);}
-
-  Init *convertValue(VarBitInit *VB) override { return nullptr; }
-  Init *convertValue(   DefInit *DI) override { return nullptr; }
-  Init *convertValue(   DagInit *DI) override { return nullptr; }
-  Init *convertValue( TypedInit *TI) override;
-  Init *convertValue(   VarInit *VI) override { return RecTy::convertValue(VI);}
-  Init *convertValue( FieldInit *FI) override { return RecTy::convertValue(FI);}
-
-  std::string getAsString() const override { return "string"; }
-
-  bool typeIsConvertibleTo(const RecTy *RHS) const override {
-    return RHS->baseClassOf(this);
-  }
+  std::string getAsString() const override;
 };
 
 /// ListRecTy - 'list<Ty>' - Represent a list of values, all of which must be of
@@ -304,29 +165,9 @@ class ListRecTy : public RecTy {
   static ListRecTy *get(RecTy *T) { return T->getListTy(); }
   RecTy *getElementType() const { return Ty; }
 
-  Init *convertValue( UnsetInit *UI) override { return (Init*)UI; }
-  Init *convertValue(   BitInit *BI) override { return nullptr; }
-  Init *convertValue(  BitsInit *BI) override { return nullptr; }
-  Init *convertValue(   IntInit *II) override { return nullptr; }
-  Init *convertValue(StringInit *SI) override { return nullptr; }
-  Init *convertValue(  ListInit *LI) override;
-  Init *convertValue(VarBitInit *VB) override { return nullptr; }
-  Init *convertValue(   DefInit *DI) override { return nullptr; }
-  Init *convertValue(   DagInit *DI) override { return nullptr; }
-  Init *convertValue(  UnOpInit *UI) override { return RecTy::convertValue(UI);}
-  Init *convertValue( BinOpInit *UI) override { return RecTy::convertValue(UI);}
-  Init *convertValue(TernOpInit *UI) override { return RecTy::convertValue(UI);}
-  Init *convertValue( TypedInit *TI) override;
-  Init *convertValue(   VarInit *VI) override { return RecTy::convertValue(VI);}
-  Init *convertValue( FieldInit *FI) override { return RecTy::convertValue(FI);}
-
   std::string getAsString() const override;
 
-  bool typeIsConvertibleTo(const RecTy *RHS) const override {
-    return RHS->baseClassOf(this);
-  }
-
-  bool baseClassOf(const RecTy*) const override;
+  bool typeIsConvertibleTo(const RecTy *RHS) const override;
 };
 
 /// DagRecTy - 'dag' - Represent a dag fragment
@@ -342,27 +183,7 @@ class DagRecTy : public RecTy {
 
   static DagRecTy *get() { return &Shared; }
 
-  Init *convertValue( UnsetInit *UI) override { return (Init*)UI; }
-  Init *convertValue(   BitInit *BI) override { return nullptr; }
-  Init *convertValue(  BitsInit *BI) override { return nullptr; }
-  Init *convertValue(   IntInit *II) override { return nullptr; }
-  Init *convertValue(StringInit *SI) override { return nullptr; }
-  Init *convertValue(  ListInit *LI) override { return nullptr; }
-  Init *convertValue(VarBitInit *VB) override { return nullptr; }
-  Init *convertValue(   DefInit *DI) override { return nullptr; }
-  Init *convertValue( UnOpInit *BO) override;
-  Init *convertValue( BinOpInit *BO) override;
-  Init *convertValue( TernOpInit *BO) override {return RecTy::convertValue(BO);}
-  Init *convertValue(   DagInit *CI) override { return (Init*)CI; }
-  Init *convertValue( TypedInit *TI) override;
-  Init *convertValue(   VarInit *VI) override { return RecTy::convertValue(VI);}
-  Init *convertValue( FieldInit *FI) override { return RecTy::convertValue(FI);}
-
-  std::string getAsString() const override { return "dag"; }
-
-  bool typeIsConvertibleTo(const RecTy *RHS) const override {
-    return RHS->baseClassOf(this);
-  }
+  std::string getAsString() const override;
 };
 
 /// RecordRecTy - '[classname]' - Represent an instance of a class, such as:
@@ -382,28 +203,9 @@ class RecordRecTy : public RecTy {
 
   Record *getRecord() const { return Rec; }
 
-  Init *convertValue( UnsetInit *UI) override { return (Init*)UI; }
-  Init *convertValue(   BitInit *BI) override { return nullptr; }
-  Init *convertValue(  BitsInit *BI) override { return nullptr; }
-  Init *convertValue(   IntInit *II) override { return nullptr; }
-  Init *convertValue(StringInit *SI) override { return nullptr; }
-  Init *convertValue(  ListInit *LI) override { return nullptr; }
-  Init *convertValue(VarBitInit *VB) override { return nullptr; }
-  Init *convertValue( UnOpInit *UI) override { return RecTy::convertValue(UI);}
-  Init *convertValue( BinOpInit *UI) override { return RecTy::convertValue(UI);}
-  Init *convertValue( TernOpInit *UI) override {return RecTy::convertValue(UI);}
-  Init *convertValue(   DefInit *DI) override;
-  Init *convertValue(   DagInit *DI) override { return nullptr; }
-  Init *convertValue( TypedInit *VI) override;
-  Init *convertValue(   VarInit *VI) override { return RecTy::convertValue(VI);}
-  Init *convertValue( FieldInit *FI) override { return RecTy::convertValue(FI);}
-
   std::string getAsString() const override;
 
-  bool typeIsConvertibleTo(const RecTy *RHS) const override {
-    return RHS->baseClassOf(this);
-  }
-  bool baseClassOf(const RecTy*) const override;
+  bool typeIsConvertibleTo(const RecTy *RHS) const override;
 };
 
 /// resolveTypes - Find a common type that T1 and T2 convert to.
@@ -486,10 +288,8 @@ class Init {
   /// invokes print on stderr.
   void dump() const;
 
-  /// convertInitializerTo - This virtual function is a simple call-back
-  /// function that should be overridden to call the appropriate
-  /// RecTy::convertValue method.
-  ///
+  /// convertInitializerTo - This virtual function converts to the appropriate
+  /// Init based on the passed in type.
   virtual Init *convertInitializerTo(RecTy *Ty) const = 0;
 
   /// convertInitializerBitRange - This method is used to implement the bitrange
@@ -566,6 +366,11 @@ class TypedInit : public Init {
 
 protected:
   explicit TypedInit(InitKind K, RecTy *T) : Init(K), Ty(T) {}
+  ~TypedInit() {
+    // If this is a DefInit we need to delete the RecordRecTy.
+    if (getKind() == IK_DefInit)
+      delete Ty;
+  }
 
 public:
   static bool classof(const Init *I) {
@@ -574,6 +379,8 @@ class TypedInit : public Init {
   }
   RecTy *getType() const { return Ty; }
 
+  Init *convertInitializerTo(RecTy *Ty) const override;
+
   Init *
   convertInitializerBitRange(const std::vector<unsigned> &Bits) const override;
   Init *
@@ -598,7 +405,6 @@ class UnsetInit : public Init {
   UnsetInit() : Init(IK_UnsetInit) {}
   UnsetInit(const UnsetInit &) = delete;
   UnsetInit &operator=(const UnsetInit &Other) = delete;
-  void anchor() override;
 
 public:
   static bool classof(const Init *I) {
@@ -606,9 +412,7 @@ class UnsetInit : public Init {
   }
   static UnsetInit *get();
 
-  Init *convertInitializerTo(RecTy *Ty) const override {
-    return Ty->convertValue(const_cast<UnsetInit *>(this));
-  }
+  Init *convertInitializerTo(RecTy *Ty) const override;
 
   Init *getBit(unsigned Bit) const override {
     return const_cast<UnsetInit*>(this);
@@ -626,7 +430,6 @@ class BitInit : public Init {
   explicit BitInit(bool V) : Init(IK_BitInit), Value(V) {}
   BitInit(const BitInit &Other) = delete;
   BitInit &operator=(BitInit &Other) = delete;
-  void anchor() override;
 
 public:
   static bool classof(const Init *I) {
@@ -636,9 +439,7 @@ class BitInit : public Init {
 
   bool getValue() const { return Value; }
 
-  Init *convertInitializerTo(RecTy *Ty) const override {
-    return Ty->convertValue(const_cast<BitInit *>(this));
-  }
+  Init *convertInitializerTo(RecTy *Ty) const override;
 
   Init *getBit(unsigned Bit) const override {
     assert(Bit < 1 && "Bit index out of range!");
@@ -671,9 +472,7 @@ class BitsInit : public TypedInit, public FoldingSetNode {
 
   unsigned getNumBits() const { return Bits.size(); }
 
-  Init *convertInitializerTo(RecTy *Ty) const override {
-    return Ty->convertValue(const_cast<BitsInit *>(this));
-  }
+  Init *convertInitializerTo(RecTy *Ty) const override;
   Init *
   convertInitializerBitRange(const std::vector<unsigned> &Bits) const override;
 
@@ -724,9 +523,7 @@ class IntInit : public TypedInit {
 
   int64_t getValue() const { return Value; }
 
-  Init *convertInitializerTo(RecTy *Ty) const override {
-    return Ty->convertValue(const_cast<IntInit *>(this));
-  }
+  Init *convertInitializerTo(RecTy *Ty) const override;
   Init *
   convertInitializerBitRange(const std::vector<unsigned> &Bits) const override;
 
@@ -755,7 +552,6 @@ class StringInit : public TypedInit {
 
   StringInit(const StringInit &Other) = delete;
   StringInit &operator=(const StringInit &Other) = delete;
-  void anchor() override;
 
 public:
   static bool classof(const Init *I) {
@@ -765,9 +561,7 @@ class StringInit : public TypedInit {
 
   const std::string &getValue() const { return Value; }
 
-  Init *convertInitializerTo(RecTy *Ty) const override {
-    return Ty->convertValue(const_cast<StringInit *>(this));
-  }
+  Init *convertInitializerTo(RecTy *Ty) const override;
 
   std::string getAsString() const override { return "\"" + Value + "\""; }
   std::string getAsUnquotedString() const override { return Value; }
@@ -809,7 +603,6 @@ class ListInit : public TypedInit, public FoldingSetNode {
 
   void Profile(FoldingSetNodeID &ID) const;
 
-  unsigned getSize() const { return Values.size(); }
   Init *getElement(unsigned i) const {
     assert(i < Values.size() && "List element index out of range!");
     return Values[i];
@@ -820,9 +613,7 @@ class ListInit : public TypedInit, public FoldingSetNode {
   Init *
     convertInitListSlice(const std::vector<unsigned> &Elements) const override;
 
-  Init *convertInitializerTo(RecTy *Ty) const override {
-    return Ty->convertValue(const_cast<ListInit *>(this));
-  }
+  Init *convertInitializerTo(RecTy *Ty) const override;
 
   /// resolveReferences - This method is used by classes that refer to other
   /// variables which may not be defined at the time they expression is formed.
@@ -835,11 +626,11 @@ class ListInit : public TypedInit, public FoldingSetNode {
 
   ArrayRef<Init*> getValues() const { return Values; }
 
-  inline const_iterator begin() const { return Values.begin(); }
-  inline const_iterator end  () const { return Values.end();   }
+  const_iterator begin() const { return Values.begin(); }
+  const_iterator end  () const { return Values.end();   }
 
-  inline size_t         size () const { return Values.size();  }
-  inline bool           empty() const { return Values.empty(); }
+  size_t         size () const { return Values.size();  }
+  bool           empty() const { return Values.empty(); }
 
   /// resolveListElementReference - This method is used to implement
   /// VarListElementInit::resolveReferences.  If the list element is resolvable
@@ -869,17 +660,13 @@ class OpInit : public TypedInit {
   // Clone - Clone this operator, replacing arguments with the new list
   virtual OpInit *clone(std::vector<Init *> &Operands) const = 0;
 
-  virtual int getNumOperands() const = 0;
-  virtual Init *getOperand(int i) const = 0;
+  virtual unsigned getNumOperands() const = 0;
+  virtual Init *getOperand(unsigned i) const = 0;
 
   // Fold - If possible, fold this to a simpler init.  Return this if not
   // possible to fold.
   virtual Init *Fold(Record *CurRec, MultiClass *CurMultiClass) const = 0;
 
-  Init *convertInitializerTo(RecTy *Ty) const override {
-    return Ty->convertValue(const_cast<OpInit *>(this));
-  }
-
   Init *resolveListElementReference(Record &R, const RecordVal *RV,
                                     unsigned Elt) const override;
 
@@ -915,8 +702,8 @@ class UnOpInit : public OpInit {
     return UnOpInit::get(getOpcode(), *Operands.begin(), getType());
   }
 
-  int getNumOperands() const override { return 1; }
-  Init *getOperand(int i) const override {
+  unsigned getNumOperands() const override { return 1; }
+  Init *getOperand(unsigned i) const override {
     assert(i == 0 && "Invalid operand id for unary operator");
     return getOperand();
   }
@@ -963,13 +750,12 @@ class BinOpInit : public OpInit {
     return BinOpInit::get(getOpcode(), Operands[0], Operands[1], getType());
   }
 
-  int getNumOperands() const override { return 2; }
-  Init *getOperand(int i) const override {
-    assert((i == 0 || i == 1) && "Invalid operand id for binary operator");
-    if (i == 0) {
-      return getLHS();
-    } else {
-      return getRHS();
+  unsigned getNumOperands() const override { return 2; }
+  Init *getOperand(unsigned i) const override {
+    switch (i) {
+    default: llvm_unreachable("Invalid operand id for binary operator");
+    case 0: return getLHS();
+    case 1: return getRHS();
     }
   }
 
@@ -1019,16 +805,13 @@ class TernOpInit : public OpInit {
                            getType());
   }
 
-  int getNumOperands() const override { return 3; }
-  Init *getOperand(int i) const override {
-    assert((i == 0 || i == 1 || i == 2) &&
-           "Invalid operand id for ternary operator");
-    if (i == 0) {
-      return getLHS();
-    } else if (i == 1) {
-      return getMHS();
-    } else {
-      return getRHS();
+  unsigned getNumOperands() const override { return 3; }
+  Init *getOperand(unsigned i) const override {
+    switch (i) {
+    default: llvm_unreachable("Invalid operand id for ternary operator");
+    case 0: return getLHS();
+    case 1: return getMHS();
+    case 2: return getRHS();
     }
   }
 
@@ -1068,10 +851,6 @@ class VarInit : public TypedInit {
   static VarInit *get(const std::string &VN, RecTy *T);
   static VarInit *get(Init *VN, RecTy *T);
 
-  Init *convertInitializerTo(RecTy *Ty) const override {
-    return Ty->convertValue(const_cast<VarInit *>(this));
-  }
-
   const std::string &getName() const;
   Init *getNameInit() const { return VarName; }
   std::string getNameInitAsString() const {
@@ -1120,9 +899,7 @@ class VarBitInit : public Init {
   }
   static VarBitInit *get(TypedInit *T, unsigned B);
 
-  Init *convertInitializerTo(RecTy *Ty) const override {
-    return Ty->convertValue(const_cast<VarBitInit *>(this));
-  }
+  Init *convertInitializerTo(RecTy *Ty) const override;
 
   Init *getBitVar() const override { return TI; }
   unsigned getBitNum() const override { return Bit; }
@@ -1159,10 +936,6 @@ class VarListElementInit : public TypedInit {
   }
   static VarListElementInit *get(TypedInit *T, unsigned E);
 
-  Init *convertInitializerTo(RecTy *Ty) const override {
-    return Ty->convertValue(const_cast<VarListElementInit *>(this));
-  }
-
   TypedInit *getVariable() const { return TI; }
   unsigned getElementNum() const { return Element; }
 
@@ -1195,9 +968,7 @@ class DefInit : public TypedInit {
   }
   static DefInit *get(Record*);
 
-  Init *convertInitializerTo(RecTy *Ty) const override {
-    return Ty->convertValue(const_cast<DefInit *>(this));
-  }
+  Init *convertInitializerTo(RecTy *Ty) const override;
 
   Record *getDef() const { return Def; }
 
@@ -1241,11 +1012,6 @@ class FieldInit : public TypedInit {
     return I->getKind() == IK_FieldInit;
   }
   static FieldInit *get(Init *R, const std::string &FN);
-  static FieldInit *get(Init *R, const Init *FN);
-
-  Init *convertInitializerTo(RecTy *Ty) const override {
-    return Ty->convertValue(const_cast<FieldInit *>(this));
-  }
 
   Init *getBit(unsigned Bit) const override;
 
@@ -1292,9 +1058,7 @@ class DagInit : public TypedInit, public FoldingSetNode {
 
   void Profile(FoldingSetNodeID &ID) const;
 
-  Init *convertInitializerTo(RecTy *Ty) const override {
-    return Ty->convertValue(const_cast<DagInit *>(this));
-  }
+  Init *convertInitializerTo(RecTy *Ty) const override;
 
   Init *getOperator() const { return Val; }
 
@@ -1344,22 +1108,21 @@ class DagInit : public TypedInit, public FoldingSetNode {
 //===----------------------------------------------------------------------===//
 
 class RecordVal {
-  Init *Name;
+  PointerIntPair<Init *, 1, bool> NameAndPrefix;
   RecTy *Ty;
-  unsigned Prefix;
   Init *Value;
 
 public:
-  RecordVal(Init *N, RecTy *T, unsigned P);
-  RecordVal(const std::string &N, RecTy *T, unsigned P);
+  RecordVal(Init *N, RecTy *T, bool P);
+  RecordVal(const std::string &N, RecTy *T, bool P);
 
   const std::string &getName() const;
-  const Init *getNameInit() const { return Name; }
+  const Init *getNameInit() const { return NameAndPrefix.getPointer(); }
   std::string getNameInitAsString() const {
     return getNameInit()->getAsUnquotedString();
   }
 
-  unsigned getPrefix() const { return Prefix; }
+  bool getPrefix() const { return NameAndPrefix.getInt(); }
   RecTy *getType() const { return Ty; }
   Init *getValue() const { return Value; }
 
@@ -1398,7 +1161,7 @@ class Record {
   // Tracks Record instances. Not owned by Record.
   RecordKeeper &TrackedRecords;
 
-  DefInit *TheInit;
+  std::unique_ptr<DefInit> TheInit;
   bool IsAnonymous;
 
   // Class-instance values can be used by other defs.  For example, Struct<i>
@@ -1408,7 +1171,7 @@ class Record {
   //     def Def : Class<Struct<i>>;
   //
   // These need to get fully resolved before instantiating any other
-  // definitions that usie them (e.g. Def).  However, inside a multiclass they
+  // definitions that use them (e.g. Def).  However, inside a multiclass they
   // can't be immediately resolved so we mark them ResolveFirst to fully
   // resolve them later as soon as the multiclass is instantiated.
   bool ResolveFirst;
@@ -1418,28 +1181,25 @@ class Record {
 
 public:
   // Constructs a record.
-  explicit Record(const std::string &N, ArrayRef<SMLoc> locs,
-                  RecordKeeper &records, bool Anonymous = false) :
-    ID(LastID++), Name(StringInit::get(N)), Locs(locs.begin(), locs.end()),
-    TrackedRecords(records), TheInit(nullptr), IsAnonymous(Anonymous),
-    ResolveFirst(false) {
-    init();
-  }
   explicit Record(Init *N, ArrayRef<SMLoc> locs, RecordKeeper &records,
                   bool Anonymous = false) :
     ID(LastID++), Name(N), Locs(locs.begin(), locs.end()),
-    TrackedRecords(records), TheInit(nullptr), IsAnonymous(Anonymous),
-    ResolveFirst(false) {
+    TrackedRecords(records), IsAnonymous(Anonymous), ResolveFirst(false) {
     init();
   }
+  explicit Record(const std::string &N, ArrayRef<SMLoc> locs,
+                  RecordKeeper &records, bool Anonymous = false)
+    : Record(StringInit::get(N), locs, records, Anonymous) {}
+
 
   // When copy-constructing a Record, we must still guarantee a globally unique
-  // ID number.  All other fields can be copied normally.
+  // ID number.  Don't copy TheInit either since it's owned by the original
+  // record. All other fields can be copied normally.
   Record(const Record &O) :
     ID(LastID++), Name(O.Name), Locs(O.Locs), TemplateArgs(O.TemplateArgs),
     Values(O.Values), SuperClasses(O.SuperClasses),
     SuperClassRanges(O.SuperClassRanges), TrackedRecords(O.TrackedRecords),
-    TheInit(O.TheInit), IsAnonymous(O.IsAnonymous),
+    IsAnonymous(O.IsAnonymous),
     ResolveFirst(O.ResolveFirst) { }
 
   static unsigned getNewUID() { return LastID++; }
@@ -1462,16 +1222,16 @@ class Record {
   /// get the corresponding DefInit.
   DefInit *getDefInit();
 
-  const std::vector<Init *> &getTemplateArgs() const {
+  ArrayRef<Init *> getTemplateArgs() const {
     return TemplateArgs;
   }
-  const std::vector<RecordVal> &getValues() const { return Values; }
-  const std::vector<Record*>   &getSuperClasses() const { return SuperClasses; }
+  ArrayRef<RecordVal> getValues() const { return Values; }
+  ArrayRef<Record *>  getSuperClasses() const { return SuperClasses; }
   ArrayRef<SMRange> getSuperClassRanges() const { return SuperClassRanges; }
 
   bool isTemplateArg(Init *Name) const {
-    for (unsigned i = 0, e = TemplateArgs.size(); i != e; ++i)
-      if (TemplateArgs[i] == Name) return true;
+    for (Init *TA : TemplateArgs)
+      if (TA == Name) return true;
     return false;
   }
   bool isTemplateArg(StringRef Name) const {
@@ -1479,16 +1239,16 @@ class Record {
   }
 
   const RecordVal *getValue(const Init *Name) const {
-    for (unsigned i = 0, e = Values.size(); i != e; ++i)
-      if (Values[i].getNameInit() == Name) return &Values[i];
+    for (const RecordVal &Val : Values)
+      if (Val.getNameInit() == Name) return &Val;
     return nullptr;
   }
   const RecordVal *getValue(StringRef Name) const {
     return getValue(StringInit::get(Name));
   }
   RecordVal *getValue(const Init *Name) {
-    for (unsigned i = 0, e = Values.size(); i != e; ++i)
-      if (Values[i].getNameInit() == Name) return &Values[i];
+    for (RecordVal &Val : Values)
+      if (Val.getNameInit() == Name) return &Val;
     return nullptr;
   }
   RecordVal *getValue(StringRef Name) {
@@ -1529,15 +1289,15 @@ class Record {
   }
 
   bool isSubClassOf(const Record *R) const {
-    for (unsigned i = 0, e = SuperClasses.size(); i != e; ++i)
-      if (SuperClasses[i] == R)
+    for (const Record *SC : SuperClasses)
+      if (SC == R)
         return true;
     return false;
   }
 
   bool isSubClassOf(StringRef Name) const {
-    for (unsigned i = 0, e = SuperClasses.size(); i != e; ++i)
-      if (SuperClasses[i]->getNameInitAsString() == Name)
+    for (const Record *SC : SuperClasses)
+      if (SC->getNameInitAsString() == Name)
         return true;
     return false;
   }
@@ -1587,7 +1347,7 @@ class Record {
 
   /// Return true if the named field is unset.
   bool isValueUnset(StringRef FieldName) const {
-    return getValueInit(FieldName) == UnsetInit::get();
+    return isa<UnsetInit>(getValueInit(FieldName));
   }
 
   /// getValueAsString - This method looks up the specified field and returns
@@ -1739,7 +1499,6 @@ struct LessRecordFieldName {
 };
 
 struct LessRecordRegister {
-  static size_t min(size_t a, size_t b) { return a < b ? a : b; }
   static bool ascii_isdigit(char x) { return x >= '0' && x <= '9'; }
 
   struct RecordParts {
diff --git a/include/llvm/TableGen/SetTheory.h b/include/llvm/TableGen/SetTheory.h
index 76e56ec6443..d4e0f53d3ef 100644
--- a/include/llvm/TableGen/SetTheory.h
+++ b/include/llvm/TableGen/SetTheory.h
@@ -47,9 +47,10 @@
 #ifndef LLVM_TABLEGEN_SETTHEORY_H
 #define LLVM_TABLEGEN_SETTHEORY_H
 
+#include "llvm/ADT/ArrayRef.h"
 #include "llvm/ADT/SetVector.h"
 #include "llvm/ADT/StringMap.h"
-#include "llvm/Support/SourceMgr.h"
+#include "llvm/Support/SMLoc.h"
 #include <map>
 #include <vector>
 
@@ -58,7 +59,6 @@ namespace llvm {
 class DagInit;
 class Init;
 class Record;
-class RecordKeeper;
 
 class SetTheory {
 public:
@@ -95,17 +95,17 @@ class SetTheory {
   ExpandMap Expansions;
 
   // Known DAG operators by name.
-  StringMap<Operator*> Operators;
+  StringMap<std::unique_ptr<Operator>> Operators;
 
   // Typed expanders by class name.
-  StringMap<Expander*> Expanders;
+  StringMap<std::unique_ptr<Expander>> Expanders;
 
 public:
   /// Create a SetTheory instance with only the standard operators.
   SetTheory();
 
   /// addExpander - Add an expander for Records with the named super class.
-  void addExpander(StringRef ClassName, Expander*);
+  void addExpander(StringRef ClassName, std::unique_ptr<Expander>);
 
   /// addFieldExpander - Add an expander for ClassName that simply evaluates
   /// FieldName in the Record to get the set elements.  That is all that is
@@ -118,7 +118,7 @@ class SetTheory {
   void addFieldExpander(StringRef ClassName, StringRef FieldName);
 
   /// addOperator - Add a DAG operator.
-  void addOperator(StringRef Name, Operator*);
+  void addOperator(StringRef Name, std::unique_ptr<Operator>);
 
   /// evaluate - Evaluate Expr and append the resulting set to Elts.
   void evaluate(Init *Expr, RecSet &Elts, ArrayRef<SMLoc> Loc);
diff --git a/include/llvm/TableGen/TableGenBackend.h b/include/llvm/TableGen/TableGenBackend.h
index 3e4f3cc917a..d226f1f1af7 100644
--- a/include/llvm/TableGen/TableGenBackend.h
+++ b/include/llvm/TableGen/TableGenBackend.h
@@ -14,10 +14,9 @@
 #ifndef LLVM_TABLEGEN_TABLEGENBACKEND_H
 #define LLVM_TABLEGEN_TABLEGENBACKEND_H
 
-#include "llvm/ADT/StringRef.h"
-
 namespace llvm {
 
+class StringRef;
 class raw_ostream;
 
 /// emitSourceFileHeader - Output an LLVM style file header to the specified
diff --git a/include/llvm/Target/Target.td b/include/llvm/Target/Target.td
index 1ff8db8c2e6..01c6e343208 100644
--- a/include/llvm/Target/Target.td
+++ b/include/llvm/Target/Target.td
@@ -381,6 +381,7 @@ class Instruction {
   bit hasPostISelHook = 0;  // To be *adjusted* after isel by target hook.
   bit hasCtrlDep   = 0;     // Does this instruction r/w ctrl-flow chains?
   bit isNotDuplicable = 0;  // Is it unsafe to duplicate this instruction?
+  bit isConvergent = 0;     // Is this instruction convergent?
   bit isAsCheapAsAMove = 0; // As cheap (or cheaper) than a move instruction.
   bit hasExtraSrcRegAllocReq = 0; // Sources have special regalloc requirement?
   bit hasExtraDefRegAllocReq = 0; // Defs have special regalloc requirement?
@@ -440,6 +441,30 @@ class Instruction {
   string PostEncoderMethod = "";
   string DecoderMethod = "";
 
+  // Is the instruction decoder method able to completely determine if the
+  // given instruction is valid or not. If the TableGen definition of the
+  // instruction specifies bitpattern A??B where A and B are static bits, the
+  // hasCompleteDecoder flag says whether the decoder method fully handles the
+  // ?? space, i.e. if it is a final arbiter for the instruction validity.
+  // If not then the decoder attempts to continue decoding when the decoder
+  // method fails.
+  //
+  // This allows to handle situations where the encoding is not fully
+  // orthogonal. Example:
+  // * InstA with bitpattern 0b0000????,
+  // * InstB with bitpattern 0b000000?? but the associated decoder method
+  //   DecodeInstB() returns Fail when ?? is 0b00 or 0b11.
+  //
+  // The decoder tries to decode a bitpattern that matches both InstA and
+  // InstB bitpatterns first as InstB (because it is the most specific
+  // encoding). In the default case (hasCompleteDecoder = 1), when
+  // DecodeInstB() returns Fail the bitpattern gets rejected. By setting
+  // hasCompleteDecoder = 0 in InstB, the decoder is informed that
+  // DecodeInstB() is not able to determine if all possible values of ?? are
+  // valid or not. If DecodeInstB() returns Fail the decoder will attempt to
+  // decode the bitpattern as InstA too.
+  bit hasCompleteDecoder = 1;
+
   /// Target-specific flags. This becomes the TSFlags field in TargetInstrDesc.
   bits<64> TSFlags = 0;
 
@@ -506,7 +531,7 @@ class Requires<list<Predicate> preds> {
 
 /// ops definition - This is just a simple marker used to identify the operand
 /// list for an instruction. outs and ins are identical both syntactically and
-/// semanticallyr; they are used to define def operands and use operands to
+/// semantically; they are used to define def operands and use operands to
 /// improve readibility. This should be used like this:
 ///     (outs R32:$dst), (ins R32:$src1, R32:$src2) or something similar.
 def ops;
@@ -594,6 +619,7 @@ class Operand<ValueType ty> : DAGOperand {
   string PrintMethod = "printOperand";
   string EncoderMethod = "";
   string DecoderMethod = "";
+  bit hasCompleteDecoder = 1;
   string OperandType = "OPERAND_UNKNOWN";
   dag MIOperandInfo = (ops);
 
@@ -871,7 +897,7 @@ def LOAD_STACK_GUARD : Instruction {
   let hasSideEffects = 0;
   bit isPseudo = 1;
 }
-def FRAME_ALLOC : Instruction {
+def LOCAL_ESCAPE : Instruction {
   // This instruction is really just a label. It has to be part of the chain so
   // that it doesn't get dropped from the DAG, but it produces nothing and has
   // no side effects.
@@ -880,6 +906,12 @@ def FRAME_ALLOC : Instruction {
   let hasSideEffects = 0;
   let hasCtrlDep = 1;
 }
+def FAULTING_LOAD_OP : Instruction {
+  let OutOperandList = (outs unknown:$dst);
+  let InOperandList = (ins variable_ops);
+  let usesCustomInserter = 1;
+  let mayLoad = 1;
+}
 }
 
 //===----------------------------------------------------------------------===//
@@ -1006,6 +1038,15 @@ class InstAlias<string Asm, dag Result, int Emit = 1> {
 
   // Predicates - Predicates that must be true for this to match.
   list<Predicate> Predicates = [];
+
+  // If the instruction specified in Result has defined an AsmMatchConverter
+  // then setting this to 1 will cause the alias to use the AsmMatchConverter
+  // function when converting the OperandVector into an MCInst instead of the
+  // function that is generated by the dag Result.
+  // Setting this to 0 will cause the alias to ignore the Result instruction's
+  // defined AsmMatchConverter and instead use the function generated by the
+  // dag Result.
+  bit UseInstAsmMatchConverter = 1;
 }
 
 //===----------------------------------------------------------------------===//
diff --git a/include/llvm/Target/TargetCallingConv.h b/include/llvm/Target/TargetCallingConv.h
index 9071bfeec7e..9d4e7a04d90 100644
--- a/include/llvm/Target/TargetCallingConv.h
+++ b/include/llvm/Target/TargetCallingConv.h
@@ -18,6 +18,7 @@
 #include "llvm/Support/DataTypes.h"
 #include "llvm/Support/MathExtras.h"
 #include <string>
+#include <limits.h>
 
 namespace llvm {
 
diff --git a/include/llvm/Target/TargetFrameLowering.h b/include/llvm/Target/TargetFrameLowering.h
index f17640f71e9..3af2227410f 100644
--- a/include/llvm/Target/TargetFrameLowering.h
+++ b/include/llvm/Target/TargetFrameLowering.h
@@ -19,6 +19,7 @@
 #include <vector>
 
 namespace llvm {
+  class BitVector;
   class CalleeSavedInfo;
   class MachineFunction;
   class RegScavenger;
@@ -130,21 +131,26 @@ class TargetFrameLowering {
 
   /// emitProlog/emitEpilog - These methods insert prolog and epilog code into
   /// the function.
-  virtual void emitPrologue(MachineFunction &MF) const = 0;
+  virtual void emitPrologue(MachineFunction &MF,
+                            MachineBasicBlock &MBB) const = 0;
   virtual void emitEpilogue(MachineFunction &MF,
                             MachineBasicBlock &MBB) const = 0;
 
   /// Adjust the prologue to have the function use segmented stacks. This works
   /// by adding a check even before the "normal" function prologue.
-  virtual void adjustForSegmentedStacks(MachineFunction &MF) const { }
+  virtual void adjustForSegmentedStacks(MachineFunction &MF,
+                                        MachineBasicBlock &PrologueMBB) const {}
 
   /// Adjust the prologue to add Erlang Run-Time System (ERTS) specific code in
   /// the assembly prologue to explicitly handle the stack.
-  virtual void adjustForHiPEPrologue(MachineFunction &MF) const { }
+  virtual void adjustForHiPEPrologue(MachineFunction &MF,
+                                     MachineBasicBlock &PrologueMBB) const {}
 
   /// Adjust the prologue to add an allocation at a fixed offset from the frame
   /// pointer.
-  virtual void adjustForFrameAllocatePrologue(MachineFunction &MF) const { }
+  virtual void
+  adjustForFrameAllocatePrologue(MachineFunction &MF,
+                                 MachineBasicBlock &PrologueMBB) const {}
 
   /// spillCalleeSavedRegisters - Issues instruction(s) to spill all callee
   /// saved registers and returns true if it isn't possible / profitable to do
@@ -168,6 +174,9 @@ class TargetFrameLowering {
     return false;
   }
 
+  /// Return true if the target needs to disable frame pointer elimination.
+  virtual bool noFramePointerElim(const MachineFunction &MF) const;
+
   /// hasFP - Return true if the specified function should have a dedicated
   /// frame pointer register. For most targets this is true only if the function
   /// has variable sized allocas or if frame pointer elimination is disabled.
@@ -218,13 +227,15 @@ class TargetFrameLowering {
     return 0;
   }
 
-  /// processFunctionBeforeCalleeSavedScan - This method is called immediately
-  /// before PrologEpilogInserter scans the physical registers used to determine
-  /// what callee saved registers should be spilled. This method is optional.
-  virtual void processFunctionBeforeCalleeSavedScan(MachineFunction &MF,
-                                             RegScavenger *RS = nullptr) const {
-
-  }
+  /// This method determines which of the registers reported by
+  /// TargetRegisterInfo::getCalleeSavedRegs() should actually get saved.
+  /// The default implementation checks populates the \p SavedRegs bitset with
+  /// all registers which are modified in the function, targets may override
+  /// this function to save additional registers.
+  /// This method also sets up the register scavenger ensuring there is a free
+  /// register or a frameindex available.
+  virtual void determineCalleeSaves(MachineFunction &MF, BitVector &SavedRegs,
+                                    RegScavenger *RS = nullptr) const;
 
   /// processFunctionBeforeFrameFinalized - This method is called immediately
   /// before the specified function's frame layout (MF.getFrameInfo()) is
@@ -249,6 +260,30 @@ class TargetFrameLowering {
     llvm_unreachable("Call Frame Pseudo Instructions do not exist on this "
                      "target!");
   }
+
+  /// Check whether or not the given \p MBB can be used as a prologue
+  /// for the target.
+  /// The prologue will be inserted first in this basic block.
+  /// This method is used by the shrink-wrapping pass to decide if
+  /// \p MBB will be correctly handled by the target.
+  /// As soon as the target enable shrink-wrapping without overriding
+  /// this method, we assume that each basic block is a valid
+  /// prologue.
+  virtual bool canUseAsPrologue(const MachineBasicBlock &MBB) const {
+    return true;
+  }
+
+  /// Check whether or not the given \p MBB can be used as a epilogue
+  /// for the target.
+  /// The epilogue will be inserted before the first terminator of that block.
+  /// This method is used by the shrink-wrapping pass to decide if
+  /// \p MBB will be correctly handled by the target.
+  /// As soon as the target enable shrink-wrapping without overriding
+  /// this method, we assume that each basic block is a valid
+  /// epilogue.
+  virtual bool canUseAsEpilogue(const MachineBasicBlock &MBB) const {
+    return true;
+  }
 };
 
 } // End llvm namespace
diff --git a/include/llvm/Target/TargetInstrInfo.h b/include/llvm/Target/TargetInstrInfo.h
index 110976a0a82..07b57713062 100644
--- a/include/llvm/Target/TargetInstrInfo.h
+++ b/include/llvm/Target/TargetInstrInfo.h
@@ -40,6 +40,7 @@ class TargetRegisterClass;
 class TargetRegisterInfo;
 class BranchProbability;
 class TargetSubtargetInfo;
+class TargetSchedModel;
 class DFAPacketizer;
 
 template<class T> class SmallVectorImpl;
@@ -53,23 +54,25 @@ class TargetInstrInfo : public MCInstrInfo {
   TargetInstrInfo(const TargetInstrInfo &) = delete;
   void operator=(const TargetInstrInfo &) = delete;
 public:
-  TargetInstrInfo(int CFSetupOpcode = -1, int CFDestroyOpcode = -1)
+  TargetInstrInfo(unsigned CFSetupOpcode = ~0u, unsigned CFDestroyOpcode = ~0u)
     : CallFrameSetupOpcode(CFSetupOpcode),
       CallFrameDestroyOpcode(CFDestroyOpcode) {
   }
 
   virtual ~TargetInstrInfo();
 
-  /// getRegClass - Givem a machine instruction descriptor, returns the register
+  /// Given a machine instruction descriptor, returns the register
   /// class constraint for OpNum, or NULL.
   const TargetRegisterClass *getRegClass(const MCInstrDesc &TID,
                                          unsigned OpNum,
                                          const TargetRegisterInfo *TRI,
                                          const MachineFunction &MF) const;
 
-  /// isTriviallyReMaterializable - Return true if the instruction is trivially
-  /// rematerializable, meaning it has no side effects and requires no operands
-  /// that aren't always available.
+  /// Return true if the instruction is trivially rematerializable, meaning it
+  /// has no side effects and requires no operands that aren't always available.
+  /// This means the only allowed uses are constants and unallocatable physical
+  /// registers so that the instructions result is independent of the place
+  /// in the function.
   bool isTriviallyReMaterializable(const MachineInstr *MI,
                                    AliasAnalysis *AA = nullptr) const {
     return MI->getOpcode() == TargetOpcode::IMPLICIT_DEF ||
@@ -79,35 +82,34 @@ class TargetInstrInfo : public MCInstrInfo {
   }
 
 protected:
-  /// isReallyTriviallyReMaterializable - For instructions with opcodes for
-  /// which the M_REMATERIALIZABLE flag is set, this hook lets the target
-  /// specify whether the instruction is actually trivially rematerializable,
-  /// taking into consideration its operands. This predicate must return false
-  /// if the instruction has any side effects other than producing a value, or
-  /// if it requres any address registers that are not always available.
+  /// For instructions with opcodes for which the M_REMATERIALIZABLE flag is
+  /// set, this hook lets the target specify whether the instruction is actually
+  /// trivially rematerializable, taking into consideration its operands. This
+  /// predicate must return false if the instruction has any side effects other
+  /// than producing a value, or if it requres any address registers that are
+  /// not always available.
+  /// Requirements must be check as stated in isTriviallyReMaterializable() .
   virtual bool isReallyTriviallyReMaterializable(const MachineInstr *MI,
                                                  AliasAnalysis *AA) const {
     return false;
   }
 
 private:
-  /// isReallyTriviallyReMaterializableGeneric - For instructions with opcodes
-  /// for which the M_REMATERIALIZABLE flag is set and the target hook
-  /// isReallyTriviallyReMaterializable returns false, this function does
-  /// target-independent tests to determine if the instruction is really
-  /// trivially rematerializable.
+  /// For instructions with opcodes for which the M_REMATERIALIZABLE flag is
+  /// set and the target hook isReallyTriviallyReMaterializable returns false,
+  /// this function does target-independent tests to determine if the
+  /// instruction is really trivially rematerializable.
   bool isReallyTriviallyReMaterializableGeneric(const MachineInstr *MI,
                                                 AliasAnalysis *AA) const;
 
 public:
-  /// getCallFrameSetup/DestroyOpcode - These methods return the opcode of the
-  /// frame setup/destroy instructions if they exist (-1 otherwise).  Some
-  /// targets use pseudo instructions in order to abstract away the difference
-  /// between operating with a frame pointer and operating without, through the
-  /// use of these two instructions.
+  /// These methods return the opcode of the frame setup/destroy instructions
+  /// if they exist (-1 otherwise).  Some targets use pseudo instructions in
+  /// order to abstract away the difference between operating with a frame
+  /// pointer and operating without, through the use of these two instructions.
   ///
-  int getCallFrameSetupOpcode() const { return CallFrameSetupOpcode; }
-  int getCallFrameDestroyOpcode() const { return CallFrameDestroyOpcode; }
+  unsigned getCallFrameSetupOpcode() const { return CallFrameSetupOpcode; }
+  unsigned getCallFrameDestroyOpcode() const { return CallFrameDestroyOpcode; }
 
   /// Returns the actual stack pointer adjustment made by an instruction
   /// as part of a call sequence. By default, only call frame setup/destroy
@@ -115,19 +117,18 @@ class TargetInstrInfo : public MCInstrInfo {
   /// to enable more fine-grained adjustment, or adjust by a different value.
   virtual int getSPAdjust(const MachineInstr *MI) const;
 
-  /// isCoalescableExtInstr - Return true if the instruction is a "coalescable"
-  /// extension instruction. That is, it's like a copy where it's legal for the
-  /// source to overlap the destination. e.g. X86::MOVSX64rr32. If this returns
-  /// true, then it's expected the pre-extension value is available as a subreg
-  /// of the result register. This also returns the sub-register index in
-  /// SubIdx.
+  /// Return true if the instruction is a "coalescable" extension instruction.
+  /// That is, it's like a copy where it's legal for the source to overlap the
+  /// destination. e.g. X86::MOVSX64rr32. If this returns true, then it's
+  /// expected the pre-extension value is available as a subreg of the result
+  /// register. This also returns the sub-register index in SubIdx.
   virtual bool isCoalescableExtInstr(const MachineInstr &MI,
                                      unsigned &SrcReg, unsigned &DstReg,
                                      unsigned &SubIdx) const {
     return false;
   }
 
-  /// isLoadFromStackSlot - If the specified machine instruction is a direct
+  /// If the specified machine instruction is a direct
   /// load from a stack slot, return the virtual or physical register number of
   /// the destination along with the FrameIndex of the loaded stack slot.  If
   /// not, return 0.  This predicate must return 0 if the instruction has
@@ -137,26 +138,24 @@ class TargetInstrInfo : public MCInstrInfo {
     return 0;
   }
 
-  /// isLoadFromStackSlotPostFE - Check for post-frame ptr elimination
-  /// stack locations as well.  This uses a heuristic so it isn't
-  /// reliable for correctness.
+  /// Check for post-frame ptr elimination stack locations as well.
+  /// This uses a heuristic so it isn't reliable for correctness.
   virtual unsigned isLoadFromStackSlotPostFE(const MachineInstr *MI,
                                              int &FrameIndex) const {
     return 0;
   }
 
-  /// hasLoadFromStackSlot - If the specified machine instruction has
-  /// a load from a stack slot, return true along with the FrameIndex
-  /// of the loaded stack slot and the machine mem operand containing
-  /// the reference.  If not, return false.  Unlike
-  /// isLoadFromStackSlot, this returns true for any instructions that
-  /// loads from the stack.  This is just a hint, as some cases may be
-  /// missed.
+  /// If the specified machine instruction has a load from a stack slot,
+  /// return true along with the FrameIndex of the loaded stack slot and the
+  /// machine mem operand containing the reference.
+  /// If not, return false.  Unlike isLoadFromStackSlot, this returns true for
+  /// any instructions that loads from the stack.  This is just a hint, as some
+  /// cases may be missed.
   virtual bool hasLoadFromStackSlot(const MachineInstr *MI,
                                     const MachineMemOperand *&MMO,
                                     int &FrameIndex) const;
 
-  /// isStoreToStackSlot - If the specified machine instruction is a direct
+  /// If the specified machine instruction is a direct
   /// store to a stack slot, return the virtual or physical register number of
   /// the source reg along with the FrameIndex of the loaded stack slot.  If
   /// not, return 0.  This predicate must return 0 if the instruction has
@@ -166,25 +165,24 @@ class TargetInstrInfo : public MCInstrInfo {
     return 0;
   }
 
-  /// isStoreToStackSlotPostFE - Check for post-frame ptr elimination
-  /// stack locations as well.  This uses a heuristic so it isn't
-  /// reliable for correctness.
+  /// Check for post-frame ptr elimination stack locations as well.
+  /// This uses a heuristic, so it isn't reliable for correctness.
   virtual unsigned isStoreToStackSlotPostFE(const MachineInstr *MI,
                                             int &FrameIndex) const {
     return 0;
   }
 
-  /// hasStoreToStackSlot - If the specified machine instruction has a
-  /// store to a stack slot, return true along with the FrameIndex of
-  /// the loaded stack slot and the machine mem operand containing the
-  /// reference.  If not, return false.  Unlike isStoreToStackSlot,
+  /// If the specified machine instruction has a store to a stack slot,
+  /// return true along with the FrameIndex of the loaded stack slot and the
+  /// machine mem operand containing the reference.
+  /// If not, return false.  Unlike isStoreToStackSlot,
   /// this returns true for any instructions that stores to the
   /// stack.  This is just a hint, as some cases may be missed.
   virtual bool hasStoreToStackSlot(const MachineInstr *MI,
                                    const MachineMemOperand *&MMO,
                                    int &FrameIndex) const;
 
-  /// isStackSlotCopy - Return true if the specified machine instruction
+  /// Return true if the specified machine instruction
   /// is a copy of one stack slot to another and has no other effect.
   /// Provide the identity of the two frame indices.
   virtual bool isStackSlotCopy(const MachineInstr *MI, int &DestFrameIndex,
@@ -209,8 +207,7 @@ class TargetInstrInfo : public MCInstrInfo {
                                  unsigned &Size, unsigned &Offset,
                                  const MachineFunction &MF) const;
 
-  /// isAsCheapAsAMove - Return true if the instruction is as cheap as a move
-  /// instruction.
+  /// Return true if the instruction is as cheap as a move instruction.
   ///
   /// Targets for different archs need to override this, and different
   /// micro-architectures can also be finely tuned inside.
@@ -218,7 +215,7 @@ class TargetInstrInfo : public MCInstrInfo {
     return MI->isAsCheapAsAMove();
   }
 
-  /// reMaterialize - Re-issue the specified 'original' instruction at the
+  /// Re-issue the specified 'original' instruction at the
   /// specific location targeting a new destination register.
   /// The register in Orig->getOperand(0).getReg() will be substituted by
   /// DestReg:SubIdx. Any existing subreg index is preserved or composed with
@@ -229,7 +226,7 @@ class TargetInstrInfo : public MCInstrInfo {
                              const MachineInstr *Orig,
                              const TargetRegisterInfo &TRI) const;
 
-  /// duplicate - Create a duplicate of the Orig instruction in MF. This is like
+  /// Create a duplicate of the Orig instruction in MF. This is like
   /// MachineFunction::CloneMachineInstr(), but the target may update operands
   /// that are required to be unique.
   ///
@@ -237,7 +234,7 @@ class TargetInstrInfo : public MCInstrInfo {
   virtual MachineInstr *duplicate(MachineInstr *Orig,
                                   MachineFunction &MF) const;
 
-  /// convertToThreeAddress - This method must be implemented by targets that
+  /// This method must be implemented by targets that
   /// set the M_CONVERTIBLE_TO_3_ADDR flag.  When this flag is set, the target
   /// may be able to convert a two-address instruction into one or more true
   /// three-address instructions on demand.  This allows the X86 target (for
@@ -253,10 +250,10 @@ class TargetInstrInfo : public MCInstrInfo {
     return nullptr;
   }
 
-  /// commuteInstruction - If a target has any instructions that are
-  /// commutable but require converting to different instructions or making
-  /// non-trivial changes to commute them, this method can overloaded to do
-  /// that.  The default implementation simply swaps the commutable operands.
+  /// If a target has any instructions that are commutable but require
+  /// converting to different instructions or making non-trivial changes to
+  /// commute them, this method can overloaded to do that.
+  /// The default implementation simply swaps the commutable operands.
   /// If NewMI is false, MI is modified in place and returned; otherwise, a
   /// new machine instruction is created and returned.  Do not call this
   /// method for a non-commutable instruction, but there may be some cases
@@ -264,8 +261,8 @@ class TargetInstrInfo : public MCInstrInfo {
   virtual MachineInstr *commuteInstruction(MachineInstr *MI,
                                            bool NewMI = false) const;
 
-  /// findCommutedOpIndices - If specified MI is commutable, return the two
-  /// operand indices that would swap value. Return false if the instruction
+  /// If specified MI is commutable, return the two operand indices that would
+  /// swap value. Return false if the instruction
   /// is not in a form which this routine understands.
   virtual bool findCommutedOpIndices(MachineInstr *MI, unsigned &SrcOpIdx1,
                                      unsigned &SrcOpIdx2) const;
@@ -349,8 +346,8 @@ class TargetInstrInfo : public MCInstrInfo {
                         RegSubRegPairAndIdx &InsertedReg) const;
 
 
-  /// produceSameValue - Return true if two machine instructions would produce
-  /// identical values. By default, this is only true when the two instructions
+  /// Return true if two machine instructions would produce identical values.
+  /// By default, this is only true when the two instructions
   /// are deemed identical except for defs. If this function is called when the
   /// IR is still in SSA form, the caller can pass the MachineRegisterInfo for
   /// aggressive checks.
@@ -358,7 +355,7 @@ class TargetInstrInfo : public MCInstrInfo {
                                 const MachineInstr *MI1,
                                 const MachineRegisterInfo *MRI = nullptr) const;
 
-  /// AnalyzeBranch - Analyze the branching code at the end of MBB, returning
+  /// Analyze the branching code at the end of MBB, returning
   /// true if it cannot be understood (e.g. it's a switch dispatch or isn't
   /// implemented for a target).  Upon success, this returns false and returns
   /// with the following information in various cases:
@@ -390,15 +387,60 @@ class TargetInstrInfo : public MCInstrInfo {
     return true;
   }
 
-  /// RemoveBranch - Remove the branching code at the end of the specific MBB.
+  /// Represents a predicate at the MachineFunction level.  The control flow a
+  /// MachineBranchPredicate represents is:
+  ///
+  ///  Reg <def>= LHS `Predicate` RHS         == ConditionDef
+  ///  if Reg then goto TrueDest else goto FalseDest
+  ///
+  struct MachineBranchPredicate {
+    enum ComparePredicate {
+      PRED_EQ,     // True if two values are equal
+      PRED_NE,     // True if two values are not equal
+      PRED_INVALID // Sentinel value
+    };
+
+    ComparePredicate Predicate;
+    MachineOperand LHS;
+    MachineOperand RHS;
+    MachineBasicBlock *TrueDest;
+    MachineBasicBlock *FalseDest;
+    MachineInstr *ConditionDef;
+
+    /// SingleUseCondition is true if ConditionDef is dead except for the
+    /// branch(es) at the end of the basic block.
+    ///
+    bool SingleUseCondition;
+
+    explicit MachineBranchPredicate()
+        : Predicate(PRED_INVALID), LHS(MachineOperand::CreateImm(0)),
+          RHS(MachineOperand::CreateImm(0)), TrueDest(nullptr),
+          FalseDest(nullptr), ConditionDef(nullptr), SingleUseCondition(false) {
+    }
+  };
+
+  /// Analyze the branching code at the end of MBB and parse it into the
+  /// MachineBranchPredicate structure if possible.  Returns false on success
+  /// and true on failure.
+  ///
+  /// If AllowModify is true, then this routine is allowed to modify the basic
+  /// block (e.g. delete instructions after the unconditional branch).
+  ///
+  virtual bool AnalyzeBranchPredicate(MachineBasicBlock &MBB,
+                                      MachineBranchPredicate &MBP,
+                                      bool AllowModify = false) const {
+    return true;
+  }
+
+  /// Remove the branching code at the end of the specific MBB.
   /// This is only invoked in cases where AnalyzeBranch returns success. It
   /// returns the number of instructions that were removed.
   virtual unsigned RemoveBranch(MachineBasicBlock &MBB) const {
     llvm_unreachable("Target didn't implement TargetInstrInfo::RemoveBranch!");
   }
 
-  /// InsertBranch - Insert branch code into the end of the specified
-  /// MachineBasicBlock.  The operands to this method are the same as those
+  /// Insert branch code into the end of the specified MachineBasicBlock.
+  /// The operands to this method are the same as those
   /// returned by AnalyzeBranch.  This is only invoked in cases where
   /// AnalyzeBranch returns success. It returns the number of instructions
   /// inserted.
@@ -409,19 +451,18 @@ class TargetInstrInfo : public MCInstrInfo {
   /// merging needs to be disabled.
   virtual unsigned InsertBranch(MachineBasicBlock &MBB, MachineBasicBlock *TBB,
                                 MachineBasicBlock *FBB,
-                                const SmallVectorImpl<MachineOperand> &Cond,
+                                ArrayRef<MachineOperand> Cond,
                                 DebugLoc DL) const {
     llvm_unreachable("Target didn't implement TargetInstrInfo::InsertBranch!");
   }
 
-  /// ReplaceTailWithBranchTo - Delete the instruction OldInst and everything
-  /// after it, replacing it with an unconditional branch to NewDest. This is
-  /// used by the tail merging pass.
+  /// Delete the instruction OldInst and everything after it, replacing it with
+  /// an unconditional branch to NewDest. This is used by the tail merging pass.
   virtual void ReplaceTailWithBranchTo(MachineBasicBlock::iterator Tail,
                                        MachineBasicBlock *NewDest) const;
 
-  /// getUnconditionalBranch - Get an instruction that performs an unconditional
-  /// branch to the given symbol.
+  /// Get an instruction that performs an unconditional branch to the given
+  /// symbol.
   virtual void
   getUnconditionalBranch(MCInst &MI,
                          const MCSymbolRefExpr *BranchTarget) const {
@@ -429,12 +470,12 @@ class TargetInstrInfo : public MCInstrInfo {
                      "TargetInstrInfo::getUnconditionalBranch!");
   }
 
-  /// getTrap - Get a machine trap instruction
+  /// Get a machine trap instruction.
   virtual void getTrap(MCInst &MI) const {
     llvm_unreachable("Target didn't implement TargetInstrInfo::getTrap!");
   }
 
-  /// getJumpInstrTableEntryBound - Get a number of bytes that suffices to hold
+  /// Get a number of bytes that suffices to hold
   /// either the instruction returned by getUnconditionalBranch or the
   /// instruction returned by getTrap. This only makes sense because
   /// getUnconditionalBranch returns a single, specific instruction. This
@@ -454,7 +495,7 @@ class TargetInstrInfo : public MCInstrInfo {
     return 0;
   }
 
-  /// isLegalToSplitMBBAt - Return true if it's legal to split the given basic
+  /// Return true if it's legal to split the given basic
   /// block at the specified instruction (i.e. instruction would be the start
   /// of a new basic block).
   virtual bool isLegalToSplitMBBAt(MachineBasicBlock &MBB,
@@ -462,7 +503,7 @@ class TargetInstrInfo : public MCInstrInfo {
     return true;
   }
 
-  /// isProfitableToIfCvt - Return true if it's profitable to predicate
+  /// Return true if it's profitable to predicate
   /// instructions with accumulated instruction latency of "NumCycles"
   /// of the specified basic block, where the probability of the instructions
   /// being executed is given by Probability, and Confidence is a measure
@@ -474,7 +515,7 @@ class TargetInstrInfo : public MCInstrInfo {
     return false;
   }
 
-  /// isProfitableToIfCvt - Second variant of isProfitableToIfCvt, this one
+  /// Second variant of isProfitableToIfCvt. This one
   /// checks for the case where two basic blocks from true and false path
   /// of a if-then-else (diamond) are predicated on mutally exclusive
   /// predicates, where the probability of the true path being taken is given
@@ -489,9 +530,9 @@ class TargetInstrInfo : public MCInstrInfo {
     return false;
   }
 
-  /// isProfitableToDupForIfCvt - Return true if it's profitable for
-  /// if-converter to duplicate instructions of specified accumulated
-  /// instruction latencies in the specified MBB to enable if-conversion.
+  /// Return true if it's profitable for if-converter to duplicate instructions
+  /// of specified accumulated instruction latencies in the specified MBB to
+  /// enable if-conversion.
   /// The probability of the instructions being executed is given by
   /// Probability, and Confidence is a measure of our confidence that it
   /// will be properly predicted.
@@ -501,7 +542,7 @@ class TargetInstrInfo : public MCInstrInfo {
     return false;
   }
 
-  /// isProfitableToUnpredicate - Return true if it's profitable to unpredicate
+  /// Return true if it's profitable to unpredicate
   /// one side of a 'diamond', i.e. two sides of if-else predicated on mutually
   /// exclusive predicates.
   /// e.g.
@@ -517,7 +558,7 @@ class TargetInstrInfo : public MCInstrInfo {
     return false;
   }
 
-  /// canInsertSelect - Return true if it is possible to insert a select
+  /// Return true if it is possible to insert a select
   /// instruction that chooses between TrueReg and FalseReg based on the
   /// condition code in Cond.
   ///
@@ -535,16 +576,15 @@ class TargetInstrInfo : public MCInstrInfo {
   /// @param TrueCycles  Latency from TrueReg to select output.
   /// @param FalseCycles Latency from FalseReg to select output.
   virtual bool canInsertSelect(const MachineBasicBlock &MBB,
-                               const SmallVectorImpl<MachineOperand> &Cond,
+                               ArrayRef<MachineOperand> Cond,
                                unsigned TrueReg, unsigned FalseReg,
                                int &CondCycles,
                                int &TrueCycles, int &FalseCycles) const {
     return false;
   }
 
-  /// insertSelect - Insert a select instruction into MBB before I that will
-  /// copy TrueReg to DstReg when Cond is true, and FalseReg to DstReg when
-  /// Cond is false.
+  /// Insert a select instruction into MBB before I that will copy TrueReg to
+  /// DstReg when Cond is true, and FalseReg to DstReg when Cond is false.
   ///
   /// This function can only be called after canInsertSelect() returned true.
   /// The condition in Cond comes from AnalyzeBranch, and it can be assumed
@@ -560,13 +600,12 @@ class TargetInstrInfo : public MCInstrInfo {
   /// @param FalseReg Virtual register to copy when Cons is false.
   virtual void insertSelect(MachineBasicBlock &MBB,
                             MachineBasicBlock::iterator I, DebugLoc DL,
-                            unsigned DstReg,
-                            const SmallVectorImpl<MachineOperand> &Cond,
+                            unsigned DstReg, ArrayRef<MachineOperand> Cond,
                             unsigned TrueReg, unsigned FalseReg) const {
     llvm_unreachable("Target didn't implement TargetInstrInfo::insertSelect!");
   }
 
-  /// analyzeSelect - Analyze the given select instruction, returning true if
+  /// Analyze the given select instruction, returning true if
   /// it cannot be understood. It is assumed that MI->isSelect() is true.
   ///
   /// When successful, return the controlling condition and the operands that
@@ -592,7 +631,7 @@ class TargetInstrInfo : public MCInstrInfo {
     return true;
   }
 
-  /// optimizeSelect - Given a select instruction that was understood by
+  /// Given a select instruction that was understood by
   /// analyzeSelect and returned Optimizable = true, attempt to optimize MI by
   /// merging it with one of its operands. Returns NULL on failure.
   ///
@@ -614,7 +653,7 @@ class TargetInstrInfo : public MCInstrInfo {
     llvm_unreachable("Target must implement TargetInstrInfo::optimizeSelect!");
   }
 
-  /// copyPhysReg - Emit instructions to copy a pair of physical registers.
+  /// Emit instructions to copy a pair of physical registers.
   ///
   /// This function should support copies within any legal register class as
   /// well as any cross-class copies created during instruction selection.
@@ -629,11 +668,10 @@ class TargetInstrInfo : public MCInstrInfo {
     llvm_unreachable("Target didn't implement TargetInstrInfo::copyPhysReg!");
   }
 
-  /// storeRegToStackSlot - Store the specified register of the given register
-  /// class to the specified stack frame index. The store instruction is to be
-  /// added to the given machine basic block before the specified machine
-  /// instruction. If isKill is true, the register operand is the last use and
-  /// must be marked kill.
+  /// Store the specified register of the given register class to the specified
+  /// stack frame index. The store instruction is to be added to the given
+  /// machine basic block before the specified machine instruction. If isKill
+  /// is true, the register operand is the last use and must be marked kill.
   virtual void storeRegToStackSlot(MachineBasicBlock &MBB,
                                    MachineBasicBlock::iterator MI,
                                    unsigned SrcReg, bool isKill, int FrameIndex,
@@ -643,10 +681,9 @@ class TargetInstrInfo : public MCInstrInfo {
                      "TargetInstrInfo::storeRegToStackSlot!");
   }
 
-  /// loadRegFromStackSlot - Load the specified register of the given register
-  /// class from the specified stack frame index. The load instruction is to be
-  /// added to the given machine basic block before the specified machine
-  /// instruction.
+  /// Load the specified register of the given register class from the specified
+  /// stack frame index. The load instruction is to be added to the given
+  /// machine basic block before the specified machine instruction.
   virtual void loadRegFromStackSlot(MachineBasicBlock &MBB,
                                     MachineBasicBlock::iterator MI,
                                     unsigned DestReg, int FrameIndex,
@@ -656,7 +693,7 @@ class TargetInstrInfo : public MCInstrInfo {
                      "TargetInstrInfo::loadRegFromStackSlot!");
   }
 
-  /// expandPostRAPseudo - This function is called for all pseudo instructions
+  /// This function is called for all pseudo instructions
   /// that remain after register allocation. Many pseudo instructions are
   /// created to help register allocation. This is the place to convert them
   /// into real instructions. The target can edit MI in place, or it can insert
@@ -666,7 +703,7 @@ class TargetInstrInfo : public MCInstrInfo {
     return false;
   }
 
-  /// foldMemoryOperand - Attempt to fold a load or store of the specified stack
+  /// Attempt to fold a load or store of the specified stack
   /// slot into the specified machine instruction for the specified operand(s).
   /// If this is possible, a new instruction is returned with the specified
   /// operand folded, otherwise NULL is returned.
@@ -675,67 +712,66 @@ class TargetInstrInfo : public MCInstrInfo {
   MachineInstr *foldMemoryOperand(MachineBasicBlock::iterator MI,
                                   ArrayRef<unsigned> Ops, int FrameIndex) const;
 
-  /// foldMemoryOperand - Same as the previous version except it allows folding
-  /// of any load and store from / to any address, not just from a specific
-  /// stack slot.
+  /// Same as the previous version except it allows folding of any load and
+  /// store from / to any address, not just from a specific stack slot.
   MachineInstr *foldMemoryOperand(MachineBasicBlock::iterator MI,
                                   ArrayRef<unsigned> Ops,
                                   MachineInstr *LoadMI) const;
 
-  /// hasPattern - return true when there is potentially a faster code sequence
-  /// for an instruction chain ending in \p Root. All potential pattern are
+  /// Return true when there is potentially a faster code sequence
+  /// for an instruction chain ending in \p Root. All potential patterns are
   /// returned in the \p Pattern vector. Pattern should be sorted in priority
   /// order since the pattern evaluator stops checking as soon as it finds a
   /// faster sequence.
   /// \param Root - Instruction that could be combined with one of its operands
-  /// \param Pattern - Vector of possible combination pattern
-
-  virtual bool hasPattern(
+  /// \param Pattern - Vector of possible combination patterns
+  virtual bool getMachineCombinerPatterns(
       MachineInstr &Root,
       SmallVectorImpl<MachineCombinerPattern::MC_PATTERN> &Pattern) const {
     return false;
   }
 
-  /// genAlternativeCodeSequence - when hasPattern() finds a pattern this
-  /// function generates the instructions that could replace the original code
-  /// sequence. The client has to decide whether the actual replacementment is
-  /// beneficial or not.
+  /// When getMachineCombinerPatterns() finds patterns, this function generates
+  /// the instructions that could replace the original code sequence. The client
+  /// has to decide whether the actual replacement is beneficial or not.
   /// \param Root - Instruction that could be combined with one of its operands
-  /// \param P - Combination pattern for Root
+  /// \param Pattern - Combination pattern for Root
   /// \param InsInstrs - Vector of new instructions that implement P
-  /// \param DelInstrs - Old instructions, including Root, that could be replaced
-  /// by InsInstr
+  /// \param DelInstrs - Old instructions, including Root, that could be
+  /// replaced by InsInstr
   /// \param InstrIdxForVirtReg - map of virtual register to instruction in
   /// InsInstr that defines it
   virtual void genAlternativeCodeSequence(
-      MachineInstr &Root, MachineCombinerPattern::MC_PATTERN P,
+      MachineInstr &Root, MachineCombinerPattern::MC_PATTERN Pattern,
       SmallVectorImpl<MachineInstr *> &InsInstrs,
       SmallVectorImpl<MachineInstr *> &DelInstrs,
       DenseMap<unsigned, unsigned> &InstrIdxForVirtReg) const {
     return;
   }
 
-  /// useMachineCombiner - return true when a target supports MachineCombiner
+  /// Return true when a target supports MachineCombiner.
   virtual bool useMachineCombiner() const { return false; }
 
 protected:
-  /// foldMemoryOperandImpl - Target-dependent implementation for
-  /// foldMemoryOperand. Target-independent code in foldMemoryOperand will
+  /// Target-dependent implementation for foldMemoryOperand.
+  /// Target-independent code in foldMemoryOperand will
   /// take care of adding a MachineMemOperand to the newly created instruction.
-  virtual MachineInstr *foldMemoryOperandImpl(MachineFunction &MF,
-                                              MachineInstr *MI,
-                                              ArrayRef<unsigned> Ops,
-                                              int FrameIndex) const {
+  /// The instruction and any auxiliary instructions necessary will be inserted
+  /// at InsertPt.
+  virtual MachineInstr *foldMemoryOperandImpl(
+      MachineFunction &MF, MachineInstr *MI, ArrayRef<unsigned> Ops,
+      MachineBasicBlock::iterator InsertPt, int FrameIndex) const {
     return nullptr;
   }
 
-  /// foldMemoryOperandImpl - Target-dependent implementation for
-  /// foldMemoryOperand. Target-independent code in foldMemoryOperand will
+  /// Target-dependent implementation for foldMemoryOperand.
+  /// Target-independent code in foldMemoryOperand will
   /// take care of adding a MachineMemOperand to the newly created instruction.
-  virtual MachineInstr *foldMemoryOperandImpl(MachineFunction &MF,
-                                              MachineInstr *MI,
-                                              ArrayRef<unsigned> Ops,
-                                              MachineInstr *LoadMI) const {
+  /// The instruction and any auxiliary instructions necessary will be inserted
+  /// at InsertPt.
+  virtual MachineInstr *foldMemoryOperandImpl(
+      MachineFunction &MF, MachineInstr *MI, ArrayRef<unsigned> Ops,
+      MachineBasicBlock::iterator InsertPt, MachineInstr *LoadMI) const {
     return nullptr;
   }
 
@@ -783,11 +819,6 @@ class TargetInstrInfo : public MCInstrInfo {
   }
 
 public:
-  /// canFoldMemoryOperand - Returns true for the specified load / store if
-  /// folding is possible.
-  virtual bool canFoldMemoryOperand(const MachineInstr *MI,
-                                    ArrayRef<unsigned> Ops) const;
-
   /// unfoldMemoryOperand - Separate a single instruction which folded a load or
   /// a store or a load and a store into two or more instruction. If this is
   /// possible, returns true as well as the new instructions by reference.
@@ -802,7 +833,7 @@ class TargetInstrInfo : public MCInstrInfo {
     return false;
   }
 
-  /// getOpcodeAfterMemoryUnfold - Returns the opcode of the would be new
+  /// Returns the opcode of the would be new
   /// instruction after load / store are unfolded from an instruction of the
   /// specified opcode. It returns zero if the specified unfolding is not
   /// possible. If LoadRegIndex is non-null, it is filled in with the operand
@@ -814,19 +845,18 @@ class TargetInstrInfo : public MCInstrInfo {
     return 0;
   }
 
-  /// areLoadsFromSameBasePtr - This is used by the pre-regalloc scheduler
-  /// to determine if two loads are loading from the same base address. It
-  /// should only return true if the base pointers are the same and the
-  /// only differences between the two addresses are the offset. It also returns
-  /// the offsets by reference.
+  /// This is used by the pre-regalloc scheduler to determine if two loads are
+  /// loading from the same base address. It should only return true if the base
+  /// pointers are the same and the only differences between the two addresses
+  /// are the offset. It also returns the offsets by reference.
   virtual bool areLoadsFromSameBasePtr(SDNode *Load1, SDNode *Load2,
                                     int64_t &Offset1, int64_t &Offset2) const {
     return false;
   }
 
-  /// shouldScheduleLoadsNear - This is a used by the pre-regalloc scheduler to
-  /// determine (in conjunction with areLoadsFromSameBasePtr) if two loads should
-  /// be scheduled togther. On some targets if two loads are loading from
+  /// This is a used by the pre-regalloc scheduler to determine (in conjunction
+  /// with areLoadsFromSameBasePtr) if two loads should be scheduled together.
+  /// On some targets if two loads are loading from
   /// addresses in the same cache line, it's better if they are scheduled
   /// together. This function takes two integers that represent the load offsets
   /// from the common base address. It returns true if it decides it's desirable
@@ -838,10 +868,11 @@ class TargetInstrInfo : public MCInstrInfo {
     return false;
   }
 
-  /// \brief Get the base register and byte offset of a load/store instr.
-  virtual bool getLdStBaseRegImmOfs(MachineInstr *LdSt,
-                                    unsigned &BaseReg, unsigned &Offset,
-                                    const TargetRegisterInfo *TRI) const {
+  /// Get the base register and byte offset of an instruction that reads/writes
+  /// memory.
+  virtual bool getMemOpBaseRegImmOfs(MachineInstr *MemOp, unsigned &BaseReg,
+                                     unsigned &Offset,
+                                     const TargetRegisterInfo *TRI) const {
     return false;
   }
 
@@ -853,23 +884,21 @@ class TargetInstrInfo : public MCInstrInfo {
     return false;
   }
 
-  /// \brief Can this target fuse the given instructions if they are scheduled
+  /// Can this target fuse the given instructions if they are scheduled
   /// adjacent.
   virtual bool shouldScheduleAdjacent(MachineInstr* First,
                                       MachineInstr *Second) const {
     return false;
   }
 
-  /// ReverseBranchCondition - Reverses the branch condition of the specified
-  /// condition list, returning false on success and true if it cannot be
-  /// reversed.
+  /// Reverses the branch condition of the specified condition list,
+  /// returning false on success and true if it cannot be reversed.
   virtual
   bool ReverseBranchCondition(SmallVectorImpl<MachineOperand> &Cond) const {
     return true;
   }
 
-  /// insertNoop - Insert a noop into the instruction stream at the specified
-  /// point.
+  /// Insert a noop into the instruction stream at the specified point.
   virtual void insertNoop(MachineBasicBlock &MBB,
                           MachineBasicBlock::iterator MI) const;
 
@@ -878,31 +907,30 @@ class TargetInstrInfo : public MCInstrInfo {
   virtual void getNoopForMachoTarget(MCInst &NopInst) const;
 
 
-  /// isPredicated - Returns true if the instruction is already predicated.
-  ///
+  /// Returns true if the instruction is already predicated.
   virtual bool isPredicated(const MachineInstr *MI) const {
     return false;
   }
 
-  /// isUnpredicatedTerminator - Returns true if the instruction is a
+  /// Returns true if the instruction is a
   /// terminator instruction that has not been predicated.
   virtual bool isUnpredicatedTerminator(const MachineInstr *MI) const;
 
-  /// PredicateInstruction - Convert the instruction into a predicated
-  /// instruction. It returns true if the operation was successful.
+  /// Convert the instruction into a predicated instruction.
+  /// It returns true if the operation was successful.
   virtual
   bool PredicateInstruction(MachineInstr *MI,
-                        const SmallVectorImpl<MachineOperand> &Pred) const;
+                            ArrayRef<MachineOperand> Pred) const;
 
-  /// SubsumesPredicate - Returns true if the first specified predicate
+  /// Returns true if the first specified predicate
   /// subsumes the second, e.g. GE subsumes GT.
   virtual
-  bool SubsumesPredicate(const SmallVectorImpl<MachineOperand> &Pred1,
-                         const SmallVectorImpl<MachineOperand> &Pred2) const {
+  bool SubsumesPredicate(ArrayRef<MachineOperand> Pred1,
+                         ArrayRef<MachineOperand> Pred2) const {
     return false;
   }
 
-  /// DefinesPredicate - If the specified instruction defines any predicate
+  /// If the specified instruction defines any predicate
   /// or condition code register(s) used for predication, returns true as well
   /// as the definition predicate(s) by reference.
   virtual bool DefinesPredicate(MachineInstr *MI,
@@ -910,22 +938,21 @@ class TargetInstrInfo : public MCInstrInfo {
     return false;
   }
 
-  /// isPredicable - Return true if the specified instruction can be predicated.
+  /// Return true if the specified instruction can be predicated.
   /// By default, this returns true for every instruction with a
   /// PredicateOperand.
   virtual bool isPredicable(MachineInstr *MI) const {
     return MI->getDesc().isPredicable();
   }
 
-  /// isSafeToMoveRegClassDefs - Return true if it's safe to move a machine
+  /// Return true if it's safe to move a machine
   /// instruction that defines the specified register class.
   virtual bool isSafeToMoveRegClassDefs(const TargetRegisterClass *RC) const {
     return true;
   }
 
-  /// isSchedulingBoundary - Test if the given instruction should be
-  /// considered a scheduling boundary. This primarily includes labels and
-  /// terminators.
+  /// Test if the given instruction should be considered a scheduling boundary.
+  /// This primarily includes labels and terminators.
   virtual bool isSchedulingBoundary(const MachineInstr *MI,
                                     const MachineBasicBlock *MBB,
                                     const MachineFunction &MF) const;
@@ -935,23 +962,20 @@ class TargetInstrInfo : public MCInstrInfo {
   virtual unsigned getInlineAsmLength(const char *Str,
                                       const MCAsmInfo &MAI) const;
 
-  /// CreateTargetHazardRecognizer - Allocate and return a hazard recognizer to
-  /// use for this target when scheduling the machine instructions before
-  /// register allocation.
+  /// Allocate and return a hazard recognizer to use for this target when
+  /// scheduling the machine instructions before register allocation.
   virtual ScheduleHazardRecognizer*
   CreateTargetHazardRecognizer(const TargetSubtargetInfo *STI,
                                const ScheduleDAG *DAG) const;
 
-  /// CreateTargetMIHazardRecognizer - Allocate and return a hazard recognizer
-  /// to use for this target when scheduling the machine instructions before
-  /// register allocation.
+  /// Allocate and return a hazard recognizer to use for this target when
+  /// scheduling the machine instructions before register allocation.
   virtual ScheduleHazardRecognizer*
   CreateTargetMIHazardRecognizer(const InstrItineraryData*,
                                  const ScheduleDAG *DAG) const;
 
-  /// CreateTargetPostRAHazardRecognizer - Allocate and return a hazard
-  /// recognizer to use for this target when scheduling the machine instructions
-  /// after register allocation.
+  /// Allocate and return a hazard recognizer to use for this target when
+  /// scheduling the machine instructions after register allocation.
   virtual ScheduleHazardRecognizer*
   CreateTargetPostRAHazardRecognizer(const InstrItineraryData*,
                                      const ScheduleDAG *DAG) const;
@@ -960,7 +984,7 @@ class TargetInstrInfo : public MCInstrInfo {
   /// targets may choose to honor.
   bool usePreRAHazardRecognizer() const;
 
-  /// analyzeCompare - For a comparison instruction, return the source registers
+  /// For a comparison instruction, return the source registers
   /// in SrcReg and SrcReg2 if having two register operands, and the value it
   /// compares against in CmpValue. Return true if the comparison instruction
   /// can be analyzed.
@@ -970,7 +994,7 @@ class TargetInstrInfo : public MCInstrInfo {
     return false;
   }
 
-  /// optimizeCompareInstr - See if the comparison instruction can be converted
+  /// See if the comparison instruction can be converted
   /// into something more efficient. E.g., on ARM most instructions can set the
   /// flags register, obviating the need for a separate CMP.
   virtual bool optimizeCompareInstr(MachineInstr *CmpInstr,
@@ -981,8 +1005,8 @@ class TargetInstrInfo : public MCInstrInfo {
   }
   virtual bool optimizeCondBranch(MachineInstr *MI) const { return false; }
 
-  /// optimizeLoadInstr - Try to remove the load by folding it to a register
-  /// operand at the use. We fold the load instructions if and only if the
+  /// Try to remove the load by folding it to a register operand at the use.
+  /// We fold the load instructions if and only if the
   /// def and use are in the same BB. We only look at one load and see
   /// whether it can be folded into MI. FoldAsLoadDefReg is the virtual register
   /// defined by the load we are trying to fold. DefMI returns the machine
@@ -995,8 +1019,8 @@ class TargetInstrInfo : public MCInstrInfo {
     return nullptr;
   }
 
-  /// FoldImmediate - 'Reg' is known to be defined by a move immediate
-  /// instruction, try to fold the immediate into the use instruction.
+  /// 'Reg' is known to be defined by a move immediate instruction,
+  /// try to fold the immediate into the use instruction.
   /// If MRI->hasOneNonDBGUse(Reg) is true, and this function returns true,
   /// then the caller may assume that DefMI has been erased from its parent
   /// block. The caller may assume that it will not be erased by this
@@ -1006,14 +1030,14 @@ class TargetInstrInfo : public MCInstrInfo {
     return false;
   }
 
-  /// getNumMicroOps - Return the number of u-operations the given machine
+  /// Return the number of u-operations the given machine
   /// instruction will be decoded to on the target cpu. The itinerary's
   /// IssueWidth is the number of microops that can be dispatched each
   /// cycle. An instruction with zero microops takes no dispatch resources.
   virtual unsigned getNumMicroOps(const InstrItineraryData *ItinData,
                                   const MachineInstr *MI) const;
 
-  /// isZeroCost - Return true for pseudo instructions that don't consume any
+  /// Return true for pseudo instructions that don't consume any
   /// machine resources in their current form. These are common cases that the
   /// scheduler should consider free, rather than conservatively handling them
   /// as instructions with no itinerary.
@@ -1025,27 +1049,27 @@ class TargetInstrInfo : public MCInstrInfo {
                                 SDNode *DefNode, unsigned DefIdx,
                                 SDNode *UseNode, unsigned UseIdx) const;
 
-  /// getOperandLatency - Compute and return the use operand latency of a given
-  /// pair of def and use.
+  /// Compute and return the use operand latency of a given pair of def and use.
   /// In most cases, the static scheduling itinerary was enough to determine the
   /// operand latency. But it may not be possible for instructions with variable
   /// number of defs / uses.
   ///
-  /// This is a raw interface to the itinerary that may be directly overriden by
-  /// a target. Use computeOperandLatency to get the best estimate of latency.
+  /// This is a raw interface to the itinerary that may be directly overridden
+  /// by a target. Use computeOperandLatency to get the best estimate of
+  /// latency.
   virtual int getOperandLatency(const InstrItineraryData *ItinData,
                                 const MachineInstr *DefMI, unsigned DefIdx,
                                 const MachineInstr *UseMI,
                                 unsigned UseIdx) const;
 
-  /// computeOperandLatency - Compute and return the latency of the given data
+  /// Compute and return the latency of the given data
   /// dependent def and use when the operand indices are already known.
   unsigned computeOperandLatency(const InstrItineraryData *ItinData,
                                  const MachineInstr *DefMI, unsigned DefIdx,
                                  const MachineInstr *UseMI, unsigned UseIdx)
     const;
 
-  /// getInstrLatency - Compute the instruction latency of a given instruction.
+  /// Compute the instruction latency of a given instruction.
   /// If the instruction has higher cost when predicated, it's returned via
   /// PredCost.
   virtual unsigned getInstrLatency(const InstrItineraryData *ItinData,
@@ -1064,36 +1088,35 @@ class TargetInstrInfo : public MCInstrInfo {
   int computeDefOperandLatency(const InstrItineraryData *ItinData,
                                const MachineInstr *DefMI) const;
 
-  /// isHighLatencyDef - Return true if this opcode has high latency to its
-  /// result.
+  /// Return true if this opcode has high latency to its result.
   virtual bool isHighLatencyDef(int opc) const { return false; }
 
-  /// hasHighOperandLatency - Compute operand latency between a def of 'Reg'
-  /// and an use in the current loop, return true if the target considered
+  /// Compute operand latency between a def of 'Reg'
+  /// and a use in the current loop. Return true if the target considered
   /// it 'high'. This is used by optimization passes such as machine LICM to
-  /// determine whether it makes sense to hoist an instruction out even in
+  /// determine whether it makes sense to hoist an instruction out even in a
   /// high register pressure situation.
   virtual
-  bool hasHighOperandLatency(const InstrItineraryData *ItinData,
+  bool hasHighOperandLatency(const TargetSchedModel &SchedModel,
                              const MachineRegisterInfo *MRI,
                              const MachineInstr *DefMI, unsigned DefIdx,
                              const MachineInstr *UseMI, unsigned UseIdx) const {
     return false;
   }
 
-  /// hasLowDefLatency - Compute operand latency of a def of 'Reg', return true
+  /// Compute operand latency of a def of 'Reg'. Return true
   /// if the target considered it 'low'.
   virtual
-  bool hasLowDefLatency(const InstrItineraryData *ItinData,
+  bool hasLowDefLatency(const TargetSchedModel &SchedModel,
                         const MachineInstr *DefMI, unsigned DefIdx) const;
 
-  /// verifyInstruction - Perform target specific instruction verification.
+  /// Perform target-specific instruction verification.
   virtual
   bool verifyInstruction(const MachineInstr *MI, StringRef &ErrInfo) const {
     return true;
   }
 
-  /// getExecutionDomain - Return the current execution domain and bit mask of
+  /// Return the current execution domain and bit mask of
   /// possible domains for instruction.
   ///
   /// Some micro-architectures have multiple execution domains, and multiple
@@ -1117,15 +1140,14 @@ class TargetInstrInfo : public MCInstrInfo {
     return std::make_pair(0, 0);
   }
 
-  /// setExecutionDomain - Change the opcode of MI to execute in Domain.
+  /// Change the opcode of MI to execute in Domain.
   ///
   /// The bit (1 << Domain) must be set in the mask returned from
   /// getExecutionDomain(MI).
-  ///
   virtual void setExecutionDomain(MachineInstr *MI, unsigned Domain) const {}
 
 
-  /// getPartialRegUpdateClearance - Returns the preferred minimum clearance
+  /// Returns the preferred minimum clearance
   /// before an instruction with an unwanted partial register update.
   ///
   /// Some instructions only write part of a register, and implicitly need to
@@ -1174,7 +1196,7 @@ class TargetInstrInfo : public MCInstrInfo {
   /// \brief Return the minimum clearance before an instruction that reads an
   /// unused register.
   ///
-  /// For example, AVX instructions may copy part of an register operand into
+  /// For example, AVX instructions may copy part of a register operand into
   /// the unused high bits of the destination register.
   ///
   /// vcvtsi2sdq %rax, %xmm0<undef>, %xmm14
@@ -1191,7 +1213,7 @@ class TargetInstrInfo : public MCInstrInfo {
     return 0;
   }
 
-  /// breakPartialRegDependency - Insert a dependency-breaking instruction
+  /// Insert a dependency-breaking instruction
   /// before MI to eliminate an unwanted dependency on OpNum.
   ///
   /// If it wasn't possible to avoid a def in the last N instructions before MI
@@ -1218,10 +1240,10 @@ class TargetInstrInfo : public MCInstrInfo {
     return nullptr;
   }
 
-  // areMemAccessesTriviallyDisjoint - Sometimes, it is possible for the target
+  // Sometimes, it is possible for the target
   // to tell, even without aliasing information, that two MIs access different
   // memory addresses. This function returns true if two MIs access different
-  // memory addresses, and false otherwise.
+  // memory addresses and false otherwise.
   virtual bool
   areMemAccessesTriviallyDisjoint(MachineInstr *MIa, MachineInstr *MIb,
                                   AliasAnalysis *AA = nullptr) const {
@@ -1232,8 +1254,26 @@ class TargetInstrInfo : public MCInstrInfo {
     return false;
   }
 
+  /// \brief Return the value to use for the MachineCSE's LookAheadLimit,
+  /// which is a heuristic used for CSE'ing phys reg defs.
+  virtual unsigned getMachineCSELookAheadLimit () const {
+    // The default lookahead is small to prevent unprofitable quadratic
+    // behavior.
+    return 5;
+  }
+
+  /// Return an array that contains the ids of the target indices (used for the
+  /// TargetIndex machine operand) and their names.
+  ///
+  /// MIR Serialization is able to serialize only the target indices that are
+  /// defined by this method.
+  virtual ArrayRef<std::pair<int, const char *>>
+  getSerializableTargetIndices() const {
+    return None;
+  }
+
 private:
-  int CallFrameSetupOpcode, CallFrameDestroyOpcode;
+  unsigned CallFrameSetupOpcode, CallFrameDestroyOpcode;
 };
 
 } // End llvm namespace
diff --git a/include/llvm/Target/TargetLowering.h b/include/llvm/Target/TargetLowering.h
index 29ecedcc643..742bf718f47 100644
--- a/include/llvm/Target/TargetLowering.h
+++ b/include/llvm/Target/TargetLowering.h
@@ -100,7 +100,8 @@ class TargetLoweringBase {
     TypeExpandFloat,     // Split this float into two of half the size.
     TypeScalarizeVector, // Replace this one-element vector with its element.
     TypeSplitVector,     // Split this vector into two of half the size.
-    TypeWidenVector      // This vector should be widened into a larger vector.
+    TypeWidenVector,     // This vector should be widened into a larger vector.
+    TypePromoteFloat     // Replace this float with a larger one.
   };
 
   /// LegalizeKind holds the legalization kind that needs to happen to EVT
@@ -125,8 +126,8 @@ class TargetLoweringBase {
 
   /// Enum that specifies what a AtomicRMWInst is expanded to, if at all. Exists
   /// because different targets have different levels of support for these
-  /// atomic RMW instructions, and also have different options w.r.t. what they should
-  /// expand to.
+  /// atomic RMW instructions, and also have different options w.r.t. what they
+  /// should expand to.
   enum class AtomicRMWExpansionKind {
     None,      // Don't expand the instruction.
     LLSC,      // Expand the instruction into loadlinked/storeconditional; used
@@ -160,26 +161,27 @@ class TargetLoweringBase {
 
 public:
   const TargetMachine &getTargetMachine() const { return TM; }
-  const DataLayout *getDataLayout() const { return TM.getDataLayout(); }
 
-  bool isBigEndian() const { return !IsLittleEndian; }
-  bool isLittleEndian() const { return IsLittleEndian; }
+  virtual bool useSoftFloat() const { return false; }
 
   /// Return the pointer type for the given address space, defaults to
   /// the pointer type from the data layout.
   /// FIXME: The default needs to be removed once all the code is updated.
-  virtual MVT getPointerTy(uint32_t /*AS*/ = 0) const;
-  unsigned getPointerSizeInBits(uint32_t AS = 0) const;
-  unsigned getPointerTypeSizeInBits(Type *Ty) const;
-  virtual MVT getScalarShiftAmountTy(EVT LHSTy) const;
+  MVT getPointerTy(const DataLayout &DL, uint32_t AS = 0) const {
+    return MVT::getIntegerVT(DL.getPointerSizeInBits(AS));
+  }
+
+  /// EVT is not used in-tree, but is used by out-of-tree target.
+  /// A documentation for this function would be nice...
+  virtual MVT getScalarShiftAmountTy(const DataLayout &, EVT) const;
 
-  EVT getShiftAmountTy(EVT LHSTy) const;
+  EVT getShiftAmountTy(EVT LHSTy, const DataLayout &DL) const;
 
   /// Returns the type to be used for the index operand of:
   /// ISD::INSERT_VECTOR_ELT, ISD::EXTRACT_VECTOR_ELT,
   /// ISD::INSERT_SUBVECTOR, and ISD::EXTRACT_SUBVECTOR
-  virtual MVT getVectorIdxTy() const {
-    return getPointerTy();
+  virtual MVT getVectorIdxTy(const DataLayout &DL) const {
+    return getPointerTy(DL);
   }
 
   /// Return true if the select operation is expensive for this target.
@@ -256,19 +258,29 @@ class TargetLoweringBase {
   /// isLoadBitCastBeneficial() - Return true if the following transform
   /// is beneficial.
   /// fold (conv (load x)) -> (load (conv*)x)
-  /// On architectures that don't natively support some vector loads efficiently,
-  /// casting the load to a smaller vector of larger types and loading
-  /// is more efficient, however, this can be undone by optimizations in
+  /// On architectures that don't natively support some vector loads
+  /// efficiently, casting the load to a smaller vector of larger types and
+  /// loading is more efficient, however, this can be undone by optimizations in
   /// dag combiner.
-  virtual bool isLoadBitCastBeneficial(EVT /* Load */, EVT /* Bitcast */) const {
+  virtual bool isLoadBitCastBeneficial(EVT /* Load */,
+                                       EVT /* Bitcast */) const {
     return true;
   }
 
+  /// Return true if it is expected to be cheaper to do a store of a non-zero
+  /// vector constant with the given size and type for the address space than to
+  /// store the individual scalar element constants.
+  virtual bool storeOfVectorConstantIsCheap(EVT MemVT,
+                                            unsigned NumElem,
+                                            unsigned AddrSpace) const {
+    return false;
+  }
+
   /// \brief Return true if it is cheap to speculate a call to intrinsic cttz.
   virtual bool isCheapToSpeculateCttz() const {
     return false;
   }
-  
+
   /// \brief Return true if it is cheap to speculate a call to intrinsic ctlz.
   virtual bool isCheapToSpeculateCtlz() const {
     return false;
@@ -315,7 +327,8 @@ class TargetLoweringBase {
   }
 
   /// Return the ValueType of the result of SETCC operations.
-  virtual EVT getSetCCResultType(LLVMContext &Context, EVT VT) const;
+  virtual EVT getSetCCResultType(const DataLayout &DL, LLVMContext &Context,
+                                 EVT VT) const;
 
   /// Return the ValueType for comparison libcalls. Comparions libcalls include
   /// floating point comparion calls, and Ordered/Unordered check calls on
@@ -571,7 +584,8 @@ class TargetLoweringBase {
   /// Return how this load with extension should be treated: either it is legal,
   /// needs to be promoted to a larger size, needs to be expanded to some other
   /// code sequence, or the target has a custom expander for it.
-  LegalizeAction getLoadExtAction(unsigned ExtType, EVT ValVT, EVT MemVT) const {
+  LegalizeAction getLoadExtAction(unsigned ExtType, EVT ValVT,
+                                  EVT MemVT) const {
     if (ValVT.isExtended() || MemVT.isExtended()) return Expand;
     unsigned ValI = (unsigned) ValVT.getSimpleVT().SimpleTy;
     unsigned MemI = (unsigned) MemVT.getSimpleVT().SimpleTy;
@@ -702,17 +716,18 @@ class TargetLoweringBase {
   /// operations except for the pointer size.  If AllowUnknown is true, this
   /// will return MVT::Other for types with no EVT counterpart (e.g. structs),
   /// otherwise it will assert.
-  EVT getValueType(Type *Ty, bool AllowUnknown = false) const {
+  EVT getValueType(const DataLayout &DL, Type *Ty,
+                   bool AllowUnknown = false) const {
     // Lower scalar pointers to native pointer types.
     if (PointerType *PTy = dyn_cast<PointerType>(Ty))
-      return getPointerTy(PTy->getAddressSpace());
+      return getPointerTy(DL, PTy->getAddressSpace());
 
     if (Ty->isVectorTy()) {
       VectorType *VTy = cast<VectorType>(Ty);
       Type *Elm = VTy->getElementType();
       // Lower vectors of pointers to native pointer types.
       if (PointerType *PT = dyn_cast<PointerType>(Elm)) {
-        EVT PointerTy(getPointerTy(PT->getAddressSpace()));
+        EVT PointerTy(getPointerTy(DL, PT->getAddressSpace()));
         Elm = PointerTy.getTypeForEVT(Ty->getContext());
       }
 
@@ -723,14 +738,15 @@ class TargetLoweringBase {
   }
 
   /// Return the MVT corresponding to this LLVM type. See getValueType.
-  MVT getSimpleValueType(Type *Ty, bool AllowUnknown = false) const {
-    return getValueType(Ty, AllowUnknown).getSimpleVT();
+  MVT getSimpleValueType(const DataLayout &DL, Type *Ty,
+                         bool AllowUnknown = false) const {
+    return getValueType(DL, Ty, AllowUnknown).getSimpleVT();
   }
 
   /// Return the desired alignment for ByVal or InAlloca aggregate function
   /// arguments in the caller parameter area.  This is the actual alignment, not
   /// its logarithm.
-  virtual unsigned getByValTypeAlignment(Type *Ty) const;
+  virtual unsigned getByValTypeAlignment(Type *Ty, const DataLayout &DL) const;
 
   /// Return the type of registers that this ValueType will eventually require.
   MVT getRegisterType(MVT VT) const {
@@ -805,8 +821,8 @@ class TargetLoweringBase {
   /// When splitting a value of the specified type into parts, does the Lo
   /// or Hi part come first?  This usually follows the endianness, except
   /// for ppcf128, where the Hi part always comes first.
-  bool hasBigEndianPartOrdering(EVT VT) const {
-    return isBigEndian() || VT == MVT::ppcf128;
+  bool hasBigEndianPartOrdering(EVT VT, const DataLayout &DL) const {
+    return DL.isBigEndian() || VT == MVT::ppcf128;
   }
 
   /// If true, the target has custom DAG combine transformations that it can
@@ -862,6 +878,14 @@ class TargetLoweringBase {
     return false;
   }
 
+  /// Return true if the target supports a memory access of this type for the
+  /// given address space and alignment. If the access is allowed, the optional
+  /// final parameter returns if the access is also fast (as defined by the
+  /// target).
+  bool allowsMemoryAccess(LLVMContext &Context, const DataLayout &DL, EVT VT,
+                          unsigned AddrSpace = 0, unsigned Alignment = 1,
+                          bool *Fast = nullptr) const;
+  
   /// Returns the target specific optimal type for load and store operations as
   /// a result of memset, memcpy, and memmove lowering.
   ///
@@ -993,7 +1017,8 @@ class TargetLoweringBase {
   int InstructionOpcodeToISD(unsigned Opcode) const;
 
   /// Estimate the cost of type-legalization and the legalized type.
-  std::pair<unsigned, MVT> getTypeLegalizationCost(Type *Ty) const;
+  std::pair<unsigned, MVT> getTypeLegalizationCost(const DataLayout &DL,
+                                                   Type *Ty) const;
 
   /// @}
 
@@ -1051,8 +1076,9 @@ class TargetLoweringBase {
   ///  seq_cst. But if they are lowered to monotonic accesses, no amount of
   ///  IR-level fences can prevent it.
   /// @{
-  virtual Instruction* emitLeadingFence(IRBuilder<> &Builder, AtomicOrdering Ord,
-          bool IsStore, bool IsLoad) const {
+  virtual Instruction *emitLeadingFence(IRBuilder<> &Builder,
+                                        AtomicOrdering Ord, bool IsStore,
+                                        bool IsLoad) const {
     if (!getInsertFencesForAtomic())
       return nullptr;
 
@@ -1062,8 +1088,9 @@ class TargetLoweringBase {
       return nullptr;
   }
 
-  virtual Instruction* emitTrailingFence(IRBuilder<> &Builder, AtomicOrdering Ord,
-          bool IsStore, bool IsLoad) const {
+  virtual Instruction *emitTrailingFence(IRBuilder<> &Builder,
+                                         AtomicOrdering Ord, bool IsStore,
+                                         bool IsLoad) const {
     if (!getInsertFencesForAtomic())
       return nullptr;
 
@@ -1108,7 +1135,8 @@ class TargetLoweringBase {
   /// it succeeds, and nullptr otherwise.
   /// If shouldExpandAtomicLoadInIR returns true on that load, it will undergo
   /// another round of expansion.
-  virtual LoadInst *lowerIdempotentRMWIntoFencedLoad(AtomicRMWInst *RMWI) const {
+  virtual LoadInst *
+  lowerIdempotentRMWIntoFencedLoad(AtomicRMWInst *RMWI) const {
     return nullptr;
   }
 
@@ -1219,11 +1247,10 @@ class TargetLoweringBase {
     HasExtractBitsInsn = hasExtractInsn;
   }
 
-  /// Tells the code generator not to expand sequence of operations into a
-  /// separate sequences that increases the amount of flow control.
-  void setJumpIsExpensive(bool isExpensive = true) {
-    JumpIsExpensive = isExpensive;
-  }
+  /// Tells the code generator not to expand logic operations on comparison
+  /// predicates into separate sequences that increase the amount of flow
+  /// control.
+  void setJumpIsExpensive(bool isExpensive = true);
 
   /// Tells the code generator that integer divide is expensive, and if
   /// possible, should be replaced by an alternate sequence of instructions not
@@ -1415,7 +1442,8 @@ class TargetLoweringBase {
   /// load/store.
   virtual bool GetAddrModeArguments(IntrinsicInst * /*I*/,
                                     SmallVectorImpl<Value*> &/*Ops*/,
-                                    Type *&/*AccessTy*/) const {
+                                    Type *&/*AccessTy*/,
+                                    unsigned AddrSpace = 0) const {
     return false;
   }
 
@@ -1440,7 +1468,12 @@ class TargetLoweringBase {
   /// The type may be VoidTy, in which case only return true if the addressing
   /// mode is legal for a load/store of any legal type.  TODO: Handle
   /// pre/postinc as well.
-  virtual bool isLegalAddressingMode(const AddrMode &AM, Type *Ty) const;
+  ///
+  /// If the address space cannot be determined, it will be -1.
+  ///
+  /// TODO: Remove default argument
+  virtual bool isLegalAddressingMode(const DataLayout &DL, const AddrMode &AM,
+                                     Type *Ty, unsigned AddrSpace) const;
 
   /// \brief Return the cost of the scaling factor used in the addressing mode
   /// represented by AM for this target, for a load/store of the specified type.
@@ -1448,9 +1481,12 @@ class TargetLoweringBase {
   /// If the AM is supported, the return value must be >= 0.
   /// If the AM is not supported, it returns a negative value.
   /// TODO: Handle pre/postinc as well.
-  virtual int getScalingFactorCost(const AddrMode &AM, Type *Ty) const {
+  /// TODO: Remove default argument
+  virtual int getScalingFactorCost(const DataLayout &DL, const AddrMode &AM,
+                                   Type *Ty, unsigned AS = 0) const {
     // Default: assume that any scaling factor used in a legal AM is free.
-    if (isLegalAddressingMode(AM, Ty)) return 0;
+    if (isLegalAddressingMode(DL, AM, Ty, AS))
+      return 0;
     return -1;
   }
 
@@ -1572,6 +1608,35 @@ class TargetLoweringBase {
     return false;
   }
 
+  /// \brief Get the maximum supported factor for interleaved memory accesses.
+  /// Default to be the minimum interleave factor: 2.
+  virtual unsigned getMaxSupportedInterleaveFactor() const { return 2; }
+
+  /// \brief Lower an interleaved load to target specific intrinsics. Return
+  /// true on success.
+  ///
+  /// \p LI is the vector load instruction.
+  /// \p Shuffles is the shufflevector list to DE-interleave the loaded vector.
+  /// \p Indices is the corresponding indices for each shufflevector.
+  /// \p Factor is the interleave factor.
+  virtual bool lowerInterleavedLoad(LoadInst *LI,
+                                    ArrayRef<ShuffleVectorInst *> Shuffles,
+                                    ArrayRef<unsigned> Indices,
+                                    unsigned Factor) const {
+    return false;
+  }
+
+  /// \brief Lower an interleaved store to target specific intrinsics. Return
+  /// true on success.
+  ///
+  /// \p SI is the vector store instruction.
+  /// \p SVI is the shufflevector to RE-interleave the stored vector.
+  /// \p Factor is the interleave factor.
+  virtual bool lowerInterleavedStore(StoreInst *SI, ShuffleVectorInst *SVI,
+                                     unsigned Factor) const {
+    return false;
+  }
+
   /// Return true if zero-extending the specific node Val to type VT2 is free
   /// (either because it's implicitly zero-extended such as ARM ldrb / ldrh or
   /// because it's folded such as X86 zero-extending loads).
@@ -1681,9 +1746,6 @@ class TargetLoweringBase {
 private:
   const TargetMachine &TM;
 
-  /// True if this is a little endian target.
-  bool IsLittleEndian;
-
   /// Tells the code generator not to expand operations into sequences that use
   /// the select operations if possible.
   bool SelectIsExpensive;
@@ -1939,7 +2001,7 @@ class TargetLoweringBase {
   unsigned MaxStoresPerMemmove;
 
   /// Maximum number of store instructions that may be substituted for a call to
-  /// memmove, used for functions with OpSize attribute.
+  /// memmove, used for functions with OptSize attribute.
   unsigned MaxStoresPerMemmoveOptSize;
 
   /// Tells the code generator that select is more expensive than a branch if
@@ -1960,7 +2022,8 @@ class TargetLoweringBase {
 
   /// Replace/modify any TargetFrameIndex operands with a targte-dependent
   /// sequence of memory operands that is recognized by PrologEpilogInserter.
-  MachineBasicBlock *emitPatchPoint(MachineInstr *MI, MachineBasicBlock *MBB) const;
+  MachineBasicBlock *emitPatchPoint(MachineInstr *MI,
+                                    MachineBasicBlock *MBB) const;
 };
 
 /// This class defines information used to lower LLVM code to legal SelectionDAG
@@ -2360,6 +2423,7 @@ class TargetLowering : public TargetLoweringBase {
     ArgListTy &getArgs() {
       return Args;
     }
+
   };
 
   /// This function lowers an abstract call to a function into an actual call.
@@ -2368,7 +2432,7 @@ class TargetLowering : public TargetLoweringBase {
   /// outgoing token chain. It calls LowerCall to do the actual lowering.
   std::pair<SDValue, SDValue> LowerCallTo(CallLoweringInfo &CLI) const;
 
-  /// This hook must be implemented to lower calls into the the specified
+  /// This hook must be implemented to lower calls into the specified
   /// DAG. The outgoing arguments to the call are described by the Outs array,
   /// and the values to be returned by the call are described by the Ins
   /// array. The implementation should fill in the InVals array with legal-type
@@ -2431,7 +2495,8 @@ class TargetLowering : public TargetLoweringBase {
   /// Return the register ID of the name passed in. Used by named register
   /// global variables extension. There is no target-independent behaviour
   /// so the default action is to bail.
-  virtual unsigned getRegisterByName(const char* RegName, EVT VT) const {
+  virtual unsigned getRegisterByName(const char* RegName, EVT VT,
+                                     SelectionDAG &DAG) const {
     report_fatal_error("Named registers not implemented for this target");
   }
 
@@ -2603,7 +2668,8 @@ class TargetLowering : public TargetLoweringBase {
   /// specific constraints and their prefixes, and also tie in the associated
   /// operand values.  If this returns an empty vector, and if the constraint
   /// string itself isn't empty, there was an error parsing.
-  virtual AsmOperandInfoVector ParseConstraints(const TargetRegisterInfo *TRI,
+  virtual AsmOperandInfoVector ParseConstraints(const DataLayout &DL,
+                                                const TargetRegisterInfo *TRI,
                                                 ImmutableCallSite CS) const;
 
   /// Examine constraint type and operand type and determine a weight value.
@@ -2625,7 +2691,7 @@ class TargetLowering : public TargetLoweringBase {
                                       SelectionDAG *DAG = nullptr) const;
 
   /// Given a constraint, return the type of constraint it is for this target.
-  virtual ConstraintType getConstraintType(const std::string &Constraint) const;
+  virtual ConstraintType getConstraintType(StringRef Constraint) const;
 
   /// Given a physical register constraint (e.g.  {edx}), return the register
   /// number and the register class for the register.
@@ -2638,10 +2704,9 @@ class TargetLowering : public TargetLoweringBase {
   /// returns a register number of 0 and a null register class pointer.
   virtual std::pair<unsigned, const TargetRegisterClass *>
   getRegForInlineAsmConstraint(const TargetRegisterInfo *TRI,
-                               const std::string &Constraint, MVT VT) const;
+                               StringRef Constraint, MVT VT) const;
 
-  virtual unsigned
-  getInlineAsmMemConstraint(const std::string &ConstraintCode) const {
+  virtual unsigned getInlineAsmMemConstraint(StringRef ConstraintCode) const {
     if (ConstraintCode == "i")
       return InlineAsm::Constraint_i;
     else if (ConstraintCode == "m")
@@ -2663,8 +2728,6 @@ class TargetLowering : public TargetLoweringBase {
   //===--------------------------------------------------------------------===//
   // Div utility functions
   //
-  SDValue BuildExactSDIV(SDValue Op1, SDValue Op2, SDLoc dl,
-                         SelectionDAG &DAG) const;
   SDValue BuildSDIV(SDNode *N, const APInt &Divisor, SelectionDAG &DAG,
                     bool IsAfterLegalization,
                     std::vector<SDNode *> *Created) const;
@@ -2677,15 +2740,17 @@ class TargetLowering : public TargetLoweringBase {
     return SDValue();
   }
 
-  /// Indicate whether this target prefers to combine the given number of FDIVs
-  /// with the same divisor.
-  virtual bool combineRepeatedFPDivisors(unsigned NumUsers) const {
-    return false;
+  /// Indicate whether this target prefers to combine FDIVs with the same
+  /// divisor. If the transform should never be done, return zero. If the
+  /// transform should be done, return the minimum number of divisor uses
+  /// that must exist.
+  virtual unsigned combineRepeatedFPDivisors() const {
+    return 0;
   }
 
   /// Hooks for building estimates in place of slower divisions and square
   /// roots.
-  
+
   /// Return a reciprocal square root estimate value for the input operand.
   /// The RefinementSteps output is the number of Newton-Raphson refinement
   /// iterations required to generate a sufficient (though not necessarily
@@ -2696,10 +2761,9 @@ class TargetLowering : public TargetLoweringBase {
   /// If that's true, then return '0' as the number of RefinementSteps to avoid
   /// any further refinement of the estimate.
   /// An empty SDValue return means no estimate sequence can be created.
-  virtual SDValue getRsqrtEstimate(SDValue Operand,
-                              DAGCombinerInfo &DCI,
-                              unsigned &RefinementSteps,
-                              bool &UseOneConstNR) const {
+  virtual SDValue getRsqrtEstimate(SDValue Operand, DAGCombinerInfo &DCI,
+                                   unsigned &RefinementSteps,
+                                   bool &UseOneConstNR) const {
     return SDValue();
   }
 
@@ -2711,8 +2775,7 @@ class TargetLowering : public TargetLoweringBase {
   /// If that's true, then return '0' as the number of RefinementSteps to avoid
   /// any further refinement of the estimate.
   /// An empty SDValue return means no estimate sequence can be created.
-  virtual SDValue getRecipEstimate(SDValue Operand,
-                                   DAGCombinerInfo &DCI,
+  virtual SDValue getRecipEstimate(SDValue Operand, DAGCombinerInfo &DCI,
                                    unsigned &RefinementSteps) const {
     return SDValue();
   }
@@ -2768,14 +2831,18 @@ class TargetLowering : public TargetLoweringBase {
   virtual bool useLoadStackGuardNode() const {
     return false;
   }
+
+  /// Lower TLS global address SDNode for target independent emulated TLS model.
+  virtual SDValue LowerToTLSEmulatedModel(const GlobalAddressSDNode *GA,
+                                          SelectionDAG &DAG) const;
 };
 
 /// Given an LLVM IR type and return type attributes, compute the return value
 /// EVTs and flags, and optionally also the offsets, if the return value is
 /// being lowered to memory.
-void GetReturnInfo(Type* ReturnType, AttributeSet attr,
+void GetReturnInfo(Type *ReturnType, AttributeSet attr,
                    SmallVectorImpl<ISD::OutputArg> &Outs,
-                   const TargetLowering &TLI);
+                   const TargetLowering &TLI, const DataLayout &DL);
 
 } // end llvm namespace
 
diff --git a/include/llvm/Target/TargetLoweringObjectFile.h b/include/llvm/Target/TargetLoweringObjectFile.h
index 62ae2372f59..7cf23b5a388 100644
--- a/include/llvm/Target/TargetLoweringObjectFile.h
+++ b/include/llvm/Target/TargetLoweringObjectFile.h
@@ -42,16 +42,15 @@ class TargetLoweringObjectFile : public MCObjectFileInfo {
   void operator=(const TargetLoweringObjectFile&) = delete;
 
 protected:
-  const DataLayout *DL;
   bool SupportIndirectSymViaGOTPCRel;
   bool SupportGOTPCRelWithOffset;
 
 public:
   MCContext &getContext() const { return *Ctx; }
 
-  TargetLoweringObjectFile() : MCObjectFileInfo(), Ctx(nullptr), DL(nullptr),
-                               SupportIndirectSymViaGOTPCRel(false),
-                               SupportGOTPCRelWithOffset(true) {}
+  TargetLoweringObjectFile()
+      : MCObjectFileInfo(), Ctx(nullptr), SupportIndirectSymViaGOTPCRel(false),
+        SupportGOTPCRelWithOffset(true) {}
 
   virtual ~TargetLoweringObjectFile();
 
@@ -60,16 +59,9 @@ class TargetLoweringObjectFile : public MCObjectFileInfo {
   /// implementations a chance to set up their default sections.
   virtual void Initialize(MCContext &ctx, const TargetMachine &TM);
 
-  virtual void emitPersonalityValue(MCStreamer &Streamer,
-                                    const TargetMachine &TM,
+  virtual void emitPersonalityValue(MCStreamer &Streamer, const DataLayout &TM,
                                     const MCSymbol *Sym) const;
 
-  /// Extract the dependent library name from a linker option string. Returns
-  /// StringRef() if the option does not specify a library.
-  virtual StringRef getDepLibFromLinkerOpt(StringRef LinkerOption) const {
-    return StringRef();
-  }
-
   /// Emit the module flags that the platform cares about.
   virtual void emitModuleFlags(MCStreamer &Streamer,
                                ArrayRef<Module::ModuleFlagEntry> Flags,
@@ -77,8 +69,9 @@ class TargetLoweringObjectFile : public MCObjectFileInfo {
 
   /// Given a constant with the SectionKind, return a section that it should be
   /// placed in.
-  virtual const MCSection *getSectionForConstant(SectionKind Kind,
-                                                 const Constant *C) const;
+  virtual MCSection *getSectionForConstant(const DataLayout &DL,
+                                           SectionKind Kind,
+                                           const Constant *C) const;
 
   /// Classify the specified global variable into a set of target independent
   /// categories embodied in SectionKind.
@@ -88,16 +81,14 @@ class TargetLoweringObjectFile : public MCObjectFileInfo {
   /// This method computes the appropriate section to emit the specified global
   /// variable or function definition. This should not be passed external (or
   /// available externally) globals.
-  const MCSection *SectionForGlobal(const GlobalValue *GV,
-                                    SectionKind Kind, Mangler &Mang,
-                                    const TargetMachine &TM) const;
+  MCSection *SectionForGlobal(const GlobalValue *GV, SectionKind Kind,
+                              Mangler &Mang, const TargetMachine &TM) const;
 
   /// This method computes the appropriate section to emit the specified global
   /// variable or function definition. This should not be passed external (or
   /// available externally) globals.
-  const MCSection *SectionForGlobal(const GlobalValue *GV,
-                                    Mangler &Mang,
-                                    const TargetMachine &TM) const {
+  MCSection *SectionForGlobal(const GlobalValue *GV, Mangler &Mang,
+                              const TargetMachine &TM) const {
     return SectionForGlobal(GV, getKindForGlobal(GV, TM), Mang, TM);
   }
 
@@ -106,9 +97,8 @@ class TargetLoweringObjectFile : public MCObjectFileInfo {
                                  bool CannotUsePrivateLabel, Mangler &Mang,
                                  const TargetMachine &TM) const;
 
-  virtual const MCSection *
-  getSectionForJumpTable(const Function &F, Mangler &Mang,
-                         const TargetMachine &TM) const;
+  virtual MCSection *getSectionForJumpTable(const Function &F, Mangler &Mang,
+                                            const TargetMachine &TM) const;
 
   virtual bool shouldPutJumpTableInFunctionSection(bool UsesLabelDifference,
                                                    const Function &F) const;
@@ -116,7 +106,7 @@ class TargetLoweringObjectFile : public MCObjectFileInfo {
   /// Targets should implement this method to assign a section to globals with
   /// an explicit section specfied. The implementation of this method can
   /// assume that GV->hasSection() is true.
-  virtual const MCSection *
+  virtual MCSection *
   getExplicitSectionGlobal(const GlobalValue *GV, SectionKind Kind,
                            Mangler &Mang, const TargetMachine &TM) const = 0;
 
@@ -150,13 +140,13 @@ class TargetLoweringObjectFile : public MCObjectFileInfo {
   getTTypeReference(const MCSymbolRefExpr *Sym, unsigned Encoding,
                     MCStreamer &Streamer) const;
 
-  virtual const MCSection *getStaticCtorSection(unsigned Priority,
-                                                const MCSymbol *KeySym) const {
+  virtual MCSection *getStaticCtorSection(unsigned Priority,
+                                          const MCSymbol *KeySym) const {
     return StaticCtorSection;
   }
 
-  virtual const MCSection *getStaticDtorSection(unsigned Priority,
-                                                const MCSymbol *KeySym) const {
+  virtual MCSection *getStaticDtorSection(unsigned Priority,
+                                          const MCSymbol *KeySym) const {
     return StaticDtorSection;
   }
 
@@ -191,10 +181,13 @@ class TargetLoweringObjectFile : public MCObjectFileInfo {
     return nullptr;
   }
 
+  virtual void emitLinkerFlagsForGlobal(raw_ostream &OS, const GlobalValue *GV,
+                                        const Mangler &Mang) const {}
+
 protected:
-  virtual const MCSection *
-  SelectSectionForGlobal(const GlobalValue *GV, SectionKind Kind,
-                         Mangler &Mang, const TargetMachine &TM) const = 0;
+  virtual MCSection *SelectSectionForGlobal(const GlobalValue *GV,
+                                            SectionKind Kind, Mangler &Mang,
+                                            const TargetMachine &TM) const = 0;
 };
 
 } // end namespace llvm
diff --git a/include/llvm/Target/TargetMachine.h b/include/llvm/Target/TargetMachine.h
index 2a1ce0483e1..d707e7c0293 100644
--- a/include/llvm/Target/TargetMachine.h
+++ b/include/llvm/Target/TargetMachine.h
@@ -15,6 +15,7 @@
 #define LLVM_TARGET_TARGETMACHINE_H
 
 #include "llvm/ADT/StringRef.h"
+#include "llvm/ADT/Triple.h"
 #include "llvm/IR/DataLayout.h"
 #include "llvm/Pass.h"
 #include "llvm/Support/CodeGen.h"
@@ -27,6 +28,7 @@ namespace llvm {
 class InstrItineraryData;
 class GlobalValue;
 class Mangler;
+class MachineFunctionInitializer;
 class MCAsmInfo;
 class MCCodeGenInfo;
 class MCContext;
@@ -68,18 +70,23 @@ class TargetMachine {
   void operator=(const TargetMachine &) = delete;
 protected: // Can only create subclasses.
   TargetMachine(const Target &T, StringRef DataLayoutString,
-                StringRef TargetTriple, StringRef CPU, StringRef FS,
+                const Triple &TargetTriple, StringRef CPU, StringRef FS,
                 const TargetOptions &Options);
 
   /// The Target that this machine was created for.
   const Target &TheTarget;
 
-  /// For ABI type size and alignment.
+  /// DataLayout for the target: keep ABI type size and alignment.
+  ///
+  /// The DataLayout is created based on the string representation provided
+  /// during construction. It is kept here only to avoid reparsing the string
+  /// but should not really be used during compilation, because it has an
+  /// internal cache that is context specific.
   const DataLayout DL;
 
   /// Triple string, CPU name, and target feature strings the TargetMachine
   /// instance is created with.
-  std::string TargetTriple;
+  Triple TargetTriple;
   std::string TargetCPU;
   std::string TargetFS;
 
@@ -103,7 +110,7 @@ class TargetMachine {
 
   const Target &getTarget() const { return TheTarget; }
 
-  StringRef getTargetTriple() const { return TargetTriple; }
+  const Triple &getTargetTriple() const { return TargetTriple; }
   StringRef getTargetCPU() const { return TargetCPU; }
   StringRef getTargetFeatureString() const { return TargetFS; }
 
@@ -123,9 +130,22 @@ class TargetMachine {
     return *static_cast<const STC*>(getSubtargetImpl(F));
   }
 
-  /// This method returns a pointer to the DataLayout for the target. It should
-  /// be unchanging for every subtarget.
-  const DataLayout *getDataLayout() const { return &DL; }
+  /// Create a DataLayout.
+  const DataLayout createDataLayout() const { return DL; }
+
+  /// Test if a DataLayout if compatible with the CodeGen for this target.
+  ///
+  /// The LLVM Module owns a DataLayout that is used for the target independent
+  /// optimizations and code generation. This hook provides a target specific
+  /// check on the validity of this DataLayout.
+  bool isCompatibleDataLayout(const DataLayout &Candidate) const {
+    return DL == Candidate;
+  }
+
+  /// Get the pointer size for this target.
+  ///
+  /// This is the only time the DataLayout in the TargetMachine is used.
+  unsigned getPointerSize() const { return DL.getPointerSize(); }
 
   /// \brief Reset the target options based on the function's attributes.
   // FIXME: Remove TargetOptions that affect per-function code generation
@@ -208,11 +228,11 @@ class TargetMachine {
   /// emitted.  Typically this will involve several steps of code generation.
   /// This method should return true if emission of this file type is not
   /// supported, or false on success.
-  virtual bool addPassesToEmitFile(PassManagerBase &, raw_pwrite_stream &,
-                                   CodeGenFileType,
-                                   bool /*DisableVerify*/ = true,
-                                   AnalysisID /*StartAfter*/ = nullptr,
-                                   AnalysisID /*StopAfter*/ = nullptr) {
+  virtual bool addPassesToEmitFile(
+      PassManagerBase &, raw_pwrite_stream &, CodeGenFileType,
+      bool /*DisableVerify*/ = true, AnalysisID /*StartBefore*/ = nullptr,
+      AnalysisID /*StartAfter*/ = nullptr, AnalysisID /*StopAfter*/ = nullptr,
+      MachineFunctionInitializer * /*MFInitializer*/ = nullptr) {
     return true;
   }
 
@@ -238,7 +258,7 @@ class TargetMachine {
 class LLVMTargetMachine : public TargetMachine {
 protected: // Can only create subclasses.
   LLVMTargetMachine(const Target &T, StringRef DataLayoutString,
-                    StringRef TargetTriple, StringRef CPU, StringRef FS,
+                    const Triple &TargetTriple, StringRef CPU, StringRef FS,
                     TargetOptions Options, Reloc::Model RM, CodeModel::Model CM,
                     CodeGenOpt::Level OL);
 
@@ -256,10 +276,11 @@ class LLVMTargetMachine : public TargetMachine {
 
   /// Add passes to the specified pass manager to get the specified file
   /// emitted.  Typically this will involve several steps of code generation.
-  bool addPassesToEmitFile(PassManagerBase &PM, raw_pwrite_stream &Out,
-                           CodeGenFileType FileType, bool DisableVerify = true,
-                           AnalysisID StartAfter = nullptr,
-                           AnalysisID StopAfter = nullptr) override;
+  bool addPassesToEmitFile(
+      PassManagerBase &PM, raw_pwrite_stream &Out, CodeGenFileType FileType,
+      bool DisableVerify = true, AnalysisID StartBefore = nullptr,
+      AnalysisID StartAfter = nullptr, AnalysisID StopAfter = nullptr,
+      MachineFunctionInitializer *MFInitializer = nullptr) override;
 
   /// Add passes to the specified pass manager to get machine code emitted with
   /// the MCJIT. This method returns true if machine code is not supported. It
diff --git a/include/llvm/Target/TargetOpcodes.h b/include/llvm/Target/TargetOpcodes.h
index afc22365eba..50197191109 100644
--- a/include/llvm/Target/TargetOpcodes.h
+++ b/include/llvm/Target/TargetOpcodes.h
@@ -118,10 +118,16 @@ enum {
   /// collectors and deoptimizations in either the callee or caller.
   STATEPOINT = 20,
 
-  /// Instruction that records the offset of a function's frame allocation in a
-  /// label. Created by the llvm.frameallocate intrinsic. It has two arguments:
-  /// the symbol for the label and the frame index of the stack allocation.
-  FRAME_ALLOC = 21,
+  /// Instruction that records the offset of a local stack allocation passed to
+  /// llvm.localescape. It has two arguments: the symbol for the label and the
+  /// frame index of the local stack allocation.
+  LOCAL_ESCAPE = 21,
+
+  /// Loading instruction that may page fault, bundled with associated
+  /// information on how to handle such a page fault.  It is intended to support
+  /// "zero cost" null checks in managed languages by allowing LLVM to fold
+  /// comparisions into existing memory operations.
+  FAULTING_LOAD_OP = 22,
 };
 } // end namespace TargetOpcode
 } // end namespace llvm
diff --git a/include/llvm/Target/TargetOptions.h b/include/llvm/Target/TargetOptions.h
index bedb71d9e9b..bd696f399d5 100644
--- a/include/llvm/Target/TargetOptions.h
+++ b/include/llvm/Target/TargetOptions.h
@@ -15,28 +15,23 @@
 #ifndef LLVM_TARGET_TARGETOPTIONS_H
 #define LLVM_TARGET_TARGETOPTIONS_H
 
+#include "llvm/Target/TargetRecip.h"
 #include "llvm/MC/MCTargetOptions.h"
 #include <string>
 
 namespace llvm {
   class MachineFunction;
+  class Module;
   class StringRef;
 
-  // Possible float ABI settings. Used with FloatABIType in TargetOptions.h.
   namespace FloatABI {
     enum ABIType {
-      Default, // Target-specific (either soft or hard depending on triple,etc).
-      Soft, // Soft float.
-      Hard  // Hard float.
+      Default, // Target-specific (either soft or hard depending on triple, etc).
+      Soft,    // Soft float.
+      Hard     // Hard float.
     };
   }
 
-  // @LOCALMOD-BEGIN
-  /// TLSUseCall - This flag enables the use of a function call to get the
-  /// thread pointer for TLS accesses, instead of using inline code.
-  extern bool TLSUseCall;
-  // @LOCALMOD-END
-
   namespace FPOpFusion {
     enum FPOpFusionMode {
       Fast,     // Enable fusion of FP ops wherever it's profitable.
@@ -63,43 +58,30 @@ namespace llvm {
     };
   }
 
-  enum class CFIntegrity {
-    Sub,             // Use subtraction-based checks.
-    Ror,             // Use rotation-based checks.
-    Add              // Use addition-based checks. This depends on having
-                     // sufficient alignment in the code and is usually not
-                     // feasible.
-  };
-
   class TargetOptions {
   public:
     TargetOptions()
-        : PrintMachineCode(false), NoFramePointerElim(false),
+        : PrintMachineCode(false),
           LessPreciseFPMADOption(false), UnsafeFPMath(false),
           NoInfsFPMath(false), NoNaNsFPMath(false),
-          HonorSignDependentRoundingFPMathOption(false), UseSoftFloat(false),
-          NoZerosInBSS(false), JITEmitDebugInfo(false),
-          JITEmitDebugInfoToDisk(false), GuaranteedTailCallOpt(false),
-          DisableTailCalls(false), StackAlignmentOverride(0),
+          HonorSignDependentRoundingFPMathOption(false),
+          NoZerosInBSS(false),
+          GuaranteedTailCallOpt(false),
+          StackAlignmentOverride(0),
           EnableFastISel(false), PositionIndependentExecutable(false),
           UseInitArray(false), DisableIntegratedAS(false),
           CompressDebugSections(false), FunctionSections(false),
           DataSections(false), UniqueSectionNames(true), TrapUnreachable(false),
-          TrapFuncName(), FloatABIType(FloatABI::Default),
-          AllowFPOpFusion(FPOpFusion::Standard), JTType(JumpTable::Single),
-          FCFI(false), ThreadModel(ThreadModel::POSIX),
-          CFIType(CFIntegrity::Sub), CFIEnforcing(false), CFIFuncName() {}
+          EmulatedTLS(false), FloatABIType(FloatABI::Default),
+          AllowFPOpFusion(FPOpFusion::Standard), Reciprocals(TargetRecip()),
+          JTType(JumpTable::Single),
+          ThreadModel(ThreadModel::POSIX) {}
 
     /// PrintMachineCode - This flag is enabled when the -print-machineinstrs
     /// option is specified on the command line, and should enable debugging
     /// output from the code generator.
     unsigned PrintMachineCode : 1;
 
-    /// NoFramePointerElim - This flag is enabled when the -disable-fp-elim is
-    /// specified on the command line.  If the target supports the frame pointer
-    /// elimination optimization, this option should disable it.
-    unsigned NoFramePointerElim : 1;
-
     /// DisableFramePointerElim - This returns true if frame pointer elimination
     /// optimization should be disabled for the given machine function.
     bool DisableFramePointerElim(const MachineFunction &MF) const;
@@ -142,26 +124,11 @@ namespace llvm {
     unsigned HonorSignDependentRoundingFPMathOption : 1;
     bool HonorSignDependentRoundingFPMath() const;
 
-    /// UseSoftFloat - This flag is enabled when the -soft-float flag is
-    /// specified on the command line.  When this flag is on, the code generator
-    /// will generate libcalls to the software floating point library instead of
-    /// target FP instructions.
-    unsigned UseSoftFloat : 1;
-
     /// NoZerosInBSS - By default some codegens place zero-initialized data to
     /// .bss section. This flag disables such behaviour (necessary, e.g. for
     /// crt*.o compiling).
     unsigned NoZerosInBSS : 1;
 
-    /// JITEmitDebugInfo - This flag indicates that the JIT should try to emit
-    /// debug information and notify a debugger about it.
-    unsigned JITEmitDebugInfo : 1;
-
-    /// JITEmitDebugInfoToDisk - This flag indicates that the JIT should write
-    /// the object files generated by the JITEmitDebugInfo flag to disk.  This
-    /// flag is hidden and is only for debugging the debug info.
-    unsigned JITEmitDebugInfoToDisk : 1;
-
     /// GuaranteedTailCallOpt - This flag is enabled when -tailcallopt is
     /// specified on the commandline. When the flag is on, participating targets
     /// will perform tail call optimization on all calls which use the fastcc
@@ -170,10 +137,6 @@ namespace llvm {
     /// as their parent function, etc.), using an alternate ABI if necessary.
     unsigned GuaranteedTailCallOpt : 1;
 
-    /// DisableTailCalls - This flag controls whether we will use tail calls.
-    /// Disabling them may be useful to maintain a correct call stack.
-    unsigned DisableTailCalls : 1;
-
     /// StackAlignmentOverride - Override default stack alignment for target.
     unsigned StackAlignmentOverride;
 
@@ -209,11 +172,9 @@ namespace llvm {
     /// Emit target-specific trap instruction for 'unreachable' IR instructions.
     unsigned TrapUnreachable : 1;
 
-    /// getTrapFunctionName - If this returns a non-empty string, this means
-    /// isel should lower Intrinsic::trap to a call to the specified function
-    /// name instead of an ISD::TRAP node.
-    std::string TrapFuncName;
-    StringRef getTrapFunctionName() const;
+    /// EmulatedTLS - This flag enables emulated TLS model, using emutls
+    /// function in the runtime library..
+    unsigned EmulatedTLS : 1;
 
     /// FloatABIType - This setting is set by -float-abi=xxx option is specfied
     /// on the command line. This setting may either be Default, Soft, or Hard.
@@ -241,32 +202,17 @@ namespace llvm {
     /// the value of this option.
     FPOpFusion::FPOpFusionMode AllowFPOpFusion;
 
+    /// This class encapsulates options for reciprocal-estimate code generation.
+    TargetRecip Reciprocals;
+    
     /// JTType - This flag specifies the type of jump-instruction table to
     /// create for functions that have the jumptable attribute.
     JumpTable::JumpTableType JTType;
 
-    /// FCFI - This flags controls whether or not forward-edge control-flow
-    /// integrity is applied.
-    bool FCFI;
-
     /// ThreadModel - This flag specifies the type of threading model to assume
     /// for things like atomics
     ThreadModel::Model ThreadModel;
 
-    /// CFIType - This flag specifies the type of control-flow integrity check
-    /// to add as a preamble to indirect calls.
-    CFIntegrity CFIType;
-
-    /// CFIEnforcing - This flags controls whether or not CFI violations cause
-    /// the program to halt.
-    bool CFIEnforcing;
-
-    /// getCFIFuncName - If this returns a non-empty string, then this is the
-    /// name of the function that will be called for each CFI violation in
-    /// non-enforcing mode.
-    std::string CFIFuncName;
-    StringRef getCFIFuncName() const;
-
     /// Machine level options.
     MCTargetOptions MCOptions;
   };
@@ -282,26 +228,19 @@ inline bool operator==(const TargetOptions &LHS,
     ARE_EQUAL(NoInfsFPMath) &&
     ARE_EQUAL(NoNaNsFPMath) &&
     ARE_EQUAL(HonorSignDependentRoundingFPMathOption) &&
-    ARE_EQUAL(UseSoftFloat) &&
     ARE_EQUAL(NoZerosInBSS) &&
-    ARE_EQUAL(JITEmitDebugInfo) &&
-    ARE_EQUAL(JITEmitDebugInfoToDisk) &&
     ARE_EQUAL(GuaranteedTailCallOpt) &&
-    ARE_EQUAL(DisableTailCalls) &&
     ARE_EQUAL(StackAlignmentOverride) &&
     ARE_EQUAL(EnableFastISel) &&
     ARE_EQUAL(PositionIndependentExecutable) &&
     ARE_EQUAL(UseInitArray) &&
     ARE_EQUAL(TrapUnreachable) &&
-    ARE_EQUAL(TrapFuncName) &&
+    ARE_EQUAL(EmulatedTLS) &&
     ARE_EQUAL(FloatABIType) &&
     ARE_EQUAL(AllowFPOpFusion) &&
+    ARE_EQUAL(Reciprocals) &&
     ARE_EQUAL(JTType) &&
-    ARE_EQUAL(FCFI) &&
     ARE_EQUAL(ThreadModel) &&
-    ARE_EQUAL(CFIType) &&
-    ARE_EQUAL(CFIEnforcing) &&
-    ARE_EQUAL(CFIFuncName) &&
     ARE_EQUAL(MCOptions);
 #undef ARE_EQUAL
 }
diff --git a/include/llvm/Target/TargetRecip.h b/include/llvm/Target/TargetRecip.h
new file mode 100644
index 00000000000..4cc3672d758
--- /dev/null
+++ b/include/llvm/Target/TargetRecip.h
@@ -0,0 +1,73 @@
+//===--------------------- llvm/Target/TargetRecip.h ------------*- C++ -*-===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This class is used to customize machine-specific reciprocal estimate code
+// generation in a target-independent way.
+// If a target does not support operations in this specification, then code
+// generation will default to using supported operations.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_TARGET_TARGETRECIP_H
+#define LLVM_TARGET_TARGETRECIP_H
+
+#include "llvm/ADT/StringRef.h"
+#include <vector>
+#include <string>
+#include <map>
+
+namespace llvm {
+
+struct TargetRecip {
+public:
+  TargetRecip();
+
+  /// Initialize all or part of the operations from command-line options or
+  /// a front end.
+  TargetRecip(const std::vector<std::string> &Args);
+  
+  /// Set whether a particular reciprocal operation is enabled and how many
+  /// refinement steps are needed when using it. Use "all" to set enablement
+  /// and refinement steps for all operations.
+  void setDefaults(const StringRef &Key, bool Enable, unsigned RefSteps);
+
+  /// Return true if the reciprocal operation has been enabled by default or
+  /// from the command-line. Return false if the operation has been disabled
+  /// by default or from the command-line.
+  bool isEnabled(const StringRef &Key) const;
+
+  /// Return the number of iterations necessary to refine the
+  /// the result of a machine instruction for the given reciprocal operation.
+  unsigned getRefinementSteps(const StringRef &Key) const;
+
+  bool operator==(const TargetRecip &Other) const;
+
+private:
+  enum {
+    Uninitialized = -1
+  };
+  
+  struct RecipParams {
+    int8_t Enabled;
+    int8_t RefinementSteps;
+    
+    RecipParams() : Enabled(Uninitialized), RefinementSteps(Uninitialized) {}
+  };
+  
+  std::map<StringRef, RecipParams> RecipMap;
+  typedef std::map<StringRef, RecipParams>::iterator RecipIter;
+  typedef std::map<StringRef, RecipParams>::const_iterator ConstRecipIter;
+
+  bool parseGlobalParams(const std::string &Arg);
+  void parseIndividualParams(const std::vector<std::string> &Args);
+};
+
+} // End llvm namespace
+
+#endif
diff --git a/include/llvm/Target/TargetRegisterInfo.h b/include/llvm/Target/TargetRegisterInfo.h
index 121b8a23252..55eab0bfe40 100644
--- a/include/llvm/Target/TargetRegisterInfo.h
+++ b/include/llvm/Target/TargetRegisterInfo.h
@@ -21,6 +21,7 @@
 #include "llvm/CodeGen/MachineValueType.h"
 #include "llvm/IR/CallingConv.h"
 #include "llvm/MC/MCRegisterInfo.h"
+#include "llvm/Support/CommandLine.h"
 #include <cassert>
 #include <functional>
 
@@ -32,6 +33,9 @@ class RegScavenger;
 template<class T> class SmallVectorImpl;
 class VirtRegMap;
 class raw_ostream;
+class LiveRegMatrix;
+
+extern cl::opt<bool> ForceStackAlign;
 
 class TargetRegisterClass {
 public:
@@ -373,6 +377,19 @@ class TargetRegisterInfo : public MCRegisterInfo {
     return SubRegIndexLaneMasks[SubIdx];
   }
 
+  /// Returns true if the given lane mask is imprecise.
+  ///
+  /// LaneMasks as given by getSubRegIndexLaneMask() have a limited number of
+  /// bits, so for targets with more than 31 disjunct subregister indices there
+  /// may be cases where:
+  ///    getSubReg(Reg,A) does not overlap getSubReg(Reg,B)
+  /// but we still have
+  ///    (getSubRegIndexLaneMask(A) & getSubRegIndexLaneMask(B)) != 0.
+  /// This function returns true in those cases.
+  static bool isImpreciseLaneMask(unsigned LaneMask) {
+    return LaneMask & 0x80000000u;
+  }
+
   /// The lane masks returned by getSubRegIndexLaneMask() above can only be
   /// used to determine if sub-registers overlap - they can't be used to
   /// determine if a set of sub-registers completely cover another
@@ -456,6 +473,10 @@ class TargetRegisterInfo : public MCRegisterInfo {
     return nullptr;
   }
 
+  /// Return all the call-preserved register masks defined for this target.
+  virtual ArrayRef<const uint32_t *> getRegMasks() const = 0;
+  virtual ArrayRef<const char *> getRegMaskNames() const = 0;
+
   /// getReservedRegs - Returns a bitset indexed by physical register number
   /// indicating if a register is a special register that has particular uses
   /// and should be considered unavailable at all times, e.g. SP, RA. This is
@@ -692,7 +713,9 @@ class TargetRegisterInfo : public MCRegisterInfo {
                                      ArrayRef<MCPhysReg> Order,
                                      SmallVectorImpl<MCPhysReg> &Hints,
                                      const MachineFunction &MF,
-                                     const VirtRegMap *VRM = nullptr) const;
+                                     const VirtRegMap *VRM = nullptr,
+                                     const LiveRegMatrix *Matrix = nullptr)
+    const;
 
   /// updateRegAllocHint - A callback to allow target a chance to update
   /// register allocation hints when a register is "changed" (e.g. coalesced)
@@ -752,7 +775,7 @@ class TargetRegisterInfo : public MCRegisterInfo {
   /// x86, if the frame register is required, the first fixed stack object is
   /// reserved as its spill slot. This tells PEI not to create a new stack frame
   /// object for the given register. It should be called only after
-  /// processFunctionBeforeCalleeSavedScan().
+  /// determineCalleeSaves().
   virtual bool hasReservedSpillSlot(const MachineFunction &MF, unsigned Reg,
                                     int &FrameIdx) const {
     return false;
@@ -764,12 +787,14 @@ class TargetRegisterInfo : public MCRegisterInfo {
     return false;
   }
 
+  /// canRealignStack - true if the stack can be realigned for the target.
+  virtual bool canRealignStack(const MachineFunction &MF) const;
+
   /// needsStackRealignment - true if storage within the function requires the
   /// stack pointer to be aligned more than the normal calling convention calls
-  /// for.
-  virtual bool needsStackRealignment(const MachineFunction &MF) const {
-    return false;
-  }
+  /// for. This cannot be overriden by the target, but canRealignStack can be
+  /// overriden.
+  bool needsStackRealignment(const MachineFunction &MF) const;
 
   /// getFrameIndexInstrOffset - Get the offset from the referenced frame
   /// index in the instruction, if there is one.
diff --git a/include/llvm/Target/TargetSelectionDAG.td b/include/llvm/Target/TargetSelectionDAG.td
index 5388962fb68..7484bb8dca7 100644
--- a/include/llvm/Target/TargetSelectionDAG.td
+++ b/include/llvm/Target/TargetSelectionDAG.td
@@ -208,6 +208,16 @@ def SDTMaskedLoad: SDTypeProfile<1, 3, [       // masked load
   SDTCisVec<0>, SDTCisPtrTy<1>, SDTCisVec<2>, SDTCisSameAs<0, 3>
 ]>;
 
+def SDTMaskedGather: SDTypeProfile<2, 3, [       // masked gather
+  SDTCisVec<0>, SDTCisVec<1>, SDTCisSameAs<0, 2>, SDTCisSameAs<1, 3>,
+  SDTCisPtrTy<4>, SDTCVecEltisVT<1, i1>, SDTCisSameNumEltsAs<0, 1>
+]>;
+
+def SDTMaskedScatter: SDTypeProfile<1, 3, [       // masked scatter
+  SDTCisVec<0>, SDTCisVec<1>, SDTCisSameAs<0, 2>, SDTCisSameNumEltsAs<0, 1>,
+  SDTCVecEltisVT<0, i1>, SDTCisPtrTy<3>
+]>;
+
 def SDTVecShuffle : SDTypeProfile<1, 2, [
   SDTCisSameAs<0, 1>, SDTCisSameAs<1, 2>
 ]>;
@@ -331,6 +341,7 @@ def externalsym : SDNode<"ISD::ExternalSymbol",       SDTPtrLeaf, [],
                          "ExternalSymbolSDNode">;
 def texternalsym: SDNode<"ISD::TargetExternalSymbol", SDTPtrLeaf, [],
                          "ExternalSymbolSDNode">;
+def mcsym: SDNode<"ISD::MCSymbol", SDTPtrLeaf, [], "MCSymbolSDNode">;
 def blockaddress : SDNode<"ISD::BlockAddress",        SDTPtrLeaf, [],
                          "BlockAddressSDNode">;
 def tblockaddress: SDNode<"ISD::TargetBlockAddress",  SDTPtrLeaf, [],
@@ -370,7 +381,13 @@ def subc       : SDNode<"ISD::SUBC"      , SDTIntBinOp,
                         [SDNPOutGlue]>;
 def sube       : SDNode<"ISD::SUBE"      , SDTIntBinOp,
                         [SDNPOutGlue, SDNPInGlue]>;
+def smin       : SDNode<"ISD::SMIN"      , SDTIntBinOp>;
+def smax       : SDNode<"ISD::SMAX"      , SDTIntBinOp>;
+def umin       : SDNode<"ISD::UMIN"      , SDTIntBinOp>;
+def umax       : SDNode<"ISD::UMAX"      , SDTIntBinOp>;
 
+def sabsdiff   : SDNode<"ISD::SABSDIFF"   , SDTIntBinOp>;
+def uabsdiff   : SDNode<"ISD::UABSDIFF"   , SDTIntBinOp>;
 def sext_inreg : SDNode<"ISD::SIGN_EXTEND_INREG", SDTExtInreg>;
 def bswap      : SDNode<"ISD::BSWAP"      , SDTIntUnaryOp>;
 def ctlz       : SDNode<"ISD::CTLZ"       , SDTIntUnaryOp>;
@@ -480,6 +497,10 @@ def masked_store : SDNode<"ISD::MSTORE",  SDTMaskedStore,
                        [SDNPHasChain, SDNPMayStore, SDNPMemOperand]>;
 def masked_load  : SDNode<"ISD::MLOAD",  SDTMaskedLoad,
                        [SDNPHasChain, SDNPMayLoad, SDNPMemOperand]>;
+def masked_scatter : SDNode<"ISD::MSCATTER",  SDTMaskedScatter,
+                       [SDNPHasChain, SDNPMayStore, SDNPMemOperand]>;
+def masked_gather  : SDNode<"ISD::MGATHER",  SDTMaskedGather,
+                       [SDNPHasChain, SDNPMayLoad, SDNPMemOperand]>;
 
 // Do not use ld, st directly. Use load, extload, sextload, zextload, store,
 // and truncst (see below).
@@ -796,6 +817,21 @@ def truncstoref64 : PatFrag<(ops node:$val, node:$ptr),
   return cast<StoreSDNode>(N)->getMemoryVT() == MVT::f64;
 }]>;
 
+def truncstorevi8 : PatFrag<(ops node:$val, node:$ptr),
+                            (truncstore node:$val, node:$ptr), [{
+  return cast<StoreSDNode>(N)->getMemoryVT().getScalarType() == MVT::i8;
+}]>;
+
+def truncstorevi16 : PatFrag<(ops node:$val, node:$ptr),
+                             (truncstore node:$val, node:$ptr), [{
+  return cast<StoreSDNode>(N)->getMemoryVT().getScalarType() == MVT::i16;
+}]>;
+
+def truncstorevi32 : PatFrag<(ops node:$val, node:$ptr),
+                             (truncstore node:$val, node:$ptr), [{
+  return cast<StoreSDNode>(N)->getMemoryVT().getScalarType() == MVT::i32;
+}]>;
+
 // indexed store fragments.
 def istore : PatFrag<(ops node:$val, node:$base, node:$offset),
                      (ist node:$val, node:$base, node:$offset), [{
diff --git a/include/llvm/Target/TargetSelectionDAGInfo.h b/include/llvm/Target/TargetSelectionDAGInfo.h
index bacdd950705..53db5aa8429 100644
--- a/include/llvm/Target/TargetSelectionDAGInfo.h
+++ b/include/llvm/Target/TargetSelectionDAGInfo.h
@@ -20,8 +20,6 @@
 
 namespace llvm {
 
-class DataLayout;
-
 //===----------------------------------------------------------------------===//
 /// TargetSelectionDAGInfo - Targets can subclass this to parameterize the
 /// SelectionDAG lowering and instruction selection process.
@@ -30,13 +28,8 @@ class TargetSelectionDAGInfo {
   TargetSelectionDAGInfo(const TargetSelectionDAGInfo &) = delete;
   void operator=(const TargetSelectionDAGInfo &) = delete;
 
-  const DataLayout *DL;
-
-protected:
-  const DataLayout *getDataLayout() const { return DL; }
-
 public:
-  explicit TargetSelectionDAGInfo(const DataLayout *DL);
+  explicit TargetSelectionDAGInfo() = default;
   virtual ~TargetSelectionDAGInfo();
 
   /// EmitTargetCodeForMemcpy - Emit target-specific code that performs a
diff --git a/include/llvm/Target/TargetSubtargetInfo.h b/include/llvm/Target/TargetSubtargetInfo.h
index 0f427901a8e..d50aa4932f8 100644
--- a/include/llvm/Target/TargetSubtargetInfo.h
+++ b/include/llvm/Target/TargetSubtargetInfo.h
@@ -15,6 +15,7 @@
 #define LLVM_TARGET_TARGETSUBTARGETINFO_H
 
 #include "llvm/CodeGen/PBQPRAConstraint.h"
+#include "llvm/CodeGen/SchedulerRegistry.h"
 #include "llvm/MC/MCSubtargetInfo.h"
 #include "llvm/Support/CodeGen.h"
 
@@ -44,9 +45,17 @@ template <typename T> class SmallVectorImpl;
 class TargetSubtargetInfo : public MCSubtargetInfo {
   TargetSubtargetInfo(const TargetSubtargetInfo &) = delete;
   void operator=(const TargetSubtargetInfo &) = delete;
+  TargetSubtargetInfo() = delete;
 
 protected: // Can only create subclasses...
-  TargetSubtargetInfo();
+  TargetSubtargetInfo(const Triple &TT, StringRef CPU, StringRef FS,
+                      ArrayRef<SubtargetFeatureKV> PF,
+                      ArrayRef<SubtargetFeatureKV> PD,
+                      const SubtargetInfoKV *ProcSched,
+                      const MCWriteProcResEntry *WPR,
+                      const MCWriteLatencyEntry *WL,
+                      const MCReadAdvanceEntry *RA, const InstrStage *IS,
+                      const unsigned *OC, const unsigned *FP);
 
 public:
   // AntiDepBreakMode - Type of anti-dependence breaking that should
@@ -73,6 +82,11 @@ class TargetSubtargetInfo : public MCSubtargetInfo {
   virtual const TargetSelectionDAGInfo *getSelectionDAGInfo() const {
     return nullptr;
   }
+  /// Target can subclass this hook to select a different DAG scheduler.
+  virtual RegisterScheduler::FunctionPassCtor
+      getDAGScheduler(CodeGenOpt::Level) const {
+    return nullptr;
+  }
 
   /// getRegisterInfo - If register information is available, return it.  If
   /// not, return null.  This is kept separate from RegInfo until RegInfo has
@@ -115,12 +129,11 @@ class TargetSubtargetInfo : public MCSubtargetInfo {
   /// can be overridden.
   virtual bool enableJoinGlobalCopies() const;
 
-  /// \brief True if the subtarget should run PostMachineScheduler.
+  /// True if the subtarget should run a scheduler after register allocation.
   ///
-  /// This only takes effect if the target has configured the
-  /// PostMachineScheduler pass to run, or if the global cl::opt flag,
-  /// MISchedPostRA, is set.
-  virtual bool enablePostMachineScheduler() const;
+  /// By default this queries the PostRAScheduling bit in the scheduling model
+  /// which is the preferred way to influence this.
+  virtual bool enablePostRAScheduler() const;
 
   /// \brief True if the subtarget should run the atomic expansion pass.
   virtual bool enableAtomicExpand() const;
diff --git a/include/llvm/Transforms/IPO.h b/include/llvm/Transforms/IPO.h
index fbd999cbc94..2ea47301bb4 100644
--- a/include/llvm/Transforms/IPO.h
+++ b/include/llvm/Transforms/IPO.h
@@ -70,6 +70,12 @@ ModulePass *createGlobalOptimizerPass();
 ///
 ModulePass *createGlobalDCEPass();
 
+//===----------------------------------------------------------------------===//
+/// This transform is designed to eliminate available external globals
+/// (functions or global variables)
+///
+ModulePass *createEliminateAvailableExternallyPass();
+
 //===----------------------------------------------------------------------===//
 /// createGVExtractionPass - If deleteFn is true, this pass deletes
 /// the specified global values. Otherwise, it deletes as much of the module as
diff --git a/include/llvm/Transforms/IPO/LowerBitSets.h b/include/llvm/Transforms/IPO/LowerBitSets.h
index 55d7d84560a..f678e86e686 100644
--- a/include/llvm/Transforms/IPO/LowerBitSets.h
+++ b/include/llvm/Transforms/IPO/LowerBitSets.h
@@ -28,6 +28,7 @@ namespace llvm {
 class DataLayout;
 class GlobalVariable;
 class Value;
+class raw_ostream;
 
 struct BitSetInfo {
   // The indices of the set bits in the bitset.
@@ -57,6 +58,8 @@ struct BitSetInfo {
   bool containsValue(const DataLayout &DL,
                      const DenseMap<GlobalVariable *, uint64_t> &GlobalLayout,
                      Value *V, uint64_t COffset = 0) const;
+
+  void print(raw_ostream &OS) const;
 };
 
 struct BitSetBuilder {
diff --git a/include/llvm/Transforms/IPO/PassManagerBuilder.h b/include/llvm/Transforms/IPO/PassManagerBuilder.h
index 5d574ae0bf0..ef01fa35053 100644
--- a/include/llvm/Transforms/IPO/PassManagerBuilder.h
+++ b/include/llvm/Transforms/IPO/PassManagerBuilder.h
@@ -81,6 +81,11 @@ class PassManagerBuilder {
     /// run after everything else.
     EP_OptimizerLast,
 
+    /// EP_VectorizerStart - This extension point allows adding optimization
+    /// passes before the vectorizer and other highly target specific
+    /// optimization passes are executed.
+    EP_VectorizerStart,
+
     /// EP_EnabledOnOptLevel0 - This extension point allows adding passes that
     /// should not be disabled by O0 optimization level. The passes will be
     /// inserted after the inlining pass.
@@ -121,6 +126,7 @@ class PassManagerBuilder {
   bool VerifyInput;
   bool VerifyOutput;
   bool MergeFunctions;
+  bool PrepareForLTO;
 
 private:
   /// ExtensionList - This is list of all of the extensions that are registered.
diff --git a/include/llvm/Transforms/Instrumentation.h b/include/llvm/Transforms/Instrumentation.h
index a1477930ed6..250e3893cb1 100644
--- a/include/llvm/Transforms/Instrumentation.h
+++ b/include/llvm/Transforms/Instrumentation.h
@@ -74,6 +74,9 @@ struct InstrProfOptions {
 
   // Add the 'noredzone' attribute to added runtime library calls.
   bool NoRedZone;
+
+  // Name of the profile file to use as output
+  std::string InstrProfileOutput;
 };
 
 /// Insert frontend instrumentation based profiling.
@@ -81,8 +84,8 @@ ModulePass *createInstrProfilingPass(
     const InstrProfOptions &Options = InstrProfOptions());
 
 // Insert AddressSanitizer (address sanity checking) instrumentation
-FunctionPass *createAddressSanitizerFunctionPass();
-ModulePass *createAddressSanitizerModulePass();
+FunctionPass *createAddressSanitizerFunctionPass(bool CompileKernel = false);
+ModulePass *createAddressSanitizerModulePass(bool CompileKernel = false);
 
 // Insert MemorySanitizer instrumentation (detection of uninitialized reads)
 FunctionPass *createMemorySanitizerPass(int TrackOrigins = 0);
@@ -95,8 +98,27 @@ ModulePass *createDataFlowSanitizerPass(
     const std::vector<std::string> &ABIListFiles = std::vector<std::string>(),
     void *(*getArgTLS)() = nullptr, void *(*getRetValTLS)() = nullptr);
 
+// Options for sanitizer coverage instrumentation.
+struct SanitizerCoverageOptions {
+  SanitizerCoverageOptions()
+      : CoverageType(SCK_None), IndirectCalls(false), TraceBB(false),
+        TraceCmp(false), Use8bitCounters(false) {}
+
+  enum Type {
+    SCK_None = 0,
+    SCK_Function,
+    SCK_BB,
+    SCK_Edge
+  } CoverageType;
+  bool IndirectCalls;
+  bool TraceBB;
+  bool TraceCmp;
+  bool Use8bitCounters;
+};
+
 // Insert SanitizerCoverage instrumentation.
-ModulePass *createSanitizerCoverageModulePass(int CoverageLevel);
+ModulePass *createSanitizerCoverageModulePass(
+    const SanitizerCoverageOptions &Options = SanitizerCoverageOptions());
 
 #if defined(__GNUC__) && defined(__linux__) && !defined(ANDROID)
 inline ModulePass *createDataFlowSanitizerPassForJIT(
@@ -110,6 +132,10 @@ inline ModulePass *createDataFlowSanitizerPassForJIT(
 // checking on loads, stores, and other memory intrinsics.
 FunctionPass *createBoundsCheckingPass();
 
+/// \brief This pass splits the stack into a safe stack and an unsafe stack to
+/// protect against stack-based overflow vulnerabilities.
+FunctionPass *createSafeStackPass();
+
 } // End llvm namespace
 
 #endif
diff --git a/include/llvm/Transforms/MinSFI.h b/include/llvm/Transforms/MinSFI.h
deleted file mode 100644
index 1c0838343dc..00000000000
--- a/include/llvm/Transforms/MinSFI.h
+++ /dev/null
@@ -1,43 +0,0 @@
-//===-- MinSFI.h - MinSFI Transformations -----------------------*- C++ -*-===//
-//
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
-//
-//===----------------------------------------------------------------------===//
-
-#ifndef LLVM_TRANSFORMS_MINSFI_H
-#define LLVM_TRANSFORMS_MINSFI_H
-
-#include "llvm/CodeGen/Passes.h"
-#include "llvm/Support/DataTypes.h"
-
-namespace llvm {
-namespace minsfi {
-
-// Returns the bit size of sandboxed pointers. The default value is 32 bits
-// and it can be set with the "-minsfi-ptrsize" command-line option.
-uint32_t GetPointerSizeInBits();
-
-// Returns the number of bytes addressable by the sandbox. This value is given
-// by the size of the sandboxed pointers.
-uint64_t GetAddressSubspaceSize();
-
-// The name of MinSFI's entry function which will be invoked by the runtime.
-extern const char EntryFunctionName[];
-
-}  // namespace minsfi
-
-FunctionPass *createSubstituteUndefsPass();
-ModulePass *createAllocateDataSegmentPass();
-ModulePass *createExpandAllocasPass();
-ModulePass *createRenameEntryPointPass();
-ModulePass *createSandboxIndirectCallsPass();
-ModulePass *createSandboxMemoryAccessesPass();
-
-void MinSFIPasses(PassManagerBase &PM);
-
-}  // namespace llvm
-
-#endif
diff --git a/include/llvm/Transforms/Scalar.h b/include/llvm/Transforms/Scalar.h
index 925ebda1317..4676c95d7cd 100644
--- a/include/llvm/Transforms/Scalar.h
+++ b/include/llvm/Transforms/Scalar.h
@@ -16,10 +16,12 @@
 #define LLVM_TRANSFORMS_SCALAR_H
 
 #include "llvm/ADT/StringRef.h"
+#include <functional>
 
 namespace llvm {
 
 class BasicBlockPass;
+class Function;
 class FunctionPass;
 class ModulePass;
 class Pass;
@@ -152,7 +154,14 @@ Pass *createLoopInterchangePass();
 //
 Pass *createLoopStrengthReducePass();
 
-Pass *createGlobalMergePass(const TargetMachine *TM, unsigned MaximalOffset);
+//===----------------------------------------------------------------------===//
+//
+// GlobalMerge - This pass merges internal (by default) globals into structs
+// to enable reuse of a base pointer by indexed addressing modes.
+// It can also be configured to focus on size optimizations only.
+//
+Pass *createGlobalMergePass(const TargetMachine *TM, unsigned MaximalOffset,
+                            bool OnlyOptimizeForSize = false);
 
 //===----------------------------------------------------------------------===//
 //
@@ -238,7 +247,8 @@ FunctionPass *createJumpThreadingPass(int Threshold = -1);
 // CFGSimplification - Merge basic blocks, eliminate unreachable blocks,
 // simplify terminator instructions, etc...
 //
-FunctionPass *createCFGSimplificationPass(int Threshold = -1);
+FunctionPass *createCFGSimplificationPass(
+    int Threshold = -1, std::function<bool(const Function &)> Ftor = nullptr);
 
 //===----------------------------------------------------------------------===//
 //
@@ -421,6 +431,13 @@ FunctionPass *
 createSeparateConstOffsetFromGEPPass(const TargetMachine *TM = nullptr,
                                      bool LowerGEP = false);
 
+//===----------------------------------------------------------------------===//
+//
+// SpeculativeExecution - Aggressively hoist instructions to enable
+// speculative execution on targets where branches are expensive.
+//
+FunctionPass *createSpeculativeExecutionPass();
+
 //===----------------------------------------------------------------------===//
 //
 // LoadCombine - Combine loads into bigger loads.
@@ -442,14 +459,14 @@ FunctionPass *createStraightLineStrengthReducePass();
 // RewriteStatepointsForGC which can be run at an arbitrary point in the pass
 // order following this pass.
 //
-ModulePass *createPlaceSafepointsPass();
+FunctionPass *createPlaceSafepointsPass();
 
 //===----------------------------------------------------------------------===//
 //
 // RewriteStatepointsForGC - Rewrite any gc.statepoints which do not yet have
 // explicit relocations to include explicit relocations.
 //
-FunctionPass *createRewriteStatepointsForGCPass();
+ModulePass *createRewriteStatepointsForGCPass();
 
 //===----------------------------------------------------------------------===//
 //
@@ -463,6 +480,12 @@ FunctionPass *createFloat2IntPass();
 //
 FunctionPass *createNaryReassociatePass();
 
+//===----------------------------------------------------------------------===//
+//
+// LoopDistribute - Distribute loops.
+//
+FunctionPass *createLoopDistributePass();
+
 } // End llvm namespace
 
 #endif
diff --git a/include/llvm/Transforms/Utils/BasicBlockUtils.h b/include/llvm/Transforms/Utils/BasicBlockUtils.h
index 710db03c45d..13c856dfdc9 100644
--- a/include/llvm/Transforms/Utils/BasicBlockUtils.h
+++ b/include/llvm/Transforms/Utils/BasicBlockUtils.h
@@ -22,7 +22,6 @@
 
 namespace llvm {
 
-class AliasAnalysis;
 class MemoryDependenceAnalysis;
 class DominatorTree;
 class LoopInfo;
@@ -40,7 +39,7 @@ void DeleteDeadBlock(BasicBlock *BB);
 /// any single-entry PHI nodes in it, fold them away.  This handles the case
 /// when all entries to the PHI nodes in a block are guaranteed equal, such as
 /// when the block has exactly one predecessor.
-void FoldSingleEntryPHINodes(BasicBlock *BB, AliasAnalysis *AA = nullptr,
+void FoldSingleEntryPHINodes(BasicBlock *BB,
                              MemoryDependenceAnalysis *MemDep = nullptr);
 
 /// DeleteDeadPHIs - Examine each PHI in the given block and delete it if it
@@ -54,7 +53,6 @@ bool DeleteDeadPHIs(BasicBlock *BB, const TargetLibraryInfo *TLI = nullptr);
 /// if possible.  The return value indicates success or failure.
 bool MergeBlockIntoPredecessor(BasicBlock *BB, DominatorTree *DT = nullptr,
                                LoopInfo *LI = nullptr,
-                               AliasAnalysis *AA = nullptr,
                                MemoryDependenceAnalysis *MemDep = nullptr);
 
 // ReplaceInstWithValue - Replace all uses of an instruction (specified by BI)
@@ -64,14 +62,16 @@ void ReplaceInstWithValue(BasicBlock::InstListType &BIL,
                           BasicBlock::iterator &BI, Value *V);
 
 // ReplaceInstWithInst - Replace the instruction specified by BI with the
-// instruction specified by I.  The original instruction is deleted and BI is
+// instruction specified by I. Copies DebugLoc from BI to I, if I doesn't
+// already have a DebugLoc. The original instruction is deleted and BI is
 // updated to point to the new instruction.
 //
 void ReplaceInstWithInst(BasicBlock::InstListType &BIL,
                          BasicBlock::iterator &BI, Instruction *I);
 
 // ReplaceInstWithInst - Replace the instruction specified by From with the
-// instruction specified by To.
+// instruction specified by To. Copies DebugLoc from BI to I, if I doesn't
+// already have a DebugLoc.
 //
 void ReplaceInstWithInst(Instruction *From, Instruction *To);
 
@@ -80,27 +80,15 @@ void ReplaceInstWithInst(Instruction *From, Instruction *To);
 /// This provides a builder interface for overriding the default options used
 /// during critical edge splitting.
 struct CriticalEdgeSplittingOptions {
-  AliasAnalysis *AA;
   DominatorTree *DT;
   LoopInfo *LI;
   bool MergeIdenticalEdges;
   bool DontDeleteUselessPHIs;
   bool PreserveLCSSA;
 
-  CriticalEdgeSplittingOptions()
-      : AA(nullptr), DT(nullptr), LI(nullptr), MergeIdenticalEdges(false),
-        DontDeleteUselessPHIs(false), PreserveLCSSA(false) {}
-
-  /// \brief Basic case of setting up all the analysis.
-  CriticalEdgeSplittingOptions(AliasAnalysis *AA, DominatorTree *DT = nullptr,
+  CriticalEdgeSplittingOptions(DominatorTree *DT = nullptr,
                                LoopInfo *LI = nullptr)
-      : AA(AA), DT(DT), LI(LI), MergeIdenticalEdges(false),
-        DontDeleteUselessPHIs(false), PreserveLCSSA(false) {}
-
-  /// \brief A common pattern is to preserve the dominator tree and loop
-  /// info but not care about AA.
-  CriticalEdgeSplittingOptions(DominatorTree *DT, LoopInfo *LI)
-      : AA(nullptr), DT(DT), LI(LI), MergeIdenticalEdges(false),
+      : DT(DT), LI(LI), MergeIdenticalEdges(false),
         DontDeleteUselessPHIs(false), PreserveLCSSA(false) {}
 
   CriticalEdgeSplittingOptions &setMergeIdenticalEdges() {
@@ -212,15 +200,13 @@ BasicBlock *SplitBlock(BasicBlock *Old, Instruction *SplitPt,
 /// It will have Suffix+".split_lp". See SplitLandingPadPredecessors for more
 /// details on this case.
 ///
-/// This currently updates the LLVM IR, AliasAnalysis, DominatorTree,
-/// DominanceFrontier, LoopInfo, and LCCSA but no other analyses.
-/// In particular, it does not preserve LoopSimplify (because it's
-/// complicated to handle the case where one of the edges being split
-/// is an exit of a loop with other exits).
+/// This currently updates the LLVM IR, DominatorTree, LoopInfo, and LCCSA but
+/// no other analyses. In particular, it does not preserve LoopSimplify
+/// (because it's complicated to handle the case where one of the edges being
+/// split is an exit of a loop with other exits).
 ///
 BasicBlock *SplitBlockPredecessors(BasicBlock *BB, ArrayRef<BasicBlock *> Preds,
                                    const char *Suffix,
-                                   AliasAnalysis *AA = nullptr,
                                    DominatorTree *DT = nullptr,
                                    LoopInfo *LI = nullptr,
                                    bool PreserveLCSSA = false);
@@ -232,17 +218,15 @@ BasicBlock *SplitBlockPredecessors(BasicBlock *BB, ArrayRef<BasicBlock *> Preds,
 /// OrigBB is clone into both of the new basic blocks. The new blocks are given
 /// the suffixes 'Suffix1' and 'Suffix2', and are returned in the NewBBs vector.
 ///
-/// This currently updates the LLVM IR, AliasAnalysis, DominatorTree,
-/// DominanceFrontier, LoopInfo, and LCCSA but no other analyses. In particular,
-/// it does not preserve LoopSimplify (because it's complicated to handle the
-/// case where one of the edges being split is an exit of a loop with other
-/// exits).
+/// This currently updates the LLVM IR, DominatorTree, LoopInfo, and LCCSA but
+/// no other analyses. In particular, it does not preserve LoopSimplify
+/// (because it's complicated to handle the case where one of the edges being
+/// split is an exit of a loop with other exits).
 ///
 void SplitLandingPadPredecessors(BasicBlock *OrigBB,
                                  ArrayRef<BasicBlock *> Preds,
                                  const char *Suffix, const char *Suffix2,
                                  SmallVectorImpl<BasicBlock *> &NewBBs,
-                                 AliasAnalysis *AA = nullptr,
                                  DominatorTree *DT = nullptr,
                                  LoopInfo *LI = nullptr,
                                  bool PreserveLCSSA = false);
diff --git a/include/llvm/Transforms/Utils/Cloning.h b/include/llvm/Transforms/Utils/Cloning.h
index cb187ec103d..2caa9a2462d 100644
--- a/include/llvm/Transforms/Utils/Cloning.h
+++ b/include/llvm/Transforms/Utils/Cloning.h
@@ -45,6 +45,7 @@ class LoopInfo;
 class AllocaInst;
 class AliasAnalysis;
 class AssumptionCacheTracker;
+class DominatorTree;
 
 /// CloneModule - Return an exact copy of the specified module
 ///
@@ -233,6 +234,21 @@ bool InlineFunction(InvokeInst *II, InlineFunctionInfo &IFI,
 bool InlineFunction(CallSite CS, InlineFunctionInfo &IFI,
                     bool InsertLifetime = true);
 
+/// \brief Clones a loop \p OrigLoop.  Returns the loop and the blocks in \p
+/// Blocks.
+///
+/// Updates LoopInfo and DominatorTree assuming the loop is dominated by block
+/// \p LoopDomBB.  Insert the new blocks before block specified in \p Before.
+Loop *cloneLoopWithPreheader(BasicBlock *Before, BasicBlock *LoopDomBB,
+                             Loop *OrigLoop, ValueToValueMapTy &VMap,
+                             const Twine &NameSuffix, LoopInfo *LI,
+                             DominatorTree *DT,
+                             SmallVectorImpl<BasicBlock *> &Blocks);
+
+/// \brief Remaps instructions in \p Blocks using the mapping in \p VMap.
+void remapInstructionsInBlocks(const SmallVectorImpl<BasicBlock *> &Blocks,
+                               ValueToValueMapTy &VMap);
+
 } // End llvm namespace
 
 #endif
diff --git a/include/llvm/Transforms/Utils/Local.h b/include/llvm/Transforms/Utils/Local.h
index e89169cdcbc..a1bb367ac7b 100644
--- a/include/llvm/Transforms/Utils/Local.h
+++ b/include/llvm/Transforms/Utils/Local.h
@@ -16,6 +16,7 @@
 #define LLVM_TRANSFORMS_UTILS_LOCAL_H
 
 #include "llvm/IR/DataLayout.h"
+#include "llvm/IR/Dominators.h"
 #include "llvm/IR/GetElementPtrTypeIterator.h"
 #include "llvm/IR/IRBuilder.h"
 #include "llvm/IR/Operator.h"
@@ -286,6 +287,10 @@ bool removeUnreachableBlocks(Function &F);
 /// Metadata not listed as known via KnownIDs is removed
 void combineMetadata(Instruction *K, const Instruction *J, ArrayRef<unsigned> KnownIDs);
 
+/// \brief Replace each use of 'From' with 'To' if that use is dominated by 
+/// the given edge.  Returns the number of replacements made.
+unsigned replaceDominatedUsesWith(Value *From, Value *To, DominatorTree &DT,
+                                  const BasicBlockEdge &Edge);
 } // End llvm namespace
 
 #endif
diff --git a/include/llvm/Transforms/Utils/LoopUtils.h b/include/llvm/Transforms/Utils/LoopUtils.h
index 6ce2aace99a..1550de01b3f 100644
--- a/include/llvm/Transforms/Utils/LoopUtils.h
+++ b/include/llvm/Transforms/Utils/LoopUtils.h
@@ -43,12 +43,37 @@ struct LICMSafetyInfo {
   {}
 };
 
-/// This POD struct holds information about a potential reduction operation.
-class ReductionInstDesc {
+/// The RecurrenceDescriptor is used to identify recurrences variables in a
+/// loop. Reduction is a special case of recurrence that has uses of the
+/// recurrence variable outside the loop. The method isReductionPHI identifies
+/// reductions that are basic recurrences.
+///
+/// Basic recurrences are defined as the summation, product, OR, AND, XOR, min,
+/// or max of a set of terms. For example: for(i=0; i<n; i++) { total +=
+/// array[i]; } is a summation of array elements. Basic recurrences are a
+/// special case of chains of recurrences (CR). See ScalarEvolution for CR
+/// references.
+
+/// This struct holds information about recurrence variables.
+class RecurrenceDescriptor {
 
 public:
-  // This enum represents the kind of minmax reduction.
-  enum MinMaxReductionKind {
+  /// This enum represents the kinds of recurrences that we support.
+  enum RecurrenceKind {
+    RK_NoRecurrence,  ///< Not a recurrence.
+    RK_IntegerAdd,    ///< Sum of integers.
+    RK_IntegerMult,   ///< Product of integers.
+    RK_IntegerOr,     ///< Bitwise or logical OR of numbers.
+    RK_IntegerAnd,    ///< Bitwise or logical AND of numbers.
+    RK_IntegerXor,    ///< Bitwise or logical XOR of numbers.
+    RK_IntegerMinMax, ///< Min/max implemented in terms of select(cmp()).
+    RK_FloatAdd,      ///< Sum of floats.
+    RK_FloatMult,     ///< Product of floats.
+    RK_FloatMinMax    ///< Min/max implemented in terms of select(cmp()).
+  };
+
+  // This enum represents the kind of minmax recurrence.
+  enum MinMaxRecurrenceKind {
     MRK_Invalid,
     MRK_UIntMin,
     MRK_UIntMax,
@@ -57,62 +82,48 @@ class ReductionInstDesc {
     MRK_FloatMin,
     MRK_FloatMax
   };
-  ReductionInstDesc(bool IsRedux, Instruction *I)
-      : IsReduction(IsRedux), PatternLastInst(I), MinMaxKind(MRK_Invalid) {}
 
-  ReductionInstDesc(Instruction *I, MinMaxReductionKind K)
-      : IsReduction(true), PatternLastInst(I), MinMaxKind(K) {}
+  RecurrenceDescriptor()
+      : StartValue(nullptr), LoopExitInstr(nullptr), Kind(RK_NoRecurrence),
+        MinMaxKind(MRK_Invalid) {}
 
-  bool isReduction() { return IsReduction; }
+  RecurrenceDescriptor(Value *Start, Instruction *Exit, RecurrenceKind K,
+                       MinMaxRecurrenceKind MK)
+      : StartValue(Start), LoopExitInstr(Exit), Kind(K), MinMaxKind(MK) {}
 
-  MinMaxReductionKind getMinMaxKind() { return MinMaxKind; }
- 
-  Instruction *getPatternInst() { return PatternLastInst; }
+  /// This POD struct holds information about a potential recurrence operation.
+  class InstDesc {
 
-private:
-  // Is this instruction a reduction candidate.
-  bool IsReduction;
-  // The last instruction in a min/max pattern (select of the select(icmp())
-  // pattern), or the current reduction instruction otherwise.
-  Instruction *PatternLastInst;
-  // If this is a min/max pattern the comparison predicate.
-  MinMaxReductionKind MinMaxKind;
-};
+  public:
+    InstDesc(bool IsRecur, Instruction *I)
+        : IsRecurrence(IsRecur), PatternLastInst(I), MinMaxKind(MRK_Invalid) {}
 
-/// This struct holds information about reduction variables.
-class ReductionDescriptor {
+    InstDesc(Instruction *I, MinMaxRecurrenceKind K)
+        : IsRecurrence(true), PatternLastInst(I), MinMaxKind(K) {}
 
-public:
-  /// This enum represents the kinds of reductions that we support.
-  enum ReductionKind {
-    RK_NoReduction,   ///< Not a reduction.
-    RK_IntegerAdd,    ///< Sum of integers.
-    RK_IntegerMult,   ///< Product of integers.
-    RK_IntegerOr,     ///< Bitwise or logical OR of numbers.
-    RK_IntegerAnd,    ///< Bitwise or logical AND of numbers.
-    RK_IntegerXor,    ///< Bitwise or logical XOR of numbers.
-    RK_IntegerMinMax, ///< Min/max implemented in terms of select(cmp()).
-    RK_FloatAdd,      ///< Sum of floats.
-    RK_FloatMult,     ///< Product of floats.
-    RK_FloatMinMax    ///< Min/max implemented in terms of select(cmp()).
-  };
+    bool isRecurrence() { return IsRecurrence; }
 
-  ReductionDescriptor()
-      : StartValue(nullptr), LoopExitInstr(nullptr), Kind(RK_NoReduction),
-        MinMaxKind(ReductionInstDesc::MRK_Invalid) {}
+    MinMaxRecurrenceKind getMinMaxKind() { return MinMaxKind; }
 
-  ReductionDescriptor(Value *Start, Instruction *Exit, ReductionKind K,
-                      ReductionInstDesc::MinMaxReductionKind MK)
-      : StartValue(Start), LoopExitInstr(Exit), Kind(K), MinMaxKind(MK) {}
+    Instruction *getPatternInst() { return PatternLastInst; }
+
+  private:
+    // Is this instruction a recurrence candidate.
+    bool IsRecurrence;
+    // The last instruction in a min/max pattern (select of the select(icmp())
+    // pattern), or the current recurrence instruction otherwise.
+    Instruction *PatternLastInst;
+    // If this is a min/max pattern the comparison predicate.
+    MinMaxRecurrenceKind MinMaxKind;
+  };
 
-  /// Returns a struct describing if the instruction 'I' can be a reduction
-  /// variable of type 'Kind'. If the reduction is a min/max pattern of
+  /// Returns a struct describing if the instruction 'I' can be a recurrence
+  /// variable of type 'Kind'. If the recurrence is a min/max pattern of
   /// select(icmp()) this function advances the instruction pointer 'I' from the
   /// compare instruction to the select instruction and stores this pointer in
   /// 'PatternLastInst' member of the returned struct.
-  static ReductionInstDesc isReductionInstr(Instruction *I, ReductionKind Kind,
-                                            ReductionInstDesc &Prev,
-                                            bool HasFunNoNaNAttr);
+  static InstDesc isRecurrenceInstr(Instruction *I, RecurrenceKind Kind,
+                                    InstDesc &Prev, bool HasFunNoNaNAttr);
 
   /// Returns true if instuction I has multiple uses in Insts
   static bool hasMultipleUsesOf(Instruction *I,
@@ -124,51 +135,48 @@ class ReductionDescriptor {
   /// Returns a struct describing if the instruction if the instruction is a
   /// Select(ICmp(X, Y), X, Y) instruction pattern corresponding to a min(X, Y)
   /// or max(X, Y).
-  static ReductionInstDesc isMinMaxSelectCmpPattern(Instruction *I,
-                                                    ReductionInstDesc &Prev);
+  static InstDesc isMinMaxSelectCmpPattern(Instruction *I, InstDesc &Prev);
 
-  /// Returns identity corresponding to the ReductionKind.
-  static Constant *getReductionIdentity(ReductionKind K, Type *Tp);
+  /// Returns identity corresponding to the RecurrenceKind.
+  static Constant *getRecurrenceIdentity(RecurrenceKind K, Type *Tp);
 
-  /// Returns the opcode of binary operation corresponding to the ReductionKind.
-  static unsigned getReductionBinOp(ReductionKind Kind);
+  /// Returns the opcode of binary operation corresponding to the
+  /// RecurrenceKind.
+  static unsigned getRecurrenceBinOp(RecurrenceKind Kind);
 
-  /// Returns a Min/Max operation corresponding to MinMaxReductionKind.
-  static Value *createMinMaxOp(IRBuilder<> &Builder,
-                               ReductionInstDesc::MinMaxReductionKind RK,
+  /// Returns a Min/Max operation corresponding to MinMaxRecurrenceKind.
+  static Value *createMinMaxOp(IRBuilder<> &Builder, MinMaxRecurrenceKind RK,
                                Value *Left, Value *Right);
 
   /// Returns true if Phi is a reduction of type Kind and adds it to the
-  /// ReductionDescriptor.
-  static bool AddReductionVar(PHINode *Phi, ReductionKind Kind, Loop *TheLoop,
+  /// RecurrenceDescriptor.
+  static bool AddReductionVar(PHINode *Phi, RecurrenceKind Kind, Loop *TheLoop,
                               bool HasFunNoNaNAttr,
-                              ReductionDescriptor &RedDes);
+                              RecurrenceDescriptor &RedDes);
 
-  /// Returns true if Phi is a reduction in TheLoop. The ReductionDescriptor is
+  /// Returns true if Phi is a reduction in TheLoop. The RecurrenceDescriptor is
   /// returned in RedDes.
   static bool isReductionPHI(PHINode *Phi, Loop *TheLoop,
-                             ReductionDescriptor &RedDes);
+                             RecurrenceDescriptor &RedDes);
 
-  ReductionKind getReductionKind() { return Kind; }
+  RecurrenceKind getRecurrenceKind() { return Kind; }
 
-  ReductionInstDesc::MinMaxReductionKind getMinMaxReductionKind() {
-    return MinMaxKind;
-  }
+  MinMaxRecurrenceKind getMinMaxRecurrenceKind() { return MinMaxKind; }
 
-  TrackingVH<Value> getReductionStartValue() { return StartValue; }
+  TrackingVH<Value> getRecurrenceStartValue() { return StartValue; }
 
   Instruction *getLoopExitInstr() { return LoopExitInstr; }
 
 private:
-  // The starting value of the reduction.
+  // The starting value of the recurrence.
   // It does not have to be zero!
   TrackingVH<Value> StartValue;
   // The instruction who's value is used outside the loop.
   Instruction *LoopExitInstr;
-  // The kind of the reduction.
-  ReductionKind Kind;
-  // If this a min/max reduction the kind of reduction.
-  ReductionInstDesc::MinMaxReductionKind MinMaxKind;
+  // The kind of the recurrence.
+  RecurrenceKind Kind;
+  // If this a min/max recurrence the kind of recurrence.
+  MinMaxRecurrenceKind MinMaxKind;
 };
 
 BasicBlock *InsertPreheaderForLoop(Loop *L, Pass *P);
@@ -180,8 +188,7 @@ BasicBlock *InsertPreheaderForLoop(Loop *L, Pass *P);
 /// will optionally update \c AliasAnalysis and \c ScalarEvolution analyses if
 /// passed into it.
 bool simplifyLoop(Loop *L, DominatorTree *DT, LoopInfo *LI, Pass *PP,
-                  AliasAnalysis *AA = nullptr, ScalarEvolution *SE = nullptr,
-                  AssumptionCache *AC = nullptr);
+                  ScalarEvolution *SE = nullptr, AssumptionCache *AC = nullptr);
 
 /// \brief Put loop into LCSSA form.
 ///
@@ -226,13 +233,13 @@ bool sinkRegion(DomTreeNode *, AliasAnalysis *, LoopInfo *, DominatorTree *,
 /// first order w.r.t the DominatorTree.  This allows us to visit definitions
 /// before uses, allowing us to hoist a loop body in one pass without iteration.
 /// Takes DomTreeNode, AliasAnalysis, LoopInfo, DominatorTree, DataLayout,
-/// TargetLibraryInfo, Loop, AliasSet information for all instructions of the 
+/// TargetLibraryInfo, Loop, AliasSet information for all instructions of the
 /// loop and loop safety information as arguments. It returns changed status.
 bool hoistRegion(DomTreeNode *, AliasAnalysis *, LoopInfo *, DominatorTree *,
                  TargetLibraryInfo *, Loop *, AliasSetTracker *,
                  LICMSafetyInfo *);
 
-/// \brief Try to promote memory values to scalars by sinking stores out of 
+/// \brief Try to promote memory values to scalars by sinking stores out of
 /// the loop and moving loads to before the loop.  We do this by looping over
 /// the stores in the loop, looking for stores to Must pointers which are 
 /// loop invariant. It takes AliasSet, Loop exit blocks vector, loop exit blocks
diff --git a/include/llvm/Transforms/Utils/LoopVersioning.h b/include/llvm/Transforms/Utils/LoopVersioning.h
new file mode 100644
index 00000000000..077ae2a11bf
--- /dev/null
+++ b/include/llvm/Transforms/Utils/LoopVersioning.h
@@ -0,0 +1,104 @@
+//===- LoopVersioning.h - Utility to version a loop -------------*- C++ -*-===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This file defines a utility class to perform loop versioning.  The versioned
+// loop speculates that otherwise may-aliasing memory accesses don't overlap and
+// emits checks to prove this.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_TRANSFORMS_UTILS_LOOPVERSIONING_H
+#define LLVM_TRANSFORMS_UTILS_LOOPVERSIONING_H
+
+#include "llvm/Transforms/Utils/ValueMapper.h"
+
+namespace llvm {
+
+class Loop;
+class LoopAccessInfo;
+class LoopInfo;
+
+/// \brief This class emits a version of the loop where run-time checks ensure
+/// that may-alias pointers can't overlap.
+///
+/// It currently only supports single-exit loops and assumes that the loop
+/// already has a preheader.
+class LoopVersioning {
+public:
+  LoopVersioning(SmallVector<RuntimePointerChecking::PointerCheck, 4> Checks,
+                 const LoopAccessInfo &LAI, Loop *L, LoopInfo *LI,
+                 DominatorTree *DT,
+                 const SmallVector<int, 8> *PtrToPartition = nullptr);
+
+  /// \brief Returns true if we need memchecks to disambiguate may-aliasing
+  /// accesses.
+  bool needsRuntimeChecks() const;
+
+  /// \brief Performs the CFG manipulation part of versioning the loop including
+  /// the DominatorTree and LoopInfo updates.
+  ///
+  /// The loop that was used to construct the class will be the "versioned" loop
+  /// i.e. the loop that will receive control if all the memchecks pass.
+  ///
+  /// This allows the loop transform pass to operate on the same loop regardless
+  /// of whether versioning was necessary or not:
+  ///
+  ///    for each loop L:
+  ///        analyze L
+  ///        if versioning is necessary version L
+  ///        transform L
+  void versionLoop(Pass *P);
+
+  /// \brief Adds the necessary PHI nodes for the versioned loops based on the
+  /// loop-defined values used outside of the loop.
+  ///
+  /// This needs to be called after versionLoop if there are defs in the loop
+  /// that are used outside the loop.  FIXME: this should be invoked internally
+  /// by versionLoop and made private.
+  void addPHINodes(const SmallVectorImpl<Instruction *> &DefsUsedOutside);
+
+  /// \brief Returns the versioned loop.  Control flows here if pointers in the
+  /// loop don't alias (i.e. all memchecks passed).  (This loop is actually the
+  /// same as the original loop that we got constructed with.)
+  Loop *getVersionedLoop() { return VersionedLoop; }
+
+  /// \brief Returns the fall-back loop.  Control flows here if pointers in the
+  /// loop may alias (i.e. one of the memchecks failed).
+  Loop *getNonVersionedLoop() { return NonVersionedLoop; }
+
+private:
+  /// \brief The original loop.  This becomes the "versioned" one.  I.e.,
+  /// control flows here if pointers in the loop don't alias.
+  Loop *VersionedLoop;
+  /// \brief The fall-back loop.  I.e. control flows here if pointers in the
+  /// loop may alias (memchecks failed).
+  Loop *NonVersionedLoop;
+
+  /// \brief For each memory pointer it contains the partitionId it is used in.
+  /// If nullptr, no partitioning is used.
+  ///
+  /// The I-th entry corresponds to I-th entry in LAI.getRuntimePointerCheck().
+  /// If the pointer is used in multiple partitions the entry is set to -1.
+  const SmallVector<int, 8> *PtrToPartition;
+
+  /// \brief This maps the instructions from VersionedLoop to their counterpart
+  /// in NonVersionedLoop.
+  ValueToValueMapTy VMap;
+
+  /// \brief The set of checks that we are versioning for.
+  SmallVector<RuntimePointerChecking::PointerCheck, 4> Checks;
+
+  /// \brief Analyses used.
+  const LoopAccessInfo &LAI;
+  LoopInfo *LI;
+  DominatorTree *DT;
+};
+}
+
+#endif
diff --git a/include/llvm/Transforms/Utils/ModuleUtils.h b/include/llvm/Transforms/Utils/ModuleUtils.h
index 907563ebe0b..0f23d34de5d 100644
--- a/include/llvm/Transforms/Utils/ModuleUtils.h
+++ b/include/llvm/Transforms/Utils/ModuleUtils.h
@@ -14,6 +14,10 @@
 #ifndef LLVM_TRANSFORMS_UTILS_MODULEUTILS_H
 #define LLVM_TRANSFORMS_UTILS_MODULEUTILS_H
 
+#include "llvm/ADT/ArrayRef.h"
+#include "llvm/ADT/StringRef.h"
+#include <utility> // for std::pair
+
 namespace llvm {
 
 class Module;
@@ -21,6 +25,9 @@ class Function;
 class GlobalValue;
 class GlobalVariable;
 class Constant;
+class StringRef;
+class Value;
+class Type;
 template <class PtrType> class SmallPtrSetImpl;
 
 /// Append F to the list of global ctors of module M with the given Priority.
@@ -43,6 +50,15 @@ GlobalVariable *collectUsedGlobalVariables(Module &M,
 // with the same name, their prototypes must match, otherwise
 // getOrInsertFunction returns a bitcast.
 Function *checkSanitizerInterfaceFunction(Constant *FuncOrBitcast);
+
+/// \brief Creates sanitizer constructor function, and calls sanitizer's init
+/// function from it.
+/// \return Returns pair of pointers to constructor, and init functions
+/// respectively.
+std::pair<Function *, Function *> createSanitizerCtorAndInitFunctions(
+    Module &M, StringRef CtorName, StringRef InitName,
+    ArrayRef<Type *> InitArgTypes, ArrayRef<Value *> InitArgs,
+    StringRef VersionCheckName = StringRef());
 } // End llvm namespace
 
 #endif //  LLVM_TRANSFORMS_UTILS_MODULEUTILS_H
diff --git a/include/llvm/Transforms/Utils/SSAUpdater.h b/include/llvm/Transforms/Utils/SSAUpdater.h
index 19e2a4a4af4..1c7b2c587a3 100644
--- a/include/llvm/Transforms/Utils/SSAUpdater.h
+++ b/include/llvm/Transforms/Utils/SSAUpdater.h
@@ -14,6 +14,7 @@
 #ifndef LLVM_TRANSFORMS_UTILS_SSAUPDATER_H
 #define LLVM_TRANSFORMS_UTILS_SSAUPDATER_H
 
+#include "llvm/ADT/ArrayRef.h"
 #include "llvm/ADT/StringRef.h"
 #include "llvm/Support/Compiler.h"
 
@@ -135,7 +136,7 @@ class LoadAndStorePromoter {
   SSAUpdater &SSA;
 
 public:
-  LoadAndStorePromoter(const SmallVectorImpl<Instruction*> &Insts,
+  LoadAndStorePromoter(ArrayRef<const Instruction*> Insts,
                        SSAUpdater &S, StringRef Name = StringRef());
   virtual ~LoadAndStorePromoter() {}
 
diff --git a/include/llvm/Transforms/Utils/VectorUtils.h b/include/llvm/Transforms/Utils/VectorUtils.h
deleted file mode 100644
index 9f0fb19d667..00000000000
--- a/include/llvm/Transforms/Utils/VectorUtils.h
+++ /dev/null
@@ -1,205 +0,0 @@
-//===- llvm/Transforms/Utils/VectorUtils.h - Vector utilities -*- C++ -*-=====//
-//
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
-//
-//===----------------------------------------------------------------------===//
-//
-// This file defines some vectorizer utilities.
-//
-//===----------------------------------------------------------------------===//
-
-#ifndef LLVM_TRANSFORMS_UTILS_VECTORUTILS_H
-#define LLVM_TRANSFORMS_UTILS_VECTORUTILS_H
-
-#include "llvm/Analysis/TargetLibraryInfo.h"
-#include "llvm/IR/IntrinsicInst.h"
-#include "llvm/IR/Intrinsics.h"
-
-namespace llvm {
-
-/// \brief Identify if the intrinsic is trivially vectorizable.
-///
-/// This method returns true if the intrinsic's argument types are all
-/// scalars for the scalar form of the intrinsic and all vectors for
-/// the vector form of the intrinsic.
-static inline bool isTriviallyVectorizable(Intrinsic::ID ID) {
-  switch (ID) {
-  case Intrinsic::sqrt:
-  case Intrinsic::sin:
-  case Intrinsic::cos:
-  case Intrinsic::exp:
-  case Intrinsic::exp2:
-  case Intrinsic::log:
-  case Intrinsic::log10:
-  case Intrinsic::log2:
-  case Intrinsic::fabs:
-  case Intrinsic::minnum:
-  case Intrinsic::maxnum:
-  case Intrinsic::copysign:
-  case Intrinsic::floor:
-  case Intrinsic::ceil:
-  case Intrinsic::trunc:
-  case Intrinsic::rint:
-  case Intrinsic::nearbyint:
-  case Intrinsic::round:
-  case Intrinsic::bswap:
-  case Intrinsic::ctpop:
-  case Intrinsic::pow:
-  case Intrinsic::fma:
-  case Intrinsic::fmuladd:
-  case Intrinsic::ctlz:
-  case Intrinsic::cttz:
-  case Intrinsic::powi:
-    return true;
-  default:
-    return false;
-  }
-}
-
-static inline bool hasVectorInstrinsicScalarOpd(Intrinsic::ID ID,
-                                         unsigned ScalarOpdIdx) {
-  switch (ID) {
-    case Intrinsic::ctlz:
-    case Intrinsic::cttz:
-    case Intrinsic::powi:
-      return (ScalarOpdIdx == 1);
-    default:
-      return false;
-  }
-}
-
-static Intrinsic::ID checkUnaryFloatSignature(const CallInst &I,
-                                              Intrinsic::ID ValidIntrinsicID) {
-  if (I.getNumArgOperands() != 1 ||
-      !I.getArgOperand(0)->getType()->isFloatingPointTy() ||
-      I.getType() != I.getArgOperand(0)->getType() ||
-      !I.onlyReadsMemory())
-    return Intrinsic::not_intrinsic;
-
-  return ValidIntrinsicID;
-}
-
-static Intrinsic::ID checkBinaryFloatSignature(const CallInst &I,
-                                               Intrinsic::ID ValidIntrinsicID) {
-  if (I.getNumArgOperands() != 2 ||
-      !I.getArgOperand(0)->getType()->isFloatingPointTy() ||
-      !I.getArgOperand(1)->getType()->isFloatingPointTy() ||
-      I.getType() != I.getArgOperand(0)->getType() ||
-      I.getType() != I.getArgOperand(1)->getType() ||
-      !I.onlyReadsMemory())
-    return Intrinsic::not_intrinsic;
-
-  return ValidIntrinsicID;
-}
-
-static Intrinsic::ID
-getIntrinsicIDForCall(CallInst *CI, const TargetLibraryInfo *TLI) {
-  // If we have an intrinsic call, check if it is trivially vectorizable.
-  if (IntrinsicInst *II = dyn_cast<IntrinsicInst>(CI)) {
-    Intrinsic::ID ID = II->getIntrinsicID();
-    if (isTriviallyVectorizable(ID) || ID == Intrinsic::lifetime_start ||
-        ID == Intrinsic::lifetime_end || ID == Intrinsic::assume)
-      return ID;
-    else
-      return Intrinsic::not_intrinsic;
-  }
-
-  if (!TLI)
-    return Intrinsic::not_intrinsic;
-
-  LibFunc::Func Func;
-  Function *F = CI->getCalledFunction();
-  // We're going to make assumptions on the semantics of the functions, check
-  // that the target knows that it's available in this environment and it does
-  // not have local linkage.
-  if (!F || F->hasLocalLinkage() || !TLI->getLibFunc(F->getName(), Func))
-    return Intrinsic::not_intrinsic;
-
-  // Otherwise check if we have a call to a function that can be turned into a
-  // vector intrinsic.
-  switch (Func) {
-  default:
-    break;
-  case LibFunc::sin:
-  case LibFunc::sinf:
-  case LibFunc::sinl:
-    return checkUnaryFloatSignature(*CI, Intrinsic::sin);
-  case LibFunc::cos:
-  case LibFunc::cosf:
-  case LibFunc::cosl:
-    return checkUnaryFloatSignature(*CI, Intrinsic::cos);
-  case LibFunc::exp:
-  case LibFunc::expf:
-  case LibFunc::expl:
-    return checkUnaryFloatSignature(*CI, Intrinsic::exp);
-  case LibFunc::exp2:
-  case LibFunc::exp2f:
-  case LibFunc::exp2l:
-    return checkUnaryFloatSignature(*CI, Intrinsic::exp2);
-  case LibFunc::log:
-  case LibFunc::logf:
-  case LibFunc::logl:
-    return checkUnaryFloatSignature(*CI, Intrinsic::log);
-  case LibFunc::log10:
-  case LibFunc::log10f:
-  case LibFunc::log10l:
-    return checkUnaryFloatSignature(*CI, Intrinsic::log10);
-  case LibFunc::log2:
-  case LibFunc::log2f:
-  case LibFunc::log2l:
-    return checkUnaryFloatSignature(*CI, Intrinsic::log2);
-  case LibFunc::fabs:
-  case LibFunc::fabsf:
-  case LibFunc::fabsl:
-    return checkUnaryFloatSignature(*CI, Intrinsic::fabs);
-  case LibFunc::fmin:
-  case LibFunc::fminf:
-  case LibFunc::fminl:
-    return checkBinaryFloatSignature(*CI, Intrinsic::minnum);
-  case LibFunc::fmax:
-  case LibFunc::fmaxf:
-  case LibFunc::fmaxl:
-    return checkBinaryFloatSignature(*CI, Intrinsic::maxnum);
-  case LibFunc::copysign:
-  case LibFunc::copysignf:
-  case LibFunc::copysignl:
-    return checkBinaryFloatSignature(*CI, Intrinsic::copysign);
-  case LibFunc::floor:
-  case LibFunc::floorf:
-  case LibFunc::floorl:
-    return checkUnaryFloatSignature(*CI, Intrinsic::floor);
-  case LibFunc::ceil:
-  case LibFunc::ceilf:
-  case LibFunc::ceill:
-    return checkUnaryFloatSignature(*CI, Intrinsic::ceil);
-  case LibFunc::trunc:
-  case LibFunc::truncf:
-  case LibFunc::truncl:
-    return checkUnaryFloatSignature(*CI, Intrinsic::trunc);
-  case LibFunc::rint:
-  case LibFunc::rintf:
-  case LibFunc::rintl:
-    return checkUnaryFloatSignature(*CI, Intrinsic::rint);
-  case LibFunc::nearbyint:
-  case LibFunc::nearbyintf:
-  case LibFunc::nearbyintl:
-    return checkUnaryFloatSignature(*CI, Intrinsic::nearbyint);
-  case LibFunc::round:
-  case LibFunc::roundf:
-  case LibFunc::roundl:
-    return checkUnaryFloatSignature(*CI, Intrinsic::round);
-  case LibFunc::pow:
-  case LibFunc::powf:
-  case LibFunc::powl:
-    return checkBinaryFloatSignature(*CI, Intrinsic::pow);
-  }
-
-  return Intrinsic::not_intrinsic;
-}
-
-} // llvm namespace
-
-#endif
diff --git a/include/llvm/module.modulemap b/include/llvm/module.modulemap
index dc6e1367034..dcc5ce1059f 100644
--- a/include/llvm/module.modulemap
+++ b/include/llvm/module.modulemap
@@ -29,6 +29,9 @@ module LLVM_Backend {
     exclude header "CodeGen/CommandFlags.h"
     exclude header "CodeGen/LinkAllAsmWriterComponents.h"
     exclude header "CodeGen/LinkAllCodegenComponents.h"
+
+    // These are intended for (repeated) textual inclusion.
+    textual header "CodeGen/DIEValue.def"
   }
 
   module Target {
@@ -46,6 +49,11 @@ module LLVM_Backend {
 module LLVM_Bitcode { requires cplusplus umbrella "Bitcode" module * { export * } }
 module LLVM_Config { requires cplusplus umbrella "Config" module * { export * } }
 
+module LLVM_DebugInfo {
+  requires cplusplus
+  module DIContext { header "DebugInfo/DIContext.h" export * }
+}
+
 module LLVM_DebugInfo_DWARF {
   requires cplusplus
 
@@ -116,6 +124,7 @@ module LLVM_IR {
   textual header "IR/DebugInfoFlags.def"
   textual header "IR/Instruction.def"
   textual header "IR/Metadata.def"
+  textual header "IR/Value.def"
 }
 
 module LLVM_IRReader { requires cplusplus umbrella "IRReader" module * { export * } }
diff --git a/lib/Analysis/AliasAnalysis.cpp b/lib/Analysis/AliasAnalysis.cpp
index 43db1763bbb..9e69342e80b 100644
--- a/lib/Analysis/AliasAnalysis.cpp
+++ b/lib/Analysis/AliasAnalysis.cpp
@@ -48,42 +48,26 @@ char AliasAnalysis::ID = 0;
 // Default chaining methods
 //===----------------------------------------------------------------------===//
 
-AliasAnalysis::AliasResult
-AliasAnalysis::alias(const Location &LocA, const Location &LocB) {
+AliasResult AliasAnalysis::alias(const MemoryLocation &LocA,
+                                 const MemoryLocation &LocB) {
   assert(AA && "AA didn't call InitializeAliasAnalysis in its run method!");
   return AA->alias(LocA, LocB);
 }
 
-bool AliasAnalysis::pointsToConstantMemory(const Location &Loc,
+bool AliasAnalysis::pointsToConstantMemory(const MemoryLocation &Loc,
                                            bool OrLocal) {
   assert(AA && "AA didn't call InitializeAliasAnalysis in its run method!");
   return AA->pointsToConstantMemory(Loc, OrLocal);
 }
 
-AliasAnalysis::Location
-AliasAnalysis::getArgLocation(ImmutableCallSite CS, unsigned ArgIdx,
-                              AliasAnalysis::ModRefResult &Mask) {
+ModRefInfo AliasAnalysis::getArgModRefInfo(ImmutableCallSite CS,
+                                           unsigned ArgIdx) {
   assert(AA && "AA didn't call InitializeAliasAnalysis in its run method!");
-  return AA->getArgLocation(CS, ArgIdx, Mask);
+  return AA->getArgModRefInfo(CS, ArgIdx);
 }
 
-void AliasAnalysis::deleteValue(Value *V) {
-  assert(AA && "AA didn't call InitializeAliasAnalysis in its run method!");
-  AA->deleteValue(V);
-}
-
-void AliasAnalysis::copyValue(Value *From, Value *To) {
-  assert(AA && "AA didn't call InitializeAliasAnalysis in its run method!");
-  AA->copyValue(From, To);
-}
-
-void AliasAnalysis::addEscapingUse(Use &U) {
-  assert(AA && "AA didn't call InitializeAliasAnalysis in its run method!");
-  AA->addEscapingUse(U);
-}
-
-AliasAnalysis::ModRefResult
-AliasAnalysis::getModRefInfo(Instruction *I, ImmutableCallSite Call) {
+ModRefInfo AliasAnalysis::getModRefInfo(Instruction *I,
+                                        ImmutableCallSite Call) {
   // We may have two calls
   if (auto CS = ImmutableCallSite(I)) {
     // Check if the two calls modify the same memory
@@ -93,108 +77,109 @@ AliasAnalysis::getModRefInfo(Instruction *I, ImmutableCallSite Call) {
     // location this memory access defines.  The best we can say
     // is that if the call references what this instruction
     // defines, it must be clobbered by this location.
-    const AliasAnalysis::Location DefLoc = AA->getLocation(I);
-    if (getModRefInfo(Call, DefLoc) != AliasAnalysis::NoModRef)
-      return AliasAnalysis::ModRef;
+    const MemoryLocation DefLoc = MemoryLocation::get(I);
+    if (getModRefInfo(Call, DefLoc) != MRI_NoModRef)
+      return MRI_ModRef;
   }
-  return AliasAnalysis::NoModRef;
+  return MRI_NoModRef;
 }
 
-AliasAnalysis::ModRefResult
-AliasAnalysis::getModRefInfo(ImmutableCallSite CS,
-                             const Location &Loc) {
+ModRefInfo AliasAnalysis::getModRefInfo(ImmutableCallSite CS,
+                                        const MemoryLocation &Loc) {
   assert(AA && "AA didn't call InitializeAliasAnalysis in its run method!");
 
-  ModRefBehavior MRB = getModRefBehavior(CS);
-  if (MRB == DoesNotAccessMemory)
-    return NoModRef;
+  auto MRB = getModRefBehavior(CS);
+  if (MRB == FMRB_DoesNotAccessMemory)
+    return MRI_NoModRef;
 
-  ModRefResult Mask = ModRef;
+  ModRefInfo Mask = MRI_ModRef;
   if (onlyReadsMemory(MRB))
-    Mask = Ref;
+    Mask = MRI_Ref;
 
   if (onlyAccessesArgPointees(MRB)) {
     bool doesAlias = false;
-    ModRefResult AllArgsMask = NoModRef;
+    ModRefInfo AllArgsMask = MRI_NoModRef;
     if (doesAccessArgPointees(MRB)) {
       for (ImmutableCallSite::arg_iterator AI = CS.arg_begin(), AE = CS.arg_end();
            AI != AE; ++AI) {
         const Value *Arg = *AI;
         if (!Arg->getType()->isPointerTy())
           continue;
-        ModRefResult ArgMask;
-        Location CSLoc =
-          getArgLocation(CS, (unsigned) std::distance(CS.arg_begin(), AI),
-                         ArgMask);
-        if (!isNoAlias(CSLoc, Loc)) {
+        unsigned ArgIdx = std::distance(CS.arg_begin(), AI);
+        MemoryLocation ArgLoc =
+            MemoryLocation::getForArgument(CS, ArgIdx, *TLI);
+        if (!isNoAlias(ArgLoc, Loc)) {
+          ModRefInfo ArgMask = getArgModRefInfo(CS, ArgIdx);
           doesAlias = true;
-          AllArgsMask = ModRefResult(AllArgsMask | ArgMask);
+          AllArgsMask = ModRefInfo(AllArgsMask | ArgMask);
         }
       }
     }
     if (!doesAlias)
-      return NoModRef;
-    Mask = ModRefResult(Mask & AllArgsMask);
+      return MRI_NoModRef;
+    Mask = ModRefInfo(Mask & AllArgsMask);
   }
 
   // If Loc is a constant memory location, the call definitely could not
   // modify the memory location.
-  if ((Mask & Mod) && pointsToConstantMemory(Loc))
-    Mask = ModRefResult(Mask & ~Mod);
+  if ((Mask & MRI_Mod) && pointsToConstantMemory(Loc))
+    Mask = ModRefInfo(Mask & ~MRI_Mod);
 
   // If this is the end of the chain, don't forward.
   if (!AA) return Mask;
 
   // Otherwise, fall back to the next AA in the chain. But we can merge
   // in any mask we've managed to compute.
-  return ModRefResult(AA->getModRefInfo(CS, Loc) & Mask);
+  return ModRefInfo(AA->getModRefInfo(CS, Loc) & Mask);
 }
 
-AliasAnalysis::ModRefResult
-AliasAnalysis::getModRefInfo(ImmutableCallSite CS1, ImmutableCallSite CS2) {
+ModRefInfo AliasAnalysis::getModRefInfo(ImmutableCallSite CS1,
+                                        ImmutableCallSite CS2) {
   assert(AA && "AA didn't call InitializeAliasAnalysis in its run method!");
 
   // If CS1 or CS2 are readnone, they don't interact.
-  ModRefBehavior CS1B = getModRefBehavior(CS1);
-  if (CS1B == DoesNotAccessMemory) return NoModRef;
+  auto CS1B = getModRefBehavior(CS1);
+  if (CS1B == FMRB_DoesNotAccessMemory)
+    return MRI_NoModRef;
 
-  ModRefBehavior CS2B = getModRefBehavior(CS2);
-  if (CS2B == DoesNotAccessMemory) return NoModRef;
+  auto CS2B = getModRefBehavior(CS2);
+  if (CS2B == FMRB_DoesNotAccessMemory)
+    return MRI_NoModRef;
 
   // If they both only read from memory, there is no dependence.
   if (onlyReadsMemory(CS1B) && onlyReadsMemory(CS2B))
-    return NoModRef;
+    return MRI_NoModRef;
 
-  AliasAnalysis::ModRefResult Mask = ModRef;
+  ModRefInfo Mask = MRI_ModRef;
 
   // If CS1 only reads memory, the only dependence on CS2 can be
   // from CS1 reading memory written by CS2.
   if (onlyReadsMemory(CS1B))
-    Mask = ModRefResult(Mask & Ref);
+    Mask = ModRefInfo(Mask & MRI_Ref);
 
   // If CS2 only access memory through arguments, accumulate the mod/ref
   // information from CS1's references to the memory referenced by
   // CS2's arguments.
   if (onlyAccessesArgPointees(CS2B)) {
-    AliasAnalysis::ModRefResult R = NoModRef;
+    ModRefInfo R = MRI_NoModRef;
     if (doesAccessArgPointees(CS2B)) {
       for (ImmutableCallSite::arg_iterator
            I = CS2.arg_begin(), E = CS2.arg_end(); I != E; ++I) {
         const Value *Arg = *I;
         if (!Arg->getType()->isPointerTy())
           continue;
-        ModRefResult ArgMask;
-        Location CS2Loc =
-          getArgLocation(CS2, (unsigned) std::distance(CS2.arg_begin(), I),
-                         ArgMask);
-        // ArgMask indicates what CS2 might do to CS2Loc, and the dependence of
+        unsigned CS2ArgIdx = std::distance(CS2.arg_begin(), I);
+        auto CS2ArgLoc = MemoryLocation::getForArgument(CS2, CS2ArgIdx, *TLI);
+
+        // ArgMask indicates what CS2 might do to CS2ArgLoc, and the dependence of
         // CS1 on that location is the inverse.
-        if (ArgMask == Mod)
-          ArgMask = ModRef;
-        else if (ArgMask == Ref)
-          ArgMask = Mod;
+        ModRefInfo ArgMask = getArgModRefInfo(CS2, CS2ArgIdx);
+        if (ArgMask == MRI_Mod)
+          ArgMask = MRI_ModRef;
+        else if (ArgMask == MRI_Ref)
+          ArgMask = MRI_Mod;
 
-        R = ModRefResult((R | (getModRefInfo(CS1, CS2Loc) & ArgMask)) & Mask);
+        R = ModRefInfo((R | (getModRefInfo(CS1, CS2ArgLoc) & ArgMask)) & Mask);
         if (R == Mask)
           break;
       }
@@ -205,23 +190,26 @@ AliasAnalysis::getModRefInfo(ImmutableCallSite CS1, ImmutableCallSite CS2) {
   // If CS1 only accesses memory through arguments, check if CS2 references
   // any of the memory referenced by CS1's arguments. If not, return NoModRef.
   if (onlyAccessesArgPointees(CS1B)) {
-    AliasAnalysis::ModRefResult R = NoModRef;
+    ModRefInfo R = MRI_NoModRef;
     if (doesAccessArgPointees(CS1B)) {
       for (ImmutableCallSite::arg_iterator
            I = CS1.arg_begin(), E = CS1.arg_end(); I != E; ++I) {
         const Value *Arg = *I;
         if (!Arg->getType()->isPointerTy())
           continue;
-        ModRefResult ArgMask;
-        Location CS1Loc = getArgLocation(
-            CS1, (unsigned)std::distance(CS1.arg_begin(), I), ArgMask);
-        // ArgMask indicates what CS1 might do to CS1Loc; if CS1 might Mod
-        // CS1Loc, then we care about either a Mod or a Ref by CS2. If CS1
+        unsigned CS1ArgIdx = std::distance(CS1.arg_begin(), I);
+        auto CS1ArgLoc = MemoryLocation::getForArgument(CS1, CS1ArgIdx, *TLI);
+
+        // ArgMask indicates what CS1 might do to CS1ArgLoc; if CS1 might Mod
+        // CS1ArgLoc, then we care about either a Mod or a Ref by CS2. If CS1
         // might Ref, then we care only about a Mod by CS2.
-        ModRefResult ArgR = getModRefInfo(CS2, CS1Loc);
-        if (((ArgMask & Mod) != NoModRef && (ArgR & ModRef) != NoModRef) ||
-            ((ArgMask & Ref) != NoModRef && (ArgR & Mod)    != NoModRef))
-          R = ModRefResult((R | ArgMask) & Mask);
+        ModRefInfo ArgMask = getArgModRefInfo(CS1, CS1ArgIdx);
+        ModRefInfo ArgR = getModRefInfo(CS2, CS1ArgLoc);
+        if (((ArgMask & MRI_Mod) != MRI_NoModRef &&
+             (ArgR & MRI_ModRef) != MRI_NoModRef) ||
+            ((ArgMask & MRI_Ref) != MRI_NoModRef &&
+             (ArgR & MRI_Mod) != MRI_NoModRef))
+          R = ModRefInfo((R | ArgMask) & Mask);
 
         if (R == Mask)
           break;
@@ -235,14 +223,13 @@ AliasAnalysis::getModRefInfo(ImmutableCallSite CS1, ImmutableCallSite CS2) {
 
   // Otherwise, fall back to the next AA in the chain. But we can merge
   // in any mask we've managed to compute.
-  return ModRefResult(AA->getModRefInfo(CS1, CS2) & Mask);
+  return ModRefInfo(AA->getModRefInfo(CS1, CS2) & Mask);
 }
 
-AliasAnalysis::ModRefBehavior
-AliasAnalysis::getModRefBehavior(ImmutableCallSite CS) {
+FunctionModRefBehavior AliasAnalysis::getModRefBehavior(ImmutableCallSite CS) {
   assert(AA && "AA didn't call InitializeAliasAnalysis in its run method!");
 
-  ModRefBehavior Min = UnknownModRefBehavior;
+  auto Min = FMRB_UnknownModRefBehavior;
 
   // Call back into the alias analysis with the other form of getModRefBehavior
   // to see if it can give a better response.
@@ -254,11 +241,10 @@ AliasAnalysis::getModRefBehavior(ImmutableCallSite CS) {
 
   // Otherwise, fall back to the next AA in the chain. But we can merge
   // in any result we've managed to compute.
-  return ModRefBehavior(AA->getModRefBehavior(CS) & Min);
+  return FunctionModRefBehavior(AA->getModRefBehavior(CS) & Min);
 }
 
-AliasAnalysis::ModRefBehavior
-AliasAnalysis::getModRefBehavior(const Function *F) {
+FunctionModRefBehavior AliasAnalysis::getModRefBehavior(const Function *F) {
   assert(AA && "AA didn't call InitializeAliasAnalysis in its run method!");
   return AA->getModRefBehavior(F);
 }
@@ -267,185 +253,120 @@ AliasAnalysis::getModRefBehavior(const Function *F) {
 // AliasAnalysis non-virtual helper method implementation
 //===----------------------------------------------------------------------===//
 
-AliasAnalysis::Location AliasAnalysis::getLocation(const LoadInst *LI) {
-  AAMDNodes AATags;
-  LI->getAAMetadata(AATags);
-
-  return Location(LI->getPointerOperand(),
-                  getTypeStoreSize(LI->getType()), AATags);
-}
-
-AliasAnalysis::Location AliasAnalysis::getLocation(const StoreInst *SI) {
-  AAMDNodes AATags;
-  SI->getAAMetadata(AATags);
-
-  return Location(SI->getPointerOperand(),
-                  getTypeStoreSize(SI->getValueOperand()->getType()), AATags);
-}
-
-AliasAnalysis::Location AliasAnalysis::getLocation(const VAArgInst *VI) {
-  AAMDNodes AATags;
-  VI->getAAMetadata(AATags);
-
-  return Location(VI->getPointerOperand(), UnknownSize, AATags);
-}
-
-AliasAnalysis::Location
-AliasAnalysis::getLocation(const AtomicCmpXchgInst *CXI) {
-  AAMDNodes AATags;
-  CXI->getAAMetadata(AATags);
-
-  return Location(CXI->getPointerOperand(),
-                  getTypeStoreSize(CXI->getCompareOperand()->getType()),
-                  AATags);
-}
-
-AliasAnalysis::Location
-AliasAnalysis::getLocation(const AtomicRMWInst *RMWI) {
-  AAMDNodes AATags;
-  RMWI->getAAMetadata(AATags);
-
-  return Location(RMWI->getPointerOperand(),
-                  getTypeStoreSize(RMWI->getValOperand()->getType()), AATags);
-}
-
-AliasAnalysis::Location
-AliasAnalysis::getLocationForSource(const MemTransferInst *MTI) {
-  uint64_t Size = UnknownSize;
-  if (ConstantInt *C = dyn_cast<ConstantInt>(MTI->getLength()))
-    Size = C->getValue().getZExtValue();
-
-  // memcpy/memmove can have AA tags. For memcpy, they apply
-  // to both the source and the destination.
-  AAMDNodes AATags;
-  MTI->getAAMetadata(AATags);
-
-  return Location(MTI->getRawSource(), Size, AATags);
-}
-
-AliasAnalysis::Location
-AliasAnalysis::getLocationForDest(const MemIntrinsic *MTI) {
-  uint64_t Size = UnknownSize;
-  if (ConstantInt *C = dyn_cast<ConstantInt>(MTI->getLength()))
-    Size = C->getValue().getZExtValue();
-
-  // memcpy/memmove can have AA tags. For memcpy, they apply
-  // to both the source and the destination.
-  AAMDNodes AATags;
-  MTI->getAAMetadata(AATags);
-
-  return Location(MTI->getRawDest(), Size, AATags);
-}
-
-
-
-AliasAnalysis::ModRefResult
-AliasAnalysis::getModRefInfo(const LoadInst *L, const Location &Loc) {
+ModRefInfo AliasAnalysis::getModRefInfo(const LoadInst *L,
+                                        const MemoryLocation &Loc) {
   // Be conservative in the face of volatile/atomic.
   if (!L->isUnordered())
-    return ModRef;
+    return MRI_ModRef;
 
   // If the load address doesn't alias the given address, it doesn't read
   // or write the specified memory.
-  if (Loc.Ptr && !alias(getLocation(L), Loc))
-    return NoModRef;
+  if (Loc.Ptr && !alias(MemoryLocation::get(L), Loc))
+    return MRI_NoModRef;
 
   // Otherwise, a load just reads.
-  return Ref;
+  return MRI_Ref;
 }
 
-AliasAnalysis::ModRefResult
-AliasAnalysis::getModRefInfo(const StoreInst *S, const Location &Loc) {
+ModRefInfo AliasAnalysis::getModRefInfo(const StoreInst *S,
+                                        const MemoryLocation &Loc) {
   // Be conservative in the face of volatile/atomic.
   if (!S->isUnordered())
-    return ModRef;
+    return MRI_ModRef;
 
   if (Loc.Ptr) {
     // If the store address cannot alias the pointer in question, then the
     // specified memory cannot be modified by the store.
-    if (!alias(getLocation(S), Loc))
-      return NoModRef;
+    if (!alias(MemoryLocation::get(S), Loc))
+      return MRI_NoModRef;
 
     // If the pointer is a pointer to constant memory, then it could not have
     // been modified by this store.
     if (pointsToConstantMemory(Loc))
-      return NoModRef;
-
+      return MRI_NoModRef;
   }
 
   // Otherwise, a store just writes.
-  return Mod;
+  return MRI_Mod;
 }
 
-AliasAnalysis::ModRefResult
-AliasAnalysis::getModRefInfo(const VAArgInst *V, const Location &Loc) {
-  // If the va_arg address cannot alias the pointer in question, then the
-  // specified memory cannot be accessed by the va_arg.
-  if (!alias(getLocation(V), Loc))
-    return NoModRef;
+ModRefInfo AliasAnalysis::getModRefInfo(const VAArgInst *V,
+                                        const MemoryLocation &Loc) {
 
-  // If the pointer is a pointer to constant memory, then it could not have been
-  // modified by this va_arg.
-  if (pointsToConstantMemory(Loc))
-    return NoModRef;
+  if (Loc.Ptr) {
+    // If the va_arg address cannot alias the pointer in question, then the
+    // specified memory cannot be accessed by the va_arg.
+    if (!alias(MemoryLocation::get(V), Loc))
+      return MRI_NoModRef;
+
+    // If the pointer is a pointer to constant memory, then it could not have
+    // been modified by this va_arg.
+    if (pointsToConstantMemory(Loc))
+      return MRI_NoModRef;
+  }
 
   // Otherwise, a va_arg reads and writes.
-  return ModRef;
+  return MRI_ModRef;
 }
 
-AliasAnalysis::ModRefResult
-AliasAnalysis::getModRefInfo(const AtomicCmpXchgInst *CX, const Location &Loc) {
+ModRefInfo AliasAnalysis::getModRefInfo(const AtomicCmpXchgInst *CX,
+                                        const MemoryLocation &Loc) {
   // Acquire/Release cmpxchg has properties that matter for arbitrary addresses.
   if (CX->getSuccessOrdering() > Monotonic)
-    return ModRef;
+    return MRI_ModRef;
 
   // If the cmpxchg address does not alias the location, it does not access it.
-  if (!alias(getLocation(CX), Loc))
-    return NoModRef;
+  if (Loc.Ptr && !alias(MemoryLocation::get(CX), Loc))
+    return MRI_NoModRef;
 
-  return ModRef;
+  return MRI_ModRef;
 }
 
-AliasAnalysis::ModRefResult
-AliasAnalysis::getModRefInfo(const AtomicRMWInst *RMW, const Location &Loc) {
+ModRefInfo AliasAnalysis::getModRefInfo(const AtomicRMWInst *RMW,
+                                        const MemoryLocation &Loc) {
   // Acquire/Release atomicrmw has properties that matter for arbitrary addresses.
   if (RMW->getOrdering() > Monotonic)
-    return ModRef;
+    return MRI_ModRef;
 
   // If the atomicrmw address does not alias the location, it does not access it.
-  if (!alias(getLocation(RMW), Loc))
-    return NoModRef;
-
-  return ModRef;
-}
-
-// FIXME: this is really just shoring-up a deficiency in alias analysis.
-// BasicAA isn't willing to spend linear time determining whether an alloca
-// was captured before or after this particular call, while we are. However,
-// with a smarter AA in place, this test is just wasting compile time.
-AliasAnalysis::ModRefResult
-AliasAnalysis::callCapturesBefore(const Instruction *I,
-                                  const AliasAnalysis::Location &MemLoc,
-                                  DominatorTree *DT) {
+  if (Loc.Ptr && !alias(MemoryLocation::get(RMW), Loc))
+    return MRI_NoModRef;
+
+  return MRI_ModRef;
+}
+
+/// \brief Return information about whether a particular call site modifies
+/// or reads the specified memory location \p MemLoc before instruction \p I
+/// in a BasicBlock. A ordered basic block \p OBB can be used to speed up
+/// instruction-ordering queries inside the BasicBlock containing \p I.
+/// FIXME: this is really just shoring-up a deficiency in alias analysis.
+/// BasicAA isn't willing to spend linear time determining whether an alloca
+/// was captured before or after this particular call, while we are. However,
+/// with a smarter AA in place, this test is just wasting compile time.
+ModRefInfo AliasAnalysis::callCapturesBefore(const Instruction *I,
+                                             const MemoryLocation &MemLoc,
+                                             DominatorTree *DT,
+                                             OrderedBasicBlock *OBB) {
   if (!DT)
-    return AliasAnalysis::ModRef;
+    return MRI_ModRef;
 
   const Value *Object = GetUnderlyingObject(MemLoc.Ptr, *DL);
   if (!isIdentifiedObject(Object) || isa<GlobalValue>(Object) ||
       isa<Constant>(Object))
-    return AliasAnalysis::ModRef;
+    return MRI_ModRef;
 
   ImmutableCallSite CS(I);
   if (!CS.getInstruction() || CS.getInstruction() == Object)
-    return AliasAnalysis::ModRef;
+    return MRI_ModRef;
 
   if (llvm::PointerMayBeCapturedBefore(Object, /* ReturnCaptures */ true,
                                        /* StoreCaptures */ true, I, DT,
-                                       /* include Object */ true))
-    return AliasAnalysis::ModRef;
+                                       /* include Object */ true,
+                                       /* OrderedBasicBlock */ OBB))
+    return MRI_ModRef;
 
   unsigned ArgNo = 0;
-  AliasAnalysis::ModRefResult R = AliasAnalysis::NoModRef;
+  ModRefInfo R = MRI_NoModRef;
   for (ImmutableCallSite::arg_iterator CI = CS.arg_begin(), CE = CS.arg_end();
        CI != CE; ++CI, ++ArgNo) {
     // Only look at the no-capture or byval pointer arguments.  If this
@@ -459,16 +380,15 @@ AliasAnalysis::callCapturesBefore(const Instruction *I,
     // is impossible to alias the pointer we're checking.  If not, we have to
     // assume that the call could touch the pointer, even though it doesn't
     // escape.
-    if (isNoAlias(AliasAnalysis::Location(*CI),
-                  AliasAnalysis::Location(Object)))
+    if (isNoAlias(MemoryLocation(*CI), MemoryLocation(Object)))
       continue;
     if (CS.doesNotAccessMemory(ArgNo))
       continue;
     if (CS.onlyReadsMemory(ArgNo)) {
-      R = AliasAnalysis::Ref;
+      R = MRI_Ref;
       continue;
     }
-    return AliasAnalysis::ModRef;
+    return MRI_ModRef;
   }
   return R;
 }
@@ -500,26 +420,26 @@ void AliasAnalysis::getAnalysisUsage(AnalysisUsage &AU) const {
 /// if known, or a conservative value otherwise.
 ///
 uint64_t AliasAnalysis::getTypeStoreSize(Type *Ty) {
-  return DL ? DL->getTypeStoreSize(Ty) : UnknownSize;
+  return DL ? DL->getTypeStoreSize(Ty) : MemoryLocation::UnknownSize;
 }
 
 /// canBasicBlockModify - Return true if it is possible for execution of the
 /// specified basic block to modify the location Loc.
 ///
 bool AliasAnalysis::canBasicBlockModify(const BasicBlock &BB,
-                                        const Location &Loc) {
-  return canInstructionRangeModRef(BB.front(), BB.back(), Loc, Mod);
+                                        const MemoryLocation &Loc) {
+  return canInstructionRangeModRef(BB.front(), BB.back(), Loc, MRI_Mod);
 }
 
 /// canInstructionRangeModRef - Return true if it is possible for the
 /// execution of the specified instructions to mod\ref (according to the
 /// mode) the location Loc. The instructions to consider are all
 /// of the instructions in the range of [I1,I2] INCLUSIVE.
-/// I1 and I2 must be in the same basic block.  
+/// I1 and I2 must be in the same basic block.
 bool AliasAnalysis::canInstructionRangeModRef(const Instruction &I1,
                                               const Instruction &I2,
-                                              const Location &Loc,
-                                              const ModRefResult Mode) {
+                                              const MemoryLocation &Loc,
+                                              const ModRefInfo Mode) {
   assert(I1.getParent() == I2.getParent() &&
          "Instructions not in same basic block!");
   BasicBlock::const_iterator I = &I1;
diff --git a/lib/Analysis/AliasAnalysisCounter.cpp b/lib/Analysis/AliasAnalysisCounter.cpp
index a1bfba1f002..6c53b41d3a1 100644
--- a/lib/Analysis/AliasAnalysisCounter.cpp
+++ b/lib/Analysis/AliasAnalysisCounter.cpp
@@ -62,15 +62,16 @@ namespace {
                  << Must*100/AASum<<"%\n\n";
         }
 
-        errs() << "  " << MRSum    << " Total Mod/Ref Queries Performed\n";
+        errs() << "  " << MRSum << " Total MRI_Mod/MRI_Ref Queries Performed\n";
         if (MRSum) {
           printLine("no mod/ref",    NoMR, MRSum);
           printLine("ref",        JustRef, MRSum);
           printLine("mod",        JustMod, MRSum);
           printLine("mod/ref",         MR, MRSum);
-          errs() << "  Mod/Ref Analysis Counter Summary: " <<NoMR*100/MRSum
-                 << "%/" << JustRef*100/MRSum << "%/" << JustMod*100/MRSum
-                 << "%/" << MR*100/MRSum <<"%\n\n";
+          errs() << "  MRI_Mod/MRI_Ref Analysis Counter Summary: "
+                 << NoMR * 100 / MRSum << "%/" << JustRef * 100 / MRSum << "%/"
+                 << JustMod * 100 / MRSum << "%/" << MR * 100 / MRSum
+                 << "%\n\n";
         }
       }
     }
@@ -98,18 +99,20 @@ namespace {
     }
     
     // FIXME: We could count these too...
-    bool pointsToConstantMemory(const Location &Loc, bool OrLocal) override {
+    bool pointsToConstantMemory(const MemoryLocation &Loc,
+                                bool OrLocal) override {
       return getAnalysis<AliasAnalysis>().pointsToConstantMemory(Loc, OrLocal);
     }
 
     // Forwarding functions: just delegate to a real AA implementation, counting
     // the number of responses...
-    AliasResult alias(const Location &LocA, const Location &LocB) override;
+    AliasResult alias(const MemoryLocation &LocA,
+                      const MemoryLocation &LocB) override;
 
-    ModRefResult getModRefInfo(ImmutableCallSite CS,
-                               const Location &Loc) override;
-    ModRefResult getModRefInfo(ImmutableCallSite CS1,
-                               ImmutableCallSite CS2) override {
+    ModRefInfo getModRefInfo(ImmutableCallSite CS,
+                             const MemoryLocation &Loc) override;
+    ModRefInfo getModRefInfo(ImmutableCallSite CS1,
+                             ImmutableCallSite CS2) override {
       return AliasAnalysis::getModRefInfo(CS1,CS2);
     }
   };
@@ -123,8 +126,8 @@ ModulePass *llvm::createAliasAnalysisCounterPass() {
   return new AliasAnalysisCounter();
 }
 
-AliasAnalysis::AliasResult
-AliasAnalysisCounter::alias(const Location &LocA, const Location &LocB) {
+AliasResult AliasAnalysisCounter::alias(const MemoryLocation &LocA,
+                                        const MemoryLocation &LocB) {
   AliasResult R = getAnalysis<AliasAnalysis>().alias(LocA, LocB);
 
   const char *AliasString = nullptr;
@@ -148,20 +151,31 @@ AliasAnalysisCounter::alias(const Location &LocA, const Location &LocB) {
   return R;
 }
 
-AliasAnalysis::ModRefResult
-AliasAnalysisCounter::getModRefInfo(ImmutableCallSite CS,
-                                    const Location &Loc) {
-  ModRefResult R = getAnalysis<AliasAnalysis>().getModRefInfo(CS, Loc);
+ModRefInfo AliasAnalysisCounter::getModRefInfo(ImmutableCallSite CS,
+                                               const MemoryLocation &Loc) {
+  ModRefInfo R = getAnalysis<AliasAnalysis>().getModRefInfo(CS, Loc);
 
   const char *MRString = nullptr;
   switch (R) {
-  case NoModRef: NoMR++;     MRString = "NoModRef"; break;
-  case Ref:      JustRef++;  MRString = "JustRef"; break;
-  case Mod:      JustMod++;  MRString = "JustMod"; break;
-  case ModRef:   MR++;       MRString = "ModRef"; break;
+  case MRI_NoModRef:
+    NoMR++;
+    MRString = "MRI_NoModRef";
+    break;
+  case MRI_Ref:
+    JustRef++;
+    MRString = "JustRef";
+    break;
+  case MRI_Mod:
+    JustMod++;
+    MRString = "JustMod";
+    break;
+  case MRI_ModRef:
+    MR++;
+    MRString = "MRI_ModRef";
+    break;
   }
 
-  if (PrintAll || (PrintAllFailures && R == ModRef)) {
+  if (PrintAll || (PrintAllFailures && R == MRI_ModRef)) {
     errs() << MRString << ":  Ptr: ";
     errs() << "[" << Loc.Size << "B] ";
     Loc.Ptr->printAsOperand(errs(), true, M);
diff --git a/lib/Analysis/AliasAnalysisEvaluator.cpp b/lib/Analysis/AliasAnalysisEvaluator.cpp
index 273eacc16e7..7114a96c4ec 100644
--- a/lib/Analysis/AliasAnalysisEvaluator.cpp
+++ b/lib/Analysis/AliasAnalysisEvaluator.cpp
@@ -47,8 +47,8 @@ static cl::opt<bool> EvalAAMD("evaluate-aa-metadata", cl::ReallyHidden);
 
 namespace {
   class AAEval : public FunctionPass {
-    unsigned NoAlias, MayAlias, PartialAlias, MustAlias;
-    unsigned NoModRef, Mod, Ref, ModRef;
+    unsigned NoAliasCount, MayAliasCount, PartialAliasCount, MustAliasCount;
+    unsigned NoModRefCount, ModCount, RefCount, ModRefCount;
 
   public:
     static char ID; // Pass identification, replacement for typeid
@@ -62,8 +62,8 @@ namespace {
     }
 
     bool doInitialization(Module &M) override {
-      NoAlias = MayAlias = PartialAlias = MustAlias = 0;
-      NoModRef = Mod = Ref = ModRef = 0;
+      NoAliasCount = MayAliasCount = PartialAliasCount = MustAliasCount = 0;
+      NoModRefCount = ModCount = RefCount = ModRefCount = 0;
 
       if (PrintAll) {
         PrintNoAlias = PrintMayAlias = true;
@@ -186,29 +186,33 @@ bool AAEval::runOnFunction(Function &F) {
   // iterate over the worklist, and run the full (n^2)/2 disambiguations
   for (SetVector<Value *>::iterator I1 = Pointers.begin(), E = Pointers.end();
        I1 != E; ++I1) {
-    uint64_t I1Size = AliasAnalysis::UnknownSize;
+    uint64_t I1Size = MemoryLocation::UnknownSize;
     Type *I1ElTy = cast<PointerType>((*I1)->getType())->getElementType();
     if (I1ElTy->isSized()) I1Size = AA.getTypeStoreSize(I1ElTy);
 
     for (SetVector<Value *>::iterator I2 = Pointers.begin(); I2 != I1; ++I2) {
-      uint64_t I2Size = AliasAnalysis::UnknownSize;
+      uint64_t I2Size = MemoryLocation::UnknownSize;
       Type *I2ElTy =cast<PointerType>((*I2)->getType())->getElementType();
       if (I2ElTy->isSized()) I2Size = AA.getTypeStoreSize(I2ElTy);
 
       switch (AA.alias(*I1, I1Size, *I2, I2Size)) {
-      case AliasAnalysis::NoAlias:
+      case NoAlias:
         PrintResults("NoAlias", PrintNoAlias, *I1, *I2, F.getParent());
-        ++NoAlias; break;
-      case AliasAnalysis::MayAlias:
+        ++NoAliasCount;
+        break;
+      case MayAlias:
         PrintResults("MayAlias", PrintMayAlias, *I1, *I2, F.getParent());
-        ++MayAlias; break;
-      case AliasAnalysis::PartialAlias:
+        ++MayAliasCount;
+        break;
+      case PartialAlias:
         PrintResults("PartialAlias", PrintPartialAlias, *I1, *I2,
                      F.getParent());
-        ++PartialAlias; break;
-      case AliasAnalysis::MustAlias:
+        ++PartialAliasCount;
+        break;
+      case MustAlias:
         PrintResults("MustAlias", PrintMustAlias, *I1, *I2, F.getParent());
-        ++MustAlias; break;
+        ++MustAliasCount;
+        break;
       }
     }
   }
@@ -219,24 +223,28 @@ bool AAEval::runOnFunction(Function &F) {
          I1 != E; ++I1) {
       for (SetVector<Value *>::iterator I2 = Stores.begin(), E2 = Stores.end();
            I2 != E2; ++I2) {
-        switch (AA.alias(AA.getLocation(cast<LoadInst>(*I1)),
-                         AA.getLocation(cast<StoreInst>(*I2)))) {
-        case AliasAnalysis::NoAlias:
+        switch (AA.alias(MemoryLocation::get(cast<LoadInst>(*I1)),
+                         MemoryLocation::get(cast<StoreInst>(*I2)))) {
+        case NoAlias:
           PrintLoadStoreResults("NoAlias", PrintNoAlias, *I1, *I2,
                                 F.getParent());
-          ++NoAlias; break;
-        case AliasAnalysis::MayAlias:
+          ++NoAliasCount;
+          break;
+        case MayAlias:
           PrintLoadStoreResults("MayAlias", PrintMayAlias, *I1, *I2,
                                 F.getParent());
-          ++MayAlias; break;
-        case AliasAnalysis::PartialAlias:
+          ++MayAliasCount;
+          break;
+        case PartialAlias:
           PrintLoadStoreResults("PartialAlias", PrintPartialAlias, *I1, *I2,
                                 F.getParent());
-          ++PartialAlias; break;
-        case AliasAnalysis::MustAlias:
+          ++PartialAliasCount;
+          break;
+        case MustAlias:
           PrintLoadStoreResults("MustAlias", PrintMustAlias, *I1, *I2,
                                 F.getParent());
-          ++MustAlias; break;
+          ++MustAliasCount;
+          break;
         }
       }
     }
@@ -245,24 +253,28 @@ bool AAEval::runOnFunction(Function &F) {
     for (SetVector<Value *>::iterator I1 = Stores.begin(), E = Stores.end();
          I1 != E; ++I1) {
       for (SetVector<Value *>::iterator I2 = Stores.begin(); I2 != I1; ++I2) {
-        switch (AA.alias(AA.getLocation(cast<StoreInst>(*I1)),
-                         AA.getLocation(cast<StoreInst>(*I2)))) {
-        case AliasAnalysis::NoAlias:
+        switch (AA.alias(MemoryLocation::get(cast<StoreInst>(*I1)),
+                         MemoryLocation::get(cast<StoreInst>(*I2)))) {
+        case NoAlias:
           PrintLoadStoreResults("NoAlias", PrintNoAlias, *I1, *I2,
                                 F.getParent());
-          ++NoAlias; break;
-        case AliasAnalysis::MayAlias:
+          ++NoAliasCount;
+          break;
+        case MayAlias:
           PrintLoadStoreResults("MayAlias", PrintMayAlias, *I1, *I2,
                                 F.getParent());
-          ++MayAlias; break;
-        case AliasAnalysis::PartialAlias:
+          ++MayAliasCount;
+          break;
+        case PartialAlias:
           PrintLoadStoreResults("PartialAlias", PrintPartialAlias, *I1, *I2,
                                 F.getParent());
-          ++PartialAlias; break;
-        case AliasAnalysis::MustAlias:
+          ++PartialAliasCount;
+          break;
+        case MustAlias:
           PrintLoadStoreResults("MustAlias", PrintMustAlias, *I1, *I2,
                                 F.getParent());
-          ++MustAlias; break;
+          ++MustAliasCount;
+          break;
         }
       }
     }
@@ -275,23 +287,27 @@ bool AAEval::runOnFunction(Function &F) {
 
     for (SetVector<Value *>::iterator V = Pointers.begin(), Ve = Pointers.end();
          V != Ve; ++V) {
-      uint64_t Size = AliasAnalysis::UnknownSize;
+      uint64_t Size = MemoryLocation::UnknownSize;
       Type *ElTy = cast<PointerType>((*V)->getType())->getElementType();
       if (ElTy->isSized()) Size = AA.getTypeStoreSize(ElTy);
 
       switch (AA.getModRefInfo(*C, *V, Size)) {
-      case AliasAnalysis::NoModRef:
+      case MRI_NoModRef:
         PrintModRefResults("NoModRef", PrintNoModRef, I, *V, F.getParent());
-        ++NoModRef; break;
-      case AliasAnalysis::Mod:
+        ++NoModRefCount;
+        break;
+      case MRI_Mod:
         PrintModRefResults("Just Mod", PrintMod, I, *V, F.getParent());
-        ++Mod; break;
-      case AliasAnalysis::Ref:
+        ++ModCount;
+        break;
+      case MRI_Ref:
         PrintModRefResults("Just Ref", PrintRef, I, *V, F.getParent());
-        ++Ref; break;
-      case AliasAnalysis::ModRef:
+        ++RefCount;
+        break;
+      case MRI_ModRef:
         PrintModRefResults("Both ModRef", PrintModRef, I, *V, F.getParent());
-        ++ModRef; break;
+        ++ModRefCount;
+        break;
       }
     }
   }
@@ -303,18 +319,22 @@ bool AAEval::runOnFunction(Function &F) {
       if (D == C)
         continue;
       switch (AA.getModRefInfo(*C, *D)) {
-      case AliasAnalysis::NoModRef:
+      case MRI_NoModRef:
         PrintModRefResults("NoModRef", PrintNoModRef, *C, *D, F.getParent());
-        ++NoModRef; break;
-      case AliasAnalysis::Mod:
+        ++NoModRefCount;
+        break;
+      case MRI_Mod:
         PrintModRefResults("Just Mod", PrintMod, *C, *D, F.getParent());
-        ++Mod; break;
-      case AliasAnalysis::Ref:
+        ++ModCount;
+        break;
+      case MRI_Ref:
         PrintModRefResults("Just Ref", PrintRef, *C, *D, F.getParent());
-        ++Ref; break;
-      case AliasAnalysis::ModRef:
+        ++RefCount;
+        break;
+      case MRI_ModRef:
         PrintModRefResults("Both ModRef", PrintModRef, *C, *D, F.getParent());
-        ++ModRef; break;
+        ++ModRefCount;
+        break;
       }
     }
   }
@@ -328,43 +348,47 @@ static void PrintPercent(unsigned Num, unsigned Sum) {
 }
 
 bool AAEval::doFinalization(Module &M) {
-  unsigned AliasSum = NoAlias + MayAlias + PartialAlias + MustAlias;
+  unsigned AliasSum =
+      NoAliasCount + MayAliasCount + PartialAliasCount + MustAliasCount;
   errs() << "===== Alias Analysis Evaluator Report =====\n";
   if (AliasSum == 0) {
     errs() << "  Alias Analysis Evaluator Summary: No pointers!\n";
   } else {
     errs() << "  " << AliasSum << " Total Alias Queries Performed\n";
-    errs() << "  " << NoAlias << " no alias responses ";
-    PrintPercent(NoAlias, AliasSum);
-    errs() << "  " << MayAlias << " may alias responses ";
-    PrintPercent(MayAlias, AliasSum);
-    errs() << "  " << PartialAlias << " partial alias responses ";
-    PrintPercent(PartialAlias, AliasSum);
-    errs() << "  " << MustAlias << " must alias responses ";
-    PrintPercent(MustAlias, AliasSum);
+    errs() << "  " << NoAliasCount << " no alias responses ";
+    PrintPercent(NoAliasCount, AliasSum);
+    errs() << "  " << MayAliasCount << " may alias responses ";
+    PrintPercent(MayAliasCount, AliasSum);
+    errs() << "  " << PartialAliasCount << " partial alias responses ";
+    PrintPercent(PartialAliasCount, AliasSum);
+    errs() << "  " << MustAliasCount << " must alias responses ";
+    PrintPercent(MustAliasCount, AliasSum);
     errs() << "  Alias Analysis Evaluator Pointer Alias Summary: "
-           << NoAlias*100/AliasSum  << "%/" << MayAlias*100/AliasSum << "%/"
-           << PartialAlias*100/AliasSum << "%/"
-           << MustAlias*100/AliasSum << "%\n";
+           << NoAliasCount * 100 / AliasSum << "%/"
+           << MayAliasCount * 100 / AliasSum << "%/"
+           << PartialAliasCount * 100 / AliasSum << "%/"
+           << MustAliasCount * 100 / AliasSum << "%\n";
   }
 
   // Display the summary for mod/ref analysis
-  unsigned ModRefSum = NoModRef + Mod + Ref + ModRef;
+  unsigned ModRefSum = NoModRefCount + ModCount + RefCount + ModRefCount;
   if (ModRefSum == 0) {
-    errs() << "  Alias Analysis Mod/Ref Evaluator Summary: no mod/ref!\n";
+    errs() << "  Alias Analysis Mod/Ref Evaluator Summary: no "
+              "mod/ref!\n";
   } else {
     errs() << "  " << ModRefSum << " Total ModRef Queries Performed\n";
-    errs() << "  " << NoModRef << " no mod/ref responses ";
-    PrintPercent(NoModRef, ModRefSum);
-    errs() << "  " << Mod << " mod responses ";
-    PrintPercent(Mod, ModRefSum);
-    errs() << "  " << Ref << " ref responses ";
-    PrintPercent(Ref, ModRefSum);
-    errs() << "  " << ModRef << " mod & ref responses ";
-    PrintPercent(ModRef, ModRefSum);
+    errs() << "  " << NoModRefCount << " no mod/ref responses ";
+    PrintPercent(NoModRefCount, ModRefSum);
+    errs() << "  " << ModCount << " mod responses ";
+    PrintPercent(ModCount, ModRefSum);
+    errs() << "  " << RefCount << " ref responses ";
+    PrintPercent(RefCount, ModRefSum);
+    errs() << "  " << ModRefCount << " mod & ref responses ";
+    PrintPercent(ModRefCount, ModRefSum);
     errs() << "  Alias Analysis Evaluator Mod/Ref Summary: "
-           << NoModRef*100/ModRefSum  << "%/" << Mod*100/ModRefSum << "%/"
-           << Ref*100/ModRefSum << "%/" << ModRef*100/ModRefSum << "%\n";
+           << NoModRefCount * 100 / ModRefSum << "%/"
+           << ModCount * 100 / ModRefSum << "%/" << RefCount * 100 / ModRefSum
+           << "%/" << ModRefCount * 100 / ModRefSum << "%\n";
   }
 
   return false;
diff --git a/lib/Analysis/AliasDebugger.cpp b/lib/Analysis/AliasDebugger.cpp
index f98b5781960..146036b0060 100644
--- a/lib/Analysis/AliasDebugger.cpp
+++ b/lib/Analysis/AliasDebugger.cpp
@@ -94,7 +94,8 @@ namespace {
     //------------------------------------------------
     // Implement the AliasAnalysis API
     //
-    AliasResult alias(const Location &LocA, const Location &LocB) override {
+    AliasResult alias(const MemoryLocation &LocA,
+                      const MemoryLocation &LocB) override {
       assert(Vals.find(LocA.Ptr) != Vals.end() &&
              "Never seen value in AA before");
       assert(Vals.find(LocB.Ptr) != Vals.end() &&
@@ -102,31 +103,22 @@ namespace {
       return AliasAnalysis::alias(LocA, LocB);
     }
 
-    ModRefResult getModRefInfo(ImmutableCallSite CS,
-                               const Location &Loc) override {
+    ModRefInfo getModRefInfo(ImmutableCallSite CS,
+                             const MemoryLocation &Loc) override {
       assert(Vals.find(Loc.Ptr) != Vals.end() && "Never seen value in AA before");
       return AliasAnalysis::getModRefInfo(CS, Loc);
     }
 
-    ModRefResult getModRefInfo(ImmutableCallSite CS1,
-                               ImmutableCallSite CS2) override {
+    ModRefInfo getModRefInfo(ImmutableCallSite CS1,
+                             ImmutableCallSite CS2) override {
       return AliasAnalysis::getModRefInfo(CS1,CS2);
     }
 
-    bool pointsToConstantMemory(const Location &Loc, bool OrLocal) override {
+    bool pointsToConstantMemory(const MemoryLocation &Loc,
+                                bool OrLocal) override {
       assert(Vals.find(Loc.Ptr) != Vals.end() && "Never seen value in AA before");
       return AliasAnalysis::pointsToConstantMemory(Loc, OrLocal);
     }
-
-    void deleteValue(Value *V) override {
-      assert(Vals.find(V) != Vals.end() && "Never seen value in AA before");
-      AliasAnalysis::deleteValue(V);
-    }
-    void copyValue(Value *From, Value *To) override {
-      Vals.insert(To);
-      AliasAnalysis::copyValue(From, To);
-    }
-
   };
 }
 
diff --git a/lib/Analysis/AliasSetTracker.cpp b/lib/Analysis/AliasSetTracker.cpp
index 4c79b9f3e8e..05bbe9b227d 100644
--- a/lib/Analysis/AliasSetTracker.cpp
+++ b/lib/Analysis/AliasSetTracker.cpp
@@ -32,11 +32,11 @@ void AliasSet::mergeSetIn(AliasSet &AS, AliasSetTracker &AST) {
   assert(!Forward && "This set is a forwarding set!!");
 
   // Update the alias and access types of this set...
-  AccessTy |= AS.AccessTy;
-  AliasTy  |= AS.AliasTy;
+  Access |= AS.Access;
+  Alias  |= AS.Alias;
   Volatile |= AS.Volatile;
 
-  if (AliasTy == MustAlias) {
+  if (Alias == SetMustAlias) {
     // Check that these two merged sets really are must aliases.  Since both
     // used to be must-alias sets, we can just check any pointer from each set
     // for aliasing.
@@ -45,14 +45,10 @@ void AliasSet::mergeSetIn(AliasSet &AS, AliasSetTracker &AST) {
     PointerRec *R = AS.getSomePointer();
 
     // If the pointers are not a must-alias pair, this set becomes a may alias.
-    if (AA.alias(AliasAnalysis::Location(L->getValue(),
-                                         L->getSize(),
-                                         L->getAAInfo()),
-                 AliasAnalysis::Location(R->getValue(),
-                                         R->getSize(),
-                                         R->getAAInfo()))
-        != AliasAnalysis::MustAlias)
-      AliasTy = MayAlias;
+    if (AA.alias(MemoryLocation(L->getValue(), L->getSize(), L->getAAInfo()),
+                 MemoryLocation(R->getValue(), R->getSize(), R->getAAInfo())) !=
+        MustAlias)
+      Alias = SetMayAlias;
   }
 
   bool ASHadUnknownInsts = !AS.UnknownInsts.empty();
@@ -105,15 +101,14 @@ void AliasSet::addPointer(AliasSetTracker &AST, PointerRec &Entry,
   if (isMustAlias() && !KnownMustAlias)
     if (PointerRec *P = getSomePointer()) {
       AliasAnalysis &AA = AST.getAliasAnalysis();
-      AliasAnalysis::AliasResult Result =
-        AA.alias(AliasAnalysis::Location(P->getValue(), P->getSize(),
-                                         P->getAAInfo()),
-                 AliasAnalysis::Location(Entry.getValue(), Size, AAInfo));
-      if (Result != AliasAnalysis::MustAlias)
-        AliasTy = MayAlias;
+      AliasResult Result =
+          AA.alias(MemoryLocation(P->getValue(), P->getSize(), P->getAAInfo()),
+                   MemoryLocation(Entry.getValue(), Size, AAInfo));
+      if (Result != MustAlias)
+        Alias = SetMayAlias;
       else                  // First entry of must alias must have maximum size!
         P->updateSizeAndAAInfo(Size, AAInfo);
-      assert(Result != AliasAnalysis::NoAlias && "Cannot be part of must set!");
+      assert(Result != NoAlias && "Cannot be part of must set!");
     }
 
   Entry.setAliasSet(this);
@@ -130,17 +125,17 @@ void AliasSet::addPointer(AliasSetTracker &AST, PointerRec &Entry,
 void AliasSet::addUnknownInst(Instruction *I, AliasAnalysis &AA) {
   if (UnknownInsts.empty())
     addRef();
-  UnknownInsts.push_back(I);
+  UnknownInsts.emplace_back(I);
 
   if (!I->mayWriteToMemory()) {
-    AliasTy = MayAlias;
-    AccessTy |= Refs;
+    Alias = SetMayAlias;
+    Access |= RefAccess;
     return;
   }
 
   // FIXME: This should use mod/ref information to make this not suck so bad
-  AliasTy = MayAlias;
-  AccessTy = ModRef;
+  Alias = SetMayAlias;
+  Access = ModRefAccess;
 }
 
 /// aliasesPointer - Return true if the specified pointer "may" (or must)
@@ -149,56 +144,51 @@ void AliasSet::addUnknownInst(Instruction *I, AliasAnalysis &AA) {
 bool AliasSet::aliasesPointer(const Value *Ptr, uint64_t Size,
                               const AAMDNodes &AAInfo,
                               AliasAnalysis &AA) const {
-  if (AliasTy == MustAlias) {
+  if (Alias == SetMustAlias) {
     assert(UnknownInsts.empty() && "Illegal must alias set!");
 
     // If this is a set of MustAliases, only check to see if the pointer aliases
     // SOME value in the set.
     PointerRec *SomePtr = getSomePointer();
     assert(SomePtr && "Empty must-alias set??");
-    return AA.alias(AliasAnalysis::Location(SomePtr->getValue(),
-                                            SomePtr->getSize(),
-                                            SomePtr->getAAInfo()),
-                    AliasAnalysis::Location(Ptr, Size, AAInfo));
+    return AA.alias(MemoryLocation(SomePtr->getValue(), SomePtr->getSize(),
+                                   SomePtr->getAAInfo()),
+                    MemoryLocation(Ptr, Size, AAInfo));
   }
 
   // If this is a may-alias set, we have to check all of the pointers in the set
   // to be sure it doesn't alias the set...
   for (iterator I = begin(), E = end(); I != E; ++I)
-    if (AA.alias(AliasAnalysis::Location(Ptr, Size, AAInfo),
-                 AliasAnalysis::Location(I.getPointer(), I.getSize(),
-                                         I.getAAInfo())))
+    if (AA.alias(MemoryLocation(Ptr, Size, AAInfo),
+                 MemoryLocation(I.getPointer(), I.getSize(), I.getAAInfo())))
       return true;
 
   // Check the unknown instructions...
   if (!UnknownInsts.empty()) {
     for (unsigned i = 0, e = UnknownInsts.size(); i != e; ++i)
       if (AA.getModRefInfo(UnknownInsts[i],
-                           AliasAnalysis::Location(Ptr, Size, AAInfo)) !=
-            AliasAnalysis::NoModRef)
+                           MemoryLocation(Ptr, Size, AAInfo)) != MRI_NoModRef)
         return true;
   }
 
   return false;
 }
 
-bool AliasSet::aliasesUnknownInst(Instruction *Inst, AliasAnalysis &AA) const {
+bool AliasSet::aliasesUnknownInst(const Instruction *Inst,
+                                  AliasAnalysis &AA) const {
   if (!Inst->mayReadOrWriteMemory())
     return false;
 
   for (unsigned i = 0, e = UnknownInsts.size(); i != e; ++i) {
-    CallSite C1(getUnknownInst(i)), C2(Inst);
-    if (!C1 || !C2 ||
-        AA.getModRefInfo(C1, C2) != AliasAnalysis::NoModRef ||
-        AA.getModRefInfo(C2, C1) != AliasAnalysis::NoModRef)
+    ImmutableCallSite C1(getUnknownInst(i)), C2(Inst);
+    if (!C1 || !C2 || AA.getModRefInfo(C1, C2) != MRI_NoModRef ||
+        AA.getModRefInfo(C2, C1) != MRI_NoModRef)
       return true;
   }
 
   for (iterator I = begin(), E = end(); I != E; ++I)
-    if (AA.getModRefInfo(Inst, AliasAnalysis::Location(I.getPointer(),
-                                                       I.getSize(),
-                                                       I.getAAInfo())) !=
-           AliasAnalysis::NoModRef)
+    if (AA.getModRefInfo(Inst, MemoryLocation(I.getPointer(), I.getSize(),
+                                              I.getAAInfo())) != MRI_NoModRef)
       return true;
 
   return false;
@@ -242,7 +232,7 @@ AliasSet *AliasSetTracker::findAliasSetForPointer(const Value *Ptr,
 /// containsPointer - Return true if the specified location is represented by
 /// this alias set, false otherwise.  This does not modify the AST object or
 /// alias sets.
-bool AliasSetTracker::containsPointer(Value *Ptr, uint64_t Size,
+bool AliasSetTracker::containsPointer(const Value *Ptr, uint64_t Size,
                                       const AAMDNodes &AAInfo) const {
   for (const_iterator I = begin(), E = end(); I != E; ++I)
     if (!I->Forward && I->aliasesPointer(Ptr, Size, AAInfo, AA))
@@ -250,7 +240,7 @@ bool AliasSetTracker::containsPointer(Value *Ptr, uint64_t Size,
   return false;
 }
 
-bool AliasSetTracker::containsUnknown(Instruction *Inst) const {
+bool AliasSetTracker::containsUnknown(const Instruction *Inst) const {
   for (const_iterator I = begin(), E = end(); I != E; ++I)
     if (!I->Forward && I->aliasesUnknownInst(Inst, AA))
       return true;
@@ -303,7 +293,7 @@ AliasSet &AliasSetTracker::getAliasSetForPointer(Value *Pointer, uint64_t Size,
 
 bool AliasSetTracker::add(Value *Ptr, uint64_t Size, const AAMDNodes &AAInfo) {
   bool NewPtr;
-  addPointer(Ptr, Size, AAInfo, AliasSet::NoModRef, NewPtr);
+  addPointer(Ptr, Size, AAInfo, AliasSet::NoAccess, NewPtr);
   return NewPtr;
 }
 
@@ -314,11 +304,11 @@ bool AliasSetTracker::add(LoadInst *LI) {
   AAMDNodes AAInfo;
   LI->getAAMetadata(AAInfo);
 
-  AliasSet::AccessType ATy = AliasSet::Refs;
+  AliasSet::AccessLattice Access = AliasSet::RefAccess;
   bool NewPtr;
   AliasSet &AS = addPointer(LI->getOperand(0),
                             AA.getTypeStoreSize(LI->getType()),
-                            AAInfo, ATy, NewPtr);
+                            AAInfo, Access, NewPtr);
   if (LI->isVolatile()) AS.setVolatile();
   return NewPtr;
 }
@@ -329,12 +319,12 @@ bool AliasSetTracker::add(StoreInst *SI) {
   AAMDNodes AAInfo;
   SI->getAAMetadata(AAInfo);
 
-  AliasSet::AccessType ATy = AliasSet::Mods;
+  AliasSet::AccessLattice Access = AliasSet::ModAccess;
   bool NewPtr;
   Value *Val = SI->getOperand(0);
   AliasSet &AS = addPointer(SI->getOperand(1),
                             AA.getTypeStoreSize(Val->getType()),
-                            AAInfo, ATy, NewPtr);
+                            AAInfo, Access, NewPtr);
   if (SI->isVolatile()) AS.setVolatile();
   return NewPtr;
 }
@@ -344,8 +334,8 @@ bool AliasSetTracker::add(VAArgInst *VAAI) {
   VAAI->getAAMetadata(AAInfo);
 
   bool NewPtr;
-  addPointer(VAAI->getOperand(0), AliasAnalysis::UnknownSize, 
-             AAInfo, AliasSet::ModRef, NewPtr);
+  addPointer(VAAI->getOperand(0), MemoryLocation::UnknownSize, AAInfo,
+             AliasSet::ModRefAccess, NewPtr);
   return NewPtr;
 }
 
@@ -404,7 +394,7 @@ void AliasSetTracker::add(const AliasSetTracker &AST) {
     for (AliasSet::iterator ASI = AS.begin(), E = AS.end(); ASI != E; ++ASI) {
       AliasSet &NewAS = addPointer(ASI.getPointer(), ASI.getSize(),
                                    ASI.getAAInfo(),
-                                   (AliasSet::AccessType)AS.AccessTy, X);
+                                   (AliasSet::AccessLattice)AS.Access, X);
       if (AS.isVolatile()) NewAS.setVolatile();
     }
   }
@@ -478,7 +468,7 @@ bool AliasSetTracker::remove(VAArgInst *VAAI) {
   VAAI->getAAMetadata(AAInfo);
 
   AliasSet *AS = findAliasSetForPointer(VAAI->getOperand(0),
-                                        AliasAnalysis::UnknownSize, AAInfo);
+                                        MemoryLocation::UnknownSize, AAInfo);
   if (!AS) return false;
   remove(*AS);
   return true;
@@ -512,9 +502,6 @@ bool AliasSetTracker::remove(Instruction *I) {
 // dangling pointers to deleted instructions.
 //
 void AliasSetTracker::deleteValue(Value *PtrVal) {
-  // Notify the alias analysis implementation that this value is gone.
-  AA.deleteValue(PtrVal);
-
   // If this is a call instruction, remove the callsite from the appropriate
   // AliasSet (if present).
   if (Instruction *Inst = dyn_cast<Instruction>(PtrVal)) {
@@ -551,9 +538,6 @@ void AliasSetTracker::deleteValue(Value *PtrVal) {
 // the tracker already knows about a value, it will ignore the request.
 //
 void AliasSetTracker::copyValue(Value *From, Value *To) {
-  // Notify the alias analysis implementation that this value is copied.
-  AA.copyValue(From, To);
-
   // First, look up the PointerRec for this pointer.
   PointerMapType::iterator I = PointerMap.find_as(From);
   if (I == PointerMap.end())
@@ -579,13 +563,13 @@ void AliasSetTracker::copyValue(Value *From, Value *To) {
 
 void AliasSet::print(raw_ostream &OS) const {
   OS << "  AliasSet[" << (const void*)this << ", " << RefCount << "] ";
-  OS << (AliasTy == MustAlias ? "must" : "may") << " alias, ";
-  switch (AccessTy) {
-  case NoModRef: OS << "No access "; break;
-  case Refs    : OS << "Ref       "; break;
-  case Mods    : OS << "Mod       "; break;
-  case ModRef  : OS << "Mod/Ref   "; break;
-  default: llvm_unreachable("Bad value for AccessTy!");
+  OS << (Alias == SetMustAlias ? "must" : "may") << " alias, ";
+  switch (Access) {
+  case NoAccess:     OS << "No access "; break;
+  case RefAccess:    OS << "Ref       "; break;
+  case ModAccess:    OS << "Mod       "; break;
+  case ModRefAccess: OS << "Mod/Ref   "; break;
+  default: llvm_unreachable("Bad value for Access!");
   }
   if (isVolatile()) OS << "[volatile] ";
   if (Forward)
diff --git a/lib/Analysis/Analysis.cpp b/lib/Analysis/Analysis.cpp
index 842ff0a14e2..3ce87f9a76c 100644
--- a/lib/Analysis/Analysis.cpp
+++ b/lib/Analysis/Analysis.cpp
@@ -27,8 +27,8 @@ void llvm::initializeAnalysis(PassRegistry &Registry) {
   initializeAliasSetPrinterPass(Registry);
   initializeNoAAPass(Registry);
   initializeBasicAliasAnalysisPass(Registry);
-  initializeBlockFrequencyInfoPass(Registry);
-  initializeBranchProbabilityInfoPass(Registry);
+  initializeBlockFrequencyInfoWrapperPassPass(Registry);
+  initializeBranchProbabilityInfoWrapperPassPass(Registry);
   initializeCostModelAnalysisPass(Registry);
   initializeCFGViewerPass(Registry);
   initializeCFGPrinterPass(Registry);
diff --git a/lib/Analysis/BasicAliasAnalysis.cpp b/lib/Analysis/BasicAliasAnalysis.cpp
index 2767e41bee0..511ea4bbfab 100644
--- a/lib/Analysis/BasicAliasAnalysis.cpp
+++ b/lib/Analysis/BasicAliasAnalysis.cpp
@@ -42,6 +42,10 @@
 #include <algorithm>
 using namespace llvm;
 
+/// Enable analysis of recursive PHI nodes.
+static cl::opt<bool> EnableRecPhiAnalysis("basicaa-recphi",
+                                          cl::Hidden, cl::init(false));
+
 /// Cutoff after which to stop analysing a set of phi nodes potentially involved
 /// in a cycle. Because we are analysing 'through' phi nodes we need to be
 /// careful with value equivalence. We use reachability to make sure a value
@@ -105,7 +109,7 @@ static uint64_t getObjectSize(const Value *V, const DataLayout &DL,
   uint64_t Size;
   if (getObjectSize(V, Size, DL, &TLI, RoundToAlign))
     return Size;
-  return AliasAnalysis::UnknownSize;
+  return MemoryLocation::UnknownSize;
 }
 
 /// isObjectSmallerThan - Return true if we can prove that the object specified
@@ -146,7 +150,7 @@ static bool isObjectSmallerThan(const Value *V, uint64_t Size,
   // reads a bit past the end given sufficient alignment.
   uint64_t ObjectSize = getObjectSize(V, DL, TLI, /*RoundToAlign*/true);
 
-  return ObjectSize != AliasAnalysis::UnknownSize && ObjectSize < Size;
+  return ObjectSize != MemoryLocation::UnknownSize && ObjectSize < Size;
 }
 
 /// isObjectSize - Return true if we can prove that the object specified
@@ -154,7 +158,7 @@ static bool isObjectSmallerThan(const Value *V, uint64_t Size,
 static bool isObjectSize(const Value *V, uint64_t Size,
                          const DataLayout &DL, const TargetLibraryInfo &TLI) {
   uint64_t ObjectSize = getObjectSize(V, DL, TLI);
-  return ObjectSize != AliasAnalysis::UnknownSize && ObjectSize == Size;
+  return ObjectSize != MemoryLocation::UnknownSize && ObjectSize == Size;
 }
 
 //===----------------------------------------------------------------------===//
@@ -459,7 +463,8 @@ namespace {
       AU.addRequired<TargetLibraryInfoWrapperPass>();
     }
 
-    AliasResult alias(const Location &LocA, const Location &LocB) override {
+    AliasResult alias(const MemoryLocation &LocA,
+                      const MemoryLocation &LocB) override {
       assert(AliasCache.empty() && "AliasCache must be cleared after use!");
       assert(notDifferentParent(LocA.Ptr, LocB.Ptr) &&
              "BasicAliasAnalysis doesn't support interprocedural queries.");
@@ -474,27 +479,27 @@ namespace {
       return Alias;
     }
 
-    ModRefResult getModRefInfo(ImmutableCallSite CS,
-                               const Location &Loc) override;
+    ModRefInfo getModRefInfo(ImmutableCallSite CS,
+                             const MemoryLocation &Loc) override;
 
-    ModRefResult getModRefInfo(ImmutableCallSite CS1,
-                               ImmutableCallSite CS2) override;
+    ModRefInfo getModRefInfo(ImmutableCallSite CS1,
+                             ImmutableCallSite CS2) override;
 
     /// pointsToConstantMemory - Chase pointers until we find a (constant
     /// global) or not.
-    bool pointsToConstantMemory(const Location &Loc, bool OrLocal) override;
+    bool pointsToConstantMemory(const MemoryLocation &Loc,
+                                bool OrLocal) override;
 
     /// Get the location associated with a pointer argument of a callsite.
-    Location getArgLocation(ImmutableCallSite CS, unsigned ArgIdx,
-                            ModRefResult &Mask) override;
+    ModRefInfo getArgModRefInfo(ImmutableCallSite CS, unsigned ArgIdx) override;
 
     /// getModRefBehavior - Return the behavior when calling the given
     /// call site.
-    ModRefBehavior getModRefBehavior(ImmutableCallSite CS) override;
+    FunctionModRefBehavior getModRefBehavior(ImmutableCallSite CS) override;
 
     /// getModRefBehavior - Return the behavior when calling the given function.
     /// For use when the call site is not known.
-    ModRefBehavior getModRefBehavior(const Function *F) override;
+    FunctionModRefBehavior getModRefBehavior(const Function *F) override;
 
     /// getAdjustedAnalysisPointer - This method is used when a pass implements
     /// an analysis interface through multiple inheritance.  If needed, it
@@ -508,7 +513,7 @@ namespace {
 
   private:
     // AliasCache - Track alias queries to guard against recursion.
-    typedef std::pair<Location, Location> LocPair;
+    typedef std::pair<MemoryLocation, MemoryLocation> LocPair;
     typedef SmallDenseMap<LocPair, AliasResult, 8> AliasCacheTy;
     AliasCacheTy AliasCache;
 
@@ -592,8 +597,8 @@ ImmutablePass *llvm::createBasicAliasAnalysisPass() {
 /// pointsToConstantMemory - Returns whether the given pointer value
 /// points to memory that is local to the function, with global constants being
 /// considered local to all functions.
-bool
-BasicAliasAnalysis::pointsToConstantMemory(const Location &Loc, bool OrLocal) {
+bool BasicAliasAnalysis::pointsToConstantMemory(const MemoryLocation &Loc,
+                                                bool OrLocal) {
   assert(Visited.empty() && "Visited must be cleared after use!");
 
   unsigned MaxLookup = 8;
@@ -637,8 +642,8 @@ BasicAliasAnalysis::pointsToConstantMemory(const Location &Loc, bool OrLocal) {
         Visited.clear();
         return AliasAnalysis::pointsToConstantMemory(Loc, OrLocal);
       }
-      for (unsigned i = 0, e = PN->getNumIncomingValues(); i != e; ++i)
-        Worklist.push_back(PN->getIncomingValue(i));
+      for (Value *IncValue : PN->incoming_values())
+        Worklist.push_back(IncValue);
       continue;
     }
 
@@ -652,6 +657,8 @@ BasicAliasAnalysis::pointsToConstantMemory(const Location &Loc, bool OrLocal) {
   return Worklist.empty();
 }
 
+// FIXME: This code is duplicated with MemoryLocation and should be hoisted to
+// some common utility location.
 static bool isMemsetPattern16(const Function *MS,
                               const TargetLibraryInfo &TLI) {
   if (TLI.has(LibFunc::memset_pattern16) &&
@@ -668,131 +675,86 @@ static bool isMemsetPattern16(const Function *MS,
 }
 
 /// getModRefBehavior - Return the behavior when calling the given call site.
-AliasAnalysis::ModRefBehavior
+FunctionModRefBehavior
 BasicAliasAnalysis::getModRefBehavior(ImmutableCallSite CS) {
   if (CS.doesNotAccessMemory())
     // Can't do better than this.
-    return DoesNotAccessMemory;
+    return FMRB_DoesNotAccessMemory;
 
-  ModRefBehavior Min = UnknownModRefBehavior;
+  FunctionModRefBehavior Min = FMRB_UnknownModRefBehavior;
 
   // If the callsite knows it only reads memory, don't return worse
   // than that.
   if (CS.onlyReadsMemory())
-    Min = OnlyReadsMemory;
+    Min = FMRB_OnlyReadsMemory;
+
+  if (CS.onlyAccessesArgMemory())
+    Min = FunctionModRefBehavior(Min & FMRB_OnlyAccessesArgumentPointees);
 
   // The AliasAnalysis base class has some smarts, lets use them.
-  return ModRefBehavior(AliasAnalysis::getModRefBehavior(CS) & Min);
+  return FunctionModRefBehavior(AliasAnalysis::getModRefBehavior(CS) & Min);
 }
 
 /// getModRefBehavior - Return the behavior when calling the given function.
 /// For use when the call site is not known.
-AliasAnalysis::ModRefBehavior
+FunctionModRefBehavior
 BasicAliasAnalysis::getModRefBehavior(const Function *F) {
   // If the function declares it doesn't access memory, we can't do better.
   if (F->doesNotAccessMemory())
-    return DoesNotAccessMemory;
+    return FMRB_DoesNotAccessMemory;
 
   // For intrinsics, we can check the table.
-  if (unsigned iid = F->getIntrinsicID()) {
+  if (Intrinsic::ID iid = F->getIntrinsicID()) {
 #define GET_INTRINSIC_MODREF_BEHAVIOR
 #include "llvm/IR/Intrinsics.gen"
 #undef GET_INTRINSIC_MODREF_BEHAVIOR
   }
 
-  ModRefBehavior Min = UnknownModRefBehavior;
+  FunctionModRefBehavior Min = FMRB_UnknownModRefBehavior;
 
   // If the function declares it only reads memory, go with that.
   if (F->onlyReadsMemory())
-    Min = OnlyReadsMemory;
+    Min = FMRB_OnlyReadsMemory;
+
+  if (F->onlyAccessesArgMemory())
+    Min = FunctionModRefBehavior(Min & FMRB_OnlyAccessesArgumentPointees);
 
   const TargetLibraryInfo &TLI =
       getAnalysis<TargetLibraryInfoWrapperPass>().getTLI();
   if (isMemsetPattern16(F, TLI))
-    Min = OnlyAccessesArgumentPointees;
+    Min = FMRB_OnlyAccessesArgumentPointees;
 
   // Otherwise be conservative.
-  return ModRefBehavior(AliasAnalysis::getModRefBehavior(F) & Min);
+  return FunctionModRefBehavior(AliasAnalysis::getModRefBehavior(F) & Min);
 }
 
-AliasAnalysis::Location
-BasicAliasAnalysis::getArgLocation(ImmutableCallSite CS, unsigned ArgIdx,
-                                   ModRefResult &Mask) {
-  Location Loc = AliasAnalysis::getArgLocation(CS, ArgIdx, Mask);
-  const TargetLibraryInfo &TLI =
-      getAnalysis<TargetLibraryInfoWrapperPass>().getTLI();
-  const IntrinsicInst *II = dyn_cast<IntrinsicInst>(CS.getInstruction());
-  if (II != nullptr)
+ModRefInfo BasicAliasAnalysis::getArgModRefInfo(ImmutableCallSite CS,
+                                                unsigned ArgIdx) {
+  if (const IntrinsicInst *II = dyn_cast<IntrinsicInst>(CS.getInstruction()))
     switch (II->getIntrinsicID()) {
-    default: break;
+    default:
+      break;
     case Intrinsic::memset:
     case Intrinsic::memcpy:
-    case Intrinsic::memmove: {
+    case Intrinsic::memmove:
       assert((ArgIdx == 0 || ArgIdx == 1) &&
              "Invalid argument index for memory intrinsic");
-      if (ConstantInt *LenCI = dyn_cast<ConstantInt>(II->getArgOperand(2)))
-        Loc.Size = LenCI->getZExtValue();
-      assert(Loc.Ptr == II->getArgOperand(ArgIdx) &&
-             "Memory intrinsic location pointer not argument?");
-      Mask = ArgIdx ? Ref : Mod;
-      break;
-    }
-    case Intrinsic::lifetime_start:
-    case Intrinsic::lifetime_end:
-    case Intrinsic::invariant_start: {
-      assert(ArgIdx == 1 && "Invalid argument index");
-      assert(Loc.Ptr == II->getArgOperand(ArgIdx) &&
-             "Intrinsic location pointer not argument?");
-      Loc.Size = cast<ConstantInt>(II->getArgOperand(0))->getZExtValue();
-      break;
-    }
-    case Intrinsic::invariant_end: {
-      assert(ArgIdx == 2 && "Invalid argument index");
-      assert(Loc.Ptr == II->getArgOperand(ArgIdx) &&
-             "Intrinsic location pointer not argument?");
-      Loc.Size = cast<ConstantInt>(II->getArgOperand(1))->getZExtValue();
-      break;
-    }
-    case Intrinsic::arm_neon_vld1: {
-      assert(ArgIdx == 0 && "Invalid argument index");
-      assert(Loc.Ptr == II->getArgOperand(ArgIdx) &&
-             "Intrinsic location pointer not argument?");
-      // LLVM's vld1 and vst1 intrinsics currently only support a single
-      // vector register.
-      if (DL)
-        Loc.Size = DL->getTypeStoreSize(II->getType());
-      break;
-    }
-    case Intrinsic::arm_neon_vst1: {
-      assert(ArgIdx == 0 && "Invalid argument index");
-      assert(Loc.Ptr == II->getArgOperand(ArgIdx) &&
-             "Intrinsic location pointer not argument?");
-      if (DL)
-        Loc.Size = DL->getTypeStoreSize(II->getArgOperand(1)->getType());
-      break;
-    }
+      return ArgIdx ? MRI_Ref : MRI_Mod;
     }
 
   // We can bound the aliasing properties of memset_pattern16 just as we can
   // for memcpy/memset.  This is particularly important because the
   // LoopIdiomRecognizer likes to turn loops into calls to memset_pattern16
   // whenever possible.
-  else if (CS.getCalledFunction() &&
-           isMemsetPattern16(CS.getCalledFunction(), TLI)) {
+  if (CS.getCalledFunction() &&
+      isMemsetPattern16(CS.getCalledFunction(), *TLI)) {
     assert((ArgIdx == 0 || ArgIdx == 1) &&
            "Invalid argument index for memset_pattern16");
-    if (ArgIdx == 1)
-      Loc.Size = 16;
-    else if (const ConstantInt *LenCI =
-             dyn_cast<ConstantInt>(CS.getArgument(2)))
-      Loc.Size = LenCI->getZExtValue();
-    assert(Loc.Ptr == CS.getArgument(ArgIdx) &&
-           "memset_pattern16 location pointer not argument?");
-    Mask = ArgIdx ? Ref : Mod;
+    return ArgIdx ? MRI_Ref : MRI_Mod;
   }
   // FIXME: Handle memset_pattern4 and memset_pattern8 also.
 
-  return Loc;
+  return AliasAnalysis::getArgModRefInfo(CS, ArgIdx);
 }
 
 static bool isAssumeIntrinsic(ImmutableCallSite CS) {
@@ -812,9 +774,8 @@ bool BasicAliasAnalysis::doInitialization(Module &M) {
 /// specified memory object.  Since we only look at local properties of this
 /// function, we really can't say much about this query.  We do, however, use
 /// simple "address taken" analysis on local objects.
-AliasAnalysis::ModRefResult
-BasicAliasAnalysis::getModRefInfo(ImmutableCallSite CS,
-                                  const Location &Loc) {
+ModRefInfo BasicAliasAnalysis::getModRefInfo(ImmutableCallSite CS,
+                                             const MemoryLocation &Loc) {
   assert(notDifferentParent(CS.getInstruction(), Loc.Ptr) &&
          "AliasAnalysis query involving multiple functions!");
 
@@ -828,7 +789,7 @@ BasicAliasAnalysis::getModRefInfo(ImmutableCallSite CS,
   if (isa<AllocaInst>(Object))
     if (const CallInst *CI = dyn_cast<CallInst>(CS.getInstruction()))
       if (CI->isTailCall())
-        return NoModRef;
+        return MRI_NoModRef;
 
   // If the pointer is to a locally allocated object that does not escape,
   // then the call can not mod/ref the pointer unless the call takes the pointer
@@ -850,34 +811,33 @@ BasicAliasAnalysis::getModRefInfo(ImmutableCallSite CS,
       // is impossible to alias the pointer we're checking.  If not, we have to
       // assume that the call could touch the pointer, even though it doesn't
       // escape.
-      if (!isNoAlias(Location(*CI), Location(Object))) {
+      if (!isNoAlias(MemoryLocation(*CI), MemoryLocation(Object))) {
         PassedAsArg = true;
         break;
       }
     }
 
     if (!PassedAsArg)
-      return NoModRef;
+      return MRI_NoModRef;
   }
 
   // While the assume intrinsic is marked as arbitrarily writing so that
   // proper control dependencies will be maintained, it never aliases any
   // particular memory location.
   if (isAssumeIntrinsic(CS))
-    return NoModRef;
+    return MRI_NoModRef;
 
   // The AliasAnalysis base class has some smarts, lets use them.
   return AliasAnalysis::getModRefInfo(CS, Loc);
 }
 
-AliasAnalysis::ModRefResult
-BasicAliasAnalysis::getModRefInfo(ImmutableCallSite CS1,
-                                  ImmutableCallSite CS2) {
+ModRefInfo BasicAliasAnalysis::getModRefInfo(ImmutableCallSite CS1,
+                                             ImmutableCallSite CS2) {
   // While the assume intrinsic is marked as arbitrarily writing so that
   // proper control dependencies will be maintained, it never aliases any
   // particular memory location.
   if (isAssumeIntrinsic(CS1) || isAssumeIntrinsic(CS2))
-    return NoModRef;
+    return MRI_NoModRef;
 
   // The AliasAnalysis base class has some smarts, lets use them.
   return AliasAnalysis::getModRefInfo(CS1, CS2);
@@ -885,10 +845,11 @@ BasicAliasAnalysis::getModRefInfo(ImmutableCallSite CS1,
 
 /// \brief Provide ad-hoc rules to disambiguate accesses through two GEP
 /// operators, both having the exact same pointer operand.
-static AliasAnalysis::AliasResult
-aliasSameBasePointerGEPs(const GEPOperator *GEP1, uint64_t V1Size,
-                         const GEPOperator *GEP2, uint64_t V2Size,
-                         const DataLayout &DL) {
+static AliasResult aliasSameBasePointerGEPs(const GEPOperator *GEP1,
+                                            uint64_t V1Size,
+                                            const GEPOperator *GEP2,
+                                            uint64_t V2Size,
+                                            const DataLayout &DL) {
 
   assert(GEP1->getPointerOperand() == GEP2->getPointerOperand() &&
          "Expected GEPs with the same pointer operand");
@@ -898,13 +859,13 @@ aliasSameBasePointerGEPs(const GEPOperator *GEP1, uint64_t V1Size,
   // We also need at least two indices (the pointer, and the struct field).
   if (GEP1->getNumIndices() != GEP2->getNumIndices() ||
       GEP1->getNumIndices() < 2)
-    return AliasAnalysis::MayAlias;
+    return MayAlias;
 
   // If we don't know the size of the accesses through both GEPs, we can't
   // determine whether the struct fields accessed can't alias.
-  if (V1Size == AliasAnalysis::UnknownSize ||
-      V2Size == AliasAnalysis::UnknownSize)
-    return AliasAnalysis::MayAlias;
+  if (V1Size == MemoryLocation::UnknownSize ||
+      V2Size == MemoryLocation::UnknownSize)
+    return MayAlias;
 
   ConstantInt *C1 =
       dyn_cast<ConstantInt>(GEP1->getOperand(GEP1->getNumOperands() - 1));
@@ -915,7 +876,7 @@ aliasSameBasePointerGEPs(const GEPOperator *GEP1, uint64_t V1Size,
   // If they're identical, the other indices might be also be dynamically
   // equal, so the GEPs can alias.
   if (!C1 || !C2 || C1 == C2)
-    return AliasAnalysis::MayAlias;
+    return MayAlias;
 
   // Find the last-indexed type of the GEP, i.e., the type you'd get if
   // you stripped the last index.
@@ -933,7 +894,7 @@ aliasSameBasePointerGEPs(const GEPOperator *GEP1, uint64_t V1Size,
   for (unsigned i = 1, e = GEP1->getNumIndices() - 1; i != e; ++i) {
     if (!isa<ArrayType>(GetElementPtrInst::getIndexedType(
             GEP1->getSourceElementType(), IntermediateIndices)))
-      return AliasAnalysis::MayAlias;
+      return MayAlias;
     IntermediateIndices.push_back(GEP1->getOperand(i + 1));
   }
 
@@ -942,7 +903,7 @@ aliasSameBasePointerGEPs(const GEPOperator *GEP1, uint64_t V1Size,
           GEP1->getSourceElementType(), IntermediateIndices));
 
   if (!LastIndexedStruct)
-    return AliasAnalysis::MayAlias;
+    return MayAlias;
 
   // We know that:
   // - both GEPs begin indexing from the exact same pointer;
@@ -971,9 +932,9 @@ aliasSameBasePointerGEPs(const GEPOperator *GEP1, uint64_t V1Size,
 
   if (EltsDontOverlap(V1Off, V1Size, V2Off, V2Size) ||
       EltsDontOverlap(V2Off, V2Size, V1Off, V1Size))
-    return AliasAnalysis::NoAlias;
+    return NoAlias;
 
-  return AliasAnalysis::MayAlias;
+  return MayAlias;
 }
 
 /// aliasGEP - Provide a bunch of ad-hoc rules to disambiguate a GEP instruction
@@ -981,13 +942,10 @@ aliasSameBasePointerGEPs(const GEPOperator *GEP1, uint64_t V1Size,
 /// anything about V2.  UnderlyingV1 is GetUnderlyingObject(GEP1, DL),
 /// UnderlyingV2 is the same for V2.
 ///
-AliasAnalysis::AliasResult
-BasicAliasAnalysis::aliasGEP(const GEPOperator *GEP1, uint64_t V1Size,
-                             const AAMDNodes &V1AAInfo,
-                             const Value *V2, uint64_t V2Size,
-                             const AAMDNodes &V2AAInfo,
-                             const Value *UnderlyingV1,
-                             const Value *UnderlyingV2) {
+AliasResult BasicAliasAnalysis::aliasGEP(
+    const GEPOperator *GEP1, uint64_t V1Size, const AAMDNodes &V1AAInfo,
+    const Value *V2, uint64_t V2Size, const AAMDNodes &V2AAInfo,
+    const Value *UnderlyingV1, const Value *UnderlyingV2) {
   int64_t GEP1BaseOffset;
   bool GEP1MaxLookupReached;
   SmallVector<VariableGEPIndex, 4> GEP1VariableIndices;
@@ -1017,8 +975,9 @@ BasicAliasAnalysis::aliasGEP(const GEPOperator *GEP1, uint64_t V1Size,
   // derived pointer.
   if (const GEPOperator *GEP2 = dyn_cast<GEPOperator>(V2)) {
     // Do the base pointers alias?
-    AliasResult BaseAlias = aliasCheck(UnderlyingV1, UnknownSize, AAMDNodes(),
-                                       UnderlyingV2, UnknownSize, AAMDNodes());
+    AliasResult BaseAlias =
+        aliasCheck(UnderlyingV1, MemoryLocation::UnknownSize, AAMDNodes(),
+                   UnderlyingV2, MemoryLocation::UnknownSize, AAMDNodes());
 
     // Check for geps of non-aliasing underlying pointers where the offsets are
     // identical.
@@ -1109,11 +1068,12 @@ BasicAliasAnalysis::aliasGEP(const GEPOperator *GEP1, uint64_t V1Size,
     // pointer, we know they cannot alias.
 
     // If both accesses are unknown size, we can't do anything useful here.
-    if (V1Size == UnknownSize && V2Size == UnknownSize)
+    if (V1Size == MemoryLocation::UnknownSize &&
+        V2Size == MemoryLocation::UnknownSize)
       return MayAlias;
 
-    AliasResult R = aliasCheck(UnderlyingV1, UnknownSize, AAMDNodes(),
-                               V2, V2Size, V2AAInfo);
+    AliasResult R = aliasCheck(UnderlyingV1, MemoryLocation::UnknownSize,
+                               AAMDNodes(), V2, V2Size, V2AAInfo);
     if (R != MustAlias)
       // If V2 may alias GEP base pointer, conservatively returns MayAlias.
       // If V2 is known not to alias GEP base pointer, then the two values
@@ -1153,7 +1113,7 @@ BasicAliasAnalysis::aliasGEP(const GEPOperator *GEP1, uint64_t V1Size,
   // greater, we know they do not overlap.
   if (GEP1BaseOffset != 0 && GEP1VariableIndices.empty()) {
     if (GEP1BaseOffset >= 0) {
-      if (V2Size != UnknownSize) {
+      if (V2Size != MemoryLocation::UnknownSize) {
         if ((uint64_t)GEP1BaseOffset < V2Size)
           return PartialAlias;
         return NoAlias;
@@ -1167,7 +1127,8 @@ BasicAliasAnalysis::aliasGEP(const GEPOperator *GEP1, uint64_t V1Size,
       // GEP1             V2
       // We need to know that V2Size is not unknown, otherwise we might have
       // stripped a gep with negative index ('gep <ptr>, -1, ...).
-      if (V1Size != UnknownSize && V2Size != UnknownSize) {
+      if (V1Size != MemoryLocation::UnknownSize &&
+          V2Size != MemoryLocation::UnknownSize) {
         if (-(uint64_t)GEP1BaseOffset < V1Size)
           return PartialAlias;
         return NoAlias;
@@ -1218,8 +1179,9 @@ BasicAliasAnalysis::aliasGEP(const GEPOperator *GEP1, uint64_t V1Size,
     // mod Modulo. Check whether that difference guarantees that the
     // two locations do not alias.
     uint64_t ModOffset = (uint64_t)GEP1BaseOffset & (Modulo - 1);
-    if (V1Size != UnknownSize && V2Size != UnknownSize &&
-        ModOffset >= V2Size && V1Size <= Modulo - ModOffset)
+    if (V1Size != MemoryLocation::UnknownSize &&
+        V2Size != MemoryLocation::UnknownSize && ModOffset >= V2Size &&
+        V1Size <= Modulo - ModOffset)
       return NoAlias;
 
     // If we know all the variables are positive, then GEP1 >= GEP1BasePtr.
@@ -1239,26 +1201,25 @@ BasicAliasAnalysis::aliasGEP(const GEPOperator *GEP1, uint64_t V1Size,
   return PartialAlias;
 }
 
-static AliasAnalysis::AliasResult
-MergeAliasResults(AliasAnalysis::AliasResult A, AliasAnalysis::AliasResult B) {
+static AliasResult MergeAliasResults(AliasResult A, AliasResult B) {
   // If the results agree, take it.
   if (A == B)
     return A;
   // A mix of PartialAlias and MustAlias is PartialAlias.
-  if ((A == AliasAnalysis::PartialAlias && B == AliasAnalysis::MustAlias) ||
-      (B == AliasAnalysis::PartialAlias && A == AliasAnalysis::MustAlias))
-    return AliasAnalysis::PartialAlias;
+  if ((A == PartialAlias && B == MustAlias) ||
+      (B == PartialAlias && A == MustAlias))
+    return PartialAlias;
   // Otherwise, we don't know anything.
-  return AliasAnalysis::MayAlias;
+  return MayAlias;
 }
 
 /// aliasSelect - Provide a bunch of ad-hoc rules to disambiguate a Select
 /// instruction against another.
-AliasAnalysis::AliasResult
-BasicAliasAnalysis::aliasSelect(const SelectInst *SI, uint64_t SISize,
-                                const AAMDNodes &SIAAInfo,
-                                const Value *V2, uint64_t V2Size,
-                                const AAMDNodes &V2AAInfo) {
+AliasResult BasicAliasAnalysis::aliasSelect(const SelectInst *SI,
+                                            uint64_t SISize,
+                                            const AAMDNodes &SIAAInfo,
+                                            const Value *V2, uint64_t V2Size,
+                                            const AAMDNodes &V2AAInfo) {
   // If the values are Selects with the same condition, we can do a more precise
   // check: just check for aliases between the values on corresponding arms.
   if (const SelectInst *SI2 = dyn_cast<SelectInst>(V2))
@@ -1288,11 +1249,10 @@ BasicAliasAnalysis::aliasSelect(const SelectInst *SI, uint64_t SISize,
 
 // aliasPHI - Provide a bunch of ad-hoc rules to disambiguate a PHI instruction
 // against another.
-AliasAnalysis::AliasResult
-BasicAliasAnalysis::aliasPHI(const PHINode *PN, uint64_t PNSize,
-                             const AAMDNodes &PNAAInfo,
-                             const Value *V2, uint64_t V2Size,
-                             const AAMDNodes &V2AAInfo) {
+AliasResult BasicAliasAnalysis::aliasPHI(const PHINode *PN, uint64_t PNSize,
+                                         const AAMDNodes &PNAAInfo,
+                                         const Value *V2, uint64_t V2Size,
+                                         const AAMDNodes &V2AAInfo) {
   // Track phi nodes we have visited. We use this information when we determine
   // value equivalence.
   VisitedPhiBBs.insert(PN->getParent());
@@ -1302,8 +1262,8 @@ BasicAliasAnalysis::aliasPHI(const PHINode *PN, uint64_t PNSize,
   // on corresponding edges.
   if (const PHINode *PN2 = dyn_cast<PHINode>(V2))
     if (PN2->getParent() == PN->getParent()) {
-      LocPair Locs(Location(PN, PNSize, PNAAInfo),
-                   Location(V2, V2Size, V2AAInfo));
+      LocPair Locs(MemoryLocation(PN, PNSize, PNAAInfo),
+                   MemoryLocation(V2, V2Size, V2AAInfo));
       if (PN > V2)
         std::swap(Locs.first, Locs.second);
       // Analyse the PHIs' inputs under the assumption that the PHIs are
@@ -1338,20 +1298,41 @@ BasicAliasAnalysis::aliasPHI(const PHINode *PN, uint64_t PNSize,
 
   SmallPtrSet<Value*, 4> UniqueSrc;
   SmallVector<Value*, 4> V1Srcs;
-  for (unsigned i = 0, e = PN->getNumIncomingValues(); i != e; ++i) {
-    Value *PV1 = PN->getIncomingValue(i);
+  bool isRecursive = false;
+  for (Value *PV1 : PN->incoming_values()) {
     if (isa<PHINode>(PV1))
       // If any of the source itself is a PHI, return MayAlias conservatively
       // to avoid compile time explosion. The worst possible case is if both
       // sides are PHI nodes. In which case, this is O(m x n) time where 'm'
       // and 'n' are the number of PHI sources.
       return MayAlias;
+
+    if (EnableRecPhiAnalysis)
+      if (GEPOperator *PV1GEP = dyn_cast<GEPOperator>(PV1)) {
+        // Check whether the incoming value is a GEP that advances the pointer
+        // result of this PHI node (e.g. in a loop). If this is the case, we
+        // would recurse and always get a MayAlias. Handle this case specially
+        // below.
+        if (PV1GEP->getPointerOperand() == PN && PV1GEP->getNumIndices() == 1 &&
+            isa<ConstantInt>(PV1GEP->idx_begin())) {
+          isRecursive = true;
+          continue;
+        }
+      }
+
     if (UniqueSrc.insert(PV1).second)
       V1Srcs.push_back(PV1);
   }
 
+  // If this PHI node is recursive, set the size of the accessed memory to
+  // unknown to represent all the possible values the GEP could advance the
+  // pointer to.
+  if (isRecursive)
+    PNSize = MemoryLocation::UnknownSize;
+
   AliasResult Alias = aliasCheck(V2, V2Size, V2AAInfo,
                                  V1Srcs[0], PNSize, PNAAInfo);
+
   // Early exit if the check of the first PHI source against V2 is MayAlias.
   // Other results are not possible.
   if (Alias == MayAlias)
@@ -1375,11 +1356,10 @@ BasicAliasAnalysis::aliasPHI(const PHINode *PN, uint64_t PNSize,
 // aliasCheck - Provide a bunch of ad-hoc rules to disambiguate in common cases,
 // such as array references.
 //
-AliasAnalysis::AliasResult
-BasicAliasAnalysis::aliasCheck(const Value *V1, uint64_t V1Size,
-                               AAMDNodes V1AAInfo,
-                               const Value *V2, uint64_t V2Size,
-                               AAMDNodes V2AAInfo) {
+AliasResult BasicAliasAnalysis::aliasCheck(const Value *V1, uint64_t V1Size,
+                                           AAMDNodes V1AAInfo, const Value *V2,
+                                           uint64_t V2Size,
+                                           AAMDNodes V2AAInfo) {
   // If either of the memory references is empty, it doesn't matter what the
   // pointer values are.
   if (V1Size == 0 || V2Size == 0)
@@ -1389,6 +1369,11 @@ BasicAliasAnalysis::aliasCheck(const Value *V1, uint64_t V1Size,
   V1 = V1->stripPointerCasts();
   V2 = V2->stripPointerCasts();
 
+  // If V1 or V2 is undef, the result is NoAlias because we can always pick a
+  // value for undef that aliases nothing in the program.
+  if (isa<UndefValue>(V1) || isa<UndefValue>(V2))
+    return NoAlias;
+
   // Are we checking for alias of the same value?
   // Because we look 'through' phi nodes we could look at "Value" pointers from
   // different iterations. We must therefore make sure that this is not the
@@ -1453,14 +1438,16 @@ BasicAliasAnalysis::aliasCheck(const Value *V1, uint64_t V1Size,
   // If the size of one access is larger than the entire object on the other
   // side, then we know such behavior is undefined and can assume no alias.
   if (DL)
-    if ((V1Size != UnknownSize && isObjectSmallerThan(O2, V1Size, *DL, *TLI)) ||
-        (V2Size != UnknownSize && isObjectSmallerThan(O1, V2Size, *DL, *TLI)))
+    if ((V1Size != MemoryLocation::UnknownSize &&
+         isObjectSmallerThan(O2, V1Size, *DL, *TLI)) ||
+        (V2Size != MemoryLocation::UnknownSize &&
+         isObjectSmallerThan(O1, V2Size, *DL, *TLI)))
       return NoAlias;
 
   // Check the cache before climbing up use-def chains. This also terminates
   // otherwise infinitely recursive queries.
-  LocPair Locs(Location(V1, V1Size, V1AAInfo),
-               Location(V2, V2Size, V2AAInfo));
+  LocPair Locs(MemoryLocation(V1, V1Size, V1AAInfo),
+               MemoryLocation(V2, V2Size, V2AAInfo));
   if (V1 > V2)
     std::swap(Locs.first, Locs.second);
   std::pair<AliasCacheTy::iterator, bool> Pair =
@@ -1507,13 +1494,15 @@ BasicAliasAnalysis::aliasCheck(const Value *V1, uint64_t V1Size,
   // accesses is accessing the entire object, then the accesses must
   // overlap in some way.
   if (DL && O1 == O2)
-    if ((V1Size != UnknownSize && isObjectSize(O1, V1Size, *DL, *TLI)) ||
-        (V2Size != UnknownSize && isObjectSize(O2, V2Size, *DL, *TLI)))
+    if ((V1Size != MemoryLocation::UnknownSize &&
+         isObjectSize(O1, V1Size, *DL, *TLI)) ||
+        (V2Size != MemoryLocation::UnknownSize &&
+         isObjectSize(O2, V2Size, *DL, *TLI)))
       return AliasCache[Locs] = PartialAlias;
 
   AliasResult Result =
-    AliasAnalysis::alias(Location(V1, V1Size, V1AAInfo),
-                         Location(V2, V2Size, V2AAInfo));
+      AliasAnalysis::alias(MemoryLocation(V1, V1Size, V1AAInfo),
+                           MemoryLocation(V2, V2Size, V2AAInfo));
   return AliasCache[Locs] = Result;
 }
 
diff --git a/lib/Analysis/BlockFrequencyInfo.cpp b/lib/Analysis/BlockFrequencyInfo.cpp
index 3d819eb596d..edd5ad9d5b7 100644
--- a/lib/Analysis/BlockFrequencyInfo.cpp
+++ b/lib/Analysis/BlockFrequencyInfo.cpp
@@ -105,45 +105,24 @@ struct DOTGraphTraits<BlockFrequencyInfo*> : public DefaultDOTGraphTraits {
 } // end namespace llvm
 #endif
 
-INITIALIZE_PASS_BEGIN(BlockFrequencyInfo, "block-freq",
-                      "Block Frequency Analysis", true, true)
-INITIALIZE_PASS_DEPENDENCY(BranchProbabilityInfo)
-INITIALIZE_PASS_DEPENDENCY(LoopInfoWrapperPass)
-INITIALIZE_PASS_END(BlockFrequencyInfo, "block-freq",
-                    "Block Frequency Analysis", true, true)
+BlockFrequencyInfo::BlockFrequencyInfo() {}
 
-char BlockFrequencyInfo::ID = 0;
-
-
-BlockFrequencyInfo::BlockFrequencyInfo() : FunctionPass(ID) {
-  initializeBlockFrequencyInfoPass(*PassRegistry::getPassRegistry());
+BlockFrequencyInfo::BlockFrequencyInfo(const Function &F,
+                                       const BranchProbabilityInfo &BPI,
+                                       const LoopInfo &LI) {
+  calculate(F, BPI, LI);
 }
 
-BlockFrequencyInfo::~BlockFrequencyInfo() {}
-
-void BlockFrequencyInfo::getAnalysisUsage(AnalysisUsage &AU) const {
-  AU.addRequired<BranchProbabilityInfo>();
-  AU.addRequired<LoopInfoWrapperPass>();
-  AU.setPreservesAll();
-}
-
-bool BlockFrequencyInfo::runOnFunction(Function &F) {
-  BranchProbabilityInfo &BPI = getAnalysis<BranchProbabilityInfo>();
-  LoopInfo &LI = getAnalysis<LoopInfoWrapperPass>().getLoopInfo();
+void BlockFrequencyInfo::calculate(const Function &F,
+                                   const BranchProbabilityInfo &BPI,
+                                   const LoopInfo &LI) {
   if (!BFI)
     BFI.reset(new ImplType);
-  BFI->doFunction(&F, &BPI, &LI);
+  BFI->calculate(F, BPI, LI);
 #ifndef NDEBUG
   if (ViewBlockFreqPropagationDAG != GVDT_None)
     view();
 #endif
-  return false;
-}
-
-void BlockFrequencyInfo::releaseMemory() { BFI.reset(); }
-
-void BlockFrequencyInfo::print(raw_ostream &O, const Module *) const {
-  if (BFI) BFI->print(O);
 }
 
 BlockFrequency BlockFrequencyInfo::getBlockFreq(const BasicBlock *BB) const {
@@ -180,3 +159,49 @@ BlockFrequencyInfo::printBlockFreq(raw_ostream &OS,
 uint64_t BlockFrequencyInfo::getEntryFreq() const {
   return BFI ? BFI->getEntryFreq() : 0;
 }
+
+void BlockFrequencyInfo::releaseMemory() { BFI.reset(); }
+
+void BlockFrequencyInfo::print(raw_ostream &OS) const {
+  if (BFI)
+    BFI->print(OS);
+}
+
+
+INITIALIZE_PASS_BEGIN(BlockFrequencyInfoWrapperPass, "block-freq",
+                      "Block Frequency Analysis", true, true)
+INITIALIZE_PASS_DEPENDENCY(BranchProbabilityInfoWrapperPass)
+INITIALIZE_PASS_DEPENDENCY(LoopInfoWrapperPass)
+INITIALIZE_PASS_END(BlockFrequencyInfoWrapperPass, "block-freq",
+                    "Block Frequency Analysis", true, true)
+
+char BlockFrequencyInfoWrapperPass::ID = 0;
+
+
+BlockFrequencyInfoWrapperPass::BlockFrequencyInfoWrapperPass()
+    : FunctionPass(ID) {
+  initializeBlockFrequencyInfoWrapperPassPass(*PassRegistry::getPassRegistry());
+}
+
+BlockFrequencyInfoWrapperPass::~BlockFrequencyInfoWrapperPass() {}
+
+void BlockFrequencyInfoWrapperPass::print(raw_ostream &OS,
+                                          const Module *) const {
+  BFI.print(OS);
+}
+
+void BlockFrequencyInfoWrapperPass::getAnalysisUsage(AnalysisUsage &AU) const {
+  AU.addRequired<BranchProbabilityInfoWrapperPass>();
+  AU.addRequired<LoopInfoWrapperPass>();
+  AU.setPreservesAll();
+}
+
+void BlockFrequencyInfoWrapperPass::releaseMemory() { BFI.releaseMemory(); }
+
+bool BlockFrequencyInfoWrapperPass::runOnFunction(Function &F) {
+  BranchProbabilityInfo &BPI =
+      getAnalysis<BranchProbabilityInfoWrapperPass>().getBPI();
+  LoopInfo &LI = getAnalysis<LoopInfoWrapperPass>().getLoopInfo();
+  BFI.calculate(F, BPI, LI);
+  return false;
+}
diff --git a/lib/Analysis/BlockFrequencyInfoImpl.cpp b/lib/Analysis/BlockFrequencyInfoImpl.cpp
index 456cee179f0..6ceda06aac1 100644
--- a/lib/Analysis/BlockFrequencyInfoImpl.cpp
+++ b/lib/Analysis/BlockFrequencyInfoImpl.cpp
@@ -286,7 +286,7 @@ bool BlockFrequencyInfoImplBase::addToDist(Distribution &Dist,
 
   if (isLoopHeader(Resolved)) {
     DEBUG(debugSuccessor("backedge"));
-    Dist.addBackedge(OuterLoop->getHeader(), Weight);
+    Dist.addBackedge(Resolved, Weight);
     return true;
   }
 
@@ -349,7 +349,10 @@ void BlockFrequencyInfoImplBase::computeLoopScale(LoopData &Loop) {
 
   // LoopScale == 1 / ExitMass
   // ExitMass == HeadMass - BackedgeMass
-  BlockMass ExitMass = BlockMass::getFull() - Loop.BackedgeMass;
+  BlockMass TotalBackedgeMass;
+  for (auto &Mass : Loop.BackedgeMass)
+    TotalBackedgeMass += Mass;
+  BlockMass ExitMass = BlockMass::getFull() - TotalBackedgeMass;
 
   // Block scale stores the inverse of the scale. If this is an infinite loop,
   // its exit mass will be zero. In this case, use an arbitrary scale for the
@@ -358,7 +361,7 @@ void BlockFrequencyInfoImplBase::computeLoopScale(LoopData &Loop) {
       ExitMass.isEmpty() ? InifiniteLoopScale : ExitMass.toScaled().inverse();
 
   DEBUG(dbgs() << " - exit-mass = " << ExitMass << " (" << BlockMass::getFull()
-               << " - " << Loop.BackedgeMass << ")\n"
+               << " - " << TotalBackedgeMass << ")\n"
                << " - scale = " << Loop.Scale << "\n");
 }
 
@@ -375,6 +378,19 @@ void BlockFrequencyInfoImplBase::packageLoop(LoopData &Loop) {
   Loop.IsPackaged = true;
 }
 
+#ifndef NDEBUG
+static void debugAssign(const BlockFrequencyInfoImplBase &BFI,
+                        const DitheringDistributer &D, const BlockNode &T,
+                        const BlockMass &M, const char *Desc) {
+  dbgs() << "  => assign " << M << " (" << D.RemMass << ")";
+  if (Desc)
+    dbgs() << " [" << Desc << "]";
+  if (T.isValid())
+    dbgs() << " to " << BFI.getBlockName(T);
+  dbgs() << "\n";
+}
+#endif
+
 void BlockFrequencyInfoImplBase::distributeMass(const BlockNode &Source,
                                                 LoopData *OuterLoop,
                                                 Distribution &Dist) {
@@ -384,25 +400,12 @@ void BlockFrequencyInfoImplBase::distributeMass(const BlockNode &Source,
   // Distribute mass to successors as laid out in Dist.
   DitheringDistributer D(Dist, Mass);
 
-#ifndef NDEBUG
-  auto debugAssign = [&](const BlockNode &T, const BlockMass &M,
-                         const char *Desc) {
-    dbgs() << "  => assign " << M << " (" << D.RemMass << ")";
-    if (Desc)
-      dbgs() << " [" << Desc << "]";
-    if (T.isValid())
-      dbgs() << " to " << getBlockName(T);
-    dbgs() << "\n";
-  };
-  (void)debugAssign;
-#endif
-
   for (const Weight &W : Dist.Weights) {
     // Check for a local edge (non-backedge and non-exit).
     BlockMass Taken = D.takeMass(W.Amount);
     if (W.Type == Weight::Local) {
       Working[W.TargetNode.Index].getMass() += Taken;
-      DEBUG(debugAssign(W.TargetNode, Taken, nullptr));
+      DEBUG(debugAssign(*this, D, W.TargetNode, Taken, nullptr));
       continue;
     }
 
@@ -411,15 +414,15 @@ void BlockFrequencyInfoImplBase::distributeMass(const BlockNode &Source,
 
     // Check for a backedge.
     if (W.Type == Weight::Backedge) {
-      OuterLoop->BackedgeMass += Taken;
-      DEBUG(debugAssign(BlockNode(), Taken, "back"));
+      OuterLoop->BackedgeMass[OuterLoop->getHeaderIndex(W.TargetNode)] += Taken;
+      DEBUG(debugAssign(*this, D, W.TargetNode, Taken, "back"));
       continue;
     }
 
     // This must be an exit.
     assert(W.Type == Weight::Exit);
     OuterLoop->Exits.push_back(std::make_pair(W.TargetNode, Taken));
-    DEBUG(debugAssign(W.TargetNode, Taken, "exit"));
+    DEBUG(debugAssign(*this, D, W.TargetNode, Taken, "exit"));
   }
 }
 
@@ -713,10 +716,44 @@ BlockFrequencyInfoImplBase::analyzeIrreducible(
 void
 BlockFrequencyInfoImplBase::updateLoopWithIrreducible(LoopData &OuterLoop) {
   OuterLoop.Exits.clear();
-  OuterLoop.BackedgeMass = BlockMass::getEmpty();
+  for (auto &Mass : OuterLoop.BackedgeMass)
+    Mass = BlockMass::getEmpty();
   auto O = OuterLoop.Nodes.begin() + 1;
   for (auto I = O, E = OuterLoop.Nodes.end(); I != E; ++I)
     if (!Working[I->Index].isPackaged())
       *O++ = *I;
   OuterLoop.Nodes.erase(O, OuterLoop.Nodes.end());
 }
+
+void BlockFrequencyInfoImplBase::adjustLoopHeaderMass(LoopData &Loop) {
+  assert(Loop.isIrreducible() && "this only makes sense on irreducible loops");
+
+  // Since the loop has more than one header block, the mass flowing back into
+  // each header will be different. Adjust the mass in each header loop to
+  // reflect the masses flowing through back edges.
+  //
+  // To do this, we distribute the initial mass using the backedge masses
+  // as weights for the distribution.
+  BlockMass LoopMass = BlockMass::getFull();
+  Distribution Dist;
+
+  DEBUG(dbgs() << "adjust-loop-header-mass:\n");
+  for (uint32_t H = 0; H < Loop.NumHeaders; ++H) {
+    auto &HeaderNode = Loop.Nodes[H];
+    auto &BackedgeMass = Loop.BackedgeMass[Loop.getHeaderIndex(HeaderNode)];
+    DEBUG(dbgs() << " - Add back edge mass for node "
+                 << getBlockName(HeaderNode) << ": " << BackedgeMass << "\n");
+    Dist.addLocal(HeaderNode, BackedgeMass.getMass());
+  }
+
+  DitheringDistributer D(Dist, LoopMass);
+
+  DEBUG(dbgs() << " Distribute loop mass " << LoopMass
+               << " to headers using above weights\n");
+  for (const Weight &W : Dist.Weights) {
+    BlockMass Taken = D.takeMass(W.Amount);
+    assert(W.Type == Weight::Local && "all weights should be local");
+    Working[W.TargetNode.Index].getMass() = Taken;
+    DEBUG(debugAssign(*this, D, W.TargetNode, Taken, nullptr));
+  }
+}
diff --git a/lib/Analysis/BranchProbabilityInfo.cpp b/lib/Analysis/BranchProbabilityInfo.cpp
index 8799a710af0..b813dca9369 100644
--- a/lib/Analysis/BranchProbabilityInfo.cpp
+++ b/lib/Analysis/BranchProbabilityInfo.cpp
@@ -27,13 +27,13 @@ using namespace llvm;
 
 #define DEBUG_TYPE "branch-prob"
 
-INITIALIZE_PASS_BEGIN(BranchProbabilityInfo, "branch-prob",
+INITIALIZE_PASS_BEGIN(BranchProbabilityInfoWrapperPass, "branch-prob",
                       "Branch Probability Analysis", false, true)
 INITIALIZE_PASS_DEPENDENCY(LoopInfoWrapperPass)
-INITIALIZE_PASS_END(BranchProbabilityInfo, "branch-prob",
+INITIALIZE_PASS_END(BranchProbabilityInfoWrapperPass, "branch-prob",
                     "Branch Probability Analysis", false, true)
 
-char BranchProbabilityInfo::ID = 0;
+char BranchProbabilityInfoWrapperPass::ID = 0;
 
 // Weights are for internal use only. They are used by heuristics to help to
 // estimate edges' probability. Example:
@@ -115,11 +115,6 @@ static const uint32_t NORMAL_WEIGHT = 16;
 // Minimum weight of an edge. Please note, that weight is NEVER 0.
 static const uint32_t MIN_WEIGHT = 1;
 
-static uint32_t getMaxWeightFor(BasicBlock *BB) {
-  return UINT32_MAX / BB->getTerminator()->getNumSuccessors();
-}
-
-
 /// \brief Calculate edge weights for successors lead to unreachable.
 ///
 /// Predict that a successor which leads necessarily to an
@@ -185,15 +180,18 @@ bool BranchProbabilityInfo::calcMetadataWeights(BasicBlock *BB) {
   if (!WeightsNode)
     return false;
 
+  // Check that the number of successors is manageable.
+  assert(TI->getNumSuccessors() < UINT32_MAX && "Too many successors");
+
   // Ensure there are weights for all of the successors. Note that the first
   // operand to the metadata node is a name, not a weight.
   if (WeightsNode->getNumOperands() != TI->getNumSuccessors() + 1)
     return false;
 
-  // Build up the final weights that will be used in a temporary buffer, but
-  // don't add them until all weihts are present. Each weight value is clamped
-  // to [1, getMaxWeightFor(BB)].
-  uint32_t WeightLimit = getMaxWeightFor(BB);
+  // Build up the final weights that will be used in a temporary buffer.
+  // Compute the sum of all weights to later decide whether they need to
+  // be scaled to fit in 32 bits.
+  uint64_t WeightSum = 0;
   SmallVector<uint32_t, 2> Weights;
   Weights.reserve(TI->getNumSuccessors());
   for (unsigned i = 1, e = WeightsNode->getNumOperands(); i != e; ++i) {
@@ -201,12 +199,26 @@ bool BranchProbabilityInfo::calcMetadataWeights(BasicBlock *BB) {
         mdconst::dyn_extract<ConstantInt>(WeightsNode->getOperand(i));
     if (!Weight)
       return false;
-    Weights.push_back(
-      std::max<uint32_t>(1, Weight->getLimitedValue(WeightLimit)));
+    assert(Weight->getValue().getActiveBits() <= 32 &&
+           "Too many bits for uint32_t");
+    Weights.push_back(Weight->getZExtValue());
+    WeightSum += Weights.back();
   }
   assert(Weights.size() == TI->getNumSuccessors() && "Checked above");
-  for (unsigned i = 0, e = TI->getNumSuccessors(); i != e; ++i)
-    setEdgeWeight(BB, i, Weights[i]);
+
+  // If the sum of weights does not fit in 32 bits, scale every weight down
+  // accordingly.
+  uint64_t ScalingFactor =
+      (WeightSum > UINT32_MAX) ? WeightSum / UINT32_MAX + 1 : 1;
+
+  WeightSum = 0;
+  for (unsigned i = 0, e = TI->getNumSuccessors(); i != e; ++i) {
+    uint32_t W = Weights[i] / ScalingFactor;
+    WeightSum += W;
+    setEdgeWeight(BB, i, W);
+  }
+  assert(WeightSum <= UINT32_MAX &&
+         "Expected weights to scale down to 32 bits");
 
   return true;
 }
@@ -307,8 +319,9 @@ bool BranchProbabilityInfo::calcPointerHeuristics(BasicBlock *BB) {
 
 // Calculate Edge Weights using "Loop Branch Heuristics". Predict backedges
 // as taken, exiting edges as not-taken.
-bool BranchProbabilityInfo::calcLoopBranchHeuristics(BasicBlock *BB) {
-  Loop *L = LI->getLoopFor(BB);
+bool BranchProbabilityInfo::calcLoopBranchHeuristics(BasicBlock *BB,
+                                                     const LoopInfo &LI) {
+  Loop *L = LI.getLoopFor(BB);
   if (!L)
     return false;
 
@@ -492,46 +505,11 @@ bool BranchProbabilityInfo::calcInvokeHeuristics(BasicBlock *BB) {
   return true;
 }
 
-void BranchProbabilityInfo::getAnalysisUsage(AnalysisUsage &AU) const {
-  AU.addRequired<LoopInfoWrapperPass>();
-  AU.setPreservesAll();
+void BranchProbabilityInfo::releaseMemory() {
+  Weights.clear();
 }
 
-bool BranchProbabilityInfo::runOnFunction(Function &F) {
-  DEBUG(dbgs() << "---- Branch Probability Info : " << F.getName()
-               << " ----\n\n");
-  LastF = &F; // Store the last function we ran on for printing.
-  LI = &getAnalysis<LoopInfoWrapperPass>().getLoopInfo();
-  assert(PostDominatedByUnreachable.empty());
-  assert(PostDominatedByColdCall.empty());
-
-  // Walk the basic blocks in post-order so that we can build up state about
-  // the successors of a block iteratively.
-  for (auto BB : post_order(&F.getEntryBlock())) {
-    DEBUG(dbgs() << "Computing probabilities for " << BB->getName() << "\n");
-    if (calcUnreachableHeuristics(BB))
-      continue;
-    if (calcMetadataWeights(BB))
-      continue;
-    if (calcColdCallHeuristics(BB))
-      continue;
-    if (calcLoopBranchHeuristics(BB))
-      continue;
-    if (calcPointerHeuristics(BB))
-      continue;
-    if (calcZeroHeuristics(BB))
-      continue;
-    if (calcFloatingPointHeuristics(BB))
-      continue;
-    calcInvokeHeuristics(BB);
-  }
-
-  PostDominatedByUnreachable.clear();
-  PostDominatedByColdCall.clear();
-  return false;
-}
-
-void BranchProbabilityInfo::print(raw_ostream &OS, const Module *) const {
+void BranchProbabilityInfo::print(raw_ostream &OS) const {
   OS << "---- Branch Probabilities ----\n";
   // We print the probabilities from the last function the analysis ran over,
   // or the function it is currently running over.
@@ -553,7 +531,7 @@ uint32_t BranchProbabilityInfo::getSumForBlock(const BasicBlock *BB) const {
     uint32_t PrevSum = Sum;
 
     Sum += Weight;
-    assert(Sum > PrevSum); (void) PrevSum;
+    assert(Sum >= PrevSum); (void) PrevSum;
   }
 
   return Sum;
@@ -616,14 +594,17 @@ uint32_t BranchProbabilityInfo::getEdgeWeight(const BasicBlock *Src,
 uint32_t BranchProbabilityInfo::
 getEdgeWeight(const BasicBlock *Src, const BasicBlock *Dst) const {
   uint32_t Weight = 0;
+  bool FoundWeight = false;
   DenseMap<Edge, uint32_t>::const_iterator MapI;
   for (succ_const_iterator I = succ_begin(Src), E = succ_end(Src); I != E; ++I)
     if (*I == Dst) {
       MapI = Weights.find(std::make_pair(Src, I.getSuccessorIndex()));
-      if (MapI != Weights.end())
+      if (MapI != Weights.end()) {
+        FoundWeight = true;
         Weight += MapI->second;
+      }
     }
-  return (Weight == 0) ? DEFAULT_WEIGHT : Weight;
+  return (!FoundWeight) ? DEFAULT_WEIGHT : Weight;
 }
 
 /// Set the edge weight for a given edge specified by PredBlock and an index
@@ -669,3 +650,54 @@ BranchProbabilityInfo::printEdgeProbability(raw_ostream &OS,
 
   return OS;
 }
+
+void BranchProbabilityInfo::calculate(Function &F, const LoopInfo& LI) {
+  DEBUG(dbgs() << "---- Branch Probability Info : " << F.getName()
+               << " ----\n\n");
+  LastF = &F; // Store the last function we ran on for printing.
+  assert(PostDominatedByUnreachable.empty());
+  assert(PostDominatedByColdCall.empty());
+
+  // Walk the basic blocks in post-order so that we can build up state about
+  // the successors of a block iteratively.
+  for (auto BB : post_order(&F.getEntryBlock())) {
+    DEBUG(dbgs() << "Computing probabilities for " << BB->getName() << "\n");
+    if (calcUnreachableHeuristics(BB))
+      continue;
+    if (calcMetadataWeights(BB))
+      continue;
+    if (calcColdCallHeuristics(BB))
+      continue;
+    if (calcLoopBranchHeuristics(BB, LI))
+      continue;
+    if (calcPointerHeuristics(BB))
+      continue;
+    if (calcZeroHeuristics(BB))
+      continue;
+    if (calcFloatingPointHeuristics(BB))
+      continue;
+    calcInvokeHeuristics(BB);
+  }
+
+  PostDominatedByUnreachable.clear();
+  PostDominatedByColdCall.clear();
+}
+
+void BranchProbabilityInfoWrapperPass::getAnalysisUsage(
+    AnalysisUsage &AU) const {
+  AU.addRequired<LoopInfoWrapperPass>();
+  AU.setPreservesAll();
+}
+
+bool BranchProbabilityInfoWrapperPass::runOnFunction(Function &F) {
+  const LoopInfo &LI = getAnalysis<LoopInfoWrapperPass>().getLoopInfo();
+  BPI.calculate(F, LI);
+  return false;
+}
+
+void BranchProbabilityInfoWrapperPass::releaseMemory() { BPI.releaseMemory(); }
+
+void BranchProbabilityInfoWrapperPass::print(raw_ostream &OS,
+                                             const Module *) const {
+  BPI.print(OS);
+}
diff --git a/lib/Analysis/CFG.cpp b/lib/Analysis/CFG.cpp
index 8ecd70b5d71..e15109bd270 100644
--- a/lib/Analysis/CFG.cpp
+++ b/lib/Analysis/CFG.cpp
@@ -126,10 +126,9 @@ static bool loopContainsBoth(const LoopInfo *LI,
   return L1 != nullptr && L1 == L2;
 }
 
-static bool isPotentiallyReachableInner(SmallVectorImpl<BasicBlock *> &Worklist,
-                                        BasicBlock *StopBB,
-                                        const DominatorTree *DT,
-                                        const LoopInfo *LI) {
+bool llvm::isPotentiallyReachableFromMany(
+    SmallVectorImpl<BasicBlock *> &Worklist, BasicBlock *StopBB,
+    const DominatorTree *DT, const LoopInfo *LI) {
   // When the stop block is unreachable, it's dominated from everywhere,
   // regardless of whether there's a path between the two blocks.
   if (DT && !DT->isReachableFromEntry(StopBB))
@@ -179,8 +178,8 @@ bool llvm::isPotentiallyReachable(const BasicBlock *A, const BasicBlock *B,
   SmallVector<BasicBlock*, 32> Worklist;
   Worklist.push_back(const_cast<BasicBlock*>(A));
 
-  return isPotentiallyReachableInner(Worklist, const_cast<BasicBlock*>(B),
-                                     DT, LI);
+  return isPotentiallyReachableFromMany(Worklist, const_cast<BasicBlock *>(B),
+                                        DT, LI);
 }
 
 bool llvm::isPotentiallyReachable(const Instruction *A, const Instruction *B,
@@ -230,7 +229,6 @@ bool llvm::isPotentiallyReachable(const Instruction *A, const Instruction *B,
   if (B->getParent() == &A->getParent()->getParent()->getEntryBlock())
     return false;
 
-  return isPotentiallyReachableInner(Worklist,
-                                     const_cast<BasicBlock*>(B->getParent()),
-                                     DT, LI);
+  return isPotentiallyReachableFromMany(
+      Worklist, const_cast<BasicBlock *>(B->getParent()), DT, LI);
 }
diff --git a/lib/Analysis/CFLAliasAnalysis.cpp b/lib/Analysis/CFLAliasAnalysis.cpp
index 3147992b9b8..fe1c088886b 100644
--- a/lib/Analysis/CFLAliasAnalysis.cpp
+++ b/lib/Analysis/CFLAliasAnalysis.cpp
@@ -14,8 +14,7 @@
 // Alias Analysis" by Zhang Q, Lyu M R, Yuan H, and Su Z. -- to summarize the
 // papers, we build a graph of the uses of a variable, where each node is a
 // memory location, and each edge is an action that happened on that memory
-// location.  The "actions" can be one of Dereference, Reference, Assign, or
-// Assign.
+// location.  The "actions" can be one of Dereference, Reference, or Assign.
 //
 // Two variables are considered as aliasing iff you can reach one value's node
 // from the other value's node and the language formed by concatenating all of
@@ -219,9 +218,10 @@ struct CFLAliasAnalysis : public ImmutablePass, public AliasAnalysis {
     return Iter->second;
   }
 
-  AliasResult query(const Location &LocA, const Location &LocB);
+  AliasResult query(const MemoryLocation &LocA, const MemoryLocation &LocB);
 
-  AliasResult alias(const Location &LocA, const Location &LocB) override {
+  AliasResult alias(const MemoryLocation &LocA,
+                    const MemoryLocation &LocB) override {
     if (LocA.Ptr == LocB.Ptr) {
       if (LocA.Size == LocB.Size) {
         return MustAlias;
@@ -305,8 +305,7 @@ class GetEdgesVisitor : public InstVisitor<GetEdgesVisitor, void> {
   }
 
   void visitPHINode(PHINode &Inst) {
-    for (unsigned I = 0, E = Inst.getNumIncomingValues(); I != E; ++I) {
-      Value *Val = Inst.getIncomingValue(I);
+    for (Value *Val : Inst.incoming_values()) {
       Output.push_back(Edge(&Inst, Val, EdgeType::Assign, AttrNone));
     }
   }
@@ -540,6 +539,19 @@ class GetEdgesVisitor : public InstVisitor<GetEdgesVisitor, void> {
     Output.push_back(Edge(&Inst, From1, EdgeType::Assign, AttrNone));
     Output.push_back(Edge(&Inst, From2, EdgeType::Assign, AttrNone));
   }
+
+  void visitConstantExpr(ConstantExpr *CE) {
+    switch (CE->getOpcode()) {
+    default:
+      llvm_unreachable("Unknown instruction type encountered!");
+// Build the switch statement using the Instruction.def file.
+#define HANDLE_INST(NUM, OPCODE, CLASS)                                        \
+  case Instruction::OPCODE:                                                    \
+    visit##OPCODE(*(CLASS *)CE);                                               \
+    break;
+#include "llvm/IR/Instruction.def"
+    }
+  }
 };
 
 // For a given instruction, we need to know which Value* to get the
@@ -742,6 +754,10 @@ static EdgeType flipWeight(EdgeType);
 static void argsToEdges(CFLAliasAnalysis &, Instruction *,
                         SmallVectorImpl<Edge> &);
 
+// Gets edges of the given ConstantExpr*, writing them to the SmallVector*.
+static void argsToEdges(CFLAliasAnalysis &, ConstantExpr *,
+                        SmallVectorImpl<Edge> &);
+
 // Gets the "Level" that one should travel in StratifiedSets
 // given an EdgeType.
 static Level directionOfEdgeType(EdgeType);
@@ -808,6 +824,13 @@ static bool hasUsefulEdges(Instruction *Inst) {
   return !isa<CmpInst>(Inst) && !isa<FenceInst>(Inst) && !IsNonInvokeTerminator;
 }
 
+static bool hasUsefulEdges(ConstantExpr *CE) {
+  // ConstantExpr doens't have terminators, invokes, or fences, so only needs
+  // to check for compares.
+  return CE->getOpcode() != Instruction::ICmp &&
+         CE->getOpcode() != Instruction::FCmp;
+}
+
 static Optional<StratifiedAttr> valueToAttrIndex(Value *Val) {
   if (isa<GlobalValue>(Val))
     return AttrGlobalIndex;
@@ -847,6 +870,13 @@ static void argsToEdges(CFLAliasAnalysis &Analysis, Instruction *Inst,
   v.visit(Inst);
 }
 
+static void argsToEdges(CFLAliasAnalysis &Analysis, ConstantExpr *CE,
+                        SmallVectorImpl<Edge> &Output) {
+  assert(hasUsefulEdges(CE) && "Expected constant expr to have 'useful' edges");
+  GetEdgesVisitor v(Analysis, Output);
+  v.visitConstantExpr(CE);
+}
+
 static Level directionOfEdgeType(EdgeType Weight) {
   switch (Weight) {
   case EdgeType::Reference:
@@ -866,25 +896,23 @@ static void constexprToEdges(CFLAliasAnalysis &Analysis,
   Worklist.push_back(&CExprToCollapse);
 
   SmallVector<Edge, 8> ConstexprEdges;
+  SmallPtrSet<ConstantExpr *, 4> Visited;
   while (!Worklist.empty()) {
     auto *CExpr = Worklist.pop_back_val();
-    std::unique_ptr<Instruction> Inst(CExpr->getAsInstruction());
 
-    if (!hasUsefulEdges(Inst.get()))
+    if (!hasUsefulEdges(CExpr))
       continue;
 
     ConstexprEdges.clear();
-    argsToEdges(Analysis, Inst.get(), ConstexprEdges);
+    argsToEdges(Analysis, CExpr, ConstexprEdges);
     for (auto &Edge : ConstexprEdges) {
-      if (Edge.From == Inst.get())
-        Edge.From = CExpr;
-      else if (auto *Nested = dyn_cast<ConstantExpr>(Edge.From))
-        Worklist.push_back(Nested);
-
-      if (Edge.To == Inst.get())
-        Edge.To = CExpr;
-      else if (auto *Nested = dyn_cast<ConstantExpr>(Edge.To))
-        Worklist.push_back(Nested);
+      if (auto *Nested = dyn_cast<ConstantExpr>(Edge.From))
+        if (Visited.insert(Nested).second)
+          Worklist.push_back(Nested);
+
+      if (auto *Nested = dyn_cast<ConstantExpr>(Edge.To))
+        if (Visited.insert(Nested).second)
+          Worklist.push_back(Nested);
     }
 
     Results.append(ConstexprEdges.begin(), ConstexprEdges.end());
@@ -1081,9 +1109,8 @@ void CFLAliasAnalysis::scan(Function *Fn) {
   Handles.push_front(FunctionHandle(Fn, this));
 }
 
-AliasAnalysis::AliasResult
-CFLAliasAnalysis::query(const AliasAnalysis::Location &LocA,
-                        const AliasAnalysis::Location &LocB) {
+AliasResult CFLAliasAnalysis::query(const MemoryLocation &LocA,
+                                    const MemoryLocation &LocB) {
   auto *ValA = const_cast<Value *>(LocA.Ptr);
   auto *ValB = const_cast<Value *>(LocB.Ptr);
 
@@ -1094,7 +1121,7 @@ CFLAliasAnalysis::query(const AliasAnalysis::Location &LocA,
     // The only times this is known to happen are when globals + InlineAsm
     // are involved
     DEBUG(dbgs() << "CFLAA: could not extract parent function information.\n");
-    return AliasAnalysis::MayAlias;
+    return MayAlias;
   }
 
   if (MaybeFnA.hasValue()) {
@@ -1112,11 +1139,11 @@ CFLAliasAnalysis::query(const AliasAnalysis::Location &LocA,
   auto &Sets = MaybeInfo->Sets;
   auto MaybeA = Sets.find(ValA);
   if (!MaybeA.hasValue())
-    return AliasAnalysis::MayAlias;
+    return MayAlias;
 
   auto MaybeB = Sets.find(ValB);
   if (!MaybeB.hasValue())
-    return AliasAnalysis::MayAlias;
+    return MayAlias;
 
   auto SetA = *MaybeA;
   auto SetB = *MaybeB;
@@ -1133,7 +1160,7 @@ CFLAliasAnalysis::query(const AliasAnalysis::Location &LocA,
   // the sets has no values that could legally be altered by changing the value
   // of an argument or global, then we don't have to be as conservative.
   if (AttrsA.any() && AttrsB.any())
-    return AliasAnalysis::MayAlias;
+    return MayAlias;
 
   // We currently unify things even if the accesses to them may not be in
   // bounds, so we can't return partial alias here because we don't
@@ -1144,9 +1171,9 @@ CFLAliasAnalysis::query(const AliasAnalysis::Location &LocA,
   // differentiate
 
   if (SetA.Index == SetB.Index)
-    return AliasAnalysis::MayAlias;
+    return MayAlias;
 
-  return AliasAnalysis::NoAlias;
+  return NoAlias;
 }
 
 bool CFLAliasAnalysis::doInitialization(Module &M) {
diff --git a/lib/Analysis/CMakeLists.txt b/lib/Analysis/CMakeLists.txt
index de189a4d38b..399c0e733b5 100644
--- a/lib/Analysis/CMakeLists.txt
+++ b/lib/Analysis/CMakeLists.txt
@@ -28,6 +28,7 @@ add_llvm_library(LLVMAnalysis
   InstructionSimplify.cpp
   Interval.cpp
   IntervalPartition.cpp
+  IteratedDominanceFrontier.cpp
   LazyCallGraph.cpp
   LazyValueInfo.cpp
   LibCallAliasAnalysis.cpp
@@ -41,8 +42,10 @@ add_llvm_library(LLVMAnalysis
   MemDerefPrinter.cpp
   MemoryBuiltins.cpp
   MemoryDependenceAnalysis.cpp
+  MemoryLocation.cpp
   ModuleDebugInfoPrinter.cpp
   NoAliasAnalysis.cpp
+  OrderedBasicBlock.cpp
   PHITransAddr.cpp
   PostDominators.cpp
   PtrUseVisitor.cpp
@@ -60,6 +63,7 @@ add_llvm_library(LLVMAnalysis
   TypeBasedAliasAnalysis.cpp
   ScopedNoAliasAA.cpp
   ValueTracking.cpp
+  VectorUtils.cpp
 
   ADDITIONAL_HEADER_DIRS
   ${LLVM_MAIN_INCLUDE_DIR}/llvm/Analysis
@@ -68,4 +72,3 @@ add_llvm_library(LLVMAnalysis
 add_dependencies(LLVMAnalysis intrinsics_gen)
 
 add_subdirectory(IPA)
-add_subdirectory(NaCl) # LOCALMOD
diff --git a/lib/Analysis/CaptureTracking.cpp b/lib/Analysis/CaptureTracking.cpp
index 5a547541795..fc23aa53d7c 100644
--- a/lib/Analysis/CaptureTracking.cpp
+++ b/lib/Analysis/CaptureTracking.cpp
@@ -21,6 +21,7 @@
 #include "llvm/Analysis/AliasAnalysis.h"
 #include "llvm/Analysis/CFG.h"
 #include "llvm/Analysis/CaptureTracking.h"
+#include "llvm/Analysis/OrderedBasicBlock.h"
 #include "llvm/IR/CallSite.h"
 #include "llvm/IR/Constants.h"
 #include "llvm/IR/Dominators.h"
@@ -57,29 +58,74 @@ namespace {
   /// Only support the case where the Value is defined in the same basic block
   /// as the given instruction and the use.
   struct CapturesBefore : public CaptureTracker {
+
     CapturesBefore(bool ReturnCaptures, const Instruction *I, DominatorTree *DT,
-                   bool IncludeI)
-      : BeforeHere(I), DT(DT), ReturnCaptures(ReturnCaptures),
-        IncludeI(IncludeI), Captured(false) {}
+                   bool IncludeI, OrderedBasicBlock *IC)
+      : OrderedBB(IC), BeforeHere(I), DT(DT),
+        ReturnCaptures(ReturnCaptures), IncludeI(IncludeI), Captured(false) {}
 
     void tooManyUses() override { Captured = true; }
 
-    bool shouldExplore(const Use *U) override {
-      Instruction *I = cast<Instruction>(U->getUser());
-      if (BeforeHere == I && !IncludeI)
-        return false;
-
+    bool isSafeToPrune(Instruction *I) {
       BasicBlock *BB = I->getParent();
       // We explore this usage only if the usage can reach "BeforeHere".
       // If use is not reachable from entry, there is no need to explore.
       if (BeforeHere != I && !DT->isReachableFromEntry(BB))
+        return true;
+
+      // Compute the case where both instructions are inside the same basic
+      // block. Since instructions in the same BB as BeforeHere are numbered in
+      // 'OrderedBB', avoid using 'dominates' and 'isPotentiallyReachable'
+      // which are very expensive for large basic blocks.
+      if (BB == BeforeHere->getParent()) {
+        // 'I' dominates 'BeforeHere' => not safe to prune.
+        //
+        // The value defined by an invoke dominates an instruction only if it
+        // dominates every instruction in UseBB. A PHI is dominated only if
+        // the instruction dominates every possible use in the UseBB. Since
+        // UseBB == BB, avoid pruning.
+        if (isa<InvokeInst>(BeforeHere) || isa<PHINode>(I) || I == BeforeHere)
+          return false;
+        if (!OrderedBB->dominates(BeforeHere, I))
+          return false;
+
+        // 'BeforeHere' comes before 'I', it's safe to prune if we also
+        // guarantee that 'I' never reaches 'BeforeHere' through a back-edge or
+        // by its successors, i.e, prune if:
+        //
+        //  (1) BB is an entry block or have no sucessors.
+        //  (2) There's no path coming back through BB sucessors.
+        if (BB == &BB->getParent()->getEntryBlock() ||
+            !BB->getTerminator()->getNumSuccessors())
+          return true;
+
+        SmallVector<BasicBlock*, 32> Worklist;
+        Worklist.append(succ_begin(BB), succ_end(BB));
+        if (!isPotentiallyReachableFromMany(Worklist, BB, DT))
+          return true;
+
         return false;
+      }
+
       // If the value is defined in the same basic block as use and BeforeHere,
       // there is no need to explore the use if BeforeHere dominates use.
       // Check whether there is a path from I to BeforeHere.
       if (BeforeHere != I && DT->dominates(BeforeHere, I) &&
           !isPotentiallyReachable(I, BeforeHere, DT))
+        return true;
+
+      return false;
+    }
+
+    bool shouldExplore(const Use *U) override {
+      Instruction *I = cast<Instruction>(U->getUser());
+
+      if (BeforeHere == I && !IncludeI)
+        return false;
+
+      if (isSafeToPrune(I))
         return false;
+
       return true;
     }
 
@@ -87,21 +133,14 @@ namespace {
       if (isa<ReturnInst>(U->getUser()) && !ReturnCaptures)
         return false;
 
-      Instruction *I = cast<Instruction>(U->getUser());
-      if (BeforeHere == I && !IncludeI)
+      if (!shouldExplore(U))
         return false;
 
-      BasicBlock *BB = I->getParent();
-      // Same logic as in shouldExplore.
-      if (BeforeHere != I && !DT->isReachableFromEntry(BB))
-        return false;
-      if (BeforeHere != I && DT->dominates(BeforeHere, I) &&
-          !isPotentiallyReachable(I, BeforeHere, DT))
-        return false;
       Captured = true;
       return true;
     }
 
+    OrderedBasicBlock *OrderedBB;
     const Instruction *BeforeHere;
     DominatorTree *DT;
 
@@ -143,21 +182,29 @@ bool llvm::PointerMayBeCaptured(const Value *V,
 /// returning the value (or part of it) from the function counts as capturing
 /// it or not.  The boolean StoreCaptures specified whether storing the value
 /// (or part of it) into memory anywhere automatically counts as capturing it
-/// or not.
+/// or not. A ordered basic block \p OBB can be used in order to speed up
+/// queries about relative order among instructions in the same basic block.
 bool llvm::PointerMayBeCapturedBefore(const Value *V, bool ReturnCaptures,
                                       bool StoreCaptures, const Instruction *I,
-                                      DominatorTree *DT, bool IncludeI) {
+                                      DominatorTree *DT, bool IncludeI,
+                                      OrderedBasicBlock *OBB) {
   assert(!isa<GlobalValue>(V) &&
          "It doesn't make sense to ask whether a global is captured.");
+  bool UseNewOBB = OBB == nullptr;
 
   if (!DT)
     return PointerMayBeCaptured(V, ReturnCaptures, StoreCaptures);
+  if (UseNewOBB)
+    OBB = new OrderedBasicBlock(I->getParent());
 
   // TODO: See comment in PointerMayBeCaptured regarding what could be done
   // with StoreCaptures.
 
-  CapturesBefore CB(ReturnCaptures, I, DT, IncludeI);
+  CapturesBefore CB(ReturnCaptures, I, DT, IncludeI, OBB);
   PointerMayBeCaptured(V, &CB);
+
+  if (UseNewOBB)
+    delete OBB;
   return CB.Captured;
 }
 
diff --git a/lib/Analysis/ConstantFolding.cpp b/lib/Analysis/ConstantFolding.cpp
index a85e8136de5..07f53bedc77 100644
--- a/lib/Analysis/ConstantFolding.cpp
+++ b/lib/Analysis/ConstantFolding.cpp
@@ -248,8 +248,7 @@ static bool IsConstantOffsetFromGlobal(Constant *C, GlobalValue *&GV,
 
   // Look through ptr->int and ptr->ptr casts.
   if (CE->getOpcode() == Instruction::PtrToInt ||
-      CE->getOpcode() == Instruction::BitCast ||
-      CE->getOpcode() == Instruction::AddrSpaceCast)
+      CE->getOpcode() == Instruction::BitCast)
     return IsConstantOffsetFromGlobal(CE->getOperand(0), GV, Offset, DL);
 
   // i32* getelementptr ([5 x i32]* @a, i32 0, i32 5)
@@ -532,6 +531,10 @@ Constant *llvm::ConstantFoldLoadFromConstPtr(Constant *C,
     if (GV->isConstant() && GV->hasDefinitiveInitializer())
       return GV->getInitializer();
 
+  if (auto *GA = dyn_cast<GlobalAlias>(C))
+    if (GA->getAliasee() && !GA->mayBeOverridden())
+      return ConstantFoldLoadFromConstPtr(GA->getAliasee(), DL);
+
   // If the loaded value isn't a constant expr, we can't handle it.
   ConstantExpr *CE = dyn_cast<ConstantExpr>(C);
   if (!CE)
@@ -895,8 +898,7 @@ Constant *llvm::ConstantFoldInstruction(Instruction *I, const DataLayout &DL,
   if (PHINode *PN = dyn_cast<PHINode>(I)) {
     Constant *CommonValue = nullptr;
 
-    for (unsigned i = 0, e = PN->getNumIncomingValues(); i != e; ++i) {
-      Value *Incoming = PN->getIncomingValue(i);
+    for (Value *Incoming : PN->incoming_values()) {
       // If the incoming value is undef then skip it.  Note that while we could
       // skip the value if it is equal to the phi node itself we choose not to
       // because that would break the rule that constant folding only applies if
@@ -1235,6 +1237,11 @@ bool llvm::canConstantFoldCallTo(const Function *F) {
   case Intrinsic::floor:
   case Intrinsic::ceil:
   case Intrinsic::sqrt:
+  case Intrinsic::sin:
+  case Intrinsic::cos:
+  case Intrinsic::trunc:
+  case Intrinsic::rint:
+  case Intrinsic::nearbyint:
   case Intrinsic::pow:
   case Intrinsic::powi:
   case Intrinsic::bswap:
@@ -1421,6 +1428,36 @@ static Constant *ConstantFoldScalarCall(StringRef Name, unsigned IntrinsicID,
         return ConstantFP::get(Ty->getContext(), V);
       }
 
+      if (IntrinsicID == Intrinsic::floor) {
+        APFloat V = Op->getValueAPF();
+        V.roundToIntegral(APFloat::rmTowardNegative);
+        return ConstantFP::get(Ty->getContext(), V);
+      }
+
+      if (IntrinsicID == Intrinsic::ceil) {
+        APFloat V = Op->getValueAPF();
+        V.roundToIntegral(APFloat::rmTowardPositive);
+        return ConstantFP::get(Ty->getContext(), V);
+      }
+
+      if (IntrinsicID == Intrinsic::trunc) {
+        APFloat V = Op->getValueAPF();
+        V.roundToIntegral(APFloat::rmTowardZero);
+        return ConstantFP::get(Ty->getContext(), V);
+      }
+
+      if (IntrinsicID == Intrinsic::rint) {
+        APFloat V = Op->getValueAPF();
+        V.roundToIntegral(APFloat::rmNearestTiesToEven);
+        return ConstantFP::get(Ty->getContext(), V);
+      }
+
+      if (IntrinsicID == Intrinsic::nearbyint) {
+        APFloat V = Op->getValueAPF();
+        V.roundToIntegral(APFloat::rmNearestTiesToEven);
+        return ConstantFP::get(Ty->getContext(), V);
+      }
+
       /// We only fold functions with finite arguments. Folding NaN and inf is
       /// likely to be aborted with an exception anyway, and some host libms
       /// have known errors raising exceptions.
@@ -1438,7 +1475,7 @@ static Constant *ConstantFoldScalarCall(StringRef Name, unsigned IntrinsicID,
         case Intrinsic::fabs:
           return ConstantFoldFP(fabs, V, Ty);
         case Intrinsic::log2:
-          return ConstantFoldFP(log2, V, Ty);
+          return ConstantFoldFP(Log2, V, Ty);
         case Intrinsic::log:
           return ConstantFoldFP(log, V, Ty);
         case Intrinsic::log10:
@@ -1447,10 +1484,10 @@ static Constant *ConstantFoldScalarCall(StringRef Name, unsigned IntrinsicID,
           return ConstantFoldFP(exp, V, Ty);
         case Intrinsic::exp2:
           return ConstantFoldFP(exp2, V, Ty);
-        case Intrinsic::floor:
-          return ConstantFoldFP(floor, V, Ty);
-        case Intrinsic::ceil:
-          return ConstantFoldFP(ceil, V, Ty);
+        case Intrinsic::sin:
+          return ConstantFoldFP(sin, V, Ty);
+        case Intrinsic::cos:
+          return ConstantFoldFP(cos, V, Ty);
       }
 
       if (!TLI)
@@ -1544,8 +1581,8 @@ static Constant *ConstantFoldScalarCall(StringRef Name, unsigned IntrinsicID,
         APFloat Val(APFloat::IEEEhalf, Op->getValue());
 
         bool lost = false;
-        APFloat::opStatus status =
-          Val.convert(APFloat::IEEEsingle, APFloat::rmNearestTiesToEven, &lost);
+        APFloat::opStatus status = Val.convert(
+            Ty->getFltSemantics(), APFloat::rmNearestTiesToEven, &lost);
 
         // Conversion is always precise.
         (void)status;
diff --git a/lib/Analysis/CostModel.cpp b/lib/Analysis/CostModel.cpp
index b529c1a70aa..da790d74524 100644
--- a/lib/Analysis/CostModel.cpp
+++ b/lib/Analysis/CostModel.cpp
@@ -383,10 +383,8 @@ unsigned CostModelAnalysis::getInstructionCost(const Instruction *I) const {
     return -1;
 
   switch (I->getOpcode()) {
-  case Instruction::GetElementPtr:{
-    Type *ValTy = I->getOperand(0)->getType()->getPointerElementType();
-    return TTI->getAddressComputationCost(ValTy);
-  }
+  case Instruction::GetElementPtr:
+    return TTI->getUserCost(I);
 
   case Instruction::Ret:
   case Instruction::PHI:
diff --git a/lib/Analysis/Delinearization.cpp b/lib/Analysis/Delinearization.cpp
index d603b7b21e3..9d157860326 100644
--- a/lib/Analysis/Delinearization.cpp
+++ b/lib/Analysis/Delinearization.cpp
@@ -115,7 +115,7 @@ void Delinearization::print(raw_ostream &O, const Module *) const {
       O << "AddRec: " << *AR << "\n";
 
       SmallVector<const SCEV *, 3> Subscripts, Sizes;
-      AR->delinearize(*SE, Subscripts, Sizes, SE->getElementSize(Inst));
+      SE->delinearize(AR, Subscripts, Sizes, SE->getElementSize(Inst));
       if (Subscripts.size() == 0 || Sizes.size() == 0 ||
           Subscripts.size() != Sizes.size()) {
         O << "failed to delinearize\n";
diff --git a/lib/Analysis/DependenceAnalysis.cpp b/lib/Analysis/DependenceAnalysis.cpp
index 3374b48c141..4826ac407d7 100644
--- a/lib/Analysis/DependenceAnalysis.cpp
+++ b/lib/Analysis/DependenceAnalysis.cpp
@@ -625,10 +625,9 @@ void Dependence::dump(raw_ostream &OS) const {
   OS << "!\n";
 }
 
-static AliasAnalysis::AliasResult underlyingObjectsAlias(AliasAnalysis *AA,
-                                                         const DataLayout &DL,
-                                                         const Value *A,
-                                                         const Value *B) {
+static AliasResult underlyingObjectsAlias(AliasAnalysis *AA,
+                                          const DataLayout &DL, const Value *A,
+                                          const Value *B) {
   const Value *AObj = GetUnderlyingObject(A, DL);
   const Value *BObj = GetUnderlyingObject(B, DL);
   return AA->alias(AObj, AA->getTypeStoreSize(AObj->getType()),
@@ -779,23 +778,56 @@ void DependenceAnalysis::collectCommonLoops(const SCEV *Expression,
   }
 }
 
-void DependenceAnalysis::unifySubscriptType(Subscript *Pair) {
-  const SCEV *Src = Pair->Src;
-  const SCEV *Dst = Pair->Dst;
-  IntegerType *SrcTy = dyn_cast<IntegerType>(Src->getType());
-  IntegerType *DstTy = dyn_cast<IntegerType>(Dst->getType());
-  if (SrcTy == nullptr || DstTy == nullptr) {
-    assert(SrcTy == DstTy && "This function only unify integer types and "
-                             "expect Src and Dst share the same type "
-                             "otherwise.");
-    return;
+void DependenceAnalysis::unifySubscriptType(ArrayRef<Subscript *> Pairs) {
+
+  unsigned widestWidthSeen = 0;
+  Type *widestType;
+
+  // Go through each pair and find the widest bit to which we need
+  // to extend all of them.
+  for (unsigned i = 0; i < Pairs.size(); i++) {
+    const SCEV *Src = Pairs[i]->Src;
+    const SCEV *Dst = Pairs[i]->Dst;
+    IntegerType *SrcTy = dyn_cast<IntegerType>(Src->getType());
+    IntegerType *DstTy = dyn_cast<IntegerType>(Dst->getType());
+    if (SrcTy == nullptr || DstTy == nullptr) {
+      assert(SrcTy == DstTy && "This function only unify integer types and "
+             "expect Src and Dst share the same type "
+             "otherwise.");
+      continue;
+    }
+    if (SrcTy->getBitWidth() > widestWidthSeen) {
+      widestWidthSeen = SrcTy->getBitWidth();
+      widestType = SrcTy;
+    }
+    if (DstTy->getBitWidth() > widestWidthSeen) {
+      widestWidthSeen = DstTy->getBitWidth();
+      widestType = DstTy;
+    }
   }
-  if (SrcTy->getBitWidth() > DstTy->getBitWidth()) {
-    // Sign-extend Dst to typeof(Src) if typeof(Src) is wider than typeof(Dst).
-    Pair->Dst = SE->getSignExtendExpr(Dst, SrcTy);
-  } else if (SrcTy->getBitWidth() < DstTy->getBitWidth()) {
-    // Sign-extend Src to typeof(Dst) if typeof(Dst) is wider than typeof(Src).
-    Pair->Src = SE->getSignExtendExpr(Src, DstTy);
+
+
+  assert(widestWidthSeen > 0);
+
+  // Now extend each pair to the widest seen.
+  for (unsigned i = 0; i < Pairs.size(); i++) {
+    const SCEV *Src = Pairs[i]->Src;
+    const SCEV *Dst = Pairs[i]->Dst;
+    IntegerType *SrcTy = dyn_cast<IntegerType>(Src->getType());
+    IntegerType *DstTy = dyn_cast<IntegerType>(Dst->getType());
+    if (SrcTy == nullptr || DstTy == nullptr) {
+      assert(SrcTy == DstTy && "This function only unify integer types and "
+             "expect Src and Dst share the same type "
+             "otherwise.");
+      continue;
+    }
+    if (SrcTy->getBitWidth() < widestWidthSeen)
+      // Sign-extend Src to widestType
+      Pairs[i]->Src = SE->getSignExtendExpr(Src, widestType);
+    if (DstTy->getBitWidth() < widestWidthSeen) {
+      // Sign-extend Dst to widestType
+      Pairs[i]->Dst = SE->getSignExtendExpr(Dst, widestType);
+    }
   }
 }
 
@@ -830,6 +862,14 @@ bool DependenceAnalysis::checkSrcSubscript(const SCEV *Src,
     return isLoopInvariant(Src, LoopNest);
   const SCEV *Start = AddRec->getStart();
   const SCEV *Step = AddRec->getStepRecurrence(*SE);
+  const SCEV *UB = SE->getBackedgeTakenCount(AddRec->getLoop());
+  if (!isa<SCEVCouldNotCompute>(UB)) {
+    if (SE->getTypeSizeInBits(Start->getType()) <
+        SE->getTypeSizeInBits(UB->getType())) {
+      if (!AddRec->getNoWrapFlags())
+        return false;
+    }
+  }
   if (!isLoopInvariant(Step, LoopNest))
     return false;
   Loops.set(mapSrcLoop(AddRec->getLoop()));
@@ -848,6 +888,14 @@ bool DependenceAnalysis::checkDstSubscript(const SCEV *Dst,
     return isLoopInvariant(Dst, LoopNest);
   const SCEV *Start = AddRec->getStart();
   const SCEV *Step = AddRec->getStepRecurrence(*SE);
+  const SCEV *UB = SE->getBackedgeTakenCount(AddRec->getLoop());
+  if (!isa<SCEVCouldNotCompute>(UB)) {
+    if (SE->getTypeSizeInBits(Start->getType()) <
+        SE->getTypeSizeInBits(UB->getType())) {
+      if (!AddRec->getNoWrapFlags())
+        return false;
+    }
+  }
   if (!isLoopInvariant(Step, LoopNest))
     return false;
   Loops.set(mapDstLoop(AddRec->getLoop()));
@@ -942,13 +990,15 @@ bool DependenceAnalysis::isKnownPredicate(ICmpInst::Predicate Pred,
 // All subscripts are all the same type.
 // Loop bound may be smaller (e.g., a char).
 // Should zero extend loop bound, since it's always >= 0.
-// This routine collects upper bound and extends if needed.
+// This routine collects upper bound and extends or truncates if needed.
+// Truncating is safe when subscripts are known not to wrap. Cases without
+// nowrap flags should have been rejected earlier.
 // Return null if no bound available.
 const SCEV *DependenceAnalysis::collectUpperBound(const Loop *L,
                                                   Type *T) const {
   if (SE->hasLoopInvariantBackedgeTakenCount(L)) {
     const SCEV *UB = SE->getBackedgeTakenCount(L);
-    return SE->getNoopOrZeroExtend(UB, T);
+    return SE->getTruncateOrZeroExtend(UB, T);
   }
   return nullptr;
 }
@@ -2919,7 +2969,7 @@ const SCEV *DependenceAnalysis::getUpperBound(BoundInfo *Bound) const {
 // return the coefficient (the step)
 // corresponding to the specified loop.
 // If there isn't one, return 0.
-// For example, given a*i + b*j + c*k, zeroing the coefficient
+// For example, given a*i + b*j + c*k, finding the coefficient
 // corresponding to the j loop would yield b.
 const SCEV *DependenceAnalysis::findCoefficient(const SCEV *Expr,
                                                 const Loop *TargetLoop)  const {
@@ -3216,8 +3266,8 @@ bool DependenceAnalysis::tryDelinearize(const SCEV *SrcSCEV,
 
   // First step: collect parametric terms in both array references.
   SmallVector<const SCEV *, 4> Terms;
-  SrcAR->collectParametricTerms(*SE, Terms);
-  DstAR->collectParametricTerms(*SE, Terms);
+  SE->collectParametricTerms(SrcAR, Terms);
+  SE->collectParametricTerms(DstAR, Terms);
 
   // Second step: find subscript sizes.
   SmallVector<const SCEV *, 4> Sizes;
@@ -3225,8 +3275,8 @@ bool DependenceAnalysis::tryDelinearize(const SCEV *SrcSCEV,
 
   // Third step: compute the access functions for each subscript.
   SmallVector<const SCEV *, 4> SrcSubscripts, DstSubscripts;
-  SrcAR->computeAccessFunctions(*SE, SrcSubscripts, Sizes);
-  DstAR->computeAccessFunctions(*SE, DstSubscripts, Sizes);
+  SE->computeAccessFunctions(SrcAR, SrcSubscripts, Sizes);
+  SE->computeAccessFunctions(DstAR, DstSubscripts, Sizes);
 
   // Fail when there is only a subscript: that's a linearized access function.
   if (SrcSubscripts.size() < 2 || DstSubscripts.size() < 2 ||
@@ -3314,16 +3364,16 @@ DependenceAnalysis::depends(Instruction *Src, Instruction *Dst,
 
   switch (underlyingObjectsAlias(AA, F->getParent()->getDataLayout(), DstPtr,
                                  SrcPtr)) {
-  case AliasAnalysis::MayAlias:
-  case AliasAnalysis::PartialAlias:
+  case MayAlias:
+  case PartialAlias:
     // cannot analyse objects if we don't understand their aliasing.
     DEBUG(dbgs() << "can't analyze may or partial alias\n");
     return make_unique<Dependence>(Src, Dst);
-  case AliasAnalysis::NoAlias:
+  case NoAlias:
     // If the objects noalias, they are distinct, accesses are independent.
     DEBUG(dbgs() << "no alias\n");
     return nullptr;
-  case AliasAnalysis::MustAlias:
+  case MustAlias:
     break; // The underlying objects alias; test accesses for dependence.
   }
 
@@ -3556,13 +3606,16 @@ DependenceAnalysis::depends(Instruction *Src, Instruction *Dst,
       SmallBitVector Sivs(Pairs);
       SmallBitVector Mivs(Pairs);
       SmallBitVector ConstrainedLevels(MaxLevels + 1);
+      SmallVector<Subscript *, 4> PairsInGroup;
       for (int SJ = Group.find_first(); SJ >= 0; SJ = Group.find_next(SJ)) {
         DEBUG(dbgs() << SJ << " ");
         if (Pair[SJ].Classification == Subscript::SIV)
           Sivs.set(SJ);
         else
           Mivs.set(SJ);
+        PairsInGroup.push_back(&Pair[SJ]);
       }
+      unifySubscriptType(PairsInGroup);
       DEBUG(dbgs() << "}\n");
       while (Sivs.any()) {
         bool Changed = false;
@@ -3760,7 +3813,7 @@ const  SCEV *DependenceAnalysis::getSplitIteration(const Dependence &Dep,
   Value *SrcPtr = getPointerOperand(Src);
   Value *DstPtr = getPointerOperand(Dst);
   assert(underlyingObjectsAlias(AA, F->getParent()->getDataLayout(), DstPtr,
-                                SrcPtr) == AliasAnalysis::MustAlias);
+                                SrcPtr) == MustAlias);
 
   // establish loop nesting levels
   establishNestingLevels(Src, Dst);
diff --git a/lib/Analysis/IPA/CallGraph.cpp b/lib/Analysis/IPA/CallGraph.cpp
index 67cf7f86e07..e2799d965a7 100644
--- a/lib/Analysis/IPA/CallGraph.cpp
+++ b/lib/Analysis/IPA/CallGraph.cpp
@@ -24,8 +24,8 @@ CallGraph::CallGraph(Module &M)
     : M(M), Root(nullptr), ExternalCallingNode(getOrInsertFunction(nullptr)),
       CallsExternalNode(new CallGraphNode(nullptr)) {
   // Add every function to the call graph.
-  for (Module::iterator I = M.begin(), E = M.end(); I != E; ++I)
-    addToCallGraph(I);
+  for (Function &F : M)
+    addToCallGraph(&F);
 
   // If we didn't find a main function, use the external call graph node
   if (!Root)
@@ -40,13 +40,11 @@ CallGraph::~CallGraph() {
 // Reset all node's use counts to zero before deleting them to prevent an
 // assertion from firing.
 #ifndef NDEBUG
-  for (FunctionMapTy::iterator I = FunctionMap.begin(), E = FunctionMap.end();
-       I != E; ++I)
-    I->second->allReferencesDropped();
+  for (auto &I : FunctionMap)
+    I.second->allReferencesDropped();
 #endif
-  for (FunctionMapTy::iterator I = FunctionMap.begin(), E = FunctionMap.end();
-       I != E; ++I)
-    delete I->second;
+  for (auto &I : FunctionMap)
+    delete I.second;
 }
 
 void CallGraph::addToCallGraph(Function *F) {
@@ -81,8 +79,10 @@ void CallGraph::addToCallGraph(Function *F) {
       CallSite CS(cast<Value>(II));
       if (CS) {
         const Function *Callee = CS.getCalledFunction();
-        if (!Callee)
+        if (!Callee || !Intrinsic::isLeaf(Callee->getIntrinsicID()))
           // Indirect calls of intrinsics are not allowed so no need to check.
+          // We can be more precise here by using TargetArg returned by
+          // Intrinsic::isLeaf.
           Node->addCalledFunction(CS, CallsExternalNode);
         else if (!Callee->isIntrinsic())
           Node->addCalledFunction(CS, getOrInsertFunction(Callee));
@@ -98,8 +98,26 @@ void CallGraph::print(raw_ostream &OS) const {
     OS << "<<null function: 0x" << Root << ">>\n";
   }
 
-  for (CallGraph::const_iterator I = begin(), E = end(); I != E; ++I)
-    I->second->print(OS);
+  // Print in a deterministic order by sorting CallGraphNodes by name.  We do
+  // this here to avoid slowing down the non-printing fast path.
+
+  SmallVector<CallGraphNode *, 16> Nodes;
+  Nodes.reserve(FunctionMap.size());
+
+  for (auto I = begin(), E = end(); I != E; ++I)
+    Nodes.push_back(I->second);
+
+  std::sort(Nodes.begin(), Nodes.end(),
+            [](CallGraphNode *LHS, CallGraphNode *RHS) {
+    if (Function *LF = LHS->getFunction())
+      if (Function *RF = RHS->getFunction())
+        return LF->getName() < RF->getName();
+
+    return RHS->getFunction() != nullptr;
+  });
+
+  for (CallGraphNode *CN : Nodes)
+    CN->print(OS);
 }
 
 #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
diff --git a/lib/Analysis/IPA/CallGraphSCCPass.cpp b/lib/Analysis/IPA/CallGraphSCCPass.cpp
index 65ba1c7c6c4..07b389a2a13 100644
--- a/lib/Analysis/IPA/CallGraphSCCPass.cpp
+++ b/lib/Analysis/IPA/CallGraphSCCPass.cpp
@@ -217,8 +217,10 @@ bool CGPassManager::RefreshCallGraph(CallGraphSCC &CurSCC,
           // another value. This can happen when constant folding happens
           // of well known functions etc.
           !CallSite(I->first) ||
-           (CallSite(I->first).getCalledFunction() &&
-            CallSite(I->first).getCalledFunction()->isIntrinsic())) {
+          (CallSite(I->first).getCalledFunction() &&
+           CallSite(I->first).getCalledFunction()->isIntrinsic() &&
+           Intrinsic::isLeaf(
+               CallSite(I->first).getCalledFunction()->getIntrinsicID()))) {
         assert(!CheckingMode &&
                "CallGraphSCCPass did not update the CallGraph correctly!");
         
@@ -449,7 +451,7 @@ bool CGPassManager::runOnModule(Module &M) {
     const std::vector<CallGraphNode *> &NodeVec = *CGI;
     CurSCC.initialize(NodeVec.data(), NodeVec.data() + NodeVec.size());
     ++CGI;
-    
+
     // At the top level, we run all the passes in this pass manager on the
     // functions in this SCC.  However, we support iterative compilation in the
     // case where a function pass devirtualizes a call to a function.  For
diff --git a/lib/Analysis/IPA/GlobalsModRef.cpp b/lib/Analysis/IPA/GlobalsModRef.cpp
index 018ae99d661..77fb41be8e2 100644
--- a/lib/Analysis/IPA/GlobalsModRef.cpp
+++ b/lib/Analysis/IPA/GlobalsModRef.cpp
@@ -16,6 +16,7 @@
 
 #include "llvm/Analysis/Passes.h"
 #include "llvm/ADT/SCCIterator.h"
+#include "llvm/ADT/SmallPtrSet.h"
 #include "llvm/ADT/Statistic.h"
 #include "llvm/Analysis/AliasAnalysis.h"
 #include "llvm/Analysis/CallGraph.h"
@@ -29,7 +30,7 @@
 #include "llvm/IR/Module.h"
 #include "llvm/Pass.h"
 #include "llvm/Support/CommandLine.h"
-#include <set>
+#include <list>
 using namespace llvm;
 
 #define DEBUG_TYPE "globalsmodref-aa"
@@ -41,163 +42,331 @@ STATISTIC(NumNoMemFunctions, "Number of functions that do not access memory");
 STATISTIC(NumReadMemFunctions, "Number of functions that only read memory");
 STATISTIC(NumIndirectGlobalVars, "Number of indirect global objects");
 
-namespace {
-  /// FunctionRecord - One instance of this structure is stored for every
-  /// function in the program.  Later, the entries for these functions are
-  /// removed if the function is found to call an external function (in which
-  /// case we know nothing about it.
-  struct FunctionRecord {
-    /// GlobalInfo - Maintain mod/ref info for all of the globals without
-    /// addresses taken that are read or written (transitively) by this
-    /// function.
-    std::map<const GlobalValue*, unsigned> GlobalInfo;
-
-    /// MayReadAnyGlobal - May read global variables, but it is not known which.
-    bool MayReadAnyGlobal;
-
-    unsigned getInfoForGlobal(const GlobalValue *GV) const {
-      unsigned Effect = MayReadAnyGlobal ? AliasAnalysis::Ref : 0;
-      std::map<const GlobalValue*, unsigned>::const_iterator I =
-        GlobalInfo.find(GV);
-      if (I != GlobalInfo.end())
-        Effect |= I->second;
-      return Effect;
-    }
-
-    /// FunctionEffect - Capture whether or not this function reads or writes to
-    /// ANY memory.  If not, we can do a lot of aggressive analysis on it.
-    unsigned FunctionEffect;
+// An option to enable unsafe alias results from the GlobalsModRef analysis.
+// When enabled, GlobalsModRef will provide no-alias results which in extremely
+// rare cases may not be conservatively correct. In particular, in the face of
+// transforms which cause assymetry between how effective GetUnderlyingObject
+// is for two pointers, it may produce incorrect results.
+//
+// These unsafe results have been returned by GMR for many years without
+// causing significant issues in the wild and so we provide a mechanism to
+// re-enable them for users of LLVM that have a particular performance
+// sensitivity and no known issues. The option also makes it easy to evaluate
+// the performance impact of these results.
+static cl::opt<bool> EnableUnsafeGlobalsModRefAliasResults(
+    "enable-unsafe-globalsmodref-alias-results", cl::init(false), cl::Hidden);
 
-    FunctionRecord() : MayReadAnyGlobal (false), FunctionEffect(0) {}
+namespace {
+/// The mod/ref information collected for a particular function.
+///
+/// We collect information about mod/ref behavior of a function here, both in
+/// general and as pertains to specific globals. We only have this detailed
+/// information when we know *something* useful about the behavior. If we
+/// saturate to fully general mod/ref, we remove the info for the function.
+class FunctionInfo {
+  typedef SmallDenseMap<const GlobalValue *, ModRefInfo, 16> GlobalInfoMapType;
+
+  /// Build a wrapper struct that has 8-byte alignment. All heap allocations
+  /// should provide this much alignment at least, but this makes it clear we
+  /// specifically rely on this amount of alignment.
+  struct LLVM_ALIGNAS(8) AlignedMap {
+    AlignedMap() {}
+    AlignedMap(const AlignedMap &Arg) : Map(Arg.Map) {}
+    GlobalInfoMapType Map;
   };
 
-  /// GlobalsModRef - The actual analysis pass.
-  class GlobalsModRef : public ModulePass, public AliasAnalysis {
-    /// NonAddressTakenGlobals - The globals that do not have their addresses
-    /// taken.
-    std::set<const GlobalValue*> NonAddressTakenGlobals;
+  /// Pointer traits for our aligned map.
+  struct AlignedMapPointerTraits {
+    static inline void *getAsVoidPointer(AlignedMap *P) { return P; }
+    static inline AlignedMap *getFromVoidPointer(void *P) {
+      return (AlignedMap *)P;
+    }
+    enum { NumLowBitsAvailable = 3 };
+    static_assert(AlignOf<AlignedMap>::Alignment >= (1 << NumLowBitsAvailable),
+                  "AlignedMap insufficiently aligned to have enough low bits.");
+  };
 
-    /// IndirectGlobals - The memory pointed to by this global is known to be
-    /// 'owned' by the global.
-    std::set<const GlobalValue*> IndirectGlobals;
+  /// The bit that flags that this function may read any global. This is
+  /// chosen to mix together with ModRefInfo bits.
+  enum { MayReadAnyGlobal = 4 };
+
+  /// Checks to document the invariants of the bit packing here.
+  static_assert((MayReadAnyGlobal & MRI_ModRef) == 0,
+                "ModRef and the MayReadAnyGlobal flag bits overlap.");
+  static_assert(((MayReadAnyGlobal | MRI_ModRef) >>
+                 AlignedMapPointerTraits::NumLowBitsAvailable) == 0,
+                "Insufficient low bits to store our flag and ModRef info.");
+
+public:
+  FunctionInfo() : Info() {}
+  ~FunctionInfo() {
+    delete Info.getPointer();
+  }
+  // Spell out the copy ond move constructors and assignment operators to get
+  // deep copy semantics and correct move semantics in the face of the
+  // pointer-int pair.
+  FunctionInfo(const FunctionInfo &Arg)
+      : Info(nullptr, Arg.Info.getInt()) {
+    if (const auto *ArgPtr = Arg.Info.getPointer())
+      Info.setPointer(new AlignedMap(*ArgPtr));
+  }
+  FunctionInfo(FunctionInfo &&Arg)
+      : Info(Arg.Info.getPointer(), Arg.Info.getInt()) {
+    Arg.Info.setPointerAndInt(nullptr, 0);
+  }
+  FunctionInfo &operator=(const FunctionInfo &RHS) {
+    delete Info.getPointer();
+    Info.setPointerAndInt(nullptr, RHS.Info.getInt());
+    if (const auto *RHSPtr = RHS.Info.getPointer())
+      Info.setPointer(new AlignedMap(*RHSPtr));
+    return *this;
+  }
+  FunctionInfo &operator=(FunctionInfo &&RHS) {
+    delete Info.getPointer();
+    Info.setPointerAndInt(RHS.Info.getPointer(), RHS.Info.getInt());
+    RHS.Info.setPointerAndInt(nullptr, 0);
+    return *this;
+  }
 
-    /// AllocsForIndirectGlobals - If an instruction allocates memory for an
-    /// indirect global, this map indicates which one.
-    std::map<const Value*, const GlobalValue*> AllocsForIndirectGlobals;
+  /// Returns the \c ModRefInfo info for this function.
+  ModRefInfo getModRefInfo() const {
+    return ModRefInfo(Info.getInt() & MRI_ModRef);
+  }
 
-    /// FunctionInfo - For each function, keep track of what globals are
-    /// modified or read.
-    std::map<const Function*, FunctionRecord> FunctionInfo;
+  /// Adds new \c ModRefInfo for this function to its state.
+  void addModRefInfo(ModRefInfo NewMRI) {
+    Info.setInt(Info.getInt() | NewMRI);
+  }
 
-  public:
-    static char ID;
-    GlobalsModRef() : ModulePass(ID) {
-      initializeGlobalsModRefPass(*PassRegistry::getPassRegistry());
+  /// Returns whether this function may read any global variable, and we don't
+  /// know which global.
+  bool mayReadAnyGlobal() const { return Info.getInt() & MayReadAnyGlobal; }
+
+  /// Sets this function as potentially reading from any global.
+  void setMayReadAnyGlobal() { Info.setInt(Info.getInt() | MayReadAnyGlobal); }
+
+  /// Returns the \c ModRefInfo info for this function w.r.t. a particular
+  /// global, which may be more precise than the general information above.
+  ModRefInfo getModRefInfoForGlobal(const GlobalValue &GV) const {
+    ModRefInfo GlobalMRI = mayReadAnyGlobal() ? MRI_Ref : MRI_NoModRef;
+    if (AlignedMap *P = Info.getPointer()) {
+      auto I = P->Map.find(&GV);
+      if (I != P->Map.end())
+        GlobalMRI = ModRefInfo(GlobalMRI | I->second);
     }
+    return GlobalMRI;
+  }
 
-    bool runOnModule(Module &M) override {
-      InitializeAliasAnalysis(this, &M.getDataLayout());
+  /// Add mod/ref info from another function into ours, saturating towards
+  /// MRI_ModRef.
+  void addFunctionInfo(const FunctionInfo &FI) {
+    addModRefInfo(FI.getModRefInfo());
 
-      // Find non-addr taken globals.
-      AnalyzeGlobals(M);
+    if (FI.mayReadAnyGlobal())
+      setMayReadAnyGlobal();
 
-      // Propagate on CG.
-      AnalyzeCallGraph(getAnalysis<CallGraphWrapperPass>().getCallGraph(), M);
-      return false;
-    }
+    if (AlignedMap *P = FI.Info.getPointer())
+      for (const auto &G : P->Map)
+        addModRefInfoForGlobal(*G.first, G.second);
+  }
 
-    void getAnalysisUsage(AnalysisUsage &AU) const override {
-      AliasAnalysis::getAnalysisUsage(AU);
-      AU.addRequired<CallGraphWrapperPass>();
-      AU.setPreservesAll();                         // Does not transform code
+  void addModRefInfoForGlobal(const GlobalValue &GV, ModRefInfo NewMRI) {
+    AlignedMap *P = Info.getPointer();
+    if (!P) {
+      P = new AlignedMap();
+      Info.setPointer(P);
     }
+    auto &GlobalMRI = P->Map[&GV];
+    GlobalMRI = ModRefInfo(GlobalMRI | NewMRI);
+  }
 
-    //------------------------------------------------
-    // Implement the AliasAnalysis API
-    //
-    AliasResult alias(const Location &LocA, const Location &LocB) override;
-    ModRefResult getModRefInfo(ImmutableCallSite CS,
-                               const Location &Loc) override;
-    ModRefResult getModRefInfo(ImmutableCallSite CS1,
-                               ImmutableCallSite CS2) override {
-      return AliasAnalysis::getModRefInfo(CS1, CS2);
-    }
+  /// Clear a global's ModRef info. Should be used when a global is being
+  /// deleted.
+  void eraseModRefInfoForGlobal(const GlobalValue &GV) {
+    if (AlignedMap *P = Info.getPointer())
+      P->Map.erase(&GV);
+  }
 
-    /// getModRefBehavior - Return the behavior of the specified function if
-    /// called from the specified call site.  The call site may be null in which
-    /// case the most generic behavior of this function should be returned.
-    ModRefBehavior getModRefBehavior(const Function *F) override {
-      ModRefBehavior Min = UnknownModRefBehavior;
-
-      if (FunctionRecord *FR = getFunctionInfo(F)) {
-        if (FR->FunctionEffect == 0)
-          Min = DoesNotAccessMemory;
-        else if ((FR->FunctionEffect & Mod) == 0)
-          Min = OnlyReadsMemory;
-      }
+private:
+  /// All of the information is encoded into a single pointer, with a three bit
+  /// integer in the low three bits. The high bit provides a flag for when this
+  /// function may read any global. The low two bits are the ModRefInfo. And
+  /// the pointer, when non-null, points to a map from GlobalValue to
+  /// ModRefInfo specific to that GlobalValue.
+  PointerIntPair<AlignedMap *, 3, unsigned, AlignedMapPointerTraits> Info;
+};
+
+/// GlobalsModRef - The actual analysis pass.
+class GlobalsModRef : public ModulePass, public AliasAnalysis {
+  /// The globals that do not have their addresses taken.
+  SmallPtrSet<const GlobalValue *, 8> NonAddressTakenGlobals;
+
+  /// IndirectGlobals - The memory pointed to by this global is known to be
+  /// 'owned' by the global.
+  SmallPtrSet<const GlobalValue *, 8> IndirectGlobals;
+
+  /// AllocsForIndirectGlobals - If an instruction allocates memory for an
+  /// indirect global, this map indicates which one.
+  DenseMap<const Value *, const GlobalValue *> AllocsForIndirectGlobals;
+
+  /// For each function, keep track of what globals are modified or read.
+  DenseMap<const Function *, FunctionInfo> FunctionInfos;
+
+  /// Handle to clear this analysis on deletion of values.
+  struct DeletionCallbackHandle final : CallbackVH {
+    GlobalsModRef &GMR;
+    std::list<DeletionCallbackHandle>::iterator I;
+
+    DeletionCallbackHandle(GlobalsModRef &GMR, Value *V)
+        : CallbackVH(V), GMR(GMR) {}
+
+    void deleted() override {
+      Value *V = getValPtr();
+      if (auto *F = dyn_cast<Function>(V))
+        GMR.FunctionInfos.erase(F);
+
+      if (GlobalValue *GV = dyn_cast<GlobalValue>(V)) {
+        if (GMR.NonAddressTakenGlobals.erase(GV)) {
+          // This global might be an indirect global.  If so, remove it and
+          // remove any AllocRelatedValues for it.
+          if (GMR.IndirectGlobals.erase(GV)) {
+            // Remove any entries in AllocsForIndirectGlobals for this global.
+            for (auto I = GMR.AllocsForIndirectGlobals.begin(),
+                      E = GMR.AllocsForIndirectGlobals.end();
+                 I != E; ++I)
+              if (I->second == GV)
+                GMR.AllocsForIndirectGlobals.erase(I);
+          }
 
-      return ModRefBehavior(AliasAnalysis::getModRefBehavior(F) & Min);
-    }
-    
-    /// getModRefBehavior - Return the behavior of the specified function if
-    /// called from the specified call site.  The call site may be null in which
-    /// case the most generic behavior of this function should be returned.
-    ModRefBehavior getModRefBehavior(ImmutableCallSite CS) override {
-      ModRefBehavior Min = UnknownModRefBehavior;
-
-      if (const Function* F = CS.getCalledFunction())
-        if (FunctionRecord *FR = getFunctionInfo(F)) {
-          if (FR->FunctionEffect == 0)
-            Min = DoesNotAccessMemory;
-          else if ((FR->FunctionEffect & Mod) == 0)
-            Min = OnlyReadsMemory;
+          // Scan the function info we have collected and remove this global
+          // from all of them.
+          for (auto &FIPair : GMR.FunctionInfos)
+            FIPair.second.eraseModRefInfoForGlobal(*GV);
         }
+      }
 
-      return ModRefBehavior(AliasAnalysis::getModRefBehavior(CS) & Min);
-    }
+      // If this is an allocation related to an indirect global, remove it.
+      GMR.AllocsForIndirectGlobals.erase(V);
 
-    void deleteValue(Value *V) override;
-    void copyValue(Value *From, Value *To) override;
-    void addEscapingUse(Use &U) override;
-
-    /// getAdjustedAnalysisPointer - This method is used when a pass implements
-    /// an analysis interface through multiple inheritance.  If needed, it
-    /// should override this to adjust the this pointer as needed for the
-    /// specified pass info.
-    void *getAdjustedAnalysisPointer(AnalysisID PI) override {
-      if (PI == &AliasAnalysis::ID)
-        return (AliasAnalysis*)this;
-      return this;
+      // And clear out the handle.
+      setValPtr(nullptr);
+      GMR.Handles.erase(I);
+      // This object is now destroyed!
     }
-    
-  private:
-    /// getFunctionInfo - Return the function info for the function, or null if
-    /// we don't have anything useful to say about it.
-    FunctionRecord *getFunctionInfo(const Function *F) {
-      std::map<const Function*, FunctionRecord>::iterator I =
-        FunctionInfo.find(F);
-      if (I != FunctionInfo.end())
-        return &I->second;
-      return nullptr;
+  };
+
+  /// List of callbacks for globals being tracked by this analysis. Note that
+  /// these objects are quite large, but we only anticipate having one per
+  /// global tracked by this analysis. There are numerous optimizations we
+  /// could perform to the memory utilization here if this becomes a problem.
+  std::list<DeletionCallbackHandle> Handles;
+
+public:
+  static char ID;
+  GlobalsModRef() : ModulePass(ID) {
+    initializeGlobalsModRefPass(*PassRegistry::getPassRegistry());
+  }
+
+  bool runOnModule(Module &M) override {
+    InitializeAliasAnalysis(this, &M.getDataLayout());
+
+    // Find non-addr taken globals.
+    AnalyzeGlobals(M);
+
+    // Propagate on CG.
+    AnalyzeCallGraph(getAnalysis<CallGraphWrapperPass>().getCallGraph(), M);
+    return false;
+  }
+
+  void getAnalysisUsage(AnalysisUsage &AU) const override {
+    AliasAnalysis::getAnalysisUsage(AU);
+    AU.addRequired<CallGraphWrapperPass>();
+    AU.setPreservesAll(); // Does not transform code
+  }
+
+  /// getAdjustedAnalysisPointer - This method is used when a pass implements
+  /// an analysis interface through multiple inheritance.  If needed, it
+  /// should override this to adjust the this pointer as needed for the
+  /// specified pass info.
+  void *getAdjustedAnalysisPointer(AnalysisID PI) override {
+    if (PI == &AliasAnalysis::ID)
+      return (AliasAnalysis *)this;
+    return this;
+  }
+
+  //------------------------------------------------
+  // Implement the AliasAnalysis API
+  //
+  AliasResult alias(const MemoryLocation &LocA,
+                    const MemoryLocation &LocB) override;
+  ModRefInfo getModRefInfo(ImmutableCallSite CS,
+                           const MemoryLocation &Loc) override;
+  ModRefInfo getModRefInfo(ImmutableCallSite CS1,
+                           ImmutableCallSite CS2) override {
+    return AliasAnalysis::getModRefInfo(CS1, CS2);
+  }
+
+  /// getModRefBehavior - Return the behavior of the specified function if
+  /// called from the specified call site.  The call site may be null in which
+  /// case the most generic behavior of this function should be returned.
+  FunctionModRefBehavior getModRefBehavior(const Function *F) override {
+    FunctionModRefBehavior Min = FMRB_UnknownModRefBehavior;
+
+    if (FunctionInfo *FI = getFunctionInfo(F)) {
+      if (FI->getModRefInfo() == MRI_NoModRef)
+        Min = FMRB_DoesNotAccessMemory;
+      else if ((FI->getModRefInfo() & MRI_Mod) == 0)
+        Min = FMRB_OnlyReadsMemory;
     }
 
-    void AnalyzeGlobals(Module &M);
-    void AnalyzeCallGraph(CallGraph &CG, Module &M);
-    bool AnalyzeUsesOfPointer(Value *V, std::vector<Function*> &Readers,
-                              std::vector<Function*> &Writers,
-                              GlobalValue *OkayStoreDest = nullptr);
-    bool AnalyzeIndirectGlobalMemory(GlobalValue *GV);
-  };
+    return FunctionModRefBehavior(AliasAnalysis::getModRefBehavior(F) & Min);
+  }
+
+  /// getModRefBehavior - Return the behavior of the specified function if
+  /// called from the specified call site.  The call site may be null in which
+  /// case the most generic behavior of this function should be returned.
+  FunctionModRefBehavior getModRefBehavior(ImmutableCallSite CS) override {
+    FunctionModRefBehavior Min = FMRB_UnknownModRefBehavior;
+
+    if (const Function *F = CS.getCalledFunction())
+      if (FunctionInfo *FI = getFunctionInfo(F)) {
+        if (FI->getModRefInfo() == MRI_NoModRef)
+          Min = FMRB_DoesNotAccessMemory;
+        else if ((FI->getModRefInfo() & MRI_Mod) == 0)
+          Min = FMRB_OnlyReadsMemory;
+      }
+
+    return FunctionModRefBehavior(AliasAnalysis::getModRefBehavior(CS) & Min);
+  }
+
+private:
+  /// Returns the function info for the function, or null if we don't have
+  /// anything useful to say about it.
+  FunctionInfo *getFunctionInfo(const Function *F) {
+    auto I = FunctionInfos.find(F);
+    if (I != FunctionInfos.end())
+      return &I->second;
+    return nullptr;
+  }
+
+  void AnalyzeGlobals(Module &M);
+  void AnalyzeCallGraph(CallGraph &CG, Module &M);
+  bool AnalyzeUsesOfPointer(Value *V,
+                            SmallPtrSetImpl<Function *> *Readers = nullptr,
+                            SmallPtrSetImpl<Function *> *Writers = nullptr,
+                            GlobalValue *OkayStoreDest = nullptr);
+  bool AnalyzeIndirectGlobalMemory(GlobalValue *GV);
+};
 }
 
 char GlobalsModRef::ID = 0;
-INITIALIZE_AG_PASS_BEGIN(GlobalsModRef, AliasAnalysis,
-                "globalsmodref-aa", "Simple mod/ref analysis for globals",    
-                false, true, false)
+INITIALIZE_AG_PASS_BEGIN(GlobalsModRef, AliasAnalysis, "globalsmodref-aa",
+                         "Simple mod/ref analysis for globals", false, true,
+                         false)
 INITIALIZE_PASS_DEPENDENCY(CallGraphWrapperPass)
-INITIALIZE_AG_PASS_END(GlobalsModRef, AliasAnalysis,
-                "globalsmodref-aa", "Simple mod/ref analysis for globals",    
-                false, true, false)
+INITIALIZE_AG_PASS_END(GlobalsModRef, AliasAnalysis, "globalsmodref-aa",
+                       "Simple mod/ref analysis for globals", false, true,
+                       false)
 
 Pass *llvm::createGlobalsModRefPass() { return new GlobalsModRef(); }
 
@@ -206,38 +375,53 @@ Pass *llvm::createGlobalsModRefPass() { return new GlobalsModRef(); }
 /// (really, their address passed to something nontrivial), record this fact,
 /// and record the functions that they are used directly in.
 void GlobalsModRef::AnalyzeGlobals(Module &M) {
-  std::vector<Function*> Readers, Writers;
-  for (Module::iterator I = M.begin(), E = M.end(); I != E; ++I)
-    if (I->hasLocalLinkage()) {
-      if (!AnalyzeUsesOfPointer(I, Readers, Writers)) {
+  SmallPtrSet<Function *, 64> TrackedFunctions;
+  for (Function &F : M)
+    if (F.hasLocalLinkage())
+      if (!AnalyzeUsesOfPointer(&F)) {
         // Remember that we are tracking this global.
-        NonAddressTakenGlobals.insert(I);
+        NonAddressTakenGlobals.insert(&F);
+        TrackedFunctions.insert(&F);
+        Handles.emplace_front(*this, &F);
+        Handles.front().I = Handles.begin();
         ++NumNonAddrTakenFunctions;
       }
-      Readers.clear(); Writers.clear();
-    }
 
-  for (Module::global_iterator I = M.global_begin(), E = M.global_end();
-       I != E; ++I)
-    if (I->hasLocalLinkage()) {
-      if (!AnalyzeUsesOfPointer(I, Readers, Writers)) {
+  SmallPtrSet<Function *, 64> Readers, Writers;
+  for (GlobalVariable &GV : M.globals())
+    if (GV.hasLocalLinkage()) {
+      if (!AnalyzeUsesOfPointer(&GV, &Readers,
+                                GV.isConstant() ? nullptr : &Writers)) {
         // Remember that we are tracking this global, and the mod/ref fns
-        NonAddressTakenGlobals.insert(I);
-
-        for (unsigned i = 0, e = Readers.size(); i != e; ++i)
-          FunctionInfo[Readers[i]].GlobalInfo[I] |= Ref;
+        NonAddressTakenGlobals.insert(&GV);
+        Handles.emplace_front(*this, &GV);
+        Handles.front().I = Handles.begin();
+
+        for (Function *Reader : Readers) {
+          if (TrackedFunctions.insert(Reader).second) {
+            Handles.emplace_front(*this, Reader);
+            Handles.front().I = Handles.begin();
+          }
+          FunctionInfos[Reader].addModRefInfoForGlobal(GV, MRI_Ref);
+        }
 
-        if (!I->isConstant())  // No need to keep track of writers to constants
-          for (unsigned i = 0, e = Writers.size(); i != e; ++i)
-            FunctionInfo[Writers[i]].GlobalInfo[I] |= Mod;
+        if (!GV.isConstant()) // No need to keep track of writers to constants
+          for (Function *Writer : Writers) {
+            if (TrackedFunctions.insert(Writer).second) {
+              Handles.emplace_front(*this, Writer);
+              Handles.front().I = Handles.begin();
+            }
+            FunctionInfos[Writer].addModRefInfoForGlobal(GV, MRI_Mod);
+          }
         ++NumNonAddrTakenGlobalVars;
 
         // If this global holds a pointer type, see if it is an indirect global.
-        if (I->getType()->getElementType()->isPointerTy() &&
-            AnalyzeIndirectGlobalMemory(I))
+        if (GV.getType()->getElementType()->isPointerTy() &&
+            AnalyzeIndirectGlobalMemory(&GV))
           ++NumIndirectGlobalVars;
       }
-      Readers.clear(); Writers.clear();
+      Readers.clear();
+      Writers.clear();
     }
 }
 
@@ -248,20 +432,23 @@ void GlobalsModRef::AnalyzeGlobals(Module &M) {
 ///
 /// If OkayStoreDest is non-null, stores into this global are allowed.
 bool GlobalsModRef::AnalyzeUsesOfPointer(Value *V,
-                                         std::vector<Function*> &Readers,
-                                         std::vector<Function*> &Writers,
+                                         SmallPtrSetImpl<Function *> *Readers,
+                                         SmallPtrSetImpl<Function *> *Writers,
                                          GlobalValue *OkayStoreDest) {
-  if (!V->getType()->isPointerTy()) return true;
+  if (!V->getType()->isPointerTy())
+    return true;
 
   for (Use &U : V->uses()) {
     User *I = U.getUser();
     if (LoadInst *LI = dyn_cast<LoadInst>(I)) {
-      Readers.push_back(LI->getParent()->getParent());
+      if (Readers)
+        Readers->insert(LI->getParent()->getParent());
     } else if (StoreInst *SI = dyn_cast<StoreInst>(I)) {
       if (V == SI->getOperand(1)) {
-        Writers.push_back(SI->getParent()->getParent());
+        if (Writers)
+          Writers->insert(SI->getParent()->getParent());
       } else if (SI->getOperand(1) != OkayStoreDest) {
-        return true;  // Storing the pointer
+        return true; // Storing the pointer
       }
     } else if (Operator::getOpcode(I) == Instruction::GetElementPtr) {
       if (AnalyzeUsesOfPointer(I, Readers, Writers))
@@ -274,14 +461,16 @@ bool GlobalsModRef::AnalyzeUsesOfPointer(Value *V,
       // passing into the function.
       if (!CS.isCallee(&U)) {
         // Detect calls to free.
-        if (isFreeCall(I, TLI))
-          Writers.push_back(CS->getParent()->getParent());
-        else
+        if (isFreeCall(I, TLI)) {
+          if (Writers)
+            Writers->insert(CS->getParent()->getParent());
+        } else {
           return true; // Argument of an unknown call.
+        }
       }
     } else if (ICmpInst *ICI = dyn_cast<ICmpInst>(I)) {
       if (!isa<ConstantPointerNull>(ICI->getOperand(1)))
-        return true;  // Allow comparison against null.
+        return true; // Allow comparison against null.
     } else {
       return true;
     }
@@ -300,7 +489,7 @@ bool GlobalsModRef::AnalyzeUsesOfPointer(Value *V,
 bool GlobalsModRef::AnalyzeIndirectGlobalMemory(GlobalValue *GV) {
   // Keep track of values related to the allocation of the memory, f.e. the
   // value produced by the malloc call and any casts.
-  std::vector<Value*> AllocRelatedValues;
+  std::vector<Value *> AllocRelatedValues;
 
   // Walk the user list of the global.  If we find anything other than a direct
   // load or store, bail out.
@@ -309,13 +498,13 @@ bool GlobalsModRef::AnalyzeIndirectGlobalMemory(GlobalValue *GV) {
       // The pointer loaded from the global can only be used in simple ways:
       // we allow addressing of it and loading storing to it.  We do *not* allow
       // storing the loaded pointer somewhere else or passing to a function.
-      std::vector<Function*> ReadersWriters;
-      if (AnalyzeUsesOfPointer(LI, ReadersWriters, ReadersWriters))
-        return false;  // Loaded pointer escapes.
+      if (AnalyzeUsesOfPointer(LI))
+        return false; // Loaded pointer escapes.
       // TODO: Could try some IP mod/ref of the loaded pointer.
     } else if (StoreInst *SI = dyn_cast<StoreInst>(U)) {
       // Storing the global itself.
-      if (SI->getOperand(0) == GV) return false;
+      if (SI->getOperand(0) == GV)
+        return false;
 
       // If storing the null pointer, ignore it.
       if (isa<ConstantPointerNull>(SI->getOperand(0)))
@@ -326,13 +515,13 @@ bool GlobalsModRef::AnalyzeIndirectGlobalMemory(GlobalValue *GV) {
                                        GV->getParent()->getDataLayout());
 
       if (!isAllocLikeFn(Ptr, TLI))
-        return false;  // Too hard to analyze.
+        return false; // Too hard to analyze.
 
       // Analyze all uses of the allocation.  If any of them are used in a
       // non-simple way (e.g. stored to another global) bail out.
-      std::vector<Function*> ReadersWriters;
-      if (AnalyzeUsesOfPointer(Ptr, ReadersWriters, ReadersWriters, GV))
-        return false;  // Loaded pointer escapes.
+      if (AnalyzeUsesOfPointer(Ptr, /*Readers*/ nullptr, /*Writers*/ nullptr,
+                               GV))
+        return false; // Loaded pointer escapes.
 
       // Remember that this allocation is related to the indirect global.
       AllocRelatedValues.push_back(Ptr);
@@ -346,9 +535,13 @@ bool GlobalsModRef::AnalyzeIndirectGlobalMemory(GlobalValue *GV) {
   // this global in AllocsForIndirectGlobals.
   while (!AllocRelatedValues.empty()) {
     AllocsForIndirectGlobals[AllocRelatedValues.back()] = GV;
+    Handles.emplace_front(*this, AllocRelatedValues.back());
+    Handles.front().I = Handles.begin();
     AllocRelatedValues.pop_back();
   }
   IndirectGlobals.insert(GV);
+  Handles.emplace_front(*this, GV);
+  Handles.front().I = Handles.begin();
   return true;
 }
 
@@ -359,22 +552,20 @@ bool GlobalsModRef::AnalyzeIndirectGlobalMemory(GlobalValue *GV) {
 void GlobalsModRef::AnalyzeCallGraph(CallGraph &CG, Module &M) {
   // We do a bottom-up SCC traversal of the call graph.  In other words, we
   // visit all callees before callers (leaf-first).
-  for (scc_iterator<CallGraph*> I = scc_begin(&CG); !I.isAtEnd(); ++I) {
+  for (scc_iterator<CallGraph *> I = scc_begin(&CG); !I.isAtEnd(); ++I) {
     const std::vector<CallGraphNode *> &SCC = *I;
     assert(!SCC.empty() && "SCC with no functions?");
 
     if (!SCC[0]->getFunction()) {
       // Calls externally - can't say anything useful.  Remove any existing
       // function records (may have been created when scanning globals).
-      for (unsigned i = 0, e = SCC.size(); i != e; ++i)
-        FunctionInfo.erase(SCC[i]->getFunction());
+      for (auto *Node : SCC)
+        FunctionInfos.erase(Node->getFunction());
       continue;
     }
 
-    FunctionRecord &FR = FunctionInfo[SCC[0]->getFunction()];
-
+    FunctionInfo &FI = FunctionInfos[SCC[0]->getFunction()];
     bool KnowNothing = false;
-    unsigned FunctionEffect = 0;
 
     // Collect the mod/ref properties due to called functions.  We only compute
     // one mod-ref set.
@@ -390,13 +581,13 @@ void GlobalsModRef::AnalyzeCallGraph(CallGraph &CG, Module &M) {
         if (F->doesNotAccessMemory()) {
           // Can't do better than that!
         } else if (F->onlyReadsMemory()) {
-          FunctionEffect |= Ref;
+          FI.addModRefInfo(MRI_Ref);
           if (!F->isIntrinsic())
             // This function might call back into the module and read a global -
             // consider every global as possibly being read by this function.
-            FR.MayReadAnyGlobal = true;
+            FI.setMayReadAnyGlobal();
         } else {
-          FunctionEffect |= ModRef;
+          FI.addModRefInfo(MRI_ModRef);
           // Can't say anything useful unless it's an intrinsic - they don't
           // read or write global variables of the kind considered here.
           KnowNothing = !F->isIntrinsic();
@@ -407,14 +598,9 @@ void GlobalsModRef::AnalyzeCallGraph(CallGraph &CG, Module &M) {
       for (CallGraphNode::iterator CI = SCC[i]->begin(), E = SCC[i]->end();
            CI != E && !KnowNothing; ++CI)
         if (Function *Callee = CI->second->getFunction()) {
-          if (FunctionRecord *CalleeFR = getFunctionInfo(Callee)) {
+          if (FunctionInfo *CalleeFI = getFunctionInfo(Callee)) {
             // Propagate function effect up.
-            FunctionEffect |= CalleeFR->FunctionEffect;
-
-            // Incorporate callee's effects on globals into our info.
-            for (const auto &G : CalleeFR->GlobalInfo)
-              FR.GlobalInfo[G.first] |= G.second;
-            FR.MayReadAnyGlobal |= CalleeFR->MayReadAnyGlobal;
+            FI.addFunctionInfo(*CalleeFI);
           } else {
             // Can't say anything about it.  However, if it is inside our SCC,
             // then nothing needs to be done.
@@ -428,59 +614,65 @@ void GlobalsModRef::AnalyzeCallGraph(CallGraph &CG, Module &M) {
     }
 
     // If we can't say anything useful about this SCC, remove all SCC functions
-    // from the FunctionInfo map.
+    // from the FunctionInfos map.
     if (KnowNothing) {
-      for (unsigned i = 0, e = SCC.size(); i != e; ++i)
-        FunctionInfo.erase(SCC[i]->getFunction());
+      for (auto *Node : SCC)
+        FunctionInfos.erase(Node->getFunction());
       continue;
     }
 
     // Scan the function bodies for explicit loads or stores.
-    for (unsigned i = 0, e = SCC.size(); i != e && FunctionEffect != ModRef;++i)
-      for (inst_iterator II = inst_begin(SCC[i]->getFunction()),
-             E = inst_end(SCC[i]->getFunction());
-           II != E && FunctionEffect != ModRef; ++II)
-        if (LoadInst *LI = dyn_cast<LoadInst>(&*II)) {
-          FunctionEffect |= Ref;
-          if (LI->isVolatile())
-            // Volatile loads may have side-effects, so mark them as writing
-            // memory (for example, a flag inside the processor).
-            FunctionEffect |= Mod;
-        } else if (StoreInst *SI = dyn_cast<StoreInst>(&*II)) {
-          FunctionEffect |= Mod;
-          if (SI->isVolatile())
-            // Treat volatile stores as reading memory somewhere.
-            FunctionEffect |= Ref;
-        } else if (isAllocationFn(&*II, TLI) || isFreeCall(&*II, TLI)) {
-          FunctionEffect |= ModRef;
-        } else if (IntrinsicInst *Intrinsic = dyn_cast<IntrinsicInst>(&*II)) {
-          // The callgraph doesn't include intrinsic calls.
-          Function *Callee = Intrinsic->getCalledFunction();
-          ModRefBehavior Behaviour = AliasAnalysis::getModRefBehavior(Callee);
-          FunctionEffect |= (Behaviour & ModRef);
+    for (auto *Node : SCC) {
+      if (FI.getModRefInfo() == MRI_ModRef)
+        break; // The mod/ref lattice saturates here.
+      for (Instruction &I : inst_range(Node->getFunction())) {
+        if (FI.getModRefInfo() == MRI_ModRef)
+          break; // The mod/ref lattice saturates here.
+
+        // We handle calls specially because the graph-relevant aspects are
+        // handled above.
+        if (auto CS = CallSite(&I)) {
+          if (isAllocationFn(&I, TLI) || isFreeCall(&I, TLI)) {
+            // FIXME: It is completely unclear why this is necessary and not
+            // handled by the above graph code.
+            FI.addModRefInfo(MRI_ModRef);
+          } else if (Function *Callee = CS.getCalledFunction()) {
+            // The callgraph doesn't include intrinsic calls.
+            if (Callee->isIntrinsic()) {
+              FunctionModRefBehavior Behaviour =
+                  AliasAnalysis::getModRefBehavior(Callee);
+              FI.addModRefInfo(ModRefInfo(Behaviour & MRI_ModRef));
+            }
+          }
+          continue;
         }
 
-    if ((FunctionEffect & Mod) == 0)
+        // All non-call instructions we use the primary predicates for whether
+        // thay read or write memory.
+        if (I.mayReadFromMemory())
+          FI.addModRefInfo(MRI_Ref);
+        if (I.mayWriteToMemory())
+          FI.addModRefInfo(MRI_Mod);
+      }
+    }
+
+    if ((FI.getModRefInfo() & MRI_Mod) == 0)
       ++NumReadMemFunctions;
-    if (FunctionEffect == 0)
+    if (FI.getModRefInfo() == MRI_NoModRef)
       ++NumNoMemFunctions;
-    FR.FunctionEffect = FunctionEffect;
 
     // Finally, now that we know the full effect on this SCC, clone the
     // information to each function in the SCC.
     for (unsigned i = 1, e = SCC.size(); i != e; ++i)
-      FunctionInfo[SCC[i]->getFunction()] = FR;
+      FunctionInfos[SCC[i]->getFunction()] = FI;
   }
 }
 
-
-
 /// alias - If one of the pointers is to a global that we are tracking, and the
 /// other is some random pointer, we know there cannot be an alias, because the
 /// address of the global isn't taken.
-AliasAnalysis::AliasResult
-GlobalsModRef::alias(const Location &LocA,
-                     const Location &LocB) {
+AliasResult GlobalsModRef::alias(const MemoryLocation &LocA,
+                                 const MemoryLocation &LocB) {
   // Get the base object these pointers point to.
   const Value *UV1 = GetUnderlyingObject(LocA.Ptr, *DL);
   const Value *UV2 = GetUnderlyingObject(LocB.Ptr, *DL);
@@ -492,14 +684,69 @@ GlobalsModRef::alias(const Location &LocA,
   if (GV1 || GV2) {
     // If the global's address is taken, pretend we don't know it's a pointer to
     // the global.
-    if (GV1 && !NonAddressTakenGlobals.count(GV1)) GV1 = nullptr;
-    if (GV2 && !NonAddressTakenGlobals.count(GV2)) GV2 = nullptr;
+    if (GV1 && !NonAddressTakenGlobals.count(GV1))
+      GV1 = nullptr;
+    if (GV2 && !NonAddressTakenGlobals.count(GV2))
+      GV2 = nullptr;
 
     // If the two pointers are derived from two different non-addr-taken
-    // globals, or if one is and the other isn't, we know these can't alias.
-    if ((GV1 || GV2) && GV1 != GV2)
+    // globals we know these can't alias.
+    if (GV1 && GV2 && GV1 != GV2)
       return NoAlias;
 
+    // If one is and the other isn't, it isn't strictly safe but we can fake
+    // this result if necessary for performance. This does not appear to be
+    // a common problem in practice.
+    if (EnableUnsafeGlobalsModRefAliasResults)
+      if ((GV1 || GV2) && GV1 != GV2)
+        return NoAlias;
+
+    // There are particular cases where we can conclude no-alias between
+    // a non-addr-taken global and some other underlying object. Specifically,
+    // a non-addr-taken global is known to not be escaped from any function. It
+    // is also incorrect for a transformation to introduce an escape of
+    // a global in a way that is observable when it was not there previously.
+    // One function being transformed to introduce an escape which could
+    // possibly be observed (via loading from a global or the return value for
+    // example) within another function is never safe. If the observation is
+    // made through non-atomic operations on different threads, it is
+    // a data-race and UB. If the observation is well defined, by being
+    // observed the transformation would have changed program behavior by
+    // introducing the observed escape, making it an invalid transform.
+    //
+    // This property does require that transformations which *temporarily*
+    // escape a global that was not previously escaped, prior to restoring
+    // it, cannot rely on the results of GMR::alias. This seems a reasonable
+    // restriction, although currently there is no way to enforce it. There is
+    // also no realistic optimization pass that would make this mistake. The
+    // closest example is a transformation pass which does reg2mem of SSA
+    // values but stores them into global variables temporarily before
+    // restoring the global variable's value. This could be useful to expose
+    // "benign" races for example. However, it seems reasonable to require that
+    // a pass which introduces escapes of global variables in this way to
+    // either not trust AA results while the escape is active, or to be forced
+    // to operate as a module pass that cannot co-exist with an alias analysis
+    // such as GMR.
+    if ((GV1 || GV2) && GV1 != GV2) {
+      const Value *UV = GV1 ? UV2 : UV1;
+
+      // In order to know that the underlying object cannot alias the
+      // non-addr-taken global, we must know that it would have to be an
+      // escape. Thus if the underlying object is a function argument, a load
+      // from a global, or the return of a function, it cannot alias.
+      if (isa<Argument>(UV) || isa<CallInst>(UV) || isa<InvokeInst>(UV)) {
+        // Arguments to functions or returns from functions are inherently
+        // escaping, so we can immediately classify those as not aliasing any
+        // non-addr-taken globals.
+        return NoAlias;
+      } else if (auto *LI = dyn_cast<LoadInst>(UV)) {
+        // A pointer loaded from a global would have been captured, and we know
+        // that GV is non-addr-taken, so no alias.
+        if (isa<GlobalValue>(LI->getPointerOperand()))
+          return NoAlias;
+      }
+    }
+
     // Otherwise if they are both derived from the same addr-taken global, we
     // can't know the two accesses don't overlap.
   }
@@ -519,25 +766,30 @@ GlobalsModRef::alias(const Location &LocA,
 
   // These pointers may also be from an allocation for the indirect global.  If
   // so, also handle them.
-  if (AllocsForIndirectGlobals.count(UV1))
-    GV1 = AllocsForIndirectGlobals[UV1];
-  if (AllocsForIndirectGlobals.count(UV2))
-    GV2 = AllocsForIndirectGlobals[UV2];
+  if (!GV1)
+    GV1 = AllocsForIndirectGlobals.lookup(UV1);
+  if (!GV2)
+    GV2 = AllocsForIndirectGlobals.lookup(UV2);
 
   // Now that we know whether the two pointers are related to indirect globals,
-  // use this to disambiguate the pointers.  If either pointer is based on an
-  // indirect global and if they are not both based on the same indirect global,
-  // they cannot alias.
-  if ((GV1 || GV2) && GV1 != GV2)
+  // use this to disambiguate the pointers. If the pointers are based on
+  // different indirect globals they cannot alias.
+  if (GV1 && GV2 && GV1 != GV2)
     return NoAlias;
 
+  // If one is based on an indirect global and the other isn't, it isn't
+  // strictly safe but we can fake this result if necessary for performance.
+  // This does not appear to be a common problem in practice.
+  if (EnableUnsafeGlobalsModRefAliasResults)
+    if ((GV1 || GV2) && GV1 != GV2)
+      return NoAlias;
+
   return AliasAnalysis::alias(LocA, LocB);
 }
 
-AliasAnalysis::ModRefResult
-GlobalsModRef::getModRefInfo(ImmutableCallSite CS,
-                             const Location &Loc) {
-  unsigned Known = ModRef;
+ModRefInfo GlobalsModRef::getModRefInfo(ImmutableCallSite CS,
+                                        const MemoryLocation &Loc) {
+  unsigned Known = MRI_ModRef;
 
   // If we are asking for mod/ref info of a direct call with a pointer to a
   // global we are tracking, return information if we have it.
@@ -547,55 +799,10 @@ GlobalsModRef::getModRefInfo(ImmutableCallSite CS,
     if (GV->hasLocalLinkage())
       if (const Function *F = CS.getCalledFunction())
         if (NonAddressTakenGlobals.count(GV))
-          if (const FunctionRecord *FR = getFunctionInfo(F))
-            Known = FR->getInfoForGlobal(GV);
-
-  if (Known == NoModRef)
-    return NoModRef; // No need to query other mod/ref analyses
-  return ModRefResult(Known & AliasAnalysis::getModRefInfo(CS, Loc));
-}
-
-
-//===----------------------------------------------------------------------===//
-// Methods to update the analysis as a result of the client transformation.
-//
-void GlobalsModRef::deleteValue(Value *V) {
-  if (GlobalValue *GV = dyn_cast<GlobalValue>(V)) {
-    if (NonAddressTakenGlobals.erase(GV)) {
-      // This global might be an indirect global.  If so, remove it and remove
-      // any AllocRelatedValues for it.
-      if (IndirectGlobals.erase(GV)) {
-        // Remove any entries in AllocsForIndirectGlobals for this global.
-        for (std::map<const Value*, const GlobalValue*>::iterator
-             I = AllocsForIndirectGlobals.begin(),
-             E = AllocsForIndirectGlobals.end(); I != E; ) {
-          if (I->second == GV) {
-            AllocsForIndirectGlobals.erase(I++);
-          } else {
-            ++I;
-          }
-        }
-      }
-    }
-  }
-
-  // Otherwise, if this is an allocation related to an indirect global, remove
-  // it.
-  AllocsForIndirectGlobals.erase(V);
-
-  AliasAnalysis::deleteValue(V);
-}
-
-void GlobalsModRef::copyValue(Value *From, Value *To) {
-  AliasAnalysis::copyValue(From, To);
-}
+          if (const FunctionInfo *FI = getFunctionInfo(F))
+            Known = FI->getModRefInfoForGlobal(*GV);
 
-void GlobalsModRef::addEscapingUse(Use &U) {
-  // For the purposes of this analysis, it is conservatively correct to treat
-  // a newly escaping value equivalently to a deleted one.  We could perhaps
-  // be more precise by processing the new use and attempting to update our
-  // saved analysis results to accommodate it.
-  deleteValue(U);
-  
-  AliasAnalysis::addEscapingUse(U);
+  if (Known == MRI_NoModRef)
+    return MRI_NoModRef; // No need to query other mod/ref analyses
+  return ModRefInfo(Known & AliasAnalysis::getModRefInfo(CS, Loc));
 }
diff --git a/lib/Analysis/IPA/InlineCost.cpp b/lib/Analysis/IPA/InlineCost.cpp
index cacf70d041b..40bfac56719 100644
--- a/lib/Analysis/IPA/InlineCost.cpp
+++ b/lib/Analysis/IPA/InlineCost.cpp
@@ -54,6 +54,11 @@ class CallAnalyzer : public InstVisitor<CallAnalyzer, bool> {
   // The called function.
   Function &F;
 
+  // The candidate callsite being analyzed. Please do not use this to do
+  // analysis in the caller function; we want the inline cost query to be
+  // easily cacheable. Instead, use the cover function paramHasAttr.
+  CallSite CandidateCS;
+
   int Threshold;
   int Cost;
 
@@ -106,6 +111,17 @@ class CallAnalyzer : public InstVisitor<CallAnalyzer, bool> {
   bool simplifyCallSite(Function *F, CallSite CS);
   ConstantInt *stripAndComputeInBoundsConstantOffsets(Value *&V);
 
+  /// Return true if the given argument to the function being considered for
+  /// inlining has the given attribute set either at the call site or the
+  /// function declaration.  Primarily used to inspect call site specific
+  /// attributes since these can be more precise than the ones on the callee
+  /// itself. 
+  bool paramHasAttr(Argument *A, Attribute::AttrKind Attr);
+  
+  /// Return true if the given value is known non null within the callee if
+  /// inlined through this particular callsite. 
+  bool isKnownNonNullInCallee(Value *V);
+
   // Custom analysis routines.
   bool analyzeBlock(BasicBlock *BB, SmallPtrSetImpl<const Value *> &EphValues);
 
@@ -140,13 +156,15 @@ class CallAnalyzer : public InstVisitor<CallAnalyzer, bool> {
   bool visitSwitchInst(SwitchInst &SI);
   bool visitIndirectBrInst(IndirectBrInst &IBI);
   bool visitResumeInst(ResumeInst &RI);
+  bool visitCleanupReturnInst(CleanupReturnInst &RI);
+  bool visitCatchReturnInst(CatchReturnInst &RI);
   bool visitUnreachableInst(UnreachableInst &I);
 
 public:
   CallAnalyzer(const TargetTransformInfo &TTI, AssumptionCacheTracker *ACT,
-               Function &Callee, int Threshold)
-      : TTI(TTI), ACT(ACT), F(Callee), Threshold(Threshold), Cost(0),
-        IsCallerRecursive(false), IsRecursiveCall(false),
+               Function &Callee, int Threshold, CallSite CSArg)
+    : TTI(TTI), ACT(ACT), F(Callee), CandidateCS(CSArg), Threshold(Threshold),
+        Cost(0), IsCallerRecursive(false), IsRecursiveCall(false),
         ExposesReturnsTwice(false), HasDynamicAlloca(false),
         ContainsNoDuplicateCall(false), HasReturn(false), HasIndirectBr(false),
         HasFrameEscape(false), AllocatedSize(0), NumInstructions(0),
@@ -496,6 +514,33 @@ bool CallAnalyzer::visitUnaryInstruction(UnaryInstruction &I) {
   return false;
 }
 
+bool CallAnalyzer::paramHasAttr(Argument *A, Attribute::AttrKind Attr) {
+  unsigned ArgNo = A->getArgNo();
+  return CandidateCS.paramHasAttr(ArgNo+1, Attr);
+}
+
+bool CallAnalyzer::isKnownNonNullInCallee(Value *V) {
+  // Does the *call site* have the NonNull attribute set on an argument?  We
+  // use the attribute on the call site to memoize any analysis done in the
+  // caller. This will also trip if the callee function has a non-null
+  // parameter attribute, but that's a less interesting case because hopefully
+  // the callee would already have been simplified based on that.
+  if (Argument *A = dyn_cast<Argument>(V))
+    if (paramHasAttr(A, Attribute::NonNull))
+      return true;
+  
+  // Is this an alloca in the caller?  This is distinct from the attribute case
+  // above because attributes aren't updated within the inliner itself and we
+  // always want to catch the alloca derived case.
+  if (isAllocaDerivedArg(V))
+    // We can actually predict the result of comparisons between an
+    // alloca-derived value and null. Note that this fires regardless of
+    // SROA firing.
+    return true;
+  
+  return false;
+}
+
 bool CallAnalyzer::visitCmpInst(CmpInst &I) {
   Value *LHS = I.getOperand(0), *RHS = I.getOperand(1);
   // First try to handle simplified comparisons.
@@ -537,18 +582,14 @@ bool CallAnalyzer::visitCmpInst(CmpInst &I) {
   }
 
   // If the comparison is an equality comparison with null, we can simplify it
-  // for any alloca-derived argument.
-  if (I.isEquality() && isa<ConstantPointerNull>(I.getOperand(1)))
-    if (isAllocaDerivedArg(I.getOperand(0))) {
-      // We can actually predict the result of comparisons between an
-      // alloca-derived value and null. Note that this fires regardless of
-      // SROA firing.
-      bool IsNotEqual = I.getPredicate() == CmpInst::ICMP_NE;
-      SimplifiedValues[&I] = IsNotEqual ? ConstantInt::getTrue(I.getType())
-                                        : ConstantInt::getFalse(I.getType());
-      return true;
-    }
-
+  // if we know the value (argument) can't be null
+  if (I.isEquality() && isa<ConstantPointerNull>(I.getOperand(1)) &&
+      isKnownNonNullInCallee(I.getOperand(0))) {
+    bool IsNotEqual = I.getPredicate() == CmpInst::ICMP_NE;
+    SimplifiedValues[&I] = IsNotEqual ? ConstantInt::getTrue(I.getType())
+                                      : ConstantInt::getFalse(I.getType());
+    return true;
+  }
   // Finally check for SROA candidates in comparisons.
   Value *SROAArg;
   DenseMap<Value *, int>::iterator CostIt;
@@ -744,7 +785,7 @@ bool CallAnalyzer::visitCallSite(CallSite CS) {
       case Intrinsic::memmove:
         // SROA can usually chew through these intrinsics, but they aren't free.
         return false;
-      case Intrinsic::frameescape:
+      case Intrinsic::localescape:
         HasFrameEscape = true;
         return false;
       }
@@ -790,7 +831,7 @@ bool CallAnalyzer::visitCallSite(CallSite CS) {
   // during devirtualization and so we want to give it a hefty bonus for
   // inlining, but cap that bonus in the event that inlining wouldn't pan
   // out. Pretend to inline the function, with a custom threshold.
-  CallAnalyzer CA(TTI, ACT, *F, InlineConstants::IndirectCallThreshold);
+  CallAnalyzer CA(TTI, ACT, *F, InlineConstants::IndirectCallThreshold, CS);
   if (CA.analyzeCall(CS)) {
     // We were able to inline the indirect call! Subtract the cost from the
     // bonus we want to apply, but don't go below zero.
@@ -864,6 +905,18 @@ bool CallAnalyzer::visitResumeInst(ResumeInst &RI) {
   return false;
 }
 
+bool CallAnalyzer::visitCleanupReturnInst(CleanupReturnInst &CRI) {
+  // FIXME: It's not clear that a single instruction is an accurate model for
+  // the inline cost of a cleanupret instruction.
+  return false;
+}
+
+bool CallAnalyzer::visitCatchReturnInst(CatchReturnInst &CRI) {
+  // FIXME: It's not clear that a single instruction is an accurate model for
+  // the inline cost of a cleanupret instruction.
+  return false;
+}
+
 bool CallAnalyzer::visitUnreachableInst(UnreachableInst &I) {
   // FIXME: It might be reasonably to discount the cost of instructions leading
   // to unreachable as they have the lowest possible impact on both runtime and
@@ -955,16 +1008,9 @@ bool CallAnalyzer::analyzeBlock(BasicBlock *BB,
         AllocatedSize > InlineConstants::TotalAllocaSizeRecursiveCaller)
       return false;
 
-    if (NumVectorInstructions > NumInstructions/2)
-      VectorBonus = FiftyPercentVectorBonus;
-    else if (NumVectorInstructions > NumInstructions/10)
-      VectorBonus = TenPercentVectorBonus;
-    else
-      VectorBonus = 0;
-
-    // Check if we've past the threshold so we don't spin in huge basic
-    // blocks that will never inline.
-    if (Cost > (Threshold + VectorBonus))
+    // Check if we've past the maximum possible threshold so we don't spin in
+    // huge basic blocks that will never inline.
+    if (Cost > Threshold)
       return false;
   }
 
@@ -1020,25 +1066,33 @@ ConstantInt *CallAnalyzer::stripAndComputeInBoundsConstantOffsets(Value *&V) {
 bool CallAnalyzer::analyzeCall(CallSite CS) {
   ++NumCallsAnalyzed;
 
-  // Track whether the post-inlining function would have more than one basic
-  // block. A single basic block is often intended for inlining. Balloon the
-  // threshold by 50% until we pass the single-BB phase.
-  bool SingleBB = true;
-  int SingleBBBonus = Threshold / 2;
-  Threshold += SingleBBBonus;
-
   // Perform some tweaks to the cost and threshold based on the direct
   // callsite information.
 
   // We want to more aggressively inline vector-dense kernels, so up the
   // threshold, and we'll lower it if the % of vector instructions gets too
-  // low.
+  // low. Note that these bonuses are some what arbitrary and evolved over time
+  // by accident as much as because they are principled bonuses.
+  //
+  // FIXME: It would be nice to remove all such bonuses. At least it would be
+  // nice to base the bonus values on something more scientific.
   assert(NumInstructions == 0);
   assert(NumVectorInstructions == 0);
-  FiftyPercentVectorBonus = Threshold;
-  TenPercentVectorBonus = Threshold / 2;
+  FiftyPercentVectorBonus = 3 * Threshold / 2;
+  TenPercentVectorBonus = 3 * Threshold / 4;
   const DataLayout &DL = F.getParent()->getDataLayout();
 
+  // Track whether the post-inlining function would have more than one basic
+  // block. A single basic block is often intended for inlining. Balloon the
+  // threshold by 50% until we pass the single-BB phase.
+  bool SingleBB = true;
+  int SingleBBBonus = Threshold / 2;
+
+  // Speculatively apply all possible bonuses to Threshold. If cost exceeds
+  // this Threshold any time, and cost cannot decrease, we can stop processing
+  // the rest of the function body.
+  Threshold += (SingleBBBonus + FiftyPercentVectorBonus);
+
   // Give out bonuses per argument, as the instructions setting them up will
   // be gone after inlining.
   for (unsigned I = 0, E = CS.arg_size(); I != E; ++I) {
@@ -1081,9 +1135,9 @@ bool CallAnalyzer::analyzeCall(CallSite CS) {
   Instruction *Instr = CS.getInstruction();
   if (InvokeInst *II = dyn_cast<InvokeInst>(Instr)) {
     if (isa<UnreachableInst>(II->getNormalDest()->begin()))
-      Threshold = 1;
+      Threshold = 0;
   } else if (isa<UnreachableInst>(++BasicBlock::iterator(Instr)))
-    Threshold = 1;
+    Threshold = 0;
 
   // If this function uses the coldcc calling convention, prefer not to inline
   // it.
@@ -1155,7 +1209,7 @@ bool CallAnalyzer::analyzeCall(CallSite CS) {
   for (unsigned Idx = 0; Idx != BBWorklist.size(); ++Idx) {
     // Bail out the moment we cross the threshold. This means we'll under-count
     // the cost, but only when undercounting doesn't matter.
-    if (Cost > (Threshold + VectorBonus))
+    if (Cost > Threshold)
       break;
 
     BasicBlock *BB = BBWorklist[Idx];
@@ -1233,7 +1287,13 @@ bool CallAnalyzer::analyzeCall(CallSite CS) {
   if (!OnlyOneCallAndLocalLinkage && ContainsNoDuplicateCall)
     return false;
 
-  Threshold += VectorBonus;
+  // We applied the maximum possible vector bonus at the beginning. Now,
+  // subtract the excess bonus, if any, from the Threshold before
+  // comparing against Cost.
+  if (NumVectorInstructions <= NumInstructions / 10)
+    Threshold -= FiftyPercentVectorBonus;
+  else if (NumVectorInstructions <= NumInstructions / 2)
+    Threshold -= (FiftyPercentVectorBonus - TenPercentVectorBonus);
 
   return Cost < Threshold;
 }
@@ -1248,12 +1308,12 @@ void CallAnalyzer::dump() {
   DEBUG_PRINT_STAT(NumConstantPtrCmps);
   DEBUG_PRINT_STAT(NumConstantPtrDiffs);
   DEBUG_PRINT_STAT(NumInstructionsSimplified);
+  DEBUG_PRINT_STAT(NumInstructions);
   DEBUG_PRINT_STAT(SROACostSavings);
   DEBUG_PRINT_STAT(SROACostSavingsLost);
   DEBUG_PRINT_STAT(ContainsNoDuplicateCall);
   DEBUG_PRINT_STAT(Cost);
   DEBUG_PRINT_STAT(Threshold);
-  DEBUG_PRINT_STAT(VectorBonus);
 #undef DEBUG_PRINT_STAT
 }
 #endif
@@ -1298,9 +1358,9 @@ static bool attributeMatches(Function *F1, Function *F2, AttrKind Attr) {
 /// \brief Test that there are no attribute conflicts between Caller and Callee
 ///        that prevent inlining.
 static bool functionsHaveCompatibleAttributes(Function *Caller,
-                                              Function *Callee) {
-  return attributeMatches(Caller, Callee, "target-cpu") &&
-         attributeMatches(Caller, Callee, "target-features") &&
+                                              Function *Callee,
+                                              TargetTransformInfo &TTI) {
+  return TTI.areInlineCompatible(Caller, Callee) &&
          attributeMatches(Caller, Callee, Attribute::SanitizeAddress) &&
          attributeMatches(Caller, Callee, Attribute::SanitizeMemory) &&
          attributeMatches(Caller, Callee, Attribute::SanitizeThread);
@@ -1322,7 +1382,8 @@ InlineCost InlineCostAnalysis::getInlineCost(CallSite CS, Function *Callee,
 
   // Never inline functions with conflicting attributes (unless callee has
   // always-inline attribute).
-  if (!functionsHaveCompatibleAttributes(CS.getCaller(), Callee))
+  if (!functionsHaveCompatibleAttributes(CS.getCaller(), Callee,
+                                         TTIWP->getTTI(*Callee)))
     return llvm::InlineCost::getNever();
 
   // Don't inline this call if the caller has the optnone attribute.
@@ -1339,7 +1400,7 @@ InlineCost InlineCostAnalysis::getInlineCost(CallSite CS, Function *Callee,
   DEBUG(llvm::dbgs() << "      Analyzing call of " << Callee->getName()
         << "...\n");
 
-  CallAnalyzer CA(TTIWP->getTTI(*Callee), ACT, *Callee, Threshold);
+  CallAnalyzer CA(TTIWP->getTTI(*Callee), ACT, *Callee, Threshold, CS);
   bool ShouldInline = CA.analyzeCall(CS);
 
   DEBUG(CA.dump());
@@ -1377,11 +1438,11 @@ bool InlineCostAnalysis::isInlineViable(Function &F) {
           cast<CallInst>(CS.getInstruction())->canReturnTwice())
         return false;
 
-      // Disallow inlining functions that call @llvm.frameescape. Doing this
+      // Disallow inlining functions that call @llvm.localescape. Doing this
       // correctly would require major changes to the inliner.
       if (CS.getCalledFunction() &&
           CS.getCalledFunction()->getIntrinsicID() ==
-              llvm::Intrinsic::frameescape)
+              llvm::Intrinsic::localescape)
         return false;
     }
   }
diff --git a/lib/Analysis/IVUsers.cpp b/lib/Analysis/IVUsers.cpp
index b88b2496b87..926787d3be9 100644
--- a/lib/Analysis/IVUsers.cpp
+++ b/lib/Analysis/IVUsers.cpp
@@ -12,8 +12,10 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "llvm/Analysis/IVUsers.h"
 #include "llvm/ADT/STLExtras.h"
+#include "llvm/Analysis/AssumptionCache.h"
+#include "llvm/Analysis/CodeMetrics.h"
+#include "llvm/Analysis/IVUsers.h"
 #include "llvm/Analysis/LoopPass.h"
 #include "llvm/Analysis/ScalarEvolutionExpressions.h"
 #include "llvm/Analysis/ValueTracking.h"
@@ -34,6 +36,7 @@ using namespace llvm;
 char IVUsers::ID = 0;
 INITIALIZE_PASS_BEGIN(IVUsers, "iv-users",
                       "Induction Variable Users", false, true)
+INITIALIZE_PASS_DEPENDENCY(AssumptionCacheTracker)
 INITIALIZE_PASS_DEPENDENCY(LoopInfoWrapperPass)
 INITIALIZE_PASS_DEPENDENCY(DominatorTreeWrapperPass)
 INITIALIZE_PASS_DEPENDENCY(ScalarEvolution)
@@ -137,6 +140,11 @@ bool IVUsers::AddUsersImpl(Instruction *I,
   if (Width > 64 || !DL.isLegalInteger(Width))
     return false;
 
+  // Don't attempt to promote ephemeral values to indvars. They will be removed
+  // later anyway.
+  if (EphValues.count(I))
+    return false;
+
   // Get the symbolic expression for this instruction.
   const SCEV *ISE = SE->getSCEV(I);
 
@@ -244,6 +252,7 @@ IVUsers::IVUsers()
 }
 
 void IVUsers::getAnalysisUsage(AnalysisUsage &AU) const {
+  AU.addRequired<AssumptionCacheTracker>();
   AU.addRequired<LoopInfoWrapperPass>();
   AU.addRequired<DominatorTreeWrapperPass>();
   AU.addRequired<ScalarEvolution>();
@@ -253,10 +262,16 @@ void IVUsers::getAnalysisUsage(AnalysisUsage &AU) const {
 bool IVUsers::runOnLoop(Loop *l, LPPassManager &LPM) {
 
   L = l;
+  AC = &getAnalysis<AssumptionCacheTracker>().getAssumptionCache(
+      *L->getHeader()->getParent());
   LI = &getAnalysis<LoopInfoWrapperPass>().getLoopInfo();
   DT = &getAnalysis<DominatorTreeWrapperPass>().getDomTree();
   SE = &getAnalysis<ScalarEvolution>();
 
+  // Collect ephemeral values so that AddUsersIfInteresting skips them.
+  EphValues.clear();
+  CodeMetrics::collectEphemeralValues(L, AC, EphValues);
+
   // Find all uses of induction variables in this loop, and categorize
   // them by stride.  Start by finding all of the PHI nodes in the header for
   // this loop.  If they are induction variables, inspect their uses.
diff --git a/lib/Analysis/InstructionSimplify.cpp b/lib/Analysis/InstructionSimplify.cpp
index d45f7bd66ff..fa42b48b6cd 100644
--- a/lib/Analysis/InstructionSimplify.cpp
+++ b/lib/Analysis/InstructionSimplify.cpp
@@ -24,6 +24,7 @@
 #include "llvm/Analysis/ConstantFolding.h"
 #include "llvm/Analysis/MemoryBuiltins.h"
 #include "llvm/Analysis/ValueTracking.h"
+#include "llvm/Analysis/VectorUtils.h"
 #include "llvm/IR/ConstantRange.h"
 #include "llvm/IR/DataLayout.h"
 #include "llvm/IR/Dominators.h"
@@ -469,8 +470,7 @@ static Value *ThreadBinOpOverPHI(unsigned Opcode, Value *LHS, Value *RHS,
 
   // Evaluate the BinOp on the incoming phi values.
   Value *CommonValue = nullptr;
-  for (unsigned i = 0, e = PI->getNumIncomingValues(); i != e; ++i) {
-    Value *Incoming = PI->getIncomingValue(i);
+  for (Value *Incoming : PI->incoming_values()) {
     // If the incoming value is the phi node itself, it can safely be skipped.
     if (Incoming == PI) continue;
     Value *V = PI == LHS ?
@@ -510,8 +510,7 @@ static Value *ThreadCmpOverPHI(CmpInst::Predicate Pred, Value *LHS, Value *RHS,
 
   // Evaluate the BinOp on the incoming phi values.
   Value *CommonValue = nullptr;
-  for (unsigned i = 0, e = PI->getNumIncomingValues(); i != e; ++i) {
-    Value *Incoming = PI->getIncomingValue(i);
+  for (Value *Incoming : PI->incoming_values()) {
     // If the incoming value is the phi node itself, it can safely be skipped.
     if (Incoming == PI) continue;
     Value *V = SimplifyCmpInst(Pred, Incoming, RHS, Q, MaxRecurse);
@@ -856,8 +855,8 @@ static Value *SimplifyFSubInst(Value *Op0, Value *Op1, FastMathFlags FMF,
       return X;
   }
 
-  // fsub nnan ninf x, x ==> 0.0
-  if (FMF.noNaNs() && FMF.noInfs() && Op0 == Op1)
+  // fsub nnan x, x ==> 0.0
+  if (FMF.noNaNs() && Op0 == Op1)
     return Constant::getNullValue(Op0->getType());
 
   return nullptr;
@@ -1128,6 +1127,21 @@ static Value *SimplifyFDivInst(Value *Op0, Value *Op1, FastMathFlags FMF,
   if (FMF.noNaNs() && FMF.noSignedZeros() && match(Op0, m_AnyZero()))
     return Op0;
 
+  if (FMF.noNaNs()) {
+    // X / X -> 1.0 is legal when NaNs are ignored.
+    if (Op0 == Op1)
+      return ConstantFP::get(Op0->getType(), 1.0);
+
+    // -X /  X -> -1.0 and
+    //  X / -X -> -1.0 are legal when NaNs are ignored.
+    // We can ignore signed zeros because +-0.0/+-0.0 is NaN and ignored.
+    if ((BinaryOperator::isFNeg(Op0, /*IgnoreZeroSign=*/true) &&
+         BinaryOperator::getFNegArgument(Op0) == Op1) ||
+        (BinaryOperator::isFNeg(Op1, /*IgnoreZeroSign=*/true) &&
+         BinaryOperator::getFNegArgument(Op1) == Op0))
+      return ConstantFP::get(Op0->getType(), -1.0);
+  }
+
   return nullptr;
 }
 
@@ -3033,7 +3047,8 @@ Value *llvm::SimplifyICmpInst(unsigned Predicate, Value *LHS, Value *RHS,
 /// SimplifyFCmpInst - Given operands for an FCmpInst, see if we can
 /// fold the result.  If not, this returns null.
 static Value *SimplifyFCmpInst(unsigned Predicate, Value *LHS, Value *RHS,
-                               const Query &Q, unsigned MaxRecurse) {
+                               FastMathFlags FMF, const Query &Q,
+                               unsigned MaxRecurse) {
   CmpInst::Predicate Pred = (CmpInst::Predicate)Predicate;
   assert(CmpInst::isFPPredicate(Pred) && "Not an FP compare!");
 
@@ -3052,6 +3067,14 @@ static Value *SimplifyFCmpInst(unsigned Predicate, Value *LHS, Value *RHS,
   if (Pred == FCmpInst::FCMP_TRUE)
     return ConstantInt::get(GetCompareTy(LHS), 1);
 
+  // UNO/ORD predicates can be trivially folded if NaNs are ignored.
+  if (FMF.noNaNs()) {
+    if (Pred == FCmpInst::FCMP_UNO)
+      return ConstantInt::get(GetCompareTy(LHS), 0);
+    if (Pred == FCmpInst::FCMP_ORD)
+      return ConstantInt::get(GetCompareTy(LHS), 1);
+  }
+
   // fcmp pred x, undef  and  fcmp pred undef, x
   // fold to true if unordered, false if ordered
   if (isa<UndefValue>(LHS) || isa<UndefValue>(RHS)) {
@@ -3138,12 +3161,96 @@ static Value *SimplifyFCmpInst(unsigned Predicate, Value *LHS, Value *RHS,
 }
 
 Value *llvm::SimplifyFCmpInst(unsigned Predicate, Value *LHS, Value *RHS,
-                              const DataLayout &DL,
+                              FastMathFlags FMF, const DataLayout &DL,
                               const TargetLibraryInfo *TLI,
                               const DominatorTree *DT, AssumptionCache *AC,
                               const Instruction *CxtI) {
-  return ::SimplifyFCmpInst(Predicate, LHS, RHS, Query(DL, TLI, DT, AC, CxtI),
-                            RecursionLimit);
+  return ::SimplifyFCmpInst(Predicate, LHS, RHS, FMF,
+                            Query(DL, TLI, DT, AC, CxtI), RecursionLimit);
+}
+
+/// SimplifyWithOpReplaced - See if V simplifies when its operand Op is
+/// replaced with RepOp.
+static const Value *SimplifyWithOpReplaced(Value *V, Value *Op, Value *RepOp,
+                                           const Query &Q,
+                                           unsigned MaxRecurse) {
+  // Trivial replacement.
+  if (V == Op)
+    return RepOp;
+
+  auto *I = dyn_cast<Instruction>(V);
+  if (!I)
+    return nullptr;
+
+  // If this is a binary operator, try to simplify it with the replaced op.
+  if (auto *B = dyn_cast<BinaryOperator>(I)) {
+    // Consider:
+    //   %cmp = icmp eq i32 %x, 2147483647
+    //   %add = add nsw i32 %x, 1
+    //   %sel = select i1 %cmp, i32 -2147483648, i32 %add
+    //
+    // We can't replace %sel with %add unless we strip away the flags.
+    if (isa<OverflowingBinaryOperator>(B))
+      if (B->hasNoSignedWrap() || B->hasNoUnsignedWrap())
+        return nullptr;
+    if (isa<PossiblyExactOperator>(B))
+      if (B->isExact())
+        return nullptr;
+
+    if (MaxRecurse) {
+      if (B->getOperand(0) == Op)
+        return SimplifyBinOp(B->getOpcode(), RepOp, B->getOperand(1), Q,
+                             MaxRecurse - 1);
+      if (B->getOperand(1) == Op)
+        return SimplifyBinOp(B->getOpcode(), B->getOperand(0), RepOp, Q,
+                             MaxRecurse - 1);
+    }
+  }
+
+  // Same for CmpInsts.
+  if (CmpInst *C = dyn_cast<CmpInst>(I)) {
+    if (MaxRecurse) {
+      if (C->getOperand(0) == Op)
+        return SimplifyCmpInst(C->getPredicate(), RepOp, C->getOperand(1), Q,
+                               MaxRecurse - 1);
+      if (C->getOperand(1) == Op)
+        return SimplifyCmpInst(C->getPredicate(), C->getOperand(0), RepOp, Q,
+                               MaxRecurse - 1);
+    }
+  }
+
+  // TODO: We could hand off more cases to instsimplify here.
+
+  // If all operands are constant after substituting Op for RepOp then we can
+  // constant fold the instruction.
+  if (Constant *CRepOp = dyn_cast<Constant>(RepOp)) {
+    // Build a list of all constant operands.
+    SmallVector<Constant *, 8> ConstOps;
+    for (unsigned i = 0, e = I->getNumOperands(); i != e; ++i) {
+      if (I->getOperand(i) == Op)
+        ConstOps.push_back(CRepOp);
+      else if (Constant *COp = dyn_cast<Constant>(I->getOperand(i)))
+        ConstOps.push_back(COp);
+      else
+        break;
+    }
+
+    // All operands were constants, fold it.
+    if (ConstOps.size() == I->getNumOperands()) {
+      if (CmpInst *C = dyn_cast<CmpInst>(I))
+        return ConstantFoldCompareInstOperands(C->getPredicate(), ConstOps[0],
+                                               ConstOps[1], Q.DL, Q.TLI);
+
+      if (LoadInst *LI = dyn_cast<LoadInst>(I))
+        if (!LI->isVolatile())
+          return ConstantFoldLoadFromConstPtr(ConstOps[0], Q.DL);
+
+      return ConstantFoldInstOperands(I->getOpcode(), I->getType(), ConstOps,
+                                      Q.DL, Q.TLI);
+    }
+  }
+
+  return nullptr;
 }
 
 /// SimplifySelectInst - Given operands for a SelectInst, see if we can fold
@@ -3174,29 +3281,28 @@ static Value *SimplifySelectInst(Value *CondVal, Value *TrueVal,
   if (isa<UndefValue>(FalseVal))   // select C, X, undef -> X
     return TrueVal;
 
-  const auto *ICI = dyn_cast<ICmpInst>(CondVal);
-  unsigned BitWidth = TrueVal->getType()->getScalarSizeInBits();
-  if (ICI && BitWidth) {
+  if (const auto *ICI = dyn_cast<ICmpInst>(CondVal)) {
+    unsigned BitWidth = Q.DL.getTypeSizeInBits(TrueVal->getType());
     ICmpInst::Predicate Pred = ICI->getPredicate();
+    Value *CmpLHS = ICI->getOperand(0);
+    Value *CmpRHS = ICI->getOperand(1);
     APInt MinSignedValue = APInt::getSignBit(BitWidth);
     Value *X;
     const APInt *Y;
     bool TrueWhenUnset;
     bool IsBitTest = false;
     if (ICmpInst::isEquality(Pred) &&
-        match(ICI->getOperand(0), m_And(m_Value(X), m_APInt(Y))) &&
-        match(ICI->getOperand(1), m_Zero())) {
+        match(CmpLHS, m_And(m_Value(X), m_APInt(Y))) &&
+        match(CmpRHS, m_Zero())) {
       IsBitTest = true;
       TrueWhenUnset = Pred == ICmpInst::ICMP_EQ;
-    } else if (Pred == ICmpInst::ICMP_SLT &&
-               match(ICI->getOperand(1), m_Zero())) {
-      X = ICI->getOperand(0);
+    } else if (Pred == ICmpInst::ICMP_SLT && match(CmpRHS, m_Zero())) {
+      X = CmpLHS;
       Y = &MinSignedValue;
       IsBitTest = true;
       TrueWhenUnset = false;
-    } else if (Pred == ICmpInst::ICMP_SGT &&
-               match(ICI->getOperand(1), m_AllOnes())) {
-      X = ICI->getOperand(0);
+    } else if (Pred == ICmpInst::ICMP_SGT && match(CmpRHS, m_AllOnes())) {
+      X = CmpLHS;
       Y = &MinSignedValue;
       IsBitTest = true;
       TrueWhenUnset = true;
@@ -3227,6 +3333,50 @@ static Value *SimplifySelectInst(Value *CondVal, Value *TrueVal,
           return TrueWhenUnset ? TrueVal : FalseVal;
       }
     }
+    if (ICI->hasOneUse()) {
+      const APInt *C;
+      if (match(CmpRHS, m_APInt(C))) {
+        // X < MIN ? T : F  -->  F
+        if (Pred == ICmpInst::ICMP_SLT && C->isMinSignedValue())
+          return FalseVal;
+        // X < MIN ? T : F  -->  F
+        if (Pred == ICmpInst::ICMP_ULT && C->isMinValue())
+          return FalseVal;
+        // X > MAX ? T : F  -->  F
+        if (Pred == ICmpInst::ICMP_SGT && C->isMaxSignedValue())
+          return FalseVal;
+        // X > MAX ? T : F  -->  F
+        if (Pred == ICmpInst::ICMP_UGT && C->isMaxValue())
+          return FalseVal;
+      }
+    }
+
+    // If we have an equality comparison then we know the value in one of the
+    // arms of the select. See if substituting this value into the arm and
+    // simplifying the result yields the same value as the other arm.
+    if (Pred == ICmpInst::ICMP_EQ) {
+      if (SimplifyWithOpReplaced(FalseVal, CmpLHS, CmpRHS, Q, MaxRecurse) ==
+              TrueVal ||
+          SimplifyWithOpReplaced(FalseVal, CmpRHS, CmpLHS, Q, MaxRecurse) ==
+              TrueVal)
+        return FalseVal;
+      if (SimplifyWithOpReplaced(TrueVal, CmpLHS, CmpRHS, Q, MaxRecurse) ==
+              FalseVal ||
+          SimplifyWithOpReplaced(TrueVal, CmpRHS, CmpLHS, Q, MaxRecurse) ==
+              FalseVal)
+        return FalseVal;
+    } else if (Pred == ICmpInst::ICMP_NE) {
+      if (SimplifyWithOpReplaced(TrueVal, CmpLHS, CmpRHS, Q, MaxRecurse) ==
+              FalseVal ||
+          SimplifyWithOpReplaced(TrueVal, CmpRHS, CmpLHS, Q, MaxRecurse) ==
+              FalseVal)
+        return TrueVal;
+      if (SimplifyWithOpReplaced(FalseVal, CmpLHS, CmpRHS, Q, MaxRecurse) ==
+              TrueVal ||
+          SimplifyWithOpReplaced(FalseVal, CmpRHS, CmpLHS, Q, MaxRecurse) ==
+              TrueVal)
+        return TrueVal;
+    }
   }
 
   return nullptr;
@@ -3371,14 +3521,89 @@ Value *llvm::SimplifyInsertValueInst(
                                    RecursionLimit);
 }
 
+/// SimplifyExtractValueInst - Given operands for an ExtractValueInst, see if we
+/// can fold the result.  If not, this returns null.
+static Value *SimplifyExtractValueInst(Value *Agg, ArrayRef<unsigned> Idxs,
+                                       const Query &, unsigned) {
+  if (auto *CAgg = dyn_cast<Constant>(Agg))
+    return ConstantFoldExtractValueInstruction(CAgg, Idxs);
+
+  // extractvalue x, (insertvalue y, elt, n), n -> elt
+  unsigned NumIdxs = Idxs.size();
+  for (auto *IVI = dyn_cast<InsertValueInst>(Agg); IVI != nullptr;
+       IVI = dyn_cast<InsertValueInst>(IVI->getAggregateOperand())) {
+    ArrayRef<unsigned> InsertValueIdxs = IVI->getIndices();
+    unsigned NumInsertValueIdxs = InsertValueIdxs.size();
+    unsigned NumCommonIdxs = std::min(NumInsertValueIdxs, NumIdxs);
+    if (InsertValueIdxs.slice(0, NumCommonIdxs) ==
+        Idxs.slice(0, NumCommonIdxs)) {
+      if (NumIdxs == NumInsertValueIdxs)
+        return IVI->getInsertedValueOperand();
+      break;
+    }
+  }
+
+  return nullptr;
+}
+
+Value *llvm::SimplifyExtractValueInst(Value *Agg, ArrayRef<unsigned> Idxs,
+                                      const DataLayout &DL,
+                                      const TargetLibraryInfo *TLI,
+                                      const DominatorTree *DT,
+                                      AssumptionCache *AC,
+                                      const Instruction *CxtI) {
+  return ::SimplifyExtractValueInst(Agg, Idxs, Query(DL, TLI, DT, AC, CxtI),
+                                    RecursionLimit);
+}
+
+/// SimplifyExtractElementInst - Given operands for an ExtractElementInst, see if we
+/// can fold the result.  If not, this returns null.
+static Value *SimplifyExtractElementInst(Value *Vec, Value *Idx, const Query &,
+                                         unsigned) {
+  if (auto *CVec = dyn_cast<Constant>(Vec)) {
+    if (auto *CIdx = dyn_cast<Constant>(Idx))
+      return ConstantFoldExtractElementInstruction(CVec, CIdx);
+
+    // The index is not relevant if our vector is a splat.
+    if (auto *Splat = CVec->getSplatValue())
+      return Splat;
+
+    if (isa<UndefValue>(Vec))
+      return UndefValue::get(Vec->getType()->getVectorElementType());
+  }
+
+  // If extracting a specified index from the vector, see if we can recursively
+  // find a previously computed scalar that was inserted into the vector.
+  if (auto *IdxC = dyn_cast<ConstantInt>(Idx)) {
+    unsigned IndexVal = IdxC->getZExtValue();
+    unsigned VectorWidth = Vec->getType()->getVectorNumElements();
+
+    // If this is extracting an invalid index, turn this into undef, to avoid
+    // crashing the code below.
+    if (IndexVal >= VectorWidth)
+      return UndefValue::get(Vec->getType()->getVectorElementType());
+
+    if (Value *Elt = findScalarElement(Vec, IndexVal))
+      return Elt;
+  }
+
+  return nullptr;
+}
+
+Value *llvm::SimplifyExtractElementInst(
+    Value *Vec, Value *Idx, const DataLayout &DL, const TargetLibraryInfo *TLI,
+    const DominatorTree *DT, AssumptionCache *AC, const Instruction *CxtI) {
+  return ::SimplifyExtractElementInst(Vec, Idx, Query(DL, TLI, DT, AC, CxtI),
+                                      RecursionLimit);
+}
+
 /// SimplifyPHINode - See if we can fold the given phi.  If not, returns null.
 static Value *SimplifyPHINode(PHINode *PN, const Query &Q) {
   // If all of the PHI's incoming values are the same then replace the PHI node
   // with the common value.
   Value *CommonValue = nullptr;
   bool HasUndefInput = false;
-  for (unsigned i = 0, e = PN->getNumIncomingValues(); i != e; ++i) {
-    Value *Incoming = PN->getIncomingValue(i);
+  for (Value *Incoming : PN->incoming_values()) {
     // If the incoming value is the phi node itself, it can safely be skipped.
     if (Incoming == PN) continue;
     if (isa<UndefValue>(Incoming)) {
@@ -3531,7 +3756,7 @@ static Value *SimplifyCmpInst(unsigned Predicate, Value *LHS, Value *RHS,
                               const Query &Q, unsigned MaxRecurse) {
   if (CmpInst::isIntPredicate((CmpInst::Predicate)Predicate))
     return SimplifyICmpInst(Predicate, LHS, RHS, Q, MaxRecurse);
-  return SimplifyFCmpInst(Predicate, LHS, RHS, Q, MaxRecurse);
+  return SimplifyFCmpInst(Predicate, LHS, RHS, FastMathFlags(), Q, MaxRecurse);
 }
 
 Value *llvm::SimplifyCmpInst(unsigned Predicate, Value *LHS, Value *RHS,
@@ -3559,14 +3784,53 @@ static bool IsIdempotent(Intrinsic::ID ID) {
 }
 
 template <typename IterTy>
-static Value *SimplifyIntrinsic(Intrinsic::ID IID, IterTy ArgBegin, IterTy ArgEnd,
+static Value *SimplifyIntrinsic(Function *F, IterTy ArgBegin, IterTy ArgEnd,
                                 const Query &Q, unsigned MaxRecurse) {
+  Intrinsic::ID IID = F->getIntrinsicID();
+  unsigned NumOperands = std::distance(ArgBegin, ArgEnd);
+  Type *ReturnType = F->getReturnType();
+
+  // Binary Ops
+  if (NumOperands == 2) {
+    Value *LHS = *ArgBegin;
+    Value *RHS = *(ArgBegin + 1);
+    if (IID == Intrinsic::usub_with_overflow ||
+        IID == Intrinsic::ssub_with_overflow) {
+      // X - X -> { 0, false }
+      if (LHS == RHS)
+        return Constant::getNullValue(ReturnType);
+
+      // X - undef -> undef
+      // undef - X -> undef
+      if (isa<UndefValue>(LHS) || isa<UndefValue>(RHS))
+        return UndefValue::get(ReturnType);
+    }
+
+    if (IID == Intrinsic::uadd_with_overflow ||
+        IID == Intrinsic::sadd_with_overflow) {
+      // X + undef -> undef
+      if (isa<UndefValue>(RHS))
+        return UndefValue::get(ReturnType);
+    }
+
+    if (IID == Intrinsic::umul_with_overflow ||
+        IID == Intrinsic::smul_with_overflow) {
+      // X * 0 -> { 0, false }
+      if (match(RHS, m_Zero()))
+        return Constant::getNullValue(ReturnType);
+
+      // X * undef -> { 0, false }
+      if (match(RHS, m_Undef()))
+        return Constant::getNullValue(ReturnType);
+    }
+  }
+
   // Perform idempotent optimizations
   if (!IsIdempotent(IID))
     return nullptr;
 
   // Unary Ops
-  if (std::distance(ArgBegin, ArgEnd) == 1)
+  if (NumOperands == 1)
     if (IntrinsicInst *II = dyn_cast<IntrinsicInst>(*ArgBegin))
       if (II->getIntrinsicID() == IID)
         return II;
@@ -3590,9 +3854,8 @@ static Value *SimplifyCall(Value *V, IterTy ArgBegin, IterTy ArgEnd,
   if (!F)
     return nullptr;
 
-  if (unsigned IID = F->getIntrinsicID())
-    if (Value *Ret =
-        SimplifyIntrinsic((Intrinsic::ID) IID, ArgBegin, ArgEnd, Q, MaxRecurse))
+  if (F->isIntrinsic())
+    if (Value *Ret = SimplifyIntrinsic(F, ArgBegin, ArgEnd, Q, MaxRecurse))
       return Ret;
 
   if (!canConstantFoldCallTo(F))
@@ -3723,9 +3986,9 @@ Value *llvm::SimplifyInstruction(Instruction *I, const DataLayout &DL,
                          I->getOperand(1), DL, TLI, DT, AC, I);
     break;
   case Instruction::FCmp:
-    Result =
-        SimplifyFCmpInst(cast<FCmpInst>(I)->getPredicate(), I->getOperand(0),
-                         I->getOperand(1), DL, TLI, DT, AC, I);
+    Result = SimplifyFCmpInst(cast<FCmpInst>(I)->getPredicate(),
+                              I->getOperand(0), I->getOperand(1),
+                              I->getFastMathFlags(), DL, TLI, DT, AC, I);
     break;
   case Instruction::Select:
     Result = SimplifySelectInst(I->getOperand(0), I->getOperand(1),
@@ -3743,6 +4006,18 @@ Value *llvm::SimplifyInstruction(Instruction *I, const DataLayout &DL,
                                      IV->getIndices(), DL, TLI, DT, AC, I);
     break;
   }
+  case Instruction::ExtractValue: {
+    auto *EVI = cast<ExtractValueInst>(I);
+    Result = SimplifyExtractValueInst(EVI->getAggregateOperand(),
+                                      EVI->getIndices(), DL, TLI, DT, AC, I);
+    break;
+  }
+  case Instruction::ExtractElement: {
+    auto *EEI = cast<ExtractElementInst>(I);
+    Result = SimplifyExtractElementInst(
+        EEI->getVectorOperand(), EEI->getIndexOperand(), DL, TLI, DT, AC, I);
+    break;
+  }
   case Instruction::PHI:
     Result = SimplifyPHINode(cast<PHINode>(I), Query(DL, TLI, DT, AC, I));
     break;
diff --git a/lib/Analysis/IteratedDominanceFrontier.cpp b/lib/Analysis/IteratedDominanceFrontier.cpp
new file mode 100644
index 00000000000..9f1edd21820
--- /dev/null
+++ b/lib/Analysis/IteratedDominanceFrontier.cpp
@@ -0,0 +1,95 @@
+//===- IteratedDominanceFrontier.cpp - Compute IDF ------------------------===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+/// \brief Compute iterated dominance frontiers using a linear time algorithm.
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/Analysis/IteratedDominanceFrontier.h"
+#include "llvm/IR/CFG.h"
+#include "llvm/IR/Dominators.h"
+#include <queue>
+
+using namespace llvm;
+
+void IDFCalculator::calculate(SmallVectorImpl<BasicBlock *> &PHIBlocks) {
+  // If we haven't computed dominator tree levels, do so now.
+  if (DomLevels.empty()) {
+    for (auto DFI = df_begin(DT.getRootNode()), DFE = df_end(DT.getRootNode());
+         DFI != DFE; ++DFI) {
+      DomLevels[*DFI] = DFI.getPathLength() - 1;
+    }
+  }
+
+  // Use a priority queue keyed on dominator tree level so that inserted nodes
+  // are handled from the bottom of the dominator tree upwards.
+  typedef std::pair<DomTreeNode *, unsigned> DomTreeNodePair;
+  typedef std::priority_queue<DomTreeNodePair, SmallVector<DomTreeNodePair, 32>,
+                              less_second> IDFPriorityQueue;
+  IDFPriorityQueue PQ;
+
+  for (BasicBlock *BB : *DefBlocks) {
+    if (DomTreeNode *Node = DT.getNode(BB))
+      PQ.push(std::make_pair(Node, DomLevels.lookup(Node)));
+  }
+
+  SmallVector<DomTreeNode *, 32> Worklist;
+  SmallPtrSet<DomTreeNode *, 32> VisitedPQ;
+  SmallPtrSet<DomTreeNode *, 32> VisitedWorklist;
+
+  while (!PQ.empty()) {
+    DomTreeNodePair RootPair = PQ.top();
+    PQ.pop();
+    DomTreeNode *Root = RootPair.first;
+    unsigned RootLevel = RootPair.second;
+
+    // Walk all dominator tree children of Root, inspecting their CFG edges with
+    // targets elsewhere on the dominator tree. Only targets whose level is at
+    // most Root's level are added to the iterated dominance frontier of the
+    // definition set.
+
+    Worklist.clear();
+    Worklist.push_back(Root);
+    VisitedWorklist.insert(Root);
+
+    while (!Worklist.empty()) {
+      DomTreeNode *Node = Worklist.pop_back_val();
+      BasicBlock *BB = Node->getBlock();
+
+      for (auto Succ : successors(BB)) {
+        DomTreeNode *SuccNode = DT.getNode(Succ);
+
+        // Quickly skip all CFG edges that are also dominator tree edges instead
+        // of catching them below.
+        if (SuccNode->getIDom() == Node)
+          continue;
+
+        unsigned SuccLevel = DomLevels.lookup(SuccNode);
+        if (SuccLevel > RootLevel)
+          continue;
+
+        if (!VisitedPQ.insert(SuccNode).second)
+          continue;
+
+        BasicBlock *SuccBB = SuccNode->getBlock();
+        if (useLiveIn && !LiveInBlocks->count(SuccBB))
+          continue;
+
+        PHIBlocks.emplace_back(SuccBB);
+        if (!DefBlocks->count(SuccBB))
+          PQ.push(std::make_pair(SuccNode, SuccLevel));
+      }
+
+      for (auto DomChild : *Node) {
+        if (VisitedWorklist.insert(DomChild).second)
+          Worklist.push_back(DomChild);
+      }
+    }
+  }
+}
diff --git a/lib/Analysis/LLVMBuild.txt b/lib/Analysis/LLVMBuild.txt
index 56d20950e4f..3039ddea4ff 100644
--- a/lib/Analysis/LLVMBuild.txt
+++ b/lib/Analysis/LLVMBuild.txt
@@ -16,7 +16,7 @@
 ;===------------------------------------------------------------------------===;
 
 [common]
-subdirectories = IPA NaCl
+subdirectories = IPA
 
 [component_0]
 type = Library
diff --git a/lib/Analysis/LazyValueInfo.cpp b/lib/Analysis/LazyValueInfo.cpp
index e6f586ac702..77e85353586 100644
--- a/lib/Analysis/LazyValueInfo.cpp
+++ b/lib/Analysis/LazyValueInfo.cpp
@@ -64,10 +64,10 @@ class LVILatticeVal {
   enum LatticeValueTy {
     /// This Value has no known value yet.
     undefined,
-    
+
     /// This Value has a specific constant value.
     constant,
-    
+
     /// This Value is known to not have the specified value.
     notconstant,
 
@@ -77,13 +77,13 @@ class LVILatticeVal {
     /// This value is not known to be constant, and we know that it has a value.
     overdefined
   };
-  
+
   /// Val: This stores the current lattice value along with the Constant* for
   /// the constant if this is a 'constant' or 'notconstant' value.
   LatticeValueTy Tag;
   Constant *Val;
   ConstantRange Range;
-  
+
 public:
   LVILatticeVal() : Tag(undefined), Val(nullptr), Range(1, true) {}
 
@@ -104,29 +104,29 @@ class LVILatticeVal {
     Res.markConstantRange(CR);
     return Res;
   }
-  
+
   bool isUndefined() const     { return Tag == undefined; }
   bool isConstant() const      { return Tag == constant; }
   bool isNotConstant() const   { return Tag == notconstant; }
   bool isConstantRange() const { return Tag == constantrange; }
   bool isOverdefined() const   { return Tag == overdefined; }
-  
+
   Constant *getConstant() const {
     assert(isConstant() && "Cannot get the constant of a non-constant!");
     return Val;
   }
-  
+
   Constant *getNotConstant() const {
     assert(isNotConstant() && "Cannot get the constant of a non-notconstant!");
     return Val;
   }
-  
+
   ConstantRange getConstantRange() const {
     assert(isConstantRange() &&
            "Cannot get the constant-range of a non-constant-range!");
     return Range;
   }
-  
+
   /// Return true if this is a change in status.
   bool markOverdefined() {
     if (isOverdefined())
@@ -150,7 +150,7 @@ class LVILatticeVal {
     Val = V;
     return true;
   }
-  
+
   /// Return true if this is a change in status.
   bool markNotConstant(Constant *V) {
     assert(V && "Marking constant with NULL");
@@ -168,27 +168,27 @@ class LVILatticeVal {
     Val = V;
     return true;
   }
-  
+
   /// Return true if this is a change in status.
   bool markConstantRange(const ConstantRange NewR) {
     if (isConstantRange()) {
       if (NewR.isEmptySet())
         return markOverdefined();
-      
+
       bool changed = Range != NewR;
       Range = NewR;
       return changed;
     }
-    
+
     assert(isUndefined());
     if (NewR.isEmptySet())
       return markOverdefined();
-    
+
     Tag = constantrange;
     Range = NewR;
     return true;
   }
-  
+
   /// Merge the specified lattice value into this one, updating this
   /// one and returning true if anything changed.
   bool mergeIn(const LVILatticeVal &RHS, const DataLayout &DL) {
@@ -267,7 +267,7 @@ class LVILatticeVal {
     return markConstantRange(NewR);
   }
 };
-  
+
 } // end anonymous namespace.
 
 namespace llvm {
@@ -297,7 +297,7 @@ namespace {
   class LazyValueInfoCache;
   struct LVIValueHandle : public CallbackVH {
     LazyValueInfoCache *Parent;
-      
+
     LVIValueHandle(Value *V, LazyValueInfoCache *P)
       : CallbackVH(V), Parent(P) { }
 
@@ -308,7 +308,7 @@ namespace {
   };
 }
 
-namespace { 
+namespace {
   /// This is the cache kept by LazyValueInfo which
   /// maintains information about queries across the clients' queries.
   class LazyValueInfoCache {
@@ -320,7 +320,7 @@ namespace {
     /// This is all of the cached information for all values,
     /// mapped from Value* to key information.
     std::map<LVIValueHandle, ValueCacheEntryTy> ValueCache;
-    
+
     /// This tracks, on a per-block basis, the set of values that are
     /// over-defined at the end of that block.  This is required
     /// for cache updating.
@@ -382,7 +382,7 @@ namespace {
                                             Instruction *BBI);
 
     void solve();
-    
+
     ValueCacheEntryTy &lookup(Value *V) {
       return ValueCache[LVIValueHandle(V, this)];
     }
@@ -402,15 +402,15 @@ namespace {
     /// value for the specified Value* that is true on the specified edge.
     LVILatticeVal getValueOnEdge(Value *V, BasicBlock *FromBB,BasicBlock *ToBB,
                                  Instruction *CxtI = nullptr);
-    
+
     /// This is the update interface to inform the cache that an edge from
     /// PredBB to OldSucc has been threaded to be from PredBB to NewSucc.
     void threadEdge(BasicBlock *PredBB,BasicBlock *OldSucc,BasicBlock *NewSucc);
-    
+
     /// This is part of the update interface to inform the cache
     /// that a block has been deleted.
     void eraseBlock(BasicBlock *BB);
-    
+
     /// clear - Empty the cache.
     void clear() {
       SeenBlocks.clear();
@@ -426,14 +426,14 @@ namespace {
 
 void LVIValueHandle::deleted() {
   typedef std::pair<AssertingVH<BasicBlock>, Value*> OverDefinedPairTy;
-  
+
   SmallVector<OverDefinedPairTy, 4> ToErase;
   for (const OverDefinedPairTy &P : Parent->OverDefinedCache)
     if (P.second == getValPtr())
       ToErase.push_back(P);
   for (const OverDefinedPairTy &P : ToErase)
     Parent->OverDefinedCache.erase(P);
-  
+
   // This erasure deallocates *this, so it MUST happen after we're done
   // using any and all members of *this.
   Parent->ValueCache.erase(*this);
@@ -516,7 +516,7 @@ bool LazyValueInfoCache::solveBlockValue(Value *Val, BasicBlock *BB) {
   // Hold off inserting this value into the Cache in case we have to return
   // false and come back later.
   LVILatticeVal Res;
-  
+
   Instruction *BBI = dyn_cast<Instruction>(Val);
   if (!BBI || BBI->getParent() != BB) {
     if (!solveBlockValueNonLocal(Res, Val, BB))
@@ -661,7 +661,7 @@ bool LazyValueInfoCache::solveBlockValueNonLocal(LVILatticeVal &BBLV,
         PointerType *PTy = cast<PointerType>(Val->getType());
         Result = LVILatticeVal::getNot(ConstantPointerNull::get(PTy));
       }
-      
+
       BBLV = Result;
       return true;
     }
@@ -674,7 +674,7 @@ bool LazyValueInfoCache::solveBlockValueNonLocal(LVILatticeVal &BBLV,
   BBLV = Result;
   return true;
 }
-  
+
 bool LazyValueInfoCache::solveBlockValuePHINode(LVILatticeVal &BBLV,
                                                 PHINode *PN, BasicBlock *BB) {
   LVILatticeVal Result;  // Start Undefined.
@@ -700,7 +700,7 @@ bool LazyValueInfoCache::solveBlockValuePHINode(LVILatticeVal &BBLV,
     if (Result.isOverdefined()) {
       DEBUG(dbgs() << " compute BB '" << BB->getName()
             << "' - overdefined because of pred.\n");
-      
+
       BBLV = Result;
       return true;
     }
@@ -765,7 +765,7 @@ bool LazyValueInfoCache::solveBlockValueConstantRange(LVILatticeVal &BBLV,
     BBLV.markOverdefined();
     return true;
   }
-  
+
   ConstantRange LHSRange = LHSVal.getConstantRange();
   ConstantRange RHSRange(1);
   IntegerType *ResultTy = cast<IntegerType>(BBI->getType());
@@ -819,7 +819,7 @@ bool LazyValueInfoCache::solveBlockValueConstantRange(LVILatticeVal &BBLV,
   case Instruction::Or:
     Result.markConstantRange(LHSRange.binaryOr(RHSRange));
     break;
-  
+
   // Unhandled instructions are overdefined.
   default:
     DEBUG(dbgs() << " compute BB '" << BB->getName()
@@ -827,7 +827,7 @@ bool LazyValueInfoCache::solveBlockValueConstantRange(LVILatticeVal &BBLV,
     Result.markOverdefined();
     break;
   }
-  
+
   BBLV = Result;
   return true;
 }
@@ -877,7 +877,7 @@ bool getValueFromFromCondition(Value *Val, ICmpInst *ICI,
 /// Val is not constrained on the edge.
 static bool getEdgeValueLocal(Value *Val, BasicBlock *BBFrom,
                               BasicBlock *BBTo, LVILatticeVal &Result) {
-  // TODO: Handle more complex conditionals.  If (v == 0 || v2 < 1) is false, we
+  // TODO: Handle more complex conditionals. If (v == 0 || v2 < 1) is false, we
   // know that v != 0.
   if (BranchInst *BI = dyn_cast<BranchInst>(BBFrom->getTerminator())) {
     // If this is a conditional branch and only one successor goes to BBTo, then
@@ -887,7 +887,7 @@ static bool getEdgeValueLocal(Value *Val, BasicBlock *BBFrom,
       bool isTrueDest = BI->getSuccessor(0) == BBTo;
       assert(BI->getSuccessor(!isTrueDest) == BBTo &&
              "BBTo isn't a successor of BBFrom");
-      
+
       // If V is the condition of the branch itself, then we know exactly what
       // it is.
       if (BI->getCondition() == Val) {
@@ -895,7 +895,7 @@ static bool getEdgeValueLocal(Value *Val, BasicBlock *BBFrom,
                               Type::getInt1Ty(Val->getContext()), isTrueDest));
         return true;
       }
-      
+
       // If the condition of the branch is an equality comparison, we may be
       // able to infer the value.
       if (ICmpInst *ICI = dyn_cast<ICmpInst>(BI->getCondition()))
@@ -997,7 +997,7 @@ LVILatticeVal LazyValueInfoCache::getValueInBlock(Value *V, BasicBlock *BB,
                                                   Instruction *CxtI) {
   DEBUG(dbgs() << "LVI Getting block end value " << *V << " at '"
         << BB->getName() << "'\n");
-  
+
   assert(BlockValueStack.empty() && BlockValueSet.empty());
   pushBlockValue(std::make_pair(BB, V));
 
@@ -1025,7 +1025,7 @@ getValueOnEdge(Value *V, BasicBlock *FromBB, BasicBlock *ToBB,
                Instruction *CxtI) {
   DEBUG(dbgs() << "LVI Getting edge value " << *V << " from '"
         << FromBB->getName() << "' to '" << ToBB->getName() << "'\n");
-  
+
   LVILatticeVal Result;
   if (!getEdgeValue(V, FromBB, ToBB, Result, CxtI)) {
     solve();
@@ -1040,24 +1040,24 @@ getValueOnEdge(Value *V, BasicBlock *FromBB, BasicBlock *ToBB,
 
 void LazyValueInfoCache::threadEdge(BasicBlock *PredBB, BasicBlock *OldSucc,
                                     BasicBlock *NewSucc) {
-  // When an edge in the graph has been threaded, values that we could not 
-  // determine a value for before (i.e. were marked overdefined) may be possible
-  // to solve now.  We do NOT try to proactively update these values.  Instead,
-  // we clear their entries from the cache, and allow lazy updating to recompute
-  // them when needed.
-  
+  // When an edge in the graph has been threaded, values that we could not
+  // determine a value for before (i.e. were marked overdefined) may be
+  // possible to solve now. We do NOT try to proactively update these values.
+  // Instead, we clear their entries from the cache, and allow lazy updating to
+  // recompute them when needed.
+
   // The updating process is fairly simple: we need to drop cached info
   // for all values that were marked overdefined in OldSucc, and for those same
   // values in any successor of OldSucc (except NewSucc) in which they were
   // also marked overdefined.
   std::vector<BasicBlock*> worklist;
   worklist.push_back(OldSucc);
-  
+
   DenseSet<Value*> ClearSet;
   for (OverDefinedPairTy &P : OverDefinedCache)
     if (P.first == OldSucc)
       ClearSet.insert(P.second);
-  
+
   // Use a worklist to perform a depth-first search of OldSucc's successors.
   // NOTE: We do not need a visited list since any blocks we have already
   // visited will have had their overdefined markers cleared already, and we
@@ -1065,10 +1065,10 @@ void LazyValueInfoCache::threadEdge(BasicBlock *PredBB, BasicBlock *OldSucc,
   while (!worklist.empty()) {
     BasicBlock *ToUpdate = worklist.back();
     worklist.pop_back();
-    
+
     // Skip blocks only accessible through NewSucc.
     if (ToUpdate == NewSucc) continue;
-    
+
     bool changed = false;
     for (Value *V : ClearSet) {
       // If a value was marked overdefined in OldSucc, and is here too...
@@ -1084,13 +1084,13 @@ void LazyValueInfoCache::threadEdge(BasicBlock *PredBB, BasicBlock *OldSucc,
       Entry.erase(CI);
       OverDefinedCache.erase(OI);
 
-      // If we removed anything, then we potentially need to update 
+      // If we removed anything, then we potentially need to update
       // blocks successors too.
       changed = true;
     }
 
     if (!changed) continue;
-    
+
     worklist.insert(worklist.end(), succ_begin(ToUpdate), succ_end(ToUpdate));
   }
 }
@@ -1158,7 +1158,7 @@ Constant *LazyValueInfo::getConstant(Value *V, BasicBlock *BB,
 }
 
 /// Determine whether the specified value is known to be a
-/// constant on the specified edge.  Return null if not.
+/// constant on the specified edge. Return null if not.
 Constant *LazyValueInfo::getConstantOnEdge(Value *V, BasicBlock *FromBB,
                                            BasicBlock *ToBB,
                                            Instruction *CxtI) {
@@ -1190,26 +1190,26 @@ static LazyValueInfo::Tristate getPredicateResult(unsigned Pred, Constant *C,
       return ResCI->isZero() ? LazyValueInfo::False : LazyValueInfo::True;
     return LazyValueInfo::Unknown;
   }
-  
+
   if (Result.isConstantRange()) {
     ConstantInt *CI = dyn_cast<ConstantInt>(C);
     if (!CI) return LazyValueInfo::Unknown;
-    
+
     ConstantRange CR = Result.getConstantRange();
     if (Pred == ICmpInst::ICMP_EQ) {
       if (!CR.contains(CI->getValue()))
         return LazyValueInfo::False;
-      
+
       if (CR.isSingleElement() && CR.contains(CI->getValue()))
         return LazyValueInfo::True;
     } else if (Pred == ICmpInst::ICMP_NE) {
       if (!CR.contains(CI->getValue()))
         return LazyValueInfo::True;
-      
+
       if (CR.isSingleElement() && CR.contains(CI->getValue()))
         return LazyValueInfo::False;
     }
-    
+
     // Handle more complex predicates.
     ConstantRange TrueValues =
         ICmpInst::makeConstantRange((ICmpInst::Predicate)Pred, CI->getValue());
@@ -1219,7 +1219,7 @@ static LazyValueInfo::Tristate getPredicateResult(unsigned Pred, Constant *C,
       return LazyValueInfo::False;
     return LazyValueInfo::Unknown;
   }
-  
+
   if (Result.isNotConstant()) {
     // If this is an equality comparison, we can try to fold it knowing that
     // "V != C1".
@@ -1240,7 +1240,7 @@ static LazyValueInfo::Tristate getPredicateResult(unsigned Pred, Constant *C,
     }
     return LazyValueInfo::Unknown;
   }
-  
+
   return LazyValueInfo::Unknown;
 }
 
@@ -1262,8 +1262,40 @@ LazyValueInfo::getPredicateAt(unsigned Pred, Value *V, Constant *C,
                               Instruction *CxtI) {
   const DataLayout &DL = CxtI->getModule()->getDataLayout();
   LVILatticeVal Result = getCache(PImpl, AC, &DL, DT).getValueAt(V, CxtI);
-
-  return getPredicateResult(Pred, C, Result, DL, TLI);
+  Tristate Ret = getPredicateResult(Pred, C, Result, DL, TLI);
+  if (Ret != Unknown)
+    return Ret;
+
+  // TODO: Move this logic inside getValueAt so that it can be cached rather
+  // than re-queried on each call. This would also allow us to merge the
+  // underlying lattice values to get more information.
+  if (CxtI) {
+    // For a comparison where the V is outside this block, it's possible
+    // that we've branched on it before. Look to see if the value is known
+    // on all incoming edges.
+    BasicBlock *BB = CxtI->getParent();
+    pred_iterator PI = pred_begin(BB), PE = pred_end(BB);
+    if (PI != PE &&
+        (!isa<Instruction>(V) ||
+         cast<Instruction>(V)->getParent() != BB)) {
+      // For predecessor edge, determine if the comparison is true or false
+      // on that edge. If they're all true or all false, we can conclude
+      // the value of the comparison in this block.
+      Tristate Baseline = getPredicateOnEdge(Pred, V, C, *PI, BB, CxtI);
+      if (Baseline != Unknown) {
+        // Check that all remaining incoming values match the first one.
+        while (++PI != PE) {
+          Tristate Ret = getPredicateOnEdge(Pred, V, C, *PI, BB, CxtI);
+          if (Ret != Baseline) break;
+        }
+        // If we terminated early, then one of the values didn't match.
+        if (PI == PE) {
+          return Baseline;
+        }
+      }
+    }
+  }
+  return Unknown;
 }
 
 void LazyValueInfo::threadEdge(BasicBlock *PredBB, BasicBlock *OldSucc,
diff --git a/lib/Analysis/LibCallAliasAnalysis.cpp b/lib/Analysis/LibCallAliasAnalysis.cpp
index f6025e3252e..c9e2585bd66 100644
--- a/lib/Analysis/LibCallAliasAnalysis.cpp
+++ b/lib/Analysis/LibCallAliasAnalysis.cpp
@@ -45,15 +45,16 @@ bool LibCallAliasAnalysis::runOnFunction(Function &F) {
 /// AnalyzeLibCallDetails - Given a call to a function with the specified
 /// LibCallFunctionInfo, see if we can improve the mod/ref footprint of the call
 /// vs the specified pointer/size.
-AliasAnalysis::ModRefResult
+ModRefInfo
 LibCallAliasAnalysis::AnalyzeLibCallDetails(const LibCallFunctionInfo *FI,
                                             ImmutableCallSite CS,
-                                            const Location &Loc) {
+                                            const MemoryLocation &Loc) {
   // If we have a function, check to see what kind of mod/ref effects it
   // has.  Start by including any info globally known about the function.
-  AliasAnalysis::ModRefResult MRInfo = FI->UniversalBehavior;
-  if (MRInfo == NoModRef) return MRInfo;
-  
+  ModRefInfo MRInfo = FI->UniversalBehavior;
+  if (MRInfo == MRI_NoModRef)
+    return MRInfo;
+
   // If that didn't tell us that the function is 'readnone', check to see
   // if we have detailed info and if 'P' is any of the locations we know
   // about.
@@ -75,7 +76,7 @@ LibCallAliasAnalysis::AnalyzeLibCallDetails(const LibCallFunctionInfo *FI,
       
       // If we find a match against a location that we 'do not' interact with,
       // learn this info into MRInfo.
-      return ModRefResult(MRInfo & ~Details[i].MRInfo);
+      return ModRefInfo(MRInfo & ~Details[i].MRInfo);
     }
     return MRInfo;
   }
@@ -83,7 +84,7 @@ LibCallAliasAnalysis::AnalyzeLibCallDetails(const LibCallFunctionInfo *FI,
   // If the details are of the 'DoesOnly' sort, we know something if the pointer
   // is a match for one of the locations in 'Details'.  Also, if we can prove
   // that the pointers is *not* one of the locations in 'Details', we know that
-  // the call is NoModRef.
+  // the call is MRI_NoModRef.
   assert(FI->DetailsType == LibCallFunctionInfo::DoesOnly);
   
   // Find out if the pointer refers to a known location.
@@ -103,16 +104,16 @@ LibCallAliasAnalysis::AnalyzeLibCallDetails(const LibCallFunctionInfo *FI,
     
     // If we know that this pointer definitely is pointing into the location,
     // merge in this information.
-    return ModRefResult(MRInfo & Details[i].MRInfo);
+    return ModRefInfo(MRInfo & Details[i].MRInfo);
   }
   
   // If we found that the pointer is guaranteed to not match any of the
   // locations in our 'DoesOnly' rule, then we know that the pointer must point
   // to some other location.  Since the libcall doesn't mod/ref any other
-  // locations, return NoModRef.
+  // locations, return MRI_NoModRef.
   if (NoneMatch)
-    return NoModRef;
-  
+    return MRI_NoModRef;
+
   // Otherwise, return any other info gained so far.
   return MRInfo;
 }
@@ -120,22 +121,22 @@ LibCallAliasAnalysis::AnalyzeLibCallDetails(const LibCallFunctionInfo *FI,
 // getModRefInfo - Check to see if the specified callsite can clobber the
 // specified memory object.
 //
-AliasAnalysis::ModRefResult
-LibCallAliasAnalysis::getModRefInfo(ImmutableCallSite CS,
-                                    const Location &Loc) {
-  ModRefResult MRInfo = ModRef;
-  
+ModRefInfo LibCallAliasAnalysis::getModRefInfo(ImmutableCallSite CS,
+                                               const MemoryLocation &Loc) {
+  ModRefInfo MRInfo = MRI_ModRef;
+
   // If this is a direct call to a function that LCI knows about, get the
   // information about the runtime function.
   if (LCI) {
     if (const Function *F = CS.getCalledFunction()) {
       if (const LibCallFunctionInfo *FI = LCI->getFunctionInfo(F)) {
-        MRInfo = ModRefResult(MRInfo & AnalyzeLibCallDetails(FI, CS, Loc));
-        if (MRInfo == NoModRef) return NoModRef;
+        MRInfo = ModRefInfo(MRInfo & AnalyzeLibCallDetails(FI, CS, Loc));
+        if (MRInfo == MRI_NoModRef)
+          return MRI_NoModRef;
       }
     }
   }
   
   // The AliasAnalysis base class has some smarts, lets use them.
-  return (ModRefResult)(MRInfo | AliasAnalysis::getModRefInfo(CS, Loc));
+  return (ModRefInfo)(MRInfo | AliasAnalysis::getModRefInfo(CS, Loc));
 }
diff --git a/lib/Analysis/LibCallSemantics.cpp b/lib/Analysis/LibCallSemantics.cpp
index 328b186b527..003c81e87b6 100644
--- a/lib/Analysis/LibCallSemantics.cpp
+++ b/lib/Analysis/LibCallSemantics.cpp
@@ -73,16 +73,15 @@ EHPersonality llvm::classifyEHPersonality(const Value *Pers) {
     .Case("__gxx_personality_v0",  EHPersonality::GNU_CXX)
     .Case("__gcc_personality_v0",  EHPersonality::GNU_C)
     .Case("__objc_personality_v0", EHPersonality::GNU_ObjC)
-    .Case("__except_handler3",     EHPersonality::MSVC_X86SEH)
-    .Case("__except_handler4",     EHPersonality::MSVC_X86SEH)
+    .Case("_except_handler3",      EHPersonality::MSVC_X86SEH)
+    .Case("_except_handler4",      EHPersonality::MSVC_X86SEH)
     .Case("__C_specific_handler",  EHPersonality::MSVC_Win64SEH)
     .Case("__CxxFrameHandler3",    EHPersonality::MSVC_CXX)
     .Default(EHPersonality::Unknown);
 }
 
-bool llvm::canSimplifyInvokeNoUnwind(const InvokeInst *II) {
-  const LandingPadInst *LP = II->getLandingPadInst();
-  EHPersonality Personality = classifyEHPersonality(LP->getPersonalityFn());
+bool llvm::canSimplifyInvokeNoUnwind(const Function *F) {
+  EHPersonality Personality = classifyEHPersonality(F->getPersonalityFn());
   // We can't simplify any invokes to nounwind functions if the personality
   // function wants to catch asynch exceptions.  The nounwind attribute only
   // implies that the function does not throw synchronous exceptions.
diff --git a/lib/Analysis/Lint.cpp b/lib/Analysis/Lint.cpp
index 65a90d7bcd8..0b9308a573a 100644
--- a/lib/Analysis/Lint.cpp
+++ b/lib/Analysis/Lint.cpp
@@ -202,8 +202,8 @@ void Lint::visitCallSite(CallSite CS) {
   Value *Callee = CS.getCalledValue();
   const DataLayout &DL = CS->getModule()->getDataLayout();
 
-  visitMemoryReference(I, Callee, AliasAnalysis::UnknownSize,
-                       0, nullptr, MemRef::Callee);
+  visitMemoryReference(I, Callee, MemoryLocation::UnknownSize, 0, nullptr,
+                       MemRef::Callee);
 
   if (Function *F = dyn_cast<Function>(findValue(Callee, DL,
                                                  /*OffsetOk=*/false))) {
@@ -244,9 +244,8 @@ void Lint::visitCallSite(CallSite CS) {
         if (Formal->hasNoAliasAttr() && Actual->getType()->isPointerTy())
           for (CallSite::arg_iterator BI = CS.arg_begin(); BI != AE; ++BI)
             if (AI != BI && (*BI)->getType()->isPointerTy()) {
-              AliasAnalysis::AliasResult Result = AA->alias(*AI, *BI);
-              Assert(Result != AliasAnalysis::MustAlias &&
-                         Result != AliasAnalysis::PartialAlias,
+              AliasResult Result = AA->alias(*AI, *BI);
+              Assert(Result != MustAlias && Result != PartialAlias,
                      "Unusual: noalias argument aliases another argument", &I);
             }
 
@@ -282,12 +281,10 @@ void Lint::visitCallSite(CallSite CS) {
     case Intrinsic::memcpy: {
       MemCpyInst *MCI = cast<MemCpyInst>(&I);
       // TODO: If the size is known, use it.
-      visitMemoryReference(I, MCI->getDest(), AliasAnalysis::UnknownSize,
-                           MCI->getAlignment(), nullptr,
-                           MemRef::Write);
-      visitMemoryReference(I, MCI->getSource(), AliasAnalysis::UnknownSize,
-                           MCI->getAlignment(), nullptr,
-                           MemRef::Read);
+      visitMemoryReference(I, MCI->getDest(), MemoryLocation::UnknownSize,
+                           MCI->getAlignment(), nullptr, MemRef::Write);
+      visitMemoryReference(I, MCI->getSource(), MemoryLocation::UnknownSize,
+                           MCI->getAlignment(), nullptr, MemRef::Read);
 
       // Check that the memcpy arguments don't overlap. The AliasAnalysis API
       // isn't expressive enough for what we really want to do. Known partial
@@ -299,27 +296,24 @@ void Lint::visitCallSite(CallSite CS) {
         if (Len->getValue().isIntN(32))
           Size = Len->getValue().getZExtValue();
       Assert(AA->alias(MCI->getSource(), Size, MCI->getDest(), Size) !=
-                 AliasAnalysis::MustAlias,
+                 MustAlias,
              "Undefined behavior: memcpy source and destination overlap", &I);
       break;
     }
     case Intrinsic::memmove: {
       MemMoveInst *MMI = cast<MemMoveInst>(&I);
       // TODO: If the size is known, use it.
-      visitMemoryReference(I, MMI->getDest(), AliasAnalysis::UnknownSize,
-                           MMI->getAlignment(), nullptr,
-                           MemRef::Write);
-      visitMemoryReference(I, MMI->getSource(), AliasAnalysis::UnknownSize,
-                           MMI->getAlignment(), nullptr,
-                           MemRef::Read);
+      visitMemoryReference(I, MMI->getDest(), MemoryLocation::UnknownSize,
+                           MMI->getAlignment(), nullptr, MemRef::Write);
+      visitMemoryReference(I, MMI->getSource(), MemoryLocation::UnknownSize,
+                           MMI->getAlignment(), nullptr, MemRef::Read);
       break;
     }
     case Intrinsic::memset: {
       MemSetInst *MSI = cast<MemSetInst>(&I);
       // TODO: If the size is known, use it.
-      visitMemoryReference(I, MSI->getDest(), AliasAnalysis::UnknownSize,
-                           MSI->getAlignment(), nullptr,
-                           MemRef::Write);
+      visitMemoryReference(I, MSI->getDest(), MemoryLocation::UnknownSize,
+                           MSI->getAlignment(), nullptr, MemRef::Write);
       break;
     }
 
@@ -328,26 +322,26 @@ void Lint::visitCallSite(CallSite CS) {
              "Undefined behavior: va_start called in a non-varargs function",
              &I);
 
-      visitMemoryReference(I, CS.getArgument(0), AliasAnalysis::UnknownSize,
-                           0, nullptr, MemRef::Read | MemRef::Write);
+      visitMemoryReference(I, CS.getArgument(0), MemoryLocation::UnknownSize, 0,
+                           nullptr, MemRef::Read | MemRef::Write);
       break;
     case Intrinsic::vacopy:
-      visitMemoryReference(I, CS.getArgument(0), AliasAnalysis::UnknownSize,
-                           0, nullptr, MemRef::Write);
-      visitMemoryReference(I, CS.getArgument(1), AliasAnalysis::UnknownSize,
-                           0, nullptr, MemRef::Read);
+      visitMemoryReference(I, CS.getArgument(0), MemoryLocation::UnknownSize, 0,
+                           nullptr, MemRef::Write);
+      visitMemoryReference(I, CS.getArgument(1), MemoryLocation::UnknownSize, 0,
+                           nullptr, MemRef::Read);
       break;
     case Intrinsic::vaend:
-      visitMemoryReference(I, CS.getArgument(0), AliasAnalysis::UnknownSize,
-                           0, nullptr, MemRef::Read | MemRef::Write);
+      visitMemoryReference(I, CS.getArgument(0), MemoryLocation::UnknownSize, 0,
+                           nullptr, MemRef::Read | MemRef::Write);
       break;
 
     case Intrinsic::stackrestore:
       // Stackrestore doesn't read or write memory, but it sets the
       // stack pointer, which the compiler may read from or write to
       // at any time, so check it for both readability and writeability.
-      visitMemoryReference(I, CS.getArgument(0), AliasAnalysis::UnknownSize,
-                           0, nullptr, MemRef::Read | MemRef::Write);
+      visitMemoryReference(I, CS.getArgument(0), MemoryLocation::UnknownSize, 0,
+                           nullptr, MemRef::Read | MemRef::Write);
       break;
 
     case Intrinsic::eh_begincatch:
@@ -435,7 +429,7 @@ void Lint::visitMemoryReference(Instruction &I,
     // OK, so the access is to a constant offset from Ptr.  Check that Ptr is
     // something we can handle and if so extract the size of this base object
     // along with its alignment.
-    uint64_t BaseSize = AliasAnalysis::UnknownSize;
+    uint64_t BaseSize = MemoryLocation::UnknownSize;
     unsigned BaseAlign = 0;
 
     if (AllocaInst *AI = dyn_cast<AllocaInst>(Base)) {
@@ -460,8 +454,8 @@ void Lint::visitMemoryReference(Instruction &I,
 
     // Accesses from before the start or after the end of the object are not
     // defined.
-    Assert(Size == AliasAnalysis::UnknownSize ||
-               BaseSize == AliasAnalysis::UnknownSize ||
+    Assert(Size == MemoryLocation::UnknownSize ||
+               BaseSize == MemoryLocation::UnknownSize ||
                (Offset >= 0 && Offset + Size <= BaseSize),
            "Undefined behavior: Buffer overflow", &I);
 
@@ -770,12 +764,12 @@ void Lint::visitAllocaInst(AllocaInst &I) {
 }
 
 void Lint::visitVAArgInst(VAArgInst &I) {
-  visitMemoryReference(I, I.getOperand(0), AliasAnalysis::UnknownSize, 0,
+  visitMemoryReference(I, I.getOperand(0), MemoryLocation::UnknownSize, 0,
                        nullptr, MemRef::Read | MemRef::Write);
 }
 
 void Lint::visitIndirectBrInst(IndirectBrInst &I) {
-  visitMemoryReference(I, I.getAddress(), AliasAnalysis::UnknownSize, 0,
+  visitMemoryReference(I, I.getAddress(), MemoryLocation::UnknownSize, 0,
                        nullptr, MemRef::Branchee);
 
   Assert(I.getNumDestinations() != 0,
diff --git a/lib/Analysis/Loads.cpp b/lib/Analysis/Loads.cpp
index aed3b04ebca..4d6d960f159 100644
--- a/lib/Analysis/Loads.cpp
+++ b/lib/Analysis/Loads.cpp
@@ -65,6 +65,12 @@ static bool AreEquivalentAddressValues(const Value *A, const Value *B) {
 bool llvm::isSafeToLoadUnconditionally(Value *V, Instruction *ScanFrom,
                                        unsigned Align) {
   const DataLayout &DL = ScanFrom->getModule()->getDataLayout();
+
+  // Zero alignment means that the load has the ABI alignment for the target
+  if (Align == 0)
+    Align = DL.getABITypeAlignment(V->getType()->getPointerElementType());
+  assert(isPowerOf2_32(Align));
+
   int64_t ByteOffset = 0;
   Value *Base = V;
   Base = GetPointerBaseWithConstantOffset(V, ByteOffset, DL);
@@ -102,7 +108,7 @@ bool llvm::isSafeToLoadUnconditionally(Value *V, Instruction *ScanFrom,
     if (Align <= BaseAlign) {
       // Check if the load is within the bounds of the underlying object.
       if (ByteOffset + LoadSize <= DL.getTypeAllocSize(BaseType) &&
-          (Align == 0 || (ByteOffset % Align) == 0))
+          ((ByteOffset % Align) == 0))
         return true;
     }
   }
@@ -128,20 +134,28 @@ bool llvm::isSafeToLoadUnconditionally(Value *V, Instruction *ScanFrom,
       return false;
 
     Value *AccessedPtr;
-    if (LoadInst *LI = dyn_cast<LoadInst>(BBI))
+    unsigned AccessedAlign;
+    if (LoadInst *LI = dyn_cast<LoadInst>(BBI)) {
       AccessedPtr = LI->getPointerOperand();
-    else if (StoreInst *SI = dyn_cast<StoreInst>(BBI))
+      AccessedAlign = LI->getAlignment();
+    } else if (StoreInst *SI = dyn_cast<StoreInst>(BBI)) {
       AccessedPtr = SI->getPointerOperand();
-    else
+      AccessedAlign = SI->getAlignment();
+    } else
+      continue;
+
+    Type *AccessedTy = AccessedPtr->getType()->getPointerElementType();
+    if (AccessedAlign == 0)
+      AccessedAlign = DL.getABITypeAlignment(AccessedTy);
+    if (AccessedAlign < Align)
       continue;
 
     // Handle trivial cases.
     if (AccessedPtr == V)
       return true;
 
-    auto *AccessedTy = cast<PointerType>(AccessedPtr->getType());
     if (AreEquivalentAddressValues(AccessedPtr->stripPointerCasts(), V) &&
-        LoadSize <= DL.getTypeStoreSize(AccessedTy->getElementType()))
+        LoadSize <= DL.getTypeStoreSize(AccessedTy))
       return true;
   }
   return false;
@@ -232,9 +246,7 @@ Value *llvm::FindAvailableLoadedValue(Value *Ptr, BasicBlock *ScanBB,
 
       // If we have alias analysis and it says the store won't modify the loaded
       // value, ignore the store.
-      if (AA &&
-          (AA->getModRefInfo(SI, StrippedPtr, AccessSize) &
-           AliasAnalysis::Mod) == 0)
+      if (AA && (AA->getModRefInfo(SI, StrippedPtr, AccessSize) & MRI_Mod) == 0)
         continue;
 
       // Otherwise the store that may or may not alias the pointer, bail out.
@@ -247,8 +259,7 @@ Value *llvm::FindAvailableLoadedValue(Value *Ptr, BasicBlock *ScanBB,
       // If alias analysis claims that it really won't modify the load,
       // ignore it.
       if (AA &&
-          (AA->getModRefInfo(Inst, StrippedPtr, AccessSize) &
-           AliasAnalysis::Mod) == 0)
+          (AA->getModRefInfo(Inst, StrippedPtr, AccessSize) & MRI_Mod) == 0)
         continue;
 
       // May modify the pointer, bail out.
diff --git a/lib/Analysis/LoopAccessAnalysis.cpp b/lib/Analysis/LoopAccessAnalysis.cpp
index 724c21f18c9..7e36a1b961f 100644
--- a/lib/Analysis/LoopAccessAnalysis.cpp
+++ b/lib/Analysis/LoopAccessAnalysis.cpp
@@ -22,7 +22,7 @@
 #include "llvm/IR/IRBuilder.h"
 #include "llvm/Support/Debug.h"
 #include "llvm/Support/raw_ostream.h"
-#include "llvm/Transforms/Utils/VectorUtils.h"
+#include "llvm/Analysis/VectorUtils.h"
 using namespace llvm;
 
 #define DEBUG_TYPE "loop-accesses"
@@ -48,6 +48,13 @@ static cl::opt<unsigned, true> RuntimeMemoryCheckThreshold(
     cl::location(VectorizerParams::RuntimeMemoryCheckThreshold), cl::init(8));
 unsigned VectorizerParams::RuntimeMemoryCheckThreshold;
 
+/// \brief The maximum iterations used to merge memory checks
+static cl::opt<unsigned> MemoryCheckMergeThreshold(
+    "memory-check-merge-threshold", cl::Hidden,
+    cl::desc("Maximum number of comparisons done when trying to merge "
+             "runtime memory checks. (default = 100)"),
+    cl::init(100));
+
 /// Maximum SIMD width.
 const unsigned VectorizerParams::MaxVectorWidth = 64;
 
@@ -112,72 +119,308 @@ const SCEV *llvm::replaceSymbolicStrideSCEV(ScalarEvolution *SE,
   return SE->getSCEV(Ptr);
 }
 
-void LoopAccessInfo::RuntimePointerCheck::insert(
-    ScalarEvolution *SE, Loop *Lp, Value *Ptr, bool WritePtr, unsigned DepSetId,
-    unsigned ASId, const ValueToValueMap &Strides) {
+void RuntimePointerChecking::insert(Loop *Lp, Value *Ptr, bool WritePtr,
+                                    unsigned DepSetId, unsigned ASId,
+                                    const ValueToValueMap &Strides) {
   // Get the stride replaced scev.
   const SCEV *Sc = replaceSymbolicStrideSCEV(SE, Strides, Ptr);
   const SCEVAddRecExpr *AR = dyn_cast<SCEVAddRecExpr>(Sc);
   assert(AR && "Invalid addrec expression");
   const SCEV *Ex = SE->getBackedgeTakenCount(Lp);
+
+  const SCEV *ScStart = AR->getStart();
   const SCEV *ScEnd = AR->evaluateAtIteration(Ex, *SE);
-  Pointers.push_back(Ptr);
-  Starts.push_back(AR->getStart());
-  Ends.push_back(ScEnd);
-  IsWritePtr.push_back(WritePtr);
-  DependencySetId.push_back(DepSetId);
-  AliasSetId.push_back(ASId);
+  const SCEV *Step = AR->getStepRecurrence(*SE);
+
+  // For expressions with negative step, the upper bound is ScStart and the
+  // lower bound is ScEnd.
+  if (const SCEVConstant *CStep = dyn_cast<const SCEVConstant>(Step)) {
+    if (CStep->getValue()->isNegative())
+      std::swap(ScStart, ScEnd);
+  } else {
+    // Fallback case: the step is not constant, but the we can still
+    // get the upper and lower bounds of the interval by using min/max
+    // expressions.
+    ScStart = SE->getUMinExpr(ScStart, ScEnd);
+    ScEnd = SE->getUMaxExpr(AR->getStart(), ScEnd);
+  }
+
+  Pointers.emplace_back(Ptr, ScStart, ScEnd, WritePtr, DepSetId, ASId, Sc);
+}
+
+SmallVector<RuntimePointerChecking::PointerCheck, 4>
+RuntimePointerChecking::generateChecks(
+    const SmallVectorImpl<int> *PtrPartition) const {
+  SmallVector<PointerCheck, 4> Checks;
+
+  for (unsigned I = 0; I < CheckingGroups.size(); ++I) {
+    for (unsigned J = I + 1; J < CheckingGroups.size(); ++J) {
+      const RuntimePointerChecking::CheckingPtrGroup &CGI = CheckingGroups[I];
+      const RuntimePointerChecking::CheckingPtrGroup &CGJ = CheckingGroups[J];
+
+      if (needsChecking(CGI, CGJ, PtrPartition))
+        Checks.push_back(std::make_pair(&CGI, &CGJ));
+    }
+  }
+  return Checks;
+}
+
+bool RuntimePointerChecking::needsChecking(
+    const CheckingPtrGroup &M, const CheckingPtrGroup &N,
+    const SmallVectorImpl<int> *PtrPartition) const {
+  for (unsigned I = 0, EI = M.Members.size(); EI != I; ++I)
+    for (unsigned J = 0, EJ = N.Members.size(); EJ != J; ++J)
+      if (needsChecking(M.Members[I], N.Members[J], PtrPartition))
+        return true;
+  return false;
+}
+
+/// Compare \p I and \p J and return the minimum.
+/// Return nullptr in case we couldn't find an answer.
+static const SCEV *getMinFromExprs(const SCEV *I, const SCEV *J,
+                                   ScalarEvolution *SE) {
+  const SCEV *Diff = SE->getMinusSCEV(J, I);
+  const SCEVConstant *C = dyn_cast<const SCEVConstant>(Diff);
+
+  if (!C)
+    return nullptr;
+  if (C->getValue()->isNegative())
+    return J;
+  return I;
+}
+
+bool RuntimePointerChecking::CheckingPtrGroup::addPointer(unsigned Index) {
+  const SCEV *Start = RtCheck.Pointers[Index].Start;
+  const SCEV *End = RtCheck.Pointers[Index].End;
+
+  // Compare the starts and ends with the known minimum and maximum
+  // of this set. We need to know how we compare against the min/max
+  // of the set in order to be able to emit memchecks.
+  const SCEV *Min0 = getMinFromExprs(Start, Low, RtCheck.SE);
+  if (!Min0)
+    return false;
+
+  const SCEV *Min1 = getMinFromExprs(End, High, RtCheck.SE);
+  if (!Min1)
+    return false;
+
+  // Update the low bound  expression if we've found a new min value.
+  if (Min0 == Start)
+    Low = Start;
+
+  // Update the high bound expression if we've found a new max value.
+  if (Min1 != End)
+    High = End;
+
+  Members.push_back(Index);
+  return true;
+}
+
+void RuntimePointerChecking::groupChecks(
+    MemoryDepChecker::DepCandidates &DepCands, bool UseDependencies) {
+  // We build the groups from dependency candidates equivalence classes
+  // because:
+  //    - We know that pointers in the same equivalence class share
+  //      the same underlying object and therefore there is a chance
+  //      that we can compare pointers
+  //    - We wouldn't be able to merge two pointers for which we need
+  //      to emit a memcheck. The classes in DepCands are already
+  //      conveniently built such that no two pointers in the same
+  //      class need checking against each other.
+
+  // We use the following (greedy) algorithm to construct the groups
+  // For every pointer in the equivalence class:
+  //   For each existing group:
+  //   - if the difference between this pointer and the min/max bounds
+  //     of the group is a constant, then make the pointer part of the
+  //     group and update the min/max bounds of that group as required.
+
+  CheckingGroups.clear();
+
+  // If we need to check two pointers to the same underlying object
+  // with a non-constant difference, we shouldn't perform any pointer
+  // grouping with those pointers. This is because we can easily get
+  // into cases where the resulting check would return false, even when
+  // the accesses are safe.
+  //
+  // The following example shows this:
+  // for (i = 0; i < 1000; ++i)
+  //   a[5000 + i * m] = a[i] + a[i + 9000]
+  //
+  // Here grouping gives a check of (5000, 5000 + 1000 * m) against
+  // (0, 10000) which is always false. However, if m is 1, there is no
+  // dependence. Not grouping the checks for a[i] and a[i + 9000] allows
+  // us to perform an accurate check in this case.
+  //
+  // The above case requires that we have an UnknownDependence between
+  // accesses to the same underlying object. This cannot happen unless
+  // ShouldRetryWithRuntimeCheck is set, and therefore UseDependencies
+  // is also false. In this case we will use the fallback path and create
+  // separate checking groups for all pointers.
+ 
+  // If we don't have the dependency partitions, construct a new
+  // checking pointer group for each pointer. This is also required
+  // for correctness, because in this case we can have checking between
+  // pointers to the same underlying object.
+  if (!UseDependencies) {
+    for (unsigned I = 0; I < Pointers.size(); ++I)
+      CheckingGroups.push_back(CheckingPtrGroup(I, *this));
+    return;
+  }
+
+  unsigned TotalComparisons = 0;
+
+  DenseMap<Value *, unsigned> PositionMap;
+  for (unsigned Index = 0; Index < Pointers.size(); ++Index)
+    PositionMap[Pointers[Index].PointerValue] = Index;
+
+  // We need to keep track of what pointers we've already seen so we
+  // don't process them twice.
+  SmallSet<unsigned, 2> Seen;
+
+  // Go through all equivalence classes, get the the "pointer check groups"
+  // and add them to the overall solution. We use the order in which accesses
+  // appear in 'Pointers' to enforce determinism.
+  for (unsigned I = 0; I < Pointers.size(); ++I) {
+    // We've seen this pointer before, and therefore already processed
+    // its equivalence class.
+    if (Seen.count(I))
+      continue;
+
+    MemoryDepChecker::MemAccessInfo Access(Pointers[I].PointerValue,
+                                           Pointers[I].IsWritePtr);
+
+    SmallVector<CheckingPtrGroup, 2> Groups;
+    auto LeaderI = DepCands.findValue(DepCands.getLeaderValue(Access));
+
+    // Because DepCands is constructed by visiting accesses in the order in
+    // which they appear in alias sets (which is deterministic) and the
+    // iteration order within an equivalence class member is only dependent on
+    // the order in which unions and insertions are performed on the
+    // equivalence class, the iteration order is deterministic.
+    for (auto MI = DepCands.member_begin(LeaderI), ME = DepCands.member_end();
+         MI != ME; ++MI) {
+      unsigned Pointer = PositionMap[MI->getPointer()];
+      bool Merged = false;
+      // Mark this pointer as seen.
+      Seen.insert(Pointer);
+
+      // Go through all the existing sets and see if we can find one
+      // which can include this pointer.
+      for (CheckingPtrGroup &Group : Groups) {
+        // Don't perform more than a certain amount of comparisons.
+        // This should limit the cost of grouping the pointers to something
+        // reasonable.  If we do end up hitting this threshold, the algorithm
+        // will create separate groups for all remaining pointers.
+        if (TotalComparisons > MemoryCheckMergeThreshold)
+          break;
+
+        TotalComparisons++;
+
+        if (Group.addPointer(Pointer)) {
+          Merged = true;
+          break;
+        }
+      }
+
+      if (!Merged)
+        // We couldn't add this pointer to any existing set or the threshold
+        // for the number of comparisons has been reached. Create a new group
+        // to hold the current pointer.
+        Groups.push_back(CheckingPtrGroup(Pointer, *this));
+    }
+
+    // We've computed the grouped checks for this partition.
+    // Save the results and continue with the next one.
+    std::copy(Groups.begin(), Groups.end(), std::back_inserter(CheckingGroups));
+  }
 }
 
-bool LoopAccessInfo::RuntimePointerCheck::needsChecking(
+bool RuntimePointerChecking::arePointersInSamePartition(
+    const SmallVectorImpl<int> &PtrToPartition, unsigned PtrIdx1,
+    unsigned PtrIdx2) {
+  return (PtrToPartition[PtrIdx1] != -1 &&
+          PtrToPartition[PtrIdx1] == PtrToPartition[PtrIdx2]);
+}
+
+bool RuntimePointerChecking::needsChecking(
     unsigned I, unsigned J, const SmallVectorImpl<int> *PtrPartition) const {
+  const PointerInfo &PointerI = Pointers[I];
+  const PointerInfo &PointerJ = Pointers[J];
+
   // No need to check if two readonly pointers intersect.
-  if (!IsWritePtr[I] && !IsWritePtr[J])
+  if (!PointerI.IsWritePtr && !PointerJ.IsWritePtr)
     return false;
 
   // Only need to check pointers between two different dependency sets.
-  if (DependencySetId[I] == DependencySetId[J])
+  if (PointerI.DependencySetId == PointerJ.DependencySetId)
     return false;
 
   // Only need to check pointers in the same alias set.
-  if (AliasSetId[I] != AliasSetId[J])
+  if (PointerI.AliasSetId != PointerJ.AliasSetId)
     return false;
 
   // If PtrPartition is set omit checks between pointers of the same partition.
-  // Partition number -1 means that the pointer is used in multiple partitions.
-  // In this case we can't omit the check.
-  if (PtrPartition && (*PtrPartition)[I] != -1 &&
-      (*PtrPartition)[I] == (*PtrPartition)[J])
+  if (PtrPartition && arePointersInSamePartition(*PtrPartition, I, J))
     return false;
 
   return true;
 }
 
-void LoopAccessInfo::RuntimePointerCheck::print(
+void RuntimePointerChecking::printChecks(
+    raw_ostream &OS, const SmallVectorImpl<PointerCheck> &Checks,
+    unsigned Depth) const {
+  unsigned N = 0;
+  for (const auto &Check : Checks) {
+    const auto &First = Check.first->Members, &Second = Check.second->Members;
+
+    OS.indent(Depth) << "Check " << N++ << ":\n";
+
+    OS.indent(Depth + 2) << "Comparing group (" << Check.first << "):\n";
+    for (unsigned K = 0; K < First.size(); ++K)
+      OS.indent(Depth + 2) << *Pointers[First[K]].PointerValue << "\n";
+
+    OS.indent(Depth + 2) << "Against group (" << Check.second << "):\n";
+    for (unsigned K = 0; K < Second.size(); ++K)
+      OS.indent(Depth + 2) << *Pointers[Second[K]].PointerValue << "\n";
+  }
+}
+
+void RuntimePointerChecking::print(
     raw_ostream &OS, unsigned Depth,
     const SmallVectorImpl<int> *PtrPartition) const {
-  unsigned NumPointers = Pointers.size();
-  if (NumPointers == 0)
-    return;
 
   OS.indent(Depth) << "Run-time memory checks:\n";
-  unsigned N = 0;
-  for (unsigned I = 0; I < NumPointers; ++I)
-    for (unsigned J = I + 1; J < NumPointers; ++J)
-      if (needsChecking(I, J, PtrPartition)) {
-        OS.indent(Depth) << N++ << ":\n";
-        OS.indent(Depth + 2) << *Pointers[I];
-        if (PtrPartition)
-          OS << " (Partition: " << (*PtrPartition)[I] << ")";
-        OS << "\n";
-        OS.indent(Depth + 2) << *Pointers[J];
-        if (PtrPartition)
-          OS << " (Partition: " << (*PtrPartition)[J] << ")";
-        OS << "\n";
-      }
+  printChecks(OS, generateChecks(PtrPartition), Depth);
+
+  OS.indent(Depth) << "Grouped accesses:\n";
+  for (unsigned I = 0; I < CheckingGroups.size(); ++I) {
+    const auto &CG = CheckingGroups[I];
+
+    OS.indent(Depth + 2) << "Group " << &CG << ":\n";
+    OS.indent(Depth + 4) << "(Low: " << *CG.Low << " High: " << *CG.High
+                         << ")\n";
+    for (unsigned J = 0; J < CG.Members.size(); ++J) {
+      OS.indent(Depth + 6) << "Member: " << *Pointers[CG.Members[J]].Expr
+                           << "\n";
+    }
+  }
+}
+
+unsigned RuntimePointerChecking::getNumberOfChecks(
+    const SmallVectorImpl<int> *PtrPartition) const {
+
+  unsigned NumPartitions = CheckingGroups.size();
+  unsigned CheckCount = 0;
+
+  for (unsigned I = 0; I < NumPartitions; ++I)
+    for (unsigned J = I + 1; J < NumPartitions; ++J)
+      if (needsChecking(CheckingGroups[I], CheckingGroups[J], PtrPartition))
+        CheckCount++;
+  return CheckCount;
 }
 
-bool LoopAccessInfo::RuntimePointerCheck::needsAnyChecking(
+bool RuntimePointerChecking::needsAnyChecking(
     const SmallVectorImpl<int> *PtrPartition) const {
   unsigned NumPointers = Pointers.size();
 
@@ -199,30 +442,33 @@ class AccessAnalysis {
   typedef PointerIntPair<Value *, 1, bool> MemAccessInfo;
   typedef SmallPtrSet<MemAccessInfo, 8> MemAccessInfoSet;
 
-  AccessAnalysis(const DataLayout &Dl, AliasAnalysis *AA,
+  AccessAnalysis(const DataLayout &Dl, AliasAnalysis *AA, LoopInfo *LI,
                  MemoryDepChecker::DepCandidates &DA)
-      : DL(Dl), AST(*AA), DepCands(DA), IsRTCheckNeeded(false) {}
+      : DL(Dl), AST(*AA), LI(LI), DepCands(DA),
+        IsRTCheckAnalysisNeeded(false) {}
 
   /// \brief Register a load  and whether it is only read from.
-  void addLoad(AliasAnalysis::Location &Loc, bool IsReadOnly) {
+  void addLoad(MemoryLocation &Loc, bool IsReadOnly) {
     Value *Ptr = const_cast<Value*>(Loc.Ptr);
-    AST.add(Ptr, AliasAnalysis::UnknownSize, Loc.AATags);
+    AST.add(Ptr, MemoryLocation::UnknownSize, Loc.AATags);
     Accesses.insert(MemAccessInfo(Ptr, false));
     if (IsReadOnly)
       ReadOnlyPtr.insert(Ptr);
   }
 
   /// \brief Register a store.
-  void addStore(AliasAnalysis::Location &Loc) {
+  void addStore(MemoryLocation &Loc) {
     Value *Ptr = const_cast<Value*>(Loc.Ptr);
-    AST.add(Ptr, AliasAnalysis::UnknownSize, Loc.AATags);
+    AST.add(Ptr, MemoryLocation::UnknownSize, Loc.AATags);
     Accesses.insert(MemAccessInfo(Ptr, true));
   }
 
   /// \brief Check whether we can check the pointers at runtime for
   /// non-intersection.
-  bool canCheckPtrAtRT(LoopAccessInfo::RuntimePointerCheck &RtCheck,
-                       unsigned &NumComparisons, ScalarEvolution *SE,
+  ///
+  /// Returns true if we need no check or if we do and we can generate them
+  /// (i.e. the pointers have computable bounds).
+  bool canCheckPtrAtRT(RuntimePointerChecking &RtCheck, ScalarEvolution *SE,
                        Loop *TheLoop, const ValueToValueMap &Strides,
                        bool ShouldCheckStride = false);
 
@@ -232,10 +478,18 @@ class AccessAnalysis {
     processMemAccesses();
   }
 
-  bool isRTCheckNeeded() { return IsRTCheckNeeded; }
-
+  /// \brief Initial processing of memory accesses determined that we need to
+  /// perform dependency checking.
+  ///
+  /// Note that this can later be cleared if we retry memcheck analysis without
+  /// dependency checking (i.e. ShouldRetryWithRuntimeCheck).
   bool isDependencyCheckNeeded() { return !CheckDeps.empty(); }
-  void resetDepChecks() { CheckDeps.clear(); }
+
+  /// We decided that no dependence analysis would be used.  Reset the state.
+  void resetDepChecks(MemoryDepChecker &DepChecker) {
+    CheckDeps.clear();
+    DepChecker.clearInterestingDependences();
+  }
 
   MemAccessInfoSet &getDependenciesToCheck() { return CheckDeps; }
 
@@ -243,7 +497,7 @@ class AccessAnalysis {
   typedef SetVector<MemAccessInfo> PtrAccessSet;
 
   /// \brief Go over all memory access and check whether runtime pointer checks
-  /// are needed /// and build sets of dependency check candidates.
+  /// are needed and build sets of dependency check candidates.
   void processMemAccesses();
 
   /// Set of all accesses.
@@ -261,12 +515,21 @@ class AccessAnalysis {
   //intrinsic property (such as TBAA metadata).
   AliasSetTracker AST;
 
+  LoopInfo *LI;
+
   /// Sets of potentially dependent accesses - members of one set share an
   /// underlying pointer. The set "CheckDeps" identfies which sets really need a
   /// dependence check.
   MemoryDepChecker::DepCandidates &DepCands;
 
-  bool IsRTCheckNeeded;
+  /// \brief Initial processing of memory accesses determined that we may need
+  /// to add memchecks.  Perform the analysis to determine the necessary checks.
+  ///
+  /// Note that, this is different from isDependencyCheckNeeded.  When we retry
+  /// memcheck analysis without dependency checking
+  /// (i.e. ShouldRetryWithRuntimeCheck), isDependencyCheckNeeded is cleared
+  /// while this remains set if we have potentially dependent accesses.
+  bool IsRTCheckAnalysisNeeded;
 };
 
 } // end anonymous namespace
@@ -282,28 +545,25 @@ static bool hasComputableBounds(ScalarEvolution *SE,
   return AR->isAffine();
 }
 
-/// \brief Check the stride of the pointer and ensure that it does not wrap in
-/// the address space.
-static int isStridedPtr(ScalarEvolution *SE, Value *Ptr, const Loop *Lp,
-                        const ValueToValueMap &StridesMap);
-
-bool AccessAnalysis::canCheckPtrAtRT(
-    LoopAccessInfo::RuntimePointerCheck &RtCheck, unsigned &NumComparisons,
-    ScalarEvolution *SE, Loop *TheLoop, const ValueToValueMap &StridesMap,
-    bool ShouldCheckStride) {
+bool AccessAnalysis::canCheckPtrAtRT(RuntimePointerChecking &RtCheck,
+                                     ScalarEvolution *SE, Loop *TheLoop,
+                                     const ValueToValueMap &StridesMap,
+                                     bool ShouldCheckStride) {
   // Find pointers with computable bounds. We are going to use this information
   // to place a runtime bound check.
   bool CanDoRT = true;
 
+  bool NeedRTCheck = false;
+  if (!IsRTCheckAnalysisNeeded) return true;
+
   bool IsDepCheckNeeded = isDependencyCheckNeeded();
-  NumComparisons = 0;
 
   // We assign a consecutive id to access from different alias sets.
   // Accesses between different groups doesn't need to be checked.
   unsigned ASId = 1;
   for (auto &AS : AST) {
-    unsigned NumReadPtrChecks = 0;
-    unsigned NumWritePtrChecks = 0;
+    int NumReadPtrChecks = 0;
+    int NumWritePtrChecks = 0;
 
     // We assign consecutive id to access from different dependence sets.
     // Accesses within the same set don't need a runtime check.
@@ -338,20 +598,26 @@ bool AccessAnalysis::canCheckPtrAtRT(
           // Each access has its own dependence set.
           DepId = RunningDepId++;
 
-        RtCheck.insert(SE, TheLoop, Ptr, IsWrite, DepId, ASId, StridesMap);
+        RtCheck.insert(TheLoop, Ptr, IsWrite, DepId, ASId, StridesMap);
 
         DEBUG(dbgs() << "LAA: Found a runtime check ptr:" << *Ptr << '\n');
       } else {
+        DEBUG(dbgs() << "LAA: Can't find bounds for ptr:" << *Ptr << '\n');
         CanDoRT = false;
       }
     }
 
-    if (IsDepCheckNeeded && CanDoRT && RunningDepId == 2)
-      NumComparisons += 0; // Only one dependence set.
-    else {
-      NumComparisons += (NumWritePtrChecks * (NumReadPtrChecks +
-                                              NumWritePtrChecks - 1));
-    }
+    // If we have at least two writes or one write and a read then we need to
+    // check them.  But there is no need to checks if there is only one
+    // dependence set for this alias set.
+    //
+    // Note that this function computes CanDoRT and NeedRTCheck independently.
+    // For example CanDoRT=false, NeedRTCheck=false means that we have a pointer
+    // for which we couldn't find the bounds but we don't actually need to emit
+    // any checks so it does not matter.
+    if (!(IsDepCheckNeeded && CanDoRT && RunningDepId == 2))
+      NeedRTCheck |= (NumWritePtrChecks >= 2 || (NumReadPtrChecks >= 1 &&
+                                                 NumWritePtrChecks >= 1));
 
     ++ASId;
   }
@@ -365,14 +631,15 @@ bool AccessAnalysis::canCheckPtrAtRT(
   for (unsigned i = 0; i < NumPointers; ++i) {
     for (unsigned j = i + 1; j < NumPointers; ++j) {
       // Only need to check pointers between two different dependency sets.
-      if (RtCheck.DependencySetId[i] == RtCheck.DependencySetId[j])
+      if (RtCheck.Pointers[i].DependencySetId ==
+          RtCheck.Pointers[j].DependencySetId)
        continue;
       // Only need to check pointers in the same alias set.
-      if (RtCheck.AliasSetId[i] != RtCheck.AliasSetId[j])
+      if (RtCheck.Pointers[i].AliasSetId != RtCheck.Pointers[j].AliasSetId)
         continue;
 
-      Value *PtrI = RtCheck.Pointers[i];
-      Value *PtrJ = RtCheck.Pointers[j];
+      Value *PtrI = RtCheck.Pointers[i].PointerValue;
+      Value *PtrJ = RtCheck.Pointers[j].PointerValue;
 
       unsigned ASi = PtrI->getType()->getPointerAddressSpace();
       unsigned ASj = PtrJ->getType()->getPointerAddressSpace();
@@ -384,7 +651,18 @@ bool AccessAnalysis::canCheckPtrAtRT(
     }
   }
 
-  return CanDoRT;
+  if (NeedRTCheck && CanDoRT)
+    RtCheck.groupChecks(DepCands, IsDepCheckNeeded);
+
+  DEBUG(dbgs() << "LAA: We need to do " << RtCheck.getNumberOfChecks(nullptr)
+               << " pointer comparisons.\n");
+
+  RtCheck.Need = NeedRTCheck;
+
+  bool CanDoRTIfNeeded = !NeedRTCheck || CanDoRT;
+  if (!CanDoRTIfNeeded)
+    RtCheck.reset();
+  return CanDoRTIfNeeded;
 }
 
 void AccessAnalysis::processMemAccesses() {
@@ -467,7 +745,7 @@ void AccessAnalysis::processMemAccesses() {
           // catch "a[i] = a[i] + " without having to do a dependence check).
           if ((IsWrite || IsReadOnlyPtr) && SetHasWrite) {
             CheckDeps.insert(Access);
-            IsRTCheckNeeded = true;
+            IsRTCheckAnalysisNeeded = true;
           }
 
           if (IsWrite)
@@ -477,7 +755,9 @@ void AccessAnalysis::processMemAccesses() {
           // underlying object.
           typedef SmallVector<Value *, 16> ValueVector;
           ValueVector TempObjects;
-          GetUnderlyingObjects(Ptr, TempObjects, DL);
+
+          GetUnderlyingObjects(Ptr, TempObjects, DL, LI);
+          DEBUG(dbgs() << "Underlying objects for pointer " << *Ptr << "\n");
           for (Value *UnderlyingObj : TempObjects) {
             UnderlyingObjToAccessMap::iterator Prev =
                 ObjToLastAccess.find(UnderlyingObj);
@@ -485,6 +765,7 @@ void AccessAnalysis::processMemAccesses() {
               DepCands.unionSets(Access, Prev->second);
 
             ObjToLastAccess[UnderlyingObj] = Access;
+            DEBUG(dbgs() << "  " << *UnderlyingObj << "\n");
           }
         }
       }
@@ -498,9 +779,57 @@ static bool isInBoundsGep(Value *Ptr) {
   return false;
 }
 
+/// \brief Return true if an AddRec pointer \p Ptr is unsigned non-wrapping,
+/// i.e. monotonically increasing/decreasing.
+static bool isNoWrapAddRec(Value *Ptr, const SCEVAddRecExpr *AR,
+                           ScalarEvolution *SE, const Loop *L) {
+  // FIXME: This should probably only return true for NUW.
+  if (AR->getNoWrapFlags(SCEV::NoWrapMask))
+    return true;
+
+  // Scalar evolution does not propagate the non-wrapping flags to values that
+  // are derived from a non-wrapping induction variable because non-wrapping
+  // could be flow-sensitive.
+  //
+  // Look through the potentially overflowing instruction to try to prove
+  // non-wrapping for the *specific* value of Ptr.
+
+  // The arithmetic implied by an inbounds GEP can't overflow.
+  auto *GEP = dyn_cast<GetElementPtrInst>(Ptr);
+  if (!GEP || !GEP->isInBounds())
+    return false;
+
+  // Make sure there is only one non-const index and analyze that.
+  Value *NonConstIndex = nullptr;
+  for (auto Index = GEP->idx_begin(); Index != GEP->idx_end(); ++Index)
+    if (!isa<ConstantInt>(*Index)) {
+      if (NonConstIndex)
+        return false;
+      NonConstIndex = *Index;
+    }
+  if (!NonConstIndex)
+    // The recurrence is on the pointer, ignore for now.
+    return false;
+
+  // The index in GEP is signed.  It is non-wrapping if it's derived from a NSW
+  // AddRec using a NSW operation.
+  if (auto *OBO = dyn_cast<OverflowingBinaryOperator>(NonConstIndex))
+    if (OBO->hasNoSignedWrap() &&
+        // Assume constant for other the operand so that the AddRec can be
+        // easily found.
+        isa<ConstantInt>(OBO->getOperand(1))) {
+      auto *OpScev = SE->getSCEV(OBO->getOperand(0));
+
+      if (auto *OpAR = dyn_cast<SCEVAddRecExpr>(OpScev))
+        return OpAR->getLoop() == L && OpAR->getNoWrapFlags(SCEV::FlagNSW);
+    }
+
+  return false;
+}
+
 /// \brief Check whether the access through \p Ptr has a constant stride.
-static int isStridedPtr(ScalarEvolution *SE, Value *Ptr, const Loop *Lp,
-                        const ValueToValueMap &StridesMap) {
+int llvm::isStridedPtr(ScalarEvolution *SE, Value *Ptr, const Loop *Lp,
+                       const ValueToValueMap &StridesMap) {
   const Type *Ty = Ptr->getType();
   assert(Ty->isPointerTy() && "Unexpected non-ptr");
 
@@ -535,7 +864,7 @@ static int isStridedPtr(ScalarEvolution *SE, Value *Ptr, const Loop *Lp,
   // to access the pointer value "0" which is undefined behavior in address
   // space 0, therefore we can also vectorize this case.
   bool IsInBoundsGEP = isInBoundsGep(Ptr);
-  bool IsNoWrapAddRec = AR->getNoWrapFlags(SCEV::NoWrapMask);
+  bool IsNoWrapAddRec = isNoWrapAddRec(Ptr, AR, SE, Lp);
   bool IsInAddressSpaceZero = PtrTy->getAddressSpace() == 0;
   if (!IsNoWrapAddRec && !IsInBoundsGEP && !IsInAddressSpaceZero) {
     DEBUG(dbgs() << "LAA: Bad stride - Pointer may wrap in the address space "
@@ -546,7 +875,7 @@ static int isStridedPtr(ScalarEvolution *SE, Value *Ptr, const Loop *Lp,
   // Check the step is constant.
   const SCEV *Step = AR->getStepRecurrence(*SE);
 
-  // Calculate the pointer stride and check if it is consecutive.
+  // Calculate the pointer stride and check if it is constant.
   const SCEVConstant *C = dyn_cast<SCEVConstant>(Step);
   if (!C) {
     DEBUG(dbgs() << "LAA: Bad stride - Not a constant strided " << *Ptr <<
@@ -667,6 +996,42 @@ bool MemoryDepChecker::couldPreventStoreLoadForward(unsigned Distance,
   return false;
 }
 
+/// \brief Check the dependence for two accesses with the same stride \p Stride.
+/// \p Distance is the positive distance and \p TypeByteSize is type size in
+/// bytes.
+///
+/// \returns true if they are independent.
+static bool areStridedAccessesIndependent(unsigned Distance, unsigned Stride,
+                                          unsigned TypeByteSize) {
+  assert(Stride > 1 && "The stride must be greater than 1");
+  assert(TypeByteSize > 0 && "The type size in byte must be non-zero");
+  assert(Distance > 0 && "The distance must be non-zero");
+
+  // Skip if the distance is not multiple of type byte size.
+  if (Distance % TypeByteSize)
+    return false;
+
+  unsigned ScaledDist = Distance / TypeByteSize;
+
+  // No dependence if the scaled distance is not multiple of the stride.
+  // E.g.
+  //      for (i = 0; i < 1024 ; i += 4)
+  //        A[i+2] = A[i] + 1;
+  //
+  // Two accesses in memory (scaled distance is 2, stride is 4):
+  //     | A[0] |      |      |      | A[4] |      |      |      |
+  //     |      |      | A[2] |      |      |      | A[6] |      |
+  //
+  // E.g.
+  //      for (i = 0; i < 1024 ; i += 3)
+  //        A[i+4] = A[i] + 1;
+  //
+  // Two accesses in memory (scaled distance is 4, stride is 3):
+  //     | A[0] |      |      | A[3] |      |      | A[6] |      |      |
+  //     |      |      |      |      | A[4] |      |      | A[7] |      |
+  return ScaledDist % Stride;
+}
+
 MemoryDepChecker::Dependence::DepType
 MemoryDepChecker::isDependent(const MemAccessInfo &A, unsigned AIdx,
                               const MemAccessInfo &B, unsigned BIdx,
@@ -715,11 +1080,11 @@ MemoryDepChecker::isDependent(const MemAccessInfo &A, unsigned AIdx,
   DEBUG(dbgs() << "LAA: Distance for " << *InstMap[AIdx] << " to "
         << *InstMap[BIdx] << ": " << *Dist << "\n");
 
-  // Need consecutive accesses. We don't want to vectorize
+  // Need accesses with constant stride. We don't want to vectorize
   // "A[B[i]] += ..." and similar code or pointer arithmetic that could wrap in
   // the address space.
   if (!StrideAPtr || !StrideBPtr || StrideAPtr != StrideBPtr){
-    DEBUG(dbgs() << "Non-consecutive pointer access\n");
+    DEBUG(dbgs() << "Pointer access with non-constant stride\n");
     return Dependence::Unknown;
   }
 
@@ -767,34 +1132,89 @@ MemoryDepChecker::isDependent(const MemAccessInfo &A, unsigned AIdx,
 
   unsigned Distance = (unsigned) Val.getZExtValue();
 
+  unsigned Stride = std::abs(StrideAPtr);
+  if (Stride > 1 &&
+      areStridedAccessesIndependent(Distance, Stride, TypeByteSize)) {
+    DEBUG(dbgs() << "LAA: Strided accesses are independent\n");
+    return Dependence::NoDep;
+  }
+
   // Bail out early if passed-in parameters make vectorization not feasible.
   unsigned ForcedFactor = (VectorizerParams::VectorizationFactor ?
                            VectorizerParams::VectorizationFactor : 1);
   unsigned ForcedUnroll = (VectorizerParams::VectorizationInterleave ?
                            VectorizerParams::VectorizationInterleave : 1);
+  // The minimum number of iterations for a vectorized/unrolled version.
+  unsigned MinNumIter = std::max(ForcedFactor * ForcedUnroll, 2U);
+
+  // It's not vectorizable if the distance is smaller than the minimum distance
+  // needed for a vectroized/unrolled version. Vectorizing one iteration in
+  // front needs TypeByteSize * Stride. Vectorizing the last iteration needs
+  // TypeByteSize (No need to plus the last gap distance).
+  //
+  // E.g. Assume one char is 1 byte in memory and one int is 4 bytes.
+  //      foo(int *A) {
+  //        int *B = (int *)((char *)A + 14);
+  //        for (i = 0 ; i < 1024 ; i += 2)
+  //          B[i] = A[i] + 1;
+  //      }
+  //
+  // Two accesses in memory (stride is 2):
+  //     | A[0] |      | A[2] |      | A[4] |      | A[6] |      |
+  //                              | B[0] |      | B[2] |      | B[4] |
+  //
+  // Distance needs for vectorizing iterations except the last iteration:
+  // 4 * 2 * (MinNumIter - 1). Distance needs for the last iteration: 4.
+  // So the minimum distance needed is: 4 * 2 * (MinNumIter - 1) + 4.
+  //
+  // If MinNumIter is 2, it is vectorizable as the minimum distance needed is
+  // 12, which is less than distance.
+  //
+  // If MinNumIter is 4 (Say if a user forces the vectorization factor to be 4),
+  // the minimum distance needed is 28, which is greater than distance. It is
+  // not safe to do vectorization.
+  unsigned MinDistanceNeeded =
+      TypeByteSize * Stride * (MinNumIter - 1) + TypeByteSize;
+  if (MinDistanceNeeded > Distance) {
+    DEBUG(dbgs() << "LAA: Failure because of positive distance " << Distance
+                 << '\n');
+    return Dependence::Backward;
+  }
 
-  // The distance must be bigger than the size needed for a vectorized version
-  // of the operation and the size of the vectorized operation must not be
-  // bigger than the currrent maximum size.
-  if (Distance < 2*TypeByteSize ||
-      2*TypeByteSize > MaxSafeDepDistBytes ||
-      Distance < TypeByteSize * ForcedUnroll * ForcedFactor) {
-    DEBUG(dbgs() << "LAA: Failure because of Positive distance "
-        << Val.getSExtValue() << '\n');
+  // Unsafe if the minimum distance needed is greater than max safe distance.
+  if (MinDistanceNeeded > MaxSafeDepDistBytes) {
+    DEBUG(dbgs() << "LAA: Failure because it needs at least "
+                 << MinDistanceNeeded << " size in bytes");
     return Dependence::Backward;
   }
 
   // Positive distance bigger than max vectorization factor.
-  MaxSafeDepDistBytes = Distance < MaxSafeDepDistBytes ?
-    Distance : MaxSafeDepDistBytes;
+  // FIXME: Should use max factor instead of max distance in bytes, which could
+  // not handle different types.
+  // E.g. Assume one char is 1 byte in memory and one int is 4 bytes.
+  //      void foo (int *A, char *B) {
+  //        for (unsigned i = 0; i < 1024; i++) {
+  //          A[i+2] = A[i] + 1;
+  //          B[i+2] = B[i] + 1;
+  //        }
+  //      }
+  //
+  // This case is currently unsafe according to the max safe distance. If we
+  // analyze the two accesses on array B, the max safe dependence distance
+  // is 2. Then we analyze the accesses on array A, the minimum distance needed
+  // is 8, which is less than 2 and forbidden vectorization, But actually
+  // both A and B could be vectorized by 2 iterations.
+  MaxSafeDepDistBytes =
+      Distance < MaxSafeDepDistBytes ? Distance : MaxSafeDepDistBytes;
 
   bool IsTrueDataDependence = (!AIsWrite && BIsWrite);
   if (IsTrueDataDependence &&
       couldPreventStoreLoadForward(Distance, TypeByteSize))
     return Dependence::BackwardVectorizableButPreventsForwarding;
 
-  DEBUG(dbgs() << "LAA: Positive distance " << Val.getSExtValue() <<
-        " with max VF = " << MaxSafeDepDistBytes / TypeByteSize << '\n');
+  DEBUG(dbgs() << "LAA: Positive distance " << Val.getSExtValue()
+               << " with max VF = "
+               << MaxSafeDepDistBytes / (TypeByteSize * Stride) << '\n');
 
   return Dependence::BackwardVectorizable;
 }
@@ -890,14 +1310,20 @@ void MemoryDepChecker::Dependence::print(
 }
 
 bool LoopAccessInfo::canAnalyzeLoop() {
+  // We need to have a loop header.
+  DEBUG(dbgs() << "LAA: Found a loop: " <<
+        TheLoop->getHeader()->getName() << '\n');
+
     // We can only analyze innermost loops.
   if (!TheLoop->empty()) {
+    DEBUG(dbgs() << "LAA: loop is not the innermost loop\n");
     emitAnalysis(LoopAccessReport() << "loop is not the innermost loop");
     return false;
   }
 
   // We must have a single backedge.
   if (TheLoop->getNumBackEdges() != 1) {
+    DEBUG(dbgs() << "LAA: loop control flow is not understood by analyzer\n");
     emitAnalysis(
         LoopAccessReport() <<
         "loop control flow is not understood by analyzer");
@@ -906,6 +1332,7 @@ bool LoopAccessInfo::canAnalyzeLoop() {
 
   // We must have a single exiting block.
   if (!TheLoop->getExitingBlock()) {
+    DEBUG(dbgs() << "LAA: loop control flow is not understood by analyzer\n");
     emitAnalysis(
         LoopAccessReport() <<
         "loop control flow is not understood by analyzer");
@@ -916,16 +1343,13 @@ bool LoopAccessInfo::canAnalyzeLoop() {
   // checked at the end of each iteration. With that we can assume that all
   // instructions in the loop are executed the same number of times.
   if (TheLoop->getExitingBlock() != TheLoop->getLoopLatch()) {
+    DEBUG(dbgs() << "LAA: loop control flow is not understood by analyzer\n");
     emitAnalysis(
         LoopAccessReport() <<
         "loop control flow is not understood by analyzer");
     return false;
   }
 
-  // We need to have a loop header.
-  DEBUG(dbgs() << "LAA: Found a loop: " <<
-        TheLoop->getHeader()->getName() << '\n');
-
   // ScalarEvolution needs to be able to find the exit count.
   const SCEV *ExitCount = SE->getBackedgeTakenCount(TheLoop);
   if (ExitCount == SE->getCouldNotCompute()) {
@@ -951,8 +1375,8 @@ void LoopAccessInfo::analyzeLoop(const ValueToValueMap &Strides) {
   unsigned NumReads = 0;
   unsigned NumReadWrites = 0;
 
-  PtrRtCheck.Pointers.clear();
-  PtrRtCheck.Need = false;
+  PtrRtChecking.Pointers.clear();
+  PtrRtChecking.Need = false;
 
   const bool IsAnnotatedParallel = TheLoop->isAnnotatedParallel();
 
@@ -1031,7 +1455,7 @@ void LoopAccessInfo::analyzeLoop(const ValueToValueMap &Strides) {
 
   MemoryDepChecker::DepCandidates DependentAccesses;
   AccessAnalysis Accesses(TheLoop->getHeader()->getModule()->getDataLayout(),
-                          AA, DependentAccesses);
+                          AA, LI, DependentAccesses);
 
   // Holds the analyzed pointers. We don't want to call GetUnderlyingObjects
   // multiple times on the same object. If the ptr is accessed twice, once
@@ -1051,7 +1475,7 @@ void LoopAccessInfo::analyzeLoop(const ValueToValueMap &Strides) {
     if (Seen.insert(Ptr).second) {
       ++NumReadWrites;
 
-      AliasAnalysis::Location Loc = AA->getLocation(ST);
+      MemoryLocation Loc = MemoryLocation::get(ST);
       // The TBAA metadata could have a control dependency on the predication
       // condition, so we cannot rely on it when determining whether or not we
       // need runtime pointer checks.
@@ -1087,7 +1511,7 @@ void LoopAccessInfo::analyzeLoop(const ValueToValueMap &Strides) {
       IsReadOnlyPtr = true;
     }
 
-    AliasAnalysis::Location Loc = AA->getLocation(LD);
+    MemoryLocation Loc = MemoryLocation::get(LD);
     // The TBAA metadata could have a control dependency on the predication
     // condition, so we cannot rely on it when determining whether or not we
     // need runtime pointer checks.
@@ -1108,36 +1532,20 @@ void LoopAccessInfo::analyzeLoop(const ValueToValueMap &Strides) {
   // Build dependence sets and check whether we need a runtime pointer bounds
   // check.
   Accesses.buildDependenceSets();
-  bool NeedRTCheck = Accesses.isRTCheckNeeded();
 
   // Find pointers with computable bounds. We are going to use this information
   // to place a runtime bound check.
-  bool CanDoRT = false;
-  if (NeedRTCheck)
-    CanDoRT = Accesses.canCheckPtrAtRT(PtrRtCheck, NumComparisons, SE, TheLoop,
-                                       Strides);
-
-  DEBUG(dbgs() << "LAA: We need to do " << NumComparisons <<
-        " pointer comparisons.\n");
-
-  // If we only have one set of dependences to check pointers among we don't
-  // need a runtime check.
-  if (NumComparisons == 0 && NeedRTCheck)
-    NeedRTCheck = false;
-
-  // Check that we found the bounds for the pointer.
-  if (CanDoRT)
-    DEBUG(dbgs() << "LAA: We can perform a memory runtime check if needed.\n");
-  else if (NeedRTCheck) {
+  bool CanDoRTIfNeeded =
+      Accesses.canCheckPtrAtRT(PtrRtChecking, SE, TheLoop, Strides);
+  if (!CanDoRTIfNeeded) {
     emitAnalysis(LoopAccessReport() << "cannot identify array bounds");
-    DEBUG(dbgs() << "LAA: We can't vectorize because we can't find " <<
-          "the array bounds.\n");
-    PtrRtCheck.reset();
+    DEBUG(dbgs() << "LAA: We can't vectorize because we can't find "
+                 << "the array bounds.\n");
     CanVecMem = false;
     return;
   }
 
-  PtrRtCheck.Need = NeedRTCheck;
+  DEBUG(dbgs() << "LAA: We can perform a memory runtime check if needed.\n");
 
   CanVecMem = true;
   if (Accesses.isDependencyCheckNeeded()) {
@@ -1148,22 +1556,21 @@ void LoopAccessInfo::analyzeLoop(const ValueToValueMap &Strides) {
 
     if (!CanVecMem && DepChecker.shouldRetryWithRuntimeCheck()) {
       DEBUG(dbgs() << "LAA: Retrying with memory checks\n");
-      NeedRTCheck = true;
 
       // Clear the dependency checks. We assume they are not needed.
-      Accesses.resetDepChecks();
+      Accesses.resetDepChecks(DepChecker);
+
+      PtrRtChecking.reset();
+      PtrRtChecking.Need = true;
 
-      PtrRtCheck.reset();
-      PtrRtCheck.Need = true;
+      CanDoRTIfNeeded =
+          Accesses.canCheckPtrAtRT(PtrRtChecking, SE, TheLoop, Strides, true);
 
-      CanDoRT = Accesses.canCheckPtrAtRT(PtrRtCheck, NumComparisons, SE,
-                                         TheLoop, Strides, true);
       // Check that we found the bounds for the pointer.
-      if (!CanDoRT && NumComparisons > 0) {
+      if (!CanDoRTIfNeeded) {
         emitAnalysis(LoopAccessReport()
                      << "cannot check memory dependencies at runtime");
         DEBUG(dbgs() << "LAA: Can't vectorize with memory checks\n");
-        PtrRtCheck.reset();
         CanVecMem = false;
         return;
       }
@@ -1174,8 +1581,8 @@ void LoopAccessInfo::analyzeLoop(const ValueToValueMap &Strides) {
 
   if (CanVecMem)
     DEBUG(dbgs() << "LAA: No unsafe dependent memory operations in loop.  We"
-                 << (NeedRTCheck ? "" : " don't")
-                 << " need a runtime memory check.\n");
+                 << (PtrRtChecking.Need ? "" : " don't")
+                 << " need runtime memory checks.\n");
   else {
     emitAnalysis(LoopAccessReport() <<
                  "unsafe dependent memory operations in loop");
@@ -1212,78 +1619,108 @@ static Instruction *getFirstInst(Instruction *FirstInst, Value *V,
   return nullptr;
 }
 
+/// \brief IR Values for the lower and upper bounds of a pointer evolution.
+struct PointerBounds {
+  Value *Start;
+  Value *End;
+};
+
+/// \brief Expand code for the lower and upper bound of the pointer group \p CG
+/// in \p TheLoop.  \return the values for the bounds.
+static PointerBounds
+expandBounds(const RuntimePointerChecking::CheckingPtrGroup *CG, Loop *TheLoop,
+             Instruction *Loc, SCEVExpander &Exp, ScalarEvolution *SE,
+             const RuntimePointerChecking &PtrRtChecking) {
+  Value *Ptr = PtrRtChecking.Pointers[CG->Members[0]].PointerValue;
+  const SCEV *Sc = SE->getSCEV(Ptr);
+
+  if (SE->isLoopInvariant(Sc, TheLoop)) {
+    DEBUG(dbgs() << "LAA: Adding RT check for a loop invariant ptr:" << *Ptr
+                 << "\n");
+    return {Ptr, Ptr};
+  } else {
+    unsigned AS = Ptr->getType()->getPointerAddressSpace();
+    LLVMContext &Ctx = Loc->getContext();
+
+    // Use this type for pointer arithmetic.
+    Type *PtrArithTy = Type::getInt8PtrTy(Ctx, AS);
+    Value *Start = nullptr, *End = nullptr;
+
+    DEBUG(dbgs() << "LAA: Adding RT check for range:\n");
+    Start = Exp.expandCodeFor(CG->Low, PtrArithTy, Loc);
+    End = Exp.expandCodeFor(CG->High, PtrArithTy, Loc);
+    DEBUG(dbgs() << "Start: " << *CG->Low << " End: " << *CG->High << "\n");
+    return {Start, End};
+  }
+}
+
+/// \brief Turns a collection of checks into a collection of expanded upper and
+/// lower bounds for both pointers in the check.
+static SmallVector<std::pair<PointerBounds, PointerBounds>, 4> expandBounds(
+    const SmallVectorImpl<RuntimePointerChecking::PointerCheck> &PointerChecks,
+    Loop *L, Instruction *Loc, ScalarEvolution *SE, SCEVExpander &Exp,
+    const RuntimePointerChecking &PtrRtChecking) {
+  SmallVector<std::pair<PointerBounds, PointerBounds>, 4> ChecksWithBounds;
+
+  // Here we're relying on the SCEV Expander's cache to only emit code for the
+  // same bounds once.
+  std::transform(
+      PointerChecks.begin(), PointerChecks.end(),
+      std::back_inserter(ChecksWithBounds),
+      [&](const RuntimePointerChecking::PointerCheck &Check) {
+        PointerBounds
+          First = expandBounds(Check.first, L, Loc, Exp, SE, PtrRtChecking),
+          Second = expandBounds(Check.second, L, Loc, Exp, SE, PtrRtChecking);
+        return std::make_pair(First, Second);
+      });
+
+  return ChecksWithBounds;
+}
+
 std::pair<Instruction *, Instruction *> LoopAccessInfo::addRuntimeCheck(
-    Instruction *Loc, const SmallVectorImpl<int> *PtrPartition) const {
-  if (!PtrRtCheck.Need)
-    return std::make_pair(nullptr, nullptr);
+    Instruction *Loc,
+    const SmallVectorImpl<RuntimePointerChecking::PointerCheck> &PointerChecks)
+    const {
 
-  unsigned NumPointers = PtrRtCheck.Pointers.size();
-  SmallVector<TrackingVH<Value> , 2> Starts;
-  SmallVector<TrackingVH<Value> , 2> Ends;
+  SCEVExpander Exp(*SE, DL, "induction");
+  auto ExpandedChecks =
+      expandBounds(PointerChecks, TheLoop, Loc, SE, Exp, PtrRtChecking);
 
   LLVMContext &Ctx = Loc->getContext();
-  SCEVExpander Exp(*SE, DL, "induction");
   Instruction *FirstInst = nullptr;
-
-  for (unsigned i = 0; i < NumPointers; ++i) {
-    Value *Ptr = PtrRtCheck.Pointers[i];
-    const SCEV *Sc = SE->getSCEV(Ptr);
-
-    if (SE->isLoopInvariant(Sc, TheLoop)) {
-      DEBUG(dbgs() << "LAA: Adding RT check for a loop invariant ptr:" <<
-            *Ptr <<"\n");
-      Starts.push_back(Ptr);
-      Ends.push_back(Ptr);
-    } else {
-      DEBUG(dbgs() << "LAA: Adding RT check for range:" << *Ptr << '\n');
-      unsigned AS = Ptr->getType()->getPointerAddressSpace();
-
-      // Use this type for pointer arithmetic.
-      Type *PtrArithTy = Type::getInt8PtrTy(Ctx, AS);
-
-      Value *Start = Exp.expandCodeFor(PtrRtCheck.Starts[i], PtrArithTy, Loc);
-      Value *End = Exp.expandCodeFor(PtrRtCheck.Ends[i], PtrArithTy, Loc);
-      Starts.push_back(Start);
-      Ends.push_back(End);
-    }
-  }
-
   IRBuilder<> ChkBuilder(Loc);
   // Our instructions might fold to a constant.
   Value *MemoryRuntimeCheck = nullptr;
-  for (unsigned i = 0; i < NumPointers; ++i) {
-    for (unsigned j = i+1; j < NumPointers; ++j) {
-      if (!PtrRtCheck.needsChecking(i, j, PtrPartition))
-        continue;
-
-      unsigned AS0 = Starts[i]->getType()->getPointerAddressSpace();
-      unsigned AS1 = Starts[j]->getType()->getPointerAddressSpace();
 
-      assert((AS0 == Ends[j]->getType()->getPointerAddressSpace()) &&
-             (AS1 == Ends[i]->getType()->getPointerAddressSpace()) &&
-             "Trying to bounds check pointers with different address spaces");
-
-      Type *PtrArithTy0 = Type::getInt8PtrTy(Ctx, AS0);
-      Type *PtrArithTy1 = Type::getInt8PtrTy(Ctx, AS1);
-
-      Value *Start0 = ChkBuilder.CreateBitCast(Starts[i], PtrArithTy0, "bc");
-      Value *Start1 = ChkBuilder.CreateBitCast(Starts[j], PtrArithTy1, "bc");
-      Value *End0 =   ChkBuilder.CreateBitCast(Ends[i],   PtrArithTy1, "bc");
-      Value *End1 =   ChkBuilder.CreateBitCast(Ends[j],   PtrArithTy0, "bc");
-
-      Value *Cmp0 = ChkBuilder.CreateICmpULE(Start0, End1, "bound0");
-      FirstInst = getFirstInst(FirstInst, Cmp0, Loc);
-      Value *Cmp1 = ChkBuilder.CreateICmpULE(Start1, End0, "bound1");
-      FirstInst = getFirstInst(FirstInst, Cmp1, Loc);
-      Value *IsConflict = ChkBuilder.CreateAnd(Cmp0, Cmp1, "found.conflict");
+  for (const auto &Check : ExpandedChecks) {
+    const PointerBounds &A = Check.first, &B = Check.second;
+    unsigned AS0 = A.Start->getType()->getPointerAddressSpace();
+    unsigned AS1 = B.Start->getType()->getPointerAddressSpace();
+
+    assert((AS0 == B.End->getType()->getPointerAddressSpace()) &&
+           (AS1 == A.End->getType()->getPointerAddressSpace()) &&
+           "Trying to bounds check pointers with different address spaces");
+
+    Type *PtrArithTy0 = Type::getInt8PtrTy(Ctx, AS0);
+    Type *PtrArithTy1 = Type::getInt8PtrTy(Ctx, AS1);
+
+    Value *Start0 = ChkBuilder.CreateBitCast(A.Start, PtrArithTy0, "bc");
+    Value *Start1 = ChkBuilder.CreateBitCast(B.Start, PtrArithTy1, "bc");
+    Value *End0 =   ChkBuilder.CreateBitCast(A.End,   PtrArithTy1, "bc");
+    Value *End1 =   ChkBuilder.CreateBitCast(B.End,   PtrArithTy0, "bc");
+
+    Value *Cmp0 = ChkBuilder.CreateICmpULE(Start0, End1, "bound0");
+    FirstInst = getFirstInst(FirstInst, Cmp0, Loc);
+    Value *Cmp1 = ChkBuilder.CreateICmpULE(Start1, End0, "bound1");
+    FirstInst = getFirstInst(FirstInst, Cmp1, Loc);
+    Value *IsConflict = ChkBuilder.CreateAnd(Cmp0, Cmp1, "found.conflict");
+    FirstInst = getFirstInst(FirstInst, IsConflict, Loc);
+    if (MemoryRuntimeCheck) {
+      IsConflict =
+          ChkBuilder.CreateOr(MemoryRuntimeCheck, IsConflict, "conflict.rdx");
       FirstInst = getFirstInst(FirstInst, IsConflict, Loc);
-      if (MemoryRuntimeCheck) {
-        IsConflict = ChkBuilder.CreateOr(MemoryRuntimeCheck, IsConflict,
-                                         "conflict.rdx");
-        FirstInst = getFirstInst(FirstInst, IsConflict, Loc);
-      }
-      MemoryRuntimeCheck = IsConflict;
     }
+    MemoryRuntimeCheck = IsConflict;
   }
 
   if (!MemoryRuntimeCheck)
@@ -1299,13 +1736,21 @@ std::pair<Instruction *, Instruction *> LoopAccessInfo::addRuntimeCheck(
   return std::make_pair(FirstInst, Check);
 }
 
+std::pair<Instruction *, Instruction *> LoopAccessInfo::addRuntimeCheck(
+    Instruction *Loc, const SmallVectorImpl<int> *PtrPartition) const {
+  if (!PtrRtChecking.Need)
+    return std::make_pair(nullptr, nullptr);
+
+  return addRuntimeCheck(Loc, PtrRtChecking.generateChecks(PtrPartition));
+}
+
 LoopAccessInfo::LoopAccessInfo(Loop *L, ScalarEvolution *SE,
                                const DataLayout &DL,
                                const TargetLibraryInfo *TLI, AliasAnalysis *AA,
-                               DominatorTree *DT,
+                               DominatorTree *DT, LoopInfo *LI,
                                const ValueToValueMap &Strides)
-    : DepChecker(SE, L), NumComparisons(0), TheLoop(L), SE(SE), DL(DL),
-      TLI(TLI), AA(AA), DT(DT), NumLoads(0), NumStores(0),
+    : PtrRtChecking(SE), DepChecker(SE, L), TheLoop(L), SE(SE), DL(DL),
+      TLI(TLI), AA(AA), DT(DT), LI(LI), NumLoads(0), NumStores(0),
       MaxSafeDepDistBytes(-1U), CanVecMem(false),
       StoreToLoopInvariantAddress(false) {
   if (canAnalyzeLoop())
@@ -1314,16 +1759,12 @@ LoopAccessInfo::LoopAccessInfo(Loop *L, ScalarEvolution *SE,
 
 void LoopAccessInfo::print(raw_ostream &OS, unsigned Depth) const {
   if (CanVecMem) {
-    if (PtrRtCheck.Need)
+    if (PtrRtChecking.Need)
       OS.indent(Depth) << "Memory dependences are safe with run-time checks\n";
     else
       OS.indent(Depth) << "Memory dependences are safe\n";
   }
 
-  OS.indent(Depth) << "Store to invariant address was "
-                   << (StoreToLoopInvariantAddress ? "" : "not ")
-                   << "found in loop.\n";
-
   if (Report)
     OS.indent(Depth) << "Report: " << Report->str() << "\n";
 
@@ -1337,8 +1778,12 @@ void LoopAccessInfo::print(raw_ostream &OS, unsigned Depth) const {
     OS.indent(Depth) << "Too many interesting dependences, not recorded\n";
 
   // List the pair of accesses need run-time checks to prove independence.
-  PtrRtCheck.print(OS, Depth);
+  PtrRtChecking.print(OS, Depth);
   OS << "\n";
+
+  OS.indent(Depth) << "Store to invariant address was "
+                   << (StoreToLoopInvariantAddress ? "" : "not ")
+                   << "found in loop.\n";
 }
 
 const LoopAccessInfo &
@@ -1352,7 +1797,8 @@ LoopAccessAnalysis::getInfo(Loop *L, const ValueToValueMap &Strides) {
 
   if (!LAI) {
     const DataLayout &DL = L->getHeader()->getModule()->getDataLayout();
-    LAI = llvm::make_unique<LoopAccessInfo>(L, SE, DL, TLI, AA, DT, Strides);
+    LAI = llvm::make_unique<LoopAccessInfo>(L, SE, DL, TLI, AA, DT, LI,
+                                            Strides);
 #ifndef NDEBUG
     LAI->NumSymbolicStrides = Strides.size();
 #endif
@@ -1363,7 +1809,6 @@ LoopAccessAnalysis::getInfo(Loop *L, const ValueToValueMap &Strides) {
 void LoopAccessAnalysis::print(raw_ostream &OS, const Module *M) const {
   LoopAccessAnalysis &LAA = *const_cast<LoopAccessAnalysis *>(this);
 
-  LoopInfo *LI = &getAnalysis<LoopInfoWrapperPass>().getLoopInfo();
   ValueToValueMap NoSymbolicStrides;
 
   for (Loop *TopLevelLoop : *LI)
@@ -1380,6 +1825,7 @@ bool LoopAccessAnalysis::runOnFunction(Function &F) {
   TLI = TLIP ? &TLIP->getTLI() : nullptr;
   AA = &getAnalysis<AliasAnalysis>();
   DT = &getAnalysis<DominatorTreeWrapperPass>().getDomTree();
+  LI = &getAnalysis<LoopInfoWrapperPass>().getLoopInfo();
 
   return false;
 }
diff --git a/lib/Analysis/LoopInfo.cpp b/lib/Analysis/LoopInfo.cpp
index 6462b0670ff..1ad4fd893bc 100644
--- a/lib/Analysis/LoopInfo.cpp
+++ b/lib/Analysis/LoopInfo.cpp
@@ -56,20 +56,16 @@ static const char *const LoopMDName = "llvm.loop";
 
 /// isLoopInvariant - Return true if the specified value is loop invariant
 ///
-bool Loop::isLoopInvariant(Value *V) const {
-  if (Instruction *I = dyn_cast<Instruction>(V))
+bool Loop::isLoopInvariant(const Value *V) const {
+  if (const Instruction *I = dyn_cast<Instruction>(V))
     return !contains(I);
   return true;  // All non-instructions are loop invariant
 }
 
 /// hasLoopInvariantOperands - Return true if all the operands of the
 /// specified instruction are loop invariant.
-bool Loop::hasLoopInvariantOperands(Instruction *I) const {
-  for (unsigned i = 0, e = I->getNumOperands(); i != e; ++i)
-    if (!isLoopInvariant(I->getOperand(i)))
-      return false;
-
-  return true;
+bool Loop::hasLoopInvariantOperands(const Instruction *I) const {
+  return all_of(I->operands(), [this](Value *V) { return isLoopInvariant(V); });
 }
 
 /// makeLoopInvariant - If the given value is an instruciton inside of the
@@ -106,8 +102,8 @@ bool Loop::makeLoopInvariant(Instruction *I, bool &Changed,
     return false;
   if (I->mayReadFromMemory())
     return false;
-  // The landingpad instruction is immobile.
-  if (isa<LandingPadInst>(I))
+  // EH block instructions are immobile.
+  if (I->isEHPad())
     return false;
   // Determine the insertion point, unless one was given.
   if (!InsertPt) {
@@ -606,6 +602,10 @@ Loop *UnloopUpdater::getNearestLoop(BasicBlock *BB, Loop *BBLoop) {
   return NearLoop;
 }
 
+LoopInfo::LoopInfo(const DominatorTreeBase<BasicBlock> &DomTree) {
+  analyze(DomTree);
+}
+
 /// updateUnloop - The last backedge has been removed from a loop--now the
 /// "unloop". Find a new parent for the blocks contained within unloop and
 /// update the loop tree. We don't necessarily have valid dominators at this
@@ -679,8 +679,8 @@ LoopInfo LoopAnalysis::run(Function &F, AnalysisManager<Function> *AM) {
   // objects. I don't want to add that kind of complexity until the scope of
   // the problem is better understood.
   LoopInfo LI;
-  LI.Analyze(AM->getResult<DominatorTreeAnalysis>(F));
-  return std::move(LI);
+  LI.analyze(AM->getResult<DominatorTreeAnalysis>(F));
+  return LI;
 }
 
 PreservedAnalyses LoopPrinterPass::run(Function &F,
@@ -702,7 +702,7 @@ INITIALIZE_PASS_END(LoopInfoWrapperPass, "loops", "Natural Loop Information",
 
 bool LoopInfoWrapperPass::runOnFunction(Function &) {
   releaseMemory();
-  LI.Analyze(getAnalysis<DominatorTreeWrapperPass>().getDomTree());
+  LI.analyze(getAnalysis<DominatorTreeWrapperPass>().getDomTree());
   return false;
 }
 
diff --git a/lib/Analysis/Makefile b/lib/Analysis/Makefile
index 426ed1699dd..4af6d350a64 100644
--- a/lib/Analysis/Makefile
+++ b/lib/Analysis/Makefile
@@ -9,7 +9,7 @@
 
 LEVEL = ../..
 LIBRARYNAME = LLVMAnalysis
-DIRS = IPA NaCl
+DIRS = IPA
 BUILD_ARCHIVE = 1
 
 include $(LEVEL)/Makefile.common
diff --git a/lib/Analysis/MemDerefPrinter.cpp b/lib/Analysis/MemDerefPrinter.cpp
index 6119a3da617..fa292a28ec8 100644
--- a/lib/Analysis/MemDerefPrinter.cpp
+++ b/lib/Analysis/MemDerefPrinter.cpp
@@ -10,6 +10,7 @@
 #include "llvm/Analysis/Passes.h"
 #include "llvm/ADT/SetVector.h"
 #include "llvm/Analysis/MemoryDependenceAnalysis.h"
+#include "llvm/Analysis/ValueTracking.h"
 #include "llvm/IR/CallSite.h"
 #include "llvm/IR/DataLayout.h"
 #include "llvm/IR/InstIterator.h"
@@ -23,7 +24,7 @@ namespace {
   struct MemDerefPrinter : public FunctionPass {
     SmallVector<Value *, 4> Vec;
 
-    static char ID; // Pass identifcation, replacement for typeid
+    static char ID; // Pass identification, replacement for typeid
     MemDerefPrinter() : FunctionPass(ID) {
       initializeMemDerefPrinterPass(*PassRegistry::getPassRegistry());
     }
@@ -53,7 +54,7 @@ bool MemDerefPrinter::runOnFunction(Function &F) {
   for (auto &I: inst_range(F)) {
     if (LoadInst *LI = dyn_cast<LoadInst>(&I)) {
       Value *PO = LI->getPointerOperand();
-      if (PO->isDereferenceablePointer(DL))
+      if (isDereferenceablePointer(PO, DL))
         Vec.push_back(PO);
     }
   }
diff --git a/lib/Analysis/MemoryDependenceAnalysis.cpp b/lib/Analysis/MemoryDependenceAnalysis.cpp
index 84769cb07d7..54ac1b61054 100644
--- a/lib/Analysis/MemoryDependenceAnalysis.cpp
+++ b/lib/Analysis/MemoryDependenceAnalysis.cpp
@@ -22,6 +22,7 @@
 #include "llvm/Analysis/InstructionSimplify.h"
 #include "llvm/Analysis/MemoryBuiltins.h"
 #include "llvm/Analysis/PHITransAddr.h"
+#include "llvm/Analysis/OrderedBasicBlock.h"
 #include "llvm/Analysis/ValueTracking.h"
 #include "llvm/IR/DataLayout.h"
 #include "llvm/IR/Dominators.h"
@@ -49,7 +50,11 @@ STATISTIC(NumCacheCompleteNonLocalPtr,
           "Number of block queries that were completely cached");
 
 // Limit for the number of instructions to scan in a block.
-static const unsigned int BlockScanLimit = 100;
+
+static cl::opt<unsigned> BlockScanLimit(
+    "memdep-block-scan-limit", cl::Hidden, cl::init(100),
+    cl::desc("The number of instructions to scan in a block in memory "
+             "dependency analysis (default = 100)"));
 
 // Limit on the number of memdep results to process.
 static const unsigned int NumResultsLimit = 100;
@@ -65,7 +70,7 @@ INITIALIZE_PASS_END(MemoryDependenceAnalysis, "memdep",
                       "Memory Dependence Analysis", false, true)
 
 MemoryDependenceAnalysis::MemoryDependenceAnalysis()
-    : FunctionPass(ID), PredCache() {
+    : FunctionPass(ID) {
   initializeMemoryDependenceAnalysisPass(*PassRegistry::getPassRegistry());
 }
 MemoryDependenceAnalysis::~MemoryDependenceAnalysis() {
@@ -79,7 +84,7 @@ void MemoryDependenceAnalysis::releaseMemory() {
   ReverseLocalDeps.clear();
   ReverseNonLocalDeps.clear();
   ReverseNonLocalPtrDeps.clear();
-  PredCache->clear();
+  PredCache.clear();
 }
 
 /// getAnalysisUsage - Does not modify anything.  It uses Alias Analysis.
@@ -96,8 +101,6 @@ bool MemoryDependenceAnalysis::runOnFunction(Function &F) {
   DominatorTreeWrapperPass *DTWP =
       getAnalysisIfAvailable<DominatorTreeWrapperPass>();
   DT = DTWP ? &DTWP->getDomTree() : nullptr;
-  if (!PredCache)
-    PredCache.reset(new PredIteratorCache());
   return false;
 }
 
@@ -120,45 +123,43 @@ static void RemoveFromReverseMap(DenseMap<Instruction*,
 /// location, fill in Loc with the details, otherwise set Loc.Ptr to null.
 /// Return a ModRefInfo value describing the general behavior of the
 /// instruction.
-static
-AliasAnalysis::ModRefResult GetLocation(const Instruction *Inst,
-                                        AliasAnalysis::Location &Loc,
-                                        AliasAnalysis *AA) {
+static ModRefInfo GetLocation(const Instruction *Inst, MemoryLocation &Loc,
+                              AliasAnalysis *AA) {
   if (const LoadInst *LI = dyn_cast<LoadInst>(Inst)) {
     if (LI->isUnordered()) {
-      Loc = AA->getLocation(LI);
-      return AliasAnalysis::Ref;
+      Loc = MemoryLocation::get(LI);
+      return MRI_Ref;
     }
     if (LI->getOrdering() == Monotonic) {
-      Loc = AA->getLocation(LI);
-      return AliasAnalysis::ModRef;
+      Loc = MemoryLocation::get(LI);
+      return MRI_ModRef;
     }
-    Loc = AliasAnalysis::Location();
-    return AliasAnalysis::ModRef;
+    Loc = MemoryLocation();
+    return MRI_ModRef;
   }
 
   if (const StoreInst *SI = dyn_cast<StoreInst>(Inst)) {
     if (SI->isUnordered()) {
-      Loc = AA->getLocation(SI);
-      return AliasAnalysis::Mod;
+      Loc = MemoryLocation::get(SI);
+      return MRI_Mod;
     }
     if (SI->getOrdering() == Monotonic) {
-      Loc = AA->getLocation(SI);
-      return AliasAnalysis::ModRef;
+      Loc = MemoryLocation::get(SI);
+      return MRI_ModRef;
     }
-    Loc = AliasAnalysis::Location();
-    return AliasAnalysis::ModRef;
+    Loc = MemoryLocation();
+    return MRI_ModRef;
   }
 
   if (const VAArgInst *V = dyn_cast<VAArgInst>(Inst)) {
-    Loc = AA->getLocation(V);
-    return AliasAnalysis::ModRef;
+    Loc = MemoryLocation::get(V);
+    return MRI_ModRef;
   }
 
   if (const CallInst *CI = isFreeCall(Inst, AA->getTargetLibraryInfo())) {
     // calls to free() deallocate the entire structure
-    Loc = AliasAnalysis::Location(CI->getArgOperand(0));
-    return AliasAnalysis::Mod;
+    Loc = MemoryLocation(CI->getArgOperand(0));
+    return MRI_Mod;
   }
 
   if (const IntrinsicInst *II = dyn_cast<IntrinsicInst>(Inst)) {
@@ -169,20 +170,20 @@ AliasAnalysis::ModRefResult GetLocation(const Instruction *Inst,
     case Intrinsic::lifetime_end:
     case Intrinsic::invariant_start:
       II->getAAMetadata(AAInfo);
-      Loc = AliasAnalysis::Location(II->getArgOperand(1),
-                                    cast<ConstantInt>(II->getArgOperand(0))
-                                      ->getZExtValue(), AAInfo);
+      Loc = MemoryLocation(
+          II->getArgOperand(1),
+          cast<ConstantInt>(II->getArgOperand(0))->getZExtValue(), AAInfo);
       // These intrinsics don't really modify the memory, but returning Mod
       // will allow them to be handled conservatively.
-      return AliasAnalysis::Mod;
+      return MRI_Mod;
     case Intrinsic::invariant_end:
       II->getAAMetadata(AAInfo);
-      Loc = AliasAnalysis::Location(II->getArgOperand(2),
-                                    cast<ConstantInt>(II->getArgOperand(1))
-                                      ->getZExtValue(), AAInfo);
+      Loc = MemoryLocation(
+          II->getArgOperand(2),
+          cast<ConstantInt>(II->getArgOperand(1))->getZExtValue(), AAInfo);
       // These intrinsics don't really modify the memory, but returning Mod
       // will allow them to be handled conservatively.
-      return AliasAnalysis::Mod;
+      return MRI_Mod;
     default:
       break;
     }
@@ -190,10 +191,10 @@ AliasAnalysis::ModRefResult GetLocation(const Instruction *Inst,
 
   // Otherwise, just do the coarse-grained thing that always works.
   if (Inst->mayWriteToMemory())
-    return AliasAnalysis::ModRef;
+    return MRI_ModRef;
   if (Inst->mayReadFromMemory())
-    return AliasAnalysis::Ref;
-  return AliasAnalysis::NoModRef;
+    return MRI_Ref;
+  return MRI_NoModRef;
 }
 
 /// getCallSiteDependencyFrom - Private helper for finding the local
@@ -214,11 +215,11 @@ getCallSiteDependencyFrom(CallSite CS, bool isReadOnlyCall,
     Instruction *Inst = --ScanIt;
 
     // If this inst is a memory op, get the pointer it accessed
-    AliasAnalysis::Location Loc;
-    AliasAnalysis::ModRefResult MR = GetLocation(Inst, Loc, AA);
+    MemoryLocation Loc;
+    ModRefInfo MR = GetLocation(Inst, Loc, AA);
     if (Loc.Ptr) {
       // A simple instruction.
-      if (AA->getModRefInfo(CS, Loc) != AliasAnalysis::NoModRef)
+      if (AA->getModRefInfo(CS, Loc) != MRI_NoModRef)
         return MemDepResult::getClobber(Inst);
       continue;
     }
@@ -228,10 +229,10 @@ getCallSiteDependencyFrom(CallSite CS, bool isReadOnlyCall,
       if (isa<DbgInfoIntrinsic>(Inst)) continue;
       // If these two calls do not interfere, look past it.
       switch (AA->getModRefInfo(CS, InstCS)) {
-      case AliasAnalysis::NoModRef:
+      case MRI_NoModRef:
         // If the two calls are the same, return InstCS as a Def, so that
         // CS can be found redundant and eliminated.
-        if (isReadOnlyCall && !(MR & AliasAnalysis::Mod) &&
+        if (isReadOnlyCall && !(MR & MRI_Mod) &&
             CS.getInstruction()->isIdenticalToWhenDefined(Inst))
           return MemDepResult::getDef(Inst);
 
@@ -245,7 +246,7 @@ getCallSiteDependencyFrom(CallSite CS, bool isReadOnlyCall,
 
     // If we could not obtain a pointer for the instruction and the instruction
     // touches memory then assume that this is a dependency.
-    if (MR != AliasAnalysis::NoModRef)
+    if (MR != MRI_NoModRef)
       return MemDepResult::getClobber(Inst);
   }
 
@@ -261,9 +262,10 @@ getCallSiteDependencyFrom(CallSite CS, bool isReadOnlyCall,
 ///
 /// MemLocBase, MemLocOffset are lazily computed here the first time the
 /// base/offs of memloc is needed.
-static bool isLoadLoadClobberIfExtendedToFullWidth(
-    const AliasAnalysis::Location &MemLoc, const Value *&MemLocBase,
-    int64_t &MemLocOffs, const LoadInst *LI) {
+static bool isLoadLoadClobberIfExtendedToFullWidth(const MemoryLocation &MemLoc,
+                                                   const Value *&MemLocBase,
+                                                   int64_t &MemLocOffs,
+                                                   const LoadInst *LI) {
   const DataLayout &DL = LI->getModule()->getDataLayout();
 
   // If we haven't already computed the base/offset of MemLoc, do so now.
@@ -370,10 +372,9 @@ static bool isVolatile(Instruction *Inst) {
 /// with reads from read-only locations.  If possible, pass the query
 /// instruction as well; this function may take advantage of the metadata
 /// annotated to the query instruction to refine the result.
-MemDepResult MemoryDependenceAnalysis::
-getPointerDependencyFrom(const AliasAnalysis::Location &MemLoc, bool isLoad,
-                         BasicBlock::iterator ScanIt, BasicBlock *BB,
-                         Instruction *QueryInst) {
+MemDepResult MemoryDependenceAnalysis::getPointerDependencyFrom(
+    const MemoryLocation &MemLoc, bool isLoad, BasicBlock::iterator ScanIt,
+    BasicBlock *BB, Instruction *QueryInst) {
 
   const Value *MemLocBase = nullptr;
   int64_t MemLocOffset = 0;
@@ -420,6 +421,12 @@ getPointerDependencyFrom(const AliasAnalysis::Location &MemLoc, bool isLoad,
 
   const DataLayout &DL = BB->getModule()->getDataLayout();
 
+  // Create a numbered basic block to lazily compute and cache instruction
+  // positions inside a BB. This is used to provide fast queries for relative
+  // position between two instructions in a BB and can be used by
+  // AliasAnalysis::callCapturesBefore.
+  OrderedBasicBlock OBB(BB);
+
   // Walk backwards through the basic block, looking for dependencies.
   while (ScanIt != BB->begin()) {
     Instruction *Inst = --ScanIt;
@@ -442,8 +449,7 @@ getPointerDependencyFrom(const AliasAnalysis::Location &MemLoc, bool isLoad,
         // pointer, not on query pointers that are indexed off of them.  It'd
         // be nice to handle that at some point (the right approach is to use
         // GetPointerBaseWithConstantOffset).
-        if (AA->isMustAlias(AliasAnalysis::Location(II->getArgOperand(1)),
-                            MemLoc))
+        if (AA->isMustAlias(MemoryLocation(II->getArgOperand(1)), MemLoc))
           return MemDepResult::getDef(II);
         continue;
       }
@@ -488,13 +494,13 @@ getPointerDependencyFrom(const AliasAnalysis::Location &MemLoc, bool isLoad,
         }
       }
 
-      AliasAnalysis::Location LoadLoc = AA->getLocation(LI);
+      MemoryLocation LoadLoc = MemoryLocation::get(LI);
 
       // If we found a pointer, check if it could be the same as our pointer.
-      AliasAnalysis::AliasResult R = AA->alias(LoadLoc, MemLoc);
+      AliasResult R = AA->alias(LoadLoc, MemLoc);
 
       if (isLoad) {
-        if (R == AliasAnalysis::NoAlias) {
+        if (R == NoAlias) {
           // If this is an over-aligned integer load (for example,
           // "load i8* %P, align 4") see if it would obviously overlap with the
           // queried location if widened to a larger load (e.g. if the queried
@@ -511,7 +517,7 @@ getPointerDependencyFrom(const AliasAnalysis::Location &MemLoc, bool isLoad,
         }
 
         // Must aliased loads are defs of each other.
-        if (R == AliasAnalysis::MustAlias)
+        if (R == MustAlias)
           return MemDepResult::getDef(Inst);
 
 #if 0 // FIXME: Temporarily disabled. GVN is cleverly rewriting loads
@@ -521,7 +527,7 @@ getPointerDependencyFrom(const AliasAnalysis::Location &MemLoc, bool isLoad,
 
         // If we have a partial alias, then return this as a clobber for the
         // client to handle.
-        if (R == AliasAnalysis::PartialAlias)
+        if (R == PartialAlias)
           return MemDepResult::getClobber(Inst);
 #endif
 
@@ -531,7 +537,7 @@ getPointerDependencyFrom(const AliasAnalysis::Location &MemLoc, bool isLoad,
       }
 
       // Stores don't depend on other no-aliased accesses.
-      if (R == AliasAnalysis::NoAlias)
+      if (R == NoAlias)
         continue;
 
       // Stores don't alias loads from read-only memory.
@@ -572,19 +578,19 @@ getPointerDependencyFrom(const AliasAnalysis::Location &MemLoc, bool isLoad,
       // If alias analysis can tell that this store is guaranteed to not modify
       // the query pointer, ignore it.  Use getModRefInfo to handle cases where
       // the query pointer points to constant memory etc.
-      if (AA->getModRefInfo(SI, MemLoc) == AliasAnalysis::NoModRef)
+      if (AA->getModRefInfo(SI, MemLoc) == MRI_NoModRef)
         continue;
 
       // Ok, this store might clobber the query pointer.  Check to see if it is
       // a must alias: in this case, we want to return this as a def.
-      AliasAnalysis::Location StoreLoc = AA->getLocation(SI);
+      MemoryLocation StoreLoc = MemoryLocation::get(SI);
 
       // If we found a pointer, check if it could be the same as our pointer.
-      AliasAnalysis::AliasResult R = AA->alias(StoreLoc, MemLoc);
+      AliasResult R = AA->alias(StoreLoc, MemLoc);
 
-      if (R == AliasAnalysis::NoAlias)
+      if (R == NoAlias)
         continue;
-      if (R == AliasAnalysis::MustAlias)
+      if (R == MustAlias)
         return MemDepResult::getDef(Inst);
       if (isInvariantLoad)
        continue;
@@ -608,7 +614,7 @@ getPointerDependencyFrom(const AliasAnalysis::Location &MemLoc, bool isLoad,
       if (isInvariantLoad)
         continue;
       // Be conservative if the accessed pointer may alias the allocation.
-      if (AA->alias(Inst, AccessPtr) != AliasAnalysis::NoAlias)
+      if (AA->alias(Inst, AccessPtr) != NoAlias)
         return MemDepResult::getClobber(Inst);
       // If the allocation is not aliased and does not read memory (like
       // strdup), it is safe to ignore.
@@ -621,17 +627,17 @@ getPointerDependencyFrom(const AliasAnalysis::Location &MemLoc, bool isLoad,
        continue;
 
     // See if this instruction (e.g. a call or vaarg) mod/ref's the pointer.
-    AliasAnalysis::ModRefResult MR = AA->getModRefInfo(Inst, MemLoc);
+    ModRefInfo MR = AA->getModRefInfo(Inst, MemLoc);
     // If necessary, perform additional analysis.
-    if (MR == AliasAnalysis::ModRef)
-      MR = AA->callCapturesBefore(Inst, MemLoc, DT);
+    if (MR == MRI_ModRef)
+      MR = AA->callCapturesBefore(Inst, MemLoc, DT, &OBB);
     switch (MR) {
-    case AliasAnalysis::NoModRef:
+    case MRI_NoModRef:
       // If the call has no effect on the queried pointer, just ignore it.
       continue;
-    case AliasAnalysis::Mod:
+    case MRI_Mod:
       return MemDepResult::getClobber(Inst);
-    case AliasAnalysis::Ref:
+    case MRI_Ref:
       // If the call is known to never store to the pointer, and if this is a
       // load query, we can safely ignore it (scan past it).
       if (isLoad)
@@ -681,11 +687,11 @@ MemDepResult MemoryDependenceAnalysis::getDependency(Instruction *QueryInst) {
     else
       LocalCache = MemDepResult::getNonFuncLocal();
   } else {
-    AliasAnalysis::Location MemLoc;
-    AliasAnalysis::ModRefResult MR = GetLocation(QueryInst, MemLoc, AA);
+    MemoryLocation MemLoc;
+    ModRefInfo MR = GetLocation(QueryInst, MemLoc, AA);
     if (MemLoc.Ptr) {
       // If we can do a pointer scan, make it happen.
-      bool isLoad = !(MR & AliasAnalysis::Mod);
+      bool isLoad = !(MR & MRI_Mod);
       if (IntrinsicInst *II = dyn_cast<IntrinsicInst>(QueryInst))
         isLoad |= II->getIntrinsicID() == Intrinsic::lifetime_start;
 
@@ -770,8 +776,8 @@ MemoryDependenceAnalysis::getNonLocalCallDependency(CallSite QueryCS) {
   } else {
     // Seed DirtyBlocks with each of the preds of QueryInst's block.
     BasicBlock *QueryBB = QueryCS.getInstruction()->getParent();
-    for (BasicBlock **PI = PredCache->GetPreds(QueryBB); *PI; ++PI)
-      DirtyBlocks.push_back(*PI);
+    for (BasicBlock *Pred : PredCache.get(QueryBB))
+      DirtyBlocks.push_back(Pred);
     ++NumUncacheNonLocal;
   }
 
@@ -856,8 +862,8 @@ MemoryDependenceAnalysis::getNonLocalCallDependency(CallSite QueryCS) {
 
       // If the block *is* completely transparent to the load, we need to check
       // the predecessors of this block.  Add them to our worklist.
-      for (BasicBlock **PI = PredCache->GetPreds(DirtyBB); *PI; ++PI)
-        DirtyBlocks.push_back(*PI);
+      for (BasicBlock *Pred : PredCache.get(DirtyBB))
+        DirtyBlocks.push_back(Pred);
     }
   }
 
@@ -874,7 +880,7 @@ MemoryDependenceAnalysis::getNonLocalCallDependency(CallSite QueryCS) {
 void MemoryDependenceAnalysis::
 getNonLocalPointerDependency(Instruction *QueryInst,
                              SmallVectorImpl<NonLocalDepResult> &Result) {
-  const AliasAnalysis::Location Loc = AA->getLocation(QueryInst);
+  const MemoryLocation Loc = MemoryLocation::get(QueryInst);
   bool isLoad = isa<LoadInst>(QueryInst);
   BasicBlock *FromBB = QueryInst->getParent();
   assert(FromBB);
@@ -926,11 +932,9 @@ getNonLocalPointerDependency(Instruction *QueryInst,
 /// Pointer/PointeeSize using either cached information in Cache or by doing a
 /// lookup (which may use dirty cache info if available).  If we do a lookup,
 /// add the result to the cache.
-MemDepResult MemoryDependenceAnalysis::
-GetNonLocalInfoForBlock(Instruction *QueryInst,
-                        const AliasAnalysis::Location &Loc,
-                        bool isLoad, BasicBlock *BB,
-                        NonLocalDepInfo *Cache, unsigned NumSortedEntries) {
+MemDepResult MemoryDependenceAnalysis::GetNonLocalInfoForBlock(
+    Instruction *QueryInst, const MemoryLocation &Loc, bool isLoad,
+    BasicBlock *BB, NonLocalDepInfo *Cache, unsigned NumSortedEntries) {
 
   // Do a binary search to see if we already have an entry for this block in
   // the cache set.  If so, find it.
@@ -1042,14 +1046,11 @@ SortNonLocalDepInfoCache(MemoryDependenceAnalysis::NonLocalDepInfo &Cache,
 /// This function returns false on success, or true to indicate that it could
 /// not compute dependence information for some reason.  This should be treated
 /// as a clobber dependence on the first instruction in the predecessor block.
-bool MemoryDependenceAnalysis::
-getNonLocalPointerDepFromBB(Instruction *QueryInst,
-                            const PHITransAddr &Pointer,
-                            const AliasAnalysis::Location &Loc,
-                            bool isLoad, BasicBlock *StartBB,
-                            SmallVectorImpl<NonLocalDepResult> &Result,
-                            DenseMap<BasicBlock*, Value*> &Visited,
-                            bool SkipFirstBlock) {
+bool MemoryDependenceAnalysis::getNonLocalPointerDepFromBB(
+    Instruction *QueryInst, const PHITransAddr &Pointer,
+    const MemoryLocation &Loc, bool isLoad, BasicBlock *StartBB,
+    SmallVectorImpl<NonLocalDepResult> &Result,
+    DenseMap<BasicBlock *, Value *> &Visited, bool SkipFirstBlock) {
   // Look up the cached info for Pointer.
   ValueIsLoadPair CacheKey(Pointer.getAddr(), isLoad);
 
@@ -1232,13 +1233,13 @@ getNonLocalPointerDepFromBB(Instruction *QueryInst,
     if (!Pointer.NeedsPHITranslationFromBlock(BB)) {
       SkipFirstBlock = false;
       SmallVector<BasicBlock*, 16> NewBlocks;
-      for (BasicBlock **PI = PredCache->GetPreds(BB); *PI; ++PI) {
+      for (BasicBlock *Pred : PredCache.get(BB)) {
         // Verify that we haven't looked at this block yet.
         std::pair<DenseMap<BasicBlock*,Value*>::iterator, bool>
-          InsertRes = Visited.insert(std::make_pair(*PI, Pointer.getAddr()));
+          InsertRes = Visited.insert(std::make_pair(Pred, Pointer.getAddr()));
         if (InsertRes.second) {
           // First time we've looked at *PI.
-          NewBlocks.push_back(*PI);
+          NewBlocks.push_back(Pred);
           continue;
         }
 
@@ -1274,15 +1275,13 @@ getNonLocalPointerDepFromBB(Instruction *QueryInst,
     Cache = nullptr;
 
     PredList.clear();
-    for (BasicBlock **PI = PredCache->GetPreds(BB); *PI; ++PI) {
-      BasicBlock *Pred = *PI;
+    for (BasicBlock *Pred : PredCache.get(BB)) {
       PredList.push_back(std::make_pair(Pred, Pointer));
 
       // Get the PHI translated pointer in this predecessor.  This can fail if
       // not translatable, in which case the getAddr() returns null.
       PHITransAddr &PredPointer = PredList.back().second;
-      PredPointer.PHITranslateValue(BB, Pred, nullptr);
-
+      PredPointer.PHITranslateValue(BB, Pred, DT, /*MustDominate=*/false);
       Value *PredPtrVal = PredPointer.getAddr();
 
       // Check to see if we have already visited this pred block with another
@@ -1465,7 +1464,7 @@ void MemoryDependenceAnalysis::invalidateCachedPointerInfo(Value *Ptr) {
 /// This needs to be done when the CFG changes, e.g., due to splitting
 /// critical edges.
 void MemoryDependenceAnalysis::invalidateCachedPredecessors() {
-  PredCache->clear();
+  PredCache.clear();
 }
 
 /// removeInstruction - Remove an instruction from the dependence analysis,
@@ -1626,7 +1625,6 @@ void MemoryDependenceAnalysis::removeInstruction(Instruction *RemInst) {
 
 
   assert(!NonLocalDeps.count(RemInst) && "RemInst got reinserted?");
-  AA->deleteValue(RemInst);
   DEBUG(verifyRemoved(RemInst));
 }
 /// verifyRemoved - Verify that the specified instruction does not occur
diff --git a/lib/Analysis/MemoryLocation.cpp b/lib/Analysis/MemoryLocation.cpp
new file mode 100644
index 00000000000..e4491261e05
--- /dev/null
+++ b/lib/Analysis/MemoryLocation.cpp
@@ -0,0 +1,174 @@
+//===- MemoryLocation.cpp - Memory location descriptions -------------------==//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/Analysis/MemoryLocation.h"
+#include "llvm/Analysis/TargetLibraryInfo.h"
+#include "llvm/IR/BasicBlock.h"
+#include "llvm/IR/DataLayout.h"
+#include "llvm/IR/Instructions.h"
+#include "llvm/IR/IntrinsicInst.h"
+#include "llvm/IR/LLVMContext.h"
+#include "llvm/IR/Module.h"
+#include "llvm/IR/Type.h"
+using namespace llvm;
+
+MemoryLocation MemoryLocation::get(const LoadInst *LI) {
+  AAMDNodes AATags;
+  LI->getAAMetadata(AATags);
+  const auto &DL = LI->getModule()->getDataLayout();
+
+  return MemoryLocation(LI->getPointerOperand(),
+                        DL.getTypeStoreSize(LI->getType()), AATags);
+}
+
+MemoryLocation MemoryLocation::get(const StoreInst *SI) {
+  AAMDNodes AATags;
+  SI->getAAMetadata(AATags);
+  const auto &DL = SI->getModule()->getDataLayout();
+
+  return MemoryLocation(SI->getPointerOperand(),
+                        DL.getTypeStoreSize(SI->getValueOperand()->getType()),
+                        AATags);
+}
+
+MemoryLocation MemoryLocation::get(const VAArgInst *VI) {
+  AAMDNodes AATags;
+  VI->getAAMetadata(AATags);
+
+  return MemoryLocation(VI->getPointerOperand(), UnknownSize, AATags);
+}
+
+MemoryLocation MemoryLocation::get(const AtomicCmpXchgInst *CXI) {
+  AAMDNodes AATags;
+  CXI->getAAMetadata(AATags);
+  const auto &DL = CXI->getModule()->getDataLayout();
+
+  return MemoryLocation(
+      CXI->getPointerOperand(),
+      DL.getTypeStoreSize(CXI->getCompareOperand()->getType()), AATags);
+}
+
+MemoryLocation MemoryLocation::get(const AtomicRMWInst *RMWI) {
+  AAMDNodes AATags;
+  RMWI->getAAMetadata(AATags);
+  const auto &DL = RMWI->getModule()->getDataLayout();
+
+  return MemoryLocation(RMWI->getPointerOperand(),
+                        DL.getTypeStoreSize(RMWI->getValOperand()->getType()),
+                        AATags);
+}
+
+MemoryLocation MemoryLocation::getForSource(const MemTransferInst *MTI) {
+  uint64_t Size = UnknownSize;
+  if (ConstantInt *C = dyn_cast<ConstantInt>(MTI->getLength()))
+    Size = C->getValue().getZExtValue();
+
+  // memcpy/memmove can have AA tags. For memcpy, they apply
+  // to both the source and the destination.
+  AAMDNodes AATags;
+  MTI->getAAMetadata(AATags);
+
+  return MemoryLocation(MTI->getRawSource(), Size, AATags);
+}
+
+MemoryLocation MemoryLocation::getForDest(const MemIntrinsic *MTI) {
+  uint64_t Size = UnknownSize;
+  if (ConstantInt *C = dyn_cast<ConstantInt>(MTI->getLength()))
+    Size = C->getValue().getZExtValue();
+
+  // memcpy/memmove can have AA tags. For memcpy, they apply
+  // to both the source and the destination.
+  AAMDNodes AATags;
+  MTI->getAAMetadata(AATags);
+
+  return MemoryLocation(MTI->getRawDest(), Size, AATags);
+}
+
+// FIXME: This code is duplicated with BasicAliasAnalysis and should be hoisted
+// to some common utility location.
+static bool isMemsetPattern16(const Function *MS,
+                              const TargetLibraryInfo &TLI) {
+  if (TLI.has(LibFunc::memset_pattern16) &&
+      MS->getName() == "memset_pattern16") {
+    FunctionType *MemsetType = MS->getFunctionType();
+    if (!MemsetType->isVarArg() && MemsetType->getNumParams() == 3 &&
+        isa<PointerType>(MemsetType->getParamType(0)) &&
+        isa<PointerType>(MemsetType->getParamType(1)) &&
+        isa<IntegerType>(MemsetType->getParamType(2)))
+      return true;
+  }
+
+  return false;
+}
+
+MemoryLocation MemoryLocation::getForArgument(ImmutableCallSite CS,
+                                              unsigned ArgIdx,
+                                              const TargetLibraryInfo &TLI) {
+  AAMDNodes AATags;
+  CS->getAAMetadata(AATags);
+  const Value *Arg = CS.getArgument(ArgIdx);
+
+  // We may be able to produce an exact size for known intrinsics.
+  if (const IntrinsicInst *II = dyn_cast<IntrinsicInst>(CS.getInstruction())) {
+    const DataLayout &DL = II->getModule()->getDataLayout();
+
+    switch (II->getIntrinsicID()) {
+    default:
+      break;
+    case Intrinsic::memset:
+    case Intrinsic::memcpy:
+    case Intrinsic::memmove:
+      assert((ArgIdx == 0 || ArgIdx == 1) &&
+             "Invalid argument index for memory intrinsic");
+      if (ConstantInt *LenCI = dyn_cast<ConstantInt>(II->getArgOperand(2)))
+        return MemoryLocation(Arg, LenCI->getZExtValue(), AATags);
+      break;
+
+    case Intrinsic::lifetime_start:
+    case Intrinsic::lifetime_end:
+    case Intrinsic::invariant_start:
+      assert(ArgIdx == 1 && "Invalid argument index");
+      return MemoryLocation(
+          Arg, cast<ConstantInt>(II->getArgOperand(0))->getZExtValue(), AATags);
+
+    case Intrinsic::invariant_end:
+      assert(ArgIdx == 2 && "Invalid argument index");
+      return MemoryLocation(
+          Arg, cast<ConstantInt>(II->getArgOperand(1))->getZExtValue(), AATags);
+
+    case Intrinsic::arm_neon_vld1:
+      assert(ArgIdx == 0 && "Invalid argument index");
+      // LLVM's vld1 and vst1 intrinsics currently only support a single
+      // vector register.
+      return MemoryLocation(Arg, DL.getTypeStoreSize(II->getType()), AATags);
+
+    case Intrinsic::arm_neon_vst1:
+      assert(ArgIdx == 0 && "Invalid argument index");
+      return MemoryLocation(
+          Arg, DL.getTypeStoreSize(II->getArgOperand(1)->getType()), AATags);
+    }
+  }
+
+  // We can bound the aliasing properties of memset_pattern16 just as we can
+  // for memcpy/memset.  This is particularly important because the
+  // LoopIdiomRecognizer likes to turn loops into calls to memset_pattern16
+  // whenever possible.
+  if (CS.getCalledFunction() &&
+      isMemsetPattern16(CS.getCalledFunction(), TLI)) {
+    assert((ArgIdx == 0 || ArgIdx == 1) &&
+           "Invalid argument index for memset_pattern16");
+    if (ArgIdx == 1)
+      return MemoryLocation(Arg, 16, AATags);
+    if (const ConstantInt *LenCI = dyn_cast<ConstantInt>(CS.getArgument(2)))
+      return MemoryLocation(Arg, LenCI->getZExtValue(), AATags);
+  }
+  // FIXME: Handle memset_pattern4 and memset_pattern8 also.
+
+  return MemoryLocation(CS.getArgument(ArgIdx), UnknownSize, AATags);
+}
diff --git a/lib/Analysis/ModuleDebugInfoPrinter.cpp b/lib/Analysis/ModuleDebugInfoPrinter.cpp
index f2a11cb461f..36c47141a45 100644
--- a/lib/Analysis/ModuleDebugInfoPrinter.cpp
+++ b/lib/Analysis/ModuleDebugInfoPrinter.cpp
@@ -72,7 +72,7 @@ void ModuleDebugInfoPrinter::print(raw_ostream &O, const Module *M) const {
   // Printing the nodes directly isn't particularly helpful (since they
   // reference other nodes that won't be printed, particularly for the
   // filenames), so just print a few useful things.
-  for (MDCompileUnit *CU : Finder.compile_units()) {
+  for (DICompileUnit *CU : Finder.compile_units()) {
     O << "Compile unit: ";
     if (const char *Lang = dwarf::LanguageString(CU->getSourceLanguage()))
       O << Lang;
@@ -82,7 +82,7 @@ void ModuleDebugInfoPrinter::print(raw_ostream &O, const Module *M) const {
     O << '\n';
   }
 
-  for (MDSubprogram *S : Finder.subprograms()) {
+  for (DISubprogram *S : Finder.subprograms()) {
     O << "Subprogram: " << S->getName();
     printFile(O, S->getFilename(), S->getDirectory(), S->getLine());
     if (!S->getLinkageName().empty())
@@ -90,7 +90,7 @@ void ModuleDebugInfoPrinter::print(raw_ostream &O, const Module *M) const {
     O << '\n';
   }
 
-  for (DIGlobalVariable GV : Finder.global_variables()) {
+  for (const DIGlobalVariable *GV : Finder.global_variables()) {
     O << "Global variable: " << GV->getName();
     printFile(O, GV->getFilename(), GV->getDirectory(), GV->getLine());
     if (!GV->getLinkageName().empty())
@@ -98,12 +98,12 @@ void ModuleDebugInfoPrinter::print(raw_ostream &O, const Module *M) const {
     O << '\n';
   }
 
-  for (const MDType *T : Finder.types()) {
+  for (const DIType *T : Finder.types()) {
     O << "Type:";
     if (!T->getName().empty())
       O << ' ' << T->getName();
     printFile(O, T->getFilename(), T->getDirectory(), T->getLine());
-    if (auto *BT = dyn_cast<MDBasicType>(T)) {
+    if (auto *BT = dyn_cast<DIBasicType>(T)) {
       O << " ";
       if (const char *Encoding =
               dwarf::AttributeEncodingString(BT->getEncoding()))
@@ -117,7 +117,7 @@ void ModuleDebugInfoPrinter::print(raw_ostream &O, const Module *M) const {
       else
         O << "unknown-tag(" << T->getTag() << ")";
     }
-    if (auto *CT = dyn_cast<MDCompositeType>(T)) {
+    if (auto *CT = dyn_cast<DICompositeType>(T)) {
       if (auto *S = CT->getRawIdentifier())
         O << " (identifier: '" << S->getString() << "')";
     }
diff --git a/lib/Analysis/NaCl/CMakeLists.txt b/lib/Analysis/NaCl/CMakeLists.txt
deleted file mode 100644
index e4d514afbf3..00000000000
--- a/lib/Analysis/NaCl/CMakeLists.txt
+++ /dev/null
@@ -1,9 +0,0 @@
-add_llvm_library(LLVMNaClAnalysis
-  PNaClABITypeChecker.cpp
-  PNaClABIProps.cpp
-  PNaClABIVerifyFunctions.cpp
-  PNaClABIVerifyModule.cpp
-  PNaClAllowedIntrinsics.cpp
-  )
-
-add_dependencies(LLVMNaClAnalysis intrinsics_gen)
diff --git a/lib/Analysis/NaCl/LLVMBuild.txt b/lib/Analysis/NaCl/LLVMBuild.txt
deleted file mode 100644
index b5e7c8a5eaf..00000000000
--- a/lib/Analysis/NaCl/LLVMBuild.txt
+++ /dev/null
@@ -1,23 +0,0 @@
-;===- ./lib/Analysis/NaCl/LLVMBuild.txt ----------------------*- Conf -*--===;
-;
-;                     The LLVM Compiler Infrastructure
-;
-; This file is distributed under the University of Illinois Open Source
-; License. See LICENSE.TXT for details.
-;
-;===------------------------------------------------------------------------===;
-;
-; This is an LLVMBuild description file for the components in this subdirectory.
-;
-; For more information on the LLVMBuild system, please see:
-;
-;   http://llvm.org/docs/LLVMBuild.html
-;
-;===------------------------------------------------------------------------===;
-
-[component_0]
-type = Library
-name = NaClAnalysis
-parent = Analysis
-library_name = NaClAnalysis
-required_libraries = Analysis Core Support
diff --git a/lib/Analysis/NaCl/Makefile b/lib/Analysis/NaCl/Makefile
deleted file mode 100644
index 7d03b1e92eb..00000000000
--- a/lib/Analysis/NaCl/Makefile
+++ /dev/null
@@ -1,14 +0,0 @@
-##===- lib/Analysis/NaCl/Makefile-------------------------*- Makefile -*-===##
-#
-#                     The LLVM Compiler Infrastructure
-#
-# This file is distributed under the University of Illinois Open Source
-# License. See LICENSE.TXT for details.
-#
-##===----------------------------------------------------------------------===##
-
-LEVEL = ../../..
-LIBRARYNAME = LLVMNaClAnalysis
-BUILD_ARCHIVE = 1
-
-include $(LEVEL)/Makefile.common
\ No newline at end of file
diff --git a/lib/Analysis/NaCl/PNaClABIProps.cpp b/lib/Analysis/NaCl/PNaClABIProps.cpp
deleted file mode 100644
index f157cba0a6c..00000000000
--- a/lib/Analysis/NaCl/PNaClABIProps.cpp
+++ /dev/null
@@ -1,103 +0,0 @@
-//===- PNaClABIProps.cpp - Verify PNaCl ABI Function Rules ----------------===//
-//
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
-//
-//===----------------------------------------------------------------------===//
-//
-// Verify function-level PNaCl ABI properties, at the construct level.
-//
-//
-//===----------------------------------------------------------------------===//
-
-#include "llvm/Analysis/NaCl.h"
-#include "llvm/Analysis/NaCl/PNaClABIProps.h"
-#include "llvm/IR/DataLayout.h"
-#include "llvm/IR/DerivedTypes.h"
-#include "llvm/IR/LLVMContext.h"
-#include "llvm/IR/Metadata.h"
-
-using namespace llvm;
-
-bool PNaClABIProps::isWhitelistedMetadata(unsigned MDKind) {
-  return MDKind == LLVMContext::MD_dbg && PNaClABIAllowDebugMetadata;
-}
-
-bool PNaClABIProps::isWhitelistedMetadata(const NamedMDNode *MD) {
-  return PNaClABIAllowDebugMetadata &&
-         (MD->getName().startswith("llvm.dbg.") ||
-          // "Debug Info Version" is in llvm.module.flags.
-          MD->getName().equals("llvm.module.flags"));
-}
-
-bool PNaClABIProps::
-isAllowedAlignment(const DataLayout *DL, uint64_t Alignment,
-                   const Type *Ty) {
-  // Non-atomic integer operations must always use "align 1", since we do not
-  // want the backend to generate code with non-portable undefined behaviour
-  // (such as misaligned access faults) if user code specifies "align 4" but
-  // uses a misaligned pointer.  As a concession to performance, we allow larger
-  // alignment values for floating point types, and we only allow vectors to be
-  // aligned by their element's size.
-  //
-  // TODO(jfb) Allow vectors to be marked as align == 1. This requires proper
-  //           testing on each supported ISA, and is probably not as common as
-  //           align == elemsize.
-  //
-  // To reduce the set of alignment values that need to be encoded in pexes, we
-  // disallow other alignment values.  We require alignments to be explicit by
-  // disallowing Alignment == 0.
-  if (Alignment > std::numeric_limits<uint64_t>::max() / CHAR_BIT)
-    return false; // No overflow assumed below.
-  else if (const VectorType *VTy = dyn_cast<VectorType>(Ty))
-    return !VTy->getElementType()->isIntegerTy(1) &&
-           (Alignment * CHAR_BIT ==
-            DL->getTypeSizeInBits(VTy->getElementType()));
-  else
-    return Alignment == 1 ||
-           (Ty->isDoubleTy() && Alignment == 8) ||
-           (Ty->isFloatTy() && Alignment == 4);
-}
-
-const char *PNaClABIProps::CallingConvName(CallingConv::ID CallingConv) {
-  // TODO(kschimpf): Add more calling conventions.
-  switch (CallingConv) {
-  case CallingConv::C:                return "ccc";
-  case CallingConv::Fast:             return "fastcc";
-  case CallingConv::Cold:             return "cold";
-  default:
-    return "unknown";
-  }
-}
-
-const char *PNaClABIProps::LinkageName(GlobalValue::LinkageTypes LT) {
-  // This logic is taken from PrintLinkage in lib/IR/AsmWriter.cpp
-  switch (LT) {
-  case GlobalValue::ExternalLinkage:      return "external";
-  case GlobalValue::PrivateLinkage:       return "private";
-  case GlobalValue::InternalLinkage:      return "internal";
-  case GlobalValue::LinkOnceAnyLinkage:   return "linkonce";
-  case GlobalValue::LinkOnceODRLinkage:   return "linkonce_odr";
-  case GlobalValue::WeakAnyLinkage:       return "weak";
-  case GlobalValue::WeakODRLinkage:       return "weak_odr";
-  case GlobalValue::CommonLinkage:        return "common";
-  case GlobalValue::AppendingLinkage:     return "appending";
-  case GlobalValue::ExternalWeakLinkage:  return "extern_weak";
-  case GlobalValue::AvailableExternallyLinkage:
-    return "available_externally";
-  }
-  llvm_unreachable("unhandled GlobalValue::LinkageTypes");
-}
-
-bool PNaClABIProps::isValidGlobalLinkage(GlobalValue::LinkageTypes Linkage) {
-  switch (Linkage) {
-    case GlobalValue::ExternalLinkage:
-      return true;
-    case GlobalValue::InternalLinkage:
-      return true;
-    default:
-      return false;
-  }
-}
diff --git a/lib/Analysis/NaCl/PNaClABITypeChecker.cpp b/lib/Analysis/NaCl/PNaClABITypeChecker.cpp
deleted file mode 100644
index 9ce28ed7961..00000000000
--- a/lib/Analysis/NaCl/PNaClABITypeChecker.cpp
+++ /dev/null
@@ -1,109 +0,0 @@
-//===- PNaClABITypeChecker.cpp - Verify PNaCl ABI rules -------------------===//
-//
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
-//
-//===----------------------------------------------------------------------===//
-//
-// Common type-checking code for module and function-level passes
-//
-//
-//===----------------------------------------------------------------------===//
-
-#include "llvm/Analysis/NaCl/PNaClABITypeChecker.h"
-#include "llvm/IR/Constant.h"
-#include "llvm/IR/Constants.h"
-#include "llvm/IR/DerivedTypes.h"
-#include "llvm/IR/Metadata.h"
-
-using namespace llvm;
-
-bool PNaClABITypeChecker::isValidParamType(const Type *Ty) {
-  if (!(isValidScalarType(Ty) || isValidVectorType(Ty)))
-    return false;
-  if (const IntegerType *IntTy = dyn_cast<IntegerType>(Ty)) {
-    // PNaCl requires function arguments and return values to be 32
-    // bits or larger.  This avoids exposing architecture
-    // ABI-dependent differences about whether arguments or return
-    // values are zero-extended when calling a function with the wrong
-    // prototype.
-    if (IntTy->getBitWidth() < 32)
-      return false;
-  }
-  return true;
-}
-
-bool PNaClABITypeChecker::isValidFunctionType(const FunctionType *FTy) {
-  if (FTy->isVarArg())
-    return false;
-  if (!isValidParamType(FTy->getReturnType()))
-    return false;
-  for (unsigned I = 0, E = FTy->getNumParams(); I < E; ++I) {
-    if (!isValidParamType(FTy->getParamType(I)))
-      return false;
-  }
-  return true;
-}
-
-namespace {
-static inline bool NaClIsValidIntType(const Type *Ty) {
-  unsigned Width = cast<const IntegerType>(Ty)->getBitWidth();
-  return Width == 1 || Width == 8 || Width == 16 ||
-      Width == 32 || Width == 64;
-}
-}
-
-bool PNaClABITypeChecker::isValidScalarType(const Type *Ty) {
-  switch (Ty->getTypeID()) {
-    case Type::IntegerTyID: {
-      return NaClIsValidIntType(Ty);
-    }
-    case Type::VoidTyID:
-    case Type::FloatTyID:
-    case Type::DoubleTyID:
-      return true;
-    default:
-      return false;
-  }
-}
-
-// TODO(jfb) Handle 64-bit int and double, and 2xi1.
-bool PNaClABITypeChecker::isValidVectorType(const Type *Ty) {
-  if (!Ty->isVectorTy())
-    return false;
-  unsigned Elems = Ty->getVectorNumElements();
-  const Type *VTy = Ty->getVectorElementType();
-
-  switch (VTy->getTypeID()) {
-  case Type::IntegerTyID: {
-    unsigned Width = cast<const IntegerType>(VTy)->getBitWidth();
-    switch (Width) {
-    case 1: return Elems == 4 || Elems == 8 || Elems == 16;
-    case 8: return Elems == 16;
-    case 16: return Elems == 8;
-    case 32: return Elems == 4;
-    default: return false;
-    }
-  }
-  case Type::FloatTyID:
-    return Elems == 4;
-  default:
-    return false;
-  }
-}
-
-namespace {
-static inline bool NaClIsValidIntArithmeticType(const Type *Ty) {
-  return Ty->isIntegerTy() && !Ty->isIntegerTy(1)
-      && NaClIsValidIntType(Ty);
-}
-
-}
-
-bool PNaClABITypeChecker::isValidIntArithmeticType(const Type *Ty) {
-  if (isValidVectorType(Ty))
-    return NaClIsValidIntArithmeticType(Ty->getVectorElementType());
-  return NaClIsValidIntArithmeticType(Ty);
-}
diff --git a/lib/Analysis/NaCl/PNaClABIVerifyFunctions.cpp b/lib/Analysis/NaCl/PNaClABIVerifyFunctions.cpp
deleted file mode 100644
index dc11f496fe4..00000000000
--- a/lib/Analysis/NaCl/PNaClABIVerifyFunctions.cpp
+++ /dev/null
@@ -1,635 +0,0 @@
-//===- PNaClABIVerifyFunctions.cpp - Verify PNaCl ABI rules ---------------===//
-//
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
-//
-//===----------------------------------------------------------------------===//
-//
-// Verify function-level PNaCl ABI requirements.
-//
-//
-//===----------------------------------------------------------------------===//
-
-#include "llvm/Analysis/NaCl/PNaClABIVerifyFunctions.h"
-#include "llvm/ADT/Twine.h"
-#include "llvm/Analysis/NaCl.h"
-#include "llvm/Analysis/NaCl/PNaClABITypeChecker.h"
-#include "llvm/Analysis/NaCl/PNaClAllowedIntrinsics.h"
-#include "llvm/IR/Function.h"
-#include "llvm/IR/Instructions.h"
-#include "llvm/IR/IntrinsicInst.h"
-#include "llvm/IR/LLVMContext.h"
-#include "llvm/IR/Metadata.h"
-#include "llvm/IR/Operator.h"
-#include "llvm/Support/raw_ostream.h"
-
-using namespace llvm;
-
-// There's no built-in way to get the name of an MDNode, so use a
-// string ostream to print it.
-static std::string getMDNodeString(unsigned Kind,
-                                   const SmallVectorImpl<StringRef> &MDNames) {
-  std::string MDName;
-  raw_string_ostream N(MDName);
-  if (Kind < MDNames.size()) {
-    N << "!" << MDNames[Kind];
-  } else {
-    N << "!<unknown kind #" << Kind << ">";
-  }
-  return N.str();
-}
-
-PNaClABIVerifyFunctions::~PNaClABIVerifyFunctions() {
-  if (ReporterIsOwned)
-    delete Reporter;
-}
-
-// A valid pointer type is either:
-//  * a pointer to a valid PNaCl scalar type (except i1), or
-//  * a pointer to a valid PNaCl vector type (except i1), or
-//  * a function pointer (with valid argument and return types).
-//
-// i1 is disallowed so that all loads and stores are a whole number of
-// bytes, and so that we do not need to define whether a store of i1
-// zero-extends.
-static bool isValidPointerType(Type *Ty) {
-  if (PointerType *PtrTy = dyn_cast<PointerType>(Ty)) {
-    if (PtrTy->getAddressSpace() != 0)
-      return false;
-    Type *EltTy = PtrTy->getElementType();
-    if (PNaClABITypeChecker::isValidScalarType(EltTy) && !EltTy->isIntegerTy(1))
-      return true;
-    if (PNaClABITypeChecker::isValidVectorType(EltTy) &&
-        !cast<VectorType>(EltTy)->getElementType()->isIntegerTy(1))
-      return true;
-    if (FunctionType *FTy = dyn_cast<FunctionType>(EltTy))
-      return PNaClABITypeChecker::isValidFunctionType(FTy);
-  }
-  return false;
-}
-
-static bool isIntrinsicFunc(const Value *Val) {
-  if (const Function *F = dyn_cast<Function>(Val))
-    return F->isIntrinsic();
-  return false;
-}
-
-// InherentPtrs may be referenced by casts -- PtrToIntInst and
-// BitCastInst -- that produce NormalizedPtrs.
-//
-// InherentPtrs exclude intrinsic functions in order to prevent taking
-// the address of an intrinsic function.  InherentPtrs include
-// intrinsic calls because some intrinsics return pointer types
-// (e.g. nacl.read.tp returns i8*).
-static bool isInherentPtr(const Value *Val) {
-  return isa<AllocaInst>(Val) ||
-         (isa<GlobalValue>(Val) && !isIntrinsicFunc(Val)) ||
-         isa<IntrinsicInst>(Val);
-}
-
-// NormalizedPtrs may be used where pointer types are required -- for
-// loads, stores, etc.  Note that this excludes ConstantExprs,
-// ConstantPointerNull and UndefValue.
-static bool isNormalizedPtr(const Value *Val) {
-  if (!isValidPointerType(Val->getType()))
-    return false;
-  // The bitcast must also be a bitcast of an InherentPtr, but we
-  // check that when visiting the bitcast instruction.
-  return isa<IntToPtrInst>(Val) || isa<BitCastInst>(Val) || isInherentPtr(Val);
-}
-
-static bool isValidScalarOperand(const Value *Val) {
-  // The types of Instructions and Arguments are checked elsewhere
-  // (when visiting the Instruction or the Function).  BasicBlocks are
-  // included here because branch instructions have BasicBlock
-  // operands.
-  if (isa<Instruction>(Val) || isa<Argument>(Val) || isa<BasicBlock>(Val))
-    return true;
-
-  // Allow some Constants.  Note that this excludes ConstantExprs.
-  return PNaClABITypeChecker::isValidScalarType(Val->getType()) &&
-         (isa<ConstantInt>(Val) ||
-          isa<ConstantFP>(Val) ||
-          isa<UndefValue>(Val));
-}
-
-static bool isValidVectorOperand(const Value *Val) {
-  // The types of Instructions and Arguments are checked elsewhere.
-  if (isa<Instruction>(Val) || isa<Argument>(Val))
-    return true;
-  // Contrary to scalars, constant vector values aren't allowed on
-  // instructions, except undefined. Constant vectors are loaded from
-  // constant global memory instead, and can be rematerialized as
-  // constants by the backend if need be.
-  return PNaClABITypeChecker::isValidVectorType(Val->getType()) &&
-         isa<UndefValue>(Val);
-}
-
-static bool hasAllowedAtomicRMWOperation(
-    const NaCl::AtomicIntrinsics::AtomicIntrinsic *I, const CallInst *Call) {
-  for (size_t P = 0; P != I->NumParams; ++P) {
-    if (I->ParamType[P] != NaCl::AtomicIntrinsics::RMW)
-      continue;
-
-    const Value *Operation = Call->getOperand(P);
-    if (!Operation)
-      return false;
-    const Constant *C = dyn_cast<Constant>(Operation);
-    if (!C)
-      return false;
-    const APInt &I = C->getUniqueInteger();
-    if (I.ule(NaCl::AtomicInvalid) || I.uge(NaCl::AtomicNum))
-      return false;
-  }
-  return true;
-}
-
-static bool
-hasAllowedAtomicMemoryOrder(const NaCl::AtomicIntrinsics::AtomicIntrinsic *I,
-                            const CallInst *Call) {
-  NaCl::MemoryOrder PreviousOrder = NaCl::MemoryOrderInvalid;
-
-  for (size_t P = 0; P != I->NumParams; ++P) {
-    if (I->ParamType[P] != NaCl::AtomicIntrinsics::Mem)
-      continue;
-
-    NaCl::MemoryOrder Order = NaCl::MemoryOrderInvalid;
-    if (const Value *MemoryOrderOperand = Call->getOperand(P))
-      if (const Constant *C = dyn_cast<Constant>(MemoryOrderOperand)) {
-        const APInt &I = C->getUniqueInteger();
-        if (I.ugt(NaCl::MemoryOrderInvalid) && I.ult(NaCl::MemoryOrderNum))
-          Order = static_cast<NaCl::MemoryOrder>(I.getLimitedValue());
-      }
-    if (Order == NaCl::MemoryOrderInvalid)
-      return false;
-
-    // Validate PNaCl restrictions.
-    switch (Order) {
-    case NaCl::MemoryOrderInvalid:
-    case NaCl::MemoryOrderNum:
-      llvm_unreachable("Invalid memory order");
-    case NaCl::MemoryOrderRelaxed:
-    case NaCl::MemoryOrderConsume:
-      // TODO(jfb) PNaCl doesn't allow relaxed or consume memory ordering.
-      return false;
-    case NaCl::MemoryOrderAcquire:
-    case NaCl::MemoryOrderRelease:
-    case NaCl::MemoryOrderAcquireRelease:
-    case NaCl::MemoryOrderSequentiallyConsistent:
-      break; // Allowed by PNaCl.
-    }
-
-    // Validate conformance to the C++11 memory model.
-    switch (I->ID) {
-    default:
-      llvm_unreachable("unexpected atomic operation");
-    case Intrinsic::nacl_atomic_load:
-      // C++11 [atomics.types.operations.req]: The order argument shall not be
-      // release nor acq_rel.
-      if (Order == NaCl::MemoryOrderRelease ||
-          Order == NaCl::MemoryOrderAcquireRelease)
-        return false;
-      break;
-    case Intrinsic::nacl_atomic_store:
-      // C++11 [atomics.types.operations.req]: The order argument shall not be
-      // consume, acquire, nor acq_rel.
-      if (Order == NaCl::MemoryOrderConsume ||
-          Order == NaCl::MemoryOrderAcquire ||
-          Order == NaCl::MemoryOrderAcquireRelease)
-        return false;
-      break;
-    case Intrinsic::nacl_atomic_rmw:
-      break; // No restriction.
-    case Intrinsic::nacl_atomic_cmpxchg:
-      // C++11 [atomics.types.operations.req]: The failure argument shall not be
-      // release nor acq_rel. The failure argument shall be no stronger than the
-      // success argument.
-      // Where the partial ordering is:
-      //   relaxed < consume < acquire           < acq_rel < seq_cst
-      //   relaxed <                     release < acq_rel < seq_cst
-      if (PreviousOrder != NaCl::MemoryOrderInvalid) { // Failure ordering.
-        NaCl::MemoryOrder Success = PreviousOrder, Failure = Order;
-        if (Failure == NaCl::MemoryOrderRelease ||
-            Failure == NaCl::MemoryOrderAcquireRelease)
-          return false;
-        if ((Success < Failure) || (Success == NaCl::MemoryOrderRelease &&
-                                    Failure != NaCl::MemoryOrderRelaxed))
-          return false;
-      }
-      break; // Success ordering has no restriction.
-    case Intrinsic::nacl_atomic_fence:
-    case Intrinsic::nacl_atomic_fence_all:
-      break; // No restrictions.
-    }
-
-    PreviousOrder = Order;
-  }
-
-  return true;
-}
-
-static bool hasAllowedLockFreeByteSize(const CallInst *Call) {
-  if (!Call->getType()->isIntegerTy())
-    return false;
-  const Value *Operation = Call->getOperand(0);
-  if (!Operation)
-    return false;
-  const Constant *C = dyn_cast<Constant>(Operation);
-  if (!C)
-    return false;
-  const APInt &I = C->getUniqueInteger();
-  // PNaCl currently only supports atomics of byte size {1,2,4,8} (which
-  // may or may not be lock-free). These values coincide with
-  // C11/C++11's supported atomic types.
-  if (I == 1 || I == 2 || I == 4 || I == 8)
-    return true;
-  return false;
-}
-
-// Check the instruction's opcode and its operands.  The operands may
-// require opcode-specific checking.
-//
-// This returns an error string if the instruction is rejected, or
-// NULL if the instruction is allowed.
-const char *PNaClABIVerifyFunctions::checkInstruction(const DataLayout *DL,
-                                                      const Instruction *Inst) {
-  // If the instruction has a single pointer operand, PtrOperandIndex is
-  // set to its operand index.
-  unsigned PtrOperandIndex = -1;
-
-  // True if we should apply the default operand checks, at the end
-  // of this function.
-  bool ApplyDefaultOperandTypeChecks = true;
-
-  switch (Inst->getOpcode()) {
-    // Disallowed instructions. Default is to disallow.
-    // We expand GetElementPtr out into arithmetic.
-    case Instruction::GetElementPtr:
-    // VAArg is expanded out by ExpandVarArgs.
-    case Instruction::VAArg:
-    // Zero-cost C++ exception handling is not supported yet.
-    case Instruction::Invoke:
-    case Instruction::LandingPad:
-    case Instruction::Resume:
-    // indirectbr may interfere with streaming
-    case Instruction::IndirectBr:
-    // TODO(jfb) Figure out ShuffleVector.
-    case Instruction::ShuffleVector:
-    // ExtractValue and InsertValue operate on struct values.
-    case Instruction::ExtractValue:
-    case Instruction::InsertValue:
-    // Atomics should become NaCl intrinsics.
-    case Instruction::AtomicCmpXchg:
-    case Instruction::AtomicRMW:
-    case Instruction::Fence:
-      return "bad instruction opcode";
-    default:
-      return "unknown instruction opcode";
-
-    // Terminator instructions
-    case Instruction::Ret:
-    case Instruction::Br:
-    case Instruction::Unreachable:
-    // Binary operations
-    case Instruction::FAdd:
-    case Instruction::FSub:
-    case Instruction::FMul:
-    case Instruction::FDiv:
-    case Instruction::FRem:
-    // Bitwise binary operations
-    case Instruction::And:
-    case Instruction::Or:
-    case Instruction::Xor:
-    // Conversion operations
-    case Instruction::Trunc:
-    case Instruction::ZExt:
-    case Instruction::SExt:
-    case Instruction::FPTrunc:
-    case Instruction::FPExt:
-    case Instruction::FPToUI:
-    case Instruction::FPToSI:
-    case Instruction::UIToFP:
-    case Instruction::SIToFP:
-    // Other operations
-    case Instruction::FCmp:
-    case Instruction::PHI:
-    case Instruction::Select:
-      break;
-
-    // The following operations are of dubious usefulness on 1-bit
-    // values.  Use of the i1 type is disallowed here so that code
-    // generators do not need to support these corner cases.
-    case Instruction::ICmp:
-    // Binary operations
-    case Instruction::Add:
-    case Instruction::Sub:
-    case Instruction::Mul:
-    case Instruction::UDiv:
-    case Instruction::SDiv:
-    case Instruction::URem:
-    case Instruction::SRem:
-    case Instruction::Shl:
-    case Instruction::LShr:
-    case Instruction::AShr: {
-      const Type *Ty = Inst->getOperand(0)->getType();
-      if (!PNaClABITypeChecker::isValidIntArithmeticType(
-              Inst->getOperand(0)->getType())) {
-        if (Ty->isIntegerTy() ||
-            (Ty->isVectorTy() && Ty->getVectorElementType()->isIntegerTy())) {
-          return "Invalid integer arithmetic type";
-        } else {
-          return "Expects integer arithmetic type";
-        }
-      }
-      ApplyDefaultOperandTypeChecks = false;
-      break;
-    }
-
-    // Vector.
-    case Instruction::ExtractElement:
-    case Instruction::InsertElement: {
-      // Insert and extract element are restricted to constant indices
-      // that are in range to prevent undefined behavior.
-      // TODO(kschimpf) Figure out way to put test into pnacl-bcdis?
-      Value *Vec = Inst->getOperand(0);
-      Value *Idx = Inst->getOperand(
-          Instruction::InsertElement == Inst->getOpcode() ? 2 : 1);
-      if (!isa<ConstantInt>(Idx))
-        return "non-constant vector insert/extract index";
-      if (!PNaClABIProps::isVectorIndexSafe(
-              cast<ConstantInt>(Idx)->getValue(),
-              cast<VectorType>(Vec->getType())->getNumElements())) {
-        return "out of range vector insert/extract index";
-      }
-      break;
-    }
-
-    // Memory accesses.
-    case Instruction::Load: {
-      const LoadInst *Load = cast<LoadInst>(Inst);
-      PtrOperandIndex = Load->getPointerOperandIndex();
-      if (Load->isAtomic())
-        return "atomic load";
-      if (Load->isVolatile())
-        return "volatile load";
-      if (!isNormalizedPtr(Inst->getOperand(PtrOperandIndex)))
-        return "bad pointer";
-      if (!PNaClABIProps::
-          isAllowedAlignment(DL, Load->getAlignment(), Load->getType()))
-        return "bad alignment";
-      break;
-    }
-    case Instruction::Store: {
-      const StoreInst *Store = cast<StoreInst>(Inst);
-      PtrOperandIndex = Store->getPointerOperandIndex();
-      if (Store->isAtomic())
-        return "atomic store";
-      if (Store->isVolatile())
-        return "volatile store";
-      if (!isNormalizedPtr(Inst->getOperand(PtrOperandIndex)))
-        return "bad pointer";
-      if (!PNaClABIProps::
-          isAllowedAlignment(DL, Store->getAlignment(),
-                             Store->getValueOperand()->getType()))
-        return "bad alignment";
-      break;
-    }
-
-    // Casts.
-    case Instruction::BitCast:
-      if (Inst->getType()->isPointerTy()) {
-        PtrOperandIndex = 0;
-        if (!isInherentPtr(Inst->getOperand(PtrOperandIndex)))
-          return "operand not InherentPtr";
-      }
-      break;
-    case Instruction::IntToPtr:
-      if (!cast<IntToPtrInst>(Inst)->getSrcTy()->isIntegerTy(32))
-        return "non-i32 inttoptr";
-      break;
-    case Instruction::PtrToInt:
-      PtrOperandIndex = 0;
-      if (!isInherentPtr(Inst->getOperand(PtrOperandIndex)))
-        return "operand not InherentPtr";
-      if (!Inst->getType()->isIntegerTy(32))
-        return "non-i32 ptrtoint";
-      break;
-
-    case Instruction::Alloca: {
-      const AllocaInst *Alloca = cast<AllocaInst>(Inst);
-      if (!PNaClABIProps::isAllocaAllocatedType(Alloca->getAllocatedType()))
-        return "non-i8 alloca";
-      if (!PNaClABIProps::isAllocaSizeType(Alloca->getArraySize()->getType()))
-        return PNaClABIProps::ExpectedAllocaSizeType();
-      break;
-    }
-
-    case Instruction::Call: {
-      const CallInst *Call = cast<CallInst>(Inst);
-      if (Call->isInlineAsm())
-        return "inline assembly";
-      if (!Call->getAttributes().isEmpty())
-        return "bad call attributes";
-      if (!PNaClABIProps::isValidCallingConv(Call->getCallingConv()))
-        return "bad calling convention";
-
-      // Intrinsic calls can have multiple pointer arguments and
-      // metadata arguments, so handle them specially.
-      // TODO(kschimpf) How can we lift this to pnacl-bcdis.
-      if (const IntrinsicInst *Call = dyn_cast<IntrinsicInst>(Inst)) {
-        if (PNaClAllowedIntrinsics::isAllowedDebugInfoIntrinsic(
-                Call->getIntrinsicID())) {
-          // If debug metadata is allowed, always allow calling debug intrinsics
-          // and assume they are correct.
-          return nullptr;
-        }
-        for (unsigned ArgNum = 0, E = Call->getNumArgOperands();
-             ArgNum < E; ++ArgNum) {
-          const Value *Arg = Call->getArgOperand(ArgNum);
-          if (!(isValidScalarOperand(Arg) ||
-                isValidVectorOperand(Arg) ||
-                isNormalizedPtr(Arg)))
-            return "bad intrinsic operand";
-        }
-
-        // Disallow alignments other than 1 on memcpy() etc., for the
-        // same reason that we disallow them on integer loads and
-        // stores.
-        if (const MemIntrinsic *MemOp = dyn_cast<MemIntrinsic>(Call)) {
-          // Avoid the getAlignment() method here because it aborts if
-          // the alignment argument is not a Constant.
-          Value *AlignArg = MemOp->getArgOperand(3);
-          if (!isa<ConstantInt>(AlignArg) ||
-              cast<ConstantInt>(AlignArg)->getZExtValue() != 1) {
-            return "bad alignment";
-          }
-        }
-
-        switch (Call->getIntrinsicID()) {
-          default: break;  // Other intrinsics don't require checks.
-          // Disallow NaCl atomic intrinsics which don't have valid
-          // constant NaCl::AtomicOperation and NaCl::MemoryOrder
-          // parameters.
-          case Intrinsic::nacl_atomic_load:
-          case Intrinsic::nacl_atomic_store:
-          case Intrinsic::nacl_atomic_rmw:
-          case Intrinsic::nacl_atomic_cmpxchg:
-          case Intrinsic::nacl_atomic_fence:
-          case Intrinsic::nacl_atomic_fence_all: {
-            // All overloads have memory order and RMW operation in the
-            // same parameter, arbitrarily use the I32 overload.
-            Type *T = Type::getInt32Ty(
-                Inst->getParent()->getParent()->getContext());
-            const NaCl::AtomicIntrinsics::AtomicIntrinsic *I =
-                AtomicIntrinsics->find(Call->getIntrinsicID(), T);
-            if (!I)
-              // All intrinsics have an I32 overload. Failure here means there
-              // is no such intrinsic.
-              return "invalid atomic intrinsic";
-            if (!hasAllowedAtomicMemoryOrder(I, Call))
-              return "invalid memory order";
-            if (!hasAllowedAtomicRMWOperation(I, Call))
-              return "invalid atomicRMW operation";
-          } break;
-          // Disallow NaCl atomic_is_lock_free intrinsics which don't
-          // have valid constant size type.
-          case Intrinsic::nacl_atomic_is_lock_free:
-            if (!hasAllowedLockFreeByteSize(Call))
-              return "invalid atomic lock-free byte size";
-            break;
-        }
-
-        // Allow the instruction and skip the later checks.
-        return NULL;
-      }
-
-      // The callee is the last operand.
-      PtrOperandIndex = Inst->getNumOperands() - 1;
-      if (!isNormalizedPtr(Inst->getOperand(PtrOperandIndex)))
-        return "bad function callee operand";
-      break;
-    }
-
-    case Instruction::Switch: {
-      // SwitchInst represents switch cases using array and vector
-      // constants, which we normally reject, so we must check
-      // SwitchInst specially here.
-      const SwitchInst *Switch = cast<SwitchInst>(Inst);
-      if (!isValidScalarOperand(Switch->getCondition()))
-        return "bad switch condition";
-      const Type *SwitchType = Switch->getCondition()->getType();
-      if (!PNaClABITypeChecker::isValidSwitchConditionType(SwitchType))
-        return PNaClABITypeChecker::ExpectedSwitchConditionType(SwitchType);
-
-      // SwitchInst requires the cases to be ConstantInts, but it
-      // doesn't require their types to be the same as the condition
-      // value, so check all the cases too.
-      for (SwitchInst::ConstCaseIt Case = Switch->case_begin(),
-             E = Switch->case_end(); Case != E; ++Case) {
-        if (!isValidScalarOperand(Case.getCaseValue()))
-          return "bad switch case";
-      }
-
-      // Allow the instruction and skip the later checks.
-      return NULL;
-    }
-  }
-
-  if (ApplyDefaultOperandTypeChecks) {
-    // Check the instruction's operands.  We have already checked any
-    // pointer operands.  Any remaining operands must be scalars or vectors.
-    for (unsigned OpNum = 0, E = Inst->getNumOperands(); OpNum < E; ++OpNum) {
-      if (OpNum != PtrOperandIndex &&
-          !(isValidScalarOperand(Inst->getOperand(OpNum)) ||
-            isValidVectorOperand(Inst->getOperand(OpNum))))
-        return "bad operand";
-    }
-  }
-
-  // Check arithmetic attributes.
-  if (const OverflowingBinaryOperator *Op =
-          dyn_cast<OverflowingBinaryOperator>(Inst)) {
-    if (Op->hasNoUnsignedWrap())
-      return "has \"nuw\" attribute";
-    if (Op->hasNoSignedWrap())
-      return "has \"nsw\" attribute";
-  }
-  if (const PossiblyExactOperator *Op =
-          dyn_cast<PossiblyExactOperator>(Inst)) {
-    if (Op->isExact())
-      return "has \"exact\" attribute";
-  }
-
-  // Allow the instruction.
-  return NULL;
-}
-
-bool PNaClABIVerifyFunctions::runOnFunction(Function &F) {
-  const DataLayout *DL = &F.getParent()->getDataLayout();
-  SmallVector<StringRef, 8> MDNames;
-  F.getContext().getMDKindNames(MDNames);
-
-  for (Function::const_iterator FI = F.begin(), FE = F.end();
-           FI != FE; ++FI) {
-    for (BasicBlock::const_iterator BBI = FI->begin(), BBE = FI->end();
-             BBI != BBE; ++BBI) {
-      const Instruction *Inst = BBI;
-      // Check the instruction opcode first.  This simplifies testing,
-      // because some instruction opcodes must be rejected out of hand
-      // (regardless of the instruction's result type) and the tests
-      // check the reason for rejection.
-      const char *Error = checkInstruction(DL, BBI);
-      // Check the instruction's result type.
-      bool BadResult = false;
-      if (!Error && !(PNaClABITypeChecker::isValidScalarType(Inst->getType()) ||
-                      PNaClABITypeChecker::isValidVectorType(Inst->getType()) ||
-                      isNormalizedPtr(Inst) ||
-                      isa<AllocaInst>(Inst))) {
-        Error = "bad result type";
-        BadResult = true;
-      }
-      if (Error) {
-        Reporter->addError()
-            << "Function " << F.getName() << " disallowed: " << Error << ": "
-            << (BadResult ? PNaClABITypeChecker::getTypeName(BBI->getType())
-                          : "") << " " << *BBI << "\n";
-      }
-
-      // Check instruction attachment metadata.
-      SmallVector<std::pair<unsigned, MDNode*>, 4> MDForInst;
-      BBI->getAllMetadata(MDForInst);
-
-      for (unsigned i = 0, e = MDForInst.size(); i != e; i++) {
-        if (!PNaClABIProps::isWhitelistedMetadata(MDForInst[i].first)) {
-          Reporter->addError()
-              << "Function " << F.getName()
-              << " has disallowed instruction metadata: "
-              << getMDNodeString(MDForInst[i].first, MDNames) << "\n";
-        }
-      }
-    }
-  }
-
-  Reporter->checkForFatalErrors();
-  return false;
-}
-
-// This method exists so that the passes can easily be run with opt -analyze.
-// In this case the default constructor is used and we want to reset the error
-// messages after each print.
-void PNaClABIVerifyFunctions::print(llvm::raw_ostream &O, const Module *M)
-    const {
-  Reporter->printErrors(O);
-  Reporter->reset();
-}
-
-char PNaClABIVerifyFunctions::ID = 0;
-INITIALIZE_PASS(PNaClABIVerifyFunctions, "verify-pnaclabi-functions",
-                "Verify functions for PNaCl", false, true)
-
-FunctionPass *llvm::createPNaClABIVerifyFunctionsPass(
-    PNaClABIErrorReporter *Reporter) {
-  return new PNaClABIVerifyFunctions(Reporter);
-}
diff --git a/lib/Analysis/NaCl/PNaClABIVerifyModule.cpp b/lib/Analysis/NaCl/PNaClABIVerifyModule.cpp
deleted file mode 100644
index d4bb151367c..00000000000
--- a/lib/Analysis/NaCl/PNaClABIVerifyModule.cpp
+++ /dev/null
@@ -1,327 +0,0 @@
-//===- PNaClABIVerifyModule.cpp - Verify PNaCl ABI rules ------------------===//
-//
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
-//
-//===----------------------------------------------------------------------===//
-//
-// Verify module-level PNaCl ABI requirements (specifically those that do not
-// require looking at the function bodies)
-//
-//
-//===----------------------------------------------------------------------===//
-
-#include "llvm/Analysis/NaCl/PNaClABIVerifyModule.h"
-#include "llvm/ADT/Twine.h"
-#include "llvm/Analysis/NaCl/PNaClABIProps.h"
-#include "llvm/Analysis/NaCl/PNaClABITypeChecker.h"
-#include "llvm/Analysis/NaCl/PNaClAllowedIntrinsics.h"
-#include "llvm/IR/Constants.h"
-#include "llvm/IR/DerivedTypes.h"
-#include "llvm/IR/Instructions.h"
-#include "llvm/Support/Debug.h"
-#include "llvm/Support/raw_ostream.h"
-
-using namespace llvm;
-
-namespace llvm {
-cl::opt<bool>
-PNaClABIAllowDebugMetadata("pnaclabi-allow-debug-metadata",
-  cl::desc("Allow debug metadata during PNaCl ABI verification."),
-  cl::init(false));
-
-cl::opt<bool>
-PNaClABIAllowMinsfiSyscalls("pnaclabi-allow-minsfi-syscalls",
-  cl::desc("Allow undefined references to MinSFI syscall functions."),
-  cl::init(false));
-
-}
-
-PNaClABIVerifyModule::~PNaClABIVerifyModule() {
-  if (ReporterIsOwned)
-    delete Reporter;
-}
-
-// MinSFI syscalls are functions with a given prefix which are left undefined
-// and later linked against their implementation inside the trusted runtime.
-// If the corresponding flag is set, do allow these external symbols in the
-// module.
-//
-// We also require the syscall declarations to have an i32 return type. This
-// is meant to prevent abusing syscalls to obtain an undefined value, e.g. by
-// invoking a syscall whose trusted implementation returns void as a function
-// which returns an integer, leaking the value of a register (see comments in
-// the SubstituteUndefs pass for more information on undef values).
-static bool isAllowedMinsfiSyscall(const Function *Func) {
-  return PNaClABIAllowMinsfiSyscalls &&
-         Func->getName().startswith("__minsfi_syscall_") &&
-         Func->getReturnType()->isIntegerTy(32);
-}
-
-// Check linkage type and section attributes, which are the same for
-// GlobalVariables and Functions.
-void PNaClABIVerifyModule::checkGlobalValue(const GlobalValue *GV) {
-  assert(!isa<GlobalAlias>(GV));
-  const char *GVTypeName = PNaClABIProps::GVTypeName(isa<Function>(GV));
-  GlobalValue::LinkageTypes Linkage = GV->getLinkage();
-  if (!PNaClABIProps::isValidGlobalLinkage(Linkage)) {
-    Reporter->addError() << GVTypeName << " " << GV->getName()
-                         << " has disallowed linkage type: "
-                         << PNaClABIProps::LinkageName(Linkage) << "\n";
-  }
-  if (Linkage == GlobalValue::ExternalLinkage) checkExternalSymbol(GV);
-  if (GV->getVisibility() != GlobalValue::DefaultVisibility) {
-    std::string Text = "unknown";
-    if (GV->getVisibility() == GlobalValue::HiddenVisibility) {
-      Text = "hidden";
-    } else if (GV->getVisibility() == GlobalValue::ProtectedVisibility) {
-      Text = "protected";
-    }
-    Reporter->addError() << GVTypeName << " " << GV->getName()
-                         << " has disallowed visibility: " << Text << "\n";
-  }
-  if (GV->hasSection()) {
-    Reporter->addError() << GVTypeName << " " << GV->getName() <<
-        " has disallowed \"section\" attribute\n";
-  }
-  if (GV->getType()->getAddressSpace() != 0) {
-    Reporter->addError() << GVTypeName << " " << GV->getName()
-                         << " has addrspace attribute (disallowed)\n";
-  }
-  // The "unnamed_addr" attribute can be used to merge duplicate
-  // definitions, but that should be done by user-toolchain
-  // optimization passes, not by the PNaCl translator.
-  if (GV->hasUnnamedAddr()) {
-    Reporter->addError() << GVTypeName << " " << GV->getName()
-                         << " has disallowed \"unnamed_addr\" attribute\n";
-  }
-}
-
-void PNaClABIVerifyModule::checkExternalSymbol(const GlobalValue *GV) {
-  if (const Function *Func = dyn_cast<const Function>(GV)) {
-    if (Func->isIntrinsic() || isAllowedMinsfiSyscall(Func))
-      return;
-  }
-
-  // We only allow __pnacl_pso_root to be a variable, not a function, to
-  // reduce the number of cases that the translator needs to handle.
-  bool ValidEntry =
-      (isa<Function>(GV) && GV->getName().equals("_start")) ||
-      (isa<GlobalVariable>(GV) && GV->getName().equals("__pnacl_pso_root"));
-  if (!ValidEntry) {
-    Reporter->addError()
-        << GV->getName()
-        << " is not a valid external symbol (disallowed)\n";
-  } else {
-    if (SeenEntryPoint) {
-      Reporter->addError() <<
-          "Module has multiple entry points (disallowed)\n";
-    }
-    SeenEntryPoint = true;
-  }
-}
-
-static bool isPtrToIntOfGlobal(const Constant *C) {
-  if (const ConstantExpr *CE = dyn_cast<ConstantExpr>(C)) {
-    return CE->getOpcode() == Instruction::PtrToInt &&
-           isa<GlobalValue>(CE->getOperand(0));
-  }
-  return false;
-}
-
-// This checks for part of the normal form produced by FlattenGlobals.
-static bool isSimpleElement(const Constant *C) {
-  // A SimpleElement is one of the following:
-  // 1) An i8 array literal or zeroinitializer:
-  //      [SIZE x i8] c"DATA"
-  //      [SIZE x i8] zeroinitializer
-  if (ArrayType *Ty = dyn_cast<ArrayType>(C->getType())) {
-    return Ty->getElementType()->isIntegerTy(8) &&
-           (isa<ConstantAggregateZero>(C) ||
-            isa<ConstantDataArray>(C));
-  }
-  // 2) A reference to a GlobalValue (a function or global variable)
-  //    with an optional byte offset added to it (the addend).
-  if (C->getType()->isIntegerTy(32)) {
-    const ConstantExpr *CE = dyn_cast<ConstantExpr>(C);
-    if (!CE)
-      return false;
-    // Without addend:  ptrtoint (TYPE* @GLOBAL to i32)
-    if (isPtrToIntOfGlobal(CE))
-      return true;
-    // With addend:  add (i32 ptrtoint (TYPE* @GLOBAL to i32), i32 ADDEND)
-    if (CE->getOpcode() == Instruction::Add &&
-        isPtrToIntOfGlobal(CE->getOperand(0)) &&
-        isa<ConstantInt>(CE->getOperand(1)))
-      return true;
-  }
-  return false;
-}
-
-// This checks for part of the normal form produced by FlattenGlobals.
-static bool isCompoundElement(const Constant *C) {
-  const ConstantStruct *CS = dyn_cast<ConstantStruct>(C);
-  if (!CS || !CS->getType()->isPacked() || CS->getType()->hasName() ||
-      CS->getNumOperands() <= 1)
-    return false;
-  for (unsigned I = 0; I < CS->getNumOperands(); ++I) {
-    if (!isSimpleElement(CS->getOperand(I)))
-      return false;
-  }
-  return true;
-}
-
-static std::string getAttributesAsString(AttributeSet Attrs) {
-  std::string AttrsAsString;
-  for (unsigned Slot = 0; Slot < Attrs.getNumSlots(); ++Slot) {
-    for (AttributeSet::iterator Attr = Attrs.begin(Slot),
-           E = Attrs.end(Slot); Attr != E; ++Attr) {
-      AttrsAsString += " ";
-      AttrsAsString += Attr->getAsString();
-    }
-  }
-  return AttrsAsString;
-}
-
-// This checks that the GlobalVariable has the normal form produced by
-// the FlattenGlobals pass.
-void PNaClABIVerifyModule::checkGlobalIsFlattened(const GlobalVariable *GV) {
-  if (!GV->hasInitializer()) {
-    Reporter->addError() << "Global variable " << GV->getName()
-                         << " has no initializer (disallowed)\n";
-    return;
-  }
-  const Constant *InitVal = GV->getInitializer();
-  if (isSimpleElement(InitVal) || isCompoundElement(InitVal))
-    return;
-  Reporter->addError() << "Global variable " << GV->getName()
-                       << " has non-flattened initializer (disallowed): "
-                       << *InitVal << "\n";
-}
-
-void PNaClABIVerifyModule::checkFunction(const Function *F,
-                                         const StringRef &Name,
-                                         PNaClAllowedIntrinsics &Intrinsics) {
-  if (F->isIntrinsic()) {
-    // Check intrinsics.
-    if (!Intrinsics.isAllowed(F)) {
-      Reporter->addError() << "Function " << F->getName()
-                           << " is a disallowed LLVM intrinsic\n";
-    }
-  } else {
-    // Check types of functions and their arguments.  Not necessary
-    // for intrinsics, whose types are fixed anyway, and which have
-    // argument types that we disallow such as i8.
-    if (!PNaClABITypeChecker::isValidFunctionType(F->getFunctionType())) {
-      Reporter->addError()
-          << "Function " << Name << " has disallowed type: "
-          << PNaClABITypeChecker::getTypeName(F->getFunctionType())
-          << "\n";
-    }
-    // This check is disabled in streaming mode because it would
-    // reject a function that is defined but not read in yet.
-    // Unfortunately this means we simply don't check this property
-    // when translating a pexe in the browser.
-    // TODO(mseaborn): Enforce this property in the bitcode reader.
-    if (!StreamingMode && F->isDeclaration() && !isAllowedMinsfiSyscall(F)) {
-      Reporter->addError() << "Function " << Name
-                           << " is declared but not defined (disallowed)\n";
-    }
-    if (!F->getAttributes().isEmpty()) {
-      Reporter->addError()
-          << "Function " << Name << " has disallowed attributes:"
-          << getAttributesAsString(F->getAttributes()) << "\n";
-    }
-    if (!PNaClABIProps::isValidCallingConv(F->getCallingConv())) {
-      Reporter->addError()
-          << "Function " << Name << " has disallowed calling convention: "
-          << PNaClABIProps::CallingConvName(F->getCallingConv()) << " ("
-          << F->getCallingConv() << ")\n";
-    }
-  }
-
-  checkGlobalValue(F);
-
-  if (F->hasGC()) {
-    Reporter->addError() << "Function " << Name <<
-        " has disallowed \"gc\" attribute\n";
-  }
-  // Knowledge of what function alignments are useful is
-  // architecture-specific and sandbox-specific, so PNaCl pexes
-  // should not be able to specify function alignment.
-  if (F->getAlignment() != 0) {
-    Reporter->addError() << "Function " << Name <<
-        " has disallowed \"align\" attribute\n";
-  }
-}
-
-bool PNaClABIVerifyModule::runOnModule(Module &M) {
-  SeenEntryPoint = false;
-  PNaClAllowedIntrinsics Intrinsics(&M.getContext());
-
-  if (!M.getModuleInlineAsm().empty()) {
-    Reporter->addError() <<
-        "Module contains disallowed top-level inline assembly\n";
-  }
-
-  for (Module::const_global_iterator MI = M.global_begin(), ME = M.global_end();
-       MI != ME; ++MI) {
-    checkGlobalIsFlattened(MI);
-    checkGlobalVariable(MI);
-
-    if (MI->isThreadLocal()) {
-      Reporter->addError() << "Variable " << MI->getName() <<
-          " has disallowed \"thread_local\" attribute\n";
-    }
-    if (MI->isExternallyInitialized()) {
-      Reporter->addError() << "Variable " << MI->getName() <<
-          " has disallowed \"externally_initialized\" attribute\n";
-    }
-  }
-
-  // No aliases allowed for now.
-  for (Module::alias_iterator MI = M.alias_begin(),
-           E = M.alias_end(); MI != E; ++MI) {
-    Reporter->addError() << "Variable " << MI->getName() <<
-        " is an alias (disallowed)\n";
-  }
-
-  for (Module::const_iterator MI = M.begin(), ME = M.end(); MI != ME; ++MI) {
-    checkFunction(MI, MI->getName(), Intrinsics);
-  }
-
-  // Check named metadata nodes
-  for (Module::const_named_metadata_iterator I = M.named_metadata_begin(),
-           E = M.named_metadata_end(); I != E; ++I) {
-    if (!PNaClABIProps::isWhitelistedMetadata(I)) {
-      Reporter->addError() << "Named metadata node " << I->getName()
-                           << " is disallowed\n";
-    }
-  }
-
-  if (!SeenEntryPoint) {
-    Reporter->addError() << "Module has no entry point (disallowed)\n";
-  }
-  Reporter->checkForFatalErrors();
-  return false;
-}
-
-// This method exists so that the passes can easily be run with opt -analyze.
-// In this case the default constructor is used and we want to reset the error
-// messages after each print (this is more of an issue for the FunctionPass
-// than the ModulePass)
-void PNaClABIVerifyModule::print(llvm::raw_ostream &O, const Module *M) const {
-  Reporter->printErrors(O);
-  Reporter->reset();
-}
-
-char PNaClABIVerifyModule::ID = 0;
-INITIALIZE_PASS(PNaClABIVerifyModule, "verify-pnaclabi-module",
-                "Verify module for PNaCl", false, true)
-
-ModulePass *llvm::createPNaClABIVerifyModulePass(
-    PNaClABIErrorReporter *Reporter, bool StreamingMode) {
-  return new PNaClABIVerifyModule(Reporter, StreamingMode);
-}
diff --git a/lib/Analysis/NaCl/PNaClAllowedIntrinsics.cpp b/lib/Analysis/NaCl/PNaClAllowedIntrinsics.cpp
deleted file mode 100644
index e895269f8ba..00000000000
--- a/lib/Analysis/NaCl/PNaClAllowedIntrinsics.cpp
+++ /dev/null
@@ -1,191 +0,0 @@
-//===- PNaClAllowedIntrinsics.cpp - Set of allowed intrinsics -------------===//
-//
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
-//
-//===----------------------------------------------------------------------===//
-//
-// Defines implementation of  class that holds set of allowed intrinsics.
-//
-// Keep 3 categories of intrinsics for now.
-// (1) Allowed always, provided the exact name and type match.
-// (2) Never allowed.
-// (3) Debug info intrinsics.
-//
-//===----------------------------------------------------------------------===//
-
-#include "llvm/Analysis/NaCl/PNaClAllowedIntrinsics.h"
-
-#include "llvm/ADT/STLExtras.h"
-#include "llvm/Analysis/NaCl.h"
-#include "llvm/IR/Function.h"
-#include "llvm/IR/Type.h"
-
-using namespace llvm;
-
-/*
-  The constructor sets up the whitelist of allowed intrinsics and their expected
-  types.  The comments in that code have some details on the allowed intrinsics.
-  Additionally, the following intrinsics are disallowed for the stated reasons:
-
-  * Trampolines depend on a target-specific-sized/aligned buffer.
-    Intrinsic::adjust_trampoline:
-    Intrinsic::init_trampoline:
-  * CXX exception handling is not stable.
-    Intrinsic::eh_dwarf_cfa:
-    Intrinsic::eh_return_i32:
-    Intrinsic::eh_return_i64:
-    Intrinsic::eh_sjlj_callsite:
-    Intrinsic::eh_sjlj_functioncontext:
-    Intrinsic::eh_sjlj_longjmp:
-    Intrinsic::eh_sjlj_lsda:
-    Intrinsic::eh_sjlj_setjmp:
-    Intrinsic::eh_typeid_for:
-    Intrinsic::eh_unwind_init:
-  * We do not want to expose addresses to the user.
-    Intrinsic::frameaddress:
-    Intrinsic::returnaddress:
-  * We do not support stack protectors.
-    Intrinsic::stackprotector:
-  * Var-args handling is done w/out intrinsics.
-    Intrinsic::vacopy:
-    Intrinsic::vaend:
-    Intrinsic::vastart:
-  * Disallow the *_with_overflow intrinsics because they return
-    struct types.  All of them can be introduced by passing -ftrapv
-    to Clang, which we do not support for now.  umul_with_overflow
-    and uadd_with_overflow are introduced by Clang for C++'s new[],
-    but ExpandArithWithOverflow expands out this use.
-    Intrinsic::sadd_with_overflow:
-    Intrinsic::ssub_with_overflow:
-    Intrinsic::uadd_with_overflow:
-    Intrinsic::usub_with_overflow:
-    Intrinsic::smul_with_overflow:
-    Intrinsic::umul_with_overflow:
-  * Disallow lifetime.start/end because the semantics of what
-    arguments they accept are not very well defined, and because it
-    would be better to do merging of stack slots in the user
-    toolchain than in the PNaCl translator.
-    See https://code.google.com/p/nativeclient/issues/detail?id=3443
-    Intrinsic::lifetime_end:
-    Intrinsic::lifetime_start:
-    Intrinsic::invariant_end:
-    Intrinsic::invariant_start:
-  * Some transcendental functions not needed yet.
-    Intrinsic::cos:
-    Intrinsic::exp:
-    Intrinsic::exp2:
-    Intrinsic::log:
-    Intrinsic::log2:
-    Intrinsic::log10:
-    Intrinsic::pow:
-    Intrinsic::powi:
-    Intrinsic::sin:
-  * We run -lower-expect to convert Intrinsic::expect into branch weights
-    and consume in the middle-end. The backend just ignores llvm.expect.
-    Intrinsic::expect:
-  * For FLT_ROUNDS macro from float.h. It works for ARM and X86
-    (but not MIPS). Also, wait until we add a set_flt_rounds intrinsic
-    before we bless this.
-    case Intrinsic::flt_rounds:
-*/
-PNaClAllowedIntrinsics::
-PNaClAllowedIntrinsics(LLVMContext *Context) : Context(Context) {
-  Type *I8Ptr = Type::getInt8PtrTy(*Context);
-  Type *I8 = Type::getInt8Ty(*Context);
-  Type *I16 = Type::getInt16Ty(*Context);
-  Type *I32 = Type::getInt32Ty(*Context);
-  Type *I64 = Type::getInt64Ty(*Context);
-  Type *Float = Type::getFloatTy(*Context);
-  Type *Double = Type::getDoubleTy(*Context);
-  Type *Vec4Float = VectorType::get(Float, 4);
-
-  // We accept bswap for a limited set of types (i16, i32, i64).  The
-  // various backends are able to generate instructions to implement
-  // the intrinsic.  Also, i16 and i64 are easy to implement as along
-  // as there is a way to do i32.
-  addIntrinsic(Intrinsic::bswap, I16);
-  addIntrinsic(Intrinsic::bswap, I32);
-  addIntrinsic(Intrinsic::bswap, I64);
-
-  // We accept cttz, ctlz, and ctpop for a limited set of types (i32, i64).
-  addIntrinsic(Intrinsic::ctlz, I32);
-  addIntrinsic(Intrinsic::ctlz, I64);
-  addIntrinsic(Intrinsic::cttz, I32);
-  addIntrinsic(Intrinsic::cttz, I64);
-  addIntrinsic(Intrinsic::ctpop, I32);
-  addIntrinsic(Intrinsic::ctpop, I64);
-
-  addIntrinsic(Intrinsic::nacl_read_tp);
-  addIntrinsic(Intrinsic::nacl_longjmp);
-  addIntrinsic(Intrinsic::nacl_setjmp);
-
-  addIntrinsic(Intrinsic::fabs, Float);
-  addIntrinsic(Intrinsic::fabs, Double);
-  addIntrinsic(Intrinsic::fabs, Vec4Float);
-
-  // For native sqrt instructions. Must guarantee when x < -0.0, sqrt(x) = NaN.
-  addIntrinsic(Intrinsic::sqrt, Float);
-  addIntrinsic(Intrinsic::sqrt, Double);
-
-  Type *AtomicTypes[] = { I8, I16, I32, I64 };
-  for (size_t T = 0, E = array_lengthof(AtomicTypes); T != E; ++T) {
-    addIntrinsic(Intrinsic::nacl_atomic_load, AtomicTypes[T]);
-    addIntrinsic(Intrinsic::nacl_atomic_store, AtomicTypes[T]);
-    addIntrinsic(Intrinsic::nacl_atomic_rmw, AtomicTypes[T]);
-    addIntrinsic(Intrinsic::nacl_atomic_cmpxchg, AtomicTypes[T]);
-  }
-  addIntrinsic(Intrinsic::nacl_atomic_fence);
-  addIntrinsic(Intrinsic::nacl_atomic_fence_all);
-
-  addIntrinsic(Intrinsic::nacl_atomic_is_lock_free);
-
-  // Stack save and restore are used to support C99 VLAs.
-  addIntrinsic(Intrinsic::stacksave);
-  addIntrinsic(Intrinsic::stackrestore);
-
-  addIntrinsic(Intrinsic::trap);
-
-  // We only allow the variants of memcpy/memmove/memset with an i32
-  // "len" argument, not an i64 argument.
-  Type *MemcpyTypes[] = { I8Ptr, I8Ptr, I32 };
-  addIntrinsic(Intrinsic::memcpy, MemcpyTypes);
-  addIntrinsic(Intrinsic::memmove, MemcpyTypes);
-  Type *MemsetTypes[] = { I8Ptr, I32 };
-  addIntrinsic(Intrinsic::memset, MemsetTypes);
-}
-
-void PNaClAllowedIntrinsics::addIntrinsic(Intrinsic::ID ID,
-                                          ArrayRef<Type *> Tys) {
-  std::string Name = Intrinsic::getName(ID, Tys);
-  FunctionType *FcnType = Intrinsic::getType(*Context, ID, Tys);
-  if (TypeMap.count(Name) >= 1) {
-    std::string Buffer;
-    raw_string_ostream StrBuf(Buffer);
-    StrBuf << "Instrinsic " << Name << " defined with multiple types: "
-           << *TypeMap[Name] << " and " << *FcnType << "\n";
-    report_fatal_error(StrBuf.str());
-  }
-  TypeMap[Name] = FcnType;
-}
-
-bool PNaClAllowedIntrinsics::isAllowed(const Function *Func) {
-  if (isIntrinsicName(Func->getName()))
-    return Func->getFunctionType() == TypeMap[Func->getName()];
-  // Check to see if debugging intrinsic, which can be allowed if
-  // command-line flag set.
-  return isAllowedDebugInfoIntrinsic(Func->getIntrinsicID());
-}
-
-bool PNaClAllowedIntrinsics::isAllowedDebugInfoIntrinsic(unsigned IntrinsicID) {
-  /* These intrinsics are allowed when debug info metadata is also allowed,
-     and we just assume that they are called correctly by the frontend. */
-  switch (IntrinsicID) {
-    default: return false;
-    case Intrinsic::dbg_declare:
-    case Intrinsic::dbg_value:
-      return PNaClABIAllowDebugMetadata;
-  }
-}
diff --git a/lib/Analysis/NoAliasAnalysis.cpp b/lib/Analysis/NoAliasAnalysis.cpp
index 203e1daf7a0..aa78ceab89c 100644
--- a/lib/Analysis/NoAliasAnalysis.cpp
+++ b/lib/Analysis/NoAliasAnalysis.cpp
@@ -41,41 +41,36 @@ namespace {
       return true;
     }
 
-    AliasResult alias(const Location &LocA, const Location &LocB) override {
+    AliasResult alias(const MemoryLocation &LocA,
+                      const MemoryLocation &LocB) override {
       return MayAlias;
     }
 
-    ModRefBehavior getModRefBehavior(ImmutableCallSite CS) override {
-      return UnknownModRefBehavior;
+    FunctionModRefBehavior getModRefBehavior(ImmutableCallSite CS) override {
+      return FMRB_UnknownModRefBehavior;
     }
-    ModRefBehavior getModRefBehavior(const Function *F) override {
-      return UnknownModRefBehavior;
+    FunctionModRefBehavior getModRefBehavior(const Function *F) override {
+      return FMRB_UnknownModRefBehavior;
     }
 
-    bool pointsToConstantMemory(const Location &Loc, bool OrLocal) override {
+    bool pointsToConstantMemory(const MemoryLocation &Loc,
+                                bool OrLocal) override {
       return false;
     }
-    Location getArgLocation(ImmutableCallSite CS, unsigned ArgIdx,
-                            ModRefResult &Mask) override {
-      Mask = ModRef;
-      AAMDNodes AATags;
-      CS->getAAMetadata(AATags);
-      return Location(CS.getArgument(ArgIdx), UnknownSize, AATags);
+    ModRefInfo getArgModRefInfo(ImmutableCallSite CS,
+                                unsigned ArgIdx) override {
+      return MRI_ModRef;
     }
 
-    ModRefResult getModRefInfo(ImmutableCallSite CS,
-                               const Location &Loc) override {
-      return ModRef;
+    ModRefInfo getModRefInfo(ImmutableCallSite CS,
+                             const MemoryLocation &Loc) override {
+      return MRI_ModRef;
     }
-    ModRefResult getModRefInfo(ImmutableCallSite CS1,
-                               ImmutableCallSite CS2) override {
-      return ModRef;
+    ModRefInfo getModRefInfo(ImmutableCallSite CS1,
+                             ImmutableCallSite CS2) override {
+      return MRI_ModRef;
     }
 
-    void deleteValue(Value *V) override {}
-    void copyValue(Value *From, Value *To) override {}
-    void addEscapingUse(Use &U) override {}
-
     /// getAdjustedAnalysisPointer - This method is used when a pass implements
     /// an analysis interface through multiple inheritance.  If needed, it
     /// should override this to adjust the this pointer as needed for the
diff --git a/lib/Analysis/OrderedBasicBlock.cpp b/lib/Analysis/OrderedBasicBlock.cpp
new file mode 100644
index 00000000000..0f0016f22cc
--- /dev/null
+++ b/lib/Analysis/OrderedBasicBlock.cpp
@@ -0,0 +1,85 @@
+//===- OrderedBasicBlock.cpp --------------------------------- -*- C++ -*-===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This file implements the OrderedBasicBlock class. OrderedBasicBlock
+// maintains an interface where clients can query if one instruction comes
+// before another in a BasicBlock. Since BasicBlock currently lacks a reliable
+// way to query relative position between instructions one can use
+// OrderedBasicBlock to do such queries. OrderedBasicBlock is lazily built on a
+// source BasicBlock and maintains an internal Instruction -> Position map. A
+// OrderedBasicBlock instance should be discarded whenever the source
+// BasicBlock changes.
+//
+// It's currently used by the CaptureTracker in order to find relative
+// positions of a pair of instructions inside a BasicBlock.
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/Analysis/OrderedBasicBlock.h"
+#include "llvm/IR/Instruction.h"
+using namespace llvm;
+
+OrderedBasicBlock::OrderedBasicBlock(const BasicBlock *BasicB)
+    : NextInstPos(0), BB(BasicB) {
+  LastInstFound = BB->end();
+}
+
+/// \brief Given no cached results, find if \p A comes before \p B in \p BB.
+/// Cache and number out instruction while walking \p BB.
+bool OrderedBasicBlock::comesBefore(const Instruction *A,
+                                    const Instruction *B) {
+  const Instruction *Inst = nullptr;
+  assert(!(LastInstFound == BB->end() && NextInstPos != 0) &&
+         "Instruction supposed to be in NumberedInsts");
+
+  // Start the search with the instruction found in the last lookup round.
+  auto II = BB->begin();
+  auto IE = BB->end();
+  if (LastInstFound != IE)
+    II = std::next(LastInstFound);
+
+  // Number all instructions up to the point where we find 'A' or 'B'.
+  for (; II != IE; ++II) {
+    Inst = cast<Instruction>(II);
+    NumberedInsts[Inst] = NextInstPos++;
+    if (Inst == A || Inst == B)
+      break;
+  }
+
+  assert(II != IE && "Instruction not found?");
+  assert((Inst == A || Inst == B) && "Should find A or B");
+  LastInstFound = II;
+  return Inst == A;
+}
+
+/// \brief Find out whether \p A dominates \p B, meaning whether \p A
+/// comes before \p B in \p BB. This is a simplification that considers
+/// cached instruction positions and ignores other basic blocks, being
+/// only relevant to compare relative instructions positions inside \p BB.
+bool OrderedBasicBlock::dominates(const Instruction *A, const Instruction *B) {
+  assert(A->getParent() == B->getParent() &&
+         "Instructions must be in the same basic block!");
+
+  // First we lookup the instructions. If they don't exist, lookup will give us
+  // back ::end(). If they both exist, we compare the numbers. Otherwise, if NA
+  // exists and NB doesn't, it means NA must come before NB because we would
+  // have numbered NB as well if it didn't. The same is true for NB. If it
+  // exists, but NA does not, NA must come after it. If neither exist, we need
+  // to number the block and cache the results (by calling comesBefore).
+  auto NAI = NumberedInsts.find(A);
+  auto NBI = NumberedInsts.find(B);
+  if (NAI != NumberedInsts.end() && NBI != NumberedInsts.end())
+    return NAI->second < NBI->second;
+  if (NAI != NumberedInsts.end())
+    return true;
+  if (NBI != NumberedInsts.end())
+    return false;
+
+  return comesBefore(A, B);
+}
diff --git a/lib/Analysis/PHITransAddr.cpp b/lib/Analysis/PHITransAddr.cpp
index 177684fdc9a..8d80c6028ba 100644
--- a/lib/Analysis/PHITransAddr.cpp
+++ b/lib/Analysis/PHITransAddr.cpp
@@ -150,7 +150,8 @@ Value *PHITransAddr::PHITranslateSubExpr(Value *V, BasicBlock *CurBB,
   if (!Inst) return V;
 
   // Determine whether 'Inst' is an input to our PHI translatable expression.
-  bool isInput = std::count(InstInputs.begin(), InstInputs.end(), Inst);
+  bool isInput =
+      std::find(InstInputs.begin(), InstInputs.end(), Inst) != InstInputs.end();
 
   // Handle inputs instructions if needed.
   if (isInput) {
@@ -243,13 +244,7 @@ Value *PHITransAddr::PHITranslateSubExpr(Value *V, BasicBlock *CurBB,
             GEPI->getNumOperands() == GEPOps.size() &&
             GEPI->getParent()->getParent() == CurBB->getParent() &&
             (!DT || DT->dominates(GEPI->getParent(), PredBB))) {
-          bool Mismatch = false;
-          for (unsigned i = 0, e = GEPOps.size(); i != e; ++i)
-            if (GEPI->getOperand(i) != GEPOps[i]) {
-              Mismatch = true;
-              break;
-            }
-          if (!Mismatch)
+          if (std::equal(GEPOps.begin(), GEPOps.end(), GEPI->op_begin()))
             return GEPI;
         }
     }
@@ -276,7 +271,8 @@ Value *PHITransAddr::PHITranslateSubExpr(Value *V, BasicBlock *CurBB,
           isNSW = isNUW = false;
 
           // If the old 'LHS' was an input, add the new 'LHS' as an input.
-          if (std::count(InstInputs.begin(), InstInputs.end(), BOp)) {
+          if (std::find(InstInputs.begin(), InstInputs.end(), BOp) !=
+              InstInputs.end()) {
             RemoveInstInputs(BOp, InstInputs);
             AddAsInput(LHS);
           }
@@ -313,21 +309,26 @@ Value *PHITransAddr::PHITranslateSubExpr(Value *V, BasicBlock *CurBB,
 
 
 /// PHITranslateValue - PHI translate the current address up the CFG from
-/// CurBB to Pred, updating our state to reflect any needed changes.  If the
-/// dominator tree DT is non-null, the translated value must dominate
+/// CurBB to Pred, updating our state to reflect any needed changes.  If
+/// 'MustDominate' is true, the translated value must dominate
 /// PredBB.  This returns true on failure and sets Addr to null.
 bool PHITransAddr::PHITranslateValue(BasicBlock *CurBB, BasicBlock *PredBB,
-                                     const DominatorTree *DT) {
+                                     const DominatorTree *DT,
+                                     bool MustDominate) {
+  assert(DT || !MustDominate);
   assert(Verify() && "Invalid PHITransAddr!");
-  Addr = PHITranslateSubExpr(Addr, CurBB, PredBB, DT);
+  if (DT && DT->isReachableFromEntry(PredBB))
+    Addr =
+        PHITranslateSubExpr(Addr, CurBB, PredBB, MustDominate ? DT : nullptr);
+  else
+    Addr = nullptr;
   assert(Verify() && "Invalid PHITransAddr!");
 
-  if (DT) {
+  if (MustDominate)
     // Make sure the value is live in the predecessor.
     if (Instruction *Inst = dyn_cast_or_null<Instruction>(Addr))
       if (!DT->dominates(Inst->getParent(), PredBB))
         Addr = nullptr;
-  }
 
   return Addr == nullptr;
 }
@@ -370,7 +371,7 @@ InsertPHITranslatedSubExpr(Value *InVal, BasicBlock *CurBB,
   // See if we have a version of this value already available and dominating
   // PredBB.  If so, there is no need to insert a new instance of it.
   PHITransAddr Tmp(InVal, DL, AC);
-  if (!Tmp.PHITranslateValue(CurBB, PredBB, &DT))
+  if (!Tmp.PHITranslateValue(CurBB, PredBB, &DT, /*MustDominate=*/true))
     return Tmp.getAddr();
 
   // If we don't have an available version of this value, it must be an
@@ -385,10 +386,10 @@ InsertPHITranslatedSubExpr(Value *InVal, BasicBlock *CurBB,
     if (!OpVal) return nullptr;
 
     // Otherwise insert a cast at the end of PredBB.
-    CastInst *New = CastInst::Create(Cast->getOpcode(),
-                                     OpVal, InVal->getType(),
-                                     InVal->getName()+".phi.trans.insert",
+    CastInst *New = CastInst::Create(Cast->getOpcode(), OpVal, InVal->getType(),
+                                     InVal->getName() + ".phi.trans.insert",
                                      PredBB->getTerminator());
+    New->setDebugLoc(Inst->getDebugLoc());
     NewInsts.push_back(New);
     return New;
   }
@@ -407,6 +408,7 @@ InsertPHITranslatedSubExpr(Value *InVal, BasicBlock *CurBB,
     GetElementPtrInst *Result = GetElementPtrInst::Create(
         GEP->getSourceElementType(), GEPOps[0], makeArrayRef(GEPOps).slice(1),
         InVal->getName() + ".phi.trans.insert", PredBB->getTerminator());
+    Result->setDebugLoc(Inst->getDebugLoc());
     Result->setIsInBounds(GEP->isInBounds());
     NewInsts.push_back(Result);
     return Result;
diff --git a/lib/Analysis/ScalarEvolution.cpp b/lib/Analysis/ScalarEvolution.cpp
index 37377f0b859..efeddf07db2 100644
--- a/lib/Analysis/ScalarEvolution.cpp
+++ b/lib/Analysis/ScalarEvolution.cpp
@@ -726,6 +726,13 @@ struct SCEVDivision : public SCEVVisitor<SCEVDivision, void> {
       return;
     }
 
+    // A simple case when N/1. The quotient is N.
+    if (Denominator->isOne()) {
+      *Quotient = Numerator;
+      *Remainder = D.Zero;
+      return;
+    }
+
     // Split the Denominator when it is a product.
     if (const SCEVMulExpr *T = dyn_cast<const SCEVMulExpr>(Denominator)) {
       const SCEV *Q, *R;
@@ -788,6 +795,14 @@ struct SCEVDivision : public SCEVVisitor<SCEVDivision, void> {
     assert(Numerator->isAffine() && "Numerator should be affine");
     divide(SE, Numerator->getStart(), Denominator, &StartQ, &StartR);
     divide(SE, Numerator->getStepRecurrence(SE), Denominator, &StepQ, &StepR);
+    // Bail out if the types do not match.
+    Type *Ty = Denominator->getType();
+    if (Ty != StartQ->getType() || Ty != StartR->getType() ||
+        Ty != StepQ->getType() || Ty != StepR->getType()) {
+      Quotient = Zero;
+      Remainder = Numerator;
+      return;
+    }
     Quotient = SE.getAddRecExpr(StartQ, StepQ, Numerator->getLoop(),
                                 Numerator->getNoWrapFlags());
     Remainder = SE.getAddRecExpr(StartR, StepR, Numerator->getLoop(),
@@ -2911,6 +2926,57 @@ ScalarEvolution::getAddRecExpr(SmallVectorImpl<const SCEV *> &Operands,
   return S;
 }
 
+const SCEV *
+ScalarEvolution::getGEPExpr(Type *PointeeType, const SCEV *BaseExpr,
+                            const SmallVectorImpl<const SCEV *> &IndexExprs,
+                            bool InBounds) {
+  // getSCEV(Base)->getType() has the same address space as Base->getType()
+  // because SCEV::getType() preserves the address space.
+  Type *IntPtrTy = getEffectiveSCEVType(BaseExpr->getType());
+  // FIXME(PR23527): Don't blindly transfer the inbounds flag from the GEP
+  // instruction to its SCEV, because the Instruction may be guarded by control
+  // flow and the no-overflow bits may not be valid for the expression in any
+  // context. This can be fixed similarly to how these flags are handled for
+  // adds.
+  SCEV::NoWrapFlags Wrap = InBounds ? SCEV::FlagNSW : SCEV::FlagAnyWrap;
+
+  const SCEV *TotalOffset = getConstant(IntPtrTy, 0);
+  // The address space is unimportant. The first thing we do on CurTy is getting
+  // its element type.
+  Type *CurTy = PointerType::getUnqual(PointeeType);
+  for (const SCEV *IndexExpr : IndexExprs) {
+    // Compute the (potentially symbolic) offset in bytes for this index.
+    if (StructType *STy = dyn_cast<StructType>(CurTy)) {
+      // For a struct, add the member offset.
+      ConstantInt *Index = cast<SCEVConstant>(IndexExpr)->getValue();
+      unsigned FieldNo = Index->getZExtValue();
+      const SCEV *FieldOffset = getOffsetOfExpr(IntPtrTy, STy, FieldNo);
+
+      // Add the field offset to the running total offset.
+      TotalOffset = getAddExpr(TotalOffset, FieldOffset);
+
+      // Update CurTy to the type of the field at Index.
+      CurTy = STy->getTypeAtIndex(Index);
+    } else {
+      // Update CurTy to its element type.
+      CurTy = cast<SequentialType>(CurTy)->getElementType();
+      // For an array, add the element offset, explicitly scaled.
+      const SCEV *ElementSize = getSizeOfExpr(IntPtrTy, CurTy);
+      // Getelementptr indices are signed.
+      IndexExpr = getTruncateOrSignExtend(IndexExpr, IntPtrTy);
+
+      // Multiply the index by the element size to compute the element offset.
+      const SCEV *LocalOffset = getMulExpr(IndexExpr, ElementSize, Wrap);
+
+      // Add the element offset to the running total offset.
+      TotalOffset = getAddExpr(TotalOffset, LocalOffset);
+    }
+  }
+
+  // Add the total offset from all the GEP indices to the base.
+  return getAddExpr(BaseExpr, TotalOffset, Wrap);
+}
+
 const SCEV *ScalarEvolution::getSMaxExpr(const SCEV *LHS,
                                          const SCEV *RHS) {
   SmallVector<const SCEV *, 2> Ops;
@@ -3250,22 +3316,25 @@ bool ScalarEvolution::checkValidity(const SCEV *S) const {
 const SCEV *ScalarEvolution::getSCEV(Value *V) {
   assert(isSCEVable(V->getType()) && "Value is not SCEVable!");
 
+  const SCEV *S = getExistingSCEV(V);
+  if (S == nullptr) {
+    S = createSCEV(V);
+    ValueExprMap.insert(std::make_pair(SCEVCallbackVH(V, this), S));
+  }
+  return S;
+}
+
+const SCEV *ScalarEvolution::getExistingSCEV(Value *V) {
+  assert(isSCEVable(V->getType()) && "Value is not SCEVable!");
+
   ValueExprMapType::iterator I = ValueExprMap.find_as(V);
   if (I != ValueExprMap.end()) {
     const SCEV *S = I->second;
     if (checkValidity(S))
       return S;
-    else
-      ValueExprMap.erase(I);
+    ValueExprMap.erase(I);
   }
-  const SCEV *S = createSCEV(V);
-
-  // The process of creating a SCEV for V may have caused other SCEVs
-  // to have been created, so it's necessary to insert the new entry
-  // from scratch, rather than trying to remember the insert position
-  // above.
-  ValueExprMap.insert(std::make_pair(SCEVCallbackVH(V, this), S));
-  return S;
+  return nullptr;
 }
 
 /// getNegativeSCEV - Return a SCEV corresponding to -V = -1*V
@@ -3685,52 +3754,16 @@ const SCEV *ScalarEvolution::createNodeForPHI(PHINode *PN) {
 /// operations. This allows them to be analyzed by regular SCEV code.
 ///
 const SCEV *ScalarEvolution::createNodeForGEP(GEPOperator *GEP) {
-  Type *IntPtrTy = getEffectiveSCEVType(GEP->getType());
   Value *Base = GEP->getOperand(0);
   // Don't attempt to analyze GEPs over unsized objects.
   if (!Base->getType()->getPointerElementType()->isSized())
     return getUnknown(GEP);
 
-  // Don't blindly transfer the inbounds flag from the GEP instruction to the
-  // Add expression, because the Instruction may be guarded by control flow
-  // and the no-overflow bits may not be valid for the expression in any
-  // context.
-  SCEV::NoWrapFlags Wrap = GEP->isInBounds() ? SCEV::FlagNSW : SCEV::FlagAnyWrap;
-
-  const SCEV *TotalOffset = getConstant(IntPtrTy, 0);
-  gep_type_iterator GTI = gep_type_begin(GEP);
-  for (GetElementPtrInst::op_iterator I = std::next(GEP->op_begin()),
-                                      E = GEP->op_end();
-       I != E; ++I) {
-    Value *Index = *I;
-    // Compute the (potentially symbolic) offset in bytes for this index.
-    if (StructType *STy = dyn_cast<StructType>(*GTI++)) {
-      // For a struct, add the member offset.
-      unsigned FieldNo = cast<ConstantInt>(Index)->getZExtValue();
-      const SCEV *FieldOffset = getOffsetOfExpr(IntPtrTy, STy, FieldNo);
-
-      // Add the field offset to the running total offset.
-      TotalOffset = getAddExpr(TotalOffset, FieldOffset);
-    } else {
-      // For an array, add the element offset, explicitly scaled.
-      const SCEV *ElementSize = getSizeOfExpr(IntPtrTy, *GTI);
-      const SCEV *IndexS = getSCEV(Index);
-      // Getelementptr indices are signed.
-      IndexS = getTruncateOrSignExtend(IndexS, IntPtrTy);
-
-      // Multiply the index by the element size to compute the element offset.
-      const SCEV *LocalOffset = getMulExpr(IndexS, ElementSize, Wrap);
-
-      // Add the element offset to the running total offset.
-      TotalOffset = getAddExpr(TotalOffset, LocalOffset);
-    }
-  }
-
-  // Get the SCEV for the GEP base.
-  const SCEV *BaseS = getSCEV(Base);
-
-  // Add the total offset from all the GEP indices to the base.
-  return getAddExpr(BaseS, TotalOffset, Wrap);
+  SmallVector<const SCEV *, 4> IndexExprs;
+  for (auto Index = GEP->idx_begin(); Index != GEP->idx_end(); ++Index)
+    IndexExprs.push_back(getSCEV(*Index));
+  return getGEPExpr(GEP->getSourceElementType(), getSCEV(Base), IndexExprs,
+                    GEP->isInBounds());
 }
 
 /// GetMinTrailingZeros - Determine the minimum number of zero bits that S is
@@ -4060,8 +4093,63 @@ ScalarEvolution::getRange(const SCEV *S,
   return setRange(S, SignHint, ConservativeResult);
 }
 
-/// createSCEV - We know that there is no SCEV for the specified value.
-/// Analyze the expression.
+SCEV::NoWrapFlags ScalarEvolution::getNoWrapFlagsFromUB(const Value *V) {
+  const BinaryOperator *BinOp = cast<BinaryOperator>(V);
+
+  // Return early if there are no flags to propagate to the SCEV.
+  SCEV::NoWrapFlags Flags = SCEV::FlagAnyWrap;
+  if (BinOp->hasNoUnsignedWrap())
+    Flags = ScalarEvolution::setFlags(Flags, SCEV::FlagNUW);
+  if (BinOp->hasNoSignedWrap())
+    Flags = ScalarEvolution::setFlags(Flags, SCEV::FlagNSW);
+  if (Flags == SCEV::FlagAnyWrap) {
+    return SCEV::FlagAnyWrap;
+  }
+
+  // Here we check that BinOp is in the header of the innermost loop
+  // containing BinOp, since we only deal with instructions in the loop
+  // header. The actual loop we need to check later will come from an add
+  // recurrence, but getting that requires computing the SCEV of the operands,
+  // which can be expensive. This check we can do cheaply to rule out some
+  // cases early.
+  Loop *innermostContainingLoop = LI->getLoopFor(BinOp->getParent());
+  if (innermostContainingLoop == nullptr ||
+      innermostContainingLoop->getHeader() != BinOp->getParent())
+    return SCEV::FlagAnyWrap;
+
+  // Only proceed if we can prove that BinOp does not yield poison.
+  if (!isKnownNotFullPoison(BinOp)) return SCEV::FlagAnyWrap;
+
+  // At this point we know that if V is executed, then it does not wrap
+  // according to at least one of NSW or NUW. If V is not executed, then we do
+  // not know if the calculation that V represents would wrap. Multiple
+  // instructions can map to the same SCEV. If we apply NSW or NUW from V to
+  // the SCEV, we must guarantee no wrapping for that SCEV also when it is
+  // derived from other instructions that map to the same SCEV. We cannot make
+  // that guarantee for cases where V is not executed. So we need to find the
+  // loop that V is considered in relation to and prove that V is executed for
+  // every iteration of that loop. That implies that the value that V
+  // calculates does not wrap anywhere in the loop, so then we can apply the
+  // flags to the SCEV.
+  //
+  // We check isLoopInvariant to disambiguate in case we are adding two
+  // recurrences from different loops, so that we know which loop to prove
+  // that V is executed in.
+  for (int OpIndex = 0; OpIndex < 2; ++OpIndex) {
+    const SCEV *Op = getSCEV(BinOp->getOperand(OpIndex));
+    if (auto *AddRec = dyn_cast<SCEVAddRecExpr>(Op)) {
+      const int OtherOpIndex = 1 - OpIndex;
+      const SCEV *OtherOp = getSCEV(BinOp->getOperand(OtherOpIndex));
+      if (isLoopInvariant(OtherOp, AddRec->getLoop()) &&
+          isGuaranteedToExecuteForEveryIteration(BinOp, AddRec->getLoop()))
+        return Flags;
+    }
+  }
+  return SCEV::FlagAnyWrap;
+}
+
+/// createSCEV - We know that there is no SCEV for the specified value.  Analyze
+/// the expression.
 ///
 const SCEV *ScalarEvolution::createSCEV(Value *V) {
   if (!isSCEVable(V->getType()))
@@ -4098,29 +4186,52 @@ const SCEV *ScalarEvolution::createSCEV(Value *V) {
     // Instead, gather up all the operands and make a single getAddExpr call.
     // LLVM IR canonical form means we need only traverse the left operands.
     //
-    // Don't apply this instruction's NSW or NUW flags to the new
-    // expression. The instruction may be guarded by control flow that the
-    // no-wrap behavior depends on. Non-control-equivalent instructions can be
-    // mapped to the same SCEV expression, and it would be incorrect to transfer
-    // NSW/NUW semantics to those operations.
+    // FIXME: Expand this handling of NSW and NUW to other instructions, like
+    // sub and mul.
     SmallVector<const SCEV *, 4> AddOps;
-    AddOps.push_back(getSCEV(U->getOperand(1)));
-    for (Value *Op = U->getOperand(0); ; Op = U->getOperand(0)) {
-      unsigned Opcode = Op->getValueID() - Value::InstructionVal;
-      if (Opcode != Instruction::Add && Opcode != Instruction::Sub)
+    for (Value *Op = U;; Op = U->getOperand(0)) {
+      U = dyn_cast<Operator>(Op);
+      unsigned Opcode = U ? U->getOpcode() : 0;
+      if (!U || (Opcode != Instruction::Add && Opcode != Instruction::Sub)) {
+        assert(Op != V && "V should be an add");
+        AddOps.push_back(getSCEV(Op));
         break;
-      U = cast<Operator>(Op);
+      }
+
+      if (auto *OpSCEV = getExistingSCEV(Op)) {
+        AddOps.push_back(OpSCEV);
+        break;
+      }
+
+      // If a NUW or NSW flag can be applied to the SCEV for this
+      // addition, then compute the SCEV for this addition by itself
+      // with a separate call to getAddExpr. We need to do that
+      // instead of pushing the operands of the addition onto AddOps,
+      // since the flags are only known to apply to this particular
+      // addition - they may not apply to other additions that can be
+      // formed with operands from AddOps.
+      //
+      // FIXME: Expand this to sub instructions.
+      if (Opcode == Instruction::Add && isa<BinaryOperator>(U)) {
+        SCEV::NoWrapFlags Flags = getNoWrapFlagsFromUB(U);
+        if (Flags != SCEV::FlagAnyWrap) {
+          AddOps.push_back(getAddExpr(getSCEV(U->getOperand(0)),
+                                      getSCEV(U->getOperand(1)), Flags));
+          break;
+        }
+      }
+
       const SCEV *Op1 = getSCEV(U->getOperand(1));
       if (Opcode == Instruction::Sub)
         AddOps.push_back(getNegativeSCEV(Op1));
       else
         AddOps.push_back(Op1);
     }
-    AddOps.push_back(getSCEV(U->getOperand(0)));
     return getAddExpr(AddOps);
   }
+
   case Instruction::Mul: {
-    // Don't transfer NSW/NUW for the same reason as AddExpr.
+    // FIXME: Transfer NSW/NUW as in AddExpr.
     SmallVector<const SCEV *, 4> MulOps;
     MulOps.push_back(getSCEV(U->getOperand(1)));
     for (Value *Op = U->getOperand(0);
@@ -4715,12 +4826,12 @@ void ScalarEvolution::forgetValue(Value *V) {
 }
 
 /// getExact - Get the exact loop backedge taken count considering all loop
-/// exits. A computable result can only be return for loops with a single exit.
-/// Returning the minimum taken count among all exits is incorrect because one
-/// of the loop's exit limit's may have been skipped. HowFarToZero assumes that
-/// the limit of each loop test is never skipped. This is a valid assumption as
-/// long as the loop exits via that test. For precise results, it is the
-/// caller's responsibility to specify the relevant loop exit using
+/// exits. A computable result can only be returned for loops with a single
+/// exit.  Returning the minimum taken count among all exits is incorrect
+/// because one of the loop's exit limit's may have been skipped. HowFarToZero
+/// assumes that the limit of each loop test is never skipped. This is a valid
+/// assumption as long as the loop exits via that test. For precise results, it
+/// is the caller's responsibility to specify the relevant loop exit using
 /// getExact(ExitingBlock, SE).
 const SCEV *
 ScalarEvolution::BackedgeTakenInfo::getExact(ScalarEvolution *SE) const {
@@ -6587,6 +6698,139 @@ bool ScalarEvolution::isKnownPredicate(ICmpInst::Predicate Pred,
   return isKnownPredicateWithRanges(Pred, LHS, RHS);
 }
 
+bool ScalarEvolution::isMonotonicPredicate(const SCEVAddRecExpr *LHS,
+                                           ICmpInst::Predicate Pred,
+                                           bool &Increasing) {
+  bool Result = isMonotonicPredicateImpl(LHS, Pred, Increasing);
+
+#ifndef NDEBUG
+  // Verify an invariant: inverting the predicate should turn a monotonically
+  // increasing change to a monotonically decreasing one, and vice versa.
+  bool IncreasingSwapped;
+  bool ResultSwapped = isMonotonicPredicateImpl(
+      LHS, ICmpInst::getSwappedPredicate(Pred), IncreasingSwapped);
+
+  assert(Result == ResultSwapped && "should be able to analyze both!");
+  if (ResultSwapped)
+    assert(Increasing == !IncreasingSwapped &&
+           "monotonicity should flip as we flip the predicate");
+#endif
+
+  return Result;
+}
+
+bool ScalarEvolution::isMonotonicPredicateImpl(const SCEVAddRecExpr *LHS,
+                                               ICmpInst::Predicate Pred,
+                                               bool &Increasing) {
+  SCEV::NoWrapFlags FlagsRequired = SCEV::FlagAnyWrap;
+  bool IncreasingOnNonNegativeStep = false;
+
+  switch (Pred) {
+  default:
+    return false; // Conservative answer
+
+  case ICmpInst::ICMP_UGT:
+  case ICmpInst::ICMP_UGE:
+    FlagsRequired = SCEV::FlagNUW;
+    IncreasingOnNonNegativeStep = true;
+    break;
+
+  case ICmpInst::ICMP_ULT:
+  case ICmpInst::ICMP_ULE:
+    FlagsRequired = SCEV::FlagNUW;
+    IncreasingOnNonNegativeStep = false;
+    break;
+
+  case ICmpInst::ICMP_SGT:
+  case ICmpInst::ICMP_SGE:
+    FlagsRequired = SCEV::FlagNSW;
+    IncreasingOnNonNegativeStep = true;
+    break;
+
+  case ICmpInst::ICMP_SLT:
+  case ICmpInst::ICMP_SLE:
+    FlagsRequired = SCEV::FlagNSW;
+    IncreasingOnNonNegativeStep = false;
+    break;
+  }
+
+  if (!LHS->getNoWrapFlags(FlagsRequired))
+    return false;
+
+  // A zero step value for LHS means the induction variable is essentially a
+  // loop invariant value. We don't really depend on the predicate actually
+  // flipping from false to true (for increasing predicates, and the other way
+  // around for decreasing predicates), all we care about is that *if* the
+  // predicate changes then it only changes from false to true.
+  //
+  // A zero step value in itself is not very useful, but there may be places
+  // where SCEV can prove X >= 0 but not prove X > 0, so it is helpful to be
+  // as general as possible.
+
+  if (isKnownNonNegative(LHS->getStepRecurrence(*this))) {
+    Increasing = IncreasingOnNonNegativeStep;
+    return true;
+  }
+
+  if (isKnownNonPositive(LHS->getStepRecurrence(*this))) {
+    Increasing = !IncreasingOnNonNegativeStep;
+    return true;
+  }
+
+  return false;
+}
+
+bool ScalarEvolution::isLoopInvariantPredicate(
+    ICmpInst::Predicate Pred, const SCEV *LHS, const SCEV *RHS, const Loop *L,
+    ICmpInst::Predicate &InvariantPred, const SCEV *&InvariantLHS,
+    const SCEV *&InvariantRHS) {
+
+  // If there is a loop-invariant, force it into the RHS, otherwise bail out.
+  if (!isLoopInvariant(RHS, L)) {
+    if (!isLoopInvariant(LHS, L))
+      return false;
+
+    std::swap(LHS, RHS);
+    Pred = ICmpInst::getSwappedPredicate(Pred);
+  }
+
+  const SCEVAddRecExpr *ArLHS = dyn_cast<SCEVAddRecExpr>(LHS);
+  if (!ArLHS || ArLHS->getLoop() != L)
+    return false;
+
+  bool Increasing;
+  if (!isMonotonicPredicate(ArLHS, Pred, Increasing))
+    return false;
+
+  // If the predicate "ArLHS `Pred` RHS" monotonically increases from false to
+  // true as the loop iterates, and the backedge is control dependent on
+  // "ArLHS `Pred` RHS" == true then we can reason as follows:
+  //
+  //   * if the predicate was false in the first iteration then the predicate
+  //     is never evaluated again, since the loop exits without taking the
+  //     backedge.
+  //   * if the predicate was true in the first iteration then it will
+  //     continue to be true for all future iterations since it is
+  //     monotonically increasing.
+  //
+  // For both the above possibilities, we can replace the loop varying
+  // predicate with its value on the first iteration of the loop (which is
+  // loop invariant).
+  //
+  // A similar reasoning applies for a monotonically decreasing predicate, by
+  // replacing true with false and false with true in the above two bullets.
+
+  auto P = Increasing ? Pred : ICmpInst::getInversePredicate(Pred);
+
+  if (!isLoopBackedgeGuardedByCond(L, P, LHS, RHS))
+    return false;
+
+  InvariantPred = Pred;
+  InvariantLHS = ArLHS->getStart();
+  InvariantRHS = RHS;
+  return true;
+}
+
 bool
 ScalarEvolution::isKnownPredicateWithRanges(ICmpInst::Predicate Pred,
                                             const SCEV *LHS, const SCEV *RHS) {
@@ -6702,7 +6946,7 @@ ScalarEvolution::isLoopBackedgeGuardedByCond(const Loop *L,
     ScalarEvolution &SE;
 
     explicit ClearWalkingBEDominatingCondsOnExit(ScalarEvolution &SE)
-        : SE(SE){};
+        : SE(SE){}
 
     ~ClearWalkingBEDominatingCondsOnExit() {
       SE.WalkingBEDominatingConds = false;
@@ -7618,11 +7862,11 @@ struct SCEVCollectTerms {
 }
 
 /// Find parametric terms in this SCEVAddRecExpr.
-void SCEVAddRecExpr::collectParametricTerms(
-    ScalarEvolution &SE, SmallVectorImpl<const SCEV *> &Terms) const {
+void ScalarEvolution::collectParametricTerms(const SCEV *Expr,
+    SmallVectorImpl<const SCEV *> &Terms) {
   SmallVector<const SCEV *, 4> Strides;
-  SCEVCollectStrides StrideCollector(SE, Strides);
-  visitAll(this, StrideCollector);
+  SCEVCollectStrides StrideCollector(*this, Strides);
+  visitAll(Expr, StrideCollector);
 
   DEBUG({
       dbgs() << "Strides:\n";
@@ -7838,19 +8082,23 @@ void ScalarEvolution::findArrayDimensions(SmallVectorImpl<const SCEV *> &Terms,
 
 /// Third step of delinearization: compute the access functions for the
 /// Subscripts based on the dimensions in Sizes.
-void SCEVAddRecExpr::computeAccessFunctions(
-    ScalarEvolution &SE, SmallVectorImpl<const SCEV *> &Subscripts,
-    SmallVectorImpl<const SCEV *> &Sizes) const {
+void ScalarEvolution::computeAccessFunctions(
+    const SCEV *Expr, SmallVectorImpl<const SCEV *> &Subscripts,
+    SmallVectorImpl<const SCEV *> &Sizes) {
 
   // Early exit in case this SCEV is not an affine multivariate function.
-  if (Sizes.empty() || !this->isAffine())
+  if (Sizes.empty())
     return;
 
-  const SCEV *Res = this;
+  if (auto AR = dyn_cast<SCEVAddRecExpr>(Expr))
+    if (!AR->isAffine())
+      return;
+
+  const SCEV *Res = Expr;
   int Last = Sizes.size() - 1;
   for (int i = Last; i >= 0; i--) {
     const SCEV *Q, *R;
-    SCEVDivision::divide(SE, Res, Sizes[i], &Q, &R);
+    SCEVDivision::divide(*this, Res, Sizes[i], &Q, &R);
 
     DEBUG({
         dbgs() << "Res: " << *Res << "\n";
@@ -7942,31 +8190,31 @@ void SCEVAddRecExpr::computeAccessFunctions(
 /// asking for the SCEV of the memory access with respect to all enclosing
 /// loops, calling SCEV->delinearize on that and printing the results.
 
-void SCEVAddRecExpr::delinearize(ScalarEvolution &SE,
+void ScalarEvolution::delinearize(const SCEV *Expr,
                                  SmallVectorImpl<const SCEV *> &Subscripts,
                                  SmallVectorImpl<const SCEV *> &Sizes,
-                                 const SCEV *ElementSize) const {
+                                 const SCEV *ElementSize) {
   // First step: collect parametric terms.
   SmallVector<const SCEV *, 4> Terms;
-  collectParametricTerms(SE, Terms);
+  collectParametricTerms(Expr, Terms);
 
   if (Terms.empty())
     return;
 
   // Second step: find subscript sizes.
-  SE.findArrayDimensions(Terms, Sizes, ElementSize);
+  findArrayDimensions(Terms, Sizes, ElementSize);
 
   if (Sizes.empty())
     return;
 
   // Third step: compute the access functions for each subscript.
-  computeAccessFunctions(SE, Subscripts, Sizes);
+  computeAccessFunctions(Expr, Subscripts, Sizes);
 
   if (Subscripts.empty())
     return;
 
   DEBUG({
-      dbgs() << "succeeded to delinearize " << *this << "\n";
+      dbgs() << "succeeded to delinearize " << *Expr << "\n";
       dbgs() << "ArrayDecl[UnknownSize]";
       for (const SCEV *S : Sizes)
         dbgs() << "[" << *S << "]";
@@ -8074,10 +8322,10 @@ void ScalarEvolution::releaseMemory() {
 
 void ScalarEvolution::getAnalysisUsage(AnalysisUsage &AU) const {
   AU.setPreservesAll();
-  AU.addRequired<AssumptionCacheTracker>();
+  AU.addRequiredTransitive<AssumptionCacheTracker>();
   AU.addRequiredTransitive<LoopInfoWrapperPass>();
   AU.addRequiredTransitive<DominatorTreeWrapperPass>();
-  AU.addRequired<TargetLibraryInfoWrapperPass>();
+  AU.addRequiredTransitive<TargetLibraryInfoWrapperPass>();
 }
 
 bool ScalarEvolution::hasLoopInvariantBackedgeTakenCount(const Loop *L) {
diff --git a/lib/Analysis/ScalarEvolutionAliasAnalysis.cpp b/lib/Analysis/ScalarEvolutionAliasAnalysis.cpp
index ccec0a877f5..6bc0d85a61f 100644
--- a/lib/Analysis/ScalarEvolutionAliasAnalysis.cpp
+++ b/lib/Analysis/ScalarEvolutionAliasAnalysis.cpp
@@ -53,7 +53,8 @@ namespace {
   private:
     void getAnalysisUsage(AnalysisUsage &AU) const override;
     bool runOnFunction(Function &F) override;
-    AliasResult alias(const Location &LocA, const Location &LocB) override;
+    AliasResult alias(const MemoryLocation &LocA,
+                      const MemoryLocation &LocB) override;
 
     Value *GetBaseValue(const SCEV *S);
   };
@@ -106,9 +107,8 @@ ScalarEvolutionAliasAnalysis::GetBaseValue(const SCEV *S) {
   return nullptr;
 }
 
-AliasAnalysis::AliasResult
-ScalarEvolutionAliasAnalysis::alias(const Location &LocA,
-                                    const Location &LocB) {
+AliasResult ScalarEvolutionAliasAnalysis::alias(const MemoryLocation &LocA,
+                                                const MemoryLocation &LocB) {
   // If either of the memory references is empty, it doesn't matter what the
   // pointer values are. This allows the code below to ignore this special
   // case.
@@ -161,12 +161,12 @@ ScalarEvolutionAliasAnalysis::alias(const Location &LocA,
   Value *AO = GetBaseValue(AS);
   Value *BO = GetBaseValue(BS);
   if ((AO && AO != LocA.Ptr) || (BO && BO != LocB.Ptr))
-    if (alias(Location(AO ? AO : LocA.Ptr,
-                       AO ? +UnknownSize : LocA.Size,
-                       AO ? AAMDNodes() : LocA.AATags),
-              Location(BO ? BO : LocB.Ptr,
-                       BO ? +UnknownSize : LocB.Size,
-                       BO ? AAMDNodes() : LocB.AATags)) == NoAlias)
+    if (alias(MemoryLocation(AO ? AO : LocA.Ptr,
+                             AO ? +MemoryLocation::UnknownSize : LocA.Size,
+                             AO ? AAMDNodes() : LocA.AATags),
+              MemoryLocation(BO ? BO : LocB.Ptr,
+                             BO ? +MemoryLocation::UnknownSize : LocB.Size,
+                             BO ? AAMDNodes() : LocB.AATags)) == NoAlias)
       return NoAlias;
 
   // Forward the query to the next analysis.
diff --git a/lib/Analysis/ScalarEvolutionExpander.cpp b/lib/Analysis/ScalarEvolutionExpander.cpp
index 0bd427bf74e..fee2a2d0d18 100644
--- a/lib/Analysis/ScalarEvolutionExpander.cpp
+++ b/lib/Analysis/ScalarEvolutionExpander.cpp
@@ -24,10 +24,12 @@
 #include "llvm/IR/IntrinsicInst.h"
 #include "llvm/IR/LLVMContext.h"
 #include "llvm/IR/Module.h"
+#include "llvm/IR/PatternMatch.h"
 #include "llvm/Support/Debug.h"
 #include "llvm/Support/raw_ostream.h"
 
 using namespace llvm;
+using namespace PatternMatch;
 
 /// ReuseOrCreateCast - Arrange for there to be a cast of V to Ty at IP,
 /// reusing an existing cast if a suitable one exists, moving an existing
@@ -751,25 +753,30 @@ Value *SCEVExpander::visitMulExpr(const SCEVMulExpr *S) {
   // out of loops.
   Value *Prod = nullptr;
   for (SmallVectorImpl<std::pair<const Loop *, const SCEV *> >::iterator
-       I = OpsAndLoops.begin(), E = OpsAndLoops.end(); I != E; ) {
+       I = OpsAndLoops.begin(), E = OpsAndLoops.end(); I != E; ++I) {
     const SCEV *Op = I->second;
     if (!Prod) {
       // This is the first operand. Just expand it.
       Prod = expand(Op);
-      ++I;
     } else if (Op->isAllOnesValue()) {
       // Instead of doing a multiply by negative one, just do a negate.
       Prod = InsertNoopCastOfTo(Prod, Ty);
       Prod = InsertBinop(Instruction::Sub, Constant::getNullValue(Ty), Prod);
-      ++I;
     } else {
       // A simple mul.
       Value *W = expandCodeFor(Op, Ty);
       Prod = InsertNoopCastOfTo(Prod, Ty);
       // Canonicalize a constant to the RHS.
       if (isa<Constant>(Prod)) std::swap(Prod, W);
-      Prod = InsertBinop(Instruction::Mul, Prod, W);
-      ++I;
+      const APInt *RHS;
+      if (match(W, m_Power2(RHS))) {
+        // Canonicalize Prod*(1<<C) to Prod<<C.
+        assert(!Ty->isVectorTy() && "vector types are not SCEVable");
+        Prod = InsertBinop(Instruction::Shl, Prod,
+                           ConstantInt::get(Ty, RHS->logBase2()));
+      } else {
+        Prod = InsertBinop(Instruction::Mul, Prod, W);
+      }
     }
   }
 
@@ -1702,7 +1709,7 @@ unsigned SCEVExpander::replaceCongruentIVs(Loop *L, const DominatorTree *DT,
 
   unsigned NumElim = 0;
   DenseMap<const SCEV *, PHINode *> ExprToIVMap;
-  // Process phis from wide to narrow. Mapping wide phis to the their truncation
+  // Process phis from wide to narrow. Map wide phis to their truncation
   // so narrow phis can reuse them.
   for (SmallVectorImpl<PHINode*>::const_iterator PIter = Phis.begin(),
          PEnd = Phis.end(); PIter != PEnd; ++PIter) {
@@ -1712,7 +1719,7 @@ unsigned SCEVExpander::replaceCongruentIVs(Loop *L, const DominatorTree *DT,
     // would confuse the logic below that expects proper IVs.
     if (Value *V = SimplifyInstruction(Phi, DL, SE.TLI, SE.DT, SE.AC)) {
       Phi->replaceAllUsesWith(V);
-      DeadInsts.push_back(Phi);
+      DeadInsts.emplace_back(Phi);
       ++NumElim;
       DEBUG_WITH_TYPE(DebugType, dbgs()
                       << "INDVARS: Eliminated constant iv: " << *Phi << '\n');
@@ -1787,7 +1794,7 @@ unsigned SCEVExpander::replaceCongruentIVs(Loop *L, const DominatorTree *DT,
             CreateTruncOrBitCast(OrigInc, IsomorphicInc->getType(), IVName);
         }
         IsomorphicInc->replaceAllUsesWith(NewInc);
-        DeadInsts.push_back(IsomorphicInc);
+        DeadInsts.emplace_back(IsomorphicInc);
       }
     }
     DEBUG_WITH_TYPE(DebugType, dbgs()
@@ -1800,13 +1807,30 @@ unsigned SCEVExpander::replaceCongruentIVs(Loop *L, const DominatorTree *DT,
       NewIV = Builder.CreateTruncOrBitCast(OrigPhiRef, Phi->getType(), IVName);
     }
     Phi->replaceAllUsesWith(NewIV);
-    DeadInsts.push_back(Phi);
+    DeadInsts.emplace_back(Phi);
   }
   return NumElim;
 }
 
 bool SCEVExpander::isHighCostExpansionHelper(
     const SCEV *S, Loop *L, SmallPtrSetImpl<const SCEV *> &Processed) {
+
+  // Zero/One operand expressions
+  switch (S->getSCEVType()) {
+  case scUnknown:
+  case scConstant:
+    return false;
+  case scTruncate:
+    return isHighCostExpansionHelper(cast<SCEVTruncateExpr>(S)->getOperand(), L,
+                                     Processed);
+  case scZeroExtend:
+    return isHighCostExpansionHelper(cast<SCEVZeroExtendExpr>(S)->getOperand(),
+                                     L, Processed);
+  case scSignExtend:
+    return isHighCostExpansionHelper(cast<SCEVSignExtendExpr>(S)->getOperand(),
+                                     L, Processed);
+  }
+
   if (!Processed.insert(S).second)
     return false;
 
@@ -1849,23 +1873,22 @@ bool SCEVExpander::isHighCostExpansionHelper(
     }
   }
 
-  // Recurse past add expressions, which commonly occur in the
+  // HowManyLessThans uses a Max expression whenever the loop is not guarded by
+  // the exit condition.
+  if (isa<SCEVSMaxExpr>(S) || isa<SCEVUMaxExpr>(S))
+    return true;
+
+  // Recurse past nary expressions, which commonly occur in the
   // BackedgeTakenCount. They may already exist in program code, and if not,
   // they are not too expensive rematerialize.
-  if (const SCEVAddExpr *Add = dyn_cast<SCEVAddExpr>(S)) {
-    for (SCEVAddExpr::op_iterator I = Add->op_begin(), E = Add->op_end();
+  if (const SCEVNAryExpr *NAry = dyn_cast<SCEVNAryExpr>(S)) {
+    for (SCEVNAryExpr::op_iterator I = NAry->op_begin(), E = NAry->op_end();
          I != E; ++I) {
       if (isHighCostExpansionHelper(*I, L, Processed))
         return true;
     }
-    return false;
   }
 
-  // HowManyLessThans uses a Max expression whenever the loop is not guarded by
-  // the exit condition.
-  if (isa<SCEVSMaxExpr>(S) || isa<SCEVUMaxExpr>(S))
-    return true;
-
   // If we haven't recognized an expensive SCEV pattern, assume it's an
   // expression produced by program code.
   return false;
diff --git a/lib/Analysis/ScopedNoAliasAA.cpp b/lib/Analysis/ScopedNoAliasAA.cpp
index 02f8b0b1384..e89fbfbf865 100644
--- a/lib/Analysis/ScopedNoAliasAA.cpp
+++ b/lib/Analysis/ScopedNoAliasAA.cpp
@@ -99,14 +99,15 @@ class ScopedNoAliasAA : public ImmutablePass, public AliasAnalysis {
 
 private:
   void getAnalysisUsage(AnalysisUsage &AU) const override;
-  AliasResult alias(const Location &LocA, const Location &LocB) override;
-  bool pointsToConstantMemory(const Location &Loc, bool OrLocal) override;
-  ModRefBehavior getModRefBehavior(ImmutableCallSite CS) override;
-  ModRefBehavior getModRefBehavior(const Function *F) override;
-  ModRefResult getModRefInfo(ImmutableCallSite CS,
-                             const Location &Loc) override;
-  ModRefResult getModRefInfo(ImmutableCallSite CS1,
-                             ImmutableCallSite CS2) override;
+  AliasResult alias(const MemoryLocation &LocA,
+                    const MemoryLocation &LocB) override;
+  bool pointsToConstantMemory(const MemoryLocation &Loc, bool OrLocal) override;
+  FunctionModRefBehavior getModRefBehavior(ImmutableCallSite CS) override;
+  FunctionModRefBehavior getModRefBehavior(const Function *F) override;
+  ModRefInfo getModRefInfo(ImmutableCallSite CS,
+                           const MemoryLocation &Loc) override;
+  ModRefInfo getModRefInfo(ImmutableCallSite CS1,
+                           ImmutableCallSite CS2) override;
 };
 }  // End of anonymous namespace
 
@@ -176,8 +177,8 @@ ScopedNoAliasAA::mayAliasInScopes(const MDNode *Scopes,
   return true;
 }
 
-AliasAnalysis::AliasResult
-ScopedNoAliasAA::alias(const Location &LocA, const Location &LocB) {
+AliasResult ScopedNoAliasAA::alias(const MemoryLocation &LocA,
+                                   const MemoryLocation &LocB) {
   if (!EnableScopedNoAlias)
     return AliasAnalysis::alias(LocA, LocB);
 
@@ -198,52 +199,51 @@ ScopedNoAliasAA::alias(const Location &LocA, const Location &LocB) {
   return AliasAnalysis::alias(LocA, LocB);
 }
 
-bool ScopedNoAliasAA::pointsToConstantMemory(const Location &Loc,
+bool ScopedNoAliasAA::pointsToConstantMemory(const MemoryLocation &Loc,
                                              bool OrLocal) {
   return AliasAnalysis::pointsToConstantMemory(Loc, OrLocal);
 }
 
-AliasAnalysis::ModRefBehavior
+FunctionModRefBehavior
 ScopedNoAliasAA::getModRefBehavior(ImmutableCallSite CS) {
   return AliasAnalysis::getModRefBehavior(CS);
 }
 
-AliasAnalysis::ModRefBehavior
-ScopedNoAliasAA::getModRefBehavior(const Function *F) {
+FunctionModRefBehavior ScopedNoAliasAA::getModRefBehavior(const Function *F) {
   return AliasAnalysis::getModRefBehavior(F);
 }
 
-AliasAnalysis::ModRefResult
-ScopedNoAliasAA::getModRefInfo(ImmutableCallSite CS, const Location &Loc) {
+ModRefInfo ScopedNoAliasAA::getModRefInfo(ImmutableCallSite CS,
+                                          const MemoryLocation &Loc) {
   if (!EnableScopedNoAlias)
     return AliasAnalysis::getModRefInfo(CS, Loc);
 
   if (!mayAliasInScopes(Loc.AATags.Scope, CS.getInstruction()->getMetadata(
                                               LLVMContext::MD_noalias)))
-    return NoModRef;
+    return MRI_NoModRef;
 
   if (!mayAliasInScopes(
           CS.getInstruction()->getMetadata(LLVMContext::MD_alias_scope),
           Loc.AATags.NoAlias))
-    return NoModRef;
+    return MRI_NoModRef;
 
   return AliasAnalysis::getModRefInfo(CS, Loc);
 }
 
-AliasAnalysis::ModRefResult
-ScopedNoAliasAA::getModRefInfo(ImmutableCallSite CS1, ImmutableCallSite CS2) {
+ModRefInfo ScopedNoAliasAA::getModRefInfo(ImmutableCallSite CS1,
+                                          ImmutableCallSite CS2) {
   if (!EnableScopedNoAlias)
     return AliasAnalysis::getModRefInfo(CS1, CS2);
 
   if (!mayAliasInScopes(
           CS1.getInstruction()->getMetadata(LLVMContext::MD_alias_scope),
           CS2.getInstruction()->getMetadata(LLVMContext::MD_noalias)))
-    return NoModRef;
+    return MRI_NoModRef;
 
   if (!mayAliasInScopes(
           CS2.getInstruction()->getMetadata(LLVMContext::MD_alias_scope),
           CS1.getInstruction()->getMetadata(LLVMContext::MD_noalias)))
-    return NoModRef;
+    return MRI_NoModRef;
 
   return AliasAnalysis::getModRefInfo(CS1, CS2);
 }
diff --git a/lib/Analysis/TargetLibraryInfo.cpp b/lib/Analysis/TargetLibraryInfo.cpp
index 88dff8b94b4..635c50ca6e5 100644
--- a/lib/Analysis/TargetLibraryInfo.cpp
+++ b/lib/Analysis/TargetLibraryInfo.cpp
@@ -220,12 +220,10 @@ static void initialize(TargetLibraryInfoImpl &TLI, const Triple &T,
     TLI.setUnavailable(LibFunc::ffs);
     TLI.setUnavailable(LibFunc::fileno);
     TLI.setUnavailable(LibFunc::flockfile);
-    // @LOCALMOD: https://code.google.com/p/nativeclient/issues/detail?id=4089
-    TLI.setUnavailable(LibFunc::Fseeko);
+    TLI.setUnavailable(LibFunc::fseeko);
     TLI.setUnavailable(LibFunc::fstat);
     TLI.setUnavailable(LibFunc::fstatvfs);
-    // @LOCALMOD (same as fseeko -> Fseeko)
-    TLI.setUnavailable(LibFunc::Ftello);
+    TLI.setUnavailable(LibFunc::ftello);
     TLI.setUnavailable(LibFunc::ftrylockfile);
     TLI.setUnavailable(LibFunc::funlockfile);
     TLI.setUnavailable(LibFunc::getc_unlocked);
@@ -469,14 +467,42 @@ void TargetLibraryInfoImpl::addVectorizableFunctionsFromVecLib(
   switch (VecLib) {
   case Accelerate: {
     const VecDesc VecFuncs[] = {
+        // Floating-Point Arithmetic and Auxiliary Functions
+        {"ceilf", "vceilf", 4},
+        {"fabsf", "vfabsf", 4},
+        {"llvm.fabs.f32", "vfabsf", 4},
+        {"floorf", "vfloorf", 4},
+        {"sqrtf", "vsqrtf", 4},
+        {"llvm.sqrt.f32", "vsqrtf", 4},
+
+        // Exponential and Logarithmic Functions
         {"expf", "vexpf", 4},
         {"llvm.exp.f32", "vexpf", 4},
+        {"expm1f", "vexpm1f", 4},
         {"logf", "vlogf", 4},
         {"llvm.log.f32", "vlogf", 4},
-        {"sqrtf", "vsqrtf", 4},
-        {"llvm.sqrt.f32", "vsqrtf", 4},
-        {"fabsf", "vfabsf", 4},
-        {"llvm.fabs.f32", "vfabsf", 4},
+        {"log1pf", "vlog1pf", 4},
+        {"log10f", "vlog10f", 4},
+        {"llvm.log10.f32", "vlog10f", 4},
+        {"logbf", "vlogbf", 4},
+
+        // Trigonometric Functions
+        {"sinf", "vsinf", 4},
+        {"llvm.sin.f32", "vsinf", 4},
+        {"cosf", "vcosf", 4},
+        {"llvm.cos.f32", "vcosf", 4},
+        {"tanf", "vtanf", 4},
+        {"asinf", "vasinf", 4},
+        {"acosf", "vacosf", 4},
+        {"atanf", "vatanf", 4},
+
+        // Hyperbolic Functions
+        {"sinhf", "vsinhf", 4},
+        {"coshf", "vcoshf", 4},
+        {"tanhf", "vtanhf", 4},
+        {"asinhf", "vasinhf", 4},
+        {"acoshf", "vacoshf", 4},
+        {"atanhf", "vatanhf", 4},
     };
     addVectorizableFunctions(VecFuncs);
     break;
diff --git a/lib/Analysis/TargetTransformInfo.cpp b/lib/Analysis/TargetTransformInfo.cpp
index a1519de25ee..731aae23a26 100644
--- a/lib/Analysis/TargetTransformInfo.cpp
+++ b/lib/Analysis/TargetTransformInfo.cpp
@@ -28,12 +28,12 @@ namespace {
 ///
 /// This is used when no target specific information is available.
 struct NoTTIImpl : TargetTransformInfoImplCRTPBase<NoTTIImpl> {
-  explicit NoTTIImpl(const DataLayout *DL)
+  explicit NoTTIImpl(const DataLayout &DL)
       : TargetTransformInfoImplCRTPBase<NoTTIImpl>(DL) {}
 };
 }
 
-TargetTransformInfo::TargetTransformInfo(const DataLayout *DL)
+TargetTransformInfo::TargetTransformInfo(const DataLayout &DL)
     : TTIImpl(new Model<NoTTIImpl>(NoTTIImpl(DL))) {}
 
 TargetTransformInfo::~TargetTransformInfo() {}
@@ -100,9 +100,10 @@ bool TargetTransformInfo::isLegalICmpImmediate(int64_t Imm) const {
 bool TargetTransformInfo::isLegalAddressingMode(Type *Ty, GlobalValue *BaseGV,
                                                 int64_t BaseOffset,
                                                 bool HasBaseReg,
-                                                int64_t Scale) const {
+                                                int64_t Scale,
+                                                unsigned AddrSpace) const {
   return TTIImpl->isLegalAddressingMode(Ty, BaseGV, BaseOffset, HasBaseReg,
-                                        Scale);
+                                        Scale, AddrSpace);
 }
 
 bool TargetTransformInfo::isLegalMaskedStore(Type *DataType,
@@ -118,15 +119,20 @@ bool TargetTransformInfo::isLegalMaskedLoad(Type *DataType,
 int TargetTransformInfo::getScalingFactorCost(Type *Ty, GlobalValue *BaseGV,
                                               int64_t BaseOffset,
                                               bool HasBaseReg,
-                                              int64_t Scale) const {
+                                              int64_t Scale,
+                                              unsigned AddrSpace) const {
   return TTIImpl->getScalingFactorCost(Ty, BaseGV, BaseOffset, HasBaseReg,
-                                       Scale);
+                                       Scale, AddrSpace);
 }
 
 bool TargetTransformInfo::isTruncateFree(Type *Ty1, Type *Ty2) const {
   return TTIImpl->isTruncateFree(Ty1, Ty2);
 }
 
+bool TargetTransformInfo::isZExtFree(Type *Ty1, Type *Ty2) const {
+  return TTIImpl->isZExtFree(Ty1, Ty2);
+}
+
 bool TargetTransformInfo::isProfitableToHoist(Instruction *I) const {
   return TTIImpl->isProfitableToHoist(I);
 }
@@ -186,8 +192,8 @@ unsigned TargetTransformInfo::getRegisterBitWidth(bool Vector) const {
   return TTIImpl->getRegisterBitWidth(Vector);
 }
 
-unsigned TargetTransformInfo::getMaxInterleaveFactor() const {
-  return TTIImpl->getMaxInterleaveFactor();
+unsigned TargetTransformInfo::getMaxInterleaveFactor(unsigned VF) const {
+  return TTIImpl->getMaxInterleaveFactor(VF);
 }
 
 unsigned TargetTransformInfo::getArithmeticInstrCost(
@@ -235,6 +241,13 @@ TargetTransformInfo::getMaskedMemoryOpCost(unsigned Opcode, Type *Src,
   return TTIImpl->getMaskedMemoryOpCost(Opcode, Src, Alignment, AddressSpace);
 }
 
+unsigned TargetTransformInfo::getInterleavedMemoryOpCost(
+    unsigned Opcode, Type *VecTy, unsigned Factor, ArrayRef<unsigned> Indices,
+    unsigned Alignment, unsigned AddressSpace) const {
+  return TTIImpl->getInterleavedMemoryOpCost(Opcode, VecTy, Factor, Indices,
+                                             Alignment, AddressSpace);
+}
+
 unsigned
 TargetTransformInfo::getIntrinsicInstrCost(Intrinsic::ID ID, Type *RetTy,
                                            ArrayRef<Type *> Tys) const {
@@ -275,6 +288,11 @@ Value *TargetTransformInfo::getOrCreateResultFromMemIntrinsic(
   return TTIImpl->getOrCreateResultFromMemIntrinsic(Inst, ExpectedType);
 }
 
+bool TargetTransformInfo::areInlineCompatible(const Function *Caller,
+                                              const Function *Callee) const {
+  return TTIImpl->areInlineCompatible(Caller, Callee);
+}
+
 TargetTransformInfo::Concept::~Concept() {}
 
 TargetIRAnalysis::TargetIRAnalysis() : TTICallback(&getDefaultTTI) {}
@@ -290,7 +308,7 @@ TargetIRAnalysis::Result TargetIRAnalysis::run(Function &F) {
 char TargetIRAnalysis::PassID;
 
 TargetIRAnalysis::Result TargetIRAnalysis::getDefaultTTI(Function &F) {
-  return Result(&F.getParent()->getDataLayout());
+  return Result(F.getParent()->getDataLayout());
 }
 
 // Register the basic pass.
diff --git a/lib/Analysis/TypeBasedAliasAnalysis.cpp b/lib/Analysis/TypeBasedAliasAnalysis.cpp
index 115872584cb..c86856d330c 100644
--- a/lib/Analysis/TypeBasedAliasAnalysis.cpp
+++ b/lib/Analysis/TypeBasedAliasAnalysis.cpp
@@ -300,14 +300,16 @@ namespace {
 
   private:
     void getAnalysisUsage(AnalysisUsage &AU) const override;
-    AliasResult alias(const Location &LocA, const Location &LocB) override;
-    bool pointsToConstantMemory(const Location &Loc, bool OrLocal) override;
-    ModRefBehavior getModRefBehavior(ImmutableCallSite CS) override;
-    ModRefBehavior getModRefBehavior(const Function *F) override;
-    ModRefResult getModRefInfo(ImmutableCallSite CS,
-                               const Location &Loc) override;
-    ModRefResult getModRefInfo(ImmutableCallSite CS1,
-                               ImmutableCallSite CS2) override;
+    AliasResult alias(const MemoryLocation &LocA,
+                      const MemoryLocation &LocB) override;
+    bool pointsToConstantMemory(const MemoryLocation &Loc,
+                                bool OrLocal) override;
+    FunctionModRefBehavior getModRefBehavior(ImmutableCallSite CS) override;
+    FunctionModRefBehavior getModRefBehavior(const Function *F) override;
+    ModRefInfo getModRefInfo(ImmutableCallSite CS,
+                             const MemoryLocation &Loc) override;
+    ModRefInfo getModRefInfo(ImmutableCallSite CS1,
+                             ImmutableCallSite CS2) override;
   };
 }  // End of anonymous namespace
 
@@ -452,9 +454,8 @@ TypeBasedAliasAnalysis::PathAliases(const MDNode *A,
   return false;
 }
 
-AliasAnalysis::AliasResult
-TypeBasedAliasAnalysis::alias(const Location &LocA,
-                              const Location &LocB) {
+AliasResult TypeBasedAliasAnalysis::alias(const MemoryLocation &LocA,
+                                          const MemoryLocation &LocB) {
   if (!EnableTBAA)
     return AliasAnalysis::alias(LocA, LocB);
 
@@ -473,7 +474,7 @@ TypeBasedAliasAnalysis::alias(const Location &LocA,
   return NoAlias;
 }
 
-bool TypeBasedAliasAnalysis::pointsToConstantMemory(const Location &Loc,
+bool TypeBasedAliasAnalysis::pointsToConstantMemory(const MemoryLocation &Loc,
                                                     bool OrLocal) {
   if (!EnableTBAA)
     return AliasAnalysis::pointsToConstantMemory(Loc, OrLocal);
@@ -490,32 +491,31 @@ bool TypeBasedAliasAnalysis::pointsToConstantMemory(const Location &Loc,
   return AliasAnalysis::pointsToConstantMemory(Loc, OrLocal);
 }
 
-AliasAnalysis::ModRefBehavior
+FunctionModRefBehavior
 TypeBasedAliasAnalysis::getModRefBehavior(ImmutableCallSite CS) {
   if (!EnableTBAA)
     return AliasAnalysis::getModRefBehavior(CS);
 
-  ModRefBehavior Min = UnknownModRefBehavior;
+  FunctionModRefBehavior Min = FMRB_UnknownModRefBehavior;
 
   // If this is an "immutable" type, we can assume the call doesn't write
   // to memory.
   if (const MDNode *M = CS.getInstruction()->getMetadata(LLVMContext::MD_tbaa))
     if ((!isStructPathTBAA(M) && TBAANode(M).TypeIsImmutable()) ||
         (isStructPathTBAA(M) && TBAAStructTagNode(M).TypeIsImmutable()))
-      Min = OnlyReadsMemory;
+      Min = FMRB_OnlyReadsMemory;
 
-  return ModRefBehavior(AliasAnalysis::getModRefBehavior(CS) & Min);
+  return FunctionModRefBehavior(AliasAnalysis::getModRefBehavior(CS) & Min);
 }
 
-AliasAnalysis::ModRefBehavior
+FunctionModRefBehavior
 TypeBasedAliasAnalysis::getModRefBehavior(const Function *F) {
   // Functions don't have metadata. Just chain to the next implementation.
   return AliasAnalysis::getModRefBehavior(F);
 }
 
-AliasAnalysis::ModRefResult
-TypeBasedAliasAnalysis::getModRefInfo(ImmutableCallSite CS,
-                                      const Location &Loc) {
+ModRefInfo TypeBasedAliasAnalysis::getModRefInfo(ImmutableCallSite CS,
+                                                 const MemoryLocation &Loc) {
   if (!EnableTBAA)
     return AliasAnalysis::getModRefInfo(CS, Loc);
 
@@ -523,14 +523,13 @@ TypeBasedAliasAnalysis::getModRefInfo(ImmutableCallSite CS,
     if (const MDNode *M =
             CS.getInstruction()->getMetadata(LLVMContext::MD_tbaa))
       if (!Aliases(L, M))
-        return NoModRef;
+        return MRI_NoModRef;
 
   return AliasAnalysis::getModRefInfo(CS, Loc);
 }
 
-AliasAnalysis::ModRefResult
-TypeBasedAliasAnalysis::getModRefInfo(ImmutableCallSite CS1,
-                                      ImmutableCallSite CS2) {
+ModRefInfo TypeBasedAliasAnalysis::getModRefInfo(ImmutableCallSite CS1,
+                                                 ImmutableCallSite CS2) {
   if (!EnableTBAA)
     return AliasAnalysis::getModRefInfo(CS1, CS2);
 
@@ -539,7 +538,7 @@ TypeBasedAliasAnalysis::getModRefInfo(ImmutableCallSite CS1,
     if (const MDNode *M2 =
             CS2.getInstruction()->getMetadata(LLVMContext::MD_tbaa))
       if (!Aliases(M1, M2))
-        return NoModRef;
+        return MRI_NoModRef;
 
   return AliasAnalysis::getModRefInfo(CS1, CS2);
 }
diff --git a/lib/Analysis/ValueTracking.cpp b/lib/Analysis/ValueTracking.cpp
index 36513018ac1..b761a5c1b43 100644
--- a/lib/Analysis/ValueTracking.cpp
+++ b/lib/Analysis/ValueTracking.cpp
@@ -17,6 +17,7 @@
 #include "llvm/Analysis/AssumptionCache.h"
 #include "llvm/Analysis/InstructionSimplify.h"
 #include "llvm/Analysis/MemoryBuiltins.h"
+#include "llvm/Analysis/LoopInfo.h"
 #include "llvm/IR/CallSite.h"
 #include "llvm/IR/ConstantRange.h"
 #include "llvm/IR/Constants.h"
@@ -31,6 +32,7 @@
 #include "llvm/IR/Metadata.h"
 #include "llvm/IR/Operator.h"
 #include "llvm/IR/PatternMatch.h"
+#include "llvm/IR/Statepoint.h"
 #include "llvm/Support/Debug.h"
 #include "llvm/Support/MathExtras.h"
 #include <cstring>
@@ -136,6 +138,21 @@ void llvm::computeKnownBits(Value *V, APInt &KnownZero, APInt &KnownOne,
                      Query(AC, safeCxtI(V, CxtI), DT));
 }
 
+bool llvm::haveNoCommonBitsSet(Value *LHS, Value *RHS, const DataLayout &DL,
+                               AssumptionCache *AC, const Instruction *CxtI,
+                               const DominatorTree *DT) {
+  assert(LHS->getType() == RHS->getType() &&
+         "LHS and RHS should have the same type");
+  assert(LHS->getType()->isIntOrIntVectorTy() &&
+         "LHS and RHS should be integers");
+  IntegerType *IT = cast<IntegerType>(LHS->getType()->getScalarType());
+  APInt LHSKnownZero(IT->getBitWidth(), 0), LHSKnownOne(IT->getBitWidth(), 0);
+  APInt RHSKnownZero(IT->getBitWidth(), 0), RHSKnownOne(IT->getBitWidth(), 0);
+  computeKnownBits(LHS, LHSKnownZero, LHSKnownOne, DL, 0, AC, CxtI, DT);
+  computeKnownBits(RHS, RHSKnownZero, RHSKnownOne, DL, 0, AC, CxtI, DT);
+  return (LHSKnownZero | RHSKnownZero).isAllOnesValue();
+}
+
 static void ComputeSignBit(Value *V, bool &KnownZero, bool &KnownOne,
                            const DataLayout &DL, unsigned Depth,
                            const Query &Q);
@@ -534,12 +551,17 @@ static void computeKnownBitsFromTrueCondition(Value *V, ICmpInst *Cmp,
     }
     break;
   case ICmpInst::ICMP_EQ:
-    if (LHS == V)
-      computeKnownBits(RHS, KnownZero, KnownOne, DL, Depth + 1, Q);
-    else if (RHS == V)
-      computeKnownBits(LHS, KnownZero, KnownOne, DL, Depth + 1, Q);
-    else
-      llvm_unreachable("missing use?");
+    {
+      APInt KnownZeroTemp(BitWidth, 0), KnownOneTemp(BitWidth, 0);
+      if (LHS == V)
+        computeKnownBits(RHS, KnownZeroTemp, KnownOneTemp, DL, Depth + 1, Q);
+      else if (RHS == V)
+        computeKnownBits(LHS, KnownZeroTemp, KnownOneTemp, DL, Depth + 1, Q);
+      else
+        llvm_unreachable("missing use?");
+      KnownZero |= KnownZeroTemp;
+      KnownOne |= KnownOneTemp;
+    }
     break;
   case ICmpInst::ICMP_ULE:
     if (LHS == V) {
@@ -919,147 +941,11 @@ static void computeKnownBitsFromAssume(Value *V, APInt &KnownZero,
   }
 }
 
-/// Determine which bits of V are known to be either zero or one and return
-/// them in the KnownZero/KnownOne bit sets.
-///
-/// NOTE: we cannot consider 'undef' to be "IsZero" here.  The problem is that
-/// we cannot optimize based on the assumption that it is zero without changing
-/// it to be an explicit zero.  If we don't change it to zero, other code could
-/// optimized based on the contradictory assumption that it is non-zero.
-/// Because instcombine aggressively folds operations with undef args anyway,
-/// this won't lose us code quality.
-///
-/// This function is defined on values with integer type, values with pointer
-/// type, and vectors of integers.  In the case
-/// where V is a vector, known zero, and known one values are the
-/// same width as the vector element, and the bit is set only if it is true
-/// for all of the elements in the vector.
-void computeKnownBits(Value *V, APInt &KnownZero, APInt &KnownOne,
-                      const DataLayout &DL, unsigned Depth, const Query &Q) {
-  assert(V && "No Value?");
-  assert(Depth <= MaxDepth && "Limit Search Depth");
+static void computeKnownBitsFromOperator(Operator *I, APInt &KnownZero,
+                                         APInt &KnownOne, const DataLayout &DL,
+                                         unsigned Depth, const Query &Q) {
   unsigned BitWidth = KnownZero.getBitWidth();
 
-  assert((V->getType()->isIntOrIntVectorTy() ||
-          V->getType()->getScalarType()->isPointerTy()) &&
-         "Not integer or pointer type!");
-  assert((DL.getTypeSizeInBits(V->getType()->getScalarType()) == BitWidth) &&
-         (!V->getType()->isIntOrIntVectorTy() ||
-          V->getType()->getScalarSizeInBits() == BitWidth) &&
-         KnownZero.getBitWidth() == BitWidth &&
-         KnownOne.getBitWidth() == BitWidth &&
-         "V, KnownOne and KnownZero should have same BitWidth");
-
-  if (ConstantInt *CI = dyn_cast<ConstantInt>(V)) {
-    // We know all of the bits for a constant!
-    KnownOne = CI->getValue();
-    KnownZero = ~KnownOne;
-    return;
-  }
-  // Null and aggregate-zero are all-zeros.
-  if (isa<ConstantPointerNull>(V) ||
-      isa<ConstantAggregateZero>(V)) {
-    KnownOne.clearAllBits();
-    KnownZero = APInt::getAllOnesValue(BitWidth);
-    return;
-  }
-  // Handle a constant vector by taking the intersection of the known bits of
-  // each element.  There is no real need to handle ConstantVector here, because
-  // we don't handle undef in any particularly useful way.
-  if (ConstantDataSequential *CDS = dyn_cast<ConstantDataSequential>(V)) {
-    // We know that CDS must be a vector of integers. Take the intersection of
-    // each element.
-    KnownZero.setAllBits(); KnownOne.setAllBits();
-    APInt Elt(KnownZero.getBitWidth(), 0);
-    for (unsigned i = 0, e = CDS->getNumElements(); i != e; ++i) {
-      Elt = CDS->getElementAsInteger(i);
-      KnownZero &= ~Elt;
-      KnownOne &= Elt;
-    }
-    return;
-  }
-
-  // The address of an aligned GlobalValue has trailing zeros.
-  if (auto *GO = dyn_cast<GlobalObject>(V)) {
-    unsigned Align = GO->getAlignment();
-    if (Align == 0) {
-      if (auto *GVar = dyn_cast<GlobalVariable>(GO)) {
-        Type *ObjectType = GVar->getType()->getElementType();
-        if (ObjectType->isSized()) {
-          // If the object is defined in the current Module, we'll be giving
-          // it the preferred alignment. Otherwise, we have to assume that it
-          // may only have the minimum ABI alignment.
-          if (!GVar->isDeclaration() && !GVar->isWeakForLinker())
-            Align = DL.getPreferredAlignment(GVar);
-          else
-            Align = DL.getABITypeAlignment(ObjectType);
-        }
-      }
-    }
-    if (Align > 0)
-      KnownZero = APInt::getLowBitsSet(BitWidth,
-                                       countTrailingZeros(Align));
-    else
-      KnownZero.clearAllBits();
-    KnownOne.clearAllBits();
-    return;
-  }
-
-  if (Argument *A = dyn_cast<Argument>(V)) {
-    unsigned Align = A->getType()->isPointerTy() ? A->getParamAlignment() : 0;
-
-    if (!Align && A->hasStructRetAttr()) {
-      // An sret parameter has at least the ABI alignment of the return type.
-      Type *EltTy = cast<PointerType>(A->getType())->getElementType();
-      if (EltTy->isSized())
-        Align = DL.getABITypeAlignment(EltTy);
-    }
-
-    if (Align)
-      KnownZero = APInt::getLowBitsSet(BitWidth, countTrailingZeros(Align));
-    else
-      KnownZero.clearAllBits();
-    KnownOne.clearAllBits();
-
-    // Don't give up yet... there might be an assumption that provides more
-    // information...
-    computeKnownBitsFromAssume(V, KnownZero, KnownOne, DL, Depth, Q);
-
-    // Or a dominating condition for that matter
-    if (EnableDomConditions && Depth <= DomConditionsMaxDepth)
-      computeKnownBitsFromDominatingCondition(V, KnownZero, KnownOne, DL,
-                                              Depth, Q);
-    return;
-  }
-
-  // Start out not knowing anything.
-  KnownZero.clearAllBits(); KnownOne.clearAllBits();
-
-  // Limit search depth.
-  // All recursive calls that increase depth must come after this.
-  if (Depth == MaxDepth)
-    return;  
-
-  // A weak GlobalAlias is totally unknown. A non-weak GlobalAlias has
-  // the bits of its aliasee.
-  if (GlobalAlias *GA = dyn_cast<GlobalAlias>(V)) {
-    if (!GA->mayBeOverridden())
-      computeKnownBits(GA->getAliasee(), KnownZero, KnownOne, DL, Depth + 1, Q);
-    return;
-  }
-
-  // Check whether a nearby assume intrinsic can determine some known bits.
-  computeKnownBitsFromAssume(V, KnownZero, KnownOne, DL, Depth, Q);
-
-  // Check whether there's a dominating condition which implies something about
-  // this value at the given context.
-  if (EnableDomConditions && Depth <= DomConditionsMaxDepth)
-    computeKnownBitsFromDominatingCondition(V, KnownZero, KnownOne, DL, Depth,
-                                            Q);
-
-  Operator *I = dyn_cast<Operator>(V);
-  if (!I) return;
-
   APInt KnownZero2(KnownZero), KnownOne2(KnownOne);
   switch (I->getOpcode()) {
   default: break;
@@ -1311,7 +1197,7 @@ void computeKnownBits(Value *V, APInt &KnownZero, APInt &KnownOne,
   }
 
   case Instruction::Alloca: {
-    AllocaInst *AI = cast<AllocaInst>(V);
+    AllocaInst *AI = cast<AllocaInst>(I);
     unsigned Align = AI->getAlignment();
     if (Align == 0)
       Align = DL.getABITypeAlignment(AI->getType()->getElementType());
@@ -1427,15 +1313,15 @@ void computeKnownBits(Value *V, APInt &KnownZero, APInt &KnownOne,
 
       KnownZero = APInt::getAllOnesValue(BitWidth);
       KnownOne = APInt::getAllOnesValue(BitWidth);
-      for (unsigned i = 0, e = P->getNumIncomingValues(); i != e; ++i) {
+      for (Value *IncValue : P->incoming_values()) {
         // Skip direct self references.
-        if (P->getIncomingValue(i) == P) continue;
+        if (IncValue == P) continue;
 
         KnownZero2 = APInt(BitWidth, 0);
         KnownOne2 = APInt(BitWidth, 0);
         // Recurse, but cap the recursion to one level, because we don't
         // want to waste time spinning around in loops.
-        computeKnownBits(P->getIncomingValue(i), KnownZero2, KnownOne2, DL,
+        computeKnownBits(IncValue, KnownZero2, KnownOne2, DL,
                          MaxDepth - 1, Q);
         KnownZero &= KnownZero2;
         KnownOne &= KnownOne2;
@@ -1506,6 +1392,151 @@ void computeKnownBits(Value *V, APInt &KnownZero, APInt &KnownOne,
       }
     }
   }
+}
+
+/// Determine which bits of V are known to be either zero or one and return
+/// them in the KnownZero/KnownOne bit sets.
+///
+/// NOTE: we cannot consider 'undef' to be "IsZero" here.  The problem is that
+/// we cannot optimize based on the assumption that it is zero without changing
+/// it to be an explicit zero.  If we don't change it to zero, other code could
+/// optimized based on the contradictory assumption that it is non-zero.
+/// Because instcombine aggressively folds operations with undef args anyway,
+/// this won't lose us code quality.
+///
+/// This function is defined on values with integer type, values with pointer
+/// type, and vectors of integers.  In the case
+/// where V is a vector, known zero, and known one values are the
+/// same width as the vector element, and the bit is set only if it is true
+/// for all of the elements in the vector.
+void computeKnownBits(Value *V, APInt &KnownZero, APInt &KnownOne,
+                      const DataLayout &DL, unsigned Depth, const Query &Q) {
+  assert(V && "No Value?");
+  assert(Depth <= MaxDepth && "Limit Search Depth");
+  unsigned BitWidth = KnownZero.getBitWidth();
+
+  assert((V->getType()->isIntOrIntVectorTy() ||
+          V->getType()->getScalarType()->isPointerTy()) &&
+         "Not integer or pointer type!");
+  assert((DL.getTypeSizeInBits(V->getType()->getScalarType()) == BitWidth) &&
+         (!V->getType()->isIntOrIntVectorTy() ||
+          V->getType()->getScalarSizeInBits() == BitWidth) &&
+         KnownZero.getBitWidth() == BitWidth &&
+         KnownOne.getBitWidth() == BitWidth &&
+         "V, KnownOne and KnownZero should have same BitWidth");
+
+  if (ConstantInt *CI = dyn_cast<ConstantInt>(V)) {
+    // We know all of the bits for a constant!
+    KnownOne = CI->getValue();
+    KnownZero = ~KnownOne;
+    return;
+  }
+  // Null and aggregate-zero are all-zeros.
+  if (isa<ConstantPointerNull>(V) ||
+      isa<ConstantAggregateZero>(V)) {
+    KnownOne.clearAllBits();
+    KnownZero = APInt::getAllOnesValue(BitWidth);
+    return;
+  }
+  // Handle a constant vector by taking the intersection of the known bits of
+  // each element.  There is no real need to handle ConstantVector here, because
+  // we don't handle undef in any particularly useful way.
+  if (ConstantDataSequential *CDS = dyn_cast<ConstantDataSequential>(V)) {
+    // We know that CDS must be a vector of integers. Take the intersection of
+    // each element.
+    KnownZero.setAllBits(); KnownOne.setAllBits();
+    APInt Elt(KnownZero.getBitWidth(), 0);
+    for (unsigned i = 0, e = CDS->getNumElements(); i != e; ++i) {
+      Elt = CDS->getElementAsInteger(i);
+      KnownZero &= ~Elt;
+      KnownOne &= Elt;
+    }
+    return;
+  }
+
+  // The address of an aligned GlobalValue has trailing zeros.
+  if (auto *GO = dyn_cast<GlobalObject>(V)) {
+    unsigned Align = GO->getAlignment();
+    if (Align == 0) {
+      if (auto *GVar = dyn_cast<GlobalVariable>(GO)) {
+        Type *ObjectType = GVar->getType()->getElementType();
+        if (ObjectType->isSized()) {
+          // If the object is defined in the current Module, we'll be giving
+          // it the preferred alignment. Otherwise, we have to assume that it
+          // may only have the minimum ABI alignment.
+          if (GVar->isStrongDefinitionForLinker())
+            Align = DL.getPreferredAlignment(GVar);
+          else
+            Align = DL.getABITypeAlignment(ObjectType);
+        }
+      }
+    }
+    if (Align > 0)
+      KnownZero = APInt::getLowBitsSet(BitWidth,
+                                       countTrailingZeros(Align));
+    else
+      KnownZero.clearAllBits();
+    KnownOne.clearAllBits();
+    return;
+  }
+
+  if (Argument *A = dyn_cast<Argument>(V)) {
+    unsigned Align = A->getType()->isPointerTy() ? A->getParamAlignment() : 0;
+
+    if (!Align && A->hasStructRetAttr()) {
+      // An sret parameter has at least the ABI alignment of the return type.
+      Type *EltTy = cast<PointerType>(A->getType())->getElementType();
+      if (EltTy->isSized())
+        Align = DL.getABITypeAlignment(EltTy);
+    }
+
+    if (Align)
+      KnownZero = APInt::getLowBitsSet(BitWidth, countTrailingZeros(Align));
+    else
+      KnownZero.clearAllBits();
+    KnownOne.clearAllBits();
+
+    // Don't give up yet... there might be an assumption that provides more
+    // information...
+    computeKnownBitsFromAssume(V, KnownZero, KnownOne, DL, Depth, Q);
+
+    // Or a dominating condition for that matter
+    if (EnableDomConditions && Depth <= DomConditionsMaxDepth)
+      computeKnownBitsFromDominatingCondition(V, KnownZero, KnownOne, DL,
+                                              Depth, Q);
+    return;
+  }
+
+  // Start out not knowing anything.
+  KnownZero.clearAllBits(); KnownOne.clearAllBits();
+
+  // Limit search depth.
+  // All recursive calls that increase depth must come after this.
+  if (Depth == MaxDepth)
+    return;
+
+  // A weak GlobalAlias is totally unknown. A non-weak GlobalAlias has
+  // the bits of its aliasee.
+  if (GlobalAlias *GA = dyn_cast<GlobalAlias>(V)) {
+    if (!GA->mayBeOverridden())
+      computeKnownBits(GA->getAliasee(), KnownZero, KnownOne, DL, Depth + 1, Q);
+    return;
+  }
+
+  if (Operator *I = dyn_cast<Operator>(V))
+    computeKnownBitsFromOperator(I, KnownZero, KnownOne, DL, Depth, Q);
+  // computeKnownBitsFromAssume and computeKnownBitsFromDominatingCondition
+  // strictly refines KnownZero and KnownOne. Therefore, we run them after
+  // computeKnownBitsFromOperator.
+
+  // Check whether a nearby assume intrinsic can determine some known bits.
+  computeKnownBitsFromAssume(V, KnownZero, KnownOne, DL, Depth, Q);
+
+  // Check whether there's a dominating condition which implies something about
+  // this value at the given context.
+  if (EnableDomConditions && Depth <= DomConditionsMaxDepth)
+    computeKnownBitsFromDominatingCondition(V, KnownZero, KnownOne, DL, Depth,
+                                            Q);
 
   assert((KnownZero & KnownOne) == 0 && "Bits known to be one AND zero?");
 }
@@ -2689,8 +2720,8 @@ static uint64_t GetStringLengthH(Value *V, SmallPtrSetImpl<PHINode*> &PHIs) {
 
     // If it was new, see if all the input strings are the same length.
     uint64_t LenSoFar = ~0ULL;
-    for (unsigned i = 0, e = PN->getNumIncomingValues(); i != e; ++i) {
-      uint64_t Len = GetStringLengthH(PN->getIncomingValue(i), PHIs);
+    for (Value *IncValue : PN->incoming_values()) {
+      uint64_t Len = GetStringLengthH(IncValue, PHIs);
       if (Len == 0) return 0; // Unknown length -> unknown.
 
       if (Len == ~0ULL) continue;
@@ -2736,6 +2767,32 @@ uint64_t llvm::GetStringLength(Value *V) {
   return Len == ~0ULL ? 1 : Len;
 }
 
+/// \brief \p PN defines a loop-variant pointer to an object.  Check if the
+/// previous iteration of the loop was referring to the same object as \p PN.
+static bool isSameUnderlyingObjectInLoop(PHINode *PN, LoopInfo *LI) {
+  // Find the loop-defined value.
+  Loop *L = LI->getLoopFor(PN->getParent());
+  if (PN->getNumIncomingValues() != 2)
+    return true;
+
+  // Find the value from previous iteration.
+  auto *PrevValue = dyn_cast<Instruction>(PN->getIncomingValue(0));
+  if (!PrevValue || LI->getLoopFor(PrevValue->getParent()) != L)
+    PrevValue = dyn_cast<Instruction>(PN->getIncomingValue(1));
+  if (!PrevValue || LI->getLoopFor(PrevValue->getParent()) != L)
+    return true;
+
+  // If a new pointer is loaded in the loop, the pointer references a different
+  // object in every iteration.  E.g.:
+  //    for (i)
+  //       int *p = a[i];
+  //       ...
+  if (auto *Load = dyn_cast<LoadInst>(PrevValue))
+    if (!L->isLoopInvariant(Load->getPointerOperand()))
+      return false;
+  return true;
+}
+
 Value *llvm::GetUnderlyingObject(Value *V, const DataLayout &DL,
                                  unsigned MaxLookup) {
   if (!V->getType()->isPointerTy())
@@ -2767,7 +2824,8 @@ Value *llvm::GetUnderlyingObject(Value *V, const DataLayout &DL,
 }
 
 void llvm::GetUnderlyingObjects(Value *V, SmallVectorImpl<Value *> &Objects,
-                                const DataLayout &DL, unsigned MaxLookup) {
+                                const DataLayout &DL, LoopInfo *LI,
+                                unsigned MaxLookup) {
   SmallPtrSet<Value *, 4> Visited;
   SmallVector<Value *, 4> Worklist;
   Worklist.push_back(V);
@@ -2785,8 +2843,20 @@ void llvm::GetUnderlyingObjects(Value *V, SmallVectorImpl<Value *> &Objects,
     }
 
     if (PHINode *PN = dyn_cast<PHINode>(P)) {
-      for (unsigned i = 0, e = PN->getNumIncomingValues(); i != e; ++i)
-        Worklist.push_back(PN->getIncomingValue(i));
+      // If this PHI changes the underlying object in every iteration of the
+      // loop, don't look through it.  Consider:
+      //   int **A;
+      //   for (i) {
+      //     Prev = Curr;     // Prev = PHI (Prev_0, Curr)
+      //     Curr = A[i];
+      //     *Prev, *Curr;
+      //
+      // Prev is tracking Curr one iteration behind so they refer to different
+      // underlying objects.
+      if (!LI || !LI->isLoopHeader(PN->getParent()) ||
+          isSameUnderlyingObjectInLoop(PN, LI))
+        for (Value *IncValue : PN->incoming_values())
+          Worklist.push_back(IncValue);
       continue;
     }
 
@@ -2807,7 +2877,175 @@ bool llvm::onlyUsedByLifetimeMarkers(const Value *V) {
   return true;
 }
 
-bool llvm::isSafeToSpeculativelyExecute(const Value *V) {
+static bool isDereferenceableFromAttribute(const Value *BV, APInt Offset,
+                                           Type *Ty, const DataLayout &DL,
+                                           const Instruction *CtxI,
+                                           const DominatorTree *DT,
+                                           const TargetLibraryInfo *TLI) {
+  assert(Offset.isNonNegative() && "offset can't be negative");
+  assert(Ty->isSized() && "must be sized");
+  
+  APInt DerefBytes(Offset.getBitWidth(), 0);
+  bool CheckForNonNull = false;
+  if (const Argument *A = dyn_cast<Argument>(BV)) {
+    DerefBytes = A->getDereferenceableBytes();
+    if (!DerefBytes.getBoolValue()) {
+      DerefBytes = A->getDereferenceableOrNullBytes();
+      CheckForNonNull = true;
+    }
+  } else if (auto CS = ImmutableCallSite(BV)) {
+    DerefBytes = CS.getDereferenceableBytes(0);
+    if (!DerefBytes.getBoolValue()) {
+      DerefBytes = CS.getDereferenceableOrNullBytes(0);
+      CheckForNonNull = true;
+    }
+  } else if (const LoadInst *LI = dyn_cast<LoadInst>(BV)) {
+    if (MDNode *MD = LI->getMetadata(LLVMContext::MD_dereferenceable)) {
+      ConstantInt *CI = mdconst::extract<ConstantInt>(MD->getOperand(0));
+      DerefBytes = CI->getLimitedValue();
+    }
+    if (!DerefBytes.getBoolValue()) {
+      if (MDNode *MD = 
+              LI->getMetadata(LLVMContext::MD_dereferenceable_or_null)) {
+        ConstantInt *CI = mdconst::extract<ConstantInt>(MD->getOperand(0));
+        DerefBytes = CI->getLimitedValue();
+      }
+      CheckForNonNull = true;
+    }
+  }
+  
+  if (DerefBytes.getBoolValue())
+    if (DerefBytes.uge(Offset + DL.getTypeStoreSize(Ty)))
+      if (!CheckForNonNull || isKnownNonNullAt(BV, CtxI, DT, TLI))
+        return true;
+
+  return false;
+}
+
+static bool isDereferenceableFromAttribute(const Value *V, const DataLayout &DL,
+                                           const Instruction *CtxI,
+                                           const DominatorTree *DT,
+                                           const TargetLibraryInfo *TLI) {
+  Type *VTy = V->getType();
+  Type *Ty = VTy->getPointerElementType();
+  if (!Ty->isSized())
+    return false;
+  
+  APInt Offset(DL.getTypeStoreSizeInBits(VTy), 0);
+  return isDereferenceableFromAttribute(V, Offset, Ty, DL, CtxI, DT, TLI);
+}
+
+/// Return true if Value is always a dereferenceable pointer.
+///
+/// Test if V is always a pointer to allocated and suitably aligned memory for
+/// a simple load or store.
+static bool isDereferenceablePointer(const Value *V, const DataLayout &DL,
+                                     const Instruction *CtxI,
+                                     const DominatorTree *DT,
+                                     const TargetLibraryInfo *TLI,
+                                     SmallPtrSetImpl<const Value *> &Visited) {
+  // Note that it is not safe to speculate into a malloc'd region because
+  // malloc may return null.
+
+  // These are obviously ok.
+  if (isa<AllocaInst>(V)) return true;
+
+  // It's not always safe to follow a bitcast, for example:
+  //   bitcast i8* (alloca i8) to i32*
+  // would result in a 4-byte load from a 1-byte alloca. However,
+  // if we're casting from a pointer from a type of larger size
+  // to a type of smaller size (or the same size), and the alignment
+  // is at least as large as for the resulting pointer type, then
+  // we can look through the bitcast.
+  if (const BitCastOperator *BC = dyn_cast<BitCastOperator>(V)) {
+    Type *STy = BC->getSrcTy()->getPointerElementType(),
+         *DTy = BC->getDestTy()->getPointerElementType();
+    if (STy->isSized() && DTy->isSized() &&
+        (DL.getTypeStoreSize(STy) >= DL.getTypeStoreSize(DTy)) &&
+        (DL.getABITypeAlignment(STy) >= DL.getABITypeAlignment(DTy)))
+      return isDereferenceablePointer(BC->getOperand(0), DL, CtxI,
+                                      DT, TLI, Visited);
+  }
+
+  // Global variables which can't collapse to null are ok.
+  if (const GlobalVariable *GV = dyn_cast<GlobalVariable>(V))
+    return !GV->hasExternalWeakLinkage();
+
+  // byval arguments are okay.
+  if (const Argument *A = dyn_cast<Argument>(V))
+    if (A->hasByValAttr())
+      return true;
+    
+  if (isDereferenceableFromAttribute(V, DL, CtxI, DT, TLI))
+    return true;
+
+  // For GEPs, determine if the indexing lands within the allocated object.
+  if (const GEPOperator *GEP = dyn_cast<GEPOperator>(V)) {
+    Type *VTy = GEP->getType();
+    Type *Ty = VTy->getPointerElementType();
+    const Value *Base = GEP->getPointerOperand();
+
+    // Conservatively require that the base pointer be fully dereferenceable.
+    if (!Visited.insert(Base).second)
+      return false;
+    if (!isDereferenceablePointer(Base, DL, CtxI,
+                                  DT, TLI, Visited))
+      return false;
+    
+    APInt Offset(DL.getPointerTypeSizeInBits(VTy), 0);
+    if (!GEP->accumulateConstantOffset(DL, Offset))
+      return false;
+    
+    // Check if the load is within the bounds of the underlying object.
+    uint64_t LoadSize = DL.getTypeStoreSize(Ty);
+    Type *BaseType = Base->getType()->getPointerElementType();
+    return (Offset + LoadSize).ule(DL.getTypeAllocSize(BaseType));
+  }
+
+  // For gc.relocate, look through relocations
+  if (const IntrinsicInst *I = dyn_cast<IntrinsicInst>(V))
+    if (I->getIntrinsicID() == Intrinsic::experimental_gc_relocate) {
+      GCRelocateOperands RelocateInst(I);
+      return isDereferenceablePointer(RelocateInst.getDerivedPtr(), DL, CtxI,
+                                      DT, TLI, Visited);
+    }
+
+  if (const AddrSpaceCastInst *ASC = dyn_cast<AddrSpaceCastInst>(V))
+    return isDereferenceablePointer(ASC->getOperand(0), DL, CtxI,
+                                    DT, TLI, Visited);
+
+  // If we don't know, assume the worst.
+  return false;
+}
+
+bool llvm::isDereferenceablePointer(const Value *V, const DataLayout &DL,
+                                    const Instruction *CtxI,
+                                    const DominatorTree *DT,
+                                    const TargetLibraryInfo *TLI) {
+  // When dereferenceability information is provided by a dereferenceable
+  // attribute, we know exactly how many bytes are dereferenceable. If we can
+  // determine the exact offset to the attributed variable, we can use that
+  // information here.
+  Type *VTy = V->getType();
+  Type *Ty = VTy->getPointerElementType();
+  if (Ty->isSized()) {
+    APInt Offset(DL.getTypeStoreSizeInBits(VTy), 0);
+    const Value *BV = V->stripAndAccumulateInBoundsConstantOffsets(DL, Offset);
+    
+    if (Offset.isNonNegative())
+      if (isDereferenceableFromAttribute(BV, Offset, Ty, DL,
+                                         CtxI, DT, TLI))
+        return true;
+  }
+
+  SmallPtrSet<const Value *, 32> Visited;
+  return ::isDereferenceablePointer(V, DL, CtxI, DT, TLI, Visited);
+}
+
+bool llvm::isSafeToSpeculativelyExecute(const Value *V,
+                                        const Instruction *CtxI,
+                                        const DominatorTree *DT,
+                                        const TargetLibraryInfo *TLI) {
   const Operator *Inst = dyn_cast<Operator>(V);
   if (!Inst)
     return false;
@@ -2854,7 +3092,7 @@ bool llvm::isSafeToSpeculativelyExecute(const Value *V) {
         LI->getParent()->getParent()->hasFnAttribute(Attribute::SanitizeThread))
       return false;
     const DataLayout &DL = LI->getModule()->getDataLayout();
-    return LI->getPointerOperand()->isDereferenceablePointer(DL);
+    return isDereferenceablePointer(LI->getPointerOperand(), DL, CtxI, DT, TLI);
   }
   case Instruction::Call: {
     if (const IntrinsicInst *II = dyn_cast<IntrinsicInst>(Inst)) {
@@ -2909,10 +3147,16 @@ bool llvm::isSafeToSpeculativelyExecute(const Value *V) {
   case Instruction::Switch:
   case Instruction::Unreachable:
   case Instruction::Fence:
-  case Instruction::LandingPad:
   case Instruction::AtomicRMW:
   case Instruction::AtomicCmpXchg:
+  case Instruction::LandingPad:
   case Instruction::Resume:
+  case Instruction::CatchPad:
+  case Instruction::CatchEndPad:
+  case Instruction::CatchRet:
+  case Instruction::CleanupPad:
+  case Instruction::CleanupRet:
+  case Instruction::TerminatePad:
     return false; // Misc instructions which have effects
   }
 }
@@ -2945,6 +3189,60 @@ bool llvm::isKnownNonNull(const Value *V, const TargetLibraryInfo *TLI) {
   return false;
 }
 
+static bool isKnownNonNullFromDominatingCondition(const Value *V,
+                                                  const Instruction *CtxI,
+                                                  const DominatorTree *DT) {
+  unsigned NumUsesExplored = 0;
+  for (auto U : V->users()) {
+    // Avoid massive lists
+    if (NumUsesExplored >= DomConditionsMaxUses)
+      break;
+    NumUsesExplored++;
+    // Consider only compare instructions uniquely controlling a branch
+    const ICmpInst *Cmp = dyn_cast<ICmpInst>(U);
+    if (!Cmp)
+      continue;
+
+    if (DomConditionsSingleCmpUse && !Cmp->hasOneUse())
+      continue;
+
+    for (auto *CmpU : Cmp->users()) {
+      const BranchInst *BI = dyn_cast<BranchInst>(CmpU);
+      if (!BI)
+        continue;
+      
+      assert(BI->isConditional() && "uses a comparison!");
+
+      BasicBlock *NonNullSuccessor = nullptr;
+      CmpInst::Predicate Pred;
+
+      if (match(const_cast<ICmpInst*>(Cmp),
+                m_c_ICmp(Pred, m_Specific(V), m_Zero()))) {
+        if (Pred == ICmpInst::ICMP_EQ)
+          NonNullSuccessor = BI->getSuccessor(1);
+        else if (Pred == ICmpInst::ICMP_NE)
+          NonNullSuccessor = BI->getSuccessor(0);
+      }
+
+      if (NonNullSuccessor) {
+        BasicBlockEdge Edge(BI->getParent(), NonNullSuccessor);
+        if (Edge.isSingleEdge() && DT->dominates(Edge, CtxI->getParent()))
+          return true;
+      }
+    }
+  }
+
+  return false;
+}
+
+bool llvm::isKnownNonNullAt(const Value *V, const Instruction *CtxI,
+                   const DominatorTree *DT, const TargetLibraryInfo *TLI) {
+  if (isKnownNonNull(V, TLI))
+    return true;
+
+  return CtxI ? ::isKnownNonNullFromDominatingCondition(V, CtxI, DT) : false;
+}
+
 OverflowResult llvm::computeOverflowForUnsignedMul(Value *LHS, Value *RHS,
                                                    const DataLayout &DL,
                                                    AssumptionCache *AC,
@@ -3023,3 +3321,294 @@ OverflowResult llvm::computeOverflowForUnsignedAdd(Value *LHS, Value *RHS,
 
   return OverflowResult::MayOverflow;
 }
+
+bool llvm::isGuaranteedToTransferExecutionToSuccessor(const Instruction *I) {
+  // FIXME: This conservative implementation can be relaxed. E.g. most
+  // atomic operations are guaranteed to terminate on most platforms
+  // and most functions terminate.
+
+  return !I->isAtomic() &&       // atomics may never succeed on some platforms
+         !isa<CallInst>(I) &&    // could throw and might not terminate
+         !isa<InvokeInst>(I) &&  // might not terminate and could throw to
+                                 //   non-successor (see bug 24185 for details).
+         !isa<ResumeInst>(I) &&  // has no successors
+         !isa<ReturnInst>(I);    // has no successors
+}
+
+bool llvm::isGuaranteedToExecuteForEveryIteration(const Instruction *I,
+                                                  const Loop *L) {
+  // The loop header is guaranteed to be executed for every iteration.
+  //
+  // FIXME: Relax this constraint to cover all basic blocks that are
+  // guaranteed to be executed at every iteration.
+  if (I->getParent() != L->getHeader()) return false;
+
+  for (const Instruction &LI : *L->getHeader()) {
+    if (&LI == I) return true;
+    if (!isGuaranteedToTransferExecutionToSuccessor(&LI)) return false;
+  }
+  llvm_unreachable("Instruction not contained in its own parent basic block.");
+}
+
+bool llvm::propagatesFullPoison(const Instruction *I) {
+  switch (I->getOpcode()) {
+    case Instruction::Add:
+    case Instruction::Sub:
+    case Instruction::Xor:
+    case Instruction::Trunc:
+    case Instruction::BitCast:
+    case Instruction::AddrSpaceCast:
+      // These operations all propagate poison unconditionally. Note that poison
+      // is not any particular value, so xor or subtraction of poison with
+      // itself still yields poison, not zero.
+      return true;
+
+    case Instruction::AShr:
+    case Instruction::SExt:
+      // For these operations, one bit of the input is replicated across
+      // multiple output bits. A replicated poison bit is still poison.
+      return true;
+
+    case Instruction::Shl: {
+      // Left shift *by* a poison value is poison. The number of
+      // positions to shift is unsigned, so no negative values are
+      // possible there. Left shift by zero places preserves poison. So
+      // it only remains to consider left shift of poison by a positive
+      // number of places.
+      //
+      // A left shift by a positive number of places leaves the lowest order bit
+      // non-poisoned. However, if such a shift has a no-wrap flag, then we can
+      // make the poison operand violate that flag, yielding a fresh full-poison
+      // value.
+      auto *OBO = cast<OverflowingBinaryOperator>(I);
+      return OBO->hasNoUnsignedWrap() || OBO->hasNoSignedWrap();
+    }
+
+    case Instruction::Mul: {
+      // A multiplication by zero yields a non-poison zero result, so we need to
+      // rule out zero as an operand. Conservatively, multiplication by a
+      // non-zero constant is not multiplication by zero.
+      //
+      // Multiplication by a non-zero constant can leave some bits
+      // non-poisoned. For example, a multiplication by 2 leaves the lowest
+      // order bit unpoisoned. So we need to consider that.
+      //
+      // Multiplication by 1 preserves poison. If the multiplication has a
+      // no-wrap flag, then we can make the poison operand violate that flag
+      // when multiplied by any integer other than 0 and 1.
+      auto *OBO = cast<OverflowingBinaryOperator>(I);
+      if (OBO->hasNoUnsignedWrap() || OBO->hasNoSignedWrap()) {
+        for (Value *V : OBO->operands()) {
+          if (auto *CI = dyn_cast<ConstantInt>(V)) {
+            // A ConstantInt cannot yield poison, so we can assume that it is
+            // the other operand that is poison.
+            return !CI->isZero();
+          }
+        }
+      }
+      return false;
+    }
+
+    case Instruction::GetElementPtr:
+      // A GEP implicitly represents a sequence of additions, subtractions,
+      // truncations, sign extensions and multiplications. The multiplications
+      // are by the non-zero sizes of some set of types, so we do not have to be
+      // concerned with multiplication by zero. If the GEP is in-bounds, then
+      // these operations are implicitly no-signed-wrap so poison is propagated
+      // by the arguments above for Add, Sub, Trunc, SExt and Mul.
+      return cast<GEPOperator>(I)->isInBounds();
+
+    default:
+      return false;
+  }
+}
+
+const Value *llvm::getGuaranteedNonFullPoisonOp(const Instruction *I) {
+  switch (I->getOpcode()) {
+    case Instruction::Store:
+      return cast<StoreInst>(I)->getPointerOperand();
+
+    case Instruction::Load:
+      return cast<LoadInst>(I)->getPointerOperand();
+
+    case Instruction::AtomicCmpXchg:
+      return cast<AtomicCmpXchgInst>(I)->getPointerOperand();
+
+    case Instruction::AtomicRMW:
+      return cast<AtomicRMWInst>(I)->getPointerOperand();
+
+    case Instruction::UDiv:
+    case Instruction::SDiv:
+    case Instruction::URem:
+    case Instruction::SRem:
+      return I->getOperand(1);
+
+    default:
+      return nullptr;
+  }
+}
+
+bool llvm::isKnownNotFullPoison(const Instruction *PoisonI) {
+  // We currently only look for uses of poison values within the same basic
+  // block, as that makes it easier to guarantee that the uses will be
+  // executed given that PoisonI is executed.
+  //
+  // FIXME: Expand this to consider uses beyond the same basic block. To do
+  // this, look out for the distinction between post-dominance and strong
+  // post-dominance.
+  const BasicBlock *BB = PoisonI->getParent();
+
+  // Set of instructions that we have proved will yield poison if PoisonI
+  // does.
+  SmallSet<const Value *, 16> YieldsPoison;
+  YieldsPoison.insert(PoisonI);
+
+  for (const Instruction *I = PoisonI, *E = BB->end(); I != E;
+       I = I->getNextNode()) {
+    if (I != PoisonI) {
+      const Value *NotPoison = getGuaranteedNonFullPoisonOp(I);
+      if (NotPoison != nullptr && YieldsPoison.count(NotPoison)) return true;
+      if (!isGuaranteedToTransferExecutionToSuccessor(I)) return false;
+    }
+
+    // Mark poison that propagates from I through uses of I.
+    if (YieldsPoison.count(I)) {
+      for (const User *User : I->users()) {
+        const Instruction *UserI = cast<Instruction>(User);
+        if (UserI->getParent() == BB && propagatesFullPoison(UserI))
+          YieldsPoison.insert(User);
+      }
+    }
+  }
+  return false;
+}
+
+static SelectPatternFlavor matchSelectPattern(ICmpInst::Predicate Pred,
+                                              Value *CmpLHS, Value *CmpRHS,
+                                              Value *TrueVal, Value *FalseVal,
+                                              Value *&LHS, Value *&RHS) {
+  LHS = CmpLHS;
+  RHS = CmpRHS;
+
+  // (icmp X, Y) ? X : Y
+  if (TrueVal == CmpLHS && FalseVal == CmpRHS) {
+    switch (Pred) {
+    default: return SPF_UNKNOWN; // Equality.
+    case ICmpInst::ICMP_UGT:
+    case ICmpInst::ICMP_UGE: return SPF_UMAX;
+    case ICmpInst::ICMP_SGT:
+    case ICmpInst::ICMP_SGE: return SPF_SMAX;
+    case ICmpInst::ICMP_ULT:
+    case ICmpInst::ICMP_ULE: return SPF_UMIN;
+    case ICmpInst::ICMP_SLT:
+    case ICmpInst::ICMP_SLE: return SPF_SMIN;
+    }
+  }
+
+  // (icmp X, Y) ? Y : X
+  if (TrueVal == CmpRHS && FalseVal == CmpLHS) {
+    switch (Pred) {
+    default: return SPF_UNKNOWN; // Equality.
+    case ICmpInst::ICMP_UGT:
+    case ICmpInst::ICMP_UGE: return SPF_UMIN;
+    case ICmpInst::ICMP_SGT:
+    case ICmpInst::ICMP_SGE: return SPF_SMIN;
+    case ICmpInst::ICMP_ULT:
+    case ICmpInst::ICMP_ULE: return SPF_UMAX;
+    case ICmpInst::ICMP_SLT:
+    case ICmpInst::ICMP_SLE: return SPF_SMAX;
+    }
+  }
+
+  if (ConstantInt *C1 = dyn_cast<ConstantInt>(CmpRHS)) {
+    if ((CmpLHS == TrueVal && match(FalseVal, m_Neg(m_Specific(CmpLHS)))) ||
+        (CmpLHS == FalseVal && match(TrueVal, m_Neg(m_Specific(CmpLHS))))) {
+
+      // ABS(X) ==> (X >s 0) ? X : -X and (X >s -1) ? X : -X
+      // NABS(X) ==> (X >s 0) ? -X : X and (X >s -1) ? -X : X
+      if (Pred == ICmpInst::ICMP_SGT && (C1->isZero() || C1->isMinusOne())) {
+        return (CmpLHS == TrueVal) ? SPF_ABS : SPF_NABS;
+      }
+
+      // ABS(X) ==> (X <s 0) ? -X : X and (X <s 1) ? -X : X
+      // NABS(X) ==> (X <s 0) ? X : -X and (X <s 1) ? X : -X
+      if (Pred == ICmpInst::ICMP_SLT && (C1->isZero() || C1->isOne())) {
+        return (CmpLHS == FalseVal) ? SPF_ABS : SPF_NABS;
+      }
+    }
+    
+    // Y >s C ? ~Y : ~C == ~Y <s ~C ? ~Y : ~C = SMIN(~Y, ~C)
+    if (const auto *C2 = dyn_cast<ConstantInt>(FalseVal)) {
+      if (C1->getType() == C2->getType() && ~C1->getValue() == C2->getValue() &&
+          (match(TrueVal, m_Not(m_Specific(CmpLHS))) ||
+           match(CmpLHS, m_Not(m_Specific(TrueVal))))) {
+        LHS = TrueVal;
+        RHS = FalseVal;
+        return SPF_SMIN;
+      }
+    }
+  }
+
+  // TODO: (X > 4) ? X : 5   -->  (X >= 5) ? X : 5  -->  MAX(X, 5)
+
+  return SPF_UNKNOWN;
+}
+
+static Constant *lookThroughCast(ICmpInst *CmpI, Value *V1, Value *V2,
+                                 Instruction::CastOps *CastOp) {
+  CastInst *CI = dyn_cast<CastInst>(V1);
+  Constant *C = dyn_cast<Constant>(V2);
+  if (!CI || !C)
+    return nullptr;
+  *CastOp = CI->getOpcode();
+
+  if (isa<SExtInst>(CI) && CmpI->isSigned()) {
+    Constant *T = ConstantExpr::getTrunc(C, CI->getSrcTy());
+    // This is only valid if the truncated value can be sign-extended
+    // back to the original value.
+    if (ConstantExpr::getSExt(T, C->getType()) == C)
+      return T;
+    return nullptr;
+  }
+  if (isa<ZExtInst>(CI) && CmpI->isUnsigned())
+    return ConstantExpr::getTrunc(C, CI->getSrcTy());
+
+  if (isa<TruncInst>(CI))
+    return ConstantExpr::getIntegerCast(C, CI->getSrcTy(), CmpI->isSigned());
+
+  return nullptr;
+}
+
+SelectPatternFlavor llvm::matchSelectPattern(Value *V,
+                                             Value *&LHS, Value *&RHS,
+                                             Instruction::CastOps *CastOp) {
+  SelectInst *SI = dyn_cast<SelectInst>(V);
+  if (!SI) return SPF_UNKNOWN;
+
+  ICmpInst *CmpI = dyn_cast<ICmpInst>(SI->getCondition());
+  if (!CmpI) return SPF_UNKNOWN;
+
+  ICmpInst::Predicate Pred = CmpI->getPredicate();
+  Value *CmpLHS = CmpI->getOperand(0);
+  Value *CmpRHS = CmpI->getOperand(1);
+  Value *TrueVal = SI->getTrueValue();
+  Value *FalseVal = SI->getFalseValue();
+
+  // Bail out early.
+  if (CmpI->isEquality())
+    return SPF_UNKNOWN;
+
+  // Deal with type mismatches.
+  if (CastOp && CmpLHS->getType() != TrueVal->getType()) {
+    if (Constant *C = lookThroughCast(CmpI, TrueVal, FalseVal, CastOp))
+      return ::matchSelectPattern(Pred, CmpLHS, CmpRHS,
+                                  cast<CastInst>(TrueVal)->getOperand(0), C,
+                                  LHS, RHS);
+    if (Constant *C = lookThroughCast(CmpI, FalseVal, TrueVal, CastOp))
+      return ::matchSelectPattern(Pred, CmpLHS, CmpRHS,
+                                  C, cast<CastInst>(FalseVal)->getOperand(0),
+                                  LHS, RHS);
+  }
+  return ::matchSelectPattern(Pred, CmpLHS, CmpRHS, TrueVal, FalseVal,
+                              LHS, RHS);
+}
diff --git a/lib/Analysis/VectorUtils.cpp b/lib/Analysis/VectorUtils.cpp
new file mode 100644
index 00000000000..67f68dc8391
--- /dev/null
+++ b/lib/Analysis/VectorUtils.cpp
@@ -0,0 +1,411 @@
+//===----------- VectorUtils.cpp - Vectorizer utility functions -----------===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This file defines vectorizer utilities.
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/Analysis/LoopInfo.h"
+#include "llvm/Analysis/ScalarEvolutionExpressions.h"
+#include "llvm/Analysis/ScalarEvolution.h"
+#include "llvm/Analysis/VectorUtils.h"
+#include "llvm/IR/GetElementPtrTypeIterator.h"
+#include "llvm/IR/PatternMatch.h"
+#include "llvm/IR/Value.h"
+
+/// \brief Identify if the intrinsic is trivially vectorizable.
+/// This method returns true if the intrinsic's argument types are all
+/// scalars for the scalar form of the intrinsic and all vectors for
+/// the vector form of the intrinsic.
+bool llvm::isTriviallyVectorizable(Intrinsic::ID ID) {
+  switch (ID) {
+  case Intrinsic::sqrt:
+  case Intrinsic::sin:
+  case Intrinsic::cos:
+  case Intrinsic::exp:
+  case Intrinsic::exp2:
+  case Intrinsic::log:
+  case Intrinsic::log10:
+  case Intrinsic::log2:
+  case Intrinsic::fabs:
+  case Intrinsic::minnum:
+  case Intrinsic::maxnum:
+  case Intrinsic::copysign:
+  case Intrinsic::floor:
+  case Intrinsic::ceil:
+  case Intrinsic::trunc:
+  case Intrinsic::rint:
+  case Intrinsic::nearbyint:
+  case Intrinsic::round:
+  case Intrinsic::bswap:
+  case Intrinsic::ctpop:
+  case Intrinsic::pow:
+  case Intrinsic::fma:
+  case Intrinsic::fmuladd:
+  case Intrinsic::ctlz:
+  case Intrinsic::cttz:
+  case Intrinsic::powi:
+    return true;
+  default:
+    return false;
+  }
+}
+
+/// \brief Identifies if the intrinsic has a scalar operand. It check for
+/// ctlz,cttz and powi special intrinsics whose argument is scalar.
+bool llvm::hasVectorInstrinsicScalarOpd(Intrinsic::ID ID,
+                                        unsigned ScalarOpdIdx) {
+  switch (ID) {
+  case Intrinsic::ctlz:
+  case Intrinsic::cttz:
+  case Intrinsic::powi:
+    return (ScalarOpdIdx == 1);
+  default:
+    return false;
+  }
+}
+
+/// \brief Check call has a unary float signature
+/// It checks following:
+/// a) call should have a single argument
+/// b) argument type should be floating point type
+/// c) call instruction type and argument type should be same
+/// d) call should only reads memory.
+/// If all these condition is met then return ValidIntrinsicID
+/// else return not_intrinsic.
+llvm::Intrinsic::ID
+llvm::checkUnaryFloatSignature(const CallInst &I,
+                               Intrinsic::ID ValidIntrinsicID) {
+  if (I.getNumArgOperands() != 1 ||
+      !I.getArgOperand(0)->getType()->isFloatingPointTy() ||
+      I.getType() != I.getArgOperand(0)->getType() || !I.onlyReadsMemory())
+    return Intrinsic::not_intrinsic;
+
+  return ValidIntrinsicID;
+}
+
+/// \brief Check call has a binary float signature
+/// It checks following:
+/// a) call should have 2 arguments.
+/// b) arguments type should be floating point type
+/// c) call instruction type and arguments type should be same
+/// d) call should only reads memory.
+/// If all these condition is met then return ValidIntrinsicID
+/// else return not_intrinsic.
+llvm::Intrinsic::ID
+llvm::checkBinaryFloatSignature(const CallInst &I,
+                                Intrinsic::ID ValidIntrinsicID) {
+  if (I.getNumArgOperands() != 2 ||
+      !I.getArgOperand(0)->getType()->isFloatingPointTy() ||
+      !I.getArgOperand(1)->getType()->isFloatingPointTy() ||
+      I.getType() != I.getArgOperand(0)->getType() ||
+      I.getType() != I.getArgOperand(1)->getType() || !I.onlyReadsMemory())
+    return Intrinsic::not_intrinsic;
+
+  return ValidIntrinsicID;
+}
+
+/// \brief Returns intrinsic ID for call.
+/// For the input call instruction it finds mapping intrinsic and returns
+/// its ID, in case it does not found it return not_intrinsic.
+llvm::Intrinsic::ID llvm::getIntrinsicIDForCall(CallInst *CI,
+                                                const TargetLibraryInfo *TLI) {
+  // If we have an intrinsic call, check if it is trivially vectorizable.
+  if (IntrinsicInst *II = dyn_cast<IntrinsicInst>(CI)) {
+    Intrinsic::ID ID = II->getIntrinsicID();
+    if (isTriviallyVectorizable(ID) || ID == Intrinsic::lifetime_start ||
+        ID == Intrinsic::lifetime_end || ID == Intrinsic::assume)
+      return ID;
+    return Intrinsic::not_intrinsic;
+  }
+
+  if (!TLI)
+    return Intrinsic::not_intrinsic;
+
+  LibFunc::Func Func;
+  Function *F = CI->getCalledFunction();
+  // We're going to make assumptions on the semantics of the functions, check
+  // that the target knows that it's available in this environment and it does
+  // not have local linkage.
+  if (!F || F->hasLocalLinkage() || !TLI->getLibFunc(F->getName(), Func))
+    return Intrinsic::not_intrinsic;
+
+  // Otherwise check if we have a call to a function that can be turned into a
+  // vector intrinsic.
+  switch (Func) {
+  default:
+    break;
+  case LibFunc::sin:
+  case LibFunc::sinf:
+  case LibFunc::sinl:
+    return checkUnaryFloatSignature(*CI, Intrinsic::sin);
+  case LibFunc::cos:
+  case LibFunc::cosf:
+  case LibFunc::cosl:
+    return checkUnaryFloatSignature(*CI, Intrinsic::cos);
+  case LibFunc::exp:
+  case LibFunc::expf:
+  case LibFunc::expl:
+    return checkUnaryFloatSignature(*CI, Intrinsic::exp);
+  case LibFunc::exp2:
+  case LibFunc::exp2f:
+  case LibFunc::exp2l:
+    return checkUnaryFloatSignature(*CI, Intrinsic::exp2);
+  case LibFunc::log:
+  case LibFunc::logf:
+  case LibFunc::logl:
+    return checkUnaryFloatSignature(*CI, Intrinsic::log);
+  case LibFunc::log10:
+  case LibFunc::log10f:
+  case LibFunc::log10l:
+    return checkUnaryFloatSignature(*CI, Intrinsic::log10);
+  case LibFunc::log2:
+  case LibFunc::log2f:
+  case LibFunc::log2l:
+    return checkUnaryFloatSignature(*CI, Intrinsic::log2);
+  case LibFunc::fabs:
+  case LibFunc::fabsf:
+  case LibFunc::fabsl:
+    return checkUnaryFloatSignature(*CI, Intrinsic::fabs);
+  case LibFunc::fmin:
+  case LibFunc::fminf:
+  case LibFunc::fminl:
+    return checkBinaryFloatSignature(*CI, Intrinsic::minnum);
+  case LibFunc::fmax:
+  case LibFunc::fmaxf:
+  case LibFunc::fmaxl:
+    return checkBinaryFloatSignature(*CI, Intrinsic::maxnum);
+  case LibFunc::copysign:
+  case LibFunc::copysignf:
+  case LibFunc::copysignl:
+    return checkBinaryFloatSignature(*CI, Intrinsic::copysign);
+  case LibFunc::floor:
+  case LibFunc::floorf:
+  case LibFunc::floorl:
+    return checkUnaryFloatSignature(*CI, Intrinsic::floor);
+  case LibFunc::ceil:
+  case LibFunc::ceilf:
+  case LibFunc::ceill:
+    return checkUnaryFloatSignature(*CI, Intrinsic::ceil);
+  case LibFunc::trunc:
+  case LibFunc::truncf:
+  case LibFunc::truncl:
+    return checkUnaryFloatSignature(*CI, Intrinsic::trunc);
+  case LibFunc::rint:
+  case LibFunc::rintf:
+  case LibFunc::rintl:
+    return checkUnaryFloatSignature(*CI, Intrinsic::rint);
+  case LibFunc::nearbyint:
+  case LibFunc::nearbyintf:
+  case LibFunc::nearbyintl:
+    return checkUnaryFloatSignature(*CI, Intrinsic::nearbyint);
+  case LibFunc::round:
+  case LibFunc::roundf:
+  case LibFunc::roundl:
+    return checkUnaryFloatSignature(*CI, Intrinsic::round);
+  case LibFunc::pow:
+  case LibFunc::powf:
+  case LibFunc::powl:
+    return checkBinaryFloatSignature(*CI, Intrinsic::pow);
+  }
+
+  return Intrinsic::not_intrinsic;
+}
+
+/// \brief Find the operand of the GEP that should be checked for consecutive
+/// stores. This ignores trailing indices that have no effect on the final
+/// pointer.
+unsigned llvm::getGEPInductionOperand(const GetElementPtrInst *Gep) {
+  const DataLayout &DL = Gep->getModule()->getDataLayout();
+  unsigned LastOperand = Gep->getNumOperands() - 1;
+  unsigned GEPAllocSize = DL.getTypeAllocSize(
+      cast<PointerType>(Gep->getType()->getScalarType())->getElementType());
+
+  // Walk backwards and try to peel off zeros.
+  while (LastOperand > 1 &&
+         match(Gep->getOperand(LastOperand), llvm::PatternMatch::m_Zero())) {
+    // Find the type we're currently indexing into.
+    gep_type_iterator GEPTI = gep_type_begin(Gep);
+    std::advance(GEPTI, LastOperand - 1);
+
+    // If it's a type with the same allocation size as the result of the GEP we
+    // can peel off the zero index.
+    if (DL.getTypeAllocSize(*GEPTI) != GEPAllocSize)
+      break;
+    --LastOperand;
+  }
+
+  return LastOperand;
+}
+
+/// \brief If the argument is a GEP, then returns the operand identified by
+/// getGEPInductionOperand. However, if there is some other non-loop-invariant
+/// operand, it returns that instead.
+llvm::Value *llvm::stripGetElementPtr(llvm::Value *Ptr, ScalarEvolution *SE,
+                                      Loop *Lp) {
+  GetElementPtrInst *GEP = dyn_cast<GetElementPtrInst>(Ptr);
+  if (!GEP)
+    return Ptr;
+
+  unsigned InductionOperand = getGEPInductionOperand(GEP);
+
+  // Check that all of the gep indices are uniform except for our induction
+  // operand.
+  for (unsigned i = 0, e = GEP->getNumOperands(); i != e; ++i)
+    if (i != InductionOperand &&
+        !SE->isLoopInvariant(SE->getSCEV(GEP->getOperand(i)), Lp))
+      return Ptr;
+  return GEP->getOperand(InductionOperand);
+}
+
+/// \brief If a value has only one user that is a CastInst, return it.
+llvm::Value *llvm::getUniqueCastUse(llvm::Value *Ptr, Loop *Lp, Type *Ty) {
+  llvm::Value *UniqueCast = nullptr;
+  for (User *U : Ptr->users()) {
+    CastInst *CI = dyn_cast<CastInst>(U);
+    if (CI && CI->getType() == Ty) {
+      if (!UniqueCast)
+        UniqueCast = CI;
+      else
+        return nullptr;
+    }
+  }
+  return UniqueCast;
+}
+
+/// \brief Get the stride of a pointer access in a loop. Looks for symbolic
+/// strides "a[i*stride]". Returns the symbolic stride, or null otherwise.
+llvm::Value *llvm::getStrideFromPointer(llvm::Value *Ptr, ScalarEvolution *SE,
+                                        Loop *Lp) {
+  const PointerType *PtrTy = dyn_cast<PointerType>(Ptr->getType());
+  if (!PtrTy || PtrTy->isAggregateType())
+    return nullptr;
+
+  // Try to remove a gep instruction to make the pointer (actually index at this
+  // point) easier analyzable. If OrigPtr is equal to Ptr we are analzying the
+  // pointer, otherwise, we are analyzing the index.
+  llvm::Value *OrigPtr = Ptr;
+
+  // The size of the pointer access.
+  int64_t PtrAccessSize = 1;
+
+  Ptr = stripGetElementPtr(Ptr, SE, Lp);
+  const SCEV *V = SE->getSCEV(Ptr);
+
+  if (Ptr != OrigPtr)
+    // Strip off casts.
+    while (const SCEVCastExpr *C = dyn_cast<SCEVCastExpr>(V))
+      V = C->getOperand();
+
+  const SCEVAddRecExpr *S = dyn_cast<SCEVAddRecExpr>(V);
+  if (!S)
+    return nullptr;
+
+  V = S->getStepRecurrence(*SE);
+  if (!V)
+    return nullptr;
+
+  // Strip off the size of access multiplication if we are still analyzing the
+  // pointer.
+  if (OrigPtr == Ptr) {
+    const DataLayout &DL = Lp->getHeader()->getModule()->getDataLayout();
+    DL.getTypeAllocSize(PtrTy->getElementType());
+    if (const SCEVMulExpr *M = dyn_cast<SCEVMulExpr>(V)) {
+      if (M->getOperand(0)->getSCEVType() != scConstant)
+        return nullptr;
+
+      const APInt &APStepVal =
+          cast<SCEVConstant>(M->getOperand(0))->getValue()->getValue();
+
+      // Huge step value - give up.
+      if (APStepVal.getBitWidth() > 64)
+        return nullptr;
+
+      int64_t StepVal = APStepVal.getSExtValue();
+      if (PtrAccessSize != StepVal)
+        return nullptr;
+      V = M->getOperand(1);
+    }
+  }
+
+  // Strip off casts.
+  Type *StripedOffRecurrenceCast = nullptr;
+  if (const SCEVCastExpr *C = dyn_cast<SCEVCastExpr>(V)) {
+    StripedOffRecurrenceCast = C->getType();
+    V = C->getOperand();
+  }
+
+  // Look for the loop invariant symbolic value.
+  const SCEVUnknown *U = dyn_cast<SCEVUnknown>(V);
+  if (!U)
+    return nullptr;
+
+  llvm::Value *Stride = U->getValue();
+  if (!Lp->isLoopInvariant(Stride))
+    return nullptr;
+
+  // If we have stripped off the recurrence cast we have to make sure that we
+  // return the value that is used in this loop so that we can replace it later.
+  if (StripedOffRecurrenceCast)
+    Stride = getUniqueCastUse(Stride, Lp, StripedOffRecurrenceCast);
+
+  return Stride;
+}
+
+/// \brief Given a vector and an element number, see if the scalar value is
+/// already around as a register, for example if it were inserted then extracted
+/// from the vector.
+llvm::Value *llvm::findScalarElement(llvm::Value *V, unsigned EltNo) {
+  assert(V->getType()->isVectorTy() && "Not looking at a vector?");
+  VectorType *VTy = cast<VectorType>(V->getType());
+  unsigned Width = VTy->getNumElements();
+  if (EltNo >= Width)  // Out of range access.
+    return UndefValue::get(VTy->getElementType());
+
+  if (Constant *C = dyn_cast<Constant>(V))
+    return C->getAggregateElement(EltNo);
+
+  if (InsertElementInst *III = dyn_cast<InsertElementInst>(V)) {
+    // If this is an insert to a variable element, we don't know what it is.
+    if (!isa<ConstantInt>(III->getOperand(2)))
+      return nullptr;
+    unsigned IIElt = cast<ConstantInt>(III->getOperand(2))->getZExtValue();
+
+    // If this is an insert to the element we are looking for, return the
+    // inserted value.
+    if (EltNo == IIElt)
+      return III->getOperand(1);
+
+    // Otherwise, the insertelement doesn't modify the value, recurse on its
+    // vector input.
+    return findScalarElement(III->getOperand(0), EltNo);
+  }
+
+  if (ShuffleVectorInst *SVI = dyn_cast<ShuffleVectorInst>(V)) {
+    unsigned LHSWidth = SVI->getOperand(0)->getType()->getVectorNumElements();
+    int InEl = SVI->getMaskValue(EltNo);
+    if (InEl < 0)
+      return UndefValue::get(VTy->getElementType());
+    if (InEl < (int)LHSWidth)
+      return findScalarElement(SVI->getOperand(0), InEl);
+    return findScalarElement(SVI->getOperand(1), InEl - LHSWidth);
+  }
+
+  // Extract a value from a vector add operation with a constant zero.
+  Value *Val = nullptr; Constant *Con = nullptr;
+  if (match(V,
+            llvm::PatternMatch::m_Add(llvm::PatternMatch::m_Value(Val),
+                                      llvm::PatternMatch::m_Constant(Con)))) {
+    if (Con->getAggregateElement(EltNo)->isNullValue())
+      return findScalarElement(Val, EltNo);
+  }
+
+  // Otherwise, we don't know.
+  return nullptr;
+}
diff --git a/lib/AsmParser/CMakeLists.txt b/lib/AsmParser/CMakeLists.txt
index 78668377d13..0d727232105 100644
--- a/lib/AsmParser/CMakeLists.txt
+++ b/lib/AsmParser/CMakeLists.txt
@@ -6,4 +6,7 @@ add_llvm_library(LLVMAsmParser
 
   ADDITIONAL_HEADER_DIRS
   ${LLVM_MAIN_INCLUDE_DIR}/llvm/Analysis
+
+  DEPENDS
+  intrinsics_gen
   )
diff --git a/lib/AsmParser/LLLexer.cpp b/lib/AsmParser/LLLexer.cpp
index a72f713c490..be951f5c7d8 100644
--- a/lib/AsmParser/LLLexer.cpp
+++ b/lib/AsmParser/LLLexer.cpp
@@ -252,7 +252,7 @@ void LLLexer::SkipLineComment() {
   }
 }
 
-/// LexAt - Lex all tokens that start with an @ character:
+/// Lex all tokens that start with an @ character.
 ///   GlobalVar   @\"[^\"]*\"
 ///   GlobalVar   @[-a-zA-Z$._][-a-zA-Z$._0-9]*
 ///   GlobalVarID @[0-9]+
@@ -375,7 +375,7 @@ lltok::Kind LLLexer::LexVar(lltok::Kind Var, lltok::Kind VarID) {
   return lltok::Error;
 }
 
-/// LexPercent - Lex all tokens that start with a % character:
+/// Lex all tokens that start with a % character.
 ///   LocalVar   ::= %\"[^\"]*\"
 ///   LocalVar   ::= %[-a-zA-Z$._][-a-zA-Z$._0-9]*
 ///   LocalVarID ::= %[0-9]+
@@ -383,7 +383,7 @@ lltok::Kind LLLexer::LexPercent() {
   return LexVar(lltok::LocalVar, lltok::LocalVarID);
 }
 
-/// LexQuote - Lex all tokens that start with a " character:
+/// Lex all tokens that start with a " character.
 ///   QuoteLabel        "[^"]+":
 ///   StringConstant    "[^"]*"
 lltok::Kind LLLexer::LexQuote() {
@@ -404,7 +404,7 @@ lltok::Kind LLLexer::LexQuote() {
   return kind;
 }
 
-/// LexExclaim:
+/// Lex all tokens that start with a ! character.
 ///    !foo
 ///    !
 lltok::Kind LLLexer::LexExclaim() {
@@ -425,7 +425,7 @@ lltok::Kind LLLexer::LexExclaim() {
   return lltok::exclaim;
 }
 
-/// LexHash - Lex all tokens that start with a # character:
+/// Lex all tokens that start with a # character.
 ///    AttrGrpID ::= #[0-9]+
 lltok::Kind LLLexer::LexHash() {
   // Handle AttrGrpID: #[0-9]+
@@ -443,7 +443,7 @@ lltok::Kind LLLexer::LexHash() {
   return lltok::Error;
 }
 
-/// LexIdentifier: Handle several related productions:
+/// Lex a label, integer type, keyword, or hexadecimal integer constant.
 ///    Label           [-a-zA-Z$._0-9]+:
 ///    IntegerType     i[0-9]+
 ///    Keyword         sdiv, float, ...
@@ -524,6 +524,7 @@ lltok::Kind LLLexer::LexIdentifier() {
   KEYWORD(undef);
   KEYWORD(null);
   KEYWORD(to);
+  KEYWORD(caller);
   KEYWORD(tail);
   KEYWORD(musttail);
   KEYWORD(target);
@@ -593,10 +594,12 @@ lltok::Kind LLLexer::LexIdentifier() {
   KEYWORD(attributes);
 
   KEYWORD(alwaysinline);
+  KEYWORD(argmemonly);
   KEYWORD(builtin);
   KEYWORD(byval);
   KEYWORD(inalloca);
   KEYWORD(cold);
+  KEYWORD(convergent);
   KEYWORD(dereferenceable);
   KEYWORD(dereferenceable_or_null);
   KEYWORD(inlinehint);
@@ -627,6 +630,7 @@ lltok::Kind LLLexer::LexIdentifier() {
   KEYWORD(ssp);
   KEYWORD(sspreq);
   KEYWORD(sspstrong);
+  KEYWORD(safestack);
   KEYWORD(sanitize_address);
   KEYWORD(sanitize_thread);
   KEYWORD(sanitize_memory);
@@ -746,6 +750,12 @@ lltok::Kind LLLexer::LexIdentifier() {
   INSTKEYWORD(extractvalue,   ExtractValue);
   INSTKEYWORD(insertvalue,    InsertValue);
   INSTKEYWORD(landingpad,     LandingPad);
+  INSTKEYWORD(cleanupret,     CleanupRet);
+  INSTKEYWORD(catchret,       CatchRet);
+  INSTKEYWORD(catchpad,     CatchPad);
+  INSTKEYWORD(terminatepad, TerminatePad);
+  INSTKEYWORD(cleanuppad,   CleanupPad);
+  INSTKEYWORD(catchendpad,  CatchEndPad);
 #undef INSTKEYWORD
 
 #define DWKEYWORD(TYPE, TOKEN)                                                 \
@@ -799,9 +809,8 @@ lltok::Kind LLLexer::LexIdentifier() {
   return lltok::Error;
 }
 
-
-/// Lex0x: Handle productions that start with 0x, knowing that it matches and
-/// that this is not a label:
+/// Lex all tokens that start with a 0x prefix, knowing they match and are not
+/// labels.
 ///    HexFPConstant     0x[0-9A-Fa-f]+
 ///    HexFP80Constant   0xK[0-9A-Fa-f]+
 ///    HexFP128Constant  0xL[0-9A-Fa-f]+
@@ -859,7 +868,7 @@ lltok::Kind LLLexer::Lex0x() {
   }
 }
 
-/// LexIdentifier: Handle several related productions:
+/// Lex tokens for a label or a numeric constant, possibly starting with -.
 ///    Label             [-a-zA-Z$._0-9]+:
 ///    NInteger          -[0-9]+
 ///    FPConstant        [-+]?[0-9]+[.][0-9]*([eE][-+]?[0-9]+)?
@@ -902,20 +911,7 @@ lltok::Kind LLLexer::LexDigitOrNegative() {
   if (CurPtr[0] != '.') {
     if (TokStart[0] == '0' && TokStart[1] == 'x')
       return Lex0x();
-    unsigned Len = CurPtr-TokStart;
-    uint32_t numBits = ((Len * 64) / 19) + 2;
-    APInt Tmp(numBits, StringRef(TokStart, Len), 10);
-    if (TokStart[0] == '-') {
-      uint32_t minBits = Tmp.getMinSignedBits();
-      if (minBits > 0 && minBits < numBits)
-        Tmp = Tmp.trunc(minBits);
-      APSIntVal = APSInt(Tmp, false);
-    } else {
-      uint32_t activeBits = Tmp.getActiveBits();
-      if (activeBits > 0 && activeBits < numBits)
-        Tmp = Tmp.trunc(activeBits);
-      APSIntVal = APSInt(Tmp, true);
-    }
+    APSIntVal = APSInt(StringRef(TokStart, CurPtr - TokStart));
     return lltok::APSInt;
   }
 
@@ -937,6 +933,7 @@ lltok::Kind LLLexer::LexDigitOrNegative() {
   return lltok::APFloat;
 }
 
+/// Lex a floating point constant starting with +.
 ///    FPConstant  [-+]?[0-9]+[.][0-9]*([eE][-+]?[0-9]+)?
 lltok::Kind LLLexer::LexPositive() {
   // If the letter after the negative is a number, this is probably not a
diff --git a/lib/AsmParser/LLParser.cpp b/lib/AsmParser/LLParser.cpp
index 546363b9cb1..6c2cd08e580 100644
--- a/lib/AsmParser/LLParser.cpp
+++ b/lib/AsmParser/LLParser.cpp
@@ -13,6 +13,7 @@
 
 #include "LLParser.h"
 #include "llvm/ADT/SmallPtrSet.h"
+#include "llvm/AsmParser/SlotMapping.h"
 #include "llvm/IR/AutoUpgrade.h"
 #include "llvm/IR/CallingConv.h"
 #include "llvm/IR/Constants.h"
@@ -47,6 +48,17 @@ bool LLParser::Run() {
          ValidateEndOfModule();
 }
 
+bool LLParser::parseStandaloneConstantValue(Constant *&C) {
+  Lex.Lex();
+
+  Type *Ty = nullptr;
+  if (ParseType(Ty) || parseConstantValue(Ty, C))
+    return true;
+  if (Lex.getKind() != lltok::Eof)
+    return Error(Lex.getLoc(), "expected end of string");
+  return false;
+}
+
 /// ValidateEndOfModule - Do final validity and sanity checks at the end of the
 /// module.
 bool LLParser::ValidateEndOfModule() {
@@ -161,6 +173,14 @@ bool LLParser::ValidateEndOfModule() {
 
   UpgradeDebugInfo(*M);
 
+  if (!Slots)
+    return false;
+  // Initialize the slot mapping.
+  // Because by this point we've parsed and validated everything, we can "steal"
+  // the mapping from LLParser as it doesn't need it anymore.
+  Slots->GlobalValues = std::move(NumberedVals);
+  Slots->MetadataNodes = std::move(NumberedMetadata);
+
   return false;
 }
 
@@ -189,7 +209,7 @@ bool LLParser::ParseTopLevelEntities() {
     // The Global variable production with no name can have many different
     // optional leading prefixes, the production is:
     // GlobalVar ::= OptionalLinkage OptionalVisibility OptionalDLLStorageClass
-    //               OptionalThreadLocal OptionalAddrSpace OptionalUnNammedAddr
+    //               OptionalThreadLocal OptionalAddrSpace OptionalUnnamedAddr
     //               ('constant'|'global') ...
     case lltok::kw_private:             // OptionalLinkage
     case lltok::kw_internal:            // OptionalLinkage
@@ -360,13 +380,14 @@ bool LLParser::ParseDeclare() {
 }
 
 /// toplevelentity
-///   ::= 'define' FunctionHeader '{' ...
+///   ::= 'define' FunctionHeader (!dbg !56)* '{' ...
 bool LLParser::ParseDefine() {
   assert(Lex.getKind() == lltok::kw_define);
   Lex.Lex();
 
   Function *F;
   return ParseFunctionHeader(F, true) ||
+         ParseOptionalFunctionMetadata(*F) ||
          ParseFunctionBody(*F);
 }
 
@@ -563,10 +584,7 @@ bool LLParser::ParseNamedMetadata() {
       NMD->addOperand(N);
     } while (EatIfPresent(lltok::comma));
 
-  if (ParseToken(lltok::rbrace, "expected end of metadata node"))
-    return true;
-
-  return false;
+  return ParseToken(lltok::rbrace, "expected end of metadata node");
 }
 
 /// ParseStandaloneMetadata:
@@ -617,12 +635,12 @@ static bool isValidVisibilityForLinkage(unsigned V, unsigned L) {
 /// ParseAlias:
 ///   ::= GlobalVar '=' OptionalLinkage OptionalVisibility
 ///                     OptionalDLLStorageClass OptionalThreadLocal
-///                     OptionalUnNammedAddr 'alias' Aliasee
+///                     OptionalUnnamedAddr 'alias' Aliasee
 ///
 /// Aliasee
 ///   ::= TypeAndValue
 ///
-/// Everything through OptionalUnNammedAddr has already been parsed.
+/// Everything through OptionalUnnamedAddr has already been parsed.
 ///
 bool LLParser::ParseAlias(const std::string &Name, LocTy NameLoc, unsigned L,
                           unsigned Visibility, unsigned DLLStorageClass,
@@ -662,18 +680,19 @@ bool LLParser::ParseAlias(const std::string &Name, LocTy NameLoc, unsigned L,
   auto *PTy = dyn_cast<PointerType>(AliaseeType);
   if (!PTy)
     return Error(AliaseeLoc, "An alias must have pointer type");
-  Type *Ty = PTy->getElementType();
-  unsigned AddrSpace = PTy->getAddressSpace();
 
   // Okay, create the alias but do not insert it into the module yet.
   std::unique_ptr<GlobalAlias> GA(
-      GlobalAlias::create(Ty, AddrSpace, (GlobalValue::LinkageTypes)Linkage,
-                          Name, Aliasee, /*Parent*/ nullptr));
+      GlobalAlias::create(PTy, (GlobalValue::LinkageTypes)Linkage, Name,
+                          Aliasee, /*Parent*/ nullptr));
   GA->setThreadLocalMode(TLM);
   GA->setVisibility((GlobalValue::VisibilityTypes)Visibility);
   GA->setDLLStorageClass((GlobalValue::DLLStorageClassTypes)DLLStorageClass);
   GA->setUnnamedAddr(UnnamedAddr);
 
+  if (Name.empty())
+    NumberedVals.push_back(GA.get());
+
   // See if this value already exists in the symbol table.  If so, it is either
   // a redefinition or a definition of a forward reference.
   if (GlobalValue *Val = M->getNamedValue(Name)) {
@@ -709,13 +728,13 @@ bool LLParser::ParseAlias(const std::string &Name, LocTy NameLoc, unsigned L,
 
 /// ParseGlobal
 ///   ::= GlobalVar '=' OptionalLinkage OptionalVisibility OptionalDLLStorageClass
-///       OptionalThreadLocal OptionalUnNammedAddr OptionalAddrSpace
+///       OptionalThreadLocal OptionalUnnamedAddr OptionalAddrSpace
 ///       OptionalExternallyInitialized GlobalType Type Const
 ///   ::= OptionalLinkage OptionalVisibility OptionalDLLStorageClass
-///       OptionalThreadLocal OptionalUnNammedAddr OptionalAddrSpace
+///       OptionalThreadLocal OptionalUnnamedAddr OptionalAddrSpace
 ///       OptionalExternallyInitialized GlobalType Type Const
 ///
-/// Everything up to and including OptionalUnNammedAddr has been parsed
+/// Everything up to and including OptionalUnnamedAddr has been parsed
 /// already.
 ///
 bool LLParser::ParseGlobal(const std::string &Name, LocTy NameLoc,
@@ -777,7 +796,7 @@ bool LLParser::ParseGlobal(const std::string &Name, LocTy NameLoc,
                             Name, nullptr, GlobalVariable::NotThreadLocal,
                             AddrSpace);
   } else {
-    if (GVal->getType()->getElementType() != Ty)
+    if (GVal->getValueType() != Ty)
       return Error(TyLoc,
             "forward reference and definition of global have different types");
 
@@ -938,33 +957,42 @@ bool LLParser::ParseFnAttributeValuePairs(AttrBuilder &B,
       B.addStackAlignmentAttr(Alignment);
       continue;
     }
-    case lltok::kw_alwaysinline:      B.addAttribute(Attribute::AlwaysInline); break;
-    case lltok::kw_builtin:           B.addAttribute(Attribute::Builtin); break;
-    case lltok::kw_cold:              B.addAttribute(Attribute::Cold); break;
-    case lltok::kw_inlinehint:        B.addAttribute(Attribute::InlineHint); break;
-    case lltok::kw_jumptable:         B.addAttribute(Attribute::JumpTable); break;
-    case lltok::kw_minsize:           B.addAttribute(Attribute::MinSize); break;
-    case lltok::kw_naked:             B.addAttribute(Attribute::Naked); break;
-    case lltok::kw_nobuiltin:         B.addAttribute(Attribute::NoBuiltin); break;
-    case lltok::kw_noduplicate:       B.addAttribute(Attribute::NoDuplicate); break;
-    case lltok::kw_noimplicitfloat:   B.addAttribute(Attribute::NoImplicitFloat); break;
-    case lltok::kw_noinline:          B.addAttribute(Attribute::NoInline); break;
-    case lltok::kw_nonlazybind:       B.addAttribute(Attribute::NonLazyBind); break;
-    case lltok::kw_noredzone:         B.addAttribute(Attribute::NoRedZone); break;
-    case lltok::kw_noreturn:          B.addAttribute(Attribute::NoReturn); break;
-    case lltok::kw_nounwind:          B.addAttribute(Attribute::NoUnwind); break;
-    case lltok::kw_optnone:           B.addAttribute(Attribute::OptimizeNone); break;
-    case lltok::kw_optsize:           B.addAttribute(Attribute::OptimizeForSize); break;
-    case lltok::kw_readnone:          B.addAttribute(Attribute::ReadNone); break;
-    case lltok::kw_readonly:          B.addAttribute(Attribute::ReadOnly); break;
-    case lltok::kw_returns_twice:     B.addAttribute(Attribute::ReturnsTwice); break;
-    case lltok::kw_ssp:               B.addAttribute(Attribute::StackProtect); break;
-    case lltok::kw_sspreq:            B.addAttribute(Attribute::StackProtectReq); break;
-    case lltok::kw_sspstrong:         B.addAttribute(Attribute::StackProtectStrong); break;
-    case lltok::kw_sanitize_address:  B.addAttribute(Attribute::SanitizeAddress); break;
-    case lltok::kw_sanitize_thread:   B.addAttribute(Attribute::SanitizeThread); break;
-    case lltok::kw_sanitize_memory:   B.addAttribute(Attribute::SanitizeMemory); break;
-    case lltok::kw_uwtable:           B.addAttribute(Attribute::UWTable); break;
+    case lltok::kw_alwaysinline: B.addAttribute(Attribute::AlwaysInline); break;
+    case lltok::kw_argmemonly: B.addAttribute(Attribute::ArgMemOnly); break;
+    case lltok::kw_builtin: B.addAttribute(Attribute::Builtin); break;
+    case lltok::kw_cold: B.addAttribute(Attribute::Cold); break;
+    case lltok::kw_convergent: B.addAttribute(Attribute::Convergent); break;
+    case lltok::kw_inlinehint: B.addAttribute(Attribute::InlineHint); break;
+    case lltok::kw_jumptable: B.addAttribute(Attribute::JumpTable); break;
+    case lltok::kw_minsize: B.addAttribute(Attribute::MinSize); break;
+    case lltok::kw_naked: B.addAttribute(Attribute::Naked); break;
+    case lltok::kw_nobuiltin: B.addAttribute(Attribute::NoBuiltin); break;
+    case lltok::kw_noduplicate: B.addAttribute(Attribute::NoDuplicate); break;
+    case lltok::kw_noimplicitfloat:
+      B.addAttribute(Attribute::NoImplicitFloat); break;
+    case lltok::kw_noinline: B.addAttribute(Attribute::NoInline); break;
+    case lltok::kw_nonlazybind: B.addAttribute(Attribute::NonLazyBind); break;
+    case lltok::kw_noredzone: B.addAttribute(Attribute::NoRedZone); break;
+    case lltok::kw_noreturn: B.addAttribute(Attribute::NoReturn); break;
+    case lltok::kw_nounwind: B.addAttribute(Attribute::NoUnwind); break;
+    case lltok::kw_optnone: B.addAttribute(Attribute::OptimizeNone); break;
+    case lltok::kw_optsize: B.addAttribute(Attribute::OptimizeForSize); break;
+    case lltok::kw_readnone: B.addAttribute(Attribute::ReadNone); break;
+    case lltok::kw_readonly: B.addAttribute(Attribute::ReadOnly); break;
+    case lltok::kw_returns_twice:
+      B.addAttribute(Attribute::ReturnsTwice); break;
+    case lltok::kw_ssp: B.addAttribute(Attribute::StackProtect); break;
+    case lltok::kw_sspreq: B.addAttribute(Attribute::StackProtectReq); break;
+    case lltok::kw_sspstrong:
+      B.addAttribute(Attribute::StackProtectStrong); break;
+    case lltok::kw_safestack: B.addAttribute(Attribute::SafeStack); break;
+    case lltok::kw_sanitize_address:
+      B.addAttribute(Attribute::SanitizeAddress); break;
+    case lltok::kw_sanitize_thread:
+      B.addAttribute(Attribute::SanitizeThread); break;
+    case lltok::kw_sanitize_memory:
+      B.addAttribute(Attribute::SanitizeMemory); break;
+    case lltok::kw_uwtable: B.addAttribute(Attribute::UWTable); break;
 
     // Error handling.
     case lltok::kw_inreg:
@@ -1248,6 +1276,7 @@ bool LLParser::ParseOptionalParamAttrs(AttrBuilder &B) {
 
     case lltok::kw_alignstack:
     case lltok::kw_alwaysinline:
+    case lltok::kw_argmemonly:
     case lltok::kw_builtin:
     case lltok::kw_inlinehint:
     case lltok::kw_jumptable:
@@ -1270,6 +1299,7 @@ bool LLParser::ParseOptionalParamAttrs(AttrBuilder &B) {
     case lltok::kw_ssp:
     case lltok::kw_sspreq:
     case lltok::kw_sspstrong:
+    case lltok::kw_safestack:
     case lltok::kw_uwtable:
       HaveError |= Error(Lex.getLoc(), "invalid use of function-only attribute");
       break;
@@ -1323,6 +1353,7 @@ bool LLParser::ParseOptionalReturnAttrs(AttrBuilder &B) {
 
     case lltok::kw_alignstack:
     case lltok::kw_alwaysinline:
+    case lltok::kw_argmemonly:
     case lltok::kw_builtin:
     case lltok::kw_cold:
     case lltok::kw_inlinehint:
@@ -1346,6 +1377,7 @@ bool LLParser::ParseOptionalReturnAttrs(AttrBuilder &B) {
     case lltok::kw_ssp:
     case lltok::kw_sspreq:
     case lltok::kw_sspstrong:
+    case lltok::kw_safestack:
     case lltok::kw_uwtable:
       HaveError |= Error(Lex.getLoc(), "invalid use of function-only attribute");
       break;
@@ -1490,31 +1522,53 @@ bool LLParser::ParseOptionalCallingConv(unsigned &CC) {
   return false;
 }
 
+/// ParseMetadataAttachment
+///   ::= !dbg !42
+bool LLParser::ParseMetadataAttachment(unsigned &Kind, MDNode *&MD) {
+  assert(Lex.getKind() == lltok::MetadataVar && "Expected metadata attachment");
+
+  std::string Name = Lex.getStrVal();
+  Kind = M->getMDKindID(Name);
+  Lex.Lex();
+
+  return ParseMDNode(MD);
+}
+
 /// ParseInstructionMetadata
 ///   ::= !dbg !42 (',' !dbg !57)*
-bool LLParser::ParseInstructionMetadata(Instruction *Inst,
-                                        PerFunctionState *PFS) {
+bool LLParser::ParseInstructionMetadata(Instruction &Inst) {
   do {
     if (Lex.getKind() != lltok::MetadataVar)
       return TokError("expected metadata after comma");
 
-    std::string Name = Lex.getStrVal();
-    unsigned MDK = M->getMDKindID(Name);
-    Lex.Lex();
-
+    unsigned MDK;
     MDNode *N;
-    if (ParseMDNode(N))
+    if (ParseMetadataAttachment(MDK, N))
       return true;
 
-    Inst->setMetadata(MDK, N);
+    Inst.setMetadata(MDK, N);
     if (MDK == LLVMContext::MD_tbaa)
-      InstsWithTBAATag.push_back(Inst);
+      InstsWithTBAATag.push_back(&Inst);
 
     // If this is the end of the list, we're done.
   } while (EatIfPresent(lltok::comma));
   return false;
 }
 
+/// ParseOptionalFunctionMetadata
+///   ::= (!dbg !57)*
+bool LLParser::ParseOptionalFunctionMetadata(Function &F) {
+  while (Lex.getKind() == lltok::MetadataVar) {
+    unsigned MDK;
+    MDNode *N;
+    if (ParseMetadataAttachment(MDK, N))
+      return true;
+
+    F.setMetadata(MDK, N);
+  }
+  return false;
+}
+
 /// ParseOptionalAlignment
 ///   ::= /* empty */
 ///   ::= 'align' 4
@@ -1883,9 +1937,9 @@ bool LLParser::ParseArgumentList(SmallVectorImpl<ArgInfo> &ArgList,
       return Error(TypeLoc, "invalid type for function argument");
 
     unsigned AttrIndex = 1;
-    ArgList.push_back(ArgInfo(TypeLoc, ArgTy,
-                              AttributeSet::get(ArgTy->getContext(),
-                                                AttrIndex++, Attrs), Name));
+    ArgList.emplace_back(TypeLoc, ArgTy, AttributeSet::get(ArgTy->getContext(),
+                                                           AttrIndex++, Attrs),
+                         std::move(Name));
 
     while (EatIfPresent(lltok::comma)) {
       // Handle ... at end of arg list.
@@ -1911,10 +1965,10 @@ bool LLParser::ParseArgumentList(SmallVectorImpl<ArgInfo> &ArgList,
       if (!ArgTy->isFirstClassType())
         return Error(TypeLoc, "invalid type for function argument");
 
-      ArgList.push_back(ArgInfo(TypeLoc, ArgTy,
-                                AttributeSet::get(ArgTy->getContext(),
-                                                  AttrIndex++, Attrs),
-                                Name));
+      ArgList.emplace_back(
+          TypeLoc, ArgTy,
+          AttributeSet::get(ArgTy->getContext(), AttrIndex++, Attrs),
+          std::move(Name));
     }
   }
 
@@ -2839,8 +2893,8 @@ bool LLParser::ParseValID(ValID &ID, PerFunctionState *PFS) {
         if (ValTy->isVectorTy() != BaseType->isVectorTy())
           return Error(ID.Loc, "getelementptr index type missmatch");
         if (ValTy->isVectorTy()) {
-          unsigned ValNumEl = cast<VectorType>(ValTy)->getNumElements();
-          unsigned PtrNumEl = cast<VectorType>(BaseType)->getNumElements();
+          unsigned ValNumEl = ValTy->getVectorNumElements();
+          unsigned PtrNumEl = BaseType->getVectorNumElements();
           if (ValNumEl != PtrNumEl)
             return Error(
                 ID.Loc,
@@ -2849,8 +2903,7 @@ bool LLParser::ParseValID(ValID &ID, PerFunctionState *PFS) {
       }
 
       SmallPtrSet<const Type*, 4> Visited;
-      if (!Indices.empty() &&
-          !BasePointerType->getElementType()->isSized(&Visited))
+      if (!Indices.empty() && !Ty->isSized(&Visited))
         return Error(ID.Loc, "base element of getelementptr must be sized");
 
       if (!GetElementPtrInst::getIndexedType(Ty, Indices))
@@ -2972,7 +3025,7 @@ bool LLParser::ParseMDTuple(MDNode *&MD, bool IsDistinct) {
 /// MDNode:
 ///  ::= !{ ... }
 ///  ::= !7
-///  ::= !MDLocation(...)
+///  ::= !DILocation(...)
 bool LLParser::ParseMDNode(MDNode *&N) {
   if (Lex.getKind() == lltok::MetadataVar)
     return ParseSpecializedMDNode(N);
@@ -3187,7 +3240,7 @@ bool LLParser::ParseMDField(LocTy Loc, StringRef Name, DIFlagField &Result) {
     if (Lex.getKind() != lltok::DIFlag)
       return TokError("expected debug info flag");
 
-    Val = DebugNode::getFlag(Lex.getStrVal());
+    Val = DINode::getFlag(Lex.getStrVal());
     if (!Val)
       return TokError(Twine("invalid debug info flag flag '") +
                       Lex.getStrVal() + "'");
@@ -3369,9 +3422,9 @@ bool LLParser::ParseSpecializedMDNode(MDNode *&N, bool IsDistinct) {
 #define GET_OR_DISTINCT(CLASS, ARGS)                                           \
   (IsDistinct ? CLASS::getDistinct ARGS : CLASS::get ARGS)
 
-/// ParseMDLocationFields:
-///   ::= !MDLocation(line: 43, column: 8, scope: !5, inlinedAt: !6)
-bool LLParser::ParseMDLocation(MDNode *&Result, bool IsDistinct) {
+/// ParseDILocationFields:
+///   ::= !DILocation(line: 43, column: 8, scope: !5, inlinedAt: !6)
+bool LLParser::ParseDILocation(MDNode *&Result, bool IsDistinct) {
 #define VISIT_MD_FIELDS(OPTIONAL, REQUIRED)                                    \
   OPTIONAL(line, LineField, );                                                 \
   OPTIONAL(column, ColumnField, );                                             \
@@ -3381,13 +3434,13 @@ bool LLParser::ParseMDLocation(MDNode *&Result, bool IsDistinct) {
 #undef VISIT_MD_FIELDS
 
   Result = GET_OR_DISTINCT(
-      MDLocation, (Context, line.Val, column.Val, scope.Val, inlinedAt.Val));
+      DILocation, (Context, line.Val, column.Val, scope.Val, inlinedAt.Val));
   return false;
 }
 
-/// ParseGenericDebugNode:
-///   ::= !GenericDebugNode(tag: 15, header: "...", operands: {...})
-bool LLParser::ParseGenericDebugNode(MDNode *&Result, bool IsDistinct) {
+/// ParseGenericDINode:
+///   ::= !GenericDINode(tag: 15, header: "...", operands: {...})
+bool LLParser::ParseGenericDINode(MDNode *&Result, bool IsDistinct) {
 #define VISIT_MD_FIELDS(OPTIONAL, REQUIRED)                                    \
   REQUIRED(tag, DwarfTagField, );                                              \
   OPTIONAL(header, MDStringField, );                                           \
@@ -3395,40 +3448,40 @@ bool LLParser::ParseGenericDebugNode(MDNode *&Result, bool IsDistinct) {
   PARSE_MD_FIELDS();
 #undef VISIT_MD_FIELDS
 
-  Result = GET_OR_DISTINCT(GenericDebugNode,
+  Result = GET_OR_DISTINCT(GenericDINode,
                            (Context, tag.Val, header.Val, operands.Val));
   return false;
 }
 
-/// ParseMDSubrange:
-///   ::= !MDSubrange(count: 30, lowerBound: 2)
-bool LLParser::ParseMDSubrange(MDNode *&Result, bool IsDistinct) {
+/// ParseDISubrange:
+///   ::= !DISubrange(count: 30, lowerBound: 2)
+bool LLParser::ParseDISubrange(MDNode *&Result, bool IsDistinct) {
 #define VISIT_MD_FIELDS(OPTIONAL, REQUIRED)                                    \
   REQUIRED(count, MDSignedField, (-1, -1, INT64_MAX));                         \
   OPTIONAL(lowerBound, MDSignedField, );
   PARSE_MD_FIELDS();
 #undef VISIT_MD_FIELDS
 
-  Result = GET_OR_DISTINCT(MDSubrange, (Context, count.Val, lowerBound.Val));
+  Result = GET_OR_DISTINCT(DISubrange, (Context, count.Val, lowerBound.Val));
   return false;
 }
 
-/// ParseMDEnumerator:
-///   ::= !MDEnumerator(value: 30, name: "SomeKind")
-bool LLParser::ParseMDEnumerator(MDNode *&Result, bool IsDistinct) {
+/// ParseDIEnumerator:
+///   ::= !DIEnumerator(value: 30, name: "SomeKind")
+bool LLParser::ParseDIEnumerator(MDNode *&Result, bool IsDistinct) {
 #define VISIT_MD_FIELDS(OPTIONAL, REQUIRED)                                    \
   REQUIRED(name, MDStringField, );                                             \
   REQUIRED(value, MDSignedField, );
   PARSE_MD_FIELDS();
 #undef VISIT_MD_FIELDS
 
-  Result = GET_OR_DISTINCT(MDEnumerator, (Context, value.Val, name.Val));
+  Result = GET_OR_DISTINCT(DIEnumerator, (Context, value.Val, name.Val));
   return false;
 }
 
-/// ParseMDBasicType:
-///   ::= !MDBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32)
-bool LLParser::ParseMDBasicType(MDNode *&Result, bool IsDistinct) {
+/// ParseDIBasicType:
+///   ::= !DIBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32)
+bool LLParser::ParseDIBasicType(MDNode *&Result, bool IsDistinct) {
 #define VISIT_MD_FIELDS(OPTIONAL, REQUIRED)                                    \
   OPTIONAL(tag, DwarfTagField, (dwarf::DW_TAG_base_type));                     \
   OPTIONAL(name, MDStringField, );                                             \
@@ -3438,16 +3491,16 @@ bool LLParser::ParseMDBasicType(MDNode *&Result, bool IsDistinct) {
   PARSE_MD_FIELDS();
 #undef VISIT_MD_FIELDS
 
-  Result = GET_OR_DISTINCT(MDBasicType, (Context, tag.Val, name.Val, size.Val,
+  Result = GET_OR_DISTINCT(DIBasicType, (Context, tag.Val, name.Val, size.Val,
                                          align.Val, encoding.Val));
   return false;
 }
 
-/// ParseMDDerivedType:
-///   ::= !MDDerivedType(tag: DW_TAG_pointer_type, name: "int", file: !0,
+/// ParseDIDerivedType:
+///   ::= !DIDerivedType(tag: DW_TAG_pointer_type, name: "int", file: !0,
 ///                      line: 7, scope: !1, baseType: !2, size: 32,
 ///                      align: 32, offset: 0, flags: 0, extraData: !3)
-bool LLParser::ParseMDDerivedType(MDNode *&Result, bool IsDistinct) {
+bool LLParser::ParseDIDerivedType(MDNode *&Result, bool IsDistinct) {
 #define VISIT_MD_FIELDS(OPTIONAL, REQUIRED)                                    \
   REQUIRED(tag, DwarfTagField, );                                              \
   OPTIONAL(name, MDStringField, );                                             \
@@ -3463,14 +3516,14 @@ bool LLParser::ParseMDDerivedType(MDNode *&Result, bool IsDistinct) {
   PARSE_MD_FIELDS();
 #undef VISIT_MD_FIELDS
 
-  Result = GET_OR_DISTINCT(MDDerivedType,
+  Result = GET_OR_DISTINCT(DIDerivedType,
                            (Context, tag.Val, name.Val, file.Val, line.Val,
                             scope.Val, baseType.Val, size.Val, align.Val,
                             offset.Val, flags.Val, extraData.Val));
   return false;
 }
 
-bool LLParser::ParseMDCompositeType(MDNode *&Result, bool IsDistinct) {
+bool LLParser::ParseDICompositeType(MDNode *&Result, bool IsDistinct) {
 #define VISIT_MD_FIELDS(OPTIONAL, REQUIRED)                                    \
   REQUIRED(tag, DwarfTagField, );                                              \
   OPTIONAL(name, MDStringField, );                                             \
@@ -3491,44 +3544,44 @@ bool LLParser::ParseMDCompositeType(MDNode *&Result, bool IsDistinct) {
 #undef VISIT_MD_FIELDS
 
   Result = GET_OR_DISTINCT(
-      MDCompositeType,
+      DICompositeType,
       (Context, tag.Val, name.Val, file.Val, line.Val, scope.Val, baseType.Val,
        size.Val, align.Val, offset.Val, flags.Val, elements.Val,
        runtimeLang.Val, vtableHolder.Val, templateParams.Val, identifier.Val));
   return false;
 }
 
-bool LLParser::ParseMDSubroutineType(MDNode *&Result, bool IsDistinct) {
+bool LLParser::ParseDISubroutineType(MDNode *&Result, bool IsDistinct) {
 #define VISIT_MD_FIELDS(OPTIONAL, REQUIRED)                                    \
   OPTIONAL(flags, DIFlagField, );                                              \
   REQUIRED(types, MDField, );
   PARSE_MD_FIELDS();
 #undef VISIT_MD_FIELDS
 
-  Result = GET_OR_DISTINCT(MDSubroutineType, (Context, flags.Val, types.Val));
+  Result = GET_OR_DISTINCT(DISubroutineType, (Context, flags.Val, types.Val));
   return false;
 }
 
-/// ParseMDFileType:
-///   ::= !MDFileType(filename: "path/to/file", directory: "/path/to/dir")
-bool LLParser::ParseMDFile(MDNode *&Result, bool IsDistinct) {
+/// ParseDIFileType:
+///   ::= !DIFileType(filename: "path/to/file", directory: "/path/to/dir")
+bool LLParser::ParseDIFile(MDNode *&Result, bool IsDistinct) {
 #define VISIT_MD_FIELDS(OPTIONAL, REQUIRED)                                    \
   REQUIRED(filename, MDStringField, );                                         \
   REQUIRED(directory, MDStringField, );
   PARSE_MD_FIELDS();
 #undef VISIT_MD_FIELDS
 
-  Result = GET_OR_DISTINCT(MDFile, (Context, filename.Val, directory.Val));
+  Result = GET_OR_DISTINCT(DIFile, (Context, filename.Val, directory.Val));
   return false;
 }
 
-/// ParseMDCompileUnit:
-///   ::= !MDCompileUnit(language: DW_LANG_C99, file: !0, producer: "clang",
+/// ParseDICompileUnit:
+///   ::= !DICompileUnit(language: DW_LANG_C99, file: !0, producer: "clang",
 ///                      isOptimized: true, flags: "-O2", runtimeVersion: 1,
 ///                      splitDebugFilename: "abc.debug", emissionKind: 1,
 ///                      enums: !1, retainedTypes: !2, subprograms: !3,
-///                      globals: !4, imports: !5)
-bool LLParser::ParseMDCompileUnit(MDNode *&Result, bool IsDistinct) {
+///                      globals: !4, imports: !5, dwoId: 0x0abcd)
+bool LLParser::ParseDICompileUnit(MDNode *&Result, bool IsDistinct) {
 #define VISIT_MD_FIELDS(OPTIONAL, REQUIRED)                                    \
   REQUIRED(language, DwarfLangField, );                                        \
   REQUIRED(file, MDField, (/* AllowNull */ false));                            \
@@ -3542,28 +3595,29 @@ bool LLParser::ParseMDCompileUnit(MDNode *&Result, bool IsDistinct) {
   OPTIONAL(retainedTypes, MDField, );                                          \
   OPTIONAL(subprograms, MDField, );                                            \
   OPTIONAL(globals, MDField, );                                                \
-  OPTIONAL(imports, MDField, );
+  OPTIONAL(imports, MDField, );                                                \
+  OPTIONAL(dwoId, MDUnsignedField, );
   PARSE_MD_FIELDS();
 #undef VISIT_MD_FIELDS
 
-  Result = GET_OR_DISTINCT(MDCompileUnit,
+  Result = GET_OR_DISTINCT(DICompileUnit,
                            (Context, language.Val, file.Val, producer.Val,
                             isOptimized.Val, flags.Val, runtimeVersion.Val,
                             splitDebugFilename.Val, emissionKind.Val, enums.Val,
                             retainedTypes.Val, subprograms.Val, globals.Val,
-                            imports.Val));
+                            imports.Val, dwoId.Val));
   return false;
 }
 
-/// ParseMDSubprogram:
-///   ::= !MDSubprogram(scope: !0, name: "foo", linkageName: "_Zfoo",
+/// ParseDISubprogram:
+///   ::= !DISubprogram(scope: !0, name: "foo", linkageName: "_Zfoo",
 ///                     file: !1, line: 7, type: !2, isLocal: false,
 ///                     isDefinition: true, scopeLine: 8, containingType: !3,
 ///                     virtuality: DW_VIRTUALTIY_pure_virtual,
 ///                     virtualIndex: 10, flags: 11,
 ///                     isOptimized: false, function: void ()* @_Z3foov,
 ///                     templateParams: !4, declaration: !5, variables: !6)
-bool LLParser::ParseMDSubprogram(MDNode *&Result, bool IsDistinct) {
+bool LLParser::ParseDISubprogram(MDNode *&Result, bool IsDistinct) {
 #define VISIT_MD_FIELDS(OPTIONAL, REQUIRED)                                    \
   OPTIONAL(scope, MDField, );                                                  \
   OPTIONAL(name, MDStringField, );                                             \
@@ -3587,7 +3641,7 @@ bool LLParser::ParseMDSubprogram(MDNode *&Result, bool IsDistinct) {
 #undef VISIT_MD_FIELDS
 
   Result = GET_OR_DISTINCT(
-      MDSubprogram, (Context, scope.Val, name.Val, linkageName.Val, file.Val,
+      DISubprogram, (Context, scope.Val, name.Val, linkageName.Val, file.Val,
                      line.Val, type.Val, isLocal.Val, isDefinition.Val,
                      scopeLine.Val, containingType.Val, virtuality.Val,
                      virtualIndex.Val, flags.Val, isOptimized.Val, function.Val,
@@ -3595,9 +3649,9 @@ bool LLParser::ParseMDSubprogram(MDNode *&Result, bool IsDistinct) {
   return false;
 }
 
-/// ParseMDLexicalBlock:
-///   ::= !MDLexicalBlock(scope: !0, file: !2, line: 7, column: 9)
-bool LLParser::ParseMDLexicalBlock(MDNode *&Result, bool IsDistinct) {
+/// ParseDILexicalBlock:
+///   ::= !DILexicalBlock(scope: !0, file: !2, line: 7, column: 9)
+bool LLParser::ParseDILexicalBlock(MDNode *&Result, bool IsDistinct) {
 #define VISIT_MD_FIELDS(OPTIONAL, REQUIRED)                                    \
   REQUIRED(scope, MDField, (/* AllowNull */ false));                           \
   OPTIONAL(file, MDField, );                                                   \
@@ -3607,13 +3661,13 @@ bool LLParser::ParseMDLexicalBlock(MDNode *&Result, bool IsDistinct) {
 #undef VISIT_MD_FIELDS
 
   Result = GET_OR_DISTINCT(
-      MDLexicalBlock, (Context, scope.Val, file.Val, line.Val, column.Val));
+      DILexicalBlock, (Context, scope.Val, file.Val, line.Val, column.Val));
   return false;
 }
 
-/// ParseMDLexicalBlockFile:
-///   ::= !MDLexicalBlockFile(scope: !0, file: !2, discriminator: 9)
-bool LLParser::ParseMDLexicalBlockFile(MDNode *&Result, bool IsDistinct) {
+/// ParseDILexicalBlockFile:
+///   ::= !DILexicalBlockFile(scope: !0, file: !2, discriminator: 9)
+bool LLParser::ParseDILexicalBlockFile(MDNode *&Result, bool IsDistinct) {
 #define VISIT_MD_FIELDS(OPTIONAL, REQUIRED)                                    \
   REQUIRED(scope, MDField, (/* AllowNull */ false));                           \
   OPTIONAL(file, MDField, );                                                   \
@@ -3621,14 +3675,14 @@ bool LLParser::ParseMDLexicalBlockFile(MDNode *&Result, bool IsDistinct) {
   PARSE_MD_FIELDS();
 #undef VISIT_MD_FIELDS
 
-  Result = GET_OR_DISTINCT(MDLexicalBlockFile,
+  Result = GET_OR_DISTINCT(DILexicalBlockFile,
                            (Context, scope.Val, file.Val, discriminator.Val));
   return false;
 }
 
-/// ParseMDNamespace:
-///   ::= !MDNamespace(scope: !0, file: !2, name: "SomeNamespace", line: 9)
-bool LLParser::ParseMDNamespace(MDNode *&Result, bool IsDistinct) {
+/// ParseDINamespace:
+///   ::= !DINamespace(scope: !0, file: !2, name: "SomeNamespace", line: 9)
+bool LLParser::ParseDINamespace(MDNode *&Result, bool IsDistinct) {
 #define VISIT_MD_FIELDS(OPTIONAL, REQUIRED)                                    \
   REQUIRED(scope, MDField, );                                                  \
   OPTIONAL(file, MDField, );                                                   \
@@ -3637,14 +3691,32 @@ bool LLParser::ParseMDNamespace(MDNode *&Result, bool IsDistinct) {
   PARSE_MD_FIELDS();
 #undef VISIT_MD_FIELDS
 
-  Result = GET_OR_DISTINCT(MDNamespace,
+  Result = GET_OR_DISTINCT(DINamespace,
                            (Context, scope.Val, file.Val, name.Val, line.Val));
   return false;
 }
 
-/// ParseMDTemplateTypeParameter:
-///   ::= !MDTemplateTypeParameter(name: "Ty", type: !1)
-bool LLParser::ParseMDTemplateTypeParameter(MDNode *&Result, bool IsDistinct) {
+/// ParseDIModule:
+///   ::= !DIModule(scope: !0, name: "SomeModule", configMacros: "-DNDEBUG",
+///                 includePath: "/usr/include", isysroot: "/")
+bool LLParser::ParseDIModule(MDNode *&Result, bool IsDistinct) {
+#define VISIT_MD_FIELDS(OPTIONAL, REQUIRED)                                    \
+  REQUIRED(scope, MDField, );                                                  \
+  REQUIRED(name, MDStringField, );                                             \
+  OPTIONAL(configMacros, MDStringField, );                                     \
+  OPTIONAL(includePath, MDStringField, );                                      \
+  OPTIONAL(isysroot, MDStringField, );
+  PARSE_MD_FIELDS();
+#undef VISIT_MD_FIELDS
+
+  Result = GET_OR_DISTINCT(DIModule, (Context, scope.Val, name.Val,
+                           configMacros.Val, includePath.Val, isysroot.Val));
+  return false;
+}
+
+/// ParseDITemplateTypeParameter:
+///   ::= !DITemplateTypeParameter(name: "Ty", type: !1)
+bool LLParser::ParseDITemplateTypeParameter(MDNode *&Result, bool IsDistinct) {
 #define VISIT_MD_FIELDS(OPTIONAL, REQUIRED)                                    \
   OPTIONAL(name, MDStringField, );                                             \
   REQUIRED(type, MDField, );
@@ -3652,14 +3724,14 @@ bool LLParser::ParseMDTemplateTypeParameter(MDNode *&Result, bool IsDistinct) {
 #undef VISIT_MD_FIELDS
 
   Result =
-      GET_OR_DISTINCT(MDTemplateTypeParameter, (Context, name.Val, type.Val));
+      GET_OR_DISTINCT(DITemplateTypeParameter, (Context, name.Val, type.Val));
   return false;
 }
 
-/// ParseMDTemplateValueParameter:
-///   ::= !MDTemplateValueParameter(tag: DW_TAG_template_value_parameter,
+/// ParseDITemplateValueParameter:
+///   ::= !DITemplateValueParameter(tag: DW_TAG_template_value_parameter,
 ///                                 name: "V", type: !1, value: i32 7)
-bool LLParser::ParseMDTemplateValueParameter(MDNode *&Result, bool IsDistinct) {
+bool LLParser::ParseDITemplateValueParameter(MDNode *&Result, bool IsDistinct) {
 #define VISIT_MD_FIELDS(OPTIONAL, REQUIRED)                                    \
   OPTIONAL(tag, DwarfTagField, (dwarf::DW_TAG_template_value_parameter));      \
   OPTIONAL(name, MDStringField, );                                             \
@@ -3668,17 +3740,17 @@ bool LLParser::ParseMDTemplateValueParameter(MDNode *&Result, bool IsDistinct) {
   PARSE_MD_FIELDS();
 #undef VISIT_MD_FIELDS
 
-  Result = GET_OR_DISTINCT(MDTemplateValueParameter,
+  Result = GET_OR_DISTINCT(DITemplateValueParameter,
                            (Context, tag.Val, name.Val, type.Val, value.Val));
   return false;
 }
 
-/// ParseMDGlobalVariable:
-///   ::= !MDGlobalVariable(scope: !0, name: "foo", linkageName: "foo",
+/// ParseDIGlobalVariable:
+///   ::= !DIGlobalVariable(scope: !0, name: "foo", linkageName: "foo",
 ///                         file: !1, line: 7, type: !2, isLocal: false,
 ///                         isDefinition: true, variable: i32* @foo,
 ///                         declaration: !3)
-bool LLParser::ParseMDGlobalVariable(MDNode *&Result, bool IsDistinct) {
+bool LLParser::ParseDIGlobalVariable(MDNode *&Result, bool IsDistinct) {
 #define VISIT_MD_FIELDS(OPTIONAL, REQUIRED)                                    \
   REQUIRED(name, MDStringField, (/* AllowEmpty */ false));                     \
   OPTIONAL(scope, MDField, );                                                  \
@@ -3693,17 +3765,17 @@ bool LLParser::ParseMDGlobalVariable(MDNode *&Result, bool IsDistinct) {
   PARSE_MD_FIELDS();
 #undef VISIT_MD_FIELDS
 
-  Result = GET_OR_DISTINCT(MDGlobalVariable,
+  Result = GET_OR_DISTINCT(DIGlobalVariable,
                            (Context, scope.Val, name.Val, linkageName.Val,
                             file.Val, line.Val, type.Val, isLocal.Val,
                             isDefinition.Val, variable.Val, declaration.Val));
   return false;
 }
 
-/// ParseMDLocalVariable:
-///   ::= !MDLocalVariable(tag: DW_TAG_arg_variable, scope: !0, name: "foo",
+/// ParseDILocalVariable:
+///   ::= !DILocalVariable(tag: DW_TAG_arg_variable, scope: !0, name: "foo",
 ///                        file: !1, line: 7, type: !2, arg: 2, flags: 7)
-bool LLParser::ParseMDLocalVariable(MDNode *&Result, bool IsDistinct) {
+bool LLParser::ParseDILocalVariable(MDNode *&Result, bool IsDistinct) {
 #define VISIT_MD_FIELDS(OPTIONAL, REQUIRED)                                    \
   REQUIRED(tag, DwarfTagField, );                                              \
   REQUIRED(scope, MDField, (/* AllowNull */ false));                           \
@@ -3711,20 +3783,20 @@ bool LLParser::ParseMDLocalVariable(MDNode *&Result, bool IsDistinct) {
   OPTIONAL(file, MDField, );                                                   \
   OPTIONAL(line, LineField, );                                                 \
   OPTIONAL(type, MDField, );                                                   \
-  OPTIONAL(arg, MDUnsignedField, (0, UINT8_MAX));                              \
+  OPTIONAL(arg, MDUnsignedField, (0, UINT16_MAX));                             \
   OPTIONAL(flags, DIFlagField, );
   PARSE_MD_FIELDS();
 #undef VISIT_MD_FIELDS
 
-  Result = GET_OR_DISTINCT(MDLocalVariable,
+  Result = GET_OR_DISTINCT(DILocalVariable,
                            (Context, tag.Val, scope.Val, name.Val, file.Val,
                             line.Val, type.Val, arg.Val, flags.Val));
   return false;
 }
 
-/// ParseMDExpression:
-///   ::= !MDExpression(0, 7, -1)
-bool LLParser::ParseMDExpression(MDNode *&Result, bool IsDistinct) {
+/// ParseDIExpression:
+///   ::= !DIExpression(0, 7, -1)
+bool LLParser::ParseDIExpression(MDNode *&Result, bool IsDistinct) {
   assert(Lex.getKind() == lltok::MetadataVar && "Expected metadata type name");
   Lex.Lex();
 
@@ -3756,14 +3828,14 @@ bool LLParser::ParseMDExpression(MDNode *&Result, bool IsDistinct) {
   if (ParseToken(lltok::rparen, "expected ')' here"))
     return true;
 
-  Result = GET_OR_DISTINCT(MDExpression, (Context, Elements));
+  Result = GET_OR_DISTINCT(DIExpression, (Context, Elements));
   return false;
 }
 
-/// ParseMDObjCProperty:
-///   ::= !MDObjCProperty(name: "foo", file: !1, line: 7, setter: "setFoo",
+/// ParseDIObjCProperty:
+///   ::= !DIObjCProperty(name: "foo", file: !1, line: 7, setter: "setFoo",
 ///                       getter: "getFoo", attributes: 7, type: !2)
-bool LLParser::ParseMDObjCProperty(MDNode *&Result, bool IsDistinct) {
+bool LLParser::ParseDIObjCProperty(MDNode *&Result, bool IsDistinct) {
 #define VISIT_MD_FIELDS(OPTIONAL, REQUIRED)                                    \
   OPTIONAL(name, MDStringField, );                                             \
   OPTIONAL(file, MDField, );                                                   \
@@ -3775,16 +3847,16 @@ bool LLParser::ParseMDObjCProperty(MDNode *&Result, bool IsDistinct) {
   PARSE_MD_FIELDS();
 #undef VISIT_MD_FIELDS
 
-  Result = GET_OR_DISTINCT(MDObjCProperty,
+  Result = GET_OR_DISTINCT(DIObjCProperty,
                            (Context, name.Val, file.Val, line.Val, setter.Val,
                             getter.Val, attributes.Val, type.Val));
   return false;
 }
 
-/// ParseMDImportedEntity:
-///   ::= !MDImportedEntity(tag: DW_TAG_imported_module, scope: !0, entity: !1,
+/// ParseDIImportedEntity:
+///   ::= !DIImportedEntity(tag: DW_TAG_imported_module, scope: !0, entity: !1,
 ///                         line: 7, name: "foo")
-bool LLParser::ParseMDImportedEntity(MDNode *&Result, bool IsDistinct) {
+bool LLParser::ParseDIImportedEntity(MDNode *&Result, bool IsDistinct) {
 #define VISIT_MD_FIELDS(OPTIONAL, REQUIRED)                                    \
   REQUIRED(tag, DwarfTagField, );                                              \
   REQUIRED(scope, MDField, );                                                  \
@@ -3794,7 +3866,7 @@ bool LLParser::ParseMDImportedEntity(MDNode *&Result, bool IsDistinct) {
   PARSE_MD_FIELDS();
 #undef VISIT_MD_FIELDS
 
-  Result = GET_OR_DISTINCT(MDImportedEntity, (Context, tag.Val, scope.Val,
+  Result = GET_OR_DISTINCT(DIImportedEntity, (Context, tag.Val, scope.Val,
                                               entity.Val, line.Val, name.Val));
   return false;
 }
@@ -3849,7 +3921,7 @@ bool LLParser::ParseValueAsMetadata(Metadata *&MD, const Twine &TypeMsg,
 ///  ::= !42
 ///  ::= !{...}
 ///  ::= !"string"
-///  ::= !MDLocation(...)
+///  ::= !DILocation(...)
 bool LLParser::ParseMetadata(Metadata *&MD, PerFunctionState *PFS) {
   if (Lex.getKind() == lltok::MetadataVar) {
     MDNode *N;
@@ -3908,13 +3980,12 @@ bool LLParser::ConvertValIDToValue(Type *Ty, ValID &ID, Value *&V,
     V = PFS->GetVal(ID.StrVal, Ty, ID.Loc);
     return V == nullptr;
   case ValID::t_InlineAsm: {
-    PointerType *PTy = dyn_cast<PointerType>(Ty);
-    FunctionType *FTy =
-      PTy ? dyn_cast<FunctionType>(PTy->getElementType()) : nullptr;
-    if (!FTy || !InlineAsm::Verify(FTy, ID.StrVal2))
+    assert(ID.FTy);
+    if (!InlineAsm::Verify(ID.FTy, ID.StrVal2))
       return Error(ID.Loc, "invalid type for inline asm constraint string");
-    V = InlineAsm::get(FTy, ID.StrVal, ID.StrVal2, ID.UIntVal&1,
-                       (ID.UIntVal>>1)&1, (InlineAsm::AsmDialect(ID.UIntVal>>2)));
+    V = InlineAsm::get(ID.FTy, ID.StrVal, ID.StrVal2, ID.UIntVal & 1,
+                       (ID.UIntVal >> 1) & 1,
+                       (InlineAsm::AsmDialect(ID.UIntVal >> 2)));
     return false;
   }
   case ValID::t_GlobalName:
@@ -4004,6 +4075,30 @@ bool LLParser::ConvertValIDToValue(Type *Ty, ValID &ID, Value *&V,
   llvm_unreachable("Invalid ValID");
 }
 
+bool LLParser::parseConstantValue(Type *Ty, Constant *&C) {
+  C = nullptr;
+  ValID ID;
+  auto Loc = Lex.getLoc();
+  if (ParseValID(ID, /*PFS=*/nullptr))
+    return true;
+  switch (ID.Kind) {
+  case ValID::t_APSInt:
+  case ValID::t_APFloat:
+  case ValID::t_Constant:
+  case ValID::t_ConstantStruct:
+  case ValID::t_PackedConstantStruct: {
+    Value *V;
+    if (ConvertValIDToValue(Ty, ID, V, /*PFS=*/nullptr))
+      return true;
+    assert(isa<Constant>(V) && "Expected a constant value");
+    C = cast<Constant>(V);
+    return false;
+  }
+  default:
+    return Error(Loc, "expected a constant value");
+  }
+}
+
 bool LLParser::ParseValue(Type *Ty, Value *&V, PerFunctionState *PFS) {
   V = nullptr;
   ValID ID;
@@ -4032,7 +4127,7 @@ bool LLParser::ParseTypeAndBasicBlock(BasicBlock *&BB, LocTy &Loc,
 /// FunctionHeader
 ///   ::= OptionalLinkage OptionalVisibility OptionalCallingConv OptRetAttrs
 ///       OptUnnamedAddr Type GlobalName '(' ArgList ')' OptFuncAttrs OptSection
-///       OptionalAlign OptGC OptionalPrefix OptionalPrologue
+///       OptionalAlign OptGC OptionalPrefix OptionalPrologue OptPersonalityFn
 bool LLParser::ParseFunctionHeader(Function *&Fn, bool isDefine) {
   // Parse the linkage.
   LocTy LinkageLoc = Lex.getLoc();
@@ -4114,6 +4209,7 @@ bool LLParser::ParseFunctionHeader(Function *&Fn, bool isDefine) {
   LocTy UnnamedAddrLoc;
   Constant *Prefix = nullptr;
   Constant *Prologue = nullptr;
+  Constant *PersonalityFn = nullptr;
   Comdat *C;
 
   if (ParseArgumentList(ArgList, isVarArg) ||
@@ -4130,7 +4226,9 @@ bool LLParser::ParseFunctionHeader(Function *&Fn, bool isDefine) {
       (EatIfPresent(lltok::kw_prefix) &&
        ParseGlobalTypeAndValue(Prefix)) ||
       (EatIfPresent(lltok::kw_prologue) &&
-       ParseGlobalTypeAndValue(Prologue)))
+       ParseGlobalTypeAndValue(Prologue)) ||
+      (EatIfPresent(lltok::kw_personality) &&
+       ParseGlobalTypeAndValue(PersonalityFn)))
     return true;
 
   if (FuncAttrs.contains(Attribute::Builtin))
@@ -4229,6 +4327,7 @@ bool LLParser::ParseFunctionHeader(Function *&Fn, bool isDefine) {
   Fn->setAlignment(Alignment);
   Fn->setSection(Section);
   Fn->setComdat(C);
+  Fn->setPersonalityFn(PersonalityFn);
   if (!GC.empty()) Fn->setGC(GC.c_str());
   Fn->setPrefixData(Prefix);
   Fn->setPrologueData(Prologue);
@@ -4388,7 +4487,7 @@ bool LLParser::ParseBasicBlock(PerFunctionState &PFS) {
       // With a normal result, we check to see if the instruction is followed by
       // a comma and metadata.
       if (EatIfPresent(lltok::comma))
-        if (ParseInstructionMetadata(Inst, &PFS))
+        if (ParseInstructionMetadata(*Inst))
           return true;
       break;
     case InstExtraComma:
@@ -4396,7 +4495,7 @@ bool LLParser::ParseBasicBlock(PerFunctionState &PFS) {
 
       // If the instruction parser ate an extra comma at the end of it, it
       // *must* be followed by metadata.
-      if (ParseInstructionMetadata(Inst, &PFS))
+      if (ParseInstructionMetadata(*Inst))
         return true;
       break;
     }
@@ -4433,6 +4532,12 @@ int LLParser::ParseInstruction(Instruction *&Inst, BasicBlock *BB,
   case lltok::kw_indirectbr:  return ParseIndirectBr(Inst, PFS);
   case lltok::kw_invoke:      return ParseInvoke(Inst, PFS);
   case lltok::kw_resume:      return ParseResume(Inst, PFS);
+  case lltok::kw_cleanupret:  return ParseCleanupRet(Inst, PFS);
+  case lltok::kw_catchret:    return ParseCatchRet(Inst, PFS);
+  case lltok::kw_catchpad:  return ParseCatchPad(Inst, PFS);
+  case lltok::kw_terminatepad: return ParseTerminatePad(Inst, PFS);
+  case lltok::kw_cleanuppad: return ParseCleanupPad(Inst, PFS);
+  case lltok::kw_catchendpad: return ParseCatchEndPad(Inst, PFS);
   // Binary Operators.
   case lltok::kw_add:
   case lltok::kw_sub:
@@ -4478,8 +4583,17 @@ int LLParser::ParseInstruction(Instruction *&Inst, BasicBlock *BB,
   case lltok::kw_and:
   case lltok::kw_or:
   case lltok::kw_xor:    return ParseLogical(Inst, PFS, KeywordVal);
-  case lltok::kw_icmp:
-  case lltok::kw_fcmp:   return ParseCompare(Inst, PFS, KeywordVal);
+  case lltok::kw_icmp:   return ParseCompare(Inst, PFS, KeywordVal);
+  case lltok::kw_fcmp: {
+    FastMathFlags FMF = EatFastMathFlagsIfPresent();
+    int Res = ParseCompare(Inst, PFS, KeywordVal);
+    if (Res != 0)
+      return Res;
+    if (FMF.any())
+      Inst->setFastMathFlags(FMF);
+    return 0;
+  }
+
   // Casts.
   case lltok::kw_trunc:
   case lltok::kw_zext:
@@ -4742,10 +4856,8 @@ bool LLParser::ParseInvoke(Instruction *&Inst, PerFunctionState &PFS) {
   // If RetType is a non-function pointer type, then this is the short syntax
   // for the call, which means that RetType is just the return type.  Infer the
   // rest of the function argument types from the arguments that are present.
-  PointerType *PFTy = nullptr;
-  FunctionType *Ty = nullptr;
-  if (!(PFTy = dyn_cast<PointerType>(RetType)) ||
-      !(Ty = dyn_cast<FunctionType>(PFTy->getElementType()))) {
+  FunctionType *Ty = dyn_cast<FunctionType>(RetType);
+  if (!Ty) {
     // Pull out the types of all of the arguments...
     std::vector<Type*> ParamTypes;
     for (unsigned i = 0, e = ArgList.size(); i != e; ++i)
@@ -4755,12 +4867,14 @@ bool LLParser::ParseInvoke(Instruction *&Inst, PerFunctionState &PFS) {
       return Error(RetTypeLoc, "Invalid result type for LLVM function");
 
     Ty = FunctionType::get(RetType, ParamTypes, false);
-    PFTy = PointerType::getUnqual(Ty);
   }
 
+  CalleeID.FTy = Ty;
+
   // Look up the callee.
   Value *Callee;
-  if (ConvertValIDToValue(PFTy, CalleeID, Callee, &PFS)) return true;
+  if (ConvertValIDToValue(PointerType::getUnqual(Ty), CalleeID, Callee, &PFS))
+    return true;
 
   // Set up the Attribute for the function.
   SmallVector<AttributeSet, 8> Attrs;
@@ -4808,7 +4922,7 @@ bool LLParser::ParseInvoke(Instruction *&Inst, PerFunctionState &PFS) {
   // Finish off the Attribute and check them
   AttributeSet PAL = AttributeSet::get(Context, Attrs);
 
-  InvokeInst *II = InvokeInst::Create(Callee, NormalBB, UnwindBB, Args);
+  InvokeInst *II = InvokeInst::Create(Ty, Callee, NormalBB, UnwindBB, Args);
   II->setCallingConv(CC);
   II->setAttributes(PAL);
   ForwardRefAttrGroups[II] = FwdRefAttrGrps;
@@ -4828,6 +4942,161 @@ bool LLParser::ParseResume(Instruction *&Inst, PerFunctionState &PFS) {
   return false;
 }
 
+bool LLParser::ParseExceptionArgs(SmallVectorImpl<Value *> &Args,
+                                  PerFunctionState &PFS) {
+  if (ParseToken(lltok::lsquare, "expected '[' in cleanuppad"))
+    return true;
+
+  while (Lex.getKind() != lltok::rsquare) {
+    // If this isn't the first argument, we need a comma.
+    if (!Args.empty() &&
+        ParseToken(lltok::comma, "expected ',' in argument list"))
+      return true;
+
+    // Parse the argument.
+    LocTy ArgLoc;
+    Type *ArgTy = nullptr;
+    if (ParseType(ArgTy, ArgLoc))
+      return true;
+
+    Value *V;
+    if (ArgTy->isMetadataTy()) {
+      if (ParseMetadataAsValue(V, PFS))
+        return true;
+    } else {
+      if (ParseValue(ArgTy, V, PFS))
+        return true;
+    }
+    Args.push_back(V);
+  }
+
+  Lex.Lex();  // Lex the ']'.
+  return false;
+}
+
+/// ParseCleanupRet
+///   ::= 'cleanupret' ('void' | TypeAndValue) unwind ('to' 'caller' | TypeAndValue)
+bool LLParser::ParseCleanupRet(Instruction *&Inst, PerFunctionState &PFS) {
+  Type *RetTy = nullptr;
+  Value *RetVal = nullptr;
+  if (ParseType(RetTy, /*AllowVoid=*/true))
+    return true;
+
+  if (!RetTy->isVoidTy())
+    if (ParseValue(RetTy, RetVal, PFS))
+      return true;
+
+  if (ParseToken(lltok::kw_unwind, "expected 'unwind' in cleanupret"))
+    return true;
+
+  BasicBlock *UnwindBB = nullptr;
+  if (Lex.getKind() == lltok::kw_to) {
+    Lex.Lex();
+    if (ParseToken(lltok::kw_caller, "expected 'caller' in cleanupret"))
+      return true;
+  } else {
+    if (ParseTypeAndBasicBlock(UnwindBB, PFS)) {
+      return true;
+    }
+  }
+
+  Inst = CleanupReturnInst::Create(Context, RetVal, UnwindBB);
+  return false;
+}
+
+/// ParseCatchRet
+///   ::= 'catchret' TypeAndValue
+bool LLParser::ParseCatchRet(Instruction *&Inst, PerFunctionState &PFS) {
+  BasicBlock *BB;
+  if (ParseTypeAndBasicBlock(BB, PFS))
+      return true;
+
+  Inst = CatchReturnInst::Create(BB);
+  return false;
+}
+
+/// ParseCatchPad
+///   ::= 'catchpad' Type ParamList 'to' TypeAndValue 'unwind' TypeAndValue
+bool LLParser::ParseCatchPad(Instruction *&Inst, PerFunctionState &PFS) {
+  Type *RetType = nullptr;
+
+  SmallVector<Value *, 8> Args;
+  if (ParseType(RetType, /*AllowVoid=*/true) || ParseExceptionArgs(Args, PFS))
+    return true;
+
+  BasicBlock *NormalBB, *UnwindBB;
+  if (ParseToken(lltok::kw_to, "expected 'to' in catchpad") ||
+      ParseTypeAndBasicBlock(NormalBB, PFS) ||
+      ParseToken(lltok::kw_unwind, "expected 'unwind' in catchpad") ||
+      ParseTypeAndBasicBlock(UnwindBB, PFS))
+    return true;
+
+  Inst = CatchPadInst::Create(RetType, NormalBB, UnwindBB, Args);
+  return false;
+}
+
+/// ParseTerminatePad
+///   ::= 'terminatepad' ParamList 'to' TypeAndValue
+bool LLParser::ParseTerminatePad(Instruction *&Inst, PerFunctionState &PFS) {
+  SmallVector<Value *, 8> Args;
+  if (ParseExceptionArgs(Args, PFS))
+    return true;
+
+  if (ParseToken(lltok::kw_unwind, "expected 'unwind' in terminatepad"))
+    return true;
+
+  BasicBlock *UnwindBB = nullptr;
+  if (Lex.getKind() == lltok::kw_to) {
+    Lex.Lex();
+    if (ParseToken(lltok::kw_caller, "expected 'caller' in terminatepad"))
+      return true;
+  } else {
+    if (ParseTypeAndBasicBlock(UnwindBB, PFS)) {
+      return true;
+    }
+  }
+
+  Inst = TerminatePadInst::Create(Context, UnwindBB, Args);
+  return false;
+}
+
+/// ParseCleanupPad
+///   ::= 'cleanuppad' ParamList
+bool LLParser::ParseCleanupPad(Instruction *&Inst, PerFunctionState &PFS) {
+  Type *RetType = nullptr;
+
+  SmallVector<Value *, 8> Args;
+  if (ParseType(RetType, /*AllowVoid=*/true) || ParseExceptionArgs(Args, PFS))
+    return true;
+
+  Inst = CleanupPadInst::Create(RetType, Args);
+  return false;
+}
+
+/// ParseCatchEndPad
+///   ::= 'catchendpad' unwind ('to' 'caller' | TypeAndValue)
+bool LLParser::ParseCatchEndPad(Instruction *&Inst, PerFunctionState &PFS) {
+  if (ParseToken(lltok::kw_unwind, "expected 'unwind' in catchendpad"))
+    return true;
+
+  BasicBlock *UnwindBB = nullptr;
+  if (Lex.getKind() == lltok::kw_to) {
+    Lex.Lex();
+    if (Lex.getKind() == lltok::kw_caller) {
+      Lex.Lex();
+    } else {
+      return true;
+    }
+  } else {
+    if (ParseTypeAndBasicBlock(UnwindBB, PFS)) {
+      return true;
+    }
+  }
+
+  Inst = CatchEndPadInst::Create(Context, UnwindBB);
+  return false;
+}
+
 //===----------------------------------------------------------------------===//
 // Binary Operators.
 //===----------------------------------------------------------------------===//
@@ -5082,14 +5351,11 @@ int LLParser::ParsePHI(Instruction *&Inst, PerFunctionState &PFS) {
 ///   ::= 'filter' TypeAndValue ( ',' TypeAndValue )*
 bool LLParser::ParseLandingPad(Instruction *&Inst, PerFunctionState &PFS) {
   Type *Ty = nullptr; LocTy TyLoc;
-  Value *PersFn; LocTy PersFnLoc;
 
-  if (ParseType(Ty, TyLoc) ||
-      ParseToken(lltok::kw_personality, "expected 'personality'") ||
-      ParseTypeAndValue(PersFn, PersFnLoc, PFS))
+  if (ParseType(Ty, TyLoc))
     return true;
 
-  std::unique_ptr<LandingPadInst> LP(LandingPadInst::Create(Ty, PersFn, 0));
+  std::unique_ptr<LandingPadInst> LP(LandingPadInst::Create(Ty, 0));
   LP->setCleanup(EatIfPresent(lltok::kw_cleanup));
 
   while (Lex.getKind() == lltok::kw_catch || Lex.getKind() == lltok::kw_filter){
@@ -5173,6 +5439,8 @@ bool LLParser::ParseCall(Instruction *&Inst, PerFunctionState &PFS,
     Ty = FunctionType::get(RetType, ParamTypes, false);
   }
 
+  CalleeID.FTy = Ty;
+
   // Look up the callee.
   Value *Callee;
   if (ConvertValIDToValue(PointerType::getUnqual(Ty), CalleeID, Callee, &PFS))
@@ -5224,7 +5492,7 @@ bool LLParser::ParseCall(Instruction *&Inst, PerFunctionState &PFS,
   // Finish off the Attribute and check them
   AttributeSet PAL = AttributeSet::get(Context, Attrs);
 
-  CallInst *CI = CallInst::Create(Callee, Args);
+  CallInst *CI = CallInst::Create(Ty, Callee, Args);
   CI->setTailCallKind(TCK);
   CI->setCallingConv(CC);
   CI->setAttributes(PAL);
@@ -5521,6 +5789,11 @@ int LLParser::ParseGetElementPtr(Instruction *&Inst, PerFunctionState &PFS) {
 
   SmallVector<Value*, 16> Indices;
   bool AteExtraComma = false;
+  // GEP returns a vector of pointers if at least one of parameters is a vector.
+  // All vector parameters should have the same vector width.
+  unsigned GEPWidth = BaseType->isVectorTy() ?
+    BaseType->getVectorNumElements() : 0;
+
   while (EatIfPresent(lltok::comma)) {
     if (Lex.getKind() == lltok::MetadataVar) {
       AteExtraComma = true;
@@ -5529,26 +5802,22 @@ int LLParser::ParseGetElementPtr(Instruction *&Inst, PerFunctionState &PFS) {
     if (ParseTypeAndValue(Val, EltLoc, PFS)) return true;
     if (!Val->getType()->getScalarType()->isIntegerTy())
       return Error(EltLoc, "getelementptr index must be an integer");
-    if (Val->getType()->isVectorTy() != Ptr->getType()->isVectorTy())
-      return Error(EltLoc, "getelementptr index type missmatch");
+
     if (Val->getType()->isVectorTy()) {
-      unsigned ValNumEl = cast<VectorType>(Val->getType())->getNumElements();
-      unsigned PtrNumEl = cast<VectorType>(Ptr->getType())->getNumElements();
-      if (ValNumEl != PtrNumEl)
+      unsigned ValNumEl = Val->getType()->getVectorNumElements();
+      if (GEPWidth && GEPWidth != ValNumEl)
         return Error(EltLoc,
           "getelementptr vector index has a wrong number of elements");
+      GEPWidth = ValNumEl;
     }
     Indices.push_back(Val);
   }
 
   SmallPtrSet<const Type*, 4> Visited;
-  if (!Indices.empty() &&
-      !BasePointerType->getElementType()->isSized(&Visited))
+  if (!Indices.empty() && !Ty->isSized(&Visited))
     return Error(Loc, "base element of getelementptr must be sized");
 
-  if (!GetElementPtrInst::getIndexedType(
-          cast<PointerType>(BaseType->getScalarType())->getElementType(),
-          Indices))
+  if (!GetElementPtrInst::getIndexedType(Ty, Indices))
     return Error(Loc, "invalid getelementptr indices");
   Inst = GetElementPtrInst::Create(Ty, Ptr, Indices);
   if (InBounds)
diff --git a/lib/AsmParser/LLParser.h b/lib/AsmParser/LLParser.h
index 117cdcba255..0c066f4b603 100644
--- a/lib/AsmParser/LLParser.h
+++ b/lib/AsmParser/LLParser.h
@@ -37,6 +37,7 @@ namespace llvm {
   class Comdat;
   class MDString;
   class MDNode;
+  struct SlotMapping;
   class StructType;
 
   /// ValID - Represents a reference of a definition of some sort with no type.
@@ -51,13 +52,14 @@ namespace llvm {
       t_Null, t_Undef, t_Zero,    // No value.
       t_EmptyArray,               // No value:  []
       t_Constant,                 // Value in ConstantVal.
-      t_InlineAsm,                // Value in StrVal/StrVal2/UIntVal.
+      t_InlineAsm,                // Value in FTy/StrVal/StrVal2/UIntVal.
       t_ConstantStruct,           // Value in ConstantStructElts.
       t_PackedConstantStruct      // Value in ConstantStructElts.
     } Kind;
 
     LLLexer::LocTy Loc;
     unsigned UIntVal;
+    FunctionType *FTy;
     std::string StrVal, StrVal2;
     APSInt APSIntVal;
     APFloat APFloatVal;
@@ -87,6 +89,7 @@ namespace llvm {
     LLVMContext &Context;
     LLLexer Lex;
     Module *M;
+    SlotMapping *Slots;
 
     // Instruction metadata resolution.  Each instruction can have a list of
     // MDRef info associated with them.
@@ -135,11 +138,14 @@ namespace llvm {
     std::map<unsigned, AttrBuilder> NumberedAttrBuilders;
 
   public:
-    LLParser(StringRef F, SourceMgr &SM, SMDiagnostic &Err, Module *m)
-        : Context(m->getContext()), Lex(F, SM, Err, m->getContext()), M(m),
-          BlockAddressPFS(nullptr) {}
+    LLParser(StringRef F, SourceMgr &SM, SMDiagnostic &Err, Module *M,
+             SlotMapping *Slots = nullptr)
+        : Context(M->getContext()), Lex(F, SM, Err, M->getContext()), M(M),
+          Slots(Slots), BlockAddressPFS(nullptr) {}
     bool Run();
 
+    bool parseStandaloneConstantValue(Constant *&C);
+
     LLVMContext &getContext() { return Context; }
 
   private:
@@ -340,6 +346,7 @@ namespace llvm {
     bool ConvertValIDToValue(Type *Ty, ValID &ID, Value *&V,
                              PerFunctionState *PFS);
 
+    bool parseConstantValue(Type *Ty, Constant *&C);
     bool ParseValue(Type *Ty, Value *&V, PerFunctionState *PFS);
     bool ParseValue(Type *Ty, Value *&V, PerFunctionState &PFS) {
       return ParseValue(Ty, V, &PFS);
@@ -378,6 +385,9 @@ namespace llvm {
                             bool IsMustTailCall = false,
                             bool InVarArgsFunc = false);
 
+    bool ParseExceptionArgs(SmallVectorImpl<Value *> &Args,
+                            PerFunctionState &PFS);
+
     // Constant Parsing.
     bool ParseValID(ValID &ID, PerFunctionState *PFS = nullptr);
     bool ParseGlobalValue(Type *Ty, Constant *&V);
@@ -392,7 +402,9 @@ namespace llvm {
     bool ParseMDNode(MDNode *&MD);
     bool ParseMDNodeTail(MDNode *&MD);
     bool ParseMDNodeVector(SmallVectorImpl<Metadata *> &MDs);
-    bool ParseInstructionMetadata(Instruction *Inst, PerFunctionState *PFS);
+    bool ParseMetadataAttachment(unsigned &Kind, MDNode *&MD);
+    bool ParseInstructionMetadata(Instruction &Inst);
+    bool ParseOptionalFunctionMetadata(Function &F);
 
     template <class FieldTy>
     bool ParseMDField(LocTy Loc, StringRef Name, FieldTy &Result);
@@ -436,6 +448,12 @@ namespace llvm {
     bool ParseIndirectBr(Instruction *&Inst, PerFunctionState &PFS);
     bool ParseInvoke(Instruction *&Inst, PerFunctionState &PFS);
     bool ParseResume(Instruction *&Inst, PerFunctionState &PFS);
+    bool ParseCleanupRet(Instruction *&Inst, PerFunctionState &PFS);
+    bool ParseCatchRet(Instruction *&Inst, PerFunctionState &PFS);
+    bool ParseCatchPad(Instruction *&Inst, PerFunctionState &PFS);
+    bool ParseTerminatePad(Instruction *&Inst, PerFunctionState &PFS);
+    bool ParseCleanupPad(Instruction *&Inst, PerFunctionState &PFS);
+    bool ParseCatchEndPad(Instruction *&Inst, PerFunctionState &PFS);
 
     bool ParseArithmetic(Instruction *&I, PerFunctionState &PFS, unsigned Opc,
                          unsigned OperandType);
diff --git a/lib/AsmParser/LLToken.h b/lib/AsmParser/LLToken.h
index 2bdc53b3e2e..19bf7e4e4c1 100644
--- a/lib/AsmParser/LLToken.h
+++ b/lib/AsmParser/LLToken.h
@@ -51,6 +51,7 @@ namespace lltok {
     kw_zeroinitializer,
     kw_undef, kw_null,
     kw_to,
+    kw_caller,
     kw_tail,
     kw_musttail,
     kw_target,
@@ -100,11 +101,13 @@ namespace lltok {
     // Attributes:
     kw_attributes,
     kw_alwaysinline,
+    kw_argmemonly,
     kw_sanitize_address,
     kw_builtin,
     kw_byval,
     kw_inalloca,
     kw_cold,
+    kw_convergent,
     kw_dereferenceable,
     kw_dereferenceable_or_null,
     kw_inlinehint,
@@ -134,6 +137,7 @@ namespace lltok {
     kw_ssp,
     kw_sspreq,
     kw_sspstrong,
+    kw_safestack,
     kw_sret,
     kw_sanitize_thread,
     kw_sanitize_memory,
@@ -174,7 +178,8 @@ namespace lltok {
     kw_landingpad, kw_personality, kw_cleanup, kw_catch, kw_filter,
 
     kw_ret, kw_br, kw_switch, kw_indirectbr, kw_invoke, kw_resume,
-    kw_unreachable,
+    kw_unreachable, kw_cleanupret, kw_catchret, kw_catchpad,
+    kw_terminatepad, kw_cleanuppad, kw_catchendpad,
 
     kw_alloca, kw_load, kw_store, kw_fence, kw_cmpxchg, kw_atomicrmw,
     kw_getelementptr,
diff --git a/lib/AsmParser/Parser.cpp b/lib/AsmParser/Parser.cpp
index 2e76c0ecc45..6135b43f00d 100644
--- a/lib/AsmParser/Parser.cpp
+++ b/lib/AsmParser/Parser.cpp
@@ -22,21 +22,23 @@
 #include <system_error>
 using namespace llvm;
 
-bool llvm::parseAssemblyInto(MemoryBufferRef F, Module &M, SMDiagnostic &Err) {
+bool llvm::parseAssemblyInto(MemoryBufferRef F, Module &M, SMDiagnostic &Err,
+                             SlotMapping *Slots) {
   SourceMgr SM;
-  std::unique_ptr<MemoryBuffer> Buf = MemoryBuffer::getMemBuffer(F, false);
+  std::unique_ptr<MemoryBuffer> Buf = MemoryBuffer::getMemBuffer(F);
   SM.AddNewSourceBuffer(std::move(Buf), SMLoc());
 
-  return LLParser(F.getBuffer(), SM, Err, &M).Run();
+  return LLParser(F.getBuffer(), SM, Err, &M, Slots).Run();
 }
 
 std::unique_ptr<Module> llvm::parseAssembly(MemoryBufferRef F,
                                             SMDiagnostic &Err,
-                                            LLVMContext &Context) {
+                                            LLVMContext &Context,
+                                            SlotMapping *Slots) {
   std::unique_ptr<Module> M =
       make_unique<Module>(F.getBufferIdentifier(), Context);
 
-  if (parseAssemblyInto(F, *M, Err))
+  if (parseAssemblyInto(F, *M, Err, Slots))
     return nullptr;
 
   return M;
@@ -44,7 +46,8 @@ std::unique_ptr<Module> llvm::parseAssembly(MemoryBufferRef F,
 
 std::unique_ptr<Module> llvm::parseAssemblyFile(StringRef Filename,
                                                 SMDiagnostic &Err,
-                                                LLVMContext &Context) {
+                                                LLVMContext &Context,
+                                                SlotMapping *Slots) {
   ErrorOr<std::unique_ptr<MemoryBuffer>> FileOrErr =
       MemoryBuffer::getFileOrSTDIN(Filename);
   if (std::error_code EC = FileOrErr.getError()) {
@@ -53,12 +56,25 @@ std::unique_ptr<Module> llvm::parseAssemblyFile(StringRef Filename,
     return nullptr;
   }
 
-  return parseAssembly(FileOrErr.get()->getMemBufferRef(), Err, Context);
+  return parseAssembly(FileOrErr.get()->getMemBufferRef(), Err, Context, Slots);
 }
 
 std::unique_ptr<Module> llvm::parseAssemblyString(StringRef AsmString,
                                                   SMDiagnostic &Err,
-                                                  LLVMContext &Context) {
+                                                  LLVMContext &Context,
+                                                  SlotMapping *Slots) {
   MemoryBufferRef F(AsmString, "<string>");
-  return parseAssembly(F, Err, Context);
+  return parseAssembly(F, Err, Context, Slots);
+}
+
+Constant *llvm::parseConstantValue(StringRef Asm, SMDiagnostic &Err,
+                                   const Module &M) {
+  SourceMgr SM;
+  std::unique_ptr<MemoryBuffer> Buf = MemoryBuffer::getMemBuffer(Asm);
+  SM.AddNewSourceBuffer(std::move(Buf), SMLoc());
+  Constant *C;
+  if (LLParser(Asm, SM, Err, const_cast<Module *>(&M))
+          .parseStandaloneConstantValue(C))
+    return nullptr;
+  return C;
 }
diff --git a/lib/Bitcode/CMakeLists.txt b/lib/Bitcode/CMakeLists.txt
index 8969ec83f56..ff7e290cad1 100644
--- a/lib/Bitcode/CMakeLists.txt
+++ b/lib/Bitcode/CMakeLists.txt
@@ -1,3 +1,2 @@
 add_subdirectory(Reader)
 add_subdirectory(Writer)
-add_subdirectory(NaCl)
diff --git a/lib/Bitcode/LLVMBuild.txt b/lib/Bitcode/LLVMBuild.txt
index 415a33dfdff..af9936bbe82 100644
--- a/lib/Bitcode/LLVMBuild.txt
+++ b/lib/Bitcode/LLVMBuild.txt
@@ -16,7 +16,7 @@
 ;===------------------------------------------------------------------------===;
 
 [common]
-subdirectories = Reader Writer NaCl
+subdirectories = Reader Writer
 
 [component_0]
 type = Group
diff --git a/lib/Bitcode/Makefile b/lib/Bitcode/Makefile
index cbaab3578cd..2d6b5ad1fe8 100644
--- a/lib/Bitcode/Makefile
+++ b/lib/Bitcode/Makefile
@@ -8,7 +8,7 @@
 ##===----------------------------------------------------------------------===##
 
 LEVEL = ../..
-PARALLEL_DIRS = Reader Writer NaCl
+PARALLEL_DIRS = Reader Writer
 
 include $(LEVEL)/Makefile.common
 
diff --git a/lib/Bitcode/NaCl/Analysis/AbbrevTrieNode.cpp b/lib/Bitcode/NaCl/Analysis/AbbrevTrieNode.cpp
deleted file mode 100644
index 7ff3e36fea4..00000000000
--- a/lib/Bitcode/NaCl/Analysis/AbbrevTrieNode.cpp
+++ /dev/null
@@ -1,295 +0,0 @@
-//===- AbbrevTrieNode.cpp ------------------------------------------------===//
-//     Implements abbreviation lookup tries.
-//
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
-//
-//===----------------------------------------------------------------------===//
-
-#include "llvm/Bitcode/NaCl/AbbrevTrieNode.h"
-#include "llvm/Bitcode/NaCl/NaClBitcodeValueDist.h"
-
-using namespace llvm;
-
-void AbbrevTrieNode::GetSuccessorLabels(SuccessorLabels &Labels) const {
-  for (SuccessorMap::const_iterator
-           IndexIter = Successors.begin(), IndexIterEnd = Successors.end();
-       IndexIter != IndexIterEnd; ++IndexIter) {
-    if (const SuccessorValueMap *ValueMap = IndexIter->second) {
-      for (SuccessorValueMap::const_iterator
-               ValueIter = ValueMap->begin(), ValueIterEnd = ValueMap->end();
-           ValueIter != ValueIterEnd; ++ValueIter) {
-        Labels.push_back(std::pair<size_t, uint64_t>(IndexIter->first,
-                                                     ValueIter->first));
-      }
-    }
-  }
-}
-
-AbbrevTrieNode::~AbbrevTrieNode() {
-  for (SuccessorMap::const_iterator
-           Iter = Successors.begin(), IterEnd = Successors.end();
-       Iter != IterEnd; ++Iter) {
-    if (const SuccessorValueMap *ValueMap = Iter->second) {
-      for (SuccessorValueMap::const_iterator
-               Iter = ValueMap->begin(), IterEnd = ValueMap->end();
-           Iter != IterEnd; ++Iter) {
-        delete Iter->second;
-      }
-      delete ValueMap;
-    }
-  }
-}
-
-void AbbrevTrieNode::Print(raw_ostream &Stream,
-                           const std::string &Indent,
-                           bool LocalOnly) const {
-  std::string IndentPlus(Indent);
-  IndentPlus.append("  ");
-  std::string IndentPlusPlus(IndentPlus);
-  IndentPlusPlus.append("  ");
-  if (! Abbreviations.empty()) {
-    Stream << Indent << "Abbreviations:\n";
-    for (std::set<AbbrevIndexPair>::const_iterator
-             Iter = Abbreviations.begin(), IterEnd = Abbreviations.end();
-         Iter != IterEnd; ++Iter) {
-      Stream << IndentPlus;
-      Iter->second->Print(Stream, /* AddNewline= */ false);
-      Stream << " (abbrev #" << Iter->first << ")\n";
-    }
-  }
-  if (LocalOnly) return;
-  if (!Successors.empty()) {
-    Stream << Indent << "Successor Map:\n";
-    SuccessorLabels Labels;
-    GetSuccessorLabels(Labels);
-    for (SuccessorLabels::const_iterator
-             Iter = Labels.begin(), IterEnd = Labels.end();
-         Iter != IterEnd; ++Iter) {
-      size_t Index = Iter->first;
-      if (Index == 0) {
-        Stream << IndentPlus << "Record.Code = " << Iter->second << "\n";
-      } else {
-        Stream << IndentPlus << "Record.Values[" << (Index-1)
-               << "] = " << Iter->second << "\n";
-      }
-      GetSuccessor(Iter->first, Iter->second)->Print(Stream, IndentPlusPlus);
-    }
-  }
-}
-
-AbbrevTrieNode *AbbrevTrieNode::
-GetSuccessor(size_t Index, uint64_t Value) const {
-  SuccessorMap::const_iterator IndexPos = Successors.find(Index);
-  if (IndexPos != Successors.end()) {
-    if (SuccessorValueMap *ValueMap = IndexPos->second) {
-      SuccessorValueMap::iterator ValuePos = ValueMap->find(Value);
-      if (ValuePos != ValueMap->end())
-        return ValuePos->second;
-    }
-  }
-  return 0;
-}
-
-bool AbbrevTrieNode::Add(NaClBitCodeAbbrev *Abbrev,
-                         size_t Index, size_t SkipIndex) {
-  if (Index >= Abbrev->getNumOperandInfos()) return false;
-  bool AddedNodes = false;
-
-  // Skip over matches that may match because they don't have constants
-  // in the index.
-  while (SkipIndex < Index) {
-    SuccessorMap::iterator Pos = Successors.find(SkipIndex);
-    if (Pos != Successors.end()) {
-      SuccessorValueMap *ValueMap = Pos->second;
-      for (SuccessorValueMap::const_iterator
-               Iter = ValueMap->begin(), IterEnd = ValueMap->end();
-           Iter != IterEnd; ++Iter) {
-        if (AbbrevTrieNode *Successor = Iter->second) {
-          if (Successor->Add(Abbrev, Index, SkipIndex+1))
-            AddedNodes = true;
-        }
-      }
-    }
-    ++SkipIndex;
-  }
-
-  // Now update successors for next matching constant in abbreviation.
-  for (; Index < Abbrev->GetMinRecordSize(); ++Index) {
-    const NaClBitCodeAbbrevOp &Op = Abbrev->getOperandInfo(Index);
-    if (Op.isLiteral()) {
-      if (Index == SkipIndex) {
-        // No preceeding nodes to match, add here.
-        SuccessorValueMap *ValueMap = Successors[Index];
-        if (ValueMap == 0) {
-          ValueMap = new SuccessorValueMap();
-          Successors[Index] = ValueMap;
-        }
-        AbbrevTrieNode *Successor = (*ValueMap)[Op.getValue()];
-        if (Successor == 0) {
-          Successor = new AbbrevTrieNode();
-          AddedNodes = true;
-          (*ValueMap)[Op.getValue()] = Successor;
-        }
-        if (Successor->Add(Abbrev, Index+1, Index+1))
-          AddedNodes = true;
-      } else {
-        // Need to match all possible prefixes before inserting constant.
-        if (Add(Abbrev, Index, SkipIndex))
-          AddedNodes = true;
-      }
-      return AddedNodes;
-    }
-  }
-  return AddedNodes;
-}
-
-void AbbrevTrieNode::Insert(AbbrevIndexPair &AbbrevPair) {
-  NaClBitCodeAbbrev *Abbrev = AbbrevPair.second;
-
-  for (std::map<size_t, SuccessorValueMap*>::iterator
-           Iter = Successors.begin(), IterEnd = Successors.end();
-       Iter != IterEnd; ++Iter) {
-    if (SuccessorValueMap *ValueMap = Iter->second) {
-      if (Iter->first < Abbrev->GetMinRecordSize()) {
-        const NaClBitCodeAbbrevOp &Op =
-            Abbrev->getOperandInfo(Iter->first);
-        if (Op.isLiteral()) {
-          uint64_t Value = Op.getValue();
-          SuccessorValueMap::iterator Pos = ValueMap->find(Value);
-          if (Pos != ValueMap->end()) {
-            if (AbbrevTrieNode *Next = Pos->second) {
-              // Found constant that will be followed, update subgraph
-              // and quit.
-              Next->Insert(AbbrevPair);
-              return;
-            }
-          }
-        } else {
-          // Not literal, so it may (or may not) be followed, depending
-          // on the value in the record that will be matched against. So
-          // add to subgraph, and here if no succeeding matching literal
-          // constants.
-          for (SuccessorValueMap::iterator
-                   Iter = ValueMap->begin(), IterEnd = ValueMap->end();
-               Iter != IterEnd; ++Iter) {
-            if (AbbrevTrieNode *Next = Iter->second)
-              Next->Insert(AbbrevPair);
-          }
-        }
-      } else if (!Abbrev->IsFixedSize()) {
-        // May match array element. So add to subgraph as well as here.
-        for (SuccessorValueMap::iterator
-                 Iter = ValueMap->begin(), IterEnd = ValueMap->end();
-             Iter != IterEnd; ++Iter) {
-          if (AbbrevTrieNode *Next = Iter->second)
-            Next->Insert(AbbrevPair);
-        }
-      }
-    }
-  }
-
-  // If reached, no guarantees that any edge was followed, so add
-  // to matches of this node.
-  Abbreviations.insert(AbbrevPair);
-}
-
-const AbbrevTrieNode *AbbrevTrieNode::
-MatchRecord(const NaClBitcodeRecordData &Record) const {
-  for (std::map<size_t, SuccessorValueMap*>::const_iterator
-           Iter = Successors.begin(), IterEnd = Successors.end();
-       Iter != IterEnd; ++Iter) {
-    if (SuccessorValueMap *ValueMap = Iter->second) {
-      if (Iter->first <= Record.Values.size()) {
-        uint64_t Value;
-        if (Iter->first == 0)
-          Value = Record.Code;
-        else
-          Value = Record.Values[Iter->first-1];
-        SuccessorValueMap::iterator Pos = ValueMap->find(Value);
-        if (Pos != ValueMap->end()) {
-          if (AbbrevTrieNode *Next = Pos->second) {
-            return Next->MatchRecord(Record);
-          }
-        }
-      } else {
-        // Map index too big, quit.
-        break;
-      }
-    }
-  }
-
-  // If reached, no refinement found, use this node.
-  return this;
-}
-
-static void ComputeAbbrevRange(NaClBitCodeAbbrev *Abbrev,
-                               size_t &MinIndex, size_t &MaxIndex) {
-  // Find the range of record lengths for which the abbreviation may
-  // apply. Note: To keep a limit on the number of copies, collapse all
-  // records with length > NaClValueIndexCutoff into the same bucket.
-  MinIndex = Abbrev->GetMinRecordSize();
-  if (MinIndex > NaClValueIndexCutoff) {
-    MinIndex = NaClValueIndexCutoff + 1;
-  }
-  MaxIndex = MinIndex;
-  if (!Abbrev->IsFixedSize()) {
-    MaxIndex = NaClValueIndexCutoff + 1;
-  }
-}
-
-// Once all nodes have been added (via calls to AddAbbrevToLookupMap),
-// this function adds the given abbreviation pair to all possible
-// matchxes in the lookup map.
-static void AddAbbrevPairToLookupMap(AbbrevLookupSizeMap &LookupMap,
-                                     AbbrevIndexPair &AbbrevPair) {
-  size_t MinIndex, MaxIndex;
-  ComputeAbbrevRange(AbbrevPair.second, MinIndex, MaxIndex);
-  for (size_t Index = MinIndex; Index <= MaxIndex; ++Index) {
-    AbbrevTrieNode *Node = LookupMap[Index];
-    assert(Node);
-    Node->Insert(AbbrevPair);
-  }
-}
-
-// Adds the given abbreviation to the corresponding lookup map, constructing
-// the map of usable lookup tries.
-static bool AddAbbrevToLookupMap(AbbrevLookupSizeMap &LookupMap,
-                                 NaClBitCodeAbbrev *Abbrev) {
-  bool Added = false;
-  size_t MinIndex, MaxIndex;
-  ComputeAbbrevRange(Abbrev, MinIndex, MaxIndex);
-  for (size_t Index = MinIndex; Index <= MaxIndex; ++Index) {
-    AbbrevTrieNode *Node = LookupMap[Index];
-    if (Node == 0) {
-      Node = new AbbrevTrieNode();
-      LookupMap[Index] = Node;
-      Added = true;
-    }
-    if (Node->Add(Abbrev)) Added = true;
-  }
-  return Added;
-}
-
-void llvm::NaClBuildAbbrevLookupMap(
-    AbbrevLookupSizeMap &LookupMap,
-    const SmallVectorImpl<NaClBitCodeAbbrev*> &Abbrevs,
-    size_t InitialIndex) {
-  // First build nodes of trie.
-  bool FixpointFound = false;
-  while (!FixpointFound) {
-    FixpointFound = true;
-    for (size_t i = InitialIndex; i < Abbrevs.size(); ++i) {
-      if (AddAbbrevToLookupMap(LookupMap, Abbrevs[i]))
-        FixpointFound = false;
-    }
-  }
-
-  // Now populate with abbreviations that apply.
-  for (size_t i = InitialIndex; i < Abbrevs.size(); ++i) {
-    AbbrevIndexPair Pair(i, Abbrevs[i]);
-    AddAbbrevPairToLookupMap(LookupMap, Pair);
-  }
-}
diff --git a/lib/Bitcode/NaCl/Analysis/CMakeLists.txt b/lib/Bitcode/NaCl/Analysis/CMakeLists.txt
deleted file mode 100644
index 45383b58365..00000000000
--- a/lib/Bitcode/NaCl/Analysis/CMakeLists.txt
+++ /dev/null
@@ -1,21 +0,0 @@
-add_llvm_library(LLVMNaClBitAnalysis
-  AbbrevTrieNode.cpp
-  NaClBitcodeAnalyzer.cpp
-  NaClBitcodeDist.cpp
-  NaClBitcodeAbbrevDist.cpp
-  NaClBitcodeBitsDist.cpp
-  NaClBitcodeBitsAndAbbrevsDist.cpp
-  NaClBitcodeCodeDist.cpp
-  NaClBitcodeBlockDist.cpp  
-  NaClBitcodeSizeDist.cpp  
-  NaClBitcodeSubblockDist.cpp
-  NaClBitcodeValueDist.cpp
-  NaClAnalyzerBlockDist.cpp
-  NaClCompress.cpp
-  NaClCompressBlockDist.cpp
-  NaClCompressCodeDist.cpp
-  NaClObjDumpStream.cpp
-  NaClObjDump.cpp
-  )
-
-add_dependencies(LLVMNaClBitAnalysis intrinsics_gen)
diff --git a/lib/Bitcode/NaCl/Analysis/LLVMBuild.txt b/lib/Bitcode/NaCl/Analysis/LLVMBuild.txt
deleted file mode 100644
index 0a86b916bd0..00000000000
--- a/lib/Bitcode/NaCl/Analysis/LLVMBuild.txt
+++ /dev/null
@@ -1,22 +0,0 @@
-;===- ./lib/Bitcode/NaCl/Analysis/LLVMBuild.txt ----------------*- Conf -*--===;
-;
-;                     The LLVM Compiler Infrastructure
-;
-; This file is distributed under the University of Illinois Open Source
-; License. See LICENSE.TXT for details.
-;
-;===------------------------------------------------------------------------===;
-;
-; This is an LLVMBuild description file for the components in this subdirectory.
-;
-; For more information on the LLVMBuild system, please see:
-;
-;   http://llvm.org/docs/LLVMBuild.html
-;
-;===------------------------------------------------------------------------===;
-
-[component_0]
-type = Library
-name = NaClBitAnalysis
-parent = NaClBitcode
-required_libraries = Core Support NaClAnalysis NaClBitReader NaClBitWriter
diff --git a/lib/Bitcode/NaCl/Analysis/Makefile b/lib/Bitcode/NaCl/Analysis/Makefile
deleted file mode 100644
index f06aa78731e..00000000000
--- a/lib/Bitcode/NaCl/Analysis/Makefile
+++ /dev/null
@@ -1,14 +0,0 @@
-##===- lib/Bitcode/NaCl/Analysis/Makefile ------------------*- Makefile -*-===##
-#
-#                     The LLVM Compiler Infrastructure
-#
-# This file is distributed under the University of Illinois Open Source
-# License. See LICENSE.TXT for details.
-#
-##===----------------------------------------------------------------------===##
-
-LEVEL = ../../../..
-LIBRARYNAME = LLVMNaClBitAnalysis
-BUILD_ARCHIVE = 1
-
-include $(LEVEL)/Makefile.common
diff --git a/lib/Bitcode/NaCl/Analysis/NaClAnalyzerBlockDist.cpp b/lib/Bitcode/NaCl/Analysis/NaClAnalyzerBlockDist.cpp
deleted file mode 100644
index d3b3be9aa60..00000000000
--- a/lib/Bitcode/NaCl/Analysis/NaClAnalyzerBlockDist.cpp
+++ /dev/null
@@ -1,48 +0,0 @@
-//===-- NaClAnalyzerBlockDist.cpp -----------------------------------------===//
-//      implements distribution maps used to collect block and record
-//      distributions for tools pnacl-bcanalyzer.
-//
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
-//
-//===----------------------------------------------------------------------===//
-
-#include "llvm/Bitcode/NaCl/NaClAnalyzerBlockDist.h"
-
-using namespace llvm;
-
-NaClAnalyzerBlockDistElement::~NaClAnalyzerBlockDistElement() {}
-
-NaClBitcodeDistElement* NaClAnalyzerBlockDistElement::
-CreateElement(NaClBitcodeDistValue Value) const {
-  return new NaClAnalyzerBlockDistElement(Value, OrderBlocksByID);
-}
-
-double NaClAnalyzerBlockDistElement::
-GetImportance(NaClBitcodeDistValue Value) const {
-  if (OrderBlocksByID)
-    // Negate importance to "undo" reverse ordering of sort.
-    return -static_cast<double>(BlockID);
-  else
-    return NaClBitcodeBlockDistElement::GetImportance(Value);
-}
-
-const SmallVectorImpl<NaClBitcodeDist*> *NaClAnalyzerBlockDistElement::
-GetNestedDistributions() const {
-  return &NestedDists;
-}
-
-void NaClAnalyzerBlockDist::AddRecord(const NaClBitcodeRecord &Record) {
-  cast<NaClAnalyzerBlockDistElement>(GetElement(Record.GetBlockID()))
-      ->GetRecordDist().AddRecord(Record);
-}
-
-void NaClAnalyzerBlockDist::AddBlock(const NaClBitcodeBlock &Block) {
-  NaClBitcodeBlockDist::AddBlock(Block);
-  if (const NaClBitcodeBlock *EncBlock = Block.GetEnclosingBlock()) {
-    cast<NaClAnalyzerBlockDistElement>(GetElement(EncBlock->GetBlockID()))
-        ->GetSubblockDist().AddBlock(Block);
-  }
-}
diff --git a/lib/Bitcode/NaCl/Analysis/NaClBitcodeAbbrevDist.cpp b/lib/Bitcode/NaCl/Analysis/NaClBitcodeAbbrevDist.cpp
deleted file mode 100644
index 6be56951438..00000000000
--- a/lib/Bitcode/NaCl/Analysis/NaClBitcodeAbbrevDist.cpp
+++ /dev/null
@@ -1,61 +0,0 @@
-//===-- NaClBitcodeAbbrevDist.cpp -------------------------------------------===//
-//      Implements distribution maps for abbreviations associated with
-//      bitcode records.
-//
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
-//
-//===----------------------------------------------------------------------===//
-
-#include "llvm/Bitcode/NaCl/NaClBitcodeAbbrevDist.h"
-
-using namespace llvm;
-
-NaClBitcodeAbbrevDistElement::~NaClBitcodeAbbrevDistElement() {}
-
-NaClBitcodeDistElement *NaClBitcodeAbbrevDistElement::CreateElement(
-    NaClBitcodeDistValue Value) const {
-  return new NaClBitcodeAbbrevDistElement();
-}
-
-void NaClBitcodeAbbrevDistElement::
-GetValueList(const NaClBitcodeRecord &Record,
-             ValueListType &ValueList) const {
-  ValueList.push_back(Record.GetAbbreviationIndex());
-}
-
-void NaClBitcodeAbbrevDistElement::AddRecord(const NaClBitcodeRecord &Record) {
-  NaClBitcodeDistElement::AddRecord(Record);
-  CodeDist.AddRecord(Record);
-}
-
-const char *NaClBitcodeAbbrevDistElement::GetTitle() const {
-  return "Abbreviation Indices";
-}
-
-const char *NaClBitcodeAbbrevDistElement::GetValueHeader() const {
-  return "  Index";
-}
-
-void NaClBitcodeAbbrevDistElement::
-PrintRowValue(raw_ostream &Stream,
-              NaClBitcodeDistValue Value,
-              const NaClBitcodeDist *Distribution) const {
-  Stream << format("%7u", Value);
-}
-
-const SmallVectorImpl<NaClBitcodeDist*> *NaClBitcodeAbbrevDistElement::
-GetNestedDistributions() const {
-  return &NestedDists;
-}
-
-NaClBitcodeAbbrevDistElement NaClBitcodeAbbrevDistElement::Sentinel;
-
-NaClBitcodeAbbrevDist::~NaClBitcodeAbbrevDist() {}
-
-NaClBitcodeDistElement* NaClBitcodeAbbrevDist::
-CreateElement(NaClBitcodeDistValue Value) const {
-  return new NaClBitcodeAbbrevDistElement(BlockID);
-}
diff --git a/lib/Bitcode/NaCl/Analysis/NaClBitcodeAnalyzer.cpp b/lib/Bitcode/NaCl/Analysis/NaClBitcodeAnalyzer.cpp
deleted file mode 100644
index f145a363f07..00000000000
--- a/lib/Bitcode/NaCl/Analysis/NaClBitcodeAnalyzer.cpp
+++ /dev/null
@@ -1,469 +0,0 @@
-//===-- NaClBitcodeAnalyzer.cpp - Bitcode Analyzer ------------------------===//
-//
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
-//
-//===----------------------------------------------------------------------===//
-
-#define DEBUG_TYPE "nacl-bitcode-analyzer"
-
-#include "llvm/ADT/STLExtras.h"
-#include "llvm/ADT/Twine.h"
-#include "llvm/Bitcode/NaCl/NaClAnalyzerBlockDist.h"
-#include "llvm/Bitcode/NaCl/NaClBitcodeAnalyzer.h"
-#include "llvm/Bitcode/NaCl/NaClBitcodeHeader.h"
-#include "llvm/Bitcode/NaCl/NaClBitcodeParser.h"
-#include "llvm/Bitcode/NaCl/NaClBitstreamReader.h"
-#include "llvm/Bitcode/NaCl/NaClLLVMBitCodes.h"
-#include "llvm/Bitcode/NaCl/NaClReaderWriter.h"
-#include "llvm/Support/Debug.h"
-#include "llvm/Support/Format.h"
-#include "llvm/Support/MemoryBuffer.h"
-#include "llvm/Support/raw_ostream.h"
-#include <algorithm>
-#include <map>
-#include <system_error>
-
-// TODO(kschimpf): Separate out into two bitcode parsers, one for
-// dumping records, and one for collecting distribution stats for
-// printing. This should simplify the code.
-
-namespace {
-
-// Generates an error message when outside parsing, and no
-// corresponding bit position is known.
-bool Error(const llvm::Twine &Err) {
-  llvm::errs() << Err << "\n";
-  return true;
-}
-
-} // End of anonymous namespace.
-
-namespace llvm {
-
-// Parses all bitcode blocks, and collects distribution of records in
-// each block.  Also dumps bitcode structure if specified (via global
-// variables).
-class PNaClBitcodeAnalyzerParser : public NaClBitcodeParser {
-public:
-  PNaClBitcodeAnalyzerParser(NaClBitstreamCursor &Cursor,
-                             raw_ostream &OS,
-                             const AnalysisDumpOptions &DumpOptions,
-                             NaClBitcodeDist *Dist)
-    : NaClBitcodeParser(Cursor),
-      IndentLevel(0),
-      OS(OS),
-      DumpOptions(DumpOptions),
-      Dist(Dist),
-      AbbrevListener(this)
-  {
-    SetListener(&AbbrevListener);
-  }
-
-  virtual ~PNaClBitcodeAnalyzerParser() {}
-
-  virtual bool ParseBlock(unsigned BlockID);
-
-  // Returns the string defining the indentation to use with respect
-  // to the current indent level.
-  const std::string &GetIndentation() {
-    size_t Size = IndentationCache.size();
-    if (IndentLevel >= Size) {
-      IndentationCache.resize(IndentLevel+1);
-      for (size_t i = Size; i <= IndentLevel; ++i) {
-        IndentationCache[i] = std::string(i*2, ' ');
-      }
-    }
-    return IndentationCache[IndentLevel];
-  }
-
-  // Keeps track of current indentation level based on block nesting.
-  unsigned IndentLevel;
-  // The output stream to print to.
-  raw_ostream &OS;
-  // The dump options to use.
-  const AnalysisDumpOptions &DumpOptions;
-  // The bitcode distribution map (if defined) to update.
-  NaClBitcodeDist *Dist;
-
-private:
-  // The set of cached, indentation strings. Used for indenting
-  // records when dumping.
-  std::vector<std::string> IndentationCache;
-  // Listener used to get abbreviations as they are read.
-  NaClBitcodeParserListener AbbrevListener;
-};
-
-// Parses a bitcode block, and collects distribution of records in that block.
-// Also dumps bitcode structure if specified (via global variables).
-class PNaClBitcodeAnalyzerBlockParser : public NaClBitcodeParser {
-public:
-  // Parses top-level block.
-  PNaClBitcodeAnalyzerBlockParser(
-      unsigned BlockID,
-      PNaClBitcodeAnalyzerParser *Parser)
-      : NaClBitcodeParser(BlockID, Parser) {
-    Initialize(BlockID, Parser);
-  }
-
-  virtual ~PNaClBitcodeAnalyzerBlockParser() {
-    if (Context->Dist) Context->Dist->AddBlock(GetBlock());
-  }
-
-  // *****************************************************
-  // This subsection Defines an XML generator for the dump.
-  // Tag. TagName, Attribute
-  // *****************************************************
-
-private:
-  // The tag name for an element.
-  std::string TagName;
-  // The number of attributes associated with a tag.
-  unsigned NumTagAttributes;
-  // The number of (indexed attribute) operands associated with a tag.
-  unsigned NumTagOperands;
-
-protected:
-
-  /// Initializes internal data used to emit an XML tag.
-  void InitializeEmitTag() {
-    TagName.clear();
-    NumTagAttributes = 0;
-    NumTagOperands = 0;
-  }
-
-  /// Emits the start of an XML start tag.
-  void EmitBeginStartTag() {
-    InitializeEmitTag();
-    Context->OS << Indent << "<";
-  }
-
-  /// Emit the start of an XML end tag.
-  void EmitBeginEndTag() {
-    InitializeEmitTag();
-    Context->OS << Indent << "</";
-  }
-
-  /// Emits the end of an empty-element XML tag.
-  void EmitEndTag() {
-    Context->OS << "/>\n";
-  }
-
-  /// Emits the End of a start/end tag for an XML element.
-  void EmitEndElementTag() {
-    Context->OS << ">\n";
-  }
-
-  /// Emits the tag name for an XML tag.
-  void EmitTagName(const std::string &ElmtName) {
-    TagName = ElmtName;
-    Context->OS << ElmtName;
-  }
-
-  /// Emits the "name=" portion of an XML tag attribute.
-  raw_ostream &EmitAttributePrefix(const std::string &AttributeName) {
-    WrapOperandsLine();
-    Context->OS << " " << AttributeName << "=";
-    ++NumTagAttributes;
-    return Context->OS;
-  }
-
-  /// Emits a string-valued XML attribute of an XML tag.
-  void EmitStringAttribute(const char *AttributeName, const std::string &Str) {
-    EmitAttributePrefix(AttributeName) << "'" << Str << "'";
-  }
-
-  /// Emits a string-valued XML attribute of an XML tag.
-  void EmitStringAttribute(const char *AttributeName, const char*Str) {
-    std::string StrStr(Str);
-    EmitStringAttribute(AttributeName, StrStr);
-  }
-
-  /// Emits an unsigned integer-valued XML attribute of an XML tag.
-  void EmitAttribute(const char *AttributeName, uint64_t Value) {
-    EmitAttributePrefix(AttributeName) << Value;
-  }
-
-  /// Emits the "opN=" portion of an XML tag (indexable) operand attribute.
-  raw_ostream &EmitOperandPrefix() {
-    std::string OpName;
-    raw_string_ostream OpNameStrm(OpName);
-    OpNameStrm << "op" << NumTagOperands;
-    ++NumTagOperands;
-    return EmitAttributePrefix(OpNameStrm.str());
-  }
-
-  /// Adds line wrap if more than "OpsPerLine" XML tag attributes are
-  /// emitted on the current line.
-  void WrapOperandsLine() {
-    if (Context->DumpOptions.OpsPerLine) {
-      if (NumTagAttributes &&
-          (NumTagAttributes % Context->DumpOptions.OpsPerLine) == 0) {
-        raw_ostream &OS = Context->OS;
-        // Last operand crossed width boundary, add newline and indent.
-        OS << "\n" << Indent << " ";
-        for (unsigned J = 0, SZ = TagName.size(); J < SZ; ++J)
-          OS << " ";
-      }
-    }
-  }
-
-  // ********************************************************
-  // This section defines how to parse the block and generate
-  // the corresponding XML.
-  // ********************************************************
-
-  // Parses nested blocks.
-  PNaClBitcodeAnalyzerBlockParser(
-      unsigned BlockID,
-      PNaClBitcodeAnalyzerBlockParser *EnclosingBlock)
-      : NaClBitcodeParser(BlockID, EnclosingBlock),
-        Context(EnclosingBlock->Context) {
-    Initialize(BlockID, EnclosingBlock->Context);
-  }
-
-  // Initialize data associated with a block.
-  void Initialize(unsigned BlockID, PNaClBitcodeAnalyzerParser *Parser) {
-    InitializeEmitTag();
-    Context = Parser;
-    if (Context->DumpOptions.DumpRecords) {
-      Indent = Parser->GetIndentation();
-    }
-    NumWords = 0;
-  }
-
-  // Increment the indentation level for dumping.
-  void IncrementIndent() {
-    Context->IndentLevel++;
-    Indent = Context->GetIndentation();
-  }
-
-  // Increment the indentation level for dumping.
-  void DecrementIndent() {
-    Context->IndentLevel--;
-    Indent = Context->GetIndentation();
-  }
-
-  // Called once the block has been entered by the bitstream reader.
-  // Argument NumWords is set to the number of words in the
-  // corresponding block.
-  virtual void EnterBlock(unsigned NumberWords) {
-    NumWords = NumberWords;
-    if (Context->DumpOptions.DumpRecords) {
-      unsigned BlockID = GetBlockID();
-      EmitBeginStartTag();
-      EmitEnterBlockTagName(BlockID);
-      if (Context->DumpOptions.DumpDetails) {
-        EmitAttribute("NumWords", NumWords);
-        EmitAttribute("BlockCodeSize", Record.GetCursor().getAbbrevIDWidth());
-      }
-      if (!Context->DumpOptions.DumpDetails &&
-          naclbitc::BLOCKINFO_BLOCK_ID == GetBlockID()) {
-        EmitEndTag();
-      } else {
-        EmitEndElementTag();
-        IncrementIndent();
-      }
-    }
-  }
-
-  // Called when the corresponding EndBlock of the block being parsed
-  // is found.
-  virtual void ExitBlock() {
-    if (Context->DumpOptions.DumpRecords) {
-      if (!Context->DumpOptions.DumpDetails &&
-          naclbitc::BLOCKINFO_BLOCK_ID == GetBlockID())
-        return;
-      DecrementIndent();
-      EmitBeginEndTag();
-      EmitExitBlockTagName(Record.GetBlockID());
-      EmitEndElementTag();
-    }
-  }
-
-  virtual void SetBID() {
-    if (!(Context->DumpOptions.DumpRecords &&
-          Context->DumpOptions.DumpDetails)) {
-      return;
-    }
-    EmitBeginStartTag();
-    EmitCodeTagName(naclbitc::BLOCKINFO_CODE_SETBID,
-                    naclbitc::BLOCKINFO_BLOCK_ID);
-    EmitStringAttribute("block",
-                        NaClBitcodeBlockDist::GetName(Record.GetValues()[0]));
-    EmitEndTag();
-  }
-
-  virtual void ProcessAbbreviation(unsigned BlockID,
-                                   NaClBitCodeAbbrev *Abbrev,
-                                   bool IsLocal) {
-    if (Context->DumpOptions.DumpDetails) {
-      EmitAbbreviation(Abbrev);
-    }
-  }
-
-  // Process the last read record in the block.
-  virtual void ProcessRecord() {
-    // Increment the # occurrences of this code.
-    if (Context->Dist) Context->Dist->AddRecord(Record);
-
-    if (Context->DumpOptions.DumpRecords) {
-      EmitBeginStartTag();
-      EmitCodeTagName(Record.GetCode(), GetBlockID(), Record.GetEntryID());
-      const NaClBitcodeRecord::RecordVector &Values = Record.GetValues();
-      for (unsigned i = 0, e = Values.size(); i != e; ++i) {
-        EmitOperandPrefix() << (int64_t)Values[i];
-      }
-      EmitEndTag();
-    }
-  }
-
-  virtual bool ParseBlock(unsigned BlockID) {
-    PNaClBitcodeAnalyzerBlockParser Parser(BlockID, this);
-    return Parser.ParseThisBlock();
-  }
-
-private:
-  /// Defines the indent level of the block being parsed.
-  std::string Indent;
-  /// Defines the number of (32-bit) words the block occupies in
-  /// the bitstream.
-  unsigned NumWords;
-  /// Refers to global parsing context.
-  PNaClBitcodeAnalyzerParser *Context;
-
-protected:
-
-  /// Emit the given abbreviation as an XML tag.
-  void EmitAbbreviation(const NaClBitCodeAbbrev *Abbrev) {
-    EmitBeginStartTag();
-    EmitTagName("DEFINE_ABBREV");
-    if (Context->DumpOptions.DumpDetails) {
-      EmitStringAttribute("abbrev", "DEFINE_ABBREV");
-    }
-    for (unsigned I = 0, IEnd = Abbrev->getNumOperandInfos(); I != IEnd; ++I) {
-      EmitAbbreviationOp(Abbrev->getOperandInfo(I));
-    }
-    EmitEndTag();
-  }
-
-  /// Emit the given abbreviation operand as an XML tag attribute.
-  void EmitAbbreviationOp(const NaClBitCodeAbbrevOp &Op) {
-    EmitOperandPrefix()
-        << "'" << NaClBitCodeAbbrevOp::getEncodingName(Op.getEncoding());
-    if (Op.hasValue()) {
-      Context->OS << "(" << Op.getValue() << ")";
-    }
-    Context->OS << "'";
-  }
-
-  /// Emits the symbolic name of the record code as the XML tag name.
-  void EmitCodeTagName(
-      unsigned CodeID, unsigned BlockID,
-      unsigned AbbreviationID = naclbitc::UNABBREV_RECORD) {
-    EmitTagName(NaClBitcodeCodeDist::GetCodeName(CodeID, BlockID));
-    if (Context->DumpOptions.DumpDetails) {
-      if (AbbreviationID == naclbitc::UNABBREV_RECORD) {
-        EmitStringAttribute("abbrev", "UNABBREVIATED");
-      } else {
-        EmitAttribute("abbrev", AbbreviationID);
-      }
-    }
-  }
-
-  /// Emits the symbolic name of the block as the XML tag name.
-  void EmitEnterBlockTagName(unsigned BlockID) {
-    EmitTagName(NaClBitcodeBlockDist::GetName(BlockID));
-    if (Context->DumpOptions.DumpDetails)
-      EmitStringAttribute("abbrev", "ENTER_SUBBLOCK");
-  }
-
-  /// Emits the symbolic name of the block as the the XML tag name.
-  void EmitExitBlockTagName(unsigned BlockID) {
-    EmitTagName(NaClBitcodeBlockDist::GetName(BlockID));
-    if (Context->DumpOptions.DumpDetails)
-      EmitStringAttribute("abbrev", "END_BLOCK");
-  }
-};
-
-bool PNaClBitcodeAnalyzerParser::ParseBlock(unsigned BlockID) {
-  PNaClBitcodeAnalyzerBlockParser Parser(BlockID, this);
-  return Parser.ParseThisBlock();
-}
-
-static void PrintSize(uint64_t Bits, raw_ostream &OS) {
-  OS << format("%lub/%.2fB/%luW", (unsigned long)Bits,
-               (double)Bits/8, (unsigned long)(Bits/32));
-}
-
-int AnalyzeBitcodeInBuffer(const std::unique_ptr<MemoryBuffer> &Buf,
-                           raw_ostream &OS,
-                           const AnalysisDumpOptions &DumpOptions) {
-  DEBUG(dbgs() << "-> AnalyzeBitcodeInBuffer\n");
-
-  if (Buf->getBufferSize() & 3)
-    return Error("Bitcode stream should be a multiple of 4 bytes in length");
-
-  const unsigned char *BufPtr = (const unsigned char *)Buf->getBufferStart();
-  const unsigned char *EndBufPtr = BufPtr + Buf->getBufferSize();
-
-  NaClBitcodeHeader Header;
-  if (Header.Read(BufPtr, EndBufPtr))
-    return Error("Invalid PNaCl bitcode header");
-
-  if (!Header.IsSupported())
-    errs() << "Warning: " << Header.Unsupported() << "\n";
-
-  if (!Header.IsReadable())
-    Error("Bitcode file is not readable");
-
-  NaClBitstreamReader StreamFile(BufPtr, EndBufPtr, Header);
-  NaClBitstreamCursor Stream(StreamFile);
-
-  unsigned NumTopBlocks = 0;
-
-  // Print out header information.
-  for (size_t i = 0, limit = Header.NumberFields(); i < limit; ++i) {
-    OS << Header.GetField(i)->Contents() << "\n";
-  }
-  if (Header.NumberFields()) OS << "\n";
-
-  // Parse the top-level structure.  We only allow blocks at the top-level.
-  NaClAnalyzerBlockDistElement DistSentinel(0, DumpOptions.OrderBlocksByID);
-  NaClAnalyzerBlockDist Dist(DistSentinel);
-  PNaClBitcodeAnalyzerParser Parser(Stream, OS, DumpOptions, &Dist);
-  while (!Stream.AtEndOfStream()) {
-    ++NumTopBlocks;
-    if (Parser.Parse()) return 1;
-  }
-
-  if (DumpOptions.DumpRecords) return 0;
-
-  uint64_t BufferSizeBits = (EndBufPtr-BufPtr)*CHAR_BIT;
-  // Print a summary
-  OS << "Total size: ";
-  PrintSize(BufferSizeBits, OS);
-  OS << "\n";
-  OS << "# Toplevel Blocks: " << NumTopBlocks << "\n";
-  OS << "\n";
-
-  if (Parser.Dist) Parser.Dist->Print(OS);
-
-  DEBUG(dbgs() << "<- AnalyzeBitcode\n");
-  return 0;
-}
-
-int AnalyzeBitcodeInFile(const StringRef &InputFilename, raw_ostream &OS,
-                         const AnalysisDumpOptions &DumpOptions) {
-  ErrorOr<std::unique_ptr<MemoryBuffer>> ErrOrFile =
-      MemoryBuffer::getFileOrSTDIN(InputFilename);
-  if (std::error_code EC = ErrOrFile.getError())
-    return Error(Twine("Error reading '") + InputFilename + "': " +
-                 EC.message());
-
-  return AnalyzeBitcodeInBuffer(ErrOrFile.get(), OS, DumpOptions);
-}
-
-} // namespace llvm
diff --git a/lib/Bitcode/NaCl/Analysis/NaClBitcodeBitsAndAbbrevsDist.cpp b/lib/Bitcode/NaCl/Analysis/NaClBitcodeBitsAndAbbrevsDist.cpp
deleted file mode 100644
index ae1d3cbcffa..00000000000
--- a/lib/Bitcode/NaCl/Analysis/NaClBitcodeBitsAndAbbrevsDist.cpp
+++ /dev/null
@@ -1,42 +0,0 @@
-//===- NaClBitcodeBitsAndAbbrevsDist.cpp ------------------*- C++ -*-===//
-//     Implements distributions of values with the corresponding
-//     number of bits and percentage of abbreviations used in PNaCl
-//     bitcode records.
-//
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
-//
-//===----------------------------------------------------------------------===//
-
-#include "llvm/Bitcode/NaCl/NaClBitcodeBitsAndAbbrevsDist.h"
-
-using namespace llvm;
-
-NaClBitcodeBitsAndAbbrevsDistElement::~NaClBitcodeBitsAndAbbrevsDistElement() {}
-
-void NaClBitcodeBitsAndAbbrevsDistElement::
-AddRecord(const NaClBitcodeRecord &Record) {
-  NaClBitcodeBitsDistElement::AddRecord(Record);
-  if (Record.UsedAnAbbreviation()) {
-    ++NumAbbrevs;
-  }
-}
-
-void NaClBitcodeBitsAndAbbrevsDistElement::
-PrintStatsHeader(raw_ostream &Stream) const {
-  NaClBitcodeBitsDistElement::PrintStatsHeader(Stream);
-  Stream << "   % Abv";
-}
-
-void NaClBitcodeBitsAndAbbrevsDistElement::
-PrintRowStats(raw_ostream &Stream,
-              const NaClBitcodeDist *Distribution) const {
-  NaClBitcodeBitsDistElement::PrintRowStats(Stream, Distribution);
-  if (GetNumAbbrevs())
-    Stream << format(" %7.2f",
-                     (double) GetNumAbbrevs()/GetNumInstances()*100.0);
-  else
-    Stream << "        ";
-}
diff --git a/lib/Bitcode/NaCl/Analysis/NaClBitcodeBitsDist.cpp b/lib/Bitcode/NaCl/Analysis/NaClBitcodeBitsDist.cpp
deleted file mode 100644
index a8bb6db4830..00000000000
--- a/lib/Bitcode/NaCl/Analysis/NaClBitcodeBitsDist.cpp
+++ /dev/null
@@ -1,40 +0,0 @@
-//===- NaClBitcodeBitsDist.cpp ----------------------------------*- C++ -*-===//
-//     Implements distributions of values with the corresponding number
-//     of bits in PNaCl bitcode.
-//
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
-//
-//===----------------------------------------------------------------------===//
-
-#include "llvm/Bitcode/NaCl/NaClBitcodeBitsDist.h"
-
-using namespace llvm;
-
-NaClBitcodeBitsDistElement::~NaClBitcodeBitsDistElement() {}
-
-void NaClBitcodeBitsDistElement::AddRecord(const NaClBitcodeRecord &Record) {
-  NaClBitcodeDistElement::AddRecord(Record);
-  TotalBits += Record.GetNumBits();
-}
-
-void NaClBitcodeBitsDistElement::AddBlock(const NaClBitcodeBlock &Block) {
-  NaClBitcodeDistElement::AddBlock(Block);
-  TotalBits += Block.GetLocalNumBits();
-}
-
-void NaClBitcodeBitsDistElement::PrintStatsHeader(raw_ostream &Stream) const {
-  NaClBitcodeDistElement::PrintStatsHeader(Stream);
-  Stream << "    # Bits    Bits/Elmt";
-}
-
-void NaClBitcodeBitsDistElement::
-PrintRowStats(raw_ostream &Stream,
-              const NaClBitcodeDist *Distribution) const {
-  NaClBitcodeDistElement::PrintRowStats(Stream, Distribution);
-  Stream << format(" %9lu %12.2f",
-                   (unsigned long) GetTotalBits(),
-                   (double) GetTotalBits()/GetNumInstances());
-}
diff --git a/lib/Bitcode/NaCl/Analysis/NaClBitcodeBlockDist.cpp b/lib/Bitcode/NaCl/Analysis/NaClBitcodeBlockDist.cpp
deleted file mode 100644
index 72198079bfb..00000000000
--- a/lib/Bitcode/NaCl/Analysis/NaClBitcodeBlockDist.cpp
+++ /dev/null
@@ -1,106 +0,0 @@
-//===-- NaClBitcodeBlockDist.cpp ---------------------------------------===//
-//      implements distribution maps for blocks within PNaCl bitcode.
-//
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
-//
-//===----------------------------------------------------------------------===//
-
-#include "llvm/Bitcode/NaCl/NaClBitcodeBlockDist.h"
-#include "llvm/Bitcode/NaCl/NaClLLVMBitCodes.h"
-
-using namespace llvm;
-
-/// GetBlockName - Return a symbolic block name if known, otherwise return
-/// null.
-static const char *GetBlockName(unsigned BlockID) {
-  // Standard blocks for all bitcode files.
-  if (BlockID < naclbitc::FIRST_APPLICATION_BLOCKID) {
-    if (BlockID == naclbitc::BLOCKINFO_BLOCK_ID)
-      return "BLOCKINFO_BLOCK";
-    return 0;
-  }
-
-  switch (BlockID) {
-  default: return 0;
-  case naclbitc::MODULE_BLOCK_ID:          return "MODULE_BLOCK";
-  case naclbitc::PARAMATTR_BLOCK_ID:       return "PARAMATTR_BLOCK";
-  case naclbitc::PARAMATTR_GROUP_BLOCK_ID: return "PARAMATTR_GROUP_BLOCK_ID";
-  case naclbitc::TYPE_BLOCK_ID_NEW:        return "TYPE_BLOCK_ID";
-  case naclbitc::CONSTANTS_BLOCK_ID:       return "CONSTANTS_BLOCK";
-  case naclbitc::FUNCTION_BLOCK_ID:        return "FUNCTION_BLOCK";
-  case naclbitc::VALUE_SYMTAB_BLOCK_ID:    return "VALUE_SYMTAB";
-  case naclbitc::METADATA_BLOCK_ID:        return "METADATA_BLOCK";
-  case naclbitc::METADATA_ATTACHMENT_ID:   return "METADATA_ATTACHMENT_BLOCK";
-  case naclbitc::USELIST_BLOCK_ID:         return "USELIST_BLOCK_ID";
-  case naclbitc::GLOBALVAR_BLOCK_ID:       return "GLOBALVAR_BLOCK";
-  }
-}
-
-NaClBitcodeBlockDistElement NaClBitcodeBlockDist::DefaultSentinal;
-
-NaClBitcodeBlockDistElement::~NaClBitcodeBlockDistElement() {}
-
-NaClBitcodeDistElement *NaClBitcodeBlockDistElement::
-CreateElement(NaClBitcodeDistValue Value) const {
-  return new NaClBitcodeBlockDistElement();
-}
-
-double NaClBitcodeBlockDistElement::
-GetImportance(NaClBitcodeDistValue Value) const {
-  return static_cast<double>(GetTotalBits());
-}
-
-const char *NaClBitcodeBlockDistElement::GetTitle() const {
-  return "Block Histogram";
-}
-
-const char *NaClBitcodeBlockDistElement::GetValueHeader() const {
-  return "Block";
-}
-
-void NaClBitcodeBlockDistElement::PrintStatsHeader(raw_ostream &Stream) const {
-  Stream << "  %File";
-  NaClBitcodeBitsDistElement::PrintStatsHeader(Stream);
-}
-
-void NaClBitcodeBlockDistElement::
-PrintRowStats(raw_ostream &Stream, const NaClBitcodeDist *Distribution) const {
-  const NaClBitcodeBlockDist *BlockDist =
-      cast<NaClBitcodeBlockDist>(Distribution);
-  Stream << format(" %6.2f",
-                   (double) GetTotalBits()/BlockDist->GetTotalBits()*100.00);
-  NaClBitcodeBitsDistElement::PrintRowStats(Stream, Distribution);
-}
-
-void NaClBitcodeBlockDistElement::
-PrintRowValue(raw_ostream &Stream, NaClBitcodeDistValue Value,
-              const NaClBitcodeDist *Distribution) const {
-  Stream << NaClBitcodeBlockDist::GetName(Value);
-}
-
-NaClBitcodeBlockDist::~NaClBitcodeBlockDist() {}
-
-uint64_t NaClBitcodeBlockDist::GetTotalBits() const {
-  uint64_t Total = 0;
-  for (NaClBitcodeDist::const_iterator Iter = begin(), IterEnd = end();
-       Iter != IterEnd; ++Iter) {
-    const NaClBitcodeBlockDistElement *Element =
-        cast<NaClBitcodeBlockDistElement>(Iter->second);
-    Total += Element->GetTotalBits();
-  }
-  return Total;
-}
-
-std::string NaClBitcodeBlockDist::GetName(unsigned BlockID) {
-  if (const char *BlockName = GetBlockName(BlockID)) {
-    return BlockName;
-  }
-
-  std::string Str;
-  raw_string_ostream StrStrm(Str);
-  StrStrm << "UnknownBlock" << BlockID;
-  return StrStrm.str();
-}
diff --git a/lib/Bitcode/NaCl/Analysis/NaClBitcodeCodeDist.cpp b/lib/Bitcode/NaCl/Analysis/NaClBitcodeCodeDist.cpp
deleted file mode 100644
index aadea756b6e..00000000000
--- a/lib/Bitcode/NaCl/Analysis/NaClBitcodeCodeDist.cpp
+++ /dev/null
@@ -1,221 +0,0 @@
-//===-- NaClBitcodeCodeDist.cpp -------------------------------------------===//
-//      Implements distribution maps for record codes within a PNaCl bitcode
-//      file.
-//
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
-//
-//===----------------------------------------------------------------------===//
-
-#include "llvm/Bitcode/NaCl/NaClBitcodeCodeDist.h"
-#include "llvm/Bitcode/NaCl/NaClLLVMBitCodes.h"
-#include "llvm/Support/raw_ostream.h"
-
-using namespace llvm;
-
-/// GetCodeName - Return a symbolic code name if known, otherwise return
-/// null.
-static const char *GetCodeName(unsigned CodeID, unsigned BlockID) {
-  // Standard blocks for all bitcode files.
-  if (BlockID < naclbitc::FIRST_APPLICATION_BLOCKID) {
-    if (BlockID == naclbitc::BLOCKINFO_BLOCK_ID) {
-      switch (CodeID) {
-      default: return 0;
-      case naclbitc::BLOCKINFO_CODE_SETBID:        return "SETBID";
-      }
-    }
-    return 0;
-  }
-
-  switch (BlockID) {
-  default: return 0;
-  case naclbitc::MODULE_BLOCK_ID:
-    switch (CodeID) {
-    default: return 0;
-    case naclbitc::MODULE_CODE_VERSION:     return "VERSION";
-    case naclbitc::MODULE_CODE_TRIPLE:      return "TRIPLE";
-    case naclbitc::MODULE_CODE_DATALAYOUT:  return "DATALAYOUT";
-    case naclbitc::MODULE_CODE_ASM:         return "ASM";
-    case naclbitc::MODULE_CODE_SECTIONNAME: return "SECTIONNAME";
-    case naclbitc::MODULE_CODE_DEPLIB:      return "DEPLIB"; // FIXME: Remove in 4.0
-    case naclbitc::MODULE_CODE_GLOBALVAR:   return "GLOBALVAR";
-    case naclbitc::MODULE_CODE_FUNCTION:    return "FUNCTION";
-    case naclbitc::MODULE_CODE_ALIAS:       return "ALIAS";
-    case naclbitc::MODULE_CODE_PURGEVALS:   return "PURGEVALS";
-    case naclbitc::MODULE_CODE_GCNAME:      return "GCNAME";
-    }
-  case naclbitc::PARAMATTR_BLOCK_ID:
-    switch (CodeID) {
-    default: return 0;
-    case naclbitc::PARAMATTR_CODE_ENTRY_OLD: return "ENTRY";
-    case naclbitc::PARAMATTR_CODE_ENTRY:     return "ENTRY";
-    case naclbitc::PARAMATTR_GRP_CODE_ENTRY: return "ENTRY";
-    }
-  case naclbitc::TYPE_BLOCK_ID_NEW:
-    switch (CodeID) {
-    default: return 0;
-    case naclbitc::TYPE_CODE_NUMENTRY:     return "NUMENTRY";
-    case naclbitc::TYPE_CODE_VOID:         return "VOID";
-    case naclbitc::TYPE_CODE_FLOAT:        return "FLOAT";
-    case naclbitc::TYPE_CODE_DOUBLE:       return "DOUBLE";
-    case naclbitc::TYPE_CODE_LABEL:        return "LABEL";
-    case naclbitc::TYPE_CODE_OPAQUE:       return "OPAQUE";
-    case naclbitc::TYPE_CODE_INTEGER:      return "INTEGER";
-    case naclbitc::TYPE_CODE_POINTER:      return "POINTER";
-    case naclbitc::TYPE_CODE_ARRAY:        return "ARRAY";
-    case naclbitc::TYPE_CODE_VECTOR:       return "VECTOR";
-    case naclbitc::TYPE_CODE_X86_FP80:     return "X86_FP80";
-    case naclbitc::TYPE_CODE_FP128:        return "FP128";
-    case naclbitc::TYPE_CODE_PPC_FP128:    return "PPC_FP128";
-    case naclbitc::TYPE_CODE_METADATA:     return "METADATA";
-    case naclbitc::TYPE_CODE_STRUCT_ANON:  return "STRUCT_ANON";
-    case naclbitc::TYPE_CODE_STRUCT_NAME:  return "STRUCT_NAME";
-    case naclbitc::TYPE_CODE_STRUCT_NAMED: return "STRUCT_NAMED";
-    case naclbitc::TYPE_CODE_FUNCTION:     return "FUNCTION";
-    }
-
-  case naclbitc::CONSTANTS_BLOCK_ID:
-    switch (CodeID) {
-    default: return 0;
-    case naclbitc::CST_CODE_SETTYPE:         return "SETTYPE";
-    case naclbitc::CST_CODE_NULL:            return "NULL";
-    case naclbitc::CST_CODE_UNDEF:           return "UNDEF";
-    case naclbitc::CST_CODE_INTEGER:         return "INTEGER";
-    case naclbitc::CST_CODE_WIDE_INTEGER:    return "WIDE_INTEGER";
-    case naclbitc::CST_CODE_FLOAT:           return "FLOAT";
-    case naclbitc::CST_CODE_AGGREGATE:       return "AGGREGATE";
-    case naclbitc::CST_CODE_STRING:          return "STRING";
-    case naclbitc::CST_CODE_CSTRING:         return "CSTRING";
-    case naclbitc::CST_CODE_CE_BINOP:        return "CE_BINOP";
-    case naclbitc::CST_CODE_CE_CAST:         return "CE_CAST";
-    case naclbitc::CST_CODE_CE_GEP:          return "CE_GEP";
-    case naclbitc::CST_CODE_CE_INBOUNDS_GEP: return "CE_INBOUNDS_GEP";
-    case naclbitc::CST_CODE_CE_SELECT:       return "CE_SELECT";
-    case naclbitc::CST_CODE_CE_EXTRACTELT:   return "CE_EXTRACTELT";
-    case naclbitc::CST_CODE_CE_INSERTELT:    return "CE_INSERTELT";
-    case naclbitc::CST_CODE_CE_SHUFFLEVEC:   return "CE_SHUFFLEVEC";
-    case naclbitc::CST_CODE_CE_CMP:          return "CE_CMP";
-    case naclbitc::CST_CODE_INLINEASM:       return "INLINEASM";
-    case naclbitc::CST_CODE_CE_SHUFVEC_EX:   return "CE_SHUFVEC_EX";
-    case naclbitc::CST_CODE_BLOCKADDRESS:    return "CST_CODE_BLOCKADDRESS";
-    case naclbitc::CST_CODE_DATA:            return "DATA";
-    }
-  case naclbitc::FUNCTION_BLOCK_ID:
-    switch (CodeID) {
-    default: return 0;
-    case naclbitc::FUNC_CODE_DECLAREBLOCKS: return "DECLAREBLOCKS";
-
-    case naclbitc::FUNC_CODE_INST_BINOP:        return "INST_BINOP";
-    case naclbitc::FUNC_CODE_INST_CAST:         return "INST_CAST";
-    case naclbitc::FUNC_CODE_INST_GEP:          return "INST_GEP";
-    case naclbitc::FUNC_CODE_INST_INBOUNDS_GEP: return "INST_INBOUNDS_GEP";
-    case naclbitc::FUNC_CODE_INST_SELECT:       return "INST_SELECT";
-    case naclbitc::FUNC_CODE_INST_EXTRACTELT:   return "INST_EXTRACTELT";
-    case naclbitc::FUNC_CODE_INST_INSERTELT:    return "INST_INSERTELT";
-    case naclbitc::FUNC_CODE_INST_SHUFFLEVEC:   return "INST_SHUFFLEVEC";
-    case naclbitc::FUNC_CODE_INST_CMP:          return "INST_CMP";
-
-    case naclbitc::FUNC_CODE_INST_RET:          return "INST_RET";
-    case naclbitc::FUNC_CODE_INST_BR:           return "INST_BR";
-    case naclbitc::FUNC_CODE_INST_SWITCH:       return "INST_SWITCH";
-    case naclbitc::FUNC_CODE_INST_INVOKE:       return "INST_INVOKE";
-    case naclbitc::FUNC_CODE_INST_UNREACHABLE:  return "INST_UNREACHABLE";
-
-    case naclbitc::FUNC_CODE_INST_PHI:          return "INST_PHI";
-    case naclbitc::FUNC_CODE_INST_ALLOCA:       return "INST_ALLOCA";
-    case naclbitc::FUNC_CODE_INST_LOAD:         return "INST_LOAD";
-    case naclbitc::FUNC_CODE_INST_VAARG:        return "INST_VAARG";
-    case naclbitc::FUNC_CODE_INST_STORE:        return "INST_STORE";
-    case naclbitc::FUNC_CODE_INST_EXTRACTVAL:   return "INST_EXTRACTVAL";
-    case naclbitc::FUNC_CODE_INST_INSERTVAL:    return "INST_INSERTVAL";
-    case naclbitc::FUNC_CODE_INST_CMP2:         return "INST_CMP2";
-    case naclbitc::FUNC_CODE_INST_VSELECT:      return "INST_VSELECT";
-    case naclbitc::FUNC_CODE_DEBUG_LOC_AGAIN:   return "DEBUG_LOC_AGAIN";
-    case naclbitc::FUNC_CODE_INST_CALL:         return "INST_CALL";
-    case naclbitc::FUNC_CODE_INST_CALL_INDIRECT: return "INST_CALL_INDIRECT";
-    case naclbitc::FUNC_CODE_DEBUG_LOC:         return "DEBUG_LOC";
-    case naclbitc::FUNC_CODE_INST_FORWARDTYPEREF: return "FORWARDTYPEREF";
-    }
-  case naclbitc::VALUE_SYMTAB_BLOCK_ID:
-    switch (CodeID) {
-    default: return 0;
-    case naclbitc::VST_CODE_ENTRY: return "ENTRY";
-    case naclbitc::VST_CODE_BBENTRY: return "BBENTRY";
-    }
-  case naclbitc::METADATA_ATTACHMENT_ID:
-    switch(CodeID) {
-    default:return 0;
-    case naclbitc::METADATA_ATTACHMENT: return "METADATA_ATTACHMENT";
-    }
-  case naclbitc::METADATA_BLOCK_ID:
-    switch(CodeID) {
-    default:return 0;
-    case naclbitc::METADATA_STRING:      return "METADATA_STRING";
-    case naclbitc::METADATA_NAME:        return "METADATA_NAME";
-    case naclbitc::METADATA_KIND:        return "METADATA_KIND";
-    case naclbitc::METADATA_NODE:        return "METADATA_NODE";
-    case naclbitc::METADATA_FN_NODE:     return "METADATA_FN_NODE";
-    case naclbitc::METADATA_NAMED_NODE:  return "METADATA_NAMED_NODE";
-    }
-  case naclbitc::GLOBALVAR_BLOCK_ID:
-    switch (CodeID) {
-    default: return 0;
-    case naclbitc::GLOBALVAR_VAR:        return "VAR";
-    case naclbitc::GLOBALVAR_COMPOUND:   return "COMPOUND";
-    case naclbitc::GLOBALVAR_ZEROFILL:   return "ZEROFILL";
-    case naclbitc::GLOBALVAR_DATA:       return "DATA";
-    case naclbitc::GLOBALVAR_RELOC:      return "RELOC";
-    case naclbitc::GLOBALVAR_COUNT:      return "COUNT";
-    }
-  }
-}
-
-NaClBitcodeCodeDistElement::~NaClBitcodeCodeDistElement() {}
-
-NaClBitcodeDistElement *NaClBitcodeCodeDistElement::CreateElement(
-    NaClBitcodeDistValue Value) const {
-  return new NaClBitcodeCodeDistElement();
-}
-
-void NaClBitcodeCodeDistElement::
-GetValueList(const NaClBitcodeRecord &Record,
-             ValueListType &ValueList) const {
-  if (Record.GetEntryKind() == NaClBitstreamEntry::Record) {
-    ValueList.push_back(Record.GetCode());
-  }
-}
-
-const char *NaClBitcodeCodeDistElement::GetTitle() const {
-  return "Record Histogram:";
-}
-
-const char *NaClBitcodeCodeDistElement::GetValueHeader() const {
-  return "Record Kind";
-}
-
-void NaClBitcodeCodeDistElement::
-PrintRowValue(raw_ostream &Stream,
-              NaClBitcodeDistValue Value,
-              const NaClBitcodeDist *Distribution) const {
-  Stream <<
-      NaClBitcodeCodeDist::
-      GetCodeName(Value,
-                  cast<NaClBitcodeCodeDist>(Distribution)->GetBlockID());
-}
-
-NaClBitcodeCodeDistElement NaClBitcodeCodeDist::DefaultSentinel;
-
-NaClBitcodeCodeDist::~NaClBitcodeCodeDist() {}
-
-std::string NaClBitcodeCodeDist::GetCodeName(unsigned CodeID,
-                                             unsigned BlockID) {
-  if (const char *CodeName = ::GetCodeName(CodeID, BlockID))
-    return CodeName;
-
-  std::string Str;
-  raw_string_ostream StrStrm(Str);
-  StrStrm << "UnknownCode" << CodeID;
-  return StrStrm.str();
-}
diff --git a/lib/Bitcode/NaCl/Analysis/NaClBitcodeDist.cpp b/lib/Bitcode/NaCl/Analysis/NaClBitcodeDist.cpp
deleted file mode 100644
index fac862dc523..00000000000
--- a/lib/Bitcode/NaCl/Analysis/NaClBitcodeDist.cpp
+++ /dev/null
@@ -1,204 +0,0 @@
-//===- NaClBitcodeDist.cpp --------------------------------------*- C++ -*-===//
-//     Internal implementation of PNaCl bitcode distributions.
-//
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
-//
-//===----------------------------------------------------------------------===//
-
-#include "llvm/Bitcode/NaCl/NaClBitcodeDist.h"
-
-using namespace llvm;
-
-NaClBitcodeDist::~NaClBitcodeDist() {
-  RemoveCachedDistribution();
-  for (const_iterator Iter = begin(), IterEnd = end();
-       Iter != IterEnd; ++Iter) {
-    delete Iter->second;
-  }
-}
-
-NaClBitcodeDistElement *NaClBitcodeDist::CreateElement(
-    NaClBitcodeDistValue Value) const {
-  return Sentinel->CreateElement(Value);
-}
-
-void NaClBitcodeDist::GetValueList(const NaClBitcodeRecord &Record,
-                                   ValueListType &ValueList) const {
-  Sentinel->GetValueList(Record, ValueList);
-}
-
-void NaClBitcodeDist::AddRecord(const NaClBitcodeRecord &Record) {
-  if (StorageKind != NaClBitcodeDist::RecordStorage)
-    return;
-  ValueListType ValueList;
-  GetValueList(Record, ValueList);
-  if (!ValueList.empty()) {
-    RemoveCachedDistribution();
-    for (ValueListIterator
-             Iter = ValueList.begin(),
-             IterEnd = ValueList.end();
-         Iter != IterEnd; ++Iter) {
-      NaClBitcodeDistElement *Element = GetElement(*Iter);
-      Element->AddRecord(Record);
-      ++Total;
-    }
-  }
-}
-
-void NaClBitcodeDist::AddBlock(const NaClBitcodeBlock &Block) {
-  if (StorageKind != NaClBitcodeDist::BlockStorage)
-    return;
-  RemoveCachedDistribution();
-  ++Total;
-  unsigned BlockID = Block.GetBlockID();
-  NaClBitcodeDistElement *Element = GetElement(BlockID);
-  Element->AddBlock(Block);
-}
-
-void NaClBitcodeDist::Print(raw_ostream &Stream,
-                            const std::string &Indent) const {
-  const Distribution *Dist = GetDistribution();
-  Stream << Indent;
-  Sentinel->PrintTitle(Stream, this);
-  Stream << Indent;
-  Sentinel->PrintHeader(Stream);
-  Stream << "\n";
-  bool NeedsHeader = false;
-  for (size_t I = 0, E = Dist->size(); I < E; ++I) {
-    if (NeedsHeader) {
-      // Reprint the header so that rows are more readable.
-      Stream << Indent << "  " << Sentinel->GetTitle() << " (continued)\n";
-      Stream << Indent;
-      Sentinel->PrintHeader(Stream);
-      Stream << "\n";
-    }
-    const DistPair &Pair = Dist->at(I);
-    Stream << Indent;
-    NaClBitcodeDistElement *Element = at(Pair.second);
-    Element->PrintRow(Stream, Pair.second, this);
-    NeedsHeader = Element->PrintNestedDistIfApplicable(Stream, Indent);
-  }
-}
-
-void NaClBitcodeDist::Sort() const {
-  RemoveCachedDistribution();
-  CachedDistribution = new Distribution();
-  for (const_iterator Iter = begin(), IterEnd = end();
-       Iter != IterEnd; ++Iter) {
-    const NaClBitcodeDistElement *Elmt = Iter->second;
-    // Only add if histogram element is non-empty.
-    if (Elmt->GetNumInstances()) {
-      double Importance = Elmt->GetImportance(Iter->first);
-      CachedDistribution->push_back(std::make_pair(Importance, Iter->first));
-    }
-  }
-  // Sort in ascending order, based on importance.
-  std::stable_sort(CachedDistribution->begin(),
-                   CachedDistribution->end());
-  // Reverse so most important appear first.
-  std::reverse(CachedDistribution->begin(),
-               CachedDistribution->end());
-}
-
-NaClBitcodeDistElement::~NaClBitcodeDistElement() {}
-
-void NaClBitcodeDistElement::AddRecord(const NaClBitcodeRecord &Record) {
-  ++NumInstances;
-}
-
-void NaClBitcodeDistElement::AddBlock(const NaClBitcodeBlock &Block) {
-  ++NumInstances;
-}
-
-void NaClBitcodeDistElement::GetValueList(const NaClBitcodeRecord &Record,
-                                          ValueListType &ValueList) const {
-  // By default, assume no record values are defined.
-}
-
-double NaClBitcodeDistElement::GetImportance(NaClBitcodeDistValue Value) const {
-  return static_cast<double>(NumInstances);
-}
-
-const char *NaClBitcodeDistElement::GetTitle() const {
-  return "Distribution";
-}
-
-void NaClBitcodeDistElement::
-PrintTitle(raw_ostream &Stream, const NaClBitcodeDist *Distribution) const {
-  Stream << GetTitle() << " (" << Distribution->size() << " elements):\n\n";
-}
-
-const char *NaClBitcodeDistElement::GetValueHeader() const {
-  return "Value";
-}
-
-void NaClBitcodeDistElement::PrintStatsHeader(raw_ostream &Stream) const {
-  Stream << "   Count %Count";
-}
-
-void NaClBitcodeDistElement::
-PrintHeader(raw_ostream &Stream) const {
-  PrintStatsHeader(Stream);
-  Stream << " " << GetValueHeader();
-}
-
-void NaClBitcodeDistElement::
-PrintRowStats(raw_ostream &Stream,
-              const NaClBitcodeDist *Distribution) const {
-  unsigned Count = GetNumInstances();
-  Stream << format("%8d %6.2f",
-                   Count,
-                   (double) Count/Distribution->GetTotal()*100.0);
-}
-
-void NaClBitcodeDistElement::
-PrintRowValue(raw_ostream &Stream,
-              NaClBitcodeDistValue Value,
-              const NaClBitcodeDist *Distribution) const {
-  std::string ValueFormat;
-  raw_string_ostream StrStream(ValueFormat);
-  StrStream << "%" << strlen(GetValueHeader()) << "d";
-  StrStream.flush();
-  Stream << format(ValueFormat.c_str(), (int) Value);
-}
-
-void NaClBitcodeDistElement::
-PrintRow(raw_ostream &Stream,
-         NaClBitcodeDistValue Value,
-         const NaClBitcodeDist *Distribution) const {
-  PrintRowStats(Stream, Distribution);
-  Stream << " ";
-  PrintRowValue(Stream, Value, Distribution);
-  Stream << "\n";
-}
-
-const SmallVectorImpl<NaClBitcodeDist*> *NaClBitcodeDistElement::
-GetNestedDistributions() const {
-  return 0;
-}
-
-bool NaClBitcodeDistElement::
-PrintNestedDistIfApplicable(raw_ostream &Stream,
-                            const std::string &Indent) const {
-  bool PrintedNestedDists = false;
-  if (const SmallVectorImpl<NaClBitcodeDist*> *Dists =
-      GetNestedDistributions()) {
-    for (SmallVectorImpl<NaClBitcodeDist*>::const_iterator
-             Iter = Dists->begin(), IterEnd = Dists->end();
-         Iter != IterEnd; ++Iter) {
-      NaClBitcodeDist *Dist = *Iter;
-      if (!Dist->empty()) {
-        if (!PrintedNestedDists) {
-          PrintedNestedDists = true;
-          Stream << "\n";
-        }
-        Dist->Print(Stream, Indent + "    ");
-        Stream << "\n";
-      }
-    }
-  }
-  return PrintedNestedDists;
-}
diff --git a/lib/Bitcode/NaCl/Analysis/NaClBitcodeSizeDist.cpp b/lib/Bitcode/NaCl/Analysis/NaClBitcodeSizeDist.cpp
deleted file mode 100644
index 9bdea988f98..00000000000
--- a/lib/Bitcode/NaCl/Analysis/NaClBitcodeSizeDist.cpp
+++ /dev/null
@@ -1,59 +0,0 @@
-//===-- NaClBitcodeSizeDist.cpp --------------------------------------------===//
-//      Implements distribution maps for value record sizes associated with
-//      bitcode records.
-//
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
-//
-//===----------------------------------------------------------------------===//
-
-#include "llvm/Bitcode/NaCl/NaClBitcodeSizeDist.h"
-
-using namespace llvm;
-
-NaClBitcodeSizeDistElement::~NaClBitcodeSizeDistElement() {}
-
-NaClBitcodeDistElement *NaClBitcodeSizeDistElement::CreateElement(
-    NaClBitcodeDistValue Value) const {
-  return new NaClBitcodeSizeDistElement();
-}
-
-void NaClBitcodeSizeDistElement::
-GetValueList(const NaClBitcodeRecord &Record,
-             ValueListType &ValueList) const {
-  unsigned Size = Record.GetValues().size();
-  // Map all sizes greater than the max value index into the same bucket.
-  if (Size > NaClValueIndexCutoff) Size = NaClValueIndexCutoff;
-  ValueList.push_back(Size);
-}
-
-void NaClBitcodeSizeDistElement::AddRecord(const NaClBitcodeRecord &Record) {
-  NaClBitcodeDistElement::AddRecord(Record);
-  ValueIndexDist.AddRecord(Record);
-}
-
-const SmallVectorImpl<NaClBitcodeDist*> *NaClBitcodeSizeDistElement::
-GetNestedDistributions() const {
-  return &NestedDists;
-}
-
-const char *NaClBitcodeSizeDistElement::GetTitle() const {
-  return "Record sizes";
-}
-
-const char *NaClBitcodeSizeDistElement::GetValueHeader() const {
-  return "   Size";
-}
-
-void NaClBitcodeSizeDistElement::
-PrintRowValue(raw_ostream &Stream,
-              NaClBitcodeDistValue Value,
-              const NaClBitcodeDist *Distribution) const {
-  Stream << format("%7u", Value);
-  // Report if we merged in GetValueList.
-  if (Value >= NaClValueIndexCutoff) Stream << "+";
-}
-
-NaClBitcodeSizeDistElement NaClBitcodeSizeDistElement::Sentinel;
diff --git a/lib/Bitcode/NaCl/Analysis/NaClBitcodeSubblockDist.cpp b/lib/Bitcode/NaCl/Analysis/NaClBitcodeSubblockDist.cpp
deleted file mode 100644
index 9560a4f98d7..00000000000
--- a/lib/Bitcode/NaCl/Analysis/NaClBitcodeSubblockDist.cpp
+++ /dev/null
@@ -1,40 +0,0 @@
-//===-- NaClBitcodeSubblockDist.cpp ---------------------------------------===//
-//      implements distribution maps for subblock values within an
-//      (externally specified) block.
-//
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
-//
-//===----------------------------------------------------------------------===//
-
-#include "llvm/Bitcode/NaCl/NaClBitcodeSubblockDist.h"
-#include "llvm/Bitcode/NaCl/NaClBitcodeBlockDist.h"
-
-using namespace llvm;
-
-NaClBitcodeSubblockDistElement NaClBitcodeSubblockDist::DefaultSentinal;
-
-NaClBitcodeSubblockDistElement::~NaClBitcodeSubblockDistElement() {}
-
-NaClBitcodeDistElement *NaClBitcodeSubblockDistElement::
-CreateElement(NaClBitcodeDistValue Value) const {
-  return new NaClBitcodeSubblockDistElement();
-}
-
-const char *NaClBitcodeSubblockDistElement::GetTitle() const {
-  return "Subblocks";
-}
-
-const char *NaClBitcodeSubblockDistElement::GetValueHeader() const {
-  return "Subblock";
-}
-
-void NaClBitcodeSubblockDistElement::
-PrintRowValue(raw_ostream &Stream, NaClBitcodeDistValue Value,
-              const NaClBitcodeDist *Distribution) const {
-  Stream << NaClBitcodeBlockDist::GetName(Value);
-}
-
-NaClBitcodeSubblockDist::~NaClBitcodeSubblockDist() {}
diff --git a/lib/Bitcode/NaCl/Analysis/NaClBitcodeValueDist.cpp b/lib/Bitcode/NaCl/Analysis/NaClBitcodeValueDist.cpp
deleted file mode 100644
index e67bebf76cb..00000000000
--- a/lib/Bitcode/NaCl/Analysis/NaClBitcodeValueDist.cpp
+++ /dev/null
@@ -1,216 +0,0 @@
-//===-- NaClBitcodeValueDist.cpp ------------------------------------------===//
-//      Implements (nested) distribution maps to separate out values at each
-//      index in a bitcode record.
-//
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
-//
-//===----------------------------------------------------------------------===//
-
-#include "llvm/ADT/STLExtras.h"
-#include "llvm/Bitcode/NaCl/NaClBitcodeValueDist.h"
-#include "llvm/Support/DataTypes.h"
-
-using namespace llvm;
-
-/// Constant defining cutoff for value distributions. All values less than
-/// this constant will be stored as a singleton range.
-static NaClBitcodeDistValue ValueDistSingletonCutoff =
-    (NaClBitcodeDistValue)1 << 6;
-
-/// Largest value possible.
-static NaClBitcodeDistValue MaxValue = ~((NaClBitcodeDistValue) 0);
-
-/// Cutoffs for value ranges, starting the first range at
-/// ValueDistSingetonCutoff, and ending the last range with MaxValue.
-static NaClBitcodeDistValue ValueCutoffs[] = {
-  (NaClBitcodeDistValue)1 << 8,
-  (NaClBitcodeDistValue)1 << 12,
-  (NaClBitcodeDistValue)1 << 16,
-  (NaClBitcodeDistValue)1 << 24,
-  (NaClBitcodeDistValue)1 << 32
-};
-
-// Converts the value to the corresponding range (value) that will
-// be stored in a value distribution map.
-NaClValueRangeIndexType
-llvm::GetNaClValueRangeIndex(NaClBitcodeDistValue Value) {
-  if (Value < ValueDistSingletonCutoff)
-    return Value;
-
-  size_t ValueCutoffsSize = array_lengthof(ValueCutoffs);
-  for (size_t i = 0; i < ValueCutoffsSize; ++i) {
-    if (Value < ValueCutoffs[i]) {
-      return ValueDistSingletonCutoff + i;
-    }
-  }
-
-  return ValueDistSingletonCutoff + ValueCutoffsSize;
-}
-
-NaClValueRangeType llvm::GetNaClValueRange(NaClValueRangeIndexType RangeIndex) {
-  if (RangeIndex < ValueDistSingletonCutoff)
-    return NaClValueRangeType(RangeIndex, RangeIndex);
-
-  size_t Index = RangeIndex - ValueDistSingletonCutoff;
-  size_t ValueCutoffsSize = array_lengthof(ValueCutoffs);
-  if (Index >= ValueCutoffsSize)
-    return NaClValueRangeType(ValueCutoffs[ValueCutoffsSize-1], MaxValue);
-  else if (Index == 0)
-    return NaClValueRangeType(ValueDistSingletonCutoff, ValueCutoffs[0]);
-  else
-    return NaClValueRangeType(ValueCutoffs[Index-1], ValueCutoffs[Index]-1);
-}
-
-NaClBitcodeValueDistElement NaClBitcodeValueDistElement::Sentinel;
-
-NaClBitcodeValueDistElement::~NaClBitcodeValueDistElement() {}
-
-NaClBitcodeDistElement *NaClBitcodeValueDistElement::CreateElement(
-      NaClBitcodeDistValue Value) const {
-  return new NaClBitcodeValueDistElement();
-}
-
-double NaClBitcodeValueDistElement::
-GetImportance(NaClBitcodeDistValue Value) const {
-  NaClValueRangeType Pair = GetNaClValueRange(Value);
-  return GetNumInstances() / static_cast<double>((Pair.second-Pair.first) + 1);
-}
-
-const char *NaClBitcodeValueDistElement::GetTitle() const {
-  return "Values";
-}
-
-const char *NaClBitcodeValueDistElement::GetValueHeader() const {
-  return "Value / Range";
-}
-
-void NaClBitcodeValueDistElement::
-PrintRowValue(raw_ostream &Stream,
-              NaClBitcodeDistValue Value,
-              const NaClBitcodeDist *Distribution) const {
-  NaClValueRangeType Range = GetNaClValueRange(Value);
-  Stream << format("%" PRIu64, Range.first);
-  if (Range.first != Range.second) {
-    Stream << " .. " << format("%" PRIu64, Range.second);
-  }
-}
-
-/// Defines the sentinel distribution element of range indices for untracked
-/// indices in the bitcode records, which need a clarifying print title.
-class NaClBitcodeUntrackedValueDistElement
-    : public NaClBitcodeValueDistElement {
-public:
-  static NaClBitcodeUntrackedValueDistElement Sentinel;
-
-  virtual ~NaClBitcodeUntrackedValueDistElement();
-
-  virtual const char *GetTitle() const;
-
-private:
-  NaClBitcodeUntrackedValueDistElement()
-      : NaClBitcodeValueDistElement()
-  {}
-};
-
-NaClBitcodeUntrackedValueDistElement
-NaClBitcodeUntrackedValueDistElement::Sentinel;
-
-NaClBitcodeUntrackedValueDistElement::~NaClBitcodeUntrackedValueDistElement() {}
-
-const char *NaClBitcodeUntrackedValueDistElement::GetTitle() const {
-  return "Values for untracked indices";
-}
-
-NaClBitcodeValueDist::
-NaClBitcodeValueDist(unsigned Index, bool AllRemainingIndices)
-    : NaClBitcodeDist(RecordStorage,
-                      (AllRemainingIndices
-                       ? &NaClBitcodeUntrackedValueDistElement::Sentinel
-                       : &NaClBitcodeValueDistElement::Sentinel),
-                      RD_ValueDist),
-      Index(Index),
-      AllRemainingIndices(AllRemainingIndices) {
-}
-
-NaClBitcodeValueDist::~NaClBitcodeValueDist() {}
-
-void NaClBitcodeValueDist::GetValueList(const NaClBitcodeRecord &Record,
-                                        ValueListType &ValueList) const {
-  const NaClBitcodeRecord::RecordVector &Values = Record.GetValues();
-  if (AllRemainingIndices) {
-    for (size_t i = Index, i_limit = Values.size(); i < i_limit; ++i) {
-      ValueList.push_back(GetNaClValueRangeIndex(Values[i]));
-    }
-  } else {
-    ValueList.push_back(GetNaClValueRangeIndex(Values[Index]));
-  }
-}
-
-NaClBitcodeValueIndexDistElement NaClBitcodeValueIndexDistElement::Sentinel;
-
-NaClBitcodeValueIndexDistElement::~NaClBitcodeValueIndexDistElement() {}
-
-NaClBitcodeDistElement *NaClBitcodeValueIndexDistElement::CreateElement(
-    NaClBitcodeDistValue Value) const {
-  return new NaClBitcodeValueIndexDistElement(Value);
-}
-
-void NaClBitcodeValueIndexDistElement::
-GetValueList(const NaClBitcodeRecord &Record, ValueListType &ValueList) const {
-  unsigned i_limit = Record.GetValues().size();
-  if (i_limit > NaClValueIndexCutoff) i_limit = NaClValueIndexCutoff;
-  for (size_t i = 0; i < i_limit; ++i) {
-    ValueList.push_back(i);
-  }
-}
-
-double NaClBitcodeValueIndexDistElement::
-GetImportance(NaClBitcodeDistValue Value) const {
-  // Since all indices (usually) have the same number of instances,
-  // that is a bad measure of importance. Rather, we will base
-  // importance in terms of the value distribution for the value
-  // index. We would like the indicies with a few, large instance
-  // counts, to appear before value indices with a uniform value
-  // distribution. To do this, we will use the sum of the squares of
-  // the number of instances for each value (i.e. sort by standard
-  // deviation).
-  double Sum = 0.0;
-  for (NaClBitcodeDist::const_iterator
-           Iter = ValueDist.begin(), IterEnd = ValueDist.end();
-       Iter != IterEnd; ++Iter) {
-    const NaClBitcodeValueDistElement *Elmt =
-        cast<NaClBitcodeValueDistElement>(Iter->second);
-    double Count = static_cast<double>(Elmt->GetImportance(Iter->first));
-    Sum += Count * Count;
-  }
-  return Sum;
-}
-
-void NaClBitcodeValueIndexDistElement::
-AddRecord(const NaClBitcodeRecord &Record) {
-  NaClBitcodeDistElement::AddRecord(Record);
-  ValueDist.AddRecord(Record);
-}
-
-const char *NaClBitcodeValueIndexDistElement::GetTitle() const {
-  return "Value indices";
-}
-
-const char *NaClBitcodeValueIndexDistElement::GetValueHeader() const {
-  return "  Index";
-}
-
-void NaClBitcodeValueIndexDistElement::
-PrintRowValue(raw_ostream &Stream,
-              NaClBitcodeDistValue Value,
-              const NaClBitcodeDist *Distribution) const {
-  Stream << format("%7u", Value);
-}
-
-const SmallVectorImpl<NaClBitcodeDist*> *NaClBitcodeValueIndexDistElement::
-GetNestedDistributions() const {
-  return &NestedDists;
-}
diff --git a/lib/Bitcode/NaCl/Analysis/NaClCompress.cpp b/lib/Bitcode/NaCl/Analysis/NaClCompress.cpp
deleted file mode 100644
index 77225aadc2f..00000000000
--- a/lib/Bitcode/NaCl/Analysis/NaClCompress.cpp
+++ /dev/null
@@ -1,1653 +0,0 @@
-//===-- NaClCompress.cpp - Bitcode (abbrev) compression -----------------===//
-//
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
-//
-//===----------------------------------------------------------------------===//
-//
-// Analyzes the data in memory buffer, and determines what
-// abbreviations can be added to compress the bitcode file. The result
-// is written to an output stream.
-//
-// A bitcode file has two types of abbreviations. The first are Global
-// abbreviations that apply to all instances of a particular type of
-// block.  These abbreviations appear in the BlockInfo block of the
-// bitcode file.
-//
-// The second type of abbreviations are local to a particular instance
-// of a block.
-//
-// For simplicity, we will only add global abbreviations. Local
-// abbreviations are converted to corresponding global abbreviations,
-// so that they can be added as global abbreviations.
-//
-// The process of compressing is done by reading the input file
-// twice. In the first round, the records are read and analyzed,
-// generating a set of (global) abbreviations to use to generate the
-// compressed output file. Then, the input file is read again, and for
-// each record, the best fitting global abbreviation is found (or it
-// figures out that leaving the record unabbreviated is best) and
-// writes the record out accordingly.
-//
-// TODO(kschimpf): The current implementation does a trivial solution
-// for the first round.  It just converts all abbreviations (local and
-// global) into global abbreviations.  Future refinements of this file
-// will generate better (and more intelligent) global abbreviations,
-// based on the records found on the first read of the bitcode file.
-//
-//===----------------------------------------------------------------------===//
-
-#include "llvm/Bitcode/NaCl/NaClCompress.h"
-
-#include "llvm/ADT/DenseMap.h"
-#include "llvm/ADT/STLExtras.h"
-#include "llvm/Bitcode/NaCl/AbbrevTrieNode.h"
-#include "llvm/Bitcode/NaCl/NaClBitcodeHeader.h"
-#include "llvm/Bitcode/NaCl/NaClBitstreamWriter.h"
-#include "llvm/Bitcode/NaCl/NaClCompressBlockDist.h"
-#include "llvm/Bitcode/NaCl/NaClReaderWriter.h"
-
-namespace {
-
-using namespace llvm;
-
-// Generates an error message when outside parsing, and no
-// corresponding bit position is known.
-static bool Error(const std::string &Err) {
-  errs() << Err << "\n";
-  return true;
-}
-
-// Prints out the abbreviation in readable form to the given Stream.
-static void printAbbrev(raw_ostream &Stream, unsigned BlockID,
-                        const NaClBitCodeAbbrev *Abbrev) {
-  Stream << "Abbrev(block " << BlockID << "): ";
-  Abbrev->Print(Stream);
-}
-
-/// type map from bitstream abbreviation index to corresponding
-/// internal abbreviation index.
-typedef std::map<unsigned, unsigned> BitstreamToInternalAbbrevMapType;
-
-/// Defines a mapping from bitstream abbreviation indices, to
-/// corresponding internal abbreviation indices.
-class AbbrevBitstreamToInternalMap {
-public:
-  AbbrevBitstreamToInternalMap()
-      : NextBitstreamAbbrevIndex(0) {}
-
-  /// Returns the bitstream abbreviaion index that will be associated
-  /// with the next internal abbreviation index.
-  unsigned getNextBitstreamAbbrevIndex() {
-    return NextBitstreamAbbrevIndex;
-  }
-
-  /// Changes the next bitstream abbreviation index to the given value.
-  void setNextBitstreamAbbrevIndex(unsigned NextIndex) {
-    NextBitstreamAbbrevIndex = NextIndex;
-  }
-
-  /// Returns true if there is an internal abbreviation index for the
-  /// given bitstream abbreviation.
-  bool definesBitstreamAbbrevIndex(unsigned Index) {
-    return BitstreamToInternalAbbrevMap.find(Index) !=
-        BitstreamToInternalAbbrevMap.end();
-  }
-
-  /// Returns the internal abbreviation index for the given bitstream
-  /// abbreviation index.
-  unsigned getInternalAbbrevIndex(unsigned Index) {
-    return BitstreamToInternalAbbrevMap.at(Index);
-  }
-
-  /// Installs the given internal abbreviation index using the next
-  /// available bitstream abbreviation index.
-  void installNewBitstreamAbbrevIndex(unsigned InternalIndex) {
-    BitstreamToInternalAbbrevMap[NextBitstreamAbbrevIndex++] = InternalIndex;
-  }
-
-private:
-  // The index of the next bitstream abbreviation to be defined.
-  unsigned NextBitstreamAbbrevIndex;
-  // Map from bitstream abbreviation index to corresponding internal
-  // abbreviation index.
-  BitstreamToInternalAbbrevMapType BitstreamToInternalAbbrevMap;
-};
-
-/// Defines the list of abbreviations associated with a block.
-class BlockAbbrevs {
-public:
-  // Vector to hold the (ordered) list of abbreviations.
-  typedef SmallVector<NaClBitCodeAbbrev*, 32> AbbrevVector;
-
-  BlockAbbrevs(unsigned BlockID)
-      : BlockID(BlockID) {
-    // backfill internal indices that don't correspond to bitstream
-    // application abbreviations, so that added abbreviations have
-    // valid abbreviation indices.
-    for (unsigned i = 0; i < naclbitc::FIRST_APPLICATION_ABBREV; ++i) {
-      // Make all non-application abbreviations look like the default
-      // abbreviation.
-      NaClBitCodeAbbrev *Abbrev = new NaClBitCodeAbbrev();
-      Abbrev->Add(NaClBitCodeAbbrevOp(NaClBitCodeAbbrevOp::Array));
-      Abbrev->Add(NaClBitCodeAbbrevOp(NaClBitCodeAbbrevOp::VBR, 6));
-      Abbrevs.push_back(Abbrev);
-    }
-    GlobalAbbrevBitstreamToInternalMap.
-        setNextBitstreamAbbrevIndex(Abbrevs.size());
-  }
-
-  ~BlockAbbrevs() {
-    for (AbbrevVector::const_iterator
-             Iter = Abbrevs.begin(), IterEnd = Abbrevs.end();
-         Iter != IterEnd; ++Iter) {
-      (*Iter)->dropRef();
-    }
-    for (AbbrevLookupSizeMap::const_iterator
-             Iter = LookupMap.begin(), IterEnd = LookupMap.end();
-         Iter != IterEnd; ++Iter) {
-      delete Iter->second;
-    }
-  }
-
-  // Constant used to denote that a given abbreviation is not in the
-  // set of abbreviations defined by the block.
-  static const int NO_SUCH_ABBREVIATION = -1;
-
-  // Returns the index to the corresponding application abbreviation,
-  // if it exists.  Otherwise return NO_SUCH_ABBREVIATION;
-  int findAbbreviation(const NaClBitCodeAbbrev *Abbrev) const {
-    for (unsigned i = getFirstApplicationAbbreviation();
-         i < getNumberAbbreviations(); ++i) {
-      if (*Abbrevs[i] == *Abbrev) return i;
-    }
-    return NO_SUCH_ABBREVIATION;
-  }
-
-  /// Adds the given abbreviation to the set of global abbreviations
-  /// defined for the block. Guarantees that duplicate abbreviations
-  /// are not added to the block. Note: Code takes ownership of
-  /// the given abbreviation. Returns true if new abbreviation.
-  /// Updates Index to the index where the abbreviation is located
-  /// in the set of abbreviations.
-  bool addAbbreviation(NaClBitCodeAbbrev *Abbrev, int &Index) {
-    Index = findAbbreviation(Abbrev);
-    if (Index != NO_SUCH_ABBREVIATION) {
-      // Already defined, don't install.
-      Abbrev->dropRef();
-      return false;
-    }
-
-    // New abbreviation. Add.
-    Index = Abbrevs.size();
-    Abbrevs.push_back(Abbrev);
-    return true;
-  }
-
-  /// Adds the given abbreviation to the set of global abbreviations
-  /// defined for the block. Guarantees that duplicate abbreviations
-  /// are not added to the block. Note: Code takes ownership of
-  /// the given abbreviation. Returns true if new abbreviation.
-  bool addAbbreviation(NaClBitCodeAbbrev *Abbrev) {
-    int Index;
-    return addAbbreviation(Abbrev, Index);
-  }
-
-  /// The block ID associated with the block.
-  unsigned getBlockID() const {
-    return BlockID;
-  }
-
-  /// Returns the index of the frist application abbreviation for the
-  /// block.
-  unsigned getFirstApplicationAbbreviation() const {
-    return naclbitc::FIRST_APPLICATION_ABBREV;
-  }
-
-  // Returns the number of abbreviations associated with the block.
-  unsigned getNumberAbbreviations() const {
-    return Abbrevs.size();
-  }
-
-  // Returns true if there is an application abbreviation.
-  bool hasApplicationAbbreviations() const {
-    return Abbrevs.size() > naclbitc::FIRST_APPLICATION_ABBREV;
-  }
-
-  /// Returns the abbreviation associated with the given abbreviation
-  /// index.
-  NaClBitCodeAbbrev *getIndexedAbbrev(unsigned index) {
-    if (index >= Abbrevs.size()) return 0;
-    return Abbrevs[index];
-  }
-
-  // Builds the corresponding fast lookup map for finding abbreviations
-  // that applies to abbreviations in the block
-  void buildAbbrevLookupSizeMap(
-      const NaClBitcodeCompressor::CompressFlags &Flags) {
-    NaClBuildAbbrevLookupMap(getLookupMap(),
-                             getAbbrevs(),
-                             getFirstApplicationAbbreviation());
-    if (Flags.ShowAbbrevLookupTries) printLookupMap(errs());
-  }
-
-  AbbrevBitstreamToInternalMap &getGlobalAbbrevBitstreamToInternalMap() {
-    return GlobalAbbrevBitstreamToInternalMap;
-  }
-
-  AbbrevLookupSizeMap &getLookupMap() {
-    return LookupMap;
-  }
-
-  // Returns lower level vector of abbreviations.
-  const AbbrevVector &getAbbrevs() const {
-    return Abbrevs;
-  }
-
-  // Returns the abbreviation (index) to use for the corresponding
-  // record, based on the abbreviations of this block.  Note: Assumes
-  // that buildAbbrevLookupSizeMap has already been called.
-  unsigned getRecordAbbrevIndex(const NaClBitcodeRecordData &Record) {
-    unsigned BestIndex = 0; // Ignored unless found candidate.
-    unsigned BestScore = 0; // Number of bits associated with BestIndex.
-    bool FoundCandidate = false;
-    NaClBitcodeValues Values(Record);
-    size_t Size = Values.size();
-
-    if (Size > NaClValueIndexCutoff) Size = NaClValueIndexCutoff+1;
-    AbbrevLookupSizeMap::const_iterator Pos = LookupMap.find(Size);
-    if (Pos != LookupMap.end()) {
-      if (const AbbrevTrieNode *Node = Pos->second) {
-        if (const AbbrevTrieNode *MatchNode =
-            Node->MatchRecord(Record)) {
-          const std::set<AbbrevIndexPair> &Abbreviations =
-              MatchNode->GetAbbreviations();
-          for (std::set<AbbrevIndexPair>::const_iterator
-                   Iter = Abbreviations.begin(),
-                   IterEnd = Abbreviations.end();
-               Iter != IterEnd; ++Iter) {
-            uint64_t NumBits = 0;
-            if (canUseAbbreviation(Values, Iter->second, NumBits)) {
-              if (!FoundCandidate || NumBits < BestScore) {
-                // Use this as candidate.
-                BestIndex = Iter->first;
-                BestScore = NumBits;
-                FoundCandidate = true;
-              }
-            }
-          }
-        }
-      }
-    }
-    if (FoundCandidate && BestScore <= unabbreviatedSize(Record)) {
-      return BestIndex;
-    }
-    return naclbitc::UNABBREV_RECORD;
-  }
-
-  // Computes the number of bits that will be generated by the
-  // corresponding read record, if no abbreviation is used.
-  static uint64_t unabbreviatedSize(const NaClBitcodeRecordData &Record) {
-    uint64_t NumBits = matchVBRBits(Record.Code, DefaultVBRBits);
-    size_t NumValues = Record.Values.size();
-    NumBits += matchVBRBits(NumValues, DefaultVBRBits);
-    for (size_t Index = 0; Index < NumValues; ++Index) {
-      NumBits += matchVBRBits(Record.Values[Index], DefaultVBRBits);
-    }
-    return NumBits;
-  }
-
-  // Returns true if the given abbreviation can be used to represent the
-  // record. Sets NumBits to the number of bits the abbreviation will
-  // generate. Note: Value of NumBits is undefined if this function
-  // return false.
-  static bool canUseAbbreviation(NaClBitcodeValues &Values,
-                                 NaClBitCodeAbbrev *Abbrev, uint64_t &NumBits) {
-    NumBits = 0;
-    unsigned OpIndex = 0;
-    unsigned OpIndexEnd = Abbrev->getNumOperandInfos();
-    size_t ValueIndex = 0;
-    size_t ValueIndexEnd = Values.size();
-    while (ValueIndex < ValueIndexEnd && OpIndex < OpIndexEnd) {
-      const NaClBitCodeAbbrevOp &Op = Abbrev->getOperandInfo(OpIndex);
-      switch (Op.getEncoding()) {
-      case NaClBitCodeAbbrevOp::Literal:
-        if (canUseSimpleAbbrevOp(Op, Values[ValueIndex], NumBits)) {
-          ++ValueIndex;
-          ++OpIndex;
-          continue;
-        } else {
-          return false;
-        }
-      case NaClBitCodeAbbrevOp::Array: {
-        assert(OpIndex+2 == OpIndexEnd);
-        const NaClBitCodeAbbrevOp &ElmtOp =
-            Abbrev->getOperandInfo(OpIndex+1);
-
-        // Add size of array.
-        NumBits += matchVBRBits(Values.size()-ValueIndex, DefaultVBRBits);
-
-        // Add size of each field.
-        for (; ValueIndex != ValueIndexEnd; ++ValueIndex) {
-          uint64_t FieldBits=0;
-          if (!canUseSimpleAbbrevOp(ElmtOp, Values[ValueIndex], FieldBits)) {
-            return false;
-          }
-          NumBits += FieldBits;
-        }
-        return true;
-      }
-      default: {
-        if (canUseSimpleAbbrevOp(Op, Values[ValueIndex], NumBits)) {
-          ++ValueIndex;
-          ++OpIndex;
-          break;
-        }
-        return false;
-      }
-      }
-    }
-    return ValueIndex == ValueIndexEnd && OpIndex == OpIndexEnd;
-  }
-
-  // Returns true if the given abbreviation Op defines a single value,
-  // and can be applied to the given Val. Adds the number of bits the
-  // abbreviation Op will generate to NumBits if Op applies.
-  static bool canUseSimpleAbbrevOp(const NaClBitCodeAbbrevOp &Op,
-                                   uint64_t Val,
-                                   uint64_t &NumBits) {
-    switch (Op.getEncoding()) {
-    case NaClBitCodeAbbrevOp::Literal:
-      return Val == Op.getValue();
-    case NaClBitCodeAbbrevOp::Array:
-      return false;
-    case NaClBitCodeAbbrevOp::Fixed: {
-      uint64_t Width = Op.getValue();
-      if (!matchFixedBits(Val, Width))
-        return false;
-      NumBits += Width;
-      return true;
-    }
-    case NaClBitCodeAbbrevOp::VBR:
-      if (unsigned Width = matchVBRBits(Val, Op.getValue())) {
-        NumBits += Width;
-        return true;
-      } else {
-        return false;
-      }
-    case NaClBitCodeAbbrevOp::Char6:
-      if (!NaClBitCodeAbbrevOp::isChar6(Val)) return false;
-      NumBits += 6;
-      return true;
-    }
-    llvm_unreachable("unhandled NaClBitCodeAbbrevOp encoding");
-  }
-
-  // Returns true if the given Val can be represented by abbreviation
-  // operand Fixed(Width).
-  static bool matchFixedBits(uint64_t Val, unsigned Width) {
-    // Note: The reader only allows up to 32 bits for fixed values.
-    if (Val & Mask32) return false;
-    if (Val & ~(~0U >> (32-Width))) return false;
-    return true;
-  }
-
-  // Returns the number of bits needed to represent Val by abbreviation
-  // operand VBR(Width). Note: Returns 0 if Val can't be represented
-  // by VBR(Width).
-  static unsigned matchVBRBits(uint64_t Val, unsigned Width) {
-    if (Width == 0) return 0;
-    unsigned NumBits = 0;
-    while (1) {
-      // values emitted Width-1 bits at a time (plus a continue bit).
-      NumBits += Width;
-      if ((Val & (1U << (Width-1))) == 0)
-        return NumBits;
-      Val >>= Width-1;
-    }
-  }
-
-private:
-  // Defines the number of bits used to print VBR array field values.
-  static const unsigned DefaultVBRBits = 6;
-  // Masks out the top-32 bits of a uint64_t value.
-  static const uint64_t Mask32 = 0xFFFFFFFF00000000;
-  // The block ID for which abbreviations are being associated.
-  unsigned BlockID;
-  // The list of abbreviations defined for the block.
-  AbbrevVector Abbrevs;
-  // The mapping from global bitstream abbreviations to the corresponding
-  // block abbreviation index (in Abbrevs).
-  AbbrevBitstreamToInternalMap GlobalAbbrevBitstreamToInternalMap;
-  // A fast lookup map for finding the abbreviation that applies
-  // to a record.
-  AbbrevLookupSizeMap LookupMap;
-
-  void printLookupMap(raw_ostream &Stream) const {
-    Stream << "------------------------------\n";
-    Stream << "Block " << getBlockID() << " abbreviation tries:\n";
-    bool IsFirstIteration = true;
-    for (AbbrevLookupSizeMap::const_iterator
-           Iter = LookupMap.begin(), IterEnd = LookupMap.end();
-         Iter != IterEnd; ++Iter) {
-      if (IsFirstIteration)
-        IsFirstIteration = false;
-      else
-        Stream << "-----\n";
-      if (Iter->second) {
-        Stream << "Index " << Iter->first << ":\n";
-        Iter->second->Print(Stream, "  ");
-      }
-    }
-    Stream << "------------------------------\n";
-  }
-};
-
-/// Defines a map from block ID's to the corresponding abbreviation
-/// map to use.
-typedef DenseMap<unsigned, BlockAbbrevs*> BlockAbbrevsMapType;
-typedef std::pair<unsigned, BlockAbbrevs*> BlockAbbrevsMapElmtType;
-
-/// Gets the corresponding block abbreviations for a block ID.
-static BlockAbbrevs *getAbbrevs(BlockAbbrevsMapType &AbbrevsMap,
-                                unsigned BlockID) {
-  BlockAbbrevs *Abbrevs = AbbrevsMap[BlockID];
-  if (Abbrevs == nullptr) {
-    Abbrevs = new BlockAbbrevs(BlockID);
-    AbbrevsMap[BlockID] = Abbrevs;
-  }
-  return Abbrevs;
-}
-
-/// Parses the bitcode file, analyzes it, and generates the
-/// corresponding lists of global abbreviations to use in the
-/// generated (compressed) bitcode file.
-class NaClAnalyzeParser : public NaClBitcodeParser {
-  NaClAnalyzeParser(const NaClAnalyzeParser&) = delete;
-  void operator=(const NaClAnalyzeParser&) = delete;
-
-public:
-  // Creates the analysis parser, which will fill the given
-  // BlockAbbrevsMap with appropriate abbreviations, after
-  // analyzing the bitcode file defined by Cursor.
-  NaClAnalyzeParser(const NaClBitcodeCompressor::CompressFlags &Flags,
-                    NaClBitstreamCursor &Cursor,
-                    BlockAbbrevsMapType &BlockAbbrevsMap)
-      : NaClBitcodeParser(Cursor), Flags(Flags),
-        BlockAbbrevsMap(BlockAbbrevsMap),
-        BlockDist(&NaClCompressBlockDistElement::Sentinel),
-        AbbrevListener(this)
-  {
-    SetListener(&AbbrevListener);
-  }
-
-  virtual ~NaClAnalyzeParser() {}
-
-  virtual bool ParseBlock(unsigned BlockID);
-
-  const NaClBitcodeCompressor::CompressFlags &Flags;
-
-  // Mapping from block ID's to the corresponding list of abbreviations
-  // associated with that block.
-  BlockAbbrevsMapType &BlockAbbrevsMap;
-
-  // Nested distribution capturing distribution of records in bitcode file.
-  NaClBitcodeBlockDist BlockDist;
-
-  // Listener used to get abbreviations as they are read.
-  NaClBitcodeParserListener AbbrevListener;
-};
-
-class NaClBlockAnalyzeParser : public NaClBitcodeParser {
-  NaClBlockAnalyzeParser(const NaClBlockAnalyzeParser&) = delete;
-  void operator=(NaClBlockAnalyzeParser&) = delete;
-
-public:
-  /// Top-level constructor to build the top-level block with the
-  /// given BlockID, and collect data (for compression) in that block.
-  NaClBlockAnalyzeParser(unsigned BlockID,
-                         NaClAnalyzeParser *Context)
-      : NaClBitcodeParser(BlockID, Context), Context(Context) {
-    init();
-  }
-
-  virtual ~NaClBlockAnalyzeParser() {
-    Context->BlockDist.AddBlock(GetBlock());
-  }
-
-protected:
-  /// Nested constructor to parse a block within a block.  Creates a
-  /// block parser to parse a block with the given BlockID, and
-  /// collect data (for compression) in that block.
-  NaClBlockAnalyzeParser(unsigned BlockID,
-                         NaClBlockAnalyzeParser *EnclosingParser)
-      : NaClBitcodeParser(BlockID, EnclosingParser),
-        Context(EnclosingParser->Context) {
-    init();
-  }
-
-public:
-  virtual bool ParseBlock(unsigned BlockID) {
-    NaClBlockAnalyzeParser Parser(BlockID, this);
-    return Parser.ParseThisBlock();
-  }
-
-  virtual void ProcessRecord() {
-    // Before processing the record, we need to rename the abbreviation
-    // index, so that we can look it up in the set of block abbreviations
-    // being defined.
-    if (Record.UsedAnAbbreviation()) {
-      unsigned AbbrevIndex = Record.GetAbbreviationIndex();
-      if (LocalAbbrevBitstreamToInternalMap.
-          definesBitstreamAbbrevIndex(AbbrevIndex)) {
-        Record.SetAbbreviationIndex(
-            LocalAbbrevBitstreamToInternalMap.
-            getInternalAbbrevIndex(AbbrevIndex));
-      } else {
-        AbbrevBitstreamToInternalMap &GlobalAbbrevBitstreamToInternalMap =
-            GlobalBlockAbbrevs->getGlobalAbbrevBitstreamToInternalMap();
-        if (GlobalAbbrevBitstreamToInternalMap.
-            definesBitstreamAbbrevIndex(AbbrevIndex)) {
-          Record.SetAbbreviationIndex(
-              GlobalAbbrevBitstreamToInternalMap.
-              getInternalAbbrevIndex(AbbrevIndex));
-        } else {
-          report_fatal_error("Bad abbreviation index in file");
-        }
-      }
-    }
-
-    cast<NaClCompressBlockDistElement>(
-        Context->BlockDist.GetElement(Record.GetBlockID()))
-        ->GetAbbrevDist().AddRecord(Record);
-  }
-
-  virtual void ProcessAbbreviation(unsigned BlockID,
-                                   NaClBitCodeAbbrev *Abbrev,
-                                   bool IsLocal) {
-    int Index;
-    addAbbreviation(BlockID, Abbrev->Simplify(), Index);
-    if (IsLocal) {
-      LocalAbbrevBitstreamToInternalMap.installNewBitstreamAbbrevIndex(Index);
-    } else {
-      getGlobalAbbrevs(BlockID)->getGlobalAbbrevBitstreamToInternalMap().
-          installNewBitstreamAbbrevIndex(Index);
-    }
-  }
-
-protected:
-  // The context (i.e. top-level) parser.
-  NaClAnalyzeParser *Context;
-
-  // The global block abbreviations associated with this block.
-  BlockAbbrevs *GlobalBlockAbbrevs;
-
-  // The local abbreviations associated with this block.
-  AbbrevBitstreamToInternalMap LocalAbbrevBitstreamToInternalMap;
-
-  /// Returns the set of (global) block abbreviations defined for the
-  /// given block ID.
-  BlockAbbrevs *getGlobalAbbrevs(unsigned BlockID) {
-    return getAbbrevs(Context->BlockAbbrevsMap, BlockID);
-  }
-
-  // Adds the abbreviation to the list of abbreviations for the given
-  // block.
-  void addAbbreviation(unsigned BlockID, NaClBitCodeAbbrev *Abbrev,
-                       int &Index) {
-    // Get block abbreviations.
-    BlockAbbrevs* Abbrevs = getGlobalAbbrevs(BlockID);
-
-    // Read abbreviation and add as a global abbreviation.
-    if (Abbrevs->addAbbreviation(Abbrev, Index)
-        && Context->Flags.TraceGeneratedAbbreviations) {
-      printAbbrev(errs(), BlockID, Abbrev);
-    }
-  }
-
-  /// Finds the index to the corresponding internal block abbreviation
-  /// for the given abbreviation.
-  int findAbbreviation(unsigned BlockID, const NaClBitCodeAbbrev *Abbrev) {
-    return getGlobalAbbrevs(BlockID)->findAbbreviation(Abbrev);
-  }
-
-  void init() {
-    GlobalBlockAbbrevs = getGlobalAbbrevs(GetBlockID());
-    LocalAbbrevBitstreamToInternalMap.setNextBitstreamAbbrevIndex(
-        GlobalBlockAbbrevs->
-        getGlobalAbbrevBitstreamToInternalMap().getNextBitstreamAbbrevIndex());
-  }
-};
-
-bool NaClAnalyzeParser::ParseBlock(unsigned BlockID) {
-  NaClBlockAnalyzeParser Parser(BlockID, this);
-  return Parser.ParseThisBlock();
-}
-
-/// Models the unrolling of an abbreviation into its sequence of
-/// individual operators. That is, unrolling arrays to match the width
-/// of the abbreviation.
-///
-/// For example, consider the abbreviation [Array(VBR(6))]. If the
-/// distribution map has data for records of size 3, and the
-/// distribution map suggests that a constant 4 appears as the second
-/// element in the record, it is nontrivial to figure out how to
-/// encorporate this into this abbrevation. Hence, we unroll the array
-/// (3 times) to get [VBR(6), VBR(6), VBR(6), Array(VBR(6))]. To
-/// update the second element to match the literal 4, we only need to
-/// replace the second element in the unrolled abbreviation resulting
-/// in [VBR(6), Lit(4), VBR(6), Array(VBR(6))].
-///
-/// After we have done appropriate substitutions, we can simplify the
-/// unrolled abbreviation by calling method Restore.
-///
-/// Note: We unroll in the form that best matches the distribution
-/// map. Hence, the code is stored as a separate operator. We also
-/// keep the array abbreviation op, for untracked elements within the
-/// distribution maps.
-class UnrolledAbbreviation {
-  void operator=(const UnrolledAbbreviation&) = delete;
-public:
-  /// Unroll the given abbreviation, assuming it has the given size
-  /// (as specified in the distribution maps).
-  ///
-  /// If argument CanBeBigger is true, then we do not assume that we
-  /// can remove the trailing array when expanding, because the
-  /// actual size of the corresponding record using this abbreviation
-  /// may be bigger.
-  UnrolledAbbreviation(NaClBitCodeAbbrev *Abbrev, unsigned Size,
-                       bool CanBeBigger = false)
-      : CodeOp(0) {
-    unsigned NextOp = 0;
-    unrollAbbrevOp(CodeOp, Abbrev, NextOp);
-    --Size;
-    for (unsigned i = 0; i < Size; ++i) {
-      // Create slot and then fill with appropriate operator.
-      AbbrevOps.push_back(CodeOp);
-      unrollAbbrevOp(AbbrevOps[i], Abbrev, NextOp);
-    }
-    if (CanBeBigger) {
-      for (; NextOp < Abbrev->getNumOperandInfos(); ++NextOp) {
-        MoreOps.push_back(Abbrev->getOperandInfo(NextOp));
-      }
-    } else if (NextOp < Abbrev->getNumOperandInfos() &&
-               !Abbrev->getOperandInfo(NextOp).isArrayOp()) {
-      errs() << (Size+1) << ": ";
-      Abbrev->Print(errs());
-      llvm_unreachable("Malformed abbreviation/size pair");
-    }
-  }
-
-  explicit UnrolledAbbreviation(const UnrolledAbbreviation &Abbrev)
-      : CodeOp(Abbrev.CodeOp),
-        AbbrevOps(Abbrev.AbbrevOps),
-        MoreOps(Abbrev.MoreOps) {
-  }
-
-  /// Prints out the abbreviation modeled by the unrolled
-  /// abbreviation.
-  void print(raw_ostream &Stream) const {
-    NaClBitCodeAbbrev *Abbrev = restore(false);
-    Abbrev->Print(Stream);
-    Abbrev->dropRef();
-  }
-
-  /// Converts the unrolled abbreviation back into a regular
-  /// abbreviation. If Simplify is true, we simplify the
-  /// unrolled abbreviation as well.
-  NaClBitCodeAbbrev *restore(bool Simplify=true) const {
-    NaClBitCodeAbbrev *Abbrev = new NaClBitCodeAbbrev();
-    Abbrev->Add(CodeOp);
-    for (std::vector<NaClBitCodeAbbrevOp>::const_iterator
-             Iter = AbbrevOps.begin(), IterEnd = AbbrevOps.end();
-         Iter != IterEnd; ++Iter) {
-      Abbrev->Add(*Iter);
-    }
-    for (std::vector<NaClBitCodeAbbrevOp>::const_iterator
-             Iter = MoreOps.begin(), IterEnd = MoreOps.end();
-         Iter != IterEnd; ++Iter) {
-      Abbrev->Add(*Iter);
-    }
-    if (Simplify) {
-      NaClBitCodeAbbrev *SimpAbbrev = Abbrev->Simplify();
-      Abbrev->dropRef();
-      return SimpAbbrev;
-    } else {
-      return Abbrev;
-    }
-  }
-
-  // The abbreviation used for the record code.
-  NaClBitCodeAbbrevOp CodeOp;
-
-  // The abbreviations used for each tracked value index.
-  std::vector<NaClBitCodeAbbrevOp> AbbrevOps;
-
-private:
-  // Any remaining abbreviation operators not part of the unrolling.
-  std::vector<NaClBitCodeAbbrevOp> MoreOps;
-
-  // Extracts out the next abbreviation operator from the abbreviation
-  // Abbrev, given the index NextOp, and assigns it to AbbrevOp
-  void unrollAbbrevOp(NaClBitCodeAbbrevOp &AbbrevOp,
-                      NaClBitCodeAbbrev *Abbrev,
-                      unsigned &NextOp) {
-    assert(NextOp < Abbrev->getNumOperandInfos());
-    const NaClBitCodeAbbrevOp &Op = Abbrev->getOperandInfo(NextOp);
-    if (Op.isArrayOp()) {
-      // Do not advance. The array operator assumes that all remaining
-      // elements should match its argument.
-      AbbrevOp = Abbrev->getOperandInfo(NextOp+1);
-    } else {
-      AbbrevOp = Op;
-      NextOp++;
-    }
-  }
-};
-
-/// Models a candidate block abbreviation, which is a blockID, and the
-/// corresponding abbreviation to be considered for addition.  Note:
-/// Constructors and assignment take ownership of the abbreviation.
-class CandBlockAbbrev {
-public:
-  CandBlockAbbrev(unsigned BlockID, NaClBitCodeAbbrev *Abbrev)
-      : BlockID(BlockID), Abbrev(Abbrev) {
-  }
-
-  CandBlockAbbrev(const CandBlockAbbrev &BlockAbbrev)
-      : BlockID(BlockAbbrev.BlockID),
-        Abbrev(BlockAbbrev.Abbrev) {
-    Abbrev->addRef();
-  }
-
-  void operator=(const CandBlockAbbrev &BlockAbbrev) {
-    Abbrev->dropRef();
-    BlockID = BlockAbbrev.BlockID;
-    Abbrev = BlockAbbrev.Abbrev;
-    Abbrev->addRef();
-  }
-
-  ~CandBlockAbbrev() {
-    Abbrev->dropRef();
-  }
-
-  /// The block ID of the candidate abbreviation.
-  unsigned getBlockID() const {
-    return BlockID;
-  }
-
-  /// The abbreviation of the candidate abbreviation.
-  const NaClBitCodeAbbrev *getAbbrev() const {
-    return Abbrev;
-  }
-
-  /// orders this against the candidate abbreviation.
-  int compare(const CandBlockAbbrev &CandAbbrev) const {
-    unsigned diff = BlockID - CandAbbrev.BlockID;
-    if (diff) return diff;
-    return Abbrev->Compare(*CandAbbrev.Abbrev);
-  }
-
-  /// Prints the candidate abbreviation to the given stream.
-  void print(raw_ostream &Stream) const {
-    printAbbrev(Stream, BlockID, Abbrev);
-  }
-
-private:
-  // The block the abbreviation applies to.
-  unsigned BlockID;
-  // The candidate abbreviation.
-  NaClBitCodeAbbrev *Abbrev;
-};
-
-static inline bool operator<(const CandBlockAbbrev &A1,
-                             const CandBlockAbbrev &A2) {
-  return A1.compare(A2) < 0;
-}
-
-/// Models the set of candidate abbreviations being considered, and
-/// the number of abbreviations associated with each candidate
-/// Abbreviation.
-///
-/// Note: Because we may have abbreviation refinements of A->B->C and
-/// A->D->C, we need to accumulate instance counts in such cases.
-class CandidateAbbrevs {
-public:
-  // Map from candidate abbreviations to the corresponding number of
-  // instances.
-  typedef std::map<CandBlockAbbrev, unsigned> AbbrevCountMap;
-  typedef AbbrevCountMap::const_iterator const_iterator;
-
-  /// Creates an empty set of candidate abbreviations, to be
-  /// (potentially) added to the given set of block abbreviations.
-  /// Argument is the (global) block abbreviations map, which is
-  /// used to determine if it already exists.
-  CandidateAbbrevs(BlockAbbrevsMapType &BlockAbbrevsMap)
-      : BlockAbbrevsMap(BlockAbbrevsMap)
-  {}
-
-  /// Adds the given (unrolled) abbreviation as a candidate
-  /// abbreviation to the given block. NumInstances is the number of
-  /// instances expected to use this candidate abbreviation. Returns
-  /// true if the corresponding candidate abbreviation is added to this
-  /// set of candidate abbreviations.
-  bool add(unsigned BlockID,
-           UnrolledAbbreviation &UnrolledAbbrev,
-           unsigned NumInstances);
-
-  /// Returns the list of candidate abbreviations in this set.
-  const AbbrevCountMap &getAbbrevsMap() const {
-    return AbbrevsMap;
-  }
-
-  /// Prints out the current contents of this set.
-  void print(raw_ostream &Stream, const char *Title = "Candidates") const {
-    Stream << "-- " << Title << ": \n";
-    for (const_iterator Iter = AbbrevsMap.begin(), IterEnd = AbbrevsMap.end();
-         Iter != IterEnd; ++Iter) {
-      Stream << format("%12u", Iter->second) << ": ";
-      Iter->first.print(Stream);
-    }
-    Stream << "--\n";
-  }
-
-private:
-  // The set of abbreviations and corresponding number instances.
-  AbbrevCountMap AbbrevsMap;
-
-  // The map of (global) abbreviations already associated with each block.
-  BlockAbbrevsMapType &BlockAbbrevsMap;
-};
-
-bool CandidateAbbrevs::add(unsigned BlockID,
-                           UnrolledAbbreviation &UnrolledAbbrev,
-                           unsigned NumInstances) {
-  // Drop if it corresponds to an existing global abbreviation.
-  NaClBitCodeAbbrev *Abbrev = UnrolledAbbrev.restore();
-  if (BlockAbbrevs* Abbrevs = BlockAbbrevsMap[BlockID]) {
-    if (Abbrevs->findAbbreviation(Abbrev) !=
-        BlockAbbrevs::NO_SUCH_ABBREVIATION) {
-      Abbrev->dropRef();
-      return false;
-    }
-  }
-
-  CandBlockAbbrev CandAbbrev(BlockID, Abbrev);
-  AbbrevCountMap::iterator Pos = AbbrevsMap.find(CandAbbrev);
-  if (Pos == AbbrevsMap.end()) {
-    AbbrevsMap[CandAbbrev] = NumInstances;
-  } else {
-    Pos->second += NumInstances;
-  }
-  return true;
-}
-
-// Look for new abbreviations in block BlockID, considering it was
-// read with the given abbreviation Abbrev, and considering changing
-// the abbreviation opererator for value Index. ValueDist is how
-// values at Index are distributed. Any found abbreviations are added
-// to the candidate abbreviations CandAbbrevs. Returns true only if we
-// have added new candidate abbreviations to CandAbbrevs.
-static bool addNewAbbreviations(unsigned BlockID,
-                                const UnrolledAbbreviation &Abbrev,
-                                unsigned Index,
-                                NaClBitcodeValueDist &ValueDist,
-                                CandidateAbbrevs &CandAbbrevs) {
-  // TODO(kschimpf): Add code to try and find a better encoding for
-  // the values, based on the distribution.
-
-  // If this index is already a literal abbreviation, no improvements can
-  // be made.
-  const NaClBitCodeAbbrevOp Op = Abbrev.AbbrevOps[Index];
-  if (Op.isLiteral()) return false;
-
-  // Search based on sorted distribution, which sorts by number of
-  // instances.  Start by trying to find possible constants to use.
-  const NaClBitcodeDist::Distribution *
-      Distribution = ValueDist.GetDistribution();
-  for (NaClBitcodeDist::Distribution::const_iterator
-           Iter = Distribution->begin(), IterEnd = Distribution->end();
-       Iter != IterEnd; ++Iter) {
-    NaClValueRangeType Range = GetNaClValueRange(Iter->second);
-    if (Range.first != Range.second) continue;  // not a constant.
-
-    // Defines a constant. Try as new candidate range.  In addition,
-    // don't try any more constant values, since this is the one with
-    // the greatest number of instances.
-    NaClBitcodeDistElement *Elmt = ValueDist.at(Range.first);
-    UnrolledAbbreviation CandAbbrev(Abbrev);
-    CandAbbrev.AbbrevOps[Index] = NaClBitCodeAbbrevOp(Range.first);
-    return CandAbbrevs.add(BlockID, CandAbbrev, Elmt->GetNumInstances());
-  }
-  return false;
-}
-
-// Look for new abbreviations in block BlockID, considering it was
-// read with the given abbreviation Abbrev. IndexDist is the
-// corresponding distribution of value indices associated with the
-// abbreviation.  Any found abbreviations are added to the candidate
-// abbreviations CandAbbrevs.
-static void addNewAbbreviations(
-    const NaClBitcodeCompressor::CompressFlags &Flags,
-    unsigned BlockID, const UnrolledAbbreviation &Abbrev,
-    NaClBitcodeDist &IndexDist, CandidateAbbrevs &CandAbbrevs) {
-  // Search based on sorted distribution, which sorts based on heuristic
-  // of best index to fix first.
-  const NaClBitcodeDist::Distribution *
-      IndexDistribution = IndexDist.GetDistribution();
-  for (NaClBitcodeDist::Distribution::const_iterator
-           IndexIter = IndexDistribution->begin(),
-           IndexIterEnd = IndexDistribution->end();
-       IndexIter != IndexIterEnd; ++IndexIter) {
-    unsigned Index = static_cast<unsigned>(IndexIter->second);
-    if (addNewAbbreviations(
-            BlockID, Abbrev, Index,
-            cast<NaClBitcodeValueIndexDistElement>(IndexDist.at(Index))
-            ->GetValueDist(),
-            CandAbbrevs)) {
-      return;
-    }
-  }
-}
-
-// Look for new abbreviations in block BlockID, considering it was
-// read with the given abbreviation Abbrev, and the given record Code.
-// SizeDist is the corresponding distribution of sizes associated with
-// the abbreviation. Any found abbreviations are added to the
-// candidate abbreviations CandAbbrevs.
-static void addNewAbbreviations(
-    const NaClBitcodeCompressor::CompressFlags &Flags,
-    unsigned BlockID, NaClBitCodeAbbrev *Abbrev, unsigned Code,
-    NaClBitcodeDist &SizeDist, CandidateAbbrevs &CandAbbrevs) {
-  const NaClBitcodeDist::Distribution *
-      SizeDistribution = SizeDist.GetDistribution();
-  for (NaClBitcodeDist::Distribution::const_iterator
-           SizeIter = SizeDistribution->begin(),
-           SizeIterEnd = SizeDistribution->end();
-       SizeIter != SizeIterEnd; ++SizeIter) {
-    unsigned Size = static_cast<unsigned>(SizeIter->second);
-    UnrolledAbbreviation UnrolledAbbrev(Abbrev, Size+1 /* Add code! */,
-                                        Size >= NaClValueIndexCutoff);
-    if (!UnrolledAbbrev.CodeOp.isLiteral()) {
-      // Try making the code a literal.
-      UnrolledAbbreviation CandAbbrev(UnrolledAbbrev);
-      CandAbbrev.CodeOp = NaClBitCodeAbbrevOp(Code);
-      CandAbbrevs.add(BlockID, CandAbbrev,
-                      SizeDist.at(Size)->GetNumInstances());
-    }
-    // Now process value indices to find candidate abbreviations.
-    addNewAbbreviations(Flags, BlockID, UnrolledAbbrev,
-        cast<NaClBitcodeSizeDistElement>(SizeDist.at(Size))
-        ->GetValueIndexDist(),
-        CandAbbrevs);
-  }
-}
-
-// Look for new abbreviations in block BlockID. Abbrevs is the map of
-// read (globally defined) abbreviations associated with the
-// BlockID. AbbrevDist is the distribution map of abbreviations
-// associated with BlockID. Any found abbreviations are added to the
-// candidate abbreviations CandAbbrevs.
-static void addNewAbbreviations(
-    const NaClBitcodeCompressor::CompressFlags &Flags, unsigned BlockID,
-    BlockAbbrevs *Abbrevs, NaClBitcodeDist &AbbrevDist,
-    CandidateAbbrevs &CandAbbrevs) {
-  const NaClBitcodeDist::Distribution *
-      AbbrevDistribution = AbbrevDist.GetDistribution();
-  for (NaClBitcodeDist::Distribution::const_iterator
-           AbbrevIter = AbbrevDistribution->begin(),
-           AbbrevIterEnd = AbbrevDistribution->end();
-       AbbrevIter != AbbrevIterEnd; ++AbbrevIter) {
-    NaClBitcodeDistValue AbbrevIndex = AbbrevIter->second;
-    NaClBitCodeAbbrev *Abbrev = Abbrevs->getIndexedAbbrev(AbbrevIndex);
-    NaClBitcodeAbbrevDistElement *AbbrevElmt =
-        cast<NaClBitcodeAbbrevDistElement>(AbbrevDist.at(AbbrevIndex));
-    NaClBitcodeDist &CodeDist = AbbrevElmt->GetCodeDist();
-
-    const NaClBitcodeDist::Distribution *
-        CodeDistribution = CodeDist.GetDistribution();
-    for (NaClBitcodeDist::Distribution::const_iterator
-             CodeIter = CodeDistribution->begin(),
-             CodeIterEnd = CodeDistribution->end();
-         CodeIter != CodeIterEnd; ++CodeIter) {
-      unsigned Code = static_cast<unsigned>(CodeIter->second);
-      addNewAbbreviations(
-          Flags, BlockID, Abbrev, Code,
-          cast<NaClCompressCodeDistElement>(CodeDist.at(CodeIter->second))
-          ->GetSizeDist(),
-          CandAbbrevs);
-    }
-  }
-}
-
-typedef std::pair<unsigned, CandBlockAbbrev> CountedAbbrevType;
-
-// Look for new abbreviations in the given block distribution map
-// BlockDist.  BlockAbbrevsMap contains the set of read global
-// abbreviations. Adds found candidate abbreviations to the set of
-// known global abbreviations.
-static void addNewAbbreviations(
-    const NaClBitcodeCompressor::CompressFlags &Flags,
-    NaClBitcodeBlockDist &BlockDist, BlockAbbrevsMapType &BlockAbbrevsMap) {
-  CandidateAbbrevs CandAbbrevs(BlockAbbrevsMap);
-  // Start by collecting candidate abbreviations.
-  const NaClBitcodeDist::Distribution *
-      Distribution = BlockDist.GetDistribution();
-  for (NaClBitcodeDist::Distribution::const_iterator
-           Iter = Distribution->begin(), IterEnd = Distribution->end();
-       Iter != IterEnd; ++Iter) {
-    unsigned BlockID = static_cast<unsigned>(Iter->second);
-    addNewAbbreviations(
-        Flags, BlockID, BlockAbbrevsMap[BlockID],
-        cast<NaClCompressBlockDistElement>(BlockDist.at(BlockID))
-        ->GetAbbrevDist(),
-        CandAbbrevs);
-  }
-  // Install candidate abbreviations.
-  //
-  // Sort the candidate abbreviations by number of instances, so that
-  // if multiple abbreviations apply, the one with the largest number
-  // of instances will be chosen when compressing a file.
-  //
-  // For example, we may have just generated two abbreviations. The
-  // first has replaced the 3rd element with the constant 4 while the
-  // second replaced the 4th element with the constant 5. The first
-  // abbreviation can apply to 300 records while the second can apply
-  // to 1000 records.  Assuming that both abbreviations shrink the
-  // record by the same number of bits, we clearly want the tool to
-  // choose the second abbreviation when selecting the abbreviation
-  // index to choose (via method getRecordAbbrevIndex).
-  //
-  // Selecting the second is important in that abbreviation are
-  // refined by successive calls to this tool. We do not want to
-  // restrict downstream refinements prematurely.  Hence, we want the
-  // tool to choose the abbreviation with the most candidates first.
-  //
-  // Since method getRecordAbbrevIndex chooses the first abbreviation
-  // that generates the least number of bits, we clearly want to make
-  // sure that the second abbreviation occurs before the first.
-  std::vector<CountedAbbrevType> Candidates;
-  for (CandidateAbbrevs::const_iterator
-           Iter = CandAbbrevs.getAbbrevsMap().begin(),
-           IterEnd = CandAbbrevs.getAbbrevsMap().end();
-       Iter != IterEnd; ++Iter) {
-    Candidates.push_back(CountedAbbrevType(Iter->second,Iter->first));
-  }
-  std::sort(Candidates.begin(), Candidates.end());
-  std::vector<CountedAbbrevType>::const_reverse_iterator
-      Iter = Candidates.rbegin(), IterEnd = Candidates.rend();
-  if (Iter == IterEnd) return;
-
-  if (Flags.TraceGeneratedAbbreviations) {
-    errs() << "-- New abbrevations:\n";
-  }
-  unsigned Min = (Iter->first >> 2);
-  for (; Iter != IterEnd; ++Iter) {
-    if (Iter->first < Min) break;
-    unsigned BlockID = Iter->second.getBlockID();
-    BlockAbbrevs *Abbrevs = BlockAbbrevsMap[BlockID];
-    NaClBitCodeAbbrev *Abbrev = Iter->second.getAbbrev()->Copy();
-    if (Flags.TraceGeneratedAbbreviations) {
-      errs() <<format("%12u", Iter->first) << ": ";
-      printAbbrev(errs(), BlockID, Abbrev);
-    }
-    Abbrevs->addAbbreviation(Abbrev);
-  }
-  if (Flags.TraceGeneratedAbbreviations) {
-    errs() << "--\n";
-  }
-}
-
-// Walks the block distribution (BlockDist), sorting entries based
-// on the distribution of blocks and abbreviations, and then
-// prints out the frequency of each abbreviation used.
-static void displayAbbreviationFrequencies(
-    raw_ostream &Output,
-    const NaClBitcodeBlockDist &BlockDist,
-    const BlockAbbrevsMapType &BlockAbbrevsMap) {
-  const NaClBitcodeDist::Distribution *BlockDistribution =
-      BlockDist.GetDistribution();
-  for (NaClBitcodeDist::Distribution::const_iterator
-           BlockIter = BlockDistribution->begin(),
-           BlockIterEnd = BlockDistribution->end();
-       BlockIter != BlockIterEnd; ++BlockIter) {
-    unsigned BlockID = static_cast<unsigned>(BlockIter->second);
-    BlockAbbrevsMapType::const_iterator BlockPos = BlockAbbrevsMap.find(BlockID);
-    if (BlockPos == BlockAbbrevsMap.end()) continue;
-    Output << "Block " << BlockID << "\n";
-    if (NaClCompressBlockDistElement *BlockElement =
-        dyn_cast<NaClCompressBlockDistElement>(BlockDist.at(BlockID))) {
-      NaClBitcodeDist &AbbrevDist = BlockElement->GetAbbrevDist();
-      const NaClBitcodeDist::Distribution *AbbrevDistribution =
-          AbbrevDist.GetDistribution();
-      unsigned Total = AbbrevDist.GetTotal();
-      for (NaClBitcodeDist::Distribution::const_iterator
-               AbbrevIter = AbbrevDistribution->begin(),
-               AbbrevIterEnd = AbbrevDistribution->end();
-           AbbrevIter != AbbrevIterEnd; ++AbbrevIter) {
-        unsigned Index = static_cast<unsigned>(AbbrevIter->second);
-        unsigned Count = AbbrevDist.at(Index)->GetNumInstances();
-        Output << format("%8u (%6.2f%%): ", Count,
-                         (double) Count/Total*100.0);
-        BlockPos->second->getIndexedAbbrev(Index)->Print(Output);
-      }
-    }
-    Output << '\n';
-  }
-}
-
-// Read in bitcode, analyze data, and figure out set of abbreviations
-// to use, from memory buffer MemBuf containing the input bitcode file.
-// If ShowAbbreviationFrequencies or Flag.ShowValueDistributions are
-// defined, the corresponding results is printed to Output.
-static bool analyzeBitcode(
-    const NaClBitcodeCompressor::CompressFlags &Flags,
-    MemoryBuffer *MemBuf,
-    raw_ostream &Output,
-    BlockAbbrevsMapType &BlockAbbrevsMap) {
-  // TODO(kschimpf): The current code only extracts abbreviations
-  // defined in the bitcode file. This code needs to be updated to
-  // collect data distributions and figure out better (global)
-  // abbreviations to use.
-
-  const unsigned char *BufPtr = (const unsigned char *)MemBuf->getBufferStart();
-  const unsigned char *EndBufPtr = BufPtr+MemBuf->getBufferSize();
-
-  // First read header and verify it is good.
-  NaClBitcodeHeader Header;
-  if (Header.Read(BufPtr, EndBufPtr))
-    return Error("Invalid PNaCl bitcode header");
-  if (!Header.IsSupported()) {
-    errs() << Header.Unsupported();
-    if (!Header.IsReadable())
-      return Error("Invalid PNaCl bitcode header");
-  }
-
-  // Create a bitstream reader to read the bitcode file.
-  NaClBitstreamReader StreamFile(BufPtr, EndBufPtr, Header);
-  NaClBitstreamCursor Stream(StreamFile);
-
-  // Parse the the bitcode file.
-  NaClAnalyzeParser Parser(Flags, Stream, BlockAbbrevsMap);
-  while (!Stream.AtEndOfStream()) {
-    if (Parser.Parse()) return true;
-  }
-
-  if (Flags.ShowAbbreviationFrequencies)
-    displayAbbreviationFrequencies(Output, Parser.BlockDist, BlockAbbrevsMap);
-  if (Flags.ShowValueDistributions)
-    Parser.BlockDist.Print(Output);
-
-  addNewAbbreviations(Flags, Parser.BlockDist, BlockAbbrevsMap);
-  return false;
-}
-
-/// Models a queue of selected abbreviation indices, for each record
-/// in all instances of a given block. Elements are added in the order
-/// they appear in the bitcode file.
-///
-/// The goal is to remove abbreviations that are not really used, from
-/// the list of candidate abbrevations, reducing the number of
-/// abbreviations needed. This is important because as the number of
-/// abbreviations increase, the number of bits needed to store the
-/// abbreviations also increase. By removing unnecessary
-/// abbreviations, this improves the ability of this executable to
-/// compress the bitcode file.
-class SelectedAbbrevsQueue {
-  SelectedAbbrevsQueue(const SelectedAbbrevsQueue &) = delete;
-  SelectedAbbrevsQueue &operator=(const SelectedAbbrevsQueue &) = delete;
-
-  // The minimum number of times an abbreviation must be used in the
-  // compressed version of the bitcode file, if it is going to be used
-  // at all.
-  static const uint32_t MinUsageCount = 5;
-
-public:
-  SelectedAbbrevsQueue() : AbbrevsIndexQueueFront(0) {}
-
-  /// Adds the given selected abbreviation index to the end of the
-  /// queue.
-  void addIndex(unsigned Index) { AbbrevsIndexQueue.push_back(Index); }
-
-  /// Removes the next selected abbreviation index from the
-  /// queue.
-  unsigned removeIndex() {
-    assert(AbbrevsIndexQueueFront < AbbrevsIndexQueue.size());
-    return AbbrevsIndexQueue[AbbrevsIndexQueueFront++];
-  }
-
-  /// Takes the abbreviation indices on the queue, determines what
-  /// subset of abbreviations should be kept, and puts them on the
-  /// list of abbreviations defined by getKeptAbbrevs. Updates the
-  /// abbreviation idices on the queue to match the corresponding kept
-  /// abbreviations.
-  ///
-  /// Should be called after the last call to AddIndex, and before
-  /// calling removeIndex.
-  void installFrequentlyUsedAbbrevs(BlockAbbrevs *Abbrevs);
-
-  /// Return the list of kept abbreviations, for the corresponding
-  /// block, in the order they should be defined.
-  const std::vector<NaClBitCodeAbbrev *> &getKeptAbbrevs() const {
-    return KeptAbbrevs;
-  }
-
-  /// Returns the maximum abbreviation index used by the kept
-  /// abbreviations.
-  unsigned getMaxKeptAbbrevIndex() const {
-    return KeptAbbrevs.size() + naclbitc::DEFAULT_MAX_ABBREV;
-  }
-
-protected:
-  // Defines a queue of abbreviations indices using a
-  // vector. AbbrevsIndexQueueFront is used to point to the front of
-  // the queue.
-  std::vector<unsigned> AbbrevsIndexQueue;
-  unsigned AbbrevsIndexQueueFront;
-  // The list of abbreviations that should be defined for the block,
-  // in the order they should be defined.
-  std::vector<NaClBitCodeAbbrev *> KeptAbbrevs;
-};
-
-void SelectedAbbrevsQueue::installFrequentlyUsedAbbrevs(BlockAbbrevs *Abbrevs) {
-  // Start by collecting usage counts for each abbreviation.
-  assert(AbbrevsIndexQueueFront == 0);
-  assert(KeptAbbrevs.empty());
-  std::map<unsigned, uint32_t> UsageMap;
-  for (unsigned Index : AbbrevsIndexQueue) {
-    if (Index != naclbitc::UNABBREV_RECORD)
-      ++UsageMap[Index];
-  }
-
-  // Install results
-  std::map<unsigned, unsigned> KeepIndexMap;
-  for (const std::pair<unsigned, uint32_t> &Pair : UsageMap) {
-    if (Pair.second >= MinUsageCount) {
-      KeepIndexMap[Pair.first] =
-          KeptAbbrevs.size() + naclbitc::FIRST_APPLICATION_ABBREV;
-      KeptAbbrevs.push_back(Abbrevs->getIndexedAbbrev(Pair.first));
-    }
-  }
-
-  // Update the queue of selected abbreviation indices to match kept
-  // abbreviations.
-  for (unsigned &AbbrevIndex : AbbrevsIndexQueue) {
-    std::map<unsigned, unsigned>::const_iterator NewPos =
-        KeepIndexMap.find(AbbrevIndex);
-    AbbrevIndex = NewPos == KeepIndexMap.end() ? naclbitc::UNABBREV_RECORD
-                                               : NewPos->second;
-  }
-}
-
-/// Models the kept queue of abbreviations associated with each block ID.
-typedef std::map<unsigned, SelectedAbbrevsQueue *> BlockAbbrevsQueueMap;
-typedef std::pair<unsigned, SelectedAbbrevsQueue *> BlockAbbrevsQueueMapElmt;
-
-/// Installs frequently used abbreviations for each of the blocks
-/// defined in AbbrevsQueueMap, based on the abbreviations in the
-/// corresponding AbbrevsMap
-static void
-installFrequentlyUsedAbbrevs(BlockAbbrevsMapType &AbbrevsMap,
-                             BlockAbbrevsQueueMap &AbbrevsQueueMap) {
-  for (const BlockAbbrevsQueueMapElmt &Pair : AbbrevsQueueMap) {
-    if (SelectedAbbrevsQueue *SelectedAbbrevs = Pair.second)
-      SelectedAbbrevs->installFrequentlyUsedAbbrevs(AbbrevsMap[Pair.first]);
-  }
-}
-
-/// Top level parser to queue selected abbreviation indices (within
-/// the bitcode file), so that we can remove unused abbreviations from
-/// the collected list of abbreviations before generating the final,
-/// compressed bitcode file.
-class NaClAssignAbbrevsParser : public NaClBitcodeParser {
-public:
-  NaClAssignAbbrevsParser(NaClBitstreamCursor &Cursor,
-                          BlockAbbrevsMapType &AbbrevsMap,
-                          BlockAbbrevsQueueMap &AbbrevsQueueMap)
-      : NaClBitcodeParser(Cursor), AbbrevsMap(AbbrevsMap),
-        AbbrevsQueueMap(AbbrevsQueueMap) {}
-
-  ~NaClAssignAbbrevsParser() final {}
-
-  bool ParseBlock(unsigned BlockID) final;
-
-  /// The abbreviations to use for the compressed bitcode.
-  BlockAbbrevsMapType &AbbrevsMap;
-
-  /// The corresponding selected abbreviation indices to for each
-  /// block.
-  BlockAbbrevsQueueMap &AbbrevsQueueMap;
-};
-
-/// Block parser to queue abbreviation indices used by the records in
-/// the block.
-class NaClAssignAbbrevsBlockParser : public NaClBitcodeParser {
-public:
-  NaClAssignAbbrevsBlockParser(unsigned BlockID,
-                               NaClAssignAbbrevsParser *Context)
-      : NaClBitcodeParser(BlockID, Context), BlockID(BlockID),
-        Context(Context) {
-    init();
-  }
-
-  ~NaClAssignAbbrevsBlockParser() final {}
-
-protected:
-  unsigned BlockID;
-  NaClAssignAbbrevsParser *Context;
-  // Cached abbreviations defined for this block,.
-  BlockAbbrevs *Abbreviations;
-  // Cached abbreviations queue to add abbreviation indices to.
-  SelectedAbbrevsQueue *Queue;
-
-  NaClAssignAbbrevsBlockParser(unsigned BlockID,
-                               NaClAssignAbbrevsBlockParser *EnclosingParser)
-      : NaClBitcodeParser(BlockID, EnclosingParser), BlockID(BlockID),
-        Context(EnclosingParser->Context) {
-    init();
-  }
-
-  void init() {
-    Abbreviations = getAbbrevs(Context->AbbrevsMap, BlockID);
-    Queue = Context->AbbrevsQueueMap[BlockID];
-    if (Queue == nullptr) {
-      Queue = new SelectedAbbrevsQueue();
-      Context->AbbrevsQueueMap[BlockID] = Queue;
-    }
-  }
-
-  bool ParseBlock(unsigned BlockID) final {
-    NaClAssignAbbrevsBlockParser Parser(BlockID, this);
-    return Parser.ParseThisBlock();
-  }
-
-  void ProcessRecord() final {
-    // Find best fitting abbreviation to use, and save.
-    Queue->addIndex(
-        Abbreviations->getRecordAbbrevIndex(Record.GetRecordData()));
-  }
-};
-
-bool NaClAssignAbbrevsParser::ParseBlock(unsigned BlockID) {
-  NaClAssignAbbrevsBlockParser Parser(BlockID, this);
-  return Parser.ParseThisBlock();
-}
-
-// Reads the bitcode in MemBuf, using the abbreviations in AbbrevsMap,
-// and queues the selected abbrevations for each record into
-// AbbrevsQueueMap.
-static bool chooseAbbrevs(MemoryBuffer *MemBuf, BlockAbbrevsMapType &AbbrevsMap,
-                          BlockAbbrevsQueueMap &AbbrevsQueueMap) {
-  const unsigned char *BufPtr = (const unsigned char *)MemBuf->getBufferStart();
-  const unsigned char *EndBufPtr = BufPtr + MemBuf->getBufferSize();
-
-  // Read header. No verification is needed since AnalyzeBitcode has
-  // already checked it.
-  NaClBitcodeHeader Header;
-  if (Header.Read(BufPtr, EndBufPtr))
-    return Error("Invalid PNaCl bitcode header");
-
-  // Create the bitcode reader.
-  NaClBitstreamReader StreamFile(BufPtr, EndBufPtr, Header);
-  NaClBitstreamCursor Stream(StreamFile);
-
-  // Set up the parser.
-  NaClAssignAbbrevsParser Parser(Stream, AbbrevsMap, AbbrevsQueueMap);
-
-  // Parse the bitcode and choose abbreviations for records.
-  while (!Stream.AtEndOfStream()) {
-    if (Parser.Parse()) {
-      installFrequentlyUsedAbbrevs(AbbrevsMap, AbbrevsQueueMap);
-      return true;
-    }
-  }
-  installFrequentlyUsedAbbrevs(AbbrevsMap, AbbrevsQueueMap);
-  return false;
-}
-
-/// Parses the input bitcode file and generates the corresponding
-/// compressed bitcode file, by replacing abbreviations in the input
-/// file with the corresponding abbreviations defined in
-/// BlockAbbrevsMap using the selected abbreviations in AbbrevsQueueMap.
-class NaClBitcodeCopyParser : public NaClBitcodeParser {
-public:
-  // Top-level constructor to build the appropriate block parser
-  // using the given BlockAbbrevsMap to define abbreviations.
-  NaClBitcodeCopyParser(const NaClBitcodeCompressor::CompressFlags &Flags,
-                        NaClBitstreamCursor &Cursor,
-                        BlockAbbrevsMapType &BlockAbbrevsMap,
-                        BlockAbbrevsQueueMap &AbbrevsQueueMap,
-                        NaClBitstreamWriter &Writer)
-      : NaClBitcodeParser(Cursor), Flags(Flags),
-        BlockAbbrevsMap(BlockAbbrevsMap), AbbrevsQueueMap(AbbrevsQueueMap),
-        Writer(Writer) {}
-
-  virtual ~NaClBitcodeCopyParser() {}
-
-  bool ParseBlock(unsigned BlockID) final;
-
-  const NaClBitcodeCompressor::CompressFlags &Flags;
-
-  // The abbreviations to use for the copied bitcode.
-  BlockAbbrevsMapType &BlockAbbrevsMap;
-
-  // Map defining the selected abbreviations for each block.
-  BlockAbbrevsQueueMap &AbbrevsQueueMap;
-
-  // The bitstream to copy the compressed bitcode into.
-  NaClBitstreamWriter &Writer;
-};
-
-class NaClBlockCopyParser : public NaClBitcodeParser {
-public:
-  // Top-level constructor to build the appropriate block parser.
-  NaClBlockCopyParser(unsigned BlockID, NaClBitcodeCopyParser *Context)
-      : NaClBitcodeParser(BlockID, Context), Context(Context),
-        BlockAbbreviations(nullptr), SelectedAbbrevs(nullptr) {
-    init();
-  }
-
-  virtual ~NaClBlockCopyParser() {}
-
-protected:
-  // The context defining state associated with the block parser.
-  NaClBitcodeCopyParser *Context;
-
-  // The block abbreviations defined for this block (initialized by
-  // EnterBlock).
-  BlockAbbrevs *BlockAbbreviations;
-
-  // The corresonding selected abbreviations.
-  SelectedAbbrevsQueue *SelectedAbbrevs;
-
-  /// Constructor to parse nested blocks.  Creates a block parser to
-  /// parse in a block with the given BlockID, and write the block
-  /// back out using the abbreviations in BlockAbbrevsMap.
-  NaClBlockCopyParser(unsigned BlockID,
-                      NaClBlockCopyParser *EnclosingParser)
-      : NaClBitcodeParser(BlockID, EnclosingParser),
-        Context(EnclosingParser->Context), BlockAbbreviations(nullptr),
-        SelectedAbbrevs(nullptr) {
-    init();
-  }
-
-  void init() {
-    unsigned BlockID = GetBlockID();
-    BlockAbbreviations = getGlobalAbbrevs(BlockID);
-    SelectedAbbrevs = Context->AbbrevsQueueMap[BlockID];
-    assert(SelectedAbbrevs);
-  }
-
-  /// Returns the set of (global) block abbreviations defined for the
-  /// given block ID.
-  BlockAbbrevs *getGlobalAbbrevs(unsigned BlockID) {
-    return getAbbrevs(Context->BlockAbbrevsMap, BlockID);
-  }
-
-  virtual bool ParseBlock(unsigned BlockID) {
-    NaClBlockCopyParser Parser(BlockID, this);
-    return Parser.ParseThisBlock();
-  }
-
-  virtual void EnterBlock(unsigned NumWords) {
-    // Enter the subblock.
-    NaClBitcodeSelectorAbbrev Selector(
-        SelectedAbbrevs->getMaxKeptAbbrevIndex());
-    if (Context->Flags.RemoveAbbreviations)
-      Selector = NaClBitcodeSelectorAbbrev();
-
-    unsigned BlockID = GetBlockID();
-    Context->Writer.EnterSubblock(BlockID, Selector);
-
-    if (BlockID != naclbitc::MODULE_BLOCK_ID
-        || Context->Flags.RemoveAbbreviations)
-      return;
-
-    // To keep things simple, we dump all abbreviations immediately
-    // inside the module block. Start by dumping module abbreviations
-    // as local abbreviations.
-    for (auto Abbrev : SelectedAbbrevs->getKeptAbbrevs()) {
-      Context->Writer.EmitAbbrev(Abbrev->Copy());
-    }
-
-    // Insert the block info block, if needed, so that nested blocks
-    // will have defined abbreviations.
-    bool HasNonmoduleAbbrevs = false;
-    for (const BlockAbbrevsQueueMapElmt &Pair : Context->AbbrevsQueueMap) {
-      if (Pair.second->getKeptAbbrevs().empty())
-        continue;
-      HasNonmoduleAbbrevs = true;
-      break;
-    }
-    if (!HasNonmoduleAbbrevs)
-      return;
-
-    Context->Writer.EnterBlockInfoBlock();
-    for (const BlockAbbrevsMapElmtType &Pair : Context->BlockAbbrevsMap) {
-      unsigned BlockID = Pair.first;
-      // Don't emit module abbreviations, since they have been
-      // emitted as local abbreviations.
-      if (BlockID == naclbitc::MODULE_BLOCK_ID)
-        continue;
-      BlockAbbrevs *Abbrevs = Pair.second;
-      if (Abbrevs == nullptr)
-        continue;
-      if (SelectedAbbrevsQueue *SelectedAbbrevs =
-              Context->AbbrevsQueueMap[BlockID]) {
-        for (auto Abbrev : SelectedAbbrevs->getKeptAbbrevs()) {
-          Context->Writer.EmitBlockInfoAbbrev(BlockID, Abbrev->Copy());
-        }
-      }
-    }
-    Context->Writer.ExitBlock();
-  }
-
-  virtual void ExitBlock() {
-    Context->Writer.ExitBlock();
-  }
-
-  virtual void ProcessRecord() {
-    const NaClBitcodeRecord::RecordVector &Values = Record.GetValues();
-    if (Context->Flags.RemoveAbbreviations) {
-      Context->Writer.EmitRecord(Record.GetCode(), Values, 0);
-      return;
-    }
-    // Find best fitting abbreviation to use, and print out the record
-    // using that abbreviation.
-    unsigned AbbrevIndex = SelectedAbbrevs->removeIndex();
-    if (AbbrevIndex == naclbitc::UNABBREV_RECORD) {
-      Context->Writer.EmitRecord(Record.GetCode(), Values, 0);
-    } else {
-      Context->Writer.EmitRecord(Record.GetCode(), Values, AbbrevIndex);
-    }
-  }
-};
-
-bool NaClBitcodeCopyParser::ParseBlock(unsigned BlockID) {
-  NaClBlockCopyParser Parser(BlockID, this);
-  return Parser.ParseThisBlock();
-}
-
-// Read in bitcode, and write it back out using the abbreviations in
-// BlockAbbrevsMap, from memory buffer MemBuf containing the input
-// bitcode file. The bitcode is copied to Output.
-static bool copyBitcode(const NaClBitcodeCompressor::CompressFlags &Flags,
-                        MemoryBuffer *MemBuf, raw_ostream &Output,
-                        BlockAbbrevsMapType &BlockAbbrevsMap,
-                        BlockAbbrevsQueueMap &AbbrevsQueueMap) {
-
-  const unsigned char *BufPtr = (const unsigned char *)MemBuf->getBufferStart();
-  const unsigned char *EndBufPtr = BufPtr + MemBuf->getBufferSize();
-
-  // Read header. No verification is needed since AnalyzeBitcode has
-  // already checked it.
-  NaClBitcodeHeader Header;
-  if (Header.Read(BufPtr, EndBufPtr))
-    return Error("Invalid PNaCl bitcode header");
-
-  // Create the bitcode reader.
-  NaClBitstreamReader StreamFile(BufPtr, EndBufPtr, Header);
-  NaClBitstreamCursor Stream(StreamFile);
-
-  // Create the bitcode writer.
-  SmallVector<char, 0> OutputBuffer;
-  OutputBuffer.reserve(256 * 1024);
-  NaClBitstreamWriter StreamWriter(OutputBuffer);
-
-  // Emit the file header.
-  NaClWriteHeader(Header, StreamWriter);
-
-  // Set up the parser.
-  NaClBitcodeCopyParser Parser(Flags, Stream, BlockAbbrevsMap, AbbrevsQueueMap,
-                               StreamWriter);
-
-  // Parse the bitcode and copy.
-  while (!Stream.AtEndOfStream()) {
-    if (Parser.Parse())
-      return true;
-  }
-
-  // Write out the copied results.
-  Output.write((char *)&OutputBuffer.front(), OutputBuffer.size());
-  return false;
-}
-
-// Build fast lookup abbreviation maps for each of the abbreviation blocks
-// defined in AbbrevsMap.
-static void buildAbbrevLookupMaps(
-    const NaClBitcodeCompressor::CompressFlags &Flags,
-    BlockAbbrevsMapType &AbbrevsMap) {
-  for (BlockAbbrevsMapType::const_iterator
-           Iter = AbbrevsMap.begin(),
-           IterEnd = AbbrevsMap.end();
-       Iter != IterEnd; ++Iter) {
-    Iter->second->buildAbbrevLookupSizeMap(Flags);
-  }
-}
-
-}  // namespace
-
-bool NaClBitcodeCompressor::analyze(MemoryBuffer *MemBuf, raw_ostream &Output) {
-  BlockAbbrevsMapType BlockAbbrevsMap;
-  return !analyzeBitcode(Flags, MemBuf, Output, BlockAbbrevsMap);
-}
-
-bool NaClBitcodeCompressor::compress(MemoryBuffer *MemBuf,
-                                     raw_ostream &BitcodeOutput,
-                                     raw_ostream &ShowOutput) {
-  BlockAbbrevsMapType BlockAbbrevsMap;
-  if (analyzeBitcode(Flags, MemBuf, ShowOutput, BlockAbbrevsMap))
-    return false;
-  buildAbbrevLookupMaps(Flags, BlockAbbrevsMap);
-  BlockAbbrevsQueueMap AbbrevsQueueMap;
-  bool Result = true;
-  if (chooseAbbrevs(MemBuf, BlockAbbrevsMap, AbbrevsQueueMap))
-    Result = false;
-  else if (copyBitcode(Flags, MemBuf, BitcodeOutput, BlockAbbrevsMap,
-                       AbbrevsQueueMap))
-    Result = false;
-  DeleteContainerSeconds(AbbrevsQueueMap);
-  return Result;
-}
diff --git a/lib/Bitcode/NaCl/Analysis/NaClCompressBlockDist.cpp b/lib/Bitcode/NaCl/Analysis/NaClCompressBlockDist.cpp
deleted file mode 100644
index 32694feac7a..00000000000
--- a/lib/Bitcode/NaCl/Analysis/NaClCompressBlockDist.cpp
+++ /dev/null
@@ -1,28 +0,0 @@
-//===-- NaClCompressBlockDist.cpp --------------------------------------------===//
-//      Implements distribution maps used to collect block and record
-//      distributions for tool pnacl-bccompress.
-//
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
-//
-//===----------------------------------------------------------------------===//
-
-#include "llvm/Bitcode/NaCl/NaClCompressBlockDist.h"
-
-using namespace llvm;
-
-NaClCompressBlockDistElement::~NaClCompressBlockDistElement() {}
-
-NaClBitcodeDistElement* NaClCompressBlockDistElement::
-CreateElement(NaClBitcodeDistValue Value) const {
-  return new NaClCompressBlockDistElement(Value);
-}
-
-const SmallVectorImpl<NaClBitcodeDist*> *NaClCompressBlockDistElement::
-GetNestedDistributions() const {
-  return &NestedDists;
-}
-
-NaClCompressBlockDistElement NaClCompressBlockDistElement::Sentinel;
diff --git a/lib/Bitcode/NaCl/Analysis/NaClCompressCodeDist.cpp b/lib/Bitcode/NaCl/Analysis/NaClCompressCodeDist.cpp
deleted file mode 100644
index 72320388e91..00000000000
--- a/lib/Bitcode/NaCl/Analysis/NaClCompressCodeDist.cpp
+++ /dev/null
@@ -1,32 +0,0 @@
-//===-- NaClCompressCodeDist.cpp ------------------------------------------===//
-//      Implements distribution maps for record codes for pnacl-bccompress.
-//
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
-//
-//===----------------------------------------------------------------------===//
-
-#include "llvm/Bitcode/NaCl/NaClCompressCodeDist.h"
-
-using namespace llvm;
-
-NaClCompressCodeDistElement::~NaClCompressCodeDistElement() {}
-
-NaClBitcodeDistElement *NaClCompressCodeDistElement::CreateElement(
-    NaClBitcodeDistValue Value) const {
-  return new NaClCompressCodeDistElement();
-}
-
-void NaClCompressCodeDistElement::AddRecord(const NaClBitcodeRecord &Record) {
-  NaClBitcodeCodeDistElement::AddRecord(Record);
-  SizeDist.AddRecord(Record);
-}
-
-const SmallVectorImpl<NaClBitcodeDist*> *
-NaClCompressCodeDistElement::GetNestedDistributions() const {
-  return &NestedDists;
-}
-
-NaClCompressCodeDistElement NaClCompressCodeDistElement::Sentinel;
diff --git a/lib/Bitcode/NaCl/Analysis/NaClObjDump.cpp b/lib/Bitcode/NaCl/Analysis/NaClObjDump.cpp
deleted file mode 100644
index c348f01025b..00000000000
--- a/lib/Bitcode/NaCl/Analysis/NaClObjDump.cpp
+++ /dev/null
@@ -1,3593 +0,0 @@
-//===-- NaClObjDump.cpp - Dump PNaCl bitcode contents ---------------------===//
-//
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
-//
-//===----------------------------------------------------------------------===//
-
-#include "llvm/ADT/STLExtras.h"
-#include "llvm/Analysis/NaCl/PNaClABIProps.h"
-#include "llvm/Analysis/NaCl/PNaClABITypeChecker.h"
-#include "llvm/Analysis/NaCl/PNaClAllowedIntrinsics.h"
-#include "llvm/Bitcode/NaCl/NaClBitcodeDecoders.h"
-#include "llvm/Bitcode/NaCl/NaClBitcodeHeader.h"
-#include "llvm/Bitcode/NaCl/NaClBitcodeParser.h"
-#include "llvm/Bitcode/NaCl/NaClBitCodes.h"
-#include "llvm/Bitcode/NaCl/NaClObjDumpStream.h"
-#include "llvm/Bitcode/NaCl/NaClReaderWriter.h"
-#include "llvm/IR/Constants.h"
-#include "llvm/IR/DataLayout.h"
-#include "llvm/IR/InstrTypes.h"
-#include "llvm/IR/LLVMContext.h"
-#include "llvm/IR/Module.h"
-#include "llvm/Support/CommandLine.h"
-#include "llvm/Support/MemoryBuffer.h"
-#include "llvm/Support/Format.h"
-#include "llvm/Support/raw_ostream.h"
-
-#include <map>
-
-namespace {
-
-using namespace llvm;
-
-static cl::opt<bool>
-ReportWarningsAsErrors(
-    "Werror",
-    cl::desc("Report warnings as errors."),
-    cl::init(false));
-
-static cl::opt<bool>
-IgnorePNaClABIChecks(
-    "ignore-pnaclabi-checks",
-    cl::desc("Ignore checking bitcode for PNaCl ABI violations"),
-    cl::init(false));
-
-/// Class to handle sign rotations in a human readable form. That is,
-/// the sign is in the low bit. The two special cases are:
-/// 1) -1 is true for i1.
-/// 2) The representation allows -0 (which is different than 0).
-class SignRotatedInt {
-public:
-  SignRotatedInt(uint64_t Value, Type* ValueType)
-      : SignedValue(Value >> 1),
-        IsNegated((Value & 0x1)
-                  && !(ValueType->isIntegerTy()
-                       && ValueType->getIntegerBitWidth() == 1)) {
-  }
-
-  SignRotatedInt()
-      : SignedValue(0), IsNegated(false) {}
-
-  explicit SignRotatedInt(const SignRotatedInt &V)
-      : SignedValue(V.SignedValue), IsNegated(V.IsNegated) {}
-
-  void operator=(const SignRotatedInt &V) {
-    SignedValue = V.SignedValue;
-    IsNegated = V.IsNegated;
-  }
-
-  void Print(raw_ostream &Strm) const {
-    if (IsNegated) Strm << "-";
-    Strm << SignedValue;
-  }
-
-private:
-  int64_t SignedValue;
-  bool IsNegated;
-};
-
-inline raw_ostream &operator<<(raw_ostream &Strm, const SignRotatedInt &V) {
-  V.Print(Strm);
-  return Strm;
-}
-
-/// Convenience class to be able to print value ids to raw_ostream's.
-/// Kinds of bitcode Ids:
-///    a : For abbreviations.
-///    b : For basic blocks.
-///    c : For local constants.
-///    f : For function addresses.
-///    g : For global variable addresses.
-///    p : For parameter arguments.
-///    t : For type values.
-///    v : For values generated by instructions.
-class BitcodeId {
-public:
-  BitcodeId(char Kind, NaClBcIndexSize_t Index)
-      : Kind(Kind), Index(Index), IsGlobal(IsGlobalKind(Kind)) {
-  }
-
-  BitcodeId(const BitcodeId &Id)
-      : Kind(Id.Kind), Index(Id.Index), IsGlobal(Id.IsGlobal) {}
-
-  BitcodeId(char Kind, NaClBcIndexSize_t Index, bool IsGlobal)
-      : Kind(Kind), Index(Index), IsGlobal(IsGlobal) {}
-
-  void operator=(const BitcodeId &Id) {
-    Kind = Id.Kind;
-    Index = Id.Index;
-    IsGlobal = Id.IsGlobal;
-  }
-
-  raw_ostream &Print(raw_ostream &Stream) const {
-    return Stream << Prefix() << Kind << Index;
-  }
-
-  char GetKind() const {
-    return Kind;
-  }
-
-  std::string GetName() const {
-    std::string Buffer;
-    raw_string_ostream StrBuf(Buffer);
-    Print(StrBuf);
-    return StrBuf.str();
-  }
-
-private:
-  char Kind;
-  NaClBcIndexSize_t Index;
-  bool IsGlobal;
-
-  // Returns true if (default) assumption for kind implies global.
-  static bool IsGlobalKind(char Kind);
-
-  // Returns the bitcode prefix character that communicates if the
-  // bitcode Id is lcoal or global.
-  char Prefix() const { return IsGlobal ? '@' : '%'; }
-};
-
-bool BitcodeId::IsGlobalKind(char Kind) {
-  switch (Kind) {
-  case 'f':
-  case 'g':
-  case 't':
-    return true;
-  case 'b':
-  case 'c':
-  case 'p':
-  case 'v':
-    return false;
-  default:
-    errs() << "Bad bitcode id, can't determine (statically) if global: "
-           << Kind << "\n";
-    report_fatal_error("Unable to continue");
-  }
-}
-
-raw_ostream &operator<<(raw_ostream &Stream, const BitcodeId &Id) {
-  return Id.Print(Stream);
-}
-
-class NaClDisBlockParser;
-
-/// The text formatter for PNaClAsm instructions.
-class AssemblyTextFormatter : public naclbitc::TextFormatter {
-public:
-  // Special directive to tokenize type expressions. Used to convert
-  // type signatures into a sequence of tokens.
-  class TypeDirective : public naclbitc::TextFormatter::Directive {
-  public:
-    TypeDirective(TextFormatter *Formatter)
-        : naclbitc::TextFormatter::Directive(Formatter),
-          Typ(0),
-          FcnId(0),
-          AddParams(false) {}
-
-    ~TypeDirective() override {}
-
-  private:
-    /// Calls the corresponding method in AssemblyTextFormatter, with
-    /// the locally stored arguments.
-    void MyApply(bool Replay) const override;
-
-    void MaybeSaveForReplay() const override {}
-
-    // The type to tokenize.
-    Type *Typ;
-    // Pointer to function id, if not NULL.
-    BitcodeId *FcnId;
-    // true if parameter id's should be added to function signature.
-    bool AddParams;
-
-    friend class AssemblyTextFormatter;
-
-    // Internal routine to allow AssemblyTextFormatter::AllocateTypeDirective
-    // to initialize a type directive.
-    void Init(Type *NewTyp, BitcodeId *NewFcnId, bool NewAddParams) {
-      Typ = NewTyp;
-      FcnId = NewFcnId;
-      AddParams = NewAddParams;
-    }
-  };
-
-  // Special directive to tokenize an abbreviation. Used to convert
-  // an abbreviation (pointer) into a sequence of tokens.
-  class AbbreviationDirective : public naclbitc::TextFormatter::Directive {
-  public:
-    AbbreviationDirective(TextFormatter *Formatter)
-        : naclbitc::TextFormatter::Directive(Formatter),
-          Abbrev(0) {}
-
-    ~AbbreviationDirective() override {}
-
-  private:
-    void MyApply(bool Replay) const override;
-
-    void MaybeSaveForReplay() const override {}
-
-    // The abbreviation to tokenize.
-    NaClBitCodeAbbrev *Abbrev;
-
-    friend class AssemblyTextFormatter;
-  };
-
-  // Special directive to tokenize an abbreviation index, if the corresponding
-  // record in the block parser used a user-defined abbreviation.
-  class AbbrevIndexDirective : public naclbitc::TextFormatter::Directive {
-  public:
-    AbbrevIndexDirective(TextFormatter *Formatter)
-        : naclbitc::TextFormatter::Directive(Formatter),
-          Record(0), NumGlobalAbbreviations(0) {}
-
-    ~AbbrevIndexDirective() override {}
-
-  private:
-    void MyApply(bool Replay) const override;
-
-    void MaybeSaveForReplay() const override {}
-
-    // The record containing the associated abbreviation.
-    NaClBitcodeRecord *Record;
-
-    // The number of global abbreviations defined for the block the
-    // record appears in.
-    NaClBcIndexSize_t NumGlobalAbbreviations;
-
-    friend class AssemblyTextFormatter;
-  };
-
-public:
-
-  /// Creates an assembly text formatter for the given dump stream.
-  AssemblyTextFormatter(naclbitc::ObjDumpStream &ObjDump)
-      : TextFormatter(ObjDump.Assembly(),
-                      std::max(20U, GetAssemblyWidth(ObjDump)),
-                      "  "),
-        Comma(this, ","),
-        Semicolon(this, ";"),
-        Colon(this, ":"),
-        Space(this),
-        OpenParen(this, "("),
-        CloseParen(this, ")"),
-        OpenAngle(this, "<"),
-        CloseAngle(this, ">"),
-        OpenCurly(this, "{"),
-        CloseCurly(this, "}"),
-        OpenSquare(this, "["),
-        CloseSquare(this, "]"),
-        Endline(this),
-        StartCluster(this),
-        FinishCluster(this)
-  {
-    ContinuationIndent = GetIndent(2);
-  }
-
-  ~AssemblyTextFormatter() override {}
-
-  naclbitc::TokenTextDirective Comma;
-  naclbitc::TokenTextDirective Semicolon;
-  naclbitc::TokenTextDirective Colon;
-  naclbitc::SpaceTextDirective Space;
-  naclbitc::OpenTextDirective OpenParen;
-  naclbitc::CloseTextDirective CloseParen;
-  naclbitc::OpenTextDirective OpenAngle;
-  naclbitc::CloseTextDirective CloseAngle;
-  naclbitc::OpenTextDirective OpenCurly;
-  naclbitc::CloseTextDirective CloseCurly;
-  naclbitc::OpenTextDirective OpenSquare;
-  naclbitc::CloseTextDirective CloseSquare;
-  naclbitc::EndlineTextDirective Endline;
-  naclbitc::StartClusteringDirective StartCluster;
-  naclbitc::FinishClusteringDirective FinishCluster;
-
-  /// Prints the given type as a sequence of tokens.
-  TypeDirective &TokenizeType(Type *Typ) {
-    return AllocateTypeDirective(Typ, 0, false);
-  }
-
-  /// Prints the named function type as a sequence of tokens.
-  /// Typ is the type signature of the function, and FunctionName
-  /// points to the name of the function.
-  TypeDirective &TokenizeFunctionType(FunctionType *Typ,
-                                      BitcodeId *FunctionName) {
-    return AllocateTypeDirective(Typ, FunctionName, false);
-  }
-
-  /// Prints the function signature of the function type. Typ is
-  /// the type signature of the function. FunctionName points to the
-  /// name of the function. Note: Unlike TokenizeFunctionType, this
-  /// method also adds the parameter names to paramter argumements.
-  TypeDirective &TokenizeFunctionSignature(FunctionType *Typ,
-                                           BitcodeId *FunctionName) {
-    return AllocateTypeDirective(Typ, FunctionName, true);
-  }
-
-  /// Prints out the abbreviation defined by Abbrev.
-  AbbreviationDirective &TokenizeAbbreviation(NaClBitCodeAbbrev *Abbrev) {
-    AbbreviationDirective *Dir = AbbrevDirMemoryPool.Allocate(this);
-    Dir->Abbrev = Abbrev;
-    return *Dir;
-  }
-
-  /// If the record was read using a user-defined abbreviation,
-  /// generates assembly tokens describing the abbreviation index
-  /// used. Otherwise, no tokens are generated. NumGlobalAbbreviations
-  /// is used to determine if the user-defined abbreviation is local
-  /// or global.
-  AbbrevIndexDirective &TokenizeAbbrevIndex(
-      NaClBitcodeRecord &Record, NaClBcIndexSize_t NumGlobalAbbreviations) {
-    AbbrevIndexDirective *Dir = AbbrevIndexDirMemoryPool.Allocate(this);
-    Dir->Record = &Record;
-    Dir->NumGlobalAbbreviations = NumGlobalAbbreviations;
-    return *Dir;
-  }
-
-private:
-  // Converts the given type to tokens, based on the values passed in
-  // by TokenizeType, TokenizeFunctionType, or TokenizeFunctionSignature.
-  void TokenizeTypeInternal(Type *Typ, BitcodeId* FcnName, bool AddParams);
-
-  // The free list of type directives.
-  naclbitc::DirectiveMemoryPool<TypeDirective> TypeDirMemoryPool;
-
-  // Allocates an instance of TypeDirective with the following fields.
-  TypeDirective &AllocateTypeDirective(Type *Typ, BitcodeId *FcnId,
-                                       bool AddParams);
-
-  // Converts the given abbreviation to tokens.
-  void TokenizeAbbreviationInternal(const NaClBitCodeAbbrev *Abbrev);
-
-  // Helper function that prints out the abbreviation expression
-  // located at Index in the given abbreviation. Updates Index to
-  // next expression in the abbreviation.
-  void TokenizeAbbrevExpression(const NaClBitCodeAbbrev *Abbrev,
-                                unsigned &Index);
-
-  // Prints out the given abbreviation operator.
-  void TokenizeAbbrevOp(const NaClBitCodeAbbrevOp &Op);
-
-  // The free list for abbreviation directives.
-  naclbitc::DirectiveMemoryPool<AbbreviationDirective> AbbrevDirMemoryPool;
-
-  // The free list for abbreviation index directives.
-  naclbitc::DirectiveMemoryPool<AbbrevIndexDirective> AbbrevIndexDirMemoryPool;
-
-  // Computes how wide the assembly portion of the dump file should be.
-  static unsigned GetAssemblyWidth(naclbitc::ObjDumpStream &ObjDump) {
-    int Diff = 80 - (ObjDump.GetRecordWidth()+1);
-    // Make sure there is enough room to print out assembly.
-    return Diff < 20 ? 20 : Diff;
-  }
-};
-
-void AssemblyTextFormatter::TypeDirective::MyApply(bool Replay) const {
-  assert(!Replay && "Shouldn't have been saved for replay");
-  AssemblyTextFormatter *AssemblyFormatter =
-      reinterpret_cast<AssemblyTextFormatter*>(Formatter);
-  AssemblyFormatter->TokenizeTypeInternal(Typ, FcnId, AddParams);
-  AssemblyFormatter->TypeDirMemoryPool.Free(const_cast<TypeDirective*>(this));
-}
-
-void AssemblyTextFormatter::AbbreviationDirective::MyApply(bool Replay) const {
-  assert(!Replay && "Shouldn't have been saved for replay");
-  AssemblyTextFormatter *AssemblyFormatter =
-      reinterpret_cast<AssemblyTextFormatter*>(Formatter);
-  AssemblyFormatter->TokenizeAbbreviationInternal(Abbrev);
-  AssemblyFormatter->AbbrevDirMemoryPool.Free(
-      const_cast<AbbreviationDirective*>(this));
-}
-
-AssemblyTextFormatter::TypeDirective &AssemblyTextFormatter::
-AllocateTypeDirective(Type *Typ, BitcodeId *FcnId, bool AddParams) {
-  TypeDirective *Element = TypeDirMemoryPool.Allocate(this);
-  Element->Init(Typ, FcnId, AddParams);
-  return *Element;
-}
-
-void AssemblyTextFormatter::TokenizeTypeInternal(
-    Type *Typ, BitcodeId* FcnName, bool AddParams) {
-  switch (Typ->getTypeID()) {
-  case Type::VoidTyID:
-    TextStream << "void";
-    break;
-  case Type::FloatTyID:
-    TextStream << "float";
-    break;
-  case Type::DoubleTyID:
-    TextStream << "double";
-    break;
-  case Type::IntegerTyID:
-    TextStream << "i" << Typ->getIntegerBitWidth();
-    break;
-  case Type::VectorTyID: {
-    VectorType *VecType = cast<VectorType>(Typ);
-    TextStream << StartCluster << OpenAngle << VecType->getNumElements()
-               << Space << "x" << Space
-               << TokenizeType(VecType->getElementType())
-               << CloseAngle << FinishCluster;
-    break;
-  }
-  case Type::FunctionTyID: {
-    FunctionType *FcnType = cast<FunctionType>(Typ);
-    size_t NumParams = FcnType->getFunctionNumParams();
-    TextStream << StartCluster
-               << TokenizeType(FcnType->getReturnType())
-               << Space
-               << StartCluster;
-    if (NumParams > 0) TextStream << StartCluster;
-    if (FcnName) TextStream << *FcnName;
-    TextStream << OpenParen << FinishCluster;
-    bool HasParamStartCluster = false;
-    for (size_t i = 0, NumParams = FcnType->getFunctionNumParams();
-         i < NumParams; ++i) {
-      if (i > 0) {
-        TextStream << Comma << FinishCluster << Space;
-        HasParamStartCluster = false;
-      }
-      TextStream << StartCluster
-                 << TokenizeType(FcnType->getFunctionParamType(i));
-      if (AddParams) TextStream << Space << BitcodeId('p', i);
-      HasParamStartCluster = true;
-    }
-    if (FcnType->isFunctionVarArg()) {
-      if (HasParamStartCluster) {
-        TextStream << Comma << FinishCluster << Space;
-      }
-      TextStream << StartCluster << "...";
-      HasParamStartCluster = true;
-    }
-    if (HasParamStartCluster) TextStream << FinishCluster;
-    TextStream << CloseParen << FinishCluster;
-    if (NumParams > 0) TextStream << FinishCluster;
-    break;
-  }
-  default:
-    report_fatal_error("Unsupported PNaCl type found");
-    break;
-  }
-}
-
-void AssemblyTextFormatter::TokenizeAbbrevOp(const NaClBitCodeAbbrevOp &Op) {
-  // Note: Mimics NaClBitCodeAbbrevOp::Print in NaClBitCodes.cpp
-  switch (Op.getEncoding()) {
-  case NaClBitCodeAbbrevOp::Literal:
-    Tokens() << Op.getValue();
-    return;
-  case NaClBitCodeAbbrevOp::Fixed:
-    Tokens() << StartCluster << "fixed" << OpenParen << Op.getValue()
-             << CloseParen << FinishCluster;
-    return;
-  case NaClBitCodeAbbrevOp::VBR:
-    Tokens() << StartCluster << "vbr" << OpenParen << Op.getValue()
-             << CloseParen << FinishCluster;
-    return;
-  case NaClBitCodeAbbrevOp::Array:
-    Tokens() << "array";
-    return;
-  case NaClBitCodeAbbrevOp::Char6:
-    Tokens() << "char6";
-    return;
-  }
-}
-
-void AssemblyTextFormatter::
-TokenizeAbbrevExpression(const NaClBitCodeAbbrev *Abbrev, unsigned &Index) {
-  // Note: Mimics PrintExpression in NaClBitCodes.cpp
-  const NaClBitCodeAbbrevOp &Op = Abbrev->getOperandInfo(Index);
-  TokenizeAbbrevOp(Op);
-  if (unsigned NumArgs = Op.NumArguments()) {
-    Tokens() << StartCluster << OpenParen;
-    for (size_t i = 0; i < NumArgs; ++i) {
-      ++Index;
-      if (i > 0) {
-        Tokens() << Comma << FinishCluster << Space << StartCluster;
-      }
-      TokenizeAbbrevExpression(Abbrev, Index);
-    }
-    Tokens() << CloseParen << FinishCluster;
-  }
-}
-
-void AssemblyTextFormatter::
-TokenizeAbbreviationInternal(const NaClBitCodeAbbrev *Abbrev) {
-  Tokens() << StartCluster << OpenAngle;
-  for (unsigned i = 0; i < Abbrev->getNumOperandInfos(); ++i) {
-    if (i > 0) {
-      Tokens() << Comma << FinishCluster << Space << StartCluster;
-    }
-    TokenizeAbbrevExpression(Abbrev, i);
-  }
-  Tokens() << CloseAngle << FinishCluster;
-}
-
-void AssemblyTextFormatter::AbbrevIndexDirective::MyApply(bool Replay) const {
-  assert(!Replay && "Shouldn't have been saved for replay");
-  if (!Record->UsedAnAbbreviation()) return;
-  NaClBcIndexSize_t Index = Record->GetAbbreviationIndex();
-  if (Index < naclbitc::FIRST_APPLICATION_ABBREV) return;
-  Index -= naclbitc::FIRST_APPLICATION_ABBREV;
-  bool IsGlobal = Index < NumGlobalAbbreviations;
-  if (!IsGlobal) Index -= NumGlobalAbbreviations;
-  BitcodeId AbbrevIndex('a', Index, IsGlobal);
-  AssemblyTextFormatter *Fmtr =
-      reinterpret_cast<AssemblyTextFormatter*>(Formatter);
-  Tokens() << Fmtr->Space << Fmtr->StartCluster << Fmtr->OpenAngle
-           << AbbrevIndex << Fmtr->CloseAngle << Fmtr->FinishCluster;
-  Fmtr->AbbrevIndexDirMemoryPool.Free(const_cast<AbbrevIndexDirective*>(this));
-}
-
-// Holds possible alignements for loads/stores etc. Used to extract
-// expected values.
-static unsigned NaClPossibleLoadStoreAlignments[] = {
-  1, 2, 4, 8
-};
-
-// Finds the expected alignments for the load/store, for type Ty.
-static void NaClGetExpectedLoadStoreAlignment(
-    const DataLayout &DL, Type *Ty, std::vector<unsigned> &ValidAlignments) {
-  for (size_t i = 0; i < array_lengthof(NaClPossibleLoadStoreAlignments); ++i) {
-    unsigned Alignment = NaClPossibleLoadStoreAlignments[i];
-    if (PNaClABIProps::isAllowedAlignment(&DL, Alignment, Ty)) {
-      ValidAlignments.push_back(Alignment);
-    }
-  }
-}
-
-/// Top-level class to parse bitcode file and transform to
-/// corresponding disassembled code.
-class NaClDisTopLevelParser : public NaClBitcodeParser {
-  NaClDisTopLevelParser(const NaClDisTopLevelParser&) = delete;
-  void operator=(const NaClDisTopLevelParser&) = delete;
-
-public:
-  NaClDisTopLevelParser(NaClBitcodeHeader &Header,
-                        NaClBitstreamCursor &Cursor,
-                        naclbitc::ObjDumpStream &ObjDump)
-      : NaClBitcodeParser(Cursor),
-        Mod("ObjDump", getGlobalContext()),
-        ObjDump(ObjDump),
-        AbbrevListener(this),
-        DL(&Mod),
-        AllowedIntrinsics(&Mod.getContext()),
-        AssemblyFormatter(ObjDump),
-        Header(Header),
-        NumFunctions(0),
-        NumGlobals(0),
-        ExpectedNumGlobals(0),
-        NumParams(0),
-        NumConstants(0),
-        NumValuedInsts(0),
-        UnknownType(Type::getVoidTy(Mod.getContext())),
-        PointerType(Type::getInt32Ty(Mod.getContext())),
-        ComparisonType(Type::getInt1Ty(Mod.getContext())),
-        NumDefinedFunctions(0) {
-    SetListener(&AbbrevListener);
-  }
-
-  ~NaClDisTopLevelParser() override {
-    // Be sure to flush any remaining errors.
-    ObjDump.Flush();
-  }
-
-  // Returns the number of errors that were sent to the ObjDump.
-  unsigned GetNumErrors() {
-    return ObjDump.GetNumErrors();
-  }
-
-  /// Generates an error with the given message.
-  bool ErrorAt(naclbitc::ErrorLevel Level, uint64_t Bit,
-               const std::string &Message) final {
-    ObjDump.ErrorAt(Level, Bit) << Message << "\n";
-    if (Level == naclbitc::Fatal)
-      ObjDump.FlushThenQuit();
-    return true;
-  }
-
-  /// Parses the top-level module block.
-  bool ParseBlock(unsigned BlockID) override;
-
-  /// Installs the given type to the next available type index.
-  void InstallType(Type *Ty) {
-    TypeIdType.push_back(Ty);
-  }
-
-  /// Returns the type associated with the given type index.
-  Type *GetType(NaClBcIndexSize_t Index) {
-    if (Index >= TypeIdType.size()) {
-      BitcodeId Id('t', Index);
-      Errors() << "Can't find definition for " << Id << "\n";
-      // Recover so that additional errors can be found.
-      return UnknownType;
-    }
-    return TypeIdType[Index];
-  }
-
-  /// Returns the number of types (currently) defined in the bitcode
-  /// file.
-  NaClBcIndexSize_t GetNumTypes() const {
-    return TypeIdType.size();
-  }
-
-  /// Returns true if Type represents a (non-zero sized) value.
-  bool isValidValueType(Type *Ty) const {
-    return Ty->isIntegerTy() || Ty->isFloatTy() || Ty->isDoubleTy()
-        || Ty->isVectorTy();
-  }
-
-  /// Installs the given type to the next available function index.
-  void InstallFunctionType(FunctionType *Ty, uint64_t BitAddress) {
-    assert(FunctionIdType.size() == NumFunctions);
-    ++NumFunctions;
-    FunctionIdType.push_back(Ty);
-    FunctionIdAddress.push_back(BitAddress);
-    // Let Valuesymtab change this to true if appropriate.
-    FunctionIdIsIntrinsic.push_back(false);
-  }
-
-  /// Returns the bit address where the function record appeared in
-  /// the bitcode file. Returns 0 if unknown.
-  uint64_t GetFunctionIdAddress(NaClBcIndexSize_t Index) {
-    if (Index >= FunctionIdAddress.size()) return 0;
-    return FunctionIdAddress[Index];
-  }
-
-  /// Sets the record bit address to the function ID address (if known).
-  void SetRecordAddressToFunctionIdAddress(NaClBcIndexSize_t Index) {
-    uint64_t Address = GetFunctionIdAddress(Index);
-    if (Address)
-      ObjDump.SetRecordBitAddress(Address);
-  }
-
-  /// Returns the type associated with the given function index.
-  FunctionType *GetFunctionType(NaClBcIndexSize_t Index) {
-    if (Index >= FunctionIdType.size()) {
-      BitcodeId Id('f', Index);
-      Errors() << "Can't find definition for " << Id << "\n";
-      Fatal();
-    }
-    return FunctionIdType[Index];
-  }
-
-  /// Marks the function associated with the given Index as intrinsic.
-  void MarkFunctionAsIntrinsic(NaClBcIndexSize_t Index) {
-    if (Index >= FunctionIdIsIntrinsic.size()) {
-      BitcodeId Id('f', Index);
-      Errors() << "Can't define " << Id << " as intrinsic, no definition";
-    }
-    FunctionIdIsIntrinsic[Index] = true;
-  }
-
-  /// Returns true if the given Index is for a function intrinsic.
-  bool IsFunctionIntrinsic(NaClBcIndexSize_t Index) {
-    if (Index >= FunctionIdIsIntrinsic.size()) return false;
-    return FunctionIdIsIntrinsic[Index];
-  }
-
-  /// Returns true if the function is an intrinsic function.
-  bool IsIntrinsicAllowed(const std::string &FuncName,
-                          const FunctionType *FuncType) {
-    return AllowedIntrinsics.isAllowed(FuncName, FuncType);
-  }
-
-  /// Returns the type of the Name'd intrinsic, if defined as an
-  /// allowed intrinsic name. Otherwise returns 0.
-  FunctionType *GetIntrinsicType(const std::string &Name) {
-    return AllowedIntrinsics.getIntrinsicType(Name);
-  }
-
-  /// Returns the number of functions (currently) defined/declared in
-  /// the bitcode file.
-  NaClBcIndexSize_t GetNumFunctions() const {
-    return NumFunctions;
-  }
-
-  /// Installs that the given function index as having a function body
-  /// (i.e. the function address uses the "define" keyword instead of
-  /// the "declare" keyword).
-  void InstallDefinedFunction(NaClBcIndexSize_t Index) {
-    DefinedFunctions.push_back(Index);
-  }
-
-  /// Returns true if there is an Index defined function block.
-  bool HasDefinedFunctionIndex(NaClBcIndexSize_t Index) const {
-    return Index < DefinedFunctions.size();
-  }
-
-  /// Returns the function index associated with the Index defined
-  /// function block.
-  NaClBcIndexSize_t GetDefinedFunctionIndex(NaClBcIndexSize_t Index) const {
-    assert(Index < DefinedFunctions.size());
-    return DefinedFunctions[NumDefinedFunctions];
-  }
-
-  /// Returns true if there is a next defined function index.
-  bool HasNextDefinedFunctionIndex() const {
-    return HasDefinedFunctionIndex(NumDefinedFunctions);
-  }
-
-  /// Returns the function index associated with the next defined function.
-  NaClBcIndexSize_t GetNextDefinedFunctionIndex() const {
-    return GetDefinedFunctionIndex(NumDefinedFunctions);
-  }
-
-  /// Returns true if the given value ID corresponds to a declared (rather
-  /// than defined) function.
-  bool IsDeclaredFunction(NaClBcIndexSize_t Index) {
-    if (Index >= NumFunctions) return false;
-    for (size_t i = 0, e = DefinedFunctions.size(); i < e; ++i) {
-      if (DefinedFunctions[i] == Index) return false;
-    }
-    return true;
-  }
-
-  /// Increments the number of (currently) defined functions in the
-  /// bitcode file by one.
-  void IncNumDefinedFunctions() {
-    ++NumDefinedFunctions;
-  }
-
-  void IncNumGlobals() {
-    NumGlobals++;
-  }
-
-  /// Returns the number of globals (currently) defined in the bitcode
-  /// file.
-  NaClBcIndexSize_t GetNumGlobals() const {
-    return NumGlobals;
-  }
-
-  /// Returns the expected number of globals (as defined by the count
-  /// record of the globals block).
-  NaClBcIndexSize_t GetExpectedNumGlobals() const {
-    return ExpectedNumGlobals;
-  }
-
-  /// Sets field ExpectedNumGlobals to the given value.
-  void SetExpectedNumGlobals(NaClBcIndexSize_t NewValue) {
-    ExpectedNumGlobals = NewValue;
-  }
-
-  /// Installs the given type to the next available parameter index.
-  void InstallParamType(Type *Ty) {
-    while (ParamIdType.size() <= NumParams) {
-      ParamIdType.push_back(UnknownType);
-    }
-    ParamIdType[NumParams++] = Ty;
-  }
-
-  /// Returns the type associated with the given parameter index.
-  Type *GetParamType(NaClBcIndexSize_t Index) {
-    if (Index >= ParamIdType.size()) {
-      BitcodeId Id('p', Index);
-      Errors() << "Can't find defintion for " << Id << "\n";
-      Fatal();
-    }
-    return ParamIdType[Index];
-  }
-
-  /// Returns the number of parameters (currently) defined in the
-  /// enclosing defined function.
-  NaClBcIndexSize_t GetNumParams() const {
-    return NumParams;
-  }
-
-  /// Installs the given type to the next available constant index.
-  void InstallConstantType(Type *Ty) {
-    while (ConstantIdType.size() <= NumConstants) {
-      ConstantIdType.push_back(UnknownType);
-    }
-    ConstantIdType[NumConstants++] = Ty;
-  }
-
-  /// Returns the type associated with the given constant index.
-  Type *GetConstantType(NaClBcIndexSize_t Index) {
-    if (Index >= ConstantIdType.size()) {
-      BitcodeId Id('c', Index);
-      Errors() << "Can't find definition for " << Id << "\n";
-      Fatal();
-    }
-    return ConstantIdType[Index];
-  }
-
-  /// Returns the number of constants (currently) defined in the
-  /// enclosing defined function.
-  NaClBcIndexSize_t GetNumConstants() const {
-    return NumConstants;
-  }
-
-  /// Installs the given type to the given instruction index.  Note:
-  /// Instruction indices are only associated with instructions that
-  /// generate values.
-  void InstallInstType(Type *Ty, NaClBcIndexSize_t Index) {
-    while (InstIdType.size() <= Index) {
-      InstIdType.push_back(UnknownType);
-    }
-    if (InstIdType[Index] != UnknownType && Ty != InstIdType[Index]) {
-      Errors() << BitcodeId('v', Index) << " defined with multiple types: "
-               << *Ty << " and " << *InstIdType[Index] << "\n";
-    }
-    InstIdType[Index] = Ty;
-  }
-
-  /// Installs the given type to the next available instruction index.
-  void InstallInstType(Type *Ty) {
-    InstallInstType(Ty, NumValuedInsts++);
-  }
-
-  /// Returns the type associated with the given instruction index.
-  /// Note: Instruction indices are only associated with instructions
-  /// that generate values.
-  Type *GetInstType(NaClBcIndexSize_t Index) {
-    if (Index >= InstIdType.size()) {
-      Errors() << "Can't find type for " << BitcodeId('v', Index) << "\n";
-      return UnknownType;
-    }
-    return InstIdType[Index];
-  }
-
-  /// Returns the number of instructions (in the defined function)
-  /// that (currently) generate values.
-  NaClBcIndexSize_t GetNumValuedInstructions() const {
-    return NumValuedInsts;
-  }
-
-  /// Resets index counters local to a defined function.
-  void ResetLocalCounters() {
-    ParamIdType.clear();
-    NumParams = 0;
-    NumConstants = 0;
-    InstIdType.clear();
-    NumValuedInsts = 0;
-  }
-
-  /// Returns the bitcode id associated with the absolute value index
-  BitcodeId GetBitcodeId(NaClBcIndexSize_t Index);
-
-  /// Returns the type associated with the given absolute value index.
-  /// If UnderlyingType is false, function indices always return a
-  /// pointer type. Otherwise, function indices returns the declared
-  /// type of the function index.
-  Type *GetValueType(NaClBcIndexSize_t Index, bool UnderlyingType = false);
-
-  /// Returns a type to use when the type is unknown.
-  Type *GetUnknownType() const {
-    return UnknownType;
-  }
-
-  /// Returns the type used to represent a pointer.
-  Type *GetPointerType() const {
-    return PointerType;
-  }
-
-  /// Returns the type used to represent comparison results.
-  Type *GetComparisonType() const {
-    return ComparisonType;
-  }
-
-  /// Returns the void type.
-  Type *GetVoidType() const {
-    return Type::getVoidTy(getGlobalContext());
-  }
-
-  /// Returns the float (32-bit) type.
-  Type *GetFloatType() const {
-    return Type::getFloatTy(getGlobalContext());
-  }
-
-  /// Returns the double (64-bit) type.
-  Type *GetDoubleType() const {
-    return Type::getDoubleTy(getGlobalContext());
-  }
-
-  /// Returns an integer type of N bits.
-  Type *GetIntegerType(unsigned N) const {
-    return Type::getIntNTy(getGlobalContext(), N);
-  }
-
-  /// Returns the number of defined global abbreviations (currently)
-  /// for the given block id.
-  NaClBcIndexSize_t GetNumGlobalAbbreviations(unsigned BlockID) {
-    return GlobalAbbrevsCountMap[BlockID];
-  }
-
-  /// Increments the current number of defined global abbreviations.
-  void IncNumGlobalAbbreviations(unsigned BlockID) {
-    ++GlobalAbbrevsCountMap[BlockID];
-  }
-
-  /// Verifies the given integer operator has the right type.  Returns
-  /// the operator.
-  const char *VerifyIntArithmeticOp(const char *Op, Type *OpTy) {
-    if (!IgnorePNaClABIChecks &&
-        !PNaClABITypeChecker::isValidIntArithmeticType(OpTy)) {
-      Errors() << Op << ": Invalid integer arithmetic type: " << *OpTy;
-    }
-    return Op;
-  }
-
-  // Checks the Alignment for loading/storing a value of type Ty. If
-  // invalid, generates an appropriate error message.
-  void VerifyMemoryAccessAlignment(const char *Op, Type *Ty,
-                                   unsigned Alignment) {
-    if (IgnorePNaClABIChecks) return;
-    if (!PNaClABIProps::isAllowedAlignment(&DL, Alignment, Ty)) {
-      raw_ostream &Errs = Errors();
-      std::vector<unsigned> Alignments;
-      NaClGetExpectedLoadStoreAlignment(DL, Ty, Alignments);
-      if (!Alignments.empty()) {
-        Errs << Op << ": Illegal alignment for " << *Ty << ". Expects: ";
-        bool isFirst = true;
-        for (auto Align : Alignments) {
-          if (isFirst)
-            isFirst = false;
-          else
-            Errs << " or ";
-          Errs << Align;
-        }
-        Errs << "\n";
-      } else {
-        Errs << Op << ": Not allowed for type: " << *Ty << "\n";
-      }
-    }
-  }
-
-  // ******************************************************
-  // The following return the corresponding methods/fields
-  // from the the assembly formatter/objdumper.
-  // ******************************************************
-
-  naclbitc::TokenTextDirective &Semicolon() {
-    return AssemblyFormatter.Semicolon;
-  }
-
-  naclbitc::TokenTextDirective &Colon() {
-    return AssemblyFormatter.Colon;
-  }
-
-  naclbitc::SpaceTextDirective &Space() {
-    return AssemblyFormatter.Space;
-  }
-
-  naclbitc::TokenTextDirective &Comma() {
-    return AssemblyFormatter.Comma;
-  }
-
-  naclbitc::OpenTextDirective &OpenParen() {
-    return AssemblyFormatter.OpenParen;
-  }
-
-  naclbitc::CloseTextDirective &CloseParen() {
-    return AssemblyFormatter.CloseParen;
-  }
-
-  naclbitc::OpenTextDirective &OpenAngle() {
-    return AssemblyFormatter.OpenAngle;
-  }
-
-  naclbitc::CloseTextDirective &CloseAngle() {
-    return AssemblyFormatter.CloseAngle;
-  }
-
-  naclbitc::OpenTextDirective &OpenCurly() {
-    return AssemblyFormatter.OpenCurly;
-  }
-
-  naclbitc::CloseTextDirective &CloseCurly() {
-    return AssemblyFormatter.CloseCurly;
-  }
-
-  naclbitc::OpenTextDirective &OpenSquare() {
-    return AssemblyFormatter.OpenSquare;
-  }
-
-  naclbitc::CloseTextDirective &CloseSquare() {
-    return AssemblyFormatter.CloseSquare;
-  }
-
-  naclbitc::EndlineTextDirective &Endline() {
-    return AssemblyFormatter.Endline;
-  }
-
-  naclbitc::StartClusteringDirective &StartCluster() {
-    return AssemblyFormatter.StartCluster;
-  }
-
-  naclbitc::FinishClusteringDirective &FinishCluster() {
-    return AssemblyFormatter.FinishCluster;
-  }
-
-  raw_ostream &Tokens() {
-    return AssemblyFormatter.Tokens();
-  }
-
-  raw_ostream &Errors() {
-    return ObjDump.Error();
-  }
-
-  raw_ostream &Warnings() {
-    if (ReportWarningsAsErrors) return Errors();
-    return ObjDump.Warning();
-  }
-
-  const std::string &GetAssemblyIndent() const {
-    return AssemblyFormatter.GetIndent();
-  }
-
-  unsigned GetAssemblyNumTabs() const {
-    return AssemblyFormatter.GetNumTabs();
-  }
-
-  void IncAssemblyIndent() {
-    AssemblyFormatter.Inc();
-  }
-
-  void DecAssemblyIndent() {
-    AssemblyFormatter.Dec();
-  }
-
-  void IncRecordIndent() {
-    ObjDump.IncRecordIndent();
-  }
-
-  void DecRecordIndent() {
-    ObjDump.DecRecordIndent();
-  }
-
-  void ObjDumpFlush() {
-    ObjDump.Flush();
-  }
-
-  void ObjDumpSetRecordBitAddress(uint64_t Bit) {
-    ObjDump.SetRecordBitAddress(Bit);
-  }
-
-  void ObjDumpWrite(uint64_t Bit,
-                    const NaClBitcodeRecordData &Record,
-                    NaClBcIndexSize_t AbbrevIndex =
-                    naclbitc::ABBREV_INDEX_NOT_SPECIFIED) {
-    ObjDump.Write(Bit, Record, AbbrevIndex);
-  }
-
-  void ObjDumpWrite(uint64_t Bit,
-                    const NaClBitcodeRecord &Record) {
-    ObjDump.Write(Bit, Record.GetRecordData(), Record.GetAbbreviationIndex());
-  }
-
-  AssemblyTextFormatter::TypeDirective &
-  TokenizeType(Type *Typ) {
-    return AssemblyFormatter.TokenizeType(Typ);
-  }
-
-  AssemblyTextFormatter::TypeDirective &
-  TokenizeFunctionType(FunctionType *Type, BitcodeId *FcnId) {
-    return AssemblyFormatter.TokenizeFunctionType(Type, FcnId);
-  }
-
-  AssemblyTextFormatter::TypeDirective &
-  TokenizeFunctionSignature(FunctionType *Typ, BitcodeId *FcnId) {
-    return AssemblyFormatter.TokenizeFunctionSignature(Typ, FcnId);
-  }
-
-  AssemblyTextFormatter::AbbreviationDirective &
-  TokenizeAbbreviation(NaClBitCodeAbbrev *Abbrev) {
-    return AssemblyFormatter.TokenizeAbbreviation(Abbrev);
-  }
-
-  AssemblyTextFormatter::AbbrevIndexDirective &
-  TokenizeAbbrevIndex(NaClBitcodeRecord &Record,
-                      NaClBcIndexSize_t NumGlobalAbbreviations) {
-    return AssemblyFormatter.TokenizeAbbrevIndex(Record,
-                                                 NumGlobalAbbreviations);
-  }
-
-private:
-  // A placeholder module for associating context and data layout.
-  Module Mod;
-  // The output stream to generate disassembly into.
-  naclbitc::ObjDumpStream &ObjDump;
-  // Listener used to get abbreviations as they are read.
-  NaClBitcodeParserListener AbbrevListener;
-  // DataLayout to use.
-  const DataLayout DL;
-  // The set of allowed intrinsics.
-  PNaClAllowedIntrinsics AllowedIntrinsics;
-  // The formatter to use to format assembly code.
-  AssemblyTextFormatter AssemblyFormatter;
-  // The header appearing before the beginning of the input stream.
-  NaClBitcodeHeader &Header;
-  // The list of known types (index i defines the type associated with
-  // type index i).
-  std::vector<Type*> TypeIdType;
-  // The list of known function signatures (index i defines the type
-  // signature associated with function index i).
-  std::vector<FunctionType*> FunctionIdType;
-  // boolean flag defining if function id is an intrinsic.
-  std::vector<bool> FunctionIdIsIntrinsic;
-  // The list of record bit addresses associated with corresponding
-  // declaration of function IDs.
-  std::vector<uint64_t> FunctionIdAddress;
-  // The number of function indices currently defined.
-  NaClBcIndexSize_t NumFunctions;
-  // The number of global indices currently defined.
-  NaClBcIndexSize_t NumGlobals;
-  // The expected number of global indices (i.e. value of count record
-  // in the globals block).
-  NaClBcIndexSize_t ExpectedNumGlobals;
-  // The list of known parameter types (index i defines the type
-  // associated with parameter index i).
-  std::vector<Type*>ParamIdType;
-  // The number of parameter indices currently defined.
-  NaClBcIndexSize_t NumParams;
-  std::vector<Type*>ConstantIdType;
-  // The number of constant indices currently defined.
-  NaClBcIndexSize_t NumConstants;
-  // The list of known instruction types (index i defines the type
-  // associated with valued instruction index i).
-  std::vector<Type*> InstIdType;
-  // The number of valued instructions currently defined.
-  NaClBcIndexSize_t NumValuedInsts;
-  // Models an unknown type.
-  Type *UnknownType;
-  // Models the pointer type.
-  Type *PointerType;
-  // Models a result of a comparison.
-  Type *ComparisonType;
-  // Keeps list of function indicies (in the order found in the bitcode)
-  // that correspond to defined functions.
-  std::vector<NaClBcIndexSize_t> DefinedFunctions;
-  // The number of function indices currently known to be defined.
-  NaClBcIndexSize_t NumDefinedFunctions;
-  // Holds the number of global abbreviations defined for each block.
-  std::map<unsigned, NaClBcIndexSize_t> GlobalAbbrevsCountMap;
-};
-
-BitcodeId NaClDisTopLevelParser::GetBitcodeId(NaClBcIndexSize_t Index) {
-  if (Index < NumFunctions) {
-    return BitcodeId('f', Index);
-  }
-  Index -= NumFunctions;
-  if (Index < ExpectedNumGlobals) {
-    return BitcodeId('g', Index);
-  }
-  Index -= ExpectedNumGlobals;
-  if (Index < NumParams) {
-    return BitcodeId('p', Index);
-  }
-  Index -= NumParams;
-  if (Index < NumConstants) {
-    return BitcodeId('c', Index);
-  }
-  Index -= NumConstants;
-  return BitcodeId('v', Index);
-}
-
-Type *NaClDisTopLevelParser::GetValueType(NaClBcIndexSize_t Index,
-                                          bool UnderlyingType) {
-  NaClBcIndexSize_t Idx = Index;
-  if (Idx < NumFunctions)
-    return UnderlyingType ? GetFunctionType(Idx) : PointerType;
-  Idx -= NumFunctions;
-  if (Idx < ExpectedNumGlobals)
-    return PointerType;
-  Idx -= ExpectedNumGlobals;
-  if (Idx < NumParams)
-    return GetParamType(Idx);
-  Idx -= NumParams;
-  if (Idx < NumConstants)
-    return GetConstantType(Idx);
-  Idx -= NumConstants;
-  return GetInstType(Idx);
-}
-
-// Base class of all block parsers for the bitcode file. Handles
-// common actions needed by derived blocks.
-//
-// Note: This class also handles blocks with unknown block ID's.
-class NaClDisBlockParser : public NaClBitcodeParser {
-protected:
-  /// Constructor for the top-level block parser.
-  NaClDisBlockParser(unsigned BlockID, NaClDisTopLevelParser *Context)
-      : NaClBitcodeParser(BlockID, Context),
-        Context(Context){
-    InitAbbreviations();
-  }
-
-  /// Constructor for nested block parsers.
-  NaClDisBlockParser(unsigned BlockID, NaClDisBlockParser *EnclosingParser)
-      : NaClBitcodeParser(BlockID, EnclosingParser),
-        Context(EnclosingParser->Context) {
-    InitAbbreviations();
-  }
-
-public:
-
-  ~NaClDisBlockParser() override {
-    // Be sure to flush any remaining errors reported at end of block.
-    ObjDumpFlush();
-  }
-
-  bool ParseBlock(unsigned BlockID) override;
-
-protected:
-  void EnterBlock(unsigned NumWords) override;
-
-  void ExitBlock() override;
-
-  void ProcessRecord() override;
-
-  void ProcessAbbreviation(unsigned BlockID, NaClBitCodeAbbrev *Abbrev,
-                           bool IsLocal) override;
-
-  void InitAbbreviations() {
-    NumGlobalAbbreviations = Context->GetNumGlobalAbbreviations(GetBlockID());
-    NumLocalAbbreviations = 0;
-  }
-
-  // Prints the block header instruction for the block. Called by EnterBlock.
-  virtual void PrintBlockHeader();
-
-  // Dumps the corresponding record for a block enter.
-  void DumpEnterBlockRecord();
-
-  // Returns the identifier for the next local abbreviation appearing in
-  // the block.
-  BitcodeId NextLocalAbbreviationId() {
-    return BitcodeId('a', NumLocalAbbreviations, false);
-  }
-
-  AssemblyTextFormatter::AbbrevIndexDirective &TokenizeAbbrevIndex() {
-    return Context->TokenizeAbbrevIndex(Record, NumGlobalAbbreviations);
-  }
-
-  // *****************************************************************
-  // The following are dispatching methods that call the corresponding
-  // method on the Context (i.e. NaClDisTopLevelParser).
-  // *****************************************************************
-
-  /// Returns a directive to tokenize the given type.
-  AssemblyTextFormatter::TypeDirective &TokenizeType(Type *Type) {
-    return Context->TokenizeType(Type);
-  }
-
-  /// Returns a directive to tokenize the given function type,
-  /// using the given function id.
-  AssemblyTextFormatter::TypeDirective
-  &TokenizeFunctionType(FunctionType *Type, BitcodeId *FcnId) {
-    return Context->TokenizeFunctionType(Type, FcnId);
-  }
-
-  /// Returns a directive to tokenize the function signature, given the
-  /// type of the function signature, and the given function id.
-  AssemblyTextFormatter::TypeDirective
-  &TokenizeFunctionSignature(FunctionType *Typ, BitcodeId *FcnId) {
-    return Context->TokenizeFunctionSignature(Typ, FcnId);
-  }
-
-  AssemblyTextFormatter::AbbreviationDirective &TokenizeAbbreviation(
-      NaClBitCodeAbbrev *Abbrev) {
-    return Context->TokenizeAbbreviation(Abbrev);
-  }
-
-  naclbitc::TokenTextDirective &Semicolon() {
-    return Context->Semicolon();
-  }
-
-  naclbitc::TokenTextDirective &Colon() {
-    return Context->Colon();
-  }
-
-  naclbitc::SpaceTextDirective &Space() {
-    return Context->Space();
-  }
-
-  naclbitc::TokenTextDirective &Comma() {
-    return Context->Comma();
-  }
-
-  naclbitc::OpenTextDirective &OpenParen() {
-    return Context->OpenParen();
-  }
-
-
-  naclbitc::CloseTextDirective &CloseParen() {
-    return Context->CloseParen();
-  }
-
-  naclbitc::OpenTextDirective &OpenAngle() {
-    return Context->OpenAngle();
-  }
-
-  naclbitc::CloseTextDirective &CloseAngle() {
-    return Context->CloseAngle();
-  }
-
-  naclbitc::OpenTextDirective &OpenCurly() {
-    return Context->OpenCurly();
-  }
-
-  naclbitc::CloseTextDirective &CloseCurly() {
-    return Context->CloseCurly();
-  }
-
-  naclbitc::OpenTextDirective &OpenSquare() {
-    return Context->OpenSquare();
-  }
-
-  naclbitc::CloseTextDirective &CloseSquare() {
-    return Context->CloseSquare();
-  }
-
-  naclbitc::EndlineTextDirective &Endline() {
-    return Context->Endline();
-  }
-
-  naclbitc::StartClusteringDirective &StartCluster() {
-    return Context->StartCluster();
-  }
-
-  naclbitc::FinishClusteringDirective &FinishCluster() {
-    return Context->FinishCluster();
-  }
-
-  raw_ostream &Tokens() {
-    return Context->Tokens();
-  }
-
-  raw_ostream &Errors() {
-    return Context->Errors();
-  }
-
-  raw_ostream &Warnings() {
-    return Context->Warnings();
-  }
-
-  bool ErrorAt(naclbitc::ErrorLevel Level, uint64_t Bit,
-               const std::string &Message) final {
-    return Context->ErrorAt(Level, Bit, Message);
-  }
-
-  const std::string &GetAssemblyIndent() const {
-    return Context->GetAssemblyIndent();
-  }
-
-  unsigned GetAssemblyNumTabs() const {
-    return Context->GetAssemblyNumTabs();
-  }
-
-  void IncAssemblyIndent() {
-    Context->IncAssemblyIndent();
-  }
-
-  void DecAssemblyIndent() {
-    Context->DecAssemblyIndent();
-  }
-
-  void IncRecordIndent() {
-    Context->IncRecordIndent();
-  }
-
-  void DecRecordIndent() {
-    Context->DecRecordIndent();
-  }
-
-  void ObjDumpFlush() {
-    Context->ObjDumpFlush();
-  }
-
-  void ObjDumpWrite(uint64_t Bit,
-                    const NaClBitcodeRecordData &Record,
-                    NaClBcIndexSize_t AbbrevIndex =
-                    naclbitc::ABBREV_INDEX_NOT_SPECIFIED) {
-    Context->ObjDumpWrite(Bit, Record, AbbrevIndex);
-  }
-
-  void ObjDumpWrite(uint64_t Bit,
-                    const NaClBitcodeRecord &Record) {
-    Context->ObjDumpWrite(Bit, Record);
-  }
-
-  void ObjDumpSetRecordBitAddress(uint64_t Bit) {
-    Context->ObjDumpSetRecordBitAddress(Bit);
-  }
-
-  void InstallType(Type *Ty) {
-    Context->InstallType(Ty);
-  }
-
-  Type *GetType(NaClBcIndexSize_t Index) {
-    return Context->GetType(Index);
-  }
-
-  NaClBcIndexSize_t GetNumTypes() const {
-    return Context->GetNumTypes();
-  }
-
-  bool isValidValueType(Type *Ty) const {
-    return Context->isValidValueType(Ty);
-  }
-
-  FunctionType *GetFunctionType(NaClBcIndexSize_t Index) {
-    return Context->GetFunctionType(Index);
-  }
-
-  NaClBcIndexSize_t GetNumFunctions() const {
-    return Context->GetNumFunctions();
-  }
-
-  void IncNumGlobals() {
-    Context->IncNumGlobals();
-  }
-
-  NaClBcIndexSize_t GetNumGlobals() const {
-    return Context->GetNumGlobals();
-  }
-
-  void InstallFunctionType(FunctionType *Ty) {
-    return Context->InstallFunctionType(Ty, Record.GetStartBit());
-  }
-
-  void InstallDefinedFunction(NaClBcIndexSize_t Index) {
-    Context->InstallDefinedFunction(Index);
-  }
-
-  BitcodeId GetBitcodeId(NaClBcIndexSize_t Id) {
-    return Context->GetBitcodeId(Id);
-  }
-
-  void InstallParamType(Type *Ty) {
-    Context->InstallParamType(Ty);
-  }
-
-  NaClBcIndexSize_t GetNumParams() const {
-    return Context->GetNumParams();
-  }
-
-  void InstallConstantType(Type *ConstantType) {
-    Context->InstallConstantType(ConstantType);
-  }
-
-  Type *GetConstantType(NaClBcIndexSize_t Index) {
-    return Context->GetConstantType(Index);
-  }
-
-  NaClBcIndexSize_t GetNumConstants() const {
-    return Context->GetNumConstants();
-  }
-
-  void InstallInstType(Type *Ty, NaClBcIndexSize_t Index) {
-    Context->InstallInstType(Ty, Index);
-  }
-
-  void InstallInstType(Type *Ty) {
-    Context->InstallInstType(Ty);
-  }
-
-  NaClBcIndexSize_t GetNumValuedInstructions() const {
-    return Context->GetNumValuedInstructions();
-  }
-
-  Type *GetValueType(NaClBcIndexSize_t Id, bool UnderlyingType = false) {
-    return Context->GetValueType(Id, UnderlyingType);
-  }
-
-  Type *GetFunctionValueType(NaClBcIndexSize_t Id) {
-    return GetValueType(Id, /* UnderlyingType = */ true);
-  }
-
-  Type *GetUnknownType() const {
-    return Context->GetUnknownType();
-  }
-
-  Type *GetPointerType() const {
-    return Context->GetPointerType();
-  }
-
-  Type *GetComparisonType() const {
-    return Context->GetComparisonType();
-  }
-
-  Type *GetVoidType() const {
-    return Context->GetVoidType();
-  }
-
-  Type *GetFloatType() const {
-    return Context->GetFloatType();
-  }
-
-  Type *GetDoubleType() const {
-    return Context->GetDoubleType();
-  }
-
-  Type *GetIntegerType(unsigned Size) const {
-    return Context->GetIntegerType(Size);
-  }
-
-  const char *VerifyIntArithmeticOp(const char *Op, Type *OpTy) {
-    return Context->VerifyIntArithmeticOp(Op, OpTy);
-  }
-
-protected:
-  // The context parser that contains decoding state.
-  NaClDisTopLevelParser *Context;
-  // The number of global abbreviations defined for this block.
-  NaClBcIndexSize_t NumGlobalAbbreviations;
-  // The current number of local abbreviations defined for this block.
-  NaClBcIndexSize_t NumLocalAbbreviations;
-};
-
-bool NaClDisBlockParser::ParseBlock(unsigned BlockId) {
-  // Only called if we don't know the details about the block.
-  ObjDumpSetRecordBitAddress(GetBlock().GetStartBit());
-  Errors() << "Don't know how to parse block " << BlockId
-           << ", when in block " << GetBlockID() << "\n";
-  NaClDisBlockParser Parser(BlockId, this);
-  return Parser.ParseThisBlock();
-}
-
-void NaClDisBlockParser::PrintBlockHeader() {
-  Errors() << "Unknown block id found: " << GetBlockID() << "\n";
-  Tokens() << "unknown" << Space() << OpenCurly()
-           << Space() << Space() << "// BlockID = "
-           << GetBlockID() << Endline();
-}
-
-void NaClDisBlockParser::EnterBlock(unsigned NumWords) {
-  PrintBlockHeader();
-  DumpEnterBlockRecord();
-  IncRecordIndent();
-  IncAssemblyIndent();
-}
-
-void NaClDisBlockParser::ExitBlock() {
-  DecAssemblyIndent();
-  DecRecordIndent();
-  Tokens() << CloseCurly() << Endline();
-  NaClBitcodeRecordData Exit;
-  Exit.Code = naclbitc::BLK_CODE_EXIT;
-  ObjDumpWrite(Record.GetStartBit(), Exit, naclbitc::END_BLOCK);
-}
-
-void NaClDisBlockParser::DumpEnterBlockRecord() {
-  // TODO(kschimpf): Better integrate this with the bitstream reader
-  // (which currently doesn't build any records).
-  NaClBitcodeRecordData Enter;
-  Enter.Code = naclbitc::BLK_CODE_ENTER;
-  Enter.Values.push_back(GetBlockID());
-  Enter.Values.push_back(Record.GetCursor().getAbbrevIDWidth());
-  ObjDumpWrite(GetBlock().GetStartBit(), Enter, naclbitc::ENTER_SUBBLOCK);
-}
-
-void NaClDisBlockParser::ProcessAbbreviation(unsigned BlockID,
-                                             NaClBitCodeAbbrev *Abbrev,
-                                             bool IsLocal) {
-  Tokens() << NextLocalAbbreviationId() << Space() << "=" << Space()
-           << "abbrev" << Space() << TokenizeAbbreviation(Abbrev)
-           << Semicolon() << Endline();
-  ++NumLocalAbbreviations;
-  ObjDumpWrite(Record.GetStartBit(), Record);
-}
-
-void NaClDisBlockParser::ProcessRecord() {
-  // Note: Only called if block is not understood. Hence, we
-  // only report the records.
-  ObjDumpWrite(Record.GetStartBit(), Record);
-}
-
-/// Parses and disassembles the blockinfo block.
-class NaClDisBlockInfoParser : public NaClDisBlockParser {
-public:
-  NaClDisBlockInfoParser(unsigned BlockID,
-                         NaClDisBlockParser *EnclosingParser)
-      : NaClDisBlockParser(BlockID, EnclosingParser) {
-  }
-
-  ~NaClDisBlockInfoParser() override {}
-
-private:
-  void PrintBlockHeader() override;
-
-  void SetBID() override;
-
-  void ProcessAbbreviation(unsigned BlockID, NaClBitCodeAbbrev *Abbrev,
-                           bool IsLocal) override;
-
-  /// Returns the abbreviation id for the next global abbreviation
-  /// to be defined for the given block id.
-  BitcodeId NextGlobalAbbreviationId(unsigned BlockID) {
-    return BitcodeId('a', Context->GetNumGlobalAbbreviations(BlockID), true);
-  }
-};
-
-void NaClDisBlockInfoParser::PrintBlockHeader() {
-  Tokens() << "abbreviations" << Space() << OpenCurly()
-           << Space() << Space() << "// BlockID = "
-           << GetBlockID() << Endline();
-}
-
-void NaClDisBlockInfoParser::SetBID() {
-  std::string BlockName;
-  uint64_t BlockID = Record.GetValues()[0];
-  switch (BlockID) {
-  case naclbitc::MODULE_BLOCK_ID:
-    Tokens() << "module";
-    break;
-  case naclbitc::CONSTANTS_BLOCK_ID:
-    Tokens() << "constants";
-    break;
-  case naclbitc::FUNCTION_BLOCK_ID:
-    Tokens() << "function";
-    break;
-  case naclbitc::VALUE_SYMTAB_BLOCK_ID:
-    Tokens() << "valuesymtab";
-    break;
-  case naclbitc::TYPE_BLOCK_ID_NEW:
-    Tokens() << "types";
-    break;
-  case naclbitc::GLOBALVAR_BLOCK_ID:
-    Tokens() << "globals";
-    break;
-  default:
-    Tokens() << "block" << OpenParen() << BlockID << CloseParen();
-    Errors() << "Block id " << BlockID << " not understood.\n";
-    break;
-  }
-  Tokens() << Colon() << Endline();
-  ObjDumpWrite(Record.GetStartBit(), Record);
-}
-
-void NaClDisBlockInfoParser::ProcessAbbreviation(unsigned BlockID,
-                                                 NaClBitCodeAbbrev *Abbrev,
-                                                 bool IsLocal) {
-  IncAssemblyIndent();
-  Tokens() << NextGlobalAbbreviationId(BlockID) << Space() << "=" << Space()
-           << "abbrev" << Space() << TokenizeAbbreviation(Abbrev)
-           << Semicolon() << Endline();
-  Context->IncNumGlobalAbbreviations(BlockID);
-  DecAssemblyIndent();
-  ObjDumpWrite(Record.GetStartBit(), Record);
-}
-
-/// Parses and disassembles the types block.
-class NaClDisTypesParser : public NaClDisBlockParser {
-public:
-  NaClDisTypesParser(unsigned BlockID,
-                     NaClDisBlockParser *EnclosingParser)
-      : NaClDisBlockParser(BlockID, EnclosingParser),
-        ExpectedNumTypes(0),
-        IsFirstRecord(true)
-  {
-  }
-
-  ~NaClDisTypesParser() override;
-
-private:
-  void PrintBlockHeader() override;
-
-  void ProcessRecord() override;
-
-  /// Returns the value id for the next type to be defined.
-  BitcodeId NextTypeId() {
-    return BitcodeId('t', GetNumTypes());
-  }
-
-  // Installs unknown type as definition of next type.
-  void InstallUnknownTypeForNextId();
-
-
-  NaClBcIndexSize_t ExpectedNumTypes;
-  bool IsFirstRecord;
-};
-
-NaClDisTypesParser::~NaClDisTypesParser() {
-  if (GetNumTypes() != ExpectedNumTypes) {
-    Errors() << "Expected " << ExpectedNumTypes << " types but found: "
-             << GetNumTypes() << "\n";
-  }
-}
-
-void NaClDisTypesParser::PrintBlockHeader() {
-  Tokens() << "types" << Space() << OpenCurly()
-           << Space() << Space() << "// BlockID = " << GetBlockID()
-           << Endline();
-}
-
-void NaClDisTypesParser::InstallUnknownTypeForNextId() {
-  Type *UnknownType = GetUnknownType();
-  Tokens() << NextTypeId() << Space() << "=" << Space()
-           << TokenizeType(UnknownType) << Semicolon()
-           << TokenizeAbbrevIndex() << Endline();
-  InstallType(UnknownType);
-}
-
-void NaClDisTypesParser::ProcessRecord() {
-  ObjDumpSetRecordBitAddress(Record.GetStartBit());
-  const NaClBitcodeRecord::RecordVector &Values = Record.GetValues();
-  switch (Record.GetCode()) {
-  case naclbitc::TYPE_CODE_NUMENTRY: {
-    // NUMENTRY: [numentries]
-    uint64_t Size = 0;
-    if (Values.size() == 1) {
-      Size = Values[0];
-    } else {
-      Errors() << "Count record should have 1 argument. Found: "
-               << Values.size() << "\n";
-    }
-    if (!IsFirstRecord) {
-      Errors() << "Count record not first record of types block\n";
-    }
-    if (Size > NaClBcIndexSize_t_Max) {
-      Errors() << "Count record size too big: " << Size << "\n";
-    }
-    Tokens() << "count" << Space() << Size << Semicolon()
-             << TokenizeAbbrevIndex() << Endline();
-    ExpectedNumTypes = Size;
-    break;
-  }
-  case naclbitc::TYPE_CODE_VOID: {
-    // VOID
-    if (!Values.empty())
-      Errors() << "Void record shouldn't have arguments. Found: "
-               << Values.size() << "\n";
-    Type *VoidType = GetVoidType();
-    Tokens() << NextTypeId() << Space() << "=" << Space()
-             << TokenizeType(VoidType) << Semicolon()
-             << TokenizeAbbrevIndex() << Endline();
-    InstallType(VoidType);
-    break;
-  }
-  case naclbitc::TYPE_CODE_FLOAT: {
-    // FLOAT
-    if (!Values.empty())
-      Errors() << "Float record shoudn't have arguments. Found: "
-               << Values.size() << "\n";
-    Type *FloatType = GetFloatType();
-    Tokens() << NextTypeId() << Space() << "=" << Space()
-             << TokenizeType(FloatType) << Semicolon()
-             << TokenizeAbbrevIndex() << Endline();
-    InstallType(FloatType);
-    break;
-  }
-  case naclbitc::TYPE_CODE_DOUBLE: {
-    // DOUBLE
-    if (!Values.empty())
-      Errors() << "Double record shound't have arguments. Found: "
-               << Values.size() << "\n";
-    Type *DoubleType = GetDoubleType();
-    Tokens() << NextTypeId() << Space() << "=" << Space()
-             << TokenizeType(DoubleType) << Semicolon()
-             << TokenizeAbbrevIndex() << Endline();
-    InstallType(DoubleType);
-    break;
-  }
-  case naclbitc::TYPE_CODE_INTEGER: {
-    // INTEGER: [width]
-    uint64_t Size;
-    if (Values.size() == 1) {
-      Size = Values[0];
-    } else {
-      Errors() << "Integer record should have one argument. Found: "
-               << Values.size() << "\n";
-      Size = 32;
-    }
-    switch (Size) {
-      case 1:
-      case 8:
-      case 16:
-      case 32:
-      case 64: {
-        break;
-      }
-      default:
-        if (!IgnorePNaClABIChecks) {
-          Errors() << "Integer record contains bad integer size: "
-                   << Size << "\n";
-          Size = 32;
-        }
-        break;
-    }
-    Type *IntType = GetIntegerType(Size);
-    Tokens() << NextTypeId() << Space() << "=" << Space()
-             << TokenizeType(IntType) << Semicolon()
-             << TokenizeAbbrevIndex() << Endline();
-    InstallType(IntType);
-    break;
-  }
-  case naclbitc::TYPE_CODE_VECTOR: {
-    // VECTOR: [numelts, eltty]
-    if (Values.size() != 2) {
-      Errors() << "Vector record should contain two arguments. Found: "
-               << Values.size() << "\n";
-      InstallUnknownTypeForNextId();
-      break;
-    }
-    Type *BaseType = GetType(Values[1]);
-    if (!(BaseType->isIntegerTy()
-          || BaseType->isFloatTy()
-          || BaseType->isDoubleTy())) {
-      Type *ErrorRecoveryTy = GetIntegerType(32);
-      Errors() << "Vectors can only be defined on primitive types. Found "
-               << *BaseType << ". Assuming " << *ErrorRecoveryTy
-               << " instead.\n";
-      BaseType = ErrorRecoveryTy;
-    }
-    uint64_t NumElements = Values[0];
-    Type *VecType = VectorType::get(BaseType, NumElements);
-    if (!IgnorePNaClABIChecks &&
-        !PNaClABITypeChecker::isValidVectorType(VecType)) {
-      Errors() << "Vector type " << *VecType << " not allowed.\n";
-    }
-    Tokens() << NextTypeId() << Space() << "=" << Space()
-             << TokenizeType(VecType) << Semicolon()
-             << TokenizeAbbrevIndex() << Endline();
-    InstallType(VecType);
-    break;
-  }
-  case naclbitc::TYPE_CODE_FUNCTION: {
-    // FUNCTION: [vararg, retty, paramty x N]
-    if (Values.size() < 2) {
-      Errors()
-          << "Function record should contain at least 2 arguments. Found: "
-          << Values.size() << "\n";
-      InstallUnknownTypeForNextId();
-      break;
-    }
-    if (Values[0]) {
-      Errors() << "Functions with variable length arguments is not supported\n";
-    }
-    Type *ReturnType = GetType(Values[1]);
-    if (!(isValidValueType(ReturnType) || ReturnType->isVoidTy())) {
-      Type *ReplaceType = GetIntegerType(32);
-      Errors() << "Invalid return type. Found: " << *ReturnType
-               << ". Assuming: " << *ReplaceType << "\n";
-      ReturnType = ReplaceType;
-    }
-    SmallVector<Type*, 8> Signature;
-    for (size_t i = 2; i < Values.size(); ++i) {
-      Type *Ty = GetType(Values[i]);
-      if (!isValidValueType(Ty)) {
-        Type *ReplaceTy = GetIntegerType(32);
-        Errors() << "Invalid type for parameter " << (i - 1) << ". Found: "
-                 << *Ty << ". Assuming: " << *ReplaceTy << "\n";
-        Ty = ReplaceTy;
-      }
-      Signature.push_back(Ty);
-    }
-    Type *FcnType =  FunctionType::get(ReturnType, Signature, Values[0]);
-    Tokens() << NextTypeId() << Space() << "=" << Space()
-             << StartCluster() << TokenizeType(FcnType)
-             << Semicolon() << FinishCluster() << TokenizeAbbrevIndex()
-             << Endline();
-    InstallType(FcnType);
-    break;
-  }
-  default:
-    Errors() << "Unknown record code in types block. Found: "
-             << Record.GetCode() << "\n";
-    break;
-  }
-  ObjDumpWrite(Record.GetStartBit(), Record);
-  IsFirstRecord = false;
-}
-
-/// Parses and disassembles the globalvars block.
-class NaClDisGlobalsParser : public NaClDisBlockParser {
-public:
-  NaClDisGlobalsParser(unsigned BlockID,
-                       NaClDisBlockParser *EnclosingParser)
-      : NaClDisBlockParser(BlockID, EnclosingParser),
-        NumInitializers(0),
-        InsideCompound(false),
-        BaseTabs(GetAssemblyNumTabs()+1) {}
-
-  ~NaClDisGlobalsParser() override {}
-
-private:
-  void PrintBlockHeader() override;
-
-  void ProcessRecord() override;
-
-  void ExitBlock() override;
-
-  // Expected number of initializers associated with last globalvars
-  // record
-  NaClBcIndexSize_t NumInitializers;
-  // True if last globalvars record was defined by a compound record.
-  bool InsideCompound;
-  // Number of tabs used to indent elements in the globals block.
-  unsigned BaseTabs;
-
-  // Returns the ID for the next defined global.
-  BitcodeId NextGlobalId() {
-    return BitcodeId('g', GetNumGlobals());
-  }
-
-  // Prints out the close initializer "}" if necessary, and fixes
-  // the indentation to match previous indentation.
-  void InsertCloseInitializer();
-
-  uint32_t GetExpectedNumGlobals() const {
-    return Context->GetExpectedNumGlobals();
-  }
-};
-
-void NaClDisGlobalsParser::PrintBlockHeader() {
-  Tokens() << "globals" << Space() << OpenCurly()
-           << Space() << Space() << "// BlockID = " << GetBlockID()
-           << Endline();
-}
-
-void NaClDisGlobalsParser::InsertCloseInitializer() {
-  if (InsideCompound) {
-    while (BaseTabs + 1 < GetAssemblyNumTabs()) DecAssemblyIndent();
-    Tokens() << CloseCurly() << Endline();
-  }
-  while (BaseTabs < GetAssemblyNumTabs()) DecAssemblyIndent();
-  ObjDumpFlush();
-  NumInitializers = 0;
-  InsideCompound = false;
-}
-
-void NaClDisGlobalsParser::ExitBlock() {
-  if (NumInitializers > 0) {
-    BitcodeId LastGlobal('g', (GetNumGlobals()-1));
-    Errors() << "More initializers for " << LastGlobal << " expected: "
-             << NumInitializers << "\n";
-  }
-  if (GetNumGlobals() != GetExpectedNumGlobals()) {
-    Errors() << "Expected " << GetExpectedNumGlobals()
-             << " globals but found: " << GetNumGlobals() << "\n";
-  }
-  NaClDisBlockParser::ExitBlock();
-}
-
-void NaClDisGlobalsParser::ProcessRecord() {
-  ObjDumpSetRecordBitAddress(Record.GetStartBit());
-  const NaClBitcodeRecord::RecordVector &Values = Record.GetValues();
-  switch (Record.GetCode()) {
-  case naclbitc::GLOBALVAR_VAR: {
-    // VAR: [align, isconst]
-    if (Values.size() != 2) {
-      Errors() << "Globalvar record expects two arguments. Found: "
-               << Values.size() << "\n";
-      break;
-    }
-    if (GetNumGlobals() == GetExpectedNumGlobals()) {
-      Errors() << "Exceeded expected number of globals: "
-               << GetExpectedNumGlobals() << "\n";
-    }
-    if (NumInitializers > 0) {
-      Errors() << "Previous initializer list too short. Expects "
-               << NumInitializers << " more initializers\n";
-      InsertCloseInitializer();
-    }
-    uint32_t Alignment = (1 << Values[0]) >> 1;
-    Tokens() << StartCluster() << (Values[1] ? "const" : "var")
-             << Space() << NextGlobalId()
-             << Comma() << FinishCluster() << Space()
-             << StartCluster() << "align" << Space() << Alignment
-             << Comma() << FinishCluster() << TokenizeAbbrevIndex()
-             << Endline();
-    IncAssemblyIndent();
-    IncNumGlobals();
-    NumInitializers = 1;
-    break;
-  }
-  case naclbitc::GLOBALVAR_COMPOUND:
-    // COMPOUND: [size]
-    if (Values.size() != 1) {
-      Errors() << "Compound record expects one argument. Found: "
-               << Values.size() << "\n";
-      break;
-    }
-    if (NumInitializers == 1) {
-      if (InsideCompound) {
-        Errors() << "Nested initialization records not allowed\n";
-        InsertCloseInitializer();
-      }
-    } else {
-      Errors() << "Compound record must follow globalvars record\n";
-      InsertCloseInitializer();
-    }
-    Tokens() << StartCluster() << "initializers" << Space()
-             << Values[0] << FinishCluster() << Space()
-             << OpenCurly() << TokenizeAbbrevIndex() << Endline();
-    // Check that ILP32 assumption met.
-    if (Values[0] > MaxNaClGlobalVarInits)
-      Errors() << "Too many initializers for global var: "
-               << Values[0] << "\n";
-    IncAssemblyIndent();
-    NumInitializers = Values[0];
-    InsideCompound = true;
-    break;
-  case naclbitc::GLOBALVAR_ZEROFILL:
-    // ZEROFILL: [size]
-    if (Values.size() != 1) {
-      Errors() << "Zerofill record expects one argument. Found: "
-               << Values.size() << "\n";
-      break;
-    }
-    if (NumInitializers == 0) {
-      Errors() << "Zerofill initializer not associated with globalvar record\n";
-    }
-    Tokens() << "zerofill" << Space() << Values[0] << Semicolon()
-             << TokenizeAbbrevIndex() << Endline();
-    --NumInitializers;
-    break;
-  case naclbitc::GLOBALVAR_DATA: {
-    // DATA: [b0, b1, ...]
-    if (Values.empty()) {
-      Errors() << "Globals data record must have arguments.\n";
-    }
-    if (NumInitializers == 0) {
-      Errors() << "Data initializer not associated with globalvar record\n";
-    }
-    Tokens() << StartCluster() << OpenCurly();
-    for (size_t i = 0; i < Values.size(); ++i) {
-      if (i > 0) Tokens() << Comma() << FinishCluster() << Space();
-      uint64_t Byte = Values[i];
-      if (Byte >= 256) {
-        Errors() << "Invalid byte value in data record: " << Byte << "\n";
-        Byte &= 0xFF;
-      }
-      if (i > 0) Tokens() << StartCluster();
-      Tokens() << format("%3u", static_cast<unsigned>(Byte));
-    }
-    Tokens() << CloseCurly() << FinishCluster() << TokenizeAbbrevIndex()
-             << Endline();
-    --NumInitializers;
-    break;
-  }
-  case naclbitc::GLOBALVAR_RELOC:
-    // RELOC: [val, [addend]]
-    if (Values.empty() || Values.size() > 2) {
-      Errors() << "Invalid reloc record size: " << Values.size() << "\n";
-      break;
-    }
-    if (NumInitializers == 0) {
-      Errors() <<
-          "Relocation initializer not associated with globalvar record\n";
-    }
-    Tokens() << "reloc" << Space() << StartCluster() << GetBitcodeId(Values[0]);
-    if (Values.size() == 2) {
-      int32_t Addend = static_cast<int32_t>(Values[1]);
-      char Operator = '+';
-      if (Addend < 0) {
-        Operator = '-';
-        Addend = -Addend;
-      }
-      Tokens() << Space()<< Operator << Space() << Addend;
-    }
-    Tokens() << Semicolon() << FinishCluster() << TokenizeAbbrevIndex()
-             << Endline();
-    --NumInitializers;
-    break;
-  case naclbitc::GLOBALVAR_COUNT: {
-    // COUNT: [n]
-    uint64_t Count = 0;
-    if (Values.size() == 1) {
-      Count = Values[0];
-    } else {
-      Errors() << "Globals count record expects one argument. Found: "
-               << Values.size() << "\n";
-    }
-    if (GetNumGlobals() != 0)
-      Errors() << "Count record not first record in block.\n";
-    Tokens() << "count" << Space() << Count << Semicolon()
-             << TokenizeAbbrevIndex() << Endline();
-    if (Count > NaClBcIndexSize_t_Max)
-      Errors() << "Count record size too big: " << Count << "\n";
-    Context->SetExpectedNumGlobals(Count);
-    break;
-  }
-  default:
-    Errors() << "Unknown record found in globals block.\n";
-  }
-
-  ObjDumpWrite(Record.GetStartBit(), Record);
-
-  if (GetNumGlobals() > 0 && NumInitializers == 0)
-    InsertCloseInitializer();
-}
-
-/// Parsers and disassembles a valuesymtab block.
-class NaClDisValueSymtabParser : public NaClDisBlockParser {
-public:
-  NaClDisValueSymtabParser(unsigned BlockID,
-                           NaClDisBlockParser *EnclosingParser)
-      : NaClDisBlockParser(BlockID, EnclosingParser) {
-  }
-
-  ~NaClDisValueSymtabParser() override {}
-
-private:
-  void PrintBlockHeader() override;
-
-  void ProcessRecord() override;
-
-  // Displays the context of the name (in Values) for the given Id.
-  void DisplayEntry(BitcodeId &Id,
-                    const NaClBitcodeRecord::RecordVector &Values);
-};
-
-void NaClDisValueSymtabParser::PrintBlockHeader() {
-  Tokens() << "valuesymtab" << Space() << OpenCurly()
-           << Space() << Space() << "// BlockID = " << GetBlockID()
-           << Endline();
-}
-
-void NaClDisValueSymtabParser::
-DisplayEntry(BitcodeId &Id,
-             const NaClBitcodeRecord::RecordVector &Values) {
-  if (Values.size() <= 1) {
-    Errors() << "Valuesymtab entry record expects 2 arguments. Found: "
-             << Values.size() << "\n";
-    return;
-  }
-  Tokens() << Id << Space() << Colon() << Space();
-  // Check if the name of the symbol is alphanumeric. If so, print
-  // as a string. Otherwise, print a sequence of bytes.
-  // Note: The check isChar6 is a test for aphanumeric + {'.', '_'}.
-  bool IsChar6 = true;  // Until proven otherwise.
-  for (size_t i = 1; i < Values.size(); ++i) {
-    uint64_t Byte = Values[i];
-    if (Byte >= 256) {
-      Errors() << "Argument " << i << " of symbol entry not byte: "
-               << Byte << "\n";
-      IsChar6 = false;
-      break;
-    }
-    if (!NaClBitCodeAbbrevOp::isChar6(Byte))
-      IsChar6 = false;
-  }
-  if (IsChar6) {
-    Tokens() << StartCluster() << "\"";
-    for (size_t i = 1; i < Values.size(); ++i) {
-      Tokens() << static_cast<char>(Values[i]);
-    }
-    Tokens() << "\"" << Semicolon();
-  } else {
-    Tokens() << StartCluster() << OpenCurly();
-    for (size_t i = 1; i < Values.size(); ++i) {
-      if (i > 1) {
-        Tokens() << Comma() << FinishCluster() << Space()
-                 << StartCluster();
-      }
-      char ch = Values[i];
-      if (NaClBitCodeAbbrevOp::isChar6(ch)) {
-        Tokens() << "'" << ch << "'";
-      } else {
-        Tokens() << format("%3u", static_cast<unsigned>(ch));
-      }
-    }
-    Tokens() << CloseCurly();
-  }
-  Tokens() << FinishCluster() << TokenizeAbbrevIndex() << Endline();
-}
-
-void NaClDisValueSymtabParser::ProcessRecord() {
-  ObjDumpSetRecordBitAddress(Record.GetStartBit());
-  const NaClBitcodeRecord::RecordVector &Values = Record.GetValues();
-  switch (Record.GetCode()) {
-  case naclbitc::VST_CODE_ENTRY: {
-    // VST_ENTRY: [valueid, namechar x N]
-    NaClBcIndexSize_t ValID = Values[0];
-    BitcodeId ID(GetBitcodeId(ValID));
-    DisplayEntry(ID, Values);
-    if (ValID < GetNumFunctions()) {
-      // Check if value is intrinsic name. If so, verify function signature.
-      std::string Name;
-      for (size_t i = 1; i < Values.size(); ++i) {
-        Name.push_back(static_cast<char>(Values[i]));
-      }
-      FunctionType *IntrinTy = Context->GetIntrinsicType(Name);
-      if (IntrinTy == 0)
-        // TODO(kschimpf): Check for _start? Complain about other names?
-        break;
-      Context->MarkFunctionAsIntrinsic(ValID);
-      // Verify that expected intrinsic function type matches declared
-      // type signature.
-      FunctionType *FcnTy = GetFunctionType(ValID);
-      if (FcnTy->getNumParams() != IntrinTy->getNumParams()) {
-        Errors() << "Intrinsic " << Name
-                 << " expects " << IntrinTy->getNumParams()
-                 << " arguments. Found: " << FcnTy->getNumParams()
-                 << "\n";
-        break;
-      }
-      if (!IgnorePNaClABIChecks && !PNaClABITypeChecker::IsPointerEquivType(
-              IntrinTy->getReturnType(), FcnTy->getReturnType())) {
-        Errors() << "Intrinsic " << Name
-                 << " expects return type " << *IntrinTy->getReturnType()
-                 << ". Found: " << *FcnTy->getReturnType() << "\n";
-        break;
-      }
-      for (size_t i = 0; i < FcnTy->getNumParams(); ++i) {
-        if (!IgnorePNaClABIChecks && !PNaClABITypeChecker::IsPointerEquivType(
-                IntrinTy->getParamType(i),
-                FcnTy->getParamType(i))) {
-          Errors() << "Intrinsic " << Name
-                   << " expects " << *IntrinTy->getParamType(i)
-                   << " for argument " << (i+1) << ". Found: "
-                   << *FcnTy->getParamType(i) << "\n";
-          break;
-        }
-      }
-    }
-    break;
-  }
-  case naclbitc::VST_CODE_BBENTRY: {
-    // [bbid, namechar x N]
-    BitcodeId ID('b', Values[0]);
-    DisplayEntry(ID, Values);
-    break;
-  }
-  default:
-    Errors() << "Unknown record in valuesymtab block.\n";
-    break;
-  }
-  ObjDumpWrite(Record.GetStartBit(), Record);
-}
-
-/// Parses and disassembles a constants block.
-class NaClDisConstantsParser : public NaClDisBlockParser {
-public:
-  NaClDisConstantsParser(unsigned BlockID,
-                         NaClDisBlockParser *EnclosingParser)
-      : NaClDisBlockParser(BlockID, EnclosingParser),
-        ConstantType(0),
-        BlockTabs(GetAssemblyNumTabs()) {}
-
-  virtual ~NaClDisConstantsParser() {
-    while (BlockTabs < GetAssemblyNumTabs()) DecAssemblyIndent();
-  }
-
-private:
-  void PrintBlockHeader() override;
-
-  void ProcessRecord() override;
-
-  /// Generates the value id for the next generated constant.
-  BitcodeId NextConstId() {
-    return BitcodeId('c', GetNumConstants());
-  }
-
-  // The type associated with the constant.
-  Type *ConstantType;
-  // The number of tabs used to indent the globals block.
-  unsigned BlockTabs;
-};
-
-void NaClDisConstantsParser::PrintBlockHeader() {
-  Tokens() << "constants" << Space() << OpenCurly()
-           << Space() << Space() << "// BlockID = " << GetBlockID()
-           << Endline();
-}
-
-void NaClDisConstantsParser::ProcessRecord() {
-  ObjDumpSetRecordBitAddress(Record.GetStartBit());
-  const NaClBitcodeRecord::RecordVector Values = Record.GetValues();
-  switch (Record.GetCode()) {
-  case naclbitc::CST_CODE_SETTYPE:
-    // SETTYPE: [typeid]
-    while (BlockTabs + 1 < GetAssemblyNumTabs()) DecAssemblyIndent();
-    if (Values.size() == 1) {
-      ConstantType = GetType(Values[0]);
-      Tokens() << TokenizeType(ConstantType) << ":"
-               << TokenizeAbbrevIndex() << Endline();
-    } else {
-      Errors() << "Settype record should have 1 argument. Found: "
-               << Values.size() << "\n";
-      // Make up a type, so we can continue.
-      ConstantType = GetIntegerType(32);
-    }
-    IncAssemblyIndent();
-    break;
-  case naclbitc::CST_CODE_UNDEF:
-    // CST_CODE_UNDEF: []
-    if (!Values.empty()) {
-      Errors() << "Undefined record should not have arguments: Found: "
-               << Values.size() << "\n";
-    }
-    Tokens() << NextConstId() << Space() << "=" << Space()
-             << StartCluster() << TokenizeType(ConstantType)
-             << Space() << "undef"
-             << Semicolon() << FinishCluster()
-             << TokenizeAbbrevIndex() << Endline();
-    InstallConstantType(ConstantType);
-    break;
-  case naclbitc::CST_CODE_INTEGER: {
-    // INTEGER: [intval]
-    SignRotatedInt Value;
-    if (Values.size() == 1) {
-      const SignRotatedInt MyValue(Values[0], ConstantType);
-      Value = MyValue;
-    } else {
-      Errors() << "Integer record should have 1 argument. Found: "
-               << Values.size() << "\n";
-    }
-    Tokens() << NextConstId() << Space() << "=" << Space() << StartCluster()
-             << StartCluster() << TokenizeType(ConstantType) << Space()
-             << Value << Semicolon() << FinishCluster() << FinishCluster()
-             << TokenizeAbbrevIndex() << Endline();
-    InstallConstantType(ConstantType);
-    break;
-  }
-  case naclbitc::CST_CODE_FLOAT: {
-    // FLOAT: [fpval]
-    // Define initially with default value zero, in case of errors.
-    const fltSemantics &FloatRep =
-        ConstantType->isFloatTy() ? APFloat::IEEEsingle : APFloat::IEEEdouble;
-    APFloat Value(APFloat::getZero(FloatRep));
-    if (Values.size() == 1) {
-      if (ConstantType->isFloatTy()) {
-        Value = APFloat(FloatRep, APInt(32, static_cast<uint32_t>(Values[0])));
-      }
-      else if (ConstantType->isDoubleTy()) {
-        Value = APFloat(FloatRep, APInt(64, Values[0]));
-      } else {
-        Errors() << "Bad floating point constant argument: "
-                 << Values[0] << "\n";
-      }
-    } else {
-      Errors() << "Float record should have 1 argument. Found: "
-               << Values.size() << "\n";
-    }
-    Tokens() << NextConstId() << Space() << "=" << Space() << StartCluster()
-             << TokenizeType(ConstantType) << Space();
-    if (ConstantType->isFloatTy()) {
-      Tokens() << format("%g", Value.convertToFloat());
-    } else {
-      Tokens() << format("%g", Value.convertToDouble());
-    }
-    Tokens() << Semicolon() << FinishCluster()
-             << TokenizeAbbrevIndex() << Endline();
-    InstallConstantType(ConstantType);
-    break;
-  }
-  default:
-    Errors() << "Unknown record in valuesymtab block.\n";
-    break;
-  }
-  ObjDumpWrite(Record.GetStartBit(), Record);
-}
-
-/// Parses and disassembles function blocks.
-class NaClDisFunctionParser : public NaClDisBlockParser {
-public:
-  NaClDisFunctionParser(unsigned BlockID,
-                        NaClDisBlockParser *EnclosingParser);
-
-  ~NaClDisFunctionParser() override {}
-
-  void PrintBlockHeader() override;
-
-  bool ParseBlock(unsigned BlockID) override;
-
-  void ProcessRecord() override;
-
-  /// Returns the absolute value index of the first instruction that
-  /// generates a value in the defined function.
-  NaClBcIndexSize_t FirstValuedInstructionId() const {
-    return GetNumFunctions() + GetNumGlobals() + GetNumParams()
-        + GetNumConstants();
-  }
-
-  /// Converts an instruction index to the corresponding absolute value
-  /// index.
-  NaClBcIndexSize_t ValuedInstToAbsId(NaClBcIndexSize_t Id) const {
-    return FirstValuedInstructionId() + Id;
-  }
-
-  /// Converts the (function) relative index to the corresponding
-  /// absolute value index.
-  NaClBcIndexSize_t RelativeToAbsId(NaClRelBcIndexSize_t Id) {
-    NaClBcIndexSize_t AbsNextId = ValuedInstToAbsId(GetNumValuedInstructions());
-    if (Id > 0 && AbsNextId < static_cast<uint32_t>(Id)) {
-      Errors() << "Invalid relative value id: " << Id
-               << " (Must be <= " << AbsNextId << ")\n";
-      AbsNextId = Id;
-    }
-    return AbsNextId - Id;
-  }
-
-  /// Converts the given absolute value index to a corresponding
-  /// valued instruction index.
-  NaClBcIndexSize_t AbsToValuedInstId(NaClBcIndexSize_t Id) const {
-    return Id - FirstValuedInstructionId();
-  }
-
-  /// Returns the text representation of the encoded binary operator,
-  /// assuming the binary operator is operating on the given type.
-  /// Generates appropriate error messages if the given type is not
-  // allowed for the given Opcode.
-  const char *GetBinop(uint32_t Opcode, Type *Type);
-
-  /// Returns the text representation of the encoded cast operator.
-  /// Generates appropriate error messages if the given Opcode is
-  /// not allowed for the given types.
-  const char *GetCastOp(uint32_t Opcode, Type *FromType, Type *ToType);
-
-  /// Returns the text representation of the encoded integer compare
-  /// predicate. Generates appropriate error message if the given
-  /// Opcode is not defined.
-  const char *GetIcmpPredicate(uint32_t Opcode);
-
-  /// Returns the text representation of the encoded floating point
-  /// compare predicate. Generates appropriate error message if the
-  /// given Opcode is not defined.
-  const char *GetFcmpPredicate(uint32_t Opcode);
-
-  /// Returns the value id for the next instruction to be defined.
-  BitcodeId NextInstId() const {
-    return BitcodeId('v', GetNumValuedInstructions());
-  }
-
-private:
-  // The function index of the function being defined.
-  NaClBcIndexSize_t FcnId;
-  // The function type for the function being defined.
-  FunctionType *FcnTy;
-  // The current basic block being printed.
-  NaClBcIndexSize_t CurrentBbIndex;
-  // The expected number of basic blocks.
-  NaClBcIndexSize_t ExpectedNumBbs;
-  // True if the previously printed instruction was a terminating
-  // instruction. Used to figure out where to insert block labels.
-  bool InstIsTerminating;
-
-  /// Returns scalar type of vector type Ty, if vector type. Otherwise
-  /// returns Ty.
-  Type *UnderlyingType(Type *Ty) {
-    return Ty->isVectorTy() ? Ty->getVectorElementType() : Ty;
-  }
-
-  bool isFloatingType(Type *Ty) {
-    return Ty->isFloatTy() || Ty->isDoubleTy();
-  }
-
-  /// Verifies that OpTy is an integer, or a vector of integers, for
-  /// operator Op. Generates error messages if appropriate. Returns Op.
-  const char *VerifyIntegerOrVectorOp(const char *Op, Type *OpTy) {
-    Type *BaseTy = OpTy;
-    if (OpTy->isVectorTy()) {
-      if (!IgnorePNaClABIChecks &&
-          !PNaClABITypeChecker::isValidVectorType(OpTy)) {
-        Errors() << Op << ": invalid vector type: " << *OpTy << "\n";
-        return Op;
-      }
-      BaseTy = OpTy->getVectorElementType();
-    }
-    if (BaseTy->isIntegerTy()) {
-      if (IgnorePNaClABIChecks ||
-          PNaClABITypeChecker::isValidScalarType(BaseTy)) {
-        return Op;
-      }
-      Errors() << Op << ": Invalid integer type: " << *OpTy << "\n";
-    } else {
-      Errors() << Op << ": Expects integer type. Found: " << *OpTy << "\n";
-    }
-    return Op;
-  }
-
-  /// Verifies that Opty is a floating point type, or a vector of a
-  /// floating points, for operator Op. Generates error messages if
-  /// appropriate. Returns Op.
-  const char *VerifyFloatingOrVectorOp(const char *Op, Type *OpTy) {
-    Type *BaseTy = OpTy;
-    if (OpTy->isVectorTy()) {
-      if (!IgnorePNaClABIChecks &&
-          !PNaClABITypeChecker::isValidVectorType(OpTy)) {
-        Errors() << Op << ": invalid vector type: " << *OpTy << "\n";
-        return Op;
-      }
-      BaseTy = OpTy->getVectorElementType();
-    }
-    if (!isFloatingType(BaseTy)) {
-      Errors() << Op << ": Expects floating point. Found "
-               << OpTy << "\n";
-      return Op;
-    }
-    if (!IgnorePNaClABIChecks &&
-        !PNaClABITypeChecker::isValidScalarType(BaseTy)) {
-      Errors() << Op << ": type not allowed: " << OpTy << "\n";
-    }
-    return Op;
-  }
-
-  /// Op is the name of the operation that called this.  Checks if
-  /// OpTy is either a (non-void) scalar, or a vector of scalars. If
-  /// not, generates an appropriate error message. Always returns Op.
-  const char *VerifyScalarOrVectorOp(const char *Op, Type *OpTy) {
-    if (IgnorePNaClABIChecks) return Op;
-    if (PNaClABITypeChecker::isValidScalarType(OpTy)) {
-      if (OpTy->isVoidTy())
-        Errors() << Op << ": Type void not allowed\n";
-    } else if (!PNaClABITypeChecker::isValidVectorType(OpTy)) {
-      Errors() << Op << ": Expects scalar/vector type. Found: "
-               << OpTy << "\n";
-    }
-    return Op;
-  }
-
-  void VerifyIndexedVector(const char *Op, NaClBcIndexSize_t VecValue,
-                           NaClBcIndexSize_t IdxValue) {
-    Type *VecType = GetValueType(VecValue);
-    Type *IdxType = GetValueType(IdxValue);
-    if (!IgnorePNaClABIChecks &&
-        !PNaClABITypeChecker::isValidVectorType(VecType)){
-      if (VecType->isVectorTy())
-        Errors() << Op << ": Vector type " << *VecType << " not allowed\n";
-      else
-        Errors() << Op << ": Vector type expected. Found: " << *VecType << "\n";
-    }
-    // Note: This restriction appears to be LLVM specific.
-    if (!IdxType->isIntegerTy(32)) {
-      Errors() << Op << ": Index not i32. Found: " << *IdxType << "\n";
-    }
-    BitcodeId IdxId(GetBitcodeId(IdxValue));
-    if (IdxId.GetKind() != 'c') {
-      Errors() << Op << ": Vector index not constant: " << IdxId << "\n";
-      // TODO(kschimpf): We should check that Idx is a constant with
-      // valid access. However, we currently don't store constant
-      // values, so it can't be tested.
-    }
-  }
-
-  /// Checks if block Value is a valid branch target for the current
-  /// function block. If not, generates an appropriate error message.
-  void VerifyBranchRange(uint64_t Value) {
-    if (0 == Value || Value >= ExpectedNumBbs) {
-      Errors() << "Branch " << BitcodeId('b', Value)
-               << " out of range. Not in [1," << ExpectedNumBbs << "]\n";
-    }
-  }
-
-  /// Convert alignment exponent (i.e. power of two (or zero)) to the
-  /// corresponding alignment to use. If alignment is too large, it generates
-  /// an error message and returns 0.
-  unsigned getAlignmentValue(uint64_t Exponent);
-};
-
-NaClDisFunctionParser::NaClDisFunctionParser(
-    unsigned BlockID,
-    NaClDisBlockParser *EnclosingParser)
-    : NaClDisBlockParser(BlockID, EnclosingParser),
-      CurrentBbIndex(-1),
-      ExpectedNumBbs(0),
-      InstIsTerminating(false) {
-  Context->ResetLocalCounters();
-  if (Context->HasNextDefinedFunctionIndex()) {
-    FcnId = Context->GetNextDefinedFunctionIndex();
-    FcnTy = GetFunctionType(FcnId);
-  } else {
-    FcnId = 0;
-    SmallVector<Type*, 8> Signature;
-    FcnTy = FunctionType::get(GetVoidType(), Signature, 0);
-    Errors() <<
-        "No corresponding defining function address for function block.\n";
-    return;
-  }
-  Context->IncNumDefinedFunctions();
-
-  // Now install parameters.
-  for (size_t Index = 0; Index < FcnTy->getFunctionNumParams(); ++Index) {
-    InstallParamType(FcnTy->getFunctionParamType(Index));
-  }
-}
-
-void NaClDisFunctionParser::PrintBlockHeader() {
-  Tokens() << "function" << Space();
-  BitcodeId FunctionId('f', FcnId);
-  bool InvalidSignature = true; // until proven otherwise.
-  if (Context->IsFunctionIntrinsic(FcnId))
-    InvalidSignature = false;
-  else if (IgnorePNaClABIChecks ||
-           PNaClABITypeChecker::isValidFunctionType(FcnTy)) {
-      InvalidSignature = false;
-  }
-  if (InvalidSignature) {
-    Errors() << "Invalid type signature for "
-             << BitcodeId('f', FcnId) << ": " << *FcnTy << "\n";
-  }
-  Tokens() << TokenizeFunctionSignature(FcnTy, &FunctionId)
-           << Space() << OpenCurly()
-           << Space() << Space() << "// BlockID = " << GetBlockID()
-           << Endline();
-}
-
-const char *NaClDisFunctionParser::GetBinop(uint32_t Opcode, Type *Ty) {
-  Instruction::BinaryOps LLVMOpcode;
-  if (!naclbitc::DecodeBinaryOpcode(Opcode, Ty, LLVMOpcode)) {
-    Errors() << "Binary opcode not understood: " << Opcode << "\n";
-    return "???";
-  }
-  switch (LLVMOpcode) {
-  default:
-    Errors() << "Binary opcode not understood: " << Opcode << "\n";
-    return "???";
-  case Instruction::Add:
-    return VerifyIntArithmeticOp("add", Ty);
-  case Instruction::FAdd:
-    return VerifyFloatingOrVectorOp("fadd", Ty);
-  case Instruction::Sub:
-    return VerifyIntArithmeticOp("sub", Ty);
-  case Instruction::FSub:
-    return VerifyFloatingOrVectorOp("fsub", Ty);
-  case Instruction::Mul:
-    return VerifyIntArithmeticOp("mul", Ty);
-  case Instruction::FMul:
-    return VerifyFloatingOrVectorOp("fmul", Ty);
-  case Instruction::UDiv:
-    return VerifyIntArithmeticOp("udiv", Ty);
-  case Instruction::SDiv:
-    return VerifyIntArithmeticOp("sdiv", Ty);
-  case Instruction::FDiv:
-    return VerifyFloatingOrVectorOp("fdiv", Ty);
-  case Instruction::URem:
-    return VerifyIntArithmeticOp("urem", Ty);
-  case Instruction::SRem:
-    return VerifyIntArithmeticOp("srem", Ty);
-  case Instruction::FRem:
-    return VerifyFloatingOrVectorOp("frem", Ty);
-  case Instruction::Shl:
-    return VerifyIntArithmeticOp("shl", Ty);
-  case Instruction::LShr:
-    return VerifyIntArithmeticOp("lshr", Ty);
-  case Instruction::AShr:
-    return VerifyIntArithmeticOp("ashr", Ty);
-  case Instruction::And:
-    return VerifyIntegerOrVectorOp("and", Ty);
-  case Instruction::Or:
-    return VerifyIntegerOrVectorOp("or", Ty);
-  case Instruction::Xor:
-    return VerifyIntegerOrVectorOp("xor", Ty);
-  }
-}
-
-const char *NaClDisFunctionParser::GetCastOp(uint32_t Opcode,
-                                             Type *FromType, Type *ToType) {
-  Instruction::CastOps Cast;
-  const char *CastName = "???";
-  if (!naclbitc::DecodeCastOpcode(Opcode, Cast)) {
-    Errors() << "Cast opcode not understood: " << Opcode << "\n";
-    return CastName;
-  }
-  switch (Cast) {
-  default:
-    Errors() << "Cast opcode not understood: " << Opcode << "\n";
-    return CastName;
-  case Instruction::BitCast:
-    CastName = "bitcast";
-    break;
-  case Instruction::Trunc:
-    CastName = "trunc";
-    break;
-  case Instruction::ZExt:
-    CastName = "zext";
-    break;
-  case Instruction::SExt:
-    CastName = "sext";
-    break;
-  case Instruction::FPToUI:
-    CastName = "fptoui";
-    break;
-  case Instruction::FPToSI:
-    CastName = "fptosi";
-    break;
-  case Instruction::UIToFP:
-    CastName = "uitofp";
-    break;
-  case Instruction::SIToFP:
-    CastName = "sitofp";
-    break;
-  case Instruction::FPTrunc:
-    CastName = "fptrunc";
-    break;
-  case Instruction::FPExt:
-    CastName = "fpext";
-    break;
-  }
-  if (!CastInst::castIsValid(Cast, FromType, ToType)) {
-    Errors() << "Invalid cast '" << CastName << "'. Not defined on "
-             << *FromType << " to " << *ToType << "\n";
-  }
-  return CastName;
-}
-
-const char *NaClDisFunctionParser::GetIcmpPredicate(uint32_t Opcode) {
-  CmpInst::Predicate Predicate;
-  if (!naclbitc::DecodeIcmpPredicate(Opcode, Predicate)) {
-    Errors() << "Icmp predicate not understood: " << Opcode << "\n";
-    return "???";
-  }
-  switch (Predicate) {
-  default:
-    Errors() << "Icmp predicate not understood: " << Opcode << "\n";
-    return "???";
-  case CmpInst::ICMP_EQ:
-    return "eq";
-  case CmpInst::ICMP_NE:
-    return "ne";
- case CmpInst::ICMP_UGT:
-    return "ugt";
-  case CmpInst::ICMP_UGE:
-    return "uge";
-  case CmpInst::ICMP_ULT:
-    return "ult";
-  case CmpInst::ICMP_ULE:
-    return "ule";
-  case CmpInst::ICMP_SGT:
-    return "sgt";
-  case CmpInst::ICMP_SGE:
-    return "sge";
-  case CmpInst::ICMP_SLT:
-    return "slt";
-  case CmpInst::ICMP_SLE:
-    return "sle";
-  }
-}
-
-const char *NaClDisFunctionParser::GetFcmpPredicate(uint32_t Opcode) {
-  CmpInst::Predicate Predicate;
-  if (!naclbitc::DecodeFcmpPredicate(Opcode, Predicate)) {
-    Errors() << "Fcmp predicate not understood: " << Opcode << "\n";
-    return "???";
-  }
-  switch (Predicate) {
-  default:
-    Errors() << "Fcmp predicate not understood: " << Opcode << "\n";
-    return "???";
-  case CmpInst::FCMP_FALSE:
-    return "false";
-  case CmpInst::FCMP_OEQ:
-    return "oeq";
-  case CmpInst::FCMP_OGT:
-    return "ogt";
-  case CmpInst::FCMP_OGE:
-    return "oge";
-  case CmpInst::FCMP_OLT:
-    return "olt";
-  case CmpInst::FCMP_OLE:
-    return "ole";
-  case CmpInst::FCMP_ONE:
-    return "one";
-  case CmpInst::FCMP_ORD:
-    return "ord";
-  case CmpInst::FCMP_UNO:
-    return "uno";
-  case CmpInst::FCMP_UEQ:
-    return "ueq";
-  case CmpInst::FCMP_UGT:
-    return "ugt";
-  case CmpInst::FCMP_UGE:
-    return "uge";
-  case CmpInst::FCMP_ULT:
-    return "ult";
-  case CmpInst::FCMP_ULE:
-    return "ule";
-  case CmpInst::FCMP_UNE:
-    return "une";
-  case CmpInst::FCMP_TRUE:
-    return "true";
-  }
-}
-
-namespace {
-
-static const unsigned MaxAlignmentExponent = 29;
-static_assert(
-    (1u << MaxAlignmentExponent) == Value::MaximumAlignment,
-    "Inconsistency between Value.MaxAlignment and PNaCl alignment limit");
-}
-
-unsigned NaClDisFunctionParser::getAlignmentValue(uint64_t Exponent) {
-  if (Exponent > MaxAlignmentExponent + 1) {
-    Errors() << "Alignment can't be greater than 2**" << MaxAlignmentExponent
-             << ". Found: 2**" << (Exponent - 1) << "\n";
-    return 0;
-  }
-  return (1 << static_cast<unsigned>(Exponent)) >> 1;
-}
-
-bool NaClDisFunctionParser::ParseBlock(unsigned BlockID) {
-  ObjDumpSetRecordBitAddress(GetBlock().GetStartBit());
-  switch (BlockID) {
-  case naclbitc::CONSTANTS_BLOCK_ID: {
-    NaClDisConstantsParser Parser(BlockID, this);
-    return Parser.ParseThisBlock();
-  }
-  case naclbitc::VALUE_SYMTAB_BLOCK_ID: {
-    if (!PNaClAllowLocalSymbolTables) break;
-    NaClDisValueSymtabParser Parser(BlockID, this);
-    return Parser.ParseThisBlock();
-  }
-  default:
-    break;
-  }
-  return NaClDisBlockParser::ParseBlock(BlockID);
-}
-
-void NaClDisFunctionParser::ProcessRecord() {
-  ObjDumpSetRecordBitAddress(Record.GetStartBit());
-  const NaClBitcodeRecord::RecordVector Values = Record.GetValues();
-  // Start by adding block label if previous instruction is terminating.
-  if (InstIsTerminating) {
-    InstIsTerminating = false;
-    ++CurrentBbIndex;
-    DecAssemblyIndent();
-    Tokens() << BitcodeId('b', CurrentBbIndex) << ":" << Endline();
-    ObjDumpFlush();
-    IncAssemblyIndent();
-  }
-  switch (Record.GetCode()) {
-  case naclbitc::FUNC_CODE_DECLAREBLOCKS:
-    // DECLAREBLOCKS: [n]
-    InstIsTerminating = true;  // Force block label on first instruction.
-    if (Values.size() != 1) {
-      Errors() << "Function blocks record expects a size argument.\n";
-      break;
-    }
-    ExpectedNumBbs = Values[0];
-    if (ExpectedNumBbs == 0) {
-      Errors() << "Functions must contain at least one block.\n";
-    }
-    Tokens() << "blocks" << Space() << ExpectedNumBbs << Semicolon();
-    break;
-  case naclbitc::FUNC_CODE_INST_BINOP: {
-    // BINOP: [opval, opval, opcode]
-    if (Values.size() != 3) {
-      Errors() << "Binop record expects 3 arguments. Found: "
-               << Values.size() << "\n";
-      break;
-    }
-    NaClBcIndexSize_t Op1 = RelativeToAbsId(Values[0]);
-    NaClBcIndexSize_t Op2 = RelativeToAbsId(Values[1]);
-    Type *Type1 = GetValueType(Op1);
-    Type *Type2 = GetValueType(Op2);
-    if (Type1 != Type2) {
-      Errors() << "Binop argument types differ: " << *Type1 << " and "
-               << *Type2 << "\n";
-    }
-    const char *Binop = GetBinop(Values[2], Type1);
-    Tokens() << NextInstId() << Space() << "=" << Space() << Binop << Space()
-             << TokenizeType(Type1) << Space() << GetBitcodeId(Op1) << Comma()
-             << Space() << GetBitcodeId(Op2) << Semicolon();
-    InstallInstType(Type1);
-    break;
-  }
-  case naclbitc::FUNC_CODE_INST_CAST: {
-    // CAST: [opval, destty, castopc]
-    if (Values.size() != 3) {
-      Errors() << "Cast record expects 3 argments. Found: "
-               << Values.size() << "\n";
-      break;
-    }
-    NaClBcIndexSize_t Op = RelativeToAbsId(Values[0]);
-    Type *FromType = GetValueType(Op);
-    Type *ToType = GetType(Values[1]);
-    const char *CastOp = GetCastOp(Values[2], FromType, ToType);
-    Tokens() << NextInstId() << Space() << "=" << Space() << CastOp << Space()
-             << TokenizeType(FromType) << Space() << GetBitcodeId(Op)
-             << Space() << "to" << Space() << TokenizeType(ToType)
-             << Semicolon();
-    InstallInstType(ToType);
-    break;
-  }
-  case naclbitc::FUNC_CODE_INST_RET: {
-    // RET: [opval?]
-    InstIsTerminating = true;
-    Tokens() << "ret" << Space();
-    switch (Values.size()) {
-    default:
-      Errors() << "Function return record expects an optional return argument. "
-               << "Found: " << Values.size() << " arguments\n";
-      break;
-    case 0:
-      Tokens() << "void";
-      break;
-    case 1: {
-      NaClBcIndexSize_t Op = RelativeToAbsId(Values[0]);
-      Tokens() << TokenizeType(GetValueType(Op)) << Space()<< GetBitcodeId(Op);
-      break;
-    }
-    }
-    Tokens() << Semicolon();
-    break;
-  }
-  case naclbitc::FUNC_CODE_INST_BR: {
-    // BR: [bb#, bb#, opval] or [bb#]
-    InstIsTerminating = true;
-    if (Values.size() != 1 && Values.size() != 3) {
-      Errors() << "Function branch record expects 1 or 3 arguments. Found: "
-               << Values.size() << "\n";
-      break;
-    }
-    Tokens() << "br" << Space();
-    if (Values.size() == 3) {
-      NaClBcIndexSize_t OpIndex = RelativeToAbsId(Values[2]);
-      Type *CondType = GetValueType(OpIndex);
-      if (CondType != GetComparisonType())
-        Errors() << "Branch condition not i1\n";
-      Tokens() << StartCluster() << TokenizeType(CondType) << Space()
-               << GetBitcodeId(OpIndex) << Comma()
-               << FinishCluster() << Space();
-    }
-    VerifyBranchRange(Values[0]);
-    Tokens() << StartCluster() << "label" << Space()
-             << BitcodeId('b', Values[0]);
-    if (Values.size() == 3) {
-      VerifyBranchRange(Values[1]);
-      Tokens() << Comma() << FinishCluster() << Space()
-               << StartCluster() << "label" << Space()
-               << BitcodeId('b', Values[1]);
-    }
-    Tokens() << Semicolon() << FinishCluster();
-    break;
-  }
-  case naclbitc::FUNC_CODE_INST_SWITCH: {
-    // SWITCH:  [opty, op, bb#, n, (1, 1, int, bb#)*]
-    InstIsTerminating = true;
-    if (Values.size() < 4) {
-      Errors()
-          << "Function switch record expects at least 4 arguments. Found: "
-          << Values.size() << "\n";
-      break;
-    }
-    Type *OpType = GetType(Values[0]);
-    NaClBcIndexSize_t CondId = RelativeToAbsId(Values[1]);
-    Type *CondType = GetValueType(CondId);
-    if (OpType != CondType)
-      Errors() << "Specified select type " << *OpType << " but found: "
-               << *CondType << "\n";
-    if (!IgnorePNaClABIChecks &&
-        !PNaClABITypeChecker::isValidSwitchConditionType(CondType)) {
-      Errors() << PNaClABITypeChecker::ExpectedSwitchConditionType(CondType)
-               << "\n";
-    }
-    NaClBcIndexSize_t DefaultBb = Values[2];
-    // TODO(kschimpf): Deal with values that are too large for NumCases.
-    size_t NumCases = Values[3];
-    VerifyBranchRange(DefaultBb);
-    Tokens() << "switch" << Space() << StartCluster() << StartCluster()
-             << TokenizeType(OpType) << Space() <<  GetBitcodeId(CondId)
-             << FinishCluster() << Space() << "{" << FinishCluster()
-             << Endline();
-    IncAssemblyIndent();
-    Tokens() << StartCluster() << "default" << ":" << Space()
-             << FinishCluster() << StartCluster() << "br" << Space() << "label"
-             << Space() << BitcodeId('b', DefaultBb) << Semicolon()
-             << FinishCluster() << Endline();
-    size_t CurIdx = 4;
-    for (size_t i = 0; i < NumCases; ++i) {
-      if (CurIdx + 3 >= Values.size()) {
-        Errors() << "Incomplete case entry in switch. At index: "
-                 << CurIdx << "\n";
-        break;
-      }
-      uint64_t NumItems = Values[CurIdx++];
-      bool IsSingleNumber = Values[CurIdx++];
-      if (NumItems != 1 || !IsSingleNumber) {
-        Errors() << "Case ranges are not supported in PNaCl\n";
-        break;
-      }
-      SignRotatedInt CaseValue(Values[CurIdx++], OpType);
-
-      // TODO(kschimpf) Check if CaseValue possible based on OpType.
-      uint64_t Label = Values[CurIdx++];
-      VerifyBranchRange(Label);
-      Tokens() << StartCluster() << TokenizeType(OpType) << Space()
-               << CaseValue << ":" << Space() << FinishCluster()
-               << StartCluster() << "br" << Space() << "label" << Space()
-               << BitcodeId('b', Label) << Semicolon() << FinishCluster()
-               << Endline();
-    }
-    DecAssemblyIndent();
-    Tokens() << "}";
-    break;
-  }
-  case naclbitc::FUNC_CODE_INST_UNREACHABLE:
-    // UNREACHABLE
-    InstIsTerminating = true;
-    if (!Values.empty()) {
-      Errors() << "Function unreachable record expects no arguments. Found: "
-               << Values.size() << "\n";
-    }
-    Tokens() << "unreachable" << Semicolon();
-    break;
-  case naclbitc::FUNC_CODE_INST_PHI: {
-    // PHI: [ty, (val0, bb0)*]
-    if (Values.size() < 3) {
-      Errors() << "Function phi record expects at least 3 arguments. Found: "
-               << Values.size() << "\n";
-      break;
-    } else if (Values.size() % 2 == 0) {
-      Errors()
-          << "Function phi records should have an odd number of arguments. "
-          << "Found: " << Values.size() << "\n";
-    }
-    Type* OpType = GetType(Values[0]);
-    Tokens() << NextInstId() << Space() << "=" << StartCluster() << Space()
-             << "phi" << Space() << TokenizeType(OpType);
-    for (size_t i = 1; i < Values.size(); i += 2) {
-      if (i > 1) Tokens() << Comma();
-      NaClBcIndexSize_t Index =
-          RelativeToAbsId(NaClDecodeSignRotatedValue(Values[i]));
-      Tokens() << FinishCluster() << Space() << StartCluster() << OpenSquare()
-               << GetBitcodeId(Index) << Comma() << Space()
-               << BitcodeId('b', Values[i+1]) << CloseSquare();
-    }
-    Tokens() << Semicolon() << FinishCluster();
-    InstallInstType(OpType);
-    break;
-  };
-  case naclbitc::FUNC_CODE_INST_ALLOCA: {
-    // ALLOCA:     [size, align]
-    if (Values.size() != 2) {
-      Errors() << "Function alloca record expects 2 arguments. Found: "
-               << Values.size() << "\n";
-      break;
-    }
-    NaClBcIndexSize_t SizeOp = RelativeToAbsId(Values[0]);
-    Type* SizeType = GetValueType(SizeOp);
-    BitcodeId SizeId(GetBitcodeId(SizeOp));
-    unsigned Alignment = getAlignmentValue(Values[1]);
-    if (!IgnorePNaClABIChecks && !PNaClABIProps::isAllocaSizeType(SizeType))
-      Errors() << PNaClABIProps::ExpectedAllocaSizeType() << "\n";
-    // TODO(kschimpf) Are there any constraints on alignment?
-    Tokens() << NextInstId() << Space() << "=" << Space() << StartCluster()
-             << "alloca" << Space() << "i8" << Comma() << FinishCluster()
-             << Space() << StartCluster() << TokenizeType(SizeType) << Space()
-             << SizeId << Comma() << FinishCluster() << Space()
-             << StartCluster() <<"align" << Space() << Alignment << Semicolon()
-             << FinishCluster();
-    InstallInstType(GetPointerType());
-    break;
-  }
-  case naclbitc::FUNC_CODE_INST_LOAD: {
-    // LOAD: [op, align, ty]
-    if (Values.size() != 3) {
-      Errors() << "Function load record expects 3 arguments. Found: "
-               << Values.size() << "\n";
-      break;
-    }
-    unsigned Alignment = getAlignmentValue(Values[1]);
-    Type *LoadType = GetType(Values[2]);
-    VerifyScalarOrVectorOp("load", LoadType);
-    Context->VerifyMemoryAccessAlignment("load", LoadType, Alignment);
-    Tokens() << NextInstId() << Space() << "=" << Space() << StartCluster()
-             << "load" << Space() << TokenizeType(LoadType) << "*" << Space()
-             << GetBitcodeId(RelativeToAbsId(Values[0])) << Comma()
-             << FinishCluster() << Space() << StartCluster()
-             << "align" << Space() << Alignment << Semicolon()
-             << FinishCluster();
-    InstallInstType(LoadType);
-    break;
-  }
-  case naclbitc::FUNC_CODE_INST_STORE: {
-    // STORE: [ptr, val, align]
-    if (Values.size() != 3) {
-      Errors() << "Function store record expects 3 arguments. Found: "
-               << Values.size() << "\n";
-      break;
-    }
-    unsigned Alignment = getAlignmentValue(Values[2]);
-    NaClBcIndexSize_t Val = RelativeToAbsId(Values[1]);
-    Type *ValType = GetValueType(Val);
-    VerifyScalarOrVectorOp("store", ValType);
-    Context->VerifyMemoryAccessAlignment("store", ValType, Alignment);
-    Tokens() << StartCluster() << "store" << Space() << TokenizeType(ValType)
-             << Space() << GetBitcodeId(Val) << Comma() << FinishCluster()
-             << Space() << StartCluster() << TokenizeType(ValType) << "*"
-             << Space() << GetBitcodeId(RelativeToAbsId(Values[0])) << Comma()
-             << FinishCluster() << Space() << StartCluster() << "align"
-             << Space() << Alignment << Semicolon() << FinishCluster();
-    break;
-  }
-  case naclbitc::FUNC_CODE_INST_CMP2: {
-    // CMP2: [opval, opval, pred]
-    if (Values.size() != 3) {
-      Errors() << "Function compare record expects 3 arguments. Found: "
-               << Values.size() << "\n";
-      break;
-    }
-    NaClBcIndexSize_t Arg1 = RelativeToAbsId(Values[0]);
-    NaClBcIndexSize_t Arg2 = RelativeToAbsId(Values[1]);
-    Type *Arg1Type = GetValueType(Arg1);
-    Type *Arg2Type = GetValueType(Arg2);
-    if (Arg1Type != Arg2Type) {
-      Errors() << "Arguments not of same type: " << *Arg1Type << " and "
-               << *Arg2Type << "\n";
-    }
-    const char *Pred = "???";
-    Tokens() << NextInstId() << Space() << "=" << Space() << StartCluster();
-    Type* BaseType = UnderlyingType(Arg1Type);
-    if (BaseType->isIntegerTy()) {
-      Pred = GetIcmpPredicate(Values[2]);
-      Tokens() << "icmp" << Space() << Pred;
-    } else if (isFloatingType(BaseType)) {
-      Pred = GetFcmpPredicate(Values[2]);
-      Tokens() << "fcmp" << Space() << Pred;
-    } else {
-      Errors() << "Compare not on integer/float type. Found: "
-               << *Arg1Type << "\n";
-      Tokens() << "cmp" << Space() << "???" "(" << Values[2] << ")";
-    }
-    Tokens() << FinishCluster() << Space() << StartCluster()
-             << TokenizeType(Arg1Type) << Space () << GetBitcodeId(Arg1)
-             << Comma() << FinishCluster() << Space() << StartCluster()
-             << GetBitcodeId(Arg2) << Semicolon() << FinishCluster();
-    Type *ResultType = GetComparisonType();
-    if (Arg1Type->isVectorTy()) {
-      ResultType = VectorType::get(ResultType,
-                                   Arg1Type->getVectorNumElements());
-    }
-    InstallInstType(ResultType);
-    break;
-  }
-  case naclbitc::FUNC_CODE_INST_VSELECT: {
-    // VSELECT:    [opval, opval, pred]
-    if (Values.size() != 3) {
-      Errors() << "Select record expects 3 arguments. Found: "
-               << Values.size() << "\n";
-      break;
-    }
-    NaClBcIndexSize_t CondValue = RelativeToAbsId(Values[2]);
-    NaClBcIndexSize_t ThenValue = RelativeToAbsId(Values[0]);
-    NaClBcIndexSize_t ElseValue = RelativeToAbsId(Values[1]);
-    Type *CondType = GetValueType(CondValue);
-    Type *ThenType = GetValueType(ThenValue);
-    Type *ElseType = GetValueType(ElseValue);
-    if (ThenType != ElseType) {
-      Errors() << "Selected arguments not of same type: "
-               << *ThenType << " and " << *ElseType << "\n";
-    }
-    Type *BaseType = UnderlyingType(ThenType);
-    if (!(BaseType->isIntegerTy() || isFloatingType(BaseType))) {
-      Errors() << "Select arguments not integer/float. Found: " << *ThenType;
-    }
-    Tokens() << NextInstId() << Space() << "=" << Space() << StartCluster()
-             << "select" << Space() << TokenizeType(CondType) << Space()
-             << GetBitcodeId(CondValue) << Comma() << FinishCluster() << Space()
-             << StartCluster() << TokenizeType(ThenType) << Space()
-             << GetBitcodeId(ThenValue) << Comma() << FinishCluster() << Space()
-             << StartCluster() << TokenizeType(ElseType) << Space()
-             << GetBitcodeId(ElseValue) << Semicolon()
-             << FinishCluster();
-    InstallInstType(ThenType);
-    break;
-  }
-  case naclbitc::FUNC_CODE_INST_EXTRACTELT: {
-    // EXTRACTELT: [opval, opval]
-    if (Values.size() != 2) {
-      Errors() << "Extract element record expects 2 arguments. Found: "
-               << Values.size() << "\n";
-      break;
-    }
-    NaClBcIndexSize_t VecValue = RelativeToAbsId(Values[0]);
-    NaClBcIndexSize_t IdxValue = RelativeToAbsId(Values[1]);
-    VerifyIndexedVector("extractelement", VecValue, IdxValue);
-    Type *VecType = GetValueType(VecValue);
-    Type *IdxType = GetValueType(IdxValue);
-    Tokens() << NextInstId() << Space() << " = " << Space() << StartCluster()
-             << "extractelement" << Space() << TokenizeType(VecType) << Space()
-             << GetBitcodeId(VecValue) << Comma() << FinishCluster() << Space()
-             << StartCluster() << TokenizeType(IdxType) << Space()
-             << GetBitcodeId(IdxValue) << Semicolon() << FinishCluster();
-    InstallInstType(UnderlyingType(VecType));
-    break;
-  }
-  case naclbitc::FUNC_CODE_INST_INSERTELT: {
-    // INSERTELT:  [opval, opval, opval]
-    NaClBcIndexSize_t VecValue = RelativeToAbsId(Values[0]);
-    NaClBcIndexSize_t EltValue = RelativeToAbsId(Values[1]);
-    NaClBcIndexSize_t IdxValue = RelativeToAbsId(Values[2]);
-    VerifyIndexedVector("insertelement", VecValue, IdxValue);
-    Type *VecType = GetValueType(VecValue);
-    Type *EltType = GetValueType(EltValue);
-    Type *IdxType = GetValueType(IdxValue);
-    if (EltType != UnderlyingType(VecType)) {
-      Errors() << "insertelement: Illegal element type " << *EltType
-               << ". Expected: " << *UnderlyingType(VecType) << "\n";
-    }
-    Tokens() << NextInstId() << Space() << " = " << Space() << StartCluster()
-             << "insertelement" << Space() << TokenizeType(VecType) << Space()
-             << GetBitcodeId(VecValue) << Comma() << FinishCluster() << Space()
-             << StartCluster() << TokenizeType(EltType) << Space()
-             << GetBitcodeId(EltValue) << Comma() << FinishCluster() << Space()
-             << StartCluster() << TokenizeType(IdxType) << Space()
-             << GetBitcodeId(IdxValue) << Semicolon() << FinishCluster();
-    InstallInstType(VecType);
-    break;
-  }
-  case naclbitc::FUNC_CODE_INST_CALL:
-  case naclbitc::FUNC_CODE_INST_CALL_INDIRECT: {
-    // CALL: [cc, fnid, arg0, arg1...]
-    // CALL_INDIRECT: [cc, fnid, returnty, args...]
-    if (Record.GetCode() == naclbitc::FUNC_CODE_INST_CALL) {
-      if (Values.size() < 2) {
-        Errors() << "Call record expects at least 2 arguments. Found: "
-                 << Values.size() << "\n";
-        break;
-      }
-    } else if (Record.GetCode() == naclbitc::FUNC_CODE_INST_CALL_INDIRECT) {
-      if (Values.size() < 3) {
-        Errors() << "Call indirect record expects at least 3 arguments. Found: "
-                 << Values.size() << "\n";
-        break;
-      }
-    }
-    unsigned IsTailCall = (Values[0] & 0x1);
-    CallingConv::ID CallingConv;
-    NaClBcIndexSize_t FcnId = RelativeToAbsId(Values[1]);
-    if (!naclbitc::DecodeCallingConv(Values[0]>>1, CallingConv))
-      Errors() << "Call unknown calling convention:" << (Values[0]>>1) << "\n";
-    if (!IgnorePNaClABIChecks &&
-        !PNaClABIProps::isValidCallingConv(CallingConv)) {
-      Errors() << "Call uses disallowed calling convention: "
-               << PNaClABIProps::CallingConvName(CallingConv) << "("
-               << CallingConv << ")\n";
-    }
-    FunctionType *FcnType = 0;
-    Type *ReturnType = 0;
-    size_t ArgIndex = 2;
-    // Flag defining if type checking should be done on argument/return
-    // types.
-    bool CheckArgRetTypes = true;
-    if (Record.GetCode() == naclbitc::FUNC_CODE_INST_CALL) {
-      Type *FcnTy = GetFunctionValueType(FcnId);
-      if (FunctionType *FcnTyp = dyn_cast<FunctionType>(FcnTy)) {
-        // TODO(kschimpf) Add back signature checking once we know how
-        // to handle intrinsics (which can violate general signature
-        // rules).
-        CheckArgRetTypes = false;
-        FcnType = FcnTyp;
-        ReturnType = FcnType->getReturnType();
-      } else {
-        Errors() << "Invalid function signature: " << *FcnTy << "\n";
-        ReturnType = GetVoidType();
-      }
-    } else {
-      ReturnType = GetType(Values[2]);
-      ArgIndex = 3;
-    }
-    if (!ReturnType->isVoidTy()) {
-      Tokens() << NextInstId() << Space() << "=" << Space();
-    }
-    if (IsTailCall) {
-      Tokens() << "tail" << Space();
-    }
-    if (CheckArgRetTypes &&
-        !IgnorePNaClABIChecks &&
-        !PNaClABITypeChecker::isValidParamType(ReturnType)) {
-      Errors() << "Invalid return type: " << *ReturnType << "\n";
-    }
-    Tokens() << "call" << Space();
-    if (CallingConv != CallingConv::C) {
-      Tokens() << PNaClABIProps::CallingConvName(CallingConv) << Space();
-    }
-    Tokens() << TokenizeType(ReturnType) << Space()
-             << StartCluster() << GetBitcodeId(FcnId) << OpenParen()
-             << StartCluster();
-    size_t ParamIndex = 0;
-    size_t NumParams = Values.size() + 1 - ArgIndex;
-    for (size_t i = ArgIndex; i < Values.size(); ++i, ++ParamIndex) {
-      NaClBcIndexSize_t ParamId = RelativeToAbsId(Values[i]);
-      Type *ParamType = GetValueType(ParamId);
-      if (CheckArgRetTypes &&
-          !IgnorePNaClABIChecks &&
-          !PNaClABITypeChecker::isValidParamType(ParamType)) {
-        Errors() << "invalid type for parameter " << i << ": "
-                 << *ParamType << "\n";
-      }
-      if (FcnType) {
-        if (ParamIndex < FcnType->getNumParams()) {
-          Type *ExpectedType = FcnType->getParamType(ParamIndex);
-          if (ParamType != ExpectedType) {
-            Warnings() << "Parameter " << (ParamIndex + 1) << " mismatch: "
-                       << *ParamType << " and " << *ExpectedType << "\n";
-          }
-        }
-        else if (ParamIndex == FcnType->getNumParams()) {
-          Warnings() << "Call expects " << FcnType->getNumParams()
-                     << " arguments. Got: " << NumParams << "\n";
-        }
-      }
-      if (i > ArgIndex) {
-        Tokens() << Comma() << FinishCluster() << Space() << StartCluster();
-      }
-      Tokens() << TokenizeType(ParamType) << Space() << GetBitcodeId(ParamId);
-    }
-    Tokens() << CloseParen() << Semicolon() << FinishCluster()
-             << FinishCluster();
-    if (!ReturnType->isVoidTy()) InstallInstType(ReturnType);
-    break;
-  }
-  case naclbitc::FUNC_CODE_INST_FORWARDTYPEREF: {
-    // TYPE: [opval, ty]
-    if (Values.size() != 2) {
-      Errors() << "Forward declare record expects 2 arguments. Found: "
-               << Values.size() << "\n";
-      break;
-    }
-    Type *OpType = GetType(Values[1]);
-    Tokens() << "declare" << Space() << StartCluster()
-             << TokenizeType(OpType) << Space()
-             << GetBitcodeId(Values[0]) << Semicolon() << FinishCluster();
-    InstallInstType(OpType, AbsToValuedInstId(Values[0]));
-    break;
-  }
-  default:
-    Errors() << "Unknown record found in module block.\n";
-    break;
-  }
-  Tokens() << TokenizeAbbrevIndex() << Endline();
-  ObjDumpWrite(Record.GetStartBit(), Record);
-}
-
-/// Parses and disassembles the module block.
-class NaClDisModuleParser : public NaClDisBlockParser {
-public:
-  NaClDisModuleParser(unsigned BlockID, NaClDisTopLevelParser *Context)
-      : NaClDisBlockParser(BlockID, Context) {
-  }
-
-  ~NaClDisModuleParser() override;
-
-  bool ParseBlock(unsigned BlockID) override;
-
-  void PrintBlockHeader() override;
-
-  void ProcessRecord() override;
-};
-
-NaClDisModuleParser::~NaClDisModuleParser() {
-  // Note: Since we can't check type signatures of most functions till
-  // we know intrinsic names, and that isn't known until the (optional)
-  // valuesymtab block is parsed, the only reasonable spot to check
-  // function signatures is once the module block has been processed.
-  NaClBcIndexSize_t NextFcnDefinedId = 0;
-  for (size_t i = 0, e = GetNumFunctions(); i < e; ++i) {
-    // Note: If the type of a function isn't a function type, that
-    // was checked in method ProcessRecord.
-    // Note: If the function was defined, the type was checked in
-    // NaClDisFunctionParser::PrintBlockHeader.
-    if (Context->HasDefinedFunctionIndex(NextFcnDefinedId)
-        && i == Context->GetDefinedFunctionIndex(NextFcnDefinedId)) {
-      ++NextFcnDefinedId;
-      continue;
-    }
-    if (FunctionType *FcnTy = dyn_cast<FunctionType>(GetFunctionValueType(i))) {
-      if (!Context->IsFunctionIntrinsic(i) &&
-          !IgnorePNaClABIChecks &&
-          !PNaClABITypeChecker::isValidFunctionType(FcnTy)) {
-        Context->SetRecordAddressToFunctionIdAddress(i);
-        Errors() << "Invalid type signature for "
-                 << BitcodeId('f', i) << ": " << *FcnTy << "\n";
-      }
-    }
-  }
-}
-
-bool NaClDisModuleParser::ParseBlock(unsigned BlockID) {
-  ObjDumpSetRecordBitAddress(GetBlock().GetStartBit());
-  switch (BlockID) {
-  case naclbitc::BLOCKINFO_BLOCK_ID: {
-    NaClDisBlockInfoParser Parser(BlockID, this);
-    return Parser.ParseThisBlock();
-  }
-  case naclbitc::TYPE_BLOCK_ID_NEW: {
-    NaClDisTypesParser Parser(BlockID, this);
-    return Parser.ParseThisBlock();
-  }
-  case naclbitc::GLOBALVAR_BLOCK_ID: {
-    NaClDisGlobalsParser Parser(BlockID, this);
-    return Parser.ParseThisBlock();
-  }
-  case naclbitc::VALUE_SYMTAB_BLOCK_ID: {
-    NaClDisValueSymtabParser Parser(BlockID, this);
-    return Parser.ParseThisBlock();
-  }
-  case naclbitc::FUNCTION_BLOCK_ID: {
-    NaClDisFunctionParser Parser(BlockID, this);
-    return Parser.ParseThisBlock();
-  }
-  default:
-    return NaClDisBlockParser::ParseBlock(BlockID);
-  }
-}
-
-void NaClDisModuleParser::PrintBlockHeader() {
-  Tokens() << "module" << Space() << OpenCurly()
-           << Space() << Space() << "// BlockID = " << GetBlockID()
-           << Endline();
-}
-
-void NaClDisModuleParser::ProcessRecord() {
-  ObjDumpSetRecordBitAddress(Record.GetStartBit());
-  const NaClBitcodeRecord::RecordVector &Values = Record.GetValues();
-  switch (Record.GetCode()) {
-  case naclbitc::MODULE_CODE_VERSION:
-    // [version#]
-    if (Values.size() != 1) {
-      Errors() << "Version record should have one argument. Found: "
-               << Values.size() << "\n";
-      break;
-    }
-    Tokens() << "version" << Space() << Values[0] << Semicolon()
-             << TokenizeAbbrevIndex() << Endline();
-    break;
-  case naclbitc::MODULE_CODE_FUNCTION: {
-    // [type, callingconv, isproto, linkage]
-    if (Values.size() != 4) {
-      Errors() << "Function record should have 4 arguments. Found: "
-               << Values.size() << "\n";
-      break;
-    }
-    bool IsProto = (Values[2] != 0);
-    Tokens() << StartCluster() << (IsProto ? "declare" : "define");
-    NaClBcIndexSize_t FcnId = GetNumFunctions();
-    BitcodeId FcnName('f', FcnId);
-    std::string FcnStrName(FcnName.GetName());
-    GlobalValue::LinkageTypes Linkage;
-    if (!naclbitc::DecodeLinkage(Values[3], Linkage)) {
-      Errors() << "Unknown linkage value: " << Values[3] << "\n";
-    } else {
-      if (!IgnorePNaClABIChecks &&
-          !PNaClABIProps::isValidGlobalLinkage(Linkage)) {
-        Errors() << "Disallowed linkage type: "
-                 << PNaClABIProps::LinkageName(Linkage) << "\n";
-      }
-      Tokens() << Space() << PNaClABIProps::LinkageName(Linkage);
-    }
-    CallingConv::ID CallingConv;
-    if (!naclbitc::DecodeCallingConv(Values[1], CallingConv)) {
-      Errors() << "Unknown calling convention value: " << Values[1] << "\n";
-    } else if (PNaClABIProps::isValidCallingConv(CallingConv)) {
-      if (CallingConv != CallingConv::C) {
-        Tokens() << Space() << PNaClABIProps::CallingConvName(CallingConv);
-      }
-    } else {
-      Errors() << "Function " << FcnStrName
-               << " has disallowed calling convention: "
-               << PNaClABIProps::CallingConvName(CallingConv)
-               << " (" << CallingConv << ")\n";
-    }
-    Tokens() << FinishCluster() << Space() << StartCluster();
-    Type *FcnType = GetType(Values[0]);
-    FunctionType *FunctionTy = dyn_cast<FunctionType>(FcnType);
-    if (FunctionTy) {
-      Tokens() << TokenizeFunctionType(FunctionTy, &FcnName);
-    } else {
-      BitcodeId FcnTypeId('t', Values[0]);
-      Errors() << "Not function type: " << FcnTypeId << " = "
-               << *FcnType << "\n";
-      Tokens() << "???";
-      SmallVector<Type*, 1> Signature;
-      FunctionTy = FunctionType::get(GetVoidType(), Signature, 0);
-    }
-    Tokens() << Semicolon() << FinishCluster() << TokenizeAbbrevIndex()
-             << Endline();
-    InstallFunctionType(FunctionTy);
-    if (!IsProto) InstallDefinedFunction(FcnId);
-    break;
-  }
-  default:
-    Errors() << "Unknown record found in module block\n";
-    break;
-  }
-  ObjDumpWrite(Record.GetStartBit(), Record);
-}
-
-bool NaClDisTopLevelParser::ParseBlock(unsigned BlockID) {
-  // Before parsing top-level module block. Describe header by
-  // reconstructing the corresponding header record.
-  NaClBitcodeRecordData HeaderRecord;
-  size_t HeaderSize = Header.getHeaderSize();
-  HeaderRecord.Code = naclbitc::BLK_CODE_HEADER;
-  NaClBitstreamCursor &Cursor = Record.GetCursor();
-  uint64_t CurPos = Cursor.GetCurrentBitNo();
-  Cursor.JumpToBit(0);
-  for (size_t i = 0; i < HeaderSize; ++i) {
-    HeaderRecord.Values.push_back(Cursor.Read(CHAR_BIT));
-  }
-  Cursor.JumpToBit(CurPos);
-  if (ObjDump.GetDumpRecords() && ObjDump.GetDumpAssembly()) {
-    if (HeaderSize >= 4) {
-      const NaClRecordVector &Values = HeaderRecord.Values;
-      Tokens() << "Magic" << Space() << "Number" << Colon()
-               << Space() << StartCluster() << StartCluster() << "'"
-               << (char) Values[0] << (char) Values[1]
-               << (char) Values[2] << (char) Values[3]
-               << "'" << FinishCluster() << Space()
-               << StartCluster() << OpenParen()
-               << Values[0] << Comma() << Space()
-               << Values[1] << Comma() << Space()
-               << Values[2] << Comma() << Space()
-               << Values[3] << CloseParen() << FinishCluster()
-               << FinishCluster() << Endline();
-    }
-    // Show interpretation of header as assembly.
-    for (size_t i = 0; i < Header.NumberFields(); ++i) {
-      Tokens() << Header.GetField(i)->Contents() << Endline();
-    }
-  }
-  ObjDump.Write(0, HeaderRecord);
-
-  if (BlockID != naclbitc::MODULE_BLOCK_ID)
-    return Error("Module block expected at top-level, but not found");
-
-  // Now parse a module block.
-  NaClDisModuleParser Parser(BlockID, this);
-  return Parser.ParseThisBlock();
-}
-
-}
-
-namespace llvm {
-
-bool NaClObjDump(MemoryBufferRef MemBuf, raw_ostream &Output,
-                 bool NoRecords, bool NoAssembly) {
-  // Create objects needed to run parser.
-  naclbitc::ObjDumpStream ObjDump(Output, !NoRecords, !NoAssembly);
-
-  if (MemBuf.getBufferSize() % 4 != 0) {
-    ObjDump.Error()
-        << "Bitcode stream should be a multiple of 4 bytes in length.\n";
-    return true;
-  }
-
-  const unsigned char *BufPtr = (const unsigned char *)MemBuf.getBufferStart();
-  const unsigned char *EndBufPtr = BufPtr+MemBuf.getBufferSize();
-
-  // Read header and verify it is good.
-  NaClBitcodeHeader Header;
-  if (Header.Read(BufPtr, EndBufPtr)) {
-    ObjDump.Error() << "Invalid PNaCl bitcode header.\n";
-    return true;
-  }
-  if (!Header.IsSupported()) {
-    ObjDump.Warning() << Header.Unsupported();
-    if (!Header.IsReadable()) {
-      ObjDump.Error() << "Invalid PNaCl bitcode header.\n";
-      return true;
-    }
-  }
-
-  // Create a bitstream reader to read the bitcode file.
-  NaClBitstreamReader InputStreamFile(BufPtr, EndBufPtr, Header);
-  NaClBitstreamCursor InputStream(InputStreamFile);
-
-  // Parse the the bitcode file.
-  ::NaClDisTopLevelParser Parser(Header, InputStream, ObjDump);
-  int NumBlocksRead = 0;
-  bool ErrorsFound = false;
-  while (!InputStream.AtEndOfStream()) {
-    ++NumBlocksRead;
-    if (Parser.Parse()) ErrorsFound = true;
-  }
-
-  if (NumBlocksRead != 1) {
-    ObjDump.Error() << "Expected 1 top level block in bitcode: Found:"
-                    << NumBlocksRead << "\n";
-    ErrorsFound = true;
-  }
-
-  ObjDump.Flush();
-  return ErrorsFound || Parser.GetNumErrors() > 0;
-}
-
-}
diff --git a/lib/Bitcode/NaCl/Analysis/NaClObjDumpStream.cpp b/lib/Bitcode/NaCl/Analysis/NaClObjDumpStream.cpp
deleted file mode 100644
index bab93cfcfa6..00000000000
--- a/lib/Bitcode/NaCl/Analysis/NaClObjDumpStream.cpp
+++ /dev/null
@@ -1,343 +0,0 @@
-//===-- NaClObjDumpStream.cpp --------------------------------------------===//
-//      Implements an objdump stream (bitcode records/assembly code).
-//
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
-//
-//===----------------------------------------------------------------------===//
-
-#include "llvm/ADT/STLExtras.h"
-#include "llvm/Bitcode/NaCl/NaClObjDumpStream.h"
-#include "llvm/Support/DataTypes.h"
-#include "llvm/Support/ErrorHandling.h"
-
-namespace llvm {
-namespace naclbitc {
-
-TextFormatter::TextFormatter(raw_ostream &BaseStream,
-                             unsigned LineWidth,
-                             const char *Tab)
-    : TextIndenter(Tab),
-      BaseStream(BaseStream),
-      TextStream(TextBuffer),
-      LineWidth(LineWidth),
-      LinePosition(0),
-      CurrentIndent(0),
-      MinLineWidth(20),
-      AtInstructionBeginning(true),
-      LineIndent(),
-      ContinuationIndent(),
-      ClusteringLevel(0),
-      ClusteredTextSize(0) {
-  if (MinLineWidth > LineWidth) MinLineWidth = LineWidth;
-}
-
-TextFormatter::~TextFormatter() {}
-
-void TextFormatter::WriteEndline() {
-  assert(!IsClustering() && "Must close clustering before ending instruction");
-  Write('\n');
-  CurrentIndent = 0;
-  AtInstructionBeginning = true;
-  LineIndent.clear();
-}
-
-std::string TextFormatter::GetToken() {
-  TextStream.flush();
-  std::string Token(TextBuffer);
-  TextBuffer.clear();
-  if (!Token.empty() && IsClustering())
-    AppendForReplay(GetTokenDirective::Allocate(this, Token));
-  return Token;
-}
-
-void TextFormatter::Write(char ch) {
-  switch (ch) {
-  case '\n':
-    BaseStream << ch;
-    LinePosition = 0;
-    break;
-  case '\t': {
-    size_t TabWidth = GetTabSize();
-    size_t NumChars = LinePosition % TabWidth;
-    if (NumChars == 0) NumChars = TabWidth;
-    for (size_t i = 0; i < NumChars; ++i) Write(' ');
-    break;
-  }
-  default:
-    if (LinePosition == 0) {
-      WriteLineIndents();
-    }
-    BaseStream << ch;
-    ++LinePosition;
-  }
-  AtInstructionBeginning = false;
-}
-
-void TextFormatter::Write(const std::string &Text) {
-  if (IsClustering()) {
-    ClusteredTextSize += Text.size();
-  } else {
-    for (std::string::const_iterator
-             Iter = Text.begin(), IterEnd = Text.end();
-         Iter != IterEnd; ++Iter) {
-      Write(*Iter);
-    }
-  }
-}
-
-void TextFormatter::StartClustering() {
-  ++ClusteringLevel;
-}
-
-void TextFormatter::FinishClustering() {
-  assert(IsClustering() && "Can't finish clustering, not in cluster!");
-  --ClusteringLevel;
-  if (IsClustering()) return;
-
-  AddLineWrapIfNeeded(ClusteredTextSize);
-
-  // Reapply the directives to generate the token text, and set
-  // indentations. Because clustering can be nested, duplicate before
-  // replaying, so that nested clusters can replayed and build its own
-  // list of clustered directives.
-  std::vector<const Directive*> Directives(ClusteredDirectives);
-  ClusteredDirectives.clear();
-  ClusteredTextSize = 0;
-  for (std::vector<const Directive*>::iterator
-           Iter = Directives.begin(),
-           IterEnd = Directives.end();
-       Iter != IterEnd; ++Iter) {
-    (*Iter)->Apply();
-  }
-}
-
-void TextFormatter::WriteLineIndents() {
-  // Add line indent to base stream.
-  if (AtInstructionBeginning) LineIndent = GetIndent();
-  BaseStream << LineIndent;
-  LinePosition += LineIndent.size();
-
-  // If not the first line, and local indent not set, add continuation
-  // indent to the base stream.
-  unsigned UseIndent = CurrentIndent;
-  if (!AtInstructionBeginning && CurrentIndent == 0) {
-    UseIndent = FixIndentValue(LinePosition + ContinuationIndent.size());
-  }
-
-  // Add any additional indents local to the current instruction
-  // being dumped to the base stream.
-  for (; LinePosition < UseIndent; ++LinePosition) {
-    BaseStream << ' ';
-  }
-}
-
-void TextFormatter::Directive::Reapply() const {
-  // Note: We don't want to store top-level start/finish cluster
-  // directives on ClusteredDirectives, so that nested replays won't
-  // reapply them.
-  bool WasClustering = IsClustering();
-  MyApply(true);
-  if (WasClustering && IsClustering())
-    Formatter->ClusteredDirectives.push_back(this);
-}
-
-
-TextFormatter::Directive *TextFormatter::GetTokenDirective::
-Allocate(TextFormatter *Formatter, const std::string &Text) {
-  GetTokenDirective *Dir = Formatter->GetTokenFreeList.Allocate(Formatter);
-  Dir->Text = Text;
-  return Dir;
-}
-
-std::string RecordTextFormatter::getBitAddress(uint64_t Bit) {
-  std::string Address = naclbitc::getBitAddress(Bit);
-  size_t AddressSize = Address.size();
-  if (AddressSize >= AddressWriteWidth)
-    return Address;
-
-  // Pad address with leading spaces.
-  std::string Buffer;
-  raw_string_ostream StrBuf(Buffer);
-  for (size_t i = AddressWriteWidth - AddressSize; i > 0; --i) {
-    StrBuf << " ";
-  }
-  StrBuf << Address;
-  return StrBuf.str();
-}
-
-RecordTextFormatter::RecordTextFormatter(ObjDumpStream *ObjDump)
-    : TextFormatter(ObjDump->Records(), 0, "  "),
-      OpenBrace(this, "<"),
-      CloseBrace(this, ">"),
-      Comma(this, ","),
-      Space(this),
-      Endline(this),
-      StartCluster(this),
-      FinishCluster(this) {
-  // Handle fact that 64-bit values can take up to 21 characters.
-  MinLineWidth = 21;
-  Label = getBitAddress(0);
-}
-
-std::string RecordTextFormatter::GetEmptyLabelColumn() {
-  std::string Buffer;
-  raw_string_ostream StrmBuffer(Buffer);
-  for (size_t i = 0; i < Label.size(); ++i) {
-    StrmBuffer << ' ';
-  }
-  StrmBuffer << '|';
-  return StrmBuffer.str();
-}
-
-void RecordTextFormatter::WriteLineIndents() {
-  if (AtInstructionBeginning) {
-    BaseStream << Label << '|';
-  } else {
-    BaseStream << GetEmptyLabelColumn();
-  }
-  LinePosition += Label.size() + 1;
-  TextFormatter::WriteLineIndents();
-}
-
-void RecordTextFormatter::WriteValues(uint64_t Bit,
-                                      const llvm::NaClBitcodeValues &Values,
-                                      int32_t AbbrevIndex) {
-  Label = getBitAddress(Bit);
-  if (AbbrevIndex != ABBREV_INDEX_NOT_SPECIFIED) {
-    TextStream << AbbrevIndex << ":" << Space;
-  }
-  TextStream << OpenBrace;
-  for (size_t i = 0; i < Values.size(); ++i) {
-    if (i > 0) {
-      TextStream << Comma << FinishCluster << Space;
-    }
-    TextStream << StartCluster << Values[i];
-  }
-  // Note: Because of record codes, Values are never empty. Hence we
-  // always need to finish the cluster for the last number printed.
-  TextStream << FinishCluster << CloseBrace << Endline;
-}
-
-unsigned ObjDumpStream::DefaultMaxErrors = 20;
-
-unsigned ObjDumpStream::ComboObjDumpSeparatorColumn = 40;
-
-unsigned ObjDumpStream::RecordObjectDumpLength = 80;
-
-ObjDumpStream::ObjDumpStream(raw_ostream &Stream,
-                             bool DumpRecords, bool DumpAssembly)
-    : Stream(Stream),
-      DumpRecords(DumpRecords),
-      DumpAssembly(DumpAssembly),
-      NumErrors(0),
-      MaxErrors(DefaultMaxErrors),
-      RecordWidth(0),
-      AssemblyBuffer(),
-      AssemblyStream(AssemblyBuffer),
-      MessageBuffer(),
-      MessageStream(MessageBuffer),
-      ColumnSeparator('|'),
-      LastKnownBit(0),
-      RecordBuffer(),
-      RecordStream(RecordBuffer),
-      RecordFormatter(this) {
-  if (DumpRecords) {
-    RecordWidth = DumpAssembly
-        ? ComboObjDumpSeparatorColumn
-        : RecordObjectDumpLength;
-    RecordFormatter.SetLineWidth(RecordWidth);
-  }
-}
-
-raw_ostream &ObjDumpStream::ErrorAt(naclbitc::ErrorLevel Level, uint64_t Bit) {
-  // Note: Since this method only prints the error line prefix, and
-  // lets the caller fill in the rest of the error.
-  if (NumErrors >= MaxErrors) {
-    // If we've hit the maximum number of errors, let Flush empty the buffers,
-    // print out reported errors, and then exit the executable.
-    Flush();
-    llvm_unreachable("Flush shouldn't return if too many errors");
-  }
-  LastKnownBit = Bit;
-  if (Level >= naclbitc::Error)
-    ++NumErrors;
-  return naclbitc::ErrorAt(Comments(), Level, Bit);
-}
-
-// Dumps the next line of text in the buffer. Returns the number of characters
-// printed.
-static size_t DumpLine(raw_ostream &Stream,
-                       const std::string &Buffer,
-                       size_t &Index,
-                       size_t Size) {
-  size_t Count = 0;
-  while (Index < Size) {
-    char ch = Buffer[Index];
-    if (ch == '\n') {
-      // At end of line, stop here.
-      ++Index;
-      return Count;
-    } else {
-      Stream << ch;
-      ++Index;
-      ++Count;
-    }
-  }
-  return Count;
-}
-
-void ObjDumpStream::Flush() {
-  // Start by flushing all buffers, so that we know the
-  // text that must be written.
-  RecordStream.flush();
-  AssemblyStream.flush();
-  MessageStream.flush();
-
-  // See if there is any record/assembly lines to print.
-  if ((DumpRecords && !RecordBuffer.empty())
-      || (DumpAssembly && !AssemblyBuffer.empty())) {
-    size_t RecordIndex = 0;
-    size_t RecordSize = DumpRecords ? RecordBuffer.size() : 0;
-    size_t AssemblyIndex = 0;
-    size_t AssemblySize = DumpAssembly ? AssemblyBuffer.size() : 0;
-
-    while (RecordIndex < RecordSize || AssemblyIndex < AssemblySize) {
-      // Dump next record line.
-      size_t Column = DumpLine(Stream, RecordBuffer, RecordIndex, RecordSize);
-      // Now move to separator if assembly is to be printed also.
-      if (DumpRecords && DumpAssembly) {
-        if (Column == 0) {
-          // Add indent filler.
-          std::string Label = RecordFormatter.GetEmptyLabelColumn();
-          Stream << Label;
-          Column += Label.size();
-        }
-        for (size_t i = Column; i < RecordWidth; ++i) {
-          Stream << ' ';
-        }
-        Stream << ColumnSeparator;
-      }
-      // Dump next assembly line.
-      DumpLine(Stream, AssemblyBuffer, AssemblyIndex, AssemblySize);
-      Stream << '\n';
-    }
-  }
-
-  // Print out messages and reset buffers.
-  Stream << MessageBuffer;
-  Stream.flush();
-  ResetBuffers();
-  if (NumErrors >= MaxErrors)
-    report_fatal_error("Too many errors, Unable to continue");
-}
-
-void ObjDumpStream::FlushThenQuit() {
-  Flush();
-  report_fatal_error("Unable to continue");
-}
-
-}
-}
diff --git a/lib/Bitcode/NaCl/CMakeLists.txt b/lib/Bitcode/NaCl/CMakeLists.txt
deleted file mode 100644
index 22faa88bd63..00000000000
--- a/lib/Bitcode/NaCl/CMakeLists.txt
+++ /dev/null
@@ -1,5 +0,0 @@
-add_subdirectory(Writer)
-add_subdirectory(Reader)
-add_subdirectory(Analysis)
-add_subdirectory(TestUtils)
-
diff --git a/lib/Bitcode/NaCl/LLVMBuild.txt b/lib/Bitcode/NaCl/LLVMBuild.txt
deleted file mode 100644
index d3e410de9f8..00000000000
--- a/lib/Bitcode/NaCl/LLVMBuild.txt
+++ /dev/null
@@ -1,24 +0,0 @@
-;===- ./lib/Bitcode/NaCl/LLVMBuild.txt ------------------------*- Conf -*--===;
-;
-;                     The LLVM Compiler Infrastructure
-;
-; This file is distributed under the University of Illinois Open Source
-; License. See LICENSE.TXT for details.
-;
-;===------------------------------------------------------------------------===;
-;
-; This is an LLVMBuild description file for the components in this subdirectory.
-;
-; For more information on the LLVMBuild system, please see:
-;
-;   http://llvm.org/docs/LLVMBuild.html
-;
-;===------------------------------------------------------------------------===;
-
-[common]
-subdirectories = Writer Reader Analysis TestUtils
-
-[component_0]
-type = Group
-name = NaClBitcode
-parent = Bitcode
diff --git a/lib/Bitcode/NaCl/Makefile b/lib/Bitcode/NaCl/Makefile
deleted file mode 100644
index 46ac6e49789..00000000000
--- a/lib/Bitcode/NaCl/Makefile
+++ /dev/null
@@ -1,14 +0,0 @@
-##===- lib/Bitcode/NaCl/Makefile ---------------------------*- Makefile -*-===##
-#
-#                     The LLVM Compiler Infrastructure
-#
-# This file is distributed under the University of Illinois Open Source
-# License. See LICENSE.TXT for details.
-#
-##===----------------------------------------------------------------------===##
-
-LEVEL = ../../..
-PARALLEL_DIRS = Writer Reader Analysis TestUtils
-
-include $(LEVEL)/Makefile.common
-
diff --git a/lib/Bitcode/NaCl/Reader/CMakeLists.txt b/lib/Bitcode/NaCl/Reader/CMakeLists.txt
deleted file mode 100644
index fa0e4e9be48..00000000000
--- a/lib/Bitcode/NaCl/Reader/CMakeLists.txt
+++ /dev/null
@@ -1,10 +0,0 @@
-add_llvm_library(LLVMNaClBitReader
-  NaClBitCodes.cpp
-  NaClBitcodeHeader.cpp
-  NaClBitcodeReader.cpp
-  NaClBitstreamReader.cpp
-  NaClBitcodeParser.cpp
-  NaClBitcodeDecoders.cpp
-  )
-
-add_dependencies(LLVMNaClBitReader intrinsics_gen)
diff --git a/lib/Bitcode/NaCl/Reader/LLVMBuild.txt b/lib/Bitcode/NaCl/Reader/LLVMBuild.txt
deleted file mode 100644
index 4c5f2c41995..00000000000
--- a/lib/Bitcode/NaCl/Reader/LLVMBuild.txt
+++ /dev/null
@@ -1,22 +0,0 @@
-;===- ./lib/Bitcode/NaClReader/LLVMBuild.txt -------------------*- Conf -*--===;
-;
-;                     The LLVM Compiler Infrastructure
-;
-; This file is distributed under the University of Illinois Open Source
-; License. See LICENSE.TXT for details.
-;
-;===------------------------------------------------------------------------===;
-;
-; This is an LLVMBuild description file for the components in this subdirectory.
-;
-; For more information on the LLVMBuild system, please see:
-;
-;   http://llvm.org/docs/LLVMBuild.html
-;
-;===------------------------------------------------------------------------===;
-
-[component_0]
-type = Library
-name = NaClBitReader
-parent = NaClBitcode
-required_libraries = Core Support NaClAnalysis
diff --git a/lib/Bitcode/NaCl/Reader/Makefile b/lib/Bitcode/NaCl/Reader/Makefile
deleted file mode 100644
index 92c75c29a41..00000000000
--- a/lib/Bitcode/NaCl/Reader/Makefile
+++ /dev/null
@@ -1,15 +0,0 @@
-##===- lib/Bitcode/NaCl/Reader/Makefile --------------------*- Makefile -*-===##
-#
-#                     The LLVM Compiler Infrastructure
-#
-# This file is distributed under the University of Illinois Open Source
-# License. See LICENSE.TXT for details.
-#
-##===----------------------------------------------------------------------===##
-
-LEVEL = ../../../..
-LIBRARYNAME = LLVMNaClBitReader
-BUILD_ARCHIVE = 1
-
-include $(LEVEL)/Makefile.common
-
diff --git a/lib/Bitcode/NaCl/Reader/NaClBitCodes.cpp b/lib/Bitcode/NaCl/Reader/NaClBitCodes.cpp
deleted file mode 100644
index 72674115194..00000000000
--- a/lib/Bitcode/NaCl/Reader/NaClBitCodes.cpp
+++ /dev/null
@@ -1,131 +0,0 @@
-//===- NaClBitcodeHeader.cpp ----------------------------------------------===//
-//     PNaCl bitcode header reader.
-//
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
-//
-//===----------------------------------------------------------------------===//
-//
-// Implementation of Bitcode abbrevations.
-//
-//===----------------------------------------------------------------------===//
-
-#include "llvm/Bitcode/NaCl/NaClBitCodes.h"
-#include "llvm/Support/raw_ostream.h"
-#include "llvm/Support/ErrorHandling.h"
-
-using namespace llvm;
-
-const bool NaClBitCodeAbbrevOp::HasValueArray[] = {
-  true,  // Literal
-  true,  // Fixed
-  true,  // VBR
-  false, // Array
-  false  // Char6
-};
-
-const char *NaClBitCodeAbbrevOp::EncodingNameArray[] = {
-  "Literal",
-  "Fixed",
-  "VBR",
-  "Array",
-  "Char6"
-};
-
-NaClBitCodeAbbrevOp::NaClBitCodeAbbrevOp(Encoding E, uint64_t Data)
-    : Enc(E), Val(Data) {
-  if (isValid(E, Data)) return;
-  std::string Buffer;
-  raw_string_ostream StrBuf(Buffer);
-  StrBuf << "Invalid NaClBitCodeAbbrevOp(" << E << ", " << Data << ")";
-  report_fatal_error(StrBuf.str());
-}
-
-bool NaClBitCodeAbbrevOp::isValid(Encoding E, uint64_t Val) {
-  switch (NaClBitCodeAbbrevOp::Encoding(E)) {
-  case Literal:
-    return true;
-  case Fixed:
-  case VBR:
-    return Val <= naclbitc::MaxAbbrevWidth;
-  case Char6:
-  case Array:
-    return Val == 0;
-  }
-  llvm_unreachable("unhandled abbreviation");
-}
-
-void NaClBitCodeAbbrevOp::Print(raw_ostream& Stream) const {
-  if (Enc == Literal) {
-    Stream << getValue();
-    return;
-  }
-  Stream << getEncodingName(Enc);
-  if (!hasValue())
-    return;
-  Stream << "(" << Val << ")";
-}
-
-static void PrintExpression(raw_ostream &Stream,
-                            const NaClBitCodeAbbrev *Abbrev,
-                            unsigned &Index) {
-  // Bail out early, in case we are incrementally building the
-  // expression and the argument is not available yet.
-  if (Index >= Abbrev->getNumOperandInfos()) return;
-
-  const NaClBitCodeAbbrevOp &Op = Abbrev->getOperandInfo(Index);
-  Op.Print(Stream);
-  if (unsigned NumArgs = Op.NumArguments()) {
-    Stream << "(";
-    for (unsigned i = 0; i < NumArgs; ++i) {
-      ++Index;
-      if (i > 0) Stream << ",";
-      PrintExpression(Stream, Abbrev, Index);
-    }
-    Stream << ")";
-  }
-}
-
-void NaClBitCodeAbbrev::Print(raw_ostream &Stream, bool AddNewLine) const {
-  Stream << "[";
-  for (unsigned i = 0; i < getNumOperandInfos(); ++i) {
-    if (i > 0) Stream << ", ";
-    PrintExpression(Stream, this, i);
-  }
-  Stream << "]";
-  if (AddNewLine) Stream << "\n";
-}
-
-NaClBitCodeAbbrev *NaClBitCodeAbbrev::Simplify() const {
-  NaClBitCodeAbbrev *Abbrev = new NaClBitCodeAbbrev();
-  for (unsigned i = 0; i < OperandList.size(); ++i) {
-    const NaClBitCodeAbbrevOp &Op = OperandList[i];
-    // Simplify if possible.  Currently, the only simplification known
-    // is to remove unnecessary operands appearing immediately before an
-    // array operator. That is, apply the simplification:
-    //    Op Array(Op) -> Array(Op)
-    assert(!Op.isArrayOp() || i == OperandList.size()-2);
-    while (Op.isArrayOp() && !Abbrev->OperandList.empty() &&
-           Abbrev->OperandList.back() == OperandList[i+1]) {
-      Abbrev->OperandList.pop_back();
-    }
-    Abbrev->OperandList.push_back(Op);
-  }
-  return Abbrev;
-}
-
-bool NaClBitCodeAbbrev::isValid() const {
-  // Verify that an array op appears can only appear if it is the
-  // second to last element.
-  unsigned NumOperands = getNumOperandInfos();
-  if (NumOperands == 0) return false;
-  for (unsigned i = 0; i < NumOperands; ++i) {
-    const NaClBitCodeAbbrevOp &Op = getOperandInfo(i);
-    if (Op.isArrayOp() && i + 2 != NumOperands)
-      // Note: Unlike LLVM bitcode, we allow literals in arrays!
-      return false;
-  }
-  return true;
-}
diff --git a/lib/Bitcode/NaCl/Reader/NaClBitcodeDecoders.cpp b/lib/Bitcode/NaCl/Reader/NaClBitcodeDecoders.cpp
deleted file mode 100644
index 450499a3519..00000000000
--- a/lib/Bitcode/NaCl/Reader/NaClBitcodeDecoders.cpp
+++ /dev/null
@@ -1,228 +0,0 @@
-//===- NaClBitcodeDecoders.cpp --------------------------------------------===//
-//     Internal implementation of decoder functions for PNaCl Bitcode files.
-//
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
-//
-//===----------------------------------------------------------------------===//
-
-#include "llvm/Bitcode/NaCl/NaClBitcodeDecoders.h"
-
-namespace llvm {
-namespace naclbitc {
-
-bool DecodeCastOpcode(uint64_t NaClOpcode,
-                      Instruction::CastOps &LLVMOpcode) {
-  switch (NaClOpcode) {
-  default:
-    LLVMOpcode = Instruction::BitCast;
-    return false;
-  case naclbitc::CAST_TRUNC:
-    LLVMOpcode = Instruction::Trunc;
-    return true;
-  case naclbitc::CAST_ZEXT:
-    LLVMOpcode = Instruction::ZExt;
-    return true;
-  case naclbitc::CAST_SEXT:
-    LLVMOpcode = Instruction::SExt;
-    return true;
-  case naclbitc::CAST_FPTOUI:
-    LLVMOpcode = Instruction::FPToUI;
-    return true;
-  case naclbitc::CAST_FPTOSI:
-    LLVMOpcode = Instruction::FPToSI;
-    return true;
-  case naclbitc::CAST_UITOFP:
-    LLVMOpcode = Instruction::UIToFP;
-    return true;
-  case naclbitc::CAST_SITOFP:
-    LLVMOpcode = Instruction::SIToFP;
-    return true;
-  case naclbitc::CAST_FPTRUNC:
-    LLVMOpcode = Instruction::FPTrunc;
-    return true;
-  case naclbitc::CAST_FPEXT:
-    LLVMOpcode = Instruction::FPExt;
-    return true;
-  case naclbitc::CAST_BITCAST:
-    LLVMOpcode = Instruction::BitCast;
-    return true;
-  }
-}
-
-bool DecodeLinkage(uint64_t NaClLinkage,
-                   GlobalValue::LinkageTypes &LLVMLinkage) {
-  switch (NaClLinkage) {
-  default:
-    LLVMLinkage = GlobalValue::InternalLinkage;
-    return false;
-  case naclbitc::LINKAGE_EXTERNAL:
-    LLVMLinkage = GlobalValue::ExternalLinkage;
-    return true;
-  case naclbitc::LINKAGE_INTERNAL:
-    LLVMLinkage = GlobalValue::InternalLinkage;
-    return true;
-  }
-}
-
-bool DecodeBinaryOpcode(uint64_t NaClOpcode, Type *Ty,
-                        Instruction::BinaryOps &LLVMOpcode) {
-  switch (NaClOpcode) {
-  default:
-    LLVMOpcode = Instruction::Add;
-    return false;
-  case naclbitc::BINOP_ADD:
-    LLVMOpcode = Ty->isFPOrFPVectorTy() ? Instruction::FAdd : Instruction::Add;
-    return true;
-  case naclbitc::BINOP_SUB:
-    LLVMOpcode = Ty->isFPOrFPVectorTy() ? Instruction::FSub : Instruction::Sub;
-    return true;
-  case naclbitc::BINOP_MUL:
-    LLVMOpcode = Ty->isFPOrFPVectorTy() ? Instruction::FMul : Instruction::Mul;
-    return true;
-  case naclbitc::BINOP_UDIV:
-    LLVMOpcode = Instruction::UDiv;
-    return true;
-  case naclbitc::BINOP_SDIV:
-    LLVMOpcode = Ty->isFPOrFPVectorTy() ? Instruction::FDiv : Instruction::SDiv;
-    return true;
-  case naclbitc::BINOP_UREM:
-    LLVMOpcode = Instruction::URem;
-    return true;
-  case naclbitc::BINOP_SREM:
-    LLVMOpcode = Ty->isFPOrFPVectorTy() ? Instruction::FRem : Instruction::SRem;
-    return true;
-  case naclbitc::BINOP_SHL:
-    LLVMOpcode = Instruction::Shl;
-    return true;
-  case naclbitc::BINOP_LSHR:
-    LLVMOpcode = Instruction::LShr;
-    return true;
-  case naclbitc::BINOP_ASHR:
-    LLVMOpcode = Instruction::AShr;
-    return true;
-  case naclbitc::BINOP_AND:
-    LLVMOpcode = Instruction::And;
-    return true;
-  case naclbitc::BINOP_OR:
-    LLVMOpcode = Instruction::Or;
-    return true;
-  case naclbitc::BINOP_XOR:
-    LLVMOpcode = Instruction::Xor;
-    return true;
-  }
-}
-
-bool DecodeCallingConv(uint64_t NaClCallingConv,
-                       CallingConv::ID &LLVMCallingConv) {
-  switch (NaClCallingConv) {
-  default:
-    LLVMCallingConv = CallingConv::C;
-    return false;
-  case naclbitc::C_CallingConv:
-    LLVMCallingConv = CallingConv::C;
-    return true;
-  }
-}
-
-bool DecodeFcmpPredicate(uint64_t NaClPredicate,
-                         CmpInst::Predicate &LLVMPredicate) {
-  switch (NaClPredicate) {
-  default:
-    LLVMPredicate = CmpInst::FCMP_FALSE;
-    return false;
-  case naclbitc::FCMP_FALSE:
-    LLVMPredicate = CmpInst::FCMP_FALSE;
-    return true;
-  case naclbitc::FCMP_OEQ:
-    LLVMPredicate = CmpInst::FCMP_OEQ;
-    return true;
-  case naclbitc::FCMP_OGT:
-    LLVMPredicate = CmpInst::FCMP_OGT;
-    return true;
-  case naclbitc::FCMP_OGE:
-    LLVMPredicate = CmpInst::FCMP_OGE;
-    return true;
-  case naclbitc::FCMP_OLT:
-    LLVMPredicate = CmpInst::FCMP_OLT;
-    return true;
-  case naclbitc::FCMP_OLE:
-    LLVMPredicate = CmpInst::FCMP_OLE;
-    return true;
-  case naclbitc::FCMP_ONE:
-    LLVMPredicate = CmpInst::FCMP_ONE;
-    return true;
-  case naclbitc::FCMP_ORD:
-    LLVMPredicate = CmpInst::FCMP_ORD;
-    return true;
-  case naclbitc::FCMP_UNO:
-    LLVMPredicate = CmpInst::FCMP_UNO;
-    return true;
-  case naclbitc::FCMP_UEQ:
-    LLVMPredicate = CmpInst::FCMP_UEQ;
-    return true;
-  case naclbitc::FCMP_UGT:
-    LLVMPredicate = CmpInst::FCMP_UGT;
-    return true;
-  case naclbitc::FCMP_UGE:
-    LLVMPredicate = CmpInst::FCMP_UGE;
-    return true;
-  case naclbitc::FCMP_ULT:
-    LLVMPredicate = CmpInst::FCMP_ULT;
-    return true;
-  case naclbitc::FCMP_ULE:
-    LLVMPredicate = CmpInst::FCMP_ULE;
-    return true;
-  case naclbitc::FCMP_UNE:
-    LLVMPredicate = CmpInst::FCMP_UNE;
-    return true;
-  case naclbitc::FCMP_TRUE:
-    LLVMPredicate = CmpInst::FCMP_TRUE;
-    return true;
-  }
-}
-
-bool DecodeIcmpPredicate(uint64_t NaClPredicate,
-                         CmpInst::Predicate &LLVMPredicate) {
-  switch (NaClPredicate) {
-  default:
-    LLVMPredicate = CmpInst::ICMP_EQ;
-    return false;
-  case naclbitc::ICMP_EQ:
-    LLVMPredicate = CmpInst::ICMP_EQ;
-    return true;
-  case naclbitc::ICMP_NE:
-    LLVMPredicate = CmpInst::ICMP_NE;
-    return true;
-  case naclbitc::ICMP_UGT:
-    LLVMPredicate = CmpInst::ICMP_UGT;
-    return true;
-  case naclbitc::ICMP_UGE:
-    LLVMPredicate = CmpInst::ICMP_UGE;
-    return true;
-  case naclbitc::ICMP_ULT:
-    LLVMPredicate = CmpInst::ICMP_ULT;
-    return true;
-  case naclbitc::ICMP_ULE:
-    LLVMPredicate = CmpInst::ICMP_ULE;
-    return true;
-  case naclbitc::ICMP_SGT:
-    LLVMPredicate = CmpInst::ICMP_SGT;
-    return true;
-  case naclbitc::ICMP_SGE:
-    LLVMPredicate = CmpInst::ICMP_SGE;
-    return true;
-  case naclbitc::ICMP_SLT:
-    LLVMPredicate = CmpInst::ICMP_SLT;
-    return true;
-  case naclbitc::ICMP_SLE:
-    LLVMPredicate = CmpInst::ICMP_SLE;
-    return true;
-  }
-}
-
-
-}
-}
diff --git a/lib/Bitcode/NaCl/Reader/NaClBitcodeHeader.cpp b/lib/Bitcode/NaCl/Reader/NaClBitcodeHeader.cpp
deleted file mode 100644
index 8a65f50829b..00000000000
--- a/lib/Bitcode/NaCl/Reader/NaClBitcodeHeader.cpp
+++ /dev/null
@@ -1,351 +0,0 @@
-//===- NaClBitcodeHeader.cpp ----------------------------------------------===//
-//     PNaCl bitcode header reader.
-//
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
-//
-//===----------------------------------------------------------------------===//
-
-#include "llvm/Bitcode/NaCl/NaClBitcodeHeader.h"
-
-#include "llvm/ADT/SmallSet.h"
-#include "llvm/Bitcode/NaCl/NaClReaderWriter.h"
-#include "llvm/Bitcode/ReaderWriter.h"
-#include "llvm/Support/ErrorHandling.h"
-#include "llvm/Support/Format.h"
-#include "llvm/Support/raw_ostream.h"
-#include "llvm/Support/StreamingMemoryObject.h"
-
-#include <cstring>
-#include <iomanip>
-#include <limits>
-
-using namespace llvm;
-
-namespace {
-
-// The name for each ID tag.
-static const char *TagName[] = {
-  "Invalid",              // kInvalid
-  "PNaCl Version",        // kPNaClVersion
-  "Align bitcode records" // kAlignBitcodeRecords
-};
-
-// The name for each field type.
-static const char *FieldTypeName[] = {
-  "uint8[]", // kBufferType
-  "uint32",  // kUInt32Type
-  "flag",    // kFlagType
-  "unknown"  // kUnknownType
-};
-
-// The type associated with each ID tag.
-static const NaClBitcodeHeaderField::FieldType ExpectedType[] = {
-  NaClBitcodeHeaderField::kUnknownType, // kInvalid
-  NaClBitcodeHeaderField::kUInt32Type,  // kPNaClVersion
-  NaClBitcodeHeaderField::kFlagType     // kAlignBitcodeRecords
-};
-
-} // end of anonymous namespace
-
-const char *NaClBitcodeHeaderField::IDName(Tag ID) {
-  return ID > kTag_MAX ? "???" : TagName[ID];
-}
-
-const char *NaClBitcodeHeaderField::TypeName(FieldType FType) {
-  return FType > kFieldType_MAX ? "???" : FieldTypeName[FType];
-}
-
-NaClBitcodeHeaderField::NaClBitcodeHeaderField()
-  : ID(kInvalid), FType(kBufferType), Len(0), Data(0) {}
-
-NaClBitcodeHeaderField::NaClBitcodeHeaderField(Tag MyID)
-  : ID(MyID), FType(kFlagType), Len(0), Data(0) {
-  assert(MyID <= kTag_MAX);
-}
-
-NaClBitcodeHeaderField::NaClBitcodeHeaderField(Tag MyID, uint32_t MyValue)
-    : ID(MyID), FType(kUInt32Type), Len(4), Data(new uint8_t[4]) {
-  assert(MyID <= kTag_MAX);
-  Data[0] = static_cast<uint8_t>(MyValue & 0xFF);
-  Data[1] = static_cast<uint8_t>((MyValue >> 8) & 0xFF);
-  Data[2] = static_cast<uint8_t>((MyValue >> 16) & 0xFF);
-  Data[3] = static_cast<uint8_t>((MyValue >> 24) & 0xFF);
-}
-
-uint32_t NaClBitcodeHeaderField::GetUInt32Value() const {
-  assert(FType == kUInt32Type && "Header field must be uint32");
-  return static_cast<uint32_t>(Data[0]) |
-         (static_cast<uint32_t>(Data[1]) << 8) |
-         (static_cast<uint32_t>(Data[2]) << 16) |
-         (static_cast<uint32_t>(Data[2]) << 24);
-}
-
-NaClBitcodeHeaderField::NaClBitcodeHeaderField(Tag MyID, size_t MyLen,
-                                               uint8_t *MyData)
-    : ID(MyID), FType(kBufferType), Len(MyLen), Data(new uint8_t[MyLen]) {
-  assert(MyID <= kTag_MAX);
-  for (size_t i = 0; i < MyLen; ++i) {
-    Data[i] = MyData[i];
-  }
-}
-
-bool NaClBitcodeHeaderField::Write(uint8_t *Buf, size_t BufLen) const {
-  size_t FieldsLen = kTagLenSize + Len;
-  size_t PadLen = (WordSize - (FieldsLen & (WordSize-1))) & (WordSize-1);
-  // Ensure buffer is large enough and that length can be represented
-  // in 32 bits
-  if (BufLen < FieldsLen + PadLen ||
-      Len > std::numeric_limits<FixedSubfield>::max())
-    return false;
-
-  WriteFixedSubfield(EncodeTypedID(), Buf);
-  WriteFixedSubfield(static_cast<FixedSubfield>(Len),
-                     Buf + sizeof(FixedSubfield));
-  memcpy(Buf + kTagLenSize, Data, Len);
-  // Pad out to word alignment
-  if (PadLen) {
-    memset(Buf + FieldsLen, 0, PadLen);
-  }
-  return true;
-}
-
-bool NaClBitcodeHeaderField::Read(const uint8_t *Buf, size_t BufLen) {
-  if (BufLen < kTagLenSize)
-    return false;
-  FixedSubfield IdField;
-  ReadFixedSubfield(&IdField, Buf);
-  FixedSubfield LengthField;
-  ReadFixedSubfield(&LengthField, Buf + sizeof(FixedSubfield));
-  size_t Length = static_cast<size_t>(LengthField);
-  if (BufLen < kTagLenSize + Length)
-    return false;
-  if (Len != Length) {
-    // Need to reallocate data buffer.
-    if (Data)
-      delete[] Data;
-    Data = new uint8_t[Length];
-  }
-  Len = Length;
-  DecodeTypedID(IdField, ID, FType);
-  memcpy(Data, Buf + kTagLenSize, Len);
-  return true;
-}
-
-std::string NaClBitcodeHeaderField::Contents() const {
-  std::string buffer;
-  raw_string_ostream ss(buffer);
-  ss << IDName() << ": ";
-  switch (FType) {
-  case kFlagType:
-    ss << "true";
-    break;
-  case kUInt32Type:
-    ss << GetUInt32Value();
-    break;
-  case kBufferType:
-    ss << "[";
-    for (size_t i = 0; i < Len; ++i) {
-      if (i)
-        ss << " ";
-      ss << format("%02x", Data[i]);
-    }
-    ss << "]";
-    break;
-  case kUnknownType:
-    ss << "unknown value";
-    break;
-  }
-  return ss.str();
-}
-
-NaClBitcodeHeader::NaClBitcodeHeader()
-    : HeaderSize(0), UnsupportedMessage(), IsSupportedFlag(false),
-      IsReadableFlag(false), PNaClVersion(0) {}
-
-NaClBitcodeHeader::~NaClBitcodeHeader() {
-  for (std::vector<NaClBitcodeHeaderField *>::const_iterator
-           Iter = Fields.begin(),
-           IterEnd = Fields.end();
-       Iter != IterEnd; ++Iter) {
-    delete *Iter;
-  }
-}
-
-bool NaClBitcodeHeader::ReadPrefix(const unsigned char *BufPtr,
-                                   const unsigned char *BufEnd,
-                                   unsigned &NumFields, unsigned &NumBytes) {
-  // Must contain PEXE.
-  if (!isNaClBitcode(BufPtr, BufEnd)) {
-    UnsupportedMessage = "Invalid PNaCl bitcode header";
-    if (isBitcode(BufPtr, BufEnd)) {
-      UnsupportedMessage += " (to run in Chrome, bitcode files must be "
-          "finalized using pnacl-finalize)";
-    }
-    return true;
-  }
-  BufPtr += WordSize;
-
-  // Read #Fields and number of bytes needed for the header.
-  if (BufPtr + WordSize > BufEnd)
-    return UnsupportedError("Bitcode read failure");
-  NumFields = static_cast<unsigned>(BufPtr[0]) |
-      (static_cast<unsigned>(BufPtr[1]) << 8);
-  NumBytes = static_cast<unsigned>(BufPtr[2]) |
-      (static_cast<unsigned>(BufPtr[3]) << 8);
-  BufPtr += WordSize;
-  return false;
-}
-
-bool NaClBitcodeHeader::ReadFields(const unsigned char *BufPtr,
-                                   const unsigned char *BufEnd,
-                                   unsigned NumFields, unsigned NumBytes) {
-  HeaderSize = NumBytes + (2 * WordSize);
-
-  // Read in each field.
-  for (size_t i = 0; i < NumFields; ++i) {
-    NaClBitcodeHeaderField *Field = new NaClBitcodeHeaderField();
-    Fields.push_back(Field);
-    if (!Field->Read(BufPtr, BufEnd - BufPtr))
-      return UnsupportedError("Bitcode read failure");
-    size_t FieldSize = Field->GetTotalSize();
-    BufPtr += FieldSize;
-  }
-  return false;
-}
-
-bool NaClBitcodeHeader::Read(const unsigned char *BufPtr,
-                             const unsigned char *BufEnd) {
-  unsigned NumFields;
-  unsigned NumBytes;
-  if (ReadPrefix(BufPtr, BufEnd, NumFields, NumBytes))
-    return true; // ReadPrefix sets UnsupportedMessage
-  BufPtr += 2 * WordSize;
-
-  if (ReadFields(BufPtr, BufEnd, NumFields, NumBytes))
-    return true; // ReadFields sets UnsupportedMessage
-  BufPtr += NumBytes;
-  InstallFields();
-  return false;
-}
-
-bool NaClBitcodeHeader::Read(MemoryObject *Bytes) {
-  unsigned NumFields;
-  unsigned NumBytes;
-  // First, read the prefix, which is 2 * WordSize, to determine the
-  // NumBytes and NumFields.
-  {
-    unsigned char Buffer[2 * WordSize];
-    if (Bytes->readBytes(Buffer, sizeof(Buffer), 0) != sizeof(Buffer))
-      return UnsupportedError("Bitcode read failure");
-    if (ReadPrefix(Buffer, Buffer + sizeof(Buffer), NumFields, NumBytes))
-      return true; // ReadPrefix sets UnsupportedMessage
-  }
-  // Then read the rest, starting after the 2 * WordSize of the prefix.
-  uint8_t *Header = new uint8_t[NumBytes];
-  bool failed =
-      Bytes->readBytes(Header, NumBytes, 2 * WordSize) != NumBytes ||
-      ReadFields(Header, Header + NumBytes, NumFields, NumBytes);
-  delete[] Header;
-  if (failed)
-    return UnsupportedError("Bitcode read failure");
-  InstallFields();
-  return false;
-}
-
-NaClBitcodeHeaderField *
-NaClBitcodeHeader::GetTaggedField(NaClBitcodeHeaderField::Tag ID) const {
-  for (std::vector<NaClBitcodeHeaderField *>::const_iterator
-           Iter = Fields.begin(),
-           IterEnd = Fields.end();
-       Iter != IterEnd; ++Iter) {
-    if ((*Iter)->GetID() == ID) {
-      return *Iter;
-    }
-  }
-  return 0;
-}
-
-NaClBitcodeHeaderField *NaClBitcodeHeader::GetField(size_t index) const {
-  if (index >= Fields.size())
-    return 0;
-  return Fields[index];
-}
-
-NaClBitcodeHeaderField *GetPNaClVersionPtr(NaClBitcodeHeader *Header) {
-  if (NaClBitcodeHeaderField *Version =
-          Header->GetTaggedField(NaClBitcodeHeaderField::kPNaClVersion)) {
-    if (Version->GetType() == NaClBitcodeHeaderField::kUInt32Type) {
-      return Version;
-    }
-  }
-  return 0;
-}
-
-void NaClBitcodeHeader::InstallFields() {
-  IsSupportedFlag = true;
-  IsReadableFlag = true;
-  AlignBitcodeRecords = false;
-  PNaClVersion = 0;
-  UnsupportedMessage.clear();
-  SmallSet<unsigned, NaClBitcodeHeaderField::kTag_MAX> FieldIDs;
-
-  auto ReportProblem = [&](bool IsReadable) {
-    UnsupportedMessage.append("\n");
-    IsSupportedFlag = false;
-    IsReadableFlag = IsReadableFlag && IsReadable;
-  };
-
-  auto ReportProblemWithContents = [&](NaClBitcodeHeaderField *Field,
-                                       bool IsReadable) {
-    UnsupportedMessage.append(": ");
-    UnsupportedMessage.append(Field->Contents());
-    ReportProblem(IsReadable);
-  };
-
-  for (size_t i = 0, e = NumberFields(); i < e; ++i) {
-    // Start by checking expected properties for any field
-    NaClBitcodeHeaderField *Field = GetField(i);
-    if (!FieldIDs.insert(Field->GetID()).second) {
-      UnsupportedMessage.append("Specified multiple times: ");
-      UnsupportedMessage.append(Field->IDName());
-      ReportProblem(false);
-      continue;
-    }
-    NaClBitcodeHeaderField::FieldType ExpectedTy = ExpectedType[Field->GetID()];
-    if (Field->GetType() != ExpectedTy) {
-      UnsupportedMessage.append("Expects type ");
-      UnsupportedMessage.append(NaClBitcodeHeaderField::TypeName(ExpectedTy));
-      ReportProblemWithContents(Field, false);
-      continue;
-    }
-    if (Field->GetType() == NaClBitcodeHeaderField::kUnknownType) {
-      UnsupportedMessage.append("Unknown value");
-      ReportProblemWithContents(Field, false);
-      continue;
-    }
-
-    // Check specific ID values and install.
-    switch (Field->GetID()) {
-    case NaClBitcodeHeaderField::kInvalid:
-      UnsupportedMessage.append("Unsupported");
-      ReportProblemWithContents(Field, false);
-      continue;
-    case NaClBitcodeHeaderField::kPNaClVersion:
-      PNaClVersion = Field->GetUInt32Value();
-      if (PNaClVersion != 2) {
-        UnsupportedMessage.append("Unsupported");
-        ReportProblemWithContents(Field, false);
-        continue;
-      }
-      break;
-    case NaClBitcodeHeaderField::kAlignBitcodeRecords:
-      AlignBitcodeRecords = true;
-      UnsupportedMessage.append("Unsupported");
-      ReportProblemWithContents(Field, true);
-      continue;
-    }
-  }
-}
diff --git a/lib/Bitcode/NaCl/Reader/NaClBitcodeParser.cpp b/lib/Bitcode/NaCl/Reader/NaClBitcodeParser.cpp
deleted file mode 100644
index 30744ec4982..00000000000
--- a/lib/Bitcode/NaCl/Reader/NaClBitcodeParser.cpp
+++ /dev/null
@@ -1,159 +0,0 @@
-//===- NaClBitcodeParser.cpp ----------------------------------------------===//
-//     Low-level bitcode driver to parse PNaCl bitcode files.
-//
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
-//
-//===----------------------------------------------------------------------===//
-
-#include "llvm/Bitcode/NaCl/NaClBitcodeParser.h"
-
-using namespace llvm;
-
-void NaClBitcodeRecordData::Print(raw_ostream &os) const {
-  os << "[" << Code;
-  for (NaClRecordVector::const_iterator
-           Iter = Values.begin(), IterEnd = Values.end();
-       Iter != IterEnd; ++Iter) {
-    os << ", " << *Iter;
-  }
-  os << "]";
-}
-
-void NaClBitcodeRecord::Print(raw_ostream& os) const {
-  Block.Print(os);
-  os << ", Code " << Data.Code << ", EntryID " << Entry.ID << ", <";
-  for (unsigned i = 0, e = Data.Values.size(); i != e; ++i) {
-    if (i > 0) os << " ";
-    os << Data.Values[i];
-  }
-  os << ">";
-}
-
-NaClBitcodeBlock::NaClBitcodeBlock(unsigned BlockID,
-                                   const NaClBitcodeRecord &Record)
-    : NaClBitcodeData(Record),
-      BlockID(BlockID),
-      EnclosingBlock(&Record.GetBlock()),
-      LocalStartBit(Record.GetStartBit())
-{}
-
-void NaClBitcodeBlock::Print(raw_ostream &os) const {
-  os << "Block " << BlockID;
-}
-
-void NaClBitcodeParserListener::BeginBlockInfoBlock(unsigned NumWords) {
-  Parser->EnterBlock(NumWords);
-}
-
-void NaClBitcodeParserListener::SetBID() {
-  Parser->Record.SetStartBit(StartBit);
-  Parser->Record.Entry.Kind = NaClBitstreamEntry::Record;
-  Parser->Record.Entry.ID = naclbitc::UNABBREV_RECORD;
-  Parser->Record.Data.Code = naclbitc::BLOCKINFO_CODE_SETBID;
-  Parser->Record.Data.Values = Values;
-  GlobalBlockID = Values[0];
-  Parser->SetBID();
-  Values.clear();
-}
-
-void NaClBitcodeParserListener::EndBlockInfoBlock() {
-  Parser->Record.SetStartBit(StartBit);
-  Parser->Record.Entry.Kind = NaClBitstreamEntry::EndBlock;
-  Parser->Record.Entry.ID = naclbitc::END_BLOCK;
-  Parser->Record.Data.Code = naclbitc::END_BLOCK;
-  Parser->Record.Data.Values.clear();
-  GlobalBlockID = naclbitc::BLOCKINFO_BLOCK_ID;
-  Parser->ExitBlock();
-}
-
-void NaClBitcodeParserListener::
-ProcessAbbreviation(NaClBitCodeAbbrev *Abbrev, bool IsLocal) {
-  Parser->Record.SetStartBit(StartBit);
-  Parser->Record.Entry.Kind = NaClBitstreamEntry::Record;
-  Parser->Record.Entry.ID = naclbitc::DEFINE_ABBREV;
-  Parser->Record.Data.Code = naclbitc::BLK_CODE_DEFINE_ABBREV;
-  Parser->Record.Data.Values = Values;
-  Parser->ProcessAbbreviation(IsLocal ? Parser->GetBlockID() : GlobalBlockID,
-                              Abbrev, IsLocal);
-}
-
-NaClBitcodeParser::~NaClBitcodeParser() {
-  if (EnclosingParser) {
-    EnclosingParser->Block.LocalStartBit += Block.GetNumBits();
-  }
-}
-
-bool NaClBitcodeParser::ErrorAt(
-    naclbitc::ErrorLevel Level, uint64_t BitPosition,
-    const std::string &Message) {
-  naclbitc::ErrorAt(*ErrStream, Level, BitPosition) << Message << "\n";
-  if (Level == naclbitc::Fatal)
-    report_fatal_error("Unable to continue");
-  return true;
-}
-
-bool NaClBitcodeParser::Parse() {
-  Record.ReadEntry();
-
-  if (Record.GetEntryKind() != NaClBitstreamEntry::SubBlock)
-    return Error("Expected block, but not found");
-
-  return ParseBlock(Record.GetEntryID());
-}
-
-bool NaClBitcodeParser::ParseBlockInfoInternal() {
-  // BLOCKINFO is a special part of the stream. Let the bitstream
-  // reader process this block.
-  bool Result = Record.GetCursor().ReadBlockInfoBlock(Listener);
-  if (Result) return Error("Malformed BlockInfoBlock");
-  return Result;
-}
-
-bool NaClBitcodeParser::ParseBlockInternal() {
-  // Regular block. Enter subblock.
-  unsigned NumWords;
-  if (Record.GetCursor().EnterSubBlock(GetBlockID(), &NumWords)) {
-    return Error("Malformed block record");
-  }
-
-  EnterBlock(NumWords);
-
-  // Process records.
-  while (1) {
-    if (Record.GetCursor().AtEndOfStream())
-      return Error("Premature end of bitstream");
-
-    // Read entry defining type of entry.
-    Record.ReadEntry();
-
-    switch (Record.GetEntryKind()) {
-    case NaClBitstreamEntry::Error:
-      return Error("malformed bitcode file");
-    case NaClBitstreamEntry::EndBlock: {
-      return false;
-    }
-    case NaClBitstreamEntry::SubBlock: {
-      if (ParseBlock(Record.GetEntryID())) return true;
-      break;
-    }
-    case NaClBitstreamEntry::Record:
-      // The interesting case.
-      if (Record.GetEntryID() == naclbitc::DEFINE_ABBREV) {
-        // Since this abbreviation is local, the listener doesn't
-        // have the start bit set (it is only set when processing
-        // the BlockInfo block). Fix this by setting it here.
-        if (Listener) Listener->StartBit = Record.GetStartBit();
-        Record.GetCursor().ReadAbbrevRecord(true, Listener);
-      } else {
-        // Read in a record.
-        Record.ReadValues();
-        ProcessRecord();
-      }
-      break;
-    }
-  }
-  return false;
-}
diff --git a/lib/Bitcode/NaCl/Reader/NaClBitcodeReader.cpp b/lib/Bitcode/NaCl/Reader/NaClBitcodeReader.cpp
deleted file mode 100644
index 4bfe85ce6d1..00000000000
--- a/lib/Bitcode/NaCl/Reader/NaClBitcodeReader.cpp
+++ /dev/null
@@ -1,2001 +0,0 @@
-//===- NaClBitcodeReader.cpp ----------------------------------------------===//
-//     Internal NaClBitcodeReader implementation
-//
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
-//
-//===----------------------------------------------------------------------===//
-
-#define DEBUG_TYPE "NaClBitcodeReader"
-
-#include "NaClBitcodeReader.h"
-#include "llvm/ADT/SmallString.h"
-#include "llvm/ADT/SmallVector.h"
-#include "llvm/Analysis/NaCl/PNaClABITypeChecker.h"
-#include "llvm/Bitcode/NaCl/NaClBitcodeDecoders.h"
-#include "llvm/Bitcode/NaCl/NaClReaderWriter.h"
-#include "llvm/IR/AutoUpgrade.h"
-#include "llvm/IR/Constants.h"
-#include "llvm/IR/DerivedTypes.h"
-#include "llvm/IR/InlineAsm.h"
-#include "llvm/IR/IntrinsicInst.h"
-#include "llvm/IR/Module.h"
-#include "llvm/IR/OperandTraits.h"
-#include "llvm/IR/Operator.h"
-#include "llvm/Support/DataStream.h"
-#include "llvm/Support/Debug.h"
-#include "llvm/Support/MathExtras.h"
-#include "llvm/Support/MemoryBuffer.h"
-#include "llvm/Support/raw_ostream.h"
-using namespace llvm;
-
-
-cl::opt<bool>
-llvm::PNaClAllowLocalSymbolTables(
-    "allow-local-symbol-tables",
-    cl::desc("Allow (function) local symbol tables in PNaCl bitcode files"),
-    cl::init(false));
-
-static_assert(sizeof(NaClBcIndexSize_t) <= sizeof(size_t),
-              "NaClBcIndexSize_t incorrectly defined");
-
-void NaClBitcodeReader::FreeState() {
-  std::vector<Type*>().swap(TypeList);
-  ValueList.clear();
-
-  std::vector<Function*>().swap(FunctionsWithBodies);
-  DeferredFunctionInfo.clear();
-}
-
-//===----------------------------------------------------------------------===//
-//  Helper functions to implement forward reference resolution, etc.
-//===----------------------------------------------------------------------===//
-
-/// ConvertToString - Convert a string from a record into an std::string, return
-/// true on failure.
-template<typename StrTy>
-static bool ConvertToString(ArrayRef<uint64_t> Record, size_t Idx,
-                            StrTy &Result) {
-  if (Idx > Record.size())
-    return true;
-
-  for (size_t i = Idx, e = Record.size(); i != e; ++i)
-    Result += (char)Record[i];
-  return false;
-}
-
-void NaClBitcodeReaderValueList::AssignValue(Value *V, NaClBcIndexSize_t Idx) {
-  assert(V);
-  if (Idx == size()) {
-    push_back(V);
-    return;
-  }
-
-  if (Idx >= size())
-    resize(Idx+1);
-
-  WeakVH &OldV = ValuePtrs[Idx];
-  if (OldV == 0) {
-    OldV = V;
-    return;
-  }
-
-  // If there was a forward reference to this value, replace it.
-  Value *PrevVal = OldV;
-  OldV->replaceAllUsesWith(V);
-  delete PrevVal;
-}
-
-void NaClBitcodeReaderValueList::OverwriteValue(Value *V,
-                                                NaClBcIndexSize_t Idx) {
-  ValuePtrs[Idx] = V;
-}
-
-Value *NaClBitcodeReaderValueList::getValueFwdRef(NaClBcIndexSize_t Idx) {
-  if (Idx >= size())
-    return 0;
-
-  if (Value *V = ValuePtrs[Idx])
-    return V;
-
-  return 0;
-}
-
-bool NaClBitcodeReaderValueList::createValueFwdRef(NaClBcIndexSize_t Idx,
-                                                   Type *Ty) {
-  if (Idx >= size())
-    resize(Idx + 1);
-
-  // Return an error if this a duplicate definition of Idx.
-  if (ValuePtrs[Idx])
-    return true;
-
-  // No type specified, must be invalid reference.
-  if (Ty == 0)
-    return true;
-
-  // Create a placeholder, which will later be RAUW'd.
-  ValuePtrs[Idx] = new Argument(Ty);
-  return false;
-}
-
-namespace {
-class NaClBitcodeErrorCategoryType : public std::error_category {
-  const char *name() const LLVM_NOEXCEPT override {
-    return "pnacl.bitcode";
-  }
-  std::string message(int IndexError) const override {
-    switch(static_cast<NaClBitcodeReader::ErrorType>(IndexError)) {
-    case NaClBitcodeReader::CouldNotFindFunctionInStream:
-      return "Unable to find function in bitcode stream.";
-    case NaClBitcodeReader::InsufficientFunctionProtos:
-      return "Insufficient function protos";
-    case NaClBitcodeReader::InvalidBitstream:
-      return "Error in bitstream format";
-    case NaClBitcodeReader::InvalidBlock:
-      return "Invalid block found in bitcode file";
-    case NaClBitcodeReader::InvalidConstantReference:
-      return "Bad constant reference";
-    case NaClBitcodeReader::InvalidDataAfterModule:
-      return "Invalid data after module";
-    case NaClBitcodeReader::InvalidInstructionWithNoBB:
-      return "No basic block for instruction";
-    case NaClBitcodeReader::InvalidMultipleBlocks:
-      return "Multiple blocks for a kind of block that should have only one";
-    case NaClBitcodeReader::InvalidRecord:
-      return "Record doesn't have expected size or structure";
-    case NaClBitcodeReader::InvalidSkippedBlock:
-      return "Unable to skip unknown block in bitcode file";
-    case NaClBitcodeReader::InvalidType:
-      return "Invalid type in record";
-    case NaClBitcodeReader::InvalidTypeForValue:
-      return "Type of value in record incorrect";
-    case NaClBitcodeReader::InvalidValue:
-      return "Invalid value in record";
-    case NaClBitcodeReader::MalformedBlock:
-      return "Malformed block. Unable to advance over block";
-    }
-    llvm_unreachable("Unknown error type!");
-  }
-};
-} // end of anonymous namespace.
-
-const std::error_category &NaClBitcodeReader::BitcodeErrorCategory() {
-  static NaClBitcodeErrorCategoryType ErrCat;
-  return ErrCat;
-}
-
-Type *NaClBitcodeReader::getTypeByID(NaClBcIndexSize_t ID) {
-  // The type table size is always specified correctly.
-  if (ID >= TypeList.size())
-    return 0;
-
-  if (Type *Ty = TypeList[ID])
-    return Ty;
-
-  // If we have a forward reference, the only possible case is when it is to a
-  // named struct.  Just create a placeholder for now.
-  return TypeList[ID] = StructType::create(Context);
-}
-
-
-//===----------------------------------------------------------------------===//
-//  Functions for parsing blocks from the bitcode file
-//===----------------------------------------------------------------------===//
-
-
-namespace {
-
-static const unsigned MaxAlignmentExponent = 29;
-static_assert(
-    (1u << MaxAlignmentExponent) == Value::MaximumAlignment,
-    "Inconsistency between Value.MaxAlignment and PNaCl alignment limit");
-}
-
-std::error_code NaClBitcodeReader::Error(ErrorType E,
-                                         const std::string &Message) const {
-  if (Verbose) {
-    naclbitc::ErrorAt(*Verbose, naclbitc::Error, Stream.GetCurrentBitNo())
-        << Message << "\n";
-  }
-  return Error(E);
-}
-
-std::error_code NaClBitcodeReader::getAlignmentValue(
-    uint64_t Exponent, unsigned &Alignment) {
-  if (Exponent > MaxAlignmentExponent + 1) {
-    std::string Buffer;
-    raw_string_ostream StrBuf(Buffer);
-    StrBuf << "Alignment can't be greater than 2**" << MaxAlignmentExponent
-           << ". Found: 2**" << (Exponent - 1);
-    return Error(InvalidValue, StrBuf.str());
-  }
-  Alignment = (1 << static_cast<unsigned>(Exponent)) >> 1;
-  return std::error_code();
-}
-
-std::error_code NaClBitcodeReader::ParseTypeTable() {
-  DEBUG(dbgs() << "-> ParseTypeTable\n");
-  if (Stream.EnterSubBlock(naclbitc::TYPE_BLOCK_ID_NEW))
-    return Error(InvalidRecord, "Malformed block record");
-
-  std::error_code result = ParseTypeTableBody();
-  if (!result)
-    DEBUG(dbgs() << "<- ParseTypeTable\n");
-  return result;
-}
-
-std::error_code NaClBitcodeReader::ParseTypeTableBody() {
-  if (!TypeList.empty())
-    return Error(InvalidMultipleBlocks, "Multiple TYPE_BLOCKs found!");
-
-  SmallVector<uint64_t, 64> Record;
-  NaClBcIndexSize_t NumRecords = 0;
-  NaClBcIndexSize_t ExpectedNumRecords = 0;
-
-  // Read all the records for this type table.
-  while (1) {
-    NaClBitstreamEntry Entry = Stream.advance(0, nullptr);
-
-    switch (Entry.Kind) {
-    case NaClBitstreamEntry::SubBlock:
-      return Error(InvalidBlock, "Invalid block found in the types block");
-    case NaClBitstreamEntry::Error:
-      return Error(MalformedBlock, "Malformed types block");
-    case NaClBitstreamEntry::EndBlock:
-      if (NumRecords != ExpectedNumRecords)
-        return Error(MalformedBlock,
-                     "Invalid forward reference in the types block");
-      return std::error_code();
-    case NaClBitstreamEntry::Record:
-      // The interesting case.
-      break;
-    }
-
-    // Read a record.
-    Record.clear();
-    Type *ResultTy = 0;
-    unsigned TypeCode = Stream.readRecord(Entry.ID, Record);
-    switch (TypeCode) {
-    default: {
-      std::string Message;
-      raw_string_ostream StrM(Message);
-      StrM << "Unknown type code in type table: " << TypeCode;
-      StrM.flush();
-      return Error(InvalidValue, Message);
-    }
-
-    case naclbitc::TYPE_CODE_NUMENTRY: { // TYPE_CODE_NUMENTRY: [numentries]
-      // TYPE_CODE_NUMENTRY contains a count of the number of types in the
-      // type list.  This allows us to reserve space.
-      if (Record.size() != 1)
-        return Error(InvalidRecord, "Invalid TYPE_CODE_NUMENTRY record");
-      uint64_t Size = Record[0];
-      if (Size > NaClBcIndexSize_t_Max)
-        return Error(InvalidValue, "Size too big in TYPE_CODE_NUMENTRY record");
-      // The code double checks that Expected size and the actual size
-      // at the end of the block. To reduce allocations we preallocate
-      // the space.
-      //
-      // However, if the number is large, we suspect that the number
-      // is (possibly) incorrect. In that case, we preallocate a
-      // smaller space.
-      TypeList.resize(std::min(Size, (uint64_t) 1000000));
-      ExpectedNumRecords = Size;
-      // No type was defined, skip the checks that follow the switch.
-      continue;
-    }
-    case naclbitc::TYPE_CODE_VOID: // VOID
-      if (Record.size() != 0)
-        return Error(InvalidRecord, "Invalid TYPE_CODE_VOID record");
-      ResultTy = Type::getVoidTy(Context);
-      break;
-
-    case naclbitc::TYPE_CODE_FLOAT: // FLOAT
-      if (Record.size() != 0)
-        return Error(InvalidRecord, "Invalid TYPE_CODE_FLOAT record");
-      ResultTy = Type::getFloatTy(Context);
-      break;
-
-    case naclbitc::TYPE_CODE_DOUBLE: // DOUBLE
-      if (Record.size() != 0)
-        return Error(InvalidRecord, "Invalid TYPE_CODE_DOUBLE record");
-      ResultTy = Type::getDoubleTy(Context);
-      break;
-
-    case naclbitc::TYPE_CODE_INTEGER: // INTEGER: [width]
-      if (Record.size() != 1)
-        return Error(InvalidRecord, "Invalid TYPE_CODE_INTEGER record");
-      // TODO(kschimpf): Should we check if Record[0] is in range?
-      ResultTy = IntegerType::get(Context, Record[0]);
-      break;
-
-    case naclbitc::TYPE_CODE_FUNCTION: {
-      // FUNCTION: [vararg, retty, paramty x N]
-      if (Record.size() < 2)
-        return Error(InvalidRecord, "Invalid TYPE_CODE_FUNCTION record");
-      SmallVector<Type *, 8> ArgTys;
-      for (size_t i = 2, e = Record.size(); i != e; ++i) {
-        if (Type *T = getTypeByID(Record[i]))
-          ArgTys.push_back(T);
-        else
-          break;
-      }
-
-      ResultTy = getTypeByID(Record[1]);
-      if (ResultTy == 0 || ArgTys.size() < Record.size() - 2)
-        return Error(InvalidType, "invalid type in function type");
-
-      ResultTy = FunctionType::get(ResultTy, ArgTys, Record[0]);
-      break;
-    }
-    case naclbitc::TYPE_CODE_VECTOR: { // VECTOR: [numelts, eltty]
-      if (Record.size() != 2)
-        return Error(InvalidRecord, "Invalid VECTOR type record");
-      if ((ResultTy = getTypeByID(Record[1])))
-        ResultTy = VectorType::get(ResultTy, Record[0]);
-      else
-        return Error(InvalidType, "invalid type in vector type");
-      break;
-    }
-    }
-
-    if (NumRecords >= ExpectedNumRecords)
-      return Error(MalformedBlock, "invalid TYPE table");
-    assert(ResultTy && "Didn't read a type?");
-    assert(TypeList[NumRecords] == 0 && "Already read type?");
-    if (NumRecords == NaClBcIndexSize_t_Max)
-      return Error(InvalidRecord, "Exceeded type index limit");
-    TypeList[NumRecords++] = ResultTy;
-  }
-  return std::error_code();
-}
-
-namespace {
-
-// Class to process globals in two passes. In the first pass, build
-// the corresponding global variables with no initializers. In the
-// second pass, add initializers. The purpose of putting off
-// initializers is to make sure that we don't need to generate
-// placeholders for relocation records, and the corresponding cost
-// of duplicating initializers when these placeholders are replaced.
-class ParseGlobalsHandler {
-  ParseGlobalsHandler(const ParseGlobalsHandler &H) = delete;
-  void operator=(const ParseGlobalsHandler &H) = delete;
-
-  NaClBitcodeReader &Reader;
-  NaClBitcodeReaderValueList &ValueList;
-  NaClBitstreamCursor &Stream;
-  LLVMContext &Context;
-  Module *TheModule;
-
-  // Holds read data record.
-  SmallVector<uint64_t, 64> Record;
-  // True when processing a global variable. Stays true until all records
-  // are processed, and the global variable is created.
-  bool ProcessingGlobal;
-  // The number of initializers needed for the global variable.
-  size_t VarInitializersNeeded;
-  NaClBcIndexSize_t FirstValueNo;
-  // The index of the next global variable.
-  NaClBcIndexSize_t NextValueNo;
-  // The number of expected global variable definitions.
-  NaClBcIndexSize_t NumGlobals;
-  // The bit to go back to to generate initializers.
-  uint64_t StartBit;
-
-  void InitPass() {
-    Stream.JumpToBit(StartBit);
-    ProcessingGlobal = false;
-    VarInitializersNeeded = 0;
-    NextValueNo = FirstValueNo;
-  }
-
-public:
-  ParseGlobalsHandler(NaClBitcodeReader &Reader,
-                      NaClBitcodeReaderValueList &ValueList,
-                      NaClBitstreamCursor &Stream,
-                      LLVMContext &Context,
-                      Module *TheModule)
-      : Reader(Reader),
-        ValueList(ValueList),
-        Stream(Stream),
-        Context(Context),
-        TheModule(TheModule),
-        FirstValueNo(ValueList.size()),
-        NumGlobals(0),
-        StartBit(Stream.GetCurrentBitNo()) {}
-
-  std::error_code GenerateGlobalVarsPass() {
-    InitPass();
-
-    // The type for the initializer of the global variable.
-    SmallVector<Type*, 10> VarType;
-    // The alignment value defined for the global variable.
-    unsigned VarAlignment = 0;
-    // True if the variable is read-only.
-    bool VarIsConstant = false;
-
-    // Read all records to build global variables without initializers.
-    while (1) {
-      NaClBitstreamEntry Entry =
-        Stream.advance(NaClBitstreamCursor::AF_DontPopBlockAtEnd, nullptr);
-      switch (Entry.Kind) {
-      case NaClBitstreamEntry::SubBlock:
-	return Reader.Error(NaClBitcodeReader::InvalidBlock,
-			    "Invalid block in the global vars block");
-      case NaClBitstreamEntry::Error:
-        return Reader.Error(NaClBitcodeReader::MalformedBlock,
-                            "Error in the global vars block");
-      case NaClBitstreamEntry::EndBlock:
-        if (ProcessingGlobal || NumGlobals != (NextValueNo - FirstValueNo))
-          return Reader.Error(NaClBitcodeReader::MalformedBlock,
-                              "Error in the global vars block");
-        return std::error_code();
-      case NaClBitstreamEntry::Record:
-        // The interesting case.
-        break;
-      }
-
-      // Read a record.
-      Record.clear();
-      unsigned Bitcode = Stream.readRecord(Entry.ID, Record);
-      switch (Bitcode) {
-      default:
-        return Reader.Error(NaClBitcodeReader::InvalidValue,
-                            "Unknown global variable entry");
-      case naclbitc::GLOBALVAR_VAR:
-        // Start the definition of a global variable.
-        if (ProcessingGlobal || Record.size() != 2)
-          return Reader.Error(NaClBitcodeReader::InvalidRecord,
-                              "Bad GLOBALVAR_VAR record");
-        ProcessingGlobal = true;
-        if (std::error_code EC =
-            Reader.getAlignmentValue(Record[0], VarAlignment))
-          return EC;
-        VarIsConstant = Record[1] != 0;
-        // Assume (by default) there is a single initializer.
-        VarInitializersNeeded = 1;
-        break;
-      case naclbitc::GLOBALVAR_COMPOUND: {
-        // Global variable has multiple initializers. Changes the
-        // default number of initializers to the given value in
-        // Record[0].
-        if (!ProcessingGlobal || !VarType.empty() ||
-            VarInitializersNeeded != 1 || Record.size() != 1)
-          return Reader.Error(NaClBitcodeReader::InvalidRecord,
-                              "Bad GLOBALVAR_COMPOUND record");
-        uint64_t Size = Record[0];
-        // Check that ILP32 assumption met.
-        if (Size > MaxNaClGlobalVarInits)
-          return Reader.Error(NaClBitcodeReader::InvalidValue,
-                              "Size too big in GLOBALVAR_COMPOUND record");
-        VarInitializersNeeded = static_cast<size_t>(Size);
-        break;
-      }
-      case naclbitc::GLOBALVAR_ZEROFILL: {
-        // Define a type that defines a sequence of zero-filled bytes.
-        if (!ProcessingGlobal || Record.size() != 1)
-          return Reader.Error(NaClBitcodeReader::InvalidRecord,
-                              "Bad GLOBALVAR_ZEROFILL record");
-        VarType.push_back(ArrayType::get(
-            Type::getInt8Ty(Context), Record[0]));
-        break;
-      }
-      case naclbitc::GLOBALVAR_DATA: {
-        // Defines a type defined by a sequence of byte values.
-        if (!ProcessingGlobal || Record.size() < 1)
-          return Reader.Error(NaClBitcodeReader::InvalidRecord,
-                              "Bad GLOBALVAR_DATA record");
-        VarType.push_back(ArrayType::get(
-            Type::getInt8Ty(Context), Record.size()));
-        break;
-      }
-      case naclbitc::GLOBALVAR_RELOC: {
-        // Define a relocation initializer type.
-        if (!ProcessingGlobal || Record.size() < 1 || Record.size() > 2)
-          return Reader.Error(NaClBitcodeReader::InvalidRecord,
-                              "Bad GLOBALVAR_RELOC record");
-        VarType.push_back(IntegerType::get(Context, 32));
-        break;
-      }
-      case naclbitc::GLOBALVAR_COUNT:
-        if (Record.size() != 1 || NumGlobals != 0)
-          return Reader.Error(NaClBitcodeReader::InvalidRecord,
-                              "Invalid global count record");
-        if (Record[0] > NaClBcIndexSize_t_Max)
-          return Reader.Error(NaClBitcodeReader::InvalidValue,
-                              "Size too big in global count record");
-        NumGlobals = static_cast<NaClBcIndexSize_t>(Record[0]);
-        break;
-      }
-
-      // If more initializers needed for global variable, continue processing.
-      if (!ProcessingGlobal || VarType.size() < VarInitializersNeeded)
-        continue;
-
-      Type *Ty = 0;
-      switch (VarType.size()) {
-      case 0:
-        return Reader.Error(
-            NaClBitcodeReader::InvalidRecord,
-            "No initializer for global variable in global vars block");
-      case 1:
-        Ty = VarType[0];
-        break;
-      default:
-        Ty = StructType::get(Context, VarType, true);
-        break;
-      }
-      GlobalVariable *GV = new GlobalVariable(
-          *TheModule, Ty, VarIsConstant,
-          GlobalValue::InternalLinkage, NULL, "");
-      GV->setAlignment(VarAlignment);
-      ValueList.AssignValue(GV, NextValueNo);
-      ++NextValueNo;
-      ProcessingGlobal = false;
-      VarAlignment = 0;
-      VarIsConstant = false;
-      VarInitializersNeeded = 0;
-      VarType.clear();
-    }
-    return std::error_code();
-  }
-
-  std::error_code GenerateGlobalVarInitsPass() {
-    InitPass();
-    // The initializer for the global variable.
-    SmallVector<Constant *, 10> VarInit;
-
-    while (1) {
-      NaClBitstreamEntry Entry =
-        Stream.advance(NaClBitstreamCursor::AF_DontAutoprocessAbbrevs, nullptr);
-      switch (Entry.Kind) {
-      case NaClBitstreamEntry::SubBlock:
-	return Reader.Error(NaClBitcodeReader::InvalidBlock,
-			    "Invalid block in the global vars block");
-      case NaClBitstreamEntry::Error:
-        return Reader.Error(NaClBitcodeReader::MalformedBlock,
-                            "Error in the global vars block");
-      case NaClBitstreamEntry::EndBlock:
-        if (ProcessingGlobal || NumGlobals != (NextValueNo - FirstValueNo))
-          return Reader.Error(NaClBitcodeReader::MalformedBlock,
-                              "Error in the global vars block");
-        return std::error_code();
-      case NaClBitstreamEntry::Record:
-        if (Entry.ID == naclbitc::DEFINE_ABBREV) {
-          Stream.SkipAbbrevRecord();
-          continue;
-        }
-        // The interesting case.
-        break;
-      }
-
-      // Read a record.
-      Record.clear();
-      unsigned Bitcode = Stream.readRecord(Entry.ID, Record);
-      switch (Bitcode) {
-      default:
-        return Reader.Error(NaClBitcodeReader::InvalidValue,
-                            "Unknown global variable entry 2");
-      case naclbitc::GLOBALVAR_VAR:
-        // Start the definition of a global variable.
-        ProcessingGlobal = true;
-        // Assume (by default) there is a single initializer.
-        VarInitializersNeeded = 1;
-        break;
-      case naclbitc::GLOBALVAR_COMPOUND:
-        // Global variable has multiple initializers. Changes the
-        // default number of initializers to the given value in
-        // Record[0].
-        if (!ProcessingGlobal || !VarInit.empty() ||
-            VarInitializersNeeded != 1 || Record.size() != 1)
-          return Reader.Error(NaClBitcodeReader::InvalidRecord,
-                              "Bad GLOBALVAR_COMPOUND record");
-        // Note: We assume VarInitializersNeeded size was checked
-        // in GenerateGlobalVarsPass.
-        VarInitializersNeeded = static_cast<size_t>(Record[0]);
-        break;
-      case naclbitc::GLOBALVAR_ZEROFILL: {
-        // Define an initializer that defines a sequence of zero-filled bytes.
-        if (!ProcessingGlobal || Record.size() != 1)
-          return Reader.Error(NaClBitcodeReader::InvalidRecord,
-                              "Bad GLOBALVAR_ZEROFILL record");
-        Type *Ty = ArrayType::get(Type::getInt8Ty(Context),
-                                  Record[0]);
-        Constant *Zero = ConstantAggregateZero::get(Ty);
-        VarInit.push_back(Zero);
-        break;
-      }
-      case naclbitc::GLOBALVAR_DATA: {
-        // Defines an initializer defined by a sequence of byte values.
-        if (!ProcessingGlobal || Record.size() < 1)
-          return Reader.Error(NaClBitcodeReader::InvalidRecord,
-                              "Bad GLOBALVAR_DATA record");
-        size_t Size = Record.size();
-        uint8_t *Buf = new uint8_t[Size];
-        assert(Buf);
-        for (size_t i = 0; i < Size; ++i)
-          Buf[i] = Record[i];
-        Constant *Init = ConstantDataArray::get(
-            Context, ArrayRef<uint8_t>(Buf, Buf + Size));
-        VarInit.push_back(Init);
-        delete[] Buf;
-        break;
-      }
-      case naclbitc::GLOBALVAR_RELOC: {
-        // Define a relocation initializer.
-        if (!ProcessingGlobal || Record.size() < 1 || Record.size() > 2)
-          return Reader.Error(NaClBitcodeReader::InvalidRecord,
-                              "Bad GLOBALVAR_RELOC record");
-        Constant *BaseVal = cast<Constant>(Reader.getFnValueByID(Record[0]));
-        Type *IntPtrType = IntegerType::get(Context, 32);
-        Constant *Val = ConstantExpr::getPtrToInt(BaseVal, IntPtrType);
-        if (Record.size() == 2) {
-          uint64_t Addend = Record[1];
-          // Note: PNaCl is ILP32, so Addend must be uint32_t.
-          if (Addend > std::numeric_limits<uint32_t>::max())
-            return Reader.Error(NaClBitcodeReader::InvalidValue,
-                                "Addend of GLOBALVAR_RELOC record too big");
-          Val = ConstantExpr::getAdd(Val, ConstantInt::get(IntPtrType,
-                                                           Addend));
-        }
-        VarInit.push_back(Val);
-        break;
-      }
-      case naclbitc::GLOBALVAR_COUNT:
-        if (Record.size() != 1)
-          return Reader.Error(NaClBitcodeReader::InvalidRecord,
-                              "Invalid global count record");
-        // Note: NumGlobals should have been set in GenerateGlobalVarsPass.
-        // Fail if methods are called in wrong order.
-        assert(NumGlobals == Record[0]);
-        break;
-      }
-
-      // If more initializers needed for global variable, continue processing.
-      if (!ProcessingGlobal || VarInit.size() < VarInitializersNeeded)
-        continue;
-
-      Constant *Init = 0;
-      switch (VarInit.size()) {
-      case 0:
-        return Reader.Error(NaClBitcodeReader::InvalidRecord,
-            "No initializer for global variable in global vars block");
-      case 1:
-        Init = VarInit[0];
-        break;
-      default:
-        Init = ConstantStruct::getAnon(Context, VarInit, true);
-        break;
-      }
-      cast<GlobalVariable>(ValueList[NextValueNo])->setInitializer(Init);
-      if (NextValueNo == NaClBcIndexSize_t_Max)
-        return Reader.Error(NaClBitcodeReader::InvalidRecord,
-                            "Exceeded value index limit");
-      ++NextValueNo;
-      ProcessingGlobal = false;
-      VarInitializersNeeded = 0;
-      VarInit.clear();
-    }
-    return std::error_code();
-  }
-};
-
-} // End anonymous namespace.
-
-std::error_code NaClBitcodeReader::ParseGlobalVars() {
-  if (Stream.EnterSubBlock(naclbitc::GLOBALVAR_BLOCK_ID))
-    return Error(InvalidRecord, "Malformed block record");
-
-  ParseGlobalsHandler PassHandler(*this, ValueList, Stream, Context, TheModule);
-  if (std::error_code EC = PassHandler.GenerateGlobalVarsPass())
-    return EC;
-  return PassHandler.GenerateGlobalVarInitsPass();
-}
-
-std::error_code NaClBitcodeReader::ParseValueSymbolTable() {
-  DEBUG(dbgs() << "-> ParseValueSymbolTable\n");
-  if (Stream.EnterSubBlock(naclbitc::VALUE_SYMTAB_BLOCK_ID))
-    return Error(InvalidRecord, "Malformed block record");
-
-  SmallVector<uint64_t, 64> Record;
-
-  // Read all the records for this value table.
-  SmallString<128> ValueName;
-  while (1) {
-    NaClBitstreamEntry Entry = Stream.advance(0, nullptr);
-
-    switch (Entry.Kind) {
-    case NaClBitstreamEntry::SubBlock:
-      return Error(InvalidBlock,
-                   "Invalid block in the value symbol table block");
-    case NaClBitstreamEntry::Error:
-      return Error(MalformedBlock, "malformed value symbol table block");
-    case NaClBitstreamEntry::EndBlock:
-      DEBUG(dbgs() << "<- ParseValueSymbolTable\n");
-      return std::error_code();
-    case NaClBitstreamEntry::Record:
-      // The interesting case.
-      break;
-    }
-
-    // Read a record.
-    Record.clear();
-    switch (Stream.readRecord(Entry.ID, Record)) {
-    default:  // Default behavior: unknown type.
-      break;
-    case naclbitc::VST_CODE_ENTRY: {  // VST_ENTRY: [valueid, namechar x N]
-      if (ConvertToString(Record, 1, ValueName))
-        return Error(InvalidRecord, "Invalid VST_ENTRY record");
-      Value *V = getFnValueByID(Record[0]);
-      if (V == nullptr)
-        return Error(InvalidValue, "Invalid Value ID in VST_ENTRY record");
-
-      V->setName(StringRef(ValueName.data(), ValueName.size()));
-      ValueName.clear();
-      break;
-    }
-    case naclbitc::VST_CODE_BBENTRY: {
-      if (ConvertToString(Record, 1, ValueName))
-        return Error(InvalidRecord, "Invalid VST_BBENTRY record");
-      BasicBlock *BB = getBasicBlock(Record[0]);
-      if (BB == 0)
-        return Error(InvalidValue, "Invalid BB ID in VST_BBENTRY record");
-
-      BB->setName(StringRef(ValueName.data(), ValueName.size()));
-      ValueName.clear();
-      break;
-    }
-    }
-  }
-}
-
-std::error_code NaClBitcodeReader::ParseConstants() {
-  DEBUG(dbgs() << "-> ParseConstants\n");
-  if (Stream.EnterSubBlock(naclbitc::CONSTANTS_BLOCK_ID))
-    return Error(InvalidRecord, "Malformed block record");
-
-  SmallVector<uint64_t, 64> Record;
-
-  // Read all the records for this value table.
-  Type *CurTy = Type::getInt32Ty(Context);
-  NaClBcIndexSize_t NextCstNo = ValueList.size();
-  while (1) {
-    NaClBitstreamEntry Entry = Stream.advance(0, nullptr);
-
-    switch (Entry.Kind) {
-    case NaClBitstreamEntry::SubBlock:
-      return Error(InvalidBlock, "Invalid block in function constants block");
-    case NaClBitstreamEntry::Error:
-      return Error(MalformedBlock, "malformed function constants block");
-    case NaClBitstreamEntry::EndBlock:
-      if (NextCstNo != ValueList.size())
-        return Error(InvalidConstantReference,
-                     "Invalid constant reference!");
-      DEBUG(dbgs() << "<- ParseConstants\n");
-      return std::error_code();
-    case NaClBitstreamEntry::Record:
-      // The interesting case.
-      break;
-    }
-
-    // Read a record.
-    Record.clear();
-    Value *V = 0;
-    unsigned BitCode = Stream.readRecord(Entry.ID, Record);
-    switch (BitCode) {
-    default: {
-      std::string Message;
-      raw_string_ostream StrM(Message);
-      StrM << "Invalid Constant code: " << BitCode;
-      StrM.flush();
-      return Error(InvalidValue, Message);
-    }
-    case naclbitc::CST_CODE_UNDEF:     // UNDEF
-      V = UndefValue::get(CurTy);
-      break;
-    case naclbitc::CST_CODE_SETTYPE:   // SETTYPE: [typeid]
-      if (Record.empty())
-        return Error(NaClBitcodeReader::InvalidRecord,
-                     "Malformed CST_SETTYPE record");
-      if (Record[0] >= TypeList.size())
-        return Error(NaClBitcodeReader::InvalidType,
-                     "Invalid Type ID in CST_SETTYPE record");
-      CurTy = TypeList[Record[0]];
-      continue;  // Skip the ValueList manipulation.
-    case naclbitc::CST_CODE_INTEGER:   // INTEGER: [intval]
-      if (!CurTy->isIntegerTy() || Record.empty())
-        return Error(InvalidRecord, "Invalid CST_INTEGER record");
-      V = ConstantInt::get(CurTy, NaClDecodeSignRotatedValue(Record[0]));
-      break;
-    case naclbitc::CST_CODE_FLOAT: {    // FLOAT: [fpval]
-      if (Record.empty())
-        return Error(NaClBitcodeReader::InvalidRecord, "Invalid FLOAT record");
-      if (CurTy->isFloatTy())
-        V = ConstantFP::get(Context, APFloat(APFloat::IEEEsingle,
-                                             APInt(32, (uint32_t)Record[0])));
-      else if (CurTy->isDoubleTy())
-        V = ConstantFP::get(Context, APFloat(APFloat::IEEEdouble,
-                                             APInt(64, Record[0])));
-      else
-        return Error(NaClBitcodeReader::InvalidRecord,
-                     "Unknown type for FLOAT record");
-      break;
-    }
-    }
-
-    ValueList.AssignValue(V, NextCstNo);
-    ++NextCstNo;
-  }
-  return std::error_code();
-}
-
-/// RememberAndSkipFunctionBody - When we see the block for a function body,
-/// remember where it is and then skip it.  This lets us lazily deserialize the
-/// functions.
-std::error_code NaClBitcodeReader::RememberAndSkipFunctionBody() {
-  DEBUG(dbgs() << "-> RememberAndSkipFunctionBody\n");
-  // Get the function we are talking about.
-  if (FunctionsWithBodies.empty())
-    return Error(InsufficientFunctionProtos,
-                 "Insufficient function protos");
-
-  Function *Fn = FunctionsWithBodies.back();
-  FunctionsWithBodies.pop_back();
-
-  // Save the current stream state.
-  uint64_t CurBit = Stream.GetCurrentBitNo();
-  DeferredFunctionInfo[Fn] = CurBit;
-
-  // Skip over the function block for now.
-  if (Stream.SkipBlock())
-    return Error(InvalidSkippedBlock, "Unable to skip function block.");
-  DEBUG(dbgs() << "<- RememberAndSkipFunctionBody\n");
-  return std::error_code();
-}
-
-std::error_code NaClBitcodeReader::GlobalCleanup() {
-  // Look for intrinsic functions which need to be upgraded at some point
-  for (Module::iterator FI = TheModule->begin(), FE = TheModule->end();
-       FI != FE; ++FI) {
-    Function *NewFn;
-    if (UpgradeIntrinsicFunction(FI, NewFn))
-      UpgradedIntrinsics.push_back(std::make_pair(FI, NewFn));
-  }
-
-  // Look for global variables which need to be renamed.
-  for (Module::global_iterator
-         GI = TheModule->global_begin(), GE = TheModule->global_end();
-       GI != GE; ++GI)
-    UpgradeGlobalVariable(GI);
-  return std::error_code();
-}
-
-FunctionType *NaClBitcodeReader::AddPointerTypesToIntrinsicType(
-    StringRef Name, FunctionType *FTy) {
-  FunctionType *IntrinsicTy = AllowedIntrinsics.getIntrinsicType(Name);
-  if (IntrinsicTy == 0) return FTy;
-
-  Type *IReturnTy = IntrinsicTy->getReturnType();
-  Type *FReturnTy = FTy->getReturnType();
-
-  if (!PNaClABITypeChecker::IsPointerEquivType(IReturnTy, FReturnTy)) {
-    std::string Buffer;
-    raw_string_ostream StrBuf(Buffer);
-    StrBuf << "Intrinsic return type mismatch for " << Name << ": "
-           << *IReturnTy << " and " << *FReturnTy;
-    report_fatal_error(StrBuf.str());
-  }
-  if (FTy->getNumParams() != IntrinsicTy->getNumParams()) {
-    std::string Buffer;
-    raw_string_ostream StrBuf(Buffer);
-    StrBuf << "Intrinsic type mistmatch for " << Name << ": "
-           << *FTy << " and " << *IntrinsicTy;
-    report_fatal_error(StrBuf.str());
-  }
-  for (size_t i = 0; i < FTy->getNumParams(); ++i) {
-    Type *IargTy = IntrinsicTy->getParamType(i);
-    Type *FargTy = FTy->getParamType(i);
-    if (!PNaClABITypeChecker::IsPointerEquivType(IargTy, FargTy)) {
-      std::string Buffer;
-      raw_string_ostream StrBuf(Buffer);
-      StrBuf << "Intrinsic type mismatch for argument " << i << " in "
-             << Name << ": " << *IargTy << " and " << *FargTy;
-      report_fatal_error(StrBuf.str());
-    }
-  }
-  return IntrinsicTy;
-}
-
-void NaClBitcodeReader::AddPointerTypesToIntrinsicParams() {
-  for (size_t Index = 0, E = ValueList.size(); Index < E; ++Index) {
-    if (Function *Func = dyn_cast<Function>(ValueList[Index])) {
-      if (Func->isIntrinsic()) {
-        FunctionType *FTy = Func->getFunctionType();
-        FunctionType *ITy = AddPointerTypesToIntrinsicType(
-            Func->getName(), FTy);
-        if (ITy == FTy) continue;
-        Function *NewIntrinsic = Function::Create(
-            ITy, GlobalValue::ExternalLinkage, "", TheModule);
-        NewIntrinsic->takeName(Func);
-        ValueList.OverwriteValue(NewIntrinsic, Index);
-        Func->eraseFromParent();
-      }
-    }
-  }
-}
-
-std::error_code NaClBitcodeReader::ParseModule(bool Resume) {
-  DEBUG(dbgs() << "-> ParseModule\n");
-  if (Resume)
-    Stream.JumpToBit(NextUnreadBit);
-  else if (Stream.EnterSubBlock(naclbitc::MODULE_BLOCK_ID))
-    return Error(InvalidRecord, "Malformed block record");
-
-  SmallVector<uint64_t, 64> Record;
-
-  // Read all the records for this module.
-  while (1) {
-    NaClBitstreamEntry Entry = Stream.advance(0, nullptr);
-
-    switch (Entry.Kind) {
-    case NaClBitstreamEntry::Error:
-      return Error(MalformedBlock, "malformed module block");
-    case NaClBitstreamEntry::EndBlock:
-      DEBUG(dbgs() << "<- ParseModule\n");
-      if (std::error_code EC = GlobalCleanup())
-        return EC;
-      if (!Stream.AtEndOfStream())
-        return Error(InvalidDataAfterModule, "Invalid data after module");
-      return std::error_code();
-    case NaClBitstreamEntry::SubBlock:
-      switch (Entry.ID) {
-      default: {
-        std::string Message;
-        raw_string_ostream StrM(Message);
-        StrM << "Unknown block ID: " << Entry.ID;
-        return Error(InvalidRecord, StrM.str());
-      }
-      case naclbitc::BLOCKINFO_BLOCK_ID:
-        if (Stream.ReadBlockInfoBlock(0))
-          return Error(MalformedBlock, "Malformed BlockInfoBlock");
-        break;
-      case naclbitc::TYPE_BLOCK_ID_NEW:
-        if (std::error_code EC = ParseTypeTable())
-          return EC;
-        break;
-      case naclbitc::GLOBALVAR_BLOCK_ID:
-        if (std::error_code EC = ParseGlobalVars())
-          return EC;
-        break;
-      case naclbitc::VALUE_SYMTAB_BLOCK_ID:
-        if (std::error_code EC = ParseValueSymbolTable())
-          return EC;
-        SeenValueSymbolTable = true;
-        // Now that we know the names of the intrinsics, we can add
-        // pointer types to the intrinsic declarations' types.
-        AddPointerTypesToIntrinsicParams();
-        break;
-      case naclbitc::FUNCTION_BLOCK_ID:
-        // If this is the first function body we've seen, reverse the
-        // FunctionsWithBodies list.
-        if (!SeenFirstFunctionBody) {
-          std::reverse(FunctionsWithBodies.begin(), FunctionsWithBodies.end());
-          if (std::error_code EC = GlobalCleanup())
-            return EC;
-          SeenFirstFunctionBody = true;
-        }
-
-        if (std::error_code EC = RememberAndSkipFunctionBody())
-          return EC;
-
-        // For streaming bitcode, suspend parsing when we reach the function
-        // bodies. Subsequent materialization calls will resume it when
-        // necessary. For streaming, the function bodies must be at the end of
-        // the bitcode. If the bitcode file is old, the symbol table will be
-        // at the end instead and will not have been seen yet. In this case,
-        // just finish the parse now.
-        if (LazyStreamer && SeenValueSymbolTable) {
-          NextUnreadBit = Stream.GetCurrentBitNo();
-          DEBUG(dbgs() << "<- ParseModule\n");
-          return std::error_code();
-        }
-        break;
-      }
-      continue;
-
-    case NaClBitstreamEntry::Record:
-      // The interesting case.
-      break;
-    }
-
-    // Read a record.
-    unsigned Selector = Stream.readRecord(Entry.ID, Record);
-    switch (Selector) {
-    default: {
-      std::string Message;
-      raw_string_ostream StrM(Message);
-      StrM << "Invalid MODULE_CODE: " << Selector;
-      StrM.flush();
-      return Error(InvalidValue, Message);
-    }
-    case naclbitc::MODULE_CODE_VERSION: {  // VERSION: [version#]
-      if (Record.size() < 1)
-        return Error(InvalidRecord, "Malformed MODULE_CODE_VERSION");
-      // Only version #1 is supported for PNaCl. Version #0 is not supported.
-      uint64_t module_version = Record[0];
-      if (module_version != 1)
-        return Error(InvalidValue, "Unknown bitstream version!");
-      break;
-    }
-    // FUNCTION:  [type, callingconv, isproto, linkage]
-    case naclbitc::MODULE_CODE_FUNCTION: {
-      if (Record.size() < 4)
-        return Error(InvalidRecord, "Invalid MODULE_CODE_FUNCTION record");
-      Type *Ty = getTypeByID(Record[0]);
-      if (!Ty)
-        return Error(InvalidType, "Invalid MODULE_CODE_FUNCTION record");
-      FunctionType *FTy = dyn_cast<FunctionType>(Ty);
-      if (!FTy)
-        return Error(InvalidType,
-                     "Function not declared with a function type!");
-
-      Function *Func = Function::Create(FTy, GlobalValue::ExternalLinkage,
-                                        "", TheModule);
-
-      CallingConv::ID CallingConv;
-      if (!naclbitc::DecodeCallingConv(Record[1], CallingConv))
-        return Error(InvalidValue,
-                     "PNaCl bitcode contains invalid calling conventions.");
-      Func->setCallingConv(CallingConv);
-      bool isProto = Record[2];
-      GlobalValue::LinkageTypes Linkage;
-      if (!naclbitc::DecodeLinkage(Record[3], Linkage))
-        return Error(InvalidValue, "Unknown linkage type");
-      Func->setLinkage(Linkage);
-      ValueList.push_back(Func);
-
-      // If this is a function with a body, remember the prototype we are
-      // creating now, so that we can match up the body with them later.
-      if (!isProto) {
-        Func->setIsMaterializable(true);
-        FunctionsWithBodies.push_back(Func);
-        if (LazyStreamer) DeferredFunctionInfo[Func] = 0;
-      }
-      break;
-    }
-    }
-    Record.clear();
-  }
-  return std::error_code();
-}
-
-const char *llvm::PNaClDataLayout =
-    "e-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-"
-    "f32:32:32-f64:64:64-p:32:32:32-v128:32:32";
-
-std::error_code NaClBitcodeReader::ParseBitcodeInto(Module *M) {
-  TheModule = 0;
-
-  // PNaCl does not support different DataLayouts in pexes, so we
-  // implicitly set the DataLayout to the following default.
-  //
-  // This is not usually needed by the backend, but it might be used
-  // by IR passes that the PNaCl translator runs.  We set this in the
-  // reader rather than in pnacl-llc so that 'opt' will also use the
-  // correct DataLayout if it is run on a pexe.
-  M->setDataLayout(PNaClDataLayout);
-
-  if (std::error_code EC = InitStream())
-    return EC;
-
-  // We expect a number of well-defined blocks, though we don't necessarily
-  // need to understand them all.
-  while (1) {
-    if (Stream.AtEndOfStream())
-      return std::error_code();
-
-    NaClBitstreamEntry Entry =
-        Stream.advance(NaClBitstreamCursor::AF_DontAutoprocessAbbrevs, nullptr);
-
-    switch (Entry.Kind) {
-    case NaClBitstreamEntry::Error:
-      return Error(MalformedBlock, "malformed module file");
-    case NaClBitstreamEntry::EndBlock:
-      return std::error_code();
-
-    case NaClBitstreamEntry::SubBlock:
-      switch (Entry.ID) {
-      case naclbitc::MODULE_BLOCK_ID:
-        // Reject multiple MODULE_BLOCK's in a single bitstream.
-        if (TheModule)
-          return Error(InvalidMultipleBlocks,
-                       "Multiple MODULE_BLOCKs in same stream");
-        TheModule = M;
-        if (std::error_code EC = ParseModule(false))
-          return EC;
-        if (LazyStreamer)
-          return std::error_code();
-        break;
-      default:
-        return Error(InvalidBlock, "Invalid top-level block found.");
-        break;
-      }
-      continue;
-    case NaClBitstreamEntry::Record:
-      // There should be no records in the top-level of blocks.
-      return Error(InvalidRecord, "Invalid record at top-level");
-    }
-  }
-}
-
-// Returns true if error occured installing I into BB.
-std::error_code NaClBitcodeReader::InstallInstruction(
-    BasicBlock *BB, Instruction *I) {
-  // Add instruction to end of current BB.  If there is no current BB, reject
-  // this file.
-  if (BB == 0) {
-    delete I;
-    return Error(InvalidInstructionWithNoBB,
-                 "Instruction with no BB, can't install");
-  }
-  BB->getInstList().push_back(I);
-  return std::error_code();
-}
-
-CastInst *
-NaClBitcodeReader::CreateCast(NaClBcIndexSize_t BBIndex,
-                              Instruction::CastOps Op,
-                              Type *CT, Value *V, bool DeferInsertion) {
-  if (BBIndex >= FunctionBBs.size())
-    report_fatal_error("CreateCast on unknown basic block");
-  BasicBlockInfo &BBInfo = FunctionBBs[BBIndex];
-  NaClBitcodeReaderCast ModeledCast(Op, CT, V);
-  CastInst *Cast = BBInfo.CastMap[ModeledCast];
-  if (Cast == NULL) {
-    Cast = CastInst::Create(Op, V, CT);
-    BBInfo.CastMap[ModeledCast] = Cast;
-    if (DeferInsertion) {
-      BBInfo.PhiCasts.push_back(Cast);
-    }
-  }
-  if (!DeferInsertion && Cast->getParent() == 0) {
-    InstallInstruction(BBInfo.BB, Cast);
-  }
-  return Cast;
-}
-
-Value *NaClBitcodeReader::ConvertOpToScalar(Value *Op,
-                                            NaClBcIndexSize_t BBIndex,
-                                            bool DeferInsertion) {
-  if (Op->getType()->isPointerTy()) {
-    return CreateCast(BBIndex, Instruction::PtrToInt, IntPtrType, Op,
-                      DeferInsertion);
-  }
-  return Op;
-}
-
-Value *NaClBitcodeReader::ConvertOpToType(Value *Op, Type *T,
-                                          NaClBcIndexSize_t BBIndex) {
-  Type *OpTy = Op->getType();
-  if (OpTy == T) return Op;
-
-  if (OpTy->isPointerTy()) {
-    if (T == IntPtrType) {
-      return ConvertOpToScalar(Op, BBIndex);
-    } else {
-      return CreateCast(BBIndex, Instruction::BitCast, T, Op);
-    }
-  } else if (OpTy == IntPtrType) {
-    return CreateCast(BBIndex, Instruction::IntToPtr, T, Op);
-  }
-
-  std::string Message;
-  raw_string_ostream StrM(Message);
-  StrM << "Can't convert " << *Op << " to type " << *T << "\n";
-  report_fatal_error(StrM.str());
-}
-
-/// ParseFunctionBody - Lazily parse the specified function body block.
-std::error_code NaClBitcodeReader::ParseFunctionBody(Function *F) {
-  DEBUG(dbgs() << "-> ParseFunctionBody\n");
-  if (Stream.EnterSubBlock(naclbitc::FUNCTION_BLOCK_ID))
-    return Error(InvalidRecord, "Malformed block record");
-
-  NaClBcIndexSize_t ModuleValueListSize = ValueList.size();
-
-  // Add all the function arguments to the value table.
-  for(Function::arg_iterator I = F->arg_begin(), E = F->arg_end(); I != E; ++I)
-    ValueList.push_back(I);
-
-  NaClBcIndexSize_t NextValueNo = ValueList.size();
-  BasicBlock *CurBB = 0;
-  NaClBcIndexSize_t CurBBNo = 0;
-
-  // Read all the records.
-  SmallVector<uint64_t, 64> Record;
-  while (1) {
-    NaClBitstreamEntry Entry = Stream.advance(0, nullptr);
-
-    switch (Entry.Kind) {
-    case NaClBitstreamEntry::Error:
-      return Error(MalformedBlock, "Bitcode error in function block");
-    case NaClBitstreamEntry::EndBlock:
-      goto OutOfRecordLoop;
-
-    case NaClBitstreamEntry::SubBlock:
-      switch (Entry.ID) {
-      default:
-        return Error(InvalidBlock, "Invalid block in function block");
-        break;
-      case naclbitc::CONSTANTS_BLOCK_ID:
-        if (std::error_code EC = ParseConstants())
-          return EC;
-        NextValueNo = ValueList.size();
-        break;
-      case naclbitc::VALUE_SYMTAB_BLOCK_ID:
-        if (PNaClAllowLocalSymbolTables) {
-          if (std::error_code EC = ParseValueSymbolTable())
-            return EC;
-        } else {
-          return Error(InvalidRecord, "Local value symbol tables not allowed");
-        }
-        break;
-      }
-      continue;
-
-    case NaClBitstreamEntry::Record:
-      // The interesting case.
-      break;
-    }
-
-    // Read a record.
-    Record.clear();
-    Instruction *I = 0;
-    unsigned BitCode = Stream.readRecord(Entry.ID, Record);
-    switch (BitCode) {
-    default: {// Default behavior: reject
-      std::string Message;
-      raw_string_ostream StrM(Message);
-      StrM << "Unknown instruction record: <" << BitCode;
-      for (size_t I = 0, E = Record.size(); I != E; ++I) {
-        StrM << " " << Record[I];
-      }
-      StrM << ">";
-      return Error(InvalidRecord, StrM.str());
-    }
-
-    case naclbitc::FUNC_CODE_DECLAREBLOCKS:     // DECLAREBLOCKS: [nblocks]
-      if (Record.size() != 1 || Record[0] == 0)
-        return Error(InvalidRecord, "Invalid DECLAREBLOCKS record");
-      // Create all the basic blocks for the function.
-      // TODO(kschimpf): Figure out how to handle size values that
-      // are too large.
-      FunctionBBs.resize(Record[0]);
-      for (size_t i = 0, e = FunctionBBs.size(); i != e; ++i) {
-        BasicBlockInfo &BBInfo = FunctionBBs[i];
-        BBInfo.BB = BasicBlock::Create(Context, "", F);
-      }
-      CurBB = FunctionBBs.at(0).BB;
-      continue;
-
-    case naclbitc::FUNC_CODE_INST_BINOP: {
-      // BINOP: [opval, opval, opcode[, flags]]
-      // Note: Only old PNaCl bitcode files may contain flags. If
-      // they are found, we ignore them.
-      size_t OpNum = 0;
-      Value *LHS, *RHS;
-      if (popValue(Record, &OpNum, NextValueNo, &LHS) ||
-          popValue(Record, &OpNum, NextValueNo, &RHS) ||
-          OpNum+1 > Record.size())
-        return Error(InvalidRecord, "Invalid BINOP record");
-
-      LHS = ConvertOpToScalar(LHS, CurBBNo);
-      RHS = ConvertOpToScalar(RHS, CurBBNo);
-
-      Instruction::BinaryOps Opc;
-      if (!naclbitc::DecodeBinaryOpcode(Record[OpNum++], LHS->getType(), Opc))
-        return Error(InvalidValue, "Invalid binary opcode in BINOP record");
-      I = BinaryOperator::Create(Opc, LHS, RHS);
-      break;
-    }
-    case naclbitc::FUNC_CODE_INST_CAST: {    // CAST: [opval, destty, castopc]
-      size_t OpNum = 0;
-      Value *Op;
-      if (popValue(Record, &OpNum, NextValueNo, &Op) ||
-          OpNum+2 != Record.size())
-        return Error(InvalidRecord, "Invalid CAST record: bad record size");
-
-      Type *ResTy = getTypeByID(Record[OpNum]);
-      if (ResTy == 0)
-        return Error(InvalidType, "Invalid CAST record: bad type ID");
-      Instruction::CastOps Opc;
-      if (!naclbitc::DecodeCastOpcode(Record[OpNum+1], Opc)) {
-        return Error(InvalidValue, "Invalid CAST record: bad opcode");
-      }
-
-      // If a ptrtoint cast was elided on the argument of the cast,
-      // add it back. Note: The casts allowed here should match the
-      // casts in NaClValueEnumerator::ExpectsScalarValue.
-      switch (Opc) {
-      case Instruction::Trunc:
-      case Instruction::ZExt:
-      case Instruction::SExt:
-      case Instruction::UIToFP:
-      case Instruction::SIToFP:
-        Op = ConvertOpToScalar(Op, CurBBNo);
-        break;
-      default:
-        break;
-      }
-
-      I = CastInst::Create(Opc, Op, ResTy);
-      break;
-    }
-
-    case naclbitc::FUNC_CODE_INST_VSELECT: {// VSELECT: [opval, opval, pred]
-      // new form of select
-      // handles select i1 or select [N x i1]
-      size_t OpNum = 0;
-      Value *TrueVal, *FalseVal, *Cond;
-      if (popValue(Record, &OpNum, NextValueNo, &TrueVal) ||
-          popValue(Record, &OpNum, NextValueNo, &FalseVal) ||
-          popValue(Record, &OpNum, NextValueNo, &Cond) ||
-          OpNum != Record.size())
-        return Error(InvalidRecord, "Invalid SELECT record");
-
-      TrueVal = ConvertOpToScalar(TrueVal, CurBBNo);
-      FalseVal = ConvertOpToScalar(FalseVal, CurBBNo);
-
-      // select condition can be either i1 or [N x i1]
-      if (VectorType* vector_type =
-          dyn_cast<VectorType>(Cond->getType())) {
-        // expect <n x i1>
-        if (vector_type->getElementType() != Type::getInt1Ty(Context))
-          return Error(InvalidTypeForValue,
-                       "Invalid SELECT vector condition type");
-      } else {
-        // expect i1
-        if (Cond->getType() != Type::getInt1Ty(Context))
-          return Error(InvalidTypeForValue, "Invalid SELECT condition type");
-      }
-
-      I = SelectInst::Create(Cond, TrueVal, FalseVal);
-      break;
-    }
-
-    case naclbitc::FUNC_CODE_INST_EXTRACTELT: { // EXTRACTELT: [opval, opval]
-      size_t OpNum = 0;
-      Value *Vec, *Idx;
-      if (popValue(Record, &OpNum, NextValueNo, &Vec) ||
-          popValue(Record, &OpNum, NextValueNo, &Idx) || OpNum != Record.size())
-        return Error(InvalidRecord, "Invalid EXTRACTELEMENT record");
-
-      // expect i32
-      if (Idx->getType() != Type::getInt32Ty(Context))
-        return Error(InvalidTypeForValue, "Invalid EXTRACTELEMENT index type");
-
-      I = ExtractElementInst::Create(Vec, Idx);
-      break;
-    }
-
-    case naclbitc::FUNC_CODE_INST_INSERTELT: { // INSERTELT: [opval,opval,opval]
-      size_t OpNum = 0;
-      Value *Vec, *Elt, *Idx;
-      if (popValue(Record, &OpNum, NextValueNo, &Vec) ||
-          popValue(Record, &OpNum, NextValueNo, &Elt) ||
-          popValue(Record, &OpNum, NextValueNo, &Idx) || OpNum != Record.size())
-        return Error(InvalidRecord, "Invalid INSERTELEMENT record");
-
-      // expect vector type
-      if (!isa<VectorType>(Vec->getType()))
-        return Error(InvalidTypeForValue, "Invalid INSERTELEMENT vector type");
-      // match vector and element types
-      if (cast<VectorType>(Vec->getType())->getElementType() != Elt->getType())
-        return Error(InvalidTypeForValue,
-                     "Mismatched INSERTELEMENT vector and element type");
-      // expect i32
-      if (Idx->getType() != Type::getInt32Ty(Context))
-        return Error(InvalidTypeForValue, "Invalid INSERTELEMENT index type");
-
-      I = InsertElementInst::Create(Vec, Elt, Idx);
-      break;
-    }
-
-    case naclbitc::FUNC_CODE_INST_CMP2: { // CMP2: [opval, opval, pred]
-      // FCmp/ICmp returning bool or vector of bool
-
-      size_t OpNum = 0;
-      Value *LHS, *RHS;
-      if (popValue(Record, &OpNum, NextValueNo, &LHS) ||
-          popValue(Record, &OpNum, NextValueNo, &RHS) ||
-          OpNum+1 != Record.size())
-        return Error(InvalidRecord, "Invalid CMP record");
-
-      LHS = ConvertOpToScalar(LHS, CurBBNo);
-      RHS = ConvertOpToScalar(RHS, CurBBNo);
-
-      CmpInst::Predicate Predicate;
-      if (LHS->getType()->isFPOrFPVectorTy()) {
-        if (!naclbitc::DecodeFcmpPredicate(Record[OpNum], Predicate))
-          return Error(
-              InvalidValue,
-              "PNaCl bitcode contains invalid floating comparison predicate");
-        I = new FCmpInst(Predicate, LHS, RHS);
-      } else {
-        if (!naclbitc::DecodeIcmpPredicate(Record[OpNum], Predicate))
-          return Error(
-              InvalidValue,
-              "PNaCl bitcode contains invalid integer comparison predicate");
-        I = new ICmpInst(Predicate, LHS, RHS);
-      }
-      break;
-    }
-
-    case naclbitc::FUNC_CODE_INST_RET: // RET: [opval<optional>]
-      {
-        size_t Size = Record.size();
-        if (Size == 0) {
-          I = ReturnInst::Create(Context);
-          break;
-        }
-
-        size_t OpNum = 0;
-        Value *Op = NULL;
-        if (popValue(Record, &OpNum, NextValueNo, &Op))
-          return Error(InvalidRecord, "Invalid RET record");
-        if (OpNum != Record.size())
-          return Error(InvalidRecord, "Invalid RET record");
-
-        I = ReturnInst::Create(Context, ConvertOpToScalar(Op, CurBBNo));
-        break;
-      }
-    case naclbitc::FUNC_CODE_INST_BR: { // BR: [bb#, bb#, opval] or [bb#]
-      if (Record.size() != 1 && Record.size() != 3)
-        return Error(InvalidRecord, "Invalid BR record");
-      BasicBlock *TrueDest = getBasicBlock(Record[0]);
-      if (TrueDest == 0)
-        return Error(InvalidRecord, "Invalid BR record");
-
-      if (Record.size() == 1) {
-        I = BranchInst::Create(TrueDest);
-      }
-      else {
-        BasicBlock *FalseDest = getBasicBlock(Record[1]);
-        Value *Cond = getValue(Record, 2, NextValueNo);
-        if (FalseDest == 0 || Cond == 0)
-          return Error(InvalidValue, "Invalid BR record");
-        if (!Cond->getType()->isIntegerTy(1)) {
-          std::string Buffer;
-          raw_string_ostream StrBuf(Buffer);
-          StrBuf << "Type of branch condition not i1. Found: "
-                 << *Cond->getType() << "\n";
-          return Error(InvalidValue, StrBuf.str());
-        }
-        I = BranchInst::Create(TrueDest, FalseDest, Cond);
-      }
-      break;
-    }
-    case naclbitc::FUNC_CODE_INST_SWITCH: { // SWITCH: [opty, op0, op1, ...]
-      if (Record.size() < 4)
-        return Error(InvalidRecord, "Invalid SWITCH record");
-      Type *OpTy = getTypeByID(Record[0]);
-      unsigned ValueBitWidth = cast<IntegerType>(OpTy)->getBitWidth();
-      if (ValueBitWidth > 64)
-        return Error(InvalidValue,
-                     "Wide integers are not supported in PNaCl bitcode");
-
-      Value *Cond = getValue(Record, 1, NextValueNo);
-      BasicBlock *Default = getBasicBlock(Record[2]);
-      if (OpTy == 0 || Cond == 0 || Default == 0)
-        return Error(InvalidRecord, "Invalid SWITCH record");
-
-      Cond = ConvertOpToScalar(Cond, CurBBNo);
-      // TODO(kschimpf): Deal with values that are too large for NumCases.
-      size_t NumCases = Record[3];
-
-      SwitchInst *SI = SwitchInst::Create(Cond, Default, NumCases);
-
-      size_t CurIdx = 4;
-      for (size_t i = 0; i != NumCases; ++i) {
-        // The PNaCl bitcode format has vestigial support for case
-        // ranges, but we no longer support reading them because
-        // no-one produced them.
-        // See https://code.google.com/p/nativeclient/issues/detail?id=3758
-        if (CurIdx + 3 >= Record.size())
-          return Error(InvalidRecord,
-                       "Incomplete case entry in SWITCH record");
-        uint64_t NumItems = Record[CurIdx++];
-        bool isSingleNumber = Record[CurIdx++];
-        if (NumItems != 1 || !isSingleNumber)
-          return Error(InvalidRecord,
-                       "Case ranges are not supported in PNaCl bitcode");
-
-        APInt CaseValue(ValueBitWidth,
-                        NaClDecodeSignRotatedValue(Record[CurIdx++]));
-        BasicBlock *DestBB = getBasicBlock(Record[CurIdx++]);
-        SI->addCase(ConstantInt::get(Context, CaseValue), DestBB);
-      }
-      I = SI;
-      break;
-    }
-    case naclbitc::FUNC_CODE_INST_UNREACHABLE: // UNREACHABLE
-      I = new UnreachableInst(Context);
-      break;
-    case naclbitc::FUNC_CODE_INST_PHI: { // PHI: [ty, val0,bb0, ...]
-      if (Record.size() < 1 || ((Record.size()-1)&1))
-        return Error(InvalidRecord, "Invalid PHI record");
-      Type *Ty = getTypeByID(Record[0]);
-      if (!Ty) return Error(InvalidType, "Invalid PHI record");
-
-      PHINode *PN = PHINode::Create(Ty, (Record.size()-1)/2);
-
-      for (size_t i = 0, e = Record.size()-1; i != e; i += 2) {
-        Value *V;
-        // With relative value IDs, it is possible that operands have
-        // negative IDs (for forward references).  Use a signed VBR
-        // representation to keep the encoding small.
-        V = getValueSigned(Record, 1+i, NextValueNo);
-        NaClBcIndexSize_t BBIndex = Record[2+i];
-        BasicBlock *BB = getBasicBlock(BBIndex);
-        if (!V || !BB)
-          return Error(InvalidValue, "Invalid PHI record");
-        if (Ty == IntPtrType) {
-          // Delay installing scalar casts until all instructions of
-          // the function are rendered. This guarantees that we insert
-          // the conversion just before the incoming edge (or use an
-          // existing conversion if already installed).
-          V = ConvertOpToScalar(V, BBIndex, /* DeferInsertion = */ true);
-        }
-        PN->addIncoming(V, BB);
-      }
-      I = PN;
-      break;
-    }
-
-    case naclbitc::FUNC_CODE_INST_ALLOCA: { // ALLOCA: [op, align]
-      if (Record.size() != 2)
-        return Error(InvalidRecord, "Invalid ALLOCA record");
-      Value *Size;
-      size_t OpNum = 0;
-      if (popValue(Record, &OpNum, NextValueNo, &Size))
-        return Error(InvalidRecord, "Invalid ALLOCA record");
-      unsigned Alignment;
-      if (std::error_code EC = getAlignmentValue(Record[1], Alignment))
-        return EC;
-      I = new AllocaInst(Type::getInt8Ty(Context), Size, Alignment);
-      break;
-    }
-    case naclbitc::FUNC_CODE_INST_LOAD: {
-      // LOAD: [op, align, ty]
-      size_t OpNum = 0;
-      Value *Op;
-      if (popValue(Record, &OpNum, NextValueNo, &Op) ||
-          Record.size() != 3)
-        return Error(InvalidRecord, "Invalid LOAD record");
-
-      // Add pointer cast to op.
-      Type *T = getTypeByID(Record[2]);
-      if (T == nullptr)
-        return Error(InvalidType, "Invalid type for load instruction");
-      Op = ConvertOpToType(Op, T->getPointerTo(), CurBBNo);
-      if (Op == nullptr)
-        return Error(InvalidTypeForValue, "Can't convert cast to type");
-      unsigned Alignment;
-      if (std::error_code EC = getAlignmentValue(Record[OpNum], Alignment))
-        return EC;
-      I = new LoadInst(Op, "", false, Alignment);
-      break;
-    }
-    case naclbitc::FUNC_CODE_INST_STORE: {
-      // STORE: [ptr, val, align]
-      size_t OpNum = 0;
-      Value *Val, *Ptr;
-      if (popValue(Record, &OpNum, NextValueNo, &Ptr) ||
-          popValue(Record, &OpNum, NextValueNo, &Val) ||
-          OpNum+1 != Record.size())
-        return Error(InvalidRecord, "Invalid STORE record");
-      Val = ConvertOpToScalar(Val, CurBBNo);
-      Ptr = ConvertOpToType(Ptr, Val->getType()->getPointerTo(), CurBBNo);
-      if (Ptr == nullptr)
-        return Error(InvalidTypeForValue, "Can't convert cast to type");
-      unsigned Alignment;
-      if (std::error_code EC = getAlignmentValue(Record[OpNum], Alignment))
-       return EC;
-      I = new StoreInst(Val, Ptr, false, Alignment);
-      break;
-    }
-    case naclbitc::FUNC_CODE_INST_CALL:
-    case naclbitc::FUNC_CODE_INST_CALL_INDIRECT: {
-      // CALL: [cc, fnid, arg0, arg1...]
-      // CALL_INDIRECT: [cc, fnid, returnty, args...]
-      if ((Record.size() < 2) ||
-          (BitCode == naclbitc::FUNC_CODE_INST_CALL_INDIRECT &&
-           Record.size() < 3))
-        return Error(InvalidRecord, "Invalid CALL record");
-
-      unsigned CCInfo = Record[0];
-
-      size_t OpNum = 1;
-      Value *Callee;
-      if (popValue(Record, &OpNum, NextValueNo, &Callee))
-        return Error(InvalidRecord, "Invalid CALL record");
-
-      // Build function type for call.
-      FunctionType *FTy = 0;
-      Type *ReturnType = 0;
-      if (BitCode == naclbitc::FUNC_CODE_INST_CALL_INDIRECT) {
-        // Callee type has been elided, add back in.
-        ReturnType = getTypeByID(Record[2]);
-        ++OpNum;
-      } else {
-        // Get type signature from callee.
-        if (PointerType *OpTy = dyn_cast<PointerType>(Callee->getType())) {
-          FTy = dyn_cast<FunctionType>(OpTy->getElementType());
-        }
-        if (FTy == 0)
-          return Error(InvalidType, "Invalid type for CALL record");
-      }
-
-      size_t NumParams = Record.size() - OpNum;
-      if (FTy && NumParams != FTy->getNumParams())
-        return Error(InvalidRecord, "Invalid CALL record");
-
-      // Process call arguments.
-      SmallVector<Value*, 6> Args;
-      for (size_t Index = 0; Index < NumParams; ++Index) {
-        Value *Arg;
-        if (popValue(Record, &OpNum, NextValueNo, &Arg)) {
-          std::string Buffer;
-          raw_string_ostream StrBuf(Buffer);
-          StrBuf << "Invalid call argument: Index " << Index;
-          return Error(InvalidValue, StrBuf.str());
-        }
-        if (FTy) {
-          // Add a cast, to a pointer type if necessary, in case this
-          // is an intrinsic call that takes a pointer argument.
-          Arg = ConvertOpToType(Arg, FTy->getParamType(Index), CurBBNo);
-        } else {
-          Arg = ConvertOpToScalar(Arg, CurBBNo);
-        }
-        if (Arg == nullptr) {
-          std::string Buffer;
-          raw_string_ostream StrBuf(Buffer);
-          StrBuf << "Unable to cast call argument to parameter type: " << Index;
-          return Error(InvalidValue, StrBuf.str());
-        }
-        Args.push_back(Arg);
-      }
-      if (FTy == nullptr) {
-        // Reconstruct the function type and cast the function pointer
-        // to it.
-        SmallVector<Type*, 6> ArgTypes;
-        for (const auto Arg : Args) {
-          ArgTypes.push_back(Arg->getType());
-        }
-        FTy = FunctionType::get(ReturnType, ArgTypes, false);
-        Callee = ConvertOpToType(Callee, FTy->getPointerTo(), CurBBNo);
-      }
-
-      // Construct call.
-      I = CallInst::Create(Callee, Args);
-      CallingConv::ID CallingConv;
-      if (!naclbitc::DecodeCallingConv(CCInfo>>1, CallingConv))
-        return Error(InvalidValue,
-                     "PNaCl bitcode contains invalid calling conventions.");
-      cast<CallInst>(I)->setCallingConv(CallingConv);
-      cast<CallInst>(I)->setTailCall(CCInfo & 1);
-      break;
-    }
-    case naclbitc::FUNC_CODE_INST_FORWARDTYPEREF:
-      // Build corresponding forward reference.
-      if (Record.size() != 2 ||
-          ValueList.createValueFwdRef(Record[0], getTypeByID(Record[1])))
-        return Error(InvalidRecord, "Invalid FORWARDTYPEREF record");
-      continue;
-    }
-
-    if (std::error_code EC = InstallInstruction(CurBB, I))
-      return EC;
-
-    // If this was a terminator instruction, move to the next block.
-    if (isa<TerminatorInst>(I)) {
-      ++CurBBNo;
-      CurBB = getBasicBlock(CurBBNo);
-    }
-
-    // Non-void values get registered in the value table for future use.
-    if (I && !I->getType()->isVoidTy()) {
-      Value *NewVal = I;
-      if (NewVal->getType()->isPointerTy() &&
-          getFnValueByID(NextValueNo)) {
-        // Forward-referenced values cannot have pointer type.
-        NewVal = ConvertOpToScalar(NewVal, CurBBNo);
-      }
-      ValueList.AssignValue(NewVal, NextValueNo++);
-    }
-  }
-
-OutOfRecordLoop:
-
-  // Add PHI conversions to corresponding incoming block, if not
-  // already in the block. Also clear all conversions after fixing
-  // PHI conversions.
-  for (size_t I = 0, NumBBs = FunctionBBs.size(); I < NumBBs; ++I) {
-    BasicBlockInfo &BBInfo = FunctionBBs[I];
-    std::vector<CastInst*> &PhiCasts = BBInfo.PhiCasts;
-    for (std::vector<CastInst*>::iterator Iter = PhiCasts.begin(),
-           IterEnd = PhiCasts.end(); Iter != IterEnd; ++Iter) {
-      CastInst *Cast = *Iter;
-      if (Cast->getParent() == 0) {
-        BasicBlock *BB = BBInfo.BB;
-        BB->getInstList().insert(BB->getTerminator(), Cast);
-      }
-    }
-    PhiCasts.clear();
-    BBInfo.CastMap.clear();
-  }
-
-  // Check the function list for unresolved values.
-  if (Argument *A = dyn_cast<Argument>(ValueList.back())) {
-    if (A->getParent() == 0) {
-      // We found at least one unresolved value.  Nuke them all to avoid leaks.
-      for (size_t i = ModuleValueListSize, e = ValueList.size(); i != e; ++i){
-        if ((A = dyn_cast<Argument>(ValueList[i])) && A->getParent() == 0) {
-          A->replaceAllUsesWith(UndefValue::get(A->getType()));
-          delete A;
-        }
-      }
-      return Error(InvalidValue, "Never resolved value found in function!");
-    }
-  }
-
-  // Trim the value list down to the size it was before we parsed this function.
-  ValueList.shrinkTo(ModuleValueListSize);
-  FunctionBBs.clear();
-  DEBUG(dbgs() << "-> ParseFunctionBody\n");
-  return std::error_code();
-}
-
-/// FindFunctionInStream - Find the function body in the bitcode stream
-std::error_code NaClBitcodeReader::FindFunctionInStream(
-    Function *F,
-    DenseMap<Function*, uint64_t>::iterator DeferredFunctionInfoIterator) {
-  while (DeferredFunctionInfoIterator->second == 0) {
-    if (Stream.AtEndOfStream())
-      return Error(CouldNotFindFunctionInStream,
-                   "Could not find Function in stream");
-    // ParseModule will parse the next body in the stream and set its
-    // position in the DeferredFunctionInfo map.
-    if (std::error_code EC = ParseModule(true))
-      return EC;
-  }
-  return std::error_code();
-}
-
-//===----------------------------------------------------------------------===//
-// GVMaterializer implementation
-//===----------------------------------------------------------------------===//
-
-void NaClBitcodeReader::releaseBuffer() { Buffer.release(); }
-
-std::error_code NaClBitcodeReader::materialize(GlobalValue *GV) {
-  Function *F = dyn_cast<Function>(GV);
-  // If it's not a function or is already material, ignore the request.
-  if (!F || !F->isMaterializable())
-    return std::error_code();
-
-  DenseMap<Function*, uint64_t>::iterator DFII = DeferredFunctionInfo.find(F);
-  assert(DFII != DeferredFunctionInfo.end() && "Deferred function not found!");
-  // If its position is recorded as 0, its body is somewhere in the stream
-  // but we haven't seen it yet.
-  if (DFII->second == 0) {
-    if (std::error_code EC = FindFunctionInStream(F, DFII)) {
-      return EC;
-    }
-  }
-
-  // Move the bit stream to the saved position of the deferred function body.
-  Stream.JumpToBit(DFII->second);
-
-  if (std::error_code EC = ParseFunctionBody(F))
-    return EC;
-  F->setIsMaterializable(false);
-
-  // Upgrade any old intrinsic calls in the function.
-  for (UpgradedIntrinsicMap::iterator I = UpgradedIntrinsics.begin(),
-       E = UpgradedIntrinsics.end(); I != E; ++I) {
-    if (I->first != I->second) {
-      for (Value::use_iterator UI = I->first->use_begin(),
-           UE = I->first->use_end(); UI != UE; ) {
-        if (CallInst* CI = dyn_cast<CallInst>(*UI++))
-          UpgradeIntrinsicCall(CI, I->second);
-      }
-    }
-  }
-
-  return std::error_code();
-}
-
-bool NaClBitcodeReader::isDematerializable(const GlobalValue *GV) const {
-  const Function *F = dyn_cast<Function>(GV);
-  if (!F || F->isDeclaration())
-    return false;
-  return DeferredFunctionInfo.count(const_cast<Function*>(F));
-}
-
-void NaClBitcodeReader::Dematerialize(GlobalValue *GV) {
-  Function *F = dyn_cast<Function>(GV);
-  // If this function isn't dematerializable, this is a noop.
-  if (!F || !isDematerializable(F))
-    return;
-
-  assert(DeferredFunctionInfo.count(F) && "No info to read function later?");
-
-  // Just forget the function body, we can remat it later.
-  F->dropAllReferences();
-  F->setIsMaterializable(true);
-}
-
-
-std::error_code NaClBitcodeReader::MaterializeModule(Module *M) {
-  assert(M == TheModule &&
-         "Can only Materialize the Module this NaClBitcodeReader is attached to.");
-  // Iterate over the module, deserializing any functions that are still on
-  // disk.
-  for (Module::iterator F = TheModule->begin(), E = TheModule->end();
-       F != E; ++F) {
-    if (F->isMaterializable()) {
-      if (std::error_code EC = materialize(F))
-        return EC;
-    }
-  }
-
-  // At this point, if there are any function bodies, the current bit is
-  // pointing to the END_BLOCK record after them. Now make sure the rest
-  // of the bits in the module have been read.
-  if (NextUnreadBit)
-    ParseModule(true);
-
-  // Upgrade any intrinsic calls that slipped through (should not happen!) and
-  // delete the old functions to clean up. We can't do this unless the entire
-  // module is materialized because there could always be another function body
-  // with calls to the old function.
-  for (std::vector<std::pair<Function*, Function*> >::iterator I =
-       UpgradedIntrinsics.begin(), E = UpgradedIntrinsics.end(); I != E; ++I) {
-    if (I->first != I->second) {
-      for (Value::use_iterator UI = I->first->use_begin(),
-           UE = I->first->use_end(); UI != UE; ) {
-        if (CallInst* CI = dyn_cast<CallInst>(*UI++))
-          UpgradeIntrinsicCall(CI, I->second);
-      }
-      if (!I->first->use_empty())
-        I->first->replaceAllUsesWith(I->second);
-      I->first->eraseFromParent();
-    }
-  }
-  std::vector<std::pair<Function*, Function*> >().swap(UpgradedIntrinsics);
-
-  return std::error_code();
-}
-
-std::vector<StructType *> NaClBitcodeReader::getIdentifiedStructTypes() const {
-  // MERGETODO(dschuff): does this need to contain anything for TypeFinder?
-  return std::vector<StructType *>();
-}
-
-std::error_code NaClBitcodeReader::InitStream() {
-  if (LazyStreamer)
-    return InitLazyStream();
-  return InitStreamFromBuffer();
-}
-
-std::error_code NaClBitcodeReader::InitStreamFromBuffer() {
-  const unsigned char *BufPtr = (const unsigned char*)Buffer->getBufferStart();
-  const unsigned char *BufEnd = BufPtr+Buffer->getBufferSize();
-
-  if (Buffer->getBufferSize() & 3)
-    return Error(InvalidBitstream,
-                 "Bitcode stream should be a multiple of 4 bytes in length");
-
-  if (Header.Read(BufPtr, BufEnd))
-    return Error(InvalidBitstream, Header.Unsupported());
-
-  if (AcceptHeader())
-    return Error(InvalidBitstream, Header.Unsupported());
-
-  StreamFile.reset(new NaClBitstreamReader(BufPtr, BufEnd, Header));
-  Stream.init(StreamFile.get());
-
-  return std::error_code();
-}
-
-std::error_code NaClBitcodeReader::InitLazyStream() {
-  if (Header.Read(LazyStreamer))
-    return Error(InvalidBitstream, Header.Unsupported());
-
-  if (AcceptHeader())
-    return Error(InvalidBitstream, Header.Unsupported());
-
-  StreamFile.reset(new NaClBitstreamReader(LazyStreamer, Header));
-  Stream.init(StreamFile.get());
-  return std::error_code();
-}
-
-//===----------------------------------------------------------------------===//
-// External interface
-//===----------------------------------------------------------------------===//
-
-/// \brief Get a lazy one-at-time loading module from bitcode.
-///
-/// This isn't always used in a lazy context.  In particular, it's also used by
-/// \a NaClParseBitcodeFile(). Compared to the upstream LLVM bitcode reader,
-/// NaCl does not support BlockAddresses, so it does not need to materialize
-/// forward-referenced functions from block address references.
-ErrorOr<Module *> llvm::getNaClLazyBitcodeModule(
-    std::unique_ptr<MemoryBuffer> &&Buffer, LLVMContext& Context,
-    raw_ostream *Verbose, bool AcceptSupportedOnly) {
-  Module *M = new Module(Buffer->getBufferIdentifier(), Context);
-  NaClBitcodeReader *R =
-      new NaClBitcodeReader(Buffer.get(), Context, Verbose, AcceptSupportedOnly);
-  M->setMaterializer(R);
-
-  auto cleanupOnError = [&](std::error_code EC) {
-    R->releaseBuffer(); // Never take ownership on error.
-    delete M;  // Also deletes R.
-    return EC;
-  };
-
-  if (std::error_code EC = R->ParseBitcodeInto(M))
-    return cleanupOnError(EC);
-
-  Buffer.release(); // The BitcodeReader owns it now.
-  return M;
-}
-
-
-Module *llvm::getNaClStreamedBitcodeModule(const std::string &name,
-                                           StreamingMemoryObject *Streamer,
-                                           LLVMContext &Context,
-                                           raw_ostream *Verbose,
-                                           std::string *ErrMsg,
-                                           bool AcceptSupportedOnly) {
-  Module *M = new Module(name, Context);
-  NaClBitcodeReader *R =
-      new NaClBitcodeReader(Streamer, Context, Verbose,
-                            AcceptSupportedOnly);
-  M->setMaterializer(R);
-  if (std::error_code EC = R->ParseBitcodeInto(M)) {
-    if (ErrMsg)
-      *ErrMsg = EC.message();
-    delete M;  // Also deletes R.
-    return nullptr;
-  }
-
-  return M;
-}
-
-ErrorOr<Module *> llvm::NaClParseBitcodeFile(
-    MemoryBufferRef Buffer, LLVMContext& Context, raw_ostream *Verbose,
-    bool AcceptSupportedOnly){
-  std::unique_ptr<MemoryBuffer> Buf = MemoryBuffer::getMemBuffer(Buffer, false);
-  ErrorOr<Module *> ModuleOrErr =
-      getNaClLazyBitcodeModule(std::move(Buf), Context, Verbose,
-                               AcceptSupportedOnly);
-  if (!ModuleOrErr)
-    return ModuleOrErr;
-  Module *M = ModuleOrErr.get();
-  // Read in the entire module, and destroy the NaClBitcodeReader.
-  if (std::error_code EC = M->materializeAllPermanently()) {
-    delete M;
-    return EC;
-  }
-
-  // TODO: Restore the use-lists to the in-memory state when the bitcode was
-  // written.  We must defer until the Module has been fully materialized.
-
-  return M;
-}
diff --git a/lib/Bitcode/NaCl/Reader/NaClBitcodeReader.h b/lib/Bitcode/NaCl/Reader/NaClBitcodeReader.h
deleted file mode 100644
index 8aac9ea9f2a..00000000000
--- a/lib/Bitcode/NaCl/Reader/NaClBitcodeReader.h
+++ /dev/null
@@ -1,376 +0,0 @@
-//===- NaClBitcodeReader.h ------------------------------------*- C++ -*-===//
-//     Internal NaClBitcodeReader implementation
-//
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
-//
-//===----------------------------------------------------------------------===//
-//
-// This header defines the NaClBitcodeReader class.
-//
-//===----------------------------------------------------------------------===//
-
-#ifndef NACL_BITCODE_READER_H
-#define NACL_BITCODE_READER_H
-
-#include "llvm/ADT/DenseMap.h"
-#include "llvm/Analysis/NaCl/PNaClAllowedIntrinsics.h"
-#include "llvm/Bitcode/NaCl/NaClBitcodeHeader.h"
-#include "llvm/Bitcode/NaCl/NaClBitstreamReader.h"
-#include "llvm/Bitcode/NaCl/NaClBitcodeDefs.h"
-#include "llvm/Bitcode/NaCl/NaClLLVMBitCodes.h"
-#include "llvm/Bitcode/NaCl/NaClReaderWriter.h"
-#include "llvm/IR/DerivedTypes.h"
-#include "llvm/IR/GVMaterializer.h"
-#include "llvm/IR/Instruction.h"
-#include "llvm/IR/OperandTraits.h"
-#include "llvm/IR/Type.h"
-#include "llvm/IR/ValueHandle.h"
-#include <vector>
-
-namespace llvm {
-
-class MemoryBuffer;
-class LLVMContext;
-class CastInst;
-
-// Models a Cast.  Used to cache casts created in a basic block by the
-// PNaCl bitcode reader.
-struct NaClBitcodeReaderCast {
-  // Fields of the conversion.
-  Instruction::CastOps Op;
-  Type *Ty;
-  Value *Val;
-
-  NaClBitcodeReaderCast(Instruction::CastOps Op, Type *Ty, Value *Val)
-    : Op(Op), Ty(Ty), Val(Val) {}
-};
-
-// Models the data structure used to hash/compare Casts in a DenseMap.
-template<>
-struct DenseMapInfo<NaClBitcodeReaderCast> {
-public:
-  static NaClBitcodeReaderCast getEmptyKey() {
-    return NaClBitcodeReaderCast(Instruction::CastOpsEnd,
-                                 DenseMapInfo<Type*>::getEmptyKey(),
-                                 DenseMapInfo<Value*>::getEmptyKey());
-  }
-  static NaClBitcodeReaderCast getTombstoneKey() {
-    return NaClBitcodeReaderCast(Instruction::CastOpsEnd,
-                                 DenseMapInfo<Type*>::getTombstoneKey(),
-                                 DenseMapInfo<Value*>::getTombstoneKey());
-  }
-  static unsigned getHashValue(const NaClBitcodeReaderCast &C) {
-    std::pair<int, std::pair<Type*, Value*> > Tuple;
-    Tuple.first = C.Op;
-    Tuple.second.first = C.Ty;
-    Tuple.second.second = C.Val;
-    return DenseMapInfo<std::pair<int,
-                                  std::pair<Type*,
-                                            Value*> > >::getHashValue(Tuple);
-  }
-  static bool isEqual(const NaClBitcodeReaderCast &LHS,
-                      const NaClBitcodeReaderCast &RHS) {
-    return LHS.Op == RHS.Op && LHS.Ty == RHS.Ty && LHS.Val == RHS.Val;
-  }
-};
-
-//===----------------------------------------------------------------------===//
-//                          NaClBitcodeReaderValueList Class
-//===----------------------------------------------------------------------===//
-
-class NaClBitcodeReaderValueList {
-  std::vector<WeakVH> ValuePtrs;
-public:
-  NaClBitcodeReaderValueList() {}
-  ~NaClBitcodeReaderValueList() {}
-
-  // vector compatibility methods
-  size_t size() const { return ValuePtrs.size(); }
-  void resize(size_t N) { ValuePtrs.resize(N); }
-  void push_back(Value *V) {
-    ValuePtrs.push_back(V);
-  }
-
-  void clear() {
-    ValuePtrs.clear();
-  }
-
-  Value *operator[](size_t i) const {
-    assert(i < ValuePtrs.size());
-    return ValuePtrs[i];
-  }
-
-  Value *back() const { return ValuePtrs.back(); }
-  void pop_back() { ValuePtrs.pop_back(); }
-  bool empty() const { return ValuePtrs.empty(); }
-  void shrinkTo(size_t N) {
-    assert(N <= size() && "Invalid shrinkTo request!");
-    ValuePtrs.resize(N);
-  }
-
-  // Declares the type of the forward-referenced value Idx.  Returns
-  // true if an error occurred.  It is an error if Idx's type has
-  // already been declared.
-  bool createValueFwdRef(NaClBcIndexSize_t Idx, Type *Ty);
-
-  // Gets the forward reference value for Idx.
-  Value *getValueFwdRef(NaClBcIndexSize_t Idx);
-
-  // Assigns V to value index Idx.
-  void AssignValue(Value *V, NaClBcIndexSize_t Idx);
-
-  // Assigns Idx to the given value, overwriting the existing entry
-  // and possibly modifying the type of the entry.
-  void OverwriteValue(Value *V, NaClBcIndexSize_t Idx);
-};
-
-
-class NaClBitcodeReader : public GVMaterializer {
-  NaClBitcodeHeader Header;  // Header fields of the PNaCl bitcode file.
-  LLVMContext &Context;
-  Module *TheModule;
-  // If non-null, stream to write verbose errors to.
-  raw_ostream *Verbose;
-  PNaClAllowedIntrinsics AllowedIntrinsics;
-  std::unique_ptr<MemoryBuffer> Buffer;
-  std::unique_ptr<NaClBitstreamReader> StreamFile;
-  NaClBitstreamCursor Stream;
-  StreamingMemoryObject *LazyStreamer;
-  uint64_t NextUnreadBit;
-  bool SeenValueSymbolTable;
-  std::vector<Type*> TypeList;
-  NaClBitcodeReaderValueList ValueList;
-
-  // Holds information about each BasicBlock in the function being read.
-  struct BasicBlockInfo {
-    // A basic block within the function being modeled.
-    BasicBlock *BB;
-    // The set of generated conversions.
-    DenseMap<NaClBitcodeReaderCast, CastInst*> CastMap;
-    // The set of generated conversions that were added for phi nodes,
-    // and may need thier parent basic block defined.
-    std::vector<CastInst*> PhiCasts;
-  };
-
-  /// FunctionBBs - While parsing a function body, this is a list of the basic
-  /// blocks for the function.
-  std::vector<BasicBlockInfo> FunctionBBs;
-
-  // When reading the module header, this list is populated with functions that
-  // have bodies later in the file.
-  std::vector<Function*> FunctionsWithBodies;
-
-  // When intrinsic functions are encountered which require upgrading they are
-  // stored here with their replacement function.
-  typedef std::vector<std::pair<Function*, Function*> > UpgradedIntrinsicMap;
-  UpgradedIntrinsicMap UpgradedIntrinsics;
-
-  // Several operations happen after the module header has been read, but
-  // before function bodies are processed. This keeps track of whether
-  // we've done this yet.
-  bool SeenFirstFunctionBody;
-
-  /// DeferredFunctionInfo - When function bodies are initially scanned, this
-  /// map contains info about where to find deferred function body in the
-  /// stream.
-  DenseMap<Function*, uint64_t> DeferredFunctionInfo;
-
-  /// \brief True if we should only accept supported bitcode format.
-  bool AcceptSupportedBitcodeOnly;
-
-  /// \brief Integer type use for PNaCl conversion of pointers.
-  Type *IntPtrType;
-
-  static const std::error_category &BitcodeErrorCategory();
-
-public:
-
-  /// Types of errors reported.
-  enum ErrorType {
-    CouldNotFindFunctionInStream // Unable to find function in bitcode stream.
-    = 1,                         // Note: Error types must not be zero!
-    InsufficientFunctionProtos,
-    InvalidBitstream,         // Error in bitstream format.
-    InvalidBlock,             // Invalid block found in bitcode.
-    InvalidConstantReference, // Bad constant reference.
-    InvalidDataAfterModule,   // Invalid data after module.
-    InvalidInstructionWithNoBB,  // No basic block for instruction.
-    InvalidMultipleBlocks,    // Multiple blocks for a kind of block that should
-                              // have only one.
-    InvalidRecord,            // Record doesn't have expected size or structure.
-    InvalidSkippedBlock,      // Unable to skip unknown block in bitcode file.
-    InvalidType,              // Invalid type in record.
-    InvalidTypeForValue,      // Type of value incorrect.
-    InvalidValue,             // Invalid value in record.
-    MalformedBlock            // Unable to advance over block.
-  };
-
-  explicit NaClBitcodeReader(MemoryBuffer *buffer, LLVMContext &C,
-                             raw_ostream *Verbose,
-                             bool AcceptSupportedOnly)
-      : Context(C), TheModule(nullptr), Verbose(Verbose), AllowedIntrinsics(&C),
-        Buffer(buffer),
-        LazyStreamer(nullptr), NextUnreadBit(0), SeenValueSymbolTable(false),
-        ValueList(),
-        SeenFirstFunctionBody(false),
-        AcceptSupportedBitcodeOnly(AcceptSupportedOnly),
-        IntPtrType(IntegerType::get(C, PNaClIntPtrTypeBitSize)) {
-  }
-  explicit NaClBitcodeReader(StreamingMemoryObject *streamer,
-                             LLVMContext &C,
-                             raw_ostream *Verbose,
-                             bool AcceptSupportedOnly)
-      : Context(C), TheModule(nullptr), Verbose(Verbose), AllowedIntrinsics(&C),
-        Buffer(nullptr),
-        LazyStreamer(streamer), NextUnreadBit(0), SeenValueSymbolTable(false),
-        ValueList(),
-        SeenFirstFunctionBody(false),
-        AcceptSupportedBitcodeOnly(AcceptSupportedOnly),
-        IntPtrType(IntegerType::get(C, PNaClIntPtrTypeBitSize)) {
-  }
-  ~NaClBitcodeReader() override {
-    FreeState();
-  }
-
-  void FreeState();
-
-  bool isDematerializable(const GlobalValue *GV) const override;
-  std::error_code materialize(GlobalValue *GV) override;
-  std::error_code MaterializeModule(Module *M) override;
-  std::vector<StructType *> getIdentifiedStructTypes() const override;
-  void Dematerialize(GlobalValue *GV) override;
-  void releaseBuffer();
-
-  std::error_code Error(ErrorType E) const {
-    return std::error_code(E, BitcodeErrorCategory());
-  }
-
-  /// Generates the corresponding verbose Message, then generates error.
-  std::error_code Error(ErrorType E, const std::string &Message) const;
-
-  /// @brief Main interface to parsing a bitcode buffer.
-  /// @returns true if an error occurred.
-  std::error_code ParseBitcodeInto(Module *M);
-
-  /// Convert alignment exponent (i.e. power of two (or zero)) to the
-  /// corresponding alignment to use. If alignment is too large, it generates
-  /// an error message and returns corresponding error code.
-  std::error_code getAlignmentValue(uint64_t Exponent, unsigned &Alignment);
-
-  // GVMaterializer interface. It's a no-op for PNaCl bitcode, which has no
-  // metadata.
-  std::error_code materializeMetadata() override { return std::error_code(); };
-
-  // GVMaterializer interface. Causes debug info to be stripped from the module
-  // on materialization. It's a no-op for PNaCl bitcode, which has no metadata.
-  void setStripDebugInfo() override {};
-
-  // Returns the value associated with ID.  The value must already exist.
-  Value *getFnValueByID(NaClBcIndexSize_t ID) {
-    return ValueList.getValueFwdRef(ID);
-  }
-
-private:
-  // Returns false if Header is acceptable.
-  bool AcceptHeader() const {
-    return !(Header.IsSupported() ||
-             (!AcceptSupportedBitcodeOnly && Header.IsReadable()));
-  }
-  uint32_t GetPNaClVersion() const {
-    return Header.GetPNaClVersion();
-  }
-  Type *getTypeByID(NaClBcIndexSize_t ID);
-  BasicBlock *getBasicBlock(NaClBcIndexSize_t ID) const {
-    if (ID >= FunctionBBs.size()) return 0; // Invalid ID
-    return FunctionBBs[ID].BB;
-  }
-
-  /// \brief Read a value out of the specified record from slot '*Slot'.
-  /// Increment *Slot past the number of slots used by the value in the record.
-  /// Return true if there is an error.
-  bool popValue(const SmallVector<uint64_t, 64> &Record, size_t *Slot,
-                NaClBcIndexSize_t InstNum, Value **ResVal) {
-    if (*Slot == Record.size()) return true;
-    // ValNo is encoded relative to the InstNum.
-    NaClBcIndexSize_t ValNo = InstNum -
-        static_cast<NaClRelBcIndexSize_t>(Record[(*Slot)++]);
-    *ResVal = getFnValueByID(ValNo);
-    return *ResVal == 0;
-  }
-
-  /// getValue -- Version of getValue that returns ResVal directly,
-  /// or 0 if there is an error.
-  Value *getValue(const SmallVector<uint64_t, 64> &Record, size_t Slot,
-                  NaClBcIndexSize_t InstNum) {
-    if (Slot == Record.size()) return 0;
-    // ValNo is encoded relative to the InstNum.
-    NaClBcIndexSize_t ValNo = InstNum -
-        static_cast<NaClRelBcIndexSize_t>(Record[Slot]);
-    return getFnValueByID(ValNo);
-  }
-
-  /// getValueSigned -- Like getValue, but decodes signed VBRs.
-  Value *getValueSigned(const SmallVector<uint64_t, 64> &Record, size_t Slot,
-                        NaClBcIndexSize_t InstNum) {
-    if (Slot == Record.size()) return 0;
-    // ValNo is encoded relative to the InstNum.
-    NaClBcIndexSize_t ValNo = InstNum -
-        static_cast<NaClRelBcIndexSize_t>(
-            NaClDecodeSignRotatedValue(Record[Slot]));
-    return getFnValueByID(ValNo);
-  }
-
-  /// \brief Create an (elided) cast instruction for basic block
-  /// BBIndex.  Op is the type of cast.  V is the value to cast.  CT
-  /// is the type to convert V to.  DeferInsertion defines whether the
-  /// generated conversion should also be installed into basic block
-  /// BBIndex.  Note: For PHI nodes, we don't insert when created
-  /// (i.e. DeferInsertion=true), since they must be inserted at the end
-  /// of the corresponding incoming basic block.
-  CastInst *CreateCast(NaClBcIndexSize_t BBIndex, Instruction::CastOps Op,
-                       Type *CT, Value *V, bool DeferInsertion = false);
-
-  /// \brief Add instructions to cast Op to the given type T into
-  /// block BBIndex.  Follows rules for pointer conversion as defined
-  /// in llvm/lib/Transforms/NaCl/ReplacePtrsWithInts.cpp.
-  ///
-  /// Returns 0 if unable to generate conversion value (also generates
-  /// an appropriate error message and calls Error).
-  Value *ConvertOpToType(Value *Op, Type *T, NaClBcIndexSize_t BBIndex);
-
-  /// \brief If Op is a scalar value, this is a nop.  If Op is a
-  /// pointer value, a PtrToInt instruction is inserted (in BBIndex)
-  /// to convert Op to an integer.  For defaults on DeferInsertion,
-  /// see comments for method CreateCast.
-  Value *ConvertOpToScalar(Value *Op, NaClBcIndexSize_t BBIndex,
-                           bool DeferInsertion = false);
-
-  /// \brief Install instruction I into basic block BB.
-  std::error_code InstallInstruction(BasicBlock *BB, Instruction *I);
-
-  FunctionType *AddPointerTypesToIntrinsicType(StringRef Name,
-                                               FunctionType *FTy);
-  void AddPointerTypesToIntrinsicParams();
-  std::error_code ParseModule(bool Resume);
-  std::error_code ParseTypeTable();
-  std::error_code ParseTypeTableBody();
-  std::error_code ParseGlobalVars();
-  std::error_code ParseValueSymbolTable();
-  std::error_code ParseConstants();
-  std::error_code RememberAndSkipFunctionBody();
-  std::error_code ParseFunctionBody(Function *F);
-  std::error_code GlobalCleanup();
-  std::error_code InitStream();
-  std::error_code InitStreamFromBuffer();
-  std::error_code InitLazyStream();
-  std::error_code FindFunctionInStream(
-      Function *F,
-      DenseMap<Function*, uint64_t>::iterator DeferredFunctionInfoIterator);
-};
-
-} // End llvm namespace
-
-#endif
diff --git a/lib/Bitcode/NaCl/Reader/NaClBitstreamReader.cpp b/lib/Bitcode/NaCl/Reader/NaClBitstreamReader.cpp
deleted file mode 100644
index bdb7255f12c..00000000000
--- a/lib/Bitcode/NaCl/Reader/NaClBitstreamReader.cpp
+++ /dev/null
@@ -1,419 +0,0 @@
-//===- NaClBitstreamReader.cpp --------------------------------------------===//
-//     NaClBitstreamReader implementation
-//
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
-//
-//===----------------------------------------------------------------------===//
-
-#include "llvm/ADT/STLExtras.h"
-#include "llvm/Bitcode/NaCl/NaClBitstreamReader.h"
-#include "llvm/Support/Format.h"
-#include "llvm/Support/raw_ostream.h"
-
-using namespace llvm;
-
-namespace {
-
-static const char *ErrorLevelName[] = {
-  "Warning",
-  "Error",
-  "Fatal"
-};
-
-} // End of anonymous namespace.
-
-std::string llvm::naclbitc::getBitAddress(uint64_t Bit) {
-  std::string Buffer;
-  raw_string_ostream Stream(Buffer);
-  Stream << (Bit / 8) << ":" << (Bit % 8);
-  return Stream.str();
-}
-
-raw_ostream &llvm::naclbitc::ErrorAt(
-    raw_ostream &Out, ErrorLevel Level, uint64_t BitPosition) {
-  assert(Level < array_lengthof(::ErrorLevelName));
-  return Out << ErrorLevelName[Level] << "("
-             << naclbitc::getBitAddress(BitPosition) << "): ";
-}
-
-//===----------------------------------------------------------------------===//
-//  NaClBitstreamCursor implementation
-//===----------------------------------------------------------------------===//
-
-void NaClBitstreamCursor::ErrorHandler::
-Fatal(const std::string &ErrorMessage) const {
-  // Default implementation is simply print message, and the bit where
-  // the error occurred.
-  std::string Buffer;
-  raw_string_ostream StrBuf(Buffer);
-  naclbitc::ErrorAt(StrBuf, naclbitc::Fatal, Cursor.GetCurrentBitNo())
-      << ErrorMessage;
-  report_fatal_error(StrBuf.str());
-}
-
-void NaClBitstreamCursor::freeState() {
-  // Free all the Abbrevs.
-  for (size_t i = 0, e = CurAbbrevs.size(); i != e; ++i)
-    CurAbbrevs[i]->dropRef();
-  CurAbbrevs.clear();
-
-  // Free all the Abbrevs in the block scope.
-  for (size_t S = 0, e = BlockScope.size(); S != e; ++S) {
-    std::vector<NaClBitCodeAbbrev*> &Abbrevs = BlockScope[S].PrevAbbrevs;
-    for (size_t i = 0, e = Abbrevs.size(); i != e; ++i)
-      Abbrevs[i]->dropRef();
-  }
-  BlockScope.clear();
-}
-
-void NaClBitstreamCursor::reportInvalidAbbrevNumber(unsigned AbbrevNo) const {
-  std::string Buffer;
-  raw_string_ostream StrBuf(Buffer);
-  StrBuf << "Invalid abbreviation # " << AbbrevNo << " defined for record";
-  ErrHandler->Fatal(StrBuf.str());
-}
-
-void NaClBitstreamCursor::reportInvalidJumpToBit(uint64_t BitNo) const {
-  std::string Buffer;
-  raw_string_ostream StrBuf(Buffer);
-  StrBuf << "Invalid jump to bit " << BitNo;
-  ErrHandler->Fatal(StrBuf.str());
-}
-
-/// EnterSubBlock - Having read the ENTER_SUBBLOCK abbrevid, enter
-/// the block, and return true if the block has an error.
-bool NaClBitstreamCursor::EnterSubBlock(unsigned BlockID, unsigned *NumWordsP) {
-  // Save the current block's state on BlockScope.
-  BlockScope.push_back(Block(CurCodeSize));
-  BlockScope.back().PrevAbbrevs.swap(CurAbbrevs);
-
-  // Add the abbrevs specific to this block to the CurAbbrevs list.
-  if (const NaClBitstreamReader::BlockInfo *Info =
-      BitStream->getBlockInfo(BlockID)) {
-    for (size_t i = 0, e = Info->Abbrevs.size(); i != e; ++i) {
-      CurAbbrevs.push_back(Info->Abbrevs[i]);
-      CurAbbrevs.back()->addRef();
-    }
-  }
-
-  // Get the codesize of this block.
-  CurCodeSize.IsFixed = true;
-  CurCodeSize.NumBits = ReadVBR(naclbitc::CodeLenWidth);
-  SkipToFourByteBoundary();
-  unsigned NumWords = Read(naclbitc::BlockSizeWidth);
-  if (NumWordsP) *NumWordsP = NumWords;
-
-  // Validate that this block is sane.
-  if (CurCodeSize.NumBits == 0 || AtEndOfStream())
-    return true;
-
-  return false;
-}
-
-void NaClBitstreamCursor::skipAbbreviatedField(const NaClBitCodeAbbrevOp &Op) {
-  // Decode the value as we are commanded.
-  switch (Op.getEncoding()) {
-  case NaClBitCodeAbbrevOp::Literal:
-    // No read necessary for literal.
-    break;
-  case NaClBitCodeAbbrevOp::Fixed:
-    (void)Read((unsigned)Op.getValue());
-    break;
-  case NaClBitCodeAbbrevOp::VBR:
-    (void)ReadVBR64((unsigned)Op.getValue());
-    break;
-  case NaClBitCodeAbbrevOp::Array:
-    // This can't happen because the abbreviation must be valid.
-    llvm_unreachable("Bad array abbreviation encoding!");
-    break;
-  case NaClBitCodeAbbrevOp::Char6:
-    (void)Read(6);
-    break;
-  }
-}
-
-/// skipRecord - Read the current record and discard it.
-void NaClBitstreamCursor::skipRecord(unsigned AbbrevID) {
-  // Skip unabbreviated records by reading past their entries.
-  if (AbbrevID == naclbitc::UNABBREV_RECORD) {
-    unsigned Code = ReadVBR(6);
-    (void)Code;
-    unsigned NumElts = ReadVBR(6);
-    for (unsigned i = 0; i != NumElts; ++i)
-      (void)ReadVBR64(6);
-    SkipToByteBoundaryIfAligned();
-    return;
-  }
-
-  const NaClBitCodeAbbrev *Abbv = getAbbrev(AbbrevID);
-
-  for (unsigned i = 0, e = Abbv->getNumOperandInfos(); i != e; ++i) {
-    const NaClBitCodeAbbrevOp &Op = Abbv->getOperandInfo(i);
-    switch (Op.getEncoding()) {
-    default:
-      skipAbbreviatedField(Op);
-      break;
-    case NaClBitCodeAbbrevOp::Literal:
-      break;
-    case NaClBitCodeAbbrevOp::Array: {
-      // Array case.  Read the number of elements as a vbr6.
-      unsigned NumElts = ReadVBR(6);
-
-      // Get the element encoding.
-      const NaClBitCodeAbbrevOp &EltEnc = Abbv->getOperandInfo(++i);
-
-      // Read all the elements.
-      for (; NumElts; --NumElts)
-        skipAbbreviatedField(EltEnc);
-      break;
-    }
-    }
-  }
-  SkipToByteBoundaryIfAligned();
-}
-
-bool NaClBitstreamCursor::readRecordAbbrevField(
-    const NaClBitCodeAbbrevOp &Op, uint64_t &Value) {
-  switch (Op.getEncoding()) {
-  case NaClBitCodeAbbrevOp::Literal:
-    Value = Op.getValue();
-    break;
-  case NaClBitCodeAbbrevOp::Array:
-    // Returns number of elements in the array.
-    Value = ReadVBR(6);
-    return true;
-  case NaClBitCodeAbbrevOp::Fixed:
-    Value = Read((unsigned)Op.getValue());
-    break;
-  case NaClBitCodeAbbrevOp::VBR:
-    Value = ReadVBR64((unsigned)Op.getValue());
-    break;
-  case NaClBitCodeAbbrevOp::Char6:
-    Value = NaClBitCodeAbbrevOp::DecodeChar6(Read(6));
-    break;
-  }
-  return false;
-}
-
-uint64_t NaClBitstreamCursor::readArrayAbbreviatedField(
-    const NaClBitCodeAbbrevOp &Op) {
-  // Decode the value as we are commanded.
-  switch (Op.getEncoding()) {
-  case NaClBitCodeAbbrevOp::Literal:
-    return Op.getValue();
-  case NaClBitCodeAbbrevOp::Fixed:
-    return Read((unsigned)Op.getValue());
-  case NaClBitCodeAbbrevOp::VBR:
-    return ReadVBR64((unsigned)Op.getValue());
-  case NaClBitCodeAbbrevOp::Array:
-    // This can't happen because the abbreviation must be valid.
-    llvm_unreachable("Bad array abbreviation encoding!");
-    break;
-  case NaClBitCodeAbbrevOp::Char6:
-    return NaClBitCodeAbbrevOp::DecodeChar6(Read(6));
-  }
-}
-
-void NaClBitstreamCursor::readArrayAbbrev(
-    const NaClBitCodeAbbrevOp &Op, unsigned NumArrayElements,
-    SmallVectorImpl<uint64_t> &Vals) {
-  for (; NumArrayElements; --NumArrayElements) {
-    Vals.push_back(readArrayAbbreviatedField(Op));
-  }
-}
-
-unsigned NaClBitstreamCursor::readRecord(unsigned AbbrevID,
-                                         SmallVectorImpl<uint64_t> &Vals) {
-  if (AbbrevID == naclbitc::UNABBREV_RECORD) {
-    unsigned Code = ReadVBR(6);
-    unsigned NumElts = ReadVBR(6);
-    for (unsigned i = 0; i != NumElts; ++i)
-      Vals.push_back(ReadVBR64(6));
-    SkipToByteBoundaryIfAligned();
-    return Code;
-  }
-
-  // Read code.
-  const NaClBitCodeAbbrev *Abbv = getAbbrev(AbbrevID);
-  uint64_t Value;
-  unsigned Code;
-  if (readRecordAbbrevField(Abbv->getOperandInfo(0), Value)) {
-    // Array found, use to read all elements.
-    if (Value == 0)
-      ErrHandler->Fatal("No code found for record!");
-    const NaClBitCodeAbbrevOp &Op = Abbv->getOperandInfo(1);
-    Code = readArrayAbbreviatedField(Op);
-    readArrayAbbrev(Op, Value - 1, Vals);
-    SkipToByteBoundaryIfAligned();
-    return Code;
-  }
-  Code = Value;
-
-  // Read arguments.
-  unsigned NumOperands = Abbv->getNumOperandInfos();
-  for (unsigned i = 1; i != NumOperands; ++i) {
-    if (readRecordAbbrevField(Abbv->getOperandInfo(i), Value)) {
-      ++i;
-      readArrayAbbrev(Abbv->getOperandInfo(i), Value, Vals);
-      SkipToByteBoundaryIfAligned();
-      return Code;
-    }
-    Vals.push_back(Value);
-  }
-  SkipToByteBoundaryIfAligned();
-  return Code;
-}
-
-
-NaClBitCodeAbbrevOp::Encoding NaClBitstreamCursor::
-getEncoding(uint64_t Value) {
-  if (!NaClBitCodeAbbrevOp::isValidEncoding(Value)) {
-    std::string Buffer;
-    raw_string_ostream StrBuf(Buffer);
-    StrBuf << "Invalid abbreviation encoding specified in bitcode file: "
-           << Value;
-    ErrHandler->Fatal(StrBuf.str());
-  }
-  return NaClBitCodeAbbrevOp::Encoding(Value);
-}
-
-void NaClBitstreamCursor::ReadAbbrevRecord(bool IsLocal,
-                                           NaClAbbrevListener *Listener) {
-  NaClBitCodeAbbrev *Abbv = new NaClBitCodeAbbrev();
-  unsigned NumOpInfo = ReadVBR(5);
-  if (Listener) Listener->Values.push_back(NumOpInfo);
-  for (unsigned i = 0; i != NumOpInfo; ++i) {
-    bool IsLiteral = Read(1) ? true : false;
-    if (Listener) Listener->Values.push_back(IsLiteral);
-    if (IsLiteral) {
-      uint64_t Value = ReadVBR64(8);
-      if (Listener) Listener->Values.push_back(Value);
-      Abbv->Add(NaClBitCodeAbbrevOp(Value));
-      continue;
-    }
-    NaClBitCodeAbbrevOp::Encoding E = getEncoding(Read(3));
-    if (Listener) Listener->Values.push_back(E);
-    if (NaClBitCodeAbbrevOp::hasValue(E)) {
-      unsigned Data = ReadVBR64(5);
-      if (Listener) Listener->Values.push_back(Data);
-
-      // As a special case, handle fixed(0) (i.e., a fixed field with zero bits)
-      // and vbr(0) as a literal zero.  This is decoded the same way, and avoids
-      // a slow path in Read() to have to handle reading zero bits.
-      if ((E == NaClBitCodeAbbrevOp::Fixed || E == NaClBitCodeAbbrevOp::VBR) &&
-          Data == 0) {
-        if (Listener) Listener->Values.push_back(0);
-        Abbv->Add(NaClBitCodeAbbrevOp(0));
-        continue;
-      }
-      if (!NaClBitCodeAbbrevOp::isValid(E, Data)) {
-        std::string Buffer;
-        raw_string_ostream StrBuf(Buffer);
-        StrBuf << "Invalid abbreviation encoding ("
-               << NaClBitCodeAbbrevOp::getEncodingName(E)
-               << ", " << Data << ")";
-        ErrHandler->Fatal(StrBuf.str());
-      }
-      Abbv->Add(NaClBitCodeAbbrevOp(E, Data));
-    } else {
-      if (!NaClBitCodeAbbrevOp::isValid(E)) {
-        std::string Buffer;
-        raw_string_ostream StrBuf(Buffer);
-        StrBuf << "Invalid abbreviation encoding ("
-               << NaClBitCodeAbbrevOp::getEncodingName(E) << ")";
-        ErrHandler->Fatal(StrBuf.str());
-      }
-      Abbv->Add(NaClBitCodeAbbrevOp(E));
-    }
-  }
-  SkipToByteBoundaryIfAligned();
-  if (!Abbv->isValid())
-    ErrHandler->Fatal("Invalid abbreviation specified in bitcode file");
-  CurAbbrevs.push_back(Abbv);
-  if (Listener) {
-    Listener->ProcessAbbreviation(Abbv, IsLocal);
-    // Reset record information of the listener.
-    Listener->Values.clear();
-    Listener->StartBit = GetCurrentBitNo();
-  }
-}
-
-void NaClBitstreamCursor::SkipAbbrevRecord() {
-  unsigned NumOpInfo = ReadVBR(5);
-  for (unsigned i = 0; i != NumOpInfo; ++i) {
-    bool IsLiteral = Read(1) ? true : false;
-    if (IsLiteral) {
-      ReadVBR64(8);
-      continue;
-    }
-    NaClBitCodeAbbrevOp::Encoding E = getEncoding(Read(3));
-    if (NaClBitCodeAbbrevOp::hasValue(E)) {
-      ReadVBR64(5);
-    }
-  }
-  SkipToByteBoundaryIfAligned();
-}
-
-bool NaClBitstreamCursor::ReadBlockInfoBlock(NaClAbbrevListener *Listener) {
-  // If this is the second stream to get to the block info block, skip it.
-  if (BitStream->hasBlockInfoRecords())
-    return SkipBlock();
-
-  unsigned NumWords;
-  if (EnterSubBlock(naclbitc::BLOCKINFO_BLOCK_ID, &NumWords)) return true;
-
-  if (Listener) Listener->BeginBlockInfoBlock(NumWords);
-
-  NaClBitcodeRecordVector Record;
-  NaClBitstreamReader::BlockInfo *CurBlockInfo = 0;
-
-  // Read records of the BlockInfo block.
-  while (1) {
-    if (Listener) Listener->StartBit = GetCurrentBitNo();
-    NaClBitstreamEntry Entry = advance(AF_DontAutoprocessAbbrevs, Listener);
-
-    switch (Entry.Kind) {
-    case llvm::NaClBitstreamEntry::SubBlock:  // PNaCl doesn't allow!
-    case llvm::NaClBitstreamEntry::Error:
-      return true;
-    case llvm::NaClBitstreamEntry::EndBlock:
-      if (Listener) Listener->EndBlockInfoBlock();
-      return false;
-    case llvm::NaClBitstreamEntry::Record:
-      // The interesting case.
-      break;
-    }
-
-    // Read abbrev records, associate them with CurBID.
-    if (Entry.ID == naclbitc::DEFINE_ABBREV) {
-      if (!CurBlockInfo) return true;
-      ReadAbbrevRecord(false, Listener);
-
-      // ReadAbbrevRecord installs the abbrev in CurAbbrevs.  Move it to the
-      // appropriate BlockInfo.
-      NaClBitCodeAbbrev *Abbv = CurAbbrevs.back();
-      CurAbbrevs.pop_back();
-      CurBlockInfo->Abbrevs.push_back(Abbv);
-      continue;
-    }
-
-    // Read a record.
-    Record.clear();
-    switch (readRecord(Entry.ID, Record)) {
-      default: 
-        // No other records should be found!
-        return true;
-      case naclbitc::BLOCKINFO_CODE_SETBID:
-        if (Record.size() < 1) return true;
-        CurBlockInfo = &BitStream->getOrCreateBlockInfo((unsigned)Record[0]);
-        if (Listener) {
-          Listener->Values = Record;
-          Listener->SetBID();
-        }
-        break;
-    }
-  }
-}
diff --git a/lib/Bitcode/NaCl/TestUtils/CMakeLists.txt b/lib/Bitcode/NaCl/TestUtils/CMakeLists.txt
deleted file mode 100644
index 36539272c2f..00000000000
--- a/lib/Bitcode/NaCl/TestUtils/CMakeLists.txt
+++ /dev/null
@@ -1,12 +0,0 @@
-add_llvm_library(LLVMNaClBitTestUtils
-  NaClBitcodeMunge.cpp
-  NaClBitcodeMungeReader.cpp
-  NaClBitcodeMungeUtils.cpp
-  NaClBitcodeMungeWriter.cpp
-  NaClBitcodeTextReader.cpp
-  NaClBitcodeTextWriter.cpp
-  NaClFuzz.cpp
-  NaClRandNumGen.cpp
-  NaClSimpleRecordFuzzer.cpp
-  )
-add_dependencies(LLVMNaClBitTestUtils intrinsics_gen)
diff --git a/lib/Bitcode/NaCl/TestUtils/LLVMBuild.txt b/lib/Bitcode/NaCl/TestUtils/LLVMBuild.txt
deleted file mode 100644
index 398448354a6..00000000000
--- a/lib/Bitcode/NaCl/TestUtils/LLVMBuild.txt
+++ /dev/null
@@ -1,22 +0,0 @@
-;===- ./lib/Bitcode/NaCl/TestUtils/LLVMBuild.txt ---------------*- Conf -*--===;
-;
-;                     The LLVM Compiler Infrastructure
-;
-; This file is distributed under the University of Illinois Open Source
-; License. See LICENSE.TXT for details.
-;
-;===------------------------------------------------------------------------===;
-;
-; This is an LLVMBuild description file for the components in this subdirectory.
-;
-; For more information on the LLVMBuild system, please see:
-;
-;   http://llvm.org/docs/LLVMBuild.html
-;
-;===------------------------------------------------------------------------===;
-
-[component_0]
-type = Library
-name = NaClBitTestUtils
-parent = NaClBitcode
-required_libraries = Core Support NaClBitReader NaClBitWriter NaClBitAnalysis
diff --git a/lib/Bitcode/NaCl/TestUtils/Makefile b/lib/Bitcode/NaCl/TestUtils/Makefile
deleted file mode 100644
index a37c51ee903..00000000000
--- a/lib/Bitcode/NaCl/TestUtils/Makefile
+++ /dev/null
@@ -1,14 +0,0 @@
-##===- lib/Bitcode/NaCl/TestUtils/Makefile -----------------*- Makefile -*-===##
-#
-#                     The LLVM Compiler Infrastructure
-#
-# This file is distributed under the University of Illinois Open Source
-# License. See LICENSE.TXT for details.
-#
-##===----------------------------------------------------------------------===##
-
-LEVEL = ../../../..
-LIBRARYNAME = LLVMNaClBitTestUtils
-BUILD_ARCHIVE = 1
-
-include $(LEVEL)/Makefile.common
diff --git a/lib/Bitcode/NaCl/TestUtils/NaClBitcodeMunge.cpp b/lib/Bitcode/NaCl/TestUtils/NaClBitcodeMunge.cpp
deleted file mode 100644
index 0d5e70c5830..00000000000
--- a/lib/Bitcode/NaCl/TestUtils/NaClBitcodeMunge.cpp
+++ /dev/null
@@ -1,168 +0,0 @@
-//===--- Bitcode/NaCl/TestUtils/NaClBitcodeMunge.cpp - Bitcode Munger -----===//
-//
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
-//
-//===----------------------------------------------------------------------===//
-//
-// Bitcode writer/munger implementation for testing.
-//
-//===----------------------------------------------------------------------===//
-
-#include "llvm/Bitcode/NaCl/NaClBitcodeMunge.h"
-
-#include "llvm/Bitcode/NaCl/NaClBitcodeHeader.h"
-#include "llvm/Bitcode/NaCl/NaClBitcodeParser.h"
-#include "llvm/Bitcode/NaCl/NaClCompress.h"
-#include "llvm/Bitcode/NaCl/NaClReaderWriter.h"
-#include "llvm/IR/LLVMContext.h"
-#include "llvm/IR/Module.h"
-#include "llvm/Support/ErrorHandling.h"
-#include "llvm/Support/MemoryBuffer.h"
-
-#include <memory>
-
-using namespace llvm;
-
-// For debugging. When true, shows each test being run.
-static bool TraceTestRuns = false;
-
-bool NaClBitcodeMunger::setupTest(const uint64_t Munges[], size_t MungesSize,
-                                  bool AddHeader) {
-  assert(DumpStream == nullptr && "Test run with DumpStream already defined");
-  assert(MungedInput.get() == nullptr
-         && "Test run with MungedInput already defined");
-  FoundErrors = false;
-  DumpResults.clear(); // Throw away any previous results.
-  std::string DumpBuffer;
-  DumpStream = new raw_string_ostream(DumpResults);
-  MungedInputBuffer.clear();
-
-  if (TraceTestRuns) {
-    errs() << "*** Run test:\n";
-  }
-
-  MungedBitcode.munge(Munges, MungesSize, RecordTerminator);
-  WriteFlags.setErrStream(getDumpStream());
-  NaClMungedBitcode::WriteResults Results =
-      MungedBitcode.writeMaybeRepair(MungedInputBuffer, AddHeader, WriteFlags);
-  if (Results.NumErrors != 0
-      && !(WriteFlags.getTryToRecover()
-           && Results.NumRepairs == Results.NumErrors)
-      && !(WriteFlags.getWriteBadAbbrevIndex()
-           && Results.WroteBadAbbrevIndex && Results.NumErrors == 1)) {
-    Error() << "Unable to generate bitcode file due to write errors\n";
-    return false;
-  }
-
-  // Add null terminator, so that we meet the requirements of the
-  // MemoryBuffer API.
-  MungedInputBuffer.push_back('\0');
-
-  MungedInput = MemoryBuffer::getMemBuffer(
-      StringRef(MungedInputBuffer.data(), MungedInputBuffer.size()-1));
-  return true;
-}
-
-bool NaClBitcodeMunger::cleanupTest() {
-  RunAsDeathTest = false;
-  WriteFlags.reset();
-  MungedBitcode.removeEdits();
-  MungedInput.reset();
-  assert(DumpStream && "Dump stream removed before cleanup!");
-  DumpStream->flush();
-  delete DumpStream;
-  DumpStream = nullptr;
-  return !FoundErrors;
-}
-
-// Return the next line of input (including eoln), starting from
-// Pos. Then increment Pos past the end of that line.
-static std::string getLine(const std::string &Input, size_t &Pos) {
-  std::string Line;
-  if (Pos >= Input.size()) {
-    Pos = std::string::npos;
-    return Line;
-  }
-  size_t Eoln = Input.find_first_of("\n", Pos);
-  if (Eoln != std::string::npos) {
-    for (size_t i = Pos; i <= Eoln; ++i)
-      Line.push_back(Input[i]);
-    Pos = Eoln + 1;
-    return Line;
-  }
-  Pos = std::string::npos;
-  return Input.substr(Pos);
-}
-
-std::string NaClBitcodeMunger::
-getLinesWithTextMatch(const std::string &Substring, bool MustBePrefix) const {
-  std::string Messages;
-  size_t LastLinePos = 0;
-  while (1) {
-    std::string Line = getLine(DumpResults, LastLinePos);
-    if (LastLinePos == std::string::npos) break;
-    size_t Pos = Line.find(Substring);
-    if (Pos != std::string::npos && (!MustBePrefix || Pos == 0)) {
-      Messages.append(Line);
-    }
-  }
-  return Messages;
-}
-
-bool NaClWriteMunger::runTest(const uint64_t Munges[], size_t MungesSize) {
-  bool AddHeader = true;
-  if (!setupTest(Munges, MungesSize, AddHeader))
-    return cleanupTest();
-  MemoryBufferRef InputRef(MungedInput->getMemBufferRef());
-  NaClMungedBitcode WrittenBitcode(MemoryBuffer::getMemBuffer(InputRef));
-  WrittenBitcode.print(WriteFlags.getErrStream());
-  return cleanupTest();
-}
-
-bool NaClObjDumpMunger::runTestWithFlags(
-    const uint64_t Munges[], size_t MungesSize, bool AddHeader,
-    bool NoRecords, bool NoAssembly) {
-  if (!setupTest(Munges, MungesSize, AddHeader))
-    return cleanupTest();
-
-  if (NaClObjDump(MungedInput.get()->getMemBufferRef(),
-                  getDumpStream(), NoRecords, NoAssembly))
-    FoundErrors = true;
-  return cleanupTest();
-}
-
-bool NaClParseBitcodeMunger::runTest(const uint64_t Munges[], size_t MungesSize,
-                                     bool VerboseErrors) {
-  bool AddHeader = true;
-  if (!setupTest(Munges, MungesSize, AddHeader))
-    return cleanupTest();
-
-  LLVMContext &Context = getGlobalContext();
-
-  raw_ostream *VerboseStrm = VerboseErrors ? &getDumpStream() : nullptr;
-  ErrorOr<Module *> ModuleOrError =
-      NaClParseBitcodeFile(MungedInput->getMemBufferRef(), Context,
-                           VerboseStrm);
-  if (ModuleOrError) {
-    if (VerboseErrors)
-      getDumpStream() << "Successful parse!\n";
-    delete ModuleOrError.get();
-  } else {
-    Error() << ModuleOrError.getError().message() << "\n";
-  }
-  return cleanupTest();
-}
-
-bool NaClCompressMunger::runTest(const uint64_t Munges[], size_t MungesSize) {
-  bool AddHeader = true;
-  if (!setupTest(Munges, MungesSize, AddHeader))
-    return cleanupTest();
-
-  NaClBitcodeCompressor Compressor;
-  if (!Compressor.compress(MungedInput.get(), getDumpStream()))
-    Error() << "Unable to compress\n";
-  return cleanupTest();
-}
diff --git a/lib/Bitcode/NaCl/TestUtils/NaClBitcodeMungeReader.cpp b/lib/Bitcode/NaCl/TestUtils/NaClBitcodeMungeReader.cpp
deleted file mode 100644
index f0d9b2a15cc..00000000000
--- a/lib/Bitcode/NaCl/TestUtils/NaClBitcodeMungeReader.cpp
+++ /dev/null
@@ -1,153 +0,0 @@
-//===- NaClBitcodeMungeReader.cpp - Read bitcode record list ----*- C++ -*-===//
-//
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
-//
-//===----------------------------------------------------------------------===//
-//
-// Implements bitcode reader for NaClBitcodeRecordList and NaClMungedBitcode.
-
-#include "llvm/Bitcode/NaCl/NaClBitcodeMungeUtils.h"
-
-#include "llvm/Bitcode/NaCl/NaClBitcodeHeader.h"
-
-using namespace llvm;
-
-namespace {
-
-class BitcodeParser;
-
-// \brief The state associated with parsing a bitcode buffer.
-class BitcodeParseState {
-  BitcodeParseState(const BitcodeParseState&) = delete;
-  BitcodeParseState &operator=(const BitcodeParseState&) = delete;
-public:
-  // \brief Construct the bitcode parse state.
-  //
-  // \param Parser The parser used to parse the bitcode.
-  // \param[out] Records Filled with parsed records.
-  BitcodeParseState(BitcodeParser *Parser,
-                    NaClBitcodeRecordList &Records);
-
-  // List to read records into.
-  NaClBitcodeRecordList &Records;
-  // Listener used to get abbreviations as they are read.
-  NaClBitcodeParserListener AbbrevListener;
-};
-
-// \brief The bitcode parser to extract bitcode records.
-class BitcodeParser : public NaClBitcodeParser {
-  BitcodeParser(const BitcodeParser &) = delete;
-  BitcodeParser &operator=(const BitcodeParser&) = delete;
-public:
-  // \brief Top-level constructor for a bitcode parser.
-  //
-  // \param Cursor The beginning position of the bitcode to parse.
-  // \param[out] Records Filled with parsed records.
-  BitcodeParser(NaClBitstreamCursor &Cursor,
-                NaClBitcodeRecordList &Records)
-      : NaClBitcodeParser(Cursor),
-        State(new BitcodeParseState(this, Records)) {
-    SetListener(&State->AbbrevListener);
-  }
-
-  ~BitcodeParser() override {
-    if (EnclosingParser == nullptr)
-      delete State;
-  }
-
-  bool ParseBlock(unsigned BlockID) override {
-    BitcodeParser NestedParser(BlockID, this);
-    return NestedParser.ParseThisBlock();
-  }
-
-  void EnterBlock(unsigned NumWords) override {
-    NaClRecordVector Values;
-    Values.push_back(GetBlockID());
-    Values.push_back(Record.GetCursor().getAbbrevIDWidth());
-    std::unique_ptr<NaClBitcodeAbbrevRecord> AbbrevRec(
-        new NaClBitcodeAbbrevRecord(naclbitc::ENTER_SUBBLOCK,
-                                    naclbitc::BLK_CODE_ENTER,
-                                    Values));
-    State->Records.push_back(std::move(AbbrevRec));
-  }
-
-  void ExitBlock() override {
-    NaClRecordVector Values;
-    std::unique_ptr<NaClBitcodeAbbrevRecord> AbbrevRec(
-        new NaClBitcodeAbbrevRecord(naclbitc::END_BLOCK,
-                                    naclbitc::BLK_CODE_EXIT,
-                                    Values));
-    State->Records.push_back(std::move(AbbrevRec));
-  }
-
-  void ProcessRecord() override {
-    std::unique_ptr<NaClBitcodeAbbrevRecord> AbbrevRec(
-        new NaClBitcodeAbbrevRecord(Record.GetAbbreviationIndex(),
-                                    Record.GetCode(),
-                                    Record.GetValues()));
-    State->Records.push_back(std::move(AbbrevRec));
-  }
-
-  void SetBID() override {
-    ProcessRecord();
-  }
-
-  void ProcessAbbreviation(unsigned BlockID,
-                           NaClBitCodeAbbrev *Abbrev,
-                           bool IsLocal) override {
-    ProcessRecord();
-  }
-
-private:
-  // \brief Nested constructor for blocks within the bitcode buffer.
-  //
-  // \param BlockID The identifying constant associated with the block.
-  // \param EnclosingParser The bitcode parser parsing the enclosing block.
-  BitcodeParser(unsigned BlockID, BitcodeParser *EnclosingParser)
-      : NaClBitcodeParser(BlockID, EnclosingParser),
-        State(EnclosingParser->State) {}
-
-  // The state of the bitcode parser.
-  BitcodeParseState* State;
-};
-
-BitcodeParseState::BitcodeParseState(BitcodeParser *Parser,
-                                     NaClBitcodeRecordList &Records)
-    : Records(Records), AbbrevListener(Parser) {}
-
-} // end of anonymous namespace
-
-void llvm::readNaClBitcodeRecordList(
-    NaClBitcodeRecordList &RecordList,
-    std::unique_ptr<MemoryBuffer> InputBuffer) {
-  if (InputBuffer->getBufferSize() % 4 != 0)
-    report_fatal_error(
-        "Bitcode stream must be a multiple of 4 bytes in length");
-
-  const unsigned char *BufPtr =
-      (const unsigned char *) InputBuffer->getBufferStart();
-  const unsigned char *EndBufPtr = BufPtr + InputBuffer->getBufferSize();
-
-  // Read header and verify it is good.
-  NaClBitcodeHeader Header;
-  if (Header.Read(BufPtr, EndBufPtr))
-    report_fatal_error("Invalid PNaCl bitcode header.\n");
-  if (!Header.IsSupported())
-    errs() << Header.Unsupported();
-  if (!Header.IsReadable())
-    report_fatal_error("Invalid PNaCl bitcode header.\n");
-
-  NaClBitstreamReader Reader(BufPtr, EndBufPtr, Header);
-  NaClBitstreamCursor Cursor(Reader);
-
-  // Parse the bitcode buffer.
-  BitcodeParser Parser(Cursor, RecordList);
-
-  while (!Cursor.AtEndOfStream()) {
-    if (Parser.Parse())
-      report_fatal_error("Malformed records founds, unable to continue");
-  }
-}
diff --git a/lib/Bitcode/NaCl/TestUtils/NaClBitcodeMungeUtils.cpp b/lib/Bitcode/NaCl/TestUtils/NaClBitcodeMungeUtils.cpp
deleted file mode 100644
index 36d532b5787..00000000000
--- a/lib/Bitcode/NaCl/TestUtils/NaClBitcodeMungeUtils.cpp
+++ /dev/null
@@ -1,313 +0,0 @@
-//===--- Bitcode/NaCl/TestUtils/NaClBitcodeMungeUtils.cpp - Munge Bitcode -===//
-//
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
-//
-//===----------------------------------------------------------------------===//
-//
-// Munge bitcode records utility class NaClMungedBitcode.
-//
-//===----------------------------------------------------------------------===//
-
-#include "llvm/Bitcode/NaCl/NaClBitcodeMungeUtils.h"
-
-#include "llvm/ADT/STLExtras.h"
-#include "llvm/Support/Format.h"
-
-using namespace llvm;
-
-namespace {
-
-// \brief Extracts an uint64_t value from the array of values.
-//
-// \param Values The array of values to extract from.
-// \param ValuesSize The length of Values.
-// \param Terminator Denotes the end of a bitcode record.
-// \param [in/out] Index The index within Values to extract the
-// integer.  Updates Index to point to the next value after
-// extraction.
-uint64_t readValue(const uint64_t Values[], size_t ValuesSize,
-                   uint64_t Terminator, size_t &Index) {
-  if (Index < ValuesSize && Values[Index] != Terminator)
-    return Values[Index++];
-  std::string Buffer;
-  raw_string_ostream StrBuf(Buffer);
-  StrBuf << "Value expected at index " << Index;
-  report_fatal_error(StrBuf.str());
-}
-
-// \brief Extracts value of Type from the array of values. Parameters
-// are the same as for readValue.
-template <class Type>
-Type readAsType(const uint64_t Values[], size_t ValuesSize, uint64_t Terminator,
-                size_t &Index) {
-  uint64_t Value = readValue(Values, ValuesSize, Terminator, Index);
-  Type ValueAsType = static_cast<Type>(Value);
-  if (Value == ValueAsType)
-    return ValueAsType;
-  std::string Buffer;
-  raw_string_ostream StrBuf(Buffer);
-  StrBuf << "Out of range value " << Value << " at index " << (Index - 1);
-  report_fatal_error(StrBuf.str());
-}
-
-// \brief Extracts an edit action from the array of values. Parameters
-// are the same as for readValue.
-NaClMungedBitcode::EditAction readEditAction(const uint64_t Values[],
-                                             size_t ValuesSize,
-                                             uint64_t Terminator,
-                                             size_t &Index) {
-  uint64_t Value = readValue(Values, ValuesSize, Terminator, Index);
-  if (Value <= NaClMungedBitcode::Replace)
-    return static_cast<NaClMungedBitcode::EditAction>(Value);
-  std::string Buffer;
-  raw_string_ostream StrBuf(Buffer);
-  StrBuf << "Edit action expected at index " << Index << ". Found: " << Value;
-  report_fatal_error(StrBuf.str());
-}
-
-} // end of anonymous namespace
-
-namespace llvm {
-
-void readNaClBitcodeRecordList(NaClBitcodeRecordList &RecordList,
-                               const uint64_t Records[],
-                               size_t RecordsSize,
-                               uint64_t RecordTerminator) {
-  for (size_t Index = 0; Index < RecordsSize;) {
-    std::unique_ptr<NaClBitcodeAbbrevRecord>
-        Rcd(new NaClBitcodeAbbrevRecord());
-    Rcd->read(Records, RecordsSize, RecordTerminator, Index);
-    RecordList.push_back(std::move(Rcd));
-  }
-}
-
-} // end of namespace llvm
-
-void NaClBitcodeAbbrevRecord::print(raw_ostream &Out) const {
-  NaClBitcodeRecordData::Print(Out << Abbrev << ": ");
-}
-
-void NaClBitcodeAbbrevRecord::read(const uint64_t Vals[], size_t ValsSize,
-                                   uint64_t Terminator, size_t &Index) {
-  Abbrev = readAsType<unsigned>(Vals, ValsSize, Terminator, Index);
-  Code = readAsType<unsigned>(Vals, ValsSize, Terminator, Index);
-  Values.clear();
-  while (Index < ValsSize) {
-    uint64_t Value = Vals[Index++];
-    if (Value == Terminator)
-      break;
-    Values.push_back(Value);
-  }
-}
-
-NaClMungedBitcode::NaClMungedBitcode(const uint64_t Records[],
-                                     size_t RecordsSize,
-                                     uint64_t RecordTerminator)
-    : BaseRecords(new NaClBitcodeRecordList()) {
-  readNaClBitcodeRecordList(*BaseRecords, Records, RecordsSize,
-                            RecordTerminator);
-}
-
-void NaClMungedBitcode::print(raw_ostream &Out) const {
-  size_t Indent = 0;
-  for (const NaClBitcodeAbbrevRecord &Record : *this) {
-    if (Indent && Record.Code == naclbitc::BLK_CODE_EXIT)
-      --Indent;
-    for (size_t i = 0; i < Indent; ++i) {
-      Out << "  ";
-    }
-    // Blank fill to make abbreviation indices right align, and then
-    // print record.
-    uint32_t Cutoff = 9999999;
-    while (Record.Abbrev <= Cutoff && Cutoff) {
-      Out << " ";
-      Cutoff /= 10;
-    }
-    Out << Record << "\n";
-    if (Record.Code == naclbitc::BLK_CODE_ENTER)
-      ++Indent;
-  }
-}
-
-NaClMungedBitcode::~NaClMungedBitcode() {
-  removeEdits();
-}
-
-void NaClMungedBitcode::addBefore(size_t RecordIndex,
-                                  NaClBitcodeAbbrevRecord &Record) {
-  assert(RecordIndex < BaseRecords->size());
-  at(BeforeInsertionsMap, RecordIndex).push_back(copy(Record));
-}
-
-void NaClMungedBitcode::addAfter(size_t RecordIndex,
-                                 NaClBitcodeAbbrevRecord &Record) {
-  assert(RecordIndex < BaseRecords->size());
-  at(AfterInsertionsMap, RecordIndex).push_back(copy(Record));
-}
-
-void NaClMungedBitcode::remove(size_t RecordIndex) {
-  assert(RecordIndex < BaseRecords->size());
-  NaClBitcodeAbbrevRecord *&RcdPtr = ReplaceMap[RecordIndex];
-  if (RcdPtr != nullptr)
-    delete RcdPtr;
-  RcdPtr = nullptr;
-}
-
-void NaClMungedBitcode::replace(size_t RecordIndex,
-                                NaClBitcodeAbbrevRecord &Record) {
-  assert(RecordIndex < BaseRecords->size());
-  NaClBitcodeAbbrevRecord *&RcdPtr = ReplaceMap[RecordIndex];
-  if (RcdPtr != nullptr)
-    delete RcdPtr;
-  RcdPtr = copy(Record);
-}
-
-NaClBitcodeAbbrevRecord *
-NaClMungedBitcode::copy(const NaClBitcodeAbbrevRecord &Record) {
-  return new NaClBitcodeAbbrevRecord(Record);
-}
-
-void NaClMungedBitcode::destroyInsertionsMap(
-    NaClMungedBitcode::InsertionsMapType &Map) {
-  for (auto Pair : Map) {
-    DeleteContainerPointers(*Pair.second);
-    delete Pair.second;
-  }
-  Map.clear();
-}
-
-void NaClMungedBitcode::removeEdits() {
-  destroyInsertionsMap(BeforeInsertionsMap);
-  destroyInsertionsMap(AfterInsertionsMap);
-  DeleteContainerSeconds(ReplaceMap);
-  ReplaceMap.clear();
-}
-
-void NaClMungedBitcode::munge(const uint64_t Munges[], size_t MungesSize,
-                              uint64_t Terminator) {
-  for (size_t Index = 0; Index < MungesSize;) {
-    size_t RecordIndex =
-        readAsType<size_t>(Munges, MungesSize, Terminator, Index);
-    if (RecordIndex >= BaseRecords->size()) {
-      std::string Buffer;
-      raw_string_ostream StrBuf(Buffer);
-      StrBuf << "Record index " << RecordIndex << " out of range. "
-             << "Must be less than " << BaseRecords->size() << "\n";
-      report_fatal_error(StrBuf.str());
-    }
-    switch (readEditAction(Munges, MungesSize, Terminator, Index)) {
-    case NaClMungedBitcode::AddBefore: {
-      NaClBitcodeAbbrevRecord Record;
-      Record.read(Munges, MungesSize, Terminator, Index);
-      addBefore(RecordIndex, Record);
-      break;
-    }
-    case NaClMungedBitcode::AddAfter: {
-      NaClBitcodeAbbrevRecord Record;
-      Record.read(Munges, MungesSize, Terminator, Index);
-      addAfter(RecordIndex, Record);
-      break;
-    }
-    case NaClMungedBitcode::Remove: {
-      remove(RecordIndex);
-      break;
-    }
-    case NaClMungedBitcode::Replace: {
-      NaClBitcodeAbbrevRecord Record;
-      Record.read(Munges, MungesSize, Terminator, Index);
-      replace(RecordIndex, Record);
-      break;
-    }
-    }
-  }
-}
-
-NaClMungedBitcodeIter NaClMungedBitcode::begin() const {
-  return NaClMungedBitcodeIter::begin(*this);
-}
-
-NaClMungedBitcodeIter NaClMungedBitcode::end() const {
-  return NaClMungedBitcodeIter::end(*this);
-}
-
-bool NaClMungedBitcodeIter::
-operator==(const NaClMungedBitcodeIter &Iter) const {
-  if (MungedBitcode != Iter.MungedBitcode || Index != Iter.Index ||
-      Position != Iter.Position)
-    return false;
-  // Deal with the fact that InsertionsIter is undefined when
-  // at the end of all records.
-  if (Index == MungedBitcode->BaseRecords->size())
-    return true;
-  return InsertionsIter == Iter.InsertionsIter;
-}
-
-NaClBitcodeAbbrevRecord &NaClMungedBitcodeIter::operator*() {
-  switch (Position) {
-  case InBeforeInsertions:
-  case InAfterInsertions:
-    assert(Index < MungedBitcode->BaseRecords->size() &&
-           InsertionsIter != InsertionsIterEnd);
-    return **InsertionsIter;
-  case AtIndex: {
-    NaClMungedBitcode::ReplaceMapType::const_iterator Pos =
-        MungedBitcode->ReplaceMap.find(Index);
-    if (Pos == MungedBitcode->ReplaceMap.end())
-      return *(*MungedBitcode->BaseRecords)[Index];
-    assert(Pos->second);
-    return *Pos->second;
-  }
-  }
-}
-
-NaClMungedBitcodeIter &NaClMungedBitcodeIter::operator++() {
-  switch (Position) {
-  case InBeforeInsertions:
-  case InAfterInsertions:
-    assert(Index < MungedBitcode->BaseRecords->size() &&
-           InsertionsIter != InsertionsIterEnd);
-    ++InsertionsIter;
-    break;
-  case AtIndex: {
-    Position = InAfterInsertions;
-    placeAt(MungedBitcode->AfterInsertionsMap, Index);
-    break;
-  }
-  }
-  updatePosition();
-  return *this;
-}
-
-void NaClMungedBitcodeIter::updatePosition() {
-  while (true) {
-    switch (Position) {
-    case InBeforeInsertions:
-      if (Index >= MungedBitcode->BaseRecords->size() ||
-          InsertionsIter != InsertionsIterEnd)
-        return;
-      Position = AtIndex;
-      break;
-    case AtIndex: {
-      NaClMungedBitcode::ReplaceMapType::const_iterator Pos =
-          MungedBitcode->ReplaceMap.find(Index);
-      // Stop looking if no replacement, or index has a replacement.
-      if (Pos == MungedBitcode->ReplaceMap.end() || Pos->second != nullptr)
-        return;
-      // Base element has been removed.
-      Position = InAfterInsertions;
-      placeAt(MungedBitcode->AfterInsertionsMap, Index);
-      break;
-    }
-    case InAfterInsertions:
-      if (InsertionsIter != InsertionsIterEnd)
-        return;
-      Position = InBeforeInsertions;
-      ++Index;
-      placeAt(MungedBitcode->BeforeInsertionsMap, Index);
-      break;
-    }
-  }
-}
diff --git a/lib/Bitcode/NaCl/TestUtils/NaClBitcodeMungeWriter.cpp b/lib/Bitcode/NaCl/TestUtils/NaClBitcodeMungeWriter.cpp
deleted file mode 100644
index 5b73be25a33..00000000000
--- a/lib/Bitcode/NaCl/TestUtils/NaClBitcodeMungeWriter.cpp
+++ /dev/null
@@ -1,631 +0,0 @@
-//===- NaClBitcodeMungeWriter.cpp - Write munged bitcode --------*- C++ -*-===//
-//
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
-//
-//===----------------------------------------------------------------------===//
-//
-// Implements method NaClMungedBitcode.write(), which writes out a munged
-// list of bitcode records using a bitstream writer.
-
-#include "llvm/Bitcode/NaCl/NaClBitcodeMungeUtils.h"
-
-#include "llvm/Bitcode/NaCl/NaClBitstreamWriter.h"
-#include "llvm/Bitcode/NaCl/NaClReaderWriter.h"
-#include "llvm/Support/raw_ostream.h"
-
-#include <set>
-
-using namespace llvm;
-
-namespace {
-
-// For debugging. When true, shows trace information of emitting
-// bitcode.
-const bool DebugEmit = false;
-
-// Description of current block scope
-struct BlockScope {
-  BlockScope(unsigned CurBlockID, size_t AbbrevIndexLimit)
-      : CurBlockID(CurBlockID), AbbrevIndexLimit(AbbrevIndexLimit),
-        OmittedAbbreviations(false) {}
-  unsigned CurBlockID;
-  // Defines the maximum value for abbreviation indices in block.
-  size_t AbbrevIndexLimit;
-  // Defines if an abbreviation definition was omitted (i.e. not
-  // written) from this block. Used to turn off writing further
-  // abbreviation definitions for this block.
-  bool OmittedAbbreviations;
-  void print(raw_ostream &Out) const {
-    Out << "BlockScope(ID=" << CurBlockID << ", AbbrevIndexLimit="
-        << AbbrevIndexLimit << "OmittedAbbreviations="
-        << OmittedAbbreviations << ")";
-  }
-};
-
-inline raw_ostream &operator<<(raw_ostream &Out, const BlockScope &Scope) {
-  Scope.print(Out);
-  return Out;
-}
-
-// State of bitcode writing.
-struct WriteState {
-  // The block ID associated with records not in any block.
-  static const unsigned UnknownWriteBlockID = UINT_MAX;
-  // The SetBID for the blockinfo block.
-  unsigned SetBID = UnknownWriteBlockID;
-  // The stack of scopes the writer is in.
-  SmallVector<BlockScope, 4> ScopeStack;
-  // The set of write flags to use.
-  const NaClMungedBitcode::WriteFlags &Flags;
-  // The results of the attempted write.
-  NaClMungedBitcode::WriteResults Results;
-  // The minimum number of bits allowed to be specified in a block.
-  const unsigned BlockMinBits;
-  // The set of block IDs for which abbreviation definitions have been
-  // omitted in the blockinfo block.
-  std::set<unsigned> BlocksWithOmittedAbbrevs;
-
-  WriteState(const NaClMungedBitcode::WriteFlags &Flags)
-      : Flags(Flags),
-        BlockMinBits(NaClBitsNeededForValue(naclbitc::DEFAULT_MAX_ABBREV)) {
-    BlockScope Scope(UnknownWriteBlockID, naclbitc::DEFAULT_MAX_ABBREV);
-    ScopeStack.push_back(Scope);
-  }
-
-  // Returns stream to print error message to.
-  raw_ostream &Error();
-
-  // Returns stream to print error message to, assuming that
-  // the error message can be repaired if Flags.TryToRecover is true.
-  raw_ostream &RecoverableError() {
-    if (Flags.getTryToRecover())
-      ++Results.NumRepairs;
-    return Error();
-  }
-
-  bool atOutermostScope() {
-    assert(!ScopeStack.empty());
-    return ScopeStack.size() == 1;
-  }
-
-  unsigned getCurWriteBlockID() const {
-    assert(!ScopeStack.empty());
-    return ScopeStack.back().CurBlockID;
-  }
-
-  unsigned getCurAbbrevIndexLimit() const {
-    assert(!ScopeStack.empty());
-    return ScopeStack.back().AbbrevIndexLimit;
-  }
-
-  // Returns whether any abbreviation definitions were not written to
-  // the bitcode buffer.
-  bool curBlockHasOmittedAbbreviations() const {
-    assert(!ScopeStack.empty());
-    return ScopeStack.back().OmittedAbbreviations
-        || BlocksWithOmittedAbbrevs.count(getCurWriteBlockID());
-  }
-
-  // Marks that an abbreviation definition is being omitted (i.e. not
-  // written) for the current block.
-  void markCurrentBlockWithOmittedAbbreviations() {
-    assert(!ScopeStack.empty());
-    ScopeStack.back().OmittedAbbreviations = true;
-    if (getCurWriteBlockID() == naclbitc::BLOCKINFO_BLOCK_ID)
-      BlocksWithOmittedAbbrevs.insert(SetBID);
-  }
-
-  // Returns true if abbreviation operand is legal. If not, logs
-  // a recoverable error message and returns false. Assumes that
-  // the caller does the actual error recovery if applicable.
-  bool verifyAbbrevOp(NaClBitCodeAbbrevOp::Encoding Encoding, uint64_t Value,
-                      const NaClBitcodeAbbrevRecord &Record);
-
-  // Converts the abbreviation record to the corresponding abbreviation.
-  // Returns nullptr if unable to build abbreviation.
-  NaClBitCodeAbbrev *buildAbbrev(const NaClBitcodeAbbrevRecord &Record);
-
-  // Gets the next value (based on Index) from the record values,
-  // assigns it to ExtractedValue, and returns true. Otherwise, logs a
-  // recoverable error message and returns false. Assumes that the
-  // caller does the actual error recovery if applicable.
-  bool LLVM_ATTRIBUTE_UNUSED_RESULT
-  nextAbbrevValue(uint64_t &ExtractedValue,
-                  const NaClBitcodeAbbrevRecord &Record, size_t &Index) {
-    if (Index >= Record.Values.size()) {
-      RecoverableError()
-          << "Malformed abbreviation found: " << Record << "\n";
-      return false;
-    }
-    ExtractedValue = Record.Values[Index++];
-    return true;
-  }
-
-  // Emits the given record to the bitcode file. Returns true if
-  // successful.
-  bool LLVM_ATTRIBUTE_UNUSED_RESULT
-  emitRecord(NaClBitstreamWriter &Writer,
-             const NaClBitcodeAbbrevRecord &Record);
-
-  // Enter the given block
-  bool LLVM_ATTRIBUTE_UNUSED_RESULT
-  enterBlock(NaClBitstreamWriter &Writer, uint64_t WriteBlockID,
-             uint64_t NumBits, const NaClBitcodeAbbrevRecord &Record);
-
-  // Exit current block and return to previous block. Silently recovers
-  // if at outermost scope.
-  bool LLVM_ATTRIBUTE_UNUSED_RESULT
-  exitBlock(NaClBitstreamWriter &Writer,
-            const NaClBitcodeAbbrevRecord *Record = nullptr) {
-    if (atOutermostScope())
-      return false;
-    Writer.ExitBlock();
-    ScopeStack.pop_back();
-    if (DebugEmit)
-      printScopeStack(errs());
-    return true;
-  }
-
-  void WriteRecord(NaClBitstreamWriter &Writer,
-                   const NaClBitcodeAbbrevRecord &Record,
-                   bool UsesDefaultAbbrev) {
-    if (UsesDefaultAbbrev)
-      Writer.EmitRecord(Record.Code, Record.Values);
-    else
-      Writer.EmitRecord(Record.Code, Record.Values, Record.Abbrev);
-  }
-
-  // Returns true if the abbreviation index defines an abbreviation
-  // that can be applied to the record.
-  bool LLVM_ATTRIBUTE_UNUSED_RESULT
-  canApplyAbbreviation(NaClBitstreamWriter &Writer,
-                       const NaClBitcodeAbbrevRecord &Record);
-
-  // Completes the write.
-  NaClMungedBitcode::WriteResults &finish(NaClBitstreamWriter &Writer,
-                                          bool RecoverSilently);
-
-  void printScopeStack(raw_ostream &Out) {
-    Out << "Scope Stack:\n";
-    for (auto &Scope : ScopeStack)
-      Out << "  " << Scope << "\n";
-  }
-};
-
-raw_ostream &WriteState::Error() {
-  ++Results.NumErrors;
-  raw_ostream &ErrStrm = Flags.getErrStream();
-  unsigned WriteBlockID = getCurWriteBlockID();
-  ErrStrm << "Error (Block ";
-  if (WriteBlockID == UnknownWriteBlockID)
-    ErrStrm << "unknown";
-  else
-    ErrStrm << WriteBlockID;
-  return ErrStrm <<  "): ";
-}
-
-bool WriteState::enterBlock(NaClBitstreamWriter &Writer, uint64_t WriteBlockID,
-                            uint64_t NumBits,
-                            const NaClBitcodeAbbrevRecord &Record) {
-  if (NumBits < BlockMinBits || NumBits > naclbitc::MaxAbbrevWidth) {
-    RecoverableError()
-        << "Block index bit limit " << NumBits << " invalid. Must be in ["
-        << BlockMinBits << ".." << naclbitc::MaxAbbrevWidth << "]: "
-        << Record << "\n";
-    if (!Flags.getTryToRecover())
-      return false;
-    NumBits = naclbitc::MaxAbbrevWidth;
-  }
-  if (WriteBlockID > UINT_MAX) {
-    RecoverableError() << "Block id must be <= " << UINT_MAX
-                       << ": " << Record << "\n";
-    if (!Flags.getTryToRecover())
-      return false;
-    WriteBlockID = UnknownWriteBlockID;
-  }
-
-  uint64_t MaxAbbrev = (static_cast<uint64_t>(1) << NumBits) - 1;
-  BlockScope Scope(WriteBlockID, MaxAbbrev);
-  ScopeStack.push_back(Scope);
-  if (DebugEmit)
-    printScopeStack(errs());
-  if (WriteBlockID == naclbitc::BLOCKINFO_BLOCK_ID) {
-    unsigned DefaultMaxBits =
-        NaClBitsNeededForValue(naclbitc::DEFAULT_MAX_ABBREV);
-    if (NumBits != DefaultMaxBits) {
-      RecoverableError()
-          << "Numbits entry for abbreviations in blockinfo block not "
-          << DefaultMaxBits << ". found " << NumBits <<
-          ": " << Record << "\n";
-      if (!Flags.getTryToRecover()) {
-        ScopeStack.pop_back();
-        if (DebugEmit)
-          printScopeStack(errs());
-        return false;
-      }
-    }
-    Writer.EnterBlockInfoBlock();
-  } else {
-    NaClBitcodeSelectorAbbrev CurCodeLen(MaxAbbrev);
-    Writer.EnterSubblock(WriteBlockID, CurCodeLen);
-  }
-  return true;
-}
-
-bool WriteState::canApplyAbbreviation(
-    NaClBitstreamWriter &Writer, const NaClBitcodeAbbrevRecord &Record) {
-  const NaClBitCodeAbbrev *Abbrev = Writer.getAbbreviation(Record.Abbrev);
-  if (Abbrev == nullptr)
-    return false;
-
-  // Merge record code into values and then match abbreviation.
-  NaClBitcodeValues Values(Record);
-  size_t ValueIndex = 0;
-  size_t ValuesSize = Values.size();
-  size_t AbbrevIndex = 0;
-  size_t AbbrevSize = Abbrev->getNumOperandInfos();
-  bool FoundArray = false;
-  while (ValueIndex < ValuesSize && AbbrevIndex < AbbrevSize) {
-    const NaClBitCodeAbbrevOp *Op = &Abbrev->getOperandInfo(AbbrevIndex++);
-    uint64_t Value = Values[ValueIndex++];
-    if (Op->getEncoding() == NaClBitCodeAbbrevOp::Array) {
-      if (AbbrevIndex + 1 != AbbrevSize)
-        return false;
-      Op = &Abbrev->getOperandInfo(AbbrevIndex);
-      --AbbrevIndex; // i.e. don't advance to next abbreviation op.
-      FoundArray = true;
-    }
-    switch (Op->getEncoding()) {
-    case NaClBitCodeAbbrevOp::Literal:
-      if (Value != Op->getValue())
-        return false;
-      continue;
-    case NaClBitCodeAbbrevOp::Fixed:
-      if (Value >= (static_cast<uint64_t>(1)
-                    << NaClBitstreamWriter::MaxEmitNumBits)
-          || NaClBitsNeededForValue(Value) > Op->getValue())
-        return false;
-      continue;
-    case NaClBitCodeAbbrevOp::VBR:
-      if (Op->getValue() < 2)
-        return false;
-      continue;
-    case NaClBitCodeAbbrevOp::Array:
-      llvm_unreachable("Array(Array) abbreviation is not legal!");
-    case NaClBitCodeAbbrevOp::Char6:
-      if (!Op->isChar6(Value))
-        return false;
-      continue;
-    }
-  }
-  return ValueIndex == ValuesSize && (FoundArray || AbbrevIndex == AbbrevSize);
-}
-
-NaClMungedBitcode::WriteResults &WriteState::finish(
-    NaClBitstreamWriter &Writer, bool RecoverSilently) {
-  // Be sure blocks are balanced.
-  while (!atOutermostScope()) {
-    if (!RecoverSilently)
-      RecoverableError() << "Missing close block.\n";
-    if (!exitBlock(Writer)) {
-      Error() << "Failed to add missing close block at end of file.\n";
-      break;
-    }
-  }
-
-  // Be sure that generated bitcode buffer is word aligned.
-  if (Writer.GetCurrentBitNo() % 4 * CHAR_BIT) {
-    if (!RecoverSilently)
-      RecoverableError() << "Written bitstream not word aligned\n";
-    // Force a repair so that the bitstream writer doesn't crash.
-    Writer.FlushToWord();
-  }
-  return Results;
-}
-
-bool WriteState::emitRecord(NaClBitstreamWriter &Writer,
-                            const NaClBitcodeAbbrevRecord &Record) {
-  size_t NumValues = Record.Values.size();
-  if (DebugEmit) {
-    errs() << "Emit " << Record.Abbrev << ": <" << Record.Code;
-    for (size_t i = 0; i < NumValues; ++i) {
-      errs() << ", " << Record.Values[i];
-    }
-    errs() << ">\n";
-  }
-
-  switch (Record.Code) {
-  case naclbitc::BLK_CODE_ENTER: {
-    if (getCurWriteBlockID() == naclbitc::BLOCKINFO_BLOCK_ID) {
-      RecoverableError() << "Can't nest blocks inside blockinfo block: "
-                         << Record << "\n";
-      return Flags.getTryToRecover();
-    }
-    uint64_t WriteBlockID = UnknownWriteBlockID;
-    uint64_t NumBits = naclbitc::MaxAbbrevWidth; // Default to safest value.
-    if (Record.Abbrev != naclbitc::ENTER_SUBBLOCK) {
-      RecoverableError()
-          << "Uses illegal abbreviation index in enter block record: "
-          << Record << "\n";
-      if (!Flags.getTryToRecover())
-        return false;
-    }
-    if (NumValues != 2) {
-      RecoverableError()
-          << "Values for enter record should be of size 2, but found "
-          << NumValues << ": " << Record << "\n";
-      if (!Flags.getTryToRecover())
-        return false;
-    }
-    if (NumValues > 0)
-      WriteBlockID = Record.Values[0];
-    if (NumValues > 1)
-      NumBits = Record.Values[1];
-    return enterBlock(Writer, WriteBlockID, NumBits, Record);
-  }
-  case naclbitc::BLK_CODE_EXIT: {
-    if (atOutermostScope()) {
-      RecoverableError()
-          << "Extraneous exit block: " << Record << "\n";
-      if (!Flags.getTryToRecover())
-        return false;
-      break;
-    }
-    if (Record.Abbrev != naclbitc::END_BLOCK) {
-      RecoverableError()
-          << "Uses illegal abbreviation index in exit block record: "
-          << Record << "\n";
-      if (!Flags.getTryToRecover())
-        return false;
-    }
-    if (NumValues != 0) {
-      RecoverableError() << "Exit block should not have values: "
-                         << Record << "\n";
-      if (!Flags.getTryToRecover())
-        return false;
-    }
-    if (!exitBlock(Writer)) {
-      Error() << "Failed to write exit block, can't continue: "
-              << Record << "\n";
-      return false;
-    }
-    break;
-  }
-  case naclbitc::BLK_CODE_DEFINE_ABBREV: {
-    if (curBlockHasOmittedAbbreviations()) {
-      // If reached, a previous abbreviation for the block was omitted. Can't
-      // generate more abbreviations without having to fix abbreviation indices.
-      RecoverableError() << "Ignoring abbreviation: " << Record << "\n";
-      return Flags.getTryToRecover();
-    }
-    if (Record.Abbrev != naclbitc::DEFINE_ABBREV) {
-      RecoverableError()
-          << "Uses illegal abbreviation index in define abbreviation record: "
-          << Record << "\n";
-      if (!Flags.getTryToRecover())
-        return false;
-    }
-    NaClBitCodeAbbrev *Abbrev = buildAbbrev(Record);
-    if (Abbrev == nullptr) {
-      markCurrentBlockWithOmittedAbbreviations();
-      return Flags.getTryToRecover();
-    }
-    if (atOutermostScope()) {
-      RecoverableError() << "Abbreviation definition not in block: "
-                         << Record << "\n";
-      return Flags.getTryToRecover();
-    }
-    if (getCurWriteBlockID() == naclbitc::BLOCKINFO_BLOCK_ID) {
-      Writer.EmitBlockInfoAbbrev(SetBID, Abbrev);
-    } else {
-      Writer.EmitAbbrev(Abbrev);
-    }
-    break;
-  }
-  case naclbitc::BLK_CODE_HEADER:
-    // Note: There is no abbreviation index here. Ignore.
-    for (uint64_t Value : Record.Values)
-      Writer.Emit(Value, 8);
-    break;
-  default:
-    bool UsesDefaultAbbrev = Record.Abbrev == naclbitc::UNABBREV_RECORD;
-    if (atOutermostScope()) {
-      RecoverableError() << "Record outside block: " << Record << "\n";
-      if (!Flags.getTryToRecover())
-        return false;
-      // Create a dummy block to hold record.
-      if (!enterBlock(Writer, UnknownWriteBlockID,
-                      naclbitc::DEFAULT_MAX_ABBREV, Record)) {
-        Error() << "Failed to recover from record outside block\n";
-        return false;
-      }
-      UsesDefaultAbbrev = true;
-    }
-    if (getCurWriteBlockID() == naclbitc::BLOCKINFO_BLOCK_ID) {
-      // Note: only abbreviations and setBID records can appear in
-      // blockinfo blocks.
-      if (Record.Code != naclbitc::BLOCKINFO_CODE_SETBID) {
-        RecoverableError() << "Record not allowed in blockinfo block: "
-                           << Record << "\n";
-        return Flags.getTryToRecover();
-      }
-      // Note: SetBID records are handled by Writer->EmitBlockInfoAbbrev,
-      // based on the SetBID value. Don't bother to generate SetBID record here.
-      // Rather just set SetBID and let call to Writer->EmitBlockInfoAbbrev
-      // generate the SetBID record.
-      if (NumValues != 1) {
-        RecoverableError() << "SetBID record expects 1 value but found "
-                           << NumValues << ": " << Record << "\n";
-        if (!Flags.getTryToRecover())
-          return false;
-        SetBID = NumValues > 0 ? Record.Values[0] : UnknownWriteBlockID;
-        return true;
-      }
-      SetBID = Record.Values[0];
-      return true;
-    }
-    if (!UsesDefaultAbbrev && !canApplyAbbreviation(Writer, Record)) {
-      if (Writer.getAbbreviation(Record.Abbrev) != nullptr) {
-        RecoverableError() << "Abbreviation doesn't apply to record: "
-                           << Record << "\n";
-        UsesDefaultAbbrev = true;
-        if (!Flags.getTryToRecover())
-          return false;
-        WriteRecord(Writer, Record, UsesDefaultAbbrev);
-        return true;
-      }
-      if (Flags.getWriteBadAbbrevIndex()) {
-        // The abbreviation is not understood by the bitcode writer,
-        // and the flag value implies that we should still write it
-        // out so that unit tests for this error condition can be
-        // applied.
-        Error() << "Uses illegal abbreviation index: " << Record << "\n";
-        // Generate bad abbreviation index so that the bitcode reader
-        // can be tested, and then quit.
-        Results.WroteBadAbbrevIndex = true;
-        Writer.EmitCode(Record.Abbrev);
-        bool RecoverSilently = true;
-        finish(Writer, RecoverSilently);
-        return false;
-      }
-      RecoverableError() << "Uses illegal abbreviation index: "
-                         << Record << "\n";
-      UsesDefaultAbbrev = true;
-      if (!Flags.getTryToRecover())
-        return false;
-      WriteRecord(Writer, Record, UsesDefaultAbbrev);
-      return true;
-    }
-    WriteRecord(Writer, Record, UsesDefaultAbbrev);
-    break;
-  }
-  return true;
-}
-
-static NaClBitCodeAbbrev *deleteAbbrev(NaClBitCodeAbbrev *Abbrev) {
-  Abbrev->dropRef();
-  return nullptr;
-}
-
-bool WriteState::verifyAbbrevOp(NaClBitCodeAbbrevOp::Encoding Encoding,
-                                uint64_t Value,
-                                const NaClBitcodeAbbrevRecord &Record) {
-  if (NaClBitCodeAbbrevOp::isValid(Encoding, Value))
-    return true;
-  RecoverableError() << "Invalid abbreviation "
-                     << NaClBitCodeAbbrevOp::getEncodingName(Encoding)
-                     << "(" << static_cast<int64_t>(Value)
-                     << ") in: " << Record << "\n";
-  return false;
-}
-
-NaClBitCodeAbbrev *WriteState::buildAbbrev(
-    const NaClBitcodeAbbrevRecord &Record) {
-  // Note: Recover by removing abbreviation.
-  NaClBitCodeAbbrev *Abbrev = new NaClBitCodeAbbrev();
-  size_t Index = 0;
-  uint64_t NumAbbrevOps;
-  if (!nextAbbrevValue(NumAbbrevOps, Record, Index))
-    return deleteAbbrev(Abbrev);
-  if (NumAbbrevOps == 0) {
-    RecoverableError() << "Abbreviation must contain at least one operator: "
-                       << Record << "\n";
-    return deleteAbbrev(Abbrev);
-  }
-  for (uint64_t Count = 0; Count < NumAbbrevOps; ++Count) {
-    uint64_t IsLiteral;
-    if (!nextAbbrevValue(IsLiteral, Record, Index))
-      return deleteAbbrev(Abbrev);
-    switch (IsLiteral) {
-    case 1: {
-      uint64_t Value;
-      if (!nextAbbrevValue(Value, Record, Index))
-        return deleteAbbrev(Abbrev);
-      if (!verifyAbbrevOp(NaClBitCodeAbbrevOp::Literal, Value, Record))
-        return deleteAbbrev(Abbrev);
-      Abbrev->Add(NaClBitCodeAbbrevOp(Value));
-      break;
-    }
-    case 0: {
-      uint64_t Kind;
-      if (!nextAbbrevValue(Kind, Record, Index))
-        return deleteAbbrev(Abbrev);
-      switch (Kind) {
-      case NaClBitCodeAbbrevOp::Fixed: {
-        uint64_t Value;
-        if (!nextAbbrevValue(Value, Record, Index))
-          return deleteAbbrev(Abbrev);
-        if (!verifyAbbrevOp(NaClBitCodeAbbrevOp::Fixed, Value, Record))
-          return deleteAbbrev(Abbrev);
-        Abbrev->Add(NaClBitCodeAbbrevOp(NaClBitCodeAbbrevOp::Fixed, Value));
-        break;
-      }
-      case NaClBitCodeAbbrevOp::VBR: {
-        uint64_t Value;
-        if (!nextAbbrevValue(Value, Record, Index))
-          return deleteAbbrev(Abbrev);
-        if (!verifyAbbrevOp(NaClBitCodeAbbrevOp::VBR, Value, Record))
-          return deleteAbbrev(Abbrev);
-        Abbrev->Add(NaClBitCodeAbbrevOp(NaClBitCodeAbbrevOp::VBR, Value));
-        break;
-      }
-      case NaClBitCodeAbbrevOp::Array:
-        if (Count + 2 != NumAbbrevOps) {
-          RecoverableError() << "Array abbreviation must be second to last: "
-                             << Record << "\n";
-          return deleteAbbrev(Abbrev);
-        }
-        Abbrev->Add(NaClBitCodeAbbrevOp(NaClBitCodeAbbrevOp::Array));
-        break;
-      case NaClBitCodeAbbrevOp::Char6:
-        Abbrev->Add(NaClBitCodeAbbrevOp(NaClBitCodeAbbrevOp::Char6));
-        break;
-      default:
-        RecoverableError() << "Unknown abbreviation kind " << Kind
-                           << ": " << Record << "\n";
-        return deleteAbbrev(Abbrev);
-      }
-      break;
-    }
-    default:
-      RecoverableError() << "Bad abbreviation operand encoding "
-                         << Record.Values[Index-1] << ": " << Record << "\n";
-      return deleteAbbrev(Abbrev);
-    }
-  }
-  if (Index != Record.Values.size()) {
-    RecoverableError() << "Error: Too many values for number of operands ("
-                       << NumAbbrevOps << "): "
-                       << Record << "\n";
-    return deleteAbbrev(Abbrev);
-  }
-  if (!Abbrev->isValid()) {
-    raw_ostream &Out = RecoverableError();
-    Abbrev->Print(Out << "Abbreviation ");
-    Out << " not valid: " << Record << "\n";
-    return deleteAbbrev(Abbrev);
-  }
-  return Abbrev;
-}
-
-} // end of anonymous namespace.
-
-NaClMungedBitcode::WriteResults NaClMungedBitcode::writeMaybeRepair(
-    SmallVectorImpl<char> &Buffer, bool AddHeader,
-    const WriteFlags &Flags) const {
-  NaClBitstreamWriter Writer(Buffer);
-  WriteState State(Flags);
-  if (AddHeader) {
-    NaClWriteHeader(Writer, true);
-  }
-  for (const NaClBitcodeAbbrevRecord &Record : *this) {
-    if (!State.emitRecord(Writer, Record))
-      break;
-  }
-  bool RecoverSilently =
-      State.Results.NumErrors > 0 && !Flags.getTryToRecover();
-  return State.finish(Writer, RecoverSilently);
-}
diff --git a/lib/Bitcode/NaCl/TestUtils/NaClBitcodeTextReader.cpp b/lib/Bitcode/NaCl/TestUtils/NaClBitcodeTextReader.cpp
deleted file mode 100644
index 720e89455ce..00000000000
--- a/lib/Bitcode/NaCl/TestUtils/NaClBitcodeTextReader.cpp
+++ /dev/null
@@ -1,291 +0,0 @@
-//===- NaClBitcodeTextReader.cpp - Read texual bitcode record list --------===//
-//
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
-//
-//===----------------------------------------------------------------------===//
-//
-// Implements bitcode reader from textual form.
-//
-// A Textual bitcode file is a sequence of textual bitcode records.
-// A Textual bitcode record is a sequence of integers, separated by
-// commas, and terminated with a semicolon followed by a newline.
-//
-// In addition, unlike the binary form of bitcode, the input has no
-// bitcode header record.
-// ===---------------------------------------------------------------------===//
-
-#include "llvm/ADT/STLExtras.h"
-#include "llvm/Bitcode/NaCl/NaClBitcodeMungeUtils.h"
-#include "llvm/Bitcode/NaCl/NaClReaderWriter.h"
-
-#include <cstring>
-
-namespace llvm {
-class Module;
-} // end of namespace llvm
-
-using namespace llvm;
-
-namespace {
-
-// Defines a text reader error code.
-enum ReaderErrorType {
-  NoCodeForRecord=1,  // Note: Error types must not be zero!
-  NoValueAfterSeparator,
-  NoSeparatorOrTerminator,
-  NoNewlineAfterTerminator,
-  BitcodeHeaderNotAllowed,
-  NoAbbreviationsAllowed,
-  UnableToWriteBitcode
-};
-
-// Defines the corresponding error messages.
-class ReaderErrorCategoryType : public std::error_category {
-  ReaderErrorCategoryType(const ReaderErrorCategoryType&) = delete;
-  void operator=(const ReaderErrorCategoryType&) = delete;
-public:
-  static const ReaderErrorCategoryType &get() {
-    return Sentinel;
-  }
-private:
-  static const ReaderErrorCategoryType Sentinel;
-  ReaderErrorCategoryType() {}
-  const char *name() const LLVM_NOEXCEPT override {
-    return "pnacl.text_bitcode";
-  }
-  std::string message(int IndexError) const override {
-    switch(static_cast<ReaderErrorType>(IndexError)) {
-    case NoCodeForRecord:
-      return "Bitcode record doesn't begin with a record code";
-    case NoValueAfterSeparator:
-      return "Value expected after separator, but not found";
-    case NoSeparatorOrTerminator:
-      return "Separator/terminator expected after value";
-    case NoNewlineAfterTerminator:
-      return "Newline expecded after terminating semicolon";
-    case BitcodeHeaderNotAllowed:
-      return "Bitcode headers not allowed in bitcode text";
-    case NoAbbreviationsAllowed:
-      return "Bitcode abbreviations not allowed in bitcode text";
-    case UnableToWriteBitcode:
-      return "Unable to generate bitcode buffer from textual bitcode records";
-    }
-    llvm_unreachable("Unknown error type!");
-  }
-  ~ReaderErrorCategoryType() override = default;
-};
-
-const ReaderErrorCategoryType ReaderErrorCategoryType::Sentinel;
-
-/// Parses text bitcode records, and adds them to an existing list of
-/// bitcode records.
-class TextRecordParser {
-  TextRecordParser(const TextRecordParser&) = delete;
-  void operator=(const TextRecordParser&) = delete;
-public:
-  /// Creates a parser to parse records from the InputBuffer, and
-  /// append them to the list of Records.
-  TextRecordParser(NaClBitcodeRecordList &Records,
-                   MemoryBuffer *InputBuffer)
-      : Records(Records), Buffer(InputBuffer->getBuffer()) {}
-
-  /// Reads in the list of bitcode records in the input buffer.
-  std::error_code read();
-
-private:
-  // The list of bitcode records to generate.
-  NaClBitcodeRecordList &Records;
-  // The input buffer to parse.
-  StringRef Buffer;
-  // The current location within the input buffer.
-  size_t Cursor = 0;
-  // The separator character.
-  static const char *Separator;
-  // The terminator character.
-  static const char *Terminator;
-  // The newline character that must follow a terminator.
-  static const char *Newline;
-  // Valid digits that can be used to define numbers.
-  static const char *Digits;
-
-  // Returns true if we have reached the end of the input buffer.
-  bool atEof() const {
-    return Cursor == Buffer.size();
-  }
-
-  // Tries to read a character in the given set of characters. Returns
-  // the character found, or 0 if not found.
-  char readChar(const char *Chars);
-
-  // Tries to read a (integral) number. If successful, Value is set to
-  // the parsed number and returns true. Otherwise false is returned.
-  // Does not check for number overflow.
-  bool readNumber(uint64_t &Value);
-
-  // Reads a record from the input buffer.
-  std::error_code readRecord();
-};
-
-const char *TextRecordParser::Newline = "\n";
-const char *TextRecordParser::Separator = ",";
-const char *TextRecordParser::Terminator = ";";
-const char *TextRecordParser::Digits = "0123456789";
-
-std::error_code TextRecordParser::read() {
-  while (!atEof()) {
-    if (std::error_code EC = readRecord())
-      return EC;
-  }
-  return std::error_code();
-}
-
-char TextRecordParser::readChar(const char *Chars) {
-  if (atEof())
-    return 0;
-  char Ch = Buffer[Cursor];
-  if (std::strchr(Chars, Ch) == 0)
-    return 0;
-  ++Cursor;
-  return Ch;
-}
-
-bool TextRecordParser::readNumber(uint64_t &Value) {
-  Value = 0;
-  bool NumberFound = false;
-  while (1) {
-    char Ch = readChar(Digits);
-    if (!Ch)
-      return NumberFound;
-    Value = (Value * 10) + (Ch - '0');
-    NumberFound = true;
-  }
-}
-
-std::error_code TextRecordParser::readRecord() {
-  // States of parser used to parse bitcode records.
-  enum ParseState {
-    // Begin parsing a record.
-    StartParse,
-    // Before a value in the record.
-    BeforeValue,
-    // Immediately after a value in the record.
-    AfterValue,
-    // Completed parsing a record.
-    FinishParse
-  } State = StartParse;
-  unsigned Code = 0;
-  NaClRecordVector Values;
-  uint64_t Number = 0;
-  while (1) {
-    switch (State) {
-      case StartParse:
-        if (!readNumber(Number)) {
-          if (atEof())
-            return std::error_code();
-          return std::error_code(NoCodeForRecord,
-                                 ReaderErrorCategoryType::get());
-        }
-        Code = Number;
-        State = AfterValue;
-        continue;
-      case BeforeValue:
-        if (!readNumber(Number))
-          return std::error_code(NoValueAfterSeparator,
-                                 ReaderErrorCategoryType::get());
-        Values.push_back(Number);
-        State = AfterValue;
-        continue;
-      case AfterValue:
-        if (readChar(Separator)) {
-          State = BeforeValue;
-          continue;
-        }
-        if (readChar(Terminator)) {
-          State = FinishParse;
-          if (!readChar(Newline))
-            return std::error_code(NoNewlineAfterTerminator,
-                                   ReaderErrorCategoryType::get());
-          continue;
-        }
-        return std::error_code(NoSeparatorOrTerminator,
-                               ReaderErrorCategoryType::get());
-      case FinishParse: {
-        unsigned Abbrev = naclbitc::UNABBREV_RECORD;
-        switch (Code) {
-        case naclbitc::BLK_CODE_ENTER:
-          Abbrev = naclbitc::ENTER_SUBBLOCK;
-          break;
-        case naclbitc::BLK_CODE_EXIT:
-          Abbrev = naclbitc::END_BLOCK;
-          break;
-        case naclbitc::BLK_CODE_HEADER:
-          return std::error_code(BitcodeHeaderNotAllowed,
-                                 ReaderErrorCategoryType::get());
-        case naclbitc::BLK_CODE_DEFINE_ABBREV:
-          return std::error_code(NoAbbreviationsAllowed,
-                                 ReaderErrorCategoryType::get());
-        default: // All other records.
-          break;
-        }
-        Records.push_back(make_unique<NaClBitcodeAbbrevRecord>(
-            Abbrev, Code, Values));
-        return std::error_code();
-      }
-    }
-  }
-}
-
-} // end of anonymous namespace
-
-std::error_code llvm::readNaClRecordTextAndBuildBitcode(
-    StringRef Filename, SmallVectorImpl<char> &Buffer, raw_ostream *Verbose) {
-  // Open the input file with text records.
-  ErrorOr<std::unique_ptr<MemoryBuffer>>
-      MemBuf(MemoryBuffer::getFileOrSTDIN(Filename));
-  if (!MemBuf)
-    return MemBuf.getError();
-
-  // Read in the bitcode text records.
-  std::unique_ptr<NaClBitcodeRecordList> Records(new NaClBitcodeRecordList());
-  if (std::error_code EC =
-      readNaClTextBcRecordList(*Records, std::move(MemBuf.get())))
-    return EC;
-
-  // Write out the records into Buffer.
-  NaClMungedBitcode Bitcode(std::move(Records));
-  NaClMungedBitcode::WriteFlags Flags;
-  if (Verbose)
-    Flags.setErrStream(*Verbose);
-  bool AddHeader = true;
-  if (!Bitcode.write(Buffer, AddHeader)) {
-    return std::error_code(UnableToWriteBitcode,
-                           ReaderErrorCategoryType::get());
-  }
-  return std::error_code();
-}
-
-std::error_code llvm::readNaClTextBcRecordList(
-    NaClBitcodeRecordList &RecordList,
-    std::unique_ptr<MemoryBuffer> InputBuffer) {
-  TextRecordParser Parser(RecordList, InputBuffer.get());
-  return Parser.read();
-}
-
-llvm::ErrorOr<llvm::Module *>
-llvm::parseNaClBitcodeText(const std::string &Filename, LLVMContext &Context,
-                           raw_ostream *Verbose) {
-  SmallVector<char, 1024> Buffer;
-
-  // Fill Buffer with corresponding bitcode records from Filename.
-  if (std::error_code EC =
-      readNaClRecordTextAndBuildBitcode(Filename, Buffer, Verbose))
-    return EC;
-
-  // Parse buffer as ordinary binary bitcode file.
-  StringRef BitcodeBuffer(Buffer.data(),  Buffer.size());
-  MemoryBufferRef MemBufRef(BitcodeBuffer, Filename);
-  return NaClParseBitcodeFile(MemBufRef, Context, Verbose);
-}
diff --git a/lib/Bitcode/NaCl/TestUtils/NaClBitcodeTextWriter.cpp b/lib/Bitcode/NaCl/TestUtils/NaClBitcodeTextWriter.cpp
deleted file mode 100644
index 2a9a46ebfbf..00000000000
--- a/lib/Bitcode/NaCl/TestUtils/NaClBitcodeTextWriter.cpp
+++ /dev/null
@@ -1,143 +0,0 @@
-//===- NaClBitcodeTextWriter.cpp - Write textual bitcode ------------------===//
-//
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
-//
-//===----------------------------------------------------------------------===//
-//
-// Implements writeNaClBitcodeRecordList(), which writes out bitcode records
-// as text.
-//
-// Note that textual bitcode records do not contain a header,
-// abbreviations, or a blockinfo block. Records are defined as a
-// sequence of integers, separated by commas, and terminated with a
-// semicolon.
-//
-// For readability, a newline is added after each record.
-//===----------------------------------------------------------------------===//
-
-#include "llvm/Bitcode/NaCl/NaClBitcodeMungeUtils.h"
-
-using namespace llvm;
-
-namespace {
-
-class TextWriter {
-  TextWriter(const TextWriter&) = delete;
-  void operator=(const TextWriter&) = delete;
-public:
-  TextWriter(SmallVectorImpl<char> &Buffer, raw_ostream &ErrStream)
-      : Buffer(Buffer), ErrStream(ErrStream), ValueStream(ValueBuffer) {}
-
-  bool emitRecord(NaClBitcodeAbbrevRecord &Record);
-
-private:
-  // Buffer to write textual bitcode records into.
-  SmallVectorImpl<char> &Buffer;
-  // Stream for error messages.
-  raw_ostream &ErrStream;
-  // Selector for number of bits to use for abbreviations.
-  NaClBitcodeSelectorAbbrev DefaultAbbrevSelector;
-  // True iff in the blockinfo block.
-  bool InBlockInfoBlock = false;
-  // String stream to convert integers to strings.
-  std::string ValueBuffer;
-  raw_string_ostream ValueStream;
-
-  void writeValue(uint64_t Value);
-
-  void writeSeparator() {
-    Buffer.push_back(',');
-  }
-
-  void writeTerminator() {
-    Buffer.push_back(';');
-    Buffer.push_back('\n');
-  }
-};
-
-void TextWriter::writeValue(uint64_t Value) {
-  ValueStream << Value;
-  ValueStream.flush();
-  for (auto ch : ValueBuffer)
-    Buffer.push_back(ch);
-  ValueBuffer.clear();
-}
-
-bool TextWriter::emitRecord(NaClBitcodeAbbrevRecord &Record) {
-  size_t NumValues = Record.Values.size();
-  switch (Record.Code) {
-  case naclbitc::BLK_CODE_ENTER:
-    // Be careful to remove all records in the blockinfo block.
-    if (InBlockInfoBlock) {
-      ErrStream << "Blocks not allowed within the blockinfo block\n";
-      return false;
-    }
-    if (NumValues != 2) {
-      ErrStream << "Block enter doesn't contain 2 values: Found: "
-                << NumValues << "\n";
-      return false;
-    }
-    if (Record.Values[0] == naclbitc::BLOCKINFO_BLOCK_ID) {
-      InBlockInfoBlock = true;
-      return true;
-    }
-    writeValue(Record.Code);
-    writeSeparator();
-    writeValue(Record.Values[0]);
-    writeSeparator();
-    // Note: Since the textual form of the bitcode doesn't have
-    // abbreviations, we simplify the number of bits field
-    // (ie. Values[1] within the record) with the default bit width.
-    writeValue(DefaultAbbrevSelector.NumBits);
-    writeTerminator();
-    return true;
-  case naclbitc::BLK_CODE_EXIT:
-    if (InBlockInfoBlock) {
-      InBlockInfoBlock = false;
-      return true;
-    }
-    if (NumValues != 0) {
-      ErrStream << "Block exit shouldn't have any values. Found: "
-                << NumValues << "\n";
-    }
-    writeValue(Record.Code);
-    writeTerminator();
-    return true;
-  case naclbitc::BLK_CODE_DEFINE_ABBREV:
-  case naclbitc::BLK_CODE_HEADER:
-    // These records are skipped in textual bitcode.
-    return true;
-  default: {
-    // Don't write records within blockinfo blocks.
-    if (InBlockInfoBlock) {
-      if (Record.Code == naclbitc::BLOCKINFO_CODE_SETBID)
-        return true;
-      ErrStream << "Invalid record found in blockinfo block\n";
-      return false;
-    }
-    writeValue(Record.Code);
-    for (const auto Value : Record.Values) {
-      writeSeparator();
-      writeValue(Value);
-    }
-    writeTerminator();
-    return true;
-  }
-  }
-}
-
-} // end of anonymous namespace
-
-bool llvm::writeNaClBitcodeRecordList(NaClBitcodeRecordList &RecordList,
-                                      SmallVectorImpl<char> &Buffer,
-                                      raw_ostream &ErrStream) {
-  TextWriter Writer(Buffer, ErrStream);
-  for (const auto &Record : RecordList) {
-    if (!Writer.emitRecord(*Record))
-      return false;
-  }
-  return true;
-}
diff --git a/lib/Bitcode/NaCl/TestUtils/NaClFuzz.cpp b/lib/Bitcode/NaCl/TestUtils/NaClFuzz.cpp
deleted file mode 100644
index 0673736277a..00000000000
--- a/lib/Bitcode/NaCl/TestUtils/NaClFuzz.cpp
+++ /dev/null
@@ -1,53 +0,0 @@
-//===- NaClFuzz.cpp - Fuzz PNaCl bitcode records --------------------------===//
-//
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
-//
-//===----------------------------------------------------------------------===//
-//
-// This file implements a basic fuzzer for a list of PNaCl bitcode records.
-//
-//===----------------------------------------------------------------------===//
-
-#include "llvm/ADT/STLExtras.h"
-#include "llvm/Bitcode/NaCl/NaClFuzz.h"
-
-using namespace llvm;
-
-namespace {
-
-// Names for edit actions.
-const char *ActionNameArray[] = {
-  "Insert",
-  "Mutate",
-  "Remove",
-  "Replace",
-  "Swap"
-};
-
-} // end of anonymous namespace
-
-namespace naclfuzz {
-
-const char *RecordFuzzer::actionName(EditAction Action) {
-  return Action < array_lengthof(ActionNameArray)
-                  ? ActionNameArray[Action] : "???";
-}
-
-RecordFuzzer::RecordFuzzer(NaClMungedBitcode &Bitcode,
-                           RandomNumberGenerator &Generator)
-    : Bitcode(Bitcode), Generator(Generator) {
-  if (Bitcode.getBaseRecords().empty())
-    report_fatal_error(
-        "Sorry, the fuzzer doesn't know how to fuzz an empty record list");
-}
-
-RecordFuzzer::~RecordFuzzer() {}
-
-void RecordFuzzer::clear() {
-  Bitcode.removeEdits();
-}
-
-} // end of namespace naclfuzz
diff --git a/lib/Bitcode/NaCl/TestUtils/NaClRandNumGen.cpp b/lib/Bitcode/NaCl/TestUtils/NaClRandNumGen.cpp
deleted file mode 100644
index 5eb89d9fe78..00000000000
--- a/lib/Bitcode/NaCl/TestUtils/NaClRandNumGen.cpp
+++ /dev/null
@@ -1,44 +0,0 @@
-//===- NaClRandNumGen.cpp - Random number generator -----------------------===//
-//
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
-//
-//===----------------------------------------------------------------------===//
-//
-// This file provides a default implementation of a random number generator.
-//
-//===----------------------------------------------------------------------===//
-
-#include "llvm/Bitcode/NaCl/NaClRandNumGen.h"
-
-#include <vector>
-#include <algorithm>
-
-namespace naclfuzz {
-
-// Put destructor in .cpp file guarantee vtable construction.
-RandomNumberGenerator::~RandomNumberGenerator() {}
-
-DefaultRandomNumberGenerator::DefaultRandomNumberGenerator(llvm::StringRef Seed)
-    : Seed(Seed) {
-  saltSeed(0);
-}
-
-uint64_t DefaultRandomNumberGenerator::operator()() {
-  return Generator();
-}
-
-void DefaultRandomNumberGenerator::saltSeed(uint64_t Salt) {
-  // Combine seed and salt and pass to generator.
-  std::vector<uint32_t> Data;
-  Data.reserve(Seed.size() + 2);
-  Data.push_back(static_cast<uint32_t>(Salt));
-  Data.push_back(static_cast<uint32_t>(Salt >> 32));
-  std::copy(Seed.begin(), Seed.end(), Data.end());
-  std::seed_seq SeedSeq(Data.begin(), Data.end());
-  Generator.seed(SeedSeq);
-}
-
-} // end of namespace naclfuzz
diff --git a/lib/Bitcode/NaCl/TestUtils/NaClSimpleRecordFuzzer.cpp b/lib/Bitcode/NaCl/TestUtils/NaClSimpleRecordFuzzer.cpp
deleted file mode 100644
index 40557a5cff1..00000000000
--- a/lib/Bitcode/NaCl/TestUtils/NaClSimpleRecordFuzzer.cpp
+++ /dev/null
@@ -1,496 +0,0 @@
-//===- NaClSimpleRecordFuzzer.cpp - Simple fuzzer of bitcode --------------===//
-//
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
-//
-//===----------------------------------------------------------------------===//
-//
-// This file implements a simple fuzzer for a list of PNaCl bitcode records.
-//
-//===----------------------------------------------------------------------===//
-
-#include "llvm/ADT/STLExtras.h"
-#include "llvm/Bitcode/NaCl/NaClFuzz.h"
-#include "llvm/Support/Format.h"
-
-#include <set>
-
-using namespace llvm;
-using namespace naclfuzz;
-
-namespace {
-
-// Counts the number of times each value in a range [0..N) is used (based
-// on the number of calls to method increment()).
-class DistCounter {
-  DistCounter(const DistCounter&) = delete;
-  void operator=(const DistCounter&) = delete;
-public:
-  explicit DistCounter(size_t DistSize)
-      : Dist(DistSize, 0), Total(0) {}
-
-  // Increments the count for the given value
-  size_t increment(size_t Value) {
-    ++Dist[Value];
-    ++Total;
-    return Value;
-  }
-
-  // Returns the end of the range being checked.
-  size_t size() const {
-    return Dist.size();
-  }
-
-  // Returns the number of times value was incremented.
-  size_t operator[](size_t Value) const {
-    return Dist[Value];
-  }
-
-  // Retrns the number of times any value in the distribution was
-  // incremented.
-  size_t getTotal() const {
-    return Total;
-  }
-private:
-  // The number of times each value was incremented.
-  std::vector<size_t> Dist;
-  // The total number of times any value was incremented.
-  size_t Total;
-};
-
-// Model weights when randomly choosing values.
-typedef unsigned WeightType;
-
-// Associates weights with values. Used to generate weighted
-// distributions (see class WeightedDistribution below).
-template<typename T>
-struct WeightedValue {
-  T Value;
-  WeightType Weight;
-};
-
-// Weighted distribution for a set of values in [0..DistSize).
-template<typename T>
-class WeightedDistribution {
-  WeightedDistribution(const WeightedDistribution&) = delete;
-  void operator=(const WeightedDistribution&) = delete;
-public:
-  typedef const WeightedValue<T> *const_iterator;
-
-  WeightedDistribution(const WeightedValue<T> Dist[],
-                       size_t DistSize,
-                       RandomNumberGenerator &Generator)
-      : Dist(Dist), DistSize(DistSize), TotalWeight(0), Generator(Generator) {
-    for (size_t i = 0; i < DistSize; ++i)
-      TotalWeight += Dist[i].Weight;
-  }
-
-  virtual ~WeightedDistribution() {}
-
-  /// Returns the number of weighted values in the distribution.
-  size_t size() const {
-    return DistSize;
-  }
-
-  /// Returns const iterator at beginning of weighted distribution.
-  const_iterator begin() const {
-    return Dist;
-  }
-
-  /// Returns const iterator at end of weighted distribution.
-  const_iterator end() const {
-    return Dist + DistSize;
-  }
-
-  /// Randomly chooses a weighted value in the distribution, based on
-  /// the weights of the distrubution.
-  virtual const WeightedValue<T> &choose() {
-    return Dist[chooseIndex()];
-  }
-
-  /// Returns the total weight associated with the distribution.
-  size_t getTotalWeight() const {
-    return TotalWeight;
-  }
-
-protected:
-  // The weighted values defining the distribution.
-  const WeightedValue<T> *Dist;
-  // The number of weighed values.
-  size_t DistSize;
-  // The total weight of the distribution (sum of weights in weighted values).
-  size_t TotalWeight;
-  // The random number generator to use.
-  RandomNumberGenerator &Generator;
-
-  // Randomly chooses an index of a weighed value in the distribution,
-  // based on the weights of the distribution.
-  size_t chooseIndex() {
-    /// TODO(kschimpf) Speed this up?
-    WeightType WeightedSum = Generator.chooseInRange(TotalWeight);
-    assert(WeightedSum < TotalWeight);
-    for (size_t Choice = 0; Choice < DistSize; ++Choice) {
-      WeightType NextWeight = Dist[Choice].Weight;
-      if (WeightedSum < NextWeight)
-        return Choice;
-      WeightedSum -= NextWeight;
-    }
-    llvm_unreachable("no index for WeightedDistribution.chooseIndex()");
-  }
-};
-
-// Defines a range [min..max].
-template<typename T>
-struct RangeType {
-  T min;
-  T max;
-};
-
-// Weighted distribution of a set of value ranges.
-template<typename T>
-class WeightedRangeDistribution : public WeightedDistribution<RangeType<T>> {
-  WeightedRangeDistribution(const WeightedRangeDistribution<T>&) = delete;
-  void operator=(const WeightedRangeDistribution<T>&) = delete;
-public:
-  WeightedRangeDistribution(const WeightedValue<RangeType<T>> Dist[],
-                            size_t DistSize,
-                            RandomNumberGenerator &Generator)
-      : WeightedDistribution<RangeType<T>>(Dist, DistSize, Generator) {}
-
-  // Choose a value from one of the value ranges.
-  T chooseValue() {
-    const RangeType<T> Range = this->choose().Value;
-    return Range.min +
-        this->Generator.chooseInRange(Range.max - Range.min + 1);
-  }
-};
-
-/// Weighted distribution with a counter, capturing the distribution
-/// of random choices for each weighted value.
-template<typename T>
-class CountedWeightedDistribution : public WeightedDistribution<T> {
-  CountedWeightedDistribution(const CountedWeightedDistribution&) = delete;
-  void operator=(const CountedWeightedDistribution&) = delete;
- public:
-  CountedWeightedDistribution(const WeightedValue<T> Dist[],
-                              size_t DistSize,
-                              RandomNumberGenerator &Generator)
-      : WeightedDistribution<T>(Dist, DistSize, Generator), Counter(DistSize) {}
-
-  const WeightedValue<T> &choose() final {
-    return this->Dist[Counter.increment(
-        WeightedDistribution<T>::chooseIndex())];
-  }
-
-  /// Returns the number of times the Index-th weighted value was
-  /// chosen.
-  size_t getChooseCount(size_t Index) const {
-    return Counter[Index];
-  }
-
-  /// Returns the total number of times method choose was called.
-  size_t getTotalChooseCount() const {
-    return Counter.getTotal();
-  }
-
-private:
-  DistCounter Counter;
-};
-
-#define ARRAY(array) array, array_lengthof(array)
-
-// Weighted distribution used to select edit actions.
-const WeightedValue<RecordFuzzer::EditAction> ActionDist[] = {
-  {RecordFuzzer::InsertRecord, 3},
-  {RecordFuzzer::MutateRecord, 5},
-  {RecordFuzzer::RemoveRecord, 1},
-  {RecordFuzzer::ReplaceRecord, 1},
-  {RecordFuzzer::SwapRecord, 1}
-};
-
-// Type of values in bitcode records.
-typedef uint64_t ValueType;
-
-// Weighted ranges for non-negative values in records.
-const WeightedValue<RangeType<ValueType>> PosValueDist[] = {
-  {{0, 6}, 100},
-  {{7, 20}, 50},
-  {{21, 40}, 10},
-  {{41, 100}, 2},
-  {{101, 4096}, 1}
-};
-
-// Distribution used to decide when use negative values in records.
-WeightedValue<bool> NegValueDist[] = {
-  {true, 1},   // i.e. make value negative.
-  {false, 100} // i.e. leave value positive.
-};
-
-// Range distribution for records sizes (must be greater than 0).
-const WeightedValue<RangeType<size_t>> RecordSizeDist[] = {
-  {{1, 3}, 1000},
-  {{4, 7},  100},
-  {{7, 100},  1}
-};
-
-// Defines valid record codes.
-typedef unsigned RecordCodeType;
-
-// Special code to signify adding random other record codes.
-static const RecordCodeType OtherRecordCode = 575757575;
-
-// List of record codes we can generate. The weights are based
-// on record counts in pnacl-llc.pexe, using how many thousand of
-// each record code appeared (or 1 if less than 1 thousand).
-WeightedValue<RecordCodeType> RecordCodeDist[] = {
-  {naclbitc::BLOCKINFO_CODE_SETBID, 1},
-  {naclbitc::MODULE_CODE_VERSION, 1},
-  {naclbitc::MODULE_CODE_FUNCTION, 7},
-  {naclbitc::TYPE_CODE_NUMENTRY, 1},
-  {naclbitc::TYPE_CODE_VOID, 1},
-  {naclbitc::TYPE_CODE_FLOAT, 1},
-  {naclbitc::TYPE_CODE_DOUBLE, 1},
-  {naclbitc::TYPE_CODE_INTEGER, 1},
-  {naclbitc::TYPE_CODE_VECTOR, 1},
-  {naclbitc::TYPE_CODE_FUNCTION, 1},
-  {naclbitc::VST_CODE_ENTRY, 1},
-  {naclbitc::VST_CODE_BBENTRY, 1},
-  {naclbitc::CST_CODE_SETTYPE, 15},
-  {naclbitc::CST_CODE_UNDEF, 1},
-  {naclbitc::CST_CODE_INTEGER, 115},
-  {naclbitc::CST_CODE_FLOAT, 1},
-  {naclbitc::GLOBALVAR_VAR, 14},
-  {naclbitc::GLOBALVAR_COMPOUND, 1},
-  {naclbitc::GLOBALVAR_ZEROFILL, 2},
-  {naclbitc::GLOBALVAR_DATA, 18},
-  {naclbitc::GLOBALVAR_RELOC, 20},
-  {naclbitc::GLOBALVAR_COUNT, 1},
-  {naclbitc::FUNC_CODE_DECLAREBLOCKS, 6},
-  {naclbitc::FUNC_CODE_INST_BINOP, 402},
-  {naclbitc::FUNC_CODE_INST_CAST, 61},
-  {naclbitc::FUNC_CODE_INST_EXTRACTELT, 1},
-  {naclbitc::FUNC_CODE_INST_INSERTELT, 1},
-  {naclbitc::FUNC_CODE_INST_RET, 7},
-  {naclbitc::FUNC_CODE_INST_BR, 223},
-  {naclbitc::FUNC_CODE_INST_SWITCH, 7},
-  {naclbitc::FUNC_CODE_INST_UNREACHABLE, 1},
-  {naclbitc::FUNC_CODE_INST_PHI, 84},
-  {naclbitc::FUNC_CODE_INST_ALLOCA, 34},
-  {naclbitc::FUNC_CODE_INST_LOAD, 225},
-  {naclbitc::FUNC_CODE_INST_STORE, 461},
-  {naclbitc::FUNC_CODE_INST_CMP2, 140},
-  {naclbitc::FUNC_CODE_INST_VSELECT, 10},
-  {naclbitc::FUNC_CODE_INST_CALL, 80},
-  {naclbitc::FUNC_CODE_INST_FORWARDTYPEREF, 36},
-  {naclbitc::FUNC_CODE_INST_CALL_INDIRECT, 5},
-  {naclbitc::BLK_CODE_ENTER, 1},
-  {naclbitc::BLK_CODE_EXIT, 1},
-  {naclbitc::BLK_CODE_DEFINE_ABBREV, 1},
-  {OtherRecordCode, 1}
-};
-
-// *Warning* The current implementation does not work on empty bitcode
-// record lists.
-class SimpleRecordFuzzer : public RecordFuzzer {
-public:
-  SimpleRecordFuzzer(NaClMungedBitcode &Bitcode,
-                     RandomNumberGenerator &Generator)
-      : RecordFuzzer(Bitcode, Generator),
-        RecordCounter(Bitcode.getBaseRecords().size()),
-        ActionWeight(ARRAY(ActionDist), Generator),
-        RecordSizeWeight(ARRAY(RecordSizeDist), Generator),
-        PosValueWeight(ARRAY(PosValueDist), Generator),
-        NegValueWeight(ARRAY(NegValueDist), Generator),
-        RecordCodeWeight(ARRAY(RecordCodeDist), Generator) {
-
-    assert(!Bitcode.getBaseRecords().empty()
-           && "Can't fuzz empty list of records");
-
-    for (const auto &RecordCode : RecordCodeWeight)
-      UsedRecordCodes.insert(RecordCode.Value);
-  }
-
-  ~SimpleRecordFuzzer() final;
-
-  bool fuzz(unsigned Count, unsigned Base) final;
-
-  void showRecordDistribution(raw_ostream &Out) const final;
-
-  void showEditDistribution(raw_ostream &Out) const final;
-
-private:
-  // Count how many edits are applied to each record in the bitcode.
-  DistCounter RecordCounter;
-  // Distribution used to randomly choose edit actions.
-  CountedWeightedDistribution<RecordFuzzer::EditAction> ActionWeight;
-  // Distribution used to randomly choose the size of created records.
-  WeightedRangeDistribution<size_t> RecordSizeWeight;
-  // Distribution used to randomly choose positive values for records.
-  WeightedRangeDistribution<ValueType> PosValueWeight;
-  // Distribution of value sign used to randomly choose value for records.
-  WeightedDistribution<bool> NegValueWeight;
-  // Distribution used to choose record codes for records.
-  WeightedDistribution<RecordCodeType> RecordCodeWeight;
-  // Filter to make sure the "other" choice for record codes will not
-  // choose any other record code in RecordCodeWeight.
-  std::set<size_t> UsedRecordCodes;
-
-  // Randomly choose an edit action.
-  RecordFuzzer::EditAction chooseAction() {
-    return ActionWeight.choose().Value;
-  }
-
-  // Randomly choose a record index from the list of records to edit.
-  size_t chooseRecordIndex() {
-    return RecordCounter.increment(
-        Generator.chooseInRange(Bitcode.getBaseRecords().size()));
-  }
-
-  // Randomly choose a record code.
-  RecordCodeType chooseRecordCode() {
-    RecordCodeType Code = RecordCodeWeight.choose().Value;
-    if (Code != OtherRecordCode)
-      return Code;
-    Code = Generator.chooseInRange(UINT_MAX);
-    while (UsedRecordCodes.count(Code)) // don't use predefined values.
-      ++Code;
-    return Code;
-  }
-
-  // Randomly choose a positive value for use in a record.
-  ValueType choosePositiveValue() {
-    return PosValueWeight.chooseValue();
-  }
-
-  // Randomly choose a positive/negative value for use in a record.
-  ValueType chooseValue() {
-    ValueType Value = choosePositiveValue();
-    if (NegValueWeight.choose().Value) {
-      // Use two's complement negation.
-      Value = ~Value + 1;
-    }
-    return Value;
-  }
-
-  // Randomly fill in a record with record values.
-  void chooseRecordValues(NaClBitcodeAbbrevRecord &Record) {
-    Record.Abbrev = naclbitc::UNABBREV_RECORD;
-    Record.Code = chooseRecordCode();
-    Record.Values.clear();
-    size_t NumValues = RecordSizeWeight.chooseValue();
-    for (size_t i = 0; i < NumValues; ++i) {
-      Record.Values.push_back(chooseValue());
-    }
-  }
-
-  // Apply the given edit action to a random record.
-  void applyAction(EditAction Action);
-
-  // Randomly mutate a record.
-  void mutateRecord(NaClBitcodeAbbrevRecord &Record) {
-    // TODO(kschimpf) Do something smarter than just changing a value
-    // in the record.
-    size_t Index = Generator.chooseInRange(Record.Values.size() + 1);
-    if (Index == 0)
-      Record.Code = chooseRecordCode();
-    else
-      Record.Values[Index - 1] = chooseValue();
-  }
-};
-
-SimpleRecordFuzzer::~SimpleRecordFuzzer() {}
-
-bool SimpleRecordFuzzer::fuzz(unsigned Count, unsigned Base) {
-  // TODO(kschimpf): Add some randomness in the number of actions selected.
-  clear();
-  size_t NumRecords = Bitcode.getBaseRecords().size();
-  size_t NumActions = NumRecords * Count / Base;
-  if (NumActions == 0)
-    NumActions = 1;
-  for (size_t i = 0; i < NumActions; ++i)
-    applyAction(chooseAction());
-  return true;
-}
-
-void SimpleRecordFuzzer::applyAction(EditAction Action) {
-  size_t Index = chooseRecordIndex();
-  switch(Action) {
-  case InsertRecord: {
-    NaClBitcodeAbbrevRecord Record;
-    chooseRecordValues(Record);
-    if (Generator.chooseInRange(2))
-      Bitcode.addBefore(Index, Record);
-    else
-      Bitcode.addAfter(Index, Record);
-    return;
-  }
-  case RemoveRecord:
-    Bitcode.remove(Index);
-    return;
-  case ReplaceRecord: {
-    NaClBitcodeAbbrevRecord Record;
-    chooseRecordValues(Record);
-    Bitcode.replace(Index, Record);
-    return;
-  }
-  case MutateRecord: {
-    NaClBitcodeAbbrevRecord Copy(*Bitcode.getBaseRecords()[Index]);
-    mutateRecord(Copy);
-    Bitcode.replace(Index, Copy);
-    return;
-  }
-  case SwapRecord: {
-    size_t Index2 = chooseRecordIndex();
-    Bitcode.replace(Index, *Bitcode.getBaseRecords()[Index2]);
-    Bitcode.replace(Index2, *Bitcode.getBaseRecords()[Index]);
-    return;
-  }
-  }
-}
-
-// Returns corresponding percentage defined by Count/Total, in a form
-// that can be printed to a raw_ostream.
-format_object<float> percentage(size_t Count, size_t Total) {
-  float percent = Total == 0.0 ? 0.0 : 100.0 * Count / Total;
-  return format("%1.0f", nearbyintf(percent));
-}
-
-void SimpleRecordFuzzer::showRecordDistribution(raw_ostream &Out) const {
-  Out << "Edit Record Distribution (Total: " << RecordCounter.getTotal()
-      << "):\n";
-  size_t Total = RecordCounter.getTotal();
-  for (size_t i = 0; i < Bitcode.getBaseRecords().size(); ++i) {
-    size_t Count = RecordCounter[i];
-    Out << "  " << format("%zd", i) << ": "
-        << Count << " (" << percentage(Count, Total) << "%)\n";
-  }
-}
-
-void SimpleRecordFuzzer::showEditDistribution(raw_ostream &Out) const {
-  size_t TotalWeight = ActionWeight.getTotalWeight();
-  size_t TotalCount = ActionWeight.getTotalChooseCount();
-  Out << "Edit Action Distribution(Total: " << TotalCount << "):\n";
-  size_t ActionIndex = 0;
-  for (const auto &Action : ActionWeight) {
-    size_t ActionCount = ActionWeight.getChooseCount(ActionIndex);
-    Out << "  " << actionName(Action.Value) <<  " - Wanted: "
-        << percentage(Action.Weight, TotalWeight)  << "%, Applied: "
-        << ActionCount  << " (" << percentage(ActionCount, TotalCount)
-        << "%)\n";
-    ++ActionIndex;
-  }
-}
-
-} // end of anonymous namespace
-
-namespace naclfuzz {
-
-RecordFuzzer *RecordFuzzer::createSimpleRecordFuzzer(
-    NaClMungedBitcode &Bitcode,
-    RandomNumberGenerator &Generator) {
-  return new SimpleRecordFuzzer(Bitcode, Generator);
-}
-
-} // end of namespace naclfuzz
diff --git a/lib/Bitcode/NaCl/Writer/CMakeLists.txt b/lib/Bitcode/NaCl/Writer/CMakeLists.txt
deleted file mode 100644
index 42e63ac2486..00000000000
--- a/lib/Bitcode/NaCl/Writer/CMakeLists.txt
+++ /dev/null
@@ -1,6 +0,0 @@
-add_llvm_library(LLVMNaClBitWriter
-  NaClBitcodeWriter.cpp
-  NaClBitcodeWriterPass.cpp
-  NaClValueEnumerator.cpp
-  )
-add_dependencies(LLVMNaClBitWriter intrinsics_gen)
diff --git a/lib/Bitcode/NaCl/Writer/LLVMBuild.txt b/lib/Bitcode/NaCl/Writer/LLVMBuild.txt
deleted file mode 100644
index 4bcdfdc87e5..00000000000
--- a/lib/Bitcode/NaCl/Writer/LLVMBuild.txt
+++ /dev/null
@@ -1,22 +0,0 @@
-;===- ./lib/Bitcode/NaCl/Writer/LLVMBuild.txt -----------------*- Conf -*--===;
-;
-;                     The LLVM Compiler Infrastructure
-;
-; This file is distributed under the University of Illinois Open Source
-; License. See LICENSE.TXT for details.
-;
-;===------------------------------------------------------------------------===;
-;
-; This is an LLVMBuild description file for the components in this subdirectory.
-;
-; For more information on the LLVMBuild system, please see:
-;
-;   http://llvm.org/docs/LLVMBuild.html
-;
-;===------------------------------------------------------------------------===;
-
-[component_0]
-type = Library
-name = NaClBitWriter
-parent = NaClBitcode
-required_libraries = Core Support NaClBitReader
diff --git a/lib/Bitcode/NaCl/Writer/Makefile b/lib/Bitcode/NaCl/Writer/Makefile
deleted file mode 100644
index 60da2d1b716..00000000000
--- a/lib/Bitcode/NaCl/Writer/Makefile
+++ /dev/null
@@ -1,15 +0,0 @@
-##===- lib/Bitcode/NaCl/Writer/Makefile --------------------*- Makefile -*-===##
-#
-#                     The LLVM Compiler Infrastructure
-#
-# This file is distributed under the University of Illinois Open Source
-# License. See LICENSE.TXT for details.
-#
-##===----------------------------------------------------------------------===##
-
-LEVEL = ../../../..
-LIBRARYNAME = LLVMNaClBitWriter
-BUILD_ARCHIVE = 1
-
-include $(LEVEL)/Makefile.common
-
diff --git a/lib/Bitcode/NaCl/Writer/NaClBitcodeWriter.cpp b/lib/Bitcode/NaCl/Writer/NaClBitcodeWriter.cpp
deleted file mode 100644
index 9d03699407f..00000000000
--- a/lib/Bitcode/NaCl/Writer/NaClBitcodeWriter.cpp
+++ /dev/null
@@ -1,1252 +0,0 @@
-//===--- Bitcode/NaCl/Writer/NaClBitcodeWriter.cpp - Bitcode Writer -------===//
-//
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
-//
-//===----------------------------------------------------------------------===//
-//
-// Bitcode writer implementation.
-//
-//===----------------------------------------------------------------------===//
-
-#define DEBUG_TYPE "NaClBitcodeWriter"
-
-#include "llvm/Bitcode/NaCl/NaClBitcodeHeader.h"
-#include "llvm/Bitcode/NaCl/NaClReaderWriter.h"
-#include "NaClValueEnumerator.h"
-#include "llvm/Bitcode/NaCl/NaClBitstreamWriter.h"
-#include "llvm/Bitcode/NaCl/NaClLLVMBitCodes.h"
-#include "llvm/IR/Constants.h"
-#include "llvm/IR/DerivedTypes.h"
-#include "llvm/IR/InlineAsm.h"
-#include "llvm/IR/Instructions.h"
-#include "llvm/IR/Module.h"
-#include "llvm/IR/Operator.h"
-#include "llvm/IR/ValueSymbolTable.h"
-#include "llvm/Support/Debug.h"
-#include "llvm/Support/CommandLine.h"
-#include "llvm/Support/ErrorHandling.h"
-#include "llvm/Support/MathExtras.h"
-#include "llvm/Support/Program.h"
-#include "llvm/Support/raw_ostream.h"
-#include <cctype>
-#include <map>
-using namespace llvm;
-
-static cl::opt<unsigned>
-PNaClVersion("pnacl-version",
-             cl::desc("Specify PNaCl bitcode version to write"),
-             cl::init(2));
-
-static cl::opt<bool>
-AlignBitcodeRecords("align-bitcode-records",
-    cl::desc("Align bitcode records in PNaCl bitcode files (experimental)"),
-    cl::init(false));
-
-/// These are manifest constants used by the bitcode writer. They do
-/// not need to be kept in sync with the reader, but need to be
-/// consistent within this file.
-///
-/// Note that for each block type GROUP, the last entry should be of
-/// the form:
-///
-///    GROUP_MAX_ABBREV = GROUP_LAST_ABBREV,
-///
-/// where GROUP_LAST_ABBREV is the last defined abbreviation. See
-/// include file "llvm/Bitcode/NaCl/NaClBitCodes.h" for more
-/// information on how groups should be defined.
-enum {
-  // VALUE_SYMTAB_BLOCK abbrev id's.
-  VST_ENTRY_8_ABBREV = naclbitc::FIRST_APPLICATION_ABBREV,
-  VST_ENTRY_7_ABBREV,
-  VST_ENTRY_6_ABBREV,
-  VST_BBENTRY_6_ABBREV,
-  VST_MAX_ABBREV = VST_BBENTRY_6_ABBREV,
-
-  // CONSTANTS_BLOCK abbrev id's.
-  CONSTANTS_SETTYPE_ABBREV = naclbitc::FIRST_APPLICATION_ABBREV,
-  CONSTANTS_INTEGER_ABBREV,
-  CONSTANTS_INTEGER_ZERO_ABBREV,
-  CONSTANTS_FLOAT_ABBREV,
-  CONSTANTS_MAX_ABBREV = CONSTANTS_FLOAT_ABBREV,
-
-  // GLOBALVAR BLOCK abbrev id's.
-  GLOBALVAR_VAR_ABBREV = naclbitc::FIRST_APPLICATION_ABBREV,
-  GLOBALVAR_COMPOUND_ABBREV,
-  GLOBALVAR_ZEROFILL_ABBREV,
-  GLOBALVAR_DATA_ABBREV,
-  GLOBALVAR_RELOC_ABBREV,
-  GLOBALVAR_RELOC_WITH_ADDEND_ABBREV,
-  GLOBALVAR_MAX_ABBREV = GLOBALVAR_RELOC_WITH_ADDEND_ABBREV,
-
-  // FUNCTION_BLOCK abbrev id's.
-  FUNCTION_INST_LOAD_ABBREV = naclbitc::FIRST_APPLICATION_ABBREV,
-  FUNCTION_INST_BINOP_ABBREV,
-  FUNCTION_INST_CAST_ABBREV,
-  FUNCTION_INST_RET_VOID_ABBREV,
-  FUNCTION_INST_RET_VAL_ABBREV,
-  FUNCTION_INST_UNREACHABLE_ABBREV,
-  FUNCTION_INST_FORWARDTYPEREF_ABBREV,
-  FUNCTION_INST_STORE_ABBREV,
-  FUNCTION_INST_MAX_ABBREV = FUNCTION_INST_STORE_ABBREV,
-
-  // TYPE_BLOCK_ID_NEW abbrev id's.
-  TYPE_FUNCTION_ABBREV = naclbitc::FIRST_APPLICATION_ABBREV,
-  TYPE_MAX_ABBREV = TYPE_FUNCTION_ABBREV
-};
-
-LLVM_ATTRIBUTE_NORETURN
-static void ReportIllegalValue(const char *ValueMessage,
-                               const Value &Value) {
-  std::string Message;
-  raw_string_ostream StrM(Message);
-  StrM << "NaCl Illegal ";
-  if (ValueMessage != 0)
-    StrM << ValueMessage << " ";
-  StrM << ": " << Value;
-  report_fatal_error(StrM.str());
-}
-
-static unsigned GetEncodedCastOpcode(unsigned Opcode, const Value &V) {
-  switch (Opcode) {
-  default: ReportIllegalValue("cast", V);
-  case Instruction::Trunc   : return naclbitc::CAST_TRUNC;
-  case Instruction::ZExt    : return naclbitc::CAST_ZEXT;
-  case Instruction::SExt    : return naclbitc::CAST_SEXT;
-  case Instruction::FPToUI  : return naclbitc::CAST_FPTOUI;
-  case Instruction::FPToSI  : return naclbitc::CAST_FPTOSI;
-  case Instruction::UIToFP  : return naclbitc::CAST_UITOFP;
-  case Instruction::SIToFP  : return naclbitc::CAST_SITOFP;
-  case Instruction::FPTrunc : return naclbitc::CAST_FPTRUNC;
-  case Instruction::FPExt   : return naclbitc::CAST_FPEXT;
-  case Instruction::BitCast : return naclbitc::CAST_BITCAST;
-  }
-}
-
-static unsigned GetEncodedBinaryOpcode(unsigned Opcode, const Value &V) {
-  switch (Opcode) {
-  default: ReportIllegalValue("binary opcode", V);
-  case Instruction::Add:
-  case Instruction::FAdd: return naclbitc::BINOP_ADD;
-  case Instruction::Sub:
-  case Instruction::FSub: return naclbitc::BINOP_SUB;
-  case Instruction::Mul:
-  case Instruction::FMul: return naclbitc::BINOP_MUL;
-  case Instruction::UDiv: return naclbitc::BINOP_UDIV;
-  case Instruction::FDiv:
-  case Instruction::SDiv: return naclbitc::BINOP_SDIV;
-  case Instruction::URem: return naclbitc::BINOP_UREM;
-  case Instruction::FRem:
-  case Instruction::SRem: return naclbitc::BINOP_SREM;
-  case Instruction::Shl:  return naclbitc::BINOP_SHL;
-  case Instruction::LShr: return naclbitc::BINOP_LSHR;
-  case Instruction::AShr: return naclbitc::BINOP_ASHR;
-  case Instruction::And:  return naclbitc::BINOP_AND;
-  case Instruction::Or:   return naclbitc::BINOP_OR;
-  case Instruction::Xor:  return naclbitc::BINOP_XOR;
-  }
-}
-
-static unsigned GetEncodedCallingConv(CallingConv::ID conv) {
-  switch (conv) {
-  default: report_fatal_error(
-      "Calling convention not supported by PNaCL bitcode");
-  case CallingConv::C: return naclbitc::C_CallingConv;
-  }
-}
-
-// Converts LLVM encoding of comparison predicates to the
-// corresponding bitcode versions.
-static unsigned GetEncodedCmpPredicate(const CmpInst &Cmp) {
-  switch (Cmp.getPredicate()) {
-  default: report_fatal_error(
-      "Comparison predicate not supported by PNaCl bitcode");
-  case CmpInst::FCMP_FALSE:
-    return naclbitc::FCMP_FALSE;
-  case CmpInst::FCMP_OEQ:
-    return naclbitc::FCMP_OEQ;
-  case CmpInst::FCMP_OGT:
-    return naclbitc::FCMP_OGT;
-  case CmpInst::FCMP_OGE:
-    return naclbitc::FCMP_OGE;
-  case CmpInst::FCMP_OLT:
-    return naclbitc::FCMP_OLT;
-  case CmpInst::FCMP_OLE:
-    return naclbitc::FCMP_OLE;
-  case CmpInst::FCMP_ONE:
-    return naclbitc::FCMP_ONE;
-  case CmpInst::FCMP_ORD:
-    return naclbitc::FCMP_ORD;
-  case CmpInst::FCMP_UNO:
-    return naclbitc::FCMP_UNO;
-  case CmpInst::FCMP_UEQ:
-    return naclbitc::FCMP_UEQ;
-  case CmpInst::FCMP_UGT:
-    return naclbitc::FCMP_UGT;
-  case CmpInst::FCMP_UGE:
-    return naclbitc::FCMP_UGE;
-  case CmpInst::FCMP_ULT:
-    return naclbitc::FCMP_ULT;
-  case CmpInst::FCMP_ULE:
-    return naclbitc::FCMP_ULE;
-  case CmpInst::FCMP_UNE:
-    return naclbitc::FCMP_UNE;
-  case CmpInst::FCMP_TRUE:
-    return naclbitc::FCMP_TRUE;
-  case CmpInst::ICMP_EQ:
-    return naclbitc::ICMP_EQ;
-  case CmpInst::ICMP_NE:
-    return naclbitc::ICMP_NE;
-  case CmpInst::ICMP_UGT:
-    return naclbitc::ICMP_UGT;
-  case CmpInst::ICMP_UGE:
-    return naclbitc::ICMP_UGE;
-  case CmpInst::ICMP_ULT:
-    return naclbitc::ICMP_ULT;
-  case CmpInst::ICMP_ULE:
-    return naclbitc::ICMP_ULE;
-  case CmpInst::ICMP_SGT:
-    return naclbitc::ICMP_SGT;
-  case CmpInst::ICMP_SGE:
-    return naclbitc::ICMP_SGE;
-  case CmpInst::ICMP_SLT:
-    return naclbitc::ICMP_SLT;
-  case CmpInst::ICMP_SLE:
-    return naclbitc::ICMP_SLE;
-  }
-}
-
-// The type of encoding to use for type ids.
-static NaClBitCodeAbbrevOp::Encoding TypeIdEncoding = NaClBitCodeAbbrevOp::VBR;
-
-// The cutoff (in number of bits) from Fixed to VBR.
-static const unsigned TypeIdVBRCutoff = 6;
-
-// The number of bits to use in the encoding of type ids.
-static unsigned TypeIdNumBits = TypeIdVBRCutoff;
-
-// Optimizes the value for TypeIdEncoding and TypeIdNumBits based
-// the actual number of types.
-static inline void OptimizeTypeIdEncoding(const NaClValueEnumerator &VE) {
-  // Note: modify to use maximum number of bits if under cutoff. Otherwise,
-  // use VBR to take advantage that frequently referenced types have
-  // small IDs.
-  unsigned NumBits = NaClBitsNeededForValue(VE.getTypes().size());
-  TypeIdNumBits = (NumBits < TypeIdVBRCutoff ? NumBits : TypeIdVBRCutoff);
-  TypeIdEncoding = NaClBitCodeAbbrevOp::Encoding(
-      NumBits <= TypeIdVBRCutoff
-      ? NaClBitCodeAbbrevOp::Fixed : NaClBitCodeAbbrevOp::VBR);
-}
-
-/// WriteTypeTable - Write out the type table for a module.
-static void WriteTypeTable(const NaClValueEnumerator &VE,
-                           NaClBitstreamWriter &Stream) {
-  DEBUG(dbgs() << "-> WriteTypeTable\n");
-  const NaClValueEnumerator::TypeList &TypeList = VE.getTypes();
-
-  Stream.EnterSubblock(naclbitc::TYPE_BLOCK_ID_NEW, TYPE_MAX_ABBREV);
-
-  SmallVector<uint64_t, 64> TypeVals;
-
-  // Abbrev for TYPE_CODE_FUNCTION.
-  NaClBitCodeAbbrev *Abbv = new NaClBitCodeAbbrev();
-  Abbv->Add(NaClBitCodeAbbrevOp(naclbitc::TYPE_CODE_FUNCTION));
-  Abbv->Add(NaClBitCodeAbbrevOp(NaClBitCodeAbbrevOp::Fixed, 1));  // isvararg
-  Abbv->Add(NaClBitCodeAbbrevOp(NaClBitCodeAbbrevOp::Array));
-  Abbv->Add(NaClBitCodeAbbrevOp(TypeIdEncoding, TypeIdNumBits));
-  if (TYPE_FUNCTION_ABBREV != Stream.EmitAbbrev(Abbv))
-    llvm_unreachable("Unexpected abbrev ordering!");
-
-  // Emit an entry count so the reader can reserve space.
-  TypeVals.push_back(TypeList.size());
-  Stream.EmitRecord(naclbitc::TYPE_CODE_NUMENTRY, TypeVals);
-  TypeVals.clear();
-
-  // Loop over all of the types, emitting each in turn.
-  for (unsigned i = 0, e = TypeList.size(); i != e; ++i) {
-    Type *T = TypeList[i];
-    int AbbrevToUse = 0;
-    unsigned Code = 0;
-
-    switch (T->getTypeID()) {
-    default: llvm_unreachable("Unknown type!");
-    case Type::VoidTyID:      Code = naclbitc::TYPE_CODE_VOID;      break;
-    case Type::FloatTyID:     Code = naclbitc::TYPE_CODE_FLOAT;     break;
-    case Type::DoubleTyID:    Code = naclbitc::TYPE_CODE_DOUBLE;    break;
-    case Type::IntegerTyID:
-      // INTEGER: [width]
-      Code = naclbitc::TYPE_CODE_INTEGER;
-      TypeVals.push_back(cast<IntegerType>(T)->getBitWidth());
-      break;
-    case Type::VectorTyID: {
-      VectorType *VT = cast<VectorType>(T);
-      // VECTOR [numelts, eltty]
-      Code = naclbitc::TYPE_CODE_VECTOR;
-      TypeVals.push_back(VT->getNumElements());
-      TypeVals.push_back(VE.getTypeID(VT->getElementType()));
-      break;
-    }
-    case Type::FunctionTyID: {
-      FunctionType *FT = cast<FunctionType>(T);
-      // FUNCTION: [isvararg, retty, paramty x N]
-      Code = naclbitc::TYPE_CODE_FUNCTION;
-      TypeVals.push_back(FT->isVarArg());
-      TypeVals.push_back(VE.getTypeID(FT->getReturnType()));
-      for (unsigned i = 0, e = FT->getNumParams(); i != e; ++i)
-        TypeVals.push_back(VE.getTypeID(FT->getParamType(i)));
-      AbbrevToUse = TYPE_FUNCTION_ABBREV;
-      break;
-    }
-    case Type::StructTyID:
-      report_fatal_error("Struct types are not supported in PNaCl bitcode");
-    case Type::ArrayTyID:
-      report_fatal_error("Array types are not supported in PNaCl bitcode");
-    }
-
-    // Emit the finished record.
-    Stream.EmitRecord(Code, TypeVals, AbbrevToUse);
-    TypeVals.clear();
-  }
-
-  Stream.ExitBlock();
-  DEBUG(dbgs() << "<- WriteTypeTable\n");
-}
-
-static unsigned getEncodedLinkage(const GlobalValue *GV) {
-  switch (GV->getLinkage()) {
-  case GlobalValue::ExternalLinkage:                 return 0;
-  case GlobalValue::InternalLinkage:                 return 3;
-  default:
-    report_fatal_error("Invalid linkage");
-  }
-}
-
-/// \brief Function to convert constant initializers for global
-/// variables into corresponding bitcode. Takes advantage that these
-/// global variable initializations are normalized (see
-/// lib/Transforms/NaCl/FlattenGlobals.cpp).
-void WriteGlobalInit(const Constant *C, unsigned GlobalVarID,
-                     SmallVectorImpl<uint32_t> &Vals,
-                     const NaClValueEnumerator &VE,
-                     NaClBitstreamWriter &Stream) {
-  if (ArrayType *Ty = dyn_cast<ArrayType>(C->getType())) {
-    if (!Ty->getElementType()->isIntegerTy(8))
-      report_fatal_error("Global array initializer not i8");
-    uint32_t Size = Ty->getNumElements();
-    if (isa<ConstantAggregateZero>(C)) {
-      Vals.push_back(Size);
-      Stream.EmitRecord(naclbitc::GLOBALVAR_ZEROFILL, Vals,
-                        GLOBALVAR_ZEROFILL_ABBREV);
-      Vals.clear();
-    } else {
-      const ConstantDataSequential *CD = cast<ConstantDataSequential>(C);
-      StringRef Data = CD->getRawDataValues();
-      for (size_t i = 0; i < Size; ++i) {
-        Vals.push_back(Data[i] & 0xFF);
-      }
-      Stream.EmitRecord(naclbitc::GLOBALVAR_DATA, Vals,
-                        GLOBALVAR_DATA_ABBREV);
-      Vals.clear();
-    }
-    return;
-  }
-  if (VE.IsIntPtrType(C->getType())) {
-    // This constant defines a relocation. Start by verifying the
-    // relocation is of the right form.
-    const ConstantExpr *CE = dyn_cast<ConstantExpr>(C);
-    if (CE == 0)
-      report_fatal_error("Global i32 initializer not constant");
-    assert(CE);
-    int32_t Addend = 0;
-    if (CE->getOpcode() == Instruction::Add) {
-      const ConstantInt *AddendConst = dyn_cast<ConstantInt>(CE->getOperand(1));
-      if (AddendConst == 0)
-        report_fatal_error("Malformed addend in global relocation initializer");
-      Addend = AddendConst->getSExtValue();
-      CE = dyn_cast<ConstantExpr>(CE->getOperand(0));
-      if (CE == 0)
-        report_fatal_error(
-            "Base of global relocation initializer not constant");
-    }
-    if (CE->getOpcode() != Instruction::PtrToInt)
-      report_fatal_error("Global relocation base doesn't contain ptrtoint");
-    GlobalValue *GV = dyn_cast<GlobalValue>(CE->getOperand(0));
-    if (GV == 0)
-      report_fatal_error(
-          "Argument of ptrtoint in global relocation no global value");
-
-    // Now generate the corresponding relocation record.
-    unsigned RelocID = VE.getValueID(GV);
-    // This is a value index.
-    unsigned AbbrevToUse = GLOBALVAR_RELOC_ABBREV;
-    Vals.push_back(RelocID);
-    if (Addend) {
-      Vals.push_back(Addend);
-      AbbrevToUse = GLOBALVAR_RELOC_WITH_ADDEND_ABBREV;
-    }
-    Stream.EmitRecord(naclbitc::GLOBALVAR_RELOC, Vals, AbbrevToUse);
-    Vals.clear();
-    return;
-  }
-  report_fatal_error("Global initializer is not a SimpleElement");
-}
-
-// Emit global variables.
-static void WriteGlobalVars(const Module *M,
-                            const NaClValueEnumerator &VE,
-                            NaClBitstreamWriter &Stream) {
-  Stream.EnterSubblock(naclbitc::GLOBALVAR_BLOCK_ID);
-  SmallVector<uint32_t, 32> Vals;
-  unsigned GlobalVarID = VE.getFirstGlobalVarID();
-
-  // Emit the number of global variables.
-
-  Vals.push_back(M->getGlobalList().size());
-  Stream.EmitRecord(naclbitc::GLOBALVAR_COUNT, Vals);
-  Vals.clear();
-
-  // Now emit each global variable.
-  for (Module::const_global_iterator
-           GV = M->global_begin(), E = M->global_end();
-       GV != E; ++GV, ++GlobalVarID) {
-    // Define the global variable.
-    Vals.push_back(Log2_32(GV->getAlignment()) + 1);
-    Vals.push_back(GV->isConstant());
-    Stream.EmitRecord(naclbitc::GLOBALVAR_VAR, Vals, GLOBALVAR_VAR_ABBREV);
-    Vals.clear();
-
-    // Add the field(s).
-    const Constant *C = GV->getInitializer();
-    if (C == 0)
-      report_fatal_error("Global variable initializer not a constant");
-    if (const ConstantStruct *CS = dyn_cast<ConstantStruct>(C)) {
-      if (!CS->getType()->isPacked())
-        report_fatal_error("Global variable type not packed");
-      if (CS->getType()->hasName())
-        report_fatal_error("Global variable type is named");
-      Vals.push_back(CS->getNumOperands());
-      Stream.EmitRecord(naclbitc::GLOBALVAR_COMPOUND, Vals,
-                        GLOBALVAR_COMPOUND_ABBREV);
-      Vals.clear();
-      for (unsigned I = 0; I < CS->getNumOperands(); ++I) {
-        WriteGlobalInit(dyn_cast<Constant>(CS->getOperand(I)), GlobalVarID,
-                        Vals, VE, Stream);
-      }
-    } else {
-      WriteGlobalInit(C, GlobalVarID, Vals, VE, Stream);
-    }
-  }
-
-  assert(GlobalVarID == VE.getFirstGlobalVarID() + VE.getNumGlobalVarIDs());
-  Stream.ExitBlock();
-}
-
-// Emit top-level description of module, including inline asm,
-// descriptors for global variables, and function prototype info.
-static void WriteModuleInfo(const Module *M, const NaClValueEnumerator &VE,
-                            NaClBitstreamWriter &Stream) {
-  DEBUG(dbgs() << "-> WriteModuleInfo\n");
-
-  // Emit the function proto information. Note: We do this before
-  // global variables, so that global variable initializations can
-  // refer to the functions without a forward reference.
-  SmallVector<unsigned, 64> Vals;
-  for (Module::const_iterator F = M->begin(), E = M->end(); F != E; ++F) {
-    // FUNCTION:  [type, callingconv, isproto, linkage]
-    Type *Ty = F->getType()->getPointerElementType();
-    Vals.push_back(VE.getTypeID(Ty));
-    Vals.push_back(GetEncodedCallingConv(F->getCallingConv()));
-    Vals.push_back(F->isDeclaration());
-    Vals.push_back(getEncodedLinkage(F));
-
-    unsigned AbbrevToUse = 0;
-    Stream.EmitRecord(naclbitc::MODULE_CODE_FUNCTION, Vals, AbbrevToUse);
-    Vals.clear();
-  }
-
-  // Emit the global variable information.
-  WriteGlobalVars(M, VE, Stream);
-  DEBUG(dbgs() << "<- WriteModuleInfo\n");
-}
-
-static void emitSignedInt64(SmallVectorImpl<uint64_t> &Vals, uint64_t V) {
-  Vals.push_back(NaClEncodeSignRotatedValue((int64_t)V));
-}
-
-static void EmitAPInt(SmallVectorImpl<uint64_t> &Vals,
-                      unsigned &Code, unsigned &AbbrevToUse, const APInt &Val) {
-  if (Val.getBitWidth() <= 64) {
-    uint64_t V = Val.getSExtValue();
-    emitSignedInt64(Vals, V);
-    Code = naclbitc::CST_CODE_INTEGER;
-    AbbrevToUse =
-        Val == 0 ? CONSTANTS_INTEGER_ZERO_ABBREV : CONSTANTS_INTEGER_ABBREV;
-  } else {
-    report_fatal_error("Wide integers are not supported");
-  }
-}
-
-static void WriteConstants(unsigned FirstVal, unsigned LastVal,
-                           const NaClValueEnumerator &VE,
-                           NaClBitstreamWriter &Stream) {
-  if (FirstVal == LastVal) return;
-
-  Stream.EnterSubblock(naclbitc::CONSTANTS_BLOCK_ID, CONSTANTS_MAX_ABBREV);
-
-  SmallVector<uint64_t, 64> Record;
-
-  const NaClValueEnumerator::ValueList &Vals = VE.getValues();
-  Type *LastTy = 0;
-  for (unsigned i = FirstVal; i != LastVal; ++i) {
-    const Value *V = Vals[i].first;
-    // If we need to switch types, do so now.
-    if (V->getType() != LastTy) {
-      LastTy = V->getType();
-      Record.push_back(VE.getTypeID(LastTy));
-      Stream.EmitRecord(naclbitc::CST_CODE_SETTYPE, Record,
-                        CONSTANTS_SETTYPE_ABBREV);
-      Record.clear();
-    }
-
-    if (isa<InlineAsm>(V)) {
-      ReportIllegalValue("inline assembly", *V);
-    }
-    const Constant *C = cast<Constant>(V);
-    unsigned Code = -1U;
-    unsigned AbbrevToUse = 0;
-    if (isa<UndefValue>(C)) {
-      Code = naclbitc::CST_CODE_UNDEF;
-    } else if (const ConstantInt *IV = dyn_cast<ConstantInt>(C)) {
-      EmitAPInt(Record, Code, AbbrevToUse, IV->getValue());
-    } else if (const ConstantFP *CFP = dyn_cast<ConstantFP>(C)) {
-      Code = naclbitc::CST_CODE_FLOAT;
-      AbbrevToUse = CONSTANTS_FLOAT_ABBREV;
-      Type *Ty = CFP->getType();
-      if (Ty->isFloatTy() || Ty->isDoubleTy()) {
-        Record.push_back(CFP->getValueAPF().bitcastToAPInt().getZExtValue());
-      } else {
-        report_fatal_error("Unknown FP type");
-      }
-    } else {
-#ifndef NDEBUG
-      C->dump();
-#endif
-      ReportIllegalValue("constant", *C);
-    }
-    Stream.EmitRecord(Code, Record, AbbrevToUse);
-    Record.clear();
-  }
-
-  Stream.ExitBlock();
-  DEBUG(dbgs() << "<- WriteConstants\n");
-}
-
-/// \brief Emits a type for the forward value reference. That is, if
-/// the ID for the given value is larger than or equal to the BaseID,
-/// the corresponding forward reference is generated.
-static void EmitFnForwardTypeRef(const Value *V,
-                                 unsigned BaseID,
-                                 NaClValueEnumerator &VE,
-                                 NaClBitstreamWriter &Stream) {
-  unsigned ValID = VE.getValueID(V);
-  if (ValID >= BaseID &&
-      VE.InsertFnForwardTypeRef(ValID)) {
-    SmallVector<unsigned, 2> Vals;
-    Vals.push_back(ValID);
-    Vals.push_back(VE.getTypeID(VE.NormalizeType(V->getType())));
-    Stream.EmitRecord(naclbitc::FUNC_CODE_INST_FORWARDTYPEREF, Vals,
-                      FUNCTION_INST_FORWARDTYPEREF_ABBREV);
-  }
-}
-
-/// pushValue - The file has to encode both the value and type id for
-/// many values, because we need to know what type to create for forward
-/// references.  However, most operands are not forward references, so this type
-/// field is not needed.
-///
-/// This function adds V's value ID to Vals.  If the value ID is higher than the
-/// instruction ID, then it is a forward reference, and it also includes the
-/// type ID.  The value ID that is written is encoded relative to the InstID.
-static void pushValue(const Value *V, unsigned InstID,
-                      SmallVector<unsigned, 64> &Vals,
-                      NaClValueEnumerator &VE,
-                      NaClBitstreamWriter &Stream) {
-  const Value *VElided = VE.ElideCasts(V);
-  EmitFnForwardTypeRef(VElided, InstID, VE, Stream);
-  unsigned ValID = VE.getValueID(VElided);
-  // Make encoding relative to the InstID.
-  Vals.push_back(InstID - ValID);
-}
-
-static void pushValue64(const Value *V, unsigned InstID,
-                        SmallVector<uint64_t, 128> &Vals,
-                        NaClValueEnumerator &VE,
-                        NaClBitstreamWriter &Stream) {
-  const Value *VElided = VE.ElideCasts(V);
-  EmitFnForwardTypeRef(VElided, InstID, VE, Stream);
-  uint64_t ValID = VE.getValueID(VElided);
-  Vals.push_back(InstID - ValID);
-}
-
-static void pushValueSigned(const Value *V, unsigned InstID,
-                            SmallVector<uint64_t, 128> &Vals,
-                            NaClValueEnumerator &VE,
-                            NaClBitstreamWriter &Stream) {
-  const Value *VElided = VE.ElideCasts(V);
-  EmitFnForwardTypeRef(VElided, InstID, VE, Stream);
-  unsigned ValID = VE.getValueID(VElided);
-  int64_t diff = ((int32_t)InstID - (int32_t)ValID);
-  emitSignedInt64(Vals, diff);
-}
-
-/// WriteInstruction - Emit an instruction to the specified stream.
-/// Returns true if instruction actually emitted.
-static bool WriteInstruction(const Instruction &I, unsigned InstID,
-                             NaClValueEnumerator &VE,
-                             NaClBitstreamWriter &Stream,
-                             SmallVector<unsigned, 64> &Vals) {
-  unsigned Code = 0;
-  unsigned AbbrevToUse = 0;
-  VE.setInstructionID(&I);
-  switch (I.getOpcode()) {
-  default:
-    if (Instruction::isCast(I.getOpcode())) {
-      // CAST:       [opval, destty, castopc]
-      if (VE.IsElidedCast(&I))
-        return false;
-      Code = naclbitc::FUNC_CODE_INST_CAST;
-      AbbrevToUse = FUNCTION_INST_CAST_ABBREV;
-      pushValue(I.getOperand(0), InstID, Vals, VE, Stream);
-      Vals.push_back(VE.getTypeID(I.getType()));
-      unsigned Opcode = I.getOpcode();
-      Vals.push_back(GetEncodedCastOpcode(Opcode, I));
-      if (Opcode == Instruction::PtrToInt ||
-          Opcode == Instruction::IntToPtr ||
-          (Opcode == Instruction::BitCast &&
-           (I.getOperand(0)->getType()->isPointerTy() ||
-            I.getType()->isPointerTy()))) {
-        ReportIllegalValue("(PNaCl ABI) pointer cast", I);
-      }
-    } else if (isa<BinaryOperator>(I)) {
-      // BINOP:      [opval, opval, opcode]
-      Code = naclbitc::FUNC_CODE_INST_BINOP;
-      AbbrevToUse = FUNCTION_INST_BINOP_ABBREV;
-      pushValue(I.getOperand(0), InstID, Vals, VE, Stream);
-      pushValue(I.getOperand(1), InstID, Vals, VE, Stream);
-      Vals.push_back(GetEncodedBinaryOpcode(I.getOpcode(), I));
-    } else {
-      ReportIllegalValue("instruction", I);
-    }
-    break;
-  case Instruction::Select:
-    Code = naclbitc::FUNC_CODE_INST_VSELECT;
-    pushValue(I.getOperand(1), InstID, Vals, VE, Stream);
-    pushValue(I.getOperand(2), InstID, Vals, VE, Stream);
-    pushValue(I.getOperand(0), InstID, Vals, VE, Stream);
-    break;
-  case Instruction::ExtractElement:
-    Code = naclbitc::FUNC_CODE_INST_EXTRACTELT;
-    pushValue(I.getOperand(0), InstID, Vals, VE, Stream);
-    pushValue(I.getOperand(1), InstID, Vals, VE, Stream);
-    break;
-  case Instruction::InsertElement:
-    Code = naclbitc::FUNC_CODE_INST_INSERTELT;
-    pushValue(I.getOperand(0), InstID, Vals, VE, Stream);
-    pushValue(I.getOperand(1), InstID, Vals, VE, Stream);
-    pushValue(I.getOperand(2), InstID, Vals, VE, Stream);
-    break;
-  case Instruction::ICmp:
-  case Instruction::FCmp:
-    // compare returning Int1Ty or vector of Int1Ty
-    Code = naclbitc::FUNC_CODE_INST_CMP2;
-    pushValue(I.getOperand(0), InstID, Vals, VE, Stream);
-    pushValue(I.getOperand(1), InstID, Vals, VE, Stream);
-    Vals.push_back(GetEncodedCmpPredicate(cast<CmpInst>(I)));
-    break;
-
-  case Instruction::Ret:
-    {
-      Code = naclbitc::FUNC_CODE_INST_RET;
-      unsigned NumOperands = I.getNumOperands();
-      if (NumOperands == 0)
-        AbbrevToUse = FUNCTION_INST_RET_VOID_ABBREV;
-      else if (NumOperands == 1) {
-        pushValue(I.getOperand(0), InstID, Vals, VE, Stream);
-        AbbrevToUse = FUNCTION_INST_RET_VAL_ABBREV;
-      } else {
-        for (unsigned i = 0, e = NumOperands; i != e; ++i)
-          pushValue(I.getOperand(i), InstID, Vals, VE, Stream);
-      }
-    }
-    break;
-  case Instruction::Br:
-    {
-      Code = naclbitc::FUNC_CODE_INST_BR;
-      const BranchInst &II = cast<BranchInst>(I);
-      Vals.push_back(VE.getValueID(II.getSuccessor(0)));
-      if (II.isConditional()) {
-        Vals.push_back(VE.getValueID(II.getSuccessor(1)));
-        pushValue(II.getCondition(), InstID, Vals, VE, Stream);
-      }
-    }
-    break;
-  case Instruction::Switch:
-    {
-      // Redefine Vals, since here we need to use 64 bit values
-      // explicitly to store large APInt numbers.
-      SmallVector<uint64_t, 128> Vals64;
-
-      Code = naclbitc::FUNC_CODE_INST_SWITCH;
-      const SwitchInst &SI = cast<SwitchInst>(I);
-
-      Vals64.push_back(VE.getTypeID(SI.getCondition()->getType()));
-      pushValue64(SI.getCondition(), InstID, Vals64, VE, Stream);
-      Vals64.push_back(VE.getValueID(SI.getDefaultDest()));
-      Vals64.push_back(SI.getNumCases());
-      for (SwitchInst::ConstCaseIt i = SI.case_begin(), e = SI.case_end();
-           i != e; ++i) {
-        // The PNaCl bitcode format has vestigial support for case
-        // ranges, but we no longer support reading or writing them,
-        // so the next two fields always have the same values.
-        // See https://code.google.com/p/nativeclient/issues/detail?id=3758
-        Vals64.push_back(1/*NumItems = 1*/);
-        Vals64.push_back(true/*IsSingleNumber = true*/);
-
-        emitSignedInt64(Vals64, i.getCaseValue()->getSExtValue());
-        Vals64.push_back(VE.getValueID(i.getCaseSuccessor()));
-      }
-
-      Stream.EmitRecord(Code, Vals64, AbbrevToUse);
-
-      // Also do expected action - clear external Vals collection:
-      Vals.clear();
-      return true;
-    }
-    break;
-  case Instruction::Unreachable:
-    Code = naclbitc::FUNC_CODE_INST_UNREACHABLE;
-    AbbrevToUse = FUNCTION_INST_UNREACHABLE_ABBREV;
-    break;
-
-  case Instruction::PHI: {
-    const PHINode &PN = cast<PHINode>(I);
-    Code = naclbitc::FUNC_CODE_INST_PHI;
-    // With the newer instruction encoding, forward references could give
-    // negative valued IDs.  This is most common for PHIs, so we use
-    // signed VBRs.
-    SmallVector<uint64_t, 128> Vals64;
-    Vals64.push_back(VE.getTypeID(PN.getType()));
-    for (unsigned i = 0, e = PN.getNumIncomingValues(); i != e; ++i) {
-      pushValueSigned(PN.getIncomingValue(i), InstID, Vals64, VE, Stream);
-      Vals64.push_back(VE.getValueID(PN.getIncomingBlock(i)));
-    }
-    // Emit a Vals64 vector and exit.
-    Stream.EmitRecord(Code, Vals64, AbbrevToUse);
-    Vals64.clear();
-    return true;
-  }
-
-  case Instruction::Alloca:
-    if (!cast<AllocaInst>(&I)->getAllocatedType()->isIntegerTy(8))
-      report_fatal_error("Type of alloca instruction is not i8");
-    Code = naclbitc::FUNC_CODE_INST_ALLOCA;
-    pushValue(I.getOperand(0), InstID, Vals, VE, Stream); // size.
-    Vals.push_back(Log2_32(cast<AllocaInst>(I).getAlignment())+1);
-    break;
-  case Instruction::Load:
-    // LOAD: [op, align, ty]
-    Code = naclbitc::FUNC_CODE_INST_LOAD;
-    pushValue(I.getOperand(0), InstID, Vals, VE, Stream);
-    AbbrevToUse = FUNCTION_INST_LOAD_ABBREV;
-    Vals.push_back(Log2_32(cast<LoadInst>(I).getAlignment())+1);
-    Vals.push_back(VE.getTypeID(I.getType()));
-    break;
-  case Instruction::Store:
-    // STORE: [ptr, val, align]
-    Code = naclbitc::FUNC_CODE_INST_STORE;
-    AbbrevToUse = FUNCTION_INST_STORE_ABBREV;
-    pushValue(I.getOperand(1), InstID, Vals, VE, Stream);
-    pushValue(I.getOperand(0), InstID, Vals, VE, Stream);
-    Vals.push_back(Log2_32(cast<StoreInst>(I).getAlignment())+1);
-    break;
-  case Instruction::Call: {
-    // CALL: [cc, fnid, args...]
-    // CALL_INDIRECT: [cc, fnid, fnty, args...]
-
-    const CallInst &Call = cast<CallInst>(I);
-    const Value* Callee = Call.getCalledValue();
-    Vals.push_back((GetEncodedCallingConv(Call.getCallingConv()) << 1)
-                   | unsigned(Call.isTailCall()));
-
-    pushValue(Callee, InstID, Vals, VE, Stream);
-
-    if (Callee == VE.ElideCasts(Callee)) {
-      // Since the call pointer has not been elided, we know that
-      // the call pointer has the type signature of the called
-      // function.  This implies that the reader can use the type
-      // signature of the callee to figure out how to add casts to
-      // the arguments.
-      Code = naclbitc::FUNC_CODE_INST_CALL;
-    } else {
-      // If the cast was elided, a pointer conversion to a pointer
-      // was applied, meaning that this is an indirect call. For the
-      // reader, this implies that we can't use the type signature
-      // of the callee to resolve elided call arguments, since it is
-      // not known. Hence, we must send the type signature to the
-      // reader.
-      Code = naclbitc::FUNC_CODE_INST_CALL_INDIRECT;
-      Vals.push_back(VE.getTypeID(I.getType()));
-    }
-
-    for (unsigned I = 0, E = Call.getNumArgOperands(); I < E; ++I) {
-      pushValue(Call.getArgOperand(I), InstID, Vals, VE, Stream);
-    }
-    break;
-  }
-  }
-
-  Stream.EmitRecord(Code, Vals, AbbrevToUse);
-  Vals.clear();
-  return true;
-}
-
-// Emit names for globals/functions etc.
-static void WriteValueSymbolTable(const ValueSymbolTable &VST,
-                                  const NaClValueEnumerator &VE,
-                                  NaClBitstreamWriter &Stream) {
-  if (VST.empty()) return;
-  Stream.EnterSubblock(naclbitc::VALUE_SYMTAB_BLOCK_ID);
-
-  // FIXME: Set up the abbrev, we know how many values there are!
-  // FIXME: We know if the type names can use 7-bit ascii.
-  SmallVector<unsigned, 64> NameVals;
-
-  for (ValueSymbolTable::const_iterator SI = VST.begin(), SE = VST.end();
-       SI != SE; ++SI) {
-    if (VE.IsElidedCast(SI->getValue())) continue;
-
-    const ValueName &Name = *SI;
-
-    // Figure out the encoding to use for the name.
-    bool is7Bit = true;
-    bool isChar6 = true;
-    for (const char *C = Name.getKeyData(), *E = C+Name.getKeyLength();
-         C != E; ++C) {
-      if (isChar6)
-        isChar6 = NaClBitCodeAbbrevOp::isChar6(*C);
-      if ((unsigned char)*C & 128) {
-        is7Bit = false;
-        break;  // don't bother scanning the rest.
-      }
-    }
-
-    unsigned AbbrevToUse = VST_ENTRY_8_ABBREV;
-
-    // VST_ENTRY:   [valueid, namechar x N]
-    // VST_BBENTRY: [bbid, namechar x N]
-    unsigned Code;
-    if (isa<BasicBlock>(SI->getValue())) {
-      Code = naclbitc::VST_CODE_BBENTRY;
-      if (isChar6)
-        AbbrevToUse = VST_BBENTRY_6_ABBREV;
-    } else {
-      Code = naclbitc::VST_CODE_ENTRY;
-      if (isChar6)
-        AbbrevToUse = VST_ENTRY_6_ABBREV;
-      else if (is7Bit)
-        AbbrevToUse = VST_ENTRY_7_ABBREV;
-    }
-
-    NameVals.push_back(VE.getValueID(SI->getValue()));
-    for (const char *P = Name.getKeyData(),
-         *E = Name.getKeyData()+Name.getKeyLength(); P != E; ++P)
-      NameVals.push_back((unsigned char)*P);
-
-    // Emit the finished record.
-    Stream.EmitRecord(Code, NameVals, AbbrevToUse);
-    NameVals.clear();
-  }
-  Stream.ExitBlock();
-}
-
-/// WriteFunction - Emit a function body to the module stream.
-static void WriteFunction(const Function &F, NaClValueEnumerator &VE,
-                          NaClBitstreamWriter &Stream) {
-  Stream.EnterSubblock(naclbitc::FUNCTION_BLOCK_ID);
-  VE.incorporateFunction(F);
-
-  SmallVector<unsigned, 64> Vals;
-
-  // Emit the number of basic blocks, so the reader can create them ahead of
-  // time.
-  Vals.push_back(VE.getBasicBlocks().size());
-  Stream.EmitRecord(naclbitc::FUNC_CODE_DECLAREBLOCKS, Vals);
-  Vals.clear();
-
-  // If there are function-local constants, emit them now.
-  unsigned CstStart, CstEnd;
-  VE.getFunctionConstantRange(CstStart, CstEnd);
-  WriteConstants(CstStart, CstEnd, VE, Stream);
-
-  // Keep a running idea of what the instruction ID is.
-  unsigned InstID = CstEnd;
-
-  // Finally, emit all the instructions, in order.
-  for (Function::const_iterator BB = F.begin(), E = F.end(); BB != E; ++BB)
-    for (BasicBlock::const_iterator I = BB->begin(), E = BB->end();
-         I != E; ++I) {
-      if (WriteInstruction(*I, InstID, VE, Stream, Vals) &&
-          !I->getType()->isVoidTy())
-        ++InstID;
-    }
-
-  // Emit names for instructions etc.
-  if (PNaClAllowLocalSymbolTables)
-    WriteValueSymbolTable(F.getValueSymbolTable(), VE, Stream);
-
-  VE.purgeFunction();
-  Stream.ExitBlock();
-}
-
-// Emit blockinfo, which defines the standard abbreviations etc.
-static void WriteBlockInfo(const NaClValueEnumerator &VE,
-                           NaClBitstreamWriter &Stream) {
-  // We only want to emit block info records for blocks that have multiple
-  // instances: CONSTANTS_BLOCK, FUNCTION_BLOCK and VALUE_SYMTAB_BLOCK.
-  // Other blocks can define their abbrevs inline.
-  Stream.EnterBlockInfoBlock();
-
-  { // 8-bit fixed-width VST_ENTRY/VST_BBENTRY strings.
-    NaClBitCodeAbbrev *Abbv = new NaClBitCodeAbbrev();
-    Abbv->Add(NaClBitCodeAbbrevOp(NaClBitCodeAbbrevOp::Fixed, 3));
-    Abbv->Add(NaClBitCodeAbbrevOp(NaClBitCodeAbbrevOp::VBR, 8));
-    Abbv->Add(NaClBitCodeAbbrevOp(NaClBitCodeAbbrevOp::Array));
-    Abbv->Add(NaClBitCodeAbbrevOp(NaClBitCodeAbbrevOp::Fixed, 8));
-    if (Stream.EmitBlockInfoAbbrev(naclbitc::VALUE_SYMTAB_BLOCK_ID,
-                                   Abbv) != VST_ENTRY_8_ABBREV)
-      llvm_unreachable("Unexpected abbrev ordering!");
-  }
-
-  { // 7-bit fixed width VST_ENTRY strings.
-    NaClBitCodeAbbrev *Abbv = new NaClBitCodeAbbrev();
-    Abbv->Add(NaClBitCodeAbbrevOp(naclbitc::VST_CODE_ENTRY));
-    Abbv->Add(NaClBitCodeAbbrevOp(NaClBitCodeAbbrevOp::VBR, 8));
-    Abbv->Add(NaClBitCodeAbbrevOp(NaClBitCodeAbbrevOp::Array));
-    Abbv->Add(NaClBitCodeAbbrevOp(NaClBitCodeAbbrevOp::Fixed, 7));
-    if (Stream.EmitBlockInfoAbbrev(naclbitc::VALUE_SYMTAB_BLOCK_ID,
-                                   Abbv) != VST_ENTRY_7_ABBREV)
-      llvm_unreachable("Unexpected abbrev ordering!");
-  }
-  { // 6-bit char6 VST_ENTRY strings.
-    NaClBitCodeAbbrev *Abbv = new NaClBitCodeAbbrev();
-    Abbv->Add(NaClBitCodeAbbrevOp(naclbitc::VST_CODE_ENTRY));
-    Abbv->Add(NaClBitCodeAbbrevOp(NaClBitCodeAbbrevOp::VBR, 8));
-    Abbv->Add(NaClBitCodeAbbrevOp(NaClBitCodeAbbrevOp::Array));
-    Abbv->Add(NaClBitCodeAbbrevOp(NaClBitCodeAbbrevOp::Char6));
-    if (Stream.EmitBlockInfoAbbrev(naclbitc::VALUE_SYMTAB_BLOCK_ID,
-                                   Abbv) != VST_ENTRY_6_ABBREV)
-      llvm_unreachable("Unexpected abbrev ordering!");
-  }
-  { // 6-bit char6 VST_BBENTRY strings.
-    NaClBitCodeAbbrev *Abbv = new NaClBitCodeAbbrev();
-    Abbv->Add(NaClBitCodeAbbrevOp(naclbitc::VST_CODE_BBENTRY));
-    Abbv->Add(NaClBitCodeAbbrevOp(NaClBitCodeAbbrevOp::VBR, 8));
-    Abbv->Add(NaClBitCodeAbbrevOp(NaClBitCodeAbbrevOp::Array));
-    Abbv->Add(NaClBitCodeAbbrevOp(NaClBitCodeAbbrevOp::Char6));
-    if (Stream.EmitBlockInfoAbbrev(naclbitc::VALUE_SYMTAB_BLOCK_ID,
-                                   Abbv) != VST_BBENTRY_6_ABBREV)
-      llvm_unreachable("Unexpected abbrev ordering!");
-  }
-
-
-
-  { // SETTYPE abbrev for CONSTANTS_BLOCK.
-    NaClBitCodeAbbrev *Abbv = new NaClBitCodeAbbrev();
-    Abbv->Add(NaClBitCodeAbbrevOp(naclbitc::CST_CODE_SETTYPE));
-    Abbv->Add(NaClBitCodeAbbrevOp(TypeIdEncoding, TypeIdNumBits));
-    if (Stream.EmitBlockInfoAbbrev(naclbitc::CONSTANTS_BLOCK_ID,
-                                   Abbv) != CONSTANTS_SETTYPE_ABBREV)
-      llvm_unreachable("Unexpected abbrev ordering!");
-  }
-
-  { // INTEGER abbrev for CONSTANTS_BLOCK.
-    NaClBitCodeAbbrev *Abbv = new NaClBitCodeAbbrev();
-    Abbv->Add(NaClBitCodeAbbrevOp(naclbitc::CST_CODE_INTEGER));
-    Abbv->Add(NaClBitCodeAbbrevOp(NaClBitCodeAbbrevOp::VBR, 8));
-    if (Stream.EmitBlockInfoAbbrev(naclbitc::CONSTANTS_BLOCK_ID,
-                                   Abbv) != CONSTANTS_INTEGER_ABBREV)
-      llvm_unreachable("Unexpected abbrev ordering!");
-  }
-  { // INTEGER_ZERO abbrev for CONSTANTS_BLOCK.
-    NaClBitCodeAbbrev *Abbv = new NaClBitCodeAbbrev();
-    Abbv->Add(NaClBitCodeAbbrevOp(naclbitc::CST_CODE_INTEGER));
-    Abbv->Add(NaClBitCodeAbbrevOp(0));
-    if (Stream.EmitBlockInfoAbbrev(naclbitc::CONSTANTS_BLOCK_ID,
-                                   Abbv) != CONSTANTS_INTEGER_ZERO_ABBREV)
-      llvm_unreachable("Unexpected abbrev ordering!");
-  }
-  { // FLOAT abbrev for CONSTANTS_BLOCK.
-    NaClBitCodeAbbrev *Abbv = new NaClBitCodeAbbrev();
-    Abbv->Add(NaClBitCodeAbbrevOp(naclbitc::CST_CODE_FLOAT));
-    Abbv->Add(NaClBitCodeAbbrevOp(NaClBitCodeAbbrevOp::VBR, 8));
-    if (Stream.EmitBlockInfoAbbrev(naclbitc::CONSTANTS_BLOCK_ID,
-                                   Abbv) != CONSTANTS_FLOAT_ABBREV)
-      llvm_unreachable("Unexpected abbrev ordering!");
-  }
-
-  // FIXME: This should only use space for first class types!
-
-  { // INST_LOAD abbrev for FUNCTION_BLOCK.
-    NaClBitCodeAbbrev *Abbv = new NaClBitCodeAbbrev();
-    Abbv->Add(NaClBitCodeAbbrevOp(naclbitc::FUNC_CODE_INST_LOAD));
-    Abbv->Add(NaClBitCodeAbbrevOp(NaClBitCodeAbbrevOp::VBR, 6)); // Ptr
-    Abbv->Add(NaClBitCodeAbbrevOp(NaClBitCodeAbbrevOp::VBR, 4)); // Align
-    // Note: The vast majority of load operations are only on integers
-    // and floats. In addition, no function types are allowed. In
-    // addition, the type IDs have been sorted based on usage, moving
-    // type IDs associated integers and floats to very low
-    // indices. Hence, we assume that we can use a smaller width for
-    // the typecast.
-    Abbv->Add(NaClBitCodeAbbrevOp(NaClBitCodeAbbrevOp::VBR, 4)); // TypeCast
-    if (Stream.EmitBlockInfoAbbrev(naclbitc::FUNCTION_BLOCK_ID,
-                                   Abbv) != FUNCTION_INST_LOAD_ABBREV)
-      llvm_unreachable("Unexpected abbrev ordering!");
-  }
-  { // INST_BINOP abbrev for FUNCTION_BLOCK.
-    NaClBitCodeAbbrev *Abbv = new NaClBitCodeAbbrev();
-    Abbv->Add(NaClBitCodeAbbrevOp(naclbitc::FUNC_CODE_INST_BINOP));
-    Abbv->Add(NaClBitCodeAbbrevOp(NaClBitCodeAbbrevOp::VBR, 6)); // LHS
-    Abbv->Add(NaClBitCodeAbbrevOp(NaClBitCodeAbbrevOp::VBR, 6)); // RHS
-    Abbv->Add(NaClBitCodeAbbrevOp(NaClBitCodeAbbrevOp::Fixed, 4)); // opc
-    if (Stream.EmitBlockInfoAbbrev(naclbitc::FUNCTION_BLOCK_ID,
-                                   Abbv) != FUNCTION_INST_BINOP_ABBREV)
-      llvm_unreachable("Unexpected abbrev ordering!");
-  }
-  { // INST_CAST abbrev for FUNCTION_BLOCK.
-    NaClBitCodeAbbrev *Abbv = new NaClBitCodeAbbrev();
-    Abbv->Add(NaClBitCodeAbbrevOp(naclbitc::FUNC_CODE_INST_CAST));
-    Abbv->Add(NaClBitCodeAbbrevOp(NaClBitCodeAbbrevOp::VBR, 6));    // OpVal
-    Abbv->Add(NaClBitCodeAbbrevOp(TypeIdEncoding, TypeIdNumBits));  // dest ty
-    Abbv->Add(NaClBitCodeAbbrevOp(NaClBitCodeAbbrevOp::Fixed, 4));  // opc
-    if (Stream.EmitBlockInfoAbbrev(naclbitc::FUNCTION_BLOCK_ID,
-                                   Abbv) != FUNCTION_INST_CAST_ABBREV)
-      llvm_unreachable("Unexpected abbrev ordering!");
-  }
-
-  { // INST_RET abbrev for FUNCTION_BLOCK.
-    NaClBitCodeAbbrev *Abbv = new NaClBitCodeAbbrev();
-    Abbv->Add(NaClBitCodeAbbrevOp(naclbitc::FUNC_CODE_INST_RET));
-    if (Stream.EmitBlockInfoAbbrev(naclbitc::FUNCTION_BLOCK_ID,
-                                   Abbv) != FUNCTION_INST_RET_VOID_ABBREV)
-      llvm_unreachable("Unexpected abbrev ordering!");
-  }
-  { // INST_RET abbrev for FUNCTION_BLOCK.
-    NaClBitCodeAbbrev *Abbv = new NaClBitCodeAbbrev();
-    Abbv->Add(NaClBitCodeAbbrevOp(naclbitc::FUNC_CODE_INST_RET));
-    Abbv->Add(NaClBitCodeAbbrevOp(NaClBitCodeAbbrevOp::VBR, 6)); // ValID
-    if (Stream.EmitBlockInfoAbbrev(naclbitc::FUNCTION_BLOCK_ID,
-                                   Abbv) != FUNCTION_INST_RET_VAL_ABBREV)
-      llvm_unreachable("Unexpected abbrev ordering!");
-  }
-  { // INST_UNREACHABLE abbrev for FUNCTION_BLOCK.
-    NaClBitCodeAbbrev *Abbv = new NaClBitCodeAbbrev();
-    Abbv->Add(NaClBitCodeAbbrevOp(naclbitc::FUNC_CODE_INST_UNREACHABLE));
-    if (Stream.EmitBlockInfoAbbrev(naclbitc::FUNCTION_BLOCK_ID,
-                                   Abbv) != FUNCTION_INST_UNREACHABLE_ABBREV)
-      llvm_unreachable("Unexpected abbrev ordering!");
-  }
-  { // INST_FORWARDTYPEREF abbrev for FUNCTION_BLOCK.
-    NaClBitCodeAbbrev *Abbv = new NaClBitCodeAbbrev();
-    Abbv->Add(NaClBitCodeAbbrevOp(naclbitc::FUNC_CODE_INST_FORWARDTYPEREF));
-    Abbv->Add(NaClBitCodeAbbrevOp(NaClBitCodeAbbrevOp::VBR, 6));
-    Abbv->Add(NaClBitCodeAbbrevOp(TypeIdEncoding, TypeIdNumBits));
-    if (Stream.EmitBlockInfoAbbrev(naclbitc::FUNCTION_BLOCK_ID,
-                                   Abbv) != FUNCTION_INST_FORWARDTYPEREF_ABBREV)
-      llvm_unreachable("Unexpected abbrev ordering!");
-  }
-  { // INST_STORE abbrev for FUNCTION_BLOCK.
-    NaClBitCodeAbbrev *Abbv = new NaClBitCodeAbbrev();
-    Abbv->Add(NaClBitCodeAbbrevOp(naclbitc::FUNC_CODE_INST_STORE));
-    Abbv->Add(NaClBitCodeAbbrevOp(NaClBitCodeAbbrevOp::VBR, 6)); // Ptr
-    Abbv->Add(NaClBitCodeAbbrevOp(NaClBitCodeAbbrevOp::VBR, 6)); // Value
-    Abbv->Add(NaClBitCodeAbbrevOp(NaClBitCodeAbbrevOp::VBR, 4)); // Align
-    if (Stream.EmitBlockInfoAbbrev(naclbitc::FUNCTION_BLOCK_ID,
-                                   Abbv) != FUNCTION_INST_STORE_ABBREV)
-      llvm_unreachable("Unexpected abbrev ordering!");
-  }
-
-  { // VAR abbrev for GLOBALVAR_BLOCK.
-    NaClBitCodeAbbrev *Abbv = new NaClBitCodeAbbrev();
-    Abbv->Add(NaClBitCodeAbbrevOp(naclbitc::GLOBALVAR_VAR));
-    Abbv->Add(NaClBitCodeAbbrevOp(NaClBitCodeAbbrevOp::VBR, 6));
-    Abbv->Add(NaClBitCodeAbbrevOp(NaClBitCodeAbbrevOp::Fixed, 1));
-    if (Stream.EmitBlockInfoAbbrev(naclbitc::GLOBALVAR_BLOCK_ID,
-                                   Abbv) != GLOBALVAR_VAR_ABBREV)
-      llvm_unreachable("Unexpected abbrev ordering!");
-  }
-  { // COMPOUND abbrev for GLOBALVAR_BLOCK.
-    NaClBitCodeAbbrev *Abbv = new NaClBitCodeAbbrev();
-    Abbv->Add(NaClBitCodeAbbrevOp(naclbitc::GLOBALVAR_COMPOUND));
-    Abbv->Add(NaClBitCodeAbbrevOp(NaClBitCodeAbbrevOp::VBR, 8));
-    if (Stream.EmitBlockInfoAbbrev(naclbitc::GLOBALVAR_BLOCK_ID,
-                                   Abbv) != GLOBALVAR_COMPOUND_ABBREV)
-      llvm_unreachable("Unexpected abbrev ordering!");
-  }
-  { // ZEROFILL abbrev for GLOBALVAR_BLOCK.
-    NaClBitCodeAbbrev *Abbv = new NaClBitCodeAbbrev();
-    Abbv->Add(NaClBitCodeAbbrevOp(naclbitc::GLOBALVAR_ZEROFILL));
-    Abbv->Add(NaClBitCodeAbbrevOp(NaClBitCodeAbbrevOp::VBR, 8));
-    if (Stream.EmitBlockInfoAbbrev(naclbitc::GLOBALVAR_BLOCK_ID,
-                                   Abbv) != GLOBALVAR_ZEROFILL_ABBREV)
-      llvm_unreachable("Unexpected abbrev ordering!");
-  }
-  { // DATA abbrev for GLOBALVAR_BLOCK.
-    NaClBitCodeAbbrev *Abbv = new NaClBitCodeAbbrev();
-    Abbv->Add(NaClBitCodeAbbrevOp(naclbitc::GLOBALVAR_DATA));
-    Abbv->Add(NaClBitCodeAbbrevOp(NaClBitCodeAbbrevOp::Array));
-    Abbv->Add(NaClBitCodeAbbrevOp(NaClBitCodeAbbrevOp::Fixed, 8));
-    if (Stream.EmitBlockInfoAbbrev(naclbitc::GLOBALVAR_BLOCK_ID,
-                                   Abbv) != GLOBALVAR_DATA_ABBREV)
-      llvm_unreachable("Unexpected abbrev ordering!");
-  }
-  { // RELOC abbrev for GLOBALVAR_BLOCK.
-    NaClBitCodeAbbrev *Abbv = new NaClBitCodeAbbrev();
-    Abbv->Add(NaClBitCodeAbbrevOp(naclbitc::GLOBALVAR_RELOC));
-    Abbv->Add(NaClBitCodeAbbrevOp(NaClBitCodeAbbrevOp::VBR, 6));
-    if (Stream.EmitBlockInfoAbbrev(naclbitc::GLOBALVAR_BLOCK_ID,
-                                   Abbv) != GLOBALVAR_RELOC_ABBREV)
-      llvm_unreachable("Unexpected abbrev ordering!");
-  }
-  { // RELOC_WITH_ADDEND_ABBREV abbrev for GLOBALVAR_BLOCK.
-    NaClBitCodeAbbrev *Abbv = new NaClBitCodeAbbrev();
-    Abbv->Add(NaClBitCodeAbbrevOp(naclbitc::GLOBALVAR_RELOC));
-    Abbv->Add(NaClBitCodeAbbrevOp(NaClBitCodeAbbrevOp::VBR, 6));
-    Abbv->Add(NaClBitCodeAbbrevOp(NaClBitCodeAbbrevOp::VBR, 6));
-    if (Stream.EmitBlockInfoAbbrev(
-            naclbitc::GLOBALVAR_BLOCK_ID,
-            Abbv) != GLOBALVAR_RELOC_WITH_ADDEND_ABBREV)
-      llvm_unreachable("Unexpected abbrev ordering!");
-  }
-
-  Stream.ExitBlock();
-}
-
-/// WriteModule - Emit the specified module to the bitstream.
-static void WriteModule(const Module *M, NaClBitstreamWriter &Stream) {
-  DEBUG(dbgs() << "-> WriteModule\n");
-  Stream.EnterSubblock(naclbitc::MODULE_BLOCK_ID);
-
-  SmallVector<unsigned, 1> Vals;
-  unsigned CurVersion = 1;
-  Vals.push_back(CurVersion);
-  Stream.EmitRecord(naclbitc::MODULE_CODE_VERSION, Vals);
-
-  // Analyze the module, enumerating globals, functions, etc.
-  NaClValueEnumerator VE(M);
-  OptimizeTypeIdEncoding(VE);
-
-  // Emit blockinfo, which defines the standard abbreviations etc.
-  WriteBlockInfo(VE, Stream);
-
-  // Emit information describing all of the types in the module.
-  WriteTypeTable(VE, Stream);
-
-  // Emit top-level description of module, including inline asm,
-  // descriptors for global variables, and function prototype info.
-  WriteModuleInfo(M, VE, Stream);
-
-  // Emit names for globals/functions etc.
-  WriteValueSymbolTable(M->getValueSymbolTable(), VE, Stream);
-
-  // Emit function bodies.
-  for (Module::const_iterator F = M->begin(), E = M->end(); F != E; ++F)
-    if (!F->isDeclaration())
-      WriteFunction(*F, VE, Stream);
-
-  Stream.ExitBlock();
-  DEBUG(dbgs() << "<- WriteModule\n");
-}
-
-// Max size for variable fields. Currently only used for writing them
-// out to files (the parsing works for arbitrary sizes).
-static const size_t kMaxVariableFieldSize = 256;
-
-void llvm::NaClWriteHeader(NaClBitstreamWriter &Stream,
-                           bool AcceptSupportedOnly) {
-  NaClBitcodeHeader Header;
-  Header.push_back(
-      new NaClBitcodeHeaderField(NaClBitcodeHeaderField::kPNaClVersion,
-                                 PNaClVersion));
-  if (AlignBitcodeRecords)
-    Header.push_back(new NaClBitcodeHeaderField(
-        NaClBitcodeHeaderField::kAlignBitcodeRecords));
-
-  Header.InstallFields();
-  if (!(Header.IsSupported() ||
-        (!AcceptSupportedOnly && Header.IsReadable()))) {
-    report_fatal_error(Header.Unsupported());
-  }
-  NaClWriteHeader(Header, Stream);
-}
-
-// Write out the given Header to the bitstream.
-void llvm::NaClWriteHeader(const NaClBitcodeHeader &Header,
-                           NaClBitstreamWriter &Stream) {
-  Stream.initFromHeader(Header);
-
-  // Emit the file magic number;
-  Stream.Emit((unsigned)'P', 8);
-  Stream.Emit((unsigned)'E', 8);
-  Stream.Emit((unsigned)'X', 8);
-  Stream.Emit((unsigned)'E', 8);
-
-  // Emit placeholder for number of bytes used to hold header fields.
-  // This value is necessary so that the streamable reader can preallocate
-  // a buffer to read the fields.
-  Stream.Emit(0, naclbitc::BlockSizeWidth);
-  unsigned BytesForHeader = 0;
-
-  unsigned NumberFields = Header.NumberFields();
-  if (NumberFields > 0xFFFF)
-    report_fatal_error("Too many header fields");
-
-  uint8_t Buffer[kMaxVariableFieldSize];
-  for (unsigned F = 0; F < NumberFields; ++F) {
-    NaClBitcodeHeaderField *Field = Header.GetField(F);
-    if (!Field->Write(Buffer, kMaxVariableFieldSize))
-      report_fatal_error("Header field too big to generate");
-    size_t limit = Field->GetTotalSize();
-    for (size_t i = 0; i < limit; i++) {
-      Stream.Emit(Buffer[i], 8);
-    }
-    BytesForHeader += limit;
-  }
-
-  if (BytesForHeader > 0xFFFF)
-    report_fatal_error("Header fields to big to save");
-
-  // Encode #fields in top two bytes, and #bytes to hold fields in
-  // bottom two bytes. Then backpatch into second word.
-  unsigned Value = NumberFields | (BytesForHeader << 16);
-  Stream.BackpatchWord(NaClBitcodeHeader::WordSize, Value);
-}
-
-/// WriteBitcodeToFile - Write the specified module to the specified output
-/// stream.
-void llvm::NaClWriteBitcodeToFile(const Module *M, raw_ostream &Out,
-                                  bool AcceptSupportedOnly) {
-  SmallVector<char, 0> Buffer;
-  Buffer.reserve(256*1024);
-
-  // Emit the module into the buffer.
-  {
-    NaClBitstreamWriter Stream(Buffer);
-    NaClWriteHeader(Stream, AcceptSupportedOnly);
-    WriteModule(M, Stream);
-  }
-
-  // Write the generated bitstream to "Out".
-  Out.write((char*)&Buffer.front(), Buffer.size());
-}
diff --git a/lib/Bitcode/NaCl/Writer/NaClBitcodeWriterPass.cpp b/lib/Bitcode/NaCl/Writer/NaClBitcodeWriterPass.cpp
deleted file mode 100644
index 9f1f9aa9b80..00000000000
--- a/lib/Bitcode/NaCl/Writer/NaClBitcodeWriterPass.cpp
+++ /dev/null
@@ -1,47 +0,0 @@
-//===- NaClBitcodeWriterPass.cpp - Bitcode writing pass -------------------===//
-//
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
-//
-//===----------------------------------------------------------------------===//
-//
-// NaClBitcodeWriterPass implementation.
-//
-//===----------------------------------------------------------------------===//
-
-#include "llvm/Bitcode/NaCl/NaClBitcodeWriterPass.h"
-#include "llvm/Bitcode/NaCl/NaClReaderWriter.h"
-#include "llvm/IR/Module.h"
-#include "llvm/IR/PassManager.h"
-#include "llvm/Pass.h"
-using namespace llvm;
-
-PreservedAnalyses NaClBitcodeWriterPass::run(Module *M) {
-  NaClWriteBitcodeToFile(M, OS);
-  return PreservedAnalyses::all();
-}
-
-namespace {
-  class NaClWriteBitcodePass : public ModulePass {
-    raw_ostream &OS; // raw_ostream to print on
-  public:
-    static char ID; // Pass identification, replacement for typeid
-    explicit NaClWriteBitcodePass(raw_ostream &o)
-      : ModulePass(ID), OS(o) {}
-
-    const char *getPassName() const override { return "NaCl Bitcode Writer"; }
-
-    bool runOnModule(Module &M) override {
-      NaClWriteBitcodeToFile(&M, OS);
-      return false;
-    }
-  };
-}
-
-char NaClWriteBitcodePass::ID = 0;
-
-ModulePass *llvm::createNaClBitcodeWriterPass(raw_ostream &Str) {
-  return new NaClWriteBitcodePass(Str);
-}
diff --git a/lib/Bitcode/NaCl/Writer/NaClValueEnumerator.cpp b/lib/Bitcode/NaCl/Writer/NaClValueEnumerator.cpp
deleted file mode 100644
index bdc7eee99ec..00000000000
--- a/lib/Bitcode/NaCl/Writer/NaClValueEnumerator.cpp
+++ /dev/null
@@ -1,503 +0,0 @@
-//===-- NaClValueEnumerator.cpp ------------------------------------------===//
-//     Number values and types for bitcode writer
-//
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
-//
-//===----------------------------------------------------------------------===//
-//
-// This file implements the NaClValueEnumerator class.
-//
-//===----------------------------------------------------------------------===//
-
-#include "NaClValueEnumerator.h"
-#include "llvm/ADT/STLExtras.h"
-#include "llvm/ADT/SmallPtrSet.h"
-#include "llvm/IR/Constants.h"
-#include "llvm/IR/DerivedTypes.h"
-#include "llvm/IR/Instructions.h"
-#include "llvm/IR/IntrinsicInst.h"
-#include "llvm/IR/Module.h"
-#include "llvm/IR/ValueSymbolTable.h"
-#include "llvm/Support/Debug.h"
-#include "llvm/Support/raw_ostream.h"
-#include <algorithm>
-#include <set>
-
-using namespace llvm;
-
-static bool isIntOrIntVectorValue(const std::pair<const Value*, unsigned> &V) {
-  return V.first->getType()->isIntOrIntVectorTy();
-}
-
-/// NaClValueEnumerator - Enumerate module-level information.
-NaClValueEnumerator::NaClValueEnumerator(const Module *M) {
-  // Create map for counting frequency of types, and set field
-  // TypeCountMap accordingly.  Note: Pointer field TypeCountMap is
-  // used to deal with the fact that types are added through various
-  // method calls in this routine. Rather than pass it as an argument,
-  // we use a field. The field is a pointer so that the memory
-  // footprint of count_map can be garbage collected when this
-  // constructor completes.
-  TypeCountMapType count_map;
-  TypeCountMap = &count_map;
-
-  IntPtrType = IntegerType::get(M->getContext(), PNaClIntPtrTypeBitSize);
-
-  // Enumerate the functions. Note: We do this before global
-  // variables, so that global variable initializations can refer to
-  // the functions without a forward reference.
-  for (Module::const_iterator I = M->begin(), E = M->end(); I != E; ++I) {
-    EnumerateValue(I);
-  }
-
-  // Enumerate the global variables.
-  FirstGlobalVarID = Values.size();
-  for (Module::const_global_iterator I = M->global_begin(),
-         E = M->global_end(); I != E; ++I)
-    EnumerateValue(I);
-  NumGlobalVarIDs = Values.size() - FirstGlobalVarID;
-
-  // Enumerate the aliases.
-  for (Module::const_alias_iterator I = M->alias_begin(), E = M->alias_end();
-       I != E; ++I)
-    EnumerateValue(I);
-
-  // Remember what is the cutoff between globalvalue's and other constants.
-  unsigned FirstConstant = Values.size();
-
-  // Skip global variable initializers since they are handled within
-  // WriteGlobalVars of file NaClBitcodeWriter.cpp.
-
-  // Enumerate the aliasees.
-  for (Module::const_alias_iterator I = M->alias_begin(), E = M->alias_end();
-       I != E; ++I)
-    EnumerateValue(I->getAliasee());
-
-  // Insert constants that are named at module level into the slot
-  // pool so that the module symbol table can refer to them...
-  EnumerateValueSymbolTable(M->getValueSymbolTable());
-
-  // Enumerate types used by function bodies and argument lists.
-  for (Module::const_iterator F = M->begin(), E = M->end(); F != E; ++F) {
-
-    for (Function::const_arg_iterator I = F->arg_begin(), E = F->arg_end();
-         I != E; ++I)
-      EnumerateType(I->getType());
-
-    for (Function::const_iterator BB = F->begin(), E = F->end(); BB != E; ++BB)
-      for (BasicBlock::const_iterator I = BB->begin(), E = BB->end(); I!=E;++I){
-        // Don't generate types for elided pointer casts!
-        if (IsElidedCast(I))
-          continue;
-
-        if (const SwitchInst *SI = dyn_cast<SwitchInst>(I)) {
-          // Handle switch instruction specially, so that we don't
-          // write out unnecessary vector/array types used to model case
-          // selectors.
-          EnumerateOperandType(SI->getCondition());
-        } else {
-          for (User::const_op_iterator OI = I->op_begin(), E = I->op_end();
-               OI != E; ++OI) {
-            EnumerateOperandType(*OI);
-          }
-        }
-        EnumerateType(I->getType());
-      }
-  }
-
-  // Optimized type indicies to put "common" expected types in with small
-  // indices.
-  OptimizeTypes(M);
-  TypeCountMap = NULL;
-
-  // Optimize constant ordering.
-  OptimizeConstants(FirstConstant, Values.size());
-}
-
-void NaClValueEnumerator::OptimizeTypes(const Module *M) {
-
-  // Sort types by count, so that we can index them based on
-  // frequency. Use indices of built TypeMap, so that order of
-  // construction is repeatable.
-  std::set<unsigned> type_counts;
-  typedef std::set<unsigned> TypeSetType;
-  std::map<unsigned, TypeSetType> usage_count_map;
-  TypeList IdType(Types);
-
-  for (TypeCountMapType::iterator iter = TypeCountMap->begin();
-       iter != TypeCountMap->end(); ++ iter) {
-    type_counts.insert(iter->second);
-    usage_count_map[iter->second].insert(TypeMap[iter->first]-1);
-  }
-
-  // Reset type tracking maps, so that we can re-enter based
-  // on fequency ordering.
-  TypeCountMap = NULL;
-  Types.clear();
-  TypeMap.clear();
-
-  // Reinsert types, based on frequency.
-  for (std::set<unsigned>::reverse_iterator count_iter = type_counts.rbegin();
-       count_iter != type_counts.rend(); ++count_iter) {
-    TypeSetType& count_types = usage_count_map[*count_iter];
-    for (TypeSetType::iterator type_iter = count_types.begin();
-         type_iter != count_types.end(); ++type_iter)
-      EnumerateType((IdType[*type_iter]), true);
-  }
-}
-
-unsigned NaClValueEnumerator::getInstructionID(const Instruction *Inst) const {
-  InstructionMapType::const_iterator I = InstructionMap.find(Inst);
-  assert(I != InstructionMap.end() && "Instruction is not mapped!");
-  return I->second;
-}
-
-void NaClValueEnumerator::setInstructionID(const Instruction *I) {
-  InstructionMap[I] = InstructionCount++;
-}
-
-unsigned NaClValueEnumerator::getValueID(const Value *V) const {
-  ValueMapType::const_iterator I = ValueMap.find(V);
-  assert(I != ValueMap.end() && "Value not in slotcalculator!");
-  return I->second-1;
-}
-
-void NaClValueEnumerator::dump() const {
-  print(dbgs(), ValueMap, "Default");
-  dbgs() << '\n';
-}
-
-void NaClValueEnumerator::print(raw_ostream &OS, const ValueMapType &Map,
-                            const char *Name) const {
-
-  OS << "Map Name: " << Name << "\n";
-  OS << "Size: " << Map.size() << "\n";
-  for (ValueMapType::const_iterator I = Map.begin(),
-         E = Map.end(); I != E; ++I) {
-
-    const Value *V = I->first;
-    if (V->hasName())
-      OS << "Value: " << V->getName();
-    else
-      OS << "Value: [null]\n";
-    V->dump();
-
-    OS << " Uses(" << std::distance(V->use_begin(),V->use_end()) << "):";
-    for (Value::const_use_iterator UI = V->use_begin(), UE = V->use_end();
-         UI != UE; ++UI) {
-      if (UI != V->use_begin())
-        OS << ",";
-      if((*UI)->hasName())
-        OS << " " << (*UI)->getName();
-      else
-        OS << " [null]";
-
-    }
-    OS <<  "\n\n";
-  }
-}
-
-// Optimize constant ordering.
-namespace {
-  struct CstSortPredicate {
-    NaClValueEnumerator &VE;
-    explicit CstSortPredicate(NaClValueEnumerator &ve) : VE(ve) {}
-    bool operator()(const std::pair<const Value*, unsigned> &LHS,
-                    const std::pair<const Value*, unsigned> &RHS) {
-      // Sort by plane.
-      if (LHS.first->getType() != RHS.first->getType())
-        return VE.getTypeID(LHS.first->getType()) <
-               VE.getTypeID(RHS.first->getType());
-      // Then by frequency.
-      return LHS.second > RHS.second;
-    }
-  };
-}
-
-/// OptimizeConstants - Reorder constant pool for denser encoding.
-void NaClValueEnumerator::OptimizeConstants(unsigned CstStart, unsigned CstEnd) {
-  if (CstStart == CstEnd || CstStart+1 == CstEnd) return;
-
-  CstSortPredicate P(*this);
-  std::stable_sort(Values.begin()+CstStart, Values.begin()+CstEnd, P);
-
-  // Ensure that integer and vector of integer constants are at the start of the
-  // constant pool.  This is important so that GEP structure indices come before
-  // gep constant exprs.
-  std::partition(Values.begin()+CstStart, Values.begin()+CstEnd,
-                 isIntOrIntVectorValue);
-
-  // Rebuild the modified portion of ValueMap.
-  for (; CstStart != CstEnd; ++CstStart)
-    ValueMap[Values[CstStart].first] = CstStart+1;
-}
-
-
-/// EnumerateValueSymbolTable - Insert all of the values in the specified symbol
-/// table into the values table.
-void NaClValueEnumerator::EnumerateValueSymbolTable(const ValueSymbolTable &VST) {
-  for (ValueSymbolTable::const_iterator VI = VST.begin(), VE = VST.end();
-       VI != VE; ++VI)
-    EnumerateValue(VI->getValue());
-}
-
-void NaClValueEnumerator::EnumerateValue(const Value *VIn) {
-  // Skip over elided values.
-  const Value *V = ElideCasts(VIn);
-  if (V != VIn) return;
-
-  assert(!V->getType()->isVoidTy() && "Can't insert void values!");
-
-  // Check to see if it's already in!
-  unsigned &ValueID = ValueMap[V];
-  if (ValueID) {
-    // Increment use count.
-    Values[ValueID-1].second++;
-    return;
-  }
-
-  // Enumerate the type of this value. Skip global values since no
-  // types are dumped for global variables.
-  if (!isa<GlobalVariable>(V))
-    EnumerateType(V->getType());
-
-  if (const Constant *C = dyn_cast<Constant>(V)) {
-    if (isa<GlobalValue>(C)) {
-      // Initializers for globals are handled explicitly elsewhere.
-    } else if (C->getNumOperands()) {
-      // If a constant has operands, enumerate them.  This makes sure that if a
-      // constant has uses (for example an array of const ints), that they are
-      // inserted also.
-
-      // We prefer to enumerate them with values before we enumerate the user
-      // itself.  This makes it more likely that we can avoid forward references
-      // in the reader.  We know that there can be no cycles in the constants
-      // graph that don't go through a global variable.
-      for (User::const_op_iterator I = C->op_begin(), E = C->op_end();
-           I != E; ++I)
-        if (!isa<BasicBlock>(*I)) // Don't enumerate BB operand to BlockAddress.
-          EnumerateValue(*I);
-
-      // Finally, add the value.  Doing this could make the ValueID reference be
-      // dangling, don't reuse it.
-      Values.push_back(std::make_pair(V, 1U));
-      ValueMap[V] = Values.size();
-      return;
-    }
-  }
-
-  // Add the value.
-  Values.push_back(std::make_pair(V, 1U));
-  ValueID = Values.size();
-}
-
-
-Type *NaClValueEnumerator::NormalizeType(Type *Ty) const {
-  if (Ty->isPointerTy())
-    return IntPtrType;
-  if (FunctionType *FTy = dyn_cast<FunctionType>(Ty)) {
-    SmallVector<Type *, 8> ArgTypes;
-    for (unsigned I = 0, E = FTy->getNumParams(); I < E; ++I)
-      ArgTypes.push_back(NormalizeType(FTy->getParamType(I)));
-    return FunctionType::get(NormalizeType(FTy->getReturnType()),
-                             ArgTypes, false);
-  }
-  return Ty;
-}
-
-void NaClValueEnumerator::EnumerateType(Type *Ty, bool InsideOptimizeTypes) {
-  // Pointer types do not need to be given type IDs.
-  if (Ty->isPointerTy())
-    Ty = Ty->getPointerElementType();
-
-  Ty = NormalizeType(Ty);
-
-  // The label type does not need to be given a type ID.
-  if (Ty->isLabelTy())
-    return;
-
-  // This function is used to enumerate types referenced by the given
-  // module. This function is called in two phases, based on the value
-  // of TypeCountMap. These phases are:
-  //
-  // (1) In this phase, InsideOptimizeTypes=false. We are collecting types
-  // and all corresponding (implicitly) referenced types. In addition,
-  // we are keeping track of the number of references to each type in
-  // TypeCountMap. These reference counts will be used by method
-  // OptimizeTypes to associate the smallest type ID's with the most
-  // referenced types.
-  //
-  // (2) In this phase, InsideOptimizeTypes=true. We are registering types
-  // based on frequency. To minimize type IDs for frequently used
-  // types, (unlike the other context) we are inserting the minimal
-  // (implicitly) referenced types needed for each type.
-  unsigned *TypeID = &TypeMap[Ty];
-
-  if (TypeCountMap) ++((*TypeCountMap)[Ty]);
-
-  // We've already seen this type.
-  if (*TypeID)
-    return;
-
-  // If it is a non-anonymous struct, mark the type as being visited so that we
-  // don't recursively visit it.  This is safe because we allow forward
-  // references of these in the bitcode reader.
-  if (StructType *STy = dyn_cast<StructType>(Ty))
-    if (!STy->isLiteral())
-      *TypeID = ~0U;
-
-  // If in the second phase (i.e. inside optimize types), don't expand
-  // pointers to structures, since we can just generate a forward
-  // reference to it. This way, we don't use up unnecessary (small) ID
-  // values just to define the pointer.
-  bool EnumerateSubtypes = true;
-  if (InsideOptimizeTypes)
-    if (PointerType *PTy = dyn_cast<PointerType>(Ty))
-      if (StructType *STy = dyn_cast<StructType>(PTy->getElementType()))
-        if (!STy->isLiteral())
-          EnumerateSubtypes = false;
-
-  // Enumerate all of the subtypes before we enumerate this type.  This ensures
-  // that the type will be enumerated in an order that can be directly built.
-  if (EnumerateSubtypes) {
-    for (Type::subtype_iterator I = Ty->subtype_begin(), E = Ty->subtype_end();
-         I != E; ++I)
-      EnumerateType(*I, InsideOptimizeTypes);
-  }
-
-  // Refresh the TypeID pointer in case the table rehashed.
-  TypeID = &TypeMap[Ty];
-
-  // Check to see if we got the pointer another way.  This can happen when
-  // enumerating recursive types that hit the base case deeper than they start.
-  //
-  // If this is actually a struct that we are treating as forward ref'able,
-  // then emit the definition now that all of its contents are available.
-  if (*TypeID && *TypeID != ~0U)
-    return;
-
-  // Add this type now that its contents are all happily enumerated.
-  Types.push_back(Ty);
-
-  *TypeID = Types.size();
-}
-
-// Enumerate the types for the specified value.  If the value is a constant,
-// walk through it, enumerating the types of the constant.
-void NaClValueEnumerator::EnumerateOperandType(const Value *V) {
-  // Note: We intentionally don't create a type id for global variables,
-  // since the type is automatically generated by the reader before any
-  // use of the global variable.
-  if (isa<GlobalVariable>(V)) return;
-
-  EnumerateType(V->getType());
-
-  if (const Constant *C = dyn_cast<Constant>(V)) {
-    // If this constant is already enumerated, ignore it, we know its type must
-    // be enumerated.
-    if (ValueMap.count(V)) return;
-
-    // This constant may have operands, make sure to enumerate the types in
-    // them.
-    for (unsigned i = 0, e = C->getNumOperands(); i != e; ++i) {
-      const Value *Op = C->getOperand(i);
-
-      // Don't enumerate basic blocks here, this happens as operands to
-      // blockaddress.
-      if (isa<BasicBlock>(Op)) continue;
-
-      EnumerateOperandType(Op);
-    }
-  }
-}
-
-void NaClValueEnumerator::incorporateFunction(const Function &F) {
-  InstructionCount = 0;
-  NumModuleValues = Values.size();
-
-  // Make sure no insertions outside of a function.
-  assert(FnForwardTypeRefs.empty());
-
-  // Adding function arguments to the value table.
-  for (Function::const_arg_iterator I = F.arg_begin(), E = F.arg_end();
-       I != E; ++I)
-    EnumerateValue(I);
-
-  FirstFuncConstantID = Values.size();
-
-  // Add all function-level constants to the value table.
-  for (Function::const_iterator BB = F.begin(), E = F.end(); BB != E; ++BB) {
-    for (BasicBlock::const_iterator I = BB->begin(), E = BB->end(); I!=E; ++I) {
-      if (const SwitchInst *SI = dyn_cast<SwitchInst>(I)) {
-        // Handle switch instruction specially, so that we don't write
-        // out unnecessary vector/array constants used to model case selectors.
-        if (isa<Constant>(SI->getCondition())) {
-          EnumerateValue(SI->getCondition());
-        }
-      } else {
-        for (User::const_op_iterator OI = I->op_begin(), E = I->op_end();
-             OI != E; ++OI) {
-          if ((isa<Constant>(*OI) && !isa<GlobalValue>(*OI)) ||
-              isa<InlineAsm>(*OI))
-            EnumerateValue(*OI);
-        }
-      }
-    }
-    BasicBlocks.push_back(BB);
-    ValueMap[BB] = BasicBlocks.size();
-  }
-
-  // Optimize the constant layout.
-  OptimizeConstants(FirstFuncConstantID, Values.size());
-
-  FirstInstID = Values.size();
-
-  // Add all of the instructions.
-  for (Function::const_iterator BB = F.begin(), E = F.end(); BB != E; ++BB) {
-    for (BasicBlock::const_iterator I = BB->begin(), E = BB->end(); I!=E; ++I) {
-      if (!I->getType()->isVoidTy())
-        EnumerateValue(I);
-    }
-  }
-}
-
-void NaClValueEnumerator::purgeFunction() {
-  /// Remove purged values from the ValueMap.
-  for (unsigned i = NumModuleValues, e = Values.size(); i != e; ++i)
-    ValueMap.erase(Values[i].first);
-  for (unsigned i = 0, e = BasicBlocks.size(); i != e; ++i)
-    ValueMap.erase(BasicBlocks[i]);
-
-  Values.resize(NumModuleValues);
-  BasicBlocks.clear();
-  FnForwardTypeRefs.clear();
-}
-
-// The normal form required by the PNaCl ABI verifier (documented in
-// ReplacePtrsWithInts.cpp) allows us to omit the following pointer
-// casts from the bitcode file.
-const Value *NaClValueEnumerator::ElideCasts(const Value *V) const {
-  if (const Instruction *I = dyn_cast<Instruction>(V)) {
-    switch (I->getOpcode()) {
-    default:
-      break;
-    case Instruction::BitCast:
-      if (I->getType()->isPointerTy()) {
-        V = I->getOperand(0);
-      }
-      break;
-    case Instruction::IntToPtr:
-      V = ElideCasts(I->getOperand(0));
-      break;
-    case Instruction::PtrToInt:
-      if (IsIntPtrType(I->getType())) {
-        V = I->getOperand(0);
-      }
-      break;
-    }
-  }
-  return V;
-}
diff --git a/lib/Bitcode/NaCl/Writer/NaClValueEnumerator.h b/lib/Bitcode/NaCl/Writer/NaClValueEnumerator.h
deleted file mode 100644
index fc64bcd398a..00000000000
--- a/lib/Bitcode/NaCl/Writer/NaClValueEnumerator.h
+++ /dev/null
@@ -1,169 +0,0 @@
-//===-- Bitcode/NaCl/Writer/NaClValueEnumerator.h - ----------*- C++ -*-===//
-//      Number values.
-//
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
-//
-//===----------------------------------------------------------------------===//
-//
-// This class gives values and types Unique ID's.
-//
-//===----------------------------------------------------------------------===//
-
-#ifndef NACL_VALUE_ENUMERATOR_H
-#define NACL_VALUE_ENUMERATOR_H
-
-#include "llvm/ADT/DenseMap.h"
-#include "llvm/ADT/SmallSet.h"
-#include "llvm/ADT/SmallVector.h"
-#include "llvm/Bitcode/NaCl/NaClReaderWriter.h"
-#include <vector>
-
-namespace llvm {
-
-class Type;
-class Value;
-class Instruction;
-class BasicBlock;
-class Function;
-class Module;
-class ValueSymbolTable;
-class raw_ostream;
-
-class NaClValueEnumerator {
-public:
-  typedef std::vector<Type*> TypeList;
-
-  // For each value, we remember its Value* and occurrence frequency.
-  typedef std::vector<std::pair<const Value*, unsigned> > ValueList;
-private:
-  // Defines unique ID's for each type.
-  typedef DenseMap<Type*, unsigned> TypeMapType;
-  TypeMapType TypeMap;
-  // Defines the number of references to each type. If defined,
-  // we are in the first pass of collecting types, and reference counts
-  // should be added to the map. If undefined, we are in the second pass
-  // that actually assigns type IDs, based on frequency counts found in
-  // the first pass.
-  typedef TypeMapType TypeCountMapType;
-  TypeCountMapType* TypeCountMap;
-
-  TypeList Types;
-
-  typedef DenseMap<const Value*, unsigned> ValueMapType;
-  ValueMapType ValueMap;
-  ValueList Values;
-
-  typedef DenseMap<const Instruction*, unsigned> InstructionMapType;
-  InstructionMapType InstructionMap;
-  unsigned InstructionCount;
-
-  /// BasicBlocks - This contains all the basic blocks for the currently
-  /// incorporated function.  Their reverse mapping is stored in ValueMap.
-  std::vector<const BasicBlock*> BasicBlocks;
-
-  /// When a function is incorporated, this is the size of the Values list
-  /// before incorporation.
-  unsigned NumModuleValues;
-
-  unsigned FirstFuncConstantID;
-  unsigned FirstInstID;
-
-  /// Holds values that have been forward referenced within a function.
-  /// Used to make sure we don't generate more forward reference declarations
-  /// than necessary.
-  SmallSet<unsigned, 32> FnForwardTypeRefs;
-
-  // The index of the first global variable ID in the bitcode file.
-  unsigned FirstGlobalVarID;
-  // The number of global variable IDs defined in the bitcode file.
-  unsigned NumGlobalVarIDs;
-
-  /// \brief Integer type use for PNaCl conversion of pointers.
-  Type *IntPtrType;
-
-  NaClValueEnumerator(const NaClValueEnumerator &) = delete;
-  void operator=(const NaClValueEnumerator &) = delete;
-public:
-  NaClValueEnumerator(const Module *M);
-
-  void dump() const;
-  void print(raw_ostream &OS, const ValueMapType &Map, const char *Name) const;
-
-  unsigned getFirstGlobalVarID() const {
-    return FirstGlobalVarID;
-  }
-
-  unsigned getNumGlobalVarIDs() const {
-    return NumGlobalVarIDs;
-  }
-
-  unsigned getValueID(const Value *V) const;
-
-  unsigned getTypeID(Type *T) const {
-    TypeMapType::const_iterator I = TypeMap.find(NormalizeType(T));
-    assert(I != TypeMap.end() && "Type not in NaClValueEnumerator!");
-    return I->second-1;
-  }
-
-  unsigned getInstructionID(const Instruction *I) const;
-  void setInstructionID(const Instruction *I);
-
-  /// getFunctionConstantRange - Return the range of values that corresponds to
-  /// function-local constants.
-  void getFunctionConstantRange(unsigned &Start, unsigned &End) const {
-    Start = FirstFuncConstantID;
-    End = FirstInstID;
-  }
-
-  /// \brief Inserts the give value into the set of known function forward
-  /// value type refs. Returns true if the value id is added to the set.
-  bool InsertFnForwardTypeRef(unsigned ValID) {
-    return FnForwardTypeRefs.insert(ValID).second;
-  }
-
-  const ValueList &getValues() const { return Values; }
-  const TypeList &getTypes() const { return Types; }
-  const std::vector<const BasicBlock*> &getBasicBlocks() const {
-    return BasicBlocks;
-  }
-
-  /// incorporateFunction/purgeFunction - If you'd like to deal with a function,
-  /// use these two methods to get its data into the NaClValueEnumerator!
-  ///
-  void incorporateFunction(const Function &F);
-  void purgeFunction();
-
-  /// \brief Returns the value after elided (cast) operations have been
-  /// removed. Returns V if unable to elide the cast.
-  const Value *ElideCasts(const Value *V) const;
-
-  /// \brief Returns true if value V is an elided (cast) operation.
-  bool IsElidedCast(const Value *V) const {
-    return V != ElideCasts(V);
-  }
-
-  /// \brief Returns true if the type of V is the integer used to
-  /// model pointers in PNaCl.
-  bool IsIntPtrType(Type *T) const {
-    return T == IntPtrType;
-  }
-
-  Type *NormalizeType(Type *Ty) const;
-
-private:
-  void OptimizeTypes(const Module *M);
-  void OptimizeConstants(unsigned CstStart, unsigned CstEnd);
-
-  void EnumerateValue(const Value *V);
-  void EnumerateType(Type *T, bool InsideOptimizeTypes=false);
-  void EnumerateOperandType(const Value *V);
-
-  void EnumerateValueSymbolTable(const ValueSymbolTable &ST);
-};
-
-} // End llvm namespace
-
-#endif
diff --git a/lib/Bitcode/Reader/BitReader.cpp b/lib/Bitcode/Reader/BitReader.cpp
index 868fbf010db..289c76e85b4 100644
--- a/lib/Bitcode/Reader/BitReader.cpp
+++ b/lib/Bitcode/Reader/BitReader.cpp
@@ -39,7 +39,7 @@ LLVMBool LLVMParseBitcodeInContext(LLVMContextRef ContextRef,
   raw_string_ostream Stream(Message);
   DiagnosticPrinterRawOStream DP(Stream);
 
-  ErrorOr<Module *> ModuleOrErr = parseBitcodeFile(
+  ErrorOr<std::unique_ptr<Module>> ModuleOrErr = parseBitcodeFile(
       Buf, Ctx, [&](const DiagnosticInfo &DI) { DI.print(DP); });
   if (ModuleOrErr.getError()) {
     if (OutMessage) {
@@ -50,7 +50,7 @@ LLVMBool LLVMParseBitcodeInContext(LLVMContextRef ContextRef,
     return 1;
   }
 
-  *OutModule = wrap(ModuleOrErr.get());
+  *OutModule = wrap(ModuleOrErr.get().release());
   return 0;
 }
 
@@ -64,7 +64,7 @@ LLVMBool LLVMGetBitcodeModuleInContext(LLVMContextRef ContextRef,
   std::string Message;
   std::unique_ptr<MemoryBuffer> Owner(unwrap(MemBuf));
 
-  ErrorOr<Module *> ModuleOrErr =
+  ErrorOr<std::unique_ptr<Module>> ModuleOrErr =
       getLazyBitcodeModule(std::move(Owner), *unwrap(ContextRef));
   Owner.release();
 
@@ -75,7 +75,7 @@ LLVMBool LLVMGetBitcodeModuleInContext(LLVMContextRef ContextRef,
     return 1;
   }
 
-  *OutM = wrap(ModuleOrErr.get());
+  *OutM = wrap(ModuleOrErr.get().release());
 
   return 0;
 
diff --git a/lib/Bitcode/Reader/BitcodeReader.cpp b/lib/Bitcode/Reader/BitcodeReader.cpp
index 3cb449e5d25..fea4c167ae3 100644
--- a/lib/Bitcode/Reader/BitcodeReader.cpp
+++ b/lib/Bitcode/Reader/BitcodeReader.cpp
@@ -44,9 +44,9 @@ enum {
 class BitcodeReaderValueList {
   std::vector<WeakVH> ValuePtrs;
 
-  /// ResolveConstants - As we resolve forward-referenced constants, we add
-  /// information about them to this vector.  This allows us to resolve them in
-  /// bulk instead of resolving each reference at a time.  See the code in
+  /// As we resolve forward-referenced constants, we add information about them
+  /// to this vector.  This allows us to resolve them in bulk instead of
+  /// resolving each reference at a time.  See the code in
   /// ResolveConstantForwardRefs for more information about this.
   ///
   /// The key of this vector is the placeholder constant, the value is the slot
@@ -63,9 +63,7 @@ class BitcodeReaderValueList {
   // vector compatibility methods
   unsigned size() const { return ValuePtrs.size(); }
   void resize(unsigned N) { ValuePtrs.resize(N); }
-  void push_back(Value *V) {
-    ValuePtrs.push_back(V);
-  }
+  void push_back(Value *V) { ValuePtrs.emplace_back(V); }
 
   void clear() {
     assert(ResolveConstants.empty() && "Constants not resolved?");
@@ -88,11 +86,11 @@ class BitcodeReaderValueList {
   Constant *getConstantFwdRef(unsigned Idx, Type *Ty);
   Value *getValueFwdRef(unsigned Idx, Type *Ty);
 
-  void AssignValue(Value *V, unsigned Idx);
+  void assignValue(Value *V, unsigned Idx);
 
-  /// ResolveConstantForwardRefs - Once all constants are read, this method bulk
-  /// resolves any forward references.
-  void ResolveConstantForwardRefs();
+  /// Once all constants are read, this method bulk resolves any forward
+  /// references.
+  void resolveConstantForwardRefs();
 };
 
 class BitcodeReaderMDValueList {
@@ -127,20 +125,19 @@ class BitcodeReaderMDValueList {
   }
 
   Metadata *getValueFwdRef(unsigned Idx);
-  void AssignValue(Metadata *MD, unsigned Idx);
+  void assignValue(Metadata *MD, unsigned Idx);
   void tryToResolveCycles();
 };
 
 class BitcodeReader : public GVMaterializer {
   LLVMContext &Context;
   DiagnosticHandlerFunction DiagnosticHandler;
-  Module *TheModule;
+  Module *TheModule = nullptr;
   std::unique_ptr<MemoryBuffer> Buffer;
   std::unique_ptr<BitstreamReader> StreamFile;
   BitstreamCursor Stream;
-  StreamingMemoryObject *LazyStreamer; // @LOCALMOD
-  uint64_t NextUnreadBit;
-  bool SeenValueSymbolTable;
+  uint64_t NextUnreadBit = 0;
+  bool SeenValueSymbolTable = false;
 
   std::vector<Type*> TypeList;
   BitcodeReaderValueList ValueList;
@@ -152,19 +149,19 @@ class BitcodeReader : public GVMaterializer {
   std::vector<std::pair<GlobalAlias*, unsigned> > AliasInits;
   std::vector<std::pair<Function*, unsigned> > FunctionPrefixes;
   std::vector<std::pair<Function*, unsigned> > FunctionPrologues;
+  std::vector<std::pair<Function*, unsigned> > FunctionPersonalityFns;
 
   SmallVector<Instruction*, 64> InstsWithTBAATag;
 
-  /// MAttributes - The set of attributes by index.  Index zero in the
-  /// file is for null, and is thus not represented here.  As such all indices
-  /// are off by one.
+  /// The set of attributes by index.  Index zero in the file is for null, and
+  /// is thus not represented here.  As such all indices are off by one.
   std::vector<AttributeSet> MAttributes;
 
   /// \brief The set of attribute groups.
   std::map<unsigned, AttributeSet> MAttributeGroups;
 
-  /// FunctionBBs - While parsing a function body, this is a list of the basic
-  /// blocks for the function.
+  /// While parsing a function body, this is a list of the basic blocks for the
+  /// function.
   std::vector<BasicBlock*> FunctionBBs;
 
   // When reading the module header, this list is populated with functions that
@@ -173,7 +170,7 @@ class BitcodeReader : public GVMaterializer {
 
   // When intrinsic functions are encountered which require upgrading they are
   // stored here with their replacement function.
-  typedef std::vector<std::pair<Function*, Function*> > UpgradedIntrinsicMap;
+  typedef DenseMap<Function*, Function*> UpgradedIntrinsicMap;
   UpgradedIntrinsicMap UpgradedIntrinsics;
 
   // Map the bitcode's custom MDKind ID to the Module's MDKind ID.
@@ -182,11 +179,10 @@ class BitcodeReader : public GVMaterializer {
   // Several operations happen after the module header has been read, but
   // before function bodies are processed. This keeps track of whether
   // we've done this yet.
-  bool SeenFirstFunctionBody;
+  bool SeenFirstFunctionBody = false;
 
-  /// DeferredFunctionInfo - When function bodies are initially scanned, this
-  /// map contains info about where to find deferred function body in the
-  /// stream.
+  /// When function bodies are initially scanned, this map contains info about
+  /// where to find deferred function body in the stream.
   DenseMap<Function*, uint64_t> DeferredFunctionInfo;
 
   /// When Metadata block is initially scanned when parsing the module, we may
@@ -200,57 +196,57 @@ class BitcodeReader : public GVMaterializer {
   DenseMap<Function *, std::vector<BasicBlock *>> BasicBlockFwdRefs;
   std::deque<Function *> BasicBlockFwdRefQueue;
 
-  /// UseRelativeIDs - Indicates that we are using a new encoding for
-  /// instruction operands where most operands in the current
-  /// FUNCTION_BLOCK are encoded relative to the instruction number,
-  /// for a more compact encoding.  Some instruction operands are not
-  /// relative to the instruction ID: basic block numbers, and types.
-  /// Once the old style function blocks have been phased out, we would
+  /// Indicates that we are using a new encoding for instruction operands where
+  /// most operands in the current FUNCTION_BLOCK are encoded relative to the
+  /// instruction number, for a more compact encoding.  Some instruction
+  /// operands are not relative to the instruction ID: basic block numbers, and
+  /// types. Once the old style function blocks have been phased out, we would
   /// not need this flag.
-  bool UseRelativeIDs;
+  bool UseRelativeIDs = false;
 
   /// True if all functions will be materialized, negating the need to process
   /// (e.g.) blockaddress forward references.
-  bool WillMaterializeAllForwardRefs;
+  bool WillMaterializeAllForwardRefs = false;
 
   /// Functions that have block addresses taken.  This is usually empty.
   SmallPtrSet<const Function *, 4> BlockAddressesTaken;
 
   /// True if any Metadata block has been materialized.
-  bool IsMetadataMaterialized;
+  bool IsMetadataMaterialized = false;
 
   bool StripDebugInfo = false;
 
 public:
-  std::error_code Error(BitcodeError E, const Twine &Message);
-  std::error_code Error(BitcodeError E);
-  std::error_code Error(const Twine &Message);
+  std::error_code error(BitcodeError E, const Twine &Message);
+  std::error_code error(BitcodeError E);
+  std::error_code error(const Twine &Message);
 
-  explicit BitcodeReader(MemoryBuffer *buffer, LLVMContext &C,
-                         DiagnosticHandlerFunction DiagnosticHandler);
-  explicit BitcodeReader(StreamingMemoryObject *streamer, LLVMContext &C, // @LOCALMOD
-                         DiagnosticHandlerFunction DiagnosticHandler);
-  ~BitcodeReader() override { FreeState(); }
+  BitcodeReader(MemoryBuffer *Buffer, LLVMContext &Context,
+                DiagnosticHandlerFunction DiagnosticHandler);
+  BitcodeReader(LLVMContext &Context,
+                DiagnosticHandlerFunction DiagnosticHandler);
+  ~BitcodeReader() override { freeState(); }
 
   std::error_code materializeForwardReferencedFunctions();
 
-  void FreeState();
+  void freeState();
 
   void releaseBuffer();
 
   bool isDematerializable(const GlobalValue *GV) const override;
   std::error_code materialize(GlobalValue *GV) override;
-  std::error_code MaterializeModule(Module *M) override;
+  std::error_code materializeModule(Module *M) override;
   std::vector<StructType *> getIdentifiedStructTypes() const override;
-  void Dematerialize(GlobalValue *GV) override;
+  void dematerialize(GlobalValue *GV) override;
 
-  /// @brief Main interface to parsing a bitcode buffer.
-  /// @returns true if an error occurred.
-  std::error_code ParseBitcodeInto(Module *M,
+  /// \brief Main interface to parsing a bitcode buffer.
+  /// \returns true if an error occurred.
+  std::error_code parseBitcodeInto(std::unique_ptr<DataStreamer> Streamer,
+                                   Module *M,
                                    bool ShouldLazyLoadMetadata = false);
 
-  /// @brief Cheap mechanism to just extract module triple
-  /// @returns true if an error occurred.
+  /// \brief Cheap mechanism to just extract module triple
+  /// \returns true if an error occurred.
   ErrorOr<std::string> parseTriple();
 
   static uint64_t decodeSignRotatedValue(uint64_t V);
@@ -284,9 +280,9 @@ class BitcodeReader : public GVMaterializer {
     return AttributeSet();
   }
 
-  /// getValueTypePair - Read a value/type pair out of the specified record from
-  /// slot 'Slot'.  Increment Slot past the number of slots used in the record.
-  /// Return true on failure.
+  /// Read a value/type pair out of the specified record from slot 'Slot'.
+  /// Increment Slot past the number of slots used in the record. Return true on
+  /// failure.
   bool getValueTypePair(SmallVectorImpl<uint64_t> &Record, unsigned &Slot,
                         unsigned InstNum, Value *&ResVal) {
     if (Slot == Record.size()) return true;
@@ -299,18 +295,18 @@ class BitcodeReader : public GVMaterializer {
       // have.
       ResVal = getFnValueByID(ValNo, nullptr);
       return ResVal == nullptr;
-    } else if (Slot == Record.size()) {
-      return true;
     }
+    if (Slot == Record.size())
+      return true;
 
     unsigned TypeNo = (unsigned)Record[Slot++];
     ResVal = getFnValueByID(ValNo, getTypeByID(TypeNo));
     return ResVal == nullptr;
   }
 
-  /// popValue - Read a value out of the specified record from slot 'Slot'.
-  /// Increment Slot past the number of slots used by the value in the record.
-  /// Return true if there is an error.
+  /// Read a value out of the specified record from slot 'Slot'. Increment Slot
+  /// past the number of slots used by the value in the record. Return true if
+  /// there is an error.
   bool popValue(SmallVectorImpl<uint64_t> &Record, unsigned &Slot,
                 unsigned InstNum, Type *Ty, Value *&ResVal) {
     if (getValue(Record, Slot, InstNum, Ty, ResVal))
@@ -320,15 +316,15 @@ class BitcodeReader : public GVMaterializer {
     return false;
   }
 
-  /// getValue -- Like popValue, but does not increment the Slot number.
+  /// Like popValue, but does not increment the Slot number.
   bool getValue(SmallVectorImpl<uint64_t> &Record, unsigned Slot,
                 unsigned InstNum, Type *Ty, Value *&ResVal) {
     ResVal = getValue(Record, Slot, InstNum, Ty);
     return ResVal == nullptr;
   }
 
-  /// getValue -- Version of getValue that returns ResVal directly,
-  /// or 0 if there is an error.
+  /// Version of getValue that returns ResVal directly, or 0 if there is an
+  /// error.
   Value *getValue(SmallVectorImpl<uint64_t> &Record, unsigned Slot,
                   unsigned InstNum, Type *Ty) {
     if (Slot == Record.size()) return nullptr;
@@ -339,7 +335,7 @@ class BitcodeReader : public GVMaterializer {
     return getFnValueByID(ValNo, Ty);
   }
 
-  /// getValueSigned -- Like getValue, but decodes signed VBRs.
+  /// Like getValue, but decodes signed VBRs.
   Value *getValueSigned(SmallVectorImpl<uint64_t> &Record, unsigned Slot,
                         unsigned InstNum, Type *Ty) {
     if (Slot == Record.size()) return nullptr;
@@ -354,29 +350,29 @@ class BitcodeReader : public GVMaterializer {
   /// corresponding alignment to use. If alignment is too large, returns
   /// a corresponding error code.
   std::error_code parseAlignmentValue(uint64_t Exponent, unsigned &Alignment);
-  std::error_code ParseAttrKind(uint64_t Code, Attribute::AttrKind *Kind);
-  std::error_code ParseModule(bool Resume, bool ShouldLazyLoadMetadata = false);
-  std::error_code ParseAttributeBlock();
-  std::error_code ParseAttributeGroupBlock();
-  std::error_code ParseTypeTable();
-  std::error_code ParseTypeTableBody();
-
-  std::error_code ParseValueSymbolTable();
-  std::error_code ParseConstants();
-  std::error_code RememberAndSkipFunctionBody();
+  std::error_code parseAttrKind(uint64_t Code, Attribute::AttrKind *Kind);
+  std::error_code parseModule(bool Resume, bool ShouldLazyLoadMetadata = false);
+  std::error_code parseAttributeBlock();
+  std::error_code parseAttributeGroupBlock();
+  std::error_code parseTypeTable();
+  std::error_code parseTypeTableBody();
+
+  std::error_code parseValueSymbolTable();
+  std::error_code parseConstants();
+  std::error_code rememberAndSkipFunctionBody();
   /// Save the positions of the Metadata blocks and skip parsing the blocks.
   std::error_code rememberAndSkipMetadata();
-  std::error_code ParseFunctionBody(Function *F);
-  std::error_code GlobalCleanup();
-  std::error_code ResolveGlobalAndAliasInits();
-  std::error_code ParseMetadata();
-  std::error_code ParseMetadataAttachment();
+  std::error_code parseFunctionBody(Function *F);
+  std::error_code globalCleanup();
+  std::error_code resolveGlobalAndAliasInits();
+  std::error_code parseMetadata();
+  std::error_code parseMetadataAttachment(Function &F);
   ErrorOr<std::string> parseModuleTriple();
-  std::error_code ParseUseLists();
-  std::error_code InitStream();
-  std::error_code InitStreamFromBuffer();
-  std::error_code InitLazyStream();
-  std::error_code FindFunctionInStream(
+  std::error_code parseUseLists();
+  std::error_code initStream(std::unique_ptr<DataStreamer> Streamer);
+  std::error_code initStreamFromBuffer();
+  std::error_code initLazyStream(std::unique_ptr<DataStreamer> Streamer);
+  std::error_code findFunctionInStream(
       Function *F,
       DenseMap<Function *, uint64_t>::iterator DeferredFunctionInfoIterator);
 };
@@ -389,29 +385,35 @@ BitcodeDiagnosticInfo::BitcodeDiagnosticInfo(std::error_code EC,
 
 void BitcodeDiagnosticInfo::print(DiagnosticPrinter &DP) const { DP << Msg; }
 
-static std::error_code Error(DiagnosticHandlerFunction DiagnosticHandler,
+static std::error_code error(DiagnosticHandlerFunction DiagnosticHandler,
                              std::error_code EC, const Twine &Message) {
   BitcodeDiagnosticInfo DI(EC, DS_Error, Message);
   DiagnosticHandler(DI);
   return EC;
 }
 
-static std::error_code Error(DiagnosticHandlerFunction DiagnosticHandler,
+static std::error_code error(DiagnosticHandlerFunction DiagnosticHandler,
                              std::error_code EC) {
-  return Error(DiagnosticHandler, EC, EC.message());
+  return error(DiagnosticHandler, EC, EC.message());
+}
+
+static std::error_code error(DiagnosticHandlerFunction DiagnosticHandler,
+                             const Twine &Message) {
+  return error(DiagnosticHandler,
+               make_error_code(BitcodeError::CorruptedBitcode), Message);
 }
 
-std::error_code BitcodeReader::Error(BitcodeError E, const Twine &Message) {
-  return ::Error(DiagnosticHandler, make_error_code(E), Message);
+std::error_code BitcodeReader::error(BitcodeError E, const Twine &Message) {
+  return ::error(DiagnosticHandler, make_error_code(E), Message);
 }
 
-std::error_code BitcodeReader::Error(const Twine &Message) {
-  return ::Error(DiagnosticHandler,
+std::error_code BitcodeReader::error(const Twine &Message) {
+  return ::error(DiagnosticHandler,
                  make_error_code(BitcodeError::CorruptedBitcode), Message);
 }
 
-std::error_code BitcodeReader::Error(BitcodeError E) {
-  return ::Error(DiagnosticHandler, make_error_code(E));
+std::error_code BitcodeReader::error(BitcodeError E) {
+  return ::error(DiagnosticHandler, make_error_code(E));
 }
 
 static DiagnosticHandlerFunction getDiagHandler(DiagnosticHandlerFunction F,
@@ -421,22 +423,17 @@ static DiagnosticHandlerFunction getDiagHandler(DiagnosticHandlerFunction F,
   return [&C](const DiagnosticInfo &DI) { C.diagnose(DI); };
 }
 
-BitcodeReader::BitcodeReader(MemoryBuffer *buffer, LLVMContext &C,
+BitcodeReader::BitcodeReader(MemoryBuffer *Buffer, LLVMContext &Context,
                              DiagnosticHandlerFunction DiagnosticHandler)
-    : Context(C), DiagnosticHandler(getDiagHandler(DiagnosticHandler, C)),
-      TheModule(nullptr), Buffer(buffer), LazyStreamer(nullptr),
-      NextUnreadBit(0), SeenValueSymbolTable(false), ValueList(C),
-      MDValueList(C), SeenFirstFunctionBody(false), UseRelativeIDs(false),
-      WillMaterializeAllForwardRefs(false), IsMetadataMaterialized(false) {}
-
-// @LOCALMOD:StreamingMemoryObject
-BitcodeReader::BitcodeReader(StreamingMemoryObject *streamer, LLVMContext &C,
+    : Context(Context),
+      DiagnosticHandler(getDiagHandler(DiagnosticHandler, Context)),
+      Buffer(Buffer), ValueList(Context), MDValueList(Context) {}
+
+BitcodeReader::BitcodeReader(LLVMContext &Context,
                              DiagnosticHandlerFunction DiagnosticHandler)
-    : Context(C), DiagnosticHandler(getDiagHandler(DiagnosticHandler, C)),
-      TheModule(nullptr), Buffer(nullptr), LazyStreamer(streamer),
-      NextUnreadBit(0), SeenValueSymbolTable(false), ValueList(C),
-      MDValueList(C), SeenFirstFunctionBody(false), UseRelativeIDs(false),
-      WillMaterializeAllForwardRefs(false), IsMetadataMaterialized(false) {}
+    : Context(Context),
+      DiagnosticHandler(getDiagHandler(DiagnosticHandler, Context)),
+      Buffer(nullptr), ValueList(Context), MDValueList(Context) {}
 
 std::error_code BitcodeReader::materializeForwardReferencedFunctions() {
   if (WillMaterializeAllForwardRefs)
@@ -458,7 +455,7 @@ std::error_code BitcodeReader::materializeForwardReferencedFunctions() {
     // isn't a trivial way to check if a function will have a body without a
     // linear search through FunctionsWithBodies, so just check it here.
     if (!F->isMaterializable())
-      return Error("Never resolved function from blockaddress");
+      return error("Never resolved function from blockaddress");
 
     // Try to materialize F.
     if (std::error_code EC = materialize(F))
@@ -471,7 +468,7 @@ std::error_code BitcodeReader::materializeForwardReferencedFunctions() {
   return std::error_code();
 }
 
-void BitcodeReader::FreeState() {
+void BitcodeReader::freeState() {
   Buffer = nullptr;
   std::vector<Type*>().swap(TypeList);
   ValueList.clear();
@@ -493,10 +490,9 @@ void BitcodeReader::FreeState() {
 //  Helper functions to implement forward reference resolution, etc.
 //===----------------------------------------------------------------------===//
 
-/// ConvertToString - Convert a string from a record into an std::string, return
-/// true on failure.
-template<typename StrTy>
-static bool ConvertToString(ArrayRef<uint64_t> Record, unsigned Idx,
+/// Convert a string from a record into an std::string, return true on failure.
+template <typename StrTy>
+static bool convertToString(ArrayRef<uint64_t> Record, unsigned Idx,
                             StrTy &Result) {
   if (Idx > Record.size())
     return true;
@@ -560,7 +556,7 @@ static GlobalValue::LinkageTypes getDecodedLinkage(unsigned Val) {
   }
 }
 
-static GlobalValue::VisibilityTypes GetDecodedVisibility(unsigned Val) {
+static GlobalValue::VisibilityTypes getDecodedVisibility(unsigned Val) {
   switch (Val) {
   default: // Map unknown visibilities to default.
   case 0: return GlobalValue::DefaultVisibility;
@@ -570,7 +566,7 @@ static GlobalValue::VisibilityTypes GetDecodedVisibility(unsigned Val) {
 }
 
 static GlobalValue::DLLStorageClassTypes
-GetDecodedDLLStorageClass(unsigned Val) {
+getDecodedDLLStorageClass(unsigned Val) {
   switch (Val) {
   default: // Map unknown values to default.
   case 0: return GlobalValue::DefaultStorageClass;
@@ -579,7 +575,7 @@ GetDecodedDLLStorageClass(unsigned Val) {
   }
 }
 
-static GlobalVariable::ThreadLocalMode GetDecodedThreadLocalMode(unsigned Val) {
+static GlobalVariable::ThreadLocalMode getDecodedThreadLocalMode(unsigned Val) {
   switch (Val) {
     case 0: return GlobalVariable::NotThreadLocal;
     default: // Map unknown non-zero value to general dynamic.
@@ -590,7 +586,7 @@ static GlobalVariable::ThreadLocalMode GetDecodedThreadLocalMode(unsigned Val) {
   }
 }
 
-static int GetDecodedCastOpcode(unsigned Val) {
+static int getDecodedCastOpcode(unsigned Val) {
   switch (Val) {
   default: return -1;
   case bitc::CAST_TRUNC   : return Instruction::Trunc;
@@ -608,31 +604,46 @@ static int GetDecodedCastOpcode(unsigned Val) {
   case bitc::CAST_ADDRSPACECAST: return Instruction::AddrSpaceCast;
   }
 }
-static int GetDecodedBinaryOpcode(unsigned Val, Type *Ty) {
+
+static int getDecodedBinaryOpcode(unsigned Val, Type *Ty) {
+  bool IsFP = Ty->isFPOrFPVectorTy();
+  // BinOps are only valid for int/fp or vector of int/fp types
+  if (!IsFP && !Ty->isIntOrIntVectorTy())
+    return -1;
+
   switch (Val) {
-  default: return -1;
+  default:
+    return -1;
   case bitc::BINOP_ADD:
-    return Ty->isFPOrFPVectorTy() ? Instruction::FAdd : Instruction::Add;
+    return IsFP ? Instruction::FAdd : Instruction::Add;
   case bitc::BINOP_SUB:
-    return Ty->isFPOrFPVectorTy() ? Instruction::FSub : Instruction::Sub;
+    return IsFP ? Instruction::FSub : Instruction::Sub;
   case bitc::BINOP_MUL:
-    return Ty->isFPOrFPVectorTy() ? Instruction::FMul : Instruction::Mul;
-  case bitc::BINOP_UDIV: return Instruction::UDiv;
+    return IsFP ? Instruction::FMul : Instruction::Mul;
+  case bitc::BINOP_UDIV:
+    return IsFP ? -1 : Instruction::UDiv;
   case bitc::BINOP_SDIV:
-    return Ty->isFPOrFPVectorTy() ? Instruction::FDiv : Instruction::SDiv;
-  case bitc::BINOP_UREM: return Instruction::URem;
+    return IsFP ? Instruction::FDiv : Instruction::SDiv;
+  case bitc::BINOP_UREM:
+    return IsFP ? -1 : Instruction::URem;
   case bitc::BINOP_SREM:
-    return Ty->isFPOrFPVectorTy() ? Instruction::FRem : Instruction::SRem;
-  case bitc::BINOP_SHL:  return Instruction::Shl;
-  case bitc::BINOP_LSHR: return Instruction::LShr;
-  case bitc::BINOP_ASHR: return Instruction::AShr;
-  case bitc::BINOP_AND:  return Instruction::And;
-  case bitc::BINOP_OR:   return Instruction::Or;
-  case bitc::BINOP_XOR:  return Instruction::Xor;
+    return IsFP ? Instruction::FRem : Instruction::SRem;
+  case bitc::BINOP_SHL:
+    return IsFP ? -1 : Instruction::Shl;
+  case bitc::BINOP_LSHR:
+    return IsFP ? -1 : Instruction::LShr;
+  case bitc::BINOP_ASHR:
+    return IsFP ? -1 : Instruction::AShr;
+  case bitc::BINOP_AND:
+    return IsFP ? -1 : Instruction::And;
+  case bitc::BINOP_OR:
+    return IsFP ? -1 : Instruction::Or;
+  case bitc::BINOP_XOR:
+    return IsFP ? -1 : Instruction::Xor;
   }
 }
 
-static AtomicRMWInst::BinOp GetDecodedRMWOperation(unsigned Val) {
+static AtomicRMWInst::BinOp getDecodedRMWOperation(unsigned Val) {
   switch (Val) {
   default: return AtomicRMWInst::BAD_BINOP;
   case bitc::RMW_XCHG: return AtomicRMWInst::Xchg;
@@ -649,7 +660,7 @@ static AtomicRMWInst::BinOp GetDecodedRMWOperation(unsigned Val) {
   }
 }
 
-static AtomicOrdering GetDecodedOrdering(unsigned Val) {
+static AtomicOrdering getDecodedOrdering(unsigned Val) {
   switch (Val) {
   case bitc::ORDERING_NOTATOMIC: return NotAtomic;
   case bitc::ORDERING_UNORDERED: return Unordered;
@@ -662,7 +673,7 @@ static AtomicOrdering GetDecodedOrdering(unsigned Val) {
   }
 }
 
-static SynchronizationScope GetDecodedSynchScope(unsigned Val) {
+static SynchronizationScope getDecodedSynchScope(unsigned Val) {
   switch (Val) {
   case bitc::SYNCHSCOPE_SINGLETHREAD: return SingleThread;
   default: // Map unknown scopes to cross-thread.
@@ -686,7 +697,22 @@ static Comdat::SelectionKind getDecodedComdatSelectionKind(unsigned Val) {
   }
 }
 
-static void UpgradeDLLImportExportLinkage(llvm::GlobalValue *GV, unsigned Val) {
+static FastMathFlags getDecodedFastMathFlags(unsigned Val) {
+  FastMathFlags FMF;
+  if (0 != (Val & FastMathFlags::UnsafeAlgebra))
+    FMF.setUnsafeAlgebra();
+  if (0 != (Val & FastMathFlags::NoNaNs))
+    FMF.setNoNaNs();
+  if (0 != (Val & FastMathFlags::NoInfs))
+    FMF.setNoInfs();
+  if (0 != (Val & FastMathFlags::NoSignedZeros))
+    FMF.setNoSignedZeros();
+  if (0 != (Val & FastMathFlags::AllowReciprocal))
+    FMF.setAllowReciprocal();
+  return FMF;
+}
+
+static void upgradeDLLImportExportLinkage(llvm::GlobalValue *GV, unsigned Val) {
   switch (Val) {
   case 5: GV->setDLLStorageClass(GlobalValue::DLLImportStorageClass); break;
   case 6: GV->setDLLStorageClass(GlobalValue::DLLExportStorageClass); break;
@@ -695,30 +721,28 @@ static void UpgradeDLLImportExportLinkage(llvm::GlobalValue *GV, unsigned Val) {
 
 namespace llvm {
 namespace {
-  /// @brief A class for maintaining the slot number definition
-  /// as a placeholder for the actual definition for forward constants defs.
-  class ConstantPlaceHolder : public ConstantExpr {
-    void operator=(const ConstantPlaceHolder &) = delete;
-  public:
-    // allocate space for exactly one operand
-    void *operator new(size_t s) {
-      return User::operator new(s, 1);
-    }
-    explicit ConstantPlaceHolder(Type *Ty, LLVMContext& Context)
-      : ConstantExpr(Ty, Instruction::UserOp1, &Op<0>(), 1) {
-      Op<0>() = UndefValue::get(Type::getInt32Ty(Context));
-    }
+/// \brief A class for maintaining the slot number definition
+/// as a placeholder for the actual definition for forward constants defs.
+class ConstantPlaceHolder : public ConstantExpr {
+  void operator=(const ConstantPlaceHolder &) = delete;
 
-    /// @brief Methods to support type inquiry through isa, cast, and dyn_cast.
-    static bool classof(const Value *V) {
-      return isa<ConstantExpr>(V) &&
-             cast<ConstantExpr>(V)->getOpcode() == Instruction::UserOp1;
-    }
+public:
+  // allocate space for exactly one operand
+  void *operator new(size_t s) { return User::operator new(s, 1); }
+  explicit ConstantPlaceHolder(Type *Ty, LLVMContext &Context)
+      : ConstantExpr(Ty, Instruction::UserOp1, &Op<0>(), 1) {
+    Op<0>() = UndefValue::get(Type::getInt32Ty(Context));
+  }
 
+  /// \brief Methods to support type inquiry through isa, cast, and dyn_cast.
+  static bool classof(const Value *V) {
+    return isa<ConstantExpr>(V) &&
+           cast<ConstantExpr>(V)->getOpcode() == Instruction::UserOp1;
+  }
 
-    /// Provide fast operand accessors
-    DECLARE_TRANSPARENT_OPERAND_ACCESSORS(Value);
-  };
+  /// Provide fast operand accessors
+  DECLARE_TRANSPARENT_OPERAND_ACCESSORS(Value);
+};
 }
 
 // FIXME: can we inherit this from ConstantExpr?
@@ -729,8 +753,7 @@ struct OperandTraits<ConstantPlaceHolder> :
 DEFINE_TRANSPARENT_OPERAND_ACCESSORS(ConstantPlaceHolder, Value)
 }
 
-
-void BitcodeReaderValueList::AssignValue(Value *V, unsigned Idx) {
+void BitcodeReaderValueList::assignValue(Value *V, unsigned Idx) {
   if (Idx == size()) {
     push_back(V);
     return;
@@ -765,7 +788,8 @@ Constant *BitcodeReaderValueList::getConstantFwdRef(unsigned Idx,
     resize(Idx + 1);
 
   if (Value *V = ValuePtrs[Idx]) {
-    assert(Ty == V->getType() && "Type mismatch in constant table!");
+    if (Ty != V->getType())
+      report_fatal_error("Type mismatch in constant table!");
     return cast<Constant>(V);
   }
 
@@ -776,11 +800,17 @@ Constant *BitcodeReaderValueList::getConstantFwdRef(unsigned Idx,
 }
 
 Value *BitcodeReaderValueList::getValueFwdRef(unsigned Idx, Type *Ty) {
+  // Bail out for a clearly invalid value. This would make us call resize(0)
+  if (Idx == UINT_MAX)
+    return nullptr;
+
   if (Idx >= size())
     resize(Idx + 1);
 
   if (Value *V = ValuePtrs[Idx]) {
-    assert((!Ty || Ty == V->getType()) && "Type mismatch in value table!");
+    // If the types don't match, it's invalid.
+    if (Ty && Ty != V->getType())
+      return nullptr;
     return V;
   }
 
@@ -793,14 +823,13 @@ Value *BitcodeReaderValueList::getValueFwdRef(unsigned Idx, Type *Ty) {
   return V;
 }
 
-/// ResolveConstantForwardRefs - Once all constants are read, this method bulk
-/// resolves any forward references.  The idea behind this is that we sometimes
-/// get constants (such as large arrays) which reference *many* forward ref
-/// constants.  Replacing each of these causes a lot of thrashing when
-/// building/reuniquing the constant.  Instead of doing this, we look at all the
-/// uses and rewrite all the place holders at once for any constant that uses
-/// a placeholder.
-void BitcodeReaderValueList::ResolveConstantForwardRefs() {
+/// Once all constants are read, this method bulk resolves any forward
+/// references.  The idea behind this is that we sometimes get constants (such
+/// as large arrays) which reference *many* forward ref constants.  Replacing
+/// each of these causes a lot of thrashing when building/reuniquing the
+/// constant.  Instead of doing this, we look at all the uses and rewrite all
+/// the place holders at once for any constant that uses a placeholder.
+void BitcodeReaderValueList::resolveConstantForwardRefs() {
   // Sort the values by-pointer so that they are efficient to look up with a
   // binary search.
   std::sort(ResolveConstants.begin(), ResolveConstants.end());
@@ -875,7 +904,7 @@ void BitcodeReaderValueList::ResolveConstantForwardRefs() {
   }
 }
 
-void BitcodeReaderMDValueList::AssignValue(Metadata *MD, unsigned Idx) {
+void BitcodeReaderMDValueList::assignValue(Metadata *MD, unsigned Idx) {
   if (Idx == size()) {
     push_back(MD);
     return;
@@ -994,12 +1023,12 @@ static void decodeLLVMAttributesForBitcode(AttrBuilder &B,
                 (EncodedAttrs & 0xffff));
 }
 
-std::error_code BitcodeReader::ParseAttributeBlock() {
+std::error_code BitcodeReader::parseAttributeBlock() {
   if (Stream.EnterSubBlock(bitc::PARAMATTR_BLOCK_ID))
-    return Error("Invalid record");
+    return error("Invalid record");
 
   if (!MAttributes.empty())
-    return Error("Invalid multiple blocks");
+    return error("Invalid multiple blocks");
 
   SmallVector<uint64_t, 64> Record;
 
@@ -1012,7 +1041,7 @@ std::error_code BitcodeReader::ParseAttributeBlock() {
     switch (Entry.Kind) {
     case BitstreamEntry::SubBlock: // Handled for us already.
     case BitstreamEntry::Error:
-      return Error("Malformed block");
+      return error("Malformed block");
     case BitstreamEntry::EndBlock:
       return std::error_code();
     case BitstreamEntry::Record:
@@ -1028,7 +1057,7 @@ std::error_code BitcodeReader::ParseAttributeBlock() {
     case bitc::PARAMATTR_CODE_ENTRY_OLD: { // ENTRY: [paramidx0, attr0, ...]
       // FIXME: Remove in 4.0.
       if (Record.size() & 1)
-        return Error("Invalid record");
+        return error("Invalid record");
 
       for (unsigned i = 0, e = Record.size(); i != e; i += 2) {
         AttrBuilder B;
@@ -1053,7 +1082,7 @@ std::error_code BitcodeReader::ParseAttributeBlock() {
 }
 
 // Returns Attribute::None on unrecognized codes.
-static Attribute::AttrKind GetAttrFromCode(uint64_t Code) {
+static Attribute::AttrKind getAttrFromCode(uint64_t Code) {
   switch (Code) {
   default:
     return Attribute::None;
@@ -1061,6 +1090,8 @@ static Attribute::AttrKind GetAttrFromCode(uint64_t Code) {
     return Attribute::Alignment;
   case bitc::ATTR_KIND_ALWAYS_INLINE:
     return Attribute::AlwaysInline;
+  case bitc::ATTR_KIND_ARGMEMONLY:
+    return Attribute::ArgMemOnly;
   case bitc::ATTR_KIND_BUILTIN:
     return Attribute::Builtin;
   case bitc::ATTR_KIND_BY_VAL:
@@ -1069,6 +1100,8 @@ static Attribute::AttrKind GetAttrFromCode(uint64_t Code) {
     return Attribute::InAlloca;
   case bitc::ATTR_KIND_COLD:
     return Attribute::Cold;
+  case bitc::ATTR_KIND_CONVERGENT:
+    return Attribute::Convergent;
   case bitc::ATTR_KIND_INLINE_HINT:
     return Attribute::InlineHint;
   case bitc::ATTR_KIND_IN_REG:
@@ -1129,6 +1162,8 @@ static Attribute::AttrKind GetAttrFromCode(uint64_t Code) {
     return Attribute::StackProtectReq;
   case bitc::ATTR_KIND_STACK_PROTECT_STRONG:
     return Attribute::StackProtectStrong;
+  case bitc::ATTR_KIND_SAFESTACK:
+    return Attribute::SafeStack;
   case bitc::ATTR_KIND_STRUCT_RET:
     return Attribute::StructRet;
   case bitc::ATTR_KIND_SANITIZE_ADDRESS:
@@ -1149,26 +1184,26 @@ std::error_code BitcodeReader::parseAlignmentValue(uint64_t Exponent,
   // Note: Alignment in bitcode files is incremented by 1, so that zero
   // can be used for default alignment.
   if (Exponent > Value::MaxAlignmentExponent + 1)
-    return Error("Invalid alignment value");
+    return error("Invalid alignment value");
   Alignment = (1 << static_cast<unsigned>(Exponent)) >> 1;
   return std::error_code();
 }
 
-std::error_code BitcodeReader::ParseAttrKind(uint64_t Code,
+std::error_code BitcodeReader::parseAttrKind(uint64_t Code,
                                              Attribute::AttrKind *Kind) {
-  *Kind = GetAttrFromCode(Code);
+  *Kind = getAttrFromCode(Code);
   if (*Kind == Attribute::None)
-    return Error(BitcodeError::CorruptedBitcode,
+    return error(BitcodeError::CorruptedBitcode,
                  "Unknown attribute kind (" + Twine(Code) + ")");
   return std::error_code();
 }
 
-std::error_code BitcodeReader::ParseAttributeGroupBlock() {
+std::error_code BitcodeReader::parseAttributeGroupBlock() {
   if (Stream.EnterSubBlock(bitc::PARAMATTR_GROUP_BLOCK_ID))
-    return Error("Invalid record");
+    return error("Invalid record");
 
   if (!MAttributeGroups.empty())
-    return Error("Invalid multiple blocks");
+    return error("Invalid multiple blocks");
 
   SmallVector<uint64_t, 64> Record;
 
@@ -1179,7 +1214,7 @@ std::error_code BitcodeReader::ParseAttributeGroupBlock() {
     switch (Entry.Kind) {
     case BitstreamEntry::SubBlock: // Handled for us already.
     case BitstreamEntry::Error:
-      return Error("Malformed block");
+      return error("Malformed block");
     case BitstreamEntry::EndBlock:
       return std::error_code();
     case BitstreamEntry::Record:
@@ -1194,7 +1229,7 @@ std::error_code BitcodeReader::ParseAttributeGroupBlock() {
       break;
     case bitc::PARAMATTR_GRP_CODE_ENTRY: { // ENTRY: [grpid, idx, a0, a1, ...]
       if (Record.size() < 3)
-        return Error("Invalid record");
+        return error("Invalid record");
 
       uint64_t GrpID = Record[0];
       uint64_t Idx = Record[1]; // Index of the object this attribute refers to.
@@ -1203,13 +1238,13 @@ std::error_code BitcodeReader::ParseAttributeGroupBlock() {
       for (unsigned i = 2, e = Record.size(); i != e; ++i) {
         if (Record[i] == 0) {        // Enum attribute
           Attribute::AttrKind Kind;
-          if (std::error_code EC = ParseAttrKind(Record[++i], &Kind))
+          if (std::error_code EC = parseAttrKind(Record[++i], &Kind))
             return EC;
 
           B.addAttribute(Kind);
         } else if (Record[i] == 1) { // Integer attribute
           Attribute::AttrKind Kind;
-          if (std::error_code EC = ParseAttrKind(Record[++i], &Kind))
+          if (std::error_code EC = parseAttrKind(Record[++i], &Kind))
             return EC;
           if (Kind == Attribute::Alignment)
             B.addAlignmentAttr(Record[++i]);
@@ -1249,16 +1284,16 @@ std::error_code BitcodeReader::ParseAttributeGroupBlock() {
   }
 }
 
-std::error_code BitcodeReader::ParseTypeTable() {
+std::error_code BitcodeReader::parseTypeTable() {
   if (Stream.EnterSubBlock(bitc::TYPE_BLOCK_ID_NEW))
-    return Error("Invalid record");
+    return error("Invalid record");
 
-  return ParseTypeTableBody();
+  return parseTypeTableBody();
 }
 
-std::error_code BitcodeReader::ParseTypeTableBody() {
+std::error_code BitcodeReader::parseTypeTableBody() {
   if (!TypeList.empty())
-    return Error("Invalid multiple blocks");
+    return error("Invalid multiple blocks");
 
   SmallVector<uint64_t, 64> Record;
   unsigned NumRecords = 0;
@@ -1272,10 +1307,10 @@ std::error_code BitcodeReader::ParseTypeTableBody() {
     switch (Entry.Kind) {
     case BitstreamEntry::SubBlock: // Handled for us already.
     case BitstreamEntry::Error:
-      return Error("Malformed block");
+      return error("Malformed block");
     case BitstreamEntry::EndBlock:
       if (NumRecords != TypeList.size())
-        return Error("Malformed block");
+        return error("Malformed block");
       return std::error_code();
     case BitstreamEntry::Record:
       // The interesting case.
@@ -1287,12 +1322,12 @@ std::error_code BitcodeReader::ParseTypeTableBody() {
     Type *ResultTy = nullptr;
     switch (Stream.readRecord(Entry.ID, Record)) {
     default:
-      return Error("Invalid value");
+      return error("Invalid value");
     case bitc::TYPE_CODE_NUMENTRY: // TYPE_CODE_NUMENTRY: [numentries]
       // TYPE_CODE_NUMENTRY contains a count of the number of types in the
       // type list.  This allows us to reserve space.
       if (Record.size() < 1)
-        return Error("Invalid record");
+        return error("Invalid record");
       TypeList.resize(Record[0]);
       continue;
     case bitc::TYPE_CODE_VOID:      // VOID
@@ -1327,25 +1362,26 @@ std::error_code BitcodeReader::ParseTypeTableBody() {
       break;
     case bitc::TYPE_CODE_INTEGER: { // INTEGER: [width]
       if (Record.size() < 1)
-        return Error("Invalid record");
+        return error("Invalid record");
 
       uint64_t NumBits = Record[0];
       if (NumBits < IntegerType::MIN_INT_BITS ||
           NumBits > IntegerType::MAX_INT_BITS)
-        return Error("Bitwidth for integer type out of range");
+        return error("Bitwidth for integer type out of range");
       ResultTy = IntegerType::get(Context, NumBits);
       break;
     }
     case bitc::TYPE_CODE_POINTER: { // POINTER: [pointee type] or
                                     //          [pointee type, address space]
       if (Record.size() < 1)
-        return Error("Invalid record");
+        return error("Invalid record");
       unsigned AddressSpace = 0;
       if (Record.size() == 2)
         AddressSpace = Record[1];
       ResultTy = getTypeByID(Record[0]);
-      if (!ResultTy)
-        return Error("Invalid type");
+      if (!ResultTy ||
+          !PointerType::isValidElementType(ResultTy))
+        return error("Invalid type");
       ResultTy = PointerType::get(ResultTy, AddressSpace);
       break;
     }
@@ -1353,7 +1389,7 @@ std::error_code BitcodeReader::ParseTypeTableBody() {
       // FIXME: attrid is dead, remove it in LLVM 4.0
       // FUNCTION: [vararg, attrid, retty, paramty x N]
       if (Record.size() < 3)
-        return Error("Invalid record");
+        return error("Invalid record");
       SmallVector<Type*, 8> ArgTys;
       for (unsigned i = 3, e = Record.size(); i != e; ++i) {
         if (Type *T = getTypeByID(Record[i]))
@@ -1364,7 +1400,7 @@ std::error_code BitcodeReader::ParseTypeTableBody() {
 
       ResultTy = getTypeByID(Record[2]);
       if (!ResultTy || ArgTys.size() < Record.size()-3)
-        return Error("Invalid type");
+        return error("Invalid type");
 
       ResultTy = FunctionType::get(ResultTy, ArgTys, Record[0]);
       break;
@@ -1372,25 +1408,28 @@ std::error_code BitcodeReader::ParseTypeTableBody() {
     case bitc::TYPE_CODE_FUNCTION: {
       // FUNCTION: [vararg, retty, paramty x N]
       if (Record.size() < 2)
-        return Error("Invalid record");
+        return error("Invalid record");
       SmallVector<Type*, 8> ArgTys;
       for (unsigned i = 2, e = Record.size(); i != e; ++i) {
-        if (Type *T = getTypeByID(Record[i]))
+        if (Type *T = getTypeByID(Record[i])) {
+          if (!FunctionType::isValidArgumentType(T))
+            return error("Invalid function argument type");
           ArgTys.push_back(T);
+        }
         else
           break;
       }
 
       ResultTy = getTypeByID(Record[1]);
       if (!ResultTy || ArgTys.size() < Record.size()-2)
-        return Error("Invalid type");
+        return error("Invalid type");
 
       ResultTy = FunctionType::get(ResultTy, ArgTys, Record[0]);
       break;
     }
     case bitc::TYPE_CODE_STRUCT_ANON: {  // STRUCT: [ispacked, eltty x N]
       if (Record.size() < 1)
-        return Error("Invalid record");
+        return error("Invalid record");
       SmallVector<Type*, 8> EltTys;
       for (unsigned i = 1, e = Record.size(); i != e; ++i) {
         if (Type *T = getTypeByID(Record[i]))
@@ -1399,21 +1438,21 @@ std::error_code BitcodeReader::ParseTypeTableBody() {
           break;
       }
       if (EltTys.size() != Record.size()-1)
-        return Error("Invalid type");
+        return error("Invalid type");
       ResultTy = StructType::get(Context, EltTys, Record[0]);
       break;
     }
     case bitc::TYPE_CODE_STRUCT_NAME:   // STRUCT_NAME: [strchr x N]
-      if (ConvertToString(Record, 0, TypeName))
-        return Error("Invalid record");
+      if (convertToString(Record, 0, TypeName))
+        return error("Invalid record");
       continue;
 
     case bitc::TYPE_CODE_STRUCT_NAMED: { // STRUCT: [ispacked, eltty x N]
       if (Record.size() < 1)
-        return Error("Invalid record");
+        return error("Invalid record");
 
       if (NumRecords >= TypeList.size())
-        return Error("Invalid TYPE table");
+        return error("Invalid TYPE table");
 
       // Check to see if this was forward referenced, if so fill in the temp.
       StructType *Res = cast_or_null<StructType>(TypeList[NumRecords]);
@@ -1432,17 +1471,17 @@ std::error_code BitcodeReader::ParseTypeTableBody() {
           break;
       }
       if (EltTys.size() != Record.size()-1)
-        return Error("Invalid record");
+        return error("Invalid record");
       Res->setBody(EltTys, Record[0]);
       ResultTy = Res;
       break;
     }
     case bitc::TYPE_CODE_OPAQUE: {       // OPAQUE: []
       if (Record.size() != 1)
-        return Error("Invalid record");
+        return error("Invalid record");
 
       if (NumRecords >= TypeList.size())
-        return Error("Invalid TYPE table");
+        return error("Invalid TYPE table");
 
       // Check to see if this was forward referenced, if so fill in the temp.
       StructType *Res = cast_or_null<StructType>(TypeList[NumRecords]);
@@ -1457,35 +1496,37 @@ std::error_code BitcodeReader::ParseTypeTableBody() {
     }
     case bitc::TYPE_CODE_ARRAY:     // ARRAY: [numelts, eltty]
       if (Record.size() < 2)
-        return Error("Invalid record");
-      if ((ResultTy = getTypeByID(Record[1])))
-        ResultTy = ArrayType::get(ResultTy, Record[0]);
-      else
-        return Error("Invalid type");
+        return error("Invalid record");
+      ResultTy = getTypeByID(Record[1]);
+      if (!ResultTy || !ArrayType::isValidElementType(ResultTy))
+        return error("Invalid type");
+      ResultTy = ArrayType::get(ResultTy, Record[0]);
       break;
     case bitc::TYPE_CODE_VECTOR:    // VECTOR: [numelts, eltty]
       if (Record.size() < 2)
-        return Error("Invalid record");
-      if ((ResultTy = getTypeByID(Record[1])))
-        ResultTy = VectorType::get(ResultTy, Record[0]);
-      else
-        return Error("Invalid type");
+        return error("Invalid record");
+      if (Record[0] == 0)
+        return error("Invalid vector length");
+      ResultTy = getTypeByID(Record[1]);
+      if (!ResultTy || !StructType::isValidElementType(ResultTy))
+        return error("Invalid type");
+      ResultTy = VectorType::get(ResultTy, Record[0]);
       break;
     }
 
     if (NumRecords >= TypeList.size())
-      return Error("Invalid TYPE table");
+      return error("Invalid TYPE table");
     if (TypeList[NumRecords])
-      return Error(
+      return error(
           "Invalid TYPE table: Only named structs can be forward referenced");
     assert(ResultTy && "Didn't read a type?");
     TypeList[NumRecords++] = ResultTy;
   }
 }
 
-std::error_code BitcodeReader::ParseValueSymbolTable() {
+std::error_code BitcodeReader::parseValueSymbolTable() {
   if (Stream.EnterSubBlock(bitc::VALUE_SYMTAB_BLOCK_ID))
-    return Error("Invalid record");
+    return error("Invalid record");
 
   SmallVector<uint64_t, 64> Record;
 
@@ -1499,7 +1540,7 @@ std::error_code BitcodeReader::ParseValueSymbolTable() {
     switch (Entry.Kind) {
     case BitstreamEntry::SubBlock: // Handled for us already.
     case BitstreamEntry::Error:
-      return Error("Malformed block");
+      return error("Malformed block");
     case BitstreamEntry::EndBlock:
       return std::error_code();
     case BitstreamEntry::Record:
@@ -1513,11 +1554,11 @@ std::error_code BitcodeReader::ParseValueSymbolTable() {
     default:  // Default behavior: unknown type.
       break;
     case bitc::VST_CODE_ENTRY: {  // VST_ENTRY: [valueid, namechar x N]
-      if (ConvertToString(Record, 1, ValueName))
-        return Error("Invalid record");
+      if (convertToString(Record, 1, ValueName))
+        return error("Invalid record");
       unsigned ValueID = Record[0];
       if (ValueID >= ValueList.size() || !ValueList[ValueID])
-        return Error("Invalid record");
+        return error("Invalid record");
       Value *V = ValueList[ValueID];
 
       V->setName(StringRef(ValueName.data(), ValueName.size()));
@@ -1533,11 +1574,11 @@ std::error_code BitcodeReader::ParseValueSymbolTable() {
       break;
     }
     case bitc::VST_CODE_BBENTRY: {
-      if (ConvertToString(Record, 1, ValueName))
-        return Error("Invalid record");
+      if (convertToString(Record, 1, ValueName))
+        return error("Invalid record");
       BasicBlock *BB = getBasicBlock(Record[0]);
       if (!BB)
-        return Error("Invalid record");
+        return error("Invalid record");
 
       BB->setName(StringRef(ValueName.data(), ValueName.size()));
       ValueName.clear();
@@ -1549,12 +1590,12 @@ std::error_code BitcodeReader::ParseValueSymbolTable() {
 
 static int64_t unrotateSign(uint64_t U) { return U & 1 ? ~(U >> 1) : U >> 1; }
 
-std::error_code BitcodeReader::ParseMetadata() {
+std::error_code BitcodeReader::parseMetadata() {
   IsMetadataMaterialized = true;
   unsigned NextMDValueNo = MDValueList.size();
 
   if (Stream.EnterSubBlock(bitc::METADATA_BLOCK_ID))
-    return Error("Invalid record");
+    return error("Invalid record");
 
   SmallVector<uint64_t, 64> Record;
 
@@ -1581,7 +1622,7 @@ std::error_code BitcodeReader::ParseMetadata() {
     switch (Entry.Kind) {
     case BitstreamEntry::SubBlock: // Handled for us already.
     case BitstreamEntry::Error:
-      return Error("Malformed block");
+      return error("Malformed block");
     case BitstreamEntry::EndBlock:
       MDValueList.tryToResolveCycles();
       return std::error_code();
@@ -1603,9 +1644,9 @@ std::error_code BitcodeReader::ParseMetadata() {
       Record.clear();
       Code = Stream.ReadCode();
 
-      // METADATA_NAME is always followed by METADATA_NAMED_NODE.
       unsigned NextBitCode = Stream.readRecord(Code, Record);
-      assert(NextBitCode == bitc::METADATA_NAMED_NODE); (void)NextBitCode;
+      if (NextBitCode != bitc::METADATA_NAMED_NODE)
+        return error("METADATA_NAME not followed by METADATA_NAMED_NODE");
 
       // Read named metadata elements.
       unsigned Size = Record.size();
@@ -1613,7 +1654,7 @@ std::error_code BitcodeReader::ParseMetadata() {
       for (unsigned i = 0; i != Size; ++i) {
         MDNode *MD = dyn_cast_or_null<MDNode>(MDValueList.getValueFwdRef(Record[i]));
         if (!MD)
-          return Error("Invalid record");
+          return error("Invalid record");
         NMD->addOperand(MD);
       }
       break;
@@ -1623,12 +1664,12 @@ std::error_code BitcodeReader::ParseMetadata() {
       // This is a LocalAsMetadata record, the only type of function-local
       // metadata.
       if (Record.size() % 2 == 1)
-        return Error("Invalid record");
+        return error("Invalid record");
 
       // If this isn't a LocalAsMetadata record, we're dropping it.  This used
       // to be legal, but there's no upgrade path.
       auto dropRecord = [&] {
-        MDValueList.AssignValue(MDNode::get(Context, None), NextMDValueNo++);
+        MDValueList.assignValue(MDNode::get(Context, None), NextMDValueNo++);
       };
       if (Record.size() != 2) {
         dropRecord();
@@ -1641,7 +1682,7 @@ std::error_code BitcodeReader::ParseMetadata() {
         break;
       }
 
-      MDValueList.AssignValue(
+      MDValueList.assignValue(
           LocalAsMetadata::get(ValueList.getValueFwdRef(Record[1], Ty)),
           NextMDValueNo++);
       break;
@@ -1649,14 +1690,14 @@ std::error_code BitcodeReader::ParseMetadata() {
     case bitc::METADATA_OLD_NODE: {
       // FIXME: Remove in 4.0.
       if (Record.size() % 2 == 1)
-        return Error("Invalid record");
+        return error("Invalid record");
 
       unsigned Size = Record.size();
       SmallVector<Metadata *, 8> Elts;
       for (unsigned i = 0; i != Size; i += 2) {
         Type *Ty = getTypeByID(Record[i]);
         if (!Ty)
-          return Error("Invalid record");
+          return error("Invalid record");
         if (Ty->isMetadataTy())
           Elts.push_back(MDValueList.getValueFwdRef(Record[i+1]));
         else if (!Ty->isVoidTy()) {
@@ -1668,18 +1709,18 @@ std::error_code BitcodeReader::ParseMetadata() {
         } else
           Elts.push_back(nullptr);
       }
-      MDValueList.AssignValue(MDNode::get(Context, Elts), NextMDValueNo++);
+      MDValueList.assignValue(MDNode::get(Context, Elts), NextMDValueNo++);
       break;
     }
     case bitc::METADATA_VALUE: {
       if (Record.size() != 2)
-        return Error("Invalid record");
+        return error("Invalid record");
 
       Type *Ty = getTypeByID(Record[0]);
       if (Ty->isMetadataTy() || Ty->isVoidTy())
-        return Error("Invalid record");
+        return error("Invalid record");
 
-      MDValueList.AssignValue(
+      MDValueList.assignValue(
           ValueAsMetadata::get(ValueList.getValueFwdRef(Record[1], Ty)),
           NextMDValueNo++);
       break;
@@ -1692,61 +1733,61 @@ std::error_code BitcodeReader::ParseMetadata() {
       Elts.reserve(Record.size());
       for (unsigned ID : Record)
         Elts.push_back(ID ? MDValueList.getValueFwdRef(ID - 1) : nullptr);
-      MDValueList.AssignValue(IsDistinct ? MDNode::getDistinct(Context, Elts)
+      MDValueList.assignValue(IsDistinct ? MDNode::getDistinct(Context, Elts)
                                          : MDNode::get(Context, Elts),
                               NextMDValueNo++);
       break;
     }
     case bitc::METADATA_LOCATION: {
       if (Record.size() != 5)
-        return Error("Invalid record");
+        return error("Invalid record");
 
       unsigned Line = Record[1];
       unsigned Column = Record[2];
       MDNode *Scope = cast<MDNode>(MDValueList.getValueFwdRef(Record[3]));
       Metadata *InlinedAt =
           Record[4] ? MDValueList.getValueFwdRef(Record[4] - 1) : nullptr;
-      MDValueList.AssignValue(
-          GET_OR_DISTINCT(MDLocation, Record[0],
+      MDValueList.assignValue(
+          GET_OR_DISTINCT(DILocation, Record[0],
                           (Context, Line, Column, Scope, InlinedAt)),
           NextMDValueNo++);
       break;
     }
     case bitc::METADATA_GENERIC_DEBUG: {
       if (Record.size() < 4)
-        return Error("Invalid record");
+        return error("Invalid record");
 
       unsigned Tag = Record[1];
       unsigned Version = Record[2];
 
       if (Tag >= 1u << 16 || Version != 0)
-        return Error("Invalid record");
+        return error("Invalid record");
 
       auto *Header = getMDString(Record[3]);
       SmallVector<Metadata *, 8> DwarfOps;
       for (unsigned I = 4, E = Record.size(); I != E; ++I)
         DwarfOps.push_back(Record[I] ? MDValueList.getValueFwdRef(Record[I] - 1)
                                      : nullptr);
-      MDValueList.AssignValue(GET_OR_DISTINCT(GenericDebugNode, Record[0],
+      MDValueList.assignValue(GET_OR_DISTINCT(GenericDINode, Record[0],
                                               (Context, Tag, Header, DwarfOps)),
                               NextMDValueNo++);
       break;
     }
     case bitc::METADATA_SUBRANGE: {
       if (Record.size() != 3)
-        return Error("Invalid record");
+        return error("Invalid record");
 
-      MDValueList.AssignValue(
-          GET_OR_DISTINCT(MDSubrange, Record[0],
+      MDValueList.assignValue(
+          GET_OR_DISTINCT(DISubrange, Record[0],
                           (Context, Record[1], unrotateSign(Record[2]))),
           NextMDValueNo++);
       break;
     }
     case bitc::METADATA_ENUMERATOR: {
       if (Record.size() != 3)
-        return Error("Invalid record");
+        return error("Invalid record");
 
-      MDValueList.AssignValue(GET_OR_DISTINCT(MDEnumerator, Record[0],
+      MDValueList.assignValue(GET_OR_DISTINCT(DIEnumerator, Record[0],
                                               (Context, unrotateSign(Record[1]),
                                                getMDString(Record[2]))),
                               NextMDValueNo++);
@@ -1754,10 +1795,10 @@ std::error_code BitcodeReader::ParseMetadata() {
     }
     case bitc::METADATA_BASIC_TYPE: {
       if (Record.size() != 6)
-        return Error("Invalid record");
+        return error("Invalid record");
 
-      MDValueList.AssignValue(
-          GET_OR_DISTINCT(MDBasicType, Record[0],
+      MDValueList.assignValue(
+          GET_OR_DISTINCT(DIBasicType, Record[0],
                           (Context, Record[1], getMDString(Record[2]),
                            Record[3], Record[4], Record[5])),
           NextMDValueNo++);
@@ -1765,10 +1806,10 @@ std::error_code BitcodeReader::ParseMetadata() {
     }
     case bitc::METADATA_DERIVED_TYPE: {
       if (Record.size() != 12)
-        return Error("Invalid record");
+        return error("Invalid record");
 
-      MDValueList.AssignValue(
-          GET_OR_DISTINCT(MDDerivedType, Record[0],
+      MDValueList.assignValue(
+          GET_OR_DISTINCT(DIDerivedType, Record[0],
                           (Context, Record[1], getMDString(Record[2]),
                            getMDOrNull(Record[3]), Record[4],
                            getMDOrNull(Record[5]), getMDOrNull(Record[6]),
@@ -1779,10 +1820,10 @@ std::error_code BitcodeReader::ParseMetadata() {
     }
     case bitc::METADATA_COMPOSITE_TYPE: {
       if (Record.size() != 16)
-        return Error("Invalid record");
+        return error("Invalid record");
 
-      MDValueList.AssignValue(
-          GET_OR_DISTINCT(MDCompositeType, Record[0],
+      MDValueList.assignValue(
+          GET_OR_DISTINCT(DICompositeType, Record[0],
                           (Context, Record[1], getMDString(Record[2]),
                            getMDOrNull(Record[3]), Record[4],
                            getMDOrNull(Record[5]), getMDOrNull(Record[6]),
@@ -1795,47 +1836,61 @@ std::error_code BitcodeReader::ParseMetadata() {
     }
     case bitc::METADATA_SUBROUTINE_TYPE: {
       if (Record.size() != 3)
-        return Error("Invalid record");
+        return error("Invalid record");
 
-      MDValueList.AssignValue(
-          GET_OR_DISTINCT(MDSubroutineType, Record[0],
+      MDValueList.assignValue(
+          GET_OR_DISTINCT(DISubroutineType, Record[0],
                           (Context, Record[1], getMDOrNull(Record[2]))),
           NextMDValueNo++);
       break;
     }
+
+    case bitc::METADATA_MODULE: {
+      if (Record.size() != 6)
+        return error("Invalid record");
+
+      MDValueList.assignValue(
+          GET_OR_DISTINCT(DIModule, Record[0],
+                          (Context, getMDOrNull(Record[1]),
+                          getMDString(Record[2]), getMDString(Record[3]),
+                          getMDString(Record[4]), getMDString(Record[5]))),
+          NextMDValueNo++);
+      break;
+    }
+
     case bitc::METADATA_FILE: {
       if (Record.size() != 3)
-        return Error("Invalid record");
+        return error("Invalid record");
 
-      MDValueList.AssignValue(
-          GET_OR_DISTINCT(MDFile, Record[0], (Context, getMDString(Record[1]),
+      MDValueList.assignValue(
+          GET_OR_DISTINCT(DIFile, Record[0], (Context, getMDString(Record[1]),
                                               getMDString(Record[2]))),
           NextMDValueNo++);
       break;
     }
     case bitc::METADATA_COMPILE_UNIT: {
-      if (Record.size() != 14)
-        return Error("Invalid record");
+      if (Record.size() < 14 || Record.size() > 15)
+        return error("Invalid record");
 
-      MDValueList.AssignValue(
-          GET_OR_DISTINCT(MDCompileUnit, Record[0],
-                          (Context, Record[1], getMDOrNull(Record[2]),
-                           getMDString(Record[3]), Record[4],
-                           getMDString(Record[5]), Record[6],
-                           getMDString(Record[7]), Record[8],
-                           getMDOrNull(Record[9]), getMDOrNull(Record[10]),
-                           getMDOrNull(Record[11]), getMDOrNull(Record[12]),
-                           getMDOrNull(Record[13]))),
+      MDValueList.assignValue(
+          GET_OR_DISTINCT(
+              DICompileUnit, Record[0],
+              (Context, Record[1], getMDOrNull(Record[2]),
+               getMDString(Record[3]), Record[4], getMDString(Record[5]),
+               Record[6], getMDString(Record[7]), Record[8],
+               getMDOrNull(Record[9]), getMDOrNull(Record[10]),
+               getMDOrNull(Record[11]), getMDOrNull(Record[12]),
+               getMDOrNull(Record[13]), Record.size() == 14 ? 0 : Record[14])),
           NextMDValueNo++);
       break;
     }
     case bitc::METADATA_SUBPROGRAM: {
       if (Record.size() != 19)
-        return Error("Invalid record");
+        return error("Invalid record");
 
-      MDValueList.AssignValue(
+      MDValueList.assignValue(
           GET_OR_DISTINCT(
-              MDSubprogram, Record[0],
+              DISubprogram, Record[0],
               (Context, getMDOrNull(Record[1]), getMDString(Record[2]),
                getMDString(Record[3]), getMDOrNull(Record[4]), Record[5],
                getMDOrNull(Record[6]), Record[7], Record[8], Record[9],
@@ -1847,10 +1902,10 @@ std::error_code BitcodeReader::ParseMetadata() {
     }
     case bitc::METADATA_LEXICAL_BLOCK: {
       if (Record.size() != 5)
-        return Error("Invalid record");
+        return error("Invalid record");
 
-      MDValueList.AssignValue(
-          GET_OR_DISTINCT(MDLexicalBlock, Record[0],
+      MDValueList.assignValue(
+          GET_OR_DISTINCT(DILexicalBlock, Record[0],
                           (Context, getMDOrNull(Record[1]),
                            getMDOrNull(Record[2]), Record[3], Record[4])),
           NextMDValueNo++);
@@ -1858,10 +1913,10 @@ std::error_code BitcodeReader::ParseMetadata() {
     }
     case bitc::METADATA_LEXICAL_BLOCK_FILE: {
       if (Record.size() != 4)
-        return Error("Invalid record");
+        return error("Invalid record");
 
-      MDValueList.AssignValue(
-          GET_OR_DISTINCT(MDLexicalBlockFile, Record[0],
+      MDValueList.assignValue(
+          GET_OR_DISTINCT(DILexicalBlockFile, Record[0],
                           (Context, getMDOrNull(Record[1]),
                            getMDOrNull(Record[2]), Record[3])),
           NextMDValueNo++);
@@ -1869,10 +1924,10 @@ std::error_code BitcodeReader::ParseMetadata() {
     }
     case bitc::METADATA_NAMESPACE: {
       if (Record.size() != 5)
-        return Error("Invalid record");
+        return error("Invalid record");
 
-      MDValueList.AssignValue(
-          GET_OR_DISTINCT(MDNamespace, Record[0],
+      MDValueList.assignValue(
+          GET_OR_DISTINCT(DINamespace, Record[0],
                           (Context, getMDOrNull(Record[1]),
                            getMDOrNull(Record[2]), getMDString(Record[3]),
                            Record[4])),
@@ -1881,9 +1936,9 @@ std::error_code BitcodeReader::ParseMetadata() {
     }
     case bitc::METADATA_TEMPLATE_TYPE: {
       if (Record.size() != 3)
-        return Error("Invalid record");
+        return error("Invalid record");
 
-      MDValueList.AssignValue(GET_OR_DISTINCT(MDTemplateTypeParameter,
+      MDValueList.assignValue(GET_OR_DISTINCT(DITemplateTypeParameter,
                                               Record[0],
                                               (Context, getMDString(Record[1]),
                                                getMDOrNull(Record[2]))),
@@ -1892,10 +1947,10 @@ std::error_code BitcodeReader::ParseMetadata() {
     }
     case bitc::METADATA_TEMPLATE_VALUE: {
       if (Record.size() != 5)
-        return Error("Invalid record");
+        return error("Invalid record");
 
-      MDValueList.AssignValue(
-          GET_OR_DISTINCT(MDTemplateValueParameter, Record[0],
+      MDValueList.assignValue(
+          GET_OR_DISTINCT(DITemplateValueParameter, Record[0],
                           (Context, Record[1], getMDString(Record[2]),
                            getMDOrNull(Record[3]), getMDOrNull(Record[4]))),
           NextMDValueNo++);
@@ -1903,10 +1958,10 @@ std::error_code BitcodeReader::ParseMetadata() {
     }
     case bitc::METADATA_GLOBAL_VAR: {
       if (Record.size() != 11)
-        return Error("Invalid record");
+        return error("Invalid record");
 
-      MDValueList.AssignValue(
-          GET_OR_DISTINCT(MDGlobalVariable, Record[0],
+      MDValueList.assignValue(
+          GET_OR_DISTINCT(DIGlobalVariable, Record[0],
                           (Context, getMDOrNull(Record[1]),
                            getMDString(Record[2]), getMDString(Record[3]),
                            getMDOrNull(Record[4]), Record[5],
@@ -1918,10 +1973,10 @@ std::error_code BitcodeReader::ParseMetadata() {
     case bitc::METADATA_LOCAL_VAR: {
       // 10th field is for the obseleted 'inlinedAt:' field.
       if (Record.size() != 9 && Record.size() != 10)
-        return Error("Invalid record");
+        return error("Invalid record");
 
-      MDValueList.AssignValue(
-          GET_OR_DISTINCT(MDLocalVariable, Record[0],
+      MDValueList.assignValue(
+          GET_OR_DISTINCT(DILocalVariable, Record[0],
                           (Context, Record[1], getMDOrNull(Record[2]),
                            getMDString(Record[3]), getMDOrNull(Record[4]),
                            Record[5], getMDOrNull(Record[6]), Record[7],
@@ -1931,20 +1986,20 @@ std::error_code BitcodeReader::ParseMetadata() {
     }
     case bitc::METADATA_EXPRESSION: {
       if (Record.size() < 1)
-        return Error("Invalid record");
+        return error("Invalid record");
 
-      MDValueList.AssignValue(
-          GET_OR_DISTINCT(MDExpression, Record[0],
+      MDValueList.assignValue(
+          GET_OR_DISTINCT(DIExpression, Record[0],
                           (Context, makeArrayRef(Record).slice(1))),
           NextMDValueNo++);
       break;
     }
     case bitc::METADATA_OBJC_PROPERTY: {
       if (Record.size() != 8)
-        return Error("Invalid record");
+        return error("Invalid record");
 
-      MDValueList.AssignValue(
-          GET_OR_DISTINCT(MDObjCProperty, Record[0],
+      MDValueList.assignValue(
+          GET_OR_DISTINCT(DIObjCProperty, Record[0],
                           (Context, getMDString(Record[1]),
                            getMDOrNull(Record[2]), Record[3],
                            getMDString(Record[4]), getMDString(Record[5]),
@@ -1954,10 +2009,10 @@ std::error_code BitcodeReader::ParseMetadata() {
     }
     case bitc::METADATA_IMPORTED_ENTITY: {
       if (Record.size() != 6)
-        return Error("Invalid record");
+        return error("Invalid record");
 
-      MDValueList.AssignValue(
-          GET_OR_DISTINCT(MDImportedEntity, Record[0],
+      MDValueList.assignValue(
+          GET_OR_DISTINCT(DIImportedEntity, Record[0],
                           (Context, Record[1], getMDOrNull(Record[2]),
                            getMDOrNull(Record[3]), Record[4],
                            getMDString(Record[5]))),
@@ -1968,19 +2023,19 @@ std::error_code BitcodeReader::ParseMetadata() {
       std::string String(Record.begin(), Record.end());
       llvm::UpgradeMDStringConstant(String);
       Metadata *MD = MDString::get(Context, String);
-      MDValueList.AssignValue(MD, NextMDValueNo++);
+      MDValueList.assignValue(MD, NextMDValueNo++);
       break;
     }
     case bitc::METADATA_KIND: {
       if (Record.size() < 2)
-        return Error("Invalid record");
+        return error("Invalid record");
 
       unsigned Kind = Record[0];
       SmallString<8> Name(Record.begin()+1, Record.end());
 
       unsigned NewKind = TheModule->getMDKindID(Name.str());
       if (!MDKindMap.insert(std::make_pair(Kind, NewKind)).second)
-        return Error("Conflicting METADATA_KIND records");
+        return error("Conflicting METADATA_KIND records");
       break;
     }
     }
@@ -1988,8 +2043,8 @@ std::error_code BitcodeReader::ParseMetadata() {
 #undef GET_OR_DISTINCT
 }
 
-/// decodeSignRotatedValue - Decode a signed value stored with the sign bit in
-/// the LSB for dense VBR encoding.
+/// Decode a signed value stored with the sign bit in the LSB for dense VBR
+/// encoding.
 uint64_t BitcodeReader::decodeSignRotatedValue(uint64_t V) {
   if ((V & 1) == 0)
     return V >> 1;
@@ -1999,18 +2054,19 @@ uint64_t BitcodeReader::decodeSignRotatedValue(uint64_t V) {
   return 1ULL << 63;
 }
 
-/// ResolveGlobalAndAliasInits - Resolve all of the initializers for global
-/// values and aliases that we can.
-std::error_code BitcodeReader::ResolveGlobalAndAliasInits() {
+/// Resolve all of the initializers for global values and aliases that we can.
+std::error_code BitcodeReader::resolveGlobalAndAliasInits() {
   std::vector<std::pair<GlobalVariable*, unsigned> > GlobalInitWorklist;
   std::vector<std::pair<GlobalAlias*, unsigned> > AliasInitWorklist;
   std::vector<std::pair<Function*, unsigned> > FunctionPrefixWorklist;
   std::vector<std::pair<Function*, unsigned> > FunctionPrologueWorklist;
+  std::vector<std::pair<Function*, unsigned> > FunctionPersonalityFnWorklist;
 
   GlobalInitWorklist.swap(GlobalInits);
   AliasInitWorklist.swap(AliasInits);
   FunctionPrefixWorklist.swap(FunctionPrefixes);
   FunctionPrologueWorklist.swap(FunctionPrologues);
+  FunctionPersonalityFnWorklist.swap(FunctionPersonalityFns);
 
   while (!GlobalInitWorklist.empty()) {
     unsigned ValID = GlobalInitWorklist.back().second;
@@ -2021,7 +2077,7 @@ std::error_code BitcodeReader::ResolveGlobalAndAliasInits() {
       if (Constant *C = dyn_cast_or_null<Constant>(ValueList[ValID]))
         GlobalInitWorklist.back().first->setInitializer(C);
       else
-        return Error("Expected a constant");
+        return error("Expected a constant");
     }
     GlobalInitWorklist.pop_back();
   }
@@ -2031,10 +2087,13 @@ std::error_code BitcodeReader::ResolveGlobalAndAliasInits() {
     if (ValID >= ValueList.size()) {
       AliasInits.push_back(AliasInitWorklist.back());
     } else {
-      if (Constant *C = dyn_cast_or_null<Constant>(ValueList[ValID]))
-        AliasInitWorklist.back().first->setAliasee(C);
-      else
-        return Error("Expected a constant");
+      Constant *C = dyn_cast_or_null<Constant>(ValueList[ValID]);
+      if (!C)
+        return error("Expected a constant");
+      GlobalAlias *Alias = AliasInitWorklist.back().first;
+      if (C->getType() != Alias->getType())
+        return error("Alias and aliasee types don't match");
+      Alias->setAliasee(C);
     }
     AliasInitWorklist.pop_back();
   }
@@ -2047,7 +2106,7 @@ std::error_code BitcodeReader::ResolveGlobalAndAliasInits() {
       if (Constant *C = dyn_cast_or_null<Constant>(ValueList[ValID]))
         FunctionPrefixWorklist.back().first->setPrefixData(C);
       else
-        return Error("Expected a constant");
+        return error("Expected a constant");
     }
     FunctionPrefixWorklist.pop_back();
   }
@@ -2060,15 +2119,28 @@ std::error_code BitcodeReader::ResolveGlobalAndAliasInits() {
       if (Constant *C = dyn_cast_or_null<Constant>(ValueList[ValID]))
         FunctionPrologueWorklist.back().first->setPrologueData(C);
       else
-        return Error("Expected a constant");
+        return error("Expected a constant");
     }
     FunctionPrologueWorklist.pop_back();
   }
 
+  while (!FunctionPersonalityFnWorklist.empty()) {
+    unsigned ValID = FunctionPersonalityFnWorklist.back().second;
+    if (ValID >= ValueList.size()) {
+      FunctionPersonalityFns.push_back(FunctionPersonalityFnWorklist.back());
+    } else {
+      if (Constant *C = dyn_cast_or_null<Constant>(ValueList[ValID]))
+        FunctionPersonalityFnWorklist.back().first->setPersonalityFn(C);
+      else
+        return error("Expected a constant");
+    }
+    FunctionPersonalityFnWorklist.pop_back();
+  }
+
   return std::error_code();
 }
 
-static APInt ReadWideAPInt(ArrayRef<uint64_t> Vals, unsigned TypeBits) {
+static APInt readWideAPInt(ArrayRef<uint64_t> Vals, unsigned TypeBits) {
   SmallVector<uint64_t, 8> Words(Vals.size());
   std::transform(Vals.begin(), Vals.end(), Words.begin(),
                  BitcodeReader::decodeSignRotatedValue);
@@ -2076,9 +2148,9 @@ static APInt ReadWideAPInt(ArrayRef<uint64_t> Vals, unsigned TypeBits) {
   return APInt(TypeBits, Words);
 }
 
-std::error_code BitcodeReader::ParseConstants() {
+std::error_code BitcodeReader::parseConstants() {
   if (Stream.EnterSubBlock(bitc::CONSTANTS_BLOCK_ID))
-    return Error("Invalid record");
+    return error("Invalid record");
 
   SmallVector<uint64_t, 64> Record;
 
@@ -2091,14 +2163,14 @@ std::error_code BitcodeReader::ParseConstants() {
     switch (Entry.Kind) {
     case BitstreamEntry::SubBlock: // Handled for us already.
     case BitstreamEntry::Error:
-      return Error("Malformed block");
+      return error("Malformed block");
     case BitstreamEntry::EndBlock:
       if (NextCstNo != ValueList.size())
-        return Error("Invalid ronstant reference");
+        return error("Invalid ronstant reference");
 
       // Once all the constants have been read, go through and resolve forward
       // references.
-      ValueList.ResolveConstantForwardRefs();
+      ValueList.resolveConstantForwardRefs();
       return std::error_code();
     case BitstreamEntry::Record:
       // The interesting case.
@@ -2116,9 +2188,9 @@ std::error_code BitcodeReader::ParseConstants() {
       break;
     case bitc::CST_CODE_SETTYPE:   // SETTYPE: [typeid]
       if (Record.empty())
-        return Error("Invalid record");
+        return error("Invalid record");
       if (Record[0] >= TypeList.size() || !TypeList[Record[0]])
-        return Error("Invalid record");
+        return error("Invalid record");
       CurTy = TypeList[Record[0]];
       continue;  // Skip the ValueList manipulation.
     case bitc::CST_CODE_NULL:      // NULL
@@ -2126,22 +2198,22 @@ std::error_code BitcodeReader::ParseConstants() {
       break;
     case bitc::CST_CODE_INTEGER:   // INTEGER: [intval]
       if (!CurTy->isIntegerTy() || Record.empty())
-        return Error("Invalid record");
+        return error("Invalid record");
       V = ConstantInt::get(CurTy, decodeSignRotatedValue(Record[0]));
       break;
     case bitc::CST_CODE_WIDE_INTEGER: {// WIDE_INTEGER: [n x intval]
       if (!CurTy->isIntegerTy() || Record.empty())
-        return Error("Invalid record");
+        return error("Invalid record");
 
-      APInt VInt = ReadWideAPInt(Record,
-                                 cast<IntegerType>(CurTy)->getBitWidth());
+      APInt VInt =
+          readWideAPInt(Record, cast<IntegerType>(CurTy)->getBitWidth());
       V = ConstantInt::get(Context, VInt);
 
       break;
     }
     case bitc::CST_CODE_FLOAT: {    // FLOAT: [fpval]
       if (Record.empty())
-        return Error("Invalid record");
+        return error("Invalid record");
       if (CurTy->isHalfTy())
         V = ConstantFP::get(Context, APFloat(APFloat::IEEEhalf,
                                              APInt(16, (uint16_t)Record[0])));
@@ -2171,7 +2243,7 @@ std::error_code BitcodeReader::ParseConstants() {
 
     case bitc::CST_CODE_AGGREGATE: {// AGGREGATE: [n x value number]
       if (Record.empty())
-        return Error("Invalid record");
+        return error("Invalid record");
 
       unsigned Size = Record.size();
       SmallVector<Constant*, 16> Elts;
@@ -2199,7 +2271,7 @@ std::error_code BitcodeReader::ParseConstants() {
     case bitc::CST_CODE_STRING:    // STRING: [values]
     case bitc::CST_CODE_CSTRING: { // CSTRING: [values]
       if (Record.empty())
-        return Error("Invalid record");
+        return error("Invalid record");
 
       SmallString<16> Elts(Record.begin(), Record.end());
       V = ConstantDataArray::getString(Context, Elts,
@@ -2208,7 +2280,7 @@ std::error_code BitcodeReader::ParseConstants() {
     }
     case bitc::CST_CODE_DATA: {// DATA: [n x value]
       if (Record.empty())
-        return Error("Invalid record");
+        return error("Invalid record");
 
       Type *EltTy = cast<SequentialType>(CurTy)->getElementType();
       unsigned Size = Record.size();
@@ -2253,15 +2325,15 @@ std::error_code BitcodeReader::ParseConstants() {
         else
           V = ConstantDataArray::get(Context, Elts);
       } else {
-        return Error("Invalid type for value");
+        return error("Invalid type for value");
       }
       break;
     }
 
     case bitc::CST_CODE_CE_BINOP: {  // CE_BINOP: [opcode, opval, opval]
       if (Record.size() < 3)
-        return Error("Invalid record");
-      int Opc = GetDecodedBinaryOpcode(Record[0], CurTy);
+        return error("Invalid record");
+      int Opc = getDecodedBinaryOpcode(Record[0], CurTy);
       if (Opc < 0) {
         V = UndefValue::get(CurTy);  // Unknown binop.
       } else {
@@ -2291,14 +2363,14 @@ std::error_code BitcodeReader::ParseConstants() {
     }
     case bitc::CST_CODE_CE_CAST: {  // CE_CAST: [opcode, opty, opval]
       if (Record.size() < 3)
-        return Error("Invalid record");
-      int Opc = GetDecodedCastOpcode(Record[0]);
+        return error("Invalid record");
+      int Opc = getDecodedCastOpcode(Record[0]);
       if (Opc < 0) {
         V = UndefValue::get(CurTy);  // Unknown cast.
       } else {
         Type *OpTy = getTypeByID(Record[1]);
         if (!OpTy)
-          return Error("Invalid record");
+          return error("Invalid record");
         Constant *Op = ValueList.getConstantFwdRef(Record[2], OpTy);
         V = UpgradeBitCastExpr(Opc, Op, CurTy);
         if (!V) V = ConstantExpr::getCast(Opc, Op, CurTy);
@@ -2315,7 +2387,7 @@ std::error_code BitcodeReader::ParseConstants() {
       while (OpNum != Record.size()) {
         Type *ElTy = getTypeByID(Record[OpNum++]);
         if (!ElTy)
-          return Error("Invalid record");
+          return error("Invalid record");
         Elts.push_back(ValueList.getConstantFwdRef(Record[OpNum++], ElTy));
       }
 
@@ -2323,7 +2395,7 @@ std::error_code BitcodeReader::ParseConstants() {
           PointeeType !=
               cast<SequentialType>(Elts[0]->getType()->getScalarType())
                   ->getElementType())
-        return Error("Explicit gep operator type does not match pointee type "
+        return error("Explicit gep operator type does not match pointee type "
                      "of pointer operand");
 
       ArrayRef<Constant *> Indices(Elts.begin() + 1, Elts.end());
@@ -2334,7 +2406,7 @@ std::error_code BitcodeReader::ParseConstants() {
     }
     case bitc::CST_CODE_CE_SELECT: {  // CE_SELECT: [opval#, opval#, opval#]
       if (Record.size() < 3)
-        return Error("Invalid record");
+        return error("Invalid record");
 
       Type *SelectorTy = Type::getInt1Ty(Context);
 
@@ -2353,22 +2425,22 @@ std::error_code BitcodeReader::ParseConstants() {
     case bitc::CST_CODE_CE_EXTRACTELT
         : { // CE_EXTRACTELT: [opty, opval, opty, opval]
       if (Record.size() < 3)
-        return Error("Invalid record");
+        return error("Invalid record");
       VectorType *OpTy =
         dyn_cast_or_null<VectorType>(getTypeByID(Record[0]));
       if (!OpTy)
-        return Error("Invalid record");
+        return error("Invalid record");
       Constant *Op0 = ValueList.getConstantFwdRef(Record[1], OpTy);
       Constant *Op1 = nullptr;
       if (Record.size() == 4) {
         Type *IdxTy = getTypeByID(Record[2]);
         if (!IdxTy)
-          return Error("Invalid record");
+          return error("Invalid record");
         Op1 = ValueList.getConstantFwdRef(Record[3], IdxTy);
       } else // TODO: Remove with llvm 4.0
         Op1 = ValueList.getConstantFwdRef(Record[2], Type::getInt32Ty(Context));
       if (!Op1)
-        return Error("Invalid record");
+        return error("Invalid record");
       V = ConstantExpr::getExtractElement(Op0, Op1);
       break;
     }
@@ -2376,7 +2448,7 @@ std::error_code BitcodeReader::ParseConstants() {
         : { // CE_INSERTELT: [opval, opval, opty, opval]
       VectorType *OpTy = dyn_cast<VectorType>(CurTy);
       if (Record.size() < 3 || !OpTy)
-        return Error("Invalid record");
+        return error("Invalid record");
       Constant *Op0 = ValueList.getConstantFwdRef(Record[0], OpTy);
       Constant *Op1 = ValueList.getConstantFwdRef(Record[1],
                                                   OpTy->getElementType());
@@ -2384,19 +2456,19 @@ std::error_code BitcodeReader::ParseConstants() {
       if (Record.size() == 4) {
         Type *IdxTy = getTypeByID(Record[2]);
         if (!IdxTy)
-          return Error("Invalid record");
+          return error("Invalid record");
         Op2 = ValueList.getConstantFwdRef(Record[3], IdxTy);
       } else // TODO: Remove with llvm 4.0
         Op2 = ValueList.getConstantFwdRef(Record[2], Type::getInt32Ty(Context));
       if (!Op2)
-        return Error("Invalid record");
+        return error("Invalid record");
       V = ConstantExpr::getInsertElement(Op0, Op1, Op2);
       break;
     }
     case bitc::CST_CODE_CE_SHUFFLEVEC: { // CE_SHUFFLEVEC: [opval, opval, opval]
       VectorType *OpTy = dyn_cast<VectorType>(CurTy);
       if (Record.size() < 3 || !OpTy)
-        return Error("Invalid record");
+        return error("Invalid record");
       Constant *Op0 = ValueList.getConstantFwdRef(Record[0], OpTy);
       Constant *Op1 = ValueList.getConstantFwdRef(Record[1], OpTy);
       Type *ShufTy = VectorType::get(Type::getInt32Ty(Context),
@@ -2410,7 +2482,7 @@ std::error_code BitcodeReader::ParseConstants() {
       VectorType *OpTy =
         dyn_cast_or_null<VectorType>(getTypeByID(Record[0]));
       if (Record.size() < 4 || !RTy || !OpTy)
-        return Error("Invalid record");
+        return error("Invalid record");
       Constant *Op0 = ValueList.getConstantFwdRef(Record[1], OpTy);
       Constant *Op1 = ValueList.getConstantFwdRef(Record[2], OpTy);
       Type *ShufTy = VectorType::get(Type::getInt32Ty(Context),
@@ -2421,10 +2493,10 @@ std::error_code BitcodeReader::ParseConstants() {
     }
     case bitc::CST_CODE_CE_CMP: {     // CE_CMP: [opty, opval, opval, pred]
       if (Record.size() < 4)
-        return Error("Invalid record");
+        return error("Invalid record");
       Type *OpTy = getTypeByID(Record[0]);
       if (!OpTy)
-        return Error("Invalid record");
+        return error("Invalid record");
       Constant *Op0 = ValueList.getConstantFwdRef(Record[1], OpTy);
       Constant *Op1 = ValueList.getConstantFwdRef(Record[2], OpTy);
 
@@ -2438,16 +2510,16 @@ std::error_code BitcodeReader::ParseConstants() {
     // FIXME: Remove with the 4.0 release.
     case bitc::CST_CODE_INLINEASM_OLD: {
       if (Record.size() < 2)
-        return Error("Invalid record");
+        return error("Invalid record");
       std::string AsmStr, ConstrStr;
       bool HasSideEffects = Record[0] & 1;
       bool IsAlignStack = Record[0] >> 1;
       unsigned AsmStrSize = Record[1];
       if (2+AsmStrSize >= Record.size())
-        return Error("Invalid record");
+        return error("Invalid record");
       unsigned ConstStrSize = Record[2+AsmStrSize];
       if (3+AsmStrSize+ConstStrSize > Record.size())
-        return Error("Invalid record");
+        return error("Invalid record");
 
       for (unsigned i = 0; i != AsmStrSize; ++i)
         AsmStr += (char)Record[2+i];
@@ -2462,17 +2534,17 @@ std::error_code BitcodeReader::ParseConstants() {
     // inteldialect).
     case bitc::CST_CODE_INLINEASM: {
       if (Record.size() < 2)
-        return Error("Invalid record");
+        return error("Invalid record");
       std::string AsmStr, ConstrStr;
       bool HasSideEffects = Record[0] & 1;
       bool IsAlignStack = (Record[0] >> 1) & 1;
       unsigned AsmDialect = Record[0] >> 2;
       unsigned AsmStrSize = Record[1];
       if (2+AsmStrSize >= Record.size())
-        return Error("Invalid record");
+        return error("Invalid record");
       unsigned ConstStrSize = Record[2+AsmStrSize];
       if (3+AsmStrSize+ConstStrSize > Record.size())
-        return Error("Invalid record");
+        return error("Invalid record");
 
       for (unsigned i = 0; i != AsmStrSize; ++i)
         AsmStr += (char)Record[2+i];
@@ -2486,14 +2558,14 @@ std::error_code BitcodeReader::ParseConstants() {
     }
     case bitc::CST_CODE_BLOCKADDRESS:{
       if (Record.size() < 3)
-        return Error("Invalid record");
+        return error("Invalid record");
       Type *FnTy = getTypeByID(Record[0]);
       if (!FnTy)
-        return Error("Invalid record");
+        return error("Invalid record");
       Function *Fn =
         dyn_cast_or_null<Function>(ValueList.getConstantFwdRef(Record[1],FnTy));
       if (!Fn)
-        return Error("Invalid record");
+        return error("Invalid record");
 
       // Don't let Fn get dematerialized.
       BlockAddressesTaken.insert(Fn);
@@ -2504,12 +2576,12 @@ std::error_code BitcodeReader::ParseConstants() {
       unsigned BBID = Record[2];
       if (!BBID)
         // Invalid reference to entry block.
-        return Error("Invalid ID");
+        return error("Invalid ID");
       if (!Fn->empty()) {
         Function::iterator BBI = Fn->begin(), BBE = Fn->end();
         for (size_t I = 0, E = BBID; I != E; ++I) {
           if (BBI == BBE)
-            return Error("Invalid ID");
+            return error("Invalid ID");
           ++BBI;
         }
         BB = BBI;
@@ -2530,14 +2602,14 @@ std::error_code BitcodeReader::ParseConstants() {
     }
     }
 
-    ValueList.AssignValue(V, NextCstNo);
+    ValueList.assignValue(V, NextCstNo);
     ++NextCstNo;
   }
 }
 
-std::error_code BitcodeReader::ParseUseLists() {
+std::error_code BitcodeReader::parseUseLists() {
   if (Stream.EnterSubBlock(bitc::USELIST_BLOCK_ID))
-    return Error("Invalid record");
+    return error("Invalid record");
 
   // Read all the records.
   SmallVector<uint64_t, 64> Record;
@@ -2547,7 +2619,7 @@ std::error_code BitcodeReader::ParseUseLists() {
     switch (Entry.Kind) {
     case BitstreamEntry::SubBlock: // Handled for us already.
     case BitstreamEntry::Error:
-      return Error("Malformed block");
+      return error("Malformed block");
     case BitstreamEntry::EndBlock:
       return std::error_code();
     case BitstreamEntry::Record:
@@ -2568,7 +2640,7 @@ std::error_code BitcodeReader::ParseUseLists() {
       unsigned RecordLength = Record.size();
       if (RecordLength < 3)
         // Records should have at least an ID and two indexes.
-        return Error("Invalid record");
+        return error("Invalid record");
       unsigned ID = Record.back();
       Record.pop_back();
 
@@ -2608,7 +2680,7 @@ std::error_code BitcodeReader::rememberAndSkipMetadata() {
 
   // Skip over the block for now.
   if (Stream.SkipBlock())
-    return Error("Invalid record");
+    return error("Invalid record");
   return std::error_code();
 }
 
@@ -2616,7 +2688,7 @@ std::error_code BitcodeReader::materializeMetadata() {
   for (uint64_t BitPos : DeferredMetadataInfo) {
     // Move the bit stream to the saved position.
     Stream.JumpToBit(BitPos);
-    if (std::error_code EC = ParseMetadata())
+    if (std::error_code EC = parseMetadata())
       return EC;
   }
   DeferredMetadataInfo.clear();
@@ -2625,13 +2697,12 @@ std::error_code BitcodeReader::materializeMetadata() {
 
 void BitcodeReader::setStripDebugInfo() { StripDebugInfo = true; }
 
-/// RememberAndSkipFunctionBody - When we see the block for a function body,
-/// remember where it is and then skip it.  This lets us lazily deserialize the
-/// functions.
-std::error_code BitcodeReader::RememberAndSkipFunctionBody() {
+/// When we see the block for a function body, remember where it is and then
+/// skip it.  This lets us lazily deserialize the functions.
+std::error_code BitcodeReader::rememberAndSkipFunctionBody() {
   // Get the function we are talking about.
   if (FunctionsWithBodies.empty())
-    return Error("Insufficient function protos");
+    return error("Insufficient function protos");
 
   Function *Fn = FunctionsWithBodies.back();
   FunctionsWithBodies.pop_back();
@@ -2642,31 +2713,26 @@ std::error_code BitcodeReader::RememberAndSkipFunctionBody() {
 
   // Skip over the function block for now.
   if (Stream.SkipBlock())
-    return Error("Invalid record");
+    return error("Invalid record");
   return std::error_code();
 }
 
-std::error_code BitcodeReader::GlobalCleanup() {
+std::error_code BitcodeReader::globalCleanup() {
   // Patch the initializers for globals and aliases up.
-  ResolveGlobalAndAliasInits();
+  resolveGlobalAndAliasInits();
   if (!GlobalInits.empty() || !AliasInits.empty())
-    return Error("Malformed global initializer set");
+    return error("Malformed global initializer set");
 
   // Look for intrinsic functions which need to be upgraded at some point
-  for (Module::iterator FI = TheModule->begin(), FE = TheModule->end();
-       FI != FE; ++FI) {
+  for (Function &F : *TheModule) {
     Function *NewFn;
-    if (UpgradeIntrinsicFunction(FI, NewFn))
-      UpgradedIntrinsics.push_back(std::make_pair(FI, NewFn));
+    if (UpgradeIntrinsicFunction(&F, NewFn))
+      UpgradedIntrinsics[&F] = NewFn;
   }
 
   // Look for global variables which need to be renamed.
-  for (Module::global_iterator
-         GI = TheModule->global_begin(), GE = TheModule->global_end();
-       GI != GE;) {
-    GlobalVariable *GV = GI++;
-    UpgradeGlobalVariable(GV);
-  }
+  for (GlobalVariable &GV : TheModule->globals())
+    UpgradeGlobalVariable(&GV);
 
   // Force deallocation of memory for these vectors to favor the client that
   // want lazy deserialization.
@@ -2675,12 +2741,12 @@ std::error_code BitcodeReader::GlobalCleanup() {
   return std::error_code();
 }
 
-std::error_code BitcodeReader::ParseModule(bool Resume,
+std::error_code BitcodeReader::parseModule(bool Resume,
                                            bool ShouldLazyLoadMetadata) {
   if (Resume)
     Stream.JumpToBit(NextUnreadBit);
   else if (Stream.EnterSubBlock(bitc::MODULE_BLOCK_ID))
-    return Error("Invalid record");
+    return error("Invalid record");
 
   SmallVector<uint64_t, 64> Record;
   std::vector<std::string> SectionTable;
@@ -2692,41 +2758,41 @@ std::error_code BitcodeReader::ParseModule(bool Resume,
 
     switch (Entry.Kind) {
     case BitstreamEntry::Error:
-      return Error("Malformed block");
+      return error("Malformed block");
     case BitstreamEntry::EndBlock:
-      return GlobalCleanup();
+      return globalCleanup();
 
     case BitstreamEntry::SubBlock:
       switch (Entry.ID) {
       default:  // Skip unknown content.
         if (Stream.SkipBlock())
-          return Error("Invalid record");
+          return error("Invalid record");
         break;
       case bitc::BLOCKINFO_BLOCK_ID:
         if (Stream.ReadBlockInfoBlock())
-          return Error("Malformed block");
+          return error("Malformed block");
         break;
       case bitc::PARAMATTR_BLOCK_ID:
-        if (std::error_code EC = ParseAttributeBlock())
+        if (std::error_code EC = parseAttributeBlock())
           return EC;
         break;
       case bitc::PARAMATTR_GROUP_BLOCK_ID:
-        if (std::error_code EC = ParseAttributeGroupBlock())
+        if (std::error_code EC = parseAttributeGroupBlock())
           return EC;
         break;
       case bitc::TYPE_BLOCK_ID_NEW:
-        if (std::error_code EC = ParseTypeTable())
+        if (std::error_code EC = parseTypeTable())
           return EC;
         break;
       case bitc::VALUE_SYMTAB_BLOCK_ID:
-        if (std::error_code EC = ParseValueSymbolTable())
+        if (std::error_code EC = parseValueSymbolTable())
           return EC;
         SeenValueSymbolTable = true;
         break;
       case bitc::CONSTANTS_BLOCK_ID:
-        if (std::error_code EC = ParseConstants())
+        if (std::error_code EC = parseConstants())
           return EC;
-        if (std::error_code EC = ResolveGlobalAndAliasInits())
+        if (std::error_code EC = resolveGlobalAndAliasInits())
           return EC;
         break;
       case bitc::METADATA_BLOCK_ID:
@@ -2736,7 +2802,7 @@ std::error_code BitcodeReader::ParseModule(bool Resume,
           break;
         }
         assert(DeferredMetadataInfo.empty() && "Unexpected deferred metadata");
-        if (std::error_code EC = ParseMetadata())
+        if (std::error_code EC = parseMetadata())
           return EC;
         break;
       case bitc::FUNCTION_BLOCK_ID:
@@ -2744,26 +2810,24 @@ std::error_code BitcodeReader::ParseModule(bool Resume,
         // FunctionsWithBodies list.
         if (!SeenFirstFunctionBody) {
           std::reverse(FunctionsWithBodies.begin(), FunctionsWithBodies.end());
-          if (std::error_code EC = GlobalCleanup())
+          if (std::error_code EC = globalCleanup())
             return EC;
           SeenFirstFunctionBody = true;
         }
 
-        if (std::error_code EC = RememberAndSkipFunctionBody())
+        if (std::error_code EC = rememberAndSkipFunctionBody())
           return EC;
-        // For streaming bitcode, suspend parsing when we reach the function
-        // bodies. Subsequent materialization calls will resume it when
-        // necessary. For streaming, the function bodies must be at the end of
-        // the bitcode. If the bitcode file is old, the symbol table will be
-        // at the end instead and will not have been seen yet. In this case,
-        // just finish the parse now.
-        if (LazyStreamer && SeenValueSymbolTable) {
+        // Suspend parsing when we reach the function bodies. Subsequent
+        // materialization calls will resume it when necessary. If the bitcode
+        // file is old, the symbol table will be at the end instead and will not
+        // have been seen yet. In this case, just finish the parse now.
+        if (SeenValueSymbolTable) {
           NextUnreadBit = Stream.GetCurrentBitNo();
           return std::error_code();
         }
         break;
       case bitc::USELIST_BLOCK_ID:
-        if (std::error_code EC = ParseUseLists())
+        if (std::error_code EC = parseUseLists())
           return EC;
         break;
       }
@@ -2780,12 +2844,12 @@ std::error_code BitcodeReader::ParseModule(bool Resume,
     default: break;  // Default behavior, ignore unknown content.
     case bitc::MODULE_CODE_VERSION: {  // VERSION: [version#]
       if (Record.size() < 1)
-        return Error("Invalid record");
+        return error("Invalid record");
       // Only version #0 and #1 are supported so far.
       unsigned module_version = Record[0];
       switch (module_version) {
         default:
-          return Error("Invalid value");
+          return error("Invalid value");
         case 0:
           UseRelativeIDs = false;
           break;
@@ -2797,50 +2861,50 @@ std::error_code BitcodeReader::ParseModule(bool Resume,
     }
     case bitc::MODULE_CODE_TRIPLE: {  // TRIPLE: [strchr x N]
       std::string S;
-      if (ConvertToString(Record, 0, S))
-        return Error("Invalid record");
+      if (convertToString(Record, 0, S))
+        return error("Invalid record");
       TheModule->setTargetTriple(S);
       break;
     }
     case bitc::MODULE_CODE_DATALAYOUT: {  // DATALAYOUT: [strchr x N]
       std::string S;
-      if (ConvertToString(Record, 0, S))
-        return Error("Invalid record");
+      if (convertToString(Record, 0, S))
+        return error("Invalid record");
       TheModule->setDataLayout(S);
       break;
     }
     case bitc::MODULE_CODE_ASM: {  // ASM: [strchr x N]
       std::string S;
-      if (ConvertToString(Record, 0, S))
-        return Error("Invalid record");
+      if (convertToString(Record, 0, S))
+        return error("Invalid record");
       TheModule->setModuleInlineAsm(S);
       break;
     }
     case bitc::MODULE_CODE_DEPLIB: {  // DEPLIB: [strchr x N]
       // FIXME: Remove in 4.0.
       std::string S;
-      if (ConvertToString(Record, 0, S))
-        return Error("Invalid record");
+      if (convertToString(Record, 0, S))
+        return error("Invalid record");
       // Ignore value.
       break;
     }
     case bitc::MODULE_CODE_SECTIONNAME: {  // SECTIONNAME: [strchr x N]
       std::string S;
-      if (ConvertToString(Record, 0, S))
-        return Error("Invalid record");
+      if (convertToString(Record, 0, S))
+        return error("Invalid record");
       SectionTable.push_back(S);
       break;
     }
     case bitc::MODULE_CODE_GCNAME: {  // SECTIONNAME: [strchr x N]
       std::string S;
-      if (ConvertToString(Record, 0, S))
-        return Error("Invalid record");
+      if (convertToString(Record, 0, S))
+        return error("Invalid record");
       GCTable.push_back(S);
       break;
     }
     case bitc::MODULE_CODE_COMDAT: { // COMDAT: [selection_kind, name]
       if (Record.size() < 2)
-        return Error("Invalid record");
+        return error("Invalid record");
       Comdat::SelectionKind SK = getDecodedComdatSelectionKind(Record[0]);
       unsigned ComdatNameSize = Record[1];
       std::string ComdatName;
@@ -2858,16 +2922,22 @@ std::error_code BitcodeReader::ParseModule(bool Resume,
     //             comdat]
     case bitc::MODULE_CODE_GLOBALVAR: {
       if (Record.size() < 6)
-        return Error("Invalid record");
+        return error("Invalid record");
       Type *Ty = getTypeByID(Record[0]);
       if (!Ty)
-        return Error("Invalid record");
-      if (!Ty->isPointerTy())
-        return Error("Invalid type for value");
-      unsigned AddressSpace = cast<PointerType>(Ty)->getAddressSpace();
-      Ty = cast<PointerType>(Ty)->getElementType();
+        return error("Invalid record");
+      bool isConstant = Record[1] & 1;
+      bool explicitType = Record[1] & 2;
+      unsigned AddressSpace;
+      if (explicitType) {
+        AddressSpace = Record[1] >> 2;
+      } else {
+        if (!Ty->isPointerTy())
+          return error("Invalid type for value");
+        AddressSpace = cast<PointerType>(Ty)->getAddressSpace();
+        Ty = cast<PointerType>(Ty)->getElementType();
+      }
 
-      bool isConstant = Record[1];
       uint64_t RawLinkage = Record[3];
       GlobalValue::LinkageTypes Linkage = getDecodedLinkage(RawLinkage);
       unsigned Alignment;
@@ -2876,18 +2946,18 @@ std::error_code BitcodeReader::ParseModule(bool Resume,
       std::string Section;
       if (Record[5]) {
         if (Record[5]-1 >= SectionTable.size())
-          return Error("Invalid ID");
+          return error("Invalid ID");
         Section = SectionTable[Record[5]-1];
       }
       GlobalValue::VisibilityTypes Visibility = GlobalValue::DefaultVisibility;
       // Local linkage must have default visibility.
       if (Record.size() > 6 && !GlobalValue::isLocalLinkage(Linkage))
         // FIXME: Change to an error if non-default in 4.0.
-        Visibility = GetDecodedVisibility(Record[6]);
+        Visibility = getDecodedVisibility(Record[6]);
 
       GlobalVariable::ThreadLocalMode TLM = GlobalVariable::NotThreadLocal;
       if (Record.size() > 7)
-        TLM = GetDecodedThreadLocalMode(Record[7]);
+        TLM = getDecodedThreadLocalMode(Record[7]);
 
       bool UnnamedAddr = false;
       if (Record.size() > 8)
@@ -2907,9 +2977,9 @@ std::error_code BitcodeReader::ParseModule(bool Resume,
       NewGV->setUnnamedAddr(UnnamedAddr);
 
       if (Record.size() > 10)
-        NewGV->setDLLStorageClass(GetDecodedDLLStorageClass(Record[10]));
+        NewGV->setDLLStorageClass(getDecodedDLLStorageClass(Record[10]));
       else
-        UpgradeDLLImportExportLinkage(NewGV, RawLinkage);
+        upgradeDLLImportExportLinkage(NewGV, RawLinkage);
 
       ValueList.push_back(NewGV);
 
@@ -2919,7 +2989,8 @@ std::error_code BitcodeReader::ParseModule(bool Resume,
 
       if (Record.size() > 11) {
         if (unsigned ComdatID = Record[11]) {
-          assert(ComdatID <= ComdatList.size());
+          if (ComdatID > ComdatList.size())
+            return error("Invalid global variable comdat ID");
           NewGV->setComdat(ComdatList[ComdatID - 1]);
         }
       } else if (hasImplicitComdat(RawLinkage)) {
@@ -2932,16 +3003,15 @@ std::error_code BitcodeReader::ParseModule(bool Resume,
     //             prologuedata, dllstorageclass, comdat, prefixdata]
     case bitc::MODULE_CODE_FUNCTION: {
       if (Record.size() < 8)
-        return Error("Invalid record");
+        return error("Invalid record");
       Type *Ty = getTypeByID(Record[0]);
       if (!Ty)
-        return Error("Invalid record");
-      if (!Ty->isPointerTy())
-        return Error("Invalid type for value");
-      FunctionType *FTy =
-        dyn_cast<FunctionType>(cast<PointerType>(Ty)->getElementType());
+        return error("Invalid record");
+      if (auto *PTy = dyn_cast<PointerType>(Ty))
+        Ty = PTy->getElementType();
+      auto *FTy = dyn_cast<FunctionType>(Ty);
       if (!FTy)
-        return Error("Invalid type for value");
+        return error("Invalid type for value");
 
       Function *Func = Function::Create(FTy, GlobalValue::ExternalLinkage,
                                         "", TheModule);
@@ -2958,16 +3028,16 @@ std::error_code BitcodeReader::ParseModule(bool Resume,
       Func->setAlignment(Alignment);
       if (Record[6]) {
         if (Record[6]-1 >= SectionTable.size())
-          return Error("Invalid ID");
+          return error("Invalid ID");
         Func->setSection(SectionTable[Record[6]-1]);
       }
       // Local linkage must have default visibility.
       if (!Func->hasLocalLinkage())
         // FIXME: Change to an error if non-default in 4.0.
-        Func->setVisibility(GetDecodedVisibility(Record[7]));
+        Func->setVisibility(getDecodedVisibility(Record[7]));
       if (Record.size() > 8 && Record[8]) {
-        if (Record[8]-1 > GCTable.size())
-          return Error("Invalid ID");
+        if (Record[8]-1 >= GCTable.size())
+          return error("Invalid ID");
         Func->setGC(GCTable[Record[8]-1].c_str());
       }
       bool UnnamedAddr = false;
@@ -2978,13 +3048,14 @@ std::error_code BitcodeReader::ParseModule(bool Resume,
         FunctionPrologues.push_back(std::make_pair(Func, Record[10]-1));
 
       if (Record.size() > 11)
-        Func->setDLLStorageClass(GetDecodedDLLStorageClass(Record[11]));
+        Func->setDLLStorageClass(getDecodedDLLStorageClass(Record[11]));
       else
-        UpgradeDLLImportExportLinkage(Func, RawLinkage);
+        upgradeDLLImportExportLinkage(Func, RawLinkage);
 
       if (Record.size() > 12) {
         if (unsigned ComdatID = Record[12]) {
-          assert(ComdatID <= ComdatList.size());
+          if (ComdatID > ComdatList.size())
+            return error("Invalid function comdat ID");
           Func->setComdat(ComdatList[ComdatID - 1]);
         }
       } else if (hasImplicitComdat(RawLinkage)) {
@@ -2994,6 +3065,9 @@ std::error_code BitcodeReader::ParseModule(bool Resume,
       if (Record.size() > 13 && Record[13] != 0)
         FunctionPrefixes.push_back(std::make_pair(Func, Record[13]-1));
 
+      if (Record.size() > 14 && Record[14] != 0)
+        FunctionPersonalityFns.push_back(std::make_pair(Func, Record[14] - 1));
+
       ValueList.push_back(Func);
 
       // If this is a function with a body, remember the prototype we are
@@ -3001,8 +3075,7 @@ std::error_code BitcodeReader::ParseModule(bool Resume,
       if (!isProto) {
         Func->setIsMaterializable(true);
         FunctionsWithBodies.push_back(Func);
-        if (LazyStreamer)
-          DeferredFunctionInfo[Func] = 0;
+        DeferredFunctionInfo[Func] = 0;
       }
       break;
     }
@@ -3010,28 +3083,27 @@ std::error_code BitcodeReader::ParseModule(bool Resume,
     // ALIAS: [alias type, aliasee val#, linkage, visibility, dllstorageclass]
     case bitc::MODULE_CODE_ALIAS: {
       if (Record.size() < 3)
-        return Error("Invalid record");
+        return error("Invalid record");
       Type *Ty = getTypeByID(Record[0]);
       if (!Ty)
-        return Error("Invalid record");
+        return error("Invalid record");
       auto *PTy = dyn_cast<PointerType>(Ty);
       if (!PTy)
-        return Error("Invalid type for value");
+        return error("Invalid type for value");
 
       auto *NewGA =
-          GlobalAlias::create(PTy->getElementType(), PTy->getAddressSpace(),
-                              getDecodedLinkage(Record[2]), "", TheModule);
+          GlobalAlias::create(PTy, getDecodedLinkage(Record[2]), "", TheModule);
       // Old bitcode files didn't have visibility field.
       // Local linkage must have default visibility.
       if (Record.size() > 3 && !NewGA->hasLocalLinkage())
         // FIXME: Change to an error if non-default in 4.0.
-        NewGA->setVisibility(GetDecodedVisibility(Record[3]));
+        NewGA->setVisibility(getDecodedVisibility(Record[3]));
       if (Record.size() > 4)
-        NewGA->setDLLStorageClass(GetDecodedDLLStorageClass(Record[4]));
+        NewGA->setDLLStorageClass(getDecodedDLLStorageClass(Record[4]));
       else
-        UpgradeDLLImportExportLinkage(NewGA, Record[2]);
+        upgradeDLLImportExportLinkage(NewGA, Record[2]);
       if (Record.size() > 5)
-        NewGA->setThreadLocalMode(GetDecodedThreadLocalMode(Record[5]));
+        NewGA->setThreadLocalMode(getDecodedThreadLocalMode(Record[5]));
       if (Record.size() > 6)
         NewGA->setUnnamedAddr(Record[6]);
       ValueList.push_back(NewGA);
@@ -3042,7 +3114,7 @@ std::error_code BitcodeReader::ParseModule(bool Resume,
     case bitc::MODULE_CODE_PURGEVALS:
       // Trim down the value list to the specified size.
       if (Record.size() < 1 || Record[0] > ValueList.size())
-        return Error("Invalid record");
+        return error("Invalid record");
       ValueList.shrinkTo(Record[0]);
       break;
     }
@@ -3050,11 +3122,12 @@ std::error_code BitcodeReader::ParseModule(bool Resume,
   }
 }
 
-std::error_code BitcodeReader::ParseBitcodeInto(Module *M,
-                                                bool ShouldLazyLoadMetadata) {
-  TheModule = nullptr;
+std::error_code
+BitcodeReader::parseBitcodeInto(std::unique_ptr<DataStreamer> Streamer,
+                                Module *M, bool ShouldLazyLoadMetadata) {
+  TheModule = M;
 
-  if (std::error_code EC = InitStream())
+  if (std::error_code EC = initStream(std::move(Streamer)))
     return EC;
 
   // Sniff for the signature.
@@ -3064,68 +3137,33 @@ std::error_code BitcodeReader::ParseBitcodeInto(Module *M,
       Stream.Read(4) != 0xC ||
       Stream.Read(4) != 0xE ||
       Stream.Read(4) != 0xD)
-    return Error("Invalid bitcode signature");
+    return error("Invalid bitcode signature");
 
   // We expect a number of well-defined blocks, though we don't necessarily
   // need to understand them all.
   while (1) {
     if (Stream.AtEndOfStream()) {
-      if (TheModule)
-        return std::error_code();
       // We didn't really read a proper Module.
-      return Error("Malformed IR file");
+      return error("Malformed IR file");
     }
 
     BitstreamEntry Entry =
       Stream.advance(BitstreamCursor::AF_DontAutoprocessAbbrevs);
 
-    switch (Entry.Kind) {
-    case BitstreamEntry::Error:
-      return Error("Malformed block");
-    case BitstreamEntry::EndBlock:
-      return std::error_code();
-
-    case BitstreamEntry::SubBlock:
-      switch (Entry.ID) {
-      case bitc::BLOCKINFO_BLOCK_ID:
-        if (Stream.ReadBlockInfoBlock())
-          return Error("Malformed block");
-        break;
-      case bitc::MODULE_BLOCK_ID:
-        // Reject multiple MODULE_BLOCK's in a single bitstream.
-        if (TheModule)
-          return Error("Invalid multiple blocks");
-        TheModule = M;
-        if (std::error_code EC = ParseModule(false, ShouldLazyLoadMetadata))
-          return EC;
-        if (LazyStreamer)
-          return std::error_code();
-        break;
-      default:
-        if (Stream.SkipBlock())
-          return Error("Invalid record");
-        break;
-      }
-      continue;
-    case BitstreamEntry::Record:
-      // There should be no records in the top-level of blocks.
+    if (Entry.Kind != BitstreamEntry::SubBlock)
+      return error("Malformed block");
 
-      // The ranlib in Xcode 4 will align archive members by appending newlines
-      // to the end of them. If this file size is a multiple of 4 but not 8, we
-      // have to read and ignore these final 4 bytes :-(
-      if (Stream.getAbbrevIDWidth() == 2 && Entry.ID == 2 &&
-          Stream.Read(6) == 2 && Stream.Read(24) == 0xa0a0a &&
-          Stream.AtEndOfStream())
-        return std::error_code();
+    if (Entry.ID == bitc::MODULE_BLOCK_ID)
+      return parseModule(false, ShouldLazyLoadMetadata);
 
-      return Error("Invalid record");
-    }
+    if (Stream.SkipBlock())
+      return error("Invalid record");
   }
 }
 
 ErrorOr<std::string> BitcodeReader::parseModuleTriple() {
   if (Stream.EnterSubBlock(bitc::MODULE_BLOCK_ID))
-    return Error("Invalid record");
+    return error("Invalid record");
 
   SmallVector<uint64_t, 64> Record;
 
@@ -3137,7 +3175,7 @@ ErrorOr<std::string> BitcodeReader::parseModuleTriple() {
     switch (Entry.Kind) {
     case BitstreamEntry::SubBlock: // Handled for us already.
     case BitstreamEntry::Error:
-      return Error("Malformed block");
+      return error("Malformed block");
     case BitstreamEntry::EndBlock:
       return Triple;
     case BitstreamEntry::Record:
@@ -3150,8 +3188,8 @@ ErrorOr<std::string> BitcodeReader::parseModuleTriple() {
     default: break;  // Default behavior, ignore unknown content.
     case bitc::MODULE_CODE_TRIPLE: {  // TRIPLE: [strchr x N]
       std::string S;
-      if (ConvertToString(Record, 0, S))
-        return Error("Invalid record");
+      if (convertToString(Record, 0, S))
+        return error("Invalid record");
       Triple = S;
       break;
     }
@@ -3162,7 +3200,7 @@ ErrorOr<std::string> BitcodeReader::parseModuleTriple() {
 }
 
 ErrorOr<std::string> BitcodeReader::parseTriple() {
-  if (std::error_code EC = InitStream())
+  if (std::error_code EC = initStream(nullptr))
     return EC;
 
   // Sniff for the signature.
@@ -3172,7 +3210,7 @@ ErrorOr<std::string> BitcodeReader::parseTriple() {
       Stream.Read(4) != 0xC ||
       Stream.Read(4) != 0xE ||
       Stream.Read(4) != 0xD)
-    return Error("Invalid bitcode signature");
+    return error("Invalid bitcode signature");
 
   // We expect a number of well-defined blocks, though we don't necessarily
   // need to understand them all.
@@ -3181,7 +3219,7 @@ ErrorOr<std::string> BitcodeReader::parseTriple() {
 
     switch (Entry.Kind) {
     case BitstreamEntry::Error:
-      return Error("Malformed block");
+      return error("Malformed block");
     case BitstreamEntry::EndBlock:
       return std::error_code();
 
@@ -3191,7 +3229,7 @@ ErrorOr<std::string> BitcodeReader::parseTriple() {
 
       // Ignore other sub-blocks.
       if (Stream.SkipBlock())
-        return Error("Malformed block");
+        return error("Malformed block");
       continue;
 
     case BitstreamEntry::Record:
@@ -3201,10 +3239,10 @@ ErrorOr<std::string> BitcodeReader::parseTriple() {
   }
 }
 
-/// ParseMetadataAttachment - Parse metadata attachments.
-std::error_code BitcodeReader::ParseMetadataAttachment() {
+/// Parse metadata attachments.
+std::error_code BitcodeReader::parseMetadataAttachment(Function &F) {
   if (Stream.EnterSubBlock(bitc::METADATA_ATTACHMENT_ID))
-    return Error("Invalid record");
+    return error("Invalid record");
 
   SmallVector<uint64_t, 64> Record;
   while (1) {
@@ -3213,7 +3251,7 @@ std::error_code BitcodeReader::ParseMetadataAttachment() {
     switch (Entry.Kind) {
     case BitstreamEntry::SubBlock: // Handled for us already.
     case BitstreamEntry::Error:
-      return Error("Malformed block");
+      return error("Malformed block");
     case BitstreamEntry::EndBlock:
       return std::error_code();
     case BitstreamEntry::Record:
@@ -3228,15 +3266,28 @@ std::error_code BitcodeReader::ParseMetadataAttachment() {
       break;
     case bitc::METADATA_ATTACHMENT: {
       unsigned RecordLength = Record.size();
-      if (Record.empty() || (RecordLength - 1) % 2 == 1)
-        return Error("Invalid record");
+      if (Record.empty())
+        return error("Invalid record");
+      if (RecordLength % 2 == 0) {
+        // A function attachment.
+        for (unsigned I = 0; I != RecordLength; I += 2) {
+          auto K = MDKindMap.find(Record[I]);
+          if (K == MDKindMap.end())
+            return error("Invalid ID");
+          Metadata *MD = MDValueList.getValueFwdRef(Record[I + 1]);
+          F.setMetadata(K->second, cast<MDNode>(MD));
+        }
+        continue;
+      }
+
+      // An instruction attachment.
       Instruction *Inst = InstructionList[Record[0]];
       for (unsigned i = 1; i != RecordLength; i = i+2) {
         unsigned Kind = Record[i];
         DenseMap<unsigned, unsigned>::iterator I =
           MDKindMap.find(Kind);
         if (I == MDKindMap.end())
-          return Error("Invalid ID");
+          return error("Invalid ID");
         Metadata *Node = MDValueList.getValueFwdRef(Record[i + 1]);
         if (isa<LocalAsMetadata>(Node))
           // Drop the attachment.  This used to be legal, but there's no
@@ -3252,10 +3303,24 @@ std::error_code BitcodeReader::ParseMetadataAttachment() {
   }
 }
 
-/// ParseFunctionBody - Lazily parse the specified function body block.
-std::error_code BitcodeReader::ParseFunctionBody(Function *F) {
+static std::error_code typeCheckLoadStoreInst(DiagnosticHandlerFunction DH,
+                                              Type *ValType, Type *PtrType) {
+  if (!isa<PointerType>(PtrType))
+    return error(DH, "Load/Store operand is not a pointer type");
+  Type *ElemType = cast<PointerType>(PtrType)->getElementType();
+
+  if (ValType && ValType != ElemType)
+    return error(DH, "Explicit load/store type does not match pointee type of "
+                     "pointer operand");
+  if (!PointerType::isLoadableOrStorableType(ElemType))
+    return error(DH, "Cannot load/store from pointer");
+  return std::error_code();
+}
+
+/// Lazily parse the specified function body block.
+std::error_code BitcodeReader::parseFunctionBody(Function *F) {
   if (Stream.EnterSubBlock(bitc::FUNCTION_BLOCK_ID))
-    return Error("Invalid record");
+    return error("Invalid record");
 
   InstructionList.clear();
   unsigned ModuleValueListSize = ValueList.size();
@@ -3286,7 +3351,7 @@ std::error_code BitcodeReader::ParseFunctionBody(Function *F) {
 
     switch (Entry.Kind) {
     case BitstreamEntry::Error:
-      return Error("Malformed block");
+      return error("Malformed block");
     case BitstreamEntry::EndBlock:
       goto OutOfRecordLoop;
 
@@ -3294,27 +3359,27 @@ std::error_code BitcodeReader::ParseFunctionBody(Function *F) {
       switch (Entry.ID) {
       default:  // Skip unknown content.
         if (Stream.SkipBlock())
-          return Error("Invalid record");
+          return error("Invalid record");
         break;
       case bitc::CONSTANTS_BLOCK_ID:
-        if (std::error_code EC = ParseConstants())
+        if (std::error_code EC = parseConstants())
           return EC;
         NextValueNo = ValueList.size();
         break;
       case bitc::VALUE_SYMTAB_BLOCK_ID:
-        if (std::error_code EC = ParseValueSymbolTable())
+        if (std::error_code EC = parseValueSymbolTable())
           return EC;
         break;
       case bitc::METADATA_ATTACHMENT_ID:
-        if (std::error_code EC = ParseMetadataAttachment())
+        if (std::error_code EC = parseMetadataAttachment(*F))
           return EC;
         break;
       case bitc::METADATA_BLOCK_ID:
-        if (std::error_code EC = ParseMetadata())
+        if (std::error_code EC = parseMetadata())
           return EC;
         break;
       case bitc::USELIST_BLOCK_ID:
-        if (std::error_code EC = ParseUseLists())
+        if (std::error_code EC = parseUseLists())
           return EC;
         break;
       }
@@ -3331,10 +3396,10 @@ std::error_code BitcodeReader::ParseFunctionBody(Function *F) {
     unsigned BitCode = Stream.readRecord(Entry.ID, Record);
     switch (BitCode) {
     default: // Default behavior: reject
-      return Error("Invalid value");
+      return error("Invalid value");
     case bitc::FUNC_CODE_DECLAREBLOCKS: {   // DECLAREBLOCKS: [nblocks]
       if (Record.size() < 1 || Record[0] == 0)
-        return Error("Invalid record");
+        return error("Invalid record");
       // Create all the basic blocks for the function.
       FunctionBBs.resize(Record[0]);
 
@@ -3347,7 +3412,7 @@ std::error_code BitcodeReader::ParseFunctionBody(Function *F) {
         auto &BBRefs = BBFRI->second;
         // Check for invalid basic block references.
         if (BBRefs.size() > FunctionBBs.size())
-          return Error("Invalid ID");
+          return error("Invalid ID");
         assert(!BBRefs.empty() && "Unexpected empty array");
         assert(!BBRefs.front() && "Invalid reference to entry block");
         for (unsigned I = 0, E = FunctionBBs.size(), RE = BBRefs.size(); I != E;
@@ -3373,7 +3438,7 @@ std::error_code BitcodeReader::ParseFunctionBody(Function *F) {
       I = getLastInstruction();
 
       if (!I)
-        return Error("Invalid record");
+        return error("Invalid record");
       I->setDebugLoc(LastLoc);
       I = nullptr;
       continue;
@@ -3381,7 +3446,7 @@ std::error_code BitcodeReader::ParseFunctionBody(Function *F) {
     case bitc::FUNC_CODE_DEBUG_LOC: {      // DEBUG_LOC: [line, col, scope, ia]
       I = getLastInstruction();
       if (!I || Record.size() < 4)
-        return Error("Invalid record");
+        return error("Invalid record");
 
       unsigned Line = Record[0], Col = Record[1];
       unsigned ScopeID = Record[2], IAID = Record[3];
@@ -3401,11 +3466,11 @@ std::error_code BitcodeReader::ParseFunctionBody(Function *F) {
       if (getValueTypePair(Record, OpNum, NextValueNo, LHS) ||
           popValue(Record, OpNum, NextValueNo, LHS->getType(), RHS) ||
           OpNum+1 > Record.size())
-        return Error("Invalid record");
+        return error("Invalid record");
 
-      int Opc = GetDecodedBinaryOpcode(Record[OpNum++], LHS->getType());
+      int Opc = getDecodedBinaryOpcode(Record[OpNum++], LHS->getType());
       if (Opc == -1)
-        return Error("Invalid record");
+        return error("Invalid record");
       I = BinaryOperator::Create((Instruction::BinaryOps)Opc, LHS, RHS);
       InstructionList.push_back(I);
       if (OpNum < Record.size()) {
@@ -3424,17 +3489,7 @@ std::error_code BitcodeReader::ParseFunctionBody(Function *F) {
           if (Record[OpNum] & (1 << bitc::PEO_EXACT))
             cast<BinaryOperator>(I)->setIsExact(true);
         } else if (isa<FPMathOperator>(I)) {
-          FastMathFlags FMF;
-          if (0 != (Record[OpNum] & FastMathFlags::UnsafeAlgebra))
-            FMF.setUnsafeAlgebra();
-          if (0 != (Record[OpNum] & FastMathFlags::NoNaNs))
-            FMF.setNoNaNs();
-          if (0 != (Record[OpNum] & FastMathFlags::NoInfs))
-            FMF.setNoInfs();
-          if (0 != (Record[OpNum] & FastMathFlags::NoSignedZeros))
-            FMF.setNoSignedZeros();
-          if (0 != (Record[OpNum] & FastMathFlags::AllowReciprocal))
-            FMF.setAllowReciprocal();
+          FastMathFlags FMF = getDecodedFastMathFlags(Record[OpNum]);
           if (FMF.any())
             I->setFastMathFlags(FMF);
         }
@@ -3447,12 +3502,12 @@ std::error_code BitcodeReader::ParseFunctionBody(Function *F) {
       Value *Op;
       if (getValueTypePair(Record, OpNum, NextValueNo, Op) ||
           OpNum+2 != Record.size())
-        return Error("Invalid record");
+        return error("Invalid record");
 
       Type *ResTy = getTypeByID(Record[OpNum]);
-      int Opc = GetDecodedCastOpcode(Record[OpNum+1]);
+      int Opc = getDecodedCastOpcode(Record[OpNum + 1]);
       if (Opc == -1 || !ResTy)
-        return Error("Invalid record");
+        return error("Invalid record");
       Instruction *Temp = nullptr;
       if ((I = UpgradeBitCastInst(Opc, Op, ResTy, Temp))) {
         if (Temp) {
@@ -3483,20 +3538,22 @@ std::error_code BitcodeReader::ParseFunctionBody(Function *F) {
 
       Value *BasePtr;
       if (getValueTypePair(Record, OpNum, NextValueNo, BasePtr))
-        return Error("Invalid record");
+        return error("Invalid record");
 
-      if (Ty &&
-          Ty !=
-              cast<SequentialType>(BasePtr->getType()->getScalarType())
-                  ->getElementType())
-        return Error(
+      if (!Ty)
+        Ty = cast<SequentialType>(BasePtr->getType()->getScalarType())
+                 ->getElementType();
+      else if (Ty !=
+               cast<SequentialType>(BasePtr->getType()->getScalarType())
+                   ->getElementType())
+        return error(
             "Explicit gep type does not match pointee type of pointer operand");
 
       SmallVector<Value*, 16> GEPIdx;
       while (OpNum != Record.size()) {
         Value *Op;
         if (getValueTypePair(Record, OpNum, NextValueNo, Op))
-          return Error("Invalid record");
+          return error("Invalid record");
         GEPIdx.push_back(Op);
       }
 
@@ -3513,24 +3570,27 @@ std::error_code BitcodeReader::ParseFunctionBody(Function *F) {
       unsigned OpNum = 0;
       Value *Agg;
       if (getValueTypePair(Record, OpNum, NextValueNo, Agg))
-        return Error("Invalid record");
+        return error("Invalid record");
+
+      unsigned RecSize = Record.size();
+      if (OpNum == RecSize)
+        return error("EXTRACTVAL: Invalid instruction with 0 indices");
 
       SmallVector<unsigned, 4> EXTRACTVALIdx;
       Type *CurTy = Agg->getType();
-      for (unsigned RecSize = Record.size();
-           OpNum != RecSize; ++OpNum) {
+      for (; OpNum != RecSize; ++OpNum) {
         bool IsArray = CurTy->isArrayTy();
         bool IsStruct = CurTy->isStructTy();
         uint64_t Index = Record[OpNum];
 
         if (!IsStruct && !IsArray)
-          return Error("EXTRACTVAL: Invalid type");
+          return error("EXTRACTVAL: Invalid type");
         if ((unsigned)Index != Index)
-          return Error("Invalid value");
+          return error("Invalid value");
         if (IsStruct && Index >= CurTy->subtypes().size())
-          return Error("EXTRACTVAL: Invalid struct index");
+          return error("EXTRACTVAL: Invalid struct index");
         if (IsArray && Index >= CurTy->getArrayNumElements())
-          return Error("EXTRACTVAL: Invalid array index");
+          return error("EXTRACTVAL: Invalid array index");
         EXTRACTVALIdx.push_back((unsigned)Index);
 
         if (IsStruct)
@@ -3549,29 +3609,30 @@ std::error_code BitcodeReader::ParseFunctionBody(Function *F) {
       unsigned OpNum = 0;
       Value *Agg;
       if (getValueTypePair(Record, OpNum, NextValueNo, Agg))
-        return Error("Invalid record");
+        return error("Invalid record");
       Value *Val;
       if (getValueTypePair(Record, OpNum, NextValueNo, Val))
-        return Error("Invalid record");
+        return error("Invalid record");
+
+      unsigned RecSize = Record.size();
+      if (OpNum == RecSize)
+        return error("INSERTVAL: Invalid instruction with 0 indices");
 
       SmallVector<unsigned, 4> INSERTVALIdx;
       Type *CurTy = Agg->getType();
-      for (unsigned RecSize = Record.size();
-           OpNum != RecSize; ++OpNum) {
+      for (; OpNum != RecSize; ++OpNum) {
         bool IsArray = CurTy->isArrayTy();
         bool IsStruct = CurTy->isStructTy();
         uint64_t Index = Record[OpNum];
 
         if (!IsStruct && !IsArray)
-          return Error("INSERTVAL: Invalid type");
-        if (!CurTy->isStructTy() && !CurTy->isArrayTy())
-          return Error("Invalid type");
+          return error("INSERTVAL: Invalid type");
         if ((unsigned)Index != Index)
-          return Error("Invalid value");
+          return error("Invalid value");
         if (IsStruct && Index >= CurTy->subtypes().size())
-          return Error("INSERTVAL: Invalid struct index");
+          return error("INSERTVAL: Invalid struct index");
         if (IsArray && Index >= CurTy->getArrayNumElements())
-          return Error("INSERTVAL: Invalid array index");
+          return error("INSERTVAL: Invalid array index");
 
         INSERTVALIdx.push_back((unsigned)Index);
         if (IsStruct)
@@ -3580,6 +3641,9 @@ std::error_code BitcodeReader::ParseFunctionBody(Function *F) {
           CurTy = CurTy->subtypes()[0];
       }
 
+      if (CurTy != Val->getType())
+        return error("Inserted value type doesn't match aggregate type");
+
       I = InsertValueInst::Create(Agg, Val, INSERTVALIdx);
       InstructionList.push_back(I);
       break;
@@ -3593,7 +3657,7 @@ std::error_code BitcodeReader::ParseFunctionBody(Function *F) {
       if (getValueTypePair(Record, OpNum, NextValueNo, TrueVal) ||
           popValue(Record, OpNum, NextValueNo, TrueVal->getType(), FalseVal) ||
           popValue(Record, OpNum, NextValueNo, Type::getInt1Ty(Context), Cond))
-        return Error("Invalid record");
+        return error("Invalid record");
 
       I = SelectInst::Create(Cond, TrueVal, FalseVal);
       InstructionList.push_back(I);
@@ -3608,18 +3672,18 @@ std::error_code BitcodeReader::ParseFunctionBody(Function *F) {
       if (getValueTypePair(Record, OpNum, NextValueNo, TrueVal) ||
           popValue(Record, OpNum, NextValueNo, TrueVal->getType(), FalseVal) ||
           getValueTypePair(Record, OpNum, NextValueNo, Cond))
-        return Error("Invalid record");
+        return error("Invalid record");
 
       // select condition can be either i1 or [N x i1]
       if (VectorType* vector_type =
           dyn_cast<VectorType>(Cond->getType())) {
         // expect <n x i1>
         if (vector_type->getElementType() != Type::getInt1Ty(Context))
-          return Error("Invalid type for value");
+          return error("Invalid type for value");
       } else {
         // expect i1
         if (Cond->getType() != Type::getInt1Ty(Context))
-          return Error("Invalid type for value");
+          return error("Invalid type for value");
       }
 
       I = SelectInst::Create(Cond, TrueVal, FalseVal);
@@ -3632,7 +3696,9 @@ std::error_code BitcodeReader::ParseFunctionBody(Function *F) {
       Value *Vec, *Idx;
       if (getValueTypePair(Record, OpNum, NextValueNo, Vec) ||
           getValueTypePair(Record, OpNum, NextValueNo, Idx))
-        return Error("Invalid record");
+        return error("Invalid record");
+      if (!Vec->getType()->isVectorTy())
+        return error("Invalid type for value");
       I = ExtractElementInst::Create(Vec, Idx);
       InstructionList.push_back(I);
       break;
@@ -3641,11 +3707,14 @@ std::error_code BitcodeReader::ParseFunctionBody(Function *F) {
     case bitc::FUNC_CODE_INST_INSERTELT: { // INSERTELT: [ty, opval,opval,opval]
       unsigned OpNum = 0;
       Value *Vec, *Elt, *Idx;
-      if (getValueTypePair(Record, OpNum, NextValueNo, Vec) ||
-          popValue(Record, OpNum, NextValueNo,
+      if (getValueTypePair(Record, OpNum, NextValueNo, Vec))
+        return error("Invalid record");
+      if (!Vec->getType()->isVectorTy())
+        return error("Invalid type for value");
+      if (popValue(Record, OpNum, NextValueNo,
                    cast<VectorType>(Vec->getType())->getElementType(), Elt) ||
           getValueTypePair(Record, OpNum, NextValueNo, Idx))
-        return Error("Invalid record");
+        return error("Invalid record");
       I = InsertElementInst::Create(Vec, Elt, Idx);
       InstructionList.push_back(I);
       break;
@@ -3656,10 +3725,12 @@ std::error_code BitcodeReader::ParseFunctionBody(Function *F) {
       Value *Vec1, *Vec2, *Mask;
       if (getValueTypePair(Record, OpNum, NextValueNo, Vec1) ||
           popValue(Record, OpNum, NextValueNo, Vec1->getType(), Vec2))
-        return Error("Invalid record");
+        return error("Invalid record");
 
       if (getValueTypePair(Record, OpNum, NextValueNo, Mask))
-        return Error("Invalid record");
+        return error("Invalid record");
+      if (!Vec1->getType()->isVectorTy() || !Vec2->getType()->isVectorTy())
+        return error("Invalid type for value");
       I = new ShuffleVectorInst(Vec1, Vec2, Mask);
       InstructionList.push_back(I);
       break;
@@ -3675,14 +3746,25 @@ std::error_code BitcodeReader::ParseFunctionBody(Function *F) {
       unsigned OpNum = 0;
       Value *LHS, *RHS;
       if (getValueTypePair(Record, OpNum, NextValueNo, LHS) ||
-          popValue(Record, OpNum, NextValueNo, LHS->getType(), RHS) ||
-          OpNum+1 != Record.size())
-        return Error("Invalid record");
+          popValue(Record, OpNum, NextValueNo, LHS->getType(), RHS))
+        return error("Invalid record");
+
+      unsigned PredVal = Record[OpNum];
+      bool IsFP = LHS->getType()->isFPOrFPVectorTy();
+      FastMathFlags FMF;
+      if (IsFP && Record.size() > OpNum+1)
+        FMF = getDecodedFastMathFlags(Record[++OpNum]);
+
+      if (OpNum+1 != Record.size())
+        return error("Invalid record");
 
       if (LHS->getType()->isFPOrFPVectorTy())
-        I = new FCmpInst((FCmpInst::Predicate)Record[OpNum], LHS, RHS);
+        I = new FCmpInst((FCmpInst::Predicate)PredVal, LHS, RHS);
       else
-        I = new ICmpInst((ICmpInst::Predicate)Record[OpNum], LHS, RHS);
+        I = new ICmpInst((ICmpInst::Predicate)PredVal, LHS, RHS);
+
+      if (FMF.any())
+        I->setFastMathFlags(FMF);
       InstructionList.push_back(I);
       break;
     }
@@ -3699,9 +3781,9 @@ std::error_code BitcodeReader::ParseFunctionBody(Function *F) {
         unsigned OpNum = 0;
         Value *Op = nullptr;
         if (getValueTypePair(Record, OpNum, NextValueNo, Op))
-          return Error("Invalid record");
+          return error("Invalid record");
         if (OpNum != Record.size())
-          return Error("Invalid record");
+          return error("Invalid record");
 
         I = ReturnInst::Create(Context, Op);
         InstructionList.push_back(I);
@@ -3709,10 +3791,10 @@ std::error_code BitcodeReader::ParseFunctionBody(Function *F) {
       }
     case bitc::FUNC_CODE_INST_BR: { // BR: [bb#, bb#, opval] or [bb#]
       if (Record.size() != 1 && Record.size() != 3)
-        return Error("Invalid record");
+        return error("Invalid record");
       BasicBlock *TrueDest = getBasicBlock(Record[0]);
       if (!TrueDest)
-        return Error("Invalid record");
+        return error("Invalid record");
 
       if (Record.size() == 1) {
         I = BranchInst::Create(TrueDest);
@@ -3723,12 +3805,140 @@ std::error_code BitcodeReader::ParseFunctionBody(Function *F) {
         Value *Cond = getValue(Record, 2, NextValueNo,
                                Type::getInt1Ty(Context));
         if (!FalseDest || !Cond)
-          return Error("Invalid record");
+          return error("Invalid record");
         I = BranchInst::Create(TrueDest, FalseDest, Cond);
         InstructionList.push_back(I);
       }
       break;
     }
+    // CLEANUPRET: [] or [ty,val] or [bb#] or [ty,val,bb#]
+    case bitc::FUNC_CODE_INST_CLEANUPRET: {
+      if (Record.size() < 2)
+        return error("Invalid record");
+      unsigned Idx = 0;
+      bool HasReturnValue = !!Record[Idx++];
+      bool HasUnwindDest = !!Record[Idx++];
+      Value *RetVal = nullptr;
+      BasicBlock *UnwindDest = nullptr;
+
+      if (HasReturnValue && getValueTypePair(Record, Idx, NextValueNo, RetVal))
+        return error("Invalid record");
+      if (HasUnwindDest) {
+        if (Idx == Record.size())
+          return error("Invalid record");
+        UnwindDest = getBasicBlock(Record[Idx++]);
+        if (!UnwindDest)
+          return error("Invalid record");
+      }
+
+      if (Record.size() != Idx)
+        return error("Invalid record");
+
+      I = CleanupReturnInst::Create(Context, RetVal, UnwindDest);
+      InstructionList.push_back(I);
+      break;
+    }
+    case bitc::FUNC_CODE_INST_CATCHRET: { // CATCHRET: [bb#]
+      if (Record.size() != 1)
+        return error("Invalid record");
+      BasicBlock *BB = getBasicBlock(Record[0]);
+      if (!BB)
+        return error("Invalid record");
+      I = CatchReturnInst::Create(BB);
+      InstructionList.push_back(I);
+      break;
+    }
+    case bitc::FUNC_CODE_INST_CATCHPAD: { // CATCHPAD: [ty,bb#,bb#,num,(ty,val)*]
+      if (Record.size() < 4)
+        return error("Invalid record");
+      unsigned Idx = 0;
+      Type *Ty = getTypeByID(Record[Idx++]);
+      if (!Ty)
+        return error("Invalid record");
+      BasicBlock *NormalBB = getBasicBlock(Record[Idx++]);
+      if (!NormalBB)
+        return error("Invalid record");
+      BasicBlock *UnwindBB = getBasicBlock(Record[Idx++]);
+      if (!UnwindBB)
+        return error("Invalid record");
+      unsigned NumArgOperands = Record[Idx++];
+      SmallVector<Value *, 2> Args;
+      for (unsigned Op = 0; Op != NumArgOperands; ++Op) {
+        Value *Val;
+        if (getValueTypePair(Record, Idx, NextValueNo, Val))
+          return error("Invalid record");
+        Args.push_back(Val);
+      }
+      if (Record.size() != Idx)
+        return error("Invalid record");
+
+      I = CatchPadInst::Create(Ty, NormalBB, UnwindBB, Args);
+      InstructionList.push_back(I);
+      break;
+    }
+    case bitc::FUNC_CODE_INST_TERMINATEPAD: { // TERMINATEPAD: [bb#,num,(ty,val)*]
+      if (Record.size() < 1)
+        return error("Invalid record");
+      unsigned Idx = 0;
+      bool HasUnwindDest = !!Record[Idx++];
+      BasicBlock *UnwindDest = nullptr;
+      if (HasUnwindDest) {
+        if (Idx == Record.size())
+          return error("Invalid record");
+        UnwindDest = getBasicBlock(Record[Idx++]);
+        if (!UnwindDest)
+          return error("Invalid record");
+      }
+      unsigned NumArgOperands = Record[Idx++];
+      SmallVector<Value *, 2> Args;
+      for (unsigned Op = 0; Op != NumArgOperands; ++Op) {
+        Value *Val;
+        if (getValueTypePair(Record, Idx, NextValueNo, Val))
+          return error("Invalid record");
+        Args.push_back(Val);
+      }
+      if (Record.size() != Idx)
+        return error("Invalid record");
+
+      I = TerminatePadInst::Create(Context, UnwindDest, Args);
+      InstructionList.push_back(I);
+      break;
+    }
+    case bitc::FUNC_CODE_INST_CLEANUPPAD: { // CLEANUPPAD: [ty, num,(ty,val)*]
+      if (Record.size() < 2)
+        return error("Invalid record");
+      unsigned Idx = 0;
+      Type *Ty = getTypeByID(Record[Idx++]);
+      if (!Ty)
+        return error("Invalid record");
+      unsigned NumArgOperands = Record[Idx++];
+      SmallVector<Value *, 2> Args;
+      for (unsigned Op = 0; Op != NumArgOperands; ++Op) {
+        Value *Val;
+        if (getValueTypePair(Record, Idx, NextValueNo, Val))
+          return error("Invalid record");
+        Args.push_back(Val);
+      }
+      if (Record.size() != Idx)
+        return error("Invalid record");
+
+      I = CleanupPadInst::Create(Ty, Args);
+      InstructionList.push_back(I);
+      break;
+    }
+    case bitc::FUNC_CODE_INST_CATCHENDPAD: { // CATCHENDPADINST: [bb#] or []
+      if (Record.size() > 1)
+        return error("Invalid record");
+      BasicBlock *BB = nullptr;
+      if (Record.size() == 1) {
+        BB = getBasicBlock(Record[0]);
+        if (!BB)
+          return error("Invalid record");
+      }
+      I = CatchEndPadInst::Create(Context, BB);
+      InstructionList.push_back(I);
+      break;
+    }
     case bitc::FUNC_CODE_INST_SWITCH: { // SWITCH: [opty, op0, op1, ...]
       // Check magic
       if ((Record[0] >> 16) == SWITCH_INST_MAGIC) {
@@ -3743,7 +3953,7 @@ std::error_code BitcodeReader::ParseFunctionBody(Function *F) {
         Value *Cond = getValue(Record, 2, NextValueNo, OpTy);
         BasicBlock *Default = getBasicBlock(Record[3]);
         if (!OpTy || !Cond || !Default)
-          return Error("Invalid record");
+          return error("Invalid record");
 
         unsigned NumCases = Record[4];
 
@@ -3761,7 +3971,7 @@ std::error_code BitcodeReader::ParseFunctionBody(Function *F) {
             unsigned ActiveWords = 1;
             if (ValueBitWidth > 64)
               ActiveWords = Record[CurIdx++];
-            Low = ReadWideAPInt(makeArrayRef(&Record[CurIdx], ActiveWords),
+            Low = readWideAPInt(makeArrayRef(&Record[CurIdx], ActiveWords),
                                 ValueBitWidth);
             CurIdx += ActiveWords;
 
@@ -3769,9 +3979,8 @@ std::error_code BitcodeReader::ParseFunctionBody(Function *F) {
               ActiveWords = 1;
               if (ValueBitWidth > 64)
                 ActiveWords = Record[CurIdx++];
-              APInt High =
-                  ReadWideAPInt(makeArrayRef(&Record[CurIdx], ActiveWords),
-                                ValueBitWidth);
+              APInt High = readWideAPInt(
+                  makeArrayRef(&Record[CurIdx], ActiveWords), ValueBitWidth);
               CurIdx += ActiveWords;
 
               // FIXME: It is not clear whether values in the range should be
@@ -3795,12 +4004,12 @@ std::error_code BitcodeReader::ParseFunctionBody(Function *F) {
       // Old SwitchInst format without case ranges.
 
       if (Record.size() < 3 || (Record.size() & 1) == 0)
-        return Error("Invalid record");
+        return error("Invalid record");
       Type *OpTy = getTypeByID(Record[0]);
       Value *Cond = getValue(Record, 1, NextValueNo, OpTy);
       BasicBlock *Default = getBasicBlock(Record[2]);
       if (!OpTy || !Cond || !Default)
-        return Error("Invalid record");
+        return error("Invalid record");
       unsigned NumCases = (Record.size()-3)/2;
       SwitchInst *SI = SwitchInst::Create(Cond, Default, NumCases);
       InstructionList.push_back(SI);
@@ -3810,7 +4019,7 @@ std::error_code BitcodeReader::ParseFunctionBody(Function *F) {
         BasicBlock *DestBB = getBasicBlock(Record[1+3+i*2]);
         if (!CaseVal || !DestBB) {
           delete SI;
-          return Error("Invalid record");
+          return error("Invalid record");
         }
         SI->addCase(CaseVal, DestBB);
       }
@@ -3819,11 +4028,11 @@ std::error_code BitcodeReader::ParseFunctionBody(Function *F) {
     }
     case bitc::FUNC_CODE_INST_INDIRECTBR: { // INDIRECTBR: [opty, op0, op1, ...]
       if (Record.size() < 2)
-        return Error("Invalid record");
+        return error("Invalid record");
       Type *OpTy = getTypeByID(Record[0]);
       Value *Address = getValue(Record, 1, NextValueNo, OpTy);
       if (!OpTy || !Address)
-        return Error("Invalid record");
+        return error("Invalid record");
       unsigned NumDests = Record.size()-2;
       IndirectBrInst *IBI = IndirectBrInst::Create(Address, NumDests);
       InstructionList.push_back(IBI);
@@ -3832,7 +4041,7 @@ std::error_code BitcodeReader::ParseFunctionBody(Function *F) {
           IBI->addDestination(DestBB);
         } else {
           delete IBI;
-          return Error("Invalid record");
+          return error("Invalid record");
         }
       }
       I = IBI;
@@ -3842,51 +4051,60 @@ std::error_code BitcodeReader::ParseFunctionBody(Function *F) {
     case bitc::FUNC_CODE_INST_INVOKE: {
       // INVOKE: [attrs, cc, normBB, unwindBB, fnty, op0,op1,op2, ...]
       if (Record.size() < 4)
-        return Error("Invalid record");
-      AttributeSet PAL = getAttributes(Record[0]);
-      unsigned CCInfo = Record[1];
-      BasicBlock *NormalBB = getBasicBlock(Record[2]);
-      BasicBlock *UnwindBB = getBasicBlock(Record[3]);
+        return error("Invalid record");
+      unsigned OpNum = 0;
+      AttributeSet PAL = getAttributes(Record[OpNum++]);
+      unsigned CCInfo = Record[OpNum++];
+      BasicBlock *NormalBB = getBasicBlock(Record[OpNum++]);
+      BasicBlock *UnwindBB = getBasicBlock(Record[OpNum++]);
+
+      FunctionType *FTy = nullptr;
+      if (CCInfo >> 13 & 1 &&
+          !(FTy = dyn_cast<FunctionType>(getTypeByID(Record[OpNum++]))))
+        return error("Explicit invoke type is not a function type");
 
-      unsigned OpNum = 4;
       Value *Callee;
       if (getValueTypePair(Record, OpNum, NextValueNo, Callee))
-        return Error("Invalid record");
+        return error("Invalid record");
 
       PointerType *CalleeTy = dyn_cast<PointerType>(Callee->getType());
-      FunctionType *FTy = !CalleeTy ? nullptr :
-        dyn_cast<FunctionType>(CalleeTy->getElementType());
-
-      // Check that the right number of fixed parameters are here.
-      if (!FTy || !NormalBB || !UnwindBB ||
-          Record.size() < OpNum+FTy->getNumParams())
-        return Error("Invalid record");
+      if (!CalleeTy)
+        return error("Callee is not a pointer");
+      if (!FTy) {
+        FTy = dyn_cast<FunctionType>(CalleeTy->getElementType());
+        if (!FTy)
+          return error("Callee is not of pointer to function type");
+      } else if (CalleeTy->getElementType() != FTy)
+        return error("Explicit invoke type does not match pointee type of "
+                     "callee operand");
+      if (Record.size() < FTy->getNumParams() + OpNum)
+        return error("Insufficient operands to call");
 
       SmallVector<Value*, 16> Ops;
       for (unsigned i = 0, e = FTy->getNumParams(); i != e; ++i, ++OpNum) {
         Ops.push_back(getValue(Record, OpNum, NextValueNo,
                                FTy->getParamType(i)));
         if (!Ops.back())
-          return Error("Invalid record");
+          return error("Invalid record");
       }
 
       if (!FTy->isVarArg()) {
         if (Record.size() != OpNum)
-          return Error("Invalid record");
+          return error("Invalid record");
       } else {
         // Read type/value pairs for varargs params.
         while (OpNum != Record.size()) {
           Value *Op;
           if (getValueTypePair(Record, OpNum, NextValueNo, Op))
-            return Error("Invalid record");
+            return error("Invalid record");
           Ops.push_back(Op);
         }
       }
 
       I = InvokeInst::Create(Callee, NormalBB, UnwindBB, Ops);
       InstructionList.push_back(I);
-      cast<InvokeInst>(I)->setCallingConv(
-        static_cast<CallingConv::ID>(CCInfo));
+      cast<InvokeInst>(I)
+          ->setCallingConv(static_cast<CallingConv::ID>(~(1U << 13) & CCInfo));
       cast<InvokeInst>(I)->setAttributes(PAL);
       break;
     }
@@ -3894,7 +4112,7 @@ std::error_code BitcodeReader::ParseFunctionBody(Function *F) {
       unsigned Idx = 0;
       Value *Val = nullptr;
       if (getValueTypePair(Record, Idx, NextValueNo, Val))
-        return Error("Invalid record");
+        return error("Invalid record");
       I = ResumeInst::Create(Val);
       InstructionList.push_back(I);
       break;
@@ -3905,10 +4123,10 @@ std::error_code BitcodeReader::ParseFunctionBody(Function *F) {
       break;
     case bitc::FUNC_CODE_INST_PHI: { // PHI: [ty, val0,bb0, ...]
       if (Record.size() < 1 || ((Record.size()-1)&1))
-        return Error("Invalid record");
+        return error("Invalid record");
       Type *Ty = getTypeByID(Record[0]);
       if (!Ty)
-        return Error("Invalid record");
+        return error("Invalid record");
 
       PHINode *PN = PHINode::Create(Ty, (Record.size()-1)/2);
       InstructionList.push_back(PN);
@@ -3924,28 +4142,42 @@ std::error_code BitcodeReader::ParseFunctionBody(Function *F) {
           V = getValue(Record, 1+i, NextValueNo, Ty);
         BasicBlock *BB = getBasicBlock(Record[2+i]);
         if (!V || !BB)
-          return Error("Invalid record");
+          return error("Invalid record");
         PN->addIncoming(V, BB);
       }
       I = PN;
       break;
     }
 
-    case bitc::FUNC_CODE_INST_LANDINGPAD: {
+    case bitc::FUNC_CODE_INST_LANDINGPAD:
+    case bitc::FUNC_CODE_INST_LANDINGPAD_OLD: {
       // LANDINGPAD: [ty, val, val, num, (id0,val0 ...)?]
       unsigned Idx = 0;
-      if (Record.size() < 4)
-        return Error("Invalid record");
+      if (BitCode == bitc::FUNC_CODE_INST_LANDINGPAD) {
+        if (Record.size() < 3)
+          return error("Invalid record");
+      } else {
+        assert(BitCode == bitc::FUNC_CODE_INST_LANDINGPAD_OLD);
+        if (Record.size() < 4)
+          return error("Invalid record");
+      }
       Type *Ty = getTypeByID(Record[Idx++]);
       if (!Ty)
-        return Error("Invalid record");
-      Value *PersFn = nullptr;
-      if (getValueTypePair(Record, Idx, NextValueNo, PersFn))
-        return Error("Invalid record");
+        return error("Invalid record");
+      if (BitCode == bitc::FUNC_CODE_INST_LANDINGPAD_OLD) {
+        Value *PersFn = nullptr;
+        if (getValueTypePair(Record, Idx, NextValueNo, PersFn))
+          return error("Invalid record");
+
+        if (!F->hasPersonalityFn())
+          F->setPersonalityFn(cast<Constant>(PersFn));
+        else if (F->getPersonalityFn() != cast<Constant>(PersFn))
+          return error("Personality function mismatch");
+      }
 
       bool IsCleanup = !!Record[Idx++];
       unsigned NumClauses = Record[Idx++];
-      LandingPadInst *LP = LandingPadInst::Create(Ty, PersFn, NumClauses);
+      LandingPadInst *LP = LandingPadInst::Create(Ty, NumClauses);
       LP->setCleanup(IsCleanup);
       for (unsigned J = 0; J != NumClauses; ++J) {
         LandingPadInst::ClauseType CT =
@@ -3954,7 +4186,7 @@ std::error_code BitcodeReader::ParseFunctionBody(Function *F) {
 
         if (getValueTypePair(Record, Idx, NextValueNo, Val)) {
           delete LP;
-          return Error("Invalid record");
+          return error("Invalid record");
         }
 
         assert((CT != LandingPadInst::Catch ||
@@ -3973,22 +4205,31 @@ std::error_code BitcodeReader::ParseFunctionBody(Function *F) {
 
     case bitc::FUNC_CODE_INST_ALLOCA: { // ALLOCA: [instty, opty, op, align]
       if (Record.size() != 4)
-        return Error("Invalid record");
-      PointerType *Ty =
-        dyn_cast_or_null<PointerType>(getTypeByID(Record[0]));
-      Type *OpTy = getTypeByID(Record[1]);
-      Value *Size = getFnValueByID(Record[2], OpTy);
+        return error("Invalid record");
       uint64_t AlignRecord = Record[3];
       const uint64_t InAllocaMask = uint64_t(1) << 5;
+      const uint64_t ExplicitTypeMask = uint64_t(1) << 6;
+      // Reserve bit 7 for SwiftError flag.
+      // const uint64_t SwiftErrorMask = uint64_t(1) << 7;
+      const uint64_t FlagMask = InAllocaMask | ExplicitTypeMask;
       bool InAlloca = AlignRecord & InAllocaMask;
+      Type *Ty = getTypeByID(Record[0]);
+      if ((AlignRecord & ExplicitTypeMask) == 0) {
+        auto *PTy = dyn_cast_or_null<PointerType>(Ty);
+        if (!PTy)
+          return error("Old-style alloca with a non-pointer type");
+        Ty = PTy->getElementType();
+      }
+      Type *OpTy = getTypeByID(Record[1]);
+      Value *Size = getFnValueByID(Record[2], OpTy);
       unsigned Align;
       if (std::error_code EC =
-          parseAlignmentValue(AlignRecord & ~InAllocaMask, Align)) {
+              parseAlignmentValue(AlignRecord & ~FlagMask, Align)) {
         return EC;
       }
       if (!Ty || !Size)
-        return Error("Invalid record");
-      AllocaInst *AI = new AllocaInst(Ty->getElementType(), Size, Align);
+        return error("Invalid record");
+      AllocaInst *AI = new AllocaInst(Ty, Size, Align);
       AI->setUsedWithInAlloca(InAlloca);
       I = AI;
       InstructionList.push_back(I);
@@ -3999,20 +4240,21 @@ std::error_code BitcodeReader::ParseFunctionBody(Function *F) {
       Value *Op;
       if (getValueTypePair(Record, OpNum, NextValueNo, Op) ||
           (OpNum + 2 != Record.size() && OpNum + 3 != Record.size()))
-        return Error("Invalid record");
+        return error("Invalid record");
 
       Type *Ty = nullptr;
       if (OpNum + 3 == Record.size())
         Ty = getTypeByID(Record[OpNum++]);
+      if (std::error_code EC =
+              typeCheckLoadStoreInst(DiagnosticHandler, Ty, Op->getType()))
+        return EC;
+      if (!Ty)
+        Ty = cast<PointerType>(Op->getType())->getElementType();
 
       unsigned Align;
       if (std::error_code EC = parseAlignmentValue(Record[OpNum], Align))
         return EC;
-      I = new LoadInst(Op, "", Record[OpNum+1], Align);
-
-      if (Ty && Ty != I->getType())
-        return Error("Explicit load type does not match pointee type of "
-                     "pointer operand");
+      I = new LoadInst(Ty, Op, "", Record[OpNum + 1], Align);
 
       InstructionList.push_back(I);
       break;
@@ -4023,40 +4265,49 @@ std::error_code BitcodeReader::ParseFunctionBody(Function *F) {
       Value *Op;
       if (getValueTypePair(Record, OpNum, NextValueNo, Op) ||
           (OpNum + 4 != Record.size() && OpNum + 5 != Record.size()))
-        return Error("Invalid record");
+        return error("Invalid record");
 
       Type *Ty = nullptr;
       if (OpNum + 5 == Record.size())
         Ty = getTypeByID(Record[OpNum++]);
+      if (std::error_code EC =
+              typeCheckLoadStoreInst(DiagnosticHandler, Ty, Op->getType()))
+        return EC;
+      if (!Ty)
+        Ty = cast<PointerType>(Op->getType())->getElementType();
 
-      AtomicOrdering Ordering = GetDecodedOrdering(Record[OpNum+2]);
+      AtomicOrdering Ordering = getDecodedOrdering(Record[OpNum + 2]);
       if (Ordering == NotAtomic || Ordering == Release ||
           Ordering == AcquireRelease)
-        return Error("Invalid record");
+        return error("Invalid record");
       if (Ordering != NotAtomic && Record[OpNum] == 0)
-        return Error("Invalid record");
-      SynchronizationScope SynchScope = GetDecodedSynchScope(Record[OpNum+3]);
+        return error("Invalid record");
+      SynchronizationScope SynchScope = getDecodedSynchScope(Record[OpNum + 3]);
 
       unsigned Align;
       if (std::error_code EC = parseAlignmentValue(Record[OpNum], Align))
         return EC;
       I = new LoadInst(Op, "", Record[OpNum+1], Align, Ordering, SynchScope);
 
-      (void)Ty;
-      assert((!Ty || Ty == I->getType()) &&
-             "Explicit type doesn't match pointee type of the first operand");
-
       InstructionList.push_back(I);
       break;
     }
-    case bitc::FUNC_CODE_INST_STORE: { // STORE2:[ptrty, ptr, val, align, vol]
+    case bitc::FUNC_CODE_INST_STORE:
+    case bitc::FUNC_CODE_INST_STORE_OLD: { // STORE2:[ptrty, ptr, val, align, vol]
       unsigned OpNum = 0;
       Value *Val, *Ptr;
       if (getValueTypePair(Record, OpNum, NextValueNo, Ptr) ||
-          popValue(Record, OpNum, NextValueNo,
-                    cast<PointerType>(Ptr->getType())->getElementType(), Val) ||
-          OpNum+2 != Record.size())
-        return Error("Invalid record");
+          (BitCode == bitc::FUNC_CODE_INST_STORE
+               ? getValueTypePair(Record, OpNum, NextValueNo, Val)
+               : popValue(Record, OpNum, NextValueNo,
+                          cast<PointerType>(Ptr->getType())->getElementType(),
+                          Val)) ||
+          OpNum + 2 != Record.size())
+        return error("Invalid record");
+
+      if (std::error_code EC = typeCheckLoadStoreInst(
+              DiagnosticHandler, Val->getType(), Ptr->getType()))
+        return EC;
       unsigned Align;
       if (std::error_code EC = parseAlignmentValue(Record[OpNum], Align))
         return EC;
@@ -4064,23 +4315,30 @@ std::error_code BitcodeReader::ParseFunctionBody(Function *F) {
       InstructionList.push_back(I);
       break;
     }
-    case bitc::FUNC_CODE_INST_STOREATOMIC: {
+    case bitc::FUNC_CODE_INST_STOREATOMIC:
+    case bitc::FUNC_CODE_INST_STOREATOMIC_OLD: {
       // STOREATOMIC: [ptrty, ptr, val, align, vol, ordering, synchscope]
       unsigned OpNum = 0;
       Value *Val, *Ptr;
       if (getValueTypePair(Record, OpNum, NextValueNo, Ptr) ||
-          popValue(Record, OpNum, NextValueNo,
-                    cast<PointerType>(Ptr->getType())->getElementType(), Val) ||
-          OpNum+4 != Record.size())
-        return Error("Invalid record");
-
-      AtomicOrdering Ordering = GetDecodedOrdering(Record[OpNum+2]);
+          (BitCode == bitc::FUNC_CODE_INST_STOREATOMIC
+               ? getValueTypePair(Record, OpNum, NextValueNo, Val)
+               : popValue(Record, OpNum, NextValueNo,
+                          cast<PointerType>(Ptr->getType())->getElementType(),
+                          Val)) ||
+          OpNum + 4 != Record.size())
+        return error("Invalid record");
+
+      if (std::error_code EC = typeCheckLoadStoreInst(
+              DiagnosticHandler, Val->getType(), Ptr->getType()))
+        return EC;
+      AtomicOrdering Ordering = getDecodedOrdering(Record[OpNum + 2]);
       if (Ordering == NotAtomic || Ordering == Acquire ||
           Ordering == AcquireRelease)
-        return Error("Invalid record");
-      SynchronizationScope SynchScope = GetDecodedSynchScope(Record[OpNum+3]);
+        return error("Invalid record");
+      SynchronizationScope SynchScope = getDecodedSynchScope(Record[OpNum + 3]);
       if (Ordering != NotAtomic && Record[OpNum] == 0)
-        return Error("Invalid record");
+        return error("Invalid record");
 
       unsigned Align;
       if (std::error_code EC = parseAlignmentValue(Record[OpNum], Align))
@@ -4089,29 +4347,35 @@ std::error_code BitcodeReader::ParseFunctionBody(Function *F) {
       InstructionList.push_back(I);
       break;
     }
+    case bitc::FUNC_CODE_INST_CMPXCHG_OLD:
     case bitc::FUNC_CODE_INST_CMPXCHG: {
       // CMPXCHG:[ptrty, ptr, cmp, new, vol, successordering, synchscope,
       //          failureordering?, isweak?]
       unsigned OpNum = 0;
       Value *Ptr, *Cmp, *New;
       if (getValueTypePair(Record, OpNum, NextValueNo, Ptr) ||
-          popValue(Record, OpNum, NextValueNo,
-                    cast<PointerType>(Ptr->getType())->getElementType(), Cmp) ||
-          popValue(Record, OpNum, NextValueNo,
-                    cast<PointerType>(Ptr->getType())->getElementType(), New) ||
-          (Record.size() < OpNum + 3 || Record.size() > OpNum + 5))
-        return Error("Invalid record");
-      AtomicOrdering SuccessOrdering = GetDecodedOrdering(Record[OpNum+1]);
+          (BitCode == bitc::FUNC_CODE_INST_CMPXCHG
+               ? getValueTypePair(Record, OpNum, NextValueNo, Cmp)
+               : popValue(Record, OpNum, NextValueNo,
+                          cast<PointerType>(Ptr->getType())->getElementType(),
+                          Cmp)) ||
+          popValue(Record, OpNum, NextValueNo, Cmp->getType(), New) ||
+          Record.size() < OpNum + 3 || Record.size() > OpNum + 5)
+        return error("Invalid record");
+      AtomicOrdering SuccessOrdering = getDecodedOrdering(Record[OpNum + 1]);
       if (SuccessOrdering == NotAtomic || SuccessOrdering == Unordered)
-        return Error("Invalid record");
-      SynchronizationScope SynchScope = GetDecodedSynchScope(Record[OpNum+2]);
+        return error("Invalid record");
+      SynchronizationScope SynchScope = getDecodedSynchScope(Record[OpNum + 2]);
 
+      if (std::error_code EC = typeCheckLoadStoreInst(
+              DiagnosticHandler, Cmp->getType(), Ptr->getType()))
+        return EC;
       AtomicOrdering FailureOrdering;
       if (Record.size() < 7)
         FailureOrdering =
             AtomicCmpXchgInst::getStrongestFailureOrdering(SuccessOrdering);
       else
-        FailureOrdering = GetDecodedOrdering(Record[OpNum+3]);
+        FailureOrdering = getDecodedOrdering(Record[OpNum + 3]);
 
       I = new AtomicCmpXchgInst(Ptr, Cmp, New, SuccessOrdering, FailureOrdering,
                                 SynchScope);
@@ -4138,15 +4402,15 @@ std::error_code BitcodeReader::ParseFunctionBody(Function *F) {
           popValue(Record, OpNum, NextValueNo,
                     cast<PointerType>(Ptr->getType())->getElementType(), Val) ||
           OpNum+4 != Record.size())
-        return Error("Invalid record");
-      AtomicRMWInst::BinOp Operation = GetDecodedRMWOperation(Record[OpNum]);
+        return error("Invalid record");
+      AtomicRMWInst::BinOp Operation = getDecodedRMWOperation(Record[OpNum]);
       if (Operation < AtomicRMWInst::FIRST_BINOP ||
           Operation > AtomicRMWInst::LAST_BINOP)
-        return Error("Invalid record");
-      AtomicOrdering Ordering = GetDecodedOrdering(Record[OpNum+2]);
+        return error("Invalid record");
+      AtomicOrdering Ordering = getDecodedOrdering(Record[OpNum + 2]);
       if (Ordering == NotAtomic || Ordering == Unordered)
-        return Error("Invalid record");
-      SynchronizationScope SynchScope = GetDecodedSynchScope(Record[OpNum+3]);
+        return error("Invalid record");
+      SynchronizationScope SynchScope = getDecodedSynchScope(Record[OpNum + 3]);
       I = new AtomicRMWInst(Operation, Ptr, Val, Ordering, SynchScope);
       cast<AtomicRMWInst>(I)->setVolatile(Record[OpNum+1]);
       InstructionList.push_back(I);
@@ -4154,12 +4418,12 @@ std::error_code BitcodeReader::ParseFunctionBody(Function *F) {
     }
     case bitc::FUNC_CODE_INST_FENCE: { // FENCE:[ordering, synchscope]
       if (2 != Record.size())
-        return Error("Invalid record");
-      AtomicOrdering Ordering = GetDecodedOrdering(Record[0]);
+        return error("Invalid record");
+      AtomicOrdering Ordering = getDecodedOrdering(Record[0]);
       if (Ordering == NotAtomic || Ordering == Unordered ||
           Ordering == Monotonic)
-        return Error("Invalid record");
-      SynchronizationScope SynchScope = GetDecodedSynchScope(Record[1]);
+        return error("Invalid record");
+      SynchronizationScope SynchScope = getDecodedSynchScope(Record[1]);
       I = new FenceInst(Context, Ordering, SynchScope);
       InstructionList.push_back(I);
       break;
@@ -4167,21 +4431,33 @@ std::error_code BitcodeReader::ParseFunctionBody(Function *F) {
     case bitc::FUNC_CODE_INST_CALL: {
       // CALL: [paramattrs, cc, fnty, fnid, arg0, arg1...]
       if (Record.size() < 3)
-        return Error("Invalid record");
+        return error("Invalid record");
+
+      unsigned OpNum = 0;
+      AttributeSet PAL = getAttributes(Record[OpNum++]);
+      unsigned CCInfo = Record[OpNum++];
 
-      AttributeSet PAL = getAttributes(Record[0]);
-      unsigned CCInfo = Record[1];
+      FunctionType *FTy = nullptr;
+      if (CCInfo >> 15 & 1 &&
+          !(FTy = dyn_cast<FunctionType>(getTypeByID(Record[OpNum++]))))
+        return error("Explicit call type is not a function type");
 
-      unsigned OpNum = 2;
       Value *Callee;
       if (getValueTypePair(Record, OpNum, NextValueNo, Callee))
-        return Error("Invalid record");
+        return error("Invalid record");
 
       PointerType *OpTy = dyn_cast<PointerType>(Callee->getType());
-      FunctionType *FTy = nullptr;
-      if (OpTy) FTy = dyn_cast<FunctionType>(OpTy->getElementType());
-      if (!FTy || Record.size() < FTy->getNumParams()+OpNum)
-        return Error("Invalid record");
+      if (!OpTy)
+        return error("Callee is not a pointer type");
+      if (!FTy) {
+        FTy = dyn_cast<FunctionType>(OpTy->getElementType());
+        if (!FTy)
+          return error("Callee is not of pointer to function type");
+      } else if (OpTy->getElementType() != FTy)
+        return error("Explicit call type does not match pointee type of "
+                     "callee operand");
+      if (Record.size() < FTy->getNumParams() + OpNum)
+        return error("Insufficient operands to call");
 
       SmallVector<Value*, 16> Args;
       // Read the fixed params.
@@ -4192,23 +4468,23 @@ std::error_code BitcodeReader::ParseFunctionBody(Function *F) {
           Args.push_back(getValue(Record, OpNum, NextValueNo,
                                   FTy->getParamType(i)));
         if (!Args.back())
-          return Error("Invalid record");
+          return error("Invalid record");
       }
 
       // Read type/value pairs for varargs params.
       if (!FTy->isVarArg()) {
         if (OpNum != Record.size())
-          return Error("Invalid record");
+          return error("Invalid record");
       } else {
         while (OpNum != Record.size()) {
           Value *Op;
           if (getValueTypePair(Record, OpNum, NextValueNo, Op))
-            return Error("Invalid record");
+            return error("Invalid record");
           Args.push_back(Op);
         }
       }
 
-      I = CallInst::Create(Callee, Args);
+      I = CallInst::Create(FTy, Callee, Args);
       InstructionList.push_back(I);
       cast<CallInst>(I)->setCallingConv(
           static_cast<CallingConv::ID>((~(1U << 14) & CCInfo) >> 1));
@@ -4223,12 +4499,12 @@ std::error_code BitcodeReader::ParseFunctionBody(Function *F) {
     }
     case bitc::FUNC_CODE_INST_VAARG: { // VAARG: [valistty, valist, instty]
       if (Record.size() < 3)
-        return Error("Invalid record");
+        return error("Invalid record");
       Type *OpTy = getTypeByID(Record[0]);
       Value *Op = getValue(Record, 1, NextValueNo, OpTy);
       Type *ResTy = getTypeByID(Record[2]);
       if (!OpTy || !Op || !ResTy)
-        return Error("Invalid record");
+        return error("Invalid record");
       I = new VAArgInst(Op, ResTy);
       InstructionList.push_back(I);
       break;
@@ -4239,7 +4515,7 @@ std::error_code BitcodeReader::ParseFunctionBody(Function *F) {
     // this file.
     if (!CurBB) {
       delete I;
-      return Error("Invalid instruction with no BB");
+      return error("Invalid instruction with no BB");
     }
     CurBB->getInstList().push_back(I);
 
@@ -4251,7 +4527,7 @@ std::error_code BitcodeReader::ParseFunctionBody(Function *F) {
 
     // Non-void values get registered in the value table for future use.
     if (I && !I->getType()->isVoidTy())
-      ValueList.AssignValue(I, NextValueNo++);
+      ValueList.assignValue(I, NextValueNo++);
   }
 
 OutOfRecordLoop:
@@ -4266,7 +4542,7 @@ std::error_code BitcodeReader::ParseFunctionBody(Function *F) {
           delete A;
         }
       }
-      return Error("Never resolved value found in function");
+      return error("Never resolved value found in function");
     }
   }
 
@@ -4281,15 +4557,15 @@ std::error_code BitcodeReader::ParseFunctionBody(Function *F) {
 }
 
 /// Find the function body in the bitcode stream
-std::error_code BitcodeReader::FindFunctionInStream(
+std::error_code BitcodeReader::findFunctionInStream(
     Function *F,
     DenseMap<Function *, uint64_t>::iterator DeferredFunctionInfoIterator) {
   while (DeferredFunctionInfoIterator->second == 0) {
     if (Stream.AtEndOfStream())
-      return Error("Could not find function in stream");
+      return error("Could not find function in stream");
     // ParseModule will parse the next body in the stream and set its
     // position in the DeferredFunctionInfo map.
-    if (std::error_code EC = ParseModule(true))
+    if (std::error_code EC = parseModule(true))
       return EC;
   }
   return std::error_code();
@@ -4314,14 +4590,14 @@ std::error_code BitcodeReader::materialize(GlobalValue *GV) {
   assert(DFII != DeferredFunctionInfo.end() && "Deferred function not found!");
   // If its position is recorded as 0, its body is somewhere in the stream
   // but we haven't seen it yet.
-  if (DFII->second == 0 && LazyStreamer)
-    if (std::error_code EC = FindFunctionInStream(F, DFII))
+  if (DFII->second == 0)
+    if (std::error_code EC = findFunctionInStream(F, DFII))
       return EC;
 
   // Move the bit stream to the saved position of the deferred function body.
   Stream.JumpToBit(DFII->second);
 
-  if (std::error_code EC = ParseFunctionBody(F))
+  if (std::error_code EC = parseFunctionBody(F))
     return EC;
   F->setIsMaterializable(false);
 
@@ -4329,14 +4605,12 @@ std::error_code BitcodeReader::materialize(GlobalValue *GV) {
     stripDebugInfo(*F);
 
   // Upgrade any old intrinsic calls in the function.
-  for (UpgradedIntrinsicMap::iterator I = UpgradedIntrinsics.begin(),
-       E = UpgradedIntrinsics.end(); I != E; ++I) {
-    if (I->first != I->second) {
-      for (auto UI = I->first->user_begin(), UE = I->first->user_end();
-           UI != UE;) {
-        if (CallInst* CI = dyn_cast<CallInst>(*UI++))
-          UpgradeIntrinsicCall(CI, I->second);
-      }
+  for (auto &I : UpgradedIntrinsics) {
+    for (auto UI = I.first->user_begin(), UE = I.first->user_end(); UI != UE;) {
+      User *U = *UI;
+      ++UI;
+      if (CallInst *CI = dyn_cast<CallInst>(U))
+        UpgradeIntrinsicCall(CI, I.second);
     }
   }
 
@@ -4358,7 +4632,7 @@ bool BitcodeReader::isDematerializable(const GlobalValue *GV) const {
   return DeferredFunctionInfo.count(const_cast<Function*>(F));
 }
 
-void BitcodeReader::Dematerialize(GlobalValue *GV) {
+void BitcodeReader::dematerialize(GlobalValue *GV) {
   Function *F = dyn_cast<Function>(GV);
   // If this function isn't dematerializable, this is a noop.
   if (!F || !isDematerializable(F))
@@ -4371,7 +4645,7 @@ void BitcodeReader::Dematerialize(GlobalValue *GV) {
   F->setIsMaterializable(true);
 }
 
-std::error_code BitcodeReader::MaterializeModule(Module *M) {
+std::error_code BitcodeReader::materializeModule(Module *M) {
   assert(M == TheModule &&
          "Can only Materialize the Module this BitcodeReader is attached to.");
 
@@ -4392,31 +4666,27 @@ std::error_code BitcodeReader::MaterializeModule(Module *M) {
   // pointing to the END_BLOCK record after them. Now make sure the rest
   // of the bits in the module have been read.
   if (NextUnreadBit)
-    ParseModule(true);
+    parseModule(true);
 
   // Check that all block address forward references got resolved (as we
   // promised above).
   if (!BasicBlockFwdRefs.empty())
-    return Error("Never resolved function from blockaddress");
+    return error("Never resolved function from blockaddress");
 
   // Upgrade any intrinsic calls that slipped through (should not happen!) and
   // delete the old functions to clean up. We can't do this unless the entire
   // module is materialized because there could always be another function body
   // with calls to the old function.
-  for (std::vector<std::pair<Function*, Function*> >::iterator I =
-       UpgradedIntrinsics.begin(), E = UpgradedIntrinsics.end(); I != E; ++I) {
-    if (I->first != I->second) {
-      for (auto UI = I->first->user_begin(), UE = I->first->user_end();
-           UI != UE;) {
-        if (CallInst* CI = dyn_cast<CallInst>(*UI++))
-          UpgradeIntrinsicCall(CI, I->second);
-      }
-      if (!I->first->use_empty())
-        I->first->replaceAllUsesWith(I->second);
-      I->first->eraseFromParent();
-    }
+  for (auto &I : UpgradedIntrinsics) {
+    for (auto *U : I.first->users()) {
+      if (CallInst *CI = dyn_cast<CallInst>(U))
+        UpgradeIntrinsicCall(CI, I.second);
+    }
+    if (!I.first->use_empty())
+      I.first->replaceAllUsesWith(I.second);
+    I.first->eraseFromParent();
   }
-  std::vector<std::pair<Function*, Function*> >().swap(UpgradedIntrinsics);
+  UpgradedIntrinsics.clear();
 
   for (unsigned I = 0, E = InstsWithTBAATag.size(); I < E; I++)
     UpgradeInstWithTBAATag(InstsWithTBAATag[I]);
@@ -4429,24 +4699,25 @@ std::vector<StructType *> BitcodeReader::getIdentifiedStructTypes() const {
   return IdentifiedStructTypes;
 }
 
-std::error_code BitcodeReader::InitStream() {
-  if (LazyStreamer)
-    return InitLazyStream();
-  return InitStreamFromBuffer();
+std::error_code
+BitcodeReader::initStream(std::unique_ptr<DataStreamer> Streamer) {
+  if (Streamer)
+    return initLazyStream(std::move(Streamer));
+  return initStreamFromBuffer();
 }
 
-std::error_code BitcodeReader::InitStreamFromBuffer() {
+std::error_code BitcodeReader::initStreamFromBuffer() {
   const unsigned char *BufPtr = (const unsigned char*)Buffer->getBufferStart();
   const unsigned char *BufEnd = BufPtr+Buffer->getBufferSize();
 
   if (Buffer->getBufferSize() & 3)
-    return Error("Invalid bitcode signature");
+    return error("Invalid bitcode signature");
 
   // If we have a wrapper header, parse it and ignore the non-bc file contents.
   // The magic number is 0x0B17C0DE stored in little endian.
   if (isBitcodeWrapper(BufPtr, BufEnd))
     if (SkipBitcodeWrapperHeader(BufPtr, BufEnd, true))
-      return Error("Invalid bitcode wrapper header");
+      return error("Invalid bitcode wrapper header");
 
   StreamFile.reset(new BitstreamReader(BufPtr, BufEnd));
   Stream.init(&*StreamFile);
@@ -4454,27 +4725,29 @@ std::error_code BitcodeReader::InitStreamFromBuffer() {
   return std::error_code();
 }
 
-std::error_code BitcodeReader::InitLazyStream() {
+std::error_code
+BitcodeReader::initLazyStream(std::unique_ptr<DataStreamer> Streamer) {
   // Check and strip off the bitcode wrapper; BitstreamReader expects never to
   // see it.
-  // @LOCALMOD Bytes -> LazyStreamer
-  std::unique_ptr<StreamingMemoryObject> OwnedBytes(LazyStreamer);
+  auto OwnedBytes =
+      llvm::make_unique<StreamingMemoryObject>(std::move(Streamer));
+  StreamingMemoryObject &Bytes = *OwnedBytes;
   StreamFile = llvm::make_unique<BitstreamReader>(std::move(OwnedBytes));
   Stream.init(&*StreamFile);
 
   unsigned char buf[16];
-  if (LazyStreamer->readBytes(buf, 16, 0) != 16)
-    return Error("Invalid bitcode signature");
+  if (Bytes.readBytes(buf, 16, 0) != 16)
+    return error("Invalid bitcode signature");
 
   if (!isBitcode(buf, buf + 16))
-    return Error("Invalid bitcode signature");
+    return error("Invalid bitcode signature");
 
   if (isBitcodeWrapper(buf, buf + 4)) {
     const unsigned char *bitcodeStart = buf;
     const unsigned char *bitcodeEnd = buf + 16;
     SkipBitcodeWrapperHeader(bitcodeStart, bitcodeEnd, false);
-    LazyStreamer->dropLeadingBytes(bitcodeStart - buf);
-    LazyStreamer->setKnownObjectSize(bitcodeEnd - bitcodeStart);
+    Bytes.dropLeadingBytes(bitcodeStart - buf);
+    Bytes.setKnownObjectSize(bitcodeEnd - bitcodeStart);
   }
   return std::error_code();
 }
@@ -4507,83 +4780,86 @@ const std::error_category &llvm::BitcodeErrorCategory() {
 // External interface
 //===----------------------------------------------------------------------===//
 
-/// \brief Get a lazy one-at-time loading module from bitcode.
-///
-/// This isn't always used in a lazy context.  In particular, it's also used by
-/// \a parseBitcodeFile().  If this is truly lazy, then we need to eagerly pull
-/// in forward-referenced functions from block address references.
-///
-/// \param[in] WillMaterializeAll Set to \c true if the caller promises to
-/// materialize everything -- in particular, if this isn't truly lazy.
-static ErrorOr<Module *>
-getLazyBitcodeModuleImpl(std::unique_ptr<MemoryBuffer> &&Buffer,
-                         LLVMContext &Context, bool WillMaterializeAll,
-                         DiagnosticHandlerFunction DiagnosticHandler,
-                         bool ShouldLazyLoadMetadata = false) {
-  Module *M = new Module(Buffer->getBufferIdentifier(), Context);
-  BitcodeReader *R =
-      new BitcodeReader(Buffer.get(), Context, DiagnosticHandler);
+static ErrorOr<std::unique_ptr<Module>>
+getBitcodeModuleImpl(std::unique_ptr<DataStreamer> Streamer, StringRef Name,
+                     BitcodeReader *R, LLVMContext &Context,
+                     bool MaterializeAll, bool ShouldLazyLoadMetadata) {
+  std::unique_ptr<Module> M = make_unique<Module>(Name, Context);
   M->setMaterializer(R);
 
   auto cleanupOnError = [&](std::error_code EC) {
     R->releaseBuffer(); // Never take ownership on error.
-    delete M;  // Also deletes R.
     return EC;
   };
 
   // Delay parsing Metadata if ShouldLazyLoadMetadata is true.
-  if (std::error_code EC = R->ParseBitcodeInto(M, ShouldLazyLoadMetadata))
+  if (std::error_code EC = R->parseBitcodeInto(std::move(Streamer), M.get(),
+                                               ShouldLazyLoadMetadata))
     return cleanupOnError(EC);
 
-  if (!WillMaterializeAll)
+  if (MaterializeAll) {
+    // Read in the entire module, and destroy the BitcodeReader.
+    if (std::error_code EC = M->materializeAllPermanently())
+      return cleanupOnError(EC);
+  } else {
     // Resolve forward references from blockaddresses.
     if (std::error_code EC = R->materializeForwardReferencedFunctions())
       return cleanupOnError(EC);
+  }
+  return std::move(M);
+}
+
+/// \brief Get a lazy one-at-time loading module from bitcode.
+///
+/// This isn't always used in a lazy context.  In particular, it's also used by
+/// \a parseBitcodeFile().  If this is truly lazy, then we need to eagerly pull
+/// in forward-referenced functions from block address references.
+///
+/// \param[in] MaterializeAll Set to \c true if we should materialize
+/// everything.
+static ErrorOr<std::unique_ptr<Module>>
+getLazyBitcodeModuleImpl(std::unique_ptr<MemoryBuffer> &&Buffer,
+                         LLVMContext &Context, bool MaterializeAll,
+                         DiagnosticHandlerFunction DiagnosticHandler,
+                         bool ShouldLazyLoadMetadata = false) {
+  BitcodeReader *R =
+      new BitcodeReader(Buffer.get(), Context, DiagnosticHandler);
+
+  ErrorOr<std::unique_ptr<Module>> Ret =
+      getBitcodeModuleImpl(nullptr, Buffer->getBufferIdentifier(), R, Context,
+                           MaterializeAll, ShouldLazyLoadMetadata);
+  if (!Ret)
+    return Ret;
 
   Buffer.release(); // The BitcodeReader owns it now.
-  return M;
+  return Ret;
 }
 
-ErrorOr<Module *>
-llvm::getLazyBitcodeModule(std::unique_ptr<MemoryBuffer> &&Buffer,
-                           LLVMContext &Context,
-                           DiagnosticHandlerFunction DiagnosticHandler,
-                           bool ShouldLazyLoadMetadata) {
+ErrorOr<std::unique_ptr<Module>> llvm::getLazyBitcodeModule(
+    std::unique_ptr<MemoryBuffer> &&Buffer, LLVMContext &Context,
+    DiagnosticHandlerFunction DiagnosticHandler, bool ShouldLazyLoadMetadata) {
   return getLazyBitcodeModuleImpl(std::move(Buffer), Context, false,
                                   DiagnosticHandler, ShouldLazyLoadMetadata);
 }
 
-ErrorOr<std::unique_ptr<Module>>
-llvm::getStreamedBitcodeModule(StringRef Name, StreamingMemoryObject *Streamer, // @LOCALMOD
-                               LLVMContext &Context,
-                               DiagnosticHandlerFunction DiagnosticHandler) {
+ErrorOr<std::unique_ptr<Module>> llvm::getStreamedBitcodeModule(
+    StringRef Name, std::unique_ptr<DataStreamer> Streamer,
+    LLVMContext &Context, DiagnosticHandlerFunction DiagnosticHandler) {
   std::unique_ptr<Module> M = make_unique<Module>(Name, Context);
-  BitcodeReader *R = new BitcodeReader(Streamer, Context, DiagnosticHandler);
-  M->setMaterializer(R);
-  if (std::error_code EC = R->ParseBitcodeInto(M.get()))
-    return EC;
-  return std::move(M);
+  BitcodeReader *R = new BitcodeReader(Context, DiagnosticHandler);
+
+  return getBitcodeModuleImpl(std::move(Streamer), Name, R, Context, false,
+                              false);
 }
 
-ErrorOr<Module *>
+ErrorOr<std::unique_ptr<Module>>
 llvm::parseBitcodeFile(MemoryBufferRef Buffer, LLVMContext &Context,
                        DiagnosticHandlerFunction DiagnosticHandler) {
   std::unique_ptr<MemoryBuffer> Buf = MemoryBuffer::getMemBuffer(Buffer, false);
-  ErrorOr<Module *> ModuleOrErr = getLazyBitcodeModuleImpl(
-      std::move(Buf), Context, true, DiagnosticHandler);
-  if (!ModuleOrErr)
-    return ModuleOrErr;
-  Module *M = ModuleOrErr.get();
-  // Read in the entire module, and destroy the BitcodeReader.
-  if (std::error_code EC = M->materializeAllPermanently()) {
-    delete M;
-    return EC;
-  }
-
+  return getLazyBitcodeModuleImpl(std::move(Buf), Context, true,
+                                  DiagnosticHandler);
   // TODO: Restore the use-lists to the in-memory state when the bitcode was
   // written.  We must defer until the Module has been fully materialized.
-
-  return M;
 }
 
 std::string
diff --git a/lib/Bitcode/Reader/BitstreamReader.cpp b/lib/Bitcode/Reader/BitstreamReader.cpp
index beaaf7a7d66..a103fbdf4a9 100644
--- a/lib/Bitcode/Reader/BitstreamReader.cpp
+++ b/lib/Bitcode/Reader/BitstreamReader.cpp
@@ -39,15 +39,16 @@ bool BitstreamCursor::EnterSubBlock(unsigned BlockID, unsigned *NumWordsP) {
 
   // Get the codesize of this block.
   CurCodeSize = ReadVBR(bitc::CodeLenWidth);
+  // We can't read more than MaxChunkSize at a time
+  if (CurCodeSize > MaxChunkSize)
+    return true;
+
   SkipToFourByteBoundary();
   unsigned NumWords = Read(bitc::BlockSizeWidth);
   if (NumWordsP) *NumWordsP = NumWords;
 
   // Validate that this block is sane.
-  if (CurCodeSize == 0 || AtEndOfStream())
-    return true;
-
-  return false;
+  return CurCodeSize == 0 || AtEndOfStream();
 }
 
 static uint64_t readAbbreviatedField(BitstreamCursor &Cursor,
@@ -60,8 +61,10 @@ static uint64_t readAbbreviatedField(BitstreamCursor &Cursor,
   case BitCodeAbbrevOp::Blob:
     llvm_unreachable("Should not reach here");
   case BitCodeAbbrevOp::Fixed:
+    assert((unsigned)Op.getEncodingData() <= Cursor.MaxChunkSize);
     return Cursor.Read((unsigned)Op.getEncodingData());
   case BitCodeAbbrevOp::VBR:
+    assert((unsigned)Op.getEncodingData() <= Cursor.MaxChunkSize);
     return Cursor.ReadVBR64((unsigned)Op.getEncodingData());
   case BitCodeAbbrevOp::Char6:
     return BitCodeAbbrevOp::DecodeChar6(Cursor.Read(6));
@@ -79,9 +82,11 @@ static void skipAbbreviatedField(BitstreamCursor &Cursor,
   case BitCodeAbbrevOp::Blob:
     llvm_unreachable("Should not reach here");
   case BitCodeAbbrevOp::Fixed:
+    assert((unsigned)Op.getEncodingData() <= Cursor.MaxChunkSize);
     Cursor.Read((unsigned)Op.getEncodingData());
     break;
   case BitCodeAbbrevOp::VBR:
+    assert((unsigned)Op.getEncodingData() <= Cursor.MaxChunkSize);
     Cursor.ReadVBR64((unsigned)Op.getEncodingData());
     break;
   case BitCodeAbbrevOp::Char6:
@@ -195,8 +200,15 @@ unsigned BitstreamCursor::readRecord(unsigned AbbrevID,
       unsigned NumElts = ReadVBR(6);
 
       // Get the element encoding.
-      assert(i+2 == e && "array op not second to last?");
+      if (i + 2 != e)
+        report_fatal_error("Array op not second to last");
       const BitCodeAbbrevOp &EltEnc = Abbv->getOperandInfo(++i);
+      if (!EltEnc.isEncoding())
+        report_fatal_error(
+            "Array element type has to be an encoding of a type");
+      if (EltEnc.getEncoding() == BitCodeAbbrevOp::Array ||
+          EltEnc.getEncoding() == BitCodeAbbrevOp::Blob)
+        report_fatal_error("Array element type can't be an Array or a Blob");
 
       // Read all the elements.
       for (; NumElts; --NumElts)
@@ -264,10 +276,18 @@ void BitstreamCursor::ReadAbbrevRecord() {
         continue;
       }
 
+      if ((E == BitCodeAbbrevOp::Fixed || E == BitCodeAbbrevOp::VBR) &&
+          Data > MaxChunkSize)
+        report_fatal_error(
+            "Fixed or VBR abbrev record with size > MaxChunkData");
+
       Abbv->Add(BitCodeAbbrevOp(E, Data));
     } else
       Abbv->Add(BitCodeAbbrevOp(E));
   }
+
+  if (Abbv->getNumOperandInfos() == 0)
+    report_fatal_error("Abbrev record with no operands");
   CurAbbrevs.push_back(Abbv);
 }
 
diff --git a/lib/Bitcode/Writer/BitcodeWriter.cpp b/lib/Bitcode/Writer/BitcodeWriter.cpp
index aa4a6a4c28a..20e31d1a102 100644
--- a/lib/Bitcode/Writer/BitcodeWriter.cpp
+++ b/lib/Bitcode/Writer/BitcodeWriter.cpp
@@ -162,10 +162,14 @@ static uint64_t getAttrKindEncoding(Attribute::AttrKind Kind) {
     return bitc::ATTR_KIND_ALIGNMENT;
   case Attribute::AlwaysInline:
     return bitc::ATTR_KIND_ALWAYS_INLINE;
+  case Attribute::ArgMemOnly:
+    return bitc::ATTR_KIND_ARGMEMONLY;
   case Attribute::Builtin:
     return bitc::ATTR_KIND_BUILTIN;
   case Attribute::ByVal:
     return bitc::ATTR_KIND_BY_VAL;
+  case Attribute::Convergent:
+    return bitc::ATTR_KIND_CONVERGENT;
   case Attribute::InAlloca:
     return bitc::ATTR_KIND_IN_ALLOCA;
   case Attribute::Cold:
@@ -230,6 +234,8 @@ static uint64_t getAttrKindEncoding(Attribute::AttrKind Kind) {
     return bitc::ATTR_KIND_STACK_PROTECT_REQ;
   case Attribute::StackProtectStrong:
     return bitc::ATTR_KIND_STACK_PROTECT_STRONG;
+  case Attribute::SafeStack:
+    return bitc::ATTR_KIND_SAFESTACK;
   case Attribute::StructRet:
     return bitc::ATTR_KIND_STRUCT_RET;
   case Attribute::SanitizeAddress:
@@ -590,7 +596,7 @@ static void WriteModuleInfo(const Module *M, const ValueEnumerator &VE,
   unsigned MaxGlobalType = 0;
   for (const GlobalValue &GV : M->globals()) {
     MaxAlignment = std::max(MaxAlignment, GV.getAlignment());
-    MaxGlobalType = std::max(MaxGlobalType, VE.getTypeID(GV.getType()));
+    MaxGlobalType = std::max(MaxGlobalType, VE.getTypeID(GV.getValueType()));
     if (GV.hasSection()) {
       // Give section names unique ID's.
       unsigned &Entry = SectionMap[GV.getSection()];
@@ -631,10 +637,12 @@ static void WriteModuleInfo(const Module *M, const ValueEnumerator &VE,
     Abbv->Add(BitCodeAbbrevOp(bitc::MODULE_CODE_GLOBALVAR));
     Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed,
                               Log2_32_Ceil(MaxGlobalType+1)));
-    Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 1));      // Constant.
-    Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6));        // Initializer.
-    Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 5));      // Linkage.
-    if (MaxAlignment == 0)                                      // Alignment.
+    Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6));   // AddrSpace << 2
+                                                           //| explicitType << 1
+                                                           //| constant
+    Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6));   // Initializer.
+    Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 5)); // Linkage.
+    if (MaxAlignment == 0)                                 // Alignment.
       Abbv->Add(BitCodeAbbrevOp(0));
     else {
       unsigned MaxEncAlignment = Log2_32(MaxAlignment)+1;
@@ -659,8 +667,8 @@ static void WriteModuleInfo(const Module *M, const ValueEnumerator &VE,
     //             linkage, alignment, section, visibility, threadlocal,
     //             unnamed_addr, externally_initialized, dllstorageclass,
     //             comdat]
-    Vals.push_back(VE.getTypeID(GV.getType()));
-    Vals.push_back(GV.isConstant());
+    Vals.push_back(VE.getTypeID(GV.getValueType()));
+    Vals.push_back(GV.getType()->getAddressSpace() << 2 | 2 | GV.isConstant());
     Vals.push_back(GV.isDeclaration() ? 0 :
                    (VE.getValueID(GV.getInitializer()) + 1));
     Vals.push_back(getEncodedLinkage(GV));
@@ -689,8 +697,8 @@ static void WriteModuleInfo(const Module *M, const ValueEnumerator &VE,
   for (const Function &F : *M) {
     // FUNCTION:  [type, callingconv, isproto, linkage, paramattrs, alignment,
     //             section, visibility, gc, unnamed_addr, prologuedata,
-    //             dllstorageclass, comdat, prefixdata]
-    Vals.push_back(VE.getTypeID(F.getType()));
+    //             dllstorageclass, comdat, prefixdata, personalityfn]
+    Vals.push_back(VE.getTypeID(F.getFunctionType()));
     Vals.push_back(F.getCallingConv());
     Vals.push_back(F.isDeclaration());
     Vals.push_back(getEncodedLinkage(F));
@@ -706,6 +714,8 @@ static void WriteModuleInfo(const Module *M, const ValueEnumerator &VE,
     Vals.push_back(F.hasComdat() ? VE.getComdatID(F.getComdat()) : 0);
     Vals.push_back(F.hasPrefixData() ? (VE.getValueID(F.getPrefixData()) + 1)
                                      : 0);
+    Vals.push_back(
+        F.hasPersonalityFn() ? (VE.getValueID(F.getPersonalityFn()) + 1) : 0);
 
     unsigned AbbrevToUse = 0;
     Stream.EmitRecord(bitc::MODULE_CODE_FUNCTION, Vals, AbbrevToUse);
@@ -782,7 +792,7 @@ static void WriteMDTuple(const MDTuple *N, const ValueEnumerator &VE,
   Record.clear();
 }
 
-static void WriteMDLocation(const MDLocation *N, const ValueEnumerator &VE,
+static void WriteDILocation(const DILocation *N, const ValueEnumerator &VE,
                             BitstreamWriter &Stream,
                             SmallVectorImpl<uint64_t> &Record,
                             unsigned Abbrev) {
@@ -796,11 +806,11 @@ static void WriteMDLocation(const MDLocation *N, const ValueEnumerator &VE,
   Record.clear();
 }
 
-static void WriteGenericDebugNode(const GenericDebugNode *N,
-                                  const ValueEnumerator &VE,
-                                  BitstreamWriter &Stream,
-                                  SmallVectorImpl<uint64_t> &Record,
-                                  unsigned Abbrev) {
+static void WriteGenericDINode(const GenericDINode *N,
+                               const ValueEnumerator &VE,
+                               BitstreamWriter &Stream,
+                               SmallVectorImpl<uint64_t> &Record,
+                               unsigned Abbrev) {
   Record.push_back(N->isDistinct());
   Record.push_back(N->getTag());
   Record.push_back(0); // Per-tag version field; unused for now.
@@ -817,7 +827,7 @@ static uint64_t rotateSign(int64_t I) {
   return I < 0 ? ~(U << 1) : U << 1;
 }
 
-static void WriteMDSubrange(const MDSubrange *N, const ValueEnumerator &,
+static void WriteDISubrange(const DISubrange *N, const ValueEnumerator &,
                             BitstreamWriter &Stream,
                             SmallVectorImpl<uint64_t> &Record,
                             unsigned Abbrev) {
@@ -829,7 +839,7 @@ static void WriteMDSubrange(const MDSubrange *N, const ValueEnumerator &,
   Record.clear();
 }
 
-static void WriteMDEnumerator(const MDEnumerator *N, const ValueEnumerator &VE,
+static void WriteDIEnumerator(const DIEnumerator *N, const ValueEnumerator &VE,
                               BitstreamWriter &Stream,
                               SmallVectorImpl<uint64_t> &Record,
                               unsigned Abbrev) {
@@ -841,7 +851,7 @@ static void WriteMDEnumerator(const MDEnumerator *N, const ValueEnumerator &VE,
   Record.clear();
 }
 
-static void WriteMDBasicType(const MDBasicType *N, const ValueEnumerator &VE,
+static void WriteDIBasicType(const DIBasicType *N, const ValueEnumerator &VE,
                              BitstreamWriter &Stream,
                              SmallVectorImpl<uint64_t> &Record,
                              unsigned Abbrev) {
@@ -856,7 +866,7 @@ static void WriteMDBasicType(const MDBasicType *N, const ValueEnumerator &VE,
   Record.clear();
 }
 
-static void WriteMDDerivedType(const MDDerivedType *N,
+static void WriteDIDerivedType(const DIDerivedType *N,
                                const ValueEnumerator &VE,
                                BitstreamWriter &Stream,
                                SmallVectorImpl<uint64_t> &Record,
@@ -878,7 +888,7 @@ static void WriteMDDerivedType(const MDDerivedType *N,
   Record.clear();
 }
 
-static void WriteMDCompositeType(const MDCompositeType *N,
+static void WriteDICompositeType(const DICompositeType *N,
                                  const ValueEnumerator &VE,
                                  BitstreamWriter &Stream,
                                  SmallVectorImpl<uint64_t> &Record,
@@ -904,7 +914,7 @@ static void WriteMDCompositeType(const MDCompositeType *N,
   Record.clear();
 }
 
-static void WriteMDSubroutineType(const MDSubroutineType *N,
+static void WriteDISubroutineType(const DISubroutineType *N,
                                   const ValueEnumerator &VE,
                                   BitstreamWriter &Stream,
                                   SmallVectorImpl<uint64_t> &Record,
@@ -917,7 +927,7 @@ static void WriteMDSubroutineType(const MDSubroutineType *N,
   Record.clear();
 }
 
-static void WriteMDFile(const MDFile *N, const ValueEnumerator &VE,
+static void WriteDIFile(const DIFile *N, const ValueEnumerator &VE,
                         BitstreamWriter &Stream,
                         SmallVectorImpl<uint64_t> &Record, unsigned Abbrev) {
   Record.push_back(N->isDistinct());
@@ -928,7 +938,7 @@ static void WriteMDFile(const MDFile *N, const ValueEnumerator &VE,
   Record.clear();
 }
 
-static void WriteMDCompileUnit(const MDCompileUnit *N,
+static void WriteDICompileUnit(const DICompileUnit *N,
                                const ValueEnumerator &VE,
                                BitstreamWriter &Stream,
                                SmallVectorImpl<uint64_t> &Record,
@@ -947,16 +957,16 @@ static void WriteMDCompileUnit(const MDCompileUnit *N,
   Record.push_back(VE.getMetadataOrNullID(N->getSubprograms().get()));
   Record.push_back(VE.getMetadataOrNullID(N->getGlobalVariables().get()));
   Record.push_back(VE.getMetadataOrNullID(N->getImportedEntities().get()));
+  Record.push_back(N->getDWOId());
 
   Stream.EmitRecord(bitc::METADATA_COMPILE_UNIT, Record, Abbrev);
   Record.clear();
 }
 
-static void WriteMDSubprogram(const MDSubprogram *N,
-                               const ValueEnumerator &VE,
-                               BitstreamWriter &Stream,
-                               SmallVectorImpl<uint64_t> &Record,
-                               unsigned Abbrev) {
+static void WriteDISubprogram(const DISubprogram *N, const ValueEnumerator &VE,
+                              BitstreamWriter &Stream,
+                              SmallVectorImpl<uint64_t> &Record,
+                              unsigned Abbrev) {
   Record.push_back(N->isDistinct());
   Record.push_back(VE.getMetadataOrNullID(N->getScope()));
   Record.push_back(VE.getMetadataOrNullID(N->getRawName()));
@@ -981,11 +991,11 @@ static void WriteMDSubprogram(const MDSubprogram *N,
   Record.clear();
 }
 
-static void WriteMDLexicalBlock(const MDLexicalBlock *N,
-                               const ValueEnumerator &VE,
-                               BitstreamWriter &Stream,
-                               SmallVectorImpl<uint64_t> &Record,
-                               unsigned Abbrev) {
+static void WriteDILexicalBlock(const DILexicalBlock *N,
+                                const ValueEnumerator &VE,
+                                BitstreamWriter &Stream,
+                                SmallVectorImpl<uint64_t> &Record,
+                                unsigned Abbrev) {
   Record.push_back(N->isDistinct());
   Record.push_back(VE.getMetadataOrNullID(N->getScope()));
   Record.push_back(VE.getMetadataOrNullID(N->getFile()));
@@ -996,7 +1006,7 @@ static void WriteMDLexicalBlock(const MDLexicalBlock *N,
   Record.clear();
 }
 
-static void WriteMDLexicalBlockFile(const MDLexicalBlockFile *N,
+static void WriteDILexicalBlockFile(const DILexicalBlockFile *N,
                                     const ValueEnumerator &VE,
                                     BitstreamWriter &Stream,
                                     SmallVectorImpl<uint64_t> &Record,
@@ -1010,7 +1020,7 @@ static void WriteMDLexicalBlockFile(const MDLexicalBlockFile *N,
   Record.clear();
 }
 
-static void WriteMDNamespace(const MDNamespace *N, const ValueEnumerator &VE,
+static void WriteDINamespace(const DINamespace *N, const ValueEnumerator &VE,
                              BitstreamWriter &Stream,
                              SmallVectorImpl<uint64_t> &Record,
                              unsigned Abbrev) {
@@ -1024,7 +1034,18 @@ static void WriteMDNamespace(const MDNamespace *N, const ValueEnumerator &VE,
   Record.clear();
 }
 
-static void WriteMDTemplateTypeParameter(const MDTemplateTypeParameter *N,
+static void WriteDIModule(const DIModule *N, const ValueEnumerator &VE,
+                          BitstreamWriter &Stream,
+                          SmallVectorImpl<uint64_t> &Record, unsigned Abbrev) {
+  Record.push_back(N->isDistinct());
+  for (auto &I : N->operands())
+    Record.push_back(VE.getMetadataOrNullID(I));
+
+  Stream.EmitRecord(bitc::METADATA_MODULE, Record, Abbrev);
+  Record.clear();
+}
+
+static void WriteDITemplateTypeParameter(const DITemplateTypeParameter *N,
                                          const ValueEnumerator &VE,
                                          BitstreamWriter &Stream,
                                          SmallVectorImpl<uint64_t> &Record,
@@ -1037,7 +1058,7 @@ static void WriteMDTemplateTypeParameter(const MDTemplateTypeParameter *N,
   Record.clear();
 }
 
-static void WriteMDTemplateValueParameter(const MDTemplateValueParameter *N,
+static void WriteDITemplateValueParameter(const DITemplateValueParameter *N,
                                           const ValueEnumerator &VE,
                                           BitstreamWriter &Stream,
                                           SmallVectorImpl<uint64_t> &Record,
@@ -1052,7 +1073,7 @@ static void WriteMDTemplateValueParameter(const MDTemplateValueParameter *N,
   Record.clear();
 }
 
-static void WriteMDGlobalVariable(const MDGlobalVariable *N,
+static void WriteDIGlobalVariable(const DIGlobalVariable *N,
                                   const ValueEnumerator &VE,
                                   BitstreamWriter &Stream,
                                   SmallVectorImpl<uint64_t> &Record,
@@ -1073,7 +1094,7 @@ static void WriteMDGlobalVariable(const MDGlobalVariable *N,
   Record.clear();
 }
 
-static void WriteMDLocalVariable(const MDLocalVariable *N,
+static void WriteDILocalVariable(const DILocalVariable *N,
                                  const ValueEnumerator &VE,
                                  BitstreamWriter &Stream,
                                  SmallVectorImpl<uint64_t> &Record,
@@ -1092,7 +1113,7 @@ static void WriteMDLocalVariable(const MDLocalVariable *N,
   Record.clear();
 }
 
-static void WriteMDExpression(const MDExpression *N, const ValueEnumerator &,
+static void WriteDIExpression(const DIExpression *N, const ValueEnumerator &,
                               BitstreamWriter &Stream,
                               SmallVectorImpl<uint64_t> &Record,
                               unsigned Abbrev) {
@@ -1105,11 +1126,11 @@ static void WriteMDExpression(const MDExpression *N, const ValueEnumerator &,
   Record.clear();
 }
 
-static void WriteMDObjCProperty(const MDObjCProperty *N,
-                                 const ValueEnumerator &VE,
-                                 BitstreamWriter &Stream,
-                                 SmallVectorImpl<uint64_t> &Record,
-                                 unsigned Abbrev) {
+static void WriteDIObjCProperty(const DIObjCProperty *N,
+                                const ValueEnumerator &VE,
+                                BitstreamWriter &Stream,
+                                SmallVectorImpl<uint64_t> &Record,
+                                unsigned Abbrev) {
   Record.push_back(N->isDistinct());
   Record.push_back(VE.getMetadataOrNullID(N->getRawName()));
   Record.push_back(VE.getMetadataOrNullID(N->getFile()));
@@ -1123,7 +1144,7 @@ static void WriteMDObjCProperty(const MDObjCProperty *N,
   Record.clear();
 }
 
-static void WriteMDImportedEntity(const MDImportedEntity *N,
+static void WriteDIImportedEntity(const DIImportedEntity *N,
                                   const ValueEnumerator &VE,
                                   BitstreamWriter &Stream,
                                   SmallVectorImpl<uint64_t> &Record,
@@ -1162,7 +1183,7 @@ static void WriteModuleMetadata(const Module *M,
 #define HANDLE_MDNODE_LEAF(CLASS) unsigned CLASS##Abbrev = 0;
 #include "llvm/IR/Metadata.def"
 
-  if (VE.hasMDLocation()) {
+  if (VE.hasDILocation()) {
     // Abbrev for METADATA_LOCATION.
     //
     // Assume the column is usually under 128, and always output the inlined-at
@@ -1174,10 +1195,10 @@ static void WriteModuleMetadata(const Module *M,
     Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8));
     Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6));
     Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6));
-    MDLocationAbbrev = Stream.EmitAbbrev(Abbv);
+    DILocationAbbrev = Stream.EmitAbbrev(Abbv);
   }
 
-  if (VE.hasGenericDebugNode()) {
+  if (VE.hasGenericDINode()) {
     // Abbrev for METADATA_GENERIC_DEBUG.
     //
     // Assume the column is usually under 128, and always output the inlined-at
@@ -1190,7 +1211,7 @@ static void WriteModuleMetadata(const Module *M,
     Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6));
     Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Array));
     Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6));
-    GenericDebugNodeAbbrev = Stream.EmitAbbrev(Abbv);
+    GenericDINodeAbbrev = Stream.EmitAbbrev(Abbv);
   }
 
   unsigned NameAbbrev = 0;
@@ -1279,6 +1300,15 @@ static void WriteMetadataAttachment(const Function &F,
   // Write metadata attachments
   // METADATA_ATTACHMENT - [m x [value, [n x [id, mdnode]]]
   SmallVector<std::pair<unsigned, MDNode *>, 4> MDs;
+  F.getAllMetadata(MDs);
+  if (!MDs.empty()) {
+    for (const auto &I : MDs) {
+      Record.push_back(I.first);
+      Record.push_back(VE.getMetadataID(I.second));
+    }
+    Stream.EmitRecord(bitc::METADATA_ATTACHMENT, Record, 0);
+    Record.clear();
+  }
 
   for (Function::const_iterator BB = F.begin(), E = F.end(); BB != E; ++BB)
     for (BasicBlock::const_iterator I = BB->begin(), E = BB->end();
@@ -1502,8 +1532,8 @@ static void WriteConstants(unsigned FirstVal, unsigned LastVal,
     } else if (isa<ConstantArray>(C) || isa<ConstantStruct>(C) ||
                isa<ConstantVector>(C)) {
       Code = bitc::CST_CODE_AGGREGATE;
-      for (unsigned i = 0, e = C->getNumOperands(); i != e; ++i)
-        Record.push_back(VE.getValueID(C->getOperand(i)));
+      for (const Value *Op : C->operands())
+        Record.push_back(VE.getValueID(Op));
       AbbrevToUse = AggregateAbbrev;
     } else if (const ConstantExpr *CE = dyn_cast<ConstantExpr>(C)) {
       switch (CE->getOpcode()) {
@@ -1731,13 +1761,17 @@ static void WriteInstruction(const Instruction &I, unsigned InstID,
     pushValue(I.getOperand(2), InstID, Vals, VE);
     break;
   case Instruction::ICmp:
-  case Instruction::FCmp:
+  case Instruction::FCmp: {
     // compare returning Int1Ty or vector of Int1Ty
     Code = bitc::FUNC_CODE_INST_CMP2;
     PushValueAndType(I.getOperand(0), InstID, Vals, VE);
     pushValue(I.getOperand(1), InstID, Vals, VE);
     Vals.push_back(cast<CmpInst>(I).getPredicate());
+    uint64_t Flags = GetOptimizationFlags(&I);
+    if (Flags != 0)
+      Vals.push_back(Flags);
     break;
+  }
 
   case Instruction::Ret:
     {
@@ -1790,15 +1824,15 @@ static void WriteInstruction(const Instruction &I, unsigned InstID,
 
   case Instruction::Invoke: {
     const InvokeInst *II = cast<InvokeInst>(&I);
-    const Value *Callee(II->getCalledValue());
-    PointerType *PTy = cast<PointerType>(Callee->getType());
-    FunctionType *FTy = cast<FunctionType>(PTy->getElementType());
+    const Value *Callee = II->getCalledValue();
+    FunctionType *FTy = II->getFunctionType();
     Code = bitc::FUNC_CODE_INST_INVOKE;
 
     Vals.push_back(VE.getAttributeID(II->getAttributes()));
-    Vals.push_back(II->getCallingConv());
+    Vals.push_back(II->getCallingConv() | 1 << 13);
     Vals.push_back(VE.getValueID(II->getNormalDest()));
     Vals.push_back(VE.getValueID(II->getUnwindDest()));
+    Vals.push_back(VE.getTypeID(FTy));
     PushValueAndType(Callee, InstID, Vals, VE);
 
     // Emit value #'s for the fixed parameters.
@@ -1817,6 +1851,64 @@ static void WriteInstruction(const Instruction &I, unsigned InstID,
     Code = bitc::FUNC_CODE_INST_RESUME;
     PushValueAndType(I.getOperand(0), InstID, Vals, VE);
     break;
+  case Instruction::CleanupRet: {
+    Code = bitc::FUNC_CODE_INST_CLEANUPRET;
+    const auto &CRI = cast<CleanupReturnInst>(I);
+    Vals.push_back(CRI.hasReturnValue());
+    Vals.push_back(CRI.hasUnwindDest());
+    if (CRI.hasReturnValue())
+      PushValueAndType(CRI.getReturnValue(), InstID, Vals, VE);
+    if (CRI.hasUnwindDest())
+      Vals.push_back(VE.getValueID(CRI.getUnwindDest()));
+    break;
+  }
+  case Instruction::CatchRet: {
+    Code = bitc::FUNC_CODE_INST_CATCHRET;
+    const auto &CRI = cast<CatchReturnInst>(I);
+    Vals.push_back(VE.getValueID(CRI.getSuccessor()));
+    break;
+  }
+  case Instruction::CatchPad: {
+    Code = bitc::FUNC_CODE_INST_CATCHPAD;
+    const auto &CPI = cast<CatchPadInst>(I);
+    Vals.push_back(VE.getTypeID(CPI.getType()));
+    Vals.push_back(VE.getValueID(CPI.getNormalDest()));
+    Vals.push_back(VE.getValueID(CPI.getUnwindDest()));
+    unsigned NumArgOperands = CPI.getNumArgOperands();
+    Vals.push_back(NumArgOperands);
+    for (unsigned Op = 0; Op != NumArgOperands; ++Op)
+      PushValueAndType(CPI.getArgOperand(Op), InstID, Vals, VE);
+    break;
+  }
+  case Instruction::TerminatePad: {
+    Code = bitc::FUNC_CODE_INST_TERMINATEPAD;
+    const auto &TPI = cast<TerminatePadInst>(I);
+    Vals.push_back(TPI.hasUnwindDest());
+    if (TPI.hasUnwindDest())
+      Vals.push_back(VE.getValueID(TPI.getUnwindDest()));
+    unsigned NumArgOperands = TPI.getNumArgOperands();
+    Vals.push_back(NumArgOperands);
+    for (unsigned Op = 0; Op != NumArgOperands; ++Op)
+      PushValueAndType(TPI.getArgOperand(Op), InstID, Vals, VE);
+    break;
+  }
+  case Instruction::CleanupPad: {
+    Code = bitc::FUNC_CODE_INST_CLEANUPPAD;
+    const auto &CPI = cast<CleanupPadInst>(I);
+    Vals.push_back(VE.getTypeID(CPI.getType()));
+    unsigned NumOperands = CPI.getNumOperands();
+    Vals.push_back(NumOperands);
+    for (unsigned Op = 0; Op != NumOperands; ++Op)
+      PushValueAndType(CPI.getOperand(Op), InstID, Vals, VE);
+    break;
+  }
+  case Instruction::CatchEndPad: {
+    Code = bitc::FUNC_CODE_INST_CATCHENDPAD;
+    const auto &CEPI = cast<CatchEndPadInst>(I);
+    if (CEPI.hasUnwindDest())
+      Vals.push_back(VE.getValueID(CEPI.getUnwindDest()));
+    break;
+  }
   case Instruction::Unreachable:
     Code = bitc::FUNC_CODE_INST_UNREACHABLE;
     AbbrevToUse = FUNCTION_INST_UNREACHABLE_ABBREV;
@@ -1844,7 +1936,6 @@ static void WriteInstruction(const Instruction &I, unsigned InstID,
     const LandingPadInst &LP = cast<LandingPadInst>(I);
     Code = bitc::FUNC_CODE_INST_LANDINGPAD;
     Vals.push_back(VE.getTypeID(LP.getType()));
-    PushValueAndType(LP.getPersonalityFn(), InstID, Vals, VE);
     Vals.push_back(LP.isCleanup());
     Vals.push_back(LP.getNumClauses());
     for (unsigned I = 0, E = LP.getNumClauses(); I != E; ++I) {
@@ -1859,15 +1950,18 @@ static void WriteInstruction(const Instruction &I, unsigned InstID,
 
   case Instruction::Alloca: {
     Code = bitc::FUNC_CODE_INST_ALLOCA;
-    Vals.push_back(VE.getTypeID(I.getType()));
+    const AllocaInst &AI = cast<AllocaInst>(I);
+    Vals.push_back(VE.getTypeID(AI.getAllocatedType()));
     Vals.push_back(VE.getTypeID(I.getOperand(0)->getType()));
     Vals.push_back(VE.getValueID(I.getOperand(0))); // size.
-    const AllocaInst &AI = cast<AllocaInst>(I);
     unsigned AlignRecord = Log2_32(AI.getAlignment()) + 1;
     assert(Log2_32(Value::MaximumAlignment) + 1 < 1 << 5 &&
            "not enough bits for maximum alignment");
     assert(AlignRecord < 1 << 5 && "alignment greater than 1 << 64");
     AlignRecord |= AI.isUsedWithInAlloca() << 5;
+    AlignRecord |= 1 << 6;
+    // Reserve bit 7 for SwiftError flag.
+    // AlignRecord |= AI.isSwiftError() << 7;
     Vals.push_back(AlignRecord);
     break;
   }
@@ -1895,7 +1989,7 @@ static void WriteInstruction(const Instruction &I, unsigned InstID,
     else
       Code = bitc::FUNC_CODE_INST_STORE;
     PushValueAndType(I.getOperand(1), InstID, Vals, VE);  // ptrty + ptr
-    pushValue(I.getOperand(0), InstID, Vals, VE);         // val.
+    PushValueAndType(I.getOperand(0), InstID, Vals, VE);  // valty + val
     Vals.push_back(Log2_32(cast<StoreInst>(I).getAlignment())+1);
     Vals.push_back(cast<StoreInst>(I).isVolatile());
     if (cast<StoreInst>(I).isAtomic()) {
@@ -1906,7 +2000,7 @@ static void WriteInstruction(const Instruction &I, unsigned InstID,
   case Instruction::AtomicCmpXchg:
     Code = bitc::FUNC_CODE_INST_CMPXCHG;
     PushValueAndType(I.getOperand(0), InstID, Vals, VE);  // ptrty + ptr
-    pushValue(I.getOperand(1), InstID, Vals, VE);         // cmp.
+    PushValueAndType(I.getOperand(1), InstID, Vals, VE);         // cmp.
     pushValue(I.getOperand(2), InstID, Vals, VE);         // newval.
     Vals.push_back(cast<AtomicCmpXchgInst>(I).isVolatile());
     Vals.push_back(GetEncodedOrdering(
@@ -1935,14 +2029,14 @@ static void WriteInstruction(const Instruction &I, unsigned InstID,
     break;
   case Instruction::Call: {
     const CallInst &CI = cast<CallInst>(I);
-    PointerType *PTy = cast<PointerType>(CI.getCalledValue()->getType());
-    FunctionType *FTy = cast<FunctionType>(PTy->getElementType());
+    FunctionType *FTy = CI.getFunctionType();
 
     Code = bitc::FUNC_CODE_INST_CALL;
 
     Vals.push_back(VE.getAttributeID(CI.getAttributes()));
     Vals.push_back((CI.getCallingConv() << 1) | unsigned(CI.isTailCall()) |
-                   unsigned(CI.isMustTailCall()) << 14);
+                   unsigned(CI.isMustTailCall()) << 14 | 1 << 15);
+    Vals.push_back(VE.getTypeID(FTy));
     PushValueAndType(CI.getCalledValue(), InstID, Vals, VE);  // Callee
 
     // Emit value #'s for the fixed parameters.
@@ -2091,9 +2185,9 @@ static void WriteFunction(const Function &F, ValueEnumerator &VE,
   // Keep a running idea of what the instruction ID is.
   unsigned InstID = CstEnd;
 
-  bool NeedsMetadataAttachment = false;
+  bool NeedsMetadataAttachment = F.hasMetadata();
 
-  MDLocation *LastDL = nullptr;
+  DILocation *LastDL = nullptr;
 
   // Finally, emit all the instructions, in order.
   for (Function::const_iterator BB = F.begin(), E = F.end(); BB != E; ++BB)
@@ -2108,7 +2202,7 @@ static void WriteFunction(const Function &F, ValueEnumerator &VE,
       NeedsMetadataAttachment |= I->hasMetadataOtherThanDebugLoc();
 
       // If the instruction has a debug location, emit it.
-      MDLocation *DL = I->getDebugLoc();
+      DILocation *DL = I->getDebugLoc();
       if (!DL)
         continue;
 
@@ -2124,6 +2218,8 @@ static void WriteFunction(const Function &F, ValueEnumerator &VE,
       Vals.push_back(VE.getMetadataOrNullID(DL->getInlinedAt()));
       Stream.EmitRecord(bitc::FUNC_CODE_DEBUG_LOC, Vals);
       Vals.clear();
+
+      LastDL = DL;
     }
 
   // Emit names for all the instructions etc.
@@ -2387,10 +2483,7 @@ enum {
 
 static void WriteInt32ToBuffer(uint32_t Value, SmallVectorImpl<char> &Buffer,
                                uint32_t &Position) {
-  Buffer[Position + 0] = (unsigned char) (Value >>  0);
-  Buffer[Position + 1] = (unsigned char) (Value >>  8);
-  Buffer[Position + 2] = (unsigned char) (Value >> 16);
-  Buffer[Position + 3] = (unsigned char) (Value >> 24);
+  support::endian::write32le(&Buffer[Position], Value);
   Position += 4;
 }
 
diff --git a/lib/Bitcode/Writer/CMakeLists.txt b/lib/Bitcode/Writer/CMakeLists.txt
index f097b097c33..82dc6b24137 100644
--- a/lib/Bitcode/Writer/CMakeLists.txt
+++ b/lib/Bitcode/Writer/CMakeLists.txt
@@ -3,4 +3,7 @@ add_llvm_library(LLVMBitWriter
   BitcodeWriter.cpp
   BitcodeWriterPass.cpp
   ValueEnumerator.cpp
+
+  DEPENDS
+  intrinsics_gen
   )
diff --git a/lib/Bitcode/Writer/ValueEnumerator.cpp b/lib/Bitcode/Writer/ValueEnumerator.cpp
index 7f576d758bd..44dd604f882 100644
--- a/lib/Bitcode/Writer/ValueEnumerator.cpp
+++ b/lib/Bitcode/Writer/ValueEnumerator.cpp
@@ -93,6 +93,9 @@ static OrderMap orderModule(const Module &M) {
     if (F.hasPrologueData())
       if (!isa<GlobalValue>(F.getPrologueData()))
         orderValue(F.getPrologueData(), OM);
+    if (F.hasPersonalityFn())
+      if (!isa<GlobalValue>(F.getPersonalityFn()))
+        orderValue(F.getPersonalityFn(), OM);
   }
   OM.LastGlobalConstantID = OM.size();
 
@@ -274,6 +277,8 @@ static UseListOrderStack predictUseListOrder(const Module &M) {
       predictValueUseListOrder(F.getPrefixData(), nullptr, OM, Stack);
     if (F.hasPrologueData())
       predictValueUseListOrder(F.getPrologueData(), nullptr, OM, Stack);
+    if (F.hasPersonalityFn())
+      predictValueUseListOrder(F.getPersonalityFn(), nullptr, OM, Stack);
   }
 
   return Stack;
@@ -285,50 +290,51 @@ static bool isIntOrIntVectorValue(const std::pair<const Value*, unsigned> &V) {
 
 ValueEnumerator::ValueEnumerator(const Module &M,
                                  bool ShouldPreserveUseListOrder)
-    : HasMDString(false), HasMDLocation(false), HasGenericDebugNode(false),
+    : HasMDString(false), HasDILocation(false), HasGenericDINode(false),
       ShouldPreserveUseListOrder(ShouldPreserveUseListOrder) {
   if (ShouldPreserveUseListOrder)
     UseListOrders = predictUseListOrder(M);
 
   // Enumerate the global variables.
-  for (Module::const_global_iterator I = M.global_begin(), E = M.global_end();
-       I != E; ++I)
-    EnumerateValue(I);
+  for (const GlobalVariable &GV : M.globals())
+    EnumerateValue(&GV);
 
   // Enumerate the functions.
-  for (Module::const_iterator I = M.begin(), E = M.end(); I != E; ++I) {
-    EnumerateValue(I);
-    EnumerateAttributes(cast<Function>(I)->getAttributes());
+  for (const Function & F : M) {
+    EnumerateValue(&F);
+    EnumerateAttributes(F.getAttributes());
   }
 
   // Enumerate the aliases.
-  for (Module::const_alias_iterator I = M.alias_begin(), E = M.alias_end();
-       I != E; ++I)
-    EnumerateValue(I);
+  for (const GlobalAlias &GA : M.aliases())
+    EnumerateValue(&GA);
 
   // Remember what is the cutoff between globalvalue's and other constants.
   unsigned FirstConstant = Values.size();
 
   // Enumerate the global variable initializers.
-  for (Module::const_global_iterator I = M.global_begin(), E = M.global_end();
-       I != E; ++I)
-    if (I->hasInitializer())
-      EnumerateValue(I->getInitializer());
+  for (const GlobalVariable &GV : M.globals())
+    if (GV.hasInitializer())
+      EnumerateValue(GV.getInitializer());
 
   // Enumerate the aliasees.
-  for (Module::const_alias_iterator I = M.alias_begin(), E = M.alias_end();
-       I != E; ++I)
-    EnumerateValue(I->getAliasee());
+  for (const GlobalAlias &GA : M.aliases())
+    EnumerateValue(GA.getAliasee());
 
   // Enumerate the prefix data constants.
-  for (Module::const_iterator I = M.begin(), E = M.end(); I != E; ++I)
-    if (I->hasPrefixData())
-      EnumerateValue(I->getPrefixData());
+  for (const Function &F : M)
+    if (F.hasPrefixData())
+      EnumerateValue(F.getPrefixData());
 
   // Enumerate the prologue data constants.
+  for (const Function &F : M)
+    if (F.hasPrologueData())
+      EnumerateValue(F.getPrologueData());
+
+  // Enumerate the personality functions.
   for (Module::const_iterator I = M.begin(), E = M.end(); I != E; ++I)
-    if (I->hasPrologueData())
-      EnumerateValue(I->getPrologueData());
+    if (I->hasPersonalityFn())
+      EnumerateValue(I->getPersonalityFn());
 
   // Enumerate the metadata type.
   //
@@ -348,6 +354,11 @@ ValueEnumerator::ValueEnumerator(const Module &M,
     for (const Argument &A : F.args())
       EnumerateType(A.getType());
 
+    // Enumerate metadata attached to this function.
+    F.getAllMetadata(MDs);
+    for (const auto &I : MDs)
+      EnumerateMetadata(I.second);
+
     for (const BasicBlock &BB : F)
       for (const Instruction &I : BB) {
         for (const Use &Op : I.operands()) {
@@ -377,7 +388,7 @@ ValueEnumerator::ValueEnumerator(const Module &M,
 
         // Don't enumerate the location directly -- it has a special record
         // type -- but enumerate its operands.
-        if (MDLocation *L = I.getDebugLoc())
+        if (DILocation *L = I.getDebugLoc())
           EnumerateMDNodeOperands(L);
       }
   }
@@ -543,8 +554,8 @@ void ValueEnumerator::EnumerateMetadata(const Metadata *MD) {
     EnumerateValue(C->getValue());
 
   HasMDString |= isa<MDString>(MD);
-  HasMDLocation |= isa<MDLocation>(MD);
-  HasGenericDebugNode |= isa<GenericDebugNode>(MD);
+  HasDILocation |= isa<DILocation>(MD);
+  HasGenericDINode |= isa<GenericDINode>(MD);
 
   // Replace the dummy ID inserted above with the correct one.  MDValueMap may
   // have changed by inserting operands, so we need a fresh lookup here.
@@ -680,9 +691,7 @@ void ValueEnumerator::EnumerateOperandType(const Value *V) {
 
   // This constant may have operands, make sure to enumerate the types in
   // them.
-  for (unsigned i = 0, e = C->getNumOperands(); i != e; ++i) {
-    const Value *Op = C->getOperand(i);
-
+  for (const Value *Op : C->operands()) {
     // Don't enumerate basic blocks here, this happens as operands to
     // blockaddress.
     if (isa<BasicBlock>(Op))
diff --git a/lib/Bitcode/Writer/ValueEnumerator.h b/lib/Bitcode/Writer/ValueEnumerator.h
index ba245a3de68..92d166e3ba9 100644
--- a/lib/Bitcode/Writer/ValueEnumerator.h
+++ b/lib/Bitcode/Writer/ValueEnumerator.h
@@ -65,8 +65,8 @@ class ValueEnumerator {
   typedef DenseMap<const Metadata *, unsigned> MetadataMapType;
   MetadataMapType MDValueMap;
   bool HasMDString;
-  bool HasMDLocation;
-  bool HasGenericDebugNode;
+  bool HasDILocation;
+  bool HasGenericDINode;
   bool ShouldPreserveUseListOrder;
 
   typedef DenseMap<AttributeSet, unsigned> AttributeGroupMapType;
@@ -121,8 +121,8 @@ class ValueEnumerator {
   }
 
   bool hasMDString() const { return HasMDString; }
-  bool hasMDLocation() const { return HasMDLocation; }
-  bool hasGenericDebugNode() const { return HasGenericDebugNode; }
+  bool hasDILocation() const { return HasDILocation; }
+  bool hasGenericDINode() const { return HasGenericDINode; }
 
   bool shouldPreserveUseListOrder() const { return ShouldPreserveUseListOrder; }
 
diff --git a/lib/CMakeLists.txt b/lib/CMakeLists.txt
index ce10998768d..d00c10f5802 100644
--- a/lib/CMakeLists.txt
+++ b/lib/CMakeLists.txt
@@ -19,3 +19,4 @@ add_subdirectory(LineEditor)
 add_subdirectory(ProfileData)
 add_subdirectory(Fuzzer)
 add_subdirectory(Passes)
+add_subdirectory(LibDriver)
diff --git a/lib/CodeGen/AggressiveAntiDepBreaker.cpp b/lib/CodeGen/AggressiveAntiDepBreaker.cpp
index 58b87e12912..fc5dcfae863 100644
--- a/lib/CodeGen/AggressiveAntiDepBreaker.cpp
+++ b/lib/CodeGen/AggressiveAntiDepBreaker.cpp
@@ -163,7 +163,7 @@ void AggressiveAntiDepBreaker::StartBlock(MachineBasicBlock *BB) {
   // all callee-saved registers. In non-return this is any
   // callee-saved register that is not saved in the prolog.
   const MachineFrameInfo *MFI = MF.getFrameInfo();
-  BitVector Pristine = MFI->getPristineRegs(BB);
+  BitVector Pristine = MFI->getPristineRegs(MF);
   for (const MCPhysReg *I = TRI->getCalleeSavedRegs(&MF); *I; ++I) {
     unsigned Reg = *I;
     if (!IsReturnBlock && !Pristine.test(Reg)) continue;
@@ -509,15 +509,8 @@ BitVector AggressiveAntiDepBreaker::GetRenameRegisters(unsigned Reg) {
   // Check all references that need rewriting for Reg. For each, use
   // the corresponding register class to narrow the set of registers
   // that are appropriate for renaming.
-  std::pair<std::multimap<unsigned,
-                     AggressiveAntiDepState::RegisterReference>::iterator,
-            std::multimap<unsigned,
-                     AggressiveAntiDepState::RegisterReference>::iterator>
-    Range = State->GetRegRefs().equal_range(Reg);
-  for (std::multimap<unsigned,
-       AggressiveAntiDepState::RegisterReference>::iterator Q = Range.first,
-       QE = Range.second; Q != QE; ++Q) {
-    const TargetRegisterClass *RC = Q->second.RC;
+  for (const auto &Q : make_range(State->GetRegRefs().equal_range(Reg))) {
+    const TargetRegisterClass *RC = Q.second.RC;
     if (!RC) continue;
 
     BitVector RCBV = TRI->getAllocatableSet(MF, RC);
@@ -685,9 +678,8 @@ bool AggressiveAntiDepBreaker::FindSuitableFreeRegisters(
 
       // We cannot rename 'Reg' to 'NewReg' if one of the uses of 'Reg' also
       // defines 'NewReg' via an early-clobber operand.
-      auto Range = RegRefs.equal_range(Reg);
-      for (auto Q = Range.first, QE = Range.second; Q != QE; ++Q) {
-        auto UseMI = Q->second.Operand->getParent();
+      for (const auto &Q : make_range(RegRefs.equal_range(Reg))) {
+        MachineInstr *UseMI = Q.second.Operand->getParent();
         int Idx = UseMI->findRegisterDefOperandIdx(NewReg, false, true, TRI);
         if (Idx == -1)
           continue;
@@ -920,23 +912,16 @@ unsigned AggressiveAntiDepBreaker::BreakAntiDependencies(
 
             // Update the references to the old register CurrReg to
             // refer to the new register NewReg.
-            std::pair<std::multimap<unsigned,
-                           AggressiveAntiDepState::RegisterReference>::iterator,
-                      std::multimap<unsigned,
-                           AggressiveAntiDepState::RegisterReference>::iterator>
-              Range = RegRefs.equal_range(CurrReg);
-            for (std::multimap<unsigned,
-                 AggressiveAntiDepState::RegisterReference>::iterator
-                   Q = Range.first, QE = Range.second; Q != QE; ++Q) {
-              Q->second.Operand->setReg(NewReg);
+            for (const auto &Q : make_range(RegRefs.equal_range(CurrReg))) {
+              Q.second.Operand->setReg(NewReg);
               // If the SU for the instruction being updated has debug
               // information related to the anti-dependency register, make
               // sure to update that as well.
-              const SUnit *SU = MISUnitMap[Q->second.Operand->getParent()];
+              const SUnit *SU = MISUnitMap[Q.second.Operand->getParent()];
               if (!SU) continue;
               for (DbgValueVector::iterator DVI = DbgValues.begin(),
                      DVE = DbgValues.end(); DVI != DVE; ++DVI)
-                if (DVI->second == Q->second.Operand->getParent())
+                if (DVI->second == Q.second.Operand->getParent())
                   UpdateDbgValue(DVI->first, AntiDepReg, NewReg);
             }
 
diff --git a/lib/CodeGen/AggressiveAntiDepBreaker.h b/lib/CodeGen/AggressiveAntiDepBreaker.h
index f9544dd1bee..eba73839660 100644
--- a/lib/CodeGen/AggressiveAntiDepBreaker.h
+++ b/lib/CodeGen/AggressiveAntiDepBreaker.h
@@ -33,7 +33,7 @@ namespace llvm {
 class RegisterClassInfo;
 
   /// Contains all the state necessary for anti-dep breaking.
-  class AggressiveAntiDepState {
+class LLVM_LIBRARY_VISIBILITY AggressiveAntiDepState {
   public:
     /// Information about a register reference within a liverange
     typedef struct {
@@ -63,11 +63,11 @@ class RegisterClassInfo;
     /// Map registers to all their references within a live range.
     std::multimap<unsigned, RegisterReference> RegRefs;
 
-    /// The index of the most recent kill (proceding bottom-up),
+    /// The index of the most recent kill (proceeding bottom-up),
     /// or ~0u if the register is not live.
     std::vector<unsigned> KillIndices;
 
-    /// The index of the most recent complete def (proceding bottom
+    /// The index of the most recent complete def (proceeding bottom
     /// up), or ~0u if the register is live.
     std::vector<unsigned> DefIndices;
 
@@ -108,8 +108,8 @@ class RegisterClassInfo;
     bool IsLive(unsigned Reg);
   };
 
-
-  class AggressiveAntiDepBreaker : public AntiDepBreaker {
+  class LLVM_LIBRARY_VISIBILITY AggressiveAntiDepBreaker
+      : public AntiDepBreaker {
     MachineFunction& MF;
     MachineRegisterInfo &MRI;
     const TargetInstrInfo *TII;
diff --git a/lib/CodeGen/AllocationOrder.cpp b/lib/CodeGen/AllocationOrder.cpp
index dc9bcff5612..40451c0d6c1 100644
--- a/lib/CodeGen/AllocationOrder.cpp
+++ b/lib/CodeGen/AllocationOrder.cpp
@@ -29,12 +29,13 @@ using namespace llvm;
 // Compare VirtRegMap::getRegAllocPref().
 AllocationOrder::AllocationOrder(unsigned VirtReg,
                                  const VirtRegMap &VRM,
-                                 const RegisterClassInfo &RegClassInfo)
+                                 const RegisterClassInfo &RegClassInfo,
+                                 const LiveRegMatrix *Matrix)
   : Pos(0) {
   const MachineFunction &MF = VRM.getMachineFunction();
   const TargetRegisterInfo *TRI = &VRM.getTargetRegInfo();
   Order = RegClassInfo.getOrder(MF.getRegInfo().getRegClass(VirtReg));
-  TRI->getRegAllocationHints(VirtReg, Order, Hints, MF, &VRM);
+  TRI->getRegAllocationHints(VirtReg, Order, Hints, MF, &VRM, Matrix);
   rewind();
 
   DEBUG({
diff --git a/lib/CodeGen/AllocationOrder.h b/lib/CodeGen/AllocationOrder.h
index 1e4eaa76ee7..2aee3a63a2b 100644
--- a/lib/CodeGen/AllocationOrder.h
+++ b/lib/CodeGen/AllocationOrder.h
@@ -24,8 +24,9 @@ namespace llvm {
 
 class RegisterClassInfo;
 class VirtRegMap;
+class LiveRegMatrix;
 
-class AllocationOrder {
+class LLVM_LIBRARY_VISIBILITY AllocationOrder {
   SmallVector<MCPhysReg, 16> Hints;
   ArrayRef<MCPhysReg> Order;
   int Pos;
@@ -37,7 +38,8 @@ class AllocationOrder {
   /// @param RegClassInfo Information about reserved and allocatable registers.
   AllocationOrder(unsigned VirtReg,
                   const VirtRegMap &VRM,
-                  const RegisterClassInfo &RegClassInfo);
+                  const RegisterClassInfo &RegClassInfo,
+                  const LiveRegMatrix *Matrix);
 
   /// Get the allocation order without reordered hints.
   ArrayRef<MCPhysReg> getOrder() const { return Order; }
diff --git a/lib/CodeGen/Analysis.cpp b/lib/CodeGen/Analysis.cpp
index 8e11fe1c9cf..98d4c8afc7b 100644
--- a/lib/CodeGen/Analysis.cpp
+++ b/lib/CodeGen/Analysis.cpp
@@ -81,27 +81,27 @@ unsigned llvm::ComputeLinearIndex(Type *Ty,
 /// If Offsets is non-null, it points to a vector to be filled in
 /// with the in-memory offsets of each of the individual values.
 ///
-void llvm::ComputeValueVTs(const TargetLowering &TLI, Type *Ty,
-                           SmallVectorImpl<EVT> &ValueVTs,
+void llvm::ComputeValueVTs(const TargetLowering &TLI, const DataLayout &DL,
+                           Type *Ty, SmallVectorImpl<EVT> &ValueVTs,
                            SmallVectorImpl<uint64_t> *Offsets,
                            uint64_t StartingOffset) {
   // Given a struct type, recursively traverse the elements.
   if (StructType *STy = dyn_cast<StructType>(Ty)) {
-    const StructLayout *SL = TLI.getDataLayout()->getStructLayout(STy);
+    const StructLayout *SL = DL.getStructLayout(STy);
     for (StructType::element_iterator EB = STy->element_begin(),
                                       EI = EB,
                                       EE = STy->element_end();
          EI != EE; ++EI)
-      ComputeValueVTs(TLI, *EI, ValueVTs, Offsets,
+      ComputeValueVTs(TLI, DL, *EI, ValueVTs, Offsets,
                       StartingOffset + SL->getElementOffset(EI - EB));
     return;
   }
   // Given an array type, recursively traverse the elements.
   if (ArrayType *ATy = dyn_cast<ArrayType>(Ty)) {
     Type *EltTy = ATy->getElementType();
-    uint64_t EltSize = TLI.getDataLayout()->getTypeAllocSize(EltTy);
+    uint64_t EltSize = DL.getTypeAllocSize(EltTy);
     for (unsigned i = 0, e = ATy->getNumElements(); i != e; ++i)
-      ComputeValueVTs(TLI, EltTy, ValueVTs, Offsets,
+      ComputeValueVTs(TLI, DL, EltTy, ValueVTs, Offsets,
                       StartingOffset + i * EltSize);
     return;
   }
@@ -109,7 +109,7 @@ void llvm::ComputeValueVTs(const TargetLowering &TLI, Type *Ty,
   if (Ty->isVoidTy())
     return;
   // Base case: we can get an EVT for this LLVM IR type.
-  ValueVTs.push_back(TLI.getValueType(Ty));
+  ValueVTs.push_back(TLI.getValueType(DL, Ty));
   if (Offsets)
     Offsets->push_back(StartingOffset);
 }
@@ -233,7 +233,8 @@ static bool isNoopBitcast(Type *T1, Type *T2,
 static const Value *getNoopInput(const Value *V,
                                  SmallVectorImpl<unsigned> &ValLoc,
                                  unsigned &DataBits,
-                                 const TargetLoweringBase &TLI) {
+                                 const TargetLoweringBase &TLI,
+                                 const DataLayout &DL) {
   while (true) {
     // Try to look through V1; if V1 is not an instruction, it can't be looked
     // through.
@@ -255,16 +256,16 @@ static const Value *getNoopInput(const Value *V,
       // Make sure this isn't a truncating or extending cast.  We could
       // support this eventually, but don't bother for now.
       if (!isa<VectorType>(I->getType()) &&
-          TLI.getPointerTy().getSizeInBits() ==
-          cast<IntegerType>(Op->getType())->getBitWidth())
+          DL.getPointerSizeInBits() ==
+              cast<IntegerType>(Op->getType())->getBitWidth())
         NoopInput = Op;
     } else if (isa<PtrToIntInst>(I)) {
       // Look through ptrtoint.
       // Make sure this isn't a truncating or extending cast.  We could
       // support this eventually, but don't bother for now.
       if (!isa<VectorType>(I->getType()) &&
-          TLI.getPointerTy().getSizeInBits() ==
-          cast<IntegerType>(I->getType())->getBitWidth())
+          DL.getPointerSizeInBits() ==
+              cast<IntegerType>(I->getType())->getBitWidth())
         NoopInput = Op;
     } else if (isa<TruncInst>(I) &&
                TLI.allowTruncateForTailCall(Op->getType(), I->getType())) {
@@ -295,8 +296,8 @@ static const Value *getNoopInput(const Value *V,
     } else if (const InsertValueInst *IVI = dyn_cast<InsertValueInst>(V)) {
       // Value may come from either the aggregate or the scalar
       ArrayRef<unsigned> InsertLoc = IVI->getIndices();
-      if (std::equal(InsertLoc.rbegin(), InsertLoc.rend(),
-                     ValLoc.rbegin())) {
+      if (ValLoc.size() >= InsertLoc.size() &&
+          std::equal(InsertLoc.begin(), InsertLoc.end(), ValLoc.rbegin())) {
         // The type being inserted is a nested sub-type of the aggregate; we
         // have to remove those initial indices to get the location we're
         // interested in for the operand.
@@ -331,14 +332,15 @@ static bool slotOnlyDiscardsData(const Value *RetVal, const Value *CallVal,
                                  SmallVectorImpl<unsigned> &RetIndices,
                                  SmallVectorImpl<unsigned> &CallIndices,
                                  bool AllowDifferingSizes,
-                                 const TargetLoweringBase &TLI) {
+                                 const TargetLoweringBase &TLI,
+                                 const DataLayout &DL) {
 
   // Trace the sub-value needed by the return value as far back up the graph as
   // possible, in the hope that it will intersect with the value produced by the
   // call. In the simple case with no "returned" attribute, the hope is actually
   // that we end up back at the tail call instruction itself.
   unsigned BitsRequired = UINT_MAX;
-  RetVal = getNoopInput(RetVal, RetIndices, BitsRequired, TLI);
+  RetVal = getNoopInput(RetVal, RetIndices, BitsRequired, TLI, DL);
 
   // If this slot in the value returned is undef, it doesn't matter what the
   // call puts there, it'll be fine.
@@ -350,7 +352,7 @@ static bool slotOnlyDiscardsData(const Value *RetVal, const Value *CallVal,
   // a "returned" attribute, the search will be blocked immediately and the loop
   // a Noop.
   unsigned BitsProvided = UINT_MAX;
-  CallVal = getNoopInput(CallVal, CallIndices, BitsProvided, TLI);
+  CallVal = getNoopInput(CallVal, CallIndices, BitsProvided, TLI, DL);
 
   // There's no hope if we can't actually trace them to (the same part of!) the
   // same value.
@@ -606,7 +608,8 @@ bool llvm::returnTypeIsEligibleForTailCall(const Function *F,
     // Finally, we can check whether the value produced by the tail call at this
     // index is compatible with the value we return.
     if (!slotOnlyDiscardsData(RetVal, CallVal, TmpRetPath, TmpCallPath,
-                              AllowDifferingSizes, TLI))
+                              AllowDifferingSizes, TLI,
+                              F->getParent()->getDataLayout()))
       return false;
 
     CallEmpty  = !nextRealType(CallSubTypes, CallPath);
diff --git a/lib/CodeGen/AntiDepBreaker.h b/lib/CodeGen/AntiDepBreaker.h
index a61a8efa4da..9f05200dcdf 100644
--- a/lib/CodeGen/AntiDepBreaker.h
+++ b/lib/CodeGen/AntiDepBreaker.h
@@ -27,7 +27,7 @@ namespace llvm {
 
 /// This class works in conjunction with the post-RA scheduler to rename
 /// registers to break register anti-dependencies (WAR hazards).
-class AntiDepBreaker {
+class LLVM_LIBRARY_VISIBILITY AntiDepBreaker {
 public:
   typedef std::vector<std::pair<MachineInstr *, MachineInstr *> > 
     DbgValueVector;
diff --git a/lib/CodeGen/AsmPrinter/ARMException.cpp b/lib/CodeGen/AsmPrinter/ARMException.cpp
index 9a16e15ecf7..462e5d19c8d 100644
--- a/lib/CodeGen/AsmPrinter/ARMException.cpp
+++ b/lib/CodeGen/AsmPrinter/ARMException.cpp
@@ -41,7 +41,7 @@ ARMException::ARMException(AsmPrinter *A) : DwarfCFIExceptionBase(A) {}
 ARMException::~ARMException() {}
 
 ARMTargetStreamer &ARMException::getTargetStreamer() {
-  MCTargetStreamer &TS = *Asm->OutStreamer.getTargetStreamer();
+  MCTargetStreamer &TS = *Asm->OutStreamer->getTargetStreamer();
   return static_cast<ARMTargetStreamer &>(TS);
 }
 
@@ -49,7 +49,7 @@ ARMTargetStreamer &ARMException::getTargetStreamer() {
 /// content.
 void ARMException::endModule() {
   if (shouldEmitCFI)
-    Asm->OutStreamer.EmitCFISections(false, true);
+    Asm->OutStreamer->EmitCFISections(false, true);
 }
 
 void ARMException::beginFunction(const MachineFunction *MF) {
@@ -61,7 +61,7 @@ void ARMException::beginFunction(const MachineFunction *MF) {
          "non-EH CFI not yet supported in prologue with EHABI lowering");
   if (MoveType == AsmPrinter::CFI_M_Debug) {
     shouldEmitCFI = true;
-    Asm->OutStreamer.EmitCFIStartProc(false);
+    Asm->OutStreamer->EmitCFIStartProc(false);
   }
 }
 
@@ -69,24 +69,32 @@ void ARMException::beginFunction(const MachineFunction *MF) {
 ///
 void ARMException::endFunction(const MachineFunction *MF) {
   ARMTargetStreamer &ATS = getTargetStreamer();
+  const Function *F = MF->getFunction();
+  const Function *Per = nullptr;
+  if (F->hasPersonalityFn())
+    Per = dyn_cast<Function>(F->getPersonalityFn()->stripPointerCasts());
+  assert(!MMI->getPersonality() || Per == MMI->getPersonality());
+  bool forceEmitPersonality =
+    F->hasPersonalityFn() && !isNoOpWithoutInvoke(classifyEHPersonality(Per)) &&
+    F->needsUnwindTableEntry();
+  bool shouldEmitPersonality = forceEmitPersonality ||
+    !MMI->getLandingPads().empty();
   if (!Asm->MF->getFunction()->needsUnwindTableEntry() &&
-      MMI->getLandingPads().empty())
+      !shouldEmitPersonality)
     ATS.emitCantUnwind();
-  else {
-    if (!MMI->getLandingPads().empty()) {
-      // Emit references to personality.
-      if (const Function *Personality = MMI->getPersonality()) {
-        MCSymbol *PerSym = Asm->getSymbol(Personality);
-        Asm->OutStreamer.EmitSymbolAttribute(PerSym, MCSA_Global);
-        ATS.emitPersonality(PerSym);
-      }
-
-      // Emit .handlerdata directive.
-      ATS.emitHandlerData();
-
-      // Emit actual exception table
-      emitExceptionTable();
+  else if (shouldEmitPersonality) {
+    // Emit references to personality.
+    if (Per) {
+      MCSymbol *PerSym = Asm->getSymbol(Per);
+      Asm->OutStreamer->EmitSymbolAttribute(PerSym, MCSA_Global);
+      ATS.emitPersonality(PerSym);
     }
+
+    // Emit .handlerdata directive.
+    ATS.emitHandlerData();
+
+    // Emit actual exception table
+    emitExceptionTable();
   }
 
   if (Asm->MAI->getExceptionHandlingType() == ExceptionHandling::ARM)
@@ -97,28 +105,26 @@ void ARMException::emitTypeInfos(unsigned TTypeEncoding) {
   const std::vector<const GlobalValue *> &TypeInfos = MMI->getTypeInfos();
   const std::vector<unsigned> &FilterIds = MMI->getFilterIds();
 
-  bool VerboseAsm = Asm->OutStreamer.isVerboseAsm();
+  bool VerboseAsm = Asm->OutStreamer->isVerboseAsm();
 
   int Entry = 0;
   // Emit the Catch TypeInfos.
   if (VerboseAsm && !TypeInfos.empty()) {
-    Asm->OutStreamer.AddComment(">> Catch TypeInfos <<");
-    Asm->OutStreamer.AddBlankLine();
+    Asm->OutStreamer->AddComment(">> Catch TypeInfos <<");
+    Asm->OutStreamer->AddBlankLine();
     Entry = TypeInfos.size();
   }
 
-  for (std::vector<const GlobalValue *>::const_reverse_iterator
-         I = TypeInfos.rbegin(), E = TypeInfos.rend(); I != E; ++I) {
-    const GlobalValue *GV = *I;
+  for (const GlobalValue *GV : reverse(TypeInfos)) {
     if (VerboseAsm)
-      Asm->OutStreamer.AddComment("TypeInfo " + Twine(Entry--));
+      Asm->OutStreamer->AddComment("TypeInfo " + Twine(Entry--));
     Asm->EmitTTypeReference(GV, TTypeEncoding);
   }
 
   // Emit the Exception Specifications.
   if (VerboseAsm && !FilterIds.empty()) {
-    Asm->OutStreamer.AddComment(">> Filter TypeInfos <<");
-    Asm->OutStreamer.AddBlankLine();
+    Asm->OutStreamer->AddComment(">> Filter TypeInfos <<");
+    Asm->OutStreamer->AddBlankLine();
     Entry = 0;
   }
   for (std::vector<unsigned>::const_iterator
@@ -127,7 +133,7 @@ void ARMException::emitTypeInfos(unsigned TTypeEncoding) {
     if (VerboseAsm) {
       --Entry;
       if (TypeID != 0)
-        Asm->OutStreamer.AddComment("FilterInfo " + Twine(Entry));
+        Asm->OutStreamer->AddComment("FilterInfo " + Twine(Entry));
     }
 
     Asm->EmitTTypeReference((TypeID == 0 ? nullptr : TypeInfos[TypeID - 1]),
diff --git a/lib/CodeGen/AsmPrinter/AddressPool.cpp b/lib/CodeGen/AsmPrinter/AddressPool.cpp
index 8dab5e59d63..8c6838394ac 100644
--- a/lib/CodeGen/AsmPrinter/AddressPool.cpp
+++ b/lib/CodeGen/AsmPrinter/AddressPool.cpp
@@ -24,12 +24,12 @@ unsigned AddressPool::getIndex(const MCSymbol *Sym, bool TLS) {
 }
 
 // Emit addresses into the section given.
-void AddressPool::emit(AsmPrinter &Asm, const MCSection *AddrSection) {
+void AddressPool::emit(AsmPrinter &Asm, MCSection *AddrSection) {
   if (Pool.empty())
     return;
 
   // Start the dwarf addr section.
-  Asm.OutStreamer.SwitchSection(AddrSection);
+  Asm.OutStreamer->SwitchSection(AddrSection);
 
   // Order the address pool entries by ID
   SmallVector<const MCExpr *, 64> Entries(Pool.size());
@@ -38,8 +38,8 @@ void AddressPool::emit(AsmPrinter &Asm, const MCSection *AddrSection) {
     Entries[I.second.Number] =
         I.second.TLS
             ? Asm.getObjFileLowering().getDebugThreadLocalSymbol(I.first)
-            : MCSymbolRefExpr::Create(I.first, Asm.OutContext);
+            : MCSymbolRefExpr::create(I.first, Asm.OutContext);
 
   for (const MCExpr *Entry : Entries)
-    Asm.OutStreamer.EmitValue(Entry, Asm.getDataLayout().getPointerSize());
+    Asm.OutStreamer->EmitValue(Entry, Asm.getDataLayout().getPointerSize());
 }
diff --git a/lib/CodeGen/AsmPrinter/AddressPool.h b/lib/CodeGen/AsmPrinter/AddressPool.h
index 802e050cdf6..211fc98c7f6 100644
--- a/lib/CodeGen/AsmPrinter/AddressPool.h
+++ b/lib/CodeGen/AsmPrinter/AddressPool.h
@@ -40,7 +40,7 @@ class AddressPool {
   /// label/symbol.
   unsigned getIndex(const MCSymbol *Sym, bool TLS = false);
 
-  void emit(AsmPrinter &Asm, const MCSection *AddrSection);
+  void emit(AsmPrinter &Asm, MCSection *AddrSection);
 
   bool isEmpty() { return Pool.empty(); }
 
diff --git a/lib/CodeGen/AsmPrinter/AsmPrinter.cpp b/lib/CodeGen/AsmPrinter/AsmPrinter.cpp
index e2eaca6b31b..bf1b628aff8 100644
--- a/lib/CodeGen/AsmPrinter/AsmPrinter.cpp
+++ b/lib/CodeGen/AsmPrinter/AsmPrinter.cpp
@@ -14,12 +14,11 @@
 #include "llvm/CodeGen/AsmPrinter.h"
 #include "DwarfDebug.h"
 #include "DwarfException.h"
-#include "Win64Exception.h"
+#include "WinException.h"
 #include "WinCodeViewLineTables.h"
 #include "llvm/ADT/SmallString.h"
 #include "llvm/ADT/Statistic.h"
 #include "llvm/Analysis/ConstantFolding.h"
-#include "llvm/Analysis/JumpInstrTableInfo.h"
 #include "llvm/CodeGen/Analysis.h"
 #include "llvm/CodeGen/GCMetadataPrinter.h"
 #include "llvm/CodeGen/MachineConstantPool.h"
@@ -40,7 +39,7 @@
 #include "llvm/MC/MCInst.h"
 #include "llvm/MC/MCSection.h"
 #include "llvm/MC/MCStreamer.h"
-#include "llvm/MC/MCSymbol.h"
+#include "llvm/MC/MCSymbolELF.h"
 #include "llvm/MC/MCValue.h"
 #include "llvm/Support/ErrorHandling.h"
 #include "llvm/Support/Format.h"
@@ -102,7 +101,7 @@ static unsigned getGVAlignmentLog2(const GlobalValue *GV, const DataLayout &DL,
 
 AsmPrinter::AsmPrinter(TargetMachine &tm, std::unique_ptr<MCStreamer> Streamer)
     : MachineFunctionPass(ID), TM(tm), MAI(tm.getMCAsmInfo()),
-      OutContext(Streamer->getContext()), OutStreamer(*Streamer.release()),
+      OutContext(Streamer->getContext()), OutStreamer(std::move(Streamer)),
       LastMI(nullptr), LastFn(0), Counter(~0U) {
   DD = nullptr;
   MMI = nullptr;
@@ -112,7 +111,7 @@ AsmPrinter::AsmPrinter(TargetMachine &tm, std::unique_ptr<MCStreamer> Streamer)
   CurrentFnBegin = nullptr;
   CurrentFnEnd = nullptr;
   GCMetadataPrinters = nullptr;
-  VerboseAsm = OutStreamer.isVerboseAsm();
+  VerboseAsm = OutStreamer->isVerboseAsm();
 }
 
 AsmPrinter::~AsmPrinter() {
@@ -124,8 +123,6 @@ AsmPrinter::~AsmPrinter() {
     delete &GCMap;
     GCMetadataPrinters = nullptr;
   }
-
-  delete &OutStreamer;
 }
 
 /// getFunctionNumber - Return a unique ID for the current function.
@@ -138,11 +135,14 @@ const TargetLoweringObjectFile &AsmPrinter::getObjFileLowering() const {
   return *TM.getObjFileLowering();
 }
 
-/// getDataLayout - Return information about data layout.
 const DataLayout &AsmPrinter::getDataLayout() const {
-  return *TM.getDataLayout();
+  return MMI->getModule()->getDataLayout();
 }
 
+// Do not use the cached DataLayout because some client use it without a Module
+// (llmv-dsymutil, llvm-dwarfdump).
+unsigned AsmPrinter::getPointerSize() const { return TM.getPointerSize(); }
+
 const MCSubtargetInfo &AsmPrinter::getSubtargetInfo() const {
   assert(MF && "getSubtargetInfo requires a valid MachineFunction!");
   return MF->getSubtarget<MCSubtargetInfo>();
@@ -153,12 +153,12 @@ void AsmPrinter::EmitToStreamer(MCStreamer &S, const MCInst &Inst) {
 }
 
 StringRef AsmPrinter::getTargetTriple() const {
-  return TM.getTargetTriple();
+  return TM.getTargetTriple().str();
 }
 
 /// getCurrentSection() - Return the current section we are emitting to.
 const MCSection *AsmPrinter::getCurrentSection() const {
-  return OutStreamer.getCurrentSection().first;
+  return OutStreamer->getCurrentSection().first;
 }
 
 
@@ -174,15 +174,14 @@ void AsmPrinter::getAnalysisUsage(AnalysisUsage &AU) const {
 
 bool AsmPrinter::doInitialization(Module &M) {
   MMI = getAnalysisIfAvailable<MachineModuleInfo>();
-  MMI->AnalyzeModule(M);
 
   // Initialize TargetLoweringObjectFile.
   const_cast<TargetLoweringObjectFile&>(getObjFileLowering())
     .Initialize(OutContext, TM);
 
-  OutStreamer.InitSections(false);
+  OutStreamer->InitSections(false);
 
-  Mang = new Mangler(TM.getDataLayout());
+  Mang = new Mangler();
 
   // Emit the version-min deplyment target directive if needed.
   //
@@ -198,9 +197,9 @@ bool AsmPrinter::doInitialization(Module &M) {
     TT.getOSVersion(Major, Minor, Update);
     // If there is a version specified, Major will be non-zero.
     if (Major)
-      OutStreamer.EmitVersionMin((TT.isMacOSX() ?
-                                  MCVM_OSXVersionMin : MCVM_IOSVersionMin),
-                                 Major, Minor, Update);
+      OutStreamer->EmitVersionMin((TT.isMacOSX() ?
+                                   MCVM_OSXVersionMin : MCVM_IOSVersionMin),
+                                  Major, Minor, Update);
   }
 
   // Allow the target to emit any magic that it wants at the start of the file.
@@ -210,7 +209,7 @@ bool AsmPrinter::doInitialization(Module &M) {
   // don't, this at least helps the user find where a global came from.
   if (MAI->hasSingleParameterDotFile()) {
     // .file "foo.c"
-    OutStreamer.EmitFileDirective(M.getModuleIdentifier());
+    OutStreamer->EmitFileDirective(M.getModuleIdentifier());
   }
 
   GCModuleInfo *MI = getAnalysisIfAvailable<GCModuleInfo>();
@@ -224,17 +223,18 @@ bool AsmPrinter::doInitialization(Module &M) {
     // We're at the module level. Construct MCSubtarget from the default CPU
     // and target triple.
     std::unique_ptr<MCSubtargetInfo> STI(TM.getTarget().createMCSubtargetInfo(
-        TM.getTargetTriple(), TM.getTargetCPU(), TM.getTargetFeatureString()));
-    OutStreamer.AddComment("Start of file scope inline assembly");
-    OutStreamer.AddBlankLine();
-    EmitInlineAsm(M.getModuleInlineAsm()+"\n", *STI);
-    OutStreamer.AddComment("End of file scope inline assembly");
-    OutStreamer.AddBlankLine();
+        TM.getTargetTriple().str(), TM.getTargetCPU(),
+        TM.getTargetFeatureString()));
+    OutStreamer->AddComment("Start of file scope inline assembly");
+    OutStreamer->AddBlankLine();
+    EmitInlineAsm(M.getModuleInlineAsm()+"\n", *STI, TM.Options.MCOptions);
+    OutStreamer->AddComment("End of file scope inline assembly");
+    OutStreamer->AddBlankLine();
   }
 
   if (MAI->doesSupportDebugInformation()) {
     bool skip_dwarf = false;
-    if (Triple(TM.getTargetTriple()).isKnownWindowsMSVCEnvironment()) {
+    if (TM.getTargetTriple().isKnownWindowsMSVCEnvironment()) {
       Handlers.push_back(HandlerInfo(new WinCodeViewLineTables(this),
                                      DbgTimerName,
                                      CodeViewLineTablesGroupName));
@@ -268,8 +268,11 @@ bool AsmPrinter::doInitialization(Module &M) {
   case ExceptionHandling::WinEH:
     switch (MAI->getWinEHEncodingType()) {
     default: llvm_unreachable("unsupported unwinding information encoding");
+    case WinEH::EncodingType::Invalid:
+      break;
+    case WinEH::EncodingType::X86:
     case WinEH::EncodingType::Itanium:
-      ES = new Win64Exception(this);
+      ES = new WinException(this);
       break;
     }
     break;
@@ -296,20 +299,20 @@ void AsmPrinter::EmitLinkage(const GlobalValue *GV, MCSymbol *GVSym) const {
   case GlobalValue::WeakODRLinkage:
     if (MAI->hasWeakDefDirective()) {
       // .globl _foo
-      OutStreamer.EmitSymbolAttribute(GVSym, MCSA_Global);
+      OutStreamer->EmitSymbolAttribute(GVSym, MCSA_Global);
 
       if (!canBeHidden(GV, *MAI))
         // .weak_definition _foo
-        OutStreamer.EmitSymbolAttribute(GVSym, MCSA_WeakDefinition);
+        OutStreamer->EmitSymbolAttribute(GVSym, MCSA_WeakDefinition);
       else
-        OutStreamer.EmitSymbolAttribute(GVSym, MCSA_WeakDefAutoPrivate);
+        OutStreamer->EmitSymbolAttribute(GVSym, MCSA_WeakDefAutoPrivate);
     } else if (MAI->hasLinkOnceDirective()) {
       // .globl _foo
-      OutStreamer.EmitSymbolAttribute(GVSym, MCSA_Global);
+      OutStreamer->EmitSymbolAttribute(GVSym, MCSA_Global);
       //NOTE: linkonce is handled by the section the symbol was assigned to.
     } else {
       // .weak _foo
-      OutStreamer.EmitSymbolAttribute(GVSym, MCSA_Weak);
+      OutStreamer->EmitSymbolAttribute(GVSym, MCSA_Weak);
     }
     return;
   case GlobalValue::AppendingLinkage:
@@ -318,7 +321,7 @@ void AsmPrinter::EmitLinkage(const GlobalValue *GV, MCSymbol *GVSym) const {
   case GlobalValue::ExternalLinkage:
     // If external or appending, declare as a global symbol.
     // .globl _foo
-    OutStreamer.EmitSymbolAttribute(GVSym, MCSA_Global);
+    OutStreamer->EmitSymbolAttribute(GVSym, MCSA_Global);
     return;
   case GlobalValue::PrivateLinkage:
   case GlobalValue::InternalLinkage:
@@ -340,8 +343,58 @@ MCSymbol *AsmPrinter::getSymbol(const GlobalValue *GV) const {
   return TM.getSymbol(GV, *Mang);
 }
 
+static MCSymbol *getOrCreateEmuTLSControlSym(MCSymbol *GVSym, MCContext &C) {
+  return C.getOrCreateSymbol(Twine("__emutls_v.") + GVSym->getName());
+}
+
+static MCSymbol *getOrCreateEmuTLSInitSym(MCSymbol *GVSym, MCContext &C) {
+  return C.getOrCreateSymbol(Twine("__emutls_t.") + GVSym->getName());
+}
+
+/// EmitEmulatedTLSControlVariable - Emit the control variable for an emulated TLS variable.
+void AsmPrinter::EmitEmulatedTLSControlVariable(const GlobalVariable *GV,
+                                                MCSymbol *EmittedSym,
+                                                bool AllZeroInitValue) {
+  // If there is init value, use .data.rel.local section;
+  // otherwise use the .data section.
+  MCSection *TLSVarSection = const_cast<MCSection*>(
+      (GV->hasInitializer() && !AllZeroInitValue)
+      ? getObjFileLowering().getDataRelLocalSection()
+      : getObjFileLowering().getDataSection());
+  OutStreamer->SwitchSection(TLSVarSection);
+  MCSymbol *GVSym = getSymbol(GV);
+  EmitLinkage(GV, EmittedSym);  // same linkage as GV
+  const DataLayout &DL = GV->getParent()->getDataLayout();
+  uint64_t Size = DL.getTypeAllocSize(GV->getType()->getElementType());
+  unsigned AlignLog = getGVAlignmentLog2(GV, DL);
+  unsigned WordSize = DL.getPointerSize();
+  unsigned Alignment = DL.getPointerABIAlignment();
+  EmitAlignment(Log2_32(Alignment));
+  OutStreamer->EmitLabel(EmittedSym);
+  OutStreamer->EmitIntValue(Size, WordSize);
+  OutStreamer->EmitIntValue((1 << AlignLog), WordSize);
+  OutStreamer->EmitIntValue(0, WordSize);
+  if (GV->hasInitializer() && !AllZeroInitValue) {
+    OutStreamer->EmitSymbolValue(
+        getOrCreateEmuTLSInitSym(GVSym, OutContext), WordSize);
+  } else
+    OutStreamer->EmitIntValue(0, WordSize);
+  if (MAI->hasDotTypeDotSizeDirective())
+    OutStreamer->emitELFSize(cast<MCSymbolELF>(EmittedSym),
+                             MCConstantExpr::create(4 * WordSize, OutContext));
+  OutStreamer->AddBlankLine();  // End of the __emutls_v.* variable.
+}
+
 /// EmitGlobalVariable - Emit the specified global variable to the .s file.
 void AsmPrinter::EmitGlobalVariable(const GlobalVariable *GV) {
+  bool IsEmuTLSVar =
+      GV->getThreadLocalMode() != llvm::GlobalVariable::NotThreadLocal &&
+      TM.Options.EmulatedTLS;
+  assert((!IsEmuTLSVar || getObjFileLowering().getDataRelLocalSection()) &&
+         "Need relocatable local section for emulated TLS variables");
+  assert(!(IsEmuTLSVar && GV->hasCommonLinkage()) &&
+         "No emulated TLS variables in the common section");
+
   if (GV->hasInitializer()) {
     // Check to see if this is a special global used by LLVM, if so, emit it.
     if (EmitSpecialLLVMGlobal(GV))
@@ -352,15 +405,22 @@ void AsmPrinter::EmitGlobalVariable(const GlobalVariable *GV) {
     if (GlobalGOTEquivs.count(getSymbol(GV)))
       return;
 
-    if (isVerbose()) {
-      GV->printAsOperand(OutStreamer.GetCommentOS(),
+    if (isVerbose() && !IsEmuTLSVar) {
+      // When printing the control variable __emutls_v.*,
+      // we don't need to print the original TLS variable name.
+      GV->printAsOperand(OutStreamer->GetCommentOS(),
                      /*PrintType=*/false, GV->getParent());
-      OutStreamer.GetCommentOS() << '\n';
+      OutStreamer->GetCommentOS() << '\n';
     }
   }
 
   MCSymbol *GVSym = getSymbol(GV);
-  EmitVisibility(GVSym, GV->getVisibility(), !GV->isDeclaration());
+  MCSymbol *EmittedSym = IsEmuTLSVar ?
+      getOrCreateEmuTLSControlSym(GVSym, OutContext) : GVSym;
+  // getOrCreateEmuTLSControlSym only creates the symbol with name and default attributes.
+  // GV's or GVSym's attributes will be used for the EmittedSym.
+
+  EmitVisibility(EmittedSym, GV->getVisibility(), !GV->isDeclaration());
 
   if (!GV->hasInitializer())   // External globals require no extra code.
     return;
@@ -371,17 +431,29 @@ void AsmPrinter::EmitGlobalVariable(const GlobalVariable *GV) {
                        "' is already defined");
 
   if (MAI->hasDotTypeDotSizeDirective())
-    OutStreamer.EmitSymbolAttribute(GVSym, MCSA_ELF_TypeObject);
+    OutStreamer->EmitSymbolAttribute(EmittedSym, MCSA_ELF_TypeObject);
 
   SectionKind GVKind = TargetLoweringObjectFile::getKindForGlobal(GV, TM);
 
-  const DataLayout *DL = TM.getDataLayout();
-  uint64_t Size = DL->getTypeAllocSize(GV->getType()->getElementType());
+  const DataLayout &DL = GV->getParent()->getDataLayout();
+  uint64_t Size = DL.getTypeAllocSize(GV->getType()->getElementType());
 
   // If the alignment is specified, we *must* obey it.  Overaligning a global
   // with a specified alignment is a prompt way to break globals emitted to
   // sections and expected to be contiguous (e.g. ObjC metadata).
-  unsigned AlignLog = getGVAlignmentLog2(GV, *DL);
+  unsigned AlignLog = getGVAlignmentLog2(GV, DL);
+
+  bool AllZeroInitValue = false;
+  const Constant *InitValue = GV->getInitializer();
+  if (isa<ConstantAggregateZero>(InitValue))
+    AllZeroInitValue = true;
+  else {
+    const ConstantInt *InitIntValue = dyn_cast<ConstantInt>(InitValue);
+    if (InitIntValue && InitIntValue->isZero())
+      AllZeroInitValue = true;
+  }
+  if (IsEmuTLSVar)
+    EmitEmulatedTLSControlVariable(GV, EmittedSym, AllZeroInitValue);
 
   for (const HandlerInfo &HI : Handlers) {
     NamedRegionTimer T(HI.TimerName, HI.TimerGroupName, TimePassesIsEnabled);
@@ -390,6 +462,8 @@ void AsmPrinter::EmitGlobalVariable(const GlobalVariable *GV) {
 
   // Handle common and BSS local symbols (.lcomm).
   if (GVKind.isCommon() || GVKind.isBSSLocal()) {
+    assert(!(IsEmuTLSVar && GVKind.isCommon()) &&
+           "No emulated TLS variables in the common section");
     if (Size == 0) Size = 1;   // .comm Foo, 0 is undefined, avoid it.
     unsigned Align = 1 << AlignLog;
 
@@ -399,16 +473,16 @@ void AsmPrinter::EmitGlobalVariable(const GlobalVariable *GV) {
         Align = 0;
 
       // .comm _foo, 42, 4
-      OutStreamer.EmitCommonSymbol(GVSym, Size, Align);
+      OutStreamer->EmitCommonSymbol(GVSym, Size, Align);
       return;
     }
 
     // Handle local BSS symbols.
     if (MAI->hasMachoZeroFillDirective()) {
-      const MCSection *TheSection =
-        getObjFileLowering().SectionForGlobal(GV, GVKind, *Mang, TM);
+      MCSection *TheSection =
+          getObjFileLowering().SectionForGlobal(GV, GVKind, *Mang, TM);
       // .zerofill __DATA, __bss, _foo, 400, 5
-      OutStreamer.EmitZerofill(TheSection, GVSym, Size, Align);
+      OutStreamer->EmitZerofill(TheSection, GVSym, Size, Align);
       return;
     }
 
@@ -420,7 +494,7 @@ void AsmPrinter::EmitGlobalVariable(const GlobalVariable *GV) {
     // Prefer to simply fall back to .local / .comm in this case.
     if (MAI->getLCOMMDirectiveAlignmentType() != LCOMM::NoAlignment) {
       // .lcomm _foo, 42
-      OutStreamer.EmitLocalCommonSymbol(GVSym, Size, Align);
+      OutStreamer->EmitLocalCommonSymbol(GVSym, Size, Align);
       return;
     }
 
@@ -428,24 +502,33 @@ void AsmPrinter::EmitGlobalVariable(const GlobalVariable *GV) {
       Align = 0;
 
     // .local _foo
-    OutStreamer.EmitSymbolAttribute(GVSym, MCSA_Local);
+    OutStreamer->EmitSymbolAttribute(GVSym, MCSA_Local);
     // .comm _foo, 42, 4
-    OutStreamer.EmitCommonSymbol(GVSym, Size, Align);
+    OutStreamer->EmitCommonSymbol(GVSym, Size, Align);
     return;
   }
 
-  const MCSection *TheSection =
-    getObjFileLowering().SectionForGlobal(GV, GVKind, *Mang, TM);
+  if (IsEmuTLSVar && AllZeroInitValue)
+    return;  // No need of initialization values.
+
+  MCSymbol *EmittedInitSym = IsEmuTLSVar ?
+      getOrCreateEmuTLSInitSym(GVSym, OutContext) : GVSym;
+  // getOrCreateEmuTLSInitSym only creates the symbol with name and default attributes.
+  // GV's or GVSym's attributes will be used for the EmittedInitSym.
+
+  MCSection *TheSection = IsEmuTLSVar ?
+      getObjFileLowering().getReadOnlySection() :
+      getObjFileLowering().SectionForGlobal(GV, GVKind, *Mang, TM);
 
   // Handle the zerofill directive on darwin, which is a special form of BSS
   // emission.
-  if (GVKind.isBSSExtern() && MAI->hasMachoZeroFillDirective()) {
+  if (GVKind.isBSSExtern() && MAI->hasMachoZeroFillDirective() && !IsEmuTLSVar) {
     if (Size == 0) Size = 1;  // zerofill of 0 bytes is undefined.
 
     // .globl _foo
-    OutStreamer.EmitSymbolAttribute(GVSym, MCSA_Global);
+    OutStreamer->EmitSymbolAttribute(GVSym, MCSA_Global);
     // .zerofill __DATA, __common, _foo, 400, 5
-    OutStreamer.EmitZerofill(TheSection, GVSym, Size, 1 << AlignLog);
+    OutStreamer->EmitZerofill(TheSection, GVSym, Size, 1 << AlignLog);
     return;
   }
 
@@ -459,62 +542,65 @@ void AsmPrinter::EmitGlobalVariable(const GlobalVariable *GV) {
   // TLOF class.  This will also make it more obvious that stuff like
   // MCStreamer::EmitTBSSSymbol is macho specific and only called from macho
   // specific code.
-  if (GVKind.isThreadLocal() && MAI->hasMachoTBSSDirective()) {
+  if (GVKind.isThreadLocal() && MAI->hasMachoTBSSDirective() && !IsEmuTLSVar) {
     // Emit the .tbss symbol
     MCSymbol *MangSym =
-      OutContext.GetOrCreateSymbol(GVSym->getName() + Twine("$tlv$init"));
+      OutContext.getOrCreateSymbol(GVSym->getName() + Twine("$tlv$init"));
 
     if (GVKind.isThreadBSS()) {
       TheSection = getObjFileLowering().getTLSBSSSection();
-      OutStreamer.EmitTBSSSymbol(TheSection, MangSym, Size, 1 << AlignLog);
+      OutStreamer->EmitTBSSSymbol(TheSection, MangSym, Size, 1 << AlignLog);
     } else if (GVKind.isThreadData()) {
-      OutStreamer.SwitchSection(TheSection);
+      OutStreamer->SwitchSection(TheSection);
 
       EmitAlignment(AlignLog, GV);
-      OutStreamer.EmitLabel(MangSym);
+      OutStreamer->EmitLabel(MangSym);
 
-      EmitGlobalConstant(GV->getInitializer());
+      EmitGlobalConstant(GV->getParent()->getDataLayout(),
+                         GV->getInitializer());
     }
 
-    OutStreamer.AddBlankLine();
+    OutStreamer->AddBlankLine();
 
     // Emit the variable struct for the runtime.
-    const MCSection *TLVSect
-      = getObjFileLowering().getTLSExtraDataSection();
+    MCSection *TLVSect = getObjFileLowering().getTLSExtraDataSection();
 
-    OutStreamer.SwitchSection(TLVSect);
+    OutStreamer->SwitchSection(TLVSect);
     // Emit the linkage here.
     EmitLinkage(GV, GVSym);
-    OutStreamer.EmitLabel(GVSym);
+    OutStreamer->EmitLabel(GVSym);
 
     // Three pointers in size:
     //   - __tlv_bootstrap - used to make sure support exists
     //   - spare pointer, used when mapped by the runtime
     //   - pointer to mangled symbol above with initializer
-    unsigned PtrSize = DL->getPointerTypeSize(GV->getType());
-    OutStreamer.EmitSymbolValue(GetExternalSymbolSymbol("_tlv_bootstrap"),
+    unsigned PtrSize = DL.getPointerTypeSize(GV->getType());
+    OutStreamer->EmitSymbolValue(GetExternalSymbolSymbol("_tlv_bootstrap"),
                                 PtrSize);
-    OutStreamer.EmitIntValue(0, PtrSize);
-    OutStreamer.EmitSymbolValue(MangSym, PtrSize);
+    OutStreamer->EmitIntValue(0, PtrSize);
+    OutStreamer->EmitSymbolValue(MangSym, PtrSize);
 
-    OutStreamer.AddBlankLine();
+    OutStreamer->AddBlankLine();
     return;
   }
 
-  OutStreamer.SwitchSection(TheSection);
+  OutStreamer->SwitchSection(TheSection);
 
-  EmitLinkage(GV, GVSym);
+  // emutls_t.* symbols are only used in the current compilation unit.
+  if (!IsEmuTLSVar)
+    EmitLinkage(GV, EmittedInitSym);
   EmitAlignment(AlignLog, GV);
 
-  OutStreamer.EmitLabel(GVSym);
+  OutStreamer->EmitLabel(EmittedInitSym);
 
-  EmitGlobalConstant(GV->getInitializer());
+  EmitGlobalConstant(GV->getParent()->getDataLayout(), GV->getInitializer());
 
   if (MAI->hasDotTypeDotSizeDirective())
     // .size foo, 42
-    OutStreamer.EmitELFSize(GVSym, MCConstantExpr::Create(Size, OutContext));
+    OutStreamer->emitELFSize(cast<MCSymbolELF>(EmittedInitSym),
+                             MCConstantExpr::create(Size, OutContext));
 
-  OutStreamer.AddBlankLine();
+  OutStreamer->AddBlankLine();
 }
 
 /// EmitFunctionHeader - This method emits the header for the current
@@ -526,7 +612,7 @@ void AsmPrinter::EmitFunctionHeader() {
   // Print the 'header' of function.
   const Function *F = MF->getFunction();
 
-  OutStreamer.SwitchSection(
+  OutStreamer->SwitchSection(
       getObjFileLowering().SectionForGlobal(F, *Mang, TM));
   EmitVisibility(CurrentFnSym, F->getVisibility());
 
@@ -535,17 +621,17 @@ void AsmPrinter::EmitFunctionHeader() {
     EmitAlignment(MF->getAlignment(), F);
 
   if (MAI->hasDotTypeDotSizeDirective())
-    OutStreamer.EmitSymbolAttribute(CurrentFnSym, MCSA_ELF_TypeFunction);
+    OutStreamer->EmitSymbolAttribute(CurrentFnSym, MCSA_ELF_TypeFunction);
 
   if (isVerbose()) {
-    F->printAsOperand(OutStreamer.GetCommentOS(),
+    F->printAsOperand(OutStreamer->GetCommentOS(),
                    /*PrintType=*/false, F->getParent());
-    OutStreamer.GetCommentOS() << '\n';
+    OutStreamer->GetCommentOS() << '\n';
   }
 
   // Emit the prefix data.
   if (F->hasPrefixData())
-    EmitGlobalConstant(F->getPrefixData());
+    EmitGlobalConstant(F->getParent()->getDataLayout(), F->getPrefixData());
 
   // Emit the CurrentFnSym.  This is a virtual function to allow targets to
   // do their wild and crazy things as required.
@@ -557,18 +643,18 @@ void AsmPrinter::EmitFunctionHeader() {
   std::vector<MCSymbol*> DeadBlockSyms;
   MMI->takeDeletedSymbolsForFunction(F, DeadBlockSyms);
   for (unsigned i = 0, e = DeadBlockSyms.size(); i != e; ++i) {
-    OutStreamer.AddComment("Address taken block that was later removed");
-    OutStreamer.EmitLabel(DeadBlockSyms[i]);
+    OutStreamer->AddComment("Address taken block that was later removed");
+    OutStreamer->EmitLabel(DeadBlockSyms[i]);
   }
 
   if (CurrentFnBegin) {
     if (MAI->useAssignmentForEHBegin()) {
-      MCSymbol *CurPos = OutContext.CreateTempSymbol();
-      OutStreamer.EmitLabel(CurPos);
-      OutStreamer.EmitAssignment(CurrentFnBegin,
-                                 MCSymbolRefExpr::Create(CurPos, OutContext));
+      MCSymbol *CurPos = OutContext.createTempSymbol();
+      OutStreamer->EmitLabel(CurPos);
+      OutStreamer->EmitAssignment(CurrentFnBegin,
+                                 MCSymbolRefExpr::create(CurPos, OutContext));
     } else {
-      OutStreamer.EmitLabel(CurrentFnBegin);
+      OutStreamer->EmitLabel(CurrentFnBegin);
     }
   }
 
@@ -580,7 +666,7 @@ void AsmPrinter::EmitFunctionHeader() {
 
   // Emit the prologue data.
   if (F->hasPrologueData())
-    EmitGlobalConstant(F->getPrologueData());
+    EmitGlobalConstant(F->getParent()->getDataLayout(), F->getPrologueData());
 }
 
 /// EmitFunctionEntryLabel - Emit the label that is the entrypoint for the
@@ -597,7 +683,7 @@ void AsmPrinter::EmitFunctionEntryLabel() {
     report_fatal_error("'" + Twine(CurrentFnSym->getName()) +
                        "' label emitted multiple times to assembly file");
 
-  return OutStreamer.EmitLabel(CurrentFnSym);
+  return OutStreamer->EmitLabel(CurrentFnSym);
 }
 
 /// emitComments - Pretty-print comments for instructions.
@@ -640,9 +726,9 @@ static void emitComments(const MachineInstr &MI, raw_ostream &CommentOS) {
 /// that is an implicit def.
 void AsmPrinter::emitImplicitDef(const MachineInstr *MI) const {
   unsigned RegNo = MI->getOperand(0).getReg();
-  OutStreamer.AddComment(Twine("implicit-def: ") +
-                         MMI->getContext().getRegisterInfo()->getName(RegNo));
-  OutStreamer.AddBlankLine();
+  OutStreamer->AddComment(Twine("implicit-def: ") +
+                          MMI->getContext().getRegisterInfo()->getName(RegNo));
+  OutStreamer->AddBlankLine();
 }
 
 static void emitKill(const MachineInstr *MI, AsmPrinter &AP) {
@@ -654,8 +740,8 @@ static void emitKill(const MachineInstr *MI, AsmPrinter &AP) {
     Str += AP.MMI->getContext().getRegisterInfo()->getName(Op.getReg());
     Str += (Op.isDef() ? "<def>" : "<kill>");
   }
-  AP.OutStreamer.AddComment(Str);
-  AP.OutStreamer.AddBlankLine();
+  AP.OutStreamer->AddComment(Str);
+  AP.OutStreamer->AddBlankLine();
 }
 
 /// emitDebugValueComment - This method handles the target-independent form
@@ -670,15 +756,15 @@ static bool emitDebugValueComment(const MachineInstr *MI, AsmPrinter &AP) {
   raw_svector_ostream OS(Str);
   OS << "DEBUG_VALUE: ";
 
-  DIVariable V = MI->getDebugVariable();
-  if (auto *SP = dyn_cast<MDSubprogram>(V->getScope())) {
+  const DILocalVariable *V = MI->getDebugVariable();
+  if (auto *SP = dyn_cast<DISubprogram>(V->getScope())) {
     StringRef Name = SP->getDisplayName();
     if (!Name.empty())
       OS << Name << ":";
   }
   OS << V->getName();
 
-  DIExpression Expr = MI->getDebugExpression();
+  const DIExpression *Expr = MI->getDebugExpression();
   if (Expr->isBitPiece())
     OS << " [bit_piece offset=" << Expr->getBitPieceOffset()
        << " size=" << Expr->getBitPieceSize() << "]";
@@ -722,7 +808,7 @@ static bool emitDebugValueComment(const MachineInstr *MI, AsmPrinter &AP) {
       // Suppress offset, it is not meaningful here.
       OS << "undef";
       // NOTE: Want this comment at start of line, don't emit with AddComment.
-      AP.OutStreamer.emitRawComment(OS.str());
+      AP.OutStreamer->emitRawComment(OS.str());
       return true;
     }
     if (Deref)
@@ -734,7 +820,7 @@ static bool emitDebugValueComment(const MachineInstr *MI, AsmPrinter &AP) {
     OS << '+' << Offset << ']';
 
   // NOTE: Want this comment at start of line, don't emit with AddComment.
-  AP.OutStreamer.emitRawComment(OS.str());
+  AP.OutStreamer->emitRawComment(OS.str());
   return true;
 }
 
@@ -775,8 +861,8 @@ void AsmPrinter::emitFrameAlloc(const MachineInstr &MI) {
   int FrameOffset = MI.getOperand(1).getImm();
 
   // Emit a symbol assignment.
-  OutStreamer.EmitAssignment(FrameAllocSym,
-                             MCConstantExpr::Create(FrameOffset, OutContext));
+  OutStreamer->EmitAssignment(FrameAllocSym,
+                             MCConstantExpr::create(FrameOffset, OutContext));
 }
 
 /// EmitFunctionBody - This method emits the body and trailer for a
@@ -812,26 +898,21 @@ void AsmPrinter::EmitFunctionBody() {
       }
 
       if (isVerbose())
-        emitComments(MI, OutStreamer.GetCommentOS());
+        emitComments(MI, OutStreamer->GetCommentOS());
 
       switch (MI.getOpcode()) {
       case TargetOpcode::CFI_INSTRUCTION:
         emitCFIInstruction(MI);
         break;
 
-      case TargetOpcode::FRAME_ALLOC:
+      case TargetOpcode::LOCAL_ESCAPE:
         emitFrameAlloc(MI);
         break;
 
       case TargetOpcode::EH_LABEL:
-      case TargetOpcode::GC_LABEL: {
-        // @LOCALMOD-START
-        if (unsigned LabelAlign = GetTargetLabelAlign(&MI))
-          EmitAlignment(LabelAlign);
-        // @LOCALMOD-END
-        OutStreamer.EmitLabel(MI.getOperand(0).getMCSymbol());
+      case TargetOpcode::GC_LABEL:
+        OutStreamer->EmitLabel(MI.getOperand(0).getMCSymbol());
         break;
-      }
       case TargetOpcode::INLINEASM:
         EmitInlineAsm(&MI);
         break;
@@ -870,12 +951,12 @@ void AsmPrinter::EmitFunctionBody() {
   if ((MAI->hasSubsectionsViaSymbols() && !HasAnyRealCode)) {
     MCInst Noop;
     MF->getSubtarget().getInstrInfo()->getNoopForMachoTarget(Noop);
-    OutStreamer.AddComment("avoids zero-length function");
+    OutStreamer->AddComment("avoids zero-length function");
 
     // Targets can opt-out of emitting the noop here by leaving the opcode
     // unspecified.
     if (Noop.getOpcode())
-      OutStreamer.EmitInstruction(Noop, getSubtargetInfo());
+      OutStreamer->EmitInstruction(Noop, getSubtargetInfo());
   }
 
   const Function *F = MF->getFunction();
@@ -885,8 +966,8 @@ void AsmPrinter::EmitFunctionBody() {
     MCSymbol *Sym = GetBlockAddressSymbol(&BB);
     if (Sym->isDefined())
       continue;
-    OutStreamer.AddComment("Address of block that was removed by CodeGen");
-    OutStreamer.EmitLabel(Sym);
+    OutStreamer->AddComment("Address of block that was removed by CodeGen");
+    OutStreamer->EmitLabel(Sym);
   }
 
   // Emit target-specific gunk after the function body.
@@ -896,7 +977,7 @@ void AsmPrinter::EmitFunctionBody() {
       MAI->hasDotTypeDotSizeDirective()) {
     // Create a symbol for the end of function.
     CurrentFnEnd = createTempSymbol("func_end");
-    OutStreamer.EmitLabel(CurrentFnEnd);
+    OutStreamer->EmitLabel(CurrentFnEnd);
   }
 
   // If the target wants a .size directive for the size of the function, emit
@@ -904,12 +985,11 @@ void AsmPrinter::EmitFunctionBody() {
   if (MAI->hasDotTypeDotSizeDirective()) {
     // We can get the size as difference between the function label and the
     // temp label.
-    const MCExpr *SizeExp =
-      MCBinaryExpr::CreateSub(MCSymbolRefExpr::Create(CurrentFnEnd, OutContext),
-                              MCSymbolRefExpr::Create(CurrentFnSymForSize,
-                                                      OutContext),
-                              OutContext);
-    OutStreamer.EmitELFSize(CurrentFnSym, SizeExp);
+    const MCExpr *SizeExp = MCBinaryExpr::createSub(
+        MCSymbolRefExpr::create(CurrentFnEnd, OutContext),
+        MCSymbolRefExpr::create(CurrentFnSymForSize, OutContext), OutContext);
+    if (auto Sym = dyn_cast<MCSymbolELF>(CurrentFnSym))
+      OutStreamer->emitELFSize(Sym, SizeExp);
   }
 
   for (const HandlerInfo &HI : Handlers) {
@@ -927,7 +1007,7 @@ void AsmPrinter::EmitFunctionBody() {
   }
   MMI->EndFunction();
 
-  OutStreamer.AddBlankLine();
+  OutStreamer->AddBlankLine();
 }
 
 /// \brief Compute the number of Global Variables that uses a Constant.
@@ -1029,7 +1109,7 @@ bool AsmPrinter::doFinalization(Module &M) {
 
   // Emit visibility info for declarations
   for (const Function &F : M) {
-    if (!F.isDeclaration())
+    if (!F.isDeclarationForLinker())
       continue;
     GlobalValue::VisibilityTypes V = F.getVisibility();
     if (V == GlobalValue::DefaultVisibility)
@@ -1045,28 +1125,27 @@ bool AsmPrinter::doFinalization(Module &M) {
   SmallVector<Module::ModuleFlagEntry, 8> ModuleFlags;
   M.getModuleFlagsMetadata(ModuleFlags);
   if (!ModuleFlags.empty())
-    TLOF.emitModuleFlags(OutStreamer, ModuleFlags, *Mang, TM);
+    TLOF.emitModuleFlags(*OutStreamer, ModuleFlags, *Mang, TM);
 
-  Triple TT(TM.getTargetTriple());
-  if (TT.isOSBinFormatELF()) {
+  if (TM.getTargetTriple().isOSBinFormatELF()) {
     MachineModuleInfoELF &MMIELF = MMI->getObjFileInfo<MachineModuleInfoELF>();
 
     // Output stubs for external and common global variables.
     MachineModuleInfoELF::SymbolListTy Stubs = MMIELF.GetGVStubList();
     if (!Stubs.empty()) {
-      OutStreamer.SwitchSection(TLOF.getDataRelSection());
-      const DataLayout *DL = TM.getDataLayout();
+      OutStreamer->SwitchSection(TLOF.getDataRelSection());
+      const DataLayout &DL = M.getDataLayout();
 
       for (const auto &Stub : Stubs) {
-        OutStreamer.EmitLabel(Stub.first);
-        OutStreamer.EmitSymbolValue(Stub.second.getPointer(),
-                                    DL->getPointerSize());
+        OutStreamer->EmitLabel(Stub.first);
+        OutStreamer->EmitSymbolValue(Stub.second.getPointer(),
+                                     DL.getPointerSize());
       }
     }
   }
 
   // Make sure we wrote out everything we need.
-  OutStreamer.Flush();
+  OutStreamer->Flush();
 
   // Finalize debug and EH information.
   for (const HandlerInfo &HI : Handlers) {
@@ -1089,31 +1168,41 @@ bool AsmPrinter::doFinalization(Module &M) {
     for (const auto &G : M.globals()) {
       if (!G.hasExternalWeakLinkage())
         continue;
-      OutStreamer.EmitSymbolAttribute(getSymbol(&G), MCSA_WeakReference);
+      OutStreamer->EmitSymbolAttribute(getSymbol(&G), MCSA_WeakReference);
     }
 
     for (const auto &F : M) {
       if (!F.hasExternalWeakLinkage())
         continue;
-      OutStreamer.EmitSymbolAttribute(getSymbol(&F), MCSA_WeakReference);
+      OutStreamer->EmitSymbolAttribute(getSymbol(&F), MCSA_WeakReference);
     }
   }
 
-  OutStreamer.AddBlankLine();
+  OutStreamer->AddBlankLine();
   for (const auto &Alias : M.aliases()) {
     MCSymbol *Name = getSymbol(&Alias);
 
     if (Alias.hasExternalLinkage() || !MAI->getWeakRefDirective())
-      OutStreamer.EmitSymbolAttribute(Name, MCSA_Global);
+      OutStreamer->EmitSymbolAttribute(Name, MCSA_Global);
     else if (Alias.hasWeakLinkage() || Alias.hasLinkOnceLinkage())
-      OutStreamer.EmitSymbolAttribute(Name, MCSA_WeakReference);
+      OutStreamer->EmitSymbolAttribute(Name, MCSA_WeakReference);
     else
       assert(Alias.hasLocalLinkage() && "Invalid alias linkage");
 
     EmitVisibility(Name, Alias.getVisibility());
 
     // Emit the directives as assignments aka .set:
-    OutStreamer.EmitAssignment(Name, lowerConstant(Alias.getAliasee()));
+    OutStreamer->EmitAssignment(Name, lowerConstant(Alias.getAliasee()));
+
+    // Set the size of the alias symbol if we can, as otherwise the alias gets
+    // the size of the aliasee which may not be correct e.g. if the alias is of
+    // a member of a struct.
+    if (MAI->hasDotTypeDotSizeDirective() && Alias.getValueType()->isSized()) {
+      const DataLayout &DL = M.getDataLayout();
+      uint64_t Size = DL.getTypeAllocSize(Alias.getValueType());
+      OutStreamer->emitELFSize(cast<MCSymbolELF>(Name),
+                               MCConstantExpr::create(Size, OutContext));
+    }
   }
 
   GCModuleInfo *MI = getAnalysisIfAvailable<GCModuleInfo>();
@@ -1127,26 +1216,26 @@ bool AsmPrinter::doFinalization(Module &M) {
 
   // Emit __morestack address if needed for indirect calls.
   if (MMI->usesMorestackAddr()) {
-    const MCSection *ReadOnlySection =
-        getObjFileLowering().getSectionForConstant(SectionKind::getReadOnly(),
-                                                   /*C=*/nullptr);
-    OutStreamer.SwitchSection(ReadOnlySection);
+    MCSection *ReadOnlySection = getObjFileLowering().getSectionForConstant(
+        getDataLayout(), SectionKind::getReadOnly(),
+        /*C=*/nullptr);
+    OutStreamer->SwitchSection(ReadOnlySection);
 
     MCSymbol *AddrSymbol =
-        OutContext.GetOrCreateSymbol(StringRef("__morestack_addr"));
-    OutStreamer.EmitLabel(AddrSymbol);
+        OutContext.getOrCreateSymbol(StringRef("__morestack_addr"));
+    OutStreamer->EmitLabel(AddrSymbol);
 
-    unsigned PtrSize = TM.getDataLayout()->getPointerSize(0);
-    OutStreamer.EmitSymbolValue(GetExternalSymbolSymbol("__morestack"),
-                                PtrSize);
+    unsigned PtrSize = M.getDataLayout().getPointerSize(0);
+    OutStreamer->EmitSymbolValue(GetExternalSymbolSymbol("__morestack"),
+                                 PtrSize);
   }
 
   // If we don't have any trampolines, then we don't require stack memory
   // to be executable. Some targets have a directive to declare this.
   Function *InitTrampolineIntrinsic = M.getFunction("llvm.init.trampoline");
   if (!InitTrampolineIntrinsic || InitTrampolineIntrinsic->use_empty())
-    if (const MCSection *S = MAI->getNonexecutableStackSection(OutContext))
-      OutStreamer.SwitchSection(S);
+    if (MCSection *S = MAI->getNonexecutableStackSection(OutContext))
+      OutStreamer->SwitchSection(S);
 
   // Allow the target to emit any magic that it wants at the end of the file,
   // after everything else has gone out.
@@ -1155,8 +1244,8 @@ bool AsmPrinter::doFinalization(Module &M) {
   delete Mang; Mang = nullptr;
   MMI = nullptr;
 
-  OutStreamer.Finish();
-  OutStreamer.reset();
+  OutStreamer->Finish();
+  OutStreamer->reset();
 
   return false;
 }
@@ -1187,12 +1276,12 @@ void AsmPrinter::SetupMachineFunction(MachineFunction &MF) {
 }
 
 namespace {
-  // SectionCPs - Keep track the alignment, constpool entries per Section.
+// Keep track the alignment, constpool entries per Section.
   struct SectionCPs {
-    const MCSection *S;
+    MCSection *S;
     unsigned Alignment;
     SmallVector<unsigned, 4> CPEs;
-    SectionCPs(const MCSection *s, unsigned a) : S(s), Alignment(a) {}
+    SectionCPs(MCSection *s, unsigned a) : S(s), Alignment(a) {}
   };
 }
 
@@ -1213,14 +1302,14 @@ void AsmPrinter::EmitConstantPool() {
     const MachineConstantPoolEntry &CPE = CP[i];
     unsigned Align = CPE.getAlignment();
 
-    SectionKind Kind =
-        CPE.getSectionKind(TM.getDataLayout());
+    SectionKind Kind = CPE.getSectionKind(&getDataLayout());
 
     const Constant *C = nullptr;
     if (!CPE.isMachineConstantPoolEntry())
       C = CPE.Val.ConstVal;
 
-    const MCSection *S = getObjFileLowering().getSectionForConstant(Kind, C);
+    MCSection *S =
+        getObjFileLowering().getSectionForConstant(getDataLayout(), Kind, C);
 
     // The number of sections are small, just do a linear search from the
     // last section to the first.
@@ -1253,7 +1342,7 @@ void AsmPrinter::EmitConstantPool() {
         continue;
 
       if (CurSection != CPSections[i].S) {
-        OutStreamer.SwitchSection(CPSections[i].S);
+        OutStreamer->SwitchSection(CPSections[i].S);
         EmitAlignment(Log2_32(CPSections[i].Alignment));
         CurSection = CPSections[i].S;
         Offset = 0;
@@ -1264,17 +1353,16 @@ void AsmPrinter::EmitConstantPool() {
       // Emit inter-object padding for alignment.
       unsigned AlignMask = CPE.getAlignment() - 1;
       unsigned NewOffset = (Offset + AlignMask) & ~AlignMask;
-      OutStreamer.EmitZeros(NewOffset - Offset);
+      OutStreamer->EmitZeros(NewOffset - Offset);
 
       Type *Ty = CPE.getType();
-      Offset = NewOffset +
-               TM.getDataLayout()->getTypeAllocSize(Ty);
+      Offset = NewOffset + getDataLayout().getTypeAllocSize(Ty);
 
-      OutStreamer.EmitLabel(Sym);
+      OutStreamer->EmitLabel(Sym);
       if (CPE.isMachineConstantPoolEntry())
         EmitMachineConstantPoolValue(CPE.Val.MachineCPVal);
       else
-        EmitGlobalConstant(CPE.Val.ConstVal);
+        EmitGlobalConstant(getDataLayout(), CPE.Val.ConstVal);
     }
   }
 }
@@ -1283,7 +1371,7 @@ void AsmPrinter::EmitConstantPool() {
 /// by the current function to the current output stream.
 ///
 void AsmPrinter::EmitJumpTableInfo() {
-  const DataLayout *DL = MF->getTarget().getDataLayout();
+  const DataLayout &DL = MF->getDataLayout();
   const MachineJumpTableInfo *MJTI = MF->getJumpTableInfo();
   if (!MJTI) return;
   if (MJTI->getEntryKind() == MachineJumpTableInfo::EK_Inline) return;
@@ -1299,18 +1387,16 @@ void AsmPrinter::EmitJumpTableInfo() {
       *F);
   if (JTInDiffSection) {
     // Drop it in the readonly section.
-    const MCSection *ReadOnlySection =
-        TLOF.getSectionForJumpTable(*F, *Mang, TM);
-    OutStreamer.SwitchSection(ReadOnlySection);
+    MCSection *ReadOnlySection = TLOF.getSectionForJumpTable(*F, *Mang, TM);
+    OutStreamer->SwitchSection(ReadOnlySection);
   }
 
-  EmitAlignment(Log2_32(
-      MJTI->getEntryAlignment(*TM.getDataLayout())));
+  EmitAlignment(Log2_32(MJTI->getEntryAlignment(DL)));
 
   // Jump tables in code sections are marked with a data_region directive
   // where that's supported.
   if (!JTInDiffSection)
-    OutStreamer.EmitDataRegion(MCDR_DataRegionJT32);
+    OutStreamer->EmitDataRegion(MCDR_DataRegionJT32);
 
   for (unsigned JTI = 0, e = JT.size(); JTI != e; ++JTI) {
     const std::vector<MachineBasicBlock*> &JTBBs = JT[JTI].MBBs;
@@ -1321,8 +1407,7 @@ void AsmPrinter::EmitJumpTableInfo() {
     // For the EK_LabelDifference32 entry, if using .set avoids a relocation,
     /// emit a .set directive for each unique entry.
     if (MJTI->getEntryKind() == MachineJumpTableInfo::EK_LabelDifference32 &&
-        MAI->doesSetDirectiveSuppressesReloc() &&
-        !UseReadOnlyJumpTables()) { // @LOCALMOD
+        MAI->doesSetDirectiveSuppressesReloc()) {
       SmallPtrSet<const MachineBasicBlock*, 16> EmittedSets;
       const TargetLowering *TLI = MF->getSubtarget().getTargetLowering();
       const MCExpr *Base = TLI->getPICJumpTableRelocBaseExpr(MF,JTI,OutContext);
@@ -1333,9 +1418,10 @@ void AsmPrinter::EmitJumpTableInfo() {
 
         // .set LJTSet, LBB32-base
         const MCExpr *LHS =
-          MCSymbolRefExpr::Create(MBB->getSymbol(), OutContext);
-        OutStreamer.EmitAssignment(GetJTSetSymbol(JTI, MBB->getNumber()),
-                                MCBinaryExpr::CreateSub(LHS, Base, OutContext));
+          MCSymbolRefExpr::create(MBB->getSymbol(), OutContext);
+        OutStreamer->EmitAssignment(GetJTSetSymbol(JTI, MBB->getNumber()),
+                                    MCBinaryExpr::createSub(LHS, Base,
+                                                            OutContext));
       }
     }
 
@@ -1343,18 +1429,18 @@ void AsmPrinter::EmitJumpTableInfo() {
     // before each jump table.  The first label is never referenced, but tells
     // the assembler and linker the extents of the jump table object.  The
     // second label is actually referenced by the code.
-    if (JTInDiffSection && DL->hasLinkerPrivateGlobalPrefix())
+    if (JTInDiffSection && DL.hasLinkerPrivateGlobalPrefix())
       // FIXME: This doesn't have to have any specific name, just any randomly
       // named and numbered 'l' label would work.  Simplify GetJTISymbol.
-      OutStreamer.EmitLabel(GetJTISymbol(JTI, true));
+      OutStreamer->EmitLabel(GetJTISymbol(JTI, true));
 
-    OutStreamer.EmitLabel(GetJTISymbol(JTI));
+    OutStreamer->EmitLabel(GetJTISymbol(JTI));
 
     for (unsigned ii = 0, ee = JTBBs.size(); ii != ee; ++ii)
       EmitJumpTableEntry(MJTI, JTBBs[ii], JTI);
   }
   if (!JTInDiffSection)
-    OutStreamer.EmitDataRegion(MCDR_DataRegionEnd);
+    OutStreamer->EmitDataRegion(MCDR_DataRegionEnd);
 }
 
 /// EmitJumpTableEntry - Emit a jump table entry for the specified MBB to the
@@ -1374,14 +1460,14 @@ void AsmPrinter::EmitJumpTableEntry(const MachineJumpTableInfo *MJTI,
   case MachineJumpTableInfo::EK_BlockAddress:
     // EK_BlockAddress - Each entry is a plain address of block, e.g.:
     //     .word LBB123
-    Value = MCSymbolRefExpr::Create(MBB->getSymbol(), OutContext);
+    Value = MCSymbolRefExpr::create(MBB->getSymbol(), OutContext);
     break;
   case MachineJumpTableInfo::EK_GPRel32BlockAddress: {
     // EK_GPRel32BlockAddress - Each entry is an address of block, encoded
     // with a relocation as gp-relative, e.g.:
     //     .gprel32 LBB123
     MCSymbol *MBBSym = MBB->getSymbol();
-    OutStreamer.EmitGPRel32Value(MCSymbolRefExpr::Create(MBBSym, OutContext));
+    OutStreamer->EmitGPRel32Value(MCSymbolRefExpr::create(MBBSym, OutContext));
     return;
   }
 
@@ -1390,7 +1476,7 @@ void AsmPrinter::EmitJumpTableEntry(const MachineJumpTableInfo *MJTI,
     // with a relocation as gp-relative, e.g.:
     //     .gpdword LBB123
     MCSymbol *MBBSym = MBB->getSymbol();
-    OutStreamer.EmitGPRel64Value(MCSymbolRefExpr::Create(MBBSym, OutContext));
+    OutStreamer->EmitGPRel64Value(MCSymbolRefExpr::create(MBBSym, OutContext));
     return;
   }
 
@@ -1402,25 +1488,23 @@ void AsmPrinter::EmitJumpTableEntry(const MachineJumpTableInfo *MJTI,
     // If the .set directive avoids relocations, this is emitted as:
     //      .set L4_5_set_123, LBB123 - LJTI1_2
     //      .word L4_5_set_123
-    if (MAI->doesSetDirectiveSuppressesReloc() &&
-        !UseReadOnlyJumpTables()) { // @LOCALMOD
-      Value = MCSymbolRefExpr::Create(GetJTSetSymbol(UID, MBB->getNumber()),
+    if (MAI->doesSetDirectiveSuppressesReloc()) {
+      Value = MCSymbolRefExpr::create(GetJTSetSymbol(UID, MBB->getNumber()),
                                       OutContext);
       break;
     }
-    Value = MCSymbolRefExpr::Create(MBB->getSymbol(), OutContext);
+    Value = MCSymbolRefExpr::create(MBB->getSymbol(), OutContext);
     const TargetLowering *TLI = MF->getSubtarget().getTargetLowering();
     const MCExpr *Base = TLI->getPICJumpTableRelocBaseExpr(MF, UID, OutContext);
-    Value = MCBinaryExpr::CreateSub(Value, Base, OutContext);
+    Value = MCBinaryExpr::createSub(Value, Base, OutContext);
     break;
   }
   }
 
   assert(Value && "Unknown entry kind!");
 
-  unsigned EntrySize =
-      MJTI->getEntrySize(*TM.getDataLayout());
-  OutStreamer.EmitValue(Value, EntrySize);
+  unsigned EntrySize = MJTI->getEntrySize(getDataLayout());
+  OutStreamer->EmitValue(Value, EntrySize);
 }
 
 
@@ -1444,25 +1528,27 @@ bool AsmPrinter::EmitSpecialLLVMGlobal(const GlobalVariable *GV) {
   assert(GV->hasInitializer() && "Not a special LLVM global!");
 
   if (GV->getName() == "llvm.global_ctors") {
-    EmitXXStructorList(GV->getInitializer(), /* isCtor */ true);
+    EmitXXStructorList(GV->getParent()->getDataLayout(), GV->getInitializer(),
+                       /* isCtor */ true);
 
     if (TM.getRelocationModel() == Reloc::Static &&
         MAI->hasStaticCtorDtorReferenceInStaticMode()) {
       StringRef Sym(".constructors_used");
-      OutStreamer.EmitSymbolAttribute(OutContext.GetOrCreateSymbol(Sym),
-                                      MCSA_Reference);
+      OutStreamer->EmitSymbolAttribute(OutContext.getOrCreateSymbol(Sym),
+                                       MCSA_Reference);
     }
     return true;
   }
 
   if (GV->getName() == "llvm.global_dtors") {
-    EmitXXStructorList(GV->getInitializer(), /* isCtor */ false);
+    EmitXXStructorList(GV->getParent()->getDataLayout(), GV->getInitializer(),
+                       /* isCtor */ false);
 
     if (TM.getRelocationModel() == Reloc::Static &&
         MAI->hasStaticCtorDtorReferenceInStaticMode()) {
       StringRef Sym(".destructors_used");
-      OutStreamer.EmitSymbolAttribute(OutContext.GetOrCreateSymbol(Sym),
-                                      MCSA_Reference);
+      OutStreamer->EmitSymbolAttribute(OutContext.getOrCreateSymbol(Sym),
+                                       MCSA_Reference);
     }
     return true;
   }
@@ -1479,7 +1565,7 @@ void AsmPrinter::EmitLLVMUsedList(const ConstantArray *InitList) {
     const GlobalValue *GV =
       dyn_cast<GlobalValue>(InitList->getOperand(i)->stripPointerCasts());
     if (GV)
-      OutStreamer.EmitSymbolAttribute(getSymbol(GV), MCSA_NoDeadStrip);
+      OutStreamer->EmitSymbolAttribute(getSymbol(GV), MCSA_NoDeadStrip);
   }
 }
 
@@ -1494,7 +1580,8 @@ struct Structor {
 
 /// EmitXXStructorList - Emit the ctor or dtor list taking into account the init
 /// priority.
-void AsmPrinter::EmitXXStructorList(const Constant *List, bool isCtor) {
+void AsmPrinter::EmitXXStructorList(const DataLayout &DL, const Constant *List,
+                                    bool isCtor) {
   // Should be an array of '{ int, void ()* }' structs.  The first value is the
   // init priority.
   if (!isa<ConstantArray>(List)) return;
@@ -1529,8 +1616,7 @@ void AsmPrinter::EmitXXStructorList(const Constant *List, bool isCtor) {
   }
 
   // Emit the function pointers in the target-specific order
-  const DataLayout *DL = TM.getDataLayout();
-  unsigned Align = Log2_32(DL->getPointerPrefAlignment());
+  unsigned Align = Log2_32(DL.getPointerPrefAlignment());
   std::stable_sort(Structors.begin(), Structors.end(),
                    [](const Structor &L,
                       const Structor &R) { return L.Priority < R.Priority; });
@@ -1545,13 +1631,13 @@ void AsmPrinter::EmitXXStructorList(const Constant *List, bool isCtor) {
 
       KeySym = getSymbol(GV);
     }
-    const MCSection *OutputSection =
+    MCSection *OutputSection =
         (isCtor ? Obj.getStaticCtorSection(S.Priority, KeySym)
                 : Obj.getStaticDtorSection(S.Priority, KeySym));
-    OutStreamer.SwitchSection(OutputSection);
-    if (OutStreamer.getCurrentSection() != OutStreamer.getPreviousSection())
+    OutStreamer->SwitchSection(OutputSection);
+    if (OutStreamer->getCurrentSection() != OutStreamer->getPreviousSection())
       EmitAlignment(Align);
-    EmitXXStructor(S.Func);
+    EmitXXStructor(DL, S.Func);
   }
 }
 
@@ -1565,7 +1651,7 @@ void AsmPrinter::EmitModuleIdents(Module &M) {
       assert(N->getNumOperands() == 1 &&
              "llvm.ident metadata entry can have only one operand");
       const MDString *S = cast<MDString>(N->getOperand(0));
-      OutStreamer.EmitIdent(S->getString());
+      OutStreamer->EmitIdent(S->getString());
     }
   }
 }
@@ -1577,19 +1663,19 @@ void AsmPrinter::EmitModuleIdents(Module &M) {
 /// EmitInt8 - Emit a byte directive and value.
 ///
 void AsmPrinter::EmitInt8(int Value) const {
-  OutStreamer.EmitIntValue(Value, 1);
+  OutStreamer->EmitIntValue(Value, 1);
 }
 
 /// EmitInt16 - Emit a short directive and value.
 ///
 void AsmPrinter::EmitInt16(int Value) const {
-  OutStreamer.EmitIntValue(Value, 2);
+  OutStreamer->EmitIntValue(Value, 2);
 }
 
 /// EmitInt32 - Emit a long directive and value.
 ///
 void AsmPrinter::EmitInt32(int Value) const {
-  OutStreamer.EmitIntValue(Value, 4);
+  OutStreamer->EmitIntValue(Value, 4);
 }
 
 /// Emit something like ".long Hi-Lo" where the size in bytes of the directive
@@ -1597,21 +1683,7 @@ void AsmPrinter::EmitInt32(int Value) const {
 /// .set if it avoids relocations.
 void AsmPrinter::EmitLabelDifference(const MCSymbol *Hi, const MCSymbol *Lo,
                                      unsigned Size) const {
-  // Get the Hi-Lo expression.
-  const MCExpr *Diff =
-    MCBinaryExpr::CreateSub(MCSymbolRefExpr::Create(Hi, OutContext),
-                            MCSymbolRefExpr::Create(Lo, OutContext),
-                            OutContext);
-
-  if (!MAI->doesSetDirectiveSuppressesReloc()) {
-    OutStreamer.EmitValue(Diff, Size);
-    return;
-  }
-
-  // Otherwise, emit with .set (aka assignment).
-  MCSymbol *SetLabel = createTempSymbol("set");
-  OutStreamer.EmitAssignment(SetLabel, Diff);
-  OutStreamer.EmitSymbolValue(SetLabel, Size);
+  OutStreamer->emitAbsoluteSymbolDiff(Hi, Lo, Size);
 }
 
 /// EmitLabelPlusOffset - Emit something like ".long Label+Offset"
@@ -1621,17 +1693,17 @@ void AsmPrinter::EmitLabelPlusOffset(const MCSymbol *Label, uint64_t Offset,
                                      unsigned Size,
                                      bool IsSectionRelative) const {
   if (MAI->needsDwarfSectionOffsetDirective() && IsSectionRelative) {
-    OutStreamer.EmitCOFFSecRel32(Label);
+    OutStreamer->EmitCOFFSecRel32(Label);
     return;
   }
 
   // Emit Label+Offset (or just Label if Offset is zero)
-  const MCExpr *Expr = MCSymbolRefExpr::Create(Label, OutContext);
+  const MCExpr *Expr = MCSymbolRefExpr::create(Label, OutContext);
   if (Offset)
-    Expr = MCBinaryExpr::CreateAdd(
-        Expr, MCConstantExpr::Create(Offset, OutContext), OutContext);
+    Expr = MCBinaryExpr::createAdd(
+        Expr, MCConstantExpr::create(Offset, OutContext), OutContext);
 
-  OutStreamer.EmitValue(Expr, Size);
+  OutStreamer->EmitValue(Expr, Size);
 }
 
 //===----------------------------------------------------------------------===//
@@ -1644,8 +1716,7 @@ void AsmPrinter::EmitLabelPlusOffset(const MCSymbol *Label, uint64_t Offset,
 //
 void AsmPrinter::EmitAlignment(unsigned NumBits, const GlobalObject *GV) const {
   if (GV)
-    NumBits = getGVAlignmentLog2(GV, *TM.getDataLayout(),
-                                 NumBits);
+    NumBits = getGVAlignmentLog2(GV, GV->getParent()->getDataLayout(), NumBits);
 
   if (NumBits == 0) return;   // 1-byte aligned: no need to emit alignment.
 
@@ -1653,9 +1724,9 @@ void AsmPrinter::EmitAlignment(unsigned NumBits, const GlobalObject *GV) const {
              static_cast<unsigned>(std::numeric_limits<unsigned>::digits) &&
          "undefined behavior");
   if (getCurrentSection()->getKind().isText())
-    OutStreamer.EmitCodeAlignment(1u << NumBits);
+    OutStreamer->EmitCodeAlignment(1u << NumBits);
   else
-    OutStreamer.EmitValueToAlignment(1u << NumBits);
+    OutStreamer->EmitValueToAlignment(1u << NumBits);
 }
 
 //===----------------------------------------------------------------------===//
@@ -1666,16 +1737,16 @@ const MCExpr *AsmPrinter::lowerConstant(const Constant *CV) {
   MCContext &Ctx = OutContext;
 
   if (CV->isNullValue() || isa<UndefValue>(CV))
-    return MCConstantExpr::Create(0, Ctx);
+    return MCConstantExpr::create(0, Ctx);
 
   if (const ConstantInt *CI = dyn_cast<ConstantInt>(CV))
-    return MCConstantExpr::Create(CI->getZExtValue(), Ctx);
+    return MCConstantExpr::create(CI->getZExtValue(), Ctx);
 
   if (const GlobalValue *GV = dyn_cast<GlobalValue>(CV))
-    return MCSymbolRefExpr::Create(getSymbol(GV), Ctx);
+    return MCSymbolRefExpr::create(getSymbol(GV), Ctx);
 
   if (const BlockAddress *BA = dyn_cast<BlockAddress>(CV))
-    return MCSymbolRefExpr::Create(GetBlockAddressSymbol(BA), Ctx);
+    return MCSymbolRefExpr::create(GetBlockAddressSymbol(BA), Ctx);
 
   const ConstantExpr *CE = dyn_cast<ConstantExpr>(CV);
   if (!CE) {
@@ -1691,7 +1762,7 @@ const MCExpr *AsmPrinter::lowerConstant(const Constant *CV) {
     // If the code isn't optimized, there may be outstanding folding
     // opportunities. Attempt to fold the expression using DataLayout as a
     // last resort before giving up.
-    if (Constant *C = ConstantFoldConstantExpression(CE, *TM.getDataLayout()))
+    if (Constant *C = ConstantFoldConstantExpression(CE, getDataLayout()))
       if (C != CE)
         return lowerConstant(C);
 
@@ -1705,18 +1776,16 @@ const MCExpr *AsmPrinter::lowerConstant(const Constant *CV) {
       report_fatal_error(OS.str());
     }
   case Instruction::GetElementPtr: {
-    const DataLayout &DL = *TM.getDataLayout();
-
     // Generate a symbolic expression for the byte address
-    APInt OffsetAI(DL.getPointerTypeSizeInBits(CE->getType()), 0);
-    cast<GEPOperator>(CE)->accumulateConstantOffset(DL, OffsetAI);
+    APInt OffsetAI(getDataLayout().getPointerTypeSizeInBits(CE->getType()), 0);
+    cast<GEPOperator>(CE)->accumulateConstantOffset(getDataLayout(), OffsetAI);
 
     const MCExpr *Base = lowerConstant(CE->getOperand(0));
     if (!OffsetAI)
       return Base;
 
     int64_t Offset = OffsetAI.getSExtValue();
-    return MCBinaryExpr::CreateAdd(Base, MCConstantExpr::Create(Offset, Ctx),
+    return MCBinaryExpr::createAdd(Base, MCConstantExpr::create(Offset, Ctx),
                                    Ctx);
   }
 
@@ -1730,7 +1799,7 @@ const MCExpr *AsmPrinter::lowerConstant(const Constant *CV) {
     return lowerConstant(CE->getOperand(0));
 
   case Instruction::IntToPtr: {
-    const DataLayout &DL = *TM.getDataLayout();
+    const DataLayout &DL = getDataLayout();
 
     // Handle casts to pointers by changing them into casts to the appropriate
     // integer type.  This promotes constant folding and simplifies this code.
@@ -1741,7 +1810,7 @@ const MCExpr *AsmPrinter::lowerConstant(const Constant *CV) {
   }
 
   case Instruction::PtrToInt: {
-    const DataLayout &DL = *TM.getDataLayout();
+    const DataLayout &DL = getDataLayout();
 
     // Support only foldable casts to/from pointers that can be eliminated by
     // changing the pointer to the appropriately sized integer type.
@@ -1759,8 +1828,8 @@ const MCExpr *AsmPrinter::lowerConstant(const Constant *CV) {
     // the high bits so we are sure to get a proper truncation if the input is
     // a constant expr.
     unsigned InBits = DL.getTypeAllocSizeInBits(Op->getType());
-    const MCExpr *MaskExpr = MCConstantExpr::Create(~0ULL >> (64-InBits), Ctx);
-    return MCBinaryExpr::CreateAnd(OpExpr, MaskExpr, Ctx);
+    const MCExpr *MaskExpr = MCConstantExpr::create(~0ULL >> (64-InBits), Ctx);
+    return MCBinaryExpr::createAnd(OpExpr, MaskExpr, Ctx);
   }
 
   // The MC library also has a right-shift operator, but it isn't consistently
@@ -1778,21 +1847,22 @@ const MCExpr *AsmPrinter::lowerConstant(const Constant *CV) {
     const MCExpr *RHS = lowerConstant(CE->getOperand(1));
     switch (CE->getOpcode()) {
     default: llvm_unreachable("Unknown binary operator constant cast expr");
-    case Instruction::Add: return MCBinaryExpr::CreateAdd(LHS, RHS, Ctx);
-    case Instruction::Sub: return MCBinaryExpr::CreateSub(LHS, RHS, Ctx);
-    case Instruction::Mul: return MCBinaryExpr::CreateMul(LHS, RHS, Ctx);
-    case Instruction::SDiv: return MCBinaryExpr::CreateDiv(LHS, RHS, Ctx);
-    case Instruction::SRem: return MCBinaryExpr::CreateMod(LHS, RHS, Ctx);
-    case Instruction::Shl: return MCBinaryExpr::CreateShl(LHS, RHS, Ctx);
-    case Instruction::And: return MCBinaryExpr::CreateAnd(LHS, RHS, Ctx);
-    case Instruction::Or:  return MCBinaryExpr::CreateOr (LHS, RHS, Ctx);
-    case Instruction::Xor: return MCBinaryExpr::CreateXor(LHS, RHS, Ctx);
+    case Instruction::Add: return MCBinaryExpr::createAdd(LHS, RHS, Ctx);
+    case Instruction::Sub: return MCBinaryExpr::createSub(LHS, RHS, Ctx);
+    case Instruction::Mul: return MCBinaryExpr::createMul(LHS, RHS, Ctx);
+    case Instruction::SDiv: return MCBinaryExpr::createDiv(LHS, RHS, Ctx);
+    case Instruction::SRem: return MCBinaryExpr::createMod(LHS, RHS, Ctx);
+    case Instruction::Shl: return MCBinaryExpr::createShl(LHS, RHS, Ctx);
+    case Instruction::And: return MCBinaryExpr::createAnd(LHS, RHS, Ctx);
+    case Instruction::Or:  return MCBinaryExpr::createOr (LHS, RHS, Ctx);
+    case Instruction::Xor: return MCBinaryExpr::createXor(LHS, RHS, Ctx);
     }
   }
   }
 }
 
-static void emitGlobalConstantImpl(const Constant *C, AsmPrinter &AP,
+static void emitGlobalConstantImpl(const DataLayout &DL, const Constant *C,
+                                   AsmPrinter &AP,
                                    const Constant *BaseCV = nullptr,
                                    uint64_t Offset = 0);
 
@@ -1812,41 +1882,31 @@ static int isRepeatedByteSequence(const ConstantDataSequential *V) {
 /// isRepeatedByteSequence - Determine whether the given value is
 /// composed of a repeated sequence of identical bytes and return the
 /// byte value.  If it is not a repeated sequence, return -1.
-static int isRepeatedByteSequence(const Value *V, TargetMachine &TM) {
-
+static int isRepeatedByteSequence(const Value *V, const DataLayout &DL) {
   if (const ConstantInt *CI = dyn_cast<ConstantInt>(V)) {
-    if (CI->getBitWidth() > 64) return -1;
+    uint64_t Size = DL.getTypeAllocSizeInBits(V->getType());
+    assert(Size % 8 == 0);
 
-    uint64_t Size =
-        TM.getDataLayout()->getTypeAllocSize(V->getType());
-    uint64_t Value = CI->getZExtValue();
+    // Extend the element to take zero padding into account.
+    APInt Value = CI->getValue().zextOrSelf(Size);
+    if (!Value.isSplat(8))
+      return -1;
 
-    // Make sure the constant is at least 8 bits long and has a power
-    // of 2 bit width.  This guarantees the constant bit width is
-    // always a multiple of 8 bits, avoiding issues with padding out
-    // to Size and other such corner cases.
-    if (CI->getBitWidth() < 8 || !isPowerOf2_64(CI->getBitWidth())) return -1;
-
-    uint8_t Byte = static_cast<uint8_t>(Value);
-
-    for (unsigned i = 1; i < Size; ++i) {
-      Value >>= 8;
-      if (static_cast<uint8_t>(Value) != Byte) return -1;
-    }
-    return Byte;
+    return Value.zextOrTrunc(8).getZExtValue();
   }
   if (const ConstantArray *CA = dyn_cast<ConstantArray>(V)) {
     // Make sure all array elements are sequences of the same repeated
     // byte.
     assert(CA->getNumOperands() != 0 && "Should be a CAZ");
-    int Byte = isRepeatedByteSequence(CA->getOperand(0), TM);
-    if (Byte == -1) return -1;
-
-    for (unsigned i = 1, e = CA->getNumOperands(); i != e; ++i) {
-      int ThisByte = isRepeatedByteSequence(CA->getOperand(i), TM);
-      if (ThisByte == -1) return -1;
-      if (Byte != ThisByte) return -1;
-    }
+    Constant *Op0 = CA->getOperand(0);
+    int Byte = isRepeatedByteSequence(Op0, DL);
+    if (Byte == -1)
+      return -1;
+
+    // All array elements must be equal.
+    for (unsigned i = 1, e = CA->getNumOperands(); i != e; ++i)
+      if (CA->getOperand(i) != Op0)
+        return -1;
     return Byte;
   }
 
@@ -1856,33 +1916,32 @@ static int isRepeatedByteSequence(const Value *V, TargetMachine &TM) {
   return -1;
 }
 
-static void emitGlobalConstantDataSequential(const ConstantDataSequential *CDS,
-                                             AsmPrinter &AP){
+static void emitGlobalConstantDataSequential(const DataLayout &DL,
+                                             const ConstantDataSequential *CDS,
+                                             AsmPrinter &AP) {
 
   // See if we can aggregate this into a .fill, if so, emit it as such.
-  int Value = isRepeatedByteSequence(CDS, AP.TM);
+  int Value = isRepeatedByteSequence(CDS, DL);
   if (Value != -1) {
-    uint64_t Bytes =
-        AP.TM.getDataLayout()->getTypeAllocSize(
-            CDS->getType());
+    uint64_t Bytes = DL.getTypeAllocSize(CDS->getType());
     // Don't emit a 1-byte object as a .fill.
     if (Bytes > 1)
-      return AP.OutStreamer.EmitFill(Bytes, Value);
+      return AP.OutStreamer->EmitFill(Bytes, Value);
   }
 
   // If this can be emitted with .ascii/.asciz, emit it as such.
   if (CDS->isString())
-    return AP.OutStreamer.EmitBytes(CDS->getAsString());
+    return AP.OutStreamer->EmitBytes(CDS->getAsString());
 
   // Otherwise, emit the values in successive locations.
   unsigned ElementByteSize = CDS->getElementByteSize();
   if (isa<IntegerType>(CDS->getElementType())) {
     for (unsigned i = 0, e = CDS->getNumElements(); i != e; ++i) {
       if (AP.isVerbose())
-        AP.OutStreamer.GetCommentOS() << format("0x%" PRIx64 "\n",
-                                                CDS->getElementAsInteger(i));
-      AP.OutStreamer.EmitIntValue(CDS->getElementAsInteger(i),
-                                  ElementByteSize);
+        AP.OutStreamer->GetCommentOS() << format("0x%" PRIx64 "\n",
+                                                 CDS->getElementAsInteger(i));
+      AP.OutStreamer->EmitIntValue(CDS->getElementAsInteger(i),
+                                   ElementByteSize);
     }
   } else if (ElementByteSize == 4) {
     // FP Constants are printed as integer constants to avoid losing
@@ -1896,8 +1955,8 @@ static void emitGlobalConstantDataSequential(const ConstantDataSequential *CDS,
 
       F = CDS->getElementAsFloat(i);
       if (AP.isVerbose())
-        AP.OutStreamer.GetCommentOS() << "float " << F << '\n';
-      AP.OutStreamer.EmitIntValue(I, 4);
+        AP.OutStreamer->GetCommentOS() << "float " << F << '\n';
+      AP.OutStreamer->EmitIntValue(I, 4);
     }
   } else {
     assert(CDS->getElementType()->isDoubleTy());
@@ -1909,66 +1968,65 @@ static void emitGlobalConstantDataSequential(const ConstantDataSequential *CDS,
 
       F = CDS->getElementAsDouble(i);
       if (AP.isVerbose())
-        AP.OutStreamer.GetCommentOS() << "double " << F << '\n';
-      AP.OutStreamer.EmitIntValue(I, 8);
+        AP.OutStreamer->GetCommentOS() << "double " << F << '\n';
+      AP.OutStreamer->EmitIntValue(I, 8);
     }
   }
 
-  const DataLayout &DL = *AP.TM.getDataLayout();
   unsigned Size = DL.getTypeAllocSize(CDS->getType());
   unsigned EmittedSize = DL.getTypeAllocSize(CDS->getType()->getElementType()) *
                         CDS->getNumElements();
   if (unsigned Padding = Size - EmittedSize)
-    AP.OutStreamer.EmitZeros(Padding);
+    AP.OutStreamer->EmitZeros(Padding);
 
 }
 
-static void emitGlobalConstantArray(const ConstantArray *CA, AsmPrinter &AP,
+static void emitGlobalConstantArray(const DataLayout &DL,
+                                    const ConstantArray *CA, AsmPrinter &AP,
                                     const Constant *BaseCV, uint64_t Offset) {
   // See if we can aggregate some values.  Make sure it can be
   // represented as a series of bytes of the constant value.
-  int Value = isRepeatedByteSequence(CA, AP.TM);
-  const DataLayout &DL = *AP.TM.getDataLayout();
+  int Value = isRepeatedByteSequence(CA, DL);
 
   if (Value != -1) {
     uint64_t Bytes = DL.getTypeAllocSize(CA->getType());
-    AP.OutStreamer.EmitFill(Bytes, Value);
+    AP.OutStreamer->EmitFill(Bytes, Value);
   }
   else {
     for (unsigned i = 0, e = CA->getNumOperands(); i != e; ++i) {
-      emitGlobalConstantImpl(CA->getOperand(i), AP, BaseCV, Offset);
+      emitGlobalConstantImpl(DL, CA->getOperand(i), AP, BaseCV, Offset);
       Offset += DL.getTypeAllocSize(CA->getOperand(i)->getType());
     }
   }
 }
 
-static void emitGlobalConstantVector(const ConstantVector *CV, AsmPrinter &AP) {
+static void emitGlobalConstantVector(const DataLayout &DL,
+                                     const ConstantVector *CV, AsmPrinter &AP) {
   for (unsigned i = 0, e = CV->getType()->getNumElements(); i != e; ++i)
-    emitGlobalConstantImpl(CV->getOperand(i), AP);
+    emitGlobalConstantImpl(DL, CV->getOperand(i), AP);
 
-  const DataLayout &DL = *AP.TM.getDataLayout();
   unsigned Size = DL.getTypeAllocSize(CV->getType());
   unsigned EmittedSize = DL.getTypeAllocSize(CV->getType()->getElementType()) *
                          CV->getType()->getNumElements();
   if (unsigned Padding = Size - EmittedSize)
-    AP.OutStreamer.EmitZeros(Padding);
+    AP.OutStreamer->EmitZeros(Padding);
 }
 
-static void emitGlobalConstantStruct(const ConstantStruct *CS, AsmPrinter &AP,
+static void emitGlobalConstantStruct(const DataLayout &DL,
+                                     const ConstantStruct *CS, AsmPrinter &AP,
                                      const Constant *BaseCV, uint64_t Offset) {
   // Print the fields in successive locations. Pad to align if needed!
-  const DataLayout *DL = AP.TM.getDataLayout();
-  unsigned Size = DL->getTypeAllocSize(CS->getType());
-  const StructLayout *Layout = DL->getStructLayout(CS->getType());
+  unsigned Size = DL.getTypeAllocSize(CS->getType());
+  const StructLayout *Layout = DL.getStructLayout(CS->getType());
   uint64_t SizeSoFar = 0;
   for (unsigned i = 0, e = CS->getNumOperands(); i != e; ++i) {
     const Constant *Field = CS->getOperand(i);
 
     // Print the actual field value.
-    emitGlobalConstantImpl(Field, AP, BaseCV, Offset+SizeSoFar);
+    emitGlobalConstantImpl(DL, Field, AP, BaseCV, Offset + SizeSoFar);
 
     // Check if padding is needed and insert one or more 0s.
-    uint64_t FieldSize = DL->getTypeAllocSize(Field->getType());
+    uint64_t FieldSize = DL.getTypeAllocSize(Field->getType());
     uint64_t PadSize = ((i == e-1 ? Size : Layout->getElementOffset(i+1))
                         - Layout->getElementOffset(i)) - FieldSize;
     SizeSoFar += FieldSize + PadSize;
@@ -1976,7 +2034,7 @@ static void emitGlobalConstantStruct(const ConstantStruct *CS, AsmPrinter &AP,
     // Insert padding - this may include padding to increase the size of the
     // current field up to the ABI size (if the struct is not packed) as well
     // as padding to ensure that the next field starts at the right offset.
-    AP.OutStreamer.EmitZeros(PadSize);
+    AP.OutStreamer->EmitZeros(PadSize);
   }
   assert(SizeSoFar == Layout->getSizeInBytes() &&
          "Layout of constant struct may be incorrect!");
@@ -1992,10 +2050,10 @@ static void emitGlobalConstantFP(const ConstantFP *CFP, AsmPrinter &AP) {
     CFP->getValueAPF().toString(StrVal);
 
     if (CFP->getType())
-      CFP->getType()->print(AP.OutStreamer.GetCommentOS());
+      CFP->getType()->print(AP.OutStreamer->GetCommentOS());
     else
-      AP.OutStreamer.GetCommentOS() << "Printing <null> Type";
-    AP.OutStreamer.GetCommentOS() << ' ' << StrVal << '\n';
+      AP.OutStreamer->GetCommentOS() << "Printing <null> Type";
+    AP.OutStreamer->GetCommentOS() << ' ' << StrVal << '\n';
   }
 
   // Now iterate through the APInt chunks, emitting them in endian-correct
@@ -2007,32 +2065,31 @@ static void emitGlobalConstantFP(const ConstantFP *CFP, AsmPrinter &AP) {
 
   // PPC's long double has odd notions of endianness compared to how LLVM
   // handles it: p[0] goes first for *big* endian on PPC.
-  if (AP.TM.getDataLayout()->isBigEndian() &&
-      !CFP->getType()->isPPC_FP128Ty()) {
+  if (AP.getDataLayout().isBigEndian() && !CFP->getType()->isPPC_FP128Ty()) {
     int Chunk = API.getNumWords() - 1;
 
     if (TrailingBytes)
-      AP.OutStreamer.EmitIntValue(p[Chunk--], TrailingBytes);
+      AP.OutStreamer->EmitIntValue(p[Chunk--], TrailingBytes);
 
     for (; Chunk >= 0; --Chunk)
-      AP.OutStreamer.EmitIntValue(p[Chunk], sizeof(uint64_t));
+      AP.OutStreamer->EmitIntValue(p[Chunk], sizeof(uint64_t));
   } else {
     unsigned Chunk;
     for (Chunk = 0; Chunk < NumBytes / sizeof(uint64_t); ++Chunk)
-      AP.OutStreamer.EmitIntValue(p[Chunk], sizeof(uint64_t));
+      AP.OutStreamer->EmitIntValue(p[Chunk], sizeof(uint64_t));
 
     if (TrailingBytes)
-      AP.OutStreamer.EmitIntValue(p[Chunk], TrailingBytes);
+      AP.OutStreamer->EmitIntValue(p[Chunk], TrailingBytes);
   }
 
   // Emit the tail padding for the long double.
-  const DataLayout &DL = *AP.TM.getDataLayout();
-  AP.OutStreamer.EmitZeros(DL.getTypeAllocSize(CFP->getType()) -
-                           DL.getTypeStoreSize(CFP->getType()));
+  const DataLayout &DL = AP.getDataLayout();
+  AP.OutStreamer->EmitZeros(DL.getTypeAllocSize(CFP->getType()) -
+                            DL.getTypeStoreSize(CFP->getType()));
 }
 
 static void emitGlobalConstantLargeInt(const ConstantInt *CI, AsmPrinter &AP) {
-  const DataLayout *DL = AP.TM.getDataLayout();
+  const DataLayout &DL = AP.getDataLayout();
   unsigned BitWidth = CI->getBitWidth();
 
   // Copy the value as we may massage the layout for constants whose bit width
@@ -2049,7 +2106,7 @@ static void emitGlobalConstantLargeInt(const ConstantInt *CI, AsmPrinter &AP) {
     // Big endian:
     // * Record the extra bits to emit.
     // * Realign the raw data to emit the chunks of 64-bits.
-    if (DL->isBigEndian()) {
+    if (DL.isBigEndian()) {
       // Basically the structure of the raw data is a chunk of 64-bits cells:
       //    0        1         BitWidth / 64
       // [chunk1][chunk2] ... [chunkN].
@@ -2070,21 +2127,20 @@ static void emitGlobalConstantLargeInt(const ConstantInt *CI, AsmPrinter &AP) {
   // quantities at a time.
   const uint64_t *RawData = Realigned.getRawData();
   for (unsigned i = 0, e = BitWidth / 64; i != e; ++i) {
-    uint64_t Val = DL->isBigEndian() ? RawData[e - i - 1] : RawData[i];
-    AP.OutStreamer.EmitIntValue(Val, 8);
+    uint64_t Val = DL.isBigEndian() ? RawData[e - i - 1] : RawData[i];
+    AP.OutStreamer->EmitIntValue(Val, 8);
   }
 
   if (ExtraBitsSize) {
     // Emit the extra bits after the 64-bits chunks.
 
     // Emit a directive that fills the expected size.
-    uint64_t Size = AP.TM.getDataLayout()->getTypeAllocSize(
-        CI->getType());
+    uint64_t Size = AP.getDataLayout().getTypeAllocSize(CI->getType());
     Size -= (BitWidth / 64) * 8;
     assert(Size && Size * 8 >= ExtraBitsSize &&
            (ExtraBits & (((uint64_t)-1) >> (64 - ExtraBitsSize)))
            == ExtraBits && "Directive too small for extra bits.");
-    AP.OutStreamer.EmitIntValue(ExtraBits, Size);
+    AP.OutStreamer->EmitIntValue(ExtraBits, Size);
   }
 }
 
@@ -2110,25 +2166,32 @@ static void handleIndirectSymViaGOTPCRel(AsmPrinter &AP, const MCExpr **ME,
   //    cstexpr := <gotequiv> - "." + <cst>
   //    cstexpr := <gotequiv> - (<foo> - <offset from @foo base>) + <cst>
   //
-  // After canonicalization by EvaluateAsRelocatable `ME` turns into:
+  // After canonicalization by evaluateAsRelocatable `ME` turns into:
   //
   //  cstexpr := <gotequiv> - <foo> + gotpcrelcst, where
   //    gotpcrelcst := <offset from @foo base> + <cst>
   //
   MCValue MV;
-  if (!(*ME)->EvaluateAsRelocatable(MV, nullptr, nullptr) || MV.isAbsolute())
+  if (!(*ME)->evaluateAsRelocatable(MV, nullptr, nullptr) || MV.isAbsolute())
+    return;
+  const MCSymbolRefExpr *SymA = MV.getSymA();
+  if (!SymA)
     return;
 
-  const MCSymbol *GOTEquivSym = &MV.getSymA()->getSymbol();
+  // Check that GOT equivalent symbol is cached.
+  const MCSymbol *GOTEquivSym = &SymA->getSymbol();
   if (!AP.GlobalGOTEquivs.count(GOTEquivSym))
     return;
 
-  const GlobalValue *BaseGV = dyn_cast<GlobalValue>(BaseCst);
+  const GlobalValue *BaseGV = dyn_cast_or_null<GlobalValue>(BaseCst);
   if (!BaseGV)
     return;
 
+  // Check for a valid base symbol
   const MCSymbol *BaseSym = AP.getSymbol(BaseGV);
-  if (BaseSym != &MV.getSymB()->getSymbol())
+  const MCSymbolRefExpr *SymB = MV.getSymB();
+
+  if (!SymB || BaseSym != &SymB->getSymbol())
     return;
 
   // Make sure to match:
@@ -2167,7 +2230,7 @@ static void handleIndirectSymViaGOTPCRel(AsmPrinter &AP, const MCExpr **ME,
   const GlobalValue *FinalGV = dyn_cast<GlobalValue>(GV->getOperand(0));
   const MCSymbol *FinalSym = AP.getSymbol(FinalGV);
   *ME = AP.getObjFileLowering().getIndirectSymViaGOTPCRel(
-      FinalSym, MV, Offset, AP.MMI, AP.OutStreamer);
+      FinalSym, MV, Offset, AP.MMI, *AP.OutStreamer);
 
   // Update GOT equivalent usage information
   --NumUses;
@@ -2175,10 +2238,10 @@ static void handleIndirectSymViaGOTPCRel(AsmPrinter &AP, const MCExpr **ME,
     AP.GlobalGOTEquivs[GOTEquivSym] = std::make_pair(GV, NumUses);
 }
 
-static void emitGlobalConstantImpl(const Constant *CV, AsmPrinter &AP,
-                                   const Constant *BaseCV, uint64_t Offset) {
-  const DataLayout *DL = AP.TM.getDataLayout();
-  uint64_t Size = DL->getTypeAllocSize(CV->getType());
+static void emitGlobalConstantImpl(const DataLayout &DL, const Constant *CV,
+                                   AsmPrinter &AP, const Constant *BaseCV,
+                                   uint64_t Offset) {
+  uint64_t Size = DL.getTypeAllocSize(CV->getType());
 
   // Globals with sub-elements such as combinations of arrays and structs
   // are handled recursively by emitGlobalConstantImpl. Keep track of the
@@ -2187,7 +2250,7 @@ static void emitGlobalConstantImpl(const Constant *CV, AsmPrinter &AP,
     BaseCV = dyn_cast<Constant>(CV->user_back());
 
   if (isa<ConstantAggregateZero>(CV) || isa<UndefValue>(CV))
-    return AP.OutStreamer.EmitZeros(Size);
+    return AP.OutStreamer->EmitZeros(Size);
 
   if (const ConstantInt *CI = dyn_cast<ConstantInt>(CV)) {
     switch (Size) {
@@ -2196,9 +2259,9 @@ static void emitGlobalConstantImpl(const Constant *CV, AsmPrinter &AP,
     case 4:
     case 8:
       if (AP.isVerbose())
-        AP.OutStreamer.GetCommentOS() << format("0x%" PRIx64 "\n",
-                                                CI->getZExtValue());
-      AP.OutStreamer.EmitIntValue(CI->getZExtValue(), Size);
+        AP.OutStreamer->GetCommentOS() << format("0x%" PRIx64 "\n",
+                                                 CI->getZExtValue());
+      AP.OutStreamer->EmitIntValue(CI->getZExtValue(), Size);
       return;
     default:
       emitGlobalConstantLargeInt(CI, AP);
@@ -2210,37 +2273,37 @@ static void emitGlobalConstantImpl(const Constant *CV, AsmPrinter &AP,
     return emitGlobalConstantFP(CFP, AP);
 
   if (isa<ConstantPointerNull>(CV)) {
-    AP.OutStreamer.EmitIntValue(0, Size);
+    AP.OutStreamer->EmitIntValue(0, Size);
     return;
   }
 
   if (const ConstantDataSequential *CDS = dyn_cast<ConstantDataSequential>(CV))
-    return emitGlobalConstantDataSequential(CDS, AP);
+    return emitGlobalConstantDataSequential(DL, CDS, AP);
 
   if (const ConstantArray *CVA = dyn_cast<ConstantArray>(CV))
-    return emitGlobalConstantArray(CVA, AP, BaseCV, Offset);
+    return emitGlobalConstantArray(DL, CVA, AP, BaseCV, Offset);
 
   if (const ConstantStruct *CVS = dyn_cast<ConstantStruct>(CV))
-    return emitGlobalConstantStruct(CVS, AP, BaseCV, Offset);
+    return emitGlobalConstantStruct(DL, CVS, AP, BaseCV, Offset);
 
   if (const ConstantExpr *CE = dyn_cast<ConstantExpr>(CV)) {
     // Look through bitcasts, which might not be able to be MCExpr'ized (e.g. of
     // vectors).
     if (CE->getOpcode() == Instruction::BitCast)
-      return emitGlobalConstantImpl(CE->getOperand(0), AP);
+      return emitGlobalConstantImpl(DL, CE->getOperand(0), AP);
 
     if (Size > 8) {
       // If the constant expression's size is greater than 64-bits, then we have
       // to emit the value in chunks. Try to constant fold the value and emit it
       // that way.
-      Constant *New = ConstantFoldConstantExpression(CE, *DL);
+      Constant *New = ConstantFoldConstantExpression(CE, DL);
       if (New && New != CE)
-        return emitGlobalConstantImpl(New, AP);
+        return emitGlobalConstantImpl(DL, New, AP);
     }
   }
 
   if (const ConstantVector *V = dyn_cast<ConstantVector>(CV))
-    return emitGlobalConstantVector(V, AP);
+    return emitGlobalConstantVector(DL, V, AP);
 
   // Otherwise, it must be a ConstantExpr.  Lower it to an MCExpr, then emit it
   // thread the streamer with EmitValue.
@@ -2252,19 +2315,18 @@ static void emitGlobalConstantImpl(const Constant *CV, AsmPrinter &AP,
   if (AP.getObjFileLowering().supportIndirectSymViaGOTPCRel())
     handleIndirectSymViaGOTPCRel(AP, &ME, BaseCV, Offset);
 
-  AP.OutStreamer.EmitValue(ME, Size);
+  AP.OutStreamer->EmitValue(ME, Size);
 }
 
 /// EmitGlobalConstant - Print a general LLVM constant to the .s file.
-void AsmPrinter::EmitGlobalConstant(const Constant *CV) {
-  uint64_t Size =
-      TM.getDataLayout()->getTypeAllocSize(CV->getType());
+void AsmPrinter::EmitGlobalConstant(const DataLayout &DL, const Constant *CV) {
+  uint64_t Size = DL.getTypeAllocSize(CV->getType());
   if (Size)
-    emitGlobalConstantImpl(CV, *this);
+    emitGlobalConstantImpl(DL, CV, *this);
   else if (MAI->hasSubsectionsViaSymbols()) {
     // If the global has zero size, emit a single byte so that two labels don't
     // look like they are at the same location.
-    OutStreamer.EmitIntValue(0, 1);
+    OutStreamer->EmitIntValue(0, 1);
   }
 }
 
@@ -2298,10 +2360,10 @@ MCSymbol *AsmPrinter::GetBlockAddressSymbol(const BasicBlock *BB) const {
 
 /// GetCPISymbol - Return the symbol for the specified constant pool entry.
 MCSymbol *AsmPrinter::GetCPISymbol(unsigned CPID) const {
-  const DataLayout *DL = TM.getDataLayout();
-  return OutContext.GetOrCreateSymbol
-    (Twine(DL->getPrivateGlobalPrefix()) + "CPI" + Twine(getFunctionNumber())
-     + "_" + Twine(CPID));
+  const DataLayout &DL = getDataLayout();
+  return OutContext.getOrCreateSymbol(Twine(DL.getPrivateGlobalPrefix()) +
+                                      "CPI" + Twine(getFunctionNumber()) + "_" +
+                                      Twine(CPID));
 }
 
 /// GetJTISymbol - Return the symbol for the specified jump table entry.
@@ -2312,10 +2374,10 @@ MCSymbol *AsmPrinter::GetJTISymbol(unsigned JTID, bool isLinkerPrivate) const {
 /// GetJTSetSymbol - Return the symbol for the specified jump table .set
 /// FIXME: privatize to AsmPrinter.
 MCSymbol *AsmPrinter::GetJTSetSymbol(unsigned UID, unsigned MBBID) const {
-  const DataLayout *DL = TM.getDataLayout();
-  return OutContext.GetOrCreateSymbol
-  (Twine(DL->getPrivateGlobalPrefix()) + Twine(getFunctionNumber()) + "_" +
-   Twine(UID) + "_set_" + Twine(MBBID));
+  const DataLayout &DL = getDataLayout();
+  return OutContext.getOrCreateSymbol(Twine(DL.getPrivateGlobalPrefix()) +
+                                      Twine(getFunctionNumber()) + "_" +
+                                      Twine(UID) + "_set_" + Twine(MBBID));
 }
 
 MCSymbol *AsmPrinter::getSymbolWithGlobalValueBase(const GlobalValue *GV,
@@ -2324,12 +2386,11 @@ MCSymbol *AsmPrinter::getSymbolWithGlobalValueBase(const GlobalValue *GV,
                                                            TM);
 }
 
-/// GetExternalSymbolSymbol - Return the MCSymbol for the specified
-/// ExternalSymbol.
+/// Return the MCSymbol for the specified ExternalSymbol.
 MCSymbol *AsmPrinter::GetExternalSymbolSymbol(StringRef Sym) const {
   SmallString<60> NameStr;
-  Mang->getNameWithPrefix(NameStr, Sym);
-  return OutContext.GetOrCreateSymbol(NameStr);
+  Mangler::getNameWithPrefix(NameStr, Sym, getDataLayout());
+  return OutContext.getOrCreateSymbol(NameStr);
 }
 
 
@@ -2374,16 +2435,16 @@ static void emitBasicBlockLoopComments(const MachineBasicBlock &MBB,
   // If this block is not a loop header, just print out what is the loop header
   // and return.
   if (Header != &MBB) {
-    AP.OutStreamer.AddComment("  in Loop: Header=BB" +
-                              Twine(AP.getFunctionNumber())+"_" +
-                              Twine(Loop->getHeader()->getNumber())+
-                              " Depth="+Twine(Loop->getLoopDepth()));
+    AP.OutStreamer->AddComment("  in Loop: Header=BB" +
+                               Twine(AP.getFunctionNumber())+"_" +
+                               Twine(Loop->getHeader()->getNumber())+
+                               " Depth="+Twine(Loop->getLoopDepth()));
     return;
   }
 
   // Otherwise, it is a loop header.  Print out information about child and
   // parent loops.
-  raw_ostream &OS = AP.OutStreamer.GetCommentOS();
+  raw_ostream &OS = AP.OutStreamer->GetCommentOS();
 
   PrintParentLoopComment(OS, Loop->getParentLoop(), AP.getFunctionNumber());
 
@@ -2414,18 +2475,17 @@ void AsmPrinter::EmitBasicBlockStart(const MachineBasicBlock &MBB) const {
   if (MBB.hasAddressTaken()) {
     const BasicBlock *BB = MBB.getBasicBlock();
     if (isVerbose())
-      OutStreamer.AddComment("Block address taken");
+      OutStreamer->AddComment("Block address taken");
 
-    std::vector<MCSymbol*> Symbols = MMI->getAddrLabelSymbolToEmit(BB);
-    for (auto *Sym : Symbols)
-      OutStreamer.EmitLabel(Sym);
+    for (MCSymbol *Sym : MMI->getAddrLabelSymbolToEmit(BB))
+      OutStreamer->EmitLabel(Sym);
   }
 
   // Print some verbose block comments.
   if (isVerbose()) {
     if (const BasicBlock *BB = MBB.getBasicBlock())
       if (BB->hasName())
-        OutStreamer.AddComment("%" + BB->getName());
+        OutStreamer->AddComment("%" + BB->getName());
     emitBasicBlockLoopComments(MBB, LI, *this);
   }
 
@@ -2433,10 +2493,10 @@ void AsmPrinter::EmitBasicBlockStart(const MachineBasicBlock &MBB) const {
   if (MBB.pred_empty() || isBlockOnlyReachableByFallthrough(&MBB)) {
     if (isVerbose()) {
       // NOTE: Want this comment at start of line, don't emit with AddComment.
-      OutStreamer.emitRawComment(" BB#" + Twine(MBB.getNumber()) + ":", false);
+      OutStreamer->emitRawComment(" BB#" + Twine(MBB.getNumber()) + ":", false);
     }
   } else {
-    OutStreamer.EmitLabel(MBB.getSymbol());
+    OutStreamer->EmitLabel(MBB.getSymbol());
   }
 }
 
@@ -2458,7 +2518,7 @@ void AsmPrinter::EmitVisibility(MCSymbol *Sym, unsigned Visibility,
   }
 
   if (Attr != MCSA_Invalid)
-    OutStreamer.EmitSymbolAttribute(Sym, Attr);
+    OutStreamer->EmitSymbolAttribute(Sym, Attr);
 }
 
 /// isBlockOnlyReachableByFallthough - Return true if the basic block has
diff --git a/lib/CodeGen/AsmPrinter/AsmPrinterDwarf.cpp b/lib/CodeGen/AsmPrinter/AsmPrinterDwarf.cpp
index 9de36da51a0..45a8820c461 100644
--- a/lib/CodeGen/AsmPrinter/AsmPrinterDwarf.cpp
+++ b/lib/CodeGen/AsmPrinter/AsmPrinterDwarf.cpp
@@ -42,30 +42,30 @@ using namespace llvm;
 /// EmitSLEB128 - emit the specified signed leb128 value.
 void AsmPrinter::EmitSLEB128(int64_t Value, const char *Desc) const {
   if (isVerbose() && Desc)
-    OutStreamer.AddComment(Desc);
+    OutStreamer->AddComment(Desc);
 
-  OutStreamer.EmitSLEB128IntValue(Value);
+  OutStreamer->EmitSLEB128IntValue(Value);
 }
 
 /// EmitULEB128 - emit the specified signed leb128 value.
 void AsmPrinter::EmitULEB128(uint64_t Value, const char *Desc,
                              unsigned PadTo) const {
   if (isVerbose() && Desc)
-    OutStreamer.AddComment(Desc);
+    OutStreamer->AddComment(Desc);
 
-  OutStreamer.EmitULEB128IntValue(Value, PadTo);
+  OutStreamer->EmitULEB128IntValue(Value, PadTo);
 }
 
 /// EmitCFAByte - Emit a .byte 42 directive for a DW_CFA_xxx value.
 void AsmPrinter::EmitCFAByte(unsigned Val) const {
   if (isVerbose()) {
     if (Val >= dwarf::DW_CFA_offset && Val < dwarf::DW_CFA_offset + 64)
-      OutStreamer.AddComment("DW_CFA_offset + Reg (" +
-                             Twine(Val - dwarf::DW_CFA_offset) + ")");
+      OutStreamer->AddComment("DW_CFA_offset + Reg (" +
+                              Twine(Val - dwarf::DW_CFA_offset) + ")");
     else
-      OutStreamer.AddComment(dwarf::CallFrameString(Val));
+      OutStreamer->AddComment(dwarf::CallFrameString(Val));
   }
-  OutStreamer.EmitIntValue(Val, 1);
+  OutStreamer->EmitIntValue(Val, 1);
 }
 
 static const char *DecodeDWARFEncoding(unsigned Encoding) {
@@ -116,13 +116,13 @@ static const char *DecodeDWARFEncoding(unsigned Encoding) {
 void AsmPrinter::EmitEncodingByte(unsigned Val, const char *Desc) const {
   if (isVerbose()) {
     if (Desc)
-      OutStreamer.AddComment(Twine(Desc) + " Encoding = " +
-                             Twine(DecodeDWARFEncoding(Val)));
+      OutStreamer->AddComment(Twine(Desc) + " Encoding = " +
+                              Twine(DecodeDWARFEncoding(Val)));
     else
-      OutStreamer.AddComment(Twine("Encoding = ") + DecodeDWARFEncoding(Val));
+      OutStreamer->AddComment(Twine("Encoding = ") + DecodeDWARFEncoding(Val));
   }
 
-  OutStreamer.EmitIntValue(Val, 1);
+  OutStreamer->EmitIntValue(Val, 1);
 }
 
 /// GetSizeOfEncodedValue - Return the size of the encoding in bytes.
@@ -134,7 +134,7 @@ unsigned AsmPrinter::GetSizeOfEncodedValue(unsigned Encoding) const {
   default:
     llvm_unreachable("Invalid encoded value.");
   case dwarf::DW_EH_PE_absptr:
-    return TM.getDataLayout()->getPointerSize();
+    return MF->getDataLayout().getPointerSize();
   case dwarf::DW_EH_PE_udata2:
     return 2;
   case dwarf::DW_EH_PE_udata4:
@@ -150,34 +150,41 @@ void AsmPrinter::EmitTTypeReference(const GlobalValue *GV,
     const TargetLoweringObjectFile &TLOF = getObjFileLowering();
 
     const MCExpr *Exp =
-        TLOF.getTTypeGlobalReference(GV, Encoding, *Mang, TM, MMI, OutStreamer);
-    OutStreamer.EmitValue(Exp, GetSizeOfEncodedValue(Encoding));
+        TLOF.getTTypeGlobalReference(GV, Encoding, *Mang, TM, MMI,
+                                     *OutStreamer);
+    OutStreamer->EmitValue(Exp, GetSizeOfEncodedValue(Encoding));
   } else
-    OutStreamer.EmitIntValue(0, GetSizeOfEncodedValue(Encoding));
+    OutStreamer->EmitIntValue(0, GetSizeOfEncodedValue(Encoding));
 }
 
-/// EmitSectionOffset - Emit the 4-byte offset of Label from the start of its
-/// section.  This can be done with a special directive if the target supports
-/// it (e.g. cygwin) or by emitting it as an offset from a label at the start
-/// of the section.
-///
-/// SectionLabel is a temporary label emitted at the start of the section that
-/// Label lives in.
-void AsmPrinter::emitSectionOffset(const MCSymbol *Label) const {
-  // On COFF targets, we have to emit the special .secrel32 directive.
-  if (MAI->needsDwarfSectionOffsetDirective()) {
-    OutStreamer.EmitCOFFSecRel32(Label);
-    return;
+void AsmPrinter::emitDwarfSymbolReference(const MCSymbol *Label,
+                                          bool ForceOffset) const {
+  if (!ForceOffset) {
+    // On COFF targets, we have to emit the special .secrel32 directive.
+    if (MAI->needsDwarfSectionOffsetDirective()) {
+      OutStreamer->EmitCOFFSecRel32(Label);
+      return;
+    }
+
+    // If the format uses relocations with dwarf, refer to the symbol directly.
+    if (MAI->doesDwarfUseRelocationsAcrossSections()) {
+      OutStreamer->EmitSymbolValue(Label, 4);
+      return;
+    }
   }
 
-  // If the format uses relocations with dwarf, refer to the symbol directly.
+  // Otherwise, emit it as a label difference from the start of the section.
+  EmitLabelDifference(Label, Label->getSection().getBeginSymbol(), 4);
+}
+
+void AsmPrinter::emitDwarfStringOffset(DwarfStringPoolEntryRef S) const {
   if (MAI->doesDwarfUseRelocationsAcrossSections()) {
-    OutStreamer.EmitSymbolValue(Label, 4);
+    emitDwarfSymbolReference(S.getSymbol());
     return;
   }
 
-  // Otherwise, emit it as a label difference from the start of the section.
-  EmitLabelDifference(Label, Label->getSection().getBeginSymbol(), 4);
+  // Just emit the offset directly; no need for symbol math.
+  EmitInt32(S.getOffset());
 }
 
 /// EmitDwarfRegOp - Emit dwarf register operation.
@@ -219,67 +226,60 @@ void AsmPrinter::emitCFIInstruction(const MCCFIInstruction &Inst) const {
   default:
     llvm_unreachable("Unexpected instruction");
   case MCCFIInstruction::OpDefCfaOffset:
-    OutStreamer.EmitCFIDefCfaOffset(Inst.getOffset());
+    OutStreamer->EmitCFIDefCfaOffset(Inst.getOffset());
     break;
   case MCCFIInstruction::OpDefCfa:
-    OutStreamer.EmitCFIDefCfa(Inst.getRegister(), Inst.getOffset());
+    OutStreamer->EmitCFIDefCfa(Inst.getRegister(), Inst.getOffset());
     break;
   case MCCFIInstruction::OpDefCfaRegister:
-    OutStreamer.EmitCFIDefCfaRegister(Inst.getRegister());
+    OutStreamer->EmitCFIDefCfaRegister(Inst.getRegister());
     break;
   case MCCFIInstruction::OpOffset:
-    OutStreamer.EmitCFIOffset(Inst.getRegister(), Inst.getOffset());
+    OutStreamer->EmitCFIOffset(Inst.getRegister(), Inst.getOffset());
     break;
   case MCCFIInstruction::OpRegister:
-    OutStreamer.EmitCFIRegister(Inst.getRegister(), Inst.getRegister2());
+    OutStreamer->EmitCFIRegister(Inst.getRegister(), Inst.getRegister2());
     break;
   case MCCFIInstruction::OpWindowSave:
-    OutStreamer.EmitCFIWindowSave();
+    OutStreamer->EmitCFIWindowSave();
     break;
   case MCCFIInstruction::OpSameValue:
-    OutStreamer.EmitCFISameValue(Inst.getRegister());
+    OutStreamer->EmitCFISameValue(Inst.getRegister());
     break;
   }
 }
 
 void AsmPrinter::emitDwarfDIE(const DIE &Die) const {
-  // Get the abbreviation for this DIE.
-  const DIEAbbrev &Abbrev = Die.getAbbrev();
-
   // Emit the code (index) for the abbreviation.
   if (isVerbose())
-    OutStreamer.AddComment("Abbrev [" + Twine(Abbrev.getNumber()) +
-                           "] 0x" + Twine::utohexstr(Die.getOffset()) +
-                           ":0x" + Twine::utohexstr(Die.getSize()) + " " +
-                           dwarf::TagString(Abbrev.getTag()));
-  EmitULEB128(Abbrev.getNumber());
-
-  const SmallVectorImpl<DIEValue *> &Values = Die.getValues();
-  const SmallVectorImpl<DIEAbbrevData> &AbbrevData = Abbrev.getData();
+    OutStreamer->AddComment("Abbrev [" + Twine(Die.getAbbrevNumber()) + "] 0x" +
+                            Twine::utohexstr(Die.getOffset()) + ":0x" +
+                            Twine::utohexstr(Die.getSize()) + " " +
+                            dwarf::TagString(Die.getTag()));
+  EmitULEB128(Die.getAbbrevNumber());
 
   // Emit the DIE attribute values.
-  for (unsigned i = 0, N = Values.size(); i < N; ++i) {
-    dwarf::Attribute Attr = AbbrevData[i].getAttribute();
-    dwarf::Form Form = AbbrevData[i].getForm();
-    assert(Form && "Too many attributes for DIE (check abbreviation)");
+  for (const auto &V : Die.values()) {
+    dwarf::Attribute Attr = V.getAttribute();
+    assert(V.getForm() && "Too many attributes for DIE (check abbreviation)");
 
     if (isVerbose()) {
-      OutStreamer.AddComment(dwarf::AttributeString(Attr));
+      OutStreamer->AddComment(dwarf::AttributeString(Attr));
       if (Attr == dwarf::DW_AT_accessibility)
-        OutStreamer.AddComment(dwarf::AccessibilityString(
-            cast<DIEInteger>(Values[i])->getValue()));
+        OutStreamer->AddComment(
+            dwarf::AccessibilityString(V.getDIEInteger().getValue()));
     }
 
     // Emit an attribute using the defined form.
-    Values[i]->EmitValue(this, Form);
+    V.EmitValue(this);
   }
 
   // Emit the DIE children if any.
-  if (Abbrev.hasChildren()) {
-    for (auto &Child : Die.getChildren())
-      emitDwarfDIE(*Child);
+  if (Die.hasChildren()) {
+    for (auto &Child : Die.children())
+      emitDwarfDIE(Child);
 
-    OutStreamer.AddComment("End Of Children Mark");
+    OutStreamer->AddComment("End Of Children Mark");
     EmitInt8(0);
   }
 }
diff --git a/lib/CodeGen/AsmPrinter/AsmPrinterInlineAsm.cpp b/lib/CodeGen/AsmPrinter/AsmPrinterInlineAsm.cpp
index 2cbe038540b..96c9dc8b1bc 100644
--- a/lib/CodeGen/AsmPrinter/AsmPrinterInlineAsm.cpp
+++ b/lib/CodeGen/AsmPrinter/AsmPrinterInlineAsm.cpp
@@ -74,13 +74,9 @@ static void srcMgrDiagHandler(const SMDiagnostic &Diag, void *diagInfo) {
 
 /// EmitInlineAsm - Emit a blob of inline asm to the output streamer.
 void AsmPrinter::EmitInlineAsm(StringRef Str, const MCSubtargetInfo &STI,
+                               const MCTargetOptions &MCOptions,
                                const MDNode *LocMDNode,
                                InlineAsm::AsmDialect Dialect) const {
-#if defined(__native_client__)
-  // Prune the generic AsmParser bits from the in-browser translator.
-  // This is normally used to parse inline asm (see createMCAsmParser below).
-  return;
-#else
   assert(!Str.empty() && "Can't emit empty inline asm block");
 
   // Remember if the buffer is nul terminated or not so we can avoid a copy.
@@ -96,9 +92,9 @@ void AsmPrinter::EmitInlineAsm(StringRef Str, const MCSubtargetInfo &STI,
   const MCAsmInfo *MCAI = TM.getMCAsmInfo();
   assert(MCAI && "No MCAsmInfo");
   if (!MCAI->useIntegratedAssembler() &&
-      !OutStreamer.isIntegratedAssemblerRequired()) {
+      !OutStreamer->isIntegratedAssemblerRequired()) {
     emitInlineAsmStart();
-    OutStreamer.EmitRawText(Str);
+    OutStreamer->EmitRawText(Str);
     emitInlineAsmEnd(STI, nullptr);
     return;
   }
@@ -129,7 +125,7 @@ void AsmPrinter::EmitInlineAsm(StringRef Str, const MCSubtargetInfo &STI,
   SrcMgr.AddNewSourceBuffer(std::move(Buffer), SMLoc());
 
   std::unique_ptr<MCAsmParser> Parser(
-      createMCAsmParser(SrcMgr, OutContext, OutStreamer, *MAI));
+      createMCAsmParser(SrcMgr, OutContext, *OutStreamer, *MAI));
 
   // Create a temporary copy of the original STI because the parser may modify
   // it. For example, when switching between arm and thumb mode. If the target
@@ -143,7 +139,7 @@ void AsmPrinter::EmitInlineAsm(StringRef Str, const MCSubtargetInfo &STI,
   // because it's not subtarget dependent.
   std::unique_ptr<MCInstrInfo> MII(TM.getTarget().createMCInstrInfo());
   std::unique_ptr<MCTargetAsmParser> TAP(TM.getTarget().createMCAsmParser(
-      TmpSTI, *Parser, *MII, TM.Options.MCOptions));
+      TmpSTI, *Parser, *MII, MCOptions));
   if (!TAP)
     report_fatal_error("Inline asm not supported by this streamer because"
                        " we don't have an asm parser for this target\n");
@@ -161,7 +157,6 @@ void AsmPrinter::EmitInlineAsm(StringRef Str, const MCSubtargetInfo &STI,
   emitInlineAsmEnd(STI, &TmpSTI);
   if (Res && !HasDiagHandler)
     report_fatal_error("Error parsing inline asm\n");
-#endif  // defined(__native_client__)
 }
 
 static void EmitMSInlineAsmStr(const char *AsmStr, const MachineInstr *MI,
@@ -407,10 +402,11 @@ static void EmitGCCInlineAsmStr(const char *AsmStr, const MachineInstr *MI,
           unsigned OpFlags = MI->getOperand(OpNo).getImm();
           ++OpNo;  // Skip over the ID number.
 
-          if (Modifier[0] == 'l')  // labels are target independent
+          if (Modifier[0] == 'l') { // Labels are target independent.
             // FIXME: What if the operand isn't an MBB, report error?
-            OS << *MI->getOperand(OpNo).getMBB()->getSymbol();
-          else {
+            const MCSymbol *Sym = MI->getOperand(OpNo).getMBB()->getSymbol();
+            Sym->print(OS, AP->MAI);
+          } else {
             if (InlineAsm::isMemKind(OpFlags)) {
               Error = AP->PrintAsmMemoryOperand(MI, OpNo, InlineAsmVariant,
                                                 Modifier[0] ? Modifier : nullptr,
@@ -438,10 +434,6 @@ static void EmitGCCInlineAsmStr(const char *AsmStr, const MachineInstr *MI,
 /// EmitInlineAsm - This method formats and emits the specified machine
 /// instruction that is an inline asm.
 void AsmPrinter::EmitInlineAsm(const MachineInstr *MI) const {
-#if defined(__native__client__)
-  // See above LOCALMOD for pruning generic AsmParsing.
-  return;
-#else
   assert(MI->isInlineAsm() && "printInlineAsm only works on inline asms");
 
   // Count the number of register definitions to find the asm string.
@@ -458,14 +450,14 @@ void AsmPrinter::EmitInlineAsm(const MachineInstr *MI) const {
   // If this asmstr is empty, just print the #APP/#NOAPP markers.
   // These are useful to see where empty asm's wound up.
   if (AsmStr[0] == 0) {
-    OutStreamer.emitRawComment(MAI->getInlineAsmStart());
-    OutStreamer.emitRawComment(MAI->getInlineAsmEnd());
+    OutStreamer->emitRawComment(MAI->getInlineAsmStart());
+    OutStreamer->emitRawComment(MAI->getInlineAsmEnd());
     return;
   }
 
   // Emit the #APP start marker.  This has to happen even if verbose-asm isn't
   // enabled, so we use emitRawComment.
-  OutStreamer.emitRawComment(MAI->getInlineAsmStart());
+  OutStreamer->emitRawComment(MAI->getInlineAsmStart());
 
   // Get the !srcloc metadata node if we have it, and decode the loc cookie from
   // it.
@@ -498,12 +490,17 @@ void AsmPrinter::EmitInlineAsm(const MachineInstr *MI) const {
   else
     EmitMSInlineAsmStr(AsmStr, MI, MMI, InlineAsmVariant, AP, LocCookie, OS);
 
-  EmitInlineAsm(OS.str(), getSubtargetInfo(), LocMD, MI->getInlineAsmDialect());
+  // Reset SanitizeAddress based on the function's attribute.
+  MCTargetOptions MCOptions = TM.Options.MCOptions;
+  MCOptions.SanitizeAddress =
+      MF->getFunction()->hasFnAttribute(Attribute::SanitizeAddress);
+
+  EmitInlineAsm(OS.str(), getSubtargetInfo(), MCOptions, LocMD,
+                MI->getInlineAsmDialect());
 
   // Emit the #NOAPP end marker.  This has to happen even if verbose-asm isn't
   // enabled, so we use emitRawComment.
-  OutStreamer.emitRawComment(MAI->getInlineAsmEnd());
-#endif  // __native_client__
+  OutStreamer->emitRawComment(MAI->getInlineAsmEnd());
 }
 
 
@@ -515,9 +512,9 @@ void AsmPrinter::EmitInlineAsm(const MachineInstr *MI) const {
 /// for their own strange codes.
 void AsmPrinter::PrintSpecial(const MachineInstr *MI, raw_ostream &OS,
                               const char *Code) const {
-  const DataLayout *DL = TM.getDataLayout();
   if (!strcmp(Code, "private")) {
-    OS << DL->getPrivateGlobalPrefix();
+    const DataLayout &DL = MF->getDataLayout();
+    OS << DL.getPrivateGlobalPrefix();
   } else if (!strcmp(Code, "comment")) {
     OS << MAI->getCommentString();
   } else if (!strcmp(Code, "uid")) {
diff --git a/lib/CodeGen/AsmPrinter/ByteStreamer.h b/lib/CodeGen/AsmPrinter/ByteStreamer.h
index 179a4d49582..0cc829fffc5 100644
--- a/lib/CodeGen/AsmPrinter/ByteStreamer.h
+++ b/lib/CodeGen/AsmPrinter/ByteStreamer.h
@@ -40,15 +40,15 @@ class APByteStreamer : public ByteStreamer {
 public:
   APByteStreamer(AsmPrinter &Asm) : AP(Asm) {}
   void EmitInt8(uint8_t Byte, const Twine &Comment) override {
-    AP.OutStreamer.AddComment(Comment);
+    AP.OutStreamer->AddComment(Comment);
     AP.EmitInt8(Byte);
   }
   void EmitSLEB128(uint64_t DWord, const Twine &Comment) override {
-    AP.OutStreamer.AddComment(Comment);
+    AP.OutStreamer->AddComment(Comment);
     AP.EmitSLEB128(DWord);
   }
   void EmitULEB128(uint64_t DWord, const Twine &Comment) override {
-    AP.OutStreamer.AddComment(Comment);
+    AP.OutStreamer->AddComment(Comment);
     AP.EmitULEB128(DWord);
   }
 };
@@ -72,26 +72,34 @@ class HashingByteStreamer : public ByteStreamer {
 class BufferByteStreamer : public ByteStreamer {
 private:
   SmallVectorImpl<char> &Buffer;
-  // FIXME: This is actually only needed for textual asm output.
   SmallVectorImpl<std::string> &Comments;
 
+  /// \brief Only verbose textual output needs comments.  This will be set to
+  /// true for that case, and false otherwise.  If false, comments passed in to
+  /// the emit methods will be ignored.
+  bool GenerateComments;
+
 public:
   BufferByteStreamer(SmallVectorImpl<char> &Buffer,
-                     SmallVectorImpl<std::string> &Comments)
-  : Buffer(Buffer), Comments(Comments) {}
+                     SmallVectorImpl<std::string> &Comments,
+                     bool GenerateComments)
+  : Buffer(Buffer), Comments(Comments), GenerateComments(GenerateComments) {}
   void EmitInt8(uint8_t Byte, const Twine &Comment) override {
     Buffer.push_back(Byte);
-    Comments.push_back(Comment.str());
+    if (GenerateComments)
+      Comments.push_back(Comment.str());
   }
   void EmitSLEB128(uint64_t DWord, const Twine &Comment) override {
     raw_svector_ostream OSE(Buffer);
     encodeSLEB128(DWord, OSE);
-    Comments.push_back(Comment.str());
+    if (GenerateComments)
+      Comments.push_back(Comment.str());
   }
   void EmitULEB128(uint64_t DWord, const Twine &Comment) override {
     raw_svector_ostream OSE(Buffer);
     encodeULEB128(DWord, OSE);
-    Comments.push_back(Comment.str());
+    if (GenerateComments)
+      Comments.push_back(Comment.str());
   }
 };
 
diff --git a/lib/CodeGen/AsmPrinter/CMakeLists.txt b/lib/CodeGen/AsmPrinter/CMakeLists.txt
index 01d2c7220ab..ba2f61a4482 100644
--- a/lib/CodeGen/AsmPrinter/CMakeLists.txt
+++ b/lib/CodeGen/AsmPrinter/CMakeLists.txt
@@ -5,6 +5,7 @@ add_llvm_library(LLVMAsmPrinter
   AsmPrinterDwarf.cpp
   AsmPrinterInlineAsm.cpp
   DbgValueHistoryCalculator.cpp
+  DebugLocStream.cpp
   DIE.cpp
   DIEHash.cpp
   DwarfAccelTable.cpp
@@ -18,7 +19,7 @@ add_llvm_library(LLVMAsmPrinter
   EHStreamer.cpp
   ErlangGCPrinter.cpp
   OcamlGCPrinter.cpp
-  Win64Exception.cpp
+  WinException.cpp
   WinCodeViewLineTables.cpp
   )
 
diff --git a/lib/CodeGen/AsmPrinter/DIE.cpp b/lib/CodeGen/AsmPrinter/DIE.cpp
index 1a706f7f0d1..e867ddcf340 100644
--- a/lib/CodeGen/AsmPrinter/DIE.cpp
+++ b/lib/CodeGen/AsmPrinter/DIE.cpp
@@ -107,6 +107,13 @@ void DIEAbbrev::print(raw_ostream &O) {
 void DIEAbbrev::dump() { print(dbgs()); }
 #endif
 
+DIEAbbrev DIE::generateAbbrev() const {
+  DIEAbbrev Abbrev(Tag, hasChildren());
+  for (const DIEValue &V : Values)
+    Abbrev.AddAttribute(V.getAttribute(), V.getForm());
+  return Abbrev;
+}
+
 /// Climb up the parent chain to get the unit DIE to which this DIE
 /// belongs.
 const DIE *DIE::getUnit() const {
@@ -128,22 +135,19 @@ const DIE *DIE::getUnitOrNull() const {
   return nullptr;
 }
 
-DIEValue *DIE::findAttribute(dwarf::Attribute Attribute) const {
-  const SmallVectorImpl<DIEValue *> &Values = getValues();
-  const DIEAbbrev &Abbrevs = getAbbrev();
-
+DIEValue DIE::findAttribute(dwarf::Attribute Attribute) const {
   // Iterate through all the attributes until we find the one we're
   // looking for, if we can't find it return NULL.
-  for (size_t i = 0; i < Values.size(); ++i)
-    if (Abbrevs.getData()[i].getAttribute() == Attribute)
-      return Values[i];
-  return nullptr;
+  for (const auto &V : values())
+    if (V.getAttribute() == Attribute)
+      return V;
+  return DIEValue();
 }
 
 #ifndef NDEBUG
 void DIE::print(raw_ostream &O, unsigned IndentCount) const {
   const std::string Indent(IndentCount, ' ');
-  bool isBlock = Abbrev.getTag() == 0;
+  bool isBlock = getTag() == 0;
 
   if (!isBlock) {
     O << Indent
@@ -153,35 +157,31 @@ void DIE::print(raw_ostream &O, unsigned IndentCount) const {
       << ", Size: " << Size << "\n";
 
     O << Indent
-      << dwarf::TagString(Abbrev.getTag())
+      << dwarf::TagString(getTag())
       << " "
-      << dwarf::ChildrenString(Abbrev.hasChildren()) << "\n";
+      << dwarf::ChildrenString(hasChildren()) << "\n";
   } else {
     O << "Size: " << Size << "\n";
   }
 
-  const SmallVectorImpl<DIEAbbrevData> &Data = Abbrev.getData();
-
   IndentCount += 2;
-  for (unsigned i = 0, N = Data.size(); i < N; ++i) {
+  unsigned I = 0;
+  for (const auto &V : Values) {
     O << Indent;
 
     if (!isBlock)
-      O << dwarf::AttributeString(Data[i].getAttribute());
+      O << dwarf::AttributeString(V.getAttribute());
     else
-      O << "Blk[" << i << "]";
+      O << "Blk[" << I++ << "]";
 
-    O <<  "  "
-      << dwarf::FormEncodingString(Data[i].getForm())
-      << " ";
-    Values[i]->print(O);
+    O << "  " << dwarf::FormEncodingString(V.getForm()) << " ";
+    V.print(O);
     O << "\n";
   }
   IndentCount -= 2;
 
-  for (unsigned j = 0, M = Children.size(); j < M; ++j) {
-    Children[j]->print(O, IndentCount+4);
-  }
+  for (const auto &Child : children())
+    Child.print(O, IndentCount + 4);
 
   if (!isBlock) O << "\n";
 }
@@ -191,9 +191,43 @@ void DIE::dump() {
 }
 #endif
 
-void DIEValue::anchor() { }
+void DIEValue::EmitValue(const AsmPrinter *AP) const {
+  switch (Ty) {
+  case isNone:
+    llvm_unreachable("Expected valid DIEValue");
+#define HANDLE_DIEVALUE(T)                                                     \
+  case is##T:                                                                  \
+    getDIE##T().EmitValue(AP, Form);                                           \
+    break;
+#include "llvm/CodeGen/DIEValue.def"
+  }
+}
+
+unsigned DIEValue::SizeOf(const AsmPrinter *AP) const {
+  switch (Ty) {
+  case isNone:
+    llvm_unreachable("Expected valid DIEValue");
+#define HANDLE_DIEVALUE(T)                                                     \
+  case is##T:                                                                  \
+    return getDIE##T().SizeOf(AP, Form);
+#include "llvm/CodeGen/DIEValue.def"
+  }
+  llvm_unreachable("Unknown DIE kind");
+}
 
 #ifndef NDEBUG
+void DIEValue::print(raw_ostream &O) const {
+  switch (Ty) {
+  case isNone:
+    llvm_unreachable("Expected valid DIEValue");
+#define HANDLE_DIEVALUE(T)                                                     \
+  case is##T:                                                                  \
+    getDIE##T().print(O);                                                      \
+    break;
+#include "llvm/CodeGen/DIEValue.def"
+  }
+}
+
 void DIEValue::dump() const {
   print(dbgs());
 }
@@ -211,7 +245,7 @@ void DIEInteger::EmitValue(const AsmPrinter *Asm, dwarf::Form Form) const {
   case dwarf::DW_FORM_flag_present:
     // Emit something to keep the lines and comments in sync.
     // FIXME: Is there a better way to do this?
-    Asm->OutStreamer.AddBlankLine();
+    Asm->OutStreamer->AddBlankLine();
     return;
   case dwarf::DW_FORM_flag:  // Fall thru
   case dwarf::DW_FORM_ref1:  // Fall thru
@@ -230,13 +264,14 @@ void DIEInteger::EmitValue(const AsmPrinter *Asm, dwarf::Form Form) const {
   case dwarf::DW_FORM_udata: Asm->EmitULEB128(Integer); return;
   case dwarf::DW_FORM_sdata: Asm->EmitSLEB128(Integer); return;
   case dwarf::DW_FORM_addr:
-    Size = Asm->getDataLayout().getPointerSize(); break;
+    Size = Asm->getPointerSize();
+    break;
   case dwarf::DW_FORM_ref_addr:
     Size = SizeOf(Asm, dwarf::DW_FORM_ref_addr);
     break;
   default: llvm_unreachable("DIE Value form not supported yet");
   }
-  Asm->OutStreamer.EmitIntValue(Integer, Size);
+  Asm->OutStreamer->EmitIntValue(Integer, Size);
 }
 
 /// SizeOf - Determine size of integer value in bytes.
@@ -260,10 +295,11 @@ unsigned DIEInteger::SizeOf(const AsmPrinter *AP, dwarf::Form Form) const {
   case dwarf::DW_FORM_GNU_addr_index: return getULEB128Size(Integer);
   case dwarf::DW_FORM_udata: return getULEB128Size(Integer);
   case dwarf::DW_FORM_sdata: return getSLEB128Size(Integer);
-  case dwarf::DW_FORM_addr:  return AP->getDataLayout().getPointerSize();
+  case dwarf::DW_FORM_addr:
+    return AP->getPointerSize();
   case dwarf::DW_FORM_ref_addr:
-    if (AP->OutStreamer.getContext().getDwarfVersion() == 2)
-      return AP->getDataLayout().getPointerSize();
+    if (AP->OutStreamer->getContext().getDwarfVersion() == 2)
+      return AP->getPointerSize();
     return sizeof(int32_t);
   default: llvm_unreachable("DIE Value form not supported yet");
   }
@@ -283,7 +319,7 @@ void DIEInteger::print(raw_ostream &O) const {
 /// EmitValue - Emit expression value.
 ///
 void DIEExpr::EmitValue(const AsmPrinter *AP, dwarf::Form Form) const {
-  AP->OutStreamer.EmitValue(Expr, SizeOf(AP, Form));
+  AP->OutStreamer->EmitValue(Expr, SizeOf(AP, Form));
 }
 
 /// SizeOf - Determine size of expression value in bytes.
@@ -292,14 +328,11 @@ unsigned DIEExpr::SizeOf(const AsmPrinter *AP, dwarf::Form Form) const {
   if (Form == dwarf::DW_FORM_data4) return 4;
   if (Form == dwarf::DW_FORM_sec_offset) return 4;
   if (Form == dwarf::DW_FORM_strp) return 4;
-  return AP->getDataLayout().getPointerSize();
+  return AP->getPointerSize();
 }
 
 #ifndef NDEBUG
-void DIEExpr::print(raw_ostream &O) const {
-  O << "Expr: ";
-  Expr->print(O);
-}
+void DIEExpr::print(raw_ostream &O) const { O << "Expr: " << *Expr; }
 #endif
 
 //===----------------------------------------------------------------------===//
@@ -321,13 +354,11 @@ unsigned DIELabel::SizeOf(const AsmPrinter *AP, dwarf::Form Form) const {
   if (Form == dwarf::DW_FORM_data4) return 4;
   if (Form == dwarf::DW_FORM_sec_offset) return 4;
   if (Form == dwarf::DW_FORM_strp) return 4;
-  return AP->getDataLayout().getPointerSize();
+  return AP->getPointerSize();
 }
 
 #ifndef NDEBUG
-void DIELabel::print(raw_ostream &O) const {
-  O << "Lbl: " << Label->getName();
-}
+void DIELabel::print(raw_ostream &O) const { O << "Lbl: " << Label->getName(); }
 #endif
 
 //===----------------------------------------------------------------------===//
@@ -346,7 +377,7 @@ unsigned DIEDelta::SizeOf(const AsmPrinter *AP, dwarf::Form Form) const {
   if (Form == dwarf::DW_FORM_data4) return 4;
   if (Form == dwarf::DW_FORM_sec_offset) return 4;
   if (Form == dwarf::DW_FORM_strp) return 4;
-  return AP->getDataLayout().getPointerSize();
+  return AP->getPointerSize();
 }
 
 #ifndef NDEBUG
@@ -362,19 +393,49 @@ void DIEDelta::print(raw_ostream &O) const {
 /// EmitValue - Emit string value.
 ///
 void DIEString::EmitValue(const AsmPrinter *AP, dwarf::Form Form) const {
-  Access->EmitValue(AP, Form);
+  assert(
+      (Form == dwarf::DW_FORM_strp || Form == dwarf::DW_FORM_GNU_str_index) &&
+      "Expected valid string form");
+
+  // Index of string in symbol table.
+  if (Form == dwarf::DW_FORM_GNU_str_index) {
+    DIEInteger(S.getIndex()).EmitValue(AP, Form);
+    return;
+  }
+
+  // Relocatable symbol.
+  assert(Form == dwarf::DW_FORM_strp);
+  if (AP->MAI->doesDwarfUseRelocationsAcrossSections()) {
+    DIELabel(S.getSymbol()).EmitValue(AP, Form);
+    return;
+  }
+
+  // Offset into symbol table.
+  DIEInteger(S.getOffset()).EmitValue(AP, Form);
 }
 
 /// SizeOf - Determine size of delta value in bytes.
 ///
 unsigned DIEString::SizeOf(const AsmPrinter *AP, dwarf::Form Form) const {
-  return Access->SizeOf(AP, Form);
+  assert(
+      (Form == dwarf::DW_FORM_strp || Form == dwarf::DW_FORM_GNU_str_index) &&
+      "Expected valid string form");
+
+  // Index of string in symbol table.
+  if (Form == dwarf::DW_FORM_GNU_str_index)
+    return DIEInteger(S.getIndex()).SizeOf(AP, Form);
+
+  // Relocatable symbol.
+  if (AP->MAI->doesDwarfUseRelocationsAcrossSections())
+    return DIELabel(S.getSymbol()).SizeOf(AP, Form);
+
+  // Offset into symbol table.
+  return DIEInteger(S.getOffset()).SizeOf(AP, Form);
 }
 
 #ifndef NDEBUG
 void DIEString::print(raw_ostream &O) const {
-  O << "String: " << Str << "\tSymbol: ";
-  Access->print(O);
+  O << "String: " << S.getString();
 }
 #endif
 
@@ -388,21 +449,21 @@ void DIEEntry::EmitValue(const AsmPrinter *AP, dwarf::Form Form) const {
 
   if (Form == dwarf::DW_FORM_ref_addr) {
     const DwarfDebug *DD = AP->getDwarfDebug();
-    unsigned Addr = Entry.getOffset();
+    unsigned Addr = Entry->getOffset();
     assert(!DD->useSplitDwarf() && "TODO: dwo files can't have relocations.");
     // For DW_FORM_ref_addr, output the offset from beginning of debug info
     // section. Entry->getOffset() returns the offset from start of the
     // compile unit.
-    DwarfCompileUnit *CU = DD->lookupUnit(Entry.getUnit());
+    DwarfCompileUnit *CU = DD->lookupUnit(Entry->getUnit());
     assert(CU && "CUDie should belong to a CU.");
     Addr += CU->getDebugInfoOffset();
     if (AP->MAI->doesDwarfUseRelocationsAcrossSections())
       AP->EmitLabelPlusOffset(CU->getSectionSym(), Addr,
                               DIEEntry::getRefAddrSize(AP));
     else
-      AP->OutStreamer.EmitIntValue(Addr, DIEEntry::getRefAddrSize(AP));
+      AP->OutStreamer->EmitIntValue(Addr, DIEEntry::getRefAddrSize(AP));
   } else
-    AP->EmitInt32(Entry.getOffset());
+    AP->EmitInt32(Entry->getOffset());
 }
 
 unsigned DIEEntry::getRefAddrSize(const AsmPrinter *AP) {
@@ -413,7 +474,7 @@ unsigned DIEEntry::getRefAddrSize(const AsmPrinter *AP) {
   const DwarfDebug *DD = AP->getDwarfDebug();
   assert(DD && "Expected Dwarf Debug info to be available");
   if (DD->getDwarfVersion() == 2)
-    return AP->getDataLayout().getPointerSize();
+    return AP->getPointerSize();
   return sizeof(int32_t);
 }
 
@@ -426,17 +487,16 @@ void DIEEntry::print(raw_ostream &O) const {
 //===----------------------------------------------------------------------===//
 // DIETypeSignature Implementation
 //===----------------------------------------------------------------------===//
-void DIETypeSignature::EmitValue(const AsmPrinter *Asm, dwarf::Form Form) const {
+void DIETypeSignature::EmitValue(const AsmPrinter *Asm,
+                                 dwarf::Form Form) const {
   assert(Form == dwarf::DW_FORM_ref_sig8);
-  Asm->OutStreamer.EmitIntValue(Unit.getTypeSignature(), 8);
+  Asm->OutStreamer->EmitIntValue(Unit->getTypeSignature(), 8);
 }
 
 #ifndef NDEBUG
 void DIETypeSignature::print(raw_ostream &O) const {
-  O << format("Type Unit: 0x%lx", Unit.getTypeSignature());
+  O << format("Type Unit: 0x%lx", Unit->getTypeSignature());
 }
-
-void DIETypeSignature::dump() const { print(dbgs()); }
 #endif
 
 //===----------------------------------------------------------------------===//
@@ -447,9 +507,8 @@ void DIETypeSignature::dump() const { print(dbgs()); }
 ///
 unsigned DIELoc::ComputeSize(const AsmPrinter *AP) const {
   if (!Size) {
-    const SmallVectorImpl<DIEAbbrevData> &AbbrevData = Abbrev.getData();
-    for (unsigned i = 0, N = Values.size(); i < N; ++i)
-      Size += Values[i]->SizeOf(AP, AbbrevData[i].getForm());
+    for (const auto &V : Values)
+      Size += V.SizeOf(AP);
   }
 
   return Size;
@@ -468,9 +527,8 @@ void DIELoc::EmitValue(const AsmPrinter *Asm, dwarf::Form Form) const {
     Asm->EmitULEB128(Size); break;
   }
 
-  const SmallVectorImpl<DIEAbbrevData> &AbbrevData = Abbrev.getData();
-  for (unsigned i = 0, N = Values.size(); i < N; ++i)
-    Values[i]->EmitValue(Asm, AbbrevData[i].getForm());
+  for (const auto &V : Values)
+    V.EmitValue(Asm);
 }
 
 /// SizeOf - Determine size of location data in bytes.
@@ -502,9 +560,8 @@ void DIELoc::print(raw_ostream &O) const {
 ///
 unsigned DIEBlock::ComputeSize(const AsmPrinter *AP) const {
   if (!Size) {
-    const SmallVectorImpl<DIEAbbrevData> &AbbrevData = Abbrev.getData();
-    for (unsigned i = 0, N = Values.size(); i < N; ++i)
-      Size += Values[i]->SizeOf(AP, AbbrevData[i].getForm());
+    for (const auto &V : Values)
+      Size += V.SizeOf(AP);
   }
 
   return Size;
@@ -521,9 +578,8 @@ void DIEBlock::EmitValue(const AsmPrinter *Asm, dwarf::Form Form) const {
   case dwarf::DW_FORM_block:  Asm->EmitULEB128(Size); break;
   }
 
-  const SmallVectorImpl<DIEAbbrevData> &AbbrevData = Abbrev.getData();
-  for (unsigned i = 0, N = Values.size(); i < N; ++i)
-    Values[i]->EmitValue(Asm, AbbrevData[i].getForm());
+  for (const auto &V : Values)
+    V.EmitValue(Asm);
 }
 
 /// SizeOf - Determine size of block data in bytes.
@@ -554,24 +610,17 @@ unsigned DIELocList::SizeOf(const AsmPrinter *AP, dwarf::Form Form) const {
     return 4;
   if (Form == dwarf::DW_FORM_sec_offset)
     return 4;
-  return AP->getDataLayout().getPointerSize();
+  return AP->getPointerSize();
 }
 
 /// EmitValue - Emit label value.
 ///
 void DIELocList::EmitValue(const AsmPrinter *AP, dwarf::Form Form) const {
   DwarfDebug *DD = AP->getDwarfDebug();
-  MCSymbol *Label = DD->getDebugLocEntries()[Index].Label;
-
-  if (AP->MAI->doesDwarfUseRelocationsAcrossSections() && !DD->useSplitDwarf())
-    AP->emitSectionOffset(Label);
-  else
-    AP->EmitLabelDifference(Label, Label->getSection().getBeginSymbol(), 4);
+  MCSymbol *Label = DD->getDebugLocs().getList(Index).Label;
+  AP->emitDwarfSymbolReference(Label, /*ForceOffset*/ DD->useSplitDwarf());
 }
 
 #ifndef NDEBUG
-void DIELocList::print(raw_ostream &O) const {
-  O << "LocList: " << Index;
-
-}
+void DIELocList::print(raw_ostream &O) const { O << "LocList: " << Index; }
 #endif
diff --git a/lib/CodeGen/AsmPrinter/DIEHash.cpp b/lib/CodeGen/AsmPrinter/DIEHash.cpp
index da7252ade7d..5e60156fdfc 100644
--- a/lib/CodeGen/AsmPrinter/DIEHash.cpp
+++ b/lib/CodeGen/AsmPrinter/DIEHash.cpp
@@ -31,19 +31,12 @@ using namespace llvm;
 /// \brief Grabs the string in whichever attribute is passed in and returns
 /// a reference to it.
 static StringRef getDIEStringAttr(const DIE &Die, uint16_t Attr) {
-  const SmallVectorImpl<DIEValue *> &Values = Die.getValues();
-  const DIEAbbrev &Abbrevs = Die.getAbbrev();
-
   // Iterate through all the attributes until we find the one we're
   // looking for, if we can't find it return an empty string.
-  for (size_t i = 0; i < Values.size(); ++i) {
-    if (Abbrevs.getData()[i].getAttribute() == Attr) {
-      DIEValue *V = Values[i];
-      assert(isa<DIEString>(V) && "String requested. Not a string.");
-      DIEString *S = cast<DIEString>(V);
-      return S->getString();
-    }
-  }
+  for (const auto &V : Die.values())
+    if (V.getAttribute() == Attr)
+      return V.getDIEString().getString();
+
   return StringRef("");
 }
 
@@ -123,20 +116,16 @@ void DIEHash::addParentContext(const DIE &Parent) {
 
 // Collect all of the attributes for a particular DIE in single structure.
 void DIEHash::collectAttributes(const DIE &Die, DIEAttrs &Attrs) {
-  const SmallVectorImpl<DIEValue *> &Values = Die.getValues();
-  const DIEAbbrev &Abbrevs = Die.getAbbrev();
-
 #define COLLECT_ATTR(NAME)                                                     \
   case dwarf::NAME:                                                            \
-    Attrs.NAME.Val = Values[i];                                                \
-    Attrs.NAME.Desc = &Abbrevs.getData()[i];                                   \
+    Attrs.NAME = V;                                                            \
     break
 
-  for (size_t i = 0, e = Values.size(); i != e; ++i) {
+  for (const auto &V : Die.values()) {
     DEBUG(dbgs() << "Attribute: "
-                 << dwarf::AttributeString(Abbrevs.getData()[i].getAttribute())
+                 << dwarf::AttributeString(V.getAttribute())
                  << " added.\n");
-    switch (Abbrevs.getData()[i].getAttribute()) {
+    switch (V.getAttribute()) {
       COLLECT_ATTR(DW_AT_name);
       COLLECT_ATTR(DW_AT_accessibility);
       COLLECT_ATTR(DW_AT_address_class);
@@ -274,28 +263,24 @@ void DIEHash::hashDIEEntry(dwarf::Attribute Attribute, dwarf::Tag Tag,
 
 // Hash all of the values in a block like set of values. This assumes that
 // all of the data is going to be added as integers.
-void DIEHash::hashBlockData(const SmallVectorImpl<DIEValue *> &Values) {
-  for (SmallVectorImpl<DIEValue *>::const_iterator I = Values.begin(),
-                                                   E = Values.end();
-       I != E; ++I)
-    Hash.update((uint64_t)cast<DIEInteger>(*I)->getValue());
+void DIEHash::hashBlockData(const DIE::const_value_range &Values) {
+  for (const auto &V : Values)
+    Hash.update((uint64_t)V.getDIEInteger().getValue());
 }
 
 // Hash the contents of a loclistptr class.
 void DIEHash::hashLocList(const DIELocList &LocList) {
   HashingByteStreamer Streamer(*this);
   DwarfDebug &DD = *AP->getDwarfDebug();
-  for (const auto &Entry :
-       DD.getDebugLocEntries()[LocList.getValue()].List)
+  const DebugLocStream &Locs = DD.getDebugLocs();
+  for (const auto &Entry : Locs.getEntries(Locs.getList(LocList.getValue())))
     DD.emitDebugLocEntry(Streamer, Entry);
 }
 
 // Hash an individual attribute \param Attr based on the type of attribute and
 // the form.
-void DIEHash::hashAttribute(AttrEntry Attr, dwarf::Tag Tag) {
-  const DIEValue *Value = Attr.Val;
-  const DIEAbbrevData *Desc = Attr.Desc;
-  dwarf::Attribute Attribute = Desc->getAttribute();
+void DIEHash::hashAttribute(DIEValue Value, dwarf::Tag Tag) {
+  dwarf::Attribute Attribute = Value.getAttribute();
 
   // Other attribute values use the letter 'A' as the marker, and the value
   // consists of the form code (encoded as an unsigned LEB128 value) followed by
@@ -304,17 +289,20 @@ void DIEHash::hashAttribute(AttrEntry Attr, dwarf::Tag Tag) {
   // computation is limited to the following: DW_FORM_sdata, DW_FORM_flag,
   // DW_FORM_string, and DW_FORM_block.
 
-  switch (Value->getType()) {
+  switch (Value.getType()) {
+  case DIEValue::isNone:
+    llvm_unreachable("Expected valid DIEValue");
+
     // 7.27 Step 3
     // ... An attribute that refers to another type entry T is processed as
     // follows:
   case DIEValue::isEntry:
-    hashDIEEntry(Attribute, Tag, cast<DIEEntry>(Value)->getEntry());
+    hashDIEEntry(Attribute, Tag, Value.getDIEEntry().getEntry());
     break;
   case DIEValue::isInteger: {
     addULEB128('A');
     addULEB128(Attribute);
-    switch (Desc->getForm()) {
+    switch (Value.getForm()) {
     case dwarf::DW_FORM_data1:
     case dwarf::DW_FORM_data2:
     case dwarf::DW_FORM_data4:
@@ -322,14 +310,14 @@ void DIEHash::hashAttribute(AttrEntry Attr, dwarf::Tag Tag) {
     case dwarf::DW_FORM_udata:
     case dwarf::DW_FORM_sdata:
       addULEB128(dwarf::DW_FORM_sdata);
-      addSLEB128((int64_t)cast<DIEInteger>(Value)->getValue());
+      addSLEB128((int64_t)Value.getDIEInteger().getValue());
       break;
     // DW_FORM_flag_present is just flag with a value of one. We still give it a
     // value so just use the value.
     case dwarf::DW_FORM_flag_present:
     case dwarf::DW_FORM_flag:
       addULEB128(dwarf::DW_FORM_flag);
-      addULEB128((int64_t)cast<DIEInteger>(Value)->getValue());
+      addULEB128((int64_t)Value.getDIEInteger().getValue());
       break;
     default:
       llvm_unreachable("Unknown integer form!");
@@ -340,7 +328,7 @@ void DIEHash::hashAttribute(AttrEntry Attr, dwarf::Tag Tag) {
     addULEB128('A');
     addULEB128(Attribute);
     addULEB128(dwarf::DW_FORM_string);
-    addString(cast<DIEString>(Value)->getString());
+    addString(Value.getDIEString().getString());
     break;
   case DIEValue::isBlock:
   case DIEValue::isLoc:
@@ -348,17 +336,17 @@ void DIEHash::hashAttribute(AttrEntry Attr, dwarf::Tag Tag) {
     addULEB128('A');
     addULEB128(Attribute);
     addULEB128(dwarf::DW_FORM_block);
-    if (isa<DIEBlock>(Value)) {
-      addULEB128(cast<DIEBlock>(Value)->ComputeSize(AP));
-      hashBlockData(cast<DIEBlock>(Value)->getValues());
-    } else if (isa<DIELoc>(Value)) {
-      addULEB128(cast<DIELoc>(Value)->ComputeSize(AP));
-      hashBlockData(cast<DIELoc>(Value)->getValues());
+    if (Value.getType() == DIEValue::isBlock) {
+      addULEB128(Value.getDIEBlock().ComputeSize(AP));
+      hashBlockData(Value.getDIEBlock().values());
+    } else if (Value.getType() == DIEValue::isLoc) {
+      addULEB128(Value.getDIELoc().ComputeSize(AP));
+      hashBlockData(Value.getDIELoc().values());
     } else {
       // We could add the block length, but that would take
       // a bit of work and not add a lot of uniqueness
       // to the hash in some way we could test.
-      hashLocList(*cast<DIELocList>(Value));
+      hashLocList(Value.getDIELocList());
     }
     break;
     // FIXME: It's uncertain whether or not we should handle this at the moment.
@@ -375,7 +363,7 @@ void DIEHash::hashAttribute(AttrEntry Attr, dwarf::Tag Tag) {
 void DIEHash::hashAttributes(const DIEAttrs &Attrs, dwarf::Tag Tag) {
 #define ADD_ATTR(ATTR)                                                         \
   {                                                                            \
-    if (ATTR.Val != 0)                                                         \
+    if (ATTR)                                                                  \
       hashAttribute(ATTR, Tag);                                                \
   }
 
@@ -463,18 +451,18 @@ void DIEHash::computeHash(const DIE &Die) {
   addAttributes(Die);
 
   // Then hash each of the children of the DIE.
-  for (auto &C : Die.getChildren()) {
+  for (auto &C : Die.children()) {
     // 7.27 Step 7
     // If C is a nested type entry or a member function entry, ...
-    if (isType(C->getTag()) || C->getTag() == dwarf::DW_TAG_subprogram) {
-      StringRef Name = getDIEStringAttr(*C, dwarf::DW_AT_name);
+    if (isType(C.getTag()) || C.getTag() == dwarf::DW_TAG_subprogram) {
+      StringRef Name = getDIEStringAttr(C, dwarf::DW_AT_name);
       // ... and has a DW_AT_name attribute
       if (!Name.empty()) {
-        hashNestedType(*C, Name);
+        hashNestedType(C, Name);
         continue;
       }
     }
-    computeHash(*C);
+    computeHash(C);
   }
 
   // Following the last (or if there are no children), append a zero byte.
diff --git a/lib/CodeGen/AsmPrinter/DIEHash.h b/lib/CodeGen/AsmPrinter/DIEHash.h
index ac014b727b7..833ca0276fd 100644
--- a/lib/CodeGen/AsmPrinter/DIEHash.h
+++ b/lib/CodeGen/AsmPrinter/DIEHash.h
@@ -26,64 +26,57 @@ class CompileUnit;
 /// \brief An object containing the capability of hashing and adding hash
 /// attributes onto a DIE.
 class DIEHash {
-
-  // The entry for a particular attribute.
-  struct AttrEntry {
-    const DIEValue *Val;
-    const DIEAbbrevData *Desc;
-  };
-
   // Collection of all attributes used in hashing a particular DIE.
   struct DIEAttrs {
-    AttrEntry DW_AT_name;
-    AttrEntry DW_AT_accessibility;
-    AttrEntry DW_AT_address_class;
-    AttrEntry DW_AT_allocated;
-    AttrEntry DW_AT_artificial;
-    AttrEntry DW_AT_associated;
-    AttrEntry DW_AT_binary_scale;
-    AttrEntry DW_AT_bit_offset;
-    AttrEntry DW_AT_bit_size;
-    AttrEntry DW_AT_bit_stride;
-    AttrEntry DW_AT_byte_size;
-    AttrEntry DW_AT_byte_stride;
-    AttrEntry DW_AT_const_expr;
-    AttrEntry DW_AT_const_value;
-    AttrEntry DW_AT_containing_type;
-    AttrEntry DW_AT_count;
-    AttrEntry DW_AT_data_bit_offset;
-    AttrEntry DW_AT_data_location;
-    AttrEntry DW_AT_data_member_location;
-    AttrEntry DW_AT_decimal_scale;
-    AttrEntry DW_AT_decimal_sign;
-    AttrEntry DW_AT_default_value;
-    AttrEntry DW_AT_digit_count;
-    AttrEntry DW_AT_discr;
-    AttrEntry DW_AT_discr_list;
-    AttrEntry DW_AT_discr_value;
-    AttrEntry DW_AT_encoding;
-    AttrEntry DW_AT_enum_class;
-    AttrEntry DW_AT_endianity;
-    AttrEntry DW_AT_explicit;
-    AttrEntry DW_AT_is_optional;
-    AttrEntry DW_AT_location;
-    AttrEntry DW_AT_lower_bound;
-    AttrEntry DW_AT_mutable;
-    AttrEntry DW_AT_ordering;
-    AttrEntry DW_AT_picture_string;
-    AttrEntry DW_AT_prototyped;
-    AttrEntry DW_AT_small;
-    AttrEntry DW_AT_segment;
-    AttrEntry DW_AT_string_length;
-    AttrEntry DW_AT_threads_scaled;
-    AttrEntry DW_AT_upper_bound;
-    AttrEntry DW_AT_use_location;
-    AttrEntry DW_AT_use_UTF8;
-    AttrEntry DW_AT_variable_parameter;
-    AttrEntry DW_AT_virtuality;
-    AttrEntry DW_AT_visibility;
-    AttrEntry DW_AT_vtable_elem_location;
-    AttrEntry DW_AT_type;
+    DIEValue DW_AT_name;
+    DIEValue DW_AT_accessibility;
+    DIEValue DW_AT_address_class;
+    DIEValue DW_AT_allocated;
+    DIEValue DW_AT_artificial;
+    DIEValue DW_AT_associated;
+    DIEValue DW_AT_binary_scale;
+    DIEValue DW_AT_bit_offset;
+    DIEValue DW_AT_bit_size;
+    DIEValue DW_AT_bit_stride;
+    DIEValue DW_AT_byte_size;
+    DIEValue DW_AT_byte_stride;
+    DIEValue DW_AT_const_expr;
+    DIEValue DW_AT_const_value;
+    DIEValue DW_AT_containing_type;
+    DIEValue DW_AT_count;
+    DIEValue DW_AT_data_bit_offset;
+    DIEValue DW_AT_data_location;
+    DIEValue DW_AT_data_member_location;
+    DIEValue DW_AT_decimal_scale;
+    DIEValue DW_AT_decimal_sign;
+    DIEValue DW_AT_default_value;
+    DIEValue DW_AT_digit_count;
+    DIEValue DW_AT_discr;
+    DIEValue DW_AT_discr_list;
+    DIEValue DW_AT_discr_value;
+    DIEValue DW_AT_encoding;
+    DIEValue DW_AT_enum_class;
+    DIEValue DW_AT_endianity;
+    DIEValue DW_AT_explicit;
+    DIEValue DW_AT_is_optional;
+    DIEValue DW_AT_location;
+    DIEValue DW_AT_lower_bound;
+    DIEValue DW_AT_mutable;
+    DIEValue DW_AT_ordering;
+    DIEValue DW_AT_picture_string;
+    DIEValue DW_AT_prototyped;
+    DIEValue DW_AT_small;
+    DIEValue DW_AT_segment;
+    DIEValue DW_AT_string_length;
+    DIEValue DW_AT_threads_scaled;
+    DIEValue DW_AT_upper_bound;
+    DIEValue DW_AT_use_location;
+    DIEValue DW_AT_use_UTF8;
+    DIEValue DW_AT_variable_parameter;
+    DIEValue DW_AT_virtuality;
+    DIEValue DW_AT_visibility;
+    DIEValue DW_AT_vtable_elem_location;
+    DIEValue DW_AT_type;
 
     // Insert any additional ones here...
   };
@@ -135,13 +128,13 @@ class DIEHash {
 
   /// \brief Hashes the data in a block like DIEValue, e.g. DW_FORM_block or
   /// DW_FORM_exprloc.
-  void hashBlockData(const SmallVectorImpl<DIEValue *> &Values);
+  void hashBlockData(const DIE::const_value_range &Values);
 
   /// \brief Hashes the contents pointed to in the .debug_loc section.
   void hashLocList(const DIELocList &LocList);
 
   /// \brief Hashes an individual attribute.
-  void hashAttribute(AttrEntry Attr, dwarf::Tag Tag);
+  void hashAttribute(DIEValue Value, dwarf::Tag Tag);
 
   /// \brief Hashes an attribute that refers to another DIE.
   void hashDIEEntry(dwarf::Attribute Attribute, dwarf::Tag Tag,
diff --git a/lib/CodeGen/AsmPrinter/DbgValueHistoryCalculator.cpp b/lib/CodeGen/AsmPrinter/DbgValueHistoryCalculator.cpp
index 1e3c5d769f1..3c46a99d084 100644
--- a/lib/CodeGen/AsmPrinter/DbgValueHistoryCalculator.cpp
+++ b/lib/CodeGen/AsmPrinter/DbgValueHistoryCalculator.cpp
@@ -204,7 +204,7 @@ void llvm::calculateDbgValueHistory(const MachineFunction *MF,
       // Use the base variable (without any DW_OP_piece expressions)
       // as index into History. The full variables including the
       // piece expressions are attached to the MI.
-      MDLocalVariable *RawVar = MI.getDebugVariable();
+      const DILocalVariable *RawVar = MI.getDebugVariable();
       assert(RawVar->isValidLocationForIntrinsic(MI.getDebugLoc()) &&
              "Expected inlined-at fields to agree");
       InlinedVariable Var(RawVar, MI.getDebugLoc()->getInlinedAt());
diff --git a/lib/CodeGen/AsmPrinter/DbgValueHistoryCalculator.h b/lib/CodeGen/AsmPrinter/DbgValueHistoryCalculator.h
index c25aaffd2d1..546d1b44378 100644
--- a/lib/CodeGen/AsmPrinter/DbgValueHistoryCalculator.h
+++ b/lib/CodeGen/AsmPrinter/DbgValueHistoryCalculator.h
@@ -17,8 +17,8 @@ namespace llvm {
 
 class MachineFunction;
 class MachineInstr;
-class MDLocalVariable;
-class MDLocation;
+class DILocalVariable;
+class DILocation;
 class TargetRegisterInfo;
 
 // For each user variable, keep a list of instruction ranges where this variable
@@ -32,7 +32,7 @@ class DbgValueHistoryMap {
 public:
   typedef std::pair<const MachineInstr *, const MachineInstr *> InstrRange;
   typedef SmallVector<InstrRange, 4> InstrRanges;
-  typedef std::pair<const MDLocalVariable *, const MDLocation *>
+  typedef std::pair<const DILocalVariable *, const DILocation *>
       InlinedVariable;
   typedef MapVector<InlinedVariable, InstrRanges> InstrRangesMap;
 
diff --git a/lib/CodeGen/AsmPrinter/DebugLocEntry.h b/lib/CodeGen/AsmPrinter/DebugLocEntry.h
index 4f6714e1a04..afffa839a60 100644
--- a/lib/CodeGen/AsmPrinter/DebugLocEntry.h
+++ b/lib/CodeGen/AsmPrinter/DebugLocEntry.h
@@ -17,7 +17,8 @@
 
 namespace llvm {
 class AsmPrinter;
-class MDNode;
+class DebugLocStream;
+
 /// \brief This struct describes location entries emitted in the .debug_loc
 /// section.
 class DebugLocEntry {
@@ -28,29 +29,25 @@ class DebugLocEntry {
 public:
   /// \brief A single location or constant.
   struct Value {
-    Value(const MDNode *Var, const MDNode *Expr, int64_t i)
-        : Variable(Var), Expression(Expr), EntryKind(E_Integer) {
+    Value(const DIExpression *Expr, int64_t i)
+        : Expression(Expr), EntryKind(E_Integer) {
       Constant.Int = i;
     }
-    Value(const MDNode *Var, const MDNode *Expr, const ConstantFP *CFP)
-        : Variable(Var), Expression(Expr), EntryKind(E_ConstantFP) {
+    Value(const DIExpression *Expr, const ConstantFP *CFP)
+        : Expression(Expr), EntryKind(E_ConstantFP) {
       Constant.CFP = CFP;
     }
-    Value(const MDNode *Var, const MDNode *Expr, const ConstantInt *CIP)
-        : Variable(Var), Expression(Expr), EntryKind(E_ConstantInt) {
+    Value(const DIExpression *Expr, const ConstantInt *CIP)
+        : Expression(Expr), EntryKind(E_ConstantInt) {
       Constant.CIP = CIP;
     }
-    Value(const MDNode *Var, const MDNode *Expr, MachineLocation Loc)
-        : Variable(Var), Expression(Expr), EntryKind(E_Location), Loc(Loc) {
-      assert(isa<MDLocalVariable>(Var));
-      assert(cast<MDExpression>(Expr)->isValid());
+    Value(const DIExpression *Expr, MachineLocation Loc)
+        : Expression(Expr), EntryKind(E_Location), Loc(Loc) {
+      assert(cast<DIExpression>(Expr)->isValid());
     }
 
-    /// The variable to which this location entry corresponds.
-    const MDNode *Variable;
-
     /// Any complex address location expression for this Value.
-    const MDNode *Expression;
+    const DIExpression *Expression;
 
     /// Type of entry that this represents.
     enum EntryType { E_Location, E_Integer, E_ConstantFP, E_ConstantInt };
@@ -74,11 +71,8 @@ class DebugLocEntry {
     const ConstantFP *getConstantFP() const { return Constant.CFP; }
     const ConstantInt *getConstantInt() const { return Constant.CIP; }
     MachineLocation getLoc() const { return Loc; }
-    DIVariable getVariable() const { return cast<MDLocalVariable>(Variable); }
     bool isBitPiece() const { return getExpression()->isBitPiece(); }
-    DIExpression getExpression() const {
-      return cast_or_null<MDExpression>(Expression);
-    }
+    const DIExpression *getExpression() const { return Expression; }
     friend bool operator==(const Value &, const Value &);
     friend bool operator<(const Value &, const Value &);
   };
@@ -87,8 +81,6 @@ class DebugLocEntry {
   /// A nonempty list of locations/constants belonging to this entry,
   /// sorted by offset.
   SmallVector<Value, 1> Values;
-  SmallString<8> DWARFBytes;
-  SmallVector<std::string, 1> Comments;
 
 public:
   DebugLocEntry(const MCSymbol *B, const MCSymbol *E, Value Val)
@@ -102,12 +94,9 @@ class DebugLocEntry {
   /// Return true if the merge was successful.
   bool MergeValues(const DebugLocEntry &Next) {
     if (Begin == Next.Begin) {
-      DIExpression Expr = cast_or_null<MDExpression>(Values[0].Expression);
-      DIVariable Var = cast_or_null<MDLocalVariable>(Values[0].Variable);
-      DIExpression NextExpr =
-          cast_or_null<MDExpression>(Next.Values[0].Expression);
-      DIVariable NextVar = cast_or_null<MDLocalVariable>(Next.Values[0].Variable);
-      if (Var == NextVar && Expr->isBitPiece() && NextExpr->isBitPiece()) {
+      auto *Expr = cast_or_null<DIExpression>(Values[0].Expression);
+      auto *NextExpr = cast_or_null<DIExpression>(Next.Values[0].Expression);
+      if (Expr->isBitPiece() && NextExpr->isBitPiece()) {
         addValues(Next.Values);
         End = Next.End;
         return true;
@@ -144,22 +133,17 @@ class DebugLocEntry {
   // Remove any duplicate entries by dropping all but the first.
   void sortUniqueValues() {
     std::sort(Values.begin(), Values.end());
-    Values.erase(std::unique(Values.begin(), Values.end(),
-                             [](const Value &A, const Value &B) {
-                   return A.getVariable() == B.getVariable() &&
-                          A.getExpression() == B.getExpression();
-                 }),
-                 Values.end());
+    Values.erase(
+        std::unique(
+            Values.begin(), Values.end(), [](const Value &A, const Value &B) {
+              return A.getExpression() == B.getExpression();
+            }),
+        Values.end());
   }
 
   /// \brief Lower this entry into a DWARF expression.
-  void finalize(const AsmPrinter &AP,
-                const DITypeIdentifierMap &TypeIdentifierMap);
-
-  /// \brief Return the lowered DWARF expression.
-  StringRef getDWARFBytes() const { return DWARFBytes; }
-  /// \brief Return the assembler comments for the lowered DWARF expression.
-  const SmallVectorImpl<std::string> &getComments() const { return Comments; }
+  void finalize(const AsmPrinter &AP, DebugLocStream::ListBuilder &List,
+                const DIBasicType *BT);
 };
 
 /// \brief Compare two Values for equality.
@@ -171,9 +155,6 @@ inline bool operator==(const DebugLocEntry::Value &A,
   if (A.Expression != B.Expression)
     return false;
 
-  if (A.Variable != B.Variable)
-    return false;
-
   switch (A.EntryKind) {
   case DebugLocEntry::Value::E_Location:
     return A.Loc == B.Loc;
diff --git a/lib/CodeGen/AsmPrinter/DebugLocList.h b/lib/CodeGen/AsmPrinter/DebugLocList.h
deleted file mode 100644
index 0f1d2edd425..00000000000
--- a/lib/CodeGen/AsmPrinter/DebugLocList.h
+++ /dev/null
@@ -1,25 +0,0 @@
-//===--- lib/CodeGen/DebugLocList.h - DWARF debug_loc list ------*- C++ -*-===//
-//
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
-//
-//===----------------------------------------------------------------------===//
-
-#ifndef LLVM_LIB_CODEGEN_ASMPRINTER_DEBUGLOCLIST_H
-#define LLVM_LIB_CODEGEN_ASMPRINTER_DEBUGLOCLIST_H
-
-#include "DebugLocEntry.h"
-#include "llvm/ADT/SmallVector.h"
-
-namespace llvm {
-class DwarfCompileUnit;
-class MCSymbol;
-struct DebugLocList {
-  MCSymbol *Label;
-  DwarfCompileUnit *CU;
-  SmallVector<DebugLocEntry, 4> List;
-};
-}
-#endif
diff --git a/lib/CodeGen/AsmPrinter/DebugLocStream.cpp b/lib/CodeGen/AsmPrinter/DebugLocStream.cpp
new file mode 100644
index 00000000000..7e8ed7104af
--- /dev/null
+++ b/lib/CodeGen/AsmPrinter/DebugLocStream.cpp
@@ -0,0 +1,46 @@
+//===- DebugLocStream.cpp - DWARF debug_loc stream --------------*- C++ -*-===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+
+#include "DebugLocStream.h"
+#include "DwarfDebug.h"
+#include "llvm/CodeGen/AsmPrinter.h"
+
+using namespace llvm;
+
+bool DebugLocStream::finalizeList(AsmPrinter &Asm) {
+  if (Lists.back().EntryOffset == Entries.size()) {
+    // Empty list.  Delete it.
+    Lists.pop_back();
+    return false;
+  }
+
+  // Real list.  Generate a label for it.
+  Lists.back().Label = Asm.createTempSymbol("debug_loc");
+  return true;
+}
+
+void DebugLocStream::finalizeEntry() {
+  if (Entries.back().ByteOffset != DWARFBytes.size())
+    return;
+
+  // The last entry was empty.  Delete it.
+  Comments.erase(Comments.begin() + Entries.back().CommentOffset,
+                 Comments.end());
+  Entries.pop_back();
+
+  assert(Lists.back().EntryOffset <= Entries.size() &&
+         "Popped off more entries than are in the list");
+}
+
+DebugLocStream::ListBuilder::~ListBuilder() {
+  if (!Locs.finalizeList(Asm))
+    return;
+  V.initializeDbgValue(&MI);
+  V.setDebugLocListIndex(ListIndex);
+}
diff --git a/lib/CodeGen/AsmPrinter/DebugLocStream.h b/lib/CodeGen/AsmPrinter/DebugLocStream.h
new file mode 100644
index 00000000000..3656e9d9509
--- /dev/null
+++ b/lib/CodeGen/AsmPrinter/DebugLocStream.h
@@ -0,0 +1,193 @@
+//===--- lib/CodeGen/DebugLocStream.h - DWARF debug_loc stream --*- C++ -*-===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIB_CODEGEN_ASMPRINTER_DEBUGLOCSTREAM_H
+#define LLVM_LIB_CODEGEN_ASMPRINTER_DEBUGLOCSTREAM_H
+
+#include "llvm/ADT/ArrayRef.h"
+#include "llvm/ADT/SmallVector.h"
+#include "ByteStreamer.h"
+
+namespace llvm {
+
+class AsmPrinter;
+class DbgVariable;
+class DwarfCompileUnit;
+class MachineInstr;
+class MCSymbol;
+
+/// \brief Byte stream of .debug_loc entries.
+///
+/// Stores a unified stream of .debug_loc entries.  There's \a List for each
+/// variable/inlined-at pair, and an \a Entry for each \a DebugLocEntry.
+///
+/// FIXME: Do we need all these temp symbols?
+/// FIXME: Why not output directly to the output stream?
+class DebugLocStream {
+public:
+  struct List {
+    DwarfCompileUnit *CU;
+    MCSymbol *Label = nullptr;
+    size_t EntryOffset;
+    List(DwarfCompileUnit *CU, size_t EntryOffset)
+        : CU(CU), EntryOffset(EntryOffset) {}
+  };
+  struct Entry {
+    const MCSymbol *BeginSym;
+    const MCSymbol *EndSym;
+    size_t ByteOffset;
+    size_t CommentOffset;
+    Entry(const MCSymbol *BeginSym, const MCSymbol *EndSym, size_t ByteOffset,
+          size_t CommentOffset)
+        : BeginSym(BeginSym), EndSym(EndSym), ByteOffset(ByteOffset),
+          CommentOffset(CommentOffset) {}
+  };
+
+private:
+  SmallVector<List, 4> Lists;
+  SmallVector<Entry, 32> Entries;
+  SmallString<256> DWARFBytes;
+  SmallVector<std::string, 32> Comments;
+
+  /// \brief Only verbose textual output needs comments.  This will be set to
+  /// true for that case, and false otherwise.
+  bool GenerateComments;
+
+public:
+  DebugLocStream(bool GenerateComments) : GenerateComments(GenerateComments) { }
+  size_t getNumLists() const { return Lists.size(); }
+  const List &getList(size_t LI) const { return Lists[LI]; }
+  ArrayRef<List> getLists() const { return Lists; }
+
+  class ListBuilder;
+  class EntryBuilder;
+
+private:
+  /// \brief Start a new .debug_loc entry list.
+  ///
+  /// Start a new .debug_loc entry list.  Return the new list's index so it can
+  /// be retrieved later via \a getList().
+  ///
+  /// Until the next call, \a startEntry() will add entries to this list.
+  size_t startList(DwarfCompileUnit *CU) {
+    size_t LI = Lists.size();
+    Lists.emplace_back(CU, Entries.size());
+    return LI;
+  }
+
+  /// Finalize a .debug_loc entry list.
+  ///
+  /// If there are no entries in this list, delete it outright.  Otherwise,
+  /// create a label with \a Asm.
+  ///
+  /// \return false iff the list is deleted.
+  bool finalizeList(AsmPrinter &Asm);
+
+  /// \brief Start a new .debug_loc entry.
+  ///
+  /// Until the next call, bytes added to the stream will be added to this
+  /// entry.
+  void startEntry(const MCSymbol *BeginSym, const MCSymbol *EndSym) {
+    Entries.emplace_back(BeginSym, EndSym, DWARFBytes.size(), Comments.size());
+  }
+
+  /// Finalize a .debug_loc entry, deleting if it's empty.
+  void finalizeEntry();
+
+public:
+  BufferByteStreamer getStreamer() {
+    return BufferByteStreamer(DWARFBytes, Comments, GenerateComments);
+  }
+
+  ArrayRef<Entry> getEntries(const List &L) const {
+    size_t LI = getIndex(L);
+    return makeArrayRef(Entries)
+        .slice(Lists[LI].EntryOffset, getNumEntries(LI));
+  }
+
+  ArrayRef<char> getBytes(const Entry &E) const {
+    size_t EI = getIndex(E);
+    return makeArrayRef(DWARFBytes.begin(), DWARFBytes.end())
+        .slice(Entries[EI].ByteOffset, getNumBytes(EI));
+  }
+  ArrayRef<std::string> getComments(const Entry &E) const {
+    size_t EI = getIndex(E);
+    return makeArrayRef(Comments)
+        .slice(Entries[EI].CommentOffset, getNumComments(EI));
+  }
+
+private:
+  size_t getIndex(const List &L) const {
+    assert(&Lists.front() <= &L && &L <= &Lists.back() &&
+           "Expected valid list");
+    return &L - &Lists.front();
+  }
+  size_t getIndex(const Entry &E) const {
+    assert(&Entries.front() <= &E && &E <= &Entries.back() &&
+           "Expected valid entry");
+    return &E - &Entries.front();
+  }
+  size_t getNumEntries(size_t LI) const {
+    if (LI + 1 == Lists.size())
+      return Entries.size() - Lists[LI].EntryOffset;
+    return Lists[LI + 1].EntryOffset - Lists[LI].EntryOffset;
+  }
+  size_t getNumBytes(size_t EI) const {
+    if (EI + 1 == Entries.size())
+      return DWARFBytes.size() - Entries[EI].ByteOffset;
+    return Entries[EI + 1].ByteOffset - Entries[EI].ByteOffset;
+  }
+  size_t getNumComments(size_t EI) const {
+    if (EI + 1 == Entries.size())
+      return Comments.size() - Entries[EI].CommentOffset;
+    return Entries[EI + 1].CommentOffset - Entries[EI].CommentOffset;
+  }
+};
+
+/// Builder for DebugLocStream lists.
+class DebugLocStream::ListBuilder {
+  DebugLocStream &Locs;
+  AsmPrinter &Asm;
+  DbgVariable &V;
+  const MachineInstr &MI;
+  size_t ListIndex;
+
+public:
+  ListBuilder(DebugLocStream &Locs, DwarfCompileUnit &CU, AsmPrinter &Asm,
+              DbgVariable &V, const MachineInstr &MI)
+      : Locs(Locs), Asm(Asm), V(V), MI(MI), ListIndex(Locs.startList(&CU)) {}
+
+  /// Finalize the list.
+  ///
+  /// If the list is empty, delete it.  Otherwise, finalize it by creating a
+  /// temp symbol in \a Asm and setting up the \a DbgVariable.
+  ~ListBuilder();
+
+  DebugLocStream &getLocs() { return Locs; }
+};
+
+/// Builder for DebugLocStream entries.
+class DebugLocStream::EntryBuilder {
+  DebugLocStream &Locs;
+
+public:
+  EntryBuilder(ListBuilder &List, const MCSymbol *Begin, const MCSymbol *End)
+      : Locs(List.getLocs()) {
+    Locs.startEntry(Begin, End);
+  }
+
+  /// Finalize the entry, deleting it if it's empty.
+  ~EntryBuilder() { Locs.finalizeEntry(); }
+
+  BufferByteStreamer getStreamer() { return Locs.getStreamer(); }
+};
+
+} // namespace llvm
+
+#endif
diff --git a/lib/CodeGen/AsmPrinter/DwarfAccelTable.cpp b/lib/CodeGen/AsmPrinter/DwarfAccelTable.cpp
index f64338ec832..f8cdde20318 100644
--- a/lib/CodeGen/AsmPrinter/DwarfAccelTable.cpp
+++ b/lib/CodeGen/AsmPrinter/DwarfAccelTable.cpp
@@ -30,14 +30,14 @@ DwarfAccelTable::DwarfAccelTable(ArrayRef<DwarfAccelTable::Atom> atomList)
     : Header(8 + (atomList.size() * 4)), HeaderData(atomList),
       Entries(Allocator) {}
 
-void DwarfAccelTable::AddName(StringRef Name, MCSymbol *StrSym, const DIE *die,
+void DwarfAccelTable::AddName(DwarfStringPoolEntryRef Name, const DIE *die,
                               char Flags) {
   assert(Data.empty() && "Already finalized!");
   // If the string is in the list already then add this die to the list
   // otherwise add a new one.
-  DataArray &DIEs = Entries[Name];
-  assert(!DIEs.StrSym || DIEs.StrSym == StrSym);
-  DIEs.StrSym = StrSym;
+  DataArray &DIEs = Entries[Name.getString()];
+  assert(!DIEs.Name || DIEs.Name == Name);
+  DIEs.Name = Name;
   DIEs.Values.push_back(new (Allocator) HashDataContents(die, Flags));
 }
 
@@ -111,27 +111,27 @@ void DwarfAccelTable::FinalizeTable(AsmPrinter *Asm, StringRef Prefix) {
 
 // Emits the header for the table via the AsmPrinter.
 void DwarfAccelTable::EmitHeader(AsmPrinter *Asm) {
-  Asm->OutStreamer.AddComment("Header Magic");
+  Asm->OutStreamer->AddComment("Header Magic");
   Asm->EmitInt32(Header.magic);
-  Asm->OutStreamer.AddComment("Header Version");
+  Asm->OutStreamer->AddComment("Header Version");
   Asm->EmitInt16(Header.version);
-  Asm->OutStreamer.AddComment("Header Hash Function");
+  Asm->OutStreamer->AddComment("Header Hash Function");
   Asm->EmitInt16(Header.hash_function);
-  Asm->OutStreamer.AddComment("Header Bucket Count");
+  Asm->OutStreamer->AddComment("Header Bucket Count");
   Asm->EmitInt32(Header.bucket_count);
-  Asm->OutStreamer.AddComment("Header Hash Count");
+  Asm->OutStreamer->AddComment("Header Hash Count");
   Asm->EmitInt32(Header.hashes_count);
-  Asm->OutStreamer.AddComment("Header Data Length");
+  Asm->OutStreamer->AddComment("Header Data Length");
   Asm->EmitInt32(Header.header_data_len);
-  Asm->OutStreamer.AddComment("HeaderData Die Offset Base");
+  Asm->OutStreamer->AddComment("HeaderData Die Offset Base");
   Asm->EmitInt32(HeaderData.die_offset_base);
-  Asm->OutStreamer.AddComment("HeaderData Atom Count");
+  Asm->OutStreamer->AddComment("HeaderData Atom Count");
   Asm->EmitInt32(HeaderData.Atoms.size());
   for (size_t i = 0; i < HeaderData.Atoms.size(); i++) {
     Atom A = HeaderData.Atoms[i];
-    Asm->OutStreamer.AddComment(dwarf::AtomTypeString(A.type));
+    Asm->OutStreamer->AddComment(dwarf::AtomTypeString(A.type));
     Asm->EmitInt16(A.type);
-    Asm->OutStreamer.AddComment(dwarf::FormEncodingString(A.form));
+    Asm->OutStreamer->AddComment(dwarf::FormEncodingString(A.form));
     Asm->EmitInt16(A.form);
   }
 }
@@ -141,7 +141,7 @@ void DwarfAccelTable::EmitHeader(AsmPrinter *Asm) {
 void DwarfAccelTable::EmitBuckets(AsmPrinter *Asm) {
   unsigned index = 0;
   for (size_t i = 0, e = Buckets.size(); i < e; ++i) {
-    Asm->OutStreamer.AddComment("Bucket " + Twine(i));
+    Asm->OutStreamer->AddComment("Bucket " + Twine(i));
     if (Buckets[i].size() != 0)
       Asm->EmitInt32(index);
     else
@@ -169,7 +169,7 @@ void DwarfAccelTable::EmitHashes(AsmPrinter *Asm) {
       uint32_t HashValue = (*HI)->HashValue;
       if (PrevHash == HashValue)
         continue;
-      Asm->OutStreamer.AddComment("Hash in Bucket " + Twine(i));
+      Asm->OutStreamer->AddComment("Hash in Bucket " + Twine(i));
       Asm->EmitInt32(HashValue);
       PrevHash = HashValue;
     }
@@ -190,12 +190,12 @@ void DwarfAccelTable::emitOffsets(AsmPrinter *Asm, const MCSymbol *SecBegin) {
       if (PrevHash == HashValue)
         continue;
       PrevHash = HashValue;
-      Asm->OutStreamer.AddComment("Offset in Bucket " + Twine(i));
-      MCContext &Context = Asm->OutStreamer.getContext();
-      const MCExpr *Sub = MCBinaryExpr::CreateSub(
-          MCSymbolRefExpr::Create((*HI)->Sym, Context),
-          MCSymbolRefExpr::Create(SecBegin, Context), Context);
-      Asm->OutStreamer.EmitValue(Sub, sizeof(uint32_t));
+      Asm->OutStreamer->AddComment("Offset in Bucket " + Twine(i));
+      MCContext &Context = Asm->OutStreamer->getContext();
+      const MCExpr *Sub = MCBinaryExpr::createSub(
+          MCSymbolRefExpr::create((*HI)->Sym, Context),
+          MCSymbolRefExpr::create(SecBegin, Context), Context);
+      Asm->OutStreamer->EmitValue(Sub, sizeof(uint32_t));
     }
   }
 }
@@ -214,10 +214,10 @@ void DwarfAccelTable::EmitData(AsmPrinter *Asm, DwarfDebug *D) {
       if (PrevHash != UINT64_MAX && PrevHash != (*HI)->HashValue)
         Asm->EmitInt32(0);
       // Remember to emit the label for our offset.
-      Asm->OutStreamer.EmitLabel((*HI)->Sym);
-      Asm->OutStreamer.AddComment((*HI)->Str);
-      Asm->emitSectionOffset((*HI)->Data.StrSym);
-      Asm->OutStreamer.AddComment("Num DIEs");
+      Asm->OutStreamer->EmitLabel((*HI)->Sym);
+      Asm->OutStreamer->AddComment((*HI)->Str);
+      Asm->emitDwarfStringOffset((*HI)->Data.Name);
+      Asm->OutStreamer->AddComment("Num DIEs");
       Asm->EmitInt32((*HI)->Data.Values.size());
       for (HashDataContents *HD : (*HI)->Data.Values) {
         // Emit the DIE offset
diff --git a/lib/CodeGen/AsmPrinter/DwarfAccelTable.h b/lib/CodeGen/AsmPrinter/DwarfAccelTable.h
index e6fdf08b364..4d81441f6a7 100644
--- a/lib/CodeGen/AsmPrinter/DwarfAccelTable.h
+++ b/lib/CodeGen/AsmPrinter/DwarfAccelTable.h
@@ -181,9 +181,8 @@ class DwarfAccelTable {
 private:
   // String Data
   struct DataArray {
-    MCSymbol *StrSym;
+    DwarfStringPoolEntryRef Name;
     std::vector<HashDataContents *> Values;
-    DataArray() : StrSym(nullptr) {}
   };
   friend struct HashData;
   struct HashData {
@@ -201,7 +200,7 @@ class DwarfAccelTable {
       O << "  Hash Value: " << format("0x%x", HashValue) << "\n";
       O << "  Symbol: ";
       if (Sym)
-        Sym->print(O);
+        O << *Sym;
       else
         O << "<none>";
       O << "\n";
@@ -245,8 +244,7 @@ class DwarfAccelTable {
   // Public Implementation
 public:
   DwarfAccelTable(ArrayRef<DwarfAccelTable::Atom>);
-  void AddName(StringRef Name, MCSymbol *StrSym, const DIE *Die,
-               char Flags = 0);
+  void AddName(DwarfStringPoolEntryRef Name, const DIE *Die, char Flags = 0);
   void FinalizeTable(AsmPrinter *, StringRef);
   void emit(AsmPrinter *, const MCSymbol *, DwarfDebug *);
 #ifndef NDEBUG
diff --git a/lib/CodeGen/AsmPrinter/DwarfCFIException.cpp b/lib/CodeGen/AsmPrinter/DwarfCFIException.cpp
index 1bee367996d..282e2ec31a9 100644
--- a/lib/CodeGen/AsmPrinter/DwarfCFIException.cpp
+++ b/lib/CodeGen/AsmPrinter/DwarfCFIException.cpp
@@ -44,7 +44,7 @@ DwarfCFIExceptionBase::DwarfCFIExceptionBase(AsmPrinter *A)
 
 void DwarfCFIExceptionBase::markFunctionEnd() {
   if (shouldEmitCFI)
-    Asm->OutStreamer.EmitCFIEndProc();
+    Asm->OutStreamer->EmitCFIEndProc();
 
   if (MMI->getLandingPads().empty())
     return;
@@ -64,7 +64,7 @@ DwarfCFIException::~DwarfCFIException() {}
 /// content.
 void DwarfCFIException::endModule() {
   if (moveTypeModule == AsmPrinter::CFI_M_Debug)
-    Asm->OutStreamer.EmitCFISections(false, true);
+    Asm->OutStreamer->EmitCFISections(false, true);
 
   // SjLj uses this pass and it doesn't need this info.
   if (!Asm->MAI->usesCFIForEH())
@@ -83,12 +83,13 @@ void DwarfCFIException::endModule() {
     if (!Personalities[i])
       continue;
     MCSymbol *Sym = Asm->getSymbol(Personalities[i]);
-    TLOF.emitPersonalityValue(Asm->OutStreamer, Asm->TM, Sym);
+    TLOF.emitPersonalityValue(*Asm->OutStreamer, Asm->getDataLayout(), Sym);
   }
 }
 
 void DwarfCFIException::beginFunction(const MachineFunction *MF) {
   shouldEmitMoves = shouldEmitPersonality = shouldEmitLSDA = false;
+  const Function *F = MF->getFunction();
 
   // If any landing pads survive, we need an EH table.
   bool hasLandingPads = !MMI->getLandingPads().empty();
@@ -104,10 +105,24 @@ void DwarfCFIException::beginFunction(const MachineFunction *MF) {
 
   const TargetLoweringObjectFile &TLOF = Asm->getObjFileLowering();
   unsigned PerEncoding = TLOF.getPersonalityEncoding();
-  const Function *Per = MMI->getPersonality();
-
-  shouldEmitPersonality = hasLandingPads &&
-    PerEncoding != dwarf::DW_EH_PE_omit && Per;
+  const Function *Per = nullptr;
+  if (F->hasPersonalityFn())
+    Per = dyn_cast<Function>(F->getPersonalityFn()->stripPointerCasts());
+  assert(!MMI->getPersonality() || Per == MMI->getPersonality());
+
+  // Emit a personality function even when there are no landing pads
+  bool forceEmitPersonality =
+      // ...if a personality function is explicitly specified
+      F->hasPersonalityFn() &&
+      // ... and it's not known to be a noop in the absence of invokes
+      !isNoOpWithoutInvoke(classifyEHPersonality(Per)) &&
+      // ... and we're not explicitly asked not to emit it
+      F->needsUnwindTableEntry();
+
+  shouldEmitPersonality =
+      (forceEmitPersonality ||
+       (hasLandingPads && PerEncoding != dwarf::DW_EH_PE_omit)) &&
+      Per;
 
   unsigned LSDAEncoding = TLOF.getLSDAEncoding();
   shouldEmitLSDA = shouldEmitPersonality &&
@@ -117,21 +132,26 @@ void DwarfCFIException::beginFunction(const MachineFunction *MF) {
   if (!shouldEmitCFI)
     return;
 
-  Asm->OutStreamer.EmitCFIStartProc(/*IsSimple=*/false);
+  Asm->OutStreamer->EmitCFIStartProc(/*IsSimple=*/false);
 
   // Indicate personality routine, if any.
   if (!shouldEmitPersonality)
     return;
 
+  // If we are forced to emit this personality, make sure to record
+  // it because it might not appear in any landingpad
+  if (forceEmitPersonality)
+    MMI->addPersonality(Per);
+
   const MCSymbol *Sym =
       TLOF.getCFIPersonalitySymbol(Per, *Asm->Mang, Asm->TM, MMI);
-  Asm->OutStreamer.EmitCFIPersonality(Sym, PerEncoding);
+  Asm->OutStreamer->EmitCFIPersonality(Sym, PerEncoding);
 
   // Provide LSDA information.
   if (!shouldEmitLSDA)
     return;
 
-  Asm->OutStreamer.EmitCFILsda(Asm->getCurExceptionSym(), LSDAEncoding);
+  Asm->OutStreamer->EmitCFILsda(Asm->getCurExceptionSym(), LSDAEncoding);
 }
 
 /// endFunction - Gather and emit post-function exception information.
diff --git a/lib/CodeGen/AsmPrinter/DwarfCompileUnit.cpp b/lib/CodeGen/AsmPrinter/DwarfCompileUnit.cpp
index 75d3b68ed3f..f29ad65129c 100644
--- a/lib/CodeGen/AsmPrinter/DwarfCompileUnit.cpp
+++ b/lib/CodeGen/AsmPrinter/DwarfCompileUnit.cpp
@@ -1,6 +1,7 @@
 #include "DwarfCompileUnit.h"
 #include "DwarfExpression.h"
 #include "llvm/CodeGen/MachineFunction.h"
+#include "llvm/IR/Constants.h"
 #include "llvm/IR/DataLayout.h"
 #include "llvm/IR/GlobalValue.h"
 #include "llvm/IR/GlobalVariable.h"
@@ -15,7 +16,7 @@
 
 namespace llvm {
 
-DwarfCompileUnit::DwarfCompileUnit(unsigned UID, DICompileUnit Node,
+DwarfCompileUnit::DwarfCompileUnit(unsigned UID, const DICompileUnit *Node,
                                    AsmPrinter *A, DwarfDebug *DW,
                                    DwarfFile *DWU)
     : DwarfUnit(UID, dwarf::DW_TAG_compile_unit, Node, A, DW, DWU),
@@ -41,8 +42,8 @@ void DwarfCompileUnit::addLabelAddress(DIE &Die, dwarf::Attribute Attribute,
     DD->addArangeLabel(SymbolCU(this, Label));
 
   unsigned idx = DD->getAddressPool().getIndex(Label);
-  DIEValue *Value = new (DIEValueAllocator) DIEInteger(idx);
-  Die.addValue(Attribute, dwarf::DW_FORM_GNU_addr_index, Value);
+  Die.addValue(DIEValueAllocator, Attribute, dwarf::DW_FORM_GNU_addr_index,
+               DIEInteger(idx));
 }
 
 void DwarfCompileUnit::addLocalLabelAddress(DIE &Die,
@@ -51,9 +52,12 @@ void DwarfCompileUnit::addLocalLabelAddress(DIE &Die,
   if (Label)
     DD->addArangeLabel(SymbolCU(this, Label));
 
-  Die.addValue(Attribute, dwarf::DW_FORM_addr,
-               Label ? (DIEValue *)new (DIEValueAllocator) DIELabel(Label)
-                     : new (DIEValueAllocator) DIEInteger(0));
+  if (Label)
+    Die.addValue(DIEValueAllocator, Attribute, dwarf::DW_FORM_addr,
+                 DIELabel(Label));
+  else
+    Die.addValue(DIEValueAllocator, Attribute, dwarf::DW_FORM_addr,
+                 DIEInteger(0));
 }
 
 unsigned DwarfCompileUnit::getOrCreateSourceID(StringRef FileName,
@@ -63,9 +67,9 @@ unsigned DwarfCompileUnit::getOrCreateSourceID(StringRef FileName,
 
   // FIXME: add a better feature test than hasRawTextSupport. Even better,
   // extend .file to support this.
-  return Asm->OutStreamer.EmitDwarfFileDirective(
+  return Asm->OutStreamer->EmitDwarfFileDirective(
       0, DirName, FileName,
-      Asm->OutStreamer.hasRawTextSupport() ? 0 : getUniqueID());
+      Asm->OutStreamer->hasRawTextSupport() ? 0 : getUniqueID());
 }
 
 // Return const expression if value is a GEP to access merged global
@@ -96,15 +100,16 @@ static const ConstantExpr *getMergedGlobalExpr(const Value *V) {
 }
 
 /// getOrCreateGlobalVariableDIE - get or create global variable DIE.
-DIE *DwarfCompileUnit::getOrCreateGlobalVariableDIE(DIGlobalVariable GV) {
+DIE *DwarfCompileUnit::getOrCreateGlobalVariableDIE(
+    const DIGlobalVariable *GV) {
   // Check for pre-existence.
   if (DIE *Die = getDIE(GV))
     return Die;
 
   assert(GV);
 
-  DIScope GVContext = GV->getScope();
-  DIType GTy = DD->resolve(GV->getType());
+  auto *GVContext = GV->getScope();
+  auto *GTy = DD->resolve(GV->getType());
 
   // Construct the context before querying for the existence of the DIE in
   // case such construction creates the DIE.
@@ -112,8 +117,7 @@ DIE *DwarfCompileUnit::getOrCreateGlobalVariableDIE(DIGlobalVariable GV) {
 
   // Add to map.
   DIE *VariableDIE = &createAndAddDIE(GV->getTag(), *ContextDIE, GV);
-  DIScope DeclContext;
-
+  DIScope *DeclContext;
   if (auto *SDMDecl = GV->getStaticDataMemberDeclaration()) {
     DeclContext = resolve(SDMDecl->getScope());
     assert(SDMDecl->isStaticMember() && "Expected static member decl");
@@ -144,31 +148,35 @@ DIE *DwarfCompileUnit::getOrCreateGlobalVariableDIE(DIGlobalVariable GV) {
   bool addToAccelTable = false;
   if (auto *Global = dyn_cast_or_null<GlobalVariable>(GV->getVariable())) {
     addToAccelTable = true;
-    DIELoc *Loc = new (DIEValueAllocator) DIELoc();
+    DIELoc *Loc = new (DIEValueAllocator) DIELoc;
     const MCSymbol *Sym = Asm->getSymbol(Global);
     if (Global->isThreadLocal()) {
-      // FIXME: Make this work with -gsplit-dwarf.
-      unsigned PointerSize = Asm->getDataLayout().getPointerSize();
-      assert((PointerSize == 4 || PointerSize == 8) &&
-             "Add support for other sizes if necessary");
-      // Based on GCC's support for TLS:
-      if (!DD->useSplitDwarf()) {
-        // 1) Start with a constNu of the appropriate pointer size
-        addUInt(*Loc, dwarf::DW_FORM_data1,
-                PointerSize == 4 ? dwarf::DW_OP_const4u : dwarf::DW_OP_const8u);
-        // 2) containing the (relocated) offset of the TLS variable
-        //    within the module's TLS block.
-        addExpr(*Loc, dwarf::DW_FORM_udata,
-                Asm->getObjFileLowering().getDebugThreadLocalSymbol(Sym));
+      if (Asm->TM.Options.EmulatedTLS) {
+        // TODO: add debug info for emulated thread local mode.
       } else {
-        addUInt(*Loc, dwarf::DW_FORM_data1, dwarf::DW_OP_GNU_const_index);
-        addUInt(*Loc, dwarf::DW_FORM_udata,
-                DD->getAddressPool().getIndex(Sym, /* TLS */ true));
+        // FIXME: Make this work with -gsplit-dwarf.
+        unsigned PointerSize = Asm->getDataLayout().getPointerSize();
+        assert((PointerSize == 4 || PointerSize == 8) &&
+               "Add support for other sizes if necessary");
+        // Based on GCC's support for TLS:
+        if (!DD->useSplitDwarf()) {
+          // 1) Start with a constNu of the appropriate pointer size
+          addUInt(*Loc, dwarf::DW_FORM_data1,
+                  PointerSize == 4 ? dwarf::DW_OP_const4u : dwarf::DW_OP_const8u);
+          // 2) containing the (relocated) offset of the TLS variable
+          //    within the module's TLS block.
+          addExpr(*Loc, dwarf::DW_FORM_udata,
+                  Asm->getObjFileLowering().getDebugThreadLocalSymbol(Sym));
+        } else {
+          addUInt(*Loc, dwarf::DW_FORM_data1, dwarf::DW_OP_GNU_const_index);
+          addUInt(*Loc, dwarf::DW_FORM_udata,
+                  DD->getAddressPool().getIndex(Sym, /* TLS */ true));
+        }
+        // 3) followed by an OP to make the debugger do a TLS lookup.
+        addUInt(*Loc, dwarf::DW_FORM_data1,
+                DD->useGNUTLSOpcode() ? dwarf::DW_OP_GNU_push_tls_address
+                                      : dwarf::DW_OP_form_tls_address);
       }
-      // 3) followed by an OP to make the debugger do a TLS lookup.
-      addUInt(*Loc, dwarf::DW_FORM_data1,
-              DD->useGNUTLSOpcode() ? dwarf::DW_OP_GNU_push_tls_address
-                                    : dwarf::DW_OP_form_tls_address);
     } else {
       DD->addArangeLabel(SymbolCU(this, Sym));
       addOpAddress(*Loc, Sym);
@@ -182,7 +190,7 @@ DIE *DwarfCompileUnit::getOrCreateGlobalVariableDIE(DIGlobalVariable GV) {
   } else if (const ConstantExpr *CE = getMergedGlobalExpr(GV->getVariable())) {
     addToAccelTable = true;
     // GV is a merged global.
-    DIELoc *Loc = new (DIEValueAllocator) DIELoc();
+    DIELoc *Loc = new (DIEValueAllocator) DIELoc;
     Value *Ptr = CE->getOperand(0);
     MCSymbol *Sym = Asm->getSymbol(cast<GlobalValue>(Ptr));
     DD->addArangeLabel(SymbolCU(this, Sym));
@@ -224,24 +232,21 @@ void DwarfCompileUnit::addRange(RangeSpan Range) {
   CURanges.back().setEnd(Range.getEnd());
 }
 
-void DwarfCompileUnit::addSectionLabel(DIE &Die, dwarf::Attribute Attribute,
-                                       const MCSymbol *Label,
-                                       const MCSymbol *Sec) {
+DIE::value_iterator
+DwarfCompileUnit::addSectionLabel(DIE &Die, dwarf::Attribute Attribute,
+                                  const MCSymbol *Label, const MCSymbol *Sec) {
   if (Asm->MAI->doesDwarfUseRelocationsAcrossSections())
-    addLabel(Die, Attribute,
-             DD->getDwarfVersion() >= 4 ? dwarf::DW_FORM_sec_offset
-                                        : dwarf::DW_FORM_data4,
-             Label);
-  else
-    addSectionDelta(Die, Attribute, Label, Sec);
+    return addLabel(Die, Attribute,
+                    DD->getDwarfVersion() >= 4 ? dwarf::DW_FORM_sec_offset
+                                               : dwarf::DW_FORM_data4,
+                    Label);
+  return addSectionDelta(Die, Attribute, Label, Sec);
 }
 
 void DwarfCompileUnit::initStmtList() {
   // Define start line table label for each Compile Unit.
   MCSymbol *LineTableStartSym =
-      Asm->OutStreamer.getDwarfLineTableSymbol(getUniqueID());
-
-  stmtListIndex = UnitDie.getValues().size();
+      Asm->OutStreamer->getDwarfLineTableSymbol(getUniqueID());
 
   // DW_AT_stmt_list is a offset of line number information for this
   // compile unit in debug_line section. For split dwarf this is
@@ -249,14 +254,13 @@ void DwarfCompileUnit::initStmtList() {
   // The line table entries are not always emitted in assembly, so it
   // is not okay to use line_table_start here.
   const TargetLoweringObjectFile &TLOF = Asm->getObjFileLowering();
-  addSectionLabel(UnitDie, dwarf::DW_AT_stmt_list, LineTableStartSym,
-                  TLOF.getDwarfLineSection()->getBeginSymbol());
+  StmtListValue =
+      addSectionLabel(UnitDie, dwarf::DW_AT_stmt_list, LineTableStartSym,
+                      TLOF.getDwarfLineSection()->getBeginSymbol());
 }
 
 void DwarfCompileUnit::applyStmtList(DIE &D) {
-  D.addValue(dwarf::DW_AT_stmt_list,
-             UnitDie.getAbbrev().getData()[stmtListIndex].getForm(),
-             UnitDie.getValues()[stmtListIndex]);
+  D.addValue(DIEValueAllocator, *StmtListValue);
 }
 
 void DwarfCompileUnit::attachLowHighPC(DIE &D, const MCSymbol *Begin,
@@ -276,7 +280,7 @@ void DwarfCompileUnit::attachLowHighPC(DIE &D, const MCSymbol *Begin,
 // Find DIE for the given subprogram and attach appropriate DW_AT_low_pc
 // and DW_AT_high_pc attributes. If there are global variables in this
 // scope then create and insert DIEs for these variables.
-DIE &DwarfCompileUnit::updateSubprogramScopeDIE(DISubprogram SP) {
+DIE &DwarfCompileUnit::updateSubprogramScopeDIE(const DISubprogram *SP) {
   DIE *SPDie = getOrCreateSubprogramDIE(SP, includeMinimalInlineScopes());
 
   attachLowHighPC(*SPDie, Asm->getFunctionBegin(), Asm->getFunctionEnd());
@@ -301,24 +305,24 @@ DIE &DwarfCompileUnit::updateSubprogramScopeDIE(DISubprogram SP) {
 
 // Construct a DIE for this scope.
 void DwarfCompileUnit::constructScopeDIE(
-    LexicalScope *Scope, SmallVectorImpl<std::unique_ptr<DIE>> &FinalChildren) {
+    LexicalScope *Scope, SmallVectorImpl<DIE *> &FinalChildren) {
   if (!Scope || !Scope->getScopeNode())
     return;
 
-  DIScope DS(Scope->getScopeNode());
+  auto *DS = Scope->getScopeNode();
 
-  assert((Scope->getInlinedAt() || !isa<MDSubprogram>(DS)) &&
+  assert((Scope->getInlinedAt() || !isa<DISubprogram>(DS)) &&
          "Only handle inlined subprograms here, use "
          "constructSubprogramScopeDIE for non-inlined "
          "subprograms");
 
-  SmallVector<std::unique_ptr<DIE>, 8> Children;
+  SmallVector<DIE *, 8> Children;
 
   // We try to create the scope DIE first, then the children DIEs. This will
   // avoid creating un-used children then removing them later when we find out
   // the scope DIE is null.
-  std::unique_ptr<DIE> ScopeDIE;
-  if (Scope->getParent() && isa<MDSubprogram>(DS)) {
+  DIE *ScopeDIE;
+  if (Scope->getParent() && isa<DISubprogram>(DS)) {
     ScopeDIE = constructInlinedScopeDIE(Scope);
     if (!ScopeDIE)
       return;
@@ -340,7 +344,7 @@ void DwarfCompileUnit::constructScopeDIE(
       // There is no need to emit empty lexical block DIE.
       for (const auto &E : DD->findImportedEntitiesForScope(DS))
         Children.push_back(
-            constructImportedEntityDIE(cast<MDImportedEntity>(E.second)));
+            constructImportedEntityDIE(cast<DIImportedEntity>(E.second)));
     }
 
     // If there are only other scopes as children, put them directly in the
@@ -362,12 +366,13 @@ void DwarfCompileUnit::constructScopeDIE(
   FinalChildren.push_back(std::move(ScopeDIE));
 }
 
-void DwarfCompileUnit::addSectionDelta(DIE &Die, dwarf::Attribute Attribute,
-                                       const MCSymbol *Hi, const MCSymbol *Lo) {
-  DIEValue *Value = new (DIEValueAllocator) DIEDelta(Hi, Lo);
-  Die.addValue(Attribute, DD->getDwarfVersion() >= 4 ? dwarf::DW_FORM_sec_offset
-                                                     : dwarf::DW_FORM_data4,
-               Value);
+DIE::value_iterator
+DwarfCompileUnit::addSectionDelta(DIE &Die, dwarf::Attribute Attribute,
+                                  const MCSymbol *Hi, const MCSymbol *Lo) {
+  return Die.addValue(DIEValueAllocator, Attribute,
+                      DD->getDwarfVersion() >= 4 ? dwarf::DW_FORM_sec_offset
+                                                 : dwarf::DW_FORM_data4,
+                      new (DIEValueAllocator) DIEDelta(Hi, Lo));
 }
 
 void DwarfCompileUnit::addScopeRangeList(DIE &ScopeDIE,
@@ -415,23 +420,22 @@ void DwarfCompileUnit::attachRangesOrLowHighPC(
 
 // This scope represents inlined body of a function. Construct DIE to
 // represent this concrete inlined copy of the function.
-std::unique_ptr<DIE>
-DwarfCompileUnit::constructInlinedScopeDIE(LexicalScope *Scope) {
+DIE *DwarfCompileUnit::constructInlinedScopeDIE(LexicalScope *Scope) {
   assert(Scope->getScopeNode());
-  DIScope DS(Scope->getScopeNode());
-  DISubprogram InlinedSP = getDISubprogram(DS);
+  auto *DS = Scope->getScopeNode();
+  auto *InlinedSP = getDISubprogram(DS);
   // Find the subprogram's DwarfCompileUnit in the SPMap in case the subprogram
   // was inlined from another compile unit.
   DIE *OriginDIE = DU->getAbstractSPDies()[InlinedSP];
   assert(OriginDIE && "Unable to find original DIE for an inlined subprogram.");
 
-  auto ScopeDIE = make_unique<DIE>(dwarf::DW_TAG_inlined_subroutine);
+  auto ScopeDIE = DIE::get(DIEValueAllocator, dwarf::DW_TAG_inlined_subroutine);
   addDIEEntry(*ScopeDIE, dwarf::DW_AT_abstract_origin, *OriginDIE);
 
   attachRangesOrLowHighPC(*ScopeDIE, Scope->getRanges());
 
   // Add the call site information to the DIE.
-  const MDLocation *IA = Scope->getInlinedAt();
+  const DILocation *IA = Scope->getInlinedAt();
   addUInt(*ScopeDIE, dwarf::DW_AT_call_file, None,
           getOrCreateSourceID(IA->getFilename(), IA->getDirectory()));
   addUInt(*ScopeDIE, dwarf::DW_AT_call_line, None, IA->getLine());
@@ -445,12 +449,11 @@ DwarfCompileUnit::constructInlinedScopeDIE(LexicalScope *Scope) {
 
 // Construct new DW_TAG_lexical_block for this scope and attach
 // DW_AT_low_pc/DW_AT_high_pc labels.
-std::unique_ptr<DIE>
-DwarfCompileUnit::constructLexicalScopeDIE(LexicalScope *Scope) {
+DIE *DwarfCompileUnit::constructLexicalScopeDIE(LexicalScope *Scope) {
   if (DD->isLexicalScopeDIENull(Scope))
     return nullptr;
 
-  auto ScopeDIE = make_unique<DIE>(dwarf::DW_TAG_lexical_block);
+  auto ScopeDIE = DIE::get(DIEValueAllocator, dwarf::DW_TAG_lexical_block);
   if (Scope->isAbstractScope())
     return ScopeDIE;
 
@@ -460,18 +463,16 @@ DwarfCompileUnit::constructLexicalScopeDIE(LexicalScope *Scope) {
 }
 
 /// constructVariableDIE - Construct a DIE for the given DbgVariable.
-std::unique_ptr<DIE> DwarfCompileUnit::constructVariableDIE(DbgVariable &DV,
-                                                            bool Abstract) {
+DIE *DwarfCompileUnit::constructVariableDIE(DbgVariable &DV, bool Abstract) {
   auto D = constructVariableDIEImpl(DV, Abstract);
   DV.setDIE(*D);
   return D;
 }
 
-std::unique_ptr<DIE>
-DwarfCompileUnit::constructVariableDIEImpl(const DbgVariable &DV,
-                                           bool Abstract) {
+DIE *DwarfCompileUnit::constructVariableDIEImpl(const DbgVariable &DV,
+                                                bool Abstract) {
   // Define variable debug information entry.
-  auto VariableDie = make_unique<DIE>(DV.getTag());
+  auto VariableDie = DIE::get(DIEValueAllocator, DV.getTag());
 
   if (Abstract) {
     applyVariableAttributes(DV, *VariableDie);
@@ -480,7 +481,7 @@ DwarfCompileUnit::constructVariableDIEImpl(const DbgVariable &DV,
 
   // Add variable address.
 
-  unsigned Offset = DV.getDotDebugLocOffset();
+  unsigned Offset = DV.getDebugLocListIndex();
   if (Offset != ~0U) {
     addLocationList(*VariableDie, dwarf::DW_AT_location, Offset);
     return VariableDie;
@@ -510,11 +511,11 @@ DwarfCompileUnit::constructVariableDIEImpl(const DbgVariable &DV,
   }
 
   // .. else use frame index.
-  if (DV.getFrameIndex().back() == ~0)
+  if (DV.getFrameIndex().empty())
     return VariableDie;
 
   auto Expr = DV.getExpression().begin();
-  DIELoc *Loc = new (DIEValueAllocator) DIELoc();
+  DIELoc *Loc = new (DIEValueAllocator) DIELoc;
   DIEDwarfExpression DwarfExpr(*Asm, *this, *Loc);
   for (auto FI : DV.getFrameIndex()) {
     unsigned FrameReg = 0;
@@ -531,17 +532,18 @@ DwarfCompileUnit::constructVariableDIEImpl(const DbgVariable &DV,
   return VariableDie;
 }
 
-std::unique_ptr<DIE> DwarfCompileUnit::constructVariableDIE(
-    DbgVariable &DV, const LexicalScope &Scope, DIE *&ObjectPointer) {
+DIE *DwarfCompileUnit::constructVariableDIE(DbgVariable &DV,
+                                            const LexicalScope &Scope,
+                                            DIE *&ObjectPointer) {
   auto Var = constructVariableDIE(DV, Scope.isAbstractScope());
   if (DV.isObjectPointer())
-    ObjectPointer = Var.get();
+    ObjectPointer = Var;
   return Var;
 }
 
-DIE *DwarfCompileUnit::createScopeChildrenDIE(
-    LexicalScope *Scope, SmallVectorImpl<std::unique_ptr<DIE>> &Children,
-    unsigned *ChildScopeCount) {
+DIE *DwarfCompileUnit::createScopeChildrenDIE(LexicalScope *Scope,
+                                              SmallVectorImpl<DIE *> &Children,
+                                              unsigned *ChildScopeCount) {
   DIE *ObjectPointer = nullptr;
 
   for (DbgVariable *DV : DU->getScopeVariables().lookup(Scope))
@@ -562,14 +564,14 @@ void DwarfCompileUnit::constructSubprogramScopeDIE(LexicalScope *Scope) {
   assert(Scope && Scope->getScopeNode());
   assert(!Scope->getInlinedAt());
   assert(!Scope->isAbstractScope());
-  DISubprogram Sub = cast<MDSubprogram>(Scope->getScopeNode());
+  auto *Sub = cast<DISubprogram>(Scope->getScopeNode());
 
   DD->getProcessedSPNodes().insert(Sub);
 
   DIE &ScopeDIE = updateSubprogramScopeDIE(Sub);
 
   // If this is a variadic function, add an unspecified parameter.
-  DITypeArray FnArgs = Sub->getType()->getTypeArray();
+  DITypeRefArray FnArgs = Sub->getType()->getTypeArray();
 
   // Collect lexical scope children first.
   // ObjectPointer might be a local (non-argument) local variable if it's a
@@ -582,13 +584,14 @@ void DwarfCompileUnit::constructSubprogramScopeDIE(LexicalScope *Scope) {
   // variadic function.
   if (FnArgs.size() > 1 && !FnArgs[FnArgs.size() - 1] &&
       !includeMinimalInlineScopes())
-    ScopeDIE.addChild(make_unique<DIE>(dwarf::DW_TAG_unspecified_parameters));
+    ScopeDIE.addChild(
+        DIE::get(DIEValueAllocator, dwarf::DW_TAG_unspecified_parameters));
 }
 
 DIE *DwarfCompileUnit::createAndAddScopeChildren(LexicalScope *Scope,
                                                  DIE &ScopeDIE) {
   // We create children when the scope DIE is not null.
-  SmallVector<std::unique_ptr<DIE>, 8> Children;
+  SmallVector<DIE *, 8> Children;
   DIE *ObjectPointer = createScopeChildrenDIE(Scope, Children);
 
   // Add children
@@ -604,15 +607,15 @@ DwarfCompileUnit::constructAbstractSubprogramScopeDIE(LexicalScope *Scope) {
   if (AbsDef)
     return;
 
-  DISubprogram SP = cast<MDSubprogram>(Scope->getScopeNode());
+  auto *SP = cast<DISubprogram>(Scope->getScopeNode());
 
   DIE *ContextDIE;
 
   if (includeMinimalInlineScopes())
     ContextDIE = &getUnitDie();
   // Some of this is duplicated from DwarfUnit::getOrCreateSubprogramDIE, with
-  // the important distinction that the DIDescriptor is not associated with the
-  // DIE (since the DIDescriptor will be associated with the concrete DIE, if
+  // the important distinction that the debug node is not associated with the
+  // DIE (since the debug node will be associated with the concrete DIE, if
   // any). It could be refactored to some common utility function.
   else if (auto *SPDecl = SP->getDeclaration()) {
     ContextDIE = &getUnitDie();
@@ -620,10 +623,9 @@ DwarfCompileUnit::constructAbstractSubprogramScopeDIE(LexicalScope *Scope) {
   } else
     ContextDIE = getOrCreateContextDIE(resolve(SP->getScope()));
 
-  // Passing null as the associated DIDescriptor because the abstract definition
+  // Passing null as the associated node because the abstract definition
   // shouldn't be found by lookup.
-  AbsDef =
-      &createAndAddDIE(dwarf::DW_TAG_subprogram, *ContextDIE, DIDescriptor());
+  AbsDef = &createAndAddDIE(dwarf::DW_TAG_subprogram, *ContextDIE, nullptr);
   applySubprogramAttributesToDefinition(SP, *AbsDef);
 
   if (!includeMinimalInlineScopes())
@@ -632,19 +634,21 @@ DwarfCompileUnit::constructAbstractSubprogramScopeDIE(LexicalScope *Scope) {
     addDIEEntry(*AbsDef, dwarf::DW_AT_object_pointer, *ObjectPointer);
 }
 
-std::unique_ptr<DIE>
-DwarfCompileUnit::constructImportedEntityDIE(const DIImportedEntity &Module) {
-  std::unique_ptr<DIE> IMDie = make_unique<DIE>((dwarf::Tag)Module->getTag());
-  insertDIE(Module, IMDie.get());
+DIE *DwarfCompileUnit::constructImportedEntityDIE(
+    const DIImportedEntity *Module) {
+  DIE *IMDie = DIE::get(DIEValueAllocator, (dwarf::Tag)Module->getTag());
+  insertDIE(Module, IMDie);
   DIE *EntityDie;
   auto *Entity = resolve(Module->getEntity());
-  if (auto *NS = dyn_cast<MDNamespace>(Entity))
+  if (auto *NS = dyn_cast<DINamespace>(Entity))
     EntityDie = getOrCreateNameSpace(NS);
-  else if (auto *SP = dyn_cast<MDSubprogram>(Entity))
+  else if (auto *M = dyn_cast<DIModule>(Entity))
+    EntityDie = getOrCreateModule(M);
+  else if (auto *SP = dyn_cast<DISubprogram>(Entity))
     EntityDie = getOrCreateSubprogramDIE(SP);
-  else if (auto *T = dyn_cast<MDType>(Entity))
+  else if (auto *T = dyn_cast<DIType>(Entity))
     EntityDie = getOrCreateTypeDIE(T);
-  else if (auto *GV = dyn_cast<MDGlobalVariable>(Entity))
+  else if (auto *GV = dyn_cast<DIGlobalVariable>(Entity))
     EntityDie = getOrCreateGlobalVariableDIE(GV);
   else
     EntityDie = getDIE(Entity);
@@ -659,7 +663,7 @@ DwarfCompileUnit::constructImportedEntityDIE(const DIImportedEntity &Module) {
   return IMDie;
 }
 
-void DwarfCompileUnit::finishSubprogramDefinition(DISubprogram SP) {
+void DwarfCompileUnit::finishSubprogramDefinition(const DISubprogram *SP) {
   DIE *D = getDIE(SP);
   if (DIE *AbsSPDIE = DU->getAbstractSPDies().lookup(SP)) {
     if (D)
@@ -676,7 +680,7 @@ void DwarfCompileUnit::finishSubprogramDefinition(DISubprogram SP) {
       applySubprogramAttributesToDefinition(SP, *D);
   }
 }
-void DwarfCompileUnit::collectDeadVariables(DISubprogram SP) {
+void DwarfCompileUnit::collectDeadVariables(const DISubprogram *SP) {
   assert(SP && "CU's subprogram list contains a non-subprogram");
   assert(SP->isDefinition() &&
          "CU's subprogram list contains a subprogram declaration");
@@ -688,8 +692,8 @@ void DwarfCompileUnit::collectDeadVariables(DISubprogram SP) {
   if (!SPDIE)
     SPDIE = getDIE(SP);
   assert(SPDIE);
-  for (DIVariable DV : Variables) {
-    DbgVariable NewVar(DV, nullptr, DIExpression(), DD);
+  for (const DILocalVariable *DV : Variables) {
+    DbgVariable NewVar(DV, /* IA */ nullptr, DD);
     auto VariableDie = constructVariableDIE(NewVar);
     applyVariableAttributes(NewVar, *VariableDie);
     SPDIE->addChild(std::move(VariableDie));
@@ -700,7 +704,7 @@ void DwarfCompileUnit::emitHeader(bool UseOffsets) {
   // Don't bother labeling the .dwo unit, as its offset isn't used.
   if (!Skeleton) {
     LabelBegin = Asm->createTempSymbol("cu_begin");
-    Asm->OutStreamer.EmitLabel(LabelBegin);
+    Asm->OutStreamer->EmitLabel(LabelBegin);
   }
 
   DwarfUnit::emitHeader(UseOffsets);
@@ -708,7 +712,7 @@ void DwarfCompileUnit::emitHeader(bool UseOffsets) {
 
 /// addGlobalName - Add a new global name to the compile unit.
 void DwarfCompileUnit::addGlobalName(StringRef Name, DIE &Die,
-                                     DIScope Context) {
+                                     const DIScope *Context) {
   if (includeMinimalInlineScopes())
     return;
   std::string FullName = getParentContextString(Context) + Name.str();
@@ -716,8 +720,8 @@ void DwarfCompileUnit::addGlobalName(StringRef Name, DIE &Die,
 }
 
 /// Add a new global type to the unit.
-void DwarfCompileUnit::addGlobalType(DIType Ty, const DIE &Die,
-                                     DIScope Context) {
+void DwarfCompileUnit::addGlobalType(const DIType *Ty, const DIE &Die,
+                                     const DIScope *Context) {
   if (includeMinimalInlineScopes())
     return;
   std::string FullName = getParentContextString(Context) + Ty->getName().str();
@@ -728,7 +732,7 @@ void DwarfCompileUnit::addGlobalType(DIType Ty, const DIE &Die,
 /// DbgVariable based on provided MachineLocation.
 void DwarfCompileUnit::addVariableAddress(const DbgVariable &DV, DIE &Die,
                                           MachineLocation Location) {
-  if (DV.variableHasComplexAddress())
+  if (DV.hasComplexAddress())
     addComplexAddress(DV, Die, dwarf::DW_AT_location, Location);
   else if (DV.isBlockByrefVariable())
     addBlockByrefAddress(DV, Die, dwarf::DW_AT_location, Location);
@@ -739,7 +743,7 @@ void DwarfCompileUnit::addVariableAddress(const DbgVariable &DV, DIE &Die,
 /// Add an address attribute to a die based on the location provided.
 void DwarfCompileUnit::addAddress(DIE &Die, dwarf::Attribute Attribute,
                                   const MachineLocation &Location) {
-  DIELoc *Loc = new (DIEValueAllocator) DIELoc();
+  DIELoc *Loc = new (DIEValueAllocator) DIELoc;
 
   bool validReg;
   if (Location.isReg())
@@ -761,10 +765,10 @@ void DwarfCompileUnit::addAddress(DIE &Die, dwarf::Attribute Attribute,
 void DwarfCompileUnit::addComplexAddress(const DbgVariable &DV, DIE &Die,
                                          dwarf::Attribute Attribute,
                                          const MachineLocation &Location) {
-  DIELoc *Loc = new (DIEValueAllocator) DIELoc();
+  DIELoc *Loc = new (DIEValueAllocator) DIELoc;
   DIEDwarfExpression DwarfExpr(*Asm, *this, *Loc);
   assert(DV.getExpression().size() == 1);
-  DIExpression Expr = DV.getExpression().back();
+  const DIExpression *Expr = DV.getExpression().back();
   bool ValidReg;
   if (Location.getOffset()) {
     ValidReg = DwarfExpr.AddMachineRegIndirect(Location.getReg(),
@@ -782,10 +786,9 @@ void DwarfCompileUnit::addComplexAddress(const DbgVariable &DV, DIE &Die,
 /// Add a Dwarf loclistptr attribute data and value.
 void DwarfCompileUnit::addLocationList(DIE &Die, dwarf::Attribute Attribute,
                                        unsigned Index) {
-  DIEValue *Value = new (DIEValueAllocator) DIELocList(Index);
   dwarf::Form Form = DD->getDwarfVersion() >= 4 ? dwarf::DW_FORM_sec_offset
                                                 : dwarf::DW_FORM_data4;
-  Die.addValue(Attribute, Form, Value);
+  Die.addValue(DIEValueAllocator, Attribute, Form, DIELocList(Index));
 }
 
 void DwarfCompileUnit::applyVariableAttributes(const DbgVariable &Var,
@@ -802,14 +805,13 @@ void DwarfCompileUnit::applyVariableAttributes(const DbgVariable &Var,
 /// Add a Dwarf expression attribute data and value.
 void DwarfCompileUnit::addExpr(DIELoc &Die, dwarf::Form Form,
                                const MCExpr *Expr) {
-  DIEValue *Value = new (DIEValueAllocator) DIEExpr(Expr);
-  Die.addValue((dwarf::Attribute)0, Form, Value);
+  Die.addValue(DIEValueAllocator, (dwarf::Attribute)0, Form, DIEExpr(Expr));
 }
 
-void DwarfCompileUnit::applySubprogramAttributesToDefinition(DISubprogram SP,
-                                                             DIE &SPDie) {
+void DwarfCompileUnit::applySubprogramAttributesToDefinition(
+    const DISubprogram *SP, DIE &SPDie) {
   auto *SPDecl = SP->getDeclaration();
-  DIScope Context = resolve(SPDecl ? SPDecl->getScope() : SP->getScope());
+  auto *Context = resolve(SPDecl ? SPDecl->getScope() : SP->getScope());
   applySubprogramAttributes(SP, SPDie, includeMinimalInlineScopes());
   addGlobalName(SP->getName(), SPDie, Context);
 }
diff --git a/lib/CodeGen/AsmPrinter/DwarfCompileUnit.h b/lib/CodeGen/AsmPrinter/DwarfCompileUnit.h
index 9484bb6f33d..509c9432bcb 100644
--- a/lib/CodeGen/AsmPrinter/DwarfCompileUnit.h
+++ b/lib/CodeGen/AsmPrinter/DwarfCompileUnit.h
@@ -31,7 +31,7 @@ class LexicalScope;
 class DwarfCompileUnit : public DwarfUnit {
   /// The attribute index of DW_AT_stmt_list in the compile unit DIE, avoiding
   /// the need to search for it in applyStmtList.
-  unsigned stmtListIndex;
+  DIE::value_iterator StmtListValue;
 
   /// Skeleton unit associated with this unit.
   DwarfCompileUnit *Skeleton;
@@ -58,15 +58,14 @@ class DwarfCompileUnit : public DwarfUnit {
 
   /// \brief Construct a DIE for the given DbgVariable without initializing the
   /// DbgVariable's DIE reference.
-  std::unique_ptr<DIE> constructVariableDIEImpl(const DbgVariable &DV,
-                                                bool Abstract);
+  DIE *constructVariableDIEImpl(const DbgVariable &DV, bool Abstract);
 
   bool isDwoUnit() const override;
 
   bool includeMinimalInlineScopes() const;
 
 public:
-  DwarfCompileUnit(unsigned UID, DICompileUnit Node, AsmPrinter *A,
+  DwarfCompileUnit(unsigned UID, const DICompileUnit *Node, AsmPrinter *A,
                    DwarfDebug *DW, DwarfFile *DWU);
 
   DwarfCompileUnit *getSkeleton() const {
@@ -79,7 +78,7 @@ class DwarfCompileUnit : public DwarfUnit {
   void applyStmtList(DIE &D);
 
   /// getOrCreateGlobalVariableDIE - get or create global variable DIE.
-  DIE *getOrCreateGlobalVariableDIE(DIGlobalVariable GV);
+  DIE *getOrCreateGlobalVariableDIE(const DIGlobalVariable *GV);
 
   /// addLabelAddress - Add a dwarf label attribute data and value using
   /// either DW_FORM_addr or DW_FORM_GNU_addr_index.
@@ -92,8 +91,8 @@ class DwarfCompileUnit : public DwarfUnit {
                             const MCSymbol *Label);
 
   /// addSectionDelta - Add a label delta attribute data and value.
-  void addSectionDelta(DIE &Die, dwarf::Attribute Attribute, const MCSymbol *Hi,
-                       const MCSymbol *Lo);
+  DIE::value_iterator addSectionDelta(DIE &Die, dwarf::Attribute Attribute,
+                                      const MCSymbol *Hi, const MCSymbol *Lo);
 
   DwarfCompileUnit &getCU() override { return *this; }
 
@@ -106,17 +105,18 @@ class DwarfCompileUnit : public DwarfUnit {
 
   /// addSectionLabel - Add a Dwarf section label attribute data and value.
   ///
-  void addSectionLabel(DIE &Die, dwarf::Attribute Attribute,
-                       const MCSymbol *Label, const MCSymbol *Sec);
+  DIE::value_iterator addSectionLabel(DIE &Die, dwarf::Attribute Attribute,
+                                      const MCSymbol *Label,
+                                      const MCSymbol *Sec);
 
   /// \brief Find DIE for the given subprogram and attach appropriate
   /// DW_AT_low_pc and DW_AT_high_pc attributes. If there are global
   /// variables in this scope then create and insert DIEs for these
   /// variables.
-  DIE &updateSubprogramScopeDIE(DISubprogram SP);
+  DIE &updateSubprogramScopeDIE(const DISubprogram *SP);
 
   void constructScopeDIE(LexicalScope *Scope,
-                         SmallVectorImpl<std::unique_ptr<DIE>> &FinalChildren);
+                         SmallVectorImpl<DIE *> &FinalChildren);
 
   /// \brief A helper function to construct a RangeSpanList for a given
   /// lexical scope.
@@ -128,23 +128,21 @@ class DwarfCompileUnit : public DwarfUnit {
                                const SmallVectorImpl<InsnRange> &Ranges);
   /// \brief This scope represents inlined body of a function. Construct
   /// DIE to represent this concrete inlined copy of the function.
-  std::unique_ptr<DIE> constructInlinedScopeDIE(LexicalScope *Scope);
+  DIE *constructInlinedScopeDIE(LexicalScope *Scope);
 
   /// \brief Construct new DW_TAG_lexical_block for this scope and
   /// attach DW_AT_low_pc/DW_AT_high_pc labels.
-  std::unique_ptr<DIE> constructLexicalScopeDIE(LexicalScope *Scope);
+  DIE *constructLexicalScopeDIE(LexicalScope *Scope);
 
   /// constructVariableDIE - Construct a DIE for the given DbgVariable.
-  std::unique_ptr<DIE> constructVariableDIE(DbgVariable &DV,
-                                            bool Abstract = false);
+  DIE *constructVariableDIE(DbgVariable &DV, bool Abstract = false);
 
-  std::unique_ptr<DIE> constructVariableDIE(DbgVariable &DV,
-                                            const LexicalScope &Scope,
-                                            DIE *&ObjectPointer);
+  DIE *constructVariableDIE(DbgVariable &DV, const LexicalScope &Scope,
+                            DIE *&ObjectPointer);
 
   /// A helper function to create children of a Scope DIE.
   DIE *createScopeChildrenDIE(LexicalScope *Scope,
-                              SmallVectorImpl<std::unique_ptr<DIE>> &Children,
+                              SmallVectorImpl<DIE *> &Children,
                               unsigned *ChildScopeCount = nullptr);
 
   /// \brief Construct a DIE for this subprogram scope.
@@ -155,12 +153,11 @@ class DwarfCompileUnit : public DwarfUnit {
   void constructAbstractSubprogramScopeDIE(LexicalScope *Scope);
 
   /// \brief Construct import_module DIE.
-  std::unique_ptr<DIE>
-  constructImportedEntityDIE(const DIImportedEntity &Module);
+  DIE *constructImportedEntityDIE(const DIImportedEntity *Module);
 
-  void finishSubprogramDefinition(DISubprogram SP);
+  void finishSubprogramDefinition(const DISubprogram *SP);
 
-  void collectDeadVariables(DISubprogram SP);
+  void collectDeadVariables(const DISubprogram *SP);
 
   /// Set the skeleton unit associated with this unit.
   void setSkeleton(DwarfCompileUnit &Skel) { Skeleton = &Skel; }
@@ -183,10 +180,11 @@ class DwarfCompileUnit : public DwarfUnit {
   }
 
   /// Add a new global name to the compile unit.
-  void addGlobalName(StringRef Name, DIE &Die, DIScope Context) override;
+  void addGlobalName(StringRef Name, DIE &Die, const DIScope *Context) override;
 
   /// Add a new global type to the compile unit.
-  void addGlobalType(DIType Ty, const DIE &Die, DIScope Context) override;
+  void addGlobalType(const DIType *Ty, const DIE &Die,
+                     const DIScope *Context) override;
 
   const StringMap<const DIE *> &getGlobalNames() const { return GlobalNames; }
   const StringMap<const DIE *> &getGlobalTypes() const { return GlobalTypes; }
@@ -214,7 +212,8 @@ class DwarfCompileUnit : public DwarfUnit {
   /// Add a Dwarf expression attribute data and value.
   void addExpr(DIELoc &Die, dwarf::Form Form, const MCExpr *Expr);
 
-  void applySubprogramAttributesToDefinition(DISubprogram SP, DIE &SPDie);
+  void applySubprogramAttributesToDefinition(const DISubprogram *SP,
+                                             DIE &SPDie);
 
   /// getRangeLists - Get the vector of range lists.
   const SmallVectorImpl<RangeSpanList> &getRangeLists() const {
diff --git a/lib/CodeGen/AsmPrinter/DwarfDebug.cpp b/lib/CodeGen/AsmPrinter/DwarfDebug.cpp
index a083393b3a3..5611ae6fcd8 100644
--- a/lib/CodeGen/AsmPrinter/DwarfDebug.cpp
+++ b/lib/CodeGen/AsmPrinter/DwarfDebug.cpp
@@ -14,6 +14,7 @@
 #include "DwarfDebug.h"
 #include "ByteStreamer.h"
 #include "DIEHash.h"
+#include "DebugLocEntry.h"
 #include "DwarfCompileUnit.h"
 #include "DwarfExpression.h"
 #include "DwarfUnit.h"
@@ -75,6 +76,17 @@ static cl::opt<bool> GenerateARangeSection("generate-arange-section",
                                            cl::desc("Generate dwarf aranges"),
                                            cl::init(false));
 
+static cl::opt<DebuggerKind>
+DebuggerTuningOpt("debugger-tune",
+                  cl::desc("Tune debug info for a particular debugger"),
+                  cl::init(DebuggerKind::Default),
+                  cl::values(
+                      clEnumValN(DebuggerKind::GDB, "gdb", "gdb"),
+                      clEnumValN(DebuggerKind::LLDB, "lldb", "lldb"),
+                      clEnumValN(DebuggerKind::SCE, "sce",
+                                 "SCE targets (e.g. PS4)"),
+                      clEnumValEnd));
+
 namespace {
 enum DefaultOnOff { Default, Enable, Disable };
 }
@@ -129,7 +141,7 @@ bool DebugLocDwarfExpression::isFrameRegister(unsigned MachineReg) {
 
 /// resolve - Look in the DwarfDebug map for the MDNode that
 /// corresponds to the reference.
-template <typename T> T *DbgVariable::resolve(TypedDebugNodeRef<T> Ref) const {
+template <typename T> T *DbgVariable::resolve(TypedDINodeRef<T> Ref) const {
   return DD->resolve(Ref);
 }
 
@@ -140,8 +152,8 @@ bool DbgVariable::isBlockByrefVariable() const {
       ->isBlockByrefStruct();
 }
 
-DIType DbgVariable::getType() const {
-  MDType *Ty = Var->getType().resolve(DD->getTypeIdentifierMap());
+const DIType *DbgVariable::getType() const {
+  DIType *Ty = Var->getType().resolve(DD->getTypeIdentifierMap());
   // FIXME: isBlockByrefVariable should be reformulated in terms of complex
   // addresses instead.
   if (Ty->isBlockByrefStruct()) {
@@ -169,17 +181,17 @@ DIType DbgVariable::getType() const {
        have a DW_AT_location that tells the debugger how to unwind through
        the pointers and __Block_byref_x_VarName struct to find the actual
        value of the variable.  The function addBlockByrefType does this.  */
-    MDType *subType = Ty;
+    DIType *subType = Ty;
     uint16_t tag = Ty->getTag();
 
     if (tag == dwarf::DW_TAG_pointer_type)
-      subType = resolve(DITypeRef(cast<MDDerivedType>(Ty)->getBaseType()));
+      subType = resolve(cast<DIDerivedType>(Ty)->getBaseType());
 
-    auto Elements = cast<MDCompositeTypeBase>(subType)->getElements();
+    auto Elements = cast<DICompositeType>(subType)->getElements();
     for (unsigned i = 0, N = Elements.size(); i < N; ++i) {
-      auto *DT = cast<MDDerivedTypeBase>(Elements[i]);
+      auto *DT = cast<DIDerivedType>(Elements[i]);
       if (getName() == DT->getName())
-        return resolve(DITypeRef(DT->getBaseType()));
+        return resolve(DT->getBaseType());
     }
   }
   return Ty;
@@ -191,37 +203,49 @@ static LLVM_CONSTEXPR DwarfAccelTable::Atom TypeAtoms[] = {
     DwarfAccelTable::Atom(dwarf::DW_ATOM_type_flags, dwarf::DW_FORM_data1)};
 
 DwarfDebug::DwarfDebug(AsmPrinter *A, Module *M)
-    : Asm(A), MMI(Asm->MMI), PrevLabel(nullptr),
-      InfoHolder(A, "info_string", DIEValueAllocator),
+    : Asm(A), MMI(Asm->MMI), DebugLocs(A->OutStreamer->isVerboseAsm()),
+      PrevLabel(nullptr), InfoHolder(A, "info_string", DIEValueAllocator),
       UsedNonDefaultText(false),
       SkeletonHolder(A, "skel_string", DIEValueAllocator),
       IsDarwin(Triple(A->getTargetTriple()).isOSDarwin()),
-      IsPS4(Triple(A->getTargetTriple()).isPS4()),
       AccelNames(DwarfAccelTable::Atom(dwarf::DW_ATOM_die_offset,
                                        dwarf::DW_FORM_data4)),
       AccelObjC(DwarfAccelTable::Atom(dwarf::DW_ATOM_die_offset,
                                       dwarf::DW_FORM_data4)),
       AccelNamespace(DwarfAccelTable::Atom(dwarf::DW_ATOM_die_offset,
                                            dwarf::DW_FORM_data4)),
-      AccelTypes(TypeAtoms) {
+      AccelTypes(TypeAtoms), DebuggerTuning(DebuggerKind::Default) {
 
   CurFn = nullptr;
   CurMI = nullptr;
+  Triple TT(Asm->getTargetTriple());
+
+  // Make sure we know our "debugger tuning."  The command-line option takes
+  // precedence; fall back to triple-based defaults.
+  if (DebuggerTuningOpt != DebuggerKind::Default)
+    DebuggerTuning = DebuggerTuningOpt;
+  else if (IsDarwin || TT.isOSFreeBSD())
+    DebuggerTuning = DebuggerKind::LLDB;
+  else if (TT.isPS4CPU())
+    DebuggerTuning = DebuggerKind::SCE;
+  else
+    DebuggerTuning = DebuggerKind::GDB;
 
-  // Turn on accelerator tables for Darwin by default, pubnames by
-  // default for non-Darwin/PS4, and handle split dwarf.
+  // Turn on accelerator tables for LLDB by default.
   if (DwarfAccelTables == Default)
-    HasDwarfAccelTables = IsDarwin;
+    HasDwarfAccelTables = tuneForLLDB();
   else
     HasDwarfAccelTables = DwarfAccelTables == Enable;
 
+  // Handle split DWARF. Off by default for now.
   if (SplitDwarf == Default)
     HasSplitDwarf = false;
   else
     HasSplitDwarf = SplitDwarf == Enable;
 
+  // Pubnames/pubtypes on by default for GDB.
   if (DwarfPubSections == Default)
-    HasDwarfPubSections = !IsDarwin && !IsPS4;
+    HasDwarfPubSections = tuneForGDB();
   else
     HasDwarfPubSections = DwarfPubSections == Enable;
 
@@ -229,11 +253,14 @@ DwarfDebug::DwarfDebug(AsmPrinter *A, Module *M)
   DwarfVersion = DwarfVersionNumber ? DwarfVersionNumber
                                     : MMI->getModule()->getDwarfVersion();
 
-  // Darwin and PS4 use the standard TLS opcode (defined in DWARF 3).
-  // Everybody else uses GNU's.
-  UseGNUTLSOpcode = !(IsDarwin || IsPS4) || DwarfVersion < 3;
+  // Work around a GDB bug. GDB doesn't support the standard opcode;
+  // SCE doesn't support GNU's; LLDB prefers the standard opcode, which
+  // is defined as of DWARF 3.
+  // See GDB bug 11616 - DW_OP_form_tls_address is unimplemented
+  // https://sourceware.org/bugzilla/show_bug.cgi?id=11616
+  UseGNUTLSOpcode = tuneForGDB() || DwarfVersion < 3;
 
-  Asm->OutStreamer.getContext().setDwarfVersion(DwarfVersion);
+  Asm->OutStreamer->getContext().setDwarfVersion(DwarfVersion);
 
   {
     NamedRegionTimer T(DbgTimerName, DWARFGroupName, TimePassesIsEnabled);
@@ -276,7 +303,7 @@ static StringRef getObjCMethodName(StringRef In) {
 // TODO: Determine whether or not we should add names for programs
 // that do not have a DW_AT_name or DW_AT_linkage_name field - this
 // is only slightly different than the lookup of non-standard ObjC names.
-void DwarfDebug::addSubprogramNames(DISubprogram SP, DIE &Die) {
+void DwarfDebug::addSubprogramNames(const DISubprogram *SP, DIE &Die) {
   if (!SP->isDefinition())
     return;
   addAccelName(SP->getName(), Die);
@@ -304,9 +331,9 @@ void DwarfDebug::addSubprogramNames(DISubprogram SP, DIE &Die) {
 bool DwarfDebug::isSubprogramContext(const MDNode *Context) {
   if (!Context)
     return false;
-  if (isa<MDSubprogram>(Context))
+  if (isa<DISubprogram>(Context))
     return true;
-  if (auto *T = dyn_cast<MDType>(Context))
+  if (auto *T = dyn_cast<DIType>(Context))
     return isSubprogramContext(resolve(T->getScope()));
   return false;
 }
@@ -362,7 +389,8 @@ void DwarfDebug::addGnuPubAttributes(DwarfUnit &U, DIE &D) const {
 
 // Create new DwarfCompileUnit for the given metadata node with tag
 // DW_TAG_compile_unit.
-DwarfCompileUnit &DwarfDebug::constructDwarfCompileUnit(DICompileUnit DIUnit) {
+DwarfCompileUnit &
+DwarfDebug::constructDwarfCompileUnit(const DICompileUnit *DIUnit) {
   StringRef FN = DIUnit->getFilename();
   CompilationDir = DIUnit->getDirectory();
 
@@ -378,8 +406,8 @@ DwarfCompileUnit &DwarfDebug::constructDwarfCompileUnit(DICompileUnit DIUnit) {
   // To avoid the compilation directory being ambiguous, let the line table
   // explicitly describe the directory of all files, never relying on the
   // compilation directory.
-  if (!Asm->OutStreamer.hasRawTextSupport() || SingleCU)
-    Asm->OutStreamer.getContext().setMCLineTableCompilationDir(
+  if (!Asm->OutStreamer->hasRawTextSupport() || SingleCU)
+    Asm->OutStreamer->getContext().setMCLineTableCompilationDir(
         NewCU.getUniqueID(), CompilationDir);
 
   NewCU.addString(Die, dwarf::DW_AT_producer, DIUnit->getProducer());
@@ -420,10 +448,9 @@ DwarfCompileUnit &DwarfDebug::constructDwarfCompileUnit(DICompileUnit DIUnit) {
 }
 
 void DwarfDebug::constructAndAddImportedEntityDIE(DwarfCompileUnit &TheCU,
-                                                  const MDNode *N) {
-  DIImportedEntity Module = cast<MDImportedEntity>(N);
-  if (DIE *D = TheCU.getOrCreateContextDIE(Module->getScope()))
-    D->addChild(TheCU.constructImportedEntityDIE(Module));
+                                                  const DIImportedEntity *N) {
+  if (DIE *D = TheCU.getOrCreateContextDIE(N->getScope()))
+    D->addChild(TheCU.constructImportedEntityDIE(N));
 }
 
 // Emit all Dwarf sections that should come prior to the content. Create
@@ -445,7 +472,7 @@ void DwarfDebug::beginModule() {
   SingleCU = CU_Nodes->getNumOperands() == 1;
 
   for (MDNode *N : CU_Nodes->operands()) {
-    DICompileUnit CUNode = cast<MDCompileUnit>(N);
+    auto *CUNode = cast<DICompileUnit>(N);
     DwarfCompileUnit &CU = constructDwarfCompileUnit(CUNode);
     for (auto *IE : CUNode->getImportedEntities())
       ScopesWithImportedEntities.push_back(std::make_pair(IE->getScope(), IE));
@@ -458,17 +485,18 @@ void DwarfDebug::beginModule() {
       CU.getOrCreateGlobalVariableDIE(GV);
     for (auto *SP : CUNode->getSubprograms())
       SPMap.insert(std::make_pair(SP, &CU));
-    for (DIType Ty : CUNode->getEnumTypes()) {
+    for (auto *Ty : CUNode->getEnumTypes()) {
       // The enum types array by design contains pointers to
       // MDNodes rather than DIRefs. Unique them here.
-      DIType UniqueTy = cast<MDType>(resolve(Ty->getRef()));
-      CU.getOrCreateTypeDIE(UniqueTy);
+      CU.getOrCreateTypeDIE(cast<DIType>(resolve(Ty->getRef())));
     }
-    for (DIType Ty : CUNode->getRetainedTypes()) {
+    for (auto *Ty : CUNode->getRetainedTypes()) {
       // The retained types array by design contains pointers to
       // MDNodes rather than DIRefs. Unique them here.
-      DIType UniqueTy = cast<MDType>(resolve(Ty->getRef()));
-      CU.getOrCreateTypeDIE(UniqueTy);
+      DIType *RT = cast<DIType>(resolve(Ty->getRef()));
+      if (!RT->isExternalTypeRef())
+        // There is no point in force-emitting a forward declaration.
+        CU.getOrCreateTypeDIE(RT);
     }
     // Emit imported_modules last so that the relevant context is already
     // available.
@@ -502,7 +530,7 @@ void DwarfDebug::finishVariableDefinitions() {
 void DwarfDebug::finishSubprogramDefinitions() {
   for (const auto &P : SPMap)
     forBothCUs(*P.second, [&](DwarfCompileUnit &CU) {
-      CU.finishSubprogramDefinition(cast<MDSubprogram>(P.first));
+      CU.finishSubprogramDefinition(cast<DISubprogram>(P.first));
     });
 }
 
@@ -513,7 +541,7 @@ void DwarfDebug::collectDeadVariables() {
 
   if (NamedMDNode *CU_Nodes = M->getNamedMetadata("llvm.dbg.cu")) {
     for (MDNode *N : CU_Nodes->operands()) {
-      DICompileUnit TheCU = cast<MDCompileUnit>(N);
+      auto *TheCU = cast<DICompileUnit>(N);
       // Construct subprogram DIE and add variables DIEs.
       DwarfCompileUnit *SPCU =
           static_cast<DwarfCompileUnit *>(CUMap.lookup(TheCU));
@@ -586,7 +614,7 @@ void DwarfDebug::finalizeModuleInfo() {
         // 2.17.3).
         U.addUInt(U.getUnitDie(), dwarf::DW_AT_low_pc, dwarf::DW_FORM_addr, 0);
       else
-        TheCU.setBaseAddress(TheCU.getRanges().front().getStart());
+        U.setBaseAddress(TheCU.getRanges().front().getStart());
       U.attachRangesOrLowHighPC(U.getUnitDie(), TheCU.takeRanges());
     }
   }
@@ -661,8 +689,9 @@ void DwarfDebug::endModule() {
 }
 
 // Find abstract variable, if any, associated with Var.
-DbgVariable *DwarfDebug::getExistingAbstractVariable(InlinedVariable IV,
-                                                     DIVariable &Cleansed) {
+DbgVariable *
+DwarfDebug::getExistingAbstractVariable(InlinedVariable IV,
+                                        const DILocalVariable *&Cleansed) {
   // More then one inlined variable corresponds to one abstract variable.
   Cleansed = IV.first;
   auto I = AbstractVariables.find(Cleansed);
@@ -672,36 +701,35 @@ DbgVariable *DwarfDebug::getExistingAbstractVariable(InlinedVariable IV,
 }
 
 DbgVariable *DwarfDebug::getExistingAbstractVariable(InlinedVariable IV) {
-  DIVariable Cleansed;
+  const DILocalVariable *Cleansed;
   return getExistingAbstractVariable(IV, Cleansed);
 }
 
-void DwarfDebug::createAbstractVariable(const DIVariable &Var,
+void DwarfDebug::createAbstractVariable(const DILocalVariable *Var,
                                         LexicalScope *Scope) {
-  auto AbsDbgVariable =
-      make_unique<DbgVariable>(Var, nullptr, DIExpression(), this);
+  auto AbsDbgVariable = make_unique<DbgVariable>(Var, /* IA */ nullptr, this);
   InfoHolder.addScopeVariable(Scope, AbsDbgVariable.get());
   AbstractVariables[Var] = std::move(AbsDbgVariable);
 }
 
 void DwarfDebug::ensureAbstractVariableIsCreated(InlinedVariable IV,
                                                  const MDNode *ScopeNode) {
-  DIVariable Cleansed;
+  const DILocalVariable *Cleansed = nullptr;
   if (getExistingAbstractVariable(IV, Cleansed))
     return;
 
   createAbstractVariable(Cleansed, LScopes.getOrCreateAbstractScope(
-                                       cast<MDLocalScope>(ScopeNode)));
+                                       cast<DILocalScope>(ScopeNode)));
 }
 
 void DwarfDebug::ensureAbstractVariableIsCreatedIfScoped(
     InlinedVariable IV, const MDNode *ScopeNode) {
-  DIVariable Cleansed;
+  const DILocalVariable *Cleansed = nullptr;
   if (getExistingAbstractVariable(IV, Cleansed))
     return;
 
   if (LexicalScope *Scope =
-          LScopes.findAbstractScope(cast_or_null<MDLocalScope>(ScopeNode)))
+          LScopes.findAbstractScope(cast_or_null<DILocalScope>(ScopeNode)))
     createAbstractVariable(Cleansed, Scope);
 }
 
@@ -722,10 +750,9 @@ void DwarfDebug::collectVariableInfoFromMMITable(
     if (!Scope)
       continue;
 
-    DIExpression Expr = cast_or_null<MDExpression>(VI.Expr);
     ensureAbstractVariableIsCreatedIfScoped(Var, Scope->getScopeNode());
-    auto RegVar =
-        make_unique<DbgVariable>(Var.first, Var.second, Expr, this, VI.Slot);
+    auto RegVar = make_unique<DbgVariable>(Var.first, Var.second, this);
+    RegVar->initializeMMI(VI.Expr, VI.Slot);
     if (InfoHolder.addScopeVariable(Scope, RegVar.get()))
       ConcreteVariables.push_back(std::move(RegVar));
   }
@@ -733,8 +760,7 @@ void DwarfDebug::collectVariableInfoFromMMITable(
 
 // Get .debug_loc entry for the instruction range starting at MI.
 static DebugLocEntry::Value getDebugLocValue(const MachineInstr *MI) {
-  const MDNode *Expr = MI->getDebugExpression();
-  const MDNode *Var = MI->getDebugVariable();
+  const DIExpression *Expr = MI->getDebugExpression();
 
   assert(MI->getNumOperands() == 4);
   if (MI->getOperand(0).isReg()) {
@@ -745,20 +771,20 @@ static DebugLocEntry::Value getDebugLocValue(const MachineInstr *MI) {
       MLoc.set(MI->getOperand(0).getReg());
     else
       MLoc.set(MI->getOperand(0).getReg(), MI->getOperand(1).getImm());
-    return DebugLocEntry::Value(Var, Expr, MLoc);
+    return DebugLocEntry::Value(Expr, MLoc);
   }
   if (MI->getOperand(0).isImm())
-    return DebugLocEntry::Value(Var, Expr, MI->getOperand(0).getImm());
+    return DebugLocEntry::Value(Expr, MI->getOperand(0).getImm());
   if (MI->getOperand(0).isFPImm())
-    return DebugLocEntry::Value(Var, Expr, MI->getOperand(0).getFPImm());
+    return DebugLocEntry::Value(Expr, MI->getOperand(0).getFPImm());
   if (MI->getOperand(0).isCImm())
-    return DebugLocEntry::Value(Var, Expr, MI->getOperand(0).getCImm());
+    return DebugLocEntry::Value(Expr, MI->getOperand(0).getCImm());
 
   llvm_unreachable("Unexpected 4-operand DBG_VALUE instruction!");
 }
 
 /// Determine whether two variable pieces overlap.
-static bool piecesOverlap(DIExpression P1, DIExpression P2) {
+static bool piecesOverlap(const DIExpression *P1, const DIExpression *P2) {
   if (!P1->isBitPiece() || !P2->isBitPiece())
     return true;
   unsigned l1 = P1->getBitPieceOffset();
@@ -810,7 +836,7 @@ DwarfDebug::buildLocationList(SmallVectorImpl<DebugLocEntry> &DebugLoc,
     }
 
     // If this piece overlaps with any open ranges, truncate them.
-    DIExpression DIExpr = Begin->getDebugExpression();
+    const DIExpression *DIExpr = Begin->getDebugExpression();
     auto Last = std::remove_if(OpenRanges.begin(), OpenRanges.end(),
                                [&](DebugLocEntry::Value R) {
       return piecesOverlap(DIExpr, R.getExpression());
@@ -858,24 +884,31 @@ DwarfDebug::buildLocationList(SmallVectorImpl<DebugLocEntry> &DebugLoc,
     // Attempt to coalesce the ranges of two otherwise identical
     // DebugLocEntries.
     auto CurEntry = DebugLoc.rbegin();
-    auto PrevEntry = std::next(CurEntry);
-    if (PrevEntry != DebugLoc.rend() && PrevEntry->MergeRanges(*CurEntry))
-      DebugLoc.pop_back();
-
     DEBUG({
       dbgs() << CurEntry->getValues().size() << " Values:\n";
-      for (auto Value : CurEntry->getValues()) {
-        Value.getVariable()->dump();
+      for (auto &Value : CurEntry->getValues())
         Value.getExpression()->dump();
-      }
       dbgs() << "-----\n";
     });
+
+    auto PrevEntry = std::next(CurEntry);
+    if (PrevEntry != DebugLoc.rend() && PrevEntry->MergeRanges(*CurEntry))
+      DebugLoc.pop_back();
   }
 }
 
+DbgVariable *DwarfDebug::createConcreteVariable(LexicalScope &Scope,
+                                                InlinedVariable IV) {
+  ensureAbstractVariableIsCreatedIfScoped(IV, Scope.getScopeNode());
+  ConcreteVariables.push_back(
+      make_unique<DbgVariable>(IV.first, IV.second, this));
+  InfoHolder.addScopeVariable(&Scope, ConcreteVariables.back().get());
+  return ConcreteVariables.back().get();
+}
 
 // Find variables for each lexical scope.
-void DwarfDebug::collectVariableInfo(DwarfCompileUnit &TheCU, DISubprogram SP,
+void DwarfDebug::collectVariableInfo(DwarfCompileUnit &TheCU,
+                                     const DISubprogram *SP,
                                      DenseSet<InlinedVariable> &Processed) {
   // Grab the variable info that was squirreled away in the MMI side-table.
   collectVariableInfoFromMMITable(Processed);
@@ -891,7 +924,7 @@ void DwarfDebug::collectVariableInfo(DwarfCompileUnit &TheCU, DISubprogram SP,
       continue;
 
     LexicalScope *Scope = nullptr;
-    if (const MDLocation *IA = IV.second)
+    if (const DILocation *IA = IV.second)
       Scope = LScopes.findInlinedScope(IV.first->getScope(), IA);
     else
       Scope = LScopes.findLexicalScope(IV.first->getScope());
@@ -900,44 +933,40 @@ void DwarfDebug::collectVariableInfo(DwarfCompileUnit &TheCU, DISubprogram SP,
       continue;
 
     Processed.insert(IV);
+    DbgVariable *RegVar = createConcreteVariable(*Scope, IV);
+
     const MachineInstr *MInsn = Ranges.front().first;
     assert(MInsn->isDebugValue() && "History must begin with debug value");
-    ensureAbstractVariableIsCreatedIfScoped(IV, Scope->getScopeNode());
-    ConcreteVariables.push_back(make_unique<DbgVariable>(MInsn, this));
-    DbgVariable *RegVar = ConcreteVariables.back().get();
-    InfoHolder.addScopeVariable(Scope, RegVar);
 
     // Check if the first DBG_VALUE is valid for the rest of the function.
-    if (Ranges.size() == 1 && Ranges.front().second == nullptr)
+    if (Ranges.size() == 1 && Ranges.front().second == nullptr) {
+      RegVar->initializeDbgValue(MInsn);
       continue;
+    }
 
     // Handle multiple DBG_VALUE instructions describing one variable.
-    RegVar->setDotDebugLocOffset(DotDebugLocEntries.size());
-
-    DotDebugLocEntries.resize(DotDebugLocEntries.size() + 1);
-    DebugLocList &LocList = DotDebugLocEntries.back();
-    LocList.CU = &TheCU;
-    LocList.Label = Asm->createTempSymbol("debug_loc");
+    DebugLocStream::ListBuilder List(DebugLocs, TheCU, *Asm, *RegVar, *MInsn);
 
     // Build the location list for this variable.
-    buildLocationList(LocList.List, Ranges);
+    SmallVector<DebugLocEntry, 8> Entries;
+    buildLocationList(Entries, Ranges);
+
+    // If the variable has an DIBasicType, extract it.  Basic types cannot have
+    // unique identifiers, so don't bother resolving the type with the
+    // identifier map.
+    const DIBasicType *BT = dyn_cast<DIBasicType>(
+        static_cast<const Metadata *>(IV.first->getType()));
+
     // Finalize the entry by lowering it into a DWARF bytestream.
-    for (auto &Entry : LocList.List)
-      Entry.finalize(*Asm, TypeIdentifierMap);
+    for (auto &Entry : Entries)
+      Entry.finalize(*Asm, List, BT);
   }
 
   // Collect info for variables that were optimized out.
-  for (DIVariable DV : SP->getVariables()) {
-    if (!Processed.insert(InlinedVariable(DV, nullptr)).second)
-      continue;
-    if (LexicalScope *Scope = LScopes.findLexicalScope(DV->getScope())) {
-      ensureAbstractVariableIsCreatedIfScoped(InlinedVariable(DV, nullptr),
-                                              Scope->getScopeNode());
-      DIExpression NoExpr;
-      ConcreteVariables.push_back(
-          make_unique<DbgVariable>(DV, nullptr, NoExpr, this));
-      InfoHolder.addScopeVariable(Scope, ConcreteVariables.back().get());
-    }
+  for (const DILocalVariable *DV : SP->getVariables()) {
+    if (Processed.insert(InlinedVariable(DV, nullptr)).second)
+      if (LexicalScope *Scope = LScopes.findLexicalScope(DV->getScope()))
+        createConcreteVariable(*Scope, InlinedVariable(DV, nullptr));
   }
 }
 
@@ -970,7 +999,7 @@ void DwarfDebug::beginInstruction(const MachineInstr *MI) {
           Flags |= DWARF2_FLAG_IS_STMT;
         }
         if (DL.getLine() !=
-            Asm->OutStreamer.getContext().getCurrentDwarfLoc().getLine())
+            Asm->OutStreamer->getContext().getCurrentDwarfLoc().getLine())
           Flags |= DWARF2_FLAG_IS_STMT;
 
         const MDNode *Scope = DL.getScope();
@@ -995,8 +1024,8 @@ void DwarfDebug::beginInstruction(const MachineInstr *MI) {
     return;
 
   if (!PrevLabel) {
-    PrevLabel = MMI->getContext().CreateTempSymbol();
-    Asm->OutStreamer.EmitLabel(PrevLabel);
+    PrevLabel = MMI->getContext().createTempSymbol();
+    Asm->OutStreamer->EmitLabel(PrevLabel);
   }
   I->second = PrevLabel;
 }
@@ -1023,8 +1052,8 @@ void DwarfDebug::endInstruction() {
 
   // We need a label after this instruction.
   if (!PrevLabel) {
-    PrevLabel = MMI->getContext().CreateTempSymbol();
-    Asm->OutStreamer.EmitLabel(PrevLabel);
+    PrevLabel = MMI->getContext().createTempSymbol();
+    Asm->OutStreamer->EmitLabel(PrevLabel);
   }
   I->second = PrevLabel;
 }
@@ -1108,11 +1137,11 @@ void DwarfDebug::beginFunction(const MachineFunction *MF) {
   // is absolute (such as an <> lookup header)))
   DwarfCompileUnit *TheCU = SPMap.lookup(FnScope->getScopeNode());
   assert(TheCU && "Unable to find compile unit!");
-  if (Asm->OutStreamer.hasRawTextSupport())
+  if (Asm->OutStreamer->hasRawTextSupport())
     // Use a single line table if we are generating assembly.
-    Asm->OutStreamer.getContext().setDwarfCompileUnitID(0);
+    Asm->OutStreamer->getContext().setDwarfCompileUnitID(0);
   else
-    Asm->OutStreamer.getContext().setDwarfCompileUnitID(TheCU->getUniqueID());
+    Asm->OutStreamer->getContext().setDwarfCompileUnitID(TheCU->getUniqueID());
 
   // Calculate history for local variables.
   calculateDbgValueHistory(MF, Asm->MF->getSubtarget().getRegisterInfo(),
@@ -1126,14 +1155,14 @@ void DwarfDebug::beginFunction(const MachineFunction *MF) {
 
     // The first mention of a function argument gets the CurrentFnBegin
     // label, so arguments are visible when breaking at function entry.
-    DIVariable DIVar = Ranges.front().first->getDebugVariable();
+    const DILocalVariable *DIVar = Ranges.front().first->getDebugVariable();
     if (DIVar->getTag() == dwarf::DW_TAG_arg_variable &&
         getDISubprogram(DIVar->getScope())->describes(MF->getFunction())) {
       LabelsBeforeInsn[Ranges.front().first] = Asm->getFunctionBegin();
       if (Ranges.front().first->getDebugExpression()->isBitPiece()) {
         // Mark all non-overlapping initial pieces.
         for (auto I = Ranges.begin(); I != Ranges.end(); ++I) {
-          DIExpression Piece = I->first->getDebugExpression();
+          const DIExpression *Piece = I->first->getDebugExpression();
           if (std::all_of(Ranges.begin(), I,
                           [&](DbgValueHistoryMap::InstrRange Pred) {
                 return !piecesOverlap(Piece, Pred.first->getDebugExpression());
@@ -1157,7 +1186,7 @@ void DwarfDebug::beginFunction(const MachineFunction *MF) {
 
   // Record beginning of function.
   PrologEndLoc = findPrologueEndLoc(MF);
-  if (MDLocation *L = PrologEndLoc) {
+  if (DILocation *L = PrologEndLoc) {
     // We'd like to list the prologue as "not statements" but GDB behaves
     // poorly if we do that. Revisit this with caution/GDB (7.5+) testing.
     auto *SP = L->getInlinedAtScope()->getSubprogram();
@@ -1181,10 +1210,10 @@ void DwarfDebug::endFunction(const MachineFunction *MF) {
   }
 
   // Set DwarfDwarfCompileUnitID in MCContext to default value.
-  Asm->OutStreamer.getContext().setDwarfCompileUnitID(0);
+  Asm->OutStreamer->getContext().setDwarfCompileUnitID(0);
 
   LexicalScope *FnScope = LScopes.getCurrentFunctionScope();
-  DISubprogram SP = cast<MDSubprogram>(FnScope->getScopeNode());
+  auto *SP = cast<DISubprogram>(FnScope->getScopeNode());
   DwarfCompileUnit &TheCU = *SPMap.lookup(SP);
 
   DenseSet<InlinedVariable> ProcessedVars;
@@ -1201,11 +1230,7 @@ void DwarfDebug::endFunction(const MachineFunction *MF) {
     assert(DbgValues.empty());
     // FIXME: This wouldn't be true in LTO with a -g (with inlining) CU followed
     // by a -gmlt CU. Add a test and remove this assertion.
-    // @LOCALMOD-BEGIN -- The PNaCl IRT build does bitcode linking
-    // and mixes -g w/ -gmlt.
-    // https://code.google.com/p/nativeclient/issues/detail?id=4026
-    // assert(AbstractVariables.empty());
-    // @LOCALMOD-END
+    assert(AbstractVariables.empty());
     LabelsBeforeInsn.clear();
     LabelsAfterInsn.clear();
     PrevLabel = nullptr;
@@ -1218,9 +1243,9 @@ void DwarfDebug::endFunction(const MachineFunction *MF) {
 #endif
   // Construct abstract scopes.
   for (LexicalScope *AScope : LScopes.getAbstractScopesList()) {
-    DISubprogram SP = cast<MDSubprogram>(AScope->getScopeNode());
+    auto *SP = cast<DISubprogram>(AScope->getScopeNode());
     // Collect info for variables that were optimized out.
-    for (DIVariable DV : SP->getVariables()) {
+    for (const DILocalVariable *DV : SP->getVariables()) {
       if (!ProcessedVars.insert(InlinedVariable(DV, nullptr)).second)
         continue;
       ensureAbstractVariableIsCreated(InlinedVariable(DV, nullptr),
@@ -1256,18 +1281,18 @@ void DwarfDebug::recordSourceLine(unsigned Line, unsigned Col, const MDNode *S,
   StringRef Dir;
   unsigned Src = 1;
   unsigned Discriminator = 0;
-  if (auto *Scope = cast_or_null<MDScope>(S)) {
+  if (auto *Scope = cast_or_null<DIScope>(S)) {
     Fn = Scope->getFilename();
     Dir = Scope->getDirectory();
-    if (auto *LBF = dyn_cast<MDLexicalBlockFile>(Scope))
+    if (auto *LBF = dyn_cast<DILexicalBlockFile>(Scope))
       Discriminator = LBF->getDiscriminator();
 
-    unsigned CUID = Asm->OutStreamer.getContext().getDwarfCompileUnitID();
+    unsigned CUID = Asm->OutStreamer->getContext().getDwarfCompileUnitID();
     Src = static_cast<DwarfCompileUnit &>(*InfoHolder.getUnits()[CUID])
               .getOrCreateSourceID(Fn, Dir);
   }
-  Asm->OutStreamer.EmitDwarfLocDirective(Src, Line, Col, Flags, 0,
-                                         Discriminator, Fn);
+  Asm->OutStreamer->EmitDwarfLocDirective(Src, Line, Col, Flags, 0,
+                                          Discriminator, Fn);
 }
 
 //===----------------------------------------------------------------------===//
@@ -1287,10 +1312,10 @@ void DwarfDebug::emitAbbreviations() {
   Holder.emitAbbrevs(Asm->getObjFileLowering().getDwarfAbbrevSection());
 }
 
-void DwarfDebug::emitAccel(DwarfAccelTable &Accel, const MCSection *Section,
+void DwarfDebug::emitAccel(DwarfAccelTable &Accel, MCSection *Section,
                            StringRef TableName) {
   Accel.FinalizeTable(Asm, TableName);
-  Asm->OutStreamer.SwitchSection(Section);
+  Asm->OutStreamer->SwitchSection(Section);
 
   // Emit the full data.
   Accel.emit(Asm, Section->getBeginSymbol(), this);
@@ -1343,9 +1368,8 @@ static dwarf::PubIndexEntryDescriptor computeIndexValue(DwarfUnit *CU,
 
   // We could have a specification DIE that has our most of our knowledge,
   // look for that now.
-  DIEValue *SpecVal = Die->findAttribute(dwarf::DW_AT_specification);
-  if (SpecVal) {
-    DIE &SpecDIE = cast<DIEEntry>(SpecVal)->getEntry();
+  if (DIEValue SpecVal = Die->findAttribute(dwarf::DW_AT_specification)) {
+    DIE &SpecDIE = SpecVal.getDIEEntry().getEntry();
     if (SpecDIE.findAttribute(dwarf::DW_AT_external))
       Linkage = dwarf::GIEL_EXTERNAL;
   } else if (Die->findAttribute(dwarf::DW_AT_external))
@@ -1381,16 +1405,16 @@ static dwarf::PubIndexEntryDescriptor computeIndexValue(DwarfUnit *CU,
 /// emitDebugPubNames - Emit visible names into a debug pubnames section.
 ///
 void DwarfDebug::emitDebugPubNames(bool GnuStyle) {
-  const MCSection *PSec =
-      GnuStyle ? Asm->getObjFileLowering().getDwarfGnuPubNamesSection()
-               : Asm->getObjFileLowering().getDwarfPubNamesSection();
+  MCSection *PSec = GnuStyle
+                        ? Asm->getObjFileLowering().getDwarfGnuPubNamesSection()
+                        : Asm->getObjFileLowering().getDwarfPubNamesSection();
 
   emitDebugPubSection(GnuStyle, PSec, "Names",
                       &DwarfCompileUnit::getGlobalNames);
 }
 
 void DwarfDebug::emitDebugPubSection(
-    bool GnuStyle, const MCSection *PSec, StringRef Name,
+    bool GnuStyle, MCSection *PSec, StringRef Name,
     const StringMap<const DIE *> &(DwarfCompileUnit::*Accessor)() const) {
   for (const auto &NU : CUMap) {
     DwarfCompileUnit *TheU = NU.second;
@@ -1404,23 +1428,23 @@ void DwarfDebug::emitDebugPubSection(
       TheU = Skeleton;
 
     // Start the dwarf pubnames section.
-    Asm->OutStreamer.SwitchSection(PSec);
+    Asm->OutStreamer->SwitchSection(PSec);
 
     // Emit the header.
-    Asm->OutStreamer.AddComment("Length of Public " + Name + " Info");
+    Asm->OutStreamer->AddComment("Length of Public " + Name + " Info");
     MCSymbol *BeginLabel = Asm->createTempSymbol("pub" + Name + "_begin");
     MCSymbol *EndLabel = Asm->createTempSymbol("pub" + Name + "_end");
     Asm->EmitLabelDifference(EndLabel, BeginLabel, 4);
 
-    Asm->OutStreamer.EmitLabel(BeginLabel);
+    Asm->OutStreamer->EmitLabel(BeginLabel);
 
-    Asm->OutStreamer.AddComment("DWARF Version");
+    Asm->OutStreamer->AddComment("DWARF Version");
     Asm->EmitInt16(dwarf::DW_PUBNAMES_VERSION);
 
-    Asm->OutStreamer.AddComment("Offset of Compilation Unit Info");
-    Asm->emitSectionOffset(TheU->getLabelBegin());
+    Asm->OutStreamer->AddComment("Offset of Compilation Unit Info");
+    Asm->emitDwarfSymbolReference(TheU->getLabelBegin());
 
-    Asm->OutStreamer.AddComment("Compilation Unit Length");
+    Asm->OutStreamer->AddComment("Compilation Unit Length");
     Asm->EmitInt32(TheU->getLength());
 
     // Emit the pubnames for this compilation unit.
@@ -1428,31 +1452,31 @@ void DwarfDebug::emitDebugPubSection(
       const char *Name = GI.getKeyData();
       const DIE *Entity = GI.second;
 
-      Asm->OutStreamer.AddComment("DIE offset");
+      Asm->OutStreamer->AddComment("DIE offset");
       Asm->EmitInt32(Entity->getOffset());
 
       if (GnuStyle) {
         dwarf::PubIndexEntryDescriptor Desc = computeIndexValue(TheU, Entity);
-        Asm->OutStreamer.AddComment(
+        Asm->OutStreamer->AddComment(
             Twine("Kind: ") + dwarf::GDBIndexEntryKindString(Desc.Kind) + ", " +
             dwarf::GDBIndexEntryLinkageString(Desc.Linkage));
         Asm->EmitInt8(Desc.toBits());
       }
 
-      Asm->OutStreamer.AddComment("External Name");
-      Asm->OutStreamer.EmitBytes(StringRef(Name, GI.getKeyLength() + 1));
+      Asm->OutStreamer->AddComment("External Name");
+      Asm->OutStreamer->EmitBytes(StringRef(Name, GI.getKeyLength() + 1));
     }
 
-    Asm->OutStreamer.AddComment("End Mark");
+    Asm->OutStreamer->AddComment("End Mark");
     Asm->EmitInt32(0);
-    Asm->OutStreamer.EmitLabel(EndLabel);
+    Asm->OutStreamer->EmitLabel(EndLabel);
   }
 }
 
 void DwarfDebug::emitDebugPubTypes(bool GnuStyle) {
-  const MCSection *PSec =
-      GnuStyle ? Asm->getObjFileLowering().getDwarfGnuPubTypesSection()
-               : Asm->getObjFileLowering().getDwarfPubTypesSection();
+  MCSection *PSec = GnuStyle
+                        ? Asm->getObjFileLowering().getDwarfGnuPubTypesSection()
+                        : Asm->getObjFileLowering().getDwarfPubTypesSection();
 
   emitDebugPubSection(GnuStyle, PSec, "Types",
                       &DwarfCompileUnit::getGlobalTypes);
@@ -1464,36 +1488,32 @@ void DwarfDebug::emitDebugStr() {
   Holder.emitStrings(Asm->getObjFileLowering().getDwarfStrSection());
 }
 
-
 void DwarfDebug::emitDebugLocEntry(ByteStreamer &Streamer,
-                                   const DebugLocEntry &Entry) {
-  auto Comment = Entry.getComments().begin();
-  auto End = Entry.getComments().end();
-  for (uint8_t Byte : Entry.getDWARFBytes())
+                                   const DebugLocStream::Entry &Entry) {
+  auto &&Comments = DebugLocs.getComments(Entry);
+  auto Comment = Comments.begin();
+  auto End = Comments.end();
+  for (uint8_t Byte : DebugLocs.getBytes(Entry))
     Streamer.EmitInt8(Byte, Comment != End ? *(Comment++) : "");
 }
 
-static void emitDebugLocValue(const AsmPrinter &AP,
-                              const DITypeIdentifierMap &TypeIdentifierMap,
+static void emitDebugLocValue(const AsmPrinter &AP, const DIBasicType *BT,
                               ByteStreamer &Streamer,
                               const DebugLocEntry::Value &Value,
                               unsigned PieceOffsetInBits) {
-  DIVariable DV = Value.getVariable();
   DebugLocDwarfExpression DwarfExpr(*AP.MF->getSubtarget().getRegisterInfo(),
                                     AP.getDwarfDebug()->getDwarfVersion(),
                                     Streamer);
   // Regular entry.
   if (Value.isInt()) {
-    MDType *T = DV->getType().resolve(TypeIdentifierMap);
-    auto *B = dyn_cast<MDBasicType>(T);
-    if (B && (B->getEncoding() == dwarf::DW_ATE_signed ||
-              B->getEncoding() == dwarf::DW_ATE_signed_char))
+    if (BT && (BT->getEncoding() == dwarf::DW_ATE_signed ||
+               BT->getEncoding() == dwarf::DW_ATE_signed_char))
       DwarfExpr.AddSignedConstant(Value.getInt());
     else
       DwarfExpr.AddUnsignedConstant(Value.getInt());
   } else if (Value.isLocation()) {
     MachineLocation Loc = Value.getLoc();
-    DIExpression Expr = Value.getExpression();
+    const DIExpression *Expr = Value.getExpression();
     if (!Expr || !Expr->getNumElements())
       // Regular entry.
       AP.EmitDwarfRegOp(Streamer, Loc);
@@ -1513,11 +1533,12 @@ static void emitDebugLocValue(const AsmPrinter &AP,
   // FIXME: ^
 }
 
-
 void DebugLocEntry::finalize(const AsmPrinter &AP,
-                             const DITypeIdentifierMap &TypeIdentifierMap) {
-  BufferByteStreamer Streamer(DWARFBytes, Comments);
-  const DebugLocEntry::Value Value = Values[0];
+                             DebugLocStream::ListBuilder &List,
+                             const DIBasicType *BT) {
+  DebugLocStream::EntryBuilder Entry(List, Begin, End);
+  BufferByteStreamer Streamer = Entry.getStreamer();
+  const DebugLocEntry::Value &Value = Values[0];
   if (Value.isBitPiece()) {
     // Emit all pieces that belong to the same variable and range.
     assert(std::all_of(Values.begin(), Values.end(), [](DebugLocEntry::Value P) {
@@ -1528,7 +1549,7 @@ void DebugLocEntry::finalize(const AsmPrinter &AP,
    
     unsigned Offset = 0;
     for (auto Piece : Values) {
-      DIExpression Expr = Piece.getExpression();
+      const DIExpression *Expr = Piece.getExpression();
       unsigned PieceOffset = Expr->getBitPieceOffset();
       unsigned PieceSize = Expr->getBitPieceSize();
       assert(Offset <= PieceOffset && "overlapping or duplicate pieces");
@@ -1542,70 +1563,66 @@ void DebugLocEntry::finalize(const AsmPrinter &AP,
       }
       Offset += PieceSize;
 
-      emitDebugLocValue(AP, TypeIdentifierMap, Streamer, Piece, PieceOffset);
+      emitDebugLocValue(AP, BT, Streamer, Piece, PieceOffset);
     }
   } else {
     assert(Values.size() == 1 && "only pieces may have >1 value");
-    emitDebugLocValue(AP, TypeIdentifierMap, Streamer, Value, 0);
+    emitDebugLocValue(AP, BT, Streamer, Value, 0);
   }
 }
 
+void DwarfDebug::emitDebugLocEntryLocation(const DebugLocStream::Entry &Entry) {
+  // Emit the size.
+  Asm->OutStreamer->AddComment("Loc expr size");
+  Asm->EmitInt16(DebugLocs.getBytes(Entry).size());
 
-void DwarfDebug::emitDebugLocEntryLocation(const DebugLocEntry &Entry) {
-  Asm->OutStreamer.AddComment("Loc expr size");
-  MCSymbol *begin = Asm->OutStreamer.getContext().CreateTempSymbol();
-  MCSymbol *end = Asm->OutStreamer.getContext().CreateTempSymbol();
-  Asm->EmitLabelDifference(end, begin, 2);
-  Asm->OutStreamer.EmitLabel(begin);
   // Emit the entry.
   APByteStreamer Streamer(*Asm);
   emitDebugLocEntry(Streamer, Entry);
-  // Close the range.
-  Asm->OutStreamer.EmitLabel(end);
 }
 
 // Emit locations into the debug loc section.
 void DwarfDebug::emitDebugLoc() {
   // Start the dwarf loc section.
-  Asm->OutStreamer.SwitchSection(
+  Asm->OutStreamer->SwitchSection(
       Asm->getObjFileLowering().getDwarfLocSection());
   unsigned char Size = Asm->getDataLayout().getPointerSize();
-  for (const auto &DebugLoc : DotDebugLocEntries) {
-    Asm->OutStreamer.EmitLabel(DebugLoc.Label);
-    const DwarfCompileUnit *CU = DebugLoc.CU;
-    for (const auto &Entry : DebugLoc.List) {
+  for (const auto &List : DebugLocs.getLists()) {
+    Asm->OutStreamer->EmitLabel(List.Label);
+    const DwarfCompileUnit *CU = List.CU;
+    for (const auto &Entry : DebugLocs.getEntries(List)) {
       // Set up the range. This range is relative to the entry point of the
       // compile unit. This is a hard coded 0 for low_pc when we're emitting
       // ranges, or the DW_AT_low_pc on the compile unit otherwise.
       if (auto *Base = CU->getBaseAddress()) {
-        Asm->EmitLabelDifference(Entry.getBeginSym(), Base, Size);
-        Asm->EmitLabelDifference(Entry.getEndSym(), Base, Size);
+        Asm->EmitLabelDifference(Entry.BeginSym, Base, Size);
+        Asm->EmitLabelDifference(Entry.EndSym, Base, Size);
       } else {
-        Asm->OutStreamer.EmitSymbolValue(Entry.getBeginSym(), Size);
-        Asm->OutStreamer.EmitSymbolValue(Entry.getEndSym(), Size);
+        Asm->OutStreamer->EmitSymbolValue(Entry.BeginSym, Size);
+        Asm->OutStreamer->EmitSymbolValue(Entry.EndSym, Size);
       }
 
       emitDebugLocEntryLocation(Entry);
     }
-    Asm->OutStreamer.EmitIntValue(0, Size);
-    Asm->OutStreamer.EmitIntValue(0, Size);
+    Asm->OutStreamer->EmitIntValue(0, Size);
+    Asm->OutStreamer->EmitIntValue(0, Size);
   }
 }
 
 void DwarfDebug::emitDebugLocDWO() {
-  Asm->OutStreamer.SwitchSection(
+  Asm->OutStreamer->SwitchSection(
       Asm->getObjFileLowering().getDwarfLocDWOSection());
-  for (const auto &DebugLoc : DotDebugLocEntries) {
-    Asm->OutStreamer.EmitLabel(DebugLoc.Label);
-    for (const auto &Entry : DebugLoc.List) {
+  for (const auto &List : DebugLocs.getLists()) {
+    Asm->OutStreamer->EmitLabel(List.Label);
+    for (const auto &Entry : DebugLocs.getEntries(List)) {
       // Just always use start_length for now - at least that's one address
       // rather than two. We could get fancier and try to, say, reuse an
       // address we know we've emitted elsewhere (the start of the function?
       // The start of the CU or CU subrange that encloses this range?)
       Asm->EmitInt8(dwarf::DW_LLE_start_length_entry);
-      unsigned idx = AddrPool.getIndex(Entry.getBeginSym());
+      unsigned idx = AddrPool.getIndex(Entry.BeginSym);
       Asm->EmitULEB128(idx);
-      Asm->EmitLabelDifference(Entry.getEndSym(), Entry.getBeginSym(), 4);
+      Asm->EmitLabelDifference(Entry.EndSym, Entry.BeginSym, 4);
 
       emitDebugLocEntryLocation(Entry);
     }
@@ -1621,13 +1638,13 @@ struct ArangeSpan {
 // address we can tie back to a CU.
 void DwarfDebug::emitDebugARanges() {
   // Provides a unique id per text section.
-  MapVector<const MCSection *, SmallVector<SymbolCU, 8>> SectionMap;
+  MapVector<MCSection *, SmallVector<SymbolCU, 8>> SectionMap;
 
   // Filter labels by section.
   for (const SymbolCU &SCU : ArangeLabels) {
     if (SCU.Sym->isInSection()) {
       // Make a note of this symbol and it's section.
-      const MCSection *Section = &SCU.Sym->getSection();
+      MCSection *Section = &SCU.Sym->getSection();
       if (!Section->getKind().isMetadata())
         SectionMap[Section].push_back(SCU);
     } else {
@@ -1640,11 +1657,11 @@ void DwarfDebug::emitDebugARanges() {
 
   // Add terminating symbols for each section.
   for (const auto &I : SectionMap) {
-    const MCSection *Section = I.first;
+    MCSection *Section = I.first;
     MCSymbol *Sym = nullptr;
 
     if (Section)
-      Sym = Asm->OutStreamer.endSection(Section);
+      Sym = Asm->OutStreamer->endSection(Section);
 
     // Insert a final terminator.
     SectionMap[Section].push_back(SymbolCU(nullptr, Sym));
@@ -1674,8 +1691,8 @@ void DwarfDebug::emitDebugARanges() {
     // Sort the symbols by offset within the section.
     std::sort(List.begin(), List.end(),
               [&](const SymbolCU &A, const SymbolCU &B) {
-      unsigned IA = A.Sym ? Asm->OutStreamer.GetSymbolOrder(A.Sym) : 0;
-      unsigned IB = B.Sym ? Asm->OutStreamer.GetSymbolOrder(B.Sym) : 0;
+      unsigned IA = A.Sym ? Asm->OutStreamer->GetSymbolOrder(A.Sym) : 0;
+      unsigned IB = B.Sym ? Asm->OutStreamer->GetSymbolOrder(B.Sym) : 0;
 
       // Symbols with no order assigned should be placed at the end.
       // (e.g. section end labels)
@@ -1704,7 +1721,7 @@ void DwarfDebug::emitDebugARanges() {
   }
 
   // Start the dwarf aranges section.
-  Asm->OutStreamer.SwitchSection(
+  Asm->OutStreamer->SwitchSection(
       Asm->getObjFileLowering().getDwarfARangesSection());
 
   unsigned PtrSize = Asm->getDataLayout().getPointerSize();
@@ -1746,18 +1763,18 @@ void DwarfDebug::emitDebugARanges() {
     ContentSize += (List.size() + 1) * TupleSize;
 
     // For each compile unit, write the list of spans it covers.
-    Asm->OutStreamer.AddComment("Length of ARange Set");
+    Asm->OutStreamer->AddComment("Length of ARange Set");
     Asm->EmitInt32(ContentSize);
-    Asm->OutStreamer.AddComment("DWARF Arange version number");
+    Asm->OutStreamer->AddComment("DWARF Arange version number");
     Asm->EmitInt16(dwarf::DW_ARANGES_VERSION);
-    Asm->OutStreamer.AddComment("Offset Into Debug Info Section");
-    Asm->emitSectionOffset(CU->getLabelBegin());
-    Asm->OutStreamer.AddComment("Address Size (in bytes)");
+    Asm->OutStreamer->AddComment("Offset Into Debug Info Section");
+    Asm->emitDwarfSymbolReference(CU->getLabelBegin());
+    Asm->OutStreamer->AddComment("Address Size (in bytes)");
     Asm->EmitInt8(PtrSize);
-    Asm->OutStreamer.AddComment("Segment Size (in bytes)");
+    Asm->OutStreamer->AddComment("Segment Size (in bytes)");
     Asm->EmitInt8(0);
 
-    Asm->OutStreamer.EmitFill(Padding, 0xff);
+    Asm->OutStreamer->EmitFill(Padding, 0xff);
 
     for (const ArangeSpan &Span : List) {
       Asm->EmitLabelReference(Span.Start, PtrSize);
@@ -1772,20 +1789,20 @@ void DwarfDebug::emitDebugARanges() {
         if (Size == 0)
           Size = 1;
 
-        Asm->OutStreamer.EmitIntValue(Size, PtrSize);
+        Asm->OutStreamer->EmitIntValue(Size, PtrSize);
       }
     }
 
-    Asm->OutStreamer.AddComment("ARange terminator");
-    Asm->OutStreamer.EmitIntValue(0, PtrSize);
-    Asm->OutStreamer.EmitIntValue(0, PtrSize);
+    Asm->OutStreamer->AddComment("ARange terminator");
+    Asm->OutStreamer->EmitIntValue(0, PtrSize);
+    Asm->OutStreamer->EmitIntValue(0, PtrSize);
   }
 }
 
 // Emit visible names into a debug ranges section.
 void DwarfDebug::emitDebugRanges() {
   // Start the dwarf ranges section.
-  Asm->OutStreamer.SwitchSection(
+  Asm->OutStreamer->SwitchSection(
       Asm->getObjFileLowering().getDwarfRangesSection());
 
   // Size for our labels.
@@ -1801,7 +1818,7 @@ void DwarfDebug::emitDebugRanges() {
     // Iterate over the misc ranges for the compile units in the module.
     for (const RangeSpanList &List : TheCU->getRangeLists()) {
       // Emit our symbol so we can find the beginning of the range.
-      Asm->OutStreamer.EmitLabel(List.getSym());
+      Asm->OutStreamer->EmitLabel(List.getSym());
 
       for (const RangeSpan &Range : List.getRanges()) {
         const MCSymbol *Begin = Range.getStart();
@@ -1812,14 +1829,14 @@ void DwarfDebug::emitDebugRanges() {
           Asm->EmitLabelDifference(Begin, Base, Size);
           Asm->EmitLabelDifference(End, Base, Size);
         } else {
-          Asm->OutStreamer.EmitSymbolValue(Begin, Size);
-          Asm->OutStreamer.EmitSymbolValue(End, Size);
+          Asm->OutStreamer->EmitSymbolValue(Begin, Size);
+          Asm->OutStreamer->EmitSymbolValue(End, Size);
         }
       }
 
       // And terminate the list with two 0 values.
-      Asm->OutStreamer.EmitIntValue(0, Size);
-      Asm->OutStreamer.EmitIntValue(0, Size);
+      Asm->OutStreamer->EmitIntValue(0, Size);
+      Asm->OutStreamer->EmitIntValue(0, Size);
     }
   }
 }
@@ -1873,9 +1890,9 @@ void DwarfDebug::emitDebugAbbrevDWO() {
 
 void DwarfDebug::emitDebugLineDWO() {
   assert(useSplitDwarf() && "No split dwarf?");
-  Asm->OutStreamer.SwitchSection(
+  Asm->OutStreamer->SwitchSection(
       Asm->getObjFileLowering().getDwarfLineDWOSection());
-  SplitTypeUnitFileTable.Emit(Asm->OutStreamer);
+  SplitTypeUnitFileTable.Emit(*Asm->OutStreamer);
 }
 
 // Emit the .debug_str.dwo section for separated dwarf. This contains the
@@ -1883,8 +1900,7 @@ void DwarfDebug::emitDebugLineDWO() {
 // sections.
 void DwarfDebug::emitDebugStrDWO() {
   assert(useSplitDwarf() && "No split dwarf?");
-  const MCSection *OffSec =
-      Asm->getObjFileLowering().getDwarfStrOffDWOSection();
+  MCSection *OffSec = Asm->getObjFileLowering().getDwarfStrOffDWOSection();
   InfoHolder.emitStrings(Asm->getObjFileLowering().getDwarfStrDWOSection(),
                          OffSec);
 }
@@ -1897,7 +1913,7 @@ MCDwarfDwoLineTable *DwarfDebug::getDwoLineTable(const DwarfCompileUnit &CU) {
   return &SplitTypeUnitFileTable;
 }
 
-static uint64_t makeTypeSignature(StringRef Identifier) {
+uint64_t DwarfDebug::makeTypeSignature(StringRef Identifier) {
   MD5 Hash;
   Hash.update(Identifier);
   // ... take the least significant 8 bytes and return those. Our MD5
@@ -1910,7 +1926,7 @@ static uint64_t makeTypeSignature(StringRef Identifier) {
 
 void DwarfDebug::addDwarfTypeUnitType(DwarfCompileUnit &CU,
                                       StringRef Identifier, DIE &RefDie,
-                                      DICompositeType CTy) {
+                                      const DICompositeType *CTy) {
   // Fast path if we're building some type units and one has already used the
   // address pool we know we're going to throw away all this work anyway, so
   // don't bother building dependent types.
@@ -1969,7 +1985,7 @@ void DwarfDebug::addDwarfTypeUnitType(DwarfCompileUnit &CU,
       // This is inefficient because all the dependent types will be rebuilt
       // from scratch, including building them in type units, discovering that
       // they depend on addresses, throwing them out and rebuilding them.
-      CU.constructTypeDIE(RefDie, CTy);
+      CU.constructTypeDIE(RefDie, cast<DICompositeType>(CTy));
       return;
     }
 
@@ -1988,27 +2004,23 @@ void DwarfDebug::addDwarfTypeUnitType(DwarfCompileUnit &CU,
 void DwarfDebug::addAccelName(StringRef Name, const DIE &Die) {
   if (!useDwarfAccelTables())
     return;
-  AccelNames.AddName(Name, InfoHolder.getStringPool().getSymbol(*Asm, Name),
-                     &Die);
+  AccelNames.AddName(InfoHolder.getStringPool().getEntry(*Asm, Name), &Die);
 }
 
 void DwarfDebug::addAccelObjC(StringRef Name, const DIE &Die) {
   if (!useDwarfAccelTables())
     return;
-  AccelObjC.AddName(Name, InfoHolder.getStringPool().getSymbol(*Asm, Name),
-                    &Die);
+  AccelObjC.AddName(InfoHolder.getStringPool().getEntry(*Asm, Name), &Die);
 }
 
 void DwarfDebug::addAccelNamespace(StringRef Name, const DIE &Die) {
   if (!useDwarfAccelTables())
     return;
-  AccelNamespace.AddName(Name, InfoHolder.getStringPool().getSymbol(*Asm, Name),
-                         &Die);
+  AccelNamespace.AddName(InfoHolder.getStringPool().getEntry(*Asm, Name), &Die);
 }
 
 void DwarfDebug::addAccelType(StringRef Name, const DIE &Die, char Flags) {
   if (!useDwarfAccelTables())
     return;
-  AccelTypes.AddName(Name, InfoHolder.getStringPool().getSymbol(*Asm, Name),
-                     &Die);
+  AccelTypes.AddName(InfoHolder.getStringPool().getEntry(*Asm, Name), &Die);
 }
diff --git a/lib/CodeGen/AsmPrinter/DwarfDebug.h b/lib/CodeGen/AsmPrinter/DwarfDebug.h
index e067f41904c..57bb8d0229d 100644
--- a/lib/CodeGen/AsmPrinter/DwarfDebug.h
+++ b/lib/CodeGen/AsmPrinter/DwarfDebug.h
@@ -16,8 +16,7 @@
 
 #include "AsmPrinterHandler.h"
 #include "DbgValueHistoryCalculator.h"
-#include "DebugLocEntry.h"
-#include "DebugLocList.h"
+#include "DebugLocStream.h"
 #include "DwarfAccelTable.h"
 #include "DwarfFile.h"
 #include "llvm/ADT/DenseMap.h"
@@ -42,6 +41,7 @@ class AsmPrinter;
 class ByteStreamer;
 class ConstantInt;
 class ConstantFP;
+class DebugLocEntry;
 class DwarfCompileUnit;
 class DwarfDebug;
 class DwarfTypeUnit;
@@ -49,7 +49,7 @@ class DwarfUnit;
 class MachineModuleInfo;
 
 //===----------------------------------------------------------------------===//
-/// \brief This class is used to record source line correspondence.
+/// This class is used to record source line correspondence.
 class SrcLineInfo {
   unsigned Line;     // Source line number.
   unsigned Column;   // Source column.
@@ -67,71 +67,91 @@ class SrcLineInfo {
 };
 
 //===----------------------------------------------------------------------===//
-/// \brief This class is used to track local variable information.
+/// This class is used to track local variable information.
 ///
-/// - Variables whose location changes over time have a DotDebugLocOffset and
-///   the other fields are not used.
+/// Variables can be created from allocas, in which case they're generated from
+/// the MMI table.  Such variables can have multiple expressions and frame
+/// indices.  The \a Expr and \a FrameIndices array must match.
 ///
-/// - Variables that are described by multiple MMI table entries have multiple
-///   expressions and frame indices.
+/// Variables can be created from \c DBG_VALUE instructions.  Those whose
+/// location changes over time use \a DebugLocListIndex, while those with a
+/// single instruction use \a MInsn and (optionally) a single entry of \a Expr.
+///
+/// Variables that have been optimized out use none of these fields.
 class DbgVariable {
-  DIVariable Var;                    /// Variable Descriptor.
-  DILocation IA;                     /// Inlined at location.
-  SmallVector<DIExpression, 1> Expr; /// Complex address location expression.
-  DIE *TheDIE;                /// Variable DIE.
-  unsigned DotDebugLocOffset; /// Offset in DotDebugLocEntries.
-  const MachineInstr *MInsn;  /// DBG_VALUE instruction of the variable.
-  SmallVector<int, 1> FrameIndex; /// Frame index of the variable.
+  const DILocalVariable *Var;                /// Variable Descriptor.
+  const DILocation *IA;                      /// Inlined at location.
+  SmallVector<const DIExpression *, 1> Expr; /// Complex address.
+  DIE *TheDIE = nullptr;                     /// Variable DIE.
+  unsigned DebugLocListIndex = ~0u;          /// Offset in DebugLocs.
+  const MachineInstr *MInsn = nullptr;       /// DBG_VALUE instruction.
+  SmallVector<int, 1> FrameIndex;            /// Frame index.
   DwarfDebug *DD;
 
 public:
-  /// Construct a DbgVariable from a DIVariable.
-  DbgVariable(DIVariable V, DILocation IA, DIExpression E, DwarfDebug *DD,
-              int FI = ~0)
-      : Var(V), IA(IA), Expr(1, E), TheDIE(nullptr), DotDebugLocOffset(~0U),
-        MInsn(nullptr), DD(DD) {
+  /// Construct a DbgVariable.
+  ///
+  /// Creates a variable without any DW_AT_location.  Call \a initializeMMI()
+  /// for MMI entries, or \a initializeDbgValue() for DBG_VALUE instructions.
+  DbgVariable(const DILocalVariable *V, const DILocation *IA, DwarfDebug *DD)
+      : Var(V), IA(IA), DD(DD) {}
+
+  /// Initialize from the MMI table.
+  void initializeMMI(const DIExpression *E, int FI) {
+    assert(Expr.empty() && "Already initialized?");
+    assert(FrameIndex.empty() && "Already initialized?");
+    assert(!MInsn && "Already initialized?");
+
+    assert((!E || E->isValid()) && "Expected valid expression");
+    assert(~FI && "Expected valid index");
+
+    Expr.push_back(E);
     FrameIndex.push_back(FI);
-    assert(!E || E->isValid());
   }
 
-  /// Construct a DbgVariable from a DEBUG_VALUE.
-  /// AbstractVar may be NULL.
-  DbgVariable(const MachineInstr *DbgValue, DwarfDebug *DD)
-      : Var(DbgValue->getDebugVariable()),
-        IA(DbgValue->getDebugLoc()->getInlinedAt()),
-        Expr(1, DbgValue->getDebugExpression()), TheDIE(nullptr),
-        DotDebugLocOffset(~0U), MInsn(DbgValue), DD(DD) {
-    FrameIndex.push_back(~0);
+  /// Initialize from a DBG_VALUE instruction.
+  void initializeDbgValue(const MachineInstr *DbgValue) {
+    assert(Expr.empty() && "Already initialized?");
+    assert(FrameIndex.empty() && "Already initialized?");
+    assert(!MInsn && "Already initialized?");
+
+    assert(Var == DbgValue->getDebugVariable() && "Wrong variable");
+    assert(IA == DbgValue->getDebugLoc()->getInlinedAt() && "Wrong inlined-at");
+
+    MInsn = DbgValue;
+    if (auto *E = DbgValue->getDebugExpression())
+      if (E->getNumElements())
+        Expr.push_back(E);
   }
 
   // Accessors.
-  DIVariable getVariable() const { return Var; }
-  DILocation getInlinedAt() const { return IA; }
-  const ArrayRef<DIExpression> getExpression() const { return Expr; }
+  const DILocalVariable *getVariable() const { return Var; }
+  const DILocation *getInlinedAt() const { return IA; }
+  const ArrayRef<const DIExpression *> getExpression() const { return Expr; }
   void setDIE(DIE &D) { TheDIE = &D; }
   DIE *getDIE() const { return TheDIE; }
-  void setDotDebugLocOffset(unsigned O) { DotDebugLocOffset = O; }
-  unsigned getDotDebugLocOffset() const { return DotDebugLocOffset; }
+  void setDebugLocListIndex(unsigned O) { DebugLocListIndex = O; }
+  unsigned getDebugLocListIndex() const { return DebugLocListIndex; }
   StringRef getName() const { return Var->getName(); }
   const MachineInstr *getMInsn() const { return MInsn; }
   const ArrayRef<int> getFrameIndex() const { return FrameIndex; }
 
   void addMMIEntry(const DbgVariable &V) {
-    assert(  DotDebugLocOffset == ~0U &&   !MInsn && "not an MMI entry");
-    assert(V.DotDebugLocOffset == ~0U && !V.MInsn && "not an MMI entry");
-    assert(V.Var == Var && "conflicting DIVariable");
+    assert(DebugLocListIndex == ~0U && !MInsn && "not an MMI entry");
+    assert(V.DebugLocListIndex == ~0U && !V.MInsn && "not an MMI entry");
+    assert(V.Var == Var && "conflicting variable");
     assert(V.IA == IA && "conflicting inlined-at location");
 
-    if (V.getFrameIndex().back() != ~0) {
-      auto E = V.getExpression();
-      auto FI = V.getFrameIndex();
-      Expr.append(E.begin(), E.end());
-      FrameIndex.append(FI.begin(), FI.end());
-    }
-    assert(Expr.size() > 1
-               ? std::all_of(Expr.begin(), Expr.end(),
-                             [](DIExpression &E) { return E->isBitPiece(); })
-               : (true && "conflicting locations for variable"));
+    assert(!FrameIndex.empty() && "Expected an MMI entry");
+    assert(!V.FrameIndex.empty() && "Expected an MMI entry");
+    assert(Expr.size() == FrameIndex.size() && "Mismatched expressions");
+    assert(V.Expr.size() == V.FrameIndex.size() && "Mismatched expressions");
+
+    Expr.append(V.Expr.begin(), V.Expr.end());
+    FrameIndex.append(V.FrameIndex.begin(), V.FrameIndex.end());
+    assert(std::all_of(Expr.begin(), Expr.end(), [](const DIExpression *E) {
+             return E && E->isBitPiece();
+           }) && "conflicting locations for variable");
   }
 
   // Translate tag to proper Dwarf tag.
@@ -141,7 +161,7 @@ class DbgVariable {
 
     return dwarf::DW_TAG_variable;
   }
-  /// \brief Return true if DbgVariable is artificial.
+  /// Return true if DbgVariable is artificial.
   bool isArtificial() const {
     if (Var->isArtificial())
       return true;
@@ -158,161 +178,190 @@ class DbgVariable {
     return false;
   }
 
-  bool variableHasComplexAddress() const {
-    assert(Var && "Invalid complex DbgVariable!");
-    assert(Expr.size() == 1 &&
-           "variableHasComplexAddress() invoked on multi-FI variable");
-    return Expr.back()->getNumElements() > 0;
+  bool hasComplexAddress() const {
+    assert(MInsn && "Expected DBG_VALUE, not MMI variable");
+    assert(FrameIndex.empty() && "Expected DBG_VALUE, not MMI variable");
+    assert(
+        (Expr.empty() || (Expr.size() == 1 && Expr.back()->getNumElements())) &&
+        "Invalid Expr for DBG_VALUE");
+    return !Expr.empty();
   }
   bool isBlockByrefVariable() const;
-  DIType getType() const;
+  const DIType *getType() const;
 
 private:
-  /// resolve - Look in the DwarfDebug map for the MDNode that
+  /// Look in the DwarfDebug map for the MDNode that
   /// corresponds to the reference.
-  template <typename T> T *resolve(TypedDebugNodeRef<T> Ref) const;
+  template <typename T> T *resolve(TypedDINodeRef<T> Ref) const;
 };
 
 
-/// \brief Helper used to pair up a symbol and its DWARF compile unit.
+/// Helper used to pair up a symbol and its DWARF compile unit.
 struct SymbolCU {
   SymbolCU(DwarfCompileUnit *CU, const MCSymbol *Sym) : Sym(Sym), CU(CU) {}
   const MCSymbol *Sym;
   DwarfCompileUnit *CU;
 };
 
-/// \brief Collects and handles dwarf debug information.
+/// Identify a debugger for "tuning" the debug info.
+///
+/// The "debugger tuning" concept allows us to present a more intuitive
+/// interface that unpacks into different sets of defaults for the various
+/// individual feature-flag settings, that suit the preferences of the
+/// various debuggers.  However, it's worth remembering that debuggers are
+/// not the only consumers of debug info, and some variations in DWARF might
+/// better be treated as target/platform issues. Fundamentally,
+/// o if the feature is useful (or not) to a particular debugger, regardless
+///   of the target, that's a tuning decision;
+/// o if the feature is useful (or not) on a particular platform, regardless
+///   of the debugger, that's a target decision.
+/// It's not impossible to see both factors in some specific case.
+///
+/// The "tuning" should be used to set defaults for individual feature flags
+/// in DwarfDebug; if a given feature has a more specific command-line option,
+/// that option should take precedence over the tuning.
+enum class DebuggerKind {
+  Default,  // No specific tuning requested.
+  GDB,      // Tune debug info for gdb.
+  LLDB,     // Tune debug info for lldb.
+  SCE       // Tune debug info for SCE targets (e.g. PS4).
+};
+
+/// Collects and handles dwarf debug information.
 class DwarfDebug : public AsmPrinterHandler {
-  // Target of Dwarf emission.
+  /// Target of Dwarf emission.
   AsmPrinter *Asm;
 
-  // Collected machine module information.
+  /// Collected machine module information.
   MachineModuleInfo *MMI;
 
-  // All DIEValues are allocated through this allocator.
+  /// All DIEValues are allocated through this allocator.
   BumpPtrAllocator DIEValueAllocator;
 
-  // Maps MDNode with its corresponding DwarfCompileUnit.
+  /// Maps MDNode with its corresponding DwarfCompileUnit.
   MapVector<const MDNode *, DwarfCompileUnit *> CUMap;
 
-  // Maps subprogram MDNode with its corresponding DwarfCompileUnit.
+  /// Maps subprogram MDNode with its corresponding DwarfCompileUnit.
   MapVector<const MDNode *, DwarfCompileUnit *> SPMap;
 
-  // Maps a CU DIE with its corresponding DwarfCompileUnit.
+  /// Maps a CU DIE with its corresponding DwarfCompileUnit.
   DenseMap<const DIE *, DwarfCompileUnit *> CUDieMap;
 
-  // List of all labels used in aranges generation.
+  /// List of all labels used in aranges generation.
   std::vector<SymbolCU> ArangeLabels;
 
-  // Size of each symbol emitted (for those symbols that have a specific size).
+  /// Size of each symbol emitted (for those symbols that have a specific size).
   DenseMap<const MCSymbol *, uint64_t> SymSize;
 
   LexicalScopes LScopes;
 
-  // Collection of abstract variables.
+  /// Collection of abstract variables.
   DenseMap<const MDNode *, std::unique_ptr<DbgVariable>> AbstractVariables;
   SmallVector<std::unique_ptr<DbgVariable>, 64> ConcreteVariables;
 
-  // Collection of DebugLocEntry. Stored in a linked list so that DIELocLists
-  // can refer to them in spite of insertions into this list.
-  SmallVector<DebugLocList, 4> DotDebugLocEntries;
+  /// Collection of DebugLocEntry. Stored in a linked list so that DIELocLists
+  /// can refer to them in spite of insertions into this list.
+  DebugLocStream DebugLocs;
 
-  // This is a collection of subprogram MDNodes that are processed to
-  // create DIEs.
+  /// This is a collection of subprogram MDNodes that are processed to
+  /// create DIEs.
   SmallPtrSet<const MDNode *, 16> ProcessedSPNodes;
 
-  // Maps instruction with label emitted before instruction.
+  /// Maps instruction with label emitted before instruction.
   DenseMap<const MachineInstr *, MCSymbol *> LabelsBeforeInsn;
 
-  // Maps instruction with label emitted after instruction.
+  /// Maps instruction with label emitted after instruction.
   DenseMap<const MachineInstr *, MCSymbol *> LabelsAfterInsn;
 
-  // History of DBG_VALUE and clobber instructions for each user variable.
-  // Variables are listed in order of appearance.
+  /// History of DBG_VALUE and clobber instructions for each user
+  /// variable.  Variables are listed in order of appearance.
   DbgValueHistoryMap DbgValues;
 
-  // Previous instruction's location information. This is used to determine
-  // label location to indicate scope boundries in dwarf debug info.
+  /// Previous instruction's location information. This is used to
+  /// determine label location to indicate scope boundries in dwarf
+  /// debug info.
   DebugLoc PrevInstLoc;
   MCSymbol *PrevLabel;
 
-  // This location indicates end of function prologue and beginning of function
-  // body.
+  /// This location indicates end of function prologue and beginning of
+  /// function body.
   DebugLoc PrologEndLoc;
 
-  // If nonnull, stores the current machine function we're processing.
+  /// If nonnull, stores the current machine function we're processing.
   const MachineFunction *CurFn;
 
-  // If nonnull, stores the current machine instruction we're processing.
+  /// If nonnull, stores the current machine instruction we're processing.
   const MachineInstr *CurMI;
 
-  // If nonnull, stores the CU in which the previous subprogram was contained.
+  /// If nonnull, stores the CU in which the previous subprogram was contained.
   const DwarfCompileUnit *PrevCU;
 
-  // As an optimization, there is no need to emit an entry in the directory
-  // table for the same directory as DW_AT_comp_dir.
+  /// As an optimization, there is no need to emit an entry in the directory
+  /// table for the same directory as DW_AT_comp_dir.
   StringRef CompilationDir;
 
-  // Holder for the file specific debug information.
+  /// Holder for the file specific debug information.
   DwarfFile InfoHolder;
 
-  // Holders for the various debug information flags that we might need to
-  // have exposed. See accessor functions below for description.
+  /// Holders for the various debug information flags that we might need to
+  /// have exposed. See accessor functions below for description.
 
-  // Holder for imported entities.
+  /// Holder for imported entities.
   typedef SmallVector<std::pair<const MDNode *, const MDNode *>, 32>
   ImportedEntityMap;
   ImportedEntityMap ScopesWithImportedEntities;
 
-  // Map from MDNodes for user-defined types to the type units that describe
-  // them.
+  /// Map from MDNodes for user-defined types to the type units that
+  /// describe them.
   DenseMap<const MDNode *, const DwarfTypeUnit *> DwarfTypeUnits;
 
-  SmallVector<std::pair<std::unique_ptr<DwarfTypeUnit>, DICompositeType>, 1>
+  SmallVector<
+      std::pair<std::unique_ptr<DwarfTypeUnit>, const DICompositeType *>, 1>
       TypeUnitsUnderConstruction;
 
-  // Whether to emit the pubnames/pubtypes sections.
+  /// Whether to emit the pubnames/pubtypes sections.
   bool HasDwarfPubSections;
 
-  // Whether or not to use AT_ranges for compilation units.
+  /// Whether or not to use AT_ranges for compilation units.
   bool HasCURanges;
 
-  // Whether we emitted a function into a section other than the default
-  // text.
+  /// Whether we emitted a function into a section other than the
+  /// default text.
   bool UsedNonDefaultText;
 
-  // Whether to use the GNU TLS opcode (instead of the standard opcode).
+  /// Whether to use the GNU TLS opcode (instead of the standard opcode).
   bool UseGNUTLSOpcode;
 
-  // Version of dwarf we're emitting.
+  /// Version of dwarf we're emitting.
   unsigned DwarfVersion;
 
-  // Maps from a type identifier to the actual MDNode.
+  /// Maps from a type identifier to the actual MDNode.
   DITypeIdentifierMap TypeIdentifierMap;
 
-  // DWARF5 Experimental Options
+  /// DWARF5 Experimental Options
+  /// @{
   bool HasDwarfAccelTables;
   bool HasSplitDwarf;
 
-  // Separated Dwarf Variables
-  // In general these will all be for bits that are left in the
-  // original object file, rather than things that are meant
-  // to be in the .dwo sections.
+  /// Separated Dwarf Variables
+  /// In general these will all be for bits that are left in the
+  /// original object file, rather than things that are meant
+  /// to be in the .dwo sections.
 
-  // Holder for the skeleton information.
+  /// Holder for the skeleton information.
   DwarfFile SkeletonHolder;
 
-  /// Store file names for type units under fission in a line table header that
-  /// will be emitted into debug_line.dwo.
-  // FIXME: replace this with a map from comp_dir to table so that we can emit
-  // multiple tables during LTO each of which uses directory 0, referencing the
-  // comp_dir of all the type units that use it.
+  /// Store file names for type units under fission in a line table
+  /// header that will be emitted into debug_line.dwo.
+  // FIXME: replace this with a map from comp_dir to table so that we
+  // can emit multiple tables during LTO each of which uses directory
+  // 0, referencing the comp_dir of all the type units that use it.
   MCDwarfDwoLineTable SplitTypeUnitFileTable;
-
-  // True iff there are multiple CUs in this module.
+  /// @}
+  
+  /// True iff there are multiple CUs in this module.
   bool SingleCU;
   bool IsDarwin;
-  bool IsPS4;
 
   AddressPool AddrPool;
 
@@ -321,7 +370,10 @@ class DwarfDebug : public AsmPrinterHandler {
   DwarfAccelTable AccelNamespace;
   DwarfAccelTable AccelTypes;
 
-  DenseMap<const Function *, DISubprogram> FunctionDIs;
+  DenseMap<const Function *, DISubprogram *> FunctionDIs;
+
+  // Identify a debugger for "tuning" the debug info.
+  DebuggerKind DebuggerTuning;
 
   MCDwarfDwoLineTable *getDwoLineTable(const DwarfCompileUnit &);
 
@@ -331,160 +383,162 @@ class DwarfDebug : public AsmPrinterHandler {
 
   typedef DbgValueHistoryMap::InlinedVariable InlinedVariable;
 
-  /// \brief Find abstract variable associated with Var.
+  /// Find abstract variable associated with Var.
   DbgVariable *getExistingAbstractVariable(InlinedVariable IV,
-                                           DIVariable &Cleansed);
+                                           const DILocalVariable *&Cleansed);
   DbgVariable *getExistingAbstractVariable(InlinedVariable IV);
-  void createAbstractVariable(const DIVariable &DV, LexicalScope *Scope);
+  void createAbstractVariable(const DILocalVariable *DV, LexicalScope *Scope);
   void ensureAbstractVariableIsCreated(InlinedVariable Var,
                                        const MDNode *Scope);
   void ensureAbstractVariableIsCreatedIfScoped(InlinedVariable Var,
                                                const MDNode *Scope);
 
-  /// \brief Construct a DIE for this abstract scope.
+  DbgVariable *createConcreteVariable(LexicalScope &Scope, InlinedVariable IV);
+
+  /// Construct a DIE for this abstract scope.
   void constructAbstractSubprogramScopeDIE(LexicalScope *Scope);
 
-  /// \brief Compute the size and offset of a DIE given an incoming Offset.
+  /// Compute the size and offset of a DIE given an incoming Offset.
   unsigned computeSizeAndOffset(DIE *Die, unsigned Offset);
 
-  /// \brief Compute the size and offset of all the DIEs.
+  /// Compute the size and offset of all the DIEs.
   void computeSizeAndOffsets();
 
-  /// \brief Collect info for variables that were optimized out.
+  /// Collect info for variables that were optimized out.
   void collectDeadVariables();
 
   void finishVariableDefinitions();
 
   void finishSubprogramDefinitions();
 
-  /// \brief Finish off debug information after all functions have been
+  /// Finish off debug information after all functions have been
   /// processed.
   void finalizeModuleInfo();
 
-  /// \brief Emit the debug info section.
+  /// Emit the debug info section.
   void emitDebugInfo();
 
-  /// \brief Emit the abbreviation section.
+  /// Emit the abbreviation section.
   void emitAbbreviations();
 
-  /// \brief Emit a specified accelerator table.
-  void emitAccel(DwarfAccelTable &Accel, const MCSection *Section,
+  /// Emit a specified accelerator table.
+  void emitAccel(DwarfAccelTable &Accel, MCSection *Section,
                  StringRef TableName);
 
-  /// \brief Emit visible names into a hashed accelerator table section.
+  /// Emit visible names into a hashed accelerator table section.
   void emitAccelNames();
 
-  /// \brief Emit objective C classes and categories into a hashed
+  /// Emit objective C classes and categories into a hashed
   /// accelerator table section.
   void emitAccelObjC();
 
-  /// \brief Emit namespace dies into a hashed accelerator table.
+  /// Emit namespace dies into a hashed accelerator table.
   void emitAccelNamespaces();
 
-  /// \brief Emit type dies into a hashed accelerator table.
+  /// Emit type dies into a hashed accelerator table.
   void emitAccelTypes();
 
-  /// \brief Emit visible names into a debug pubnames section.
+  /// Emit visible names into a debug pubnames section.
   /// \param GnuStyle determines whether or not we want to emit
   /// additional information into the table ala newer gcc for gdb
   /// index.
   void emitDebugPubNames(bool GnuStyle = false);
 
-  /// \brief Emit visible types into a debug pubtypes section.
+  /// Emit visible types into a debug pubtypes section.
   /// \param GnuStyle determines whether or not we want to emit
   /// additional information into the table ala newer gcc for gdb
   /// index.
   void emitDebugPubTypes(bool GnuStyle = false);
 
   void emitDebugPubSection(
-      bool GnuStyle, const MCSection *PSec, StringRef Name,
+      bool GnuStyle, MCSection *PSec, StringRef Name,
       const StringMap<const DIE *> &(DwarfCompileUnit::*Accessor)() const);
 
-  /// \brief Emit visible names into a debug str section.
+  /// Emit visible names into a debug str section.
   void emitDebugStr();
 
-  /// \brief Emit visible names into a debug loc section.
+  /// Emit visible names into a debug loc section.
   void emitDebugLoc();
 
-  /// \brief Emit visible names into a debug loc dwo section.
+  /// Emit visible names into a debug loc dwo section.
   void emitDebugLocDWO();
 
-  /// \brief Emit visible names into a debug aranges section.
+  /// Emit visible names into a debug aranges section.
   void emitDebugARanges();
 
-  /// \brief Emit visible names into a debug ranges section.
+  /// Emit visible names into a debug ranges section.
   void emitDebugRanges();
 
-  /// \brief Emit inline info using custom format.
+  /// Emit inline info using custom format.
   void emitDebugInlineInfo();
 
   /// DWARF 5 Experimental Split Dwarf Emitters
 
-  /// \brief Initialize common features of skeleton units.
+  /// Initialize common features of skeleton units.
   void initSkeletonUnit(const DwarfUnit &U, DIE &Die,
                         std::unique_ptr<DwarfUnit> NewU);
 
-  /// \brief Construct the split debug info compile unit for the debug info
+  /// Construct the split debug info compile unit for the debug info
   /// section.
   DwarfCompileUnit &constructSkeletonCU(const DwarfCompileUnit &CU);
 
-  /// \brief Construct the split debug info compile unit for the debug info
+  /// Construct the split debug info compile unit for the debug info
   /// section.
   DwarfTypeUnit &constructSkeletonTU(DwarfTypeUnit &TU);
 
-  /// \brief Emit the debug info dwo section.
+  /// Emit the debug info dwo section.
   void emitDebugInfoDWO();
 
-  /// \brief Emit the debug abbrev dwo section.
+  /// Emit the debug abbrev dwo section.
   void emitDebugAbbrevDWO();
 
-  /// \brief Emit the debug line dwo section.
+  /// Emit the debug line dwo section.
   void emitDebugLineDWO();
 
-  /// \brief Emit the debug str dwo section.
+  /// Emit the debug str dwo section.
   void emitDebugStrDWO();
 
   /// Flags to let the linker know we have emitted new style pubnames. Only
   /// emit it here if we don't have a skeleton CU for split dwarf.
   void addGnuPubAttributes(DwarfUnit &U, DIE &D) const;
 
-  /// \brief Create new DwarfCompileUnit for the given metadata node with tag
+  /// Create new DwarfCompileUnit for the given metadata node with tag
   /// DW_TAG_compile_unit.
-  DwarfCompileUnit &constructDwarfCompileUnit(DICompileUnit DIUnit);
+  DwarfCompileUnit &constructDwarfCompileUnit(const DICompileUnit *DIUnit);
 
-  /// \brief Construct imported_module or imported_declaration DIE.
+  /// Construct imported_module or imported_declaration DIE.
   void constructAndAddImportedEntityDIE(DwarfCompileUnit &TheCU,
-                                        const MDNode *N);
+                                        const DIImportedEntity *N);
 
-  /// \brief Register a source line with debug info. Returns the unique
+  /// Register a source line with debug info. Returns the unique
   /// label that was emitted and which provides correspondence to the
   /// source line list.
   void recordSourceLine(unsigned Line, unsigned Col, const MDNode *Scope,
                         unsigned Flags);
 
-  /// \brief Indentify instructions that are marking the beginning of or
+  /// Indentify instructions that are marking the beginning of or
   /// ending of a scope.
   void identifyScopeMarkers();
 
-  /// \brief Populate LexicalScope entries with variables' info.
-  void collectVariableInfo(DwarfCompileUnit &TheCU, DISubprogram SP,
+  /// Populate LexicalScope entries with variables' info.
+  void collectVariableInfo(DwarfCompileUnit &TheCU, const DISubprogram *SP,
                            DenseSet<InlinedVariable> &ProcessedVars);
 
-  /// \brief Build the location list for all DBG_VALUEs in the
+  /// Build the location list for all DBG_VALUEs in the
   /// function that describe the same variable.
   void buildLocationList(SmallVectorImpl<DebugLocEntry> &DebugLoc,
                          const DbgValueHistoryMap::InstrRanges &Ranges);
 
-  /// \brief Collect variable information from the side table maintained
+  /// Collect variable information from the side table maintained
   /// by MMI.
   void collectVariableInfoFromMMITable(DenseSet<InlinedVariable> &P);
 
-  /// \brief Ensure that a label will be emitted before MI.
+  /// Ensure that a label will be emitted before MI.
   void requestLabelBeforeInsn(const MachineInstr *MI) {
     LabelsBeforeInsn.insert(std::make_pair(MI, nullptr));
   }
 
-  /// \brief Ensure that a label will be emitted after MI.
+  /// Ensure that a label will be emitted after MI.
   void requestLabelAfterInsn(const MachineInstr *MI) {
     LabelsAfterInsn.insert(std::make_pair(MI, nullptr));
   }
@@ -497,50 +551,62 @@ class DwarfDebug : public AsmPrinterHandler {
 
   ~DwarfDebug() override;
 
-  /// \brief Emit all Dwarf sections that should come prior to the
+  /// Emit all Dwarf sections that should come prior to the
   /// content.
   void beginModule();
 
-  /// \brief Emit all Dwarf sections that should come after the content.
+  /// Emit all Dwarf sections that should come after the content.
   void endModule() override;
 
-  /// \brief Gather pre-function debug information.
+  /// Gather pre-function debug information.
   void beginFunction(const MachineFunction *MF) override;
 
-  /// \brief Gather and emit post-function debug information.
+  /// Gather and emit post-function debug information.
   void endFunction(const MachineFunction *MF) override;
 
-  /// \brief Process beginning of an instruction.
+  /// Process beginning of an instruction.
   void beginInstruction(const MachineInstr *MI) override;
 
-  /// \brief Process end of an instruction.
+  /// Process end of an instruction.
   void endInstruction() override;
 
-  /// \brief Add a DIE to the set of types that we're going to pull into
+  /// Perform an MD5 checksum of \p Identifier and return the lower 64 bits.
+  static uint64_t makeTypeSignature(StringRef Identifier);
+
+  /// Add a DIE to the set of types that we're going to pull into
   /// type units.
   void addDwarfTypeUnitType(DwarfCompileUnit &CU, StringRef Identifier,
-                            DIE &Die, DICompositeType CTy);
+                            DIE &Die, const DICompositeType *CTy);
 
-  /// \brief Add a label so that arange data can be generated for it.
+  /// Add a label so that arange data can be generated for it.
   void addArangeLabel(SymbolCU SCU) { ArangeLabels.push_back(SCU); }
 
-  /// \brief For symbols that have a size designated (e.g. common symbols),
+  /// For symbols that have a size designated (e.g. common symbols),
   /// this tracks that size.
   void setSymbolSize(const MCSymbol *Sym, uint64_t Size) override {
     SymSize[Sym] = Size;
   }
 
-  /// \brief Returns whether to use DW_OP_GNU_push_tls_address, instead of the
+  /// Returns whether to use DW_OP_GNU_push_tls_address, instead of the
   /// standard DW_OP_form_tls_address opcode
   bool useGNUTLSOpcode() const { return UseGNUTLSOpcode; }
 
+  /// \defgroup DebuggerTuning Predicates to tune DWARF for a given debugger.
+  ///
+  /// Returns whether we are "tuning" for a given debugger.
+  /// @{
+  bool tuneForGDB() const { return DebuggerTuning == DebuggerKind::GDB; }
+  bool tuneForLLDB() const { return DebuggerTuning == DebuggerKind::LLDB; }
+  bool tuneForSCE() const { return DebuggerTuning == DebuggerKind::SCE; }
+  /// @}
+
   // Experimental DWARF5 features.
 
-  /// \brief Returns whether or not to emit tables that dwarf consumers can
+  /// Returns whether or not to emit tables that dwarf consumers can
   /// use to accelerate lookup.
   bool useDwarfAccelTables() const { return HasDwarfAccelTables; }
 
-  /// \brief Returns whether or not to change the current debug info for the
+  /// Returns whether or not to change the current debug info for the
   /// split dwarf proposal support.
   bool useSplitDwarf() const { return HasSplitDwarf; }
 
@@ -552,33 +618,22 @@ class DwarfDebug : public AsmPrinterHandler {
   void setPrevCU(const DwarfCompileUnit *PrevCU) { this->PrevCU = PrevCU; }
 
   /// Returns the entries for the .debug_loc section.
-  const SmallVectorImpl<DebugLocList> &
-  getDebugLocEntries() const {
-    return DotDebugLocEntries;
-  }
+  const DebugLocStream &getDebugLocs() const { return DebugLocs; }
 
-  /// \brief Emit an entry for the debug loc section. This can be used to
+  /// Emit an entry for the debug loc section. This can be used to
   /// handle an entry that's going to be emitted into the debug loc section.
   void emitDebugLocEntry(ByteStreamer &Streamer,
-                         const DebugLocEntry &Entry);
-  /// \brief emit a single value for the debug loc section.
-  void emitDebugLocValue(ByteStreamer &Streamer,
-                         const DebugLocEntry::Value &Value,
-                         unsigned PieceOffsetInBits = 0);
-  /// Emits an optimal (=sorted) sequence of DW_OP_pieces.
-  void emitLocPieces(ByteStreamer &Streamer,
-                     const DITypeIdentifierMap &Map,
-                     ArrayRef<DebugLocEntry::Value> Values);
+                         const DebugLocStream::Entry &Entry);
 
   /// Emit the location for a debug loc entry, including the size header.
-  void emitDebugLocEntryLocation(const DebugLocEntry &Entry);
+  void emitDebugLocEntryLocation(const DebugLocStream::Entry &Entry);
 
   /// Find the MDNode for the given reference.
-  template <typename T> T *resolve(TypedDebugNodeRef<T> Ref) const {
+  template <typename T> T *resolve(TypedDINodeRef<T> Ref) const {
     return Ref.resolve(TypeIdentifierMap);
   }
 
-  /// \brief Return the TypeIdentifierMap.
+  /// Return the TypeIdentifierMap.
   const DITypeIdentifierMap &getTypeIdentifierMap() const {
     return TypeIdentifierMap;
   }
@@ -591,7 +646,7 @@ class DwarfDebug : public AsmPrinterHandler {
   /// or another context nested inside a subprogram.
   bool isSubprogramContext(const MDNode *Context);
 
-  void addSubprogramNames(DISubprogram SP, DIE &Die);
+  void addSubprogramNames(const DISubprogram *SP, DIE &Die);
 
   AddressPool &getAddressPool() { return AddrPool; }
 
@@ -613,14 +668,14 @@ class DwarfDebug : public AsmPrinterHandler {
         less_first()));
   }
 
-  /// \brief A helper function to check whether the DIE for a given Scope is
+  /// A helper function to check whether the DIE for a given Scope is
   /// going to be null.
   bool isLexicalScopeDIENull(LexicalScope *Scope);
 
-  /// \brief Return Label preceding the instruction.
+  /// Return Label preceding the instruction.
   MCSymbol *getLabelBeforeInsn(const MachineInstr *MI);
 
-  /// \brief Return Label immediately following the instruction.
+  /// Return Label immediately following the instruction.
   MCSymbol *getLabelAfterInsn(const MachineInstr *MI);
 
   // FIXME: Sink these functions down into DwarfFile/Dwarf*Unit.
diff --git a/lib/CodeGen/AsmPrinter/DwarfException.h b/lib/CodeGen/AsmPrinter/DwarfException.h
index a4fd36f0339..f4667b4a346 100644
--- a/lib/CodeGen/AsmPrinter/DwarfException.h
+++ b/lib/CodeGen/AsmPrinter/DwarfException.h
@@ -21,7 +21,7 @@ namespace llvm {
 class MachineFunction;
 class ARMTargetStreamer;
 
-class DwarfCFIExceptionBase : public EHStreamer {
+class LLVM_LIBRARY_VISIBILITY DwarfCFIExceptionBase : public EHStreamer {
 protected:
   DwarfCFIExceptionBase(AsmPrinter *A);
 
@@ -31,7 +31,7 @@ class DwarfCFIExceptionBase : public EHStreamer {
   void markFunctionEnd() override;
 };
 
-class DwarfCFIException : public DwarfCFIExceptionBase {
+class LLVM_LIBRARY_VISIBILITY DwarfCFIException : public DwarfCFIExceptionBase {
   /// Per-function flag to indicate if .cfi_personality should be emitted.
   bool shouldEmitPersonality;
 
@@ -61,7 +61,7 @@ class DwarfCFIException : public DwarfCFIExceptionBase {
   void endFunction(const MachineFunction *) override;
 };
 
-class ARMException : public DwarfCFIExceptionBase {
+class LLVM_LIBRARY_VISIBILITY ARMException : public DwarfCFIExceptionBase {
   void emitTypeInfos(unsigned TTypeEncoding) override;
   ARMTargetStreamer &getTargetStreamer();
 
diff --git a/lib/CodeGen/AsmPrinter/DwarfExpression.cpp b/lib/CodeGen/AsmPrinter/DwarfExpression.cpp
index e576c93035e..a2799b8d630 100644
--- a/lib/CodeGen/AsmPrinter/DwarfExpression.cpp
+++ b/lib/CodeGen/AsmPrinter/DwarfExpression.cpp
@@ -192,7 +192,7 @@ static unsigned getOffsetOrZero(unsigned OffsetInBits,
   return OffsetInBits;
 }
 
-bool DwarfExpression::AddMachineRegExpression(DIExpression Expr,
+bool DwarfExpression::AddMachineRegExpression(const DIExpression *Expr,
                                               unsigned MachineReg,
                                               unsigned PieceOffsetInBits) {
   auto I = Expr->expr_op_begin();
@@ -240,8 +240,8 @@ bool DwarfExpression::AddMachineRegExpression(DIExpression Expr,
   return true;
 }
 
-void DwarfExpression::AddExpression(MDExpression::expr_op_iterator I,
-                                    MDExpression::expr_op_iterator E,
+void DwarfExpression::AddExpression(DIExpression::expr_op_iterator I,
+                                    DIExpression::expr_op_iterator E,
                                     unsigned PieceOffsetInBits) {
   for (; I != E; ++I) {
     switch (I->getOp()) {
@@ -259,7 +259,7 @@ void DwarfExpression::AddExpression(MDExpression::expr_op_iterator I,
       EmitOp(dwarf::DW_OP_deref);
       break;
     default:
-      llvm_unreachable("unhandled opcode found in DIExpression");
+      llvm_unreachable("unhandled opcode found in expression");
     }
   }
 }
diff --git a/lib/CodeGen/AsmPrinter/DwarfExpression.h b/lib/CodeGen/AsmPrinter/DwarfExpression.h
index a8b65f50434..78ec937a6b6 100644
--- a/lib/CodeGen/AsmPrinter/DwarfExpression.h
+++ b/lib/CodeGen/AsmPrinter/DwarfExpression.h
@@ -88,17 +88,18 @@ class DwarfExpression {
   /// Emit an unsigned constant.
   void AddUnsignedConstant(unsigned Value);
 
-  /// Emit an entire DIExpression on top of a machine register location.
+  /// \brief Emit an entire expression on top of a machine register location.
+  ///
   /// \param PieceOffsetInBits If this is one piece out of a fragmented
   /// location, this is the offset of the piece inside the entire variable.
   /// \return false if no DWARF register exists for MachineReg.
-  bool AddMachineRegExpression(DIExpression Expr, unsigned MachineReg,
+  bool AddMachineRegExpression(const DIExpression *Expr, unsigned MachineReg,
                                unsigned PieceOffsetInBits = 0);
   /// Emit a the operations remaining the DIExpressionIterator I.
   /// \param PieceOffsetInBits If this is one piece out of a fragmented
   /// location, this is the offset of the piece inside the entire variable.
-  void AddExpression(MDExpression::expr_op_iterator I,
-                     MDExpression::expr_op_iterator E,
+  void AddExpression(DIExpression::expr_op_iterator I,
+                     DIExpression::expr_op_iterator E,
                      unsigned PieceOffsetInBits = 0);
 };
 
diff --git a/lib/CodeGen/AsmPrinter/DwarfFile.cpp b/lib/CodeGen/AsmPrinter/DwarfFile.cpp
index 32adb40f5a2..51b27b462a7 100644
--- a/lib/CodeGen/AsmPrinter/DwarfFile.cpp
+++ b/lib/CodeGen/AsmPrinter/DwarfFile.cpp
@@ -20,25 +20,34 @@ namespace llvm {
 DwarfFile::DwarfFile(AsmPrinter *AP, StringRef Pref, BumpPtrAllocator &DA)
     : Asm(AP), StrPool(DA, *Asm, Pref) {}
 
-DwarfFile::~DwarfFile() {}
+DwarfFile::~DwarfFile() {
+  for (DIEAbbrev *Abbrev : Abbreviations)
+    Abbrev->~DIEAbbrev();
+}
 
 // Define a unique number for the abbreviation.
 //
-void DwarfFile::assignAbbrevNumber(DIEAbbrev &Abbrev) {
-  // Check the set for priors.
-  DIEAbbrev *InSet = AbbreviationsSet.GetOrInsertNode(&Abbrev);
-
-  // If it's newly added.
-  if (InSet == &Abbrev) {
-    // Add to abbreviation list.
-    Abbreviations.push_back(&Abbrev);
-
-    // Assign the vector position + 1 as its number.
-    Abbrev.setNumber(Abbreviations.size());
-  } else {
-    // Assign existing abbreviation number.
-    Abbrev.setNumber(InSet->getNumber());
+DIEAbbrev &DwarfFile::assignAbbrevNumber(DIE &Die) {
+  FoldingSetNodeID ID;
+  DIEAbbrev Abbrev = Die.generateAbbrev();
+  Abbrev.Profile(ID);
+
+  void *InsertPos;
+  if (DIEAbbrev *Existing =
+          AbbreviationsSet.FindNodeOrInsertPos(ID, InsertPos)) {
+    Die.setAbbrevNumber(Existing->getNumber());
+    return *Existing;
   }
+
+  // Move the abbreviation to the heap and assign a number.
+  DIEAbbrev *New = new (AbbrevAllocator) DIEAbbrev(std::move(Abbrev));
+  Abbreviations.push_back(New);
+  New->setNumber(Abbreviations.size());
+  Die.setAbbrevNumber(Abbreviations.size());
+
+  // Store it for lookup.
+  AbbreviationsSet.InsertNode(New, InsertPos);
+  return *New;
 }
 
 void DwarfFile::addUnit(std::unique_ptr<DwarfUnit> U) {
@@ -50,8 +59,8 @@ void DwarfFile::addUnit(std::unique_ptr<DwarfUnit> U) {
 void DwarfFile::emitUnits(bool UseOffsets) {
   for (const auto &TheU : CUs) {
     DIE &Die = TheU->getUnitDie();
-    const MCSection *USection = TheU->getSection();
-    Asm->OutStreamer.SwitchSection(USection);
+    MCSection *USection = TheU->getSection();
+    Asm->OutStreamer->SwitchSection(USection);
 
     TheU->emitHeader(UseOffsets);
 
@@ -83,10 +92,7 @@ void DwarfFile::computeSizeAndOffsets() {
 // CU. It returns the offset after laying out the DIE.
 unsigned DwarfFile::computeSizeAndOffset(DIE &Die, unsigned Offset) {
   // Record the abbreviation.
-  assignAbbrevNumber(Die.getAbbrev());
-
-  // Get the abbreviation for this DIE.
-  const DIEAbbrev &Abbrev = Die.getAbbrev();
+  const DIEAbbrev &Abbrev = assignAbbrevNumber(Die);
 
   // Set DIE offset
   Die.setOffset(Offset);
@@ -94,23 +100,18 @@ unsigned DwarfFile::computeSizeAndOffset(DIE &Die, unsigned Offset) {
   // Start the size with the size of abbreviation code.
   Offset += getULEB128Size(Die.getAbbrevNumber());
 
-  const SmallVectorImpl<DIEValue *> &Values = Die.getValues();
-  const SmallVectorImpl<DIEAbbrevData> &AbbrevData = Abbrev.getData();
-
   // Size the DIE attribute values.
-  for (unsigned i = 0, N = Values.size(); i < N; ++i)
+  for (const auto &V : Die.values())
     // Size attribute value.
-    Offset += Values[i]->SizeOf(Asm, AbbrevData[i].getForm());
-
-  // Get the children.
-  const auto &Children = Die.getChildren();
+    Offset += V.SizeOf(Asm);
 
   // Size the DIE children if any.
-  if (!Children.empty()) {
+  if (Die.hasChildren()) {
+    (void)Abbrev;
     assert(Abbrev.hasChildren() && "Children flag not set");
 
-    for (auto &Child : Children)
-      Offset = computeSizeAndOffset(*Child, Offset);
+    for (auto &Child : Die.children())
+      Offset = computeSizeAndOffset(Child, Offset);
 
     // End of children marker.
     Offset += sizeof(int8_t);
@@ -120,24 +121,23 @@ unsigned DwarfFile::computeSizeAndOffset(DIE &Die, unsigned Offset) {
   return Offset;
 }
 
-void DwarfFile::emitAbbrevs(const MCSection *Section) {
+void DwarfFile::emitAbbrevs(MCSection *Section) {
   // Check to see if it is worth the effort.
   if (!Abbreviations.empty()) {
     // Start the debug abbrev section.
-    Asm->OutStreamer.SwitchSection(Section);
+    Asm->OutStreamer->SwitchSection(Section);
     Asm->emitDwarfAbbrevs(Abbreviations);
   }
 }
 
 // Emit strings into a string section.
-void DwarfFile::emitStrings(const MCSection *StrSection,
-                            const MCSection *OffsetSection) {
+void DwarfFile::emitStrings(MCSection *StrSection, MCSection *OffsetSection) {
   StrPool.emit(*Asm, StrSection, OffsetSection);
 }
 
 bool DwarfFile::addScopeVariable(LexicalScope *LS, DbgVariable *Var) {
   SmallVectorImpl<DbgVariable *> &Vars = ScopeVariables[LS];
-  DIVariable DV = Var->getVariable();
+  const DILocalVariable *DV = Var->getVariable();
   // Variables with positive arg numbers are parameters.
   if (unsigned ArgNum = DV->getArg()) {
     // Keep all parameters in order at the start of the variable list to ensure
diff --git a/lib/CodeGen/AsmPrinter/DwarfFile.h b/lib/CodeGen/AsmPrinter/DwarfFile.h
index c9de6665b01..8402027edd6 100644
--- a/lib/CodeGen/AsmPrinter/DwarfFile.h
+++ b/lib/CodeGen/AsmPrinter/DwarfFile.h
@@ -28,15 +28,17 @@ class DwarfUnit;
 class DIEAbbrev;
 class MCSymbol;
 class DIE;
-class DISubprogram;
 class LexicalScope;
 class StringRef;
 class DwarfDebug;
 class MCSection;
+class MDNode;
 class DwarfFile {
   // Target of Dwarf emission, used for sizing of abbreviations.
   AsmPrinter *Asm;
 
+  BumpPtrAllocator AbbrevAllocator;
+
   // Used to uniquely define abbreviations.
   FoldingSet<DIEAbbrev> AbbreviationsSet;
 
@@ -57,7 +59,7 @@ class DwarfFile {
   /// Maps MDNodes for type system with the corresponding DIEs. These DIEs can
   /// be shared across CUs, that is why we keep the map here instead
   /// of in DwarfCompileUnit.
-  DenseMap<const MDNode *, DIE *> MDTypeNodeToDieMap;
+  DenseMap<const MDNode *, DIE *> DITypeNodeToDieMap;
 
 public:
   DwarfFile(AsmPrinter *AP, StringRef Pref, BumpPtrAllocator &DA);
@@ -72,8 +74,11 @@ class DwarfFile {
   /// \brief Compute the size and offset of all the DIEs.
   void computeSizeAndOffsets();
 
-  /// \brief Define a unique number for the abbreviation.
-  void assignAbbrevNumber(DIEAbbrev &Abbrev);
+  /// Define a unique number for the abbreviation.
+  ///
+  /// Compute the abbreviation for \c Die, look up its unique number, and
+  /// return a reference to it in the uniquing table.
+  DIEAbbrev &assignAbbrevNumber(DIE &Die);
 
   /// \brief Add a unit to the list of CUs.
   void addUnit(std::unique_ptr<DwarfUnit> U);
@@ -83,11 +88,10 @@ class DwarfFile {
   void emitUnits(bool UseOffsets);
 
   /// \brief Emit a set of abbreviations to the specific section.
-  void emitAbbrevs(const MCSection *);
+  void emitAbbrevs(MCSection *);
 
   /// \brief Emit all of the strings to the section given.
-  void emitStrings(const MCSection *StrSection,
-                   const MCSection *OffsetSection = nullptr);
+  void emitStrings(MCSection *StrSection, MCSection *OffsetSection = nullptr);
 
   /// \brief Returns the string pool.
   DwarfStringPool &getStringPool() { return StrPool; }
@@ -104,10 +108,10 @@ class DwarfFile {
   }
 
   void insertDIE(const MDNode *TypeMD, DIE *Die) {
-    MDTypeNodeToDieMap.insert(std::make_pair(TypeMD, Die));
+    DITypeNodeToDieMap.insert(std::make_pair(TypeMD, Die));
   }
   DIE *getDIE(const MDNode *TypeMD) {
-    return MDTypeNodeToDieMap.lookup(TypeMD);
+    return DITypeNodeToDieMap.lookup(TypeMD);
   }
 };
 }
diff --git a/lib/CodeGen/AsmPrinter/DwarfStringPool.cpp b/lib/CodeGen/AsmPrinter/DwarfStringPool.cpp
index 165ef16dfbc..2066f745e31 100644
--- a/lib/CodeGen/AsmPrinter/DwarfStringPool.cpp
+++ b/lib/CodeGen/AsmPrinter/DwarfStringPool.cpp
@@ -8,63 +8,67 @@
 //===----------------------------------------------------------------------===//
 
 #include "DwarfStringPool.h"
+#include "llvm/CodeGen/AsmPrinter.h"
+#include "llvm/MC/MCAsmInfo.h"
 #include "llvm/MC/MCStreamer.h"
 
 using namespace llvm;
 
-static std::pair<MCSymbol *, unsigned> &
-getEntry(AsmPrinter &Asm,
-         StringMap<std::pair<MCSymbol *, unsigned>, BumpPtrAllocator &> &Pool,
-         StringRef Prefix, StringRef Str) {
-  std::pair<MCSymbol *, unsigned> &Entry = Pool[Str];
-  if (!Entry.first) {
-    Entry.second = Pool.size() - 1;
-    Entry.first = Asm.createTempSymbol(Prefix);
-  }
-  return Entry;
-}
+DwarfStringPool::DwarfStringPool(BumpPtrAllocator &A, AsmPrinter &Asm,
+                                 StringRef Prefix)
+    : Pool(A), Prefix(Prefix),
+      ShouldCreateSymbols(Asm.MAI->doesDwarfUseRelocationsAcrossSections()) {}
 
-MCSymbol *DwarfStringPool::getSymbol(AsmPrinter &Asm, StringRef Str) {
-  return getEntry(Asm, Pool, Prefix, Str).first;
-}
+DwarfStringPool::EntryRef DwarfStringPool::getEntry(AsmPrinter &Asm,
+                                                    StringRef Str) {
+  auto I = Pool.insert(std::make_pair(Str, EntryTy()));
+  if (I.second) {
+    auto &Entry = I.first->second;
+    Entry.Index = Pool.size() - 1;
+    Entry.Offset = NumBytes;
+    Entry.Symbol = ShouldCreateSymbols ? Asm.createTempSymbol(Prefix) : nullptr;
 
-unsigned DwarfStringPool::getIndex(AsmPrinter &Asm, StringRef Str) {
-  return getEntry(Asm, Pool, Prefix, Str).second;
+    NumBytes += Str.size() + 1;
+    assert(NumBytes > Entry.Offset && "Unexpected overflow");
+  }
+  return EntryRef(*I.first);
 }
 
-void DwarfStringPool::emit(AsmPrinter &Asm, const MCSection *StrSection,
-                           const MCSection *OffsetSection) {
+void DwarfStringPool::emit(AsmPrinter &Asm, MCSection *StrSection,
+                           MCSection *OffsetSection) {
   if (Pool.empty())
     return;
 
   // Start the dwarf str section.
-  Asm.OutStreamer.SwitchSection(StrSection);
+  Asm.OutStreamer->SwitchSection(StrSection);
 
   // Get all of the string pool entries and put them in an array by their ID so
   // we can sort them.
-  SmallVector<const StringMapEntry<std::pair<MCSymbol *, unsigned>> *, 64>
-  Entries(Pool.size());
+  SmallVector<const StringMapEntry<EntryTy> *, 64> Entries(Pool.size());
 
   for (const auto &E : Pool)
-    Entries[E.getValue().second] = &E;
+    Entries[E.getValue().Index] = &E;
 
   for (const auto &Entry : Entries) {
+    assert(ShouldCreateSymbols == static_cast<bool>(Entry->getValue().Symbol) &&
+           "Mismatch between setting and entry");
+
     // Emit a label for reference from debug information entries.
-    Asm.OutStreamer.EmitLabel(Entry->getValue().first);
+    if (ShouldCreateSymbols)
+      Asm.OutStreamer->EmitLabel(Entry->getValue().Symbol);
 
     // Emit the string itself with a terminating null byte.
-    Asm.OutStreamer.EmitBytes(
+    Asm.OutStreamer->AddComment("string offset=" +
+                                Twine(Entry->getValue().Offset));
+    Asm.OutStreamer->EmitBytes(
         StringRef(Entry->getKeyData(), Entry->getKeyLength() + 1));
   }
 
   // If we've got an offset section go ahead and emit that now as well.
   if (OffsetSection) {
-    Asm.OutStreamer.SwitchSection(OffsetSection);
-    unsigned offset = 0;
+    Asm.OutStreamer->SwitchSection(OffsetSection);
     unsigned size = 4; // FIXME: DWARF64 is 8.
-    for (const auto &Entry : Entries) {
-      Asm.OutStreamer.EmitIntValue(offset, size);
-      offset += Entry->getKeyLength() + 1;
-    }
+    for (const auto &Entry : Entries)
+      Asm.OutStreamer->EmitIntValue(Entry->getValue().Offset, size);
   }
 }
diff --git a/lib/CodeGen/AsmPrinter/DwarfStringPool.h b/lib/CodeGen/AsmPrinter/DwarfStringPool.h
index 63e34123d85..93a168485a5 100644
--- a/lib/CodeGen/AsmPrinter/DwarfStringPool.h
+++ b/lib/CodeGen/AsmPrinter/DwarfStringPool.h
@@ -11,12 +11,13 @@
 #define LLVM_LIB_CODEGEN_ASMPRINTER_DWARFSTRINGPOOL_H
 
 #include "llvm/ADT/StringMap.h"
-#include "llvm/CodeGen/AsmPrinter.h"
+#include "llvm/CodeGen/DwarfStringPoolEntry.h"
 #include "llvm/Support/Allocator.h"
 #include <utility>
 
 namespace llvm {
 
+class AsmPrinter;
 class MCSymbol;
 class MCSection;
 class StringRef;
@@ -25,25 +26,24 @@ class StringRef;
 // A String->Symbol mapping of strings used by indirect
 // references.
 class DwarfStringPool {
-  StringMap<std::pair<MCSymbol *, unsigned>, BumpPtrAllocator &> Pool;
+  typedef DwarfStringPoolEntry EntryTy;
+  StringMap<EntryTy, BumpPtrAllocator &> Pool;
   StringRef Prefix;
+  unsigned NumBytes = 0;
+  bool ShouldCreateSymbols;
 
 public:
-  DwarfStringPool(BumpPtrAllocator &A, AsmPrinter &Asm, StringRef Prefix)
-      : Pool(A), Prefix(Prefix) {}
+  typedef DwarfStringPoolEntryRef EntryRef;
 
-  void emit(AsmPrinter &Asm, const MCSection *StrSection,
-            const MCSection *OffsetSection = nullptr);
+  DwarfStringPool(BumpPtrAllocator &A, AsmPrinter &Asm, StringRef Prefix);
 
-  /// \brief Returns an entry into the string pool with the given
-  /// string text.
-  MCSymbol *getSymbol(AsmPrinter &Asm, StringRef Str);
-
-  /// \brief Returns the index into the string pool with the given
-  /// string text.
-  unsigned getIndex(AsmPrinter &Asm, StringRef Str);
+  void emit(AsmPrinter &Asm, MCSection *StrSection,
+            MCSection *OffsetSection = nullptr);
 
   bool empty() const { return Pool.empty(); }
+
+  /// Get a reference to an entry in the string pool.
+  EntryRef getEntry(AsmPrinter &Asm, StringRef Str);
 };
 }
 #endif
diff --git a/lib/CodeGen/AsmPrinter/DwarfUnit.cpp b/lib/CodeGen/AsmPrinter/DwarfUnit.cpp
index 91546529877..2240868229b 100644
--- a/lib/CodeGen/AsmPrinter/DwarfUnit.cpp
+++ b/lib/CodeGen/AsmPrinter/DwarfUnit.cpp
@@ -63,15 +63,14 @@ bool DIEDwarfExpression::isFrameRegister(unsigned MachineReg) {
   return MachineReg == TRI.getFrameRegister(*AP.MF);
 }
 
-
-/// Unit - Unit constructor.
-DwarfUnit::DwarfUnit(unsigned UID, dwarf::Tag UnitTag, DICompileUnit Node,
-                     AsmPrinter *A, DwarfDebug *DW, DwarfFile *DWU)
-    : UniqueID(UID), CUNode(Node), UnitDie(UnitTag), DebugInfoOffset(0), Asm(A),
-      DD(DW), DU(DWU), IndexTyDie(nullptr), Section(nullptr) {
+DwarfUnit::DwarfUnit(unsigned UID, dwarf::Tag UnitTag,
+                     const DICompileUnit *Node, AsmPrinter *A, DwarfDebug *DW,
+                     DwarfFile *DWU)
+    : UniqueID(UID), CUNode(Node),
+      UnitDie(*DIE::get(DIEValueAllocator, UnitTag)), DebugInfoOffset(0),
+      Asm(A), DD(DW), DU(DWU), IndexTyDie(nullptr), Section(nullptr) {
   assert(UnitTag == dwarf::DW_TAG_compile_unit ||
          UnitTag == dwarf::DW_TAG_type_unit);
-  DIEIntegerOne = new (DIEValueAllocator) DIEInteger(1);
 }
 
 DwarfTypeUnit::DwarfTypeUnit(unsigned UID, DwarfCompileUnit &CU, AsmPrinter *A,
@@ -83,7 +82,6 @@ DwarfTypeUnit::DwarfTypeUnit(unsigned UID, DwarfCompileUnit &CU, AsmPrinter *A,
     addSectionOffset(UnitDie, dwarf::DW_AT_stmt_list, 0);
 }
 
-/// ~Unit - Destructor for compile unit.
 DwarfUnit::~DwarfUnit() {
   for (unsigned j = 0, M = DIEBlocks.size(); j < M; ++j)
     DIEBlocks[j]->~DIEBlock();
@@ -91,15 +89,6 @@ DwarfUnit::~DwarfUnit() {
     DIELocs[j]->~DIELoc();
 }
 
-/// createDIEEntry - Creates a new DIEEntry to be a proxy for a debug
-/// information entry.
-DIEEntry *DwarfUnit::createDIEEntry(DIE &Entry) {
-  DIEEntry *Value = new (DIEValueAllocator) DIEEntry(Entry);
-  return Value;
-}
-
-/// getDefaultLowerBound - Return the default lower bound for an array. If the
-/// DWARF version doesn't handle the language, return -1.
 int64_t DwarfUnit::getDefaultLowerBound() const {
   switch (getLanguage()) {
   default:
@@ -167,7 +156,7 @@ int64_t DwarfUnit::getDefaultLowerBound() const {
 }
 
 /// Check whether the DIE for this MDNode can be shared across CUs.
-static bool isShareableAcrossCUs(DIDescriptor D) {
+static bool isShareableAcrossCUs(const DINode *D) {
   // When the MDNode can be part of the type system, the DIE can be shared
   // across CUs.
   // Combining type units and cross-CU DIE sharing is lower value (since
@@ -175,25 +164,18 @@ static bool isShareableAcrossCUs(DIDescriptor D) {
   // level already) but may be implementable for some value in projects
   // building multiple independent libraries with LTO and then linking those
   // together.
-  return (isa<MDType>(D) ||
-          (isa<MDSubprogram>(D) && !cast<MDSubprogram>(D)->isDefinition())) &&
+  return (isa<DIType>(D) ||
+          (isa<DISubprogram>(D) && !cast<DISubprogram>(D)->isDefinition())) &&
          !GenerateDwarfTypeUnits;
 }
 
-/// getDIE - Returns the debug information entry map slot for the
-/// specified debug variable. We delegate the request to DwarfDebug
-/// when the DIE for this MDNode can be shared across CUs. The mappings
-/// will be kept in DwarfDebug for shareable DIEs.
-DIE *DwarfUnit::getDIE(DIDescriptor D) const {
+DIE *DwarfUnit::getDIE(const DINode *D) const {
   if (isShareableAcrossCUs(D))
     return DU->getDIE(D);
   return MDNodeToDieMap.lookup(D);
 }
 
-/// insertDIE - Insert DIE into the map. We delegate the request to DwarfDebug
-/// when the DIE for this MDNode can be shared across CUs. The mappings
-/// will be kept in DwarfDebug for shareable DIEs.
-void DwarfUnit::insertDIE(DIDescriptor Desc, DIE *D) {
+void DwarfUnit::insertDIE(const DINode *Desc, DIE *D) {
   if (isShareableAcrossCUs(Desc)) {
     DU->insertDIE(Desc, D);
     return;
@@ -201,37 +183,31 @@ void DwarfUnit::insertDIE(DIDescriptor Desc, DIE *D) {
   MDNodeToDieMap.insert(std::make_pair(Desc, D));
 }
 
-/// addFlag - Add a flag that is true.
 void DwarfUnit::addFlag(DIE &Die, dwarf::Attribute Attribute) {
   if (DD->getDwarfVersion() >= 4)
-    Die.addValue(Attribute, dwarf::DW_FORM_flag_present, DIEIntegerOne);
+    Die.addValue(DIEValueAllocator, Attribute, dwarf::DW_FORM_flag_present,
+                 DIEInteger(1));
   else
-    Die.addValue(Attribute, dwarf::DW_FORM_flag, DIEIntegerOne);
+    Die.addValue(DIEValueAllocator, Attribute, dwarf::DW_FORM_flag,
+                 DIEInteger(1));
 }
 
-/// addUInt - Add an unsigned integer attribute data and value.
-///
 void DwarfUnit::addUInt(DIE &Die, dwarf::Attribute Attribute,
                         Optional<dwarf::Form> Form, uint64_t Integer) {
   if (!Form)
     Form = DIEInteger::BestForm(false, Integer);
-  DIEValue *Value = Integer == 1 ? DIEIntegerOne : new (DIEValueAllocator)
-                        DIEInteger(Integer);
-  Die.addValue(Attribute, *Form, Value);
+  Die.addValue(DIEValueAllocator, Attribute, *Form, DIEInteger(Integer));
 }
 
 void DwarfUnit::addUInt(DIE &Block, dwarf::Form Form, uint64_t Integer) {
   addUInt(Block, (dwarf::Attribute)0, Form, Integer);
 }
 
-/// addSInt - Add an signed integer attribute data and value.
-///
 void DwarfUnit::addSInt(DIE &Die, dwarf::Attribute Attribute,
                         Optional<dwarf::Form> Form, int64_t Integer) {
   if (!Form)
     Form = DIEInteger::BestForm(true, Integer);
-  DIEValue *Value = new (DIEValueAllocator) DIEInteger(Integer);
-  Die.addValue(Attribute, *Form, Value);
+  Die.addValue(DIEValueAllocator, Attribute, *Form, DIEInteger(Integer));
 }
 
 void DwarfUnit::addSInt(DIELoc &Die, Optional<dwarf::Form> Form,
@@ -239,57 +215,23 @@ void DwarfUnit::addSInt(DIELoc &Die, Optional<dwarf::Form> Form,
   addSInt(Die, (dwarf::Attribute)0, Form, Integer);
 }
 
-/// addString - Add a string attribute data and value. We always emit a
-/// reference to the string pool instead of immediate strings so that DIEs have
-/// more predictable sizes. In the case of split dwarf we emit an index
-/// into another table which gets us the static offset into the string
-/// table.
 void DwarfUnit::addString(DIE &Die, dwarf::Attribute Attribute,
                           StringRef String) {
-  if (!isDwoUnit())
-    return addLocalString(Die, Attribute, String);
-
-  addIndexedString(Die, Attribute, String);
+  Die.addValue(DIEValueAllocator, Attribute,
+               isDwoUnit() ? dwarf::DW_FORM_GNU_str_index : dwarf::DW_FORM_strp,
+               DIEString(DU->getStringPool().getEntry(*Asm, String)));
 }
 
-void DwarfUnit::addIndexedString(DIE &Die, dwarf::Attribute Attribute,
-                                 StringRef String) {
-  unsigned idx = DU->getStringPool().getIndex(*Asm, String);
-  DIEValue *Value = new (DIEValueAllocator) DIEInteger(idx);
-  DIEValue *Str = new (DIEValueAllocator) DIEString(Value, String);
-  Die.addValue(Attribute, dwarf::DW_FORM_GNU_str_index, Str);
-}
-
-/// addLocalString - Add a string attribute data and value. This is guaranteed
-/// to be in the local string pool instead of indirected.
-void DwarfUnit::addLocalString(DIE &Die, dwarf::Attribute Attribute,
-                               StringRef String) {
-  const TargetLoweringObjectFile &TLOF = Asm->getObjFileLowering();
-  MCSymbol *Symb = DU->getStringPool().getSymbol(*Asm, String);
-  DIEValue *Value;
-  if (Asm->MAI->doesDwarfUseRelocationsAcrossSections())
-    Value = new (DIEValueAllocator) DIELabel(Symb);
-  else
-    Value = new (DIEValueAllocator)
-        DIEDelta(Symb, TLOF.getDwarfStrSection()->getBeginSymbol());
-  DIEValue *Str = new (DIEValueAllocator) DIEString(Value, String);
-  Die.addValue(Attribute, dwarf::DW_FORM_strp, Str);
-}
-
-/// addLabel - Add a Dwarf label attribute data and value.
-///
-void DwarfUnit::addLabel(DIE &Die, dwarf::Attribute Attribute, dwarf::Form Form,
-                         const MCSymbol *Label) {
-  DIEValue *Value = new (DIEValueAllocator) DIELabel(Label);
-  Die.addValue(Attribute, Form, Value);
+DIE::value_iterator DwarfUnit::addLabel(DIE &Die, dwarf::Attribute Attribute,
+                                        dwarf::Form Form,
+                                        const MCSymbol *Label) {
+  return Die.addValue(DIEValueAllocator, Attribute, Form, DIELabel(Label));
 }
 
 void DwarfUnit::addLabel(DIELoc &Die, dwarf::Form Form, const MCSymbol *Label) {
   addLabel(Die, (dwarf::Attribute)0, Form, Label);
 }
 
-/// addSectionOffset - Add an offset into a section attribute data and value.
-///
 void DwarfUnit::addSectionOffset(DIE &Die, dwarf::Attribute Attribute,
                                  uint64_t Integer) {
   if (DD->getDwarfVersion() >= 4)
@@ -303,9 +245,6 @@ unsigned DwarfTypeUnit::getOrCreateSourceID(StringRef FileName, StringRef DirNam
                         : getCU().getOrCreateSourceID(FileName, DirName);
 }
 
-/// addOpAddress - Add a dwarf op address data and value using the
-/// form given and an op of either DW_FORM_addr or DW_FORM_GNU_addr_index.
-///
 void DwarfUnit::addOpAddress(DIELoc &Die, const MCSymbol *Sym) {
   if (!DD->useSplitDwarf()) {
     addUInt(Die, dwarf::DW_FORM_data1, dwarf::DW_OP_addr);
@@ -319,14 +258,12 @@ void DwarfUnit::addOpAddress(DIELoc &Die, const MCSymbol *Sym) {
 
 void DwarfUnit::addLabelDelta(DIE &Die, dwarf::Attribute Attribute,
                               const MCSymbol *Hi, const MCSymbol *Lo) {
-  DIEValue *Value = new (DIEValueAllocator) DIEDelta(Hi, Lo);
-  Die.addValue(Attribute, dwarf::DW_FORM_data4, Value);
+  Die.addValue(DIEValueAllocator, Attribute, dwarf::DW_FORM_data4,
+               new (DIEValueAllocator) DIEDelta(Hi, Lo));
 }
 
-/// addDIEEntry - Add a DIE attribute data and value.
-///
 void DwarfUnit::addDIEEntry(DIE &Die, dwarf::Attribute Attribute, DIE &Entry) {
-  addDIEEntry(Die, Attribute, createDIEEntry(Entry));
+  addDIEEntry(Die, Attribute, DIEEntry(Entry));
 }
 
 void DwarfUnit::addDIETypeSignature(DIE &Die, const DwarfTypeUnit &Type) {
@@ -336,53 +273,54 @@ void DwarfUnit::addDIETypeSignature(DIE &Die, const DwarfTypeUnit &Type) {
   // and think this is a full definition.
   addFlag(Die, dwarf::DW_AT_declaration);
 
-  Die.addValue(dwarf::DW_AT_signature, dwarf::DW_FORM_ref_sig8,
-               new (DIEValueAllocator) DIETypeSignature(Type));
+  Die.addValue(DIEValueAllocator, dwarf::DW_AT_signature,
+               dwarf::DW_FORM_ref_sig8, DIETypeSignature(Type));
+}
+
+void DwarfUnit::addDIETypeSignature(DIE &Die, dwarf::Attribute Attribute,
+                                    StringRef Identifier) {
+  uint64_t Signature = DD->makeTypeSignature(Identifier);
+  Die.addValue(DIEValueAllocator, Attribute, dwarf::DW_FORM_ref_sig8,
+               DIEInteger(Signature));
 }
 
 void DwarfUnit::addDIEEntry(DIE &Die, dwarf::Attribute Attribute,
-                            DIEEntry *Entry) {
+                            DIEEntry Entry) {
   const DIE *DieCU = Die.getUnitOrNull();
-  const DIE *EntryCU = Entry->getEntry().getUnitOrNull();
+  const DIE *EntryCU = Entry.getEntry().getUnitOrNull();
   if (!DieCU)
     // We assume that Die belongs to this CU, if it is not linked to any CU yet.
     DieCU = &getUnitDie();
   if (!EntryCU)
     EntryCU = &getUnitDie();
-  Die.addValue(Attribute,
+  Die.addValue(DIEValueAllocator, Attribute,
                EntryCU == DieCU ? dwarf::DW_FORM_ref4 : dwarf::DW_FORM_ref_addr,
                Entry);
 }
 
-/// Create a DIE with the given Tag, add the DIE to its parent, and
-/// call insertDIE if MD is not null.
-DIE &DwarfUnit::createAndAddDIE(unsigned Tag, DIE &Parent, DIDescriptor N) {
+DIE &DwarfUnit::createAndAddDIE(unsigned Tag, DIE &Parent, const DINode *N) {
   assert(Tag != dwarf::DW_TAG_auto_variable &&
          Tag != dwarf::DW_TAG_arg_variable);
-  Parent.addChild(make_unique<DIE>((dwarf::Tag)Tag));
-  DIE &Die = *Parent.getChildren().back();
+  DIE &Die = Parent.addChild(DIE::get(DIEValueAllocator, (dwarf::Tag)Tag));
   if (N)
     insertDIE(N, &Die);
   return Die;
 }
 
-/// addBlock - Add block data.
-///
 void DwarfUnit::addBlock(DIE &Die, dwarf::Attribute Attribute, DIELoc *Loc) {
   Loc->ComputeSize(Asm);
   DIELocs.push_back(Loc); // Memoize so we can call the destructor later on.
-  Die.addValue(Attribute, Loc->BestForm(DD->getDwarfVersion()), Loc);
+  Die.addValue(DIEValueAllocator, Attribute,
+               Loc->BestForm(DD->getDwarfVersion()), Loc);
 }
 
 void DwarfUnit::addBlock(DIE &Die, dwarf::Attribute Attribute,
                          DIEBlock *Block) {
   Block->ComputeSize(Asm);
   DIEBlocks.push_back(Block); // Memoize so we can call the destructor later on.
-  Die.addValue(Attribute, Block->BestForm(), Block);
+  Die.addValue(DIEValueAllocator, Attribute, Block->BestForm(), Block);
 }
 
-/// addSourceLine - Add location information to specified debug information
-/// entry.
 void DwarfUnit::addSourceLine(DIE &Die, unsigned Line, StringRef File,
                               StringRef Directory) {
   if (Line == 0)
@@ -394,54 +332,41 @@ void DwarfUnit::addSourceLine(DIE &Die, unsigned Line, StringRef File,
   addUInt(Die, dwarf::DW_AT_decl_line, None, Line);
 }
 
-/// addSourceLine - Add location information to specified debug information
-/// entry.
-void DwarfUnit::addSourceLine(DIE &Die, DIVariable V) {
+void DwarfUnit::addSourceLine(DIE &Die, const DILocalVariable *V) {
   assert(V);
 
   addSourceLine(Die, V->getLine(), V->getScope()->getFilename(),
                 V->getScope()->getDirectory());
 }
 
-/// addSourceLine - Add location information to specified debug information
-/// entry.
-void DwarfUnit::addSourceLine(DIE &Die, DIGlobalVariable G) {
+void DwarfUnit::addSourceLine(DIE &Die, const DIGlobalVariable *G) {
   assert(G);
 
   addSourceLine(Die, G->getLine(), G->getFilename(), G->getDirectory());
 }
 
-/// addSourceLine - Add location information to specified debug information
-/// entry.
-void DwarfUnit::addSourceLine(DIE &Die, DISubprogram SP) {
+void DwarfUnit::addSourceLine(DIE &Die, const DISubprogram *SP) {
   assert(SP);
 
   addSourceLine(Die, SP->getLine(), SP->getFilename(), SP->getDirectory());
 }
 
-/// addSourceLine - Add location information to specified debug information
-/// entry.
-void DwarfUnit::addSourceLine(DIE &Die, DIType Ty) {
+void DwarfUnit::addSourceLine(DIE &Die, const DIType *Ty) {
   assert(Ty);
 
   addSourceLine(Die, Ty->getLine(), Ty->getFilename(), Ty->getDirectory());
 }
 
-/// addSourceLine - Add location information to specified debug information
-/// entry.
-void DwarfUnit::addSourceLine(DIE &Die, DIObjCProperty Ty) {
+void DwarfUnit::addSourceLine(DIE &Die, const DIObjCProperty *Ty) {
   assert(Ty);
 
   addSourceLine(Die, Ty->getLine(), Ty->getFilename(), Ty->getDirectory());
 }
 
-/// addSourceLine - Add location information to specified debug information
-/// entry.
-void DwarfUnit::addSourceLine(DIE &Die, DINameSpace NS) {
+void DwarfUnit::addSourceLine(DIE &Die, const DINamespace *NS) {
   addSourceLine(Die, NS->getLine(), NS->getFilename(), NS->getDirectory());
 }
 
-/// addRegisterOp - Add register operand.
 bool DwarfUnit::addRegisterOpPiece(DIELoc &TheDie, unsigned Reg,
                                    unsigned SizeInBits, unsigned OffsetInBits) {
   DIEDwarfExpression Expr(*Asm, *this, TheDie);
@@ -449,7 +374,6 @@ bool DwarfUnit::addRegisterOpPiece(DIELoc &TheDie, unsigned Reg,
   return true;
 }
 
-/// addRegisterOffset - Add register offset.
 bool DwarfUnit::addRegisterOffset(DIELoc &TheDie, unsigned Reg,
                                   int64_t Offset) {
   DIEDwarfExpression Expr(*Asm, *this, TheDie);
@@ -510,36 +434,30 @@ bool DwarfUnit::addRegisterOffset(DIELoc &TheDie, unsigned Reg,
 
    That is what this function does.  */
 
-/// addBlockByrefAddress - Start with the address based on the location
-/// provided, and generate the DWARF information necessary to find the
-/// actual Block variable (navigating the Block struct) based on the
-/// starting location.  Add the DWARF information to the die.  For
-/// more information, read large comment just above here.
-///
 void DwarfUnit::addBlockByrefAddress(const DbgVariable &DV, DIE &Die,
                                      dwarf::Attribute Attribute,
                                      const MachineLocation &Location) {
-  DIType Ty = DV.getType();
-  DIType TmpTy = Ty;
+  const DIType *Ty = DV.getType();
+  const DIType *TmpTy = Ty;
   uint16_t Tag = Ty->getTag();
   bool isPointer = false;
 
   StringRef varName = DV.getName();
 
   if (Tag == dwarf::DW_TAG_pointer_type) {
-    DIDerivedType DTy = cast<MDDerivedType>(Ty);
+    auto *DTy = cast<DIDerivedType>(Ty);
     TmpTy = resolve(DTy->getBaseType());
     isPointer = true;
   }
 
   // Find the __forwarding field and the variable field in the __Block_byref
   // struct.
-  DIArray Fields = cast<MDCompositeTypeBase>(TmpTy)->getElements();
-  DIDerivedType varField;
-  DIDerivedType forwardingField;
+  DINodeArray Fields = cast<DICompositeType>(TmpTy)->getElements();
+  const DIDerivedType *varField = nullptr;
+  const DIDerivedType *forwardingField = nullptr;
 
   for (unsigned i = 0, N = Fields.size(); i < N; ++i) {
-    DIDerivedType DT = cast<MDDerivedTypeBase>(Fields[i]);
+    auto *DT = cast<DIDerivedType>(Fields[i]);
     StringRef fieldName = DT->getName();
     if (fieldName == "__forwarding")
       forwardingField = DT;
@@ -553,7 +471,7 @@ void DwarfUnit::addBlockByrefAddress(const DbgVariable &DV, DIE &Die,
 
   // Decode the original location, and use that as the start of the byref
   // variable's location.
-  DIELoc *Loc = new (DIEValueAllocator) DIELoc();
+  DIELoc *Loc = new (DIEValueAllocator) DIELoc;
 
   bool validReg;
   if (Location.isReg())
@@ -594,38 +512,39 @@ void DwarfUnit::addBlockByrefAddress(const DbgVariable &DV, DIE &Die,
 }
 
 /// Return true if type encoding is unsigned.
-static bool isUnsignedDIType(DwarfDebug *DD, DIType Ty) {
-  if (DIDerivedType DTy = dyn_cast<MDDerivedTypeBase>(Ty)) {
+static bool isUnsignedDIType(DwarfDebug *DD, const DIType *Ty) {
+  if (auto *CTy = dyn_cast<DICompositeType>(Ty)) {
+    // FIXME: Enums without a fixed underlying type have unknown signedness
+    // here, leading to incorrectly emitted constants.
+    if (CTy->getTag() == dwarf::DW_TAG_enumeration_type)
+      return false;
+
+    // (Pieces of) aggregate types that get hacked apart by SROA may be
+    // represented by a constant. Encode them as unsigned bytes.
+    return true;
+  }
+
+  if (auto *DTy = dyn_cast<DIDerivedType>(Ty)) {
     dwarf::Tag T = (dwarf::Tag)Ty->getTag();
     // Encode pointer constants as unsigned bytes. This is used at least for
     // null pointer constant emission.
-    // (Pieces of) aggregate types that get hacked apart by SROA may also be
-    // represented by a constant. Encode them as unsigned bytes.
     // FIXME: reference and rvalue_reference /probably/ shouldn't be allowed
     // here, but accept them for now due to a bug in SROA producing bogus
     // dbg.values.
-    if (T == dwarf::DW_TAG_array_type ||
-        T == dwarf::DW_TAG_class_type ||
-        T == dwarf::DW_TAG_pointer_type ||
+    if (T == dwarf::DW_TAG_pointer_type ||
         T == dwarf::DW_TAG_ptr_to_member_type ||
         T == dwarf::DW_TAG_reference_type ||
-        T == dwarf::DW_TAG_rvalue_reference_type ||
-        T == dwarf::DW_TAG_structure_type ||
-        T == dwarf::DW_TAG_union_type)
+        T == dwarf::DW_TAG_rvalue_reference_type)
       return true;
     assert(T == dwarf::DW_TAG_typedef || T == dwarf::DW_TAG_const_type ||
            T == dwarf::DW_TAG_volatile_type ||
-           T == dwarf::DW_TAG_restrict_type ||
-           T == dwarf::DW_TAG_enumeration_type);
-    if (DITypeRef Deriv = DTy->getBaseType())
-      return isUnsignedDIType(DD, DD->resolve(Deriv));
-    // FIXME: Enums without a fixed underlying type have unknown signedness
-    // here, leading to incorrectly emitted constants.
-    assert(DTy->getTag() == dwarf::DW_TAG_enumeration_type);
-    return false;
+           T == dwarf::DW_TAG_restrict_type);
+    DITypeRef Deriv = DTy->getBaseType();
+    assert(Deriv && "Expected valid base type");
+    return isUnsignedDIType(DD, DD->resolve(Deriv));
   }
 
-  DIBasicType BTy = cast<MDBasicType>(Ty);
+  auto *BTy = cast<DIBasicType>(Ty);
   unsigned Encoding = BTy->getEncoding();
   assert((Encoding == dwarf::DW_ATE_unsigned ||
           Encoding == dwarf::DW_ATE_unsigned_char ||
@@ -643,7 +562,7 @@ static bool isUnsignedDIType(DwarfDebug *DD, DIType Ty) {
 }
 
 /// If this type is derived from a base type then return base type size.
-static uint64_t getBaseTypeSize(DwarfDebug *DD, DIDerivedType Ty) {
+static uint64_t getBaseTypeSize(DwarfDebug *DD, const DIDerivedType *Ty) {
   unsigned Tag = Ty->getTag();
 
   if (Tag != dwarf::DW_TAG_member && Tag != dwarf::DW_TAG_typedef &&
@@ -662,16 +581,15 @@ static uint64_t getBaseTypeSize(DwarfDebug *DD, DIDerivedType Ty) {
       BaseType->getTag() == dwarf::DW_TAG_rvalue_reference_type)
     return Ty->getSizeInBits();
 
-  if (auto *DT = dyn_cast<MDDerivedTypeBase>(BaseType))
+  if (auto *DT = dyn_cast<DIDerivedType>(BaseType))
     return getBaseTypeSize(DD, DT);
 
   return BaseType->getSizeInBits();
 }
 
-/// addConstantFPValue - Add constant value entry in variable DIE.
 void DwarfUnit::addConstantFPValue(DIE &Die, const MachineOperand &MO) {
   assert(MO.isFPImm() && "Invalid machine operand!");
-  DIEBlock *Block = new (DIEValueAllocator) DIEBlock();
+  DIEBlock *Block = new (DIEValueAllocator) DIEBlock;
   APFloat FPImm = MO.getFPImm()->getValueAPF();
 
   // Get the raw data form of the floating point.
@@ -691,20 +609,18 @@ void DwarfUnit::addConstantFPValue(DIE &Die, const MachineOperand &MO) {
   addBlock(Die, dwarf::DW_AT_const_value, Block);
 }
 
-/// addConstantFPValue - Add constant value entry in variable DIE.
 void DwarfUnit::addConstantFPValue(DIE &Die, const ConstantFP *CFP) {
   // Pass this down to addConstantValue as an unsigned bag of bits.
   addConstantValue(Die, CFP->getValueAPF().bitcastToAPInt(), true);
 }
 
-/// addConstantValue - Add constant value entry in variable DIE.
-void DwarfUnit::addConstantValue(DIE &Die, const ConstantInt *CI, DIType Ty) {
+void DwarfUnit::addConstantValue(DIE &Die, const ConstantInt *CI,
+                                 const DIType *Ty) {
   addConstantValue(Die, CI->getValue(), Ty);
 }
 
-/// addConstantValue - Add constant value entry in variable DIE.
 void DwarfUnit::addConstantValue(DIE &Die, const MachineOperand &MO,
-                                 DIType Ty) {
+                                 const DIType *Ty) {
   assert(MO.isImm() && "Invalid machine operand!");
 
   addConstantValue(Die, isUnsignedDIType(DD, Ty), MO.getImm());
@@ -717,11 +633,10 @@ void DwarfUnit::addConstantValue(DIE &Die, bool Unsigned, uint64_t Val) {
           Unsigned ? dwarf::DW_FORM_udata : dwarf::DW_FORM_sdata, Val);
 }
 
-void DwarfUnit::addConstantValue(DIE &Die, const APInt &Val, DIType Ty) {
+void DwarfUnit::addConstantValue(DIE &Die, const APInt &Val, const DIType *Ty) {
   addConstantValue(Die, Val, isUnsignedDIType(DD, Ty));
 }
 
-// addConstantValue - Add constant value entry in variable DIE.
 void DwarfUnit::addConstantValue(DIE &Die, const APInt &Val, bool Unsigned) {
   unsigned CIBitWidth = Val.getBitWidth();
   if (CIBitWidth <= 64) {
@@ -730,7 +645,7 @@ void DwarfUnit::addConstantValue(DIE &Die, const APInt &Val, bool Unsigned) {
     return;
   }
 
-  DIEBlock *Block = new (DIEValueAllocator) DIEBlock();
+  DIEBlock *Block = new (DIEValueAllocator) DIEBlock;
 
   // Get the raw data form of the large APInt.
   const uint64_t *Ptr64 = Val.getRawData();
@@ -751,7 +666,6 @@ void DwarfUnit::addConstantValue(DIE &Die, const APInt &Val, bool Unsigned) {
   addBlock(Die, dwarf::DW_AT_const_value, Block);
 }
 
-// Add a linkage name to the DIE.
 void DwarfUnit::addLinkageName(DIE &Die, StringRef LinkageName) {
   if (!LinkageName.empty())
     addString(Die,
@@ -760,33 +674,30 @@ void DwarfUnit::addLinkageName(DIE &Die, StringRef LinkageName) {
               GlobalValue::getRealLinkageName(LinkageName));
 }
 
-/// addTemplateParams - Add template parameters into buffer.
-void DwarfUnit::addTemplateParams(DIE &Buffer, DIArray TParams) {
+void DwarfUnit::addTemplateParams(DIE &Buffer, DINodeArray TParams) {
   // Add template parameters.
-  for (unsigned i = 0, e = TParams.size(); i != e; ++i) {
-    DIDescriptor Element = TParams[i];
-    if (auto *TTP = dyn_cast<MDTemplateTypeParameter>(Element))
+  for (const auto *Element : TParams) {
+    if (auto *TTP = dyn_cast<DITemplateTypeParameter>(Element))
       constructTemplateTypeParameterDIE(Buffer, TTP);
-    else if (auto *TVP = dyn_cast<MDTemplateValueParameter>(Element))
+    else if (auto *TVP = dyn_cast<DITemplateValueParameter>(Element))
       constructTemplateValueParameterDIE(Buffer, TVP);
   }
 }
 
-/// getOrCreateContextDIE - Get context owner's DIE.
-DIE *DwarfUnit::getOrCreateContextDIE(DIScope Context) {
-  if (!Context || isa<MDFile>(Context))
+DIE *DwarfUnit::getOrCreateContextDIE(const DIScope *Context) {
+  if (!Context || isa<DIFile>(Context))
     return &getUnitDie();
-  if (auto *T = dyn_cast<MDType>(Context))
+  if (auto *T = dyn_cast<DIType>(Context))
     return getOrCreateTypeDIE(T);
-  if (auto *NS = dyn_cast<MDNamespace>(Context))
+  if (auto *NS = dyn_cast<DINamespace>(Context))
     return getOrCreateNameSpace(NS);
-  if (auto *SP = dyn_cast<MDSubprogram>(Context))
+  if (auto *SP = dyn_cast<DISubprogram>(Context))
     return getOrCreateSubprogramDIE(SP);
   return getDIE(Context);
 }
 
-DIE *DwarfUnit::createTypeDIE(DICompositeType Ty) {
-  DIScope Context = resolve(Ty->getScope());
+DIE *DwarfUnit::createTypeDIE(const DICompositeType *Ty) {
+  auto *Context = resolve(Ty->getScope());
   DIE *ContextDIE = getOrCreateContextDIE(Context);
 
   if (DIE *TyDIE = getDIE(Ty))
@@ -795,30 +706,28 @@ DIE *DwarfUnit::createTypeDIE(DICompositeType Ty) {
   // Create new type.
   DIE &TyDIE = createAndAddDIE(Ty->getTag(), *ContextDIE, Ty);
 
-  constructTypeDIE(TyDIE, Ty);
+  constructTypeDIE(TyDIE, cast<DICompositeType>(Ty));
 
-  updateAcceleratorTables(Context, Ty, TyDIE);
+  if (!Ty->isExternalTypeRef())
+    updateAcceleratorTables(Context, Ty, TyDIE);
   return &TyDIE;
 }
 
-/// getOrCreateTypeDIE - Find existing DIE or create new DIE for the
-/// given DIType.
 DIE *DwarfUnit::getOrCreateTypeDIE(const MDNode *TyNode) {
   if (!TyNode)
     return nullptr;
 
-  auto *Ty = cast<MDType>(TyNode);
+  auto *Ty = cast<DIType>(TyNode);
   assert(Ty == resolve(Ty->getRef()) &&
          "type was not uniqued, possible ODR violation.");
 
   // DW_TAG_restrict_type is not supported in DWARF2
   if (Ty->getTag() == dwarf::DW_TAG_restrict_type && DD->getDwarfVersion() <= 2)
-    return getOrCreateTypeDIE(
-        resolve(DITypeRef(cast<MDDerivedType>(Ty)->getBaseType())));
+    return getOrCreateTypeDIE(resolve(cast<DIDerivedType>(Ty)->getBaseType()));
 
   // Construct the context before querying for the existence of the DIE in case
   // such construction creates the DIE.
-  DIScope Context = resolve(Ty->getScope());
+  auto *Context = resolve(Ty->getScope());
   DIE *ContextDIE = getOrCreateContextDIE(Context);
   assert(ContextDIE);
 
@@ -830,9 +739,11 @@ DIE *DwarfUnit::getOrCreateTypeDIE(const MDNode *TyNode) {
 
   updateAcceleratorTables(Context, Ty, TyDIE);
 
-  if (auto *BT = dyn_cast<MDBasicType>(Ty))
+  if (auto *BT = dyn_cast<DIBasicType>(Ty))
     constructTypeDIE(TyDIE, BT);
-  else if (DICompositeType CTy = dyn_cast<MDCompositeTypeBase>(Ty)) {
+  else if (auto *STy = dyn_cast<DISubroutineType>(Ty))
+    constructTypeDIE(TyDIE, STy);
+  else if (auto *CTy = dyn_cast<DICompositeType>(Ty)) {
     if (GenerateDwarfTypeUnits && !Ty->isForwardDecl())
       if (MDString *TypeId = CTy->getRawIdentifier()) {
         DD->addDwarfTypeUnitType(getCU(), TypeId->getString(), TyDIE, CTy);
@@ -841,17 +752,17 @@ DIE *DwarfUnit::getOrCreateTypeDIE(const MDNode *TyNode) {
       }
     constructTypeDIE(TyDIE, CTy);
   } else {
-    constructTypeDIE(TyDIE, cast<MDDerivedType>(Ty));
+    constructTypeDIE(TyDIE, cast<DIDerivedType>(Ty));
   }
 
   return &TyDIE;
 }
 
-void DwarfUnit::updateAcceleratorTables(DIScope Context, DIType Ty,
-                                        const DIE &TyDIE) {
+void DwarfUnit::updateAcceleratorTables(const DIScope *Context,
+                                        const DIType *Ty, const DIE &TyDIE) {
   if (!Ty->getName().empty() && !Ty->isForwardDecl()) {
     bool IsImplementation = 0;
-    if (auto *CT = dyn_cast<MDCompositeTypeBase>(Ty)) {
+    if (auto *CT = dyn_cast<DICompositeType>(Ty)) {
       // A runtime language of 0 actually means C/C++ and that any
       // non-negative value is some version of Objective-C/C++.
       IsImplementation = CT->getRuntimeLang() == 0 || CT->isObjcClassComplete();
@@ -859,39 +770,19 @@ void DwarfUnit::updateAcceleratorTables(DIScope Context, DIType Ty,
     unsigned Flags = IsImplementation ? dwarf::DW_FLAG_type_implementation : 0;
     DD->addAccelType(Ty->getName(), TyDIE, Flags);
 
-    if (!Context || isa<MDCompileUnit>(Context) || isa<MDFile>(Context) ||
-        isa<MDNamespace>(Context))
+    if (!Context || isa<DICompileUnit>(Context) || isa<DIFile>(Context) ||
+        isa<DINamespace>(Context))
       addGlobalType(Ty, TyDIE, Context);
   }
 }
 
-/// addType - Add a new type attribute to the specified entity.
-void DwarfUnit::addType(DIE &Entity, DIType Ty, dwarf::Attribute Attribute) {
+void DwarfUnit::addType(DIE &Entity, const DIType *Ty,
+                        dwarf::Attribute Attribute) {
   assert(Ty && "Trying to add a type that doesn't exist?");
-
-  // Check for pre-existence.
-  DIEEntry *Entry = getDIEEntry(Ty);
-  // If it exists then use the existing value.
-  if (Entry) {
-    addDIEEntry(Entity, Attribute, Entry);
-    return;
-  }
-
-  // Construct type.
-  DIE *Buffer = getOrCreateTypeDIE(Ty);
-
-  // Set up proxy.
-  Entry = createDIEEntry(*Buffer);
-  insertDIEEntry(Ty, Entry);
-  addDIEEntry(Entity, Attribute, Entry);
+  addDIEEntry(Entity, Attribute, DIEEntry(*getOrCreateTypeDIE(Ty)));
 }
 
-/// getParentContextString - Walks the metadata parent chain in a language
-/// specific manner (using the compile unit language) and returns
-/// it as a string. This is done at the metadata level because DIEs may
-/// not currently have been added to the parent context and walking the
-/// DIEs looking for names is more expensive than walking the metadata.
-std::string DwarfUnit::getParentContextString(DIScope Context) const {
+std::string DwarfUnit::getParentContextString(const DIScope *Context) const {
   if (!Context)
     return "";
 
@@ -900,8 +791,8 @@ std::string DwarfUnit::getParentContextString(DIScope Context) const {
     return "";
 
   std::string CS;
-  SmallVector<DIScope, 1> Parents;
-  while (!isa<MDCompileUnit>(Context)) {
+  SmallVector<const DIScope *, 1> Parents;
+  while (!isa<DICompileUnit>(Context)) {
     Parents.push_back(Context);
     if (Context->getScope())
       Context = resolve(Context->getScope());
@@ -913,12 +804,9 @@ std::string DwarfUnit::getParentContextString(DIScope Context) const {
 
   // Reverse iterate over our list to go from the outermost construct to the
   // innermost.
-  for (SmallVectorImpl<DIScope>::reverse_iterator I = Parents.rbegin(),
-                                                  E = Parents.rend();
-       I != E; ++I) {
-    const MDScope *Ctx = *I;
+  for (const DIScope *Ctx : make_range(Parents.rbegin(), Parents.rend())) {
     StringRef Name = Ctx->getName();
-    if (Name.empty() && isa<MDNamespace>(Ctx))
+    if (Name.empty() && isa<DINamespace>(Ctx))
       Name = "(anonymous namespace)";
     if (!Name.empty()) {
       CS += Name;
@@ -928,8 +816,7 @@ std::string DwarfUnit::getParentContextString(DIScope Context) const {
   return CS;
 }
 
-/// constructTypeDIE - Construct basic type die from DIBasicType.
-void DwarfUnit::constructTypeDIE(DIE &Buffer, DIBasicType BTy) {
+void DwarfUnit::constructTypeDIE(DIE &Buffer, const DIBasicType *BTy) {
   // Get core information.
   StringRef Name = BTy->getName();
   // Add name if not anonymous or intermediate type.
@@ -947,15 +834,14 @@ void DwarfUnit::constructTypeDIE(DIE &Buffer, DIBasicType BTy) {
   addUInt(Buffer, dwarf::DW_AT_byte_size, None, Size);
 }
 
-/// constructTypeDIE - Construct derived type die from DIDerivedType.
-void DwarfUnit::constructTypeDIE(DIE &Buffer, DIDerivedType DTy) {
+void DwarfUnit::constructTypeDIE(DIE &Buffer, const DIDerivedType *DTy) {
   // Get core information.
   StringRef Name = DTy->getName();
   uint64_t Size = DTy->getSizeInBits() >> 3;
   uint16_t Tag = Buffer.getTag();
 
   // Map to main type, void will not have a type.
-  DIType FromTy = resolve(DTy->getBaseType());
+  const DIType *FromTy = resolve(DTy->getBaseType());
   if (FromTy)
     addType(Buffer, FromTy);
 
@@ -971,16 +857,15 @@ void DwarfUnit::constructTypeDIE(DIE &Buffer, DIDerivedType DTy) {
   if (Tag == dwarf::DW_TAG_ptr_to_member_type)
     addDIEEntry(
         Buffer, dwarf::DW_AT_containing_type,
-        *getOrCreateTypeDIE(resolve(cast<MDDerivedType>(DTy)->getClassType())));
+        *getOrCreateTypeDIE(resolve(cast<DIDerivedType>(DTy)->getClassType())));
   // Add source line info if available and TyDesc is not a forward declaration.
   if (!DTy->isForwardDecl())
     addSourceLine(Buffer, DTy);
 }
 
-/// constructSubprogramArguments - Construct function argument DIEs.
-void DwarfUnit::constructSubprogramArguments(DIE &Buffer, DITypeArray Args) {
+void DwarfUnit::constructSubprogramArguments(DIE &Buffer, DITypeRefArray Args) {
   for (unsigned i = 1, N = Args.size(); i < N; ++i) {
-    DIType Ty = resolve(Args[i]);
+    const DIType *Ty = resolve(Args[i]);
     if (!Ty) {
       assert(i == N-1 && "Unspecified parameter must be the last argument");
       createAndAddDIE(dwarf::DW_TAG_unspecified_parameters, Buffer);
@@ -993,8 +878,42 @@ void DwarfUnit::constructSubprogramArguments(DIE &Buffer, DITypeArray Args) {
   }
 }
 
-/// constructTypeDIE - Construct type DIE from DICompositeType.
-void DwarfUnit::constructTypeDIE(DIE &Buffer, DICompositeType CTy) {
+void DwarfUnit::constructTypeDIE(DIE &Buffer, const DISubroutineType *CTy) {
+  // Add return type.  A void return won't have a type.
+  auto Elements = cast<DISubroutineType>(CTy)->getTypeArray();
+  if (Elements.size())
+    if (auto RTy = resolve(Elements[0]))
+      addType(Buffer, RTy);
+
+  bool isPrototyped = true;
+  if (Elements.size() == 2 && !Elements[1])
+    isPrototyped = false;
+
+  constructSubprogramArguments(Buffer, Elements);
+
+  // Add prototype flag if we're dealing with a C language and the function has
+  // been prototyped.
+  uint16_t Language = getLanguage();
+  if (isPrototyped &&
+      (Language == dwarf::DW_LANG_C89 || Language == dwarf::DW_LANG_C99 ||
+       Language == dwarf::DW_LANG_ObjC))
+    addFlag(Buffer, dwarf::DW_AT_prototyped);
+
+  if (CTy->isLValueReference())
+    addFlag(Buffer, dwarf::DW_AT_reference);
+
+  if (CTy->isRValueReference())
+    addFlag(Buffer, dwarf::DW_AT_rvalue_reference);
+}
+
+void DwarfUnit::constructTypeDIE(DIE &Buffer, const DICompositeType *CTy) {
+  if (CTy->isExternalTypeRef()) {
+    StringRef Identifier = CTy->getIdentifier();
+    assert(!Identifier.empty() && "external type ref without identifier");
+    addFlag(Buffer, dwarf::DW_AT_declaration);
+    return addDIETypeSignature(Buffer, dwarf::DW_AT_signature, Identifier);
+  }
+
   // Add name if not anonymous or intermediate type.
   StringRef Name = CTy->getName();
 
@@ -1008,45 +927,17 @@ void DwarfUnit::constructTypeDIE(DIE &Buffer, DICompositeType CTy) {
   case dwarf::DW_TAG_enumeration_type:
     constructEnumTypeDIE(Buffer, CTy);
     break;
-  case dwarf::DW_TAG_subroutine_type: {
-    // Add return type. A void return won't have a type.
-    auto Elements = cast<MDSubroutineType>(CTy)->getTypeArray();
-    if (Elements.size())
-      if (auto RTy = resolve(Elements[0]))
-        addType(Buffer, RTy);
-
-    bool isPrototyped = true;
-    if (Elements.size() == 2 && !Elements[1])
-      isPrototyped = false;
-
-    constructSubprogramArguments(Buffer, Elements);
-
-    // Add prototype flag if we're dealing with a C language and the
-    // function has been prototyped.
-    uint16_t Language = getLanguage();
-    if (isPrototyped &&
-        (Language == dwarf::DW_LANG_C89 || Language == dwarf::DW_LANG_C99 ||
-         Language == dwarf::DW_LANG_ObjC))
-      addFlag(Buffer, dwarf::DW_AT_prototyped);
-
-    if (CTy->isLValueReference())
-      addFlag(Buffer, dwarf::DW_AT_reference);
-
-    if (CTy->isRValueReference())
-      addFlag(Buffer, dwarf::DW_AT_rvalue_reference);
-  } break;
   case dwarf::DW_TAG_structure_type:
   case dwarf::DW_TAG_union_type:
   case dwarf::DW_TAG_class_type: {
     // Add elements to structure type.
-    DIArray Elements = CTy->getElements();
-    for (unsigned i = 0, N = Elements.size(); i < N; ++i) {
-      DIDescriptor Element = Elements[i];
+    DINodeArray Elements = CTy->getElements();
+    for (const auto *Element : Elements) {
       if (!Element)
         continue;
-      if (auto *SP = dyn_cast<MDSubprogram>(Element))
+      if (auto *SP = dyn_cast<DISubprogram>(Element))
         getOrCreateSubprogramDIE(SP);
-      else if (DIDerivedType DDTy = dyn_cast<MDDerivedTypeBase>(Element)) {
+      else if (auto *DDTy = dyn_cast<DIDerivedType>(Element)) {
         if (DDTy->getTag() == dwarf::DW_TAG_friend) {
           DIE &ElemDie = createAndAddDIE(dwarf::DW_TAG_friend, Buffer);
           addType(ElemDie, resolve(DDTy->getBaseType()), dwarf::DW_AT_friend);
@@ -1055,12 +946,12 @@ void DwarfUnit::constructTypeDIE(DIE &Buffer, DICompositeType CTy) {
         } else {
           constructMemberDIE(Buffer, DDTy);
         }
-      } else if (DIObjCProperty Property = dyn_cast<MDObjCProperty>(Element)) {
+      } else if (auto *Property = dyn_cast<DIObjCProperty>(Element)) {
         DIE &ElemDie = createAndAddDIE(Property->getTag(), Buffer);
         StringRef PropertyName = Property->getName();
         addString(ElemDie, dwarf::DW_AT_APPLE_property_name, PropertyName);
         if (Property->getType())
-          addType(ElemDie, Property->getType());
+          addType(ElemDie, resolve(Property->getType()));
         addSourceLine(ElemDie, Property);
         StringRef GetterName = Property->getGetterName();
         if (!GetterName.empty())
@@ -1071,12 +962,6 @@ void DwarfUnit::constructTypeDIE(DIE &Buffer, DICompositeType CTy) {
         if (unsigned PropertyAttributes = Property->getAttributes())
           addUInt(ElemDie, dwarf::DW_AT_APPLE_property_attribute, None,
                   PropertyAttributes);
-
-        DIEEntry *Entry = getDIEEntry(Element);
-        if (!Entry) {
-          Entry = createDIEEntry(ElemDie);
-          insertDIEEntry(Element, Entry);
-        }
       }
     }
 
@@ -1085,8 +970,8 @@ void DwarfUnit::constructTypeDIE(DIE &Buffer, DICompositeType CTy) {
 
     // This is outside the DWARF spec, but GDB expects a DW_AT_containing_type
     // inside C++ composite types to point to the base class with the vtable.
-    if (DICompositeType ContainingType =
-            dyn_cast_or_null<MDCompositeType>(resolve(CTy->getVTableHolder())))
+    if (auto *ContainingType =
+            dyn_cast_or_null<DICompositeType>(resolve(CTy->getVTableHolder())))
       addDIEEntry(Buffer, dwarf::DW_AT_containing_type,
                   *getOrCreateTypeDIE(ContainingType));
 
@@ -1136,10 +1021,8 @@ void DwarfUnit::constructTypeDIE(DIE &Buffer, DICompositeType CTy) {
   }
 }
 
-/// constructTemplateTypeParameterDIE - Construct new DIE for the given
-/// DITemplateTypeParameter.
-void DwarfUnit::constructTemplateTypeParameterDIE(DIE &Buffer,
-                                                  DITemplateTypeParameter TP) {
+void DwarfUnit::constructTemplateTypeParameterDIE(
+    DIE &Buffer, const DITemplateTypeParameter *TP) {
   DIE &ParamDIE =
       createAndAddDIE(dwarf::DW_TAG_template_type_parameter, Buffer);
   // Add the type if it exists, it could be void and therefore no type.
@@ -1149,11 +1032,8 @@ void DwarfUnit::constructTemplateTypeParameterDIE(DIE &Buffer,
     addString(ParamDIE, dwarf::DW_AT_name, TP->getName());
 }
 
-/// constructTemplateValueParameterDIE - Construct new DIE for the given
-/// DITemplateValueParameter.
-void
-DwarfUnit::constructTemplateValueParameterDIE(DIE &Buffer,
-                                              DITemplateValueParameter VP) {
+void DwarfUnit::constructTemplateValueParameterDIE(
+    DIE &Buffer, const DITemplateValueParameter *VP) {
   DIE &ParamDIE = createAndAddDIE(VP->getTag(), Buffer);
 
   // Add the type if there is one, template template and template parameter
@@ -1168,7 +1048,7 @@ DwarfUnit::constructTemplateValueParameterDIE(DIE &Buffer,
     else if (GlobalValue *GV = mdconst::dyn_extract<GlobalValue>(Val)) {
       // For declaration non-type template parameters (such as global values and
       // functions)
-      DIELoc *Loc = new (DIEValueAllocator) DIELoc();
+      DIELoc *Loc = new (DIEValueAllocator) DIELoc;
       addOpAddress(*Loc, Asm->getSymbol(GV));
       // Emit DW_OP_stack_value to use the address as the immediate value of the
       // parameter, rather than a pointer to it.
@@ -1184,8 +1064,7 @@ DwarfUnit::constructTemplateValueParameterDIE(DIE &Buffer,
   }
 }
 
-/// getOrCreateNameSpace - Create a DIE for DINameSpace.
-DIE *DwarfUnit::getOrCreateNameSpace(DINameSpace NS) {
+DIE *DwarfUnit::getOrCreateNameSpace(const DINamespace *NS) {
   // Construct the context before querying for the existence of the DIE in case
   // such construction creates the DIE.
   DIE *ContextDIE = getOrCreateContextDIE(NS->getScope());
@@ -1205,8 +1084,31 @@ DIE *DwarfUnit::getOrCreateNameSpace(DINameSpace NS) {
   return &NDie;
 }
 
-/// getOrCreateSubprogramDIE - Create new DIE using SP.
-DIE *DwarfUnit::getOrCreateSubprogramDIE(DISubprogram SP, bool Minimal) {
+DIE *DwarfUnit::getOrCreateModule(const DIModule *M) {
+  // Construct the context before querying for the existence of the DIE in case
+  // such construction creates the DIE.
+  DIE *ContextDIE = getOrCreateContextDIE(M->getScope());
+
+  if (DIE *MDie = getDIE(M))
+    return MDie;
+  DIE &MDie = createAndAddDIE(dwarf::DW_TAG_module, *ContextDIE, M);
+
+  if (!M->getName().empty()) {
+    addString(MDie, dwarf::DW_AT_name, M->getName());
+    addGlobalName(M->getName(), MDie, M->getScope());
+  }
+  if (!M->getConfigurationMacros().empty())
+    addString(MDie, dwarf::DW_AT_LLVM_config_macros,
+              M->getConfigurationMacros());
+  if (!M->getIncludePath().empty())
+    addString(MDie, dwarf::DW_AT_LLVM_include_path, M->getIncludePath());
+  if (!M->getISysRoot().empty())
+    addString(MDie, dwarf::DW_AT_LLVM_isysroot, M->getISysRoot());
+  
+  return &MDie;
+}
+
+DIE *DwarfUnit::getOrCreateSubprogramDIE(const DISubprogram *SP, bool Minimal) {
   // Construct the context before querying for the existence of the DIE in case
   // such construction creates the DIE (as is the case for member function
   // declarations).
@@ -1237,7 +1139,7 @@ DIE *DwarfUnit::getOrCreateSubprogramDIE(DISubprogram SP, bool Minimal) {
   return &SPDie;
 }
 
-bool DwarfUnit::applySubprogramDefinitionAttributes(DISubprogram SP,
+bool DwarfUnit::applySubprogramDefinitionAttributes(const DISubprogram *SP,
                                                     DIE &SPDie) {
   DIE *DeclDie = nullptr;
   StringRef DeclLinkageName;
@@ -1269,7 +1171,7 @@ bool DwarfUnit::applySubprogramDefinitionAttributes(DISubprogram SP,
   return true;
 }
 
-void DwarfUnit::applySubprogramAttributes(DISubprogram SP, DIE &SPDie,
+void DwarfUnit::applySubprogramAttributes(const DISubprogram *SP, DIE &SPDie,
                                           bool Minimal) {
   if (!Minimal)
     if (applySubprogramDefinitionAttributes(SP, SPDie))
@@ -1293,7 +1195,7 @@ void DwarfUnit::applySubprogramAttributes(DISubprogram SP, DIE &SPDie,
        Language == dwarf::DW_LANG_ObjC))
     addFlag(SPDie, dwarf::DW_AT_prototyped);
 
-  DISubroutineType SPTy = SP->getType();
+  const DISubroutineType *SPTy = SP->getType();
   assert(SPTy->getTag() == dwarf::DW_TAG_subroutine_type &&
          "the type of a subprogram should be a subroutine");
 
@@ -1355,8 +1257,8 @@ void DwarfUnit::applySubprogramAttributes(DISubprogram SP, DIE &SPDie,
     addFlag(SPDie, dwarf::DW_AT_explicit);
 }
 
-/// constructSubrangeDIE - Construct subrange DIE from DISubrange.
-void DwarfUnit::constructSubrangeDIE(DIE &Buffer, DISubrange SR, DIE *IndexTy) {
+void DwarfUnit::constructSubrangeDIE(DIE &Buffer, const DISubrange *SR,
+                                     DIE *IndexTy) {
   DIE &DW_Subrange = createAndAddDIE(dwarf::DW_TAG_subrange_type, Buffer);
   addDIEEntry(DW_Subrange, dwarf::DW_AT_type, *IndexTy);
 
@@ -1389,8 +1291,7 @@ DIE *DwarfUnit::getIndexTyDie() {
   return IndexTyDie;
 }
 
-/// constructArrayTypeDIE - Construct array type DIE from DICompositeType.
-void DwarfUnit::constructArrayTypeDIE(DIE &Buffer, DICompositeType CTy) {
+void DwarfUnit::constructArrayTypeDIE(DIE &Buffer, const DICompositeType *CTy) {
   if (CTy->isVector())
     addFlag(Buffer, dwarf::DW_AT_GNU_vector);
 
@@ -1403,22 +1304,21 @@ void DwarfUnit::constructArrayTypeDIE(DIE &Buffer, DICompositeType CTy) {
   DIE *IdxTy = getIndexTyDie();
 
   // Add subranges to array type.
-  DIArray Elements = CTy->getElements();
+  DINodeArray Elements = CTy->getElements();
   for (unsigned i = 0, N = Elements.size(); i < N; ++i) {
     // FIXME: Should this really be such a loose cast?
-    if (auto *Element = dyn_cast_or_null<DebugNode>(Elements[i]))
+    if (auto *Element = dyn_cast_or_null<DINode>(Elements[i]))
       if (Element->getTag() == dwarf::DW_TAG_subrange_type)
-        constructSubrangeDIE(Buffer, cast<MDSubrange>(Element), IdxTy);
+        constructSubrangeDIE(Buffer, cast<DISubrange>(Element), IdxTy);
   }
 }
 
-/// constructEnumTypeDIE - Construct an enum type DIE from DICompositeType.
-void DwarfUnit::constructEnumTypeDIE(DIE &Buffer, DICompositeType CTy) {
-  DIArray Elements = CTy->getElements();
+void DwarfUnit::constructEnumTypeDIE(DIE &Buffer, const DICompositeType *CTy) {
+  DINodeArray Elements = CTy->getElements();
 
   // Add enumerators to enumeration type.
   for (unsigned i = 0, N = Elements.size(); i < N; ++i) {
-    auto *Enum = dyn_cast_or_null<MDEnumerator>(Elements[i]);
+    auto *Enum = dyn_cast_or_null<DIEnumerator>(Elements[i]);
     if (Enum) {
       DIE &Enumerator = createAndAddDIE(dwarf::DW_TAG_enumerator, Buffer);
       StringRef Name = Enum->getName();
@@ -1428,21 +1328,18 @@ void DwarfUnit::constructEnumTypeDIE(DIE &Buffer, DICompositeType CTy) {
               Value);
     }
   }
-  DIType DTy = resolve(CTy->getBaseType());
+  const DIType *DTy = resolve(CTy->getBaseType());
   if (DTy) {
     addType(Buffer, DTy);
     addFlag(Buffer, dwarf::DW_AT_enum_class);
   }
 }
 
-/// constructContainingTypeDIEs - Construct DIEs for types that contain
-/// vtables.
 void DwarfUnit::constructContainingTypeDIEs() {
-  for (DenseMap<DIE *, const MDNode *>::iterator CI = ContainingTypeMap.begin(),
-                                                 CE = ContainingTypeMap.end();
+  for (auto CI = ContainingTypeMap.begin(), CE = ContainingTypeMap.end();
        CI != CE; ++CI) {
     DIE &SPDie = *CI->first;
-    DIDescriptor D(CI->second);
+    const DINode *D = CI->second;
     if (!D)
       continue;
     DIE *NDie = getDIE(D);
@@ -1452,11 +1349,7 @@ void DwarfUnit::constructContainingTypeDIEs() {
   }
 }
 
-/// constructMemberDIE - Construct member DIE from DIDerivedType.
-void DwarfUnit::constructMemberDIE(DIE &Buffer, DIDerivedType DT_) {
-  // Downcast to MDDerivedType.
-  const MDDerivedType *DT = cast<MDDerivedType>(DT_);
-
+void DwarfUnit::constructMemberDIE(DIE &Buffer, const DIDerivedType *DT) {
   DIE &MemberDie = createAndAddDIE(DT->getTag(), Buffer);
   StringRef Name = DT->getName();
   if (!Name.empty())
@@ -1472,7 +1365,7 @@ void DwarfUnit::constructMemberDIE(DIE &Buffer, DIDerivedType DT_) {
     // expression to extract appropriate offset from vtable.
     // BaseAddr = ObAddr + *((*ObAddr) - Offset)
 
-    DIELoc *VBaseLocationDie = new (DIEValueAllocator) DIELoc();
+    DIELoc *VBaseLocationDie = new (DIEValueAllocator) DIELoc;
     addUInt(*VBaseLocationDie, dwarf::DW_FORM_data1, dwarf::DW_OP_dup);
     addUInt(*VBaseLocationDie, dwarf::DW_FORM_data1, dwarf::DW_OP_deref);
     addUInt(*VBaseLocationDie, dwarf::DW_FORM_data1, dwarf::DW_OP_constu);
@@ -1491,27 +1384,47 @@ void DwarfUnit::constructMemberDIE(DIE &Buffer, DIDerivedType DT_) {
       // Handle bitfield, assume bytes are 8 bits.
       addUInt(MemberDie, dwarf::DW_AT_byte_size, None, FieldSize/8);
       addUInt(MemberDie, dwarf::DW_AT_bit_size, None, Size);
-
+      //
+      // The DWARF 2 DW_AT_bit_offset is counting the bits between the most
+      // significant bit of the aligned storage unit containing the bit field to
+      // the most significan bit of the bit field.
+      //
+      // FIXME: DWARF 4 states that DW_AT_data_bit_offset (which
+      // counts from the beginning, regardless of endianness) should
+      // be used instead.
+      //
+      //
+      // Struct      Align       Align       Align
+      // v           v           v           v
+      // +-----------+-----*-----+-----*-----+--
+      // | ...             |b1|b2|b3|b4|
+      // +-----------+-----*-----+-----*-----+--
+      // |           |     |<-- Size ->|     |
+      // |<---- Offset --->|           |<--->|
+      // |           |     |              \_ DW_AT_bit_offset (little endian)
+      // |           |<--->|
+      // |<--------->|  \_ StartBitOffset = DW_AT_bit_offset (big endian)
+      //     \                            = DW_AT_data_bit_offset (biendian)
+      //      \_ OffsetInBytes
       uint64_t Offset = DT->getOffsetInBits();
-      uint64_t AlignMask = ~(DT->getAlignInBits() - 1);
-      uint64_t HiMark = (Offset + FieldSize) & AlignMask;
-      uint64_t FieldOffset = (HiMark - FieldSize);
-      Offset -= FieldOffset;
-
-      // Maybe we need to work from the other end.
-      if (Asm->getDataLayout().isLittleEndian())
-        Offset = FieldSize - (Offset + Size);
-      addUInt(MemberDie, dwarf::DW_AT_bit_offset, None, Offset);
-
-      // Here DW_AT_data_member_location points to the anonymous
-      // field that includes this bit field.
-      OffsetInBytes = FieldOffset >> 3;
+      uint64_t Align = DT->getAlignInBits() ? DT->getAlignInBits() : FieldSize;
+      uint64_t AlignMask = ~(Align - 1);
+      // The bits from the start of the storage unit to the start of the field.
+      uint64_t StartBitOffset = Offset - (Offset & AlignMask);
+      // The endian-dependent DWARF 2 offset.
+      uint64_t DwarfBitOffset = Asm->getDataLayout().isLittleEndian()
+        ? OffsetToAlignment(Offset + Size, Align)
+        : StartBitOffset;
+
+      // The byte offset of the field's aligned storage unit inside the struct.
+      OffsetInBytes = (Offset - StartBitOffset) / 8;
+      addUInt(MemberDie, dwarf::DW_AT_bit_offset, None, DwarfBitOffset);
     } else
       // This is not a bitfield.
-      OffsetInBytes = DT->getOffsetInBits() >> 3;
+      OffsetInBytes = DT->getOffsetInBits() / 8;
 
     if (DD->getDwarfVersion() <= 2) {
-      DIELoc *MemLocationDie = new (DIEValueAllocator) DIELoc();
+      DIELoc *MemLocationDie = new (DIEValueAllocator) DIELoc;
       addUInt(*MemLocationDie, dwarf::DW_FORM_data1, dwarf::DW_OP_plus_uconst);
       addUInt(*MemLocationDie, dwarf::DW_FORM_udata, OffsetInBytes);
       addBlock(MemberDie, dwarf::DW_AT_data_member_location, MemLocationDie);
@@ -1535,18 +1448,16 @@ void DwarfUnit::constructMemberDIE(DIE &Buffer, DIDerivedType DT_) {
             dwarf::DW_VIRTUALITY_virtual);
 
   // Objective-C properties.
-  if (MDNode *PNode = DT->getObjCProperty())
-    if (DIEEntry *PropertyDie = getDIEEntry(PNode))
-      MemberDie.addValue(dwarf::DW_AT_APPLE_property, dwarf::DW_FORM_ref4,
-                         PropertyDie);
+  if (DINode *PNode = DT->getObjCProperty())
+    if (DIE *PDie = getDIE(PNode))
+      MemberDie.addValue(DIEValueAllocator, dwarf::DW_AT_APPLE_property,
+                         dwarf::DW_FORM_ref4, DIEEntry(*PDie));
 
   if (DT->isArtificial())
     addFlag(MemberDie, dwarf::DW_AT_artificial);
 }
 
-/// getOrCreateStaticMemberDIE - Create new DIE for C++ static member.
-DIE *DwarfUnit::getOrCreateStaticMemberDIE(DIDerivedType DT_) {
-  const MDDerivedType *DT = cast_or_null<MDDerivedType>(DT_);
+DIE *DwarfUnit::getOrCreateStaticMemberDIE(const DIDerivedType *DT) {
   if (!DT)
     return nullptr;
 
@@ -1561,7 +1472,7 @@ DIE *DwarfUnit::getOrCreateStaticMemberDIE(DIDerivedType DT_) {
 
   DIE &StaticMemberDIE = createAndAddDIE(DT->getTag(), *ContextDIE, DT);
 
-  DIType Ty = resolve(DT->getBaseType());
+  const DIType *Ty = resolve(DT->getBaseType());
 
   addString(StaticMemberDIE, dwarf::DW_AT_name, DT->getName());
   addType(StaticMemberDIE, Ty);
@@ -1591,39 +1502,37 @@ DIE *DwarfUnit::getOrCreateStaticMemberDIE(DIDerivedType DT_) {
 
 void DwarfUnit::emitHeader(bool UseOffsets) {
   // Emit size of content not including length itself
-  Asm->OutStreamer.AddComment("Length of Unit");
+  Asm->OutStreamer->AddComment("Length of Unit");
   Asm->EmitInt32(getHeaderSize() + UnitDie.getSize());
 
-  Asm->OutStreamer.AddComment("DWARF version number");
+  Asm->OutStreamer->AddComment("DWARF version number");
   Asm->EmitInt16(DD->getDwarfVersion());
-  Asm->OutStreamer.AddComment("Offset Into Abbrev. Section");
+  Asm->OutStreamer->AddComment("Offset Into Abbrev. Section");
 
   // We share one abbreviations table across all units so it's always at the
   // start of the section. Use a relocatable offset where needed to ensure
   // linking doesn't invalidate that offset.
   const TargetLoweringObjectFile &TLOF = Asm->getObjFileLowering();
-  if (!UseOffsets)
-    Asm->emitSectionOffset(TLOF.getDwarfAbbrevSection()->getBeginSymbol());
-  else
-    Asm->EmitInt32(0);
+  Asm->emitDwarfSymbolReference(TLOF.getDwarfAbbrevSection()->getBeginSymbol(),
+                                UseOffsets);
 
-  Asm->OutStreamer.AddComment("Address Size (in bytes)");
+  Asm->OutStreamer->AddComment("Address Size (in bytes)");
   Asm->EmitInt8(Asm->getDataLayout().getPointerSize());
 }
 
-void DwarfUnit::initSection(const MCSection *Section) {
+void DwarfUnit::initSection(MCSection *Section) {
   assert(!this->Section);
   this->Section = Section;
 }
 
 void DwarfTypeUnit::emitHeader(bool UseOffsets) {
   DwarfUnit::emitHeader(UseOffsets);
-  Asm->OutStreamer.AddComment("Type Signature");
-  Asm->OutStreamer.EmitIntValue(TypeSignature, sizeof(TypeSignature));
-  Asm->OutStreamer.AddComment("Type DIE Offset");
+  Asm->OutStreamer->AddComment("Type Signature");
+  Asm->OutStreamer->EmitIntValue(TypeSignature, sizeof(TypeSignature));
+  Asm->OutStreamer->AddComment("Type DIE Offset");
   // In a skeleton type unit there is no type DIE so emit a zero offset.
-  Asm->OutStreamer.EmitIntValue(Ty ? Ty->getOffset() : 0,
-                                sizeof(Ty->getOffset()));
+  Asm->OutStreamer->EmitIntValue(Ty ? Ty->getOffset() : 0,
+                                 sizeof(Ty->getOffset()));
 }
 
 bool DwarfTypeUnit::isDwoUnit() const {
diff --git a/lib/CodeGen/AsmPrinter/DwarfUnit.h b/lib/CodeGen/AsmPrinter/DwarfUnit.h
index b354255d709..28fb71cc462 100644
--- a/lib/CodeGen/AsmPrinter/DwarfUnit.h
+++ b/lib/CodeGen/AsmPrinter/DwarfUnit.h
@@ -63,77 +63,71 @@ class RangeSpanList {
 };
 
 //===----------------------------------------------------------------------===//
-/// Unit - This dwarf writer support class manages information associated
-/// with a source file.
+/// This dwarf writer support class manages information associated with a
+/// source file.
 class DwarfUnit {
 protected:
-  /// UniqueID - a numeric ID unique among all CUs in the module
+  /// A numeric ID unique among all CUs in the module
   unsigned UniqueID;
 
-  /// Node - MDNode for the compile unit.
-  DICompileUnit CUNode;
+  /// MDNode for the compile unit.
+  const DICompileUnit *CUNode;
+
+  // All DIEValues are allocated through this allocator.
+  BumpPtrAllocator DIEValueAllocator;
 
   /// Unit debug information entry.
-  DIE UnitDie;
+  DIE &UnitDie;
 
   /// Offset of the UnitDie from beginning of debug info section.
   unsigned DebugInfoOffset;
 
-  /// Asm - Target of Dwarf emission.
+  /// Target of Dwarf emission.
   AsmPrinter *Asm;
 
   // Holders for some common dwarf information.
   DwarfDebug *DD;
   DwarfFile *DU;
 
-  /// IndexTyDie - An anonymous type for index type.  Owned by UnitDie.
+  /// An anonymous type for index type.  Owned by UnitDie.
   DIE *IndexTyDie;
 
-  /// MDNodeToDieMap - Tracks the mapping of unit level debug information
-  /// variables to debug information entries.
+  /// Tracks the mapping of unit level debug information variables to debug
+  /// information entries.
   DenseMap<const MDNode *, DIE *> MDNodeToDieMap;
 
-  /// MDNodeToDIEEntryMap - Tracks the mapping of unit level debug information
-  /// descriptors to debug information entries using a DIEEntry proxy.
-  DenseMap<const MDNode *, DIEEntry *> MDNodeToDIEEntryMap;
-
-  /// DIEBlocks - A list of all the DIEBlocks in use.
+  /// A list of all the DIEBlocks in use.
   std::vector<DIEBlock *> DIEBlocks;
-  
-  /// DIELocs - A list of all the DIELocs in use.
+
+  /// A list of all the DIELocs in use.
   std::vector<DIELoc *> DIELocs;
 
-  /// ContainingTypeMap - This map is used to keep track of subprogram DIEs that
-  /// need DW_AT_containing_type attribute. This attribute points to a DIE that
+  /// This map is used to keep track of subprogram DIEs that need
+  /// DW_AT_containing_type attribute. This attribute points to a DIE that
   /// corresponds to the MDNode mapped with the subprogram DIE.
-  DenseMap<DIE *, const MDNode *> ContainingTypeMap;
-
-  // DIEValueAllocator - All DIEValues are allocated through this allocator.
-  BumpPtrAllocator DIEValueAllocator;
-
-  // DIEIntegerOne - A preallocated DIEValue because 1 is used frequently.
-  DIEInteger *DIEIntegerOne;
+  DenseMap<DIE *, const DINode *> ContainingTypeMap;
 
   /// The section this unit will be emitted in.
-  const MCSection *Section;
+  MCSection *Section;
 
-  DwarfUnit(unsigned UID, dwarf::Tag, DICompileUnit CU, AsmPrinter *A,
+  DwarfUnit(unsigned UID, dwarf::Tag, const DICompileUnit *CU, AsmPrinter *A,
             DwarfDebug *DW, DwarfFile *DWU);
 
-
   /// Add a string attribute data and value.
+  ///
+  /// This is guaranteed to be in the local string pool instead of indirected.
   void addLocalString(DIE &Die, dwarf::Attribute Attribute, StringRef Str);
 
   void addIndexedString(DIE &Die, dwarf::Attribute Attribute, StringRef Str);
 
-  bool applySubprogramDefinitionAttributes(DISubprogram SP, DIE &SPDie);
+  bool applySubprogramDefinitionAttributes(const DISubprogram *SP, DIE &SPDie);
 
 public:
   virtual ~DwarfUnit();
 
-  void initSection(const MCSection *Section);
+  void initSection(MCSection *Section);
 
-  const MCSection *getSection() const {
+  MCSection *getSection() const {
     assert(Section);
     return Section;
   }
@@ -142,179 +136,189 @@ class DwarfUnit {
   AsmPrinter* getAsmPrinter() const { return Asm; }
   unsigned getUniqueID() const { return UniqueID; }
   uint16_t getLanguage() const { return CUNode->getSourceLanguage(); }
-  DICompileUnit getCUNode() const { return CUNode; }
+  const DICompileUnit *getCUNode() const { return CUNode; }
   DIE &getUnitDie() { return UnitDie; }
 
   unsigned getDebugInfoOffset() const { return DebugInfoOffset; }
   void setDebugInfoOffset(unsigned DbgInfoOff) { DebugInfoOffset = DbgInfoOff; }
 
-  /// hasContent - Return true if this compile unit has something to write out.
-  bool hasContent() const { return !UnitDie.getChildren().empty(); }
+  /// Return true if this compile unit has something to write out.
+  bool hasContent() const { return UnitDie.hasChildren(); }
 
-  /// getParentContextString - Get a string containing the language specific
-  /// context for a global name.
-  std::string getParentContextString(DIScope Context) const;
+  /// Get string containing language specific context for a global name.
+  ///
+  /// Walks the metadata parent chain in a language specific manner (using the
+  /// compile unit language) and returns it as a string. This is done at the
+  /// metadata level because DIEs may not currently have been added to the
+  /// parent context and walking the DIEs looking for names is more expensive
+  /// than walking the metadata.
+  std::string getParentContextString(const DIScope *Context) const;
 
   /// Add a new global name to the compile unit.
-  virtual void addGlobalName(StringRef Name, DIE &Die, DIScope Context) {}
+  virtual void addGlobalName(StringRef Name, DIE &Die, const DIScope *Context) {
+  }
 
   /// Add a new global type to the compile unit.
-  virtual void addGlobalType(DIType Ty, const DIE &Die, DIScope Context) {}
+  virtual void addGlobalType(const DIType *Ty, const DIE &Die,
+                             const DIScope *Context) {}
 
-  /// addAccelNamespace - Add a new name to the namespace accelerator table.
+  /// Add a new name to the namespace accelerator table.
   void addAccelNamespace(StringRef Name, const DIE &Die);
 
-  /// getDIE - Returns the debug information entry map slot for the
-  /// specified debug variable. We delegate the request to DwarfDebug
-  /// when the MDNode can be part of the type system, since DIEs for
-  /// the type system can be shared across CUs and the mappings are
-  /// kept in DwarfDebug.
-  DIE *getDIE(DIDescriptor D) const;
+  /// Returns the DIE map slot for the specified debug variable.
+  ///
+  /// We delegate the request to DwarfDebug when the MDNode can be part of the
+  /// type system, since DIEs for the type system can be shared across CUs and
+  /// the mappings are kept in DwarfDebug.
+  DIE *getDIE(const DINode *D) const;
 
-  /// getDIELoc - Returns a fresh newly allocated DIELoc.
-  DIELoc *getDIELoc() { return new (DIEValueAllocator) DIELoc(); }
+  /// Returns a fresh newly allocated DIELoc.
+  DIELoc *getDIELoc() { return new (DIEValueAllocator) DIELoc; }
 
-  /// insertDIE - Insert DIE into the map. We delegate the request to DwarfDebug
-  /// when the MDNode can be part of the type system, since DIEs for
-  /// the type system can be shared across CUs and the mappings are
-  /// kept in DwarfDebug.
-  void insertDIE(DIDescriptor Desc, DIE *D);
+  /// Insert DIE into the map.
+  ///
+  /// We delegate the request to DwarfDebug when the MDNode can be part of the
+  /// type system, since DIEs for the type system can be shared across CUs and
+  /// the mappings are kept in DwarfDebug.
+  void insertDIE(const DINode *Desc, DIE *D);
 
-  /// addFlag - Add a flag that is true to the DIE.
+  /// Add a flag that is true to the DIE.
   void addFlag(DIE &Die, dwarf::Attribute Attribute);
 
-  /// addUInt - Add an unsigned integer attribute data and value.
+  /// Add an unsigned integer attribute data and value.
   void addUInt(DIE &Die, dwarf::Attribute Attribute, Optional<dwarf::Form> Form,
                uint64_t Integer);
 
   void addUInt(DIE &Block, dwarf::Form Form, uint64_t Integer);
 
-  /// addSInt - Add an signed integer attribute data and value.
+  /// Add an signed integer attribute data and value.
   void addSInt(DIE &Die, dwarf::Attribute Attribute, Optional<dwarf::Form> Form,
                int64_t Integer);
 
   void addSInt(DIELoc &Die, Optional<dwarf::Form> Form, int64_t Integer);
 
-  /// addString - Add a string attribute data and value.
+  /// Add a string attribute data and value.
+  ///
+  /// We always emit a reference to the string pool instead of immediate
+  /// strings so that DIEs have more predictable sizes. In the case of split
+  /// dwarf we emit an index into another table which gets us the static offset
+  /// into the string table.
   void addString(DIE &Die, dwarf::Attribute Attribute, StringRef Str);
 
-  /// addLabel - Add a Dwarf label attribute data and value.
-  void addLabel(DIE &Die, dwarf::Attribute Attribute, dwarf::Form Form,
-                const MCSymbol *Label);
+  /// Add a Dwarf label attribute data and value.
+  DIE::value_iterator addLabel(DIE &Die, dwarf::Attribute Attribute,
+                               dwarf::Form Form, const MCSymbol *Label);
 
   void addLabel(DIELoc &Die, dwarf::Form Form, const MCSymbol *Label);
 
-  /// addSectionOffset - Add an offset into a section attribute data and value.
-  ///
+  /// Add an offset into a section attribute data and value.
   void addSectionOffset(DIE &Die, dwarf::Attribute Attribute, uint64_t Integer);
 
-  /// addOpAddress - Add a dwarf op address data and value using the
-  /// form given and an op of either DW_FORM_addr or DW_FORM_GNU_addr_index.
+  /// Add a dwarf op address data and value using the form given and an
+  /// op of either DW_FORM_addr or DW_FORM_GNU_addr_index.
   void addOpAddress(DIELoc &Die, const MCSymbol *Label);
 
-  /// addLabelDelta - Add a label delta attribute data and value.
+  /// Add a label delta attribute data and value.
   void addLabelDelta(DIE &Die, dwarf::Attribute Attribute, const MCSymbol *Hi,
                      const MCSymbol *Lo);
 
-  /// addDIEEntry - Add a DIE attribute data and value.
+  /// Add a DIE attribute data and value.
   void addDIEEntry(DIE &Die, dwarf::Attribute Attribute, DIE &Entry);
 
-  /// addDIEEntry - Add a DIE attribute data and value.
-  void addDIEEntry(DIE &Die, dwarf::Attribute Attribute, DIEEntry *Entry);
+  /// Add a DIE attribute data and value.
+  void addDIEEntry(DIE &Die, dwarf::Attribute Attribute, DIEEntry Entry);
 
+  /// Add a type's DW_AT_signature and set the  declaration flag.
   void addDIETypeSignature(DIE &Die, const DwarfTypeUnit &Type);
+  /// Add an attribute containing the type signature for a unique identifier.
+  void addDIETypeSignature(DIE &Die, dwarf::Attribute Attribute,
+                           StringRef Identifier);
 
-  /// addBlock - Add block data.
+  /// Add block data.
   void addBlock(DIE &Die, dwarf::Attribute Attribute, DIELoc *Block);
 
-  /// addBlock - Add block data.
+  /// Add block data.
   void addBlock(DIE &Die, dwarf::Attribute Attribute, DIEBlock *Block);
 
-  /// addSourceLine - Add location information to specified debug information
-  /// entry.
+  /// Add location information to specified debug information entry.
   void addSourceLine(DIE &Die, unsigned Line, StringRef File,
                      StringRef Directory);
-  void addSourceLine(DIE &Die, DIVariable V);
-  void addSourceLine(DIE &Die, DIGlobalVariable G);
-  void addSourceLine(DIE &Die, DISubprogram SP);
-  void addSourceLine(DIE &Die, DIType Ty);
-  void addSourceLine(DIE &Die, DINameSpace NS);
-  void addSourceLine(DIE &Die, DIObjCProperty Ty);
-
-  /// addConstantValue - Add constant value entry in variable DIE.
-  void addConstantValue(DIE &Die, const MachineOperand &MO, DIType Ty);
-  void addConstantValue(DIE &Die, const ConstantInt *CI, DIType Ty);
-  void addConstantValue(DIE &Die, const APInt &Val, DIType Ty);
+  void addSourceLine(DIE &Die, const DILocalVariable *V);
+  void addSourceLine(DIE &Die, const DIGlobalVariable *G);
+  void addSourceLine(DIE &Die, const DISubprogram *SP);
+  void addSourceLine(DIE &Die, const DIType *Ty);
+  void addSourceLine(DIE &Die, const DINamespace *NS);
+  void addSourceLine(DIE &Die, const DIObjCProperty *Ty);
+
+  /// Add constant value entry in variable DIE.
+  void addConstantValue(DIE &Die, const MachineOperand &MO, const DIType *Ty);
+  void addConstantValue(DIE &Die, const ConstantInt *CI, const DIType *Ty);
+  void addConstantValue(DIE &Die, const APInt &Val, const DIType *Ty);
   void addConstantValue(DIE &Die, const APInt &Val, bool Unsigned);
   void addConstantValue(DIE &Die, bool Unsigned, uint64_t Val);
 
-  /// addConstantFPValue - Add constant value entry in variable DIE.
+  /// Add constant value entry in variable DIE.
   void addConstantFPValue(DIE &Die, const MachineOperand &MO);
   void addConstantFPValue(DIE &Die, const ConstantFP *CFP);
 
-  /// \brief Add a linkage name, if it isn't empty.
+  /// Add a linkage name, if it isn't empty.
   void addLinkageName(DIE &Die, StringRef LinkageName);
 
-  /// addTemplateParams - Add template parameters in buffer.
-  void addTemplateParams(DIE &Buffer, DIArray TParams);
+  /// Add template parameters in buffer.
+  void addTemplateParams(DIE &Buffer, DINodeArray TParams);
 
-  /// \brief Add register operand.
+  /// Add register operand.
   /// \returns false if the register does not exist, e.g., because it was never
   /// materialized.
   bool addRegisterOpPiece(DIELoc &TheDie, unsigned Reg,
                           unsigned SizeInBits = 0, unsigned OffsetInBits = 0);
 
-  /// \brief Add register offset.
+  /// Add register offset.
   /// \returns false if the register does not exist, e.g., because it was never
   /// materialized.
   bool addRegisterOffset(DIELoc &TheDie, unsigned Reg, int64_t Offset);
 
   // FIXME: Should be reformulated in terms of addComplexAddress.
-  /// addBlockByrefAddress - Start with the address based on the location
-  /// provided, and generate the DWARF information necessary to find the
-  /// actual Block variable (navigating the Block struct) based on the
-  /// starting location.  Add the DWARF information to the die.  Obsolete,
-  /// please use addComplexAddress instead.
+  /// Start with the address based on the location provided, and generate the
+  /// DWARF information necessary to find the actual Block variable (navigating
+  /// the Block struct) based on the starting location.  Add the DWARF
+  /// information to the die.  Obsolete, please use addComplexAddress instead.
   void addBlockByrefAddress(const DbgVariable &DV, DIE &Die,
                             dwarf::Attribute Attribute,
                             const MachineLocation &Location);
 
-  /// addType - Add a new type attribute to the specified entity. This takes
-  /// and attribute parameter because DW_AT_friend attributes are also
-  /// type references.
-  void addType(DIE &Entity, DIType Ty,
+  /// Add a new type attribute to the specified entity.
+  ///
+  /// This takes and attribute parameter because DW_AT_friend attributes are
+  /// also type references.
+  void addType(DIE &Entity, const DIType *Ty,
                dwarf::Attribute Attribute = dwarf::DW_AT_type);
 
-  /// getOrCreateNameSpace - Create a DIE for DINameSpace.
-  DIE *getOrCreateNameSpace(DINameSpace NS);
-
-  /// getOrCreateSubprogramDIE - Create new DIE using SP.
-  DIE *getOrCreateSubprogramDIE(DISubprogram SP, bool Minimal = false);
+  DIE *getOrCreateNameSpace(const DINamespace *NS);
+  DIE *getOrCreateModule(const DIModule *M);
+  DIE *getOrCreateSubprogramDIE(const DISubprogram *SP, bool Minimal = false);
 
-  void applySubprogramAttributes(DISubprogram SP, DIE &SPDie,
+  void applySubprogramAttributes(const DISubprogram *SP, DIE &SPDie,
                                  bool Minimal = false);
 
-  /// getOrCreateTypeDIE - Find existing DIE or create new DIE for the
-  /// given DIType.
+  /// Find existing DIE or create new DIE for the given type.
   DIE *getOrCreateTypeDIE(const MDNode *N);
 
-  /// getOrCreateContextDIE - Get context owner's DIE.
-  DIE *createTypeDIE(DICompositeType Ty);
+  /// Get context owner's DIE.
+  DIE *createTypeDIE(const DICompositeType *Ty);
 
-  /// getOrCreateContextDIE - Get context owner's DIE.
-  DIE *getOrCreateContextDIE(DIScope Context);
+  /// Get context owner's DIE.
+  DIE *getOrCreateContextDIE(const DIScope *Context);
 
-  /// constructContainingTypeDIEs - Construct DIEs for types that contain
-  /// vtables.
+  /// Construct DIEs for types that contain vtables.
   void constructContainingTypeDIEs();
 
-  /// constructSubprogramArguments - Construct function argument DIEs.
-  void constructSubprogramArguments(DIE &Buffer, DITypeArray Args);
+  /// Construct function argument DIEs.
+  void constructSubprogramArguments(DIE &Buffer, DITypeRefArray Args);
 
   /// Create a DIE with the given Tag, add the DIE to its parent, and
   /// call insertDIE if MD is not null.
-  DIE &createAndAddDIE(unsigned Tag, DIE &Parent,
-                       DIDescriptor N = DIDescriptor());
+  DIE &createAndAddDIE(unsigned Tag, DIE &Parent, const DINode *N = nullptr);
 
   /// Compute the size of a header for this unit, not including the initial
   /// length field.
@@ -329,81 +333,50 @@ class DwarfUnit {
 
   virtual DwarfCompileUnit &getCU() = 0;
 
-  /// constructTypeDIE - Construct type DIE from DICompositeType.
-  void constructTypeDIE(DIE &Buffer, DICompositeType CTy);
+  void constructTypeDIE(DIE &Buffer, const DICompositeType *CTy);
 
 protected:
-  /// getOrCreateStaticMemberDIE - Create new static data member DIE.
-  DIE *getOrCreateStaticMemberDIE(DIDerivedType DT);
+  /// Create new static data member DIE.
+  DIE *getOrCreateStaticMemberDIE(const DIDerivedType *DT);
 
   /// Look up the source ID with the given directory and source file names. If
   /// none currently exists, create a new ID and insert it in the line table.
   virtual unsigned getOrCreateSourceID(StringRef File, StringRef Directory) = 0;
 
-  /// resolve - Look in the DwarfDebug map for the MDNode that
-  /// corresponds to the reference.
-  template <typename T> T *resolve(TypedDebugNodeRef<T> Ref) const {
+  /// Look in the DwarfDebug map for the MDNode that corresponds to the
+  /// reference.
+  template <typename T> T *resolve(TypedDINodeRef<T> Ref) const {
     return DD->resolve(Ref);
   }
 
 private:
-  /// constructTypeDIE - Construct basic type die from DIBasicType.
-  void constructTypeDIE(DIE &Buffer, DIBasicType BTy);
-
-  /// constructTypeDIE - Construct derived type die from DIDerivedType.
-  void constructTypeDIE(DIE &Buffer, DIDerivedType DTy);
-
-  /// constructSubrangeDIE - Construct subrange DIE from DISubrange.
-  void constructSubrangeDIE(DIE &Buffer, DISubrange SR, DIE *IndexTy);
-
-  /// constructArrayTypeDIE - Construct array type DIE from DICompositeType.
-  void constructArrayTypeDIE(DIE &Buffer, DICompositeType CTy);
-
-  /// constructEnumTypeDIE - Construct enum type DIE from DIEnumerator.
-  void constructEnumTypeDIE(DIE &Buffer, DICompositeType CTy);
-
-  /// constructMemberDIE - Construct member DIE from DIDerivedType.
-  void constructMemberDIE(DIE &Buffer, DIDerivedType DT);
-
-  /// constructTemplateTypeParameterDIE - Construct new DIE for the given
-  /// DITemplateTypeParameter.
+  void constructTypeDIE(DIE &Buffer, const DIBasicType *BTy);
+  void constructTypeDIE(DIE &Buffer, const DIDerivedType *DTy);
+  void constructTypeDIE(DIE &Buffer, const DISubroutineType *DTy);
+  void constructSubrangeDIE(DIE &Buffer, const DISubrange *SR, DIE *IndexTy);
+  void constructArrayTypeDIE(DIE &Buffer, const DICompositeType *CTy);
+  void constructEnumTypeDIE(DIE &Buffer, const DICompositeType *CTy);
+  void constructMemberDIE(DIE &Buffer, const DIDerivedType *DT);
   void constructTemplateTypeParameterDIE(DIE &Buffer,
-                                         DITemplateTypeParameter TP);
-
-  /// constructTemplateValueParameterDIE - Construct new DIE for the given
-  /// DITemplateValueParameter.
+                                         const DITemplateTypeParameter *TP);
   void constructTemplateValueParameterDIE(DIE &Buffer,
-                                          DITemplateValueParameter TVP);
+                                          const DITemplateValueParameter *TVP);
 
-  /// getLowerBoundDefault - Return the default lower bound for an array. If the
-  /// DWARF version doesn't handle the language, return -1.
+  /// Return the default lower bound for an array.
+  ///
+  /// If the DWARF version doesn't handle the language, return -1.
   int64_t getDefaultLowerBound() const;
 
-  /// getDIEEntry - Returns the debug information entry for the specified
-  /// debug variable.
-  DIEEntry *getDIEEntry(const MDNode *N) const {
-    return MDNodeToDIEEntryMap.lookup(N);
-  }
-
-  /// insertDIEEntry - Insert debug information entry into the map.
-  void insertDIEEntry(const MDNode *N, DIEEntry *E) {
-    MDNodeToDIEEntryMap.insert(std::make_pair(N, E));
-  }
-
-  // getIndexTyDie - Get an anonymous type for index type.
+  /// Get an anonymous type for index type.
   DIE *getIndexTyDie();
 
-  // setIndexTyDie - Set D as anonymous type for index which can be reused
-  // later.
+  /// Set D as anonymous type for index which can be reused later.
   void setIndexTyDie(DIE *D) { IndexTyDie = D; }
 
-  /// createDIEEntry - Creates a new DIEEntry to be a proxy for a debug
-  /// information entry.
-  DIEEntry *createDIEEntry(DIE &Entry);
-
   /// If this is a named finished type then include it in the list of types for
   /// the accelerator tables.
-  void updateAcceleratorTables(DIScope Context, DIType Ty, const DIE &TyDIE);
+  void updateAcceleratorTables(const DIScope *Context, const DIType *Ty,
+                               const DIE &TyDIE);
 
   virtual bool isDwoUnit() const = 0;
 };
diff --git a/lib/CodeGen/AsmPrinter/EHStreamer.cpp b/lib/CodeGen/AsmPrinter/EHStreamer.cpp
index 6f64d8f790a..e24dcb1bffd 100644
--- a/lib/CodeGen/AsmPrinter/EHStreamer.cpp
+++ b/lib/CodeGen/AsmPrinter/EHStreamer.cpp
@@ -309,7 +309,7 @@ computeCallSiteTable(SmallVectorImpl<CallSiteEntry> &CallSites,
   // If some instruction between the previous try-range and the end of the
   // function may throw, create a call-site entry with no landing pad for the
   // region following the try-range.
-  if (SawPotentiallyThrowing && !IsSJLJ) {
+  if (SawPotentiallyThrowing && !IsSJLJ && LastLabel != nullptr) {
     CallSiteEntry Site = { LastLabel, nullptr, nullptr, 0 };
     CallSites.push_back(Site);
   }
@@ -388,7 +388,7 @@ void EHStreamer::emitExceptionTable() {
   }
 
   // Type infos.
-  const MCSection *LSDASection = Asm->getObjFileLowering().getLSDASection();
+  MCSection *LSDASection = Asm->getObjFileLowering().getLSDASection();
   unsigned TTypeEncoding;
   unsigned TypeFormatSize;
 
@@ -434,15 +434,15 @@ void EHStreamer::emitExceptionTable() {
   // Sometimes we want not to emit the data into separate section (e.g. ARM
   // EHABI). In this case LSDASection will be NULL.
   if (LSDASection)
-    Asm->OutStreamer.SwitchSection(LSDASection);
+    Asm->OutStreamer->SwitchSection(LSDASection);
   Asm->EmitAlignment(2);
 
   // Emit the LSDA.
   MCSymbol *GCCETSym =
-    Asm->OutContext.GetOrCreateSymbol(Twine("GCC_except_table")+
+    Asm->OutContext.getOrCreateSymbol(Twine("GCC_except_table")+
                                       Twine(Asm->getFunctionNumber()));
-  Asm->OutStreamer.EmitLabel(GCCETSym);
-  Asm->OutStreamer.EmitLabel(Asm->getCurExceptionSym());
+  Asm->OutStreamer->EmitLabel(GCCETSym);
+  Asm->OutStreamer->EmitLabel(Asm->getCurExceptionSym());
 
   // Emit the LSDA header.
   Asm->EmitEncodingByte(dwarf::DW_EH_PE_omit, "@LPStart");
@@ -486,7 +486,7 @@ void EHStreamer::emitExceptionTable() {
     SizeAlign = 0;
   }
 
-  bool VerboseAsm = Asm->OutStreamer.isVerboseAsm();
+  bool VerboseAsm = Asm->OutStreamer->isVerboseAsm();
 
   // SjLj Exception handling
   if (IsSJLJ) {
@@ -504,8 +504,8 @@ void EHStreamer::emitExceptionTable() {
       // Offset of the landing pad, counted in 16-byte bundles relative to the
       // @LPStart address.
       if (VerboseAsm) {
-        Asm->OutStreamer.AddComment(">> Call Site " + Twine(idx) + " <<");
-        Asm->OutStreamer.AddComment("  On exception at call site "+Twine(idx));
+        Asm->OutStreamer->AddComment(">> Call Site " + Twine(idx) + " <<");
+        Asm->OutStreamer->AddComment("  On exception at call site "+Twine(idx));
       }
       Asm->EmitULEB128(idx);
 
@@ -514,10 +514,10 @@ void EHStreamer::emitExceptionTable() {
       // the action table), and 0 indicates that there are no actions.
       if (VerboseAsm) {
         if (S.Action == 0)
-          Asm->OutStreamer.AddComment("  Action: cleanup");
+          Asm->OutStreamer->AddComment("  Action: cleanup");
         else
-          Asm->OutStreamer.AddComment("  Action: " +
-                                      Twine((S.Action - 1) / 2 + 1));
+          Asm->OutStreamer->AddComment("  Action: " +
+                                       Twine((S.Action - 1) / 2 + 1));
       }
       Asm->EmitULEB128(S.Action);
     }
@@ -566,24 +566,24 @@ void EHStreamer::emitExceptionTable() {
       // number of 16-byte bundles. The first call site is counted relative to
       // the start of the procedure fragment.
       if (VerboseAsm)
-        Asm->OutStreamer.AddComment(">> Call Site " + Twine(++Entry) + " <<");
+        Asm->OutStreamer->AddComment(">> Call Site " + Twine(++Entry) + " <<");
       Asm->EmitLabelDifference(BeginLabel, EHFuncBeginSym, 4);
       if (VerboseAsm)
-        Asm->OutStreamer.AddComment(Twine("  Call between ") +
-                                    BeginLabel->getName() + " and " +
-                                    EndLabel->getName());
+        Asm->OutStreamer->AddComment(Twine("  Call between ") +
+                                     BeginLabel->getName() + " and " +
+                                     EndLabel->getName());
       Asm->EmitLabelDifference(EndLabel, BeginLabel, 4);
 
       // Offset of the landing pad, counted in 16-byte bundles relative to the
       // @LPStart address.
       if (!S.LPad) {
         if (VerboseAsm)
-          Asm->OutStreamer.AddComment("    has no landing pad");
-        Asm->OutStreamer.EmitIntValue(0, 4/*size*/);
+          Asm->OutStreamer->AddComment("    has no landing pad");
+        Asm->OutStreamer->EmitIntValue(0, 4/*size*/);
       } else {
         if (VerboseAsm)
-          Asm->OutStreamer.AddComment(Twine("    jumps to ") +
-                                      S.LPad->LandingPadLabel->getName());
+          Asm->OutStreamer->AddComment(Twine("    jumps to ") +
+                                       S.LPad->LandingPadLabel->getName());
         Asm->EmitLabelDifference(S.LPad->LandingPadLabel, EHFuncBeginSym, 4);
       }
 
@@ -592,10 +592,10 @@ void EHStreamer::emitExceptionTable() {
       // the action table), and 0 indicates that there are no actions.
       if (VerboseAsm) {
         if (S.Action == 0)
-          Asm->OutStreamer.AddComment("  On action: cleanup");
+          Asm->OutStreamer->AddComment("  On action: cleanup");
         else
-          Asm->OutStreamer.AddComment("  On action: " +
-                                      Twine((S.Action - 1) / 2 + 1));
+          Asm->OutStreamer->AddComment("  On action: " +
+                                       Twine((S.Action - 1) / 2 + 1));
       }
       Asm->EmitULEB128(S.Action);
     }
@@ -609,7 +609,7 @@ void EHStreamer::emitExceptionTable() {
 
     if (VerboseAsm) {
       // Emit comments that decode the action table.
-      Asm->OutStreamer.AddComment(">> Action Record " + Twine(++Entry) + " <<");
+      Asm->OutStreamer->AddComment(">> Action Record " + Twine(++Entry) + " <<");
     }
 
     // Type Filter
@@ -618,13 +618,13 @@ void EHStreamer::emitExceptionTable() {
     //   type of the catch clauses or the types in the exception specification.
     if (VerboseAsm) {
       if (Action.ValueForTypeID > 0)
-        Asm->OutStreamer.AddComment("  Catch TypeInfo " +
-                                    Twine(Action.ValueForTypeID));
+        Asm->OutStreamer->AddComment("  Catch TypeInfo " +
+                                     Twine(Action.ValueForTypeID));
       else if (Action.ValueForTypeID < 0)
-        Asm->OutStreamer.AddComment("  Filter TypeInfo " +
-                                    Twine(Action.ValueForTypeID));
+        Asm->OutStreamer->AddComment("  Filter TypeInfo " +
+                                     Twine(Action.ValueForTypeID));
       else
-        Asm->OutStreamer.AddComment("  Cleanup");
+        Asm->OutStreamer->AddComment("  Cleanup");
     }
     Asm->EmitSLEB128(Action.ValueForTypeID);
 
@@ -634,10 +634,10 @@ void EHStreamer::emitExceptionTable() {
     //   or 0 if there is no next action record.
     if (VerboseAsm) {
       if (Action.NextAction == 0) {
-        Asm->OutStreamer.AddComment("  No further actions");
+        Asm->OutStreamer->AddComment("  No further actions");
       } else {
         unsigned NextAction = Entry + (Action.NextAction + 1) / 2;
-        Asm->OutStreamer.AddComment("  Continue to action "+Twine(NextAction));
+        Asm->OutStreamer->AddComment("  Continue to action "+Twine(NextAction));
       }
     }
     Asm->EmitSLEB128(Action.NextAction);
@@ -652,28 +652,27 @@ void EHStreamer::emitTypeInfos(unsigned TTypeEncoding) {
   const std::vector<const GlobalValue *> &TypeInfos = MMI->getTypeInfos();
   const std::vector<unsigned> &FilterIds = MMI->getFilterIds();
 
-  bool VerboseAsm = Asm->OutStreamer.isVerboseAsm();
+  bool VerboseAsm = Asm->OutStreamer->isVerboseAsm();
 
   int Entry = 0;
   // Emit the Catch TypeInfos.
   if (VerboseAsm && !TypeInfos.empty()) {
-    Asm->OutStreamer.AddComment(">> Catch TypeInfos <<");
-    Asm->OutStreamer.AddBlankLine();
+    Asm->OutStreamer->AddComment(">> Catch TypeInfos <<");
+    Asm->OutStreamer->AddBlankLine();
     Entry = TypeInfos.size();
   }
 
-  for (std::vector<const GlobalValue *>::const_reverse_iterator
-         I = TypeInfos.rbegin(), E = TypeInfos.rend(); I != E; ++I) {
-    const GlobalValue *GV = *I;
+  for (const GlobalValue *GV : make_range(TypeInfos.rbegin(),
+                                          TypeInfos.rend())) {
     if (VerboseAsm)
-      Asm->OutStreamer.AddComment("TypeInfo " + Twine(Entry--));
+      Asm->OutStreamer->AddComment("TypeInfo " + Twine(Entry--));
     Asm->EmitTTypeReference(GV, TTypeEncoding);
   }
 
   // Emit the Exception Specifications.
   if (VerboseAsm && !FilterIds.empty()) {
-    Asm->OutStreamer.AddComment(">> Filter TypeInfos <<");
-    Asm->OutStreamer.AddBlankLine();
+    Asm->OutStreamer->AddComment(">> Filter TypeInfos <<");
+    Asm->OutStreamer->AddBlankLine();
     Entry = 0;
   }
   for (std::vector<unsigned>::const_iterator
@@ -682,7 +681,7 @@ void EHStreamer::emitTypeInfos(unsigned TTypeEncoding) {
     if (VerboseAsm) {
       --Entry;
       if (isFilterEHSelector(TypeID))
-        Asm->OutStreamer.AddComment("FilterInfo " + Twine(Entry));
+        Asm->OutStreamer->AddComment("FilterInfo " + Twine(Entry));
     }
 
     Asm->EmitULEB128(TypeID);
diff --git a/lib/CodeGen/AsmPrinter/EHStreamer.h b/lib/CodeGen/AsmPrinter/EHStreamer.h
index 65973fab6b2..e42e082acbf 100644
--- a/lib/CodeGen/AsmPrinter/EHStreamer.h
+++ b/lib/CodeGen/AsmPrinter/EHStreamer.h
@@ -30,7 +30,7 @@ template <typename T>
 class SmallVectorImpl;
 
 /// Emits exception handling directives.
-class EHStreamer : public AsmPrinterHandler {
+class LLVM_LIBRARY_VISIBILITY EHStreamer : public AsmPrinterHandler {
 protected:
   /// Target of directive emission.
   AsmPrinter *Asm;
diff --git a/lib/CodeGen/AsmPrinter/ErlangGCPrinter.cpp b/lib/CodeGen/AsmPrinter/ErlangGCPrinter.cpp
index 97a3234c28f..6a023b998b3 100644
--- a/lib/CodeGen/AsmPrinter/ErlangGCPrinter.cpp
+++ b/lib/CodeGen/AsmPrinter/ErlangGCPrinter.cpp
@@ -47,11 +47,11 @@ void llvm::linkErlangGCPrinter() {}
 
 void ErlangGCPrinter::finishAssembly(Module &M, GCModuleInfo &Info,
                                      AsmPrinter &AP) {
-  MCStreamer &OS = AP.OutStreamer;
-  unsigned IntPtrSize = AP.TM.getDataLayout()->getPointerSize();
+  MCStreamer &OS = *AP.OutStreamer;
+  unsigned IntPtrSize = M.getDataLayout().getPointerSize();
 
   // Put this in a custom .note section.
-  AP.OutStreamer.SwitchSection(
+  OS.SwitchSection(
       AP.getObjFileLowering().getContext().getELFSection(".note.gc",
                                                          ELF::SHT_PROGBITS, 0));
 
diff --git a/lib/CodeGen/AsmPrinter/OcamlGCPrinter.cpp b/lib/CodeGen/AsmPrinter/OcamlGCPrinter.cpp
index 76d6a065815..c09ef6adea6 100644
--- a/lib/CodeGen/AsmPrinter/OcamlGCPrinter.cpp
+++ b/lib/CodeGen/AsmPrinter/OcamlGCPrinter.cpp
@@ -58,20 +58,20 @@ static void EmitCamlGlobal(const Module &M, AsmPrinter &AP, const char *Id) {
   SymName[Letter] = toupper(SymName[Letter]);
 
   SmallString<128> TmpStr;
-  AP.Mang->getNameWithPrefix(TmpStr, SymName);
+  Mangler::getNameWithPrefix(TmpStr, SymName, M.getDataLayout());
 
-  MCSymbol *Sym = AP.OutContext.GetOrCreateSymbol(TmpStr);
+  MCSymbol *Sym = AP.OutContext.getOrCreateSymbol(TmpStr);
 
-  AP.OutStreamer.EmitSymbolAttribute(Sym, MCSA_Global);
-  AP.OutStreamer.EmitLabel(Sym);
+  AP.OutStreamer->EmitSymbolAttribute(Sym, MCSA_Global);
+  AP.OutStreamer->EmitLabel(Sym);
 }
 
 void OcamlGCMetadataPrinter::beginAssembly(Module &M, GCModuleInfo &Info,
                                            AsmPrinter &AP) {
-  AP.OutStreamer.SwitchSection(AP.getObjFileLowering().getTextSection());
+  AP.OutStreamer->SwitchSection(AP.getObjFileLowering().getTextSection());
   EmitCamlGlobal(M, AP, "code_begin");
 
-  AP.OutStreamer.SwitchSection(AP.getObjFileLowering().getDataSection());
+  AP.OutStreamer->SwitchSection(AP.getObjFileLowering().getDataSection());
   EmitCamlGlobal(M, AP, "data_begin");
 }
 
@@ -93,18 +93,18 @@ void OcamlGCMetadataPrinter::beginAssembly(Module &M, GCModuleInfo &Info,
 ///
 void OcamlGCMetadataPrinter::finishAssembly(Module &M, GCModuleInfo &Info,
                                             AsmPrinter &AP) {
-  unsigned IntPtrSize = AP.TM.getDataLayout()->getPointerSize();
+  unsigned IntPtrSize = M.getDataLayout().getPointerSize();
 
-  AP.OutStreamer.SwitchSection(AP.getObjFileLowering().getTextSection());
+  AP.OutStreamer->SwitchSection(AP.getObjFileLowering().getTextSection());
   EmitCamlGlobal(M, AP, "code_end");
 
-  AP.OutStreamer.SwitchSection(AP.getObjFileLowering().getDataSection());
+  AP.OutStreamer->SwitchSection(AP.getObjFileLowering().getDataSection());
   EmitCamlGlobal(M, AP, "data_end");
 
   // FIXME: Why does ocaml emit this??
-  AP.OutStreamer.EmitIntValue(0, IntPtrSize);
+  AP.OutStreamer->EmitIntValue(0, IntPtrSize);
 
-  AP.OutStreamer.SwitchSection(AP.getObjFileLowering().getDataSection());
+  AP.OutStreamer->SwitchSection(AP.getObjFileLowering().getDataSection());
   EmitCamlGlobal(M, AP, "frametable");
 
   int NumDescriptors = 0;
@@ -146,9 +146,9 @@ void OcamlGCMetadataPrinter::finishAssembly(Module &M, GCModuleInfo &Info,
                          Twine(uintptr_t(&FI)) + ")");
     }
 
-    AP.OutStreamer.AddComment("live roots for " +
-                              Twine(FI.getFunction().getName()));
-    AP.OutStreamer.AddBlankLine();
+    AP.OutStreamer->AddComment("live roots for " +
+                               Twine(FI.getFunction().getName()));
+    AP.OutStreamer->AddBlankLine();
 
     for (GCFunctionInfo::iterator J = FI.begin(), JE = FI.end(); J != JE; ++J) {
       size_t LiveCount = FI.live_size(J);
@@ -160,7 +160,7 @@ void OcamlGCMetadataPrinter::finishAssembly(Module &M, GCModuleInfo &Info,
                            Twine(LiveCount) + " >= 65536.");
       }
 
-      AP.OutStreamer.EmitSymbolValue(J->Label, IntPtrSize);
+      AP.OutStreamer->EmitSymbolValue(J->Label, IntPtrSize);
       AP.EmitInt16(FrameSize);
       AP.EmitInt16(LiveCount);
 
diff --git a/lib/CodeGen/AsmPrinter/Win64Exception.cpp b/lib/CodeGen/AsmPrinter/Win64Exception.cpp
deleted file mode 100644
index f89d36455f4..00000000000
--- a/lib/CodeGen/AsmPrinter/Win64Exception.cpp
+++ /dev/null
@@ -1,503 +0,0 @@
-//===-- CodeGen/AsmPrinter/Win64Exception.cpp - Dwarf Exception Impl ------===//
-//
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
-//
-//===----------------------------------------------------------------------===//
-//
-// This file contains support for writing Win64 exception info into asm files.
-//
-//===----------------------------------------------------------------------===//
-
-#include "Win64Exception.h"
-#include "llvm/ADT/SmallString.h"
-#include "llvm/ADT/StringExtras.h"
-#include "llvm/ADT/Twine.h"
-#include "llvm/CodeGen/AsmPrinter.h"
-#include "llvm/CodeGen/MachineFrameInfo.h"
-#include "llvm/CodeGen/MachineFunction.h"
-#include "llvm/CodeGen/MachineModuleInfo.h"
-#include "llvm/CodeGen/WinEHFuncInfo.h"
-#include "llvm/IR/DataLayout.h"
-#include "llvm/IR/Mangler.h"
-#include "llvm/IR/Module.h"
-#include "llvm/MC/MCAsmInfo.h"
-#include "llvm/MC/MCContext.h"
-#include "llvm/MC/MCExpr.h"
-#include "llvm/MC/MCSection.h"
-#include "llvm/MC/MCStreamer.h"
-#include "llvm/MC/MCSymbol.h"
-#include "llvm/Support/Dwarf.h"
-#include "llvm/Support/ErrorHandling.h"
-#include "llvm/Support/FormattedStream.h"
-#include "llvm/Target/TargetFrameLowering.h"
-#include "llvm/Target/TargetLoweringObjectFile.h"
-#include "llvm/Target/TargetOptions.h"
-#include "llvm/Target/TargetRegisterInfo.h"
-using namespace llvm;
-
-Win64Exception::Win64Exception(AsmPrinter *A)
-  : EHStreamer(A), shouldEmitPersonality(false), shouldEmitLSDA(false),
-    shouldEmitMoves(false) {}
-
-Win64Exception::~Win64Exception() {}
-
-/// endModule - Emit all exception information that should come after the
-/// content.
-void Win64Exception::endModule() {
-}
-
-void Win64Exception::beginFunction(const MachineFunction *MF) {
-  shouldEmitMoves = shouldEmitPersonality = shouldEmitLSDA = false;
-
-  // If any landing pads survive, we need an EH table.
-  bool hasLandingPads = !MMI->getLandingPads().empty();
-
-  shouldEmitMoves = Asm->needsSEHMoves();
-
-  const TargetLoweringObjectFile &TLOF = Asm->getObjFileLowering();
-  unsigned PerEncoding = TLOF.getPersonalityEncoding();
-  const Function *Per = MF->getMMI().getPersonality();
-
-  shouldEmitPersonality = hasLandingPads &&
-    PerEncoding != dwarf::DW_EH_PE_omit && Per;
-
-  unsigned LSDAEncoding = TLOF.getLSDAEncoding();
-  shouldEmitLSDA = shouldEmitPersonality &&
-    LSDAEncoding != dwarf::DW_EH_PE_omit;
-
-
-  // If this was an outlined handler, we need to define the label corresponding
-  // to the offset of the parent frame relative to the stack pointer after the
-  // prologue.
-  const Function *F = MF->getFunction();
-  const Function *ParentF = MMI->getWinEHParent(F);
-  if (F != ParentF) {
-    WinEHFuncInfo &FuncInfo = MMI->getWinEHFuncInfo(ParentF);
-    auto I = FuncInfo.CatchHandlerParentFrameObjOffset.find(F);
-    if (I != FuncInfo.CatchHandlerParentFrameObjOffset.end()) {
-      MCSymbol *HandlerTypeParentFrameOffset =
-          Asm->OutContext.getOrCreateParentFrameOffsetSymbol(
-              GlobalValue::getRealLinkageName(F->getName()));
-
-      // Emit a symbol assignment.
-      Asm->OutStreamer.EmitAssignment(
-          HandlerTypeParentFrameOffset,
-          MCConstantExpr::Create(I->second, Asm->OutContext));
-    }
-  }
-
-  if (!shouldEmitPersonality && !shouldEmitMoves)
-    return;
-
-  Asm->OutStreamer.EmitWinCFIStartProc(Asm->CurrentFnSym);
-
-  if (!shouldEmitPersonality)
-    return;
-
-  const MCSymbol *PersHandlerSym =
-      TLOF.getCFIPersonalitySymbol(Per, *Asm->Mang, Asm->TM, MMI);
-  Asm->OutStreamer.EmitWinEHHandler(PersHandlerSym, true, true);
-}
-
-/// endFunction - Gather and emit post-function exception information.
-///
-void Win64Exception::endFunction(const MachineFunction *MF) {
-  if (!shouldEmitPersonality && !shouldEmitMoves)
-    return;
-
-  EHPersonality Per = MMI->getPersonalityType();
-
-  // Get rid of any dead landing pads if we're not using a Windows EH scheme. In
-  // Windows EH schemes, the landing pad is not actually reachable. It only
-  // exists so that we can emit the right table data.
-  if (!isMSVCEHPersonality(Per))
-    MMI->TidyLandingPads();
-
-  if (shouldEmitPersonality) {
-    Asm->OutStreamer.PushSection();
-
-    // Emit an UNWIND_INFO struct describing the prologue.
-    Asm->OutStreamer.EmitWinEHHandlerData();
-
-    // Emit the tables appropriate to the personality function in use. If we
-    // don't recognize the personality, assume it uses an Itanium-style LSDA.
-    if (Per == EHPersonality::MSVC_Win64SEH)
-      emitCSpecificHandlerTable();
-    else if (Per == EHPersonality::MSVC_CXX)
-      emitCXXFrameHandler3Table(MF);
-    else
-      emitExceptionTable();
-
-    Asm->OutStreamer.PopSection();
-  }
-  Asm->OutStreamer.EmitWinCFIEndProc();
-}
-
-const MCExpr *Win64Exception::createImageRel32(const MCSymbol *Value) {
-  if (!Value)
-    return MCConstantExpr::Create(0, Asm->OutContext);
-  return MCSymbolRefExpr::Create(Value, MCSymbolRefExpr::VK_COFF_IMGREL32,
-                                 Asm->OutContext);
-}
-
-const MCExpr *Win64Exception::createImageRel32(const GlobalValue *GV) {
-  if (!GV)
-    return MCConstantExpr::Create(0, Asm->OutContext);
-  return createImageRel32(Asm->getSymbol(GV));
-}
-
-/// Emit the language-specific data that __C_specific_handler expects.  This
-/// handler lives in the x64 Microsoft C runtime and allows catching or cleaning
-/// up after faults with __try, __except, and __finally.  The typeinfo values
-/// are not really RTTI data, but pointers to filter functions that return an
-/// integer (1, 0, or -1) indicating how to handle the exception. For __finally
-/// blocks and other cleanups, the landing pad label is zero, and the filter
-/// function is actually a cleanup handler with the same prototype.  A catch-all
-/// entry is modeled with a null filter function field and a non-zero landing
-/// pad label.
-///
-/// Possible filter function return values:
-///   EXCEPTION_EXECUTE_HANDLER (1):
-///     Jump to the landing pad label after cleanups.
-///   EXCEPTION_CONTINUE_SEARCH (0):
-///     Continue searching this table or continue unwinding.
-///   EXCEPTION_CONTINUE_EXECUTION (-1):
-///     Resume execution at the trapping PC.
-///
-/// Inferred table structure:
-///   struct Table {
-///     int NumEntries;
-///     struct Entry {
-///       imagerel32 LabelStart;
-///       imagerel32 LabelEnd;
-///       imagerel32 FilterOrFinally;  // One means catch-all.
-///       imagerel32 LabelLPad;        // Zero means __finally.
-///     } Entries[NumEntries];
-///   };
-void Win64Exception::emitCSpecificHandlerTable() {
-  const std::vector<LandingPadInfo> &PadInfos = MMI->getLandingPads();
-
-  // Simplifying assumptions for first implementation:
-  // - Cleanups are not implemented.
-  // - Filters are not implemented.
-
-  // The Itanium LSDA table sorts similar landing pads together to simplify the
-  // actions table, but we don't need that.
-  SmallVector<const LandingPadInfo *, 64> LandingPads;
-  LandingPads.reserve(PadInfos.size());
-  for (const auto &LP : PadInfos)
-    LandingPads.push_back(&LP);
-
-  // Compute label ranges for call sites as we would for the Itanium LSDA, but
-  // use an all zero action table because we aren't using these actions.
-  SmallVector<unsigned, 64> FirstActions;
-  FirstActions.resize(LandingPads.size());
-  SmallVector<CallSiteEntry, 64> CallSites;
-  computeCallSiteTable(CallSites, LandingPads, FirstActions);
-
-  MCSymbol *EHFuncBeginSym = Asm->getFunctionBegin();
-  MCSymbol *EHFuncEndSym = Asm->getFunctionEnd();
-
-  // Emit the number of table entries.
-  unsigned NumEntries = 0;
-  for (const CallSiteEntry &CSE : CallSites) {
-    if (!CSE.LPad)
-      continue; // Ignore gaps.
-    for (int Selector : CSE.LPad->TypeIds) {
-      // Ignore C++ filter clauses in SEH.
-      // FIXME: Implement cleanup clauses.
-      if (isCatchEHSelector(Selector))
-        ++NumEntries;
-    }
-  }
-  Asm->OutStreamer.EmitIntValue(NumEntries, 4);
-
-  // Emit the four-label records for each call site entry. The table has to be
-  // sorted in layout order, and the call sites should already be sorted.
-  for (const CallSiteEntry &CSE : CallSites) {
-    // Ignore gaps. Unlike the Itanium model, unwinding through a frame without
-    // an EH table entry will propagate the exception rather than terminating
-    // the program.
-    if (!CSE.LPad)
-      continue;
-    const LandingPadInfo *LPad = CSE.LPad;
-
-    // Compute the label range. We may reuse the function begin and end labels
-    // rather than forming new ones.
-    const MCExpr *Begin =
-        createImageRel32(CSE.BeginLabel ? CSE.BeginLabel : EHFuncBeginSym);
-    const MCExpr *End;
-    if (CSE.EndLabel) {
-      // The interval is half-open, so we have to add one to include the return
-      // address of the last invoke in the range.
-      End = MCBinaryExpr::CreateAdd(createImageRel32(CSE.EndLabel),
-                                    MCConstantExpr::Create(1, Asm->OutContext),
-                                    Asm->OutContext);
-    } else {
-      End = createImageRel32(EHFuncEndSym);
-    }
-
-    // These aren't really type info globals, they are actually pointers to
-    // filter functions ordered by selector. The zero selector is used for
-    // cleanups, so slot zero corresponds to selector 1.
-    const std::vector<const GlobalValue *> &SelectorToFilter = MMI->getTypeInfos();
-
-    // Do a parallel iteration across typeids and clause labels, skipping filter
-    // clauses.
-    size_t NextClauseLabel = 0;
-    for (size_t I = 0, E = LPad->TypeIds.size(); I < E; ++I) {
-      // AddLandingPadInfo stores the clauses in reverse, but there is a FIXME
-      // to change that.
-      int Selector = LPad->TypeIds[E - I - 1];
-
-      // Ignore C++ filter clauses in SEH.
-      // FIXME: Implement cleanup clauses.
-      if (!isCatchEHSelector(Selector))
-        continue;
-
-      Asm->OutStreamer.EmitValue(Begin, 4);
-      Asm->OutStreamer.EmitValue(End, 4);
-      if (isCatchEHSelector(Selector)) {
-        assert(unsigned(Selector - 1) < SelectorToFilter.size());
-        const GlobalValue *TI = SelectorToFilter[Selector - 1];
-        if (TI) // Emit the filter function pointer.
-          Asm->OutStreamer.EmitValue(createImageRel32(Asm->getSymbol(TI)), 4);
-        else  // Otherwise, this is a "catch i8* null", or catch all.
-          Asm->OutStreamer.EmitIntValue(1, 4);
-      }
-      MCSymbol *ClauseLabel = LPad->ClauseLabels[NextClauseLabel++];
-      Asm->OutStreamer.EmitValue(createImageRel32(ClauseLabel), 4);
-    }
-  }
-}
-
-void Win64Exception::emitCXXFrameHandler3Table(const MachineFunction *MF) {
-  const Function *F = MF->getFunction();
-  const Function *ParentF = MMI->getWinEHParent(F);
-  auto &OS = Asm->OutStreamer;
-  WinEHFuncInfo &FuncInfo = MMI->getWinEHFuncInfo(ParentF);
-
-  StringRef ParentLinkageName =
-      GlobalValue::getRealLinkageName(ParentF->getName());
-
-  MCSymbol *FuncInfoXData =
-      Asm->OutContext.GetOrCreateSymbol(Twine("$cppxdata$", ParentLinkageName));
-  OS.EmitValue(createImageRel32(FuncInfoXData), 4);
-
-  // The Itanium LSDA table sorts similar landing pads together to simplify the
-  // actions table, but we don't need that.
-  SmallVector<const LandingPadInfo *, 64> LandingPads;
-  const std::vector<LandingPadInfo> &PadInfos = MMI->getLandingPads();
-  LandingPads.reserve(PadInfos.size());
-  for (const auto &LP : PadInfos)
-    LandingPads.push_back(&LP);
-
-  RangeMapType PadMap;
-  computePadMap(LandingPads, PadMap);
-
-  // The end label of the previous invoke or nounwind try-range.
-  MCSymbol *LastLabel = Asm->getFunctionBegin();
-
-  // Whether there is a potentially throwing instruction (currently this means
-  // an ordinary call) between the end of the previous try-range and now.
-  bool SawPotentiallyThrowing = false;
-
-  int LastEHState = -2;
-
-  // The parent function and the catch handlers contribute to the 'ip2state'
-  // table.
-  for (const auto &MBB : *MF) {
-    for (const auto &MI : MBB) {
-      if (!MI.isEHLabel()) {
-        if (MI.isCall())
-          SawPotentiallyThrowing |= !callToNoUnwindFunction(&MI);
-        continue;
-      }
-
-      // End of the previous try-range?
-      MCSymbol *BeginLabel = MI.getOperand(0).getMCSymbol();
-      if (BeginLabel == LastLabel)
-        SawPotentiallyThrowing = false;
-
-      // Beginning of a new try-range?
-      RangeMapType::const_iterator L = PadMap.find(BeginLabel);
-      if (L == PadMap.end())
-        // Nope, it was just some random label.
-        continue;
-
-      const PadRange &P = L->second;
-      const LandingPadInfo *LandingPad = LandingPads[P.PadIndex];
-      assert(BeginLabel == LandingPad->BeginLabels[P.RangeIndex] &&
-             "Inconsistent landing pad map!");
-
-      if (SawPotentiallyThrowing) {
-        FuncInfo.IPToStateList.push_back(std::make_pair(LastLabel, -1));
-        SawPotentiallyThrowing = false;
-        LastEHState = -1;
-      }
-
-      if (LandingPad->WinEHState != LastEHState)
-        FuncInfo.IPToStateList.push_back(
-            std::make_pair(BeginLabel, LandingPad->WinEHState));
-      LastEHState = LandingPad->WinEHState;
-      LastLabel = LandingPad->EndLabels[P.RangeIndex];
-    }
-  }
-
-  // Defer emission until we've visited the parent function and all the catch
-  // handlers.  Cleanups don't contribute to the ip2state table yet, so don't
-  // count them.
-  if (ParentF != F && !FuncInfo.CatchHandlerMaxState.count(F))
-    return;
-  ++FuncInfo.NumIPToStateFuncsVisited;
-  if (FuncInfo.NumIPToStateFuncsVisited != FuncInfo.CatchHandlerMaxState.size())
-    return;
-
-  MCSymbol *UnwindMapXData = nullptr;
-  MCSymbol *TryBlockMapXData = nullptr;
-  MCSymbol *IPToStateXData = nullptr;
-  if (!FuncInfo.UnwindMap.empty())
-    UnwindMapXData = Asm->OutContext.GetOrCreateSymbol(
-        Twine("$stateUnwindMap$", ParentLinkageName));
-  if (!FuncInfo.TryBlockMap.empty())
-    TryBlockMapXData = Asm->OutContext.GetOrCreateSymbol(
-        Twine("$tryMap$", ParentLinkageName));
-  if (!FuncInfo.IPToStateList.empty())
-    IPToStateXData = Asm->OutContext.GetOrCreateSymbol(
-        Twine("$ip2state$", ParentLinkageName));
-
-  // FuncInfo {
-  //   uint32_t           MagicNumber
-  //   int32_t            MaxState;
-  //   UnwindMapEntry    *UnwindMap;
-  //   uint32_t           NumTryBlocks;
-  //   TryBlockMapEntry  *TryBlockMap;
-  //   uint32_t           IPMapEntries;
-  //   IPToStateMapEntry *IPToStateMap;
-  //   uint32_t           UnwindHelp; // (x64/ARM only)
-  //   ESTypeList        *ESTypeList;
-  //   int32_t            EHFlags;
-  // }
-  // EHFlags & 1 -> Synchronous exceptions only, no async exceptions.
-  // EHFlags & 2 -> ???
-  // EHFlags & 4 -> The function is noexcept(true), unwinding can't continue.
-  OS.EmitLabel(FuncInfoXData);
-  OS.EmitIntValue(0x19930522, 4);                      // MagicNumber
-  OS.EmitIntValue(FuncInfo.UnwindMap.size(), 4);       // MaxState
-  OS.EmitValue(createImageRel32(UnwindMapXData), 4);   // UnwindMap
-  OS.EmitIntValue(FuncInfo.TryBlockMap.size(), 4);     // NumTryBlocks
-  OS.EmitValue(createImageRel32(TryBlockMapXData), 4); // TryBlockMap
-  OS.EmitIntValue(FuncInfo.IPToStateList.size(), 4);   // IPMapEntries
-  OS.EmitValue(createImageRel32(IPToStateXData), 4);   // IPToStateMap
-  OS.EmitIntValue(FuncInfo.UnwindHelpFrameOffset, 4);  // UnwindHelp
-  OS.EmitIntValue(0, 4);                               // ESTypeList
-  OS.EmitIntValue(1, 4);                               // EHFlags
-
-  // UnwindMapEntry {
-  //   int32_t ToState;
-  //   void  (*Action)();
-  // };
-  if (UnwindMapXData) {
-    OS.EmitLabel(UnwindMapXData);
-    for (const WinEHUnwindMapEntry &UME : FuncInfo.UnwindMap) {
-      OS.EmitIntValue(UME.ToState, 4);                // ToState
-      OS.EmitValue(createImageRel32(UME.Cleanup), 4); // Action
-    }
-  }
-
-  // TryBlockMap {
-  //   int32_t      TryLow;
-  //   int32_t      TryHigh;
-  //   int32_t      CatchHigh;
-  //   int32_t      NumCatches;
-  //   HandlerType *HandlerArray;
-  // };
-  if (TryBlockMapXData) {
-    OS.EmitLabel(TryBlockMapXData);
-    SmallVector<MCSymbol *, 1> HandlerMaps;
-    for (size_t I = 0, E = FuncInfo.TryBlockMap.size(); I != E; ++I) {
-      WinEHTryBlockMapEntry &TBME = FuncInfo.TryBlockMap[I];
-      MCSymbol *HandlerMapXData = nullptr;
-
-      if (!TBME.HandlerArray.empty())
-        HandlerMapXData =
-            Asm->OutContext.GetOrCreateSymbol(Twine("$handlerMap$")
-                                                  .concat(Twine(I))
-                                                  .concat("$")
-                                                  .concat(ParentLinkageName));
-
-      HandlerMaps.push_back(HandlerMapXData);
-
-      int CatchHigh = -1;
-      for (WinEHHandlerType &HT : TBME.HandlerArray)
-        CatchHigh =
-            std::max(CatchHigh, FuncInfo.CatchHandlerMaxState[HT.Handler]);
-
-      assert(TBME.TryLow <= TBME.TryHigh);
-      assert(CatchHigh > TBME.TryHigh);
-      OS.EmitIntValue(TBME.TryLow, 4);                    // TryLow
-      OS.EmitIntValue(TBME.TryHigh, 4);                   // TryHigh
-      OS.EmitIntValue(CatchHigh, 4);                      // CatchHigh
-      OS.EmitIntValue(TBME.HandlerArray.size(), 4);       // NumCatches
-      OS.EmitValue(createImageRel32(HandlerMapXData), 4); // HandlerArray
-    }
-
-    for (size_t I = 0, E = FuncInfo.TryBlockMap.size(); I != E; ++I) {
-      WinEHTryBlockMapEntry &TBME = FuncInfo.TryBlockMap[I];
-      MCSymbol *HandlerMapXData = HandlerMaps[I];
-      if (!HandlerMapXData)
-        continue;
-      // HandlerType {
-      //   int32_t         Adjectives;
-      //   TypeDescriptor *Type;
-      //   int32_t         CatchObjOffset;
-      //   void          (*Handler)();
-      //   int32_t         ParentFrameOffset; // x64 only
-      // };
-      OS.EmitLabel(HandlerMapXData);
-      for (const WinEHHandlerType &HT : TBME.HandlerArray) {
-        MCSymbol *ParentFrameOffset =
-            Asm->OutContext.getOrCreateParentFrameOffsetSymbol(
-                GlobalValue::getRealLinkageName(HT.Handler->getName()));
-        const MCSymbolRefExpr *ParentFrameOffsetRef = MCSymbolRefExpr::Create(
-            ParentFrameOffset, MCSymbolRefExpr::VK_None, Asm->OutContext);
-
-        // Get the frame escape label with the offset of the catch object. If
-        // the index is -1, then there is no catch object, and we should emit an
-        // offset of zero, indicating that no copy will occur.
-        const MCExpr *FrameAllocOffsetRef = nullptr;
-        if (HT.CatchObjRecoverIdx >= 0) {
-          MCSymbol *FrameAllocOffset =
-              Asm->OutContext.getOrCreateFrameAllocSymbol(
-                  GlobalValue::getRealLinkageName(ParentF->getName()),
-                  HT.CatchObjRecoverIdx);
-          FrameAllocOffsetRef = MCSymbolRefExpr::Create(
-              FrameAllocOffset, MCSymbolRefExpr::VK_None, Asm->OutContext);
-        } else {
-          FrameAllocOffsetRef = MCConstantExpr::Create(0, Asm->OutContext);
-        }
-
-        OS.EmitIntValue(HT.Adjectives, 4);                    // Adjectives
-        OS.EmitValue(createImageRel32(HT.TypeDescriptor), 4); // Type
-        OS.EmitValue(FrameAllocOffsetRef, 4);                 // CatchObjOffset
-        OS.EmitValue(createImageRel32(HT.Handler), 4);        // Handler
-        OS.EmitValue(ParentFrameOffsetRef, 4);                // ParentFrameOffset
-      }
-    }
-  }
-
-  // IPToStateMapEntry {
-  //   void   *IP;
-  //   int32_t State;
-  // };
-  if (IPToStateXData) {
-    OS.EmitLabel(IPToStateXData);
-    for (auto &IPStatePair : FuncInfo.IPToStateList) {
-      OS.EmitValue(createImageRel32(IPStatePair.first), 4); // IP
-      OS.EmitIntValue(IPStatePair.second, 4);               // State
-    }
-  }
-}
diff --git a/lib/CodeGen/AsmPrinter/Win64Exception.h b/lib/CodeGen/AsmPrinter/Win64Exception.h
deleted file mode 100644
index 5f4237fc015..00000000000
--- a/lib/CodeGen/AsmPrinter/Win64Exception.h
+++ /dev/null
@@ -1,61 +0,0 @@
-//===-- Win64Exception.h - Windows Exception Handling ----------*- C++ -*--===//
-//
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
-//
-//===----------------------------------------------------------------------===//
-//
-// This file contains support for writing windows exception info into asm files.
-//
-//===----------------------------------------------------------------------===//
-
-#ifndef LLVM_LIB_CODEGEN_ASMPRINTER_WIN64EXCEPTION_H
-#define LLVM_LIB_CODEGEN_ASMPRINTER_WIN64EXCEPTION_H
-
-#include "EHStreamer.h"
-
-namespace llvm {
-class GlobalValue;
-class MachineFunction;
-class MCExpr;
-
-class Win64Exception : public EHStreamer {
-  /// Per-function flag to indicate if personality info should be emitted.
-  bool shouldEmitPersonality;
-
-  /// Per-function flag to indicate if the LSDA should be emitted.
-  bool shouldEmitLSDA;
-
-  /// Per-function flag to indicate if frame moves info should be emitted.
-  bool shouldEmitMoves;
-
-  void emitCSpecificHandlerTable();
-
-  void emitCXXFrameHandler3Table(const MachineFunction *MF);
-
-  const MCExpr *createImageRel32(const MCSymbol *Value);
-  const MCExpr *createImageRel32(const GlobalValue *GV);
-
-public:
-  //===--------------------------------------------------------------------===//
-  // Main entry points.
-  //
-  Win64Exception(AsmPrinter *A);
-  ~Win64Exception() override;
-
-  /// Emit all exception information that should come after the content.
-  void endModule() override;
-
-  /// Gather pre-function exception information.  Assumes being emitted
-  /// immediately after the function entry point.
-  void beginFunction(const MachineFunction *MF) override;
-
-  /// Gather and emit post-function exception information.
-  void endFunction(const MachineFunction *) override;
-};
-}
-
-#endif
-
diff --git a/lib/CodeGen/AsmPrinter/WinCodeViewLineTables.cpp b/lib/CodeGen/AsmPrinter/WinCodeViewLineTables.cpp
index 276e7dfb2eb..6610ac78f8c 100644
--- a/lib/CodeGen/AsmPrinter/WinCodeViewLineTables.cpp
+++ b/lib/CodeGen/AsmPrinter/WinCodeViewLineTables.cpp
@@ -20,11 +20,11 @@ namespace llvm {
 
 StringRef WinCodeViewLineTables::getFullFilepath(const MDNode *S) {
   assert(S);
-  assert((isa<MDCompileUnit>(S) || isa<MDFile>(S) || isa<MDSubprogram>(S) ||
-          isa<MDLexicalBlockBase>(S)) &&
+  assert((isa<DICompileUnit>(S) || isa<DIFile>(S) || isa<DISubprogram>(S) ||
+          isa<DILexicalBlockBase>(S)) &&
          "Unexpected scope info");
 
-  auto *Scope = cast<MDScope>(S);
+  auto *Scope = cast<DIScope>(S);
   StringRef Dir = Scope->getDirectory(),
             Filename = Scope->getFilename();
   char *&Result = DirAndFilenameToFilepathMap[std::make_pair(Dir, Filename)];
@@ -94,10 +94,10 @@ void WinCodeViewLineTables::maybeRecordLocation(DebugLoc DL,
   }
   FileNameRegistry.add(Filename);
 
-  MCSymbol *MCL = Asm->MMI->getContext().CreateTempSymbol();
-  Asm->OutStreamer.EmitLabel(MCL);
+  MCSymbol *MCL = Asm->MMI->getContext().createTempSymbol();
+  Asm->OutStreamer->EmitLabel(MCL);
   CurFn->Instrs.push_back(MCL);
-  InstrInfo[MCL] = InstrInfoTy(Filename, DL.getLine());
+  InstrInfo[MCL] = InstrInfoTy(Filename, DL.getLine(), DL.getCol());
 }
 
 WinCodeViewLineTables::WinCodeViewLineTables(AsmPrinter *AP)
@@ -120,7 +120,7 @@ void WinCodeViewLineTables::endModule() {
     return;
 
   assert(Asm != nullptr);
-  Asm->OutStreamer.SwitchSection(
+  Asm->OutStreamer->SwitchSection(
       Asm->getObjFileLowering().getCOFFDebugSymbolsSection());
   Asm->EmitInt32(COFF::DEBUG_SECTION_MAGIC);
 
@@ -135,7 +135,7 @@ void WinCodeViewLineTables::endModule() {
     emitDebugInfoForFunction(VisitedFunctions[I]);
 
   // This subsection holds a file index to offset in string table table.
-  Asm->OutStreamer.AddComment("File index to string table offset subsection");
+  Asm->OutStreamer->AddComment("File index to string table offset subsection");
   Asm->EmitInt32(COFF::DEBUG_INDEX_SUBSECTION);
   size_t NumFilenames = FileNameRegistry.Infos.size();
   Asm->EmitInt32(8 * NumFilenames);
@@ -148,7 +148,7 @@ void WinCodeViewLineTables::endModule() {
   }
 
   // This subsection holds the string table.
-  Asm->OutStreamer.AddComment("String table");
+  Asm->OutStreamer->AddComment("String table");
   Asm->EmitInt32(COFF::DEBUG_STRING_TABLE_SUBSECTION);
   Asm->EmitInt32(FileNameRegistry.LastOffset);
   // The payload starts with a null character.
@@ -156,12 +156,12 @@ void WinCodeViewLineTables::endModule() {
 
   for (size_t I = 0, E = FileNameRegistry.Filenames.size(); I != E; ++I) {
     // Just emit unique filenames one by one, separated by a null character.
-    Asm->OutStreamer.EmitBytes(FileNameRegistry.Filenames[I]);
+    Asm->OutStreamer->EmitBytes(FileNameRegistry.Filenames[I]);
     Asm->EmitInt8(0);
   }
 
   // No more subsections. Fill with zeros to align the end of the section by 4.
-  Asm->OutStreamer.EmitFill((-FileNameRegistry.LastOffset) % 4, 0);
+  Asm->OutStreamer->EmitFill((-FileNameRegistry.LastOffset) % 4, 0);
 
   clear();
 }
@@ -171,10 +171,10 @@ static void EmitLabelDiff(MCStreamer &Streamer,
                           unsigned int Size = 4) {
   MCSymbolRefExpr::VariantKind Variant = MCSymbolRefExpr::VK_None;
   MCContext &Context = Streamer.getContext();
-  const MCExpr *FromRef = MCSymbolRefExpr::Create(From, Variant, Context),
-               *ToRef   = MCSymbolRefExpr::Create(To, Variant, Context);
+  const MCExpr *FromRef = MCSymbolRefExpr::create(From, Variant, Context),
+               *ToRef   = MCSymbolRefExpr::create(To, Variant, Context);
   const MCExpr *AddrDelta =
-      MCBinaryExpr::Create(MCBinaryExpr::Sub, ToRef, FromRef, Context);
+      MCBinaryExpr::create(MCBinaryExpr::Sub, ToRef, FromRef, Context);
   Streamer.EmitValue(AddrDelta, Size);
 }
 
@@ -191,7 +191,7 @@ void WinCodeViewLineTables::emitDebugInfoForFunction(const Function *GV) {
 
   StringRef GVName = GV->getName();
   StringRef FuncName;
-  if (DISubprogram SP = getDISubprogram(GV))
+  if (auto *SP = getDISubprogram(GV))
     FuncName = SP->getDisplayName();
 
   // FIXME Clang currently sets DisplayName to "bar" for a C++
@@ -201,41 +201,41 @@ void WinCodeViewLineTables::emitDebugInfoForFunction(const Function *GV) {
   if (GVName.startswith("\01?"))
     FuncName = GVName.substr(1);
   // Emit a symbol subsection, required by VS2012+ to find function boundaries.
-  MCSymbol *SymbolsBegin = Asm->MMI->getContext().CreateTempSymbol(),
-           *SymbolsEnd = Asm->MMI->getContext().CreateTempSymbol();
-  Asm->OutStreamer.AddComment("Symbol subsection for " + Twine(FuncName));
+  MCSymbol *SymbolsBegin = Asm->MMI->getContext().createTempSymbol(),
+           *SymbolsEnd = Asm->MMI->getContext().createTempSymbol();
+  Asm->OutStreamer->AddComment("Symbol subsection for " + Twine(FuncName));
   Asm->EmitInt32(COFF::DEBUG_SYMBOL_SUBSECTION);
-  EmitLabelDiff(Asm->OutStreamer, SymbolsBegin, SymbolsEnd);
-  Asm->OutStreamer.EmitLabel(SymbolsBegin);
+  EmitLabelDiff(*Asm->OutStreamer, SymbolsBegin, SymbolsEnd);
+  Asm->OutStreamer->EmitLabel(SymbolsBegin);
   {
-    MCSymbol *ProcSegmentBegin = Asm->MMI->getContext().CreateTempSymbol(),
-             *ProcSegmentEnd = Asm->MMI->getContext().CreateTempSymbol();
-    EmitLabelDiff(Asm->OutStreamer, ProcSegmentBegin, ProcSegmentEnd, 2);
-    Asm->OutStreamer.EmitLabel(ProcSegmentBegin);
+    MCSymbol *ProcSegmentBegin = Asm->MMI->getContext().createTempSymbol(),
+             *ProcSegmentEnd = Asm->MMI->getContext().createTempSymbol();
+    EmitLabelDiff(*Asm->OutStreamer, ProcSegmentBegin, ProcSegmentEnd, 2);
+    Asm->OutStreamer->EmitLabel(ProcSegmentBegin);
 
     Asm->EmitInt16(COFF::DEBUG_SYMBOL_TYPE_PROC_START);
     // Some bytes of this segment don't seem to be required for basic debugging,
     // so just fill them with zeroes.
-    Asm->OutStreamer.EmitFill(12, 0);
+    Asm->OutStreamer->EmitFill(12, 0);
     // This is the important bit that tells the debugger where the function
     // code is located and what's its size:
-    EmitLabelDiff(Asm->OutStreamer, Fn, FI.End);
-    Asm->OutStreamer.EmitFill(12, 0);
-    Asm->OutStreamer.EmitCOFFSecRel32(Fn);
-    Asm->OutStreamer.EmitCOFFSectionIndex(Fn);
+    EmitLabelDiff(*Asm->OutStreamer, Fn, FI.End);
+    Asm->OutStreamer->EmitFill(12, 0);
+    Asm->OutStreamer->EmitCOFFSecRel32(Fn);
+    Asm->OutStreamer->EmitCOFFSectionIndex(Fn);
     Asm->EmitInt8(0);
     // Emit the function display name as a null-terminated string.
-    Asm->OutStreamer.EmitBytes(FuncName);
+    Asm->OutStreamer->EmitBytes(FuncName);
     Asm->EmitInt8(0);
-    Asm->OutStreamer.EmitLabel(ProcSegmentEnd);
+    Asm->OutStreamer->EmitLabel(ProcSegmentEnd);
 
     // We're done with this function.
     Asm->EmitInt16(0x0002);
     Asm->EmitInt16(COFF::DEBUG_SYMBOL_TYPE_PROC_END);
   }
-  Asm->OutStreamer.EmitLabel(SymbolsEnd);
+  Asm->OutStreamer->EmitLabel(SymbolsEnd);
   // Every subsection must be aligned to a 4-byte boundary.
-  Asm->OutStreamer.EmitFill((-FuncName.size()) % 4, 0);
+  Asm->OutStreamer->EmitFill((-FuncName.size()) % 4, 0);
 
   // PCs/Instructions are grouped into segments sharing the same filename.
   // Pre-calculate the lengths (in instructions) of these segments and store
@@ -254,42 +254,58 @@ void WinCodeViewLineTables::emitDebugInfoForFunction(const Function *GV) {
   FilenameSegmentLengths[LastSegmentEnd] = FI.Instrs.size() - LastSegmentEnd;
 
   // Emit a line table subsection, requred to do PC-to-file:line lookup.
-  Asm->OutStreamer.AddComment("Line table subsection for " + Twine(FuncName));
+  Asm->OutStreamer->AddComment("Line table subsection for " + Twine(FuncName));
   Asm->EmitInt32(COFF::DEBUG_LINE_TABLE_SUBSECTION);
-  MCSymbol *LineTableBegin = Asm->MMI->getContext().CreateTempSymbol(),
-           *LineTableEnd = Asm->MMI->getContext().CreateTempSymbol();
-  EmitLabelDiff(Asm->OutStreamer, LineTableBegin, LineTableEnd);
-  Asm->OutStreamer.EmitLabel(LineTableBegin);
+  MCSymbol *LineTableBegin = Asm->MMI->getContext().createTempSymbol(),
+           *LineTableEnd = Asm->MMI->getContext().createTempSymbol();
+  EmitLabelDiff(*Asm->OutStreamer, LineTableBegin, LineTableEnd);
+  Asm->OutStreamer->EmitLabel(LineTableBegin);
 
   // Identify the function this subsection is for.
-  Asm->OutStreamer.EmitCOFFSecRel32(Fn);
-  Asm->OutStreamer.EmitCOFFSectionIndex(Fn);
-  // Insert padding after a 16-bit section index.
-  Asm->EmitInt16(0);
+  Asm->OutStreamer->EmitCOFFSecRel32(Fn);
+  Asm->OutStreamer->EmitCOFFSectionIndex(Fn);
+  // Insert flags after a 16-bit section index.
+  Asm->EmitInt16(COFF::DEBUG_LINE_TABLES_HAVE_COLUMN_RECORDS);
 
   // Length of the function's code, in bytes.
-  EmitLabelDiff(Asm->OutStreamer, Fn, FI.End);
+  EmitLabelDiff(*Asm->OutStreamer, Fn, FI.End);
 
   // PC-to-linenumber lookup table:
   MCSymbol *FileSegmentEnd = nullptr;
+
+  // The start of the last segment:
+  size_t LastSegmentStart = 0;
+
+  auto FinishPreviousChunk = [&] {
+    if (!FileSegmentEnd)
+      return;
+    for (size_t ColSegI = LastSegmentStart,
+                ColSegEnd = ColSegI + FilenameSegmentLengths[LastSegmentStart];
+         ColSegI != ColSegEnd; ++ColSegI) {
+      unsigned ColumnNumber = InstrInfo[FI.Instrs[ColSegI]].ColumnNumber;
+      Asm->EmitInt16(ColumnNumber); // Start column
+      Asm->EmitInt16(ColumnNumber); // End column
+    }
+    Asm->OutStreamer->EmitLabel(FileSegmentEnd);
+  };
+
   for (size_t J = 0, F = FI.Instrs.size(); J != F; ++J) {
     MCSymbol *Instr = FI.Instrs[J];
     assert(InstrInfo.count(Instr));
 
     if (FilenameSegmentLengths.count(J)) {
       // We came to a beginning of a new filename segment.
-      if (FileSegmentEnd)
-        Asm->OutStreamer.EmitLabel(FileSegmentEnd);
+      FinishPreviousChunk();
       StringRef CurFilename = InstrInfo[FI.Instrs[J]].Filename;
       assert(FileNameRegistry.Infos.count(CurFilename));
       size_t IndexInStringTable =
           FileNameRegistry.Infos[CurFilename].FilenameID;
       // Each segment starts with the offset of the filename
       // in the string table.
-      Asm->OutStreamer.AddComment(
+      Asm->OutStreamer->AddComment(
           "Segment for file '" + Twine(CurFilename) + "' begins");
-      MCSymbol *FileSegmentBegin = Asm->MMI->getContext().CreateTempSymbol();
-      Asm->OutStreamer.EmitLabel(FileSegmentBegin);
+      MCSymbol *FileSegmentBegin = Asm->MMI->getContext().createTempSymbol();
+      Asm->OutStreamer->EmitLabel(FileSegmentBegin);
       Asm->EmitInt32(8 * IndexInStringTable);
 
       // Number of PC records in the lookup table.
@@ -298,18 +314,18 @@ void WinCodeViewLineTables::emitDebugInfoForFunction(const Function *GV) {
 
       // Full size of the segment for this filename, including the prev two
       // records.
-      FileSegmentEnd = Asm->MMI->getContext().CreateTempSymbol();
-      EmitLabelDiff(Asm->OutStreamer, FileSegmentBegin, FileSegmentEnd);
+      FileSegmentEnd = Asm->MMI->getContext().createTempSymbol();
+      EmitLabelDiff(*Asm->OutStreamer, FileSegmentBegin, FileSegmentEnd);
+      LastSegmentStart = J;
     }
 
     // The first PC with the given linenumber and the linenumber itself.
-    EmitLabelDiff(Asm->OutStreamer, Fn, Instr);
+    EmitLabelDiff(*Asm->OutStreamer, Fn, Instr);
     Asm->EmitInt32(InstrInfo[Instr].LineNumber);
   }
 
-  if (FileSegmentEnd)
-    Asm->OutStreamer.EmitLabel(FileSegmentEnd);
-  Asm->OutStreamer.EmitLabel(LineTableEnd);
+  FinishPreviousChunk();
+  Asm->OutStreamer->EmitLabel(LineTableEnd);
 }
 
 void WinCodeViewLineTables::beginFunction(const MachineFunction *MF) {
diff --git a/lib/CodeGen/AsmPrinter/WinCodeViewLineTables.h b/lib/CodeGen/AsmPrinter/WinCodeViewLineTables.h
index c66d141837d..43d1a432712 100644
--- a/lib/CodeGen/AsmPrinter/WinCodeViewLineTables.h
+++ b/lib/CodeGen/AsmPrinter/WinCodeViewLineTables.h
@@ -29,7 +29,7 @@
 
 namespace llvm {
 /// \brief Collects and handles line tables information in a CodeView format.
-class WinCodeViewLineTables : public AsmPrinterHandler {
+class LLVM_LIBRARY_VISIBILITY WinCodeViewLineTables : public AsmPrinterHandler {
   AsmPrinter *Asm;
   DebugLoc PrevInstLoc;
 
@@ -52,11 +52,13 @@ class WinCodeViewLineTables : public AsmPrinterHandler {
   struct InstrInfoTy {
     StringRef Filename;
     unsigned LineNumber;
+    unsigned ColumnNumber;
 
-    InstrInfoTy() : LineNumber(0) {}
+    InstrInfoTy() : LineNumber(0), ColumnNumber(0) {}
 
-    InstrInfoTy(StringRef Filename, unsigned LineNumber)
-        : Filename(Filename), LineNumber(LineNumber) {}
+    InstrInfoTy(StringRef Filename, unsigned LineNumber, unsigned ColumnNumber)
+        : Filename(Filename), LineNumber(LineNumber),
+          ColumnNumber(ColumnNumber) {}
   };
   DenseMap<MCSymbol *, InstrInfoTy> InstrInfo;
 
diff --git a/lib/CodeGen/AsmPrinter/WinException.cpp b/lib/CodeGen/AsmPrinter/WinException.cpp
new file mode 100644
index 00000000000..71c77815e28
--- /dev/null
+++ b/lib/CodeGen/AsmPrinter/WinException.cpp
@@ -0,0 +1,668 @@
+//===-- CodeGen/AsmPrinter/WinException.cpp - Dwarf Exception Impl ------===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This file contains support for writing Win64 exception info into asm files.
+//
+//===----------------------------------------------------------------------===//
+
+#include "WinException.h"
+#include "llvm/ADT/SmallString.h"
+#include "llvm/ADT/StringExtras.h"
+#include "llvm/ADT/Twine.h"
+#include "llvm/CodeGen/AsmPrinter.h"
+#include "llvm/CodeGen/MachineFrameInfo.h"
+#include "llvm/CodeGen/MachineFunction.h"
+#include "llvm/CodeGen/MachineModuleInfo.h"
+#include "llvm/CodeGen/WinEHFuncInfo.h"
+#include "llvm/IR/DataLayout.h"
+#include "llvm/IR/Mangler.h"
+#include "llvm/IR/Module.h"
+#include "llvm/MC/MCAsmInfo.h"
+#include "llvm/MC/MCContext.h"
+#include "llvm/MC/MCExpr.h"
+#include "llvm/MC/MCSection.h"
+#include "llvm/MC/MCStreamer.h"
+#include "llvm/MC/MCSymbol.h"
+#include "llvm/MC/MCWin64EH.h"
+#include "llvm/Support/Dwarf.h"
+#include "llvm/Support/ErrorHandling.h"
+#include "llvm/Support/FormattedStream.h"
+#include "llvm/Target/TargetFrameLowering.h"
+#include "llvm/Target/TargetLoweringObjectFile.h"
+#include "llvm/Target/TargetOptions.h"
+#include "llvm/Target/TargetRegisterInfo.h"
+using namespace llvm;
+
+WinException::WinException(AsmPrinter *A) : EHStreamer(A) {
+  // MSVC's EH tables are always composed of 32-bit words.  All known 64-bit
+  // platforms use an imagerel32 relocation to refer to symbols.
+  useImageRel32 = (A->getDataLayout().getPointerSizeInBits() == 64);
+}
+
+WinException::~WinException() {}
+
+/// endModule - Emit all exception information that should come after the
+/// content.
+void WinException::endModule() {
+  auto &OS = *Asm->OutStreamer;
+  const Module *M = MMI->getModule();
+  for (const Function &F : *M)
+    if (F.hasFnAttribute("safeseh"))
+      OS.EmitCOFFSafeSEH(Asm->getSymbol(&F));
+}
+
+void WinException::beginFunction(const MachineFunction *MF) {
+  shouldEmitMoves = shouldEmitPersonality = shouldEmitLSDA = false;
+
+  // If any landing pads survive, we need an EH table.
+  bool hasLandingPads = !MMI->getLandingPads().empty();
+
+  const Function *F = MF->getFunction();
+  const Function *ParentF = MMI->getWinEHParent(F);
+
+  shouldEmitMoves = Asm->needsSEHMoves();
+
+  const TargetLoweringObjectFile &TLOF = Asm->getObjFileLowering();
+  unsigned PerEncoding = TLOF.getPersonalityEncoding();
+  const Function *Per = nullptr;
+  if (F->hasPersonalityFn())
+    Per = dyn_cast<Function>(F->getPersonalityFn()->stripPointerCasts());
+
+  bool forceEmitPersonality =
+    F->hasPersonalityFn() && !isNoOpWithoutInvoke(classifyEHPersonality(Per)) &&
+    F->needsUnwindTableEntry();
+
+  shouldEmitPersonality = forceEmitPersonality || (hasLandingPads &&
+    PerEncoding != dwarf::DW_EH_PE_omit && Per);
+
+  unsigned LSDAEncoding = TLOF.getLSDAEncoding();
+  shouldEmitLSDA = shouldEmitPersonality &&
+    LSDAEncoding != dwarf::DW_EH_PE_omit;
+
+  // If we're not using CFI, we don't want the CFI or the personality. If
+  // WinEHPrepare outlined something, we should emit the LSDA.
+  if (!Asm->MAI->usesWindowsCFI()) {
+    bool HasOutlinedChildren =
+        F->hasFnAttribute("wineh-parent") && F == ParentF;
+    shouldEmitLSDA = HasOutlinedChildren;
+    shouldEmitPersonality = false;
+    return;
+  }
+
+  // If this was an outlined handler, we need to define the label corresponding
+  // to the offset of the parent frame relative to the stack pointer after the
+  // prologue.
+  if (F != ParentF) {
+    WinEHFuncInfo &FuncInfo = MMI->getWinEHFuncInfo(ParentF);
+    auto I = FuncInfo.CatchHandlerParentFrameObjOffset.find(F);
+    if (I != FuncInfo.CatchHandlerParentFrameObjOffset.end()) {
+      MCSymbol *HandlerTypeParentFrameOffset =
+          Asm->OutContext.getOrCreateParentFrameOffsetSymbol(
+              GlobalValue::getRealLinkageName(F->getName()));
+
+      // Emit a symbol assignment.
+      Asm->OutStreamer->EmitAssignment(
+          HandlerTypeParentFrameOffset,
+          MCConstantExpr::create(I->second, Asm->OutContext));
+    }
+  }
+
+  if (shouldEmitMoves || shouldEmitPersonality)
+    Asm->OutStreamer->EmitWinCFIStartProc(Asm->CurrentFnSym);
+
+  if (shouldEmitPersonality) {
+    const MCSymbol *PersHandlerSym =
+        TLOF.getCFIPersonalitySymbol(Per, *Asm->Mang, Asm->TM, MMI);
+    Asm->OutStreamer->EmitWinEHHandler(PersHandlerSym, true, true);
+  }
+}
+
+/// endFunction - Gather and emit post-function exception information.
+///
+void WinException::endFunction(const MachineFunction *MF) {
+  if (!shouldEmitPersonality && !shouldEmitMoves && !shouldEmitLSDA)
+    return;
+
+  const Function *F = MF->getFunction();
+  EHPersonality Per = EHPersonality::Unknown;
+  if (F->hasPersonalityFn())
+    Per = classifyEHPersonality(F->getPersonalityFn());
+
+  // Get rid of any dead landing pads if we're not using a Windows EH scheme. In
+  // Windows EH schemes, the landing pad is not actually reachable. It only
+  // exists so that we can emit the right table data.
+  if (!isMSVCEHPersonality(Per))
+    MMI->TidyLandingPads();
+
+  if (shouldEmitPersonality || shouldEmitLSDA) {
+    Asm->OutStreamer->PushSection();
+
+    if (shouldEmitMoves || shouldEmitPersonality) {
+      // Emit an UNWIND_INFO struct describing the prologue.
+      Asm->OutStreamer->EmitWinEHHandlerData();
+    } else {
+      // Just switch sections to the right xdata section. This use of
+      // CurrentFnSym assumes that we only emit the LSDA when ending the parent
+      // function.
+      MCSection *XData = WinEH::UnwindEmitter::getXDataSection(
+          Asm->CurrentFnSym, Asm->OutContext);
+      Asm->OutStreamer->SwitchSection(XData);
+    }
+
+    // Emit the tables appropriate to the personality function in use. If we
+    // don't recognize the personality, assume it uses an Itanium-style LSDA.
+    if (Per == EHPersonality::MSVC_Win64SEH)
+      emitCSpecificHandlerTable();
+    else if (Per == EHPersonality::MSVC_X86SEH)
+      emitExceptHandlerTable(MF);
+    else if (Per == EHPersonality::MSVC_CXX)
+      emitCXXFrameHandler3Table(MF);
+    else
+      emitExceptionTable();
+
+    Asm->OutStreamer->PopSection();
+  }
+
+  if (shouldEmitMoves)
+    Asm->OutStreamer->EmitWinCFIEndProc();
+}
+
+const MCExpr *WinException::create32bitRef(const MCSymbol *Value) {
+  if (!Value)
+    return MCConstantExpr::create(0, Asm->OutContext);
+  return MCSymbolRefExpr::create(Value, useImageRel32
+                                            ? MCSymbolRefExpr::VK_COFF_IMGREL32
+                                            : MCSymbolRefExpr::VK_None,
+                                 Asm->OutContext);
+}
+
+const MCExpr *WinException::create32bitRef(const GlobalValue *GV) {
+  if (!GV)
+    return MCConstantExpr::create(0, Asm->OutContext);
+  return create32bitRef(Asm->getSymbol(GV));
+}
+
+/// Emit the language-specific data that __C_specific_handler expects.  This
+/// handler lives in the x64 Microsoft C runtime and allows catching or cleaning
+/// up after faults with __try, __except, and __finally.  The typeinfo values
+/// are not really RTTI data, but pointers to filter functions that return an
+/// integer (1, 0, or -1) indicating how to handle the exception. For __finally
+/// blocks and other cleanups, the landing pad label is zero, and the filter
+/// function is actually a cleanup handler with the same prototype.  A catch-all
+/// entry is modeled with a null filter function field and a non-zero landing
+/// pad label.
+///
+/// Possible filter function return values:
+///   EXCEPTION_EXECUTE_HANDLER (1):
+///     Jump to the landing pad label after cleanups.
+///   EXCEPTION_CONTINUE_SEARCH (0):
+///     Continue searching this table or continue unwinding.
+///   EXCEPTION_CONTINUE_EXECUTION (-1):
+///     Resume execution at the trapping PC.
+///
+/// Inferred table structure:
+///   struct Table {
+///     int NumEntries;
+///     struct Entry {
+///       imagerel32 LabelStart;
+///       imagerel32 LabelEnd;
+///       imagerel32 FilterOrFinally;  // One means catch-all.
+///       imagerel32 LabelLPad;        // Zero means __finally.
+///     } Entries[NumEntries];
+///   };
+void WinException::emitCSpecificHandlerTable() {
+  const std::vector<LandingPadInfo> &PadInfos = MMI->getLandingPads();
+
+  // Simplifying assumptions for first implementation:
+  // - Cleanups are not implemented.
+  // - Filters are not implemented.
+
+  // The Itanium LSDA table sorts similar landing pads together to simplify the
+  // actions table, but we don't need that.
+  SmallVector<const LandingPadInfo *, 64> LandingPads;
+  LandingPads.reserve(PadInfos.size());
+  for (const auto &LP : PadInfos)
+    LandingPads.push_back(&LP);
+
+  // Compute label ranges for call sites as we would for the Itanium LSDA, but
+  // use an all zero action table because we aren't using these actions.
+  SmallVector<unsigned, 64> FirstActions;
+  FirstActions.resize(LandingPads.size());
+  SmallVector<CallSiteEntry, 64> CallSites;
+  computeCallSiteTable(CallSites, LandingPads, FirstActions);
+
+  MCSymbol *EHFuncBeginSym = Asm->getFunctionBegin();
+  MCSymbol *EHFuncEndSym = Asm->getFunctionEnd();
+
+  // Emit the number of table entries.
+  unsigned NumEntries = 0;
+  for (const CallSiteEntry &CSE : CallSites) {
+    if (!CSE.LPad)
+      continue; // Ignore gaps.
+    NumEntries += CSE.LPad->SEHHandlers.size();
+  }
+  Asm->OutStreamer->EmitIntValue(NumEntries, 4);
+
+  // If there are no actions, we don't need to iterate again.
+  if (NumEntries == 0)
+    return;
+
+  // Emit the four-label records for each call site entry. The table has to be
+  // sorted in layout order, and the call sites should already be sorted.
+  for (const CallSiteEntry &CSE : CallSites) {
+    // Ignore gaps. Unlike the Itanium model, unwinding through a frame without
+    // an EH table entry will propagate the exception rather than terminating
+    // the program.
+    if (!CSE.LPad)
+      continue;
+    const LandingPadInfo *LPad = CSE.LPad;
+
+    // Compute the label range. We may reuse the function begin and end labels
+    // rather than forming new ones.
+    const MCExpr *Begin =
+        create32bitRef(CSE.BeginLabel ? CSE.BeginLabel : EHFuncBeginSym);
+    const MCExpr *End;
+    if (CSE.EndLabel) {
+      // The interval is half-open, so we have to add one to include the return
+      // address of the last invoke in the range.
+      End = MCBinaryExpr::createAdd(create32bitRef(CSE.EndLabel),
+                                    MCConstantExpr::create(1, Asm->OutContext),
+                                    Asm->OutContext);
+    } else {
+      End = create32bitRef(EHFuncEndSym);
+    }
+
+    // Emit an entry for each action.
+    for (SEHHandler Handler : LPad->SEHHandlers) {
+      Asm->OutStreamer->EmitValue(Begin, 4);
+      Asm->OutStreamer->EmitValue(End, 4);
+
+      // Emit the filter or finally function pointer, if present. Otherwise,
+      // emit '1' to indicate a catch-all.
+      const Function *F = Handler.FilterOrFinally;
+      if (F)
+        Asm->OutStreamer->EmitValue(create32bitRef(Asm->getSymbol(F)), 4);
+      else
+        Asm->OutStreamer->EmitIntValue(1, 4);
+
+      // Emit the recovery address, if present. Otherwise, this must be a
+      // finally.
+      const BlockAddress *BA = Handler.RecoverBA;
+      if (BA)
+        Asm->OutStreamer->EmitValue(
+            create32bitRef(Asm->GetBlockAddressSymbol(BA)), 4);
+      else
+        Asm->OutStreamer->EmitIntValue(0, 4);
+    }
+  }
+}
+
+void WinException::emitCXXFrameHandler3Table(const MachineFunction *MF) {
+  const Function *F = MF->getFunction();
+  const Function *ParentF = MMI->getWinEHParent(F);
+  auto &OS = *Asm->OutStreamer;
+  WinEHFuncInfo &FuncInfo = MMI->getWinEHFuncInfo(ParentF);
+
+  StringRef ParentLinkageName =
+      GlobalValue::getRealLinkageName(ParentF->getName());
+
+  MCSymbol *FuncInfoXData = nullptr;
+  if (shouldEmitPersonality) {
+    FuncInfoXData = Asm->OutContext.getOrCreateSymbol(
+        Twine("$cppxdata$", ParentLinkageName));
+    OS.EmitValue(create32bitRef(FuncInfoXData), 4);
+
+    extendIP2StateTable(MF, ParentF, FuncInfo);
+
+    // Defer emission until we've visited the parent function and all the catch
+    // handlers.  Cleanups don't contribute to the ip2state table, so don't count
+    // them.
+    if (ParentF != F && !FuncInfo.CatchHandlerMaxState.count(F))
+      return;
+    ++FuncInfo.NumIPToStateFuncsVisited;
+    if (FuncInfo.NumIPToStateFuncsVisited != FuncInfo.CatchHandlerMaxState.size())
+      return;
+  } else {
+    FuncInfoXData = Asm->OutContext.getOrCreateLSDASymbol(ParentLinkageName);
+    emitEHRegistrationOffsetLabel(FuncInfo, ParentLinkageName);
+  }
+
+  MCSymbol *UnwindMapXData = nullptr;
+  MCSymbol *TryBlockMapXData = nullptr;
+  MCSymbol *IPToStateXData = nullptr;
+  if (!FuncInfo.UnwindMap.empty())
+    UnwindMapXData = Asm->OutContext.getOrCreateSymbol(
+        Twine("$stateUnwindMap$", ParentLinkageName));
+  if (!FuncInfo.TryBlockMap.empty())
+    TryBlockMapXData = Asm->OutContext.getOrCreateSymbol(
+        Twine("$tryMap$", ParentLinkageName));
+  if (!FuncInfo.IPToStateList.empty())
+    IPToStateXData = Asm->OutContext.getOrCreateSymbol(
+        Twine("$ip2state$", ParentLinkageName));
+
+  // FuncInfo {
+  //   uint32_t           MagicNumber
+  //   int32_t            MaxState;
+  //   UnwindMapEntry    *UnwindMap;
+  //   uint32_t           NumTryBlocks;
+  //   TryBlockMapEntry  *TryBlockMap;
+  //   uint32_t           IPMapEntries; // always 0 for x86
+  //   IPToStateMapEntry *IPToStateMap; // always 0 for x86
+  //   uint32_t           UnwindHelp;   // non-x86 only
+  //   ESTypeList        *ESTypeList;
+  //   int32_t            EHFlags;
+  // }
+  // EHFlags & 1 -> Synchronous exceptions only, no async exceptions.
+  // EHFlags & 2 -> ???
+  // EHFlags & 4 -> The function is noexcept(true), unwinding can't continue.
+  OS.EmitValueToAlignment(4);
+  OS.EmitLabel(FuncInfoXData);
+  OS.EmitIntValue(0x19930522, 4);                      // MagicNumber
+  OS.EmitIntValue(FuncInfo.UnwindMap.size(), 4);       // MaxState
+  OS.EmitValue(create32bitRef(UnwindMapXData), 4);     // UnwindMap
+  OS.EmitIntValue(FuncInfo.TryBlockMap.size(), 4);     // NumTryBlocks
+  OS.EmitValue(create32bitRef(TryBlockMapXData), 4);   // TryBlockMap
+  OS.EmitIntValue(FuncInfo.IPToStateList.size(), 4);   // IPMapEntries
+  OS.EmitValue(create32bitRef(IPToStateXData), 4);     // IPToStateMap
+  if (Asm->MAI->usesWindowsCFI())
+    OS.EmitIntValue(FuncInfo.UnwindHelpFrameOffset, 4); // UnwindHelp
+  OS.EmitIntValue(0, 4);                               // ESTypeList
+  OS.EmitIntValue(1, 4);                               // EHFlags
+
+  // UnwindMapEntry {
+  //   int32_t ToState;
+  //   void  (*Action)();
+  // };
+  if (UnwindMapXData) {
+    OS.EmitLabel(UnwindMapXData);
+    for (const WinEHUnwindMapEntry &UME : FuncInfo.UnwindMap) {
+      OS.EmitIntValue(UME.ToState, 4);                // ToState
+      OS.EmitValue(create32bitRef(UME.Cleanup), 4);   // Action
+    }
+  }
+
+  // TryBlockMap {
+  //   int32_t      TryLow;
+  //   int32_t      TryHigh;
+  //   int32_t      CatchHigh;
+  //   int32_t      NumCatches;
+  //   HandlerType *HandlerArray;
+  // };
+  if (TryBlockMapXData) {
+    OS.EmitLabel(TryBlockMapXData);
+    SmallVector<MCSymbol *, 1> HandlerMaps;
+    for (size_t I = 0, E = FuncInfo.TryBlockMap.size(); I != E; ++I) {
+      WinEHTryBlockMapEntry &TBME = FuncInfo.TryBlockMap[I];
+      MCSymbol *HandlerMapXData = nullptr;
+
+      if (!TBME.HandlerArray.empty())
+        HandlerMapXData =
+            Asm->OutContext.getOrCreateSymbol(Twine("$handlerMap$")
+                                                  .concat(Twine(I))
+                                                  .concat("$")
+                                                  .concat(ParentLinkageName));
+
+      HandlerMaps.push_back(HandlerMapXData);
+
+      int CatchHigh = -1;
+      for (WinEHHandlerType &HT : TBME.HandlerArray)
+        CatchHigh =
+            std::max(CatchHigh, FuncInfo.CatchHandlerMaxState[HT.Handler]);
+
+      assert(TBME.TryLow <= TBME.TryHigh);
+      OS.EmitIntValue(TBME.TryLow, 4);                    // TryLow
+      OS.EmitIntValue(TBME.TryHigh, 4);                   // TryHigh
+      OS.EmitIntValue(CatchHigh, 4);                      // CatchHigh
+      OS.EmitIntValue(TBME.HandlerArray.size(), 4);       // NumCatches
+      OS.EmitValue(create32bitRef(HandlerMapXData), 4);   // HandlerArray
+    }
+
+    for (size_t I = 0, E = FuncInfo.TryBlockMap.size(); I != E; ++I) {
+      WinEHTryBlockMapEntry &TBME = FuncInfo.TryBlockMap[I];
+      MCSymbol *HandlerMapXData = HandlerMaps[I];
+      if (!HandlerMapXData)
+        continue;
+      // HandlerType {
+      //   int32_t         Adjectives;
+      //   TypeDescriptor *Type;
+      //   int32_t         CatchObjOffset;
+      //   void          (*Handler)();
+      //   int32_t         ParentFrameOffset; // x64 only
+      // };
+      OS.EmitLabel(HandlerMapXData);
+      for (const WinEHHandlerType &HT : TBME.HandlerArray) {
+        // Get the frame escape label with the offset of the catch object. If
+        // the index is -1, then there is no catch object, and we should emit an
+        // offset of zero, indicating that no copy will occur.
+        const MCExpr *FrameAllocOffsetRef = nullptr;
+        if (HT.CatchObjRecoverIdx >= 0) {
+          MCSymbol *FrameAllocOffset =
+              Asm->OutContext.getOrCreateFrameAllocSymbol(
+                  GlobalValue::getRealLinkageName(ParentF->getName()),
+                  HT.CatchObjRecoverIdx);
+          FrameAllocOffsetRef = MCSymbolRefExpr::create(
+              FrameAllocOffset, MCSymbolRefExpr::VK_None, Asm->OutContext);
+        } else {
+          FrameAllocOffsetRef = MCConstantExpr::create(0, Asm->OutContext);
+        }
+
+        OS.EmitIntValue(HT.Adjectives, 4);                    // Adjectives
+        OS.EmitValue(create32bitRef(HT.TypeDescriptor), 4);   // Type
+        OS.EmitValue(FrameAllocOffsetRef, 4);                 // CatchObjOffset
+        OS.EmitValue(create32bitRef(HT.Handler), 4);          // Handler
+
+        if (shouldEmitPersonality) {
+          MCSymbol *ParentFrameOffset =
+              Asm->OutContext.getOrCreateParentFrameOffsetSymbol(
+                  GlobalValue::getRealLinkageName(HT.Handler->getName()));
+          const MCSymbolRefExpr *ParentFrameOffsetRef = MCSymbolRefExpr::create(
+              ParentFrameOffset, Asm->OutContext);
+          OS.EmitValue(ParentFrameOffsetRef, 4); // ParentFrameOffset
+        }
+      }
+    }
+  }
+
+  // IPToStateMapEntry {
+  //   void   *IP;
+  //   int32_t State;
+  // };
+  if (IPToStateXData) {
+    OS.EmitLabel(IPToStateXData);
+    for (auto &IPStatePair : FuncInfo.IPToStateList) {
+      OS.EmitValue(create32bitRef(IPStatePair.first), 4);   // IP
+      OS.EmitIntValue(IPStatePair.second, 4);               // State
+    }
+  }
+}
+
+void WinException::extendIP2StateTable(const MachineFunction *MF,
+                                       const Function *ParentF,
+                                       WinEHFuncInfo &FuncInfo) {
+  const Function *F = MF->getFunction();
+
+  // The Itanium LSDA table sorts similar landing pads together to simplify the
+  // actions table, but we don't need that.
+  SmallVector<const LandingPadInfo *, 64> LandingPads;
+  const std::vector<LandingPadInfo> &PadInfos = MMI->getLandingPads();
+  LandingPads.reserve(PadInfos.size());
+  for (const auto &LP : PadInfos)
+    LandingPads.push_back(&LP);
+
+  RangeMapType PadMap;
+  computePadMap(LandingPads, PadMap);
+
+  // The end label of the previous invoke or nounwind try-range.
+  MCSymbol *LastLabel = Asm->getFunctionBegin();
+
+  // Whether there is a potentially throwing instruction (currently this means
+  // an ordinary call) between the end of the previous try-range and now.
+  bool SawPotentiallyThrowing = false;
+
+  int LastEHState = -2;
+
+  // The parent function and the catch handlers contribute to the 'ip2state'
+  // table.
+
+  // Include ip2state entries for the beginning of the main function and
+  // for catch handler functions.
+  if (F == ParentF) {
+    FuncInfo.IPToStateList.push_back(std::make_pair(LastLabel, -1));
+    LastEHState = -1;
+  } else if (FuncInfo.HandlerBaseState.count(F)) {
+    FuncInfo.IPToStateList.push_back(
+        std::make_pair(LastLabel, FuncInfo.HandlerBaseState[F]));
+    LastEHState = FuncInfo.HandlerBaseState[F];
+  }
+  for (const auto &MBB : *MF) {
+    for (const auto &MI : MBB) {
+      if (!MI.isEHLabel()) {
+        if (MI.isCall())
+          SawPotentiallyThrowing |= !callToNoUnwindFunction(&MI);
+        continue;
+      }
+
+      // End of the previous try-range?
+      MCSymbol *BeginLabel = MI.getOperand(0).getMCSymbol();
+      if (BeginLabel == LastLabel)
+        SawPotentiallyThrowing = false;
+
+      // Beginning of a new try-range?
+      RangeMapType::const_iterator L = PadMap.find(BeginLabel);
+      if (L == PadMap.end())
+        // Nope, it was just some random label.
+        continue;
+
+      const PadRange &P = L->second;
+      const LandingPadInfo *LandingPad = LandingPads[P.PadIndex];
+      assert(BeginLabel == LandingPad->BeginLabels[P.RangeIndex] &&
+             "Inconsistent landing pad map!");
+
+      // FIXME: Should this be using FuncInfo.HandlerBaseState?
+      if (SawPotentiallyThrowing && LastEHState != -1) {
+        FuncInfo.IPToStateList.push_back(std::make_pair(LastLabel, -1));
+        SawPotentiallyThrowing = false;
+        LastEHState = -1;
+      }
+
+      if (LandingPad->WinEHState != LastEHState)
+        FuncInfo.IPToStateList.push_back(
+            std::make_pair(BeginLabel, LandingPad->WinEHState));
+      LastEHState = LandingPad->WinEHState;
+      LastLabel = LandingPad->EndLabels[P.RangeIndex];
+    }
+  }
+}
+
+void WinException::emitEHRegistrationOffsetLabel(const WinEHFuncInfo &FuncInfo,
+                                                 StringRef FLinkageName) {
+  // Outlined helpers called by the EH runtime need to know the offset of the EH
+  // registration in order to recover the parent frame pointer. Now that we know
+  // we've code generated the parent, we can emit the label assignment that
+  // those helpers use to get the offset of the registration node.
+  assert(FuncInfo.EHRegNodeEscapeIndex != INT_MAX &&
+         "no EH reg node localescape index");
+  MCSymbol *ParentFrameOffset =
+      Asm->OutContext.getOrCreateParentFrameOffsetSymbol(FLinkageName);
+  MCSymbol *RegistrationOffsetSym = Asm->OutContext.getOrCreateFrameAllocSymbol(
+      FLinkageName, FuncInfo.EHRegNodeEscapeIndex);
+  const MCExpr *RegistrationOffsetSymRef =
+      MCSymbolRefExpr::create(RegistrationOffsetSym, Asm->OutContext);
+  Asm->OutStreamer->EmitAssignment(ParentFrameOffset, RegistrationOffsetSymRef);
+}
+
+/// Emit the language-specific data that _except_handler3 and 4 expect. This is
+/// functionally equivalent to the __C_specific_handler table, except it is
+/// indexed by state number instead of IP.
+void WinException::emitExceptHandlerTable(const MachineFunction *MF) {
+  MCStreamer &OS = *Asm->OutStreamer;
+  const Function *F = MF->getFunction();
+  StringRef FLinkageName = GlobalValue::getRealLinkageName(F->getName());
+
+  WinEHFuncInfo &FuncInfo = MMI->getWinEHFuncInfo(F);
+  emitEHRegistrationOffsetLabel(FuncInfo, FLinkageName);
+
+  // Emit the __ehtable label that we use for llvm.x86.seh.lsda.
+  MCSymbol *LSDALabel = Asm->OutContext.getOrCreateLSDASymbol(FLinkageName);
+  OS.EmitValueToAlignment(4);
+  OS.EmitLabel(LSDALabel);
+
+  const Function *Per =
+      dyn_cast<Function>(F->getPersonalityFn()->stripPointerCasts());
+  StringRef PerName = Per->getName();
+  int BaseState = -1;
+  if (PerName == "_except_handler4") {
+    // The LSDA for _except_handler4 starts with this struct, followed by the
+    // scope table:
+    //
+    // struct EH4ScopeTable {
+    //   int32_t GSCookieOffset;
+    //   int32_t GSCookieXOROffset;
+    //   int32_t EHCookieOffset;
+    //   int32_t EHCookieXOROffset;
+    //   ScopeTableEntry ScopeRecord[];
+    // };
+    //
+    // Only the EHCookieOffset field appears to vary, and it appears to be the
+    // offset from the final saved SP value to the retaddr.
+    OS.EmitIntValue(-2, 4);
+    OS.EmitIntValue(0, 4);
+    // FIXME: Calculate.
+    OS.EmitIntValue(9999, 4);
+    OS.EmitIntValue(0, 4);
+    BaseState = -2;
+  }
+
+  // Build a list of pointers to LandingPadInfos and then sort by WinEHState.
+  const std::vector<LandingPadInfo> &PadInfos = MMI->getLandingPads();
+  SmallVector<const LandingPadInfo *, 4> LPads;
+  LPads.reserve((PadInfos.size()));
+  for (const LandingPadInfo &LPInfo : PadInfos)
+    LPads.push_back(&LPInfo);
+  std::sort(LPads.begin(), LPads.end(),
+            [](const LandingPadInfo *L, const LandingPadInfo *R) {
+              return L->WinEHState < R->WinEHState;
+            });
+
+  // For each action in each lpad, emit one of these:
+  // struct ScopeTableEntry {
+  //   int32_t EnclosingLevel;
+  //   int32_t (__cdecl *Filter)();
+  //   void *HandlerOrFinally;
+  // };
+  //
+  // The "outermost" action will use BaseState as its enclosing level. Each
+  // other action will refer to the previous state as its enclosing level.
+  int CurState = 0;
+  for (const LandingPadInfo *LPInfo : LPads) {
+    int EnclosingLevel = BaseState;
+    assert(CurState + int(LPInfo->SEHHandlers.size()) - 1 ==
+               LPInfo->WinEHState &&
+           "gaps in the SEH scope table");
+    for (auto I = LPInfo->SEHHandlers.rbegin(), E = LPInfo->SEHHandlers.rend();
+         I != E; ++I) {
+      const SEHHandler &Handler = *I;
+      const BlockAddress *BA = Handler.RecoverBA;
+      const Function *F = Handler.FilterOrFinally;
+      assert(F && "cannot catch all in 32-bit SEH without filter function");
+      const MCExpr *FilterOrNull =
+          create32bitRef(BA ? Asm->getSymbol(F) : nullptr);
+      const MCExpr *ExceptOrFinally = create32bitRef(
+          BA ? Asm->GetBlockAddressSymbol(BA) : Asm->getSymbol(F));
+
+      OS.EmitIntValue(EnclosingLevel, 4);
+      OS.EmitValue(FilterOrNull, 4);
+      OS.EmitValue(ExceptOrFinally, 4);
+
+      // The next state unwinds to this state.
+      EnclosingLevel = CurState;
+      CurState++;
+    }
+  }
+}
diff --git a/lib/CodeGen/AsmPrinter/WinException.h b/lib/CodeGen/AsmPrinter/WinException.h
new file mode 100644
index 00000000000..669c9cc366b
--- /dev/null
+++ b/lib/CodeGen/AsmPrinter/WinException.h
@@ -0,0 +1,81 @@
+//===-- WinException.h - Windows Exception Handling ----------*- C++ -*--===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This file contains support for writing windows exception info into asm files.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIB_CODEGEN_ASMPRINTER_WIN64EXCEPTION_H
+#define LLVM_LIB_CODEGEN_ASMPRINTER_WIN64EXCEPTION_H
+
+#include "EHStreamer.h"
+
+namespace llvm {
+class Function;
+class GlobalValue;
+class MachineFunction;
+class MCExpr;
+struct WinEHFuncInfo;
+
+class LLVM_LIBRARY_VISIBILITY WinException : public EHStreamer {
+  /// Per-function flag to indicate if personality info should be emitted.
+  bool shouldEmitPersonality = false;
+
+  /// Per-function flag to indicate if the LSDA should be emitted.
+  bool shouldEmitLSDA = false;
+
+  /// Per-function flag to indicate if frame moves info should be emitted.
+  bool shouldEmitMoves = false;
+
+  /// True if this is a 64-bit target and we should use image relative offsets.
+  bool useImageRel32 = false;
+
+  void emitCSpecificHandlerTable();
+
+  /// Emit the EH table data for 32-bit and 64-bit functions using
+  /// the __CxxFrameHandler3 personality.
+  void emitCXXFrameHandler3Table(const MachineFunction *MF);
+
+  /// Emit the EH table data for _except_handler3 and _except_handler4
+  /// personality functions. These are only used on 32-bit and do not use CFI
+  /// tables.
+  void emitExceptHandlerTable(const MachineFunction *MF);
+
+  void extendIP2StateTable(const MachineFunction *MF, const Function *ParentF,
+                           WinEHFuncInfo &FuncInfo);
+
+  /// Emits the label used with llvm.x86.seh.recoverfp, which is used by
+  /// outlined funclets.
+  void emitEHRegistrationOffsetLabel(const WinEHFuncInfo &FuncInfo,
+                                     StringRef FLinkageName);
+
+  const MCExpr *create32bitRef(const MCSymbol *Value);
+  const MCExpr *create32bitRef(const GlobalValue *GV);
+
+public:
+  //===--------------------------------------------------------------------===//
+  // Main entry points.
+  //
+  WinException(AsmPrinter *A);
+  ~WinException() override;
+
+  /// Emit all exception information that should come after the content.
+  void endModule() override;
+
+  /// Gather pre-function exception information.  Assumes being emitted
+  /// immediately after the function entry point.
+  void beginFunction(const MachineFunction *MF) override;
+
+  /// Gather and emit post-function exception information.
+  void endFunction(const MachineFunction *) override;
+};
+}
+
+#endif
+
diff --git a/lib/CodeGen/AtomicExpandPass.cpp b/lib/CodeGen/AtomicExpandPass.cpp
index fa17108b2a8..530ab46db03 100644
--- a/lib/CodeGen/AtomicExpandPass.cpp
+++ b/lib/CodeGen/AtomicExpandPass.cpp
@@ -464,7 +464,7 @@ bool AtomicExpand::expandAtomicCmpXchg(AtomicCmpXchgInst *CI) {
   Value *ShouldStore =
       Builder.CreateICmpEQ(Loaded, CI->getCompareOperand(), "should_store");
 
-  // If the the cmpxchg doesn't actually need any ordering when it fails, we can
+  // If the cmpxchg doesn't actually need any ordering when it fails, we can
   // jump straight past that fence instruction (if it exists).
   Builder.CreateCondBr(ShouldStore, TryStoreBB, FailureBB);
 
diff --git a/lib/CodeGen/BasicTargetTransformInfo.cpp b/lib/CodeGen/BasicTargetTransformInfo.cpp
index 82f5c482408..db00910cd01 100644
--- a/lib/CodeGen/BasicTargetTransformInfo.cpp
+++ b/lib/CodeGen/BasicTargetTransformInfo.cpp
@@ -34,4 +34,5 @@ cl::opt<unsigned>
                                     cl::Hidden);
 
 BasicTTIImpl::BasicTTIImpl(const TargetMachine *TM, Function &F)
-    : BaseT(TM), ST(TM->getSubtargetImpl(F)), TLI(ST->getTargetLowering()) {}
+    : BaseT(TM, F.getParent()->getDataLayout()), ST(TM->getSubtargetImpl(F)),
+      TLI(ST->getTargetLowering()) {}
diff --git a/lib/CodeGen/BranchFolding.cpp b/lib/CodeGen/BranchFolding.cpp
index b3ceb91b1ef..fbf1504b548 100644
--- a/lib/CodeGen/BranchFolding.cpp
+++ b/lib/CodeGen/BranchFolding.cpp
@@ -12,7 +12,8 @@
 // it then removes.
 //
 // Note that this pass must be run after register allocation, it cannot handle
-// SSA form.
+// SSA form. It also must handle virtual registers for targets that emit virtual
+// ISA (e.g. NVPTX).
 //
 //===----------------------------------------------------------------------===//
 
@@ -22,7 +23,6 @@
 #include "llvm/ADT/Statistic.h"
 #include "llvm/CodeGen/MachineBlockFrequencyInfo.h"
 #include "llvm/CodeGen/MachineBranchProbabilityInfo.h"
-#include "llvm/CodeGen/MachineConstantPool.h" //  @LOCALMOD
 #include "llvm/CodeGen/MachineFunctionPass.h"
 #include "llvm/CodeGen/MachineJumpTableInfo.h"
 #include "llvm/CodeGen/MachineMemOperand.h"
@@ -151,9 +151,13 @@ bool BranchFolder::OptimizeImpDefsBlock(MachineBasicBlock *MBB) {
     if (!I->isImplicitDef())
       break;
     unsigned Reg = I->getOperand(0).getReg();
-    for (MCSubRegIterator SubRegs(Reg, TRI, /*IncludeSelf=*/true);
-         SubRegs.isValid(); ++SubRegs)
-      ImpDefRegs.insert(*SubRegs);
+    if (TargetRegisterInfo::isPhysicalRegister(Reg)) {
+      for (MCSubRegIterator SubRegs(Reg, TRI, /*IncludeSelf=*/true);
+           SubRegs.isValid(); ++SubRegs)
+        ImpDefRegs.insert(*SubRegs);
+    } else {
+      ImpDefRegs.insert(Reg);
+    }
     ++I;
   }
   if (ImpDefRegs.empty())
@@ -249,20 +253,6 @@ bool BranchFolder::OptimizeFunction(MachineFunction &MF,
       }
   }
 
-  // @LOCALMOD-START
-  // This currently only used on ARM targets where the ConstantPool
-  // subclass is overloading getJumpTableIndex()
-  const std::vector<MachineConstantPoolEntry>& CPs =
-    MF.getConstantPool()->getConstants();
-  for (unsigned i = 0, e = CPs.size(); i != e; ++i) {
-    if (!CPs[i].isMachineConstantPoolEntry()) continue;
-    unsigned *JTIndex = CPs[i].Val.MachineCPVal->getJumpTableIndex();
-    if (!JTIndex) continue;
-    // Remember that this JT is live.
-    JTIsLive.set(*JTIndex);
-  }
-  // @LOCALMOD-END
-
   // Finally, remove dead jump tables.  This happens when the
   // indirect jump was unreachable (and thus deleted).
   for (unsigned i = 0, e = JTIsLive.size(); i != e; ++i)
@@ -285,11 +275,17 @@ static unsigned HashMachineInstr(const MachineInstr *MI) {
   for (unsigned i = 0, e = MI->getNumOperands(); i != e; ++i) {
     const MachineOperand &Op = MI->getOperand(i);
 
-    // Merge in bits from the operand if easy.
+    // Merge in bits from the operand if easy. We can't use MachineOperand's
+    // hash_code here because it's not deterministic and we sort by hash value
+    // later.
     unsigned OperandHash = 0;
     switch (Op.getType()) {
-    case MachineOperand::MO_Register:          OperandHash = Op.getReg(); break;
-    case MachineOperand::MO_Immediate:         OperandHash = Op.getImm(); break;
+    case MachineOperand::MO_Register:
+      OperandHash = Op.getReg();
+      break;
+    case MachineOperand::MO_Immediate:
+      OperandHash = Op.getImm();
+      break;
     case MachineOperand::MO_MachineBasicBlock:
       OperandHash = Op.getMBB()->getNumber();
       break;
@@ -304,27 +300,20 @@ static unsigned HashMachineInstr(const MachineInstr *MI) {
       // pull in the offset.
       OperandHash = Op.getOffset();
       break;
-    default: break;
+    default:
+      break;
     }
 
-    Hash += ((OperandHash << 3) | Op.getType()) << (i&31);
+    Hash += ((OperandHash << 3) | Op.getType()) << (i & 31);
   }
   return Hash;
 }
 
 /// HashEndOfMBB - Hash the last instruction in the MBB.
 static unsigned HashEndOfMBB(const MachineBasicBlock *MBB) {
-  MachineBasicBlock::const_iterator I = MBB->end();
-  if (I == MBB->begin())
-    return 0;   // Empty MBB.
-
-  --I;
-  // Skip debug info so it will not affect codegen.
-  while (I->isDebugValue()) {
-    if (I==MBB->begin())
-      return 0;      // MBB empty except for debug info.
-    --I;
-  }
+  MachineBasicBlock::const_iterator I = MBB->getLastNonDebugInstr();
+  if (I == MBB->end())
+    return 0;
 
   return HashMachineInstr(I);
 }
@@ -1133,25 +1122,15 @@ bool BranchFolder::OptimizeBranches(MachineFunction &MF) {
 // Blocks should be considered empty if they contain only debug info;
 // else the debug info would affect codegen.
 static bool IsEmptyBlock(MachineBasicBlock *MBB) {
-  if (MBB->empty())
-    return true;
-  for (MachineBasicBlock::iterator MBBI = MBB->begin(), MBBE = MBB->end();
-       MBBI!=MBBE; ++MBBI) {
-    if (!MBBI->isDebugValue())
-      return false;
-  }
-  return true;
+  return MBB->getFirstNonDebugInstr() == MBB->end();
 }
 
 // Blocks with only debug info and branches should be considered the same
 // as blocks with only branches.
 static bool IsBranchOnlyBlock(MachineBasicBlock *MBB) {
-  MachineBasicBlock::iterator MBBI, MBBE;
-  for (MBBI = MBB->begin(), MBBE = MBB->end(); MBBI!=MBBE; ++MBBI) {
-    if (!MBBI->isDebugValue())
-      break;
-  }
-  return (MBBI->isBranch());
+  MachineBasicBlock::iterator I = MBB->getFirstNonDebugInstr();
+  assert(I != MBB->end() && "empty block!");
+  return I->isBranch();
 }
 
 /// IsBetterFallthrough - Return true if it would be clearly better to
@@ -1164,36 +1143,24 @@ static bool IsBetterFallthrough(MachineBasicBlock *MBB1,
   // MBB1 doesn't, we prefer to fall through into MBB1.  This allows us to
   // optimize branches that branch to either a return block or an assert block
   // into a fallthrough to the return.
-  if (IsEmptyBlock(MBB1) || IsEmptyBlock(MBB2)) return false;
+  MachineBasicBlock::iterator MBB1I = MBB1->getLastNonDebugInstr();
+  MachineBasicBlock::iterator MBB2I = MBB2->getLastNonDebugInstr();
+  if (MBB1I == MBB1->end() || MBB2I == MBB2->end())
+    return false;
 
   // If there is a clear successor ordering we make sure that one block
   // will fall through to the next
   if (MBB1->isSuccessor(MBB2)) return true;
   if (MBB2->isSuccessor(MBB1)) return false;
 
-  // Neither block consists entirely of debug info (per IsEmptyBlock check),
-  // so we needn't test for falling off the beginning here.
-  MachineBasicBlock::iterator MBB1I = --MBB1->end();
-  while (MBB1I->isDebugValue())
-    --MBB1I;
-  MachineBasicBlock::iterator MBB2I = --MBB2->end();
-  while (MBB2I->isDebugValue())
-    --MBB2I;
   return MBB2I->isCall() && !MBB1I->isCall();
 }
 
 /// getBranchDebugLoc - Find and return, if any, the DebugLoc of the branch
-/// instructions on the block. Always use the DebugLoc of the first
-/// branching instruction found unless its absent, in which case use the
-/// DebugLoc of the second if present.
+/// instructions on the block.
 static DebugLoc getBranchDebugLoc(MachineBasicBlock &MBB) {
-  MachineBasicBlock::iterator I = MBB.end();
-  if (I == MBB.begin())
-    return DebugLoc();
-  --I;
-  while (I->isDebugValue() && I != MBB.begin())
-    --I;
-  if (I->isBranch())
+  MachineBasicBlock::iterator I = MBB.getLastNonDebugInstr();
+  if (I != MBB.end() && I->isBranch())
     return I->getDebugLoc();
   return DebugLoc();
 }
@@ -1218,6 +1185,11 @@ bool BranchFolder::OptimizeBlock(MachineBasicBlock *MBB) {
 
     if (FallThrough == MF.end()) {
       // TODO: Simplify preds to not branch here if possible!
+    } else if (FallThrough->isLandingPad()) {
+      // Don't rewrite to a landing pad fallthough.  That could lead to the case
+      // where a BB jumps to more than one landing pad.
+      // TODO: Is it ever worth rewriting predecessors which don't already
+      // jump to a landing pad, and so can safely jump to the fallthrough?
     } else {
       // Rewrite all predecessors of the old block to go to the fallthrough
       // instead.
@@ -1413,19 +1385,10 @@ bool BranchFolder::OptimizeBlock(MachineBasicBlock *MBB) {
       // If the only things remaining in the block are debug info, remove these
       // as well, so this will behave the same as an empty block in non-debug
       // mode.
-      if (!MBB->empty()) {
-        bool NonDebugInfoFound = false;
-        for (MachineBasicBlock::iterator I = MBB->begin(), E = MBB->end();
-             I != E; ++I) {
-          if (!I->isDebugValue()) {
-            NonDebugInfoFound = true;
-            break;
-          }
-        }
-        if (!NonDebugInfoFound)
-          // Make the block empty, losing the debug info (we could probably
-          // improve this in some cases.)
-          MBB->erase(MBB->begin(), MBB->end());
+      if (IsEmptyBlock(MBB)) {
+        // Make the block empty, losing the debug info (we could probably
+        // improve this in some cases.)
+        MBB->erase(MBB->begin(), MBB->end());
       }
       // If this block is just an unconditional branch to CurTBB, we can
       // usually completely eliminate the block.  The only case we cannot
@@ -1615,6 +1578,17 @@ static MachineBasicBlock *findFalseBlock(MachineBasicBlock *BB,
   return nullptr;
 }
 
+template <class Container>
+static void addRegAndItsAliases(unsigned Reg, const TargetRegisterInfo *TRI,
+                                Container &Set) {
+  if (TargetRegisterInfo::isPhysicalRegister(Reg)) {
+    for (MCRegAliasIterator AI(Reg, TRI, true); AI.isValid(); ++AI)
+      Set.insert(*AI);
+  } else {
+    Set.insert(Reg);
+  }
+}
+
 /// findHoistingInsertPosAndDeps - Find the location to move common instructions
 /// in successors to. The location is usually just before the terminator,
 /// however if the terminator is a conditional branch and its previous
@@ -1640,8 +1614,7 @@ MachineBasicBlock::iterator findHoistingInsertPosAndDeps(MachineBasicBlock *MBB,
     if (!Reg)
       continue;
     if (MO.isUse()) {
-      for (MCRegAliasIterator AI(Reg, TRI, true); AI.isValid(); ++AI)
-        Uses.insert(*AI);
+      addRegAndItsAliases(Reg, TRI, Uses);
     } else {
       if (!MO.isDead())
         // Don't try to hoist code in the rare case the terminator defines a
@@ -1650,8 +1623,7 @@ MachineBasicBlock::iterator findHoistingInsertPosAndDeps(MachineBasicBlock *MBB,
 
       // If the terminator defines a register, make sure we don't hoist
       // the instruction whose def might be clobbered by the terminator.
-      for (MCRegAliasIterator AI(Reg, TRI, true); AI.isValid(); ++AI)
-        Defs.insert(*AI);
+      addRegAndItsAliases(Reg, TRI, Defs);
     }
   }
 
@@ -1693,8 +1665,7 @@ MachineBasicBlock::iterator findHoistingInsertPosAndDeps(MachineBasicBlock *MBB,
   // Also avoid moving code above predicated instruction since it's hard to
   // reason about register liveness with predicated instruction.
   bool DontMoveAcrossStore = true;
-  if (!PI->isSafeToMove(TII, nullptr, DontMoveAcrossStore) ||
-      TII->isPredicated(PI))
+  if (!PI->isSafeToMove(nullptr, DontMoveAcrossStore) || TII->isPredicated(PI))
     return MBB->end();
 
 
@@ -1708,15 +1679,15 @@ MachineBasicBlock::iterator findHoistingInsertPosAndDeps(MachineBasicBlock *MBB,
     if (!Reg)
       continue;
     if (MO.isUse()) {
-      for (MCRegAliasIterator AI(Reg, TRI, true); AI.isValid(); ++AI)
-        Uses.insert(*AI);
+      addRegAndItsAliases(Reg, TRI, Uses);
     } else {
       if (Uses.erase(Reg)) {
-        for (MCSubRegIterator SubRegs(Reg, TRI); SubRegs.isValid(); ++SubRegs)
-          Uses.erase(*SubRegs); // Use sub-registers to be conservative
+        if (TargetRegisterInfo::isPhysicalRegister(Reg)) {
+          for (MCSubRegIterator SubRegs(Reg, TRI); SubRegs.isValid(); ++SubRegs)
+            Uses.erase(*SubRegs); // Use sub-registers to be conservative
+        }
       }
-      for (MCRegAliasIterator AI(Reg, TRI, true); AI.isValid(); ++AI)
-        Defs.insert(*AI);
+      addRegAndItsAliases(Reg, TRI, Defs);
     }
   }
 
@@ -1832,7 +1803,7 @@ bool BranchFolder::HoistCommonCodeInSuccs(MachineBasicBlock *MBB) {
       break;
 
     bool DontMoveAcrossStore = true;
-    if (!TIB->isSafeToMove(TII, nullptr, DontMoveAcrossStore))
+    if (!TIB->isSafeToMove(nullptr, DontMoveAcrossStore))
       break;
 
     // Remove kills from LocalDefsSet, these registers had short live ranges.
@@ -1843,8 +1814,12 @@ bool BranchFolder::HoistCommonCodeInSuccs(MachineBasicBlock *MBB) {
       unsigned Reg = MO.getReg();
       if (!Reg || !LocalDefsSet.count(Reg))
         continue;
-      for (MCRegAliasIterator AI(Reg, TRI, true); AI.isValid(); ++AI)
-        LocalDefsSet.erase(*AI);
+      if (TargetRegisterInfo::isPhysicalRegister(Reg)) {
+        for (MCRegAliasIterator AI(Reg, TRI, true); AI.isValid(); ++AI)
+          LocalDefsSet.erase(*AI);
+      } else {
+        LocalDefsSet.erase(Reg);
+      }
     }
 
     // Track local defs so we can update liveins.
@@ -1856,8 +1831,7 @@ bool BranchFolder::HoistCommonCodeInSuccs(MachineBasicBlock *MBB) {
       if (!Reg)
         continue;
       LocalDefs.push_back(Reg);
-      for (MCRegAliasIterator AI(Reg, TRI, true); AI.isValid(); ++AI)
-        LocalDefsSet.insert(*AI);
+      addRegAndItsAliases(Reg, TRI, LocalDefsSet);
     }
 
     HasDups = true;
diff --git a/lib/CodeGen/BranchFolding.h b/lib/CodeGen/BranchFolding.h
index 3653a2ccd62..46c05dc0600 100644
--- a/lib/CodeGen/BranchFolding.h
+++ b/lib/CodeGen/BranchFolding.h
@@ -24,7 +24,7 @@ namespace llvm {
   class TargetInstrInfo;
   class TargetRegisterInfo;
 
-  class BranchFolder {
+  class LLVM_LIBRARY_VISIBILITY BranchFolder {
   public:
     explicit BranchFolder(bool defaultEnableTailMerge, bool CommonHoist,
                           const MachineBlockFrequencyInfo &MBFI,
diff --git a/lib/CodeGen/CMakeLists.txt b/lib/CodeGen/CMakeLists.txt
index ef5763855a8..eb7552970d3 100644
--- a/lib/CodeGen/CMakeLists.txt
+++ b/lib/CodeGen/CMakeLists.txt
@@ -9,6 +9,7 @@ add_llvm_library(LLVMCodeGen
   CallingConvLower.cpp
   CodeGen.cpp
   CodeGenPrepare.cpp
+  CoreCLRGC.cpp
   CriticalAntiDepBreaker.cpp
   DFAPacketizer.cpp
   DeadMachineInstructionElim.cpp
@@ -19,14 +20,17 @@ add_llvm_library(LLVMCodeGen
   ExecutionDepsFix.cpp
   ExpandISelPseudos.cpp
   ExpandPostRAPseudos.cpp
+  FaultMaps.cpp
   GCMetadata.cpp
   GCMetadataPrinter.cpp
   GCRootLowering.cpp
   GCStrategy.cpp
   GlobalMerge.cpp
   IfConversion.cpp
+  ImplicitNullChecks.cpp
   InlineSpiller.cpp
   InterferenceCache.cpp
+  InterleavedAccessPass.cpp
   IntrinsicLowering.cpp
   LLVMTargetMachine.cpp
   LatencyPriorityQueue.cpp
@@ -70,6 +74,8 @@ add_llvm_library(LLVMCodeGen
   MachineSink.cpp
   MachineTraceMetrics.cpp
   MachineVerifier.cpp
+  MIRPrinter.cpp
+  MIRPrintingPass.cpp
   OcamlGC.cpp
   OptimizePHIs.cpp
   PHIElimination.cpp
@@ -93,6 +99,7 @@ add_llvm_library(LLVMCodeGen
   ScheduleDAGInstrs.cpp
   ScheduleDAGPrinter.cpp
   ScoreboardHazardRecognizer.cpp
+  ShrinkWrap.cpp
   ShadowStackGC.cpp
   ShadowStackGCLowering.cpp
   SjLjEHPrepare.cpp
@@ -127,3 +134,4 @@ add_dependencies(LLVMCodeGen intrinsics_gen)
 
 add_subdirectory(SelectionDAG)
 add_subdirectory(AsmPrinter)
+add_subdirectory(MIRParser)
diff --git a/lib/CodeGen/CallingConvLower.cpp b/lib/CodeGen/CallingConvLower.cpp
index 034ffb34b9c..fb29b1db7a4 100644
--- a/lib/CodeGen/CallingConvLower.cpp
+++ b/lib/CodeGen/CallingConvLower.cpp
@@ -37,9 +37,9 @@ CCState::CCState(CallingConv::ID CC, bool isVarArg, MachineFunction &mf,
   UsedRegs.resize((TRI.getNumRegs()+31)/32);
 }
 
-// HandleByVal - Allocate space on the stack large enough to pass an argument
-// by value. The size and alignment information of the argument is encoded in
-// its parameter attribute.
+/// Allocate space on the stack large enough to pass an argument by value.
+/// The size and alignment information of the argument is encoded in
+/// its parameter attribute.
 void CCState::HandleByVal(unsigned ValNo, MVT ValVT,
                           MVT LocVT, CCValAssign::LocInfo LocInfo,
                           int MinSize, int MinAlign,
@@ -57,13 +57,13 @@ void CCState::HandleByVal(unsigned ValNo, MVT ValVT,
   addLoc(CCValAssign::getMem(ValNo, ValVT, Offset, LocVT, LocInfo));
 }
 
-/// MarkAllocated - Mark a register and all of its aliases as allocated.
+/// Mark a register and all of its aliases as allocated.
 void CCState::MarkAllocated(unsigned Reg) {
   for (MCRegAliasIterator AI(Reg, &TRI, true); AI.isValid(); ++AI)
     UsedRegs[*AI/32] |= 1 << (*AI&31);
 }
 
-/// AnalyzeFormalArguments - Analyze an array of argument values,
+/// Analyze an array of argument values,
 /// incorporating info about the formals into this state.
 void
 CCState::AnalyzeFormalArguments(const SmallVectorImpl<ISD::InputArg> &Ins,
@@ -83,8 +83,8 @@ CCState::AnalyzeFormalArguments(const SmallVectorImpl<ISD::InputArg> &Ins,
   }
 }
 
-/// CheckReturn - Analyze the return values of a function, returning true if
-/// the return can be performed without sret-demotion, and false otherwise.
+/// Analyze the return values of a function, returning true if the return can
+/// be performed without sret-demotion and false otherwise.
 bool CCState::CheckReturn(const SmallVectorImpl<ISD::OutputArg> &Outs,
                           CCAssignFn Fn) {
   // Determine which register each value should be copied into.
@@ -97,7 +97,7 @@ bool CCState::CheckReturn(const SmallVectorImpl<ISD::OutputArg> &Outs,
   return true;
 }
 
-/// AnalyzeReturn - Analyze the returned values of a return,
+/// Analyze the returned values of a return,
 /// incorporating info about the result values into this state.
 void CCState::AnalyzeReturn(const SmallVectorImpl<ISD::OutputArg> &Outs,
                             CCAssignFn Fn) {
@@ -115,7 +115,7 @@ void CCState::AnalyzeReturn(const SmallVectorImpl<ISD::OutputArg> &Outs,
   }
 }
 
-/// AnalyzeCallOperands - Analyze the outgoing arguments to a call,
+/// Analyze the outgoing arguments to a call,
 /// incorporating info about the passed values into this state.
 void CCState::AnalyzeCallOperands(const SmallVectorImpl<ISD::OutputArg> &Outs,
                                   CCAssignFn Fn) {
@@ -133,8 +133,7 @@ void CCState::AnalyzeCallOperands(const SmallVectorImpl<ISD::OutputArg> &Outs,
   }
 }
 
-/// AnalyzeCallOperands - Same as above except it takes vectors of types
-/// and argument flags.
+/// Same as above except it takes vectors of types and argument flags.
 void CCState::AnalyzeCallOperands(SmallVectorImpl<MVT> &ArgVTs,
                                   SmallVectorImpl<ISD::ArgFlagsTy> &Flags,
                                   CCAssignFn Fn) {
@@ -152,8 +151,8 @@ void CCState::AnalyzeCallOperands(SmallVectorImpl<MVT> &ArgVTs,
   }
 }
 
-/// AnalyzeCallResult - Analyze the return values of a call,
-/// incorporating info about the passed values into this state.
+/// Analyze the return values of a call, incorporating info about the passed
+/// values into this state.
 void CCState::AnalyzeCallResult(const SmallVectorImpl<ISD::InputArg> &Ins,
                                 CCAssignFn Fn) {
   for (unsigned i = 0, e = Ins.size(); i != e; ++i) {
@@ -169,8 +168,7 @@ void CCState::AnalyzeCallResult(const SmallVectorImpl<ISD::InputArg> &Ins,
   }
 }
 
-/// AnalyzeCallResult - Same as above except it's specialized for calls which
-/// produce a single value.
+/// Same as above except it's specialized for calls that produce a single value.
 void CCState::AnalyzeCallResult(MVT VT, CCAssignFn Fn) {
   if (Fn(0, VT, VT, CCValAssign::Full, ISD::ArgFlagsTy(), *this)) {
 #ifndef NDEBUG
diff --git a/lib/CodeGen/CodeGen.cpp b/lib/CodeGen/CodeGen.cpp
index da66639d02f..155c5ecec77 100644
--- a/lib/CodeGen/CodeGen.cpp
+++ b/lib/CodeGen/CodeGen.cpp
@@ -42,6 +42,7 @@ void llvm::initializeCodeGen(PassRegistry &Registry) {
   initializeMachineBlockPlacementPass(Registry);
   initializeMachineBlockPlacementStatsPass(Registry);
   initializeMachineCSEPass(Registry);
+  initializeImplicitNullChecksPass(Registry);
   initializeMachineCombinerPass(Registry);
   initializeMachineCopyPropagationPass(Registry);
   initializeMachineDominatorTreePass(Registry);
@@ -61,6 +62,7 @@ void llvm::initializeCodeGen(PassRegistry &Registry) {
   initializePostRASchedulerPass(Registry);
   initializeProcessImplicitDefsPass(Registry);
   initializeRegisterCoalescerPass(Registry);
+  initializeShrinkWrapPass(Registry);
   initializeSlotIndexesPass(Registry);
   initializeStackColoringPass(Registry);
   initializeStackMapLivenessPass(Registry);
diff --git a/lib/CodeGen/CodeGenPrepare.cpp b/lib/CodeGen/CodeGenPrepare.cpp
index 35376e162d3..2554ccf0f7b 100644
--- a/lib/CodeGen/CodeGenPrepare.cpp
+++ b/lib/CodeGen/CodeGenPrepare.cpp
@@ -109,11 +109,7 @@ static cl::opt<bool> StressExtLdPromotion(
 
 namespace {
 typedef SmallPtrSet<Instruction *, 16> SetOfInstrs;
-struct TypeIsSExt {
-  Type *Ty;
-  bool IsSExt;
-  TypeIsSExt(Type *Ty, bool IsSExt) : Ty(Ty), IsSExt(IsSExt) {}
-};
+typedef PointerIntPair<Type *, 1, bool> TypeIsSExt;
 typedef DenseMap<Instruction *, TypeIsSExt> InstrToOrigTy;
 class TypePromotionTransaction;
 
@@ -135,8 +131,8 @@ class TypePromotionTransaction;
     /// multiple load/stores of the same address.
     ValueMap<Value*, Value*> SunkAddrs;
 
-    /// Keeps track of all truncates inserted for the current function.
-    SetOfInstrs InsertedTruncsSet;
+    /// Keeps track of all instructions inserted for the current function.
+    SetOfInstrs InsertedInsts;
     /// Keeps track of the type of the related instruction before their
     /// promotion for the current function.
     InstrToOrigTy PromotedInsts;
@@ -147,10 +143,13 @@ class TypePromotionTransaction;
     /// OptSize - True if optimizing for size.
     bool OptSize;
 
+    /// DataLayout for the Function being processed.
+    const DataLayout *DL;
+
   public:
     static char ID; // Pass identification, replacement for typeid
     explicit CodeGenPrepare(const TargetMachine *TM = nullptr)
-        : FunctionPass(ID), TM(TM), TLI(nullptr), TTI(nullptr) {
+        : FunctionPass(ID), TM(TM), TLI(nullptr), TTI(nullptr), DL(nullptr) {
         initializeCodeGenPreparePass(*PassRegistry::getPassRegistry());
       }
     bool runOnFunction(Function &F) override;
@@ -170,7 +169,8 @@ class TypePromotionTransaction;
     void EliminateMostlyEmptyBlock(BasicBlock *BB);
     bool OptimizeBlock(BasicBlock &BB, bool& ModifiedDT);
     bool OptimizeInst(Instruction *I, bool& ModifiedDT);
-    bool OptimizeMemoryInst(Instruction *I, Value *Addr, Type *AccessTy);
+    bool OptimizeMemoryInst(Instruction *I, Value *Addr,
+                            Type *AccessTy, unsigned AS);
     bool OptimizeInlineAsmInst(CallInst *CS);
     bool OptimizeCallInst(CallInst *CI, bool& ModifiedDT);
     bool MoveExtToFormExtLoad(Instruction *&I);
@@ -202,9 +202,11 @@ bool CodeGenPrepare::runOnFunction(Function &F) {
   if (skipOptnoneFunction(F))
     return false;
 
+  DL = &F.getParent()->getDataLayout();
+
   bool EverMadeChange = false;
   // Clear per function information.
-  InsertedTruncsSet.clear();
+  InsertedInsts.clear();
   PromotedInsts.clear();
 
   ModifiedDT = false;
@@ -531,8 +533,8 @@ static void computeBaseDerivedRelocateMap(
   for (auto &U : AllRelocateCalls) {
     GCRelocateOperands ThisRelocate(U);
     IntrinsicInst *I = cast<IntrinsicInst>(U);
-    auto K = std::make_pair(ThisRelocate.basePtrIndex(),
-                            ThisRelocate.derivedPtrIndex());
+    auto K = std::make_pair(ThisRelocate.getBasePtrIndex(),
+                            ThisRelocate.getDerivedPtrIndex());
     RelocateIdxMap.insert(std::make_pair(K, I));
   }
   for (auto &Item : RelocateIdxMap) {
@@ -581,15 +583,15 @@ simplifyRelocatesOffABase(IntrinsicInst *RelocatedBase,
     GCRelocateOperands MasterRelocate(RelocatedBase);
     GCRelocateOperands ThisRelocate(ToReplace);
 
-    assert(ThisRelocate.basePtrIndex() == MasterRelocate.basePtrIndex() &&
+    assert(ThisRelocate.getBasePtrIndex() == MasterRelocate.getBasePtrIndex() &&
            "Not relocating a derived object of the original base object");
-    if (ThisRelocate.basePtrIndex() == ThisRelocate.derivedPtrIndex()) {
+    if (ThisRelocate.getBasePtrIndex() == ThisRelocate.getDerivedPtrIndex()) {
       // A duplicate relocate call. TODO: coalesce duplicates.
       continue;
     }
 
-    Value *Base = ThisRelocate.basePtr();
-    auto Derived = dyn_cast<GetElementPtrInst>(ThisRelocate.derivedPtr());
+    Value *Base = ThisRelocate.getBasePtr();
+    auto Derived = dyn_cast<GetElementPtrInst>(ThisRelocate.getDerivedPtr());
     if (!Derived || Derived->getPointerOperand() != Base)
       continue;
 
@@ -598,15 +600,50 @@ simplifyRelocatesOffABase(IntrinsicInst *RelocatedBase,
       continue;
 
     // Create a Builder and replace the target callsite with a gep
-    IRBuilder<> Builder(ToReplace);
+    assert(RelocatedBase->getNextNode() && "Should always have one since it's not a terminator");
+
+    // Insert after RelocatedBase
+    IRBuilder<> Builder(RelocatedBase->getNextNode());
     Builder.SetCurrentDebugLocation(ToReplace->getDebugLoc());
+
+    // If gc_relocate does not match the actual type, cast it to the right type.
+    // In theory, there must be a bitcast after gc_relocate if the type does not
+    // match, and we should reuse it to get the derived pointer. But it could be
+    // cases like this:
+    // bb1:
+    //  ...
+    //  %g1 = call coldcc i8 addrspace(1)* @llvm.experimental.gc.relocate.p1i8(...)
+    //  br label %merge
+    //
+    // bb2:
+    //  ...
+    //  %g2 = call coldcc i8 addrspace(1)* @llvm.experimental.gc.relocate.p1i8(...)
+    //  br label %merge
+    //
+    // merge:
+    //  %p1 = phi i8 addrspace(1)* [ %g1, %bb1 ], [ %g2, %bb2 ]
+    //  %cast = bitcast i8 addrspace(1)* %p1 in to i32 addrspace(1)*
+    //
+    // In this case, we can not find the bitcast any more. So we insert a new bitcast
+    // no matter there is already one or not. In this way, we can handle all cases, and
+    // the extra bitcast should be optimized away in later passes.
+    Instruction *ActualRelocatedBase = RelocatedBase;
+    if (RelocatedBase->getType() != Base->getType()) {
+      ActualRelocatedBase =
+          cast<Instruction>(Builder.CreateBitCast(RelocatedBase, Base->getType()));
+    }
     Value *Replacement = Builder.CreateGEP(
-        Derived->getSourceElementType(), RelocatedBase, makeArrayRef(OffsetV));
+        Derived->getSourceElementType(), ActualRelocatedBase, makeArrayRef(OffsetV));
     Instruction *ReplacementInst = cast<Instruction>(Replacement);
-    ReplacementInst->removeFromParent();
-    ReplacementInst->insertAfter(RelocatedBase);
     Replacement->takeName(ToReplace);
-    ToReplace->replaceAllUsesWith(Replacement);
+    // If the newly generated derived pointer's type does not match the original derived
+    // pointer's type, cast the new derived pointer to match it. Same reasoning as above.
+    Instruction *ActualReplacement = ReplacementInst;
+    if (ReplacementInst->getType() != ToReplace->getType()) {
+      ActualReplacement =
+          cast<Instruction>(Builder.CreateBitCast(ReplacementInst, ToReplace->getType()));
+    }
+    ToReplace->replaceAllUsesWith(ActualReplacement);
     ToReplace->eraseFromParent();
 
     MadeChange = true;
@@ -717,10 +754,11 @@ static bool SinkCast(CastInst *CI) {
 ///
 /// Return true if any changes are made.
 ///
-static bool OptimizeNoopCopyExpression(CastInst *CI, const TargetLowering &TLI){
+static bool OptimizeNoopCopyExpression(CastInst *CI, const TargetLowering &TLI,
+                                       const DataLayout &DL) {
   // If this is a noop copy,
-  EVT SrcVT = TLI.getValueType(CI->getOperand(0)->getType());
-  EVT DstVT = TLI.getValueType(CI->getType());
+  EVT SrcVT = TLI.getValueType(DL, CI->getOperand(0)->getType());
+  EVT DstVT = TLI.getValueType(DL, CI->getType());
 
   // This is an fp<->int conversion?
   if (SrcVT.isInteger() != DstVT.isInteger())
@@ -885,7 +923,7 @@ static bool isExtractBitsCandidateUse(Instruction *User) {
 static bool
 SinkShiftAndTruncate(BinaryOperator *ShiftI, Instruction *User, ConstantInt *CI,
                      DenseMap<BasicBlock *, BinaryOperator *> &InsertedShifts,
-                     const TargetLowering &TLI) {
+                     const TargetLowering &TLI, const DataLayout &DL) {
   BasicBlock *UserBB = User->getParent();
   DenseMap<BasicBlock *, CastInst *> InsertedTruncs;
   TruncInst *TruncI = dyn_cast<TruncInst>(User);
@@ -911,7 +949,7 @@ SinkShiftAndTruncate(BinaryOperator *ShiftI, Instruction *User, ConstantInt *CI,
     // approximation; some nodes' legality is determined by the
     // operand or other means. There's no good way to find out though.
     if (TLI.isOperationLegalOrCustom(
-            ISDOpcode, TLI.getValueType(TruncUser->getType(), true)))
+            ISDOpcode, TLI.getValueType(DL, TruncUser->getType(), true)))
       continue;
 
     // Don't bother for PHI nodes.
@@ -969,13 +1007,14 @@ SinkShiftAndTruncate(BinaryOperator *ShiftI, Instruction *User, ConstantInt *CI,
 /// instruction.
 /// Return true if any changes are made.
 static bool OptimizeExtractBits(BinaryOperator *ShiftI, ConstantInt *CI,
-                                const TargetLowering &TLI) {
+                                const TargetLowering &TLI,
+                                const DataLayout &DL) {
   BasicBlock *DefBB = ShiftI->getParent();
 
   /// Only insert instructions in each block once.
   DenseMap<BasicBlock *, BinaryOperator *> InsertedShifts;
 
-  bool shiftIsLegal = TLI.isTypeLegal(TLI.getValueType(ShiftI->getType()));
+  bool shiftIsLegal = TLI.isTypeLegal(TLI.getValueType(DL, ShiftI->getType()));
 
   bool MadeChange = false;
   for (Value::user_iterator UI = ShiftI->user_begin(), E = ShiftI->user_end();
@@ -1012,9 +1051,10 @@ static bool OptimizeExtractBits(BinaryOperator *ShiftI, ConstantInt *CI,
       if (isa<TruncInst>(User) && shiftIsLegal
           // If the type of the truncate is legal, no trucate will be
           // introduced in other basic blocks.
-          && (!TLI.isTypeLegal(TLI.getValueType(User->getType()))))
+          &&
+          (!TLI.isTypeLegal(TLI.getValueType(DL, User->getType()))))
         MadeChange =
-            SinkShiftAndTruncate(ShiftI, User, CI, InsertedShifts, TLI);
+            SinkShiftAndTruncate(ShiftI, User, CI, InsertedShifts, TLI, DL);
 
       continue;
     }
@@ -1271,12 +1311,10 @@ bool CodeGenPrepare::OptimizeCallInst(CallInst *CI, bool& ModifiedDT) {
       return true;
   }
 
-  const DataLayout *TD = TLI ? TLI->getDataLayout() : nullptr;
-
   // Align the pointer arguments to this call if the target thinks it's a good
   // idea
   unsigned MinSize, PrefAlign;
-  if (TLI && TD && TLI->shouldAlignPointerArgs(CI, MinSize, PrefAlign)) {
+  if (TLI && TLI->shouldAlignPointerArgs(CI, MinSize, PrefAlign)) {
     for (auto &Arg : CI->arg_operands()) {
       // We want to align both objects whose address is used directly and
       // objects whose address is used in casts and GEPs, though it only makes
@@ -1284,36 +1322,34 @@ bool CodeGenPrepare::OptimizeCallInst(CallInst *CI, bool& ModifiedDT) {
       // if size - offset meets the size threshold.
       if (!Arg->getType()->isPointerTy())
         continue;
-      APInt Offset(TD->getPointerSizeInBits(
-                     cast<PointerType>(Arg->getType())->getAddressSpace()), 0);
-      Value *Val = Arg->stripAndAccumulateInBoundsConstantOffsets(*TD, Offset);
+      APInt Offset(DL->getPointerSizeInBits(
+                       cast<PointerType>(Arg->getType())->getAddressSpace()),
+                   0);
+      Value *Val = Arg->stripAndAccumulateInBoundsConstantOffsets(*DL, Offset);
       uint64_t Offset2 = Offset.getLimitedValue();
       if ((Offset2 & (PrefAlign-1)) != 0)
         continue;
       AllocaInst *AI;
-      if ((AI = dyn_cast<AllocaInst>(Val)) &&
-          AI->getAlignment() < PrefAlign &&
-          TD->getTypeAllocSize(AI->getAllocatedType()) >= MinSize + Offset2)
+      if ((AI = dyn_cast<AllocaInst>(Val)) && AI->getAlignment() < PrefAlign &&
+          DL->getTypeAllocSize(AI->getAllocatedType()) >= MinSize + Offset2)
         AI->setAlignment(PrefAlign);
       // Global variables can only be aligned if they are defined in this
       // object (i.e. they are uniquely initialized in this object), and
       // over-aligning global variables that have an explicit section is
       // forbidden.
       GlobalVariable *GV;
-      if ((GV = dyn_cast<GlobalVariable>(Val)) &&
-          GV->hasUniqueInitializer() &&
-          !GV->hasSection() &&
-          GV->getAlignment() < PrefAlign &&
-          TD->getTypeAllocSize(
-            GV->getType()->getElementType()) >= MinSize + Offset2)
+      if ((GV = dyn_cast<GlobalVariable>(Val)) && GV->hasUniqueInitializer() &&
+          !GV->hasSection() && GV->getAlignment() < PrefAlign &&
+          DL->getTypeAllocSize(GV->getType()->getElementType()) >=
+              MinSize + Offset2)
         GV->setAlignment(PrefAlign);
     }
     // If this is a memcpy (or similar) then we may be able to improve the
     // alignment
     if (MemIntrinsic *MI = dyn_cast<MemIntrinsic>(CI)) {
-      unsigned Align = getKnownAlignment(MI->getDest(), *TD);
+      unsigned Align = getKnownAlignment(MI->getDest(), *DL);
       if (MemTransferInst *MTI = dyn_cast<MemTransferInst>(MI))
-        Align = std::min(Align, getKnownAlignment(MTI->getSource(), *TD));
+        Align = std::min(Align, getKnownAlignment(MTI->getSource(), *DL));
       if (Align > MI->getAlignment())
         MI->setAlignment(ConstantInt::get(MI->getAlignmentType(), Align));
     }
@@ -1362,14 +1398,31 @@ bool CodeGenPrepare::OptimizeCallInst(CallInst *CI, bool& ModifiedDT) {
       }
       return false;
     }
+    case Intrinsic::aarch64_stlxr:
+    case Intrinsic::aarch64_stxr: {
+      ZExtInst *ExtVal = dyn_cast<ZExtInst>(CI->getArgOperand(0));
+      if (!ExtVal || !ExtVal->hasOneUse() ||
+          ExtVal->getParent() == CI->getParent())
+        return false;
+      // Sink a zext feeding stlxr/stxr before it, so it can be folded into it.
+      ExtVal->moveBefore(CI);
+      // Mark this instruction as "inserted by CGP", so that other
+      // optimizations don't touch it.
+      InsertedInsts.insert(ExtVal);
+      return true;
+    }
     }
 
     if (TLI) {
+      // Unknown address space.
+      // TODO: Target hook to pick which address space the intrinsic cares
+      // about?
+      unsigned AddrSpace = ~0u;
       SmallVector<Value*, 2> PtrOps;
       Type *AccessTy;
-      if (TLI->GetAddrModeArguments(II, PtrOps, AccessTy))
+      if (TLI->GetAddrModeArguments(II, PtrOps, AccessTy, AddrSpace))
         while (!PtrOps.empty())
-          if (OptimizeMemoryInst(II, PtrOps.pop_back_val(), AccessTy))
+          if (OptimizeMemoryInst(II, PtrOps.pop_back_val(), AccessTy, AddrSpace))
             return true;
     }
   }
@@ -2046,18 +2099,20 @@ class AddressingModeMatcher {
   SmallVectorImpl<Instruction*> &AddrModeInsts;
   const TargetMachine &TM;
   const TargetLowering &TLI;
+  const DataLayout &DL;
 
   /// AccessTy/MemoryInst - This is the type for the access (e.g. double) and
   /// the memory instruction that we're computing this address for.
   Type *AccessTy;
+  unsigned AddrSpace;
   Instruction *MemoryInst;
 
   /// AddrMode - This is the addressing mode that we're building up.  This is
   /// part of the return value of this addressing mode matching stuff.
   ExtAddrMode &AddrMode;
 
-  /// The truncate instruction inserted by other CodeGenPrepare optimizations.
-  const SetOfInstrs &InsertedTruncs;
+  /// The instructions inserted by other CodeGenPrepare optimizations.
+  const SetOfInstrs &InsertedInsts;
   /// A map from the instructions to their type before promotion.
   InstrToOrigTy &PromotedInsts;
   /// The ongoing transaction where every action should be registered.
@@ -2069,15 +2124,17 @@ class AddressingModeMatcher {
   bool IgnoreProfitability;
 
   AddressingModeMatcher(SmallVectorImpl<Instruction *> &AMI,
-                        const TargetMachine &TM, Type *AT, Instruction *MI,
-                        ExtAddrMode &AM, const SetOfInstrs &InsertedTruncs,
+                        const TargetMachine &TM, Type *AT, unsigned AS,
+                        Instruction *MI, ExtAddrMode &AM,
+                        const SetOfInstrs &InsertedInsts,
                         InstrToOrigTy &PromotedInsts,
                         TypePromotionTransaction &TPT)
       : AddrModeInsts(AMI), TM(TM),
         TLI(*TM.getSubtargetImpl(*MI->getParent()->getParent())
                  ->getTargetLowering()),
-        AccessTy(AT), MemoryInst(MI), AddrMode(AM),
-        InsertedTruncs(InsertedTruncs), PromotedInsts(PromotedInsts), TPT(TPT) {
+        DL(MI->getModule()->getDataLayout()), AccessTy(AT), AddrSpace(AS),
+        MemoryInst(MI), AddrMode(AM), InsertedInsts(InsertedInsts),
+        PromotedInsts(PromotedInsts), TPT(TPT) {
     IgnoreProfitability = false;
   }
 public:
@@ -2085,22 +2142,21 @@ class AddressingModeMatcher {
   /// Match - Find the maximal addressing mode that a load/store of V can fold,
   /// give an access type of AccessTy.  This returns a list of involved
   /// instructions in AddrModeInsts.
-  /// \p InsertedTruncs The truncate instruction inserted by other
-  /// CodeGenPrepare
+  /// \p InsertedInsts The instructions inserted by other CodeGenPrepare
   /// optimizations.
   /// \p PromotedInsts maps the instructions to their type before promotion.
   /// \p The ongoing transaction where every action should be registered.
-  static ExtAddrMode Match(Value *V, Type *AccessTy,
+  static ExtAddrMode Match(Value *V, Type *AccessTy, unsigned AS,
                            Instruction *MemoryInst,
                            SmallVectorImpl<Instruction*> &AddrModeInsts,
                            const TargetMachine &TM,
-                           const SetOfInstrs &InsertedTruncs,
+                           const SetOfInstrs &InsertedInsts,
                            InstrToOrigTy &PromotedInsts,
                            TypePromotionTransaction &TPT) {
     ExtAddrMode Result;
 
-    bool Success = AddressingModeMatcher(AddrModeInsts, TM, AccessTy,
-                                         MemoryInst, Result, InsertedTruncs,
+    bool Success = AddressingModeMatcher(AddrModeInsts, TM, AccessTy, AS,
+                                         MemoryInst, Result, InsertedInsts,
                                          PromotedInsts, TPT).MatchAddr(V, 0);
     (void)Success; assert(Success && "Couldn't select *anything*?");
     return Result;
@@ -2145,7 +2201,7 @@ bool AddressingModeMatcher::MatchScaledValue(Value *ScaleReg, int64_t Scale,
   TestAddrMode.ScaledReg = ScaleReg;
 
   // If the new address isn't legal, bail out.
-  if (!TLI.isLegalAddressingMode(TestAddrMode, AccessTy))
+  if (!TLI.isLegalAddressingMode(DL, TestAddrMode, AccessTy, AddrSpace))
     return false;
 
   // It was legal, so commit it.
@@ -2162,7 +2218,7 @@ bool AddressingModeMatcher::MatchScaledValue(Value *ScaleReg, int64_t Scale,
 
     // If this addressing mode is legal, commit it and remember that we folded
     // this instruction.
-    if (TLI.isLegalAddressingMode(TestAddrMode, AccessTy)) {
+    if (TLI.isLegalAddressingMode(DL, TestAddrMode, AccessTy, AddrSpace)) {
       AddrModeInsts.push_back(cast<Instruction>(ScaleReg));
       AddrMode = TestAddrMode;
       return true;
@@ -2208,7 +2264,8 @@ static bool MightBeFoldableInst(Instruction *I) {
 /// \note \p Val is assumed to be the product of some type promotion.
 /// Therefore if \p Val has an undefined state in \p TLI, this is assumed
 /// to be legal, as the non-promoted value would have had the same state.
-static bool isPromotedInstructionLegal(const TargetLowering &TLI, Value *Val) {
+static bool isPromotedInstructionLegal(const TargetLowering &TLI,
+                                       const DataLayout &DL, Value *Val) {
   Instruction *PromotedInst = dyn_cast<Instruction>(Val);
   if (!PromotedInst)
     return false;
@@ -2218,7 +2275,7 @@ static bool isPromotedInstructionLegal(const TargetLowering &TLI, Value *Val) {
     return true;
   // Otherwise, check if the promoted instruction is legal or not.
   return TLI.isOperationLegalOrCustom(
-      ISDOpcode, TLI.getValueType(PromotedInst->getType()));
+      ISDOpcode, TLI.getValueType(DL, PromotedInst->getType()));
 }
 
 /// \brief Hepler class to perform type promotion.
@@ -2309,12 +2366,12 @@ class TypePromotionHelper {
   /// action to promote the operand of \p Ext instead of using Ext.
   /// \return NULL if no promotable action is possible with the current
   /// sign extension.
-  /// \p InsertedTruncs keeps track of all the truncate instructions inserted by
-  /// the others CodeGenPrepare optimizations. This information is important
+  /// \p InsertedInsts keeps track of all the instructions inserted by the
+  /// other CodeGenPrepare optimizations. This information is important
   /// because we do not want to promote these instructions as CodeGenPrepare
   /// will reinsert them later. Thus creating an infinite loop: create/remove.
   /// \p PromotedInsts maps the instructions to their type before promotion.
-  static Action getAction(Instruction *Ext, const SetOfInstrs &InsertedTruncs,
+  static Action getAction(Instruction *Ext, const SetOfInstrs &InsertedInsts,
                           const TargetLowering &TLI,
                           const InstrToOrigTy &PromotedInsts);
 };
@@ -2372,8 +2429,8 @@ bool TypePromotionHelper::canGetThrough(const Instruction *Inst,
   // #1 get the type of the operand and check the kind of the extended bits.
   const Type *OpndType;
   InstrToOrigTy::const_iterator It = PromotedInsts.find(Opnd);
-  if (It != PromotedInsts.end() && It->second.IsSExt == IsSExt)
-    OpndType = It->second.Ty;
+  if (It != PromotedInsts.end() && It->second.getInt() == IsSExt)
+    OpndType = It->second.getPointer();
   else if ((IsSExt && isa<SExtInst>(Opnd)) || (!IsSExt && isa<ZExtInst>(Opnd)))
     OpndType = Opnd->getOperand(0)->getType();
   else
@@ -2387,7 +2444,7 @@ bool TypePromotionHelper::canGetThrough(const Instruction *Inst,
 }
 
 TypePromotionHelper::Action TypePromotionHelper::getAction(
-    Instruction *Ext, const SetOfInstrs &InsertedTruncs,
+    Instruction *Ext, const SetOfInstrs &InsertedInsts,
     const TargetLowering &TLI, const InstrToOrigTy &PromotedInsts) {
   assert((isa<SExtInst>(Ext) || isa<ZExtInst>(Ext)) &&
          "Unexpected instruction type");
@@ -2403,7 +2460,7 @@ TypePromotionHelper::Action TypePromotionHelper::getAction(
   // Do not promote if the operand has been added by codegenprepare.
   // Otherwise, it means we are undoing an optimization that is likely to be
   // redone, thus causing potential infinite loop.
-  if (isa<TruncInst>(ExtOpnd) && InsertedTruncs.count(ExtOpnd))
+  if (isa<TruncInst>(ExtOpnd) && InsertedInsts.count(ExtOpnd))
     return nullptr;
 
   // SExt or Trunc instructions.
@@ -2592,7 +2649,7 @@ bool AddressingModeMatcher::IsPromotionProfitable(
   // The promotion is neutral but it may help folding the sign extension in
   // loads for instance.
   // Check that we did not create an illegal instruction.
-  return isPromotedInstructionLegal(TLI, PromotedOperand);
+  return isPromotedInstructionLegal(TLI, DL, PromotedOperand);
 }
 
 /// MatchOperationAddr - Given an instruction or constant expr, see if we can
@@ -2620,14 +2677,15 @@ bool AddressingModeMatcher::MatchOperationAddr(User *AddrInst, unsigned Opcode,
   case Instruction::PtrToInt:
     // PtrToInt is always a noop, as we know that the int type is pointer sized.
     return MatchAddr(AddrInst->getOperand(0), Depth);
-  case Instruction::IntToPtr:
+  case Instruction::IntToPtr: {
+    auto AS = AddrInst->getType()->getPointerAddressSpace();
+    auto PtrTy = MVT::getIntegerVT(DL.getPointerSizeInBits(AS));
     // This inttoptr is a no-op if the integer type is pointer sized.
-    if (TLI.getValueType(AddrInst->getOperand(0)->getType()) ==
-        TLI.getPointerTy(AddrInst->getType()->getPointerAddressSpace()))
+    if (TLI.getValueType(DL, AddrInst->getOperand(0)->getType()) == PtrTy)
       return MatchAddr(AddrInst->getOperand(0), Depth);
     return false;
+  }
   case Instruction::BitCast:
-  case Instruction::AddrSpaceCast:
     // BitCast is always a noop, and we can handle it as long as it is
     // int->int or pointer->pointer (we don't want int<->fp or something).
     if ((AddrInst->getOperand(0)->getType()->isPointerTy() ||
@@ -2638,6 +2696,14 @@ bool AddressingModeMatcher::MatchOperationAddr(User *AddrInst, unsigned Opcode,
         AddrInst->getOperand(0)->getType() != AddrInst->getType())
       return MatchAddr(AddrInst->getOperand(0), Depth);
     return false;
+  case Instruction::AddrSpaceCast: {
+    unsigned SrcAS
+      = AddrInst->getOperand(0)->getType()->getPointerAddressSpace();
+    unsigned DestAS = AddrInst->getType()->getPointerAddressSpace();
+    if (TLI.isNoopAddrSpaceCast(SrcAS, DestAS))
+      return MatchAddr(AddrInst->getOperand(0), Depth);
+    return false;
+  }
   case Instruction::Add: {
     // Check to see if we can merge in the RHS then the LHS.  If so, we win.
     ExtAddrMode BackupAddrMode = AddrMode;
@@ -2691,16 +2757,15 @@ bool AddressingModeMatcher::MatchOperationAddr(User *AddrInst, unsigned Opcode,
     unsigned VariableScale = 0;
 
     int64_t ConstantOffset = 0;
-    const DataLayout *TD = TLI.getDataLayout();
     gep_type_iterator GTI = gep_type_begin(AddrInst);
     for (unsigned i = 1, e = AddrInst->getNumOperands(); i != e; ++i, ++GTI) {
       if (StructType *STy = dyn_cast<StructType>(*GTI)) {
-        const StructLayout *SL = TD->getStructLayout(STy);
+        const StructLayout *SL = DL.getStructLayout(STy);
         unsigned Idx =
           cast<ConstantInt>(AddrInst->getOperand(i))->getZExtValue();
         ConstantOffset += SL->getElementOffset(Idx);
       } else {
-        uint64_t TypeSize = TD->getTypeAllocSize(GTI.getIndexedType());
+        uint64_t TypeSize = DL.getTypeAllocSize(GTI.getIndexedType());
         if (ConstantInt *CI = dyn_cast<ConstantInt>(AddrInst->getOperand(i))) {
           ConstantOffset += CI->getSExtValue()*TypeSize;
         } else if (TypeSize) {  // Scales of zero don't do anything.
@@ -2719,7 +2784,8 @@ bool AddressingModeMatcher::MatchOperationAddr(User *AddrInst, unsigned Opcode,
     // just add it to the disp field and check validity.
     if (VariableOperand == -1) {
       AddrMode.BaseOffs += ConstantOffset;
-      if (ConstantOffset == 0 || TLI.isLegalAddressingMode(AddrMode, AccessTy)){
+      if (ConstantOffset == 0 ||
+          TLI.isLegalAddressingMode(DL, AddrMode, AccessTy, AddrSpace)) {
         // Check to see if we can fold the base pointer in too.
         if (MatchAddr(AddrInst->getOperand(0), Depth+1))
           return true;
@@ -2779,7 +2845,7 @@ bool AddressingModeMatcher::MatchOperationAddr(User *AddrInst, unsigned Opcode,
     // Try to move this ext out of the way of the addressing mode.
     // Ask for a method for doing so.
     TypePromotionHelper::Action TPH =
-        TypePromotionHelper::getAction(Ext, InsertedTruncs, TLI, PromotedInsts);
+        TypePromotionHelper::getAction(Ext, InsertedInsts, TLI, PromotedInsts);
     if (!TPH)
       return false;
 
@@ -2842,14 +2908,14 @@ bool AddressingModeMatcher::MatchAddr(Value *Addr, unsigned Depth) {
   if (ConstantInt *CI = dyn_cast<ConstantInt>(Addr)) {
     // Fold in immediates if legal for the target.
     AddrMode.BaseOffs += CI->getSExtValue();
-    if (TLI.isLegalAddressingMode(AddrMode, AccessTy))
+    if (TLI.isLegalAddressingMode(DL, AddrMode, AccessTy, AddrSpace))
       return true;
     AddrMode.BaseOffs -= CI->getSExtValue();
   } else if (GlobalValue *GV = dyn_cast<GlobalValue>(Addr)) {
     // If this is a global variable, try to fold it into the addressing mode.
     if (!AddrMode.BaseGV) {
       AddrMode.BaseGV = GV;
-      if (TLI.isLegalAddressingMode(AddrMode, AccessTy))
+      if (TLI.isLegalAddressingMode(DL, AddrMode, AccessTy, AddrSpace))
         return true;
       AddrMode.BaseGV = nullptr;
     }
@@ -2893,7 +2959,7 @@ bool AddressingModeMatcher::MatchAddr(Value *Addr, unsigned Depth) {
     AddrMode.HasBaseReg = true;
     AddrMode.BaseReg = Addr;
     // Still check for legality in case the target supports [imm] but not [i+r].
-    if (TLI.isLegalAddressingMode(AddrMode, AccessTy))
+    if (TLI.isLegalAddressingMode(DL, AddrMode, AccessTy, AddrSpace))
       return true;
     AddrMode.HasBaseReg = false;
     AddrMode.BaseReg = nullptr;
@@ -2903,7 +2969,7 @@ bool AddressingModeMatcher::MatchAddr(Value *Addr, unsigned Depth) {
   if (AddrMode.Scale == 0) {
     AddrMode.Scale = 1;
     AddrMode.ScaledReg = Addr;
-    if (TLI.isLegalAddressingMode(AddrMode, AccessTy))
+    if (TLI.isLegalAddressingMode(DL, AddrMode, AccessTy, AddrSpace))
       return true;
     AddrMode.Scale = 0;
     AddrMode.ScaledReg = nullptr;
@@ -2922,7 +2988,8 @@ static bool IsOperandAMemoryOperand(CallInst *CI, InlineAsm *IA, Value *OpVal,
   const TargetLowering *TLI = TM.getSubtargetImpl(*F)->getTargetLowering();
   const TargetRegisterInfo *TRI = TM.getSubtargetImpl(*F)->getRegisterInfo();
   TargetLowering::AsmOperandInfoVector TargetConstraints =
-      TLI->ParseConstraints(TRI, ImmutableCallSite(CI));
+      TLI->ParseConstraints(F->getParent()->getDataLayout(), TRI,
+                            ImmutableCallSite(CI));
   for (unsigned i = 0, e = TargetConstraints.size(); i != e; ++i) {
     TargetLowering::AsmOperandInfo &OpInfo = TargetConstraints[i];
 
@@ -3084,9 +3151,11 @@ IsProfitableToFoldIntoAddressingMode(Instruction *I, ExtAddrMode &AMBefore,
     // Get the access type of this use.  If the use isn't a pointer, we don't
     // know what it accesses.
     Value *Address = User->getOperand(OpNo);
-    if (!Address->getType()->isPointerTy())
+    PointerType *AddrTy = dyn_cast<PointerType>(Address->getType());
+    if (!AddrTy)
       return false;
-    Type *AddressAccessTy = Address->getType()->getPointerElementType();
+    Type *AddressAccessTy = AddrTy->getElementType();
+    unsigned AS = AddrTy->getAddressSpace();
 
     // Do a match against the root of this address, ignoring profitability. This
     // will tell us if the addressing mode for the memory operation will
@@ -3094,8 +3163,8 @@ IsProfitableToFoldIntoAddressingMode(Instruction *I, ExtAddrMode &AMBefore,
     ExtAddrMode Result;
     TypePromotionTransaction::ConstRestorationPt LastKnownGood =
         TPT.getRestorationPoint();
-    AddressingModeMatcher Matcher(MatchedAddrModeInsts, TM, AddressAccessTy,
-                                  MemoryInst, Result, InsertedTruncs,
+    AddressingModeMatcher Matcher(MatchedAddrModeInsts, TM, AddressAccessTy, AS,
+                                  MemoryInst, Result, InsertedInsts,
                                   PromotedInsts, TPT);
     Matcher.IgnoreProfitability = true;
     bool Success = Matcher.MatchAddr(Address, 0);
@@ -3137,7 +3206,7 @@ static bool IsNonLocalValue(Value *V, BasicBlock *BB) {
 /// This method is used to optimize both load/store and inline asms with memory
 /// operands.
 bool CodeGenPrepare::OptimizeMemoryInst(Instruction *MemoryInst, Value *Addr,
-                                        Type *AccessTy) {
+                                        Type *AccessTy, unsigned AddrSpace) {
   Value *Repl = Addr;
 
   // Try to collapse single-value PHI nodes.  This is necessary to undo
@@ -3169,16 +3238,16 @@ bool CodeGenPrepare::OptimizeMemoryInst(Instruction *MemoryInst, Value *Addr,
 
     // For a PHI node, push all of its incoming values.
     if (PHINode *P = dyn_cast<PHINode>(V)) {
-      for (unsigned i = 0, e = P->getNumIncomingValues(); i != e; ++i)
-        worklist.push_back(P->getIncomingValue(i));
+      for (Value *IncValue : P->incoming_values())
+        worklist.push_back(IncValue);
       continue;
     }
 
     // For non-PHIs, determine the addressing mode being computed.
     SmallVector<Instruction*, 16> NewAddrModeInsts;
     ExtAddrMode NewAddrMode = AddressingModeMatcher::Match(
-        V, AccessTy, MemoryInst, NewAddrModeInsts, *TM, InsertedTruncsSet,
-        PromotedInsts, TPT);
+      V, AccessTy, AddrSpace, MemoryInst, NewAddrModeInsts, *TM,
+      InsertedInsts, PromotedInsts, TPT);
 
     // This check is broken into two cases with very similar code to avoid using
     // getNumUses() as much as possible. Some values have a lot of uses, so
@@ -3260,7 +3329,7 @@ bool CodeGenPrepare::OptimizeMemoryInst(Instruction *MemoryInst, Value *Addr,
     // prevents new inttoptr/ptrtoint pairs from degrading AA capabilities.
     DEBUG(dbgs() << "CGP: SINKING nonlocal addrmode: " << AddrMode << " for "
                  << *MemoryInst << "\n");
-    Type *IntPtrTy = TLI->getDataLayout()->getIntPtrType(Addr->getType());
+    Type *IntPtrTy = DL->getIntPtrType(Addr->getType());
     Value *ResultPtr = nullptr, *ResultIndex = nullptr;
 
     // First, find the pointer.
@@ -3379,7 +3448,7 @@ bool CodeGenPrepare::OptimizeMemoryInst(Instruction *MemoryInst, Value *Addr,
   } else {
     DEBUG(dbgs() << "CGP: SINKING nonlocal addrmode: " << AddrMode << " for "
                  << *MemoryInst << "\n");
-    Type *IntPtrTy = TLI->getDataLayout()->getIntPtrType(Addr->getType());
+    Type *IntPtrTy = DL->getIntPtrType(Addr->getType());
     Value *Result = nullptr;
 
     // Start with the base register. Do this first so that subsequent address
@@ -3481,8 +3550,8 @@ bool CodeGenPrepare::OptimizeInlineAsmInst(CallInst *CS) {
 
   const TargetRegisterInfo *TRI =
       TM->getSubtargetImpl(*CS->getParent()->getParent())->getRegisterInfo();
-  TargetLowering::AsmOperandInfoVector
-    TargetConstraints = TLI->ParseConstraints(TRI, CS);
+  TargetLowering::AsmOperandInfoVector TargetConstraints =
+      TLI->ParseConstraints(*DL, TRI, CS);
   unsigned ArgNo = 0;
   for (unsigned i = 0, e = TargetConstraints.size(); i != e; ++i) {
     TargetLowering::AsmOperandInfo &OpInfo = TargetConstraints[i];
@@ -3493,7 +3562,7 @@ bool CodeGenPrepare::OptimizeInlineAsmInst(CallInst *CS) {
     if (OpInfo.ConstraintType == TargetLowering::C_Memory &&
         OpInfo.isIndirect) {
       Value *OpVal = CS->getArgOperand(ArgNo++);
-      MadeChange |= OptimizeMemoryInst(CS, OpVal, OpVal->getType());
+      MadeChange |= OptimizeMemoryInst(CS, OpVal, OpVal->getType(), ~0u);
     } else if (OpInfo.Type == InlineAsm::isInput)
       ArgNo++;
   }
@@ -3590,7 +3659,7 @@ bool CodeGenPrepare::ExtLdPromotion(TypePromotionTransaction &TPT,
       continue;
     // Get the action to perform the promotion.
     TypePromotionHelper::Action TPH = TypePromotionHelper::getAction(
-        I, InsertedTruncsSet, *TLI, PromotedInsts);
+        I, InsertedInsts, *TLI, PromotedInsts);
     // Check if we can promote.
     if (!TPH)
       continue;
@@ -3616,7 +3685,7 @@ bool CodeGenPrepare::ExtLdPromotion(TypePromotionTransaction &TPT,
     TotalCreatedInstsCost -= ExtCost;
     if (!StressExtLdPromotion &&
         (TotalCreatedInstsCost > 1 ||
-         !isPromotedInstructionLegal(*TLI, PromotedVal))) {
+         !isPromotedInstructionLegal(*TLI, *DL, PromotedVal))) {
       // The promotion is not profitable, rollback to the previous state.
       TPT.rollback(LastKnownGood);
       continue;
@@ -3671,8 +3740,8 @@ bool CodeGenPrepare::MoveExtToFormExtLoad(Instruction *&I) {
   if (!HasPromoted && LI->getParent() == I->getParent())
     return false;
 
-  EVT VT = TLI->getValueType(I->getType());
-  EVT LoadVT = TLI->getValueType(LI->getType());
+  EVT VT = TLI->getValueType(*DL, I->getType());
+  EVT LoadVT = TLI->getValueType(*DL, LI->getType());
 
   // If the load has other users and the truncate is not free, this probably
   // isn't worthwhile.
@@ -3766,7 +3835,7 @@ bool CodeGenPrepare::OptimizeExtUses(Instruction *I) {
     if (!InsertedTrunc) {
       BasicBlock::iterator InsertPt = UserBB->getFirstInsertionPt();
       InsertedTrunc = new TruncInst(I, Src->getType(), "", InsertPt);
-      InsertedTruncsSet.insert(InsertedTrunc);
+      InsertedInsts.insert(InsertedTrunc);
     }
 
     // Replace a use of the {s|z}ext source with a use of the result.
@@ -3949,6 +4018,9 @@ namespace {
 /// Assuming both extractelement and store can be combine, we get rid of the
 /// transition.
 class VectorPromoteHelper {
+  /// DataLayout associated with the current module.
+  const DataLayout &DL;
+
   /// Used to perform some checks on the legality of vector operations.
   const TargetLowering &TLI;
 
@@ -4022,7 +4094,8 @@ class VectorPromoteHelper {
     unsigned Align = ST->getAlignment();
     // Check if this store is supported.
     if (!TLI.allowsMisalignedMemoryAccesses(
-            TLI.getValueType(ST->getValueOperand()->getType()), AS, Align)) {
+            TLI.getValueType(DL, ST->getValueOperand()->getType()), AS,
+            Align)) {
       // If this is not supported, there is no way we can combine
       // the extract with the store.
       return false;
@@ -4117,9 +4190,10 @@ class VectorPromoteHelper {
   }
 
 public:
-  VectorPromoteHelper(const TargetLowering &TLI, const TargetTransformInfo &TTI,
-                      Instruction *Transition, unsigned CombineCost)
-      : TLI(TLI), TTI(TTI), Transition(Transition),
+  VectorPromoteHelper(const DataLayout &DL, const TargetLowering &TLI,
+                      const TargetTransformInfo &TTI, Instruction *Transition,
+                      unsigned CombineCost)
+      : DL(DL), TLI(TLI), TTI(TTI), Transition(Transition),
         StoreExtractCombineCost(CombineCost), CombineInst(nullptr) {
     assert(Transition && "Do not know how to promote null");
   }
@@ -4155,7 +4229,7 @@ class VectorPromoteHelper {
       return false;
     return StressStoreExtract ||
            TLI.isOperationLegalOrCustom(
-               ISDOpcode, TLI.getValueType(getTransitionType(), true));
+               ISDOpcode, TLI.getValueType(DL, getTransitionType(), true));
   }
 
   /// \brief Check whether or not \p Use can be combined
@@ -4259,7 +4333,7 @@ bool CodeGenPrepare::OptimizeExtractElementInst(Instruction *Inst) {
   //      we do not do that for now.
   BasicBlock *Parent = Inst->getParent();
   DEBUG(dbgs() << "Found an interesting transition: " << *Inst << '\n');
-  VectorPromoteHelper VPH(*TLI, *TTI, Inst, CombineCost);
+  VectorPromoteHelper VPH(*DL, *TLI, *TTI, Inst, CombineCost);
   // If the transition has more than one use, assume this is not going to be
   // beneficial.
   while (Inst->hasOneUse()) {
@@ -4295,12 +4369,16 @@ bool CodeGenPrepare::OptimizeExtractElementInst(Instruction *Inst) {
 }
 
 bool CodeGenPrepare::OptimizeInst(Instruction *I, bool& ModifiedDT) {
+  // Bail out if we inserted the instruction to prevent optimizations from
+  // stepping on each other's toes.
+  if (InsertedInsts.count(I))
+    return false;
+
   if (PHINode *P = dyn_cast<PHINode>(I)) {
     // It is possible for very late stage optimizations (such as SimplifyCFG)
     // to introduce PHI nodes too late to be cleaned up.  If we detect such a
     // trivial PHI, go ahead and zap it here.
-    const DataLayout &DL = I->getModule()->getDataLayout();
-    if (Value *V = SimplifyInstruction(P, DL, TLInfo, nullptr)) {
+    if (Value *V = SimplifyInstruction(P, *DL, TLInfo, nullptr)) {
       P->replaceAllUsesWith(V);
       P->eraseFromParent();
       ++NumPHIsElim;
@@ -4319,15 +4397,16 @@ bool CodeGenPrepare::OptimizeInst(Instruction *I, bool& ModifiedDT) {
     if (isa<Constant>(CI->getOperand(0)))
       return false;
 
-    if (TLI && OptimizeNoopCopyExpression(CI, *TLI))
+    if (TLI && OptimizeNoopCopyExpression(CI, *TLI, *DL))
       return true;
 
     if (isa<ZExtInst>(I) || isa<SExtInst>(I)) {
       /// Sink a zext or sext into its user blocks if the target type doesn't
       /// fit in one register
-      if (TLI && TLI->getTypeAction(CI->getContext(),
-                                    TLI->getValueType(CI->getType())) ==
-                     TargetLowering::TypeExpandInteger) {
+      if (TLI &&
+          TLI->getTypeAction(CI->getContext(),
+                             TLI->getValueType(*DL, CI->getType())) ==
+              TargetLowering::TypeExpandInteger) {
         return SinkCast(CI);
       } else {
         bool MadeChange = MoveExtToFormExtLoad(I);
@@ -4342,15 +4421,19 @@ bool CodeGenPrepare::OptimizeInst(Instruction *I, bool& ModifiedDT) {
       return OptimizeCmpExpression(CI);
 
   if (LoadInst *LI = dyn_cast<LoadInst>(I)) {
-    if (TLI)
-      return OptimizeMemoryInst(I, I->getOperand(0), LI->getType());
+    if (TLI) {
+      unsigned AS = LI->getPointerAddressSpace();
+      return OptimizeMemoryInst(I, I->getOperand(0), LI->getType(), AS);
+    }
     return false;
   }
 
   if (StoreInst *SI = dyn_cast<StoreInst>(I)) {
-    if (TLI)
+    if (TLI) {
+      unsigned AS = SI->getPointerAddressSpace();
       return OptimizeMemoryInst(I, SI->getOperand(1),
-                                SI->getOperand(0)->getType());
+                                SI->getOperand(0)->getType(), AS);
+    }
     return false;
   }
 
@@ -4360,7 +4443,7 @@ bool CodeGenPrepare::OptimizeInst(Instruction *I, bool& ModifiedDT) {
                 BinOp->getOpcode() == Instruction::LShr)) {
     ConstantInt *CI = dyn_cast<ConstantInt>(BinOp->getOperand(1));
     if (TLI && CI && TLI->hasExtractBitsInsn())
-      return OptimizeExtractBits(BinOp, CI, *TLI);
+      return OptimizeExtractBits(BinOp, CI, *TLI, *DL);
 
     return false;
   }
diff --git a/lib/CodeGen/CoreCLRGC.cpp b/lib/CodeGen/CoreCLRGC.cpp
new file mode 100644
index 00000000000..28c97ba71bd
--- /dev/null
+++ b/lib/CodeGen/CoreCLRGC.cpp
@@ -0,0 +1,54 @@
+//===-- CoreCLRGC.cpp - CoreCLR Runtime GC Strategy -----------------------===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This file contains a GCStrategy for the CoreCLR Runtime.
+// The strategy is similar to Statepoint-example GC, but differs from it in
+// certain aspects, such as:
+// 1) Base-pointers need not be explicitly tracked and reported for
+//    interior pointers
+// 2) Uses a different format for encoding stack-maps
+// 3) Location of Safe-point polls: polls are only needed before loop-back edges
+//    and before tail-calls (not needed at function-entry)
+//
+// The above differences in behavior are to be implemented in upcoming checkins.
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/CodeGen/GCStrategy.h"
+#include "llvm/IR/DerivedTypes.h"
+#include "llvm/IR/Value.h"
+
+using namespace llvm;
+
+namespace {
+class CoreCLRGC : public GCStrategy {
+public:
+  CoreCLRGC() {
+    UseStatepoints = true;
+    // These options are all gc.root specific, we specify them so that the
+    // gc.root lowering code doesn't run.
+    InitRoots = false;
+    NeededSafePoints = 0;
+    UsesMetadata = false;
+    CustomRoots = false;
+  }
+  Optional<bool> isGCManagedPointer(const Value *V) const override {
+    // Method is only valid on pointer typed values.
+    PointerType *PT = cast<PointerType>(V->getType());
+    // We pick addrspace(1) as our GC managed heap.
+    return (1 == PT->getAddressSpace());
+  }
+};
+}
+
+static GCRegistry::Add<CoreCLRGC> X("coreclr", "CoreCLR-compatible GC");
+
+namespace llvm {
+void linkCoreCLRGC() {}
+}
diff --git a/lib/CodeGen/CriticalAntiDepBreaker.cpp b/lib/CodeGen/CriticalAntiDepBreaker.cpp
index 3d62d488760..dba280fd5aa 100644
--- a/lib/CodeGen/CriticalAntiDepBreaker.cpp
+++ b/lib/CodeGen/CriticalAntiDepBreaker.cpp
@@ -71,7 +71,7 @@ void CriticalAntiDepBreaker::StartBlock(MachineBasicBlock *BB) {
   // all callee-saved registers. In non-return this is any
   // callee-saved register that is not saved in the prolog.
   const MachineFrameInfo *MFI = MF.getFrameInfo();
-  BitVector Pristine = MFI->getPristineRegs(BB);
+  BitVector Pristine = MFI->getPristineRegs(MF);
   for (const MCPhysReg *I = TRI->getCalleeSavedRegs(&MF); *I; ++I) {
     if (!IsReturnBlock && !Pristine.test(*I)) continue;
     for (MCRegAliasIterator AI(*I, TRI, true); AI.isValid(); ++AI) {
diff --git a/lib/CodeGen/CriticalAntiDepBreaker.h b/lib/CodeGen/CriticalAntiDepBreaker.h
index af011a0a65f..10b873959ad 100644
--- a/lib/CodeGen/CriticalAntiDepBreaker.h
+++ b/lib/CodeGen/CriticalAntiDepBreaker.h
@@ -31,7 +31,7 @@ class RegisterClassInfo;
 class TargetInstrInfo;
 class TargetRegisterInfo;
 
-  class CriticalAntiDepBreaker : public AntiDepBreaker {
+class LLVM_LIBRARY_VISIBILITY CriticalAntiDepBreaker : public AntiDepBreaker {
     MachineFunction& MF;
     MachineRegisterInfo &MRI;
     const TargetInstrInfo *TII;
diff --git a/lib/CodeGen/DeadMachineInstructionElim.cpp b/lib/CodeGen/DeadMachineInstructionElim.cpp
index c17a35d1c73..39c259d7c79 100644
--- a/lib/CodeGen/DeadMachineInstructionElim.cpp
+++ b/lib/CodeGen/DeadMachineInstructionElim.cpp
@@ -60,12 +60,12 @@ bool DeadMachineInstructionElim::isDead(const MachineInstr *MI) const {
     return false;
 
   // Don't delete frame allocation labels.
-  if (MI->getOpcode() == TargetOpcode::FRAME_ALLOC)
+  if (MI->getOpcode() == TargetOpcode::LOCAL_ESCAPE)
     return false;
 
   // Don't delete instructions with side effects.
   bool SawStore = false;
-  if (!MI->isSafeToMove(TII, nullptr, SawStore) && !MI->isPHI())
+  if (!MI->isSafeToMove(nullptr, SawStore) && !MI->isPHI())
     return false;
 
   // Examine each operand.
@@ -101,26 +101,23 @@ bool DeadMachineInstructionElim::runOnMachineFunction(MachineFunction &MF) {
   // Loop over all instructions in all blocks, from bottom to top, so that it's
   // more likely that chains of dependent but ultimately dead instructions will
   // be cleaned up.
-  for (MachineFunction::reverse_iterator I = MF.rbegin(), E = MF.rend();
-       I != E; ++I) {
-    MachineBasicBlock *MBB = &*I;
-
+  for (MachineBasicBlock &MBB : make_range(MF.rbegin(), MF.rend())) {
     // Start out assuming that reserved registers are live out of this block.
     LivePhysRegs = MRI->getReservedRegs();
 
     // Add live-ins from sucessors to LivePhysRegs. Normally, physregs are not
     // live across blocks, but some targets (x86) can have flags live out of a
     // block.
-    for (MachineBasicBlock::succ_iterator S = MBB->succ_begin(),
-           E = MBB->succ_end(); S != E; S++)
+    for (MachineBasicBlock::succ_iterator S = MBB.succ_begin(),
+           E = MBB.succ_end(); S != E; S++)
       for (MachineBasicBlock::livein_iterator LI = (*S)->livein_begin();
            LI != (*S)->livein_end(); LI++)
         LivePhysRegs.set(*LI);
 
     // Now scan the instructions and delete dead ones, tracking physreg
     // liveness as we go.
-    for (MachineBasicBlock::reverse_iterator MII = MBB->rbegin(),
-         MIE = MBB->rend(); MII != MIE; ) {
+    for (MachineBasicBlock::reverse_iterator MII = MBB.rbegin(),
+         MIE = MBB.rend(); MII != MIE; ) {
       MachineInstr *MI = &*MII;
 
       // If the instruction is dead, delete it!
@@ -132,7 +129,7 @@ bool DeadMachineInstructionElim::runOnMachineFunction(MachineFunction &MF) {
         MI->eraseFromParentAndMarkDBGValuesForRemoval();
         AnyChanges = true;
         ++NumDeletes;
-        MIE = MBB->rend();
+        MIE = MBB.rend();
         // MII is now pointing to the next instruction to process,
         // so don't increment it.
         continue;
diff --git a/lib/CodeGen/DwarfEHPrepare.cpp b/lib/CodeGen/DwarfEHPrepare.cpp
index 42656fb08db..e019dfbc8f7 100644
--- a/lib/CodeGen/DwarfEHPrepare.cpp
+++ b/lib/CodeGen/DwarfEHPrepare.cpp
@@ -181,27 +181,22 @@ size_t DwarfEHPrepare::pruneUnreachableResumes(
 bool DwarfEHPrepare::InsertUnwindResumeCalls(Function &Fn) {
   SmallVector<ResumeInst*, 16> Resumes;
   SmallVector<LandingPadInst*, 16> CleanupLPads;
-  bool FoundLP = false;
   for (BasicBlock &BB : Fn) {
     if (auto *RI = dyn_cast<ResumeInst>(BB.getTerminator()))
       Resumes.push_back(RI);
-    if (auto *LP = BB.getLandingPadInst()) {
+    if (auto *LP = BB.getLandingPadInst())
       if (LP->isCleanup())
         CleanupLPads.push_back(LP);
-      // Check the personality on the first landingpad. Don't do anything if
-      // it's for MSVC.
-      if (!FoundLP) {
-        FoundLP = true;
-        EHPersonality Pers = classifyEHPersonality(LP->getPersonalityFn());
-        if (isMSVCEHPersonality(Pers))
-          return false;
-      }
-    }
   }
 
   if (Resumes.empty())
     return false;
 
+  // Check the personality, don't do anything if it's for MSVC.
+  EHPersonality Pers = classifyEHPersonality(Fn.getPersonalityFn());
+  if (isMSVCEHPersonality(Pers))
+    return false;
+
   LLVMContext &Ctx = Fn.getContext();
 
   size_t ResumesLeft = pruneUnreachableResumes(Fn, Resumes, CleanupLPads);
diff --git a/lib/CodeGen/EarlyIfConversion.cpp b/lib/CodeGen/EarlyIfConversion.cpp
index 6cde4c24915..fbc4d97c498 100644
--- a/lib/CodeGen/EarlyIfConversion.cpp
+++ b/lib/CodeGen/EarlyIfConversion.cpp
@@ -220,27 +220,27 @@ bool SSAIfConv::canSpeculateInstrs(MachineBasicBlock *MBB) {
 
     // We never speculate stores, so an AA pointer isn't necessary.
     bool DontMoveAcrossStore = true;
-    if (!I->isSafeToMove(TII, nullptr, DontMoveAcrossStore)) {
+    if (!I->isSafeToMove(nullptr, DontMoveAcrossStore)) {
       DEBUG(dbgs() << "Can't speculate: " << *I);
       return false;
     }
 
     // Check for any dependencies on Head instructions.
-    for (MIOperands MO(I); MO.isValid(); ++MO) {
-      if (MO->isRegMask()) {
+    for (const MachineOperand &MO : I->operands()) {
+      if (MO.isRegMask()) {
         DEBUG(dbgs() << "Won't speculate regmask: " << *I);
         return false;
       }
-      if (!MO->isReg())
+      if (!MO.isReg())
         continue;
-      unsigned Reg = MO->getReg();
+      unsigned Reg = MO.getReg();
 
       // Remember clobbered regunits.
-      if (MO->isDef() && TargetRegisterInfo::isPhysicalRegister(Reg))
+      if (MO.isDef() && TargetRegisterInfo::isPhysicalRegister(Reg))
         for (MCRegUnitIterator Units(Reg, TRI); Units.isValid(); ++Units)
           ClobberedRegUnits.set(*Units);
 
-      if (!MO->readsReg() || !TargetRegisterInfo::isVirtualRegister(Reg))
+      if (!MO.readsReg() || !TargetRegisterInfo::isVirtualRegister(Reg))
         continue;
       MachineInstr *DefMI = MRI->getVRegDef(Reg);
       if (!DefMI || DefMI->getParent() != Head)
@@ -284,19 +284,19 @@ bool SSAIfConv::findInsertionPoint() {
     }
 
     // Update live regunits.
-    for (MIOperands MO(I); MO.isValid(); ++MO) {
+    for (const MachineOperand &MO : I->operands()) {
       // We're ignoring regmask operands. That is conservatively correct.
-      if (!MO->isReg())
+      if (!MO.isReg())
         continue;
-      unsigned Reg = MO->getReg();
+      unsigned Reg = MO.getReg();
       if (!TargetRegisterInfo::isPhysicalRegister(Reg))
         continue;
       // I clobbers Reg, so it isn't live before I.
-      if (MO->isDef())
+      if (MO.isDef())
         for (MCRegUnitIterator Units(Reg, TRI); Units.isValid(); ++Units)
           LiveRegUnits.erase(*Units);
       // Unless I reads Reg.
-      if (MO->readsReg())
+      if (MO.readsReg())
         Reads.push_back(Reg);
     }
     // Anything read by I is live before I.
@@ -479,11 +479,20 @@ void SSAIfConv::rewritePHIOperands() {
   // Convert all PHIs to select instructions inserted before FirstTerm.
   for (unsigned i = 0, e = PHIs.size(); i != e; ++i) {
     PHIInfo &PI = PHIs[i];
+    unsigned DstReg = 0;
+    
     DEBUG(dbgs() << "If-converting " << *PI.PHI);
-    unsigned PHIDst = PI.PHI->getOperand(0).getReg();
-    unsigned DstReg = MRI->createVirtualRegister(MRI->getRegClass(PHIDst));
-    TII->insertSelect(*Head, FirstTerm, HeadDL, DstReg, Cond, PI.TReg, PI.FReg);
-    DEBUG(dbgs() << "          --> " << *std::prev(FirstTerm));
+    if (PI.TReg == PI.FReg) {
+      // We do not need the select instruction if both incoming values are
+      // equal.
+      DstReg = PI.TReg;
+    } else {
+      unsigned PHIDst = PI.PHI->getOperand(0).getReg();
+      DstReg = MRI->createVirtualRegister(MRI->getRegClass(PHIDst));
+      TII->insertSelect(*Head, FirstTerm, HeadDL,
+                         DstReg, Cond, PI.TReg, PI.FReg);
+      DEBUG(dbgs() << "          --> " << *std::prev(FirstTerm));
+    }
 
     // Rewrite PHI operands TPred -> (DstReg, Head), remove FPred.
     for (unsigned i = PI.PHI->getNumOperands(); i != 1; i -= 2) {
diff --git a/lib/CodeGen/ExecutionDepsFix.cpp b/lib/CodeGen/ExecutionDepsFix.cpp
index 5b09cf1a0fd..cbc3b80a6a5 100644
--- a/lib/CodeGen/ExecutionDepsFix.cpp
+++ b/lib/CodeGen/ExecutionDepsFix.cpp
@@ -559,12 +559,11 @@ void ExeDepsFix::processUndefReads(MachineBasicBlock *MBB) {
   MachineInstr *UndefMI = UndefReads.back().first;
   unsigned OpIdx = UndefReads.back().second;
 
-  for (MachineBasicBlock::reverse_iterator I = MBB->rbegin(), E = MBB->rend();
-       I != E; ++I) {
+  for (MachineInstr &I : make_range(MBB->rbegin(), MBB->rend())) {
     // Update liveness, including the current instruction's defs.
-    LiveRegSet.stepBackward(*I);
+    LiveRegSet.stepBackward(I);
 
-    if (UndefMI == &*I) {
+    if (UndefMI == &I) {
       if (!LiveRegSet.contains(UndefMI->getOperand(OpIdx).getReg()))
         TII->breakPartialRegDependency(UndefMI, OpIdx, TRI);
 
@@ -733,12 +732,14 @@ bool ExeDepsFix::runOnMachineFunction(MachineFunction &mf) {
   // If no relevant registers are used in the function, we can skip it
   // completely.
   bool anyregs = false;
+  const MachineRegisterInfo &MRI = mf.getRegInfo();
   for (TargetRegisterClass::const_iterator I = RC->begin(), E = RC->end();
-       I != E; ++I)
-    if (MF->getRegInfo().isPhysRegUsed(*I)) {
-      anyregs = true;
-      break;
-    }
+       I != E && !anyregs; ++I)
+    for (MCRegAliasIterator AI(*I, TRI, true); AI.isValid(); ++AI)
+      if (!MRI.reg_nodbg_empty(*AI)) {
+        anyregs = true;
+        break;
+      }
   if (!anyregs) return false;
 
   // Initialize the AliasMap on the first use.
diff --git a/lib/CodeGen/FaultMaps.cpp b/lib/CodeGen/FaultMaps.cpp
new file mode 100644
index 00000000000..2acafafdb9f
--- /dev/null
+++ b/lib/CodeGen/FaultMaps.cpp
@@ -0,0 +1,150 @@
+//===---------------------------- FaultMaps.cpp ---------------------------===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/CodeGen/FaultMaps.h"
+
+#include "llvm/CodeGen/AsmPrinter.h"
+#include "llvm/MC/MCContext.h"
+#include "llvm/MC/MCExpr.h"
+#include "llvm/MC/MCObjectFileInfo.h"
+#include "llvm/MC/MCStreamer.h"
+#include "llvm/Support/Debug.h"
+
+using namespace llvm;
+
+#define DEBUG_TYPE "faultmaps"
+
+static const int FaultMapVersion = 1;
+const char *FaultMaps::WFMP = "Fault Maps: ";
+
+FaultMaps::FaultMaps(AsmPrinter &AP) : AP(AP) {}
+
+void FaultMaps::recordFaultingOp(FaultKind FaultTy,
+                                 const MCSymbol *HandlerLabel) {
+  MCContext &OutContext = AP.OutStreamer->getContext();
+  MCSymbol *FaultingLabel = OutContext.createTempSymbol();
+
+  AP.OutStreamer->EmitLabel(FaultingLabel);
+
+  const MCExpr *FaultingOffset = MCBinaryExpr::createSub(
+      MCSymbolRefExpr::create(FaultingLabel, OutContext),
+      MCSymbolRefExpr::create(AP.CurrentFnSymForSize, OutContext), OutContext);
+
+  const MCExpr *HandlerOffset = MCBinaryExpr::createSub(
+      MCSymbolRefExpr::create(HandlerLabel, OutContext),
+      MCSymbolRefExpr::create(AP.CurrentFnSymForSize, OutContext), OutContext);
+
+  FunctionInfos[AP.CurrentFnSym].emplace_back(FaultTy, FaultingOffset,
+                                              HandlerOffset);
+}
+
+void FaultMaps::serializeToFaultMapSection() {
+  if (FunctionInfos.empty())
+    return;
+
+  MCContext &OutContext = AP.OutStreamer->getContext();
+  MCStreamer &OS = *AP.OutStreamer;
+
+  // Create the section.
+  MCSection *FaultMapSection =
+      OutContext.getObjectFileInfo()->getFaultMapSection();
+  OS.SwitchSection(FaultMapSection);
+
+  // Emit a dummy symbol to force section inclusion.
+  OS.EmitLabel(OutContext.getOrCreateSymbol(Twine("__LLVM_FaultMaps")));
+
+  DEBUG(dbgs() << "********** Fault Map Output **********\n");
+
+  // Header
+  OS.EmitIntValue(FaultMapVersion, 1); // Version.
+  OS.EmitIntValue(0, 1);               // Reserved.
+  OS.EmitIntValue(0, 2);               // Reserved.
+
+  DEBUG(dbgs() << WFMP << "#functions = " << FunctionInfos.size() << "\n");
+  OS.EmitIntValue(FunctionInfos.size(), 4);
+
+  DEBUG(dbgs() << WFMP << "functions:\n");
+
+  for (const auto &FFI : FunctionInfos)
+    emitFunctionInfo(FFI.first, FFI.second);
+}
+
+void FaultMaps::emitFunctionInfo(const MCSymbol *FnLabel,
+                                 const FunctionFaultInfos &FFI) {
+  MCStreamer &OS = *AP.OutStreamer;
+
+  DEBUG(dbgs() << WFMP << "  function addr: " << *FnLabel << "\n");
+  OS.EmitSymbolValue(FnLabel, 8);
+
+  DEBUG(dbgs() << WFMP << "  #faulting PCs: " << FFI.size() << "\n");
+  OS.EmitIntValue(FFI.size(), 4);
+
+  OS.EmitIntValue(0, 4); // Reserved
+
+  for (auto &Fault : FFI) {
+    DEBUG(dbgs() << WFMP << "    fault type: "
+          << faultTypeToString(Fault.Kind) << "\n");
+    OS.EmitIntValue(Fault.Kind, 4);
+
+    DEBUG(dbgs() << WFMP << "    faulting PC offset: "
+          << *Fault.FaultingOffsetExpr << "\n");
+    OS.EmitValue(Fault.FaultingOffsetExpr, 4);
+
+    DEBUG(dbgs() << WFMP << "    fault handler PC offset: "
+          << *Fault.HandlerOffsetExpr << "\n");
+    OS.EmitValue(Fault.HandlerOffsetExpr, 4);
+  }
+}
+
+
+const char *FaultMaps::faultTypeToString(FaultMaps::FaultKind FT) {
+  switch (FT) {
+  default:
+    llvm_unreachable("unhandled fault type!");
+
+  case FaultMaps::FaultingLoad:
+    return "FaultingLoad";
+  }
+}
+
+raw_ostream &llvm::
+operator<<(raw_ostream &OS,
+           const FaultMapParser::FunctionFaultInfoAccessor &FFI) {
+  OS << "Fault kind: "
+     << FaultMaps::faultTypeToString((FaultMaps::FaultKind)FFI.getFaultKind())
+     << ", faulting PC offset: " << FFI.getFaultingPCOffset()
+     << ", handling PC offset: " << FFI.getHandlerPCOffset();
+  return OS;
+}
+
+raw_ostream &llvm::
+operator<<(raw_ostream &OS, const FaultMapParser::FunctionInfoAccessor &FI) {
+  OS << "FunctionAddress: " << format_hex(FI.getFunctionAddr(), 8)
+     << ", NumFaultingPCs: " << FI.getNumFaultingPCs() << "\n";
+  for (unsigned i = 0, e = FI.getNumFaultingPCs(); i != e; ++i)
+    OS << FI.getFunctionFaultInfoAt(i) << "\n";
+  return OS;
+}
+
+raw_ostream &llvm::operator<<(raw_ostream &OS, const FaultMapParser &FMP) {
+  OS << "Version: " << format_hex(FMP.getFaultMapVersion(), 2) << "\n";
+  OS << "NumFunctions: " << FMP.getNumFunctions() << "\n";
+
+  if (FMP.getNumFunctions() == 0)
+    return OS;
+
+  FaultMapParser::FunctionInfoAccessor FI;
+
+  for (unsigned i = 0, e = FMP.getNumFunctions(); i != e; ++i) {
+    FI = (i == 0) ? FMP.getFirstFunctionInfo() : FI.getNextFunctionInfo();
+    OS << FI;
+  }
+
+  return OS;
+}
diff --git a/lib/CodeGen/GCMetadata.cpp b/lib/CodeGen/GCMetadata.cpp
index 16cd9e83b4a..c8116a453d2 100644
--- a/lib/CodeGen/GCMetadata.cpp
+++ b/lib/CodeGen/GCMetadata.cpp
@@ -148,7 +148,6 @@ bool Printer::doFinalization(Module &M) {
   return false;
 }
 
-
 GCStrategy *GCModuleInfo::getGCStrategy(const StringRef Name) {
   // TODO: Arguably, just doing a linear search would be faster for small N
   auto NMI = GCStrategyMap.find(Name);
@@ -165,5 +164,14 @@ GCStrategy *GCModuleInfo::getGCStrategy(const StringRef Name) {
     }
   }
 
-  report_fatal_error(std::string("unsupported GC: ") + Name);
+  if (GCRegistry::begin() == GCRegistry::end()) {
+    // In normal operation, the registry should not be empty.  There should 
+    // be the builtin GCs if nothing else.  The most likely scenario here is
+    // that we got here without running the initializers used by the Registry 
+    // itself and it's registration mechanism.
+    const std::string error = ("unsupported GC: " + Name).str() + 
+      " (did you remember to link and initialize the CodeGen library?)";
+    report_fatal_error(error);
+  } else
+    report_fatal_error(std::string("unsupported GC: ") + Name);
 }
diff --git a/lib/CodeGen/GCRootLowering.cpp b/lib/CodeGen/GCRootLowering.cpp
index ac351656a4a..d8edd7e4063 100644
--- a/lib/CodeGen/GCRootLowering.cpp
+++ b/lib/CodeGen/GCRootLowering.cpp
@@ -142,7 +142,7 @@ static bool CouldBecomeSafePoint(Instruction *I) {
   // llvm.gcroot is safe because it doesn't do anything at runtime.
   if (CallInst *CI = dyn_cast<CallInst>(I))
     if (Function *F = CI->getCalledFunction())
-      if (unsigned IID = F->getIntrinsicID())
+      if (Intrinsic::ID IID = F->getIntrinsicID())
         if (IID == Intrinsic::gcroot)
           return false;
 
@@ -272,7 +272,7 @@ void GCMachineCodeAnalysis::getAnalysisUsage(AnalysisUsage &AU) const {
 MCSymbol *GCMachineCodeAnalysis::InsertLabel(MachineBasicBlock &MBB,
                                              MachineBasicBlock::iterator MI,
                                              DebugLoc DL) const {
-  MCSymbol *Label = MBB.getParent()->getContext().CreateTempSymbol();
+  MCSymbol *Label = MBB.getParent()->getContext().createTempSymbol();
   BuildMI(MBB, MI, DL, TII->get(TargetOpcode::GC_LABEL)).addSym(Label);
   return Label;
 }
diff --git a/lib/CodeGen/GlobalMerge.cpp b/lib/CodeGen/GlobalMerge.cpp
index 2157bcbe7d4..6f9e8394081 100644
--- a/lib/CodeGen/GlobalMerge.cpp
+++ b/lib/CodeGen/GlobalMerge.cpp
@@ -49,9 +49,19 @@
 //  str     r0, [r5], #4
 //
 //  note that we saved 2 registers here almostly "for free".
+//
+// However, merging globals can have tradeoffs:
+// - it confuses debuggers, tools, and users
+// - it makes linker optimizations less useful (order files, LOHs, ...)
+// - it forces usage of indexed addressing (which isn't necessarily "free")
+// - it can increase register pressure when the uses are disparate enough.
+// 
+// We use heuristics to discover the best global grouping we can (cf cl::opts).
 // ===---------------------------------------------------------------------===//
 
 #include "llvm/Transforms/Scalar.h"
+#include "llvm/ADT/DenseMap.h"
+#include "llvm/ADT/SmallBitVector.h"
 #include "llvm/ADT/SmallPtrSet.h"
 #include "llvm/ADT/Statistic.h"
 #include "llvm/CodeGen/Passes.h"
@@ -66,9 +76,12 @@
 #include "llvm/IR/Module.h"
 #include "llvm/Pass.h"
 #include "llvm/Support/CommandLine.h"
+#include "llvm/Support/Debug.h"
+#include "llvm/Support/raw_ostream.h"
 #include "llvm/Target/TargetLowering.h"
 #include "llvm/Target/TargetLoweringObjectFile.h"
 #include "llvm/Target/TargetSubtargetInfo.h"
+#include <algorithm>
 using namespace llvm;
 
 #define DEBUG_TYPE "global-merge"
@@ -79,6 +92,15 @@ EnableGlobalMerge("enable-global-merge", cl::Hidden,
                   cl::desc("Enable the global merge pass"),
                   cl::init(true));
 
+static cl::opt<bool> GlobalMergeGroupByUse(
+    "global-merge-group-by-use", cl::Hidden,
+    cl::desc("Improve global merge pass to look at uses"), cl::init(true));
+
+static cl::opt<bool> GlobalMergeIgnoreSingleUse(
+    "global-merge-ignore-single-use", cl::Hidden,
+    cl::desc("Improve global merge pass to ignore globals only used alone"),
+    cl::init(true));
+
 static cl::opt<bool>
 EnableGlobalMergeOnConst("global-merge-on-const", cl::Hidden,
                          cl::desc("Enable global merge pass on constants"),
@@ -95,15 +117,25 @@ STATISTIC(NumMerged, "Number of globals merged");
 namespace {
   class GlobalMerge : public FunctionPass {
     const TargetMachine *TM;
-    const DataLayout *DL;
     // FIXME: Infer the maximum possible offset depending on the actual users
     // (these max offsets are different for the users inside Thumb or ARM
     // functions), see the code that passes in the offset in the ARM backend
     // for more information.
     unsigned MaxOffset;
 
+    /// Whether we should try to optimize for size only.
+    /// Currently, this applies a dead simple heuristic: only consider globals
+    /// used in minsize functions for merging.
+    /// FIXME: This could learn about optsize, and be used in the cost model.
+    bool OnlyOptimizeForSize;
+
     bool doMerge(SmallVectorImpl<GlobalVariable*> &Globals,
                  Module &M, bool isConst, unsigned AddrSpace) const;
+    /// \brief Merge everything in \p Globals for which the corresponding bit
+    /// in \p GlobalSet is set.
+    bool doMerge(SmallVectorImpl<GlobalVariable *> &Globals,
+                 const BitVector &GlobalSet, Module &M, bool isConst,
+                 unsigned AddrSpace) const;
 
     /// \brief Check if the given variable has been identified as must keep
     /// \pre setMustKeepGlobalVariables must have been called on the Module that
@@ -125,9 +157,10 @@ namespace {
   public:
     static char ID;             // Pass identification, replacement for typeid.
     explicit GlobalMerge(const TargetMachine *TM = nullptr,
-                         unsigned MaximalOffset = 0)
-        : FunctionPass(ID), TM(TM), DL(TM->getDataLayout()),
-          MaxOffset(MaximalOffset) {
+                         unsigned MaximalOffset = 0,
+                         bool OnlyOptimizeForSize = false)
+        : FunctionPass(ID), TM(TM), MaxOffset(MaximalOffset),
+          OnlyOptimizeForSize(OnlyOptimizeForSize) {
       initializeGlobalMergePass(*PassRegistry::getPassRegistry());
     }
 
@@ -154,37 +187,249 @@ INITIALIZE_PASS_END(GlobalMerge, "global-merge", "Merge global variables",
 
 bool GlobalMerge::doMerge(SmallVectorImpl<GlobalVariable*> &Globals,
                           Module &M, bool isConst, unsigned AddrSpace) const {
+  auto &DL = M.getDataLayout();
   // FIXME: Find better heuristics
-  std::stable_sort(Globals.begin(), Globals.end(),
-                   [this](const GlobalVariable *GV1, const GlobalVariable *GV2) {
-    Type *Ty1 = cast<PointerType>(GV1->getType())->getElementType();
-    Type *Ty2 = cast<PointerType>(GV2->getType())->getElementType();
+  std::stable_sort(
+      Globals.begin(), Globals.end(),
+      [&DL](const GlobalVariable *GV1, const GlobalVariable *GV2) {
+        Type *Ty1 = cast<PointerType>(GV1->getType())->getElementType();
+        Type *Ty2 = cast<PointerType>(GV2->getType())->getElementType();
+
+        return (DL.getTypeAllocSize(Ty1) < DL.getTypeAllocSize(Ty2));
+      });
+
+  // If we want to just blindly group all globals together, do so.
+  if (!GlobalMergeGroupByUse) {
+    BitVector AllGlobals(Globals.size());
+    AllGlobals.set();
+    return doMerge(Globals, AllGlobals, M, isConst, AddrSpace);
+  }
+
+  // If we want to be smarter, look at all uses of each global, to try to
+  // discover all sets of globals used together, and how many times each of
+  // these sets occured.
+  //
+  // Keep this reasonably efficient, by having an append-only list of all sets
+  // discovered so far (UsedGlobalSet), and mapping each "together-ness" unit of
+  // code (currently, a Function) to the set of globals seen so far that are
+  // used together in that unit (GlobalUsesByFunction).
+  //
+  // When we look at the Nth global, we now that any new set is either:
+  // - the singleton set {N}, containing this global only, or
+  // - the union of {N} and a previously-discovered set, containing some
+  //   combination of the previous N-1 globals.
+  // Using that knowledge, when looking at the Nth global, we can keep:
+  // - a reference to the singleton set {N} (CurGVOnlySetIdx)
+  // - a list mapping each previous set to its union with {N} (EncounteredUGS),
+  //   if it actually occurs.
+
+  // We keep track of the sets of globals used together "close enough".
+  struct UsedGlobalSet {
+    UsedGlobalSet(size_t Size) : Globals(Size), UsageCount(1) {}
+    BitVector Globals;
+    unsigned UsageCount;
+  };
+
+  // Each set is unique in UsedGlobalSets.
+  std::vector<UsedGlobalSet> UsedGlobalSets;
+
+  // Avoid repeating the create-global-set pattern.
+  auto CreateGlobalSet = [&]() -> UsedGlobalSet & {
+    UsedGlobalSets.emplace_back(Globals.size());
+    return UsedGlobalSets.back();
+  };
+
+  // The first set is the empty set.
+  CreateGlobalSet().UsageCount = 0;
+
+  // We define "close enough" to be "in the same function".
+  // FIXME: Grouping uses by function is way too aggressive, so we should have
+  // a better metric for distance between uses.
+  // The obvious alternative would be to group by BasicBlock, but that's in
+  // turn too conservative..
+  // Anything in between wouldn't be trivial to compute, so just stick with
+  // per-function grouping.
+
+  // The value type is an index into UsedGlobalSets.
+  // The default (0) conveniently points to the empty set.
+  DenseMap<Function *, size_t /*UsedGlobalSetIdx*/> GlobalUsesByFunction;
+
+  // Now, look at each merge-eligible global in turn.
+
+  // Keep track of the sets we already encountered to which we added the
+  // current global.
+  // Each element matches the same-index element in UsedGlobalSets.
+  // This lets us efficiently tell whether a set has already been expanded to
+  // include the current global.
+  std::vector<size_t> EncounteredUGS;
+
+  for (size_t GI = 0, GE = Globals.size(); GI != GE; ++GI) {
+    GlobalVariable *GV = Globals[GI];
+
+    // Reset the encountered sets for this global...
+    std::fill(EncounteredUGS.begin(), EncounteredUGS.end(), 0);
+    // ...and grow it in case we created new sets for the previous global.
+    EncounteredUGS.resize(UsedGlobalSets.size());
+
+    // We might need to create a set that only consists of the current global.
+    // Keep track of its index into UsedGlobalSets.
+    size_t CurGVOnlySetIdx = 0;
+
+    // For each global, look at all its Uses.
+    for (auto &U : GV->uses()) {
+      // This Use might be a ConstantExpr.  We're interested in Instruction
+      // users, so look through ConstantExpr...
+      Use *UI, *UE;
+      if (ConstantExpr *CE = dyn_cast<ConstantExpr>(U.getUser())) {
+        if (CE->use_empty())
+          continue;
+        UI = &*CE->use_begin();
+        UE = nullptr;
+      } else if (isa<Instruction>(U.getUser())) {
+        UI = &U;
+        UE = UI->getNext();
+      } else {
+        continue;
+      }
+
+      // ...to iterate on all the instruction users of the global.
+      // Note that we iterate on Uses and not on Users to be able to getNext().
+      for (; UI != UE; UI = UI->getNext()) {
+        Instruction *I = dyn_cast<Instruction>(UI->getUser());
+        if (!I)
+          continue;
+
+        Function *ParentFn = I->getParent()->getParent();
+
+        // If we're only optimizing for size, ignore non-minsize functions.
+        if (OnlyOptimizeForSize &&
+            !ParentFn->hasFnAttribute(Attribute::MinSize))
+          continue;
+
+        size_t UGSIdx = GlobalUsesByFunction[ParentFn];
+
+        // If this is the first global the basic block uses, map it to the set
+        // consisting of this global only.
+        if (!UGSIdx) {
+          // If that set doesn't exist yet, create it.
+          if (!CurGVOnlySetIdx) {
+            CurGVOnlySetIdx = UsedGlobalSets.size();
+            CreateGlobalSet().Globals.set(GI);
+          } else {
+            ++UsedGlobalSets[CurGVOnlySetIdx].UsageCount;
+          }
+
+          GlobalUsesByFunction[ParentFn] = CurGVOnlySetIdx;
+          continue;
+        }
+
+        // If we already encountered this BB, just increment the counter.
+        if (UsedGlobalSets[UGSIdx].Globals.test(GI)) {
+          ++UsedGlobalSets[UGSIdx].UsageCount;
+          continue;
+        }
+
+        // If not, the previous set wasn't actually used in this function.
+        --UsedGlobalSets[UGSIdx].UsageCount;
+
+        // If we already expanded the previous set to include this global, just
+        // reuse that expanded set.
+        if (size_t ExpandedIdx = EncounteredUGS[UGSIdx]) {
+          ++UsedGlobalSets[ExpandedIdx].UsageCount;
+          GlobalUsesByFunction[ParentFn] = ExpandedIdx;
+          continue;
+        }
+
+        // If not, create a new set consisting of the union of the previous set
+        // and this global.  Mark it as encountered, so we can reuse it later.
+        GlobalUsesByFunction[ParentFn] = EncounteredUGS[UGSIdx] =
+            UsedGlobalSets.size();
+
+        UsedGlobalSet &NewUGS = CreateGlobalSet();
+        NewUGS.Globals.set(GI);
+        NewUGS.Globals |= UsedGlobalSets[UGSIdx].Globals;
+      }
+    }
+  }
+
+  // Now we found a bunch of sets of globals used together.  We accumulated
+  // the number of times we encountered the sets (i.e., the number of blocks
+  // that use that exact set of globals).
+  //
+  // Multiply that by the size of the set to give us a crude profitability
+  // metric.
+  std::sort(UsedGlobalSets.begin(), UsedGlobalSets.end(),
+            [](const UsedGlobalSet &UGS1, const UsedGlobalSet &UGS2) {
+              return UGS1.Globals.count() * UGS1.UsageCount <
+                     UGS2.Globals.count() * UGS2.UsageCount;
+            });
+
+  // We can choose to merge all globals together, but ignore globals never used
+  // with another global.  This catches the obviously non-profitable cases of
+  // having a single global, but is aggressive enough for any other case.
+  if (GlobalMergeIgnoreSingleUse) {
+    BitVector AllGlobals(Globals.size());
+    for (size_t i = 0, e = UsedGlobalSets.size(); i != e; ++i) {
+      const UsedGlobalSet &UGS = UsedGlobalSets[e - i - 1];
+      if (UGS.UsageCount == 0)
+        continue;
+      if (UGS.Globals.count() > 1)
+        AllGlobals |= UGS.Globals;
+    }
+    return doMerge(Globals, AllGlobals, M, isConst, AddrSpace);
+  }
 
-    return (DL->getTypeAllocSize(Ty1) < DL->getTypeAllocSize(Ty2));
-  });
+  // Starting from the sets with the best (=biggest) profitability, find a
+  // good combination.
+  // The ideal (and expensive) solution can only be found by trying all
+  // combinations, looking for the one with the best profitability.
+  // Don't be smart about it, and just pick the first compatible combination,
+  // starting with the sets with the best profitability.
+  BitVector PickedGlobals(Globals.size());
+  bool Changed = false;
+
+  for (size_t i = 0, e = UsedGlobalSets.size(); i != e; ++i) {
+    const UsedGlobalSet &UGS = UsedGlobalSets[e - i - 1];
+    if (UGS.UsageCount == 0)
+      continue;
+    if (PickedGlobals.anyCommon(UGS.Globals))
+      continue;
+    PickedGlobals |= UGS.Globals;
+    // If the set only contains one global, there's no point in merging.
+    // Ignore the global for inclusion in other sets though, so keep it in
+    // PickedGlobals.
+    if (UGS.Globals.count() < 2)
+      continue;
+    Changed |= doMerge(Globals, UGS.Globals, M, isConst, AddrSpace);
+  }
+
+  return Changed;
+}
+
+bool GlobalMerge::doMerge(SmallVectorImpl<GlobalVariable *> &Globals,
+                          const BitVector &GlobalSet, Module &M, bool isConst,
+                          unsigned AddrSpace) const {
 
   Type *Int32Ty = Type::getInt32Ty(M.getContext());
+  auto &DL = M.getDataLayout();
 
   assert(Globals.size() > 1);
 
-  // FIXME: This simple solution merges globals all together as maximum as
-  // possible. However, with this solution it would be hard to remove dead
-  // global symbols at link-time. An alternative solution could be checking
-  // global symbols references function by function, and make the symbols
-  // being referred in the same function merged and we would probably need
-  // to introduce heuristic algorithm to solve the merge conflict from
-  // different functions.
-  for (size_t i = 0, e = Globals.size(); i != e; ) {
-    size_t j = 0;
+  DEBUG(dbgs() << " Trying to merge set, starts with #"
+               << GlobalSet.find_first() << "\n");
+
+  ssize_t i = GlobalSet.find_first();
+  while (i != -1) {
+    ssize_t j = 0;
     uint64_t MergedSize = 0;
     std::vector<Type*> Tys;
     std::vector<Constant*> Inits;
 
     bool HasExternal = false;
     GlobalVariable *TheFirstExternal = 0;
-    for (j = i; j != e; ++j) {
+    for (j = i; j != -1; j = GlobalSet.find_next(j)) {
       Type *Ty = Globals[j]->getType()->getElementType();
-      MergedSize += DL->getTypeAllocSize(Ty);
+      MergedSize += DL.getTypeAllocSize(Ty);
       if (MergedSize > MaxOffset) {
         break;
       }
@@ -215,13 +460,13 @@ bool GlobalMerge::doMerge(SmallVectorImpl<GlobalVariable*> &Globals,
                     : "_MergedGlobals",
         nullptr, GlobalVariable::NotThreadLocal, AddrSpace);
 
-    for (size_t k = i; k < j; ++k) {
+    for (ssize_t k = i, idx = 0; k != j; k = GlobalSet.find_next(k)) {
       GlobalValue::LinkageTypes Linkage = Globals[k]->getLinkage();
       std::string Name = Globals[k]->getName();
 
       Constant *Idx[2] = {
         ConstantInt::get(Int32Ty, 0),
-        ConstantInt::get(Int32Ty, k-i)
+        ConstantInt::get(Int32Ty, idx++)
       };
       Constant *GEP =
           ConstantExpr::getInBoundsGetElementPtr(MergedTy, MergedGV, Idx);
@@ -231,8 +476,7 @@ bool GlobalMerge::doMerge(SmallVectorImpl<GlobalVariable*> &Globals,
       if (Linkage != GlobalValue::InternalLinkage) {
         // Generate a new alias...
         auto *PTy = cast<PointerType>(GEP->getType());
-        GlobalAlias::create(PTy->getElementType(), PTy->getAddressSpace(),
-                            Linkage, Name, GEP, &M);
+        GlobalAlias::create(PTy, Linkage, Name, GEP, &M);
       }
 
       NumMerged++;
@@ -284,6 +528,7 @@ bool GlobalMerge::doInitialization(Module &M) {
   if (!EnableGlobalMerge)
     return false;
 
+  auto &DL = M.getDataLayout();
   DenseMap<unsigned, SmallVector<GlobalVariable*, 16> > Globals, ConstGlobals,
                                                         BSSGlobals;
   bool Changed = false;
@@ -306,9 +551,9 @@ bool GlobalMerge::doInitialization(Module &M) {
     unsigned AddressSpace = PT->getAddressSpace();
 
     // Ignore fancy-aligned globals for now.
-    unsigned Alignment = DL->getPreferredAlignment(I);
+    unsigned Alignment = DL.getPreferredAlignment(I);
     Type *Ty = I->getType()->getElementType();
-    if (Alignment > DL->getABITypeAlignment(Ty))
+    if (Alignment > DL.getABITypeAlignment(Ty))
       continue;
 
     // Ignore all 'special' globals.
@@ -320,7 +565,7 @@ bool GlobalMerge::doInitialization(Module &M) {
     if (isMustKeepGlobalVariable(I))
       continue;
 
-    if (DL->getTypeAllocSize(Ty) < MaxOffset) {
+    if (DL.getTypeAllocSize(Ty) < MaxOffset) {
       if (TargetLoweringObjectFile::getKindForGlobal(I, *TM).isBSSLocal())
         BSSGlobals[AddressSpace].push_back(I);
       else if (I->isConstant())
@@ -358,6 +603,7 @@ bool GlobalMerge::doFinalization(Module &M) {
   return false;
 }
 
-Pass *llvm::createGlobalMergePass(const TargetMachine *TM, unsigned Offset) {
-  return new GlobalMerge(TM, Offset);
+Pass *llvm::createGlobalMergePass(const TargetMachine *TM, unsigned Offset,
+                                  bool OnlyOptimizeForSize) {
+  return new GlobalMerge(TM, Offset, OnlyOptimizeForSize);
 }
diff --git a/lib/CodeGen/IfConversion.cpp b/lib/CodeGen/IfConversion.cpp
index b8799a563fc..ee0532bfc63 100644
--- a/lib/CodeGen/IfConversion.cpp
+++ b/lib/CodeGen/IfConversion.cpp
@@ -24,7 +24,6 @@
 #include "llvm/CodeGen/MachineModuleInfo.h"
 #include "llvm/CodeGen/MachineRegisterInfo.h"
 #include "llvm/CodeGen/TargetSchedule.h"
-#include "llvm/MC/MCInstrItineraries.h"
 #include "llvm/Support/CommandLine.h"
 #include "llvm/Support/Debug.h"
 #include "llvm/Support/ErrorHandling.h"
@@ -171,9 +170,12 @@ namespace {
     bool PreRegAlloc;
     bool MadeChange;
     int FnNum;
+    std::function<bool(const Function &)> PredicateFtor;
+
   public:
     static char ID;
-    IfConverter() : MachineFunctionPass(ID), FnNum(-1) {
+    IfConverter(std::function<bool(const Function &)> Ftor = nullptr)
+        : MachineFunctionPass(ID), FnNum(-1), PredicateFtor(Ftor) {
       initializeIfConverterPass(*PassRegistry::getPassRegistry());
     }
 
@@ -195,8 +197,7 @@ namespace {
     bool ValidDiamond(BBInfo &TrueBBI, BBInfo &FalseBBI,
                       unsigned &Dups1, unsigned &Dups2) const;
     void ScanInstructions(BBInfo &BBI);
-    BBInfo &AnalyzeBlock(MachineBasicBlock *BB,
-                         std::vector<IfcvtToken*> &Tokens);
+    void AnalyzeBlock(MachineBasicBlock *MBB, std::vector<IfcvtToken*> &Tokens);
     bool FeasibilityAnalysis(BBInfo &BBI, SmallVectorImpl<MachineOperand> &Cond,
                              bool isTriangle = false, bool RevBranch = false);
     void AnalyzeBlocks(MachineFunction &MF, std::vector<IfcvtToken*> &Tokens);
@@ -271,6 +272,9 @@ INITIALIZE_PASS_DEPENDENCY(MachineBranchProbabilityInfo)
 INITIALIZE_PASS_END(IfConverter, "if-converter", "If Converter", false, false)
 
 bool IfConverter::runOnMachineFunction(MachineFunction &MF) {
+  if (PredicateFtor && !PredicateFtor(*MF.getFunction()))
+    return false;
+
   const TargetSubtargetInfo &ST = MF.getSubtarget();
   TLI = ST.getTargetLowering();
   TII = ST.getInstrInfo();
@@ -759,155 +763,185 @@ bool IfConverter::FeasibilityAnalysis(BBInfo &BBI,
 /// AnalyzeBlock - Analyze the structure of the sub-CFG starting from
 /// the specified block. Record its successors and whether it looks like an
 /// if-conversion candidate.
-IfConverter::BBInfo &IfConverter::AnalyzeBlock(MachineBasicBlock *BB,
-                                             std::vector<IfcvtToken*> &Tokens) {
-  BBInfo &BBI = BBAnalysis[BB->getNumber()];
+void IfConverter::AnalyzeBlock(MachineBasicBlock *MBB,
+                               std::vector<IfcvtToken*> &Tokens) {
+  struct BBState {
+    BBState(MachineBasicBlock *BB) : MBB(BB), SuccsAnalyzed(false) {}
+    MachineBasicBlock *MBB;
+
+    /// This flag is true if MBB's successors have been analyzed.
+    bool SuccsAnalyzed;
+  };
 
-  if (BBI.IsAnalyzed || BBI.IsBeingAnalyzed)
-    return BBI;
+  // Push MBB to the stack.
+  SmallVector<BBState, 16> BBStack(1, MBB);
 
-  BBI.BB = BB;
-  BBI.IsBeingAnalyzed = true;
+  while (!BBStack.empty()) {
+    BBState &State = BBStack.back();
+    MachineBasicBlock *BB = State.MBB;
+    BBInfo &BBI = BBAnalysis[BB->getNumber()];
 
-  ScanInstructions(BBI);
+    if (!State.SuccsAnalyzed) {
+      if (BBI.IsAnalyzed || BBI.IsBeingAnalyzed) {
+        BBStack.pop_back();
+        continue;
+      }
 
-  // Unanalyzable or ends with fallthrough or unconditional branch, or if is not
-  // considered for ifcvt anymore.
-  if (!BBI.IsBrAnalyzable || BBI.BrCond.empty() || BBI.IsDone) {
-    BBI.IsBeingAnalyzed = false;
-    BBI.IsAnalyzed = true;
-    return BBI;
-  }
+      BBI.BB = BB;
+      BBI.IsBeingAnalyzed = true;
 
-  // Do not ifcvt if either path is a back edge to the entry block.
-  if (BBI.TrueBB == BB || BBI.FalseBB == BB) {
-    BBI.IsBeingAnalyzed = false;
-    BBI.IsAnalyzed = true;
-    return BBI;
-  }
+      ScanInstructions(BBI);
 
-  // Do not ifcvt if true and false fallthrough blocks are the same.
-  if (!BBI.FalseBB) {
-    BBI.IsBeingAnalyzed = false;
-    BBI.IsAnalyzed = true;
-    return BBI;
-  }
+      // Unanalyzable or ends with fallthrough or unconditional branch, or if is
+      // not considered for ifcvt anymore.
+      if (!BBI.IsBrAnalyzable || BBI.BrCond.empty() || BBI.IsDone) {
+        BBI.IsBeingAnalyzed = false;
+        BBI.IsAnalyzed = true;
+        BBStack.pop_back();
+        continue;
+      }
 
-  BBInfo &TrueBBI  = AnalyzeBlock(BBI.TrueBB, Tokens);
-  BBInfo &FalseBBI = AnalyzeBlock(BBI.FalseBB, Tokens);
+      // Do not ifcvt if either path is a back edge to the entry block.
+      if (BBI.TrueBB == BB || BBI.FalseBB == BB) {
+        BBI.IsBeingAnalyzed = false;
+        BBI.IsAnalyzed = true;
+        BBStack.pop_back();
+        continue;
+      }
 
-  if (TrueBBI.IsDone && FalseBBI.IsDone) {
-    BBI.IsBeingAnalyzed = false;
-    BBI.IsAnalyzed = true;
-    return BBI;
-  }
+      // Do not ifcvt if true and false fallthrough blocks are the same.
+      if (!BBI.FalseBB) {
+        BBI.IsBeingAnalyzed = false;
+        BBI.IsAnalyzed = true;
+        BBStack.pop_back();
+        continue;
+      }
 
-  SmallVector<MachineOperand, 4> RevCond(BBI.BrCond.begin(), BBI.BrCond.end());
-  bool CanRevCond = !TII->ReverseBranchCondition(RevCond);
+      // Push the False and True blocks to the stack.
+      State.SuccsAnalyzed = true;
+      BBStack.push_back(BBI.FalseBB);
+      BBStack.push_back(BBI.TrueBB);
+      continue;
+    }
 
-  unsigned Dups = 0;
-  unsigned Dups2 = 0;
-  bool TNeedSub = !TrueBBI.Predicate.empty();
-  bool FNeedSub = !FalseBBI.Predicate.empty();
-  bool Enqueued = false;
+    BBInfo &TrueBBI = BBAnalysis[BBI.TrueBB->getNumber()];
+    BBInfo &FalseBBI = BBAnalysis[BBI.FalseBB->getNumber()];
 
-  BranchProbability Prediction = MBPI->getEdgeProbability(BB, TrueBBI.BB);
+    if (TrueBBI.IsDone && FalseBBI.IsDone) {
+      BBI.IsBeingAnalyzed = false;
+      BBI.IsAnalyzed = true;
+      BBStack.pop_back();
+      continue;
+    }
 
-  if (CanRevCond && ValidDiamond(TrueBBI, FalseBBI, Dups, Dups2) &&
-      MeetIfcvtSizeLimit(*TrueBBI.BB, (TrueBBI.NonPredSize - (Dups + Dups2) +
-                                       TrueBBI.ExtraCost), TrueBBI.ExtraCost2,
-                         *FalseBBI.BB, (FalseBBI.NonPredSize - (Dups + Dups2) +
-                                        FalseBBI.ExtraCost),FalseBBI.ExtraCost2,
-                         Prediction) &&
-      FeasibilityAnalysis(TrueBBI, BBI.BrCond) &&
-      FeasibilityAnalysis(FalseBBI, RevCond)) {
-    // Diamond:
-    //   EBB
-    //   / \_
-    //  |   |
-    // TBB FBB
-    //   \ /
-    //  TailBB
-    // Note TailBB can be empty.
-    Tokens.push_back(new IfcvtToken(BBI, ICDiamond, TNeedSub|FNeedSub, Dups,
-                                    Dups2));
-    Enqueued = true;
-  }
+    SmallVector<MachineOperand, 4>
+        RevCond(BBI.BrCond.begin(), BBI.BrCond.end());
+    bool CanRevCond = !TII->ReverseBranchCondition(RevCond);
 
-  if (ValidTriangle(TrueBBI, FalseBBI, false, Dups, Prediction) &&
-      MeetIfcvtSizeLimit(*TrueBBI.BB, TrueBBI.NonPredSize + TrueBBI.ExtraCost,
-                         TrueBBI.ExtraCost2, Prediction) &&
-      FeasibilityAnalysis(TrueBBI, BBI.BrCond, true)) {
-    // Triangle:
-    //   EBB
-    //   | \_
-    //   |  |
-    //   | TBB
-    //   |  /
-    //   FBB
-    Tokens.push_back(new IfcvtToken(BBI, ICTriangle, TNeedSub, Dups));
-    Enqueued = true;
-  }
+    unsigned Dups = 0;
+    unsigned Dups2 = 0;
+    bool TNeedSub = !TrueBBI.Predicate.empty();
+    bool FNeedSub = !FalseBBI.Predicate.empty();
+    bool Enqueued = false;
 
-  if (ValidTriangle(TrueBBI, FalseBBI, true, Dups, Prediction) &&
-      MeetIfcvtSizeLimit(*TrueBBI.BB, TrueBBI.NonPredSize + TrueBBI.ExtraCost,
-                         TrueBBI.ExtraCost2, Prediction) &&
-      FeasibilityAnalysis(TrueBBI, BBI.BrCond, true, true)) {
-    Tokens.push_back(new IfcvtToken(BBI, ICTriangleRev, TNeedSub, Dups));
-    Enqueued = true;
-  }
+    BranchProbability Prediction = MBPI->getEdgeProbability(BB, TrueBBI.BB);
 
-  if (ValidSimple(TrueBBI, Dups, Prediction) &&
-      MeetIfcvtSizeLimit(*TrueBBI.BB, TrueBBI.NonPredSize + TrueBBI.ExtraCost,
-                         TrueBBI.ExtraCost2, Prediction) &&
-      FeasibilityAnalysis(TrueBBI, BBI.BrCond)) {
-    // Simple (split, no rejoin):
-    //   EBB
-    //   | \_
-    //   |  |
-    //   | TBB---> exit
-    //   |
-    //   FBB
-    Tokens.push_back(new IfcvtToken(BBI, ICSimple, TNeedSub, Dups));
-    Enqueued = true;
-  }
+    if (CanRevCond && ValidDiamond(TrueBBI, FalseBBI, Dups, Dups2) &&
+        MeetIfcvtSizeLimit(*TrueBBI.BB, (TrueBBI.NonPredSize - (Dups + Dups2) +
+                                         TrueBBI.ExtraCost), TrueBBI.ExtraCost2,
+                           *FalseBBI.BB, (FalseBBI.NonPredSize - (Dups + Dups2) +
+                                        FalseBBI.ExtraCost),FalseBBI.ExtraCost2,
+                         Prediction) &&
+        FeasibilityAnalysis(TrueBBI, BBI.BrCond) &&
+        FeasibilityAnalysis(FalseBBI, RevCond)) {
+      // Diamond:
+      //   EBB
+      //   / \_
+      //  |   |
+      // TBB FBB
+      //   \ /
+      //  TailBB
+      // Note TailBB can be empty.
+      Tokens.push_back(new IfcvtToken(BBI, ICDiamond, TNeedSub|FNeedSub, Dups,
+                                      Dups2));
+      Enqueued = true;
+    }
 
-  if (CanRevCond) {
-    // Try the other path...
-    if (ValidTriangle(FalseBBI, TrueBBI, false, Dups,
-                      Prediction.getCompl()) &&
-        MeetIfcvtSizeLimit(*FalseBBI.BB,
-                           FalseBBI.NonPredSize + FalseBBI.ExtraCost,
-                           FalseBBI.ExtraCost2, Prediction.getCompl()) &&
-        FeasibilityAnalysis(FalseBBI, RevCond, true)) {
-      Tokens.push_back(new IfcvtToken(BBI, ICTriangleFalse, FNeedSub, Dups));
+    if (ValidTriangle(TrueBBI, FalseBBI, false, Dups, Prediction) &&
+        MeetIfcvtSizeLimit(*TrueBBI.BB, TrueBBI.NonPredSize + TrueBBI.ExtraCost,
+                           TrueBBI.ExtraCost2, Prediction) &&
+        FeasibilityAnalysis(TrueBBI, BBI.BrCond, true)) {
+      // Triangle:
+      //   EBB
+      //   | \_
+      //   |  |
+      //   | TBB
+      //   |  /
+      //   FBB
+      Tokens.push_back(new IfcvtToken(BBI, ICTriangle, TNeedSub, Dups));
       Enqueued = true;
     }
 
-    if (ValidTriangle(FalseBBI, TrueBBI, true, Dups,
-                      Prediction.getCompl()) &&
-        MeetIfcvtSizeLimit(*FalseBBI.BB,
-                           FalseBBI.NonPredSize + FalseBBI.ExtraCost,
-                           FalseBBI.ExtraCost2, Prediction.getCompl()) &&
-        FeasibilityAnalysis(FalseBBI, RevCond, true, true)) {
-      Tokens.push_back(new IfcvtToken(BBI, ICTriangleFRev, FNeedSub, Dups));
+    if (ValidTriangle(TrueBBI, FalseBBI, true, Dups, Prediction) &&
+        MeetIfcvtSizeLimit(*TrueBBI.BB, TrueBBI.NonPredSize + TrueBBI.ExtraCost,
+                           TrueBBI.ExtraCost2, Prediction) &&
+        FeasibilityAnalysis(TrueBBI, BBI.BrCond, true, true)) {
+      Tokens.push_back(new IfcvtToken(BBI, ICTriangleRev, TNeedSub, Dups));
       Enqueued = true;
     }
 
-    if (ValidSimple(FalseBBI, Dups, Prediction.getCompl()) &&
-        MeetIfcvtSizeLimit(*FalseBBI.BB,
-                           FalseBBI.NonPredSize + FalseBBI.ExtraCost,
-                           FalseBBI.ExtraCost2, Prediction.getCompl()) &&
-        FeasibilityAnalysis(FalseBBI, RevCond)) {
-      Tokens.push_back(new IfcvtToken(BBI, ICSimpleFalse, FNeedSub, Dups));
+    if (ValidSimple(TrueBBI, Dups, Prediction) &&
+        MeetIfcvtSizeLimit(*TrueBBI.BB, TrueBBI.NonPredSize + TrueBBI.ExtraCost,
+                           TrueBBI.ExtraCost2, Prediction) &&
+        FeasibilityAnalysis(TrueBBI, BBI.BrCond)) {
+      // Simple (split, no rejoin):
+      //   EBB
+      //   | \_
+      //   |  |
+      //   | TBB---> exit
+      //   |
+      //   FBB
+      Tokens.push_back(new IfcvtToken(BBI, ICSimple, TNeedSub, Dups));
       Enqueued = true;
     }
-  }
 
-  BBI.IsEnqueued = Enqueued;
-  BBI.IsBeingAnalyzed = false;
-  BBI.IsAnalyzed = true;
-  return BBI;
+    if (CanRevCond) {
+      // Try the other path...
+      if (ValidTriangle(FalseBBI, TrueBBI, false, Dups,
+                        Prediction.getCompl()) &&
+          MeetIfcvtSizeLimit(*FalseBBI.BB,
+                             FalseBBI.NonPredSize + FalseBBI.ExtraCost,
+                             FalseBBI.ExtraCost2, Prediction.getCompl()) &&
+          FeasibilityAnalysis(FalseBBI, RevCond, true)) {
+        Tokens.push_back(new IfcvtToken(BBI, ICTriangleFalse, FNeedSub, Dups));
+        Enqueued = true;
+      }
+
+      if (ValidTriangle(FalseBBI, TrueBBI, true, Dups,
+                        Prediction.getCompl()) &&
+          MeetIfcvtSizeLimit(*FalseBBI.BB,
+                             FalseBBI.NonPredSize + FalseBBI.ExtraCost,
+                           FalseBBI.ExtraCost2, Prediction.getCompl()) &&
+        FeasibilityAnalysis(FalseBBI, RevCond, true, true)) {
+        Tokens.push_back(new IfcvtToken(BBI, ICTriangleFRev, FNeedSub, Dups));
+        Enqueued = true;
+      }
+
+      if (ValidSimple(FalseBBI, Dups, Prediction.getCompl()) &&
+          MeetIfcvtSizeLimit(*FalseBBI.BB,
+                             FalseBBI.NonPredSize + FalseBBI.ExtraCost,
+                             FalseBBI.ExtraCost2, Prediction.getCompl()) &&
+          FeasibilityAnalysis(FalseBBI, RevCond)) {
+        Tokens.push_back(new IfcvtToken(BBI, ICSimpleFalse, FNeedSub, Dups));
+        Enqueued = true;
+      }
+    }
+
+    BBI.IsEnqueued = Enqueued;
+    BBI.IsBeingAnalyzed = false;
+    BBI.IsAnalyzed = true;
+    BBStack.pop_back();
+  }
 }
 
 /// AnalyzeBlocks - Analyze all blocks and find entries for all if-conversion
@@ -975,26 +1009,37 @@ void IfConverter::RemoveExtraEdges(BBInfo &BBI) {
 /// Behaves like LiveRegUnits::StepForward() but also adds implicit uses to all
 /// values defined in MI which are not live/used by MI.
 static void UpdatePredRedefs(MachineInstr *MI, LivePhysRegs &Redefs) {
-  for (ConstMIBundleOperands Ops(MI); Ops.isValid(); ++Ops) {
-    if (!Ops->isReg() || !Ops->isKill())
+  SmallVector<std::pair<unsigned, const MachineOperand*>, 4> Clobbers;
+  Redefs.stepForward(*MI, Clobbers);
+
+  // Now add the implicit uses for each of the clobbered values.
+  for (auto Reg : Clobbers) {
+    // FIXME: Const cast here is nasty, but better than making StepForward
+    // take a mutable instruction instead of const.
+    MachineOperand &Op = const_cast<MachineOperand&>(*Reg.second);
+    MachineInstr *OpMI = Op.getParent();
+    MachineInstrBuilder MIB(*OpMI->getParent()->getParent(), OpMI);
+    if (Op.isRegMask()) {
+      // First handle regmasks.  They clobber any entries in the mask which
+      // means that we need a def for those registers.
+      MIB.addReg(Reg.first, RegState::Implicit | RegState::Undef);
+
+      // We also need to add an implicit def of this register for the later
+      // use to read from.
+      // For the register allocator to have allocated a register clobbered
+      // by the call which is used later, it must be the case that
+      // the call doesn't return.
+      MIB.addReg(Reg.first, RegState::Implicit | RegState::Define);
       continue;
-    unsigned Reg = Ops->getReg();
-    if (Reg == 0)
-      continue;
-    Redefs.removeReg(Reg);
-  }
-  for (MIBundleOperands Ops(MI); Ops.isValid(); ++Ops) {
-    if (!Ops->isReg() || !Ops->isDef())
-      continue;
-    unsigned Reg = Ops->getReg();
-    if (Reg == 0 || Redefs.contains(Reg))
-      continue;
-    Redefs.addReg(Reg);
-
-    MachineOperand &Op = *Ops;
-    MachineInstr *MI = Op.getParent();
-    MachineInstrBuilder MIB(*MI->getParent()->getParent(), MI);
-    MIB.addReg(Reg, RegState::Implicit | RegState::Undef);
+    }
+    assert(Op.isReg() && "Register operand required");
+    if (Op.isDead()) {
+      // If we found a dead def, but it needs to be live, then remove the dead
+      // flag.
+      if (Redefs.contains(Op.getReg()))
+        Op.setIsDead(false);
+    }
+    MIB.addReg(Reg.first, RegState::Implicit | RegState::Undef);
   }
 }
 
@@ -1339,15 +1384,9 @@ bool IfConverter::IfConvertDiamond(BBInfo &BBI, IfcvtKind Kind,
   Redefs.addLiveIns(BBI1->BB);
 
   // Remove the duplicated instructions at the beginnings of both paths.
-  MachineBasicBlock::iterator DI1 = BBI1->BB->begin();
-  MachineBasicBlock::iterator DI2 = BBI2->BB->begin();
-  MachineBasicBlock::iterator DIE1 = BBI1->BB->end();
-  MachineBasicBlock::iterator DIE2 = BBI2->BB->end();
   // Skip dbg_value instructions
-  while (DI1 != DIE1 && DI1->isDebugValue())
-    ++DI1;
-  while (DI2 != DIE2 && DI2->isDebugValue())
-    ++DI2;
+  MachineBasicBlock::iterator DI1 = BBI1->BB->getFirstNonDebugInstr();
+  MachineBasicBlock::iterator DI2 = BBI2->BB->getFirstNonDebugInstr();
   BBI1->NonPredSize -= NumDups1;
   BBI2->NonPredSize -= NumDups1;
 
@@ -1374,7 +1413,8 @@ bool IfConverter::IfConvertDiamond(BBInfo &BBI, IfcvtKind Kind,
 
   for (MachineBasicBlock::const_iterator I = BBI1->BB->begin(), E = DI1; I != E;
        ++I) {
-    Redefs.stepForward(*I);
+    SmallVector<std::pair<unsigned, const MachineOperand*>, 4> IgnoredClobbers;
+    Redefs.stepForward(*I, IgnoredClobbers);
   }
   BBI.BB->splice(BBI.BB->end(), BBI1->BB, BBI1->BB->begin(), DI1);
   BBI2->BB->erase(BBI2->BB->begin(), DI2);
@@ -1508,10 +1548,9 @@ bool IfConverter::IfConvertDiamond(BBInfo &BBI, IfcvtKind Kind,
 }
 
 static bool MaySpeculate(const MachineInstr *MI,
-                         SmallSet<unsigned, 4> &LaterRedefs,
-                         const TargetInstrInfo *TII) {
+                         SmallSet<unsigned, 4> &LaterRedefs) {
   bool SawStore = true;
-  if (!MI->isSafeToMove(TII, nullptr, SawStore))
+  if (!MI->isSafeToMove(nullptr, SawStore))
     return false;
 
   for (unsigned i = 0, e = MI->getNumOperands(); i != e; ++i) {
@@ -1542,7 +1581,7 @@ void IfConverter::PredicateBlock(BBInfo &BBI,
     // It may be possible not to predicate an instruction if it's the 'true'
     // side of a diamond and the 'false' side may re-define the instruction's
     // defs.
-    if (MaySpec && MaySpeculate(I, *LaterRedefs, TII)) {
+    if (MaySpec && MaySpeculate(I, *LaterRedefs)) {
       AnyUnpred = true;
       continue;
     }
@@ -1681,3 +1720,8 @@ void IfConverter::MergeBlocks(BBInfo &ToBBI, BBInfo &FromBBI, bool AddEdges) {
   ToBBI.IsAnalyzed = false;
   FromBBI.IsAnalyzed = false;
 }
+
+FunctionPass *
+llvm::createIfConverter(std::function<bool(const Function &)> Ftor) {
+  return new IfConverter(Ftor);
+}
diff --git a/lib/CodeGen/ImplicitNullChecks.cpp b/lib/CodeGen/ImplicitNullChecks.cpp
new file mode 100644
index 00000000000..fbe057791c5
--- /dev/null
+++ b/lib/CodeGen/ImplicitNullChecks.cpp
@@ -0,0 +1,348 @@
+//===-- ImplicitNullChecks.cpp - Fold null checks into memory accesses ----===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This pass turns explicit null checks of the form
+//
+//   test %r10, %r10
+//   je throw_npe
+//   movl (%r10), %esi
+//   ...
+//
+// to
+//
+//   faulting_load_op("movl (%r10), %esi", throw_npe)
+//   ...
+//
+// With the help of a runtime that understands the .fault_maps section,
+// faulting_load_op branches to throw_npe if executing movl (%r10), %esi incurs
+// a page fault.
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/ADT/DenseSet.h"
+#include "llvm/ADT/SmallVector.h"
+#include "llvm/ADT/Statistic.h"
+#include "llvm/CodeGen/Passes.h"
+#include "llvm/CodeGen/MachineFunction.h"
+#include "llvm/CodeGen/MachineMemOperand.h"
+#include "llvm/CodeGen/MachineOperand.h"
+#include "llvm/CodeGen/MachineFunctionPass.h"
+#include "llvm/CodeGen/MachineInstrBuilder.h"
+#include "llvm/CodeGen/MachineRegisterInfo.h"
+#include "llvm/CodeGen/MachineModuleInfo.h"
+#include "llvm/IR/BasicBlock.h"
+#include "llvm/IR/Instruction.h"
+#include "llvm/Support/CommandLine.h"
+#include "llvm/Support/Debug.h"
+#include "llvm/Target/TargetSubtargetInfo.h"
+#include "llvm/Target/TargetInstrInfo.h"
+
+using namespace llvm;
+
+static cl::opt<unsigned> PageSize("imp-null-check-page-size",
+                                  cl::desc("The page size of the target in "
+                                           "bytes"),
+                                  cl::init(4096));
+
+#define DEBUG_TYPE "implicit-null-checks"
+
+STATISTIC(NumImplicitNullChecks,
+          "Number of explicit null checks made implicit");
+
+namespace {
+
+class ImplicitNullChecks : public MachineFunctionPass {
+  /// Represents one null check that can be made implicit.
+  struct NullCheck {
+    // The memory operation the null check can be folded into.
+    MachineInstr *MemOperation;
+
+    // The instruction actually doing the null check (Ptr != 0).
+    MachineInstr *CheckOperation;
+
+    // The block the check resides in.
+    MachineBasicBlock *CheckBlock;
+
+    // The block branched to if the pointer is non-null.
+    MachineBasicBlock *NotNullSucc;
+
+    // The block branched to if the pointer is null.
+    MachineBasicBlock *NullSucc;
+
+    NullCheck()
+        : MemOperation(), CheckOperation(), CheckBlock(), NotNullSucc(),
+          NullSucc() {}
+
+    explicit NullCheck(MachineInstr *memOperation, MachineInstr *checkOperation,
+                       MachineBasicBlock *checkBlock,
+                       MachineBasicBlock *notNullSucc,
+                       MachineBasicBlock *nullSucc)
+        : MemOperation(memOperation), CheckOperation(checkOperation),
+          CheckBlock(checkBlock), NotNullSucc(notNullSucc), NullSucc(nullSucc) {
+    }
+  };
+
+  const TargetInstrInfo *TII = nullptr;
+  const TargetRegisterInfo *TRI = nullptr;
+  MachineModuleInfo *MMI = nullptr;
+
+  bool analyzeBlockForNullChecks(MachineBasicBlock &MBB,
+                                 SmallVectorImpl<NullCheck> &NullCheckList);
+  MachineInstr *insertFaultingLoad(MachineInstr *LoadMI, MachineBasicBlock *MBB,
+                                   MCSymbol *HandlerLabel);
+  void rewriteNullChecks(ArrayRef<NullCheck> NullCheckList);
+
+public:
+  static char ID;
+
+  ImplicitNullChecks() : MachineFunctionPass(ID) {
+    initializeImplicitNullChecksPass(*PassRegistry::getPassRegistry());
+  }
+
+  bool runOnMachineFunction(MachineFunction &MF) override;
+};
+}
+
+bool ImplicitNullChecks::runOnMachineFunction(MachineFunction &MF) {
+  TII = MF.getSubtarget().getInstrInfo();
+  TRI = MF.getRegInfo().getTargetRegisterInfo();
+  MMI = &MF.getMMI();
+
+  SmallVector<NullCheck, 16> NullCheckList;
+
+  for (auto &MBB : MF)
+    analyzeBlockForNullChecks(MBB, NullCheckList);
+
+  if (!NullCheckList.empty())
+    rewriteNullChecks(NullCheckList);
+
+  return !NullCheckList.empty();
+}
+
+/// Analyze MBB to check if its terminating branch can be turned into an
+/// implicit null check.  If yes, append a description of the said null check to
+/// NullCheckList and return true, else return false.
+bool ImplicitNullChecks::analyzeBlockForNullChecks(
+    MachineBasicBlock &MBB, SmallVectorImpl<NullCheck> &NullCheckList) {
+  typedef TargetInstrInfo::MachineBranchPredicate MachineBranchPredicate;
+
+  MDNode *BranchMD =
+      MBB.getBasicBlock()
+          ? MBB.getBasicBlock()->getTerminator()->getMetadata("make.implicit")
+          : nullptr;
+  if (!BranchMD)
+    return false;
+
+  MachineBranchPredicate MBP;
+
+  if (TII->AnalyzeBranchPredicate(MBB, MBP, true))
+    return false;
+
+  // Is the predicate comparing an integer to zero?
+  if (!(MBP.LHS.isReg() && MBP.RHS.isImm() && MBP.RHS.getImm() == 0 &&
+        (MBP.Predicate == MachineBranchPredicate::PRED_NE ||
+         MBP.Predicate == MachineBranchPredicate::PRED_EQ)))
+    return false;
+
+  // If we cannot erase the test instruction itself, then making the null check
+  // implicit does not buy us much.
+  if (!MBP.SingleUseCondition)
+    return false;
+
+  MachineBasicBlock *NotNullSucc, *NullSucc;
+
+  if (MBP.Predicate == MachineBranchPredicate::PRED_NE) {
+    NotNullSucc = MBP.TrueDest;
+    NullSucc = MBP.FalseDest;
+  } else {
+    NotNullSucc = MBP.FalseDest;
+    NullSucc = MBP.TrueDest;
+  }
+
+  // We handle the simplest case for now.  We can potentially do better by using
+  // the machine dominator tree.
+  if (NotNullSucc->pred_size() != 1)
+    return false;
+
+  // Starting with a code fragment like:
+  //
+  //   test %RAX, %RAX
+  //   jne LblNotNull
+  //
+  //  LblNull:
+  //   callq throw_NullPointerException
+  //
+  //  LblNotNull:
+  //   Inst0
+  //   Inst1
+  //   ...
+  //   Def = Load (%RAX + <offset>)
+  //   ...
+  //
+  //
+  // we want to end up with
+  //
+  //   Def = TrappingLoad (%RAX + <offset>), LblNull
+  //   jmp LblNotNull ;; explicit or fallthrough
+  //
+  //  LblNotNull:
+  //   Inst0
+  //   Inst1
+  //   ...
+  //
+  //  LblNull:
+  //   callq throw_NullPointerException
+  //
+
+  unsigned PointerReg = MBP.LHS.getReg();
+
+  // As we scan NotNullSucc for a suitable load instruction, we keep track of
+  // the registers defined and used by the instructions we scan past.  This bit
+  // of information lets us decide if it is legal to hoist the load instruction
+  // we find (if we do find such an instruction) to before NotNullSucc.
+  DenseSet<unsigned> RegDefs, RegUses;
+
+  // Returns true if it is safe to reorder MI to before NotNullSucc.
+  auto IsSafeToHoist = [&](MachineInstr *MI) {
+    // Right now we don't want to worry about LLVM's memory model.  This can be
+    // made more precise later.
+    for (auto *MMO : MI->memoperands())
+      if (!MMO->isUnordered())
+        return false;
+
+    for (auto &MO : MI->operands()) {
+      if (MO.isReg() && MO.getReg()) {
+        for (unsigned Reg : RegDefs)
+          if (TRI->regsOverlap(Reg, MO.getReg()))
+            return false;  // We found a write-after-write or read-after-write
+
+        if (MO.isDef())
+          for (unsigned Reg : RegUses)
+            if (TRI->regsOverlap(Reg, MO.getReg()))
+              return false;  // We found a write-after-read
+      }
+    }
+
+    return true;
+  };
+
+  for (auto MII = NotNullSucc->begin(), MIE = NotNullSucc->end(); MII != MIE;
+       ++MII) {
+    MachineInstr *MI = &*MII;
+    unsigned BaseReg, Offset;
+    if (TII->getMemOpBaseRegImmOfs(MI, BaseReg, Offset, TRI))
+      if (MI->mayLoad() && !MI->isPredicable() && BaseReg == PointerReg &&
+          Offset < PageSize && MI->getDesc().getNumDefs() <= 1 &&
+          IsSafeToHoist(MI)) {
+        NullCheckList.emplace_back(MI, MBP.ConditionDef, &MBB, NotNullSucc,
+                                   NullSucc);
+        return true;
+      }
+
+    // MI did not match our criteria for conversion to a trapping load.  Check
+    // if we can continue looking.
+
+    if (MI->mayStore() || MI->hasUnmodeledSideEffects())
+      return false;
+
+    for (auto *MMO : MI->memoperands())
+      // Right now we don't want to worry about LLVM's memory model.
+      if (!MMO->isUnordered())
+        return false;
+
+    // It _may_ be okay to reorder a later load instruction across MI.  Make a
+    // note of its operands so that we can make the legality check if we find a
+    // suitable load instruction:
+
+    for (auto &MO : MI->operands()) {
+      if (!MO.isReg() || !MO.getReg())
+        continue;
+
+      if (MO.isDef())
+        RegDefs.insert(MO.getReg());
+      else
+        RegUses.insert(MO.getReg());
+    }
+  }
+
+  return false;
+}
+
+/// Wrap a machine load instruction, LoadMI, into a FAULTING_LOAD_OP machine
+/// instruction.  The FAULTING_LOAD_OP instruction does the same load as LoadMI
+/// (defining the same register), and branches to HandlerLabel if the load
+/// faults.  The FAULTING_LOAD_OP instruction is inserted at the end of MBB.
+MachineInstr *ImplicitNullChecks::insertFaultingLoad(MachineInstr *LoadMI,
+                                                     MachineBasicBlock *MBB,
+                                                     MCSymbol *HandlerLabel) {
+  const unsigned NoRegister = 0; // Guaranteed to be the NoRegister value for
+                                 // all targets.
+
+  DebugLoc DL;
+  unsigned NumDefs = LoadMI->getDesc().getNumDefs();
+  assert(NumDefs <= 1 && "other cases unhandled!");
+
+  unsigned DefReg = NoRegister;
+  if (NumDefs != 0) {
+    DefReg = LoadMI->defs().begin()->getReg();
+    assert(std::distance(LoadMI->defs().begin(), LoadMI->defs().end()) == 1 &&
+           "expected exactly one def!");
+  }
+
+  auto MIB = BuildMI(MBB, DL, TII->get(TargetOpcode::FAULTING_LOAD_OP), DefReg)
+                 .addSym(HandlerLabel)
+                 .addImm(LoadMI->getOpcode());
+
+  for (auto &MO : LoadMI->uses())
+    MIB.addOperand(MO);
+
+  MIB.setMemRefs(LoadMI->memoperands_begin(), LoadMI->memoperands_end());
+
+  return MIB;
+}
+
+/// Rewrite the null checks in NullCheckList into implicit null checks.
+void ImplicitNullChecks::rewriteNullChecks(
+    ArrayRef<ImplicitNullChecks::NullCheck> NullCheckList) {
+  DebugLoc DL;
+
+  for (auto &NC : NullCheckList) {
+    MCSymbol *HandlerLabel = MMI->getContext().createTempSymbol();
+
+    // Remove the conditional branch dependent on the null check.
+    unsigned BranchesRemoved = TII->RemoveBranch(*NC.CheckBlock);
+    (void)BranchesRemoved;
+    assert(BranchesRemoved > 0 && "expected at least one branch!");
+
+    // Insert a faulting load where the conditional branch was originally.  We
+    // check earlier ensures that this bit of code motion is legal.  We do not
+    // touch the successors list for any basic block since we haven't changed
+    // control flow, we've just made it implicit.
+    insertFaultingLoad(NC.MemOperation, NC.CheckBlock, HandlerLabel);
+    NC.MemOperation->eraseFromParent();
+    NC.CheckOperation->eraseFromParent();
+
+    // Insert an *unconditional* branch to not-null successor.
+    TII->InsertBranch(*NC.CheckBlock, NC.NotNullSucc, nullptr, /*Cond=*/None,
+                      DL);
+
+    // Emit the HandlerLabel as an EH_LABEL.
+    BuildMI(*NC.NullSucc, NC.NullSucc->begin(), DL,
+            TII->get(TargetOpcode::EH_LABEL)).addSym(HandlerLabel);
+
+    NumImplicitNullChecks++;
+  }
+}
+
+char ImplicitNullChecks::ID = 0;
+char &llvm::ImplicitNullChecksID = ImplicitNullChecks::ID;
+INITIALIZE_PASS_BEGIN(ImplicitNullChecks, "implicit-null-checks",
+                      "Implicit null checks", false, false)
+INITIALIZE_PASS_END(ImplicitNullChecks, "implicit-null-checks",
+                    "Implicit null checks", false, false)
diff --git a/lib/CodeGen/InlineSpiller.cpp b/lib/CodeGen/InlineSpiller.cpp
index c7e7e58f397..9989f233d09 100644
--- a/lib/CodeGen/InlineSpiller.cpp
+++ b/lib/CodeGen/InlineSpiller.cpp
@@ -921,7 +921,7 @@ bool InlineSpiller::reMaterializeFor(LiveInterval &VirtReg,
 
   // Replace operands
   for (unsigned i = 0, e = Ops.size(); i != e; ++i) {
-    MachineOperand &MO = MI->getOperand(Ops[i].second);
+    MachineOperand &MO = Ops[i].first->getOperand(Ops[i].second);
     if (MO.isReg() && MO.isUse() && MO.getReg() == VirtReg.reg) {
       MO.setReg(NewVReg);
       MO.setIsKill();
@@ -1100,6 +1100,7 @@ foldMemoryOperand(ArrayRef<std::pair<MachineInstr*, unsigned> > Ops,
   SmallVector<unsigned, 8> FoldOps;
   for (unsigned i = 0, e = Ops.size(); i != e; ++i) {
     unsigned Idx = Ops[i].second;
+    assert(MI == Ops[i].first && "Instruction conflict during operand folding");
     MachineOperand &MO = MI->getOperand(Idx);
     if (MO.isImplicit()) {
       ImpReg = MO.getReg();
@@ -1232,7 +1233,7 @@ void InlineSpiller::spillAroundUses(unsigned Reg) {
       DebugLoc DL = MI->getDebugLoc();
       DEBUG(dbgs() << "Modifying debug info due to spill:" << "\t" << *MI);
       MachineBasicBlock *MBB = MI->getParent();
-      assert(cast<MDLocalVariable>(Var)->isValidLocationForIntrinsic(DL) &&
+      assert(cast<DILocalVariable>(Var)->isValidLocationForIntrinsic(DL) &&
              "Expected inlined-at fields to agree");
       BuildMI(*MBB, MBB->erase(MI), DL, TII.get(TargetOpcode::DBG_VALUE))
           .addFrameIndex(StackSlot)
diff --git a/lib/CodeGen/InterferenceCache.h b/lib/CodeGen/InterferenceCache.h
index 6519a806e57..18aa5c7c5ad 100644
--- a/lib/CodeGen/InterferenceCache.h
+++ b/lib/CodeGen/InterferenceCache.h
@@ -21,7 +21,7 @@ namespace llvm {
 
 class LiveIntervals;
 
-class InterferenceCache {
+class LLVM_LIBRARY_VISIBILITY InterferenceCache {
   const TargetRegisterInfo *TRI;
   LiveIntervalUnion *LIUArray;
   MachineFunction *MF;
diff --git a/lib/CodeGen/InterleavedAccessPass.cpp b/lib/CodeGen/InterleavedAccessPass.cpp
new file mode 100644
index 00000000000..53c8adc05d7
--- /dev/null
+++ b/lib/CodeGen/InterleavedAccessPass.cpp
@@ -0,0 +1,286 @@
+//=----------------------- InterleavedAccessPass.cpp -----------------------==//
+//
+// The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This file implements the Interleaved Access pass, which identifies
+// interleaved memory accesses and transforms into target specific intrinsics.
+//
+// An interleaved load reads data from memory into several vectors, with
+// DE-interleaving the data on a factor. An interleaved store writes several
+// vectors to memory with RE-interleaving the data on a factor.
+//
+// As interleaved accesses are hard to be identified in CodeGen (mainly because
+// the VECTOR_SHUFFLE DAG node is quite different from the shufflevector IR),
+// we identify and transform them to intrinsics in this pass. So the intrinsics
+// can be easily matched into target specific instructions later in CodeGen.
+//
+// E.g. An interleaved load (Factor = 2):
+//        %wide.vec = load <8 x i32>, <8 x i32>* %ptr
+//        %v0 = shuffle <8 x i32> %wide.vec, <8 x i32> undef, <0, 2, 4, 6>
+//        %v1 = shuffle <8 x i32> %wide.vec, <8 x i32> undef, <1, 3, 5, 7>
+//
+// It could be transformed into a ld2 intrinsic in AArch64 backend or a vld2
+// intrinsic in ARM backend.
+//
+// E.g. An interleaved store (Factor = 3):
+//        %i.vec = shuffle <8 x i32> %v0, <8 x i32> %v1,
+//                                    <0, 4, 8, 1, 5, 9, 2, 6, 10, 3, 7, 11>
+//        store <12 x i32> %i.vec, <12 x i32>* %ptr
+//
+// It could be transformed into a st3 intrinsic in AArch64 backend or a vst3
+// intrinsic in ARM backend.
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/CodeGen/Passes.h"
+#include "llvm/IR/InstIterator.h"
+#include "llvm/Support/Debug.h"
+#include "llvm/Support/MathExtras.h"
+#include "llvm/Support/raw_ostream.h"
+#include "llvm/Target/TargetLowering.h"
+#include "llvm/Target/TargetSubtargetInfo.h"
+
+using namespace llvm;
+
+#define DEBUG_TYPE "interleaved-access"
+
+static cl::opt<bool> LowerInterleavedAccesses(
+    "lower-interleaved-accesses",
+    cl::desc("Enable lowering interleaved accesses to intrinsics"),
+    cl::init(false), cl::Hidden);
+
+static unsigned MaxFactor; // The maximum supported interleave factor.
+
+namespace llvm {
+static void initializeInterleavedAccessPass(PassRegistry &);
+}
+
+namespace {
+
+class InterleavedAccess : public FunctionPass {
+
+public:
+  static char ID;
+  InterleavedAccess(const TargetMachine *TM = nullptr)
+      : FunctionPass(ID), TM(TM), TLI(nullptr) {
+    initializeInterleavedAccessPass(*PassRegistry::getPassRegistry());
+  }
+
+  const char *getPassName() const override { return "Interleaved Access Pass"; }
+
+  bool runOnFunction(Function &F) override;
+
+private:
+  const TargetMachine *TM;
+  const TargetLowering *TLI;
+
+  /// \brief Transform an interleaved load into target specific intrinsics.
+  bool lowerInterleavedLoad(LoadInst *LI,
+                            SmallVector<Instruction *, 32> &DeadInsts);
+
+  /// \brief Transform an interleaved store into target specific intrinsics.
+  bool lowerInterleavedStore(StoreInst *SI,
+                             SmallVector<Instruction *, 32> &DeadInsts);
+};
+} // end anonymous namespace.
+
+char InterleavedAccess::ID = 0;
+INITIALIZE_TM_PASS(InterleavedAccess, "interleaved-access",
+    "Lower interleaved memory accesses to target specific intrinsics",
+    false, false)
+
+FunctionPass *llvm::createInterleavedAccessPass(const TargetMachine *TM) {
+  return new InterleavedAccess(TM);
+}
+
+/// \brief Check if the mask is a DE-interleave mask of the given factor
+/// \p Factor like:
+///     <Index, Index+Factor, ..., Index+(NumElts-1)*Factor>
+static bool isDeInterleaveMaskOfFactor(ArrayRef<int> Mask, unsigned Factor,
+                                       unsigned &Index) {
+  // Check all potential start indices from 0 to (Factor - 1).
+  for (Index = 0; Index < Factor; Index++) {
+    unsigned i = 0;
+
+    // Check that elements are in ascending order by Factor. Ignore undef
+    // elements.
+    for (; i < Mask.size(); i++)
+      if (Mask[i] >= 0 && static_cast<unsigned>(Mask[i]) != Index + i * Factor)
+        break;
+
+    if (i == Mask.size())
+      return true;
+  }
+
+  return false;
+}
+
+/// \brief Check if the mask is a DE-interleave mask for an interleaved load.
+///
+/// E.g. DE-interleave masks (Factor = 2) could be:
+///     <0, 2, 4, 6>    (mask of index 0 to extract even elements)
+///     <1, 3, 5, 7>    (mask of index 1 to extract odd elements)
+static bool isDeInterleaveMask(ArrayRef<int> Mask, unsigned &Factor,
+                               unsigned &Index) {
+  if (Mask.size() < 2)
+    return false;
+
+  // Check potential Factors.
+  for (Factor = 2; Factor <= MaxFactor; Factor++)
+    if (isDeInterleaveMaskOfFactor(Mask, Factor, Index))
+      return true;
+
+  return false;
+}
+
+/// \brief Check if the mask is RE-interleave mask for an interleaved store.
+///
+/// I.e. <0, NumSubElts, ... , NumSubElts*(Factor - 1), 1, NumSubElts + 1, ...>
+///
+/// E.g. The RE-interleave mask (Factor = 2) could be:
+///     <0, 4, 1, 5, 2, 6, 3, 7>
+static bool isReInterleaveMask(ArrayRef<int> Mask, unsigned &Factor) {
+  unsigned NumElts = Mask.size();
+  if (NumElts < 4)
+    return false;
+
+  // Check potential Factors.
+  for (Factor = 2; Factor <= MaxFactor; Factor++) {
+    if (NumElts % Factor)
+      continue;
+
+    unsigned NumSubElts = NumElts / Factor;
+    if (!isPowerOf2_32(NumSubElts))
+      continue;
+
+    // Check whether each element matchs the RE-interleaved rule. Ignore undef
+    // elements.
+    unsigned i = 0;
+    for (; i < NumElts; i++)
+      if (Mask[i] >= 0 &&
+          static_cast<unsigned>(Mask[i]) !=
+              (i % Factor) * NumSubElts + i / Factor)
+        break;
+
+    // Find a RE-interleaved mask of current factor.
+    if (i == NumElts)
+      return true;
+  }
+
+  return false;
+}
+
+bool InterleavedAccess::lowerInterleavedLoad(
+    LoadInst *LI, SmallVector<Instruction *, 32> &DeadInsts) {
+  if (!LI->isSimple())
+    return false;
+
+  SmallVector<ShuffleVectorInst *, 4> Shuffles;
+
+  // Check if all users of this load are shufflevectors.
+  for (auto UI = LI->user_begin(), E = LI->user_end(); UI != E; UI++) {
+    ShuffleVectorInst *SVI = dyn_cast<ShuffleVectorInst>(*UI);
+    if (!SVI || !isa<UndefValue>(SVI->getOperand(1)))
+      return false;
+
+    Shuffles.push_back(SVI);
+  }
+
+  if (Shuffles.empty())
+    return false;
+
+  unsigned Factor, Index;
+
+  // Check if the first shufflevector is DE-interleave shuffle.
+  if (!isDeInterleaveMask(Shuffles[0]->getShuffleMask(), Factor, Index))
+    return false;
+
+  // Holds the corresponding index for each DE-interleave shuffle.
+  SmallVector<unsigned, 4> Indices;
+  Indices.push_back(Index);
+
+  Type *VecTy = Shuffles[0]->getType();
+
+  // Check if other shufflevectors are also DE-interleaved of the same type
+  // and factor as the first shufflevector.
+  for (unsigned i = 1; i < Shuffles.size(); i++) {
+    if (Shuffles[i]->getType() != VecTy)
+      return false;
+
+    if (!isDeInterleaveMaskOfFactor(Shuffles[i]->getShuffleMask(), Factor,
+                                    Index))
+      return false;
+
+    Indices.push_back(Index);
+  }
+
+  DEBUG(dbgs() << "IA: Found an interleaved load: " << *LI << "\n");
+
+  // Try to create target specific intrinsics to replace the load and shuffles.
+  if (!TLI->lowerInterleavedLoad(LI, Shuffles, Indices, Factor))
+    return false;
+
+  for (auto SVI : Shuffles)
+    DeadInsts.push_back(SVI);
+
+  DeadInsts.push_back(LI);
+  return true;
+}
+
+bool InterleavedAccess::lowerInterleavedStore(
+    StoreInst *SI, SmallVector<Instruction *, 32> &DeadInsts) {
+  if (!SI->isSimple())
+    return false;
+
+  ShuffleVectorInst *SVI = dyn_cast<ShuffleVectorInst>(SI->getValueOperand());
+  if (!SVI || !SVI->hasOneUse())
+    return false;
+
+  // Check if the shufflevector is RE-interleave shuffle.
+  unsigned Factor;
+  if (!isReInterleaveMask(SVI->getShuffleMask(), Factor))
+    return false;
+
+  DEBUG(dbgs() << "IA: Found an interleaved store: " << *SI << "\n");
+
+  // Try to create target specific intrinsics to replace the store and shuffle.
+  if (!TLI->lowerInterleavedStore(SI, SVI, Factor))
+    return false;
+
+  // Already have a new target specific interleaved store. Erase the old store.
+  DeadInsts.push_back(SI);
+  DeadInsts.push_back(SVI);
+  return true;
+}
+
+bool InterleavedAccess::runOnFunction(Function &F) {
+  if (!TM || !LowerInterleavedAccesses)
+    return false;
+
+  DEBUG(dbgs() << "*** " << getPassName() << ": " << F.getName() << "\n");
+
+  TLI = TM->getSubtargetImpl(F)->getTargetLowering();
+  MaxFactor = TLI->getMaxSupportedInterleaveFactor();
+
+  // Holds dead instructions that will be erased later.
+  SmallVector<Instruction *, 32> DeadInsts;
+  bool Changed = false;
+
+  for (auto &I : inst_range(F)) {
+    if (LoadInst *LI = dyn_cast<LoadInst>(&I))
+      Changed |= lowerInterleavedLoad(LI, DeadInsts);
+
+    if (StoreInst *SI = dyn_cast<StoreInst>(&I))
+      Changed |= lowerInterleavedStore(SI, DeadInsts);
+  }
+
+  for (auto I : DeadInsts)
+    I->eraseFromParent();
+
+  return Changed;
+}
diff --git a/lib/CodeGen/LLVMBuild.txt b/lib/CodeGen/LLVMBuild.txt
index fee0347ea65..18ed77607c6 100644
--- a/lib/CodeGen/LLVMBuild.txt
+++ b/lib/CodeGen/LLVMBuild.txt
@@ -16,10 +16,10 @@
 ;===------------------------------------------------------------------------===;
 
 [common]
-subdirectories = AsmPrinter SelectionDAG
+subdirectories = AsmPrinter SelectionDAG MIRParser
 
 [component_0]
 type = Library
 name = CodeGen
 parent = Libraries
-required_libraries = Analysis Core MC Scalar Support Target TransformUtils
+required_libraries = Analysis Core Instrumentation MC Scalar Support Target TransformUtils
diff --git a/lib/CodeGen/LLVMTargetMachine.cpp b/lib/CodeGen/LLVMTargetMachine.cpp
index 8ee1f7cd48b..37299eb664c 100644
--- a/lib/CodeGen/LLVMTargetMachine.cpp
+++ b/lib/CodeGen/LLVMTargetMachine.cpp
@@ -43,16 +43,17 @@ EnableFastISelOption("fast-isel", cl::Hidden,
   cl::desc("Enable the \"fast\" instruction selector"));
 
 void LLVMTargetMachine::initAsmInfo() {
-  MRI = TheTarget.createMCRegInfo(getTargetTriple());
+  MRI = TheTarget.createMCRegInfo(getTargetTriple().str());
   MII = TheTarget.createMCInstrInfo();
   // FIXME: Having an MCSubtargetInfo on the target machine is a hack due
   // to some backends having subtarget feature dependent module level
   // code generation. This is similar to the hack in the AsmPrinter for
   // module level assembly etc.
-  STI = TheTarget.createMCSubtargetInfo(getTargetTriple(), getTargetCPU(),
+  STI = TheTarget.createMCSubtargetInfo(getTargetTriple().str(), getTargetCPU(),
                                         getTargetFeatureString());
 
-  MCAsmInfo *TmpAsmInfo = TheTarget.createMCAsmInfo(*MRI, getTargetTriple());
+  MCAsmInfo *TmpAsmInfo =
+      TheTarget.createMCAsmInfo(*MRI, getTargetTriple().str());
   // TargetSelect.h moved to a different directory between LLVM 2.9 and 3.0,
   // and if the old one gets included then MCAsmInfo will be NULL and
   // we'll crash later.
@@ -72,12 +73,12 @@ void LLVMTargetMachine::initAsmInfo() {
 
 LLVMTargetMachine::LLVMTargetMachine(const Target &T,
                                      StringRef DataLayoutString,
-                                     StringRef Triple, StringRef CPU,
+                                     const Triple &TT, StringRef CPU,
                                      StringRef FS, TargetOptions Options,
                                      Reloc::Model RM, CodeModel::Model CM,
                                      CodeGenOpt::Level OL)
-    : TargetMachine(T, DataLayoutString, Triple, CPU, FS, Options) {
-  CodeGenInfo = T.createMCCodeGenInfo(Triple, RM, CM, OL);
+    : TargetMachine(T, DataLayoutString, TT, CPU, FS, Options) {
+  CodeGenInfo = T.createMCCodeGenInfo(TT.str(), RM, CM, OL);
 }
 
 TargetIRAnalysis LLVMTargetMachine::getTargetIRAnalysis() {
@@ -87,11 +88,11 @@ TargetIRAnalysis LLVMTargetMachine::getTargetIRAnalysis() {
 }
 
 /// addPassesToX helper drives creation and initialization of TargetPassConfig.
-static MCContext *addPassesToGenerateCode(LLVMTargetMachine *TM,
-                                          PassManagerBase &PM,
-                                          bool DisableVerify,
-                                          AnalysisID StartAfter,
-                                          AnalysisID StopAfter) {
+static MCContext *
+addPassesToGenerateCode(LLVMTargetMachine *TM, PassManagerBase &PM,
+                        bool DisableVerify, AnalysisID StartBefore,
+                        AnalysisID StartAfter, AnalysisID StopAfter,
+                        MachineFunctionInitializer *MFInitializer = nullptr) {
 
   // Add internal analysis passes from the target machine.
   PM.add(createTargetTransformInfoWrapperPass(TM->getTargetIRAnalysis()));
@@ -99,7 +100,7 @@ static MCContext *addPassesToGenerateCode(LLVMTargetMachine *TM,
   // Targets may override createPassConfig to provide a target-specific
   // subclass.
   TargetPassConfig *PassConfig = TM->createPassConfig(PM);
-  PassConfig->setStartStopPasses(StartAfter, StopAfter);
+  PassConfig->setStartStopPasses(StartBefore, StartAfter, StopAfter);
 
   // Set PassConfig options provided by TargetMachine.
   PassConfig->setDisableVerify(DisableVerify);
@@ -121,7 +122,7 @@ static MCContext *addPassesToGenerateCode(LLVMTargetMachine *TM,
   PM.add(MMI);
 
   // Set up a MachineFunction for the rest of CodeGen to work on.
-  PM.add(new MachineFunctionAnalysis(*TM));
+  PM.add(new MachineFunctionAnalysis(*TM, MFInitializer));
 
   // Enable FastISel with -fast, but allow that to be overridden.
   if (EnableFastISelOption == cl::BOU_TRUE ||
@@ -142,23 +143,17 @@ static MCContext *addPassesToGenerateCode(LLVMTargetMachine *TM,
 
 bool LLVMTargetMachine::addPassesToEmitFile(
     PassManagerBase &PM, raw_pwrite_stream &Out, CodeGenFileType FileType,
-    bool DisableVerify, AnalysisID StartAfter, AnalysisID StopAfter) {
-  // @LOCALMOD: When there is no ModulePassManager in the PMStack, the
-  // JumpInstrTablesPass will silently be dropped and not added to the pass
-  // manager, because it is a module pass.
+    bool DisableVerify, AnalysisID StartBefore, AnalysisID StartAfter,
+    AnalysisID StopAfter, MachineFunctionInitializer *MFInitializer) {
   // Add common CodeGen passes.
-  MCContext *Context = addPassesToGenerateCode(this, PM, DisableVerify,
-                                               StartAfter, StopAfter);
+  MCContext *Context =
+      addPassesToGenerateCode(this, PM, DisableVerify, StartBefore, StartAfter,
+                              StopAfter, MFInitializer);
   if (!Context)
     return true;
 
   if (StopAfter) {
-    // FIXME: The intent is that this should eventually write out a YAML file,
-    // containing the LLVM IR, the machine-level IR (when stopping after a
-    // machine-level pass), and whatever other information is needed to
-    // deserialize the code and resume compilation.  For now, just write the
-    // LLVM IR.
-    PM.add(createPrintModulePass(Out));
+    PM.add(createPrintMIRPass(outs()));
     return false;
   }
 
@@ -175,15 +170,15 @@ bool LLVMTargetMachine::addPassesToEmitFile(
   switch (FileType) {
   case CGFT_AssemblyFile: {
     MCInstPrinter *InstPrinter = getTarget().createMCInstPrinter(
-        Triple(getTargetTriple()), MAI.getAssemblerDialect(), MAI, MII, MRI);
+        getTargetTriple(), MAI.getAssemblerDialect(), MAI, MII, MRI);
 
     // Create a code emitter if asked to show the encoding.
     MCCodeEmitter *MCE = nullptr;
     if (Options.MCOptions.ShowMCEncoding)
       MCE = getTarget().createMCCodeEmitter(MII, MRI, *Context);
 
-    MCAsmBackend *MAB = getTarget().createMCAsmBackend(MRI, getTargetTriple(),
-                                                       TargetCPU);
+    MCAsmBackend *MAB =
+        getTarget().createMCAsmBackend(MRI, getTargetTriple().str(), TargetCPU);
     auto FOut = llvm::make_unique<formatted_raw_ostream>(Out);
     MCStreamer *S = getTarget().createAsmStreamer(
         *Context, std::move(FOut), Options.MCOptions.AsmVerbose,
@@ -196,12 +191,15 @@ bool LLVMTargetMachine::addPassesToEmitFile(
     // Create the code emitter for the target if it exists.  If not, .o file
     // emission fails.
     MCCodeEmitter *MCE = getTarget().createMCCodeEmitter(MII, MRI, *Context);
-    MCAsmBackend *MAB = getTarget().createMCAsmBackend(MRI, getTargetTriple(),
-                                                       TargetCPU);
+    MCAsmBackend *MAB =
+        getTarget().createMCAsmBackend(MRI, getTargetTriple().str(), TargetCPU);
     if (!MCE || !MAB)
       return true;
 
-    Triple T(getTargetTriple());
+    // Don't waste memory on names of temp labels.
+    Context->setUseNamesOnTempLabels(false);
+
+    Triple T(getTargetTriple().str());
     AsmStreamer.reset(getTarget().createMCObjectStreamer(
         T, *Context, *MAB, Out, MCE, STI, Options.MCOptions.MCRelaxAll,
         /*DWARFMustBeAtTheEnd*/ true));
@@ -234,7 +232,8 @@ bool LLVMTargetMachine::addPassesToEmitMC(PassManagerBase &PM, MCContext *&Ctx,
                                           raw_pwrite_stream &Out,
                                           bool DisableVerify) {
   // Add common CodeGen passes.
-  Ctx = addPassesToGenerateCode(this, PM, DisableVerify, nullptr, nullptr);
+  Ctx = addPassesToGenerateCode(this, PM, DisableVerify, nullptr, nullptr,
+                                nullptr);
   if (!Ctx)
     return true;
 
@@ -246,12 +245,12 @@ bool LLVMTargetMachine::addPassesToEmitMC(PassManagerBase &PM, MCContext *&Ctx,
   const MCRegisterInfo &MRI = *getMCRegisterInfo();
   MCCodeEmitter *MCE =
       getTarget().createMCCodeEmitter(*getMCInstrInfo(), MRI, *Ctx);
-  MCAsmBackend *MAB = getTarget().createMCAsmBackend(MRI, getTargetTriple(),
-                                                     TargetCPU);
+  MCAsmBackend *MAB =
+      getTarget().createMCAsmBackend(MRI, getTargetTriple().str(), TargetCPU);
   if (!MCE || !MAB)
     return true;
 
-  Triple T(getTargetTriple());
+  const Triple &T = getTargetTriple();
   const MCSubtargetInfo &STI = *getMCSubtargetInfo();
   std::unique_ptr<MCStreamer> AsmStreamer(getTarget().createMCObjectStreamer(
       T, *Ctx, *MAB, Out, MCE, STI, Options.MCOptions.MCRelaxAll,
diff --git a/lib/CodeGen/LexicalScopes.cpp b/lib/CodeGen/LexicalScopes.cpp
index df3182a93ea..be61a20424b 100644
--- a/lib/CodeGen/LexicalScopes.cpp
+++ b/lib/CodeGen/LexicalScopes.cpp
@@ -59,10 +59,10 @@ void LexicalScopes::extractLexicalScopes(
   for (const auto &MBB : *MF) {
     const MachineInstr *RangeBeginMI = nullptr;
     const MachineInstr *PrevMI = nullptr;
-    const MDLocation *PrevDL = nullptr;
+    const DILocation *PrevDL = nullptr;
     for (const auto &MInsn : MBB) {
       // Check if instruction has valid location information.
-      const MDLocation *MIDL = MInsn.getDebugLoc();
+      const DILocation *MIDL = MInsn.getDebugLoc();
       if (!MIDL) {
         PrevMI = &MInsn;
         continue;
@@ -106,14 +106,14 @@ void LexicalScopes::extractLexicalScopes(
 
 /// findLexicalScope - Find lexical scope, either regular or inlined, for the
 /// given DebugLoc. Return NULL if not found.
-LexicalScope *LexicalScopes::findLexicalScope(const MDLocation *DL) {
-  MDLocalScope *Scope = DL->getScope();
+LexicalScope *LexicalScopes::findLexicalScope(const DILocation *DL) {
+  DILocalScope *Scope = DL->getScope();
   if (!Scope)
     return nullptr;
 
   // The scope that we were created with could have an extra file - which
   // isn't what we care about in this case.
-  if (auto *File = dyn_cast<MDLexicalBlockFile>(Scope))
+  if (auto *File = dyn_cast<DILexicalBlockFile>(Scope))
     Scope = File->getScope();
 
   if (auto *IA = DL->getInlinedAt()) {
@@ -125,8 +125,8 @@ LexicalScope *LexicalScopes::findLexicalScope(const MDLocation *DL) {
 
 /// getOrCreateLexicalScope - Find lexical scope for the given DebugLoc. If
 /// not available then create new lexical scope.
-LexicalScope *LexicalScopes::getOrCreateLexicalScope(const MDLocalScope *Scope,
-                                                     const MDLocation *IA) {
+LexicalScope *LexicalScopes::getOrCreateLexicalScope(const DILocalScope *Scope,
+                                                     const DILocation *IA) {
   if (IA) {
     // Create an abstract scope for inlined function.
     getOrCreateAbstractScope(Scope);
@@ -139,17 +139,17 @@ LexicalScope *LexicalScopes::getOrCreateLexicalScope(const MDLocalScope *Scope,
 
 /// getOrCreateRegularScope - Find or create a regular lexical scope.
 LexicalScope *
-LexicalScopes::getOrCreateRegularScope(const MDLocalScope *Scope) {
-  if (auto *File = dyn_cast<MDLexicalBlockFile>(Scope))
+LexicalScopes::getOrCreateRegularScope(const DILocalScope *Scope) {
+  if (auto *File = dyn_cast<DILexicalBlockFile>(Scope))
     Scope = File->getScope();
 
   auto I = LexicalScopeMap.find(Scope);
   if (I != LexicalScopeMap.end())
     return &I->second;
 
-  // FIXME: Should the following dyn_cast be MDLexicalBlock?
+  // FIXME: Should the following dyn_cast be DILexicalBlock?
   LexicalScope *Parent = nullptr;
-  if (auto *Block = dyn_cast<MDLexicalBlockBase>(Scope))
+  if (auto *Block = dyn_cast<DILexicalBlockBase>(Scope))
     Parent = getOrCreateLexicalScope(Block->getScope());
   I = LexicalScopeMap.emplace(std::piecewise_construct,
                               std::forward_as_tuple(Scope),
@@ -157,18 +157,8 @@ LexicalScopes::getOrCreateRegularScope(const MDLocalScope *Scope) {
                                                     false)).first;
 
   if (!Parent) {
-    assert(cast<MDSubprogram>(Scope)->describes(MF->getFunction()));
-    // @LOCALMOD-BEGIN
-    // This currently asserts when mixing -g and -g0 compilation
-    // units + LTO. Debug info from a few inlined functions are not
-    // marked as inline, so we end up in getOrCreateRegularScope
-    // instead of getOrCreateInlinedScope.
-    // This is reproducible w/ the pnacl-llc.nexe build and
-    // setting the env var PNACL_PRUNE=false.
-    // https://code.google.com/p/nativeclient/issues/detail?id=4026
-    //assert(cast<MDSubprogram>(Scope)->describes(MF->getFunction()));
-    //assert(!CurrentFnLexicalScope);
-    // @LOCALMOD-END
+    assert(cast<DISubprogram>(Scope)->describes(MF->getFunction()));
+    assert(!CurrentFnLexicalScope);
     CurrentFnLexicalScope = &I->second;
   }
 
@@ -177,15 +167,15 @@ LexicalScopes::getOrCreateRegularScope(const MDLocalScope *Scope) {
 
 /// getOrCreateInlinedScope - Find or create an inlined lexical scope.
 LexicalScope *
-LexicalScopes::getOrCreateInlinedScope(const MDLocalScope *Scope,
-                                       const MDLocation *InlinedAt) {
-  std::pair<const MDLocalScope *, const MDLocation *> P(Scope, InlinedAt);
+LexicalScopes::getOrCreateInlinedScope(const DILocalScope *Scope,
+                                       const DILocation *InlinedAt) {
+  std::pair<const DILocalScope *, const DILocation *> P(Scope, InlinedAt);
   auto I = InlinedLexicalScopeMap.find(P);
   if (I != InlinedLexicalScopeMap.end())
     return &I->second;
 
   LexicalScope *Parent;
-  if (auto *Block = dyn_cast<MDLexicalBlockBase>(Scope))
+  if (auto *Block = dyn_cast<DILexicalBlockBase>(Scope))
     Parent = getOrCreateInlinedScope(Block->getScope(), InlinedAt);
   else
     Parent = getOrCreateLexicalScope(InlinedAt);
@@ -200,25 +190,25 @@ LexicalScopes::getOrCreateInlinedScope(const MDLocalScope *Scope,
 
 /// getOrCreateAbstractScope - Find or create an abstract lexical scope.
 LexicalScope *
-LexicalScopes::getOrCreateAbstractScope(const MDLocalScope *Scope) {
+LexicalScopes::getOrCreateAbstractScope(const DILocalScope *Scope) {
   assert(Scope && "Invalid Scope encoding!");
 
-  if (auto *File = dyn_cast<MDLexicalBlockFile>(Scope))
+  if (auto *File = dyn_cast<DILexicalBlockFile>(Scope))
     Scope = File->getScope();
   auto I = AbstractScopeMap.find(Scope);
   if (I != AbstractScopeMap.end())
     return &I->second;
 
-  // FIXME: Should the following isa be MDLexicalBlock?
+  // FIXME: Should the following isa be DILexicalBlock?
   LexicalScope *Parent = nullptr;
-  if (auto *Block = dyn_cast<MDLexicalBlockBase>(Scope))
+  if (auto *Block = dyn_cast<DILexicalBlockBase>(Scope))
     Parent = getOrCreateAbstractScope(Block->getScope());
 
   I = AbstractScopeMap.emplace(std::piecewise_construct,
                                std::forward_as_tuple(Scope),
                                std::forward_as_tuple(Parent, Scope,
                                                      nullptr, true)).first;
-  if (isa<MDSubprogram>(Scope))
+  if (isa<DISubprogram>(Scope))
     AbstractScopesList.push_back(&I->second);
   return &I->second;
 }
@@ -279,7 +269,7 @@ void LexicalScopes::assignInstructionRanges(
 /// have machine instructions that belong to lexical scope identified by
 /// DebugLoc.
 void LexicalScopes::getMachineBasicBlocks(
-    const MDLocation *DL, SmallPtrSetImpl<const MachineBasicBlock *> &MBBs) {
+    const DILocation *DL, SmallPtrSetImpl<const MachineBasicBlock *> &MBBs) {
   MBBs.clear();
   LexicalScope *Scope = getOrCreateLexicalScope(DL);
   if (!Scope)
@@ -302,7 +292,7 @@ void LexicalScopes::getMachineBasicBlocks(
 
 /// dominates - Return true if DebugLoc's lexical scope dominates at least one
 /// machine instruction's lexical scope in a given machine basic block.
-bool LexicalScopes::dominates(const MDLocation *DL, MachineBasicBlock *MBB) {
+bool LexicalScopes::dominates(const DILocation *DL, MachineBasicBlock *MBB) {
   LexicalScope *Scope = getOrCreateLexicalScope(DL);
   if (!Scope)
     return false;
@@ -314,7 +304,7 @@ bool LexicalScopes::dominates(const MDLocation *DL, MachineBasicBlock *MBB) {
   bool Result = false;
   for (MachineBasicBlock::iterator I = MBB->begin(), E = MBB->end(); I != E;
        ++I) {
-    if (const MDLocation *IDL = I->getDebugLoc())
+    if (const DILocation *IDL = I->getDebugLoc())
       if (LexicalScope *IScope = getOrCreateLexicalScope(IDL))
         if (Scope->dominates(IScope))
           return true;
diff --git a/lib/CodeGen/LiveDebugVariables.cpp b/lib/CodeGen/LiveDebugVariables.cpp
index c2993db6c5a..15715513452 100644
--- a/lib/CodeGen/LiveDebugVariables.cpp
+++ b/lib/CodeGen/LiveDebugVariables.cpp
@@ -158,7 +158,7 @@ class UserValue {
   UserValue *getNext() const { return next; }
 
   /// match - Does this UserValue match the parameters?
-  bool match(const MDNode *Var, const MDNode *Expr, const MDLocation *IA,
+  bool match(const MDNode *Var, const MDNode *Expr, const DILocation *IA,
              unsigned Offset, bool indirect) const {
     return Var == Variable && Expr == Expression && dl->getInlinedAt() == IA &&
            Offset == offset && indirect == IsIndirect;
@@ -362,7 +362,7 @@ static void printDebugLoc(DebugLoc DL, raw_ostream &CommentOS,
   if (!DL)
     return;
 
-  auto *Scope = cast<MDScope>(DL.getScope());
+  auto *Scope = cast<DIScope>(DL.getScope());
   // Omit the directory, because it's likely to be long and uninteresting.
   CommentOS << Scope->getFilename();
   CommentOS << ':' << DL.getLine();
@@ -378,8 +378,8 @@ static void printDebugLoc(DebugLoc DL, raw_ostream &CommentOS,
   CommentOS << " ]";
 }
 
-static void printExtendedName(raw_ostream &OS, const MDLocalVariable *V,
-                              const MDLocation *DL) {
+static void printExtendedName(raw_ostream &OS, const DILocalVariable *V,
+                              const DILocation *DL) {
   const LLVMContext &Ctx = V->getContext();
   StringRef Res = V->getName();
   if (!Res.empty())
@@ -394,7 +394,7 @@ static void printExtendedName(raw_ostream &OS, const MDLocalVariable *V,
 }
 
 void UserValue::print(raw_ostream &OS, const TargetRegisterInfo *TRI) {
-  DIVariable DV = cast<MDLocalVariable>(Variable);
+  auto *DV = cast<DILocalVariable>(Variable);
   OS << "!\"";
   printExtendedName(OS, DV, dl);
 
@@ -981,7 +981,7 @@ void UserValue::insertDebugValue(MachineBasicBlock *MBB, SlotIndex Idx,
   MachineOperand &Loc = locations[LocNo];
   ++NumInsertedDebugValues;
 
-  assert(cast<MDLocalVariable>(Variable)
+  assert(cast<DILocalVariable>(Variable)
              ->isValidLocationForIntrinsic(getDebugLoc()) &&
          "Expected inlined-at fields to agree");
   if (Loc.isReg())
diff --git a/lib/CodeGen/LiveDebugVariables.h b/lib/CodeGen/LiveDebugVariables.h
index fe296bc4cb5..694aa1770c9 100644
--- a/lib/CodeGen/LiveDebugVariables.h
+++ b/lib/CodeGen/LiveDebugVariables.h
@@ -31,9 +31,10 @@ class LiveInterval;
 class LiveIntervals;
 class VirtRegMap;
 
-class LiveDebugVariables : public MachineFunctionPass {
+class LLVM_LIBRARY_VISIBILITY LiveDebugVariables : public MachineFunctionPass {
   void *pImpl;
-  DenseMap<const Function*, DISubprogram> FunctionDIs;
+  DenseMap<const Function *, DISubprogram *> FunctionDIs;
+
 public:
   static char ID; // Pass identification, replacement for typeid
 
diff --git a/lib/CodeGen/LiveInterval.cpp b/lib/CodeGen/LiveInterval.cpp
index d75e4417cb0..3dc4113c639 100644
--- a/lib/CodeGen/LiveInterval.cpp
+++ b/lib/CodeGen/LiveInterval.cpp
@@ -1110,6 +1110,8 @@ void LiveInterval::verify(const MachineRegisterInfo *MRI) const {
 
     // subrange mask should not contained in maximum lane mask for the vreg.
     assert((Mask & ~MaxMask) == 0);
+    // empty subranges must be removed.
+    assert(!SR.empty());
 
     SR.verify();
     // Main liverange should cover subrange.
diff --git a/lib/CodeGen/LiveIntervalAnalysis.cpp b/lib/CodeGen/LiveIntervalAnalysis.cpp
index adca4cc738e..9738dac65ad 100644
--- a/lib/CodeGen/LiveIntervalAnalysis.cpp
+++ b/lib/CodeGen/LiveIntervalAnalysis.cpp
@@ -223,11 +223,11 @@ void LiveIntervals::computeRegMasks() {
     RMB.first = RegMaskSlots.size();
     for (MachineBasicBlock::iterator MI = MBB->begin(), ME = MBB->end();
          MI != ME; ++MI)
-      for (MIOperands MO(MI); MO.isValid(); ++MO) {
-        if (!MO->isRegMask())
+      for (const MachineOperand &MO : MI->operands()) {
+        if (!MO.isRegMask())
           continue;
           RegMaskSlots.push_back(Indexes->getInstructionIndex(MI).getRegSlot());
-          RegMaskBits.push_back(MO->getRegMask());
+          RegMaskBits.push_back(MO.getRegMask());
       }
     // Compute the number of register mask instructions in this block.
     RMB.second = RegMaskSlots.size() - RMB.first;
@@ -396,9 +396,6 @@ static void extendSegmentsToUses(LiveRange &LR, const SlotIndexes &Indexes,
   }
 }
 
-/// shrinkToUses - After removing some uses of a register, shrink its live
-/// range to just the remaining uses. This method does not compute reaching
-/// defs for new uses, and it doesn't remove dead defs.
 bool LiveIntervals::shrinkToUses(LiveInterval *li,
                                  SmallVectorImpl<MachineInstr*> *dead) {
   DEBUG(dbgs() << "Shrink: " << *li << '\n');
@@ -406,9 +403,14 @@ bool LiveIntervals::shrinkToUses(LiveInterval *li,
          && "Can only shrink virtual registers");
 
   // Shrink subregister live ranges.
+  bool NeedsCleanup = false;
   for (LiveInterval::SubRange &S : li->subranges()) {
     shrinkToUses(S, li->reg);
+    if (S.empty())
+      NeedsCleanup = true;
   }
+  if (NeedsCleanup)
+    li->removeEmptySubRanges();
 
   // Find all the values used, including PHI kills.
   ShrinkToUsesWorkList WorkList;
@@ -927,23 +929,23 @@ class LiveIntervals::HMEditor {
   void updateAllRanges(MachineInstr *MI) {
     DEBUG(dbgs() << "handleMove " << OldIdx << " -> " << NewIdx << ": " << *MI);
     bool hasRegMask = false;
-    for (MIOperands MO(MI); MO.isValid(); ++MO) {
-      if (MO->isRegMask())
+    for (MachineOperand &MO : MI->operands()) {
+      if (MO.isRegMask())
         hasRegMask = true;
-      if (!MO->isReg())
+      if (!MO.isReg())
         continue;
       // Aggressively clear all kill flags.
       // They are reinserted by VirtRegRewriter.
-      if (MO->isUse())
-        MO->setIsKill(false);
+      if (MO.isUse())
+        MO.setIsKill(false);
 
-      unsigned Reg = MO->getReg();
+      unsigned Reg = MO.getReg();
       if (!Reg)
         continue;
       if (TargetRegisterInfo::isVirtualRegister(Reg)) {
         LiveInterval &LI = LIS.getInterval(Reg);
         if (LI.hasSubRanges()) {
-          unsigned SubReg = MO->getSubReg();
+          unsigned SubReg = MO.getSubReg();
           unsigned LaneMask = TRI.getSubRegIndexLaneMask(SubReg);
           for (LiveInterval::SubRange &S : LI.subranges()) {
             if ((S.LaneMask & LaneMask) == 0)
diff --git a/lib/CodeGen/LivePhysRegs.cpp b/lib/CodeGen/LivePhysRegs.cpp
index 89567eff517..cbd98e3f345 100644
--- a/lib/CodeGen/LivePhysRegs.cpp
+++ b/lib/CodeGen/LivePhysRegs.cpp
@@ -14,6 +14,8 @@
 //===----------------------------------------------------------------------===//
 
 #include "llvm/CodeGen/LivePhysRegs.h"
+#include "llvm/CodeGen/MachineFrameInfo.h"
+#include "llvm/CodeGen/MachineFunction.h"
 #include "llvm/CodeGen/MachineInstrBundle.h"
 #include "llvm/Support/Debug.h"
 #include "llvm/Support/raw_ostream.h"
@@ -22,12 +24,17 @@ using namespace llvm;
 
 /// \brief Remove all registers from the set that get clobbered by the register
 /// mask.
-void LivePhysRegs::removeRegsInMask(const MachineOperand &MO) {
+/// The clobbers set will be the list of live registers clobbered
+/// by the regmask.
+void LivePhysRegs::removeRegsInMask(const MachineOperand &MO,
+        SmallVectorImpl<std::pair<unsigned, const MachineOperand*>> *Clobbers) {
   SparseSet<unsigned>::iterator LRI = LiveRegs.begin();
   while (LRI != LiveRegs.end()) {
-    if (MO.clobbersPhysReg(*LRI))
+    if (MO.clobbersPhysReg(*LRI)) {
+      if (Clobbers)
+        Clobbers->push_back(std::make_pair(*LRI, &MO));
       LRI = LiveRegs.erase(LRI);
-    else
+    } else
       ++LRI;
   }
 }
@@ -45,7 +52,7 @@ void LivePhysRegs::stepBackward(const MachineInstr &MI) {
         continue;
       removeReg(Reg);
     } else if (O->isRegMask())
-      removeRegsInMask(*O);
+      removeRegsInMask(*O, nullptr);
   }
 
   // Add uses to the set.
@@ -63,8 +70,8 @@ void LivePhysRegs::stepBackward(const MachineInstr &MI) {
 /// killed-uses, add defs. This is the not recommended way, because it depends
 /// on accurate kill flags. If possible use stepBackwards() instead of this
 /// function.
-void LivePhysRegs::stepForward(const MachineInstr &MI) {
-  SmallVector<unsigned, 4> Defs;
+void LivePhysRegs::stepForward(const MachineInstr &MI,
+        SmallVectorImpl<std::pair<unsigned, const MachineOperand*>> &Clobbers) {
   // Remove killed registers from the set.
   for (ConstMIBundleOperands O(&MI); O.isValid(); ++O) {
     if (O->isReg()) {
@@ -72,8 +79,9 @@ void LivePhysRegs::stepForward(const MachineInstr &MI) {
       if (Reg == 0)
         continue;
       if (O->isDef()) {
-        if (!O->isDead())
-          Defs.push_back(Reg);
+        // Note, dead defs are still recorded.  The caller should decide how to
+        // handle them.
+        Clobbers.push_back(std::make_pair(Reg, &*O));
       } else {
         if (!O->isKill())
           continue;
@@ -81,12 +89,16 @@ void LivePhysRegs::stepForward(const MachineInstr &MI) {
         removeReg(Reg);
       }
     } else if (O->isRegMask())
-      removeRegsInMask(*O);
+      removeRegsInMask(*O, &Clobbers);
   }
 
   // Add defs to the set.
-  for (unsigned i = 0, e = Defs.size(); i != e; ++i)
-    addReg(Defs[i]);
+  for (auto Reg : Clobbers) {
+    // Skip dead defs.  They shouldn't be added to the set.
+    if (Reg.second->isReg() && Reg.second->isDead())
+      continue;
+    addReg(Reg.first);
+  }
 }
 
 /// Prin the currently live registers to OS.
@@ -113,3 +125,42 @@ void LivePhysRegs::dump() const {
   dbgs() << "  " << *this;
 #endif
 }
+
+/// Add live-in registers of basic block \p MBB to \p LiveRegs.
+static void addLiveIns(LivePhysRegs &LiveRegs, const MachineBasicBlock &MBB) {
+  for (unsigned Reg : make_range(MBB.livein_begin(), MBB.livein_end()))
+    LiveRegs.addReg(Reg);
+}
+
+/// Add pristine registers to the given \p LiveRegs. This function removes
+/// actually saved callee save registers when \p InPrologueEpilogue is false.
+static void addPristines(LivePhysRegs &LiveRegs, const MachineFunction &MF,
+                         const TargetRegisterInfo &TRI) {
+  const MachineFrameInfo &MFI = *MF.getFrameInfo();
+  if (!MFI.isCalleeSavedInfoValid())
+    return;
+
+  for (const MCPhysReg *CSR = TRI.getCalleeSavedRegs(&MF); CSR && *CSR; ++CSR)
+    LiveRegs.addReg(*CSR);
+  for (const CalleeSavedInfo &Info : MFI.getCalleeSavedInfo())
+    LiveRegs.removeReg(Info.getReg());
+}
+
+void LivePhysRegs::addLiveOuts(const MachineBasicBlock *MBB,
+                               bool AddPristines) {
+  if (AddPristines) {
+    const MachineFunction &MF = *MBB->getParent();
+    addPristines(*this, MF, *TRI);
+  }
+  for (const MachineBasicBlock *Succ : MBB->successors())
+    ::addLiveIns(*this, *Succ);
+}
+
+void LivePhysRegs::addLiveIns(const MachineBasicBlock *MBB,
+                              bool AddPristines) {
+  if (AddPristines) {
+    const MachineFunction &MF = *MBB->getParent();
+    addPristines(*this, MF, *TRI);
+  }
+  ::addLiveIns(*this, *MBB);
+}
diff --git a/lib/CodeGen/LiveRangeCalc.cpp b/lib/CodeGen/LiveRangeCalc.cpp
index 45e7265c8f4..bb2877ae31a 100644
--- a/lib/CodeGen/LiveRangeCalc.cpp
+++ b/lib/CodeGen/LiveRangeCalc.cpp
@@ -272,13 +272,19 @@ bool LiveRangeCalc::findReachingDefs(LiveRange &LR, MachineBasicBlock &UseMBB,
 #ifndef NDEBUG
     if (MBB->pred_empty()) {
       MBB->getParent()->verify();
+      errs() << "Use of " << PrintReg(PhysReg)
+             << " does not have a corresponding definition on every path:\n";
+      const MachineInstr *MI = Indexes->getInstructionFromIndex(Use);
+      if (MI != nullptr)
+        errs() << Use << " " << *MI;
       llvm_unreachable("Use not jointly dominated by defs.");
     }
 
     if (TargetRegisterInfo::isPhysicalRegister(PhysReg) &&
         !MBB->isLiveIn(PhysReg)) {
       MBB->getParent()->verify();
-      errs() << "The register needs to be live in to BB#" << MBB->getNumber()
+      errs() << "The register " << PrintReg(PhysReg)
+             << " needs to be live in to BB#" << MBB->getNumber()
              << ", but is missing from the live-in list.\n";
       llvm_unreachable("Invalid global physical register");
     }
diff --git a/lib/CodeGen/LiveRangeEdit.cpp b/lib/CodeGen/LiveRangeEdit.cpp
index 0edc897dc31..08bbe0c3f37 100644
--- a/lib/CodeGen/LiveRangeEdit.cpp
+++ b/lib/CodeGen/LiveRangeEdit.cpp
@@ -196,7 +196,7 @@ bool LiveRangeEdit::foldAsLoad(LiveInterval *LI,
   // We also need to make sure it is safe to move the load.
   // Assume there are stores between DefMI and UseMI.
   bool SawStore = true;
-  if (!DefMI->isSafeToMove(&TII, nullptr, SawStore))
+  if (!DefMI->isSafeToMove(nullptr, SawStore))
     return false;
 
   DEBUG(dbgs() << "Try to fold single def: " << *DefMI
@@ -218,6 +218,22 @@ bool LiveRangeEdit::foldAsLoad(LiveInterval *LI,
   return true;
 }
 
+bool LiveRangeEdit::useIsKill(const LiveInterval &LI,
+                              const MachineOperand &MO) const {
+  const MachineInstr *MI = MO.getParent();
+  SlotIndex Idx = LIS.getInstructionIndex(MI).getRegSlot();
+  if (LI.Query(Idx).isKill())
+    return true;
+  const TargetRegisterInfo &TRI = *MRI.getTargetRegisterInfo();
+  unsigned SubReg = MO.getSubReg();
+  unsigned LaneMask = TRI.getSubRegIndexLaneMask(SubReg);
+  for (const LiveInterval::SubRange &S : LI.subranges()) {
+    if ((S.LaneMask & LaneMask) != 0 && S.Query(Idx).isKill())
+      return true;
+  }
+  return false;
+}
+
 /// Find all live intervals that need to shrink, then remove the instruction.
 void LiveRangeEdit::eliminateDeadDef(MachineInstr *MI, ToShrinkSet &ToShrink) {
   assert(MI->allDefsAreDead() && "Def isn't really dead");
@@ -235,7 +251,7 @@ void LiveRangeEdit::eliminateDeadDef(MachineInstr *MI, ToShrinkSet &ToShrink) {
 
   // Use the same criteria as DeadMachineInstructionElim.
   bool SawStore = false;
-  if (!MI->isSafeToMove(&TII, nullptr, SawStore)) {
+  if (!MI->isSafeToMove(nullptr, SawStore)) {
     DEBUG(dbgs() << "Can't delete: " << Idx << '\t' << *MI);
     return;
   }
@@ -266,9 +282,8 @@ void LiveRangeEdit::eliminateDeadDef(MachineInstr *MI, ToShrinkSet &ToShrink) {
     // unlikely to change anything. We typically don't want to shrink the
     // PIC base register that has lots of uses everywhere.
     // Always shrink COPY uses that probably come from live range splitting.
-    if (MI->readsVirtualRegister(Reg) &&
-        (MI->isCopy() || MOI->isDef() || MRI.hasOneNonDBGUse(Reg) ||
-         LI.Query(Idx).isKill()))
+    if ((MI->readsVirtualRegister(Reg) && (MI->isCopy() || MOI->isDef())) ||
+        (MOI->readsReg() && (MRI.hasOneNonDBGUse(Reg) || useIsKill(LI, *MOI))))
       ToShrink.insert(&LI);
 
     // Remove defined value.
diff --git a/lib/CodeGen/LiveRegMatrix.cpp b/lib/CodeGen/LiveRegMatrix.cpp
index 154ce6fc122..000151acd73 100644
--- a/lib/CodeGen/LiveRegMatrix.cpp
+++ b/lib/CodeGen/LiveRegMatrix.cpp
@@ -15,12 +15,12 @@
 #include "RegisterCoalescer.h"
 #include "llvm/ADT/Statistic.h"
 #include "llvm/CodeGen/LiveIntervalAnalysis.h"
-#include "llvm/CodeGen/MachineRegisterInfo.h"
 #include "llvm/CodeGen/VirtRegMap.h"
 #include "llvm/Support/Debug.h"
 #include "llvm/Support/Format.h"
 #include "llvm/Support/raw_ostream.h"
 #include "llvm/Target/TargetRegisterInfo.h"
+#include "llvm/Target/TargetSubtargetInfo.h"
 
 using namespace llvm;
 
@@ -49,7 +49,6 @@ void LiveRegMatrix::getAnalysisUsage(AnalysisUsage &AU) const {
 
 bool LiveRegMatrix::runOnMachineFunction(MachineFunction &MF) {
   TRI = MF.getSubtarget().getRegisterInfo();
-  MRI = &MF.getRegInfo();
   LIS = &getAnalysis<LiveIntervals>();
   VRM = &getAnalysis<VirtRegMap>();
 
@@ -101,7 +100,6 @@ void LiveRegMatrix::assign(LiveInterval &VirtReg, unsigned PhysReg) {
                << " to " << PrintReg(PhysReg, TRI) << ':');
   assert(!VRM->hasPhys(VirtReg.reg) && "Duplicate VirtReg assignment");
   VRM->assignVirt2Phys(VirtReg.reg, PhysReg);
-  MRI->setPhysRegUsed(PhysReg);
 
   foreachUnit(TRI, VirtReg, PhysReg, [&](unsigned Unit,
                                          const LiveRange &Range) {
@@ -131,6 +129,14 @@ void LiveRegMatrix::unassign(LiveInterval &VirtReg) {
   DEBUG(dbgs() << '\n');
 }
 
+bool LiveRegMatrix::isPhysRegUsed(unsigned PhysReg) const {
+  for (MCRegUnitIterator Unit(PhysReg, TRI); Unit.isValid(); ++Unit) {
+    if (!Matrix[*Unit].empty())
+      return true;
+  }
+  return false;
+}
+
 bool LiveRegMatrix::checkRegMaskInterference(LiveInterval &VirtReg,
                                              unsigned PhysReg) {
   // Check if the cached information is valid.
diff --git a/lib/CodeGen/LiveVariables.cpp b/lib/CodeGen/LiveVariables.cpp
index 11deb813dde..b355393e76f 100644
--- a/lib/CodeGen/LiveVariables.cpp
+++ b/lib/CodeGen/LiveVariables.cpp
@@ -738,45 +738,22 @@ bool LiveVariables::VarInfo::isLiveIn(const MachineBasicBlock &MBB,
 bool LiveVariables::isLiveOut(unsigned Reg, const MachineBasicBlock &MBB) {
   LiveVariables::VarInfo &VI = getVarInfo(Reg);
 
+  SmallPtrSet<const MachineBasicBlock *, 8> Kills;
+  for (unsigned i = 0, e = VI.Kills.size(); i != e; ++i)
+    Kills.insert(VI.Kills[i]->getParent());
+
   // Loop over all of the successors of the basic block, checking to see if
   // the value is either live in the block, or if it is killed in the block.
-  SmallVector<MachineBasicBlock*, 8> OpSuccBlocks;
-  for (MachineBasicBlock::const_succ_iterator SI = MBB.succ_begin(),
-         E = MBB.succ_end(); SI != E; ++SI) {
-    MachineBasicBlock *SuccMBB = *SI;
-
+  for (const MachineBasicBlock *SuccMBB : MBB.successors()) {
     // Is it alive in this successor?
     unsigned SuccIdx = SuccMBB->getNumber();
     if (VI.AliveBlocks.test(SuccIdx))
       return true;
-    OpSuccBlocks.push_back(SuccMBB);
+    // Or is it live because there is a use in a successor that kills it?
+    if (Kills.count(SuccMBB))
+      return true;
   }
 
-  // Check to see if this value is live because there is a use in a successor
-  // that kills it.
-  switch (OpSuccBlocks.size()) {
-  case 1: {
-    MachineBasicBlock *SuccMBB = OpSuccBlocks[0];
-    for (unsigned i = 0, e = VI.Kills.size(); i != e; ++i)
-      if (VI.Kills[i]->getParent() == SuccMBB)
-        return true;
-    break;
-  }
-  case 2: {
-    MachineBasicBlock *SuccMBB1 = OpSuccBlocks[0], *SuccMBB2 = OpSuccBlocks[1];
-    for (unsigned i = 0, e = VI.Kills.size(); i != e; ++i)
-      if (VI.Kills[i]->getParent() == SuccMBB1 ||
-          VI.Kills[i]->getParent() == SuccMBB2)
-        return true;
-    break;
-  }
-  default:
-    std::sort(OpSuccBlocks.begin(), OpSuccBlocks.end());
-    for (unsigned i = 0, e = VI.Kills.size(); i != e; ++i)
-      if (std::binary_search(OpSuccBlocks.begin(), OpSuccBlocks.end(),
-                             VI.Kills[i]->getParent()))
-        return true;
-  }
   return false;
 }
 
diff --git a/lib/CodeGen/MIRParser/CMakeLists.txt b/lib/CodeGen/MIRParser/CMakeLists.txt
new file mode 100644
index 00000000000..7e757f68208
--- /dev/null
+++ b/lib/CodeGen/MIRParser/CMakeLists.txt
@@ -0,0 +1,7 @@
+add_llvm_library(LLVMMIRParser
+  MILexer.cpp
+  MIParser.cpp
+  MIRParser.cpp
+  )
+
+add_dependencies(LLVMMIRParser intrinsics_gen)
diff --git a/lib/CodeGen/MIRParser/LLVMBuild.txt b/lib/CodeGen/MIRParser/LLVMBuild.txt
new file mode 100644
index 00000000000..2852124786e
--- /dev/null
+++ b/lib/CodeGen/MIRParser/LLVMBuild.txt
@@ -0,0 +1,22 @@
+;===- ./lib/CodeGen/MIRParser/LLVMBuild.txt --------------------*- Conf -*--===;
+;
+;                     The LLVM Compiler Infrastructure
+;
+; This file is distributed under the University of Illinois Open Source
+; License. See LICENSE.TXT for details.
+;
+;===------------------------------------------------------------------------===;
+;
+; This is an LLVMBuild description file for the components in this subdirectory.
+;
+; For more information on the LLVMBuild system, please see:
+;
+;   http://llvm.org/docs/LLVMBuild.html
+;
+;===------------------------------------------------------------------------===;
+
+[component_0]
+type = Library
+name = MIRParser
+parent = CodeGen
+required_libraries = AsmParser CodeGen Core MC Support Target
diff --git a/lib/CodeGen/MIRParser/MILexer.cpp b/lib/CodeGen/MIRParser/MILexer.cpp
new file mode 100644
index 00000000000..3c0dc4c0ded
--- /dev/null
+++ b/lib/CodeGen/MIRParser/MILexer.cpp
@@ -0,0 +1,390 @@
+//===- MILexer.cpp - Machine instructions lexer implementation ----------===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This file implements the lexing of machine instructions.
+//
+//===----------------------------------------------------------------------===//
+
+#include "MILexer.h"
+#include "llvm/ADT/StringExtras.h"
+#include "llvm/ADT/StringSwitch.h"
+#include "llvm/ADT/Twine.h"
+#include <cctype>
+
+using namespace llvm;
+
+namespace {
+
+/// This class provides a way to iterate and get characters from the source
+/// string.
+class Cursor {
+  const char *Ptr;
+  const char *End;
+
+public:
+  Cursor(NoneType) : Ptr(nullptr), End(nullptr) {}
+
+  explicit Cursor(StringRef Str) {
+    Ptr = Str.data();
+    End = Ptr + Str.size();
+  }
+
+  bool isEOF() const { return Ptr == End; }
+
+  char peek(int I = 0) const { return End - Ptr <= I ? 0 : Ptr[I]; }
+
+  void advance(unsigned I = 1) { Ptr += I; }
+
+  StringRef remaining() const { return StringRef(Ptr, End - Ptr); }
+
+  StringRef upto(Cursor C) const {
+    assert(C.Ptr >= Ptr && C.Ptr <= End);
+    return StringRef(Ptr, C.Ptr - Ptr);
+  }
+
+  StringRef::iterator location() const { return Ptr; }
+
+  operator bool() const { return Ptr != nullptr; }
+};
+
+} // end anonymous namespace
+
+/// Skip the leading whitespace characters and return the updated cursor.
+static Cursor skipWhitespace(Cursor C) {
+  while (isspace(C.peek()))
+    C.advance();
+  return C;
+}
+
+/// Return true if the given character satisfies the following regular
+/// expression: [-a-zA-Z$._0-9]
+static bool isIdentifierChar(char C) {
+  return isalpha(C) || isdigit(C) || C == '_' || C == '-' || C == '.' ||
+         C == '$';
+}
+
+void MIToken::unescapeQuotedStringValue(std::string &Str) const {
+  assert(isStringValueQuoted() && "String value isn't quoted");
+  StringRef Value = Range.drop_front(StringOffset);
+  assert(Value.front() == '"' && Value.back() == '"');
+  Cursor C = Cursor(Value.substr(1, Value.size() - 2));
+
+  Str.clear();
+  Str.reserve(C.remaining().size());
+  while (!C.isEOF()) {
+    char Char = C.peek();
+    if (Char == '\\') {
+      if (C.peek(1) == '\\') {
+        // Two '\' become one
+        Str += '\\';
+        C.advance(2);
+        continue;
+      }
+      if (isxdigit(C.peek(1)) && isxdigit(C.peek(2))) {
+        Str += hexDigitValue(C.peek(1)) * 16 + hexDigitValue(C.peek(2));
+        C.advance(3);
+        continue;
+      }
+    }
+    Str += Char;
+    C.advance();
+  }
+}
+
+/// Lex a string constant using the following regular expression: \"[^\"]*\"
+static Cursor lexStringConstant(
+    Cursor C,
+    function_ref<void(StringRef::iterator Loc, const Twine &)> ErrorCallback) {
+  assert(C.peek() == '"');
+  for (C.advance(); C.peek() != '"'; C.advance()) {
+    if (C.isEOF()) {
+      ErrorCallback(
+          C.location(),
+          "end of machine instruction reached before the closing '\"'");
+      return None;
+    }
+  }
+  C.advance();
+  return C;
+}
+
+static Cursor lexName(
+    Cursor C, MIToken &Token, MIToken::TokenKind Type,
+    MIToken::TokenKind QuotedType, unsigned PrefixLength,
+    function_ref<void(StringRef::iterator Loc, const Twine &)> ErrorCallback) {
+  auto Range = C;
+  C.advance(PrefixLength);
+  if (C.peek() == '"') {
+    if (Cursor R = lexStringConstant(C, ErrorCallback)) {
+      Token = MIToken(QuotedType, Range.upto(R), PrefixLength);
+      return R;
+    }
+    Token = MIToken(MIToken::Error, Range.remaining());
+    return Range;
+  }
+  while (isIdentifierChar(C.peek()))
+    C.advance();
+  Token = MIToken(Type, Range.upto(C), PrefixLength);
+  return C;
+}
+
+static MIToken::TokenKind getIdentifierKind(StringRef Identifier) {
+  return StringSwitch<MIToken::TokenKind>(Identifier)
+      .Case("_", MIToken::underscore)
+      .Case("implicit", MIToken::kw_implicit)
+      .Case("implicit-def", MIToken::kw_implicit_define)
+      .Case("dead", MIToken::kw_dead)
+      .Case("killed", MIToken::kw_killed)
+      .Case("undef", MIToken::kw_undef)
+      .Case("frame-setup", MIToken::kw_frame_setup)
+      .Case("debug-location", MIToken::kw_debug_location)
+      .Case(".cfi_offset", MIToken::kw_cfi_offset)
+      .Case(".cfi_def_cfa_register", MIToken::kw_cfi_def_cfa_register)
+      .Case(".cfi_def_cfa_offset", MIToken::kw_cfi_def_cfa_offset)
+      .Case(".cfi_def_cfa", MIToken::kw_cfi_def_cfa)
+      .Case("blockaddress", MIToken::kw_blockaddress)
+      .Case("target-index", MIToken::kw_target_index)
+      .Default(MIToken::Identifier);
+}
+
+static Cursor maybeLexIdentifier(Cursor C, MIToken &Token) {
+  if (!isalpha(C.peek()) && C.peek() != '_' && C.peek() != '.')
+    return None;
+  auto Range = C;
+  while (isIdentifierChar(C.peek()))
+    C.advance();
+  auto Identifier = Range.upto(C);
+  Token = MIToken(getIdentifierKind(Identifier), Identifier);
+  return C;
+}
+
+static Cursor maybeLexMachineBasicBlock(
+    Cursor C, MIToken &Token,
+    function_ref<void(StringRef::iterator Loc, const Twine &)> ErrorCallback) {
+  if (!C.remaining().startswith("%bb."))
+    return None;
+  auto Range = C;
+  C.advance(4); // Skip '%bb.'
+  if (!isdigit(C.peek())) {
+    Token = MIToken(MIToken::Error, C.remaining());
+    ErrorCallback(C.location(), "expected a number after '%bb.'");
+    return C;
+  }
+  auto NumberRange = C;
+  while (isdigit(C.peek()))
+    C.advance();
+  StringRef Number = NumberRange.upto(C);
+  unsigned StringOffset = 4 + Number.size(); // Drop '%bb.<id>'
+  if (C.peek() == '.') {
+    C.advance(); // Skip '.'
+    ++StringOffset;
+    while (isIdentifierChar(C.peek()))
+      C.advance();
+  }
+  Token = MIToken(MIToken::MachineBasicBlock, Range.upto(C), APSInt(Number),
+                  StringOffset);
+  return C;
+}
+
+static Cursor maybeLexIndex(Cursor C, MIToken &Token, StringRef Rule,
+                            MIToken::TokenKind Kind) {
+  if (!C.remaining().startswith(Rule) || !isdigit(C.peek(Rule.size())))
+    return None;
+  auto Range = C;
+  C.advance(Rule.size());
+  auto NumberRange = C;
+  while (isdigit(C.peek()))
+    C.advance();
+  Token = MIToken(Kind, Range.upto(C), APSInt(NumberRange.upto(C)));
+  return C;
+}
+
+static Cursor maybeLexIndexAndName(Cursor C, MIToken &Token, StringRef Rule,
+                                   MIToken::TokenKind Kind) {
+  if (!C.remaining().startswith(Rule) || !isdigit(C.peek(Rule.size())))
+    return None;
+  auto Range = C;
+  C.advance(Rule.size());
+  auto NumberRange = C;
+  while (isdigit(C.peek()))
+    C.advance();
+  StringRef Number = NumberRange.upto(C);
+  unsigned StringOffset = Rule.size() + Number.size();
+  if (C.peek() == '.') {
+    C.advance();
+    ++StringOffset;
+    while (isIdentifierChar(C.peek()))
+      C.advance();
+  }
+  Token = MIToken(Kind, Range.upto(C), APSInt(Number), StringOffset);
+  return C;
+}
+
+static Cursor maybeLexJumpTableIndex(Cursor C, MIToken &Token) {
+  return maybeLexIndex(C, Token, "%jump-table.", MIToken::JumpTableIndex);
+}
+
+static Cursor maybeLexStackObject(Cursor C, MIToken &Token) {
+  return maybeLexIndexAndName(C, Token, "%stack.", MIToken::StackObject);
+}
+
+static Cursor maybeLexFixedStackObject(Cursor C, MIToken &Token) {
+  return maybeLexIndex(C, Token, "%fixed-stack.", MIToken::FixedStackObject);
+}
+
+static Cursor maybeLexConstantPoolItem(Cursor C, MIToken &Token) {
+  return maybeLexIndex(C, Token, "%const.", MIToken::ConstantPoolItem);
+}
+
+static Cursor maybeLexIRBlock(
+    Cursor C, MIToken &Token,
+    function_ref<void(StringRef::iterator Loc, const Twine &)> ErrorCallback) {
+  const StringRef Rule = "%ir-block.";
+  if (!C.remaining().startswith(Rule))
+    return None;
+  if (isdigit(C.peek(Rule.size())))
+    return maybeLexIndex(C, Token, Rule, MIToken::IRBlock);
+  return lexName(C, Token, MIToken::NamedIRBlock, MIToken::QuotedNamedIRBlock,
+                 Rule.size(), ErrorCallback);
+}
+
+static Cursor lexVirtualRegister(Cursor C, MIToken &Token) {
+  auto Range = C;
+  C.advance(); // Skip '%'
+  auto NumberRange = C;
+  while (isdigit(C.peek()))
+    C.advance();
+  Token = MIToken(MIToken::VirtualRegister, Range.upto(C),
+                  APSInt(NumberRange.upto(C)));
+  return C;
+}
+
+static Cursor maybeLexRegister(Cursor C, MIToken &Token) {
+  if (C.peek() != '%')
+    return None;
+  if (isdigit(C.peek(1)))
+    return lexVirtualRegister(C, Token);
+  auto Range = C;
+  C.advance(); // Skip '%'
+  while (isIdentifierChar(C.peek()))
+    C.advance();
+  Token = MIToken(MIToken::NamedRegister, Range.upto(C),
+                  /*StringOffset=*/1); // Drop the '%'
+  return C;
+}
+
+static Cursor maybeLexGlobalValue(
+    Cursor C, MIToken &Token,
+    function_ref<void(StringRef::iterator Loc, const Twine &)> ErrorCallback) {
+  if (C.peek() != '@')
+    return None;
+  if (!isdigit(C.peek(1)))
+    return lexName(C, Token, MIToken::NamedGlobalValue,
+                   MIToken::QuotedNamedGlobalValue, /*PrefixLength=*/1,
+                   ErrorCallback);
+  auto Range = C;
+  C.advance(1); // Skip the '@'
+  auto NumberRange = C;
+  while (isdigit(C.peek()))
+    C.advance();
+  Token =
+      MIToken(MIToken::GlobalValue, Range.upto(C), APSInt(NumberRange.upto(C)));
+  return C;
+}
+
+static Cursor maybeLexExternalSymbol(
+    Cursor C, MIToken &Token,
+    function_ref<void(StringRef::iterator Loc, const Twine &)> ErrorCallback) {
+  if (C.peek() != '$')
+    return None;
+  return lexName(C, Token, MIToken::ExternalSymbol,
+                 MIToken::QuotedExternalSymbol,
+                 /*PrefixLength=*/1, ErrorCallback);
+}
+
+static Cursor maybeLexIntegerLiteral(Cursor C, MIToken &Token) {
+  if (!isdigit(C.peek()) && (C.peek() != '-' || !isdigit(C.peek(1))))
+    return None;
+  auto Range = C;
+  C.advance();
+  while (isdigit(C.peek()))
+    C.advance();
+  StringRef StrVal = Range.upto(C);
+  Token = MIToken(MIToken::IntegerLiteral, StrVal, APSInt(StrVal));
+  return C;
+}
+
+static MIToken::TokenKind symbolToken(char C) {
+  switch (C) {
+  case ',':
+    return MIToken::comma;
+  case '=':
+    return MIToken::equal;
+  case ':':
+    return MIToken::colon;
+  case '!':
+    return MIToken::exclaim;
+  case '(':
+    return MIToken::lparen;
+  case ')':
+    return MIToken::rparen;
+  default:
+    return MIToken::Error;
+  }
+}
+
+static Cursor maybeLexSymbol(Cursor C, MIToken &Token) {
+  auto Kind = symbolToken(C.peek());
+  if (Kind == MIToken::Error)
+    return None;
+  auto Range = C;
+  C.advance();
+  Token = MIToken(Kind, Range.upto(C));
+  return C;
+}
+
+StringRef llvm::lexMIToken(
+    StringRef Source, MIToken &Token,
+    function_ref<void(StringRef::iterator Loc, const Twine &)> ErrorCallback) {
+  auto C = skipWhitespace(Cursor(Source));
+  if (C.isEOF()) {
+    Token = MIToken(MIToken::Eof, C.remaining());
+    return C.remaining();
+  }
+
+  if (Cursor R = maybeLexIdentifier(C, Token))
+    return R.remaining();
+  if (Cursor R = maybeLexMachineBasicBlock(C, Token, ErrorCallback))
+    return R.remaining();
+  if (Cursor R = maybeLexJumpTableIndex(C, Token))
+    return R.remaining();
+  if (Cursor R = maybeLexStackObject(C, Token))
+    return R.remaining();
+  if (Cursor R = maybeLexFixedStackObject(C, Token))
+    return R.remaining();
+  if (Cursor R = maybeLexConstantPoolItem(C, Token))
+    return R.remaining();
+  if (Cursor R = maybeLexIRBlock(C, Token, ErrorCallback))
+    return R.remaining();
+  if (Cursor R = maybeLexRegister(C, Token))
+    return R.remaining();
+  if (Cursor R = maybeLexGlobalValue(C, Token, ErrorCallback))
+    return R.remaining();
+  if (Cursor R = maybeLexExternalSymbol(C, Token, ErrorCallback))
+    return R.remaining();
+  if (Cursor R = maybeLexIntegerLiteral(C, Token))
+    return R.remaining();
+  if (Cursor R = maybeLexSymbol(C, Token))
+    return R.remaining();
+
+  Token = MIToken(MIToken::Error, C.remaining());
+  ErrorCallback(C.location(),
+                Twine("unexpected character '") + Twine(C.peek()) + "'");
+  return C.remaining();
+}
diff --git a/lib/CodeGen/MIRParser/MILexer.h b/lib/CodeGen/MIRParser/MILexer.h
new file mode 100644
index 00000000000..31c3575bba9
--- /dev/null
+++ b/lib/CodeGen/MIRParser/MILexer.h
@@ -0,0 +1,157 @@
+//===- MILexer.h - Lexer for machine instructions -------------------------===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This file declares the function that lexes the machine instruction source
+// string.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIB_CODEGEN_MIRPARSER_MILEXER_H
+#define LLVM_LIB_CODEGEN_MIRPARSER_MILEXER_H
+
+#include "llvm/ADT/APSInt.h"
+#include "llvm/ADT/StringRef.h"
+#include "llvm/ADT/STLExtras.h"
+#include <functional>
+
+namespace llvm {
+
+class Twine;
+
+/// A token produced by the machine instruction lexer.
+struct MIToken {
+  enum TokenKind {
+    // Markers
+    Eof,
+    Error,
+
+    // Tokens with no info.
+    comma,
+    equal,
+    underscore,
+    colon,
+    exclaim,
+    lparen,
+    rparen,
+
+    // Keywords
+    kw_implicit,
+    kw_implicit_define,
+    kw_dead,
+    kw_killed,
+    kw_undef,
+    kw_frame_setup,
+    kw_debug_location,
+    kw_cfi_offset,
+    kw_cfi_def_cfa_register,
+    kw_cfi_def_cfa_offset,
+    kw_cfi_def_cfa,
+    kw_blockaddress,
+    kw_target_index,
+
+    // Identifier tokens
+    Identifier,
+    NamedRegister,
+    MachineBasicBlock,
+    StackObject,
+    FixedStackObject,
+    NamedGlobalValue,
+    QuotedNamedGlobalValue,
+    GlobalValue,
+    ExternalSymbol,
+    QuotedExternalSymbol,
+
+    // Other tokens
+    IntegerLiteral,
+    VirtualRegister,
+    ConstantPoolItem,
+    JumpTableIndex,
+    NamedIRBlock,
+    QuotedNamedIRBlock,
+    IRBlock,
+  };
+
+private:
+  TokenKind Kind;
+  unsigned StringOffset;
+  StringRef Range;
+  APSInt IntVal;
+
+public:
+  MIToken(TokenKind Kind, StringRef Range, unsigned StringOffset = 0)
+      : Kind(Kind), StringOffset(StringOffset), Range(Range) {}
+
+  MIToken(TokenKind Kind, StringRef Range, const APSInt &IntVal,
+          unsigned StringOffset = 0)
+      : Kind(Kind), StringOffset(StringOffset), Range(Range), IntVal(IntVal) {}
+
+  TokenKind kind() const { return Kind; }
+
+  bool isError() const { return Kind == Error; }
+
+  bool isRegister() const {
+    return Kind == NamedRegister || Kind == underscore ||
+           Kind == VirtualRegister;
+  }
+
+  bool isRegisterFlag() const {
+    return Kind == kw_implicit || Kind == kw_implicit_define ||
+           Kind == kw_dead || Kind == kw_killed || Kind == kw_undef;
+  }
+
+  bool is(TokenKind K) const { return Kind == K; }
+
+  bool isNot(TokenKind K) const { return Kind != K; }
+
+  StringRef::iterator location() const { return Range.begin(); }
+
+  bool isStringValueQuoted() const {
+    return Kind == QuotedNamedGlobalValue || Kind == QuotedExternalSymbol ||
+           Kind == QuotedNamedIRBlock;
+  }
+
+  /// Return the token's raw string value.
+  ///
+  /// If the string value is quoted, this method returns that quoted string as
+  /// it is, without unescaping the string value.
+  StringRef rawStringValue() const { return Range.drop_front(StringOffset); }
+
+  /// Return token's string value.
+  ///
+  /// Expects the string value to be unquoted.
+  StringRef stringValue() const {
+    assert(!isStringValueQuoted() && "String value is quoted");
+    return Range.drop_front(StringOffset);
+  }
+
+  /// Unescapes the token's string value.
+  ///
+  /// Expects the string value to be quoted.
+  void unescapeQuotedStringValue(std::string &Str) const;
+
+  const APSInt &integerValue() const { return IntVal; }
+
+  bool hasIntegerValue() const {
+    return Kind == IntegerLiteral || Kind == MachineBasicBlock ||
+           Kind == StackObject || Kind == FixedStackObject ||
+           Kind == GlobalValue || Kind == VirtualRegister ||
+           Kind == ConstantPoolItem || Kind == JumpTableIndex ||
+           Kind == IRBlock;
+  }
+};
+
+/// Consume a single machine instruction token in the given source and return
+/// the remaining source string.
+StringRef lexMIToken(
+    StringRef Source, MIToken &Token,
+    function_ref<void(StringRef::iterator, const Twine &)> ErrorCallback);
+
+} // end namespace llvm
+
+#endif
diff --git a/lib/CodeGen/MIRParser/MIParser.cpp b/lib/CodeGen/MIRParser/MIParser.cpp
new file mode 100644
index 00000000000..c71eeb99fca
--- /dev/null
+++ b/lib/CodeGen/MIRParser/MIParser.cpp
@@ -0,0 +1,1070 @@
+//===- MIParser.cpp - Machine instructions parser implementation ----------===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This file implements the parsing of machine instructions.
+//
+//===----------------------------------------------------------------------===//
+
+#include "MIParser.h"
+#include "MILexer.h"
+#include "llvm/ADT/StringMap.h"
+#include "llvm/AsmParser/SlotMapping.h"
+#include "llvm/CodeGen/MachineBasicBlock.h"
+#include "llvm/CodeGen/MachineFunction.h"
+#include "llvm/CodeGen/MachineFrameInfo.h"
+#include "llvm/CodeGen/MachineInstr.h"
+#include "llvm/CodeGen/MachineInstrBuilder.h"
+#include "llvm/CodeGen/MachineModuleInfo.h"
+#include "llvm/IR/Instructions.h"
+#include "llvm/IR/Constants.h"
+#include "llvm/IR/Module.h"
+#include "llvm/IR/ModuleSlotTracker.h"
+#include "llvm/IR/ValueSymbolTable.h"
+#include "llvm/Support/raw_ostream.h"
+#include "llvm/Support/SourceMgr.h"
+#include "llvm/Target/TargetSubtargetInfo.h"
+#include "llvm/Target/TargetInstrInfo.h"
+
+using namespace llvm;
+
+namespace {
+
+struct StringValueUtility {
+  StringRef String;
+  std::string UnescapedString;
+
+  StringValueUtility(const MIToken &Token) {
+    if (Token.isStringValueQuoted()) {
+      Token.unescapeQuotedStringValue(UnescapedString);
+      String = UnescapedString;
+      return;
+    }
+    String = Token.stringValue();
+  }
+
+  operator StringRef() const { return String; }
+};
+
+/// A wrapper struct around the 'MachineOperand' struct that includes a source
+/// range.
+struct MachineOperandWithLocation {
+  MachineOperand Operand;
+  StringRef::iterator Begin;
+  StringRef::iterator End;
+
+  MachineOperandWithLocation(const MachineOperand &Operand,
+                             StringRef::iterator Begin, StringRef::iterator End)
+      : Operand(Operand), Begin(Begin), End(End) {}
+};
+
+class MIParser {
+  SourceMgr &SM;
+  MachineFunction &MF;
+  SMDiagnostic &Error;
+  StringRef Source, CurrentSource;
+  MIToken Token;
+  const PerFunctionMIParsingState &PFS;
+  /// Maps from indices to unnamed global values and metadata nodes.
+  const SlotMapping &IRSlots;
+  /// Maps from instruction names to op codes.
+  StringMap<unsigned> Names2InstrOpCodes;
+  /// Maps from register names to registers.
+  StringMap<unsigned> Names2Regs;
+  /// Maps from register mask names to register masks.
+  StringMap<const uint32_t *> Names2RegMasks;
+  /// Maps from subregister names to subregister indices.
+  StringMap<unsigned> Names2SubRegIndices;
+  /// Maps from slot numbers to function's unnamed basic blocks.
+  DenseMap<unsigned, const BasicBlock *> Slots2BasicBlocks;
+  /// Maps from target index names to target indices.
+  StringMap<int> Names2TargetIndices;
+
+public:
+  MIParser(SourceMgr &SM, MachineFunction &MF, SMDiagnostic &Error,
+           StringRef Source, const PerFunctionMIParsingState &PFS,
+           const SlotMapping &IRSlots);
+
+  void lex();
+
+  /// Report an error at the current location with the given message.
+  ///
+  /// This function always return true.
+  bool error(const Twine &Msg);
+
+  /// Report an error at the given location with the given message.
+  ///
+  /// This function always return true.
+  bool error(StringRef::iterator Loc, const Twine &Msg);
+
+  bool parse(MachineInstr *&MI);
+  bool parseStandaloneMBB(MachineBasicBlock *&MBB);
+  bool parseStandaloneNamedRegister(unsigned &Reg);
+  bool parseStandaloneVirtualRegister(unsigned &Reg);
+  bool parseStandaloneIRBlockReference(const BasicBlock *&BB);
+
+  bool parseRegister(unsigned &Reg);
+  bool parseRegisterFlag(unsigned &Flags);
+  bool parseSubRegisterIndex(unsigned &SubReg);
+  bool parseRegisterOperand(MachineOperand &Dest, bool IsDef = false);
+  bool parseImmediateOperand(MachineOperand &Dest);
+  bool parseMBBReference(MachineBasicBlock *&MBB);
+  bool parseMBBOperand(MachineOperand &Dest);
+  bool parseStackObjectOperand(MachineOperand &Dest);
+  bool parseFixedStackObjectOperand(MachineOperand &Dest);
+  bool parseGlobalValue(GlobalValue *&GV);
+  bool parseGlobalAddressOperand(MachineOperand &Dest);
+  bool parseConstantPoolIndexOperand(MachineOperand &Dest);
+  bool parseJumpTableIndexOperand(MachineOperand &Dest);
+  bool parseExternalSymbolOperand(MachineOperand &Dest);
+  bool parseMDNode(MDNode *&Node);
+  bool parseMetadataOperand(MachineOperand &Dest);
+  bool parseCFIOffset(int &Offset);
+  bool parseCFIRegister(unsigned &Reg);
+  bool parseCFIOperand(MachineOperand &Dest);
+  bool parseIRBlock(BasicBlock *&BB, const Function &F);
+  bool parseBlockAddressOperand(MachineOperand &Dest);
+  bool parseTargetIndexOperand(MachineOperand &Dest);
+  bool parseMachineOperand(MachineOperand &Dest);
+
+private:
+  /// Convert the integer literal in the current token into an unsigned integer.
+  ///
+  /// Return true if an error occurred.
+  bool getUnsigned(unsigned &Result);
+
+  /// If the current token is of the given kind, consume it and return false.
+  /// Otherwise report an error and return true.
+  bool expectAndConsume(MIToken::TokenKind TokenKind);
+
+  void initNames2InstrOpCodes();
+
+  /// Try to convert an instruction name to an opcode. Return true if the
+  /// instruction name is invalid.
+  bool parseInstrName(StringRef InstrName, unsigned &OpCode);
+
+  bool parseInstruction(unsigned &OpCode, unsigned &Flags);
+
+  bool verifyImplicitOperands(ArrayRef<MachineOperandWithLocation> Operands,
+                              const MCInstrDesc &MCID);
+
+  void initNames2Regs();
+
+  /// Try to convert a register name to a register number. Return true if the
+  /// register name is invalid.
+  bool getRegisterByName(StringRef RegName, unsigned &Reg);
+
+  void initNames2RegMasks();
+
+  /// Check if the given identifier is a name of a register mask.
+  ///
+  /// Return null if the identifier isn't a register mask.
+  const uint32_t *getRegMask(StringRef Identifier);
+
+  void initNames2SubRegIndices();
+
+  /// Check if the given identifier is a name of a subregister index.
+  ///
+  /// Return 0 if the name isn't a subregister index class.
+  unsigned getSubRegIndex(StringRef Name);
+
+  void initSlots2BasicBlocks();
+
+  const BasicBlock *getIRBlock(unsigned Slot);
+
+  void initNames2TargetIndices();
+
+  /// Try to convert a name of target index to the corresponding target index.
+  ///
+  /// Return true if the name isn't a name of a target index.
+  bool getTargetIndex(StringRef Name, int &Index);
+};
+
+} // end anonymous namespace
+
+MIParser::MIParser(SourceMgr &SM, MachineFunction &MF, SMDiagnostic &Error,
+                   StringRef Source, const PerFunctionMIParsingState &PFS,
+                   const SlotMapping &IRSlots)
+    : SM(SM), MF(MF), Error(Error), Source(Source), CurrentSource(Source),
+      Token(MIToken::Error, StringRef()), PFS(PFS), IRSlots(IRSlots) {}
+
+void MIParser::lex() {
+  CurrentSource = lexMIToken(
+      CurrentSource, Token,
+      [this](StringRef::iterator Loc, const Twine &Msg) { error(Loc, Msg); });
+}
+
+bool MIParser::error(const Twine &Msg) { return error(Token.location(), Msg); }
+
+bool MIParser::error(StringRef::iterator Loc, const Twine &Msg) {
+  assert(Loc >= Source.data() && Loc <= (Source.data() + Source.size()));
+  Error = SMDiagnostic(
+      SM, SMLoc(),
+      SM.getMemoryBuffer(SM.getMainFileID())->getBufferIdentifier(), 1,
+      Loc - Source.data(), SourceMgr::DK_Error, Msg.str(), Source, None, None);
+  return true;
+}
+
+static const char *toString(MIToken::TokenKind TokenKind) {
+  switch (TokenKind) {
+  case MIToken::comma:
+    return "','";
+  case MIToken::equal:
+    return "'='";
+  case MIToken::lparen:
+    return "'('";
+  case MIToken::rparen:
+    return "')'";
+  default:
+    return "<unknown token>";
+  }
+}
+
+bool MIParser::expectAndConsume(MIToken::TokenKind TokenKind) {
+  if (Token.isNot(TokenKind))
+    return error(Twine("expected ") + toString(TokenKind));
+  lex();
+  return false;
+}
+
+bool MIParser::parse(MachineInstr *&MI) {
+  lex();
+
+  // Parse any register operands before '='
+  MachineOperand MO = MachineOperand::CreateImm(0);
+  SmallVector<MachineOperandWithLocation, 8> Operands;
+  while (Token.isRegister() || Token.isRegisterFlag()) {
+    auto Loc = Token.location();
+    if (parseRegisterOperand(MO, /*IsDef=*/true))
+      return true;
+    Operands.push_back(MachineOperandWithLocation(MO, Loc, Token.location()));
+    if (Token.isNot(MIToken::comma))
+      break;
+    lex();
+  }
+  if (!Operands.empty() && expectAndConsume(MIToken::equal))
+    return true;
+
+  unsigned OpCode, Flags = 0;
+  if (Token.isError() || parseInstruction(OpCode, Flags))
+    return true;
+
+  // TODO: Parse the bundle instruction flags and memory operands.
+
+  // Parse the remaining machine operands.
+  while (Token.isNot(MIToken::Eof) && Token.isNot(MIToken::kw_debug_location)) {
+    auto Loc = Token.location();
+    if (parseMachineOperand(MO))
+      return true;
+    Operands.push_back(MachineOperandWithLocation(MO, Loc, Token.location()));
+    if (Token.is(MIToken::Eof))
+      break;
+    if (Token.isNot(MIToken::comma))
+      return error("expected ',' before the next machine operand");
+    lex();
+  }
+
+  DebugLoc DebugLocation;
+  if (Token.is(MIToken::kw_debug_location)) {
+    lex();
+    if (Token.isNot(MIToken::exclaim))
+      return error("expected a metadata node after 'debug-location'");
+    MDNode *Node = nullptr;
+    if (parseMDNode(Node))
+      return true;
+    DebugLocation = DebugLoc(Node);
+  }
+
+  const auto &MCID = MF.getSubtarget().getInstrInfo()->get(OpCode);
+  if (!MCID.isVariadic()) {
+    // FIXME: Move the implicit operand verification to the machine verifier.
+    if (verifyImplicitOperands(Operands, MCID))
+      return true;
+  }
+
+  // TODO: Check for extraneous machine operands.
+  MI = MF.CreateMachineInstr(MCID, DebugLocation, /*NoImplicit=*/true);
+  MI->setFlags(Flags);
+  for (const auto &Operand : Operands)
+    MI->addOperand(MF, Operand.Operand);
+  return false;
+}
+
+bool MIParser::parseStandaloneMBB(MachineBasicBlock *&MBB) {
+  lex();
+  if (Token.isNot(MIToken::MachineBasicBlock))
+    return error("expected a machine basic block reference");
+  if (parseMBBReference(MBB))
+    return true;
+  lex();
+  if (Token.isNot(MIToken::Eof))
+    return error(
+        "expected end of string after the machine basic block reference");
+  return false;
+}
+
+bool MIParser::parseStandaloneNamedRegister(unsigned &Reg) {
+  lex();
+  if (Token.isNot(MIToken::NamedRegister))
+    return error("expected a named register");
+  if (parseRegister(Reg))
+    return 0;
+  lex();
+  if (Token.isNot(MIToken::Eof))
+    return error("expected end of string after the register reference");
+  return false;
+}
+
+bool MIParser::parseStandaloneVirtualRegister(unsigned &Reg) {
+  lex();
+  if (Token.isNot(MIToken::VirtualRegister))
+    return error("expected a virtual register");
+  if (parseRegister(Reg))
+    return 0;
+  lex();
+  if (Token.isNot(MIToken::Eof))
+    return error("expected end of string after the register reference");
+  return false;
+}
+
+bool MIParser::parseStandaloneIRBlockReference(const BasicBlock *&BB) {
+  lex();
+  if (Token.isNot(MIToken::IRBlock))
+    return error("expected an IR block reference");
+  unsigned SlotNumber = 0;
+  if (getUnsigned(SlotNumber))
+    return true;
+  BB = getIRBlock(SlotNumber);
+  if (!BB)
+    return error(Twine("use of undefined IR block '%ir-block.") +
+                 Twine(SlotNumber) + "'");
+  lex();
+  if (Token.isNot(MIToken::Eof))
+    return error("expected end of string after the IR block reference");
+  return false;
+}
+
+static const char *printImplicitRegisterFlag(const MachineOperand &MO) {
+  assert(MO.isImplicit());
+  return MO.isDef() ? "implicit-def" : "implicit";
+}
+
+static std::string getRegisterName(const TargetRegisterInfo *TRI,
+                                   unsigned Reg) {
+  assert(TargetRegisterInfo::isPhysicalRegister(Reg) && "expected phys reg");
+  return StringRef(TRI->getName(Reg)).lower();
+}
+
+bool MIParser::verifyImplicitOperands(
+    ArrayRef<MachineOperandWithLocation> Operands, const MCInstrDesc &MCID) {
+  if (MCID.isCall())
+    // We can't verify call instructions as they can contain arbitrary implicit
+    // register and register mask operands.
+    return false;
+
+  // Gather all the expected implicit operands.
+  SmallVector<MachineOperand, 4> ImplicitOperands;
+  if (MCID.ImplicitDefs)
+    for (const uint16_t *ImpDefs = MCID.getImplicitDefs(); *ImpDefs; ++ImpDefs)
+      ImplicitOperands.push_back(
+          MachineOperand::CreateReg(*ImpDefs, true, true));
+  if (MCID.ImplicitUses)
+    for (const uint16_t *ImpUses = MCID.getImplicitUses(); *ImpUses; ++ImpUses)
+      ImplicitOperands.push_back(
+          MachineOperand::CreateReg(*ImpUses, false, true));
+
+  const auto *TRI = MF.getSubtarget().getRegisterInfo();
+  assert(TRI && "Expected target register info");
+  size_t I = ImplicitOperands.size(), J = Operands.size();
+  while (I) {
+    --I;
+    if (J) {
+      --J;
+      const auto &ImplicitOperand = ImplicitOperands[I];
+      const auto &Operand = Operands[J].Operand;
+      if (ImplicitOperand.isIdenticalTo(Operand))
+        continue;
+      if (Operand.isReg() && Operand.isImplicit()) {
+        return error(Operands[J].Begin,
+                     Twine("expected an implicit register operand '") +
+                         printImplicitRegisterFlag(ImplicitOperand) + " %" +
+                         getRegisterName(TRI, ImplicitOperand.getReg()) + "'");
+      }
+    }
+    // TODO: Fix source location when Operands[J].end is right before '=', i.e:
+    // insead of reporting an error at this location:
+    //            %eax = MOV32r0
+    //                 ^
+    // report the error at the following location:
+    //            %eax = MOV32r0
+    //                          ^
+    return error(J < Operands.size() ? Operands[J].End : Token.location(),
+                 Twine("missing implicit register operand '") +
+                     printImplicitRegisterFlag(ImplicitOperands[I]) + " %" +
+                     getRegisterName(TRI, ImplicitOperands[I].getReg()) + "'");
+  }
+  return false;
+}
+
+bool MIParser::parseInstruction(unsigned &OpCode, unsigned &Flags) {
+  if (Token.is(MIToken::kw_frame_setup)) {
+    Flags |= MachineInstr::FrameSetup;
+    lex();
+  }
+  if (Token.isNot(MIToken::Identifier))
+    return error("expected a machine instruction");
+  StringRef InstrName = Token.stringValue();
+  if (parseInstrName(InstrName, OpCode))
+    return error(Twine("unknown machine instruction name '") + InstrName + "'");
+  lex();
+  return false;
+}
+
+bool MIParser::parseRegister(unsigned &Reg) {
+  switch (Token.kind()) {
+  case MIToken::underscore:
+    Reg = 0;
+    break;
+  case MIToken::NamedRegister: {
+    StringRef Name = Token.stringValue();
+    if (getRegisterByName(Name, Reg))
+      return error(Twine("unknown register name '") + Name + "'");
+    break;
+  }
+  case MIToken::VirtualRegister: {
+    unsigned ID;
+    if (getUnsigned(ID))
+      return true;
+    const auto RegInfo = PFS.VirtualRegisterSlots.find(ID);
+    if (RegInfo == PFS.VirtualRegisterSlots.end())
+      return error(Twine("use of undefined virtual register '%") + Twine(ID) +
+                   "'");
+    Reg = RegInfo->second;
+    break;
+  }
+  // TODO: Parse other register kinds.
+  default:
+    llvm_unreachable("The current token should be a register");
+  }
+  return false;
+}
+
+bool MIParser::parseRegisterFlag(unsigned &Flags) {
+  switch (Token.kind()) {
+  case MIToken::kw_implicit:
+    Flags |= RegState::Implicit;
+    break;
+  case MIToken::kw_implicit_define:
+    Flags |= RegState::ImplicitDefine;
+    break;
+  case MIToken::kw_dead:
+    Flags |= RegState::Dead;
+    break;
+  case MIToken::kw_killed:
+    Flags |= RegState::Kill;
+    break;
+  case MIToken::kw_undef:
+    Flags |= RegState::Undef;
+    break;
+  // TODO: report an error when we specify the same flag more than once.
+  // TODO: parse the other register flags.
+  default:
+    llvm_unreachable("The current token should be a register flag");
+  }
+  lex();
+  return false;
+}
+
+bool MIParser::parseSubRegisterIndex(unsigned &SubReg) {
+  assert(Token.is(MIToken::colon));
+  lex();
+  if (Token.isNot(MIToken::Identifier))
+    return error("expected a subregister index after ':'");
+  auto Name = Token.stringValue();
+  SubReg = getSubRegIndex(Name);
+  if (!SubReg)
+    return error(Twine("use of unknown subregister index '") + Name + "'");
+  lex();
+  return false;
+}
+
+bool MIParser::parseRegisterOperand(MachineOperand &Dest, bool IsDef) {
+  unsigned Reg;
+  unsigned Flags = IsDef ? RegState::Define : 0;
+  while (Token.isRegisterFlag()) {
+    if (parseRegisterFlag(Flags))
+      return true;
+  }
+  if (!Token.isRegister())
+    return error("expected a register after register flags");
+  if (parseRegister(Reg))
+    return true;
+  lex();
+  unsigned SubReg = 0;
+  if (Token.is(MIToken::colon)) {
+    if (parseSubRegisterIndex(SubReg))
+      return true;
+  }
+  Dest = MachineOperand::CreateReg(
+      Reg, Flags & RegState::Define, Flags & RegState::Implicit,
+      Flags & RegState::Kill, Flags & RegState::Dead, Flags & RegState::Undef,
+      /*isEarlyClobber=*/false, SubReg);
+  return false;
+}
+
+bool MIParser::parseImmediateOperand(MachineOperand &Dest) {
+  assert(Token.is(MIToken::IntegerLiteral));
+  const APSInt &Int = Token.integerValue();
+  if (Int.getMinSignedBits() > 64)
+    // TODO: Replace this with an error when we can parse CIMM Machine Operands.
+    llvm_unreachable("Can't parse large integer literals yet!");
+  Dest = MachineOperand::CreateImm(Int.getExtValue());
+  lex();
+  return false;
+}
+
+bool MIParser::getUnsigned(unsigned &Result) {
+  assert(Token.hasIntegerValue() && "Expected a token with an integer value");
+  const uint64_t Limit = uint64_t(std::numeric_limits<unsigned>::max()) + 1;
+  uint64_t Val64 = Token.integerValue().getLimitedValue(Limit);
+  if (Val64 == Limit)
+    return error("expected 32-bit integer (too large)");
+  Result = Val64;
+  return false;
+}
+
+bool MIParser::parseMBBReference(MachineBasicBlock *&MBB) {
+  assert(Token.is(MIToken::MachineBasicBlock));
+  unsigned Number;
+  if (getUnsigned(Number))
+    return true;
+  auto MBBInfo = PFS.MBBSlots.find(Number);
+  if (MBBInfo == PFS.MBBSlots.end())
+    return error(Twine("use of undefined machine basic block #") +
+                 Twine(Number));
+  MBB = MBBInfo->second;
+  if (!Token.stringValue().empty() && Token.stringValue() != MBB->getName())
+    return error(Twine("the name of machine basic block #") + Twine(Number) +
+                 " isn't '" + Token.stringValue() + "'");
+  return false;
+}
+
+bool MIParser::parseMBBOperand(MachineOperand &Dest) {
+  MachineBasicBlock *MBB;
+  if (parseMBBReference(MBB))
+    return true;
+  Dest = MachineOperand::CreateMBB(MBB);
+  lex();
+  return false;
+}
+
+bool MIParser::parseStackObjectOperand(MachineOperand &Dest) {
+  assert(Token.is(MIToken::StackObject));
+  unsigned ID;
+  if (getUnsigned(ID))
+    return true;
+  auto ObjectInfo = PFS.StackObjectSlots.find(ID);
+  if (ObjectInfo == PFS.StackObjectSlots.end())
+    return error(Twine("use of undefined stack object '%stack.") + Twine(ID) +
+                 "'");
+  StringRef Name;
+  if (const auto *Alloca =
+          MF.getFrameInfo()->getObjectAllocation(ObjectInfo->second))
+    Name = Alloca->getName();
+  if (!Token.stringValue().empty() && Token.stringValue() != Name)
+    return error(Twine("the name of the stack object '%stack.") + Twine(ID) +
+                 "' isn't '" + Token.stringValue() + "'");
+  lex();
+  Dest = MachineOperand::CreateFI(ObjectInfo->second);
+  return false;
+}
+
+bool MIParser::parseFixedStackObjectOperand(MachineOperand &Dest) {
+  assert(Token.is(MIToken::FixedStackObject));
+  unsigned ID;
+  if (getUnsigned(ID))
+    return true;
+  auto ObjectInfo = PFS.FixedStackObjectSlots.find(ID);
+  if (ObjectInfo == PFS.FixedStackObjectSlots.end())
+    return error(Twine("use of undefined fixed stack object '%fixed-stack.") +
+                 Twine(ID) + "'");
+  lex();
+  Dest = MachineOperand::CreateFI(ObjectInfo->second);
+  return false;
+}
+
+bool MIParser::parseGlobalValue(GlobalValue *&GV) {
+  switch (Token.kind()) {
+  case MIToken::NamedGlobalValue:
+  case MIToken::QuotedNamedGlobalValue: {
+    StringValueUtility Name(Token);
+    const Module *M = MF.getFunction()->getParent();
+    GV = M->getNamedValue(Name);
+    if (!GV)
+      return error(Twine("use of undefined global value '@") +
+                   Token.rawStringValue() + "'");
+    break;
+  }
+  case MIToken::GlobalValue: {
+    unsigned GVIdx;
+    if (getUnsigned(GVIdx))
+      return true;
+    if (GVIdx >= IRSlots.GlobalValues.size())
+      return error(Twine("use of undefined global value '@") + Twine(GVIdx) +
+                   "'");
+    GV = IRSlots.GlobalValues[GVIdx];
+    break;
+  }
+  default:
+    llvm_unreachable("The current token should be a global value");
+  }
+  return false;
+}
+
+bool MIParser::parseGlobalAddressOperand(MachineOperand &Dest) {
+  GlobalValue *GV = nullptr;
+  if (parseGlobalValue(GV))
+    return true;
+  Dest = MachineOperand::CreateGA(GV, /*Offset=*/0);
+  // TODO: Parse offset and target flags.
+  lex();
+  return false;
+}
+
+bool MIParser::parseConstantPoolIndexOperand(MachineOperand &Dest) {
+  assert(Token.is(MIToken::ConstantPoolItem));
+  unsigned ID;
+  if (getUnsigned(ID))
+    return true;
+  auto ConstantInfo = PFS.ConstantPoolSlots.find(ID);
+  if (ConstantInfo == PFS.ConstantPoolSlots.end())
+    return error("use of undefined constant '%const." + Twine(ID) + "'");
+  lex();
+  // TODO: Parse offset and target flags.
+  Dest = MachineOperand::CreateCPI(ID, /*Offset=*/0);
+  return false;
+}
+
+bool MIParser::parseJumpTableIndexOperand(MachineOperand &Dest) {
+  assert(Token.is(MIToken::JumpTableIndex));
+  unsigned ID;
+  if (getUnsigned(ID))
+    return true;
+  auto JumpTableEntryInfo = PFS.JumpTableSlots.find(ID);
+  if (JumpTableEntryInfo == PFS.JumpTableSlots.end())
+    return error("use of undefined jump table '%jump-table." + Twine(ID) + "'");
+  lex();
+  // TODO: Parse target flags.
+  Dest = MachineOperand::CreateJTI(JumpTableEntryInfo->second);
+  return false;
+}
+
+bool MIParser::parseExternalSymbolOperand(MachineOperand &Dest) {
+  assert(Token.is(MIToken::ExternalSymbol) ||
+         Token.is(MIToken::QuotedExternalSymbol));
+  StringValueUtility Name(Token);
+  const char *Symbol = MF.createExternalSymbolName(Name);
+  lex();
+  // TODO: Parse the target flags.
+  Dest = MachineOperand::CreateES(Symbol);
+  return false;
+}
+
+bool MIParser::parseMDNode(MDNode *&Node) {
+  assert(Token.is(MIToken::exclaim));
+  auto Loc = Token.location();
+  lex();
+  if (Token.isNot(MIToken::IntegerLiteral) || Token.integerValue().isSigned())
+    return error("expected metadata id after '!'");
+  unsigned ID;
+  if (getUnsigned(ID))
+    return true;
+  auto NodeInfo = IRSlots.MetadataNodes.find(ID);
+  if (NodeInfo == IRSlots.MetadataNodes.end())
+    return error(Loc, "use of undefined metadata '!" + Twine(ID) + "'");
+  lex();
+  Node = NodeInfo->second.get();
+  return false;
+}
+
+bool MIParser::parseMetadataOperand(MachineOperand &Dest) {
+  MDNode *Node = nullptr;
+  if (parseMDNode(Node))
+    return true;
+  Dest = MachineOperand::CreateMetadata(Node);
+  return false;
+}
+
+bool MIParser::parseCFIOffset(int &Offset) {
+  if (Token.isNot(MIToken::IntegerLiteral))
+    return error("expected a cfi offset");
+  if (Token.integerValue().getMinSignedBits() > 32)
+    return error("expected a 32 bit integer (the cfi offset is too large)");
+  Offset = (int)Token.integerValue().getExtValue();
+  lex();
+  return false;
+}
+
+bool MIParser::parseCFIRegister(unsigned &Reg) {
+  if (Token.isNot(MIToken::NamedRegister))
+    return error("expected a cfi register");
+  unsigned LLVMReg;
+  if (parseRegister(LLVMReg))
+    return true;
+  const auto *TRI = MF.getSubtarget().getRegisterInfo();
+  assert(TRI && "Expected target register info");
+  int DwarfReg = TRI->getDwarfRegNum(LLVMReg, true);
+  if (DwarfReg < 0)
+    return error("invalid DWARF register");
+  Reg = (unsigned)DwarfReg;
+  lex();
+  return false;
+}
+
+bool MIParser::parseCFIOperand(MachineOperand &Dest) {
+  auto Kind = Token.kind();
+  lex();
+  auto &MMI = MF.getMMI();
+  int Offset;
+  unsigned Reg;
+  unsigned CFIIndex;
+  switch (Kind) {
+  case MIToken::kw_cfi_offset:
+    if (parseCFIRegister(Reg) || expectAndConsume(MIToken::comma) ||
+        parseCFIOffset(Offset))
+      return true;
+    CFIIndex =
+        MMI.addFrameInst(MCCFIInstruction::createOffset(nullptr, Reg, Offset));
+    break;
+  case MIToken::kw_cfi_def_cfa_register:
+    if (parseCFIRegister(Reg))
+      return true;
+    CFIIndex =
+        MMI.addFrameInst(MCCFIInstruction::createDefCfaRegister(nullptr, Reg));
+    break;
+  case MIToken::kw_cfi_def_cfa_offset:
+    if (parseCFIOffset(Offset))
+      return true;
+    // NB: MCCFIInstruction::createDefCfaOffset negates the offset.
+    CFIIndex = MMI.addFrameInst(
+        MCCFIInstruction::createDefCfaOffset(nullptr, -Offset));
+    break;
+  case MIToken::kw_cfi_def_cfa:
+    if (parseCFIRegister(Reg) || expectAndConsume(MIToken::comma) ||
+        parseCFIOffset(Offset))
+      return true;
+    // NB: MCCFIInstruction::createDefCfa negates the offset.
+    CFIIndex =
+        MMI.addFrameInst(MCCFIInstruction::createDefCfa(nullptr, Reg, -Offset));
+    break;
+  default:
+    // TODO: Parse the other CFI operands.
+    llvm_unreachable("The current token should be a cfi operand");
+  }
+  Dest = MachineOperand::CreateCFIIndex(CFIIndex);
+  return false;
+}
+
+bool MIParser::parseIRBlock(BasicBlock *&BB, const Function &F) {
+  switch (Token.kind()) {
+  case MIToken::NamedIRBlock:
+  case MIToken::QuotedNamedIRBlock: {
+    StringValueUtility Name(Token);
+    BB = dyn_cast_or_null<BasicBlock>(F.getValueSymbolTable().lookup(Name));
+    if (!BB)
+      return error(Twine("use of undefined IR block '%ir-block.") +
+                   Token.rawStringValue() + "'");
+    break;
+  }
+  case MIToken::IRBlock: {
+    unsigned SlotNumber = 0;
+    if (getUnsigned(SlotNumber))
+      return true;
+    BB = const_cast<BasicBlock *>(getIRBlock(SlotNumber));
+    if (!BB)
+      return error(Twine("use of undefined IR block '%ir-block.") +
+                   Twine(SlotNumber) + "'");
+    break;
+  }
+  default:
+    llvm_unreachable("The current token should be an IR block reference");
+  }
+  return false;
+}
+
+bool MIParser::parseBlockAddressOperand(MachineOperand &Dest) {
+  assert(Token.is(MIToken::kw_blockaddress));
+  lex();
+  if (expectAndConsume(MIToken::lparen))
+    return true;
+  if (Token.isNot(MIToken::GlobalValue) &&
+      Token.isNot(MIToken::NamedGlobalValue) &&
+      Token.isNot(MIToken::QuotedNamedGlobalValue))
+    return error("expected a global value");
+  GlobalValue *GV = nullptr;
+  if (parseGlobalValue(GV))
+    return true;
+  auto *F = dyn_cast<Function>(GV);
+  if (!F)
+    return error("expected an IR function reference");
+  lex();
+  if (expectAndConsume(MIToken::comma))
+    return true;
+  BasicBlock *BB = nullptr;
+  if (Token.isNot(MIToken::IRBlock) && Token.isNot(MIToken::NamedIRBlock) &&
+      Token.isNot(MIToken::QuotedNamedIRBlock))
+    return error("expected an IR block reference");
+  if (parseIRBlock(BB, *F))
+    return true;
+  lex();
+  if (expectAndConsume(MIToken::rparen))
+    return true;
+  // TODO: parse offset and target flags.
+  Dest = MachineOperand::CreateBA(BlockAddress::get(F, BB), /*Offset=*/0);
+  return false;
+}
+
+bool MIParser::parseTargetIndexOperand(MachineOperand &Dest) {
+  assert(Token.is(MIToken::kw_target_index));
+  lex();
+  if (expectAndConsume(MIToken::lparen))
+    return true;
+  if (Token.isNot(MIToken::Identifier))
+    return error("expected the name of the target index");
+  int Index = 0;
+  if (getTargetIndex(Token.stringValue(), Index))
+    return error("use of undefined target index '" + Token.stringValue() + "'");
+  lex();
+  if (expectAndConsume(MIToken::rparen))
+    return true;
+  // TODO: Parse the offset and target flags.
+  Dest = MachineOperand::CreateTargetIndex(unsigned(Index), /*Offset=*/0);
+  return false;
+}
+
+bool MIParser::parseMachineOperand(MachineOperand &Dest) {
+  switch (Token.kind()) {
+  case MIToken::kw_implicit:
+  case MIToken::kw_implicit_define:
+  case MIToken::kw_dead:
+  case MIToken::kw_killed:
+  case MIToken::kw_undef:
+  case MIToken::underscore:
+  case MIToken::NamedRegister:
+  case MIToken::VirtualRegister:
+    return parseRegisterOperand(Dest);
+  case MIToken::IntegerLiteral:
+    return parseImmediateOperand(Dest);
+  case MIToken::MachineBasicBlock:
+    return parseMBBOperand(Dest);
+  case MIToken::StackObject:
+    return parseStackObjectOperand(Dest);
+  case MIToken::FixedStackObject:
+    return parseFixedStackObjectOperand(Dest);
+  case MIToken::GlobalValue:
+  case MIToken::NamedGlobalValue:
+  case MIToken::QuotedNamedGlobalValue:
+    return parseGlobalAddressOperand(Dest);
+  case MIToken::ConstantPoolItem:
+    return parseConstantPoolIndexOperand(Dest);
+  case MIToken::JumpTableIndex:
+    return parseJumpTableIndexOperand(Dest);
+  case MIToken::ExternalSymbol:
+  case MIToken::QuotedExternalSymbol:
+    return parseExternalSymbolOperand(Dest);
+  case MIToken::exclaim:
+    return parseMetadataOperand(Dest);
+  case MIToken::kw_cfi_offset:
+  case MIToken::kw_cfi_def_cfa_register:
+  case MIToken::kw_cfi_def_cfa_offset:
+  case MIToken::kw_cfi_def_cfa:
+    return parseCFIOperand(Dest);
+  case MIToken::kw_blockaddress:
+    return parseBlockAddressOperand(Dest);
+  case MIToken::kw_target_index:
+    return parseTargetIndexOperand(Dest);
+  case MIToken::Error:
+    return true;
+  case MIToken::Identifier:
+    if (const auto *RegMask = getRegMask(Token.stringValue())) {
+      Dest = MachineOperand::CreateRegMask(RegMask);
+      lex();
+      break;
+    }
+  // fallthrough
+  default:
+    // TODO: parse the other machine operands.
+    return error("expected a machine operand");
+  }
+  return false;
+}
+
+void MIParser::initNames2InstrOpCodes() {
+  if (!Names2InstrOpCodes.empty())
+    return;
+  const auto *TII = MF.getSubtarget().getInstrInfo();
+  assert(TII && "Expected target instruction info");
+  for (unsigned I = 0, E = TII->getNumOpcodes(); I < E; ++I)
+    Names2InstrOpCodes.insert(std::make_pair(StringRef(TII->getName(I)), I));
+}
+
+bool MIParser::parseInstrName(StringRef InstrName, unsigned &OpCode) {
+  initNames2InstrOpCodes();
+  auto InstrInfo = Names2InstrOpCodes.find(InstrName);
+  if (InstrInfo == Names2InstrOpCodes.end())
+    return true;
+  OpCode = InstrInfo->getValue();
+  return false;
+}
+
+void MIParser::initNames2Regs() {
+  if (!Names2Regs.empty())
+    return;
+  // The '%noreg' register is the register 0.
+  Names2Regs.insert(std::make_pair("noreg", 0));
+  const auto *TRI = MF.getSubtarget().getRegisterInfo();
+  assert(TRI && "Expected target register info");
+  for (unsigned I = 0, E = TRI->getNumRegs(); I < E; ++I) {
+    bool WasInserted =
+        Names2Regs.insert(std::make_pair(StringRef(TRI->getName(I)).lower(), I))
+            .second;
+    (void)WasInserted;
+    assert(WasInserted && "Expected registers to be unique case-insensitively");
+  }
+}
+
+bool MIParser::getRegisterByName(StringRef RegName, unsigned &Reg) {
+  initNames2Regs();
+  auto RegInfo = Names2Regs.find(RegName);
+  if (RegInfo == Names2Regs.end())
+    return true;
+  Reg = RegInfo->getValue();
+  return false;
+}
+
+void MIParser::initNames2RegMasks() {
+  if (!Names2RegMasks.empty())
+    return;
+  const auto *TRI = MF.getSubtarget().getRegisterInfo();
+  assert(TRI && "Expected target register info");
+  ArrayRef<const uint32_t *> RegMasks = TRI->getRegMasks();
+  ArrayRef<const char *> RegMaskNames = TRI->getRegMaskNames();
+  assert(RegMasks.size() == RegMaskNames.size());
+  for (size_t I = 0, E = RegMasks.size(); I < E; ++I)
+    Names2RegMasks.insert(
+        std::make_pair(StringRef(RegMaskNames[I]).lower(), RegMasks[I]));
+}
+
+const uint32_t *MIParser::getRegMask(StringRef Identifier) {
+  initNames2RegMasks();
+  auto RegMaskInfo = Names2RegMasks.find(Identifier);
+  if (RegMaskInfo == Names2RegMasks.end())
+    return nullptr;
+  return RegMaskInfo->getValue();
+}
+
+void MIParser::initNames2SubRegIndices() {
+  if (!Names2SubRegIndices.empty())
+    return;
+  const TargetRegisterInfo *TRI = MF.getSubtarget().getRegisterInfo();
+  for (unsigned I = 1, E = TRI->getNumSubRegIndices(); I < E; ++I)
+    Names2SubRegIndices.insert(
+        std::make_pair(StringRef(TRI->getSubRegIndexName(I)).lower(), I));
+}
+
+unsigned MIParser::getSubRegIndex(StringRef Name) {
+  initNames2SubRegIndices();
+  auto SubRegInfo = Names2SubRegIndices.find(Name);
+  if (SubRegInfo == Names2SubRegIndices.end())
+    return 0;
+  return SubRegInfo->getValue();
+}
+
+void MIParser::initSlots2BasicBlocks() {
+  if (!Slots2BasicBlocks.empty())
+    return;
+  const auto &F = *MF.getFunction();
+  ModuleSlotTracker MST(F.getParent());
+  MST.incorporateFunction(F);
+  for (auto &BB : F) {
+    if (BB.hasName())
+      continue;
+    int Slot = MST.getLocalSlot(&BB);
+    if (Slot == -1)
+      continue;
+    Slots2BasicBlocks.insert(std::make_pair(unsigned(Slot), &BB));
+  }
+}
+
+const BasicBlock *MIParser::getIRBlock(unsigned Slot) {
+  initSlots2BasicBlocks();
+  auto BlockInfo = Slots2BasicBlocks.find(Slot);
+  if (BlockInfo == Slots2BasicBlocks.end())
+    return nullptr;
+  return BlockInfo->second;
+}
+
+void MIParser::initNames2TargetIndices() {
+  if (!Names2TargetIndices.empty())
+    return;
+  const auto *TII = MF.getSubtarget().getInstrInfo();
+  assert(TII && "Expected target instruction info");
+  auto Indices = TII->getSerializableTargetIndices();
+  for (const auto &I : Indices)
+    Names2TargetIndices.insert(std::make_pair(StringRef(I.second), I.first));
+}
+
+bool MIParser::getTargetIndex(StringRef Name, int &Index) {
+  initNames2TargetIndices();
+  auto IndexInfo = Names2TargetIndices.find(Name);
+  if (IndexInfo == Names2TargetIndices.end())
+    return true;
+  Index = IndexInfo->second;
+  return false;
+}
+
+bool llvm::parseMachineInstr(MachineInstr *&MI, SourceMgr &SM,
+                             MachineFunction &MF, StringRef Src,
+                             const PerFunctionMIParsingState &PFS,
+                             const SlotMapping &IRSlots, SMDiagnostic &Error) {
+  return MIParser(SM, MF, Error, Src, PFS, IRSlots).parse(MI);
+}
+
+bool llvm::parseMBBReference(MachineBasicBlock *&MBB, SourceMgr &SM,
+                             MachineFunction &MF, StringRef Src,
+                             const PerFunctionMIParsingState &PFS,
+                             const SlotMapping &IRSlots, SMDiagnostic &Error) {
+  return MIParser(SM, MF, Error, Src, PFS, IRSlots).parseStandaloneMBB(MBB);
+}
+
+bool llvm::parseNamedRegisterReference(unsigned &Reg, SourceMgr &SM,
+                                       MachineFunction &MF, StringRef Src,
+                                       const PerFunctionMIParsingState &PFS,
+                                       const SlotMapping &IRSlots,
+                                       SMDiagnostic &Error) {
+  return MIParser(SM, MF, Error, Src, PFS, IRSlots)
+      .parseStandaloneNamedRegister(Reg);
+}
+
+bool llvm::parseVirtualRegisterReference(unsigned &Reg, SourceMgr &SM,
+                                         MachineFunction &MF, StringRef Src,
+                                         const PerFunctionMIParsingState &PFS,
+                                         const SlotMapping &IRSlots,
+                                         SMDiagnostic &Error) {
+  return MIParser(SM, MF, Error, Src, PFS, IRSlots)
+      .parseStandaloneVirtualRegister(Reg);
+}
+
+bool llvm::parseIRBlockReference(const BasicBlock *&BB, SourceMgr &SM,
+                                 MachineFunction &MF, StringRef Src,
+                                 const PerFunctionMIParsingState &PFS,
+                                 const SlotMapping &IRSlots,
+                                 SMDiagnostic &Error) {
+  return MIParser(SM, MF, Error, Src, PFS, IRSlots)
+      .parseStandaloneIRBlockReference(BB);
+}
diff --git a/lib/CodeGen/MIRParser/MIParser.h b/lib/CodeGen/MIRParser/MIParser.h
new file mode 100644
index 00000000000..92698f6e6d1
--- /dev/null
+++ b/lib/CodeGen/MIRParser/MIParser.h
@@ -0,0 +1,67 @@
+//===- MIParser.h - Machine Instructions Parser ---------------------------===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This file declares the function that parses the machine instructions.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIB_CODEGEN_MIRPARSER_MIPARSER_H
+#define LLVM_LIB_CODEGEN_MIRPARSER_MIPARSER_H
+
+#include "llvm/ADT/DenseMap.h"
+#include "llvm/ADT/StringRef.h"
+
+namespace llvm {
+
+class BasicBlock;
+class MachineBasicBlock;
+class MachineInstr;
+class MachineFunction;
+struct SlotMapping;
+class SMDiagnostic;
+class SourceMgr;
+
+struct PerFunctionMIParsingState {
+  DenseMap<unsigned, MachineBasicBlock *> MBBSlots;
+  DenseMap<unsigned, unsigned> VirtualRegisterSlots;
+  DenseMap<unsigned, int> FixedStackObjectSlots;
+  DenseMap<unsigned, int> StackObjectSlots;
+  DenseMap<unsigned, unsigned> ConstantPoolSlots;
+  DenseMap<unsigned, unsigned> JumpTableSlots;
+};
+
+bool parseMachineInstr(MachineInstr *&MI, SourceMgr &SM, MachineFunction &MF,
+                       StringRef Src, const PerFunctionMIParsingState &PFS,
+                       const SlotMapping &IRSlots, SMDiagnostic &Error);
+
+bool parseMBBReference(MachineBasicBlock *&MBB, SourceMgr &SM,
+                       MachineFunction &MF, StringRef Src,
+                       const PerFunctionMIParsingState &PFS,
+                       const SlotMapping &IRSlots, SMDiagnostic &Error);
+
+bool parseNamedRegisterReference(unsigned &Reg, SourceMgr &SM,
+                                 MachineFunction &MF, StringRef Src,
+                                 const PerFunctionMIParsingState &PFS,
+                                 const SlotMapping &IRSlots,
+                                 SMDiagnostic &Error);
+
+bool parseVirtualRegisterReference(unsigned &Reg, SourceMgr &SM,
+                                   MachineFunction &MF, StringRef Src,
+                                   const PerFunctionMIParsingState &PFS,
+                                   const SlotMapping &IRSlots,
+                                   SMDiagnostic &Error);
+
+bool parseIRBlockReference(const BasicBlock *&BB, SourceMgr &SM,
+                           MachineFunction &MF, StringRef Src,
+                           const PerFunctionMIParsingState &PFS,
+                           const SlotMapping &IRSlots, SMDiagnostic &Error);
+
+} // end namespace llvm
+
+#endif
diff --git a/lib/CodeGen/MIRParser/MIRParser.cpp b/lib/CodeGen/MIRParser/MIRParser.cpp
new file mode 100644
index 00000000000..1bf4684e924
--- /dev/null
+++ b/lib/CodeGen/MIRParser/MIRParser.cpp
@@ -0,0 +1,698 @@
+//===- MIRParser.cpp - MIR serialization format parser implementation -----===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This file implements the class that parses the optional LLVM IR and machine
+// functions that are stored in MIR files.
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/CodeGen/MIRParser/MIRParser.h"
+#include "MIParser.h"
+#include "llvm/ADT/DenseMap.h"
+#include "llvm/ADT/StringRef.h"
+#include "llvm/ADT/StringMap.h"
+#include "llvm/ADT/STLExtras.h"
+#include "llvm/AsmParser/Parser.h"
+#include "llvm/AsmParser/SlotMapping.h"
+#include "llvm/CodeGen/MachineConstantPool.h"
+#include "llvm/CodeGen/MachineFunction.h"
+#include "llvm/CodeGen/MachineFrameInfo.h"
+#include "llvm/CodeGen/MachineRegisterInfo.h"
+#include "llvm/CodeGen/MIRYamlMapping.h"
+#include "llvm/IR/BasicBlock.h"
+#include "llvm/IR/DiagnosticInfo.h"
+#include "llvm/IR/Instructions.h"
+#include "llvm/IR/LLVMContext.h"
+#include "llvm/IR/Module.h"
+#include "llvm/IR/ValueSymbolTable.h"
+#include "llvm/Support/LineIterator.h"
+#include "llvm/Support/SMLoc.h"
+#include "llvm/Support/SourceMgr.h"
+#include "llvm/Support/MemoryBuffer.h"
+#include "llvm/Support/YAMLTraits.h"
+#include <memory>
+
+using namespace llvm;
+
+namespace llvm {
+
+/// This class implements the parsing of LLVM IR that's embedded inside a MIR
+/// file.
+class MIRParserImpl {
+  SourceMgr SM;
+  StringRef Filename;
+  LLVMContext &Context;
+  StringMap<std::unique_ptr<yaml::MachineFunction>> Functions;
+  SlotMapping IRSlots;
+  /// Maps from register class names to register classes.
+  StringMap<const TargetRegisterClass *> Names2RegClasses;
+
+public:
+  MIRParserImpl(std::unique_ptr<MemoryBuffer> Contents, StringRef Filename,
+                LLVMContext &Context);
+
+  void reportDiagnostic(const SMDiagnostic &Diag);
+
+  /// Report an error with the given message at unknown location.
+  ///
+  /// Always returns true.
+  bool error(const Twine &Message);
+
+  /// Report an error with the given message at the given location.
+  ///
+  /// Always returns true.
+  bool error(SMLoc Loc, const Twine &Message);
+
+  /// Report a given error with the location translated from the location in an
+  /// embedded string literal to a location in the MIR file.
+  ///
+  /// Always returns true.
+  bool error(const SMDiagnostic &Error, SMRange SourceRange);
+
+  /// Try to parse the optional LLVM module and the machine functions in the MIR
+  /// file.
+  ///
+  /// Return null if an error occurred.
+  std::unique_ptr<Module> parse();
+
+  /// Parse the machine function in the current YAML document.
+  ///
+  /// \param NoLLVMIR - set to true when the MIR file doesn't have LLVM IR.
+  /// A dummy IR function is created and inserted into the given module when
+  /// this parameter is true.
+  ///
+  /// Return true if an error occurred.
+  bool parseMachineFunction(yaml::Input &In, Module &M, bool NoLLVMIR);
+
+  /// Initialize the machine function to the state that's described in the MIR
+  /// file.
+  ///
+  /// Return true if error occurred.
+  bool initializeMachineFunction(MachineFunction &MF);
+
+  /// Initialize the machine basic block using it's YAML representation.
+  ///
+  /// Return true if an error occurred.
+  bool initializeMachineBasicBlock(MachineFunction &MF, MachineBasicBlock &MBB,
+                                   const yaml::MachineBasicBlock &YamlMBB,
+                                   const PerFunctionMIParsingState &PFS);
+
+  bool initializeRegisterInfo(MachineFunction &MF,
+                              const yaml::MachineFunction &YamlMF,
+                              PerFunctionMIParsingState &PFS);
+
+  bool initializeFrameInfo(MachineFunction &MF,
+                           const yaml::MachineFunction &YamlMF,
+                           PerFunctionMIParsingState &PFS);
+
+  bool parseCalleeSavedRegister(MachineFunction &MF,
+                                PerFunctionMIParsingState &PFS,
+                                std::vector<CalleeSavedInfo> &CSIInfo,
+                                const yaml::StringValue &RegisterSource,
+                                int FrameIdx);
+
+  bool initializeConstantPool(MachineConstantPool &ConstantPool,
+                              const yaml::MachineFunction &YamlMF,
+                              const MachineFunction &MF,
+                              DenseMap<unsigned, unsigned> &ConstantPoolSlots);
+
+  bool initializeJumpTableInfo(MachineFunction &MF,
+                               const yaml::MachineJumpTable &YamlJTI,
+                               PerFunctionMIParsingState &PFS);
+
+private:
+  bool parseMBBReference(MachineBasicBlock *&MBB,
+                         const yaml::StringValue &Source, MachineFunction &MF,
+                         const PerFunctionMIParsingState &PFS);
+
+  /// Return a MIR diagnostic converted from an MI string diagnostic.
+  SMDiagnostic diagFromMIStringDiag(const SMDiagnostic &Error,
+                                    SMRange SourceRange);
+
+  /// Return a MIR diagnostic converted from an LLVM assembly diagnostic.
+  SMDiagnostic diagFromLLVMAssemblyDiag(const SMDiagnostic &Error,
+                                        SMRange SourceRange);
+
+  /// Create an empty function with the given name.
+  void createDummyFunction(StringRef Name, Module &M);
+
+  void initNames2RegClasses(const MachineFunction &MF);
+
+  /// Check if the given identifier is a name of a register class.
+  ///
+  /// Return null if the name isn't a register class.
+  const TargetRegisterClass *getRegClass(const MachineFunction &MF,
+                                         StringRef Name);
+};
+
+} // end namespace llvm
+
+MIRParserImpl::MIRParserImpl(std::unique_ptr<MemoryBuffer> Contents,
+                             StringRef Filename, LLVMContext &Context)
+    : SM(), Filename(Filename), Context(Context) {
+  SM.AddNewSourceBuffer(std::move(Contents), SMLoc());
+}
+
+bool MIRParserImpl::error(const Twine &Message) {
+  Context.diagnose(DiagnosticInfoMIRParser(
+      DS_Error, SMDiagnostic(Filename, SourceMgr::DK_Error, Message.str())));
+  return true;
+}
+
+bool MIRParserImpl::error(SMLoc Loc, const Twine &Message) {
+  Context.diagnose(DiagnosticInfoMIRParser(
+      DS_Error, SM.GetMessage(Loc, SourceMgr::DK_Error, Message)));
+  return true;
+}
+
+bool MIRParserImpl::error(const SMDiagnostic &Error, SMRange SourceRange) {
+  assert(Error.getKind() == SourceMgr::DK_Error && "Expected an error");
+  reportDiagnostic(diagFromMIStringDiag(Error, SourceRange));
+  return true;
+}
+
+void MIRParserImpl::reportDiagnostic(const SMDiagnostic &Diag) {
+  DiagnosticSeverity Kind;
+  switch (Diag.getKind()) {
+  case SourceMgr::DK_Error:
+    Kind = DS_Error;
+    break;
+  case SourceMgr::DK_Warning:
+    Kind = DS_Warning;
+    break;
+  case SourceMgr::DK_Note:
+    Kind = DS_Note;
+    break;
+  }
+  Context.diagnose(DiagnosticInfoMIRParser(Kind, Diag));
+}
+
+static void handleYAMLDiag(const SMDiagnostic &Diag, void *Context) {
+  reinterpret_cast<MIRParserImpl *>(Context)->reportDiagnostic(Diag);
+}
+
+std::unique_ptr<Module> MIRParserImpl::parse() {
+  yaml::Input In(SM.getMemoryBuffer(SM.getMainFileID())->getBuffer(),
+                 /*Ctxt=*/nullptr, handleYAMLDiag, this);
+  In.setContext(&In);
+
+  if (!In.setCurrentDocument()) {
+    if (In.error())
+      return nullptr;
+    // Create an empty module when the MIR file is empty.
+    return llvm::make_unique<Module>(Filename, Context);
+  }
+
+  std::unique_ptr<Module> M;
+  bool NoLLVMIR = false;
+  // Parse the block scalar manually so that we can return unique pointer
+  // without having to go trough YAML traits.
+  if (const auto *BSN =
+          dyn_cast_or_null<yaml::BlockScalarNode>(In.getCurrentNode())) {
+    SMDiagnostic Error;
+    M = parseAssembly(MemoryBufferRef(BSN->getValue(), Filename), Error,
+                      Context, &IRSlots);
+    if (!M) {
+      reportDiagnostic(diagFromLLVMAssemblyDiag(Error, BSN->getSourceRange()));
+      return M;
+    }
+    In.nextDocument();
+    if (!In.setCurrentDocument())
+      return M;
+  } else {
+    // Create an new, empty module.
+    M = llvm::make_unique<Module>(Filename, Context);
+    NoLLVMIR = true;
+  }
+
+  // Parse the machine functions.
+  do {
+    if (parseMachineFunction(In, *M, NoLLVMIR))
+      return nullptr;
+    In.nextDocument();
+  } while (In.setCurrentDocument());
+
+  return M;
+}
+
+bool MIRParserImpl::parseMachineFunction(yaml::Input &In, Module &M,
+                                         bool NoLLVMIR) {
+  auto MF = llvm::make_unique<yaml::MachineFunction>();
+  yaml::yamlize(In, *MF, false);
+  if (In.error())
+    return true;
+  auto FunctionName = MF->Name;
+  if (Functions.find(FunctionName) != Functions.end())
+    return error(Twine("redefinition of machine function '") + FunctionName +
+                 "'");
+  Functions.insert(std::make_pair(FunctionName, std::move(MF)));
+  if (NoLLVMIR)
+    createDummyFunction(FunctionName, M);
+  else if (!M.getFunction(FunctionName))
+    return error(Twine("function '") + FunctionName +
+                 "' isn't defined in the provided LLVM IR");
+  return false;
+}
+
+void MIRParserImpl::createDummyFunction(StringRef Name, Module &M) {
+  auto &Context = M.getContext();
+  Function *F = cast<Function>(M.getOrInsertFunction(
+      Name, FunctionType::get(Type::getVoidTy(Context), false)));
+  BasicBlock *BB = BasicBlock::Create(Context, "entry", F);
+  new UnreachableInst(Context, BB);
+}
+
+bool MIRParserImpl::initializeMachineFunction(MachineFunction &MF) {
+  auto It = Functions.find(MF.getName());
+  if (It == Functions.end())
+    return error(Twine("no machine function information for function '") +
+                 MF.getName() + "' in the MIR file");
+  // TODO: Recreate the machine function.
+  const yaml::MachineFunction &YamlMF = *It->getValue();
+  if (YamlMF.Alignment)
+    MF.setAlignment(YamlMF.Alignment);
+  MF.setExposesReturnsTwice(YamlMF.ExposesReturnsTwice);
+  MF.setHasInlineAsm(YamlMF.HasInlineAsm);
+  PerFunctionMIParsingState PFS;
+  if (initializeRegisterInfo(MF, YamlMF, PFS))
+    return true;
+  if (!YamlMF.Constants.empty()) {
+    auto *ConstantPool = MF.getConstantPool();
+    assert(ConstantPool && "Constant pool must be created");
+    if (initializeConstantPool(*ConstantPool, YamlMF, MF,
+                               PFS.ConstantPoolSlots))
+      return true;
+  }
+
+  const auto &F = *MF.getFunction();
+  for (const auto &YamlMBB : YamlMF.BasicBlocks) {
+    const BasicBlock *BB = nullptr;
+    const yaml::StringValue &Name = YamlMBB.Name;
+    const yaml::StringValue &IRBlock = YamlMBB.IRBlock;
+    if (!Name.Value.empty()) {
+      BB = dyn_cast_or_null<BasicBlock>(
+          F.getValueSymbolTable().lookup(Name.Value));
+      if (!BB)
+        return error(Name.SourceRange.Start,
+                     Twine("basic block '") + Name.Value +
+                         "' is not defined in the function '" + MF.getName() +
+                         "'");
+    }
+    if (!IRBlock.Value.empty()) {
+      // TODO: Report an error when both name and ir block are specified.
+      SMDiagnostic Error;
+      if (parseIRBlockReference(BB, SM, MF, IRBlock.Value, PFS, IRSlots, Error))
+        return error(Error, IRBlock.SourceRange);
+    }
+    auto *MBB = MF.CreateMachineBasicBlock(BB);
+    MF.insert(MF.end(), MBB);
+    bool WasInserted =
+        PFS.MBBSlots.insert(std::make_pair(YamlMBB.ID, MBB)).second;
+    if (!WasInserted)
+      return error(Twine("redefinition of machine basic block with id #") +
+                   Twine(YamlMBB.ID));
+  }
+
+  if (YamlMF.BasicBlocks.empty())
+    return error(Twine("machine function '") + Twine(MF.getName()) +
+                 "' requires at least one machine basic block in its body");
+  // Initialize the frame information after creating all the MBBs so that the
+  // MBB references in the frame information can be resolved.
+  if (initializeFrameInfo(MF, YamlMF, PFS))
+    return true;
+  // Initialize the jump table after creating all the MBBs so that the MBB
+  // references can be resolved.
+  if (!YamlMF.JumpTableInfo.Entries.empty() &&
+      initializeJumpTableInfo(MF, YamlMF.JumpTableInfo, PFS))
+    return true;
+  // Initialize the machine basic blocks after creating them all so that the
+  // machine instructions parser can resolve the MBB references.
+  unsigned I = 0;
+  for (const auto &YamlMBB : YamlMF.BasicBlocks) {
+    if (initializeMachineBasicBlock(MF, *MF.getBlockNumbered(I++), YamlMBB,
+                                    PFS))
+      return true;
+  }
+  // FIXME: This is a temporary workaround until the reserved registers can be
+  // serialized.
+  MF.getRegInfo().freezeReservedRegs(MF);
+  MF.verify();
+  return false;
+}
+
+bool MIRParserImpl::initializeMachineBasicBlock(
+    MachineFunction &MF, MachineBasicBlock &MBB,
+    const yaml::MachineBasicBlock &YamlMBB,
+    const PerFunctionMIParsingState &PFS) {
+  MBB.setAlignment(YamlMBB.Alignment);
+  if (YamlMBB.AddressTaken)
+    MBB.setHasAddressTaken();
+  MBB.setIsLandingPad(YamlMBB.IsLandingPad);
+  SMDiagnostic Error;
+  // Parse the successors.
+  const auto &Weights = YamlMBB.SuccessorWeights;
+  bool HasWeights = !Weights.empty();
+  if (HasWeights && Weights.size() != YamlMBB.Successors.size()) {
+    bool IsFew = Weights.size() < YamlMBB.Successors.size();
+    return error(IsFew ? Weights.back().SourceRange.End
+                       : Weights[YamlMBB.Successors.size()].SourceRange.Start,
+                 Twine("too ") + (IsFew ? "few" : "many") +
+                     " successor weights, expected " +
+                     Twine(YamlMBB.Successors.size()) + ", have " +
+                     Twine(Weights.size()));
+  }
+  size_t SuccessorIndex = 0;
+  for (const auto &MBBSource : YamlMBB.Successors) {
+    MachineBasicBlock *SuccMBB = nullptr;
+    if (parseMBBReference(SuccMBB, MBBSource, MF, PFS))
+      return true;
+    // TODO: Report an error when adding the same successor more than once.
+    MBB.addSuccessor(SuccMBB, HasWeights ? Weights[SuccessorIndex++].Value : 0);
+  }
+  // Parse the liveins.
+  for (const auto &LiveInSource : YamlMBB.LiveIns) {
+    unsigned Reg = 0;
+    if (parseNamedRegisterReference(Reg, SM, MF, LiveInSource.Value, PFS,
+                                    IRSlots, Error))
+      return error(Error, LiveInSource.SourceRange);
+    MBB.addLiveIn(Reg);
+  }
+  // Parse the instructions.
+  for (const auto &MISource : YamlMBB.Instructions) {
+    MachineInstr *MI = nullptr;
+    if (parseMachineInstr(MI, SM, MF, MISource.Value, PFS, IRSlots, Error))
+      return error(Error, MISource.SourceRange);
+    MBB.insert(MBB.end(), MI);
+  }
+  return false;
+}
+
+bool MIRParserImpl::initializeRegisterInfo(MachineFunction &MF,
+                                           const yaml::MachineFunction &YamlMF,
+                                           PerFunctionMIParsingState &PFS) {
+  MachineRegisterInfo &RegInfo = MF.getRegInfo();
+  assert(RegInfo.isSSA());
+  if (!YamlMF.IsSSA)
+    RegInfo.leaveSSA();
+  assert(RegInfo.tracksLiveness());
+  if (!YamlMF.TracksRegLiveness)
+    RegInfo.invalidateLiveness();
+  RegInfo.enableSubRegLiveness(YamlMF.TracksSubRegLiveness);
+
+  SMDiagnostic Error;
+  // Parse the virtual register information.
+  for (const auto &VReg : YamlMF.VirtualRegisters) {
+    const auto *RC = getRegClass(MF, VReg.Class.Value);
+    if (!RC)
+      return error(VReg.Class.SourceRange.Start,
+                   Twine("use of undefined register class '") +
+                       VReg.Class.Value + "'");
+    unsigned Reg = RegInfo.createVirtualRegister(RC);
+    if (!PFS.VirtualRegisterSlots.insert(std::make_pair(VReg.ID.Value, Reg))
+             .second)
+      return error(VReg.ID.SourceRange.Start,
+                   Twine("redefinition of virtual register '%") +
+                       Twine(VReg.ID.Value) + "'");
+    if (!VReg.PreferredRegister.Value.empty()) {
+      unsigned PreferredReg = 0;
+      if (parseNamedRegisterReference(PreferredReg, SM, MF,
+                                      VReg.PreferredRegister.Value, PFS,
+                                      IRSlots, Error))
+        return error(Error, VReg.PreferredRegister.SourceRange);
+      RegInfo.setSimpleHint(Reg, PreferredReg);
+    }
+  }
+
+  // Parse the liveins.
+  for (const auto &LiveIn : YamlMF.LiveIns) {
+    unsigned Reg = 0;
+    if (parseNamedRegisterReference(Reg, SM, MF, LiveIn.Register.Value, PFS,
+                                    IRSlots, Error))
+      return error(Error, LiveIn.Register.SourceRange);
+    unsigned VReg = 0;
+    if (!LiveIn.VirtualRegister.Value.empty()) {
+      if (parseVirtualRegisterReference(
+              VReg, SM, MF, LiveIn.VirtualRegister.Value, PFS, IRSlots, Error))
+        return error(Error, LiveIn.VirtualRegister.SourceRange);
+    }
+    RegInfo.addLiveIn(Reg, VReg);
+  }
+  return false;
+}
+
+bool MIRParserImpl::initializeFrameInfo(MachineFunction &MF,
+                                        const yaml::MachineFunction &YamlMF,
+                                        PerFunctionMIParsingState &PFS) {
+  MachineFrameInfo &MFI = *MF.getFrameInfo();
+  const Function &F = *MF.getFunction();
+  const yaml::MachineFrameInfo &YamlMFI = YamlMF.FrameInfo;
+  MFI.setFrameAddressIsTaken(YamlMFI.IsFrameAddressTaken);
+  MFI.setReturnAddressIsTaken(YamlMFI.IsReturnAddressTaken);
+  MFI.setHasStackMap(YamlMFI.HasStackMap);
+  MFI.setHasPatchPoint(YamlMFI.HasPatchPoint);
+  MFI.setStackSize(YamlMFI.StackSize);
+  MFI.setOffsetAdjustment(YamlMFI.OffsetAdjustment);
+  if (YamlMFI.MaxAlignment)
+    MFI.ensureMaxAlignment(YamlMFI.MaxAlignment);
+  MFI.setAdjustsStack(YamlMFI.AdjustsStack);
+  MFI.setHasCalls(YamlMFI.HasCalls);
+  MFI.setMaxCallFrameSize(YamlMFI.MaxCallFrameSize);
+  MFI.setHasOpaqueSPAdjustment(YamlMFI.HasOpaqueSPAdjustment);
+  MFI.setHasVAStart(YamlMFI.HasVAStart);
+  MFI.setHasMustTailInVarArgFunc(YamlMFI.HasMustTailInVarArgFunc);
+  if (!YamlMFI.SavePoint.Value.empty()) {
+    MachineBasicBlock *MBB = nullptr;
+    if (parseMBBReference(MBB, YamlMFI.SavePoint, MF, PFS))
+      return true;
+    MFI.setSavePoint(MBB);
+  }
+  if (!YamlMFI.RestorePoint.Value.empty()) {
+    MachineBasicBlock *MBB = nullptr;
+    if (parseMBBReference(MBB, YamlMFI.RestorePoint, MF, PFS))
+      return true;
+    MFI.setRestorePoint(MBB);
+  }
+
+  std::vector<CalleeSavedInfo> CSIInfo;
+  // Initialize the fixed frame objects.
+  for (const auto &Object : YamlMF.FixedStackObjects) {
+    int ObjectIdx;
+    if (Object.Type != yaml::FixedMachineStackObject::SpillSlot)
+      ObjectIdx = MFI.CreateFixedObject(Object.Size, Object.Offset,
+                                        Object.IsImmutable, Object.IsAliased);
+    else
+      ObjectIdx = MFI.CreateFixedSpillStackObject(Object.Size, Object.Offset);
+    MFI.setObjectAlignment(ObjectIdx, Object.Alignment);
+    // TODO: Report an error when objects are redefined.
+    PFS.FixedStackObjectSlots.insert(std::make_pair(Object.ID, ObjectIdx));
+    if (parseCalleeSavedRegister(MF, PFS, CSIInfo, Object.CalleeSavedRegister,
+                                 ObjectIdx))
+      return true;
+  }
+
+  // Initialize the ordinary frame objects.
+  for (const auto &Object : YamlMF.StackObjects) {
+    int ObjectIdx;
+    const AllocaInst *Alloca = nullptr;
+    const yaml::StringValue &Name = Object.Name;
+    if (!Name.Value.empty()) {
+      Alloca = dyn_cast_or_null<AllocaInst>(
+          F.getValueSymbolTable().lookup(Name.Value));
+      if (!Alloca)
+        return error(Name.SourceRange.Start,
+                     "alloca instruction named '" + Name.Value +
+                         "' isn't defined in the function '" + F.getName() +
+                         "'");
+    }
+    if (Object.Type == yaml::MachineStackObject::VariableSized)
+      ObjectIdx = MFI.CreateVariableSizedObject(Object.Alignment, Alloca);
+    else
+      ObjectIdx = MFI.CreateStackObject(
+          Object.Size, Object.Alignment,
+          Object.Type == yaml::MachineStackObject::SpillSlot, Alloca);
+    MFI.setObjectOffset(ObjectIdx, Object.Offset);
+    // TODO: Report an error when objects are redefined.
+    PFS.StackObjectSlots.insert(std::make_pair(Object.ID, ObjectIdx));
+    if (parseCalleeSavedRegister(MF, PFS, CSIInfo, Object.CalleeSavedRegister,
+                                 ObjectIdx))
+      return true;
+  }
+  MFI.setCalleeSavedInfo(CSIInfo);
+  if (!CSIInfo.empty())
+    MFI.setCalleeSavedInfoValid(true);
+  return false;
+}
+
+bool MIRParserImpl::parseCalleeSavedRegister(
+    MachineFunction &MF, PerFunctionMIParsingState &PFS,
+    std::vector<CalleeSavedInfo> &CSIInfo,
+    const yaml::StringValue &RegisterSource, int FrameIdx) {
+  if (RegisterSource.Value.empty())
+    return false;
+  unsigned Reg = 0;
+  SMDiagnostic Error;
+  if (parseNamedRegisterReference(Reg, SM, MF, RegisterSource.Value, PFS,
+                                  IRSlots, Error))
+    return error(Error, RegisterSource.SourceRange);
+  CSIInfo.push_back(CalleeSavedInfo(Reg, FrameIdx));
+  return false;
+}
+
+bool MIRParserImpl::initializeConstantPool(
+    MachineConstantPool &ConstantPool, const yaml::MachineFunction &YamlMF,
+    const MachineFunction &MF,
+    DenseMap<unsigned, unsigned> &ConstantPoolSlots) {
+  const auto &M = *MF.getFunction()->getParent();
+  SMDiagnostic Error;
+  for (const auto &YamlConstant : YamlMF.Constants) {
+    const Constant *Value = dyn_cast_or_null<Constant>(
+        parseConstantValue(YamlConstant.Value.Value, Error, M));
+    if (!Value)
+      return error(Error, YamlConstant.Value.SourceRange);
+    unsigned Alignment =
+        YamlConstant.Alignment
+            ? YamlConstant.Alignment
+            : M.getDataLayout().getPrefTypeAlignment(Value->getType());
+    unsigned Index = ConstantPool.getConstantPoolIndex(Value, Alignment);
+    if (!ConstantPoolSlots.insert(std::make_pair(YamlConstant.ID.Value, Index))
+             .second)
+      return error(YamlConstant.ID.SourceRange.Start,
+                   Twine("redefinition of constant pool item '%const.") +
+                       Twine(YamlConstant.ID.Value) + "'");
+  }
+  return false;
+}
+
+bool MIRParserImpl::initializeJumpTableInfo(
+    MachineFunction &MF, const yaml::MachineJumpTable &YamlJTI,
+    PerFunctionMIParsingState &PFS) {
+  MachineJumpTableInfo *JTI = MF.getOrCreateJumpTableInfo(YamlJTI.Kind);
+  SMDiagnostic Error;
+  for (const auto &Entry : YamlJTI.Entries) {
+    std::vector<MachineBasicBlock *> Blocks;
+    for (const auto &MBBSource : Entry.Blocks) {
+      MachineBasicBlock *MBB = nullptr;
+      if (parseMBBReference(MBB, MBBSource.Value, MF, PFS))
+        return true;
+      Blocks.push_back(MBB);
+    }
+    unsigned Index = JTI->createJumpTableIndex(Blocks);
+    // TODO: Report an error when the same jump table slot ID is redefined.
+    PFS.JumpTableSlots.insert(std::make_pair(Entry.ID, Index));
+  }
+  return false;
+}
+
+bool MIRParserImpl::parseMBBReference(MachineBasicBlock *&MBB,
+                                      const yaml::StringValue &Source,
+                                      MachineFunction &MF,
+                                      const PerFunctionMIParsingState &PFS) {
+  SMDiagnostic Error;
+  if (llvm::parseMBBReference(MBB, SM, MF, Source.Value, PFS, IRSlots, Error))
+    return error(Error, Source.SourceRange);
+  return false;
+}
+
+SMDiagnostic MIRParserImpl::diagFromMIStringDiag(const SMDiagnostic &Error,
+                                                 SMRange SourceRange) {
+  assert(SourceRange.isValid() && "Invalid source range");
+  SMLoc Loc = SourceRange.Start;
+  bool HasQuote = Loc.getPointer() < SourceRange.End.getPointer() &&
+                  *Loc.getPointer() == '\'';
+  // Translate the location of the error from the location in the MI string to
+  // the corresponding location in the MIR file.
+  Loc = Loc.getFromPointer(Loc.getPointer() + Error.getColumnNo() +
+                           (HasQuote ? 1 : 0));
+
+  // TODO: Translate any source ranges as well.
+  return SM.GetMessage(Loc, Error.getKind(), Error.getMessage(), None,
+                       Error.getFixIts());
+}
+
+SMDiagnostic MIRParserImpl::diagFromLLVMAssemblyDiag(const SMDiagnostic &Error,
+                                                     SMRange SourceRange) {
+  assert(SourceRange.isValid());
+
+  // Translate the location of the error from the location in the llvm IR string
+  // to the corresponding location in the MIR file.
+  auto LineAndColumn = SM.getLineAndColumn(SourceRange.Start);
+  unsigned Line = LineAndColumn.first + Error.getLineNo() - 1;
+  unsigned Column = Error.getColumnNo();
+  StringRef LineStr = Error.getLineContents();
+  SMLoc Loc = Error.getLoc();
+
+  // Get the full line and adjust the column number by taking the indentation of
+  // LLVM IR into account.
+  for (line_iterator L(*SM.getMemoryBuffer(SM.getMainFileID()), false), E;
+       L != E; ++L) {
+    if (L.line_number() == Line) {
+      LineStr = *L;
+      Loc = SMLoc::getFromPointer(LineStr.data());
+      auto Indent = LineStr.find(Error.getLineContents());
+      if (Indent != StringRef::npos)
+        Column += Indent;
+      break;
+    }
+  }
+
+  return SMDiagnostic(SM, Loc, Filename, Line, Column, Error.getKind(),
+                      Error.getMessage(), LineStr, Error.getRanges(),
+                      Error.getFixIts());
+}
+
+void MIRParserImpl::initNames2RegClasses(const MachineFunction &MF) {
+  if (!Names2RegClasses.empty())
+    return;
+  const TargetRegisterInfo *TRI = MF.getSubtarget().getRegisterInfo();
+  for (unsigned I = 0, E = TRI->getNumRegClasses(); I < E; ++I) {
+    const auto *RC = TRI->getRegClass(I);
+    Names2RegClasses.insert(
+        std::make_pair(StringRef(TRI->getRegClassName(RC)).lower(), RC));
+  }
+}
+
+const TargetRegisterClass *MIRParserImpl::getRegClass(const MachineFunction &MF,
+                                                      StringRef Name) {
+  initNames2RegClasses(MF);
+  auto RegClassInfo = Names2RegClasses.find(Name);
+  if (RegClassInfo == Names2RegClasses.end())
+    return nullptr;
+  return RegClassInfo->getValue();
+}
+
+MIRParser::MIRParser(std::unique_ptr<MIRParserImpl> Impl)
+    : Impl(std::move(Impl)) {}
+
+MIRParser::~MIRParser() {}
+
+std::unique_ptr<Module> MIRParser::parseLLVMModule() { return Impl->parse(); }
+
+bool MIRParser::initializeMachineFunction(MachineFunction &MF) {
+  return Impl->initializeMachineFunction(MF);
+}
+
+std::unique_ptr<MIRParser> llvm::createMIRParserFromFile(StringRef Filename,
+                                                         SMDiagnostic &Error,
+                                                         LLVMContext &Context) {
+  auto FileOrErr = MemoryBuffer::getFile(Filename);
+  if (std::error_code EC = FileOrErr.getError()) {
+    Error = SMDiagnostic(Filename, SourceMgr::DK_Error,
+                         "Could not open input file: " + EC.message());
+    return nullptr;
+  }
+  return createMIRParser(std::move(FileOrErr.get()), Context);
+}
+
+std::unique_ptr<MIRParser>
+llvm::createMIRParser(std::unique_ptr<MemoryBuffer> Contents,
+                      LLVMContext &Context) {
+  auto Filename = Contents->getBufferIdentifier();
+  return llvm::make_unique<MIRParser>(
+      llvm::make_unique<MIRParserImpl>(std::move(Contents), Filename, Context));
+}
diff --git a/lib/CodeGen/MIRParser/Makefile b/lib/CodeGen/MIRParser/Makefile
new file mode 100644
index 00000000000..c02d18806a9
--- /dev/null
+++ b/lib/CodeGen/MIRParser/Makefile
@@ -0,0 +1,13 @@
+##===- lib/CodeGen/MIRParser/Makefile ----------------------*- Makefile -*-===##
+#
+#                     The LLVM Compiler Infrastructure
+#
+# This file is distributed under the University of Illinois Open Source
+# License. See LICENSE.TXT for details.
+#
+##===----------------------------------------------------------------------===##
+
+LEVEL = ../../..
+LIBRARYNAME = LLVMMIRParser
+
+include $(LEVEL)/Makefile.common
diff --git a/lib/CodeGen/MIRPrinter.cpp b/lib/CodeGen/MIRPrinter.cpp
new file mode 100644
index 00000000000..f34cef7bcb4
--- /dev/null
+++ b/lib/CodeGen/MIRPrinter.cpp
@@ -0,0 +1,635 @@
+//===- MIRPrinter.cpp - MIR serialization format printer ------------------===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This file implements the class that prints out the LLVM IR and machine
+// functions using the MIR serialization format.
+//
+//===----------------------------------------------------------------------===//
+
+#include "MIRPrinter.h"
+#include "llvm/ADT/STLExtras.h"
+#include "llvm/CodeGen/MachineConstantPool.h"
+#include "llvm/CodeGen/MachineFunction.h"
+#include "llvm/CodeGen/MachineFrameInfo.h"
+#include "llvm/CodeGen/MachineModuleInfo.h"
+#include "llvm/CodeGen/MachineRegisterInfo.h"
+#include "llvm/CodeGen/MIRYamlMapping.h"
+#include "llvm/IR/BasicBlock.h"
+#include "llvm/IR/Constants.h"
+#include "llvm/IR/Instructions.h"
+#include "llvm/IR/IRPrintingPasses.h"
+#include "llvm/IR/Module.h"
+#include "llvm/IR/ModuleSlotTracker.h"
+#include "llvm/Support/MemoryBuffer.h"
+#include "llvm/Support/raw_ostream.h"
+#include "llvm/Support/YAMLTraits.h"
+#include "llvm/Target/TargetInstrInfo.h"
+#include "llvm/Target/TargetSubtargetInfo.h"
+
+using namespace llvm;
+
+namespace {
+
+/// This structure describes how to print out stack object references.
+struct FrameIndexOperand {
+  std::string Name;
+  unsigned ID;
+  bool IsFixed;
+
+  FrameIndexOperand(StringRef Name, unsigned ID, bool IsFixed)
+      : Name(Name.str()), ID(ID), IsFixed(IsFixed) {}
+
+  /// Return an ordinary stack object reference.
+  static FrameIndexOperand create(StringRef Name, unsigned ID) {
+    return FrameIndexOperand(Name, ID, /*IsFixed=*/false);
+  }
+
+  /// Return a fixed stack object reference.
+  static FrameIndexOperand createFixed(unsigned ID) {
+    return FrameIndexOperand("", ID, /*IsFixed=*/true);
+  }
+};
+
+} // end anonymous namespace
+
+namespace llvm {
+
+/// This class prints out the machine functions using the MIR serialization
+/// format.
+class MIRPrinter {
+  raw_ostream &OS;
+  DenseMap<const uint32_t *, unsigned> RegisterMaskIds;
+  /// Maps from stack object indices to operand indices which will be used when
+  /// printing frame index machine operands.
+  DenseMap<int, FrameIndexOperand> StackObjectOperandMapping;
+
+public:
+  MIRPrinter(raw_ostream &OS) : OS(OS) {}
+
+  void print(const MachineFunction &MF);
+
+  void convert(yaml::MachineFunction &MF, const MachineRegisterInfo &RegInfo,
+               const TargetRegisterInfo *TRI);
+  void convert(ModuleSlotTracker &MST, yaml::MachineFrameInfo &YamlMFI,
+               const MachineFrameInfo &MFI);
+  void convert(yaml::MachineFunction &MF,
+               const MachineConstantPool &ConstantPool);
+  void convert(ModuleSlotTracker &MST, yaml::MachineJumpTable &YamlJTI,
+               const MachineJumpTableInfo &JTI);
+  void convert(ModuleSlotTracker &MST, yaml::MachineBasicBlock &YamlMBB,
+               const MachineBasicBlock &MBB);
+  void convertStackObjects(yaml::MachineFunction &MF,
+                           const MachineFrameInfo &MFI,
+                           const TargetRegisterInfo *TRI);
+
+private:
+  void initRegisterMaskIds(const MachineFunction &MF);
+};
+
+} // end namespace llvm
+
+namespace {
+
+/// This class prints out the machine instructions using the MIR serialization
+/// format.
+class MIPrinter {
+  raw_ostream &OS;
+  ModuleSlotTracker &MST;
+  const DenseMap<const uint32_t *, unsigned> &RegisterMaskIds;
+  const DenseMap<int, FrameIndexOperand> &StackObjectOperandMapping;
+
+public:
+  MIPrinter(raw_ostream &OS, ModuleSlotTracker &MST,
+            const DenseMap<const uint32_t *, unsigned> &RegisterMaskIds,
+            const DenseMap<int, FrameIndexOperand> &StackObjectOperandMapping)
+      : OS(OS), MST(MST), RegisterMaskIds(RegisterMaskIds),
+        StackObjectOperandMapping(StackObjectOperandMapping) {}
+
+  void print(const MachineInstr &MI);
+  void printMBBReference(const MachineBasicBlock &MBB);
+  void printIRBlockReference(const BasicBlock &BB);
+  void printStackObjectReference(int FrameIndex);
+  void print(const MachineOperand &Op, const TargetRegisterInfo *TRI);
+
+  void print(const MCCFIInstruction &CFI, const TargetRegisterInfo *TRI);
+};
+
+} // end anonymous namespace
+
+namespace llvm {
+namespace yaml {
+
+/// This struct serializes the LLVM IR module.
+template <> struct BlockScalarTraits<Module> {
+  static void output(const Module &Mod, void *Ctxt, raw_ostream &OS) {
+    Mod.print(OS, nullptr);
+  }
+  static StringRef input(StringRef Str, void *Ctxt, Module &Mod) {
+    llvm_unreachable("LLVM Module is supposed to be parsed separately");
+    return "";
+  }
+};
+
+} // end namespace yaml
+} // end namespace llvm
+
+static void printReg(unsigned Reg, raw_ostream &OS,
+                     const TargetRegisterInfo *TRI) {
+  // TODO: Print Stack Slots.
+  if (!Reg)
+    OS << '_';
+  else if (TargetRegisterInfo::isVirtualRegister(Reg))
+    OS << '%' << TargetRegisterInfo::virtReg2Index(Reg);
+  else if (Reg < TRI->getNumRegs())
+    OS << '%' << StringRef(TRI->getName(Reg)).lower();
+  else
+    llvm_unreachable("Can't print this kind of register yet");
+}
+
+static void printReg(unsigned Reg, yaml::StringValue &Dest,
+                     const TargetRegisterInfo *TRI) {
+  raw_string_ostream OS(Dest.Value);
+  printReg(Reg, OS, TRI);
+}
+
+void MIRPrinter::print(const MachineFunction &MF) {
+  initRegisterMaskIds(MF);
+
+  yaml::MachineFunction YamlMF;
+  YamlMF.Name = MF.getName();
+  YamlMF.Alignment = MF.getAlignment();
+  YamlMF.ExposesReturnsTwice = MF.exposesReturnsTwice();
+  YamlMF.HasInlineAsm = MF.hasInlineAsm();
+  convert(YamlMF, MF.getRegInfo(), MF.getSubtarget().getRegisterInfo());
+  ModuleSlotTracker MST(MF.getFunction()->getParent());
+  MST.incorporateFunction(*MF.getFunction());
+  convert(MST, YamlMF.FrameInfo, *MF.getFrameInfo());
+  convertStackObjects(YamlMF, *MF.getFrameInfo(),
+                      MF.getSubtarget().getRegisterInfo());
+  if (const auto *ConstantPool = MF.getConstantPool())
+    convert(YamlMF, *ConstantPool);
+  if (const auto *JumpTableInfo = MF.getJumpTableInfo())
+    convert(MST, YamlMF.JumpTableInfo, *JumpTableInfo);
+  for (const auto &MBB : MF) {
+    yaml::MachineBasicBlock YamlMBB;
+    convert(MST, YamlMBB, MBB);
+    YamlMF.BasicBlocks.push_back(YamlMBB);
+  }
+  yaml::Output Out(OS);
+  Out << YamlMF;
+}
+
+void MIRPrinter::convert(yaml::MachineFunction &MF,
+                         const MachineRegisterInfo &RegInfo,
+                         const TargetRegisterInfo *TRI) {
+  MF.IsSSA = RegInfo.isSSA();
+  MF.TracksRegLiveness = RegInfo.tracksLiveness();
+  MF.TracksSubRegLiveness = RegInfo.subRegLivenessEnabled();
+
+  // Print the virtual register definitions.
+  for (unsigned I = 0, E = RegInfo.getNumVirtRegs(); I < E; ++I) {
+    unsigned Reg = TargetRegisterInfo::index2VirtReg(I);
+    yaml::VirtualRegisterDefinition VReg;
+    VReg.ID = I;
+    VReg.Class =
+        StringRef(TRI->getRegClassName(RegInfo.getRegClass(Reg))).lower();
+    unsigned PreferredReg = RegInfo.getSimpleHint(Reg);
+    if (PreferredReg)
+      printReg(PreferredReg, VReg.PreferredRegister, TRI);
+    MF.VirtualRegisters.push_back(VReg);
+  }
+
+  // Print the live ins.
+  for (auto I = RegInfo.livein_begin(), E = RegInfo.livein_end(); I != E; ++I) {
+    yaml::MachineFunctionLiveIn LiveIn;
+    printReg(I->first, LiveIn.Register, TRI);
+    if (I->second)
+      printReg(I->second, LiveIn.VirtualRegister, TRI);
+    MF.LiveIns.push_back(LiveIn);
+  }
+}
+
+void MIRPrinter::convert(ModuleSlotTracker &MST,
+                         yaml::MachineFrameInfo &YamlMFI,
+                         const MachineFrameInfo &MFI) {
+  YamlMFI.IsFrameAddressTaken = MFI.isFrameAddressTaken();
+  YamlMFI.IsReturnAddressTaken = MFI.isReturnAddressTaken();
+  YamlMFI.HasStackMap = MFI.hasStackMap();
+  YamlMFI.HasPatchPoint = MFI.hasPatchPoint();
+  YamlMFI.StackSize = MFI.getStackSize();
+  YamlMFI.OffsetAdjustment = MFI.getOffsetAdjustment();
+  YamlMFI.MaxAlignment = MFI.getMaxAlignment();
+  YamlMFI.AdjustsStack = MFI.adjustsStack();
+  YamlMFI.HasCalls = MFI.hasCalls();
+  YamlMFI.MaxCallFrameSize = MFI.getMaxCallFrameSize();
+  YamlMFI.HasOpaqueSPAdjustment = MFI.hasOpaqueSPAdjustment();
+  YamlMFI.HasVAStart = MFI.hasVAStart();
+  YamlMFI.HasMustTailInVarArgFunc = MFI.hasMustTailInVarArgFunc();
+  if (MFI.getSavePoint()) {
+    raw_string_ostream StrOS(YamlMFI.SavePoint.Value);
+    MIPrinter(StrOS, MST, RegisterMaskIds, StackObjectOperandMapping)
+        .printMBBReference(*MFI.getSavePoint());
+  }
+  if (MFI.getRestorePoint()) {
+    raw_string_ostream StrOS(YamlMFI.RestorePoint.Value);
+    MIPrinter(StrOS, MST, RegisterMaskIds, StackObjectOperandMapping)
+        .printMBBReference(*MFI.getRestorePoint());
+  }
+}
+
+void MIRPrinter::convertStackObjects(yaml::MachineFunction &MF,
+                                     const MachineFrameInfo &MFI,
+                                     const TargetRegisterInfo *TRI) {
+  // Process fixed stack objects.
+  unsigned ID = 0;
+  for (int I = MFI.getObjectIndexBegin(); I < 0; ++I) {
+    if (MFI.isDeadObjectIndex(I))
+      continue;
+
+    yaml::FixedMachineStackObject YamlObject;
+    YamlObject.ID = ID;
+    YamlObject.Type = MFI.isSpillSlotObjectIndex(I)
+                          ? yaml::FixedMachineStackObject::SpillSlot
+                          : yaml::FixedMachineStackObject::DefaultType;
+    YamlObject.Offset = MFI.getObjectOffset(I);
+    YamlObject.Size = MFI.getObjectSize(I);
+    YamlObject.Alignment = MFI.getObjectAlignment(I);
+    YamlObject.IsImmutable = MFI.isImmutableObjectIndex(I);
+    YamlObject.IsAliased = MFI.isAliasedObjectIndex(I);
+    MF.FixedStackObjects.push_back(YamlObject);
+    StackObjectOperandMapping.insert(
+        std::make_pair(I, FrameIndexOperand::createFixed(ID++)));
+  }
+
+  // Process ordinary stack objects.
+  ID = 0;
+  for (int I = 0, E = MFI.getObjectIndexEnd(); I < E; ++I) {
+    if (MFI.isDeadObjectIndex(I))
+      continue;
+
+    yaml::MachineStackObject YamlObject;
+    YamlObject.ID = ID;
+    if (const auto *Alloca = MFI.getObjectAllocation(I))
+      YamlObject.Name.Value =
+          Alloca->hasName() ? Alloca->getName() : "<unnamed alloca>";
+    YamlObject.Type = MFI.isSpillSlotObjectIndex(I)
+                          ? yaml::MachineStackObject::SpillSlot
+                          : MFI.isVariableSizedObjectIndex(I)
+                                ? yaml::MachineStackObject::VariableSized
+                                : yaml::MachineStackObject::DefaultType;
+    YamlObject.Offset = MFI.getObjectOffset(I);
+    YamlObject.Size = MFI.getObjectSize(I);
+    YamlObject.Alignment = MFI.getObjectAlignment(I);
+
+    MF.StackObjects.push_back(YamlObject);
+    StackObjectOperandMapping.insert(std::make_pair(
+        I, FrameIndexOperand::create(YamlObject.Name.Value, ID++)));
+  }
+
+  for (const auto &CSInfo : MFI.getCalleeSavedInfo()) {
+    yaml::StringValue Reg;
+    printReg(CSInfo.getReg(), Reg, TRI);
+    auto StackObjectInfo = StackObjectOperandMapping.find(CSInfo.getFrameIdx());
+    assert(StackObjectInfo != StackObjectOperandMapping.end() &&
+           "Invalid stack object index");
+    const FrameIndexOperand &StackObject = StackObjectInfo->second;
+    if (StackObject.IsFixed)
+      MF.FixedStackObjects[StackObject.ID].CalleeSavedRegister = Reg;
+    else
+      MF.StackObjects[StackObject.ID].CalleeSavedRegister = Reg;
+  }
+}
+
+void MIRPrinter::convert(yaml::MachineFunction &MF,
+                         const MachineConstantPool &ConstantPool) {
+  unsigned ID = 0;
+  for (const MachineConstantPoolEntry &Constant : ConstantPool.getConstants()) {
+    // TODO: Serialize target specific constant pool entries.
+    if (Constant.isMachineConstantPoolEntry())
+      llvm_unreachable("Can't print target specific constant pool entries yet");
+
+    yaml::MachineConstantPoolValue YamlConstant;
+    std::string Str;
+    raw_string_ostream StrOS(Str);
+    Constant.Val.ConstVal->printAsOperand(StrOS);
+    YamlConstant.ID = ID++;
+    YamlConstant.Value = StrOS.str();
+    YamlConstant.Alignment = Constant.getAlignment();
+    MF.Constants.push_back(YamlConstant);
+  }
+}
+
+void MIRPrinter::convert(ModuleSlotTracker &MST,
+                         yaml::MachineJumpTable &YamlJTI,
+                         const MachineJumpTableInfo &JTI) {
+  YamlJTI.Kind = JTI.getEntryKind();
+  unsigned ID = 0;
+  for (const auto &Table : JTI.getJumpTables()) {
+    std::string Str;
+    yaml::MachineJumpTable::Entry Entry;
+    Entry.ID = ID++;
+    for (const auto *MBB : Table.MBBs) {
+      raw_string_ostream StrOS(Str);
+      MIPrinter(StrOS, MST, RegisterMaskIds, StackObjectOperandMapping)
+          .printMBBReference(*MBB);
+      Entry.Blocks.push_back(StrOS.str());
+      Str.clear();
+    }
+    YamlJTI.Entries.push_back(Entry);
+  }
+}
+
+void MIRPrinter::convert(ModuleSlotTracker &MST,
+                         yaml::MachineBasicBlock &YamlMBB,
+                         const MachineBasicBlock &MBB) {
+  assert(MBB.getNumber() >= 0 && "Invalid MBB number");
+  YamlMBB.ID = (unsigned)MBB.getNumber();
+  if (const auto *BB = MBB.getBasicBlock()) {
+    if (BB->hasName()) {
+      YamlMBB.Name.Value = BB->getName();
+    } else {
+      int Slot = MST.getLocalSlot(BB);
+      if (Slot == -1)
+        YamlMBB.IRBlock.Value = "<badref>";
+      else
+        YamlMBB.IRBlock.Value = (Twine("%ir-block.") + Twine(Slot)).str();
+    }
+  }
+  YamlMBB.Alignment = MBB.getAlignment();
+  YamlMBB.AddressTaken = MBB.hasAddressTaken();
+  YamlMBB.IsLandingPad = MBB.isLandingPad();
+  for (const auto *SuccMBB : MBB.successors()) {
+    std::string Str;
+    raw_string_ostream StrOS(Str);
+    MIPrinter(StrOS, MST, RegisterMaskIds, StackObjectOperandMapping)
+        .printMBBReference(*SuccMBB);
+    YamlMBB.Successors.push_back(StrOS.str());
+  }
+  if (MBB.hasSuccessorWeights()) {
+    for (auto I = MBB.succ_begin(), E = MBB.succ_end(); I != E; ++I)
+      YamlMBB.SuccessorWeights.push_back(
+          yaml::UnsignedValue(MBB.getSuccWeight(I)));
+  }
+  // Print the live in registers.
+  const auto *TRI = MBB.getParent()->getSubtarget().getRegisterInfo();
+  assert(TRI && "Expected target register info");
+  for (auto I = MBB.livein_begin(), E = MBB.livein_end(); I != E; ++I) {
+    std::string Str;
+    raw_string_ostream StrOS(Str);
+    printReg(*I, StrOS, TRI);
+    YamlMBB.LiveIns.push_back(StrOS.str());
+  }
+  // Print the machine instructions.
+  YamlMBB.Instructions.reserve(MBB.size());
+  std::string Str;
+  for (const auto &MI : MBB) {
+    raw_string_ostream StrOS(Str);
+    MIPrinter(StrOS, MST, RegisterMaskIds, StackObjectOperandMapping).print(MI);
+    YamlMBB.Instructions.push_back(StrOS.str());
+    Str.clear();
+  }
+}
+
+void MIRPrinter::initRegisterMaskIds(const MachineFunction &MF) {
+  const auto *TRI = MF.getSubtarget().getRegisterInfo();
+  unsigned I = 0;
+  for (const uint32_t *Mask : TRI->getRegMasks())
+    RegisterMaskIds.insert(std::make_pair(Mask, I++));
+}
+
+void MIPrinter::print(const MachineInstr &MI) {
+  const auto &SubTarget = MI.getParent()->getParent()->getSubtarget();
+  const auto *TRI = SubTarget.getRegisterInfo();
+  assert(TRI && "Expected target register info");
+  const auto *TII = SubTarget.getInstrInfo();
+  assert(TII && "Expected target instruction info");
+  if (MI.isCFIInstruction())
+    assert(MI.getNumOperands() == 1 && "Expected 1 operand in CFI instruction");
+
+  unsigned I = 0, E = MI.getNumOperands();
+  for (; I < E && MI.getOperand(I).isReg() && MI.getOperand(I).isDef() &&
+         !MI.getOperand(I).isImplicit();
+       ++I) {
+    if (I)
+      OS << ", ";
+    print(MI.getOperand(I), TRI);
+  }
+
+  if (I)
+    OS << " = ";
+  if (MI.getFlag(MachineInstr::FrameSetup))
+    OS << "frame-setup ";
+  OS << TII->getName(MI.getOpcode());
+  // TODO: Print the bundling instruction flags, machine mem operands.
+  if (I < E)
+    OS << ' ';
+
+  bool NeedComma = false;
+  for (; I < E; ++I) {
+    if (NeedComma)
+      OS << ", ";
+    print(MI.getOperand(I), TRI);
+    NeedComma = true;
+  }
+
+  if (MI.getDebugLoc()) {
+    if (NeedComma)
+      OS << ',';
+    OS << " debug-location ";
+    MI.getDebugLoc()->printAsOperand(OS, MST);
+  }
+}
+
+void MIPrinter::printMBBReference(const MachineBasicBlock &MBB) {
+  OS << "%bb." << MBB.getNumber();
+  if (const auto *BB = MBB.getBasicBlock()) {
+    if (BB->hasName())
+      OS << '.' << BB->getName();
+  }
+}
+
+void MIPrinter::printIRBlockReference(const BasicBlock &BB) {
+  OS << "%ir-block.";
+  if (BB.hasName()) {
+    printLLVMNameWithoutPrefix(OS, BB.getName());
+    return;
+  }
+  int Slot = MST.getLocalSlot(&BB);
+  if (Slot == -1)
+    OS << "<badref>";
+  else
+    OS << Slot;
+}
+
+void MIPrinter::printStackObjectReference(int FrameIndex) {
+  auto ObjectInfo = StackObjectOperandMapping.find(FrameIndex);
+  assert(ObjectInfo != StackObjectOperandMapping.end() &&
+         "Invalid frame index");
+  const FrameIndexOperand &Operand = ObjectInfo->second;
+  if (Operand.IsFixed) {
+    OS << "%fixed-stack." << Operand.ID;
+    return;
+  }
+  OS << "%stack." << Operand.ID;
+  if (!Operand.Name.empty())
+    OS << '.' << Operand.Name;
+}
+
+static const char *getTargetIndexName(const MachineFunction &MF, int Index) {
+  const auto *TII = MF.getSubtarget().getInstrInfo();
+  assert(TII && "expected instruction info");
+  auto Indices = TII->getSerializableTargetIndices();
+  for (const auto &I : Indices) {
+    if (I.first == Index) {
+      return I.second;
+    }
+  }
+  return nullptr;
+}
+
+void MIPrinter::print(const MachineOperand &Op, const TargetRegisterInfo *TRI) {
+  switch (Op.getType()) {
+  case MachineOperand::MO_Register:
+    // TODO: Print the other register flags.
+    if (Op.isImplicit())
+      OS << (Op.isDef() ? "implicit-def " : "implicit ");
+    if (Op.isDead())
+      OS << "dead ";
+    if (Op.isKill())
+      OS << "killed ";
+    if (Op.isUndef())
+      OS << "undef ";
+    printReg(Op.getReg(), OS, TRI);
+    // Print the sub register.
+    if (Op.getSubReg() != 0)
+      OS << ':' << TRI->getSubRegIndexName(Op.getSubReg());
+    break;
+  case MachineOperand::MO_Immediate:
+    OS << Op.getImm();
+    break;
+  case MachineOperand::MO_MachineBasicBlock:
+    printMBBReference(*Op.getMBB());
+    break;
+  case MachineOperand::MO_FrameIndex:
+    printStackObjectReference(Op.getIndex());
+    break;
+  case MachineOperand::MO_ConstantPoolIndex:
+    OS << "%const." << Op.getIndex();
+    // TODO: Print offset and target flags.
+    break;
+  case MachineOperand::MO_TargetIndex: {
+    OS << "target-index(";
+    if (const auto *Name = getTargetIndexName(
+            *Op.getParent()->getParent()->getParent(), Op.getIndex()))
+      OS << Name;
+    else
+      OS << "<unknown>";
+    OS << ')';
+    // TODO: Print the offset and target flags.
+    break;
+  }
+  case MachineOperand::MO_JumpTableIndex:
+    OS << "%jump-table." << Op.getIndex();
+    // TODO: Print target flags.
+    break;
+  case MachineOperand::MO_ExternalSymbol:
+    OS << '$';
+    printLLVMNameWithoutPrefix(OS, Op.getSymbolName());
+    // TODO: Print the target flags.
+    break;
+  case MachineOperand::MO_GlobalAddress:
+    Op.getGlobal()->printAsOperand(OS, /*PrintType=*/false, MST);
+    // TODO: Print offset and target flags.
+    break;
+  case MachineOperand::MO_BlockAddress:
+    OS << "blockaddress(";
+    Op.getBlockAddress()->getFunction()->printAsOperand(OS, /*PrintType=*/false,
+                                                        MST);
+    OS << ", ";
+    printIRBlockReference(*Op.getBlockAddress()->getBasicBlock());
+    OS << ')';
+    // TODO: Print offset and target flags.
+    break;
+  case MachineOperand::MO_RegisterMask: {
+    auto RegMaskInfo = RegisterMaskIds.find(Op.getRegMask());
+    if (RegMaskInfo != RegisterMaskIds.end())
+      OS << StringRef(TRI->getRegMaskNames()[RegMaskInfo->second]).lower();
+    else
+      llvm_unreachable("Can't print this machine register mask yet.");
+    break;
+  }
+  case MachineOperand::MO_Metadata:
+    Op.getMetadata()->printAsOperand(OS, MST);
+    break;
+  case MachineOperand::MO_CFIIndex: {
+    const auto &MMI = Op.getParent()->getParent()->getParent()->getMMI();
+    print(MMI.getFrameInstructions()[Op.getCFIIndex()], TRI);
+    break;
+  }
+  default:
+    // TODO: Print the other machine operands.
+    llvm_unreachable("Can't print this machine operand at the moment");
+  }
+}
+
+static void printCFIRegister(unsigned DwarfReg, raw_ostream &OS,
+                             const TargetRegisterInfo *TRI) {
+  int Reg = TRI->getLLVMRegNum(DwarfReg, true);
+  if (Reg == -1) {
+    OS << "<badreg>";
+    return;
+  }
+  printReg(Reg, OS, TRI);
+}
+
+void MIPrinter::print(const MCCFIInstruction &CFI,
+                      const TargetRegisterInfo *TRI) {
+  switch (CFI.getOperation()) {
+  case MCCFIInstruction::OpOffset:
+    OS << ".cfi_offset ";
+    if (CFI.getLabel())
+      OS << "<mcsymbol> ";
+    printCFIRegister(CFI.getRegister(), OS, TRI);
+    OS << ", " << CFI.getOffset();
+    break;
+  case MCCFIInstruction::OpDefCfaRegister:
+    OS << ".cfi_def_cfa_register ";
+    if (CFI.getLabel())
+      OS << "<mcsymbol> ";
+    printCFIRegister(CFI.getRegister(), OS, TRI);
+    break;
+  case MCCFIInstruction::OpDefCfaOffset:
+    OS << ".cfi_def_cfa_offset ";
+    if (CFI.getLabel())
+      OS << "<mcsymbol> ";
+    OS << CFI.getOffset();
+    break;
+  case MCCFIInstruction::OpDefCfa:
+    OS << ".cfi_def_cfa ";
+    if (CFI.getLabel())
+      OS << "<mcsymbol> ";
+    printCFIRegister(CFI.getRegister(), OS, TRI);
+    OS << ", " << CFI.getOffset();
+    break;
+  default:
+    // TODO: Print the other CFI Operations.
+    OS << "<unserializable cfi operation>";
+    break;
+  }
+}
+
+void llvm::printMIR(raw_ostream &OS, const Module &M) {
+  yaml::Output Out(OS);
+  Out << const_cast<Module &>(M);
+}
+
+void llvm::printMIR(raw_ostream &OS, const MachineFunction &MF) {
+  MIRPrinter Printer(OS);
+  Printer.print(MF);
+}
diff --git a/lib/CodeGen/MIRPrinter.h b/lib/CodeGen/MIRPrinter.h
new file mode 100644
index 00000000000..16aa9038b6b
--- /dev/null
+++ b/lib/CodeGen/MIRPrinter.h
@@ -0,0 +1,33 @@
+//===- MIRPrinter.h - MIR serialization format printer --------------------===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This file declares the functions that print out the LLVM IR and the machine
+// functions using the MIR serialization format.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIB_CODEGEN_MIRPRINTER_H
+#define LLVM_LIB_CODEGEN_MIRPRINTER_H
+
+namespace llvm {
+
+class MachineFunction;
+class Module;
+class raw_ostream;
+
+/// Print LLVM IR using the MIR serialization format to the given output stream.
+void printMIR(raw_ostream &OS, const Module &M);
+
+/// Print a machine function using the MIR serialization format to the given
+/// output stream.
+void printMIR(raw_ostream &OS, const MachineFunction &MF);
+
+} // end namespace llvm
+
+#endif
diff --git a/lib/CodeGen/MIRPrintingPass.cpp b/lib/CodeGen/MIRPrintingPass.cpp
new file mode 100644
index 00000000000..13d61e65d7e
--- /dev/null
+++ b/lib/CodeGen/MIRPrintingPass.cpp
@@ -0,0 +1,71 @@
+//===- MIRPrintingPass.cpp - Pass that prints out using the MIR format ----===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This file implements a pass that prints out the LLVM module using the MIR
+// serialization format.
+//
+//===----------------------------------------------------------------------===//
+
+#include "MIRPrinter.h"
+#include "llvm/CodeGen/Passes.h"
+#include "llvm/CodeGen/MachineFunctionPass.h"
+#include "llvm/CodeGen/MIRYamlMapping.h"
+#include "llvm/Support/Debug.h"
+#include "llvm/Support/raw_ostream.h"
+
+using namespace llvm;
+
+namespace {
+
+/// This pass prints out the LLVM IR to an output stream using the MIR
+/// serialization format.
+struct MIRPrintingPass : public MachineFunctionPass {
+  static char ID;
+  raw_ostream &OS;
+  std::string MachineFunctions;
+
+  MIRPrintingPass() : MachineFunctionPass(ID), OS(dbgs()) {}
+  MIRPrintingPass(raw_ostream &OS) : MachineFunctionPass(ID), OS(OS) {}
+
+  const char *getPassName() const override { return "MIR Printing Pass"; }
+
+  void getAnalysisUsage(AnalysisUsage &AU) const override {
+    AU.setPreservesAll();
+    MachineFunctionPass::getAnalysisUsage(AU);
+  }
+
+  virtual bool runOnMachineFunction(MachineFunction &MF) override {
+    std::string Str;
+    raw_string_ostream StrOS(Str);
+    printMIR(StrOS, MF);
+    MachineFunctions.append(StrOS.str());
+    return false;
+  }
+
+  virtual bool doFinalization(Module &M) override {
+    printMIR(OS, M);
+    OS << MachineFunctions;
+    return false;
+  }
+};
+
+char MIRPrintingPass::ID = 0;
+
+} // end anonymous namespace
+
+char &llvm::MIRPrintingPassID = MIRPrintingPass::ID;
+INITIALIZE_PASS(MIRPrintingPass, "mir-printer", "MIR Printer", false, false)
+
+namespace llvm {
+
+MachineFunctionPass *createPrintMIRPass(raw_ostream &OS) {
+  return new MIRPrintingPass(OS);
+}
+
+} // end namespace llvm
diff --git a/lib/CodeGen/MachineBasicBlock.cpp b/lib/CodeGen/MachineBasicBlock.cpp
index 34131bbc16a..5d3f7ebaed2 100644
--- a/lib/CodeGen/MachineBasicBlock.cpp
+++ b/lib/CodeGen/MachineBasicBlock.cpp
@@ -24,6 +24,7 @@
 #include "llvm/CodeGen/SlotIndexes.h"
 #include "llvm/IR/BasicBlock.h"
 #include "llvm/IR/DataLayout.h"
+#include "llvm/IR/ModuleSlotTracker.h"
 #include "llvm/MC/MCAsmInfo.h"
 #include "llvm/MC/MCContext.h"
 #include "llvm/Support/Debug.h"
@@ -53,7 +54,7 @@ MCSymbol *MachineBasicBlock::getSymbol() const {
     const MachineFunction *MF = getParent();
     MCContext &Ctx = MF->getContext();
     const char *Prefix = Ctx.getAsmInfo()->getPrivateLabelPrefix();
-    CachedMCSymbol = Ctx.GetOrCreateSymbol(Twine(Prefix) + "BB" +
+    CachedMCSymbol = Ctx.getOrCreateSymbol(Twine(Prefix) + "BB" +
                                            Twine(MF->getFunctionNumber()) +
                                            "_" + Twine(getNumber()));
   }
@@ -171,9 +172,8 @@ MachineBasicBlock::iterator MachineBasicBlock::getFirstTerminator() {
   return I;
 }
 
-MachineBasicBlock::const_iterator
-MachineBasicBlock::getFirstTerminator() const {
-  const_iterator B = begin(), E = end(), I = E;
+MachineBasicBlock::instr_iterator MachineBasicBlock::getFirstInstrTerminator() {
+  instr_iterator B = instr_begin(), E = instr_end(), I = E;
   while (I != B && ((--I)->isTerminator() || I->isDebugValue()))
     ; /*noop */
   while (I != E && !I->isTerminator())
@@ -181,11 +181,10 @@ MachineBasicBlock::getFirstTerminator() const {
   return I;
 }
 
-MachineBasicBlock::instr_iterator MachineBasicBlock::getFirstInstrTerminator() {
-  instr_iterator B = instr_begin(), E = instr_end(), I = E;
-  while (I != B && ((--I)->isTerminator() || I->isDebugValue()))
-    ; /*noop */
-  while (I != E && !I->isTerminator())
+MachineBasicBlock::iterator MachineBasicBlock::getFirstNonDebugInstr() {
+  // Skip over begin-of-block dbg_value instructions.
+  iterator I = begin(), E = end();
+  while (I != E && I->isDebugValue())
     ++I;
   return I;
 }
@@ -204,21 +203,6 @@ MachineBasicBlock::iterator MachineBasicBlock::getLastNonDebugInstr() {
   return end();
 }
 
-MachineBasicBlock::const_iterator
-MachineBasicBlock::getLastNonDebugInstr() const {
-  // Skip over end-of-block dbg_value instructions.
-  const_instr_iterator B = instr_begin(), I = instr_end();
-  while (I != B) {
-    --I;
-    // Return instruction that starts a bundle.
-    if (I->isDebugValue() || I->isInsideBundle())
-      continue;
-    return I;
-  }
-  // The block is all debug values.
-  return end();
-}
-
 const MachineBasicBlock *MachineBasicBlock::getLandingPadSuccessor() const {
   // A block with a landing pad successor only has one other successor.
   if (succ_size() > 2)
@@ -261,6 +245,20 @@ void MachineBasicBlock::print(raw_ostream &OS, SlotIndexes *Indexes) const {
        << " is null\n";
     return;
   }
+  const Function *F = MF->getFunction();
+  const Module *M = F ? F->getParent() : nullptr;
+  ModuleSlotTracker MST(M);
+  print(OS, MST, Indexes);
+}
+
+void MachineBasicBlock::print(raw_ostream &OS, ModuleSlotTracker &MST,
+                              SlotIndexes *Indexes) const {
+  const MachineFunction *MF = getParent();
+  if (!MF) {
+    OS << "Can't print out MachineBasicBlock because parent MachineFunction"
+       << " is null\n";
+    return;
+  }
 
   if (Indexes)
     OS << Indexes->getMBBStartIdx(this) << '\t';
@@ -270,7 +268,7 @@ void MachineBasicBlock::print(raw_ostream &OS, SlotIndexes *Indexes) const {
   const char *Comma = "";
   if (const BasicBlock *LBB = getBasicBlock()) {
     OS << Comma << "derived from LLVM BB ";
-    LBB->printAsOperand(OS, /*PrintType=*/false);
+    LBB->printAsOperand(OS, /*PrintType=*/false, MST);
     Comma = ", ";
   }
   if (isLandingPad()) { OS << Comma << "EH LANDING PAD"; Comma = ", "; }
@@ -307,7 +305,7 @@ void MachineBasicBlock::print(raw_ostream &OS, SlotIndexes *Indexes) const {
     OS << '\t';
     if (I->isInsideBundle())
       OS << "  * ";
-    I->print(OS);
+    I->print(OS, MST);
   }
 
   // Print the successors of this block according to the CFG.
@@ -1129,21 +1127,19 @@ getWeightIterator(MachineBasicBlock::const_succ_iterator I) const {
 /// instructions after (searching just for defs) MI.
 MachineBasicBlock::LivenessQueryResult
 MachineBasicBlock::computeRegisterLiveness(const TargetRegisterInfo *TRI,
-                                           unsigned Reg, MachineInstr *MI,
-                                           unsigned Neighborhood) {
+                                           unsigned Reg, const_iterator Before,
+                                           unsigned Neighborhood) const {
   unsigned N = Neighborhood;
-  MachineBasicBlock *MBB = MI->getParent();
-
-  // Start by searching backwards from MI, looking for kills, reads or defs.
 
-  MachineBasicBlock::iterator I(MI);
+  // Start by searching backwards from Before, looking for kills, reads or defs.
+  const_iterator I(Before);
   // If this is the first insn in the block, don't search backwards.
-  if (I != MBB->begin()) {
+  if (I != begin()) {
     do {
       --I;
 
       MachineOperandIteratorBase::PhysRegInfo Analysis =
-        MIOperands(I).analyzePhysReg(Reg, TRI);
+        ConstMIOperands(I).analyzePhysReg(Reg, TRI);
 
       if (Analysis.Defines)
         // Outputs happen after inputs so they take precedence if both are
@@ -1158,15 +1154,15 @@ MachineBasicBlock::computeRegisterLiveness(const TargetRegisterInfo *TRI,
         // Defined or read without a previous kill - live.
         return Analysis.Reads ? LQR_Live : LQR_OverlappingLive;
 
-    } while (I != MBB->begin() && --N > 0);
+    } while (I != begin() && --N > 0);
   }
 
   // Did we get to the start of the block?
-  if (I == MBB->begin()) {
+  if (I == begin()) {
     // If so, the register's state is definitely defined by the live-in state.
     for (MCRegAliasIterator RAI(Reg, TRI, /*IncludeSelf=*/true);
          RAI.isValid(); ++RAI) {
-      if (MBB->isLiveIn(*RAI))
+      if (isLiveIn(*RAI))
         return (*RAI == Reg) ? LQR_Live : LQR_OverlappingLive;
     }
 
@@ -1175,13 +1171,13 @@ MachineBasicBlock::computeRegisterLiveness(const TargetRegisterInfo *TRI,
 
   N = Neighborhood;
 
-  // Try searching forwards from MI, looking for reads or defs.
-  I = MachineBasicBlock::iterator(MI);
+  // Try searching forwards from Before, looking for reads or defs.
+  I = const_iterator(Before);
   // If this is the last insn in the block, don't search forwards.
-  if (I != MBB->end()) {
-    for (++I; I != MBB->end() && N > 0; ++I, --N) {
+  if (I != end()) {
+    for (++I; I != end() && N > 0; ++I, --N) {
       MachineOperandIteratorBase::PhysRegInfo Analysis =
-        MIOperands(I).analyzePhysReg(Reg, TRI);
+        ConstMIOperands(I).analyzePhysReg(Reg, TRI);
 
       if (Analysis.ReadsOverlap)
         // Used, therefore must have been live.
diff --git a/lib/CodeGen/MachineBlockFrequencyInfo.cpp b/lib/CodeGen/MachineBlockFrequencyInfo.cpp
index 9151d99089d..10b0e929f2b 100644
--- a/lib/CodeGen/MachineBlockFrequencyInfo.cpp
+++ b/lib/CodeGen/MachineBlockFrequencyInfo.cpp
@@ -143,7 +143,7 @@ bool MachineBlockFrequencyInfo::runOnMachineFunction(MachineFunction &F) {
   MachineLoopInfo &MLI = getAnalysis<MachineLoopInfo>();
   if (!MBFI)
     MBFI.reset(new ImplType);
-  MBFI->doFunction(&F, &MBPI, &MLI);
+  MBFI->calculate(F, MBPI, MLI);
 #ifndef NDEBUG
   if (ViewMachineBlockFreqPropagationDAG != GVDT_None) {
     view();
diff --git a/lib/CodeGen/MachineBlockPlacement.cpp b/lib/CodeGen/MachineBlockPlacement.cpp
index 2969bad4ff9..d7a7dd1c378 100644
--- a/lib/CodeGen/MachineBlockPlacement.cpp
+++ b/lib/CodeGen/MachineBlockPlacement.cpp
@@ -1224,4 +1224,3 @@ bool MachineBlockPlacementStats::runOnMachineFunction(MachineFunction &F) {
 
   return false;
 }
-
diff --git a/lib/CodeGen/MachineCSE.cpp b/lib/CodeGen/MachineCSE.cpp
index f72d72a2e87..87aaaa0834c 100644
--- a/lib/CodeGen/MachineCSE.cpp
+++ b/lib/CodeGen/MachineCSE.cpp
@@ -48,7 +48,7 @@ namespace {
     MachineRegisterInfo *MRI;
   public:
     static char ID; // Pass identification
-    MachineCSE() : MachineFunctionPass(ID), LookAheadLimit(5), CurrVN(0) {
+    MachineCSE() : MachineFunctionPass(ID), LookAheadLimit(0), CurrVN(0) {
       initializeMachineCSEPass(*PassRegistry::getPassRegistry());
     }
 
@@ -69,7 +69,7 @@ namespace {
     }
 
   private:
-    const unsigned LookAheadLimit;
+    unsigned LookAheadLimit;
     typedef RecyclingAllocator<BumpPtrAllocator,
         ScopedHashTableVal<MachineInstr*, unsigned> > AllocatorTy;
     typedef ScopedHashTable<MachineInstr*, unsigned,
@@ -716,5 +716,6 @@ bool MachineCSE::runOnMachineFunction(MachineFunction &MF) {
   MRI = &MF.getRegInfo();
   AA = &getAnalysis<AliasAnalysis>();
   DT = &getAnalysis<MachineDominatorTree>();
+  LookAheadLimit = TII->getMachineCSELookAheadLimit();
   return PerformCSE(DT->getRootNode());
 }
diff --git a/lib/CodeGen/MachineCombiner.cpp b/lib/CodeGen/MachineCombiner.cpp
index 41045ac0bab..7ffb41b64df 100644
--- a/lib/CodeGen/MachineCombiner.cpp
+++ b/lib/CodeGen/MachineCombiner.cpp
@@ -67,10 +67,11 @@ class MachineCombiner : public MachineFunctionPass {
   unsigned getLatency(MachineInstr *Root, MachineInstr *NewRoot,
                       MachineTraceMetrics::Trace BlockTrace);
   bool
-  preservesCriticalPathLen(MachineBasicBlock *MBB, MachineInstr *Root,
+  improvesCriticalPathLen(MachineBasicBlock *MBB, MachineInstr *Root,
                            MachineTraceMetrics::Trace BlockTrace,
                            SmallVectorImpl<MachineInstr *> &InsInstrs,
-                           DenseMap<unsigned, unsigned> &InstrIdxForVirtReg);
+                           DenseMap<unsigned, unsigned> &InstrIdxForVirtReg,
+                           bool NewCodeHasLessInsts);
   bool preservesResourceLen(MachineBasicBlock *MBB,
                             MachineTraceMetrics::Trace BlockTrace,
                             SmallVectorImpl<MachineInstr *> &InsInstrs,
@@ -123,7 +124,8 @@ MachineCombiner::getDepth(SmallVectorImpl<MachineInstr *> &InsInstrs,
                           MachineTraceMetrics::Trace BlockTrace) {
 
   SmallVector<unsigned, 16> InstrDepth;
-  assert(TSchedModel.hasInstrSchedModel() && "Missing machine model\n");
+  assert(TSchedModel.hasInstrSchedModelOrItineraries() &&
+         "Missing machine model\n");
 
   // For each instruction in the new sequence compute the depth based on the
   // operands. Use the trace information when possible. For new operands which
@@ -131,8 +133,7 @@ MachineCombiner::getDepth(SmallVectorImpl<MachineInstr *> &InsInstrs,
   for (auto *InstrPtr : InsInstrs) { // for each Use
     unsigned IDepth = 0;
     DEBUG(dbgs() << "NEW INSTR "; InstrPtr->dump(); dbgs() << "\n";);
-    for (unsigned i = 0, e = InstrPtr->getNumOperands(); i != e; ++i) {
-      const MachineOperand &MO = InstrPtr->getOperand(i);
+    for (const MachineOperand &MO : InstrPtr->operands()) {
       // Check for virtual register operand.
       if (!(MO.isReg() && TargetRegisterInfo::isVirtualRegister(MO.getReg())))
         continue;
@@ -181,13 +182,13 @@ MachineCombiner::getDepth(SmallVectorImpl<MachineInstr *> &InsInstrs,
 unsigned MachineCombiner::getLatency(MachineInstr *Root, MachineInstr *NewRoot,
                                      MachineTraceMetrics::Trace BlockTrace) {
 
-  assert(TSchedModel.hasInstrSchedModel() && "Missing machine model\n");
+  assert(TSchedModel.hasInstrSchedModelOrItineraries() &&
+         "Missing machine model\n");
 
   // Check each definition in NewRoot and compute the latency
   unsigned NewRootLatency = 0;
 
-  for (unsigned i = 0, e = NewRoot->getNumOperands(); i != e; ++i) {
-    const MachineOperand &MO = NewRoot->getOperand(i);
+  for (const MachineOperand &MO : NewRoot->operands()) {
     // Check for virtual register operand.
     if (!(MO.isReg() && TargetRegisterInfo::isVirtualRegister(MO.getReg())))
       continue;
@@ -210,29 +211,35 @@ unsigned MachineCombiner::getLatency(MachineInstr *Root, MachineInstr *NewRoot,
   return NewRootLatency;
 }
 
-/// True when the new instruction sequence does not
-/// lengthen the critical path. The DAGCombine code sequence ends in MI
-/// (Machine Instruction) Root. The new code sequence ends in MI NewRoot. A
-/// necessary condition for the new sequence to replace the old sequence is that
-/// it cannot lengthen the critical path. This is decided by the formula
+/// True when the new instruction sequence does not lengthen the critical path
+/// and the new sequence has less instructions or the new sequence improves the
+/// critical path.
+/// The DAGCombine code sequence ends in MI (Machine Instruction) Root.
+/// The new code sequence ends in MI NewRoot. A necessary condition for the new
+/// sequence to replace the old sequence is that it cannot lengthen the critical
+/// path. This is decided by the formula:
 /// (NewRootDepth + NewRootLatency) <= (RootDepth + RootLatency + RootSlack)).
-/// The slack is the number of cycles Root can be delayed before the critical
-/// patch becomes longer.
-bool MachineCombiner::preservesCriticalPathLen(
+/// If the new sequence has an equal length critical path but does not reduce
+/// the number of instructions (NewCodeHasLessInsts is false), then it is not
+/// considered an improvement. The slack is the number of cycles Root can be
+/// delayed before the critical patch becomes longer.
+bool MachineCombiner::improvesCriticalPathLen(
     MachineBasicBlock *MBB, MachineInstr *Root,
     MachineTraceMetrics::Trace BlockTrace,
     SmallVectorImpl<MachineInstr *> &InsInstrs,
-    DenseMap<unsigned, unsigned> &InstrIdxForVirtReg) {
+    DenseMap<unsigned, unsigned> &InstrIdxForVirtReg,
+    bool NewCodeHasLessInsts) {
 
-  assert(TSchedModel.hasInstrSchedModel() && "Missing machine model\n");
-  // NewRoot is the last instruction in the \p InsInstrs vector
-  // Get depth and latency of NewRoot
+  assert(TSchedModel.hasInstrSchedModelOrItineraries() &&
+         "Missing machine model\n");
+  // NewRoot is the last instruction in the \p InsInstrs vector.
+  // Get depth and latency of NewRoot.
   unsigned NewRootIdx = InsInstrs.size() - 1;
   MachineInstr *NewRoot = InsInstrs[NewRootIdx];
   unsigned NewRootDepth = getDepth(InsInstrs, InstrIdxForVirtReg, BlockTrace);
   unsigned NewRootLatency = getLatency(Root, NewRoot, BlockTrace);
 
-  // Get depth, latency and slack of Root
+  // Get depth, latency and slack of Root.
   unsigned RootDepth = BlockTrace.getInstrCycles(Root).Depth;
   unsigned RootLatency = TSchedModel.computeInstrLatency(Root);
   unsigned RootSlack = BlockTrace.getInstrSlack(Root);
@@ -247,9 +254,13 @@ bool MachineCombiner::preservesCriticalPathLen(
         dbgs() << " RootDepth + RootLatency + RootSlack "
                << RootDepth + RootLatency + RootSlack << "\n";);
 
-  /// True when the new sequence does not lenghten the critical path.
-  return ((NewRootDepth + NewRootLatency) <=
-          (RootDepth + RootLatency + RootSlack));
+  unsigned NewCycleCount = NewRootDepth + NewRootLatency;
+  unsigned OldCycleCount = RootDepth + RootLatency + RootSlack;
+  
+  if (NewCodeHasLessInsts)
+    return NewCycleCount <= OldCycleCount;
+  else
+    return NewCycleCount < OldCycleCount;
 }
 
 /// helper routine to convert instructions into SC
@@ -268,6 +279,8 @@ bool MachineCombiner::preservesResourceLen(
     MachineBasicBlock *MBB, MachineTraceMetrics::Trace BlockTrace,
     SmallVectorImpl<MachineInstr *> &InsInstrs,
     SmallVectorImpl<MachineInstr *> &DelInstrs) {
+  if (!TSchedModel.hasInstrSchedModel())
+    return true;
 
   // Compute current resource length
 
@@ -286,7 +299,7 @@ bool MachineCombiner::preservesResourceLen(
   ArrayRef<const MCSchedClassDesc *> MSCInsArr = makeArrayRef(InsInstrsSC);
   ArrayRef<const MCSchedClassDesc *> MSCDelArr = makeArrayRef(DelInstrsSC);
 
-  // Compute new resource length
+  // Compute new resource length.
   unsigned ResLenAfterCombine =
       BlockTrace.getResourceLength(MBBarr, MSCInsArr, MSCDelArr);
 
@@ -302,7 +315,7 @@ bool MachineCombiner::preservesResourceLen(
 bool MachineCombiner::doSubstitute(unsigned NewSize, unsigned OldSize) {
   if (OptSize && (NewSize < OldSize))
     return true;
-  if (!TSchedModel.hasInstrSchedModel())
+  if (!TSchedModel.hasInstrSchedModelOrItineraries())
     return true;
   return false;
 }
@@ -324,7 +337,7 @@ bool MachineCombiner::combineInstructions(MachineBasicBlock *MBB) {
     auto &MI = *BlockIter++;
 
     DEBUG(dbgs() << "INSTR "; MI.dump(); dbgs() << "\n";);
-    SmallVector<MachineCombinerPattern::MC_PATTERN, 16> Pattern;
+    SmallVector<MachineCombinerPattern::MC_PATTERN, 16> Patterns;
     // The motivating example is:
     //
     //     MUL  Other        MUL_op1 MUL_op2  Other
@@ -347,11 +360,11 @@ bool MachineCombiner::combineInstructions(MachineBasicBlock *MBB) {
     //
     // The algorithm does not try to evaluate all patterns and pick the best.
     // This is only an artificial restriction though. In practice there is
-    // mostly one pattern and hasPattern() can order patterns based on an
-    // internal cost heuristic.
+    // mostly one pattern, and getMachineCombinerPatterns() can order patterns
+    // based on an internal cost heuristic.
 
-    if (TII->hasPattern(MI, Pattern)) {
-      for (auto P : Pattern) {
+    if (TII->getMachineCombinerPatterns(MI, Patterns)) {
+      for (auto P : Patterns) {
         SmallVector<MachineInstr *, 16> InsInstrs;
         SmallVector<MachineInstr *, 16> DelInstrs;
         DenseMap<unsigned, unsigned> InstrIdxForVirtReg;
@@ -361,22 +374,24 @@ bool MachineCombiner::combineInstructions(MachineBasicBlock *MBB) {
         Traces->verifyAnalysis();
         TII->genAlternativeCodeSequence(MI, P, InsInstrs, DelInstrs,
                                         InstrIdxForVirtReg);
+        unsigned NewInstCount = InsInstrs.size();
+        unsigned OldInstCount = DelInstrs.size();
         // Found pattern, but did not generate alternative sequence.
         // This can happen e.g. when an immediate could not be materialized
         // in a single instruction.
-        if (!InsInstrs.size())
+        if (!NewInstCount)
           continue;
         // Substitute when we optimize for codesize and the new sequence has
         // fewer instructions OR
-        // the new sequence neither lenghten the critical path nor increases
+        // the new sequence neither lengthens the critical path nor increases
         // resource pressure.
-        if (doSubstitute(InsInstrs.size(), DelInstrs.size()) ||
-            (preservesCriticalPathLen(MBB, &MI, BlockTrace, InsInstrs,
-                                      InstrIdxForVirtReg) &&
+        if (doSubstitute(NewInstCount, OldInstCount) ||
+            (improvesCriticalPathLen(MBB, &MI, BlockTrace, InsInstrs,
+                                      InstrIdxForVirtReg,
+                                      NewInstCount < OldInstCount) &&
              preservesResourceLen(MBB, BlockTrace, InsInstrs, DelInstrs))) {
           for (auto *InstrPtr : InsInstrs)
-            MBB->insert((MachineBasicBlock::iterator) & MI,
-                        (MachineInstr *)InstrPtr);
+            MBB->insert((MachineBasicBlock::iterator) &MI, InstrPtr);
           for (auto *InstrPtr : DelInstrs)
             InstrPtr->eraseFromParentAndMarkDBGValuesForRemoval();
 
@@ -385,15 +400,14 @@ bool MachineCombiner::combineInstructions(MachineBasicBlock *MBB) {
 
           Traces->invalidate(MBB);
           Traces->verifyAnalysis();
-          // Eagerly stop after the first pattern fired
+          // Eagerly stop after the first pattern fires.
           break;
         } else {
           // Cleanup instructions of the alternative code sequence. There is no
           // use for them.
-          for (auto *InstrPtr : InsInstrs) {
-            MachineFunction *MF = MBB->getParent();
-            MF->DeleteMachineInstr((MachineInstr *)InstrPtr);
-          }
+          MachineFunction *MF = MBB->getParent();
+          for (auto *InstrPtr : InsInstrs)
+            MF->DeleteMachineInstr(InstrPtr);
         }
         InstrIdxForVirtReg.clear();
       }
diff --git a/lib/CodeGen/MachineCopyPropagation.cpp b/lib/CodeGen/MachineCopyPropagation.cpp
index 96111225db5..a6863412132 100644
--- a/lib/CodeGen/MachineCopyPropagation.cpp
+++ b/lib/CodeGen/MachineCopyPropagation.cpp
@@ -54,7 +54,6 @@ namespace {
                                  SourceMap &SrcMap,
                                  DenseMap<unsigned, MachineInstr*> &AvailCopyMap);
     bool CopyPropagateBlock(MachineBasicBlock &MBB);
-    void removeCopy(MachineInstr *MI);
   };
 }
 char MachineCopyPropagation::ID = 0;
@@ -127,13 +126,6 @@ static bool isNopCopy(MachineInstr *CopyMI, unsigned Def, unsigned Src,
   return false;
 }
 
-// Remove MI from the function because it has been determined it is dead.
-// Turn it into a noop KILL instruction as opposed to removing it to
-// maintain imp-use/imp-def chains.
-void MachineCopyPropagation::removeCopy(MachineInstr *MI) {
-  MI->setDesc(TII->get(TargetOpcode::KILL));
-}
-
 bool MachineCopyPropagation::CopyPropagateBlock(MachineBasicBlock &MBB) {
   SmallSetVector<MachineInstr*, 8> MaybeDeadCopies;  // Candidates for deletion
   DenseMap<unsigned, MachineInstr*> AvailCopyMap;    // Def -> available copies map
@@ -183,7 +175,7 @@ bool MachineCopyPropagation::CopyPropagateBlock(MachineBasicBlock &MBB) {
           for (MachineBasicBlock::iterator I = CopyMI, E = MI; I != E; ++I)
             I->clearRegisterKills(Def, TRI);
 
-          removeCopy(MI);
+          MI->eraseFromParent();
           Changed = true;
           ++NumDeletes;
           continue;
@@ -266,6 +258,14 @@ bool MachineCopyPropagation::CopyPropagateBlock(MachineBasicBlock &MBB) {
           MaybeDeadCopies.remove(CI->second);
         }
       }
+      // Treat undef use like defs for copy propagation but not for
+      // dead copy. We would need to do a liveness check to be sure the copy
+      // is dead for undef uses.
+      // The backends are allowed to do whatever they want with undef value
+      // and we cannot be sure this register will not be rewritten to break
+      // some false dependencies for the hardware for instance.
+      if (MO.isUndef())
+        Defs.push_back(Reg);
     }
 
     // The instruction has a register mask operand which means that it clobbers
@@ -283,7 +283,7 @@ bool MachineCopyPropagation::CopyPropagateBlock(MachineBasicBlock &MBB) {
           continue;
         DEBUG(dbgs() << "MCP: Removing copy due to regmask clobbering: ";
               (*DI)->dump());
-        removeCopy(*DI);
+        (*DI)->eraseFromParent();
         Changed = true;
         ++NumDeletes;
       }
@@ -319,7 +319,7 @@ bool MachineCopyPropagation::CopyPropagateBlock(MachineBasicBlock &MBB) {
            DI = MaybeDeadCopies.begin(), DE = MaybeDeadCopies.end();
          DI != DE; ++DI) {
       if (!MRI->isReserved((*DI)->getOperand(0).getReg())) {
-        removeCopy(*DI);
+        (*DI)->eraseFromParent();
         Changed = true;
         ++NumDeletes;
       }
diff --git a/lib/CodeGen/MachineDominators.cpp b/lib/CodeGen/MachineDominators.cpp
index 467a2e4eb42..3f04bb0b532 100644
--- a/lib/CodeGen/MachineDominators.cpp
+++ b/lib/CodeGen/MachineDominators.cpp
@@ -19,8 +19,8 @@
 using namespace llvm;
 
 namespace llvm {
-TEMPLATE_INSTANTIATION(class DomTreeNodeBase<MachineBasicBlock>);
-TEMPLATE_INSTANTIATION(class DominatorTreeBase<MachineBasicBlock>);
+template class DomTreeNodeBase<MachineBasicBlock>;
+template class DominatorTreeBase<MachineBasicBlock>;
 }
 
 char MachineDominatorTree::ID = 0;
diff --git a/lib/CodeGen/MachineFunction.cpp b/lib/CodeGen/MachineFunction.cpp
index 448531fb716..bd6f8771e7b 100644
--- a/lib/CodeGen/MachineFunction.cpp
+++ b/lib/CodeGen/MachineFunction.cpp
@@ -19,6 +19,7 @@
 #include "llvm/Analysis/ConstantFolding.h"
 #include "llvm/CodeGen/MachineConstantPool.h"
 #include "llvm/CodeGen/MachineFrameInfo.h"
+#include "llvm/CodeGen/MachineFunctionInitializer.h"
 #include "llvm/CodeGen/MachineFunctionPass.h"
 #include "llvm/CodeGen/MachineInstr.h"
 #include "llvm/CodeGen/MachineJumpTableInfo.h"
@@ -28,6 +29,8 @@
 #include "llvm/IR/DataLayout.h"
 #include "llvm/IR/DebugInfo.h"
 #include "llvm/IR/Function.h"
+#include "llvm/IR/Module.h"
+#include "llvm/IR/ModuleSlotTracker.h"
 #include "llvm/MC/MCAsmInfo.h"
 #include "llvm/MC/MCContext.h"
 #include "llvm/Support/Debug.h"
@@ -41,11 +44,13 @@ using namespace llvm;
 
 #define DEBUG_TYPE "codegen"
 
+void MachineFunctionInitializer::anchor() {}
+
 //===----------------------------------------------------------------------===//
 // MachineFunction implementation
 //===----------------------------------------------------------------------===//
 
-// Out of line virtual method.
+// Out-of-line virtual method.
 MachineFunctionInfo::~MachineFunctionInfo() {}
 
 void ilist_traits<MachineBasicBlock>::deleteNode(MachineBasicBlock *MBB) {
@@ -70,7 +75,7 @@ MachineFunction::MachineFunction(const Function *F, const TargetMachine &TM,
   if (Fn->hasFnAttribute(Attribute::StackAlignment))
     FrameInfo->ensureMaxAlignment(Fn->getFnStackAlignment());
 
-  ConstantPool = new (Allocator) MachineConstantPool(TM);
+  ConstantPool = new (Allocator) MachineConstantPool(getDataLayout());
   Alignment = STI->getTargetLowering()->getMinFunctionAlignment();
 
   // FIXME: Shouldn't use pref alignment if explicit alignment is set on Fn.
@@ -80,6 +85,10 @@ MachineFunction::MachineFunction(const Function *F, const TargetMachine &TM,
 
   FunctionNumber = FunctionNum;
   JumpTableInfo = nullptr;
+
+  assert(TM.isCompatibleDataLayout(getDataLayout()) &&
+         "Can't create a MachineFunction using a Module with a "
+         "Target-incompatible DataLayout attached\n");
 }
 
 MachineFunction::~MachineFunction() {
@@ -114,8 +123,12 @@ MachineFunction::~MachineFunction() {
   }
 }
 
-/// getOrCreateJumpTableInfo - Get the JumpTableInfo for this function, if it
-/// does already exist, allocate one.
+const DataLayout &MachineFunction::getDataLayout() const {
+  return Fn->getParent()->getDataLayout();
+}
+
+/// Get the JumpTableInfo for this function.
+/// If it does not already exist, allocate one.
 MachineJumpTableInfo *MachineFunction::
 getOrCreateJumpTableInfo(unsigned EntryKind) {
   if (JumpTableInfo) return JumpTableInfo;
@@ -130,11 +143,10 @@ bool MachineFunction::shouldSplitStack() {
   return getFunction()->hasFnAttribute("split-stack");
 }
 
-/// RenumberBlocks - This discards all of the MachineBasicBlock numbers and
-/// recomputes them.  This guarantees that the MBB numbers are sequential,
-/// dense, and match the ordering of the blocks within the function.  If a
-/// specific MachineBasicBlock is specified, only that block and those after
-/// it are renumbered.
+/// This discards all of the MachineBasicBlock numbers and recomputes them.
+/// This guarantees that the MBB numbers are sequential, dense, and match the
+/// ordering of the blocks within the function.  If a specific MachineBasicBlock
+/// is specified, only that block and those after it are renumbered.
 void MachineFunction::RenumberBlocks(MachineBasicBlock *MBB) {
   if (empty()) { MBBNumbering.clear(); return; }
   MachineFunction::iterator MBBI, E = end();
@@ -172,9 +184,7 @@ void MachineFunction::RenumberBlocks(MachineBasicBlock *MBB) {
   MBBNumbering.resize(BlockNo);
 }
 
-/// CreateMachineInstr - Allocate a new MachineInstr. Use this instead
-/// of `new MachineInstr'.
-///
+/// Allocate a new MachineInstr. Use this instead of `new MachineInstr'.
 MachineInstr *
 MachineFunction::CreateMachineInstr(const MCInstrDesc &MCID,
                                     DebugLoc DL, bool NoImp) {
@@ -182,17 +192,15 @@ MachineFunction::CreateMachineInstr(const MCInstrDesc &MCID,
     MachineInstr(*this, MCID, DL, NoImp);
 }
 
-/// CloneMachineInstr - Create a new MachineInstr which is a copy of the
-/// 'Orig' instruction, identical in all ways except the instruction
-/// has no parent, prev, or next.
-///
+/// Create a new MachineInstr which is a copy of the 'Orig' instruction,
+/// identical in all ways except the instruction has no parent, prev, or next.
 MachineInstr *
 MachineFunction::CloneMachineInstr(const MachineInstr *Orig) {
   return new (InstructionRecycler.Allocate<MachineInstr>(Allocator))
              MachineInstr(*this, *Orig);
 }
 
-/// DeleteMachineInstr - Delete the given MachineInstr.
+/// Delete the given MachineInstr.
 ///
 /// This function also serves as the MachineInstr destructor - the real
 /// ~MachineInstr() destructor must be empty.
@@ -208,17 +216,15 @@ MachineFunction::DeleteMachineInstr(MachineInstr *MI) {
   InstructionRecycler.Deallocate(Allocator, MI);
 }
 
-/// CreateMachineBasicBlock - Allocate a new MachineBasicBlock. Use this
-/// instead of `new MachineBasicBlock'.
-///
+/// Allocate a new MachineBasicBlock. Use this instead of
+/// `new MachineBasicBlock'.
 MachineBasicBlock *
 MachineFunction::CreateMachineBasicBlock(const BasicBlock *bb) {
   return new (BasicBlockRecycler.Allocate<MachineBasicBlock>(Allocator))
              MachineBasicBlock(*this, bb);
 }
 
-/// DeleteMachineBasicBlock - Delete the given MachineBasicBlock.
-///
+/// Delete the given MachineBasicBlock.
 void
 MachineFunction::DeleteMachineBasicBlock(MachineBasicBlock *MBB) {
   assert(MBB->getParent() == this && "MBB parent mismatch!");
@@ -320,6 +326,13 @@ MachineFunction::extractStoreMemRefs(MachineInstr::mmo_iterator Begin,
   return std::make_pair(Result, Result + Num);
 }
 
+const char *MachineFunction::createExternalSymbolName(StringRef Name) {
+  char *Dest = Allocator.Allocate<char>(Name.size() + 1);
+  std::copy(Name.begin(), Name.end(), Dest);
+  Dest[Name.size()] = 0;
+  return Dest;
+}
+
 #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
 void MachineFunction::dump() const {
   print(dbgs());
@@ -365,9 +378,11 @@ void MachineFunction::print(raw_ostream &OS, SlotIndexes *Indexes) const {
     OS << '\n';
   }
 
+  ModuleSlotTracker MST(getFunction()->getParent());
+  MST.incorporateFunction(*getFunction());
   for (const auto &BB : *this) {
     OS << '\n';
-    BB.print(OS, Indexes);
+    BB.print(OS, MST, Indexes);
   }
 
   OS << "\n# End machine code for function " << getName() << ".\n\n";
@@ -430,7 +445,7 @@ void MachineFunction::viewCFGOnly() const
 #endif // NDEBUG
 }
 
-/// addLiveIn - Add the specified physical register as a live-in value and
+/// Add the specified physical register as a live-in value and
 /// create a corresponding virtual register for it.
 unsigned MachineFunction::addLiveIn(unsigned PReg,
                                     const TargetRegisterClass *RC) {
@@ -454,37 +469,35 @@ unsigned MachineFunction::addLiveIn(unsigned PReg,
   return VReg;
 }
 
-/// getJTISymbol - Return the MCSymbol for the specified non-empty jump table.
+/// Return the MCSymbol for the specified non-empty jump table.
 /// If isLinkerPrivate is specified, an 'l' label is returned, otherwise a
 /// normal 'L' label is returned.
 MCSymbol *MachineFunction::getJTISymbol(unsigned JTI, MCContext &Ctx,
                                         bool isLinkerPrivate) const {
-  const DataLayout *DL = getTarget().getDataLayout();
+  const DataLayout &DL = getDataLayout();
   assert(JumpTableInfo && "No jump tables");
   assert(JTI < JumpTableInfo->getJumpTables().size() && "Invalid JTI!");
 
-  const char *Prefix = isLinkerPrivate ? DL->getLinkerPrivateGlobalPrefix() :
-                                         DL->getPrivateGlobalPrefix();
+  const char *Prefix = isLinkerPrivate ? DL.getLinkerPrivateGlobalPrefix()
+                                       : DL.getPrivateGlobalPrefix();
   SmallString<60> Name;
   raw_svector_ostream(Name)
     << Prefix << "JTI" << getFunctionNumber() << '_' << JTI;
-  return Ctx.GetOrCreateSymbol(Name);
+  return Ctx.getOrCreateSymbol(Name);
 }
 
-/// getPICBaseSymbol - Return a function-local symbol to represent the PIC
-/// base.
+/// Return a function-local symbol to represent the PIC base.
 MCSymbol *MachineFunction::getPICBaseSymbol() const {
-  const DataLayout *DL = getTarget().getDataLayout();
-  return Ctx.GetOrCreateSymbol(Twine(DL->getPrivateGlobalPrefix())+
-                               Twine(getFunctionNumber())+"$pb");
+  const DataLayout &DL = getDataLayout();
+  return Ctx.getOrCreateSymbol(Twine(DL.getPrivateGlobalPrefix()) +
+                               Twine(getFunctionNumber()) + "$pb");
 }
 
 //===----------------------------------------------------------------------===//
 //  MachineFrameInfo implementation
 //===----------------------------------------------------------------------===//
 
-/// ensureMaxAlignment - Make sure the function is at least Align bytes
-/// aligned.
+/// Make sure the function is at least Align bytes aligned.
 void MachineFrameInfo::ensureMaxAlignment(unsigned Align) {
   if (!StackRealignable || !RealignOption)
     assert(Align <= StackAlignment &&
@@ -492,7 +505,7 @@ void MachineFrameInfo::ensureMaxAlignment(unsigned Align) {
   if (MaxAlignment < Align) MaxAlignment = Align;
 }
 
-/// clampStackAlignment - Clamp the alignment if requested and emit a warning.
+/// Clamp the alignment if requested and emit a warning.
 static inline unsigned clampStackAlignment(bool ShouldClamp, unsigned Align,
                                            unsigned StackAlign) {
   if (!ShouldClamp || Align <= StackAlign)
@@ -503,9 +516,8 @@ static inline unsigned clampStackAlignment(bool ShouldClamp, unsigned Align,
   return StackAlign;
 }
 
-/// CreateStackObject - Create a new statically sized stack object, returning
-/// a nonnegative identifier to represent it.
-///
+/// Create a new statically sized stack object, returning a nonnegative
+/// identifier to represent it.
 int MachineFrameInfo::CreateStackObject(uint64_t Size, unsigned Alignment,
                       bool isSS, const AllocaInst *Alloca) {
   assert(Size != 0 && "Cannot allocate zero size stack objects!");
@@ -519,10 +531,8 @@ int MachineFrameInfo::CreateStackObject(uint64_t Size, unsigned Alignment,
   return Index;
 }
 
-/// CreateSpillStackObject - Create a new statically sized stack object that
-/// represents a spill slot, returning a nonnegative identifier to represent
-/// it.
-///
+/// Create a new statically sized stack object that represents a spill slot,
+/// returning a nonnegative identifier to represent it.
 int MachineFrameInfo::CreateSpillStackObject(uint64_t Size,
                                              unsigned Alignment) {
   Alignment = clampStackAlignment(!StackRealignable || !RealignOption,
@@ -533,11 +543,9 @@ int MachineFrameInfo::CreateSpillStackObject(uint64_t Size,
   return Index;
 }
 
-/// CreateVariableSizedObject - Notify the MachineFrameInfo object that a
-/// variable sized object has been created.  This must be created whenever a
-/// variable sized object is created, whether or not the index returned is
-/// actually used.
-///
+/// Notify the MachineFrameInfo object that a variable sized object has been
+/// created. This must be created whenever a variable sized object is created,
+/// whether or not the index returned is actually used.
 int MachineFrameInfo::CreateVariableSizedObject(unsigned Alignment,
                                                 const AllocaInst *Alloca) {
   HasVarSizedObjects = true;
@@ -548,11 +556,10 @@ int MachineFrameInfo::CreateVariableSizedObject(unsigned Alignment,
   return (int)Objects.size()-NumFixedObjects-1;
 }
 
-/// CreateFixedObject - Create a new object at a fixed location on the stack.
+/// Create a new object at a fixed location on the stack.
 /// All fixed objects should be created before other objects are created for
 /// efficiency. By default, fixed objects are immutable. This returns an
 /// index with a negative value.
-///
 int MachineFrameInfo::CreateFixedObject(uint64_t Size, int64_t SPOffset,
                                         bool Immutable, bool isAliased) {
   assert(Size != 0 && "Cannot allocate zero size fixed stack objects!");
@@ -569,8 +576,8 @@ int MachineFrameInfo::CreateFixedObject(uint64_t Size, int64_t SPOffset,
   return -++NumFixedObjects;
 }
 
-/// CreateFixedSpillStackObject - Create a spill slot at a fixed location
-/// on the stack.  Returns an index with a negative value.
+/// Create a spill slot at a fixed location on the stack.
+/// Returns an index with a negative value.
 int MachineFrameInfo::CreateFixedSpillStackObject(uint64_t Size,
                                                   int64_t SPOffset) {
   unsigned Align = MinAlign(SPOffset, StackAlignment);
@@ -584,12 +591,8 @@ int MachineFrameInfo::CreateFixedSpillStackObject(uint64_t Size,
   return -++NumFixedObjects;
 }
 
-BitVector
-MachineFrameInfo::getPristineRegs(const MachineBasicBlock *MBB) const {
-  assert(MBB && "MBB must be valid");
-  const MachineFunction *MF = MBB->getParent();
-  assert(MF && "MBB must be part of a MachineFunction");
-  const TargetRegisterInfo *TRI = MF->getSubtarget().getRegisterInfo();
+BitVector MachineFrameInfo::getPristineRegs(const MachineFunction &MF) const {
+  const TargetRegisterInfo *TRI = MF.getSubtarget().getRegisterInfo();
   BitVector BV(TRI->getNumRegs());
 
   // Before CSI is calculated, no registers are considered pristine. They can be
@@ -597,14 +600,10 @@ MachineFrameInfo::getPristineRegs(const MachineBasicBlock *MBB) const {
   if (!isCalleeSavedInfoValid())
     return BV;
 
-  for (const MCPhysReg *CSR = TRI->getCalleeSavedRegs(MF); CSR && *CSR; ++CSR)
+  for (const MCPhysReg *CSR = TRI->getCalleeSavedRegs(&MF); CSR && *CSR; ++CSR)
     BV.set(*CSR);
 
-  // The entry MBB always has all CSRs pristine.
-  if (MBB == &MF->front())
-    return BV;
-
-  // On other MBBs the saved CSRs are not pristine.
+  // Saved CSRs are not pristine.
   const std::vector<CalleeSavedInfo> &CSI = getCalleeSavedInfo();
   for (std::vector<CalleeSavedInfo>::const_iterator I = CSI.begin(),
          E = CSI.end(); I != E; ++I)
@@ -708,7 +707,7 @@ void MachineFrameInfo::dump(const MachineFunction &MF) const {
 //  MachineJumpTableInfo implementation
 //===----------------------------------------------------------------------===//
 
-/// getEntrySize - Return the size of each entry in the jump table.
+/// Return the size of each entry in the jump table.
 unsigned MachineJumpTableInfo::getEntrySize(const DataLayout &TD) const {
   // The size of a jump table entry is 4 bytes unless the entry is just the
   // address of a block, in which case it is the pointer size.
@@ -727,7 +726,7 @@ unsigned MachineJumpTableInfo::getEntrySize(const DataLayout &TD) const {
   llvm_unreachable("Unknown jump table encoding!");
 }
 
-/// getEntryAlignment - Return the alignment of each entry in the jump table.
+/// Return the alignment of each entry in the jump table.
 unsigned MachineJumpTableInfo::getEntryAlignment(const DataLayout &TD) const {
   // The alignment of a jump table entry is the alignment of int32 unless the
   // entry is just the address of a block, in which case it is the pointer
@@ -747,8 +746,7 @@ unsigned MachineJumpTableInfo::getEntryAlignment(const DataLayout &TD) const {
   llvm_unreachable("Unknown jump table encoding!");
 }
 
-/// createJumpTableIndex - Create a new jump table entry in the jump table info.
-///
+/// Create a new jump table entry in the jump table info.
 unsigned MachineJumpTableInfo::createJumpTableIndex(
                                const std::vector<MachineBasicBlock*> &DestBBs) {
   assert(!DestBBs.empty() && "Cannot create an empty jump table!");
@@ -756,8 +754,8 @@ unsigned MachineJumpTableInfo::createJumpTableIndex(
   return JumpTables.size()-1;
 }
 
-/// ReplaceMBBInJumpTables - If Old is the target of any jump tables, update
-/// the jump tables to branch to New instead.
+/// If Old is the target of any jump tables, update the jump tables to branch
+/// to New instead.
 bool MachineJumpTableInfo::ReplaceMBBInJumpTables(MachineBasicBlock *Old,
                                                   MachineBasicBlock *New) {
   assert(Old != New && "Not making a change?");
@@ -767,8 +765,8 @@ bool MachineJumpTableInfo::ReplaceMBBInJumpTables(MachineBasicBlock *Old,
   return MadeChange;
 }
 
-/// ReplaceMBBInJumpTable - If Old is a target of the jump tables, update
-/// the jump table to branch to New instead.
+/// If Old is a target of the jump tables, update the jump table to branch to
+/// New instead.
 bool MachineJumpTableInfo::ReplaceMBBInJumpTable(unsigned Idx,
                                                  MachineBasicBlock *Old,
                                                  MachineBasicBlock *New) {
@@ -808,10 +806,6 @@ void MachineJumpTableInfo::dump() const { print(dbgs()); }
 
 void MachineConstantPoolValue::anchor() { }
 
-const DataLayout *MachineConstantPool::getDataLayout() const {
-  return TM.getDataLayout();
-}
-
 Type *MachineConstantPoolEntry::getType() const {
   if (isMachineConstantPoolEntry())
     return Val.MachineCPVal->getType();
@@ -866,10 +860,10 @@ MachineConstantPool::~MachineConstantPool() {
     delete *I;
 }
 
-/// CanShareConstantPoolEntry - Test whether the given two constants
-/// can be allocated the same constant pool entry.
+/// Test whether the given two constants can be allocated the same constant pool
+/// entry.
 static bool CanShareConstantPoolEntry(const Constant *A, const Constant *B,
-                                      const DataLayout *TD) {
+                                      const DataLayout &DL) {
   // Handle the trivial case quickly.
   if (A == B) return true;
 
@@ -883,8 +877,8 @@ static bool CanShareConstantPoolEntry(const Constant *A, const Constant *B,
     return false;
 
   // For now, only support constants with the same size.
-  uint64_t StoreSize = TD->getTypeStoreSize(A->getType());
-  if (StoreSize != TD->getTypeStoreSize(B->getType()) || StoreSize > 128)
+  uint64_t StoreSize = DL.getTypeStoreSize(A->getType());
+  if (StoreSize != DL.getTypeStoreSize(B->getType()) || StoreSize > 128)
     return false;
 
   Type *IntTy = IntegerType::get(A->getContext(), StoreSize*8);
@@ -895,24 +889,22 @@ static bool CanShareConstantPoolEntry(const Constant *A, const Constant *B,
   // DataLayout.
   if (isa<PointerType>(A->getType()))
     A = ConstantFoldInstOperands(Instruction::PtrToInt, IntTy,
-                                 const_cast<Constant *>(A), *TD);
+                                 const_cast<Constant *>(A), DL);
   else if (A->getType() != IntTy)
     A = ConstantFoldInstOperands(Instruction::BitCast, IntTy,
-                                 const_cast<Constant *>(A), *TD);
+                                 const_cast<Constant *>(A), DL);
   if (isa<PointerType>(B->getType()))
     B = ConstantFoldInstOperands(Instruction::PtrToInt, IntTy,
-                                 const_cast<Constant *>(B), *TD);
+                                 const_cast<Constant *>(B), DL);
   else if (B->getType() != IntTy)
     B = ConstantFoldInstOperands(Instruction::BitCast, IntTy,
-                                 const_cast<Constant *>(B), *TD);
+                                 const_cast<Constant *>(B), DL);
 
   return A == B;
 }
 
-/// getConstantPoolIndex - Create a new entry in the constant pool or return
-/// an existing one.  User must specify the log2 of the minimum required
-/// alignment for the object.
-///
+/// Create a new entry in the constant pool or return an existing one.
+/// User must specify the log2 of the minimum required alignment for the object.
 unsigned MachineConstantPool::getConstantPoolIndex(const Constant *C,
                                                    unsigned Alignment) {
   assert(Alignment && "Alignment must be specified!");
@@ -923,8 +915,7 @@ unsigned MachineConstantPool::getConstantPoolIndex(const Constant *C,
   // FIXME, this could be made much more efficient for large constant pools.
   for (unsigned i = 0, e = Constants.size(); i != e; ++i)
     if (!Constants[i].isMachineConstantPoolEntry() &&
-        CanShareConstantPoolEntry(Constants[i].Val.ConstVal, C,
-                                  getDataLayout())) {
+        CanShareConstantPoolEntry(Constants[i].Val.ConstVal, C, DL)) {
       if ((unsigned)Constants[i].getAlignment() < Alignment)
         Constants[i].Alignment = Alignment;
       return i;
diff --git a/lib/CodeGen/MachineFunctionAnalysis.cpp b/lib/CodeGen/MachineFunctionAnalysis.cpp
index f6f34ba9d92..338cd1e2203 100644
--- a/lib/CodeGen/MachineFunctionAnalysis.cpp
+++ b/lib/CodeGen/MachineFunctionAnalysis.cpp
@@ -15,12 +15,14 @@
 #include "llvm/CodeGen/GCMetadata.h"
 #include "llvm/CodeGen/MachineFunction.h"
 #include "llvm/CodeGen/MachineModuleInfo.h"
+#include "llvm/CodeGen/MachineFunctionInitializer.h"
 using namespace llvm;
 
 char MachineFunctionAnalysis::ID = 0;
 
-MachineFunctionAnalysis::MachineFunctionAnalysis(const TargetMachine &tm) :
-  FunctionPass(ID), TM(tm), MF(nullptr) {
+MachineFunctionAnalysis::MachineFunctionAnalysis(
+    const TargetMachine &tm, MachineFunctionInitializer *MFInitializer)
+    : FunctionPass(ID), TM(tm), MF(nullptr), MFInitializer(MFInitializer) {
   initializeMachineModuleInfoPass(*PassRegistry::getPassRegistry());
 }
 
@@ -47,6 +49,8 @@ bool MachineFunctionAnalysis::runOnFunction(Function &F) {
   assert(!MF && "MachineFunctionAnalysis already initialized!");
   MF = new MachineFunction(&F, TM, NextFnNum++,
                            getAnalysis<MachineModuleInfo>());
+  if (MFInitializer)
+    MFInitializer->initializeMachineFunction(*MF);
   return false;
 }
 
diff --git a/lib/CodeGen/MachineInstr.cpp b/lib/CodeGen/MachineInstr.cpp
index d15411026cb..fdc4226ad92 100644
--- a/lib/CodeGen/MachineInstr.cpp
+++ b/lib/CodeGen/MachineInstr.cpp
@@ -28,6 +28,7 @@
 #include "llvm/IR/LLVMContext.h"
 #include "llvm/IR/Metadata.h"
 #include "llvm/IR/Module.h"
+#include "llvm/IR/ModuleSlotTracker.h"
 #include "llvm/IR/Type.h"
 #include "llvm/IR/Value.h"
 #include "llvm/MC/MCInstrDesc.h"
@@ -141,6 +142,28 @@ void MachineOperand::ChangeToFPImmediate(const ConstantFP *FPImm) {
   Contents.CFP = FPImm;
 }
 
+void MachineOperand::ChangeToES(const char *SymName, unsigned char TargetFlags) {
+  assert((!isReg() || !isTied()) &&
+         "Cannot change a tied operand into an external symbol");
+
+  removeRegFromUses();
+
+  OpKind = MO_ExternalSymbol;
+  Contents.OffsetedInfo.Val.SymbolName = SymName;
+  setOffset(0); // Offset is always 0.
+  setTargetFlags(TargetFlags);
+}
+
+void MachineOperand::ChangeToMCSymbol(MCSymbol *Sym) {
+  assert((!isReg() || !isTied()) &&
+         "Cannot change a tied operand into an MCSymbol");
+
+  removeRegFromUses();
+
+  OpKind = MO_MCSymbol;
+  Contents.Sym = Sym;
+}
+
 /// ChangeToRegister - Replace this operand with a new register operand of
 /// the specified value.  If an operand is known to be an register already,
 /// the setReg method should be used.
@@ -274,10 +297,14 @@ hash_code llvm::hash_value(const MachineOperand &MO) {
   llvm_unreachable("Invalid machine operand type");
 }
 
-/// print - Print the specified machine operand.
-///
 void MachineOperand::print(raw_ostream &OS,
                            const TargetRegisterInfo *TRI) const {
+  ModuleSlotTracker DummyMST(nullptr);
+  print(OS, DummyMST, TRI);
+}
+
+void MachineOperand::print(raw_ostream &OS, ModuleSlotTracker &MST,
+                           const TargetRegisterInfo *TRI) const {
   switch (getType()) {
   case MachineOperand::MO_Register:
     OS << PrintReg(getReg(), TRI, getSubReg());
@@ -299,8 +326,8 @@ void MachineOperand::print(raw_ostream &OS,
         if (isUndef() && getSubReg())
           OS << ",read-undef";
       } else if (isImplicit()) {
-          OS << "imp-use";
-          NeedComma = true;
+        OS << "imp-use";
+        NeedComma = true;
       }
 
       if (isKill()) {
@@ -365,7 +392,7 @@ void MachineOperand::print(raw_ostream &OS,
     break;
   case MachineOperand::MO_GlobalAddress:
     OS << "<ga:";
-    getGlobal()->printAsOperand(OS, /*PrintType=*/false);
+    getGlobal()->printAsOperand(OS, /*PrintType=*/false, MST);
     if (getOffset()) OS << "+" << getOffset();
     OS << '>';
     break;
@@ -376,7 +403,7 @@ void MachineOperand::print(raw_ostream &OS,
     break;
   case MachineOperand::MO_BlockAddress:
     OS << '<';
-    getBlockAddress()->printAsOperand(OS, /*PrintType=*/false);
+    getBlockAddress()->printAsOperand(OS, /*PrintType=*/false, MST);
     if (getOffset()) OS << "+" << getOffset();
     OS << '>';
     break;
@@ -388,7 +415,7 @@ void MachineOperand::print(raw_ostream &OS,
     break;
   case MachineOperand::MO_Metadata:
     OS << '<';
-    getMetadata()->printAsOperand(OS);
+    getMetadata()->printAsOperand(OS, MST);
     OS << '>';
     break;
   case MachineOperand::MO_MCSymbol:
@@ -483,63 +510,66 @@ uint64_t MachineMemOperand::getAlignment() const {
   return MinAlign(getBaseAlignment(), getOffset());
 }
 
-raw_ostream &llvm::operator<<(raw_ostream &OS, const MachineMemOperand &MMO) {
-  assert((MMO.isLoad() || MMO.isStore()) &&
+void MachineMemOperand::print(raw_ostream &OS) const {
+  ModuleSlotTracker DummyMST(nullptr);
+  print(OS, DummyMST);
+}
+void MachineMemOperand::print(raw_ostream &OS, ModuleSlotTracker &MST) const {
+  assert((isLoad() || isStore()) &&
          "SV has to be a load, store or both.");
 
-  if (MMO.isVolatile())
+  if (isVolatile())
     OS << "Volatile ";
 
-  if (MMO.isLoad())
+  if (isLoad())
     OS << "LD";
-  if (MMO.isStore())
+  if (isStore())
     OS << "ST";
-  OS << MMO.getSize();
+  OS << getSize();
 
   // Print the address information.
   OS << "[";
-  if (const Value *V = MMO.getValue())
-    V->printAsOperand(OS, /*PrintType=*/false);
-  else if (const PseudoSourceValue *PSV = MMO.getPseudoValue())
+  if (const Value *V = getValue())
+    V->printAsOperand(OS, /*PrintType=*/false, MST);
+  else if (const PseudoSourceValue *PSV = getPseudoValue())
     PSV->printCustom(OS);
   else
     OS << "<unknown>";
 
-  unsigned AS = MMO.getAddrSpace();
+  unsigned AS = getAddrSpace();
   if (AS != 0)
     OS << "(addrspace=" << AS << ')';
 
   // If the alignment of the memory reference itself differs from the alignment
   // of the base pointer, print the base alignment explicitly, next to the base
   // pointer.
-  if (MMO.getBaseAlignment() != MMO.getAlignment())
-    OS << "(align=" << MMO.getBaseAlignment() << ")";
+  if (getBaseAlignment() != getAlignment())
+    OS << "(align=" << getBaseAlignment() << ")";
 
-  if (MMO.getOffset() != 0)
-    OS << "+" << MMO.getOffset();
+  if (getOffset() != 0)
+    OS << "+" << getOffset();
   OS << "]";
 
   // Print the alignment of the reference.
-  if (MMO.getBaseAlignment() != MMO.getAlignment() ||
-      MMO.getBaseAlignment() != MMO.getSize())
-    OS << "(align=" << MMO.getAlignment() << ")";
+  if (getBaseAlignment() != getAlignment() || getBaseAlignment() != getSize())
+    OS << "(align=" << getAlignment() << ")";
 
   // Print TBAA info.
-  if (const MDNode *TBAAInfo = MMO.getAAInfo().TBAA) {
+  if (const MDNode *TBAAInfo = getAAInfo().TBAA) {
     OS << "(tbaa=";
     if (TBAAInfo->getNumOperands() > 0)
-      TBAAInfo->getOperand(0)->printAsOperand(OS);
+      TBAAInfo->getOperand(0)->printAsOperand(OS, MST);
     else
       OS << "<unknown>";
     OS << ")";
   }
 
   // Print AA scope info.
-  if (const MDNode *ScopeInfo = MMO.getAAInfo().Scope) {
+  if (const MDNode *ScopeInfo = getAAInfo().Scope) {
     OS << "(alias.scope=";
     if (ScopeInfo->getNumOperands() > 0)
       for (unsigned i = 0, ie = ScopeInfo->getNumOperands(); i != ie; ++i) {
-        ScopeInfo->getOperand(i)->printAsOperand(OS);
+        ScopeInfo->getOperand(i)->printAsOperand(OS, MST);
         if (i != ie-1)
           OS << ",";
       }
@@ -549,11 +579,11 @@ raw_ostream &llvm::operator<<(raw_ostream &OS, const MachineMemOperand &MMO) {
   }
 
   // Print AA noalias scope info.
-  if (const MDNode *NoAliasInfo = MMO.getAAInfo().NoAlias) {
+  if (const MDNode *NoAliasInfo = getAAInfo().NoAlias) {
     OS << "(noalias=";
     if (NoAliasInfo->getNumOperands() > 0)
       for (unsigned i = 0, ie = NoAliasInfo->getNumOperands(); i != ie; ++i) {
-        NoAliasInfo->getOperand(i)->printAsOperand(OS);
+        NoAliasInfo->getOperand(i)->printAsOperand(OS, MST);
         if (i != ie-1)
           OS << ",";
       }
@@ -563,10 +593,11 @@ raw_ostream &llvm::operator<<(raw_ostream &OS, const MachineMemOperand &MMO) {
   }
 
   // Print nontemporal info.
-  if (MMO.isNonTemporal())
+  if (isNonTemporal())
     OS << "(nontemporal)";
 
-  return OS;
+  if (isInvariant())
+    OS << "(invariant)";
 }
 
 //===----------------------------------------------------------------------===//
@@ -1070,9 +1101,8 @@ const TargetRegisterClass *MachineInstr::getRegClassConstraintEffectForVReg(
           OpndIt.getOperandNo(), Reg, CurRC, TII, TRI);
   else
     // Otherwise, just check the current operands.
-    for (ConstMIOperands OpndIt(this); OpndIt.isValid() && CurRC; ++OpndIt)
-      CurRC = getRegClassConstraintEffectForVRegImpl(OpndIt.getOperandNo(), Reg,
-                                                     CurRC, TII, TRI);
+    for (unsigned i = 0, e = NumOperands; i < e && CurRC; ++i)
+      CurRC = getRegClassConstraintEffectForVRegImpl(i, Reg, CurRC, TII, TRI);
   return CurRC;
 }
 
@@ -1344,9 +1374,7 @@ void MachineInstr::substituteRegister(unsigned FromReg,
 /// isSafeToMove - Return true if it is safe to move this instruction. If
 /// SawStore is set to true, it means that there is a store (or call) between
 /// the instruction's location and its intended destination.
-bool MachineInstr::isSafeToMove(const TargetInstrInfo *TII,
-                                AliasAnalysis *AA,
-                                bool &SawStore) const {
+bool MachineInstr::isSafeToMove(AliasAnalysis *AA, bool &SawStore) const {
   // Ignore stuff that we obviously can't move.
   //
   // Treat volatile loads as stores. This is not strictly necessary for
@@ -1431,9 +1459,9 @@ bool MachineInstr::isInvariantLoad(AliasAnalysis *AA) const {
 
     if (const Value *V = (*I)->getValue()) {
       // If we have an AliasAnalysis, ask it whether the memory is constant.
-      if (AA && AA->pointsToConstantMemory(
-                      AliasAnalysis::Location(V, (*I)->getSize(),
-                                              (*I)->getAAInfo())))
+      if (AA &&
+          AA->pointsToConstantMemory(
+              MemoryLocation(V, (*I)->getSize(), (*I)->getAAInfo())))
         continue;
     }
 
@@ -1504,6 +1532,17 @@ void MachineInstr::dump() const {
 }
 
 void MachineInstr::print(raw_ostream &OS, bool SkipOpers) const {
+  const Module *M = nullptr;
+  if (const MachineBasicBlock *MBB = getParent())
+    if (const MachineFunction *MF = MBB->getParent())
+      M = MF->getFunction()->getParent();
+
+  ModuleSlotTracker MST(M);
+  print(OS, MST, SkipOpers);
+}
+
+void MachineInstr::print(raw_ostream &OS, ModuleSlotTracker &MST,
+                         bool SkipOpers) const {
   // We can be a bit tidier if we know the MachineFunction.
   const MachineFunction *MF = nullptr;
   const TargetRegisterInfo *TRI = nullptr;
@@ -1528,7 +1567,7 @@ void MachineInstr::print(raw_ostream &OS, bool SkipOpers) const {
          !getOperand(StartOp).isImplicit();
        ++StartOp) {
     if (StartOp != 0) OS << ", ";
-    getOperand(StartOp).print(OS, TRI);
+    getOperand(StartOp).print(OS, MST, TRI);
     unsigned Reg = getOperand(StartOp).getReg();
     if (TargetRegisterInfo::isVirtualRegister(Reg))
       VirtRegs.push_back(Reg);
@@ -1555,7 +1594,7 @@ void MachineInstr::print(raw_ostream &OS, bool SkipOpers) const {
   if (isInlineAsm() && e >= InlineAsm::MIOp_FirstOperand) {
     // Print asm string.
     OS << " ";
-    getOperand(InlineAsm::MIOp_AsmString).print(OS, TRI);
+    getOperand(InlineAsm::MIOp_AsmString).print(OS, MST, TRI);
 
     // Print HasSideEffects, MayLoad, MayStore, IsAlignStack
     unsigned ExtraInfo = getOperand(InlineAsm::MIOp_ExtraInfo).getImm();
@@ -1619,11 +1658,11 @@ void MachineInstr::print(raw_ostream &OS, bool SkipOpers) const {
     }
     if (isDebugValue() && MO.isMetadata()) {
       // Pretty print DBG_VALUE instructions.
-      DIVariable DIV = dyn_cast<MDLocalVariable>(MO.getMetadata());
+      auto *DIV = dyn_cast<DILocalVariable>(MO.getMetadata());
       if (DIV && !DIV->getName().empty())
         OS << "!\"" << DIV->getName() << '\"';
       else
-        MO.print(OS, TRI);
+        MO.print(OS, MST, TRI);
     } else if (TRI && (isInsertSubreg() || isRegSequence()) && MO.isImm()) {
       OS << TRI->getSubRegIndexName(MO.getImm());
     } else if (i == AsmDescOp && MO.isImm()) {
@@ -1657,7 +1696,7 @@ void MachineInstr::print(raw_ostream &OS, bool SkipOpers) const {
       // Compute the index of the next operand descriptor.
       AsmDescOp += 1 + InlineAsm::getNumOperandRegisters(Flag);
     } else
-      MO.print(OS, TRI);
+      MO.print(OS, MST, TRI);
   }
 
   // Briefly indicate whether any call clobbers were omitted.
@@ -1682,7 +1721,7 @@ void MachineInstr::print(raw_ostream &OS, bool SkipOpers) const {
     OS << " mem:";
     for (mmo_iterator i = memoperands_begin(), e = memoperands_end();
          i != e; ++i) {
-      OS << **i;
+      (*i)->print(OS, MST);
       if (std::next(i) != e)
         OS << " ";
     }
@@ -1710,7 +1749,7 @@ void MachineInstr::print(raw_ostream &OS, bool SkipOpers) const {
   // Print debug location information.
   if (isDebugValue() && getOperand(e - 2).isMetadata()) {
     if (!HaveSemi) OS << ";";
-    DIVariable DV = cast<MDLocalVariable>(getOperand(e - 2).getMetadata());
+    auto *DV = cast<DILocalVariable>(getOperand(e - 2).getMetadata());
     OS << " line no:" <<  DV->getLine();
     if (auto *InlinedAt = debugLoc->getInlinedAt()) {
       DebugLoc InlinedAtDL(InlinedAt);
diff --git a/lib/CodeGen/MachineInstrBundle.cpp b/lib/CodeGen/MachineInstrBundle.cpp
index 0690f08d495..cd820ee1ac5 100644
--- a/lib/CodeGen/MachineInstrBundle.cpp
+++ b/lib/CodeGen/MachineInstrBundle.cpp
@@ -23,11 +23,15 @@ namespace {
   class UnpackMachineBundles : public MachineFunctionPass {
   public:
     static char ID; // Pass identification
-    UnpackMachineBundles() : MachineFunctionPass(ID) {
+    UnpackMachineBundles(std::function<bool(const Function &)> Ftor = nullptr)
+        : MachineFunctionPass(ID), PredicateFtor(Ftor) {
       initializeUnpackMachineBundlesPass(*PassRegistry::getPassRegistry());
     }
 
     bool runOnMachineFunction(MachineFunction &MF) override;
+
+  private:
+    std::function<bool(const Function &)> PredicateFtor;
   };
 } // end anonymous namespace
 
@@ -37,6 +41,9 @@ INITIALIZE_PASS(UnpackMachineBundles, "unpack-mi-bundles",
                 "Unpack machine instruction bundles", false, false)
 
 bool UnpackMachineBundles::runOnMachineFunction(MachineFunction &MF) {
+  if (PredicateFtor && !PredicateFtor(*MF.getFunction()))
+    return false;
+
   bool Changed = false;
   for (MachineFunction::iterator I = MF.begin(), E = MF.end(); I != E; ++I) {
     MachineBasicBlock *MBB = &*I;
@@ -69,6 +76,10 @@ bool UnpackMachineBundles::runOnMachineFunction(MachineFunction &MF) {
   return Changed;
 }
 
+FunctionPass *
+llvm::createUnpackMachineBundles(std::function<bool(const Function &)> Ftor) {
+  return new UnpackMachineBundles(Ftor);
+}
 
 namespace {
   class FinalizeMachineBundles : public MachineFunctionPass {
diff --git a/lib/CodeGen/MachineLICM.cpp b/lib/CodeGen/MachineLICM.cpp
index 9756f13e7e5..e9ea5ed9648 100644
--- a/lib/CodeGen/MachineLICM.cpp
+++ b/lib/CodeGen/MachineLICM.cpp
@@ -27,7 +27,7 @@
 #include "llvm/CodeGen/MachineMemOperand.h"
 #include "llvm/CodeGen/MachineRegisterInfo.h"
 #include "llvm/CodeGen/PseudoSourceValue.h"
-#include "llvm/MC/MCInstrItineraries.h"
+#include "llvm/CodeGen/TargetSchedule.h"
 #include "llvm/Support/CommandLine.h"
 #include "llvm/Support/Debug.h"
 #include "llvm/Support/raw_ostream.h"
@@ -74,7 +74,7 @@ namespace {
     const TargetRegisterInfo *TRI;
     const MachineFrameInfo *MFI;
     MachineRegisterInfo *MRI;
-    const InstrItineraryData *InstrItins;
+    TargetSchedModel SchedModel;
     bool PreRegAlloc;
 
     // Various analyses that we use...
@@ -338,12 +338,13 @@ bool MachineLICM::runOnMachineFunction(MachineFunction &MF) {
     return false;
 
   Changed = FirstInLoop = false;
-  TII = MF.getSubtarget().getInstrInfo();
-  TLI = MF.getSubtarget().getTargetLowering();
-  TRI = MF.getSubtarget().getRegisterInfo();
+  const TargetSubtargetInfo &ST = MF.getSubtarget();
+  TII = ST.getInstrInfo();
+  TLI = ST.getTargetLowering();
+  TRI = ST.getRegisterInfo();
   MFI = MF.getFrameInfo();
   MRI = &MF.getRegInfo();
-  InstrItins = MF.getSubtarget().getInstrItineraryData();
+  SchedModel.init(ST.getSchedModel(), &ST, TII);
 
   PreRegAlloc = MRI->isSSA();
 
@@ -934,7 +935,7 @@ static bool isLoadFromGOTOrConstantPool(MachineInstr &MI) {
 bool MachineLICM::IsLICMCandidate(MachineInstr &I) {
   // Check if it's safe to move the instruction.
   bool DontMoveAcrossStore = true;
-  if (!I.isSafeToMove(TII, AA, DontMoveAcrossStore))
+  if (!I.isSafeToMove(AA, DontMoveAcrossStore))
     return false;
 
   // If it is load then check if it is guaranteed to execute by making sure that
@@ -1012,10 +1013,10 @@ bool MachineLICM::HasLoopPHIUse(const MachineInstr *MI) const {
   SmallVector<const MachineInstr*, 8> Work(1, MI);
   do {
     MI = Work.pop_back_val();
-    for (ConstMIOperands MO(MI); MO.isValid(); ++MO) {
-      if (!MO->isReg() || !MO->isDef())
+    for (const MachineOperand &MO : MI->operands()) {
+      if (!MO.isReg() || !MO.isDef())
         continue;
-      unsigned Reg = MO->getReg();
+      unsigned Reg = MO.getReg();
       if (!TargetRegisterInfo::isVirtualRegister(Reg))
         continue;
       for (MachineInstr &UseMI : MRI->use_instructions(Reg)) {
@@ -1046,7 +1047,7 @@ bool MachineLICM::HasLoopPHIUse(const MachineInstr *MI) const {
 /// it 'high'.
 bool MachineLICM::HasHighOperandLatency(MachineInstr &MI,
                                         unsigned DefIdx, unsigned Reg) const {
-  if (!InstrItins || InstrItins->isEmpty() || MRI->use_nodbg_empty(Reg))
+  if (MRI->use_nodbg_empty(Reg))
     return false;
 
   for (MachineInstr &UseMI : MRI->use_nodbg_instructions(Reg)) {
@@ -1062,7 +1063,7 @@ bool MachineLICM::HasHighOperandLatency(MachineInstr &MI,
       if (MOReg != Reg)
         continue;
 
-      if (TII->hasHighOperandLatency(InstrItins, MRI, &MI, DefIdx, &UseMI, i))
+      if (TII->hasHighOperandLatency(SchedModel, MRI, &MI, DefIdx, &UseMI, i))
         return true;
     }
 
@@ -1078,8 +1079,6 @@ bool MachineLICM::HasHighOperandLatency(MachineInstr &MI,
 bool MachineLICM::IsCheapInstruction(MachineInstr &MI) const {
   if (TII->isAsCheapAsAMove(&MI) || MI.isCopyLike())
     return true;
-  if (!InstrItins || InstrItins->isEmpty())
-    return false;
 
   bool isCheap = false;
   unsigned NumDefs = MI.getDesc().getNumDefs();
@@ -1092,7 +1091,7 @@ bool MachineLICM::IsCheapInstruction(MachineInstr &MI) const {
     if (TargetRegisterInfo::isPhysicalRegister(Reg))
       continue;
 
-    if (!TII->hasLowDefLatency(InstrItins, &MI, i))
+    if (!TII->hasLowDefLatency(SchedModel, &MI, i))
       return false;
     isCheap = true;
   }
diff --git a/lib/CodeGen/MachineLoopInfo.cpp b/lib/CodeGen/MachineLoopInfo.cpp
index ce6abdd870b..e19e6e30a01 100644
--- a/lib/CodeGen/MachineLoopInfo.cpp
+++ b/lib/CodeGen/MachineLoopInfo.cpp
@@ -37,7 +37,7 @@ char &llvm::MachineLoopInfoID = MachineLoopInfo::ID;
 
 bool MachineLoopInfo::runOnMachineFunction(MachineFunction &) {
   releaseMemory();
-  LI.Analyze(getAnalysis<MachineDominatorTree>().getBase());
+  LI.analyze(getAnalysis<MachineDominatorTree>().getBase());
   return false;
 }
 
diff --git a/lib/CodeGen/MachineModuleInfo.cpp b/lib/CodeGen/MachineModuleInfo.cpp
index e8bd1f8e6d9..6a206249d83 100644
--- a/lib/CodeGen/MachineModuleInfo.cpp
+++ b/lib/CodeGen/MachineModuleInfo.cpp
@@ -54,9 +54,8 @@ class MMIAddrLabelMapCallbackPtr : CallbackVH {
 class MMIAddrLabelMap {
   MCContext &Context;
   struct AddrLabelSymEntry {
-    /// Symbols - The symbols for the label.  This is a pointer union that is
-    /// either one symbol (the common case) or a list of symbols.
-    PointerUnion<MCSymbol *, std::vector<MCSymbol*>*> Symbols;
+    /// Symbols - The symbols for the label.
+    TinyPtrVector<MCSymbol *> Symbols;
 
     Function *Fn;   // The containing function of the BasicBlock.
     unsigned Index; // The index in BBCallbacks for the BasicBlock.
@@ -80,16 +79,9 @@ class MMIAddrLabelMap {
   ~MMIAddrLabelMap() {
     assert(DeletedAddrLabelsNeedingEmission.empty() &&
            "Some labels for deleted blocks never got emitted");
-
-    // Deallocate any of the 'list of symbols' case.
-    for (DenseMap<AssertingVH<BasicBlock>, AddrLabelSymEntry>::iterator
-         I = AddrLabelSymbols.begin(), E = AddrLabelSymbols.end(); I != E; ++I)
-      if (I->second.Symbols.is<std::vector<MCSymbol*>*>())
-        delete I->second.Symbols.get<std::vector<MCSymbol*>*>();
   }
 
-  MCSymbol *getAddrLabelSymbol(BasicBlock *BB);
-  std::vector<MCSymbol*> getAddrLabelSymbolToEmit(BasicBlock *BB);
+  ArrayRef<MCSymbol *> getAddrLabelSymbolToEmit(BasicBlock *BB);
 
   void takeDeletedSymbolsForFunction(Function *F,
                                      std::vector<MCSymbol*> &Result);
@@ -99,49 +91,27 @@ class MMIAddrLabelMap {
 };
 }
 
-MCSymbol *MMIAddrLabelMap::getAddrLabelSymbol(BasicBlock *BB) {
+ArrayRef<MCSymbol *> MMIAddrLabelMap::getAddrLabelSymbolToEmit(BasicBlock *BB) {
   assert(BB->hasAddressTaken() &&
          "Shouldn't get label for block without address taken");
   AddrLabelSymEntry &Entry = AddrLabelSymbols[BB];
 
   // If we already had an entry for this block, just return it.
-  if (!Entry.Symbols.isNull()) {
+  if (!Entry.Symbols.empty()) {
     assert(BB->getParent() == Entry.Fn && "Parent changed");
-    if (Entry.Symbols.is<MCSymbol*>())
-      return Entry.Symbols.get<MCSymbol*>();
-    return (*Entry.Symbols.get<std::vector<MCSymbol*>*>())[0];
+    return Entry.Symbols;
   }
 
   // Otherwise, this is a new entry, create a new symbol for it and add an
   // entry to BBCallbacks so we can be notified if the BB is deleted or RAUWd.
-  BBCallbacks.push_back(BB);
+  BBCallbacks.emplace_back(BB);
   BBCallbacks.back().setMap(this);
-  Entry.Index = BBCallbacks.size()-1;
+  Entry.Index = BBCallbacks.size() - 1;
   Entry.Fn = BB->getParent();
-  MCSymbol *Result = Context.CreateTempSymbol();
-  Entry.Symbols = Result;
-  return Result;
-}
-
-std::vector<MCSymbol*>
-MMIAddrLabelMap::getAddrLabelSymbolToEmit(BasicBlock *BB) {
-  assert(BB->hasAddressTaken() &&
-         "Shouldn't get label for block without address taken");
-  AddrLabelSymEntry &Entry = AddrLabelSymbols[BB];
-
-  std::vector<MCSymbol*> Result;
-
-  // If we already had an entry for this block, just return it.
-  if (Entry.Symbols.isNull())
-    Result.push_back(getAddrLabelSymbol(BB));
-  else if (MCSymbol *Sym = Entry.Symbols.dyn_cast<MCSymbol*>())
-    Result.push_back(Sym);
-  else
-    Result = *Entry.Symbols.get<std::vector<MCSymbol*>*>();
-  return Result;
+  Entry.Symbols.push_back(Context.createTempSymbol());
+  return Entry.Symbols;
 }
 
-
 /// takeDeletedSymbolsForFunction - If we have any deleted symbols for F, return
 /// them.
 void MMIAddrLabelMap::
@@ -162,16 +132,15 @@ void MMIAddrLabelMap::UpdateForDeletedBlock(BasicBlock *BB) {
   // If the block got deleted, there is no need for the symbol.  If the symbol
   // was already emitted, we can just forget about it, otherwise we need to
   // queue it up for later emission when the function is output.
-  AddrLabelSymEntry Entry = AddrLabelSymbols[BB];
+  AddrLabelSymEntry Entry = std::move(AddrLabelSymbols[BB]);
   AddrLabelSymbols.erase(BB);
-  assert(!Entry.Symbols.isNull() && "Didn't have a symbol, why a callback?");
+  assert(!Entry.Symbols.empty() && "Didn't have a symbol, why a callback?");
   BBCallbacks[Entry.Index] = nullptr;  // Clear the callback.
 
   assert((BB->getParent() == nullptr || BB->getParent() == Entry.Fn) &&
          "Block/parent mismatch");
 
-  // Handle both the single and the multiple symbols cases.
-  if (MCSymbol *Sym = Entry.Symbols.dyn_cast<MCSymbol*>()) {
+  for (MCSymbol *Sym : Entry.Symbols) {
     if (Sym->isDefined())
       return;
 
@@ -180,64 +149,29 @@ void MMIAddrLabelMap::UpdateForDeletedBlock(BasicBlock *BB) {
     // for the containing Function.  Since the block is being deleted, its
     // parent may already be removed, we have to get the function from 'Entry'.
     DeletedAddrLabelsNeedingEmission[Entry.Fn].push_back(Sym);
-  } else {
-    std::vector<MCSymbol*> *Syms = Entry.Symbols.get<std::vector<MCSymbol*>*>();
-
-    for (unsigned i = 0, e = Syms->size(); i != e; ++i) {
-      MCSymbol *Sym = (*Syms)[i];
-      if (Sym->isDefined()) continue;  // Ignore already emitted labels.
-
-      // If the block is not yet defined, we need to emit it at the end of the
-      // function.  Add the symbol to the DeletedAddrLabelsNeedingEmission list
-      // for the containing Function.  Since the block is being deleted, its
-      // parent may already be removed, we have to get the function from
-      // 'Entry'.
-      DeletedAddrLabelsNeedingEmission[Entry.Fn].push_back(Sym);
-    }
-
-    // The entry is deleted, free the memory associated with the symbol list.
-    delete Syms;
   }
 }
 
 void MMIAddrLabelMap::UpdateForRAUWBlock(BasicBlock *Old, BasicBlock *New) {
   // Get the entry for the RAUW'd block and remove it from our map.
-  AddrLabelSymEntry OldEntry = AddrLabelSymbols[Old];
+  AddrLabelSymEntry OldEntry = std::move(AddrLabelSymbols[Old]);
   AddrLabelSymbols.erase(Old);
-  assert(!OldEntry.Symbols.isNull() && "Didn't have a symbol, why a callback?");
+  assert(!OldEntry.Symbols.empty() && "Didn't have a symbol, why a callback?");
 
   AddrLabelSymEntry &NewEntry = AddrLabelSymbols[New];
 
   // If New is not address taken, just move our symbol over to it.
-  if (NewEntry.Symbols.isNull()) {
+  if (NewEntry.Symbols.empty()) {
     BBCallbacks[OldEntry.Index].setPtr(New);    // Update the callback.
-    NewEntry = OldEntry;     // Set New's entry.
+    NewEntry = std::move(OldEntry);             // Set New's entry.
     return;
   }
 
   BBCallbacks[OldEntry.Index] = nullptr;    // Update the callback.
 
-  // Otherwise, we need to add the old symbol to the new block's set.  If it is
-  // just a single entry, upgrade it to a symbol list.
-  if (MCSymbol *PrevSym = NewEntry.Symbols.dyn_cast<MCSymbol*>()) {
-    std::vector<MCSymbol*> *SymList = new std::vector<MCSymbol*>();
-    SymList->push_back(PrevSym);
-    NewEntry.Symbols = SymList;
-  }
-
-  std::vector<MCSymbol*> *SymList =
-    NewEntry.Symbols.get<std::vector<MCSymbol*>*>();
-
-  // If the old entry was a single symbol, add it.
-  if (MCSymbol *Sym = OldEntry.Symbols.dyn_cast<MCSymbol*>()) {
-    SymList->push_back(Sym);
-    return;
-  }
-
-  // Otherwise, concatenate the list.
-  std::vector<MCSymbol*> *Syms =OldEntry.Symbols.get<std::vector<MCSymbol*>*>();
-  SymList->insert(SymList->end(), Syms->begin(), Syms->end());
-  delete Syms;
+  // Otherwise, we need to add the old symbols to the new block's set.
+  NewEntry.Symbols.insert(NewEntry.Symbols.end(), OldEntry.Symbols.begin(),
+                          OldEntry.Symbols.end());
 }
 
 
@@ -273,8 +207,8 @@ bool MachineModuleInfo::doInitialization(Module &M) {
 
   ObjFileMMI = nullptr;
   CurCallSite = 0;
-  CallsEHReturn = 0;
-  CallsUnwindInit = 0;
+  CallsEHReturn = false;
+  CallsUnwindInit = false;
   DbgInfoAvailable = UsesVAFloatArgument = UsesMorestackAddr = false;
   // Always emit some info, by default "no personality" info.
   Personalities.push_back(nullptr);
@@ -308,50 +242,23 @@ void MachineModuleInfo::EndFunction() {
 
   // Clean up exception info.
   LandingPads.clear();
+  PersonalityTypeCache = EHPersonality::Unknown;
   CallSiteMap.clear();
   TypeInfos.clear();
   FilterIds.clear();
   FilterEnds.clear();
-  CallsEHReturn = 0;
-  CallsUnwindInit = 0;
+  CallsEHReturn = false;
+  CallsUnwindInit = false;
   VariableDbgInfos.clear();
 }
 
-/// AnalyzeModule - Scan the module for global debug information.
-///
-void MachineModuleInfo::AnalyzeModule(const Module &M) {
-  // Insert functions in the llvm.used array (but not llvm.compiler.used) into
-  // UsedFunctions.
-  const GlobalVariable *GV = M.getGlobalVariable("llvm.used");
-  if (!GV || !GV->hasInitializer()) return;
-
-  // Should be an array of 'i8*'.
-  const ConstantArray *InitList = cast<ConstantArray>(GV->getInitializer());
-
-  for (unsigned i = 0, e = InitList->getNumOperands(); i != e; ++i)
-    if (const Function *F =
-          dyn_cast<Function>(InitList->getOperand(i)->stripPointerCasts()))
-      UsedFunctions.insert(F);
-}
-
 //===- Address of Block Management ----------------------------------------===//
 
-
-/// getAddrLabelSymbol - Return the symbol to be used for the specified basic
-/// block when its address is taken.  This cannot be its normal LBB label
-/// because the block may be accessed outside its containing function.
-MCSymbol *MachineModuleInfo::getAddrLabelSymbol(const BasicBlock *BB) {
-  // Lazily create AddrLabelSymbols.
-  if (!AddrLabelSymbols)
-    AddrLabelSymbols = new MMIAddrLabelMap(Context);
-  return AddrLabelSymbols->getAddrLabelSymbol(const_cast<BasicBlock*>(BB));
-}
-
 /// getAddrLabelSymbolToEmit - Return the symbol to be used for the specified
 /// basic block when its address is taken.  If other blocks were RAUW'd to
 /// this one, we may have to emit them as well, return the whole set.
-std::vector<MCSymbol*> MachineModuleInfo::
-getAddrLabelSymbolToEmit(const BasicBlock *BB) {
+ArrayRef<MCSymbol *>
+MachineModuleInfo::getAddrLabelSymbolToEmit(const BasicBlock *BB) {
   // Lazily create AddrLabelSymbols.
   if (!AddrLabelSymbols)
     AddrLabelSymbols = new MMIAddrLabelMap(Context);
@@ -401,7 +308,7 @@ void MachineModuleInfo::addInvoke(MachineBasicBlock *LandingPad,
 /// addLandingPad - Provide the label of a try LandingPad block.
 ///
 MCSymbol *MachineModuleInfo::addLandingPad(MachineBasicBlock *LandingPad) {
-  MCSymbol *LandingPadLabel = Context.CreateTempSymbol();
+  MCSymbol *LandingPadLabel = Context.createTempSymbol();
   LandingPadInfo &LP = getOrCreateLandingPadInfo(LandingPad);
   LP.LandingPadLabel = LandingPadLabel;
   return LandingPadLabel;
@@ -413,7 +320,10 @@ void MachineModuleInfo::addPersonality(MachineBasicBlock *LandingPad,
                                        const Function *Personality) {
   LandingPadInfo &LP = getOrCreateLandingPadInfo(LandingPad);
   LP.Personality = Personality;
+  addPersonality(Personality);
+}
 
+void MachineModuleInfo::addPersonality(const Function *Personality) {
   for (unsigned i = 0; i < Personalities.size(); ++i)
     if (Personalities[i] == Personality)
       return;
@@ -461,12 +371,23 @@ void MachineModuleInfo::addCleanup(MachineBasicBlock *LandingPad) {
   LP.TypeIds.push_back(0);
 }
 
-MCSymbol *
-MachineModuleInfo::addClauseForLandingPad(MachineBasicBlock *LandingPad) {
-  MCSymbol *ClauseLabel = Context.CreateTempSymbol();
+void MachineModuleInfo::addSEHCatchHandler(MachineBasicBlock *LandingPad,
+                                           const Function *Filter,
+                                           const BlockAddress *RecoverBA) {
+  LandingPadInfo &LP = getOrCreateLandingPadInfo(LandingPad);
+  SEHHandler Handler;
+  Handler.FilterOrFinally = Filter;
+  Handler.RecoverBA = RecoverBA;
+  LP.SEHHandlers.push_back(Handler);
+}
+
+void MachineModuleInfo::addSEHCleanupHandler(MachineBasicBlock *LandingPad,
+                                             const Function *Cleanup) {
   LandingPadInfo &LP = getOrCreateLandingPadInfo(LandingPad);
-  LP.ClauseLabels.push_back(ClauseLabel);
-  return ClauseLabel;
+  SEHHandler Handler;
+  Handler.FilterOrFinally = Cleanup;
+  Handler.RecoverBA = nullptr;
+  LP.SEHHandlers.push_back(Handler);
 }
 
 /// TidyLandingPads - Remap landing pad labels and remove any deleted landing
diff --git a/lib/CodeGen/MachineRegisterInfo.cpp b/lib/CodeGen/MachineRegisterInfo.cpp
index 278a8f24d63..d3704efb631 100644
--- a/lib/CodeGen/MachineRegisterInfo.cpp
+++ b/lib/CodeGen/MachineRegisterInfo.cpp
@@ -13,6 +13,7 @@
 
 #include "llvm/CodeGen/MachineRegisterInfo.h"
 #include "llvm/CodeGen/MachineInstrBuilder.h"
+#include "llvm/IR/Function.h"
 #include "llvm/Support/raw_os_ostream.h"
 #include "llvm/Target/TargetInstrInfo.h"
 #include "llvm/Target/TargetMachine.h"
@@ -28,7 +29,6 @@ MachineRegisterInfo::MachineRegisterInfo(const MachineFunction *MF)
     TracksSubRegLiveness(false) {
   VRegInfo.reserve(256);
   RegAllocHints.reserve(256);
-  UsedRegUnits.resize(getTargetRegisterInfo()->getNumRegUnits());
   UsedPhysRegMask.resize(getTargetRegisterInfo()->getNumRegs());
 
   // Create the physreg use/def lists.
@@ -116,6 +116,8 @@ void MachineRegisterInfo::clearVirtRegs() {
   }
 #endif
   VRegInfo.clear();
+  for (auto &I : LiveIns)
+    I.second = 0;
 }
 
 void MachineRegisterInfo::verifyUseList(unsigned Reg) const {
@@ -441,3 +443,49 @@ void MachineRegisterInfo::markUsesInDebugValueAsUndef(unsigned Reg) const {
       UseMI->getOperand(0).setReg(0U);
   }
 }
+
+static const Function *getCalledFunction(const MachineInstr &MI) {
+  for (const MachineOperand &MO : MI.operands()) {
+    if (!MO.isGlobal())
+      continue;
+    const Function *Func = dyn_cast<Function>(MO.getGlobal());
+    if (Func != nullptr)
+      return Func;
+  }
+  return nullptr;
+}
+
+static bool isNoReturnDef(const MachineOperand &MO) {
+  // Anything which is not a noreturn function is a real def.
+  const MachineInstr &MI = *MO.getParent();
+  if (!MI.isCall())
+    return false;
+  const MachineBasicBlock &MBB = *MI.getParent();
+  if (!MBB.succ_empty())
+    return false;
+  const MachineFunction &MF = *MBB.getParent();
+  // We need to keep correct unwind information even if the function will
+  // not return, since the runtime may need it.
+  if (MF.getFunction()->hasFnAttribute(Attribute::UWTable))
+    return false;
+  const Function *Called = getCalledFunction(MI);
+  if (Called == nullptr || !Called->hasFnAttribute(Attribute::NoReturn)
+      || !Called->hasFnAttribute(Attribute::NoUnwind))
+    return false;
+
+  return true;
+}
+
+bool MachineRegisterInfo::isPhysRegModified(unsigned PhysReg) const {
+  if (UsedPhysRegMask.test(PhysReg))
+    return true;
+  const TargetRegisterInfo *TRI = getTargetRegisterInfo();
+  for (MCRegAliasIterator AI(PhysReg, TRI, true); AI.isValid(); ++AI) {
+    for (const MachineOperand &MO : make_range(def_begin(*AI), def_end())) {
+      if (isNoReturnDef(MO))
+        continue;
+      return true;
+    }
+  }
+  return false;
+}
diff --git a/lib/CodeGen/MachineScheduler.cpp b/lib/CodeGen/MachineScheduler.cpp
index a52d05fa6fa..f4823681cf2 100644
--- a/lib/CodeGen/MachineScheduler.cpp
+++ b/lib/CodeGen/MachineScheduler.cpp
@@ -347,7 +347,7 @@ bool PostMachineScheduler::runOnMachineFunction(MachineFunction &mf) {
   if (skipOptnoneFunction(*mf.getFunction()))
     return false;
 
-  if (!mf.getSubtarget().enablePostMachineScheduler()) {
+  if (!mf.getSubtarget().enablePostRAScheduler()) {
     DEBUG(dbgs() << "Subtarget disables post-MI-sched.\n");
     return false;
   }
@@ -943,8 +943,9 @@ updateScheduledPressure(const SUnit *SU,
     unsigned Limit = RegClassInfo->getRegPressureSetLimit(ID);
     if (NewMaxPressure[ID] >= Limit - 2) {
       DEBUG(dbgs() << "  " << TRI->getRegPressureSetName(ID) << ": "
-            << NewMaxPressure[ID] << " > " << Limit << "(+ "
-            << BotRPTracker.getLiveThru()[ID] << " livethru)\n");
+            << NewMaxPressure[ID]
+            << ((NewMaxPressure[ID] > Limit) ? " > " : " <= ") << Limit
+            << "(+ " << BotRPTracker.getLiveThru()[ID] << " livethru)\n");
     }
   }
 }
@@ -1270,7 +1271,7 @@ void LoadClusterMutation::clusterNeighboringLoads(ArrayRef<SUnit*> Loads,
     SUnit *SU = Loads[Idx];
     unsigned BaseReg;
     unsigned Offset;
-    if (TII->getLdStBaseRegImmOfs(SU->getInstr(), BaseReg, Offset, TRI))
+    if (TII->getMemOpBaseRegImmOfs(SU->getInstr(), BaseReg, Offset, TRI))
       LoadRecords.push_back(LoadInfo(SU, BaseReg, Offset));
   }
   if (LoadRecords.size() < 2)
@@ -1348,25 +1349,49 @@ namespace {
 /// \brief Post-process the DAG to create cluster edges between instructions
 /// that may be fused by the processor into a single operation.
 class MacroFusion : public ScheduleDAGMutation {
-  const TargetInstrInfo *TII;
+  const TargetInstrInfo &TII;
+  const TargetRegisterInfo &TRI;
 public:
-  MacroFusion(const TargetInstrInfo *tii): TII(tii) {}
+  MacroFusion(const TargetInstrInfo &TII, const TargetRegisterInfo &TRI)
+    : TII(TII), TRI(TRI) {}
 
   void apply(ScheduleDAGMI *DAG) override;
 };
 } // anonymous
 
+/// Returns true if \p MI reads a register written by \p Other.
+static bool HasDataDep(const TargetRegisterInfo &TRI, const MachineInstr &MI,
+                       const MachineInstr &Other) {
+  for (const MachineOperand &MO : MI.uses()) {
+    if (!MO.isReg() || !MO.readsReg())
+      continue;
+
+    unsigned Reg = MO.getReg();
+    if (Other.modifiesRegister(Reg, &TRI))
+      return true;
+  }
+  return false;
+}
+
 /// \brief Callback from DAG postProcessing to create cluster edges to encourage
 /// fused operations.
 void MacroFusion::apply(ScheduleDAGMI *DAG) {
   // For now, assume targets can only fuse with the branch.
-  MachineInstr *Branch = DAG->ExitSU.getInstr();
+  SUnit &ExitSU = DAG->ExitSU;
+  MachineInstr *Branch = ExitSU.getInstr();
   if (!Branch)
     return;
 
-  for (unsigned Idx = DAG->SUnits.size(); Idx > 0;) {
-    SUnit *SU = &DAG->SUnits[--Idx];
-    if (!TII->shouldScheduleAdjacent(SU->getInstr(), Branch))
+  for (SUnit &SU : DAG->SUnits) {
+    // SUnits with successors can't be schedule in front of the ExitSU.
+    if (!SU.Succs.empty())
+      continue;
+    // We only care if the node writes to a register that the branch reads.
+    MachineInstr *Pred = SU.getInstr();
+    if (!HasDataDep(TRI, *Branch, *Pred))
+      continue;
+
+    if (!TII.shouldScheduleAdjacent(Pred, Branch))
       continue;
 
     // Create a single weak edge from SU to ExitSU. The only effect is to cause
@@ -1375,11 +1400,11 @@ void MacroFusion::apply(ScheduleDAGMI *DAG) {
     // scheduling cannot prioritize ExitSU anyway. To defer top-down scheduling
     // of SU, we could create an artificial edge from the deepest root, but it
     // hasn't been needed yet.
-    bool Success = DAG->addEdge(&DAG->ExitSU, SDep(SU, SDep::Cluster));
+    bool Success = DAG->addEdge(&ExitSU, SDep(&SU, SDep::Cluster));
     (void)Success;
     assert(Success && "No DAG nodes should be reachable from ExitSU");
 
-    DEBUG(dbgs() << "Macro Fuse SU(" << SU->NodeNum << ")\n");
+    DEBUG(dbgs() << "Macro Fuse SU(" << SU.NodeNum << ")\n");
     break;
   }
 }
@@ -2149,7 +2174,7 @@ void GenericSchedulerBase::setPolicy(CandPolicy &Policy,
                                      bool IsPostRA,
                                      SchedBoundary &CurrZone,
                                      SchedBoundary *OtherZone) {
-  // Apply preemptive heuristics based on the the total latency and resources
+  // Apply preemptive heuristics based on the total latency and resources
   // inside and outside this zone. Potential stalls should be considered before
   // following this policy.
 
@@ -2611,8 +2636,7 @@ void GenericScheduler::tryCandidate(SchedCandidate &Cand,
                  TryCand, Cand, PhysRegCopy))
     return;
 
-  // Avoid exceeding the target's limit. If signed PSetID is negative, it is
-  // invalid; convert it to INT_MAX to give it lowest priority.
+  // Avoid exceeding the target's limit.
   if (DAG->isTrackingPressure() && tryPressure(TryCand.RPDelta.Excess,
                                                Cand.RPDelta.Excess,
                                                TryCand, Cand, RegExcess))
@@ -2887,7 +2911,7 @@ static ScheduleDAGInstrs *createGenericSchedLive(MachineSchedContext *C) {
   if (EnableLoadCluster && DAG->TII->enableClusterLoads())
     DAG->addMutation(make_unique<LoadClusterMutation>(DAG->TII, DAG->TRI));
   if (EnableMacroFusion)
-    DAG->addMutation(make_unique<MacroFusion>(DAG->TII));
+    DAG->addMutation(make_unique<MacroFusion>(*DAG->TII, *DAG->TRI));
   return DAG;
 }
 
diff --git a/lib/CodeGen/MachineSink.cpp b/lib/CodeGen/MachineSink.cpp
index 8337793d960..1b9be50068a 100644
--- a/lib/CodeGen/MachineSink.cpp
+++ b/lib/CodeGen/MachineSink.cpp
@@ -19,6 +19,7 @@
 #include "llvm/CodeGen/Passes.h"
 #include "llvm/ADT/SetVector.h"
 #include "llvm/ADT/SmallSet.h"
+#include "llvm/ADT/SparseBitVector.h"
 #include "llvm/ADT/Statistic.h"
 #include "llvm/Analysis/AliasAnalysis.h"
 #include "llvm/CodeGen/MachineBlockFrequencyInfo.h"
@@ -70,6 +71,11 @@ namespace {
     // will be split.
     SetVector<std::pair<MachineBasicBlock*,MachineBasicBlock*> > ToSplit;
 
+    SparseBitVector<> RegsToClearKillFlags;
+
+    typedef std::map<MachineBasicBlock *, SmallVector<MachineBasicBlock *, 4>>
+        AllSuccsCache;
+
   public:
     static char ID; // Pass identification
     MachineSinking() : MachineFunctionPass(ID) {
@@ -117,18 +123,24 @@ namespace {
                                    MachineBasicBlock *From,
                                    MachineBasicBlock *To,
                                    bool BreakPHIEdge);
-    bool SinkInstruction(MachineInstr *MI, bool &SawStore);
+    bool SinkInstruction(MachineInstr *MI, bool &SawStore,
+                         AllSuccsCache &AllSuccessors);
     bool AllUsesDominatedByBlock(unsigned Reg, MachineBasicBlock *MBB,
                                  MachineBasicBlock *DefMBB,
                                  bool &BreakPHIEdge, bool &LocalUse) const;
     MachineBasicBlock *FindSuccToSinkTo(MachineInstr *MI, MachineBasicBlock *MBB,
-               bool &BreakPHIEdge);
+               bool &BreakPHIEdge, AllSuccsCache &AllSuccessors);
     bool isProfitableToSinkTo(unsigned Reg, MachineInstr *MI,
                               MachineBasicBlock *MBB,
-                              MachineBasicBlock *SuccToSinkTo);
+                              MachineBasicBlock *SuccToSinkTo,
+                              AllSuccsCache &AllSuccessors);
 
     bool PerformTrivialForwardCoalescing(MachineInstr *MI,
                                          MachineBasicBlock *MBB);
+
+    SmallVector<MachineBasicBlock *, 4> &
+    GetAllSortedSuccessors(MachineInstr *MI, MachineBasicBlock *MBB,
+                           AllSuccsCache &AllSuccessors) const;
   };
 } // end anonymous namespace
 
@@ -266,9 +278,8 @@ bool MachineSinking::runOnMachineFunction(MachineFunction &MF) {
     // Process all basic blocks.
     CEBCandidates.clear();
     ToSplit.clear();
-    for (MachineFunction::iterator I = MF.begin(), E = MF.end();
-         I != E; ++I)
-      MadeChange |= ProcessBlock(*I);
+    for (auto &MBB: MF)
+      MadeChange |= ProcessBlock(MBB);
 
     // If we have anything we marked as toSplit, split it now.
     for (auto &Pair : ToSplit) {
@@ -287,6 +298,12 @@ bool MachineSinking::runOnMachineFunction(MachineFunction &MF) {
     if (!MadeChange) break;
     EverMadeChange = true;
   }
+
+  // Now clear any kill flags for recorded registers.
+  for (auto I : RegsToClearKillFlags)
+    MRI->clearKillFlags(I);
+  RegsToClearKillFlags.clear();
+
   return EverMadeChange;
 }
 
@@ -301,6 +318,9 @@ bool MachineSinking::ProcessBlock(MachineBasicBlock &MBB) {
 
   bool MadeChange = false;
 
+  // Cache all successors, sorted by frequency info and loop depth.
+  AllSuccsCache AllSuccessors;
+
   // Walk the basic block bottom-up.  Remember if we saw a store.
   MachineBasicBlock::iterator I = MBB.end();
   --I;
@@ -323,7 +343,7 @@ bool MachineSinking::ProcessBlock(MachineBasicBlock &MBB) {
       continue;
     }
 
-    if (SinkInstruction(MI, SawStore))
+    if (SinkInstruction(MI, SawStore, AllSuccessors))
       ++NumSunk, MadeChange = true;
 
     // If we just processed the first instruction in the block, we're done.
@@ -475,7 +495,8 @@ static void collectDebugValues(MachineInstr *MI,
 /// isProfitableToSinkTo - Return true if it is profitable to sink MI.
 bool MachineSinking::isProfitableToSinkTo(unsigned Reg, MachineInstr *MI,
                                           MachineBasicBlock *MBB,
-                                          MachineBasicBlock *SuccToSinkTo) {
+                                          MachineBasicBlock *SuccToSinkTo,
+                                          AllSuccsCache &AllSuccessors) {
   assert (MI && "Invalid MachineInstr!");
   assert (SuccToSinkTo && "Invalid SinkTo Candidate BB");
 
@@ -505,18 +526,66 @@ bool MachineSinking::isProfitableToSinkTo(unsigned Reg, MachineInstr *MI,
   // can further profitably sinked into another block in next round.
   bool BreakPHIEdge = false;
   // FIXME - If finding successor is compile time expensive then cache results.
-  if (MachineBasicBlock *MBB2 = FindSuccToSinkTo(MI, SuccToSinkTo, BreakPHIEdge))
-    return isProfitableToSinkTo(Reg, MI, SuccToSinkTo, MBB2);
+  if (MachineBasicBlock *MBB2 =
+          FindSuccToSinkTo(MI, SuccToSinkTo, BreakPHIEdge, AllSuccessors))
+    return isProfitableToSinkTo(Reg, MI, SuccToSinkTo, MBB2, AllSuccessors);
 
   // If SuccToSinkTo is final destination and it is a post dominator of current
   // block then it is not profitable to sink MI into SuccToSinkTo block.
   return false;
 }
 
+/// Get the sorted sequence of successors for this MachineBasicBlock, possibly
+/// computing it if it was not already cached.
+SmallVector<MachineBasicBlock *, 4> &
+MachineSinking::GetAllSortedSuccessors(MachineInstr *MI, MachineBasicBlock *MBB,
+                                       AllSuccsCache &AllSuccessors) const {
+
+  // Do we have the sorted successors in cache ?
+  auto Succs = AllSuccessors.find(MBB);
+  if (Succs != AllSuccessors.end())
+    return Succs->second;
+
+  SmallVector<MachineBasicBlock *, 4> AllSuccs(MBB->succ_begin(),
+                                               MBB->succ_end());
+
+  // Handle cases where sinking can happen but where the sink point isn't a
+  // successor. For example:
+  //
+  //   x = computation
+  //   if () {} else {}
+  //   use x
+  //
+  const std::vector<MachineDomTreeNode *> &Children =
+    DT->getNode(MBB)->getChildren();
+  for (const auto &DTChild : Children)
+    // DomTree children of MBB that have MBB as immediate dominator are added.
+    if (DTChild->getIDom()->getBlock() == MI->getParent() &&
+        // Skip MBBs already added to the AllSuccs vector above.
+        !MBB->isSuccessor(DTChild->getBlock()))
+      AllSuccs.push_back(DTChild->getBlock());
+
+  // Sort Successors according to their loop depth or block frequency info.
+  std::stable_sort(
+      AllSuccs.begin(), AllSuccs.end(),
+      [this](const MachineBasicBlock *L, const MachineBasicBlock *R) {
+        uint64_t LHSFreq = MBFI ? MBFI->getBlockFreq(L).getFrequency() : 0;
+        uint64_t RHSFreq = MBFI ? MBFI->getBlockFreq(R).getFrequency() : 0;
+        bool HasBlockFreq = LHSFreq != 0 && RHSFreq != 0;
+        return HasBlockFreq ? LHSFreq < RHSFreq
+                            : LI->getLoopDepth(L) < LI->getLoopDepth(R);
+      });
+
+  auto it = AllSuccessors.insert(std::make_pair(MBB, AllSuccs));
+
+  return it.first->second;
+}
+
 /// FindSuccToSinkTo - Find a successor to sink this instruction to.
 MachineBasicBlock *MachineSinking::FindSuccToSinkTo(MachineInstr *MI,
                                    MachineBasicBlock *MBB,
-                                   bool &BreakPHIEdge) {
+                                   bool &BreakPHIEdge,
+                                   AllSuccsCache &AllSuccessors) {
 
   assert (MI && "Invalid MachineInstr!");
   assert (MBB && "Invalid MachineBasicBlock!");
@@ -570,38 +639,8 @@ MachineBasicBlock *MachineSinking::FindSuccToSinkTo(MachineInstr *MI,
       // we should sink to. If we have reliable block frequency information
       // (frequency != 0) available, give successors with smaller frequencies
       // higher priority, otherwise prioritize smaller loop depths.
-      SmallVector<MachineBasicBlock*, 4> Succs(MBB->succ_begin(),
-                                               MBB->succ_end());
-
-      // Handle cases where sinking can happen but where the sink point isn't a
-      // successor. For example:
-      //
-      //   x = computation
-      //   if () {} else {}
-      //   use x
-      //
-      const std::vector<MachineDomTreeNode *> &Children =
-        DT->getNode(MBB)->getChildren();
-      for (const auto &DTChild : Children)
-        // DomTree children of MBB that have MBB as immediate dominator are added.
-        if (DTChild->getIDom()->getBlock() == MI->getParent() &&
-            // Skip MBBs already added to the Succs vector above.
-            !MBB->isSuccessor(DTChild->getBlock()))
-          Succs.push_back(DTChild->getBlock());
-
-      // Sort Successors according to their loop depth or block frequency info.
-      std::stable_sort(
-          Succs.begin(), Succs.end(),
-          [this](const MachineBasicBlock *L, const MachineBasicBlock *R) {
-            uint64_t LHSFreq = MBFI ? MBFI->getBlockFreq(L).getFrequency() : 0;
-            uint64_t RHSFreq = MBFI ? MBFI->getBlockFreq(R).getFrequency() : 0;
-            bool HasBlockFreq = LHSFreq != 0 && RHSFreq != 0;
-            return HasBlockFreq ? LHSFreq < RHSFreq
-                                : LI->getLoopDepth(L) < LI->getLoopDepth(R);
-          });
-      for (SmallVectorImpl<MachineBasicBlock *>::iterator SI = Succs.begin(),
-             E = Succs.end(); SI != E; ++SI) {
-        MachineBasicBlock *SuccBlock = *SI;
+      for (MachineBasicBlock *SuccBlock :
+           GetAllSortedSuccessors(MI, MBB, AllSuccessors)) {
         bool LocalUse = false;
         if (AllUsesDominatedByBlock(Reg, SuccBlock, MBB,
                                     BreakPHIEdge, LocalUse)) {
@@ -616,7 +655,7 @@ MachineBasicBlock *MachineSinking::FindSuccToSinkTo(MachineInstr *MI,
       // If we couldn't find a block to sink to, ignore this instruction.
       if (!SuccToSinkTo)
         return nullptr;
-      if (!isProfitableToSinkTo(Reg, MI, MBB, SuccToSinkTo))
+      if (!isProfitableToSinkTo(Reg, MI, MBB, SuccToSinkTo, AllSuccessors))
         return nullptr;
     }
   }
@@ -636,14 +675,19 @@ MachineBasicBlock *MachineSinking::FindSuccToSinkTo(MachineInstr *MI,
 
 /// SinkInstruction - Determine whether it is safe to sink the specified machine
 /// instruction out of its current block into a successor.
-bool MachineSinking::SinkInstruction(MachineInstr *MI, bool &SawStore) {
+bool MachineSinking::SinkInstruction(MachineInstr *MI, bool &SawStore,
+                                     AllSuccsCache &AllSuccessors) {
   // Don't sink insert_subreg, subreg_to_reg, reg_sequence. These are meant to
   // be close to the source to make it easier to coalesce.
   if (AvoidsSinking(MI, MRI))
     return false;
 
   // Check if it's safe to move the instruction.
-  if (!MI->isSafeToMove(TII, AA, SawStore))
+  if (!MI->isSafeToMove(AA, SawStore))
+    return false;
+
+  // Convergent operations may only be moved to control equivalent locations.
+  if (MI->isConvergent())
     return false;
 
   // FIXME: This should include support for sinking instructions within the
@@ -656,7 +700,8 @@ bool MachineSinking::SinkInstruction(MachineInstr *MI, bool &SawStore) {
 
   bool BreakPHIEdge = false;
   MachineBasicBlock *ParentBlock = MI->getParent();
-  MachineBasicBlock *SuccToSinkTo = FindSuccToSinkTo(MI, ParentBlock, BreakPHIEdge);
+  MachineBasicBlock *SuccToSinkTo =
+      FindSuccToSinkTo(MI, ParentBlock, BreakPHIEdge, AllSuccessors);
 
   // If there are no outputs, it must have side-effects.
   if (!SuccToSinkTo)
@@ -684,7 +729,7 @@ bool MachineSinking::SinkInstruction(MachineInstr *MI, bool &SawStore) {
     // other code paths.
     bool TryBreak = false;
     bool store = true;
-    if (!MI->isSafeToMove(TII, AA, store)) {
+    if (!MI->isSafeToMove(AA, store)) {
       DEBUG(dbgs() << " *** NOTE: Won't sink load along critical edge.\n");
       TryBreak = true;
     }
@@ -755,7 +800,13 @@ bool MachineSinking::SinkInstruction(MachineInstr *MI, bool &SawStore) {
 
   // Conservatively, clear any kill flags, since it's possible that they are no
   // longer correct.
-  MI->clearKillInfo();
+  // Note that we have to clear the kill flags for any register this instruction
+  // uses as we may sink over another instruction which currently kills the
+  // used registers.
+  for (MachineOperand &MO : MI->operands()) {
+    if (MO.isReg() && MO.isUse())
+      RegsToClearKillFlags.set(MO.getReg()); // Remember to clear kill flags.
+  }
 
   return true;
 }
diff --git a/lib/CodeGen/MachineTraceMetrics.cpp b/lib/CodeGen/MachineTraceMetrics.cpp
index 5dc7c216994..d9a6b68462e 100644
--- a/lib/CodeGen/MachineTraceMetrics.cpp
+++ b/lib/CodeGen/MachineTraceMetrics.cpp
@@ -320,9 +320,7 @@ MinInstrCountEnsemble::pickTracePred(const MachineBasicBlock *MBB) {
   unsigned CurCount = MTM.getResources(MBB)->InstrCount;
   const MachineBasicBlock *Best = nullptr;
   unsigned BestDepth = 0;
-  for (MachineBasicBlock::const_pred_iterator
-       I = MBB->pred_begin(), E = MBB->pred_end(); I != E; ++I) {
-    const MachineBasicBlock *Pred = *I;
+  for (const MachineBasicBlock *Pred : MBB->predecessors()) {
     const MachineTraceMetrics::TraceBlockInfo *PredTBI =
       getDepthResources(Pred);
     // Ignore cycles that aren't natural loops.
@@ -344,9 +342,7 @@ MinInstrCountEnsemble::pickTraceSucc(const MachineBasicBlock *MBB) {
   const MachineLoop *CurLoop = getLoopFor(MBB);
   const MachineBasicBlock *Best = nullptr;
   unsigned BestHeight = 0;
-  for (MachineBasicBlock::const_succ_iterator
-       I = MBB->succ_begin(), E = MBB->succ_end(); I != E; ++I) {
-    const MachineBasicBlock *Succ = *I;
+  for (const MachineBasicBlock *Succ : MBB->successors()) {
     // Don't consider back-edges.
     if (CurLoop && Succ == CurLoop->getHeader())
       continue;
@@ -513,18 +509,17 @@ MachineTraceMetrics::Ensemble::invalidate(const MachineBasicBlock *BadMBB) {
             << " height.\n");
       // Find any MBB predecessors that have MBB as their preferred successor.
       // They are the only ones that need to be invalidated.
-      for (MachineBasicBlock::const_pred_iterator
-           I = MBB->pred_begin(), E = MBB->pred_end(); I != E; ++I) {
-        TraceBlockInfo &TBI = BlockInfo[(*I)->getNumber()];
+      for (const MachineBasicBlock *Pred : MBB->predecessors()) {
+        TraceBlockInfo &TBI = BlockInfo[Pred->getNumber()];
         if (!TBI.hasValidHeight())
           continue;
         if (TBI.Succ == MBB) {
           TBI.invalidateHeight();
-          WorkList.push_back(*I);
+          WorkList.push_back(Pred);
           continue;
         }
         // Verify that TBI.Succ is actually a *I successor.
-        assert((!TBI.Succ || (*I)->isSuccessor(TBI.Succ)) && "CFG changed");
+        assert((!TBI.Succ || Pred->isSuccessor(TBI.Succ)) && "CFG changed");
       }
     } while (!WorkList.empty());
   }
@@ -539,18 +534,17 @@ MachineTraceMetrics::Ensemble::invalidate(const MachineBasicBlock *BadMBB) {
             << " depth.\n");
       // Find any MBB successors that have MBB as their preferred predecessor.
       // They are the only ones that need to be invalidated.
-      for (MachineBasicBlock::const_succ_iterator
-           I = MBB->succ_begin(), E = MBB->succ_end(); I != E; ++I) {
-        TraceBlockInfo &TBI = BlockInfo[(*I)->getNumber()];
+      for (const MachineBasicBlock *Succ : MBB->successors()) {
+        TraceBlockInfo &TBI = BlockInfo[Succ->getNumber()];
         if (!TBI.hasValidDepth())
           continue;
         if (TBI.Pred == MBB) {
           TBI.invalidateDepth();
-          WorkList.push_back(*I);
+          WorkList.push_back(Succ);
           continue;
         }
         // Verify that TBI.Pred is actually a *I predecessor.
-        assert((!TBI.Pred || (*I)->isPredecessor(TBI.Pred)) && "CFG changed");
+        assert((!TBI.Pred || Succ->isPredecessor(TBI.Pred)) && "CFG changed");
       }
     } while (!WorkList.empty());
   }
@@ -630,11 +624,17 @@ struct DataDep {
 static bool getDataDeps(const MachineInstr *UseMI,
                         SmallVectorImpl<DataDep> &Deps,
                         const MachineRegisterInfo *MRI) {
+  // Debug values should not be included in any calculations.
+  if (UseMI->isDebugValue())
+    return false;
+  
   bool HasPhysRegs = false;
-  for (ConstMIOperands MO(UseMI); MO.isValid(); ++MO) {
-    if (!MO->isReg())
+  for (MachineInstr::const_mop_iterator I = UseMI->operands_begin(),
+       E = UseMI->operands_end(); I != E; ++I) {
+    const MachineOperand &MO = *I;
+    if (!MO.isReg())
       continue;
-    unsigned Reg = MO->getReg();
+    unsigned Reg = MO.getReg();
     if (!Reg)
       continue;
     if (TargetRegisterInfo::isPhysicalRegister(Reg)) {
@@ -642,8 +642,8 @@ static bool getDataDeps(const MachineInstr *UseMI,
       continue;
     }
     // Collect virtual register reads.
-    if (MO->readsReg())
-      Deps.push_back(DataDep(MRI, Reg, MO.getOperandNo()));
+    if (MO.readsReg())
+      Deps.push_back(DataDep(MRI, Reg, UseMI->getOperandNo(I)));
   }
   return HasPhysRegs;
 }
@@ -694,28 +694,30 @@ static void updatePhysDepsDownwards(const MachineInstr *UseMI,
   SmallVector<unsigned, 8> Kills;
   SmallVector<unsigned, 8> LiveDefOps;
 
-  for (ConstMIOperands MO(UseMI); MO.isValid(); ++MO) {
-    if (!MO->isReg())
+  for (MachineInstr::const_mop_iterator MI = UseMI->operands_begin(),
+       ME = UseMI->operands_end(); MI != ME; ++MI) {
+    const MachineOperand &MO = *MI;
+    if (!MO.isReg())
       continue;
-    unsigned Reg = MO->getReg();
+    unsigned Reg = MO.getReg();
     if (!TargetRegisterInfo::isPhysicalRegister(Reg))
       continue;
     // Track live defs and kills for updating RegUnits.
-    if (MO->isDef()) {
-      if (MO->isDead())
+    if (MO.isDef()) {
+      if (MO.isDead())
         Kills.push_back(Reg);
       else
-        LiveDefOps.push_back(MO.getOperandNo());
-    } else if (MO->isKill())
+        LiveDefOps.push_back(UseMI->getOperandNo(MI));
+    } else if (MO.isKill())
       Kills.push_back(Reg);
     // Identify dependencies.
-    if (!MO->readsReg())
+    if (!MO.readsReg())
       continue;
     for (MCRegUnitIterator Units(Reg, TRI); Units.isValid(); ++Units) {
       SparseSet<LiveRegUnit>::iterator I = RegUnits.find(*Units);
       if (I == RegUnits.end())
         continue;
-      Deps.push_back(DataDep(I->MI, I->Op, MO.getOperandNo()));
+      Deps.push_back(DataDep(I->MI, I->Op, UseMI->getOperandNo(MI)));
       break;
     }
   }
@@ -829,8 +831,7 @@ computeInstrDepths(const MachineBasicBlock *MBB) {
 
       // Filter and process dependencies, computing the earliest issue cycle.
       unsigned Cycle = 0;
-      for (unsigned i = 0, e = Deps.size(); i != e; ++i) {
-        const DataDep &Dep = Deps[i];
+      for (const DataDep &Dep : Deps) {
         const TraceBlockInfo&DepTBI =
           BlockInfo[Dep.DefMI->getParent()->getNumber()];
         // Ignore dependencies from outside the current trace.
@@ -868,15 +869,18 @@ static unsigned updatePhysDepsUpwards(const MachineInstr *MI, unsigned Height,
                                       const TargetInstrInfo *TII,
                                       const TargetRegisterInfo *TRI) {
   SmallVector<unsigned, 8> ReadOps;
-  for (ConstMIOperands MO(MI); MO.isValid(); ++MO) {
-    if (!MO->isReg())
+
+  for (MachineInstr::const_mop_iterator MOI = MI->operands_begin(),
+       MOE = MI->operands_end(); MOI != MOE; ++MOI) {
+    const MachineOperand &MO = *MOI;
+    if (!MO.isReg())
       continue;
-    unsigned Reg = MO->getReg();
+    unsigned Reg = MO.getReg();
     if (!TargetRegisterInfo::isPhysicalRegister(Reg))
       continue;
-    if (MO->readsReg())
-      ReadOps.push_back(MO.getOperandNo());
-    if (!MO->isDef())
+    if (MO.readsReg())
+      ReadOps.push_back(MI->getOperandNo(MOI));
+    if (!MO.isDef())
       continue;
     // This is a def of Reg. Remove corresponding entries from RegUnits, and
     // update MI Height to consider the physreg dependencies.
@@ -889,7 +893,7 @@ static unsigned updatePhysDepsUpwards(const MachineInstr *MI, unsigned Height,
         // We may not know the UseMI of this dependency, if it came from the
         // live-in list. SchedModel can handle a NULL UseMI.
         DepHeight += SchedModel
-          .computeOperandLatency(MI, MO.getOperandNo(), I->MI, I->Op);
+          .computeOperandLatency(MI, MI->getOperandNo(MOI), I->MI, I->Op);
       }
       Height = std::max(Height, DepHeight);
       // This regunit is dead above MI.
@@ -996,8 +1000,7 @@ computeInstrHeights(const MachineBasicBlock *MBB) {
   // MBB is the highest precomputed block in the trace.
   if (MBB) {
     TraceBlockInfo &TBI = BlockInfo[MBB->getNumber()];
-    for (unsigned i = 0, e = TBI.LiveIns.size(); i != e; ++i) {
-      LiveInReg LI = TBI.LiveIns[i];
+    for (LiveInReg &LI : TBI.LiveIns) {
       if (TargetRegisterInfo::isVirtualRegister(LI.Reg)) {
         // For virtual registers, the def latency is included.
         unsigned &Height = Heights[MTM.MRI->getVRegDef(LI.Reg)];
@@ -1085,9 +1088,9 @@ computeInstrHeights(const MachineBasicBlock *MBB) {
                                       MTM.SchedModel, MTM.TII, MTM.TRI);
 
       // Update the required height of any virtual registers read by MI.
-      for (unsigned i = 0, e = Deps.size(); i != e; ++i)
-        if (pushDepHeight(Deps[i], MI, Cycle, Heights, MTM.SchedModel, MTM.TII))
-          addLiveIns(Deps[i].DefMI, Deps[i].DefOp, Stack);
+      for (const DataDep &Dep : Deps)
+        if (pushDepHeight(Dep, MI, Cycle, Heights, MTM.SchedModel, MTM.TII))
+          addLiveIns(Dep.DefMI, Dep.DefOp, Stack);
 
       InstrCycles &MICycles = Cycles[MI];
       MICycles.Height = Cycle;
@@ -1103,8 +1106,7 @@ computeInstrHeights(const MachineBasicBlock *MBB) {
     // Update virtual live-in heights. They were added by addLiveIns() with a 0
     // height because the final height isn't known until now.
     DEBUG(dbgs() << "BB#" << MBB->getNumber() <<  " Live-ins:");
-    for (unsigned i = 0, e = TBI.LiveIns.size(); i != e; ++i) {
-      LiveInReg &LIR = TBI.LiveIns[i];
+    for (LiveInReg &LIR : TBI.LiveIns) {
       const MachineInstr *DefMI = MTM.MRI->getVRegDef(LIR.Reg);
       LIR.Height = Heights.lookup(DefMI);
       DEBUG(dbgs() << ' ' << PrintReg(LIR.Reg) << '@' << LIR.Height);
@@ -1130,11 +1132,16 @@ computeInstrHeights(const MachineBasicBlock *MBB) {
 
 MachineTraceMetrics::Trace
 MachineTraceMetrics::Ensemble::getTrace(const MachineBasicBlock *MBB) {
-  // FIXME: Check cache tags, recompute as needed.
-  computeTrace(MBB);
-  computeInstrDepths(MBB);
-  computeInstrHeights(MBB);
-  return Trace(*this, BlockInfo[MBB->getNumber()]);
+  TraceBlockInfo &TBI = BlockInfo[MBB->getNumber()];
+
+  if (!TBI.hasValidDepth() || !TBI.hasValidHeight())
+    computeTrace(MBB);
+  if (!TBI.HasValidInstrDepths)
+    computeInstrDepths(MBB);
+  if (!TBI.HasValidInstrHeights)
+    computeInstrHeights(MBB);
+  
+  return Trace(*this, TBI);
 }
 
 unsigned
@@ -1203,8 +1210,7 @@ unsigned MachineTraceMetrics::Trace::getResourceLength(
                             unsigned ResourceIdx)
                          ->unsigned {
     unsigned Cycles = 0;
-    for (unsigned I = 0; I != Instrs.size(); ++I) {
-      const MCSchedClassDesc *SC = Instrs[I];
+    for (const MCSchedClassDesc *SC : Instrs) {
       if (!SC->isValid())
         continue;
       for (TargetSchedModel::ProcResIter
@@ -1222,8 +1228,8 @@ unsigned MachineTraceMetrics::Trace::getResourceLength(
 
   for (unsigned K = 0; K != PRDepths.size(); ++K) {
     unsigned PRCycles = PRDepths[K] + PRHeights[K];
-    for (unsigned I = 0; I != Extrablocks.size(); ++I)
-      PRCycles += TE.MTM.getProcResourceCycles(Extrablocks[I]->getNumber())[K];
+    for (const MachineBasicBlock *MBB : Extrablocks)
+      PRCycles += TE.MTM.getProcResourceCycles(MBB->getNumber())[K];
     PRCycles += extraCycles(ExtraInstrs, K);
     PRCycles -= extraCycles(RemoveInstrs, K);
     PRMax = std::max(PRMax, PRCycles);
@@ -1234,8 +1240,8 @@ unsigned MachineTraceMetrics::Trace::getResourceLength(
   // Instrs: #instructions in current trace outside current block.
   unsigned Instrs = TBI.InstrDepth + TBI.InstrHeight;
   // Add instruction count from the extra blocks.
-  for (unsigned i = 0, e = Extrablocks.size(); i != e; ++i)
-    Instrs += TE.MTM.getResources(Extrablocks[i])->InstrCount;
+  for (const MachineBasicBlock *MBB : Extrablocks)
+    Instrs += TE.MTM.getResources(MBB)->InstrCount;
   Instrs += ExtraInstrs.size();
   Instrs -= RemoveInstrs.size();
   if (unsigned IW = TE.MTM.SchedModel.getIssueWidth())
diff --git a/lib/CodeGen/MachineVerifier.cpp b/lib/CodeGen/MachineVerifier.cpp
index 991241e0f74..a5e5bc992ef 100644
--- a/lib/CodeGen/MachineVerifier.cpp
+++ b/lib/CodeGen/MachineVerifier.cpp
@@ -329,8 +329,18 @@ bool MachineVerifier::runOnMachineFunction(MachineFunction &MF) {
       } else if (!CurBundle)
         report("No bundle header", MBBI);
       visitMachineInstrBefore(MBBI);
-      for (unsigned I = 0, E = MBBI->getNumOperands(); I != E; ++I)
-        visitMachineOperand(&MBBI->getOperand(I), I);
+      for (unsigned I = 0, E = MBBI->getNumOperands(); I != E; ++I) {
+        const MachineInstr &MI = *MBBI;
+        const MachineOperand &Op = MI.getOperand(I);
+        if (Op.getParent() != &MI) {
+          // Make sure to use correct addOperand / RemoveOperand / ChangeTo
+          // functions when replacing operands of a MachineInstr.
+          report("Instruction has operand with wrong parent set", &MI);
+        }
+
+        visitMachineOperand(&Op, I);
+      }
+
       visitMachineInstrAfter(MBBI);
 
       // Was this the last bundled instruction?
@@ -684,7 +694,7 @@ MachineVerifier::visitMachineBasicBlockBefore(const MachineBasicBlock *MBB) {
 
   const MachineFrameInfo *MFI = MF->getFrameInfo();
   assert(MFI && "Function has no frame info");
-  BitVector PR = MFI->getPristineRegs(MBB);
+  BitVector PR = MFI->getPristineRegs(*MF);
   for (int I = PR.find_first(); I>0; I = PR.find_next(I)) {
     for (MCSubRegIterator SubRegs(I, TRI, /*IncludeSelf=*/true);
          SubRegs.isValid(); ++SubRegs)
@@ -1661,6 +1671,8 @@ void MachineVerifier::verifyLiveInterval(const LiveInterval &LI) {
       report("Lane masks of sub ranges overlap in live interval", MF, LI);
     if ((SR.LaneMask & ~MaxMask) != 0)
       report("Subrange lanemask is invalid", MF, LI);
+    if (SR.empty())
+      report("Subrange must not be empty", MF, SR, LI.reg, SR.LaneMask);
     Mask |= SR.LaneMask;
     verifyLiveRange(SR, LI.reg, SR.LaneMask);
     if (!LI.covers(SR))
@@ -1706,8 +1718,8 @@ namespace {
 /// by a FrameDestroy <n>, stack adjustments are identical on all
 /// CFG edges to a merge point, and frame is destroyed at end of a return block.
 void MachineVerifier::verifyStackFrame() {
-  int FrameSetupOpcode   = TII->getCallFrameSetupOpcode();
-  int FrameDestroyOpcode = TII->getCallFrameDestroyOpcode();
+  unsigned FrameSetupOpcode   = TII->getCallFrameSetupOpcode();
+  unsigned FrameDestroyOpcode = TII->getCallFrameDestroyOpcode();
 
   SmallVector<StackStateOfBB, 8> SPState;
   SPState.resize(MF->getNumBlockIDs());
diff --git a/lib/CodeGen/Makefile b/lib/CodeGen/Makefile
index 4ab3e3c0013..96f7ca58513 100644
--- a/lib/CodeGen/Makefile
+++ b/lib/CodeGen/Makefile
@@ -9,7 +9,7 @@
 
 LEVEL = ../..
 LIBRARYNAME = LLVMCodeGen
-PARALLEL_DIRS = SelectionDAG AsmPrinter
+PARALLEL_DIRS = SelectionDAG AsmPrinter MIRParser
 BUILD_ARCHIVE = 1
 
 include $(LEVEL)/Makefile.common
diff --git a/lib/CodeGen/PHIElimination.cpp b/lib/CodeGen/PHIElimination.cpp
index d51419083d0..d3433018004 100644
--- a/lib/CodeGen/PHIElimination.cpp
+++ b/lib/CodeGen/PHIElimination.cpp
@@ -88,8 +88,8 @@ namespace {
 
     // These functions are temporary abstractions around LiveVariables and
     // LiveIntervals, so they can go away when LiveVariables does.
-    bool isLiveIn(unsigned Reg, MachineBasicBlock *MBB);
-    bool isLiveOutPastPHIs(unsigned Reg, MachineBasicBlock *MBB);
+    bool isLiveIn(unsigned Reg, const MachineBasicBlock *MBB);
+    bool isLiveOutPastPHIs(unsigned Reg, const MachineBasicBlock *MBB);
 
     typedef std::pair<unsigned, unsigned> BBVRegPair;
     typedef DenseMap<BBVRegPair, unsigned> VRegPHIUse;
@@ -143,16 +143,16 @@ bool PHIElimination::runOnMachineFunction(MachineFunction &MF) {
   // updating LiveIntervals, so we disable it.
   if (!DisableEdgeSplitting && (LV || LIS)) {
     MachineLoopInfo *MLI = getAnalysisIfAvailable<MachineLoopInfo>();
-    for (MachineFunction::iterator I = MF.begin(), E = MF.end(); I != E; ++I)
-      Changed |= SplitPHIEdges(MF, *I, MLI);
+    for (auto &MBB : MF)
+      Changed |= SplitPHIEdges(MF, MBB, MLI);
   }
 
   // Populate VRegPHIUseCount
   analyzePHINodes(MF);
 
   // Eliminate PHI instructions by inserting copies into predecessor blocks.
-  for (MachineFunction::iterator I = MF.begin(), E = MF.end(); I != E; ++I)
-    Changed |= EliminatePHINodes(MF, *I);
+  for (auto &MBB : MF)
+    Changed |= EliminatePHINodes(MF, MBB);
 
   // Remove dead IMPLICIT_DEF instructions.
   for (MachineInstr *DefMI : ImpDefs) {
@@ -623,7 +623,7 @@ bool PHIElimination::SplitPHIEdges(MachineFunction &MF,
   return Changed;
 }
 
-bool PHIElimination::isLiveIn(unsigned Reg, MachineBasicBlock *MBB) {
+bool PHIElimination::isLiveIn(unsigned Reg, const MachineBasicBlock *MBB) {
   assert((LV || LIS) &&
          "isLiveIn() requires either LiveVariables or LiveIntervals");
   if (LIS)
@@ -632,7 +632,8 @@ bool PHIElimination::isLiveIn(unsigned Reg, MachineBasicBlock *MBB) {
     return LV->isLiveIn(Reg, *MBB);
 }
 
-bool PHIElimination::isLiveOutPastPHIs(unsigned Reg, MachineBasicBlock *MBB) {
+bool PHIElimination::isLiveOutPastPHIs(unsigned Reg,
+                                       const MachineBasicBlock *MBB) {
   assert((LV || LIS) &&
          "isLiveOutPastPHIs() requires either LiveVariables or LiveIntervals");
   // LiveVariables considers uses in PHIs to be in the predecessor basic block,
@@ -642,11 +643,9 @@ bool PHIElimination::isLiveOutPastPHIs(unsigned Reg, MachineBasicBlock *MBB) {
   // out of the block.
   if (LIS) {
     const LiveInterval &LI = LIS->getInterval(Reg);
-    for (MachineBasicBlock::succ_iterator SI = MBB->succ_begin(),
-         SE = MBB->succ_end(); SI != SE; ++SI) {
-      if (LI.liveAt(LIS->getMBBStartIdx(*SI)))
+    for (const MachineBasicBlock *SI : MBB->successors())
+      if (LI.liveAt(LIS->getMBBStartIdx(SI)))
         return true;
-    }
     return false;
   } else {
     return LV->isLiveOut(Reg, *MBB);
diff --git a/lib/CodeGen/Passes.cpp b/lib/CodeGen/Passes.cpp
index c128414c0b3..024d166a498 100644
--- a/lib/CodeGen/Passes.cpp
+++ b/lib/CodeGen/Passes.cpp
@@ -24,6 +24,7 @@
 #include "llvm/Support/Debug.h"
 #include "llvm/Support/ErrorHandling.h"
 #include "llvm/Support/raw_ostream.h"
+#include "llvm/Transforms/Instrumentation.h"
 #include "llvm/Transforms/Scalar.h"
 #include "llvm/Transforms/Utils/SymbolRewriter.h"
 
@@ -52,7 +53,10 @@ static cl::opt<bool> DisableMachineLICM("disable-machine-licm", cl::Hidden,
 static cl::opt<bool> DisableMachineCSE("disable-machine-cse", cl::Hidden,
     cl::desc("Disable Machine Common Subexpression Elimination"));
 static cl::opt<cl::boolOrDefault>
-OptimizeRegAlloc("optimize-regalloc", cl::Hidden,
+    EnableShrinkWrapOpt("enable-shrink-wrap", cl::Hidden,
+                        cl::desc("enable the shrink-wrapping pass"));
+static cl::opt<cl::boolOrDefault> OptimizeRegAlloc(
+    "optimize-regalloc", cl::Hidden,
     cl::desc("Enable optimized register allocation compilation path."));
 static cl::opt<bool> DisablePostRAMachineLICM("disable-postra-machine-licm",
     cl::Hidden,
@@ -69,6 +73,10 @@ static cl::opt<bool> DisableCopyProp("disable-copyprop", cl::Hidden,
     cl::desc("Disable Copy Propagation pass"));
 static cl::opt<bool> DisablePartialLibcallInlining("disable-partial-libcall-inlining",
     cl::Hidden, cl::desc("Disable Partial Libcall Inlining"));
+static cl::opt<bool> EnableImplicitNullChecks(
+    "enable-implicit-null-checks",
+    cl::desc("Fold null checks into faulting memory operations"),
+    cl::init(false));
 static cl::opt<bool> PrintLSR("print-lsr-output", cl::Hidden,
     cl::desc("Print LLVM IR produced by the loop-reduce pass"));
 static cl::opt<bool> PrintISelInput("print-isel-input", cl::Hidden,
@@ -206,10 +214,10 @@ TargetPassConfig::~TargetPassConfig() {
 // Out of line constructor provides default values for pass options and
 // registers all common codegen passes.
 TargetPassConfig::TargetPassConfig(TargetMachine *tm, PassManagerBase &pm)
-  : ImmutablePass(ID), PM(&pm), StartAfter(nullptr), StopAfter(nullptr),
-    Started(true), Stopped(false), AddingMachinePasses(false), TM(tm),
-    Impl(nullptr), Initialized(false), DisableVerify(false),
-    EnableTailMerge(true) {
+    : ImmutablePass(ID), PM(&pm), StartBefore(nullptr), StartAfter(nullptr),
+      StopAfter(nullptr), Started(true), Stopped(false),
+      AddingMachinePasses(false), TM(tm), Impl(nullptr), Initialized(false),
+      DisableVerify(false), EnableTailMerge(true), EnableShrinkWrap(false) {
 
   Impl = new PassConfigImpl();
 
@@ -280,6 +288,8 @@ void TargetPassConfig::addPass(Pass *P, bool verifyAfter, bool printAfter) {
   // and shouldn't reference it.
   AnalysisID PassID = P->getPassID();
 
+  if (StartBefore == PassID)
+    Started = true;
   if (Started && !Stopped) {
     std::string Banner;
     // Construct banner message before PM->add() as that may delete the pass.
@@ -292,6 +302,24 @@ void TargetPassConfig::addPass(Pass *P, bool verifyAfter, bool printAfter) {
       if (verifyAfter)
         addVerifyPass(Banner);
     }
+
+    // Add the passes after the pass P if there is any.
+    for (SmallVectorImpl<std::pair<AnalysisID, IdentifyingPassPtr> >::iterator
+             I = Impl->InsertedPasses.begin(),
+             E = Impl->InsertedPasses.end();
+         I != E; ++I) {
+      if ((*I).first == PassID) {
+        assert((*I).second.isValid() && "Illegal Pass ID!");
+        Pass *NP;
+        if ((*I).second.isInstance())
+          NP = (*I).second.getInstance();
+        else {
+          NP = Pass::createPass((*I).second.getID());
+          assert(NP && "Pass ID not registered");
+        }
+        addPass(NP, false, false);
+      }
+    }
   } else {
     delete P;
   }
@@ -326,22 +354,6 @@ AnalysisID TargetPassConfig::addPass(AnalysisID PassID, bool verifyAfter,
   AnalysisID FinalID = P->getPassID();
   addPass(P, verifyAfter, printAfter); // Ends the lifetime of P.
 
-  // Add the passes after the pass P if there is any.
-  for (SmallVectorImpl<std::pair<AnalysisID, IdentifyingPassPtr> >::iterator
-         I = Impl->InsertedPasses.begin(), E = Impl->InsertedPasses.end();
-       I != E; ++I) {
-    if ((*I).first == PassID) {
-      assert((*I).second.isValid() && "Illegal Pass ID!");
-      Pass *NP;
-      if ((*I).second.isInstance())
-        NP = (*I).second.getInstance();
-      else {
-        NP = Pass::createPass((*I).second.getID());
-        assert(NP && "Pass ID not registered");
-      }
-      addPass(NP, false, false);
-    }
-  }
   return FinalID;
 }
 
@@ -412,7 +424,7 @@ void TargetPassConfig::addPassesToHandleExceptions() {
     // removed from the parent invoke(s). This could happen when a landing
     // pad is shared by multiple invokes and is also a target of a normal
     // edge from elsewhere.
-    addPass(createSjLjEHPreparePass(TM));
+    addPass(createSjLjEHPreparePass());
     // FALLTHROUGH
   case ExceptionHandling::DwarfCFI:
   case ExceptionHandling::ARM:
@@ -447,6 +459,9 @@ void TargetPassConfig::addCodeGenPrepare() {
 void TargetPassConfig::addISelPrepare() {
   addPreISel();
 
+  // Add both the safe stack and the stack protection passes: each of them will
+  // only protect functions that have corresponding attributes.
+  addPass(createSafeStackPass());
   addPass(createStackProtectorPass(TM));
 
   if (PrintISelInput)
@@ -524,6 +539,8 @@ void TargetPassConfig::addMachinePasses() {
   addPostRegAlloc();
 
   // Insert prolog/epilog code.  Eliminate abstract frame index references...
+  if (getEnableShrinkWrap())
+    addPass(&ShrinkWrapID);
   addPass(&PrologEpilogCodeInserterID);
 
   /// Add passes that optimize machine instructions after register allocation.
@@ -536,6 +553,9 @@ void TargetPassConfig::addMachinePasses() {
   // Run pre-sched2 passes.
   addPreSched2();
 
+  if (EnableImplicitNullChecks)
+    addPass(&ImplicitNullChecksID);
+
   // Second pass scheduler.
   if (getOptLevel() != CodeGenOpt::None) {
     if (MISchedPostRA)
@@ -599,6 +619,21 @@ void TargetPassConfig::addMachineSSAOptimization() {
   addPass(&DeadMachineInstructionElimID);
 }
 
+bool TargetPassConfig::getEnableShrinkWrap() const {
+  switch (EnableShrinkWrapOpt) {
+  case cl::BOU_UNSET:
+    return EnableShrinkWrap && getOptLevel() != CodeGenOpt::None;
+  // If EnableShrinkWrap is set, it takes precedence on whatever the
+  // target sets. The rational is that we assume we want to test
+  // something related to shrink-wrapping.
+  case cl::BOU_TRUE:
+    return true;
+  case cl::BOU_FALSE:
+    return false;
+  }
+  llvm_unreachable("Invalid shrink-wrapping state");
+}
+
 //===---------------------------------------------------------------------===//
 /// Register Allocation Pass Configuration
 //===---------------------------------------------------------------------===//
diff --git a/lib/CodeGen/PeepholeOptimizer.cpp b/lib/CodeGen/PeepholeOptimizer.cpp
index ebe05e3f273..ef213e4347f 100644
--- a/lib/CodeGen/PeepholeOptimizer.cpp
+++ b/lib/CodeGen/PeepholeOptimizer.cpp
@@ -107,6 +107,8 @@ STATISTIC(NumUncoalescableCopies, "Number of uncoalescable copies optimized");
 STATISTIC(NumRewrittenCopies, "Number of copies rewritten");
 
 namespace {
+  class ValueTrackerResult;
+
   class PeepholeOptimizer : public MachineFunctionPass {
     const TargetInstrInfo *TII;
     const TargetRegisterInfo *TRI;
@@ -171,6 +173,55 @@ namespace {
     }
   };
 
+  /// \brief Helper class to hold a reply for ValueTracker queries. Contains the
+  /// returned sources for a given search and the instructions where the sources
+  /// were tracked from.
+  class ValueTrackerResult {
+  private:
+    /// Track all sources found by one ValueTracker query.
+    SmallVector<TargetInstrInfo::RegSubRegPair, 2> RegSrcs;
+
+    /// Instruction using the sources in 'RegSrcs'.
+    const MachineInstr *Inst;
+
+  public:
+    ValueTrackerResult() : Inst(nullptr) {}
+    ValueTrackerResult(unsigned Reg, unsigned SubReg) : Inst(nullptr) {
+      addSource(Reg, SubReg);
+    }
+
+    bool isValid() const { return getNumSources() > 0; }
+
+    void setInst(const MachineInstr *I) { Inst = I; }
+    const MachineInstr *getInst() const { return Inst; }
+
+    void clear() {
+      RegSrcs.clear();
+      Inst = nullptr;
+    }
+
+    void addSource(unsigned SrcReg, unsigned SrcSubReg) {
+      RegSrcs.push_back(TargetInstrInfo::RegSubRegPair(SrcReg, SrcSubReg));
+    }
+
+    void setSource(int Idx, unsigned SrcReg, unsigned SrcSubReg) {
+      assert(Idx < getNumSources() && "Reg pair source out of index");
+      RegSrcs[Idx] = TargetInstrInfo::RegSubRegPair(SrcReg, SrcSubReg);
+    }
+
+    int getNumSources() const { return RegSrcs.size(); }
+
+    unsigned getSrcReg(int Idx) const {
+      assert(Idx < getNumSources() && "Reg source out of index");
+      return RegSrcs[Idx].Reg;
+    }
+
+    unsigned getSrcSubReg(int Idx) const {
+      assert(Idx < getNumSources() && "SubReg source out of index");
+      return RegSrcs[Idx].SubReg;
+    }
+  };
+
   /// \brief Helper class to track the possible sources of a value defined by
   /// a (chain of) copy related instructions.
   /// Given a definition (instruction and definition index), this class
@@ -213,23 +264,23 @@ namespace {
 
     /// \brief Dispatcher to the right underlying implementation of
     /// getNextSource.
-    bool getNextSourceImpl(unsigned &SrcReg, unsigned &SrcSubReg);
+    ValueTrackerResult getNextSourceImpl();
     /// \brief Specialized version of getNextSource for Copy instructions.
-    bool getNextSourceFromCopy(unsigned &SrcReg, unsigned &SrcSubReg);
+    ValueTrackerResult getNextSourceFromCopy();
     /// \brief Specialized version of getNextSource for Bitcast instructions.
-    bool getNextSourceFromBitcast(unsigned &SrcReg, unsigned &SrcSubReg);
+    ValueTrackerResult getNextSourceFromBitcast();
     /// \brief Specialized version of getNextSource for RegSequence
     /// instructions.
-    bool getNextSourceFromRegSequence(unsigned &SrcReg, unsigned &SrcSubReg);
+    ValueTrackerResult getNextSourceFromRegSequence();
     /// \brief Specialized version of getNextSource for InsertSubreg
     /// instructions.
-    bool getNextSourceFromInsertSubreg(unsigned &SrcReg, unsigned &SrcSubReg);
+    ValueTrackerResult getNextSourceFromInsertSubreg();
     /// \brief Specialized version of getNextSource for ExtractSubreg
     /// instructions.
-    bool getNextSourceFromExtractSubreg(unsigned &SrcReg, unsigned &SrcSubReg);
+    ValueTrackerResult getNextSourceFromExtractSubreg();
     /// \brief Specialized version of getNextSource for SubregToReg
     /// instructions.
-    bool getNextSourceFromSubregToReg(unsigned &SrcReg, unsigned &SrcSubReg);
+    ValueTrackerResult getNextSourceFromSubregToReg();
 
   public:
     /// \brief Create a ValueTracker instance for the value defined by \p Reg.
@@ -276,16 +327,10 @@ namespace {
 
     /// \brief Following the use-def chain, get the next available source
     /// for the tracked value.
-    /// When the returned value is not nullptr, \p SrcReg gives the register
-    /// that contain the tracked value.
-    /// \note The sub register index returned in \p SrcSubReg must be used
-    /// on \p SrcReg to access the actual value.
-    /// \return Unless the returned value is nullptr (i.e., no source found),
-    /// \p SrcReg gives the register of the next source used in the returned
-    /// instruction and \p SrcSubReg the sub-register index to be used on that
-    /// source to get the tracked value. When nullptr is returned, no
-    /// alternative source has been found.
-    const MachineInstr *getNextSource(unsigned &SrcReg, unsigned &SrcSubReg);
+    /// \return A ValueTrackerResult containing the a set of registers
+    /// and sub registers with tracked values. A ValueTrackerResult with
+    /// an empty set of registers means no source was found.
+    ValueTrackerResult getNextSource();
 
     /// \brief Get the last register where the initial value can be found.
     /// Initially this is the register of the definition.
@@ -560,11 +605,11 @@ bool PeepholeOptimizer::findNextSource(unsigned &Reg, unsigned &SubReg) {
   // or find a more suitable source.
   ValueTracker ValTracker(Reg, DefSubReg, *MRI, !DisableAdvCopyOpt, TII);
   do {
-    unsigned CopySrcReg, CopySrcSubReg;
-    if (!ValTracker.getNextSource(CopySrcReg, CopySrcSubReg))
+    ValueTrackerResult Res = ValTracker.getNextSource();
+    if (!Res.isValid())
       break;
-    Src = CopySrcReg;
-    SrcSubReg = CopySrcSubReg;
+    Src = Res.getSrcReg(0);
+    SrcSubReg = Res.getSrcSubReg(0);
 
     // Do not extend the live-ranges of physical registers as they add
     // constraints to the register allocator.
@@ -653,7 +698,7 @@ class CopyRewriter {
 
   /// \brief Rewrite the current source with \p NewReg and \p NewSubReg
   /// if possible.
-  /// \return True if the rewritting was possible, false otherwise.
+  /// \return True if the rewriting was possible, false otherwise.
   virtual bool RewriteCurrentSource(unsigned NewReg, unsigned NewSubReg) {
     if (!CopyLike.isCopy() || CurrentSrcIdx != 1)
       return false;
@@ -662,6 +707,91 @@ class CopyRewriter {
     MOSrc.setSubReg(NewSubReg);
     return true;
   }
+
+  /// \brief Rewrite the current source with \p NewSrcReg and \p NewSecSubReg
+  /// by creating a new COPY instruction. \p DefReg and \p DefSubReg contain the
+  /// definition to be rewritten from the original copylike instruction.
+  /// \return The new COPY if the rewriting was possible, nullptr otherwise.
+  /// This is needed to handle Uncoalescable copies, since they are copy
+  /// like instructions that aren't recognized by the register allocator.
+  virtual MachineInstr *RewriteCurrentSource(unsigned DefReg,
+                                             unsigned DefSubReg,
+                                             unsigned NewSrcReg,
+                                             unsigned NewSrcSubReg) {
+    return nullptr;
+  }
+};
+
+/// \brief Helper class to rewrite uncoalescable copy like instructions
+/// into new COPY (coalescable friendly) instructions.
+class UncoalescableRewriter : public CopyRewriter {
+protected:
+  const TargetInstrInfo &TII;
+  MachineRegisterInfo   &MRI;
+  /// The number of defs in the bitcast
+  unsigned NumDefs;
+
+public:
+  UncoalescableRewriter(MachineInstr &MI, const TargetInstrInfo &TII,
+                         MachineRegisterInfo &MRI)
+      : CopyRewriter(MI), TII(TII), MRI(MRI) {
+    NumDefs = MI.getDesc().getNumDefs();
+  }
+
+  /// \brief Get the next rewritable def source (TrackReg, TrackSubReg)
+  /// All such sources need to be considered rewritable in order to
+  /// rewrite a uncoalescable copy-like instruction. This method return
+  /// each definition that must be checked if rewritable.
+  ///
+  bool getNextRewritableSource(unsigned &SrcReg, unsigned &SrcSubReg,
+                               unsigned &TrackReg,
+                               unsigned &TrackSubReg) override {
+    // Find the next non-dead definition and continue from there.
+    if (CurrentSrcIdx == NumDefs)
+      return false;
+
+    while (CopyLike.getOperand(CurrentSrcIdx).isDead()) {
+      ++CurrentSrcIdx;
+      if (CurrentSrcIdx == NumDefs)
+        return false;
+    }
+
+    // What we track are the alternative sources of the definition.
+    const MachineOperand &MODef = CopyLike.getOperand(CurrentSrcIdx);
+    TrackReg = MODef.getReg();
+    TrackSubReg = MODef.getSubReg();
+
+    CurrentSrcIdx++;
+    return true;
+  }
+
+  /// \brief Rewrite the current source with \p NewSrcReg and \p NewSrcSubReg
+  /// by creating a new COPY instruction. \p DefReg and \p DefSubReg contain the
+  /// definition to be rewritten from the original copylike instruction.
+  /// \return The new COPY if the rewriting was possible, nullptr otherwise.
+  MachineInstr *RewriteCurrentSource(unsigned DefReg, unsigned DefSubReg,
+                                     unsigned NewSrcReg,
+                                     unsigned NewSrcSubReg) override {
+    assert(!TargetRegisterInfo::isPhysicalRegister(DefReg) &&
+           "We do not rewrite physical registers");
+
+    const TargetRegisterClass *DefRC = MRI.getRegClass(DefReg);
+    unsigned NewVR = MRI.createVirtualRegister(DefRC);
+
+    MachineInstr *NewCopy =
+        BuildMI(*CopyLike.getParent(), &CopyLike, CopyLike.getDebugLoc(),
+                TII.get(TargetOpcode::COPY), NewVR)
+            .addReg(NewSrcReg, 0, NewSrcSubReg);
+
+    NewCopy->getOperand(0).setSubReg(DefSubReg);
+    if (DefSubReg)
+      NewCopy->getOperand(0).setIsUndef();
+
+    MRI.replaceRegWith(DefReg, NewVR);
+    MRI.clearKillFlags(NewVR);
+
+    return NewCopy;
+  }
 };
 
 /// \brief Specialized rewriter for INSERT_SUBREG instruction.
@@ -850,7 +980,13 @@ class RegSequenceRewriter : public CopyRewriter {
 /// \return A pointer to a dynamically allocated CopyRewriter or nullptr
 /// if no rewriter works for \p MI.
 static CopyRewriter *getCopyRewriter(MachineInstr &MI,
-                                     const TargetInstrInfo &TII) {
+                                     const TargetInstrInfo &TII,
+                                     MachineRegisterInfo &MRI) {
+  // Handle uncoalescable copy-like instructions.
+  if (MI.isBitcast() || (MI.isRegSequenceLike() || MI.isInsertSubregLike() ||
+                         MI.isExtractSubregLike()))
+    return new UncoalescableRewriter(MI, TII, MRI);
+
   switch (MI.getOpcode()) {
   default:
     return nullptr;
@@ -889,7 +1025,7 @@ bool PeepholeOptimizer::optimizeCoalescableCopy(MachineInstr *MI) {
 
   bool Changed = false;
   // Get the right rewriter for the current copy.
-  std::unique_ptr<CopyRewriter> CpyRewriter(getCopyRewriter(*MI, *TII));
+  std::unique_ptr<CopyRewriter> CpyRewriter(getCopyRewriter(*MI, *TII, *MRI));
   // If none exists, bails out.
   if (!CpyRewriter)
     return false;
@@ -899,9 +1035,8 @@ bool PeepholeOptimizer::optimizeCoalescableCopy(MachineInstr *MI) {
                                               TrackSubReg)) {
     unsigned NewSrc = TrackReg;
     unsigned NewSubReg = TrackSubReg;
-    // Try to find a more suitable source.
-    // If we failed to do so, or get the actual source,
-    // move to the next source.
+    // Try to find a more suitable source. If we failed to do so, or get the
+    // actual source, move to the next source.
     if (!findNextSource(NewSrc, NewSubReg) || SrcReg == NewSrc)
       continue;
     // Rewrite source.
@@ -939,45 +1074,47 @@ bool PeepholeOptimizer::optimizeUncoalescableCopy(
   SmallVector<
       std::pair<TargetInstrInfo::RegSubRegPair, TargetInstrInfo::RegSubRegPair>,
       4> RewritePairs;
-  for (const MachineOperand &MODef : MI->defs()) {
-    if (MODef.isDead())
-      // We can ignore those.
-      continue;
+  // Get the right rewriter for the current copy.
+  std::unique_ptr<CopyRewriter> CpyRewriter(getCopyRewriter(*MI, *TII, *MRI));
+  // If none exists, bails out.
+  if (!CpyRewriter)
+    return false;
 
+  // Rewrite each rewritable source by generating new COPYs. This works
+  // differently from optimizeCoalescableCopy since it first makes sure that all
+  // definitions can be rewritten.
+  unsigned SrcReg, SrcSubReg, TrackReg, TrackSubReg;
+  while (CpyRewriter->getNextRewritableSource(SrcReg, SrcSubReg, TrackReg,
+                                              TrackSubReg)) {
     // If a physical register is here, this is probably for a good reason.
     // Do not rewrite that.
-    if (TargetRegisterInfo::isPhysicalRegister(MODef.getReg()))
+    if (TargetRegisterInfo::isPhysicalRegister(TrackReg))
       return false;
 
     // If we do not know how to rewrite this definition, there is no point
     // in trying to kill this instruction.
-    TargetInstrInfo::RegSubRegPair Def(MODef.getReg(), MODef.getSubReg());
+    TargetInstrInfo::RegSubRegPair Def(TrackReg, TrackSubReg);
     TargetInstrInfo::RegSubRegPair Src = Def;
     if (!findNextSource(Src.Reg, Src.SubReg))
       return false;
+
     RewritePairs.push_back(std::make_pair(Def, Src));
   }
+
   // The change is possible for all defs, do it.
   for (const auto &PairDefSrc : RewritePairs) {
     const auto &Def = PairDefSrc.first;
     const auto &Src = PairDefSrc.second;
+
     // Rewrite the "copy" in a way the register coalescer understands.
-    assert(!TargetRegisterInfo::isPhysicalRegister(Def.Reg) &&
-           "We do not rewrite physical registers");
-    const TargetRegisterClass *DefRC = MRI->getRegClass(Def.Reg);
-    unsigned NewVR = MRI->createVirtualRegister(DefRC);
-    MachineInstr *NewCopy = BuildMI(*MI->getParent(), MI, MI->getDebugLoc(),
-                                    TII->get(TargetOpcode::COPY),
-                                    NewVR).addReg(Src.Reg, 0, Src.SubReg);
-    NewCopy->getOperand(0).setSubReg(Def.SubReg);
-    if (Def.SubReg)
-      NewCopy->getOperand(0).setIsUndef();
-    LocalMIs.insert(NewCopy);
-    MRI->replaceRegWith(Def.Reg, NewVR);
-    MRI->clearKillFlags(NewVR);
-    // We extended the lifetime of Src.
-    // Clear the kill flags to account for that.
+    MachineInstr *NewCopy = CpyRewriter->RewriteCurrentSource(
+        Def.Reg, Def.SubReg, Src.Reg, Src.SubReg);
+    assert(NewCopy && "Should be able to always generate a new copy");
+
+    // We extended the lifetime of Src and clear the kill flags to
+    // account for that.
     MRI->clearKillFlags(Src.Reg);
+    LocalMIs.insert(NewCopy);
   }
   // MI is now dead.
   MI->eraseFromParent();
@@ -1190,8 +1327,7 @@ bool PeepholeOptimizer::runOnMachineFunction(MachineFunction &MF) {
   return Changed;
 }
 
-bool ValueTracker::getNextSourceFromCopy(unsigned &SrcReg,
-                                         unsigned &SrcSubReg) {
+ValueTrackerResult ValueTracker::getNextSourceFromCopy() {
   assert(Def->isCopy() && "Invalid definition");
   // Copy instruction are supposed to be: Def = Src.
   // If someone breaks this assumption, bad things will happen everywhere.
@@ -1200,29 +1336,26 @@ bool ValueTracker::getNextSourceFromCopy(unsigned &SrcReg,
   if (Def->getOperand(DefIdx).getSubReg() != DefSubReg)
     // If we look for a different subreg, it means we want a subreg of src.
     // Bails as we do not support composing subreg yet.
-    return false;
+    return ValueTrackerResult();
   // Otherwise, we want the whole source.
   const MachineOperand &Src = Def->getOperand(1);
-  SrcReg = Src.getReg();
-  SrcSubReg = Src.getSubReg();
-  return true;
+  return ValueTrackerResult(Src.getReg(), Src.getSubReg());
 }
 
-bool ValueTracker::getNextSourceFromBitcast(unsigned &SrcReg,
-                                            unsigned &SrcSubReg) {
+ValueTrackerResult ValueTracker::getNextSourceFromBitcast() {
   assert(Def->isBitcast() && "Invalid definition");
 
   // Bail if there are effects that a plain copy will not expose.
   if (Def->hasUnmodeledSideEffects())
-    return false;
+    return ValueTrackerResult();
 
   // Bitcasts with more than one def are not supported.
   if (Def->getDesc().getNumDefs() != 1)
-    return false;
+    return ValueTrackerResult();
   if (Def->getOperand(DefIdx).getSubReg() != DefSubReg)
     // If we look for a different subreg, it means we want a subreg of the src.
     // Bails as we do not support composing subreg yet.
-    return false;
+    return ValueTrackerResult();
 
   unsigned SrcIdx = Def->getNumOperands();
   for (unsigned OpIdx = DefIdx + 1, EndOpIdx = SrcIdx; OpIdx != EndOpIdx;
@@ -1233,17 +1366,14 @@ bool ValueTracker::getNextSourceFromBitcast(unsigned &SrcReg,
     assert(!MO.isDef() && "We should have skipped all the definitions by now");
     if (SrcIdx != EndOpIdx)
       // Multiple sources?
-      return false;
+      return ValueTrackerResult();
     SrcIdx = OpIdx;
   }
   const MachineOperand &Src = Def->getOperand(SrcIdx);
-  SrcReg = Src.getReg();
-  SrcSubReg = Src.getSubReg();
-  return true;
+  return ValueTrackerResult(Src.getReg(), Src.getSubReg());
 }
 
-bool ValueTracker::getNextSourceFromRegSequence(unsigned &SrcReg,
-                                                unsigned &SrcSubReg) {
+ValueTrackerResult ValueTracker::getNextSourceFromRegSequence() {
   assert((Def->isRegSequence() || Def->isRegSequenceLike()) &&
          "Invalid definition");
 
@@ -1262,16 +1392,16 @@ bool ValueTracker::getNextSourceFromRegSequence(unsigned &SrcReg,
     // have this case.
     // If we can ascertain (or force) that this never happens, we could
     // turn that into an assertion.
-    return false;
+    return ValueTrackerResult();
 
   if (!TII)
     // We could handle the REG_SEQUENCE here, but we do not want to
     // duplicate the code from the generic TII.
-    return false;
+    return ValueTrackerResult();
 
   SmallVector<TargetInstrInfo::RegSubRegPairAndIdx, 8> RegSeqInputRegs;
   if (!TII->getRegSequenceInputs(*Def, DefIdx, RegSeqInputRegs))
-    return false;
+    return ValueTrackerResult();
 
   // We are looking at:
   // Def = REG_SEQUENCE v0, sub0, v1, sub1, ...
@@ -1280,22 +1410,19 @@ bool ValueTracker::getNextSourceFromRegSequence(unsigned &SrcReg,
     if (RegSeqInput.SubIdx == DefSubReg) {
       if (RegSeqInput.SubReg)
         // Bails if we have to compose sub registers.
-        return false;
+        return ValueTrackerResult();
 
-      SrcReg = RegSeqInput.Reg;
-      SrcSubReg = RegSeqInput.SubReg;
-      return true;
+      return ValueTrackerResult(RegSeqInput.Reg, RegSeqInput.SubReg);
     }
   }
 
   // If the subreg we are tracking is super-defined by another subreg,
   // we could follow this value. However, this would require to compose
   // the subreg and we do not do that for now.
-  return false;
+  return ValueTrackerResult();
 }
 
-bool ValueTracker::getNextSourceFromInsertSubreg(unsigned &SrcReg,
-                                                 unsigned &SrcSubReg) {
+ValueTrackerResult ValueTracker::getNextSourceFromInsertSubreg() {
   assert((Def->isInsertSubreg() || Def->isInsertSubregLike()) &&
          "Invalid definition");
 
@@ -1303,17 +1430,17 @@ bool ValueTracker::getNextSourceFromInsertSubreg(unsigned &SrcReg,
     // If we are composing subreg, bails out.
     // Same remark as getNextSourceFromRegSequence.
     // I.e., this may be turned into an assert.
-    return false;
+    return ValueTrackerResult();
 
   if (!TII)
     // We could handle the REG_SEQUENCE here, but we do not want to
     // duplicate the code from the generic TII.
-    return false;
+    return ValueTrackerResult();
 
   TargetInstrInfo::RegSubRegPair BaseReg;
   TargetInstrInfo::RegSubRegPairAndIdx InsertedReg;
   if (!TII->getInsertSubregInputs(*Def, DefIdx, BaseReg, InsertedReg))
-    return false;
+    return ValueTrackerResult();
 
   // We are looking at:
   // Def = INSERT_SUBREG v0, v1, sub1
@@ -1323,9 +1450,7 @@ bool ValueTracker::getNextSourceFromInsertSubreg(unsigned &SrcReg,
 
   // #1 Check if the inserted register matches the required sub index.
   if (InsertedReg.SubIdx == DefSubReg) {
-    SrcReg = InsertedReg.Reg;
-    SrcSubReg = InsertedReg.SubReg;
-    return true;
+    return ValueTrackerResult(InsertedReg.Reg, InsertedReg.SubReg);
   }
   // #2 Otherwise, if the sub register we are looking for is not partial
   // defined by the inserted element, we can look through the main
@@ -1336,7 +1461,7 @@ bool ValueTracker::getNextSourceFromInsertSubreg(unsigned &SrcReg,
   // subregisters, bails out.
   if (MRI.getRegClass(MODef.getReg()) != MRI.getRegClass(BaseReg.Reg) ||
       BaseReg.SubReg)
-    return false;
+    return ValueTrackerResult();
 
   // Get the TRI and check if the inserted sub-register overlaps with the
   // sub-register we are tracking.
@@ -1344,16 +1469,13 @@ bool ValueTracker::getNextSourceFromInsertSubreg(unsigned &SrcReg,
   if (!TRI ||
       (TRI->getSubRegIndexLaneMask(DefSubReg) &
        TRI->getSubRegIndexLaneMask(InsertedReg.SubIdx)) != 0)
-    return false;
+    return ValueTrackerResult();
   // At this point, the value is available in v0 via the same subreg
   // we used for Def.
-  SrcReg = BaseReg.Reg;
-  SrcSubReg = DefSubReg;
-  return true;
+  return ValueTrackerResult(BaseReg.Reg, DefSubReg);
 }
 
-bool ValueTracker::getNextSourceFromExtractSubreg(unsigned &SrcReg,
-                                                  unsigned &SrcSubReg) {
+ValueTrackerResult ValueTracker::getNextSourceFromExtractSubreg() {
   assert((Def->isExtractSubreg() ||
           Def->isExtractSubregLike()) && "Invalid definition");
   // We are looking at:
@@ -1362,29 +1484,26 @@ bool ValueTracker::getNextSourceFromExtractSubreg(unsigned &SrcReg,
   // Bails if we have to compose sub registers.
   // Indeed, if DefSubReg != 0, we would have to compose it with sub0.
   if (DefSubReg)
-    return false;
+    return ValueTrackerResult();
 
   if (!TII)
     // We could handle the EXTRACT_SUBREG here, but we do not want to
     // duplicate the code from the generic TII.
-    return false;
+    return ValueTrackerResult();
 
   TargetInstrInfo::RegSubRegPairAndIdx ExtractSubregInputReg;
   if (!TII->getExtractSubregInputs(*Def, DefIdx, ExtractSubregInputReg))
-    return false;
+    return ValueTrackerResult();
 
   // Bails if we have to compose sub registers.
   // Likewise, if v0.subreg != 0, we would have to compose v0.subreg with sub0.
   if (ExtractSubregInputReg.SubReg)
-    return false;
+    return ValueTrackerResult();
   // Otherwise, the value is available in the v0.sub0.
-  SrcReg = ExtractSubregInputReg.Reg;
-  SrcSubReg = ExtractSubregInputReg.SubIdx;
-  return true;
+  return ValueTrackerResult(ExtractSubregInputReg.Reg, ExtractSubregInputReg.SubIdx);
 }
 
-bool ValueTracker::getNextSourceFromSubregToReg(unsigned &SrcReg,
-                                                unsigned &SrcSubReg) {
+ValueTrackerResult ValueTracker::getNextSourceFromSubregToReg() {
   assert(Def->isSubregToReg() && "Invalid definition");
   // We are looking at:
   // Def = SUBREG_TO_REG Imm, v0, sub0
@@ -1394,71 +1513,71 @@ bool ValueTracker::getNextSourceFromSubregToReg(unsigned &SrcReg,
   // we track are included in sub0 and if yes, we would have to
   // determine the right subreg in v0.
   if (DefSubReg != Def->getOperand(3).getImm())
-    return false;
+    return ValueTrackerResult();
   // Bails if we have to compose sub registers.
   // Likewise, if v0.subreg != 0, we would have to compose it with sub0.
   if (Def->getOperand(2).getSubReg())
-    return false;
+    return ValueTrackerResult();
 
-  SrcReg = Def->getOperand(2).getReg();
-  SrcSubReg = Def->getOperand(3).getImm();
-  return true;
+  return ValueTrackerResult(Def->getOperand(2).getReg(),
+                            Def->getOperand(3).getImm());
 }
 
-bool ValueTracker::getNextSourceImpl(unsigned &SrcReg, unsigned &SrcSubReg) {
+ValueTrackerResult ValueTracker::getNextSourceImpl() {
   assert(Def && "This method needs a valid definition");
 
   assert(
       (DefIdx < Def->getDesc().getNumDefs() || Def->getDesc().isVariadic()) &&
       Def->getOperand(DefIdx).isDef() && "Invalid DefIdx");
   if (Def->isCopy())
-    return getNextSourceFromCopy(SrcReg, SrcSubReg);
+    return getNextSourceFromCopy();
   if (Def->isBitcast())
-    return getNextSourceFromBitcast(SrcReg, SrcSubReg);
+    return getNextSourceFromBitcast();
   // All the remaining cases involve "complex" instructions.
   // Bails if we did not ask for the advanced tracking.
   if (!UseAdvancedTracking)
-    return false;
+    return ValueTrackerResult();
   if (Def->isRegSequence() || Def->isRegSequenceLike())
-    return getNextSourceFromRegSequence(SrcReg, SrcSubReg);
+    return getNextSourceFromRegSequence();
   if (Def->isInsertSubreg() || Def->isInsertSubregLike())
-    return getNextSourceFromInsertSubreg(SrcReg, SrcSubReg);
+    return getNextSourceFromInsertSubreg();
   if (Def->isExtractSubreg() || Def->isExtractSubregLike())
-    return getNextSourceFromExtractSubreg(SrcReg, SrcSubReg);
+    return getNextSourceFromExtractSubreg();
   if (Def->isSubregToReg())
-    return getNextSourceFromSubregToReg(SrcReg, SrcSubReg);
-  return false;
+    return getNextSourceFromSubregToReg();
+  return ValueTrackerResult();
 }
 
-const MachineInstr *ValueTracker::getNextSource(unsigned &SrcReg,
-                                                unsigned &SrcSubReg) {
+ValueTrackerResult ValueTracker::getNextSource() {
   // If we reach a point where we cannot move up in the use-def chain,
   // there is nothing we can get.
   if (!Def)
-    return nullptr;
+    return ValueTrackerResult();
 
-  const MachineInstr *PrevDef = nullptr;
-  // Try to find the next source.
-  if (getNextSourceImpl(SrcReg, SrcSubReg)) {
+  ValueTrackerResult Res = getNextSourceImpl();
+  if (Res.isValid()) {
     // Update definition, definition index, and subregister for the
     // next call of getNextSource.
     // Update the current register.
-    Reg = SrcReg;
-    // Update the return value before moving up in the use-def chain.
-    PrevDef = Def;
+    bool OneRegSrc = Res.getNumSources() == 1;
+    if (OneRegSrc)
+      Reg = Res.getSrcReg(0);
+    // Update the result before moving up in the use-def chain
+    // with the instruction containing the last found sources.
+    Res.setInst(Def);
+
     // If we can still move up in the use-def chain, move to the next
     // defintion.
-    if (!TargetRegisterInfo::isPhysicalRegister(Reg)) {
+    if (!TargetRegisterInfo::isPhysicalRegister(Reg) && OneRegSrc) {
       Def = MRI.getVRegDef(Reg);
       DefIdx = MRI.def_begin(Reg).getOperandNo();
-      DefSubReg = SrcSubReg;
-      return PrevDef;
+      DefSubReg = Res.getSrcSubReg(0);
+      return Res;
     }
   }
   // If we end up here, this means we will not be able to find another source
-  // for the next iteration.
-  // Make sure any new call to getNextSource bails out early by cutting the
-  // use-def chain.
+  // for the next iteration. Make sure any new call to getNextSource bails out
+  // early by cutting the use-def chain.
   Def = nullptr;
-  return PrevDef;
+  return Res;
 }
diff --git a/lib/CodeGen/PostRASchedulerList.cpp b/lib/CodeGen/PostRASchedulerList.cpp
index 55f08e496de..6f76116da1e 100644
--- a/lib/CodeGen/PostRASchedulerList.cpp
+++ b/lib/CodeGen/PostRASchedulerList.cpp
@@ -257,7 +257,7 @@ bool PostRAScheduler::enablePostRAScheduler(
     TargetSubtargetInfo::RegClassVector &CriticalPathRCs) const {
   Mode = ST.getAntiDepBreakMode();
   ST.getCriticalPathRCs(CriticalPathRCs);
-  return ST.enablePostMachineScheduler() &&
+  return ST.enablePostRAScheduler() &&
          OptLevel >= ST.getOptLevelToEnablePostRAScheduler();
 }
 
diff --git a/lib/CodeGen/ProcessImplicitDefs.cpp b/lib/CodeGen/ProcessImplicitDefs.cpp
index b1538006e72..5f819498348 100644
--- a/lib/CodeGen/ProcessImplicitDefs.cpp
+++ b/lib/CodeGen/ProcessImplicitDefs.cpp
@@ -68,8 +68,8 @@ bool ProcessImplicitDefs::canTurnIntoImplicitDef(MachineInstr *MI) {
       !MI->isRegSequence() &&
       !MI->isPHI())
     return false;
-  for (MIOperands MO(MI); MO.isValid(); ++MO)
-    if (MO->isReg() && MO->isUse() && MO->readsReg())
+  for (const MachineOperand &MO : MI->operands())
+    if (MO.isReg() && MO.isUse() && MO.readsReg())
       return false;
   return true;
 }
@@ -100,17 +100,17 @@ void ProcessImplicitDefs::processImplicitDef(MachineInstr *MI) {
   MachineBasicBlock::instr_iterator UserE = MI->getParent()->instr_end();
   bool Found = false;
   for (++UserMI; UserMI != UserE; ++UserMI) {
-    for (MIOperands MO(UserMI); MO.isValid(); ++MO) {
-      if (!MO->isReg())
+    for (MachineOperand &MO : UserMI->operands()) {
+      if (!MO.isReg())
         continue;
-      unsigned UserReg = MO->getReg();
+      unsigned UserReg = MO.getReg();
       if (!TargetRegisterInfo::isPhysicalRegister(UserReg) ||
           !TRI->regsOverlap(Reg, UserReg))
         continue;
       // UserMI uses or redefines Reg. Set <undef> flags on all uses.
       Found = true;
-      if (MO->isUse())
-        MO->setIsUndef();
+      if (MO.isUse())
+        MO.setIsUndef();
     }
     if (Found)
       break;
diff --git a/lib/CodeGen/PrologEpilogInserter.cpp b/lib/CodeGen/PrologEpilogInserter.cpp
index 5334a633ead..b2fdee6c8e4 100644
--- a/lib/CodeGen/PrologEpilogInserter.cpp
+++ b/lib/CodeGen/PrologEpilogInserter.cpp
@@ -71,9 +71,9 @@ class PEI : public MachineFunctionPass {
   // stack frame indexes.
   unsigned MinCSFrameIndex, MaxCSFrameIndex;
 
-  // Entry and return blocks of the current function.
-  MachineBasicBlock *EntryBlock;
-  SmallVector<MachineBasicBlock *, 4> ReturnBlocks;
+  // Save and Restore blocks of the current function.
+  MachineBasicBlock *SaveBlock;
+  SmallVector<MachineBasicBlock *, 4> RestoreBlocks;
 
   // Flag to control whether to use the register scavenger to resolve
   // frame index materialization registers. Set according to
@@ -82,7 +82,8 @@ class PEI : public MachineFunctionPass {
 
   void calculateSets(MachineFunction &Fn);
   void calculateCallsInformation(MachineFunction &Fn);
-  void calculateCalleeSavedRegisters(MachineFunction &Fn);
+  void assignCalleeSavedSpillSlots(MachineFunction &Fn,
+                                   const BitVector &SavedRegs);
   void insertCSRSpillsAndRestores(MachineFunction &Fn);
   void calculateFrameObjectOffsets(MachineFunction &Fn);
   void replaceFrameIndices(MachineFunction &Fn);
@@ -92,7 +93,7 @@ class PEI : public MachineFunctionPass {
   void insertPrologEpilogCode(MachineFunction &Fn);
 
   // Convenience for recognizing return blocks.
-  bool isReturnBlock(MachineBasicBlock *MBB);
+  bool isReturnBlock(const MachineBasicBlock *MBB) const;
 };
 } // namespace
 
@@ -127,26 +128,37 @@ void PEI::getAnalysisUsage(AnalysisUsage &AU) const {
   MachineFunctionPass::getAnalysisUsage(AU);
 }
 
-bool PEI::isReturnBlock(MachineBasicBlock* MBB) {
+bool PEI::isReturnBlock(const MachineBasicBlock* MBB) const {
   return (MBB && !MBB->empty() && MBB->back().isReturn());
 }
 
 /// Compute the set of return blocks
 void PEI::calculateSets(MachineFunction &Fn) {
-  // Sets used to compute spill, restore placement sets.
-  const std::vector<CalleeSavedInfo> &CSI =
-    Fn.getFrameInfo()->getCalleeSavedInfo();
-
-  // If no CSRs used, we are done.
-  if (CSI.empty())
+  const MachineFrameInfo *MFI = Fn.getFrameInfo();
+
+  // Even when we do not change any CSR, we still want to insert the
+  // prologue and epilogue of the function.
+  // So set the save points for those.
+
+  // Use the points found by shrink-wrapping, if any.
+  if (MFI->getSavePoint()) {
+    SaveBlock = MFI->getSavePoint();
+    assert(MFI->getRestorePoint() && "Both restore and save must be set");
+    MachineBasicBlock *RestoreBlock = MFI->getRestorePoint();
+    // If RestoreBlock does not have any successor and is not a return block
+    // then the end point is unreachable and we do not need to insert any
+    // epilogue.
+    if (!RestoreBlock->succ_empty() || isReturnBlock(RestoreBlock))
+      RestoreBlocks.push_back(RestoreBlock);
     return;
+  }
 
   // Save refs to entry and return blocks.
-  EntryBlock = Fn.begin();
+  SaveBlock = Fn.begin();
   for (MachineFunction::iterator MBB = Fn.begin(), E = Fn.end();
        MBB != E; ++MBB)
     if (isReturnBlock(MBB))
-      ReturnBlocks.push_back(MBB);
+      RestoreBlocks.push_back(MBB);
 
   return;
 }
@@ -172,13 +184,12 @@ bool PEI::runOnMachineFunction(MachineFunction &Fn) {
   // instructions.
   calculateCallsInformation(Fn);
 
-  // Allow the target machine to make some adjustments to the function
-  // e.g. UsedPhysRegs before calculateCalleeSavedRegisters.
-  TFI->processFunctionBeforeCalleeSavedScan(Fn, RS);
+  // Determine which of the registers in the callee save list should be saved.
+  BitVector SavedRegs;
+  TFI->determineCalleeSaves(Fn, SavedRegs, RS);
 
-  // Scan the function for modified callee saved registers and insert spill code
-  // for any callee saved registers that are modified.
-  calculateCalleeSavedRegisters(Fn);
+  // Insert spill code for any callee saved registers that are modified.
+  assignCalleeSavedSpillSlots(Fn, SavedRegs);
 
   // Determine placement of CSR spill/restore code:
   // place all spills in the entry block, all restores in return blocks.
@@ -226,7 +237,7 @@ bool PEI::runOnMachineFunction(MachineFunction &Fn) {
   }
 
   delete RS;
-  ReturnBlocks.clear();
+  RestoreBlocks.clear();
   return true;
 }
 
@@ -242,12 +253,12 @@ void PEI::calculateCallsInformation(MachineFunction &Fn) {
   bool AdjustsStack = MFI->adjustsStack();
 
   // Get the function call frame set-up and tear-down instruction opcode
-  int FrameSetupOpcode   = TII.getCallFrameSetupOpcode();
-  int FrameDestroyOpcode = TII.getCallFrameDestroyOpcode();
+  unsigned FrameSetupOpcode = TII.getCallFrameSetupOpcode();
+  unsigned FrameDestroyOpcode = TII.getCallFrameDestroyOpcode();
 
   // Early exit for targets which have no call frame setup/destroy pseudo
   // instructions.
-  if (FrameSetupOpcode == -1 && FrameDestroyOpcode == -1)
+  if (FrameSetupOpcode == ~0u && FrameDestroyOpcode == ~0u)
     return;
 
   std::vector<MachineBasicBlock::iterator> FrameSDOps;
@@ -284,39 +295,27 @@ void PEI::calculateCallsInformation(MachineFunction &Fn) {
   }
 }
 
-
-/// calculateCalleeSavedRegisters - Scan the function for modified callee saved
-/// registers.
-void PEI::calculateCalleeSavedRegisters(MachineFunction &F) {
-  const TargetRegisterInfo *RegInfo = F.getSubtarget().getRegisterInfo();
-  const TargetFrameLowering *TFI = F.getSubtarget().getFrameLowering();
-  MachineFrameInfo *MFI = F.getFrameInfo();
-
-  // Get the callee saved register list...
-  const MCPhysReg *CSRegs = RegInfo->getCalleeSavedRegs(&F);
-
+void PEI::assignCalleeSavedSpillSlots(MachineFunction &F,
+                                      const BitVector &SavedRegs) {
   // These are used to keep track the callee-save area. Initialize them.
   MinCSFrameIndex = INT_MAX;
   MaxCSFrameIndex = 0;
 
-  // Early exit for targets which have no callee saved registers.
-  if (!CSRegs || CSRegs[0] == 0)
+  if (SavedRegs.empty())
     return;
 
-  // In Naked functions we aren't going to save any registers.
-  if (F.getFunction()->hasFnAttribute(Attribute::Naked))
-    return;
+  const TargetRegisterInfo *RegInfo = F.getSubtarget().getRegisterInfo();
+  const MCPhysReg *CSRegs = RegInfo->getCalleeSavedRegs(&F);
 
   std::vector<CalleeSavedInfo> CSI;
   for (unsigned i = 0; CSRegs[i]; ++i) {
     unsigned Reg = CSRegs[i];
-    // Functions which call __builtin_unwind_init get all their registers saved.
-    if (F.getRegInfo().isPhysRegUsed(Reg) || F.getMMI().callsUnwindInit()) {
-      // If the reg is modified, save it!
+    if (SavedRegs.test(Reg))
       CSI.push_back(CalleeSavedInfo(Reg));
-    }
   }
 
+  const TargetFrameLowering *TFI = F.getSubtarget().getFrameLowering();
+  MachineFrameInfo *MFI = F.getFrameInfo();
   if (!TFI->assignCalleeSavedSpillSlots(F, RegInfo, CSI)) {
     // If target doesn't implement this, use generic code.
 
@@ -372,6 +371,61 @@ void PEI::calculateCalleeSavedRegisters(MachineFunction &F) {
   MFI->setCalleeSavedInfo(CSI);
 }
 
+/// Helper function to update the liveness information for the callee-saved
+/// registers.
+static void updateLiveness(MachineFunction &MF) {
+  MachineFrameInfo *MFI = MF.getFrameInfo();
+  // Visited will contain all the basic blocks that are in the region
+  // where the callee saved registers are alive:
+  // - Anything that is not Save or Restore -> LiveThrough.
+  // - Save -> LiveIn.
+  // - Restore -> LiveOut.
+  // The live-out is not attached to the block, so no need to keep
+  // Restore in this set.
+  SmallPtrSet<MachineBasicBlock *, 8> Visited;
+  SmallVector<MachineBasicBlock *, 8> WorkList;
+  MachineBasicBlock *Entry = &MF.front();
+  MachineBasicBlock *Save = MFI->getSavePoint();
+
+  if (!Save)
+    Save = Entry;
+
+  if (Entry != Save) {
+    WorkList.push_back(Entry);
+    Visited.insert(Entry);
+  }
+  Visited.insert(Save);
+
+  MachineBasicBlock *Restore = MFI->getRestorePoint();
+  if (Restore)
+    // By construction Restore cannot be visited, otherwise it
+    // means there exists a path to Restore that does not go
+    // through Save.
+    WorkList.push_back(Restore);
+
+  while (!WorkList.empty()) {
+    const MachineBasicBlock *CurBB = WorkList.pop_back_val();
+    // By construction, the region that is after the save point is
+    // dominated by the Save and post-dominated by the Restore.
+    if (CurBB == Save)
+      continue;
+    // Enqueue all the successors not already visited.
+    // Those are by construction either before Save or after Restore.
+    for (MachineBasicBlock *SuccBB : CurBB->successors())
+      if (Visited.insert(SuccBB).second)
+        WorkList.push_back(SuccBB);
+  }
+
+  const std::vector<CalleeSavedInfo> &CSI = MFI->getCalleeSavedInfo();
+
+  for (unsigned i = 0, e = CSI.size(); i != e; ++i) {
+    for (MachineBasicBlock *MBB : Visited)
+      // Add the callee-saved register as live-in.
+      // It's killed at the spill.
+      MBB->addLiveIn(CSI[i].getReg());
+  }
+}
+
 /// insertCSRSpillsAndRestores - Insert spill and restore code for
 /// callee saved registers used in the function.
 ///
@@ -392,26 +446,22 @@ void PEI::insertCSRSpillsAndRestores(MachineFunction &Fn) {
   MachineBasicBlock::iterator I;
 
   // Spill using target interface.
-  I = EntryBlock->begin();
-  if (!TFI->spillCalleeSavedRegisters(*EntryBlock, I, CSI, TRI)) {
+  I = SaveBlock->begin();
+  if (!TFI->spillCalleeSavedRegisters(*SaveBlock, I, CSI, TRI)) {
     for (unsigned i = 0, e = CSI.size(); i != e; ++i) {
-      // Add the callee-saved register as live-in.
-      // It's killed at the spill.
-      EntryBlock->addLiveIn(CSI[i].getReg());
-
       // Insert the spill to the stack frame.
       unsigned Reg = CSI[i].getReg();
       const TargetRegisterClass *RC = TRI->getMinimalPhysRegClass(Reg);
-      TII.storeRegToStackSlot(*EntryBlock, I, Reg, true, CSI[i].getFrameIdx(),
+      TII.storeRegToStackSlot(*SaveBlock, I, Reg, true, CSI[i].getFrameIdx(),
                               RC, TRI);
     }
   }
+  // Update the live-in information of all the blocks up to the save point.
+  updateLiveness(Fn);
 
   // Restore using target interface.
-  for (unsigned ri = 0, re = ReturnBlocks.size(); ri != re; ++ri) {
-    MachineBasicBlock *MBB = ReturnBlocks[ri];
+  for (MachineBasicBlock *MBB : RestoreBlocks) {
     I = MBB->end();
-    --I;
 
     // Skip over all terminator instructions, which are part of the return
     // sequence.
@@ -721,21 +771,18 @@ void PEI::insertPrologEpilogCode(MachineFunction &Fn) {
   const TargetFrameLowering &TFI = *Fn.getSubtarget().getFrameLowering();
 
   // Add prologue to the function...
-  TFI.emitPrologue(Fn);
+  TFI.emitPrologue(Fn, *SaveBlock);
 
-  // Add epilogue to restore the callee-save registers in each exiting block
-  for (MachineFunction::iterator I = Fn.begin(), E = Fn.end(); I != E; ++I) {
-    // If last instruction is a return instruction, add an epilogue
-    if (!I->empty() && I->back().isReturn())
-      TFI.emitEpilogue(Fn, *I);
-  }
+  // Add epilogue to restore the callee-save registers in each exiting block.
+  for (MachineBasicBlock *RestoreBlock : RestoreBlocks)
+    TFI.emitEpilogue(Fn, *RestoreBlock);
 
   // Emit additional code that is required to support segmented stacks, if
   // we've been asked for it.  This, when linked with a runtime with support
   // for segmented stacks (libgcc is one), will result in allocating stack
   // space in small chunks instead of one large contiguous block.
   if (Fn.shouldSplitStack())
-    TFI.adjustForSegmentedStacks(Fn);
+    TFI.adjustForSegmentedStacks(Fn, *SaveBlock);
 
   // Emit additional code that is required to explicitly handle the stack in
   // HiPE native code (if needed) when loaded in the Erlang/OTP runtime. The
@@ -743,7 +790,7 @@ void PEI::insertPrologEpilogCode(MachineFunction &Fn) {
   // different conditional check and another BIF for allocating more stack
   // space.
   if (Fn.getFunction()->getCallingConv() == CallingConv::HiPE)
-    TFI.adjustForHiPEPrologue(Fn);
+    TFI.adjustForHiPEPrologue(Fn, *SaveBlock);
 }
 
 /// replaceFrameIndices - Replace all MO_FrameIndex operands with physical
@@ -810,8 +857,8 @@ void PEI::replaceFrameIndices(MachineBasicBlock *BB, MachineFunction &Fn,
   const TargetInstrInfo &TII = *Fn.getSubtarget().getInstrInfo();
   const TargetRegisterInfo &TRI = *Fn.getSubtarget().getRegisterInfo();
   const TargetFrameLowering *TFI = Fn.getSubtarget().getFrameLowering();
-  int FrameSetupOpcode   = TII.getCallFrameSetupOpcode();
-  int FrameDestroyOpcode = TII.getCallFrameDestroyOpcode();
+  unsigned FrameSetupOpcode = TII.getCallFrameSetupOpcode();
+  unsigned FrameDestroyOpcode = TII.getCallFrameDestroyOpcode();
 
   if (RS && !FrameIndexVirtualScavenging) RS->enterBasicBlock(BB);
 
@@ -979,12 +1026,8 @@ PEI::scavengeFrameVirtualRegs(MachineFunction &Fn) {
           // Replace this reference to the virtual register with the
           // scratch register.
           assert (ScratchReg && "Missing scratch register!");
-          MachineRegisterInfo &MRI = Fn.getRegInfo();
           Fn.getRegInfo().replaceRegWith(Reg, ScratchReg);
           
-          // Make sure MRI now accounts this register as used.
-          MRI.setPhysRegUsed(ScratchReg);
-
           // Because this instruction was processed by the RS before this
           // register was allocated, make sure that the RS now records the
           // register as being used.
diff --git a/lib/CodeGen/RegAllocBasic.cpp b/lib/CodeGen/RegAllocBasic.cpp
index 0090332a812..d9660541d92 100644
--- a/lib/CodeGen/RegAllocBasic.cpp
+++ b/lib/CodeGen/RegAllocBasic.cpp
@@ -223,7 +223,7 @@ unsigned RABasic::selectOrSplit(LiveInterval &VirtReg,
   SmallVector<unsigned, 8> PhysRegSpillCands;
 
   // Check for an available register in this class.
-  AllocationOrder Order(VirtReg.reg, *VRM, RegClassInfo);
+  AllocationOrder Order(VirtReg.reg, *VRM, RegClassInfo, Matrix);
   while (unsigned PhysReg = Order.next()) {
     // Check for interference in PhysReg
     switch (Matrix->checkInterference(VirtReg, PhysReg)) {
diff --git a/lib/CodeGen/RegAllocFast.cpp b/lib/CodeGen/RegAllocFast.cpp
index c311c7ba2b4..660bb4f0e34 100644
--- a/lib/CodeGen/RegAllocFast.cpp
+++ b/lib/CodeGen/RegAllocFast.cpp
@@ -302,7 +302,7 @@ void RAFast::spillVirtReg(MachineBasicBlock::iterator MI,
       bool IsIndirect = DBG->isIndirectDebugValue();
       uint64_t Offset = IsIndirect ? DBG->getOperand(1).getImm() : 0;
       DebugLoc DL = DBG->getDebugLoc();
-      assert(cast<MDLocalVariable>(Var)->isValidLocationForIntrinsic(DL) &&
+      assert(cast<DILocalVariable>(Var)->isValidLocationForIntrinsic(DL) &&
              "Expected inlined-at fields to agree");
       MachineInstr *NewDV =
           BuildMI(*MBB, MI, DL, TII->get(TargetOpcode::DBG_VALUE))
@@ -874,7 +874,7 @@ void RAFast::AllocateBasicBlock() {
               DebugLoc DL = MI->getDebugLoc();
               MachineBasicBlock *MBB = MI->getParent();
               assert(
-                  cast<MDLocalVariable>(Var)->isValidLocationForIntrinsic(DL) &&
+                  cast<DILocalVariable>(Var)->isValidLocationForIntrinsic(DL) &&
                   "Expected inlined-at fields to agree");
               MachineInstr *NewDV = BuildMI(*MBB, MBB->erase(MI), DL,
                                             TII->get(TargetOpcode::DBG_VALUE))
@@ -986,10 +986,6 @@ void RAFast::AllocateBasicBlock() {
       }
     }
 
-    for (UsedInInstrSet::iterator
-         I = UsedInInstr.begin(), E = UsedInInstr.end(); I != E; ++I)
-      MRI->setRegUnitUsed(*I);
-
     // Track registers defined by instruction - early clobbers and tied uses at
     // this point.
     UsedInInstr.clear();
@@ -1050,10 +1046,6 @@ void RAFast::AllocateBasicBlock() {
       killVirtReg(VirtDead[i]);
     VirtDead.clear();
 
-    for (UsedInInstrSet::iterator
-         I = UsedInInstr.begin(), E = UsedInInstr.end(); I != E; ++I)
-      MRI->setRegUnitUsed(*I);
-
     if (CopyDst && CopyDst == CopySrc && CopyDstSub == CopySrcSub) {
       DEBUG(dbgs() << "-- coalescing: " << *MI);
       Coalesced.push_back(MI);
@@ -1103,12 +1095,6 @@ bool RAFast::runOnMachineFunction(MachineFunction &Fn) {
     AllocateBasicBlock();
   }
 
-  // Add the clobber lists for all the instructions we skipped earlier.
-  for (const MCInstrDesc *Desc : SkippedInstrs)
-    if (const uint16_t *Defs = Desc->getImplicitDefs())
-      while (*Defs)
-        MRI->setPhysRegUsed(*Defs++);
-
   // All machine operands and other references to virtual registers have been
   // replaced. Remove the virtual registers.
   MRI->clearVirtRegs();
diff --git a/lib/CodeGen/RegAllocGreedy.cpp b/lib/CodeGen/RegAllocGreedy.cpp
index 26f42c93323..72e20d1d98e 100644
--- a/lib/CodeGen/RegAllocGreedy.cpp
+++ b/lib/CodeGen/RegAllocGreedy.cpp
@@ -86,6 +86,14 @@ static cl::opt<bool> EnableLocalReassignment(
              "may be compile time intensive"),
     cl::init(false));
 
+static cl::opt<bool> EnableDeferredSpilling(
+    "enable-deferred-spilling", cl::Hidden,
+    cl::desc("Instead of spilling a variable right away, defer the actual "
+             "code insertion to the end of the allocation. That way the "
+             "allocator might still find a suitable coloring for this "
+             "variable because of other evicted variables."),
+    cl::init(false));
+
 // FIXME: Find a good default for this flag and remove the flag.
 static cl::opt<unsigned>
 CSRFirstTimeCost("regalloc-csr-first-time-cost",
@@ -157,6 +165,11 @@ class RAGreedy : public MachineFunctionPass,
     /// Live range will be spilled.  No more splitting will be attempted.
     RS_Spill,
 
+
+    /// Live range is in memory. Because of other evictions, it might get moved
+    /// in a register in the end.
+    RS_Memory,
+
     /// There is nothing more we can do to this live range.  Abort compilation
     /// if it can't be assigned.
     RS_Done
@@ -400,6 +413,8 @@ class RAGreedy : public MachineFunctionPass,
   typedef SmallVector<HintInfo, 4> HintsInfo;
   BlockFrequency getBrokenHintFreq(const HintsInfo &, unsigned);
   void collectHintInfo(unsigned, HintsInfo &);
+
+  bool isUnusedCalleeSavedReg(unsigned PhysReg) const;
 };
 } // end anonymous namespace
 
@@ -412,6 +427,7 @@ const char *const RAGreedy::StageName[] = {
     "RS_Split",
     "RS_Split2",
     "RS_Spill",
+    "RS_Memory",
     "RS_Done"
 };
 #endif
@@ -534,6 +550,13 @@ void RAGreedy::enqueue(PQueue &CurQueue, LiveInterval *LI) {
     // Unsplit ranges that couldn't be allocated immediately are deferred until
     // everything else has been allocated.
     Prio = Size;
+  } else if (ExtraRegInfo[Reg].Stage == RS_Memory) {
+    // Memory operand should be considered last.
+    // Change the priority such that Memory operand are assigned in
+    // the reverse order that they came in.
+    // TODO: Make this a member variable and probably do something about hints.
+    static unsigned MemOp = 0;
+    Prio = MemOp++;
   } else {
     // Giant live ranges fall back to the global assignment heuristic, which
     // prevents excessive spilling in pathological cases.
@@ -635,7 +658,7 @@ unsigned RAGreedy::tryAssign(LiveInterval &VirtReg,
 //===----------------------------------------------------------------------===//
 
 unsigned RAGreedy::canReassign(LiveInterval &VirtReg, unsigned PrevReg) {
-  AllocationOrder Order(VirtReg.reg, *VRM, RegClassInfo);
+  AllocationOrder Order(VirtReg.reg, *VRM, RegClassInfo, Matrix);
   unsigned PhysReg;
   while ((PhysReg = Order.next())) {
     if (PhysReg == PrevReg)
@@ -816,6 +839,16 @@ void RAGreedy::evictInterference(LiveInterval &VirtReg, unsigned PhysReg,
   }
 }
 
+/// Returns true if the given \p PhysReg is a callee saved register and has not
+/// been used for allocation yet.
+bool RAGreedy::isUnusedCalleeSavedReg(unsigned PhysReg) const {
+  unsigned CSR = RegClassInfo.getLastCalleeSavedAlias(PhysReg);
+  if (CSR == 0)
+    return false;
+
+  return !Matrix->isPhysRegUsed(PhysReg);
+}
+
 /// tryEvict - Try to evict all interferences for a physreg.
 /// @param  VirtReg Currently unassigned virtual register.
 /// @param  Order   Physregs to try.
@@ -861,13 +894,12 @@ unsigned RAGreedy::tryEvict(LiveInterval &VirtReg,
       continue;
     // The first use of a callee-saved register in a function has cost 1.
     // Don't start using a CSR when the CostPerUseLimit is low.
-    if (CostPerUseLimit == 1)
-     if (unsigned CSR = RegClassInfo.getLastCalleeSavedAlias(PhysReg))
-       if (!MRI->isPhysRegUsed(CSR)) {
-         DEBUG(dbgs() << PrintReg(PhysReg, TRI) << " would clobber CSR "
-                      << PrintReg(CSR, TRI) << '\n');
-         continue;
-       }
+    if (CostPerUseLimit == 1 && isUnusedCalleeSavedReg(PhysReg)) {
+      DEBUG(dbgs() << PrintReg(PhysReg, TRI) << " would clobber CSR "
+            << PrintReg(RegClassInfo.getLastCalleeSavedAlias(PhysReg), TRI)
+            << '\n');
+      continue;
+    }
 
     if (!canEvictInterference(VirtReg, PhysReg, false, BestCost))
       continue;
@@ -1348,9 +1380,8 @@ unsigned RAGreedy::calculateRegionSplitCost(LiveInterval &VirtReg,
   unsigned BestCand = NoCand;
   Order.rewind();
   while (unsigned PhysReg = Order.next()) {
-   if (unsigned CSR = RegClassInfo.getLastCalleeSavedAlias(PhysReg))
-     if (IgnoreCSR && !MRI->isPhysRegUsed(CSR))
-       continue;
+    if (IgnoreCSR && isUnusedCalleeSavedReg(PhysReg))
+      continue;
 
     // Discard bad candidates before we run out of interference cache cursors.
     // This will only affect register classes with a lot of registers (>32).
@@ -2134,7 +2165,8 @@ unsigned RAGreedy::tryLastChanceRecoloring(LiveInterval &VirtReg,
       unsigned ItVirtReg = (*It)->reg;
       if (VRM->hasPhys(ItVirtReg))
         Matrix->unassign(**It);
-      Matrix->assign(**It, VirtRegToPhysReg[ItVirtReg]);
+      unsigned ItPhysReg = VirtRegToPhysReg[ItVirtReg];
+      Matrix->assign(**It, ItPhysReg);
     }
   }
 
@@ -2439,18 +2471,13 @@ unsigned RAGreedy::selectOrSplitImpl(LiveInterval &VirtReg,
                                      unsigned Depth) {
   unsigned CostPerUseLimit = ~0u;
   // First try assigning a free register.
-  AllocationOrder Order(VirtReg.reg, *VRM, RegClassInfo);
+  AllocationOrder Order(VirtReg.reg, *VRM, RegClassInfo, Matrix);
   if (unsigned PhysReg = tryAssign(VirtReg, Order, NewVRegs)) {
-    // We check other options if we are using a CSR for the first time.
-    bool CSRFirstUse = false;
-    if (unsigned CSR = RegClassInfo.getLastCalleeSavedAlias(PhysReg))
-      if (!MRI->isPhysRegUsed(CSR))
-        CSRFirstUse = true;
-
     // When NewVRegs is not empty, we may have made decisions such as evicting
     // a virtual register, go with the earlier decisions and use the physical
     // register.
-    if (CSRCost.getFrequency() && CSRFirstUse && NewVRegs.empty()) {
+    if (CSRCost.getFrequency() && isUnusedCalleeSavedReg(PhysReg) &&
+        NewVRegs.empty()) {
       unsigned CSRReg = tryAssignCSRFirstTime(VirtReg, Order, PhysReg,
                                               CostPerUseLimit, NewVRegs);
       if (CSRReg || !NewVRegs.empty())
@@ -2506,13 +2533,23 @@ unsigned RAGreedy::selectOrSplitImpl(LiveInterval &VirtReg,
     return PhysReg;
 
   // Finally spill VirtReg itself.
-  NamedRegionTimer T("Spiller", TimerGroupName, TimePassesIsEnabled);
-  LiveRangeEdit LRE(&VirtReg, NewVRegs, *MF, *LIS, VRM, this);
-  spiller().spill(LRE);
-  setStage(NewVRegs.begin(), NewVRegs.end(), RS_Done);
+  if (EnableDeferredSpilling && getStage(VirtReg) < RS_Memory) {
+    // TODO: This is experimental and in particular, we do not model
+    // the live range splitting done by spilling correctly.
+    // We would need a deep integration with the spiller to do the
+    // right thing here. Anyway, that is still good for early testing.
+    setStage(VirtReg, RS_Memory);
+    DEBUG(dbgs() << "Do as if this register is in memory\n");
+    NewVRegs.push_back(VirtReg.reg);
+  } else {
+    NamedRegionTimer T("Spiller", TimerGroupName, TimePassesIsEnabled);
+    LiveRangeEdit LRE(&VirtReg, NewVRegs, *MF, *LIS, VRM, this);
+    spiller().spill(LRE);
+    setStage(NewVRegs.begin(), NewVRegs.end(), RS_Done);
 
-  if (VerifyEnabled)
-    MF->verify(this, "After spilling");
+    if (VerifyEnabled)
+      MF->verify(this, "After spilling");
+  }
 
   // The live virtual register requesting allocation was spilled, so tell
   // the caller not to allocate anything during this round.
diff --git a/lib/CodeGen/RegisterCoalescer.cpp b/lib/CodeGen/RegisterCoalescer.cpp
index 5d958a61e6a..c911b9b47ea 100644
--- a/lib/CodeGen/RegisterCoalescer.cpp
+++ b/lib/CodeGen/RegisterCoalescer.cpp
@@ -60,7 +60,7 @@ EnableJoining("join-liveintervals",
 
 static cl::opt<bool> UseTerminalRule("terminal-rule",
                                      cl::desc("Apply the terminal rule"),
-                                     cl::init(false));
+                                     cl::init(false), cl::Hidden);
 
 /// Temporary flag to test critical edge unsplitting.
 static cl::opt<bool>
@@ -194,7 +194,7 @@ namespace {
 
     /// If the source of a copy is defined by a
     /// trivial computation, replace the copy by rematerialize the definition.
-    bool reMaterializeTrivialDef(CoalescerPair &CP, MachineInstr *CopyMI,
+    bool reMaterializeTrivialDef(const CoalescerPair &CP, MachineInstr *CopyMI,
                                  bool &IsDefCopy);
 
     /// Return true if a copy involving a physreg should be joined.
@@ -224,6 +224,32 @@ namespace {
     /// Dst, we can drop \p Copy.
     bool applyTerminalRule(const MachineInstr &Copy) const;
 
+    /// Check whether or not \p LI is composed by multiple connected
+    /// components and if that is the case, fix that.
+    void splitNewRanges(LiveInterval *LI) {
+      ConnectedVNInfoEqClasses ConEQ(*LIS);
+      unsigned NumComps = ConEQ.Classify(LI);
+      if (NumComps <= 1)
+        return;
+      SmallVector<LiveInterval*, 8> NewComps(1, LI);
+      for (unsigned i = 1; i != NumComps; ++i) {
+        unsigned VReg = MRI->createVirtualRegister(MRI->getRegClass(LI->reg));
+        NewComps.push_back(&LIS->createEmptyInterval(VReg));
+      }
+
+      ConEQ.Distribute(&NewComps[0], *MRI);
+    }
+
+    /// Wrapper method for \see LiveIntervals::shrinkToUses.
+    /// This method does the proper fixing of the live-ranges when the afore
+    /// mentioned method returns true.
+    void shrinkToUses(LiveInterval *LI,
+                      SmallVectorImpl<MachineInstr * > *Dead = nullptr) {
+      if (LIS->shrinkToUses(LI, Dead))
+        // We may have created multiple connected components, split them.
+        splitNewRanges(LI);
+    }
+
   public:
     static char ID; ///< Class identification, replacement for typeinfo
     RegisterCoalescer() : MachineFunctionPass(ID) {
@@ -556,7 +582,7 @@ bool RegisterCoalescer::adjustCopiesBackFrom(const CoalescerPair &CP,
   // will also add the isKill marker.
   CopyMI->substituteRegister(IntA.reg, IntB.reg, 0, *TRI);
   if (AS->end == CopyIdx)
-    LIS->shrinkToUses(&IntA);
+    shrinkToUses(&IntA);
 
   ++numExtends;
   return true;
@@ -851,7 +877,7 @@ static bool definesFullReg(const MachineInstr &MI, unsigned Reg) {
   return false;
 }
 
-bool RegisterCoalescer::reMaterializeTrivialDef(CoalescerPair &CP,
+bool RegisterCoalescer::reMaterializeTrivialDef(const CoalescerPair &CP,
                                                 MachineInstr *CopyMI,
                                                 bool &IsDefCopy) {
   IsDefCopy = false;
@@ -882,7 +908,7 @@ bool RegisterCoalescer::reMaterializeTrivialDef(CoalescerPair &CP,
   if (!definesFullReg(*DefMI, SrcReg))
     return false;
   bool SawStore = false;
-  if (!DefMI->isSafeToMove(TII, AA, SawStore))
+  if (!DefMI->isSafeToMove(AA, SawStore))
     return false;
   const MCInstrDesc &MCID = DefMI->getDesc();
   if (MCID.getNumDefs() != 1)
@@ -929,6 +955,28 @@ bool RegisterCoalescer::reMaterializeTrivialDef(CoalescerPair &CP,
   TII->reMaterialize(*MBB, MII, DstReg, SrcIdx, DefMI, *TRI);
   MachineInstr *NewMI = std::prev(MII);
 
+  // In a situation like the following:
+  //     %vreg0:subreg = instr              ; DefMI, subreg = DstIdx
+  //     %vreg1        = copy %vreg0:subreg ; CopyMI, SrcIdx = 0
+  // instead of widening %vreg1 to the register class of %vreg0 simply do:
+  //     %vreg1 = instr
+  const TargetRegisterClass *NewRC = CP.getNewRC();
+  if (DstIdx != 0) {
+    MachineOperand &DefMO = NewMI->getOperand(0);
+    if (DefMO.getSubReg() == DstIdx) {
+      assert(SrcIdx == 0 && CP.isFlipped()
+             && "Shouldn't have SrcIdx+DstIdx at this point");
+      const TargetRegisterClass *DstRC = MRI->getRegClass(DstReg);
+      const TargetRegisterClass *CommonRC =
+        TRI->getCommonSubClass(DefRC, DstRC);
+      if (CommonRC != nullptr) {
+        NewRC = CommonRC;
+        DstIdx = 0;
+        DefMO.setSubReg(0);
+      }
+    }
+  }
+
   LIS->ReplaceMachineInstrInMaps(CopyMI, NewMI);
   CopyMI->eraseFromParent();
   ErasedInstrs.insert(CopyMI);
@@ -940,15 +988,14 @@ bool RegisterCoalescer::reMaterializeTrivialDef(CoalescerPair &CP,
   for (unsigned i = NewMI->getDesc().getNumOperands(),
          e = NewMI->getNumOperands(); i != e; ++i) {
     MachineOperand &MO = NewMI->getOperand(i);
-    if (MO.isReg()) {
-      assert(MO.isDef() && MO.isImplicit() && MO.isDead() &&
+    if (MO.isReg() && MO.isDef()) {
+      assert(MO.isImplicit() && MO.isDead() &&
              TargetRegisterInfo::isPhysicalRegister(MO.getReg()));
       NewMIImplDefs.push_back(MO.getReg());
     }
   }
 
   if (TargetRegisterInfo::isVirtualRegister(DstReg)) {
-    const TargetRegisterClass *NewRC = CP.getNewRC();
     unsigned NewIdx = NewMI->getOperand(0).getSubReg();
 
     if (DefRC != nullptr) {
@@ -1024,7 +1071,7 @@ bool RegisterCoalescer::reMaterializeTrivialDef(CoalescerPair &CP,
   ++NumReMats;
 
   // The source interval can become smaller because we removed a use.
-  LIS->shrinkToUses(&SrcInt, &DeadDefs);
+  shrinkToUses(&SrcInt, &DeadDefs);
   if (!DeadDefs.empty()) {
     // If the virtual SrcReg is completely eliminated, update all DBG_VALUEs
     // to describe DstReg instead.
@@ -1402,10 +1449,11 @@ bool RegisterCoalescer::joinCopy(MachineInstr *CopyMI, bool &Again) {
                    << format("%04X", S.LaneMask) << ")\n");
       LIS->shrinkToUses(S, LI.reg);
     }
+    LI.removeEmptySubRanges();
   }
   if (ShrinkMainRange) {
     LiveInterval &LI = LIS->getInterval(CP.getDstReg());
-    LIS->shrinkToUses(&LI);
+    shrinkToUses(&LI);
   }
 
   // SrcReg is guaranteed to be the register whose live interval that is
@@ -1483,6 +1531,14 @@ bool RegisterCoalescer::joinReservedPhysReg(CoalescerPair &CP) {
         DEBUG(dbgs() << "\t\tInterference (read): " << *MI);
         return false;
       }
+
+      // We must also check for clobbers caused by regmasks.
+      for (const auto &MO : MI->operands()) {
+        if (MO.isRegMask() && MO.clobbersPhysReg(DstReg)) {
+          DEBUG(dbgs() << "\t\tInterference (regmask clobber): " << *MI);
+          return false;
+        }
+      }
     }
 
     // We're going to remove the copy which defines a physical reserved
@@ -1787,12 +1843,12 @@ class JoinVals {
 unsigned JoinVals::computeWriteLanes(const MachineInstr *DefMI, bool &Redef)
   const {
   unsigned L = 0;
-  for (ConstMIOperands MO(DefMI); MO.isValid(); ++MO) {
-    if (!MO->isReg() || MO->getReg() != Reg || !MO->isDef())
+  for (const MachineOperand &MO : DefMI->operands()) {
+    if (!MO.isReg() || MO.getReg() != Reg || !MO.isDef())
       continue;
     L |= TRI->getSubRegIndexLaneMask(
-           TRI->composeSubRegIndices(SubIdx, MO->getSubReg()));
-    if (MO->readsReg())
+           TRI->composeSubRegIndices(SubIdx, MO.getSubReg()));
+    if (MO.readsReg())
       Redef = true;
   }
   return L;
@@ -2177,13 +2233,13 @@ bool JoinVals::usesLanes(const MachineInstr *MI, unsigned Reg, unsigned SubIdx,
                          unsigned Lanes) const {
   if (MI->isDebugValue())
     return false;
-  for (ConstMIOperands MO(MI); MO.isValid(); ++MO) {
-    if (!MO->isReg() || MO->isDef() || MO->getReg() != Reg)
+  for (const MachineOperand &MO : MI->operands()) {
+    if (!MO.isReg() || MO.isDef() || MO.getReg() != Reg)
       continue;
-    if (!MO->readsReg())
+    if (!MO.readsReg())
       continue;
     if (Lanes & TRI->getSubRegIndexLaneMask(
-                  TRI->composeSubRegIndices(SubIdx, MO->getSubReg())))
+                  TRI->composeSubRegIndices(SubIdx, MO.getSubReg())))
       return true;
   }
   return false;
@@ -2292,11 +2348,11 @@ void JoinVals::pruneValues(JoinVals &Other,
           // Remove <def,read-undef> flags. This def is now a partial redef.
           // Also remove <def,dead> flags since the joined live range will
           // continue past this instruction.
-          for (MIOperands MO(Indexes->getInstructionFromIndex(Def));
-               MO.isValid(); ++MO) {
-            if (MO->isReg() && MO->isDef() && MO->getReg() == Reg) {
-              MO->setIsUndef(EraseImpDef);
-              MO->setIsDead(false);
+          for (MachineOperand &MO :
+               Indexes->getInstructionFromIndex(Def)->operands()) {
+            if (MO.isReg() && MO.isDef() && MO.getReg() == Reg) {
+              MO.setIsUndef(EraseImpDef);
+              MO.setIsDead(false);
             }
           }
         }
@@ -2586,7 +2642,8 @@ bool RegisterCoalescer::joinVirtRegs(CoalescerPair &CP) {
       // "overflow bit" 32. As a workaround we drop all subregister ranges
       // which means we loose some precision but are back to a well defined
       // state.
-      assert((CP.getNewRC()->getLaneMask() & 0x80000000u)
+      assert(TargetRegisterInfo::isImpreciseLaneMask(
+             CP.getNewRC()->getLaneMask())
              && "SubRange merge should only fail when merging into bit 32.");
       DEBUG(dbgs() << "\tSubrange join aborted!\n");
       LHS.clearSubRanges();
@@ -2613,7 +2670,7 @@ bool RegisterCoalescer::joinVirtRegs(CoalescerPair &CP) {
   LHSVals.eraseInstrs(ErasedInstrs, ShrinkRegs);
   RHSVals.eraseInstrs(ErasedInstrs, ShrinkRegs);
   while (!ShrinkRegs.empty())
-    LIS->shrinkToUses(&LIS->getInterval(ShrinkRegs.pop_back_val()));
+    shrinkToUses(&LIS->getInterval(ShrinkRegs.pop_back_val()));
 
   // Join RHS into LHS.
   LHS.join(RHS, LHSVals.getAssignments(), RHSVals.getAssignments(), NewVNInfo);
diff --git a/lib/CodeGen/RegisterPressure.cpp b/lib/CodeGen/RegisterPressure.cpp
index 36341039acf..c3786e552a1 100644
--- a/lib/CodeGen/RegisterPressure.cpp
+++ b/lib/CodeGen/RegisterPressure.cpp
@@ -60,11 +60,11 @@ void RegisterPressure::dump(const TargetRegisterInfo *TRI) const {
   dumpRegSetPressure(MaxSetPressure, TRI);
   dbgs() << "Live In: ";
   for (unsigned i = 0, e = LiveInRegs.size(); i < e; ++i)
-    dbgs() << PrintReg(LiveInRegs[i], TRI) << " ";
+    dbgs() << PrintVRegOrUnit(LiveInRegs[i], TRI) << " ";
   dbgs() << '\n';
   dbgs() << "Live Out: ";
   for (unsigned i = 0, e = LiveOutRegs.size(); i < e; ++i)
-    dbgs() << PrintReg(LiveOutRegs[i], TRI) << " ";
+    dbgs() << PrintVRegOrUnit(LiveOutRegs[i], TRI) << " ";
   dbgs() << '\n';
 }
 
@@ -77,6 +77,16 @@ void RegPressureTracker::dump() const {
   P.dump(TRI);
 }
 
+void PressureDiff::dump(const TargetRegisterInfo &TRI) const {
+  for (const PressureChange &Change : *this) {
+    if (!Change.isValid() || Change.getUnitInc() == 0)
+      continue;
+    dbgs() << "    " << TRI.getRegPressureSetName(Change.getPSet())
+           << " " << Change.getUnitInc();
+  }
+  dbgs() << '\n';
+}
+
 /// Increase the current pressure as impacted by these registers and bump
 /// the high water mark if needed.
 void RegPressureTracker::increaseRegPressure(ArrayRef<unsigned> RegUnits) {
@@ -750,9 +760,11 @@ void RegPressureTracker::bumpUpwardPressure(const MachineInstr *MI) {
 ///
 /// This assumes that the current LiveOut set is sufficient.
 ///
-/// FIXME: This is expensive for an on-the-fly query. We need to cache the
-/// result per-SUnit with enough information to adjust for the current
-/// scheduling position. But this works as a proof of concept.
+/// This is expensive for an on-the-fly query because it calls
+/// bumpUpwardPressure to recompute the pressure sets based on current
+/// liveness. This mainly exists to verify correctness, e.g. with
+/// -verify-misched. getUpwardPressureDelta is the fast version of this query
+/// that uses the per-SUnit cache of the PressureDiff.
 void RegPressureTracker::
 getMaxUpwardPressureDelta(const MachineInstr *MI, PressureDiff *PDiff,
                           RegPressureDelta &Delta,
@@ -785,6 +797,8 @@ getMaxUpwardPressureDelta(const MachineInstr *MI, PressureDiff *PDiff,
   RegPressureDelta Delta2;
   getUpwardPressureDelta(MI, *PDiff, Delta2, CriticalPSets, MaxPressureLimit);
   if (Delta != Delta2) {
+    dbgs() << "PDiff: ";
+    PDiff->dump(*TRI);
     dbgs() << "DELTA: " << *MI;
     if (Delta.Excess.isValid())
       dbgs() << "Excess1 " << TRI->getRegPressureSetName(Delta.Excess.getPSet())
@@ -809,10 +823,8 @@ getMaxUpwardPressureDelta(const MachineInstr *MI, PressureDiff *PDiff,
 #endif
 }
 
-/// This is a prototype of the fast version of querying register pressure that
-/// does not directly depend on current liveness. It's still slow because we
-/// recompute pressure change on-the-fly. This implementation only exists to
-/// prove correctness.
+/// This is the fast version of querying register pressure that does not
+/// directly depend on current liveness.
 ///
 /// @param Delta captures information needed for heuristics.
 ///
@@ -950,6 +962,11 @@ void RegPressureTracker::bumpDownwardPressure(const MachineInstr *MI) {
 /// register units of that pressure set introduced by this instruction.
 ///
 /// This assumes that the current LiveIn set is sufficient.
+///
+/// This is expensive for an on-the-fly query because it calls
+/// bumpDownwardPressure to recompute the pressure sets based on current
+/// liveness. We don't yet have a fast version of downward pressure tracking
+/// analagous to getUpwardPressureDelta.
 void RegPressureTracker::
 getMaxDownwardPressureDelta(const MachineInstr *MI, RegPressureDelta &Delta,
                             ArrayRef<PressureChange> CriticalPSets,
diff --git a/lib/CodeGen/RegisterScavenging.cpp b/lib/CodeGen/RegisterScavenging.cpp
index 7626dd29c00..4176686d1f7 100644
--- a/lib/CodeGen/RegisterScavenging.cpp
+++ b/lib/CodeGen/RegisterScavenging.cpp
@@ -55,7 +55,8 @@ void RegScavenger::initRegState() {
     setRegUsed(*I);
 
   // Pristine CSRs are also unavailable.
-  BitVector PR = MBB->getParent()->getFrameInfo()->getPristineRegs(MBB);
+  const MachineFunction &MF = *MBB->getParent();
+  BitVector PR = MF.getFrameInfo()->getPristineRegs(MF);
   for (int I = PR.find_first(); I>0; I = PR.find_next(I))
     setRegUsed(I);
 }
@@ -102,10 +103,6 @@ void RegScavenger::determineKillsAndDefs() {
 
   // Find out which registers are early clobbered, killed, defined, and marked
   // def-dead in this instruction.
-  // FIXME: The scavenger is not predication aware. If the instruction is
-  // predicated, conservatively assume "kill" markers do not actually kill the
-  // register. Similarly ignores "dead" markers.
-  bool isPred = TII->isPredicated(MI);
   KillRegUnits.reset();
   DefRegUnits.reset();
   for (unsigned i = 0, e = MI->getNumOperands(); i != e; ++i) {
@@ -123,7 +120,7 @@ void RegScavenger::determineKillsAndDefs() {
       }
       
       // Apply the mask.
-      (isPred ? DefRegUnits : KillRegUnits) |= TmpRegUnits;
+      KillRegUnits |= TmpRegUnits;
     }
     if (!MO.isReg())
       continue;
@@ -135,11 +132,11 @@ void RegScavenger::determineKillsAndDefs() {
       // Ignore undef uses.
       if (MO.isUndef())
         continue;
-      if (!isPred && MO.isKill())
+      if (MO.isKill())
         addRegUnits(KillRegUnits, Reg);
     } else {
       assert(MO.isDef());
-      if (!isPred && MO.isDead())
+      if (MO.isDead())
         addRegUnits(KillRegUnits, Reg);
       else
         addRegUnits(DefRegUnits, Reg);
diff --git a/lib/CodeGen/ScheduleDAGInstrs.cpp b/lib/CodeGen/ScheduleDAGInstrs.cpp
index 17dd729a19a..9c07f5be8b6 100644
--- a/lib/CodeGen/ScheduleDAGInstrs.cpp
+++ b/lib/CodeGen/ScheduleDAGInstrs.cpp
@@ -20,6 +20,7 @@
 #include "llvm/Analysis/ValueTracking.h"
 #include "llvm/CodeGen/LiveIntervalAnalysis.h"
 #include "llvm/CodeGen/MachineFunctionPass.h"
+#include "llvm/CodeGen/MachineFrameInfo.h"
 #include "llvm/CodeGen/MachineInstrBuilder.h"
 #include "llvm/CodeGen/MachineMemOperand.h"
 #include "llvm/CodeGen/MachineRegisterInfo.h"
@@ -27,7 +28,6 @@
 #include "llvm/CodeGen/RegisterPressure.h"
 #include "llvm/CodeGen/ScheduleDFS.h"
 #include "llvm/IR/Operator.h"
-#include "llvm/MC/MCInstrItineraries.h"
 #include "llvm/Support/CommandLine.h"
 #include "llvm/Support/Debug.h"
 #include "llvm/Support/Format.h"
@@ -143,6 +143,13 @@ static void getUnderlyingObjectsForInstr(const MachineInstr *MI,
 
   if (const PseudoSourceValue *PSV =
       (*MI->memoperands_begin())->getPseudoValue()) {
+    // Function that contain tail calls don't have unique PseudoSourceValue
+    // objects. Two PseudoSourceValues might refer to the same or overlapping
+    // locations. The client code calling this function assumes this is not the
+    // case. So return a conservative answer of no known object.
+    if (MFI->hasTailCall())
+      return;
+
     // For now, ignore PseudoSourceValues which may alias LLVM IR values
     // because the code that uses this function has no way to cope with
     // such aliases.
@@ -160,10 +167,7 @@ static void getUnderlyingObjectsForInstr(const MachineInstr *MI,
   SmallVector<Value *, 4> Objs;
   getUnderlyingObjects(V, Objs, DL);
 
-  for (SmallVectorImpl<Value *>::iterator I = Objs.begin(), IE = Objs.end();
-         I != IE; ++I) {
-    V = *I;
-
+  for (Value *V : Objs) {
     if (!isIdentifiedObject(V)) {
       Objects.clear();
       return;
@@ -495,10 +499,9 @@ static inline bool isUnsafeMemoryObject(MachineInstr *MI,
 
   SmallVector<Value *, 4> Objs;
   getUnderlyingObjects(V, Objs, DL);
-  for (SmallVectorImpl<Value *>::iterator I = Objs.begin(),
-         IE = Objs.end(); I != IE; ++I) {
+  for (Value *V : Objs) {
     // Does this pointer refer to a distinct and identifiable object?
-    if (!isIdentifiedObject(*I))
+    if (!isIdentifiedObject(V))
       return true;
   }
 
@@ -571,13 +574,13 @@ static bool MIsNeedChainEdge(AliasAnalysis *AA, const MachineFrameInfo *MFI,
   int64_t Overlapa = MMOa->getSize() + MMOa->getOffset() - MinOffset;
   int64_t Overlapb = MMOb->getSize() + MMOb->getOffset() - MinOffset;
 
-  AliasAnalysis::AliasResult AAResult = AA->alias(
-      AliasAnalysis::Location(MMOa->getValue(), Overlapa,
-                              UseTBAA ? MMOa->getAAInfo() : AAMDNodes()),
-      AliasAnalysis::Location(MMOb->getValue(), Overlapb,
-                              UseTBAA ? MMOb->getAAInfo() : AAMDNodes()));
+  AliasResult AAResult =
+      AA->alias(MemoryLocation(MMOa->getValue(), Overlapa,
+                               UseTBAA ? MMOa->getAAInfo() : AAMDNodes()),
+                MemoryLocation(MMOb->getValue(), Overlapb,
+                               UseTBAA ? MMOb->getAAInfo() : AAMDNodes()));
 
-  return (AAResult != AliasAnalysis::NoAlias);
+  return (AAResult != NoAlias);
 }
 
 /// This recursive function iterates over chain deps of SUb looking for
@@ -887,7 +890,7 @@ void ScheduleDAGInstrs::buildSchedGraph(AliasAnalysis *AA,
       BarrierChain = SU;
       // This is a barrier event that acts as a pivotal node in the DAG,
       // so it is safe to clear list of exposed nodes.
-      adjustChainDeps(AA, MFI, *TM.getDataLayout(), SU, &ExitSU, RejectMemNodes,
+      adjustChainDeps(AA, MFI, MF.getDataLayout(), SU, &ExitSU, RejectMemNodes,
                       TrueMemOrderLatency);
       RejectMemNodes.clear();
       NonAliasMemDefs.clear();
@@ -900,27 +903,27 @@ void ScheduleDAGInstrs::buildSchedGraph(AliasAnalysis *AA,
         unsigned ChainLatency = 0;
         if (AliasChain->getInstr()->mayLoad())
           ChainLatency = TrueMemOrderLatency;
-        addChainDependency(AAForDep, MFI, *TM.getDataLayout(), SU, AliasChain,
+        addChainDependency(AAForDep, MFI, MF.getDataLayout(), SU, AliasChain,
                            RejectMemNodes, ChainLatency);
       }
       AliasChain = SU;
       for (unsigned k = 0, m = PendingLoads.size(); k != m; ++k)
-        addChainDependency(AAForDep, MFI, *TM.getDataLayout(), SU,
+        addChainDependency(AAForDep, MFI, MF.getDataLayout(), SU,
                            PendingLoads[k], RejectMemNodes,
                            TrueMemOrderLatency);
       for (MapVector<ValueType, std::vector<SUnit *> >::iterator I =
            AliasMemDefs.begin(), E = AliasMemDefs.end(); I != E; ++I) {
         for (unsigned i = 0, e = I->second.size(); i != e; ++i)
-          addChainDependency(AAForDep, MFI, *TM.getDataLayout(), SU,
+          addChainDependency(AAForDep, MFI, MF.getDataLayout(), SU,
                              I->second[i], RejectMemNodes);
       }
       for (MapVector<ValueType, std::vector<SUnit *> >::iterator I =
            AliasMemUses.begin(), E = AliasMemUses.end(); I != E; ++I) {
         for (unsigned i = 0, e = I->second.size(); i != e; ++i)
-          addChainDependency(AAForDep, MFI, *TM.getDataLayout(), SU,
+          addChainDependency(AAForDep, MFI, MF.getDataLayout(), SU,
                              I->second[i], RejectMemNodes, TrueMemOrderLatency);
       }
-      adjustChainDeps(AA, MFI, *TM.getDataLayout(), SU, &ExitSU, RejectMemNodes,
+      adjustChainDeps(AA, MFI, MF.getDataLayout(), SU, &ExitSU, RejectMemNodes,
                       TrueMemOrderLatency);
       PendingLoads.clear();
       AliasMemDefs.clear();
@@ -934,7 +937,7 @@ void ScheduleDAGInstrs::buildSchedGraph(AliasAnalysis *AA,
         BarrierChain->addPred(SDep(SU, SDep::Barrier));
 
       UnderlyingObjectsVector Objs;
-      getUnderlyingObjectsForInstr(MI, MFI, Objs, *TM.getDataLayout());
+      getUnderlyingObjectsForInstr(MI, MFI, Objs, MF.getDataLayout());
 
       if (Objs.empty()) {
         // Treat all other stores conservatively.
@@ -958,7 +961,7 @@ void ScheduleDAGInstrs::buildSchedGraph(AliasAnalysis *AA,
           ((ThisMayAlias) ? AliasMemDefs.end() : NonAliasMemDefs.end());
         if (I != IE) {
           for (unsigned i = 0, e = I->second.size(); i != e; ++i)
-            addChainDependency(AAForDep, MFI, *TM.getDataLayout(), SU,
+            addChainDependency(AAForDep, MFI, MF.getDataLayout(), SU,
                                I->second[i], RejectMemNodes, 0, true);
 
           // If we're not using AA, then we only need one store per object.
@@ -983,7 +986,7 @@ void ScheduleDAGInstrs::buildSchedGraph(AliasAnalysis *AA,
           ((ThisMayAlias) ? AliasMemUses.end() : NonAliasMemUses.end());
         if (J != JE) {
           for (unsigned i = 0, e = J->second.size(); i != e; ++i)
-            addChainDependency(AAForDep, MFI, *TM.getDataLayout(), SU,
+            addChainDependency(AAForDep, MFI, MF.getDataLayout(), SU,
                                J->second[i], RejectMemNodes,
                                TrueMemOrderLatency, true);
           J->second.clear();
@@ -993,15 +996,15 @@ void ScheduleDAGInstrs::buildSchedGraph(AliasAnalysis *AA,
         // Add dependencies from all the PendingLoads, i.e. loads
         // with no underlying object.
         for (unsigned k = 0, m = PendingLoads.size(); k != m; ++k)
-          addChainDependency(AAForDep, MFI, *TM.getDataLayout(), SU,
+          addChainDependency(AAForDep, MFI, MF.getDataLayout(), SU,
                              PendingLoads[k], RejectMemNodes,
                              TrueMemOrderLatency);
         // Add dependence on alias chain, if needed.
         if (AliasChain)
-          addChainDependency(AAForDep, MFI, *TM.getDataLayout(), SU, AliasChain,
+          addChainDependency(AAForDep, MFI, MF.getDataLayout(), SU, AliasChain,
                              RejectMemNodes);
       }
-      adjustChainDeps(AA, MFI, *TM.getDataLayout(), SU, &ExitSU, RejectMemNodes,
+      adjustChainDeps(AA, MFI, MF.getDataLayout(), SU, &ExitSU, RejectMemNodes,
                       TrueMemOrderLatency);
     } else if (MI->mayLoad()) {
       bool MayAlias = true;
@@ -1009,7 +1012,7 @@ void ScheduleDAGInstrs::buildSchedGraph(AliasAnalysis *AA,
         // Invariant load, no chain dependencies needed!
       } else {
         UnderlyingObjectsVector Objs;
-        getUnderlyingObjectsForInstr(MI, MFI, Objs, *TM.getDataLayout());
+        getUnderlyingObjectsForInstr(MI, MFI, Objs, MF.getDataLayout());
 
         if (Objs.empty()) {
           // A load with no underlying object. Depend on all
@@ -1017,7 +1020,7 @@ void ScheduleDAGInstrs::buildSchedGraph(AliasAnalysis *AA,
           for (MapVector<ValueType, std::vector<SUnit *> >::iterator I =
                  AliasMemDefs.begin(), E = AliasMemDefs.end(); I != E; ++I)
             for (unsigned i = 0, e = I->second.size(); i != e; ++i)
-              addChainDependency(AAForDep, MFI, *TM.getDataLayout(), SU,
+              addChainDependency(AAForDep, MFI, MF.getDataLayout(), SU,
                                  I->second[i], RejectMemNodes);
 
           PendingLoads.push_back(SU);
@@ -1041,7 +1044,7 @@ void ScheduleDAGInstrs::buildSchedGraph(AliasAnalysis *AA,
             ((ThisMayAlias) ? AliasMemDefs.end() : NonAliasMemDefs.end());
           if (I != IE)
             for (unsigned i = 0, e = I->second.size(); i != e; ++i)
-              addChainDependency(AAForDep, MFI, *TM.getDataLayout(), SU,
+              addChainDependency(AAForDep, MFI, MF.getDataLayout(), SU,
                                  I->second[i], RejectMemNodes, 0, true);
           if (ThisMayAlias)
             AliasMemUses[V].push_back(SU);
@@ -1049,11 +1052,11 @@ void ScheduleDAGInstrs::buildSchedGraph(AliasAnalysis *AA,
             NonAliasMemUses[V].push_back(SU);
         }
         if (MayAlias)
-          adjustChainDeps(AA, MFI, *TM.getDataLayout(), SU, &ExitSU,
+          adjustChainDeps(AA, MFI, MF.getDataLayout(), SU, &ExitSU,
                           RejectMemNodes, /*Latency=*/0);
         // Add dependencies on alias and barrier chains, if needed.
         if (MayAlias && AliasChain)
-          addChainDependency(AAForDep, MFI, *TM.getDataLayout(), SU, AliasChain,
+          addChainDependency(AAForDep, MFI, MF.getDataLayout(), SU, AliasChain,
                              RejectMemNodes);
         if (BarrierChain)
           BarrierChain->addPred(SDep(SU, SDep::Barrier));
@@ -1088,22 +1091,65 @@ void ScheduleDAGInstrs::startBlockForKills(MachineBasicBlock *BB) {
   }
 }
 
+/// \brief If we change a kill flag on the bundle instruction implicit register
+/// operands, then we also need to propagate that to any instructions inside
+/// the bundle which had the same kill state.
+static void toggleBundleKillFlag(MachineInstr *MI, unsigned Reg,
+                                 bool NewKillState) {
+  if (MI->getOpcode() != TargetOpcode::BUNDLE)
+    return;
+
+  // Walk backwards from the last instruction in the bundle to the first.
+  // Once we set a kill flag on an instruction, we bail out, as otherwise we
+  // might set it on too many operands.  We will clear as many flags as we
+  // can though.
+  MachineBasicBlock::instr_iterator Begin = MI;
+  MachineBasicBlock::instr_iterator End = getBundleEnd(MI);
+  while (Begin != End) {
+    for (MachineOperand &MO : (--End)->operands()) {
+      if (!MO.isReg() || MO.isDef() || Reg != MO.getReg())
+        continue;
+
+      // DEBUG_VALUE nodes do not contribute to code generation and should
+      // always be ignored.  Failure to do so may result in trying to modify
+      // KILL flags on DEBUG_VALUE nodes, which is distressing.
+      if (MO.isDebug())
+        continue;
+
+      // If the register has the internal flag then it could be killing an
+      // internal def of the register.  In this case, just skip.  We only want
+      // to toggle the flag on operands visible outside the bundle.
+      if (MO.isInternalRead())
+        continue;
+
+      if (MO.isKill() == NewKillState)
+        continue;
+      MO.setIsKill(NewKillState);
+      if (NewKillState)
+        return;
+    }
+  }
+}
+
 bool ScheduleDAGInstrs::toggleKillFlag(MachineInstr *MI, MachineOperand &MO) {
   // Setting kill flag...
   if (!MO.isKill()) {
     MO.setIsKill(true);
+    toggleBundleKillFlag(MI, MO.getReg(), true);
     return false;
   }
 
   // If MO itself is live, clear the kill flag...
   if (LiveRegs.test(MO.getReg())) {
     MO.setIsKill(false);
+    toggleBundleKillFlag(MI, MO.getReg(), false);
     return false;
   }
 
   // If any subreg of MO is live, then create an imp-def for that
   // subreg and keep MO marked as killed.
   MO.setIsKill(false);
+  toggleBundleKillFlag(MI, MO.getReg(), false);
   bool AllDead = true;
   const unsigned SuperReg = MO.getReg();
   MachineInstrBuilder MIB(MF, MI);
@@ -1114,8 +1160,10 @@ bool ScheduleDAGInstrs::toggleKillFlag(MachineInstr *MI, MachineOperand &MO) {
     }
   }
 
-  if(AllDead)
+  if(AllDead) {
     MO.setIsKill(true);
+    toggleBundleKillFlag(MI, MO.getReg(), true);
+  }
   return false;
 }
 
@@ -1188,6 +1236,12 @@ void ScheduleDAGInstrs::fixupKills(MachineBasicBlock *MBB) {
         // Warning: toggleKillFlag may invalidate MO.
         toggleKillFlag(MI, MO);
         DEBUG(MI->dump());
+        DEBUG(if (MI->getOpcode() == TargetOpcode::BUNDLE) {
+          MachineBasicBlock::instr_iterator Begin = MI;
+          MachineBasicBlock::instr_iterator End = getBundleEnd(MI);
+          while (++Begin != End)
+            DEBUG(Begin->dump());
+        });
       }
 
       killedRegs.set(Reg);
diff --git a/lib/CodeGen/SelectionDAG/DAGCombiner.cpp b/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
index 22fd6d68101..104bd8e237b 100644
--- a/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
+++ b/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
@@ -255,6 +255,7 @@ namespace {
     SDValue visitSRA(SDNode *N);
     SDValue visitSRL(SDNode *N);
     SDValue visitRotate(SDNode *N);
+    SDValue visitBSWAP(SDNode *N);
     SDValue visitCTLZ(SDNode *N);
     SDValue visitCTLZ_ZERO_UNDEF(SDNode *N);
     SDValue visitCTTZ(SDNode *N);
@@ -268,6 +269,7 @@ namespace {
     SDValue visitZERO_EXTEND(SDNode *N);
     SDValue visitANY_EXTEND(SDNode *N);
     SDValue visitSIGN_EXTEND_INREG(SDNode *N);
+    SDValue visitSIGN_EXTEND_VECTOR_INREG(SDNode *N);
     SDValue visitTRUNCATE(SDNode *N);
     SDValue visitBITCAST(SDNode *N);
     SDValue visitBUILD_PAIR(SDNode *N);
@@ -307,6 +309,12 @@ namespace {
     SDValue visitINSERT_SUBVECTOR(SDNode *N);
     SDValue visitMLOAD(SDNode *N);
     SDValue visitMSTORE(SDNode *N);
+    SDValue visitMGATHER(SDNode *N);
+    SDValue visitMSCATTER(SDNode *N);
+    SDValue visitFP_TO_FP16(SDNode *N);
+
+    SDValue visitFADDForFMACombine(SDNode *N);
+    SDValue visitFSUBForFMACombine(SDNode *N);
 
     SDValue XformToShuffleWithZero(SDNode *N);
     SDValue ReassociateOps(unsigned Opc, SDLoc DL, SDValue LHS, SDValue RHS);
@@ -330,6 +338,7 @@ namespace {
                                          unsigned HiOp);
     SDValue CombineConsecutiveLoads(SDNode *N, EVT VT);
     SDValue CombineExtLoad(SDNode *N);
+    SDValue combineRepeatedFPDivisors(SDNode *N);
     SDValue ConstantFoldBITCASTofBUILD_VECTOR(SDNode *, EVT);
     SDValue BuildSDIV(SDNode *N);
     SDValue BuildSDIVPow2(SDNode *N);
@@ -380,6 +389,13 @@ namespace {
       unsigned SequenceNum;
     };
 
+    /// This is a helper function for MergeStoresOfConstantsOrVecElts. Returns a
+    /// constant build_vector of the stored constant values in Stores.
+    SDValue getMergedConstantVectorStore(SelectionDAG &DAG,
+                                         SDLoc SL,
+                                         ArrayRef<MemOpLink> Stores,
+                                         EVT Ty) const;
+
     /// This is a helper function for MergeConsecutiveStores. When the source
     /// elements of the consecutive stores are all constants or all extracted
     /// vector elements, try to merge them into one larger store.
@@ -388,6 +404,13 @@ namespace {
                                          EVT MemVT, unsigned NumElem,
                                          bool IsConstantSrc, bool UseVector);
 
+    /// This is a helper function for MergeConsecutiveStores.
+    /// Stores that may be merged are placed in StoreNodes.
+    /// Loads that may alias with those stores are placed in AliasLoadNodes.
+    void getStoreMergeAndAliasCandidates(
+        StoreSDNode* St, SmallVectorImpl<MemOpLink> &StoreNodes,
+        SmallVectorImpl<LSBaseSDNode*> &AliasLoadNodes);
+    
     /// Merge consecutive store operations into a wide store.
     /// This optimization uses wide integers or vectors when possible.
     /// \return True if some memory operations were changed.
@@ -421,8 +444,9 @@ namespace {
       assert(LHSTy.isInteger() && "Shift amount is not an integer type!");
       if (LHSTy.isVector())
         return LHSTy;
-      return LegalTypes ? TLI.getScalarShiftAmountTy(LHSTy)
-                        : TLI.getPointerTy();
+      auto &DL = DAG.getDataLayout();
+      return LegalTypes ? TLI.getScalarShiftAmountTy(DL, LHSTy)
+                        : TLI.getPointerTy(DL);
     }
 
     /// This method returns true if we are running before type legalization or
@@ -434,7 +458,7 @@ namespace {
 
     /// Convenience wrapper around TargetLowering::getSetCCResultType
     EVT getSetCCResultType(EVT VT) const {
-      return TLI.getSetCCResultType(*DAG.getContext(), VT);
+      return TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT);
     }
   };
 }
@@ -588,7 +612,7 @@ static SDValue GetNegatedExpression(SDValue Op, SelectionDAG &DAG,
   case ISD::ConstantFP: {
     APFloat V = cast<ConstantFPSDNode>(Op)->getValueAPF();
     V.changeSign();
-    return DAG.getConstantFP(V, Op.getValueType());
+    return DAG.getConstantFP(V, SDLoc(Op), Op.getValueType());
   }
   case ISD::FADD:
     // FIXME: determine better conditions for this xform.
@@ -612,7 +636,7 @@ static SDValue GetNegatedExpression(SDValue Op, SelectionDAG &DAG,
 
     // fold (fneg (fsub 0, B)) -> B
     if (ConstantFPSDNode *N0CFP = dyn_cast<ConstantFPSDNode>(Op.getOperand(0)))
-      if (N0CFP->getValueAPF().isZero())
+      if (N0CFP->isZero())
         return Op.getOperand(1);
 
     // fold (fneg (fsub A, B)) -> (fsub B, A)
@@ -770,7 +794,7 @@ SDValue DAGCombiner::ReassociateOps(unsigned Opc, SDLoc DL,
     if (SDNode *L = isConstantIntBuildVectorOrConstantInt(N0.getOperand(1))) {
       if (SDNode *R = isConstantIntBuildVectorOrConstantInt(N1)) {
         // reassoc. (op (op x, c1), c2) -> (op x, (op c1, c2))
-        if (SDValue OpNode = DAG.FoldConstantArithmetic(Opc, VT, L, R))
+        if (SDValue OpNode = DAG.FoldConstantArithmetic(Opc, DL, VT, L, R))
           return DAG.getNode(Opc, DL, VT, N0.getOperand(0), OpNode);
         return SDValue();
       }
@@ -790,7 +814,7 @@ SDValue DAGCombiner::ReassociateOps(unsigned Opc, SDLoc DL,
     if (SDNode *R = isConstantIntBuildVectorOrConstantInt(N1.getOperand(1))) {
       if (SDNode *L = isConstantIntBuildVectorOrConstantInt(N0)) {
         // reassoc. (op c2, (op x, c1)) -> (op x, (op c1, c2))
-        if (SDValue OpNode = DAG.FoldConstantArithmetic(Opc, VT, R, L))
+        if (SDValue OpNode = DAG.FoldConstantArithmetic(Opc, DL, VT, R, L))
           return DAG.getNode(Opc, DL, VT, N1.getOperand(0), OpNode);
         return SDValue();
       }
@@ -1170,8 +1194,8 @@ bool DAGCombiner::recursivelyDeleteUnusedNodes(SDNode *N) {
       continue;
 
     if (N->use_empty()) {
-      for (unsigned i = 0, e = N->getNumOperands(); i != e; ++i)
-        Nodes.insert(N->getOperand(i).getNode());
+      for (const SDValue &ChildN : N->op_values())
+        Nodes.insert(ChildN.getNode());
 
       removeFromWorklist(N);
       DAG.DeleteNode(N);
@@ -1193,9 +1217,8 @@ void DAGCombiner::Run(CombineLevel AtLevel) {
   LegalTypes = Level >= AfterLegalizeTypes;
 
   // Add all the dag nodes to the worklist.
-  for (SelectionDAG::allnodes_iterator I = DAG.allnodes_begin(),
-       E = DAG.allnodes_end(); I != E; ++I)
-    AddToWorklist(I);
+  for (SDNode &Node : DAG.allnodes())
+    AddToWorklist(&Node);
 
   // Create a dummy node (which is not added to allnodes), that adds a reference
   // to the root node, preventing it from being deleted, and tracking any
@@ -1244,9 +1267,9 @@ void DAGCombiner::Run(CombineLevel AtLevel) {
     // worklist as well. Because the worklist uniques things already, this
     // won't repeatedly process the same operand.
     CombinedNodes.insert(N);
-    for (unsigned i = 0, e = N->getNumOperands(); i != e; ++i)
-      if (!CombinedNodes.count(N->getOperand(i).getNode()))
-        AddToWorklist(N->getOperand(i).getNode());
+    for (const SDValue &ChildN : N->op_values())
+      if (!CombinedNodes.count(ChildN.getNode()))
+        AddToWorklist(ChildN.getNode());
 
     SDValue RV = combine(N);
 
@@ -1328,6 +1351,7 @@ SDValue DAGCombiner::visit(SDNode *N) {
   case ISD::SRL:                return visitSRL(N);
   case ISD::ROTR:
   case ISD::ROTL:               return visitRotate(N);
+  case ISD::BSWAP:              return visitBSWAP(N);
   case ISD::CTLZ:               return visitCTLZ(N);
   case ISD::CTLZ_ZERO_UNDEF:    return visitCTLZ_ZERO_UNDEF(N);
   case ISD::CTTZ:               return visitCTTZ(N);
@@ -1341,6 +1365,7 @@ SDValue DAGCombiner::visit(SDNode *N) {
   case ISD::ZERO_EXTEND:        return visitZERO_EXTEND(N);
   case ISD::ANY_EXTEND:         return visitANY_EXTEND(N);
   case ISD::SIGN_EXTEND_INREG:  return visitSIGN_EXTEND_INREG(N);
+  case ISD::SIGN_EXTEND_VECTOR_INREG: return visitSIGN_EXTEND_VECTOR_INREG(N);
   case ISD::TRUNCATE:           return visitTRUNCATE(N);
   case ISD::BITCAST:            return visitBITCAST(N);
   case ISD::BUILD_PAIR:         return visitBUILD_PAIR(N);
@@ -1378,8 +1403,11 @@ SDValue DAGCombiner::visit(SDNode *N) {
   case ISD::VECTOR_SHUFFLE:     return visitVECTOR_SHUFFLE(N);
   case ISD::SCALAR_TO_VECTOR:   return visitSCALAR_TO_VECTOR(N);
   case ISD::INSERT_SUBVECTOR:   return visitINSERT_SUBVECTOR(N);
+  case ISD::MGATHER:            return visitMGATHER(N);
   case ISD::MLOAD:              return visitMLOAD(N);
+  case ISD::MSCATTER:           return visitMSCATTER(N);
   case ISD::MSTORE:             return visitMSTORE(N);
+  case ISD::FP_TO_FP16:         return visitFP_TO_FP16(N);
   }
   return SDValue();
 }
@@ -1443,11 +1471,9 @@ SDValue DAGCombiner::combine(SDNode *N) {
     if (isa<ConstantSDNode>(N0) || !isa<ConstantSDNode>(N1)) {
       SDValue Ops[] = {N1, N0};
       SDNode *CSENode;
-      if (const BinaryWithFlagsSDNode *BinNode =
-              dyn_cast<BinaryWithFlagsSDNode>(N)) {
-        CSENode = DAG.getNodeIfExists(
-            N->getOpcode(), N->getVTList(), Ops, BinNode->hasNoUnsignedWrap(),
-            BinNode->hasNoSignedWrap(), BinNode->isExact());
+      if (const auto *BinNode = dyn_cast<BinaryWithFlagsSDNode>(N)) {
+        CSENode = DAG.getNodeIfExists(N->getOpcode(), N->getVTList(), Ops,
+                                      &BinNode->Flags);
       } else {
         CSENode = DAG.getNodeIfExists(N->getOpcode(), N->getVTList(), Ops);
       }
@@ -1498,8 +1524,7 @@ SDValue DAGCombiner::visitTokenFactor(SDNode *N) {
     SDNode *TF = TFs[i];
 
     // Check each of the operands.
-    for (unsigned i = 0, ie = TF->getNumOperands(); i != ie; ++i) {
-      SDValue Op = TF->getOperand(i);
+    for (const SDValue &Op : TF->op_values()) {
 
       switch (Op.getOpcode()) {
       case ISD::EntryToken:
@@ -1570,6 +1595,33 @@ SDValue DAGCombiner::visitMERGE_VALUES(SDNode *N) {
   return SDValue(N, 0);   // Return N so it doesn't get rechecked!
 }
 
+static bool isNullConstant(SDValue V) {
+  ConstantSDNode *Const = dyn_cast<ConstantSDNode>(V);
+  return Const != nullptr && Const->isNullValue();
+}
+
+static bool isNullFPConstant(SDValue V) {
+  ConstantFPSDNode *Const = dyn_cast<ConstantFPSDNode>(V);
+  return Const != nullptr && Const->isZero() && !Const->isNegative();
+}
+
+static bool isAllOnesConstant(SDValue V) {
+  ConstantSDNode *Const = dyn_cast<ConstantSDNode>(V);
+  return Const != nullptr && Const->isAllOnesValue();
+}
+
+static bool isOneConstant(SDValue V) {
+  ConstantSDNode *Const = dyn_cast<ConstantSDNode>(V);
+  return Const != nullptr && Const->isOne();
+}
+
+/// If \p N is a ContantSDNode with isOpaque() == false return it casted to a
+/// ContantSDNode pointer else nullptr.
+static ConstantSDNode *getAsNonOpaqueConstant(SDValue N) {
+  ConstantSDNode *Const = dyn_cast<ConstantSDNode>(N);
+  return Const != nullptr && !Const->isOpaque() ? Const : nullptr;
+}
+
 SDValue DAGCombiner::visitADD(SDNode *N) {
   SDValue N0 = N->getOperand(0);
   SDValue N1 = N->getOperand(1);
@@ -1593,16 +1645,16 @@ SDValue DAGCombiner::visitADD(SDNode *N) {
   if (N1.getOpcode() == ISD::UNDEF)
     return N1;
   // fold (add c1, c2) -> c1+c2
-  ConstantSDNode *N0C = dyn_cast<ConstantSDNode>(N0);
-  ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(N1);
+  ConstantSDNode *N0C = getAsNonOpaqueConstant(N0);
+  ConstantSDNode *N1C = getAsNonOpaqueConstant(N1);
   if (N0C && N1C)
-    return DAG.FoldConstantArithmetic(ISD::ADD, VT, N0C, N1C);
+    return DAG.FoldConstantArithmetic(ISD::ADD, SDLoc(N), VT, N0C, N1C);
   // canonicalize constant to RHS
   if (isConstantIntBuildVectorOrConstantInt(N0) &&
      !isConstantIntBuildVectorOrConstantInt(N1))
     return DAG.getNode(ISD::ADD, SDLoc(N), VT, N1, N0);
   // fold (add x, 0) -> x
-  if (N1C && N1C->isNullValue())
+  if (isNullConstant(N1))
     return N0;
   // fold (add Sym, c) -> Sym+c
   if (GlobalAddressSDNode *GA = dyn_cast<GlobalAddressSDNode>(N0))
@@ -1613,21 +1665,21 @@ SDValue DAGCombiner::visitADD(SDNode *N) {
                                     (uint64_t)N1C->getSExtValue());
   // fold ((c1-A)+c2) -> (c1+c2)-A
   if (N1C && N0.getOpcode() == ISD::SUB)
-    if (ConstantSDNode *N0C = dyn_cast<ConstantSDNode>(N0.getOperand(0)))
-      return DAG.getNode(ISD::SUB, SDLoc(N), VT,
+    if (ConstantSDNode *N0C = getAsNonOpaqueConstant(N0.getOperand(0))) {
+      SDLoc DL(N);
+      return DAG.getNode(ISD::SUB, DL, VT,
                          DAG.getConstant(N1C->getAPIntValue()+
-                                         N0C->getAPIntValue(), VT),
+                                         N0C->getAPIntValue(), DL, VT),
                          N0.getOperand(1));
+    }
   // reassociate add
   if (SDValue RADD = ReassociateOps(ISD::ADD, SDLoc(N), N0, N1))
     return RADD;
   // fold ((0-A) + B) -> B-A
-  if (N0.getOpcode() == ISD::SUB && isa<ConstantSDNode>(N0.getOperand(0)) &&
-      cast<ConstantSDNode>(N0.getOperand(0))->isNullValue())
+  if (N0.getOpcode() == ISD::SUB && isNullConstant(N0.getOperand(0)))
     return DAG.getNode(ISD::SUB, SDLoc(N), VT, N1, N0.getOperand(1));
   // fold (A + (0-B)) -> A-B
-  if (N1.getOpcode() == ISD::SUB && isa<ConstantSDNode>(N1.getOperand(0)) &&
-      cast<ConstantSDNode>(N1.getOperand(0))->isNullValue())
+  if (N1.getOpcode() == ISD::SUB && isNullConstant(N1.getOperand(0)))
     return DAG.getNode(ISD::SUB, SDLoc(N), VT, N0, N1.getOperand(1));
   // fold (A+(B-A)) -> B
   if (N1.getOpcode() == ISD::SUB && N0 == N1.getOperand(1))
@@ -1687,34 +1739,27 @@ SDValue DAGCombiner::visitADD(SDNode *N) {
   }
 
   // fold (add x, shl(0 - y, n)) -> sub(x, shl(y, n))
-  if (N1.getOpcode() == ISD::SHL &&
-      N1.getOperand(0).getOpcode() == ISD::SUB)
-    if (ConstantSDNode *C =
-          dyn_cast<ConstantSDNode>(N1.getOperand(0).getOperand(0)))
-      if (C->getAPIntValue() == 0)
-        return DAG.getNode(ISD::SUB, SDLoc(N), VT, N0,
-                           DAG.getNode(ISD::SHL, SDLoc(N), VT,
-                                       N1.getOperand(0).getOperand(1),
-                                       N1.getOperand(1)));
-  if (N0.getOpcode() == ISD::SHL &&
-      N0.getOperand(0).getOpcode() == ISD::SUB)
-    if (ConstantSDNode *C =
-          dyn_cast<ConstantSDNode>(N0.getOperand(0).getOperand(0)))
-      if (C->getAPIntValue() == 0)
-        return DAG.getNode(ISD::SUB, SDLoc(N), VT, N1,
-                           DAG.getNode(ISD::SHL, SDLoc(N), VT,
-                                       N0.getOperand(0).getOperand(1),
-                                       N0.getOperand(1)));
+  if (N1.getOpcode() == ISD::SHL && N1.getOperand(0).getOpcode() == ISD::SUB &&
+      isNullConstant(N1.getOperand(0).getOperand(0)))
+    return DAG.getNode(ISD::SUB, SDLoc(N), VT, N0,
+                       DAG.getNode(ISD::SHL, SDLoc(N), VT,
+                                   N1.getOperand(0).getOperand(1),
+                                   N1.getOperand(1)));
+  if (N0.getOpcode() == ISD::SHL && N0.getOperand(0).getOpcode() == ISD::SUB &&
+      isNullConstant(N0.getOperand(0).getOperand(0)))
+    return DAG.getNode(ISD::SUB, SDLoc(N), VT, N1,
+                       DAG.getNode(ISD::SHL, SDLoc(N), VT,
+                                   N0.getOperand(0).getOperand(1),
+                                   N0.getOperand(1)));
 
   if (N1.getOpcode() == ISD::AND) {
     SDValue AndOp0 = N1.getOperand(0);
-    ConstantSDNode *AndOp1 = dyn_cast<ConstantSDNode>(N1->getOperand(1));
     unsigned NumSignBits = DAG.ComputeNumSignBits(AndOp0);
     unsigned DestBits = VT.getScalarType().getSizeInBits();
 
     // (add z, (and (sbbl x, x), 1)) -> (sub z, (sbbl x, x))
     // and similar xforms where the inner op is either ~0 or 0.
-    if (NumSignBits == DestBits && AndOp1 && AndOp1->isOne()) {
+    if (NumSignBits == DestBits && isOneConstant(N1->getOperand(1))) {
       SDLoc DL(N);
       return DAG.getNode(ISD::SUB, DL, VT, N->getOperand(0), AndOp0);
     }
@@ -1735,7 +1780,7 @@ SDValue DAGCombiner::visitADD(SDNode *N) {
     if (TN->getVT() == MVT::i1) {
       SDLoc DL(N);
       SDValue ZExt = DAG.getNode(ISD::AND, DL, VT, N1.getOperand(0),
-                                 DAG.getConstant(1, VT));
+                                 DAG.getConstant(1, DL, VT));
       return DAG.getNode(ISD::SUB, DL, VT, N0, ZExt);
     }
   }
@@ -1761,7 +1806,7 @@ SDValue DAGCombiner::visitADDC(SDNode *N) {
     return DAG.getNode(ISD::ADDC, SDLoc(N), N->getVTList(), N1, N0);
 
   // fold (addc x, 0) -> x + no carry out
-  if (N1C && N1C->isNullValue())
+  if (isNullConstant(N1))
     return CombineTo(N, N0, DAG.getNode(ISD::CARRY_FALSE,
                                         SDLoc(N), MVT::Glue));
 
@@ -1809,9 +1854,9 @@ static SDValue tryFoldToZero(SDLoc DL, const TargetLowering &TLI, EVT VT,
                              SelectionDAG &DAG,
                              bool LegalOperations, bool LegalTypes) {
   if (!VT.isVector())
-    return DAG.getConstant(0, VT);
+    return DAG.getConstant(0, DL, VT);
   if (!LegalOperations || TLI.isOperationLegal(ISD::BUILD_VECTOR, VT))
-    return DAG.getConstant(0, VT);
+    return DAG.getConstant(0, DL, VT);
   return SDValue();
 }
 
@@ -1835,16 +1880,18 @@ SDValue DAGCombiner::visitSUB(SDNode *N) {
   if (N0 == N1)
     return tryFoldToZero(SDLoc(N), TLI, VT, DAG, LegalOperations, LegalTypes);
   // fold (sub c1, c2) -> c1-c2
-  ConstantSDNode *N0C = dyn_cast<ConstantSDNode>(N0.getNode());
-  ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(N1.getNode());
+  ConstantSDNode *N0C = getAsNonOpaqueConstant(N0);
+  ConstantSDNode *N1C = getAsNonOpaqueConstant(N1);
   if (N0C && N1C)
-    return DAG.FoldConstantArithmetic(ISD::SUB, VT, N0C, N1C);
+    return DAG.FoldConstantArithmetic(ISD::SUB, SDLoc(N), VT, N0C, N1C);
   // fold (sub x, c) -> (add x, -c)
-  if (N1C)
-    return DAG.getNode(ISD::ADD, SDLoc(N), VT, N0,
-                       DAG.getConstant(-N1C->getAPIntValue(), VT));
+  if (N1C) {
+    SDLoc DL(N);
+    return DAG.getNode(ISD::ADD, DL, VT, N0,
+                       DAG.getConstant(-N1C->getAPIntValue(), DL, VT));
+  }
   // Canonicalize (sub -1, x) -> ~x, i.e. (xor x, -1)
-  if (N0C && N0C->isAllOnesValue())
+  if (isAllOnesConstant(N0))
     return DAG.getNode(ISD::XOR, SDLoc(N), VT, N1, N0);
   // fold A-(A-B) -> B
   if (N1.getOpcode() == ISD::SUB && N0 == N1.getOperand(0))
@@ -1859,9 +1906,10 @@ SDValue DAGCombiner::visitSUB(SDNode *N) {
   ConstantSDNode *N1C1 = N1.getOpcode() != ISD::ADD ? nullptr :
     dyn_cast<ConstantSDNode>(N1.getOperand(1).getNode());
   if (N1.getOpcode() == ISD::ADD && N0C && N1C1) {
+    SDLoc DL(N);
     SDValue NewC = DAG.getConstant(N0C->getAPIntValue() - N1C1->getAPIntValue(),
-                                   VT);
-    return DAG.getNode(ISD::SUB, SDLoc(N), VT, NewC,
+                                   DL, VT);
+    return DAG.getNode(ISD::SUB, DL, VT, NewC,
                        N1.getOperand(0));
   }
   // fold ((A+(B+or-C))-B) -> A+or-C
@@ -1902,7 +1950,7 @@ SDValue DAGCombiner::visitSUB(SDNode *N) {
       if (GlobalAddressSDNode *GB = dyn_cast<GlobalAddressSDNode>(N1))
         if (GA->getGlobal() == GB->getGlobal())
           return DAG.getConstant((uint64_t)GA->getOffset() - GB->getOffset(),
-                                 VT);
+                                 SDLoc(N), VT);
     }
 
   // sub X, (sextinreg Y i1) -> add X, (and Y 1)
@@ -1911,7 +1959,7 @@ SDValue DAGCombiner::visitSUB(SDNode *N) {
     if (TN->getVT() == MVT::i1) {
       SDLoc DL(N);
       SDValue ZExt = DAG.getNode(ISD::AND, DL, VT, N1.getOperand(0),
-                                 DAG.getConstant(1, VT));
+                                 DAG.getConstant(1, DL, VT));
       return DAG.getNode(ISD::ADD, DL, VT, N0, ZExt);
     }
   }
@@ -1931,20 +1979,20 @@ SDValue DAGCombiner::visitSUBC(SDNode *N) {
                                  MVT::Glue));
 
   // fold (subc x, x) -> 0 + no borrow
-  if (N0 == N1)
-    return CombineTo(N, DAG.getConstant(0, VT),
-                     DAG.getNode(ISD::CARRY_FALSE, SDLoc(N),
+  if (N0 == N1) {
+    SDLoc DL(N);
+    return CombineTo(N, DAG.getConstant(0, DL, VT),
+                     DAG.getNode(ISD::CARRY_FALSE, DL,
                                  MVT::Glue));
+  }
 
   // fold (subc x, 0) -> x + no borrow
-  ConstantSDNode *N0C = dyn_cast<ConstantSDNode>(N0);
-  ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(N1);
-  if (N1C && N1C->isNullValue())
+  if (isNullConstant(N1))
     return CombineTo(N, N0, DAG.getNode(ISD::CARRY_FALSE, SDLoc(N),
                                         MVT::Glue));
 
   // Canonicalize (sub -1, x) -> ~x, i.e. (xor x, -1) + no borrow
-  if (N0C && N0C->isAllOnesValue())
+  if (isAllOnesConstant(N0))
     return CombineTo(N, DAG.getNode(ISD::XOR, SDLoc(N), VT, N1, N0),
                      DAG.getNode(ISD::CARRY_FALSE, SDLoc(N),
                                  MVT::Glue));
@@ -1971,10 +2019,12 @@ SDValue DAGCombiner::visitMUL(SDNode *N) {
 
   // fold (mul x, undef) -> 0
   if (N0.getOpcode() == ISD::UNDEF || N1.getOpcode() == ISD::UNDEF)
-    return DAG.getConstant(0, VT);
+    return DAG.getConstant(0, SDLoc(N), VT);
 
   bool N0IsConst = false;
   bool N1IsConst = false;
+  bool N1IsOpaqueConst = false;
+  bool N0IsOpaqueConst = false;
   APInt ConstValue0, ConstValue1;
   // fold vector ops
   if (VT.isVector()) {
@@ -1985,16 +2035,21 @@ SDValue DAGCombiner::visitMUL(SDNode *N) {
     N1IsConst = isConstantSplatVector(N1.getNode(), ConstValue1);
   } else {
     N0IsConst = isa<ConstantSDNode>(N0);
-    if (N0IsConst)
+    if (N0IsConst) {
       ConstValue0 = cast<ConstantSDNode>(N0)->getAPIntValue();
+      N0IsOpaqueConst = cast<ConstantSDNode>(N0)->isOpaque();
+    }
     N1IsConst = isa<ConstantSDNode>(N1);
-    if (N1IsConst)
+    if (N1IsConst) {
       ConstValue1 = cast<ConstantSDNode>(N1)->getAPIntValue();
+      N1IsOpaqueConst = cast<ConstantSDNode>(N1)->isOpaque();
+    }
   }
 
   // fold (mul c1, c2) -> c1*c2
-  if (N0IsConst && N1IsConst)
-    return DAG.FoldConstantArithmetic(ISD::MUL, VT, N0.getNode(), N1.getNode());
+  if (N0IsConst && N1IsConst && !N0IsOpaqueConst && !N1IsOpaqueConst)
+    return DAG.FoldConstantArithmetic(ISD::MUL, SDLoc(N), VT,
+                                      N0.getNode(), N1.getNode());
 
   // canonicalize constant to RHS (vector doesn't have to splat)
   if (isConstantIntBuildVectorOrConstantInt(N0) &&
@@ -2011,23 +2066,30 @@ SDValue DAGCombiner::visitMUL(SDNode *N) {
   if (N1IsConst && ConstValue1 == 1 && IsFullSplat)
     return N0;
   // fold (mul x, -1) -> 0-x
-  if (N1IsConst && ConstValue1.isAllOnesValue())
-    return DAG.getNode(ISD::SUB, SDLoc(N), VT,
-                       DAG.getConstant(0, VT), N0);
+  if (N1IsConst && ConstValue1.isAllOnesValue()) {
+    SDLoc DL(N);
+    return DAG.getNode(ISD::SUB, DL, VT,
+                       DAG.getConstant(0, DL, VT), N0);
+  }
   // fold (mul x, (1 << c)) -> x << c
-  if (N1IsConst && ConstValue1.isPowerOf2() && IsFullSplat)
-    return DAG.getNode(ISD::SHL, SDLoc(N), VT, N0,
-                       DAG.getConstant(ConstValue1.logBase2(),
+  if (N1IsConst && !N1IsOpaqueConst && ConstValue1.isPowerOf2() &&
+      IsFullSplat) {
+    SDLoc DL(N);
+    return DAG.getNode(ISD::SHL, DL, VT, N0,
+                       DAG.getConstant(ConstValue1.logBase2(), DL,
                                        getShiftAmountTy(N0.getValueType())));
+  }
   // fold (mul x, -(1 << c)) -> -(x << c) or (-x) << c
-  if (N1IsConst && (-ConstValue1).isPowerOf2() && IsFullSplat) {
+  if (N1IsConst && !N1IsOpaqueConst && (-ConstValue1).isPowerOf2() &&
+      IsFullSplat) {
     unsigned Log2Val = (-ConstValue1).logBase2();
+    SDLoc DL(N);
     // FIXME: If the input is something that is easily negated (e.g. a
     // single-use add), we should put the negate there.
-    return DAG.getNode(ISD::SUB, SDLoc(N), VT,
-                       DAG.getConstant(0, VT),
-                       DAG.getNode(ISD::SHL, SDLoc(N), VT, N0,
-                            DAG.getConstant(Log2Val,
+    return DAG.getNode(ISD::SUB, DL, VT,
+                       DAG.getConstant(0, DL, VT),
+                       DAG.getNode(ISD::SHL, DL, VT, N0,
+                            DAG.getConstant(Log2Val, DL,
                                       getShiftAmountTy(N0.getValueType()))));
   }
 
@@ -2097,15 +2159,17 @@ SDValue DAGCombiner::visitSDIV(SDNode *N) {
   // fold (sdiv c1, c2) -> c1/c2
   ConstantSDNode *N0C = isConstOrConstSplat(N0);
   ConstantSDNode *N1C = isConstOrConstSplat(N1);
-  if (N0C && N1C && !N1C->isNullValue())
-    return DAG.FoldConstantArithmetic(ISD::SDIV, VT, N0C, N1C);
+  if (N0C && N1C && !N0C->isOpaque() && !N1C->isOpaque())
+    return DAG.FoldConstantArithmetic(ISD::SDIV, SDLoc(N), VT, N0C, N1C);
   // fold (sdiv X, 1) -> X
-  if (N1C && N1C->getAPIntValue() == 1LL)
+  if (N1C && N1C->isOne())
     return N0;
   // fold (sdiv X, -1) -> 0-X
-  if (N1C && N1C->isAllOnesValue())
-    return DAG.getNode(ISD::SUB, SDLoc(N), VT,
-                       DAG.getConstant(0, VT), N0);
+  if (N1C && N1C->isAllOnesValue()) {
+    SDLoc DL(N);
+    return DAG.getNode(ISD::SUB, DL, VT,
+                       DAG.getConstant(0, DL, VT), N0);
+  }
   // If we know the sign bits of both operands are zero, strength reduce to a
   // udiv instead.  Handles (X&15) /s 4 -> X&15 >> 2
   if (!VT.isVector()) {
@@ -2115,37 +2179,43 @@ SDValue DAGCombiner::visitSDIV(SDNode *N) {
   }
 
   // fold (sdiv X, pow2) -> simple ops after legalize
-  if (N1C && !N1C->isNullValue() && (N1C->getAPIntValue().isPowerOf2() ||
-                                     (-N1C->getAPIntValue()).isPowerOf2())) {
+  // FIXME: We check for the exact bit here because the generic lowering gives
+  // better results in that case. The target-specific lowering should learn how
+  // to handle exact sdivs efficiently.
+  if (N1C && !N1C->isNullValue() && !N1C->isOpaque() &&
+      !cast<BinaryWithFlagsSDNode>(N)->Flags.hasExact() &&
+      (N1C->getAPIntValue().isPowerOf2() ||
+       (-N1C->getAPIntValue()).isPowerOf2())) {
     // If dividing by powers of two is cheap, then don't perform the following
     // fold.
     if (TLI.isPow2SDivCheap())
       return SDValue();
 
     // Target-specific implementation of sdiv x, pow2.
-    SDValue Res = BuildSDIVPow2(N);
-    if (Res.getNode())
+    if (SDValue Res = BuildSDIVPow2(N))
       return Res;
 
     unsigned lg2 = N1C->getAPIntValue().countTrailingZeros();
+    SDLoc DL(N);
 
     // Splat the sign bit into the register
     SDValue SGN =
-        DAG.getNode(ISD::SRA, SDLoc(N), VT, N0,
-                    DAG.getConstant(VT.getScalarSizeInBits() - 1,
+        DAG.getNode(ISD::SRA, DL, VT, N0,
+                    DAG.getConstant(VT.getScalarSizeInBits() - 1, DL,
                                     getShiftAmountTy(N0.getValueType())));
     AddToWorklist(SGN.getNode());
 
     // Add (N0 < 0) ? abs2 - 1 : 0;
     SDValue SRL =
-        DAG.getNode(ISD::SRL, SDLoc(N), VT, SGN,
-                    DAG.getConstant(VT.getScalarSizeInBits() - lg2,
+        DAG.getNode(ISD::SRL, DL, VT, SGN,
+                    DAG.getConstant(VT.getScalarSizeInBits() - lg2, DL,
                                     getShiftAmountTy(SGN.getValueType())));
-    SDValue ADD = DAG.getNode(ISD::ADD, SDLoc(N), VT, N0, SRL);
+    SDValue ADD = DAG.getNode(ISD::ADD, DL, VT, N0, SRL);
     AddToWorklist(SRL.getNode());
     AddToWorklist(ADD.getNode());    // Divide by pow2
-    SDValue SRA = DAG.getNode(ISD::SRA, SDLoc(N), VT, ADD,
-                  DAG.getConstant(lg2, getShiftAmountTy(ADD.getValueType())));
+    SDValue SRA = DAG.getNode(ISD::SRA, DL, VT, ADD,
+                  DAG.getConstant(lg2, DL,
+                                  getShiftAmountTy(ADD.getValueType())));
 
     // If we're dividing by a positive value, we're done.  Otherwise, we must
     // negate the result.
@@ -2153,19 +2223,18 @@ SDValue DAGCombiner::visitSDIV(SDNode *N) {
       return SRA;
 
     AddToWorklist(SRA.getNode());
-    return DAG.getNode(ISD::SUB, SDLoc(N), VT, DAG.getConstant(0, VT), SRA);
+    return DAG.getNode(ISD::SUB, DL, VT, DAG.getConstant(0, DL, VT), SRA);
   }
 
   // If integer divide is expensive and we satisfy the requirements, emit an
   // alternate sequence.
-  if (N1C && !TLI.isIntDivCheap()) {
-    SDValue Op = BuildSDIV(N);
-    if (Op.getNode()) return Op;
-  }
+  if (N1C && !TLI.isIntDivCheap())
+    if (SDValue Op = BuildSDIV(N))
+      return Op;
 
   // undef / X -> 0
   if (N0.getOpcode() == ISD::UNDEF)
-    return DAG.getConstant(0, VT);
+    return DAG.getConstant(0, SDLoc(N), VT);
   // X / undef -> undef
   if (N1.getOpcode() == ISD::UNDEF)
     return N1;
@@ -2186,37 +2255,41 @@ SDValue DAGCombiner::visitUDIV(SDNode *N) {
   // fold (udiv c1, c2) -> c1/c2
   ConstantSDNode *N0C = isConstOrConstSplat(N0);
   ConstantSDNode *N1C = isConstOrConstSplat(N1);
-  if (N0C && N1C && !N1C->isNullValue())
-    return DAG.FoldConstantArithmetic(ISD::UDIV, VT, N0C, N1C);
+  if (N0C && N1C)
+    if (SDValue Folded = DAG.FoldConstantArithmetic(ISD::UDIV, SDLoc(N), VT,
+                                                    N0C, N1C))
+      return Folded;
   // fold (udiv x, (1 << c)) -> x >>u c
-  if (N1C && N1C->getAPIntValue().isPowerOf2())
-    return DAG.getNode(ISD::SRL, SDLoc(N), VT, N0,
-                       DAG.getConstant(N1C->getAPIntValue().logBase2(),
+  if (N1C && !N1C->isOpaque() && N1C->getAPIntValue().isPowerOf2()) {
+    SDLoc DL(N);
+    return DAG.getNode(ISD::SRL, DL, VT, N0,
+                       DAG.getConstant(N1C->getAPIntValue().logBase2(), DL,
                                        getShiftAmountTy(N0.getValueType())));
+  }
   // fold (udiv x, (shl c, y)) -> x >>u (log2(c)+y) iff c is power of 2
   if (N1.getOpcode() == ISD::SHL) {
-    if (ConstantSDNode *SHC = dyn_cast<ConstantSDNode>(N1.getOperand(0))) {
+    if (ConstantSDNode *SHC = getAsNonOpaqueConstant(N1.getOperand(0))) {
       if (SHC->getAPIntValue().isPowerOf2()) {
         EVT ADDVT = N1.getOperand(1).getValueType();
-        SDValue Add = DAG.getNode(ISD::ADD, SDLoc(N), ADDVT,
+        SDLoc DL(N);
+        SDValue Add = DAG.getNode(ISD::ADD, DL, ADDVT,
                                   N1.getOperand(1),
                                   DAG.getConstant(SHC->getAPIntValue()
                                                                   .logBase2(),
-                                                  ADDVT));
+                                                  DL, ADDVT));
         AddToWorklist(Add.getNode());
-        return DAG.getNode(ISD::SRL, SDLoc(N), VT, N0, Add);
+        return DAG.getNode(ISD::SRL, DL, VT, N0, Add);
       }
     }
   }
   // fold (udiv x, c) -> alternate
-  if (N1C && !TLI.isIntDivCheap()) {
-    SDValue Op = BuildUDIV(N);
-    if (Op.getNode()) return Op;
-  }
+  if (N1C && !TLI.isIntDivCheap())
+    if (SDValue Op = BuildUDIV(N))
+      return Op;
 
   // undef / X -> 0
   if (N0.getOpcode() == ISD::UNDEF)
-    return DAG.getConstant(0, VT);
+    return DAG.getConstant(0, SDLoc(N), VT);
   // X / undef -> undef
   if (N1.getOpcode() == ISD::UNDEF)
     return N1;
@@ -2232,8 +2305,10 @@ SDValue DAGCombiner::visitSREM(SDNode *N) {
   // fold (srem c1, c2) -> c1%c2
   ConstantSDNode *N0C = isConstOrConstSplat(N0);
   ConstantSDNode *N1C = isConstOrConstSplat(N1);
-  if (N0C && N1C && !N1C->isNullValue())
-    return DAG.FoldConstantArithmetic(ISD::SREM, VT, N0C, N1C);
+  if (N0C && N1C)
+    if (SDValue Folded = DAG.FoldConstantArithmetic(ISD::SREM, SDLoc(N), VT,
+                                                    N0C, N1C))
+      return Folded;
   // If we know the sign bits of both operands are zero, strength reduce to a
   // urem instead.  Handles (X & 0x0FFFFFFF) %s 16 -> X&15
   if (!VT.isVector()) {
@@ -2258,7 +2333,7 @@ SDValue DAGCombiner::visitSREM(SDNode *N) {
 
   // undef % X -> 0
   if (N0.getOpcode() == ISD::UNDEF)
-    return DAG.getConstant(0, VT);
+    return DAG.getConstant(0, SDLoc(N), VT);
   // X % undef -> undef
   if (N1.getOpcode() == ISD::UNDEF)
     return N1;
@@ -2274,22 +2349,28 @@ SDValue DAGCombiner::visitUREM(SDNode *N) {
   // fold (urem c1, c2) -> c1%c2
   ConstantSDNode *N0C = isConstOrConstSplat(N0);
   ConstantSDNode *N1C = isConstOrConstSplat(N1);
-  if (N0C && N1C && !N1C->isNullValue())
-    return DAG.FoldConstantArithmetic(ISD::UREM, VT, N0C, N1C);
+  if (N0C && N1C)
+    if (SDValue Folded = DAG.FoldConstantArithmetic(ISD::UREM, SDLoc(N), VT,
+                                                    N0C, N1C))
+      return Folded;
   // fold (urem x, pow2) -> (and x, pow2-1)
-  if (N1C && !N1C->isNullValue() && N1C->getAPIntValue().isPowerOf2())
-    return DAG.getNode(ISD::AND, SDLoc(N), VT, N0,
-                       DAG.getConstant(N1C->getAPIntValue()-1,VT));
+  if (N1C && !N1C->isNullValue() && !N1C->isOpaque() &&
+      N1C->getAPIntValue().isPowerOf2()) {
+    SDLoc DL(N);
+    return DAG.getNode(ISD::AND, DL, VT, N0,
+                       DAG.getConstant(N1C->getAPIntValue() - 1, DL, VT));
+  }
   // fold (urem x, (shl pow2, y)) -> (and x, (add (shl pow2, y), -1))
   if (N1.getOpcode() == ISD::SHL) {
-    if (ConstantSDNode *SHC = dyn_cast<ConstantSDNode>(N1.getOperand(0))) {
+    if (ConstantSDNode *SHC = getAsNonOpaqueConstant(N1.getOperand(0))) {
       if (SHC->getAPIntValue().isPowerOf2()) {
+        SDLoc DL(N);
         SDValue Add =
-          DAG.getNode(ISD::ADD, SDLoc(N), VT, N1,
-                 DAG.getConstant(APInt::getAllOnesValue(VT.getSizeInBits()),
+          DAG.getNode(ISD::ADD, DL, VT, N1,
+                 DAG.getConstant(APInt::getAllOnesValue(VT.getSizeInBits()), DL,
                                  VT));
         AddToWorklist(Add.getNode());
-        return DAG.getNode(ISD::AND, SDLoc(N), VT, N0, Add);
+        return DAG.getNode(ISD::AND, DL, VT, N0, Add);
       }
     }
   }
@@ -2311,7 +2392,7 @@ SDValue DAGCombiner::visitUREM(SDNode *N) {
 
   // undef % X -> 0
   if (N0.getOpcode() == ISD::UNDEF)
-    return DAG.getConstant(0, VT);
+    return DAG.getConstant(0, SDLoc(N), VT);
   // X % undef -> undef
   if (N1.getOpcode() == ISD::UNDEF)
     return N1;
@@ -2322,21 +2403,23 @@ SDValue DAGCombiner::visitUREM(SDNode *N) {
 SDValue DAGCombiner::visitMULHS(SDNode *N) {
   SDValue N0 = N->getOperand(0);
   SDValue N1 = N->getOperand(1);
-  ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(N1);
   EVT VT = N->getValueType(0);
   SDLoc DL(N);
 
   // fold (mulhs x, 0) -> 0
-  if (N1C && N1C->isNullValue())
+  if (isNullConstant(N1))
     return N1;
   // fold (mulhs x, 1) -> (sra x, size(x)-1)
-  if (N1C && N1C->getAPIntValue() == 1)
-    return DAG.getNode(ISD::SRA, SDLoc(N), N0.getValueType(), N0,
+  if (isOneConstant(N1)) {
+    SDLoc DL(N);
+    return DAG.getNode(ISD::SRA, DL, N0.getValueType(), N0,
                        DAG.getConstant(N0.getValueType().getSizeInBits() - 1,
+                                       DL,
                                        getShiftAmountTy(N0.getValueType())));
+  }
   // fold (mulhs x, undef) -> 0
   if (N0.getOpcode() == ISD::UNDEF || N1.getOpcode() == ISD::UNDEF)
-    return DAG.getConstant(0, VT);
+    return DAG.getConstant(0, SDLoc(N), VT);
 
   // If the type twice as wide is legal, transform the mulhs to a wider multiply
   // plus a shift.
@@ -2349,7 +2432,8 @@ SDValue DAGCombiner::visitMULHS(SDNode *N) {
       N1 = DAG.getNode(ISD::SIGN_EXTEND, DL, NewVT, N1);
       N1 = DAG.getNode(ISD::MUL, DL, NewVT, N0, N1);
       N1 = DAG.getNode(ISD::SRL, DL, NewVT, N1,
-            DAG.getConstant(SimpleSize, getShiftAmountTy(N1.getValueType())));
+            DAG.getConstant(SimpleSize, DL,
+                            getShiftAmountTy(N1.getValueType())));
       return DAG.getNode(ISD::TRUNCATE, DL, VT, N1);
     }
   }
@@ -2360,19 +2444,18 @@ SDValue DAGCombiner::visitMULHS(SDNode *N) {
 SDValue DAGCombiner::visitMULHU(SDNode *N) {
   SDValue N0 = N->getOperand(0);
   SDValue N1 = N->getOperand(1);
-  ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(N1);
   EVT VT = N->getValueType(0);
   SDLoc DL(N);
 
   // fold (mulhu x, 0) -> 0
-  if (N1C && N1C->isNullValue())
+  if (isNullConstant(N1))
     return N1;
   // fold (mulhu x, 1) -> 0
-  if (N1C && N1C->getAPIntValue() == 1)
-    return DAG.getConstant(0, N0.getValueType());
+  if (isOneConstant(N1))
+    return DAG.getConstant(0, DL, N0.getValueType());
   // fold (mulhu x, undef) -> 0
   if (N0.getOpcode() == ISD::UNDEF || N1.getOpcode() == ISD::UNDEF)
-    return DAG.getConstant(0, VT);
+    return DAG.getConstant(0, DL, VT);
 
   // If the type twice as wide is legal, transform the mulhu to a wider multiply
   // plus a shift.
@@ -2385,7 +2468,8 @@ SDValue DAGCombiner::visitMULHU(SDNode *N) {
       N1 = DAG.getNode(ISD::ZERO_EXTEND, DL, NewVT, N1);
       N1 = DAG.getNode(ISD::MUL, DL, NewVT, N0, N1);
       N1 = DAG.getNode(ISD::SRL, DL, NewVT, N1,
-            DAG.getConstant(SimpleSize, getShiftAmountTy(N1.getValueType())));
+            DAG.getConstant(SimpleSize, DL,
+                            getShiftAmountTy(N1.getValueType())));
       return DAG.getNode(ISD::TRUNCATE, DL, VT, N1);
     }
   }
@@ -2445,8 +2529,8 @@ SDValue DAGCombiner::SimplifyNodeWithTwoResults(SDNode *N, unsigned LoOp,
 }
 
 SDValue DAGCombiner::visitSMUL_LOHI(SDNode *N) {
-  SDValue Res = SimplifyNodeWithTwoResults(N, ISD::MUL, ISD::MULHS);
-  if (Res.getNode()) return Res;
+  if (SDValue Res = SimplifyNodeWithTwoResults(N, ISD::MUL, ISD::MULHS))
+    return Res;
 
   EVT VT = N->getValueType(0);
   SDLoc DL(N);
@@ -2463,7 +2547,8 @@ SDValue DAGCombiner::visitSMUL_LOHI(SDNode *N) {
       Lo = DAG.getNode(ISD::MUL, DL, NewVT, Lo, Hi);
       // Compute the high part as N1.
       Hi = DAG.getNode(ISD::SRL, DL, NewVT, Lo,
-            DAG.getConstant(SimpleSize, getShiftAmountTy(Lo.getValueType())));
+            DAG.getConstant(SimpleSize, DL,
+                            getShiftAmountTy(Lo.getValueType())));
       Hi = DAG.getNode(ISD::TRUNCATE, DL, VT, Hi);
       // Compute the low part as N0.
       Lo = DAG.getNode(ISD::TRUNCATE, DL, VT, Lo);
@@ -2475,8 +2560,8 @@ SDValue DAGCombiner::visitSMUL_LOHI(SDNode *N) {
 }
 
 SDValue DAGCombiner::visitUMUL_LOHI(SDNode *N) {
-  SDValue Res = SimplifyNodeWithTwoResults(N, ISD::MUL, ISD::MULHU);
-  if (Res.getNode()) return Res;
+  if (SDValue Res = SimplifyNodeWithTwoResults(N, ISD::MUL, ISD::MULHU))
+    return Res;
 
   EVT VT = N->getValueType(0);
   SDLoc DL(N);
@@ -2493,7 +2578,8 @@ SDValue DAGCombiner::visitUMUL_LOHI(SDNode *N) {
       Lo = DAG.getNode(ISD::MUL, DL, NewVT, Lo, Hi);
       // Compute the high part as N1.
       Hi = DAG.getNode(ISD::SRL, DL, NewVT, Lo,
-            DAG.getConstant(SimpleSize, getShiftAmountTy(Lo.getValueType())));
+            DAG.getConstant(SimpleSize, DL,
+                            getShiftAmountTy(Lo.getValueType())));
       Hi = DAG.getNode(ISD::TRUNCATE, DL, VT, Hi);
       // Compute the low part as N0.
       Lo = DAG.getNode(ISD::TRUNCATE, DL, VT, Lo);
@@ -2525,15 +2611,15 @@ SDValue DAGCombiner::visitUMULO(SDNode *N) {
 }
 
 SDValue DAGCombiner::visitSDIVREM(SDNode *N) {
-  SDValue Res = SimplifyNodeWithTwoResults(N, ISD::SDIV, ISD::SREM);
-  if (Res.getNode()) return Res;
+  if (SDValue Res = SimplifyNodeWithTwoResults(N, ISD::SDIV, ISD::SREM))
+    return Res;
 
   return SDValue();
 }
 
 SDValue DAGCombiner::visitUDIVREM(SDNode *N) {
-  SDValue Res = SimplifyNodeWithTwoResults(N, ISD::UDIV, ISD::UREM);
-  if (Res.getNode()) return Res;
+  if (SDValue Res = SimplifyNodeWithTwoResults(N, ISD::UDIV, ISD::UREM))
+    return Res;
 
   return SDValue();
 }
@@ -2649,7 +2735,7 @@ SDValue DAGCombiner::SimplifyBinOpWithSameOpcodeHands(SDNode *N) {
       // build vector of all zeros that might be illegal at this stage.
       if (N->getOpcode() == ISD::XOR && ShOp.getOpcode() != ISD::UNDEF) {
         if (!LegalTypes)
-          ShOp = DAG.getConstant(0, VT);
+          ShOp = DAG.getConstant(0, SDLoc(N), VT);
         else
           ShOp = SDValue();
       }
@@ -2670,7 +2756,7 @@ SDValue DAGCombiner::SimplifyBinOpWithSameOpcodeHands(SDNode *N) {
       ShOp = N0->getOperand(0);
       if (N->getOpcode() == ISD::XOR && ShOp.getOpcode() != ISD::UNDEF) {
         if (!LegalTypes)
-          ShOp = DAG.getConstant(0, VT);
+          ShOp = DAG.getConstant(0, SDLoc(N), VT);
         else
           ShOp = SDValue();
       }
@@ -2701,7 +2787,7 @@ SDValue DAGCombiner::visitANDLike(SDValue N0, SDValue N1,
 
   // fold (and x, undef) -> 0
   if (N0.getOpcode() == ISD::UNDEF || N1.getOpcode() == ISD::UNDEF)
-    return DAG.getConstant(0, VT);
+    return DAG.getConstant(0, SDLoc(LocReference), VT);
   // fold (and (setcc x), (setcc y)) -> (setcc (and x, y))
   SDValue LL, LR, RL, RR, CC0, CC1;
   if (isSetCCEquivalent(N0, LL, LR, CC0) && isSetCCEquivalent(N1, RL, RR, CC1)){
@@ -2711,39 +2797,42 @@ SDValue DAGCombiner::visitANDLike(SDValue N0, SDValue N1,
     if (LR == RR && isa<ConstantSDNode>(LR) && Op0 == Op1 &&
         LL.getValueType().isInteger()) {
       // fold (and (seteq X, 0), (seteq Y, 0)) -> (seteq (or X, Y), 0)
-      if (cast<ConstantSDNode>(LR)->isNullValue() && Op1 == ISD::SETEQ) {
+      if (isNullConstant(LR) && Op1 == ISD::SETEQ) {
         SDValue ORNode = DAG.getNode(ISD::OR, SDLoc(N0),
                                      LR.getValueType(), LL, RL);
         AddToWorklist(ORNode.getNode());
         return DAG.getSetCC(SDLoc(LocReference), VT, ORNode, LR, Op1);
       }
-      // fold (and (seteq X, -1), (seteq Y, -1)) -> (seteq (and X, Y), -1)
-      if (cast<ConstantSDNode>(LR)->isAllOnesValue() && Op1 == ISD::SETEQ) {
-        SDValue ANDNode = DAG.getNode(ISD::AND, SDLoc(N0),
-                                      LR.getValueType(), LL, RL);
-        AddToWorklist(ANDNode.getNode());
-        return DAG.getSetCC(SDLoc(LocReference), VT, ANDNode, LR, Op1);
-      }
-      // fold (and (setgt X,  -1), (setgt Y,  -1)) -> (setgt (or X, Y), -1)
-      if (cast<ConstantSDNode>(LR)->isAllOnesValue() && Op1 == ISD::SETGT) {
-        SDValue ORNode = DAG.getNode(ISD::OR, SDLoc(N0),
-                                     LR.getValueType(), LL, RL);
-        AddToWorklist(ORNode.getNode());
-        return DAG.getSetCC(SDLoc(LocReference), VT, ORNode, LR, Op1);
+      if (isAllOnesConstant(LR)) {
+        // fold (and (seteq X, -1), (seteq Y, -1)) -> (seteq (and X, Y), -1)
+        if (Op1 == ISD::SETEQ) {
+          SDValue ANDNode = DAG.getNode(ISD::AND, SDLoc(N0),
+                                        LR.getValueType(), LL, RL);
+          AddToWorklist(ANDNode.getNode());
+          return DAG.getSetCC(SDLoc(LocReference), VT, ANDNode, LR, Op1);
+        }
+        // fold (and (setgt X, -1), (setgt Y, -1)) -> (setgt (or X, Y), -1)
+        if (Op1 == ISD::SETGT) {
+          SDValue ORNode = DAG.getNode(ISD::OR, SDLoc(N0),
+                                       LR.getValueType(), LL, RL);
+          AddToWorklist(ORNode.getNode());
+          return DAG.getSetCC(SDLoc(LocReference), VT, ORNode, LR, Op1);
+        }
       }
     }
     // Simplify (and (setne X, 0), (setne X, -1)) -> (setuge (add X, 1), 2)
     if (LL == RL && isa<ConstantSDNode>(LR) && isa<ConstantSDNode>(RR) &&
         Op0 == Op1 && LL.getValueType().isInteger() &&
-      Op0 == ISD::SETNE && ((cast<ConstantSDNode>(LR)->isNullValue() &&
-                                 cast<ConstantSDNode>(RR)->isAllOnesValue()) ||
-                                (cast<ConstantSDNode>(LR)->isAllOnesValue() &&
-                                 cast<ConstantSDNode>(RR)->isNullValue()))) {
-      SDValue ADDNode = DAG.getNode(ISD::ADD, SDLoc(N0), LL.getValueType(),
-                                    LL, DAG.getConstant(1, LL.getValueType()));
+      Op0 == ISD::SETNE && ((isNullConstant(LR) && isAllOnesConstant(RR)) ||
+                            (isAllOnesConstant(LR) && isNullConstant(RR)))) {
+      SDLoc DL(N0);
+      SDValue ADDNode = DAG.getNode(ISD::ADD, DL, LL.getValueType(),
+                                    LL, DAG.getConstant(1, DL,
+                                                        LL.getValueType()));
       AddToWorklist(ADDNode.getNode());
       return DAG.getSetCC(SDLoc(LocReference), VT, ADDNode,
-                          DAG.getConstant(2, LL.getValueType()), ISD::SETUGE);
+                          DAG.getConstant(2, DL, LL.getValueType()),
+                          ISD::SETUGE);
     }
     // canonicalize equivalent to ll == rl
     if (LL == RR && LR == RL) {
@@ -2778,9 +2867,10 @@ SDValue DAGCombiner::visitANDLike(SDValue N0, SDValue N1,
           if (DAG.MaskedValueIsZero(N0.getOperand(1), Mask)) {
             ADDC |= Mask;
             if (TLI.isLegalAddImmediate(ADDC.getSExtValue())) {
+              SDLoc DL(N0);
               SDValue NewAdd =
-                DAG.getNode(ISD::ADD, SDLoc(N0), VT,
-                            N0.getOperand(0), DAG.getConstant(ADDC, VT));
+                DAG.getNode(ISD::ADD, DL, VT,
+                            N0.getOperand(0), DAG.getConstant(ADDC, DL, VT));
               CombineTo(N0.getNode(), NewAdd);
               // Return N so it doesn't get rechecked!
               return SDValue(LocReference, 0);
@@ -2810,13 +2900,13 @@ SDValue DAGCombiner::visitAND(SDNode *N) {
       return DAG.getConstant(
           APInt::getNullValue(
               N0.getValueType().getScalarType().getSizeInBits()),
-          N0.getValueType());
+          SDLoc(N), N0.getValueType());
     if (ISD::isBuildVectorAllZeros(N1.getNode()))
       // do not return N1, because undef node may exist in N1
       return DAG.getConstant(
           APInt::getNullValue(
               N1.getValueType().getScalarType().getSizeInBits()),
-          N1.getValueType());
+          SDLoc(N), N1.getValueType());
 
     // fold (and x, -1) -> x, vector edition
     if (ISD::isBuildVectorAllOnes(N0.getNode()))
@@ -2826,22 +2916,22 @@ SDValue DAGCombiner::visitAND(SDNode *N) {
   }
 
   // fold (and c1, c2) -> c1&c2
-  ConstantSDNode *N0C = dyn_cast<ConstantSDNode>(N0);
+  ConstantSDNode *N0C = getAsNonOpaqueConstant(N0);
   ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(N1);
-  if (N0C && N1C)
-    return DAG.FoldConstantArithmetic(ISD::AND, VT, N0C, N1C);
+  if (N0C && N1C && !N1C->isOpaque())
+    return DAG.FoldConstantArithmetic(ISD::AND, SDLoc(N), VT, N0C, N1C);
   // canonicalize constant to RHS
   if (isConstantIntBuildVectorOrConstantInt(N0) &&
      !isConstantIntBuildVectorOrConstantInt(N1))
     return DAG.getNode(ISD::AND, SDLoc(N), VT, N1, N0);
   // fold (and x, -1) -> x
-  if (N1C && N1C->isAllOnesValue())
+  if (isAllOnesConstant(N1))
     return N0;
   // if (and x, c) is known to be zero, return 0
   unsigned BitWidth = VT.getScalarType().getSizeInBits();
   if (N1C && DAG.MaskedValueIsZero(SDValue(N, 0),
                                    APInt::getAllOnesValue(BitWidth)))
-    return DAG.getConstant(0, VT);
+    return DAG.getConstant(0, SDLoc(N), VT);
   // reassociate and
   if (SDValue RAND = ReassociateOps(ISD::AND, SDLoc(N), N0, N1))
     return RAND;
@@ -3019,12 +3109,13 @@ SDValue DAGCombiner::visitAND(SDNode *N) {
           // For big endian targets, we need to add an offset to the pointer
           // to load the correct bytes.  For little endian systems, we merely
           // need to read fewer bytes from the same pointer.
-          if (TLI.isBigEndian()) {
+          if (DAG.getDataLayout().isBigEndian()) {
             unsigned LVTStoreBytes = LoadedVT.getStoreSize();
             unsigned EVTStoreBytes = ExtVT.getStoreSize();
             unsigned PtrOff = LVTStoreBytes - EVTStoreBytes;
-            NewPtr = DAG.getNode(ISD::ADD, SDLoc(LN0), PtrType,
-                                 NewPtr, DAG.getConstant(PtrOff, PtrType));
+            SDLoc DL(LN0);
+            NewPtr = DAG.getNode(ISD::ADD, DL, PtrType,
+                                 NewPtr, DAG.getConstant(PtrOff, DL, PtrType));
             Alignment = MinAlign(Alignment, PtrOff);
           }
 
@@ -3048,10 +3139,9 @@ SDValue DAGCombiner::visitAND(SDNode *N) {
     return Combined;
 
   // Simplify: (and (op x...), (op y...))  -> (op (and x, y))
-  if (N0.getOpcode() == N1.getOpcode()) {
-    SDValue Tmp = SimplifyBinOpWithSameOpcodeHands(N);
-    if (Tmp.getNode()) return Tmp;
-  }
+  if (N0.getOpcode() == N1.getOpcode())
+    if (SDValue Tmp = SimplifyBinOpWithSameOpcodeHands(N))
+      return Tmp;
 
   // fold (and (sign_extend_inreg x, i16 to i32), 1) -> (and x, 1)
   // fold (and (sra)) -> (and (srl)) when possible.
@@ -3208,9 +3298,12 @@ SDValue DAGCombiner::MatchBSwapHWordLow(SDNode *N, SDValue N0, SDValue N1,
   }
 
   SDValue Res = DAG.getNode(ISD::BSWAP, SDLoc(N), VT, N00);
-  if (OpSizeInBits > 16)
-    Res = DAG.getNode(ISD::SRL, SDLoc(N), VT, Res,
-                      DAG.getConstant(OpSizeInBits-16, getShiftAmountTy(VT)));
+  if (OpSizeInBits > 16) {
+    SDLoc DL(N);
+    Res = DAG.getNode(ISD::SRL, DL, VT, Res,
+                      DAG.getConstant(OpSizeInBits - 16, DL,
+                                      getShiftAmountTy(VT)));
+  }
   return Res;
 }
 
@@ -3348,19 +3441,20 @@ SDValue DAGCombiner::MatchBSwapHWord(SDNode *N, SDValue N0, SDValue N1) {
   if (Parts[0] != Parts[1] || Parts[0] != Parts[2] || Parts[0] != Parts[3])
     return SDValue();
 
-  SDValue BSwap = DAG.getNode(ISD::BSWAP, SDLoc(N), VT,
-                              SDValue(Parts[0],0));
+  SDLoc DL(N);
+  SDValue BSwap = DAG.getNode(ISD::BSWAP, DL, VT,
+                              SDValue(Parts[0], 0));
 
   // Result of the bswap should be rotated by 16. If it's not legal, then
   // do  (x << 16) | (x >> 16).
-  SDValue ShAmt = DAG.getConstant(16, getShiftAmountTy(VT));
+  SDValue ShAmt = DAG.getConstant(16, DL, getShiftAmountTy(VT));
   if (TLI.isOperationLegalOrCustom(ISD::ROTL, VT))
-    return DAG.getNode(ISD::ROTL, SDLoc(N), VT, BSwap, ShAmt);
+    return DAG.getNode(ISD::ROTL, DL, VT, BSwap, ShAmt);
   if (TLI.isOperationLegalOrCustom(ISD::ROTR, VT))
-    return DAG.getNode(ISD::ROTR, SDLoc(N), VT, BSwap, ShAmt);
-  return DAG.getNode(ISD::OR, SDLoc(N), VT,
-                     DAG.getNode(ISD::SHL, SDLoc(N), VT, BSwap, ShAmt),
-                     DAG.getNode(ISD::SRL, SDLoc(N), VT, BSwap, ShAmt));
+    return DAG.getNode(ISD::ROTR, DL, VT, BSwap, ShAmt);
+  return DAG.getNode(ISD::OR, DL, VT,
+                     DAG.getNode(ISD::SHL, DL, VT, BSwap, ShAmt),
+                     DAG.getNode(ISD::SRL, DL, VT, BSwap, ShAmt));
 }
 
 /// This contains all DAGCombine rules which reduce two values combined by
@@ -3371,7 +3465,8 @@ SDValue DAGCombiner::visitORLike(SDValue N0, SDValue N1, SDNode *LocReference) {
   if (!LegalOperations &&
       (N0.getOpcode() == ISD::UNDEF || N1.getOpcode() == ISD::UNDEF)) {
     EVT EltVT = VT.isVector() ? VT.getVectorElementType() : VT;
-    return DAG.getConstant(APInt::getAllOnesValue(EltVT.getSizeInBits()), VT);
+    return DAG.getConstant(APInt::getAllOnesValue(EltVT.getSizeInBits()),
+                           SDLoc(LocReference), VT);
   }
   // fold (or (setcc x), (setcc y)) -> (setcc (or x, y))
   SDValue LL, LR, RL, RR, CC0, CC1;
@@ -3379,12 +3474,10 @@ SDValue DAGCombiner::visitORLike(SDValue N0, SDValue N1, SDNode *LocReference) {
     ISD::CondCode Op0 = cast<CondCodeSDNode>(CC0)->get();
     ISD::CondCode Op1 = cast<CondCodeSDNode>(CC1)->get();
 
-    if (LR == RR && isa<ConstantSDNode>(LR) && Op0 == Op1 &&
-        LL.getValueType().isInteger()) {
+    if (LR == RR && Op0 == Op1 && LL.getValueType().isInteger()) {
       // fold (or (setne X, 0), (setne Y, 0)) -> (setne (or X, Y), 0)
       // fold (or (setlt X, 0), (setlt Y, 0)) -> (setne (or X, Y), 0)
-      if (cast<ConstantSDNode>(LR)->isNullValue() &&
-          (Op1 == ISD::SETNE || Op1 == ISD::SETLT)) {
+      if (isNullConstant(LR) && (Op1 == ISD::SETNE || Op1 == ISD::SETLT)) {
         SDValue ORNode = DAG.getNode(ISD::OR, SDLoc(LR),
                                      LR.getValueType(), LL, RL);
         AddToWorklist(ORNode.getNode());
@@ -3392,8 +3485,7 @@ SDValue DAGCombiner::visitORLike(SDValue N0, SDValue N1, SDNode *LocReference) {
       }
       // fold (or (setne X, -1), (setne Y, -1)) -> (setne (and X, Y), -1)
       // fold (or (setgt X, -1), (setgt Y  -1)) -> (setgt (and X, Y), -1)
-      if (cast<ConstantSDNode>(LR)->isAllOnesValue() &&
-          (Op1 == ISD::SETNE || Op1 == ISD::SETGT)) {
+      if (isAllOnesConstant(LR) && (Op1 == ISD::SETNE || Op1 == ISD::SETGT)) {
         SDValue ANDNode = DAG.getNode(ISD::AND, SDLoc(LR),
                                       LR.getValueType(), LL, RL);
         AddToWorklist(ANDNode.getNode());
@@ -3419,25 +3511,29 @@ SDValue DAGCombiner::visitORLike(SDValue N0, SDValue N1, SDNode *LocReference) {
   }
 
   // (or (and X, C1), (and Y, C2))  -> (and (or X, Y), C3) if possible.
-  if (N0.getOpcode() == ISD::AND &&
-      N1.getOpcode() == ISD::AND &&
-      N0.getOperand(1).getOpcode() == ISD::Constant &&
-      N1.getOperand(1).getOpcode() == ISD::Constant &&
+  if (N0.getOpcode() == ISD::AND && N1.getOpcode() == ISD::AND &&
       // Don't increase # computations.
       (N0.getNode()->hasOneUse() || N1.getNode()->hasOneUse())) {
     // We can only do this xform if we know that bits from X that are set in C2
     // but not in C1 are already zero.  Likewise for Y.
-    const APInt &LHSMask =
-      cast<ConstantSDNode>(N0.getOperand(1))->getAPIntValue();
-    const APInt &RHSMask =
-      cast<ConstantSDNode>(N1.getOperand(1))->getAPIntValue();
-
-    if (DAG.MaskedValueIsZero(N0.getOperand(0), RHSMask&~LHSMask) &&
-        DAG.MaskedValueIsZero(N1.getOperand(0), LHSMask&~RHSMask)) {
-      SDValue X = DAG.getNode(ISD::OR, SDLoc(N0), VT,
-                              N0.getOperand(0), N1.getOperand(0));
-      return DAG.getNode(ISD::AND, SDLoc(LocReference), VT, X,
-                         DAG.getConstant(LHSMask | RHSMask, VT));
+    if (const ConstantSDNode *N0O1C =
+        getAsNonOpaqueConstant(N0.getOperand(1))) {
+      if (const ConstantSDNode *N1O1C =
+          getAsNonOpaqueConstant(N1.getOperand(1))) {
+        // We can only do this xform if we know that bits from X that are set in
+        // C2 but not in C1 are already zero.  Likewise for Y.
+        const APInt &LHSMask = N0O1C->getAPIntValue();
+        const APInt &RHSMask = N1O1C->getAPIntValue();
+
+        if (DAG.MaskedValueIsZero(N0.getOperand(0), RHSMask&~LHSMask) &&
+            DAG.MaskedValueIsZero(N1.getOperand(0), LHSMask&~RHSMask)) {
+          SDValue X = DAG.getNode(ISD::OR, SDLoc(N0), VT,
+                                  N0.getOperand(0), N1.getOperand(0));
+          SDLoc DL(LocReference);
+          return DAG.getNode(ISD::AND, DL, VT, X,
+                             DAG.getConstant(LHSMask | RHSMask, DL, VT));
+        }
+      }
     }
   }
 
@@ -3477,13 +3573,13 @@ SDValue DAGCombiner::visitOR(SDNode *N) {
       return DAG.getConstant(
           APInt::getAllOnesValue(
               N0.getValueType().getScalarType().getSizeInBits()),
-          N0.getValueType());
+          SDLoc(N), N0.getValueType());
     if (ISD::isBuildVectorAllOnes(N1.getNode()))
       // do not return N1, because undef node may exist in N1
       return DAG.getConstant(
           APInt::getAllOnesValue(
               N1.getValueType().getScalarType().getSizeInBits()),
-          N1.getValueType());
+          SDLoc(N), N1.getValueType());
 
     // fold (or (shuf A, V_0, MA), (shuf B, V_0, MB)) -> (shuf A, B, Mask1)
     // fold (or (shuf A, V_0, MA), (shuf B, V_0, MB)) -> (shuf B, A, Mask2)
@@ -3543,19 +3639,19 @@ SDValue DAGCombiner::visitOR(SDNode *N) {
   }
 
   // fold (or c1, c2) -> c1|c2
-  ConstantSDNode *N0C = dyn_cast<ConstantSDNode>(N0);
+  ConstantSDNode *N0C = getAsNonOpaqueConstant(N0);
   ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(N1);
-  if (N0C && N1C)
-    return DAG.FoldConstantArithmetic(ISD::OR, VT, N0C, N1C);
+  if (N0C && N1C && !N1C->isOpaque())
+    return DAG.FoldConstantArithmetic(ISD::OR, SDLoc(N), VT, N0C, N1C);
   // canonicalize constant to RHS
   if (isConstantIntBuildVectorOrConstantInt(N0) &&
      !isConstantIntBuildVectorOrConstantInt(N1))
     return DAG.getNode(ISD::OR, SDLoc(N), VT, N1, N0);
   // fold (or x, 0) -> x
-  if (N1C && N1C->isNullValue())
+  if (isNullConstant(N1))
     return N0;
   // fold (or x, -1) -> -1
-  if (N1C && N1C->isAllOnesValue())
+  if (isAllOnesConstant(N1))
     return N1;
   // fold (or x, c) -> c iff (x & ~c) == 0
   if (N1C && DAG.MaskedValueIsZero(N0, ~N1C->getAPIntValue()))
@@ -3565,11 +3661,9 @@ SDValue DAGCombiner::visitOR(SDNode *N) {
     return Combined;
 
   // Recognize halfword bswaps as (bswap + rotl 16) or (bswap + shl 16)
-  SDValue BSwap = MatchBSwapHWord(N, N0, N1);
-  if (BSwap.getNode())
+  if (SDValue BSwap = MatchBSwapHWord(N, N0, N1))
     return BSwap;
-  BSwap = MatchBSwapHWordLow(N, N0, N1);
-  if (BSwap.getNode())
+  if (SDValue BSwap = MatchBSwapHWordLow(N, N0, N1))
     return BSwap;
 
   // reassociate or
@@ -3581,7 +3675,8 @@ SDValue DAGCombiner::visitOR(SDNode *N) {
              isa<ConstantSDNode>(N0.getOperand(1))) {
     ConstantSDNode *C1 = cast<ConstantSDNode>(N0.getOperand(1));
     if ((C1->getAPIntValue() & N1C->getAPIntValue()) != 0) {
-      if (SDValue COR = DAG.FoldConstantArithmetic(ISD::OR, VT, N1C, C1))
+      if (SDValue COR = DAG.FoldConstantArithmetic(ISD::OR, SDLoc(N1), VT,
+                                                   N1C, C1))
         return DAG.getNode(
             ISD::AND, SDLoc(N), VT,
             DAG.getNode(ISD::OR, SDLoc(N0), VT, N0.getOperand(0), N1), COR);
@@ -3589,10 +3684,9 @@ SDValue DAGCombiner::visitOR(SDNode *N) {
     }
   }
   // Simplify: (or (op x...), (op y...))  -> (op (or x, y))
-  if (N0.getOpcode() == N1.getOpcode()) {
-    SDValue Tmp = SimplifyBinOpWithSameOpcodeHands(N);
-    if (Tmp.getNode()) return Tmp;
-  }
+  if (N0.getOpcode() == N1.getOpcode())
+    if (SDValue Tmp = SimplifyBinOpWithSameOpcodeHands(N))
+      return Tmp;
 
   // See if this is some rotate idiom.
   if (SDNode *Rot = MatchRotate(N0, N1, SDLoc(N)))
@@ -3821,7 +3915,7 @@ SDNode *DAGCombiner::MatchRotate(SDValue LHS, SDValue RHS, SDLoc DL) {
         Mask &= cast<ConstantSDNode>(RHSMask)->getAPIntValue() | LHSBits;
       }
 
-      Rot = DAG.getNode(ISD::AND, DL, VT, Rot, DAG.getConstant(Mask, VT));
+      Rot = DAG.getNode(ISD::AND, DL, VT, Rot, DAG.getConstant(Mask, DL, VT));
     }
 
     return Rot.getNode();
@@ -3879,23 +3973,23 @@ SDValue DAGCombiner::visitXOR(SDNode *N) {
 
   // fold (xor undef, undef) -> 0. This is a common idiom (misuse).
   if (N0.getOpcode() == ISD::UNDEF && N1.getOpcode() == ISD::UNDEF)
-    return DAG.getConstant(0, VT);
+    return DAG.getConstant(0, SDLoc(N), VT);
   // fold (xor x, undef) -> undef
   if (N0.getOpcode() == ISD::UNDEF)
     return N0;
   if (N1.getOpcode() == ISD::UNDEF)
     return N1;
   // fold (xor c1, c2) -> c1^c2
-  ConstantSDNode *N0C = dyn_cast<ConstantSDNode>(N0);
-  ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(N1);
+  ConstantSDNode *N0C = getAsNonOpaqueConstant(N0);
+  ConstantSDNode *N1C = getAsNonOpaqueConstant(N1);
   if (N0C && N1C)
-    return DAG.FoldConstantArithmetic(ISD::XOR, VT, N0C, N1C);
+    return DAG.FoldConstantArithmetic(ISD::XOR, SDLoc(N), VT, N0C, N1C);
   // canonicalize constant to RHS
   if (isConstantIntBuildVectorOrConstantInt(N0) &&
      !isConstantIntBuildVectorOrConstantInt(N1))
     return DAG.getNode(ISD::XOR, SDLoc(N), VT, N1, N0);
   // fold (xor x, 0) -> x
-  if (N1C && N1C->isNullValue())
+  if (isNullConstant(N1))
     return N0;
   // reassociate xor
   if (SDValue RXOR = ReassociateOps(ISD::XOR, SDLoc(N), N0, N1))
@@ -3923,18 +4017,19 @@ SDValue DAGCombiner::visitXOR(SDNode *N) {
   }
 
   // fold (not (zext (setcc x, y))) -> (zext (not (setcc x, y)))
-  if (N1C && N1C->getAPIntValue() == 1 && N0.getOpcode() == ISD::ZERO_EXTEND &&
+  if (isOneConstant(N1) && N0.getOpcode() == ISD::ZERO_EXTEND &&
       N0.getNode()->hasOneUse() &&
       isSetCCEquivalent(N0.getOperand(0), LHS, RHS, CC)){
     SDValue V = N0.getOperand(0);
-    V = DAG.getNode(ISD::XOR, SDLoc(N0), V.getValueType(), V,
-                    DAG.getConstant(1, V.getValueType()));
+    SDLoc DL(N0);
+    V = DAG.getNode(ISD::XOR, DL, V.getValueType(), V,
+                    DAG.getConstant(1, DL, V.getValueType()));
     AddToWorklist(V.getNode());
     return DAG.getNode(ISD::ZERO_EXTEND, SDLoc(N), VT, V);
   }
 
   // fold (not (or x, y)) -> (and (not x), (not y)) iff x or y are setcc
-  if (N1C && N1C->getAPIntValue() == 1 && VT == MVT::i1 &&
+  if (isOneConstant(N1) && VT == MVT::i1 &&
       (N0.getOpcode() == ISD::OR || N0.getOpcode() == ISD::AND)) {
     SDValue LHS = N0.getOperand(0), RHS = N0.getOperand(1);
     if (isOneUseSetCC(RHS) || isOneUseSetCC(LHS)) {
@@ -3946,7 +4041,7 @@ SDValue DAGCombiner::visitXOR(SDNode *N) {
     }
   }
   // fold (not (or x, y)) -> (and (not x), (not y)) iff x or y are constants
-  if (N1C && N1C->isAllOnesValue() &&
+  if (isAllOnesConstant(N1) &&
       (N0.getOpcode() == ISD::OR || N0.getOpcode() == ISD::AND)) {
     SDValue LHS = N0.getOperand(0), RHS = N0.getOperand(1);
     if (isa<ConstantSDNode>(RHS) || isa<ConstantSDNode>(LHS)) {
@@ -3967,16 +4062,18 @@ SDValue DAGCombiner::visitXOR(SDNode *N) {
   }
   // fold (xor (xor x, c1), c2) -> (xor x, (xor c1, c2))
   if (N1C && N0.getOpcode() == ISD::XOR) {
-    ConstantSDNode *N00C = dyn_cast<ConstantSDNode>(N0.getOperand(0));
-    ConstantSDNode *N01C = dyn_cast<ConstantSDNode>(N0.getOperand(1));
-    if (N00C)
-      return DAG.getNode(ISD::XOR, SDLoc(N), VT, N0.getOperand(1),
+    if (const ConstantSDNode *N00C = getAsNonOpaqueConstant(N0.getOperand(0))) {
+      SDLoc DL(N);
+      return DAG.getNode(ISD::XOR, DL, VT, N0.getOperand(1),
                          DAG.getConstant(N1C->getAPIntValue() ^
-                                         N00C->getAPIntValue(), VT));
-    if (N01C)
-      return DAG.getNode(ISD::XOR, SDLoc(N), VT, N0.getOperand(0),
+                                         N00C->getAPIntValue(), DL, VT));
+    }
+    if (const ConstantSDNode *N01C = getAsNonOpaqueConstant(N0.getOperand(1))) {
+      SDLoc DL(N);
+      return DAG.getNode(ISD::XOR, DL, VT, N0.getOperand(0),
                          DAG.getConstant(N1C->getAPIntValue() ^
-                                         N01C->getAPIntValue(), VT));
+                                         N01C->getAPIntValue(), DL, VT));
+    }
   }
   // fold (xor x, x) -> 0
   if (N0 == N1)
@@ -4000,19 +4097,17 @@ SDValue DAGCombiner::visitXOR(SDNode *N) {
   //   consistent result.
   // - Pushing the zero left requires shifting one bits in from the right.
   // A rotate left of ~1 is a nice way of achieving the desired result.
-  if (TLI.isOperationLegalOrCustom(ISD::ROTL, VT))
-    if (auto *N1C = dyn_cast<ConstantSDNode>(N1.getNode()))
-      if (N0.getOpcode() == ISD::SHL)
-        if (auto *ShlLHS = dyn_cast<ConstantSDNode>(N0.getOperand(0)))
-          if (N1C->isAllOnesValue() && ShlLHS->isOne())
-            return DAG.getNode(ISD::ROTL, SDLoc(N), VT, DAG.getConstant(~1, VT),
-                               N0.getOperand(1));
+  if (TLI.isOperationLegalOrCustom(ISD::ROTL, VT) && N0.getOpcode() == ISD::SHL
+      && isAllOnesConstant(N1) && isOneConstant(N0.getOperand(0))) {
+    SDLoc DL(N);
+    return DAG.getNode(ISD::ROTL, DL, VT, DAG.getConstant(~1, DL, VT),
+                       N0.getOperand(1));
+  }
 
   // Simplify: xor (op x...), (op y...)  -> (op (xor x, y))
-  if (N0.getOpcode() == N1.getOpcode()) {
-    SDValue Tmp = SimplifyBinOpWithSameOpcodeHands(N);
-    if (Tmp.getNode()) return Tmp;
-  }
+  if (N0.getOpcode() == N1.getOpcode())
+    if (SDValue Tmp = SimplifyBinOpWithSameOpcodeHands(N))
+      return Tmp;
 
   // Simplify the expression using non-local knowledge.
   if (!VT.isVector() &&
@@ -4025,10 +4120,6 @@ SDValue DAGCombiner::visitXOR(SDNode *N) {
 /// Handle transforms common to the three shifts, when the shift amount is a
 /// constant.
 SDValue DAGCombiner::visitShiftByConstant(SDNode *N, ConstantSDNode *Amt) {
-  // We can't and shouldn't fold opaque constants.
-  if (Amt->isOpaque())
-    return SDValue();
-
   SDNode *LHS = N->getOperand(0).getNode();
   if (!LHS->hasOneUse()) return SDValue();
 
@@ -4055,8 +4146,8 @@ SDValue DAGCombiner::visitShiftByConstant(SDNode *N, ConstantSDNode *Amt) {
   }
 
   // We require the RHS of the binop to be a constant and not opaque as well.
-  ConstantSDNode *BinOpCst = dyn_cast<ConstantSDNode>(LHS->getOperand(1));
-  if (!BinOpCst || BinOpCst->isOpaque()) return SDValue();
+  ConstantSDNode *BinOpCst = getAsNonOpaqueConstant(LHS->getOperand(1));
+  if (!BinOpCst) return SDValue();
 
   // FIXME: disable this unless the input to the binop is a shift by a constant.
   // If it is not a shift, it pessimizes some common cases like:
@@ -4109,14 +4200,17 @@ SDValue DAGCombiner::distributeTruncateThroughAnd(SDNode *N) {
     SDValue N01 = N->getOperand(0).getOperand(1);
 
     if (ConstantSDNode *N01C = isConstOrConstSplat(N01)) {
-      EVT TruncVT = N->getValueType(0);
-      SDValue N00 = N->getOperand(0).getOperand(0);
-      APInt TruncC = N01C->getAPIntValue();
-      TruncC = TruncC.trunc(TruncVT.getScalarSizeInBits());
+      if (!N01C->isOpaque()) {
+        EVT TruncVT = N->getValueType(0);
+        SDValue N00 = N->getOperand(0).getOperand(0);
+        APInt TruncC = N01C->getAPIntValue();
+        TruncC = TruncC.trunc(TruncVT.getScalarSizeInBits());
+        SDLoc DL(N);
 
-      return DAG.getNode(ISD::AND, SDLoc(N), TruncVT,
-                         DAG.getNode(ISD::TRUNCATE, SDLoc(N), TruncVT, N00),
-                         DAG.getConstant(TruncC, TruncVT));
+        return DAG.getNode(ISD::AND, DL, TruncVT,
+                           DAG.getNode(ISD::TRUNCATE, DL, TruncVT, N00),
+                           DAG.getConstant(TruncC, DL, TruncVT));
+      }
     }
   }
 
@@ -4159,7 +4253,8 @@ SDValue DAGCombiner::visitSHL(SDNode *N) {
         if (N01CV && N01CV->isConstant() && N00.getOpcode() == ISD::SETCC &&
             TLI.getBooleanContents(N00.getOperand(0).getValueType()) ==
                 TargetLowering::ZeroOrNegativeOneBooleanContent) {
-          if (SDValue C = DAG.FoldConstantArithmetic(ISD::SHL, VT, N01CV, N1CV))
+          if (SDValue C = DAG.FoldConstantArithmetic(ISD::SHL, SDLoc(N), VT,
+                                                     N01CV, N1CV))
             return DAG.getNode(ISD::AND, SDLoc(N), VT, N00, C);
         }
       } else {
@@ -4169,25 +4264,25 @@ SDValue DAGCombiner::visitSHL(SDNode *N) {
   }
 
   // fold (shl c1, c2) -> c1<<c2
-  ConstantSDNode *N0C = dyn_cast<ConstantSDNode>(N0);
-  if (N0C && N1C)
-    return DAG.FoldConstantArithmetic(ISD::SHL, VT, N0C, N1C);
+  ConstantSDNode *N0C = getAsNonOpaqueConstant(N0);
+  if (N0C && N1C && !N1C->isOpaque())
+    return DAG.FoldConstantArithmetic(ISD::SHL, SDLoc(N), VT, N0C, N1C);
   // fold (shl 0, x) -> 0
-  if (N0C && N0C->isNullValue())
+  if (isNullConstant(N0))
     return N0;
   // fold (shl x, c >= size(x)) -> undef
-  if (N1C && N1C->getZExtValue() >= OpSizeInBits)
+  if (N1C && N1C->getAPIntValue().uge(OpSizeInBits))
     return DAG.getUNDEF(VT);
   // fold (shl x, 0) -> x
   if (N1C && N1C->isNullValue())
     return N0;
   // fold (shl undef, x) -> 0
   if (N0.getOpcode() == ISD::UNDEF)
-    return DAG.getConstant(0, VT);
+    return DAG.getConstant(0, SDLoc(N), VT);
   // if (shl x, c) is known to be zero, return 0
   if (DAG.MaskedValueIsZero(SDValue(N, 0),
                             APInt::getAllOnesValue(OpSizeInBits)))
-    return DAG.getConstant(0, VT);
+    return DAG.getConstant(0, SDLoc(N), VT);
   // fold (shl x, (trunc (and y, c))) -> (shl x, (and (trunc y), (trunc c))).
   if (N1.getOpcode() == ISD::TRUNCATE &&
       N1.getOperand(0).getOpcode() == ISD::AND) {
@@ -4204,10 +4299,11 @@ SDValue DAGCombiner::visitSHL(SDNode *N) {
     if (ConstantSDNode *N0C1 = isConstOrConstSplat(N0.getOperand(1))) {
       uint64_t c1 = N0C1->getZExtValue();
       uint64_t c2 = N1C->getZExtValue();
+      SDLoc DL(N);
       if (c1 + c2 >= OpSizeInBits)
-        return DAG.getConstant(0, VT);
-      return DAG.getNode(ISD::SHL, SDLoc(N), VT, N0.getOperand(0),
-                         DAG.getConstant(c1 + c2, N1.getValueType()));
+        return DAG.getConstant(0, DL, VT);
+      return DAG.getNode(ISD::SHL, DL, VT, N0.getOperand(0),
+                         DAG.getConstant(c1 + c2, DL, N1.getValueType()));
     }
   }
 
@@ -4227,12 +4323,13 @@ SDValue DAGCombiner::visitSHL(SDNode *N) {
       EVT InnerShiftVT = N0Op0.getValueType();
       uint64_t InnerShiftSize = InnerShiftVT.getScalarSizeInBits();
       if (c2 >= OpSizeInBits - InnerShiftSize) {
+        SDLoc DL(N0);
         if (c1 + c2 >= OpSizeInBits)
-          return DAG.getConstant(0, VT);
-        return DAG.getNode(ISD::SHL, SDLoc(N0), VT,
-                           DAG.getNode(N0.getOpcode(), SDLoc(N0), VT,
+          return DAG.getConstant(0, DL, VT);
+        return DAG.getNode(ISD::SHL, DL, VT,
+                           DAG.getNode(N0.getOpcode(), DL, VT,
                                        N0Op0->getOperand(0)),
-                           DAG.getConstant(c1 + c2, N1.getValueType()));
+                           DAG.getConstant(c1 + c2, DL, N1.getValueType()));
       }
     }
   }
@@ -4250,8 +4347,10 @@ SDValue DAGCombiner::visitSHL(SDNode *N) {
         if (c1 == c2) {
           SDValue NewOp0 = N0.getOperand(0);
           EVT CountVT = NewOp0.getOperand(1).getValueType();
-          SDValue NewSHL = DAG.getNode(ISD::SHL, SDLoc(N), NewOp0.getValueType(),
-                                       NewOp0, DAG.getConstant(c2, CountVT));
+          SDLoc DL(N);
+          SDValue NewSHL = DAG.getNode(ISD::SHL, DL, NewOp0.getValueType(),
+                                       NewOp0,
+                                       DAG.getConstant(c2, DL, CountVT));
           AddToWorklist(NewSHL.getNode());
           return DAG.getNode(ISD::ZERO_EXTEND, SDLoc(N0), VT, NewSHL);
         }
@@ -4259,6 +4358,22 @@ SDValue DAGCombiner::visitSHL(SDNode *N) {
     }
   }
 
+  // fold (shl (sr[la] exact X,  C1), C2) -> (shl    X, (C2-C1)) if C1 <= C2
+  // fold (shl (sr[la] exact X,  C1), C2) -> (sr[la] X, (C2-C1)) if C1  > C2
+  if (N1C && (N0.getOpcode() == ISD::SRL || N0.getOpcode() == ISD::SRA) &&
+      cast<BinaryWithFlagsSDNode>(N0)->Flags.hasExact()) {
+    if (ConstantSDNode *N0C1 = isConstOrConstSplat(N0.getOperand(1))) {
+      uint64_t C1 = N0C1->getZExtValue();
+      uint64_t C2 = N1C->getZExtValue();
+      SDLoc DL(N);
+      if (C1 <= C2)
+        return DAG.getNode(ISD::SHL, DL, VT, N0.getOperand(0),
+                           DAG.getConstant(C2 - C1, DL, N1.getValueType()));
+      return DAG.getNode(N0.getOpcode(), DL, VT, N0.getOperand(0),
+                         DAG.getConstant(C1 - C2, DL, N1.getValueType()));
+    }
+  }
+
   // fold (shl (srl x, c1), c2) -> (and (shl x, (sub c2, c1), MASK) or
   //                               (and (srl x, (sub c1, c2), MASK)
   // Only fold this if the inner shift has no other uses -- if it does, folding
@@ -4272,25 +4387,30 @@ SDValue DAGCombiner::visitSHL(SDNode *N) {
         SDValue Shift;
         if (c2 > c1) {
           Mask = Mask.shl(c2 - c1);
-          Shift = DAG.getNode(ISD::SHL, SDLoc(N), VT, N0.getOperand(0),
-                              DAG.getConstant(c2 - c1, N1.getValueType()));
+          SDLoc DL(N);
+          Shift = DAG.getNode(ISD::SHL, DL, VT, N0.getOperand(0),
+                              DAG.getConstant(c2 - c1, DL, N1.getValueType()));
         } else {
           Mask = Mask.lshr(c1 - c2);
-          Shift = DAG.getNode(ISD::SRL, SDLoc(N), VT, N0.getOperand(0),
-                              DAG.getConstant(c1 - c2, N1.getValueType()));
+          SDLoc DL(N);
+          Shift = DAG.getNode(ISD::SRL, DL, VT, N0.getOperand(0),
+                              DAG.getConstant(c1 - c2, DL, N1.getValueType()));
         }
-        return DAG.getNode(ISD::AND, SDLoc(N0), VT, Shift,
-                           DAG.getConstant(Mask, VT));
+        SDLoc DL(N0);
+        return DAG.getNode(ISD::AND, DL, VT, Shift,
+                           DAG.getConstant(Mask, DL, VT));
       }
     }
   }
   // fold (shl (sra x, c1), c1) -> (and x, (shl -1, c1))
   if (N1C && N0.getOpcode() == ISD::SRA && N1 == N0.getOperand(1)) {
     unsigned BitSize = VT.getScalarSizeInBits();
+    SDLoc DL(N);
     SDValue HiBitsMask =
       DAG.getConstant(APInt::getHighBitsSet(BitSize,
-                                            BitSize - N1C->getZExtValue()), VT);
-    return DAG.getNode(ISD::AND, SDLoc(N), VT, N0.getOperand(0),
+                                            BitSize - N1C->getZExtValue()),
+                      DL, VT);
+    return DAG.getNode(ISD::AND, DL, VT, N0.getOperand(0),
                        HiBitsMask);
   }
 
@@ -4306,11 +4426,9 @@ SDValue DAGCombiner::visitSHL(SDNode *N) {
     return DAG.getNode(ISD::ADD, SDLoc(N), VT, Shl0, Shl1);
   }
 
-  if (N1C) {
-    SDValue NewSHL = visitShiftByConstant(N, N1C);
-    if (NewSHL.getNode())
+  if (N1C && !N1C->isOpaque())
+    if (SDValue NewSHL = visitShiftByConstant(N, N1C))
       return NewSHL;
-  }
 
   return SDValue();
 }
@@ -4331,14 +4449,14 @@ SDValue DAGCombiner::visitSRA(SDNode *N) {
   }
 
   // fold (sra c1, c2) -> (sra c1, c2)
-  ConstantSDNode *N0C = dyn_cast<ConstantSDNode>(N0);
-  if (N0C && N1C)
-    return DAG.FoldConstantArithmetic(ISD::SRA, VT, N0C, N1C);
+  ConstantSDNode *N0C = getAsNonOpaqueConstant(N0);
+  if (N0C && N1C && !N1C->isOpaque())
+    return DAG.FoldConstantArithmetic(ISD::SRA, SDLoc(N), VT, N0C, N1C);
   // fold (sra 0, x) -> 0
-  if (N0C && N0C->isNullValue())
+  if (isNullConstant(N0))
     return N0;
   // fold (sra -1, x) -> -1
-  if (N0C && N0C->isAllOnesValue())
+  if (isAllOnesConstant(N0))
     return N0;
   // fold (sra x, (setge c, size(x))) -> undef
   if (N1C && N1C->getZExtValue() >= OpSizeInBits)
@@ -4366,8 +4484,9 @@ SDValue DAGCombiner::visitSRA(SDNode *N) {
       unsigned Sum = N1C->getZExtValue() + C1->getZExtValue();
       if (Sum >= OpSizeInBits)
         Sum = OpSizeInBits - 1;
-      return DAG.getNode(ISD::SRA, SDLoc(N), VT, N0.getOperand(0),
-                         DAG.getConstant(Sum, N1.getValueType()));
+      SDLoc DL(N);
+      return DAG.getNode(ISD::SRA, DL, VT, N0.getOperand(0),
+                         DAG.getConstant(Sum, DL, N1.getValueType()));
     }
   }
 
@@ -4399,14 +4518,15 @@ SDValue DAGCombiner::visitSRA(SDNode *N) {
           TLI.isOperationLegalOrCustom(ISD::TRUNCATE, VT) &&
           TLI.isTruncateFree(VT, TruncVT)) {
 
-          SDValue Amt = DAG.getConstant(ShiftAmt,
-              getShiftAmountTy(N0.getOperand(0).getValueType()));
-          SDValue Shift = DAG.getNode(ISD::SRL, SDLoc(N0), VT,
-                                      N0.getOperand(0), Amt);
-          SDValue Trunc = DAG.getNode(ISD::TRUNCATE, SDLoc(N0), TruncVT,
-                                      Shift);
-          return DAG.getNode(ISD::SIGN_EXTEND, SDLoc(N),
-                             N->getValueType(0), Trunc);
+        SDLoc DL(N);
+        SDValue Amt = DAG.getConstant(ShiftAmt, DL,
+            getShiftAmountTy(N0.getOperand(0).getValueType()));
+        SDValue Shift = DAG.getNode(ISD::SRL, DL, VT,
+                                    N0.getOperand(0), Amt);
+        SDValue Trunc = DAG.getNode(ISD::TRUNCATE, DL, TruncVT,
+                                    Shift);
+        return DAG.getNode(ISD::SIGN_EXTEND, DL,
+                           N->getValueType(0), Trunc);
       }
     }
   }
@@ -4433,12 +4553,13 @@ SDValue DAGCombiner::visitSRA(SDNode *N) {
       EVT LargeVT = N0Op0.getValueType();
 
       if (LargeVT.getScalarSizeInBits() - OpSizeInBits == LargeShiftVal) {
+        SDLoc DL(N);
         SDValue Amt =
-          DAG.getConstant(LargeShiftVal + N1C->getZExtValue(),
+          DAG.getConstant(LargeShiftVal + N1C->getZExtValue(), DL,
                           getShiftAmountTy(N0Op0.getOperand(0).getValueType()));
-        SDValue SRA = DAG.getNode(ISD::SRA, SDLoc(N), LargeVT,
+        SDValue SRA = DAG.getNode(ISD::SRA, DL, LargeVT,
                                   N0Op0.getOperand(0), Amt);
-        return DAG.getNode(ISD::TRUNCATE, SDLoc(N), VT, SRA);
+        return DAG.getNode(ISD::TRUNCATE, DL, VT, SRA);
       }
     }
   }
@@ -4452,11 +4573,9 @@ SDValue DAGCombiner::visitSRA(SDNode *N) {
   if (DAG.SignBitIsZero(N0))
     return DAG.getNode(ISD::SRL, SDLoc(N), VT, N0, N1);
 
-  if (N1C) {
-    SDValue NewSRA = visitShiftByConstant(N, N1C);
-    if (NewSRA.getNode())
+  if (N1C && !N1C->isOpaque())
+    if (SDValue NewSRA = visitShiftByConstant(N, N1C))
       return NewSRA;
-  }
 
   return SDValue();
 }
@@ -4477,11 +4596,11 @@ SDValue DAGCombiner::visitSRL(SDNode *N) {
   }
 
   // fold (srl c1, c2) -> c1 >>u c2
-  ConstantSDNode *N0C = dyn_cast<ConstantSDNode>(N0);
-  if (N0C && N1C)
-    return DAG.FoldConstantArithmetic(ISD::SRL, VT, N0C, N1C);
+  ConstantSDNode *N0C = getAsNonOpaqueConstant(N0);
+  if (N0C && N1C && !N1C->isOpaque())
+    return DAG.FoldConstantArithmetic(ISD::SRL, SDLoc(N), VT, N0C, N1C);
   // fold (srl 0, x) -> 0
-  if (N0C && N0C->isNullValue())
+  if (isNullConstant(N0))
     return N0;
   // fold (srl x, c >= size(x)) -> undef
   if (N1C && N1C->getZExtValue() >= OpSizeInBits)
@@ -4492,17 +4611,18 @@ SDValue DAGCombiner::visitSRL(SDNode *N) {
   // if (srl x, c) is known to be zero, return 0
   if (N1C && DAG.MaskedValueIsZero(SDValue(N, 0),
                                    APInt::getAllOnesValue(OpSizeInBits)))
-    return DAG.getConstant(0, VT);
+    return DAG.getConstant(0, SDLoc(N), VT);
 
   // fold (srl (srl x, c1), c2) -> 0 or (srl x, (add c1, c2))
   if (N1C && N0.getOpcode() == ISD::SRL) {
     if (ConstantSDNode *N01C = isConstOrConstSplat(N0.getOperand(1))) {
       uint64_t c1 = N01C->getZExtValue();
       uint64_t c2 = N1C->getZExtValue();
+      SDLoc DL(N);
       if (c1 + c2 >= OpSizeInBits)
-        return DAG.getConstant(0, VT);
-      return DAG.getNode(ISD::SRL, SDLoc(N), VT, N0.getOperand(0),
-                         DAG.getConstant(c1 + c2, N1.getValueType()));
+        return DAG.getConstant(0, DL, VT);
+      return DAG.getNode(ISD::SRL, DL, VT, N0.getOperand(0),
+                         DAG.getConstant(c1 + c2, DL, N1.getValueType()));
     }
   }
 
@@ -4518,12 +4638,14 @@ SDValue DAGCombiner::visitSRL(SDNode *N) {
     uint64_t InnerShiftSize = InnerShiftVT.getScalarType().getSizeInBits();
     // This is only valid if the OpSizeInBits + c1 = size of inner shift.
     if (c1 + OpSizeInBits == InnerShiftSize) {
+      SDLoc DL(N0);
       if (c1 + c2 >= InnerShiftSize)
-        return DAG.getConstant(0, VT);
-      return DAG.getNode(ISD::TRUNCATE, SDLoc(N0), VT,
-                         DAG.getNode(ISD::SRL, SDLoc(N0), InnerShiftVT,
+        return DAG.getConstant(0, DL, VT);
+      return DAG.getNode(ISD::TRUNCATE, DL, VT,
+                         DAG.getNode(ISD::SRL, DL, InnerShiftVT,
                                      N0.getOperand(0)->getOperand(0),
-                                     DAG.getConstant(c1 + c2, ShiftCountVT)));
+                                     DAG.getConstant(c1 + c2, DL,
+                                                     ShiftCountVT)));
     }
   }
 
@@ -4532,8 +4654,9 @@ SDValue DAGCombiner::visitSRL(SDNode *N) {
     unsigned BitSize = N0.getScalarValueSizeInBits();
     if (BitSize <= 64) {
       uint64_t ShAmt = N1C->getZExtValue() + 64 - BitSize;
-      return DAG.getNode(ISD::AND, SDLoc(N), VT, N0.getOperand(0),
-                         DAG.getConstant(~0ULL >> ShAmt, VT));
+      SDLoc DL(N);
+      return DAG.getNode(ISD::AND, DL, VT, N0.getOperand(0),
+                         DAG.getConstant(~0ULL >> ShAmt, DL, VT));
     }
   }
 
@@ -4547,14 +4670,17 @@ SDValue DAGCombiner::visitSRL(SDNode *N) {
 
     if (!LegalTypes || TLI.isTypeDesirableForOp(ISD::SRL, SmallVT)) {
       uint64_t ShiftAmt = N1C->getZExtValue();
-      SDValue SmallShift = DAG.getNode(ISD::SRL, SDLoc(N0), SmallVT,
+      SDLoc DL0(N0);
+      SDValue SmallShift = DAG.getNode(ISD::SRL, DL0, SmallVT,
                                        N0.getOperand(0),
-                          DAG.getConstant(ShiftAmt, getShiftAmountTy(SmallVT)));
+                          DAG.getConstant(ShiftAmt, DL0,
+                                          getShiftAmountTy(SmallVT)));
       AddToWorklist(SmallShift.getNode());
       APInt Mask = APInt::getAllOnesValue(OpSizeInBits).lshr(ShiftAmt);
-      return DAG.getNode(ISD::AND, SDLoc(N), VT,
-                         DAG.getNode(ISD::ANY_EXTEND, SDLoc(N), VT, SmallShift),
-                         DAG.getConstant(Mask, VT));
+      SDLoc DL(N);
+      return DAG.getNode(ISD::AND, DL, VT,
+                         DAG.getNode(ISD::ANY_EXTEND, DL, VT, SmallShift),
+                         DAG.getConstant(Mask, DL, VT));
     }
   }
 
@@ -4573,12 +4699,12 @@ SDValue DAGCombiner::visitSRL(SDNode *N) {
 
     // If any of the input bits are KnownOne, then the input couldn't be all
     // zeros, thus the result of the srl will always be zero.
-    if (KnownOne.getBoolValue()) return DAG.getConstant(0, VT);
+    if (KnownOne.getBoolValue()) return DAG.getConstant(0, SDLoc(N0), VT);
 
     // If all of the bits input the to ctlz node are known to be zero, then
     // the result of the ctlz is "32" and the result of the shift is one.
     APInt UnknownBits = ~KnownZero;
-    if (UnknownBits == 0) return DAG.getConstant(1, VT);
+    if (UnknownBits == 0) return DAG.getConstant(1, SDLoc(N0), VT);
 
     // Otherwise, check to see if there is exactly one bit input to the ctlz.
     if ((UnknownBits & (UnknownBits - 1)) == 0) {
@@ -4590,13 +4716,16 @@ SDValue DAGCombiner::visitSRL(SDNode *N) {
       SDValue Op = N0.getOperand(0);
 
       if (ShAmt) {
-        Op = DAG.getNode(ISD::SRL, SDLoc(N0), VT, Op,
-                  DAG.getConstant(ShAmt, getShiftAmountTy(Op.getValueType())));
+        SDLoc DL(N0);
+        Op = DAG.getNode(ISD::SRL, DL, VT, Op,
+                  DAG.getConstant(ShAmt, DL,
+                                  getShiftAmountTy(Op.getValueType())));
         AddToWorklist(Op.getNode());
       }
 
-      return DAG.getNode(ISD::XOR, SDLoc(N), VT,
-                         Op, DAG.getConstant(1, VT));
+      SDLoc DL(N);
+      return DAG.getNode(ISD::XOR, DL, VT,
+                         Op, DAG.getConstant(1, DL, VT));
     }
   }
 
@@ -4613,7 +4742,7 @@ SDValue DAGCombiner::visitSRL(SDNode *N) {
   if (N1C && SimplifyDemandedBits(SDValue(N, 0)))
     return SDValue(N, 0);
 
-  if (N1C) {
+  if (N1C && !N1C->isOpaque()) {
     SDValue NewSRL = visitShiftByConstant(N, N1C);
     if (NewSRL.getNode())
       return NewSRL;
@@ -4656,12 +4785,25 @@ SDValue DAGCombiner::visitSRL(SDNode *N) {
   return SDValue();
 }
 
+SDValue DAGCombiner::visitBSWAP(SDNode *N) {
+  SDValue N0 = N->getOperand(0);
+  EVT VT = N->getValueType(0);
+
+  // fold (bswap c1) -> c2
+  if (isConstantIntBuildVectorOrConstantInt(N0))
+    return DAG.getNode(ISD::BSWAP, SDLoc(N), VT, N0);
+  // fold (bswap (bswap x)) -> x
+  if (N0.getOpcode() == ISD::BSWAP)
+    return N0->getOperand(0);
+  return SDValue();
+}
+
 SDValue DAGCombiner::visitCTLZ(SDNode *N) {
   SDValue N0 = N->getOperand(0);
   EVT VT = N->getValueType(0);
 
   // fold (ctlz c1) -> c2
-  if (isa<ConstantSDNode>(N0))
+  if (isConstantIntBuildVectorOrConstantInt(N0))
     return DAG.getNode(ISD::CTLZ, SDLoc(N), VT, N0);
   return SDValue();
 }
@@ -4671,7 +4813,7 @@ SDValue DAGCombiner::visitCTLZ_ZERO_UNDEF(SDNode *N) {
   EVT VT = N->getValueType(0);
 
   // fold (ctlz_zero_undef c1) -> c2
-  if (isa<ConstantSDNode>(N0))
+  if (isConstantIntBuildVectorOrConstantInt(N0))
     return DAG.getNode(ISD::CTLZ_ZERO_UNDEF, SDLoc(N), VT, N0);
   return SDValue();
 }
@@ -4681,7 +4823,7 @@ SDValue DAGCombiner::visitCTTZ(SDNode *N) {
   EVT VT = N->getValueType(0);
 
   // fold (cttz c1) -> c2
-  if (isa<ConstantSDNode>(N0))
+  if (isConstantIntBuildVectorOrConstantInt(N0))
     return DAG.getNode(ISD::CTTZ, SDLoc(N), VT, N0);
   return SDValue();
 }
@@ -4691,7 +4833,7 @@ SDValue DAGCombiner::visitCTTZ_ZERO_UNDEF(SDNode *N) {
   EVT VT = N->getValueType(0);
 
   // fold (cttz_zero_undef c1) -> c2
-  if (isa<ConstantSDNode>(N0))
+  if (isConstantIntBuildVectorOrConstantInt(N0))
     return DAG.getNode(ISD::CTTZ_ZERO_UNDEF, SDLoc(N), VT, N0);
   return SDValue();
 }
@@ -4701,7 +4843,7 @@ SDValue DAGCombiner::visitCTPOP(SDNode *N) {
   EVT VT = N->getValueType(0);
 
   // fold (ctpop c1) -> c2
-  if (isa<ConstantSDNode>(N0))
+  if (isConstantIntBuildVectorOrConstantInt(N0))
     return DAG.getNode(ISD::CTPOP, SDLoc(N), VT, N0);
   return SDValue();
 }
@@ -4753,16 +4895,13 @@ SDValue DAGCombiner::visitSELECT(SDNode *N) {
   // fold (select C, X, X) -> X
   if (N1 == N2)
     return N1;
-  // fold (select true, X, Y) -> X
-  ConstantSDNode *N0C = dyn_cast<ConstantSDNode>(N0);
-  if (N0C && !N0C->isNullValue())
-    return N1;
-  // fold (select false, X, Y) -> Y
-  if (N0C && N0C->isNullValue())
-    return N2;
+  if (const ConstantSDNode *N0C = dyn_cast<const ConstantSDNode>(N0)) {
+    // fold (select true, X, Y) -> X
+    // fold (select false, X, Y) -> Y
+    return !N0C->isNullValue() ? N1 : N2;
+  }
   // fold (select C, 1, X) -> (or C, X)
-  ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(N1);
-  if (VT == MVT::i1 && N1C && N1C->getAPIntValue() == 1)
+  if (VT == MVT::i1 && isOneConstant(N1))
     return DAG.getNode(ISD::OR, SDLoc(N), VT, N0, N2);
   // fold (select C, 0, 1) -> (xor C, 1)
   // We can't do this reliably if integer based booleans have different contents
@@ -4773,47 +4912,49 @@ SDValue DAGCombiner::visitSELECT(SDNode *N) {
   // undiscoverable (or not reasonably discoverable). For example, it could be
   // in another basic block or it could require searching a complicated
   // expression.
-  ConstantSDNode *N2C = dyn_cast<ConstantSDNode>(N2);
   if (VT.isInteger() &&
       (VT0 == MVT::i1 || (VT0.isInteger() &&
                           TLI.getBooleanContents(false, false) ==
                               TLI.getBooleanContents(false, true) &&
                           TLI.getBooleanContents(false, false) ==
                               TargetLowering::ZeroOrOneBooleanContent)) &&
-      N1C && N2C && N1C->isNullValue() && N2C->getAPIntValue() == 1) {
+      isNullConstant(N1) && isOneConstant(N2)) {
     SDValue XORNode;
-    if (VT == VT0)
-      return DAG.getNode(ISD::XOR, SDLoc(N), VT0,
-                         N0, DAG.getConstant(1, VT0));
-    XORNode = DAG.getNode(ISD::XOR, SDLoc(N0), VT0,
-                          N0, DAG.getConstant(1, VT0));
+    if (VT == VT0) {
+      SDLoc DL(N);
+      return DAG.getNode(ISD::XOR, DL, VT0,
+                         N0, DAG.getConstant(1, DL, VT0));
+    }
+    SDLoc DL0(N0);
+    XORNode = DAG.getNode(ISD::XOR, DL0, VT0,
+                          N0, DAG.getConstant(1, DL0, VT0));
     AddToWorklist(XORNode.getNode());
     if (VT.bitsGT(VT0))
       return DAG.getNode(ISD::ZERO_EXTEND, SDLoc(N), VT, XORNode);
     return DAG.getNode(ISD::TRUNCATE, SDLoc(N), VT, XORNode);
   }
   // fold (select C, 0, X) -> (and (not C), X)
-  if (VT == VT0 && VT == MVT::i1 && N1C && N1C->isNullValue()) {
+  if (VT == VT0 && VT == MVT::i1 && isNullConstant(N1)) {
     SDValue NOTNode = DAG.getNOT(SDLoc(N0), N0, VT);
     AddToWorklist(NOTNode.getNode());
     return DAG.getNode(ISD::AND, SDLoc(N), VT, NOTNode, N2);
   }
   // fold (select C, X, 1) -> (or (not C), X)
-  if (VT == VT0 && VT == MVT::i1 && N2C && N2C->getAPIntValue() == 1) {
+  if (VT == VT0 && VT == MVT::i1 && isOneConstant(N2)) {
     SDValue NOTNode = DAG.getNOT(SDLoc(N0), N0, VT);
     AddToWorklist(NOTNode.getNode());
     return DAG.getNode(ISD::OR, SDLoc(N), VT, NOTNode, N1);
   }
   // fold (select C, X, 0) -> (and C, X)
-  if (VT == MVT::i1 && N2C && N2C->isNullValue())
+  if (VT == MVT::i1 && isNullConstant(N2))
     return DAG.getNode(ISD::AND, SDLoc(N), VT, N0, N1);
   // fold (select X, X, Y) -> (or X, Y)
   // fold (select X, 1, Y) -> (or X, Y)
-  if (VT == MVT::i1 && (N0 == N1 || (N1C && N1C->getAPIntValue() == 1)))
+  if (VT == MVT::i1 && (N0 == N1 || isOneConstant(N1)))
     return DAG.getNode(ISD::OR, SDLoc(N), VT, N0, N2);
   // fold (select X, Y, X) -> (and X, Y)
   // fold (select X, Y, 0) -> (and X, Y)
-  if (VT == MVT::i1 && (N0 == N2 || (N2C && N2C->getAPIntValue() == 0)))
+  if (VT == MVT::i1 && (N0 == N2 || isNullConstant(N2)))
     return DAG.getNode(ISD::AND, SDLoc(N), VT, N0, N1);
 
   // If we can fold this based on the true/false value, do so.
@@ -4991,6 +5132,67 @@ static SDValue ConvertSelectToConcatVector(SDNode *N, SelectionDAG &DAG) {
       TopHalf->isNullValue() ? RHS->getOperand(1) : LHS->getOperand(1));
 }
 
+SDValue DAGCombiner::visitMSCATTER(SDNode *N) {
+
+  if (Level >= AfterLegalizeTypes)
+    return SDValue();
+
+  MaskedScatterSDNode *MSC = cast<MaskedScatterSDNode>(N);
+  SDValue Mask = MSC->getMask();
+  SDValue Data  = MSC->getValue();
+  SDLoc DL(N);
+
+  // If the MSCATTER data type requires splitting and the mask is provided by a
+  // SETCC, then split both nodes and its operands before legalization. This
+  // prevents the type legalizer from unrolling SETCC into scalar comparisons
+  // and enables future optimizations (e.g. min/max pattern matching on X86).
+  if (Mask.getOpcode() != ISD::SETCC)
+    return SDValue();
+
+  // Check if any splitting is required.
+  if (TLI.getTypeAction(*DAG.getContext(), Data.getValueType()) !=
+      TargetLowering::TypeSplitVector)
+    return SDValue();
+  SDValue MaskLo, MaskHi, Lo, Hi;
+  std::tie(MaskLo, MaskHi) = SplitVSETCC(Mask.getNode(), DAG);
+
+  EVT LoVT, HiVT;
+  std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(MSC->getValueType(0));
+
+  SDValue Chain = MSC->getChain();
+
+  EVT MemoryVT = MSC->getMemoryVT();
+  unsigned Alignment = MSC->getOriginalAlignment();
+
+  EVT LoMemVT, HiMemVT;
+  std::tie(LoMemVT, HiMemVT) = DAG.GetSplitDestVTs(MemoryVT);
+
+  SDValue DataLo, DataHi;
+  std::tie(DataLo, DataHi) = DAG.SplitVector(Data, DL);
+
+  SDValue BasePtr = MSC->getBasePtr();
+  SDValue IndexLo, IndexHi;
+  std::tie(IndexLo, IndexHi) = DAG.SplitVector(MSC->getIndex(), DL);
+
+  MachineMemOperand *MMO = DAG.getMachineFunction().
+    getMachineMemOperand(MSC->getPointerInfo(),
+                          MachineMemOperand::MOStore,  LoMemVT.getStoreSize(),
+                          Alignment, MSC->getAAInfo(), MSC->getRanges());
+
+  SDValue OpsLo[] = { Chain, DataLo, MaskLo, BasePtr, IndexLo };
+  Lo = DAG.getMaskedScatter(DAG.getVTList(MVT::Other), DataLo.getValueType(),
+                            DL, OpsLo, MMO);
+
+  SDValue OpsHi[] = {Chain, DataHi, MaskHi, BasePtr, IndexHi};
+  Hi = DAG.getMaskedScatter(DAG.getVTList(MVT::Other), DataHi.getValueType(),
+                            DL, OpsHi, MMO);
+
+  AddToWorklist(Lo.getNode());
+  AddToWorklist(Hi.getNode());
+
+  return DAG.getNode(ISD::TokenFactor, DL, MVT::Other, Lo, Hi);
+}
+
 SDValue DAGCombiner::visitMSTORE(SDNode *N) {
 
   if (Level >= AfterLegalizeTypes)
@@ -5046,7 +5248,7 @@ SDValue DAGCombiner::visitMSTORE(SDNode *N) {
 
     unsigned IncrementSize = LoMemVT.getSizeInBits()/8;
     Ptr = DAG.getNode(ISD::ADD, DL, Ptr.getValueType(), Ptr,
-                      DAG.getConstant(IncrementSize, Ptr.getValueType()));
+                      DAG.getConstant(IncrementSize, DL, Ptr.getValueType()));
 
     MMO = DAG.getMachineFunction().
       getMachineMemOperand(MST->getPointerInfo(),
@@ -5065,6 +5267,83 @@ SDValue DAGCombiner::visitMSTORE(SDNode *N) {
   return SDValue();
 }
 
+SDValue DAGCombiner::visitMGATHER(SDNode *N) {
+
+  if (Level >= AfterLegalizeTypes)
+    return SDValue();
+
+  MaskedGatherSDNode *MGT = dyn_cast<MaskedGatherSDNode>(N);
+  SDValue Mask = MGT->getMask();
+  SDLoc DL(N);
+
+  // If the MGATHER result requires splitting and the mask is provided by a
+  // SETCC, then split both nodes and its operands before legalization. This
+  // prevents the type legalizer from unrolling SETCC into scalar comparisons
+  // and enables future optimizations (e.g. min/max pattern matching on X86).
+
+  if (Mask.getOpcode() != ISD::SETCC)
+    return SDValue();
+
+  EVT VT = N->getValueType(0);
+
+  // Check if any splitting is required.
+  if (TLI.getTypeAction(*DAG.getContext(), VT) !=
+      TargetLowering::TypeSplitVector)
+    return SDValue();
+
+  SDValue MaskLo, MaskHi, Lo, Hi;
+  std::tie(MaskLo, MaskHi) = SplitVSETCC(Mask.getNode(), DAG);
+
+  SDValue Src0 = MGT->getValue();
+  SDValue Src0Lo, Src0Hi;
+  std::tie(Src0Lo, Src0Hi) = DAG.SplitVector(Src0, DL);
+
+  EVT LoVT, HiVT;
+  std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(VT);
+
+  SDValue Chain = MGT->getChain();
+  EVT MemoryVT = MGT->getMemoryVT();
+  unsigned Alignment = MGT->getOriginalAlignment();
+
+  EVT LoMemVT, HiMemVT;
+  std::tie(LoMemVT, HiMemVT) = DAG.GetSplitDestVTs(MemoryVT);
+
+  SDValue BasePtr = MGT->getBasePtr();
+  SDValue Index = MGT->getIndex();
+  SDValue IndexLo, IndexHi;
+  std::tie(IndexLo, IndexHi) = DAG.SplitVector(Index, DL);
+
+  MachineMemOperand *MMO = DAG.getMachineFunction().
+    getMachineMemOperand(MGT->getPointerInfo(),
+                          MachineMemOperand::MOLoad,  LoMemVT.getStoreSize(),
+                          Alignment, MGT->getAAInfo(), MGT->getRanges());
+
+  SDValue OpsLo[] = { Chain, Src0Lo, MaskLo, BasePtr, IndexLo };
+  Lo = DAG.getMaskedGather(DAG.getVTList(LoVT, MVT::Other), LoVT, DL, OpsLo,
+                            MMO);
+
+  SDValue OpsHi[] = {Chain, Src0Hi, MaskHi, BasePtr, IndexHi};
+  Hi = DAG.getMaskedGather(DAG.getVTList(HiVT, MVT::Other), HiVT, DL, OpsHi,
+                            MMO);
+
+  AddToWorklist(Lo.getNode());
+  AddToWorklist(Hi.getNode());
+
+  // Build a factor node to remember that this load is independent of the
+  // other one.
+  Chain = DAG.getNode(ISD::TokenFactor, DL, MVT::Other, Lo.getValue(1),
+                      Hi.getValue(1));
+
+  // Legalized the chain result - switch anything that used the old chain to
+  // use the new one.
+  DAG.ReplaceAllUsesOfValueWith(SDValue(MGT, 1), Chain);
+
+  SDValue GatherRes = DAG.getNode(ISD::CONCAT_VECTORS, DL, VT, Lo, Hi);
+
+  SDValue RetOps[] = { GatherRes, Chain };
+  return DAG.getMergeValues(RetOps, DL);
+}
+
 SDValue DAGCombiner::visitMLOAD(SDNode *N) {
 
   if (Level >= AfterLegalizeTypes)
@@ -5121,7 +5400,7 @@ SDValue DAGCombiner::visitMLOAD(SDNode *N) {
 
     unsigned IncrementSize = LoMemVT.getSizeInBits()/8;
     Ptr = DAG.getNode(ISD::ADD, DL, Ptr.getValueType(), Ptr,
-                      DAG.getConstant(IncrementSize, Ptr.getValueType()));
+                      DAG.getConstant(IncrementSize, DL, Ptr.getValueType()));
 
     MMO = DAG.getMachineFunction().
     getMachineMemOperand(MLD->getPointerInfo(),
@@ -5180,7 +5459,7 @@ SDValue DAGCombiner::visitVSELECT(SDNode *N) {
       EVT VT = LHS.getValueType();
       SDValue Shift = DAG.getNode(
           ISD::SRA, DL, VT, LHS,
-          DAG.getConstant(VT.getScalarType().getSizeInBits() - 1, VT));
+          DAG.getConstant(VT.getScalarType().getSizeInBits() - 1, DL, VT));
       SDValue Add = DAG.getNode(ISD::ADD, DL, VT, LHS, Shift);
       AddToWorklist(Shift.getNode());
       AddToWorklist(Add.getNode());
@@ -5188,6 +5467,9 @@ SDValue DAGCombiner::visitVSELECT(SDNode *N) {
     }
   }
 
+  if (SimplifySelectOps(N, N1, N2))
+    return SDValue(N, 0);  // Don't revisit N.
+
   // If the VSELECT result requires splitting and the mask is provided by a
   // SETCC, then split both nodes and its operands before legalization. This
   // prevents the type legalizer from unrolling SETCC into scalar comparisons
@@ -5286,12 +5568,12 @@ SDValue DAGCombiner::visitSETCC(SDNode *N) {
                        SDLoc(N));
 }
 
-// tryToFoldExtendOfConstant - Try to fold a sext/zext/aext
-// dag node into a ConstantSDNode or a build_vector of constants.
-// This function is called by the DAGCombiner when visiting sext/zext/aext
-// dag nodes (see for example method DAGCombiner::visitSIGN_EXTEND).
-// Vector extends are not folded if operations are legal; this is to
-// avoid introducing illegal build_vector dag nodes.
+/// Try to fold a sext/zext/aext dag node into a ConstantSDNode or 
+/// a build_vector of constants.
+/// This function is called by the DAGCombiner when visiting sext/zext/aext
+/// dag nodes (see for example method DAGCombiner::visitSIGN_EXTEND).
+/// Vector extends are not folded if operations are legal; this is to
+/// avoid introducing illegal build_vector dag nodes.
 static SDNode *tryToFoldExtendOfConstant(SDNode *N, const TargetLowering &TLI,
                                          SelectionDAG &DAG, bool LegalTypes,
                                          bool LegalOperations) {
@@ -5300,7 +5582,8 @@ static SDNode *tryToFoldExtendOfConstant(SDNode *N, const TargetLowering &TLI,
   EVT VT = N->getValueType(0);
 
   assert((Opcode == ISD::SIGN_EXTEND || Opcode == ISD::ZERO_EXTEND ||
-         Opcode == ISD::ANY_EXTEND) && "Expected EXTEND dag node in input!");
+         Opcode == ISD::ANY_EXTEND || Opcode == ISD::SIGN_EXTEND_VECTOR_INREG)
+         && "Expected EXTEND dag node in input!");
 
   // fold (sext c1) -> c1
   // fold (zext c1) -> c1
@@ -5320,9 +5603,8 @@ static SDNode *tryToFoldExtendOfConstant(SDNode *N, const TargetLowering &TLI,
   // We can fold this node into a build_vector.
   unsigned VTBits = SVT.getSizeInBits();
   unsigned EVTBits = N0->getValueType(0).getScalarType().getSizeInBits();
-  unsigned ShAmt = VTBits - EVTBits;
   SmallVector<SDValue, 8> Elts;
-  unsigned NumElts = N0->getNumOperands();
+  unsigned NumElts = VT.getVectorNumElements();
   SDLoc DL(N);
 
   for (unsigned i=0; i != NumElts; ++i) {
@@ -5332,14 +5614,14 @@ static SDNode *tryToFoldExtendOfConstant(SDNode *N, const TargetLowering &TLI,
       continue;
     }
 
-    ConstantSDNode *CurrentND = cast<ConstantSDNode>(Op);
-    const APInt &C = APInt(VTBits, CurrentND->getAPIntValue().getZExtValue());
-    if (Opcode == ISD::SIGN_EXTEND)
-      Elts.push_back(DAG.getConstant(C.shl(ShAmt).ashr(ShAmt).getZExtValue(),
-                                     SVT));
+    SDLoc DL(Op);
+    // Get the constant value and if needed trunc it to the size of the type.
+    // Nodes like build_vector might have constants wider than the scalar type.
+    APInt C = cast<ConstantSDNode>(Op)->getAPIntValue().zextOrTrunc(EVTBits);
+    if (Opcode == ISD::SIGN_EXTEND || Opcode == ISD::SIGN_EXTEND_VECTOR_INREG)
+      Elts.push_back(DAG.getConstant(C.sext(VTBits), DL, SVT));
     else
-      Elts.push_back(DAG.getConstant(C.shl(ShAmt).lshr(ShAmt).getZExtValue(),
-                                     SVT));
+      Elts.push_back(DAG.getConstant(C.zext(VTBits), DL, SVT));
   }
 
   return DAG.getNode(ISD::BUILD_VECTOR, DL, VT, Elts).getNode();
@@ -5505,7 +5787,7 @@ SDValue DAGCombiner::CombineExtLoad(SDNode *N) {
         Align, LN0->getAAInfo());
 
     BasePtr = DAG.getNode(ISD::ADD, DL, BasePtr.getValueType(), BasePtr,
-                          DAG.getConstant(Stride, BasePtr.getValueType()));
+                          DAG.getConstant(Stride, DL, BasePtr.getValueType()));
 
     Loads.push_back(SplitLoad.getValue(0));
     Chains.push_back(SplitLoad.getValue(1));
@@ -5543,8 +5825,7 @@ SDValue DAGCombiner::visitSIGN_EXTEND(SDNode *N) {
   if (N0.getOpcode() == ISD::TRUNCATE) {
     // fold (sext (truncate (load x))) -> (sext (smaller load x))
     // fold (sext (truncate (srl (load x), c))) -> (sext (smaller load (x+c/n)))
-    SDValue NarrowLoad = ReduceLoadWidth(N0.getNode());
-    if (NarrowLoad.getNode()) {
+    if (SDValue NarrowLoad = ReduceLoadWidth(N0.getNode())) {
       SDNode* oye = N0.getNode()->getOperand(0).getNode();
       if (NarrowLoad.getNode() != N0.getNode()) {
         CombineTo(N0.getNode(), NarrowLoad);
@@ -5668,14 +5949,15 @@ SDValue DAGCombiner::visitSIGN_EXTEND(SDNode *N) {
                                          LN0->getMemOperand());
         APInt Mask = cast<ConstantSDNode>(N0.getOperand(1))->getAPIntValue();
         Mask = Mask.sext(VT.getSizeInBits());
-        SDValue And = DAG.getNode(N0.getOpcode(), SDLoc(N), VT,
-                                  ExtLoad, DAG.getConstant(Mask, VT));
+        SDLoc DL(N);
+        SDValue And = DAG.getNode(N0.getOpcode(), DL, VT,
+                                  ExtLoad, DAG.getConstant(Mask, DL, VT));
         SDValue Trunc = DAG.getNode(ISD::TRUNCATE,
                                     SDLoc(N0.getOperand(0)),
                                     N0.getOperand(0).getValueType(), ExtLoad);
         CombineTo(N, And);
         CombineTo(N0.getOperand(0).getNode(), Trunc, ExtLoad.getValue(1));
-        ExtendSetCCUses(SetCCs, Trunc, ExtLoad, SDLoc(N),
+        ExtendSetCCUses(SetCCs, Trunc, ExtLoad, DL,
                         ISD::SIGN_EXTEND);
         return SDValue(N, 0);   // Return N so it doesn't get rechecked!
       }
@@ -5718,11 +6000,12 @@ SDValue DAGCombiner::visitSIGN_EXTEND(SDNode *N) {
 
     // sext(setcc x, y, cc) -> (select (setcc x, y, cc), -1, 0)
     unsigned ElementWidth = VT.getScalarType().getSizeInBits();
+    SDLoc DL(N);
     SDValue NegOne =
-      DAG.getConstant(APInt::getAllOnesValue(ElementWidth), VT);
+      DAG.getConstant(APInt::getAllOnesValue(ElementWidth), DL, VT);
     SDValue SCC =
-      SimplifySelectCC(SDLoc(N), N0.getOperand(0), N0.getOperand(1),
-                       NegOne, DAG.getConstant(0, VT),
+      SimplifySelectCC(DL, N0.getOperand(0), N0.getOperand(1),
+                       NegOne, DAG.getConstant(0, DL, VT),
                        cast<CondCodeSDNode>(N0.getOperand(2))->get(), true);
     if (SCC.getNode()) return SCC;
 
@@ -5734,7 +6017,7 @@ SDValue DAGCombiner::visitSIGN_EXTEND(SDNode *N) {
         SDValue SetCC = DAG.getSetCC(DL, SetCCVT,
                                      N0.getOperand(0), N0.getOperand(1), CC);
         return DAG.getSelect(DL, VT, SetCC,
-                             NegOne, DAG.getConstant(0, VT));
+                             NegOne, DAG.getConstant(0, DL, VT));
       }
     }
   }
@@ -5768,11 +6051,9 @@ static bool isTruncateOf(SelectionDAG &DAG, SDValue N, SDValue &Op,
   SDValue Op1 = N->getOperand(1);
   assert(Op0.getValueType() == Op1.getValueType());
 
-  ConstantSDNode *COp0 = dyn_cast<ConstantSDNode>(Op0);
-  ConstantSDNode *COp1 = dyn_cast<ConstantSDNode>(Op1);
-  if (COp0 && COp0->isNullValue())
+  if (isNullConstant(Op0))
     Op = Op1;
-  else if (COp1 && COp1->isNullValue())
+  else if (isNullConstant(Op1))
     Op = Op0;
   else
     return false;
@@ -5826,8 +6107,7 @@ SDValue DAGCombiner::visitZERO_EXTEND(SDNode *N) {
   // fold (zext (truncate (load x))) -> (zext (smaller load x))
   // fold (zext (truncate (srl (load x), c))) -> (zext (small load (x+c/n)))
   if (N0.getOpcode() == ISD::TRUNCATE) {
-    SDValue NarrowLoad = ReduceLoadWidth(N0.getNode());
-    if (NarrowLoad.getNode()) {
+    if (SDValue NarrowLoad = ReduceLoadWidth(N0.getNode())) {
       SDNode* oye = N0.getNode()->getOperand(0).getNode();
       if (NarrowLoad.getNode() != N0.getNode()) {
         CombineTo(N0.getNode(), NarrowLoad);
@@ -5844,8 +6124,7 @@ SDValue DAGCombiner::visitZERO_EXTEND(SDNode *N) {
 
     // fold (zext (truncate (load x))) -> (zext (smaller load x))
     // fold (zext (truncate (srl (load x), c))) -> (zext (smaller load (x+c/n)))
-    SDValue NarrowLoad = ReduceLoadWidth(N0.getNode());
-    if (NarrowLoad.getNode()) {
+    if (SDValue NarrowLoad = ReduceLoadWidth(N0.getNode())) {
       SDNode* oye = N0.getNode()->getOperand(0).getNode();
       if (NarrowLoad.getNode() != N0.getNode()) {
         CombineTo(N0.getNode(), NarrowLoad);
@@ -5883,8 +6162,9 @@ SDValue DAGCombiner::visitZERO_EXTEND(SDNode *N) {
     }
     APInt Mask = cast<ConstantSDNode>(N0.getOperand(1))->getAPIntValue();
     Mask = Mask.zext(VT.getSizeInBits());
-    return DAG.getNode(ISD::AND, SDLoc(N), VT,
-                       X, DAG.getConstant(Mask, VT));
+    SDLoc DL(N);
+    return DAG.getNode(ISD::AND, DL, VT,
+                       X, DAG.getConstant(Mask, DL, VT));
   }
 
   // fold (zext (load x)) -> (zext (truncate (zextload x)))
@@ -5944,14 +6224,15 @@ SDValue DAGCombiner::visitZERO_EXTEND(SDNode *N) {
                                          LN0->getMemOperand());
         APInt Mask = cast<ConstantSDNode>(N0.getOperand(1))->getAPIntValue();
         Mask = Mask.zext(VT.getSizeInBits());
-        SDValue And = DAG.getNode(N0.getOpcode(), SDLoc(N), VT,
-                                  ExtLoad, DAG.getConstant(Mask, VT));
+        SDLoc DL(N);
+        SDValue And = DAG.getNode(N0.getOpcode(), DL, VT,
+                                  ExtLoad, DAG.getConstant(Mask, DL, VT));
         SDValue Trunc = DAG.getNode(ISD::TRUNCATE,
                                     SDLoc(N0.getOperand(0)),
                                     N0.getOperand(0).getValueType(), ExtLoad);
         CombineTo(N, And);
         CombineTo(N0.getOperand(0).getNode(), Trunc, ExtLoad.getValue(1));
-        ExtendSetCCUses(SetCCs, Trunc, ExtLoad, SDLoc(N),
+        ExtendSetCCUses(SetCCs, Trunc, ExtLoad, DL,
                         ISD::ZERO_EXTEND);
         return SDValue(N, 0);   // Return N so it doesn't get rechecked!
       }
@@ -5989,19 +6270,20 @@ SDValue DAGCombiner::visitZERO_EXTEND(SDNode *N) {
       // zext(setcc) -> (and (vsetcc), (1, 1, ...) for vectors.
       // Only do this before legalize for now.
       EVT EltVT = VT.getVectorElementType();
+      SDLoc DL(N);
       SmallVector<SDValue,8> OneOps(VT.getVectorNumElements(),
-                                    DAG.getConstant(1, EltVT));
+                                    DAG.getConstant(1, DL, EltVT));
       if (VT.getSizeInBits() == N0VT.getSizeInBits())
         // We know that the # elements of the results is the same as the
         // # elements of the compare (and the # elements of the compare result
         // for that matter).  Check to see that they are the same size.  If so,
         // we know that the element size of the sext'd result matches the
         // element size of the compare operands.
-        return DAG.getNode(ISD::AND, SDLoc(N), VT,
-                           DAG.getSetCC(SDLoc(N), VT, N0.getOperand(0),
+        return DAG.getNode(ISD::AND, DL, VT,
+                           DAG.getSetCC(DL, VT, N0.getOperand(0),
                                          N0.getOperand(1),
                                  cast<CondCodeSDNode>(N0.getOperand(2))->get()),
-                           DAG.getNode(ISD::BUILD_VECTOR, SDLoc(N), VT,
+                           DAG.getNode(ISD::BUILD_VECTOR, DL, VT,
                                        OneOps));
 
       // If the desired elements are smaller or larger than the source
@@ -6014,18 +6296,19 @@ SDValue DAGCombiner::visitZERO_EXTEND(SDNode *N) {
         EVT::getVectorVT(*DAG.getContext(), MatchingElementType,
                          N0VT.getVectorNumElements());
       SDValue VsetCC =
-        DAG.getSetCC(SDLoc(N), MatchingVectorType, N0.getOperand(0),
+        DAG.getSetCC(DL, MatchingVectorType, N0.getOperand(0),
                       N0.getOperand(1),
                       cast<CondCodeSDNode>(N0.getOperand(2))->get());
-      return DAG.getNode(ISD::AND, SDLoc(N), VT,
-                         DAG.getSExtOrTrunc(VsetCC, SDLoc(N), VT),
-                         DAG.getNode(ISD::BUILD_VECTOR, SDLoc(N), VT, OneOps));
+      return DAG.getNode(ISD::AND, DL, VT,
+                         DAG.getSExtOrTrunc(VsetCC, DL, VT),
+                         DAG.getNode(ISD::BUILD_VECTOR, DL, VT, OneOps));
     }
 
     // zext(setcc x,y,cc) -> select_cc x, y, 1, 0, cc
+    SDLoc DL(N);
     SDValue SCC =
-      SimplifySelectCC(SDLoc(N), N0.getOperand(0), N0.getOperand(1),
-                       DAG.getConstant(1, VT), DAG.getConstant(0, VT),
+      SimplifySelectCC(DL, N0.getOperand(0), N0.getOperand(1),
+                       DAG.getConstant(1, DL, VT), DAG.getConstant(0, DL, VT),
                        cast<CondCodeSDNode>(N0.getOperand(2))->get(), true);
     if (SCC.getNode()) return SCC;
   }
@@ -6117,8 +6400,9 @@ SDValue DAGCombiner::visitANY_EXTEND(SDNode *N) {
     }
     APInt Mask = cast<ConstantSDNode>(N0.getOperand(1))->getAPIntValue();
     Mask = Mask.zext(VT.getSizeInBits());
-    return DAG.getNode(ISD::AND, SDLoc(N), VT,
-                       X, DAG.getConstant(Mask, VT));
+    SDLoc DL(N);
+    return DAG.getNode(ISD::AND, DL, VT,
+                       X, DAG.getConstant(Mask, DL, VT));
   }
 
   // fold (aext (load x)) -> (aext (truncate (extload x)))
@@ -6201,9 +6485,10 @@ SDValue DAGCombiner::visitANY_EXTEND(SDNode *N) {
     }
 
     // aext(setcc x,y,cc) -> select_cc x, y, 1, 0, cc
+    SDLoc DL(N);
     SDValue SCC =
-      SimplifySelectCC(SDLoc(N), N0.getOperand(0), N0.getOperand(1),
-                       DAG.getConstant(1, VT), DAG.getConstant(0, VT),
+      SimplifySelectCC(DL, N0.getOperand(0), N0.getOperand(1),
+                       DAG.getConstant(1, DL, VT), DAG.getConstant(0, DL, VT),
                        cast<CondCodeSDNode>(N0.getOperand(2))->get(), true);
     if (SCC.getNode())
       return SCC;
@@ -6224,7 +6509,7 @@ SDValue DAGCombiner::GetDemandedBits(SDValue V, const APInt &Mask) {
     const APInt &CVal = CV->getAPIntValue();
     APInt NewVal = CVal & Mask;
     if (NewVal != CVal)
-      return DAG.getConstant(NewVal, V.getValueType());
+      return DAG.getConstant(NewVal, SDLoc(V), V.getValueType());
     break;
   }
   case ISD::OR:
@@ -6239,15 +6524,14 @@ SDValue DAGCombiner::GetDemandedBits(SDValue V, const APInt &Mask) {
     // Only look at single-use SRLs.
     if (!V.getNode()->hasOneUse())
       break;
-    if (ConstantSDNode *RHSC = dyn_cast<ConstantSDNode>(V.getOperand(1))) {
+    if (ConstantSDNode *RHSC = getAsNonOpaqueConstant(V.getOperand(1))) {
       // See if we can recursively simplify the LHS.
       unsigned Amt = RHSC->getZExtValue();
 
       // Watch out for shift count overflow though.
       if (Amt >= Mask.getBitWidth()) break;
       APInt NewMask = Mask << Amt;
-      SDValue SimplifyLHS = GetDemandedBits(V.getOperand(0), NewMask);
-      if (SimplifyLHS.getNode())
+      if (SDValue SimplifyLHS = GetDemandedBits(V.getOperand(0), NewMask))
         return DAG.getNode(ISD::SRL, SDLoc(V), V.getValueType(),
                            SimplifyLHS, V.getOperand(1));
     }
@@ -6376,7 +6660,7 @@ SDValue DAGCombiner::ReduceLoadWidth(SDNode *N) {
 
   // For big endian targets, we need to adjust the offset to the pointer to
   // load the correct bytes.
-  if (TLI.isBigEndian()) {
+  if (DAG.getDataLayout().isBigEndian()) {
     unsigned LVTStoreBits = LN0->getMemoryVT().getStoreSizeInBits();
     unsigned EVTStoreBits = ExtVT.getStoreSizeInBits();
     ShAmt = LVTStoreBits - EVTStoreBits - ShAmt;
@@ -6384,9 +6668,10 @@ SDValue DAGCombiner::ReduceLoadWidth(SDNode *N) {
 
   uint64_t PtrOff = ShAmt / 8;
   unsigned NewAlign = MinAlign(LN0->getAlignment(), PtrOff);
-  SDValue NewPtr = DAG.getNode(ISD::ADD, SDLoc(LN0),
+  SDLoc DL(LN0);
+  SDValue NewPtr = DAG.getNode(ISD::ADD, DL,
                                PtrType, LN0->getBasePtr(),
-                               DAG.getConstant(PtrOff, PtrType));
+                               DAG.getConstant(PtrOff, DL, PtrType));
   AddToWorklist(NewPtr.getNode());
 
   SDValue Load;
@@ -6415,11 +6700,12 @@ SDValue DAGCombiner::ReduceLoadWidth(SDNode *N) {
     // no larger than the source) then the useful bits of the result are
     // zero; we can't simply return the shortened shift, because the result
     // of that operation is undefined.
+    SDLoc DL(N0);
     if (ShLeftAmt >= VT.getSizeInBits())
-      Result = DAG.getConstant(0, VT);
+      Result = DAG.getConstant(0, DL, VT);
     else
-      Result = DAG.getNode(ISD::SHL, SDLoc(N0), VT,
-                          Result, DAG.getConstant(ShLeftAmt, ShImmTy));
+      Result = DAG.getNode(ISD::SHL, DL, VT,
+                          Result, DAG.getConstant(ShLeftAmt, DL, ShImmTy));
   }
 
   // Return the new loaded value.
@@ -6469,8 +6755,7 @@ SDValue DAGCombiner::visitSIGN_EXTEND_INREG(SDNode *N) {
 
   // fold (sext_in_reg (load x)) -> (smaller sextload x)
   // fold (sext_in_reg (srl (load x), c)) -> (smaller sextload (x+c/evtbits))
-  SDValue NarrowLoad = ReduceLoadWidth(N);
-  if (NarrowLoad.getNode())
+  if (SDValue NarrowLoad = ReduceLoadWidth(N))
     return NarrowLoad;
 
   // fold (sext_in_reg (srl X, 24), i8) -> (sra X, 24)
@@ -6546,7 +6831,7 @@ SDValue DAGCombiner::visitSIGN_EXTEND_INREG(SDNode *N) {
       ConstantSDNode *CurrentND = cast<ConstantSDNode>(Op);
       const APInt &C = APInt(VTBits, CurrentND->getAPIntValue().getZExtValue());
       Elts.push_back(DAG.getConstant(C.shl(ShAmt).ashr(ShAmt).getZExtValue(),
-                                     Op.getValueType()));
+                                     SDLoc(Op), Op.getValueType()));
     }
 
     return DAG.getNode(ISD::BUILD_VECTOR, SDLoc(N), VT, Elts);
@@ -6555,10 +6840,24 @@ SDValue DAGCombiner::visitSIGN_EXTEND_INREG(SDNode *N) {
   return SDValue();
 }
 
+SDValue DAGCombiner::visitSIGN_EXTEND_VECTOR_INREG(SDNode *N) {
+  SDValue N0 = N->getOperand(0);
+  EVT VT = N->getValueType(0);
+
+  if (N0.getOpcode() == ISD::UNDEF)
+    return DAG.getUNDEF(VT);
+
+  if (SDNode *Res = tryToFoldExtendOfConstant(N, TLI, DAG, LegalTypes,
+                                              LegalOperations))
+    return SDValue(Res, 0);
+
+  return SDValue();
+}
+
 SDValue DAGCombiner::visitTRUNCATE(SDNode *N) {
   SDValue N0 = N->getOperand(0);
   EVT VT = N->getValueType(0);
-  bool isLE = TLI.isLittleEndian();
+  bool isLE = DAG.getDataLayout().isLittleEndian();
 
   // noop truncate
   if (N0.getValueType() == N->getValueType(0))
@@ -6611,15 +6910,16 @@ SDValue DAGCombiner::visitTRUNCATE(SDNode *N) {
     SDValue EltNo = N0->getOperand(1);
     if (isa<ConstantSDNode>(EltNo) && isTypeLegal(NVT)) {
       int Elt = cast<ConstantSDNode>(EltNo)->getZExtValue();
-      EVT IndexTy = TLI.getVectorIdxTy();
+      EVT IndexTy = TLI.getVectorIdxTy(DAG.getDataLayout());
       int Index = isLE ? (Elt*SizeRatio) : (Elt*SizeRatio + (SizeRatio-1));
 
       SDValue V = DAG.getNode(ISD::BITCAST, SDLoc(N),
                               NVT, N0.getOperand(0));
 
+      SDLoc DL(N);
       return DAG.getNode(ISD::EXTRACT_VECTOR_ELT,
-                         SDLoc(N), TrTy, V,
-                         DAG.getConstant(Index, IndexTy));
+                         DL, TrTy, V,
+                         DAG.getConstant(Index, DL, IndexTy));
     }
   }
 
@@ -6682,9 +6982,9 @@ SDValue DAGCombiner::visitTRUNCATE(SDNode *N) {
   // fold (truncate (load x)) -> (smaller load x)
   // fold (truncate (srl (load x), c)) -> (smaller load (x+c/evtbits))
   if (!LegalTypes || TLI.isTypeDesirableForOp(N0.getOpcode(), VT)) {
-    SDValue Reduced = ReduceLoadWidth(N);
-    if (Reduced.getNode())
+    if (SDValue Reduced = ReduceLoadWidth(N))
       return Reduced;
+
     // Handle the case where the load remains an extending load even
     // after truncation.
     if (N0.hasOneUse() && ISD::isUNINDEXEDLoad(N0.getNode())) {
@@ -6777,8 +7077,8 @@ SDValue DAGCombiner::CombineConsecutiveLoads(SDNode *N, EVT VT) {
       !LD2->isVolatile() &&
       DAG.isConsecutiveLoad(LD2, LD1, LD1VT.getSizeInBits()/8, 1)) {
     unsigned Align = LD1->getAlignment();
-    unsigned NewAlign = TLI.getDataLayout()->
-      getABITypeAlignment(VT.getTypeForEVT(*DAG.getContext()));
+    unsigned NewAlign = DAG.getDataLayout().getABITypeAlignment(
+        VT.getTypeForEVT(*DAG.getContext()));
 
     if (NewAlign <= Align &&
         (!LegalOperations || TLI.isOperationLegal(ISD::LOAD, VT)))
@@ -6834,13 +7134,13 @@ SDValue DAGCombiner::visitBITCAST(SDNode *N) {
       // Do not change the width of a volatile load.
       !cast<LoadSDNode>(N0)->isVolatile() &&
       // Do not remove the cast if the types differ in endian layout.
-      TLI.hasBigEndianPartOrdering(N0.getValueType()) ==
-      TLI.hasBigEndianPartOrdering(VT) &&
+      TLI.hasBigEndianPartOrdering(N0.getValueType(), DAG.getDataLayout()) ==
+          TLI.hasBigEndianPartOrdering(VT, DAG.getDataLayout()) &&
       (!LegalOperations || TLI.isOperationLegal(ISD::LOAD, VT)) &&
       TLI.isLoadBitCastBeneficial(N0.getValueType(), VT)) {
     LoadSDNode *LN0 = cast<LoadSDNode>(N0);
-    unsigned Align = TLI.getDataLayout()->
-      getABITypeAlignment(VT.getTypeForEVT(*DAG.getContext()));
+    unsigned Align = DAG.getDataLayout().getABITypeAlignment(
+        VT.getTypeForEVT(*DAG.getContext()));
     unsigned OrigAlign = LN0->getAlignment();
 
     if (Align <= OrigAlign) {
@@ -6865,13 +7165,14 @@ SDValue DAGCombiner::visitBITCAST(SDNode *N) {
                                   N0.getOperand(0));
     AddToWorklist(NewConv.getNode());
 
+    SDLoc DL(N);
     APInt SignBit = APInt::getSignBit(VT.getSizeInBits());
     if (N0.getOpcode() == ISD::FNEG)
-      return DAG.getNode(ISD::XOR, SDLoc(N), VT,
-                         NewConv, DAG.getConstant(SignBit, VT));
+      return DAG.getNode(ISD::XOR, DL, VT,
+                         NewConv, DAG.getConstant(SignBit, DL, VT));
     assert(N0.getOpcode() == ISD::FABS);
-    return DAG.getNode(ISD::AND, SDLoc(N), VT,
-                       NewConv, DAG.getConstant(~SignBit, VT));
+    return DAG.getNode(ISD::AND, DL, VT,
+                       NewConv, DAG.getConstant(~SignBit, DL, VT));
   }
 
   // fold (bitconvert (fcopysign cst, x)) ->
@@ -6896,9 +7197,11 @@ SDValue DAGCombiner::visitBITCAST(SDNode *N) {
       } else if (OrigXWidth > VTWidth) {
         // To get the sign bit in the right place, we have to shift it right
         // before truncating.
-        X = DAG.getNode(ISD::SRL, SDLoc(X),
+        SDLoc DL(X);
+        X = DAG.getNode(ISD::SRL, DL,
                         X.getValueType(), X,
-                        DAG.getConstant(OrigXWidth-VTWidth, X.getValueType()));
+                        DAG.getConstant(OrigXWidth-VTWidth, DL,
+                                        X.getValueType()));
         AddToWorklist(X.getNode());
         X = DAG.getNode(ISD::TRUNCATE, SDLoc(X), VT, X);
         AddToWorklist(X.getNode());
@@ -6906,13 +7209,13 @@ SDValue DAGCombiner::visitBITCAST(SDNode *N) {
 
       APInt SignBit = APInt::getSignBit(VT.getSizeInBits());
       X = DAG.getNode(ISD::AND, SDLoc(X), VT,
-                      X, DAG.getConstant(SignBit, VT));
+                      X, DAG.getConstant(SignBit, SDLoc(X), VT));
       AddToWorklist(X.getNode());
 
       SDValue Cst = DAG.getNode(ISD::BITCAST, SDLoc(N0),
                                 VT, N0.getOperand(0));
       Cst = DAG.getNode(ISD::AND, SDLoc(Cst), VT,
-                        Cst, DAG.getConstant(~SignBit, VT));
+                        Cst, DAG.getConstant(~SignBit, SDLoc(Cst), VT));
       AddToWorklist(Cst.getNode());
 
       return DAG.getNode(ISD::OR, SDLoc(N), VT, X, Cst);
@@ -6920,13 +7223,56 @@ SDValue DAGCombiner::visitBITCAST(SDNode *N) {
   }
 
   // bitconvert(build_pair(ld, ld)) -> ld iff load locations are consecutive.
-  if (N0.getOpcode() == ISD::BUILD_PAIR) {
-    SDValue CombineLD = CombineConsecutiveLoads(N0.getNode(), VT);
-    if (CombineLD.getNode())
+  if (N0.getOpcode() == ISD::BUILD_PAIR)
+    if (SDValue CombineLD = CombineConsecutiveLoads(N0.getNode(), VT))
       return CombineLD;
-  }
 
-  return SDValue();
+  // Remove double bitcasts from shuffles - this is often a legacy of
+  // XformToShuffleWithZero being used to combine bitmaskings (of
+  // float vectors bitcast to integer vectors) into shuffles.
+  // bitcast(shuffle(bitcast(s0),bitcast(s1))) -> shuffle(s0,s1)
+  if (Level < AfterLegalizeDAG && TLI.isTypeLegal(VT) && VT.isVector() &&
+      N0->getOpcode() == ISD::VECTOR_SHUFFLE &&
+      VT.getVectorNumElements() >= N0.getValueType().getVectorNumElements() &&
+      !(VT.getVectorNumElements() % N0.getValueType().getVectorNumElements())) {
+    ShuffleVectorSDNode *SVN = cast<ShuffleVectorSDNode>(N0);
+
+    // If operands are a bitcast, peek through if it casts the original VT.
+    // If operands are a constant, just bitcast back to original VT.
+    auto PeekThroughBitcast = [&](SDValue Op) {
+      if (Op.getOpcode() == ISD::BITCAST &&
+          Op.getOperand(0).getValueType() == VT)
+        return SDValue(Op.getOperand(0));
+      if (ISD::isBuildVectorOfConstantSDNodes(Op.getNode()) ||
+          ISD::isBuildVectorOfConstantFPSDNodes(Op.getNode()))
+        return DAG.getNode(ISD::BITCAST, SDLoc(N), VT, Op);
+      return SDValue();
+    };
+
+    SDValue SV0 = PeekThroughBitcast(N0->getOperand(0));
+    SDValue SV1 = PeekThroughBitcast(N0->getOperand(1));
+    if (!(SV0 && SV1))
+      return SDValue();
+
+    int MaskScale =
+        VT.getVectorNumElements() / N0.getValueType().getVectorNumElements();
+    SmallVector<int, 8> NewMask;
+    for (int M : SVN->getMask())
+      for (int i = 0; i != MaskScale; ++i)
+        NewMask.push_back(M < 0 ? -1 : M * MaskScale + i);
+
+    bool LegalMask = TLI.isShuffleMaskLegal(NewMask, VT);
+    if (!LegalMask) {
+      std::swap(SV0, SV1);
+      ShuffleVectorSDNode::commuteMask(NewMask);
+      LegalMask = TLI.isShuffleMaskLegal(NewMask, VT);
+    }
+
+    if (LegalMask)
+      return DAG.getVectorShuffle(VT, SDLoc(N), SV0, SV1, NewMask);
+  }
+
+  return SDValue();
 }
 
 SDValue DAGCombiner::visitBUILD_PAIR(SDNode *N) {
@@ -6960,8 +7306,7 @@ ConstantFoldBITCASTofBUILD_VECTOR(SDNode *BV, EVT DstEltVT) {
                                      DstEltVT, BV->getOperand(0)));
 
     SmallVector<SDValue, 8> Ops;
-    for (unsigned i = 0, e = BV->getNumOperands(); i != e; ++i) {
-      SDValue Op = BV->getOperand(i);
+    for (SDValue Op : BV->op_values()) {
       // If the vector element type is not legal, the BUILD_VECTOR operands
       // are promoted and implicitly truncated.  Make that explicit here.
       if (Op.getValueType() != SrcEltVT)
@@ -6994,6 +7339,8 @@ ConstantFoldBITCASTofBUILD_VECTOR(SDNode *BV, EVT DstEltVT) {
     return ConstantFoldBITCASTofBUILD_VECTOR(Tmp, DstEltVT);
   }
 
+  SDLoc DL(BV);
+
   // Okay, we know the src/dst types are both integers of differing types.
   // Handling growing first.
   assert(SrcEltVT.isInteger() && DstEltVT.isInteger());
@@ -7003,7 +7350,7 @@ ConstantFoldBITCASTofBUILD_VECTOR(SDNode *BV, EVT DstEltVT) {
     SmallVector<SDValue, 8> Ops;
     for (unsigned i = 0, e = BV->getNumOperands(); i != e;
          i += NumInputsPerOutput) {
-      bool isLE = TLI.isLittleEndian();
+      bool isLE = DAG.getDataLayout().isLittleEndian();
       APInt NewBits = APInt(DstBitSize, 0);
       bool EltIsUndef = true;
       for (unsigned j = 0; j != NumInputsPerOutput; ++j) {
@@ -7020,11 +7367,11 @@ ConstantFoldBITCASTofBUILD_VECTOR(SDNode *BV, EVT DstEltVT) {
       if (EltIsUndef)
         Ops.push_back(DAG.getUNDEF(DstEltVT));
       else
-        Ops.push_back(DAG.getConstant(NewBits, DstEltVT));
+        Ops.push_back(DAG.getConstant(NewBits, DL, DstEltVT));
     }
 
     EVT VT = EVT::getVectorVT(*DAG.getContext(), DstEltVT, Ops.size());
-    return DAG.getNode(ISD::BUILD_VECTOR, SDLoc(BV), VT, Ops);
+    return DAG.getNode(ISD::BUILD_VECTOR, DL, VT, Ops);
   }
 
   // Finally, this must be the case where we are shrinking elements: each input
@@ -7034,43 +7381,63 @@ ConstantFoldBITCASTofBUILD_VECTOR(SDNode *BV, EVT DstEltVT) {
                             NumOutputsPerInput*BV->getNumOperands());
   SmallVector<SDValue, 8> Ops;
 
-  for (unsigned i = 0, e = BV->getNumOperands(); i != e; ++i) {
-    if (BV->getOperand(i).getOpcode() == ISD::UNDEF) {
+  for (const SDValue &Op : BV->op_values()) {
+    if (Op.getOpcode() == ISD::UNDEF) {
       Ops.append(NumOutputsPerInput, DAG.getUNDEF(DstEltVT));
       continue;
     }
 
-    APInt OpVal = cast<ConstantSDNode>(BV->getOperand(i))->
+    APInt OpVal = cast<ConstantSDNode>(Op)->
                   getAPIntValue().zextOrTrunc(SrcBitSize);
 
     for (unsigned j = 0; j != NumOutputsPerInput; ++j) {
       APInt ThisVal = OpVal.trunc(DstBitSize);
-      Ops.push_back(DAG.getConstant(ThisVal, DstEltVT));
+      Ops.push_back(DAG.getConstant(ThisVal, DL, DstEltVT));
       OpVal = OpVal.lshr(DstBitSize);
     }
 
     // For big endian targets, swap the order of the pieces of each element.
-    if (TLI.isBigEndian())
+    if (DAG.getDataLayout().isBigEndian())
       std::reverse(Ops.end()-NumOutputsPerInput, Ops.end());
   }
 
-  return DAG.getNode(ISD::BUILD_VECTOR, SDLoc(BV), VT, Ops);
+  return DAG.getNode(ISD::BUILD_VECTOR, DL, VT, Ops);
 }
 
-// Attempt different variants of (fadd (fmul a, b), c) -> fma or fmad
-static SDValue performFaddFmulCombines(unsigned FusedOpcode,
-                                       bool Aggressive,
-                                       SDNode *N,
-                                       const TargetLowering &TLI,
-                                       SelectionDAG &DAG) {
+/// Try to perform FMA combining on a given FADD node.
+SDValue DAGCombiner::visitFADDForFMACombine(SDNode *N) {
   SDValue N0 = N->getOperand(0);
   SDValue N1 = N->getOperand(1);
   EVT VT = N->getValueType(0);
+  SDLoc SL(N);
+
+  const TargetOptions &Options = DAG.getTarget().Options;
+  bool UnsafeFPMath = (Options.AllowFPOpFusion == FPOpFusion::Fast ||
+                       Options.UnsafeFPMath);
+
+  // Floating-point multiply-add with intermediate rounding.
+  bool HasFMAD = (LegalOperations &&
+                  TLI.isOperationLegal(ISD::FMAD, VT));
+
+  // Floating-point multiply-add without intermediate rounding.
+  bool HasFMA = ((!LegalOperations ||
+                  TLI.isOperationLegalOrCustom(ISD::FMA, VT)) &&
+                 TLI.isFMAFasterThanFMulAndFAdd(VT) &&
+                 UnsafeFPMath);
+
+  // No valid opcode, do not combine.
+  if (!HasFMAD && !HasFMA)
+    return SDValue();
+
+  // Always prefer FMAD to FMA for precision.
+  unsigned int PreferredFusedOpcode = HasFMAD ? ISD::FMAD : ISD::FMA;
+  bool Aggressive = TLI.enableAggressiveFMAFusion(VT);
+  bool LookThroughFPExt = TLI.isFPExtFree(VT);
 
   // fold (fadd (fmul x, y), z) -> (fma x, y, z)
   if (N0.getOpcode() == ISD::FMUL &&
       (Aggressive || N0->hasOneUse())) {
-    return DAG.getNode(FusedOpcode, SDLoc(N), VT,
+    return DAG.getNode(PreferredFusedOpcode, SL, VT,
                        N0.getOperand(0), N0.getOperand(1), N1);
   }
 
@@ -7078,53 +7445,176 @@ static SDValue performFaddFmulCombines(unsigned FusedOpcode,
   // Note: Commutes FADD operands.
   if (N1.getOpcode() == ISD::FMUL &&
       (Aggressive || N1->hasOneUse())) {
-    return DAG.getNode(FusedOpcode, SDLoc(N), VT,
+    return DAG.getNode(PreferredFusedOpcode, SL, VT,
                        N1.getOperand(0), N1.getOperand(1), N0);
   }
 
+  // Look through FP_EXTEND nodes to do more combining.
+  if (UnsafeFPMath && LookThroughFPExt) {
+    // fold (fadd (fpext (fmul x, y)), z) -> (fma (fpext x), (fpext y), z)
+    if (N0.getOpcode() == ISD::FP_EXTEND) {
+      SDValue N00 = N0.getOperand(0);
+      if (N00.getOpcode() == ISD::FMUL)
+        return DAG.getNode(PreferredFusedOpcode, SL, VT,
+                           DAG.getNode(ISD::FP_EXTEND, SL, VT,
+                                       N00.getOperand(0)),
+                           DAG.getNode(ISD::FP_EXTEND, SL, VT,
+                                       N00.getOperand(1)), N1);
+    }
+
+    // fold (fadd x, (fpext (fmul y, z))) -> (fma (fpext y), (fpext z), x)
+    // Note: Commutes FADD operands.
+    if (N1.getOpcode() == ISD::FP_EXTEND) {
+      SDValue N10 = N1.getOperand(0);
+      if (N10.getOpcode() == ISD::FMUL)
+        return DAG.getNode(PreferredFusedOpcode, SL, VT,
+                           DAG.getNode(ISD::FP_EXTEND, SL, VT,
+                                       N10.getOperand(0)),
+                           DAG.getNode(ISD::FP_EXTEND, SL, VT,
+                                       N10.getOperand(1)), N0);
+    }
+  }
+
   // More folding opportunities when target permits.
-  if (Aggressive) {
+  if ((UnsafeFPMath || HasFMAD)  && Aggressive) {
     // fold (fadd (fma x, y, (fmul u, v)), z) -> (fma x, y (fma u, v, z))
-    if (N0.getOpcode() == ISD::FMA &&
+    if (N0.getOpcode() == PreferredFusedOpcode &&
         N0.getOperand(2).getOpcode() == ISD::FMUL) {
-      return DAG.getNode(FusedOpcode, SDLoc(N), VT,
+      return DAG.getNode(PreferredFusedOpcode, SL, VT,
                          N0.getOperand(0), N0.getOperand(1),
-                         DAG.getNode(FusedOpcode, SDLoc(N), VT,
+                         DAG.getNode(PreferredFusedOpcode, SL, VT,
                                      N0.getOperand(2).getOperand(0),
                                      N0.getOperand(2).getOperand(1),
                                      N1));
     }
 
     // fold (fadd x, (fma y, z, (fmul u, v)) -> (fma y, z (fma u, v, x))
-    if (N1->getOpcode() == ISD::FMA &&
+    if (N1->getOpcode() == PreferredFusedOpcode &&
         N1.getOperand(2).getOpcode() == ISD::FMUL) {
-      return DAG.getNode(FusedOpcode, SDLoc(N), VT,
+      return DAG.getNode(PreferredFusedOpcode, SL, VT,
                          N1.getOperand(0), N1.getOperand(1),
-                         DAG.getNode(FusedOpcode, SDLoc(N), VT,
+                         DAG.getNode(PreferredFusedOpcode, SL, VT,
                                      N1.getOperand(2).getOperand(0),
                                      N1.getOperand(2).getOperand(1),
                                      N0));
     }
+
+    if (UnsafeFPMath && LookThroughFPExt) {
+      // fold (fadd (fma x, y, (fpext (fmul u, v))), z)
+      //   -> (fma x, y, (fma (fpext u), (fpext v), z))
+      auto FoldFAddFMAFPExtFMul = [&] (
+          SDValue X, SDValue Y, SDValue U, SDValue V, SDValue Z) {
+        return DAG.getNode(PreferredFusedOpcode, SL, VT, X, Y,
+                           DAG.getNode(PreferredFusedOpcode, SL, VT,
+                                       DAG.getNode(ISD::FP_EXTEND, SL, VT, U),
+                                       DAG.getNode(ISD::FP_EXTEND, SL, VT, V),
+                                       Z));
+      };
+      if (N0.getOpcode() == PreferredFusedOpcode) {
+        SDValue N02 = N0.getOperand(2);
+        if (N02.getOpcode() == ISD::FP_EXTEND) {
+          SDValue N020 = N02.getOperand(0);
+          if (N020.getOpcode() == ISD::FMUL)
+            return FoldFAddFMAFPExtFMul(N0.getOperand(0), N0.getOperand(1),
+                                        N020.getOperand(0), N020.getOperand(1),
+                                        N1);
+        }
+      }
+
+      // fold (fadd (fpext (fma x, y, (fmul u, v))), z)
+      //   -> (fma (fpext x), (fpext y), (fma (fpext u), (fpext v), z))
+      // FIXME: This turns two single-precision and one double-precision
+      // operation into two double-precision operations, which might not be
+      // interesting for all targets, especially GPUs.
+      auto FoldFAddFPExtFMAFMul = [&] (
+          SDValue X, SDValue Y, SDValue U, SDValue V, SDValue Z) {
+        return DAG.getNode(PreferredFusedOpcode, SL, VT,
+                           DAG.getNode(ISD::FP_EXTEND, SL, VT, X),
+                           DAG.getNode(ISD::FP_EXTEND, SL, VT, Y),
+                           DAG.getNode(PreferredFusedOpcode, SL, VT,
+                                       DAG.getNode(ISD::FP_EXTEND, SL, VT, U),
+                                       DAG.getNode(ISD::FP_EXTEND, SL, VT, V),
+                                       Z));
+      };
+      if (N0.getOpcode() == ISD::FP_EXTEND) {
+        SDValue N00 = N0.getOperand(0);
+        if (N00.getOpcode() == PreferredFusedOpcode) {
+          SDValue N002 = N00.getOperand(2);
+          if (N002.getOpcode() == ISD::FMUL)
+            return FoldFAddFPExtFMAFMul(N00.getOperand(0), N00.getOperand(1),
+                                        N002.getOperand(0), N002.getOperand(1),
+                                        N1);
+        }
+      }
+
+      // fold (fadd x, (fma y, z, (fpext (fmul u, v)))
+      //   -> (fma y, z, (fma (fpext u), (fpext v), x))
+      if (N1.getOpcode() == PreferredFusedOpcode) {
+        SDValue N12 = N1.getOperand(2);
+        if (N12.getOpcode() == ISD::FP_EXTEND) {
+          SDValue N120 = N12.getOperand(0);
+          if (N120.getOpcode() == ISD::FMUL)
+            return FoldFAddFMAFPExtFMul(N1.getOperand(0), N1.getOperand(1),
+                                        N120.getOperand(0), N120.getOperand(1),
+                                        N0);
+        }
+      }
+
+      // fold (fadd x, (fpext (fma y, z, (fmul u, v)))
+      //   -> (fma (fpext y), (fpext z), (fma (fpext u), (fpext v), x))
+      // FIXME: This turns two single-precision and one double-precision
+      // operation into two double-precision operations, which might not be
+      // interesting for all targets, especially GPUs.
+      if (N1.getOpcode() == ISD::FP_EXTEND) {
+        SDValue N10 = N1.getOperand(0);
+        if (N10.getOpcode() == PreferredFusedOpcode) {
+          SDValue N102 = N10.getOperand(2);
+          if (N102.getOpcode() == ISD::FMUL)
+            return FoldFAddFPExtFMAFMul(N10.getOperand(0), N10.getOperand(1),
+                                        N102.getOperand(0), N102.getOperand(1),
+                                        N0);
+        }
+      }
+    }
   }
 
   return SDValue();
 }
 
-static SDValue performFsubFmulCombines(unsigned FusedOpcode,
-                                       bool Aggressive,
-                                       SDNode *N,
-                                       const TargetLowering &TLI,
-                                       SelectionDAG &DAG) {
+/// Try to perform FMA combining on a given FSUB node.
+SDValue DAGCombiner::visitFSUBForFMACombine(SDNode *N) {
   SDValue N0 = N->getOperand(0);
   SDValue N1 = N->getOperand(1);
   EVT VT = N->getValueType(0);
-
   SDLoc SL(N);
 
+  const TargetOptions &Options = DAG.getTarget().Options;
+  bool UnsafeFPMath = (Options.AllowFPOpFusion == FPOpFusion::Fast ||
+                       Options.UnsafeFPMath);
+
+  // Floating-point multiply-add with intermediate rounding.
+  bool HasFMAD = (LegalOperations &&
+                  TLI.isOperationLegal(ISD::FMAD, VT));
+
+  // Floating-point multiply-add without intermediate rounding.
+  bool HasFMA = ((!LegalOperations ||
+                  TLI.isOperationLegalOrCustom(ISD::FMA, VT)) &&
+                 TLI.isFMAFasterThanFMulAndFAdd(VT) &&
+                 UnsafeFPMath);
+
+  // No valid opcode, do not combine.
+  if (!HasFMAD && !HasFMA)
+    return SDValue();
+
+  // Always prefer FMAD to FMA for precision.
+  unsigned int PreferredFusedOpcode = HasFMAD ? ISD::FMAD : ISD::FMA;
+  bool Aggressive = TLI.enableAggressiveFMAFusion(VT);
+  bool LookThroughFPExt = TLI.isFPExtFree(VT);
+
   // fold (fsub (fmul x, y), z) -> (fma x, y, (fneg z))
   if (N0.getOpcode() == ISD::FMUL &&
       (Aggressive || N0->hasOneUse())) {
-    return DAG.getNode(FusedOpcode, SL, VT,
+    return DAG.getNode(PreferredFusedOpcode, SL, VT,
                        N0.getOperand(0), N0.getOperand(1),
                        DAG.getNode(ISD::FNEG, SL, VT, N1));
   }
@@ -7133,7 +7623,7 @@ static SDValue performFsubFmulCombines(unsigned FusedOpcode,
   // Note: Commutes FSUB operands.
   if (N1.getOpcode() == ISD::FMUL &&
       (Aggressive || N1->hasOneUse()))
-    return DAG.getNode(FusedOpcode, SL, VT,
+    return DAG.getNode(PreferredFusedOpcode, SL, VT,
                        DAG.getNode(ISD::FNEG, SL, VT,
                                    N1.getOperand(0)),
                        N1.getOperand(1), N0);
@@ -7144,41 +7634,214 @@ static SDValue performFsubFmulCombines(unsigned FusedOpcode,
       (Aggressive || (N0->hasOneUse() && N0.getOperand(0).hasOneUse()))) {
     SDValue N00 = N0.getOperand(0).getOperand(0);
     SDValue N01 = N0.getOperand(0).getOperand(1);
-    return DAG.getNode(FusedOpcode, SL, VT,
+    return DAG.getNode(PreferredFusedOpcode, SL, VT,
                        DAG.getNode(ISD::FNEG, SL, VT, N00), N01,
                        DAG.getNode(ISD::FNEG, SL, VT, N1));
   }
 
+  // Look through FP_EXTEND nodes to do more combining.
+  if (UnsafeFPMath && LookThroughFPExt) {
+    // fold (fsub (fpext (fmul x, y)), z)
+    //   -> (fma (fpext x), (fpext y), (fneg z))
+    if (N0.getOpcode() == ISD::FP_EXTEND) {
+      SDValue N00 = N0.getOperand(0);
+      if (N00.getOpcode() == ISD::FMUL)
+        return DAG.getNode(PreferredFusedOpcode, SL, VT,
+                           DAG.getNode(ISD::FP_EXTEND, SL, VT,
+                                       N00.getOperand(0)),
+                           DAG.getNode(ISD::FP_EXTEND, SL, VT,
+                                       N00.getOperand(1)),
+                           DAG.getNode(ISD::FNEG, SL, VT, N1));
+    }
+
+    // fold (fsub x, (fpext (fmul y, z)))
+    //   -> (fma (fneg (fpext y)), (fpext z), x)
+    // Note: Commutes FSUB operands.
+    if (N1.getOpcode() == ISD::FP_EXTEND) {
+      SDValue N10 = N1.getOperand(0);
+      if (N10.getOpcode() == ISD::FMUL)
+        return DAG.getNode(PreferredFusedOpcode, SL, VT,
+                           DAG.getNode(ISD::FNEG, SL, VT,
+                                       DAG.getNode(ISD::FP_EXTEND, SL, VT,
+                                                   N10.getOperand(0))),
+                           DAG.getNode(ISD::FP_EXTEND, SL, VT,
+                                       N10.getOperand(1)),
+                           N0);
+    }
+
+    // fold (fsub (fpext (fneg (fmul, x, y))), z)
+    //   -> (fneg (fma (fpext x), (fpext y), z))
+    // Note: This could be removed with appropriate canonicalization of the
+    // input expression into (fneg (fadd (fpext (fmul, x, y)), z). However, the
+    // orthogonal flags -fp-contract=fast and -enable-unsafe-fp-math prevent
+    // from implementing the canonicalization in visitFSUB.
+    if (N0.getOpcode() == ISD::FP_EXTEND) {
+      SDValue N00 = N0.getOperand(0);
+      if (N00.getOpcode() == ISD::FNEG) {
+        SDValue N000 = N00.getOperand(0);
+        if (N000.getOpcode() == ISD::FMUL) {
+          return DAG.getNode(ISD::FNEG, SL, VT,
+                             DAG.getNode(PreferredFusedOpcode, SL, VT,
+                                         DAG.getNode(ISD::FP_EXTEND, SL, VT,
+                                                     N000.getOperand(0)),
+                                         DAG.getNode(ISD::FP_EXTEND, SL, VT,
+                                                     N000.getOperand(1)),
+                                         N1));
+        }
+      }
+    }
+
+    // fold (fsub (fneg (fpext (fmul, x, y))), z)
+    //   -> (fneg (fma (fpext x)), (fpext y), z)
+    // Note: This could be removed with appropriate canonicalization of the
+    // input expression into (fneg (fadd (fpext (fmul, x, y)), z). However, the
+    // orthogonal flags -fp-contract=fast and -enable-unsafe-fp-math prevent
+    // from implementing the canonicalization in visitFSUB.
+    if (N0.getOpcode() == ISD::FNEG) {
+      SDValue N00 = N0.getOperand(0);
+      if (N00.getOpcode() == ISD::FP_EXTEND) {
+        SDValue N000 = N00.getOperand(0);
+        if (N000.getOpcode() == ISD::FMUL) {
+          return DAG.getNode(ISD::FNEG, SL, VT,
+                             DAG.getNode(PreferredFusedOpcode, SL, VT,
+                                         DAG.getNode(ISD::FP_EXTEND, SL, VT,
+                                                     N000.getOperand(0)),
+                                         DAG.getNode(ISD::FP_EXTEND, SL, VT,
+                                                     N000.getOperand(1)),
+                                         N1));
+        }
+      }
+    }
+
+  }
+
   // More folding opportunities when target permits.
-  if (Aggressive) {
+  if ((UnsafeFPMath || HasFMAD) && Aggressive) {
     // fold (fsub (fma x, y, (fmul u, v)), z)
     //   -> (fma x, y (fma u, v, (fneg z)))
-    if (N0.getOpcode() == FusedOpcode &&
+    if (N0.getOpcode() == PreferredFusedOpcode &&
         N0.getOperand(2).getOpcode() == ISD::FMUL) {
-      return DAG.getNode(FusedOpcode, SDLoc(N), VT,
+      return DAG.getNode(PreferredFusedOpcode, SL, VT,
                          N0.getOperand(0), N0.getOperand(1),
-                         DAG.getNode(FusedOpcode, SDLoc(N), VT,
+                         DAG.getNode(PreferredFusedOpcode, SL, VT,
                                      N0.getOperand(2).getOperand(0),
                                      N0.getOperand(2).getOperand(1),
-                                     DAG.getNode(ISD::FNEG, SDLoc(N), VT,
+                                     DAG.getNode(ISD::FNEG, SL, VT,
                                                  N1)));
     }
 
     // fold (fsub x, (fma y, z, (fmul u, v)))
     //   -> (fma (fneg y), z, (fma (fneg u), v, x))
-    if (N1.getOpcode() == FusedOpcode &&
+    if (N1.getOpcode() == PreferredFusedOpcode &&
         N1.getOperand(2).getOpcode() == ISD::FMUL) {
       SDValue N20 = N1.getOperand(2).getOperand(0);
       SDValue N21 = N1.getOperand(2).getOperand(1);
-      return DAG.getNode(FusedOpcode, SDLoc(N), VT,
-                         DAG.getNode(ISD::FNEG, SDLoc(N), VT,
+      return DAG.getNode(PreferredFusedOpcode, SL, VT,
+                         DAG.getNode(ISD::FNEG, SL, VT,
                                      N1.getOperand(0)),
                          N1.getOperand(1),
-                         DAG.getNode(FusedOpcode, SDLoc(N), VT,
-                                     DAG.getNode(ISD::FNEG, SDLoc(N),  VT,
-                                                 N20),
+                         DAG.getNode(PreferredFusedOpcode, SL, VT,
+                                     DAG.getNode(ISD::FNEG, SL, VT, N20),
+
                                      N21, N0));
     }
+
+    if (UnsafeFPMath && LookThroughFPExt) {
+      // fold (fsub (fma x, y, (fpext (fmul u, v))), z)
+      //   -> (fma x, y (fma (fpext u), (fpext v), (fneg z)))
+      if (N0.getOpcode() == PreferredFusedOpcode) {
+        SDValue N02 = N0.getOperand(2);
+        if (N02.getOpcode() == ISD::FP_EXTEND) {
+          SDValue N020 = N02.getOperand(0);
+          if (N020.getOpcode() == ISD::FMUL)
+            return DAG.getNode(PreferredFusedOpcode, SL, VT,
+                               N0.getOperand(0), N0.getOperand(1),
+                               DAG.getNode(PreferredFusedOpcode, SL, VT,
+                                           DAG.getNode(ISD::FP_EXTEND, SL, VT,
+                                                       N020.getOperand(0)),
+                                           DAG.getNode(ISD::FP_EXTEND, SL, VT,
+                                                       N020.getOperand(1)),
+                                           DAG.getNode(ISD::FNEG, SL, VT,
+                                                       N1)));
+        }
+      }
+
+      // fold (fsub (fpext (fma x, y, (fmul u, v))), z)
+      //   -> (fma (fpext x), (fpext y),
+      //           (fma (fpext u), (fpext v), (fneg z)))
+      // FIXME: This turns two single-precision and one double-precision
+      // operation into two double-precision operations, which might not be
+      // interesting for all targets, especially GPUs.
+      if (N0.getOpcode() == ISD::FP_EXTEND) {
+        SDValue N00 = N0.getOperand(0);
+        if (N00.getOpcode() == PreferredFusedOpcode) {
+          SDValue N002 = N00.getOperand(2);
+          if (N002.getOpcode() == ISD::FMUL)
+            return DAG.getNode(PreferredFusedOpcode, SL, VT,
+                               DAG.getNode(ISD::FP_EXTEND, SL, VT,
+                                           N00.getOperand(0)),
+                               DAG.getNode(ISD::FP_EXTEND, SL, VT,
+                                           N00.getOperand(1)),
+                               DAG.getNode(PreferredFusedOpcode, SL, VT,
+                                           DAG.getNode(ISD::FP_EXTEND, SL, VT,
+                                                       N002.getOperand(0)),
+                                           DAG.getNode(ISD::FP_EXTEND, SL, VT,
+                                                       N002.getOperand(1)),
+                                           DAG.getNode(ISD::FNEG, SL, VT,
+                                                       N1)));
+        }
+      }
+
+      // fold (fsub x, (fma y, z, (fpext (fmul u, v))))
+      //   -> (fma (fneg y), z, (fma (fneg (fpext u)), (fpext v), x))
+      if (N1.getOpcode() == PreferredFusedOpcode &&
+        N1.getOperand(2).getOpcode() == ISD::FP_EXTEND) {
+        SDValue N120 = N1.getOperand(2).getOperand(0);
+        if (N120.getOpcode() == ISD::FMUL) {
+          SDValue N1200 = N120.getOperand(0);
+          SDValue N1201 = N120.getOperand(1);
+          return DAG.getNode(PreferredFusedOpcode, SL, VT,
+                             DAG.getNode(ISD::FNEG, SL, VT, N1.getOperand(0)),
+                             N1.getOperand(1),
+                             DAG.getNode(PreferredFusedOpcode, SL, VT,
+                                         DAG.getNode(ISD::FNEG, SL, VT,
+                                             DAG.getNode(ISD::FP_EXTEND, SL,
+                                                         VT, N1200)),
+                                         DAG.getNode(ISD::FP_EXTEND, SL, VT,
+                                                     N1201),
+                                         N0));
+        }
+      }
+
+      // fold (fsub x, (fpext (fma y, z, (fmul u, v))))
+      //   -> (fma (fneg (fpext y)), (fpext z),
+      //           (fma (fneg (fpext u)), (fpext v), x))
+      // FIXME: This turns two single-precision and one double-precision
+      // operation into two double-precision operations, which might not be
+      // interesting for all targets, especially GPUs.
+      if (N1.getOpcode() == ISD::FP_EXTEND &&
+        N1.getOperand(0).getOpcode() == PreferredFusedOpcode) {
+        SDValue N100 = N1.getOperand(0).getOperand(0);
+        SDValue N101 = N1.getOperand(0).getOperand(1);
+        SDValue N102 = N1.getOperand(0).getOperand(2);
+        if (N102.getOpcode() == ISD::FMUL) {
+          SDValue N1020 = N102.getOperand(0);
+          SDValue N1021 = N102.getOperand(1);
+          return DAG.getNode(PreferredFusedOpcode, SL, VT,
+                             DAG.getNode(ISD::FNEG, SL, VT,
+                                         DAG.getNode(ISD::FP_EXTEND, SL, VT,
+                                                     N100)),
+                             DAG.getNode(ISD::FP_EXTEND, SL, VT, N101),
+                             DAG.getNode(PreferredFusedOpcode, SL, VT,
+                                         DAG.getNode(ISD::FNEG, SL, VT,
+                                             DAG.getNode(ISD::FP_EXTEND, SL,
+                                                         VT, N1020)),
+                                         DAG.getNode(ISD::FP_EXTEND, SL, VT,
+                                                     N1021),
+                                         N0));
+        }
+      }
+    }
   }
 
   return SDValue();
@@ -7190,6 +7853,7 @@ SDValue DAGCombiner::visitFADD(SDNode *N) {
   ConstantFPSDNode *N0CFP = dyn_cast<ConstantFPSDNode>(N0);
   ConstantFPSDNode *N1CFP = dyn_cast<ConstantFPSDNode>(N1);
   EVT VT = N->getValueType(0);
+  SDLoc DL(N);
   const TargetOptions &Options = DAG.getTarget().Options;
 
   // fold vector ops
@@ -7199,22 +7863,22 @@ SDValue DAGCombiner::visitFADD(SDNode *N) {
 
   // fold (fadd c1, c2) -> c1 + c2
   if (N0CFP && N1CFP)
-    return DAG.getNode(ISD::FADD, SDLoc(N), VT, N0, N1);
+    return DAG.getNode(ISD::FADD, DL, VT, N0, N1);
 
   // canonicalize constant to RHS
   if (N0CFP && !N1CFP)
-    return DAG.getNode(ISD::FADD, SDLoc(N), VT, N1, N0);
+    return DAG.getNode(ISD::FADD, DL, VT, N1, N0);
 
   // fold (fadd A, (fneg B)) -> (fsub A, B)
   if ((!LegalOperations || TLI.isOperationLegalOrCustom(ISD::FSUB, VT)) &&
       isNegatibleForFree(N1, LegalOperations, TLI, &Options) == 2)
-    return DAG.getNode(ISD::FSUB, SDLoc(N), VT, N0,
+    return DAG.getNode(ISD::FSUB, DL, VT, N0,
                        GetNegatedExpression(N1, DAG, LegalOperations));
 
   // fold (fadd (fneg A), B) -> (fsub B, A)
   if ((!LegalOperations || TLI.isOperationLegalOrCustom(ISD::FSUB, VT)) &&
       isNegatibleForFree(N0, LegalOperations, TLI, &Options) == 2)
-    return DAG.getNode(ISD::FSUB, SDLoc(N), VT, N1,
+    return DAG.getNode(ISD::FSUB, DL, VT, N1,
                        GetNegatedExpression(N0, DAG, LegalOperations));
 
   // If 'unsafe math' is enabled, fold lots of things.
@@ -7224,23 +7888,22 @@ SDValue DAGCombiner::visitFADD(SDNode *N) {
     bool AllowNewConst = (Level < AfterLegalizeDAG);
 
     // fold (fadd A, 0) -> A
-    if (N1CFP && N1CFP->getValueAPF().isZero())
+    if (N1CFP && N1CFP->isZero())
       return N0;
 
     // fold (fadd (fadd x, c1), c2) -> (fadd x, (fadd c1, c2))
     if (N1CFP && N0.getOpcode() == ISD::FADD && N0.getNode()->hasOneUse() &&
         isa<ConstantFPSDNode>(N0.getOperand(1)))
-      return DAG.getNode(ISD::FADD, SDLoc(N), VT, N0.getOperand(0),
-                         DAG.getNode(ISD::FADD, SDLoc(N), VT,
-                                     N0.getOperand(1), N1));
+      return DAG.getNode(ISD::FADD, DL, VT, N0.getOperand(0),
+                         DAG.getNode(ISD::FADD, DL, VT, N0.getOperand(1), N1));
 
     // If allowed, fold (fadd (fneg x), x) -> 0.0
     if (AllowNewConst && N0.getOpcode() == ISD::FNEG && N0.getOperand(0) == N1)
-      return DAG.getConstantFP(0.0, VT);
+      return DAG.getConstantFP(0.0, DL, VT);
 
     // If allowed, fold (fadd x, (fneg x)) -> 0.0
     if (AllowNewConst && N1.getOpcode() == ISD::FNEG && N1.getOperand(0) == N0)
-      return DAG.getConstantFP(0.0, VT);
+      return DAG.getConstantFP(0.0, DL, VT);
 
     // We can fold chains of FADD's of the same value into multiplications.
     // This transform is not safe in general because we are reducing the number
@@ -7252,21 +7915,18 @@ SDValue DAGCombiner::visitFADD(SDNode *N) {
 
         // (fadd (fmul x, c), x) -> (fmul x, c+1)
         if (CFP01 && !CFP00 && N0.getOperand(0) == N1) {
-          SDValue NewCFP = DAG.getNode(ISD::FADD, SDLoc(N), VT,
-                                       SDValue(CFP01, 0),
-                                       DAG.getConstantFP(1.0, VT));
-          return DAG.getNode(ISD::FMUL, SDLoc(N), VT, N1, NewCFP);
+          SDValue NewCFP = DAG.getNode(ISD::FADD, DL, VT, SDValue(CFP01, 0),
+                                       DAG.getConstantFP(1.0, DL, VT));
+          return DAG.getNode(ISD::FMUL, DL, VT, N1, NewCFP);
         }
 
         // (fadd (fmul x, c), (fadd x, x)) -> (fmul x, c+2)
         if (CFP01 && !CFP00 && N1.getOpcode() == ISD::FADD &&
             N1.getOperand(0) == N1.getOperand(1) &&
             N0.getOperand(0) == N1.getOperand(0)) {
-          SDValue NewCFP = DAG.getNode(ISD::FADD, SDLoc(N), VT,
-                                       SDValue(CFP01, 0),
-                                       DAG.getConstantFP(2.0, VT));
-          return DAG.getNode(ISD::FMUL, SDLoc(N), VT,
-                             N0.getOperand(0), NewCFP);
+          SDValue NewCFP = DAG.getNode(ISD::FADD, DL, VT, SDValue(CFP01, 0),
+                                       DAG.getConstantFP(2.0, DL, VT));
+          return DAG.getNode(ISD::FMUL, DL, VT, N0.getOperand(0), NewCFP);
         }
       }
 
@@ -7276,20 +7936,18 @@ SDValue DAGCombiner::visitFADD(SDNode *N) {
 
         // (fadd x, (fmul x, c)) -> (fmul x, c+1)
         if (CFP11 && !CFP10 && N1.getOperand(0) == N0) {
-          SDValue NewCFP = DAG.getNode(ISD::FADD, SDLoc(N), VT,
-                                       SDValue(CFP11, 0),
-                                       DAG.getConstantFP(1.0, VT));
-          return DAG.getNode(ISD::FMUL, SDLoc(N), VT, N0, NewCFP);
+          SDValue NewCFP = DAG.getNode(ISD::FADD, DL, VT, SDValue(CFP11, 0),
+                                       DAG.getConstantFP(1.0, DL, VT));
+          return DAG.getNode(ISD::FMUL, DL, VT, N0, NewCFP);
         }
 
         // (fadd (fadd x, x), (fmul x, c)) -> (fmul x, c+2)
         if (CFP11 && !CFP10 && N0.getOpcode() == ISD::FADD &&
             N0.getOperand(0) == N0.getOperand(1) &&
             N1.getOperand(0) == N0.getOperand(0)) {
-          SDValue NewCFP = DAG.getNode(ISD::FADD, SDLoc(N), VT,
-                                       SDValue(CFP11, 0),
-                                       DAG.getConstantFP(2.0, VT));
-          return DAG.getNode(ISD::FMUL, SDLoc(N), VT, N1.getOperand(0), NewCFP);
+          SDValue NewCFP = DAG.getNode(ISD::FADD, DL, VT, SDValue(CFP11, 0),
+                                       DAG.getConstantFP(2.0, DL, VT));
+          return DAG.getNode(ISD::FMUL, DL, VT, N1.getOperand(0), NewCFP);
         }
       }
 
@@ -7297,18 +7955,20 @@ SDValue DAGCombiner::visitFADD(SDNode *N) {
         ConstantFPSDNode *CFP = dyn_cast<ConstantFPSDNode>(N0.getOperand(0));
         // (fadd (fadd x, x), x) -> (fmul x, 3.0)
         if (!CFP && N0.getOperand(0) == N0.getOperand(1) &&
-            (N0.getOperand(0) == N1))
-          return DAG.getNode(ISD::FMUL, SDLoc(N), VT,
-                             N1, DAG.getConstantFP(3.0, VT));
+            (N0.getOperand(0) == N1)) {
+          return DAG.getNode(ISD::FMUL, DL, VT,
+                             N1, DAG.getConstantFP(3.0, DL, VT));
+        }
       }
 
       if (N1.getOpcode() == ISD::FADD && AllowNewConst) {
         ConstantFPSDNode *CFP10 = dyn_cast<ConstantFPSDNode>(N1.getOperand(0));
         // (fadd x, (fadd x, x)) -> (fmul x, 3.0)
         if (!CFP10 && N1.getOperand(0) == N1.getOperand(1) &&
-            N1.getOperand(0) == N0)
-          return DAG.getNode(ISD::FMUL, SDLoc(N), VT,
-                             N0, DAG.getConstantFP(3.0, VT));
+            N1.getOperand(0) == N0) {
+          return DAG.getNode(ISD::FMUL, DL, VT,
+                             N0, DAG.getConstantFP(3.0, DL, VT));
+        }
       }
 
       // (fadd (fadd x, x), (fadd x, x)) -> (fmul x, 4.0)
@@ -7316,61 +7976,18 @@ SDValue DAGCombiner::visitFADD(SDNode *N) {
           N0.getOpcode() == ISD::FADD && N1.getOpcode() == ISD::FADD &&
           N0.getOperand(0) == N0.getOperand(1) &&
           N1.getOperand(0) == N1.getOperand(1) &&
-          N0.getOperand(0) == N1.getOperand(0))
-        return DAG.getNode(ISD::FMUL, SDLoc(N), VT,
-                           N0.getOperand(0), DAG.getConstantFP(4.0, VT));
+          N0.getOperand(0) == N1.getOperand(0)) {
+        return DAG.getNode(ISD::FMUL, DL, VT,
+                           N0.getOperand(0), DAG.getConstantFP(4.0, DL, VT));
+      }
     }
   } // enable-unsafe-fp-math
 
-  if (LegalOperations && TLI.isOperationLegal(ISD::FMAD, VT)) {
-    // Assume if there is an fmad instruction that it should be aggressively
-    // used.
-    if (SDValue Fused = performFaddFmulCombines(ISD::FMAD, true, N, TLI, DAG))
-      return Fused;
-  }
-
   // FADD -> FMA combines:
-  if ((Options.AllowFPOpFusion == FPOpFusion::Fast || Options.UnsafeFPMath) &&
-      TLI.isFMAFasterThanFMulAndFAdd(VT) &&
-      (!LegalOperations || TLI.isOperationLegalOrCustom(ISD::FMA, VT))) {
-
-    if (!TLI.isOperationLegal(ISD::FMAD, VT)) {
-      // Don't form FMA if we are preferring FMAD.
-      if (SDValue Fused
-          = performFaddFmulCombines(ISD::FMA,
-                                    TLI.enableAggressiveFMAFusion(VT),
-                                    N, TLI, DAG)) {
-        return Fused;
-      }
-    }
-
-    // When FP_EXTEND nodes are free on the target, and there is an opportunity
-    // to combine into FMA, arrange such nodes accordingly.
-    if (TLI.isFPExtFree(VT)) {
-
-      // fold (fadd (fpext (fmul x, y)), z) -> (fma (fpext x), (fpext y), z)
-      if (N0.getOpcode() == ISD::FP_EXTEND) {
-        SDValue N00 = N0.getOperand(0);
-        if (N00.getOpcode() == ISD::FMUL)
-          return DAG.getNode(ISD::FMA, SDLoc(N), VT,
-                             DAG.getNode(ISD::FP_EXTEND, SDLoc(N), VT,
-                                         N00.getOperand(0)),
-                             DAG.getNode(ISD::FP_EXTEND, SDLoc(N), VT,
-                                         N00.getOperand(1)), N1);
-      }
-
-      // fold (fadd x, (fpext (fmul y, z)), z) -> (fma (fpext y), (fpext z), x)
-      // Note: Commutes FADD operands.
-      if (N1.getOpcode() == ISD::FP_EXTEND) {
-        SDValue N10 = N1.getOperand(0);
-        if (N10.getOpcode() == ISD::FMUL)
-          return DAG.getNode(ISD::FMA, SDLoc(N), VT,
-                             DAG.getNode(ISD::FP_EXTEND, SDLoc(N), VT,
-                                         N10.getOperand(0)),
-                             DAG.getNode(ISD::FP_EXTEND, SDLoc(N), VT,
-                                         N10.getOperand(1)), N0);
-      }
-    }
+  SDValue Fused = visitFADDForFMACombine(N);
+  if (Fused) {
+    AddToWorklist(Fused.getNode());
+    return Fused;
   }
 
   return SDValue();
@@ -7392,7 +8009,7 @@ SDValue DAGCombiner::visitFSUB(SDNode *N) {
 
   // fold (fsub c1, c2) -> c1-c2
   if (N0CFP && N1CFP)
-    return DAG.getNode(ISD::FSUB, SDLoc(N), VT, N0, N1);
+    return DAG.getNode(ISD::FSUB, dl, VT, N0, N1);
 
   // fold (fsub A, (fneg B)) -> (fadd A, B)
   if (isNegatibleForFree(N1, LegalOperations, TLI, &Options))
@@ -7402,11 +8019,11 @@ SDValue DAGCombiner::visitFSUB(SDNode *N) {
   // If 'unsafe math' is enabled, fold lots of things.
   if (Options.UnsafeFPMath) {
     // (fsub A, 0) -> A
-    if (N1CFP && N1CFP->getValueAPF().isZero())
+    if (N1CFP && N1CFP->isZero())
       return N0;
 
     // (fsub 0, B) -> -B
-    if (N0CFP && N0CFP->getValueAPF().isZero()) {
+    if (N0CFP && N0CFP->isZero()) {
       if (isNegatibleForFree(N1, LegalOperations, TLI, &Options))
         return GetNegatedExpression(N1, DAG, LegalOperations);
       if (!LegalOperations || TLI.isOperationLegal(ISD::FNEG, VT))
@@ -7415,7 +8032,7 @@ SDValue DAGCombiner::visitFSUB(SDNode *N) {
 
     // (fsub x, x) -> 0.0
     if (N0 == N1)
-      return DAG.getConstantFP(0.0f, VT);
+      return DAG.getConstantFP(0.0f, dl, VT);
 
     // (fsub x, (fadd x, y)) -> (fneg y)
     // (fsub x, (fadd y, x)) -> (fneg y)
@@ -7431,96 +8048,11 @@ SDValue DAGCombiner::visitFSUB(SDNode *N) {
     }
   }
 
-  if (LegalOperations && TLI.isOperationLegal(ISD::FMAD, VT)) {
-    // Assume if there is an fmad instruction that it should be aggressively
-    // used.
-    if (SDValue Fused = performFsubFmulCombines(ISD::FMAD, true, N, TLI, DAG))
-      return Fused;
-  }
-
   // FSUB -> FMA combines:
-  if ((Options.AllowFPOpFusion == FPOpFusion::Fast || Options.UnsafeFPMath) &&
-      TLI.isFMAFasterThanFMulAndFAdd(VT) &&
-      (!LegalOperations || TLI.isOperationLegalOrCustom(ISD::FMA, VT))) {
-
-    if (!TLI.isOperationLegal(ISD::FMAD, VT)) {
-      // Don't form FMA if we are preferring FMAD.
-
-      if (SDValue Fused
-          = performFsubFmulCombines(ISD::FMA,
-                                    TLI.enableAggressiveFMAFusion(VT),
-                                    N, TLI, DAG)) {
-        return Fused;
-      }
-    }
-
-    // When FP_EXTEND nodes are free on the target, and there is an opportunity
-    // to combine into FMA, arrange such nodes accordingly.
-    if (TLI.isFPExtFree(VT)) {
-      // fold (fsub (fpext (fmul x, y)), z)
-      //   -> (fma (fpext x), (fpext y), (fneg z))
-      if (N0.getOpcode() == ISD::FP_EXTEND) {
-        SDValue N00 = N0.getOperand(0);
-        if (N00.getOpcode() == ISD::FMUL)
-          return DAG.getNode(ISD::FMA, SDLoc(N), VT,
-                             DAG.getNode(ISD::FP_EXTEND, SDLoc(N), VT,
-                                         N00.getOperand(0)),
-                             DAG.getNode(ISD::FP_EXTEND, SDLoc(N), VT,
-                                         N00.getOperand(1)),
-                             DAG.getNode(ISD::FNEG, SDLoc(N), VT, N1));
-      }
-
-      // fold (fsub x, (fpext (fmul y, z)))
-      //   -> (fma (fneg (fpext y)), (fpext z), x)
-      // Note: Commutes FSUB operands.
-      if (N1.getOpcode() == ISD::FP_EXTEND) {
-        SDValue N10 = N1.getOperand(0);
-        if (N10.getOpcode() == ISD::FMUL)
-          return DAG.getNode(ISD::FMA, SDLoc(N), VT,
-                             DAG.getNode(ISD::FNEG, SDLoc(N), VT,
-                                         DAG.getNode(ISD::FP_EXTEND, SDLoc(N),
-                                                     VT, N10.getOperand(0))),
-                             DAG.getNode(ISD::FP_EXTEND, SDLoc(N), VT,
-                                         N10.getOperand(1)),
-                             N0);
-      }
-
-      // fold (fsub (fpext (fneg (fmul, x, y))), z)
-      //   -> (fma (fneg (fpext x)), (fpext y), (fneg z))
-      if (N0.getOpcode() == ISD::FP_EXTEND) {
-        SDValue N00 = N0.getOperand(0);
-        if (N00.getOpcode() == ISD::FNEG) {
-          SDValue N000 = N00.getOperand(0);
-          if (N000.getOpcode() == ISD::FMUL) {
-            return DAG.getNode(ISD::FMA, dl, VT,
-                               DAG.getNode(ISD::FNEG, dl, VT,
-                                           DAG.getNode(ISD::FP_EXTEND, SDLoc(N),
-                                                       VT, N000.getOperand(0))),
-                               DAG.getNode(ISD::FP_EXTEND, SDLoc(N), VT,
-                                           N000.getOperand(1)),
-                               DAG.getNode(ISD::FNEG, dl, VT, N1));
-          }
-        }
-      }
-
-      // fold (fsub (fneg (fpext (fmul, x, y))), z)
-      //   -> (fma (fneg (fpext x)), (fpext y), (fneg z))
-      if (N0.getOpcode() == ISD::FNEG) {
-        SDValue N00 = N0.getOperand(0);
-        if (N00.getOpcode() == ISD::FP_EXTEND) {
-          SDValue N000 = N00.getOperand(0);
-          if (N000.getOpcode() == ISD::FMUL) {
-            return DAG.getNode(ISD::FMA, dl, VT,
-                               DAG.getNode(ISD::FNEG, dl, VT,
-                                           DAG.getNode(ISD::FP_EXTEND, SDLoc(N),
-                                           VT, N000.getOperand(0))),
-                               DAG.getNode(ISD::FP_EXTEND, SDLoc(N), VT,
-                                           N000.getOperand(1)),
-                               DAG.getNode(ISD::FNEG, dl, VT, N1));
-          }
-        }
-      }
-    }
+  SDValue Fused = visitFSUBForFMACombine(N);
+  if (Fused) {
+    AddToWorklist(Fused.getNode());
+    return Fused;
   }
 
   return SDValue();
@@ -7532,6 +8064,7 @@ SDValue DAGCombiner::visitFMUL(SDNode *N) {
   ConstantFPSDNode *N0CFP = isConstOrConstSplatFP(N0);
   ConstantFPSDNode *N1CFP = isConstOrConstSplatFP(N1);
   EVT VT = N->getValueType(0);
+  SDLoc DL(N);
   const TargetOptions &Options = DAG.getTarget().Options;
 
   // fold vector ops
@@ -7543,12 +8076,12 @@ SDValue DAGCombiner::visitFMUL(SDNode *N) {
 
   // fold (fmul c1, c2) -> c1*c2
   if (N0CFP && N1CFP)
-    return DAG.getNode(ISD::FMUL, SDLoc(N), VT, N0, N1);
+    return DAG.getNode(ISD::FMUL, DL, VT, N0, N1);
 
   // canonicalize constant to RHS
   if (isConstantFPBuildVectorOrConstantFP(N0) &&
      !isConstantFPBuildVectorOrConstantFP(N1))
-    return DAG.getNode(ISD::FMUL, SDLoc(N), VT, N1, N0);
+    return DAG.getNode(ISD::FMUL, DL, VT, N1, N0);
 
   // fold (fmul A, 1.0) -> A
   if (N1CFP && N1CFP->isExactlyValue(1.0))
@@ -7556,7 +8089,7 @@ SDValue DAGCombiner::visitFMUL(SDNode *N) {
 
   if (Options.UnsafeFPMath) {
     // fold (fmul A, 0) -> 0
-    if (N1CFP && N1CFP->getValueAPF().isZero())
+    if (N1CFP && N1CFP->isZero())
       return N1;
 
     // fold (fmul (fmul x, c1), c2) -> (fmul x, (fmul c1, c2))
@@ -7569,7 +8102,7 @@ SDValue DAGCombiner::visitFMUL(SDNode *N) {
       auto *BV1 = dyn_cast<BuildVectorSDNode>(N1);
       auto *BV00 = dyn_cast<BuildVectorSDNode>(N00);
       auto *BV01 = dyn_cast<BuildVectorSDNode>(N01);
-      
+
       // Check 1: Make sure that the first operand of the inner multiply is NOT
       // a constant. Otherwise, we may induce infinite looping.
       if (!(isConstOrConstSplatFP(N00) || (BV00 && BV00->isConstant()))) {
@@ -7577,9 +8110,8 @@ SDValue DAGCombiner::visitFMUL(SDNode *N) {
         // the second operand of the outer multiply are constants.
         if ((N1CFP && isConstOrConstSplatFP(N01)) ||
             (BV1 && BV01 && BV1->isConstant() && BV01->isConstant())) {
-          SDLoc SL(N);
-          SDValue MulConsts = DAG.getNode(ISD::FMUL, SL, VT, N01, N1);
-          return DAG.getNode(ISD::FMUL, SL, VT, N00, MulConsts);
+          SDValue MulConsts = DAG.getNode(ISD::FMUL, DL, VT, N01, N1);
+          return DAG.getNode(ISD::FMUL, DL, VT, N00, MulConsts);
         }
       }
     }
@@ -7588,22 +8120,23 @@ SDValue DAGCombiner::visitFMUL(SDNode *N) {
     // Undo the fmul 2.0, x -> fadd x, x transformation, since if it occurs
     // during an early run of DAGCombiner can prevent folding with fmuls
     // inserted during lowering.
-    if (N0.getOpcode() == ISD::FADD && N0.getOperand(0) == N0.getOperand(1)) {
-      SDLoc SL(N);
-      const SDValue Two = DAG.getConstantFP(2.0, VT);
-      SDValue MulConsts = DAG.getNode(ISD::FMUL, SL, VT, Two, N1);
-      return DAG.getNode(ISD::FMUL, SDLoc(N), VT, N0.getOperand(0), MulConsts);
+    if (N0.getOpcode() == ISD::FADD &&
+        (N0.getOperand(0) == N0.getOperand(1)) &&
+        N0.hasOneUse()) {
+      const SDValue Two = DAG.getConstantFP(2.0, DL, VT);
+      SDValue MulConsts = DAG.getNode(ISD::FMUL, DL, VT, Two, N1);
+      return DAG.getNode(ISD::FMUL, DL, VT, N0.getOperand(0), MulConsts);
     }
   }
 
   // fold (fmul X, 2.0) -> (fadd X, X)
   if (N1CFP && N1CFP->isExactlyValue(+2.0))
-    return DAG.getNode(ISD::FADD, SDLoc(N), VT, N0, N0);
+    return DAG.getNode(ISD::FADD, DL, VT, N0, N0);
 
   // fold (fmul X, -1.0) -> (fneg X)
   if (N1CFP && N1CFP->isExactlyValue(-1.0))
     if (!LegalOperations || TLI.isOperationLegal(ISD::FNEG, VT))
-      return DAG.getNode(ISD::FNEG, SDLoc(N), VT, N0);
+      return DAG.getNode(ISD::FNEG, DL, VT, N0);
 
   // fold (fmul (fneg X), (fneg Y)) -> (fmul X, Y)
   if (char LHSNeg = isNegatibleForFree(N0, LegalOperations, TLI, &Options)) {
@@ -7611,7 +8144,7 @@ SDValue DAGCombiner::visitFMUL(SDNode *N) {
       // Both can be negated for free, check to see if at least one is cheaper
       // negated.
       if (LHSNeg == 2 || RHSNeg == 2)
-        return DAG.getNode(ISD::FMUL, SDLoc(N), VT,
+        return DAG.getNode(ISD::FMUL, DL, VT,
                            GetNegatedExpression(N0, DAG, LegalOperations),
                            GetNegatedExpression(N1, DAG, LegalOperations));
     }
@@ -7690,19 +8223,79 @@ SDValue DAGCombiner::visitFMA(SDNode *N) {
   if (Options.UnsafeFPMath && N1CFP && N0 == N2)
     return DAG.getNode(ISD::FMUL, dl, VT, N0,
                        DAG.getNode(ISD::FADD, dl, VT,
-                                   N1, DAG.getConstantFP(1.0, VT)));
+                                   N1, DAG.getConstantFP(1.0, dl, VT)));
 
   // (fma x, c, (fneg x)) -> (fmul x, (c-1))
   if (Options.UnsafeFPMath && N1CFP &&
       N2.getOpcode() == ISD::FNEG && N2.getOperand(0) == N0)
     return DAG.getNode(ISD::FMUL, dl, VT, N0,
                        DAG.getNode(ISD::FADD, dl, VT,
-                                   N1, DAG.getConstantFP(-1.0, VT)));
+                                   N1, DAG.getConstantFP(-1.0, dl, VT)));
 
 
   return SDValue();
 }
 
+// Combine multiple FDIVs with the same divisor into multiple FMULs by the
+// reciprocal.
+// E.g., (a / D; b / D;) -> (recip = 1.0 / D; a * recip; b * recip)
+// Notice that this is not always beneficial. One reason is different target
+// may have different costs for FDIV and FMUL, so sometimes the cost of two
+// FDIVs may be lower than the cost of one FDIV and two FMULs. Another reason
+// is the critical path is increased from "one FDIV" to "one FDIV + one FMUL".
+SDValue DAGCombiner::combineRepeatedFPDivisors(SDNode *N) {
+  if (!DAG.getTarget().Options.UnsafeFPMath)
+    return SDValue();
+  
+  // Skip if current node is a reciprocal.
+  SDValue N0 = N->getOperand(0);
+  ConstantFPSDNode *N0CFP = dyn_cast<ConstantFPSDNode>(N0);
+  if (N0CFP && N0CFP->isExactlyValue(1.0))
+    return SDValue();
+  
+  // Exit early if the target does not want this transform or if there can't
+  // possibly be enough uses of the divisor to make the transform worthwhile.
+  SDValue N1 = N->getOperand(1);
+  unsigned MinUses = TLI.combineRepeatedFPDivisors();
+  if (!MinUses || N1->use_size() < MinUses)
+    return SDValue();
+
+  // Find all FDIV users of the same divisor.
+  // Use a set because duplicates may be present in the user list.
+  SetVector<SDNode *> Users;
+  for (auto *U : N1->uses())
+    if (U->getOpcode() == ISD::FDIV && U->getOperand(1) == N1)
+      Users.insert(U);
+
+  // Now that we have the actual number of divisor uses, make sure it meets
+  // the minimum threshold specified by the target.
+  if (Users.size() < MinUses)
+    return SDValue();
+
+  EVT VT = N->getValueType(0);
+  SDLoc DL(N);
+  SDValue FPOne = DAG.getConstantFP(1.0, DL, VT);
+  // FIXME: This optimization requires some level of fast-math, so the
+  // created reciprocal node should at least have the 'allowReciprocal'
+  // fast-math-flag set.
+  SDValue Reciprocal = DAG.getNode(ISD::FDIV, DL, VT, FPOne, N1);
+
+  // Dividend / Divisor -> Dividend * Reciprocal
+  for (auto *U : Users) {
+    SDValue Dividend = U->getOperand(0);
+    if (Dividend != FPOne) {
+      SDValue NewNode = DAG.getNode(ISD::FMUL, SDLoc(U), VT, Dividend,
+                                    Reciprocal);
+      CombineTo(U, NewNode);
+    } else if (U != Reciprocal.getNode()) {
+      // In the absence of fast-math-flags, this user node is always the
+      // same node as Reciprocal, but with FMF they may be different nodes.
+      CombineTo(U, Reciprocal);
+    }
+  }
+  return SDValue(N, 0);  // N was replaced.
+}
+
 SDValue DAGCombiner::visitFDIV(SDNode *N) {
   SDValue N0 = N->getOperand(0);
   SDValue N1 = N->getOperand(1);
@@ -7737,8 +8330,8 @@ SDValue DAGCombiner::visitFDIV(SDNode *N) {
            // TLI.isOperationLegalOrCustom(llvm::ISD::ConstantFP, VT) ||
            TLI.isOperationLegal(llvm::ISD::ConstantFP, VT) ||
            TLI.isFPImmLegal(Recip, VT)))
-        return DAG.getNode(ISD::FMUL, SDLoc(N), VT, N0,
-                           DAG.getConstantFP(Recip, VT));
+        return DAG.getNode(ISD::FMUL, DL, VT, N0,
+                           DAG.getConstantFP(Recip, DL, VT));
     }
 
     // If this FDIV is part of a reciprocal square root, it may be folded
@@ -7803,43 +8396,8 @@ SDValue DAGCombiner::visitFDIV(SDNode *N) {
     }
   }
 
-  // Combine multiple FDIVs with the same divisor into multiple FMULs by the
-  // reciprocal.
-  // E.g., (a / D; b / D;) -> (recip = 1.0 / D; a * recip; b * recip)
-  // Notice that this is not always beneficial. One reason is different target
-  // may have different costs for FDIV and FMUL, so sometimes the cost of two
-  // FDIVs may be lower than the cost of one FDIV and two FMULs. Another reason
-  // is the critical path is increased from "one FDIV" to "one FDIV + one FMUL".
-  if (Options.UnsafeFPMath) {
-    // Skip if current node is a reciprocal.
-    if (N0CFP && N0CFP->isExactlyValue(1.0))
-      return SDValue();
-
-    SmallVector<SDNode *, 4> Users;
-    // Find all FDIV users of the same divisor.
-    for (SDNode::use_iterator UI = N1.getNode()->use_begin(),
-                              UE = N1.getNode()->use_end();
-         UI != UE; ++UI) {
-      SDNode *User = UI.getUse().getUser();
-      if (User->getOpcode() == ISD::FDIV && User->getOperand(1) == N1)
-        Users.push_back(User);
-    }
-
-    if (TLI.combineRepeatedFPDivisors(Users.size())) {
-      SDValue FPOne = DAG.getConstantFP(1.0, VT); // floating point 1.0
-      SDValue Reciprocal = DAG.getNode(ISD::FDIV, SDLoc(N), VT, FPOne, N1);
-
-      // Dividend / Divisor -> Dividend * Reciprocal
-      for (auto I = Users.begin(), E = Users.end(); I != E; ++I) {
-        if ((*I)->getOperand(0) != FPOne) {
-          SDValue NewNode = DAG.getNode(ISD::FMUL, SDLoc(*I), VT,
-                                        (*I)->getOperand(0), Reciprocal);
-          DAG.ReplaceAllUsesWith(*I, NewNode.getNode());
-        }
-      }
-      return SDValue();
-    }
-  }
+  if (SDValue CombineRepeatedDivisors = combineRepeatedFPDivisors(N))
+    return CombineRepeatedDivisors;
 
   return SDValue();
 }
@@ -7859,29 +8417,29 @@ SDValue DAGCombiner::visitFREM(SDNode *N) {
 }
 
 SDValue DAGCombiner::visitFSQRT(SDNode *N) {
-  if (DAG.getTarget().Options.UnsafeFPMath &&
-      !TLI.isFsqrtCheap()) {
-    // Compute this as X * (1/sqrt(X)) = X * (X ** -0.5)
-    if (SDValue RV = BuildRsqrtEstimate(N->getOperand(0))) {
-      EVT VT = RV.getValueType();
-      RV = DAG.getNode(ISD::FMUL, SDLoc(N), VT, N->getOperand(0), RV);
-      AddToWorklist(RV.getNode());
+  if (!DAG.getTarget().Options.UnsafeFPMath || TLI.isFsqrtCheap())
+    return SDValue();
 
-      // Unfortunately, RV is now NaN if the input was exactly 0.
-      // Select out this case and force the answer to 0.
-      SDValue Zero = DAG.getConstantFP(0.0, VT);
-      SDValue ZeroCmp =
-        DAG.getSetCC(SDLoc(N), TLI.getSetCCResultType(*DAG.getContext(), VT),
-                     N->getOperand(0), Zero, ISD::SETEQ);
-      AddToWorklist(ZeroCmp.getNode());
-      AddToWorklist(RV.getNode());
+  // Compute this as X * (1/sqrt(X)) = X * (X ** -0.5)
+  SDValue RV = BuildRsqrtEstimate(N->getOperand(0));
+  if (!RV)
+    return SDValue();
+  
+  EVT VT = RV.getValueType();
+  SDLoc DL(N);
+  RV = DAG.getNode(ISD::FMUL, DL, VT, N->getOperand(0), RV);
+  AddToWorklist(RV.getNode());
 
-      RV = DAG.getNode(VT.isVector() ? ISD::VSELECT : ISD::SELECT,
-                       SDLoc(N), VT, ZeroCmp, Zero, RV);
-      return RV;
-    }
-  }
-  return SDValue();
+  // Unfortunately, RV is now NaN if the input was exactly 0.
+  // Select out this case and force the answer to 0.
+  SDValue Zero = DAG.getConstantFP(0.0, DL, VT);
+  EVT CCVT = TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT);
+  SDValue ZeroCmp = DAG.getSetCC(DL, CCVT, N->getOperand(0), Zero, ISD::SETEQ);
+  AddToWorklist(ZeroCmp.getNode());
+  AddToWorklist(RV.getNode());
+
+  return DAG.getNode(VT.isVector() ? ISD::VSELECT : ISD::SELECT, DL, VT,
+                     ZeroCmp, Zero, RV);
 }
 
 SDValue DAGCombiner::visitFCOPYSIGN(SDNode *N) {
@@ -7962,11 +8520,12 @@ SDValue DAGCombiner::visitSINT_TO_FP(SDNode *N) {
         !VT.isVector() &&
         (!LegalOperations ||
          TLI.isOperationLegalOrCustom(llvm::ISD::ConstantFP, VT))) {
+      SDLoc DL(N);
       SDValue Ops[] =
         { N0.getOperand(0), N0.getOperand(1),
-          DAG.getConstantFP(-1.0, VT) , DAG.getConstantFP(0.0, VT),
+          DAG.getConstantFP(-1.0, DL, VT), DAG.getConstantFP(0.0, DL, VT),
           N0.getOperand(2) };
-      return DAG.getNode(ISD::SELECT_CC, SDLoc(N), VT, Ops);
+      return DAG.getNode(ISD::SELECT_CC, DL, VT, Ops);
     }
 
     // fold (sint_to_fp (zext (setcc x, y, cc))) ->
@@ -7975,11 +8534,12 @@ SDValue DAGCombiner::visitSINT_TO_FP(SDNode *N) {
         N0.getOperand(0).getOpcode() == ISD::SETCC &&!VT.isVector() &&
         (!LegalOperations ||
          TLI.isOperationLegalOrCustom(llvm::ISD::ConstantFP, VT))) {
+      SDLoc DL(N);
       SDValue Ops[] =
         { N0.getOperand(0).getOperand(0), N0.getOperand(0).getOperand(1),
-          DAG.getConstantFP(1.0, VT) , DAG.getConstantFP(0.0, VT),
+          DAG.getConstantFP(1.0, DL, VT), DAG.getConstantFP(0.0, DL, VT),
           N0.getOperand(0).getOperand(2) };
-      return DAG.getNode(ISD::SELECT_CC, SDLoc(N), VT, Ops);
+      return DAG.getNode(ISD::SELECT_CC, DL, VT, Ops);
     }
   }
 
@@ -8014,11 +8574,12 @@ SDValue DAGCombiner::visitUINT_TO_FP(SDNode *N) {
     if (N0.getOpcode() == ISD::SETCC && !VT.isVector() &&
         (!LegalOperations ||
          TLI.isOperationLegalOrCustom(llvm::ISD::ConstantFP, VT))) {
+      SDLoc DL(N);
       SDValue Ops[] =
         { N0.getOperand(0), N0.getOperand(1),
-          DAG.getConstantFP(1.0, VT),  DAG.getConstantFP(0.0, VT),
+          DAG.getConstantFP(1.0, DL, VT), DAG.getConstantFP(0.0, DL, VT),
           N0.getOperand(2) };
-      return DAG.getNode(ISD::SELECT_CC, SDLoc(N), VT, Ops);
+      return DAG.getNode(ISD::SELECT_CC, DL, VT, Ops);
     }
   }
 
@@ -8071,11 +8632,10 @@ static SDValue FoldIntToFPToInt(SDNode *N, SelectionDAG &DAG) {
 
 SDValue DAGCombiner::visitFP_TO_SINT(SDNode *N) {
   SDValue N0 = N->getOperand(0);
-  ConstantFPSDNode *N0CFP = dyn_cast<ConstantFPSDNode>(N0);
   EVT VT = N->getValueType(0);
 
   // fold (fp_to_sint c1fp) -> c1
-  if (N0CFP)
+  if (isConstantFPBuildVectorOrConstantFP(N0))
     return DAG.getNode(ISD::FP_TO_SINT, SDLoc(N), VT, N0);
 
   return FoldIntToFPToInt(N, DAG);
@@ -8083,11 +8643,10 @@ SDValue DAGCombiner::visitFP_TO_SINT(SDNode *N) {
 
 SDValue DAGCombiner::visitFP_TO_UINT(SDNode *N) {
   SDValue N0 = N->getOperand(0);
-  ConstantFPSDNode *N0CFP = dyn_cast<ConstantFPSDNode>(N0);
   EVT VT = N->getValueType(0);
 
   // fold (fp_to_uint c1fp) -> c1
-  if (N0CFP)
+  if (isConstantFPBuildVectorOrConstantFP(N0))
     return DAG.getNode(ISD::FP_TO_UINT, SDLoc(N), VT, N0);
 
   return FoldIntToFPToInt(N, DAG);
@@ -8116,9 +8675,11 @@ SDValue DAGCombiner::visitFP_ROUND(SDNode *N) {
     // single-step fp_round we want to fold to.
     // In other words, double rounding isn't the same as rounding.
     // Also, this is a value preserving truncation iff both fp_round's are.
-    if (DAG.getTarget().Options.UnsafeFPMath || N0IsTrunc)
-      return DAG.getNode(ISD::FP_ROUND, SDLoc(N), VT, N0.getOperand(0),
-                         DAG.getIntPtrConstant(NIsTrunc && N0IsTrunc));
+    if (DAG.getTarget().Options.UnsafeFPMath || N0IsTrunc) {
+      SDLoc DL(N);
+      return DAG.getNode(ISD::FP_ROUND, DL, VT, N0.getOperand(0),
+                         DAG.getIntPtrConstant(NIsTrunc && N0IsTrunc, DL));
+    }
   }
 
   // fold (fp_round (copysign X, Y)) -> (copysign (fp_round X), Y)
@@ -8141,8 +8702,9 @@ SDValue DAGCombiner::visitFP_ROUND_INREG(SDNode *N) {
 
   // fold (fp_round_inreg c1fp) -> c1fp
   if (N0CFP && isTypeLegal(EVT)) {
-    SDValue Round = DAG.getConstantFP(*N0CFP->getConstantFPValue(), EVT);
-    return DAG.getNode(ISD::FP_EXTEND, SDLoc(N), VT, Round);
+    SDLoc DL(N);
+    SDValue Round = DAG.getConstantFP(*N0CFP->getConstantFPValue(), DL, EVT);
+    return DAG.getNode(ISD::FP_EXTEND, DL, VT, Round);
   }
 
   return SDValue();
@@ -8161,6 +8723,11 @@ SDValue DAGCombiner::visitFP_EXTEND(SDNode *N) {
   if (isConstantFPBuildVectorOrConstantFP(N0))
     return DAG.getNode(ISD::FP_EXTEND, SDLoc(N), VT, N0);
 
+  // fold (fp_extend (fp16_to_fp op)) -> (fp16_to_fp op)
+  if (N0.getOpcode() == ISD::FP16_TO_FP &&
+      TLI.getOperationAction(ISD::FP16_TO_FP, VT) == TargetLowering::Legal)
+    return DAG.getNode(ISD::FP16_TO_FP, SDLoc(N), VT, N0.getOperand(0));
+
   // Turn fp_extend(fp_round(X, 1)) -> x since the fp_round doesn't affect the
   // value of X.
   if (N0.getOpcode() == ISD::FP_ROUND
@@ -8184,7 +8751,8 @@ SDValue DAGCombiner::visitFP_EXTEND(SDNode *N) {
     CombineTo(N, ExtLoad);
     CombineTo(N0.getNode(),
               DAG.getNode(ISD::FP_ROUND, SDLoc(N0),
-                          N0.getValueType(), ExtLoad, DAG.getIntPtrConstant(1)),
+                          N0.getValueType(), ExtLoad,
+                          DAG.getIntPtrConstant(1, SDLoc(N0))),
               ExtLoad.getValue(1));
     return SDValue(N, 0);   // Return N so it doesn't get rechecked!
   }
@@ -8256,15 +8824,17 @@ SDValue DAGCombiner::visitFNEG(SDNode *N) {
         // For a scalar, just generate 0x80...
         SignMask = APInt::getSignBit(IntVT.getSizeInBits());
       }
-      Int = DAG.getNode(ISD::XOR, SDLoc(N0), IntVT, Int,
-                        DAG.getConstant(SignMask, IntVT));
+      SDLoc DL0(N0);
+      Int = DAG.getNode(ISD::XOR, DL0, IntVT, Int,
+                        DAG.getConstant(SignMask, DL0, IntVT));
       AddToWorklist(Int.getNode());
       return DAG.getNode(ISD::BITCAST, SDLoc(N), VT, Int);
     }
   }
 
   // (fneg (fmul c, x)) -> (fmul -c, x)
-  if (N0.getOpcode() == ISD::FMUL) {
+  if (N0.getOpcode() == ISD::FMUL &&
+      (N0.getNode()->hasOneUse() || !TLI.isFNegFree(VT))) {
     ConstantFPSDNode *CFP1 = dyn_cast<ConstantFPSDNode>(N0.getOperand(1));
     if (CFP1) {
       APFloat CVal = CFP1->getValueAPF();
@@ -8290,7 +8860,7 @@ SDValue DAGCombiner::visitFMINNUM(SDNode *N) {
   if (N0CFP && N1CFP) {
     const APFloat &C0 = N0CFP->getValueAPF();
     const APFloat &C1 = N1CFP->getValueAPF();
-    return DAG.getConstantFP(minnum(C0, C1), N->getValueType(0));
+    return DAG.getConstantFP(minnum(C0, C1), SDLoc(N), N->getValueType(0));
   }
 
   if (N0CFP) {
@@ -8311,7 +8881,7 @@ SDValue DAGCombiner::visitFMAXNUM(SDNode *N) {
   if (N0CFP && N1CFP) {
     const APFloat &C0 = N0CFP->getValueAPF();
     const APFloat &C1 = N1CFP->getValueAPF();
-    return DAG.getConstantFP(maxnum(C0, C1), N->getValueType(0));
+    return DAG.getConstantFP(maxnum(C0, C1), SDLoc(N), N->getValueType(0));
   }
 
   if (N0CFP) {
@@ -8358,8 +8928,9 @@ SDValue DAGCombiner::visitFABS(SDNode *N) {
         // For a scalar, just generate 0x7f...
         SignMask = ~APInt::getSignBit(IntVT.getSizeInBits());
       }
-      Int = DAG.getNode(ISD::AND, SDLoc(N0), IntVT, Int,
-                        DAG.getConstant(SignMask, IntVT));
+      SDLoc DL(N0);
+      Int = DAG.getNode(ISD::AND, DL, IntVT, Int,
+                        DAG.getConstant(SignMask, DL, IntVT));
       AddToWorklist(Int.getNode());
       return DAG.getNode(ISD::BITCAST, SDLoc(N), N->getValueType(0), Int);
     }
@@ -8429,13 +9000,14 @@ SDValue DAGCombiner::visitBRCOND(SDNode *N) {
 
         if (AndConst.isPowerOf2() &&
             cast<ConstantSDNode>(Op1)->getAPIntValue()==AndConst.logBase2()) {
+          SDLoc DL(N);
           SDValue SetCC =
-            DAG.getSetCC(SDLoc(N),
+            DAG.getSetCC(DL,
                          getSetCCResultType(Op0.getValueType()),
-                         Op0, DAG.getConstant(0, Op0.getValueType()),
+                         Op0, DAG.getConstant(0, DL, Op0.getValueType()),
                          ISD::SETNE);
 
-          SDValue NewBRCond = DAG.getNode(ISD::BRCOND, SDLoc(N),
+          SDValue NewBRCond = DAG.getNode(ISD::BRCOND, DL,
                                           MVT::Other, Chain, SetCC, N2);
           // Don't add the new BRCond into the worklist or else SimplifySelectCC
           // will convert it back to (X & C1) >> C2.
@@ -8488,12 +9060,11 @@ SDValue DAGCombiner::visitBRCOND(SDNode *N) {
 
     if (Op0.getOpcode() != ISD::SETCC && Op1.getOpcode() != ISD::SETCC) {
       bool Equal = false;
-      if (ConstantSDNode *RHSCI = dyn_cast<ConstantSDNode>(Op0))
-        if (RHSCI->getAPIntValue() == 1 && Op0.hasOneUse() &&
-            Op0.getOpcode() == ISD::XOR) {
-          TheXor = Op0.getNode();
-          Equal = true;
-        }
+      if (isOneConstant(Op0) && Op0.hasOneUse() &&
+          Op0.getOpcode() == ISD::XOR) {
+        TheXor = Op0.getNode();
+        Equal = true;
+      }
 
       EVT SetCCVT = N1.getValueType();
       if (LegalTypes)
@@ -8548,14 +9119,18 @@ static bool canFoldInAddressingMode(SDNode *N, SDNode *Use,
                                     SelectionDAG &DAG,
                                     const TargetLowering &TLI) {
   EVT VT;
+  unsigned AS;
+
   if (LoadSDNode *LD  = dyn_cast<LoadSDNode>(Use)) {
     if (LD->isIndexed() || LD->getBasePtr().getNode() != N)
       return false;
-    VT = Use->getValueType(0);
+    VT = LD->getMemoryVT();
+    AS = LD->getAddressSpace();
   } else if (StoreSDNode *ST  = dyn_cast<StoreSDNode>(Use)) {
     if (ST->isIndexed() || ST->getBasePtr().getNode() != N)
       return false;
-    VT = ST->getValue().getValueType();
+    VT = ST->getMemoryVT();
+    AS = ST->getAddressSpace();
   } else
     return false;
 
@@ -8579,7 +9154,8 @@ static bool canFoldInAddressingMode(SDNode *N, SDNode *Use,
   } else
     return false;
 
-  return TLI.isLegalAddressingMode(AM, VT.getTypeForEVT(*DAG.getContext()));
+  return TLI.isLegalAddressingMode(DAG.getDataLayout(), AM,
+                                   VT.getTypeForEVT(*DAG.getContext()), AS);
 }
 
 /// Try turning a load/store into a pre-indexed load/store when the base
@@ -8638,8 +9214,7 @@ bool DAGCombiner::CombineToPreIndexedLoadStore(SDNode *N) {
   }
 
   // Don't create a indexed load / store with zero offset.
-  if (isa<ConstantSDNode>(Offset) &&
-      cast<ConstantSDNode>(Offset)->isNullValue())
+  if (isNullConstant(Offset))
     return false;
 
   // Try turning it into a pre-indexed load / store except when:
@@ -8667,24 +9242,25 @@ bool DAGCombiner::CombineToPreIndexedLoadStore(SDNode *N) {
   // a copy of the original base pointer.
   SmallVector<SDNode *, 16> OtherUses;
   if (isa<ConstantSDNode>(Offset))
-    for (SDNode *Use : BasePtr.getNode()->uses()) {
-      if (Use == Ptr.getNode())
+    for (SDNode::use_iterator UI = BasePtr.getNode()->use_begin(),
+                              UE = BasePtr.getNode()->use_end();
+         UI != UE; ++UI) {
+      SDUse &Use = UI.getUse();
+      // Skip the use that is Ptr and uses of other results from BasePtr's
+      // node (important for nodes that return multiple results).
+      if (Use.getUser() == Ptr.getNode() || Use != BasePtr)
         continue;
 
-      if (Use->isPredecessorOf(N))
+      if (Use.getUser()->isPredecessorOf(N))
         continue;
 
-      if (Use->getOpcode() != ISD::ADD && Use->getOpcode() != ISD::SUB) {
+      if (Use.getUser()->getOpcode() != ISD::ADD &&
+          Use.getUser()->getOpcode() != ISD::SUB) {
         OtherUses.clear();
         break;
       }
 
-      SDValue Op0 = Use->getOperand(0), Op1 = Use->getOperand(1);
-      if (Op1.getNode() == BasePtr.getNode())
-        std::swap(Op0, Op1);
-      assert(Op0.getNode() == BasePtr.getNode() &&
-             "Use of ADD/SUB but not an operand");
-
+      SDValue Op1 = Use.getUser()->getOperand((UI.getOperandNo() + 1) & 1);
       if (!isa<ConstantSDNode>(Op1)) {
         OtherUses.clear();
         break;
@@ -8696,7 +9272,7 @@ bool DAGCombiner::CombineToPreIndexedLoadStore(SDNode *N) {
         break;
       }
 
-      OtherUses.push_back(Use);
+      OtherUses.push_back(Use.getUser());
     }
 
   if (Swapped)
@@ -8789,12 +9365,14 @@ bool DAGCombiner::CombineToPreIndexedLoadStore(SDNode *N) {
     if (X1 * Y0 * Y1 < 0) CNV = CNV + Offset1;
     else CNV = CNV - Offset1;
 
+    SDLoc DL(OtherUses[i]);
+
     // We can now generate the new expression.
-    SDValue NewOp1 = DAG.getConstant(CNV, CN->getValueType(0));
+    SDValue NewOp1 = DAG.getConstant(CNV, DL, CN->getValueType(0));
     SDValue NewOp2 = Result.getValue(isLoad ? 1 : 0);
 
     SDValue NewUse = DAG.getNode(Opcode,
-                                 SDLoc(OtherUses[i]),
+                                 DL,
                                  OtherUses[i]->getValueType(0), NewOp1, NewOp2);
     DAG.ReplaceAllUsesOfValueWith(SDValue(OtherUses[i], 0), NewUse);
     deleteAndRecombine(OtherUses[i]);
@@ -8852,8 +9430,7 @@ bool DAGCombiner::CombineToPostIndexedLoadStore(SDNode *N) {
     ISD::MemIndexedMode AM = ISD::UNINDEXED;
     if (TLI.getPostIndexedAddressParts(N, Op, BasePtr, Offset, AM, DAG)) {
       // Don't create a indexed load / store with zero offset.
-      if (isa<ConstantSDNode>(Offset) &&
-          cast<ConstantSDNode>(Offset)->isNullValue())
+      if (isNullConstant(Offset))
         continue;
 
       // Try turning it into a post-indexed load / store except when
@@ -8943,7 +9520,7 @@ SDValue DAGCombiner::SplitIndexingFromLoad(LoadSDNode *LD) {
          "Cannot split out indexing using opaque target constants");
   if (Inc.getOpcode() == ISD::TargetConstant) {
     ConstantSDNode *ConstInc = cast<ConstantSDNode>(Inc);
-    Inc = DAG.getConstant(*ConstInc->getConstantIntValue(),
+    Inc = DAG.getConstant(*ConstInc->getConstantIntValue(), SDLoc(Inc),
                           ConstInc->getValueType(0));
   }
 
@@ -9303,8 +9880,7 @@ struct LoadedSlice {
   /// \pre DAG != nullptr.
   uint64_t getOffsetFromBase() const {
     assert(DAG && "Missing context.");
-    bool IsBigEndian =
-        DAG->getTargetLoweringInfo().getDataLayout()->isBigEndian();
+    bool IsBigEndian = DAG->getDataLayout().isBigEndian();
     assert(!(Shift & 0x7) && "Shifts not aligned on Bytes are not supported.");
     uint64_t Offset = Shift / 8;
     unsigned TySizeInBytes = Origin->getValueSizeInBits(0) / 8;
@@ -9336,8 +9912,9 @@ struct LoadedSlice {
     if (Offset) {
       // BaseAddr = BaseAddr + Offset.
       EVT ArithType = BaseAddr.getValueType();
-      BaseAddr = DAG->getNode(ISD::ADD, SDLoc(Origin), ArithType, BaseAddr,
-                              DAG->getConstant(Offset, ArithType));
+      SDLoc DL(Origin);
+      BaseAddr = DAG->getNode(ISD::ADD, DL, ArithType, BaseAddr,
+                              DAG->getConstant(Offset, DL, ArithType));
     }
 
     // Create the type of the loaded slice according to its size.
@@ -9386,7 +9963,7 @@ struct LoadedSlice {
 
     // Check if it will be merged with the load.
     // 1. Check the alignment constraint.
-    unsigned RequiredAlignment = TLI.getDataLayout()->getABITypeAlignment(
+    unsigned RequiredAlignment = DAG->getDataLayout().getABITypeAlignment(
         ResVT.getTypeForEVT(*DAG->getContext()));
 
     if (RequiredAlignment > getAlignment())
@@ -9667,8 +10244,8 @@ CheckForMaskedLoad(SDValue V, SDValue Ptr, SDValue Chain) {
     return Result; // Fail.
   else {
     bool isOk = false;
-    for (unsigned i = 0, e = Chain->getNumOperands(); i != e; ++i)
-      if (Chain->getOperand(i).getNode() == LD) {
+    for (const SDValue &ChainOp : Chain->op_values())
+      if (ChainOp.getNode() == LD) {
         isOk = true;
         break;
       }
@@ -9743,24 +10320,27 @@ ShrinkLoadReplaceStoreWithStore(const std::pair<unsigned, unsigned> &MaskInfo,
 
   // Okay, we can do this!  Replace the 'St' store with a store of IVal that is
   // shifted by ByteShift and truncated down to NumBytes.
-  if (ByteShift)
-    IVal = DAG.getNode(ISD::SRL, SDLoc(IVal), IVal.getValueType(), IVal,
-                       DAG.getConstant(ByteShift*8,
+  if (ByteShift) {
+    SDLoc DL(IVal);
+    IVal = DAG.getNode(ISD::SRL, DL, IVal.getValueType(), IVal,
+                       DAG.getConstant(ByteShift*8, DL,
                                     DC->getShiftAmountTy(IVal.getValueType())));
+  }
 
   // Figure out the offset for the store and the alignment of the access.
   unsigned StOffset;
   unsigned NewAlign = St->getAlignment();
 
-  if (DAG.getTargetLoweringInfo().isLittleEndian())
+  if (DAG.getDataLayout().isLittleEndian())
     StOffset = ByteShift;
   else
     StOffset = IVal.getValueType().getStoreSize() - ByteShift - NumBytes;
 
   SDValue Ptr = St->getBasePtr();
   if (StOffset) {
-    Ptr = DAG.getNode(ISD::ADD, SDLoc(IVal), Ptr.getValueType(),
-                      Ptr, DAG.getConstant(StOffset, Ptr.getValueType()));
+    SDLoc DL(IVal);
+    Ptr = DAG.getNode(ISD::ADD, DL, Ptr.getValueType(),
+                      Ptr, DAG.getConstant(StOffset, DL, Ptr.getValueType()));
     NewAlign = MinAlign(NewAlign, StOffset);
   }
 
@@ -9864,17 +10444,18 @@ SDValue DAGCombiner::ReduceLoadOpStoreWidth(SDNode *N) {
       uint64_t PtrOff = ShAmt / 8;
       // For big endian targets, we need to adjust the offset to the pointer to
       // load the correct bytes.
-      if (TLI.isBigEndian())
+      if (DAG.getDataLayout().isBigEndian())
         PtrOff = (BitWidth + 7 - NewBW) / 8 - PtrOff;
 
       unsigned NewAlign = MinAlign(LD->getAlignment(), PtrOff);
       Type *NewVTTy = NewVT.getTypeForEVT(*DAG.getContext());
-      if (NewAlign < TLI.getDataLayout()->getABITypeAlignment(NewVTTy))
+      if (NewAlign < DAG.getDataLayout().getABITypeAlignment(NewVTTy))
         return SDValue();
 
       SDValue NewPtr = DAG.getNode(ISD::ADD, SDLoc(LD),
                                    Ptr.getValueType(), Ptr,
-                                   DAG.getConstant(PtrOff, Ptr.getValueType()));
+                                   DAG.getConstant(PtrOff, SDLoc(LD),
+                                                   Ptr.getValueType()));
       SDValue NewLD = DAG.getLoad(NewVT, SDLoc(N0),
                                   LD->getChain(), NewPtr,
                                   LD->getPointerInfo().getWithOffset(PtrOff),
@@ -9882,7 +10463,8 @@ SDValue DAGCombiner::ReduceLoadOpStoreWidth(SDNode *N) {
                                   LD->isInvariant(), NewAlign,
                                   LD->getAAInfo());
       SDValue NewVal = DAG.getNode(Opc, SDLoc(Value), NewVT, NewLD,
-                                   DAG.getConstant(NewImm, NewVT));
+                                   DAG.getConstant(NewImm, SDLoc(Value),
+                                                   NewVT));
       SDValue NewST = DAG.getStore(Chain, SDLoc(N),
                                    NewVal, NewPtr,
                                    ST->getPointerInfo().getWithOffset(PtrOff),
@@ -9931,7 +10513,7 @@ SDValue DAGCombiner::TransformFPLoadStorePair(SDNode *N) {
     unsigned LDAlign = LD->getAlignment();
     unsigned STAlign = ST->getAlignment();
     Type *IntVTTy = IntVT.getTypeForEVT(*DAG.getContext());
-    unsigned ABIAlign = TLI.getDataLayout()->getABITypeAlignment(IntVTTy);
+    unsigned ABIAlign = DAG.getDataLayout().getABITypeAlignment(IntVTTy);
     if (LDAlign < ABIAlign || STAlign < ABIAlign)
       return SDValue();
 
@@ -10046,6 +10628,18 @@ struct BaseIndexOffset {
 };
 } // namespace
 
+SDValue DAGCombiner::getMergedConstantVectorStore(SelectionDAG &DAG,
+                                                  SDLoc SL,
+                                                  ArrayRef<MemOpLink> Stores,
+                                                  EVT Ty) const {
+  SmallVector<SDValue, 8> BuildVector;
+
+  for (unsigned I = 0, E = Ty.getVectorNumElements(); I != E; ++I)
+    BuildVector.push_back(cast<StoreSDNode>(Stores[I].MemNode)->getValue());
+
+  return DAG.getNode(ISD::BUILD_VECTOR, SL, Ty, BuildVector);
+}
+
 bool DAGCombiner::MergeStoresOfConstantsOrVecElts(
                   SmallVectorImpl<MemOpLink> &StoreNodes, EVT MemVT,
                   unsigned NumElem, bool IsConstantSrc, bool UseVector) {
@@ -10076,12 +10670,7 @@ bool DAGCombiner::MergeStoresOfConstantsOrVecElts(
     EVT Ty = EVT::getVectorVT(*DAG.getContext(), MemVT, NumElem);
     assert(TLI.isTypeLegal(Ty) && "Illegal vector store");
     if (IsConstantSrc) {
-      // A vector store with a constant source implies that the constant is
-      // zero; we only handle merging stores of constant zeros because the zero
-      // can be materialized without a load.
-      // It may be beneficial to loosen this restriction to allow non-zero
-      // store merging.
-      StoredVal = DAG.getConstant(0, Ty);
+      StoredVal = getMergedConstantVectorStore(DAG, DL, StoreNodes, Ty);
     } else {
       SmallVector<SDValue, 8> Ops;
       for (unsigned i = 0; i < NumElem ; ++i) {
@@ -10101,29 +10690,29 @@ bool DAGCombiner::MergeStoresOfConstantsOrVecElts(
     // elements, so this path implies a store of constants.
     assert(IsConstantSrc && "Merged vector elements should use vector store");
 
-    unsigned StoreBW = NumElem * ElementSizeBytes * 8;
-    APInt StoreInt(StoreBW, 0);
+    unsigned SizeInBits = NumElem * ElementSizeBytes * 8;
+    APInt StoreInt(SizeInBits, 0);
 
     // Construct a single integer constant which is made of the smaller
     // constant inputs.
-    bool IsLE = TLI.isLittleEndian();
+    bool IsLE = DAG.getDataLayout().isLittleEndian();
     for (unsigned i = 0; i < NumElem ; ++i) {
       unsigned Idx = IsLE ? (NumElem - 1 - i) : i;
       StoreSDNode *St  = cast<StoreSDNode>(StoreNodes[Idx].MemNode);
       SDValue Val = St->getValue();
-      StoreInt <<= ElementSizeBytes*8;
+      StoreInt <<= ElementSizeBytes * 8;
       if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Val)) {
-        StoreInt |= C->getAPIntValue().zext(StoreBW);
+        StoreInt |= C->getAPIntValue().zext(SizeInBits);
       } else if (ConstantFPSDNode *C = dyn_cast<ConstantFPSDNode>(Val)) {
-        StoreInt |= C->getValueAPF().bitcastToAPInt().zext(StoreBW);
+        StoreInt |= C->getValueAPF().bitcastToAPInt().zext(SizeInBits);
       } else {
         llvm_unreachable("Invalid constant element type");
       }
     }
 
     // Create the new Load and Store operations.
-    EVT StoreTy = EVT::getIntegerVT(*DAG.getContext(), StoreBW);
-    StoredVal = DAG.getConstant(StoreInt, StoreTy);
+    EVT StoreTy = EVT::getIntegerVT(*DAG.getContext(), SizeInBits);
+    StoredVal = DAG.getConstant(StoreInt, DL, StoreTy);
   }
 
   SDValue NewStore = DAG.getStore(LatestOp->getChain(), DL, StoredVal,
@@ -10157,58 +10746,25 @@ bool DAGCombiner::MergeStoresOfConstantsOrVecElts(
   return true;
 }
 
-bool DAGCombiner::MergeConsecutiveStores(StoreSDNode* St) {
-  if (OptLevel == CodeGenOpt::None)
-    return false;
-
-  EVT MemVT = St->getMemoryVT();
-  int64_t ElementSizeBytes = MemVT.getSizeInBits()/8;
-  bool NoVectors = DAG.getMachineFunction().getFunction()->hasFnAttribute(
-      Attribute::NoImplicitFloat);
-
-  // Don't merge vectors into wider inputs.
-  if (MemVT.isVector() || !MemVT.isSimple())
-    return false;
-
-  // Perform an early exit check. Do not bother looking at stored values that
-  // are not constants, loads, or extracted vector elements.
-  SDValue StoredVal = St->getValue();
-  bool IsLoadSrc = isa<LoadSDNode>(StoredVal);
-  bool IsConstantSrc = isa<ConstantSDNode>(StoredVal) ||
-                       isa<ConstantFPSDNode>(StoredVal);
-  bool IsExtractVecEltSrc = (StoredVal.getOpcode() == ISD::EXTRACT_VECTOR_ELT);
-
-  if (!IsConstantSrc && !IsLoadSrc && !IsExtractVecEltSrc)
-    return false;
-
-  // Only look at ends of store sequences.
-  SDValue Chain = SDValue(St, 0);
-  if (Chain->hasOneUse() && Chain->use_begin()->getOpcode() == ISD::STORE)
-    return false;
-
+void DAGCombiner::getStoreMergeAndAliasCandidates(
+    StoreSDNode* St, SmallVectorImpl<MemOpLink> &StoreNodes,
+    SmallVectorImpl<LSBaseSDNode*> &AliasLoadNodes) {
   // This holds the base pointer, index, and the offset in bytes from the base
   // pointer.
   BaseIndexOffset BasePtr = BaseIndexOffset::match(St->getBasePtr());
 
   // We must have a base and an offset.
   if (!BasePtr.Base.getNode())
-    return false;
+    return;
 
   // Do not handle stores to undef base pointers.
   if (BasePtr.Base.getOpcode() == ISD::UNDEF)
-    return false;
-
-  // Save the LoadSDNodes that we find in the chain.
-  // We need to make sure that these nodes do not interfere with
-  // any of the store nodes.
-  SmallVector<LSBaseSDNode*, 8> AliasLoadNodes;
-
-  // Save the StoreSDNodes that we find in the chain.
-  SmallVector<MemOpLink, 8> StoreNodes;
+    return;
 
   // Walk up the chain and look for nodes with offsets from the same
   // base pointer. Stop when reaching an instruction with a different kind
   // or instruction which has a different base pointer.
+  EVT MemVT = St->getMemoryVT();
   unsigned Seq = 0;
   StoreSDNode *Index = St;
   while (Index) {
@@ -10223,10 +10779,6 @@ bool DAGCombiner::MergeConsecutiveStores(StoreSDNode* St) {
     if (!Ptr.equalBaseIndex(BasePtr))
       break;
 
-    // Check that the alignment is the same.
-    if (Index->getAlignment() != St->getAlignment())
-      break;
-
     // The memory operands must not be volatile.
     if (Index->isVolatile() || Index->isIndexed())
       break;
@@ -10240,11 +10792,6 @@ bool DAGCombiner::MergeConsecutiveStores(StoreSDNode* St) {
     if (Index->getMemoryVT() != MemVT)
       break;
 
-    // We do not allow unaligned stores because we want to prevent overriding
-    // stores.
-    if (Index->getAlignment()*8 != MemVT.getSizeInBits())
-      break;
-
     // We found a potential memory operand to merge.
     StoreNodes.push_back(MemOpLink(Index, Ptr.Offset, Seq++));
 
@@ -10274,7 +10821,51 @@ bool DAGCombiner::MergeConsecutiveStores(StoreSDNode* St) {
       }
     }
   }
+}
+
+bool DAGCombiner::MergeConsecutiveStores(StoreSDNode* St) {
+  if (OptLevel == CodeGenOpt::None)
+    return false;
+
+  EVT MemVT = St->getMemoryVT();
+  int64_t ElementSizeBytes = MemVT.getSizeInBits() / 8;
+  bool NoVectors = DAG.getMachineFunction().getFunction()->hasFnAttribute(
+      Attribute::NoImplicitFloat);
+
+  // This function cannot currently deal with non-byte-sized memory sizes.
+  if (ElementSizeBytes * 8 != MemVT.getSizeInBits())
+    return false;
+
+  // Don't merge vectors into wider inputs.
+  if (MemVT.isVector() || !MemVT.isSimple())
+    return false;
+
+  // Perform an early exit check. Do not bother looking at stored values that
+  // are not constants, loads, or extracted vector elements.
+  SDValue StoredVal = St->getValue();
+  bool IsLoadSrc = isa<LoadSDNode>(StoredVal);
+  bool IsConstantSrc = isa<ConstantSDNode>(StoredVal) ||
+                       isa<ConstantFPSDNode>(StoredVal);
+  bool IsExtractVecEltSrc = (StoredVal.getOpcode() == ISD::EXTRACT_VECTOR_ELT);
+
+  if (!IsConstantSrc && !IsLoadSrc && !IsExtractVecEltSrc)
+    return false;
 
+  // Only look at ends of store sequences.
+  SDValue Chain = SDValue(St, 0);
+  if (Chain->hasOneUse() && Chain->use_begin()->getOpcode() == ISD::STORE)
+    return false;
+
+  // Save the LoadSDNodes that we find in the chain.
+  // We need to make sure that these nodes do not interfere with
+  // any of the store nodes.
+  SmallVector<LSBaseSDNode*, 8> AliasLoadNodes;
+  
+  // Save the StoreSDNodes that we find in the chain.
+  SmallVector<MemOpLink, 8> StoreNodes;
+
+  getStoreMergeAndAliasCandidates(St, StoreNodes, AliasLoadNodes);
+  
   // Check if there is anything to merge.
   if (StoreNodes.size() < 2)
     return false;
@@ -10318,6 +10909,10 @@ bool DAGCombiner::MergeConsecutiveStores(StoreSDNode* St) {
 
   // The node with the lowest store address.
   LSBaseSDNode *FirstInChain = StoreNodes[0].MemNode;
+  unsigned FirstStoreAS = FirstInChain->getAddressSpace();
+  unsigned FirstStoreAlign = FirstInChain->getAlignment();
+  LLVMContext &Context = *DAG.getContext();
+  const DataLayout &DL = DAG.getDataLayout();
 
   // Store the constants into memory as one consecutive store.
   if (IsConstantSrc) {
@@ -10338,29 +10933,44 @@ bool DAGCombiner::MergeConsecutiveStores(StoreSDNode* St) {
       }
 
       // Find a legal type for the constant store.
-      unsigned StoreBW = (i+1) * ElementSizeBytes * 8;
-      EVT StoreTy = EVT::getIntegerVT(*DAG.getContext(), StoreBW);
-      if (TLI.isTypeLegal(StoreTy))
+      unsigned SizeInBits = (i+1) * ElementSizeBytes * 8;
+      EVT StoreTy = EVT::getIntegerVT(Context, SizeInBits);
+      if (TLI.isTypeLegal(StoreTy) &&
+          TLI.allowsMemoryAccess(Context, DL, StoreTy, FirstStoreAS,
+                                 FirstStoreAlign)) {
         LastLegalType = i+1;
       // Or check whether a truncstore is legal.
-      else if (TLI.getTypeAction(*DAG.getContext(), StoreTy) ==
-               TargetLowering::TypePromoteInteger) {
+      } else if (TLI.getTypeAction(Context, StoreTy) ==
+                 TargetLowering::TypePromoteInteger) {
         EVT LegalizedStoredValueTy =
-          TLI.getTypeToTransformTo(*DAG.getContext(), StoredVal.getValueType());
-        if (TLI.isTruncStoreLegal(LegalizedStoredValueTy, StoreTy))
-          LastLegalType = i+1;
+          TLI.getTypeToTransformTo(Context, StoredVal.getValueType());
+        if (TLI.isTruncStoreLegal(LegalizedStoredValueTy, StoreTy) &&
+            TLI.allowsMemoryAccess(Context, DL, LegalizedStoredValueTy,
+                                   FirstStoreAS, FirstStoreAlign)) {
+          LastLegalType = i + 1;
+        }
       }
 
       // Find a legal type for the vector store.
-      EVT Ty = EVT::getVectorVT(*DAG.getContext(), MemVT, i+1);
-      if (TLI.isTypeLegal(Ty))
+      EVT Ty = EVT::getVectorVT(Context, MemVT, i+1);
+      if (TLI.isTypeLegal(Ty) &&
+          TLI.allowsMemoryAccess(Context, DL, Ty, FirstStoreAS,
+                                 FirstStoreAlign)) {
         LastLegalVectorType = i + 1;
+      }
     }
 
-    // We only use vectors if the constant is known to be zero and the
-    // function is not marked with the noimplicitfloat attribute.
-    if (NonZero || NoVectors)
+
+    // We only use vectors if the constant is known to be zero or the target
+    // allows it and the function is not marked with the noimplicitfloat
+    // attribute.
+    if (NoVectors) {
       LastLegalVectorType = 0;
+    } else if (NonZero && !TLI.storeOfVectorConstantIsCheap(MemVT,
+                                                            LastLegalVectorType,
+                                                            FirstStoreAS)) {
+      LastLegalVectorType = 0;
+    }
 
     // Check if we found a legal integer type to store.
     if (LastLegalType == 0 && LastLegalVectorType == 0)
@@ -10389,8 +10999,10 @@ bool DAGCombiner::MergeConsecutiveStores(StoreSDNode* St) {
         return false;
 
       // Find a legal type for the vector store.
-      EVT Ty = EVT::getVectorVT(*DAG.getContext(), MemVT, i+1);
-      if (TLI.isTypeLegal(Ty))
+      EVT Ty = EVT::getVectorVT(Context, MemVT, i+1);
+      if (TLI.isTypeLegal(Ty) &&
+          TLI.allowsMemoryAccess(Context, DL, Ty, FirstStoreAS,
+                                 FirstStoreAlign))
         NumElem = i + 1;
     }
 
@@ -10417,10 +11029,6 @@ bool DAGCombiner::MergeConsecutiveStores(StoreSDNode* St) {
     if (!Ld->hasNUsesOfValue(1, 0))
       break;
 
-    // Check that the alignment is the same as the stores.
-    if (Ld->getAlignment() != St->getAlignment())
-      break;
-
     // The memory operands must not be volatile.
     if (Ld->isVolatile() || Ld->isIndexed())
       break;
@@ -10458,6 +11066,10 @@ bool DAGCombiner::MergeConsecutiveStores(StoreSDNode* St) {
       St->getAlignment() >= RequiredAlignment)
     return false;
 
+  LoadSDNode *FirstLoad = cast<LoadSDNode>(LoadNodes[0].MemNode);
+  unsigned FirstLoadAS = FirstLoad->getAddressSpace();
+  unsigned FirstLoadAlign = FirstLoad->getAlignment();
+
   // Scan the memory operations on the chain and find the first non-consecutive
   // load memory address. These variables hold the index in the store node
   // array.
@@ -10466,7 +11078,7 @@ bool DAGCombiner::MergeConsecutiveStores(StoreSDNode* St) {
   unsigned LastLegalVectorType = 0;
   unsigned LastLegalIntegerType = 0;
   StartAddress = LoadNodes[0].OffsetFromBase;
-  SDValue FirstChain = LoadNodes[0].MemNode->getChain();
+  SDValue FirstChain = FirstLoad->getChain();
   for (unsigned i = 1; i < LoadNodes.size(); ++i) {
     // All loads much share the same chain.
     if (LoadNodes[i].MemNode->getChain() != FirstChain)
@@ -10478,24 +11090,37 @@ bool DAGCombiner::MergeConsecutiveStores(StoreSDNode* St) {
     LastConsecutiveLoad = i;
 
     // Find a legal type for the vector store.
-    EVT StoreTy = EVT::getVectorVT(*DAG.getContext(), MemVT, i+1);
-    if (TLI.isTypeLegal(StoreTy))
+    EVT StoreTy = EVT::getVectorVT(Context, MemVT, i+1);
+    if (TLI.isTypeLegal(StoreTy) &&
+        TLI.allowsMemoryAccess(Context, DL, StoreTy, FirstStoreAS,
+                               FirstStoreAlign) &&
+        TLI.allowsMemoryAccess(Context, DL, StoreTy, FirstLoadAS,
+                               FirstLoadAlign)) {
       LastLegalVectorType = i + 1;
+    }
 
     // Find a legal type for the integer store.
-    unsigned StoreBW = (i+1) * ElementSizeBytes * 8;
-    StoreTy = EVT::getIntegerVT(*DAG.getContext(), StoreBW);
-    if (TLI.isTypeLegal(StoreTy))
+    unsigned SizeInBits = (i+1) * ElementSizeBytes * 8;
+    StoreTy = EVT::getIntegerVT(Context, SizeInBits);
+    if (TLI.isTypeLegal(StoreTy) &&
+        TLI.allowsMemoryAccess(Context, DL, StoreTy, FirstStoreAS,
+                               FirstStoreAlign) &&
+        TLI.allowsMemoryAccess(Context, DL, StoreTy, FirstLoadAS,
+                               FirstLoadAlign))
       LastLegalIntegerType = i + 1;
     // Or check whether a truncstore and extload is legal.
-    else if (TLI.getTypeAction(*DAG.getContext(), StoreTy) ==
+    else if (TLI.getTypeAction(Context, StoreTy) ==
              TargetLowering::TypePromoteInteger) {
       EVT LegalizedStoredValueTy =
-        TLI.getTypeToTransformTo(*DAG.getContext(), StoreTy);
+        TLI.getTypeToTransformTo(Context, StoreTy);
       if (TLI.isTruncStoreLegal(LegalizedStoredValueTy, StoreTy) &&
           TLI.isLoadExtLegal(ISD::ZEXTLOAD, LegalizedStoredValueTy, StoreTy) &&
           TLI.isLoadExtLegal(ISD::SEXTLOAD, LegalizedStoredValueTy, StoreTy) &&
-          TLI.isLoadExtLegal(ISD::EXTLOAD, LegalizedStoredValueTy, StoreTy))
+          TLI.isLoadExtLegal(ISD::EXTLOAD, LegalizedStoredValueTy, StoreTy) &&
+          TLI.allowsMemoryAccess(Context, DL, LegalizedStoredValueTy,
+                                 FirstStoreAS, FirstStoreAlign) &&
+          TLI.allowsMemoryAccess(Context, DL, LegalizedStoredValueTy,
+                                 FirstLoadAS, FirstLoadAlign))
         LastLegalIntegerType = i+1;
     }
   }
@@ -10530,27 +11155,22 @@ bool DAGCombiner::MergeConsecutiveStores(StoreSDNode* St) {
   // to memory.
   EVT JointMemOpVT;
   if (UseVectorTy) {
-    JointMemOpVT = EVT::getVectorVT(*DAG.getContext(), MemVT, NumElem);
+    JointMemOpVT = EVT::getVectorVT(Context, MemVT, NumElem);
   } else {
-    unsigned StoreBW = NumElem * ElementSizeBytes * 8;
-    JointMemOpVT = EVT::getIntegerVT(*DAG.getContext(), StoreBW);
+    unsigned SizeInBits = NumElem * ElementSizeBytes * 8;
+    JointMemOpVT = EVT::getIntegerVT(Context, SizeInBits);
   }
 
   SDLoc LoadDL(LoadNodes[0].MemNode);
   SDLoc StoreDL(StoreNodes[0].MemNode);
 
-  LoadSDNode *FirstLoad = cast<LoadSDNode>(LoadNodes[0].MemNode);
-  SDValue NewLoad = DAG.getLoad(JointMemOpVT, LoadDL,
-                                FirstLoad->getChain(),
-                                FirstLoad->getBasePtr(),
-                                FirstLoad->getPointerInfo(),
-                                false, false, false,
-                                FirstLoad->getAlignment());
-
-  SDValue NewStore = DAG.getStore(LatestOp->getChain(), StoreDL, NewLoad,
-                                  FirstInChain->getBasePtr(),
-                                  FirstInChain->getPointerInfo(), false, false,
-                                  FirstInChain->getAlignment());
+  SDValue NewLoad = DAG.getLoad(
+      JointMemOpVT, LoadDL, FirstLoad->getChain(), FirstLoad->getBasePtr(),
+      FirstLoad->getPointerInfo(), false, false, false, FirstLoadAlign);
+
+  SDValue NewStore = DAG.getStore(
+      LatestOp->getChain(), StoreDL, NewLoad, FirstInChain->getBasePtr(),
+      FirstInChain->getPointerInfo(), false, false, FirstStoreAlign);
 
   // Replace one of the loads with the new load.
   LoadSDNode *Ld = cast<LoadSDNode>(LoadNodes[0].MemNode);
@@ -10592,8 +11212,8 @@ SDValue DAGCombiner::visitSTORE(SDNode *N) {
       ST->isUnindexed()) {
     unsigned OrigAlign = ST->getAlignment();
     EVT SVT = Value.getOperand(0).getValueType();
-    unsigned Align = TLI.getDataLayout()->
-      getABITypeAlignment(SVT.getTypeForEVT(*DAG.getContext()));
+    unsigned Align = DAG.getDataLayout().getABITypeAlignment(
+        SVT.getTypeForEVT(*DAG.getContext()));
     if (Align <= OrigAlign &&
         ((!LegalOperations && !ST->isVolatile()) ||
          TLI.isOperationLegalOrCustom(ISD::STORE, SVT)))
@@ -10625,8 +11245,10 @@ SDValue DAGCombiner::visitSTORE(SDNode *N) {
       case MVT::f32:
         if ((isTypeLegal(MVT::i32) && !LegalOperations && !ST->isVolatile()) ||
             TLI.isOperationLegalOrCustom(ISD::STORE, MVT::i32)) {
+          ;
           Tmp = DAG.getConstant((uint32_t)CFP->getValueAPF().
-                              bitcastToAPInt().getZExtValue(), MVT::i32);
+                              bitcastToAPInt().getZExtValue(), SDLoc(CFP),
+                              MVT::i32);
           return DAG.getStore(Chain, SDLoc(N), Tmp,
                               Ptr, ST->getMemOperand());
         }
@@ -10635,8 +11257,9 @@ SDValue DAGCombiner::visitSTORE(SDNode *N) {
         if ((TLI.isTypeLegal(MVT::i64) && !LegalOperations &&
              !ST->isVolatile()) ||
             TLI.isOperationLegalOrCustom(ISD::STORE, MVT::i64)) {
+          ;
           Tmp = DAG.getConstant(CFP->getValueAPF().bitcastToAPInt().
-                                getZExtValue(), MVT::i64);
+                                getZExtValue(), SDLoc(CFP), MVT::i64);
           return DAG.getStore(Chain, SDLoc(N), Tmp,
                               Ptr, ST->getMemOperand());
         }
@@ -10647,27 +11270,30 @@ SDValue DAGCombiner::visitSTORE(SDNode *N) {
           // argument passing.  Since this is so common, custom legalize the
           // 64-bit integer store into two 32-bit stores.
           uint64_t Val = CFP->getValueAPF().bitcastToAPInt().getZExtValue();
-          SDValue Lo = DAG.getConstant(Val & 0xFFFFFFFF, MVT::i32);
-          SDValue Hi = DAG.getConstant(Val >> 32, MVT::i32);
-          if (TLI.isBigEndian()) std::swap(Lo, Hi);
+          SDValue Lo = DAG.getConstant(Val & 0xFFFFFFFF, SDLoc(CFP), MVT::i32);
+          SDValue Hi = DAG.getConstant(Val >> 32, SDLoc(CFP), MVT::i32);
+          if (DAG.getDataLayout().isBigEndian())
+            std::swap(Lo, Hi);
 
           unsigned Alignment = ST->getAlignment();
           bool isVolatile = ST->isVolatile();
           bool isNonTemporal = ST->isNonTemporal();
           AAMDNodes AAInfo = ST->getAAInfo();
 
+          SDLoc DL(N);
+
           SDValue St0 = DAG.getStore(Chain, SDLoc(ST), Lo,
                                      Ptr, ST->getPointerInfo(),
                                      isVolatile, isNonTemporal,
                                      ST->getAlignment(), AAInfo);
-          Ptr = DAG.getNode(ISD::ADD, SDLoc(N), Ptr.getValueType(), Ptr,
-                            DAG.getConstant(4, Ptr.getValueType()));
+          Ptr = DAG.getNode(ISD::ADD, DL, Ptr.getValueType(), Ptr,
+                            DAG.getConstant(4, DL, Ptr.getValueType()));
           Alignment = MinAlign(Alignment, 4U);
           SDValue St1 = DAG.getStore(Chain, SDLoc(ST), Hi,
                                      Ptr, ST->getPointerInfo().getWithOffset(4),
                                      isVolatile, isNonTemporal,
                                      Alignment, AAInfo);
-          return DAG.getNode(ISD::TokenFactor, SDLoc(N), MVT::Other,
+          return DAG.getNode(ISD::TokenFactor, DL, MVT::Other,
                              St0, St1);
         }
 
@@ -10896,7 +11522,7 @@ SDValue DAGCombiner::ReplaceExtractVectorEltOfLoadWithNarrowedLoad(
   EVT ResultVT = EVE->getValueType(0);
   EVT VecEltVT = InVecVT.getVectorElementType();
   unsigned Align = OriginalLoad->getAlignment();
-  unsigned NewAlign = TLI.getDataLayout()->getABITypeAlignment(
+  unsigned NewAlign = DAG.getDataLayout().getABITypeAlignment(
       VecEltVT.getTypeForEVT(*DAG.getContext()));
 
   if (NewAlign > Align || !TLI.isOperationLegalOrCustom(ISD::LOAD, VecEltVT))
@@ -10908,24 +11534,20 @@ SDValue DAGCombiner::ReplaceExtractVectorEltOfLoadWithNarrowedLoad(
   SDValue Offset;
   EVT PtrType = NewPtr.getValueType();
   MachinePointerInfo MPI;
+  SDLoc DL(EVE);
   if (auto *ConstEltNo = dyn_cast<ConstantSDNode>(EltNo)) {
     int Elt = ConstEltNo->getZExtValue();
     unsigned PtrOff = VecEltVT.getSizeInBits() * Elt / 8;
-    if (TLI.isBigEndian())
-      PtrOff = InVecVT.getSizeInBits() / 8 - PtrOff;
-    Offset = DAG.getConstant(PtrOff, PtrType);
+    Offset = DAG.getConstant(PtrOff, DL, PtrType);
     MPI = OriginalLoad->getPointerInfo().getWithOffset(PtrOff);
   } else {
+    Offset = DAG.getZExtOrTrunc(EltNo, DL, PtrType);
     Offset = DAG.getNode(
-        ISD::MUL, SDLoc(EVE), EltNo.getValueType(), EltNo,
-        DAG.getConstant(VecEltVT.getStoreSize(), EltNo.getValueType()));
-    if (TLI.isBigEndian())
-      Offset = DAG.getNode(
-          ISD::SUB, SDLoc(EVE), EltNo.getValueType(),
-          DAG.getConstant(InVecVT.getStoreSize(), EltNo.getValueType()), Offset);
+        ISD::MUL, DL, PtrType, Offset,
+        DAG.getConstant(VecEltVT.getStoreSize(), DL, PtrType));
     MPI = OriginalLoad->getPointerInfo();
   }
-  NewPtr = DAG.getNode(ISD::ADD, SDLoc(EVE), PtrType, NewPtr, Offset);
+  NewPtr = DAG.getNode(ISD::ADD, DL, PtrType, NewPtr, Offset);
 
   // The replacement we need to do here is a little tricky: we need to
   // replace an extractelement of a load with a load.
@@ -11034,9 +11656,9 @@ SDValue DAGCombiner::visitEXTRACT_VECTOR_ELT(SDNode *N) {
     // scalar_to_vector here as well.
 
     if (!LegalOperations) {
-      EVT IndexTy = TLI.getVectorIdxTy();
-      return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SDLoc(N), NVT,
-                         SVInVec, DAG.getConstant(OrigElt, IndexTy));
+      EVT IndexTy = TLI.getVectorIdxTy(DAG.getDataLayout());
+      return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SDLoc(N), NVT, SVInVec,
+                         DAG.getConstant(OrigElt, SDLoc(SVOp), IndexTy));
     }
   }
 
@@ -11125,7 +11747,7 @@ SDValue DAGCombiner::visitEXTRACT_VECTOR_ELT(SDNode *N) {
       if (ISD::isNormalLoad(InVec.getNode())) {
         LN0 = cast<LoadSDNode>(InVec);
         Elt = (Idx < (int)NumElems) ? Idx : Idx - (int)NumElems;
-        EltNo = DAG.getConstant(Elt, EltNo.getValueType());
+        EltNo = DAG.getConstant(Elt, SDLoc(EltNo), EltNo.getValueType());
       }
     }
 
@@ -11211,11 +11833,11 @@ SDValue DAGCombiner::reduceBuildVecExtToExtBuildVec(SDNode *N) {
   if (!ValidTypes)
     return SDValue();
 
-  bool isLE = TLI.isLittleEndian();
+  bool isLE = DAG.getDataLayout().isLittleEndian();
   unsigned ElemRatio = OutScalarTy.getSizeInBits()/SourceType.getSizeInBits();
   assert(ElemRatio > 1 && "Invalid element size ratio");
   SDValue Filler = AllAnyExt ? DAG.getUNDEF(SourceType):
-                               DAG.getConstant(0, SourceType);
+                               DAG.getConstant(0, SDLoc(N), SourceType);
 
   unsigned NewBVElems = ElemRatio * VT.getVectorNumElements();
   SmallVector<SDValue, 8> Ops(NewBVElems, Filler);
@@ -11360,10 +11982,7 @@ SDValue DAGCombiner::visitBUILD_VECTOR(SDNode *N) {
     if (Op.getOpcode() == ISD::UNDEF) continue;
 
     // See if we can combine this build_vector into a blend with a zero vector.
-    if (!VecIn2.getNode() && ((Op.getOpcode() == ISD::Constant &&
-        cast<ConstantSDNode>(Op.getNode())->isNullValue()) ||
-        (Op.getOpcode() == ISD::ConstantFP &&
-        cast<ConstantFPSDNode>(Op.getNode())->getValueAPF().isZero()))) {
+    if (!VecIn2.getNode() && (isNullConstant(Op) || isNullFPConstant(Op))) {
       UsesZeroVector = true;
       continue;
     }
@@ -11468,17 +12087,20 @@ SDValue DAGCombiner::visitBUILD_VECTOR(SDNode *N) {
 
         // Try to replace VecIn1 with two extract_subvectors
         // No need to update the masks, they should still be correct.
-        VecIn2 = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, VT, VecIn1,
-          DAG.getConstant(VT.getVectorNumElements(), TLI.getVectorIdxTy()));
-        VecIn1 = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, VT, VecIn1,
-          DAG.getConstant(0, TLI.getVectorIdxTy()));
+        VecIn2 = DAG.getNode(
+            ISD::EXTRACT_SUBVECTOR, dl, VT, VecIn1,
+            DAG.getConstant(VT.getVectorNumElements(), dl,
+                            TLI.getVectorIdxTy(DAG.getDataLayout())));
+        VecIn1 = DAG.getNode(
+            ISD::EXTRACT_SUBVECTOR, dl, VT, VecIn1,
+            DAG.getConstant(0, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
       } else
         return SDValue();
     }
 
     if (UsesZeroVector)
-      VecIn2 = VT.isInteger() ? DAG.getConstant(0, VT) :
-                                DAG.getConstantFP(0.0, VT);
+      VecIn2 = VT.isInteger() ? DAG.getConstant(0, dl, VT) :
+                                DAG.getConstantFP(0.0, dl, VT);
     else
       // If VecIn2 is unused then change it to undef.
       VecIn2 = VecIn2.getNode() ? VecIn2 : DAG.getUNDEF(VT);
@@ -11538,7 +12160,7 @@ static SDValue combineConcatVectorOfScalars(SDNode *N, SelectionDAG &DAG) {
   }
 
   // If any of the operands is a floating point scalar bitcast to a vector,
-  // use floating point types throughout, and bitcast everything.  
+  // use floating point types throughout, and bitcast everything.
   // Replace UNDEFs by another scalar UNDEF node, of the final desired type.
   if (AnyFP) {
     SVT = EVT::getFloatingPointVT(OpVT.getSizeInBits());
@@ -12349,6 +12971,16 @@ SDValue DAGCombiner::visitINSERT_SUBVECTOR(SDNode *N) {
   return SDValue();
 }
 
+SDValue DAGCombiner::visitFP_TO_FP16(SDNode *N) {
+  SDValue N0 = N->getOperand(0);
+
+  // fold (fp_to_fp16 (fp16_to_fp op)) -> op
+  if (N0->getOpcode() == ISD::FP16_TO_FP)
+    return N0->getOperand(0);
+
+  return SDValue();
+}
+
 /// Returns a vector_shuffle if it able to transform an AND to a vector_shuffle
 /// with the destination vector and a zero vector.
 /// e.g. AND V, <0xffffffff, 0, 0xffffffff, 0>. ==>
@@ -12359,7 +12991,7 @@ SDValue DAGCombiner::XformToShuffleWithZero(SDNode *N) {
   SDValue RHS = N->getOperand(1);
   SDLoc dl(N);
 
-  // Make sure we're not running after operation legalization where it 
+  // Make sure we're not running after operation legalization where it
   // may have custom lowered the vector shuffles.
   if (LegalOperations)
     return SDValue();
@@ -12376,12 +13008,9 @@ SDValue DAGCombiner::XformToShuffleWithZero(SDNode *N) {
 
     for (unsigned i = 0; i != NumElts; ++i) {
       SDValue Elt = RHS.getOperand(i);
-      if (!isa<ConstantSDNode>(Elt))
-        return SDValue();
-
-      if (cast<ConstantSDNode>(Elt)->isAllOnesValue())
+      if (isAllOnesConstant(Elt))
         Indices.push_back(i);
-      else if (cast<ConstantSDNode>(Elt)->isNullValue())
+      else if (isNullConstant(Elt))
         Indices.push_back(NumElts+i);
       else
         return SDValue();
@@ -12395,8 +13024,8 @@ SDValue DAGCombiner::XformToShuffleWithZero(SDNode *N) {
     // Return the new VECTOR_SHUFFLE node.
     EVT EltVT = RVT.getVectorElementType();
     SmallVector<SDValue,8> ZeroOps(RVT.getVectorNumElements(),
-                                   DAG.getConstant(0, EltVT));
-    SDValue Zero = DAG.getNode(ISD::BUILD_VECTOR, SDLoc(N), RVT, ZeroOps);
+                                   DAG.getConstant(0, dl, EltVT));
+    SDValue Zero = DAG.getNode(ISD::BUILD_VECTOR, dl, RVT, ZeroOps);
     LHS = DAG.getNode(ISD::BITCAST, dl, RVT, LHS);
     SDValue Shuf = DAG.getVectorShuffle(RVT, dl, LHS, Zero, &Indices[0]);
     return DAG.getNode(ISD::BITCAST, dl, VT, Shuf);
@@ -12433,10 +13062,8 @@ SDValue DAGCombiner::SimplifyVBinOp(SDNode *N) {
       // Can't fold divide by zero.
       if (N->getOpcode() == ISD::SDIV || N->getOpcode() == ISD::UDIV ||
           N->getOpcode() == ISD::FDIV) {
-        if ((RHSOp.getOpcode() == ISD::Constant &&
-             cast<ConstantSDNode>(RHSOp.getNode())->isNullValue()) ||
-            (RHSOp.getOpcode() == ISD::ConstantFP &&
-             cast<ConstantFPSDNode>(RHSOp.getNode())->getValueAPF().isZero()))
+        if (isNullConstant(RHSOp) || (RHSOp.getOpcode() == ISD::ConstantFP &&
+             cast<ConstantFPSDNode>(RHSOp.getNode())->isZero()))
           break;
       }
 
@@ -12528,6 +13155,38 @@ SDValue DAGCombiner::SimplifySelect(SDLoc DL, SDValue N0,
 bool DAGCombiner::SimplifySelectOps(SDNode *TheSelect, SDValue LHS,
                                     SDValue RHS) {
 
+  // fold (select (setcc x, -0.0, *lt), NaN, (fsqrt x))
+  // The select + setcc is redundant, because fsqrt returns NaN for X < -0.
+  if (const ConstantFPSDNode *NaN = isConstOrConstSplatFP(LHS)) {
+    if (NaN->isNaN() && RHS.getOpcode() == ISD::FSQRT) {
+      // We have: (select (setcc ?, ?, ?), NaN, (fsqrt ?))
+      SDValue Sqrt = RHS;
+      ISD::CondCode CC;
+      SDValue CmpLHS;
+      const ConstantFPSDNode *NegZero = nullptr;
+
+      if (TheSelect->getOpcode() == ISD::SELECT_CC) {
+        CC = dyn_cast<CondCodeSDNode>(TheSelect->getOperand(4))->get();
+        CmpLHS = TheSelect->getOperand(0);
+        NegZero = isConstOrConstSplatFP(TheSelect->getOperand(1));
+      } else {
+        // SELECT or VSELECT
+        SDValue Cmp = TheSelect->getOperand(0);
+        if (Cmp.getOpcode() == ISD::SETCC) {
+          CC = dyn_cast<CondCodeSDNode>(Cmp.getOperand(2))->get();
+          CmpLHS = Cmp.getOperand(0);
+          NegZero = isConstOrConstSplatFP(Cmp.getOperand(1));
+        }
+      }
+      if (NegZero && NegZero->isNegative() && NegZero->isZero() &&
+          Sqrt.getOperand(0) == CmpLHS && (CC == ISD::SETOLT ||
+          CC == ISD::SETULT || CC == ISD::SETLT)) {
+        // We have: (select (setcc x, -0.0, *lt), NaN, (fsqrt x))
+        CombineTo(TheSelect, Sqrt);
+        return true;
+      }
+    }
+  }
   // Cannot simplify select with vector condition
   if (TheSelect->getOperand(0).getValueType().isVector()) return false;
 
@@ -12549,6 +13208,9 @@ bool DAGCombiner::SimplifySelectOps(SDNode *TheSelect, SDValue LHS,
     if (LHS.getOperand(0) != RHS.getOperand(0) ||
         // Do not let this transformation reduce the number of volatile loads.
         LLD->isVolatile() || RLD->isVolatile() ||
+        // FIXME: If either is a pre/post inc/dec load,
+        // we'd need to split out the address adjustment.
+        LLD->isIndexed() || RLD->isIndexed() ||
         // If this is an EXTLOAD, the VT's must match.
         LLD->getMemoryVT() != RLD->getMemoryVT() ||
         // If this is an EXTLOAD, the kind of extension must match.
@@ -12650,25 +13312,22 @@ SDValue DAGCombiner::SimplifySelectCC(SDLoc DL, SDValue N0, SDValue N1,
   EVT VT = N2.getValueType();
   ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(N1.getNode());
   ConstantSDNode *N2C = dyn_cast<ConstantSDNode>(N2.getNode());
-  ConstantSDNode *N3C = dyn_cast<ConstantSDNode>(N3.getNode());
 
   // Determine if the condition we're dealing with is constant
   SDValue SCC = SimplifySetCC(getSetCCResultType(N0.getValueType()),
                               N0, N1, CC, DL, false);
   if (SCC.getNode()) AddToWorklist(SCC.getNode());
-  ConstantSDNode *SCCC = dyn_cast_or_null<ConstantSDNode>(SCC.getNode());
 
-  // fold select_cc true, x, y -> x
-  if (SCCC && !SCCC->isNullValue())
-    return N2;
-  // fold select_cc false, x, y -> y
-  if (SCCC && SCCC->isNullValue())
-    return N3;
+  if (ConstantSDNode *SCCC = dyn_cast_or_null<ConstantSDNode>(SCC.getNode())) {
+    // fold select_cc true, x, y -> x
+    // fold select_cc false, x, y -> y
+    return !SCCC->isNullValue() ? N2 : N3;
+  }
 
   // Check to see if we can simplify the select into an fabs node
   if (ConstantFPSDNode *CFP = dyn_cast<ConstantFPSDNode>(N1)) {
     // Allow either -0.0 or 0.0
-    if (CFP->getValueAPF().isZero()) {
+    if (CFP->isZero()) {
       // select (setg[te] X, +/-0.0), X, fneg(X) -> fabs
       if ((CC == ISD::SETGE || CC == ISD::SETGT) &&
           N0 == N2 && N3.getOpcode() == ISD::FNEG &&
@@ -12706,19 +13365,20 @@ SDValue DAGCombiner::SimplifySelectCC(SDLoc DL, SDValue N0, SDValue N1,
           const_cast<ConstantFP*>(TV->getConstantFPValue())
         };
         Type *FPTy = Elts[0]->getType();
-        const DataLayout &TD = *TLI.getDataLayout();
+        const DataLayout &TD = DAG.getDataLayout();
 
         // Create a ConstantArray of the two constants.
         Constant *CA = ConstantArray::get(ArrayType::get(FPTy, 2), Elts);
-        SDValue CPIdx = DAG.getConstantPool(CA, TLI.getPointerTy(),
-                                            TD.getPrefTypeAlignment(FPTy));
+        SDValue CPIdx =
+            DAG.getConstantPool(CA, TLI.getPointerTy(DAG.getDataLayout()),
+                                TD.getPrefTypeAlignment(FPTy));
         unsigned Alignment = cast<ConstantPoolSDNode>(CPIdx)->getAlignment();
 
         // Get the offsets to the 0 and 1 element of the array so that we can
         // select between them.
-        SDValue Zero = DAG.getIntPtrConstant(0);
+        SDValue Zero = DAG.getIntPtrConstant(0, DL);
         unsigned EltSize = (unsigned)TD.getTypeAllocSize(Elts[0]->getType());
-        SDValue One = DAG.getIntPtrConstant(EltSize);
+        SDValue One = DAG.getIntPtrConstant(EltSize, SDLoc(FV));
 
         SDValue Cond = DAG.getSetCC(DL,
                                     getSetCCResultType(N0.getValueType()),
@@ -12733,24 +13393,23 @@ SDValue DAGCombiner::SimplifySelectCC(SDLoc DL, SDValue N0, SDValue N1,
         return DAG.getLoad(TV->getValueType(0), DL, DAG.getEntryNode(), CPIdx,
                            MachinePointerInfo::getConstantPool(), false,
                            false, false, Alignment);
-
       }
     }
 
   // Check to see if we can perform the "gzip trick", transforming
   // (select_cc setlt X, 0, A, 0) -> (and (sra X, (sub size(X), 1), A)
-  if (N1C && N3C && N3C->isNullValue() && CC == ISD::SETLT &&
-      (N1C->isNullValue() ||                         // (a < 0) ? b : 0
-       (N1C->getAPIntValue() == 1 && N0 == N2))) {   // (a < 1) ? a : 0
+  if (isNullConstant(N3) && CC == ISD::SETLT &&
+      (isNullConstant(N1) ||                 // (a < 0) ? b : 0
+       (isOneConstant(N1) && N0 == N2))) {   // (a < 1) ? a : 0
     EVT XType = N0.getValueType();
     EVT AType = N2.getValueType();
     if (XType.bitsGE(AType)) {
       // and (sra X, size(X)-1, A) -> "and (srl X, C2), A" iff A is a
       // single-bit constant.
-      if (N2C && ((N2C->getAPIntValue() & (N2C->getAPIntValue()-1)) == 0)) {
+      if (N2C && ((N2C->getAPIntValue() & (N2C->getAPIntValue() - 1)) == 0)) {
         unsigned ShCtV = N2C->getAPIntValue().logBase2();
-        ShCtV = XType.getSizeInBits()-ShCtV-1;
-        SDValue ShCt = DAG.getConstant(ShCtV,
+        ShCtV = XType.getSizeInBits() - ShCtV - 1;
+        SDValue ShCt = DAG.getConstant(ShCtV, SDLoc(N0),
                                        getShiftAmountTy(N0.getValueType()));
         SDValue Shift = DAG.getNode(ISD::SRL, SDLoc(N0),
                                     XType, N0, ShCt);
@@ -12766,7 +13425,8 @@ SDValue DAGCombiner::SimplifySelectCC(SDLoc DL, SDValue N0, SDValue N1,
 
       SDValue Shift = DAG.getNode(ISD::SRA, SDLoc(N0),
                                   XType, N0,
-                                  DAG.getConstant(XType.getSizeInBits()-1,
+                                  DAG.getConstant(XType.getSizeInBits() - 1,
+                                                  SDLoc(N0),
                                          getShiftAmountTy(N0.getValueType())));
       AddToWorklist(Shift.getNode());
 
@@ -12786,23 +13446,21 @@ SDValue DAGCombiner::SimplifySelectCC(SDLoc DL, SDValue N0, SDValue N1,
   // single bit-test can be materialized as an all-ones register with
   // shift-left and shift-right-arith.
   if (CC == ISD::SETEQ && N0->getOpcode() == ISD::AND &&
-      N0->getValueType(0) == VT &&
-      N1C && N1C->isNullValue() &&
-      N2C && N2C->isNullValue()) {
+      N0->getValueType(0) == VT && isNullConstant(N1) && isNullConstant(N2)) {
     SDValue AndLHS = N0->getOperand(0);
     ConstantSDNode *ConstAndRHS = dyn_cast<ConstantSDNode>(N0->getOperand(1));
     if (ConstAndRHS && ConstAndRHS->getAPIntValue().countPopulation() == 1) {
       // Shift the tested bit over the sign bit.
       APInt AndMask = ConstAndRHS->getAPIntValue();
       SDValue ShlAmt =
-        DAG.getConstant(AndMask.countLeadingZeros(),
+        DAG.getConstant(AndMask.countLeadingZeros(), SDLoc(AndLHS),
                         getShiftAmountTy(AndLHS.getValueType()));
       SDValue Shl = DAG.getNode(ISD::SHL, SDLoc(N0), VT, AndLHS, ShlAmt);
 
       // Now arithmetic right shift it all the way over, so the result is either
       // all-ones, or zero.
       SDValue ShrAmt =
-        DAG.getConstant(AndMask.getBitWidth()-1,
+        DAG.getConstant(AndMask.getBitWidth() - 1, SDLoc(Shl),
                         getShiftAmountTy(Shl.getValueType()));
       SDValue Shr = DAG.getNode(ISD::SRA, SDLoc(N0), VT, Shl, ShrAmt);
 
@@ -12811,13 +13469,13 @@ SDValue DAGCombiner::SimplifySelectCC(SDLoc DL, SDValue N0, SDValue N1,
   }
 
   // fold select C, 16, 0 -> shl C, 4
-  if (N2C && N3C && N3C->isNullValue() && N2C->getAPIntValue().isPowerOf2() &&
+  if (N2C && isNullConstant(N3) && N2C->getAPIntValue().isPowerOf2() &&
       TLI.getBooleanContents(N0.getValueType()) ==
           TargetLowering::ZeroOrOneBooleanContent) {
 
     // If the caller doesn't want us to simplify this into a zext of a compare,
     // don't do it.
-    if (NotExtCompare && N2C->getAPIntValue() == 1)
+    if (NotExtCompare && N2C->isOne())
       return SDValue();
 
     // Get a SetCC of the condition
@@ -12845,13 +13503,13 @@ SDValue DAGCombiner::SimplifySelectCC(SDLoc DL, SDValue N0, SDValue N1,
       AddToWorklist(SCC.getNode());
       AddToWorklist(Temp.getNode());
 
-      if (N2C->getAPIntValue() == 1)
+      if (N2C->isOne())
         return Temp;
 
       // shl setcc result by log2 n2c
       return DAG.getNode(
           ISD::SHL, DL, N2.getValueType(), Temp,
-          DAG.getConstant(N2C->getAPIntValue().logBase2(),
+          DAG.getConstant(N2C->getAPIntValue().logBase2(), SDLoc(Temp),
                           getShiftAmountTy(Temp.getValueType())));
     }
   }
@@ -12859,7 +13517,7 @@ SDValue DAGCombiner::SimplifySelectCC(SDLoc DL, SDValue N0, SDValue N1,
   // Check to see if this is the equivalent of setcc
   // FIXME: Turn all of these into setcc if setcc if setcc is legal
   // otherwise, go ahead with the folds.
-  if (0 && N3C && N3C->isNullValue() && N2C && (N2C->getAPIntValue() == 1ULL)) {
+  if (0 && isNullConstant(N3) && isOneConstant(N2)) {
     EVT XType = N0.getValueType();
     if (!LegalOperations ||
         TLI.isOperationLegal(ISD::SETCC, getSetCCResultType(XType))) {
@@ -12870,30 +13528,34 @@ SDValue DAGCombiner::SimplifySelectCC(SDLoc DL, SDValue N0, SDValue N1,
     }
 
     // fold (seteq X, 0) -> (srl (ctlz X, log2(size(X))))
-    if (N1C && N1C->isNullValue() && CC == ISD::SETEQ &&
+    if (isNullConstant(N1) && CC == ISD::SETEQ &&
         (!LegalOperations ||
          TLI.isOperationLegal(ISD::CTLZ, XType))) {
       SDValue Ctlz = DAG.getNode(ISD::CTLZ, SDLoc(N0), XType, N0);
       return DAG.getNode(ISD::SRL, DL, XType, Ctlz,
                          DAG.getConstant(Log2_32(XType.getSizeInBits()),
+                                         SDLoc(Ctlz),
                                        getShiftAmountTy(Ctlz.getValueType())));
     }
     // fold (setgt X, 0) -> (srl (and (-X, ~X), size(X)-1))
-    if (N1C && N1C->isNullValue() && CC == ISD::SETGT) {
-      SDValue NegN0 = DAG.getNode(ISD::SUB, SDLoc(N0),
-                                  XType, DAG.getConstant(0, XType), N0);
-      SDValue NotN0 = DAG.getNOT(SDLoc(N0), N0, XType);
+    if (isNullConstant(N1) && CC == ISD::SETGT) {
+      SDLoc DL(N0);
+      SDValue NegN0 = DAG.getNode(ISD::SUB, DL,
+                                  XType, DAG.getConstant(0, DL, XType), N0);
+      SDValue NotN0 = DAG.getNOT(DL, N0, XType);
       return DAG.getNode(ISD::SRL, DL, XType,
                          DAG.getNode(ISD::AND, DL, XType, NegN0, NotN0),
-                         DAG.getConstant(XType.getSizeInBits()-1,
+                         DAG.getConstant(XType.getSizeInBits() - 1, DL,
                                          getShiftAmountTy(XType)));
     }
     // fold (setgt X, -1) -> (xor (srl (X, size(X)-1), 1))
-    if (N1C && N1C->isAllOnesValue() && CC == ISD::SETGT) {
-      SDValue Sign = DAG.getNode(ISD::SRL, SDLoc(N0), XType, N0,
-                                 DAG.getConstant(XType.getSizeInBits()-1,
+    if (isAllOnesConstant(N1) && CC == ISD::SETGT) {
+      SDLoc DL(N0);
+      SDValue Sign = DAG.getNode(ISD::SRL, DL, XType, N0,
+                                 DAG.getConstant(XType.getSizeInBits() - 1, DL,
                                          getShiftAmountTy(N0.getValueType())));
-      return DAG.getNode(ISD::XOR, DL, XType, Sign, DAG.getConstant(1, XType));
+      return DAG.getNode(ISD::XOR, DL, XType, Sign, DAG.getConstant(1, DL,
+                                                                    XType));
     }
   }
 
@@ -12916,11 +13578,12 @@ SDValue DAGCombiner::SimplifySelectCC(SDLoc DL, SDValue N0, SDValue N1,
 
     EVT XType = N0.getValueType();
     if (SubC && SubC->isNullValue() && XType.isInteger()) {
-      SDValue Shift = DAG.getNode(ISD::SRA, SDLoc(N0), XType,
+      SDLoc DL(N0);
+      SDValue Shift = DAG.getNode(ISD::SRA, DL, XType,
                                   N0,
-                                  DAG.getConstant(XType.getSizeInBits()-1,
+                                  DAG.getConstant(XType.getSizeInBits() - 1, DL,
                                          getShiftAmountTy(N0.getValueType())));
-      SDValue Add = DAG.getNode(ISD::ADD, SDLoc(N0),
+      SDValue Add = DAG.getNode(ISD::ADD, DL,
                                 XType, N0, Shift);
       AddToWorklist(Shift.getNode());
       AddToWorklist(Add.getNode());
@@ -12950,7 +13613,7 @@ SDValue DAGCombiner::BuildSDIV(SDNode *N) {
     return SDValue();
 
   // Avoid division by zero.
-  if (!C->getAPIntValue())
+  if (C->isNullValue())
     return SDValue();
 
   std::vector<SDNode*> Built;
@@ -12970,7 +13633,7 @@ SDValue DAGCombiner::BuildSDIVPow2(SDNode *N) {
     return SDValue();
 
   // Avoid division by zero.
-  if (!C->getAPIntValue())
+  if (C->isNullValue())
     return SDValue();
 
   std::vector<SDNode *> Built;
@@ -12991,7 +13654,7 @@ SDValue DAGCombiner::BuildUDIV(SDNode *N) {
     return SDValue();
 
   // Avoid division by zero.
-  if (!C->getAPIntValue())
+  if (C->isNullValue())
     return SDValue();
 
   std::vector<SDNode*> Built;
@@ -13021,7 +13684,7 @@ SDValue DAGCombiner::BuildReciprocalEstimate(SDValue Op) {
       //     does not require additional intermediate precision]
       EVT VT = Op.getValueType();
       SDLoc DL(Op);
-      SDValue FPOne = DAG.getConstantFP(1.0, VT);
+      SDValue FPOne = DAG.getConstantFP(1.0, DL, VT);
 
       AddToWorklist(Est.getNode());
 
@@ -13056,7 +13719,7 @@ SDValue DAGCombiner::BuildRsqrtNROneConst(SDValue Arg, SDValue Est,
                                           unsigned Iterations) {
   EVT VT = Arg.getValueType();
   SDLoc DL(Arg);
-  SDValue ThreeHalves = DAG.getConstantFP(1.5, VT);
+  SDValue ThreeHalves = DAG.getConstantFP(1.5, DL, VT);
 
   // We now need 0.5 * Arg which we can write as (1.5 * Arg - Arg) so that
   // this entire sequence requires only one FP constant.
@@ -13092,8 +13755,8 @@ SDValue DAGCombiner::BuildRsqrtNRTwoConst(SDValue Arg, SDValue Est,
                                           unsigned Iterations) {
   EVT VT = Arg.getValueType();
   SDLoc DL(Arg);
-  SDValue MinusThree = DAG.getConstantFP(-3.0, VT);
-  SDValue MinusHalf = DAG.getConstantFP(-0.5, VT);
+  SDValue MinusThree = DAG.getConstantFP(-3.0, DL, VT);
+  SDValue MinusHalf = DAG.getConstantFP(-0.5, DL, VT);
 
   // Newton iterations: Est = -0.5 * Est * (-3.0 + Arg * Est * Est)
   for (unsigned i = 0; i < Iterations; ++i) {
@@ -13181,6 +13844,15 @@ bool DAGCombiner::isAlias(LSBaseSDNode *Op0, LSBaseSDNode *Op1) const {
   // If they are both volatile then they cannot be reordered.
   if (Op0->isVolatile() && Op1->isVolatile()) return true;
 
+  // If one operation reads from invariant memory, and the other may store, they
+  // cannot alias. These should really be checking the equivalent of mayWrite,
+  // but it only matters for memory nodes other than load /store.
+  if (Op0->isInvariant() && Op1->writeMem())
+    return false;
+
+  if (Op1->isInvariant() && Op0->writeMem())
+    return false;
+
   // Gather base node and offset information.
   SDValue Base1, Base2;
   int64_t Offset1, Offset2;
@@ -13249,14 +13921,12 @@ bool DAGCombiner::isAlias(LSBaseSDNode *Op0, LSBaseSDNode *Op1) const {
         Op0->getSrcValueOffset() - MinOffset;
     int64_t Overlap2 = (Op1->getMemoryVT().getSizeInBits() >> 3) +
         Op1->getSrcValueOffset() - MinOffset;
-    AliasAnalysis::AliasResult AAResult =
-        AA.alias(AliasAnalysis::Location(Op0->getMemOperand()->getValue(),
-                                         Overlap1,
-                                         UseTBAA ? Op0->getAAInfo() : AAMDNodes()),
-                 AliasAnalysis::Location(Op1->getMemOperand()->getValue(),
-                                         Overlap2,
-                                         UseTBAA ? Op1->getAAInfo() : AAMDNodes()));
-    if (AAResult == AliasAnalysis::NoAlias)
+    AliasResult AAResult =
+        AA.alias(MemoryLocation(Op0->getMemOperand()->getValue(), Overlap1,
+                                UseTBAA ? Op0->getAAInfo() : AAMDNodes()),
+                 MemoryLocation(Op1->getMemOperand()->getValue(), Overlap2,
+                                UseTBAA ? Op1->getAAInfo() : AAMDNodes()));
+    if (AAResult == NoAlias)
       return false;
   }
 
@@ -13282,8 +13952,7 @@ void DAGCombiner::GatherAllAliases(SDNode *N, SDValue OriginalChain,
   // aliases list.  If not, then continue up the chain looking for the next
   // candidate.
   while (!Chains.empty()) {
-    SDValue Chain = Chains.back();
-    Chains.pop_back();
+    SDValue Chain = Chains.pop_back_val();
 
     // For TokenFactor nodes, look at each operand and only continue up the
     // chain until we find two aliases.  If we've seen two aliases, assume we'll
@@ -13390,7 +14059,7 @@ void DAGCombiner::GatherAllAliases(SDNode *N, SDValue OriginalChain,
          UIE = M->use_end(); UI != UIE; ++UI)
       if (UI.getUse().getValueType() == MVT::Other &&
           Visited.insert(*UI).second) {
-        if (isa<MemIntrinsicSDNode>(*UI) || isa<MemSDNode>(*UI)) {
+        if (isa<MemSDNode>(*UI)) {
           // We've not visited this use, and we care about it (it could have an
           // ordering dependency with the original node).
           Aliases.clear();
diff --git a/lib/CodeGen/SelectionDAG/FastISel.cpp b/lib/CodeGen/SelectionDAG/FastISel.cpp
index 5ffb826956e..2b9ba2c1b53 100644
--- a/lib/CodeGen/SelectionDAG/FastISel.cpp
+++ b/lib/CodeGen/SelectionDAG/FastISel.cpp
@@ -59,6 +59,7 @@
 #include "llvm/IR/GlobalVariable.h"
 #include "llvm/IR/Instructions.h"
 #include "llvm/IR/IntrinsicInst.h"
+#include "llvm/IR/Mangler.h"
 #include "llvm/IR/Operator.h"
 #include "llvm/Support/Debug.h"
 #include "llvm/Support/ErrorHandling.h"
@@ -165,7 +166,7 @@ bool FastISel::hasTrivialKill(const Value *V) {
 }
 
 unsigned FastISel::getRegForValue(const Value *V) {
-  EVT RealVT = TLI.getValueType(V->getType(), /*AllowUnknown=*/true);
+  EVT RealVT = TLI.getValueType(DL, V->getType(), /*AllowUnknown=*/true);
   // Don't handle non-simple values in FastISel.
   if (!RealVT.isSimple())
     return 0;
@@ -227,7 +228,7 @@ unsigned FastISel::materializeConstant(const Value *V, MVT VT) {
     if (!Reg) {
       // Try to emit the constant by using an integer constant with a cast.
       const APFloat &Flt = CF->getValueAPF();
-      EVT IntVT = TLI.getPointerTy();
+      EVT IntVT = TLI.getPointerTy(DL);
 
       uint64_t x[2];
       uint32_t IntBitWidth = IntVT.getSizeInBits();
@@ -320,7 +321,7 @@ std::pair<unsigned, bool> FastISel::getRegForGEPIndex(const Value *Idx) {
   bool IdxNIsKill = hasTrivialKill(Idx);
 
   // If the index is smaller or larger than intptr_t, truncate or extend it.
-  MVT PtrVT = TLI.getPointerTy();
+  MVT PtrVT = TLI.getPointerTy(DL);
   EVT IdxVT = EVT::getEVT(Idx->getType(), /*HandleUnknown=*/false);
   if (IdxVT.bitsLT(PtrVT)) {
     IdxN = fastEmit_r(IdxVT.getSimpleVT(), PtrVT, ISD::SIGN_EXTEND, IdxN,
@@ -492,7 +493,7 @@ bool FastISel::selectGetElementPtr(const User *I) {
   // FIXME: What's a good SWAG number for MaxOffs?
   uint64_t MaxOffs = 2048;
   Type *Ty = I->getOperand(0)->getType();
-  MVT VT = TLI.getPointerTy();
+  MVT VT = TLI.getPointerTy(DL);
   for (GetElementPtrInst::const_op_iterator OI = I->op_begin() + 1,
                                             E = I->op_end();
        OI != E; ++OI) {
@@ -701,6 +702,15 @@ bool FastISel::lowerCallOperands(const CallInst *CI, unsigned ArgIdx,
   return lowerCallTo(CLI);
 }
 
+FastISel::CallLoweringInfo &FastISel::CallLoweringInfo::setCallee(
+    const DataLayout &DL, MCContext &Ctx, CallingConv::ID CC, Type *ResultTy,
+    const char *Target, ArgListTy &&ArgsList, unsigned FixedArgs) {
+  SmallString<32> MangledName;
+  Mangler::getNameWithPrefix(MangledName, Target, DL);
+  MCSymbol *Sym = Ctx.getOrCreateSymbol(MangledName);
+  return setCallee(CC, ResultTy, Sym, std::move(ArgsList), FixedArgs);
+}
+
 bool FastISel::selectPatchpoint(const CallInst *I) {
   // void|i64 @llvm.experimental.patchpoint.void|i64(i64 <id>,
   //                                                 i32 <numBytes>,
@@ -711,7 +721,7 @@ bool FastISel::selectPatchpoint(const CallInst *I) {
   CallingConv::ID CC = I->getCallingConv();
   bool IsAnyRegCC = CC == CallingConv::AnyReg;
   bool HasDef = !I->getType()->isVoidTy();
-  Value *Callee = I->getOperand(PatchPointOpers::TargetPos);
+  Value *Callee = I->getOperand(PatchPointOpers::TargetPos)->stripPointerCasts();
 
   // Get the real number of arguments participating in the call <numArgs>
   assert(isa<ConstantInt>(I->getOperand(PatchPointOpers::NArgPos)) &&
@@ -757,23 +767,25 @@ bool FastISel::selectPatchpoint(const CallInst *I) {
       cast<ConstantInt>(I->getOperand(PatchPointOpers::NBytesPos));
   Ops.push_back(MachineOperand::CreateImm(NumBytes->getZExtValue()));
 
-  // Assume that the callee is a constant address or null pointer.
-  // FIXME: handle function symbols in the future.
-  uint64_t CalleeAddr;
-  if (const auto *C = dyn_cast<IntToPtrInst>(Callee))
-    CalleeAddr = cast<ConstantInt>(C->getOperand(0))->getZExtValue();
-  else if (const auto *C = dyn_cast<ConstantExpr>(Callee)) {
-    if (C->getOpcode() == Instruction::IntToPtr)
-      CalleeAddr = cast<ConstantInt>(C->getOperand(0))->getZExtValue();
-    else
+  // Add the call target.
+  if (const auto *C = dyn_cast<IntToPtrInst>(Callee)) {
+    uint64_t CalleeConstAddr =
+      cast<ConstantInt>(C->getOperand(0))->getZExtValue();
+    Ops.push_back(MachineOperand::CreateImm(CalleeConstAddr));
+  } else if (const auto *C = dyn_cast<ConstantExpr>(Callee)) {
+    if (C->getOpcode() == Instruction::IntToPtr) {
+      uint64_t CalleeConstAddr =
+        cast<ConstantInt>(C->getOperand(0))->getZExtValue();
+      Ops.push_back(MachineOperand::CreateImm(CalleeConstAddr));
+    } else
       llvm_unreachable("Unsupported ConstantExpr.");
+  } else if (const auto *GV = dyn_cast<GlobalValue>(Callee)) {
+    Ops.push_back(MachineOperand::CreateGA(GV, 0));
   } else if (isa<ConstantPointerNull>(Callee))
-    CalleeAddr = 0;
+    Ops.push_back(MachineOperand::CreateImm(0));
   else
     llvm_unreachable("Unsupported callee address.");
 
-  Ops.push_back(MachineOperand::CreateImm(CalleeAddr));
-
   // Adjust <numArgs> to account for any arguments that have been passed on
   // the stack instead.
   unsigned NumCallRegArgs = IsAnyRegCC ? NumArgs : CLI.OutRegs.size();
@@ -854,6 +866,15 @@ static AttributeSet getReturnAttrs(FastISel::CallLoweringInfo &CLI) {
 
 bool FastISel::lowerCallTo(const CallInst *CI, const char *SymName,
                            unsigned NumArgs) {
+  MCContext &Ctx = MF->getContext();
+  SmallString<32> MangledName;
+  Mangler::getNameWithPrefix(MangledName, SymName, DL);
+  MCSymbol *Sym = Ctx.getOrCreateSymbol(MangledName);
+  return lowerCallTo(CI, Sym, NumArgs);
+}
+
+bool FastISel::lowerCallTo(const CallInst *CI, MCSymbol *Symbol,
+                           unsigned NumArgs) {
   ImmutableCallSite CS(CI);
 
   PointerType *PT = cast<PointerType>(CS.getCalledValue()->getType());
@@ -878,7 +899,7 @@ bool FastISel::lowerCallTo(const CallInst *CI, const char *SymName,
   }
 
   CallLoweringInfo CLI;
-  CLI.setCallee(RetTy, FTy, SymName, std::move(Args), CS, NumArgs);
+  CLI.setCallee(RetTy, FTy, Symbol, std::move(Args), CS, NumArgs);
 
   return lowerCallTo(CLI);
 }
@@ -887,10 +908,10 @@ bool FastISel::lowerCallTo(CallLoweringInfo &CLI) {
   // Handle the incoming return values from the call.
   CLI.clearIns();
   SmallVector<EVT, 4> RetTys;
-  ComputeValueVTs(TLI, CLI.RetTy, RetTys);
+  ComputeValueVTs(TLI, DL, CLI.RetTy, RetTys);
 
   SmallVector<ISD::OutputArg, 4> Outs;
-  GetReturnInfo(CLI.RetTy, getReturnAttrs(CLI), Outs, TLI);
+  GetReturnInfo(CLI.RetTy, getReturnAttrs(CLI), Outs, TLI, DL);
 
   bool CanLowerReturn = TLI.CanLowerReturn(
       CLI.CallConv, *FuncInfo.MF, CLI.IsVarArg, Outs, CLI.RetTy->getContext());
@@ -955,7 +976,7 @@ bool FastISel::lowerCallTo(CallLoweringInfo &CLI) {
       // not there, but there are cases it cannot get right.
       unsigned FrameAlign = Arg.Alignment;
       if (!FrameAlign)
-        FrameAlign = TLI.getByValTypeAlignment(ElementTy);
+        FrameAlign = TLI.getByValTypeAlignment(ElementTy, DL);
       Flags.setByValSize(FrameSize);
       Flags.setByValAlign(FrameAlign);
     }
@@ -1088,8 +1109,8 @@ bool FastISel::selectIntrinsicCall(const IntrinsicInst *II) {
   }
   case Intrinsic::dbg_declare: {
     const DbgDeclareInst *DI = cast<DbgDeclareInst>(II);
-    DIVariable DIVar = DI->getVariable();
-    if (!DIVar || !FuncInfo.MF->getMMI().hasDebugInfo()) {
+    assert(DI->getVariable() && "Missing variable");
+    if (!FuncInfo.MF->getMMI().hasDebugInfo()) {
       DEBUG(dbgs() << "Dropping debug info for " << *DI << "\n");
       return true;
     }
@@ -1224,8 +1245,8 @@ bool FastISel::selectIntrinsicCall(const IntrinsicInst *II) {
 }
 
 bool FastISel::selectCast(const User *I, unsigned Opcode) {
-  EVT SrcVT = TLI.getValueType(I->getOperand(0)->getType());
-  EVT DstVT = TLI.getValueType(I->getType());
+  EVT SrcVT = TLI.getValueType(DL, I->getOperand(0)->getType());
+  EVT DstVT = TLI.getValueType(DL, I->getType());
 
   if (SrcVT == MVT::Other || !SrcVT.isSimple() || DstVT == MVT::Other ||
       !DstVT.isSimple())
@@ -1267,8 +1288,8 @@ bool FastISel::selectBitCast(const User *I) {
   }
 
   // Bitcasts of other values become reg-reg copies or BITCAST operators.
-  EVT SrcEVT = TLI.getValueType(I->getOperand(0)->getType());
-  EVT DstEVT = TLI.getValueType(I->getType());
+  EVT SrcEVT = TLI.getValueType(DL, I->getOperand(0)->getType());
+  EVT DstEVT = TLI.getValueType(DL, I->getType());
   if (SrcEVT == MVT::Other || DstEVT == MVT::Other ||
       !TLI.isTypeLegal(SrcEVT) || !TLI.isTypeLegal(DstEVT))
     // Unhandled type. Halt "fast" selection and bail.
@@ -1329,7 +1350,7 @@ bool FastISel::selectInstruction(const Instruction *I) {
 
     // Don't handle Intrinsic::trap if a trap funciton is specified.
     if (F && F->getIntrinsicID() == Intrinsic::trap &&
-        !TM.Options.getTrapFunctionName().empty())
+        Call->hasFnAttr("trap-func-name"))
       return false;
   }
 
@@ -1392,7 +1413,7 @@ bool FastISel::selectFNeg(const User *I) {
   bool OpRegIsKill = hasTrivialKill(I);
 
   // If the target has ISD::FNEG, use it.
-  EVT VT = TLI.getValueType(I->getType());
+  EVT VT = TLI.getValueType(DL, I->getType());
   unsigned ResultReg = fastEmit_r(VT.getSimpleVT(), VT.getSimpleVT(), ISD::FNEG,
                                   OpReg, OpRegIsKill);
   if (ResultReg) {
@@ -1435,7 +1456,7 @@ bool FastISel::selectExtractValue(const User *U) {
 
   // Make sure we only try to handle extracts with a legal result.  But also
   // allow i1 because it's easy.
-  EVT RealVT = TLI.getValueType(EVI->getType(), /*AllowUnknown=*/true);
+  EVT RealVT = TLI.getValueType(DL, EVI->getType(), /*AllowUnknown=*/true);
   if (!RealVT.isSimple())
     return false;
   MVT VT = RealVT.getSimpleVT();
@@ -1459,7 +1480,7 @@ bool FastISel::selectExtractValue(const User *U) {
   unsigned VTIndex = ComputeLinearIndex(AggTy, EVI->getIndices());
 
   SmallVector<EVT, 4> AggValueVTs;
-  ComputeValueVTs(TLI, AggTy, AggValueVTs);
+  ComputeValueVTs(TLI, DL, AggTy, AggValueVTs);
 
   for (unsigned i = 0; i < VTIndex; i++)
     ResultReg += TLI.getNumRegisters(FuncInfo.Fn->getContext(), AggValueVTs[i]);
@@ -1561,8 +1582,8 @@ bool FastISel::selectOperator(const User *I, unsigned Opcode) {
 
   case Instruction::IntToPtr: // Deliberate fall-through.
   case Instruction::PtrToInt: {
-    EVT SrcVT = TLI.getValueType(I->getOperand(0)->getType());
-    EVT DstVT = TLI.getValueType(I->getType());
+    EVT SrcVT = TLI.getValueType(DL, I->getOperand(0)->getType());
+    EVT DstVT = TLI.getValueType(DL, I->getType());
     if (DstVT.bitsGT(SrcVT))
       return selectCast(I, ISD::ZERO_EXTEND);
     if (DstVT.bitsLT(SrcVT))
@@ -1591,7 +1612,7 @@ FastISel::FastISel(FunctionLoweringInfo &FuncInfo,
                    bool SkipTargetIndependentISel)
     : FuncInfo(FuncInfo), MF(FuncInfo.MF), MRI(FuncInfo.MF->getRegInfo()),
       MFI(*FuncInfo.MF->getFrameInfo()), MCP(*FuncInfo.MF->getConstantPool()),
-      TM(FuncInfo.MF->getTarget()), DL(*TM.getDataLayout()),
+      TM(FuncInfo.MF->getTarget()), DL(MF->getDataLayout()),
       TII(*MF->getSubtarget().getInstrInfo()),
       TLI(*MF->getSubtarget().getTargetLowering()),
       TRI(*MF->getSubtarget().getRegisterInfo()), LibInfo(LibInfo),
@@ -1673,6 +1694,7 @@ unsigned FastISel::fastEmit_ri_(MVT VT, unsigned Opcode, unsigned Op0,
   if (ResultReg)
     return ResultReg;
   unsigned MaterialReg = fastEmit_i(ImmType, ImmType, ISD::Constant, Imm);
+  bool IsImmKill = true;
   if (!MaterialReg) {
     // This is a bit ugly/slow, but failing here means falling out of
     // fast-isel, which would be very slow.
@@ -1681,9 +1703,15 @@ unsigned FastISel::fastEmit_ri_(MVT VT, unsigned Opcode, unsigned Op0,
     MaterialReg = getRegForValue(ConstantInt::get(ITy, Imm));
     if (!MaterialReg)
       return 0;
+    // FIXME: If the materialized register here has no uses yet then this
+    // will be the first use and we should be able to mark it as killed.
+    // However, the local value area for materialising constant expressions
+    // grows down, not up, which means that any constant expressions we generate
+    // later which also use 'Imm' could be after this instruction and therefore
+    // after this kill.
+    IsImmKill = false;
   }
-  return fastEmit_rr(VT, VT, Opcode, Op0, Op0IsKill, MaterialReg,
-                     /*IsKill=*/true);
+  return fastEmit_rr(VT, VT, Opcode, Op0, Op0IsKill, MaterialReg, IsImmKill);
 }
 
 unsigned FastISel::createResultReg(const TargetRegisterClass *RC) {
@@ -2009,7 +2037,7 @@ bool FastISel::handlePHINodesInSuccessorBlocks(const BasicBlock *LLVMBB) {
       // own moves. Second, this check is necessary because FastISel doesn't
       // use CreateRegs to create registers, so it always creates
       // exactly one register for each non-void instruction.
-      EVT VT = TLI.getValueType(PN->getType(), /*AllowUnknown=*/true);
+      EVT VT = TLI.getValueType(DL, PN->getType(), /*AllowUnknown=*/true);
       if (VT == MVT::Other || !TLI.isTypeLegal(VT)) {
         // Handle integer promotions, though, because they're common and easy.
         if (!(VT == MVT::i1 || VT == MVT::i8 || VT == MVT::i16)) {
diff --git a/lib/CodeGen/SelectionDAG/FunctionLoweringInfo.cpp b/lib/CodeGen/SelectionDAG/FunctionLoweringInfo.cpp
index 4b8ae32e9a5..cc306cbf5ae 100644
--- a/lib/CodeGen/SelectionDAG/FunctionLoweringInfo.cpp
+++ b/lib/CodeGen/SelectionDAG/FunctionLoweringInfo.cpp
@@ -80,28 +80,6 @@ static ISD::NodeType getPreferredExtendForValue(const Value *V) {
   return ExtendKind;
 }
 
-namespace {
-struct WinEHNumbering {
-  WinEHNumbering(WinEHFuncInfo &FuncInfo) : FuncInfo(FuncInfo), NextState(0) {}
-
-  WinEHFuncInfo &FuncInfo;
-  int NextState;
-
-  SmallVector<ActionHandler *, 4> HandlerStack;
-  SmallPtrSet<const Function *, 4> VisitedHandlers;
-
-  int currentEHNumber() const {
-    return HandlerStack.empty() ? -1 : HandlerStack.back()->getEHState();
-  }
-
-  void createUnwindMapEntry(int ToState, ActionHandler *AH);
-  void createTryBlockMapEntry(int TryLow, int TryHigh,
-                              ArrayRef<CatchHandler *> Handlers);
-  void processCallSite(ArrayRef<ActionHandler *> Actions, ImmutableCallSite CS);
-  void calculateStateNumbers(const Function &F);
-};
-}
-
 void FunctionLoweringInfo::set(const Function &fn, MachineFunction &mf,
                                SelectionDAG *DAG) {
   Fn = &fn;
@@ -112,7 +90,8 @@ void FunctionLoweringInfo::set(const Function &fn, MachineFunction &mf,
 
   // Check whether the function can return without sret-demotion.
   SmallVector<ISD::OutputArg, 4> Outs;
-  GetReturnInfo(Fn->getReturnType(), Fn->getAttributes(), Outs, *TLI);
+  GetReturnInfo(Fn->getReturnType(), Fn->getAttributes(), Outs, *TLI,
+                mf.getDataLayout());
   CanLowerReturn = TLI->CanLowerReturn(Fn->getCallingConv(), *MF,
                                        Fn->isVarArg(), Outs, Fn->getContext());
 
@@ -128,9 +107,9 @@ void FunctionLoweringInfo::set(const Function &fn, MachineFunction &mf,
         if (AI->isStaticAlloca()) {
           const ConstantInt *CUI = cast<ConstantInt>(AI->getArraySize());
           Type *Ty = AI->getAllocatedType();
-          uint64_t TySize = TLI->getDataLayout()->getTypeAllocSize(Ty);
+          uint64_t TySize = MF->getDataLayout().getTypeAllocSize(Ty);
           unsigned Align =
-              std::max((unsigned)TLI->getDataLayout()->getPrefTypeAlignment(Ty),
+              std::max((unsigned)MF->getDataLayout().getPrefTypeAlignment(Ty),
                        AI->getAlignment());
 
           TySize *= CUI->getZExtValue();   // Get total allocated size.
@@ -140,10 +119,10 @@ void FunctionLoweringInfo::set(const Function &fn, MachineFunction &mf,
             MF->getFrameInfo()->CreateStackObject(TySize, Align, false, AI);
 
         } else {
-          unsigned Align = std::max(
-              (unsigned)TLI->getDataLayout()->getPrefTypeAlignment(
-                AI->getAllocatedType()),
-              AI->getAlignment());
+          unsigned Align =
+              std::max((unsigned)MF->getDataLayout().getPrefTypeAlignment(
+                           AI->getAllocatedType()),
+                       AI->getAlignment());
           unsigned StackAlign =
               MF->getSubtarget().getFrameLowering()->getStackAlignment();
           if (Align <= StackAlign)
@@ -160,7 +139,7 @@ void FunctionLoweringInfo::set(const Function &fn, MachineFunction &mf,
           unsigned SP = TLI->getStackPointerRegisterToSaveRestore();
           const TargetRegisterInfo *TRI = MF->getSubtarget().getRegisterInfo();
           std::vector<TargetLowering::AsmOperandInfo> Ops =
-              TLI->ParseConstraints(TRI, CS);
+              TLI->ParseConstraints(Fn->getParent()->getDataLayout(), TRI, CS);
           for (size_t I = 0, E = Ops.size(); I != E; ++I) {
             TargetLowering::AsmOperandInfo &Op = Ops[I];
             if (Op.Type == InlineAsm::isClobber) {
@@ -170,7 +149,7 @@ void FunctionLoweringInfo::set(const Function &fn, MachineFunction &mf,
                   TLI->getRegForInlineAsmConstraint(TRI, Op.ConstraintCode,
                                                     Op.ConstraintVT);
               if (PhysReg.first == SP)
-                MF->getFrameInfo()->setHasInlineAsmWithSPAdjust(true);
+                MF->getFrameInfo()->setHasOpaqueSPAdjustment(true);
             }
           }
         }
@@ -202,8 +181,9 @@ void FunctionLoweringInfo::set(const Function &fn, MachineFunction &mf,
       // during the initial isel pass through the IR so that it is done
       // in a predictable order.
       if (const DbgDeclareInst *DI = dyn_cast<DbgDeclareInst>(I)) {
-        DIVariable DIVar = DI->getVariable();
-        if (MMI.hasDebugInfo() && DIVar && DI->getDebugLoc()) {
+        assert(DI->getVariable() && "Missing variable");
+        assert(DI->getDebugLoc() && "Missing location");
+        if (MMI.hasDebugInfo()) {
           // Don't handle byval struct arguments or VLAs, for example.
           // Non-byval arguments are handled here (they refer to the stack
           // temporary alloca at this point).
@@ -257,7 +237,7 @@ void FunctionLoweringInfo::set(const Function &fn, MachineFunction &mf,
       assert(PHIReg && "PHI node does not have an assigned virtual register!");
 
       SmallVector<EVT, 4> ValueVTs;
-      ComputeValueVTs(*TLI, PN->getType(), ValueVTs);
+      ComputeValueVTs(*TLI, MF->getDataLayout(), PN->getType(), ValueVTs);
       for (unsigned vti = 0, vte = ValueVTs.size(); vti != vte; ++vti) {
         EVT VT = ValueVTs[vti];
         unsigned NumRegisters = TLI->getNumRegisters(Fn->getContext(), VT);
@@ -270,180 +250,80 @@ void FunctionLoweringInfo::set(const Function &fn, MachineFunction &mf,
   }
 
   // Mark landing pad blocks.
-  for (BB = Fn->begin(); BB != EB; ++BB)
+  SmallVector<const LandingPadInst *, 4> LPads;
+  for (BB = Fn->begin(); BB != EB; ++BB) {
     if (const auto *Invoke = dyn_cast<InvokeInst>(BB->getTerminator()))
       MBBMap[Invoke->getSuccessor(1)]->setIsLandingPad();
-
-  // Calculate EH numbers for WinEH.
-  if (fn.hasFnAttribute("wineh-parent")) {
-    const Function *WinEHParentFn = MMI.getWinEHParent(&fn);
-    WinEHFuncInfo &FI = MMI.getWinEHFuncInfo(WinEHParentFn);
-    if (FI.LandingPadStateMap.empty()) {
-      WinEHNumbering Num(FI);
-      Num.calculateStateNumbers(*WinEHParentFn);
-      // Pop everything on the handler stack.
-      Num.processCallSite(None, ImmutableCallSite());
-    }
+    if (BB->isLandingPad())
+      LPads.push_back(BB->getLandingPadInst());
   }
-}
-
-void WinEHNumbering::createUnwindMapEntry(int ToState, ActionHandler *AH) {
-  WinEHUnwindMapEntry UME;
-  UME.ToState = ToState;
-  if (auto *CH = dyn_cast_or_null<CleanupHandler>(AH))
-    UME.Cleanup = cast<Function>(CH->getHandlerBlockOrFunc());
-  else
-    UME.Cleanup = nullptr;
-  FuncInfo.UnwindMap.push_back(UME);
-}
 
-void WinEHNumbering::createTryBlockMapEntry(int TryLow, int TryHigh,
-                                            ArrayRef<CatchHandler *> Handlers) {
-  WinEHTryBlockMapEntry TBME;
-  TBME.TryLow = TryLow;
-  TBME.TryHigh = TryHigh;
-  assert(TBME.TryLow <= TBME.TryHigh);
-  for (CatchHandler *CH : Handlers) {
-    WinEHHandlerType HT;
-    if (CH->getSelector()->isNullValue()) {
-      HT.Adjectives = 0x40;
-      HT.TypeDescriptor = nullptr;
-    } else {
-      auto *GV = cast<GlobalVariable>(CH->getSelector()->stripPointerCasts());
-      // Selectors are always pointers to GlobalVariables with 'struct' type.
-      // The struct has two fields, adjectives and a type descriptor.
-      auto *CS = cast<ConstantStruct>(GV->getInitializer());
-      HT.Adjectives =
-          cast<ConstantInt>(CS->getAggregateElement(0U))->getZExtValue();
-      HT.TypeDescriptor =
-          cast<GlobalVariable>(CS->getAggregateElement(1)->stripPointerCasts());
-    }
-    HT.Handler = cast<Function>(CH->getHandlerBlockOrFunc());
-    HT.CatchObjRecoverIdx = CH->getExceptionVarIndex();
-    TBME.HandlerArray.push_back(HT);
-  }
-  FuncInfo.TryBlockMap.push_back(TBME);
-}
-
-static void print_name(const Value *V) {
-#ifndef NDEBUG
-  if (!V) {
-    DEBUG(dbgs() << "null");
+  // If this is an MSVC EH personality, we need to do a bit more work.
+  EHPersonality Personality = EHPersonality::Unknown;
+  if (Fn->hasPersonalityFn())
+    Personality = classifyEHPersonality(Fn->getPersonalityFn());
+  if (!isMSVCEHPersonality(Personality))
     return;
-  }
 
-  if (const auto *F = dyn_cast<Function>(V))
-    DEBUG(dbgs() << F->getName());
-  else
-    DEBUG(V->dump());
-#endif
-}
-
-void WinEHNumbering::processCallSite(ArrayRef<ActionHandler *> Actions,
-                                     ImmutableCallSite CS) {
-  int FirstMismatch = 0;
-  for (int E = std::min(HandlerStack.size(), Actions.size()); FirstMismatch < E;
-       ++FirstMismatch) {
-    if (HandlerStack[FirstMismatch]->getHandlerBlockOrFunc() !=
-        Actions[FirstMismatch]->getHandlerBlockOrFunc())
-      break;
-    delete Actions[FirstMismatch];
-  }
-
-  bool EnteringScope = (int)Actions.size() > FirstMismatch;
-
-  // Don't recurse while we are looping over the handler stack.  Instead, defer
-  // the numbering of the catch handlers until we are done popping.
-  SmallVector<CatchHandler *, 4> PoppedCatches;
-  for (int I = HandlerStack.size() - 1; I >= FirstMismatch; --I) {
-    if (auto *CH = dyn_cast<CatchHandler>(HandlerStack.back())) {
-      PoppedCatches.push_back(CH);
-    } else {
-      // Delete cleanup handlers
-      delete HandlerStack.back();
-    }
-    HandlerStack.pop_back();
+  if (Personality == EHPersonality::MSVC_Win64SEH ||
+      Personality == EHPersonality::MSVC_X86SEH) {
+    addSEHHandlersForLPads(LPads);
   }
 
-  // We need to create a new state number if we are exiting a try scope and we
-  // will not push any more actions.
-  int TryHigh = NextState - 1;
-  if (!EnteringScope && !PoppedCatches.empty()) {
-    createUnwindMapEntry(currentEHNumber(), nullptr);
-    ++NextState;
+  WinEHFuncInfo &EHInfo = MMI.getWinEHFuncInfo(&fn);
+  if (Personality == EHPersonality::MSVC_CXX) {
+    const Function *WinEHParentFn = MMI.getWinEHParent(&fn);
+    calculateWinCXXEHStateNumbers(WinEHParentFn, EHInfo);
   }
 
-  int LastTryLowIdx = 0;
-  for (int I = 0, E = PoppedCatches.size(); I != E; ++I) {
-    CatchHandler *CH = PoppedCatches[I];
-    if (I + 1 == E || CH->getEHState() != PoppedCatches[I + 1]->getEHState()) {
-      int TryLow = CH->getEHState();
-      auto Handlers =
-          makeArrayRef(&PoppedCatches[LastTryLowIdx], I - LastTryLowIdx + 1);
-      createTryBlockMapEntry(TryLow, TryHigh, Handlers);
-      LastTryLowIdx = I + 1;
+  // Copy the state numbers to LandingPadInfo for the current function, which
+  // could be a handler or the parent. This should happen for 32-bit SEH and
+  // C++ EH.
+  if (Personality == EHPersonality::MSVC_CXX ||
+      Personality == EHPersonality::MSVC_X86SEH) {
+    for (const LandingPadInst *LP : LPads) {
+      MachineBasicBlock *LPadMBB = MBBMap[LP->getParent()];
+      MMI.addWinEHState(LPadMBB, EHInfo.LandingPadStateMap[LP]);
     }
   }
+}
 
-  for (CatchHandler *CH : PoppedCatches) {
-    if (auto *F = dyn_cast<Function>(CH->getHandlerBlockOrFunc()))
-      calculateStateNumbers(*F);
-    delete CH;
-  }
-
-  bool LastActionWasCatch = false;
-  for (size_t I = FirstMismatch; I != Actions.size(); ++I) {
-    // We can reuse eh states when pushing two catches for the same invoke.
-    bool CurrActionIsCatch = isa<CatchHandler>(Actions[I]);
-    // FIXME: Reenable this optimization!
-    if (CurrActionIsCatch && LastActionWasCatch && false) {
-      Actions[I]->setEHState(currentEHNumber());
-    } else {
-      createUnwindMapEntry(currentEHNumber(), Actions[I]);
-      Actions[I]->setEHState(NextState);
-      NextState++;
-      DEBUG(dbgs() << "Creating unwind map entry for: (");
-      print_name(Actions[I]->getHandlerBlockOrFunc());
-      DEBUG(dbgs() << ", " << currentEHNumber() << ")\n");
-    }
-    HandlerStack.push_back(Actions[I]);
-    LastActionWasCatch = CurrActionIsCatch;
-  }
+void FunctionLoweringInfo::addSEHHandlersForLPads(
+    ArrayRef<const LandingPadInst *> LPads) {
+  MachineModuleInfo &MMI = MF->getMMI();
 
-  DEBUG(dbgs() << "In EHState " << currentEHNumber() << " for CallSite: ");
-  print_name(CS ? CS.getCalledValue() : nullptr);
-  DEBUG(dbgs() << '\n');
-}
+  // Iterate over all landing pads with llvm.eh.actions calls.
+  for (const LandingPadInst *LP : LPads) {
+    const IntrinsicInst *ActionsCall =
+        dyn_cast<IntrinsicInst>(LP->getNextNode());
+    if (!ActionsCall ||
+        ActionsCall->getIntrinsicID() != Intrinsic::eh_actions)
+      continue;
 
-void WinEHNumbering::calculateStateNumbers(const Function &F) {
-  auto I = VisitedHandlers.insert(&F);
-  if (!I.second)
-    return; // We've already visited this handler, don't renumber it.
-
-  DEBUG(dbgs() << "Calculating state numbers for: " << F.getName() << '\n');
-  SmallVector<ActionHandler *, 4> ActionList;
-  for (const BasicBlock &BB : F) {
-    for (const Instruction &I : BB) {
-      const auto *CI = dyn_cast<CallInst>(&I);
-      if (!CI || CI->doesNotThrow())
-        continue;
-      processCallSite(None, CI);
+    // Parse the llvm.eh.actions call we found.
+    MachineBasicBlock *LPadMBB = MBBMap[LP->getParent()];
+    SmallVector<std::unique_ptr<ActionHandler>, 4> Actions;
+    parseEHActions(ActionsCall, Actions);
+
+    // Iterate EH actions from most to least precedence, which means
+    // iterating in reverse.
+    for (auto I = Actions.rbegin(), E = Actions.rend(); I != E; ++I) {
+      ActionHandler *Action = I->get();
+      if (auto *CH = dyn_cast<CatchHandler>(Action)) {
+        const auto *Filter =
+            dyn_cast<Function>(CH->getSelector()->stripPointerCasts());
+        assert((Filter || CH->getSelector()->isNullValue()) &&
+               "expected function or catch-all");
+        const auto *RecoverBA =
+            cast<BlockAddress>(CH->getHandlerBlockOrFunc());
+        MMI.addSEHCatchHandler(LPadMBB, Filter, RecoverBA);
+      } else {
+        assert(isa<CleanupHandler>(Action));
+        const auto *Fini = cast<Function>(Action->getHandlerBlockOrFunc());
+        MMI.addSEHCleanupHandler(LPadMBB, Fini);
+      }
     }
-    const auto *II = dyn_cast<InvokeInst>(BB.getTerminator());
-    if (!II)
-      continue;
-    const LandingPadInst *LPI = II->getLandingPadInst();
-    auto *ActionsCall = dyn_cast<IntrinsicInst>(LPI->getNextNode());
-    if (!ActionsCall)
-      continue;
-    assert(ActionsCall->getIntrinsicID() == Intrinsic::eh_actions);
-    parseEHActions(ActionsCall, ActionList);
-    processCallSite(ActionList, II);
-    ActionList.clear();
-    FuncInfo.LandingPadStateMap[LPI] = currentEHNumber();
   }
-
-  FuncInfo.CatchHandlerMaxState[&F] = NextState - 1;
 }
 
 /// clear - Clear out all the function-specific state. This returns this
@@ -466,6 +346,7 @@ void FunctionLoweringInfo::clear() {
   ByValArgFrameIndexMap.clear();
   RegFixups.clear();
   StatepointStackSlots.clear();
+  StatepointRelocatedValues.clear();
   PreferredExtendType.clear();
 }
 
@@ -486,7 +367,7 @@ unsigned FunctionLoweringInfo::CreateRegs(Type *Ty) {
   const TargetLowering *TLI = MF->getSubtarget().getTargetLowering();
 
   SmallVector<EVT, 4> ValueVTs;
-  ComputeValueVTs(*TLI, Ty, ValueVTs);
+  ComputeValueVTs(*TLI, MF->getDataLayout(), Ty, ValueVTs);
 
   unsigned FirstReg = 0;
   for (unsigned Value = 0, e = ValueVTs.size(); Value != e; ++Value) {
@@ -533,7 +414,7 @@ void FunctionLoweringInfo::ComputePHILiveOutRegInfo(const PHINode *PN) {
     return;
 
   SmallVector<EVT, 1> ValueVTs;
-  ComputeValueVTs(*TLI, Ty, ValueVTs);
+  ComputeValueVTs(*TLI, MF->getDataLayout(), Ty, ValueVTs);
   assert(ValueVTs.size() == 1 &&
          "PHIs with non-vector integer types should have a single VT.");
   EVT IntVT = ValueVTs[0];
@@ -666,8 +547,10 @@ void llvm::ComputeUsesVAFloatArgument(const CallInst &I,
 /// landingpad instruction and add them to the specified machine module info.
 void llvm::AddLandingPadInfo(const LandingPadInst &I, MachineModuleInfo &MMI,
                              MachineBasicBlock *MBB) {
-  MMI.addPersonality(MBB,
-                     cast<Function>(I.getPersonalityFn()->stripPointerCasts()));
+  MMI.addPersonality(
+      MBB,
+      cast<Function>(
+          I.getParent()->getParent()->getPersonalityFn()->stripPointerCasts()));
 
   if (I.isCleanup())
     MMI.addCleanup(MBB);
diff --git a/lib/CodeGen/SelectionDAG/InstrEmitter.cpp b/lib/CodeGen/SelectionDAG/InstrEmitter.cpp
index 64d606a2e2c..5ec10308dc2 100644
--- a/lib/CodeGen/SelectionDAG/InstrEmitter.cpp
+++ b/lib/CodeGen/SelectionDAG/InstrEmitter.cpp
@@ -406,10 +406,10 @@ void InstrEmitter::AddOperand(MachineInstrBuilder &MIB,
     Type *Type = CP->getType();
     // MachineConstantPool wants an explicit alignment.
     if (Align == 0) {
-      Align = MF->getTarget().getDataLayout()->getPrefTypeAlignment(Type);
+      Align = MF->getDataLayout().getPrefTypeAlignment(Type);
       if (Align == 0) {
         // Alignment of vector types.  FIXME!
-        Align = MF->getTarget().getDataLayout()->getTypeAllocSize(Type);
+        Align = MF->getDataLayout().getTypeAllocSize(Type);
       }
     }
 
@@ -422,6 +422,8 @@ void InstrEmitter::AddOperand(MachineInstrBuilder &MIB,
     MIB.addConstantPoolIndex(Idx, Offset, CP->getTargetFlags());
   } else if (ExternalSymbolSDNode *ES = dyn_cast<ExternalSymbolSDNode>(Op)) {
     MIB.addExternalSymbol(ES->getSymbol(), ES->getTargetFlags());
+  } else if (auto *SymNode = dyn_cast<MCSymbolSDNode>(Op)) {
+    MIB.addSym(SymNode->getMCSymbol());
   } else if (BlockAddressSDNode *BA = dyn_cast<BlockAddressSDNode>(Op)) {
     MIB.addBlockAddress(BA->getBlockAddress(),
                         BA->getOffset(),
@@ -650,7 +652,7 @@ InstrEmitter::EmitDbgValue(SDDbgValue *SD,
   MDNode *Var = SD->getVariable();
   MDNode *Expr = SD->getExpression();
   DebugLoc DL = SD->getDebugLoc();
-  assert(cast<MDLocalVariable>(Var)->isValidLocationForIntrinsic(DL) &&
+  assert(cast<DILocalVariable>(Var)->isValidLocationForIntrinsic(DL) &&
          "Expected inlined-at fields to agree");
 
   if (SD->getKind() == SDDbgValue::FRAMEIX) {
@@ -953,6 +955,9 @@ EmitSpecialNode(SDNode *Node, bool IsClone, bool IsCloned,
     // Remember to operand index of the group flags.
     SmallVector<unsigned, 8> GroupIdx;
 
+    // Remember registers that are part of early-clobber defs.
+    SmallVector<unsigned, 8> ECRegs;
+
     // Add all of the operand registers to the instruction.
     for (unsigned i = InlineAsm::Op_FirstOperand; i != NumOps;) {
       unsigned Flags =
@@ -981,6 +986,7 @@ EmitSpecialNode(SDNode *Node, bool IsClone, bool IsCloned,
           unsigned Reg = cast<RegisterSDNode>(Node->getOperand(i))->getReg();
           MIB.addReg(Reg, RegState::Define | RegState::EarlyClobber |
                   getImplRegState(TargetRegisterInfo::isPhysicalRegister(Reg)));
+          ECRegs.push_back(Reg);
         }
         break;
       case InlineAsm::Kind_RegUse:  // Use of register.
@@ -1006,6 +1012,19 @@ EmitSpecialNode(SDNode *Node, bool IsClone, bool IsCloned,
       }
     }
 
+    // GCC inline assembly allows input operands to also be early-clobber
+    // output operands (so long as the operand is written only after it's
+    // used), but this does not match the semantics of our early-clobber flag.
+    // If an early-clobber operand register is also an input operand register,
+    // then remove the early-clobber flag.
+    for (unsigned Reg : ECRegs) {
+      if (MIB->readsRegister(Reg, TRI)) {
+        MachineOperand *MO = MIB->findRegisterDefOperand(Reg, false, TRI);
+        assert(MO && "No def operand for clobbered register?");
+        MO->setIsEarlyClobber(false);
+      }
+    }
+
     // Get the mdnode from the asm if it exists and add it to the instruction.
     SDValue MDV = Node->getOperand(InlineAsm::Op_MDNode);
     const MDNode *MD = cast<MDNodeSDNode>(MDV)->getMD();
diff --git a/lib/CodeGen/SelectionDAG/InstrEmitter.h b/lib/CodeGen/SelectionDAG/InstrEmitter.h
index 7b86f7dd8de..3b24d93c74f 100644
--- a/lib/CodeGen/SelectionDAG/InstrEmitter.h
+++ b/lib/CodeGen/SelectionDAG/InstrEmitter.h
@@ -26,7 +26,7 @@ class MachineInstrBuilder;
 class MCInstrDesc;
 class SDDbgValue;
 
-class InstrEmitter {
+class LLVM_LIBRARY_VISIBILITY InstrEmitter {
   MachineFunction *MF;
   MachineRegisterInfo *MRI;
   const TargetInstrInfo *TII;
diff --git a/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp b/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp
index 4a28a4bd806..a253faa9760 100644
--- a/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp
+++ b/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp
@@ -65,7 +65,7 @@ class SelectionDAGLegalize {
   SmallSetVector<SDNode *, 16> *UpdatedNodes;
 
   EVT getSetCCResultType(EVT VT) const {
-    return TLI.getSetCCResultType(*DAG.getContext(), VT);
+    return TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT);
   }
 
   // Libcall insertion helpers.
@@ -249,7 +249,7 @@ SelectionDAGLegalize::ExpandConstantFP(ConstantFPSDNode *CFP, bool UseCP) {
   ConstantFP *LLVMC = const_cast<ConstantFP*>(CFP->getConstantFPValue());
   if (!UseCP) {
     assert((VT == MVT::f64 || VT == MVT::f32) && "Invalid type expansion");
-    return DAG.getConstant(LLVMC->getValueAPF().bitcastToAPInt(),
+    return DAG.getConstant(LLVMC->getValueAPF().bitcastToAPInt(), dl,
                            (VT == MVT::f64) ? MVT::i64 : MVT::i32);
   }
 
@@ -269,7 +269,8 @@ SelectionDAGLegalize::ExpandConstantFP(ConstantFPSDNode *CFP, bool UseCP) {
     }
   }
 
-  SDValue CPIdx = DAG.getConstantPool(LLVMC, TLI.getPointerTy());
+  SDValue CPIdx =
+      DAG.getConstantPool(LLVMC, TLI.getPointerTy(DAG.getDataLayout()));
   unsigned Alignment = cast<ConstantPoolSDNode>(CPIdx)->getAlignment();
   if (Extend) {
     SDValue Result =
@@ -331,7 +332,8 @@ static void ExpandUnalignedStore(StoreSDNode *ST, SelectionDAG &DAG,
     SDValue Store = DAG.getTruncStore(Chain, dl,
                                       Val, StackPtr, MachinePointerInfo(),
                                       StoredVT, false, false, 0);
-    SDValue Increment = DAG.getConstant(RegBytes, TLI.getPointerTy(AS));
+    SDValue Increment = DAG.getConstant(
+        RegBytes, dl, TLI.getPointerTy(DAG.getDataLayout(), AS));
     SmallVector<SDValue, 8> Stores;
     unsigned Offset = 0;
 
@@ -385,24 +387,27 @@ static void ExpandUnalignedStore(StoreSDNode *ST, SelectionDAG &DAG,
   int IncrementSize = NumBits / 8;
 
   // Divide the stored value in two parts.
-  SDValue ShiftAmount = DAG.getConstant(NumBits,
-                                      TLI.getShiftAmountTy(Val.getValueType()));
+  SDValue ShiftAmount =
+      DAG.getConstant(NumBits, dl, TLI.getShiftAmountTy(Val.getValueType(),
+                                                        DAG.getDataLayout()));
   SDValue Lo = Val;
   SDValue Hi = DAG.getNode(ISD::SRL, dl, VT, Val, ShiftAmount);
 
   // Store the two parts
   SDValue Store1, Store2;
-  Store1 = DAG.getTruncStore(Chain, dl, TLI.isLittleEndian()?Lo:Hi, Ptr,
-                             ST->getPointerInfo(), NewStoredVT,
+  Store1 = DAG.getTruncStore(Chain, dl,
+                             DAG.getDataLayout().isLittleEndian() ? Lo : Hi,
+                             Ptr, ST->getPointerInfo(), NewStoredVT,
                              ST->isVolatile(), ST->isNonTemporal(), Alignment);
 
   Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr,
-                    DAG.getConstant(IncrementSize, TLI.getPointerTy(AS)));
+                    DAG.getConstant(IncrementSize, dl,
+                                    TLI.getPointerTy(DAG.getDataLayout(), AS)));
   Alignment = MinAlign(Alignment, IncrementSize);
-  Store2 = DAG.getTruncStore(Chain, dl, TLI.isLittleEndian()?Hi:Lo, Ptr,
-                             ST->getPointerInfo().getWithOffset(IncrementSize),
-                             NewStoredVT, ST->isVolatile(), ST->isNonTemporal(),
-                             Alignment, ST->getAAInfo());
+  Store2 = DAG.getTruncStore(
+      Chain, dl, DAG.getDataLayout().isLittleEndian() ? Hi : Lo, Ptr,
+      ST->getPointerInfo().getWithOffset(IncrementSize), NewStoredVT,
+      ST->isVolatile(), ST->isNonTemporal(), Alignment, ST->getAAInfo());
 
   SDValue Result =
     DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Store1, Store2);
@@ -448,7 +453,8 @@ ExpandUnalignedLoad(LoadSDNode *LD, SelectionDAG &DAG,
     // Make sure the stack slot is also aligned for the register type.
     SDValue StackBase = DAG.CreateStackTemporary(LoadedVT, RegVT);
 
-    SDValue Increment = DAG.getConstant(RegBytes, TLI.getPointerTy());
+    SDValue Increment =
+        DAG.getConstant(RegBytes, dl, TLI.getPointerTy(DAG.getDataLayout()));
     SmallVector<SDValue, 8> Stores;
     SDValue StackPtr = StackBase;
     unsigned Offset = 0;
@@ -522,13 +528,13 @@ ExpandUnalignedLoad(LoadSDNode *LD, SelectionDAG &DAG,
 
   // Load the value in two parts
   SDValue Lo, Hi;
-  if (TLI.isLittleEndian()) {
+  if (DAG.getDataLayout().isLittleEndian()) {
     Lo = DAG.getExtLoad(ISD::ZEXTLOAD, dl, VT, Chain, Ptr, LD->getPointerInfo(),
                         NewLoadedVT, LD->isVolatile(),
                         LD->isNonTemporal(), LD->isInvariant(), Alignment,
                         LD->getAAInfo());
     Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr,
-                      DAG.getConstant(IncrementSize, Ptr.getValueType()));
+                      DAG.getConstant(IncrementSize, dl, Ptr.getValueType()));
     Hi = DAG.getExtLoad(HiExtType, dl, VT, Chain, Ptr,
                         LD->getPointerInfo().getWithOffset(IncrementSize),
                         NewLoadedVT, LD->isVolatile(),
@@ -540,7 +546,7 @@ ExpandUnalignedLoad(LoadSDNode *LD, SelectionDAG &DAG,
                         LD->isNonTemporal(), LD->isInvariant(), Alignment,
                         LD->getAAInfo());
     Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr,
-                      DAG.getConstant(IncrementSize, Ptr.getValueType()));
+                      DAG.getConstant(IncrementSize, dl, Ptr.getValueType()));
     Lo = DAG.getExtLoad(ISD::ZEXTLOAD, dl, VT, Chain, Ptr,
                         LD->getPointerInfo().getWithOffset(IncrementSize),
                         NewLoadedVT, LD->isVolatile(),
@@ -549,8 +555,9 @@ ExpandUnalignedLoad(LoadSDNode *LD, SelectionDAG &DAG,
   }
 
   // aggregate the two parts
-  SDValue ShiftAmount = DAG.getConstant(NumBits,
-                                       TLI.getShiftAmountTy(Hi.getValueType()));
+  SDValue ShiftAmount =
+      DAG.getConstant(NumBits, dl, TLI.getShiftAmountTy(Hi.getValueType(),
+                                                        DAG.getDataLayout()));
   SDValue Result = DAG.getNode(ISD::SHL, dl, VT, Hi, ShiftAmount);
   Result = DAG.getNode(ISD::OR, dl, VT, Result, Lo);
 
@@ -581,7 +588,7 @@ PerformInsertVectorEltInMemory(SDValue Vec, SDValue Val, SDValue Idx,
   EVT VT    = Tmp1.getValueType();
   EVT EltVT = VT.getVectorElementType();
   EVT IdxVT = Tmp3.getValueType();
-  EVT PtrVT = TLI.getPointerTy();
+  EVT PtrVT = TLI.getPointerTy(DAG.getDataLayout());
   SDValue StackPtr = DAG.CreateStackTemporary(VT);
 
   int SPFI = cast<FrameIndexSDNode>(StackPtr.getNode())->getIndex();
@@ -592,11 +599,11 @@ PerformInsertVectorEltInMemory(SDValue Vec, SDValue Val, SDValue Idx,
                             false, false, 0);
 
   // Truncate or zero extend offset to target pointer type.
-  unsigned CastOpc = IdxVT.bitsGT(PtrVT) ? ISD::TRUNCATE : ISD::ZERO_EXTEND;
-  Tmp3 = DAG.getNode(CastOpc, dl, PtrVT, Tmp3);
+  Tmp3 = DAG.getZExtOrTrunc(Tmp3, dl, PtrVT);
   // Add the offset to the index.
   unsigned EltSize = EltVT.getSizeInBits()/8;
-  Tmp3 = DAG.getNode(ISD::MUL, dl, IdxVT, Tmp3,DAG.getConstant(EltSize, IdxVT));
+  Tmp3 = DAG.getNode(ISD::MUL, dl, IdxVT, Tmp3,
+                     DAG.getConstant(EltSize, dl, IdxVT));
   SDValue StackPtr2 = DAG.getNode(ISD::ADD, dl, IdxVT, Tmp3, StackPtr);
   // Store the scalar value.
   Ch = DAG.getTruncStore(Ch, dl, Tmp2, StackPtr2, MachinePointerInfo(), EltVT,
@@ -655,7 +662,7 @@ SDValue SelectionDAGLegalize::OptimizeFloatStore(StoreSDNode* ST) {
         TLI.isTypeLegal(MVT::i32)) {
       SDValue Con = DAG.getConstant(CFP->getValueAPF().
                                       bitcastToAPInt().zextOrTrunc(32),
-                              MVT::i32);
+                                    SDLoc(CFP), MVT::i32);
       return DAG.getStore(Chain, dl, Con, Ptr, ST->getPointerInfo(),
                           isVolatile, isNonTemporal, Alignment, AAInfo);
     }
@@ -664,7 +671,7 @@ SDValue SelectionDAGLegalize::OptimizeFloatStore(StoreSDNode* ST) {
       // If this target supports 64-bit registers, do a single 64-bit store.
       if (TLI.isTypeLegal(MVT::i64)) {
         SDValue Con = DAG.getConstant(CFP->getValueAPF().bitcastToAPInt().
-                                  zextOrTrunc(64), MVT::i64);
+                                      zextOrTrunc(64), SDLoc(CFP), MVT::i64);
         return DAG.getStore(Chain, dl, Con, Ptr, ST->getPointerInfo(),
                             isVolatile, isNonTemporal, Alignment, AAInfo);
       }
@@ -673,15 +680,16 @@ SDValue SelectionDAGLegalize::OptimizeFloatStore(StoreSDNode* ST) {
         // Otherwise, if the target supports 32-bit registers, use 2 32-bit
         // stores.  If the target supports neither 32- nor 64-bits, this
         // xform is certainly not worth it.
-        const APInt &IntVal =CFP->getValueAPF().bitcastToAPInt();
-        SDValue Lo = DAG.getConstant(IntVal.trunc(32), MVT::i32);
-        SDValue Hi = DAG.getConstant(IntVal.lshr(32).trunc(32), MVT::i32);
-        if (TLI.isBigEndian()) std::swap(Lo, Hi);
+        const APInt &IntVal = CFP->getValueAPF().bitcastToAPInt();
+        SDValue Lo = DAG.getConstant(IntVal.trunc(32), dl, MVT::i32);
+        SDValue Hi = DAG.getConstant(IntVal.lshr(32).trunc(32), dl, MVT::i32);
+        if (DAG.getDataLayout().isBigEndian())
+          std::swap(Lo, Hi);
 
         Lo = DAG.getStore(Chain, dl, Lo, Ptr, ST->getPointerInfo(), isVolatile,
                           isNonTemporal, Alignment, AAInfo);
         Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr,
-                          DAG.getConstant(4, Ptr.getValueType()));
+                          DAG.getConstant(4, dl, Ptr.getValueType()));
         Hi = DAG.getStore(Chain, dl, Hi, Ptr,
                           ST->getPointerInfo().getWithOffset(4),
                           isVolatile, isNonTemporal, MinAlign(Alignment, 4U),
@@ -719,14 +727,12 @@ void SelectionDAGLegalize::LegalizeStoreOps(SDNode *Node) {
         case TargetLowering::Legal: {
           // If this is an unaligned store and the target doesn't support it,
           // expand it.
+          EVT MemVT = ST->getMemoryVT();
           unsigned AS = ST->getAddressSpace();
           unsigned Align = ST->getAlignment();
-          if (!TLI.allowsMisalignedMemoryAccesses(ST->getMemoryVT(), AS, Align)) {
-            Type *Ty = ST->getMemoryVT().getTypeForEVT(*DAG.getContext());
-            unsigned ABIAlignment= TLI.getDataLayout()->getABITypeAlignment(Ty);
-            if (Align < ABIAlignment)
-              ExpandUnalignedStore(cast<StoreSDNode>(Node), DAG, TLI, this);
-          }
+          const DataLayout &DL = DAG.getDataLayout();
+          if (!TLI.allowsMemoryAccess(*DAG.getContext(), DL, MemVT, AS, Align))
+            ExpandUnalignedStore(cast<StoreSDNode>(Node), DAG, TLI, this);
           break;
         }
         case TargetLowering::Custom: {
@@ -755,6 +761,7 @@ void SelectionDAGLegalize::LegalizeStoreOps(SDNode *Node) {
 
       EVT StVT = ST->getMemoryVT();
       unsigned StWidth = StVT.getSizeInBits();
+      auto &DL = DAG.getDataLayout();
 
       if (StWidth != StVT.getStoreSizeInBits()) {
         // Promote to a byte-sized store with upper bits zero if not
@@ -781,7 +788,7 @@ void SelectionDAGLegalize::LegalizeStoreOps(SDNode *Node) {
         SDValue Lo, Hi;
         unsigned IncrementSize;
 
-        if (TLI.isLittleEndian()) {
+        if (DL.isLittleEndian()) {
           // TRUNCSTORE:i24 X -> TRUNCSTORE:i16 X, TRUNCSTORE@+2:i8 (srl X, 16)
           // Store the bottom RoundWidth bits.
           Lo = DAG.getTruncStore(Chain, dl, Value, Ptr, ST->getPointerInfo(),
@@ -792,10 +799,12 @@ void SelectionDAGLegalize::LegalizeStoreOps(SDNode *Node) {
           // Store the remaining ExtraWidth bits.
           IncrementSize = RoundWidth / 8;
           Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr,
-                            DAG.getConstant(IncrementSize, Ptr.getValueType()));
-          Hi = DAG.getNode(ISD::SRL, dl, Value.getValueType(), Value,
-                           DAG.getConstant(RoundWidth,
-                                   TLI.getShiftAmountTy(Value.getValueType())));
+                            DAG.getConstant(IncrementSize, dl,
+                                            Ptr.getValueType()));
+          Hi = DAG.getNode(
+              ISD::SRL, dl, Value.getValueType(), Value,
+              DAG.getConstant(RoundWidth, dl,
+                              TLI.getShiftAmountTy(Value.getValueType(), DL)));
           Hi = DAG.getTruncStore(Chain, dl, Hi, Ptr,
                              ST->getPointerInfo().getWithOffset(IncrementSize),
                                  ExtraVT, isVolatile, isNonTemporal,
@@ -804,9 +813,10 @@ void SelectionDAGLegalize::LegalizeStoreOps(SDNode *Node) {
           // Big endian - avoid unaligned stores.
           // TRUNCSTORE:i24 X -> TRUNCSTORE:i16 (srl X, 8), TRUNCSTORE@+2:i8 X
           // Store the top RoundWidth bits.
-          Hi = DAG.getNode(ISD::SRL, dl, Value.getValueType(), Value,
-                           DAG.getConstant(ExtraWidth,
-                                   TLI.getShiftAmountTy(Value.getValueType())));
+          Hi = DAG.getNode(
+              ISD::SRL, dl, Value.getValueType(), Value,
+              DAG.getConstant(ExtraWidth, dl,
+                              TLI.getShiftAmountTy(Value.getValueType(), DL)));
           Hi = DAG.getTruncStore(Chain, dl, Hi, Ptr, ST->getPointerInfo(),
                                  RoundVT, isVolatile, isNonTemporal, Alignment,
                                  AAInfo);
@@ -814,7 +824,8 @@ void SelectionDAGLegalize::LegalizeStoreOps(SDNode *Node) {
           // Store the remaining ExtraWidth bits.
           IncrementSize = RoundWidth / 8;
           Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr,
-                            DAG.getConstant(IncrementSize, Ptr.getValueType()));
+                            DAG.getConstant(IncrementSize, dl,
+                                            Ptr.getValueType()));
           Lo = DAG.getTruncStore(Chain, dl, Value, Ptr,
                               ST->getPointerInfo().getWithOffset(IncrementSize),
                                  ExtraVT, isVolatile, isNonTemporal,
@@ -829,16 +840,13 @@ void SelectionDAGLegalize::LegalizeStoreOps(SDNode *Node) {
                                         StVT.getSimpleVT())) {
         default: llvm_unreachable("This action is not supported yet!");
         case TargetLowering::Legal: {
+          EVT MemVT = ST->getMemoryVT();
           unsigned AS = ST->getAddressSpace();
           unsigned Align = ST->getAlignment();
           // If this is an unaligned store and the target doesn't support it,
           // expand it.
-          if (!TLI.allowsMisalignedMemoryAccesses(ST->getMemoryVT(), AS, Align)) {
-            Type *Ty = ST->getMemoryVT().getTypeForEVT(*DAG.getContext());
-            unsigned ABIAlignment= TLI.getDataLayout()->getABITypeAlignment(Ty);
-            if (Align < ABIAlignment)
-              ExpandUnalignedStore(cast<StoreSDNode>(Node), DAG, TLI, this);
-          }
+          if (!TLI.allowsMemoryAccess(*DAG.getContext(), DL, MemVT, AS, Align))
+            ExpandUnalignedStore(cast<StoreSDNode>(Node), DAG, TLI, this);
           break;
         }
         case TargetLowering::Custom: {
@@ -881,18 +889,14 @@ void SelectionDAGLegalize::LegalizeLoadOps(SDNode *Node) {
     switch (TLI.getOperationAction(Node->getOpcode(), VT)) {
     default: llvm_unreachable("This action is not supported yet!");
     case TargetLowering::Legal: {
+      EVT MemVT = LD->getMemoryVT();
       unsigned AS = LD->getAddressSpace();
       unsigned Align = LD->getAlignment();
+      const DataLayout &DL = DAG.getDataLayout();
       // If this is an unaligned load and the target doesn't support it,
       // expand it.
-      if (!TLI.allowsMisalignedMemoryAccesses(LD->getMemoryVT(), AS, Align)) {
-        Type *Ty = LD->getMemoryVT().getTypeForEVT(*DAG.getContext());
-        unsigned ABIAlignment =
-          TLI.getDataLayout()->getABITypeAlignment(Ty);
-        if (Align < ABIAlignment){
-          ExpandUnalignedLoad(cast<LoadSDNode>(Node), DAG, TLI, RVal, RChain);
-        }
-      }
+      if (!TLI.allowsMemoryAccess(*DAG.getContext(), DL, MemVT, AS, Align))
+        ExpandUnalignedLoad(cast<LoadSDNode>(Node), DAG, TLI, RVal, RChain);
       break;
     }
     case TargetLowering::Custom: {
@@ -992,8 +996,9 @@ void SelectionDAGLegalize::LegalizeLoadOps(SDNode *Node) {
     EVT ExtraVT = EVT::getIntegerVT(*DAG.getContext(), ExtraWidth);
     SDValue Lo, Hi, Ch;
     unsigned IncrementSize;
+    auto &DL = DAG.getDataLayout();
 
-    if (TLI.isLittleEndian()) {
+    if (DL.isLittleEndian()) {
       // EXTLOAD:i24 -> ZEXTLOAD:i16 | (shl EXTLOAD@+2:i8, 16)
       // Load the bottom RoundWidth bits.
       Lo = DAG.getExtLoad(ISD::ZEXTLOAD, dl, Node->getValueType(0),
@@ -1004,7 +1009,8 @@ void SelectionDAGLegalize::LegalizeLoadOps(SDNode *Node) {
       // Load the remaining ExtraWidth bits.
       IncrementSize = RoundWidth / 8;
       Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr,
-                         DAG.getConstant(IncrementSize, Ptr.getValueType()));
+                         DAG.getConstant(IncrementSize, dl,
+                                         Ptr.getValueType()));
       Hi = DAG.getExtLoad(ExtType, dl, Node->getValueType(0), Chain, Ptr,
                           LD->getPointerInfo().getWithOffset(IncrementSize),
                           ExtraVT, isVolatile, isNonTemporal, isInvariant,
@@ -1016,9 +1022,10 @@ void SelectionDAGLegalize::LegalizeLoadOps(SDNode *Node) {
                        Hi.getValue(1));
 
       // Move the top bits to the right place.
-      Hi = DAG.getNode(ISD::SHL, dl, Hi.getValueType(), Hi,
-                       DAG.getConstant(RoundWidth,
-                                      TLI.getShiftAmountTy(Hi.getValueType())));
+      Hi = DAG.getNode(
+          ISD::SHL, dl, Hi.getValueType(), Hi,
+          DAG.getConstant(RoundWidth, dl,
+                          TLI.getShiftAmountTy(Hi.getValueType(), DL)));
 
       // Join the hi and lo parts.
       Value = DAG.getNode(ISD::OR, dl, Node->getValueType(0), Lo, Hi);
@@ -1033,7 +1040,8 @@ void SelectionDAGLegalize::LegalizeLoadOps(SDNode *Node) {
       // Load the remaining ExtraWidth bits.
       IncrementSize = RoundWidth / 8;
       Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr,
-                         DAG.getConstant(IncrementSize, Ptr.getValueType()));
+                         DAG.getConstant(IncrementSize, dl,
+                                         Ptr.getValueType()));
       Lo = DAG.getExtLoad(ISD::ZEXTLOAD,
                           dl, Node->getValueType(0), Chain, Ptr,
                           LD->getPointerInfo().getWithOffset(IncrementSize),
@@ -1046,9 +1054,10 @@ void SelectionDAGLegalize::LegalizeLoadOps(SDNode *Node) {
                        Hi.getValue(1));
 
       // Move the top bits to the right place.
-      Hi = DAG.getNode(ISD::SHL, dl, Hi.getValueType(), Hi,
-                       DAG.getConstant(ExtraWidth,
-                                      TLI.getShiftAmountTy(Hi.getValueType())));
+      Hi = DAG.getNode(
+          ISD::SHL, dl, Hi.getValueType(), Hi,
+          DAG.getConstant(ExtraWidth, dl,
+                          TLI.getShiftAmountTy(Hi.getValueType(), DL)));
 
       // Join the hi and lo parts.
       Value = DAG.getNode(ISD::OR, dl, Node->getValueType(0), Lo, Hi);
@@ -1074,18 +1083,14 @@ void SelectionDAGLegalize::LegalizeLoadOps(SDNode *Node) {
           Chain = Res.getValue(1);
         }
       } else {
-        // If this is an unaligned load and the target doesn't support
-        // it, expand it.
+        // If this is an unaligned load and the target doesn't support it,
+        // expand it.
         EVT MemVT = LD->getMemoryVT();
         unsigned AS = LD->getAddressSpace();
         unsigned Align = LD->getAlignment();
-        if (!TLI.allowsMisalignedMemoryAccesses(MemVT, AS, Align)) {
-          Type *Ty = LD->getMemoryVT().getTypeForEVT(*DAG.getContext());
-          unsigned ABIAlignment = TLI.getDataLayout()->getABITypeAlignment(Ty);
-          if (Align < ABIAlignment){
-            ExpandUnalignedLoad(cast<LoadSDNode>(Node), DAG, TLI, Value, Chain);
-          }
-        }
+        const DataLayout &DL = DAG.getDataLayout();
+        if (!TLI.allowsMemoryAccess(*DAG.getContext(), DL, MemVT, AS, Align))
+          ExpandUnalignedLoad(cast<LoadSDNode>(Node), DAG, TLI, Value, Chain);
       }
       break;
     }
@@ -1160,17 +1165,18 @@ void SelectionDAGLegalize::LegalizeOp(SDNode *Node) {
   if (Node->getOpcode() == ISD::TargetConstant) // Allow illegal target nodes.
     return;
 
+#ifndef NDEBUG
   for (unsigned i = 0, e = Node->getNumValues(); i != e; ++i)
     assert(TLI.getTypeAction(*DAG.getContext(), Node->getValueType(i)) ==
              TargetLowering::TypeLegal &&
            "Unexpected illegal type!");
 
-  for (unsigned i = 0, e = Node->getNumOperands(); i != e; ++i)
+  for (const SDValue &Op : Node->op_values())
     assert((TLI.getTypeAction(*DAG.getContext(),
-                              Node->getOperand(i).getValueType()) ==
-              TargetLowering::TypeLegal ||
-            Node->getOperand(i).getOpcode() == ISD::TargetConstant) &&
-           "Unexpected illegal type!");
+                              Op.getValueType()) == TargetLowering::TypeLegal ||
+                              Op.getOpcode() == ISD::TargetConstant) &&
+                              "Unexpected illegal type!");
+#endif
 
   // Figure out the correct action; the way to query this varies by opcode
   TargetLowering::LegalizeAction Action = TargetLowering::Legal;
@@ -1240,18 +1246,13 @@ void SelectionDAGLegalize::LegalizeOp(SDNode *Node) {
     break;
   case ISD::EXTRACT_ELEMENT:
   case ISD::FLT_ROUNDS_:
-  case ISD::SADDO:
-  case ISD::SSUBO:
-  case ISD::UADDO:
-  case ISD::USUBO:
-  case ISD::SMULO:
-  case ISD::UMULO:
   case ISD::FPOWI:
   case ISD::MERGE_VALUES:
   case ISD::EH_RETURN:
   case ISD::FRAME_TO_ARGS_OFFSET:
   case ISD::EH_SJLJ_SETJMP:
   case ISD::EH_SJLJ_LONGJMP:
+  case ISD::EH_SJLJ_SETUP_DISPATCH:
     // These operations lie about being legal: when they claim to be legal,
     // they should actually be expanded.
     Action = TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0));
@@ -1437,9 +1438,9 @@ SDValue SelectionDAGLegalize::ExpandExtractFromVectorThroughStack(SDValue Op) {
   unsigned EltSize =
       Vec.getValueType().getVectorElementType().getSizeInBits()/8;
   Idx = DAG.getNode(ISD::MUL, dl, Idx.getValueType(), Idx,
-                    DAG.getConstant(EltSize, Idx.getValueType()));
+                    DAG.getConstant(EltSize, SDLoc(Vec), Idx.getValueType()));
 
-  Idx = DAG.getZExtOrTrunc(Idx, dl, TLI.getPointerTy());
+  Idx = DAG.getZExtOrTrunc(Idx, dl, TLI.getPointerTy(DAG.getDataLayout()));
   StackPtr = DAG.getNode(ISD::ADD, dl, Idx.getValueType(), Idx, StackPtr);
 
   SDValue NewLoad;
@@ -1490,8 +1491,8 @@ SDValue SelectionDAGLegalize::ExpandInsertToVectorThroughStack(SDValue Op) {
       Vec.getValueType().getVectorElementType().getSizeInBits()/8;
 
   Idx = DAG.getNode(ISD::MUL, dl, Idx.getValueType(), Idx,
-                    DAG.getConstant(EltSize, Idx.getValueType()));
-  Idx = DAG.getZExtOrTrunc(Idx, dl, TLI.getPointerTy());
+                    DAG.getConstant(EltSize, SDLoc(Vec), Idx.getValueType()));
+  Idx = DAG.getZExtOrTrunc(Idx, dl, TLI.getPointerTy(DAG.getDataLayout()));
 
   SDValue SubStackPtr = DAG.getNode(ISD::ADD, dl, Idx.getValueType(), Idx,
                                     StackPtr);
@@ -1527,7 +1528,7 @@ SDValue SelectionDAGLegalize::ExpandVectorBuildThroughStack(SDNode* Node) {
 
     unsigned Offset = TypeByteSize*i;
 
-    SDValue Idx = DAG.getConstant(Offset, FIPtr.getValueType());
+    SDValue Idx = DAG.getConstant(Offset, dl, FIPtr.getValueType());
     Idx = DAG.getNode(ISD::ADD, dl, FIPtr.getValueType(), FIPtr, Idx);
 
     // If the destination vector element type is narrower than the source
@@ -1569,15 +1570,16 @@ SDValue SelectionDAGLegalize::ExpandFCOPYSIGN(SDNode* Node) {
     // Convert to an integer with the same sign bit.
     SignBit = DAG.getNode(ISD::BITCAST, dl, IVT, Tmp2);
   } else {
+    auto &DL = DAG.getDataLayout();
     // Store the float to memory, then load the sign part out as an integer.
-    MVT LoadTy = TLI.getPointerTy();
+    MVT LoadTy = TLI.getPointerTy(DL);
     // First create a temporary that is aligned for both the load and store.
     SDValue StackPtr = DAG.CreateStackTemporary(FloatVT, LoadTy);
     // Then store the float to it.
     SDValue Ch =
       DAG.getStore(DAG.getEntryNode(), dl, Tmp2, StackPtr, MachinePointerInfo(),
                    false, false, 0);
-    if (TLI.isBigEndian()) {
+    if (DL.isBigEndian()) {
       assert(FloatVT.isByteSized() && "Unsupported floating point type!");
       // Load out a legal integer with the same sign bit as the float.
       SignBit = DAG.getLoad(LoadTy, dl, Ch, StackPtr, MachinePointerInfo(),
@@ -1589,7 +1591,8 @@ SDValue SelectionDAGLegalize::ExpandFCOPYSIGN(SDNode* Node) {
       unsigned Strides = (FloatVT.getSizeInBits()-1)/LoadTy.getSizeInBits();
       unsigned ByteOffset = (Strides * LoadTy.getSizeInBits()) / 8;
       LoadPtr = DAG.getNode(ISD::ADD, dl, LoadPtr.getValueType(), LoadPtr,
-                           DAG.getConstant(ByteOffset, LoadPtr.getValueType()));
+                           DAG.getConstant(ByteOffset, dl,
+                                           LoadPtr.getValueType()));
       // Load a legal integer containing the sign bit.
       SignBit = DAG.getLoad(LoadTy, dl, Ch, LoadPtr, MachinePointerInfo(),
                             false, false, false, 0);
@@ -1598,14 +1601,16 @@ SDValue SelectionDAGLegalize::ExpandFCOPYSIGN(SDNode* Node) {
         (FloatVT.getSizeInBits() - 8 * ByteOffset);
       assert(BitShift < LoadTy.getSizeInBits() && "Pointer advanced wrong?");
       if (BitShift)
-        SignBit = DAG.getNode(ISD::SHL, dl, LoadTy, SignBit,
-                              DAG.getConstant(BitShift,
-                                 TLI.getShiftAmountTy(SignBit.getValueType())));
+        SignBit = DAG.getNode(
+            ISD::SHL, dl, LoadTy, SignBit,
+            DAG.getConstant(BitShift, dl,
+                            TLI.getShiftAmountTy(SignBit.getValueType(), DL)));
     }
   }
   // Now get the sign bit proper, by seeing whether the value is negative.
   SignBit = DAG.getSetCC(dl, getSetCCResultType(SignBit.getValueType()),
-                         SignBit, DAG.getConstant(0, SignBit.getValueType()),
+                         SignBit,
+                         DAG.getConstant(0, dl, SignBit.getValueType()),
                          ISD::SETLT);
   // Get the absolute value of the result.
   SDValue AbsVal = DAG.getNode(ISD::FABS, dl, Tmp1.getValueType(), Tmp1);
@@ -1630,8 +1635,7 @@ void SelectionDAGLegalize::ExpandDYNAMIC_STACKALLOC(SDNode* Node,
 
   // Chain the dynamic stack allocation so that it doesn't modify the stack
   // pointer when other instructions are using the stack.
-  Chain = DAG.getCALLSEQ_START(Chain, DAG.getIntPtrConstant(0, true),
-                               SDLoc(Node));
+  Chain = DAG.getCALLSEQ_START(Chain, DAG.getIntPtrConstant(0, dl, true), dl);
 
   SDValue Size  = Tmp2.getOperand(1);
   SDValue SP = DAG.getCopyFromReg(Chain, dl, SPReg, VT);
@@ -1642,12 +1646,11 @@ void SelectionDAGLegalize::ExpandDYNAMIC_STACKALLOC(SDNode* Node,
   Tmp1 = DAG.getNode(ISD::SUB, dl, VT, SP, Size);       // Value
   if (Align > StackAlign)
     Tmp1 = DAG.getNode(ISD::AND, dl, VT, Tmp1,
-                       DAG.getConstant(-(uint64_t)Align, VT));
+                       DAG.getConstant(-(uint64_t)Align, dl, VT));
   Chain = DAG.getCopyToReg(Chain, dl, SPReg, Tmp1);     // Output chain
 
-  Tmp2 = DAG.getCALLSEQ_END(Chain,  DAG.getIntPtrConstant(0, true),
-                            DAG.getIntPtrConstant(0, true), SDValue(),
-                            SDLoc(Node));
+  Tmp2 = DAG.getCALLSEQ_END(Chain, DAG.getIntPtrConstant(0, dl, true),
+                            DAG.getIntPtrConstant(0, dl, true), SDValue(), dl);
 
   Results.push_back(Tmp1);
   Results.push_back(Tmp2);
@@ -1777,9 +1780,8 @@ SDValue SelectionDAGLegalize::EmitStackConvert(SDValue SrcOp,
                                                EVT DestVT,
                                                SDLoc dl) {
   // Create the stack frame object.
-  unsigned SrcAlign =
-    TLI.getDataLayout()->getPrefTypeAlignment(SrcOp.getValueType().
-                                              getTypeForEVT(*DAG.getContext()));
+  unsigned SrcAlign = DAG.getDataLayout().getPrefTypeAlignment(
+      SrcOp.getValueType().getTypeForEVT(*DAG.getContext()));
   SDValue FIPtr = DAG.CreateStackTemporary(SlotVT, SrcAlign);
 
   FrameIndexSDNode *StackPtrFI = cast<FrameIndexSDNode>(FIPtr);
@@ -1790,7 +1792,7 @@ SDValue SelectionDAGLegalize::EmitStackConvert(SDValue SrcOp,
   unsigned SlotSize = SlotVT.getSizeInBits();
   unsigned DestSize = DestVT.getSizeInBits();
   Type *DestType = DestVT.getTypeForEVT(*DAG.getContext());
-  unsigned DestAlign = TLI.getDataLayout()->getPrefTypeAlignment(DestType);
+  unsigned DestAlign = DAG.getDataLayout().getPrefTypeAlignment(DestType);
 
   // Emit a store to the stack slot.  Use a truncstore if the input value is
   // later than DestVT.
@@ -1994,7 +1996,8 @@ SDValue SelectionDAGLegalize::ExpandBUILD_VECTOR(SDNode *Node) {
       }
     }
     Constant *CP = ConstantVector::get(CV);
-    SDValue CPIdx = DAG.getConstantPool(CP, TLI.getPointerTy());
+    SDValue CPIdx =
+        DAG.getConstantPool(CP, TLI.getPointerTy(DAG.getDataLayout()));
     unsigned Alignment = cast<ConstantPoolSDNode>(CPIdx)->getAlignment();
     return DAG.getLoad(VT, dl, DAG.getEntryNode(), CPIdx,
                        MachinePointerInfo::getConstantPool(),
@@ -2048,16 +2051,17 @@ SDValue SelectionDAGLegalize::ExpandLibCall(RTLIB::Libcall LC, SDNode *Node,
                                             bool isSigned) {
   TargetLowering::ArgListTy Args;
   TargetLowering::ArgListEntry Entry;
-  for (unsigned i = 0, e = Node->getNumOperands(); i != e; ++i) {
-    EVT ArgVT = Node->getOperand(i).getValueType();
+  for (const SDValue &Op : Node->op_values()) {
+    EVT ArgVT = Op.getValueType();
     Type *ArgTy = ArgVT.getTypeForEVT(*DAG.getContext());
-    Entry.Node = Node->getOperand(i); Entry.Ty = ArgTy;
+    Entry.Node = Op;
+    Entry.Ty = ArgTy;
     Entry.isSExt = isSigned;
     Entry.isZExt = !isSigned;
     Args.push_back(Entry);
   }
   SDValue Callee = DAG.getExternalSymbol(TLI.getLibcallName(LC),
-                                         TLI.getPointerTy());
+                                         TLI.getPointerTy(DAG.getDataLayout()));
 
   Type *RetTy = Node->getValueType(0).getTypeForEVT(*DAG.getContext());
 
@@ -2105,7 +2109,7 @@ SDValue SelectionDAGLegalize::ExpandLibCall(RTLIB::Libcall LC, EVT RetVT,
     Args.push_back(Entry);
   }
   SDValue Callee = DAG.getExternalSymbol(TLI.getLibcallName(LC),
-                                         TLI.getPointerTy());
+                                         TLI.getPointerTy(DAG.getDataLayout()));
 
   Type *RetTy = RetVT.getTypeForEVT(*DAG.getContext());
 
@@ -2139,7 +2143,7 @@ SelectionDAGLegalize::ExpandChainLibCall(RTLIB::Libcall LC,
     Args.push_back(Entry);
   }
   SDValue Callee = DAG.getExternalSymbol(TLI.getLibcallName(LC),
-                                         TLI.getPointerTy());
+                                         TLI.getPointerTy(DAG.getDataLayout()));
 
   Type *RetTy = Node->getValueType(0).getTypeForEVT(*DAG.getContext());
 
@@ -2257,10 +2261,11 @@ SelectionDAGLegalize::ExpandDivRemLibCall(SDNode *Node,
 
   TargetLowering::ArgListTy Args;
   TargetLowering::ArgListEntry Entry;
-  for (unsigned i = 0, e = Node->getNumOperands(); i != e; ++i) {
-    EVT ArgVT = Node->getOperand(i).getValueType();
+  for (const SDValue &Op : Node->op_values()) {
+    EVT ArgVT = Op.getValueType();
     Type *ArgTy = ArgVT.getTypeForEVT(*DAG.getContext());
-    Entry.Node = Node->getOperand(i); Entry.Ty = ArgTy;
+    Entry.Node = Op;
+    Entry.Ty = ArgTy;
     Entry.isSExt = isSigned;
     Entry.isZExt = !isSigned;
     Args.push_back(Entry);
@@ -2275,7 +2280,7 @@ SelectionDAGLegalize::ExpandDivRemLibCall(SDNode *Node,
   Args.push_back(Entry);
 
   SDValue Callee = DAG.getExternalSymbol(TLI.getLibcallName(LC),
-                                         TLI.getPointerTy());
+                                         TLI.getPointerTy(DAG.getDataLayout()));
 
   SDLoc dl(Node);
   TargetLowering::CallLoweringInfo CLI(DAG);
@@ -2387,7 +2392,7 @@ SelectionDAGLegalize::ExpandSinCosLibCall(SDNode *Node,
   Args.push_back(Entry);
 
   SDValue Callee = DAG.getExternalSymbol(TLI.getLibcallName(LC),
-                                         TLI.getPointerTy());
+                                         TLI.getPointerTy(DAG.getDataLayout()));
 
   SDLoc dl(Node);
   TargetLowering::CallLoweringInfo CLI(DAG);
@@ -2418,19 +2423,20 @@ SDValue SelectionDAGLegalize::ExpandLegalINT_TO_FP(bool isSigned,
     SDValue StackSlot = DAG.CreateStackTemporary(MVT::f64);
 
     // word offset constant for Hi/Lo address computation
-    SDValue WordOff = DAG.getConstant(sizeof(int), StackSlot.getValueType());
+    SDValue WordOff = DAG.getConstant(sizeof(int), dl,
+                                      StackSlot.getValueType());
     // set up Hi and Lo (into buffer) address based on endian
     SDValue Hi = StackSlot;
     SDValue Lo = DAG.getNode(ISD::ADD, dl, StackSlot.getValueType(),
                              StackSlot, WordOff);
-    if (TLI.isLittleEndian())
+    if (DAG.getDataLayout().isLittleEndian())
       std::swap(Hi, Lo);
 
     // if signed map to unsigned space
     SDValue Op0Mapped;
     if (isSigned) {
       // constant used to invert sign bit (signed to unsigned mapping)
-      SDValue SignBit = DAG.getConstant(0x80000000u, MVT::i32);
+      SDValue SignBit = DAG.getConstant(0x80000000u, dl, MVT::i32);
       Op0Mapped = DAG.getNode(ISD::XOR, dl, MVT::i32, Op0, SignBit);
     } else {
       Op0Mapped = Op0;
@@ -2440,7 +2446,7 @@ SDValue SelectionDAGLegalize::ExpandLegalINT_TO_FP(bool isSigned,
                                   Op0Mapped, Lo, MachinePointerInfo(),
                                   false, false, 0);
     // initial hi portion of constructed double
-    SDValue InitialHi = DAG.getConstant(0x43300000u, MVT::i32);
+    SDValue InitialHi = DAG.getConstant(0x43300000u, dl, MVT::i32);
     // store the hi of the constructed double - biased exponent
     SDValue Store2 = DAG.getStore(Store1, dl, InitialHi, Hi,
                                   MachinePointerInfo(),
@@ -2452,7 +2458,7 @@ SDValue SelectionDAGLegalize::ExpandLegalINT_TO_FP(bool isSigned,
     SDValue Bias = DAG.getConstantFP(isSigned ?
                                      BitsToDouble(0x4330000080000000ULL) :
                                      BitsToDouble(0x4330000000000000ULL),
-                                     MVT::f64);
+                                     dl, MVT::f64);
     // subtract the bias
     SDValue Sub = DAG.getNode(ISD::FSUB, dl, MVT::f64, Load, Bias);
     // final result
@@ -2463,7 +2469,7 @@ SDValue SelectionDAGLegalize::ExpandLegalINT_TO_FP(bool isSigned,
       Result = Sub;
     } else if (DestVT.bitsLT(MVT::f64)) {
       Result = DAG.getNode(ISD::FP_ROUND, dl, DestVT, Sub,
-                           DAG.getIntPtrConstant(0));
+                           DAG.getIntPtrConstant(0, dl));
     } else if (DestVT.bitsGT(MVT::f64)) {
       Result = DAG.getNode(ISD::FP_EXTEND, dl, DestVT, Sub);
     }
@@ -2479,15 +2485,16 @@ SDValue SelectionDAGLegalize::ExpandLegalINT_TO_FP(bool isSigned,
   // TODO: Generalize this for use with other types.
   if (Op0.getValueType() == MVT::i64 && DestVT == MVT::f64) {
     SDValue TwoP52 =
-      DAG.getConstant(UINT64_C(0x4330000000000000), MVT::i64);
+      DAG.getConstant(UINT64_C(0x4330000000000000), dl, MVT::i64);
     SDValue TwoP84PlusTwoP52 =
-      DAG.getConstantFP(BitsToDouble(UINT64_C(0x4530000000100000)), MVT::f64);
+      DAG.getConstantFP(BitsToDouble(UINT64_C(0x4530000000100000)), dl,
+                        MVT::f64);
     SDValue TwoP84 =
-      DAG.getConstant(UINT64_C(0x4530000000000000), MVT::i64);
+      DAG.getConstant(UINT64_C(0x4530000000000000), dl, MVT::i64);
 
     SDValue Lo = DAG.getZeroExtendInReg(Op0, dl, MVT::i32);
     SDValue Hi = DAG.getNode(ISD::SRL, dl, MVT::i64, Op0,
-                             DAG.getConstant(32, MVT::i64));
+                             DAG.getConstant(32, dl, MVT::i64));
     SDValue LoOr = DAG.getNode(ISD::OR, dl, MVT::i64, Lo, TwoP52);
     SDValue HiOr = DAG.getNode(ISD::OR, dl, MVT::i64, Hi, TwoP84);
     SDValue LoFlt = DAG.getNode(ISD::BITCAST, dl, MVT::f64, LoOr);
@@ -2505,10 +2512,10 @@ SDValue SelectionDAGLegalize::ExpandLegalINT_TO_FP(bool isSigned,
     if (!isSigned) {
       SDValue Fast = DAG.getNode(ISD::SINT_TO_FP, dl, MVT::f32, Op0);
 
-      SDValue ShiftConst =
-          DAG.getConstant(1, TLI.getShiftAmountTy(Op0.getValueType()));
+      SDValue ShiftConst = DAG.getConstant(
+          1, dl, TLI.getShiftAmountTy(Op0.getValueType(), DAG.getDataLayout()));
       SDValue Shr = DAG.getNode(ISD::SRL, dl, MVT::i64, Op0, ShiftConst);
-      SDValue AndConst = DAG.getConstant(1, MVT::i64);
+      SDValue AndConst = DAG.getConstant(1, dl, MVT::i64);
       SDValue And = DAG.getNode(ISD::AND, dl, MVT::i64, Op0, AndConst);
       SDValue Or = DAG.getNode(ISD::OR, dl, MVT::i64, And, Shr);
 
@@ -2520,47 +2527,52 @@ SDValue SelectionDAGLegalize::ExpandLegalINT_TO_FP(bool isSigned,
       // thing most of the time.  This would be a good candidate for a
       //pseudo-op, or, even better, for whole-function isel.
       SDValue SignBitTest = DAG.getSetCC(dl, getSetCCResultType(MVT::i64),
-        Op0, DAG.getConstant(0, MVT::i64), ISD::SETLT);
+        Op0, DAG.getConstant(0, dl, MVT::i64), ISD::SETLT);
       return DAG.getSelect(dl, MVT::f32, SignBitTest, Slow, Fast);
     }
 
     // Otherwise, implement the fully general conversion.
 
     SDValue And = DAG.getNode(ISD::AND, dl, MVT::i64, Op0,
-         DAG.getConstant(UINT64_C(0xfffffffffffff800), MVT::i64));
+         DAG.getConstant(UINT64_C(0xfffffffffffff800), dl, MVT::i64));
     SDValue Or = DAG.getNode(ISD::OR, dl, MVT::i64, And,
-         DAG.getConstant(UINT64_C(0x800), MVT::i64));
+         DAG.getConstant(UINT64_C(0x800), dl, MVT::i64));
     SDValue And2 = DAG.getNode(ISD::AND, dl, MVT::i64, Op0,
-         DAG.getConstant(UINT64_C(0x7ff), MVT::i64));
-    SDValue Ne = DAG.getSetCC(dl, getSetCCResultType(MVT::i64),
-                   And2, DAG.getConstant(UINT64_C(0), MVT::i64), ISD::SETNE);
+         DAG.getConstant(UINT64_C(0x7ff), dl, MVT::i64));
+    SDValue Ne = DAG.getSetCC(dl, getSetCCResultType(MVT::i64), And2,
+                              DAG.getConstant(UINT64_C(0), dl, MVT::i64),
+                              ISD::SETNE);
     SDValue Sel = DAG.getSelect(dl, MVT::i64, Ne, Or, Op0);
-    SDValue Ge = DAG.getSetCC(dl, getSetCCResultType(MVT::i64),
-                   Op0, DAG.getConstant(UINT64_C(0x0020000000000000), MVT::i64),
-                   ISD::SETUGE);
+    SDValue Ge = DAG.getSetCC(dl, getSetCCResultType(MVT::i64), Op0,
+                              DAG.getConstant(UINT64_C(0x0020000000000000), dl,
+                                              MVT::i64),
+                              ISD::SETUGE);
     SDValue Sel2 = DAG.getSelect(dl, MVT::i64, Ge, Sel, Op0);
-    EVT SHVT = TLI.getShiftAmountTy(Sel2.getValueType());
+    EVT SHVT = TLI.getShiftAmountTy(Sel2.getValueType(), DAG.getDataLayout());
 
     SDValue Sh = DAG.getNode(ISD::SRL, dl, MVT::i64, Sel2,
-                             DAG.getConstant(32, SHVT));
+                             DAG.getConstant(32, dl, SHVT));
     SDValue Trunc = DAG.getNode(ISD::TRUNCATE, dl, MVT::i32, Sh);
     SDValue Fcvt = DAG.getNode(ISD::UINT_TO_FP, dl, MVT::f64, Trunc);
     SDValue TwoP32 =
-      DAG.getConstantFP(BitsToDouble(UINT64_C(0x41f0000000000000)), MVT::f64);
+      DAG.getConstantFP(BitsToDouble(UINT64_C(0x41f0000000000000)), dl,
+                        MVT::f64);
     SDValue Fmul = DAG.getNode(ISD::FMUL, dl, MVT::f64, TwoP32, Fcvt);
     SDValue Lo = DAG.getNode(ISD::TRUNCATE, dl, MVT::i32, Sel2);
     SDValue Fcvt2 = DAG.getNode(ISD::UINT_TO_FP, dl, MVT::f64, Lo);
     SDValue Fadd = DAG.getNode(ISD::FADD, dl, MVT::f64, Fmul, Fcvt2);
     return DAG.getNode(ISD::FP_ROUND, dl, MVT::f32, Fadd,
-                       DAG.getIntPtrConstant(0));
+                       DAG.getIntPtrConstant(0, dl));
   }
 
   SDValue Tmp1 = DAG.getNode(ISD::SINT_TO_FP, dl, DestVT, Op0);
 
   SDValue SignSet = DAG.getSetCC(dl, getSetCCResultType(Op0.getValueType()),
-                                 Op0, DAG.getConstant(0, Op0.getValueType()),
+                                 Op0,
+                                 DAG.getConstant(0, dl, Op0.getValueType()),
                                  ISD::SETLT);
-  SDValue Zero = DAG.getIntPtrConstant(0), Four = DAG.getIntPtrConstant(4);
+  SDValue Zero = DAG.getIntPtrConstant(0, dl),
+          Four = DAG.getIntPtrConstant(4, dl);
   SDValue CstOffset = DAG.getSelect(dl, Zero.getValueType(),
                                     SignSet, Four, Zero);
 
@@ -2575,11 +2587,13 @@ SDValue SelectionDAGLegalize::ExpandLegalINT_TO_FP(bool isSigned,
   case MVT::i32: FF = 0x4F800000ULL; break;  // 2^32 (as a float)
   case MVT::i64: FF = 0x5F800000ULL; break;  // 2^64 (as a float)
   }
-  if (TLI.isLittleEndian()) FF <<= 32;
+  if (DAG.getDataLayout().isLittleEndian())
+    FF <<= 32;
   Constant *FudgeFactor = ConstantInt::get(
                                        Type::getInt64Ty(*DAG.getContext()), FF);
 
-  SDValue CPIdx = DAG.getConstantPool(FudgeFactor, TLI.getPointerTy());
+  SDValue CPIdx =
+      DAG.getConstantPool(FudgeFactor, TLI.getPointerTy(DAG.getDataLayout()));
   unsigned Alignment = cast<ConstantPoolSDNode>(CPIdx)->getAlignment();
   CPIdx = DAG.getNode(ISD::ADD, dl, CPIdx.getValueType(), CPIdx, CstOffset);
   Alignment = std::min(Alignment, 4u);
@@ -2690,39 +2704,46 @@ SDValue SelectionDAGLegalize::PromoteLegalFP_TO_INT(SDValue LegalOp,
 /// Open code the operations for BSWAP of the specified operation.
 SDValue SelectionDAGLegalize::ExpandBSWAP(SDValue Op, SDLoc dl) {
   EVT VT = Op.getValueType();
-  EVT SHVT = TLI.getShiftAmountTy(VT);
+  EVT SHVT = TLI.getShiftAmountTy(VT, DAG.getDataLayout());
   SDValue Tmp1, Tmp2, Tmp3, Tmp4, Tmp5, Tmp6, Tmp7, Tmp8;
   switch (VT.getSimpleVT().SimpleTy) {
   default: llvm_unreachable("Unhandled Expand type in BSWAP!");
   case MVT::i16:
-    Tmp2 = DAG.getNode(ISD::SHL, dl, VT, Op, DAG.getConstant(8, SHVT));
-    Tmp1 = DAG.getNode(ISD::SRL, dl, VT, Op, DAG.getConstant(8, SHVT));
+    Tmp2 = DAG.getNode(ISD::SHL, dl, VT, Op, DAG.getConstant(8, dl, SHVT));
+    Tmp1 = DAG.getNode(ISD::SRL, dl, VT, Op, DAG.getConstant(8, dl, SHVT));
     return DAG.getNode(ISD::OR, dl, VT, Tmp1, Tmp2);
   case MVT::i32:
-    Tmp4 = DAG.getNode(ISD::SHL, dl, VT, Op, DAG.getConstant(24, SHVT));
-    Tmp3 = DAG.getNode(ISD::SHL, dl, VT, Op, DAG.getConstant(8, SHVT));
-    Tmp2 = DAG.getNode(ISD::SRL, dl, VT, Op, DAG.getConstant(8, SHVT));
-    Tmp1 = DAG.getNode(ISD::SRL, dl, VT, Op, DAG.getConstant(24, SHVT));
-    Tmp3 = DAG.getNode(ISD::AND, dl, VT, Tmp3, DAG.getConstant(0xFF0000, VT));
-    Tmp2 = DAG.getNode(ISD::AND, dl, VT, Tmp2, DAG.getConstant(0xFF00, VT));
+    Tmp4 = DAG.getNode(ISD::SHL, dl, VT, Op, DAG.getConstant(24, dl, SHVT));
+    Tmp3 = DAG.getNode(ISD::SHL, dl, VT, Op, DAG.getConstant(8, dl, SHVT));
+    Tmp2 = DAG.getNode(ISD::SRL, dl, VT, Op, DAG.getConstant(8, dl, SHVT));
+    Tmp1 = DAG.getNode(ISD::SRL, dl, VT, Op, DAG.getConstant(24, dl, SHVT));
+    Tmp3 = DAG.getNode(ISD::AND, dl, VT, Tmp3,
+                       DAG.getConstant(0xFF0000, dl, VT));
+    Tmp2 = DAG.getNode(ISD::AND, dl, VT, Tmp2, DAG.getConstant(0xFF00, dl, VT));
     Tmp4 = DAG.getNode(ISD::OR, dl, VT, Tmp4, Tmp3);
     Tmp2 = DAG.getNode(ISD::OR, dl, VT, Tmp2, Tmp1);
     return DAG.getNode(ISD::OR, dl, VT, Tmp4, Tmp2);
   case MVT::i64:
-    Tmp8 = DAG.getNode(ISD::SHL, dl, VT, Op, DAG.getConstant(56, SHVT));
-    Tmp7 = DAG.getNode(ISD::SHL, dl, VT, Op, DAG.getConstant(40, SHVT));
-    Tmp6 = DAG.getNode(ISD::SHL, dl, VT, Op, DAG.getConstant(24, SHVT));
-    Tmp5 = DAG.getNode(ISD::SHL, dl, VT, Op, DAG.getConstant(8, SHVT));
-    Tmp4 = DAG.getNode(ISD::SRL, dl, VT, Op, DAG.getConstant(8, SHVT));
-    Tmp3 = DAG.getNode(ISD::SRL, dl, VT, Op, DAG.getConstant(24, SHVT));
-    Tmp2 = DAG.getNode(ISD::SRL, dl, VT, Op, DAG.getConstant(40, SHVT));
-    Tmp1 = DAG.getNode(ISD::SRL, dl, VT, Op, DAG.getConstant(56, SHVT));
-    Tmp7 = DAG.getNode(ISD::AND, dl, VT, Tmp7, DAG.getConstant(255ULL<<48, VT));
-    Tmp6 = DAG.getNode(ISD::AND, dl, VT, Tmp6, DAG.getConstant(255ULL<<40, VT));
-    Tmp5 = DAG.getNode(ISD::AND, dl, VT, Tmp5, DAG.getConstant(255ULL<<32, VT));
-    Tmp4 = DAG.getNode(ISD::AND, dl, VT, Tmp4, DAG.getConstant(255ULL<<24, VT));
-    Tmp3 = DAG.getNode(ISD::AND, dl, VT, Tmp3, DAG.getConstant(255ULL<<16, VT));
-    Tmp2 = DAG.getNode(ISD::AND, dl, VT, Tmp2, DAG.getConstant(255ULL<<8 , VT));
+    Tmp8 = DAG.getNode(ISD::SHL, dl, VT, Op, DAG.getConstant(56, dl, SHVT));
+    Tmp7 = DAG.getNode(ISD::SHL, dl, VT, Op, DAG.getConstant(40, dl, SHVT));
+    Tmp6 = DAG.getNode(ISD::SHL, dl, VT, Op, DAG.getConstant(24, dl, SHVT));
+    Tmp5 = DAG.getNode(ISD::SHL, dl, VT, Op, DAG.getConstant(8, dl, SHVT));
+    Tmp4 = DAG.getNode(ISD::SRL, dl, VT, Op, DAG.getConstant(8, dl, SHVT));
+    Tmp3 = DAG.getNode(ISD::SRL, dl, VT, Op, DAG.getConstant(24, dl, SHVT));
+    Tmp2 = DAG.getNode(ISD::SRL, dl, VT, Op, DAG.getConstant(40, dl, SHVT));
+    Tmp1 = DAG.getNode(ISD::SRL, dl, VT, Op, DAG.getConstant(56, dl, SHVT));
+    Tmp7 = DAG.getNode(ISD::AND, dl, VT, Tmp7,
+                       DAG.getConstant(255ULL<<48, dl, VT));
+    Tmp6 = DAG.getNode(ISD::AND, dl, VT, Tmp6,
+                       DAG.getConstant(255ULL<<40, dl, VT));
+    Tmp5 = DAG.getNode(ISD::AND, dl, VT, Tmp5,
+                       DAG.getConstant(255ULL<<32, dl, VT));
+    Tmp4 = DAG.getNode(ISD::AND, dl, VT, Tmp4,
+                       DAG.getConstant(255ULL<<24, dl, VT));
+    Tmp3 = DAG.getNode(ISD::AND, dl, VT, Tmp3,
+                       DAG.getConstant(255ULL<<16, dl, VT));
+    Tmp2 = DAG.getNode(ISD::AND, dl, VT, Tmp2,
+                       DAG.getConstant(255ULL<<8 , dl, VT));
     Tmp8 = DAG.getNode(ISD::OR, dl, VT, Tmp8, Tmp7);
     Tmp6 = DAG.getNode(ISD::OR, dl, VT, Tmp6, Tmp5);
     Tmp4 = DAG.getNode(ISD::OR, dl, VT, Tmp4, Tmp3);
@@ -2740,7 +2761,7 @@ SDValue SelectionDAGLegalize::ExpandBitCount(unsigned Opc, SDValue Op,
   default: llvm_unreachable("Cannot expand this yet!");
   case ISD::CTPOP: {
     EVT VT = Op.getValueType();
-    EVT ShVT = TLI.getShiftAmountTy(VT);
+    EVT ShVT = TLI.getShiftAmountTy(VT, DAG.getDataLayout());
     unsigned Len = VT.getSizeInBits();
 
     assert(VT.isInteger() && Len <= 128 && Len % 8 == 0 &&
@@ -2749,34 +2770,38 @@ SDValue SelectionDAGLegalize::ExpandBitCount(unsigned Opc, SDValue Op,
     // This is the "best" algorithm from
     // http://graphics.stanford.edu/~seander/bithacks.html#CountBitsSetParallel
 
-    SDValue Mask55 = DAG.getConstant(APInt::getSplat(Len, APInt(8, 0x55)), VT);
-    SDValue Mask33 = DAG.getConstant(APInt::getSplat(Len, APInt(8, 0x33)), VT);
-    SDValue Mask0F = DAG.getConstant(APInt::getSplat(Len, APInt(8, 0x0F)), VT);
-    SDValue Mask01 = DAG.getConstant(APInt::getSplat(Len, APInt(8, 0x01)), VT);
+    SDValue Mask55 = DAG.getConstant(APInt::getSplat(Len, APInt(8, 0x55)),
+                                     dl, VT);
+    SDValue Mask33 = DAG.getConstant(APInt::getSplat(Len, APInt(8, 0x33)),
+                                     dl, VT);
+    SDValue Mask0F = DAG.getConstant(APInt::getSplat(Len, APInt(8, 0x0F)),
+                                     dl, VT);
+    SDValue Mask01 = DAG.getConstant(APInt::getSplat(Len, APInt(8, 0x01)),
+                                     dl, VT);
 
     // v = v - ((v >> 1) & 0x55555555...)
     Op = DAG.getNode(ISD::SUB, dl, VT, Op,
                      DAG.getNode(ISD::AND, dl, VT,
                                  DAG.getNode(ISD::SRL, dl, VT, Op,
-                                             DAG.getConstant(1, ShVT)),
+                                             DAG.getConstant(1, dl, ShVT)),
                                  Mask55));
     // v = (v & 0x33333333...) + ((v >> 2) & 0x33333333...)
     Op = DAG.getNode(ISD::ADD, dl, VT,
                      DAG.getNode(ISD::AND, dl, VT, Op, Mask33),
                      DAG.getNode(ISD::AND, dl, VT,
                                  DAG.getNode(ISD::SRL, dl, VT, Op,
-                                             DAG.getConstant(2, ShVT)),
+                                             DAG.getConstant(2, dl, ShVT)),
                                  Mask33));
     // v = (v + (v >> 4)) & 0x0F0F0F0F...
     Op = DAG.getNode(ISD::AND, dl, VT,
                      DAG.getNode(ISD::ADD, dl, VT, Op,
                                  DAG.getNode(ISD::SRL, dl, VT, Op,
-                                             DAG.getConstant(4, ShVT))),
+                                             DAG.getConstant(4, dl, ShVT))),
                      Mask0F);
     // v = (v * 0x01010101...) >> (Len - 8)
     Op = DAG.getNode(ISD::SRL, dl, VT,
                      DAG.getNode(ISD::MUL, dl, VT, Op, Mask01),
-                     DAG.getConstant(Len - 8, ShVT));
+                     DAG.getConstant(Len - 8, dl, ShVT));
 
     return Op;
   }
@@ -2794,10 +2819,10 @@ SDValue SelectionDAGLegalize::ExpandBitCount(unsigned Opc, SDValue Op,
     //
     // Ref: "Hacker's Delight" by Henry Warren
     EVT VT = Op.getValueType();
-    EVT ShVT = TLI.getShiftAmountTy(VT);
+    EVT ShVT = TLI.getShiftAmountTy(VT, DAG.getDataLayout());
     unsigned len = VT.getSizeInBits();
     for (unsigned i = 0; (1U << i) <= (len / 2); ++i) {
-      SDValue Tmp3 = DAG.getConstant(1ULL << i, ShVT);
+      SDValue Tmp3 = DAG.getConstant(1ULL << i, dl, ShVT);
       Op = DAG.getNode(ISD::OR, dl, VT, Op,
                        DAG.getNode(ISD::SRL, dl, VT, Op, Tmp3));
     }
@@ -2816,12 +2841,12 @@ SDValue SelectionDAGLegalize::ExpandBitCount(unsigned Opc, SDValue Op,
     SDValue Tmp3 = DAG.getNode(ISD::AND, dl, VT,
                                DAG.getNOT(dl, Op, VT),
                                DAG.getNode(ISD::SUB, dl, VT, Op,
-                                           DAG.getConstant(1, VT)));
+                                           DAG.getConstant(1, dl, VT)));
     // If ISD::CTLZ is legal and CTPOP isn't, then do that instead.
     if (!TLI.isOperationLegalOrCustom(ISD::CTPOP, VT) &&
         TLI.isOperationLegalOrCustom(ISD::CTLZ, VT))
       return DAG.getNode(ISD::SUB, dl, VT,
-                         DAG.getConstant(VT.getSizeInBits(), VT),
+                         DAG.getConstant(VT.getSizeInBits(), dl, VT),
                          DAG.getNode(ISD::CTLZ, dl, VT, Tmp3));
     return DAG.getNode(ISD::CTPOP, dl, VT, Tmp3);
   }
@@ -2857,10 +2882,10 @@ void SelectionDAGLegalize::ExpandNode(SDNode *Node) {
   case ISD::FRAMEADDR:
   case ISD::RETURNADDR:
   case ISD::FRAME_TO_ARGS_OFFSET:
-    Results.push_back(DAG.getConstant(0, Node->getValueType(0)));
+    Results.push_back(DAG.getConstant(0, dl, Node->getValueType(0)));
     break;
   case ISD::FLT_ROUNDS_:
-    Results.push_back(DAG.getConstant(1, Node->getValueType(0)));
+    Results.push_back(DAG.getConstant(1, dl, Node->getValueType(0)));
     break;
   case ISD::EH_RETURN:
   case ISD::EH_LABEL:
@@ -2874,7 +2899,7 @@ void SelectionDAGLegalize::ExpandNode(SDNode *Node) {
   case ISD::EH_SJLJ_SETJMP:
     // If the target didn't expand this, just return 'zero' and preserve the
     // chain.
-    Results.push_back(DAG.getConstant(0, MVT::i32));
+    Results.push_back(DAG.getConstant(0, dl, MVT::i32));
     Results.push_back(Node->getOperand(0));
     break;
   case ISD::ATOMIC_FENCE: {
@@ -2883,10 +2908,12 @@ void SelectionDAGLegalize::ExpandNode(SDNode *Node) {
     TargetLowering::ArgListTy Args;
 
     TargetLowering::CallLoweringInfo CLI(DAG);
-    CLI.setDebugLoc(dl).setChain(Node->getOperand(0))
-      .setCallee(CallingConv::C, Type::getVoidTy(*DAG.getContext()),
-                 DAG.getExternalSymbol("__sync_synchronize",
-                 TLI.getPointerTy()), std::move(Args), 0);
+    CLI.setDebugLoc(dl)
+        .setChain(Node->getOperand(0))
+        .setCallee(CallingConv::C, Type::getVoidTy(*DAG.getContext()),
+                   DAG.getExternalSymbol("__sync_synchronize",
+                                         TLI.getPointerTy(DAG.getDataLayout())),
+                   std::move(Args), 0);
 
     std::pair<SDValue, SDValue> CallResult = TLI.LowerCallTo(CLI);
 
@@ -2895,7 +2922,7 @@ void SelectionDAGLegalize::ExpandNode(SDNode *Node) {
   }
   case ISD::ATOMIC_LOAD: {
     // There is no libcall for atomic load; fake it with ATOMIC_CMP_SWAP.
-    SDValue Zero = DAG.getConstant(0, Node->getValueType(0));
+    SDValue Zero = DAG.getConstant(0, dl, Node->getValueType(0));
     SDVTList VTs = DAG.getVTList(Node->getValueType(0), MVT::Other);
     SDValue Swap = DAG.getAtomicCmpSwap(
         ISD::ATOMIC_CMP_SWAP, dl, cast<AtomicSDNode>(Node)->getMemoryVT(), VTs,
@@ -2971,10 +2998,10 @@ void SelectionDAGLegalize::ExpandNode(SDNode *Node) {
   case ISD::UNDEF: {
     EVT VT = Node->getValueType(0);
     if (VT.isInteger())
-      Results.push_back(DAG.getConstant(0, VT));
+      Results.push_back(DAG.getConstant(0, dl, VT));
     else {
       assert(VT.isFloatingPoint() && "Unknown value type!");
-      Results.push_back(DAG.getConstantFP(0, VT));
+      Results.push_back(DAG.getConstantFP(0, dl, VT));
     }
     break;
   }
@@ -2982,10 +3009,12 @@ void SelectionDAGLegalize::ExpandNode(SDNode *Node) {
     // If this operation is not supported, lower it to 'abort()' call
     TargetLowering::ArgListTy Args;
     TargetLowering::CallLoweringInfo CLI(DAG);
-    CLI.setDebugLoc(dl).setChain(Node->getOperand(0))
-      .setCallee(CallingConv::C, Type::getVoidTy(*DAG.getContext()),
-                 DAG.getExternalSymbol("abort", TLI.getPointerTy()),
-                 std::move(Args), 0);
+    CLI.setDebugLoc(dl)
+        .setChain(Node->getOperand(0))
+        .setCallee(CallingConv::C, Type::getVoidTy(*DAG.getContext()),
+                   DAG.getExternalSymbol("abort",
+                                         TLI.getPointerTy(DAG.getDataLayout())),
+                   std::move(Args), 0);
     std::pair<SDValue, SDValue> CallResult = TLI.LowerCallTo(CLI);
 
     Results.push_back(CallResult.second);
@@ -3008,12 +3037,12 @@ void SelectionDAGLegalize::ExpandNode(SDNode *Node) {
     // SAR.  However, it is doubtful that any exist.
     EVT ExtraVT = cast<VTSDNode>(Node->getOperand(1))->getVT();
     EVT VT = Node->getValueType(0);
-    EVT ShiftAmountTy = TLI.getShiftAmountTy(VT);
+    EVT ShiftAmountTy = TLI.getShiftAmountTy(VT, DAG.getDataLayout());
     if (VT.isVector())
       ShiftAmountTy = VT;
     unsigned BitsDiff = VT.getScalarType().getSizeInBits() -
                         ExtraVT.getScalarType().getSizeInBits();
-    SDValue ShiftCst = DAG.getConstant(BitsDiff, ShiftAmountTy);
+    SDValue ShiftCst = DAG.getConstant(BitsDiff, dl, ShiftAmountTy);
     Tmp1 = DAG.getNode(ISD::SHL, dl, Node->getValueType(0),
                        Node->getOperand(0), ShiftCst);
     Tmp1 = DAG.getNode(ISD::SRA, dl, Node->getValueType(0), Tmp1, ShiftCst);
@@ -3051,7 +3080,7 @@ void SelectionDAGLegalize::ExpandNode(SDNode *Node) {
                 APInt::getNullValue(VT.getSizeInBits()));
     APInt x = APInt::getSignBit(NVT.getSizeInBits());
     (void)apf.convertFromAPInt(x, false, APFloat::rmNearestTiesToEven);
-    Tmp1 = DAG.getConstantFP(apf, VT);
+    Tmp1 = DAG.getConstantFP(apf, dl, VT);
     Tmp2 = DAG.getSetCC(dl, getSetCCResultType(VT),
                         Node->getOperand(0),
                         Tmp1, ISD::SETLT);
@@ -3060,7 +3089,7 @@ void SelectionDAGLegalize::ExpandNode(SDNode *Node) {
                         DAG.getNode(ISD::FSUB, dl, VT,
                                     Node->getOperand(0), Tmp1));
     False = DAG.getNode(ISD::XOR, dl, NVT, False,
-                        DAG.getConstant(x, NVT));
+                        DAG.getConstant(x, dl, NVT));
     Tmp1 = DAG.getSelect(dl, NVT, Tmp2, True, False);
     Results.push_back(Tmp1);
     break;
@@ -3072,28 +3101,28 @@ void SelectionDAGLegalize::ExpandNode(SDNode *Node) {
     Tmp2 = Node->getOperand(1);
     unsigned Align = Node->getConstantOperandVal(3);
 
-    SDValue VAListLoad = DAG.getLoad(TLI.getPointerTy(), dl, Tmp1, Tmp2,
-                                     MachinePointerInfo(V),
-                                     false, false, false, 0);
+    SDValue VAListLoad =
+        DAG.getLoad(TLI.getPointerTy(DAG.getDataLayout()), dl, Tmp1, Tmp2,
+                    MachinePointerInfo(V), false, false, false, 0);
     SDValue VAList = VAListLoad;
 
     if (Align > TLI.getMinStackArgumentAlignment()) {
       assert(((Align & (Align-1)) == 0) && "Expected Align to be a power of 2");
 
       VAList = DAG.getNode(ISD::ADD, dl, VAList.getValueType(), VAList,
-                           DAG.getConstant(Align - 1,
+                           DAG.getConstant(Align - 1, dl,
                                            VAList.getValueType()));
 
       VAList = DAG.getNode(ISD::AND, dl, VAList.getValueType(), VAList,
-                           DAG.getConstant(-(int64_t)Align,
+                           DAG.getConstant(-(int64_t)Align, dl,
                                            VAList.getValueType()));
     }
 
     // Increment the pointer, VAList, to the next vaarg
     Tmp3 = DAG.getNode(ISD::ADD, dl, VAList.getValueType(), VAList,
-                       DAG.getConstant(TLI.getDataLayout()->
-                          getTypeAllocSize(VT.getTypeForEVT(*DAG.getContext())),
-                                       VAList.getValueType()));
+                       DAG.getConstant(DAG.getDataLayout().getTypeAllocSize(
+                                           VT.getTypeForEVT(*DAG.getContext())),
+                                       dl, VAList.getValueType()));
     // Store the incremented VAList to the legalized pointer
     Tmp3 = DAG.getStore(VAListLoad.getValue(1), dl, Tmp3, Tmp2,
                         MachinePointerInfo(V), false, false, 0);
@@ -3108,9 +3137,9 @@ void SelectionDAGLegalize::ExpandNode(SDNode *Node) {
     // output, returning the chain.
     const Value *VD = cast<SrcValueSDNode>(Node->getOperand(3))->getValue();
     const Value *VS = cast<SrcValueSDNode>(Node->getOperand(4))->getValue();
-    Tmp1 = DAG.getLoad(TLI.getPointerTy(), dl, Node->getOperand(0),
-                       Node->getOperand(2), MachinePointerInfo(VS),
-                       false, false, false, 0);
+    Tmp1 = DAG.getLoad(TLI.getPointerTy(DAG.getDataLayout()), dl,
+                       Node->getOperand(0), Node->getOperand(2),
+                       MachinePointerInfo(VS), false, false, false, 0);
     Tmp1 = DAG.getStore(Tmp1.getValue(1), dl, Tmp1, Node->getOperand(1),
                         MachinePointerInfo(VD), false, false, 0);
     Results.push_back(Tmp1);
@@ -3205,14 +3234,14 @@ void SelectionDAGLegalize::ExpandNode(SDNode *Node) {
       }
       unsigned Idx = Mask[i];
       if (Idx < NumElems)
-        Ops.push_back(DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, EltVT,
-                                  Op0,
-                                  DAG.getConstant(Idx, TLI.getVectorIdxTy())));
+        Ops.push_back(DAG.getNode(
+            ISD::EXTRACT_VECTOR_ELT, dl, EltVT, Op0,
+            DAG.getConstant(Idx, dl, TLI.getVectorIdxTy(DAG.getDataLayout()))));
       else
-        Ops.push_back(DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, EltVT,
-                                  Op1,
-                                  DAG.getConstant(Idx - NumElems,
-                                                  TLI.getVectorIdxTy())));
+        Ops.push_back(DAG.getNode(
+            ISD::EXTRACT_VECTOR_ELT, dl, EltVT, Op1,
+            DAG.getConstant(Idx - NumElems, dl,
+                            TLI.getVectorIdxTy(DAG.getDataLayout()))));
     }
 
     Tmp1 = DAG.getNode(ISD::BUILD_VECTOR, dl, VT, Ops);
@@ -3226,8 +3255,10 @@ void SelectionDAGLegalize::ExpandNode(SDNode *Node) {
     if (cast<ConstantSDNode>(Node->getOperand(1))->getZExtValue()) {
       // 1 -> Hi
       Tmp1 = DAG.getNode(ISD::SRL, dl, OpTy, Node->getOperand(0),
-                         DAG.getConstant(OpTy.getSizeInBits()/2,
-                    TLI.getShiftAmountTy(Node->getOperand(0).getValueType())));
+                         DAG.getConstant(OpTy.getSizeInBits() / 2, dl,
+                                         TLI.getShiftAmountTy(
+                                             Node->getOperand(0).getValueType(),
+                                             DAG.getDataLayout())));
       Tmp1 = DAG.getNode(ISD::TRUNCATE, dl, Node->getValueType(0), Tmp1);
     } else {
       // 0 -> Lo
@@ -3264,7 +3295,7 @@ void SelectionDAGLegalize::ExpandNode(SDNode *Node) {
     break;
   case ISD::FNEG:
     // Expand Y = FNEG(X) ->  Y = SUB -0.0, X
-    Tmp1 = DAG.getConstantFP(-0.0, Node->getValueType(0));
+    Tmp1 = DAG.getConstantFP(-0.0, dl, Node->getValueType(0));
     Tmp1 = DAG.getNode(ISD::FSUB, dl, Node->getValueType(0), Tmp1,
                        Node->getOperand(0));
     Results.push_back(Tmp1);
@@ -3273,7 +3304,7 @@ void SelectionDAGLegalize::ExpandNode(SDNode *Node) {
     // Expand Y = FABS(X) -> Y = (X >u 0.0) ? X : fneg(X).
     EVT VT = Node->getValueType(0);
     Tmp1 = Node->getOperand(0);
-    Tmp2 = DAG.getConstantFP(0.0, VT);
+    Tmp2 = DAG.getConstantFP(0.0, dl, VT);
     Tmp2 = DAG.getSetCC(dl, getSetCCResultType(Tmp1.getValueType()),
                         Tmp1, Tmp2, ISD::SETUGT);
     Tmp3 = DAG.getNode(ISD::FNEG, dl, VT, Tmp1);
@@ -3281,6 +3312,26 @@ void SelectionDAGLegalize::ExpandNode(SDNode *Node) {
     Results.push_back(Tmp1);
     break;
   }
+  case ISD::SMIN:
+  case ISD::SMAX:
+  case ISD::UMIN:
+  case ISD::UMAX: {
+    // Expand Y = MAX(A, B) -> Y = (A > B) ? A : B
+    ISD::CondCode Pred;
+    switch (Node->getOpcode()) {
+    default: llvm_unreachable("How did we get here?");
+    case ISD::SMAX: Pred = ISD::SETGT; break;
+    case ISD::SMIN: Pred = ISD::SETLT; break;
+    case ISD::UMAX: Pred = ISD::SETUGT; break;
+    case ISD::UMIN: Pred = ISD::SETULT; break;
+    }
+    Tmp1 = Node->getOperand(0);
+    Tmp2 = Node->getOperand(1);
+    Tmp1 = DAG.getSelectCC(dl, Tmp1, Tmp2, Tmp1, Tmp2, Pred);
+    Results.push_back(Tmp1);
+    break;
+  }
+    
   case ISD::FMINNUM:
     Results.push_back(ExpandFPLibCall(Node, RTLIB::FMIN_F32, RTLIB::FMIN_F64,
                                       RTLIB::FMIN_F80, RTLIB::FMIN_F128,
@@ -3438,7 +3489,7 @@ void SelectionDAGLegalize::ExpandNode(SDNode *Node) {
     break;
   }
   case ISD::FP_TO_FP16: {
-    if (!TM.Options.UseSoftFloat && TM.Options.UnsafeFPMath) {
+    if (!TLI.useSoftFloat() && TM.Options.UnsafeFPMath) {
       SDValue Op = Node->getOperand(0);
       MVT SVT = Op.getSimpleValueType();
       if ((SVT == MVT::f64 || SVT == MVT::f80) &&
@@ -3446,7 +3497,7 @@ void SelectionDAGLegalize::ExpandNode(SDNode *Node) {
         // Under fastmath, we can expand this node into a fround followed by
         // a float-half conversion.
         SDValue FloatVal = DAG.getNode(ISD::FP_ROUND, dl, MVT::f32, Op,
-                                       DAG.getIntPtrConstant(0));
+                                       DAG.getIntPtrConstant(0, dl));
         Results.push_back(
             DAG.getNode(ISD::FP_TO_FP16, dl, MVT::i16, FloatVal));
         break;
@@ -3487,8 +3538,9 @@ void SelectionDAGLegalize::ExpandNode(SDNode *Node) {
            TLI.isOperationLegalOrCustom(ISD::XOR, VT) &&
            "Don't know how to expand this subtraction!");
     Tmp1 = DAG.getNode(ISD::XOR, dl, VT, Node->getOperand(1),
-               DAG.getConstant(APInt::getAllOnesValue(VT.getSizeInBits()), VT));
-    Tmp1 = DAG.getNode(ISD::ADD, dl, VT, Tmp1, DAG.getConstant(1, VT));
+               DAG.getConstant(APInt::getAllOnesValue(VT.getSizeInBits()), dl,
+                               VT));
+    Tmp1 = DAG.getNode(ISD::ADD, dl, VT, Tmp1, DAG.getConstant(1, dl, VT));
     Results.push_back(DAG.getNode(ISD::ADD, dl, VT, Node->getOperand(0), Tmp1));
     break;
   }
@@ -3604,8 +3656,9 @@ void SelectionDAGLegalize::ExpandNode(SDNode *Node) {
         TLI.expandMUL(Node, Lo, Hi, HalfType, DAG)) {
       Lo = DAG.getNode(ISD::ZERO_EXTEND, dl, VT, Lo);
       Hi = DAG.getNode(ISD::ANY_EXTEND, dl, VT, Hi);
-      SDValue Shift = DAG.getConstant(HalfType.getSizeInBits(),
-                                      TLI.getShiftAmountTy(HalfType));
+      SDValue Shift =
+          DAG.getConstant(HalfType.getSizeInBits(), dl,
+                          TLI.getShiftAmountTy(HalfType, DAG.getDataLayout()));
       Hi = DAG.getNode(ISD::SHL, dl, VT, Hi, Shift);
       Results.push_back(DAG.getNode(ISD::OR, dl, VT, Lo, Hi));
       break;
@@ -3629,7 +3682,7 @@ void SelectionDAGLegalize::ExpandNode(SDNode *Node) {
     EVT ResultType = Node->getValueType(1);
     EVT OType = getSetCCResultType(Node->getValueType(0));
 
-    SDValue Zero = DAG.getConstant(0, LHS.getValueType());
+    SDValue Zero = DAG.getConstant(0, dl, LHS.getValueType());
 
     //   LHSSign -> LHS >= 0
     //   RHSSign -> RHS >= 0
@@ -3695,9 +3748,9 @@ void SelectionDAGLegalize::ExpandNode(SDNode *Node) {
       RHS = DAG.getNode(Ops[isSigned][2], dl, WideVT, RHS);
       Tmp1 = DAG.getNode(ISD::MUL, dl, WideVT, LHS, RHS);
       BottomHalf = DAG.getNode(ISD::EXTRACT_ELEMENT, dl, VT, Tmp1,
-                               DAG.getIntPtrConstant(0));
+                               DAG.getIntPtrConstant(0, dl));
       TopHalf = DAG.getNode(ISD::EXTRACT_ELEMENT, dl, VT, Tmp1,
-                            DAG.getIntPtrConstant(1));
+                            DAG.getIntPtrConstant(1, dl));
     } else {
       // We can fall back to a libcall with an illegal type for the MUL if we
       // have a libcall big enough.
@@ -3717,10 +3770,14 @@ void SelectionDAGLegalize::ExpandNode(SDNode *Node) {
       // The high part is obtained by SRA'ing all but one of the bits of low
       // part.
       unsigned LoSize = VT.getSizeInBits();
-      SDValue HiLHS = DAG.getNode(ISD::SRA, dl, VT, RHS,
-                                DAG.getConstant(LoSize-1, TLI.getPointerTy()));
-      SDValue HiRHS = DAG.getNode(ISD::SRA, dl, VT, LHS,
-                                DAG.getConstant(LoSize-1, TLI.getPointerTy()));
+      SDValue HiLHS =
+          DAG.getNode(ISD::SRA, dl, VT, RHS,
+                      DAG.getConstant(LoSize - 1, dl,
+                                      TLI.getPointerTy(DAG.getDataLayout())));
+      SDValue HiRHS =
+          DAG.getNode(ISD::SRA, dl, VT, LHS,
+                      DAG.getConstant(LoSize - 1, dl,
+                                      TLI.getPointerTy(DAG.getDataLayout())));
 
       // Here we're passing the 2 arguments explicitly as 4 arguments that are
       // pre-lowered to the correct types. This all depends upon WideVT not
@@ -3729,9 +3786,9 @@ void SelectionDAGLegalize::ExpandNode(SDNode *Node) {
       SDValue Args[] = { LHS, HiLHS, RHS, HiRHS };
       SDValue Ret = ExpandLibCall(LC, WideVT, Args, 4, isSigned, dl);
       BottomHalf = DAG.getNode(ISD::EXTRACT_ELEMENT, dl, VT, Ret,
-                               DAG.getIntPtrConstant(0));
+                               DAG.getIntPtrConstant(0, dl));
       TopHalf = DAG.getNode(ISD::EXTRACT_ELEMENT, dl, VT, Ret,
-                            DAG.getIntPtrConstant(1));
+                            DAG.getIntPtrConstant(1, dl));
       // Ret is a node with an illegal type. Because such things are not
       // generally permitted during this phase of legalization, make sure the
       // node has no more uses. The above EXTRACT_ELEMENT nodes should have been
@@ -3741,14 +3798,15 @@ void SelectionDAGLegalize::ExpandNode(SDNode *Node) {
     }
 
     if (isSigned) {
-      Tmp1 = DAG.getConstant(VT.getSizeInBits() - 1,
-                             TLI.getShiftAmountTy(BottomHalf.getValueType()));
+      Tmp1 = DAG.getConstant(
+          VT.getSizeInBits() - 1, dl,
+          TLI.getShiftAmountTy(BottomHalf.getValueType(), DAG.getDataLayout()));
       Tmp1 = DAG.getNode(ISD::SRA, dl, VT, BottomHalf, Tmp1);
       TopHalf = DAG.getSetCC(dl, getSetCCResultType(VT), TopHalf, Tmp1,
                              ISD::SETNE);
     } else {
       TopHalf = DAG.getSetCC(dl, getSetCCResultType(VT), TopHalf,
-                             DAG.getConstant(0, VT), ISD::SETNE);
+                             DAG.getConstant(0, dl, VT), ISD::SETNE);
     }
     Results.push_back(BottomHalf);
     Results.push_back(TopHalf);
@@ -3758,9 +3816,10 @@ void SelectionDAGLegalize::ExpandNode(SDNode *Node) {
     EVT PairTy = Node->getValueType(0);
     Tmp1 = DAG.getNode(ISD::ZERO_EXTEND, dl, PairTy, Node->getOperand(0));
     Tmp2 = DAG.getNode(ISD::ANY_EXTEND, dl, PairTy, Node->getOperand(1));
-    Tmp2 = DAG.getNode(ISD::SHL, dl, PairTy, Tmp2,
-                       DAG.getConstant(PairTy.getSizeInBits()/2,
-                                       TLI.getShiftAmountTy(PairTy)));
+    Tmp2 = DAG.getNode(
+        ISD::SHL, dl, PairTy, Tmp2,
+        DAG.getConstant(PairTy.getSizeInBits() / 2, dl,
+                        TLI.getShiftAmountTy(PairTy, DAG.getDataLayout())));
     Results.push_back(DAG.getNode(ISD::OR, dl, PairTy, Tmp1, Tmp2));
     break;
   }
@@ -3774,7 +3833,7 @@ void SelectionDAGLegalize::ExpandNode(SDNode *Node) {
                              cast<CondCodeSDNode>(Tmp1.getOperand(2))->get());
     } else {
       Tmp1 = DAG.getSelectCC(dl, Tmp1,
-                             DAG.getConstant(0, Tmp1.getValueType()),
+                             DAG.getConstant(0, dl, Tmp1.getValueType()),
                              Tmp2, Tmp3, ISD::SETNE);
     }
     Results.push_back(Tmp1);
@@ -3784,14 +3843,14 @@ void SelectionDAGLegalize::ExpandNode(SDNode *Node) {
     SDValue Table = Node->getOperand(1);
     SDValue Index = Node->getOperand(2);
 
-    EVT PTy = TLI.getPointerTy();
+    EVT PTy = TLI.getPointerTy(DAG.getDataLayout());
 
-    const DataLayout &TD = *TLI.getDataLayout();
+    const DataLayout &TD = DAG.getDataLayout();
     unsigned EntrySize =
       DAG.getMachineFunction().getJumpTableInfo()->getEntrySize(TD);
 
-    Index = DAG.getNode(ISD::MUL, dl, Index.getValueType(),
-                       Index, DAG.getConstant(EntrySize, Index.getValueType()));
+    Index = DAG.getNode(ISD::MUL, dl, Index.getValueType(), Index,
+                        DAG.getConstant(EntrySize, dl, Index.getValueType()));
     SDValue Addr = DAG.getNode(ISD::ADD, dl, Index.getValueType(),
                                Index, Table);
 
@@ -3825,10 +3884,10 @@ void SelectionDAGLegalize::ExpandNode(SDNode *Node) {
       // We test only the i1 bit.  Skip the AND if UNDEF.
       Tmp3 = (Tmp2.getOpcode() == ISD::UNDEF) ? Tmp2 :
         DAG.getNode(ISD::AND, dl, Tmp2.getValueType(), Tmp2,
-                    DAG.getConstant(1, Tmp2.getValueType()));
+                    DAG.getConstant(1, dl, Tmp2.getValueType()));
       Tmp1 = DAG.getNode(ISD::BR_CC, dl, MVT::Other, Tmp1,
                          DAG.getCondCode(ISD::SETNE), Tmp3,
-                         DAG.getConstant(0, Tmp3.getValueType()),
+                         DAG.getConstant(0, dl, Tmp3.getValueType()),
                          Node->getOperand(2));
     }
     Results.push_back(Tmp1);
@@ -3870,7 +3929,8 @@ void SelectionDAGLegalize::ExpandNode(SDNode *Node) {
       break;
     }
     Tmp1 = DAG.getNode(ISD::SELECT_CC, dl, VT, Tmp1, Tmp2,
-                       DAG.getConstant(TrueValue, VT), DAG.getConstant(0, VT),
+                       DAG.getConstant(TrueValue, dl, VT),
+                       DAG.getConstant(0, dl, VT),
                        Tmp3);
     Results.push_back(Tmp1);
     break;
@@ -3891,7 +3951,8 @@ void SelectionDAGLegalize::ExpandNode(SDNode *Node) {
       assert(!TLI.isOperationExpand(ISD::SELECT, VT) &&
              "Cannot expand ISD::SELECT_CC when ISD::SELECT also needs to be "
              "expanded.");
-      EVT CCVT = TLI.getSetCCResultType(*DAG.getContext(), CmpVT);
+      EVT CCVT =
+          TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), CmpVT);
       SDValue Cond = DAG.getNode(ISD::SETCC, dl, CCVT, Tmp1, Tmp2, CC);
       Results.push_back(DAG.getSelect(dl, VT, Cond, Tmp3, Tmp4));
       break;
@@ -3938,7 +3999,7 @@ void SelectionDAGLegalize::ExpandNode(SDNode *Node) {
         Tmp1 = DAG.getNode(ISD::SELECT_CC, dl, Node->getValueType(0),
                            Tmp1, Tmp2, Tmp3, Tmp4, CC);
       } else {
-        Tmp2 = DAG.getConstant(0, Tmp1.getValueType());
+        Tmp2 = DAG.getConstant(0, dl, Tmp1.getValueType());
         CC = DAG.getCondCode(ISD::SETNE);
         Tmp1 = DAG.getNode(ISD::SELECT_CC, dl, Node->getValueType(0), Tmp1,
                            Tmp2, Tmp3, Tmp4, CC);
@@ -3969,7 +4030,7 @@ void SelectionDAGLegalize::ExpandNode(SDNode *Node) {
       Tmp1 = DAG.getNode(ISD::BR_CC, dl, Node->getValueType(0), Tmp1,
                          Tmp4, Tmp2, Tmp3, Node->getOperand(4));
     } else {
-      Tmp3 = DAG.getConstant(0, Tmp2.getValueType());
+      Tmp3 = DAG.getConstant(0, dl, Tmp2.getValueType());
       Tmp4 = DAG.getCondCode(ISD::SETNE);
       Tmp1 = DAG.getNode(ISD::BR_CC, dl, Node->getValueType(0), Tmp1, Tmp4,
                          Tmp2, Tmp3, Node->getOperand(4));
@@ -3991,14 +4052,12 @@ void SelectionDAGLegalize::ExpandNode(SDNode *Node) {
 
     SmallVector<SDValue, 8> Scalars;
     for (unsigned Idx = 0; Idx < NumElem; Idx++) {
-      SDValue Ex = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl,
-                               VT.getScalarType(),
-                               Node->getOperand(0), DAG.getConstant(Idx,
-                                                    TLI.getVectorIdxTy()));
-      SDValue Sh = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl,
-                               VT.getScalarType(),
-                               Node->getOperand(1), DAG.getConstant(Idx,
-                                                    TLI.getVectorIdxTy()));
+      SDValue Ex = DAG.getNode(
+          ISD::EXTRACT_VECTOR_ELT, dl, VT.getScalarType(), Node->getOperand(0),
+          DAG.getConstant(Idx, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
+      SDValue Sh = DAG.getNode(
+          ISD::EXTRACT_VECTOR_ELT, dl, VT.getScalarType(), Node->getOperand(1),
+          DAG.getConstant(Idx, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
       Scalars.push_back(DAG.getNode(Node->getOpcode(), dl,
                                     VT.getScalarType(), Ex, Sh));
     }
@@ -4052,16 +4111,16 @@ void SelectionDAGLegalize::PromoteNode(SDNode *Node) {
     if (Node->getOpcode() == ISD::CTTZ) {
       // FIXME: This should set a bit in the zero extended value instead.
       Tmp2 = DAG.getSetCC(dl, getSetCCResultType(NVT),
-                          Tmp1, DAG.getConstant(NVT.getSizeInBits(), NVT),
+                          Tmp1, DAG.getConstant(NVT.getSizeInBits(), dl, NVT),
                           ISD::SETEQ);
       Tmp1 = DAG.getSelect(dl, NVT, Tmp2,
-                           DAG.getConstant(OVT.getSizeInBits(), NVT), Tmp1);
+                           DAG.getConstant(OVT.getSizeInBits(), dl, NVT), Tmp1);
     } else if (Node->getOpcode() == ISD::CTLZ ||
                Node->getOpcode() == ISD::CTLZ_ZERO_UNDEF) {
       // Tmp1 = Tmp1 - (sizeinbits(NVT) - sizeinbits(Old VT))
       Tmp1 = DAG.getNode(ISD::SUB, dl, NVT, Tmp1,
                           DAG.getConstant(NVT.getSizeInBits() -
-                                          OVT.getSizeInBits(), NVT));
+                                          OVT.getSizeInBits(), dl, NVT));
     }
     Results.push_back(DAG.getNode(ISD::TRUNCATE, dl, OVT, Tmp1));
     break;
@@ -4069,8 +4128,10 @@ void SelectionDAGLegalize::PromoteNode(SDNode *Node) {
     unsigned DiffBits = NVT.getSizeInBits() - OVT.getSizeInBits();
     Tmp1 = DAG.getNode(ISD::ZERO_EXTEND, dl, NVT, Node->getOperand(0));
     Tmp1 = DAG.getNode(ISD::BSWAP, dl, NVT, Tmp1);
-    Tmp1 = DAG.getNode(ISD::SRL, dl, NVT, Tmp1,
-                          DAG.getConstant(DiffBits, TLI.getShiftAmountTy(NVT)));
+    Tmp1 = DAG.getNode(
+        ISD::SRL, dl, NVT, Tmp1,
+        DAG.getConstant(DiffBits, dl,
+                        TLI.getShiftAmountTy(NVT, DAG.getDataLayout())));
     Results.push_back(Tmp1);
     break;
   }
@@ -4160,7 +4221,7 @@ void SelectionDAGLegalize::PromoteNode(SDNode *Node) {
       Tmp1 = DAG.getNode(TruncOp, dl, Node->getValueType(0), Tmp1);
     else
       Tmp1 = DAG.getNode(TruncOp, dl, Node->getValueType(0), Tmp1,
-                         DAG.getIntPtrConstant(0));
+                         DAG.getIntPtrConstant(0, dl));
     Results.push_back(Tmp1);
     break;
   }
@@ -4217,7 +4278,7 @@ void SelectionDAGLegalize::PromoteNode(SDNode *Node) {
     Tmp2 = DAG.getNode(ISD::FP_EXTEND, dl, NVT, Node->getOperand(1));
     Tmp3 = DAG.getNode(Node->getOpcode(), dl, NVT, Tmp1, Tmp2);
     Results.push_back(DAG.getNode(ISD::FP_ROUND, dl, OVT,
-                                  Tmp3, DAG.getIntPtrConstant(0)));
+                                  Tmp3, DAG.getIntPtrConstant(0, dl)));
     break;
   }
   case ISD::FMA: {
@@ -4227,7 +4288,7 @@ void SelectionDAGLegalize::PromoteNode(SDNode *Node) {
     Results.push_back(
         DAG.getNode(ISD::FP_ROUND, dl, OVT,
                     DAG.getNode(Node->getOpcode(), dl, NVT, Tmp1, Tmp2, Tmp3),
-                    DAG.getIntPtrConstant(0)));
+                    DAG.getIntPtrConstant(0, dl)));
     break;
   }
   case ISD::FPOWI: {
@@ -4235,7 +4296,7 @@ void SelectionDAGLegalize::PromoteNode(SDNode *Node) {
     Tmp2 = Node->getOperand(1);
     Tmp3 = DAG.getNode(Node->getOpcode(), dl, NVT, Tmp1, Tmp2);
     Results.push_back(DAG.getNode(ISD::FP_ROUND, dl, OVT,
-                                  Tmp3, DAG.getIntPtrConstant(0)));
+                                  Tmp3, DAG.getIntPtrConstant(0, dl)));
     break;
   }
   case ISD::FFLOOR:
@@ -4257,7 +4318,7 @@ void SelectionDAGLegalize::PromoteNode(SDNode *Node) {
     Tmp1 = DAG.getNode(ISD::FP_EXTEND, dl, NVT, Node->getOperand(0));
     Tmp2 = DAG.getNode(Node->getOpcode(), dl, NVT, Tmp1);
     Results.push_back(DAG.getNode(ISD::FP_ROUND, dl, OVT,
-                                  Tmp2, DAG.getIntPtrConstant(0)));
+                                  Tmp2, DAG.getIntPtrConstant(0, dl)));
     break;
   }
   }
diff --git a/lib/CodeGen/SelectionDAG/LegalizeFloatTypes.cpp b/lib/CodeGen/SelectionDAG/LegalizeFloatTypes.cpp
index b5967155434..3c50a415573 100644
--- a/lib/CodeGen/SelectionDAG/LegalizeFloatTypes.cpp
+++ b/lib/CodeGen/SelectionDAG/LegalizeFloatTypes.cpp
@@ -131,7 +131,7 @@ SDValue DAGTypeLegalizer::SoftenFloatRes_BUILD_PAIR(SDNode *N) {
 }
 
 SDValue DAGTypeLegalizer::SoftenFloatRes_ConstantFP(ConstantFPSDNode *N) {
-  return DAG.getConstant(N->getValueAPF().bitcastToAPInt(),
+  return DAG.getConstant(N->getValueAPF().bitcastToAPInt(), SDLoc(N),
                          TLI.getTypeToTransformTo(*DAG.getContext(),
                                                   N->getValueType(0)));
 }
@@ -149,8 +149,8 @@ SDValue DAGTypeLegalizer::SoftenFloatRes_FABS(SDNode *N) {
 
   // Mask = ~(1 << (Size-1))
   APInt API = APInt::getAllOnesValue(Size);
-  API.clearBit(Size-1);
-  SDValue Mask = DAG.getConstant(API, NVT);
+  API.clearBit(Size - 1);
+  SDValue Mask = DAG.getConstant(API, SDLoc(N), NVT);
   SDValue Op = GetSoftenedFloat(N->getOperand(0));
   return DAG.getNode(ISD::AND, SDLoc(N), NVT, Op, Mask);
 }
@@ -218,30 +218,36 @@ SDValue DAGTypeLegalizer::SoftenFloatRes_FCOPYSIGN(SDNode *N) {
   unsigned RSize = RVT.getSizeInBits();
 
   // First get the sign bit of second operand.
-  SDValue SignBit = DAG.getNode(ISD::SHL, dl, RVT, DAG.getConstant(1, RVT),
-                                  DAG.getConstant(RSize - 1,
-                                                  TLI.getShiftAmountTy(RVT)));
+  SDValue SignBit = DAG.getNode(
+      ISD::SHL, dl, RVT, DAG.getConstant(1, dl, RVT),
+      DAG.getConstant(RSize - 1, dl,
+                      TLI.getShiftAmountTy(RVT, DAG.getDataLayout())));
   SignBit = DAG.getNode(ISD::AND, dl, RVT, RHS, SignBit);
 
   // Shift right or sign-extend it if the two operands have different types.
   int SizeDiff = RVT.getSizeInBits() - LVT.getSizeInBits();
   if (SizeDiff > 0) {
-    SignBit = DAG.getNode(ISD::SRL, dl, RVT, SignBit,
-                          DAG.getConstant(SizeDiff,
-                                 TLI.getShiftAmountTy(SignBit.getValueType())));
+    SignBit =
+        DAG.getNode(ISD::SRL, dl, RVT, SignBit,
+                    DAG.getConstant(SizeDiff, dl,
+                                    TLI.getShiftAmountTy(SignBit.getValueType(),
+                                                         DAG.getDataLayout())));
     SignBit = DAG.getNode(ISD::TRUNCATE, dl, LVT, SignBit);
   } else if (SizeDiff < 0) {
     SignBit = DAG.getNode(ISD::ANY_EXTEND, dl, LVT, SignBit);
-    SignBit = DAG.getNode(ISD::SHL, dl, LVT, SignBit,
-                          DAG.getConstant(-SizeDiff,
-                                 TLI.getShiftAmountTy(SignBit.getValueType())));
+    SignBit =
+        DAG.getNode(ISD::SHL, dl, LVT, SignBit,
+                    DAG.getConstant(-SizeDiff, dl,
+                                    TLI.getShiftAmountTy(SignBit.getValueType(),
+                                                         DAG.getDataLayout())));
   }
 
   // Clear the sign bit of the first operand.
-  SDValue Mask = DAG.getNode(ISD::SHL, dl, LVT, DAG.getConstant(1, LVT),
-                               DAG.getConstant(LSize - 1,
-                                               TLI.getShiftAmountTy(LVT)));
-  Mask = DAG.getNode(ISD::SUB, dl, LVT, Mask, DAG.getConstant(1, LVT));
+  SDValue Mask = DAG.getNode(
+      ISD::SHL, dl, LVT, DAG.getConstant(1, dl, LVT),
+      DAG.getConstant(LSize - 1, dl,
+                      TLI.getShiftAmountTy(LVT, DAG.getDataLayout())));
+  Mask = DAG.getNode(ISD::SUB, dl, LVT, Mask, DAG.getConstant(1, dl, LVT));
   LHS = DAG.getNode(ISD::AND, dl, LVT, LHS, Mask);
 
   // Or the value with the sign bit.
@@ -386,8 +392,9 @@ SDValue DAGTypeLegalizer::SoftenFloatRes_FNEARBYINT(SDNode *N) {
 
 SDValue DAGTypeLegalizer::SoftenFloatRes_FNEG(SDNode *N) {
   EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
+  SDLoc dl(N);
   // Expand Y = FNEG(X) -> Y = SUB -0.0, X
-  SDValue Ops[2] = { DAG.getConstantFP(-0.0, N->getValueType(0)),
+  SDValue Ops[2] = { DAG.getConstantFP(-0.0, dl, N->getValueType(0)),
                      GetSoftenedFloat(N->getOperand(0)) };
   return TLI.makeLibCall(DAG, GetFPLibCall(N->getValueType(0),
                                            RTLIB::SUB_F32,
@@ -395,7 +402,7 @@ SDValue DAGTypeLegalizer::SoftenFloatRes_FNEG(SDNode *N) {
                                            RTLIB::SUB_F80,
                                            RTLIB::SUB_F128,
                                            RTLIB::SUB_PPCF128),
-                         NVT, Ops, 2, false, SDLoc(N)).first;
+                         NVT, Ops, 2, false, dl).first;
 }
 
 SDValue DAGTypeLegalizer::SoftenFloatRes_FP_EXTEND(SDNode *N) {
@@ -755,7 +762,7 @@ SDValue DAGTypeLegalizer::SoftenFloatOp_BR_CC(SDNode *N) {
   // If softenSetCCOperands returned a scalar, we need to compare the result
   // against zero to select between true and false values.
   if (!NewRHS.getNode()) {
-    NewRHS = DAG.getConstant(0, NewLHS.getValueType());
+    NewRHS = DAG.getConstant(0, SDLoc(N), NewLHS.getValueType());
     CCCode = ISD::SETNE;
   }
 
@@ -794,7 +801,7 @@ SDValue DAGTypeLegalizer::SoftenFloatOp_SELECT_CC(SDNode *N) {
   // If softenSetCCOperands returned a scalar, we need to compare the result
   // against zero to select between true and false values.
   if (!NewRHS.getNode()) {
-    NewRHS = DAG.getConstant(0, NewLHS.getValueType());
+    NewRHS = DAG.getConstant(0, SDLoc(N), NewLHS.getValueType());
     CCCode = ISD::SETNE;
   }
 
@@ -837,7 +844,7 @@ SDValue DAGTypeLegalizer::SoftenFloatOp_STORE(SDNode *N, unsigned OpNo) {
   if (ST->isTruncatingStore())
     // Do an FP_ROUND followed by a non-truncating store.
     Val = BitConvertToInteger(DAG.getNode(ISD::FP_ROUND, dl, ST->getMemoryVT(),
-                                          Val, DAG.getIntPtrConstant(0)));
+                                          Val, DAG.getIntPtrConstant(0, dl)));
   else
     Val = GetSoftenedFloat(Val);
 
@@ -927,12 +934,13 @@ void DAGTypeLegalizer::ExpandFloatRes_ConstantFP(SDNode *N, SDValue &Lo,
   assert(NVT.getSizeInBits() == integerPartWidth &&
          "Do not know how to expand this float constant!");
   APInt C = cast<ConstantFPSDNode>(N)->getValueAPF().bitcastToAPInt();
+  SDLoc dl(N);
   Lo = DAG.getConstantFP(APFloat(DAG.EVTToAPFloatSemantics(NVT),
                                  APInt(integerPartWidth, C.getRawData()[1])),
-                         NVT);
+                         dl, NVT);
   Hi = DAG.getConstantFP(APFloat(DAG.EVTToAPFloatSemantics(NVT),
                                  APInt(integerPartWidth, C.getRawData()[0])),
-                         NVT);
+                         dl, NVT);
 }
 
 void DAGTypeLegalizer::ExpandFloatRes_FABS(SDNode *N, SDValue &Lo,
@@ -1136,9 +1144,10 @@ void DAGTypeLegalizer::ExpandFloatRes_FNEG(SDNode *N, SDValue &Lo,
 void DAGTypeLegalizer::ExpandFloatRes_FP_EXTEND(SDNode *N, SDValue &Lo,
                                                 SDValue &Hi) {
   EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
-  Hi = DAG.getNode(ISD::FP_EXTEND, SDLoc(N), NVT, N->getOperand(0));
+  SDLoc dl(N);
+  Hi = DAG.getNode(ISD::FP_EXTEND, dl, NVT, N->getOperand(0));
   Lo = DAG.getConstantFP(APFloat(DAG.EVTToAPFloatSemantics(NVT),
-                                 APInt(NVT.getSizeInBits(), 0)), NVT);
+                                 APInt(NVT.getSizeInBits(), 0)), dl, NVT);
 }
 
 void DAGTypeLegalizer::ExpandFloatRes_FPOW(SDNode *N,
@@ -1262,7 +1271,7 @@ void DAGTypeLegalizer::ExpandFloatRes_LOAD(SDNode *N, SDValue &Lo,
 
   // The low part is zero.
   Lo = DAG.getConstantFP(APFloat(DAG.EVTToAPFloatSemantics(NVT),
-                                 APInt(NVT.getSizeInBits(), 0)), NVT);
+                                 APInt(NVT.getSizeInBits(), 0)), dl, NVT);
 
   // Modified the chain - switch anything that used the old chain to use the
   // new one.
@@ -1287,7 +1296,7 @@ void DAGTypeLegalizer::ExpandFloatRes_XINT_TO_FP(SDNode *N, SDValue &Lo,
     Src = DAG.getNode(isSigned ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND, dl,
                       MVT::i32, Src);
     Lo = DAG.getConstantFP(APFloat(DAG.EVTToAPFloatSemantics(NVT),
-                                   APInt(NVT.getSizeInBits(), 0)), NVT);
+                                   APInt(NVT.getSizeInBits(), 0)), dl, NVT);
     Hi = DAG.getNode(ISD::SINT_TO_FP, dl, NVT, Src);
   } else {
     RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
@@ -1335,8 +1344,8 @@ void DAGTypeLegalizer::ExpandFloatRes_XINT_TO_FP(SDNode *N, SDValue &Lo,
   Lo = DAG.getNode(ISD::FADD, dl, VT, Hi,
                    DAG.getConstantFP(APFloat(APFloat::PPCDoubleDouble,
                                              APInt(128, Parts)),
-                                     MVT::ppcf128));
-  Lo = DAG.getSelectCC(dl, Src, DAG.getConstant(0, SrcVT),
+                                     dl, MVT::ppcf128));
+  Lo = DAG.getSelectCC(dl, Src, DAG.getConstant(0, dl, SrcVT),
                        Lo, Hi, ISD::SETLT);
   GetPairElements(Lo, Lo, Hi);
 }
@@ -1436,7 +1445,7 @@ SDValue DAGTypeLegalizer::ExpandFloatOp_BR_CC(SDNode *N) {
   // If ExpandSetCCOperands returned a scalar, we need to compare the result
   // against zero to select between true and false values.
   if (!NewRHS.getNode()) {
-    NewRHS = DAG.getConstant(0, NewLHS.getValueType());
+    NewRHS = DAG.getConstant(0, SDLoc(N), NewLHS.getValueType());
     CCCode = ISD::SETNE;
   }
 
@@ -1479,7 +1488,7 @@ SDValue DAGTypeLegalizer::ExpandFloatOp_FP_TO_SINT(SDNode *N) {
     SDValue Res = DAG.getNode(ISD::FP_ROUND_INREG, dl, MVT::ppcf128,
                               N->getOperand(0), DAG.getValueType(MVT::f64));
     Res = DAG.getNode(ISD::FP_ROUND, dl, MVT::f64, Res,
-                      DAG.getIntPtrConstant(1));
+                      DAG.getIntPtrConstant(1, dl));
     return DAG.getNode(ISD::FP_TO_SINT, dl, MVT::i32, Res);
   }
 
@@ -1499,7 +1508,7 @@ SDValue DAGTypeLegalizer::ExpandFloatOp_FP_TO_UINT(SDNode *N) {
            "Logic only correct for ppcf128!");
     const uint64_t TwoE31[] = {0x41e0000000000000LL, 0};
     APFloat APF = APFloat(APFloat::PPCDoubleDouble, APInt(128, TwoE31));
-    SDValue Tmp = DAG.getConstantFP(APF, MVT::ppcf128);
+    SDValue Tmp = DAG.getConstantFP(APF, dl, MVT::ppcf128);
     //  X>=2^31 ? (int)(X-2^31)+0x80000000 : (int)X
     // FIXME: generated code sucks.
     return DAG.getSelectCC(dl, N->getOperand(0), Tmp,
@@ -1509,7 +1518,8 @@ SDValue DAGTypeLegalizer::ExpandFloatOp_FP_TO_UINT(SDNode *N) {
                                                                MVT::ppcf128,
                                                                N->getOperand(0),
                                                                Tmp)),
-                                       DAG.getConstant(0x80000000, MVT::i32)),
+                                       DAG.getConstant(0x80000000, dl,
+                                                       MVT::i32)),
                            DAG.getNode(ISD::FP_TO_SINT, dl,
                                        MVT::i32, N->getOperand(0)),
                            ISD::SETGE);
@@ -1529,7 +1539,7 @@ SDValue DAGTypeLegalizer::ExpandFloatOp_SELECT_CC(SDNode *N) {
   // If ExpandSetCCOperands returned a scalar, we need to compare the result
   // against zero to select between true and false values.
   if (!NewRHS.getNode()) {
-    NewRHS = DAG.getConstant(0, NewLHS.getValueType());
+    NewRHS = DAG.getConstant(0, SDLoc(N), NewLHS.getValueType());
     CCCode = ISD::SETNE;
   }
 
@@ -1579,3 +1589,420 @@ SDValue DAGTypeLegalizer::ExpandFloatOp_STORE(SDNode *N, unsigned OpNo) {
   return DAG.getTruncStore(Chain, SDLoc(N), Hi, Ptr,
                            ST->getMemoryVT(), ST->getMemOperand());
 }
+
+//===----------------------------------------------------------------------===//
+//  Float Operand Promotion
+//===----------------------------------------------------------------------===//
+//
+
+static ISD::NodeType GetPromotionOpcode(EVT OpVT, EVT RetVT) {
+  if (OpVT == MVT::f16) {
+      return ISD::FP16_TO_FP;
+  } else if (RetVT == MVT::f16) {
+      return ISD::FP_TO_FP16;
+  }
+
+  report_fatal_error("Attempt at an invalid promotion-related conversion");
+}
+
+bool DAGTypeLegalizer::PromoteFloatOperand(SDNode *N, unsigned OpNo) {
+  SDValue R = SDValue();
+
+  // Nodes that use a promotion-requiring floating point operand, but doesn't
+  // produce a promotion-requiring floating point result, need to be legalized
+  // to use the promoted float operand.  Nodes that produce at least one
+  // promotion-requiring floating point result have their operands legalized as
+  // a part of PromoteFloatResult.
+  switch (N->getOpcode()) {
+    default:
+      llvm_unreachable("Do not know how to promote this operator's operand!");
+
+    case ISD::BITCAST:    R = PromoteFloatOp_BITCAST(N, OpNo); break;
+    case ISD::FCOPYSIGN:  R = PromoteFloatOp_FCOPYSIGN(N, OpNo); break;
+    case ISD::FP_TO_SINT:
+    case ISD::FP_TO_UINT: R = PromoteFloatOp_FP_TO_XINT(N, OpNo); break;
+    case ISD::FP_EXTEND:  R = PromoteFloatOp_FP_EXTEND(N, OpNo); break;
+    case ISD::SELECT_CC:  R = PromoteFloatOp_SELECT_CC(N, OpNo); break;
+    case ISD::SETCC:      R = PromoteFloatOp_SETCC(N, OpNo); break;
+    case ISD::STORE:      R = PromoteFloatOp_STORE(N, OpNo); break;
+  }
+
+  if (R.getNode())
+    ReplaceValueWith(SDValue(N, 0), R);
+  return false;
+}
+
+SDValue DAGTypeLegalizer::PromoteFloatOp_BITCAST(SDNode *N, unsigned OpNo) {
+  SDValue Op = N->getOperand(0);
+  EVT OpVT = Op->getValueType(0);
+
+  EVT IVT = EVT::getIntegerVT(*DAG.getContext(), OpVT.getSizeInBits());
+  assert (IVT == N->getValueType(0) && "Bitcast to type of different size");
+
+  SDValue Promoted = GetPromotedFloat(N->getOperand(0));
+  EVT PromotedVT = Promoted->getValueType(0);
+
+  // Convert the promoted float value to the desired IVT.
+  return DAG.getNode(GetPromotionOpcode(PromotedVT, OpVT), SDLoc(N), IVT,
+                     Promoted);
+}
+
+// Promote Operand 1 of FCOPYSIGN.  Operand 0 ought to be handled by
+// PromoteFloatRes_FCOPYSIGN.
+SDValue DAGTypeLegalizer::PromoteFloatOp_FCOPYSIGN(SDNode *N, unsigned OpNo) {
+  assert (OpNo == 1 && "Only Operand 1 must need promotion here");
+  SDValue Op1 = GetPromotedFloat(N->getOperand(1));
+
+  return DAG.getNode(N->getOpcode(), SDLoc(N), N->getValueType(0),
+                     N->getOperand(0), Op1);
+}
+
+// Convert the promoted float value to the desired integer type
+SDValue DAGTypeLegalizer::PromoteFloatOp_FP_TO_XINT(SDNode *N, unsigned OpNo) {
+  SDValue Op = GetPromotedFloat(N->getOperand(0));
+  return DAG.getNode(N->getOpcode(), SDLoc(N), N->getValueType(0), Op);
+}
+
+SDValue DAGTypeLegalizer::PromoteFloatOp_FP_EXTEND(SDNode *N, unsigned OpNo) {
+  SDValue Op = GetPromotedFloat(N->getOperand(0));
+  EVT VT = N->getValueType(0);
+
+  // Desired VT is same as promoted type.  Use promoted float directly.
+  if (VT == Op->getValueType(0))
+    return Op;
+
+  // Else, extend the promoted float value to the desired VT.
+  return DAG.getNode(ISD::FP_EXTEND, SDLoc(N), VT, Op);
+}
+
+// Promote the float operands used for comparison.  The true- and false-
+// operands have the same type as the result and are promoted, if needed, by
+// PromoteFloatRes_SELECT_CC
+SDValue DAGTypeLegalizer::PromoteFloatOp_SELECT_CC(SDNode *N, unsigned OpNo) {
+  SDValue LHS = GetPromotedFloat(N->getOperand(0));
+  SDValue RHS = GetPromotedFloat(N->getOperand(1));
+
+  return DAG.getNode(ISD::SELECT_CC, SDLoc(N), N->getValueType(0),
+                     LHS, RHS, N->getOperand(2), N->getOperand(3),
+                     N->getOperand(4));
+}
+
+// Construct a SETCC that compares the promoted values and sets the conditional
+// code.
+SDValue DAGTypeLegalizer::PromoteFloatOp_SETCC(SDNode *N, unsigned OpNo) {
+  EVT VT = N->getValueType(0);
+  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
+  SDValue Op0 = GetPromotedFloat(N->getOperand(0));
+  SDValue Op1 = GetPromotedFloat(N->getOperand(1));
+  ISD::CondCode CCCode = cast<CondCodeSDNode>(N->getOperand(2))->get();
+
+  return DAG.getSetCC(SDLoc(N), NVT, Op0, Op1, CCCode);
+
+}
+
+// Lower the promoted Float down to the integer value of same size and construct
+// a STORE of the integer value.
+SDValue DAGTypeLegalizer::PromoteFloatOp_STORE(SDNode *N, unsigned OpNo) {
+  StoreSDNode *ST = cast<StoreSDNode>(N);
+  SDValue Val = ST->getValue();
+  SDLoc DL(N);
+
+  SDValue Promoted = GetPromotedFloat(Val);
+  EVT VT = ST->getOperand(1)->getValueType(0);
+  EVT IVT = EVT::getIntegerVT(*DAG.getContext(), VT.getSizeInBits());
+
+  SDValue NewVal;
+  NewVal = DAG.getNode(GetPromotionOpcode(Promoted.getValueType(), VT), DL,
+                       IVT, Promoted);
+
+  return DAG.getStore(ST->getChain(), DL, NewVal, ST->getBasePtr(),
+                      ST->getMemOperand());
+}
+
+//===----------------------------------------------------------------------===//
+//  Float Result Promotion
+//===----------------------------------------------------------------------===//
+
+void DAGTypeLegalizer::PromoteFloatResult(SDNode *N, unsigned ResNo) {
+  SDValue R = SDValue();
+
+  switch (N->getOpcode()) {
+    // These opcodes cannot appear if promotion of FP16 is done in the backend
+    // instead of Clang
+    case ISD::FP16_TO_FP:
+    case ISD::FP_TO_FP16:
+    default:
+      llvm_unreachable("Do not know how to promote this operator's result!");
+
+    case ISD::BITCAST:    R = PromoteFloatRes_BITCAST(N); break;
+    case ISD::ConstantFP: R = PromoteFloatRes_ConstantFP(N); break;
+    case ISD::EXTRACT_VECTOR_ELT:
+                          R = PromoteFloatRes_EXTRACT_VECTOR_ELT(N); break;
+    case ISD::FCOPYSIGN:  R = PromoteFloatRes_FCOPYSIGN(N); break;
+
+    // Unary FP Operations
+    case ISD::FABS:
+    case ISD::FCEIL:
+    case ISD::FCOS:
+    case ISD::FEXP:
+    case ISD::FEXP2:
+    case ISD::FFLOOR:
+    case ISD::FLOG:
+    case ISD::FLOG2:
+    case ISD::FLOG10:
+    case ISD::FNEARBYINT:
+    case ISD::FNEG:
+    case ISD::FRINT:
+    case ISD::FROUND:
+    case ISD::FSIN:
+    case ISD::FSQRT:
+    case ISD::FTRUNC:     R = PromoteFloatRes_UnaryOp(N); break;
+
+    // Binary FP Operations
+    case ISD::FADD:
+    case ISD::FDIV:
+    case ISD::FMAXNUM:
+    case ISD::FMINNUM:
+    case ISD::FMUL:
+    case ISD::FPOW:
+    case ISD::FREM:
+    case ISD::FSUB:       R = PromoteFloatRes_BinOp(N); break;
+
+    case ISD::FMA:        // FMA is same as FMAD
+    case ISD::FMAD:       R = PromoteFloatRes_FMAD(N); break;
+
+    case ISD::FPOWI:      R = PromoteFloatRes_FPOWI(N); break;
+
+    case ISD::FP_ROUND:   R = PromoteFloatRes_FP_ROUND(N); break;
+    case ISD::LOAD:       R = PromoteFloatRes_LOAD(N); break;
+    case ISD::SELECT:     R = PromoteFloatRes_SELECT(N); break;
+    case ISD::SELECT_CC:  R = PromoteFloatRes_SELECT_CC(N); break;
+
+    case ISD::SINT_TO_FP:
+    case ISD::UINT_TO_FP: R = PromoteFloatRes_XINT_TO_FP(N); break;
+    case ISD::UNDEF:      R = PromoteFloatRes_UNDEF(N); break;
+
+  }
+
+  if (R.getNode())
+    SetPromotedFloat(SDValue(N, ResNo), R);
+}
+
+// Bitcast from i16 to f16:  convert the i16 to a f32 value instead.
+// At this point, it is not possible to determine if the bitcast value is
+// eventually stored to memory or promoted to f32 or promoted to a floating
+// point at a higher precision.  Some of these cases are handled by FP_EXTEND,
+// STORE promotion handlers.
+SDValue DAGTypeLegalizer::PromoteFloatRes_BITCAST(SDNode *N) {
+  EVT VT = N->getValueType(0);
+  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
+  return DAG.getNode(GetPromotionOpcode(VT, NVT), SDLoc(N), NVT,
+                     N->getOperand(0));
+}
+
+SDValue DAGTypeLegalizer::PromoteFloatRes_ConstantFP(SDNode *N) {
+  ConstantFPSDNode *CFPNode = cast<ConstantFPSDNode>(N);
+  EVT VT = N->getValueType(0);
+  SDLoc DL(N);
+
+  // Get the (bit-cast) APInt of the APFloat and build an integer constant
+  EVT IVT = EVT::getIntegerVT(*DAG.getContext(), VT.getSizeInBits());
+  SDValue C = DAG.getConstant(CFPNode->getValueAPF().bitcastToAPInt(), DL,
+                              IVT);
+
+  // Convert the Constant to the desired FP type
+  // FIXME We might be able to do the conversion during compilation and get rid
+  // of it from the object code
+  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
+  return DAG.getNode(GetPromotionOpcode(VT, NVT), DL, NVT, C);
+}
+
+// If the Index operand is a constant, try to redirect the extract operation to
+// the correct legalized vector.  If not, bit-convert the input vector to
+// equivalent integer vector.  Extract the element as an (bit-cast) integer
+// value and convert it to the promoted type.
+SDValue DAGTypeLegalizer::PromoteFloatRes_EXTRACT_VECTOR_ELT(SDNode *N) {
+  SDLoc DL(N);
+
+  // If the index is constant, try to extract the value from the legalized
+  // vector type.
+  if (isa<ConstantSDNode>(N->getOperand(1))) {
+    SDValue Vec = N->getOperand(0);
+    SDValue Idx = N->getOperand(1);
+    EVT VecVT = Vec->getValueType(0);
+    EVT EltVT = VecVT.getVectorElementType();
+
+    uint64_t IdxVal = cast<ConstantSDNode>(Idx)->getZExtValue();
+
+    switch (getTypeAction(VecVT)) {
+    default: break;
+    case TargetLowering::TypeScalarizeVector: {
+      SDValue Res = GetScalarizedVector(N->getOperand(0));
+      ReplaceValueWith(SDValue(N, 0), Res);
+      return SDValue();
+    }
+    case TargetLowering::TypeWidenVector: {
+      Vec = GetWidenedVector(Vec);
+      SDValue Res = DAG.getNode(N->getOpcode(), DL, EltVT, Vec, Idx);
+      ReplaceValueWith(SDValue(N, 0), Res);
+      return SDValue();
+    }
+    case TargetLowering::TypeSplitVector: {
+      SDValue Lo, Hi;
+      GetSplitVector(Vec, Lo, Hi);
+
+      uint64_t LoElts = Lo.getValueType().getVectorNumElements();
+      SDValue Res;
+      if (IdxVal < LoElts)
+        Res = DAG.getNode(N->getOpcode(), DL, EltVT, Lo, Idx);
+      else
+        Res = DAG.getNode(N->getOpcode(), DL, EltVT, Hi,
+                          DAG.getConstant(IdxVal - LoElts, DL,
+                                          Idx.getValueType()));
+      ReplaceValueWith(SDValue(N, 0), Res);
+      return SDValue();
+    }
+
+    }
+  }
+
+  // Bit-convert the input vector to the equivalent integer vector
+  SDValue NewOp = BitConvertVectorToIntegerVector(N->getOperand(0));
+  EVT IVT = NewOp.getValueType().getVectorElementType();
+
+  // Extract the element as an (bit-cast) integer value
+  SDValue NewVal = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, IVT,
+                               NewOp, N->getOperand(1));
+
+  // Convert the element to the desired FP type
+  EVT VT = N->getValueType(0);
+  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
+  return DAG.getNode(GetPromotionOpcode(VT, NVT), SDLoc(N), NVT, NewVal);
+}
+
+// FCOPYSIGN(X, Y) returns the value of X with the sign of Y.  If the result
+// needs promotion, so does the argument X.  Note that Y, if needed, will be
+// handled during operand promotion.
+SDValue DAGTypeLegalizer::PromoteFloatRes_FCOPYSIGN(SDNode *N) {
+  EVT VT = N->getValueType(0);
+  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
+  SDValue Op0 = GetPromotedFloat(N->getOperand(0));
+
+  SDValue Op1 = N->getOperand(1);
+
+  return DAG.getNode(N->getOpcode(), SDLoc(N), NVT, Op0, Op1);
+}
+
+// Unary operation where the result and the operand have PromoteFloat type
+// action.  Construct a new SDNode with the promoted float value of the old
+// operand.
+SDValue DAGTypeLegalizer::PromoteFloatRes_UnaryOp(SDNode *N) {
+  EVT VT = N->getValueType(0);
+  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
+  SDValue Op = GetPromotedFloat(N->getOperand(0));
+
+  return DAG.getNode(N->getOpcode(), SDLoc(N), NVT, Op);
+}
+
+// Binary operations where the result and both operands have PromoteFloat type
+// action.  Construct a new SDNode with the promoted float values of the old
+// operands.
+SDValue DAGTypeLegalizer::PromoteFloatRes_BinOp(SDNode *N) {
+  EVT VT = N->getValueType(0);
+  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
+  SDValue Op0 = GetPromotedFloat(N->getOperand(0));
+  SDValue Op1 = GetPromotedFloat(N->getOperand(1));
+
+  return DAG.getNode(N->getOpcode(), SDLoc(N), NVT, Op0, Op1);
+}
+
+SDValue DAGTypeLegalizer::PromoteFloatRes_FMAD(SDNode *N) {
+  EVT VT = N->getValueType(0);
+  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
+  SDValue Op0 = GetPromotedFloat(N->getOperand(0));
+  SDValue Op1 = GetPromotedFloat(N->getOperand(1));
+  SDValue Op2 = GetPromotedFloat(N->getOperand(2));
+
+  return DAG.getNode(N->getOpcode(), SDLoc(N), NVT, Op0, Op1, Op2);
+}
+
+// Promote the Float (first) operand and retain the Integer (second) operand
+SDValue DAGTypeLegalizer::PromoteFloatRes_FPOWI(SDNode *N) {
+  EVT VT = N->getValueType(0);
+  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
+  SDValue Op0 = GetPromotedFloat(N->getOperand(0));
+  SDValue Op1 = N->getOperand(1);
+
+  return DAG.getNode(N->getOpcode(), SDLoc(N), NVT, Op0, Op1);
+}
+
+// Explicit operation to reduce precision.  Reduce the value to half precision
+// and promote it back to the legal type.
+SDValue DAGTypeLegalizer::PromoteFloatRes_FP_ROUND(SDNode *N) {
+  SDLoc DL(N);
+
+  SDValue Op = N->getOperand(0);
+  EVT VT = N->getValueType(0);
+  EVT OpVT = Op->getValueType(0);
+  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
+  EVT IVT = EVT::getIntegerVT(*DAG.getContext(), VT.getSizeInBits());
+
+  // Round promoted float to desired precision
+  SDValue Round = DAG.getNode(GetPromotionOpcode(OpVT, VT), DL, IVT, Op);
+  // Promote it back to the legal output type
+  return DAG.getNode(GetPromotionOpcode(VT, NVT), DL, NVT, Round);
+}
+
+SDValue DAGTypeLegalizer::PromoteFloatRes_LOAD(SDNode *N) {
+  LoadSDNode *L = cast<LoadSDNode>(N);
+  EVT VT = N->getValueType(0);
+
+  // Load the value as an integer value with the same number of bits
+  EVT IVT = EVT::getIntegerVT(*DAG.getContext(), VT.getSizeInBits());
+  SDValue newL = DAG.getLoad(L->getAddressingMode(), L->getExtensionType(),
+                   IVT, SDLoc(N), L->getChain(), L->getBasePtr(),
+                   L->getOffset(), L->getPointerInfo(), IVT, L->isVolatile(),
+                   L->isNonTemporal(), false, L->getAlignment(),
+                   L->getAAInfo());
+  // Legalize the chain result by replacing uses of the old value chain with the
+  // new one
+  ReplaceValueWith(SDValue(N, 1), newL.getValue(1));
+
+  // Convert the integer value to the desired FP type
+  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
+  return DAG.getNode(GetPromotionOpcode(VT, NVT), SDLoc(N), NVT, newL);
+}
+
+// Construct a new SELECT node with the promoted true- and false- values.
+SDValue DAGTypeLegalizer::PromoteFloatRes_SELECT(SDNode *N) {
+  SDValue TrueVal = GetPromotedFloat(N->getOperand(1));
+  SDValue FalseVal = GetPromotedFloat(N->getOperand(2));
+
+  return DAG.getNode(ISD::SELECT, SDLoc(N), TrueVal->getValueType(0),
+                     N->getOperand(0), TrueVal, FalseVal);
+}
+
+// Construct a new SELECT_CC node with the promoted true- and false- values.
+// The operands used for comparison are promoted by PromoteFloatOp_SELECT_CC.
+SDValue DAGTypeLegalizer::PromoteFloatRes_SELECT_CC(SDNode *N) {
+  SDValue TrueVal = GetPromotedFloat(N->getOperand(2));
+  SDValue FalseVal = GetPromotedFloat(N->getOperand(3));
+
+  return DAG.getNode(ISD::SELECT_CC, SDLoc(N), N->getValueType(0),
+                     N->getOperand(0), N->getOperand(1), TrueVal, FalseVal,
+                     N->getOperand(4));
+}
+
+// Construct a SDNode that transforms the SINT or UINT operand to the promoted
+// float type.
+SDValue DAGTypeLegalizer::PromoteFloatRes_XINT_TO_FP(SDNode *N) {
+  EVT VT = N->getValueType(0);
+  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
+  return DAG.getNode(N->getOpcode(), SDLoc(N), NVT, N->getOperand(0));
+}
+
+SDValue DAGTypeLegalizer::PromoteFloatRes_UNDEF(SDNode *N) {
+  return DAG.getUNDEF(TLI.getTypeToTransformTo(*DAG.getContext(),
+                                               N->getValueType(0)));
+}
+
diff --git a/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp b/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp
index 25e80b9c736..511239ce477 100644
--- a/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp
+++ b/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp
@@ -71,6 +71,10 @@ void DAGTypeLegalizer::PromoteIntegerResult(SDNode *N, unsigned ResNo) {
   case ISD::VSELECT:     Res = PromoteIntRes_VSELECT(N); break;
   case ISD::SELECT_CC:   Res = PromoteIntRes_SELECT_CC(N); break;
   case ISD::SETCC:       Res = PromoteIntRes_SETCC(N); break;
+  case ISD::SMIN:
+  case ISD::SMAX:
+  case ISD::UMIN:
+  case ISD::UMAX:        Res = PromoteIntRes_SimpleIntBinOp(N); break;
   case ISD::SHL:         Res = PromoteIntRes_SHL(N); break;
   case ISD::SIGN_EXTEND_INREG:
                          Res = PromoteIntRes_SIGN_EXTEND_INREG(N); break;
@@ -142,6 +146,10 @@ void DAGTypeLegalizer::PromoteIntegerResult(SDNode *N, unsigned ResNo) {
   case ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS:
     Res = PromoteIntRes_AtomicCmpSwap(cast<AtomicSDNode>(N), ResNo);
     break;
+  case ISD::UABSDIFF:
+  case ISD::SABSDIFF:
+    Res = PromoteIntRes_SimpleIntBinOp(N);
+    break;
   }
 
   // If the result is null then the sub-method took care of registering it.
@@ -251,6 +259,16 @@ SDValue DAGTypeLegalizer::PromoteIntRes_BITCAST(SDNode *N) {
   case TargetLowering::TypeSoftenFloat:
     // Promote the integer operand by hand.
     return DAG.getNode(ISD::ANY_EXTEND, dl, NOutVT, GetSoftenedFloat(InOp));
+  case TargetLowering::TypePromoteFloat: {
+    // Convert the promoted float by hand.
+    if (NOutVT.bitsEq(NInVT)) {
+      SDValue PromotedOp = GetPromotedFloat(InOp);
+      SDValue Trunc = DAG.getNode(ISD::FP_TO_FP16, dl, NOutVT, PromotedOp);
+      return DAG.getNode(ISD::AssertZext, dl, NOutVT, Trunc,
+                         DAG.getValueType(OutVT));
+    }
+    break;
+  }
   case TargetLowering::TypeExpandInteger:
   case TargetLowering::TypeExpandFloat:
     break;
@@ -268,7 +286,7 @@ SDValue DAGTypeLegalizer::PromoteIntRes_BITCAST(SDNode *N) {
     Lo = BitConvertToInteger(Lo);
     Hi = BitConvertToInteger(Hi);
 
-    if (TLI.isBigEndian())
+    if (DAG.getDataLayout().isBigEndian())
       std::swap(Lo, Hi);
 
     InOp = DAG.getNode(ISD::ANY_EXTEND, dl,
@@ -296,8 +314,10 @@ SDValue DAGTypeLegalizer::PromoteIntRes_BSWAP(SDNode *N) {
   SDLoc dl(N);
 
   unsigned DiffBits = NVT.getScalarSizeInBits() - OVT.getScalarSizeInBits();
-  return DAG.getNode(ISD::SRL, dl, NVT, DAG.getNode(ISD::BSWAP, dl, NVT, Op),
-                     DAG.getConstant(DiffBits, TLI.getShiftAmountTy(NVT)));
+  return DAG.getNode(
+      ISD::SRL, dl, NVT, DAG.getNode(ISD::BSWAP, dl, NVT, Op),
+      DAG.getConstant(DiffBits, dl,
+                      TLI.getShiftAmountTy(NVT, DAG.getDataLayout())));
 }
 
 SDValue DAGTypeLegalizer::PromoteIntRes_BUILD_PAIR(SDNode *N) {
@@ -345,7 +365,7 @@ SDValue DAGTypeLegalizer::PromoteIntRes_CTLZ(SDNode *N) {
   // Subtract off the extra leading bits in the bigger type.
   return DAG.getNode(
       ISD::SUB, dl, NVT, Op,
-      DAG.getConstant(NVT.getScalarSizeInBits() - OVT.getScalarSizeInBits(),
+      DAG.getConstant(NVT.getScalarSizeInBits() - OVT.getScalarSizeInBits(), dl,
                       NVT));
 }
 
@@ -366,7 +386,7 @@ SDValue DAGTypeLegalizer::PromoteIntRes_CTTZ(SDNode *N) {
     // the top of the original type.
     auto TopBit = APInt::getOneBitSet(NVT.getScalarSizeInBits(),
                                       OVT.getScalarSizeInBits());
-    Op = DAG.getNode(ISD::OR, dl, NVT, Op, DAG.getConstant(TopBit, NVT));
+    Op = DAG.getNode(ISD::OR, dl, NVT, Op, DAG.getConstant(TopBit, dl, NVT));
   }
   return DAG.getNode(N->getOpcode(), dl, NVT, Op);
 }
@@ -588,10 +608,13 @@ SDValue DAGTypeLegalizer::PromoteIntRes_SETCC(SDNode *N) {
 }
 
 SDValue DAGTypeLegalizer::PromoteIntRes_SHL(SDNode *N) {
-  SDValue Res = GetPromotedInteger(N->getOperand(0));
-  SDValue Amt = N->getOperand(1);
-  Amt = Amt.getValueType().isVector() ? ZExtPromotedInteger(Amt) : Amt;
-  return DAG.getNode(ISD::SHL, SDLoc(N), Res.getValueType(), Res, Amt);
+  SDValue LHS = N->getOperand(0);
+  SDValue RHS = N->getOperand(1);
+  if (getTypeAction(LHS.getValueType()) == TargetLowering::TypePromoteInteger)
+    LHS = GetPromotedInteger(LHS);
+  if (getTypeAction(RHS.getValueType()) == TargetLowering::TypePromoteInteger)
+    RHS = ZExtPromotedInteger(RHS);
+  return DAG.getNode(ISD::SHL, SDLoc(N), LHS.getValueType(), LHS, RHS);
 }
 
 SDValue DAGTypeLegalizer::PromoteIntRes_SIGN_EXTEND_INREG(SDNode *N) {
@@ -611,19 +634,25 @@ SDValue DAGTypeLegalizer::PromoteIntRes_SimpleIntBinOp(SDNode *N) {
 }
 
 SDValue DAGTypeLegalizer::PromoteIntRes_SRA(SDNode *N) {
+  SDValue LHS = N->getOperand(0);
+  SDValue RHS = N->getOperand(1);
   // The input value must be properly sign extended.
-  SDValue Res = SExtPromotedInteger(N->getOperand(0));
-  SDValue Amt = N->getOperand(1);
-  Amt = Amt.getValueType().isVector() ? ZExtPromotedInteger(Amt) : Amt;
-  return DAG.getNode(ISD::SRA, SDLoc(N), Res.getValueType(), Res, Amt);
+  if (getTypeAction(LHS.getValueType()) == TargetLowering::TypePromoteInteger)
+    LHS = SExtPromotedInteger(LHS);
+  if (getTypeAction(RHS.getValueType()) == TargetLowering::TypePromoteInteger)
+    RHS = ZExtPromotedInteger(RHS);
+  return DAG.getNode(ISD::SRA, SDLoc(N), LHS.getValueType(), LHS, RHS);
 }
 
 SDValue DAGTypeLegalizer::PromoteIntRes_SRL(SDNode *N) {
+  SDValue LHS = N->getOperand(0);
+  SDValue RHS = N->getOperand(1);
   // The input value must be properly zero extended.
-  SDValue Res = ZExtPromotedInteger(N->getOperand(0));
-  SDValue Amt = N->getOperand(1);
-  Amt = Amt.getValueType().isVector() ? ZExtPromotedInteger(Amt) : Amt;
-  return DAG.getNode(ISD::SRL, SDLoc(N), Res.getValueType(), Res, Amt);
+  if (getTypeAction(LHS.getValueType()) == TargetLowering::TypePromoteInteger)
+    LHS = ZExtPromotedInteger(LHS);
+  if (getTypeAction(RHS.getValueType()) == TargetLowering::TypePromoteInteger)
+    RHS = ZExtPromotedInteger(RHS);
+  return DAG.getNode(ISD::SRL, SDLoc(N), LHS.getValueType(), LHS, RHS);
 }
 
 SDValue DAGTypeLegalizer::PromoteIntRes_TRUNCATE(SDNode *N) {
@@ -723,9 +752,11 @@ SDValue DAGTypeLegalizer::PromoteIntRes_XMULO(SDNode *N, unsigned ResNo) {
   if (N->getOpcode() == ISD::UMULO) {
     // Unsigned overflow occurred if the high part is non-zero.
     SDValue Hi = DAG.getNode(ISD::SRL, DL, Mul.getValueType(), Mul,
-                             DAG.getIntPtrConstant(SmallVT.getSizeInBits()));
+                             DAG.getIntPtrConstant(SmallVT.getSizeInBits(),
+                                                   DL));
     Overflow = DAG.getSetCC(DL, N->getValueType(1), Hi,
-                            DAG.getConstant(0, Hi.getValueType()), ISD::SETNE);
+                            DAG.getConstant(0, DL, Hi.getValueType()),
+                            ISD::SETNE);
   } else {
     // Signed overflow occurred if the high part does not sign extend the low.
     SDValue SExt = DAG.getNode(ISD::SIGN_EXTEND_INREG, DL, Mul.getValueType(),
@@ -774,7 +805,7 @@ SDValue DAGTypeLegalizer::PromoteIntRes_VAARG(SDNode *N) {
   }
 
   // Handle endianness of the load.
-  if (TLI.isBigEndian())
+  if (DAG.getDataLayout().isBigEndian())
     std::reverse(Parts.begin(), Parts.end());
 
   // Assemble the parts in the promoted type.
@@ -784,8 +815,8 @@ SDValue DAGTypeLegalizer::PromoteIntRes_VAARG(SDNode *N) {
     SDValue Part = DAG.getNode(ISD::ZERO_EXTEND, dl, NVT, Parts[i]);
     // Shift it to the right position and "or" it in.
     Part = DAG.getNode(ISD::SHL, dl, NVT, Part,
-                       DAG.getConstant(i * RegVT.getSizeInBits(),
-                                       TLI.getPointerTy()));
+                       DAG.getConstant(i * RegVT.getSizeInBits(), dl,
+                                       TLI.getPointerTy(DAG.getDataLayout())));
     Res = DAG.getNode(ISD::OR, dl, NVT, Res, Part);
   }
 
@@ -852,6 +883,7 @@ bool DAGTypeLegalizer::PromoteIntegerOperand(SDNode *N, unsigned OpNo) {
   case ISD::FP16_TO_FP:
   case ISD::UINT_TO_FP:   Res = PromoteIntOp_UINT_TO_FP(N); break;
   case ISD::ZERO_EXTEND:  Res = PromoteIntOp_ZERO_EXTEND(N); break;
+  case ISD::EXTRACT_SUBVECTOR: Res = PromoteIntOp_EXTRACT_SUBVECTOR(N); break;
 
   case ISD::SHL:
   case ISD::SRA:
@@ -977,7 +1009,8 @@ SDValue DAGTypeLegalizer::PromoteIntOp_BUILD_PAIR(SDNode *N) {
   SDLoc dl(N);
 
   Hi = DAG.getNode(ISD::SHL, dl, N->getValueType(0), Hi,
-                   DAG.getConstant(OVT.getSizeInBits(), TLI.getPointerTy()));
+                   DAG.getConstant(OVT.getSizeInBits(), dl,
+                                   TLI.getPointerTy(DAG.getDataLayout())));
   return DAG.getNode(ISD::OR, dl, N->getValueType(0), Lo, Hi);
 }
 
@@ -1036,7 +1069,7 @@ SDValue DAGTypeLegalizer::PromoteIntOp_INSERT_VECTOR_ELT(SDNode *N,
 
   // Promote the index.
   SDValue Idx = DAG.getZExtOrTrunc(N->getOperand(2), SDLoc(N),
-                                   TLI.getVectorIdxTy());
+                                   TLI.getVectorIdxTy(DAG.getDataLayout()));
   return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0),
                                 N->getOperand(1), Idx), 0);
 }
@@ -1147,7 +1180,7 @@ SDValue DAGTypeLegalizer::PromoteIntOp_MSTORE(MaskedStoreSDNode *N, unsigned OpN
 
         unsigned NumConcat = WidenNumElts / MaskNumElts;
         SmallVector<SDValue, 16> Ops(NumConcat);
-        SDValue ZeroVal = DAG.getConstant(0, MaskVT);
+        SDValue ZeroVal = DAG.getConstant(0, dl, MaskVT);
         Ops[0] = Mask;
         for (unsigned i = 1; i != NumConcat; ++i)
           Ops[i] = ZeroVal;
@@ -1329,9 +1362,9 @@ std::pair <SDValue, SDValue> DAGTypeLegalizer::ExpandAtomic(SDNode *Node) {
   return ExpandChainLibCall(LC, Node, false);
 }
 
-/// ExpandShiftByConstant - N is a shift by a value that needs to be expanded,
+/// N is a shift by a value that needs to be expanded,
 /// and the shift amount is a constant 'Amt'.  Expand the operation.
-void DAGTypeLegalizer::ExpandShiftByConstant(SDNode *N, unsigned Amt,
+void DAGTypeLegalizer::ExpandShiftByConstant(SDNode *N, const APInt &Amt,
                                              SDValue &Lo, SDValue &Hi) {
   SDLoc DL(N);
   // Expand the incoming operand to be shifted, so that we have its parts
@@ -1352,14 +1385,14 @@ void DAGTypeLegalizer::ExpandShiftByConstant(SDNode *N, unsigned Amt,
   EVT ShTy = N->getOperand(1).getValueType();
 
   if (N->getOpcode() == ISD::SHL) {
-    if (Amt > VTBits) {
-      Lo = Hi = DAG.getConstant(0, NVT);
-    } else if (Amt > NVTBits) {
-      Lo = DAG.getConstant(0, NVT);
+    if (Amt.ugt(VTBits)) {
+      Lo = Hi = DAG.getConstant(0, DL, NVT);
+    } else if (Amt.ugt(NVTBits)) {
+      Lo = DAG.getConstant(0, DL, NVT);
       Hi = DAG.getNode(ISD::SHL, DL,
-                       NVT, InL, DAG.getConstant(Amt-NVTBits, ShTy));
+                       NVT, InL, DAG.getConstant(Amt - NVTBits, DL, ShTy));
     } else if (Amt == NVTBits) {
-      Lo = DAG.getConstant(0, NVT);
+      Lo = DAG.getConstant(0, DL, NVT);
       Hi = InL;
     } else if (Amt == 1 &&
                TLI.isOperationLegalOrCustom(ISD::ADDC,
@@ -1371,58 +1404,57 @@ void DAGTypeLegalizer::ExpandShiftByConstant(SDNode *N, unsigned Amt,
       SDValue HiOps[3] = { InH, InH, Lo.getValue(1) };
       Hi = DAG.getNode(ISD::ADDE, DL, VTList, HiOps);
     } else {
-      Lo = DAG.getNode(ISD::SHL, DL, NVT, InL, DAG.getConstant(Amt, ShTy));
+      Lo = DAG.getNode(ISD::SHL, DL, NVT, InL, DAG.getConstant(Amt, DL, ShTy));
       Hi = DAG.getNode(ISD::OR, DL, NVT,
                        DAG.getNode(ISD::SHL, DL, NVT, InH,
-                                   DAG.getConstant(Amt, ShTy)),
+                                   DAG.getConstant(Amt, DL, ShTy)),
                        DAG.getNode(ISD::SRL, DL, NVT, InL,
-                                   DAG.getConstant(NVTBits-Amt, ShTy)));
+                                   DAG.getConstant(-Amt + NVTBits, DL, ShTy)));
     }
     return;
   }
 
   if (N->getOpcode() == ISD::SRL) {
-    if (Amt > VTBits) {
-      Lo = DAG.getConstant(0, NVT);
-      Hi = DAG.getConstant(0, NVT);
-    } else if (Amt > NVTBits) {
+    if (Amt.ugt(VTBits)) {
+      Lo = Hi = DAG.getConstant(0, DL, NVT);
+    } else if (Amt.ugt(NVTBits)) {
       Lo = DAG.getNode(ISD::SRL, DL,
-                       NVT, InH, DAG.getConstant(Amt-NVTBits,ShTy));
-      Hi = DAG.getConstant(0, NVT);
+                       NVT, InH, DAG.getConstant(Amt - NVTBits, DL, ShTy));
+      Hi = DAG.getConstant(0, DL, NVT);
     } else if (Amt == NVTBits) {
       Lo = InH;
-      Hi = DAG.getConstant(0, NVT);
+      Hi = DAG.getConstant(0, DL, NVT);
     } else {
       Lo = DAG.getNode(ISD::OR, DL, NVT,
                        DAG.getNode(ISD::SRL, DL, NVT, InL,
-                                   DAG.getConstant(Amt, ShTy)),
+                                   DAG.getConstant(Amt, DL, ShTy)),
                        DAG.getNode(ISD::SHL, DL, NVT, InH,
-                                   DAG.getConstant(NVTBits-Amt, ShTy)));
-      Hi = DAG.getNode(ISD::SRL, DL, NVT, InH, DAG.getConstant(Amt, ShTy));
+                                   DAG.getConstant(-Amt + NVTBits, DL, ShTy)));
+      Hi = DAG.getNode(ISD::SRL, DL, NVT, InH, DAG.getConstant(Amt, DL, ShTy));
     }
     return;
   }
 
   assert(N->getOpcode() == ISD::SRA && "Unknown shift!");
-  if (Amt > VTBits) {
+  if (Amt.ugt(VTBits)) {
     Hi = Lo = DAG.getNode(ISD::SRA, DL, NVT, InH,
-                          DAG.getConstant(NVTBits-1, ShTy));
-  } else if (Amt > NVTBits) {
+                          DAG.getConstant(NVTBits - 1, DL, ShTy));
+  } else if (Amt.ugt(NVTBits)) {
     Lo = DAG.getNode(ISD::SRA, DL, NVT, InH,
-                     DAG.getConstant(Amt-NVTBits, ShTy));
+                     DAG.getConstant(Amt - NVTBits, DL, ShTy));
     Hi = DAG.getNode(ISD::SRA, DL, NVT, InH,
-                     DAG.getConstant(NVTBits-1, ShTy));
+                     DAG.getConstant(NVTBits - 1, DL, ShTy));
   } else if (Amt == NVTBits) {
     Lo = InH;
     Hi = DAG.getNode(ISD::SRA, DL, NVT, InH,
-                     DAG.getConstant(NVTBits-1, ShTy));
+                     DAG.getConstant(NVTBits - 1, DL, ShTy));
   } else {
     Lo = DAG.getNode(ISD::OR, DL, NVT,
                      DAG.getNode(ISD::SRL, DL, NVT, InL,
-                                 DAG.getConstant(Amt, ShTy)),
+                                 DAG.getConstant(Amt, DL, ShTy)),
                      DAG.getNode(ISD::SHL, DL, NVT, InH,
-                                 DAG.getConstant(NVTBits-Amt, ShTy)));
-    Hi = DAG.getNode(ISD::SRA, DL, NVT, InH, DAG.getConstant(Amt, ShTy));
+                                 DAG.getConstant(-Amt + NVTBits, DL, ShTy)));
+    Hi = DAG.getNode(ISD::SRA, DL, NVT, InH, DAG.getConstant(Amt, DL, ShTy));
   }
 }
 
@@ -1458,21 +1490,21 @@ ExpandShiftWithKnownAmountBit(SDNode *N, SDValue &Lo, SDValue &Hi) {
   if (KnownOne.intersects(HighBitMask)) {
     // Mask out the high bit, which we know is set.
     Amt = DAG.getNode(ISD::AND, dl, ShTy, Amt,
-                      DAG.getConstant(~HighBitMask, ShTy));
+                      DAG.getConstant(~HighBitMask, dl, ShTy));
 
     switch (N->getOpcode()) {
     default: llvm_unreachable("Unknown shift");
     case ISD::SHL:
-      Lo = DAG.getConstant(0, NVT);              // Low part is zero.
+      Lo = DAG.getConstant(0, dl, NVT);              // Low part is zero.
       Hi = DAG.getNode(ISD::SHL, dl, NVT, InL, Amt); // High part from Lo part.
       return true;
     case ISD::SRL:
-      Hi = DAG.getConstant(0, NVT);              // Hi part is zero.
+      Hi = DAG.getConstant(0, dl, NVT);              // Hi part is zero.
       Lo = DAG.getNode(ISD::SRL, dl, NVT, InH, Amt); // Lo part from Hi part.
       return true;
     case ISD::SRA:
       Hi = DAG.getNode(ISD::SRA, dl, NVT, InH,       // Sign extend high part.
-                       DAG.getConstant(NVTBits-1, ShTy));
+                       DAG.getConstant(NVTBits - 1, dl, ShTy));
       Lo = DAG.getNode(ISD::SRA, dl, NVT, InH, Amt); // Lo part from Hi part.
       return true;
     }
@@ -1485,7 +1517,7 @@ ExpandShiftWithKnownAmountBit(SDNode *N, SDValue &Lo, SDValue &Hi) {
     // shift if x is zero.  We can use XOR here because x is known to be smaller
     // than 32.
     SDValue Amt2 = DAG.getNode(ISD::XOR, dl, ShTy, Amt,
-                               DAG.getConstant(NVTBits-1, ShTy));
+                               DAG.getConstant(NVTBits - 1, dl, ShTy));
 
     unsigned Op1, Op2;
     switch (N->getOpcode()) {
@@ -1501,7 +1533,7 @@ ExpandShiftWithKnownAmountBit(SDNode *N, SDValue &Lo, SDValue &Hi) {
 
     // Use a little trick to get the bits that move from Lo to Hi. First
     // shift by one bit.
-    SDValue Sh1 = DAG.getNode(Op2, dl, NVT, InL, DAG.getConstant(1, ShTy));
+    SDValue Sh1 = DAG.getNode(Op2, dl, NVT, InL, DAG.getConstant(1, dl, ShTy));
     // Then compute the remaining shift with amount-1.
     SDValue Sh2 = DAG.getNode(Op2, dl, NVT, Sh1, Amt2);
 
@@ -1532,11 +1564,14 @@ ExpandShiftWithUnknownAmountBit(SDNode *N, SDValue &Lo, SDValue &Hi) {
   SDValue InL, InH;
   GetExpandedInteger(N->getOperand(0), InL, InH);
 
-  SDValue NVBitsNode = DAG.getConstant(NVTBits, ShTy);
+  SDValue NVBitsNode = DAG.getConstant(NVTBits, dl, ShTy);
   SDValue AmtExcess = DAG.getNode(ISD::SUB, dl, ShTy, Amt, NVBitsNode);
   SDValue AmtLack = DAG.getNode(ISD::SUB, dl, ShTy, NVBitsNode, Amt);
   SDValue isShort = DAG.getSetCC(dl, getSetCCResultType(ShTy),
                                  Amt, NVBitsNode, ISD::SETULT);
+  SDValue isZero = DAG.getSetCC(dl, getSetCCResultType(ShTy),
+                                Amt, DAG.getConstant(0, dl, ShTy),
+                                ISD::SETEQ);
 
   SDValue LoS, HiS, LoL, HiL;
   switch (N->getOpcode()) {
@@ -1546,16 +1581,15 @@ ExpandShiftWithUnknownAmountBit(SDNode *N, SDValue &Lo, SDValue &Hi) {
     LoS = DAG.getNode(ISD::SHL, dl, NVT, InL, Amt);
     HiS = DAG.getNode(ISD::OR, dl, NVT,
                       DAG.getNode(ISD::SHL, dl, NVT, InH, Amt),
-    // FIXME: If Amt is zero, the following shift generates an undefined result
-    // on some architectures.
                       DAG.getNode(ISD::SRL, dl, NVT, InL, AmtLack));
 
     // Long: ShAmt >= NVTBits
-    LoL = DAG.getConstant(0, NVT);                        // Lo part is zero.
+    LoL = DAG.getConstant(0, dl, NVT);                    // Lo part is zero.
     HiL = DAG.getNode(ISD::SHL, dl, NVT, InL, AmtExcess); // Hi from Lo part.
 
     Lo = DAG.getSelect(dl, NVT, isShort, LoS, LoL);
-    Hi = DAG.getSelect(dl, NVT, isShort, HiS, HiL);
+    Hi = DAG.getSelect(dl, NVT, isZero, InH,
+                       DAG.getSelect(dl, NVT, isShort, HiS, HiL));
     return true;
   case ISD::SRL:
     // Short: ShAmt < NVTBits
@@ -1567,10 +1601,11 @@ ExpandShiftWithUnknownAmountBit(SDNode *N, SDValue &Lo, SDValue &Hi) {
                       DAG.getNode(ISD::SHL, dl, NVT, InH, AmtLack));
 
     // Long: ShAmt >= NVTBits
-    HiL = DAG.getConstant(0, NVT);                        // Hi part is zero.
+    HiL = DAG.getConstant(0, dl, NVT);                    // Hi part is zero.
     LoL = DAG.getNode(ISD::SRL, dl, NVT, InH, AmtExcess); // Lo from Hi part.
 
-    Lo = DAG.getSelect(dl, NVT, isShort, LoS, LoL);
+    Lo = DAG.getSelect(dl, NVT, isZero, InL,
+                       DAG.getSelect(dl, NVT, isShort, LoS, LoL));
     Hi = DAG.getSelect(dl, NVT, isShort, HiS, HiL);
     return true;
   case ISD::SRA:
@@ -1578,16 +1613,15 @@ ExpandShiftWithUnknownAmountBit(SDNode *N, SDValue &Lo, SDValue &Hi) {
     HiS = DAG.getNode(ISD::SRA, dl, NVT, InH, Amt);
     LoS = DAG.getNode(ISD::OR, dl, NVT,
                       DAG.getNode(ISD::SRL, dl, NVT, InL, Amt),
-    // FIXME: If Amt is zero, the following shift generates an undefined result
-    // on some architectures.
                       DAG.getNode(ISD::SHL, dl, NVT, InH, AmtLack));
 
     // Long: ShAmt >= NVTBits
     HiL = DAG.getNode(ISD::SRA, dl, NVT, InH,             // Sign of Hi part.
-                      DAG.getConstant(NVTBits-1, ShTy));
+                      DAG.getConstant(NVTBits - 1, dl, ShTy));
     LoL = DAG.getNode(ISD::SRA, dl, NVT, InH, AmtExcess); // Lo from Hi part.
 
-    Lo = DAG.getSelect(dl, NVT, isShort, LoS, LoL);
+    Lo = DAG.getSelect(dl, NVT, isZero, InL,
+                       DAG.getSelect(dl, NVT, isShort, LoS, LoL));
     Hi = DAG.getSelect(dl, NVT, isShort, HiS, HiL);
     return true;
   }
@@ -1629,18 +1663,50 @@ void DAGTypeLegalizer::ExpandIntRes_ADDSUB(SDNode *N,
     return;
   }
 
+  bool hasOVF =
+    TLI.isOperationLegalOrCustom(N->getOpcode() == ISD::ADD ?
+                                   ISD::UADDO : ISD::USUBO,
+                                 TLI.getTypeToExpandTo(*DAG.getContext(), NVT));
+  if (hasOVF) {
+    SDVTList VTList = DAG.getVTList(NVT, NVT);
+    TargetLoweringBase::BooleanContent BoolType = TLI.getBooleanContents(NVT);
+    int RevOpc;
+    if (N->getOpcode() == ISD::ADD) {
+      RevOpc = ISD::SUB;
+      Lo = DAG.getNode(ISD::UADDO, dl, VTList, LoOps);
+      Hi = DAG.getNode(ISD::ADD, dl, NVT, makeArrayRef(HiOps, 2));
+    } else {
+      RevOpc = ISD::ADD;
+      Lo = DAG.getNode(ISD::USUBO, dl, VTList, LoOps);
+      Hi = DAG.getNode(ISD::SUB, dl, NVT, makeArrayRef(HiOps, 2));
+    }
+    SDValue OVF = Lo.getValue(1);
+
+    switch (BoolType) {
+    case TargetLoweringBase::UndefinedBooleanContent:
+      OVF = DAG.getNode(ISD::AND, dl, NVT, DAG.getConstant(1, dl, NVT), OVF);
+      // Fallthrough
+    case TargetLoweringBase::ZeroOrOneBooleanContent:
+      Hi = DAG.getNode(N->getOpcode(), dl, NVT, Hi, OVF);
+      break;
+    case TargetLoweringBase::ZeroOrNegativeOneBooleanContent:
+      Hi = DAG.getNode(RevOpc, dl, NVT, Hi, OVF);
+    }
+    return;
+  }
+
   if (N->getOpcode() == ISD::ADD) {
     Lo = DAG.getNode(ISD::ADD, dl, NVT, LoOps);
     Hi = DAG.getNode(ISD::ADD, dl, NVT, makeArrayRef(HiOps, 2));
     SDValue Cmp1 = DAG.getSetCC(dl, getSetCCResultType(NVT), Lo, LoOps[0],
                                 ISD::SETULT);
     SDValue Carry1 = DAG.getSelect(dl, NVT, Cmp1,
-                                   DAG.getConstant(1, NVT),
-                                   DAG.getConstant(0, NVT));
+                                   DAG.getConstant(1, dl, NVT),
+                                   DAG.getConstant(0, dl, NVT));
     SDValue Cmp2 = DAG.getSetCC(dl, getSetCCResultType(NVT), Lo, LoOps[1],
                                 ISD::SETULT);
     SDValue Carry2 = DAG.getSelect(dl, NVT, Cmp2,
-                                   DAG.getConstant(1, NVT), Carry1);
+                                   DAG.getConstant(1, dl, NVT), Carry1);
     Hi = DAG.getNode(ISD::ADD, dl, NVT, Hi, Carry2);
   } else {
     Lo = DAG.getNode(ISD::SUB, dl, NVT, LoOps);
@@ -1649,8 +1715,8 @@ void DAGTypeLegalizer::ExpandIntRes_ADDSUB(SDNode *N,
       DAG.getSetCC(dl, getSetCCResultType(LoOps[0].getValueType()),
                    LoOps[0], LoOps[1], ISD::SETULT);
     SDValue Borrow = DAG.getSelect(dl, NVT, Cmp,
-                                   DAG.getConstant(1, NVT),
-                                   DAG.getConstant(0, NVT));
+                                   DAG.getConstant(1, dl, NVT),
+                                   DAG.getConstant(0, dl, NVT));
     Hi = DAG.getNode(ISD::SUB, dl, NVT, Hi, Borrow);
   }
 }
@@ -1747,7 +1813,8 @@ void DAGTypeLegalizer::ExpandIntRes_AssertSext(SDNode *N,
     Lo = DAG.getNode(ISD::AssertSext, dl, NVT, Lo, DAG.getValueType(EVT));
     // The high part replicates the sign bit of Lo, make it explicit.
     Hi = DAG.getNode(ISD::SRA, dl, NVT, Lo,
-                     DAG.getConstant(NVTBits-1, TLI.getPointerTy()));
+                     DAG.getConstant(NVTBits - 1, dl,
+                                     TLI.getPointerTy(DAG.getDataLayout())));
   }
 }
 
@@ -1767,7 +1834,7 @@ void DAGTypeLegalizer::ExpandIntRes_AssertZext(SDNode *N,
   } else {
     Lo = DAG.getNode(ISD::AssertZext, dl, NVT, Lo, DAG.getValueType(EVT));
     // The high part must be zero, make it explicit.
-    Hi = DAG.getConstant(0, NVT);
+    Hi = DAG.getConstant(0, dl, NVT);
   }
 }
 
@@ -1787,8 +1854,9 @@ void DAGTypeLegalizer::ExpandIntRes_Constant(SDNode *N,
   const APInt &Cst = Constant->getAPIntValue();
   bool IsTarget = Constant->isTargetOpcode();
   bool IsOpaque = Constant->isOpaque();
-  Lo = DAG.getConstant(Cst.trunc(NBitWidth), NVT, IsTarget, IsOpaque);
-  Hi = DAG.getConstant(Cst.lshr(NBitWidth).trunc(NBitWidth), NVT, IsTarget,
+  SDLoc dl(N);
+  Lo = DAG.getConstant(Cst.trunc(NBitWidth), dl, NVT, IsTarget, IsOpaque);
+  Hi = DAG.getConstant(Cst.lshr(NBitWidth).trunc(NBitWidth), dl, NVT, IsTarget,
                        IsOpaque);
 }
 
@@ -1800,15 +1868,16 @@ void DAGTypeLegalizer::ExpandIntRes_CTLZ(SDNode *N,
   EVT NVT = Lo.getValueType();
 
   SDValue HiNotZero = DAG.getSetCC(dl, getSetCCResultType(NVT), Hi,
-                                   DAG.getConstant(0, NVT), ISD::SETNE);
+                                   DAG.getConstant(0, dl, NVT), ISD::SETNE);
 
   SDValue LoLZ = DAG.getNode(N->getOpcode(), dl, NVT, Lo);
   SDValue HiLZ = DAG.getNode(ISD::CTLZ_ZERO_UNDEF, dl, NVT, Hi);
 
   Lo = DAG.getSelect(dl, NVT, HiNotZero, HiLZ,
                      DAG.getNode(ISD::ADD, dl, NVT, LoLZ,
-                                 DAG.getConstant(NVT.getSizeInBits(), NVT)));
-  Hi = DAG.getConstant(0, NVT);
+                                 DAG.getConstant(NVT.getSizeInBits(), dl,
+                                                 NVT)));
+  Hi = DAG.getConstant(0, dl, NVT);
 }
 
 void DAGTypeLegalizer::ExpandIntRes_CTPOP(SDNode *N,
@@ -1819,7 +1888,7 @@ void DAGTypeLegalizer::ExpandIntRes_CTPOP(SDNode *N,
   EVT NVT = Lo.getValueType();
   Lo = DAG.getNode(ISD::ADD, dl, NVT, DAG.getNode(ISD::CTPOP, dl, NVT, Lo),
                    DAG.getNode(ISD::CTPOP, dl, NVT, Hi));
-  Hi = DAG.getConstant(0, NVT);
+  Hi = DAG.getConstant(0, dl, NVT);
 }
 
 void DAGTypeLegalizer::ExpandIntRes_CTTZ(SDNode *N,
@@ -1830,22 +1899,27 @@ void DAGTypeLegalizer::ExpandIntRes_CTTZ(SDNode *N,
   EVT NVT = Lo.getValueType();
 
   SDValue LoNotZero = DAG.getSetCC(dl, getSetCCResultType(NVT), Lo,
-                                   DAG.getConstant(0, NVT), ISD::SETNE);
+                                   DAG.getConstant(0, dl, NVT), ISD::SETNE);
 
   SDValue LoLZ = DAG.getNode(ISD::CTTZ_ZERO_UNDEF, dl, NVT, Lo);
   SDValue HiLZ = DAG.getNode(N->getOpcode(), dl, NVT, Hi);
 
   Lo = DAG.getSelect(dl, NVT, LoNotZero, LoLZ,
                      DAG.getNode(ISD::ADD, dl, NVT, HiLZ,
-                                 DAG.getConstant(NVT.getSizeInBits(), NVT)));
-  Hi = DAG.getConstant(0, NVT);
+                                 DAG.getConstant(NVT.getSizeInBits(), dl,
+                                                 NVT)));
+  Hi = DAG.getConstant(0, dl, NVT);
 }
 
 void DAGTypeLegalizer::ExpandIntRes_FP_TO_SINT(SDNode *N, SDValue &Lo,
                                                SDValue &Hi) {
   SDLoc dl(N);
   EVT VT = N->getValueType(0);
+
   SDValue Op = N->getOperand(0);
+  if (getTypeAction(Op.getValueType()) == TargetLowering::TypePromoteFloat)
+    Op = GetPromotedFloat(Op);
+
   RTLIB::Libcall LC = RTLIB::getFPTOSINT(Op.getValueType(), VT);
   assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unexpected fp-to-sint conversion!");
   SplitInteger(TLI.makeLibCall(DAG, LC, VT, &Op, 1, true/*irrelevant*/,
@@ -1857,7 +1931,11 @@ void DAGTypeLegalizer::ExpandIntRes_FP_TO_UINT(SDNode *N, SDValue &Lo,
                                                SDValue &Hi) {
   SDLoc dl(N);
   EVT VT = N->getValueType(0);
+
   SDValue Op = N->getOperand(0);
+  if (getTypeAction(Op.getValueType()) == TargetLowering::TypePromoteFloat)
+    Op = GetPromotedFloat(Op);
+
   RTLIB::Libcall LC = RTLIB::getFPTOUINT(Op.getValueType(), VT);
   assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unexpected fp-to-uint conversion!");
   SplitInteger(TLI.makeLibCall(DAG, LC, VT, &Op, 1, false/*irrelevant*/,
@@ -1903,16 +1981,17 @@ void DAGTypeLegalizer::ExpandIntRes_LOAD(LoadSDNode *N,
       // lo part.
       unsigned LoSize = Lo.getValueType().getSizeInBits();
       Hi = DAG.getNode(ISD::SRA, dl, NVT, Lo,
-                       DAG.getConstant(LoSize-1, TLI.getPointerTy()));
+                       DAG.getConstant(LoSize - 1, dl,
+                                       TLI.getPointerTy(DAG.getDataLayout())));
     } else if (ExtType == ISD::ZEXTLOAD) {
       // The high part is just a zero.
-      Hi = DAG.getConstant(0, NVT);
+      Hi = DAG.getConstant(0, dl, NVT);
     } else {
       assert(ExtType == ISD::EXTLOAD && "Unknown extload!");
       // The high part is undefined.
       Hi = DAG.getUNDEF(NVT);
     }
-  } else if (TLI.isLittleEndian()) {
+  } else if (DAG.getDataLayout().isLittleEndian()) {
     // Little-endian - low bits are at low addresses.
     Lo = DAG.getLoad(NVT, dl, Ch, Ptr, N->getPointerInfo(),
                      isVolatile, isNonTemporal, isInvariant, Alignment,
@@ -1925,7 +2004,7 @@ void DAGTypeLegalizer::ExpandIntRes_LOAD(LoadSDNode *N,
     // Increment the pointer to the other half.
     unsigned IncrementSize = NVT.getSizeInBits()/8;
     Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr,
-                      DAG.getConstant(IncrementSize, Ptr.getValueType()));
+                      DAG.getConstant(IncrementSize, dl, Ptr.getValueType()));
     Hi = DAG.getExtLoad(ExtType, dl, NVT, Ch, Ptr,
                         N->getPointerInfo().getWithOffset(IncrementSize), NEVT,
                         isVolatile, isNonTemporal, isInvariant,
@@ -1952,7 +2031,7 @@ void DAGTypeLegalizer::ExpandIntRes_LOAD(LoadSDNode *N,
 
     // Increment the pointer to the other half.
     Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr,
-                      DAG.getConstant(IncrementSize, Ptr.getValueType()));
+                      DAG.getConstant(IncrementSize, dl, Ptr.getValueType()));
     // Load the rest of the low bits.
     Lo = DAG.getExtLoad(ISD::ZEXTLOAD, dl, NVT, Ch, Ptr,
                         N->getPointerInfo().getWithOffset(IncrementSize),
@@ -1967,15 +2046,16 @@ void DAGTypeLegalizer::ExpandIntRes_LOAD(LoadSDNode *N,
 
     if (ExcessBits < NVT.getSizeInBits()) {
       // Transfer low bits from the bottom of Hi to the top of Lo.
-      Lo = DAG.getNode(ISD::OR, dl, NVT, Lo,
-                       DAG.getNode(ISD::SHL, dl, NVT, Hi,
-                                   DAG.getConstant(ExcessBits,
-                                                   TLI.getPointerTy())));
+      Lo = DAG.getNode(
+          ISD::OR, dl, NVT, Lo,
+          DAG.getNode(ISD::SHL, dl, NVT, Hi,
+                      DAG.getConstant(ExcessBits, dl,
+                                      TLI.getPointerTy(DAG.getDataLayout()))));
       // Move high bits to the right position in Hi.
-      Hi = DAG.getNode(ExtType == ISD::SEXTLOAD ? ISD::SRA : ISD::SRL, dl,
-                       NVT, Hi,
-                       DAG.getConstant(NVT.getSizeInBits() - ExcessBits,
-                                       TLI.getPointerTy()));
+      Hi = DAG.getNode(ExtType == ISD::SEXTLOAD ? ISD::SRA : ISD::SRL, dl, NVT,
+                       Hi,
+                       DAG.getConstant(NVT.getSizeInBits() - ExcessBits, dl,
+                                       TLI.getPointerTy(DAG.getDataLayout())));
     }
   }
 
@@ -2050,7 +2130,7 @@ void DAGTypeLegalizer::ExpandIntRes_SADDSUBO(SDNode *Node,
   //   Overflow -> (LHSSign != RHSSign) && (LHSSign != SumSign)
   //
   EVT OType = Node->getValueType(1);
-  SDValue Zero = DAG.getConstant(0, LHS.getValueType());
+  SDValue Zero = DAG.getConstant(0, dl, LHS.getValueType());
 
   SDValue LHSSign = DAG.getSetCC(dl, OType, LHS, Zero, ISD::SETGE);
   SDValue RHSSign = DAG.getSetCC(dl, OType, RHS, Zero, ISD::SETGE);
@@ -2071,6 +2151,13 @@ void DAGTypeLegalizer::ExpandIntRes_SDIV(SDNode *N,
                                          SDValue &Lo, SDValue &Hi) {
   EVT VT = N->getValueType(0);
   SDLoc dl(N);
+  SDValue Ops[2] = { N->getOperand(0), N->getOperand(1) };
+
+  if (TLI.getOperationAction(ISD::SDIVREM, VT) == TargetLowering::Custom) {
+    SDValue Res = DAG.getNode(ISD::SDIVREM, dl, DAG.getVTList(VT, VT), Ops);
+    SplitInteger(Res.getValue(0), Lo, Hi);
+    return;
+  }
 
   RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
   if (VT == MVT::i16)
@@ -2083,7 +2170,6 @@ void DAGTypeLegalizer::ExpandIntRes_SDIV(SDNode *N,
     LC = RTLIB::SDIV_I128;
   assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported SDIV!");
 
-  SDValue Ops[2] = { N->getOperand(0), N->getOperand(1) };
   SplitInteger(TLI.makeLibCall(DAG, LC, VT, Ops, 2, true, dl).first, Lo, Hi);
 }
 
@@ -2095,7 +2181,7 @@ void DAGTypeLegalizer::ExpandIntRes_Shift(SDNode *N,
   // If we can emit an efficient shift operation, do so now.  Check to see if
   // the RHS is a constant.
   if (ConstantSDNode *CN = dyn_cast<ConstantSDNode>(N->getOperand(1)))
-    return ExpandShiftByConstant(N, CN->getZExtValue(), Lo, Hi);
+    return ExpandShiftByConstant(N, CN->getAPIntValue(), Lo, Hi);
 
   // If we can determine that the high bit of the shift is zero or one, even if
   // the low bits are variable, emit this shift in an optimized form.
@@ -2128,7 +2214,7 @@ void DAGTypeLegalizer::ExpandIntRes_Shift(SDNode *N,
     // have an illegal type.  Fix that first by casting the operand, otherwise
     // the new SHL_PARTS operation would need further legalization.
     SDValue ShiftOp = N->getOperand(1);
-    EVT ShiftTy = TLI.getShiftAmountTy(VT);
+    EVT ShiftTy = TLI.getShiftAmountTy(VT, DAG.getDataLayout());
     assert(ShiftTy.getScalarType().getSizeInBits() >=
            Log2_32_Ceil(VT.getScalarType().getSizeInBits()) &&
            "ShiftAmountTy is too small to cover the range of this type!");
@@ -2198,8 +2284,9 @@ void DAGTypeLegalizer::ExpandIntRes_SIGN_EXTEND(SDNode *N,
     Lo = DAG.getNode(ISD::SIGN_EXTEND, dl, NVT, N->getOperand(0));
     // The high part is obtained by SRA'ing all but one of the bits of low part.
     unsigned LoSize = NVT.getSizeInBits();
-    Hi = DAG.getNode(ISD::SRA, dl, NVT, Lo,
-                     DAG.getConstant(LoSize-1, TLI.getPointerTy()));
+    Hi = DAG.getNode(
+        ISD::SRA, dl, NVT, Lo,
+        DAG.getConstant(LoSize - 1, dl, TLI.getPointerTy(DAG.getDataLayout())));
   } else {
     // For example, extension of an i48 to an i64.  The operand type necessarily
     // promotes to the result type, so will end up being expanded too.
@@ -2233,8 +2320,8 @@ ExpandIntRes_SIGN_EXTEND_INREG(SDNode *N, SDValue &Lo, SDValue &Hi) {
     // The high part gets the sign extension from the lo-part.  This handles
     // things like sextinreg V:i64 from i8.
     Hi = DAG.getNode(ISD::SRA, dl, Hi.getValueType(), Lo,
-                     DAG.getConstant(Hi.getValueType().getSizeInBits()-1,
-                                     TLI.getPointerTy()));
+                     DAG.getConstant(Hi.getValueType().getSizeInBits() - 1, dl,
+                                     TLI.getPointerTy(DAG.getDataLayout())));
   } else {
     // For example, extension of an i48 to an i64.  Leave the low part alone,
     // sext_inreg the high part.
@@ -2250,6 +2337,13 @@ void DAGTypeLegalizer::ExpandIntRes_SREM(SDNode *N,
                                          SDValue &Lo, SDValue &Hi) {
   EVT VT = N->getValueType(0);
   SDLoc dl(N);
+  SDValue Ops[2] = { N->getOperand(0), N->getOperand(1) };
+
+  if (TLI.getOperationAction(ISD::SDIVREM, VT) == TargetLowering::Custom) {
+    SDValue Res = DAG.getNode(ISD::SDIVREM, dl, DAG.getVTList(VT, VT), Ops);
+    SplitInteger(Res.getValue(1), Lo, Hi);
+    return;
+  }
 
   RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
   if (VT == MVT::i16)
@@ -2262,7 +2356,6 @@ void DAGTypeLegalizer::ExpandIntRes_SREM(SDNode *N,
     LC = RTLIB::SREM_I128;
   assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported SREM!");
 
-  SDValue Ops[2] = { N->getOperand(0), N->getOperand(1) };
   SplitInteger(TLI.makeLibCall(DAG, LC, VT, Ops, 2, true, dl).first, Lo, Hi);
 }
 
@@ -2271,9 +2364,10 @@ void DAGTypeLegalizer::ExpandIntRes_TRUNCATE(SDNode *N,
   EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
   SDLoc dl(N);
   Lo = DAG.getNode(ISD::TRUNCATE, dl, NVT, N->getOperand(0));
-  Hi = DAG.getNode(ISD::SRL, dl,
-                   N->getOperand(0).getValueType(), N->getOperand(0),
-                   DAG.getConstant(NVT.getSizeInBits(), TLI.getPointerTy()));
+  Hi = DAG.getNode(ISD::SRL, dl, N->getOperand(0).getValueType(),
+                   N->getOperand(0),
+                   DAG.getConstant(NVT.getSizeInBits(), dl,
+                                   TLI.getPointerTy(DAG.getDataLayout())));
   Hi = DAG.getNode(ISD::TRUNCATE, dl, NVT, Hi);
 }
 
@@ -2315,21 +2409,21 @@ void DAGTypeLegalizer::ExpandIntRes_XMULO(SDNode *N,
     // A divide for UMULO will be faster than a function call. Select to
     // make sure we aren't using 0.
     SDValue isZero = DAG.getSetCC(dl, getSetCCResultType(VT),
-                                  RHS, DAG.getConstant(0, VT), ISD::SETEQ);
+                                  RHS, DAG.getConstant(0, dl, VT), ISD::SETEQ);
     SDValue NotZero = DAG.getSelect(dl, VT, isZero,
-                                    DAG.getConstant(1, VT), RHS);
+                                    DAG.getConstant(1, dl, VT), RHS);
     SDValue DIV = DAG.getNode(ISD::UDIV, dl, VT, MUL, NotZero);
     SDValue Overflow = DAG.getSetCC(dl, N->getValueType(1), DIV, LHS,
                                     ISD::SETNE);
     Overflow = DAG.getSelect(dl, N->getValueType(1), isZero,
-                             DAG.getConstant(0, N->getValueType(1)),
+                             DAG.getConstant(0, dl, N->getValueType(1)),
                              Overflow);
     ReplaceValueWith(SDValue(N, 1), Overflow);
     return;
   }
 
   Type *RetTy = VT.getTypeForEVT(*DAG.getContext());
-  EVT PtrVT = TLI.getPointerTy();
+  EVT PtrVT = TLI.getPointerTy(DAG.getDataLayout());
   Type *PtrTy = PtrVT.getTypeForEVT(*DAG.getContext());
 
   // Replace this with a libcall that will check overflow.
@@ -2345,15 +2439,15 @@ void DAGTypeLegalizer::ExpandIntRes_XMULO(SDNode *N,
   SDValue Temp = DAG.CreateStackTemporary(PtrVT);
   // Temporary for the overflow value, default it to zero.
   SDValue Chain = DAG.getStore(DAG.getEntryNode(), dl,
-                               DAG.getConstant(0, PtrVT), Temp,
+                               DAG.getConstant(0, dl, PtrVT), Temp,
                                MachinePointerInfo(), false, false, 0);
 
   TargetLowering::ArgListTy Args;
   TargetLowering::ArgListEntry Entry;
-  for (unsigned i = 0, e = N->getNumOperands(); i != e; ++i) {
-    EVT ArgVT = N->getOperand(i).getValueType();
+  for (const SDValue &Op : N->op_values()) {
+    EVT ArgVT = Op.getValueType();
     Type *ArgTy = ArgVT.getTypeForEVT(*DAG.getContext());
-    Entry.Node = N->getOperand(i);
+    Entry.Node = Op;
     Entry.Ty = ArgTy;
     Entry.isSExt = true;
     Entry.isZExt = false;
@@ -2380,7 +2474,7 @@ void DAGTypeLegalizer::ExpandIntRes_XMULO(SDNode *N,
   SDValue Temp2 = DAG.getLoad(PtrVT, dl, CallInfo.second, Temp,
                               MachinePointerInfo(), false, false, false, 0);
   SDValue Ofl = DAG.getSetCC(dl, N->getValueType(1), Temp2,
-                             DAG.getConstant(0, PtrVT),
+                             DAG.getConstant(0, dl, PtrVT),
                              ISD::SETNE);
   // Use the overflow from the libcall everywhere.
   ReplaceValueWith(SDValue(N, 1), Ofl);
@@ -2390,6 +2484,13 @@ void DAGTypeLegalizer::ExpandIntRes_UDIV(SDNode *N,
                                          SDValue &Lo, SDValue &Hi) {
   EVT VT = N->getValueType(0);
   SDLoc dl(N);
+  SDValue Ops[2] = { N->getOperand(0), N->getOperand(1) };
+
+  if (TLI.getOperationAction(ISD::UDIVREM, VT) == TargetLowering::Custom) {
+    SDValue Res = DAG.getNode(ISD::UDIVREM, dl, DAG.getVTList(VT, VT), Ops);
+    SplitInteger(Res.getValue(0), Lo, Hi);
+    return;
+  }
 
   RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
   if (VT == MVT::i16)
@@ -2402,7 +2503,6 @@ void DAGTypeLegalizer::ExpandIntRes_UDIV(SDNode *N,
     LC = RTLIB::UDIV_I128;
   assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported UDIV!");
 
-  SDValue Ops[2] = { N->getOperand(0), N->getOperand(1) };
   SplitInteger(TLI.makeLibCall(DAG, LC, VT, Ops, 2, false, dl).first, Lo, Hi);
 }
 
@@ -2410,6 +2510,13 @@ void DAGTypeLegalizer::ExpandIntRes_UREM(SDNode *N,
                                          SDValue &Lo, SDValue &Hi) {
   EVT VT = N->getValueType(0);
   SDLoc dl(N);
+  SDValue Ops[2] = { N->getOperand(0), N->getOperand(1) };
+
+  if (TLI.getOperationAction(ISD::UDIVREM, VT) == TargetLowering::Custom) {
+    SDValue Res = DAG.getNode(ISD::UDIVREM, dl, DAG.getVTList(VT, VT), Ops);
+    SplitInteger(Res.getValue(1), Lo, Hi);
+    return;
+  }
 
   RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
   if (VT == MVT::i16)
@@ -2422,7 +2529,6 @@ void DAGTypeLegalizer::ExpandIntRes_UREM(SDNode *N,
     LC = RTLIB::UREM_I128;
   assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported UREM!");
 
-  SDValue Ops[2] = { N->getOperand(0), N->getOperand(1) };
   SplitInteger(TLI.makeLibCall(DAG, LC, VT, Ops, 2, false, dl).first, Lo, Hi);
 }
 
@@ -2434,7 +2540,7 @@ void DAGTypeLegalizer::ExpandIntRes_ZERO_EXTEND(SDNode *N,
   if (Op.getValueType().bitsLE(NVT)) {
     // The low part is zero extension of the input (degenerates to a copy).
     Lo = DAG.getNode(ISD::ZERO_EXTEND, dl, NVT, N->getOperand(0));
-    Hi = DAG.getConstant(0, NVT);   // The high part is just a zero.
+    Hi = DAG.getConstant(0, dl, NVT);   // The high part is just a zero.
   } else {
     // For example, extension of an i48 to an i64.  The operand type necessarily
     // promotes to the result type, so will end up being expanded too.
@@ -2459,7 +2565,7 @@ void DAGTypeLegalizer::ExpandIntRes_ATOMIC_LOAD(SDNode *N,
   SDLoc dl(N);
   EVT VT = cast<AtomicSDNode>(N)->getMemoryVT();
   SDVTList VTs = DAG.getVTList(VT, MVT::i1, MVT::Other);
-  SDValue Zero = DAG.getConstant(0, VT);
+  SDValue Zero = DAG.getConstant(0, dl, VT);
   SDValue Swap = DAG.getAtomicCmpSwap(
       ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS, dl,
       cast<AtomicSDNode>(N)->getMemoryVT(), VTs, N->getOperand(0),
@@ -2560,7 +2666,7 @@ void DAGTypeLegalizer::IntegerExpandSetCCOperands(SDValue &NewLHS,
     NewLHS = DAG.getNode(ISD::XOR, dl, LHSLo.getValueType(), LHSLo, RHSLo);
     NewRHS = DAG.getNode(ISD::XOR, dl, LHSLo.getValueType(), LHSHi, RHSHi);
     NewLHS = DAG.getNode(ISD::OR, dl, NewLHS.getValueType(), NewLHS, NewRHS);
-    NewRHS = DAG.getConstant(0, NewLHS.getValueType());
+    NewRHS = DAG.getConstant(0, dl, NewLHS.getValueType());
     return;
   }
 
@@ -2649,7 +2755,7 @@ SDValue DAGTypeLegalizer::ExpandIntOp_BR_CC(SDNode *N) {
   // If ExpandSetCCOperands returned a scalar, we need to compare the result
   // against zero to select between true and false values.
   if (!NewRHS.getNode()) {
-    NewRHS = DAG.getConstant(0, NewLHS.getValueType());
+    NewRHS = DAG.getConstant(0, SDLoc(N), NewLHS.getValueType());
     CCCode = ISD::SETNE;
   }
 
@@ -2667,7 +2773,7 @@ SDValue DAGTypeLegalizer::ExpandIntOp_SELECT_CC(SDNode *N) {
   // If ExpandSetCCOperands returned a scalar, we need to compare the result
   // against zero to select between true and false values.
   if (!NewRHS.getNode()) {
-    NewRHS = DAG.getConstant(0, NewLHS.getValueType());
+    NewRHS = DAG.getConstant(0, SDLoc(N), NewLHS.getValueType());
     CCCode = ISD::SETNE;
   }
 
@@ -2748,7 +2854,7 @@ SDValue DAGTypeLegalizer::ExpandIntOp_STORE(StoreSDNode *N, unsigned OpNo) {
                              Alignment, AAInfo);
   }
 
-  if (TLI.isLittleEndian()) {
+  if (DAG.getDataLayout().isLittleEndian()) {
     // Little-endian - low bits are at low addresses.
     GetExpandedInteger(N->getValue(), Lo, Hi);
 
@@ -2762,7 +2868,7 @@ SDValue DAGTypeLegalizer::ExpandIntOp_STORE(StoreSDNode *N, unsigned OpNo) {
     // Increment the pointer to the other half.
     unsigned IncrementSize = NVT.getSizeInBits()/8;
     Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr,
-                      DAG.getConstant(IncrementSize, Ptr.getValueType()));
+                      DAG.getConstant(IncrementSize, dl, Ptr.getValueType()));
     Hi = DAG.getTruncStore(Ch, dl, Hi, Ptr,
                            N->getPointerInfo().getWithOffset(IncrementSize),
                            NEVT, isVolatile, isNonTemporal,
@@ -2784,12 +2890,13 @@ SDValue DAGTypeLegalizer::ExpandIntOp_STORE(StoreSDNode *N, unsigned OpNo) {
   if (ExcessBits < NVT.getSizeInBits()) {
     // Transfer high bits from the top of Lo to the bottom of Hi.
     Hi = DAG.getNode(ISD::SHL, dl, NVT, Hi,
-                     DAG.getConstant(NVT.getSizeInBits() - ExcessBits,
-                                     TLI.getPointerTy()));
-    Hi = DAG.getNode(ISD::OR, dl, NVT, Hi,
-                     DAG.getNode(ISD::SRL, dl, NVT, Lo,
-                                 DAG.getConstant(ExcessBits,
-                                                 TLI.getPointerTy())));
+                     DAG.getConstant(NVT.getSizeInBits() - ExcessBits, dl,
+                                     TLI.getPointerTy(DAG.getDataLayout())));
+    Hi = DAG.getNode(
+        ISD::OR, dl, NVT, Hi,
+        DAG.getNode(ISD::SRL, dl, NVT, Lo,
+                    DAG.getConstant(ExcessBits, dl,
+                                    TLI.getPointerTy(DAG.getDataLayout()))));
   }
 
   // Store both the high bits and maybe some of the low bits.
@@ -2798,7 +2905,7 @@ SDValue DAGTypeLegalizer::ExpandIntOp_STORE(StoreSDNode *N, unsigned OpNo) {
 
   // Increment the pointer to the other half.
   Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr,
-                    DAG.getConstant(IncrementSize, Ptr.getValueType()));
+                    DAG.getConstant(IncrementSize, dl, Ptr.getValueType()));
   // Store the lowest ExcessBits bits in the second half.
   Lo = DAG.getTruncStore(Ch, dl, Lo, Ptr,
                          N->getPointerInfo().getWithOffset(IncrementSize),
@@ -2854,18 +2961,20 @@ SDValue DAGTypeLegalizer::ExpandIntOp_UINT_TO_FP(SDNode *N) {
     GetExpandedInteger(Op, Lo, Hi);
     SDValue SignSet = DAG.getSetCC(dl,
                                    getSetCCResultType(Hi.getValueType()),
-                                   Hi, DAG.getConstant(0, Hi.getValueType()),
+                                   Hi,
+                                   DAG.getConstant(0, dl, Hi.getValueType()),
                                    ISD::SETLT);
 
     // Build a 64 bit pair (0, FF) in the constant pool, with FF in the lo bits.
-    SDValue FudgePtr = DAG.getConstantPool(
-                               ConstantInt::get(*DAG.getContext(), FF.zext(64)),
-                                           TLI.getPointerTy());
+    SDValue FudgePtr =
+        DAG.getConstantPool(ConstantInt::get(*DAG.getContext(), FF.zext(64)),
+                            TLI.getPointerTy(DAG.getDataLayout()));
 
     // Get a pointer to FF if the sign bit was set, or to 0 otherwise.
-    SDValue Zero = DAG.getIntPtrConstant(0);
-    SDValue Four = DAG.getIntPtrConstant(4);
-    if (TLI.isBigEndian()) std::swap(Zero, Four);
+    SDValue Zero = DAG.getIntPtrConstant(0, dl);
+    SDValue Four = DAG.getIntPtrConstant(4, dl);
+    if (DAG.getDataLayout().isBigEndian())
+      std::swap(Zero, Four);
     SDValue Offset = DAG.getSelect(dl, Zero.getValueType(), SignSet,
                                    Zero, Four);
     unsigned Alignment = cast<ConstantPoolSDNode>(FudgePtr)->getAlignment();
@@ -2922,7 +3031,7 @@ SDValue DAGTypeLegalizer::PromoteIntRes_EXTRACT_SUBVECTOR(SDNode *N) {
 
     // Extract the element from the original vector.
     SDValue Index = DAG.getNode(ISD::ADD, dl, BaseIdx.getValueType(),
-      BaseIdx, DAG.getConstant(i, BaseIdx.getValueType()));
+      BaseIdx, DAG.getConstant(i, dl, BaseIdx.getValueType()));
     SDValue Ext = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl,
       InVT.getVectorElementType(), N->getOperand(0), Index);
 
@@ -3015,9 +3124,9 @@ SDValue DAGTypeLegalizer::PromoteIntRes_CONCAT_VECTORS(SDNode *N) {
   for (unsigned i = 0; i < NumOperands; ++i) {
     SDValue Op = N->getOperand(i);
     for (unsigned j = 0; j < NumElem; ++j) {
-      SDValue Ext = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl,
-                                InElemTy, Op, DAG.getConstant(j,
-                                              TLI.getVectorIdxTy()));
+      SDValue Ext = DAG.getNode(
+          ISD::EXTRACT_VECTOR_ELT, dl, InElemTy, Op,
+          DAG.getConstant(j, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
       Ops[i * NumElem + j] = DAG.getNode(ISD::ANY_EXTEND, dl, OutElemTy, Ext);
     }
   }
@@ -3044,7 +3153,8 @@ SDValue DAGTypeLegalizer::PromoteIntRes_INSERT_VECTOR_ELT(SDNode *N) {
 SDValue DAGTypeLegalizer::PromoteIntOp_EXTRACT_VECTOR_ELT(SDNode *N) {
   SDLoc dl(N);
   SDValue V0 = GetPromotedInteger(N->getOperand(0));
-  SDValue V1 = DAG.getZExtOrTrunc(N->getOperand(1), dl, TLI.getVectorIdxTy());
+  SDValue V1 = DAG.getZExtOrTrunc(N->getOperand(1), dl,
+                                  TLI.getVectorIdxTy(DAG.getDataLayout()));
   SDValue Ext = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl,
     V0->getValueType(0).getScalarType(), V0, V1);
 
@@ -3054,6 +3164,16 @@ SDValue DAGTypeLegalizer::PromoteIntOp_EXTRACT_VECTOR_ELT(SDNode *N) {
   return DAG.getAnyExtOrTrunc(Ext, dl, N->getValueType(0));
 }
 
+SDValue DAGTypeLegalizer::PromoteIntOp_EXTRACT_SUBVECTOR(SDNode *N) {
+  SDLoc dl(N);
+  SDValue V0 = GetPromotedInteger(N->getOperand(0));
+  MVT InVT = V0.getValueType().getSimpleVT();
+  MVT OutVT = MVT::getVectorVT(InVT.getVectorElementType(),
+                               N->getValueType(0).getVectorNumElements());
+  SDValue Ext = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, OutVT, V0, N->getOperand(1));
+  return DAG.getNode(ISD::TRUNCATE, dl, N->getValueType(0), Ext);
+}
+
 SDValue DAGTypeLegalizer::PromoteIntOp_CONCAT_VECTORS(SDNode *N) {
   SDLoc dl(N);
   unsigned NumElems = N->getNumOperands();
@@ -3071,8 +3191,9 @@ SDValue DAGTypeLegalizer::PromoteIntOp_CONCAT_VECTORS(SDNode *N) {
 
     for (unsigned i=0; i<NumElem; ++i) {
       // Extract element from incoming vector
-      SDValue Ex = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, SclrTy,
-      Incoming, DAG.getConstant(i, TLI.getVectorIdxTy()));
+      SDValue Ex = DAG.getNode(
+          ISD::EXTRACT_VECTOR_ELT, dl, SclrTy, Incoming,
+          DAG.getConstant(i, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
       SDValue Tr = DAG.getNode(ISD::TRUNCATE, dl, RetSclrTy, Ex);
       NewOps.push_back(Tr);
     }
diff --git a/lib/CodeGen/SelectionDAG/LegalizeTypes.cpp b/lib/CodeGen/SelectionDAG/LegalizeTypes.cpp
index ebf6b28259e..a97842a9402 100644
--- a/lib/CodeGen/SelectionDAG/LegalizeTypes.cpp
+++ b/lib/CodeGen/SelectionDAG/LegalizeTypes.cpp
@@ -73,21 +73,20 @@ void DAGTypeLegalizer::PerformExpensiveChecks() {
   // (for example because it was created but not used).  In general, we cannot
   // distinguish between new nodes and deleted nodes.
   SmallVector<SDNode*, 16> NewNodes;
-  for (SelectionDAG::allnodes_iterator I = DAG.allnodes_begin(),
-       E = DAG.allnodes_end(); I != E; ++I) {
+  for (SDNode &Node : DAG.allnodes()) {
     // Remember nodes marked NewNode - they are subject to extra checking below.
-    if (I->getNodeId() == NewNode)
-      NewNodes.push_back(I);
+    if (Node.getNodeId() == NewNode)
+      NewNodes.push_back(&Node);
 
-    for (unsigned i = 0, e = I->getNumValues(); i != e; ++i) {
-      SDValue Res(I, i);
+    for (unsigned i = 0, e = Node.getNumValues(); i != e; ++i) {
+      SDValue Res(&Node, i);
       bool Failed = false;
 
       unsigned Mapped = 0;
       if (ReplacedValues.find(Res) != ReplacedValues.end()) {
         Mapped |= 1;
         // Check that remapped values are only used by nodes marked NewNode.
-        for (SDNode::use_iterator UI = I->use_begin(), UE = I->use_end();
+        for (SDNode::use_iterator UI = Node.use_begin(), UE = Node.use_end();
              UI != UE; ++UI)
           if (UI.getUse().getResNo() == i)
             assert(UI->getNodeId() == NewNode &&
@@ -119,16 +118,16 @@ void DAGTypeLegalizer::PerformExpensiveChecks() {
       if (WidenedVectors.find(Res) != WidenedVectors.end())
         Mapped |= 128;
 
-      if (I->getNodeId() != Processed) {
+      if (Node.getNodeId() != Processed) {
         // Since we allow ReplacedValues to map deleted nodes, it may map nodes
         // marked NewNode too, since a deleted node may have been reallocated as
         // another node that has not been seen by the LegalizeTypes machinery.
-        if ((I->getNodeId() == NewNode && Mapped > 1) ||
-            (I->getNodeId() != NewNode && Mapped != 0)) {
+        if ((Node.getNodeId() == NewNode && Mapped > 1) ||
+            (Node.getNodeId() != NewNode && Mapped != 0)) {
           dbgs() << "Unprocessed value in a map!";
           Failed = true;
         }
-      } else if (isTypeLegal(Res.getValueType()) || IgnoreNodeResults(I)) {
+      } else if (isTypeLegal(Res.getValueType()) || IgnoreNodeResults(&Node)) {
         if (Mapped > 1) {
           dbgs() << "Value with legal type was transformed!";
           Failed = true;
@@ -194,13 +193,12 @@ bool DAGTypeLegalizer::run() {
   // Walk all nodes in the graph, assigning them a NodeId of 'ReadyToProcess'
   // (and remembering them) if they are leaves and assigning 'Unanalyzed' if
   // non-leaves.
-  for (SelectionDAG::allnodes_iterator I = DAG.allnodes_begin(),
-       E = DAG.allnodes_end(); I != E; ++I) {
-    if (I->getNumOperands() == 0) {
-      I->setNodeId(ReadyToProcess);
-      Worklist.push_back(I);
+  for (SDNode &Node : DAG.allnodes()) {
+    if (Node.getNumOperands() == 0) {
+      Node.setNodeId(ReadyToProcess);
+      Worklist.push_back(&Node);
     } else {
-      I->setNodeId(Unanalyzed);
+      Node.setNodeId(Unanalyzed);
     }
   }
 
@@ -259,6 +257,10 @@ bool DAGTypeLegalizer::run() {
         WidenVectorResult(N, i);
         Changed = true;
         goto NodeDone;
+      case TargetLowering::TypePromoteFloat:
+        PromoteFloatResult(N, i);
+        Changed = true;
+        goto NodeDone;
       }
     }
 
@@ -308,6 +310,10 @@ bool DAGTypeLegalizer::run() {
         NeedsReanalyzing = WidenVectorOperand(N, i);
         Changed = true;
         break;
+      case TargetLowering::TypePromoteFloat:
+        NeedsReanalyzing = PromoteFloatOperand(N, i);
+        Changed = true;
+        break;
       }
       break;
     }
@@ -401,40 +407,39 @@ bool DAGTypeLegalizer::run() {
   // In a debug build, scan all the nodes to make sure we found them all.  This
   // ensures that there are no cycles and that everything got processed.
 #ifndef NDEBUG
-  for (SelectionDAG::allnodes_iterator I = DAG.allnodes_begin(),
-       E = DAG.allnodes_end(); I != E; ++I) {
+  for (SDNode &Node : DAG.allnodes()) {
     bool Failed = false;
 
     // Check that all result types are legal.
-    if (!IgnoreNodeResults(I))
-      for (unsigned i = 0, NumVals = I->getNumValues(); i < NumVals; ++i)
-        if (!isTypeLegal(I->getValueType(i))) {
+    if (!IgnoreNodeResults(&Node))
+      for (unsigned i = 0, NumVals = Node.getNumValues(); i < NumVals; ++i)
+        if (!isTypeLegal(Node.getValueType(i))) {
           dbgs() << "Result type " << i << " illegal!\n";
           Failed = true;
         }
 
     // Check that all operand types are legal.
-    for (unsigned i = 0, NumOps = I->getNumOperands(); i < NumOps; ++i)
-      if (!IgnoreNodeResults(I->getOperand(i).getNode()) &&
-          !isTypeLegal(I->getOperand(i).getValueType())) {
+    for (unsigned i = 0, NumOps = Node.getNumOperands(); i < NumOps; ++i)
+      if (!IgnoreNodeResults(Node.getOperand(i).getNode()) &&
+          !isTypeLegal(Node.getOperand(i).getValueType())) {
         dbgs() << "Operand type " << i << " illegal!\n";
         Failed = true;
       }
 
-    if (I->getNodeId() != Processed) {
-       if (I->getNodeId() == NewNode)
+    if (Node.getNodeId() != Processed) {
+       if (Node.getNodeId() == NewNode)
          dbgs() << "New node not analyzed?\n";
-       else if (I->getNodeId() == Unanalyzed)
+       else if (Node.getNodeId() == Unanalyzed)
          dbgs() << "Unanalyzed node not noticed?\n";
-       else if (I->getNodeId() > 0)
+       else if (Node.getNodeId() > 0)
          dbgs() << "Operand not processed?\n";
-       else if (I->getNodeId() == ReadyToProcess)
+       else if (Node.getNodeId() == ReadyToProcess)
          dbgs() << "Not added to worklist?\n";
        Failed = true;
     }
 
     if (Failed) {
-      I->dump(&DAG); dbgs() << "\n";
+      Node.dump(&DAG); dbgs() << "\n";
       llvm_unreachable(nullptr);
     }
   }
@@ -753,6 +758,17 @@ void DAGTypeLegalizer::SetSoftenedFloat(SDValue Op, SDValue Result) {
   OpEntry = Result;
 }
 
+void DAGTypeLegalizer::SetPromotedFloat(SDValue Op, SDValue Result) {
+  assert(Result.getValueType() ==
+         TLI.getTypeToTransformTo(*DAG.getContext(), Op.getValueType()) &&
+         "Invalid type for promoted float");
+  AnalyzeNewValue(Result);
+
+  SDValue &OpEntry = PromotedFloats[Op];
+  assert(!OpEntry.getNode() && "Node is already promoted!");
+  OpEntry = Result;
+}
+
 void DAGTypeLegalizer::SetScalarizedVector(SDValue Op, SDValue Result) {
   // Note that in some cases vector operation operands may be greater than
   // the vector element type. For example BUILD_VECTOR of type <1 x i1> with
@@ -978,22 +994,22 @@ void DAGTypeLegalizer::GetPairElements(SDValue Pair,
   SDLoc dl(Pair);
   EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), Pair.getValueType());
   Lo = DAG.getNode(ISD::EXTRACT_ELEMENT, dl, NVT, Pair,
-                   DAG.getIntPtrConstant(0));
+                   DAG.getIntPtrConstant(0, dl));
   Hi = DAG.getNode(ISD::EXTRACT_ELEMENT, dl, NVT, Pair,
-                   DAG.getIntPtrConstant(1));
+                   DAG.getIntPtrConstant(1, dl));
 }
 
 SDValue DAGTypeLegalizer::GetVectorElementPointer(SDValue VecPtr, EVT EltVT,
                                                   SDValue Index) {
   SDLoc dl(Index);
   // Make sure the index type is big enough to compute in.
-  Index = DAG.getZExtOrTrunc(Index, dl, TLI.getPointerTy());
+  Index = DAG.getZExtOrTrunc(Index, dl, TLI.getPointerTy(DAG.getDataLayout()));
 
   // Calculate the element offset and add it to the pointer.
   unsigned EltSize = EltVT.getSizeInBits() / 8; // FIXME: should be ABI size.
 
   Index = DAG.getNode(ISD::MUL, dl, Index.getValueType(), Index,
-                      DAG.getConstant(EltSize, Index.getValueType()));
+                      DAG.getConstant(EltSize, dl, Index.getValueType()));
   return DAG.getNode(ISD::ADD, dl, Index.getValueType(), Index, VecPtr);
 }
 
@@ -1010,7 +1026,8 @@ SDValue DAGTypeLegalizer::JoinIntegers(SDValue Lo, SDValue Hi) {
   Lo = DAG.getNode(ISD::ZERO_EXTEND, dlLo, NVT, Lo);
   Hi = DAG.getNode(ISD::ANY_EXTEND, dlHi, NVT, Hi);
   Hi = DAG.getNode(ISD::SHL, dlHi, NVT, Hi,
-                   DAG.getConstant(LVT.getSizeInBits(), TLI.getPointerTy()));
+                   DAG.getConstant(LVT.getSizeInBits(), dlHi,
+                                   TLI.getPointerTy(DAG.getDataLayout())));
   return DAG.getNode(ISD::OR, dlHi, NVT, Lo, Hi);
 }
 
@@ -1059,7 +1076,7 @@ DAGTypeLegalizer::ExpandChainLibCall(RTLIB::Libcall LC,
     Args.push_back(Entry);
   }
   SDValue Callee = DAG.getExternalSymbol(TLI.getLibcallName(LC),
-                                         TLI.getPointerTy());
+                                         TLI.getPointerTy(DAG.getDataLayout()));
 
   Type *RetTy = Node->getValueType(0).getTypeForEVT(*DAG.getContext());
 
@@ -1096,7 +1113,8 @@ void DAGTypeLegalizer::SplitInteger(SDValue Op,
          Op.getValueType().getSizeInBits() && "Invalid integer splitting!");
   Lo = DAG.getNode(ISD::TRUNCATE, dl, LoVT, Op);
   Hi = DAG.getNode(ISD::SRL, dl, Op.getValueType(), Op,
-                   DAG.getConstant(LoVT.getSizeInBits(), TLI.getPointerTy()));
+                   DAG.getConstant(LoVT.getSizeInBits(), dl,
+                                   TLI.getPointerTy(DAG.getDataLayout())));
   Hi = DAG.getNode(ISD::TRUNCATE, dl, HiVT, Hi);
 }
 
diff --git a/lib/CodeGen/SelectionDAG/LegalizeTypes.h b/lib/CodeGen/SelectionDAG/LegalizeTypes.h
index 9de85d7e777..d1131a74cf1 100644
--- a/lib/CodeGen/SelectionDAG/LegalizeTypes.h
+++ b/lib/CodeGen/SelectionDAG/LegalizeTypes.h
@@ -73,7 +73,7 @@ class LLVM_LIBRARY_VISIBILITY DAGTypeLegalizer {
   }
 
   EVT getSetCCResultType(EVT VT) const {
-    return TLI.getSetCCResultType(*DAG.getContext(), VT);
+    return TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT);
   }
 
   /// IgnoreNodeResults - Pretend all of this node's results are legal.
@@ -93,6 +93,11 @@ class LLVM_LIBRARY_VISIBILITY DAGTypeLegalizer {
   /// the same size, this map indicates the converted value to use.
   SmallDenseMap<SDValue, SDValue, 8> SoftenedFloats;
 
+  /// PromotedFloats - For floating point nodes that have a smaller precision
+  /// than the smallest supported precision, this map indicates what promoted
+  /// value to use.
+  SmallDenseMap<SDValue, SDValue, 8> PromotedFloats;
+
   /// ExpandedFloats - For float nodes that need to be expanded this map
   /// indicates which operands are the expanded version of the input.
   SmallDenseMap<SDValue, std::pair<SDValue, SDValue>, 8> ExpandedFloats;
@@ -162,7 +167,7 @@ class LLVM_LIBRARY_VISIBILITY DAGTypeLegalizer {
   SDValue GetVectorElementPointer(SDValue VecPtr, EVT EltVT, SDValue Index);
   SDValue JoinIntegers(SDValue Lo, SDValue Hi);
   SDValue LibCallify(RTLIB::Libcall LC, SDNode *N, bool isSigned);
-  
+
   std::pair<SDValue, SDValue> ExpandChainLibCall(RTLIB::Libcall LC,
                                                  SDNode *Node, bool isSigned);
   std::pair<SDValue, SDValue> ExpandAtomic(SDNode *Node);
@@ -273,6 +278,7 @@ class LLVM_LIBRARY_VISIBILITY DAGTypeLegalizer {
   SDValue PromoteIntOp_INSERT_VECTOR_ELT(SDNode *N, unsigned OpNo);
   SDValue PromoteIntOp_EXTRACT_ELEMENT(SDNode *N);
   SDValue PromoteIntOp_EXTRACT_VECTOR_ELT(SDNode *N);
+  SDValue PromoteIntOp_EXTRACT_SUBVECTOR(SDNode *N);
   SDValue PromoteIntOp_CONCAT_VECTORS(SDNode *N);
   SDValue PromoteIntOp_SCALAR_TO_VECTOR(SDNode *N);
   SDValue PromoteIntOp_SELECT(SDNode *N, unsigned OpNo);
@@ -341,7 +347,7 @@ class LLVM_LIBRARY_VISIBILITY DAGTypeLegalizer {
 
   void ExpandIntRes_ATOMIC_LOAD       (SDNode *N, SDValue &Lo, SDValue &Hi);
 
-  void ExpandShiftByConstant(SDNode *N, unsigned Amt,
+  void ExpandShiftByConstant(SDNode *N, const APInt &Amt,
                              SDValue &Lo, SDValue &Hi);
   bool ExpandShiftWithKnownAmountBit(SDNode *N, SDValue &Lo, SDValue &Hi);
   bool ExpandShiftWithUnknownAmountBit(SDNode *N, SDValue &Lo, SDValue &Hi);
@@ -499,6 +505,44 @@ class LLVM_LIBRARY_VISIBILITY DAGTypeLegalizer {
   void FloatExpandSetCCOperands(SDValue &NewLHS, SDValue &NewRHS,
                                 ISD::CondCode &CCCode, SDLoc dl);
 
+
+  //===--------------------------------------------------------------------===//
+  // Float promotion support: LegalizeFloatTypes.cpp
+  //===--------------------------------------------------------------------===//
+
+  SDValue GetPromotedFloat(SDValue Op) {
+    SDValue &PromotedOp = PromotedFloats[Op];
+    RemapValue(PromotedOp);
+    assert(PromotedOp.getNode() && "Operand wasn't promoted?");
+    return PromotedOp;
+  }
+  void SetPromotedFloat(SDValue Op, SDValue Result);
+
+  void PromoteFloatResult(SDNode *N, unsigned ResNo);
+  SDValue PromoteFloatRes_BITCAST(SDNode *N);
+  SDValue PromoteFloatRes_BinOp(SDNode *N);
+  SDValue PromoteFloatRes_ConstantFP(SDNode *N);
+  SDValue PromoteFloatRes_EXTRACT_VECTOR_ELT(SDNode *N);
+  SDValue PromoteFloatRes_FCOPYSIGN(SDNode *N);
+  SDValue PromoteFloatRes_FMAD(SDNode *N);
+  SDValue PromoteFloatRes_FPOWI(SDNode *N);
+  SDValue PromoteFloatRes_FP_ROUND(SDNode *N);
+  SDValue PromoteFloatRes_LOAD(SDNode *N);
+  SDValue PromoteFloatRes_SELECT(SDNode *N);
+  SDValue PromoteFloatRes_SELECT_CC(SDNode *N);
+  SDValue PromoteFloatRes_UnaryOp(SDNode *N);
+  SDValue PromoteFloatRes_UNDEF(SDNode *N);
+  SDValue PromoteFloatRes_XINT_TO_FP(SDNode *N);
+
+  bool PromoteFloatOperand(SDNode *N, unsigned ResNo);
+  SDValue PromoteFloatOp_BITCAST(SDNode *N, unsigned OpNo);
+  SDValue PromoteFloatOp_FCOPYSIGN(SDNode *N, unsigned OpNo);
+  SDValue PromoteFloatOp_FP_EXTEND(SDNode *N, unsigned OpNo);
+  SDValue PromoteFloatOp_FP_TO_XINT(SDNode *N, unsigned OpNo);
+  SDValue PromoteFloatOp_STORE(SDNode *N, unsigned OpNo);
+  SDValue PromoteFloatOp_SELECT_CC(SDNode *N, unsigned OpNo);
+  SDValue PromoteFloatOp_SETCC(SDNode *N, unsigned OpNo);
+
   //===--------------------------------------------------------------------===//
   // Scalarization Support: LegalizeVectorTypes.cpp
   //===--------------------------------------------------------------------===//
@@ -582,6 +626,7 @@ class LLVM_LIBRARY_VISIBILITY DAGTypeLegalizer {
   void SplitVecRes_INSERT_VECTOR_ELT(SDNode *N, SDValue &Lo, SDValue &Hi);
   void SplitVecRes_LOAD(LoadSDNode *N, SDValue &Lo, SDValue &Hi);
   void SplitVecRes_MLOAD(MaskedLoadSDNode *N, SDValue &Lo, SDValue &Hi);
+  void SplitVecRes_MGATHER(MaskedGatherSDNode *N, SDValue &Lo, SDValue &Hi);
   void SplitVecRes_SCALAR_TO_VECTOR(SDNode *N, SDValue &Lo, SDValue &Hi);
   void SplitVecRes_SIGN_EXTEND_INREG(SDNode *N, SDValue &Lo, SDValue &Hi);
   void SplitVecRes_SETCC(SDNode *N, SDValue &Lo, SDValue &Hi);
@@ -593,13 +638,15 @@ class LLVM_LIBRARY_VISIBILITY DAGTypeLegalizer {
   bool SplitVectorOperand(SDNode *N, unsigned OpNo);
   SDValue SplitVecOp_VSELECT(SDNode *N, unsigned OpNo);
   SDValue SplitVecOp_UnaryOp(SDNode *N);
-  SDValue SplitVecOp_TruncateHelper(SDNode *N, unsigned TruncateOp);
+  SDValue SplitVecOp_TruncateHelper(SDNode *N);
 
   SDValue SplitVecOp_BITCAST(SDNode *N);
   SDValue SplitVecOp_EXTRACT_SUBVECTOR(SDNode *N);
   SDValue SplitVecOp_EXTRACT_VECTOR_ELT(SDNode *N);
   SDValue SplitVecOp_STORE(StoreSDNode *N, unsigned OpNo);
   SDValue SplitVecOp_MSTORE(MaskedStoreSDNode *N, unsigned OpNo);
+  SDValue SplitVecOp_MSCATTER(MaskedScatterSDNode *N, unsigned OpNo);
+  SDValue SplitVecOp_MGATHER(MaskedGatherSDNode *N, unsigned OpNo);
   SDValue SplitVecOp_CONCAT_VECTORS(SDNode *N);
   SDValue SplitVecOp_VSETCC(SDNode *N);
   SDValue SplitVecOp_FP_ROUND(SDNode *N);
diff --git a/lib/CodeGen/SelectionDAG/LegalizeTypesGeneric.cpp b/lib/CodeGen/SelectionDAG/LegalizeTypesGeneric.cpp
index 38829b6ac07..14d8f776208 100644
--- a/lib/CodeGen/SelectionDAG/LegalizeTypesGeneric.cpp
+++ b/lib/CodeGen/SelectionDAG/LegalizeTypesGeneric.cpp
@@ -50,6 +50,9 @@ void DAGTypeLegalizer::ExpandRes_BITCAST(SDNode *N, SDValue &Lo, SDValue &Hi) {
     case TargetLowering::TypeLegal:
     case TargetLowering::TypePromoteInteger:
       break;
+    case TargetLowering::TypePromoteFloat:
+      llvm_unreachable("Bitcast of a promotion-needing float should never need"
+                       "expansion");
     case TargetLowering::TypeSoftenFloat:
       // Convert the integer operand instead.
       SplitInteger(GetSoftenedFloat(InOp), Lo, Hi);
@@ -57,18 +60,20 @@ void DAGTypeLegalizer::ExpandRes_BITCAST(SDNode *N, SDValue &Lo, SDValue &Hi) {
       Hi = DAG.getNode(ISD::BITCAST, dl, NOutVT, Hi);
       return;
     case TargetLowering::TypeExpandInteger:
-    case TargetLowering::TypeExpandFloat:
+    case TargetLowering::TypeExpandFloat: {
+      auto &DL = DAG.getDataLayout();
       // Convert the expanded pieces of the input.
       GetExpandedOp(InOp, Lo, Hi);
-      if (TLI.hasBigEndianPartOrdering(InVT) !=
-          TLI.hasBigEndianPartOrdering(OutVT))
+      if (TLI.hasBigEndianPartOrdering(InVT, DL) !=
+          TLI.hasBigEndianPartOrdering(OutVT, DL))
         std::swap(Lo, Hi);
       Lo = DAG.getNode(ISD::BITCAST, dl, NOutVT, Lo);
       Hi = DAG.getNode(ISD::BITCAST, dl, NOutVT, Hi);
       return;
+    }
     case TargetLowering::TypeSplitVector:
       GetSplitVector(InOp, Lo, Hi);
-      if (TLI.hasBigEndianPartOrdering(OutVT))
+      if (TLI.hasBigEndianPartOrdering(OutVT, DAG.getDataLayout()))
         std::swap(Lo, Hi);
       Lo = DAG.getNode(ISD::BITCAST, dl, NOutVT, Lo);
       Hi = DAG.getNode(ISD::BITCAST, dl, NOutVT, Hi);
@@ -85,7 +90,7 @@ void DAGTypeLegalizer::ExpandRes_BITCAST(SDNode *N, SDValue &Lo, SDValue &Hi) {
       EVT LoVT, HiVT;
       std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(InVT);
       std::tie(Lo, Hi) = DAG.SplitVector(InOp, dl, LoVT, HiVT);
-      if (TLI.hasBigEndianPartOrdering(OutVT))
+      if (TLI.hasBigEndianPartOrdering(OutVT, DAG.getDataLayout()))
         std::swap(Lo, Hi);
       Lo = DAG.getNode(ISD::BITCAST, dl, NOutVT, Lo);
       Hi = DAG.getNode(ISD::BITCAST, dl, NOutVT, Hi);
@@ -116,9 +121,9 @@ void DAGTypeLegalizer::ExpandRes_BITCAST(SDNode *N, SDValue &Lo, SDValue &Hi) {
 
       SmallVector<SDValue, 8> Vals;
       for (unsigned i = 0; i < NumElems; ++i)
-        Vals.push_back(DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, ElemVT,
-                                   CastInOp, DAG.getConstant(i,
-                                             TLI.getVectorIdxTy())));
+        Vals.push_back(DAG.getNode(
+            ISD::EXTRACT_VECTOR_ELT, dl, ElemVT, CastInOp,
+            DAG.getConstant(i, dl, TLI.getVectorIdxTy(DAG.getDataLayout()))));
 
       // Build Lo, Hi pair by pairing extracted elements if needed.
       unsigned Slot = 0;
@@ -128,7 +133,7 @@ void DAGTypeLegalizer::ExpandRes_BITCAST(SDNode *N, SDValue &Lo, SDValue &Hi) {
         SDValue LHS = Vals[Slot];
         SDValue RHS = Vals[Slot + 1];
 
-        if (TLI.isBigEndian())
+        if (DAG.getDataLayout().isBigEndian())
           std::swap(LHS, RHS);
 
         Vals.push_back(DAG.getNode(ISD::BUILD_PAIR, dl,
@@ -140,7 +145,7 @@ void DAGTypeLegalizer::ExpandRes_BITCAST(SDNode *N, SDValue &Lo, SDValue &Hi) {
       Lo = Vals[Slot++];
       Hi = Vals[Slot++];
 
-      if (TLI.isBigEndian())
+      if (DAG.getDataLayout().isBigEndian())
         std::swap(Lo, Hi);
 
       return;
@@ -152,9 +157,8 @@ void DAGTypeLegalizer::ExpandRes_BITCAST(SDNode *N, SDValue &Lo, SDValue &Hi) {
 
   // Create the stack frame object.  Make sure it is aligned for both
   // the source and expanded destination types.
-  unsigned Alignment =
-    TLI.getDataLayout()->getPrefTypeAlignment(NOutVT.
-                                              getTypeForEVT(*DAG.getContext()));
+  unsigned Alignment = DAG.getDataLayout().getPrefTypeAlignment(
+      NOutVT.getTypeForEVT(*DAG.getContext()));
   SDValue StackPtr = DAG.CreateStackTemporary(InVT, Alignment);
   int SPFI = cast<FrameIndexSDNode>(StackPtr.getNode())->getIndex();
   MachinePointerInfo PtrInfo = MachinePointerInfo::getFixedStack(SPFI);
@@ -170,7 +174,7 @@ void DAGTypeLegalizer::ExpandRes_BITCAST(SDNode *N, SDValue &Lo, SDValue &Hi) {
   // Increment the pointer to the other half.
   unsigned IncrementSize = NOutVT.getSizeInBits() / 8;
   StackPtr = DAG.getNode(ISD::ADD, dl, StackPtr.getValueType(), StackPtr,
-                         DAG.getConstant(IncrementSize,
+                         DAG.getConstant(IncrementSize, dl,
                                          StackPtr.getValueType()));
 
   // Load the second half from the stack slot.
@@ -179,7 +183,7 @@ void DAGTypeLegalizer::ExpandRes_BITCAST(SDNode *N, SDValue &Lo, SDValue &Hi) {
                    false, false, MinAlign(Alignment, IncrementSize));
 
   // Handle endianness of the load.
-  if (TLI.hasBigEndianPartOrdering(OutVT))
+  if (TLI.hasBigEndianPartOrdering(OutVT, DAG.getDataLayout()))
     std::swap(Lo, Hi);
 }
 
@@ -235,10 +239,10 @@ void DAGTypeLegalizer::ExpandRes_EXTRACT_VECTOR_ELT(SDNode *N, SDValue &Lo,
   Lo = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, NewVT, NewVec, Idx);
 
   Idx = DAG.getNode(ISD::ADD, dl, Idx.getValueType(), Idx,
-                    DAG.getConstant(1, Idx.getValueType()));
+                    DAG.getConstant(1, dl, Idx.getValueType()));
   Hi = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, NewVT, NewVec, Idx);
 
-  if (TLI.isBigEndian())
+  if (DAG.getDataLayout().isBigEndian())
     std::swap(Lo, Hi);
 }
 
@@ -267,7 +271,7 @@ void DAGTypeLegalizer::ExpandRes_NormalLoad(SDNode *N, SDValue &Lo,
   // Increment the pointer to the other half.
   unsigned IncrementSize = NVT.getSizeInBits() / 8;
   Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr,
-                    DAG.getConstant(IncrementSize, Ptr.getValueType()));
+                    DAG.getConstant(IncrementSize, dl, Ptr.getValueType()));
   Hi = DAG.getLoad(NVT, dl, Chain, Ptr,
                    LD->getPointerInfo().getWithOffset(IncrementSize),
                    isVolatile, isNonTemporal, isInvariant,
@@ -279,7 +283,7 @@ void DAGTypeLegalizer::ExpandRes_NormalLoad(SDNode *N, SDValue &Lo,
                       Hi.getValue(1));
 
   // Handle endianness of the load.
-  if (TLI.hasBigEndianPartOrdering(ValueVT))
+  if (TLI.hasBigEndianPartOrdering(ValueVT, DAG.getDataLayout()))
     std::swap(Lo, Hi);
 
   // Modified the chain - switch anything that used the old chain to use
@@ -299,7 +303,7 @@ void DAGTypeLegalizer::ExpandRes_VAARG(SDNode *N, SDValue &Lo, SDValue &Hi) {
   Hi = DAG.getVAArg(NVT, dl, Lo.getValue(1), Ptr, N->getOperand(2), 0);
 
   // Handle endianness of the load.
-  if (TLI.hasBigEndianPartOrdering(OVT))
+  if (TLI.hasBigEndianPartOrdering(OVT, DAG.getDataLayout()))
     std::swap(Lo, Hi);
 
   // Modified the chain - switch anything that used the old chain to use
@@ -322,7 +326,7 @@ void DAGTypeLegalizer::IntegerToVector(SDValue Op, unsigned NumElements,
   if (NumElements > 1) {
     NumElements >>= 1;
     SplitInteger(Op, Parts[0], Parts[1]);
-      if (TLI.isBigEndian())
+    if (DAG.getDataLayout().isBigEndian())
         std::swap(Parts[0], Parts[1]);
     IntegerToVector(Parts[0], NumElements, Ops, EltVT);
     IntegerToVector(Parts[1], NumElements, Ops, EltVT);
@@ -386,7 +390,7 @@ SDValue DAGTypeLegalizer::ExpandOp_BUILD_VECTOR(SDNode *N) {
   for (unsigned i = 0; i < NumElts; ++i) {
     SDValue Lo, Hi;
     GetExpandedOp(N->getOperand(i), Lo, Hi);
-    if (TLI.isBigEndian())
+    if (DAG.getDataLayout().isBigEndian())
       std::swap(Lo, Hi);
     NewElts.push_back(Lo);
     NewElts.push_back(Hi);
@@ -428,7 +432,7 @@ SDValue DAGTypeLegalizer::ExpandOp_INSERT_VECTOR_ELT(SDNode *N) {
 
   SDValue Lo, Hi;
   GetExpandedOp(Val, Lo, Hi);
-  if (TLI.isBigEndian())
+  if (DAG.getDataLayout().isBigEndian())
     std::swap(Lo, Hi);
 
   SDValue Idx = N->getOperand(2);
@@ -436,7 +440,7 @@ SDValue DAGTypeLegalizer::ExpandOp_INSERT_VECTOR_ELT(SDNode *N) {
   NewVec = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, NewVecVT, NewVec, Lo, Idx);
   Idx = DAG.getNode(ISD::ADD, dl,
                     Idx.getValueType(), Idx,
-                    DAG.getConstant(1, Idx.getValueType()));
+                    DAG.getConstant(1, dl, Idx.getValueType()));
   NewVec =  DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, NewVecVT, NewVec, Hi, Idx);
 
   // Convert the new vector to the old vector type.
@@ -478,14 +482,14 @@ SDValue DAGTypeLegalizer::ExpandOp_NormalStore(SDNode *N, unsigned OpNo) {
   SDValue Lo, Hi;
   GetExpandedOp(St->getValue(), Lo, Hi);
 
-  if (TLI.hasBigEndianPartOrdering(ValueVT))
+  if (TLI.hasBigEndianPartOrdering(ValueVT, DAG.getDataLayout()))
     std::swap(Lo, Hi);
 
   Lo = DAG.getStore(Chain, dl, Lo, Ptr, St->getPointerInfo(),
                     isVolatile, isNonTemporal, Alignment, AAInfo);
 
   Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr,
-                    DAG.getConstant(IncrementSize, Ptr.getValueType()));
+                    DAG.getConstant(IncrementSize, dl, Ptr.getValueType()));
   Hi = DAG.getStore(Chain, dl, Hi, Ptr,
                     St->getPointerInfo().getWithOffset(IncrementSize),
                     isVolatile, isNonTemporal,
diff --git a/lib/CodeGen/SelectionDAG/LegalizeVectorOps.cpp b/lib/CodeGen/SelectionDAG/LegalizeVectorOps.cpp
index 408d5eda4e7..0f25a610724 100644
--- a/lib/CodeGen/SelectionDAG/LegalizeVectorOps.cpp
+++ b/lib/CodeGen/SelectionDAG/LegalizeVectorOps.cpp
@@ -105,6 +105,7 @@ class VectorLegalizer {
   SDValue ExpandLoad(SDValue Op);
   SDValue ExpandStore(SDValue Op);
   SDValue ExpandFNEG(SDValue Op);
+  SDValue ExpandABSDIFF(SDValue Op);
 
   /// \brief Implements vector promotion.
   ///
@@ -191,11 +192,12 @@ SDValue VectorLegalizer::LegalizeOp(SDValue Op) {
 
   // Legalize the operands
   SmallVector<SDValue, 8> Ops;
-  for (unsigned i = 0, e = Node->getNumOperands(); i != e; ++i)
-    Ops.push_back(LegalizeOp(Node->getOperand(i)));
+  for (const SDValue &Op : Node->op_values())
+    Ops.push_back(LegalizeOp(Op));
 
   SDValue Result = SDValue(DAG.UpdateNodeOperands(Op.getNode(), Ops), 0);
 
+  bool HasVectorValue = false;
   if (Op.getOpcode() == ISD::LOAD) {
     LoadSDNode *LD = cast<LoadSDNode>(Op.getNode());
     ISD::LoadExtType ExtType = LD->getExtensionType();
@@ -243,9 +245,9 @@ SDValue VectorLegalizer::LegalizeOp(SDValue Op) {
         Changed = true;
         return LegalizeOp(ExpandStore(Op));
       }
-  }
+  } else if (Op.getOpcode() == ISD::MSCATTER)
+    HasVectorValue = true;
 
-  bool HasVectorValue = false;
   for (SDNode::value_iterator J = Node->value_begin(), E = Node->value_end();
        J != E;
        ++J)
@@ -321,6 +323,12 @@ SDValue VectorLegalizer::LegalizeOp(SDValue Op) {
   case ISD::ANY_EXTEND_VECTOR_INREG:
   case ISD::SIGN_EXTEND_VECTOR_INREG:
   case ISD::ZERO_EXTEND_VECTOR_INREG:
+  case ISD::SMIN:
+  case ISD::SMAX:
+  case ISD::UMIN:
+  case ISD::UMAX:
+  case ISD::UABSDIFF:
+  case ISD::SABSDIFF:
     QueryType = Node->getValueType(0);
     break;
   case ISD::FP_ROUND_INREG:
@@ -330,6 +338,9 @@ SDValue VectorLegalizer::LegalizeOp(SDValue Op) {
   case ISD::UINT_TO_FP:
     QueryType = Node->getOperand(0).getValueType();
     break;
+  case ISD::MSCATTER:
+    QueryType = cast<MaskedScatterSDNode>(Node)->getValue().getValueType();
+    break;
   }
 
   switch (TLI.getOperationAction(Node->getOpcode(), QueryType)) {
@@ -407,7 +418,7 @@ SDValue VectorLegalizer::Promote(SDValue Op) {
   if ((VT.isFloatingPoint() && NVT.isFloatingPoint()) ||
       (VT.isVector() && VT.getVectorElementType().isFloatingPoint() &&
        NVT.isVector() && NVT.getVectorElementType().isFloatingPoint()))
-    return DAG.getNode(ISD::FP_ROUND, dl, VT, Op, DAG.getIntPtrConstant(0));
+    return DAG.getNode(ISD::FP_ROUND, dl, VT, Op, DAG.getIntPtrConstant(0, dl));
   else
     return DAG.getNode(ISD::BITCAST, dl, VT, Op);
 }
@@ -495,7 +506,7 @@ SDValue VectorLegalizer::ExpandLoad(SDValue Op) {
     // Instead, we load all significant words, mask bits off, and concatenate
     // them to form each element. Finally, they are extended to destination
     // scalar type to build the destination vector.
-    EVT WideVT = TLI.getPointerTy();
+    EVT WideVT = TLI.getPointerTy(DAG.getDataLayout());
 
     assert(WideVT.isRound() &&
            "Could not handle the sophisticated case when the widest integer is"
@@ -536,7 +547,8 @@ SDValue VectorLegalizer::ExpandLoad(SDValue Op) {
       RemainingBytes -= LoadBytes;
       Offset += LoadBytes;
       BasePTR = DAG.getNode(ISD::ADD, dl, BasePTR.getValueType(), BasePTR,
-                            DAG.getConstant(LoadBytes, BasePTR.getValueType()));
+                            DAG.getConstant(LoadBytes, dl,
+                                            BasePTR.getValueType()));
 
       LoadVals.push_back(ScalarLoad.getValue(0));
       LoadChains.push_back(ScalarLoad.getValue(1));
@@ -544,7 +556,7 @@ SDValue VectorLegalizer::ExpandLoad(SDValue Op) {
 
     // Extract bits, pack and extend/trunc them into destination type.
     unsigned SrcEltBits = SrcEltVT.getSizeInBits();
-    SDValue SrcEltBitMask = DAG.getConstant((1U << SrcEltBits) - 1, WideVT);
+    SDValue SrcEltBitMask = DAG.getConstant((1U << SrcEltBits) - 1, dl, WideVT);
 
     unsigned BitOffset = 0;
     unsigned WideIdx = 0;
@@ -554,7 +566,8 @@ SDValue VectorLegalizer::ExpandLoad(SDValue Op) {
       SDValue Lo, Hi, ShAmt;
 
       if (BitOffset < WideBits) {
-        ShAmt = DAG.getConstant(BitOffset, TLI.getShiftAmountTy(WideVT));
+        ShAmt = DAG.getConstant(
+            BitOffset, dl, TLI.getShiftAmountTy(WideVT, DAG.getDataLayout()));
         Lo = DAG.getNode(ISD::SRL, dl, WideVT, LoadVals[WideIdx], ShAmt);
         Lo = DAG.getNode(ISD::AND, dl, WideVT, Lo, SrcEltBitMask);
       }
@@ -564,8 +577,9 @@ SDValue VectorLegalizer::ExpandLoad(SDValue Op) {
         WideIdx++;
         BitOffset -= WideBits;
         if (BitOffset > 0) {
-          ShAmt = DAG.getConstant(SrcEltBits - BitOffset,
-                                  TLI.getShiftAmountTy(WideVT));
+          ShAmt = DAG.getConstant(
+              SrcEltBits - BitOffset, dl,
+              TLI.getShiftAmountTy(WideVT, DAG.getDataLayout()));
           Hi = DAG.getNode(ISD::SHL, dl, WideVT, LoadVals[WideIdx], ShAmt);
           Hi = DAG.getNode(ISD::AND, dl, WideVT, Hi, SrcEltBitMask);
         }
@@ -583,8 +597,9 @@ SDValue VectorLegalizer::ExpandLoad(SDValue Op) {
         Lo = DAG.getZExtOrTrunc(Lo, dl, DstEltVT);
         break;
       case ISD::SEXTLOAD:
-        ShAmt = DAG.getConstant(WideBits - SrcEltBits,
-                                TLI.getShiftAmountTy(WideVT));
+        ShAmt =
+            DAG.getConstant(WideBits - SrcEltBits, dl,
+                            TLI.getShiftAmountTy(WideVT, DAG.getDataLayout()));
         Lo = DAG.getNode(ISD::SHL, dl, WideVT, Lo, ShAmt);
         Lo = DAG.getNode(ISD::SRA, dl, WideVT, Lo, ShAmt);
         Lo = DAG.getSExtOrTrunc(Lo, dl, DstEltVT);
@@ -604,7 +619,7 @@ SDValue VectorLegalizer::ExpandLoad(SDValue Op) {
                 MinAlign(LD->getAlignment(), Idx * Stride), LD->getAAInfo());
 
       BasePTR = DAG.getNode(ISD::ADD, dl, BasePTR.getValueType(), BasePTR,
-                         DAG.getConstant(Stride, BasePTR.getValueType()));
+                         DAG.getConstant(Stride, dl, BasePTR.getValueType()));
 
       Vals.push_back(ScalarLoad.getValue(0));
       LoadChains.push_back(ScalarLoad.getValue(1));
@@ -654,8 +669,9 @@ SDValue VectorLegalizer::ExpandStore(SDValue Op) {
   // and save them into memory individually.
   SmallVector<SDValue, 8> Stores;
   for (unsigned Idx = 0; Idx < NumElem; Idx++) {
-    SDValue Ex = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl,
-               RegSclVT, Value, DAG.getConstant(Idx, TLI.getVectorIdxTy()));
+    SDValue Ex = DAG.getNode(
+        ISD::EXTRACT_VECTOR_ELT, dl, RegSclVT, Value,
+        DAG.getConstant(Idx, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
 
     // This scalar TruncStore may be illegal, but we legalize it later.
     SDValue Store = DAG.getTruncStore(Chain, dl, Ex, BasePTR,
@@ -664,7 +680,7 @@ SDValue VectorLegalizer::ExpandStore(SDValue Op) {
                AAInfo);
 
     BasePTR = DAG.getNode(ISD::ADD, dl, BasePTR.getValueType(), BasePTR,
-                               DAG.getConstant(Stride, BasePTR.getValueType()));
+                          DAG.getConstant(Stride, dl, BasePTR.getValueType()));
 
     Stores.push_back(Store);
   }
@@ -695,11 +711,36 @@ SDValue VectorLegalizer::Expand(SDValue Op) {
     return ExpandFNEG(Op);
   case ISD::SETCC:
     return UnrollVSETCC(Op);
+  case ISD::UABSDIFF:
+  case ISD::SABSDIFF:
+    return ExpandABSDIFF(Op);
   default:
     return DAG.UnrollVectorOp(Op.getNode());
   }
 }
 
+SDValue VectorLegalizer::ExpandABSDIFF(SDValue Op) {
+  SDLoc dl(Op);
+  SDValue Tmp1, Tmp2, Tmp3, Tmp4;
+  EVT VT = Op.getValueType();
+  SDNodeFlags Flags;
+  Flags.setNoSignedWrap(Op->getOpcode() == ISD::SABSDIFF);
+
+  Tmp2 = Op.getOperand(0);
+  Tmp3 = Op.getOperand(1);
+  Tmp1 = DAG.getNode(ISD::SUB, dl, VT, Tmp2, Tmp3, &Flags);
+  Tmp2 =
+      DAG.getNode(ISD::SUB, dl, VT, DAG.getConstant(0, dl, VT), Tmp1, &Flags);
+  Tmp4 = DAG.getNode(
+      ISD::SETCC, dl,
+      TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT), Tmp2,
+      DAG.getConstant(0, dl, VT),
+      DAG.getCondCode(Op->getOpcode() == ISD::SABSDIFF ? ISD::SETLT
+                                                       : ISD::SETULT));
+  Tmp1 = DAG.getNode(ISD::VSELECT, dl, VT, Tmp4, Tmp1, Tmp2);
+  return Tmp1;
+}
+
 SDValue VectorLegalizer::ExpandSELECT(SDValue Op) {
   // Lower a select instruction where the condition is a scalar and the
   // operands are vectors. Lower this select to VSELECT and implement it
@@ -734,8 +775,9 @@ SDValue VectorLegalizer::ExpandSELECT(SDValue Op) {
   EVT BitTy = MaskTy.getScalarType();
 
   Mask = DAG.getSelect(DL, BitTy, Mask,
-          DAG.getConstant(APInt::getAllOnesValue(BitTy.getSizeInBits()), BitTy),
-          DAG.getConstant(0, BitTy));
+          DAG.getConstant(APInt::getAllOnesValue(BitTy.getSizeInBits()), DL,
+                          BitTy),
+          DAG.getConstant(0, DL, BitTy));
 
   // Broadcast the mask so that the entire vector is all-one or all zero.
   SmallVector<SDValue, 8> Ops(NumElem, Mask);
@@ -748,7 +790,7 @@ SDValue VectorLegalizer::ExpandSELECT(SDValue Op) {
   Op2 = DAG.getNode(ISD::BITCAST, DL, MaskTy, Op2);
 
   SDValue AllOnes = DAG.getConstant(
-            APInt::getAllOnesValue(BitTy.getSizeInBits()), MaskTy);
+            APInt::getAllOnesValue(BitTy.getSizeInBits()), DL, MaskTy);
   SDValue NotMask = DAG.getNode(ISD::XOR, DL, MaskTy, Mask, AllOnes);
 
   Op1 = DAG.getNode(ISD::AND, DL, MaskTy, Op1, Mask);
@@ -770,7 +812,7 @@ SDValue VectorLegalizer::ExpandSEXTINREG(SDValue Op) {
 
   unsigned BW = VT.getScalarType().getSizeInBits();
   unsigned OrigBW = OrigTy.getScalarType().getSizeInBits();
-  SDValue ShiftSz = DAG.getConstant(BW - OrigBW, VT);
+  SDValue ShiftSz = DAG.getConstant(BW - OrigBW, DL, VT);
 
   Op = Op.getOperand(0);
   Op =   DAG.getNode(ISD::SHL, DL, VT, Op, ShiftSz);
@@ -793,7 +835,7 @@ SDValue VectorLegalizer::ExpandANY_EXTEND_VECTOR_INREG(SDValue Op) {
 
   // Place the extended lanes into the correct locations.
   int ExtLaneScale = NumSrcElements / NumElements;
-  int EndianOffset = TLI.isBigEndian() ? ExtLaneScale - 1 : 0;
+  int EndianOffset = DAG.getDataLayout().isBigEndian() ? ExtLaneScale - 1 : 0;
   for (int i = 0; i < NumElements; ++i)
     ShuffleMask[i * ExtLaneScale + EndianOffset] = i;
 
@@ -817,7 +859,7 @@ SDValue VectorLegalizer::ExpandSIGN_EXTEND_VECTOR_INREG(SDValue Op) {
   // without full scalarization than the sign extension does.
   unsigned EltWidth = VT.getVectorElementType().getSizeInBits();
   unsigned SrcEltWidth = SrcVT.getVectorElementType().getSizeInBits();
-  SDValue ShiftAmount = DAG.getConstant(EltWidth - SrcEltWidth, VT);
+  SDValue ShiftAmount = DAG.getConstant(EltWidth - SrcEltWidth, DL, VT);
   return DAG.getNode(ISD::SRA, DL, VT,
                      DAG.getNode(ISD::SHL, DL, VT, Op, ShiftAmount),
                      ShiftAmount);
@@ -836,7 +878,7 @@ SDValue VectorLegalizer::ExpandZERO_EXTEND_VECTOR_INREG(SDValue Op) {
 
   // Build up a zero vector to blend into this one.
   EVT SrcScalarVT = SrcVT.getScalarType();
-  SDValue ScalarZero = DAG.getTargetConstant(0, SrcScalarVT);
+  SDValue ScalarZero = DAG.getTargetConstant(0, DL, SrcScalarVT);
   SmallVector<SDValue, 4> BuildVectorOperands(NumSrcElements, ScalarZero);
   SDValue Zero = DAG.getNode(ISD::BUILD_VECTOR, DL, SrcVT, BuildVectorOperands);
 
@@ -848,7 +890,7 @@ SDValue VectorLegalizer::ExpandZERO_EXTEND_VECTOR_INREG(SDValue Op) {
     ShuffleMask.push_back(i);
 
   int ExtLaneScale = NumSrcElements / NumElements;
-  int EndianOffset = TLI.isBigEndian() ? ExtLaneScale - 1 : 0;
+  int EndianOffset = DAG.getDataLayout().isBigEndian() ? ExtLaneScale - 1 : 0;
   for (int i = 0; i < NumElements; ++i)
     ShuffleMask[i * ExtLaneScale + EndianOffset] = NumSrcElements + i;
 
@@ -917,7 +959,7 @@ SDValue VectorLegalizer::ExpandVSELECT(SDValue Op) {
   Op2 = DAG.getNode(ISD::BITCAST, DL, VT, Op2);
 
   SDValue AllOnes = DAG.getConstant(
-    APInt::getAllOnesValue(VT.getScalarType().getSizeInBits()), VT);
+    APInt::getAllOnesValue(VT.getScalarType().getSizeInBits()), DL, VT);
   SDValue NotMask = DAG.getNode(ISD::XOR, DL, VT, Mask, AllOnes);
 
   Op1 = DAG.getNode(ISD::AND, DL, VT, Op1, Mask);
@@ -940,16 +982,16 @@ SDValue VectorLegalizer::ExpandUINT_TO_FLOAT(SDValue Op) {
       "Elements in vector-UINT_TO_FP must be 32 or 64 bits wide");
 
   unsigned BW = SVT.getSizeInBits();
-  SDValue HalfWord = DAG.getConstant(BW/2, VT);
+  SDValue HalfWord = DAG.getConstant(BW/2, DL, VT);
 
   // Constants to clear the upper part of the word.
   // Notice that we can also use SHL+SHR, but using a constant is slightly
   // faster on x86.
   uint64_t HWMask = (SVT.getSizeInBits()==64)?0x00000000FFFFFFFF:0x0000FFFF;
-  SDValue HalfWordMask = DAG.getConstant(HWMask, VT);
+  SDValue HalfWordMask = DAG.getConstant(HWMask, DL, VT);
 
   // Two to the power of half-word-size.
-  SDValue TWOHW = DAG.getConstantFP((1<<(BW/2)), Op.getValueType());
+  SDValue TWOHW = DAG.getConstantFP(1 << (BW/2), DL, Op.getValueType());
 
   // Clear upper part of LO, lower HI
   SDValue HI = DAG.getNode(ISD::SRL, DL, VT, Op.getOperand(0), HalfWord);
@@ -968,8 +1010,9 @@ SDValue VectorLegalizer::ExpandUINT_TO_FLOAT(SDValue Op) {
 
 SDValue VectorLegalizer::ExpandFNEG(SDValue Op) {
   if (TLI.isOperationLegalOrCustom(ISD::FSUB, Op.getValueType())) {
-    SDValue Zero = DAG.getConstantFP(-0.0, Op.getValueType());
-    return DAG.getNode(ISD::FSUB, SDLoc(Op), Op.getValueType(),
+    SDLoc DL(Op);
+    SDValue Zero = DAG.getConstantFP(-0.0, DL, Op.getValueType());
+    return DAG.getNode(ISD::FSUB, DL, Op.getValueType(),
                        Zero, Op.getOperand(0));
   }
   return DAG.UnrollVectorOp(Op.getNode());
@@ -984,17 +1027,20 @@ SDValue VectorLegalizer::UnrollVSETCC(SDValue Op) {
   SDLoc dl(Op);
   SmallVector<SDValue, 8> Ops(NumElems);
   for (unsigned i = 0; i < NumElems; ++i) {
-    SDValue LHSElem = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, TmpEltVT, LHS,
-                                  DAG.getConstant(i, TLI.getVectorIdxTy()));
-    SDValue RHSElem = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, TmpEltVT, RHS,
-                                  DAG.getConstant(i, TLI.getVectorIdxTy()));
+    SDValue LHSElem = DAG.getNode(
+        ISD::EXTRACT_VECTOR_ELT, dl, TmpEltVT, LHS,
+        DAG.getConstant(i, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
+    SDValue RHSElem = DAG.getNode(
+        ISD::EXTRACT_VECTOR_ELT, dl, TmpEltVT, RHS,
+        DAG.getConstant(i, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
     Ops[i] = DAG.getNode(ISD::SETCC, dl,
-                         TLI.getSetCCResultType(*DAG.getContext(), TmpEltVT),
+                         TLI.getSetCCResultType(DAG.getDataLayout(),
+                                                *DAG.getContext(), TmpEltVT),
                          LHSElem, RHSElem, CC);
     Ops[i] = DAG.getSelect(dl, EltVT, Ops[i],
                            DAG.getConstant(APInt::getAllOnesValue
-                                           (EltVT.getSizeInBits()), EltVT),
-                           DAG.getConstant(0, EltVT));
+                                           (EltVT.getSizeInBits()), dl, EltVT),
+                           DAG.getConstant(0, dl, EltVT));
   }
   return DAG.getNode(ISD::BUILD_VECTOR, dl, VT, Ops);
 }
diff --git a/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp b/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp
index f000902260e..5f9afc9cfc5 100644
--- a/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp
+++ b/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp
@@ -251,8 +251,9 @@ SDValue DAGTypeLegalizer::ScalarizeVecRes_UnaryOp(SDNode *N) {
     Op = GetScalarizedVector(Op);
   } else {
     EVT VT = OpVT.getVectorElementType();
-    Op = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT, Op,
-                      DAG.getConstant(0, TLI.getVectorIdxTy()));
+    Op = DAG.getNode(
+        ISD::EXTRACT_VECTOR_ELT, DL, VT, Op,
+        DAG.getConstant(0, DL, TLI.getVectorIdxTy(DAG.getDataLayout())));
   }
   return DAG.getNode(N->getOpcode(), SDLoc(N), DestVT, Op);
 }
@@ -308,7 +309,7 @@ SDValue DAGTypeLegalizer::ScalarizeVecRes_VSELECT(SDNode *N) {
                VecBool == TargetLowering::ZeroOrNegativeOneBooleanContent);
         // Vector read from all ones, scalar expects a single 1 so mask.
         Cond = DAG.getNode(ISD::AND, SDLoc(N), CondVT,
-                           Cond, DAG.getConstant(1, CondVT));
+                           Cond, DAG.getConstant(1, SDLoc(N), CondVT));
         break;
       case TargetLowering::ZeroOrNegativeOneBooleanContent:
         assert(VecBool == TargetLowering::UndefinedBooleanContent ||
@@ -384,10 +385,12 @@ SDValue DAGTypeLegalizer::ScalarizeVecRes_VSETCC(SDNode *N) {
     RHS = GetScalarizedVector(RHS);
   } else {
     EVT VT = OpVT.getVectorElementType();
-    LHS = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT, LHS,
-                      DAG.getConstant(0, TLI.getVectorIdxTy()));
-    RHS = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT, RHS,
-                      DAG.getConstant(0, TLI.getVectorIdxTy()));
+    LHS = DAG.getNode(
+        ISD::EXTRACT_VECTOR_ELT, DL, VT, LHS,
+        DAG.getConstant(0, DL, TLI.getVectorIdxTy(DAG.getDataLayout())));
+    RHS = DAG.getNode(
+        ISD::EXTRACT_VECTOR_ELT, DL, VT, RHS,
+        DAG.getConstant(0, DL, TLI.getVectorIdxTy(DAG.getDataLayout())));
   }
 
   // Turn it into a scalar SETCC.
@@ -600,6 +603,9 @@ void DAGTypeLegalizer::SplitVectorResult(SDNode *N, unsigned ResNo) {
   case ISD::MLOAD:
     SplitVecRes_MLOAD(cast<MaskedLoadSDNode>(N), Lo, Hi);
     break;
+  case ISD::MGATHER:
+    SplitVecRes_MGATHER(cast<MaskedGatherSDNode>(N), Lo, Hi);
+    break;
   case ISD::SETCC:
     SplitVecRes_SETCC(N, Lo, Hi);
     break;
@@ -668,6 +674,12 @@ void DAGTypeLegalizer::SplitVectorResult(SDNode *N, unsigned ResNo) {
   case ISD::UREM:
   case ISD::SREM:
   case ISD::FREM:
+  case ISD::SMIN:
+  case ISD::SMAX:
+  case ISD::UMIN:
+  case ISD::UMAX:
+  case ISD::UABSDIFF:
+  case ISD::SABSDIFF:
     SplitVecRes_BinOp(N, Lo, Hi);
     break;
   case ISD::FMA:
@@ -723,6 +735,7 @@ void DAGTypeLegalizer::SplitVecRes_BITCAST(SDNode *N, SDValue &Lo,
   switch (getTypeAction(InVT)) {
   case TargetLowering::TypeLegal:
   case TargetLowering::TypePromoteInteger:
+  case TargetLowering::TypePromoteFloat:
   case TargetLowering::TypeSoftenFloat:
   case TargetLowering::TypeScalarizeVector:
   case TargetLowering::TypeWidenVector:
@@ -734,7 +747,7 @@ void DAGTypeLegalizer::SplitVecRes_BITCAST(SDNode *N, SDValue &Lo,
     // expanded pieces.
     if (LoVT == HiVT) {
       GetExpandedOp(InOp, Lo, Hi);
-      if (TLI.isBigEndian())
+      if (DAG.getDataLayout().isBigEndian())
         std::swap(Lo, Hi);
       Lo = DAG.getNode(ISD::BITCAST, dl, LoVT, Lo);
       Hi = DAG.getNode(ISD::BITCAST, dl, HiVT, Hi);
@@ -753,12 +766,12 @@ void DAGTypeLegalizer::SplitVecRes_BITCAST(SDNode *N, SDValue &Lo,
   // In the general case, convert the input to an integer and split it by hand.
   EVT LoIntVT = EVT::getIntegerVT(*DAG.getContext(), LoVT.getSizeInBits());
   EVT HiIntVT = EVT::getIntegerVT(*DAG.getContext(), HiVT.getSizeInBits());
-  if (TLI.isBigEndian())
+  if (DAG.getDataLayout().isBigEndian())
     std::swap(LoIntVT, HiIntVT);
 
   SplitInteger(BitConvertToInteger(InOp), LoIntVT, HiIntVT, Lo, Hi);
 
-  if (TLI.isBigEndian())
+  if (DAG.getDataLayout().isBigEndian())
     std::swap(Lo, Hi);
   Lo = DAG.getNode(ISD::BITCAST, dl, LoVT, Lo);
   Hi = DAG.getNode(ISD::BITCAST, dl, HiVT, Hi);
@@ -810,8 +823,8 @@ void DAGTypeLegalizer::SplitVecRes_EXTRACT_SUBVECTOR(SDNode *N, SDValue &Lo,
   Lo = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, LoVT, Vec, Idx);
   uint64_t IdxVal = cast<ConstantSDNode>(Idx)->getZExtValue();
   Hi = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, HiVT, Vec,
-                   DAG.getConstant(IdxVal + LoVT.getVectorNumElements(),
-                                   TLI.getVectorIdxTy()));
+                   DAG.getConstant(IdxVal + LoVT.getVectorNumElements(), dl,
+                                   TLI.getVectorIdxTy(DAG.getDataLayout())));
 }
 
 void DAGTypeLegalizer::SplitVecRes_INSERT_SUBVECTOR(SDNode *N, SDValue &Lo,
@@ -832,7 +845,7 @@ void DAGTypeLegalizer::SplitVecRes_INSERT_SUBVECTOR(SDNode *N, SDValue &Lo,
   // Store the new subvector into the specified index.
   SDValue SubVecPtr = GetVectorElementPointer(StackPtr, SubVecVT, Idx);
   Type *VecType = VecVT.getTypeForEVT(*DAG.getContext());
-  unsigned Alignment = TLI.getDataLayout()->getPrefTypeAlignment(VecType);
+  unsigned Alignment = DAG.getDataLayout().getPrefTypeAlignment(VecType);
   Store = DAG.getStore(Store, dl, SubVec, SubVecPtr, MachinePointerInfo(),
                        false, false, 0);
 
@@ -844,7 +857,7 @@ void DAGTypeLegalizer::SplitVecRes_INSERT_SUBVECTOR(SDNode *N, SDValue &Lo,
   unsigned IncrementSize = Lo.getValueType().getSizeInBits() / 8;
   StackPtr =
       DAG.getNode(ISD::ADD, dl, StackPtr.getValueType(), StackPtr,
-                  DAG.getConstant(IncrementSize, StackPtr.getValueType()));
+                  DAG.getConstant(IncrementSize, dl, StackPtr.getValueType()));
 
   // Load the Hi part from the stack slot.
   Hi = DAG.getLoad(Hi.getValueType(), dl, Store, StackPtr, MachinePointerInfo(),
@@ -890,9 +903,10 @@ void DAGTypeLegalizer::SplitVecRes_INSERT_VECTOR_ELT(SDNode *N, SDValue &Lo,
       Lo = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl,
                        Lo.getValueType(), Lo, Elt, Idx);
     else
-      Hi = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, Hi.getValueType(), Hi, Elt,
-                       DAG.getConstant(IdxVal - LoNumElts,
-                                       TLI.getVectorIdxTy()));
+      Hi =
+          DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, Hi.getValueType(), Hi, Elt,
+                      DAG.getConstant(IdxVal - LoNumElts, dl,
+                                      TLI.getVectorIdxTy(DAG.getDataLayout())));
     return;
   }
 
@@ -911,8 +925,7 @@ void DAGTypeLegalizer::SplitVecRes_INSERT_VECTOR_ELT(SDNode *N, SDValue &Lo,
   // so use a truncating store.
   SDValue EltPtr = GetVectorElementPointer(StackPtr, EltVT, Idx);
   Type *VecType = VecVT.getTypeForEVT(*DAG.getContext());
-  unsigned Alignment =
-    TLI.getDataLayout()->getPrefTypeAlignment(VecType);
+  unsigned Alignment = DAG.getDataLayout().getPrefTypeAlignment(VecType);
   Store = DAG.getTruncStore(Store, dl, Elt, EltPtr, MachinePointerInfo(), EltVT,
                             false, false, 0);
 
@@ -923,7 +936,8 @@ void DAGTypeLegalizer::SplitVecRes_INSERT_VECTOR_ELT(SDNode *N, SDValue &Lo,
   // Increment the pointer to the other part.
   unsigned IncrementSize = Lo.getValueType().getSizeInBits() / 8;
   StackPtr = DAG.getNode(ISD::ADD, dl, StackPtr.getValueType(), StackPtr,
-                       DAG.getConstant(IncrementSize, StackPtr.getValueType()));
+                         DAG.getConstant(IncrementSize, dl,
+                                         StackPtr.getValueType()));
 
   // Load the Hi part from the stack slot.
   Hi = DAG.getLoad(Hi.getValueType(), dl, Store, StackPtr, MachinePointerInfo(),
@@ -966,7 +980,7 @@ void DAGTypeLegalizer::SplitVecRes_LOAD(LoadSDNode *LD, SDValue &Lo,
 
   unsigned IncrementSize = LoMemVT.getSizeInBits()/8;
   Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr,
-                    DAG.getConstant(IncrementSize, Ptr.getValueType()));
+                    DAG.getConstant(IncrementSize, dl, Ptr.getValueType()));
   Hi = DAG.getLoad(ISD::UNINDEXED, ExtType, HiVT, dl, Ch, Ptr, Offset,
                    LD->getPointerInfo().getWithOffset(IncrementSize),
                    HiMemVT, isVolatile, isNonTemporal, isInvariant, Alignment,
@@ -1021,7 +1035,7 @@ void DAGTypeLegalizer::SplitVecRes_MLOAD(MaskedLoadSDNode *MLD,
 
   unsigned IncrementSize = LoMemVT.getSizeInBits()/8;
   Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr,
-                    DAG.getConstant(IncrementSize, Ptr.getValueType()));
+                    DAG.getConstant(IncrementSize, dl, Ptr.getValueType()));
 
   MMO = DAG.getMachineFunction().
     getMachineMemOperand(MLD->getPointerInfo(), 
@@ -1043,6 +1057,54 @@ void DAGTypeLegalizer::SplitVecRes_MLOAD(MaskedLoadSDNode *MLD,
 
 }
 
+void DAGTypeLegalizer::SplitVecRes_MGATHER(MaskedGatherSDNode *MGT,
+                                         SDValue &Lo, SDValue &Hi) {
+  EVT LoVT, HiVT;
+  SDLoc dl(MGT);
+  std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(MGT->getValueType(0));
+
+  SDValue Ch = MGT->getChain();
+  SDValue Ptr = MGT->getBasePtr();
+  SDValue Mask = MGT->getMask();
+  unsigned Alignment = MGT->getOriginalAlignment();
+
+  SDValue MaskLo, MaskHi;
+  std::tie(MaskLo, MaskHi) = DAG.SplitVector(Mask, dl);
+
+  EVT MemoryVT = MGT->getMemoryVT();
+  EVT LoMemVT, HiMemVT;
+  std::tie(LoMemVT, HiMemVT) = DAG.GetSplitDestVTs(MemoryVT);
+
+  SDValue Src0Lo, Src0Hi;
+  std::tie(Src0Lo, Src0Hi) = DAG.SplitVector(MGT->getValue(), dl);
+
+  SDValue IndexHi, IndexLo;
+  std::tie(IndexLo, IndexHi) = DAG.SplitVector(MGT->getIndex(), dl);
+
+  MachineMemOperand *MMO = DAG.getMachineFunction().
+    getMachineMemOperand(MGT->getPointerInfo(), 
+                         MachineMemOperand::MOLoad,  LoMemVT.getStoreSize(),
+                         Alignment, MGT->getAAInfo(), MGT->getRanges());
+
+  SDValue OpsLo[] = {Ch, Src0Lo, MaskLo, Ptr, IndexLo};
+  Lo = DAG.getMaskedGather(DAG.getVTList(LoVT, MVT::Other), LoVT, dl, OpsLo,
+                           MMO);
+
+  SDValue OpsHi[] = {Ch, Src0Hi, MaskHi, Ptr, IndexHi};
+  Hi = DAG.getMaskedGather(DAG.getVTList(HiVT, MVT::Other), HiVT, dl, OpsHi,
+                           MMO);
+
+  // Build a factor node to remember that this load is independent of the
+  // other one.
+  Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1),
+                   Hi.getValue(1));
+
+  // Legalized the chain result - switch anything that used the old chain to
+  // use the new one.
+  ReplaceValueWith(SDValue(MGT, 1), Ch);
+}
+
+
 void DAGTypeLegalizer::SplitVecRes_SETCC(SDNode *N, SDValue &Lo, SDValue &Hi) {
   assert(N->getValueType(0).isVector() &&
          N->getOperand(0).getValueType().isVector() &&
@@ -1235,9 +1297,9 @@ void DAGTypeLegalizer::SplitVecRes_VECTOR_SHUFFLE(ShuffleVectorSDNode *N,
         Idx -= Input * NewElts;
 
         // Extract the vector element by hand.
-        SVOps.push_back(DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, EltVT,
-                                    Inputs[Input], DAG.getConstant(Idx,
-                                                   TLI.getVectorIdxTy())));
+        SVOps.push_back(DAG.getNode(
+            ISD::EXTRACT_VECTOR_ELT, dl, EltVT, Inputs[Input],
+            DAG.getConstant(Idx, dl, TLI.getVectorIdxTy(DAG.getDataLayout()))));
       }
 
       // Construct the Lo/Hi output using a BUILD_VECTOR.
@@ -1294,7 +1356,7 @@ bool DAGTypeLegalizer::SplitVectorOperand(SDNode *N, unsigned OpNo) {
     case ISD::EXTRACT_VECTOR_ELT:Res = SplitVecOp_EXTRACT_VECTOR_ELT(N); break;
     case ISD::CONCAT_VECTORS:    Res = SplitVecOp_CONCAT_VECTORS(N); break;
     case ISD::TRUNCATE:
-      Res = SplitVecOp_TruncateHelper(N, ISD::TRUNCATE);
+      Res = SplitVecOp_TruncateHelper(N);
       break;
     case ISD::FP_ROUND:          Res = SplitVecOp_FP_ROUND(N); break;
     case ISD::STORE:
@@ -1303,20 +1365,26 @@ bool DAGTypeLegalizer::SplitVectorOperand(SDNode *N, unsigned OpNo) {
     case ISD::MSTORE:
       Res = SplitVecOp_MSTORE(cast<MaskedStoreSDNode>(N), OpNo);
       break;
+    case ISD::MSCATTER:
+      Res = SplitVecOp_MSCATTER(cast<MaskedScatterSDNode>(N), OpNo);
+      break;
+    case ISD::MGATHER:
+      Res = SplitVecOp_MGATHER(cast<MaskedGatherSDNode>(N), OpNo);
+      break;
     case ISD::VSELECT:
       Res = SplitVecOp_VSELECT(N, OpNo);
       break;
     case ISD::FP_TO_SINT:
     case ISD::FP_TO_UINT:
       if (N->getValueType(0).bitsLT(N->getOperand(0)->getValueType(0)))
-        Res = SplitVecOp_TruncateHelper(N, ISD::TRUNCATE);
+        Res = SplitVecOp_TruncateHelper(N);
       else
         Res = SplitVecOp_UnaryOp(N);
       break;
     case ISD::SINT_TO_FP:
     case ISD::UINT_TO_FP:
       if (N->getValueType(0).bitsLT(N->getOperand(0)->getValueType(0)))
-        Res = SplitVecOp_TruncateHelper(N, ISD::FTRUNC);
+        Res = SplitVecOp_TruncateHelper(N);
       else
         Res = SplitVecOp_UnaryOp(N);
       break;
@@ -1327,10 +1395,8 @@ bool DAGTypeLegalizer::SplitVectorOperand(SDNode *N, unsigned OpNo) {
     case ISD::SIGN_EXTEND:
     case ISD::ZERO_EXTEND:
     case ISD::ANY_EXTEND:
-      Res = SplitVecOp_UnaryOp(N);
-      break;
     case ISD::FTRUNC:
-      Res = SplitVecOp_TruncateHelper(N, ISD::FTRUNC);
+      Res = SplitVecOp_UnaryOp(N);
       break;
     }
   }
@@ -1410,7 +1476,7 @@ SDValue DAGTypeLegalizer::SplitVecOp_BITCAST(SDNode *N) {
   Lo = BitConvertToInteger(Lo);
   Hi = BitConvertToInteger(Hi);
 
-  if (TLI.isBigEndian())
+  if (DAG.getDataLayout().isBigEndian())
     std::swap(Lo, Hi);
 
   return DAG.getNode(ISD::BITCAST, SDLoc(N), N->getValueType(0),
@@ -1434,7 +1500,8 @@ SDValue DAGTypeLegalizer::SplitVecOp_EXTRACT_SUBVECTOR(SDNode *N) {
     return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, SubVT, Lo, Idx);
   } else {
     return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, SubVT, Hi,
-                       DAG.getConstant(IdxVal - LoElts, Idx.getValueType()));
+                       DAG.getConstant(IdxVal - LoElts, dl,
+                                       Idx.getValueType()));
   }
 }
 
@@ -1455,7 +1522,7 @@ SDValue DAGTypeLegalizer::SplitVecOp_EXTRACT_VECTOR_ELT(SDNode *N) {
     if (IdxVal < LoElts)
       return SDValue(DAG.UpdateNodeOperands(N, Lo, Idx), 0);
     return SDValue(DAG.UpdateNodeOperands(N, Hi,
-                                  DAG.getConstant(IdxVal - LoElts,
+                                  DAG.getConstant(IdxVal - LoElts, SDLoc(N),
                                                   Idx.getValueType())), 0);
   }
 
@@ -1476,6 +1543,68 @@ SDValue DAGTypeLegalizer::SplitVecOp_EXTRACT_VECTOR_ELT(SDNode *N) {
                         MachinePointerInfo(), EltVT, false, false, false, 0);
 }
 
+SDValue DAGTypeLegalizer::SplitVecOp_MGATHER(MaskedGatherSDNode *MGT,
+                                             unsigned OpNo) {
+  EVT LoVT, HiVT;
+  SDLoc dl(MGT);
+  std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(MGT->getValueType(0));
+
+  SDValue Ch = MGT->getChain();
+  SDValue Ptr = MGT->getBasePtr();
+  SDValue Index = MGT->getIndex();
+  SDValue Mask = MGT->getMask();
+  unsigned Alignment = MGT->getOriginalAlignment();
+
+  SDValue MaskLo, MaskHi;
+  std::tie(MaskLo, MaskHi) = DAG.SplitVector(Mask, dl);
+
+  EVT MemoryVT = MGT->getMemoryVT();
+  EVT LoMemVT, HiMemVT;
+  std::tie(LoMemVT, HiMemVT) = DAG.GetSplitDestVTs(MemoryVT);
+
+  SDValue Src0Lo, Src0Hi;
+  std::tie(Src0Lo, Src0Hi) = DAG.SplitVector(MGT->getValue(), dl);
+
+  SDValue IndexHi, IndexLo;
+  if (Index.getNode())
+    std::tie(IndexLo, IndexHi) = DAG.SplitVector(Index, dl);
+  else
+    IndexLo = IndexHi = Index;
+
+  MachineMemOperand *MMO = DAG.getMachineFunction().
+    getMachineMemOperand(MGT->getPointerInfo(), 
+                         MachineMemOperand::MOLoad,  LoMemVT.getStoreSize(),
+                         Alignment, MGT->getAAInfo(), MGT->getRanges());
+
+  SDValue OpsLo[] = {Ch, Src0Lo, MaskLo, Ptr, IndexLo};
+  SDValue Lo = DAG.getMaskedGather(DAG.getVTList(LoVT, MVT::Other), LoVT, dl,
+                                   OpsLo, MMO);
+
+  MMO = DAG.getMachineFunction().
+    getMachineMemOperand(MGT->getPointerInfo(), 
+                         MachineMemOperand::MOLoad,  HiMemVT.getStoreSize(),
+                         Alignment, MGT->getAAInfo(),
+                         MGT->getRanges());
+
+  SDValue OpsHi[] = {Ch, Src0Hi, MaskHi, Ptr, IndexHi};
+  SDValue Hi = DAG.getMaskedGather(DAG.getVTList(HiVT, MVT::Other), HiVT, dl,
+                                   OpsHi, MMO);
+
+  // Build a factor node to remember that this load is independent of the
+  // other one.
+  Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1),
+                   Hi.getValue(1));
+
+  // Legalized the chain result - switch anything that used the old chain to
+  // use the new one.
+  ReplaceValueWith(SDValue(MGT, 1), Ch);
+
+  SDValue Res = DAG.getNode(ISD::CONCAT_VECTORS, dl, MGT->getValueType(0), Lo,
+                            Hi);
+  ReplaceValueWith(SDValue(MGT, 0), Res);
+  return SDValue();
+}
+
 SDValue DAGTypeLegalizer::SplitVecOp_MSTORE(MaskedStoreSDNode *N,
                                             unsigned OpNo) {
   SDValue Ch  = N->getChain();
@@ -1511,7 +1640,7 @@ SDValue DAGTypeLegalizer::SplitVecOp_MSTORE(MaskedStoreSDNode *N,
 
   unsigned IncrementSize = LoMemVT.getSizeInBits()/8;
   Ptr = DAG.getNode(ISD::ADD, DL, Ptr.getValueType(), Ptr,
-                    DAG.getConstant(IncrementSize, Ptr.getValueType()));
+                    DAG.getConstant(IncrementSize, DL, Ptr.getValueType()));
 
   MMO = DAG.getMachineFunction().
     getMachineMemOperand(N->getPointerInfo(), 
@@ -1521,11 +1650,64 @@ SDValue DAGTypeLegalizer::SplitVecOp_MSTORE(MaskedStoreSDNode *N,
   Hi = DAG.getMaskedStore(Ch, DL, DataHi, Ptr, MaskHi, HiMemVT, MMO,
                           N->isTruncatingStore());
 
-
   // Build a factor node to remember that this store is independent of the
   // other one.
   return DAG.getNode(ISD::TokenFactor, DL, MVT::Other, Lo, Hi);
+}
+
+SDValue DAGTypeLegalizer::SplitVecOp_MSCATTER(MaskedScatterSDNode *N,
+                                              unsigned OpNo) {
+  SDValue Ch  = N->getChain();
+  SDValue Ptr = N->getBasePtr();
+  SDValue Mask = N->getMask();
+  SDValue Index = N->getIndex();
+  SDValue Data = N->getValue();
+  EVT MemoryVT = N->getMemoryVT();
+  unsigned Alignment = N->getOriginalAlignment();
+  SDLoc DL(N);
+
+  EVT LoMemVT, HiMemVT;
+  std::tie(LoMemVT, HiMemVT) = DAG.GetSplitDestVTs(MemoryVT);
+
+  SDValue DataLo, DataHi;
+  GetSplitVector(Data, DataLo, DataHi);
+  SDValue MaskLo, MaskHi;
+  GetSplitVector(Mask, MaskLo, MaskHi);
+
+    SDValue PtrLo, PtrHi;
+  if (Ptr.getValueType().isVector()) // gather form vector of pointers
+    std::tie(PtrLo, PtrHi) = DAG.SplitVector(Ptr, DL);
+  else
+    PtrLo = PtrHi = Ptr;
+
+  SDValue IndexHi, IndexLo;
+  if (Index.getNode())
+    std::tie(IndexLo, IndexHi) = DAG.SplitVector(Index, DL);
+  else
+    IndexLo = IndexHi = Index;
+
+  SDValue Lo, Hi;
+  MachineMemOperand *MMO = DAG.getMachineFunction().
+    getMachineMemOperand(N->getPointerInfo(), 
+                         MachineMemOperand::MOStore, LoMemVT.getStoreSize(),
+                         Alignment, N->getAAInfo(), N->getRanges());
+
+  SDValue OpsLo[] = {Ch, DataLo, MaskLo, PtrLo, IndexLo};
+  Lo = DAG.getMaskedScatter(DAG.getVTList(MVT::Other), DataLo.getValueType(),
+                            DL, OpsLo, MMO);
 
+  MMO = DAG.getMachineFunction().
+    getMachineMemOperand(N->getPointerInfo(), 
+                         MachineMemOperand::MOStore,  HiMemVT.getStoreSize(),
+                         Alignment, N->getAAInfo(), N->getRanges());
+
+  SDValue OpsHi[] = {Ch, DataHi, MaskHi, PtrHi, IndexHi};
+  Hi = DAG.getMaskedScatter(DAG.getVTList(MVT::Other), DataHi.getValueType(),
+                            DL, OpsHi, MMO);
+
+  // Build a factor node to remember that this store is independent of the
+  // other one.
+  return DAG.getNode(ISD::TokenFactor, DL, MVT::Other, Lo, Hi);
 }
 
 SDValue DAGTypeLegalizer::SplitVecOp_STORE(StoreSDNode *N, unsigned OpNo) {
@@ -1558,7 +1740,7 @@ SDValue DAGTypeLegalizer::SplitVecOp_STORE(StoreSDNode *N, unsigned OpNo) {
 
   // Increment the pointer to the other half.
   Ptr = DAG.getNode(ISD::ADD, DL, Ptr.getValueType(), Ptr,
-                    DAG.getConstant(IncrementSize, Ptr.getValueType()));
+                    DAG.getConstant(IncrementSize, DL, Ptr.getValueType()));
 
   if (isTruncating)
     Hi = DAG.getTruncStore(Ch, DL, Hi, Ptr,
@@ -1582,21 +1764,19 @@ SDValue DAGTypeLegalizer::SplitVecOp_CONCAT_VECTORS(SDNode *N) {
   // a new CONCAT_VECTORS node with elements that are half-wide.
   SmallVector<SDValue, 32> Elts;
   EVT EltVT = N->getValueType(0).getVectorElementType();
-  for (unsigned op = 0, e = N->getNumOperands(); op != e; ++op) {
-    SDValue Op = N->getOperand(op);
+  for (const SDValue &Op : N->op_values()) {
     for (unsigned i = 0, e = Op.getValueType().getVectorNumElements();
          i != e; ++i) {
-      Elts.push_back(DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, EltVT,
-                                 Op, DAG.getConstant(i, TLI.getVectorIdxTy())));
-
+      Elts.push_back(DAG.getNode(
+          ISD::EXTRACT_VECTOR_ELT, DL, EltVT, Op,
+          DAG.getConstant(i, DL, TLI.getVectorIdxTy(DAG.getDataLayout()))));
     }
   }
 
   return DAG.getNode(ISD::BUILD_VECTOR, DL, N->getValueType(0), Elts);
 }
 
-SDValue DAGTypeLegalizer::SplitVecOp_TruncateHelper(SDNode *N,
-                                                    unsigned TruncateOp) {
+SDValue DAGTypeLegalizer::SplitVecOp_TruncateHelper(SDNode *N) {
   // The result type is legal, but the input type is illegal.  If splitting
   // ends up with the result type of each half still being legal, just
   // do that.  If, however, that would result in an illegal result type,
@@ -1618,6 +1798,8 @@ SDValue DAGTypeLegalizer::SplitVecOp_TruncateHelper(SDNode *N,
   EVT InVT = InVec->getValueType(0);
   EVT OutVT = N->getValueType(0);
   unsigned NumElements = OutVT.getVectorNumElements();
+  bool IsFloat = OutVT.isFloatingPoint();
+  
   // Widening should have already made sure this is a power-two vector
   // if we're trying to split it at all. assert() that's true, just in case.
   assert(!(NumElements & 1) && "Splitting vector, but not in half!");
@@ -1636,7 +1818,9 @@ SDValue DAGTypeLegalizer::SplitVecOp_TruncateHelper(SDNode *N,
   SDValue InLoVec, InHiVec;
   std::tie(InLoVec, InHiVec) = DAG.SplitVector(InVec, DL);
   // Truncate them to 1/2 the element size.
-  EVT HalfElementVT = EVT::getIntegerVT(*DAG.getContext(), InElementSize/2);
+  EVT HalfElementVT = IsFloat ?
+    EVT::getFloatingPointVT(InElementSize/2) :
+    EVT::getIntegerVT(*DAG.getContext(), InElementSize/2);
   EVT HalfVT = EVT::getVectorVT(*DAG.getContext(), HalfElementVT,
                                 NumElements/2);
   SDValue HalfLo = DAG.getNode(N->getOpcode(), DL, HalfVT, InLoVec);
@@ -1649,7 +1833,11 @@ SDValue DAGTypeLegalizer::SplitVecOp_TruncateHelper(SDNode *N,
   // type. This should normally be something that ends up being legal directly,
   // but in theory if a target has very wide vectors and an annoyingly
   // restricted set of legal types, this split can chain to build things up.
-  return DAG.getNode(TruncateOp, DL, OutVT, InterVec);
+  return IsFloat
+             ? DAG.getNode(ISD::FP_ROUND, DL, OutVT, InterVec,
+                           DAG.getTargetConstant(
+                               0, DL, TLI.getPointerTy(DAG.getDataLayout())))
+             : DAG.getNode(ISD::TRUNCATE, DL, OutVT, InterVec);
 }
 
 SDValue DAGTypeLegalizer::SplitVecOp_VSETCC(SDNode *N) {
@@ -1879,10 +2067,12 @@ SDValue DAGTypeLegalizer::WidenVecRes_BinaryCanTrap(SDNode *N) {
   // }
   while (CurNumElts != 0) {
     while (CurNumElts >= NumElts) {
-      SDValue EOp1 = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, VT, InOp1,
-                                 DAG.getConstant(Idx, TLI.getVectorIdxTy()));
-      SDValue EOp2 = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, VT, InOp2,
-                                 DAG.getConstant(Idx, TLI.getVectorIdxTy()));
+      SDValue EOp1 = DAG.getNode(
+          ISD::EXTRACT_SUBVECTOR, dl, VT, InOp1,
+          DAG.getConstant(Idx, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
+      SDValue EOp2 = DAG.getNode(
+          ISD::EXTRACT_SUBVECTOR, dl, VT, InOp2,
+          DAG.getConstant(Idx, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
       ConcatOps[ConcatEnd++] = DAG.getNode(Opcode, dl, VT, EOp1, EOp2);
       Idx += NumElts;
       CurNumElts -= NumElts;
@@ -1894,12 +2084,12 @@ SDValue DAGTypeLegalizer::WidenVecRes_BinaryCanTrap(SDNode *N) {
 
     if (NumElts == 1) {
       for (unsigned i = 0; i != CurNumElts; ++i, ++Idx) {
-        SDValue EOp1 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, WidenEltVT,
-                                   InOp1, DAG.getConstant(Idx,
-                                                         TLI.getVectorIdxTy()));
-        SDValue EOp2 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, WidenEltVT,
-                                   InOp2, DAG.getConstant(Idx,
-                                                         TLI.getVectorIdxTy()));
+        SDValue EOp1 = DAG.getNode(
+            ISD::EXTRACT_VECTOR_ELT, dl, WidenEltVT, InOp1,
+            DAG.getConstant(Idx, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
+        SDValue EOp2 = DAG.getNode(
+            ISD::EXTRACT_VECTOR_ELT, dl, WidenEltVT, InOp2,
+            DAG.getConstant(Idx, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
         ConcatOps[ConcatEnd++] = DAG.getNode(Opcode, dl, WidenEltVT,
                                              EOp1, EOp2);
       }
@@ -1936,9 +2126,9 @@ SDValue DAGTypeLegalizer::WidenVecRes_BinaryCanTrap(SDNode *N) {
       SDValue VecOp = DAG.getUNDEF(NextVT);
       unsigned NumToInsert = ConcatEnd - Idx - 1;
       for (unsigned i = 0, OpIdx = Idx+1; i < NumToInsert; i++, OpIdx++) {
-        VecOp = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, NextVT, VecOp,
-                            ConcatOps[OpIdx], DAG.getConstant(i,
-                                                         TLI.getVectorIdxTy()));
+        VecOp = DAG.getNode(
+            ISD::INSERT_VECTOR_ELT, dl, NextVT, VecOp, ConcatOps[OpIdx],
+            DAG.getConstant(i, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
       }
       ConcatOps[Idx+1] = VecOp;
       ConcatEnd = Idx + 2;
@@ -2024,9 +2214,9 @@ SDValue DAGTypeLegalizer::WidenVecRes_Convert(SDNode *N) {
     }
 
     if (InVTNumElts % WidenNumElts == 0) {
-      SDValue InVal = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, InWidenVT,
-                                  InOp, DAG.getConstant(0,
-                                                        TLI.getVectorIdxTy()));
+      SDValue InVal = DAG.getNode(
+          ISD::EXTRACT_SUBVECTOR, DL, InWidenVT, InOp,
+          DAG.getConstant(0, DL, TLI.getVectorIdxTy(DAG.getDataLayout())));
       // Extract the input and convert the shorten input vector.
       if (N->getNumOperands() == 1)
         return DAG.getNode(Opcode, DL, WidenVT, InVal);
@@ -2040,8 +2230,9 @@ SDValue DAGTypeLegalizer::WidenVecRes_Convert(SDNode *N) {
   unsigned MinElts = std::min(InVTNumElts, WidenNumElts);
   unsigned i;
   for (i=0; i < MinElts; ++i) {
-    SDValue Val = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, InEltVT, InOp,
-                              DAG.getConstant(i, TLI.getVectorIdxTy()));
+    SDValue Val = DAG.getNode(
+        ISD::EXTRACT_VECTOR_ELT, DL, InEltVT, InOp,
+        DAG.getConstant(i, DL, TLI.getVectorIdxTy(DAG.getDataLayout())));
     if (N->getNumOperands() == 1)
       Ops[i] = DAG.getNode(Opcode, DL, EltVT, Val);
     else
@@ -2129,6 +2320,7 @@ SDValue DAGTypeLegalizer::WidenVecRes_BITCAST(SDNode *N) {
       return DAG.getNode(ISD::BITCAST, dl, WidenVT, InOp);
     break;
   case TargetLowering::TypeSoftenFloat:
+  case TargetLowering::TypePromoteFloat:
   case TargetLowering::TypeExpandInteger:
   case TargetLowering::TypeExpandFloat:
   case TargetLowering::TypeScalarizeVector:
@@ -2266,8 +2458,9 @@ SDValue DAGTypeLegalizer::WidenVecRes_CONCAT_VECTORS(SDNode *N) {
     if (InputWidened)
       InOp = GetWidenedVector(InOp);
     for (unsigned j=0; j < NumInElts; ++j)
-      Ops[Idx++] = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, EltVT, InOp,
-                               DAG.getConstant(j, TLI.getVectorIdxTy()));
+      Ops[Idx++] = DAG.getNode(
+          ISD::EXTRACT_VECTOR_ELT, dl, EltVT, InOp,
+          DAG.getConstant(j, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
   }
   SDValue UndefVal = DAG.getUNDEF(EltVT);
   for (; Idx < WidenNumElts; ++Idx)
@@ -2324,8 +2517,9 @@ SDValue DAGTypeLegalizer::WidenVecRes_CONVERT_RNDSAT(SDNode *N) {
 
     if (InVTNumElts % WidenNumElts == 0) {
       // Extract the input and convert the shorten input vector.
-      InOp = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, InWidenVT, InOp,
-                         DAG.getConstant(0, TLI.getVectorIdxTy()));
+      InOp = DAG.getNode(
+          ISD::EXTRACT_SUBVECTOR, dl, InWidenVT, InOp,
+          DAG.getConstant(0, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
       return DAG.getConvertRndSat(WidenVT, dl, InOp, DTyOp, STyOp, RndOp,
                                   SatOp, CvtCode);
     }
@@ -2340,8 +2534,9 @@ SDValue DAGTypeLegalizer::WidenVecRes_CONVERT_RNDSAT(SDNode *N) {
   unsigned MinElts = std::min(InVTNumElts, WidenNumElts);
   unsigned i;
   for (i=0; i < MinElts; ++i) {
-    SDValue ExtVal = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, InEltVT, InOp,
-                                 DAG.getConstant(i, TLI.getVectorIdxTy()));
+    SDValue ExtVal = DAG.getNode(
+        ISD::EXTRACT_VECTOR_ELT, dl, InEltVT, InOp,
+        DAG.getConstant(i, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
     Ops[i] = DAG.getConvertRndSat(WidenVT, dl, ExtVal, DTyOp, STyOp, RndOp,
                                   SatOp, CvtCode);
   }
@@ -2383,8 +2578,10 @@ SDValue DAGTypeLegalizer::WidenVecRes_EXTRACT_SUBVECTOR(SDNode *N) {
   unsigned NumElts = VT.getVectorNumElements();
   unsigned i;
   for (i=0; i < NumElts; ++i)
-    Ops[i] = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, EltVT, InOp,
-                         DAG.getConstant(IdxVal+i, TLI.getVectorIdxTy()));
+    Ops[i] =
+        DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, EltVT, InOp,
+                    DAG.getConstant(IdxVal + i, dl,
+                                    TLI.getVectorIdxTy(DAG.getDataLayout())));
 
   SDValue UndefVal = DAG.getUNDEF(EltVT);
   for (; i < WidenNumElts; ++i)
@@ -2447,7 +2644,7 @@ SDValue DAGTypeLegalizer::WidenVecRes_MLOAD(MaskedLoadSDNode *N) {
 
     unsigned NumConcat = WidenNumElts / MaskNumElts;
     SmallVector<SDValue, 16> Ops(NumConcat);
-    SDValue ZeroVal = DAG.getConstant(0, MaskVT);
+    SDValue ZeroVal = DAG.getConstant(0, dl, MaskVT);
     Ops[0] = Mask;
     for (unsigned i = 1; i != NumConcat; ++i)
       Ops[i] = ZeroVal;
@@ -2685,12 +2882,13 @@ SDValue DAGTypeLegalizer::WidenVecOp_EXTEND(SDNode *N) {
         assert(FixedVT.getVectorNumElements() != InVT.getVectorNumElements() &&
                "We can't have the same type as we started with!");
         if (FixedVT.getVectorNumElements() > InVT.getVectorNumElements())
-          InOp = DAG.getNode(ISD::INSERT_SUBVECTOR, DL, FixedVT,
-                             DAG.getUNDEF(FixedVT), InOp,
-                             DAG.getConstant(0, TLI.getVectorIdxTy()));
+          InOp = DAG.getNode(
+              ISD::INSERT_SUBVECTOR, DL, FixedVT, DAG.getUNDEF(FixedVT), InOp,
+              DAG.getConstant(0, DL, TLI.getVectorIdxTy(DAG.getDataLayout())));
         else
-          InOp = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, FixedVT, InOp,
-                             DAG.getConstant(0, TLI.getVectorIdxTy()));
+          InOp = DAG.getNode(
+              ISD::EXTRACT_SUBVECTOR, DL, FixedVT, InOp,
+              DAG.getConstant(0, DL, TLI.getVectorIdxTy(DAG.getDataLayout())));
         break;
       }
     }
@@ -2733,9 +2931,11 @@ SDValue DAGTypeLegalizer::WidenVecOp_Convert(SDNode *N) {
   unsigned Opcode = N->getOpcode();
   SmallVector<SDValue, 16> Ops(NumElts);
   for (unsigned i=0; i < NumElts; ++i)
-    Ops[i] = DAG.getNode(Opcode, dl, EltVT,
-                         DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, InEltVT, InOp,
-                                     DAG.getConstant(i, TLI.getVectorIdxTy())));
+    Ops[i] = DAG.getNode(
+        Opcode, dl, EltVT,
+        DAG.getNode(
+            ISD::EXTRACT_VECTOR_ELT, dl, InEltVT, InOp,
+            DAG.getConstant(i, dl, TLI.getVectorIdxTy(DAG.getDataLayout()))));
 
   return DAG.getNode(ISD::BUILD_VECTOR, dl, VT, Ops);
 }
@@ -2755,8 +2955,9 @@ SDValue DAGTypeLegalizer::WidenVecOp_BITCAST(SDNode *N) {
     EVT NewVT = EVT::getVectorVT(*DAG.getContext(), VT, NewNumElts);
     if (TLI.isTypeLegal(NewVT)) {
       SDValue BitOp = DAG.getNode(ISD::BITCAST, dl, NewVT, InOp);
-      return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, VT, BitOp,
-                         DAG.getConstant(0, TLI.getVectorIdxTy()));
+      return DAG.getNode(
+          ISD::EXTRACT_VECTOR_ELT, dl, VT, BitOp,
+          DAG.getConstant(0, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
     }
   }
 
@@ -2783,8 +2984,9 @@ SDValue DAGTypeLegalizer::WidenVecOp_CONCAT_VECTORS(SDNode *N) {
     if (getTypeAction(InOp.getValueType()) == TargetLowering::TypeWidenVector)
       InOp = GetWidenedVector(InOp);
     for (unsigned j=0; j < NumInElts; ++j)
-      Ops[Idx++] = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, EltVT, InOp,
-                               DAG.getConstant(j, TLI.getVectorIdxTy()));
+      Ops[Idx++] = DAG.getNode(
+          ISD::EXTRACT_VECTOR_ELT, dl, EltVT, InOp,
+          DAG.getConstant(j, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
   }
   return DAG.getNode(ISD::BUILD_VECTOR, dl, VT, Ops);
 }
@@ -2839,7 +3041,7 @@ SDValue DAGTypeLegalizer::WidenVecOp_MSTORE(SDNode *N, unsigned OpNo) {
 
     unsigned NumConcat = WidenNumElts / MaskNumElts;
     SmallVector<SDValue, 16> Ops(NumConcat);
-    SDValue ZeroVal = DAG.getConstant(0, MaskVT);
+    SDValue ZeroVal = DAG.getConstant(0, dl, MaskVT);
     Ops[0] = Mask;
     for (unsigned i = 1; i != NumConcat; ++i)
       Ops[i] = ZeroVal;
@@ -2865,7 +3067,8 @@ SDValue DAGTypeLegalizer::WidenVecOp_SETCC(SDNode *N) {
 
   // Get a new SETCC node to compare the newly widened operands.
   // Only some of the compared elements are legal.
-  EVT SVT = TLI.getSetCCResultType(*DAG.getContext(), InOp0.getValueType());
+  EVT SVT = TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(),
+                                   InOp0.getValueType());
   SDValue WideSETCC = DAG.getNode(ISD::SETCC, SDLoc(N),
                      SVT, InOp0, InOp1, N->getOperand(2));
 
@@ -2873,9 +3076,9 @@ SDValue DAGTypeLegalizer::WidenVecOp_SETCC(SDNode *N) {
   EVT ResVT = EVT::getVectorVT(*DAG.getContext(),
                                SVT.getVectorElementType(),
                                N->getValueType(0).getVectorNumElements());
-  SDValue CC = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl,
-                           ResVT, WideSETCC, DAG.getConstant(0,
-                                             TLI.getVectorIdxTy()));
+  SDValue CC = DAG.getNode(
+      ISD::EXTRACT_SUBVECTOR, dl, ResVT, WideSETCC,
+      DAG.getConstant(0, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
 
   return PromoteTargetBoolean(CC, N->getValueType(0));
 }
@@ -2913,7 +3116,10 @@ static EVT FindMemType(SelectionDAG& DAG, const TargetLowering &TLI,
     unsigned MemVTWidth = MemVT.getSizeInBits();
     if (MemVT.getSizeInBits() <= WidenEltWidth)
       break;
-    if (TLI.isTypeLegal(MemVT) && (WidenWidth % MemVTWidth) == 0 &&
+    auto Action = TLI.getTypeAction(*DAG.getContext(), MemVT);
+    if ((Action == TargetLowering::TypeLegal ||
+         Action == TargetLowering::TypePromoteInteger) &&
+        (WidenWidth % MemVTWidth) == 0 &&
         isPowerOf2_32(WidenWidth / MemVTWidth) &&
         (MemVTWidth <= Width ||
          (Align!=0 && MemVTWidth<=AlignInBits && MemVTWidth<=Width+WidenEx))) {
@@ -2968,8 +3174,9 @@ static SDValue BuildVectorFromScalar(SelectionDAG& DAG, EVT VecTy,
       Idx = Idx * LdTy.getSizeInBits() / NewLdTy.getSizeInBits();
       LdTy = NewLdTy;
     }
-    VecOp = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, NewVecVT, VecOp, LdOps[i],
-                        DAG.getConstant(Idx++, TLI.getVectorIdxTy()));
+    VecOp = DAG.getNode(
+        ISD::INSERT_VECTOR_ELT, dl, NewVecVT, VecOp, LdOps[i],
+        DAG.getConstant(Idx++, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
   }
   return DAG.getNode(ISD::BITCAST, dl, VecTy, VecOp);
 }
@@ -3040,7 +3247,7 @@ SDValue DAGTypeLegalizer::GenWidenVectorLoads(SmallVectorImpl<SDValue> &LdChain,
     unsigned Increment = NewVTWidth / 8;
     Offset += Increment;
     BasePtr = DAG.getNode(ISD::ADD, dl, BasePtr.getValueType(), BasePtr,
-                          DAG.getConstant(Increment, BasePtr.getValueType()));
+                          DAG.getConstant(Increment, dl, BasePtr.getValueType()));
 
     SDValue L;
     if (LdWidth < NewVTWidth) {
@@ -3166,7 +3373,7 @@ DAGTypeLegalizer::GenWidenVectorExtLoads(SmallVectorImpl<SDValue> &LdChain,
   for (i=1; i < NumElts; ++i, Offset += Increment) {
     SDValue NewBasePtr = DAG.getNode(ISD::ADD, dl, BasePtr.getValueType(),
                                      BasePtr,
-                                     DAG.getConstant(Offset,
+                                     DAG.getConstant(Offset, dl,
                                                      BasePtr.getValueType()));
     Ops[i] = DAG.getExtLoad(ExtType, dl, EltVT, Chain, NewBasePtr,
                             LD->getPointerInfo().getWithOffset(Offset), LdEltVT,
@@ -3216,8 +3423,9 @@ void DAGTypeLegalizer::GenWidenVectorStores(SmallVectorImpl<SDValue> &StChain,
     if (NewVT.isVector()) {
       unsigned NumVTElts = NewVT.getVectorNumElements();
       do {
-        SDValue EOp = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, NewVT, ValOp,
-                                   DAG.getConstant(Idx, TLI.getVectorIdxTy()));
+        SDValue EOp = DAG.getNode(
+            ISD::EXTRACT_SUBVECTOR, dl, NewVT, ValOp,
+            DAG.getConstant(Idx, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
         StChain.push_back(DAG.getStore(Chain, dl, EOp, BasePtr,
                                     ST->getPointerInfo().getWithOffset(Offset),
                                        isVolatile, isNonTemporal,
@@ -3226,7 +3434,8 @@ void DAGTypeLegalizer::GenWidenVectorStores(SmallVectorImpl<SDValue> &StChain,
         Offset += Increment;
         Idx += NumVTElts;
         BasePtr = DAG.getNode(ISD::ADD, dl, BasePtr.getValueType(), BasePtr,
-                              DAG.getConstant(Increment, BasePtr.getValueType()));
+                              DAG.getConstant(Increment, dl,
+                                              BasePtr.getValueType()));
       } while (StWidth != 0 && StWidth >= NewVTWidth);
     } else {
       // Cast the vector to the scalar type we can store
@@ -3236,8 +3445,10 @@ void DAGTypeLegalizer::GenWidenVectorStores(SmallVectorImpl<SDValue> &StChain,
       // Readjust index position based on new vector type
       Idx = Idx * ValEltWidth / NewVTWidth;
       do {
-        SDValue EOp = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, NewVT, VecOp,
-                      DAG.getConstant(Idx++, TLI.getVectorIdxTy()));
+        SDValue EOp = DAG.getNode(
+            ISD::EXTRACT_VECTOR_ELT, dl, NewVT, VecOp,
+            DAG.getConstant(Idx++, dl,
+                            TLI.getVectorIdxTy(DAG.getDataLayout())));
         StChain.push_back(DAG.getStore(Chain, dl, EOp, BasePtr,
                                     ST->getPointerInfo().getWithOffset(Offset),
                                        isVolatile, isNonTemporal,
@@ -3245,7 +3456,8 @@ void DAGTypeLegalizer::GenWidenVectorStores(SmallVectorImpl<SDValue> &StChain,
         StWidth -= NewVTWidth;
         Offset += Increment;
         BasePtr = DAG.getNode(ISD::ADD, dl, BasePtr.getValueType(), BasePtr,
-                            DAG.getConstant(Increment, BasePtr.getValueType()));
+                              DAG.getConstant(Increment, dl,
+                                              BasePtr.getValueType()));
       } while (StWidth != 0 && StWidth >= NewVTWidth);
       // Restore index back to be relative to the original widen element type
       Idx = Idx * NewVTWidth / ValEltWidth;
@@ -3282,8 +3494,9 @@ DAGTypeLegalizer::GenWidenVectorTruncStores(SmallVectorImpl<SDValue> &StChain,
   EVT ValEltVT = ValVT.getVectorElementType();
   unsigned Increment = ValEltVT.getSizeInBits() / 8;
   unsigned NumElts = StVT.getVectorNumElements();
-  SDValue EOp = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, ValEltVT, ValOp,
-                            DAG.getConstant(0, TLI.getVectorIdxTy()));
+  SDValue EOp = DAG.getNode(
+      ISD::EXTRACT_VECTOR_ELT, dl, ValEltVT, ValOp,
+      DAG.getConstant(0, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
   StChain.push_back(DAG.getTruncStore(Chain, dl, EOp, BasePtr,
                                       ST->getPointerInfo(), StEltVT,
                                       isVolatile, isNonTemporal, Align,
@@ -3291,10 +3504,12 @@ DAGTypeLegalizer::GenWidenVectorTruncStores(SmallVectorImpl<SDValue> &StChain,
   unsigned Offset = Increment;
   for (unsigned i=1; i < NumElts; ++i, Offset += Increment) {
     SDValue NewBasePtr = DAG.getNode(ISD::ADD, dl, BasePtr.getValueType(),
-                                     BasePtr, DAG.getConstant(Offset,
-                                                       BasePtr.getValueType()));
-    SDValue EOp = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, ValEltVT, ValOp,
-                            DAG.getConstant(0, TLI.getVectorIdxTy()));
+                                     BasePtr,
+                                     DAG.getConstant(Offset, dl,
+                                                     BasePtr.getValueType()));
+    SDValue EOp = DAG.getNode(
+        ISD::EXTRACT_VECTOR_ELT, dl, ValEltVT, ValOp,
+        DAG.getConstant(0, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
     StChain.push_back(DAG.getTruncStore(Chain, dl, EOp, NewBasePtr,
                                       ST->getPointerInfo().getWithOffset(Offset),
                                         StEltVT, isVolatile, isNonTemporal,
@@ -3330,8 +3545,9 @@ SDValue DAGTypeLegalizer::ModifyToType(SDValue InOp, EVT NVT) {
   }
 
   if (WidenNumElts < InNumElts && InNumElts % WidenNumElts)
-    return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, NVT, InOp,
-                       DAG.getConstant(0, TLI.getVectorIdxTy()));
+    return DAG.getNode(
+        ISD::EXTRACT_SUBVECTOR, dl, NVT, InOp,
+        DAG.getConstant(0, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
 
   // Fall back to extract and build.
   SmallVector<SDValue, 16> Ops(WidenNumElts);
@@ -3339,8 +3555,9 @@ SDValue DAGTypeLegalizer::ModifyToType(SDValue InOp, EVT NVT) {
   unsigned MinNumElts = std::min(WidenNumElts, InNumElts);
   unsigned Idx;
   for (Idx = 0; Idx < MinNumElts; ++Idx)
-    Ops[Idx] = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, EltVT, InOp,
-                           DAG.getConstant(Idx, TLI.getVectorIdxTy()));
+    Ops[Idx] = DAG.getNode(
+        ISD::EXTRACT_VECTOR_ELT, dl, EltVT, InOp,
+        DAG.getConstant(Idx, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
 
   SDValue UndefVal = DAG.getUNDEF(EltVT);
   for ( ; Idx < WidenNumElts; ++Idx)
diff --git a/lib/CodeGen/SelectionDAG/SDNodeDbgValue.h b/lib/CodeGen/SelectionDAG/SDNodeDbgValue.h
index bce69d79ab1..c27f8de601f 100644
--- a/lib/CodeGen/SelectionDAG/SDNodeDbgValue.h
+++ b/lib/CodeGen/SelectionDAG/SDNodeDbgValue.h
@@ -35,7 +35,6 @@ class SDDbgValue {
     FRAMEIX = 2             // value is contents of a stack location
   };
 private:
-  enum DbgValueKind kind;
   union {
     struct {
       SDNode *Node;         // valid for expressions
@@ -46,17 +45,18 @@ class SDDbgValue {
   } u;
   MDNode *Var;
   MDNode *Expr;
-  bool IsIndirect;
   uint64_t Offset;
   DebugLoc DL;
   unsigned Order;
-  bool Invalid;
+  enum DbgValueKind kind;
+  bool IsIndirect;
+  bool Invalid = false;
+
 public:
   // Constructor for non-constants.
   SDDbgValue(MDNode *Var, MDNode *Expr, SDNode *N, unsigned R, bool indir,
              uint64_t off, DebugLoc dl, unsigned O)
-      : Var(Var), Expr(Expr), IsIndirect(indir), Offset(off), DL(dl), Order(O),
-        Invalid(false) {
+      : Var(Var), Expr(Expr), Offset(off), DL(dl), Order(O), IsIndirect(indir) {
     kind = SDNODE;
     u.s.Node = N;
     u.s.ResNo = R;
@@ -65,8 +65,7 @@ class SDDbgValue {
   // Constructor for constants.
   SDDbgValue(MDNode *Var, MDNode *Expr, const Value *C, uint64_t off,
              DebugLoc dl, unsigned O)
-      : Var(Var), Expr(Expr), IsIndirect(false), Offset(off), DL(dl), Order(O),
-        Invalid(false) {
+      : Var(Var), Expr(Expr), Offset(off), DL(dl), Order(O), IsIndirect(false) {
     kind = CONST;
     u.Const = C;
   }
@@ -74,8 +73,7 @@ class SDDbgValue {
   // Constructor for frame indices.
   SDDbgValue(MDNode *Var, MDNode *Expr, unsigned FI, uint64_t off, DebugLoc dl,
              unsigned O)
-      : Var(Var), Expr(Expr), IsIndirect(false), Offset(off), DL(dl), Order(O),
-        Invalid(false) {
+      : Var(Var), Expr(Expr), Offset(off), DL(dl), Order(O), IsIndirect(false) {
     kind = FRAMEIX;
     u.FrameIx = FI;
   }
diff --git a/lib/CodeGen/SelectionDAG/ScheduleDAGFast.cpp b/lib/CodeGen/SelectionDAG/ScheduleDAGFast.cpp
index 61a3fd72871..34e1a700108 100644
--- a/lib/CodeGen/SelectionDAG/ScheduleDAGFast.cpp
+++ b/lib/CodeGen/SelectionDAG/ScheduleDAGFast.cpp
@@ -227,8 +227,7 @@ SUnit *ScheduleDAGFast::CopyAndMoveSuccessors(SUnit *SU) {
     else if (VT == MVT::Other)
       TryUnfold = true;
   }
-  for (unsigned i = 0, e = N->getNumOperands(); i != e; ++i) {
-    const SDValue &Op = N->getOperand(i);
+  for (const SDValue &Op : N->op_values()) {
     MVT VT = Op.getNode()->getSimpleValueType(Op.getResNo());
     if (VT == MVT::Glue)
       return nullptr;
@@ -726,9 +725,8 @@ void ScheduleDAGLinearize::Schedule() {
 
   SmallVector<SDNode*, 8> Glues;
   unsigned DAGSize = 0;
-  for (SelectionDAG::allnodes_iterator I = DAG->allnodes_begin(),
-         E = DAG->allnodes_end(); I != E; ++I) {
-    SDNode *N = I;
+  for (SDNode &Node : DAG->allnodes()) {
+    SDNode *N = &Node;
 
     // Use node id to record degree.
     unsigned Degree = N->use_size();
diff --git a/lib/CodeGen/SelectionDAG/ScheduleDAGRRList.cpp b/lib/CodeGen/SelectionDAG/ScheduleDAGRRList.cpp
index fd0fa31842b..e9bd52034ff 100644
--- a/lib/CodeGen/SelectionDAG/ScheduleDAGRRList.cpp
+++ b/lib/CodeGen/SelectionDAG/ScheduleDAGRRList.cpp
@@ -415,8 +415,8 @@ static bool IsChainDependent(SDNode *Outer, SDNode *Inner,
     // to get to the CALLSEQ_BEGIN, but we need to find the path with the
     // most nesting in order to ensure that we find the corresponding match.
     if (N->getOpcode() == ISD::TokenFactor) {
-      for (unsigned i = 0, e = N->getNumOperands(); i != e; ++i)
-        if (IsChainDependent(N->getOperand(i).getNode(), Inner, NestLevel, TII))
+      for (const SDValue &Op : N->op_values())
+        if (IsChainDependent(Op.getNode(), Inner, NestLevel, TII))
           return true;
       return false;
     }
@@ -433,9 +433,9 @@ static bool IsChainDependent(SDNode *Outer, SDNode *Inner,
       }
     }
     // Otherwise, find the chain and continue climbing.
-    for (unsigned i = 0, e = N->getNumOperands(); i != e; ++i)
-      if (N->getOperand(i).getValueType() == MVT::Other) {
-        N = N->getOperand(i).getNode();
+    for (const SDValue &Op : N->op_values())
+      if (Op.getValueType() == MVT::Other) {
+        N = Op.getNode();
         goto found_chain_operand;
       }
     return false;
@@ -464,10 +464,10 @@ FindCallSeqStart(SDNode *N, unsigned &NestLevel, unsigned &MaxNest,
     if (N->getOpcode() == ISD::TokenFactor) {
       SDNode *Best = nullptr;
       unsigned BestMaxNest = MaxNest;
-      for (unsigned i = 0, e = N->getNumOperands(); i != e; ++i) {
+      for (const SDValue &Op : N->op_values()) {
         unsigned MyNestLevel = NestLevel;
         unsigned MyMaxNest = MaxNest;
-        if (SDNode *New = FindCallSeqStart(N->getOperand(i).getNode(),
+        if (SDNode *New = FindCallSeqStart(Op.getNode(),
                                            MyNestLevel, MyMaxNest, TII))
           if (!Best || (MyMaxNest > BestMaxNest)) {
             Best = New;
@@ -493,9 +493,9 @@ FindCallSeqStart(SDNode *N, unsigned &NestLevel, unsigned &MaxNest,
       }
     }
     // Otherwise, find the chain and continue climbing.
-    for (unsigned i = 0, e = N->getNumOperands(); i != e; ++i)
-      if (N->getOperand(i).getValueType() == MVT::Other) {
-        N = N->getOperand(i).getNode();
+    for (const SDValue &Op : N->op_values())
+      if (Op.getValueType() == MVT::Other) {
+        N = Op.getNode();
         goto found_chain_operand;
       }
     return nullptr;
@@ -848,17 +848,26 @@ void ScheduleDAGRRList::UnscheduleNodeBottomUp(SUnit *SU) {
       }
     }
 
-  for (SUnit::succ_iterator I = SU->Succs.begin(), E = SU->Succs.end();
-       I != E; ++I) {
-    if (I->isAssignedRegDep()) {
-      if (!LiveRegDefs[I->getReg()])
+  for (auto &Succ : SU->Succs) {
+    if (Succ.isAssignedRegDep()) {
+      auto Reg = Succ.getReg();
+      if (!LiveRegDefs[Reg])
         ++NumLiveRegs;
       // This becomes the nearest def. Note that an earlier def may still be
       // pending if this is a two-address node.
-      LiveRegDefs[I->getReg()] = SU;
-      if (LiveRegGens[I->getReg()] == nullptr ||
-          I->getSUnit()->getHeight() < LiveRegGens[I->getReg()]->getHeight())
-        LiveRegGens[I->getReg()] = I->getSUnit();
+      LiveRegDefs[Reg] = SU;
+
+      // Update LiveRegGen only if was empty before this unscheduling.
+      // This is to avoid incorrect updating LiveRegGen set in previous run.
+      if (!LiveRegGens[Reg]) {
+        // Find the successor with the lowest height.
+        LiveRegGens[Reg] = Succ.getSUnit();
+        for (auto &Succ2 : SU->Succs) {
+          if (Succ2.isAssignedRegDep() && Succ2.getReg() == Reg &&
+              Succ2.getSUnit()->getHeight() < LiveRegGens[Reg]->getHeight())
+            LiveRegGens[Reg] = Succ2.getSUnit();
+        }
+      }
     }
   }
   if (SU->getHeight() < MinAvailableCycle)
@@ -951,8 +960,7 @@ SUnit *ScheduleDAGRRList::CopyAndMoveSuccessors(SUnit *SU) {
     else if (VT == MVT::Other)
       TryUnfold = true;
   }
-  for (unsigned i = 0, e = N->getNumOperands(); i != e; ++i) {
-    const SDValue &Op = N->getOperand(i);
+  for (const SDValue &Op : N->op_values()) {
     MVT VT = Op.getNode()->getSimpleValueType(Op.getResNo());
     if (VT == MVT::Glue)
       return nullptr;
@@ -1247,10 +1255,9 @@ static void CheckForLiveRegDefMasked(SUnit *SU, const uint32_t *RegMask,
 
 /// getNodeRegMask - Returns the register mask attached to an SDNode, if any.
 static const uint32_t *getNodeRegMask(const SDNode *N) {
-  for (unsigned i = 0, e = N->getNumOperands(); i != e; ++i)
-    if (const RegisterMaskSDNode *Op =
-        dyn_cast<RegisterMaskSDNode>(N->getOperand(i).getNode()))
-      return Op->getRegMask();
+  for (const SDValue &Op : N->op_values())
+    if (const auto *RegOp = dyn_cast<RegisterMaskSDNode>(Op.getNode()))
+      return RegOp->getRegMask();
   return nullptr;
 }
 
diff --git a/lib/CodeGen/SelectionDAG/ScheduleDAGSDNodes.cpp b/lib/CodeGen/SelectionDAG/ScheduleDAGSDNodes.cpp
index 3853ada5fb0..2a6c853a1d1 100644
--- a/lib/CodeGen/SelectionDAG/ScheduleDAGSDNodes.cpp
+++ b/lib/CodeGen/SelectionDAG/ScheduleDAGSDNodes.cpp
@@ -71,7 +71,7 @@ SUnit *ScheduleDAGSDNodes::newSUnit(SDNode *N) {
   if (!SUnits.empty())
     Addr = &SUnits[0];
 #endif
-  SUnits.push_back(SUnit(N, (unsigned)SUnits.size()));
+  SUnits.emplace_back(N, (unsigned)SUnits.size());
   assert((Addr == nullptr || Addr == &SUnits[0]) &&
          "SUnits std::vector reallocated on the fly!");
   SUnits.back().OrigNode = &SUnits.back();
@@ -289,9 +289,8 @@ void ScheduleDAGSDNodes::ClusterNeighboringLoads(SDNode *Node) {
 /// ClusterNodes - Cluster certain nodes which should be scheduled together.
 ///
 void ScheduleDAGSDNodes::ClusterNodes() {
-  for (SelectionDAG::allnodes_iterator NI = DAG->allnodes_begin(),
-       E = DAG->allnodes_end(); NI != E; ++NI) {
-    SDNode *Node = &*NI;
+  for (SDNode &NI : DAG->allnodes()) {
+    SDNode *Node = &NI;
     if (!Node || !Node->isMachineOpcode())
       continue;
 
@@ -308,9 +307,8 @@ void ScheduleDAGSDNodes::BuildSchedUnits() {
   // to their associated SUnits by holding SUnits table indices. A value
   // of -1 means the SDNode does not yet have an associated SUnit.
   unsigned NumNodes = 0;
-  for (SelectionDAG::allnodes_iterator NI = DAG->allnodes_begin(),
-       E = DAG->allnodes_end(); NI != E; ++NI) {
-    NI->setNodeId(-1);
+  for (SDNode &NI : DAG->allnodes()) {
+    NI.setNodeId(-1);
     ++NumNodes;
   }
 
@@ -332,9 +330,9 @@ void ScheduleDAGSDNodes::BuildSchedUnits() {
     SDNode *NI = Worklist.pop_back_val();
 
     // Add all operands to the worklist unless they've already been added.
-    for (unsigned i = 0, e = NI->getNumOperands(); i != e; ++i)
-      if (Visited.insert(NI->getOperand(i).getNode()).second)
-        Worklist.push_back(NI->getOperand(i).getNode());
+    for (const SDValue &Op : NI->op_values())
+      if (Visited.insert(Op.getNode()).second)
+        Worklist.push_back(Op.getNode());
 
     if (isPassiveNode(NI))  // Leaf node, e.g. a TargetImmediate.
       continue;
diff --git a/lib/CodeGen/SelectionDAG/ScheduleDAGSDNodes.h b/lib/CodeGen/SelectionDAG/ScheduleDAGSDNodes.h
index 6351fa2c4a2..159c28cd2a6 100644
--- a/lib/CodeGen/SelectionDAG/ScheduleDAGSDNodes.h
+++ b/lib/CodeGen/SelectionDAG/ScheduleDAGSDNodes.h
@@ -64,6 +64,7 @@ namespace llvm {
       if (isa<TargetIndexSDNode>(Node))    return true;
       if (isa<JumpTableSDNode>(Node))      return true;
       if (isa<ExternalSymbolSDNode>(Node)) return true;
+      if (isa<MCSymbolSDNode>(Node))       return true;
       if (isa<BlockAddressSDNode>(Node))   return true;
       if (Node->getOpcode() == ISD::EntryToken ||
           isa<MDNodeSDNode>(Node)) return true;
diff --git a/lib/CodeGen/SelectionDAG/SelectionDAG.cpp b/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
index 770f0b226ee..4122ce06b04 100644
--- a/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
+++ b/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
@@ -49,6 +49,7 @@
 #include "llvm/Target/TargetSubtargetInfo.h"
 #include <algorithm>
 #include <cmath>
+#include <utility>
 
 using namespace llvm;
 
@@ -150,8 +151,8 @@ bool ISD::isBuildVectorAllZeros(const SDNode *N) {
   if (N->getOpcode() != ISD::BUILD_VECTOR) return false;
 
   bool IsAllUndef = true;
-  for (unsigned i = 0, e = N->getNumOperands(); i < e; ++i) {
-    if (N->getOperand(i).getOpcode() == ISD::UNDEF)
+  for (const SDValue &Op : N->op_values()) {
+    if (Op.getOpcode() == ISD::UNDEF)
       continue;
     IsAllUndef = false;
     // Do not accept build_vectors that aren't all constants or which have non-0
@@ -162,12 +163,11 @@ bool ISD::isBuildVectorAllZeros(const SDNode *N) {
     // We only want to check enough bits to cover the vector elements, because
     // we care if the resultant vector is all zeros, not whether the individual
     // constants are.
-    SDValue Zero = N->getOperand(i);
     unsigned EltSize = N->getValueType(0).getVectorElementType().getSizeInBits();
-    if (ConstantSDNode *CN = dyn_cast<ConstantSDNode>(Zero)) {
+    if (ConstantSDNode *CN = dyn_cast<ConstantSDNode>(Op)) {
       if (CN->getAPIntValue().countTrailingZeros() < EltSize)
         return false;
-    } else if (ConstantFPSDNode *CFPN = dyn_cast<ConstantFPSDNode>(Zero)) {
+    } else if (ConstantFPSDNode *CFPN = dyn_cast<ConstantFPSDNode>(Op)) {
       if (CFPN->getValueAPF().bitcastToAPInt().countTrailingZeros() < EltSize)
         return false;
     } else
@@ -186,8 +186,7 @@ bool ISD::isBuildVectorOfConstantSDNodes(const SDNode *N) {
   if (N->getOpcode() != ISD::BUILD_VECTOR)
     return false;
 
-  for (unsigned i = 0, e = N->getNumOperands(); i != e; ++i) {
-    SDValue Op = N->getOperand(i);
+  for (const SDValue &Op : N->op_values()) {
     if (Op.getOpcode() == ISD::UNDEF)
       continue;
     if (!isa<ConstantSDNode>(Op))
@@ -202,8 +201,7 @@ bool ISD::isBuildVectorOfConstantFPSDNodes(const SDNode *N) {
   if (N->getOpcode() != ISD::BUILD_VECTOR)
     return false;
 
-  for (unsigned i = 0, e = N->getNumOperands(); i != e; ++i) {
-    SDValue Op = N->getOperand(i);
+  for (const SDValue &Op : N->op_values()) {
     if (Op.getOpcode() == ISD::UNDEF)
       continue;
     if (!isa<ConstantFPSDNode>(Op))
@@ -243,8 +241,8 @@ bool ISD::allOperandsUndef(const SDNode *N) {
   if (N->getNumOperands() == 0)
     return false;
 
-  for (unsigned i = 0, e = N->getNumOperands(); i != e ; ++i)
-    if (N->getOperand(i).getOpcode() != ISD::UNDEF)
+  for (const SDValue &Op : N->op_values())
+    if (Op.getOpcode() != ISD::UNDEF)
       return false;
 
   return true;
@@ -399,19 +397,24 @@ static void AddNodeIDOperands(FoldingSetNodeID &ID,
     ID.AddInteger(Op.getResNo());
   }
 }
+/// Add logical or fast math flag values to FoldingSetNodeID value.
+static void AddNodeIDFlags(FoldingSetNodeID &ID, unsigned Opcode,
+                           const SDNodeFlags *Flags) {
+  if (!Flags || !isBinOpWithFlags(Opcode))
+    return;
 
-static void AddBinaryNodeIDCustom(FoldingSetNodeID &ID, bool nuw, bool nsw,
-                                  bool exact) {
-  ID.AddBoolean(nuw);
-  ID.AddBoolean(nsw);
-  ID.AddBoolean(exact);
+  unsigned RawFlags = Flags->getRawFlags();
+  // If no flags are set, do not alter the ID. We must match the ID of nodes
+  // that were created without explicitly specifying flags. This also saves time
+  // and allows a gradual increase in API usage of the optional optimization
+  // flags.
+  if (RawFlags != 0)
+    ID.AddInteger(RawFlags);
 }
 
-/// AddBinaryNodeIDCustom - Add BinarySDNodes special infos
-static void AddBinaryNodeIDCustom(FoldingSetNodeID &ID, unsigned Opcode,
-                                  bool nuw, bool nsw, bool exact) {
-  if (isBinOpWithFlags(Opcode))
-    AddBinaryNodeIDCustom(ID, nuw, nsw, exact);
+static void AddNodeIDFlags(FoldingSetNodeID &ID, const SDNode *N) {
+  if (auto *Node = dyn_cast<BinaryWithFlagsSDNode>(N))
+    AddNodeIDFlags(ID, Node->getOpcode(), &Node->Flags);
 }
 
 static void AddNodeIDNode(FoldingSetNodeID &ID, unsigned short OpC,
@@ -421,12 +424,12 @@ static void AddNodeIDNode(FoldingSetNodeID &ID, unsigned short OpC,
   AddNodeIDOperands(ID, OpList);
 }
 
-/// AddNodeIDCustom - If this is an SDNode with special info, add this info to
-/// the NodeID data.
+/// If this is an SDNode with special info, add this info to the NodeID data.
 static void AddNodeIDCustom(FoldingSetNodeID &ID, const SDNode *N) {
   switch (N->getOpcode()) {
   case ISD::TargetExternalSymbol:
   case ISD::ExternalSymbol:
+  case ISD::MCSymbol:
     llvm_unreachable("Should only be used on nodes with operands");
   default: break;  // Normal nodes don't need extra info.
   case ISD::TargetConstant:
@@ -506,19 +509,6 @@ static void AddNodeIDCustom(FoldingSetNodeID &ID, const SDNode *N) {
     ID.AddInteger(ST->getPointerInfo().getAddrSpace());
     break;
   }
-  case ISD::SDIV:
-  case ISD::UDIV:
-  case ISD::SRA:
-  case ISD::SRL:
-  case ISD::MUL:
-  case ISD::ADD:
-  case ISD::SUB:
-  case ISD::SHL: {
-    const BinaryWithFlagsSDNode *BinNode = cast<BinaryWithFlagsSDNode>(N);
-    AddBinaryNodeIDCustom(ID, N->getOpcode(), BinNode->hasNoUnsignedWrap(),
-                          BinNode->hasNoSignedWrap(), BinNode->isExact());
-    break;
-  }
   case ISD::ATOMIC_CMP_SWAP:
   case ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS:
   case ISD::ATOMIC_SWAP:
@@ -562,6 +552,8 @@ static void AddNodeIDCustom(FoldingSetNodeID &ID, const SDNode *N) {
   }
   } // end switch (N->getOpcode())
 
+  AddNodeIDFlags(ID, N);
+
   // Target specific memory nodes could also have address spaces to check.
   if (N->isTargetMemoryOpcode())
     ID.AddInteger(cast<MemSDNode>(N)->getPointerInfo().getAddrSpace());
@@ -632,9 +624,9 @@ void SelectionDAG::RemoveDeadNodes() {
   SmallVector<SDNode*, 128> DeadNodes;
 
   // Add all obviously-dead nodes to the DeadNodes worklist.
-  for (allnodes_iterator I = allnodes_begin(), E = allnodes_end(); I != E; ++I)
-    if (I->use_empty())
-      DeadNodes.push_back(I);
+  for (SDNode &Node : allnodes())
+    if (Node.use_empty())
+      DeadNodes.push_back(&Node);
 
   RemoveDeadNodes(DeadNodes);
 
@@ -802,6 +794,11 @@ bool SelectionDAG::RemoveNodeFromCSEMaps(SDNode *N) {
                                                     ESN->getTargetFlags()));
     break;
   }
+  case ISD::MCSymbol: {
+    auto *MCSN = cast<MCSymbolSDNode>(N);
+    Erased = MCSymbols.erase(MCSN->getMCSymbol());
+    break;
+  }
   case ISD::VALUETYPE: {
     EVT VT = cast<VTSDNode>(N)->getVT();
     if (VT.isExtended()) {
@@ -876,7 +873,7 @@ SDNode *SelectionDAG::FindModifiedNodeSlot(SDNode *N, SDValue Op,
   FoldingSetNodeID ID;
   AddNodeIDNode(ID, N->getOpcode(), N->getVTList(), Ops);
   AddNodeIDCustom(ID, N);
-  SDNode *Node = CSEMap.FindNodeOrInsertPos(ID, InsertPos);
+  SDNode *Node = FindNodeOrInsertPos(ID, N->getDebugLoc(), InsertPos);
   return Node;
 }
 
@@ -894,7 +891,7 @@ SDNode *SelectionDAG::FindModifiedNodeSlot(SDNode *N,
   FoldingSetNodeID ID;
   AddNodeIDNode(ID, N->getOpcode(), N->getVTList(), Ops);
   AddNodeIDCustom(ID, N);
-  SDNode *Node = CSEMap.FindNodeOrInsertPos(ID, InsertPos);
+  SDNode *Node = FindNodeOrInsertPos(ID, N->getDebugLoc(), InsertPos);
   return Node;
 }
 
@@ -911,7 +908,7 @@ SDNode *SelectionDAG::FindModifiedNodeSlot(SDNode *N, ArrayRef<SDValue> Ops,
   FoldingSetNodeID ID;
   AddNodeIDNode(ID, N->getOpcode(), N->getVTList(), Ops);
   AddNodeIDCustom(ID, N);
-  SDNode *Node = CSEMap.FindNodeOrInsertPos(ID, InsertPos);
+  SDNode *Node = FindNodeOrInsertPos(ID, N->getDebugLoc(), InsertPos);
   return Node;
 }
 
@@ -923,7 +920,7 @@ unsigned SelectionDAG::getEVTAlignment(EVT VT) const {
                    PointerType::get(Type::getInt8Ty(*getContext()), 0) :
                    VT.getTypeForEVT(*getContext());
 
-  return TLI->getDataLayout()->getABITypeAlignment(Ty);
+  return getDataLayout().getABITypeAlignment(Ty);
 }
 
 // EntryNode could meaningfully have debug info if we can find it...
@@ -958,14 +955,16 @@ void SelectionDAG::allnodes_clear() {
 
 BinarySDNode *SelectionDAG::GetBinarySDNode(unsigned Opcode, SDLoc DL,
                                             SDVTList VTs, SDValue N1,
-                                            SDValue N2, bool nuw, bool nsw,
-                                            bool exact) {
+                                            SDValue N2,
+                                            const SDNodeFlags *Flags) {
   if (isBinOpWithFlags(Opcode)) {
+    // If no flags were passed in, use a default flags object.
+    SDNodeFlags F;
+    if (Flags == nullptr)
+      Flags = &F;
+
     BinaryWithFlagsSDNode *FN = new (NodeAllocator) BinaryWithFlagsSDNode(
-        Opcode, DL.getIROrder(), DL.getDebugLoc(), VTs, N1, N2);
-    FN->setHasNoUnsignedWrap(nuw);
-    FN->setHasNoSignedWrap(nsw);
-    FN->setIsExact(exact);
+        Opcode, DL.getIROrder(), DL.getDebugLoc(), VTs, N1, N2, *Flags);
 
     return FN;
   }
@@ -975,6 +974,40 @@ BinarySDNode *SelectionDAG::GetBinarySDNode(unsigned Opcode, SDLoc DL,
   return N;
 }
 
+SDNode *SelectionDAG::FindNodeOrInsertPos(const FoldingSetNodeID &ID,
+                                          void *&InsertPos) {
+  SDNode *N = CSEMap.FindNodeOrInsertPos(ID, InsertPos);
+  if (N) {
+    switch (N->getOpcode()) {
+    default: break;
+    case ISD::Constant:
+    case ISD::ConstantFP:
+      llvm_unreachable("Querying for Constant and ConstantFP nodes requires "
+                       "debug location.  Use another overload.");
+    }
+  }
+  return N;
+}
+
+SDNode *SelectionDAG::FindNodeOrInsertPos(const FoldingSetNodeID &ID,
+                                          DebugLoc DL, void *&InsertPos) {
+  SDNode *N = CSEMap.FindNodeOrInsertPos(ID, InsertPos);
+  if (N) {
+    switch (N->getOpcode()) {
+    default: break; // Process only regular (non-target) constant nodes.
+    case ISD::Constant:
+    case ISD::ConstantFP:
+      // Erase debug location from the node if the node is used at several
+      // different places to do not propagate one location to all uses as it
+      // leads to incorrect debug info.
+      if (N->getDebugLoc() != DL)
+        N->setDebugLoc(DebugLoc());
+      break;
+    }
+  }
+  return N;
+}
+
 void SelectionDAG::clear() {
   allnodes_clear();
   OperandAllocator.Reset();
@@ -983,6 +1016,7 @@ void SelectionDAG::clear() {
   ExtendedValueTypeNodes.clear();
   ExternalSymbols.clear();
   TargetExternalSymbols.clear();
+  MCSymbols.clear();
   std::fill(CondCodeNodes.begin(), CondCodeNodes.end(),
             static_cast<CondCodeSDNode*>(nullptr));
   std::fill(ValueTypeNodes.begin(), ValueTypeNodes.end(),
@@ -1030,7 +1064,7 @@ SDValue SelectionDAG::getZeroExtendInReg(SDValue Op, SDLoc DL, EVT VT) {
   APInt Imm = APInt::getLowBitsSet(BitWidth,
                                    VT.getSizeInBits());
   return getNode(ISD::AND, DL, Op.getValueType(), Op,
-                 getConstant(Imm, Op.getValueType()));
+                 getConstant(Imm, DL, Op.getValueType()));
 }
 
 SDValue SelectionDAG::getAnyExtendVectorInReg(SDValue Op, SDLoc DL, EVT VT) {
@@ -1068,7 +1102,7 @@ SDValue SelectionDAG::getZeroExtendVectorInReg(SDValue Op, SDLoc DL, EVT VT) {
 SDValue SelectionDAG::getNOT(SDLoc DL, SDValue Val, EVT VT) {
   EVT EltVT = VT.getScalarType();
   SDValue NegOne =
-    getConstant(APInt::getAllOnesValue(EltVT.getSizeInBits()), VT);
+    getConstant(APInt::getAllOnesValue(EltVT.getSizeInBits()), DL, VT);
   return getNode(ISD::XOR, DL, VT, Val, NegOne);
 }
 
@@ -1078,31 +1112,33 @@ SDValue SelectionDAG::getLogicalNOT(SDLoc DL, SDValue Val, EVT VT) {
   switch (TLI->getBooleanContents(VT)) {
     case TargetLowering::ZeroOrOneBooleanContent:
     case TargetLowering::UndefinedBooleanContent:
-      TrueValue = getConstant(1, VT);
+      TrueValue = getConstant(1, DL, VT);
       break;
     case TargetLowering::ZeroOrNegativeOneBooleanContent:
-      TrueValue = getConstant(APInt::getAllOnesValue(EltVT.getSizeInBits()),
+      TrueValue = getConstant(APInt::getAllOnesValue(EltVT.getSizeInBits()), DL,
                               VT);
       break;
   }
   return getNode(ISD::XOR, DL, VT, Val, TrueValue);
 }
 
-SDValue SelectionDAG::getConstant(uint64_t Val, EVT VT, bool isT, bool isO) {
+SDValue SelectionDAG::getConstant(uint64_t Val, SDLoc DL, EVT VT, bool isT,
+                                  bool isO) {
   EVT EltVT = VT.getScalarType();
   assert((EltVT.getSizeInBits() >= 64 ||
          (uint64_t)((int64_t)Val >> EltVT.getSizeInBits()) + 1 < 2) &&
          "getConstant with a uint64_t value that doesn't fit in the type!");
-  return getConstant(APInt(EltVT.getSizeInBits(), Val), VT, isT, isO);
+  return getConstant(APInt(EltVT.getSizeInBits(), Val), DL, VT, isT, isO);
 }
 
-SDValue SelectionDAG::getConstant(const APInt &Val, EVT VT, bool isT, bool isO)
+SDValue SelectionDAG::getConstant(const APInt &Val, SDLoc DL, EVT VT, bool isT,
+                                  bool isO)
 {
-  return getConstant(*ConstantInt::get(*Context, Val), VT, isT, isO);
+  return getConstant(*ConstantInt::get(*Context, Val), DL, VT, isT, isO);
 }
 
-SDValue SelectionDAG::getConstant(const ConstantInt &Val, EVT VT, bool isT,
-                                  bool isO) {
+SDValue SelectionDAG::getConstant(const ConstantInt &Val, SDLoc DL, EVT VT,
+                                  bool isT, bool isO) {
   assert(VT.isInteger() && "Cannot create FP integer constant!");
 
   EVT EltVT = VT.getScalarType();
@@ -1141,13 +1177,13 @@ SDValue SelectionDAG::getConstant(const ConstantInt &Val, EVT VT, bool isT,
     SmallVector<SDValue, 2> EltParts;
     for (unsigned i = 0; i < ViaVecNumElts / VT.getVectorNumElements(); ++i) {
       EltParts.push_back(getConstant(NewVal.lshr(i * ViaEltSizeInBits)
-                                           .trunc(ViaEltSizeInBits),
+                                           .trunc(ViaEltSizeInBits), DL,
                                      ViaEltVT, isT, isO));
     }
 
     // EltParts is currently in little endian order. If we actually want
     // big-endian order then reverse it now.
-    if (TLI->isBigEndian())
+    if (getDataLayout().isBigEndian())
       std::reverse(EltParts.begin(), EltParts.end());
 
     // The elements must be reversed when the element order is different
@@ -1176,12 +1212,13 @@ SDValue SelectionDAG::getConstant(const ConstantInt &Val, EVT VT, bool isT,
   ID.AddBoolean(isO);
   void *IP = nullptr;
   SDNode *N = nullptr;
-  if ((N = CSEMap.FindNodeOrInsertPos(ID, IP)))
+  if ((N = FindNodeOrInsertPos(ID, DL.getDebugLoc(), IP)))
     if (!VT.isVector())
       return SDValue(N, 0);
 
   if (!N) {
-    N = new (NodeAllocator) ConstantSDNode(isT, isO, Elt, EltVT);
+    N = new (NodeAllocator) ConstantSDNode(isT, isO, Elt, DL.getDebugLoc(),
+                                           EltVT);
     CSEMap.InsertNode(N, IP);
     InsertNode(N);
   }
@@ -1195,16 +1232,17 @@ SDValue SelectionDAG::getConstant(const ConstantInt &Val, EVT VT, bool isT,
   return Result;
 }
 
-SDValue SelectionDAG::getIntPtrConstant(uint64_t Val, bool isTarget) {
-  return getConstant(Val, TLI->getPointerTy(), isTarget);
+SDValue SelectionDAG::getIntPtrConstant(uint64_t Val, SDLoc DL, bool isTarget) {
+  return getConstant(Val, DL, TLI->getPointerTy(getDataLayout()), isTarget);
 }
 
-
-SDValue SelectionDAG::getConstantFP(const APFloat& V, EVT VT, bool isTarget) {
-  return getConstantFP(*ConstantFP::get(*getContext(), V), VT, isTarget);
+SDValue SelectionDAG::getConstantFP(const APFloat& V, SDLoc DL, EVT VT,
+                                    bool isTarget) {
+  return getConstantFP(*ConstantFP::get(*getContext(), V), DL, VT, isTarget);
 }
 
-SDValue SelectionDAG::getConstantFP(const ConstantFP& V, EVT VT, bool isTarget){
+SDValue SelectionDAG::getConstantFP(const ConstantFP& V, SDLoc DL, EVT VT,
+                                    bool isTarget){
   assert(VT.isFloatingPoint() && "Cannot create integer FP constant!");
 
   EVT EltVT = VT.getScalarType();
@@ -1218,12 +1256,13 @@ SDValue SelectionDAG::getConstantFP(const ConstantFP& V, EVT VT, bool isTarget){
   ID.AddPointer(&V);
   void *IP = nullptr;
   SDNode *N = nullptr;
-  if ((N = CSEMap.FindNodeOrInsertPos(ID, IP)))
+  if ((N = FindNodeOrInsertPos(ID, DL.getDebugLoc(), IP)))
     if (!VT.isVector())
       return SDValue(N, 0);
 
   if (!N) {
-    N = new (NodeAllocator) ConstantFPSDNode(isTarget, &V, EltVT);
+    N = new (NodeAllocator) ConstantFPSDNode(isTarget, &V, DL.getDebugLoc(),
+                                             EltVT);
     CSEMap.InsertNode(N, IP);
     InsertNode(N);
   }
@@ -1232,25 +1271,25 @@ SDValue SelectionDAG::getConstantFP(const ConstantFP& V, EVT VT, bool isTarget){
   if (VT.isVector()) {
     SmallVector<SDValue, 8> Ops;
     Ops.assign(VT.getVectorNumElements(), Result);
-    // FIXME SDLoc info might be appropriate here
     Result = getNode(ISD::BUILD_VECTOR, SDLoc(), VT, Ops);
   }
   return Result;
 }
 
-SDValue SelectionDAG::getConstantFP(double Val, EVT VT, bool isTarget) {
+SDValue SelectionDAG::getConstantFP(double Val, SDLoc DL, EVT VT,
+                                    bool isTarget) {
   EVT EltVT = VT.getScalarType();
   if (EltVT==MVT::f32)
-    return getConstantFP(APFloat((float)Val), VT, isTarget);
+    return getConstantFP(APFloat((float)Val), DL, VT, isTarget);
   else if (EltVT==MVT::f64)
-    return getConstantFP(APFloat(Val), VT, isTarget);
+    return getConstantFP(APFloat(Val), DL, VT, isTarget);
   else if (EltVT==MVT::f80 || EltVT==MVT::f128 || EltVT==MVT::ppcf128 ||
            EltVT==MVT::f16) {
     bool ignored;
     APFloat apf = APFloat(Val);
     apf.convert(EVTToAPFloatSemantics(EltVT), APFloat::rmNearestTiesToEven,
                 &ignored);
-    return getConstantFP(apf, VT, isTarget);
+    return getConstantFP(apf, DL, VT, isTarget);
   } else
     llvm_unreachable("Unsupported type in getConstantFP");
 }
@@ -1263,7 +1302,7 @@ SDValue SelectionDAG::getGlobalAddress(const GlobalValue *GV, SDLoc DL,
          "Cannot set target flags on target-independent globals");
 
   // Truncate (with sign-extension) the offset value to the pointer size.
-  unsigned BitWidth = TLI->getPointerTypeSizeInBits(GV->getType());
+  unsigned BitWidth = getDataLayout().getPointerTypeSizeInBits(GV->getType());
   if (BitWidth < 64)
     Offset = SignExtend64(Offset, BitWidth);
 
@@ -1280,7 +1319,7 @@ SDValue SelectionDAG::getGlobalAddress(const GlobalValue *GV, SDLoc DL,
   ID.AddInteger(TargetFlags);
   ID.AddInteger(GV->getType()->getAddressSpace());
   void *IP = nullptr;
-  if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP))
+  if (SDNode *E = FindNodeOrInsertPos(ID, DL.getDebugLoc(), IP))
     return SDValue(E, 0);
 
   SDNode *N = new (NodeAllocator) GlobalAddressSDNode(Opc, DL.getIROrder(),
@@ -1297,7 +1336,7 @@ SDValue SelectionDAG::getFrameIndex(int FI, EVT VT, bool isTarget) {
   AddNodeIDNode(ID, Opc, getVTList(VT), None);
   ID.AddInteger(FI);
   void *IP = nullptr;
-  if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP))
+  if (SDNode *E = FindNodeOrInsertPos(ID, IP))
     return SDValue(E, 0);
 
   SDNode *N = new (NodeAllocator) FrameIndexSDNode(FI, VT, isTarget);
@@ -1316,7 +1355,7 @@ SDValue SelectionDAG::getJumpTable(int JTI, EVT VT, bool isTarget,
   ID.AddInteger(JTI);
   ID.AddInteger(TargetFlags);
   void *IP = nullptr;
-  if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP))
+  if (SDNode *E = FindNodeOrInsertPos(ID, IP))
     return SDValue(E, 0);
 
   SDNode *N = new (NodeAllocator) JumpTableSDNode(JTI, VT, isTarget,
@@ -1333,7 +1372,7 @@ SDValue SelectionDAG::getConstantPool(const Constant *C, EVT VT,
   assert((TargetFlags == 0 || isTarget) &&
          "Cannot set target flags on target-independent globals");
   if (Alignment == 0)
-    Alignment = TLI->getDataLayout()->getPrefTypeAlignment(C->getType());
+    Alignment = getDataLayout().getPrefTypeAlignment(C->getType());
   unsigned Opc = isTarget ? ISD::TargetConstantPool : ISD::ConstantPool;
   FoldingSetNodeID ID;
   AddNodeIDNode(ID, Opc, getVTList(VT), None);
@@ -1342,7 +1381,7 @@ SDValue SelectionDAG::getConstantPool(const Constant *C, EVT VT,
   ID.AddPointer(C);
   ID.AddInteger(TargetFlags);
   void *IP = nullptr;
-  if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP))
+  if (SDNode *E = FindNodeOrInsertPos(ID, IP))
     return SDValue(E, 0);
 
   SDNode *N = new (NodeAllocator) ConstantPoolSDNode(isTarget, C, VT, Offset,
@@ -1360,7 +1399,7 @@ SDValue SelectionDAG::getConstantPool(MachineConstantPoolValue *C, EVT VT,
   assert((TargetFlags == 0 || isTarget) &&
          "Cannot set target flags on target-independent globals");
   if (Alignment == 0)
-    Alignment = TLI->getDataLayout()->getPrefTypeAlignment(C->getType());
+    Alignment = getDataLayout().getPrefTypeAlignment(C->getType());
   unsigned Opc = isTarget ? ISD::TargetConstantPool : ISD::ConstantPool;
   FoldingSetNodeID ID;
   AddNodeIDNode(ID, Opc, getVTList(VT), None);
@@ -1369,7 +1408,7 @@ SDValue SelectionDAG::getConstantPool(MachineConstantPoolValue *C, EVT VT,
   C->addSelectionDAGCSEId(ID);
   ID.AddInteger(TargetFlags);
   void *IP = nullptr;
-  if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP))
+  if (SDNode *E = FindNodeOrInsertPos(ID, IP))
     return SDValue(E, 0);
 
   SDNode *N = new (NodeAllocator) ConstantPoolSDNode(isTarget, C, VT, Offset,
@@ -1387,7 +1426,7 @@ SDValue SelectionDAG::getTargetIndex(int Index, EVT VT, int64_t Offset,
   ID.AddInteger(Offset);
   ID.AddInteger(TargetFlags);
   void *IP = nullptr;
-  if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP))
+  if (SDNode *E = FindNodeOrInsertPos(ID, IP))
     return SDValue(E, 0);
 
   SDNode *N = new (NodeAllocator) TargetIndexSDNode(Index, VT, Offset,
@@ -1402,7 +1441,7 @@ SDValue SelectionDAG::getBasicBlock(MachineBasicBlock *MBB) {
   AddNodeIDNode(ID, ISD::BasicBlock, getVTList(MVT::Other), None);
   ID.AddPointer(MBB);
   void *IP = nullptr;
-  if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP))
+  if (SDNode *E = FindNodeOrInsertPos(ID, IP))
     return SDValue(E, 0);
 
   SDNode *N = new (NodeAllocator) BasicBlockSDNode(MBB);
@@ -1433,6 +1472,15 @@ SDValue SelectionDAG::getExternalSymbol(const char *Sym, EVT VT) {
   return SDValue(N, 0);
 }
 
+SDValue SelectionDAG::getMCSymbol(MCSymbol *Sym, EVT VT) {
+  SDNode *&N = MCSymbols[Sym];
+  if (N)
+    return SDValue(N, 0);
+  N = new (NodeAllocator) MCSymbolSDNode(Sym, VT);
+  InsertNode(N);
+  return SDValue(N, 0);
+}
+
 SDValue SelectionDAG::getTargetExternalSymbol(const char *Sym, EVT VT,
                                               unsigned char TargetFlags) {
   SDNode *&N =
@@ -1615,7 +1663,7 @@ SDValue SelectionDAG::getVectorShuffle(EVT VT, SDLoc dl, SDValue N1,
     ID.AddInteger(MaskVec[i]);
 
   void* IP = nullptr;
-  if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP))
+  if (SDNode *E = FindNodeOrInsertPos(ID, dl.getDebugLoc(), IP))
     return SDValue(E, 0);
 
   // Allocate the mask array for the node out of the BumpPtrAllocator, since
@@ -1657,7 +1705,7 @@ SDValue SelectionDAG::getConvertRndSat(EVT VT, SDLoc dl,
   SDValue Ops[] = { Val, DTy, STy, Rnd, Sat };
   AddNodeIDNode(ID, ISD::CONVERT_RNDSAT, getVTList(VT), Ops);
   void* IP = nullptr;
-  if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP))
+  if (SDNode *E = FindNodeOrInsertPos(ID, dl.getDebugLoc(), IP))
     return SDValue(E, 0);
 
   CvtRndSatSDNode *N = new (NodeAllocator) CvtRndSatSDNode(VT, dl.getIROrder(),
@@ -1673,7 +1721,7 @@ SDValue SelectionDAG::getRegister(unsigned RegNo, EVT VT) {
   AddNodeIDNode(ID, ISD::Register, getVTList(VT), None);
   ID.AddInteger(RegNo);
   void *IP = nullptr;
-  if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP))
+  if (SDNode *E = FindNodeOrInsertPos(ID, IP))
     return SDValue(E, 0);
 
   SDNode *N = new (NodeAllocator) RegisterSDNode(RegNo, VT);
@@ -1687,7 +1735,7 @@ SDValue SelectionDAG::getRegisterMask(const uint32_t *RegMask) {
   AddNodeIDNode(ID, ISD::RegisterMask, getVTList(MVT::Untyped), None);
   ID.AddPointer(RegMask);
   void *IP = nullptr;
-  if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP))
+  if (SDNode *E = FindNodeOrInsertPos(ID, IP))
     return SDValue(E, 0);
 
   SDNode *N = new (NodeAllocator) RegisterMaskSDNode(RegMask);
@@ -1702,7 +1750,7 @@ SDValue SelectionDAG::getEHLabel(SDLoc dl, SDValue Root, MCSymbol *Label) {
   AddNodeIDNode(ID, ISD::EH_LABEL, getVTList(MVT::Other), Ops);
   ID.AddPointer(Label);
   void *IP = nullptr;
-  if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP))
+  if (SDNode *E = FindNodeOrInsertPos(ID, IP))
     return SDValue(E, 0);
 
   SDNode *N = new (NodeAllocator) EHLabelSDNode(dl.getIROrder(),
@@ -1725,7 +1773,7 @@ SDValue SelectionDAG::getBlockAddress(const BlockAddress *BA, EVT VT,
   ID.AddInteger(Offset);
   ID.AddInteger(TargetFlags);
   void *IP = nullptr;
-  if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP))
+  if (SDNode *E = FindNodeOrInsertPos(ID, IP))
     return SDValue(E, 0);
 
   SDNode *N = new (NodeAllocator) BlockAddressSDNode(Opc, VT, BA, Offset,
@@ -1744,7 +1792,7 @@ SDValue SelectionDAG::getSrcValue(const Value *V) {
   ID.AddPointer(V);
 
   void *IP = nullptr;
-  if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP))
+  if (SDNode *E = FindNodeOrInsertPos(ID, IP))
     return SDValue(E, 0);
 
   SDNode *N = new (NodeAllocator) SrcValueSDNode(V);
@@ -1760,7 +1808,7 @@ SDValue SelectionDAG::getMDNode(const MDNode *MD) {
   ID.AddPointer(MD);
 
   void *IP = nullptr;
-  if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP))
+  if (SDNode *E = FindNodeOrInsertPos(ID, IP))
     return SDValue(E, 0);
 
   SDNode *N = new (NodeAllocator) MDNodeSDNode(MD);
@@ -1769,6 +1817,13 @@ SDValue SelectionDAG::getMDNode(const MDNode *MD) {
   return SDValue(N, 0);
 }
 
+SDValue SelectionDAG::getBitcast(EVT VT, SDValue V) {
+  if (VT == V.getValueType())
+    return V;
+
+  return getNode(ISD::BITCAST, SDLoc(V), VT, V);
+}
+
 /// getAddrSpaceCast - Return an AddrSpaceCastSDNode.
 SDValue SelectionDAG::getAddrSpaceCast(SDLoc dl, EVT VT, SDValue Ptr,
                                        unsigned SrcAS, unsigned DestAS) {
@@ -1779,7 +1834,7 @@ SDValue SelectionDAG::getAddrSpaceCast(SDLoc dl, EVT VT, SDValue Ptr,
   ID.AddInteger(DestAS);
 
   void *IP = nullptr;
-  if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP))
+  if (SDNode *E = FindNodeOrInsertPos(ID, dl.getDebugLoc(), IP))
     return SDValue(E, 0);
 
   SDNode *N = new (NodeAllocator) AddrSpaceCastSDNode(dl.getIROrder(),
@@ -1794,11 +1849,10 @@ SDValue SelectionDAG::getAddrSpaceCast(SDLoc dl, EVT VT, SDValue Ptr,
 /// the target's desired shift amount type.
 SDValue SelectionDAG::getShiftAmountOperand(EVT LHSTy, SDValue Op) {
   EVT OpTy = Op.getValueType();
-  EVT ShTy = TLI->getShiftAmountTy(LHSTy);
+  EVT ShTy = TLI->getShiftAmountTy(LHSTy, getDataLayout());
   if (OpTy == ShTy || OpTy.isVector()) return Op;
 
-  ISD::NodeType Opcode = OpTy.bitsGT(ShTy) ?  ISD::TRUNCATE : ISD::ZERO_EXTEND;
-  return getNode(Opcode, SDLoc(Op), ShTy, Op);
+  return getZExtOrTrunc(Op, SDLoc(Op), ShTy);
 }
 
 /// CreateStackTemporary - Create a stack temporary, suitable for holding the
@@ -1808,26 +1862,25 @@ SDValue SelectionDAG::CreateStackTemporary(EVT VT, unsigned minAlign) {
   unsigned ByteSize = VT.getStoreSize();
   Type *Ty = VT.getTypeForEVT(*getContext());
   unsigned StackAlign =
-  std::max((unsigned)TLI->getDataLayout()->getPrefTypeAlignment(Ty), minAlign);
+      std::max((unsigned)getDataLayout().getPrefTypeAlignment(Ty), minAlign);
 
   int FrameIdx = FrameInfo->CreateStackObject(ByteSize, StackAlign, false);
-  return getFrameIndex(FrameIdx, TLI->getPointerTy());
+  return getFrameIndex(FrameIdx, TLI->getPointerTy(getDataLayout()));
 }
 
 /// CreateStackTemporary - Create a stack temporary suitable for holding
 /// either of the specified value types.
 SDValue SelectionDAG::CreateStackTemporary(EVT VT1, EVT VT2) {
-  unsigned Bytes = std::max(VT1.getStoreSizeInBits(),
-                            VT2.getStoreSizeInBits())/8;
+  unsigned Bytes = std::max(VT1.getStoreSize(), VT2.getStoreSize());
   Type *Ty1 = VT1.getTypeForEVT(*getContext());
   Type *Ty2 = VT2.getTypeForEVT(*getContext());
-  const DataLayout *TD = TLI->getDataLayout();
-  unsigned Align = std::max(TD->getPrefTypeAlignment(Ty1),
-                            TD->getPrefTypeAlignment(Ty2));
+  const DataLayout &DL = getDataLayout();
+  unsigned Align =
+      std::max(DL.getPrefTypeAlignment(Ty1), DL.getPrefTypeAlignment(Ty2));
 
   MachineFrameInfo *FrameInfo = getMachineFunction().getFrameInfo();
   int FrameIdx = FrameInfo->CreateStackObject(Bytes, Align, false);
-  return getFrameIndex(FrameIdx, TLI->getPointerTy());
+  return getFrameIndex(FrameIdx, TLI->getPointerTy(getDataLayout()));
 }
 
 SDValue SelectionDAG::FoldSetCC(EVT VT, SDValue N1,
@@ -1836,13 +1889,14 @@ SDValue SelectionDAG::FoldSetCC(EVT VT, SDValue N1,
   switch (Cond) {
   default: break;
   case ISD::SETFALSE:
-  case ISD::SETFALSE2: return getConstant(0, VT);
+  case ISD::SETFALSE2: return getConstant(0, dl, VT);
   case ISD::SETTRUE:
   case ISD::SETTRUE2: {
     TargetLowering::BooleanContent Cnt =
         TLI->getBooleanContents(N1->getValueType(0));
     return getConstant(
-        Cnt == TargetLowering::ZeroOrNegativeOneBooleanContent ? -1ULL : 1, VT);
+        Cnt == TargetLowering::ZeroOrNegativeOneBooleanContent ? -1ULL : 1, dl,
+        VT);
   }
 
   case ISD::SETOEQ:
@@ -1859,69 +1913,69 @@ SDValue SelectionDAG::FoldSetCC(EVT VT, SDValue N1,
     break;
   }
 
-  if (ConstantSDNode *N2C = dyn_cast<ConstantSDNode>(N2.getNode())) {
+  if (ConstantSDNode *N2C = dyn_cast<ConstantSDNode>(N2)) {
     const APInt &C2 = N2C->getAPIntValue();
-    if (ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(N1.getNode())) {
+    if (ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(N1)) {
       const APInt &C1 = N1C->getAPIntValue();
 
       switch (Cond) {
       default: llvm_unreachable("Unknown integer setcc!");
-      case ISD::SETEQ:  return getConstant(C1 == C2, VT);
-      case ISD::SETNE:  return getConstant(C1 != C2, VT);
-      case ISD::SETULT: return getConstant(C1.ult(C2), VT);
-      case ISD::SETUGT: return getConstant(C1.ugt(C2), VT);
-      case ISD::SETULE: return getConstant(C1.ule(C2), VT);
-      case ISD::SETUGE: return getConstant(C1.uge(C2), VT);
-      case ISD::SETLT:  return getConstant(C1.slt(C2), VT);
-      case ISD::SETGT:  return getConstant(C1.sgt(C2), VT);
-      case ISD::SETLE:  return getConstant(C1.sle(C2), VT);
-      case ISD::SETGE:  return getConstant(C1.sge(C2), VT);
+      case ISD::SETEQ:  return getConstant(C1 == C2, dl, VT);
+      case ISD::SETNE:  return getConstant(C1 != C2, dl, VT);
+      case ISD::SETULT: return getConstant(C1.ult(C2), dl, VT);
+      case ISD::SETUGT: return getConstant(C1.ugt(C2), dl, VT);
+      case ISD::SETULE: return getConstant(C1.ule(C2), dl, VT);
+      case ISD::SETUGE: return getConstant(C1.uge(C2), dl, VT);
+      case ISD::SETLT:  return getConstant(C1.slt(C2), dl, VT);
+      case ISD::SETGT:  return getConstant(C1.sgt(C2), dl, VT);
+      case ISD::SETLE:  return getConstant(C1.sle(C2), dl, VT);
+      case ISD::SETGE:  return getConstant(C1.sge(C2), dl, VT);
       }
     }
   }
-  if (ConstantFPSDNode *N1C = dyn_cast<ConstantFPSDNode>(N1.getNode())) {
-    if (ConstantFPSDNode *N2C = dyn_cast<ConstantFPSDNode>(N2.getNode())) {
+  if (ConstantFPSDNode *N1C = dyn_cast<ConstantFPSDNode>(N1)) {
+    if (ConstantFPSDNode *N2C = dyn_cast<ConstantFPSDNode>(N2)) {
       APFloat::cmpResult R = N1C->getValueAPF().compare(N2C->getValueAPF());
       switch (Cond) {
       default: break;
       case ISD::SETEQ:  if (R==APFloat::cmpUnordered)
                           return getUNDEF(VT);
                         // fall through
-      case ISD::SETOEQ: return getConstant(R==APFloat::cmpEqual, VT);
+      case ISD::SETOEQ: return getConstant(R==APFloat::cmpEqual, dl, VT);
       case ISD::SETNE:  if (R==APFloat::cmpUnordered)
                           return getUNDEF(VT);
                         // fall through
       case ISD::SETONE: return getConstant(R==APFloat::cmpGreaterThan ||
-                                           R==APFloat::cmpLessThan, VT);
+                                           R==APFloat::cmpLessThan, dl, VT);
       case ISD::SETLT:  if (R==APFloat::cmpUnordered)
                           return getUNDEF(VT);
                         // fall through
-      case ISD::SETOLT: return getConstant(R==APFloat::cmpLessThan, VT);
+      case ISD::SETOLT: return getConstant(R==APFloat::cmpLessThan, dl, VT);
       case ISD::SETGT:  if (R==APFloat::cmpUnordered)
                           return getUNDEF(VT);
                         // fall through
-      case ISD::SETOGT: return getConstant(R==APFloat::cmpGreaterThan, VT);
+      case ISD::SETOGT: return getConstant(R==APFloat::cmpGreaterThan, dl, VT);
       case ISD::SETLE:  if (R==APFloat::cmpUnordered)
                           return getUNDEF(VT);
                         // fall through
       case ISD::SETOLE: return getConstant(R==APFloat::cmpLessThan ||
-                                           R==APFloat::cmpEqual, VT);
+                                           R==APFloat::cmpEqual, dl, VT);
       case ISD::SETGE:  if (R==APFloat::cmpUnordered)
                           return getUNDEF(VT);
                         // fall through
       case ISD::SETOGE: return getConstant(R==APFloat::cmpGreaterThan ||
-                                           R==APFloat::cmpEqual, VT);
-      case ISD::SETO:   return getConstant(R!=APFloat::cmpUnordered, VT);
-      case ISD::SETUO:  return getConstant(R==APFloat::cmpUnordered, VT);
+                                           R==APFloat::cmpEqual, dl, VT);
+      case ISD::SETO:   return getConstant(R!=APFloat::cmpUnordered, dl, VT);
+      case ISD::SETUO:  return getConstant(R==APFloat::cmpUnordered, dl, VT);
       case ISD::SETUEQ: return getConstant(R==APFloat::cmpUnordered ||
-                                           R==APFloat::cmpEqual, VT);
-      case ISD::SETUNE: return getConstant(R!=APFloat::cmpEqual, VT);
+                                           R==APFloat::cmpEqual, dl, VT);
+      case ISD::SETUNE: return getConstant(R!=APFloat::cmpEqual, dl, VT);
       case ISD::SETULT: return getConstant(R==APFloat::cmpUnordered ||
-                                           R==APFloat::cmpLessThan, VT);
+                                           R==APFloat::cmpLessThan, dl, VT);
       case ISD::SETUGT: return getConstant(R==APFloat::cmpGreaterThan ||
-                                           R==APFloat::cmpUnordered, VT);
-      case ISD::SETULE: return getConstant(R!=APFloat::cmpGreaterThan, VT);
-      case ISD::SETUGE: return getConstant(R!=APFloat::cmpLessThan, VT);
+                                           R==APFloat::cmpUnordered, dl, VT);
+      case ISD::SETULE: return getConstant(R!=APFloat::cmpGreaterThan, dl, VT);
+      case ISD::SETUGE: return getConstant(R!=APFloat::cmpLessThan, dl, VT);
       }
     } else {
       // Ensure that the constant occurs on the RHS.
@@ -2299,15 +2353,24 @@ void SelectionDAG::computeKnownBits(SDValue Op, APInt &KnownZero,
     // Output known-0 bits are known if clear or set in both the low clear bits
     // common to both LHS & RHS.  For example, 8+(X<<3) is known to have the
     // low 3 bits clear.
+    // Output known-0 bits are also known if the top bits of each input are
+    // known to be clear. For example, if one input has the top 10 bits clear
+    // and the other has the top 8 bits clear, we know the top 7 bits of the
+    // output must be clear.
     computeKnownBits(Op.getOperand(0), KnownZero2, KnownOne2, Depth+1);
-    unsigned KnownZeroOut = KnownZero2.countTrailingOnes();
+    unsigned KnownZeroHigh = KnownZero2.countLeadingOnes();
+    unsigned KnownZeroLow = KnownZero2.countTrailingOnes();
 
     computeKnownBits(Op.getOperand(1), KnownZero2, KnownOne2, Depth+1);
-    KnownZeroOut = std::min(KnownZeroOut,
+    KnownZeroHigh = std::min(KnownZeroHigh,
+                             KnownZero2.countLeadingOnes());
+    KnownZeroLow = std::min(KnownZeroLow,
                             KnownZero2.countTrailingOnes());
 
     if (Op.getOpcode() == ISD::ADD) {
-      KnownZero |= APInt::getLowBitsSet(BitWidth, KnownZeroOut);
+      KnownZero |= APInt::getLowBitsSet(BitWidth, KnownZeroLow);
+      if (KnownZeroHigh > 1)
+        KnownZero |= APInt::getHighBitsSet(BitWidth, KnownZeroHigh - 1);
       break;
     }
 
@@ -2315,8 +2378,8 @@ void SelectionDAG::computeKnownBits(SDValue Op, APInt &KnownZero,
     // information if we know (at least) that the low two bits are clear.  We
     // then return to the caller that the low bit is unknown but that other bits
     // are known zero.
-    if (KnownZeroOut >= 2) // ADDE
-      KnownZero |= APInt::getBitsSet(BitWidth, 1, KnownZeroOut);
+    if (KnownZeroLow >= 2) // ADDE
+      KnownZero |= APInt::getBitsSet(BitWidth, 1, KnownZeroLow);
     break;
   }
   case ISD::SREM:
@@ -2383,6 +2446,19 @@ void SelectionDAG::computeKnownBits(SDValue Op, APInt &KnownZero,
     KnownOne = KnownOne.trunc(BitWidth);
     break;
   }
+  case ISD::SMIN:
+  case ISD::SMAX:
+  case ISD::UMIN:
+  case ISD::UMAX: {
+    APInt Op0Zero, Op0One;
+    APInt Op1Zero, Op1One;
+    computeKnownBits(Op.getOperand(0), Op0Zero, Op0One, Depth);
+    computeKnownBits(Op.getOperand(1), Op1Zero, Op1One, Depth);
+
+    KnownZero = Op0Zero & Op1Zero;
+    KnownOne = Op0One & Op1One;
+    break;
+  }
   case ISD::FrameIndex:
   case ISD::TargetFrameIndex:
     if (unsigned Align = InferPtrAlignment(Op)) {
@@ -2486,7 +2562,15 @@ unsigned SelectionDAG::ComputeNumSignBits(SDValue Op, unsigned Depth) const{
     if (Tmp == 1) return 1;  // Early out.
     Tmp2 = ComputeNumSignBits(Op.getOperand(2), Depth+1);
     return std::min(Tmp, Tmp2);
-
+  case ISD::SMIN:
+  case ISD::SMAX:
+  case ISD::UMIN:
+  case ISD::UMAX:
+    Tmp = ComputeNumSignBits(Op.getOperand(0), Depth + 1);
+    if (Tmp == 1)
+      return 1;  // Early out.
+    Tmp2 = ComputeNumSignBits(Op.getOperand(1), Depth + 1);
+    return std::min(Tmp, Tmp2);
   case ISD::SADDO:
   case ISD::UADDO:
   case ISD::SSUBO:
@@ -2718,7 +2802,7 @@ SDValue SelectionDAG::getNode(unsigned Opcode, SDLoc DL, EVT VT) {
   FoldingSetNodeID ID;
   AddNodeIDNode(ID, Opcode, getVTList(VT), None);
   void *IP = nullptr;
-  if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP))
+  if (SDNode *E = FindNodeOrInsertPos(ID, DL.getDebugLoc(), IP))
     return SDValue(E, 0);
 
   SDNode *N = new (NodeAllocator) SDNode(Opcode, DL.getIROrder(),
@@ -2736,17 +2820,17 @@ SDValue SelectionDAG::getNode(unsigned Opcode, SDLoc DL,
   // doesn't create new constants with different values. Nevertheless, the
   // opaque flag is preserved during folding to prevent future folding with
   // other constants.
-  if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Operand.getNode())) {
+  if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Operand)) {
     const APInt &Val = C->getAPIntValue();
     switch (Opcode) {
     default: break;
     case ISD::SIGN_EXTEND:
-      return getConstant(Val.sextOrTrunc(VT.getSizeInBits()), VT,
+      return getConstant(Val.sextOrTrunc(VT.getSizeInBits()), DL, VT,
                          C->isTargetOpcode(), C->isOpaque());
     case ISD::ANY_EXTEND:
     case ISD::ZERO_EXTEND:
     case ISD::TRUNCATE:
-      return getConstant(Val.zextOrTrunc(VT.getSizeInBits()), VT,
+      return getConstant(Val.zextOrTrunc(VT.getSizeInBits()), DL, VT,
                          C->isTargetOpcode(), C->isOpaque());
     case ISD::UINT_TO_FP:
     case ISD::SINT_TO_FP: {
@@ -2755,59 +2839,59 @@ SDValue SelectionDAG::getNode(unsigned Opcode, SDLoc DL,
       (void)apf.convertFromAPInt(Val,
                                  Opcode==ISD::SINT_TO_FP,
                                  APFloat::rmNearestTiesToEven);
-      return getConstantFP(apf, VT);
+      return getConstantFP(apf, DL, VT);
     }
     case ISD::BITCAST:
       if (VT == MVT::f16 && C->getValueType(0) == MVT::i16)
-        return getConstantFP(APFloat(APFloat::IEEEhalf, Val), VT);
+        return getConstantFP(APFloat(APFloat::IEEEhalf, Val), DL, VT);
       if (VT == MVT::f32 && C->getValueType(0) == MVT::i32)
-        return getConstantFP(APFloat(APFloat::IEEEsingle, Val), VT);
+        return getConstantFP(APFloat(APFloat::IEEEsingle, Val), DL, VT);
       else if (VT == MVT::f64 && C->getValueType(0) == MVT::i64)
-        return getConstantFP(APFloat(APFloat::IEEEdouble, Val), VT);
+        return getConstantFP(APFloat(APFloat::IEEEdouble, Val), DL, VT);
       break;
     case ISD::BSWAP:
-      return getConstant(Val.byteSwap(), VT, C->isTargetOpcode(),
+      return getConstant(Val.byteSwap(), DL, VT, C->isTargetOpcode(),
                          C->isOpaque());
     case ISD::CTPOP:
-      return getConstant(Val.countPopulation(), VT, C->isTargetOpcode(),
+      return getConstant(Val.countPopulation(), DL, VT, C->isTargetOpcode(),
                          C->isOpaque());
     case ISD::CTLZ:
     case ISD::CTLZ_ZERO_UNDEF:
-      return getConstant(Val.countLeadingZeros(), VT, C->isTargetOpcode(),
+      return getConstant(Val.countLeadingZeros(), DL, VT, C->isTargetOpcode(),
                          C->isOpaque());
     case ISD::CTTZ:
     case ISD::CTTZ_ZERO_UNDEF:
-      return getConstant(Val.countTrailingZeros(), VT, C->isTargetOpcode(),
+      return getConstant(Val.countTrailingZeros(), DL, VT, C->isTargetOpcode(),
                          C->isOpaque());
     }
   }
 
   // Constant fold unary operations with a floating point constant operand.
-  if (ConstantFPSDNode *C = dyn_cast<ConstantFPSDNode>(Operand.getNode())) {
+  if (ConstantFPSDNode *C = dyn_cast<ConstantFPSDNode>(Operand)) {
     APFloat V = C->getValueAPF();    // make copy
     switch (Opcode) {
     case ISD::FNEG:
       V.changeSign();
-      return getConstantFP(V, VT);
+      return getConstantFP(V, DL, VT);
     case ISD::FABS:
       V.clearSign();
-      return getConstantFP(V, VT);
+      return getConstantFP(V, DL, VT);
     case ISD::FCEIL: {
       APFloat::opStatus fs = V.roundToIntegral(APFloat::rmTowardPositive);
       if (fs == APFloat::opOK || fs == APFloat::opInexact)
-        return getConstantFP(V, VT);
+        return getConstantFP(V, DL, VT);
       break;
     }
     case ISD::FTRUNC: {
       APFloat::opStatus fs = V.roundToIntegral(APFloat::rmTowardZero);
       if (fs == APFloat::opOK || fs == APFloat::opInexact)
-        return getConstantFP(V, VT);
+        return getConstantFP(V, DL, VT);
       break;
     }
     case ISD::FFLOOR: {
       APFloat::opStatus fs = V.roundToIntegral(APFloat::rmTowardNegative);
       if (fs == APFloat::opOK || fs == APFloat::opInexact)
-        return getConstantFP(V, VT);
+        return getConstantFP(V, DL, VT);
       break;
     }
     case ISD::FP_EXTEND: {
@@ -2816,7 +2900,7 @@ SDValue SelectionDAG::getNode(unsigned Opcode, SDLoc DL,
       // FIXME need to be more flexible about rounding mode.
       (void)V.convert(EVTToAPFloatSemantics(VT),
                       APFloat::rmNearestTiesToEven, &ignored);
-      return getConstantFP(V, VT);
+      return getConstantFP(V, DL, VT);
     }
     case ISD::FP_TO_SINT:
     case ISD::FP_TO_UINT: {
@@ -2830,44 +2914,73 @@ SDValue SelectionDAG::getNode(unsigned Opcode, SDLoc DL,
       if (s==APFloat::opInvalidOp)     // inexact is OK, in fact usual
         break;
       APInt api(VT.getSizeInBits(), x);
-      return getConstant(api, VT);
+      return getConstant(api, DL, VT);
     }
     case ISD::BITCAST:
       if (VT == MVT::i16 && C->getValueType(0) == MVT::f16)
-        return getConstant((uint16_t)V.bitcastToAPInt().getZExtValue(), VT);
+        return getConstant((uint16_t)V.bitcastToAPInt().getZExtValue(), DL, VT);
       else if (VT == MVT::i32 && C->getValueType(0) == MVT::f32)
-        return getConstant((uint32_t)V.bitcastToAPInt().getZExtValue(), VT);
+        return getConstant((uint32_t)V.bitcastToAPInt().getZExtValue(), DL, VT);
       else if (VT == MVT::i64 && C->getValueType(0) == MVT::f64)
-        return getConstant(V.bitcastToAPInt().getZExtValue(), VT);
+        return getConstant(V.bitcastToAPInt().getZExtValue(), DL, VT);
       break;
     }
   }
 
   // Constant fold unary operations with a vector integer or float operand.
-  if (BuildVectorSDNode *BV = dyn_cast<BuildVectorSDNode>(Operand.getNode())) {
+  if (BuildVectorSDNode *BV = dyn_cast<BuildVectorSDNode>(Operand)) {
     if (BV->isConstant()) {
       switch (Opcode) {
       default:
         // FIXME: Entirely reasonable to perform folding of other unary
         // operations here as the need arises.
         break;
-      case ISD::TRUNCATE:
-        // Constant build vector truncation can be done with the original scalar
-        // operands but with a new build vector with the truncated value type.
-        return getNode(ISD::BUILD_VECTOR, DL, VT, BV->ops());
       case ISD::FNEG:
       case ISD::FABS:
       case ISD::FCEIL:
       case ISD::FTRUNC:
       case ISD::FFLOOR:
       case ISD::FP_EXTEND:
+      case ISD::FP_TO_SINT:
+      case ISD::FP_TO_UINT:
+      case ISD::TRUNCATE:
       case ISD::UINT_TO_FP:
-      case ISD::SINT_TO_FP: {
+      case ISD::SINT_TO_FP:
+      case ISD::BSWAP:
+      case ISD::CTLZ:
+      case ISD::CTLZ_ZERO_UNDEF:
+      case ISD::CTTZ:
+      case ISD::CTTZ_ZERO_UNDEF:
+      case ISD::CTPOP: {
+        EVT SVT = VT.getScalarType();
+        EVT InVT = BV->getValueType(0);
+        EVT InSVT = InVT.getScalarType();
+
+        // Find legal integer scalar type for constant promotion and
+        // ensure that its scalar size is at least as large as source.
+        EVT LegalSVT = SVT;
+        if (SVT.isInteger()) {
+          LegalSVT = TLI->getTypeToTransformTo(*getContext(), SVT);
+          if (LegalSVT.bitsLT(SVT)) break;
+        }
+
         // Let the above scalar folding handle the folding of each element.
         SmallVector<SDValue, 8> Ops;
         for (int i = 0, e = VT.getVectorNumElements(); i != e; ++i) {
           SDValue OpN = BV->getOperand(i);
-          OpN = getNode(Opcode, DL, VT.getVectorElementType(), OpN);
+          EVT OpVT = OpN.getValueType();
+
+          // Build vector (integer) scalar operands may need implicit
+          // truncation - do this before constant folding.
+          if (OpVT.isInteger() && OpVT.bitsGT(InSVT))
+            OpN = getNode(ISD::TRUNCATE, DL, InSVT, OpN);
+
+          OpN = getNode(Opcode, DL, SVT, OpN);
+
+          // Legalize the (integer) scalar constant if necessary.
+          if (LegalSVT != SVT)
+            OpN = getNode(ISD::ANY_EXTEND, DL, LegalSVT, OpN);
+
           if (OpN.getOpcode() != ISD::UNDEF &&
               OpN.getOpcode() != ISD::Constant &&
               OpN.getOpcode() != ISD::ConstantFP)
@@ -2914,7 +3027,7 @@ SDValue SelectionDAG::getNode(unsigned Opcode, SDLoc DL,
       return getNode(OpOpcode, DL, VT, Operand.getNode()->getOperand(0));
     else if (OpOpcode == ISD::UNDEF)
       // sext(undef) = 0, because the top bits will all be the same.
-      return getConstant(0, VT);
+      return getConstant(0, DL, VT);
     break;
   case ISD::ZERO_EXTEND:
     assert(VT.isInteger() && Operand.getValueType().isInteger() &&
@@ -2931,7 +3044,7 @@ SDValue SelectionDAG::getNode(unsigned Opcode, SDLoc DL,
                      Operand.getNode()->getOperand(0));
     else if (OpOpcode == ISD::UNDEF)
       // zext(undef) = 0, because the top bits will be zero.
-      return getConstant(0, VT);
+      return getConstant(0, DL, VT);
     break;
   case ISD::ANY_EXTEND:
     assert(VT.isInteger() && Operand.getValueType().isInteger() &&
@@ -2983,6 +3096,14 @@ SDValue SelectionDAG::getNode(unsigned Opcode, SDLoc DL,
     if (OpOpcode == ISD::UNDEF)
       return getUNDEF(VT);
     break;
+  case ISD::BSWAP:
+    assert(VT.isInteger() && VT == Operand.getValueType() &&
+           "Invalid BSWAP!");
+    assert((VT.getScalarSizeInBits() % 16 == 0) &&
+           "BSWAP types must be a multiple of 16 bits!");
+    if (OpOpcode == ISD::UNDEF)
+      return getUNDEF(VT);
+    break;
   case ISD::BITCAST:
     // Basic sanity checking.
     assert(VT.getSizeInBits() == Operand.getValueType().getSizeInBits()
@@ -3030,7 +3151,7 @@ SDValue SelectionDAG::getNode(unsigned Opcode, SDLoc DL,
     SDValue Ops[1] = { Operand };
     AddNodeIDNode(ID, Opcode, VTs, Ops);
     void *IP = nullptr;
-    if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP))
+    if (SDNode *E = FindNodeOrInsertPos(ID, DL.getDebugLoc(), IP))
       return SDValue(E, 0);
 
     N = new (NodeAllocator) UnarySDNode(Opcode, DL.getIROrder(),
@@ -3045,7 +3166,54 @@ SDValue SelectionDAG::getNode(unsigned Opcode, SDLoc DL,
   return SDValue(N, 0);
 }
 
-SDValue SelectionDAG::FoldConstantArithmetic(unsigned Opcode, EVT VT,
+static std::pair<APInt, bool> FoldValue(unsigned Opcode, const APInt &C1,
+                                        const APInt &C2) {
+  switch (Opcode) {
+  case ISD::ADD:  return std::make_pair(C1 + C2, true);
+  case ISD::SUB:  return std::make_pair(C1 - C2, true);
+  case ISD::MUL:  return std::make_pair(C1 * C2, true);
+  case ISD::AND:  return std::make_pair(C1 & C2, true);
+  case ISD::OR:   return std::make_pair(C1 | C2, true);
+  case ISD::XOR:  return std::make_pair(C1 ^ C2, true);
+  case ISD::SHL:  return std::make_pair(C1 << C2, true);
+  case ISD::SRL:  return std::make_pair(C1.lshr(C2), true);
+  case ISD::SRA:  return std::make_pair(C1.ashr(C2), true);
+  case ISD::ROTL: return std::make_pair(C1.rotl(C2), true);
+  case ISD::ROTR: return std::make_pair(C1.rotr(C2), true);
+  case ISD::UDIV:
+    if (!C2.getBoolValue())
+      break;
+    return std::make_pair(C1.udiv(C2), true);
+  case ISD::UREM:
+    if (!C2.getBoolValue())
+      break;
+    return std::make_pair(C1.urem(C2), true);
+  case ISD::SDIV:
+    if (!C2.getBoolValue())
+      break;
+    return std::make_pair(C1.sdiv(C2), true);
+  case ISD::SREM:
+    if (!C2.getBoolValue())
+      break;
+    return std::make_pair(C1.srem(C2), true);
+  }
+  return std::make_pair(APInt(1, 0), false);
+}
+
+SDValue SelectionDAG::FoldConstantArithmetic(unsigned Opcode, SDLoc DL, EVT VT,
+                                             const ConstantSDNode *Cst1,
+                                             const ConstantSDNode *Cst2) {
+  if (Cst1->isOpaque() || Cst2->isOpaque())
+    return SDValue();
+
+  std::pair<APInt, bool> Folded = FoldValue(Opcode, Cst1->getAPIntValue(),
+                                            Cst2->getAPIntValue());
+  if (!Folded.second)
+    return SDValue();
+  return getConstant(Folded.first, DL, VT);
+}
+
+SDValue SelectionDAG::FoldConstantArithmetic(unsigned Opcode, SDLoc DL, EVT VT,
                                              SDNode *Cst1, SDNode *Cst2) {
   // If the opcode is a target-specific ISD node, there's nothing we can
   // do here and the operand rules may not line up with the below, so
@@ -3053,116 +3221,59 @@ SDValue SelectionDAG::FoldConstantArithmetic(unsigned Opcode, EVT VT,
   if (Opcode >= ISD::BUILTIN_OP_END)
     return SDValue();
 
-  SmallVector<std::pair<ConstantSDNode *, ConstantSDNode *>, 4> Inputs;
-  SmallVector<SDValue, 4> Outputs;
-  EVT SVT = VT.getScalarType();
+  // Handle the case of two scalars.
+  if (const ConstantSDNode *Scalar1 = dyn_cast<ConstantSDNode>(Cst1)) {
+    if (const ConstantSDNode *Scalar2 = dyn_cast<ConstantSDNode>(Cst2)) {
+      if (SDValue Folded =
+          FoldConstantArithmetic(Opcode, DL, VT, Scalar1, Scalar2)) {
+        if (!VT.isVector())
+          return Folded;
+        SmallVector<SDValue, 4> Outputs;
+        // We may have a vector type but a scalar result. Create a splat.
+        Outputs.resize(VT.getVectorNumElements(), Outputs.back());
+        // Build a big vector out of the scalar elements we generated.
+        return getNode(ISD::BUILD_VECTOR, SDLoc(), VT, Outputs);
+      } else {
+        return SDValue();
+      }
+    }
+  }
 
-  ConstantSDNode *Scalar1 = dyn_cast<ConstantSDNode>(Cst1);
-  ConstantSDNode *Scalar2 = dyn_cast<ConstantSDNode>(Cst2);
-  if (Scalar1 && Scalar2 && (Scalar1->isOpaque() || Scalar2->isOpaque()))
+  // For vectors extract each constant element into Inputs so we can constant
+  // fold them individually.
+  BuildVectorSDNode *BV1 = dyn_cast<BuildVectorSDNode>(Cst1);
+  BuildVectorSDNode *BV2 = dyn_cast<BuildVectorSDNode>(Cst2);
+  if (!BV1 || !BV2)
     return SDValue();
 
-  if (Scalar1 && Scalar2)
-    // Scalar instruction.
-    Inputs.push_back(std::make_pair(Scalar1, Scalar2));
-  else {
-    // For vectors extract each constant element into Inputs so we can constant
-    // fold them individually.
-    BuildVectorSDNode *BV1 = dyn_cast<BuildVectorSDNode>(Cst1);
-    BuildVectorSDNode *BV2 = dyn_cast<BuildVectorSDNode>(Cst2);
-    if (!BV1 || !BV2)
-      return SDValue();
-
-    assert(BV1->getNumOperands() == BV2->getNumOperands() && "Out of sync!");
+  assert(BV1->getNumOperands() == BV2->getNumOperands() && "Out of sync!");
 
-    for (unsigned I = 0, E = BV1->getNumOperands(); I != E; ++I) {
-      ConstantSDNode *V1 = dyn_cast<ConstantSDNode>(BV1->getOperand(I));
-      ConstantSDNode *V2 = dyn_cast<ConstantSDNode>(BV2->getOperand(I));
-      if (!V1 || !V2) // Not a constant, bail.
-        return SDValue();
-
-      if (V1->isOpaque() || V2->isOpaque())
-        return SDValue();
-
-      // Avoid BUILD_VECTOR nodes that perform implicit truncation.
-      // FIXME: This is valid and could be handled by truncating the APInts.
-      if (V1->getValueType(0) != SVT || V2->getValueType(0) != SVT)
-        return SDValue();
+  EVT SVT = VT.getScalarType();
+  SmallVector<SDValue, 4> Outputs;
+  for (unsigned I = 0, E = BV1->getNumOperands(); I != E; ++I) {
+    ConstantSDNode *V1 = dyn_cast<ConstantSDNode>(BV1->getOperand(I));
+    ConstantSDNode *V2 = dyn_cast<ConstantSDNode>(BV2->getOperand(I));
+    if (!V1 || !V2) // Not a constant, bail.
+      return SDValue();
 
-      Inputs.push_back(std::make_pair(V1, V2));
-    }
-  }
+    if (V1->isOpaque() || V2->isOpaque())
+      return SDValue();
 
-  // We have a number of constant values, constant fold them element by element.
-  for (unsigned I = 0, E = Inputs.size(); I != E; ++I) {
-    const APInt &C1 = Inputs[I].first->getAPIntValue();
-    const APInt &C2 = Inputs[I].second->getAPIntValue();
+    // Avoid BUILD_VECTOR nodes that perform implicit truncation.
+    // FIXME: This is valid and could be handled by truncating the APInts.
+    if (V1->getValueType(0) != SVT || V2->getValueType(0) != SVT)
+      return SDValue();
 
-    switch (Opcode) {
-    case ISD::ADD:
-      Outputs.push_back(getConstant(C1 + C2, SVT));
-      break;
-    case ISD::SUB:
-      Outputs.push_back(getConstant(C1 - C2, SVT));
-      break;
-    case ISD::MUL:
-      Outputs.push_back(getConstant(C1 * C2, SVT));
-      break;
-    case ISD::UDIV:
-      if (!C2.getBoolValue())
-        return SDValue();
-      Outputs.push_back(getConstant(C1.udiv(C2), SVT));
-      break;
-    case ISD::UREM:
-      if (!C2.getBoolValue())
-        return SDValue();
-      Outputs.push_back(getConstant(C1.urem(C2), SVT));
-      break;
-    case ISD::SDIV:
-      if (!C2.getBoolValue())
-        return SDValue();
-      Outputs.push_back(getConstant(C1.sdiv(C2), SVT));
-      break;
-    case ISD::SREM:
-      if (!C2.getBoolValue())
-        return SDValue();
-      Outputs.push_back(getConstant(C1.srem(C2), SVT));
-      break;
-    case ISD::AND:
-      Outputs.push_back(getConstant(C1 & C2, SVT));
-      break;
-    case ISD::OR:
-      Outputs.push_back(getConstant(C1 | C2, SVT));
-      break;
-    case ISD::XOR:
-      Outputs.push_back(getConstant(C1 ^ C2, SVT));
-      break;
-    case ISD::SHL:
-      Outputs.push_back(getConstant(C1 << C2, SVT));
-      break;
-    case ISD::SRL:
-      Outputs.push_back(getConstant(C1.lshr(C2), SVT));
-      break;
-    case ISD::SRA:
-      Outputs.push_back(getConstant(C1.ashr(C2), SVT));
-      break;
-    case ISD::ROTL:
-      Outputs.push_back(getConstant(C1.rotl(C2), SVT));
-      break;
-    case ISD::ROTR:
-      Outputs.push_back(getConstant(C1.rotr(C2), SVT));
-      break;
-    default:
+    // Fold one vector element.
+    std::pair<APInt, bool> Folded = FoldValue(Opcode, V1->getAPIntValue(),
+                                              V2->getAPIntValue());
+    if (!Folded.second)
       return SDValue();
-    }
+    Outputs.push_back(getConstant(Folded.first, DL, SVT));
   }
 
-  assert((Scalar1 && Scalar2) || (VT.getVectorNumElements() == Outputs.size() &&
-                                  "Expected a scalar or vector!"));
-
-  // Handle the scalar case first.
-  if (!VT.isVector())
-    return Outputs.back();
+  assert(VT.getVectorNumElements() == Outputs.size() &&
+         "Vector size mismatch!");
 
   // We may have a vector type but a scalar result. Create a splat.
   Outputs.resize(VT.getVectorNumElements(), Outputs.back());
@@ -3172,9 +3283,9 @@ SDValue SelectionDAG::FoldConstantArithmetic(unsigned Opcode, EVT VT,
 }
 
 SDValue SelectionDAG::getNode(unsigned Opcode, SDLoc DL, EVT VT, SDValue N1,
-                              SDValue N2, bool nuw, bool nsw, bool exact) {
-  ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(N1.getNode());
-  ConstantSDNode *N2C = dyn_cast<ConstantSDNode>(N2.getNode());
+                              SDValue N2, const SDNodeFlags *Flags) {
+  ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(N1);
+  ConstantSDNode *N2C = dyn_cast<ConstantSDNode>(N2);
   switch (Opcode) {
   default: break;
   case ISD::TokenFactor:
@@ -3198,6 +3309,18 @@ SDValue SelectionDAG::getNode(unsigned Opcode, SDLoc DL, EVT VT, SDValue N1,
       SmallVector<SDValue, 16> Elts(N1.getNode()->op_begin(),
                                     N1.getNode()->op_end());
       Elts.append(N2.getNode()->op_begin(), N2.getNode()->op_end());
+
+      // BUILD_VECTOR requires all inputs to be of the same type, find the
+      // maximum type and extend them all.
+      EVT SVT = VT.getScalarType();
+      for (SDValue Op : Elts)
+        SVT = (SVT.bitsLT(Op.getValueType()) ? Op.getValueType() : SVT);
+      if (SVT.bitsGT(VT.getScalarType()))
+        for (SDValue &Op : Elts)
+          Op = TLI->isZExtFree(Op.getValueType(), SVT)
+             ? getZExtOrTrunc(Op, DL, SVT)
+             : getSExtOrTrunc(Op, DL, SVT);
+
       return getNode(ISD::BUILD_VECTOR, DL, VT, Elts);
     }
     break;
@@ -3362,12 +3485,35 @@ SDValue SelectionDAG::getNode(unsigned Opcode, SDLoc DL, EVT VT, SDValue N1,
     assert(EVT.bitsLE(VT) && "Not extending!");
     if (EVT == VT) return N1;  // Not actually extending
 
+    auto SignExtendInReg = [&](APInt Val) {
+      unsigned FromBits = EVT.getScalarType().getSizeInBits();
+      Val <<= Val.getBitWidth() - FromBits;
+      Val = Val.ashr(Val.getBitWidth() - FromBits);
+      return getConstant(Val, DL, VT.getScalarType());
+    };
+
     if (N1C) {
       APInt Val = N1C->getAPIntValue();
-      unsigned FromBits = EVT.getScalarType().getSizeInBits();
-      Val <<= Val.getBitWidth()-FromBits;
-      Val = Val.ashr(Val.getBitWidth()-FromBits);
-      return getConstant(Val, VT);
+      return SignExtendInReg(Val);
+    }
+    if (ISD::isBuildVectorOfConstantSDNodes(N1.getNode())) {
+      SmallVector<SDValue, 8> Ops;
+      for (int i = 0, e = VT.getVectorNumElements(); i != e; ++i) {
+        SDValue Op = N1.getOperand(i);
+        if (Op.getValueType() != VT.getScalarType()) break;
+        if (Op.getOpcode() == ISD::UNDEF) {
+          Ops.push_back(Op);
+          continue;
+        }
+        if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op)) {
+          APInt Val = C->getAPIntValue();
+          Ops.push_back(SignExtendInReg(Val));
+          continue;
+        }
+        break;
+      }
+      if (Ops.size() == VT.getVectorNumElements())
+        return getNode(ISD::BUILD_VECTOR, DL, VT, Ops);
     }
     break;
   }
@@ -3376,6 +3522,10 @@ SDValue SelectionDAG::getNode(unsigned Opcode, SDLoc DL, EVT VT, SDValue N1,
     if (N1.getOpcode() == ISD::UNDEF)
       return getUNDEF(VT);
 
+    // EXTRACT_VECTOR_ELT of out-of-bounds element is an UNDEF
+    if (N2C && N2C->getZExtValue() >= N1.getValueType().getVectorNumElements())
+      return getUNDEF(VT);
+
     // EXTRACT_VECTOR_ELT of CONCAT_VECTORS is often formed while lowering is
     // expanding copies of large vectors from registers.
     if (N2C &&
@@ -3385,7 +3535,7 @@ SDValue SelectionDAG::getNode(unsigned Opcode, SDLoc DL, EVT VT, SDValue N1,
         N1.getOperand(0).getValueType().getVectorNumElements();
       return getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT,
                      N1.getOperand(N2C->getZExtValue() / Factor),
-                     getConstant(N2C->getZExtValue() % Factor,
+                     getConstant(N2C->getZExtValue() % Factor, DL,
                                  N2.getValueType()));
     }
 
@@ -3410,7 +3560,7 @@ SDValue SelectionDAG::getNode(unsigned Opcode, SDLoc DL, EVT VT, SDValue N1,
       // if the indices are known different, extract the element from
       // the original vector.
       SDValue N1Op2 = N1.getOperand(2);
-      ConstantSDNode *N1Op2C = dyn_cast<ConstantSDNode>(N1Op2.getNode());
+      ConstantSDNode *N1Op2C = dyn_cast<ConstantSDNode>(N1Op2);
 
       if (N1Op2C && N2C) {
         if (N1Op2C->getZExtValue() == N2C->getZExtValue()) {
@@ -3442,7 +3592,7 @@ SDValue SelectionDAG::getNode(unsigned Opcode, SDLoc DL, EVT VT, SDValue N1,
       unsigned ElementSize = VT.getSizeInBits();
       unsigned Shift = ElementSize * N2C->getZExtValue();
       APInt ShiftedVal = C->getAPIntValue().lshr(Shift);
-      return getConstant(ShiftedVal.trunc(ElementSize), VT);
+      return getConstant(ShiftedVal.trunc(ElementSize), DL, VT);
     }
     break;
   case ISD::EXTRACT_SUBVECTOR: {
@@ -3456,9 +3606,9 @@ SDValue SelectionDAG::getNode(unsigned Opcode, SDLoc DL, EVT VT, SDValue N1,
       assert(VT.getSimpleVT() <= N1.getSimpleValueType() &&
              "Extract subvector must be from larger vector to smaller vector!");
 
-      if (isa<ConstantSDNode>(Index.getNode())) {
+      if (isa<ConstantSDNode>(Index)) {
         assert((VT.getVectorNumElements() +
-                cast<ConstantSDNode>(Index.getNode())->getZExtValue()
+                cast<ConstantSDNode>(Index)->getZExtValue()
                 <= N1.getValueType().getVectorNumElements())
                && "Extract subvector overflow!");
       }
@@ -3473,7 +3623,7 @@ SDValue SelectionDAG::getNode(unsigned Opcode, SDLoc DL, EVT VT, SDValue N1,
 
   // Perform trivial constant folding.
   if (SDValue SV =
-          FoldConstantArithmetic(Opcode, VT, N1.getNode(), N2.getNode()))
+          FoldConstantArithmetic(Opcode, DL, VT, N1.getNode(), N2.getNode()))
     return SV;
 
   // Canonicalize constant to RHS if commutative.
@@ -3484,8 +3634,8 @@ SDValue SelectionDAG::getNode(unsigned Opcode, SDLoc DL, EVT VT, SDValue N1,
 
   // Constant fold FP operations.
   bool HasFPExceptions = TLI->hasFloatingPointExceptions();
-  ConstantFPSDNode *N1CFP = dyn_cast<ConstantFPSDNode>(N1.getNode());
-  ConstantFPSDNode *N2CFP = dyn_cast<ConstantFPSDNode>(N2.getNode());
+  ConstantFPSDNode *N1CFP = dyn_cast<ConstantFPSDNode>(N1);
+  ConstantFPSDNode *N2CFP = dyn_cast<ConstantFPSDNode>(N2);
   if (N1CFP) {
     if (!N2CFP && isCommutativeBinOp(Opcode)) {
       // Canonicalize constant to RHS if commutative.
@@ -3498,35 +3648,35 @@ SDValue SelectionDAG::getNode(unsigned Opcode, SDLoc DL, EVT VT, SDValue N1,
       case ISD::FADD:
         s = V1.add(V2, APFloat::rmNearestTiesToEven);
         if (!HasFPExceptions || s != APFloat::opInvalidOp)
-          return getConstantFP(V1, VT);
+          return getConstantFP(V1, DL, VT);
         break;
       case ISD::FSUB:
         s = V1.subtract(V2, APFloat::rmNearestTiesToEven);
         if (!HasFPExceptions || s!=APFloat::opInvalidOp)
-          return getConstantFP(V1, VT);
+          return getConstantFP(V1, DL, VT);
         break;
       case ISD::FMUL:
         s = V1.multiply(V2, APFloat::rmNearestTiesToEven);
         if (!HasFPExceptions || s!=APFloat::opInvalidOp)
-          return getConstantFP(V1, VT);
+          return getConstantFP(V1, DL, VT);
         break;
       case ISD::FDIV:
         s = V1.divide(V2, APFloat::rmNearestTiesToEven);
         if (!HasFPExceptions || (s!=APFloat::opInvalidOp &&
                                  s!=APFloat::opDivByZero)) {
-          return getConstantFP(V1, VT);
+          return getConstantFP(V1, DL, VT);
         }
         break;
       case ISD::FREM :
         s = V1.mod(V2, APFloat::rmNearestTiesToEven);
         if (!HasFPExceptions || (s!=APFloat::opInvalidOp &&
                                  s!=APFloat::opDivByZero)) {
-          return getConstantFP(V1, VT);
+          return getConstantFP(V1, DL, VT);
         }
         break;
       case ISD::FCOPYSIGN:
         V1.copySign(V2);
-        return getConstantFP(V1, VT);
+        return getConstantFP(V1, DL, VT);
       default: break;
       }
     }
@@ -3538,7 +3688,7 @@ SDValue SelectionDAG::getNode(unsigned Opcode, SDLoc DL, EVT VT, SDValue N1,
       // FIXME need to be more flexible about rounding mode.
       (void)V.convert(EVTToAPFloatSemantics(VT),
                       APFloat::rmNearestTiesToEven, &ignored);
-      return getConstantFP(V, VT);
+      return getConstantFP(V, DL, VT);
     }
   }
 
@@ -3563,7 +3713,7 @@ SDValue SelectionDAG::getNode(unsigned Opcode, SDLoc DL, EVT VT, SDValue N1,
       case ISD::SRL:
       case ISD::SHL:
         if (!VT.isVector())
-          return getConstant(0, VT);    // fold op(undef, arg2) -> 0
+          return getConstant(0, DL, VT);    // fold op(undef, arg2) -> 0
         // For vectors, we can't easily build an all zero vector, just return
         // the LHS.
         return N2;
@@ -3578,7 +3728,7 @@ SDValue SelectionDAG::getNode(unsigned Opcode, SDLoc DL, EVT VT, SDValue N1,
       if (N1.getOpcode() == ISD::UNDEF)
         // Handle undef ^ undef -> 0 special case. This is a common
         // idiom (misuse).
-        return getConstant(0, VT);
+        return getConstant(0, DL, VT);
       // fallthrough
     case ISD::ADD:
     case ISD::ADDC:
@@ -3602,13 +3752,13 @@ SDValue SelectionDAG::getNode(unsigned Opcode, SDLoc DL, EVT VT, SDValue N1,
     case ISD::SRL:
     case ISD::SHL:
       if (!VT.isVector())
-        return getConstant(0, VT);  // fold op(arg1, undef) -> 0
+        return getConstant(0, DL, VT);  // fold op(arg1, undef) -> 0
       // For vectors, we can't easily build an all zero vector, just return
       // the LHS.
       return N1;
     case ISD::OR:
       if (!VT.isVector())
-        return getConstant(APInt::getAllOnesValue(VT.getSizeInBits()), VT);
+        return getConstant(APInt::getAllOnesValue(VT.getSizeInBits()), DL, VT);
       // For vectors, we can't easily build an all one vector, just return
       // the LHS.
       return N1;
@@ -3620,22 +3770,20 @@ SDValue SelectionDAG::getNode(unsigned Opcode, SDLoc DL, EVT VT, SDValue N1,
   // Memoize this node if possible.
   BinarySDNode *N;
   SDVTList VTs = getVTList(VT);
-  const bool BinOpHasFlags = isBinOpWithFlags(Opcode);
   if (VT != MVT::Glue) {
     SDValue Ops[] = {N1, N2};
     FoldingSetNodeID ID;
     AddNodeIDNode(ID, Opcode, VTs, Ops);
-    if (BinOpHasFlags)
-      AddBinaryNodeIDCustom(ID, Opcode, nuw, nsw, exact);
+    AddNodeIDFlags(ID, Opcode, Flags);
     void *IP = nullptr;
-    if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP))
+    if (SDNode *E = FindNodeOrInsertPos(ID, DL.getDebugLoc(), IP))
       return SDValue(E, 0);
 
-    N = GetBinarySDNode(Opcode, DL, VTs, N1, N2, nuw, nsw, exact);
+    N = GetBinarySDNode(Opcode, DL, VTs, N1, N2, Flags);
 
     CSEMap.InsertNode(N, IP);
   } else {
-    N = GetBinarySDNode(Opcode, DL, VTs, N1, N2, nuw, nsw, exact);
+    N = GetBinarySDNode(Opcode, DL, VTs, N1, N2, Flags);
   }
 
   InsertNode(N);
@@ -3645,7 +3793,7 @@ SDValue SelectionDAG::getNode(unsigned Opcode, SDLoc DL, EVT VT, SDValue N1,
 SDValue SelectionDAG::getNode(unsigned Opcode, SDLoc DL, EVT VT,
                               SDValue N1, SDValue N2, SDValue N3) {
   // Perform various simplifications.
-  ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(N1.getNode());
+  ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(N1);
   switch (Opcode) {
   case ISD::FMA: {
     ConstantFPSDNode *N1CFP = dyn_cast<ConstantFPSDNode>(N1);
@@ -3658,7 +3806,7 @@ SDValue SelectionDAG::getNode(unsigned Opcode, SDLoc DL, EVT VT,
       APFloat::opStatus s =
         V1.fusedMultiplyAdd(V2, V3, APFloat::rmNearestTiesToEven);
       if (!TLI->hasFloatingPointExceptions() || s != APFloat::opInvalidOp)
-        return getConstantFP(V1, VT);
+        return getConstantFP(V1, DL, VT);
     }
     break;
   }
@@ -3703,9 +3851,9 @@ SDValue SelectionDAG::getNode(unsigned Opcode, SDLoc DL, EVT VT,
              "Dest and insert subvector source types must match!");
       assert(N2.getSimpleValueType() <= N1.getSimpleValueType() &&
              "Insert subvector must be from smaller vector to larger vector!");
-      if (isa<ConstantSDNode>(Index.getNode())) {
+      if (isa<ConstantSDNode>(Index)) {
         assert((N2.getValueType().getVectorNumElements() +
-                cast<ConstantSDNode>(Index.getNode())->getZExtValue()
+                cast<ConstantSDNode>(Index)->getZExtValue()
                 <= VT.getVectorNumElements())
                && "Insert subvector overflow!");
       }
@@ -3731,7 +3879,7 @@ SDValue SelectionDAG::getNode(unsigned Opcode, SDLoc DL, EVT VT,
     FoldingSetNodeID ID;
     AddNodeIDNode(ID, Opcode, VTs, Ops);
     void *IP = nullptr;
-    if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP))
+    if (SDNode *E = FindNodeOrInsertPos(ID, DL.getDebugLoc(), IP))
       return SDValue(E, 0);
 
     N = new (NodeAllocator) TernarySDNode(Opcode, DL.getIROrder(),
@@ -3793,8 +3941,9 @@ static SDValue getMemsetValue(SDValue Value, EVT VT, SelectionDAG &DAG,
     assert(C->getAPIntValue().getBitWidth() == 8);
     APInt Val = APInt::getSplat(NumBits, C->getAPIntValue());
     if (VT.isInteger())
-      return DAG.getConstant(Val, VT);
-    return DAG.getConstantFP(APFloat(DAG.EVTToAPFloatSemantics(VT), Val), VT);
+      return DAG.getConstant(Val, dl, VT);
+    return DAG.getConstantFP(APFloat(DAG.EVTToAPFloatSemantics(VT), Val), dl,
+                             VT);
   }
 
   assert(Value.getValueType() == MVT::i8 && "memset with non-byte fill value?");
@@ -3808,7 +3957,7 @@ static SDValue getMemsetValue(SDValue Value, EVT VT, SelectionDAG &DAG,
     // required length.
     APInt Magic = APInt::getSplat(NumBits, APInt(8, 0x01));
     Value = DAG.getNode(ISD::MUL, dl, IntVT, Value,
-                        DAG.getConstant(Magic, IntVT));
+                        DAG.getConstant(Magic, dl, IntVT));
   }
 
   if (VT != Value.getValueType() && !VT.isInteger())
@@ -3831,15 +3980,16 @@ static SDValue getMemsetStringVal(EVT VT, SDLoc dl, SelectionDAG &DAG,
   // Handle vector with all elements zero.
   if (Str.empty()) {
     if (VT.isInteger())
-      return DAG.getConstant(0, VT);
+      return DAG.getConstant(0, dl, VT);
     else if (VT == MVT::f32 || VT == MVT::f64 || VT == MVT::f128)
-      return DAG.getConstantFP(0.0, VT);
+      return DAG.getConstantFP(0.0, dl, VT);
     else if (VT.isVector()) {
       unsigned NumElts = VT.getVectorNumElements();
       MVT EltVT = (VT.getVectorElementType() == MVT::f32) ? MVT::i32 : MVT::i64;
       return DAG.getNode(ISD::BITCAST, dl, VT,
-                         DAG.getConstant(0, EVT::getVectorVT(*DAG.getContext(),
-                                                             EltVT, NumElts)));
+                         DAG.getConstant(0, dl,
+                                         EVT::getVectorVT(*DAG.getContext(),
+                                                          EltVT, NumElts)));
     } else
       llvm_unreachable("Expected type!");
   }
@@ -3850,7 +4000,7 @@ static SDValue getMemsetStringVal(EVT VT, SDLoc dl, SelectionDAG &DAG,
   unsigned NumBytes = std::min(NumVTBytes, unsigned(Str.size()));
 
   APInt Val(NumVTBits, 0);
-  if (TLI.isLittleEndian()) {
+  if (DAG.getDataLayout().isLittleEndian()) {
     for (unsigned i = 0; i != NumBytes; ++i)
       Val |= (uint64_t)(unsigned char)Str[i] << i*8;
   } else {
@@ -3862,7 +4012,7 @@ static SDValue getMemsetStringVal(EVT VT, SDLoc dl, SelectionDAG &DAG,
   // of a load, then it is cost effective to turn the load into the immediate.
   Type *Ty = VT.getTypeForEVT(*DAG.getContext());
   if (TLI.shouldConvertConstantLoadToIntImm(Val, Ty))
-    return DAG.getConstant(Val, VT);
+    return DAG.getConstant(Val, dl, VT);
   return SDValue(nullptr, 0);
 }
 
@@ -3872,7 +4022,7 @@ static SDValue getMemBasePlusOffset(SDValue Base, unsigned Offset, SDLoc dl,
                                       SelectionDAG &DAG) {
   EVT VT = Base.getValueType();
   return DAG.getNode(ISD::ADD, dl,
-                     VT, Base, DAG.getConstant(Offset, VT));
+                     VT, Base, DAG.getConstant(Offset, dl, VT));
 }
 
 /// isMemSrcFromString - Returns true if memcpy source is a string constant.
@@ -3894,10 +4044,10 @@ static bool isMemSrcFromString(SDValue Src, StringRef &Str) {
   return getConstantStringInfo(G->getGlobal(), Str, SrcDelta, false);
 }
 
-/// FindOptimalMemOpLowering - Determines the optimial series memory ops
-/// to replace the memset / memcpy. Return true if the number of memory ops
-/// is below the threshold. It returns the types of the sequence of
-/// memory ops to perform memset / memcpy by reference.
+/// Determines the optimal series of memory ops to replace the memset / memcpy.
+/// Return true if the number of memory ops is below the threshold (Limit).
+/// It returns the types of the sequence of memory ops to perform
+/// memset / memcpy by reference.
 static bool FindOptimalMemOpLowering(std::vector<EVT> &MemOps,
                                      unsigned Limit, uint64_t Size,
                                      unsigned DstAlign, unsigned SrcAlign,
@@ -3922,9 +4072,9 @@ static bool FindOptimalMemOpLowering(std::vector<EVT> &MemOps,
 
   if (VT == MVT::Other) {
     unsigned AS = 0;
-    if (DstAlign >= TLI.getDataLayout()->getPointerPrefAlignment(AS) ||
+    if (DstAlign >= DAG.getDataLayout().getPointerPrefAlignment(AS) ||
         TLI.allowsMisalignedMemoryAccesses(VT, AS, DstAlign)) {
-      VT = TLI.getPointerTy();
+      VT = TLI.getPointerTy(DAG.getDataLayout());
     } else {
       switch (DstAlign & 7) {
       case 0:  VT = MVT::i64; break;
@@ -4001,6 +4151,18 @@ static bool FindOptimalMemOpLowering(std::vector<EVT> &MemOps,
   return true;
 }
 
+static bool shouldLowerMemFuncForSize(const MachineFunction &MF) {
+  const Function *F = MF.getFunction();
+  bool HasMinSize = F->hasFnAttribute(Attribute::MinSize);
+  bool HasOptSize = F->hasFnAttribute(Attribute::OptimizeForSize);
+
+  // On Darwin, -Os means optimize for size without hurting performance, so
+  // only really optimize for size when -Oz (MinSize) is used.
+  if (MF.getTarget().getTargetTriple().isOSDarwin())
+    return HasMinSize;
+  return HasOptSize || HasMinSize;
+}
+
 static SDValue getMemcpyLoadsAndStores(SelectionDAG &DAG, SDLoc dl,
                                        SDValue Chain, SDValue Dst,
                                        SDValue Src, uint64_t Size,
@@ -4021,7 +4183,7 @@ static SDValue getMemcpyLoadsAndStores(SelectionDAG &DAG, SDLoc dl,
   bool DstAlignCanChange = false;
   MachineFunction &MF = DAG.getMachineFunction();
   MachineFrameInfo *MFI = MF.getFrameInfo();
-  bool OptSize = MF.getFunction()->hasFnAttribute(Attribute::OptimizeForSize);
+  bool OptSize = shouldLowerMemFuncForSize(MF);
   FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Dst);
   if (FI && !MFI->isFixedObjectIndex(FI->getIndex()))
     DstAlignCanChange = true;
@@ -4041,14 +4203,14 @@ static SDValue getMemcpyLoadsAndStores(SelectionDAG &DAG, SDLoc dl,
 
   if (DstAlignCanChange) {
     Type *Ty = MemOps[0].getTypeForEVT(*DAG.getContext());
-    unsigned NewAlign = (unsigned) TLI.getDataLayout()->getABITypeAlignment(Ty);
+    unsigned NewAlign = (unsigned)DAG.getDataLayout().getABITypeAlignment(Ty);
 
     // Don't promote to an alignment that would require dynamic stack
     // realignment.
     const TargetRegisterInfo *TRI = MF.getSubtarget().getRegisterInfo();
     if (!TRI->needsStackRealignment(MF))
-       while (NewAlign > Align &&
-             TLI.getDataLayout()->exceedsNaturalStackAlignment(NewAlign))
+      while (NewAlign > Align &&
+             DAG.getDataLayout().exceedsNaturalStackAlignment(NewAlign))
           NewAlign /= 2;
 
     if (NewAlign > Align) {
@@ -4134,7 +4296,7 @@ static SDValue getMemmoveLoadsAndStores(SelectionDAG &DAG, SDLoc dl,
   bool DstAlignCanChange = false;
   MachineFunction &MF = DAG.getMachineFunction();
   MachineFrameInfo *MFI = MF.getFrameInfo();
-  bool OptSize = MF.getFunction()->hasFnAttribute(Attribute::OptimizeForSize);
+  bool OptSize = shouldLowerMemFuncForSize(MF);
   FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Dst);
   if (FI && !MFI->isFixedObjectIndex(FI->getIndex()))
     DstAlignCanChange = true;
@@ -4150,7 +4312,7 @@ static SDValue getMemmoveLoadsAndStores(SelectionDAG &DAG, SDLoc dl,
 
   if (DstAlignCanChange) {
     Type *Ty = MemOps[0].getTypeForEVT(*DAG.getContext());
-    unsigned NewAlign = (unsigned) TLI.getDataLayout()->getABITypeAlignment(Ty);
+    unsigned NewAlign = (unsigned)DAG.getDataLayout().getABITypeAlignment(Ty);
     if (NewAlign > Align) {
       // Give the stack frame object a larger alignment if needed.
       if (MFI->getObjectAlignment(FI->getIndex()) < NewAlign)
@@ -4228,7 +4390,7 @@ static SDValue getMemsetStores(SelectionDAG &DAG, SDLoc dl,
   bool DstAlignCanChange = false;
   MachineFunction &MF = DAG.getMachineFunction();
   MachineFrameInfo *MFI = MF.getFrameInfo();
-  bool OptSize = MF.getFunction()->hasFnAttribute(Attribute::OptimizeForSize);
+  bool OptSize = shouldLowerMemFuncForSize(MF);
   FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Dst);
   if (FI && !MFI->isFixedObjectIndex(FI->getIndex()))
     DstAlignCanChange = true;
@@ -4241,7 +4403,7 @@ static SDValue getMemsetStores(SelectionDAG &DAG, SDLoc dl,
 
   if (DstAlignCanChange) {
     Type *Ty = MemOps[0].getTypeForEVT(*DAG.getContext());
-    unsigned NewAlign = (unsigned) TLI.getDataLayout()->getABITypeAlignment(Ty);
+    unsigned NewAlign = (unsigned)DAG.getDataLayout().getABITypeAlignment(Ty);
     if (NewAlign > Align) {
       // Give the stack frame object a larger alignment if needed.
       if (MFI->getObjectAlignment(FI->getIndex()) < NewAlign)
@@ -4344,19 +4506,21 @@ SDValue SelectionDAG::getMemcpy(SDValue Chain, SDLoc dl, SDValue Dst,
   // Emit a library call.
   TargetLowering::ArgListTy Args;
   TargetLowering::ArgListEntry Entry;
-  Entry.Ty = TLI->getDataLayout()->getIntPtrType(*getContext());
+  Entry.Ty = getDataLayout().getIntPtrType(*getContext());
   Entry.Node = Dst; Args.push_back(Entry);
   Entry.Node = Src; Args.push_back(Entry);
   Entry.Node = Size; Args.push_back(Entry);
   // FIXME: pass in SDLoc
   TargetLowering::CallLoweringInfo CLI(*this);
-  CLI.setDebugLoc(dl).setChain(Chain)
-    .setCallee(TLI->getLibcallCallingConv(RTLIB::MEMCPY),
-               Type::getVoidTy(*getContext()),
-               getExternalSymbol(TLI->getLibcallName(RTLIB::MEMCPY),
-                                 TLI->getPointerTy()), std::move(Args), 0)
-    .setDiscardResult()
-    .setTailCall(isTailCall);
+  CLI.setDebugLoc(dl)
+      .setChain(Chain)
+      .setCallee(TLI->getLibcallCallingConv(RTLIB::MEMCPY),
+                 Type::getVoidTy(*getContext()),
+                 getExternalSymbol(TLI->getLibcallName(RTLIB::MEMCPY),
+                                   TLI->getPointerTy(getDataLayout())),
+                 std::move(Args), 0)
+      .setDiscardResult()
+      .setTailCall(isTailCall);
 
   std::pair<SDValue,SDValue> CallResult = TLI->LowerCallTo(CLI);
   return CallResult.second;
@@ -4400,19 +4564,21 @@ SDValue SelectionDAG::getMemmove(SDValue Chain, SDLoc dl, SDValue Dst,
   // Emit a library call.
   TargetLowering::ArgListTy Args;
   TargetLowering::ArgListEntry Entry;
-  Entry.Ty = TLI->getDataLayout()->getIntPtrType(*getContext());
+  Entry.Ty = getDataLayout().getIntPtrType(*getContext());
   Entry.Node = Dst; Args.push_back(Entry);
   Entry.Node = Src; Args.push_back(Entry);
   Entry.Node = Size; Args.push_back(Entry);
   // FIXME:  pass in SDLoc
   TargetLowering::CallLoweringInfo CLI(*this);
-  CLI.setDebugLoc(dl).setChain(Chain)
-    .setCallee(TLI->getLibcallCallingConv(RTLIB::MEMMOVE),
-               Type::getVoidTy(*getContext()),
-               getExternalSymbol(TLI->getLibcallName(RTLIB::MEMMOVE),
-                                 TLI->getPointerTy()), std::move(Args), 0)
-    .setDiscardResult()
-    .setTailCall(isTailCall);
+  CLI.setDebugLoc(dl)
+      .setChain(Chain)
+      .setCallee(TLI->getLibcallCallingConv(RTLIB::MEMMOVE),
+                 Type::getVoidTy(*getContext()),
+                 getExternalSymbol(TLI->getLibcallName(RTLIB::MEMMOVE),
+                                   TLI->getPointerTy(getDataLayout())),
+                 std::move(Args), 0)
+      .setDiscardResult()
+      .setTailCall(isTailCall);
 
   std::pair<SDValue,SDValue> CallResult = TLI->LowerCallTo(CLI);
   return CallResult.second;
@@ -4450,7 +4616,7 @@ SDValue SelectionDAG::getMemset(SDValue Chain, SDLoc dl, SDValue Dst,
   }
 
   // Emit a library call.
-  Type *IntPtrTy = TLI->getDataLayout()->getIntPtrType(*getContext());
+  Type *IntPtrTy = getDataLayout().getIntPtrType(*getContext());
   TargetLowering::ArgListTy Args;
   TargetLowering::ArgListEntry Entry;
   Entry.Node = Dst; Entry.Ty = IntPtrTy;
@@ -4464,13 +4630,15 @@ SDValue SelectionDAG::getMemset(SDValue Chain, SDLoc dl, SDValue Dst,
 
   // FIXME: pass in SDLoc
   TargetLowering::CallLoweringInfo CLI(*this);
-  CLI.setDebugLoc(dl).setChain(Chain)
-    .setCallee(TLI->getLibcallCallingConv(RTLIB::MEMSET),
-               Type::getVoidTy(*getContext()),
-               getExternalSymbol(TLI->getLibcallName(RTLIB::MEMSET),
-                                 TLI->getPointerTy()), std::move(Args), 0)
-    .setDiscardResult()
-    .setTailCall(isTailCall);
+  CLI.setDebugLoc(dl)
+      .setChain(Chain)
+      .setCallee(TLI->getLibcallCallingConv(RTLIB::MEMSET),
+                 Type::getVoidTy(*getContext()),
+                 getExternalSymbol(TLI->getLibcallName(RTLIB::MEMSET),
+                                   TLI->getPointerTy(getDataLayout())),
+                 std::move(Args), 0)
+      .setDiscardResult()
+      .setTailCall(isTailCall);
 
   std::pair<SDValue,SDValue> CallResult = TLI->LowerCallTo(CLI);
   return CallResult.second;
@@ -4487,7 +4655,7 @@ SDValue SelectionDAG::getAtomic(unsigned Opcode, SDLoc dl, EVT MemVT,
   AddNodeIDNode(ID, Opcode, VTList, Ops);
   ID.AddInteger(MMO->getPointerInfo().getAddrSpace());
   void* IP = nullptr;
-  if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) {
+  if (SDNode *E = FindNodeOrInsertPos(ID, dl.getDebugLoc(), IP)) {
     cast<AtomicSDNode>(E)->refineAlignment(MMO);
     return SDValue(E, 0);
   }
@@ -4692,7 +4860,7 @@ SelectionDAG::getMemIntrinsicNode(unsigned Opcode, SDLoc dl, SDVTList VTList,
     AddNodeIDNode(ID, Opcode, VTList, Ops);
     ID.AddInteger(MMO->getPointerInfo().getAddrSpace());
     void *IP = nullptr;
-    if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) {
+    if (SDNode *E = FindNodeOrInsertPos(ID, dl.getDebugLoc(), IP)) {
       cast<MemIntrinsicSDNode>(E)->refineAlignment(MMO);
       return SDValue(E, 0);
     }
@@ -4814,7 +4982,7 @@ SelectionDAG::getLoad(ISD::MemIndexedMode AM, ISD::LoadExtType ExtType,
                                      MMO->isInvariant()));
   ID.AddInteger(MMO->getPointerInfo().getAddrSpace());
   void *IP = nullptr;
-  if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) {
+  if (SDNode *E = FindNodeOrInsertPos(ID, dl.getDebugLoc(), IP)) {
     cast<LoadSDNode>(E)->refineAlignment(MMO);
     return SDValue(E, 0);
   }
@@ -4922,7 +5090,7 @@ SDValue SelectionDAG::getStore(SDValue Chain, SDLoc dl, SDValue Val,
                                      MMO->isNonTemporal(), MMO->isInvariant()));
   ID.AddInteger(MMO->getPointerInfo().getAddrSpace());
   void *IP = nullptr;
-  if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) {
+  if (SDNode *E = FindNodeOrInsertPos(ID, dl.getDebugLoc(), IP)) {
     cast<StoreSDNode>(E)->refineAlignment(MMO);
     return SDValue(E, 0);
   }
@@ -4991,7 +5159,7 @@ SDValue SelectionDAG::getTruncStore(SDValue Chain, SDLoc dl, SDValue Val,
                                      MMO->isNonTemporal(), MMO->isInvariant()));
   ID.AddInteger(MMO->getPointerInfo().getAddrSpace());
   void *IP = nullptr;
-  if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) {
+  if (SDNode *E = FindNodeOrInsertPos(ID, dl.getDebugLoc(), IP)) {
     cast<StoreSDNode>(E)->refineAlignment(MMO);
     return SDValue(E, 0);
   }
@@ -5017,7 +5185,7 @@ SelectionDAG::getIndexedStore(SDValue OrigStore, SDLoc dl, SDValue Base,
   ID.AddInteger(ST->getRawSubclassData());
   ID.AddInteger(ST->getPointerInfo().getAddrSpace());
   void *IP = nullptr;
-  if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP))
+  if (SDNode *E = FindNodeOrInsertPos(ID, dl.getDebugLoc(), IP))
     return SDValue(E, 0);
 
   SDNode *N = new (NodeAllocator) StoreSDNode(Ops, dl.getIROrder(),
@@ -5046,7 +5214,7 @@ SelectionDAG::getMaskedLoad(EVT VT, SDLoc dl, SDValue Chain,
                                      MMO->isInvariant()));
   ID.AddInteger(MMO->getPointerInfo().getAddrSpace());
   void *IP = nullptr;
-  if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) {
+  if (SDNode *E = FindNodeOrInsertPos(ID, dl.getDebugLoc(), IP)) {
     cast<MaskedLoadSDNode>(E)->refineAlignment(MMO);
     return SDValue(E, 0);
   }
@@ -5073,7 +5241,7 @@ SDValue SelectionDAG::getMaskedStore(SDValue Chain, SDLoc dl, SDValue Val,
                                      MMO->isNonTemporal(), MMO->isInvariant()));
   ID.AddInteger(MMO->getPointerInfo().getAddrSpace());
   void *IP = nullptr;
-  if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) {
+  if (SDNode *E = FindNodeOrInsertPos(ID, dl.getDebugLoc(), IP)) {
     cast<MaskedStoreSDNode>(E)->refineAlignment(MMO);
     return SDValue(E, 0);
   }
@@ -5085,11 +5253,60 @@ SDValue SelectionDAG::getMaskedStore(SDValue Chain, SDLoc dl, SDValue Val,
   return SDValue(N, 0);
 }
 
+SDValue
+SelectionDAG::getMaskedGather(SDVTList VTs, EVT VT, SDLoc dl,
+                              ArrayRef<SDValue> Ops,
+                              MachineMemOperand *MMO) {
+
+  FoldingSetNodeID ID;
+  AddNodeIDNode(ID, ISD::MGATHER, VTs, Ops);
+  ID.AddInteger(VT.getRawBits());
+  ID.AddInteger(encodeMemSDNodeFlags(ISD::NON_EXTLOAD, ISD::UNINDEXED,
+                                     MMO->isVolatile(),
+                                     MMO->isNonTemporal(),
+                                     MMO->isInvariant()));
+  ID.AddInteger(MMO->getPointerInfo().getAddrSpace());
+  void *IP = nullptr;
+  if (SDNode *E = FindNodeOrInsertPos(ID, dl.getDebugLoc(), IP)) {
+    cast<MaskedGatherSDNode>(E)->refineAlignment(MMO);
+    return SDValue(E, 0);
+  }
+  MaskedGatherSDNode *N = 
+    new (NodeAllocator) MaskedGatherSDNode(dl.getIROrder(), dl.getDebugLoc(),
+                                           Ops, VTs, VT, MMO);
+  CSEMap.InsertNode(N, IP);
+  InsertNode(N);
+  return SDValue(N, 0);
+}
+
+SDValue SelectionDAG::getMaskedScatter(SDVTList VTs, EVT VT, SDLoc dl,
+                                       ArrayRef<SDValue> Ops,
+                                       MachineMemOperand *MMO) {
+  FoldingSetNodeID ID;
+  AddNodeIDNode(ID, ISD::MSCATTER, VTs, Ops);
+  ID.AddInteger(VT.getRawBits());
+  ID.AddInteger(encodeMemSDNodeFlags(false, ISD::UNINDEXED, MMO->isVolatile(),
+                                     MMO->isNonTemporal(),
+                                     MMO->isInvariant()));
+  ID.AddInteger(MMO->getPointerInfo().getAddrSpace());
+  void *IP = nullptr;
+  if (SDNode *E = FindNodeOrInsertPos(ID, dl.getDebugLoc(), IP)) {
+    cast<MaskedScatterSDNode>(E)->refineAlignment(MMO);
+    return SDValue(E, 0);
+  }
+  SDNode *N =
+    new (NodeAllocator) MaskedScatterSDNode(dl.getIROrder(), dl.getDebugLoc(),
+                                            Ops, VTs, VT, MMO);
+  CSEMap.InsertNode(N, IP);
+  InsertNode(N);
+  return SDValue(N, 0);
+}
+
 SDValue SelectionDAG::getVAArg(EVT VT, SDLoc dl,
                                SDValue Chain, SDValue Ptr,
                                SDValue SV,
                                unsigned Align) {
-  SDValue Ops[] = { Chain, Ptr, SV, getTargetConstant(Align, MVT::i32) };
+  SDValue Ops[] = { Chain, Ptr, SV, getTargetConstant(Align, dl, MVT::i32) };
   return getNode(ISD::VAARG, dl, getVTList(VT, MVT::Other), Ops);
 }
 
@@ -5149,7 +5366,7 @@ SDValue SelectionDAG::getNode(unsigned Opcode, SDLoc DL, EVT VT,
     AddNodeIDNode(ID, Opcode, VTs, Ops);
     void *IP = nullptr;
 
-    if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP))
+    if (SDNode *E = FindNodeOrInsertPos(ID, DL.getDebugLoc(), IP))
       return SDValue(E, 0);
 
     N = new (NodeAllocator) SDNode(Opcode, DL.getIROrder(), DL.getDebugLoc(),
@@ -5204,7 +5421,7 @@ SDValue SelectionDAG::getNode(unsigned Opcode, SDLoc DL, SDVTList VTList,
     FoldingSetNodeID ID;
     AddNodeIDNode(ID, Opcode, VTList, Ops);
     void *IP = nullptr;
-    if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP))
+    if (SDNode *E = FindNodeOrInsertPos(ID, DL.getDebugLoc(), IP))
       return SDValue(E, 0);
 
     if (NumOps == 1) {
@@ -5630,7 +5847,7 @@ SDNode *SelectionDAG::MorphNodeTo(SDNode *N, unsigned Opc,
   if (VTs.VTs[VTs.NumVTs-1] != MVT::Glue) {
     FoldingSetNodeID ID;
     AddNodeIDNode(ID, Opc, VTs, Ops);
-    if (SDNode *ON = CSEMap.FindNodeOrInsertPos(ID, IP))
+    if (SDNode *ON = FindNodeOrInsertPos(ID, N->getDebugLoc(), IP))
       return UpdadeSDLocOnMergedSDNode(ON, SDLoc(N));
   }
 
@@ -5836,7 +6053,7 @@ SelectionDAG::getMachineNode(unsigned Opcode, SDLoc DL, SDVTList VTs,
     FoldingSetNodeID ID;
     AddNodeIDNode(ID, ~Opcode, VTs, OpsArray);
     IP = nullptr;
-    if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) {
+    if (SDNode *E = FindNodeOrInsertPos(ID, DL.getDebugLoc(), IP)) {
       return cast<MachineSDNode>(UpdadeSDLocOnMergedSDNode(E, DL));
     }
   }
@@ -5868,7 +6085,7 @@ SelectionDAG::getMachineNode(unsigned Opcode, SDLoc DL, SDVTList VTs,
 SDValue
 SelectionDAG::getTargetExtractSubreg(int SRIdx, SDLoc DL, EVT VT,
                                      SDValue Operand) {
-  SDValue SRIdxVal = getTargetConstant(SRIdx, MVT::i32);
+  SDValue SRIdxVal = getTargetConstant(SRIdx, DL, MVT::i32);
   SDNode *Subreg = getMachineNode(TargetOpcode::EXTRACT_SUBREG, DL,
                                   VT, Operand, SRIdxVal);
   return SDValue(Subreg, 0);
@@ -5879,7 +6096,7 @@ SelectionDAG::getTargetExtractSubreg(int SRIdx, SDLoc DL, EVT VT,
 SDValue
 SelectionDAG::getTargetInsertSubreg(int SRIdx, SDLoc DL, EVT VT,
                                     SDValue Operand, SDValue Subreg) {
-  SDValue SRIdxVal = getTargetConstant(SRIdx, MVT::i32);
+  SDValue SRIdxVal = getTargetConstant(SRIdx, DL, MVT::i32);
   SDNode *Result = getMachineNode(TargetOpcode::INSERT_SUBREG, DL,
                                   VT, Operand, Subreg, SRIdxVal);
   return SDValue(Result, 0);
@@ -5888,15 +6105,14 @@ SelectionDAG::getTargetInsertSubreg(int SRIdx, SDLoc DL, EVT VT,
 /// getNodeIfExists - Get the specified node if it's already available, or
 /// else return NULL.
 SDNode *SelectionDAG::getNodeIfExists(unsigned Opcode, SDVTList VTList,
-                                      ArrayRef<SDValue> Ops, bool nuw, bool nsw,
-                                      bool exact) {
+                                      ArrayRef<SDValue> Ops,
+                                      const SDNodeFlags *Flags) {
   if (VTList.VTs[VTList.NumVTs - 1] != MVT::Glue) {
     FoldingSetNodeID ID;
     AddNodeIDNode(ID, Opcode, VTList, Ops);
-    if (isBinOpWithFlags(Opcode))
-      AddBinaryNodeIDCustom(ID, nuw, nsw, exact);
+    AddNodeIDFlags(ID, Opcode, Flags);
     void *IP = nullptr;
-    if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP))
+    if (SDNode *E = FindNodeOrInsertPos(ID, DebugLoc(), IP))
       return E;
   }
   return nullptr;
@@ -5908,27 +6124,28 @@ SDNode *SelectionDAG::getNodeIfExists(unsigned Opcode, SDVTList VTList,
 SDDbgValue *SelectionDAG::getDbgValue(MDNode *Var, MDNode *Expr, SDNode *N,
                                       unsigned R, bool IsIndirect, uint64_t Off,
                                       DebugLoc DL, unsigned O) {
-  assert(cast<MDLocalVariable>(Var)->isValidLocationForIntrinsic(DL) &&
+  assert(cast<DILocalVariable>(Var)->isValidLocationForIntrinsic(DL) &&
          "Expected inlined-at fields to agree");
-  return new (Allocator) SDDbgValue(Var, Expr, N, R, IsIndirect, Off, DL, O);
+  return new (DbgInfo->getAlloc())
+      SDDbgValue(Var, Expr, N, R, IsIndirect, Off, DL, O);
 }
 
 /// Constant
 SDDbgValue *SelectionDAG::getConstantDbgValue(MDNode *Var, MDNode *Expr,
                                               const Value *C, uint64_t Off,
                                               DebugLoc DL, unsigned O) {
-  assert(cast<MDLocalVariable>(Var)->isValidLocationForIntrinsic(DL) &&
+  assert(cast<DILocalVariable>(Var)->isValidLocationForIntrinsic(DL) &&
          "Expected inlined-at fields to agree");
-  return new (Allocator) SDDbgValue(Var, Expr, C, Off, DL, O);
+  return new (DbgInfo->getAlloc()) SDDbgValue(Var, Expr, C, Off, DL, O);
 }
 
 /// FrameIndex
 SDDbgValue *SelectionDAG::getFrameIndexDbgValue(MDNode *Var, MDNode *Expr,
                                                 unsigned FI, uint64_t Off,
                                                 DebugLoc DL, unsigned O) {
-  assert(cast<MDLocalVariable>(Var)->isValidLocationForIntrinsic(DL) &&
+  assert(cast<DILocalVariable>(Var)->isValidLocationForIntrinsic(DL) &&
          "Expected inlined-at fields to agree");
-  return new (Allocator) SDDbgValue(Var, Expr, FI, Off, DL, O);
+  return new (DbgInfo->getAlloc()) SDDbgValue(Var, Expr, FI, Off, DL, O);
 }
 
 namespace {
@@ -6263,8 +6480,8 @@ unsigned SelectionDAG::AssignTopologicalOrder() {
 
   // Visit all the nodes. As we iterate, move nodes into sorted order,
   // such that by the time the end is reached all nodes will be sorted.
-  for (allnodes_iterator I = allnodes_begin(),E = allnodes_end(); I != E; ++I) {
-    SDNode *N = I;
+  for (SDNode &Node : allnodes()) {
+    SDNode *N = &Node;
     checkForCycles(N, this);
     // N is in sorted position, so all its uses have one less operand
     // that needs to be sorted.
@@ -6286,8 +6503,9 @@ unsigned SelectionDAG::AssignTopologicalOrder() {
         P->setNodeId(Degree);
       }
     }
-    if (I == SortedPos) {
+    if (&Node == SortedPos) {
 #ifndef NDEBUG
+      allnodes_iterator I = N;
       SDNode *S = ++I;
       dbgs() << "Overran sorted position:\n";
       S->dumprFull(this); dbgs() << "\n";
@@ -6463,7 +6681,7 @@ bool SDNode::hasAnyUseOfValue(unsigned Value) const {
 
 /// isOnlyUserOf - Return true if this node is the only use of N.
 ///
-bool SDNode::isOnlyUserOf(SDNode *N) const {
+bool SDNode::isOnlyUserOf(const SDNode *N) const {
   bool Seen = false;
   for (SDNode::use_iterator I = N->use_begin(), E = N->use_end(); I != E; ++I) {
     SDNode *User = *I;
@@ -6478,16 +6696,16 @@ bool SDNode::isOnlyUserOf(SDNode *N) const {
 
 /// isOperand - Return true if this node is an operand of N.
 ///
-bool SDValue::isOperandOf(SDNode *N) const {
-  for (unsigned i = 0, e = N->getNumOperands(); i != e; ++i)
-    if (*this == N->getOperand(i))
+bool SDValue::isOperandOf(const SDNode *N) const {
+  for (const SDValue &Op : N->op_values())
+    if (*this == Op)
       return true;
   return false;
 }
 
-bool SDNode::isOperandOf(SDNode *N) const {
-  for (unsigned i = 0, e = N->NumOperands; i != e; ++i)
-    if (this == N->OperandList[i].getNode())
+bool SDNode::isOperandOf(const SDNode *N) const {
+  for (const SDValue &Op : N->op_values())
+    if (this == Op.getNode())
       return true;
   return false;
 }
@@ -6548,8 +6766,8 @@ SDNode::hasPredecessorHelper(const SDNode *N,
   // Haven't visited N yet. Continue the search.
   while (!Worklist.empty()) {
     const SDNode *M = Worklist.pop_back_val();
-    for (unsigned i = 0, e = M->getNumOperands(); i != e; ++i) {
-      SDNode *Op = M->getOperand(i).getNode();
+    for (const SDValue &OpV : M->op_values()) {
+      SDNode *Op = OpV.getNode();
       if (Visited.insert(Op).second)
         Worklist.push_back(Op);
       if (Op == N)
@@ -6591,10 +6809,9 @@ SDValue SelectionDAG::UnrollVectorOp(SDNode *N, unsigned ResNE) {
       if (OperandVT.isVector()) {
         // A vector operand; extract a single element.
         EVT OperandEltVT = OperandVT.getVectorElementType();
-        Operands[j] = getNode(ISD::EXTRACT_VECTOR_ELT, dl,
-                              OperandEltVT,
-                              Operand,
-                              getConstant(i, TLI->getVectorIdxTy()));
+        Operands[j] =
+            getNode(ISD::EXTRACT_VECTOR_ELT, dl, OperandEltVT, Operand,
+                    getConstant(i, dl, TLI->getVectorIdxTy(getDataLayout())));
       } else {
         // A scalar operand; just use it as is.
         Operands[j] = Operand;
@@ -6698,10 +6915,10 @@ unsigned SelectionDAG::InferPtrAlignment(SDValue Ptr) const {
   const GlobalValue *GV;
   int64_t GVOffset = 0;
   if (TLI->isGAPlusOffset(Ptr.getNode(), GV, GVOffset)) {
-    unsigned PtrWidth = TLI->getPointerTypeSizeInBits(GV->getType());
+    unsigned PtrWidth = getDataLayout().getPointerTypeSizeInBits(GV->getType());
     APInt KnownZero(PtrWidth, 0), KnownOne(PtrWidth, 0);
     llvm::computeKnownBits(const_cast<GlobalValue *>(GV), KnownZero, KnownOne,
-                           *TLI->getDataLayout());
+                           getDataLayout());
     unsigned AlignBits = KnownZero.countTrailingOnes();
     unsigned Align = AlignBits ? 1 << std::min(31U, AlignBits) : 0;
     if (Align)
@@ -6757,9 +6974,10 @@ SelectionDAG::SplitVector(const SDValue &N, const SDLoc &DL, const EVT &LoVT,
          "More vector elements requested than available!");
   SDValue Lo, Hi;
   Lo = getNode(ISD::EXTRACT_SUBVECTOR, DL, LoVT, N,
-               getConstant(0, TLI->getVectorIdxTy()));
+               getConstant(0, DL, TLI->getVectorIdxTy(getDataLayout())));
   Hi = getNode(ISD::EXTRACT_SUBVECTOR, DL, HiVT, N,
-               getConstant(LoVT.getVectorNumElements(), TLI->getVectorIdxTy()));
+               getConstant(LoVT.getVectorNumElements(), DL,
+                           TLI->getVectorIdxTy(getDataLayout())));
   return std::make_pair(Lo, Hi);
 }
 
@@ -6771,11 +6989,11 @@ void SelectionDAG::ExtractVectorElements(SDValue Op,
     Count = VT.getVectorNumElements();
 
   EVT EltVT = VT.getVectorElementType();
-  EVT IdxTy = TLI->getVectorIdxTy();
+  EVT IdxTy = TLI->getVectorIdxTy(getDataLayout());
   SDLoc SL(Op);
   for (unsigned i = Start, e = Start + Count; i != e; ++i) {
     Args.push_back(getNode(ISD::EXTRACT_VECTOR_ELT, SL, EltVT,
-                           Op, getConstant(i, IdxTy)));
+                           Op, getConstant(i, SL, IdxTy)));
   }
 }
 
@@ -6886,19 +7104,17 @@ SDValue BuildVectorSDNode::getSplatValue(BitVector *UndefElements) const {
 
 ConstantSDNode *
 BuildVectorSDNode::getConstantSplatNode(BitVector *UndefElements) const {
-  return dyn_cast_or_null<ConstantSDNode>(
-      getSplatValue(UndefElements).getNode());
+  return dyn_cast_or_null<ConstantSDNode>(getSplatValue(UndefElements));
 }
 
 ConstantFPSDNode *
 BuildVectorSDNode::getConstantFPSplatNode(BitVector *UndefElements) const {
-  return dyn_cast_or_null<ConstantFPSDNode>(
-      getSplatValue(UndefElements).getNode());
+  return dyn_cast_or_null<ConstantFPSDNode>(getSplatValue(UndefElements));
 }
 
 bool BuildVectorSDNode::isConstant() const {
-  for (unsigned i = 0, e = getNumOperands(); i != e; ++i) {
-    unsigned Opc = getOperand(i).getOpcode();
+  for (const SDValue &Op : op_values()) {
+    unsigned Opc = Op.getOpcode();
     if (Opc != ISD::UNDEF && Opc != ISD::Constant && Opc != ISD::ConstantFP)
       return false;
   }
@@ -6939,8 +7155,8 @@ static void checkForCyclesHelper(const SDNode *N,
     abort();
   }
 
-  for(unsigned i = 0, e = N->getNumOperands(); i != e; ++i)
-    checkForCyclesHelper(N->getOperand(i).getNode(), Visited, Checked, DAG);
+  for (const SDValue &Op : N->op_values())
+    checkForCyclesHelper(Op.getNode(), Visited, Checked, DAG);
 
   Checked.insert(N);
   Visited.erase(N);
diff --git a/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp b/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
index 32d2aae488e..f15c95f4d5e 100644
--- a/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
+++ b/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
@@ -22,7 +22,6 @@
 #include "llvm/Analysis/ConstantFolding.h"
 #include "llvm/Analysis/TargetLibraryInfo.h"
 #include "llvm/Analysis/ValueTracking.h"
-#include "llvm/CodeGen/Analysis.h"
 #include "llvm/CodeGen/FastISel.h"
 #include "llvm/CodeGen/FunctionLoweringInfo.h"
 #include "llvm/CodeGen/GCMetadata.h"
@@ -79,12 +78,16 @@ LimitFPPrecision("limit-float-precision",
                  cl::location(LimitFloatPrecision),
                  cl::init(0));
 
+static cl::opt<bool>
+EnableFMFInDAG("enable-fmf-dag", cl::init(true), cl::Hidden,
+                cl::desc("Enable fast-math-flags for DAG nodes"));
+
 // Limit the width of DAG chains. This is important in general to prevent
-// prevent DAG-based analysis from blowing up. For example, alias analysis and
+// DAG-based analysis from blowing up. For example, alias analysis and
 // load clustering may not complete in reasonable time. It is difficult to
 // recognize and avoid this situation within each individual analysis, and
 // future analyses are likely to have the same behavior. Limiting DAG width is
-// the safe approach, and will be especially important with global DAGs.
+// the safe approach and will be especially important with global DAGs.
 //
 // MaxParallelChains default is arbitrarily high to avoid affecting
 // optimization, but could be lowered to improve compile time. Any ld-ld-st-st
@@ -143,7 +146,7 @@ static SDValue getCopyFromParts(SelectionDAG &DAG, SDLoc DL,
         Hi = DAG.getNode(ISD::BITCAST, DL, HalfVT, Parts[1]);
       }
 
-      if (TLI.isBigEndian())
+      if (DAG.getDataLayout().isBigEndian())
         std::swap(Lo, Hi);
 
       Val = DAG.getNode(ISD::BUILD_PAIR, DL, RoundVT, Lo, Hi);
@@ -157,13 +160,14 @@ static SDValue getCopyFromParts(SelectionDAG &DAG, SDLoc DL,
 
         // Combine the round and odd parts.
         Lo = Val;
-        if (TLI.isBigEndian())
+        if (DAG.getDataLayout().isBigEndian())
           std::swap(Lo, Hi);
         EVT TotalVT = EVT::getIntegerVT(*DAG.getContext(), NumParts * PartBits);
         Hi = DAG.getNode(ISD::ANY_EXTEND, DL, TotalVT, Hi);
-        Hi = DAG.getNode(ISD::SHL, DL, TotalVT, Hi,
-                         DAG.getConstant(Lo.getValueType().getSizeInBits(),
-                                         TLI.getPointerTy()));
+        Hi =
+            DAG.getNode(ISD::SHL, DL, TotalVT, Hi,
+                        DAG.getConstant(Lo.getValueType().getSizeInBits(), DL,
+                                        TLI.getPointerTy(DAG.getDataLayout())));
         Lo = DAG.getNode(ISD::ZERO_EXTEND, DL, TotalVT, Lo);
         Val = DAG.getNode(ISD::OR, DL, TotalVT, Lo, Hi);
       }
@@ -174,7 +178,7 @@ static SDValue getCopyFromParts(SelectionDAG &DAG, SDLoc DL,
       SDValue Lo, Hi;
       Lo = DAG.getNode(ISD::BITCAST, DL, EVT(MVT::f64), Parts[0]);
       Hi = DAG.getNode(ISD::BITCAST, DL, EVT(MVT::f64), Parts[1]);
-      if (TLI.hasBigEndianPartOrdering(ValueVT))
+      if (TLI.hasBigEndianPartOrdering(ValueVT, DAG.getDataLayout()))
         std::swap(Lo, Hi);
       Val = DAG.getNode(ISD::BUILD_PAIR, DL, ValueVT, Lo, Hi);
     } else {
@@ -208,8 +212,9 @@ static SDValue getCopyFromParts(SelectionDAG &DAG, SDLoc DL,
   if (PartEVT.isFloatingPoint() && ValueVT.isFloatingPoint()) {
     // FP_ROUND's are always exact here.
     if (ValueVT.bitsLT(Val.getValueType()))
-      return DAG.getNode(ISD::FP_ROUND, DL, ValueVT, Val,
-                         DAG.getTargetConstant(1, TLI.getPointerTy()));
+      return DAG.getNode(
+          ISD::FP_ROUND, DL, ValueVT, Val,
+          DAG.getTargetConstant(1, DL, TLI.getPointerTy(DAG.getDataLayout())));
 
     return DAG.getNode(ISD::FP_EXTEND, DL, ValueVT, Val);
   }
@@ -258,8 +263,9 @@ static SDValue getCopyFromPartsVector(SelectionDAG &DAG, SDLoc DL,
     assert(NumRegs == NumParts && "Part count doesn't match vector breakdown!");
     NumParts = NumRegs; // Silence a compiler warning.
     assert(RegisterVT == PartVT && "Part type doesn't match vector breakdown!");
-    assert(RegisterVT == Parts[0].getSimpleValueType() &&
-           "Part type doesn't match part!");
+    assert(RegisterVT.getSizeInBits() ==
+           Parts[0].getSimpleValueType().getSizeInBits() &&
+           "Part type sizes don't match!");
 
     // Assemble the parts into intermediate operands.
     SmallVector<SDValue, 8> Ops(NumIntermediates);
@@ -301,8 +307,9 @@ static SDValue getCopyFromPartsVector(SelectionDAG &DAG, SDLoc DL,
     if (PartEVT.getVectorElementType() == ValueVT.getVectorElementType()) {
       assert(PartEVT.getVectorNumElements() > ValueVT.getVectorNumElements() &&
              "Cannot narrow, it would be a lossy transformation");
-      return DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, ValueVT, Val,
-                         DAG.getConstant(0, TLI.getVectorIdxTy()));
+      return DAG.getNode(
+          ISD::EXTRACT_SUBVECTOR, DL, ValueVT, Val,
+          DAG.getConstant(0, DL, TLI.getVectorIdxTy(DAG.getDataLayout())));
     }
 
     // Vector/Vector bitcast.
@@ -312,9 +319,7 @@ static SDValue getCopyFromPartsVector(SelectionDAG &DAG, SDLoc DL,
     assert(PartEVT.getVectorNumElements() == ValueVT.getVectorNumElements() &&
       "Cannot handle this kind of promotion");
     // Promoted vector extract
-    bool Smaller = ValueVT.bitsLE(PartEVT);
-    return DAG.getNode((Smaller ? ISD::TRUNCATE : ISD::ANY_EXTEND),
-                       DL, ValueVT, Val);
+    return DAG.getAnyExtOrTrunc(Val, DL, ValueVT);
 
   }
 
@@ -332,11 +337,8 @@ static SDValue getCopyFromPartsVector(SelectionDAG &DAG, SDLoc DL,
   }
 
   if (ValueVT.getVectorNumElements() == 1 &&
-      ValueVT.getVectorElementType() != PartEVT) {
-    bool Smaller = ValueVT.bitsLE(PartEVT);
-    Val = DAG.getNode((Smaller ? ISD::TRUNCATE : ISD::ANY_EXTEND),
-                       DL, ValueVT.getScalarType(), Val);
-  }
+      ValueVT.getVectorElementType() != PartEVT)
+    Val = DAG.getAnyExtOrTrunc(Val, DL, ValueVT.getScalarType());
 
   return DAG.getNode(ISD::BUILD_VECTOR, DL, ValueVT, Val);
 }
@@ -358,10 +360,10 @@ static void getCopyToParts(SelectionDAG &DAG, SDLoc DL,
   if (ValueVT.isVector())
     return getCopyToPartsVector(DAG, DL, Val, Parts, NumParts, PartVT, V);
 
-  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
   unsigned PartBits = PartVT.getSizeInBits();
   unsigned OrigNumParts = NumParts;
-  assert(TLI.isTypeLegal(PartVT) && "Copying to an illegal type!");
+  assert(DAG.getTargetLoweringInfo().isTypeLegal(PartVT) &&
+         "Copying to an illegal type!");
 
   if (NumParts == 0)
     return;
@@ -426,10 +428,10 @@ static void getCopyToParts(SelectionDAG &DAG, SDLoc DL,
     unsigned RoundBits = RoundParts * PartBits;
     unsigned OddParts = NumParts - RoundParts;
     SDValue OddVal = DAG.getNode(ISD::SRL, DL, ValueVT, Val,
-                                 DAG.getIntPtrConstant(RoundBits));
+                                 DAG.getIntPtrConstant(RoundBits, DL));
     getCopyToParts(DAG, DL, OddVal, Parts + RoundParts, OddParts, PartVT, V);
 
-    if (TLI.isBigEndian())
+    if (DAG.getDataLayout().isBigEndian())
       // The odd parts were reversed by getCopyToParts - unreverse them.
       std::reverse(Parts + RoundParts, Parts + NumParts);
 
@@ -453,9 +455,9 @@ static void getCopyToParts(SelectionDAG &DAG, SDLoc DL,
       SDValue &Part1 = Parts[i+StepSize/2];
 
       Part1 = DAG.getNode(ISD::EXTRACT_ELEMENT, DL,
-                          ThisVT, Part0, DAG.getIntPtrConstant(1));
+                          ThisVT, Part0, DAG.getIntPtrConstant(1, DL));
       Part0 = DAG.getNode(ISD::EXTRACT_ELEMENT, DL,
-                          ThisVT, Part0, DAG.getIntPtrConstant(0));
+                          ThisVT, Part0, DAG.getIntPtrConstant(0, DL));
 
       if (ThisBits == PartBits && ThisVT != PartVT) {
         Part0 = DAG.getNode(ISD::BITCAST, DL, PartVT, Part0);
@@ -464,7 +466,7 @@ static void getCopyToParts(SelectionDAG &DAG, SDLoc DL,
     }
   }
 
-  if (TLI.isBigEndian())
+  if (DAG.getDataLayout().isBigEndian())
     std::reverse(Parts, Parts + OrigNumParts);
 }
 
@@ -493,9 +495,9 @@ static void getCopyToPartsVector(SelectionDAG &DAG, SDLoc DL,
       // undef elements.
       SmallVector<SDValue, 16> Ops;
       for (unsigned i = 0, e = ValueVT.getVectorNumElements(); i != e; ++i)
-        Ops.push_back(DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL,
-                                  ElementVT, Val, DAG.getConstant(i,
-                                                  TLI.getVectorIdxTy())));
+        Ops.push_back(DAG.getNode(
+            ISD::EXTRACT_VECTOR_ELT, DL, ElementVT, Val,
+            DAG.getConstant(i, DL, TLI.getVectorIdxTy(DAG.getDataLayout()))));
 
       for (unsigned i = ValueVT.getVectorNumElements(),
            e = PartVT.getVectorNumElements(); i != e; ++i)
@@ -513,19 +515,16 @@ static void getCopyToPartsVector(SelectionDAG &DAG, SDLoc DL,
                PartEVT.getVectorNumElements() == ValueVT.getVectorNumElements()) {
 
       // Promoted vector extract
-      bool Smaller = PartEVT.bitsLE(ValueVT);
-      Val = DAG.getNode((Smaller ? ISD::TRUNCATE : ISD::ANY_EXTEND),
-                        DL, PartVT, Val);
+      Val = DAG.getAnyExtOrTrunc(Val, DL, PartVT);
     } else{
       // Vector -> scalar conversion.
       assert(ValueVT.getVectorNumElements() == 1 &&
              "Only trivial vector-to-scalar conversions should get here!");
-      Val = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL,
-                        PartVT, Val, DAG.getConstant(0, TLI.getVectorIdxTy()));
+      Val = DAG.getNode(
+          ISD::EXTRACT_VECTOR_ELT, DL, PartVT, Val,
+          DAG.getConstant(0, DL, TLI.getVectorIdxTy(DAG.getDataLayout())));
 
-      bool Smaller = ValueVT.bitsLE(PartVT);
-      Val = DAG.getNode((Smaller ? ISD::TRUNCATE : ISD::ANY_EXTEND),
-                         DL, PartVT, Val);
+      Val = DAG.getAnyExtOrTrunc(Val, DL, PartVT);
     }
 
     Parts[0] = Val;
@@ -549,14 +548,14 @@ static void getCopyToPartsVector(SelectionDAG &DAG, SDLoc DL,
   SmallVector<SDValue, 8> Ops(NumIntermediates);
   for (unsigned i = 0; i != NumIntermediates; ++i) {
     if (IntermediateVT.isVector())
-      Ops[i] = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL,
-                           IntermediateVT, Val,
-                   DAG.getConstant(i * (NumElements / NumIntermediates),
-                                   TLI.getVectorIdxTy()));
+      Ops[i] =
+          DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, IntermediateVT, Val,
+                      DAG.getConstant(i * (NumElements / NumIntermediates), DL,
+                                      TLI.getVectorIdxTy(DAG.getDataLayout())));
     else
-      Ops[i] = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL,
-                           IntermediateVT, Val,
-                           DAG.getConstant(i, TLI.getVectorIdxTy()));
+      Ops[i] = DAG.getNode(
+          ISD::EXTRACT_VECTOR_ELT, DL, IntermediateVT, Val,
+          DAG.getConstant(i, DL, TLI.getVectorIdxTy(DAG.getDataLayout())));
   }
 
   // Split the intermediate operands into legal parts.
@@ -577,93 +576,24 @@ static void getCopyToPartsVector(SelectionDAG &DAG, SDLoc DL,
   }
 }
 
-namespace {
-  /// RegsForValue - This struct represents the registers (physical or virtual)
-  /// that a particular set of values is assigned, and the type information
-  /// about the value. The most common situation is to represent one value at a
-  /// time, but struct or array values are handled element-wise as multiple
-  /// values.  The splitting of aggregates is performed recursively, so that we
-  /// never have aggregate-typed registers. The values at this point do not
-  /// necessarily have legal types, so each value may require one or more
-  /// registers of some legal type.
-  ///
-  struct RegsForValue {
-    /// ValueVTs - The value types of the values, which may not be legal, and
-    /// may need be promoted or synthesized from one or more registers.
-    ///
-    SmallVector<EVT, 4> ValueVTs;
+RegsForValue::RegsForValue() {}
 
-    /// RegVTs - The value types of the registers. This is the same size as
-    /// ValueVTs and it records, for each value, what the type of the assigned
-    /// register or registers are. (Individual values are never synthesized
-    /// from more than one type of register.)
-    ///
-    /// With virtual registers, the contents of RegVTs is redundant with TLI's
-    /// getRegisterType member function, however when with physical registers
-    /// it is necessary to have a separate record of the types.
-    ///
-    SmallVector<MVT, 4> RegVTs;
-
-    /// Regs - This list holds the registers assigned to the values.
-    /// Each legal or promoted value requires one register, and each
-    /// expanded value requires multiple registers.
-    ///
-    SmallVector<unsigned, 4> Regs;
-
-    RegsForValue() {}
-
-    RegsForValue(const SmallVector<unsigned, 4> &regs,
-                 MVT regvt, EVT valuevt)
-      : ValueVTs(1, valuevt), RegVTs(1, regvt), Regs(regs) {}
-
-    RegsForValue(LLVMContext &Context, const TargetLowering &tli,
-                 unsigned Reg, Type *Ty) {
-      ComputeValueVTs(tli, Ty, ValueVTs);
-
-      for (unsigned Value = 0, e = ValueVTs.size(); Value != e; ++Value) {
-        EVT ValueVT = ValueVTs[Value];
-        unsigned NumRegs = tli.getNumRegisters(Context, ValueVT);
-        MVT RegisterVT = tli.getRegisterType(Context, ValueVT);
-        for (unsigned i = 0; i != NumRegs; ++i)
-          Regs.push_back(Reg + i);
-        RegVTs.push_back(RegisterVT);
-        Reg += NumRegs;
-      }
-    }
+RegsForValue::RegsForValue(const SmallVector<unsigned, 4> &regs, MVT regvt,
+                           EVT valuevt)
+    : ValueVTs(1, valuevt), RegVTs(1, regvt), Regs(regs) {}
 
-    /// append - Add the specified values to this one.
-    void append(const RegsForValue &RHS) {
-      ValueVTs.append(RHS.ValueVTs.begin(), RHS.ValueVTs.end());
-      RegVTs.append(RHS.RegVTs.begin(), RHS.RegVTs.end());
-      Regs.append(RHS.Regs.begin(), RHS.Regs.end());
-    }
+RegsForValue::RegsForValue(LLVMContext &Context, const TargetLowering &TLI,
+                           const DataLayout &DL, unsigned Reg, Type *Ty) {
+  ComputeValueVTs(TLI, DL, Ty, ValueVTs);
 
-    /// getCopyFromRegs - Emit a series of CopyFromReg nodes that copies from
-    /// this value and returns the result as a ValueVTs value.  This uses
-    /// Chain/Flag as the input and updates them for the output Chain/Flag.
-    /// If the Flag pointer is NULL, no flag is used.
-    SDValue getCopyFromRegs(SelectionDAG &DAG, FunctionLoweringInfo &FuncInfo,
-                            SDLoc dl,
-                            SDValue &Chain, SDValue *Flag,
-                            const Value *V = nullptr) const;
-
-    /// getCopyToRegs - Emit a series of CopyToReg nodes that copies the
-    /// specified value into the registers specified by this object.  This uses
-    /// Chain/Flag as the input and updates them for the output Chain/Flag.
-    /// If the Flag pointer is NULL, no flag is used.
-    void
-    getCopyToRegs(SDValue Val, SelectionDAG &DAG, SDLoc dl, SDValue &Chain,
-                  SDValue *Flag, const Value *V,
-                  ISD::NodeType PreferredExtendType = ISD::ANY_EXTEND) const;
-
-    /// AddInlineAsmOperands - Add this value to the specified inlineasm node
-    /// operand list.  This adds the code marker, matching input operand index
-    /// (if applicable), and includes the number of values added into it.
-    void AddInlineAsmOperands(unsigned Kind,
-                              bool HasMatching, unsigned MatchingIdx,
-                              SelectionDAG &DAG,
-                              std::vector<SDValue> &Ops) const;
-  };
+  for (EVT ValueVT : ValueVTs) {
+    unsigned NumRegs = TLI.getNumRegisters(Context, ValueVT);
+    MVT RegisterVT = TLI.getRegisterType(Context, ValueVT);
+    for (unsigned i = 0; i != NumRegs; ++i)
+      Regs.push_back(Reg + i);
+    RegVTs.push_back(RegisterVT);
+    Reg += NumRegs;
+  }
 }
 
 /// getCopyFromRegs - Emit a series of CopyFromReg nodes that copies from
@@ -722,7 +652,7 @@ SDValue RegsForValue::getCopyFromRegs(SelectionDAG &DAG,
         // The current value is a zero.
         // Explicitly express that as it would be easier for
         // optimizations to kick in.
-        Parts[i] = DAG.getConstant(0, RegisterVT);
+        Parts[i] = DAG.getConstant(0, dl, RegisterVT);
         continue;
       }
 
@@ -824,7 +754,7 @@ void RegsForValue::getCopyToRegs(SDValue Val, SelectionDAG &DAG, SDLoc dl,
 /// operand list.  This adds the code marker and includes the number of
 /// values added into it.
 void RegsForValue::AddInlineAsmOperands(unsigned Code, bool HasMatching,
-                                        unsigned MatchingIdx,
+                                        unsigned MatchingIdx, SDLoc dl,
                                         SelectionDAG &DAG,
                                         std::vector<SDValue> &Ops) const {
   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
@@ -844,7 +774,7 @@ void RegsForValue::AddInlineAsmOperands(unsigned Code, bool HasMatching,
     Flag = InlineAsm::getFlagWordForRegClass(Flag, RC->getID());
   }
 
-  SDValue Res = DAG.getTargetConstant(Flag, MVT::i32);
+  SDValue Res = DAG.getTargetConstant(Flag, dl, MVT::i32);
   Ops.push_back(Res);
 
   unsigned SP = TLI.getStackPointerRegisterToSaveRestore();
@@ -859,7 +789,7 @@ void RegsForValue::AddInlineAsmOperands(unsigned Code, bool HasMatching,
       if (TheReg == SP && Code == InlineAsm::Kind_Clobber) {
         // If we clobbered the stack pointer, MFI should know about it.
         assert(DAG.getMachineFunction().getFrameInfo()->
-            hasInlineAsmWithSPAdjust());
+            hasOpaqueSPAdjustment());
       }
     }
   }
@@ -870,7 +800,7 @@ void SelectionDAGBuilder::init(GCFunctionInfo *gfi, AliasAnalysis &aa,
   AA = &aa;
   GFI = gfi;
   LibInfo = li;
-  DL = DAG.getTarget().getDataLayout();
+  DL = &DAG.getDataLayout();
   Context = DAG.getContext();
   LPadToCallSiteMap.clear();
 }
@@ -998,8 +928,8 @@ void SelectionDAGBuilder::resolveDanglingDebugInfo(const Value *V,
     const DbgValueInst *DI = DDI.getDI();
     DebugLoc dl = DDI.getdl();
     unsigned DbgSDNodeOrder = DDI.getSDNodeOrder();
-    MDLocalVariable *Variable = DI->getVariable();
-    MDExpression *Expr = DI->getExpression();
+    DILocalVariable *Variable = DI->getVariable();
+    DIExpression *Expr = DI->getExpression();
     assert(Variable->isValidLocationForIntrinsic(dl) &&
            "Expected inlined-at fields to agree");
     uint64_t Offset = DI->getOffset();
@@ -1023,18 +953,18 @@ void SelectionDAGBuilder::resolveDanglingDebugInfo(const Value *V,
 /// emit CopyFromReg of the specified type Ty. Return empty SDValue() otherwise.
 SDValue SelectionDAGBuilder::getCopyFromRegs(const Value *V, Type *Ty) {
   DenseMap<const Value *, unsigned>::iterator It = FuncInfo.ValueMap.find(V);
-  SDValue res;
+  SDValue Result;
 
   if (It != FuncInfo.ValueMap.end()) {
     unsigned InReg = It->second;
-    RegsForValue RFV(*DAG.getContext(), DAG.getTargetLoweringInfo(), InReg,
-                     Ty);
+    RegsForValue RFV(*DAG.getContext(), DAG.getTargetLoweringInfo(),
+                     DAG.getDataLayout(), InReg, Ty);
     SDValue Chain = DAG.getEntryNode();
-    res = RFV.getCopyFromRegs(DAG, FuncInfo, getCurSDLoc(), Chain, nullptr, V);
-    resolveDanglingDebugInfo(V, res);
+    Result = RFV.getCopyFromRegs(DAG, FuncInfo, getCurSDLoc(), Chain, nullptr, V);
+    resolveDanglingDebugInfo(V, Result);
   }
 
-  return res;
+  return Result;
 }
 
 /// getValue - Return an SDValue for the given Value.
@@ -1059,12 +989,27 @@ SDValue SelectionDAGBuilder::getValue(const Value *V) {
   return Val;
 }
 
+// Return true if SDValue exists for the given Value
+bool SelectionDAGBuilder::findValue(const Value *V) const {
+  return (NodeMap.find(V) != NodeMap.end()) ||
+    (FuncInfo.ValueMap.find(V) != FuncInfo.ValueMap.end());
+}
+
 /// getNonRegisterValue - Return an SDValue for the given Value, but
 /// don't look in FuncInfo.ValueMap for a virtual register.
 SDValue SelectionDAGBuilder::getNonRegisterValue(const Value *V) {
   // If we already have an SDValue for this value, use it.
   SDValue &N = NodeMap[V];
-  if (N.getNode()) return N;
+  if (N.getNode()) {
+    if (isa<ConstantSDNode>(N) || isa<ConstantFPSDNode>(N)) {
+      // Remove the debug location from the node as the node is about to be used
+      // in a location which may differ from the original debug location.  This
+      // is relevant to Constant and ConstantFP nodes because they can appear
+      // as constant expressions inside PHI nodes.
+      N->setDebugLoc(DebugLoc());
+    }
+    return N;
+  }
 
   // Otherwise create a new SDValue and remember it.
   SDValue Val = getValueImpl(V);
@@ -1079,21 +1024,22 @@ SDValue SelectionDAGBuilder::getValueImpl(const Value *V) {
   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
 
   if (const Constant *C = dyn_cast<Constant>(V)) {
-    EVT VT = TLI.getValueType(V->getType(), true);
+    EVT VT = TLI.getValueType(DAG.getDataLayout(), V->getType(), true);
 
     if (const ConstantInt *CI = dyn_cast<ConstantInt>(C))
-      return DAG.getConstant(*CI, VT);
+      return DAG.getConstant(*CI, getCurSDLoc(), VT);
 
     if (const GlobalValue *GV = dyn_cast<GlobalValue>(C))
       return DAG.getGlobalAddress(GV, getCurSDLoc(), VT);
 
     if (isa<ConstantPointerNull>(C)) {
       unsigned AS = V->getType()->getPointerAddressSpace();
-      return DAG.getConstant(0, TLI.getPointerTy(AS));
+      return DAG.getConstant(0, getCurSDLoc(),
+                             TLI.getPointerTy(DAG.getDataLayout(), AS));
     }
 
     if (const ConstantFP *CFP = dyn_cast<ConstantFP>(C))
-      return DAG.getConstantFP(*CFP, VT);
+      return DAG.getConstantFP(*CFP, getCurSDLoc(), VT);
 
     if (isa<UndefValue>(C) && !V->getType()->isAggregateType())
       return DAG.getUNDEF(VT);
@@ -1143,7 +1089,7 @@ SDValue SelectionDAGBuilder::getValueImpl(const Value *V) {
              "Unknown struct or array constant!");
 
       SmallVector<EVT, 4> ValueVTs;
-      ComputeValueVTs(TLI, C->getType(), ValueVTs);
+      ComputeValueVTs(TLI, DAG.getDataLayout(), C->getType(), ValueVTs);
       unsigned NumElts = ValueVTs.size();
       if (NumElts == 0)
         return SDValue(); // empty struct
@@ -1153,9 +1099,9 @@ SDValue SelectionDAGBuilder::getValueImpl(const Value *V) {
         if (isa<UndefValue>(C))
           Constants[i] = DAG.getUNDEF(EltVT);
         else if (EltVT.isFloatingPoint())
-          Constants[i] = DAG.getConstantFP(0, EltVT);
+          Constants[i] = DAG.getConstantFP(0, getCurSDLoc(), EltVT);
         else
-          Constants[i] = DAG.getConstant(0, EltVT);
+          Constants[i] = DAG.getConstant(0, getCurSDLoc(), EltVT);
       }
 
       return DAG.getMergeValues(Constants, getCurSDLoc());
@@ -1175,13 +1121,14 @@ SDValue SelectionDAGBuilder::getValueImpl(const Value *V) {
         Ops.push_back(getValue(CV->getOperand(i)));
     } else {
       assert(isa<ConstantAggregateZero>(C) && "Unknown vector constant!");
-      EVT EltVT = TLI.getValueType(VecTy->getElementType());
+      EVT EltVT =
+          TLI.getValueType(DAG.getDataLayout(), VecTy->getElementType());
 
       SDValue Op;
       if (EltVT.isFloatingPoint())
-        Op = DAG.getConstantFP(0, EltVT);
+        Op = DAG.getConstantFP(0, getCurSDLoc(), EltVT);
       else
-        Op = DAG.getConstant(0, EltVT);
+        Op = DAG.getConstant(0, getCurSDLoc(), EltVT);
       Ops.assign(NumElements, Op);
     }
 
@@ -1195,13 +1142,15 @@ SDValue SelectionDAGBuilder::getValueImpl(const Value *V) {
     DenseMap<const AllocaInst*, int>::iterator SI =
       FuncInfo.StaticAllocaMap.find(AI);
     if (SI != FuncInfo.StaticAllocaMap.end())
-      return DAG.getFrameIndex(SI->second, TLI.getPointerTy());
+      return DAG.getFrameIndex(SI->second,
+                               TLI.getPointerTy(DAG.getDataLayout()));
   }
 
   // If this is an instruction which fast-isel has deferred, select it now.
   if (const Instruction *Inst = dyn_cast<Instruction>(V)) {
     unsigned InReg = FuncInfo.InitializeRegForValue(Inst);
-    RegsForValue RFV(*DAG.getContext(), TLI, InReg, Inst->getType());
+    RegsForValue RFV(*DAG.getContext(), TLI, DAG.getDataLayout(), InReg,
+                     Inst->getType());
     SDValue Chain = DAG.getEntryNode();
     return RFV.getCopyFromRegs(DAG, FuncInfo, getCurSDLoc(), Chain, nullptr, V);
   }
@@ -1209,8 +1158,33 @@ SDValue SelectionDAGBuilder::getValueImpl(const Value *V) {
   llvm_unreachable("Can't get register for value!");
 }
 
+void SelectionDAGBuilder::visitCleanupRet(const CleanupReturnInst &I) {
+  report_fatal_error("visitCleanupRet not yet implemented!");
+}
+
+void SelectionDAGBuilder::visitCatchEndPad(const CatchEndPadInst &I) {
+  report_fatal_error("visitCatchEndPad not yet implemented!");
+}
+
+void SelectionDAGBuilder::visitCatchRet(const CatchReturnInst &I) {
+  report_fatal_error("visitCatchRet not yet implemented!");
+}
+
+void SelectionDAGBuilder::visitCatchPad(const CatchPadInst &I) {
+  report_fatal_error("visitCatchPad not yet implemented!");
+}
+
+void SelectionDAGBuilder::visitTerminatePad(const TerminatePadInst &TPI) {
+  report_fatal_error("visitTerminatePad not yet implemented!");
+}
+
+void SelectionDAGBuilder::visitCleanupPad(const CleanupPadInst &CPI) {
+  report_fatal_error("visitCleanupPad not yet implemented!");
+}
+
 void SelectionDAGBuilder::visitRet(const ReturnInst &I) {
   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
+  auto &DL = DAG.getDataLayout();
   SDValue Chain = getControlRoot();
   SmallVector<ISD::OutputArg, 8> Outs;
   SmallVector<SDValue, 8> OutVals;
@@ -1223,7 +1197,7 @@ void SelectionDAGBuilder::visitRet(const ReturnInst &I) {
     // Leave Outs empty so that LowerReturn won't try to load return
     // registers the usual way.
     SmallVector<EVT, 1> PtrValueVTs;
-    ComputeValueVTs(TLI, PointerType::getUnqual(F->getReturnType()),
+    ComputeValueVTs(TLI, DL, PointerType::getUnqual(F->getReturnType()),
                     PtrValueVTs);
 
     SDValue RetPtr = DAG.getRegister(DemoteReg, PtrValueVTs[0]);
@@ -1231,14 +1205,15 @@ void SelectionDAGBuilder::visitRet(const ReturnInst &I) {
 
     SmallVector<EVT, 4> ValueVTs;
     SmallVector<uint64_t, 4> Offsets;
-    ComputeValueVTs(TLI, I.getOperand(0)->getType(), ValueVTs, &Offsets);
+    ComputeValueVTs(TLI, DL, I.getOperand(0)->getType(), ValueVTs, &Offsets);
     unsigned NumValues = ValueVTs.size();
 
     SmallVector<SDValue, 4> Chains(NumValues);
     for (unsigned i = 0; i != NumValues; ++i) {
       SDValue Add = DAG.getNode(ISD::ADD, getCurSDLoc(),
                                 RetPtr.getValueType(), RetPtr,
-                                DAG.getIntPtrConstant(Offsets[i]));
+                                DAG.getIntPtrConstant(Offsets[i],
+                                                      getCurSDLoc()));
       Chains[i] =
         DAG.getStore(Chain, getCurSDLoc(),
                      SDValue(RetOp.getNode(), RetOp.getResNo() + i),
@@ -1250,7 +1225,7 @@ void SelectionDAGBuilder::visitRet(const ReturnInst &I) {
                         MVT::Other, Chains);
   } else if (I.getNumOperands() != 0) {
     SmallVector<EVT, 4> ValueVTs;
-    ComputeValueVTs(TLI, I.getOperand(0)->getType(), ValueVTs);
+    ComputeValueVTs(TLI, DL, I.getOperand(0)->getType(), ValueVTs);
     unsigned NumValues = ValueVTs.size();
     if (NumValues) {
       SDValue RetOp = getValue(I.getOperand(0));
@@ -1424,13 +1399,11 @@ SelectionDAGBuilder::EmitBranchForMergedCondition(const Value *Cond,
       ISD::CondCode Condition;
       if (const ICmpInst *IC = dyn_cast<ICmpInst>(Cond)) {
         Condition = getICmpCondCode(IC->getPredicate());
-      } else if (const FCmpInst *FC = dyn_cast<FCmpInst>(Cond)) {
+      } else {
+        const FCmpInst *FC = cast<FCmpInst>(Cond);
         Condition = getFCmpCondCode(FC->getPredicate());
         if (TM.Options.NoNaNsFPMath)
           Condition = getFCmpCodeWithoutNaN(Condition);
-      } else {
-        (void)Condition; // silence warning.
-        llvm_unreachable("Unknown compare instruction");
       }
 
       CaseBlock CB(Condition, BOp->getOperand(0), BOp->getOperand(1), nullptr,
@@ -1460,7 +1433,8 @@ void SelectionDAGBuilder::FindMergedConditions(const Value *Cond,
                                                MachineBasicBlock *FBB,
                                                MachineBasicBlock *CurBB,
                                                MachineBasicBlock *SwitchBB,
-                                               unsigned Opc, uint32_t TWeight,
+                                               Instruction::BinaryOps Opc,
+                                               uint32_t TWeight,
                                                uint32_t FWeight) {
   // If this node is not part of the or/and tree, emit it as a branch.
   const Instruction *BOp = dyn_cast<Instruction>(Cond);
@@ -1493,8 +1467,8 @@ void SelectionDAGBuilder::FindMergedConditions(const Value *Cond,
     // We have flexibility in setting Prob for BB1 and Prob for TmpBB.
     // The requirement is that
     //   TrueProb for BB1 + (FalseProb for BB1 * TrueProb for TmpBB)
-    //     = TrueProb for orignal BB.
-    // Assuming the orignal weights are A and B, one choice is to set BB1's
+    //     = TrueProb for original BB.
+    // Assuming the original weights are A and B, one choice is to set BB1's
     // weights to A and A+2B, and set TmpBB's weights to A and 2B. This choice
     // assumes that
     //   TrueProb for BB1 == FalseProb for BB1 * TrueProb for TmpBB.
@@ -1529,8 +1503,8 @@ void SelectionDAGBuilder::FindMergedConditions(const Value *Cond,
     // We have flexibility in setting Prob for BB1 and Prob for TmpBB.
     // The requirement is that
     //   FalseProb for BB1 + (TrueProb for BB1 * FalseProb for TmpBB)
-    //     = FalseProb for orignal BB.
-    // Assuming the orignal weights are A and B, one choice is to set BB1's
+    //     = FalseProb for original BB.
+    // Assuming the original weights are A and B, one choice is to set BB1's
     // weights to 2A+B and B, and set TmpBB's weights to 2A and B. This choice
     // assumes that
     //   FalseProb for BB1 == TrueProb for BB1 * FalseProb for TmpBB.
@@ -1683,7 +1657,7 @@ void SelectionDAGBuilder::visitSwitchCase(CaseBlock &CB,
       Cond = CondLHS;
     else if (CB.CmpRHS == ConstantInt::getFalse(*DAG.getContext()) &&
              CB.CC == ISD::SETEQ) {
-      SDValue True = DAG.getConstant(1, CondLHS.getValueType());
+      SDValue True = DAG.getConstant(1, dl, CondLHS.getValueType());
       Cond = DAG.getNode(ISD::XOR, dl, CondLHS.getValueType(), CondLHS, True);
     } else
       Cond = DAG.getSetCC(dl, MVT::i1, CondLHS, getValue(CB.CmpRHS), CB.CC);
@@ -1697,13 +1671,13 @@ void SelectionDAGBuilder::visitSwitchCase(CaseBlock &CB,
     EVT VT = CmpOp.getValueType();
 
     if (cast<ConstantInt>(CB.CmpLHS)->isMinValue(true)) {
-      Cond = DAG.getSetCC(dl, MVT::i1, CmpOp, DAG.getConstant(High, VT),
+      Cond = DAG.getSetCC(dl, MVT::i1, CmpOp, DAG.getConstant(High, dl, VT),
                           ISD::SETLE);
     } else {
       SDValue SUB = DAG.getNode(ISD::SUB, dl,
-                                VT, CmpOp, DAG.getConstant(Low, VT));
+                                VT, CmpOp, DAG.getConstant(Low, dl, VT));
       Cond = DAG.getSetCC(dl, MVT::i1, SUB,
-                          DAG.getConstant(High-Low, VT), ISD::SETULE);
+                          DAG.getConstant(High-Low, dl, VT), ISD::SETULE);
     }
   }
 
@@ -1718,7 +1692,7 @@ void SelectionDAGBuilder::visitSwitchCase(CaseBlock &CB,
   // fall through to the lhs instead of the rhs block.
   if (CB.TrueBB == NextBlock(SwitchBB)) {
     std::swap(CB.TrueBB, CB.FalseBB);
-    SDValue True = DAG.getConstant(1, Cond.getValueType());
+    SDValue True = DAG.getConstant(1, dl, Cond.getValueType());
     Cond = DAG.getNode(ISD::XOR, dl, Cond.getValueType(), Cond, True);
   }
 
@@ -1739,7 +1713,7 @@ void SelectionDAGBuilder::visitSwitchCase(CaseBlock &CB,
 void SelectionDAGBuilder::visitJumpTable(JumpTable &JT) {
   // Emit the code for the jump table
   assert(JT.Reg != -1U && "Should lower JT Header first!");
-  EVT PTy = DAG.getTargetLoweringInfo().getPointerTy();
+  EVT PTy = DAG.getTargetLoweringInfo().getPointerTy(DAG.getDataLayout());
   SDValue Index = DAG.getCopyFromReg(getControlRoot(), getCurSDLoc(),
                                      JT.Reg, PTy);
   SDValue Table = DAG.getJumpTable(JT.JTI, PTy);
@@ -1754,13 +1728,15 @@ void SelectionDAGBuilder::visitJumpTable(JumpTable &JT) {
 void SelectionDAGBuilder::visitJumpTableHeader(JumpTable &JT,
                                                JumpTableHeader &JTH,
                                                MachineBasicBlock *SwitchBB) {
+  SDLoc dl = getCurSDLoc();
+
   // Subtract the lowest switch case value from the value being switched on and
   // conditional branch to default mbb if the result is greater than the
   // difference between smallest and largest cases.
   SDValue SwitchOp = getValue(JTH.SValue);
   EVT VT = SwitchOp.getValueType();
-  SDValue Sub = DAG.getNode(ISD::SUB, getCurSDLoc(), VT, SwitchOp,
-                            DAG.getConstant(JTH.First, VT));
+  SDValue Sub = DAG.getNode(ISD::SUB, dl, VT, SwitchOp,
+                            DAG.getConstant(JTH.First, dl, VT));
 
   // The SDNode we just created, which holds the value being switched on minus
   // the smallest case value, needs to be copied to a virtual register so it
@@ -1768,28 +1744,29 @@ void SelectionDAGBuilder::visitJumpTableHeader(JumpTable &JT,
   // This value may be smaller or larger than the target's pointer type, and
   // therefore require extension or truncating.
   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
-  SwitchOp = DAG.getZExtOrTrunc(Sub, getCurSDLoc(), TLI.getPointerTy());
+  SwitchOp = DAG.getZExtOrTrunc(Sub, dl, TLI.getPointerTy(DAG.getDataLayout()));
 
-  unsigned JumpTableReg = FuncInfo.CreateReg(TLI.getPointerTy());
-  SDValue CopyTo = DAG.getCopyToReg(getControlRoot(), getCurSDLoc(),
+  unsigned JumpTableReg =
+      FuncInfo.CreateReg(TLI.getPointerTy(DAG.getDataLayout()));
+  SDValue CopyTo = DAG.getCopyToReg(getControlRoot(), dl,
                                     JumpTableReg, SwitchOp);
   JT.Reg = JumpTableReg;
 
   // Emit the range check for the jump table, and branch to the default block
   // for the switch statement if the value being switched on exceeds the largest
   // case in the switch.
-  SDValue CMP =
-      DAG.getSetCC(getCurSDLoc(), TLI.getSetCCResultType(*DAG.getContext(),
-                                                         Sub.getValueType()),
-                   Sub, DAG.getConstant(JTH.Last - JTH.First, VT), ISD::SETUGT);
+  SDValue CMP = DAG.getSetCC(
+      dl, TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(),
+                                 Sub.getValueType()),
+      Sub, DAG.getConstant(JTH.Last - JTH.First, dl, VT), ISD::SETUGT);
 
-  SDValue BrCond = DAG.getNode(ISD::BRCOND, getCurSDLoc(),
+  SDValue BrCond = DAG.getNode(ISD::BRCOND, dl,
                                MVT::Other, CopyTo, CMP,
                                DAG.getBasicBlock(JT.Default));
 
   // Avoid emitting unnecessary branches to the next block.
   if (JT.MBB != NextBlock(SwitchBB))
-    BrCond = DAG.getNode(ISD::BR, getCurSDLoc(), MVT::Other, BrCond,
+    BrCond = DAG.getNode(ISD::BR, dl, MVT::Other, BrCond,
                          DAG.getBasicBlock(JT.MBB));
 
   DAG.setRoot(BrCond);
@@ -1806,7 +1783,7 @@ void SelectionDAGBuilder::visitSPDescriptorParent(StackProtectorDescriptor &SPD,
 
   // First create the loads to the guard/stack slot for the comparison.
   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
-  EVT PtrTy = TLI.getPointerTy();
+  EVT PtrTy = TLI.getPointerTy(DAG.getDataLayout());
 
   MachineFrameInfo *MFI = ParentBB->getParent()->getFrameInfo();
   int FI = MFI->getStackProtectorIndex();
@@ -1815,10 +1792,10 @@ void SelectionDAGBuilder::visitSPDescriptorParent(StackProtectorDescriptor &SPD,
   SDValue GuardPtr = getValue(IRGuard);
   SDValue StackSlotPtr = DAG.getFrameIndex(FI, PtrTy);
 
-  unsigned Align =
-    TLI.getDataLayout()->getPrefTypeAlignment(IRGuard->getType());
+  unsigned Align = DL->getPrefTypeAlignment(IRGuard->getType());
 
   SDValue Guard;
+  SDLoc dl = getCurSDLoc();
 
   // If GuardReg is set and useLoadStackGuardNode returns true, retrieve the
   // guard value from the virtual register holding the value. Otherwise, emit a
@@ -1826,34 +1803,34 @@ void SelectionDAGBuilder::visitSPDescriptorParent(StackProtectorDescriptor &SPD,
   unsigned GuardReg = SPD.getGuardReg();
 
   if (GuardReg && TLI.useLoadStackGuardNode())
-    Guard = DAG.getCopyFromReg(DAG.getEntryNode(), getCurSDLoc(), GuardReg,
+    Guard = DAG.getCopyFromReg(DAG.getEntryNode(), dl, GuardReg,
                                PtrTy);
   else
-    Guard = DAG.getLoad(PtrTy, getCurSDLoc(), DAG.getEntryNode(),
+    Guard = DAG.getLoad(PtrTy, dl, DAG.getEntryNode(),
                         GuardPtr, MachinePointerInfo(IRGuard, 0),
                         true, false, false, Align);
 
-  SDValue StackSlot = DAG.getLoad(PtrTy, getCurSDLoc(), DAG.getEntryNode(),
+  SDValue StackSlot = DAG.getLoad(PtrTy, dl, DAG.getEntryNode(),
                                   StackSlotPtr,
                                   MachinePointerInfo::getFixedStack(FI),
                                   true, false, false, Align);
 
   // Perform the comparison via a subtract/getsetcc.
   EVT VT = Guard.getValueType();
-  SDValue Sub = DAG.getNode(ISD::SUB, getCurSDLoc(), VT, Guard, StackSlot);
+  SDValue Sub = DAG.getNode(ISD::SUB, dl, VT, Guard, StackSlot);
 
-  SDValue Cmp =
-      DAG.getSetCC(getCurSDLoc(), TLI.getSetCCResultType(*DAG.getContext(),
-                                                         Sub.getValueType()),
-                   Sub, DAG.getConstant(0, VT), ISD::SETNE);
+  SDValue Cmp = DAG.getSetCC(dl, TLI.getSetCCResultType(DAG.getDataLayout(),
+                                                        *DAG.getContext(),
+                                                        Sub.getValueType()),
+                             Sub, DAG.getConstant(0, dl, VT), ISD::SETNE);
 
   // If the sub is not 0, then we know the guard/stackslot do not equal, so
   // branch to failure MBB.
-  SDValue BrCond = DAG.getNode(ISD::BRCOND, getCurSDLoc(),
+  SDValue BrCond = DAG.getNode(ISD::BRCOND, dl,
                                MVT::Other, StackSlot.getOperand(0),
                                Cmp, DAG.getBasicBlock(SPD.getFailureMBB()));
   // Otherwise branch to success MBB.
-  SDValue Br = DAG.getNode(ISD::BR, getCurSDLoc(),
+  SDValue Br = DAG.getNode(ISD::BR, dl,
                            MVT::Other, BrCond,
                            DAG.getBasicBlock(SPD.getSuccessMBB()));
 
@@ -1881,18 +1858,20 @@ SelectionDAGBuilder::visitSPDescriptorFailure(StackProtectorDescriptor &SPD) {
 /// suitable for "bit tests"
 void SelectionDAGBuilder::visitBitTestHeader(BitTestBlock &B,
                                              MachineBasicBlock *SwitchBB) {
+  SDLoc dl = getCurSDLoc();
+
   // Subtract the minimum value
   SDValue SwitchOp = getValue(B.SValue);
   EVT VT = SwitchOp.getValueType();
-  SDValue Sub = DAG.getNode(ISD::SUB, getCurSDLoc(), VT, SwitchOp,
-                            DAG.getConstant(B.First, VT));
+  SDValue Sub = DAG.getNode(ISD::SUB, dl, VT, SwitchOp,
+                            DAG.getConstant(B.First, dl, VT));
 
   // Check range
   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
-  SDValue RangeCmp =
-      DAG.getSetCC(getCurSDLoc(), TLI.getSetCCResultType(*DAG.getContext(),
-                                                         Sub.getValueType()),
-                   Sub, DAG.getConstant(B.Range, VT), ISD::SETUGT);
+  SDValue RangeCmp = DAG.getSetCC(
+      dl, TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(),
+                                 Sub.getValueType()),
+      Sub, DAG.getConstant(B.Range, dl, VT), ISD::SETUGT);
 
   // Determine the type of the test operands.
   bool UsePtrType = false;
@@ -1908,27 +1887,26 @@ void SelectionDAGBuilder::visitBitTestHeader(BitTestBlock &B,
       }
   }
   if (UsePtrType) {
-    VT = TLI.getPointerTy();
-    Sub = DAG.getZExtOrTrunc(Sub, getCurSDLoc(), VT);
+    VT = TLI.getPointerTy(DAG.getDataLayout());
+    Sub = DAG.getZExtOrTrunc(Sub, dl, VT);
   }
 
   B.RegVT = VT.getSimpleVT();
   B.Reg = FuncInfo.CreateReg(B.RegVT);
-  SDValue CopyTo = DAG.getCopyToReg(getControlRoot(), getCurSDLoc(),
-                                    B.Reg, Sub);
+  SDValue CopyTo = DAG.getCopyToReg(getControlRoot(), dl, B.Reg, Sub);
 
   MachineBasicBlock* MBB = B.Cases[0].ThisBB;
 
   addSuccessorWithWeight(SwitchBB, B.Default);
   addSuccessorWithWeight(SwitchBB, MBB);
 
-  SDValue BrRange = DAG.getNode(ISD::BRCOND, getCurSDLoc(),
+  SDValue BrRange = DAG.getNode(ISD::BRCOND, dl,
                                 MVT::Other, CopyTo, RangeCmp,
                                 DAG.getBasicBlock(B.Default));
 
   // Avoid emitting unnecessary branches to the next block.
   if (MBB != NextBlock(SwitchBB))
-    BrRange = DAG.getNode(ISD::BR, getCurSDLoc(), MVT::Other, CopyTo,
+    BrRange = DAG.getNode(ISD::BR, dl, MVT::Other, BrRange,
                           DAG.getBasicBlock(MBB));
 
   DAG.setRoot(BrRange);
@@ -1941,9 +1919,9 @@ void SelectionDAGBuilder::visitBitTestCase(BitTestBlock &BB,
                                            unsigned Reg,
                                            BitTestCase &B,
                                            MachineBasicBlock *SwitchBB) {
+  SDLoc dl = getCurSDLoc();
   MVT VT = BB.RegVT;
-  SDValue ShiftOp = DAG.getCopyFromReg(getControlRoot(), getCurSDLoc(),
-                                       Reg, VT);
+  SDValue ShiftOp = DAG.getCopyFromReg(getControlRoot(), dl, Reg, VT);
   SDValue Cmp;
   unsigned PopCount = countPopulation(B.Mask);
   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
@@ -1951,24 +1929,26 @@ void SelectionDAGBuilder::visitBitTestCase(BitTestBlock &BB,
     // Testing for a single bit; just compare the shift count with what it
     // would need to be to shift a 1 bit in that position.
     Cmp = DAG.getSetCC(
-        getCurSDLoc(), TLI.getSetCCResultType(*DAG.getContext(), VT), ShiftOp,
-        DAG.getConstant(countTrailingZeros(B.Mask), VT), ISD::SETEQ);
+        dl, TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT),
+        ShiftOp, DAG.getConstant(countTrailingZeros(B.Mask), dl, VT),
+        ISD::SETEQ);
   } else if (PopCount == BB.Range) {
     // There is only one zero bit in the range, test for it directly.
     Cmp = DAG.getSetCC(
-        getCurSDLoc(), TLI.getSetCCResultType(*DAG.getContext(), VT), ShiftOp,
-        DAG.getConstant(countTrailingOnes(B.Mask), VT), ISD::SETNE);
+        dl, TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT),
+        ShiftOp, DAG.getConstant(countTrailingOnes(B.Mask), dl, VT),
+        ISD::SETNE);
   } else {
     // Make desired shift
-    SDValue SwitchVal = DAG.getNode(ISD::SHL, getCurSDLoc(), VT,
-                                    DAG.getConstant(1, VT), ShiftOp);
+    SDValue SwitchVal = DAG.getNode(ISD::SHL, dl, VT,
+                                    DAG.getConstant(1, dl, VT), ShiftOp);
 
     // Emit bit tests and jumps
-    SDValue AndOp = DAG.getNode(ISD::AND, getCurSDLoc(),
-                                VT, SwitchVal, DAG.getConstant(B.Mask, VT));
-    Cmp = DAG.getSetCC(getCurSDLoc(),
-                       TLI.getSetCCResultType(*DAG.getContext(), VT), AndOp,
-                       DAG.getConstant(0, VT), ISD::SETNE);
+    SDValue AndOp = DAG.getNode(ISD::AND, dl,
+                                VT, SwitchVal, DAG.getConstant(B.Mask, dl, VT));
+    Cmp = DAG.getSetCC(
+        dl, TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT),
+        AndOp, DAG.getConstant(0, dl, VT), ISD::SETNE);
   }
 
   // The branch weight from SwitchBB to B.TargetBB is B.ExtraWeight.
@@ -1976,13 +1956,13 @@ void SelectionDAGBuilder::visitBitTestCase(BitTestBlock &BB,
   // The branch weight from SwitchBB to NextMBB is BranchWeightToNext.
   addSuccessorWithWeight(SwitchBB, NextMBB, BranchWeightToNext);
 
-  SDValue BrAnd = DAG.getNode(ISD::BRCOND, getCurSDLoc(),
+  SDValue BrAnd = DAG.getNode(ISD::BRCOND, dl,
                               MVT::Other, getControlRoot(),
                               Cmp, DAG.getBasicBlock(B.TargetBB));
 
   // Avoid emitting unnecessary branches to the next block.
   if (NextMBB != NextBlock(SwitchBB))
-    BrAnd = DAG.getNode(ISD::BR, getCurSDLoc(), MVT::Other, BrAnd,
+    BrAnd = DAG.getNode(ISD::BR, dl, MVT::Other, BrAnd,
                         DAG.getBasicBlock(NextMBB));
 
   DAG.setRoot(BrAnd);
@@ -2055,7 +2035,8 @@ void SelectionDAGBuilder::visitLandingPad(const LandingPadInst &LP) {
     return;
 
   SmallVector<EVT, 2> ValueVTs;
-  ComputeValueVTs(TLI, LP.getType(), ValueVTs);
+  SDLoc dl = getCurSDLoc();
+  ComputeValueVTs(TLI, DAG.getDataLayout(), LP.getType(), ValueVTs);
   assert(ValueVTs.size() == 2 && "Only two-valued landingpads are supported");
 
   // Get the two live-in registers as SDValues. The physregs have already been
@@ -2063,630 +2044,57 @@ void SelectionDAGBuilder::visitLandingPad(const LandingPadInst &LP) {
   SDValue Ops[2];
   if (FuncInfo.ExceptionPointerVirtReg) {
     Ops[0] = DAG.getZExtOrTrunc(
-        DAG.getCopyFromReg(DAG.getEntryNode(), getCurSDLoc(),
-                           FuncInfo.ExceptionPointerVirtReg, TLI.getPointerTy()),
-        getCurSDLoc(), ValueVTs[0]);
+        DAG.getCopyFromReg(DAG.getEntryNode(), dl,
+                           FuncInfo.ExceptionPointerVirtReg,
+                           TLI.getPointerTy(DAG.getDataLayout())),
+        dl, ValueVTs[0]);
   } else {
-    Ops[0] = DAG.getConstant(0, TLI.getPointerTy());
+    Ops[0] = DAG.getConstant(0, dl, TLI.getPointerTy(DAG.getDataLayout()));
   }
   Ops[1] = DAG.getZExtOrTrunc(
-      DAG.getCopyFromReg(DAG.getEntryNode(), getCurSDLoc(),
-                         FuncInfo.ExceptionSelectorVirtReg, TLI.getPointerTy()),
-      getCurSDLoc(), ValueVTs[1]);
+      DAG.getCopyFromReg(DAG.getEntryNode(), dl,
+                         FuncInfo.ExceptionSelectorVirtReg,
+                         TLI.getPointerTy(DAG.getDataLayout())),
+      dl, ValueVTs[1]);
 
   // Merge into one.
-  SDValue Res = DAG.getNode(ISD::MERGE_VALUES, getCurSDLoc(),
+  SDValue Res = DAG.getNode(ISD::MERGE_VALUES, dl,
                             DAG.getVTList(ValueVTs), Ops);
   setValue(&LP, Res);
 }
 
-unsigned
-SelectionDAGBuilder::visitLandingPadClauseBB(GlobalValue *ClauseGV,
-                                             MachineBasicBlock *LPadBB) {
-  SDValue Chain = getControlRoot();
-
-  // Get the typeid that we will dispatch on later.
-  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
-  const TargetRegisterClass *RC = TLI.getRegClassFor(TLI.getPointerTy());
-  unsigned VReg = FuncInfo.MF->getRegInfo().createVirtualRegister(RC);
-  unsigned TypeID = DAG.getMachineFunction().getMMI().getTypeIDFor(ClauseGV);
-  SDValue Sel = DAG.getConstant(TypeID, TLI.getPointerTy());
-  Chain = DAG.getCopyToReg(Chain, getCurSDLoc(), VReg, Sel);
-
-  // Branch to the main landing pad block.
-  MachineBasicBlock *ClauseMBB = FuncInfo.MBB;
-  ClauseMBB->addSuccessor(LPadBB);
-  DAG.setRoot(DAG.getNode(ISD::BR, getCurSDLoc(), MVT::Other, Chain,
-                          DAG.getBasicBlock(LPadBB)));
-  return VReg;
-}
-
-/// handleSmallSwitchCaseRange - Emit a series of specific tests (suitable for
-/// small case ranges).
-bool SelectionDAGBuilder::handleSmallSwitchRange(CaseRec& CR,
-                                                 CaseRecVector& WorkList,
-                                                 const Value* SV,
-                                                 MachineBasicBlock *Default,
-                                                 MachineBasicBlock *SwitchBB) {
-  // Size is the number of Cases represented by this range.
-  size_t Size = CR.Range.second - CR.Range.first;
-  if (Size > 3)
-    return false;
-
-  // Get the MachineFunction which holds the current MBB.  This is used when
-  // inserting any additional MBBs necessary to represent the switch.
-  MachineFunction *CurMF = FuncInfo.MF;
-
-  // Figure out which block is immediately after the current one.
-  MachineBasicBlock *NextMBB = nullptr;
-  MachineFunction::iterator BBI = CR.CaseBB;
-  if (++BBI != FuncInfo.MF->end())
-    NextMBB = BBI;
-
-  BranchProbabilityInfo *BPI = FuncInfo.BPI;
-  // If any two of the cases has the same destination, and if one value
-  // is the same as the other, but has one bit unset that the other has set,
-  // use bit manipulation to do two compares at once.  For example:
-  // "if (X == 6 || X == 4)" -> "if ((X|2) == 6)"
-  // TODO: This could be extended to merge any 2 cases in switches with 3 cases.
-  // TODO: Handle cases where CR.CaseBB != SwitchBB.
-  if (Size == 2 && CR.CaseBB == SwitchBB) {
-    Case &Small = *CR.Range.first;
-    Case &Big = *(CR.Range.second-1);
-
-    if (Small.Low == Small.High && Big.Low == Big.High && Small.BB == Big.BB) {
-      const APInt& SmallValue = Small.Low->getValue();
-      const APInt& BigValue = Big.Low->getValue();
-
-      // Check that there is only one bit different.
-      if (BigValue.countPopulation() == SmallValue.countPopulation() + 1 &&
-          (SmallValue | BigValue) == BigValue) {
-        // Isolate the common bit.
-        APInt CommonBit = BigValue & ~SmallValue;
-        assert((SmallValue | CommonBit) == BigValue &&
-               CommonBit.countPopulation() == 1 && "Not a common bit?");
-
-        SDValue CondLHS = getValue(SV);
-        EVT VT = CondLHS.getValueType();
-        SDLoc DL = getCurSDLoc();
-
-        SDValue Or = DAG.getNode(ISD::OR, DL, VT, CondLHS,
-                                 DAG.getConstant(CommonBit, VT));
-        SDValue Cond = DAG.getSetCC(DL, MVT::i1,
-                                    Or, DAG.getConstant(BigValue, VT),
-                                    ISD::SETEQ);
-
-        // Update successor info.
-        // Both Small and Big will jump to Small.BB, so we sum up the weights.
-        addSuccessorWithWeight(SwitchBB, Small.BB,
-                               Small.ExtraWeight + Big.ExtraWeight);
-        addSuccessorWithWeight(SwitchBB, Default,
-          // The default destination is the first successor in IR.
-          BPI ? BPI->getEdgeWeight(SwitchBB->getBasicBlock(), (unsigned)0) : 0);
-
-        // Insert the true branch.
-        SDValue BrCond = DAG.getNode(ISD::BRCOND, DL, MVT::Other,
-                                     getControlRoot(), Cond,
-                                     DAG.getBasicBlock(Small.BB));
-
-        // Insert the false branch.
-        BrCond = DAG.getNode(ISD::BR, DL, MVT::Other, BrCond,
-                             DAG.getBasicBlock(Default));
-
-        DAG.setRoot(BrCond);
-        return true;
-      }
-    }
-  }
-
-  // Order cases by weight so the most likely case will be checked first.
-  uint32_t UnhandledWeights = 0;
-  if (BPI) {
-    for (CaseItr I = CR.Range.first, IE = CR.Range.second; I != IE; ++I) {
-      uint32_t IWeight = I->ExtraWeight;
-      UnhandledWeights += IWeight;
-      for (CaseItr J = CR.Range.first; J < I; ++J) {
-        uint32_t JWeight = J->ExtraWeight;
-        if (IWeight > JWeight)
-          std::swap(*I, *J);
-      }
-    }
-  }
-  // Rearrange the case blocks so that the last one falls through if possible.
-  Case &BackCase = *(CR.Range.second-1);
-  if (Size > 1 && NextMBB && Default != NextMBB && BackCase.BB != NextMBB) {
-    // The last case block won't fall through into 'NextMBB' if we emit the
-    // branches in this order.  See if rearranging a case value would help.
-    // We start at the bottom as it's the case with the least weight.
-    for (Case *I = &*(CR.Range.second-2), *E = &*CR.Range.first-1; I != E; --I)
-      if (I->BB == NextMBB) {
-        std::swap(*I, BackCase);
-        break;
-      }
-  }
-
-  // Create a CaseBlock record representing a conditional branch to
-  // the Case's target mbb if the value being switched on SV is equal
-  // to C.
-  MachineBasicBlock *CurBlock = CR.CaseBB;
-  for (CaseItr I = CR.Range.first, E = CR.Range.second; I != E; ++I) {
-    MachineBasicBlock *FallThrough;
-    if (I != E-1) {
-      FallThrough = CurMF->CreateMachineBasicBlock(CurBlock->getBasicBlock());
-      CurMF->insert(BBI, FallThrough);
-
-      // Put SV in a virtual register to make it available from the new blocks.
-      ExportFromCurrentBlock(SV);
-    } else {
-      // If the last case doesn't match, go to the default block.
-      FallThrough = Default;
-    }
-
-    const Value *RHS, *LHS, *MHS;
-    ISD::CondCode CC;
-    if (I->High == I->Low) {
-      // This is just small small case range :) containing exactly 1 case
-      CC = ISD::SETEQ;
-      LHS = SV; RHS = I->High; MHS = nullptr;
-    } else {
-      CC = ISD::SETLE;
-      LHS = I->Low; MHS = SV; RHS = I->High;
-    }
-
-    // The false weight should be sum of all un-handled cases.
-    UnhandledWeights -= I->ExtraWeight;
-    CaseBlock CB(CC, LHS, RHS, MHS, /* truebb */ I->BB, /* falsebb */ FallThrough,
-                 /* me */ CurBlock,
-                 /* trueweight */ I->ExtraWeight,
-                 /* falseweight */ UnhandledWeights);
-
-    // If emitting the first comparison, just call visitSwitchCase to emit the
-    // code into the current block.  Otherwise, push the CaseBlock onto the
-    // vector to be later processed by SDISel, and insert the node's MBB
-    // before the next MBB.
-    if (CurBlock == SwitchBB)
-      visitSwitchCase(CB, SwitchBB);
-    else
-      SwitchCases.push_back(CB);
-
-    CurBlock = FallThrough;
-  }
-
-  return true;
-}
-
-static inline bool areJTsAllowed(const TargetLowering &TLI) {
-  return TLI.isOperationLegalOrCustom(ISD::BR_JT, MVT::Other) ||
-         TLI.isOperationLegalOrCustom(ISD::BRIND, MVT::Other);
-}
-
-static APInt ComputeRange(const APInt &First, const APInt &Last) {
-  uint32_t BitWidth = std::max(Last.getBitWidth(), First.getBitWidth()) + 1;
-  APInt LastExt = Last.sext(BitWidth), FirstExt = First.sext(BitWidth);
-  return (LastExt - FirstExt + 1ULL);
-}
-
-/// handleJTSwitchCase - Emit jumptable for current switch case range
-bool SelectionDAGBuilder::handleJTSwitchCase(CaseRec &CR,
-                                             CaseRecVector &WorkList,
-                                             const Value *SV,
-                                             MachineBasicBlock *Default,
-                                             MachineBasicBlock *SwitchBB) {
-  Case& FrontCase = *CR.Range.first;
-  Case& BackCase  = *(CR.Range.second-1);
-
-  const APInt &First = FrontCase.Low->getValue();
-  const APInt &Last  = BackCase.High->getValue();
-
-  APInt TSize(First.getBitWidth(), 0);
-  for (CaseItr I = CR.Range.first, E = CR.Range.second; I != E; ++I)
-    TSize += I->size();
-
-  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
-  if (!areJTsAllowed(TLI) || TSize.ult(TLI.getMinimumJumpTableEntries()))
-    return false;
-
-  APInt Range = ComputeRange(First, Last);
-  // The density is TSize / Range. Require at least 40%.
-  // It should not be possible for IntTSize to saturate for sane code, but make
-  // sure we handle Range saturation correctly.
-  uint64_t IntRange = Range.getLimitedValue(UINT64_MAX/10);
-  uint64_t IntTSize = TSize.getLimitedValue(UINT64_MAX/10);
-  if (IntTSize * 10 < IntRange * 4)
-    return false;
-
-  DEBUG(dbgs() << "Lowering jump table\n"
-               << "First entry: " << First << ". Last entry: " << Last << '\n'
-               << "Range: " << Range << ". Size: " << TSize << ".\n\n");
-
-  // Get the MachineFunction which holds the current MBB.  This is used when
-  // inserting any additional MBBs necessary to represent the switch.
-  MachineFunction *CurMF = FuncInfo.MF;
-
-  // Figure out which block is immediately after the current one.
-  MachineFunction::iterator BBI = CR.CaseBB;
-  ++BBI;
-
-  const BasicBlock *LLVMBB = CR.CaseBB->getBasicBlock();
-
-  // Create a new basic block to hold the code for loading the address
-  // of the jump table, and jumping to it.  Update successor information;
-  // we will either branch to the default case for the switch, or the jump
-  // table.
-  MachineBasicBlock *JumpTableBB = CurMF->CreateMachineBasicBlock(LLVMBB);
-  CurMF->insert(BBI, JumpTableBB);
-
-  addSuccessorWithWeight(CR.CaseBB, Default);
-  addSuccessorWithWeight(CR.CaseBB, JumpTableBB);
-
-  // Build a vector of destination BBs, corresponding to each target
-  // of the jump table. If the value of the jump table slot corresponds to
-  // a case statement, push the case's BB onto the vector, otherwise, push
-  // the default BB.
-  std::vector<MachineBasicBlock*> DestBBs;
-  APInt TEI = First;
-  for (CaseItr I = CR.Range.first, E = CR.Range.second; I != E; ++TEI) {
-    const APInt &Low = I->Low->getValue();
-    const APInt &High = I->High->getValue();
-
-    if (Low.sle(TEI) && TEI.sle(High)) {
-      DestBBs.push_back(I->BB);
-      if (TEI==High)
-        ++I;
-    } else {
-      DestBBs.push_back(Default);
-    }
-  }
-
-  // Calculate weight for each unique destination in CR.
-  DenseMap<MachineBasicBlock*, uint32_t> DestWeights;
-  if (FuncInfo.BPI) {
-    for (CaseItr I = CR.Range.first, E = CR.Range.second; I != E; ++I)
-      DestWeights[I->BB] += I->ExtraWeight;
-  }
-
-  // Update successor info. Add one edge to each unique successor.
-  BitVector SuccsHandled(CR.CaseBB->getParent()->getNumBlockIDs());
-  for (MachineBasicBlock *DestBB : DestBBs) {
-    if (!SuccsHandled[DestBB->getNumber()]) {
-      SuccsHandled[DestBB->getNumber()] = true;
-      auto I = DestWeights.find(DestBB);
-      addSuccessorWithWeight(JumpTableBB, DestBB,
-                             I != DestWeights.end() ? I->second : 0);
-    }
-  }
-
-  // Create a jump table index for this jump table.
-  unsigned JTEncoding = TLI.getJumpTableEncoding();
-  unsigned JTI = CurMF->getOrCreateJumpTableInfo(JTEncoding)
-                       ->createJumpTableIndex(DestBBs);
-
-  // Set the jump table information so that we can codegen it as a second
-  // MachineBasicBlock
-  JumpTable JT(-1U, JTI, JumpTableBB, Default);
-  JumpTableHeader JTH(First, Last, SV, CR.CaseBB, (CR.CaseBB == SwitchBB));
-  if (CR.CaseBB == SwitchBB)
-    visitJumpTableHeader(JT, JTH, SwitchBB);
-
-  JTCases.push_back(JumpTableBlock(JTH, JT));
-  return true;
-}
-
-/// handleBTSplitSwitchCase - emit comparison and split binary search tree into
-/// 2 subtrees.
-bool SelectionDAGBuilder::handleBTSplitSwitchCase(CaseRec& CR,
-                                                  CaseRecVector& WorkList,
-                                                  const Value* SV,
-                                                  MachineBasicBlock* SwitchBB) {
-  Case& FrontCase = *CR.Range.first;
-  Case& BackCase  = *(CR.Range.second-1);
-
-  // Size is the number of Cases represented by this range.
-  unsigned Size = CR.Range.second - CR.Range.first;
-
-  const APInt &First = FrontCase.Low->getValue();
-  const APInt &Last  = BackCase.High->getValue();
-  double FMetric = 0;
-  CaseItr Pivot = CR.Range.first + Size/2;
-
-  // Select optimal pivot, maximizing sum density of LHS and RHS. This will
-  // (heuristically) allow us to emit JumpTable's later.
-  APInt TSize(First.getBitWidth(), 0);
-  for (CaseItr I = CR.Range.first, E = CR.Range.second;
-       I!=E; ++I)
-    TSize += I->size();
-
-  APInt LSize = FrontCase.size();
-  APInt RSize = TSize-LSize;
-  DEBUG(dbgs() << "Selecting best pivot: \n"
-               << "First: " << First << ", Last: " << Last <<'\n'
-               << "LSize: " << LSize << ", RSize: " << RSize << '\n');
-  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
-  for (CaseItr I = CR.Range.first, J=I+1, E = CR.Range.second;
-       J!=E; ++I, ++J) {
-    const APInt &LEnd = I->High->getValue();
-    const APInt &RBegin = J->Low->getValue();
-    APInt Range = ComputeRange(LEnd, RBegin);
-    assert((Range - 2ULL).isNonNegative() &&
-           "Invalid case distance");
-    // Use volatile double here to avoid excess precision issues on some hosts,
-    // e.g. that use 80-bit X87 registers.
-    // Only consider the density of sub-ranges that actually have sufficient
-    // entries to be lowered as a jump table.
-    volatile double LDensity =
-        LSize.ult(TLI.getMinimumJumpTableEntries())
-            ? 0.0
-            : LSize.roundToDouble() / (LEnd - First + 1ULL).roundToDouble();
-    volatile double RDensity =
-        RSize.ult(TLI.getMinimumJumpTableEntries())
-            ? 0.0
-            : RSize.roundToDouble() / (Last - RBegin + 1ULL).roundToDouble();
-    volatile double Metric = Range.logBase2() * (LDensity + RDensity);
-    // Should always split in some non-trivial place
-    DEBUG(dbgs() <<"=>Step\n"
-                 << "LEnd: " << LEnd << ", RBegin: " << RBegin << '\n'
-                 << "LDensity: " << LDensity
-                 << ", RDensity: " << RDensity << '\n'
-                 << "Metric: " << Metric << '\n');
-    if (FMetric < Metric) {
-      Pivot = J;
-      FMetric = Metric;
-      DEBUG(dbgs() << "Current metric set to: " << FMetric << '\n');
-    }
-
-    LSize += J->size();
-    RSize -= J->size();
-  }
-
-  if (FMetric == 0 || !areJTsAllowed(TLI))
-    Pivot = CR.Range.first + Size/2;
-  splitSwitchCase(CR, Pivot, WorkList, SV, SwitchBB);
-  return true;
-}
-
-void SelectionDAGBuilder::splitSwitchCase(CaseRec &CR, CaseItr Pivot,
-                                          CaseRecVector &WorkList,
-                                          const Value *SV,
-                                          MachineBasicBlock *SwitchBB) {
-  // Get the MachineFunction which holds the current MBB.  This is used when
-  // inserting any additional MBBs necessary to represent the switch.
-  MachineFunction *CurMF = FuncInfo.MF;
-
-  // Figure out which block is immediately after the current one.
-  MachineFunction::iterator BBI = CR.CaseBB;
-  ++BBI;
-
-  const BasicBlock *LLVMBB = CR.CaseBB->getBasicBlock();
-
-  CaseRange LHSR(CR.Range.first, Pivot);
-  CaseRange RHSR(Pivot, CR.Range.second);
-  const ConstantInt *C = Pivot->Low;
-  MachineBasicBlock *FalseBB = nullptr, *TrueBB = nullptr;
-
-  // We know that we branch to the LHS if the Value being switched on is
-  // less than the Pivot value, C.  We use this to optimize our binary
-  // tree a bit, by recognizing that if SV is greater than or equal to the
-  // LHS's Case Value, and that Case Value is exactly one less than the
-  // Pivot's Value, then we can branch directly to the LHS's Target,
-  // rather than creating a leaf node for it.
-  if ((LHSR.second - LHSR.first) == 1 && LHSR.first->High == CR.GE &&
-      C->getValue() == (CR.GE->getValue() + 1LL)) {
-    TrueBB = LHSR.first->BB;
-  } else {
-    TrueBB = CurMF->CreateMachineBasicBlock(LLVMBB);
-    CurMF->insert(BBI, TrueBB);
-    WorkList.push_back(CaseRec(TrueBB, C, CR.GE, LHSR));
-
-    // Put SV in a virtual register to make it available from the new blocks.
-    ExportFromCurrentBlock(SV);
-  }
-
-  // Similar to the optimization above, if the Value being switched on is
-  // known to be less than the Constant CR.LT, and the current Case Value
-  // is CR.LT - 1, then we can branch directly to the target block for
-  // the current Case Value, rather than emitting a RHS leaf node for it.
-  if ((RHSR.second - RHSR.first) == 1 && CR.LT &&
-      RHSR.first->Low->getValue() == (CR.LT->getValue() - 1LL)) {
-    FalseBB = RHSR.first->BB;
-  } else {
-    FalseBB = CurMF->CreateMachineBasicBlock(LLVMBB);
-    CurMF->insert(BBI, FalseBB);
-    WorkList.push_back(CaseRec(FalseBB, CR.LT, C, RHSR));
-
-    // Put SV in a virtual register to make it available from the new blocks.
-    ExportFromCurrentBlock(SV);
-  }
-
-  // Create a CaseBlock record representing a conditional branch to
-  // the LHS node if the value being switched on SV is less than C.
-  // Otherwise, branch to LHS.
-  CaseBlock CB(ISD::SETLT, SV, C, nullptr, TrueBB, FalseBB, CR.CaseBB);
-
-  if (CR.CaseBB == SwitchBB)
-    visitSwitchCase(CB, SwitchBB);
-  else
-    SwitchCases.push_back(CB);
-}
-
-/// handleBitTestsSwitchCase - if current case range has few destination and
-/// range span less, than machine word bitwidth, encode case range into series
-/// of masks and emit bit tests with these masks.
-bool SelectionDAGBuilder::handleBitTestsSwitchCase(CaseRec& CR,
-                                                   CaseRecVector& WorkList,
-                                                   const Value* SV,
-                                                   MachineBasicBlock* Default,
-                                                   MachineBasicBlock* SwitchBB) {
-  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
-  EVT PTy = TLI.getPointerTy();
-  unsigned IntPtrBits = PTy.getSizeInBits();
-
-  Case& FrontCase = *CR.Range.first;
-  Case& BackCase  = *(CR.Range.second-1);
-
-  // Get the MachineFunction which holds the current MBB.  This is used when
-  // inserting any additional MBBs necessary to represent the switch.
-  MachineFunction *CurMF = FuncInfo.MF;
-
-  // If target does not have legal shift left, do not emit bit tests at all.
-  if (!TLI.isOperationLegal(ISD::SHL, PTy))
-    return false;
-
-  size_t numCmps = 0;
-  for (CaseItr I = CR.Range.first, E = CR.Range.second; I != E; ++I) {
-    // Single case counts one, case range - two.
-    numCmps += (I->Low == I->High ? 1 : 2);
-  }
-
-  // Count unique destinations
-  SmallSet<MachineBasicBlock*, 4> Dests;
-  for (CaseItr I = CR.Range.first, E = CR.Range.second; I != E; ++I) {
-    Dests.insert(I->BB);
-    if (Dests.size() > 3)
-      // Don't bother the code below, if there are too much unique destinations
-      return false;
-  }
-  DEBUG(dbgs() << "Total number of unique destinations: "
-        << Dests.size() << '\n'
-        << "Total number of comparisons: " << numCmps << '\n');
-
-  // Compute span of values.
-  const APInt& minValue = FrontCase.Low->getValue();
-  const APInt& maxValue = BackCase.High->getValue();
-  APInt cmpRange = maxValue - minValue;
-
-  DEBUG(dbgs() << "Compare range: " << cmpRange << '\n'
-               << "Low bound: " << minValue << '\n'
-               << "High bound: " << maxValue << '\n');
-
-  if (cmpRange.uge(IntPtrBits) ||
-      (!(Dests.size() == 1 && numCmps >= 3) &&
-       !(Dests.size() == 2 && numCmps >= 5) &&
-       !(Dests.size() >= 3 && numCmps >= 6)))
-    return false;
-
-  DEBUG(dbgs() << "Emitting bit tests\n");
-  APInt lowBound = APInt::getNullValue(cmpRange.getBitWidth());
-
-  // Optimize the case where all the case values fit in a
-  // word without having to subtract minValue. In this case,
-  // we can optimize away the subtraction.
-  if (minValue.isNonNegative() && maxValue.slt(IntPtrBits)) {
-    cmpRange = maxValue;
-  } else {
-    lowBound = minValue;
-  }
-
-  CaseBitsVector CasesBits;
-  unsigned i, count = 0;
-
-  for (CaseItr I = CR.Range.first, E = CR.Range.second; I!=E; ++I) {
-    MachineBasicBlock* Dest = I->BB;
-    for (i = 0; i < count; ++i)
-      if (Dest == CasesBits[i].BB)
-        break;
-
-    if (i == count) {
-      assert((count < 3) && "Too much destinations to test!");
-      CasesBits.push_back(CaseBits(0, Dest, 0, 0/*Weight*/));
-      count++;
-    }
-
-    const APInt& lowValue = I->Low->getValue();
-    const APInt& highValue = I->High->getValue();
-
-    uint64_t lo = (lowValue - lowBound).getZExtValue();
-    uint64_t hi = (highValue - lowBound).getZExtValue();
-    CasesBits[i].ExtraWeight += I->ExtraWeight;
-
-    for (uint64_t j = lo; j <= hi; j++) {
-      CasesBits[i].Mask |=  1ULL << j;
-      CasesBits[i].Bits++;
-    }
-
-  }
-  std::sort(CasesBits.begin(), CasesBits.end(), CaseBitsCmp());
-
-  BitTestInfo BTC;
-
-  // Figure out which block is immediately after the current one.
-  MachineFunction::iterator BBI = CR.CaseBB;
-  ++BBI;
-
-  const BasicBlock *LLVMBB = CR.CaseBB->getBasicBlock();
-
-  DEBUG(dbgs() << "Cases:\n");
-  for (unsigned i = 0, e = CasesBits.size(); i!=e; ++i) {
-    DEBUG(dbgs() << "Mask: " << CasesBits[i].Mask
-                 << ", Bits: " << CasesBits[i].Bits
-                 << ", BB: " << CasesBits[i].BB << '\n');
-
-    MachineBasicBlock *CaseBB = CurMF->CreateMachineBasicBlock(LLVMBB);
-    CurMF->insert(BBI, CaseBB);
-    BTC.push_back(BitTestCase(CasesBits[i].Mask,
-                              CaseBB,
-                              CasesBits[i].BB, CasesBits[i].ExtraWeight));
-
-    // Put SV in a virtual register to make it available from the new blocks.
-    ExportFromCurrentBlock(SV);
-  }
-
-  BitTestBlock BTB(lowBound, cmpRange, SV,
-                   -1U, MVT::Other, (CR.CaseBB == SwitchBB),
-                   CR.CaseBB, Default, std::move(BTC));
-
-  if (CR.CaseBB == SwitchBB)
-    visitBitTestHeader(BTB, SwitchBB);
-
-  BitTestCases.push_back(std::move(BTB));
-
-  return true;
-}
-
-void SelectionDAGBuilder::Clusterify(CaseVector &Cases, const SwitchInst *SI) {
-  BranchProbabilityInfo *BPI = FuncInfo.BPI;
+void SelectionDAGBuilder::sortAndRangeify(CaseClusterVector &Clusters) {
+#ifndef NDEBUG
+  for (const CaseCluster &CC : Clusters)
+    assert(CC.Low == CC.High && "Input clusters must be single-case");
+#endif
 
-  // Extract cases from the switch and sort them.
-  typedef std::pair<const ConstantInt*, unsigned> CasePair;
-  std::vector<CasePair> Sorted;
-  Sorted.reserve(SI->getNumCases());
-  for (auto I : SI->cases())
-    Sorted.push_back(std::make_pair(I.getCaseValue(), I.getSuccessorIndex()));
-  std::sort(Sorted.begin(), Sorted.end(), [](CasePair a, CasePair b) {
-    return a.first->getValue().slt(b.first->getValue());
+  std::sort(Clusters.begin(), Clusters.end(),
+            [](const CaseCluster &a, const CaseCluster &b) {
+    return a.Low->getValue().slt(b.Low->getValue());
   });
 
-  // Merge adjacent cases with the same destination, build Cases vector.
-  assert(Cases.empty() && "Cases should be empty before Clusterify;");
-  Cases.reserve(SI->getNumCases());
-  MachineBasicBlock *PreviousSucc = nullptr;
-  for (CasePair &CP : Sorted) {
-    const ConstantInt *CaseVal = CP.first;
-    unsigned SuccIndex = CP.second;
-    MachineBasicBlock *Succ = FuncInfo.MBBMap[SI->getSuccessor(SuccIndex)];
-    uint32_t Weight = BPI ? BPI->getEdgeWeight(SI->getParent(), SuccIndex) : 0;
-
-    if (PreviousSucc == Succ &&
-        (CaseVal->getValue() - Cases.back().High->getValue()) == 1) {
+  // Merge adjacent clusters with the same destination.
+  const unsigned N = Clusters.size();
+  unsigned DstIndex = 0;
+  for (unsigned SrcIndex = 0; SrcIndex < N; ++SrcIndex) {
+    CaseCluster &CC = Clusters[SrcIndex];
+    const ConstantInt *CaseVal = CC.Low;
+    MachineBasicBlock *Succ = CC.MBB;
+
+    if (DstIndex != 0 && Clusters[DstIndex - 1].MBB == Succ &&
+        (CaseVal->getValue() - Clusters[DstIndex - 1].High->getValue()) == 1) {
       // If this case has the same successor and is a neighbour, merge it into
       // the previous cluster.
-      Cases.back().High = CaseVal;
-      Cases.back().ExtraWeight += Weight;
+      Clusters[DstIndex - 1].High = CaseVal;
+      Clusters[DstIndex - 1].Weight += CC.Weight;
+      assert(Clusters[DstIndex - 1].Weight >= CC.Weight && "Weight overflow!");
     } else {
-      Cases.push_back(Case(CaseVal, CaseVal, Succ, Weight));
+      std::memmove(&Clusters[DstIndex++], &Clusters[SrcIndex],
+                   sizeof(Clusters[SrcIndex]));
     }
-
-    PreviousSucc = Succ;
   }
-
-  DEBUG({
-      size_t numCmps = 0;
-      for (auto &I : Cases)
-        // A range counts double, since it requires two compares.
-        numCmps += I.Low != I.High ? 2 : 1;
-
-      dbgs() << "Clusterify finished. Total clusters: " << Cases.size()
-             << ". Total compares: " << numCmps << '\n';
-    });
+  Clusters.resize(DstIndex);
 }
 
 void SelectionDAGBuilder::UpdateSplitBlock(MachineBasicBlock *First,
@@ -2702,90 +2110,6 @@ void SelectionDAGBuilder::UpdateSplitBlock(MachineBasicBlock *First,
       BitTestCases[i].Parent = Last;
 }
 
-void SelectionDAGBuilder::visitSwitch(const SwitchInst &SI) {
-  MachineBasicBlock *SwitchMBB = FuncInfo.MBB;
-
-  // Create a vector of Cases, sorted so that we can efficiently create a binary
-  // search tree from them.
-  CaseVector Cases;
-  Clusterify(Cases, &SI);
-
-  // Get the default destination MBB.
-  MachineBasicBlock *Default = FuncInfo.MBBMap[SI.getDefaultDest()];
-
-  if (isa<UnreachableInst>(SI.getDefaultDest()->getFirstNonPHIOrDbg()) &&
-      !Cases.empty()) {
-    // Replace an unreachable default destination with the most popular case
-    // destination.
-    DenseMap<const BasicBlock *, unsigned> Popularity;
-    unsigned MaxPop = 0;
-    const BasicBlock *MaxBB = nullptr;
-    for (auto I : SI.cases()) {
-      const BasicBlock *BB = I.getCaseSuccessor();
-      if (++Popularity[BB] > MaxPop) {
-        MaxPop = Popularity[BB];
-        MaxBB = BB;
-      }
-    }
-
-    // Set new default.
-    assert(MaxPop > 0);
-    assert(MaxBB);
-    Default = FuncInfo.MBBMap[MaxBB];
-
-    // Remove cases that were pointing to the destination that is now the default.
-    Cases.erase(std::remove_if(Cases.begin(), Cases.end(),
-                               [&](const Case &C) { return C.BB == Default; }),
-                Cases.end());
-  }
-
-  // If there is only the default destination, go there directly.
-  if (Cases.empty()) {
-    // Update machine-CFG edges.
-    SwitchMBB->addSuccessor(Default);
-
-    // If this is not a fall-through branch, emit the branch.
-    if (Default != NextBlock(SwitchMBB)) {
-      DAG.setRoot(DAG.getNode(ISD::BR, getCurSDLoc(), MVT::Other,
-                              getControlRoot(), DAG.getBasicBlock(Default)));
-    }
-    return;
-  }
-
-  // Get the Value to be switched on.
-  const Value *SV = SI.getCondition();
-
-  // Push the initial CaseRec onto the worklist
-  CaseRecVector WorkList;
-  WorkList.push_back(CaseRec(SwitchMBB,nullptr,nullptr,
-                             CaseRange(Cases.begin(),Cases.end())));
-
-  while (!WorkList.empty()) {
-    // Grab a record representing a case range to process off the worklist
-    CaseRec CR = WorkList.back();
-    WorkList.pop_back();
-
-    if (handleBitTestsSwitchCase(CR, WorkList, SV, Default, SwitchMBB))
-      continue;
-
-    // If the range has few cases (two or less) emit a series of specific
-    // tests.
-    if (handleSmallSwitchRange(CR, WorkList, SV, Default, SwitchMBB))
-      continue;
-
-    // If the switch has more than N blocks, and is at least 40% dense, and the
-    // target supports indirect branches, then emit a jump table rather than
-    // lowering the switch to a binary tree of conditional branches.
-    // N defaults to 4 and is controlled via TLS.getMinimumJumpTableEntries().
-    if (handleJTSwitchCase(CR, WorkList, SV, Default, SwitchMBB))
-      continue;
-
-    // Emit binary tree. We need to pick a pivot, and push left and right ranges
-    // onto the worklist. Leafs are handled via handleSmallSwitchRange() call.
-    handleBTSplitSwitchCase(CR, WorkList, SV, SwitchMBB);
-  }
-}
-
 void SelectionDAGBuilder::visitIndirectBr(const IndirectBrInst &I) {
   MachineBasicBlock *IndirectBrMBB = FuncInfo.MBB;
 
@@ -2832,6 +2156,8 @@ void SelectionDAGBuilder::visitBinary(const User &I, unsigned OpCode) {
   bool nuw = false;
   bool nsw = false;
   bool exact = false;
+  FastMathFlags FMF;
+
   if (const OverflowingBinaryOperator *OFBinOp =
           dyn_cast<const OverflowingBinaryOperator>(&I)) {
     nuw = OFBinOp->hasNoUnsignedWrap();
@@ -2840,9 +2166,22 @@ void SelectionDAGBuilder::visitBinary(const User &I, unsigned OpCode) {
   if (const PossiblyExactOperator *ExactOp =
           dyn_cast<const PossiblyExactOperator>(&I))
     exact = ExactOp->isExact();
-
+  if (const FPMathOperator *FPOp = dyn_cast<const FPMathOperator>(&I))
+    FMF = FPOp->getFastMathFlags();
+
+  SDNodeFlags Flags;
+  Flags.setExact(exact);
+  Flags.setNoSignedWrap(nsw);
+  Flags.setNoUnsignedWrap(nuw);
+  if (EnableFMFInDAG) {
+    Flags.setAllowReciprocal(FMF.allowReciprocal());
+    Flags.setNoInfs(FMF.noInfs());
+    Flags.setNoNaNs(FMF.noNaNs());
+    Flags.setNoSignedZeros(FMF.noSignedZeros());
+    Flags.setUnsafeAlgebra(FMF.unsafeAlgebra());
+  }
   SDValue BinNodeValue = DAG.getNode(OpCode, getCurSDLoc(), Op1.getValueType(),
-                                     Op1, Op2, nuw, nsw, exact);
+                                     Op1, Op2, &Flags);
   setValue(&I, BinNodeValue);
 }
 
@@ -2850,8 +2189,8 @@ void SelectionDAGBuilder::visitShift(const User &I, unsigned Opcode) {
   SDValue Op1 = getValue(I.getOperand(0));
   SDValue Op2 = getValue(I.getOperand(1));
 
-  EVT ShiftTy =
-      DAG.getTargetLoweringInfo().getShiftAmountTy(Op2.getValueType());
+  EVT ShiftTy = DAG.getTargetLoweringInfo().getShiftAmountTy(
+      Op2.getValueType(), DAG.getDataLayout());
 
   // Coerce the shift amount to the right type if we can.
   if (!I.getType()->isVectorTy() && Op2.getValueType() != ShiftTy) {
@@ -2890,9 +2229,12 @@ void SelectionDAGBuilder::visitShift(const User &I, unsigned Opcode) {
             dyn_cast<const PossiblyExactOperator>(&I))
       exact = ExactOp->isExact();
   }
-
+  SDNodeFlags Flags;
+  Flags.setExact(exact);
+  Flags.setNoSignedWrap(nsw);
+  Flags.setNoUnsignedWrap(nuw);
   SDValue Res = DAG.getNode(Opcode, getCurSDLoc(), Op1.getValueType(), Op1, Op2,
-                            nuw, nsw, exact);
+                            &Flags);
   setValue(&I, Res);
 }
 
@@ -2900,17 +2242,11 @@ void SelectionDAGBuilder::visitSDiv(const User &I) {
   SDValue Op1 = getValue(I.getOperand(0));
   SDValue Op2 = getValue(I.getOperand(1));
 
-  // Turn exact SDivs into multiplications.
-  // FIXME: This should be in DAGCombiner, but it doesn't have access to the
-  // exact bit.
-  if (isa<BinaryOperator>(&I) && cast<BinaryOperator>(&I)->isExact() &&
-      !isa<ConstantSDNode>(Op1) &&
-      isa<ConstantSDNode>(Op2) && !cast<ConstantSDNode>(Op2)->isNullValue())
-    setValue(&I, DAG.getTargetLoweringInfo()
-                     .BuildExactSDIV(Op1, Op2, getCurSDLoc(), DAG));
-  else
-    setValue(&I, DAG.getNode(ISD::SDIV, getCurSDLoc(), Op1.getValueType(),
-                             Op1, Op2));
+  SDNodeFlags Flags;
+  Flags.setExact(isa<PossiblyExactOperator>(&I) &&
+                 cast<PossiblyExactOperator>(&I)->isExact());
+  setValue(&I, DAG.getNode(ISD::SDIV, getCurSDLoc(), Op1.getValueType(), Op1,
+                           Op2, &Flags));
 }
 
 void SelectionDAGBuilder::visitICmp(const User &I) {
@@ -2923,7 +2259,8 @@ void SelectionDAGBuilder::visitICmp(const User &I) {
   SDValue Op2 = getValue(I.getOperand(1));
   ISD::CondCode Opcode = getICmpCondCode(predicate);
 
-  EVT DestVT = DAG.getTargetLoweringInfo().getValueType(I.getType());
+  EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),
+                                                        I.getType());
   setValue(&I, DAG.getSetCC(getCurSDLoc(), DestVT, Op1, Op2, Opcode));
 }
 
@@ -2938,31 +2275,65 @@ void SelectionDAGBuilder::visitFCmp(const User &I) {
   ISD::CondCode Condition = getFCmpCondCode(predicate);
   if (TM.Options.NoNaNsFPMath)
     Condition = getFCmpCodeWithoutNaN(Condition);
-  EVT DestVT = DAG.getTargetLoweringInfo().getValueType(I.getType());
+  EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),
+                                                        I.getType());
   setValue(&I, DAG.getSetCC(getCurSDLoc(), DestVT, Op1, Op2, Condition));
 }
 
 void SelectionDAGBuilder::visitSelect(const User &I) {
   SmallVector<EVT, 4> ValueVTs;
-  ComputeValueVTs(DAG.getTargetLoweringInfo(), I.getType(), ValueVTs);
+  ComputeValueVTs(DAG.getTargetLoweringInfo(), DAG.getDataLayout(), I.getType(),
+                  ValueVTs);
   unsigned NumValues = ValueVTs.size();
   if (NumValues == 0) return;
 
   SmallVector<SDValue, 4> Values(NumValues);
   SDValue Cond     = getValue(I.getOperand(0));
-  SDValue TrueVal  = getValue(I.getOperand(1));
-  SDValue FalseVal = getValue(I.getOperand(2));
+  SDValue LHSVal   = getValue(I.getOperand(1));
+  SDValue RHSVal   = getValue(I.getOperand(2));
+  auto BaseOps = {Cond};
   ISD::NodeType OpCode = Cond.getValueType().isVector() ?
     ISD::VSELECT : ISD::SELECT;
 
-  for (unsigned i = 0; i != NumValues; ++i)
+  // Min/max matching is only viable if all output VTs are the same.
+  if (std::equal(ValueVTs.begin(), ValueVTs.end(), ValueVTs.begin())) {
+    Value *LHS, *RHS;
+    SelectPatternFlavor SPF = matchSelectPattern(const_cast<User*>(&I), LHS, RHS);
+    ISD::NodeType Opc = ISD::DELETED_NODE;
+    switch (SPF) {
+    case SPF_UMAX: Opc = ISD::UMAX; break;
+    case SPF_UMIN: Opc = ISD::UMIN; break;
+    case SPF_SMAX: Opc = ISD::SMAX; break;
+    case SPF_SMIN: Opc = ISD::SMIN; break;
+    default: break;
+    }
+
+    EVT VT = ValueVTs[0];
+    LLVMContext &Ctx = *DAG.getContext();
+    auto &TLI = DAG.getTargetLoweringInfo();
+    while (TLI.getTypeAction(Ctx, VT) == TargetLoweringBase::TypeSplitVector)
+      VT = TLI.getTypeToTransformTo(Ctx, VT);
+
+    if (Opc != ISD::DELETED_NODE && TLI.isOperationLegalOrCustom(Opc, VT) &&
+        // If the underlying comparison instruction is used by any other instruction,
+        // the consumed instructions won't be destroyed, so it is not profitable
+        // to convert to a min/max.
+        cast<SelectInst>(&I)->getCondition()->hasOneUse()) {
+      OpCode = Opc;
+      LHSVal = getValue(LHS);
+      RHSVal = getValue(RHS);
+      BaseOps = {};
+    }
+  }
+
+  for (unsigned i = 0; i != NumValues; ++i) {
+    SmallVector<SDValue, 3> Ops(BaseOps.begin(), BaseOps.end());
+    Ops.push_back(SDValue(LHSVal.getNode(), LHSVal.getResNo() + i));
+    Ops.push_back(SDValue(RHSVal.getNode(), RHSVal.getResNo() + i));
     Values[i] = DAG.getNode(OpCode, getCurSDLoc(),
-                            TrueVal.getNode()->getValueType(TrueVal.getResNo()+i),
-                            Cond,
-                            SDValue(TrueVal.getNode(),
-                                    TrueVal.getResNo() + i),
-                            SDValue(FalseVal.getNode(),
-                                    FalseVal.getResNo() + i));
+                            LHSVal.getNode()->getValueType(LHSVal.getResNo()+i),
+                            Ops);
+  }
 
   setValue(&I, DAG.getNode(ISD::MERGE_VALUES, getCurSDLoc(),
                            DAG.getVTList(ValueVTs), Values));
@@ -2971,7 +2342,8 @@ void SelectionDAGBuilder::visitSelect(const User &I) {
 void SelectionDAGBuilder::visitTrunc(const User &I) {
   // TruncInst cannot be a no-op cast because sizeof(src) > sizeof(dest).
   SDValue N = getValue(I.getOperand(0));
-  EVT DestVT = DAG.getTargetLoweringInfo().getValueType(I.getType());
+  EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),
+                                                        I.getType());
   setValue(&I, DAG.getNode(ISD::TRUNCATE, getCurSDLoc(), DestVT, N));
 }
 
@@ -2979,7 +2351,8 @@ void SelectionDAGBuilder::visitZExt(const User &I) {
   // ZExt cannot be a no-op cast because sizeof(src) < sizeof(dest).
   // ZExt also can't be a cast to bool for same reason. So, nothing much to do
   SDValue N = getValue(I.getOperand(0));
-  EVT DestVT = DAG.getTargetLoweringInfo().getValueType(I.getType());
+  EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),
+                                                        I.getType());
   setValue(&I, DAG.getNode(ISD::ZERO_EXTEND, getCurSDLoc(), DestVT, N));
 }
 
@@ -2987,51 +2360,59 @@ void SelectionDAGBuilder::visitSExt(const User &I) {
   // SExt cannot be a no-op cast because sizeof(src) < sizeof(dest).
   // SExt also can't be a cast to bool for same reason. So, nothing much to do
   SDValue N = getValue(I.getOperand(0));
-  EVT DestVT = DAG.getTargetLoweringInfo().getValueType(I.getType());
+  EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),
+                                                        I.getType());
   setValue(&I, DAG.getNode(ISD::SIGN_EXTEND, getCurSDLoc(), DestVT, N));
 }
 
 void SelectionDAGBuilder::visitFPTrunc(const User &I) {
   // FPTrunc is never a no-op cast, no need to check
   SDValue N = getValue(I.getOperand(0));
+  SDLoc dl = getCurSDLoc();
   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
-  EVT DestVT = TLI.getValueType(I.getType());
-  setValue(&I, DAG.getNode(ISD::FP_ROUND, getCurSDLoc(), DestVT, N,
-                           DAG.getTargetConstant(0, TLI.getPointerTy())));
+  EVT DestVT = TLI.getValueType(DAG.getDataLayout(), I.getType());
+  setValue(&I, DAG.getNode(ISD::FP_ROUND, dl, DestVT, N,
+                           DAG.getTargetConstant(
+                               0, dl, TLI.getPointerTy(DAG.getDataLayout()))));
 }
 
 void SelectionDAGBuilder::visitFPExt(const User &I) {
   // FPExt is never a no-op cast, no need to check
   SDValue N = getValue(I.getOperand(0));
-  EVT DestVT = DAG.getTargetLoweringInfo().getValueType(I.getType());
+  EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),
+                                                        I.getType());
   setValue(&I, DAG.getNode(ISD::FP_EXTEND, getCurSDLoc(), DestVT, N));
 }
 
 void SelectionDAGBuilder::visitFPToUI(const User &I) {
   // FPToUI is never a no-op cast, no need to check
   SDValue N = getValue(I.getOperand(0));
-  EVT DestVT = DAG.getTargetLoweringInfo().getValueType(I.getType());
+  EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),
+                                                        I.getType());
   setValue(&I, DAG.getNode(ISD::FP_TO_UINT, getCurSDLoc(), DestVT, N));
 }
 
 void SelectionDAGBuilder::visitFPToSI(const User &I) {
   // FPToSI is never a no-op cast, no need to check
   SDValue N = getValue(I.getOperand(0));
-  EVT DestVT = DAG.getTargetLoweringInfo().getValueType(I.getType());
+  EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),
+                                                        I.getType());
   setValue(&I, DAG.getNode(ISD::FP_TO_SINT, getCurSDLoc(), DestVT, N));
 }
 
 void SelectionDAGBuilder::visitUIToFP(const User &I) {
   // UIToFP is never a no-op cast, no need to check
   SDValue N = getValue(I.getOperand(0));
-  EVT DestVT = DAG.getTargetLoweringInfo().getValueType(I.getType());
+  EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),
+                                                        I.getType());
   setValue(&I, DAG.getNode(ISD::UINT_TO_FP, getCurSDLoc(), DestVT, N));
 }
 
 void SelectionDAGBuilder::visitSIToFP(const User &I) {
   // SIToFP is never a no-op cast, no need to check
   SDValue N = getValue(I.getOperand(0));
-  EVT DestVT = DAG.getTargetLoweringInfo().getValueType(I.getType());
+  EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),
+                                                        I.getType());
   setValue(&I, DAG.getNode(ISD::SINT_TO_FP, getCurSDLoc(), DestVT, N));
 }
 
@@ -3039,7 +2420,8 @@ void SelectionDAGBuilder::visitPtrToInt(const User &I) {
   // What to do depends on the size of the integer and the size of the pointer.
   // We can either truncate, zero extend, or no-op, accordingly.
   SDValue N = getValue(I.getOperand(0));
-  EVT DestVT = DAG.getTargetLoweringInfo().getValueType(I.getType());
+  EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),
+                                                        I.getType());
   setValue(&I, DAG.getZExtOrTrunc(N, getCurSDLoc(), DestVT));
 }
 
@@ -3047,25 +2429,28 @@ void SelectionDAGBuilder::visitIntToPtr(const User &I) {
   // What to do depends on the size of the integer and the size of the pointer.
   // We can either truncate, zero extend, or no-op, accordingly.
   SDValue N = getValue(I.getOperand(0));
-  EVT DestVT = DAG.getTargetLoweringInfo().getValueType(I.getType());
+  EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),
+                                                        I.getType());
   setValue(&I, DAG.getZExtOrTrunc(N, getCurSDLoc(), DestVT));
 }
 
 void SelectionDAGBuilder::visitBitCast(const User &I) {
   SDValue N = getValue(I.getOperand(0));
-  EVT DestVT = DAG.getTargetLoweringInfo().getValueType(I.getType());
+  SDLoc dl = getCurSDLoc();
+  EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),
+                                                        I.getType());
 
   // BitCast assures us that source and destination are the same size so this is
   // either a BITCAST or a no-op.
   if (DestVT != N.getValueType())
-    setValue(&I, DAG.getNode(ISD::BITCAST, getCurSDLoc(),
+    setValue(&I, DAG.getNode(ISD::BITCAST, dl,
                              DestVT, N)); // convert types.
   // Check if the original LLVM IR Operand was a ConstantInt, because getValue()
   // might fold any kind of constant expression to an integer constant and that
   // is not what we are looking for. Only regcognize a bitcast of a genuine
   // constant integer as an opaque constant.
   else if(ConstantInt *C = dyn_cast<ConstantInt>(I.getOperand(0)))
-    setValue(&I, DAG.getConstant(C->getValue(), DestVT, /*isTarget=*/false,
+    setValue(&I, DAG.getConstant(C->getValue(), dl, DestVT, /*isTarget=*/false,
                                  /*isOpaque*/true));
   else
     setValue(&I, N);            // noop cast.
@@ -3075,7 +2460,7 @@ void SelectionDAGBuilder::visitAddrSpaceCast(const User &I) {
   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
   const Value *SV = I.getOperand(0);
   SDValue N = getValue(SV);
-  EVT DestVT = TLI.getValueType(I.getType());
+  EVT DestVT = TLI.getValueType(DAG.getDataLayout(), I.getType());
 
   unsigned SrcAS = SV->getType()->getPointerAddressSpace();
   unsigned DestAS = I.getType()->getPointerAddressSpace();
@@ -3090,19 +2475,21 @@ void SelectionDAGBuilder::visitInsertElement(const User &I) {
   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
   SDValue InVec = getValue(I.getOperand(0));
   SDValue InVal = getValue(I.getOperand(1));
-  SDValue InIdx = DAG.getSExtOrTrunc(getValue(I.getOperand(2)),
-                                     getCurSDLoc(), TLI.getVectorIdxTy());
+  SDValue InIdx = DAG.getSExtOrTrunc(getValue(I.getOperand(2)), getCurSDLoc(),
+                                     TLI.getVectorIdxTy(DAG.getDataLayout()));
   setValue(&I, DAG.getNode(ISD::INSERT_VECTOR_ELT, getCurSDLoc(),
-                           TLI.getValueType(I.getType()), InVec, InVal, InIdx));
+                           TLI.getValueType(DAG.getDataLayout(), I.getType()),
+                           InVec, InVal, InIdx));
 }
 
 void SelectionDAGBuilder::visitExtractElement(const User &I) {
   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
   SDValue InVec = getValue(I.getOperand(0));
-  SDValue InIdx = DAG.getSExtOrTrunc(getValue(I.getOperand(1)),
-                                     getCurSDLoc(), TLI.getVectorIdxTy());
+  SDValue InIdx = DAG.getSExtOrTrunc(getValue(I.getOperand(1)), getCurSDLoc(),
+                                     TLI.getVectorIdxTy(DAG.getDataLayout()));
   setValue(&I, DAG.getNode(ISD::EXTRACT_VECTOR_ELT, getCurSDLoc(),
-                           TLI.getValueType(I.getType()), InVec, InIdx));
+                           TLI.getValueType(DAG.getDataLayout(), I.getType()),
+                           InVec, InIdx));
 }
 
 // Utility for visitShuffleVector - Return true if every element in Mask,
@@ -3125,7 +2512,7 @@ void SelectionDAGBuilder::visitShuffleVector(const User &I) {
   unsigned MaskNumElts = Mask.size();
 
   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
-  EVT VT = TLI.getValueType(I.getType());
+  EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType());
   EVT SrcVT = Src1.getValueType();
   unsigned SrcNumElts = SrcVT.getVectorNumElements();
 
@@ -3243,10 +2630,13 @@ void SelectionDAGBuilder::visitShuffleVector(const User &I) {
         SDValue &Src = Input == 0 ? Src1 : Src2;
         if (RangeUse[Input] == 0)
           Src = DAG.getUNDEF(VT);
-        else
+        else {
+          SDLoc dl = getCurSDLoc();
           Src = DAG.getNode(
-              ISD::EXTRACT_SUBVECTOR, getCurSDLoc(), VT, Src,
-              DAG.getConstant(StartIdx[Input], TLI.getVectorIdxTy()));
+              ISD::EXTRACT_SUBVECTOR, dl, VT, Src,
+              DAG.getConstant(StartIdx[Input], dl,
+                              TLI.getVectorIdxTy(DAG.getDataLayout())));
+        }
       }
 
       // Calculate new mask.
@@ -3272,7 +2662,8 @@ void SelectionDAGBuilder::visitShuffleVector(const User &I) {
   // replacing the shuffle with extract and build vector.
   // to insert and build vector.
   EVT EltVT = VT.getVectorElementType();
-  EVT IdxVT = TLI.getVectorIdxTy();
+  EVT IdxVT = TLI.getVectorIdxTy(DAG.getDataLayout());
+  SDLoc dl = getCurSDLoc();
   SmallVector<SDValue,8> Ops;
   for (unsigned i = 0; i != MaskNumElts; ++i) {
     int Idx = Mask[i];
@@ -3284,14 +2675,14 @@ void SelectionDAGBuilder::visitShuffleVector(const User &I) {
       SDValue &Src = Idx < (int)SrcNumElts ? Src1 : Src2;
       if (Idx >= (int)SrcNumElts) Idx -= SrcNumElts;
 
-      Res = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, getCurSDLoc(),
-                        EltVT, Src, DAG.getConstant(Idx, IdxVT));
+      Res = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl,
+                        EltVT, Src, DAG.getConstant(Idx, dl, IdxVT));
     }
 
     Ops.push_back(Res);
   }
 
-  setValue(&I, DAG.getNode(ISD::BUILD_VECTOR, getCurSDLoc(), VT, Ops));
+  setValue(&I, DAG.getNode(ISD::BUILD_VECTOR, dl, VT, Ops));
 }
 
 void SelectionDAGBuilder::visitInsertValue(const InsertValueInst &I) {
@@ -3306,9 +2697,9 @@ void SelectionDAGBuilder::visitInsertValue(const InsertValueInst &I) {
 
   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
   SmallVector<EVT, 4> AggValueVTs;
-  ComputeValueVTs(TLI, AggTy, AggValueVTs);
+  ComputeValueVTs(TLI, DAG.getDataLayout(), AggTy, AggValueVTs);
   SmallVector<EVT, 4> ValValueVTs;
-  ComputeValueVTs(TLI, ValTy, ValValueVTs);
+  ComputeValueVTs(TLI, DAG.getDataLayout(), ValTy, ValValueVTs);
 
   unsigned NumAggValues = AggValueVTs.size();
   unsigned NumValValues = ValValueVTs.size();
@@ -3352,7 +2743,7 @@ void SelectionDAGBuilder::visitExtractValue(const ExtractValueInst &I) {
 
   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
   SmallVector<EVT, 4> ValValueVTs;
-  ComputeValueVTs(TLI, ValTy, ValValueVTs);
+  ComputeValueVTs(TLI, DAG.getDataLayout(), ValTy, ValValueVTs);
 
   unsigned NumValValues = ValValueVTs.size();
 
@@ -3383,7 +2774,18 @@ void SelectionDAGBuilder::visitGetElementPtr(const User &I) {
   Type *Ty = Op0->getType()->getScalarType();
   unsigned AS = Ty->getPointerAddressSpace();
   SDValue N = getValue(Op0);
+  SDLoc dl = getCurSDLoc();
 
+  // Normalize Vector GEP - all scalar operands should be converted to the
+  // splat vector.
+  unsigned VectorWidth = I.getType()->isVectorTy() ?
+    cast<VectorType>(I.getType())->getVectorNumElements() : 0;
+
+  if (VectorWidth && !N.getValueType().isVector()) {
+    MVT VT = MVT::getVectorVT(N.getValueType().getSimpleVT(), VectorWidth);
+    SmallVector<SDValue, 16> Ops(VectorWidth, N);
+    N = DAG.getNode(ISD::BUILD_VECTOR, dl, VT, Ops);
+  }
   for (GetElementPtrInst::const_op_iterator OI = I.op_begin()+1, E = I.op_end();
        OI != E; ++OI) {
     const Value *Idx = *OI;
@@ -3392,50 +2794,64 @@ void SelectionDAGBuilder::visitGetElementPtr(const User &I) {
       if (Field) {
         // N = N + Offset
         uint64_t Offset = DL->getStructLayout(StTy)->getElementOffset(Field);
-        N = DAG.getNode(ISD::ADD, getCurSDLoc(), N.getValueType(), N,
-                        DAG.getConstant(Offset, N.getValueType()));
+        N = DAG.getNode(ISD::ADD, dl, N.getValueType(), N,
+                        DAG.getConstant(Offset, dl, N.getValueType()));
       }
 
       Ty = StTy->getElementType(Field);
     } else {
       Ty = cast<SequentialType>(Ty)->getElementType();
-      MVT PtrTy = DAG.getTargetLoweringInfo().getPointerTy(AS);
+      MVT PtrTy =
+          DAG.getTargetLoweringInfo().getPointerTy(DAG.getDataLayout(), AS);
       unsigned PtrSize = PtrTy.getSizeInBits();
       APInt ElementSize(PtrSize, DL->getTypeAllocSize(Ty));
 
-      // If this is a constant subscript, handle it quickly.
-      if (const auto *CI = dyn_cast<ConstantInt>(Idx)) {
+      // If this is a scalar constant or a splat vector of constants,
+      // handle it quickly.
+      const auto *CI = dyn_cast<ConstantInt>(Idx);
+      if (!CI && isa<ConstantDataVector>(Idx) &&
+          cast<ConstantDataVector>(Idx)->getSplatValue())
+        CI = cast<ConstantInt>(cast<ConstantDataVector>(Idx)->getSplatValue());
+
+      if (CI) {
         if (CI->isZero())
           continue;
         APInt Offs = ElementSize * CI->getValue().sextOrTrunc(PtrSize);
-        SDValue OffsVal = DAG.getConstant(Offs, PtrTy);
-        N = DAG.getNode(ISD::ADD, getCurSDLoc(), N.getValueType(), N, OffsVal);
+        SDValue OffsVal = VectorWidth ?
+          DAG.getConstant(Offs, dl, MVT::getVectorVT(PtrTy, VectorWidth)) :
+          DAG.getConstant(Offs, dl, PtrTy);
+        N = DAG.getNode(ISD::ADD, dl, N.getValueType(), N, OffsVal);
         continue;
       }
 
       // N = N + Idx * ElementSize;
       SDValue IdxN = getValue(Idx);
 
+      if (!IdxN.getValueType().isVector() && VectorWidth) {
+        MVT VT = MVT::getVectorVT(IdxN.getValueType().getSimpleVT(), VectorWidth);
+        SmallVector<SDValue, 16> Ops(VectorWidth, IdxN);
+        IdxN = DAG.getNode(ISD::BUILD_VECTOR, dl, VT, Ops);      
+      }
       // If the index is smaller or larger than intptr_t, truncate or extend
       // it.
-      IdxN = DAG.getSExtOrTrunc(IdxN, getCurSDLoc(), N.getValueType());
+      IdxN = DAG.getSExtOrTrunc(IdxN, dl, N.getValueType());
 
       // If this is a multiply by a power of two, turn it into a shl
       // immediately.  This is a very common case.
       if (ElementSize != 1) {
         if (ElementSize.isPowerOf2()) {
           unsigned Amt = ElementSize.logBase2();
-          IdxN = DAG.getNode(ISD::SHL, getCurSDLoc(),
+          IdxN = DAG.getNode(ISD::SHL, dl,
                              N.getValueType(), IdxN,
-                             DAG.getConstant(Amt, IdxN.getValueType()));
+                             DAG.getConstant(Amt, dl, IdxN.getValueType()));
         } else {
-          SDValue Scale = DAG.getConstant(ElementSize, IdxN.getValueType());
-          IdxN = DAG.getNode(ISD::MUL, getCurSDLoc(),
+          SDValue Scale = DAG.getConstant(ElementSize, dl, IdxN.getValueType());
+          IdxN = DAG.getNode(ISD::MUL, dl,
                              N.getValueType(), IdxN, Scale);
         }
       }
 
-      N = DAG.getNode(ISD::ADD, getCurSDLoc(),
+      N = DAG.getNode(ISD::ADD, dl,
                       N.getValueType(), N, IdxN);
     }
   }
@@ -3449,22 +2865,23 @@ void SelectionDAGBuilder::visitAlloca(const AllocaInst &I) {
   if (FuncInfo.StaticAllocaMap.count(&I))
     return;   // getValue will auto-populate this.
 
+  SDLoc dl = getCurSDLoc();
   Type *Ty = I.getAllocatedType();
   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
-  uint64_t TySize = TLI.getDataLayout()->getTypeAllocSize(Ty);
+  auto &DL = DAG.getDataLayout();
+  uint64_t TySize = DL.getTypeAllocSize(Ty);
   unsigned Align =
-      std::max((unsigned)TLI.getDataLayout()->getPrefTypeAlignment(Ty),
-               I.getAlignment());
+      std::max((unsigned)DL.getPrefTypeAlignment(Ty), I.getAlignment());
 
   SDValue AllocSize = getValue(I.getArraySize());
 
-  EVT IntPtr = TLI.getPointerTy();
+  EVT IntPtr = TLI.getPointerTy(DAG.getDataLayout());
   if (AllocSize.getValueType() != IntPtr)
-    AllocSize = DAG.getZExtOrTrunc(AllocSize, getCurSDLoc(), IntPtr);
+    AllocSize = DAG.getZExtOrTrunc(AllocSize, dl, IntPtr);
 
-  AllocSize = DAG.getNode(ISD::MUL, getCurSDLoc(), IntPtr,
+  AllocSize = DAG.getNode(ISD::MUL, dl, IntPtr,
                           AllocSize,
-                          DAG.getConstant(TySize, IntPtr));
+                          DAG.getConstant(TySize, dl, IntPtr));
 
   // Handle alignment.  If the requested alignment is less than or equal to
   // the stack alignment, ignore it.  If the size is greater than or equal to
@@ -3476,18 +2893,19 @@ void SelectionDAGBuilder::visitAlloca(const AllocaInst &I) {
 
   // Round the size of the allocation up to the stack alignment size
   // by add SA-1 to the size.
-  AllocSize = DAG.getNode(ISD::ADD, getCurSDLoc(),
+  AllocSize = DAG.getNode(ISD::ADD, dl,
                           AllocSize.getValueType(), AllocSize,
-                          DAG.getIntPtrConstant(StackAlign-1));
+                          DAG.getIntPtrConstant(StackAlign - 1, dl));
 
   // Mask out the low bits for alignment purposes.
-  AllocSize = DAG.getNode(ISD::AND, getCurSDLoc(),
+  AllocSize = DAG.getNode(ISD::AND, dl,
                           AllocSize.getValueType(), AllocSize,
-                          DAG.getIntPtrConstant(~(uint64_t)(StackAlign-1)));
+                          DAG.getIntPtrConstant(~(uint64_t)(StackAlign - 1),
+                                                dl));
 
-  SDValue Ops[] = { getRoot(), AllocSize, DAG.getIntPtrConstant(Align) };
+  SDValue Ops[] = { getRoot(), AllocSize, DAG.getIntPtrConstant(Align, dl) };
   SDVTList VTs = DAG.getVTList(AllocSize.getValueType(), MVT::Other);
-  SDValue DSA = DAG.getNode(ISD::DYNAMIC_STACKALLOC, getCurSDLoc(), VTs, Ops);
+  SDValue DSA = DAG.getNode(ISD::DYNAMIC_STACKALLOC, dl, VTs, Ops);
   setValue(&I, DSA);
   DAG.setRoot(DSA.getValue(1));
 
@@ -3505,7 +2923,17 @@ void SelectionDAGBuilder::visitLoad(const LoadInst &I) {
 
   bool isVolatile = I.isVolatile();
   bool isNonTemporal = I.getMetadata(LLVMContext::MD_nontemporal) != nullptr;
-  bool isInvariant = I.getMetadata(LLVMContext::MD_invariant_load) != nullptr;
+
+  // The IR notion of invariant_load only guarantees that all *non-faulting*
+  // invariant loads result in the same value.  The MI notion of invariant load
+  // guarantees that the load can be legally moved to any location within its
+  // containing function.  The MI notion of invariant_load is stronger than the
+  // IR notion of invariant_load -- an MI invariant_load is an IR invariant_load
+  // with a guarantee that the location being loaded from is dereferenceable
+  // throughout the function's lifetime.
+
+  bool isInvariant = I.getMetadata(LLVMContext::MD_invariant_load) != nullptr &&
+                     isDereferenceablePointer(SV, DAG.getDataLayout());
   unsigned Alignment = I.getAlignment();
 
   AAMDNodes AAInfo;
@@ -3515,7 +2943,7 @@ void SelectionDAGBuilder::visitLoad(const LoadInst &I) {
   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
   SmallVector<EVT, 4> ValueVTs;
   SmallVector<uint64_t, 4> Offsets;
-  ComputeValueVTs(TLI, Ty, ValueVTs, &Offsets);
+  ComputeValueVTs(TLI, DAG.getDataLayout(), Ty, ValueVTs, &Offsets);
   unsigned NumValues = ValueVTs.size();
   if (NumValues == 0)
     return;
@@ -3526,7 +2954,7 @@ void SelectionDAGBuilder::visitLoad(const LoadInst &I) {
     // Serialize volatile loads with other side effects.
     Root = getRoot();
   else if (AA->pointsToConstantMemory(
-             AliasAnalysis::Location(SV, AA->getTypeStoreSize(Ty), AAInfo))) {
+               MemoryLocation(SV, AA->getTypeStoreSize(Ty), AAInfo))) {
     // Do not serialize (non-volatile) loads of constant memory with anything.
     Root = DAG.getEntryNode();
     ConstantMemory = true;
@@ -3535,12 +2963,13 @@ void SelectionDAGBuilder::visitLoad(const LoadInst &I) {
     Root = DAG.getRoot();
   }
 
+  SDLoc dl = getCurSDLoc();
+
   if (isVolatile)
-    Root = TLI.prepareVolatileOrAtomicLoad(Root, getCurSDLoc(), DAG);
+    Root = TLI.prepareVolatileOrAtomicLoad(Root, dl, DAG);
 
   SmallVector<SDValue, 4> Values(NumValues);
-  SmallVector<SDValue, 4> Chains(std::min(unsigned(MaxParallelChains),
-                                          NumValues));
+  SmallVector<SDValue, 4> Chains(std::min(MaxParallelChains, NumValues));
   EVT PtrVT = Ptr.getValueType();
   unsigned ChainI = 0;
   for (unsigned i = 0; i != NumValues; ++i, ++ChainI) {
@@ -3552,15 +2981,15 @@ void SelectionDAGBuilder::visitLoad(const LoadInst &I) {
     // (MaxParallelChains should always remain as failsafe).
     if (ChainI == MaxParallelChains) {
       assert(PendingLoads.empty() && "PendingLoads must be serialized first");
-      SDValue Chain = DAG.getNode(ISD::TokenFactor, getCurSDLoc(), MVT::Other,
+      SDValue Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other,
                                   makeArrayRef(Chains.data(), ChainI));
       Root = Chain;
       ChainI = 0;
     }
-    SDValue A = DAG.getNode(ISD::ADD, getCurSDLoc(),
+    SDValue A = DAG.getNode(ISD::ADD, dl,
                             PtrVT, Ptr,
-                            DAG.getConstant(Offsets[i], PtrVT));
-    SDValue L = DAG.getLoad(ValueVTs[i], getCurSDLoc(), Root,
+                            DAG.getConstant(Offsets[i], dl, PtrVT));
+    SDValue L = DAG.getLoad(ValueVTs[i], dl, Root,
                             A, MachinePointerInfo(SV, Offsets[i]), isVolatile,
                             isNonTemporal, isInvariant, Alignment, AAInfo,
                             Ranges);
@@ -3570,7 +2999,7 @@ void SelectionDAGBuilder::visitLoad(const LoadInst &I) {
   }
 
   if (!ConstantMemory) {
-    SDValue Chain = DAG.getNode(ISD::TokenFactor, getCurSDLoc(), MVT::Other,
+    SDValue Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other,
                                 makeArrayRef(Chains.data(), ChainI));
     if (isVolatile)
       DAG.setRoot(Chain);
@@ -3578,7 +3007,7 @@ void SelectionDAGBuilder::visitLoad(const LoadInst &I) {
       PendingLoads.push_back(Chain);
   }
 
-  setValue(&I, DAG.getNode(ISD::MERGE_VALUES, getCurSDLoc(),
+  setValue(&I, DAG.getNode(ISD::MERGE_VALUES, dl,
                            DAG.getVTList(ValueVTs), Values));
 }
 
@@ -3591,8 +3020,8 @@ void SelectionDAGBuilder::visitStore(const StoreInst &I) {
 
   SmallVector<EVT, 4> ValueVTs;
   SmallVector<uint64_t, 4> Offsets;
-  ComputeValueVTs(DAG.getTargetLoweringInfo(), SrcV->getType(),
-                  ValueVTs, &Offsets);
+  ComputeValueVTs(DAG.getTargetLoweringInfo(), DAG.getDataLayout(),
+                  SrcV->getType(), ValueVTs, &Offsets);
   unsigned NumValues = ValueVTs.size();
   if (NumValues == 0)
     return;
@@ -3604,12 +3033,12 @@ void SelectionDAGBuilder::visitStore(const StoreInst &I) {
   SDValue Ptr = getValue(PtrV);
 
   SDValue Root = getRoot();
-  SmallVector<SDValue, 4> Chains(std::min(unsigned(MaxParallelChains),
-                                          NumValues));
+  SmallVector<SDValue, 4> Chains(std::min(MaxParallelChains, NumValues));
   EVT PtrVT = Ptr.getValueType();
   bool isVolatile = I.isVolatile();
   bool isNonTemporal = I.getMetadata(LLVMContext::MD_nontemporal) != nullptr;
   unsigned Alignment = I.getAlignment();
+  SDLoc dl = getCurSDLoc();
 
   AAMDNodes AAInfo;
   I.getAAMetadata(AAInfo);
@@ -3618,21 +3047,21 @@ void SelectionDAGBuilder::visitStore(const StoreInst &I) {
   for (unsigned i = 0; i != NumValues; ++i, ++ChainI) {
     // See visitLoad comments.
     if (ChainI == MaxParallelChains) {
-      SDValue Chain = DAG.getNode(ISD::TokenFactor, getCurSDLoc(), MVT::Other,
+      SDValue Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other,
                                   makeArrayRef(Chains.data(), ChainI));
       Root = Chain;
       ChainI = 0;
     }
-    SDValue Add = DAG.getNode(ISD::ADD, getCurSDLoc(), PtrVT, Ptr,
-                              DAG.getConstant(Offsets[i], PtrVT));
-    SDValue St = DAG.getStore(Root, getCurSDLoc(),
+    SDValue Add = DAG.getNode(ISD::ADD, dl, PtrVT, Ptr,
+                              DAG.getConstant(Offsets[i], dl, PtrVT));
+    SDValue St = DAG.getStore(Root, dl,
                               SDValue(Src.getNode(), Src.getResNo() + i),
                               Add, MachinePointerInfo(PtrV, Offsets[i]),
                               isVolatile, isNonTemporal, Alignment, AAInfo);
     Chains[ChainI] = St;
   }
 
-  SDValue StoreNode = DAG.getNode(ISD::TokenFactor, getCurSDLoc(), MVT::Other,
+  SDValue StoreNode = DAG.getNode(ISD::TokenFactor, dl, MVT::Other,
                                   makeArrayRef(Chains.data(), ChainI));
   DAG.setRoot(StoreNode);
 }
@@ -3664,6 +3093,95 @@ void SelectionDAGBuilder::visitMaskedStore(const CallInst &I) {
   setValue(&I, StoreNode);
 }
 
+// Gather/scatter receive a vector of pointers.
+// This vector of pointers may be represented as a base pointer + vector of 
+// indices, it depends on GEP and instruction preceeding GEP
+// that calculates indices
+static bool getUniformBase(Value *& Ptr, SDValue& Base, SDValue& Index,
+                           SelectionDAGBuilder* SDB) {
+
+  assert (Ptr->getType()->isVectorTy() && "Uexpected pointer type");
+  GetElementPtrInst *Gep = dyn_cast<GetElementPtrInst>(Ptr);
+  if (!Gep || Gep->getNumOperands() > 2)
+    return false;
+  ShuffleVectorInst *ShuffleInst = 
+    dyn_cast<ShuffleVectorInst>(Gep->getPointerOperand());
+  if (!ShuffleInst || !ShuffleInst->getMask()->isNullValue() ||
+      cast<Instruction>(ShuffleInst->getOperand(0))->getOpcode() !=
+      Instruction::InsertElement)
+    return false;
+
+  Ptr = cast<InsertElementInst>(ShuffleInst->getOperand(0))->getOperand(1);
+
+  SelectionDAG& DAG = SDB->DAG;
+  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
+  // Check is the Ptr is inside current basic block
+  // If not, look for the shuffle instruction
+  if (SDB->findValue(Ptr))
+    Base = SDB->getValue(Ptr);
+  else if (SDB->findValue(ShuffleInst)) {
+    SDValue ShuffleNode = SDB->getValue(ShuffleInst);
+    SDLoc sdl = ShuffleNode;
+    Base = DAG.getNode(
+        ISD::EXTRACT_VECTOR_ELT, sdl,
+        ShuffleNode.getValueType().getScalarType(), ShuffleNode,
+        DAG.getConstant(0, sdl, TLI.getVectorIdxTy(DAG.getDataLayout())));
+    SDB->setValue(Ptr, Base);
+  }
+  else
+    return false;
+
+  Value *IndexVal = Gep->getOperand(1);
+  if (SDB->findValue(IndexVal)) {
+    Index = SDB->getValue(IndexVal);
+
+    if (SExtInst* Sext = dyn_cast<SExtInst>(IndexVal)) {
+      IndexVal = Sext->getOperand(0);
+      if (SDB->findValue(IndexVal))
+        Index = SDB->getValue(IndexVal);
+    }
+    return true;
+  }
+  return false;
+}
+
+void SelectionDAGBuilder::visitMaskedScatter(const CallInst &I) {
+  SDLoc sdl = getCurSDLoc();
+
+  // llvm.masked.scatter.*(Src0, Ptrs, alignemt, Mask)
+  Value  *Ptr = I.getArgOperand(1);
+  SDValue Src0 = getValue(I.getArgOperand(0));
+  SDValue Mask = getValue(I.getArgOperand(3));
+  EVT VT = Src0.getValueType();
+  unsigned Alignment = (cast<ConstantInt>(I.getArgOperand(2)))->getZExtValue();
+  if (!Alignment)
+    Alignment = DAG.getEVTAlignment(VT);
+  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
+
+  AAMDNodes AAInfo;
+  I.getAAMetadata(AAInfo);
+
+  SDValue Base;
+  SDValue Index;
+  Value *BasePtr = Ptr;
+  bool UniformBase = getUniformBase(BasePtr, Base, Index, this);
+
+  Value *MemOpBasePtr = UniformBase ? BasePtr : nullptr;
+  MachineMemOperand *MMO = DAG.getMachineFunction().
+    getMachineMemOperand(MachinePointerInfo(MemOpBasePtr),
+                         MachineMemOperand::MOStore,  VT.getStoreSize(),
+                         Alignment, AAInfo);
+  if (!UniformBase) {
+    Base = DAG.getTargetConstant(0, sdl, TLI.getPointerTy(DAG.getDataLayout()));
+    Index = getValue(Ptr);
+  }
+  SDValue Ops[] = { getRoot(), Src0, Mask, Base, Index };
+  SDValue Scatter = DAG.getMaskedScatter(DAG.getVTList(MVT::Other), VT, sdl,
+                                         Ops, MMO);
+  DAG.setRoot(Scatter);
+  setValue(&I, Scatter);
+}
+
 void SelectionDAGBuilder::visitMaskedLoad(const CallInst &I) {
   SDLoc sdl = getCurSDLoc();
 
@@ -3674,7 +3192,7 @@ void SelectionDAGBuilder::visitMaskedLoad(const CallInst &I) {
   SDValue Mask = getValue(I.getArgOperand(2));
 
   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
-  EVT VT = TLI.getValueType(I.getType());
+  EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType());
   unsigned Alignment = (cast<ConstantInt>(I.getArgOperand(1)))->getZExtValue();
   if (!Alignment)
     Alignment = DAG.getEVTAlignment(VT);
@@ -3684,10 +3202,8 @@ void SelectionDAGBuilder::visitMaskedLoad(const CallInst &I) {
   const MDNode *Ranges = I.getMetadata(LLVMContext::MD_range);
 
   SDValue InChain = DAG.getRoot();
-  if (AA->pointsToConstantMemory(
-      AliasAnalysis::Location(PtrOperand,
-                              AA->getTypeStoreSize(I.getType()),
-                              AAInfo))) {
+  if (AA->pointsToConstantMemory(MemoryLocation(
+          PtrOperand, AA->getTypeStoreSize(I.getType()), AAInfo))) {
     // Do not serialize (non-volatile) loads of constant memory with anything.
     InChain = DAG.getEntryNode();
   }
@@ -3705,6 +3221,58 @@ void SelectionDAGBuilder::visitMaskedLoad(const CallInst &I) {
   setValue(&I, Load);
 }
 
+void SelectionDAGBuilder::visitMaskedGather(const CallInst &I) {
+  SDLoc sdl = getCurSDLoc();
+
+  // @llvm.masked.gather.*(Ptrs, alignment, Mask, Src0)
+  Value  *Ptr = I.getArgOperand(0);
+  SDValue Src0 = getValue(I.getArgOperand(3));
+  SDValue Mask = getValue(I.getArgOperand(2));
+
+  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
+  EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType());
+  unsigned Alignment = (cast<ConstantInt>(I.getArgOperand(1)))->getZExtValue();
+  if (!Alignment)
+    Alignment = DAG.getEVTAlignment(VT);
+
+  AAMDNodes AAInfo;
+  I.getAAMetadata(AAInfo);
+  const MDNode *Ranges = I.getMetadata(LLVMContext::MD_range);
+
+  SDValue Root = DAG.getRoot();
+  SDValue Base;
+  SDValue Index;
+  Value *BasePtr = Ptr;
+  bool UniformBase = getUniformBase(BasePtr, Base, Index, this);
+  bool ConstantMemory = false;
+  if (UniformBase &&
+      AA->pointsToConstantMemory(
+          MemoryLocation(BasePtr, AA->getTypeStoreSize(I.getType()), AAInfo))) {
+    // Do not serialize (non-volatile) loads of constant memory with anything.
+    Root = DAG.getEntryNode();
+    ConstantMemory = true;
+  }
+
+  MachineMemOperand *MMO =
+    DAG.getMachineFunction().
+    getMachineMemOperand(MachinePointerInfo(UniformBase ? BasePtr : nullptr),
+                         MachineMemOperand::MOLoad,  VT.getStoreSize(),
+                         Alignment, AAInfo, Ranges);
+
+  if (!UniformBase) {
+    Base = DAG.getTargetConstant(0, sdl, TLI.getPointerTy(DAG.getDataLayout()));
+    Index = getValue(Ptr);
+  }
+  SDValue Ops[] = { Root, Src0, Mask, Base, Index };
+  SDValue Gather = DAG.getMaskedGather(DAG.getVTList(VT, MVT::Other), VT, sdl,
+                                       Ops, MMO);
+
+  SDValue OutChain = Gather.getValue(1);
+  if (!ConstantMemory)
+    PendingLoads.push_back(OutChain);
+  setValue(&I, Gather);
+}
+
 void SelectionDAGBuilder::visitAtomicCmpXchg(const AtomicCmpXchgInst &I) {
   SDLoc dl = getCurSDLoc();
   AtomicOrdering SuccessOrder = I.getSuccessOrdering();
@@ -3769,8 +3337,10 @@ void SelectionDAGBuilder::visitFence(const FenceInst &I) {
   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
   SDValue Ops[3];
   Ops[0] = getRoot();
-  Ops[1] = DAG.getConstant(I.getOrdering(), TLI.getPointerTy());
-  Ops[2] = DAG.getConstant(I.getSynchScope(), TLI.getPointerTy());
+  Ops[1] = DAG.getConstant(I.getOrdering(), dl,
+                           TLI.getPointerTy(DAG.getDataLayout()));
+  Ops[2] = DAG.getConstant(I.getSynchScope(), dl,
+                           TLI.getPointerTy(DAG.getDataLayout()));
   DAG.setRoot(DAG.getNode(ISD::ATOMIC_FENCE, dl, MVT::Other, Ops));
 }
 
@@ -3782,7 +3352,7 @@ void SelectionDAGBuilder::visitAtomicLoad(const LoadInst &I) {
   SDValue InChain = getRoot();
 
   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
-  EVT VT = TLI.getValueType(I.getType());
+  EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType());
 
   if (I.getAlignment() < VT.getSizeInBits() / 8)
     report_fatal_error("Cannot generate unaligned atomic load");
@@ -3817,7 +3387,8 @@ void SelectionDAGBuilder::visitAtomicStore(const StoreInst &I) {
   SDValue InChain = getRoot();
 
   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
-  EVT VT = TLI.getValueType(I.getValueOperand()->getType());
+  EVT VT =
+      TLI.getValueType(DAG.getDataLayout(), I.getValueOperand()->getType());
 
   if (I.getAlignment() < VT.getSizeInBits() / 8)
     report_fatal_error("Cannot generate unaligned atomic store");
@@ -3859,7 +3430,8 @@ void SelectionDAGBuilder::visitTargetIntrinsic(const CallInst &I,
   // Add the intrinsic ID as an integer operand if it's not a target intrinsic.
   if (!IsTgtIntrinsic || Info.opc == ISD::INTRINSIC_VOID ||
       Info.opc == ISD::INTRINSIC_W_CHAIN)
-    Ops.push_back(DAG.getTargetConstant(Intrinsic, TLI.getPointerTy()));
+    Ops.push_back(DAG.getTargetConstant(Intrinsic, getCurSDLoc(),
+                                        TLI.getPointerTy(DAG.getDataLayout())));
 
   // Add all operands of the call to the operand list.
   for (unsigned i = 0, e = I.getNumArgOperands(); i != e; ++i) {
@@ -3868,7 +3440,7 @@ void SelectionDAGBuilder::visitTargetIntrinsic(const CallInst &I,
   }
 
   SmallVector<EVT, 4> ValueVTs;
-  ComputeValueVTs(TLI, I.getType(), ValueVTs);
+  ComputeValueVTs(TLI, DAG.getDataLayout(), I.getType(), ValueVTs);
 
   if (HasChain)
     ValueVTs.push_back(MVT::Other);
@@ -3902,7 +3474,7 @@ void SelectionDAGBuilder::visitTargetIntrinsic(const CallInst &I,
 
   if (!I.getType()->isVoidTy()) {
     if (VectorType *PTy = dyn_cast<VectorType>(I.getType())) {
-      EVT VT = TLI.getValueType(PTy);
+      EVT VT = TLI.getValueType(DAG.getDataLayout(), PTy);
       Result = DAG.getNode(ISD::BITCAST, getCurSDLoc(), VT, Result);
     }
 
@@ -3919,9 +3491,9 @@ void SelectionDAGBuilder::visitTargetIntrinsic(const CallInst &I,
 static SDValue
 GetSignificand(SelectionDAG &DAG, SDValue Op, SDLoc dl) {
   SDValue t1 = DAG.getNode(ISD::AND, dl, MVT::i32, Op,
-                           DAG.getConstant(0x007fffff, MVT::i32));
+                           DAG.getConstant(0x007fffff, dl, MVT::i32));
   SDValue t2 = DAG.getNode(ISD::OR, dl, MVT::i32, t1,
-                           DAG.getConstant(0x3f800000, MVT::i32));
+                           DAG.getConstant(0x3f800000, dl, MVT::i32));
   return DAG.getNode(ISD::BITCAST, dl, MVT::f32, t2);
 }
 
@@ -3934,18 +3506,19 @@ static SDValue
 GetExponent(SelectionDAG &DAG, SDValue Op, const TargetLowering &TLI,
             SDLoc dl) {
   SDValue t0 = DAG.getNode(ISD::AND, dl, MVT::i32, Op,
-                           DAG.getConstant(0x7f800000, MVT::i32));
-  SDValue t1 = DAG.getNode(ISD::SRL, dl, MVT::i32, t0,
-                           DAG.getConstant(23, TLI.getPointerTy()));
+                           DAG.getConstant(0x7f800000, dl, MVT::i32));
+  SDValue t1 = DAG.getNode(
+      ISD::SRL, dl, MVT::i32, t0,
+      DAG.getConstant(23, dl, TLI.getPointerTy(DAG.getDataLayout())));
   SDValue t2 = DAG.getNode(ISD::SUB, dl, MVT::i32, t1,
-                           DAG.getConstant(127, MVT::i32));
+                           DAG.getConstant(127, dl, MVT::i32));
   return DAG.getNode(ISD::SINT_TO_FP, dl, MVT::f32, t2);
 }
 
 /// getF32Constant - Get 32-bit floating point constant.
 static SDValue
-getF32Constant(SelectionDAG &DAG, unsigned Flt) {
-  return DAG.getConstantFP(APFloat(APFloat::IEEEsingle, APInt(32, Flt)),
+getF32Constant(SelectionDAG &DAG, unsigned Flt, SDLoc dl) {
+  return DAG.getConstantFP(APFloat(APFloat::IEEEsingle, APInt(32, Flt)), dl,
                            MVT::f32);
 }
 
@@ -3961,7 +3534,8 @@ static SDValue getLimitedPrecisionExp2(SDValue t0, SDLoc dl,
   //   IntegerPartOfX <<= 23;
   IntegerPartOfX = DAG.getNode(
       ISD::SHL, dl, MVT::i32, IntegerPartOfX,
-      DAG.getConstant(23, DAG.getTargetLoweringInfo().getPointerTy()));
+      DAG.getConstant(23, dl, DAG.getTargetLoweringInfo().getPointerTy(
+                                  DAG.getDataLayout())));
 
   SDValue TwoToFractionalPartOfX;
   if (LimitFloatPrecision <= 6) {
@@ -3973,12 +3547,12 @@ static SDValue getLimitedPrecisionExp2(SDValue t0, SDLoc dl,
     //
     // error 0.0144103317, which is 6 bits
     SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
-                             getF32Constant(DAG, 0x3e814304));
+                             getF32Constant(DAG, 0x3e814304, dl));
     SDValue t3 = DAG.getNode(ISD::FADD, dl, MVT::f32, t2,
-                             getF32Constant(DAG, 0x3f3c50c8));
+                             getF32Constant(DAG, 0x3f3c50c8, dl));
     SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
     TwoToFractionalPartOfX = DAG.getNode(ISD::FADD, dl, MVT::f32, t4,
-                                         getF32Constant(DAG, 0x3f7f5e7e));
+                                         getF32Constant(DAG, 0x3f7f5e7e, dl));
   } else if (LimitFloatPrecision <= 12) {
     // For floating-point precision of 12:
     //
@@ -3989,15 +3563,15 @@ static SDValue getLimitedPrecisionExp2(SDValue t0, SDLoc dl,
     //
     // error 0.000107046256, which is 13 to 14 bits
     SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
-                             getF32Constant(DAG, 0x3da235e3));
+                             getF32Constant(DAG, 0x3da235e3, dl));
     SDValue t3 = DAG.getNode(ISD::FADD, dl, MVT::f32, t2,
-                             getF32Constant(DAG, 0x3e65b8f3));
+                             getF32Constant(DAG, 0x3e65b8f3, dl));
     SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
     SDValue t5 = DAG.getNode(ISD::FADD, dl, MVT::f32, t4,
-                             getF32Constant(DAG, 0x3f324b07));
+                             getF32Constant(DAG, 0x3f324b07, dl));
     SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X);
     TwoToFractionalPartOfX = DAG.getNode(ISD::FADD, dl, MVT::f32, t6,
-                                         getF32Constant(DAG, 0x3f7ff8fd));
+                                         getF32Constant(DAG, 0x3f7ff8fd, dl));
   } else { // LimitFloatPrecision <= 18
     // For floating-point precision of 18:
     //
@@ -4010,24 +3584,24 @@ static SDValue getLimitedPrecisionExp2(SDValue t0, SDLoc dl,
     //               (0.136028312e-2f + 0.157059148e-3f *x)*x)*x)*x)*x)*x;
     // error 2.47208000*10^(-7), which is better than 18 bits
     SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
-                             getF32Constant(DAG, 0x3924b03e));
+                             getF32Constant(DAG, 0x3924b03e, dl));
     SDValue t3 = DAG.getNode(ISD::FADD, dl, MVT::f32, t2,
-                             getF32Constant(DAG, 0x3ab24b87));
+                             getF32Constant(DAG, 0x3ab24b87, dl));
     SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
     SDValue t5 = DAG.getNode(ISD::FADD, dl, MVT::f32, t4,
-                             getF32Constant(DAG, 0x3c1d8c17));
+                             getF32Constant(DAG, 0x3c1d8c17, dl));
     SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X);
     SDValue t7 = DAG.getNode(ISD::FADD, dl, MVT::f32, t6,
-                             getF32Constant(DAG, 0x3d634a1d));
+                             getF32Constant(DAG, 0x3d634a1d, dl));
     SDValue t8 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t7, X);
     SDValue t9 = DAG.getNode(ISD::FADD, dl, MVT::f32, t8,
-                             getF32Constant(DAG, 0x3e75fe14));
+                             getF32Constant(DAG, 0x3e75fe14, dl));
     SDValue t10 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t9, X);
     SDValue t11 = DAG.getNode(ISD::FADD, dl, MVT::f32, t10,
-                              getF32Constant(DAG, 0x3f317234));
+                              getF32Constant(DAG, 0x3f317234, dl));
     SDValue t12 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t11, X);
     TwoToFractionalPartOfX = DAG.getNode(ISD::FADD, dl, MVT::f32, t12,
-                                         getF32Constant(DAG, 0x3f800000));
+                                         getF32Constant(DAG, 0x3f800000, dl));
   }
 
   // Add the exponent into the result in integer domain.
@@ -4049,7 +3623,7 @@ static SDValue expandExp(SDLoc dl, SDValue Op, SelectionDAG &DAG,
     //   #define LOG2OFe 1.4426950f
     //   t0 = Op * LOG2OFe
     SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, Op,
-                             getF32Constant(DAG, 0x3fb8aa3b));
+                             getF32Constant(DAG, 0x3fb8aa3b, dl));
     return getLimitedPrecisionExp2(t0, dl, DAG);
   }
 
@@ -4068,7 +3642,7 @@ static SDValue expandLog(SDLoc dl, SDValue Op, SelectionDAG &DAG,
     // Scale the exponent by log(2) [0.69314718f].
     SDValue Exp = GetExponent(DAG, Op1, TLI, dl);
     SDValue LogOfExponent = DAG.getNode(ISD::FMUL, dl, MVT::f32, Exp,
-                                        getF32Constant(DAG, 0x3f317218));
+                                        getF32Constant(DAG, 0x3f317218, dl));
 
     // Get the significand and build it into a floating-point number with
     // exponent of 1.
@@ -4084,12 +3658,12 @@ static SDValue expandLog(SDLoc dl, SDValue Op, SelectionDAG &DAG,
       //
       // error 0.0034276066, which is better than 8 bits
       SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
-                               getF32Constant(DAG, 0xbe74c456));
+                               getF32Constant(DAG, 0xbe74c456, dl));
       SDValue t1 = DAG.getNode(ISD::FADD, dl, MVT::f32, t0,
-                               getF32Constant(DAG, 0x3fb3a2b1));
+                               getF32Constant(DAG, 0x3fb3a2b1, dl));
       SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);
       LogOfMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t2,
-                                  getF32Constant(DAG, 0x3f949a29));
+                                  getF32Constant(DAG, 0x3f949a29, dl));
     } else if (LimitFloatPrecision <= 12) {
       // For floating-point precision of 12:
       //
@@ -4101,18 +3675,18 @@ static SDValue expandLog(SDLoc dl, SDValue Op, SelectionDAG &DAG,
       //
       // error 0.000061011436, which is 14 bits
       SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
-                               getF32Constant(DAG, 0xbd67b6d6));
+                               getF32Constant(DAG, 0xbd67b6d6, dl));
       SDValue t1 = DAG.getNode(ISD::FADD, dl, MVT::f32, t0,
-                               getF32Constant(DAG, 0x3ee4f4b8));
+                               getF32Constant(DAG, 0x3ee4f4b8, dl));
       SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);
       SDValue t3 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t2,
-                               getF32Constant(DAG, 0x3fbc278b));
+                               getF32Constant(DAG, 0x3fbc278b, dl));
       SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
       SDValue t5 = DAG.getNode(ISD::FADD, dl, MVT::f32, t4,
-                               getF32Constant(DAG, 0x40348e95));
+                               getF32Constant(DAG, 0x40348e95, dl));
       SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X);
       LogOfMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t6,
-                                  getF32Constant(DAG, 0x3fdef31a));
+                                  getF32Constant(DAG, 0x3fdef31a, dl));
     } else { // LimitFloatPrecision <= 18
       // For floating-point precision of 18:
       //
@@ -4126,24 +3700,24 @@ static SDValue expandLog(SDLoc dl, SDValue Op, SelectionDAG &DAG,
       //
       // error 0.0000023660568, which is better than 18 bits
       SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
-                               getF32Constant(DAG, 0xbc91e5ac));
+                               getF32Constant(DAG, 0xbc91e5ac, dl));
       SDValue t1 = DAG.getNode(ISD::FADD, dl, MVT::f32, t0,
-                               getF32Constant(DAG, 0x3e4350aa));
+                               getF32Constant(DAG, 0x3e4350aa, dl));
       SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);
       SDValue t3 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t2,
-                               getF32Constant(DAG, 0x3f60d3e3));
+                               getF32Constant(DAG, 0x3f60d3e3, dl));
       SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
       SDValue t5 = DAG.getNode(ISD::FADD, dl, MVT::f32, t4,
-                               getF32Constant(DAG, 0x4011cdf0));
+                               getF32Constant(DAG, 0x4011cdf0, dl));
       SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X);
       SDValue t7 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t6,
-                               getF32Constant(DAG, 0x406cfd1c));
+                               getF32Constant(DAG, 0x406cfd1c, dl));
       SDValue t8 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t7, X);
       SDValue t9 = DAG.getNode(ISD::FADD, dl, MVT::f32, t8,
-                               getF32Constant(DAG, 0x408797cb));
+                               getF32Constant(DAG, 0x408797cb, dl));
       SDValue t10 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t9, X);
       LogOfMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t10,
-                                  getF32Constant(DAG, 0x4006dcab));
+                                  getF32Constant(DAG, 0x4006dcab, dl));
     }
 
     return DAG.getNode(ISD::FADD, dl, MVT::f32, LogOfExponent, LogOfMantissa);
@@ -4178,12 +3752,12 @@ static SDValue expandLog2(SDLoc dl, SDValue Op, SelectionDAG &DAG,
       //
       // error 0.0049451742, which is more than 7 bits
       SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
-                               getF32Constant(DAG, 0xbeb08fe0));
+                               getF32Constant(DAG, 0xbeb08fe0, dl));
       SDValue t1 = DAG.getNode(ISD::FADD, dl, MVT::f32, t0,
-                               getF32Constant(DAG, 0x40019463));
+                               getF32Constant(DAG, 0x40019463, dl));
       SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);
       Log2ofMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t2,
-                                   getF32Constant(DAG, 0x3fd6633d));
+                                   getF32Constant(DAG, 0x3fd6633d, dl));
     } else if (LimitFloatPrecision <= 12) {
       // For floating-point precision of 12:
       //
@@ -4195,18 +3769,18 @@ static SDValue expandLog2(SDLoc dl, SDValue Op, SelectionDAG &DAG,
       //
       // error 0.0000876136000, which is better than 13 bits
       SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
-                               getF32Constant(DAG, 0xbda7262e));
+                               getF32Constant(DAG, 0xbda7262e, dl));
       SDValue t1 = DAG.getNode(ISD::FADD, dl, MVT::f32, t0,
-                               getF32Constant(DAG, 0x3f25280b));
+                               getF32Constant(DAG, 0x3f25280b, dl));
       SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);
       SDValue t3 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t2,
-                               getF32Constant(DAG, 0x4007b923));
+                               getF32Constant(DAG, 0x4007b923, dl));
       SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
       SDValue t5 = DAG.getNode(ISD::FADD, dl, MVT::f32, t4,
-                               getF32Constant(DAG, 0x40823e2f));
+                               getF32Constant(DAG, 0x40823e2f, dl));
       SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X);
       Log2ofMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t6,
-                                   getF32Constant(DAG, 0x4020d29c));
+                                   getF32Constant(DAG, 0x4020d29c, dl));
     } else { // LimitFloatPrecision <= 18
       // For floating-point precision of 18:
       //
@@ -4221,24 +3795,24 @@ static SDValue expandLog2(SDLoc dl, SDValue Op, SelectionDAG &DAG,
       //
       // error 0.0000018516, which is better than 18 bits
       SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
-                               getF32Constant(DAG, 0xbcd2769e));
+                               getF32Constant(DAG, 0xbcd2769e, dl));
       SDValue t1 = DAG.getNode(ISD::FADD, dl, MVT::f32, t0,
-                               getF32Constant(DAG, 0x3e8ce0b9));
+                               getF32Constant(DAG, 0x3e8ce0b9, dl));
       SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);
       SDValue t3 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t2,
-                               getF32Constant(DAG, 0x3fa22ae7));
+                               getF32Constant(DAG, 0x3fa22ae7, dl));
       SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
       SDValue t5 = DAG.getNode(ISD::FADD, dl, MVT::f32, t4,
-                               getF32Constant(DAG, 0x40525723));
+                               getF32Constant(DAG, 0x40525723, dl));
       SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X);
       SDValue t7 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t6,
-                               getF32Constant(DAG, 0x40aaf200));
+                               getF32Constant(DAG, 0x40aaf200, dl));
       SDValue t8 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t7, X);
       SDValue t9 = DAG.getNode(ISD::FADD, dl, MVT::f32, t8,
-                               getF32Constant(DAG, 0x40c39dad));
+                               getF32Constant(DAG, 0x40c39dad, dl));
       SDValue t10 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t9, X);
       Log2ofMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t10,
-                                   getF32Constant(DAG, 0x4042902c));
+                                   getF32Constant(DAG, 0x4042902c, dl));
     }
 
     return DAG.getNode(ISD::FADD, dl, MVT::f32, LogOfExponent, Log2ofMantissa);
@@ -4259,7 +3833,7 @@ static SDValue expandLog10(SDLoc dl, SDValue Op, SelectionDAG &DAG,
     // Scale the exponent by log10(2) [0.30102999f].
     SDValue Exp = GetExponent(DAG, Op1, TLI, dl);
     SDValue LogOfExponent = DAG.getNode(ISD::FMUL, dl, MVT::f32, Exp,
-                                        getF32Constant(DAG, 0x3e9a209a));
+                                        getF32Constant(DAG, 0x3e9a209a, dl));
 
     // Get the significand and build it into a floating-point number with
     // exponent of 1.
@@ -4275,12 +3849,12 @@ static SDValue expandLog10(SDLoc dl, SDValue Op, SelectionDAG &DAG,
       //
       // error 0.0014886165, which is 6 bits
       SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
-                               getF32Constant(DAG, 0xbdd49a13));
+                               getF32Constant(DAG, 0xbdd49a13, dl));
       SDValue t1 = DAG.getNode(ISD::FADD, dl, MVT::f32, t0,
-                               getF32Constant(DAG, 0x3f1c0789));
+                               getF32Constant(DAG, 0x3f1c0789, dl));
       SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);
       Log10ofMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t2,
-                                    getF32Constant(DAG, 0x3f011300));
+                                    getF32Constant(DAG, 0x3f011300, dl));
     } else if (LimitFloatPrecision <= 12) {
       // For floating-point precision of 12:
       //
@@ -4291,15 +3865,15 @@ static SDValue expandLog10(SDLoc dl, SDValue Op, SelectionDAG &DAG,
       //
       // error 0.00019228036, which is better than 12 bits
       SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
-                               getF32Constant(DAG, 0x3d431f31));
+                               getF32Constant(DAG, 0x3d431f31, dl));
       SDValue t1 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t0,
-                               getF32Constant(DAG, 0x3ea21fb2));
+                               getF32Constant(DAG, 0x3ea21fb2, dl));
       SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);
       SDValue t3 = DAG.getNode(ISD::FADD, dl, MVT::f32, t2,
-                               getF32Constant(DAG, 0x3f6ae232));
+                               getF32Constant(DAG, 0x3f6ae232, dl));
       SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
       Log10ofMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t4,
-                                    getF32Constant(DAG, 0x3f25f7c3));
+                                    getF32Constant(DAG, 0x3f25f7c3, dl));
     } else { // LimitFloatPrecision <= 18
       // For floating-point precision of 18:
       //
@@ -4312,21 +3886,21 @@ static SDValue expandLog10(SDLoc dl, SDValue Op, SelectionDAG &DAG,
       //
       // error 0.0000037995730, which is better than 18 bits
       SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
-                               getF32Constant(DAG, 0x3c5d51ce));
+                               getF32Constant(DAG, 0x3c5d51ce, dl));
       SDValue t1 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t0,
-                               getF32Constant(DAG, 0x3e00685a));
+                               getF32Constant(DAG, 0x3e00685a, dl));
       SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);
       SDValue t3 = DAG.getNode(ISD::FADD, dl, MVT::f32, t2,
-                               getF32Constant(DAG, 0x3efb6798));
+                               getF32Constant(DAG, 0x3efb6798, dl));
       SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
       SDValue t5 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t4,
-                               getF32Constant(DAG, 0x3f88d192));
+                               getF32Constant(DAG, 0x3f88d192, dl));
       SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X);
       SDValue t7 = DAG.getNode(ISD::FADD, dl, MVT::f32, t6,
-                               getF32Constant(DAG, 0x3fc4316c));
+                               getF32Constant(DAG, 0x3fc4316c, dl));
       SDValue t8 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t7, X);
       Log10ofMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t8,
-                                    getF32Constant(DAG, 0x3f57ce70));
+                                    getF32Constant(DAG, 0x3f57ce70, dl));
     }
 
     return DAG.getNode(ISD::FADD, dl, MVT::f32, LogOfExponent, Log10ofMantissa);
@@ -4368,7 +3942,7 @@ static SDValue expandPow(SDLoc dl, SDValue LHS, SDValue RHS,
     //   #define LOG2OF10 3.3219281f
     //   t0 = Op * LOG2OF10;
     SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, RHS,
-                             getF32Constant(DAG, 0x40549a78));
+                             getF32Constant(DAG, 0x40549a78, dl));
     return getLimitedPrecisionExp2(t0, dl, DAG);
   }
 
@@ -4391,7 +3965,7 @@ static SDValue ExpandPowI(SDLoc DL, SDValue LHS, SDValue RHS,
 
     // powi(x, 0) -> 1.0
     if (Val == 0)
-      return DAG.getConstantFP(1.0, LHS.getValueType());
+      return DAG.getConstantFP(1.0, DL, LHS.getValueType());
 
     const Function *F = DAG.getMachineFunction().getFunction();
     if (!F->hasFnAttribute(Attribute::OptimizeForSize) ||
@@ -4420,7 +3994,7 @@ static SDValue ExpandPowI(SDLoc DL, SDValue LHS, SDValue RHS,
       // If the original was negative, invert the result, producing 1/(x*x*x).
       if (RHSC->getSExtValue() < 0)
         Res = DAG.getNode(ISD::FDIV, DL, LHS.getValueType(),
-                          DAG.getConstantFP(1.0, LHS.getValueType()), Res);
+                          DAG.getConstantFP(1.0, DL, LHS.getValueType()), Res);
       return Res;
     }
   }
@@ -4451,8 +4025,8 @@ static unsigned getTruncatedArgReg(const SDValue &N) {
 /// argument, create the corresponding DBG_VALUE machine instruction for it now.
 /// At the end of instruction selection, they will be inserted to the entry BB.
 bool SelectionDAGBuilder::EmitFuncArgumentDbgValue(
-    const Value *V, MDLocalVariable *Variable, MDExpression *Expr,
-    MDLocation *DL, int64_t Offset, bool IsIndirect, const SDValue &N) {
+    const Value *V, DILocalVariable *Variable, DIExpression *Expr,
+    DILocation *DL, int64_t Offset, bool IsIndirect, const SDValue &N) {
   const Argument *Arg = dyn_cast<Argument>(V);
   if (!Arg)
     return false;
@@ -4463,8 +4037,7 @@ bool SelectionDAGBuilder::EmitFuncArgumentDbgValue(
   // Ignore inlined function arguments here.
   //
   // FIXME: Should we be checking DL->inlinedAt() to determine this?
-  DIVariable DV(Variable);
-  if (!DV->getScope()->getSubprogram()->describes(MF.getFunction()))
+  if (!Variable->getScope()->getSubprogram()->describes(MF.getFunction()))
     return false;
 
   Optional<MachineOperand> Op;
@@ -4549,25 +4122,31 @@ SelectionDAGBuilder::visitIntrinsicCall(const CallInst &I, unsigned Intrinsic) {
   case Intrinsic::vaend:    visitVAEnd(I); return nullptr;
   case Intrinsic::vacopy:   visitVACopy(I); return nullptr;
   case Intrinsic::returnaddress:
-    setValue(&I, DAG.getNode(ISD::RETURNADDR, sdl, TLI.getPointerTy(),
+    setValue(&I, DAG.getNode(ISD::RETURNADDR, sdl,
+                             TLI.getPointerTy(DAG.getDataLayout()),
                              getValue(I.getArgOperand(0))));
     return nullptr;
   case Intrinsic::frameaddress:
-    setValue(&I, DAG.getNode(ISD::FRAMEADDR, sdl, TLI.getPointerTy(),
+    setValue(&I, DAG.getNode(ISD::FRAMEADDR, sdl,
+                             TLI.getPointerTy(DAG.getDataLayout()),
                              getValue(I.getArgOperand(0))));
     return nullptr;
   case Intrinsic::read_register: {
     Value *Reg = I.getArgOperand(0);
+    SDValue Chain = getRoot();
     SDValue RegName =
         DAG.getMDNode(cast<MDNode>(cast<MetadataAsValue>(Reg)->getMetadata()));
-    EVT VT = TLI.getValueType(I.getType());
-    setValue(&I, DAG.getNode(ISD::READ_REGISTER, sdl, VT, RegName));
+    EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType());
+    Res = DAG.getNode(ISD::READ_REGISTER, sdl,
+      DAG.getVTList(VT, MVT::Other), Chain, RegName);
+    setValue(&I, Res);
+    DAG.setRoot(Res.getValue(1));
     return nullptr;
   }
   case Intrinsic::write_register: {
     Value *Reg = I.getArgOperand(0);
     Value *RegValue = I.getArgOperand(1);
-    SDValue Chain = getValue(RegValue).getOperand(0);
+    SDValue Chain = getRoot();
     SDValue RegName =
         DAG.getMDNode(cast<MDNode>(cast<MetadataAsValue>(Reg)->getMetadata()));
     DAG.setRoot(DAG.getNode(ISD::WRITE_REGISTER, sdl, MVT::Other, Chain,
@@ -4647,11 +4226,11 @@ SelectionDAGBuilder::visitIntrinsicCall(const CallInst &I, unsigned Intrinsic) {
   }
   case Intrinsic::dbg_declare: {
     const DbgDeclareInst &DI = cast<DbgDeclareInst>(I);
-    MDLocalVariable *Variable = DI.getVariable();
-    MDExpression *Expression = DI.getExpression();
+    DILocalVariable *Variable = DI.getVariable();
+    DIExpression *Expression = DI.getExpression();
     const Value *Address = DI.getAddress();
-    DIVariable DIVar = Variable;
-    if (!Address || !DIVar) {
+    assert(Variable && "Missing variable");
+    if (!Address) {
       DEBUG(dbgs() << "Dropping debug info for " << DI << "\n");
       return nullptr;
     }
@@ -4672,9 +4251,8 @@ SelectionDAGBuilder::visitIntrinsicCall(const CallInst &I, unsigned Intrinsic) {
       if (const BitCastInst *BCI = dyn_cast<BitCastInst>(Address))
         Address = BCI->getOperand(0);
       // Parameters are handled specially.
-      bool isParameter =
-        (DIVariable(Variable)->getTag() == dwarf::DW_TAG_arg_variable ||
-         isa<Argument>(Address));
+      bool isParameter = Variable->getTag() == dwarf::DW_TAG_arg_variable ||
+                         isa<Argument>(Address);
 
       const AllocaInst *AI = dyn_cast<AllocaInst>(Address);
 
@@ -4728,12 +4306,10 @@ SelectionDAGBuilder::visitIntrinsicCall(const CallInst &I, unsigned Intrinsic) {
   }
   case Intrinsic::dbg_value: {
     const DbgValueInst &DI = cast<DbgValueInst>(I);
-    DIVariable DIVar = DI.getVariable();
-    if (!DIVar)
-      return nullptr;
+    assert(DI.getVariable() && "Missing variable");
 
-    MDLocalVariable *Variable = DI.getVariable();
-    MDExpression *Expression = DI.getExpression();
+    DILocalVariable *Variable = DI.getVariable();
+    DIExpression *Expression = DI.getExpression();
     uint64_t Offset = DI.getOffset();
     const Value *V = DI.getValue();
     if (!V)
@@ -4793,7 +4369,7 @@ SelectionDAGBuilder::visitIntrinsicCall(const CallInst &I, unsigned Intrinsic) {
     // Find the type id for the given typeinfo.
     GlobalValue *GV = ExtractTypeInfo(I.getArgOperand(0));
     unsigned TypeID = DAG.getMachineFunction().getMMI().getTypeIDFor(GV);
-    Res = DAG.getConstant(TypeID, MVT::i32);
+    Res = DAG.getConstant(TypeID, sdl, MVT::i32);
     setValue(&I, Res);
     return nullptr;
   }
@@ -4812,14 +4388,15 @@ SelectionDAGBuilder::visitIntrinsicCall(const CallInst &I, unsigned Intrinsic) {
     return nullptr;
   case Intrinsic::eh_dwarf_cfa: {
     SDValue CfaArg = DAG.getSExtOrTrunc(getValue(I.getArgOperand(0)), sdl,
-                                        TLI.getPointerTy());
+                                        TLI.getPointerTy(DAG.getDataLayout()));
     SDValue Offset = DAG.getNode(ISD::ADD, sdl,
                                  CfaArg.getValueType(),
                                  DAG.getNode(ISD::FRAME_TO_ARGS_OFFSET, sdl,
                                              CfaArg.getValueType()),
                                  CfaArg);
-    SDValue FA = DAG.getNode(ISD::FRAMEADDR, sdl, TLI.getPointerTy(),
-                             DAG.getConstant(0, TLI.getPointerTy()));
+    SDValue FA = DAG.getNode(
+        ISD::FRAMEADDR, sdl, TLI.getPointerTy(DAG.getDataLayout()),
+        DAG.getConstant(0, sdl, TLI.getPointerTy(DAG.getDataLayout())));
     setValue(&I, DAG.getNode(ISD::ADD, sdl, FA.getValueType(),
                              FA, Offset));
     return nullptr;
@@ -4857,10 +4434,21 @@ SelectionDAGBuilder::visitIntrinsicCall(const CallInst &I, unsigned Intrinsic) {
                             getRoot(), getValue(I.getArgOperand(0))));
     return nullptr;
   }
+  case Intrinsic::eh_sjlj_setup_dispatch: {
+    DAG.setRoot(DAG.getNode(ISD::EH_SJLJ_SETUP_DISPATCH, sdl, MVT::Other,
+                            getRoot()));
+    return nullptr;
+  }
 
+  case Intrinsic::masked_gather:
+    visitMaskedGather(I);
+    return nullptr;
   case Intrinsic::masked_load:
     visitMaskedLoad(I);
     return nullptr;
+  case Intrinsic::masked_scatter:
+    visitMaskedScatter(I);
+    return nullptr;
   case Intrinsic::masked_store:
     visitMaskedStore(I);
     return nullptr;
@@ -4913,12 +4501,12 @@ SelectionDAGBuilder::visitIntrinsicCall(const CallInst &I, unsigned Intrinsic) {
     // We must do this early because v2i32 is not a legal type.
     SDValue ShOps[2];
     ShOps[0] = ShAmt;
-    ShOps[1] = DAG.getConstant(0, MVT::i32);
+    ShOps[1] = DAG.getConstant(0, sdl, MVT::i32);
     ShAmt =  DAG.getNode(ISD::BUILD_VECTOR, sdl, ShAmtVT, ShOps);
-    EVT DestVT = TLI.getValueType(I.getType());
+    EVT DestVT = TLI.getValueType(DAG.getDataLayout(), I.getType());
     ShAmt = DAG.getNode(ISD::BITCAST, sdl, DestVT, ShAmt);
     Res = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, sdl, DestVT,
-                       DAG.getConstant(NewIntrinsic, MVT::i32),
+                       DAG.getConstant(NewIntrinsic, sdl, MVT::i32),
                        getValue(I.getArgOperand(0)), ShAmt);
     setValue(&I, Res);
     return nullptr;
@@ -4945,7 +4533,7 @@ SelectionDAGBuilder::visitIntrinsicCall(const CallInst &I, unsigned Intrinsic) {
     case Intrinsic::convertus:  Code = ISD::CVT_US; break;
     case Intrinsic::convertuu:  Code = ISD::CVT_UU; break;
     }
-    EVT DestVT = TLI.getValueType(I.getType());
+    EVT DestVT = TLI.getValueType(DAG.getDataLayout(), I.getType());
     const Value *Op1 = I.getArgOperand(0);
     Res = DAG.getConvertRndSat(DestVT, sdl, getValue(Op1),
                                DAG.getValueType(DestVT),
@@ -5035,7 +4623,7 @@ SelectionDAGBuilder::visitIntrinsicCall(const CallInst &I, unsigned Intrinsic) {
                              getValue(I.getArgOperand(2))));
     return nullptr;
   case Intrinsic::fmuladd: {
-    EVT VT = TLI.getValueType(I.getType());
+    EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType());
     if (TM.Options.AllowFPOpFusion != FPOpFusion::Strict &&
         TLI.isFMAFasterThanFMulAndFAdd(VT)) {
       setValue(&I, DAG.getNode(ISD::FMA, sdl,
@@ -5060,13 +4648,14 @@ SelectionDAGBuilder::visitIntrinsicCall(const CallInst &I, unsigned Intrinsic) {
     setValue(&I, DAG.getNode(ISD::BITCAST, sdl, MVT::i16,
                              DAG.getNode(ISD::FP_ROUND, sdl, MVT::f16,
                                          getValue(I.getArgOperand(0)),
-                                         DAG.getTargetConstant(0, MVT::i32))));
+                                         DAG.getTargetConstant(0, sdl,
+                                                               MVT::i32))));
     return nullptr;
   case Intrinsic::convert_from_fp16:
-    setValue(&I,
-             DAG.getNode(ISD::FP_EXTEND, sdl, TLI.getValueType(I.getType()),
-                         DAG.getNode(ISD::BITCAST, sdl, MVT::f16,
-                                     getValue(I.getArgOperand(0)))));
+    setValue(&I, DAG.getNode(ISD::FP_EXTEND, sdl,
+                             TLI.getValueType(DAG.getDataLayout(), I.getType()),
+                             DAG.getNode(ISD::BITCAST, sdl, MVT::f16,
+                                         getValue(I.getArgOperand(0)))));
     return nullptr;
   case Intrinsic::pcmarker: {
     SDValue Tmp = getValue(I.getArgOperand(0));
@@ -5086,6 +4675,18 @@ SelectionDAGBuilder::visitIntrinsicCall(const CallInst &I, unsigned Intrinsic) {
                              getValue(I.getArgOperand(0)).getValueType(),
                              getValue(I.getArgOperand(0))));
     return nullptr;
+  case Intrinsic::uabsdiff:
+    setValue(&I, DAG.getNode(ISD::UABSDIFF, sdl,
+                             getValue(I.getArgOperand(0)).getValueType(),
+                             getValue(I.getArgOperand(0)),
+                             getValue(I.getArgOperand(1))));
+    return nullptr;
+  case Intrinsic::sabsdiff:
+    setValue(&I, DAG.getNode(ISD::SABSDIFF, sdl,
+                             getValue(I.getArgOperand(0)).getValueType(),
+                             getValue(I.getArgOperand(0)),
+                             getValue(I.getArgOperand(1))));
+    return nullptr;
   case Intrinsic::cttz: {
     SDValue Arg = getValue(I.getArgOperand(0));
     ConstantInt *CI = cast<ConstantInt>(I.getArgOperand(1));
@@ -5110,8 +4711,9 @@ SelectionDAGBuilder::visitIntrinsicCall(const CallInst &I, unsigned Intrinsic) {
   }
   case Intrinsic::stacksave: {
     SDValue Op = getRoot();
-    Res = DAG.getNode(ISD::STACKSAVE, sdl,
-                      DAG.getVTList(TLI.getPointerTy(), MVT::Other), Op);
+    Res = DAG.getNode(
+        ISD::STACKSAVE, sdl,
+        DAG.getVTList(TLI.getPointerTy(DAG.getDataLayout()), MVT::Other), Op);
     setValue(&I, Res);
     DAG.setRoot(Res.getValue(1));
     return nullptr;
@@ -5125,7 +4727,7 @@ SelectionDAGBuilder::visitIntrinsicCall(const CallInst &I, unsigned Intrinsic) {
     // Emit code into the DAG to store the stack guard onto the stack.
     MachineFunction &MF = DAG.getMachineFunction();
     MachineFrameInfo *MFI = MF.getFrameInfo();
-    EVT PtrTy = TLI.getPointerTy();
+    EVT PtrTy = TLI.getPointerTy(DAG.getDataLayout());
     SDValue Src, Chain = getRoot();
     const Value *Ptr = cast<LoadInst>(I.getArgOperand(0))->getPointerOperand();
     const GlobalVariable *GV = dyn_cast<GlobalVariable>(Ptr);
@@ -5188,9 +4790,9 @@ SelectionDAGBuilder::visitIntrinsicCall(const CallInst &I, unsigned Intrinsic) {
     EVT Ty = Arg.getValueType();
 
     if (CI->isZero())
-      Res = DAG.getConstant(-1ULL, Ty);
+      Res = DAG.getConstant(-1ULL, sdl, Ty);
     else
-      Res = DAG.getConstant(0, Ty);
+      Res = DAG.getConstant(0, sdl, Ty);
 
     setValue(&I, Res);
     return nullptr;
@@ -5223,7 +4825,7 @@ SelectionDAGBuilder::visitIntrinsicCall(const CallInst &I, unsigned Intrinsic) {
   }
   case Intrinsic::adjust_trampoline: {
     setValue(&I, DAG.getNode(ISD::ADJUST_TRAMPOLINE, sdl,
-                             TLI.getPointerTy(),
+                             TLI.getPointerTy(DAG.getDataLayout()),
                              getValue(I.getArgOperand(0))));
     return nullptr;
   }
@@ -5251,7 +4853,10 @@ SelectionDAGBuilder::visitIntrinsicCall(const CallInst &I, unsigned Intrinsic) {
 
   case Intrinsic::debugtrap:
   case Intrinsic::trap: {
-    StringRef TrapFuncName = TM.Options.getTrapFunctionName();
+    StringRef TrapFuncName =
+        I.getAttributes()
+            .getAttribute(AttributeSet::FunctionIndex, "trap-func-name")
+            .getValueAsString();
     if (TrapFuncName.empty()) {
       ISD::NodeType Op = (Intrinsic == Intrinsic::trap) ?
         ISD::TRAP : ISD::DEBUGTRAP;
@@ -5261,10 +4866,11 @@ SelectionDAGBuilder::visitIntrinsicCall(const CallInst &I, unsigned Intrinsic) {
     TargetLowering::ArgListTy Args;
 
     TargetLowering::CallLoweringInfo CLI(DAG);
-    CLI.setDebugLoc(sdl).setChain(getRoot())
-      .setCallee(CallingConv::C, I.getType(),
-                 DAG.getExternalSymbol(TrapFuncName.data(), TLI.getPointerTy()),
-                 std::move(Args), 0);
+    CLI.setDebugLoc(sdl).setChain(getRoot()).setCallee(
+        CallingConv::C, I.getType(),
+        DAG.getExternalSymbol(TrapFuncName.data(),
+                              TLI.getPointerTy(DAG.getDataLayout())),
+        std::move(Args), 0);
 
     std::pair<SDValue, SDValue> Result = TLI.LowerCallTo(CLI);
     DAG.setRoot(Result.second);
@@ -5340,7 +4946,8 @@ SelectionDAGBuilder::visitIntrinsicCall(const CallInst &I, unsigned Intrinsic) {
 
       SDValue Ops[2];
       Ops[0] = getRoot();
-      Ops[1] = DAG.getFrameIndex(FI, TLI.getPointerTy(), true);
+      Ops[1] =
+          DAG.getFrameIndex(FI, TLI.getPointerTy(DAG.getDataLayout()), true);
       unsigned Opcode = (IsStart ? ISD::LIFETIME_START : ISD::LIFETIME_END);
 
       Res = DAG.getNode(Opcode, sdl, MVT::Other, Ops);
@@ -5350,7 +4957,7 @@ SelectionDAGBuilder::visitIntrinsicCall(const CallInst &I, unsigned Intrinsic) {
   }
   case Intrinsic::invariant_start:
     // Discard region information.
-    setValue(&I, DAG.getUNDEF(TLI.getPointerTy()));
+    setValue(&I, DAG.getUNDEF(TLI.getPointerTy(DAG.getDataLayout())));
     return nullptr;
   case Intrinsic::invariant_end:
     // Discard region information.
@@ -5370,7 +4977,7 @@ SelectionDAGBuilder::visitIntrinsicCall(const CallInst &I, unsigned Intrinsic) {
   case Intrinsic::clear_cache:
     return TLI.getClearCacheBuiltinName();
   case Intrinsic::eh_actions:
-    setValue(&I, DAG.getUNDEF(TLI.getPointerTy()));
+    setValue(&I, DAG.getUNDEF(TLI.getPointerTy(DAG.getDataLayout())));
     return nullptr;
   case Intrinsic::donothing:
     // ignore
@@ -5402,11 +5009,11 @@ SelectionDAGBuilder::visitIntrinsicCall(const CallInst &I, unsigned Intrinsic) {
   case Intrinsic::instrprof_increment:
     llvm_unreachable("instrprof failed to lower an increment");
 
-  case Intrinsic::frameescape: {
+  case Intrinsic::localescape: {
     MachineFunction &MF = DAG.getMachineFunction();
     const TargetInstrInfo *TII = DAG.getSubtarget().getInstrInfo();
 
-    // Directly emit some FRAME_ALLOC machine instrs. Label assignment emission
+    // Directly emit some LOCAL_ESCAPE machine instrs. Label assignment emission
     // is the same on all targets.
     for (unsigned Idx = 0, E = I.getNumArgOperands(); Idx < E; ++Idx) {
       Value *Arg = I.getArgOperand(Idx)->stripPointerCasts();
@@ -5420,7 +5027,7 @@ SelectionDAGBuilder::visitIntrinsicCall(const CallInst &I, unsigned Intrinsic) {
           MF.getMMI().getContext().getOrCreateFrameAllocSymbol(
               GlobalValue::getRealLinkageName(MF.getName()), Idx);
       BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, dl,
-              TII->get(TargetOpcode::FRAME_ALLOC))
+              TII->get(TargetOpcode::LOCAL_ESCAPE))
           .addSym(FrameAllocSym)
           .addFrameIndex(FI);
     }
@@ -5428,10 +5035,10 @@ SelectionDAGBuilder::visitIntrinsicCall(const CallInst &I, unsigned Intrinsic) {
     return nullptr;
   }
 
-  case Intrinsic::framerecover: {
-    // i8* @llvm.framerecover(i8* %fn, i8* %fp, i32 %idx)
+  case Intrinsic::localrecover: {
+    // i8* @llvm.localrecover(i8* %fn, i8* %fp, i32 %idx)
     MachineFunction &MF = DAG.getMachineFunction();
-    MVT PtrVT = TLI.getPointerTy(0);
+    MVT PtrVT = TLI.getPointerTy(DAG.getDataLayout(), 0);
 
     // Get the symbol that defines the frame offset.
     auto *Fn = cast<Function>(I.getArgOperand(0)->stripPointerCasts());
@@ -5441,13 +5048,11 @@ SelectionDAGBuilder::visitIntrinsicCall(const CallInst &I, unsigned Intrinsic) {
         MF.getMMI().getContext().getOrCreateFrameAllocSymbol(
             GlobalValue::getRealLinkageName(Fn->getName()), IdxVal);
 
-    // Create a TargetExternalSymbol for the label to avoid any target lowering
+    // Create a MCSymbol for the label to avoid any target lowering
     // that would make this PC relative.
-    StringRef Name = FrameAllocSym->getName();
-    assert(Name.data()[Name.size()] == '\0' && "not null terminated");
-    SDValue OffsetSym = DAG.getTargetExternalSymbol(Name.data(), PtrVT);
+    SDValue OffsetSym = DAG.getMCSymbol(FrameAllocSym, PtrVT);
     SDValue OffsetVal =
-        DAG.getNode(ISD::FRAME_ALLOC_RECOVER, sdl, PtrVT, OffsetSym);
+        DAG.getNode(ISD::LOCAL_RECOVER, sdl, PtrVT, OffsetSym);
 
     // Add the offset to the FP.
     Value *FP = I.getArgOperand(1);
@@ -5460,6 +5065,19 @@ SelectionDAGBuilder::visitIntrinsicCall(const CallInst &I, unsigned Intrinsic) {
   case Intrinsic::eh_begincatch:
   case Intrinsic::eh_endcatch:
     llvm_unreachable("begin/end catch intrinsics not lowered in codegen");
+  case Intrinsic::eh_exceptioncode: {
+    unsigned Reg = TLI.getExceptionPointerRegister();
+    assert(Reg && "cannot get exception code on this platform");
+    MVT PtrVT = TLI.getPointerTy(DAG.getDataLayout());
+    const TargetRegisterClass *PtrRC = TLI.getRegClassFor(PtrVT);
+    assert(FuncInfo.MBB->isLandingPad() && "eh.exceptioncode in non-lpad");
+    unsigned VReg = FuncInfo.MBB->addLiveIn(Reg, PtrRC);
+    SDValue N =
+        DAG.getCopyFromReg(DAG.getEntryNode(), getCurSDLoc(), VReg, PtrVT);
+    N = DAG.getZExtOrTrunc(N, getCurSDLoc(), MVT::i32);
+    setValue(&I, N);
+    return nullptr;
+  }
   }
 }
 
@@ -5472,7 +5090,7 @@ SelectionDAGBuilder::lowerInvokable(TargetLowering::CallLoweringInfo &CLI,
   if (LandingPad) {
     // Insert a label before the invoke call to mark the try range.  This can be
     // used to detect deletion of the invoke via the MachineModuleInfo.
-    BeginLabel = MMI.getContext().CreateTempSymbol();
+    BeginLabel = MMI.getContext().createTempSymbol();
 
     // For SjLj, keep track of which landing pads go with which invokes
     // so as to maintain the ordering of pads in the LSDA.
@@ -5515,7 +5133,7 @@ SelectionDAGBuilder::lowerInvokable(TargetLowering::CallLoweringInfo &CLI,
   if (LandingPad) {
     // Insert a label at the end of the invoke call to mark the try range.  This
     // can be used to detect deletion of the invoke via the MachineModuleInfo.
-    MCSymbol *EndLabel = MMI.getContext().CreateTempSymbol();
+    MCSymbol *EndLabel = MMI.getContext().createTempSymbol();
     DAG.setRoot(DAG.getEHLabel(getCurSDLoc(), getRoot(), EndLabel));
 
     // Inform MachineModuleInfo of range.
@@ -5634,7 +5252,8 @@ static SDValue getMemCmpLoad(const Value *PtrVal, MVT LoadVT,
 void SelectionDAGBuilder::processIntegerCallValue(const Instruction &I,
                                                   SDValue Value,
                                                   bool IsSigned) {
-  EVT VT = DAG.getTargetLoweringInfo().getValueType(I.getType(), true);
+  EVT VT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),
+                                                    I.getType(), true);
   if (IsSigned)
     Value = DAG.getSExtOrTrunc(Value, getCurSDLoc(), VT);
   else
@@ -5659,8 +5278,9 @@ bool SelectionDAGBuilder::visitMemCmpCall(const CallInst &I) {
   const Value *Size = I.getArgOperand(2);
   const ConstantInt *CSize = dyn_cast<ConstantInt>(Size);
   if (CSize && CSize->getZExtValue() == 0) {
-    EVT CallVT = DAG.getTargetLoweringInfo().getValueType(I.getType(), true);
-    setValue(&I, DAG.getConstant(0, CallVT));
+    EVT CallVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),
+                                                          I.getType(), true);
+    setValue(&I, DAG.getConstant(0, getCurSDLoc(), CallVT));
     return true;
   }
 
@@ -5943,7 +5563,7 @@ void SelectionDAGBuilder::visitCall(const CallInst &I) {
             return;
         }
       }
-      if (unsigned IID = F->getIntrinsicID()) {
+      if (Intrinsic::ID IID = F->getIntrinsicID()) {
         RenameFn = visitIntrinsicCall(I, IID);
         if (!RenameFn)
           return;
@@ -6096,8 +5716,9 @@ void SelectionDAGBuilder::visitCall(const CallInst &I) {
   if (!RenameFn)
     Callee = getValue(I.getCalledValue());
   else
-    Callee = DAG.getExternalSymbol(RenameFn,
-                                   DAG.getTargetLoweringInfo().getPointerTy());
+    Callee = DAG.getExternalSymbol(
+        RenameFn,
+        DAG.getTargetLoweringInfo().getPointerTy(DAG.getDataLayout()));
 
   // Check if we can potentially perform a tail call. More detailed checking is
   // be done within LowerCallTo, after more information about the call is known.
@@ -6126,13 +5747,12 @@ class SDISelAsmOperandInfo : public TargetLowering::AsmOperandInfo {
   /// getCallOperandValEVT - Return the EVT of the Value* that this operand
   /// corresponds to.  If there is no Value* for this operand, it returns
   /// MVT::Other.
-  EVT getCallOperandValEVT(LLVMContext &Context,
-                           const TargetLowering &TLI,
-                           const DataLayout *DL) const {
+  EVT getCallOperandValEVT(LLVMContext &Context, const TargetLowering &TLI,
+                           const DataLayout &DL) const {
     if (!CallOperandVal) return MVT::Other;
 
     if (isa<BasicBlock>(CallOperandVal))
-      return TLI.getPointerTy();
+      return TLI.getPointerTy(DL);
 
     llvm::Type *OpTy = CallOperandVal->getType();
 
@@ -6154,7 +5774,7 @@ class SDISelAsmOperandInfo : public TargetLowering::AsmOperandInfo {
     // If OpTy is not a single value, it may be a struct/union that we
     // can tile with integers.
     if (!OpTy->isSingleValueType() && OpTy->isSized()) {
-      unsigned BitSize = DL->getTypeSizeInBits(OpTy);
+      unsigned BitSize = DL.getTypeSizeInBits(OpTy);
       switch (BitSize) {
       default: break;
       case 1:
@@ -6168,7 +5788,7 @@ class SDISelAsmOperandInfo : public TargetLowering::AsmOperandInfo {
       }
     }
 
-    return TLI.getValueType(OpTy, true);
+    return TLI.getValueType(DL, OpTy, true);
   }
 };
 
@@ -6294,8 +5914,8 @@ void SelectionDAGBuilder::visitInlineAsm(ImmutableCallSite CS) {
   SDISelAsmOperandInfoVector ConstraintOperands;
 
   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
-  TargetLowering::AsmOperandInfoVector TargetConstraints =
-      TLI.ParseConstraints(DAG.getSubtarget().getRegisterInfo(), CS);
+  TargetLowering::AsmOperandInfoVector TargetConstraints = TLI.ParseConstraints(
+      DAG.getDataLayout(), DAG.getSubtarget().getRegisterInfo(), CS);
 
   bool hasMemory = false;
 
@@ -6320,10 +5940,11 @@ void SelectionDAGBuilder::visitInlineAsm(ImmutableCallSite CS) {
       // corresponding argument.
       assert(!CS.getType()->isVoidTy() && "Bad inline asm!");
       if (StructType *STy = dyn_cast<StructType>(CS.getType())) {
-        OpVT = TLI.getSimpleValueType(STy->getElementType(ResNo));
+        OpVT = TLI.getSimpleValueType(DAG.getDataLayout(),
+                                      STy->getElementType(ResNo));
       } else {
         assert(ResNo == 0 && "Asm only has one result!");
-        OpVT = TLI.getSimpleValueType(CS.getType());
+        OpVT = TLI.getSimpleValueType(DAG.getDataLayout(), CS.getType());
       }
       ++ResNo;
       break;
@@ -6344,8 +5965,8 @@ void SelectionDAGBuilder::visitInlineAsm(ImmutableCallSite CS) {
         OpInfo.CallOperand = getValue(OpInfo.CallOperandVal);
       }
 
-      OpVT =
-          OpInfo.getCallOperandValEVT(*DAG.getContext(), TLI, DL).getSimpleVT();
+      OpVT = OpInfo.getCallOperandValEVT(*DAG.getContext(), TLI,
+                                         DAG.getDataLayout()).getSimpleVT();
     }
 
     OpInfo.ConstraintVT = OpVT;
@@ -6433,17 +6054,19 @@ void SelectionDAGBuilder::visitInlineAsm(ImmutableCallSite CS) {
       const Value *OpVal = OpInfo.CallOperandVal;
       if (isa<ConstantFP>(OpVal) || isa<ConstantInt>(OpVal) ||
           isa<ConstantVector>(OpVal) || isa<ConstantDataVector>(OpVal)) {
-        OpInfo.CallOperand = DAG.getConstantPool(cast<Constant>(OpVal),
-                                                 TLI.getPointerTy());
+        OpInfo.CallOperand = DAG.getConstantPool(
+            cast<Constant>(OpVal), TLI.getPointerTy(DAG.getDataLayout()));
       } else {
         // Otherwise, create a stack slot and emit a store to it before the
         // asm.
         Type *Ty = OpVal->getType();
-        uint64_t TySize = TLI.getDataLayout()->getTypeAllocSize(Ty);
-        unsigned Align  = TLI.getDataLayout()->getPrefTypeAlignment(Ty);
+        auto &DL = DAG.getDataLayout();
+        uint64_t TySize = DL.getTypeAllocSize(Ty);
+        unsigned Align = DL.getPrefTypeAlignment(Ty);
         MachineFunction &MF = DAG.getMachineFunction();
         int SSFI = MF.getFrameInfo()->CreateStackObject(TySize, Align, false);
-        SDValue StackSlot = DAG.getFrameIndex(SSFI, TLI.getPointerTy());
+        SDValue StackSlot =
+            DAG.getFrameIndex(SSFI, TLI.getPointerTy(DAG.getDataLayout()));
         Chain = DAG.getStore(Chain, getCurSDLoc(),
                              OpInfo.CallOperand, StackSlot,
                              MachinePointerInfo::getFixedStack(SSFI),
@@ -6478,9 +6101,8 @@ void SelectionDAGBuilder::visitInlineAsm(ImmutableCallSite CS) {
   // AsmNodeOperands - The operands for the ISD::INLINEASM node.
   std::vector<SDValue> AsmNodeOperands;
   AsmNodeOperands.push_back(SDValue());  // reserve space for input chain
-  AsmNodeOperands.push_back(
-          DAG.getTargetExternalSymbol(IA->getAsmString().c_str(),
-                                      TLI.getPointerTy()));
+  AsmNodeOperands.push_back(DAG.getTargetExternalSymbol(
+      IA->getAsmString().c_str(), TLI.getPointerTy(DAG.getDataLayout())));
 
   // If we have a !srcloc metadata node associated with it, we want to attach
   // this to the ultimately generated inline asm machineinstr.  To do this, we
@@ -6520,8 +6142,8 @@ void SelectionDAGBuilder::visitInlineAsm(ImmutableCallSite CS) {
     }
   }
 
-  AsmNodeOperands.push_back(DAG.getTargetConstant(ExtraInfo,
-                                                  TLI.getPointerTy()));
+  AsmNodeOperands.push_back(DAG.getTargetConstant(
+      ExtraInfo, getCurSDLoc(), TLI.getPointerTy(DAG.getDataLayout())));
 
   // Loop over all of the inputs, copying the operand values into the
   // appropriate registers and processing the output regs.
@@ -6548,7 +6170,8 @@ void SelectionDAGBuilder::visitInlineAsm(ImmutableCallSite CS) {
         // Add information to the INLINEASM node to know about this output.
         unsigned OpFlags = InlineAsm::getFlagWord(InlineAsm::Kind_Mem, 1);
         OpFlags = InlineAsm::getFlagWordForMem(OpFlags, ConstraintID);
-        AsmNodeOperands.push_back(DAG.getTargetConstant(OpFlags, MVT::i32));
+        AsmNodeOperands.push_back(DAG.getTargetConstant(OpFlags, getCurSDLoc(),
+                                                        MVT::i32));
         AsmNodeOperands.push_back(OpInfo.CallOperand);
         break;
       }
@@ -6583,7 +6206,7 @@ void SelectionDAGBuilder::visitInlineAsm(ImmutableCallSite CS) {
           .AddInlineAsmOperands(OpInfo.isEarlyClobber
                                     ? InlineAsm::Kind_RegDefEarlyClobber
                                     : InlineAsm::Kind_RegDef,
-                                false, 0, DAG, AsmNodeOperands);
+                                false, 0, getCurSDLoc(), DAG, AsmNodeOperands);
       break;
     }
     case InlineAsm::isInput: {
@@ -6638,11 +6261,12 @@ void SelectionDAGBuilder::visitInlineAsm(ImmutableCallSite CS) {
               return;
             }
           }
+          SDLoc dl = getCurSDLoc();
           // Use the produced MatchedRegs object to
-          MatchedRegs.getCopyToRegs(InOperandVal, DAG, getCurSDLoc(),
+          MatchedRegs.getCopyToRegs(InOperandVal, DAG, dl,
                                     Chain, &Flag, CS.getInstruction());
           MatchedRegs.AddInlineAsmOperands(InlineAsm::Kind_RegUse,
-                                           true, OpInfo.getMatchedOperand(),
+                                           true, OpInfo.getMatchedOperand(), dl,
                                            DAG, AsmNodeOperands);
           break;
         }
@@ -6655,8 +6279,8 @@ void SelectionDAGBuilder::visitInlineAsm(ImmutableCallSite CS) {
         OpFlag = InlineAsm::convertMemFlagWordToMatchingFlagWord(OpFlag);
         OpFlag = InlineAsm::getFlagWordForMatchingOp(OpFlag,
                                                     OpInfo.getMatchedOperand());
-        AsmNodeOperands.push_back(DAG.getTargetConstant(OpFlag,
-                                                        TLI.getPointerTy()));
+        AsmNodeOperands.push_back(DAG.getTargetConstant(
+            OpFlag, getCurSDLoc(), TLI.getPointerTy(DAG.getDataLayout())));
         AsmNodeOperands.push_back(AsmNodeOperands[CurOp+1]);
         break;
       }
@@ -6681,15 +6305,16 @@ void SelectionDAGBuilder::visitInlineAsm(ImmutableCallSite CS) {
         // Add information to the INLINEASM node to know about this input.
         unsigned ResOpType =
           InlineAsm::getFlagWord(InlineAsm::Kind_Imm, Ops.size());
-        AsmNodeOperands.push_back(DAG.getTargetConstant(ResOpType,
-                                                        TLI.getPointerTy()));
+        AsmNodeOperands.push_back(DAG.getTargetConstant(
+            ResOpType, getCurSDLoc(), TLI.getPointerTy(DAG.getDataLayout())));
         AsmNodeOperands.insert(AsmNodeOperands.end(), Ops.begin(), Ops.end());
         break;
       }
 
       if (OpInfo.ConstraintType == TargetLowering::C_Memory) {
         assert(OpInfo.isIndirect && "Operand must be indirect to be a mem!");
-        assert(InOperandVal.getValueType() == TLI.getPointerTy() &&
+        assert(InOperandVal.getValueType() ==
+                   TLI.getPointerTy(DAG.getDataLayout()) &&
                "Memory operands expect pointer values");
 
         unsigned ConstraintID =
@@ -6700,7 +6325,9 @@ void SelectionDAGBuilder::visitInlineAsm(ImmutableCallSite CS) {
         // Add information to the INLINEASM node to know about this input.
         unsigned ResOpType = InlineAsm::getFlagWord(InlineAsm::Kind_Mem, 1);
         ResOpType = InlineAsm::getFlagWordForMem(ResOpType, ConstraintID);
-        AsmNodeOperands.push_back(DAG.getTargetConstant(ResOpType, MVT::i32));
+        AsmNodeOperands.push_back(DAG.getTargetConstant(ResOpType,
+                                                        getCurSDLoc(),
+                                                        MVT::i32));
         AsmNodeOperands.push_back(InOperandVal);
         break;
       }
@@ -6728,11 +6355,13 @@ void SelectionDAGBuilder::visitInlineAsm(ImmutableCallSite CS) {
         return;
       }
 
-      OpInfo.AssignedRegs.getCopyToRegs(InOperandVal, DAG, getCurSDLoc(),
+      SDLoc dl = getCurSDLoc();
+
+      OpInfo.AssignedRegs.getCopyToRegs(InOperandVal, DAG, dl,
                                         Chain, &Flag, CS.getInstruction());
 
       OpInfo.AssignedRegs.AddInlineAsmOperands(InlineAsm::Kind_RegUse, false, 0,
-                                               DAG, AsmNodeOperands);
+                                               dl, DAG, AsmNodeOperands);
       break;
     }
     case InlineAsm::isClobber: {
@@ -6740,7 +6369,7 @@ void SelectionDAGBuilder::visitInlineAsm(ImmutableCallSite CS) {
       // allocator is aware that the physreg got clobbered.
       if (!OpInfo.AssignedRegs.Regs.empty())
         OpInfo.AssignedRegs.AddInlineAsmOperands(InlineAsm::Kind_Clobber,
-                                                 false, 0, DAG,
+                                                 false, 0, getCurSDLoc(), DAG,
                                                  AsmNodeOperands);
       break;
     }
@@ -6763,7 +6392,7 @@ void SelectionDAGBuilder::visitInlineAsm(ImmutableCallSite CS) {
 
     // FIXME: Why don't we do this for inline asms with MRVs?
     if (CS.getType()->isSingleValueType() && CS.getType()->isSized()) {
-      EVT ResultType = TLI.getValueType(CS.getType());
+      EVT ResultType = TLI.getValueType(DAG.getDataLayout(), CS.getType());
 
       // If any of the results of the inline asm is a vector, it may have the
       // wrong width/num elts.  This can happen for register classes that can
@@ -6829,9 +6458,9 @@ void SelectionDAGBuilder::visitVAStart(const CallInst &I) {
 
 void SelectionDAGBuilder::visitVAArg(const VAArgInst &I) {
   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
-  const DataLayout &DL = *TLI.getDataLayout();
-  SDValue V = DAG.getVAArg(TLI.getValueType(I.getType()), getCurSDLoc(),
-                           getRoot(), getValue(I.getOperand(0)),
+  const DataLayout &DL = DAG.getDataLayout();
+  SDValue V = DAG.getVAArg(TLI.getValueType(DAG.getDataLayout(), I.getType()),
+                           getCurSDLoc(), getRoot(), getValue(I.getOperand(0)),
                            DAG.getSrcValue(I.getOperand(0)),
                            DL.getABITypeAlignment(I.getType()));
   setValue(&I, V);
@@ -6864,7 +6493,7 @@ void SelectionDAGBuilder::visitVACopy(const CallInst &I) {
 std::pair<SDValue, SDValue>
 SelectionDAGBuilder::lowerCallOperands(ImmutableCallSite CS, unsigned ArgIdx,
                                        unsigned NumArgs, SDValue Callee,
-                                       bool UseVoidTy,
+                                       Type *ReturnTy,
                                        MachineBasicBlock *LandingPad,
                                        bool IsPatchPoint) {
   TargetLowering::ArgListTy Args;
@@ -6885,10 +6514,9 @@ SelectionDAGBuilder::lowerCallOperands(ImmutableCallSite CS, unsigned ArgIdx,
     Args.push_back(Entry);
   }
 
-  Type *retTy = UseVoidTy ? Type::getVoidTy(*DAG.getContext()) : CS->getType();
   TargetLowering::CallLoweringInfo CLI(DAG);
   CLI.setDebugLoc(getCurSDLoc()).setChain(getRoot())
-    .setCallee(CS.getCallingConv(), retTy, Callee, std::move(Args), NumArgs)
+    .setCallee(CS.getCallingConv(), ReturnTy, Callee, std::move(Args), NumArgs)
     .setDiscardResult(CS->use_empty()).setIsPatchPoint(IsPatchPoint);
 
   return lowerInvokable(CLI, LandingPad);
@@ -6912,19 +6540,19 @@ SelectionDAGBuilder::lowerCallOperands(ImmutableCallSite CS, unsigned ArgIdx,
 /// only available in a register, then the runtime would need to trap when
 /// execution reaches the StackMap in order to read the alloca's location.
 static void addStackMapLiveVars(ImmutableCallSite CS, unsigned StartIdx,
-                                SmallVectorImpl<SDValue> &Ops,
+                                SDLoc DL, SmallVectorImpl<SDValue> &Ops,
                                 SelectionDAGBuilder &Builder) {
   for (unsigned i = StartIdx, e = CS.arg_size(); i != e; ++i) {
     SDValue OpVal = Builder.getValue(CS.getArgument(i));
     if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(OpVal)) {
       Ops.push_back(
-        Builder.DAG.getTargetConstant(StackMaps::ConstantOp, MVT::i64));
+        Builder.DAG.getTargetConstant(StackMaps::ConstantOp, DL, MVT::i64));
       Ops.push_back(
-        Builder.DAG.getTargetConstant(C->getSExtValue(), MVT::i64));
+        Builder.DAG.getTargetConstant(C->getSExtValue(), DL, MVT::i64));
     } else if (FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(OpVal)) {
       const TargetLowering &TLI = Builder.DAG.getTargetLoweringInfo();
-      Ops.push_back(
-        Builder.DAG.getTargetFrameIndex(FI->getIndex(), TLI.getPointerTy()));
+      Ops.push_back(Builder.DAG.getTargetFrameIndex(
+          FI->getIndex(), TLI.getPointerTy(Builder.DAG.getDataLayout())));
     } else
       Ops.push_back(OpVal);
   }
@@ -6942,7 +6570,7 @@ void SelectionDAGBuilder::visitStackmap(const CallInst &CI) {
 
   SDLoc DL = getCurSDLoc();
   Callee = getValue(CI.getCalledValue());
-  NullPtr = DAG.getIntPtrConstant(0, true);
+  NullPtr = DAG.getIntPtrConstant(0, DL, true);
 
   // The stackmap intrinsic only records the live variables (the arguemnts
   // passed to it) and emits NOPS (if requested). Unlike the patchpoint
@@ -6960,13 +6588,14 @@ void SelectionDAGBuilder::visitStackmap(const CallInst &CI) {
   // Add the <id> and <numBytes> constants.
   SDValue IDVal = getValue(CI.getOperand(PatchPointOpers::IDPos));
   Ops.push_back(DAG.getTargetConstant(
-                  cast<ConstantSDNode>(IDVal)->getZExtValue(), MVT::i64));
+                  cast<ConstantSDNode>(IDVal)->getZExtValue(), DL, MVT::i64));
   SDValue NBytesVal = getValue(CI.getOperand(PatchPointOpers::NBytesPos));
   Ops.push_back(DAG.getTargetConstant(
-                  cast<ConstantSDNode>(NBytesVal)->getZExtValue(), MVT::i32));
+                  cast<ConstantSDNode>(NBytesVal)->getZExtValue(), DL,
+                  MVT::i32));
 
   // Push live variables for the stack map.
-  addStackMapLiveVars(&CI, 2, Ops, *this);
+  addStackMapLiveVars(&CI, 2, DL, Ops, *this);
 
   // We are not pushing any register mask info here on the operands list,
   // because the stackmap doesn't clobber anything.
@@ -7005,7 +6634,17 @@ void SelectionDAGBuilder::visitPatchpoint(ImmutableCallSite CS,
   CallingConv::ID CC = CS.getCallingConv();
   bool IsAnyRegCC = CC == CallingConv::AnyReg;
   bool HasDef = !CS->getType()->isVoidTy();
-  SDValue Callee = getValue(CS->getOperand(2)); // <target>
+  SDLoc dl = getCurSDLoc();
+  SDValue Callee = getValue(CS->getOperand(PatchPointOpers::TargetPos));
+
+  // Handle immediate and symbolic callees.
+  if (auto* ConstCallee = dyn_cast<ConstantSDNode>(Callee))
+    Callee = DAG.getIntPtrConstant(ConstCallee->getZExtValue(), dl,
+                                   /*isTarget=*/true);
+  else if (auto* SymbolicCallee = dyn_cast<GlobalAddressSDNode>(Callee))
+    Callee =  DAG.getTargetGlobalAddress(SymbolicCallee->getGlobal(),
+                                         SDLoc(SymbolicCallee),
+                                         SymbolicCallee->getValueType(0));
 
   // Get the real number of arguments participating in the call <numArgs>
   SDValue NArgVal = getValue(CS.getArgument(PatchPointOpers::NArgPos));
@@ -7019,8 +6658,10 @@ void SelectionDAGBuilder::visitPatchpoint(ImmutableCallSite CS,
 
   // For AnyRegCC the arguments are lowered later on manually.
   unsigned NumCallArgs = IsAnyRegCC ? 0 : NumArgs;
+  Type *ReturnTy =
+    IsAnyRegCC ? Type::getVoidTy(*DAG.getContext()) : CS->getType();
   std::pair<SDValue, SDValue> Result =
-    lowerCallOperands(CS, NumMetaOpers, NumCallArgs, Callee, IsAnyRegCC,
+    lowerCallOperands(CS, NumMetaOpers, NumCallArgs, Callee, ReturnTy,
                       LandingPad, true);
 
   SDNode *CallEnd = Result.second.getNode();
@@ -7040,26 +6681,24 @@ void SelectionDAGBuilder::visitPatchpoint(ImmutableCallSite CS,
   // Add the <id> and <numBytes> constants.
   SDValue IDVal = getValue(CS->getOperand(PatchPointOpers::IDPos));
   Ops.push_back(DAG.getTargetConstant(
-                  cast<ConstantSDNode>(IDVal)->getZExtValue(), MVT::i64));
+                  cast<ConstantSDNode>(IDVal)->getZExtValue(), dl, MVT::i64));
   SDValue NBytesVal = getValue(CS->getOperand(PatchPointOpers::NBytesPos));
   Ops.push_back(DAG.getTargetConstant(
-                  cast<ConstantSDNode>(NBytesVal)->getZExtValue(), MVT::i32));
+                  cast<ConstantSDNode>(NBytesVal)->getZExtValue(), dl,
+                  MVT::i32));
 
-  // Assume that the Callee is a constant address.
-  // FIXME: handle function symbols in the future.
-  Ops.push_back(
-    DAG.getIntPtrConstant(cast<ConstantSDNode>(Callee)->getZExtValue(),
-                          /*isTarget=*/true));
+  // Add the callee.
+  Ops.push_back(Callee);
 
   // Adjust <numArgs> to account for any arguments that have been passed on the
   // stack instead.
   // Call Node: Chain, Target, {Args}, RegMask, [Glue]
   unsigned NumCallRegArgs = Call->getNumOperands() - (HasGlue ? 4 : 3);
   NumCallRegArgs = IsAnyRegCC ? NumArgs : NumCallRegArgs;
-  Ops.push_back(DAG.getTargetConstant(NumCallRegArgs, MVT::i32));
+  Ops.push_back(DAG.getTargetConstant(NumCallRegArgs, dl, MVT::i32));
 
   // Add the calling convention
-  Ops.push_back(DAG.getTargetConstant((unsigned)CC, MVT::i32));
+  Ops.push_back(DAG.getTargetConstant((unsigned)CC, dl, MVT::i32));
 
   // Add the arguments we omitted previously. The register allocator should
   // place these in any free register.
@@ -7072,7 +6711,7 @@ void SelectionDAGBuilder::visitPatchpoint(ImmutableCallSite CS,
   Ops.append(Call->op_begin() + 2, e);
 
   // Push live variables for the stack map.
-  addStackMapLiveVars(CS, NumMetaOpers + NumArgs, Ops, *this);
+  addStackMapLiveVars(CS, NumMetaOpers + NumArgs, dl, Ops, *this);
 
   // Push the register mask info.
   if (HasGlue)
@@ -7093,7 +6732,7 @@ void SelectionDAGBuilder::visitPatchpoint(ImmutableCallSite CS,
     // Create the return types based on the intrinsic definition
     const TargetLowering &TLI = DAG.getTargetLoweringInfo();
     SmallVector<EVT, 3> ValueVTs;
-    ComputeValueVTs(TLI, CS->getType(), ValueVTs);
+    ComputeValueVTs(TLI, DAG.getDataLayout(), CS->getType(), ValueVTs);
     assert(ValueVTs.size() == 1 && "Expected only one return value type.");
 
     // There is always a chain and a glue type at the end
@@ -7105,7 +6744,7 @@ void SelectionDAGBuilder::visitPatchpoint(ImmutableCallSite CS,
 
   // Replace the target specific call node with a PATCHPOINT node.
   MachineSDNode *MN = DAG.getMachineNode(TargetOpcode::PATCHPOINT,
-                                         getCurSDLoc(), NodeTys, Ops);
+                                         dl, NodeTys, Ops);
 
   // Update the NodeMap.
   if (HasDef) {
@@ -7157,10 +6796,11 @@ TargetLowering::LowerCallTo(TargetLowering::CallLoweringInfo &CLI) const {
   Type *OrigRetTy = CLI.RetTy;
   SmallVector<EVT, 4> RetTys;
   SmallVector<uint64_t, 4> Offsets;
-  ComputeValueVTs(*this, CLI.RetTy, RetTys, &Offsets);
+  auto &DL = CLI.DAG.getDataLayout();
+  ComputeValueVTs(*this, DL, CLI.RetTy, RetTys, &Offsets);
 
   SmallVector<ISD::OutputArg, 4> Outs;
-  GetReturnInfo(CLI.RetTy, getReturnAttrs(CLI), Outs, *this);
+  GetReturnInfo(CLI.RetTy, getReturnAttrs(CLI), Outs, *this, DL);
 
   bool CanLowerReturn =
       this->CanLowerReturn(CLI.CallConv, CLI.DAG.getMachineFunction(),
@@ -7172,13 +6812,13 @@ TargetLowering::LowerCallTo(TargetLowering::CallLoweringInfo &CLI) const {
     // FIXME: equivalent assert?
     // assert(!CS.hasInAllocaArgument() &&
     //        "sret demotion is incompatible with inalloca");
-    uint64_t TySize = getDataLayout()->getTypeAllocSize(CLI.RetTy);
-    unsigned Align  = getDataLayout()->getPrefTypeAlignment(CLI.RetTy);
+    uint64_t TySize = DL.getTypeAllocSize(CLI.RetTy);
+    unsigned Align = DL.getPrefTypeAlignment(CLI.RetTy);
     MachineFunction &MF = CLI.DAG.getMachineFunction();
     DemoteStackIdx = MF.getFrameInfo()->CreateStackObject(TySize, Align, false);
     Type *StackSlotPtrType = PointerType::getUnqual(CLI.RetTy);
 
-    DemoteStackSlot = CLI.DAG.getFrameIndex(DemoteStackIdx, getPointerTy());
+    DemoteStackSlot = CLI.DAG.getFrameIndex(DemoteStackIdx, getPointerTy(DL));
     ArgListEntry Entry;
     Entry.Node = DemoteStackSlot;
     Entry.Ty = StackSlotPtrType;
@@ -7223,7 +6863,7 @@ TargetLowering::LowerCallTo(TargetLowering::CallLoweringInfo &CLI) const {
   ArgListTy &Args = CLI.getArgs();
   for (unsigned i = 0, e = Args.size(); i != e; ++i) {
     SmallVector<EVT, 4> ValueVTs;
-    ComputeValueVTs(*this, Args[i].Ty, ValueVTs);
+    ComputeValueVTs(*this, DL, Args[i].Ty, ValueVTs);
     Type *FinalType = Args[i].Ty;
     if (Args[i].isByVal)
       FinalType = cast<PointerType>(Args[i].Ty)->getElementType();
@@ -7236,7 +6876,7 @@ TargetLowering::LowerCallTo(TargetLowering::CallLoweringInfo &CLI) const {
       SDValue Op = SDValue(Args[i].Node.getNode(),
                            Args[i].Node.getResNo() + Value);
       ISD::ArgFlagsTy Flags;
-      unsigned OriginalAlignment = getDataLayout()->getABITypeAlignment(ArgTy);
+      unsigned OriginalAlignment = DL.getABITypeAlignment(ArgTy);
 
       if (Args[i].isZExt)
         Flags.setZExt();
@@ -7260,14 +6900,14 @@ TargetLowering::LowerCallTo(TargetLowering::CallLoweringInfo &CLI) const {
       if (Args[i].isByVal || Args[i].isInAlloca) {
         PointerType *Ty = cast<PointerType>(Args[i].Ty);
         Type *ElementTy = Ty->getElementType();
-        Flags.setByValSize(getDataLayout()->getTypeAllocSize(ElementTy));
+        Flags.setByValSize(DL.getTypeAllocSize(ElementTy));
         // For ByVal, alignment should come from FE.  BE will guess if this
         // info is not there but there are cases it cannot get right.
         unsigned FrameAlign;
         if (Args[i].Alignment)
           FrameAlign = Args[i].Alignment;
         else
-          FrameAlign = getByValTypeAlignment(ElementTy);
+          FrameAlign = getByValTypeAlignment(ElementTy, DL);
         Flags.setByValAlign(FrameAlign);
       }
       if (Args[i].isNest)
@@ -7362,7 +7002,7 @@ TargetLowering::LowerCallTo(TargetLowering::CallLoweringInfo &CLI) const {
     SmallVector<EVT, 1> PVTs;
     Type *PtrRetTy = PointerType::getUnqual(OrigRetTy);
 
-    ComputeValueVTs(*this, PtrRetTy, PVTs);
+    ComputeValueVTs(*this, DL, PtrRetTy, PVTs);
     assert(PVTs.size() == 1 && "Pointers should fit in one register");
     EVT PtrVT = PVTs[0];
 
@@ -7372,7 +7012,8 @@ TargetLowering::LowerCallTo(TargetLowering::CallLoweringInfo &CLI) const {
 
     for (unsigned i = 0; i < NumValues; ++i) {
       SDValue Add = CLI.DAG.getNode(ISD::ADD, CLI.DL, PtrVT, DemoteStackSlot,
-                                    CLI.DAG.getConstant(Offsets[i], PtrVT));
+                                    CLI.DAG.getConstant(Offsets[i], CLI.DL,
+                                                        PtrVT));
       SDValue L = CLI.DAG.getLoad(
           RetTys[i], CLI.DL, CLI.Chain, Add,
           MachinePointerInfo::getFixedStack(DemoteStackIdx, Offsets[i]), false,
@@ -7435,7 +7076,8 @@ SelectionDAGBuilder::CopyValueToVirtualRegister(const Value *V, unsigned Reg) {
   assert(!TargetRegisterInfo::isPhysicalRegister(Reg) && "Is a physreg");
 
   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
-  RegsForValue RFV(V->getContext(), TLI, Reg, V->getType());
+  RegsForValue RFV(V->getContext(), TLI, DAG.getDataLayout(), Reg,
+                   V->getType());
   SDValue Chain = DAG.getEntryNode();
 
   ISD::NodeType ExtendType = (FuncInfo.PreferredExtendType.find(V) ==
@@ -7468,13 +7110,14 @@ static bool isOnlyUsedInEntryBlock(const Argument *A, bool FastISel) {
 void SelectionDAGISel::LowerArguments(const Function &F) {
   SelectionDAG &DAG = SDB->DAG;
   SDLoc dl = SDB->getCurSDLoc();
-  const DataLayout *DL = TLI->getDataLayout();
+  const DataLayout &DL = DAG.getDataLayout();
   SmallVector<ISD::InputArg, 16> Ins;
 
   if (!FuncInfo->CanLowerReturn) {
     // Put in an sret pointer parameter before all the other parameters.
     SmallVector<EVT, 1> ValueVTs;
-    ComputeValueVTs(*TLI, PointerType::getUnqual(F.getReturnType()), ValueVTs);
+    ComputeValueVTs(*TLI, DAG.getDataLayout(),
+                    PointerType::getUnqual(F.getReturnType()), ValueVTs);
 
     // NOTE: Assuming that a pointer will never break down to more than one VT
     // or one register.
@@ -7491,7 +7134,7 @@ void SelectionDAGISel::LowerArguments(const Function &F) {
   for (Function::const_arg_iterator I = F.arg_begin(), E = F.arg_end();
        I != E; ++I, ++Idx) {
     SmallVector<EVT, 4> ValueVTs;
-    ComputeValueVTs(*TLI, I->getType(), ValueVTs);
+    ComputeValueVTs(*TLI, DAG.getDataLayout(), I->getType(), ValueVTs);
     bool isArgValueUsed = !I->use_empty();
     unsigned PartBase = 0;
     Type *FinalType = I->getType();
@@ -7504,7 +7147,7 @@ void SelectionDAGISel::LowerArguments(const Function &F) {
       EVT VT = ValueVTs[Value];
       Type *ArgTy = VT.getTypeForEVT(*DAG.getContext());
       ISD::ArgFlagsTy Flags;
-      unsigned OriginalAlignment = DL->getABITypeAlignment(ArgTy);
+      unsigned OriginalAlignment = DL.getABITypeAlignment(ArgTy);
 
       if (F.getAttributes().hasAttribute(Idx, Attribute::ZExt))
         Flags.setZExt();
@@ -7528,14 +7171,14 @@ void SelectionDAGISel::LowerArguments(const Function &F) {
       if (Flags.isByVal() || Flags.isInAlloca()) {
         PointerType *Ty = cast<PointerType>(I->getType());
         Type *ElementTy = Ty->getElementType();
-        Flags.setByValSize(DL->getTypeAllocSize(ElementTy));
+        Flags.setByValSize(DL.getTypeAllocSize(ElementTy));
         // For ByVal, alignment should be passed from FE.  BE will guess if
         // this info is not there but there are cases it cannot get right.
         unsigned FrameAlign;
         if (F.getParamAlignment(Idx))
           FrameAlign = F.getParamAlignment(Idx);
         else
-          FrameAlign = TLI->getByValTypeAlignment(ElementTy);
+          FrameAlign = TLI->getByValTypeAlignment(ElementTy, DL);
         Flags.setByValAlign(FrameAlign);
       }
       if (F.getAttributes().hasAttribute(Idx, Attribute::Nest))
@@ -7591,7 +7234,8 @@ void SelectionDAGISel::LowerArguments(const Function &F) {
     // Create a virtual register for the sret pointer, and put in a copy
     // from the sret argument into it.
     SmallVector<EVT, 1> ValueVTs;
-    ComputeValueVTs(*TLI, PointerType::getUnqual(F.getReturnType()), ValueVTs);
+    ComputeValueVTs(*TLI, DAG.getDataLayout(),
+                    PointerType::getUnqual(F.getReturnType()), ValueVTs);
     MVT VT = ValueVTs[0].getSimpleVT();
     MVT RegVT = TLI->getRegisterType(*CurDAG->getContext(), VT);
     ISD::NodeType AssertOp = ISD::DELETED_NODE;
@@ -7615,7 +7259,7 @@ void SelectionDAGISel::LowerArguments(const Function &F) {
       ++I, ++Idx) {
     SmallVector<SDValue, 4> ArgValues;
     SmallVector<EVT, 4> ValueVTs;
-    ComputeValueVTs(*TLI, I->getType(), ValueVTs);
+    ComputeValueVTs(*TLI, DAG.getDataLayout(), I->getType(), ValueVTs);
     unsigned NumValues = ValueVTs.size();
 
     // If this argument is unused then remember its value. It is used to generate
@@ -7762,7 +7406,7 @@ SelectionDAGBuilder::HandlePHINodesInSuccessorBlocks(const BasicBlock *LLVMBB) {
       // the input for this MBB.
       SmallVector<EVT, 4> ValueVTs;
       const TargetLowering &TLI = DAG.getTargetLoweringInfo();
-      ComputeValueVTs(TLI, PN->getType(), ValueVTs);
+      ComputeValueVTs(TLI, DAG.getDataLayout(), PN->getType(), ValueVTs);
       for (unsigned vti = 0, vte = ValueVTs.size(); vti != vte; ++vti) {
         EVT VT = ValueVTs[vti];
         unsigned NumRegisters = TLI.getNumRegisters(*DAG.getContext(), VT);
@@ -7817,3 +7461,862 @@ void SelectionDAGBuilder::updateDAGForMaybeTailCall(SDValue MaybeTC) {
     HasTailCall = true;
 }
 
+bool SelectionDAGBuilder::isDense(const CaseClusterVector &Clusters,
+                                  unsigned *TotalCases, unsigned First,
+                                  unsigned Last) {
+  assert(Last >= First);
+  assert(TotalCases[Last] >= TotalCases[First]);
+
+  APInt LowCase = Clusters[First].Low->getValue();
+  APInt HighCase = Clusters[Last].High->getValue();
+  assert(LowCase.getBitWidth() == HighCase.getBitWidth());
+
+  // FIXME: A range of consecutive cases has 100% density, but only requires one
+  // comparison to lower. We should discriminate against such consecutive ranges
+  // in jump tables.
+
+  uint64_t Diff = (HighCase - LowCase).getLimitedValue((UINT64_MAX - 1) / 100);
+  uint64_t Range = Diff + 1;
+
+  uint64_t NumCases =
+      TotalCases[Last] - (First == 0 ? 0 : TotalCases[First - 1]);
+
+  assert(NumCases < UINT64_MAX / 100);
+  assert(Range >= NumCases);
+
+  return NumCases * 100 >= Range * MinJumpTableDensity;
+}
+
+static inline bool areJTsAllowed(const TargetLowering &TLI) {
+  return TLI.isOperationLegalOrCustom(ISD::BR_JT, MVT::Other) ||
+         TLI.isOperationLegalOrCustom(ISD::BRIND, MVT::Other);
+}
+
+bool SelectionDAGBuilder::buildJumpTable(CaseClusterVector &Clusters,
+                                         unsigned First, unsigned Last,
+                                         const SwitchInst *SI,
+                                         MachineBasicBlock *DefaultMBB,
+                                         CaseCluster &JTCluster) {
+  assert(First <= Last);
+
+  uint32_t Weight = 0;
+  unsigned NumCmps = 0;
+  std::vector<MachineBasicBlock*> Table;
+  DenseMap<MachineBasicBlock*, uint32_t> JTWeights;
+  for (unsigned I = First; I <= Last; ++I) {
+    assert(Clusters[I].Kind == CC_Range);
+    Weight += Clusters[I].Weight;
+    assert(Weight >= Clusters[I].Weight && "Weight overflow!");
+    APInt Low = Clusters[I].Low->getValue();
+    APInt High = Clusters[I].High->getValue();
+    NumCmps += (Low == High) ? 1 : 2;
+    if (I != First) {
+      // Fill the gap between this and the previous cluster.
+      APInt PreviousHigh = Clusters[I - 1].High->getValue();
+      assert(PreviousHigh.slt(Low));
+      uint64_t Gap = (Low - PreviousHigh).getLimitedValue() - 1;
+      for (uint64_t J = 0; J < Gap; J++)
+        Table.push_back(DefaultMBB);
+    }
+    uint64_t ClusterSize = (High - Low).getLimitedValue() + 1;
+    for (uint64_t J = 0; J < ClusterSize; ++J)
+      Table.push_back(Clusters[I].MBB);
+    JTWeights[Clusters[I].MBB] += Clusters[I].Weight;
+  }
+
+  unsigned NumDests = JTWeights.size();
+  if (isSuitableForBitTests(NumDests, NumCmps,
+                            Clusters[First].Low->getValue(),
+                            Clusters[Last].High->getValue())) {
+    // Clusters[First..Last] should be lowered as bit tests instead.
+    return false;
+  }
+
+  // Create the MBB that will load from and jump through the table.
+  // Note: We create it here, but it's not inserted into the function yet.
+  MachineFunction *CurMF = FuncInfo.MF;
+  MachineBasicBlock *JumpTableMBB =
+      CurMF->CreateMachineBasicBlock(SI->getParent());
+
+  // Add successors. Note: use table order for determinism.
+  SmallPtrSet<MachineBasicBlock *, 8> Done;
+  for (MachineBasicBlock *Succ : Table) {
+    if (Done.count(Succ))
+      continue;
+    addSuccessorWithWeight(JumpTableMBB, Succ, JTWeights[Succ]);
+    Done.insert(Succ);
+  }
+
+  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
+  unsigned JTI = CurMF->getOrCreateJumpTableInfo(TLI.getJumpTableEncoding())
+                     ->createJumpTableIndex(Table);
+
+  // Set up the jump table info.
+  JumpTable JT(-1U, JTI, JumpTableMBB, nullptr);
+  JumpTableHeader JTH(Clusters[First].Low->getValue(),
+                      Clusters[Last].High->getValue(), SI->getCondition(),
+                      nullptr, false);
+  JTCases.emplace_back(std::move(JTH), std::move(JT));
+
+  JTCluster = CaseCluster::jumpTable(Clusters[First].Low, Clusters[Last].High,
+                                     JTCases.size() - 1, Weight);
+  return true;
+}
+
+void SelectionDAGBuilder::findJumpTables(CaseClusterVector &Clusters,
+                                         const SwitchInst *SI,
+                                         MachineBasicBlock *DefaultMBB) {
+#ifndef NDEBUG
+  // Clusters must be non-empty, sorted, and only contain Range clusters.
+  assert(!Clusters.empty());
+  for (CaseCluster &C : Clusters)
+    assert(C.Kind == CC_Range);
+  for (unsigned i = 1, e = Clusters.size(); i < e; ++i)
+    assert(Clusters[i - 1].High->getValue().slt(Clusters[i].Low->getValue()));
+#endif
+
+  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
+  if (!areJTsAllowed(TLI))
+    return;
+
+  const int64_t N = Clusters.size();
+  const unsigned MinJumpTableSize = TLI.getMinimumJumpTableEntries();
+
+  // TotalCases[i]: Total nbr of cases in Clusters[0..i].
+  SmallVector<unsigned, 8> TotalCases(N);
+
+  for (unsigned i = 0; i < N; ++i) {
+    APInt Hi = Clusters[i].High->getValue();
+    APInt Lo = Clusters[i].Low->getValue();
+    TotalCases[i] = (Hi - Lo).getLimitedValue() + 1;
+    if (i != 0)
+      TotalCases[i] += TotalCases[i - 1];
+  }
+
+  if (N >= MinJumpTableSize && isDense(Clusters, &TotalCases[0], 0, N - 1)) {
+    // Cheap case: the whole range might be suitable for jump table.
+    CaseCluster JTCluster;
+    if (buildJumpTable(Clusters, 0, N - 1, SI, DefaultMBB, JTCluster)) {
+      Clusters[0] = JTCluster;
+      Clusters.resize(1);
+      return;
+    }
+  }
+
+  // The algorithm below is not suitable for -O0.
+  if (TM.getOptLevel() == CodeGenOpt::None)
+    return;
+
+  // Split Clusters into minimum number of dense partitions. The algorithm uses
+  // the same idea as Kannan & Proebsting "Correction to 'Producing Good Code
+  // for the Case Statement'" (1994), but builds the MinPartitions array in
+  // reverse order to make it easier to reconstruct the partitions in ascending
+  // order. In the choice between two optimal partitionings, it picks the one
+  // which yields more jump tables.
+
+  // MinPartitions[i] is the minimum nbr of partitions of Clusters[i..N-1].
+  SmallVector<unsigned, 8> MinPartitions(N);
+  // LastElement[i] is the last element of the partition starting at i.
+  SmallVector<unsigned, 8> LastElement(N);
+  // NumTables[i]: nbr of >= MinJumpTableSize partitions from Clusters[i..N-1].
+  SmallVector<unsigned, 8> NumTables(N);
+
+  // Base case: There is only one way to partition Clusters[N-1].
+  MinPartitions[N - 1] = 1;
+  LastElement[N - 1] = N - 1;
+  assert(MinJumpTableSize > 1);
+  NumTables[N - 1] = 0;
+
+  // Note: loop indexes are signed to avoid underflow.
+  for (int64_t i = N - 2; i >= 0; i--) {
+    // Find optimal partitioning of Clusters[i..N-1].
+    // Baseline: Put Clusters[i] into a partition on its own.
+    MinPartitions[i] = MinPartitions[i + 1] + 1;
+    LastElement[i] = i;
+    NumTables[i] = NumTables[i + 1];
+
+    // Search for a solution that results in fewer partitions.
+    for (int64_t j = N - 1; j > i; j--) {
+      // Try building a partition from Clusters[i..j].
+      if (isDense(Clusters, &TotalCases[0], i, j)) {
+        unsigned NumPartitions = 1 + (j == N - 1 ? 0 : MinPartitions[j + 1]);
+        bool IsTable = j - i + 1 >= MinJumpTableSize;
+        unsigned Tables = IsTable + (j == N - 1 ? 0 : NumTables[j + 1]);
+
+        // If this j leads to fewer partitions, or same number of partitions
+        // with more lookup tables, it is a better partitioning.
+        if (NumPartitions < MinPartitions[i] ||
+            (NumPartitions == MinPartitions[i] && Tables > NumTables[i])) {
+          MinPartitions[i] = NumPartitions;
+          LastElement[i] = j;
+          NumTables[i] = Tables;
+        }
+      }
+    }
+  }
+
+  // Iterate over the partitions, replacing some with jump tables in-place.
+  unsigned DstIndex = 0;
+  for (unsigned First = 0, Last; First < N; First = Last + 1) {
+    Last = LastElement[First];
+    assert(Last >= First);
+    assert(DstIndex <= First);
+    unsigned NumClusters = Last - First + 1;
+
+    CaseCluster JTCluster;
+    if (NumClusters >= MinJumpTableSize &&
+        buildJumpTable(Clusters, First, Last, SI, DefaultMBB, JTCluster)) {
+      Clusters[DstIndex++] = JTCluster;
+    } else {
+      for (unsigned I = First; I <= Last; ++I)
+        std::memmove(&Clusters[DstIndex++], &Clusters[I], sizeof(Clusters[I]));
+    }
+  }
+  Clusters.resize(DstIndex);
+}
+
+bool SelectionDAGBuilder::rangeFitsInWord(const APInt &Low, const APInt &High) {
+  // FIXME: Using the pointer type doesn't seem ideal.
+  uint64_t BW = DAG.getDataLayout().getPointerSizeInBits();
+  uint64_t Range = (High - Low).getLimitedValue(UINT64_MAX - 1) + 1;
+  return Range <= BW;
+}
+
+bool SelectionDAGBuilder::isSuitableForBitTests(unsigned NumDests,
+                                                unsigned NumCmps,
+                                                const APInt &Low,
+                                                const APInt &High) {
+  // FIXME: I don't think NumCmps is the correct metric: a single case and a
+  // range of cases both require only one branch to lower. Just looking at the
+  // number of clusters and destinations should be enough to decide whether to
+  // build bit tests.
+
+  // To lower a range with bit tests, the range must fit the bitwidth of a
+  // machine word.
+  if (!rangeFitsInWord(Low, High))
+    return false;
+
+  // Decide whether it's profitable to lower this range with bit tests. Each
+  // destination requires a bit test and branch, and there is an overall range
+  // check branch. For a small number of clusters, separate comparisons might be
+  // cheaper, and for many destinations, splitting the range might be better.
+  return (NumDests == 1 && NumCmps >= 3) ||
+         (NumDests == 2 && NumCmps >= 5) ||
+         (NumDests == 3 && NumCmps >= 6);
+}
+
+bool SelectionDAGBuilder::buildBitTests(CaseClusterVector &Clusters,
+                                        unsigned First, unsigned Last,
+                                        const SwitchInst *SI,
+                                        CaseCluster &BTCluster) {
+  assert(First <= Last);
+  if (First == Last)
+    return false;
+
+  BitVector Dests(FuncInfo.MF->getNumBlockIDs());
+  unsigned NumCmps = 0;
+  for (int64_t I = First; I <= Last; ++I) {
+    assert(Clusters[I].Kind == CC_Range);
+    Dests.set(Clusters[I].MBB->getNumber());
+    NumCmps += (Clusters[I].Low == Clusters[I].High) ? 1 : 2;
+  }
+  unsigned NumDests = Dests.count();
+
+  APInt Low = Clusters[First].Low->getValue();
+  APInt High = Clusters[Last].High->getValue();
+  assert(Low.slt(High));
+
+  if (!isSuitableForBitTests(NumDests, NumCmps, Low, High))
+    return false;
+
+  APInt LowBound;
+  APInt CmpRange;
+
+  const int BitWidth = DAG.getTargetLoweringInfo()
+                           .getPointerTy(DAG.getDataLayout())
+                           .getSizeInBits();
+  assert(rangeFitsInWord(Low, High) && "Case range must fit in bit mask!");
+
+  if (Low.isNonNegative() && High.slt(BitWidth)) {
+    // Optimize the case where all the case values fit in a
+    // word without having to subtract minValue. In this case,
+    // we can optimize away the subtraction.
+    LowBound = APInt::getNullValue(Low.getBitWidth());
+    CmpRange = High;
+  } else {
+    LowBound = Low;
+    CmpRange = High - Low;
+  }
+
+  CaseBitsVector CBV;
+  uint32_t TotalWeight = 0;
+  for (unsigned i = First; i <= Last; ++i) {
+    // Find the CaseBits for this destination.
+    unsigned j;
+    for (j = 0; j < CBV.size(); ++j)
+      if (CBV[j].BB == Clusters[i].MBB)
+        break;
+    if (j == CBV.size())
+      CBV.push_back(CaseBits(0, Clusters[i].MBB, 0, 0));
+    CaseBits *CB = &CBV[j];
+
+    // Update Mask, Bits and ExtraWeight.
+    uint64_t Lo = (Clusters[i].Low->getValue() - LowBound).getZExtValue();
+    uint64_t Hi = (Clusters[i].High->getValue() - LowBound).getZExtValue();
+    assert(Hi >= Lo && Hi < 64 && "Invalid bit case!");
+    CB->Mask |= (-1ULL >> (63 - (Hi - Lo))) << Lo;
+    CB->Bits += Hi - Lo + 1;
+    CB->ExtraWeight += Clusters[i].Weight;
+    TotalWeight += Clusters[i].Weight;
+    assert(TotalWeight >= Clusters[i].Weight && "Weight overflow!");
+  }
+
+  BitTestInfo BTI;
+  std::sort(CBV.begin(), CBV.end(), [](const CaseBits &a, const CaseBits &b) {
+    // Sort by weight first, number of bits second.
+    if (a.ExtraWeight != b.ExtraWeight)
+      return a.ExtraWeight > b.ExtraWeight;
+    return a.Bits > b.Bits;
+  });
+
+  for (auto &CB : CBV) {
+    MachineBasicBlock *BitTestBB =
+        FuncInfo.MF->CreateMachineBasicBlock(SI->getParent());
+    BTI.push_back(BitTestCase(CB.Mask, BitTestBB, CB.BB, CB.ExtraWeight));
+  }
+  BitTestCases.emplace_back(std::move(LowBound), std::move(CmpRange),
+                            SI->getCondition(), -1U, MVT::Other, false, nullptr,
+                            nullptr, std::move(BTI));
+
+  BTCluster = CaseCluster::bitTests(Clusters[First].Low, Clusters[Last].High,
+                                    BitTestCases.size() - 1, TotalWeight);
+  return true;
+}
+
+void SelectionDAGBuilder::findBitTestClusters(CaseClusterVector &Clusters,
+                                              const SwitchInst *SI) {
+// Partition Clusters into as few subsets as possible, where each subset has a
+// range that fits in a machine word and has <= 3 unique destinations.
+
+#ifndef NDEBUG
+  // Clusters must be sorted and contain Range or JumpTable clusters.
+  assert(!Clusters.empty());
+  assert(Clusters[0].Kind == CC_Range || Clusters[0].Kind == CC_JumpTable);
+  for (const CaseCluster &C : Clusters)
+    assert(C.Kind == CC_Range || C.Kind == CC_JumpTable);
+  for (unsigned i = 1; i < Clusters.size(); ++i)
+    assert(Clusters[i-1].High->getValue().slt(Clusters[i].Low->getValue()));
+#endif
+
+  // The algorithm below is not suitable for -O0.
+  if (TM.getOptLevel() == CodeGenOpt::None)
+    return;
+
+  // If target does not have legal shift left, do not emit bit tests at all.
+  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
+  EVT PTy = TLI.getPointerTy(DAG.getDataLayout());
+  if (!TLI.isOperationLegal(ISD::SHL, PTy))
+    return;
+
+  int BitWidth = PTy.getSizeInBits();
+  const int64_t N = Clusters.size();
+
+  // MinPartitions[i] is the minimum nbr of partitions of Clusters[i..N-1].
+  SmallVector<unsigned, 8> MinPartitions(N);
+  // LastElement[i] is the last element of the partition starting at i.
+  SmallVector<unsigned, 8> LastElement(N);
+
+  // FIXME: This might not be the best algorithm for finding bit test clusters.
+
+  // Base case: There is only one way to partition Clusters[N-1].
+  MinPartitions[N - 1] = 1;
+  LastElement[N - 1] = N - 1;
+
+  // Note: loop indexes are signed to avoid underflow.
+  for (int64_t i = N - 2; i >= 0; --i) {
+    // Find optimal partitioning of Clusters[i..N-1].
+    // Baseline: Put Clusters[i] into a partition on its own.
+    MinPartitions[i] = MinPartitions[i + 1] + 1;
+    LastElement[i] = i;
+
+    // Search for a solution that results in fewer partitions.
+    // Note: the search is limited by BitWidth, reducing time complexity.
+    for (int64_t j = std::min(N - 1, i + BitWidth - 1); j > i; --j) {
+      // Try building a partition from Clusters[i..j].
+
+      // Check the range.
+      if (!rangeFitsInWord(Clusters[i].Low->getValue(),
+                           Clusters[j].High->getValue()))
+        continue;
+
+      // Check nbr of destinations and cluster types.
+      // FIXME: This works, but doesn't seem very efficient.
+      bool RangesOnly = true;
+      BitVector Dests(FuncInfo.MF->getNumBlockIDs());
+      for (int64_t k = i; k <= j; k++) {
+        if (Clusters[k].Kind != CC_Range) {
+          RangesOnly = false;
+          break;
+        }
+        Dests.set(Clusters[k].MBB->getNumber());
+      }
+      if (!RangesOnly || Dests.count() > 3)
+        break;
+
+      // Check if it's a better partition.
+      unsigned NumPartitions = 1 + (j == N - 1 ? 0 : MinPartitions[j + 1]);
+      if (NumPartitions < MinPartitions[i]) {
+        // Found a better partition.
+        MinPartitions[i] = NumPartitions;
+        LastElement[i] = j;
+      }
+    }
+  }
+
+  // Iterate over the partitions, replacing with bit-test clusters in-place.
+  unsigned DstIndex = 0;
+  for (unsigned First = 0, Last; First < N; First = Last + 1) {
+    Last = LastElement[First];
+    assert(First <= Last);
+    assert(DstIndex <= First);
+
+    CaseCluster BitTestCluster;
+    if (buildBitTests(Clusters, First, Last, SI, BitTestCluster)) {
+      Clusters[DstIndex++] = BitTestCluster;
+    } else {
+      size_t NumClusters = Last - First + 1;
+      std::memmove(&Clusters[DstIndex], &Clusters[First],
+                   sizeof(Clusters[0]) * NumClusters);
+      DstIndex += NumClusters;
+    }
+  }
+  Clusters.resize(DstIndex);
+}
+
+void SelectionDAGBuilder::lowerWorkItem(SwitchWorkListItem W, Value *Cond,
+                                        MachineBasicBlock *SwitchMBB,
+                                        MachineBasicBlock *DefaultMBB) {
+  MachineFunction *CurMF = FuncInfo.MF;
+  MachineBasicBlock *NextMBB = nullptr;
+  MachineFunction::iterator BBI = W.MBB;
+  if (++BBI != FuncInfo.MF->end())
+    NextMBB = BBI;
+
+  unsigned Size = W.LastCluster - W.FirstCluster + 1;
+
+  BranchProbabilityInfo *BPI = FuncInfo.BPI;
+
+  if (Size == 2 && W.MBB == SwitchMBB) {
+    // If any two of the cases has the same destination, and if one value
+    // is the same as the other, but has one bit unset that the other has set,
+    // use bit manipulation to do two compares at once.  For example:
+    // "if (X == 6 || X == 4)" -> "if ((X|2) == 6)"
+    // TODO: This could be extended to merge any 2 cases in switches with 3
+    // cases.
+    // TODO: Handle cases where W.CaseBB != SwitchBB.
+    CaseCluster &Small = *W.FirstCluster;
+    CaseCluster &Big = *W.LastCluster;
+
+    if (Small.Low == Small.High && Big.Low == Big.High &&
+        Small.MBB == Big.MBB) {
+      const APInt &SmallValue = Small.Low->getValue();
+      const APInt &BigValue = Big.Low->getValue();
+
+      // Check that there is only one bit different.
+      APInt CommonBit = BigValue ^ SmallValue;
+      if (CommonBit.isPowerOf2()) {
+        SDValue CondLHS = getValue(Cond);
+        EVT VT = CondLHS.getValueType();
+        SDLoc DL = getCurSDLoc();
+
+        SDValue Or = DAG.getNode(ISD::OR, DL, VT, CondLHS,
+                                 DAG.getConstant(CommonBit, DL, VT));
+        SDValue Cond = DAG.getSetCC(
+            DL, MVT::i1, Or, DAG.getConstant(BigValue | SmallValue, DL, VT),
+            ISD::SETEQ);
+
+        // Update successor info.
+        // Both Small and Big will jump to Small.BB, so we sum up the weights.
+        addSuccessorWithWeight(SwitchMBB, Small.MBB, Small.Weight + Big.Weight);
+        addSuccessorWithWeight(
+            SwitchMBB, DefaultMBB,
+            // The default destination is the first successor in IR.
+            BPI ? BPI->getEdgeWeight(SwitchMBB->getBasicBlock(), (unsigned)0)
+                : 0);
+
+        // Insert the true branch.
+        SDValue BrCond =
+            DAG.getNode(ISD::BRCOND, DL, MVT::Other, getControlRoot(), Cond,
+                        DAG.getBasicBlock(Small.MBB));
+        // Insert the false branch.
+        BrCond = DAG.getNode(ISD::BR, DL, MVT::Other, BrCond,
+                             DAG.getBasicBlock(DefaultMBB));
+
+        DAG.setRoot(BrCond);
+        return;
+      }
+    }
+  }
+
+  if (TM.getOptLevel() != CodeGenOpt::None) {
+    // Order cases by weight so the most likely case will be checked first.
+    std::sort(W.FirstCluster, W.LastCluster + 1,
+              [](const CaseCluster &a, const CaseCluster &b) {
+      return a.Weight > b.Weight;
+    });
+
+    // Rearrange the case blocks so that the last one falls through if possible
+    // without without changing the order of weights.
+    for (CaseClusterIt I = W.LastCluster; I > W.FirstCluster; ) {
+      --I;
+      if (I->Weight > W.LastCluster->Weight)
+        break;
+      if (I->Kind == CC_Range && I->MBB == NextMBB) {
+        std::swap(*I, *W.LastCluster);
+        break;
+      }
+    }
+  }
+
+  // Compute total weight.
+  uint32_t UnhandledWeights = 0;
+  for (CaseClusterIt I = W.FirstCluster; I <= W.LastCluster; ++I) {
+    UnhandledWeights += I->Weight;
+    assert(UnhandledWeights >= I->Weight && "Weight overflow!");
+  }
+
+  MachineBasicBlock *CurMBB = W.MBB;
+  for (CaseClusterIt I = W.FirstCluster, E = W.LastCluster; I <= E; ++I) {
+    MachineBasicBlock *Fallthrough;
+    if (I == W.LastCluster) {
+      // For the last cluster, fall through to the default destination.
+      Fallthrough = DefaultMBB;
+    } else {
+      Fallthrough = CurMF->CreateMachineBasicBlock(CurMBB->getBasicBlock());
+      CurMF->insert(BBI, Fallthrough);
+      // Put Cond in a virtual register to make it available from the new blocks.
+      ExportFromCurrentBlock(Cond);
+    }
+
+    switch (I->Kind) {
+      case CC_JumpTable: {
+        // FIXME: Optimize away range check based on pivot comparisons.
+        JumpTableHeader *JTH = &JTCases[I->JTCasesIndex].first;
+        JumpTable *JT = &JTCases[I->JTCasesIndex].second;
+
+        // The jump block hasn't been inserted yet; insert it here.
+        MachineBasicBlock *JumpMBB = JT->MBB;
+        CurMF->insert(BBI, JumpMBB);
+        addSuccessorWithWeight(CurMBB, Fallthrough);
+        addSuccessorWithWeight(CurMBB, JumpMBB);
+
+        // The jump table header will be inserted in our current block, do the
+        // range check, and fall through to our fallthrough block.
+        JTH->HeaderBB = CurMBB;
+        JT->Default = Fallthrough; // FIXME: Move Default to JumpTableHeader.
+
+        // If we're in the right place, emit the jump table header right now.
+        if (CurMBB == SwitchMBB) {
+          visitJumpTableHeader(*JT, *JTH, SwitchMBB);
+          JTH->Emitted = true;
+        }
+        break;
+      }
+      case CC_BitTests: {
+        // FIXME: Optimize away range check based on pivot comparisons.
+        BitTestBlock *BTB = &BitTestCases[I->BTCasesIndex];
+
+        // The bit test blocks haven't been inserted yet; insert them here.
+        for (BitTestCase &BTC : BTB->Cases)
+          CurMF->insert(BBI, BTC.ThisBB);
+
+        // Fill in fields of the BitTestBlock.
+        BTB->Parent = CurMBB;
+        BTB->Default = Fallthrough;
+
+        // If we're in the right place, emit the bit test header header right now.
+        if (CurMBB ==SwitchMBB) {
+          visitBitTestHeader(*BTB, SwitchMBB);
+          BTB->Emitted = true;
+        }
+        break;
+      }
+      case CC_Range: {
+        const Value *RHS, *LHS, *MHS;
+        ISD::CondCode CC;
+        if (I->Low == I->High) {
+          // Check Cond == I->Low.
+          CC = ISD::SETEQ;
+          LHS = Cond;
+          RHS=I->Low;
+          MHS = nullptr;
+        } else {
+          // Check I->Low <= Cond <= I->High.
+          CC = ISD::SETLE;
+          LHS = I->Low;
+          MHS = Cond;
+          RHS = I->High;
+        }
+
+        // The false weight is the sum of all unhandled cases.
+        UnhandledWeights -= I->Weight;
+        CaseBlock CB(CC, LHS, RHS, MHS, I->MBB, Fallthrough, CurMBB, I->Weight,
+                     UnhandledWeights);
+
+        if (CurMBB == SwitchMBB)
+          visitSwitchCase(CB, SwitchMBB);
+        else
+          SwitchCases.push_back(CB);
+
+        break;
+      }
+    }
+    CurMBB = Fallthrough;
+  }
+}
+
+unsigned SelectionDAGBuilder::caseClusterRank(const CaseCluster &CC,
+                                              CaseClusterIt First,
+                                              CaseClusterIt Last) {
+  return std::count_if(First, Last + 1, [&](const CaseCluster &X) {
+    if (X.Weight != CC.Weight)
+      return X.Weight > CC.Weight;
+
+    // Ties are broken by comparing the case value.
+    return X.Low->getValue().slt(CC.Low->getValue());
+  });
+}
+
+void SelectionDAGBuilder::splitWorkItem(SwitchWorkList &WorkList,
+                                        const SwitchWorkListItem &W,
+                                        Value *Cond,
+                                        MachineBasicBlock *SwitchMBB) {
+  assert(W.FirstCluster->Low->getValue().slt(W.LastCluster->Low->getValue()) &&
+         "Clusters not sorted?");
+
+  assert(W.LastCluster - W.FirstCluster + 1 >= 2 && "Too small to split!");
+
+  // Balance the tree based on branch weights to create a near-optimal (in terms
+  // of search time given key frequency) binary search tree. See e.g. Kurt
+  // Mehlhorn "Nearly Optimal Binary Search Trees" (1975).
+  CaseClusterIt LastLeft = W.FirstCluster;
+  CaseClusterIt FirstRight = W.LastCluster;
+  uint32_t LeftWeight = LastLeft->Weight;
+  uint32_t RightWeight = FirstRight->Weight;
+
+  // Move LastLeft and FirstRight towards each other from opposite directions to
+  // find a partitioning of the clusters which balances the weight on both
+  // sides. If LeftWeight and RightWeight are equal, alternate which side is
+  // taken to ensure 0-weight nodes are distributed evenly.
+  unsigned I = 0;
+  while (LastLeft + 1 < FirstRight) {
+    if (LeftWeight < RightWeight || (LeftWeight == RightWeight && (I & 1)))
+      LeftWeight += (++LastLeft)->Weight;
+    else
+      RightWeight += (--FirstRight)->Weight;
+    I++;
+  }
+
+  for (;;) {
+    // Our binary search tree differs from a typical BST in that ours can have up
+    // to three values in each leaf. The pivot selection above doesn't take that
+    // into account, which means the tree might require more nodes and be less
+    // efficient. We compensate for this here.
+
+    unsigned NumLeft = LastLeft - W.FirstCluster + 1;
+    unsigned NumRight = W.LastCluster - FirstRight + 1;
+
+    if (std::min(NumLeft, NumRight) < 3 && std::max(NumLeft, NumRight) > 3) {
+      // If one side has less than 3 clusters, and the other has more than 3,
+      // consider taking a cluster from the other side.
+
+      if (NumLeft < NumRight) {
+        // Consider moving the first cluster on the right to the left side.
+        CaseCluster &CC = *FirstRight;
+        unsigned RightSideRank = caseClusterRank(CC, FirstRight, W.LastCluster);
+        unsigned LeftSideRank = caseClusterRank(CC, W.FirstCluster, LastLeft);
+        if (LeftSideRank <= RightSideRank) {
+          // Moving the cluster to the left does not demote it.
+          ++LastLeft;
+          ++FirstRight;
+          continue;
+        }
+      } else {
+        assert(NumRight < NumLeft);
+        // Consider moving the last element on the left to the right side.
+        CaseCluster &CC = *LastLeft;
+        unsigned LeftSideRank = caseClusterRank(CC, W.FirstCluster, LastLeft);
+        unsigned RightSideRank = caseClusterRank(CC, FirstRight, W.LastCluster);
+        if (RightSideRank <= LeftSideRank) {
+          // Moving the cluster to the right does not demot it.
+          --LastLeft;
+          --FirstRight;
+          continue;
+        }
+      }
+    }
+    break;
+  }
+
+  assert(LastLeft + 1 == FirstRight);
+  assert(LastLeft >= W.FirstCluster);
+  assert(FirstRight <= W.LastCluster);
+
+  // Use the first element on the right as pivot since we will make less-than
+  // comparisons against it.
+  CaseClusterIt PivotCluster = FirstRight;
+  assert(PivotCluster > W.FirstCluster);
+  assert(PivotCluster <= W.LastCluster);
+
+  CaseClusterIt FirstLeft = W.FirstCluster;
+  CaseClusterIt LastRight = W.LastCluster;
+
+  const ConstantInt *Pivot = PivotCluster->Low;
+
+  // New blocks will be inserted immediately after the current one.
+  MachineFunction::iterator BBI = W.MBB;
+  ++BBI;
+
+  // We will branch to the LHS if Value < Pivot. If LHS is a single cluster,
+  // we can branch to its destination directly if it's squeezed exactly in
+  // between the known lower bound and Pivot - 1.
+  MachineBasicBlock *LeftMBB;
+  if (FirstLeft == LastLeft && FirstLeft->Kind == CC_Range &&
+      FirstLeft->Low == W.GE &&
+      (FirstLeft->High->getValue() + 1LL) == Pivot->getValue()) {
+    LeftMBB = FirstLeft->MBB;
+  } else {
+    LeftMBB = FuncInfo.MF->CreateMachineBasicBlock(W.MBB->getBasicBlock());
+    FuncInfo.MF->insert(BBI, LeftMBB);
+    WorkList.push_back({LeftMBB, FirstLeft, LastLeft, W.GE, Pivot});
+    // Put Cond in a virtual register to make it available from the new blocks.
+    ExportFromCurrentBlock(Cond);
+  }
+
+  // Similarly, we will branch to the RHS if Value >= Pivot. If RHS is a
+  // single cluster, RHS.Low == Pivot, and we can branch to its destination
+  // directly if RHS.High equals the current upper bound.
+  MachineBasicBlock *RightMBB;
+  if (FirstRight == LastRight && FirstRight->Kind == CC_Range &&
+      W.LT && (FirstRight->High->getValue() + 1ULL) == W.LT->getValue()) {
+    RightMBB = FirstRight->MBB;
+  } else {
+    RightMBB = FuncInfo.MF->CreateMachineBasicBlock(W.MBB->getBasicBlock());
+    FuncInfo.MF->insert(BBI, RightMBB);
+    WorkList.push_back({RightMBB, FirstRight, LastRight, Pivot, W.LT});
+    // Put Cond in a virtual register to make it available from the new blocks.
+    ExportFromCurrentBlock(Cond);
+  }
+
+  // Create the CaseBlock record that will be used to lower the branch.
+  CaseBlock CB(ISD::SETLT, Cond, Pivot, nullptr, LeftMBB, RightMBB, W.MBB,
+               LeftWeight, RightWeight);
+
+  if (W.MBB == SwitchMBB)
+    visitSwitchCase(CB, SwitchMBB);
+  else
+    SwitchCases.push_back(CB);
+}
+
+void SelectionDAGBuilder::visitSwitch(const SwitchInst &SI) {
+  // Extract cases from the switch.
+  BranchProbabilityInfo *BPI = FuncInfo.BPI;
+  CaseClusterVector Clusters;
+  Clusters.reserve(SI.getNumCases());
+  for (auto I : SI.cases()) {
+    MachineBasicBlock *Succ = FuncInfo.MBBMap[I.getCaseSuccessor()];
+    const ConstantInt *CaseVal = I.getCaseValue();
+    uint32_t Weight =
+        BPI ? BPI->getEdgeWeight(SI.getParent(), I.getSuccessorIndex()) : 0;
+    Clusters.push_back(CaseCluster::range(CaseVal, CaseVal, Succ, Weight));
+  }
+
+  MachineBasicBlock *DefaultMBB = FuncInfo.MBBMap[SI.getDefaultDest()];
+
+  // Cluster adjacent cases with the same destination. We do this at all
+  // optimization levels because it's cheap to do and will make codegen faster
+  // if there are many clusters.
+  sortAndRangeify(Clusters);
+
+  if (TM.getOptLevel() != CodeGenOpt::None) {
+    // Replace an unreachable default with the most popular destination.
+    // FIXME: Exploit unreachable default more aggressively.
+    bool UnreachableDefault =
+        isa<UnreachableInst>(SI.getDefaultDest()->getFirstNonPHIOrDbg());
+    if (UnreachableDefault && !Clusters.empty()) {
+      DenseMap<const BasicBlock *, unsigned> Popularity;
+      unsigned MaxPop = 0;
+      const BasicBlock *MaxBB = nullptr;
+      for (auto I : SI.cases()) {
+        const BasicBlock *BB = I.getCaseSuccessor();
+        if (++Popularity[BB] > MaxPop) {
+          MaxPop = Popularity[BB];
+          MaxBB = BB;
+        }
+      }
+      // Set new default.
+      assert(MaxPop > 0 && MaxBB);
+      DefaultMBB = FuncInfo.MBBMap[MaxBB];
+
+      // Remove cases that were pointing to the destination that is now the
+      // default.
+      CaseClusterVector New;
+      New.reserve(Clusters.size());
+      for (CaseCluster &CC : Clusters) {
+        if (CC.MBB != DefaultMBB)
+          New.push_back(CC);
+      }
+      Clusters = std::move(New);
+    }
+  }
+
+  // If there is only the default destination, jump there directly.
+  MachineBasicBlock *SwitchMBB = FuncInfo.MBB;
+  if (Clusters.empty()) {
+    SwitchMBB->addSuccessor(DefaultMBB);
+    if (DefaultMBB != NextBlock(SwitchMBB)) {
+      DAG.setRoot(DAG.getNode(ISD::BR, getCurSDLoc(), MVT::Other,
+                              getControlRoot(), DAG.getBasicBlock(DefaultMBB)));
+    }
+    return;
+  }
+
+  findJumpTables(Clusters, &SI, DefaultMBB);
+  findBitTestClusters(Clusters, &SI);
+
+  DEBUG({
+    dbgs() << "Case clusters: ";
+    for (const CaseCluster &C : Clusters) {
+      if (C.Kind == CC_JumpTable) dbgs() << "JT:";
+      if (C.Kind == CC_BitTests) dbgs() << "BT:";
+
+      C.Low->getValue().print(dbgs(), true);
+      if (C.Low != C.High) {
+        dbgs() << '-';
+        C.High->getValue().print(dbgs(), true);
+      }
+      dbgs() << ' ';
+    }
+    dbgs() << '\n';
+  });
+
+  assert(!Clusters.empty());
+  SwitchWorkList WorkList;
+  CaseClusterIt First = Clusters.begin();
+  CaseClusterIt Last = Clusters.end() - 1;
+  WorkList.push_back({SwitchMBB, First, Last, nullptr, nullptr});
+
+  while (!WorkList.empty()) {
+    SwitchWorkListItem W = WorkList.back();
+    WorkList.pop_back();
+    unsigned NumClusters = W.LastCluster - W.FirstCluster + 1;
+
+    if (NumClusters > 3 && TM.getOptLevel() != CodeGenOpt::None) {
+      // For optimized builds, lower large range as a balanced binary tree.
+      splitWorkItem(WorkList, W, SI.getCondition(), SwitchMBB);
+      continue;
+    }
+
+    lowerWorkItem(W, SI.getCondition(), SwitchMBB, DefaultMBB);
+  }
+}
diff --git a/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.h b/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.h
index a27f470df17..f71190d6375 100644
--- a/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.h
+++ b/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.h
@@ -17,6 +17,7 @@
 #include "StatepointLowering.h"
 #include "llvm/ADT/APInt.h"
 #include "llvm/ADT/DenseMap.h"
+#include "llvm/CodeGen/Analysis.h"
 #include "llvm/CodeGen/SelectionDAG.h"
 #include "llvm/CodeGen/SelectionDAGNodes.h"
 #include "llvm/IR/CallSite.h"
@@ -134,26 +135,65 @@ class SelectionDAGBuilder {
   /// SDNodes we create.
   unsigned SDNodeOrder;
 
-  /// Case - A struct to record the Value for a switch case, and the
-  /// case's target basic block.
-  struct Case {
-    const ConstantInt *Low;
-    const ConstantInt *High;
-    MachineBasicBlock* BB;
-    uint32_t ExtraWeight;
+  enum CaseClusterKind {
+    /// A cluster of adjacent case labels with the same destination, or just one
+    /// case.
+    CC_Range,
+    /// A cluster of cases suitable for jump table lowering.
+    CC_JumpTable,
+    /// A cluster of cases suitable for bit test lowering.
+    CC_BitTests
+  };
 
-    Case() : Low(nullptr), High(nullptr), BB(nullptr), ExtraWeight(0) { }
-    Case(const ConstantInt *low, const ConstantInt *high, MachineBasicBlock *bb,
-         uint32_t extraweight) : Low(low), High(high), BB(bb),
-         ExtraWeight(extraweight) { }
+  /// A cluster of case labels.
+  struct CaseCluster {
+    CaseClusterKind Kind;
+    const ConstantInt *Low, *High;
+    union {
+      MachineBasicBlock *MBB;
+      unsigned JTCasesIndex;
+      unsigned BTCasesIndex;
+    };
+    uint32_t Weight;
+
+    static CaseCluster range(const ConstantInt *Low, const ConstantInt *High,
+                             MachineBasicBlock *MBB, uint32_t Weight) {
+      CaseCluster C;
+      C.Kind = CC_Range;
+      C.Low = Low;
+      C.High = High;
+      C.MBB = MBB;
+      C.Weight = Weight;
+      return C;
+    }
 
-    APInt size() const {
-      const APInt &rHigh = High->getValue();
-      const APInt &rLow  = Low->getValue();
-      return (rHigh - rLow + 1ULL);
+    static CaseCluster jumpTable(const ConstantInt *Low,
+                                 const ConstantInt *High, unsigned JTCasesIndex,
+                                 uint32_t Weight) {
+      CaseCluster C;
+      C.Kind = CC_JumpTable;
+      C.Low = Low;
+      C.High = High;
+      C.JTCasesIndex = JTCasesIndex;
+      C.Weight = Weight;
+      return C;
+    }
+
+    static CaseCluster bitTests(const ConstantInt *Low, const ConstantInt *High,
+                                unsigned BTCasesIndex, uint32_t Weight) {
+      CaseCluster C;
+      C.Kind = CC_BitTests;
+      C.Low = Low;
+      C.High = High;
+      C.BTCasesIndex = BTCasesIndex;
+      C.Weight = Weight;
+      return C;
     }
   };
 
+  typedef std::vector<CaseCluster> CaseClusterVector;
+  typedef CaseClusterVector::iterator CaseClusterIt;
+
   struct CaseBits {
     uint64_t Mask;
     MachineBasicBlock* BB;
@@ -163,42 +203,14 @@ class SelectionDAGBuilder {
     CaseBits(uint64_t mask, MachineBasicBlock* bb, unsigned bits,
              uint32_t Weight):
       Mask(mask), BB(bb), Bits(bits), ExtraWeight(Weight) { }
-  };
 
-  typedef std::vector<Case>           CaseVector;
-  typedef std::vector<CaseBits>       CaseBitsVector;
-  typedef CaseVector::iterator        CaseItr;
-  typedef std::pair<CaseItr, CaseItr> CaseRange;
-
-  /// CaseRec - A struct with ctor used in lowering switches to a binary tree
-  /// of conditional branches.
-  struct CaseRec {
-    CaseRec(MachineBasicBlock *bb, const ConstantInt *lt, const ConstantInt *ge,
-            CaseRange r) :
-    CaseBB(bb), LT(lt), GE(ge), Range(r) {}
-
-    /// CaseBB - The MBB in which to emit the compare and branch
-    MachineBasicBlock *CaseBB;
-    /// LT, GE - If nonzero, we know the current case value must be less-than or
-    /// greater-than-or-equal-to these Constants.
-    const ConstantInt *LT;
-    const ConstantInt *GE;
-    /// Range - A pair of iterators representing the range of case values to be
-    /// processed at this point in the binary search tree.
-    CaseRange Range;
+    CaseBits() : Mask(0), BB(nullptr), Bits(0), ExtraWeight(0) {}
   };
 
-  typedef std::vector<CaseRec> CaseRecVector;
-
-  struct CaseBitsCmp {
-    bool operator()(const CaseBits &C1, const CaseBits &C2) {
-      return C1.Bits > C2.Bits;
-    }
-  };
+  typedef std::vector<CaseBits> CaseBitsVector;
 
-  /// Populate Cases with the cases in SI, clustering adjacent cases with the
-  /// same destination together.
-  void Clusterify(CaseVector &Cases, const SwitchInst *SI);
+  /// Sort Clusters and merge adjacent cases.
+  void sortAndRangeify(CaseClusterVector &Clusters);
 
   /// CaseBlock - This structure is used to communicate between
   /// SelectionDAGBuilder and SDISel for the code generation of additional basic
@@ -288,6 +300,63 @@ class SelectionDAGBuilder {
     BitTestInfo Cases;
   };
 
+  /// Minimum jump table density, in percent.
+  enum { MinJumpTableDensity = 40 };
+
+  /// Check whether a range of clusters is dense enough for a jump table.
+  bool isDense(const CaseClusterVector &Clusters, unsigned *TotalCases,
+               unsigned First, unsigned Last);
+
+  /// Build a jump table cluster from Clusters[First..Last]. Returns false if it
+  /// decides it's not a good idea.
+  bool buildJumpTable(CaseClusterVector &Clusters, unsigned First,
+                      unsigned Last, const SwitchInst *SI,
+                      MachineBasicBlock *DefaultMBB, CaseCluster &JTCluster);
+
+  /// Find clusters of cases suitable for jump table lowering.
+  void findJumpTables(CaseClusterVector &Clusters, const SwitchInst *SI,
+                      MachineBasicBlock *DefaultMBB);
+
+  /// Check whether the range [Low,High] fits in a machine word.
+  bool rangeFitsInWord(const APInt &Low, const APInt &High);
+
+  /// Check whether these clusters are suitable for lowering with bit tests based
+  /// on the number of destinations, comparison metric, and range.
+  bool isSuitableForBitTests(unsigned NumDests, unsigned NumCmps,
+                             const APInt &Low, const APInt &High);
+
+  /// Build a bit test cluster from Clusters[First..Last]. Returns false if it
+  /// decides it's not a good idea.
+  bool buildBitTests(CaseClusterVector &Clusters, unsigned First, unsigned Last,
+                     const SwitchInst *SI, CaseCluster &BTCluster);
+
+  /// Find clusters of cases suitable for bit test lowering.
+  void findBitTestClusters(CaseClusterVector &Clusters, const SwitchInst *SI);
+
+  struct SwitchWorkListItem {
+    MachineBasicBlock *MBB;
+    CaseClusterIt FirstCluster;
+    CaseClusterIt LastCluster;
+    const ConstantInt *GE;
+    const ConstantInt *LT;
+  };
+  typedef SmallVector<SwitchWorkListItem, 4> SwitchWorkList;
+
+  /// Determine the rank by weight of CC in [First,Last]. If CC has more weight
+  /// than each cluster in the range, its rank is 0.
+  static unsigned caseClusterRank(const CaseCluster &CC, CaseClusterIt First,
+                                  CaseClusterIt Last);
+
+  /// Emit comparison and split W into two subtrees.
+  void splitWorkItem(SwitchWorkList &WorkList, const SwitchWorkListItem &W,
+                     Value *Cond, MachineBasicBlock *SwitchMBB);
+
+  /// Lower W.
+  void lowerWorkItem(SwitchWorkListItem W, Value *Cond,
+                     MachineBasicBlock *SwitchMBB,
+                     MachineBasicBlock *DefaultMBB);
+
+
   /// A class which encapsulates all of the information needed to generate a
   /// stack protector check and signals to isel via its state being initialized
   /// that a stack protector needs to be generated.
@@ -604,6 +673,8 @@ class SelectionDAGBuilder {
   // generate the debug data structures now that we've seen its definition.
   void resolveDanglingDebugInfo(const Value *V, SDValue Val);
   SDValue getValue(const Value *V);
+  bool findValue(const Value *V) const;
+
   SDValue getNonRegisterValue(const Value *V);
   SDValue getValueImpl(const Value *V);
 
@@ -613,12 +684,6 @@ class SelectionDAGBuilder {
     N = NewN;
   }
 
-  void removeValue(const Value *V) {
-    // This is to support hack in lowerCallFromStatepoint
-    // Should be removed when hack is resolved
-    NodeMap.erase(V);
-  }
-
   void setUnusedArgValue(const Value *V, SDValue NewN) {
     SDValue &N = UnusedArgNodeMap[V];
     assert(!N.getNode() && "Already set a value for this node!");
@@ -627,7 +692,8 @@ class SelectionDAGBuilder {
 
   void FindMergedConditions(const Value *Cond, MachineBasicBlock *TBB,
                             MachineBasicBlock *FBB, MachineBasicBlock *CurBB,
-                            MachineBasicBlock *SwitchBB, unsigned Opc,
+                            MachineBasicBlock *SwitchBB,
+                            Instruction::BinaryOps Opc,
                             uint32_t TW, uint32_t FW);
   void EmitBranchForMergedCondition(const Value *Cond, MachineBasicBlock *TBB,
                                     MachineBasicBlock *FBB,
@@ -646,7 +712,7 @@ class SelectionDAGBuilder {
           unsigned ArgIdx,
           unsigned NumArgs,
           SDValue Callee,
-          bool UseVoidTy = false,
+          Type *ReturnTy,
           MachineBasicBlock *LandingPad = nullptr,
           bool IsPatchPoint = false);
 
@@ -669,29 +735,12 @@ class SelectionDAGBuilder {
   void visitSwitch(const SwitchInst &I);
   void visitIndirectBr(const IndirectBrInst &I);
   void visitUnreachable(const UnreachableInst &I);
-
-  // Helpers for visitSwitch
-  bool handleSmallSwitchRange(CaseRec& CR,
-                              CaseRecVector& WorkList,
-                              const Value* SV,
-                              MachineBasicBlock* Default,
-                              MachineBasicBlock *SwitchBB);
-  bool handleJTSwitchCase(CaseRec& CR,
-                          CaseRecVector& WorkList,
-                          const Value* SV,
-                          MachineBasicBlock* Default,
-                          MachineBasicBlock *SwitchBB);
-  bool handleBTSplitSwitchCase(CaseRec& CR,
-                               CaseRecVector& WorkList,
-                               const Value* SV,
-                               MachineBasicBlock *SwitchBB);
-  void splitSwitchCase(CaseRec &CR, CaseItr Pivot, CaseRecVector &WorkList,
-                       const Value *SV, MachineBasicBlock *SwitchBB);
-  bool handleBitTestsSwitchCase(CaseRec& CR,
-                                CaseRecVector& WorkList,
-                                const Value* SV,
-                                MachineBasicBlock* Default,
-                                MachineBasicBlock *SwitchBB);
+  void visitCleanupRet(const CleanupReturnInst &I);
+  void visitCatchEndPad(const CatchEndPadInst &I);
+  void visitCatchRet(const CatchReturnInst &I);
+  void visitCatchPad(const CatchPadInst &I);
+  void visitTerminatePad(const TerminatePadInst &TPI);
+  void visitCleanupPad(const CleanupPadInst &CPI);
 
   uint32_t getEdgeWeight(const MachineBasicBlock *Src,
                          const MachineBasicBlock *Dst) const;
@@ -713,8 +762,6 @@ class SelectionDAGBuilder {
   void visitJumpTable(JumpTable &JT);
   void visitJumpTableHeader(JumpTable &JT, JumpTableHeader &JTH,
                             MachineBasicBlock *SwitchBB);
-  unsigned visitLandingPadClauseBB(GlobalValue *ClauseGV,
-                                   MachineBasicBlock *LPadMBB);
 
 private:
   // These all get lowered before this pass.
@@ -774,6 +821,8 @@ class SelectionDAGBuilder {
   void visitStore(const StoreInst &I);
   void visitMaskedLoad(const CallInst &I);
   void visitMaskedStore(const CallInst &I);
+  void visitMaskedGather(const CallInst &I);
+  void visitMaskedScatter(const CallInst &I);
   void visitAtomicCmpXchg(const AtomicCmpXchgInst &I);
   void visitAtomicRMW(const AtomicRMWInst &I);
   void visitFence(const FenceInst &I);
@@ -822,8 +871,8 @@ class SelectionDAGBuilder {
   /// EmitFuncArgumentDbgValue - If V is an function argument then create
   /// corresponding DBG_VALUE machine instruction for it now. At the end of
   /// instruction selection, they will be inserted to the entry BB.
-  bool EmitFuncArgumentDbgValue(const Value *V, MDLocalVariable *Variable,
-                                MDExpression *Expr, MDLocation *DL,
+  bool EmitFuncArgumentDbgValue(const Value *V, DILocalVariable *Variable,
+                                DIExpression *Expr, DILocation *DL,
                                 int64_t Offset, bool IsIndirect,
                                 const SDValue &N);
 
@@ -835,6 +884,80 @@ class SelectionDAGBuilder {
   void updateDAGForMaybeTailCall(SDValue MaybeTC);
 };
 
+/// RegsForValue - This struct represents the registers (physical or virtual)
+/// that a particular set of values is assigned, and the type information about
+/// the value. The most common situation is to represent one value at a time,
+/// but struct or array values are handled element-wise as multiple values.  The
+/// splitting of aggregates is performed recursively, so that we never have
+/// aggregate-typed registers. The values at this point do not necessarily have
+/// legal types, so each value may require one or more registers of some legal
+/// type.
+///
+struct RegsForValue {
+  /// ValueVTs - The value types of the values, which may not be legal, and
+  /// may need be promoted or synthesized from one or more registers.
+  ///
+  SmallVector<EVT, 4> ValueVTs;
+
+  /// RegVTs - The value types of the registers. This is the same size as
+  /// ValueVTs and it records, for each value, what the type of the assigned
+  /// register or registers are. (Individual values are never synthesized
+  /// from more than one type of register.)
+  ///
+  /// With virtual registers, the contents of RegVTs is redundant with TLI's
+  /// getRegisterType member function, however when with physical registers
+  /// it is necessary to have a separate record of the types.
+  ///
+  SmallVector<MVT, 4> RegVTs;
+
+  /// Regs - This list holds the registers assigned to the values.
+  /// Each legal or promoted value requires one register, and each
+  /// expanded value requires multiple registers.
+  ///
+  SmallVector<unsigned, 4> Regs;
+
+  RegsForValue();
+
+  RegsForValue(const SmallVector<unsigned, 4> &regs, MVT regvt, EVT valuevt);
+
+  RegsForValue(LLVMContext &Context, const TargetLowering &TLI,
+               const DataLayout &DL, unsigned Reg, Type *Ty);
+
+  /// append - Add the specified values to this one.
+  void append(const RegsForValue &RHS) {
+    ValueVTs.append(RHS.ValueVTs.begin(), RHS.ValueVTs.end());
+    RegVTs.append(RHS.RegVTs.begin(), RHS.RegVTs.end());
+    Regs.append(RHS.Regs.begin(), RHS.Regs.end());
+  }
+
+  /// getCopyFromRegs - Emit a series of CopyFromReg nodes that copies from
+  /// this value and returns the result as a ValueVTs value.  This uses
+  /// Chain/Flag as the input and updates them for the output Chain/Flag.
+  /// If the Flag pointer is NULL, no flag is used.
+  SDValue getCopyFromRegs(SelectionDAG &DAG, FunctionLoweringInfo &FuncInfo,
+                          SDLoc dl,
+                          SDValue &Chain, SDValue *Flag,
+                          const Value *V = nullptr) const;
+
+  /// getCopyToRegs - Emit a series of CopyToReg nodes that copies the specified
+  /// value into the registers specified by this object.  This uses Chain/Flag
+  /// as the input and updates them for the output Chain/Flag.  If the Flag
+  /// pointer is nullptr, no flag is used.  If V is not nullptr, then it is used
+  /// in printing better diagnostic messages on error.
+  void
+  getCopyToRegs(SDValue Val, SelectionDAG &DAG, SDLoc dl, SDValue &Chain,
+                SDValue *Flag, const Value *V = nullptr,
+                ISD::NodeType PreferredExtendType = ISD::ANY_EXTEND) const;
+
+  /// AddInlineAsmOperands - Add this value to the specified inlineasm node
+  /// operand list.  This adds the code marker, matching input operand index
+  /// (if applicable), and includes the number of values added into it.
+  void AddInlineAsmOperands(unsigned Kind,
+                            bool HasMatching, unsigned MatchingIdx, SDLoc dl,
+                            SelectionDAG &DAG,
+                            std::vector<SDValue> &Ops) const;
+};
+
 } // end namespace llvm
 
 #endif
diff --git a/lib/CodeGen/SelectionDAG/SelectionDAGDumper.cpp b/lib/CodeGen/SelectionDAG/SelectionDAGDumper.cpp
index 636c0a741a7..883cc2573a6 100644
--- a/lib/CodeGen/SelectionDAG/SelectionDAGDumper.cpp
+++ b/lib/CodeGen/SelectionDAG/SelectionDAGDumper.cpp
@@ -95,13 +95,14 @@ std::string SDNode::getOperationName(const SelectionDAG *G) const {
   case ISD::GLOBAL_OFFSET_TABLE:        return "GLOBAL_OFFSET_TABLE";
   case ISD::RETURNADDR:                 return "RETURNADDR";
   case ISD::FRAMEADDR:                  return "FRAMEADDR";
-  case ISD::FRAME_ALLOC_RECOVER:        return "FRAME_ALLOC_RECOVER";
+  case ISD::LOCAL_RECOVER:        return "LOCAL_RECOVER";
   case ISD::READ_REGISTER:              return "READ_REGISTER";
   case ISD::WRITE_REGISTER:             return "WRITE_REGISTER";
   case ISD::FRAME_TO_ARGS_OFFSET:       return "FRAME_TO_ARGS_OFFSET";
   case ISD::EH_RETURN:                  return "EH_RETURN";
   case ISD::EH_SJLJ_SETJMP:             return "EH_SJLJ_SETJMP";
   case ISD::EH_SJLJ_LONGJMP:            return "EH_SJLJ_LONGJMP";
+  case ISD::EH_SJLJ_SETUP_DISPATCH:     return "EH_SJLJ_SETUP_DISPATCH";
   case ISD::ConstantPool:               return "ConstantPool";
   case ISD::TargetIndex:                return "TargetIndex";
   case ISD::ExternalSymbol:             return "ExternalSymbol";
@@ -130,6 +131,7 @@ std::string SDNode::getOperationName(const SelectionDAG *G) const {
   case ISD::TargetJumpTable:            return "TargetJumpTable";
   case ISD::TargetConstantPool:         return "TargetConstantPool";
   case ISD::TargetExternalSymbol:       return "TargetExternalSymbol";
+  case ISD::MCSymbol:                   return "MCSymbol";
   case ISD::TargetBlockAddress:         return "TargetBlockAddress";
 
   case ISD::CopyToReg:                  return "CopyToReg";
@@ -193,6 +195,10 @@ std::string SDNode::getOperationName(const SelectionDAG *G) const {
   case ISD::FCOPYSIGN:                  return "fcopysign";
   case ISD::FGETSIGN:                   return "fgetsign";
   case ISD::FPOW:                       return "fpow";
+  case ISD::SMIN:                       return "smin";
+  case ISD::SMAX:                       return "smax";
+  case ISD::UMIN:                       return "umin";
+  case ISD::UMAX:                       return "umax";
 
   case ISD::FPOWI:                      return "fpowi";
   case ISD::SETCC:                      return "setcc";
@@ -220,6 +226,8 @@ std::string SDNode::getOperationName(const SelectionDAG *G) const {
   case ISD::SHL_PARTS:                  return "shl_parts";
   case ISD::SRA_PARTS:                  return "sra_parts";
   case ISD::SRL_PARTS:                  return "srl_parts";
+  case ISD::UABSDIFF:                   return "uabsdiff";
+  case ISD::SABSDIFF:                   return "sabsdiff";
 
   // Conversion operators.
   case ISD::SIGN_EXTEND:                return "sign_extend";
@@ -273,6 +281,8 @@ std::string SDNode::getOperationName(const SelectionDAG *G) const {
   case ISD::STORE:                      return "store";
   case ISD::MLOAD:                      return "masked_load";
   case ISD::MSTORE:                     return "masked_store";
+  case ISD::MGATHER:                    return "masked_gather";
+  case ISD::MSCATTER:                   return "masked_scatter";
   case ISD::VAARG:                      return "vaarg";
   case ISD::VACOPY:                     return "vacopy";
   case ISD::VAEND:                      return "vaend";
@@ -286,6 +296,8 @@ std::string SDNode::getOperationName(const SelectionDAG *G) const {
   case ISD::DEBUGTRAP:                  return "debugtrap";
   case ISD::LIFETIME_START:             return "lifetime.start";
   case ISD::LIFETIME_END:               return "lifetime.end";
+  case ISD::GC_TRANSITION_START:        return "gc_transition.start";
+  case ISD::GC_TRANSITION_END:          return "gc_transition.end";
 
   // Bit manipulation
   case ISD::BSWAP:                      return "bswap";
@@ -523,7 +535,7 @@ void SDNode::print_details(raw_ostream &OS, const SelectionDAG *G) const {
   if (!G)
     return;
 
-  MDLocation *L = getDebugLoc();
+  DILocation *L = getDebugLoc();
   if (!L)
     return;
 
@@ -537,12 +549,12 @@ void SDNode::print_details(raw_ostream &OS, const SelectionDAG *G) const {
 }
 
 static void DumpNodes(const SDNode *N, unsigned indent, const SelectionDAG *G) {
-  for (unsigned i = 0, e = N->getNumOperands(); i != e; ++i)
-    if (N->getOperand(i).getNode()->hasOneUse())
-      DumpNodes(N->getOperand(i).getNode(), indent+2, G);
+  for (const SDValue &Op : N->op_values())
+    if (Op.getNode()->hasOneUse())
+      DumpNodes(Op.getNode(), indent+2, G);
     else
       dbgs() << "\n" << std::string(indent+2, ' ')
-             << (void*)N->getOperand(i).getNode() << ": <multiple use>";
+             << (void*)Op.getNode() << ": <multiple use>";
 
   dbgs() << '\n';
   dbgs().indent(indent);
@@ -599,10 +611,8 @@ static void DumpNodesr(raw_ostream &OS, const SDNode *N, unsigned indent,
   OS << "\n";
 
   // Dump children that have grandchildren on their own line(s).
-  for (unsigned i = 0, e = N->getNumOperands(); i != e; ++i) {
-    const SDNode *child = N->getOperand(i).getNode();
-    DumpNodesr(OS, child, indent+2, G, once);
-  }
+  for (const SDValue &Op : N->op_values())
+    DumpNodesr(OS, Op.getNode(), indent+2, G, once);
 }
 
 void SDNode::dumpr() const {
@@ -628,12 +638,12 @@ static void printrWithDepthHelper(raw_ostream &OS, const SDNode *N,
   if (depth < 1)
     return;
 
-  for (unsigned i = 0, e = N->getNumOperands(); i != e; ++i) {
+  for (const SDValue &Op : N->op_values()) {
     // Don't follow chain operands.
-    if (N->getOperand(i).getValueType() == MVT::Other)
+    if (Op.getValueType() == MVT::Other)
       continue;
     OS << '\n';
-    printrWithDepthHelper(OS, N->getOperand(i).getNode(), G, depth-1, indent+2);
+    printrWithDepthHelper(OS, Op.getNode(), G, depth-1, indent+2);
   }
 }
 
diff --git a/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp b/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp
index 1e116dddafa..6b5fcb095e2 100644
--- a/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp
+++ b/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp
@@ -293,6 +293,11 @@ namespace llvm {
     const TargetLowering *TLI = IS->TLI;
     const TargetSubtargetInfo &ST = IS->MF->getSubtarget();
 
+    // Try first to see if the Target has its own way of selecting a scheduler
+    if (auto *SchedulerCtor = ST.getDAGScheduler(OptLevel)) {
+      return SchedulerCtor(IS, OptLevel);
+    }
+
     if (OptLevel == CodeGenOpt::None ||
         (ST.enableMachineScheduler() && ST.enableMachineSchedDefaultSched()) ||
         TLI->getSchedulingPreference() == Sched::Source)
@@ -351,7 +356,8 @@ SelectionDAGISel::SelectionDAGISel(TargetMachine &tm,
   DAGSize(0) {
     initializeGCModuleInfoPass(*PassRegistry::getPassRegistry());
     initializeAliasAnalysisAnalysisGroup(*PassRegistry::getPassRegistry());
-    initializeBranchProbabilityInfoPass(*PassRegistry::getPassRegistry());
+    initializeBranchProbabilityInfoWrapperPassPass(
+        *PassRegistry::getPassRegistry());
     initializeTargetLibraryInfoWrapperPassPass(
         *PassRegistry::getPassRegistry());
   }
@@ -369,7 +375,7 @@ void SelectionDAGISel::getAnalysisUsage(AnalysisUsage &AU) const {
   AU.addPreserved<GCModuleInfo>();
   AU.addRequired<TargetLibraryInfoWrapperPass>();
   if (UseMBPI && OptLevel != CodeGenOpt::None)
-    AU.addRequired<BranchProbabilityInfo>();
+    AU.addRequired<BranchProbabilityInfoWrapperPass>();
   MachineFunctionPass::getAnalysisUsage(AU);
 }
 
@@ -380,7 +386,7 @@ void SelectionDAGISel::getAnalysisUsage(AnalysisUsage &AU) const {
 ///
 /// This is required for correctness, so it must be done at -O0.
 ///
-static void SplitCriticalSideEffectEdges(Function &Fn, AliasAnalysis *AA) {
+static void SplitCriticalSideEffectEdges(Function &Fn) {
   // Loop for blocks with phi nodes.
   for (Function::iterator BB = Fn.begin(), E = Fn.end(); BB != E; ++BB) {
     PHINode *PN = dyn_cast<PHINode>(BB->begin());
@@ -406,7 +412,7 @@ static void SplitCriticalSideEffectEdges(Function &Fn, AliasAnalysis *AA) {
         // Okay, we have to split this edge.
         SplitCriticalEdge(
             Pred->getTerminator(), GetSuccessorNumber(Pred, BB),
-            CriticalEdgeSplittingOptions(AA).setMergeIdenticalEdges());
+            CriticalEdgeSplittingOptions().setMergeIdenticalEdges());
         goto ReprocessBlock;
       }
   }
@@ -443,13 +449,13 @@ bool SelectionDAGISel::runOnMachineFunction(MachineFunction &mf) {
 
   DEBUG(dbgs() << "\n\n\n=== " << Fn.getName() << "\n");
 
-  SplitCriticalSideEffectEdges(const_cast<Function&>(Fn), AA);
+  SplitCriticalSideEffectEdges(const_cast<Function &>(Fn));
 
   CurDAG->init(*MF);
   FuncInfo->set(Fn, *MF, CurDAG);
 
   if (UseMBPI && OptLevel != CodeGenOpt::None)
-    FuncInfo->BPI = &getAnalysis<BranchProbabilityInfo>();
+    FuncInfo->BPI = &getAnalysis<BranchProbabilityInfoWrapperPass>().getBPI();
   else
     FuncInfo->BPI = nullptr;
 
@@ -504,7 +510,7 @@ bool SelectionDAGISel::runOnMachineFunction(MachineFunction &mf) {
       DebugLoc DL = MI->getDebugLoc();
       bool IsIndirect = MI->isIndirectDebugValue();
       unsigned Offset = IsIndirect ? MI->getOperand(1).getImm() : 0;
-      assert(cast<MDLocalVariable>(Variable)->isValidLocationForIntrinsic(DL) &&
+      assert(cast<DILocalVariable>(Variable)->isValidLocationForIntrinsic(DL) &&
              "Expected inlined-at fields to agree");
       // Def is never a terminator here, so it is ok to increment InsertPos.
       BuildMI(*EntryMBB, ++InsertPos, DL, TII->get(TargetOpcode::DBG_VALUE),
@@ -578,6 +584,13 @@ bool SelectionDAGISel::runOnMachineFunction(MachineFunction &mf) {
         TargetRegisterInfo::isVirtualRegister(To))
       MRI.constrainRegClass(To, MRI.getRegClass(From));
     // Replace it.
+
+
+    // Replacing one register with another won't touch the kill flags.
+    // We need to conservatively clear the kill flags as a kill on the old
+    // register might dominate existing uses of the new register.
+    if (!MRI.use_empty(To))
+      MRI.clearKillFlags(From);
     MRI.replaceRegWith(From, To);
   }
 
@@ -630,9 +643,9 @@ void SelectionDAGISel::ComputeLiveOutVRegInfo() {
       continue;
 
     // Otherwise, add all chain operands to the worklist.
-    for (unsigned i = 0, e = N->getNumOperands(); i != e; ++i)
-      if (N->getOperand(i).getValueType() == MVT::Other)
-        Worklist.push_back(N->getOperand(i).getNode());
+    for (const SDValue &Op : N->op_values())
+      if (Op.getValueType() == MVT::Other)
+        Worklist.push_back(Op.getNode());
 
     // If this is a CopyToReg with a vreg dest, process it.
     if (N->getOpcode() != ISD::CopyToReg)
@@ -911,10 +924,11 @@ void SelectionDAGISel::DoInstructionSelection() {
 
 /// PrepareEHLandingPad - Emit an EH_LABEL, set up live-in registers, and
 /// do other setup for EH landing-pad blocks.
-void SelectionDAGISel::PrepareEHLandingPad() {
+bool SelectionDAGISel::PrepareEHLandingPad() {
   MachineBasicBlock *MBB = FuncInfo->MBB;
 
-  const TargetRegisterClass *PtrRC = TLI->getRegClassFor(TLI->getPointerTy());
+  const TargetRegisterClass *PtrRC =
+      TLI->getRegClassFor(TLI->getPointerTy(CurDAG->getDataLayout()));
 
   // Add a label to mark the beginning of the landing pad.  Deletion of the
   // landing pad can thus be detected via the MachineModuleInfo.
@@ -931,18 +945,20 @@ void SelectionDAGISel::PrepareEHLandingPad() {
   // pad into several BBs.
   const BasicBlock *LLVMBB = MBB->getBasicBlock();
   const LandingPadInst *LPadInst = LLVMBB->getLandingPadInst();
-  MF->getMMI().addPersonality(
-      MBB, cast<Function>(LPadInst->getPersonalityFn()->stripPointerCasts()));
+  MF->getMMI().addPersonality(MBB, cast<Function>(LPadInst->getParent()
+                                                      ->getParent()
+                                                      ->getPersonalityFn()
+                                                      ->stripPointerCasts()));
   EHPersonality Personality = MF->getMMI().getPersonalityType();
 
   if (isMSVCEHPersonality(Personality)) {
     SmallVector<MachineBasicBlock *, 4> ClauseBBs;
-    const IntrinsicInst *Actions =
+    const IntrinsicInst *ActionsCall =
         dyn_cast<IntrinsicInst>(LLVMBB->getFirstInsertionPt());
     // Get all invoke BBs that unwind to this landingpad.
     SmallVector<MachineBasicBlock *, 4> InvokeBBs(MBB->pred_begin(),
                                                   MBB->pred_end());
-    if (Actions && Actions->getIntrinsicID() == Intrinsic::eh_actions) {
+    if (ActionsCall && ActionsCall->getIntrinsicID() == Intrinsic::eh_actions) {
       // If this is a call to llvm.eh.actions followed by indirectbr, then we've
       // run WinEHPrepare, and we should remove this block from the machine CFG.
       // Mark the targets of the indirectbr as landingpads instead.
@@ -951,55 +967,9 @@ void SelectionDAGISel::PrepareEHLandingPad() {
         // Add the edge from the invoke to the clause.
         for (MachineBasicBlock *InvokeBB : InvokeBBs)
           InvokeBB->addSuccessor(ClauseBB);
-      }
-    } else {
-      // Otherwise, we haven't done the preparation, and we need to invent some
-      // clause basic blocks that branch into the landingpad.
-      // FIXME: Remove this code once SEH preparation works.
-
-      // Make virtual registers and a series of labels that fill in values for
-      // the clauses.
-      auto &RI = MF->getRegInfo();
-      FuncInfo->ExceptionSelectorVirtReg = RI.createVirtualRegister(PtrRC);
-
-      // Emit separate machine basic blocks with separate labels for each clause
-      // before the main landing pad block.
-      MachineInstrBuilder SelectorPHI = BuildMI(
-          *MBB, MBB->begin(), SDB->getCurDebugLoc(),
-          TII->get(TargetOpcode::PHI), FuncInfo->ExceptionSelectorVirtReg);
-      for (unsigned I = 0, E = LPadInst->getNumClauses(); I != E; ++I) {
-        // Skip filter clauses, we can't implement them.
-        if (LPadInst->isFilter(I))
-          continue;
-
-        MachineBasicBlock *ClauseBB = MF->CreateMachineBasicBlock(LLVMBB);
-        MF->insert(MBB, ClauseBB);
-
-        // Add the edge from the invoke to the clause.
-        for (MachineBasicBlock *InvokeBB : InvokeBBs)
-          InvokeBB->addSuccessor(ClauseBB);
 
         // Mark the clause as a landing pad or MI passes will delete it.
         ClauseBB->setIsLandingPad();
-
-        GlobalValue *ClauseGV = ExtractTypeInfo(LPadInst->getClause(I));
-
-        // Start the BB with a label.
-        MCSymbol *ClauseLabel = MF->getMMI().addClauseForLandingPad(MBB);
-        BuildMI(*ClauseBB, ClauseBB->begin(), SDB->getCurDebugLoc(), II)
-            .addSym(ClauseLabel);
-
-        // Construct a simple BB that defines a register with the typeid
-        // constant.
-        FuncInfo->MBB = ClauseBB;
-        FuncInfo->InsertPt = ClauseBB->end();
-        unsigned VReg = SDB->visitLandingPadClauseBB(ClauseGV, MBB);
-        CurDAG->setRoot(SDB->getRoot());
-        SDB->clear();
-        CodeGenAndEmitDAG();
-
-        // Add the typeid virtual register to the phi in the main landing pad.
-        SelectorPHI.addReg(VReg).addMBB(ClauseBB);
       }
     }
 
@@ -1007,17 +977,8 @@ void SelectionDAGISel::PrepareEHLandingPad() {
     for (MachineBasicBlock *InvokeBB : InvokeBBs)
       InvokeBB->removeSuccessor(MBB);
 
-    // Restore FuncInfo back to its previous state and select the main landing
-    // pad block.
-    FuncInfo->MBB = MBB;
-    FuncInfo->InsertPt = MBB->end();
-
-    // Transfer EH state number assigned to the IR block to the MBB.
-    if (Personality == EHPersonality::MSVC_CXX) {
-      WinEHFuncInfo &FI = MF->getMMI().getWinEHFuncInfo(MF->getFunction());
-      MF->getMMI().addWinEHState(MBB, FI.LandingPadStateMap[LPadInst]);
-    }
-    return;
+    // Don't select instructions for the landingpad.
+    return false;
   }
 
   // Mark exception register as live in.
@@ -1027,6 +988,8 @@ void SelectionDAGISel::PrepareEHLandingPad() {
   // Mark exception selector register as live in.
   if (unsigned Reg = TLI->getExceptionSelectorRegister())
     FuncInfo->ExceptionSelectorVirtReg = MBB->addLiveIn(Reg, PtrRC);
+
+  return true;
 }
 
 /// isFoldedOrDeadInstruction - Return true if the specified instruction is
@@ -1197,7 +1160,8 @@ void SelectionDAGISel::SelectAllBasicBlocks(const Function &Fn) {
     FuncInfo->ExceptionPointerVirtReg = 0;
     FuncInfo->ExceptionSelectorVirtReg = 0;
     if (LLVMBB->isLandingPad())
-      PrepareEHLandingPad();
+      if (!PrepareEHLandingPad())
+        continue;
 
     // Before doing SelectionDAG ISel, see if FastISel has been requested.
     if (FastIS) {
@@ -1459,21 +1423,15 @@ SelectionDAGISel::FinishBasicBlock() {
                  << FuncInfo->PHINodesToUpdate[i].first
                  << ", " << FuncInfo->PHINodesToUpdate[i].second << ")\n");
 
-  const bool MustUpdatePHINodes = SDB->SwitchCases.empty() &&
-                                  SDB->JTCases.empty() &&
-                                  SDB->BitTestCases.empty();
-
   // Next, now that we know what the last MBB the LLVM BB expanded is, update
   // PHI nodes in successors.
-  if (MustUpdatePHINodes) {
-    for (unsigned i = 0, e = FuncInfo->PHINodesToUpdate.size(); i != e; ++i) {
-      MachineInstrBuilder PHI(*MF, FuncInfo->PHINodesToUpdate[i].first);
-      assert(PHI->isPHI() &&
-             "This is not a machine PHI node that we are updating!");
-      if (!FuncInfo->MBB->isSuccessor(PHI->getParent()))
-        continue;
-      PHI.addReg(FuncInfo->PHINodesToUpdate[i].second).addMBB(FuncInfo->MBB);
-    }
+  for (unsigned i = 0, e = FuncInfo->PHINodesToUpdate.size(); i != e; ++i) {
+    MachineInstrBuilder PHI(*MF, FuncInfo->PHINodesToUpdate[i].first);
+    assert(PHI->isPHI() &&
+           "This is not a machine PHI node that we are updating!");
+    if (!FuncInfo->MBB->isSuccessor(PHI->getParent()))
+      continue;
+    PHI.addReg(FuncInfo->PHINodesToUpdate[i].second).addMBB(FuncInfo->MBB);
   }
 
   // Handle stack protector.
@@ -1518,10 +1476,6 @@ SelectionDAGISel::FinishBasicBlock() {
     SDB->SPDescriptor.resetPerBBState();
   }
 
-  // If we updated PHI Nodes, return early.
-  if (MustUpdatePHINodes)
-    return;
-
   for (unsigned i = 0, e = SDB->BitTestCases.size(); i != e; ++i) {
     // Lower header first, if it wasn't already lowered
     if (!SDB->BitTestCases[i].Emitted) {
@@ -1635,16 +1589,6 @@ SelectionDAGISel::FinishBasicBlock() {
   }
   SDB->JTCases.clear();
 
-  // If the switch block involved a branch to one of the actual successors, we
-  // need to update PHI nodes in that block.
-  for (unsigned i = 0, e = FuncInfo->PHINodesToUpdate.size(); i != e; ++i) {
-    MachineInstrBuilder PHI(*MF, FuncInfo->PHINodesToUpdate[i].first);
-    assert(PHI->isPHI() &&
-           "This is not a machine PHI node that we are updating!");
-    if (FuncInfo->MBB->isSuccessor(PHI->getParent()))
-      PHI.addReg(FuncInfo->PHINodesToUpdate[i].second).addMBB(FuncInfo->MBB);
-  }
-
   // If we generated any switch lowering information, build and codegen any
   // additional DAGs necessary.
   for (unsigned i = 0, e = SDB->SwitchCases.size(); i != e; ++i) {
@@ -1704,14 +1648,7 @@ SelectionDAGISel::FinishBasicBlock() {
 /// one preferred by the target.
 ///
 ScheduleDAGSDNodes *SelectionDAGISel::CreateScheduler() {
-  RegisterScheduler::FunctionPassCtor Ctor = RegisterScheduler::getDefault();
-
-  if (!Ctor) {
-    Ctor = ISHeuristic;
-    RegisterScheduler::setDefault(Ctor);
-  }
-
-  return Ctor(this, OptLevel);
+  return ISHeuristic(this, OptLevel);
 }
 
 //===----------------------------------------------------------------------===//
@@ -1782,11 +1719,10 @@ bool SelectionDAGISel::CheckOrMask(SDValue LHS, ConstantSDNode *RHS,
   return false;
 }
 
-
 /// SelectInlineAsmMemoryOperands - Calls to this are automatically generated
 /// by tblgen.  Others should not call it.
 void SelectionDAGISel::
-SelectInlineAsmMemoryOperands(std::vector<SDValue> &Ops) {
+SelectInlineAsmMemoryOperands(std::vector<SDValue> &Ops, SDLoc DL) {
   std::vector<SDValue> InOps;
   std::swap(InOps, Ops);
 
@@ -1832,7 +1768,7 @@ SelectInlineAsmMemoryOperands(std::vector<SDValue> &Ops) {
       // Add this to the output node.
       unsigned NewFlags =
         InlineAsm::getFlagWord(InlineAsm::Kind_Mem, SelOps.size());
-      Ops.push_back(CurDAG->getTargetConstant(NewFlags, MVT::i32));
+      Ops.push_back(CurDAG->getTargetConstant(NewFlags, DL, MVT::i32));
       Ops.insert(Ops.end(), SelOps.begin(), SelOps.end());
       i += 2;
     }
@@ -1878,12 +1814,12 @@ static bool findNonImmUse(SDNode *Use, SDNode* Def, SDNode *ImmedUse,
   if (!Visited.insert(Use).second)
     return false;
 
-  for (unsigned i = 0, e = Use->getNumOperands(); i != e; ++i) {
+  for (const SDValue &Op : Use->op_values()) {
     // Ignore chain uses, they are validated by HandleMergeInputChains.
-    if (Use->getOperand(i).getValueType() == MVT::Other && IgnoreChains)
+    if (Op.getValueType() == MVT::Other && IgnoreChains)
       continue;
 
-    SDNode *N = Use->getOperand(i).getNode();
+    SDNode *N = Op.getNode();
     if (N == Def) {
       if (Use == ImmedUse || Use == Root)
         continue;  // We are not looking for immediate use.
@@ -1978,11 +1914,13 @@ bool SelectionDAGISel::IsLegalToFold(SDValue N, SDNode *U, SDNode *Root,
 }
 
 SDNode *SelectionDAGISel::Select_INLINEASM(SDNode *N) {
+  SDLoc DL(N);
+
   std::vector<SDValue> Ops(N->op_begin(), N->op_end());
-  SelectInlineAsmMemoryOperands(Ops);
+  SelectInlineAsmMemoryOperands(Ops, DL);
 
   const EVT VTs[] = {MVT::Other, MVT::Glue};
-  SDValue New = CurDAG->getNode(ISD::INLINEASM, SDLoc(N), VTs, Ops);
+  SDValue New = CurDAG->getNode(ISD::INLINEASM, DL, VTs, Ops);
   New->setNodeId(-1);
   return New.getNode();
 }
@@ -1990,12 +1928,13 @@ SDNode *SelectionDAGISel::Select_INLINEASM(SDNode *N) {
 SDNode
 *SelectionDAGISel::Select_READ_REGISTER(SDNode *Op) {
   SDLoc dl(Op);
-  MDNodeSDNode *MD = dyn_cast<MDNodeSDNode>(Op->getOperand(0));
+  MDNodeSDNode *MD = dyn_cast<MDNodeSDNode>(Op->getOperand(1));
   const MDString *RegStr = dyn_cast<MDString>(MD->getMD()->getOperand(0));
   unsigned Reg =
-      TLI->getRegisterByName(RegStr->getString().data(), Op->getValueType(0));
+      TLI->getRegisterByName(RegStr->getString().data(), Op->getValueType(0),
+                             *CurDAG);
   SDValue New = CurDAG->getCopyFromReg(
-                        CurDAG->getEntryNode(), dl, Reg, Op->getValueType(0));
+                        Op->getOperand(0), dl, Reg, Op->getValueType(0));
   New->setNodeId(-1);
   return New.getNode();
 }
@@ -2006,9 +1945,10 @@ SDNode
   MDNodeSDNode *MD = dyn_cast<MDNodeSDNode>(Op->getOperand(1));
   const MDString *RegStr = dyn_cast<MDString>(MD->getMD()->getOperand(0));
   unsigned Reg = TLI->getRegisterByName(RegStr->getString().data(),
-                                        Op->getOperand(2).getValueType());
+                                        Op->getOperand(2).getValueType(),
+                                        *CurDAG);
   SDValue New = CurDAG->getCopyToReg(
-                        CurDAG->getEntryNode(), dl, Reg, Op->getOperand(2));
+                        Op->getOperand(0), dl, Reg, Op->getOperand(2));
   New->setNodeId(-1);
   return New.getNode();
 }
@@ -2274,10 +2214,10 @@ HandleMergeInputChains(SmallVectorImpl<SDNode*> &ChainNodesMatched,
 
     // If we have a token factor, we want to add all inputs of the token factor
     // that are not part of the pattern we're matching.
-    for (unsigned op = 0, e = N->getNumOperands(); op != e; ++op) {
+    for (const SDValue &Op : N->op_values()) {
       if (!std::count(ChainNodesMatched.begin(), ChainNodesMatched.end(),
-                      N->getOperand(op).getNode()))
-        InputChains.push_back(N->getOperand(op));
+                      Op.getNode()))
+        InputChains.push_back(Op);
     }
   }
 
@@ -2391,21 +2331,23 @@ CheckOpcode(const unsigned char *MatcherTable, unsigned &MatcherIndex,
 }
 
 LLVM_ATTRIBUTE_ALWAYS_INLINE static bool
-CheckType(const unsigned char *MatcherTable, unsigned &MatcherIndex,
-          SDValue N, const TargetLowering *TLI) {
+CheckType(const unsigned char *MatcherTable, unsigned &MatcherIndex, SDValue N,
+          const TargetLowering *TLI, const DataLayout &DL) {
   MVT::SimpleValueType VT = (MVT::SimpleValueType)MatcherTable[MatcherIndex++];
   if (N.getValueType() == VT) return true;
 
   // Handle the case when VT is iPTR.
-  return VT == MVT::iPTR && N.getValueType() == TLI->getPointerTy();
+  return VT == MVT::iPTR && N.getValueType() == TLI->getPointerTy(DL);
 }
 
 LLVM_ATTRIBUTE_ALWAYS_INLINE static bool
 CheckChildType(const unsigned char *MatcherTable, unsigned &MatcherIndex,
-               SDValue N, const TargetLowering *TLI, unsigned ChildNo) {
+               SDValue N, const TargetLowering *TLI, const DataLayout &DL,
+               unsigned ChildNo) {
   if (ChildNo >= N.getNumOperands())
     return false;  // Match fails if out of range child #.
-  return ::CheckType(MatcherTable, MatcherIndex, N.getOperand(ChildNo), TLI);
+  return ::CheckType(MatcherTable, MatcherIndex, N.getOperand(ChildNo), TLI,
+                     DL);
 }
 
 LLVM_ATTRIBUTE_ALWAYS_INLINE static bool
@@ -2417,13 +2359,13 @@ CheckCondCode(const unsigned char *MatcherTable, unsigned &MatcherIndex,
 
 LLVM_ATTRIBUTE_ALWAYS_INLINE static bool
 CheckValueType(const unsigned char *MatcherTable, unsigned &MatcherIndex,
-               SDValue N, const TargetLowering *TLI) {
+               SDValue N, const TargetLowering *TLI, const DataLayout &DL) {
   MVT::SimpleValueType VT = (MVT::SimpleValueType)MatcherTable[MatcherIndex++];
   if (cast<VTSDNode>(N)->getVT() == VT)
     return true;
 
   // Handle the case when VT is iPTR.
-  return VT == MVT::iPTR && cast<VTSDNode>(N)->getVT() == TLI->getPointerTy();
+  return VT == MVT::iPTR && cast<VTSDNode>(N)->getVT() == TLI->getPointerTy(DL);
 }
 
 LLVM_ATTRIBUTE_ALWAYS_INLINE static bool
@@ -2506,7 +2448,8 @@ static unsigned IsPredicateKnownToFail(const unsigned char *Table,
     Result = !::CheckOpcode(Table, Index, N.getNode());
     return Index;
   case SelectionDAGISel::OPC_CheckType:
-    Result = !::CheckType(Table, Index, N, SDISel.TLI);
+    Result = !::CheckType(Table, Index, N, SDISel.TLI,
+                          SDISel.CurDAG->getDataLayout());
     return Index;
   case SelectionDAGISel::OPC_CheckChild0Type:
   case SelectionDAGISel::OPC_CheckChild1Type:
@@ -2516,15 +2459,16 @@ static unsigned IsPredicateKnownToFail(const unsigned char *Table,
   case SelectionDAGISel::OPC_CheckChild5Type:
   case SelectionDAGISel::OPC_CheckChild6Type:
   case SelectionDAGISel::OPC_CheckChild7Type:
-    Result = !::CheckChildType(Table, Index, N, SDISel.TLI,
-                               Table[Index - 1] -
-                                   SelectionDAGISel::OPC_CheckChild0Type);
+    Result = !::CheckChildType(
+                 Table, Index, N, SDISel.TLI, SDISel.CurDAG->getDataLayout(),
+                 Table[Index - 1] - SelectionDAGISel::OPC_CheckChild0Type);
     return Index;
   case SelectionDAGISel::OPC_CheckCondCode:
     Result = !::CheckCondCode(Table, Index, N);
     return Index;
   case SelectionDAGISel::OPC_CheckValueType:
-    Result = !::CheckValueType(Table, Index, N, SDISel.TLI);
+    Result = !::CheckValueType(Table, Index, N, SDISel.TLI,
+                               SDISel.CurDAG->getDataLayout());
     return Index;
   case SelectionDAGISel::OPC_CheckInteger:
     Result = !::CheckInteger(Table, Index, N);
@@ -2624,6 +2568,7 @@ SelectCodeCommon(SDNode *NodeToMatch, const unsigned char *MatcherTable,
   case ISD::TargetConstantPool:
   case ISD::TargetFrameIndex:
   case ISD::TargetExternalSymbol:
+  case ISD::MCSymbol:
   case ISD::TargetBlockAddress:
   case ISD::TargetJumpTable:
   case ISD::TargetGlobalTLSAddress:
@@ -2877,7 +2822,8 @@ SelectCodeCommon(SDNode *NodeToMatch, const unsigned char *MatcherTable,
       continue;
 
     case OPC_CheckType:
-      if (!::CheckType(MatcherTable, MatcherIndex, N, TLI))
+      if (!::CheckType(MatcherTable, MatcherIndex, N, TLI,
+                       CurDAG->getDataLayout()))
         break;
       continue;
 
@@ -2925,7 +2871,7 @@ SelectCodeCommon(SDNode *NodeToMatch, const unsigned char *MatcherTable,
 
         MVT CaseVT = (MVT::SimpleValueType)MatcherTable[MatcherIndex++];
         if (CaseVT == MVT::iPTR)
-          CaseVT = TLI->getPointerTy();
+          CaseVT = TLI->getPointerTy(CurDAG->getDataLayout());
 
         // If the VT matches, then we will execute this case.
         if (CurNodeVT == CaseVT)
@@ -2948,14 +2894,16 @@ SelectCodeCommon(SDNode *NodeToMatch, const unsigned char *MatcherTable,
     case OPC_CheckChild4Type: case OPC_CheckChild5Type:
     case OPC_CheckChild6Type: case OPC_CheckChild7Type:
       if (!::CheckChildType(MatcherTable, MatcherIndex, N, TLI,
-                            Opcode-OPC_CheckChild0Type))
+                            CurDAG->getDataLayout(),
+                            Opcode - OPC_CheckChild0Type))
         break;
       continue;
     case OPC_CheckCondCode:
       if (!::CheckCondCode(MatcherTable, MatcherIndex, N)) break;
       continue;
     case OPC_CheckValueType:
-      if (!::CheckValueType(MatcherTable, MatcherIndex, N, TLI))
+      if (!::CheckValueType(MatcherTable, MatcherIndex, N, TLI,
+                            CurDAG->getDataLayout()))
         break;
       continue;
     case OPC_CheckInteger:
@@ -3004,7 +2952,8 @@ SelectCodeCommon(SDNode *NodeToMatch, const unsigned char *MatcherTable,
       if (Val & 128)
         Val = GetVBR(Val, MatcherTable, MatcherIndex);
       RecordedNodes.push_back(std::pair<SDValue, SDNode*>(
-                              CurDAG->getTargetConstant(Val, VT), nullptr));
+                              CurDAG->getTargetConstant(Val, SDLoc(NodeToMatch),
+                                                        VT), nullptr));
       continue;
     }
     case OPC_EmitRegister: {
@@ -3036,10 +2985,12 @@ SelectCodeCommon(SDNode *NodeToMatch, const unsigned char *MatcherTable,
 
       if (Imm->getOpcode() == ISD::Constant) {
         const ConstantInt *Val=cast<ConstantSDNode>(Imm)->getConstantIntValue();
-        Imm = CurDAG->getConstant(*Val, Imm.getValueType(), true);
+        Imm = CurDAG->getConstant(*Val, SDLoc(NodeToMatch), Imm.getValueType(),
+                                  true);
       } else if (Imm->getOpcode() == ISD::ConstantFP) {
         const ConstantFP *Val=cast<ConstantFPSDNode>(Imm)->getConstantFPValue();
-        Imm = CurDAG->getConstantFP(*Val, Imm.getValueType(), true);
+        Imm = CurDAG->getConstantFP(*Val, SDLoc(NodeToMatch),
+                                    Imm.getValueType(), true);
       }
 
       RecordedNodes.push_back(std::make_pair(Imm, RecordedNodes[RecNo].second));
@@ -3155,7 +3106,7 @@ SelectCodeCommon(SDNode *NodeToMatch, const unsigned char *MatcherTable,
         MVT::SimpleValueType VT =
           (MVT::SimpleValueType)MatcherTable[MatcherIndex++];
         if (VT == MVT::iPTR)
-          VT = TLI->getPointerTy().SimpleTy;
+          VT = TLI->getPointerTy(CurDAG->getDataLayout()).SimpleTy;
         VTs.push_back(VT);
       }
 
diff --git a/lib/CodeGen/SelectionDAG/StatepointLowering.cpp b/lib/CodeGen/SelectionDAG/StatepointLowering.cpp
index a9ffa728228..c62428d308e 100644
--- a/lib/CodeGen/SelectionDAG/StatepointLowering.cpp
+++ b/lib/CodeGen/SelectionDAG/StatepointLowering.cpp
@@ -38,13 +38,19 @@ STATISTIC(NumOfStatepoints, "Number of statepoint nodes encountered");
 STATISTIC(StatepointMaxSlotsRequired,
           "Maximum number of stack slots required for a singe statepoint");
 
-void
-StatepointLoweringState::startNewStatepoint(SelectionDAGBuilder &Builder) {
+static void pushStackMapConstant(SmallVectorImpl<SDValue>& Ops,
+                                 SelectionDAGBuilder &Builder, uint64_t Value) {
+  SDLoc L = Builder.getCurSDLoc();
+  Ops.push_back(Builder.DAG.getTargetConstant(StackMaps::ConstantOp, L,
+                                              MVT::i64));
+  Ops.push_back(Builder.DAG.getTargetConstant(Value, L, MVT::i64));
+}
+
+void StatepointLoweringState::startNewStatepoint(SelectionDAGBuilder &Builder) {
   // Consistency check
   assert(PendingGCRelocateCalls.empty() &&
          "Trying to visit statepoint before finished processing previous one");
   Locations.clear();
-  RelocLocations.clear();
   NextSlotToAllocate = 0;
   // Need to resize this on each safepoint - we need the two to stay in
   // sync and the clear patterns of a SelectionDAGBuilder have no relation
@@ -54,9 +60,9 @@ StatepointLoweringState::startNewStatepoint(SelectionDAGBuilder &Builder) {
     AllocatedStackSlots[i] = false;
   }
 }
+
 void StatepointLoweringState::clear() {
   Locations.clear();
-  RelocLocations.clear();
   AllocatedStackSlots.clear();
   assert(PendingGCRelocateCalls.empty() &&
          "cleared before statepoint sequence completed");
@@ -107,84 +113,137 @@ StatepointLoweringState::allocateStackSlot(EVT ValueType,
   llvm_unreachable("infinite loop?");
 }
 
+/// Utility function for reservePreviousStackSlotForValue. Tries to find
+/// stack slot index to which we have spilled value for previous statepoints.
+/// LookUpDepth specifies maximum DFS depth this function is allowed to look.
+static Optional<int> findPreviousSpillSlot(const Value *Val,
+                                           SelectionDAGBuilder &Builder,
+                                           int LookUpDepth) {
+  // Can not look any futher - give up now
+  if (LookUpDepth <= 0)
+    return Optional<int>();
+
+  // Spill location is known for gc relocates
+  if (isGCRelocate(Val)) {
+    GCRelocateOperands RelocOps(cast<Instruction>(Val));
+
+    FunctionLoweringInfo::StatepointSpilledValueMapTy &SpillMap =
+        Builder.FuncInfo.StatepointRelocatedValues[RelocOps.getStatepoint()];
+
+    auto It = SpillMap.find(RelocOps.getDerivedPtr());
+    if (It == SpillMap.end())
+      return Optional<int>();
+
+    return It->second;
+  }
+
+  // Look through bitcast instructions.
+  if (const BitCastInst *Cast = dyn_cast<BitCastInst>(Val)) {
+    return findPreviousSpillSlot(Cast->getOperand(0), Builder, LookUpDepth - 1);
+  }
+
+  // Look through phi nodes
+  // All incoming values should have same known stack slot, otherwise result
+  // is unknown.
+  if (const PHINode *Phi = dyn_cast<PHINode>(Val)) {
+    Optional<int> MergedResult = None;
+
+    for (auto &IncomingValue : Phi->incoming_values()) {
+      Optional<int> SpillSlot =
+          findPreviousSpillSlot(IncomingValue, Builder, LookUpDepth - 1);
+      if (!SpillSlot.hasValue())
+        return Optional<int>();
+
+      if (MergedResult.hasValue() && *MergedResult != *SpillSlot)
+        return Optional<int>();
+
+      MergedResult = SpillSlot;
+    }
+    return MergedResult;
+  }
+
+  // TODO: We can do better for PHI nodes. In cases like this:
+  //   ptr = phi(relocated_pointer, not_relocated_pointer)
+  //   statepoint(ptr)
+  // We will return that stack slot for ptr is unknown. And later we might
+  // assign different stack slots for ptr and relocated_pointer. This limits
+  // llvm's ability to remove redundant stores.
+  // Unfortunately it's hard to accomplish in current infrastructure.
+  // We use this function to eliminate spill store completely, while
+  // in example we still need to emit store, but instead of any location
+  // we need to use special "preferred" location.
+
+  // TODO: handle simple updates.  If a value is modified and the original
+  // value is no longer live, it would be nice to put the modified value in the
+  // same slot.  This allows folding of the memory accesses for some
+  // instructions types (like an increment).
+  //   statepoint (i)
+  //   i1 = i+1
+  //   statepoint (i1)
+  // However we need to be careful for cases like this:
+  //   statepoint(i)
+  //   i1 = i+1
+  //   statepoint(i, i1)
+  // Here we want to reserve spill slot for 'i', but not for 'i+1'. If we just
+  // put handling of simple modifications in this function like it's done
+  // for bitcasts we might end up reserving i's slot for 'i+1' because order in
+  // which we visit values is unspecified.
+
+  // Don't know any information about this instruction
+  return Optional<int>();
+}
+
 /// Try to find existing copies of the incoming values in stack slots used for
 /// statepoint spilling.  If we can find a spill slot for the incoming value,
 /// mark that slot as allocated, and reuse the same slot for this safepoint.
 /// This helps to avoid series of loads and stores that only serve to resuffle
 /// values on the stack between calls.
-static void reservePreviousStackSlotForValue(SDValue Incoming,
+static void reservePreviousStackSlotForValue(const Value *IncomingValue,
                                              SelectionDAGBuilder &Builder) {
 
+  SDValue Incoming = Builder.getValue(IncomingValue);
+
   if (isa<ConstantSDNode>(Incoming) || isa<FrameIndexSDNode>(Incoming)) {
     // We won't need to spill this, so no need to check for previously
     // allocated stack slots
     return;
   }
 
-  SDValue Loc = Builder.StatepointLowering.getLocation(Incoming);
-  if (Loc.getNode()) {
+  SDValue OldLocation = Builder.StatepointLowering.getLocation(Incoming);
+  if (OldLocation.getNode())
     // duplicates in input
     return;
-  }
-
-  // Search back for the load from a stack slot pattern to find the original
-  // slot we allocated for this value.  We could extend this to deal with
-  // simple modification patterns, but simple dealing with trivial load/store
-  // sequences helps a lot already.
-  if (LoadSDNode *Load = dyn_cast<LoadSDNode>(Incoming)) {
-    if (auto *FI = dyn_cast<FrameIndexSDNode>(Load->getBasePtr())) {
-      const int Index = FI->getIndex();
-      auto Itr = std::find(Builder.FuncInfo.StatepointStackSlots.begin(),
-                           Builder.FuncInfo.StatepointStackSlots.end(), Index);
-      if (Itr == Builder.FuncInfo.StatepointStackSlots.end()) {
-        // not one of the lowering stack slots, can't reuse!
-        // TODO: Actually, we probably could reuse the stack slot if the value
-        // hasn't changed at all, but we'd need to look for intervening writes
-        return;
-      } else {
-        // This is one of our dedicated lowering slots
-        const int Offset =
-            std::distance(Builder.FuncInfo.StatepointStackSlots.begin(), Itr);
-        if (Builder.StatepointLowering.isStackSlotAllocated(Offset)) {
-          // stack slot already assigned to someone else, can't use it!
-          // TODO: currently we reserve space for gc arguments after doing
-          // normal allocation for deopt arguments.  We should reserve for
-          // _all_ deopt and gc arguments, then start allocating.  This
-          // will prevent some moves being inserted when vm state changes,
-          // but gc state doesn't between two calls.
-          return;
-        }
-        // Reserve this stack slot
-        Builder.StatepointLowering.reserveStackSlot(Offset);
-      }
 
-      // Cache this slot so we find it when going through the normal
-      // assignment loop.
-      SDValue Loc =
-          Builder.DAG.getTargetFrameIndex(Index, Incoming.getValueType());
+  const int LookUpDepth = 6;
+  Optional<int> Index =
+      findPreviousSpillSlot(IncomingValue, Builder, LookUpDepth);
+  if (!Index.hasValue())
+    return;
 
-      Builder.StatepointLowering.setLocation(Incoming, Loc);
-    }
+  auto Itr = std::find(Builder.FuncInfo.StatepointStackSlots.begin(),
+                       Builder.FuncInfo.StatepointStackSlots.end(), *Index);
+  assert(Itr != Builder.FuncInfo.StatepointStackSlots.end() &&
+         "value spilled to the unknown stack slot");
+
+  // This is one of our dedicated lowering slots
+  const int Offset =
+      std::distance(Builder.FuncInfo.StatepointStackSlots.begin(), Itr);
+  if (Builder.StatepointLowering.isStackSlotAllocated(Offset)) {
+    // stack slot already assigned to someone else, can't use it!
+    // TODO: currently we reserve space for gc arguments after doing
+    // normal allocation for deopt arguments.  We should reserve for
+    // _all_ deopt and gc arguments, then start allocating.  This
+    // will prevent some moves being inserted when vm state changes,
+    // but gc state doesn't between two calls.
+    return;
   }
+  // Reserve this stack slot
+  Builder.StatepointLowering.reserveStackSlot(Offset);
 
-  // TODO: handle case where a reloaded value flows through a phi to
-  // another safepoint.  e.g.
-  // bb1:
-  //  a' = relocated...
-  // bb2: % pred: bb1, bb3, bb4, etc.
-  //  a_phi = phi(a', ...)
-  // statepoint ... a_phi
-  // NOTE: This will require reasoning about cross basic block values.  This is
-  // decidedly non trivial and this might not be the right place to do it.  We
-  // don't really have the information we need here...
-
-  // TODO: handle simple updates.  If a value is modified and the original
-  // value is no longer live, it would be nice to put the modified value in the
-  // same slot.  This allows folding of the memory accesses for some
-  // instructions types (like an increment).
-  // statepoint (i)
-  // i1 = i+1
-  // statepoint (i1)
+  // Cache this slot so we find it when going through the normal
+  // assignment loop.
+  SDValue Loc = Builder.DAG.getTargetFrameIndex(*Index, Incoming.getValueType());
+  Builder.StatepointLowering.setLocation(Incoming, Loc);
 }
 
 /// Remove any duplicate (as SDValues) from the derived pointer pairs.  This
@@ -223,33 +282,67 @@ static void removeDuplicatesGCPtrs(SmallVectorImpl<const Value *> &Bases,
 /// Extract call from statepoint, lower it and return pointer to the
 /// call node. Also update NodeMap so that getValue(statepoint) will
 /// reference lowered call result
-static SDNode *lowerCallFromStatepoint(ImmutableStatepoint StatepointSite,
-                                       MachineBasicBlock *LandingPad,
-                                       SelectionDAGBuilder &Builder) {
-
-  ImmutableCallSite CS(StatepointSite.getCallSite());
-
-  // Lower the actual call itself - This is a bit of a hack, but we want to
-  // avoid modifying the actual lowering code.  This is similiar in intent to
-  // the LowerCallOperands mechanism used by PATCHPOINT, but is structured
-  // differently.  Hopefully, this is slightly more robust w.r.t. calling
-  // convention, return values, and other function attributes.
-  Value *ActualCallee = const_cast<Value *>(StatepointSite.actualCallee());
-
-  std::vector<Value *> Args;
-  CallInst::const_op_iterator arg_begin = StatepointSite.call_args_begin();
-  CallInst::const_op_iterator arg_end = StatepointSite.call_args_end();
-  Args.insert(Args.end(), arg_begin, arg_end);
-  // TODO: remove the creation of a new instruction!  We should not be
-  // modifying the IR (even temporarily) at this point.
-  CallInst *Tmp = CallInst::Create(ActualCallee, Args);
-  Tmp->setTailCall(CS.isTailCall());
-  Tmp->setCallingConv(CS.getCallingConv());
-  Tmp->setAttributes(CS.getAttributes());
-  Builder.LowerCallTo(Tmp, Builder.getValue(ActualCallee), false, LandingPad);
-
-  // Handle the return value of the call iff any.
-  const bool HasDef = !Tmp->getType()->isVoidTy();
+static SDNode *
+lowerCallFromStatepoint(ImmutableStatepoint ISP, MachineBasicBlock *LandingPad,
+                        SelectionDAGBuilder &Builder,
+                        SmallVectorImpl<SDValue> &PendingExports) {
+
+  ImmutableCallSite CS(ISP.getCallSite());
+
+  SDValue ActualCallee;
+
+  if (ISP.getNumPatchBytes() > 0) {
+    // If we've been asked to emit a nop sequence instead of a call instruction
+    // for this statepoint then don't lower the call target, but use a constant
+    // `null` instead.  Not lowering the call target lets statepoint clients get
+    // away without providing a physical address for the symbolic call target at
+    // link time.
+
+    const auto &TLI = Builder.DAG.getTargetLoweringInfo();
+    const auto &DL = Builder.DAG.getDataLayout();
+
+    unsigned AS = ISP.getCalledValue()->getType()->getPointerAddressSpace();
+    ActualCallee = Builder.DAG.getConstant(0, Builder.getCurSDLoc(),
+                                           TLI.getPointerTy(DL, AS));
+  } else
+    ActualCallee = Builder.getValue(ISP.getCalledValue());
+
+  assert(CS.getCallingConv() != CallingConv::AnyReg &&
+         "anyregcc is not supported on statepoints!");
+
+  Type *DefTy = ISP.getActualReturnType();
+  bool HasDef = !DefTy->isVoidTy();
+
+  SDValue ReturnValue, CallEndVal;
+  std::tie(ReturnValue, CallEndVal) = Builder.lowerCallOperands(
+      ISP.getCallSite(), ImmutableStatepoint::CallArgsBeginPos,
+      ISP.getNumCallArgs(), ActualCallee, DefTy, LandingPad,
+      false /* IsPatchPoint */);
+
+  SDNode *CallEnd = CallEndVal.getNode();
+
+  // Get a call instruction from the call sequence chain.  Tail calls are not
+  // allowed.  The following code is essentially reverse engineering X86's
+  // LowerCallTo.
+  //
+  // We are expecting DAG to have the following form:
+  //
+  // ch = eh_label (only in case of invoke statepoint)
+  //   ch, glue = callseq_start ch
+  //   ch, glue = X86::Call ch, glue
+  //   ch, glue = callseq_end ch, glue
+  //   get_return_value ch, glue
+  //
+  // get_return_value can either be a CopyFromReg to grab the return value from
+  // %RAX, or it can be a LOAD to load a value returned by reference via a stack
+  // slot.
+
+  if (HasDef && (CallEnd->getOpcode() == ISD::CopyFromReg ||
+                 CallEnd->getOpcode() == ISD::LOAD))
+    CallEnd = CallEnd->getOperand(0).getNode();
+
+  assert(CallEnd->getOpcode() == ISD::CALLSEQ_END && "expected!");
+
   if (HasDef) {
     if (CS.isInvoke()) {
       // Result value will be used in different basic block for invokes
@@ -259,62 +352,29 @@ static SDNode *lowerCallFromStatepoint(ImmutableStatepoint StatepointSite,
       // register with correct type and save value into it manually.
       // TODO: To eliminate this problem we can remove gc.result intrinsics
       //       completelly and make statepoint call to return a tuple.
-      unsigned reg = Builder.FuncInfo.CreateRegs(Tmp->getType());
-      Builder.CopyValueToVirtualRegister(Tmp, reg);
-      Builder.FuncInfo.ValueMap[CS.getInstruction()] = reg;
-    }
-    else {
+      unsigned Reg = Builder.FuncInfo.CreateRegs(ISP.getActualReturnType());
+      RegsForValue RFV(
+          *Builder.DAG.getContext(), Builder.DAG.getTargetLoweringInfo(),
+          Builder.DAG.getDataLayout(), Reg, ISP.getActualReturnType());
+      SDValue Chain = Builder.DAG.getEntryNode();
+
+      RFV.getCopyToRegs(ReturnValue, Builder.DAG, Builder.getCurSDLoc(), Chain,
+                        nullptr);
+      PendingExports.push_back(Chain);
+      Builder.FuncInfo.ValueMap[CS.getInstruction()] = Reg;
+    } else {
       // The value of the statepoint itself will be the value of call itself.
       // We'll replace the actually call node shortly.  gc_result will grab
       // this value.
-      Builder.setValue(CS.getInstruction(), Builder.getValue(Tmp));
+      Builder.setValue(CS.getInstruction(), ReturnValue);
     }
   } else {
     // The token value is never used from here on, just generate a poison value
-    Builder.setValue(CS.getInstruction(), Builder.DAG.getIntPtrConstant(-1));
+    Builder.setValue(CS.getInstruction(),
+                     Builder.DAG.getIntPtrConstant(-1, Builder.getCurSDLoc()));
   }
-  // Remove the fake entry we created so we don't have a hanging reference
-  // after we delete this node.
-  Builder.removeValue(Tmp);
-  delete Tmp;
-  Tmp = nullptr;
-
-  // Search for the call node
-  // The following code is essentially reverse engineering X86's
-  // LowerCallTo.
-  // We are expecting DAG to have the following form:
-  // ch = eh_label (only in case of invoke statepoint)
-  //   ch, glue = callseq_start ch
-  //   ch, glue = X86::Call ch, glue
-  //   ch, glue = callseq_end ch, glue
-  // ch = eh_label ch (only in case of invoke statepoint)
-  //
-  // DAG root will be either last eh_label or callseq_end.
-    
-  SDNode *CallNode = nullptr;
-
-  // We just emitted a call, so it should be last thing generated
-  SDValue Chain = Builder.DAG.getRoot();
-
-  // Find closest CALLSEQ_END walking back through lowered nodes if needed
-  SDNode *CallEnd = Chain.getNode();
-  int Sanity = 0;
-  while (CallEnd->getOpcode() != ISD::CALLSEQ_END) {
-    assert(CallEnd->getNumOperands() >= 1 &&
-           CallEnd->getOperand(0).getValueType() == MVT::Other);
-
-    CallEnd = CallEnd->getOperand(0).getNode();
-
-    assert(Sanity < 20 && "should have found call end already");
-    Sanity++;
-  }
-  assert(CallEnd->getOpcode() == ISD::CALLSEQ_END &&
-         "Expected a callseq node.");
-  assert(CallEnd->getGluedNode());
 
-  // Step back inside the CALLSEQ
-  CallNode = CallEnd->getGluedNode();
-  return CallNode;
+  return CallEnd->getOperand(0).getNode();
 }
 
 /// Callect all gc pointers coming into statepoint intrinsic, clean them up,
@@ -324,17 +384,14 @@ static SDNode *lowerCallFromStatepoint(ImmutableStatepoint StatepointSite,
 ///   Relocs - the gc_relocate corresponding to each base/ptr pair
 /// Elements of this arrays should be in one-to-one correspondence with each
 /// other i.e Bases[i], Ptrs[i] are from the same gcrelocate call
-static void
-getIncomingStatepointGCValues(SmallVectorImpl<const Value *> &Bases,
-                              SmallVectorImpl<const Value *> &Ptrs,
-                              SmallVectorImpl<const Value *> &Relocs,
-                              ImmutableStatepoint StatepointSite,
-                              SelectionDAGBuilder &Builder) {
-  for (GCRelocateOperands relocateOpers :
-         StatepointSite.getRelocates(StatepointSite)) {
+static void getIncomingStatepointGCValues(
+    SmallVectorImpl<const Value *> &Bases, SmallVectorImpl<const Value *> &Ptrs,
+    SmallVectorImpl<const Value *> &Relocs, ImmutableStatepoint StatepointSite,
+    SelectionDAGBuilder &Builder) {
+  for (GCRelocateOperands relocateOpers : StatepointSite.getRelocates()) {
     Relocs.push_back(relocateOpers.getUnderlyingCallSite().getInstruction());
-    Bases.push_back(relocateOpers.basePtr());
-    Ptrs.push_back(relocateOpers.derivedPtr());
+    Bases.push_back(relocateOpers.getBasePtr());
+    Ptrs.push_back(relocateOpers.getDerivedPtr());
   }
 
   // Remove any redundant llvm::Values which map to the same SDValue as another
@@ -393,14 +450,12 @@ static void lowerIncomingStatepointValue(SDValue Incoming,
     // such in the stackmap.  This is required so that the consumer can
     // parse any internal format to the deopt state.  It also handles null
     // pointers and other constant pointers in GC states
-    Ops.push_back(
-        Builder.DAG.getTargetConstant(StackMaps::ConstantOp, MVT::i64));
-    Ops.push_back(Builder.DAG.getTargetConstant(C->getSExtValue(), MVT::i64));
+    pushStackMapConstant(Ops, Builder, C->getSExtValue());
   } else if (FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Incoming)) {
     // This handles allocas as arguments to the statepoint (this is only
     // really meaningful for a deopt value.  For GC, we'd be trying to
     // relocate the address of the alloca itself?)
-    Ops.push_back(Builder.DAG.getTargetFrameIndex(FI->getIndex(), 
+    Ops.push_back(Builder.DAG.getTargetFrameIndex(FI->getIndex(),
                                                   Incoming.getValueType()));
   } else {
     // Otherwise, locate a spill slot and explicitly spill it so it
@@ -433,8 +488,8 @@ static void lowerStatepointMetaArgs(SmallVectorImpl<SDValue> &Ops,
   // be: deopt argument length, deopt arguments.., gc arguments...
 
   SmallVector<const Value *, 64> Bases, Ptrs, Relocations;
-  getIncomingStatepointGCValues(Bases, Ptrs, Relocations,
-                                StatepointSite, Builder);
+  getIncomingStatepointGCValues(Bases, Ptrs, Relocations, StatepointSite,
+                                Builder);
 
 #ifndef NDEBUG
   // Check that each of the gc pointer and bases we've gotten out of the
@@ -465,37 +520,27 @@ static void lowerStatepointMetaArgs(SmallVectorImpl<SDValue> &Ops,
   }
 #endif
 
-
-
   // Before we actually start lowering (and allocating spill slots for values),
   // reserve any stack slots which we judge to be profitable to reuse for a
   // particular value.  This is purely an optimization over the code below and
   // doesn't change semantics at all.  It is important for performance that we
   // reserve slots for both deopt and gc values before lowering either.
-  for (auto I = StatepointSite.vm_state_begin() + 1,
-            E = StatepointSite.vm_state_end();
-       I != E; ++I) {
-    Value *V = *I;
-    SDValue Incoming = Builder.getValue(V);
-    reservePreviousStackSlotForValue(Incoming, Builder);
+  for (const Value *V : StatepointSite.vm_state_args()) {
+    reservePreviousStackSlotForValue(V, Builder);
   }
-  for (unsigned i = 0; i < Bases.size() * 2; ++i) {
-    // Even elements will contain base, odd elements - derived ptr
-    const Value *V = i % 2 ? Bases[i / 2] : Ptrs[i / 2];
-    SDValue Incoming = Builder.getValue(V);
-    reservePreviousStackSlotForValue(Incoming, Builder);
+  for (unsigned i = 0; i < Bases.size(); ++i) {
+    reservePreviousStackSlotForValue(Bases[i], Builder);
+    reservePreviousStackSlotForValue(Ptrs[i], Builder);
   }
 
   // First, prefix the list with the number of unique values to be
   // lowered.  Note that this is the number of *Values* not the
   // number of SDValues required to lower them.
-  const int NumVMSArgs = StatepointSite.numTotalVMSArgs();
-  Ops.push_back(
-      Builder.DAG.getTargetConstant(StackMaps::ConstantOp, MVT::i64));
-  Ops.push_back(Builder.DAG.getTargetConstant(NumVMSArgs, MVT::i64));
+  const int NumVMSArgs = StatepointSite.getNumTotalVMSArgs();
+  pushStackMapConstant(Ops, Builder, NumVMSArgs);
 
-  assert(NumVMSArgs + 1 == std::distance(StatepointSite.vm_state_begin(),
-                                         StatepointSite.vm_state_end()));
+  assert(NumVMSArgs == std::distance(StatepointSite.vm_state_begin(),
+                                     StatepointSite.vm_state_end()));
 
   // The vm state arguments are lowered in an opaque manner.  We do
   // not know what type of values are contained within.  We skip the
@@ -503,10 +548,7 @@ static void lowerStatepointMetaArgs(SmallVectorImpl<SDValue> &Ops,
   // explicitly just above.  We could have left it in the loop and
   // not done it explicitly, but it's far easier to understand this
   // way.
-  for (auto I = StatepointSite.vm_state_begin() + 1,
-            E = StatepointSite.vm_state_end();
-       I != E; ++I) {
-    const Value *V = *I;
+  for (const Value *V : StatepointSite.vm_state_args()) {
     SDValue Incoming = Builder.getValue(V);
     lowerIncomingStatepointValue(Incoming, Ops, Builder);
   }
@@ -516,25 +558,59 @@ static void lowerStatepointMetaArgs(SmallVectorImpl<SDValue> &Ops,
   // arrays interwoven with each (lowered) base pointer immediately followed by
   // it's (lowered) derived pointer.  i.e
   // (base[0], ptr[0], base[1], ptr[1], ...)
-  for (unsigned i = 0; i < Bases.size() * 2; ++i) {
-    // Even elements will contain base, odd elements - derived ptr
-    const Value *V = i % 2 ? Bases[i / 2] : Ptrs[i / 2];
-    SDValue Incoming = Builder.getValue(V);
-    lowerIncomingStatepointValue(Incoming, Ops, Builder);
+  for (unsigned i = 0; i < Bases.size(); ++i) {
+    const Value *Base = Bases[i];
+    lowerIncomingStatepointValue(Builder.getValue(Base), Ops, Builder);
+
+    const Value *Ptr = Ptrs[i];
+    lowerIncomingStatepointValue(Builder.getValue(Ptr), Ops, Builder);
   }
 
-  // If there are any explicit spill slots passed to the statepoint, record 
+  // If there are any explicit spill slots passed to the statepoint, record
   // them, but otherwise do not do anything special.  These are user provided
-  // allocas and give control over placement to the consumer.  In this case, 
+  // allocas and give control over placement to the consumer.  In this case,
   // it is the contents of the slot which may get updated, not the pointer to
   // the alloca
   for (Value *V : StatepointSite.gc_args()) {
     SDValue Incoming = Builder.getValue(V);
     if (FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Incoming)) {
       // This handles allocas as arguments to the statepoint
-      Ops.push_back(Builder.DAG.getTargetFrameIndex(FI->getIndex(), 
+      Ops.push_back(Builder.DAG.getTargetFrameIndex(FI->getIndex(),
                                                     Incoming.getValueType()));
+    }
+  }
 
+  // Record computed locations for all lowered values.
+  // This can not be embedded in lowering loops as we need to record *all*
+  // values, while previous loops account only values with unique SDValues.
+  const Instruction *StatepointInstr =
+    StatepointSite.getCallSite().getInstruction();
+  FunctionLoweringInfo::StatepointSpilledValueMapTy &SpillMap =
+    Builder.FuncInfo.StatepointRelocatedValues[StatepointInstr];
+
+  for (GCRelocateOperands RelocateOpers : StatepointSite.getRelocates()) {
+    const Value *V = RelocateOpers.getDerivedPtr();
+    SDValue SDV = Builder.getValue(V);
+    SDValue Loc = Builder.StatepointLowering.getLocation(SDV);
+
+    if (Loc.getNode()) {
+      SpillMap[V] = cast<FrameIndexSDNode>(Loc)->getIndex();
+    } else {
+      // Record value as visited, but not spilled. This is case for allocas
+      // and constants. For this values we can avoid emiting spill load while
+      // visiting corresponding gc_relocate.
+      // Actually we do not need to record them in this map at all.
+      // We do this only to check that we are not relocating any unvisited value.
+      SpillMap[V] = None;
+
+      // Default llvm mechanisms for exporting values which are used in
+      // different basic blocks does not work for gc relocates.
+      // Note that it would be incorrect to teach llvm that all relocates are
+      // uses of the corresponging values so that it would automatically
+      // export them. Relocates of the spilled values does not use original
+      // value.
+      if (StatepointSite.getCallSite().isInvoke())
+        Builder.ExportFromCurrentBlock(V);
     }
   }
 }
@@ -547,9 +623,8 @@ void SelectionDAGBuilder::visitStatepoint(const CallInst &CI) {
   LowerStatepoint(ImmutableStatepoint(&CI));
 }
 
-void
-SelectionDAGBuilder::LowerStatepoint(ImmutableStatepoint ISP,
-                                     MachineBasicBlock *LandingPad/*=nullptr*/) {
+void SelectionDAGBuilder::LowerStatepoint(
+    ImmutableStatepoint ISP, MachineBasicBlock *LandingPad /*=nullptr*/) {
   // The basic scheme here is that information about both the original call and
   // the safepoint is encoded in the CallInst.  We create a temporary call and
   // lower it, then reverse engineer the calling sequence.
@@ -561,11 +636,14 @@ SelectionDAGBuilder::LowerStatepoint(ImmutableStatepoint ISP,
   ImmutableCallSite CS(ISP.getCallSite());
 
 #ifndef NDEBUG
-  // Consistency check
-  for (const User *U : CS->users()) {
-    const CallInst *Call = cast<CallInst>(U);
-    if (isGCRelocate(Call))
-      StatepointLowering.scheduleRelocCall(*Call);
+  // Consistency check. Don't do this for invokes. It would be too
+  // expensive to preserve this information across different basic blocks
+  if (!CS.isInvoke()) {
+    for (const User *U : CS->users()) {
+      const CallInst *Call = cast<CallInst>(U);
+      if (isGCRelocate(Call))
+        StatepointLowering.scheduleRelocCall(*Call);
+    }
   }
 #endif
 
@@ -576,39 +654,82 @@ SelectionDAGBuilder::LowerStatepoint(ImmutableStatepoint ISP,
   ISP.verify();
 
   // Check that the associated GCStrategy expects to encounter statepoints.
-  // TODO: This if should become an assert.  For now, we allow the GCStrategy
-  // to be optional for backwards compatibility.  This will only last a short
-  // period (i.e. a couple of weeks).
   assert(GFI->getStrategy().useStatepoints() &&
          "GCStrategy does not expect to encounter statepoints");
 #endif
 
   // Lower statepoint vmstate and gcstate arguments
-  SmallVector<SDValue, 10> LoweredArgs;
-  lowerStatepointMetaArgs(LoweredArgs, ISP, *this);
+  SmallVector<SDValue, 10> LoweredMetaArgs;
+  lowerStatepointMetaArgs(LoweredMetaArgs, ISP, *this);
 
   // Get call node, we will replace it later with statepoint
-  SDNode *CallNode = lowerCallFromStatepoint(ISP, LandingPad, *this);
+  SDNode *CallNode =
+      lowerCallFromStatepoint(ISP, LandingPad, *this, PendingExports);
+
+  // Construct the actual GC_TRANSITION_START, STATEPOINT, and GC_TRANSITION_END
+  // nodes with all the appropriate arguments and return values.
+
+  // Call Node: Chain, Target, {Args}, RegMask, [Glue]
+  SDValue Chain = CallNode->getOperand(0);
+
+  SDValue Glue;
+  bool CallHasIncomingGlue = CallNode->getGluedNode();
+  if (CallHasIncomingGlue) {
+    // Glue is always last operand
+    Glue = CallNode->getOperand(CallNode->getNumOperands() - 1);
+  }
+
+  // Build the GC_TRANSITION_START node if necessary.
+  //
+  // The operands to the GC_TRANSITION_{START,END} nodes are laid out in the
+  // order in which they appear in the call to the statepoint intrinsic. If
+  // any of the operands is a pointer-typed, that operand is immediately
+  // followed by a SRCVALUE for the pointer that may be used during lowering
+  // (e.g. to form MachinePointerInfo values for loads/stores).
+  const bool IsGCTransition =
+      (ISP.getFlags() & (uint64_t)StatepointFlags::GCTransition) ==
+          (uint64_t)StatepointFlags::GCTransition;
+  if (IsGCTransition) {
+    SmallVector<SDValue, 8> TSOps;
+
+    // Add chain
+    TSOps.push_back(Chain);
+
+    // Add GC transition arguments
+    for (const Value *V : ISP.gc_transition_args()) {
+      TSOps.push_back(getValue(V));
+      if (V->getType()->isPointerTy())
+        TSOps.push_back(DAG.getSrcValue(V));
+    }
+
+    // Add glue if necessary
+    if (CallHasIncomingGlue)
+      TSOps.push_back(Glue);
 
-  // Construct the actual STATEPOINT node with all the appropriate arguments
-  // and return values.
+    SDVTList NodeTys = DAG.getVTList(MVT::Other, MVT::Glue);
+
+    SDValue GCTransitionStart =
+        DAG.getNode(ISD::GC_TRANSITION_START, getCurSDLoc(), NodeTys, TSOps);
+
+    Chain = GCTransitionStart.getValue(0);
+    Glue = GCTransitionStart.getValue(1);
+  }
 
   // TODO: Currently, all of these operands are being marked as read/write in
   // PrologEpilougeInserter.cpp, we should special case the VMState arguments
   // and flags to be read-only.
   SmallVector<SDValue, 40> Ops;
 
+  // Add the <id> and <numBytes> constants.
+  Ops.push_back(DAG.getTargetConstant(ISP.getID(), getCurSDLoc(), MVT::i64));
+  Ops.push_back(
+      DAG.getTargetConstant(ISP.getNumPatchBytes(), getCurSDLoc(), MVT::i32));
+
   // Calculate and push starting position of vmstate arguments
-  // Call Node: Chain, Target, {Args}, RegMask, [Glue]
-  SDValue Glue;
-  if (CallNode->getGluedNode()) {
-    // Glue is always last operand
-    Glue = CallNode->getOperand(CallNode->getNumOperands() - 1);
-  }
   // Get number of arguments incoming directly into call node
   unsigned NumCallRegArgs =
-      CallNode->getNumOperands() - (Glue.getNode() ? 4 : 3);
-  Ops.push_back(DAG.getTargetConstant(NumCallRegArgs, MVT::i32));
+      CallNode->getNumOperands() - (CallHasIncomingGlue ? 4 : 3);
+  Ops.push_back(DAG.getTargetConstant(NumCallRegArgs, getCurSDLoc(), MVT::i32));
 
   // Add call target
   SDValue CallTarget = SDValue(CallNode->getOperand(1).getNode(), 0);
@@ -617,29 +738,30 @@ SelectionDAGBuilder::LowerStatepoint(ImmutableStatepoint ISP,
   // Add call arguments
   // Get position of register mask in the call
   SDNode::op_iterator RegMaskIt;
-  if (Glue.getNode())
+  if (CallHasIncomingGlue)
     RegMaskIt = CallNode->op_end() - 2;
   else
     RegMaskIt = CallNode->op_end() - 1;
   Ops.insert(Ops.end(), CallNode->op_begin() + 2, RegMaskIt);
 
-  // Add a leading constant argument with the Flags and the calling convention
-  // masked together
-  CallingConv::ID CallConv = CS.getCallingConv();
-  int Flags = cast<ConstantInt>(CS.getArgument(2))->getZExtValue();
-  assert(Flags == 0 && "not expected to be used");
-  Ops.push_back(DAG.getTargetConstant(StackMaps::ConstantOp, MVT::i64));
-  Ops.push_back(
-      DAG.getTargetConstant(Flags | ((unsigned)CallConv << 1), MVT::i64));
+  // Add a constant argument for the calling convention
+  pushStackMapConstant(Ops, *this, CS.getCallingConv());
+
+  // Add a constant argument for the flags
+  uint64_t Flags = ISP.getFlags();
+  assert(
+      ((Flags & ~(uint64_t)StatepointFlags::MaskAll) == 0)
+          && "unknown flag used");
+  pushStackMapConstant(Ops, *this, Flags);
 
   // Insert all vmstate and gcstate arguments
-  Ops.insert(Ops.end(), LoweredArgs.begin(), LoweredArgs.end());
+  Ops.insert(Ops.end(), LoweredMetaArgs.begin(), LoweredMetaArgs.end());
 
   // Add register mask from call node
   Ops.push_back(*RegMaskIt);
 
   // Add chain
-  Ops.push_back(CallNode->getOperand(0));
+  Ops.push_back(Chain);
 
   // Same for the glue, but we add it only if original call had it
   if (Glue.getNode())
@@ -649,11 +771,41 @@ SelectionDAGBuilder::LowerStatepoint(ImmutableStatepoint ISP,
   // input.  This allows someone else to chain off us as needed.
   SDVTList NodeTys = DAG.getVTList(MVT::Other, MVT::Glue);
 
-  SDNode *StatepointMCNode = DAG.getMachineNode(TargetOpcode::STATEPOINT,
-                                                getCurSDLoc(), NodeTys, Ops);
+  SDNode *StatepointMCNode =
+      DAG.getMachineNode(TargetOpcode::STATEPOINT, getCurSDLoc(), NodeTys, Ops);
+
+  SDNode *SinkNode = StatepointMCNode;
+
+  // Build the GC_TRANSITION_END node if necessary.
+  //
+  // See the comment above regarding GC_TRANSITION_START for the layout of
+  // the operands to the GC_TRANSITION_END node.
+  if (IsGCTransition) {
+    SmallVector<SDValue, 8> TEOps;
+
+    // Add chain
+    TEOps.push_back(SDValue(StatepointMCNode, 0));
+
+    // Add GC transition arguments
+    for (const Value *V : ISP.gc_transition_args()) {
+      TEOps.push_back(getValue(V));
+      if (V->getType()->isPointerTy())
+        TEOps.push_back(DAG.getSrcValue(V));
+    }
+
+    // Add glue
+    TEOps.push_back(SDValue(StatepointMCNode, 1));
+
+    SDVTList NodeTys = DAG.getVTList(MVT::Other, MVT::Glue);
+
+    SDValue GCTransitionStart =
+        DAG.getNode(ISD::GC_TRANSITION_END, getCurSDLoc(), NodeTys, TEOps);
+
+    SinkNode = GCTransitionStart.getNode();
+  }
 
   // Replace original call
-  DAG.ReplaceAllUsesWith(CallNode, StatepointMCNode); // This may update Root
+  DAG.ReplaceAllUsesWith(CallNode, SinkNode); // This may update Root
   // Remove originall call node
   DAG.DeleteNode(CallNode);
 
@@ -671,8 +823,7 @@ void SelectionDAGBuilder::visitGCResult(const CallInst &CI) {
   // The result value of the gc_result is simply the result of the actual
   // call.  We've already emitted this, so just grab the value.
   Instruction *I = cast<Instruction>(CI.getArgOperand(0));
-  assert(isStatepoint(I) &&
-         "first argument must be a statepoint token");
+  assert(isStatepoint(I) && "first argument must be a statepoint token");
 
   if (isa<InvokeInst>(I)) {
     // For invokes we should have stored call result in a virtual register.
@@ -681,56 +832,63 @@ void SelectionDAGBuilder::visitGCResult(const CallInst &CI) {
     // different, and getValue() will use CopyFromReg of the wrong type,
     // which is always i32 in our case.
     PointerType *CalleeType = cast<PointerType>(
-                                ImmutableStatepoint(I).actualCallee()->getType());
-    Type *RetTy = cast<FunctionType>(
-                                CalleeType->getElementType())->getReturnType();
+        ImmutableStatepoint(I).getCalledValue()->getType());
+    Type *RetTy =
+        cast<FunctionType>(CalleeType->getElementType())->getReturnType();
     SDValue CopyFromReg = getCopyFromRegs(I, RetTy);
 
     assert(CopyFromReg.getNode());
     setValue(&CI, CopyFromReg);
-  }
-  else {
+  } else {
     setValue(&CI, getValue(I));
   }
 }
 
 void SelectionDAGBuilder::visitGCRelocate(const CallInst &CI) {
+  GCRelocateOperands RelocateOpers(&CI);
+
 #ifndef NDEBUG
   // Consistency check
-  StatepointLowering.relocCallVisited(CI);
+  // We skip this check for invoke statepoints. It would be too expensive to
+  // preserve validation info through different basic blocks.
+  if (!RelocateOpers.isTiedToInvoke()) {
+    StatepointLowering.relocCallVisited(CI);
+  }
 #endif
 
-  GCRelocateOperands relocateOpers(&CI);
-  SDValue SD = getValue(relocateOpers.derivedPtr());
+  const Value *DerivedPtr = RelocateOpers.getDerivedPtr();
+  SDValue SD = getValue(DerivedPtr);
+
+  FunctionLoweringInfo::StatepointSpilledValueMapTy &SpillMap =
+    FuncInfo.StatepointRelocatedValues[RelocateOpers.getStatepoint()];
+
+  // We should have recorded location for this pointer
+  assert(SpillMap.count(DerivedPtr) && "Relocating not lowered gc value");
+  Optional<int> DerivedPtrLocation = SpillMap[DerivedPtr];
 
-  if (isa<ConstantSDNode>(SD) || isa<FrameIndexSDNode>(SD)) {
-    // We didn't need to spill these special cases (constants and allocas).
-    // See the handling in spillIncomingValueForStatepoint for detail.
+  // We didn't need to spill these special cases (constants and allocas).
+  // See the handling in spillIncomingValueForStatepoint for detail.
+  if (!DerivedPtrLocation) {
     setValue(&CI, SD);
     return;
   }
 
-  SDValue Loc = StatepointLowering.getRelocLocation(SD);
-  // Emit new load if we did not emit it before
-  if (!Loc.getNode()) {
-    SDValue SpillSlot = StatepointLowering.getLocation(SD);
-    int FI = cast<FrameIndexSDNode>(SpillSlot)->getIndex();
-
-    // Be conservative: flush all pending loads
-    // TODO: Probably we can be less restrictive on this,
-    // it may allow more scheduling opprtunities
-    SDValue Chain = getRoot();
+  SDValue SpillSlot = DAG.getTargetFrameIndex(*DerivedPtrLocation,
+                                              SD.getValueType());
 
-    Loc = DAG.getLoad(SpillSlot.getValueType(), getCurSDLoc(), Chain,
-                      SpillSlot, MachinePointerInfo::getFixedStack(FI), false,
-                      false, false, 0);
+  // Be conservative: flush all pending loads
+  // TODO: Probably we can be less restrictive on this,
+  // it may allow more scheduling opprtunities
+  SDValue Chain = getRoot();
 
-    StatepointLowering.setRelocLocation(SD, Loc);
+  SDValue SpillLoad =
+    DAG.getLoad(SpillSlot.getValueType(), getCurSDLoc(), Chain, SpillSlot,
+                MachinePointerInfo::getFixedStack(*DerivedPtrLocation),
+                false, false, false, 0);
 
-    // Again, be conservative, don't emit pending loads
-    DAG.setRoot(Loc.getValue(1));
-  }
+  // Again, be conservative, don't emit pending loads
+  DAG.setRoot(SpillLoad.getValue(1));
 
-  assert(Loc.getNode());
-  setValue(&CI, Loc);
+  assert(SpillLoad.getNode());
+  setValue(&CI, SpillLoad);
 }
diff --git a/lib/CodeGen/SelectionDAG/StatepointLowering.h b/lib/CodeGen/SelectionDAG/StatepointLowering.h
index 673112cf8bb..82d0c62f1c3 100644
--- a/lib/CodeGen/SelectionDAG/StatepointLowering.h
+++ b/lib/CodeGen/SelectionDAG/StatepointLowering.h
@@ -30,8 +30,7 @@ class SelectionDAGBuilder;
 /// works in concert with information in FunctionLoweringInfo.
 class StatepointLoweringState {
 public:
-  StatepointLoweringState() : NextSlotToAllocate(0) {
-  }
+  StatepointLoweringState() : NextSlotToAllocate(0) {}
 
   /// Reset all state tracking for a newly encountered safepoint.  Also
   /// performs some consistency checking.
@@ -57,25 +56,6 @@ class StatepointLoweringState {
     Locations[val] = Location;
   }
 
-  /// Returns the relocated value for a given input pointer. Will
-  /// return SDValue() if this value hasn't yet been reloaded from
-  /// it's stack slot after the statepoint.  Otherwise, the value
-  /// has already been reloaded and the SDValue of that reload will
-  /// be returned. Note that VMState values are spilled but not
-  /// reloaded (since they don't change at the safepoint unless
-  /// also listed in the GC pointer section) and will thus never
-  /// be in this map
-  SDValue getRelocLocation(SDValue val) {
-    if (!RelocLocations.count(val))
-      return SDValue();
-    return RelocLocations[val];
-  }
-  void setRelocLocation(SDValue val, SDValue Location) {
-    assert(!RelocLocations.count(val) &&
-           "Trying to allocate already allocated location");
-    RelocLocations[val] = Location;
-  }
-
   /// Record the fact that we expect to encounter a given gc_relocate
   /// before the next statepoint.  If we don't see it, we'll report
   /// an assertion.
@@ -118,8 +98,6 @@ class StatepointLoweringState {
   /// Maps pre-relocation value (gc pointer directly incoming into statepoint)
   /// into it's location (currently only stack slots)
   DenseMap<SDValue, SDValue> Locations;
-  /// Map pre-relocated value into it's new relocated location
-  DenseMap<SDValue, SDValue> RelocLocations;
 
   /// A boolean indicator for each slot listed in the FunctionInfo as to
   /// whether it has been used in the current statepoint.  Since we try to
diff --git a/lib/CodeGen/SelectionDAG/TargetLowering.cpp b/lib/CodeGen/SelectionDAG/TargetLowering.cpp
index ddbf0b2dd53..ffffaf0fe6f 100644
--- a/lib/CodeGen/SelectionDAG/TargetLowering.cpp
+++ b/lib/CodeGen/SelectionDAG/TargetLowering.cpp
@@ -102,7 +102,8 @@ TargetLowering::makeLibCall(SelectionDAG &DAG,
   }
   if (LC == RTLIB::UNKNOWN_LIBCALL)
     report_fatal_error("Unsupported library call operation!");
-  SDValue Callee = DAG.getExternalSymbol(getLibcallName(LC), getPointerTy());
+  SDValue Callee = DAG.getExternalSymbol(getLibcallName(LC),
+                                         getPointerTy(DAG.getDataLayout()));
 
   Type *RetTy = RetVT.getTypeForEVT(*DAG.getContext());
   TargetLowering::CallLoweringInfo CLI(DAG);
@@ -114,7 +115,6 @@ TargetLowering::makeLibCall(SelectionDAG &DAG,
   return LowerCallTo(CLI);
 }
 
-
 /// SoftenSetCCOperands - Soften the operands of a comparison.  This code is
 /// shared among BR_CC, SELECT_CC, and SETCC handlers.
 void TargetLowering::softenSetCCOperands(SelectionDAG &DAG, EVT VT,
@@ -126,6 +126,7 @@ void TargetLowering::softenSetCCOperands(SelectionDAG &DAG, EVT VT,
 
   // Expand into one or more soft-fp libcall(s).
   RTLIB::Libcall LC1 = RTLIB::UNKNOWN_LIBCALL, LC2 = RTLIB::UNKNOWN_LIBCALL;
+  bool ShouldInvertCC = false;
   switch (CCCode) {
   case ISD::SETEQ:
   case ISD::SETOEQ:
@@ -165,34 +166,38 @@ void TargetLowering::softenSetCCOperands(SelectionDAG &DAG, EVT VT,
     LC1 = (VT == MVT::f32) ? RTLIB::O_F32 :
           (VT == MVT::f64) ? RTLIB::O_F64 : RTLIB::O_F128;
     break;
-  default:
+  case ISD::SETONE:
+    // SETONE = SETOLT | SETOGT
+    LC1 = (VT == MVT::f32) ? RTLIB::OLT_F32 :
+          (VT == MVT::f64) ? RTLIB::OLT_F64 : RTLIB::OLT_F128;
+    LC2 = (VT == MVT::f32) ? RTLIB::OGT_F32 :
+          (VT == MVT::f64) ? RTLIB::OGT_F64 : RTLIB::OGT_F128;
+    break;
+  case ISD::SETUEQ:
     LC1 = (VT == MVT::f32) ? RTLIB::UO_F32 :
           (VT == MVT::f64) ? RTLIB::UO_F64 : RTLIB::UO_F128;
+    LC2 = (VT == MVT::f32) ? RTLIB::OEQ_F32 :
+          (VT == MVT::f64) ? RTLIB::OEQ_F64 : RTLIB::OEQ_F128;
+    break;
+  default:
+    // Invert CC for unordered comparisons
+    ShouldInvertCC = true;
     switch (CCCode) {
-    case ISD::SETONE:
-      // SETONE = SETOLT | SETOGT
-      LC1 = (VT == MVT::f32) ? RTLIB::OLT_F32 :
-            (VT == MVT::f64) ? RTLIB::OLT_F64 : RTLIB::OLT_F128;
-      // Fallthrough
-    case ISD::SETUGT:
-      LC2 = (VT == MVT::f32) ? RTLIB::OGT_F32 :
-            (VT == MVT::f64) ? RTLIB::OGT_F64 : RTLIB::OGT_F128;
-      break;
-    case ISD::SETUGE:
-      LC2 = (VT == MVT::f32) ? RTLIB::OGE_F32 :
-            (VT == MVT::f64) ? RTLIB::OGE_F64 : RTLIB::OGE_F128;
-      break;
     case ISD::SETULT:
-      LC2 = (VT == MVT::f32) ? RTLIB::OLT_F32 :
-            (VT == MVT::f64) ? RTLIB::OLT_F64 : RTLIB::OLT_F128;
+      LC1 = (VT == MVT::f32) ? RTLIB::OGE_F32 :
+            (VT == MVT::f64) ? RTLIB::OGE_F64 : RTLIB::OGE_F128;
       break;
     case ISD::SETULE:
-      LC2 = (VT == MVT::f32) ? RTLIB::OLE_F32 :
+      LC1 = (VT == MVT::f32) ? RTLIB::OGT_F32 :
+            (VT == MVT::f64) ? RTLIB::OGT_F64 : RTLIB::OGT_F128;
+      break;
+    case ISD::SETUGT:
+      LC1 = (VT == MVT::f32) ? RTLIB::OLE_F32 :
             (VT == MVT::f64) ? RTLIB::OLE_F64 : RTLIB::OLE_F128;
       break;
-    case ISD::SETUEQ:
-      LC2 = (VT == MVT::f32) ? RTLIB::OEQ_F32 :
-            (VT == MVT::f64) ? RTLIB::OEQ_F64 : RTLIB::OEQ_F128;
+    case ISD::SETUGE:
+      LC1 = (VT == MVT::f32) ? RTLIB::OLT_F32 :
+            (VT == MVT::f64) ? RTLIB::OLT_F64 : RTLIB::OLT_F128;
       break;
     default: llvm_unreachable("Do not know how to soften this setcc!");
     }
@@ -202,18 +207,24 @@ void TargetLowering::softenSetCCOperands(SelectionDAG &DAG, EVT VT,
   EVT RetVT = getCmpLibcallReturnType();
   SDValue Ops[2] = { NewLHS, NewRHS };
   NewLHS = makeLibCall(DAG, LC1, RetVT, Ops, 2, false/*sign irrelevant*/,
-                       dl).first;
-  NewRHS = DAG.getConstant(0, RetVT);
+					   dl).first;
+  NewRHS = DAG.getConstant(0, dl, RetVT);
+
   CCCode = getCmpLibcallCC(LC1);
+  if (ShouldInvertCC)
+    CCCode = getSetCCInverse(CCCode, /*isInteger=*/true);
+
   if (LC2 != RTLIB::UNKNOWN_LIBCALL) {
-    SDValue Tmp = DAG.getNode(ISD::SETCC, dl,
-                              getSetCCResultType(*DAG.getContext(), RetVT),
-                              NewLHS, NewRHS, DAG.getCondCode(CCCode));
+    SDValue Tmp = DAG.getNode(
+        ISD::SETCC, dl,
+        getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), RetVT),
+        NewLHS, NewRHS, DAG.getCondCode(CCCode));
     NewLHS = makeLibCall(DAG, LC2, RetVT, Ops, 2, false/*sign irrelevant*/,
-                         dl).first;
-    NewLHS = DAG.getNode(ISD::SETCC, dl,
-                         getSetCCResultType(*DAG.getContext(), RetVT), NewLHS,
-                         NewRHS, DAG.getCondCode(getCmpLibcallCC(LC2)));
+      dl).first;
+    NewLHS = DAG.getNode(
+        ISD::SETCC, dl,
+        getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), RetVT),
+        NewLHS, NewRHS, DAG.getCondCode(getCmpLibcallCC(LC2)));
     NewLHS = DAG.getNode(ISD::OR, dl, Tmp.getValueType(), Tmp, NewLHS);
     NewRHS = SDValue();
   }
@@ -242,7 +253,7 @@ SDValue TargetLowering::getPICJumpTableRelocBase(SDValue Table,
 
   if ((JTEncoding == MachineJumpTableInfo::EK_GPRel64BlockAddress) ||
       (JTEncoding == MachineJumpTableInfo::EK_GPRel32BlockAddress))
-    return DAG.getGLOBAL_OFFSET_TABLE(getPointerTy(0));
+    return DAG.getGLOBAL_OFFSET_TABLE(getPointerTy(DAG.getDataLayout()));
 
   return Table;
 }
@@ -254,7 +265,7 @@ const MCExpr *
 TargetLowering::getPICJumpTableRelocBaseExpr(const MachineFunction *MF,
                                              unsigned JTI,MCContext &Ctx) const{
   // The normal PIC reloc base is the label at the start of the jump table.
-  return MCSymbolRefExpr::Create(MF->getJTISymbol(JTI, Ctx), Ctx);
+  return MCSymbolRefExpr::create(MF->getJTISymbol(JTI, Ctx), Ctx);
 }
 
 bool
@@ -265,9 +276,7 @@ TargetLowering::isOffsetFoldingLegal(const GlobalAddressSDNode *GA) const {
 
   // In dynamic-no-pic mode, assume that known defined values are safe.
   if (getTargetMachine().getRelocationModel() == Reloc::DynamicNoPIC &&
-      GA &&
-      !GA->getGlobal()->isDeclaration() &&
-      !GA->getGlobal()->isWeakForLinker())
+      GA && GA->getGlobal()->isStrongDefinitionForLinker())
     return true;
 
   // Otherwise assume nothing is safe.
@@ -305,7 +314,7 @@ bool TargetLowering::TargetLoweringOpt::ShrinkDemandedConstant(SDValue Op,
       SDValue New = DAG.getNode(Op.getOpcode(), dl, VT, Op.getOperand(0),
                                 DAG.getConstant(Demanded &
                                                 C->getAPIntValue(),
-                                                VT));
+                                                dl, VT));
       return CombineTo(Op, New);
     }
 
@@ -383,6 +392,7 @@ bool TargetLowering::SimplifyDemandedBits(SDValue Op,
          "Mask size mismatches value type size!");
   APInt NewMask = DemandedMask;
   SDLoc dl(Op);
+  auto &DL = TLO.DAG.getDataLayout();
 
   // Don't know anything.
   KnownZero = KnownOne = APInt(BitWidth, 0);
@@ -449,7 +459,7 @@ bool TargetLowering::SimplifyDemandedBits(SDValue Op,
       return TLO.CombineTo(Op, Op.getOperand(1));
     // If all of the demanded bits in the inputs are known zeros, return zero.
     if ((NewMask & (KnownZero|KnownZero2)) == NewMask)
-      return TLO.CombineTo(Op, TLO.DAG.getConstant(0, Op.getValueType()));
+      return TLO.CombineTo(Op, TLO.DAG.getConstant(0, dl, Op.getValueType()));
     // If the RHS is a constant, see if we can simplify it.
     if (TLO.ShrinkDemandedConstant(Op, ~KnownZero2 & NewMask))
       return true;
@@ -537,7 +547,7 @@ bool TargetLowering::SimplifyDemandedBits(SDValue Op,
     if ((NewMask & (KnownZero|KnownOne)) == NewMask) { // all known on one side
       if (KnownOne == KnownOne2) { // set bits are the same on both sides
         EVT VT = Op.getValueType();
-        SDValue ANDC = TLO.DAG.getConstant(~KnownOne & NewMask, VT);
+        SDValue ANDC = TLO.DAG.getConstant(~KnownOne & NewMask, dl, VT);
         return TLO.CombineTo(Op, TLO.DAG.getNode(ISD::AND, dl, VT,
                                                  Op.getOperand(0), ANDC));
       }
@@ -553,7 +563,7 @@ bool TargetLowering::SimplifyDemandedBits(SDValue Op,
         if (Expanded != C->getAPIntValue()) {
           EVT VT = Op.getValueType();
           SDValue New = TLO.DAG.getNode(Op.getOpcode(), dl,VT, Op.getOperand(0),
-                                          TLO.DAG.getConstant(Expanded, VT));
+                                        TLO.DAG.getConstant(Expanded, dl, VT));
           return TLO.CombineTo(Op, New);
         }
         // if it already has all the bits set, nothing to change
@@ -626,7 +636,7 @@ bool TargetLowering::SimplifyDemandedBits(SDValue Op,
           }
 
           SDValue NewSA =
-            TLO.DAG.getConstant(Diff, Op.getOperand(1).getValueType());
+            TLO.DAG.getConstant(Diff, dl, Op.getOperand(1).getValueType());
           EVT VT = Op.getValueType();
           return TLO.CombineTo(Op, TLO.DAG.getNode(Opc, dl, VT,
                                                    InOp.getOperand(0), NewSA));
@@ -645,12 +655,12 @@ bool TargetLowering::SimplifyDemandedBits(SDValue Op,
         unsigned InnerBits = InnerVT.getSizeInBits();
         if (ShAmt < InnerBits && NewMask.lshr(InnerBits) == 0 &&
             isTypeDesirableForOp(ISD::SHL, InnerVT)) {
-          EVT ShTy = getShiftAmountTy(InnerVT);
+          EVT ShTy = getShiftAmountTy(InnerVT, DL);
           if (!APInt(BitWidth, ShAmt).isIntN(ShTy.getSizeInBits()))
             ShTy = InnerVT;
           SDValue NarrowShl =
             TLO.DAG.getNode(ISD::SHL, dl, InnerVT, InnerOp,
-                            TLO.DAG.getConstant(ShAmt, ShTy));
+                            TLO.DAG.getConstant(ShAmt, dl, ShTy));
           return
             TLO.CombineTo(Op,
                           TLO.DAG.getNode(ISD::ANY_EXTEND, dl, Op.getValueType(),
@@ -672,7 +682,7 @@ bool TargetLowering::SimplifyDemandedBits(SDValue Op,
               NewMask.lshr(InnerBits - InnerShAmt + ShAmt) == 0 &&
               NewMask.trunc(ShAmt) == 0) {
             SDValue NewSA =
-              TLO.DAG.getConstant(ShAmt - InnerShAmt,
+              TLO.DAG.getConstant(ShAmt - InnerShAmt, dl,
                                   Op.getOperand(1).getValueType());
             EVT VT = Op.getValueType();
             SDValue NewExt = TLO.DAG.getNode(ISD::ANY_EXTEND, dl, VT,
@@ -700,6 +710,13 @@ bool TargetLowering::SimplifyDemandedBits(SDValue Op,
       if (ShAmt >= BitWidth)
         break;
 
+      APInt InDemandedMask = (NewMask << ShAmt);
+
+      // If the shift is exact, then it does demand the low bits (and knows that
+      // they are zero).
+      if (cast<BinaryWithFlagsSDNode>(Op)->Flags.hasExact())
+        InDemandedMask |= APInt::getLowBitsSet(BitWidth, ShAmt);
+
       // If this is ((X << C1) >>u ShAmt), see if we can simplify this into a
       // single shift.  We can do this if the top bits (which are shifted out)
       // are never demanded.
@@ -715,14 +732,14 @@ bool TargetLowering::SimplifyDemandedBits(SDValue Op,
           }
 
           SDValue NewSA =
-            TLO.DAG.getConstant(Diff, Op.getOperand(1).getValueType());
+            TLO.DAG.getConstant(Diff, dl, Op.getOperand(1).getValueType());
           return TLO.CombineTo(Op, TLO.DAG.getNode(Opc, dl, VT,
                                                    InOp.getOperand(0), NewSA));
         }
       }
 
       // Compute the new bits that are at the top now.
-      if (SimplifyDemandedBits(InOp, (NewMask << ShAmt),
+      if (SimplifyDemandedBits(InOp, InDemandedMask,
                                KnownZero, KnownOne, TLO, Depth+1))
         return true;
       assert((KnownZero & KnownOne) == 0 && "Bits known to be one AND zero?");
@@ -753,6 +770,11 @@ bool TargetLowering::SimplifyDemandedBits(SDValue Op,
 
       APInt InDemandedMask = (NewMask << ShAmt);
 
+      // If the shift is exact, then it does demand the low bits (and knows that
+      // they are zero).
+      if (cast<BinaryWithFlagsSDNode>(Op)->Flags.hasExact())
+        InDemandedMask |= APInt::getLowBitsSet(BitWidth, ShAmt);
+
       // If any of the demanded bits are produced by the sign extension, we also
       // demand the input sign bit.
       APInt HighBits = APInt::getHighBitsSet(BitWidth, ShAmt);
@@ -771,16 +793,19 @@ bool TargetLowering::SimplifyDemandedBits(SDValue Op,
 
       // If the input sign bit is known to be zero, or if none of the top bits
       // are demanded, turn this into an unsigned shift right.
-      if (KnownZero.intersects(SignBit) || (HighBits & ~NewMask) == HighBits)
-        return TLO.CombineTo(Op, TLO.DAG.getNode(ISD::SRL, dl, VT,
-                                                 Op.getOperand(0),
-                                                 Op.getOperand(1)));
+      if (KnownZero.intersects(SignBit) || (HighBits & ~NewMask) == HighBits) {
+        SDNodeFlags Flags;
+        Flags.setExact(cast<BinaryWithFlagsSDNode>(Op)->Flags.hasExact());
+        return TLO.CombineTo(Op,
+                             TLO.DAG.getNode(ISD::SRL, dl, VT, Op.getOperand(0),
+                                             Op.getOperand(1), &Flags));
+      }
 
       int Log2 = NewMask.exactLogBase2();
       if (Log2 >= 0) {
         // The bit must come from the sign.
         SDValue NewSA =
-          TLO.DAG.getConstant(BitWidth - 1 - Log2,
+          TLO.DAG.getConstant(BitWidth - 1 - Log2, dl,
                               Op.getOperand(1).getValueType());
         return TLO.CombineTo(Op, TLO.DAG.getNode(ISD::SRL, dl, VT,
                                                  Op.getOperand(0), NewSA));
@@ -809,9 +834,10 @@ bool TargetLowering::SimplifyDemandedBits(SDValue Op,
         // for scalar types after legalization.
         EVT ShiftAmtTy = Op.getValueType();
         if (TLO.LegalTypes() && !ShiftAmtTy.isVector())
-          ShiftAmtTy = getShiftAmountTy(ShiftAmtTy);
+          ShiftAmtTy = getShiftAmountTy(ShiftAmtTy, DL);
 
-        SDValue ShiftAmt = TLO.DAG.getConstant(BitWidth - ShAmt, ShiftAmtTy);
+        SDValue ShiftAmt = TLO.DAG.getConstant(BitWidth - ShAmt, dl,
+                                               ShiftAmtTy);
         return TLO.CombineTo(Op, TLO.DAG.getNode(ISD::SHL, dl,
                                                  Op.getValueType(), InOp,
                                                  ShiftAmt));
@@ -993,8 +1019,8 @@ bool TargetLowering::SimplifyDemandedBits(SDValue Op,
         SDValue Shift = In.getOperand(1);
         if (TLO.LegalTypes()) {
           uint64_t ShVal = ShAmt->getZExtValue();
-          Shift =
-            TLO.DAG.getConstant(ShVal, getShiftAmountTy(Op.getValueType()));
+          Shift = TLO.DAG.getConstant(ShVal, dl,
+                                      getShiftAmountTy(Op.getValueType(), DL));
         }
 
         APInt HighBits = APInt::getHighBitsSet(OperandBitWidth,
@@ -1052,7 +1078,7 @@ bool TargetLowering::SimplifyDemandedBits(SDValue Op,
         if (!OpVTLegal && OpVTSizeInBits > 32)
           Sign = TLO.DAG.getNode(ISD::ZERO_EXTEND, dl, Op.getValueType(), Sign);
         unsigned ShVal = Op.getValueType().getSizeInBits()-1;
-        SDValue ShAmt = TLO.DAG.getConstant(ShVal, Op.getValueType());
+        SDValue ShAmt = TLO.DAG.getConstant(ShVal, dl, Op.getValueType());
         return TLO.CombineTo(Op, TLO.DAG.getNode(ISD::SHL, dl,
                                                  Op.getValueType(),
                                                  Sign, ShAmt));
@@ -1085,8 +1111,19 @@ bool TargetLowering::SimplifyDemandedBits(SDValue Op,
 
   // If we know the value of all of the demanded bits, return this as a
   // constant.
-  if ((NewMask & (KnownZero|KnownOne)) == NewMask)
-    return TLO.CombineTo(Op, TLO.DAG.getConstant(KnownOne, Op.getValueType()));
+  if ((NewMask & (KnownZero|KnownOne)) == NewMask) {
+    // Avoid folding to a constant if any OpaqueConstant is involved.
+    const SDNode *N = Op.getNode();
+    for (SDNodeIterator I = SDNodeIterator::begin(N),
+         E = SDNodeIterator::end(N); I != E; ++I) {
+      SDNode *Op = *I;
+      if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op))
+        if (C->isOpaque())
+          return false;
+    }
+    return TLO.CombineTo(Op,
+                         TLO.DAG.getConstant(KnownOne, dl, Op.getValueType()));
+  }
 
   return false;
 }
@@ -1222,13 +1259,14 @@ TargetLowering::SimplifySetCC(EVT VT, SDValue N0, SDValue N1,
   switch (Cond) {
   default: break;
   case ISD::SETFALSE:
-  case ISD::SETFALSE2: return DAG.getConstant(0, VT);
+  case ISD::SETFALSE2: return DAG.getConstant(0, dl, VT);
   case ISD::SETTRUE:
   case ISD::SETTRUE2: {
     TargetLowering::BooleanContent Cnt =
         getBooleanContents(N0->getValueType(0));
     return DAG.getConstant(
-        Cnt == TargetLowering::ZeroOrNegativeOneBooleanContent ? -1ULL : 1, VT);
+        Cnt == TargetLowering::ZeroOrNegativeOneBooleanContent ? -1ULL : 1, dl,
+        VT);
   }
   }
 
@@ -1262,7 +1300,7 @@ TargetLowering::SimplifySetCC(EVT VT, SDValue N0, SDValue N1,
           // (srl (ctlz x), 5) == 1  -> X == 0
           Cond = ISD::SETEQ;
         }
-        SDValue Zero = DAG.getConstant(0, N0.getValueType());
+        SDValue Zero = DAG.getConstant(0, dl, N0.getValueType());
         return DAG.getSetCC(dl, VT, N0.getOperand(0).getOperand(0),
                             Zero, Cond);
       }
@@ -1283,10 +1321,10 @@ TargetLowering::SimplifySetCC(EVT VT, SDValue N0, SDValue N1,
       // (ctpop x) u> 1 -> (x & x-1) != 0
       if ((Cond == ISD::SETULT && C1 == 2) || (Cond == ISD::SETUGT && C1 == 1)){
         SDValue Sub = DAG.getNode(ISD::SUB, dl, CTVT, CTOp,
-                                  DAG.getConstant(1, CTVT));
+                                  DAG.getConstant(1, dl, CTVT));
         SDValue And = DAG.getNode(ISD::AND, dl, CTVT, CTOp, Sub);
         ISD::CondCode CC = Cond == ISD::SETULT ? ISD::SETEQ : ISD::SETNE;
-        return DAG.getSetCC(dl, VT, And, DAG.getConstant(0, CTVT), CC);
+        return DAG.getSetCC(dl, VT, And, DAG.getConstant(0, dl, CTVT), CC);
       }
 
       // TODO: (ctpop x) == 1 -> x && (x & x-1) == 0 iff ctpop is illegal.
@@ -1340,7 +1378,7 @@ TargetLowering::SimplifySetCC(EVT VT, SDValue N0, SDValue N1,
         if (isTypeDesirableForOp(ISD::SETCC, MinVT)) {
           // Will get folded away.
           SDValue Trunc = DAG.getNode(ISD::TRUNCATE, dl, MinVT, PreExt);
-          SDValue C = DAG.getConstant(C1.trunc(MinBits), MinVT);
+          SDValue C = DAG.getConstant(C1.trunc(MinBits), dl, MinVT);
           return DAG.getSetCC(dl, VT, Trunc, C, Cond);
         }
       }
@@ -1372,7 +1410,7 @@ TargetLowering::SimplifySetCC(EVT VT, SDValue N0, SDValue N1,
           APInt newMask = APInt::getLowBitsSet(maskWidth, width);
           for (unsigned offset=0; offset<origWidth/width; offset++) {
             if ((newMask & Mask) == Mask) {
-              if (!getDataLayout()->isLittleEndian())
+              if (!DAG.getDataLayout().isLittleEndian())
                 bestOffset = (origWidth/width - offset - 1) * (width/8);
               else
                 bestOffset = (uint64_t)offset * (width/8);
@@ -1391,7 +1429,7 @@ TargetLowering::SimplifySetCC(EVT VT, SDValue N0, SDValue N1,
           SDValue Ptr = Lod->getBasePtr();
           if (bestOffset != 0)
             Ptr = DAG.getNode(ISD::ADD, dl, PtrType, Lod->getBasePtr(),
-                              DAG.getConstant(bestOffset, PtrType));
+                              DAG.getConstant(bestOffset, dl, PtrType));
           unsigned NewAlign = MinAlign(Lod->getAlignment(), bestOffset);
           SDValue NewLoad = DAG.getLoad(newVT, dl, Lod->getChain(), Ptr,
                                 Lod->getPointerInfo().getWithOffset(bestOffset),
@@ -1399,8 +1437,8 @@ TargetLowering::SimplifySetCC(EVT VT, SDValue N0, SDValue N1,
           return DAG.getSetCC(dl, VT,
                               DAG.getNode(ISD::AND, dl, newVT, NewLoad,
                                       DAG.getConstant(bestMask.trunc(bestWidth),
-                                                      newVT)),
-                              DAG.getConstant(0LL, newVT), Cond);
+                                                      dl, newVT)),
+                              DAG.getConstant(0LL, dl, newVT), Cond);
         }
       }
     }
@@ -1416,18 +1454,18 @@ TargetLowering::SimplifySetCC(EVT VT, SDValue N0, SDValue N1,
         switch (Cond) {
         case ISD::SETUGT:
         case ISD::SETUGE:
-        case ISD::SETEQ: return DAG.getConstant(0, VT);
+        case ISD::SETEQ: return DAG.getConstant(0, dl, VT);
         case ISD::SETULT:
         case ISD::SETULE:
-        case ISD::SETNE: return DAG.getConstant(1, VT);
+        case ISD::SETNE: return DAG.getConstant(1, dl, VT);
         case ISD::SETGT:
         case ISD::SETGE:
           // True if the sign bit of C1 is set.
-          return DAG.getConstant(C1.isNegative(), VT);
+          return DAG.getConstant(C1.isNegative(), dl, VT);
         case ISD::SETLT:
         case ISD::SETLE:
           // True if the sign bit of C1 isn't set.
-          return DAG.getConstant(C1.isNonNegative(), VT);
+          return DAG.getConstant(C1.isNonNegative(), dl, VT);
         default:
           break;
         }
@@ -1445,8 +1483,9 @@ TargetLowering::SimplifySetCC(EVT VT, SDValue N0, SDValue N1,
         if (DCI.isBeforeLegalizeOps() ||
             (isOperationLegal(ISD::SETCC, newVT) &&
              getCondCodeAction(Cond, newVT.getSimpleVT()) == Legal)) {
-          EVT NewSetCCVT = getSetCCResultType(*DAG.getContext(), newVT);
-          SDValue NewConst = DAG.getConstant(C1.trunc(InSize), newVT);
+          EVT NewSetCCVT =
+              getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), newVT);
+          SDValue NewConst = DAG.getConstant(C1.trunc(InSize), dl, newVT);
 
           SDValue NewSetCC = DAG.getSetCC(dl, NewSetCCVT, N0.getOperand(0),
                                           NewConst, Cond);
@@ -1467,7 +1506,7 @@ TargetLowering::SimplifySetCC(EVT VT, SDValue N0, SDValue N1,
       // If the constant doesn't fit into the number of bits for the source of
       // the sign extension, it is impossible for both sides to be equal.
       if (C1.getMinSignedBits() > ExtSrcTyBits)
-        return DAG.getConstant(Cond == ISD::SETNE, VT);
+        return DAG.getConstant(Cond == ISD::SETNE, dl, VT);
 
       SDValue ZextOp;
       EVT Op0Ty = N0.getOperand(0).getValueType();
@@ -1476,7 +1515,7 @@ TargetLowering::SimplifySetCC(EVT VT, SDValue N0, SDValue N1,
       } else {
         APInt Imm = APInt::getLowBitsSet(ExtDstTyBits, ExtSrcTyBits);
         ZextOp = DAG.getNode(ISD::AND, dl, Op0Ty, N0.getOperand(0),
-                              DAG.getConstant(Imm, Op0Ty));
+                              DAG.getConstant(Imm, dl, Op0Ty));
       }
       if (!DCI.isCalledByLegalizer())
         DCI.AddToWorklist(ZextOp.getNode());
@@ -1485,7 +1524,7 @@ TargetLowering::SimplifySetCC(EVT VT, SDValue N0, SDValue N1,
                           DAG.getConstant(C1 & APInt::getLowBitsSet(
                                                               ExtDstTyBits,
                                                               ExtSrcTyBits),
-                                          ExtDstTy),
+                                          dl, ExtDstTy),
                           Cond);
     } else if ((N1C->isNullValue() || N1C->getAPIntValue() == 1) &&
                 (Cond == ISD::SETEQ || Cond == ISD::SETNE)) {
@@ -1555,20 +1594,20 @@ TargetLowering::SimplifySetCC(EVT VT, SDValue N0, SDValue N1,
           if (Op0.getValueType().bitsGT(VT))
             Op0 = DAG.getNode(ISD::AND, dl, VT,
                           DAG.getNode(ISD::TRUNCATE, dl, VT, Op0.getOperand(0)),
-                          DAG.getConstant(1, VT));
+                          DAG.getConstant(1, dl, VT));
           else if (Op0.getValueType().bitsLT(VT))
             Op0 = DAG.getNode(ISD::AND, dl, VT,
                         DAG.getNode(ISD::ANY_EXTEND, dl, VT, Op0.getOperand(0)),
-                        DAG.getConstant(1, VT));
+                        DAG.getConstant(1, dl, VT));
 
           return DAG.getSetCC(dl, VT, Op0,
-                              DAG.getConstant(0, Op0.getValueType()),
+                              DAG.getConstant(0, dl, Op0.getValueType()),
                               Cond == ISD::SETEQ ? ISD::SETNE : ISD::SETEQ);
         }
         if (Op0.getOpcode() == ISD::AssertZext &&
             cast<VTSDNode>(Op0.getOperand(1))->getVT() == MVT::i1)
           return DAG.getSetCC(dl, VT, Op0,
-                              DAG.getConstant(0, Op0.getValueType()),
+                              DAG.getConstant(0, dl, Op0.getValueType()),
                               Cond == ISD::SETEQ ? ISD::SETNE : ISD::SETEQ);
       }
     }
@@ -1585,7 +1624,7 @@ TargetLowering::SimplifySetCC(EVT VT, SDValue N0, SDValue N1,
 
     // Canonicalize GE/LE comparisons to use GT/LT comparisons.
     if (Cond == ISD::SETGE || Cond == ISD::SETUGE) {
-      if (C1 == MinVal) return DAG.getConstant(1, VT);   // X >= MIN --> true
+      if (C1 == MinVal) return DAG.getConstant(1, dl, VT);  // X >= MIN --> true
       // X >= C0 --> X > (C0 - 1)
       APInt C = C1 - 1;
       ISD::CondCode NewCC = (Cond == ISD::SETGE) ? ISD::SETGT : ISD::SETUGT;
@@ -1594,13 +1633,13 @@ TargetLowering::SimplifySetCC(EVT VT, SDValue N0, SDValue N1,
           (!N1C->isOpaque() || (N1C->isOpaque() && C.getBitWidth() <= 64 &&
                                 isLegalICmpImmediate(C.getSExtValue())))) {
         return DAG.getSetCC(dl, VT, N0,
-                            DAG.getConstant(C, N1.getValueType()),
+                            DAG.getConstant(C, dl, N1.getValueType()),
                             NewCC);
       }
     }
 
     if (Cond == ISD::SETLE || Cond == ISD::SETULE) {
-      if (C1 == MaxVal) return DAG.getConstant(1, VT);   // X <= MAX --> true
+      if (C1 == MaxVal) return DAG.getConstant(1, dl, VT);  // X <= MAX --> true
       // X <= C0 --> X < (C0 + 1)
       APInt C = C1 + 1;
       ISD::CondCode NewCC = (Cond == ISD::SETLE) ? ISD::SETLT : ISD::SETULT;
@@ -1609,19 +1648,19 @@ TargetLowering::SimplifySetCC(EVT VT, SDValue N0, SDValue N1,
           (!N1C->isOpaque() || (N1C->isOpaque() && C.getBitWidth() <= 64 &&
                                 isLegalICmpImmediate(C.getSExtValue())))) {
         return DAG.getSetCC(dl, VT, N0,
-                            DAG.getConstant(C, N1.getValueType()),
+                            DAG.getConstant(C, dl, N1.getValueType()),
                             NewCC);
       }
     }
 
     if ((Cond == ISD::SETLT || Cond == ISD::SETULT) && C1 == MinVal)
-      return DAG.getConstant(0, VT);      // X < MIN --> false
+      return DAG.getConstant(0, dl, VT);      // X < MIN --> false
     if ((Cond == ISD::SETGE || Cond == ISD::SETUGE) && C1 == MinVal)
-      return DAG.getConstant(1, VT);      // X >= MIN --> true
+      return DAG.getConstant(1, dl, VT);      // X >= MIN --> true
     if ((Cond == ISD::SETGT || Cond == ISD::SETUGT) && C1 == MaxVal)
-      return DAG.getConstant(0, VT);      // X > MAX --> false
+      return DAG.getConstant(0, dl, VT);      // X > MAX --> false
     if ((Cond == ISD::SETLE || Cond == ISD::SETULE) && C1 == MaxVal)
-      return DAG.getConstant(1, VT);      // X <= MAX --> true
+      return DAG.getConstant(1, dl, VT);      // X <= MAX --> true
 
     // Canonicalize setgt X, Min --> setne X, Min
     if ((Cond == ISD::SETGT || Cond == ISD::SETUGT) && C1 == MinVal)
@@ -1633,12 +1672,12 @@ TargetLowering::SimplifySetCC(EVT VT, SDValue N0, SDValue N1,
     // If we have setult X, 1, turn it into seteq X, 0
     if ((Cond == ISD::SETLT || Cond == ISD::SETULT) && C1 == MinVal+1)
       return DAG.getSetCC(dl, VT, N0,
-                          DAG.getConstant(MinVal, N0.getValueType()),
+                          DAG.getConstant(MinVal, dl, N0.getValueType()),
                           ISD::SETEQ);
     // If we have setugt X, Max-1, turn it into seteq X, Max
     if ((Cond == ISD::SETGT || Cond == ISD::SETUGT) && C1 == MaxVal-1)
       return DAG.getSetCC(dl, VT, N0,
-                          DAG.getConstant(MaxVal, N0.getValueType()),
+                          DAG.getConstant(MaxVal, dl, N0.getValueType()),
                           ISD::SETEQ);
 
     // If we have "setcc X, C0", check to see if we can shrink the immediate
@@ -1648,14 +1687,14 @@ TargetLowering::SimplifySetCC(EVT VT, SDValue N0, SDValue N1,
     if (Cond == ISD::SETUGT &&
         C1 == APInt::getSignedMaxValue(OperandBitSize))
       return DAG.getSetCC(dl, VT, N0,
-                          DAG.getConstant(0, N1.getValueType()),
+                          DAG.getConstant(0, dl, N1.getValueType()),
                           ISD::SETLT);
 
     // SETULT X, SINTMIN  -> SETGT X, -1
     if (Cond == ISD::SETULT &&
         C1 == APInt::getSignedMinValue(OperandBitSize)) {
       SDValue ConstMinusOne =
-          DAG.getConstant(APInt::getAllOnesValue(OperandBitSize),
+          DAG.getConstant(APInt::getAllOnesValue(OperandBitSize), dl,
                           N1.getValueType());
       return DAG.getSetCC(dl, VT, N0, ConstMinusOne, ISD::SETGT);
     }
@@ -1664,17 +1703,20 @@ TargetLowering::SimplifySetCC(EVT VT, SDValue N0, SDValue N1,
     if ((Cond == ISD::SETEQ || Cond == ISD::SETNE) &&
         (VT == N0.getValueType() ||
          (isTypeLegal(VT) && VT.bitsLE(N0.getValueType()))) &&
-        N0.getOpcode() == ISD::AND)
+        N0.getOpcode() == ISD::AND) {
+      auto &DL = DAG.getDataLayout();
       if (ConstantSDNode *AndRHS =
                   dyn_cast<ConstantSDNode>(N0.getOperand(1))) {
-        EVT ShiftTy = DCI.isBeforeLegalize() ?
-          getPointerTy() : getShiftAmountTy(N0.getValueType());
+        EVT ShiftTy = DCI.isBeforeLegalize()
+                          ? getPointerTy(DL)
+                          : getShiftAmountTy(N0.getValueType(), DL);
         if (Cond == ISD::SETNE && C1 == 0) {// (X & 8) != 0  -->  (X & 8) >> 3
           // Perform the xform if the AND RHS is a single bit.
           if (AndRHS->getAPIntValue().isPowerOf2()) {
             return DAG.getNode(ISD::TRUNCATE, dl, VT,
                               DAG.getNode(ISD::SRL, dl, N0.getValueType(), N0,
-                   DAG.getConstant(AndRHS->getAPIntValue().logBase2(), ShiftTy)));
+                   DAG.getConstant(AndRHS->getAPIntValue().logBase2(), dl,
+                                   ShiftTy)));
           }
         } else if (Cond == ISD::SETEQ && C1 == AndRHS->getAPIntValue()) {
           // (X & 8) == 8  -->  (X & 8) >> 3
@@ -1682,10 +1724,12 @@ TargetLowering::SimplifySetCC(EVT VT, SDValue N0, SDValue N1,
           if (C1.isPowerOf2()) {
             return DAG.getNode(ISD::TRUNCATE, dl, VT,
                                DAG.getNode(ISD::SRL, dl, N0.getValueType(), N0,
-                                      DAG.getConstant(C1.logBase2(), ShiftTy)));
+                                      DAG.getConstant(C1.logBase2(), dl,
+                                                      ShiftTy)));
           }
         }
       }
+    }
 
     if (C1.getMinSignedBits() <= 64 &&
         !isLegalICmpImmediate(C1.getSExtValue())) {
@@ -1697,12 +1741,15 @@ TargetLowering::SimplifySetCC(EVT VT, SDValue N0, SDValue N1,
           const APInt &AndRHSC = AndRHS->getAPIntValue();
           if ((-AndRHSC).isPowerOf2() && (AndRHSC & C1) == C1) {
             unsigned ShiftBits = AndRHSC.countTrailingZeros();
-            EVT ShiftTy = DCI.isBeforeLegalize() ?
-              getPointerTy() : getShiftAmountTy(N0.getValueType());
+            auto &DL = DAG.getDataLayout();
+            EVT ShiftTy = DCI.isBeforeLegalize()
+                              ? getPointerTy(DL)
+                              : getShiftAmountTy(N0.getValueType(), DL);
             EVT CmpTy = N0.getValueType();
             SDValue Shift = DAG.getNode(ISD::SRL, dl, CmpTy, N0.getOperand(0),
-                                        DAG.getConstant(ShiftBits, ShiftTy));
-            SDValue CmpRHS = DAG.getConstant(C1.lshr(ShiftBits), CmpTy);
+                                        DAG.getConstant(ShiftBits, dl,
+                                                        ShiftTy));
+            SDValue CmpRHS = DAG.getConstant(C1.lshr(ShiftBits), dl, CmpTy);
             return DAG.getSetCC(dl, VT, Shift, CmpRHS, Cond);
           }
         }
@@ -1724,13 +1771,16 @@ TargetLowering::SimplifySetCC(EVT VT, SDValue N0, SDValue N1,
           ShiftBits = C1.countTrailingZeros();
         }
         NewC = NewC.lshr(ShiftBits);
-        if (ShiftBits && isLegalICmpImmediate(NewC.getSExtValue())) {
-          EVT ShiftTy = DCI.isBeforeLegalize() ?
-            getPointerTy() : getShiftAmountTy(N0.getValueType());
+        if (ShiftBits && NewC.getMinSignedBits() <= 64 &&
+          isLegalICmpImmediate(NewC.getSExtValue())) {
+          auto &DL = DAG.getDataLayout();
+          EVT ShiftTy = DCI.isBeforeLegalize()
+                            ? getPointerTy(DL)
+                            : getShiftAmountTy(N0.getValueType(), DL);
           EVT CmpTy = N0.getValueType();
           SDValue Shift = DAG.getNode(ISD::SRL, dl, CmpTy, N0,
-                                      DAG.getConstant(ShiftBits, ShiftTy));
-          SDValue CmpRHS = DAG.getConstant(NewC, CmpTy);
+                                      DAG.getConstant(ShiftBits, dl, ShiftTy));
+          SDValue CmpRHS = DAG.getConstant(NewC, dl, CmpTy);
           return DAG.getSetCC(dl, VT, Shift, CmpRHS, NewCond);
         }
       }
@@ -1749,9 +1799,9 @@ TargetLowering::SimplifySetCC(EVT VT, SDValue N0, SDValue N1,
       switch (ISD::getUnorderedFlavor(Cond)) {
       default: llvm_unreachable("Unknown flavor!");
       case 0:  // Known false.
-        return DAG.getConstant(0, VT);
+        return DAG.getConstant(0, dl, VT);
       case 1:  // Known true.
-        return DAG.getConstant(1, VT);
+        return DAG.getConstant(1, dl, VT);
       case 2:  // Undefined.
         return DAG.getUNDEF(VT);
       }
@@ -1818,13 +1868,13 @@ TargetLowering::SimplifySetCC(EVT VT, SDValue N0, SDValue N1,
 
     // We can always fold X == X for integer setcc's.
     if (N0.getValueType().isInteger()) {
-      return DAG.getConstant(EqVal, VT);
+      return DAG.getConstant(EqVal, dl, VT);
     }
     unsigned UOF = ISD::getUnorderedFlavor(Cond);
     if (UOF == 2)   // FP operators that are undefined on NaNs.
-      return DAG.getConstant(EqVal, VT);
+      return DAG.getConstant(EqVal, dl, VT);
     if (UOF == unsigned(ISD::isTrueWhenEqual(Cond)))
-      return DAG.getConstant(EqVal, VT);
+      return DAG.getConstant(EqVal, dl, VT);
     // Otherwise, we can't fold it.  However, we can simplify it to SETUO/SETO
     // if it is not already.
     ISD::CondCode NewCond = UOF == 0 ? ISD::SETO : ISD::SETUO;
@@ -1865,7 +1915,7 @@ TargetLowering::SimplifySetCC(EVT VT, SDValue N0, SDValue N1,
             return DAG.getSetCC(dl, VT, N0.getOperand(0),
                                 DAG.getConstant(RHSC->getAPIntValue()-
                                                 LHSR->getAPIntValue(),
-                                N0.getValueType()), Cond);
+                                dl, N0.getValueType()), Cond);
           }
 
           // Turn (X^C1) == C2 into X == C1^C2 iff X&~C1 = 0.
@@ -1877,7 +1927,7 @@ TargetLowering::SimplifySetCC(EVT VT, SDValue N0, SDValue N1,
                 DAG.getSetCC(dl, VT, N0.getOperand(0),
                              DAG.getConstant(LHSR->getAPIntValue() ^
                                                RHSC->getAPIntValue(),
-                                             N0.getValueType()),
+                                             dl, N0.getValueType()),
                              Cond);
         }
 
@@ -1888,7 +1938,7 @@ TargetLowering::SimplifySetCC(EVT VT, SDValue N0, SDValue N1,
               DAG.getSetCC(dl, VT, N0.getOperand(1),
                            DAG.getConstant(SUBC->getAPIntValue() -
                                              RHSC->getAPIntValue(),
-                                           N0.getValueType()),
+                                           dl, N0.getValueType()),
                            Cond);
           }
         }
@@ -1905,16 +1955,20 @@ TargetLowering::SimplifySetCC(EVT VT, SDValue N0, SDValue N1,
       if (!LegalRHSImm || N0.getNode()->hasOneUse()) {
         if (N0.getOperand(0) == N1)
           return DAG.getSetCC(dl, VT, N0.getOperand(1),
-                              DAG.getConstant(0, N0.getValueType()), Cond);
+                              DAG.getConstant(0, dl, N0.getValueType()), Cond);
         if (N0.getOperand(1) == N1) {
           if (DAG.isCommutativeBinOp(N0.getOpcode()))
             return DAG.getSetCC(dl, VT, N0.getOperand(0),
-                                DAG.getConstant(0, N0.getValueType()), Cond);
+                                DAG.getConstant(0, dl, N0.getValueType()),
+                                Cond);
           if (N0.getNode()->hasOneUse()) {
             assert(N0.getOpcode() == ISD::SUB && "Unexpected operation!");
+            auto &DL = DAG.getDataLayout();
             // (Z-X) == X  --> Z == X<<1
-            SDValue SH = DAG.getNode(ISD::SHL, dl, N1.getValueType(), N1,
-                       DAG.getConstant(1, getShiftAmountTy(N1.getValueType())));
+            SDValue SH = DAG.getNode(
+                ISD::SHL, dl, N1.getValueType(), N1,
+                DAG.getConstant(1, dl,
+                                getShiftAmountTy(N1.getValueType(), DL)));
             if (!DCI.isCalledByLegalizer())
               DCI.AddToWorklist(SH.getNode());
             return DAG.getSetCC(dl, VT, N0.getOperand(0), SH, Cond);
@@ -1928,16 +1982,18 @@ TargetLowering::SimplifySetCC(EVT VT, SDValue N0, SDValue N1,
       // Simplify  X == (X+Z) -->  Z == 0
       if (N1.getOperand(0) == N0)
         return DAG.getSetCC(dl, VT, N1.getOperand(1),
-                        DAG.getConstant(0, N1.getValueType()), Cond);
+                        DAG.getConstant(0, dl, N1.getValueType()), Cond);
       if (N1.getOperand(1) == N0) {
         if (DAG.isCommutativeBinOp(N1.getOpcode()))
           return DAG.getSetCC(dl, VT, N1.getOperand(0),
-                          DAG.getConstant(0, N1.getValueType()), Cond);
+                          DAG.getConstant(0, dl, N1.getValueType()), Cond);
         if (N1.getNode()->hasOneUse()) {
           assert(N1.getOpcode() == ISD::SUB && "Unexpected operation!");
+          auto &DL = DAG.getDataLayout();
           // X == (Z-X)  --> X<<1 == Z
-          SDValue SH = DAG.getNode(ISD::SHL, dl, N1.getValueType(), N0,
-                       DAG.getConstant(1, getShiftAmountTy(N0.getValueType())));
+          SDValue SH = DAG.getNode(
+              ISD::SHL, dl, N1.getValueType(), N0,
+              DAG.getConstant(1, dl, getShiftAmountTy(N0.getValueType(), DL)));
           if (!DCI.isCalledByLegalizer())
             DCI.AddToWorklist(SH.getNode());
           return DAG.getSetCC(dl, VT, SH, N1.getOperand(0), Cond);
@@ -1955,7 +2011,7 @@ TargetLowering::SimplifySetCC(EVT VT, SDValue N0, SDValue N1,
           Cond = ISD::getSetCCInverse(Cond, /*isInteger=*/true);
           if (DCI.isBeforeLegalizeOps() ||
               isCondCodeLegal(Cond, N0.getSimpleValueType())) {
-            SDValue Zero = DAG.getConstant(0, N1.getValueType());
+            SDValue Zero = DAG.getConstant(0, dl, N1.getValueType());
             return DAG.getSetCC(dl, VT, N0, Zero, Cond);
           }
         }
@@ -1966,7 +2022,7 @@ TargetLowering::SimplifySetCC(EVT VT, SDValue N0, SDValue N1,
           Cond = ISD::getSetCCInverse(Cond, /*isInteger=*/true);
           if (DCI.isBeforeLegalizeOps() ||
               isCondCodeLegal(Cond, N1.getSimpleValueType())) {
-            SDValue Zero = DAG.getConstant(0, N0.getValueType());
+            SDValue Zero = DAG.getConstant(0, dl, N0.getValueType());
             return DAG.getSetCC(dl, VT, N1, Zero, Cond);
           }
         }
@@ -2070,9 +2126,8 @@ PerformDAGCombine(SDNode *N, DAGCombinerInfo &DCI) const {
 //  Inline Assembler Implementation Methods
 //===----------------------------------------------------------------------===//
 
-
 TargetLowering::ConstraintType
-TargetLowering::getConstraintType(const std::string &Constraint) const {
+TargetLowering::getConstraintType(StringRef Constraint) const {
   unsigned S = Constraint.size();
 
   if (S == 1) {
@@ -2105,7 +2160,7 @@ TargetLowering::getConstraintType(const std::string &Constraint) const {
   }
 
   if (S > 1 && Constraint[0] == '{' && Constraint[S-1] == '}') {
-    if (S == 8 && !Constraint.compare(1, 6, "memory", 6))  // "{memory}"
+    if (S == 8 && Constraint.substr(1, 6) == "memory") // "{memory}"
       return C_Memory;
     return C_Register;
   }
@@ -2171,8 +2226,8 @@ void TargetLowering::LowerAsmOperandForConstraint(SDValue Op,
         Ops.push_back(DAG.getTargetGlobalAddress(GA->getGlobal(),
                                                  C ? SDLoc(C) : SDLoc(),
                                                  Op.getValueType(), Offs));
-        return;
       }
+      return;
     }
     if (C) {   // just C, no GV.
       // Simple constants are not allowed for 's'.
@@ -2181,9 +2236,9 @@ void TargetLowering::LowerAsmOperandForConstraint(SDValue Op,
         // now; without this it would get ZExt'd later in
         // ScheduleDAGSDNodes::EmitNode, which is very generic.
         Ops.push_back(DAG.getTargetConstant(C->getAPIntValue().getSExtValue(),
-                                            MVT::i64));
-        return;
+                                            SDLoc(C), MVT::i64));
       }
+      return;
     }
     break;
   }
@@ -2192,7 +2247,7 @@ void TargetLowering::LowerAsmOperandForConstraint(SDValue Op,
 
 std::pair<unsigned, const TargetRegisterClass *>
 TargetLowering::getRegForInlineAsmConstraint(const TargetRegisterInfo *RI,
-                                             const std::string &Constraint,
+                                             StringRef Constraint,
                                              MVT VT) const {
   if (Constraint.empty() || Constraint[0] != '{')
     return std::make_pair(0u, static_cast<TargetRegisterClass*>(nullptr));
@@ -2258,7 +2313,8 @@ unsigned TargetLowering::AsmOperandInfo::getMatchedOperand() const {
 /// If this returns an empty vector, and if the constraint string itself
 /// isn't empty, there was an error parsing.
 TargetLowering::AsmOperandInfoVector
-TargetLowering::ParseConstraints(const TargetRegisterInfo *TRI,
+TargetLowering::ParseConstraints(const DataLayout &DL,
+                                 const TargetRegisterInfo *TRI,
                                  ImmutableCallSite CS) const {
   /// ConstraintOperands - Information about all of the constraints.
   AsmOperandInfoVector ConstraintOperands;
@@ -2294,10 +2350,11 @@ TargetLowering::ParseConstraints(const TargetRegisterInfo *TRI,
       assert(!CS.getType()->isVoidTy() &&
              "Bad inline asm!");
       if (StructType *STy = dyn_cast<StructType>(CS.getType())) {
-        OpInfo.ConstraintVT = getSimpleValueType(STy->getElementType(ResNo));
+        OpInfo.ConstraintVT =
+            getSimpleValueType(DL, STy->getElementType(ResNo));
       } else {
         assert(ResNo == 0 && "Asm only has one result!");
-        OpInfo.ConstraintVT = getSimpleValueType(CS.getType());
+        OpInfo.ConstraintVT = getSimpleValueType(DL, CS.getType());
       }
       ++ResNo;
       break;
@@ -2326,7 +2383,7 @@ TargetLowering::ParseConstraints(const TargetRegisterInfo *TRI,
       // If OpTy is not a single value, it may be a struct/union that we
       // can tile with integers.
       if (!OpTy->isSingleValueType() && OpTy->isSized()) {
-        unsigned BitSize = getDataLayout()->getTypeSizeInBits(OpTy);
+        unsigned BitSize = DL.getTypeSizeInBits(OpTy);
         switch (BitSize) {
         default: break;
         case 1:
@@ -2340,8 +2397,7 @@ TargetLowering::ParseConstraints(const TargetRegisterInfo *TRI,
           break;
         }
       } else if (PointerType *PT = dyn_cast<PointerType>(OpTy)) {
-        unsigned PtrSize
-          = getDataLayout()->getPointerSizeInBits(PT->getAddressSpace());
+        unsigned PtrSize = DL.getPointerSizeInBits(PT->getAddressSpace());
         OpInfo.ConstraintVT = MVT::getIntegerVT(PtrSize);
       } else {
         OpInfo.ConstraintVT = MVT::getVT(OpTy, true);
@@ -2639,19 +2695,22 @@ void TargetLowering::ComputeConstraintToUse(AsmOperandInfo &OpInfo,
 
 /// \brief Given an exact SDIV by a constant, create a multiplication
 /// with the multiplicative inverse of the constant.
-SDValue TargetLowering::BuildExactSDIV(SDValue Op1, SDValue Op2, SDLoc dl,
-                                       SelectionDAG &DAG) const {
-  ConstantSDNode *C = cast<ConstantSDNode>(Op2);
-  APInt d = C->getAPIntValue();
+static SDValue BuildExactSDIV(const TargetLowering &TLI, SDValue Op1, APInt d,
+                              SDLoc dl, SelectionDAG &DAG,
+                              std::vector<SDNode *> &Created) {
   assert(d != 0 && "Division by zero!");
 
   // Shift the value upfront if it is even, so the LSB is one.
   unsigned ShAmt = d.countTrailingZeros();
   if (ShAmt) {
     // TODO: For UDIV use SRL instead of SRA.
-    SDValue Amt = DAG.getConstant(ShAmt, getShiftAmountTy(Op1.getValueType()));
-    Op1 = DAG.getNode(ISD::SRA, dl, Op1.getValueType(), Op1, Amt, false, false,
-                      true);
+    SDValue Amt =
+        DAG.getConstant(ShAmt, dl, TLI.getShiftAmountTy(Op1.getValueType(),
+                                                        DAG.getDataLayout()));
+    SDNodeFlags Flags;
+    Flags.setExact(true);
+    Op1 = DAG.getNode(ISD::SRA, dl, Op1.getValueType(), Op1, Amt, &Flags);
+    Created.push_back(Op1.getNode());
     d = d.ashr(ShAmt);
   }
 
@@ -2660,8 +2719,10 @@ SDValue TargetLowering::BuildExactSDIV(SDValue Op1, SDValue Op2, SDLoc dl,
   while ((t = d*xn) != 1)
     xn *= APInt(d.getBitWidth(), 2) - t;
 
-  Op2 = DAG.getConstant(xn, Op1.getValueType());
-  return DAG.getNode(ISD::MUL, dl, Op1.getValueType(), Op1, Op2);
+  SDValue Op2 = DAG.getConstant(xn, dl, Op1.getValueType());
+  SDValue Mul = DAG.getNode(ISD::MUL, dl, Op1.getValueType(), Op1, Op2);
+  Created.push_back(Mul.getNode());
+  return Mul;
 }
 
 /// \brief Given an ISD::SDIV node expressing a divide by constant,
@@ -2681,6 +2742,10 @@ SDValue TargetLowering::BuildSDIV(SDNode *N, const APInt &Divisor,
   if (!isTypeLegal(VT))
     return SDValue();
 
+  // If the sdiv has an 'exact' bit we can use a simpler lowering.
+  if (cast<BinaryWithFlagsSDNode>(N)->Flags.hasExact())
+    return BuildExactSDIV(*this, N->getOperand(0), Divisor, dl, DAG, *Created);
+
   APInt::ms magics = Divisor.magic();
 
   // Multiply the numerator (operand 0) by the magic value
@@ -2689,12 +2754,12 @@ SDValue TargetLowering::BuildSDIV(SDNode *N, const APInt &Divisor,
   if (IsAfterLegalization ? isOperationLegal(ISD::MULHS, VT) :
                             isOperationLegalOrCustom(ISD::MULHS, VT))
     Q = DAG.getNode(ISD::MULHS, dl, VT, N->getOperand(0),
-                    DAG.getConstant(magics.m, VT));
+                    DAG.getConstant(magics.m, dl, VT));
   else if (IsAfterLegalization ? isOperationLegal(ISD::SMUL_LOHI, VT) :
                                  isOperationLegalOrCustom(ISD::SMUL_LOHI, VT))
     Q = SDValue(DAG.getNode(ISD::SMUL_LOHI, dl, DAG.getVTList(VT, VT),
                               N->getOperand(0),
-                              DAG.getConstant(magics.m, VT)).getNode(), 1);
+                              DAG.getConstant(magics.m, dl, VT)).getNode(), 1);
   else
     return SDValue();       // No mulhs or equvialent
   // If d > 0 and m < 0, add the numerator
@@ -2707,16 +2772,19 @@ SDValue TargetLowering::BuildSDIV(SDNode *N, const APInt &Divisor,
     Q = DAG.getNode(ISD::SUB, dl, VT, Q, N->getOperand(0));
     Created->push_back(Q.getNode());
   }
+  auto &DL = DAG.getDataLayout();
   // Shift right algebraic if shift value is nonzero
   if (magics.s > 0) {
-    Q = DAG.getNode(ISD::SRA, dl, VT, Q,
-                 DAG.getConstant(magics.s, getShiftAmountTy(Q.getValueType())));
+    Q = DAG.getNode(
+        ISD::SRA, dl, VT, Q,
+        DAG.getConstant(magics.s, dl, getShiftAmountTy(Q.getValueType(), DL)));
     Created->push_back(Q.getNode());
   }
   // Extract the sign bit and add it to the quotient
-  SDValue T = DAG.getNode(ISD::SRL, dl, VT, Q,
-                          DAG.getConstant(VT.getScalarSizeInBits() - 1,
-                                          getShiftAmountTy(Q.getValueType())));
+  SDValue T =
+      DAG.getNode(ISD::SRL, dl, VT, Q,
+                  DAG.getConstant(VT.getScalarSizeInBits() - 1, dl,
+                                  getShiftAmountTy(Q.getValueType(), DL)));
   Created->push_back(T.getNode());
   return DAG.getNode(ISD::ADD, dl, VT, Q, T);
 }
@@ -2732,6 +2800,7 @@ SDValue TargetLowering::BuildUDIV(SDNode *N, const APInt &Divisor,
 
   EVT VT = N->getValueType(0);
   SDLoc dl(N);
+  auto &DL = DAG.getDataLayout();
 
   // Check to see if we can do this.
   // FIXME: We should be more aggressive here.
@@ -2748,8 +2817,9 @@ SDValue TargetLowering::BuildUDIV(SDNode *N, const APInt &Divisor,
   // the divided value upfront.
   if (magics.a != 0 && !Divisor[0]) {
     unsigned Shift = Divisor.countTrailingZeros();
-    Q = DAG.getNode(ISD::SRL, dl, VT, Q,
-                    DAG.getConstant(Shift, getShiftAmountTy(Q.getValueType())));
+    Q = DAG.getNode(
+        ISD::SRL, dl, VT, Q,
+        DAG.getConstant(Shift, dl, getShiftAmountTy(Q.getValueType(), DL)));
     Created->push_back(Q.getNode());
 
     // Get magic number for the shifted divisor.
@@ -2761,11 +2831,11 @@ SDValue TargetLowering::BuildUDIV(SDNode *N, const APInt &Divisor,
   // FIXME: We should support doing a MUL in a wider type
   if (IsAfterLegalization ? isOperationLegal(ISD::MULHU, VT) :
                             isOperationLegalOrCustom(ISD::MULHU, VT))
-    Q = DAG.getNode(ISD::MULHU, dl, VT, Q, DAG.getConstant(magics.m, VT));
+    Q = DAG.getNode(ISD::MULHU, dl, VT, Q, DAG.getConstant(magics.m, dl, VT));
   else if (IsAfterLegalization ? isOperationLegal(ISD::UMUL_LOHI, VT) :
                                  isOperationLegalOrCustom(ISD::UMUL_LOHI, VT))
     Q = SDValue(DAG.getNode(ISD::UMUL_LOHI, dl, DAG.getVTList(VT, VT), Q,
-                            DAG.getConstant(magics.m, VT)).getNode(), 1);
+                            DAG.getConstant(magics.m, dl, VT)).getNode(), 1);
   else
     return SDValue();       // No mulhu or equvialent
 
@@ -2774,18 +2844,22 @@ SDValue TargetLowering::BuildUDIV(SDNode *N, const APInt &Divisor,
   if (magics.a == 0) {
     assert(magics.s < Divisor.getBitWidth() &&
            "We shouldn't generate an undefined shift!");
-    return DAG.getNode(ISD::SRL, dl, VT, Q,
-                 DAG.getConstant(magics.s, getShiftAmountTy(Q.getValueType())));
+    return DAG.getNode(
+        ISD::SRL, dl, VT, Q,
+        DAG.getConstant(magics.s, dl, getShiftAmountTy(Q.getValueType(), DL)));
   } else {
     SDValue NPQ = DAG.getNode(ISD::SUB, dl, VT, N->getOperand(0), Q);
     Created->push_back(NPQ.getNode());
-    NPQ = DAG.getNode(ISD::SRL, dl, VT, NPQ,
-                      DAG.getConstant(1, getShiftAmountTy(NPQ.getValueType())));
+    NPQ = DAG.getNode(
+        ISD::SRL, dl, VT, NPQ,
+        DAG.getConstant(1, dl, getShiftAmountTy(NPQ.getValueType(), DL)));
     Created->push_back(NPQ.getNode());
     NPQ = DAG.getNode(ISD::ADD, dl, VT, NPQ, Q);
     Created->push_back(NPQ.getNode());
-    return DAG.getNode(ISD::SRL, dl, VT, NPQ,
-             DAG.getConstant(magics.s-1, getShiftAmountTy(NPQ.getValueType())));
+    return DAG.getNode(
+        ISD::SRL, dl, VT, NPQ,
+        DAG.getConstant(magics.s - 1, dl,
+                        getShiftAmountTy(NPQ.getValueType(), DL)));
   }
 }
 
@@ -2871,8 +2945,9 @@ bool TargetLowering::expandMUL(SDNode *N, SDValue &Lo, SDValue &Hi, EVT HiLoVT,
     if (!LH.getNode() && !RH.getNode() &&
         isOperationLegalOrCustom(ISD::SRL, VT) &&
         isOperationLegalOrCustom(ISD::TRUNCATE, HiLoVT)) {
+      auto &DL = DAG.getDataLayout();
       unsigned ShiftAmt = VT.getSizeInBits() - HiLoVT.getSizeInBits();
-      SDValue Shift = DAG.getConstant(ShiftAmt, getShiftAmountTy(VT));
+      SDValue Shift = DAG.getConstant(ShiftAmt, dl, getShiftAmountTy(VT, DL));
       LH = DAG.getNode(ISD::SRL, dl, VT, N->getOperand(0), Shift);
       LH = DAG.getNode(ISD::TRUNCATE, dl, HiLoVT, LH);
       RH = DAG.getNode(ISD::SRL, dl, VT, N->getOperand(1), Shift);
@@ -2922,49 +2997,93 @@ bool TargetLowering::expandFP_TO_SINT(SDNode *Node, SDValue &Result,
   // https://github.com/llvm-mirror/compiler-rt/blob/master/lib/builtins/fixsfdi.c
   EVT IntVT = EVT::getIntegerVT(*DAG.getContext(),
                                 VT.getSizeInBits());
-  SDValue ExponentMask = DAG.getConstant(0x7F800000, IntVT);
-  SDValue ExponentLoBit = DAG.getConstant(23, IntVT);
-  SDValue Bias = DAG.getConstant(127, IntVT);
-  SDValue SignMask = DAG.getConstant(APInt::getSignBit(VT.getSizeInBits()),
+  SDValue ExponentMask = DAG.getConstant(0x7F800000, dl, IntVT);
+  SDValue ExponentLoBit = DAG.getConstant(23, dl, IntVT);
+  SDValue Bias = DAG.getConstant(127, dl, IntVT);
+  SDValue SignMask = DAG.getConstant(APInt::getSignBit(VT.getSizeInBits()), dl,
                                      IntVT);
-  SDValue SignLowBit = DAG.getConstant(VT.getSizeInBits() - 1, IntVT);
-  SDValue MantissaMask = DAG.getConstant(0x007FFFFF, IntVT);
+  SDValue SignLowBit = DAG.getConstant(VT.getSizeInBits() - 1, dl, IntVT);
+  SDValue MantissaMask = DAG.getConstant(0x007FFFFF, dl, IntVT);
 
   SDValue Bits = DAG.getNode(ISD::BITCAST, dl, IntVT, Node->getOperand(0));
 
-  SDValue ExponentBits = DAG.getNode(ISD::SRL, dl, IntVT,
-      DAG.getNode(ISD::AND, dl, IntVT, Bits, ExponentMask),
-      DAG.getZExtOrTrunc(ExponentLoBit, dl, getShiftAmountTy(IntVT)));
+  auto &DL = DAG.getDataLayout();
+  SDValue ExponentBits = DAG.getNode(
+      ISD::SRL, dl, IntVT, DAG.getNode(ISD::AND, dl, IntVT, Bits, ExponentMask),
+      DAG.getZExtOrTrunc(ExponentLoBit, dl, getShiftAmountTy(IntVT, DL)));
   SDValue Exponent = DAG.getNode(ISD::SUB, dl, IntVT, ExponentBits, Bias);
 
-  SDValue Sign = DAG.getNode(ISD::SRA, dl, IntVT,
-      DAG.getNode(ISD::AND, dl, IntVT, Bits, SignMask),
-      DAG.getZExtOrTrunc(SignLowBit, dl, getShiftAmountTy(IntVT)));
+  SDValue Sign = DAG.getNode(
+      ISD::SRA, dl, IntVT, DAG.getNode(ISD::AND, dl, IntVT, Bits, SignMask),
+      DAG.getZExtOrTrunc(SignLowBit, dl, getShiftAmountTy(IntVT, DL)));
   Sign = DAG.getSExtOrTrunc(Sign, dl, NVT);
 
   SDValue R = DAG.getNode(ISD::OR, dl, IntVT,
       DAG.getNode(ISD::AND, dl, IntVT, Bits, MantissaMask),
-      DAG.getConstant(0x00800000, IntVT));
+      DAG.getConstant(0x00800000, dl, IntVT));
 
   R = DAG.getZExtOrTrunc(R, dl, NVT);
 
-
-  R = DAG.getSelectCC(dl, Exponent, ExponentLoBit,
-     DAG.getNode(ISD::SHL, dl, NVT, R,
-                 DAG.getZExtOrTrunc(
-                    DAG.getNode(ISD::SUB, dl, IntVT, Exponent, ExponentLoBit),
-                    dl, getShiftAmountTy(IntVT))),
-     DAG.getNode(ISD::SRL, dl, NVT, R,
-                 DAG.getZExtOrTrunc(
-                    DAG.getNode(ISD::SUB, dl, IntVT, ExponentLoBit, Exponent),
-                    dl, getShiftAmountTy(IntVT))),
-     ISD::SETGT);
+  R = DAG.getSelectCC(
+      dl, Exponent, ExponentLoBit,
+      DAG.getNode(ISD::SHL, dl, NVT, R,
+                  DAG.getZExtOrTrunc(
+                      DAG.getNode(ISD::SUB, dl, IntVT, Exponent, ExponentLoBit),
+                      dl, getShiftAmountTy(IntVT, DL))),
+      DAG.getNode(ISD::SRL, dl, NVT, R,
+                  DAG.getZExtOrTrunc(
+                      DAG.getNode(ISD::SUB, dl, IntVT, ExponentLoBit, Exponent),
+                      dl, getShiftAmountTy(IntVT, DL))),
+      ISD::SETGT);
 
   SDValue Ret = DAG.getNode(ISD::SUB, dl, NVT,
       DAG.getNode(ISD::XOR, dl, NVT, R, Sign),
       Sign);
 
-  Result = DAG.getSelectCC(dl, Exponent, DAG.getConstant(0, IntVT),
-      DAG.getConstant(0, NVT), Ret, ISD::SETLT);
+  Result = DAG.getSelectCC(dl, Exponent, DAG.getConstant(0, dl, IntVT),
+      DAG.getConstant(0, dl, NVT), Ret, ISD::SETLT);
   return true;
 }
+
+//===----------------------------------------------------------------------===//
+// Implementation of Emulated TLS Model
+//===----------------------------------------------------------------------===//
+
+SDValue TargetLowering::LowerToTLSEmulatedModel(const GlobalAddressSDNode *GA,
+                                                SelectionDAG &DAG) const {
+  // Access to address of TLS varialbe xyz is lowered to a function call:
+  //   __emutls_get_address( address of global variable named "__emutls_v.xyz" )
+  EVT PtrVT = getPointerTy(DAG.getDataLayout());
+  PointerType *VoidPtrType = Type::getInt8PtrTy(*DAG.getContext());
+  SDLoc dl(GA);
+
+  ArgListTy Args;
+  ArgListEntry Entry;
+  std::string NameString = ("__emutls_v." + GA->getGlobal()->getName()).str();
+  Module *VariableModule = const_cast<Module*>(GA->getGlobal()->getParent());
+  StringRef EmuTlsVarName(NameString);
+  GlobalVariable *EmuTlsVar = VariableModule->getNamedGlobal(EmuTlsVarName);
+  if (!EmuTlsVar)
+    EmuTlsVar = dyn_cast_or_null<GlobalVariable>(
+        VariableModule->getOrInsertGlobal(EmuTlsVarName, VoidPtrType));
+  Entry.Node = DAG.getGlobalAddress(EmuTlsVar, dl, PtrVT);
+  Entry.Ty = VoidPtrType;
+  Args.push_back(Entry);
+
+  SDValue EmuTlsGetAddr = DAG.getExternalSymbol("__emutls_get_address", PtrVT);
+
+  TargetLowering::CallLoweringInfo CLI(DAG);
+  CLI.setDebugLoc(dl).setChain(DAG.getEntryNode());
+  CLI.setCallee(CallingConv::C, VoidPtrType, EmuTlsGetAddr, std::move(Args), 0);
+  std::pair<SDValue, SDValue> CallResult = LowerCallTo(CLI);
+
+  // TLSADDR will be codegen'ed as call. Inform MFI that function has calls.
+  // At last for X86 targets, maybe good for other targets too?
+  MachineFrameInfo *MFI = DAG.getMachineFunction().getFrameInfo();
+  MFI->setAdjustsStack(true);  // Is this only for X86 target?
+  MFI->setHasCalls(true);
+
+  assert((GA->getOffset() == 0) &&
+         "Emulated TLS must have zero offset in GlobalAddressSDNode");
+  return CallResult.first;
+}
diff --git a/lib/CodeGen/SelectionDAG/TargetSelectionDAGInfo.cpp b/lib/CodeGen/SelectionDAG/TargetSelectionDAGInfo.cpp
index 0e89bad5f26..00db9425684 100644
--- a/lib/CodeGen/SelectionDAG/TargetSelectionDAGInfo.cpp
+++ b/lib/CodeGen/SelectionDAG/TargetSelectionDAGInfo.cpp
@@ -15,9 +15,5 @@
 #include "llvm/Target/TargetMachine.h"
 using namespace llvm;
 
-TargetSelectionDAGInfo::TargetSelectionDAGInfo(const DataLayout *DL)
-  : DL(DL) {
-}
-
 TargetSelectionDAGInfo::~TargetSelectionDAGInfo() {
 }
diff --git a/lib/CodeGen/ShadowStackGCLowering.cpp b/lib/CodeGen/ShadowStackGCLowering.cpp
index 7c0b2bb4569..e7b2a8e72d2 100644
--- a/lib/CodeGen/ShadowStackGCLowering.cpp
+++ b/lib/CodeGen/ShadowStackGCLowering.cpp
@@ -144,10 +144,14 @@ class EscapeEnumerator {
       BasicBlock *CleanupBB = BasicBlock::Create(C, CleanupBBName, &F);
       Type *ExnTy =
           StructType::get(Type::getInt8PtrTy(C), Type::getInt32Ty(C), nullptr);
-      Constant *PersFn = F.getParent()->getOrInsertFunction(
-          "__gcc_personality_v0", FunctionType::get(Type::getInt32Ty(C), true));
+      if (!F.hasPersonalityFn()) {
+        Constant *PersFn = F.getParent()->getOrInsertFunction(
+            "__gcc_personality_v0",
+            FunctionType::get(Type::getInt32Ty(C), true));
+        F.setPersonalityFn(PersFn);
+      }
       LandingPadInst *LPad =
-          LandingPadInst::Create(ExnTy, PersFn, 1, "cleanup.lpad", CleanupBB);
+          LandingPadInst::Create(ExnTy, 1, "cleanup.lpad", CleanupBB);
       LPad->setCleanup(true);
       ResumeInst *RI = ResumeInst::Create(LPad, CleanupBB);
 
diff --git a/lib/CodeGen/ShrinkWrap.cpp b/lib/CodeGen/ShrinkWrap.cpp
new file mode 100644
index 00000000000..4463cc7d3c5
--- /dev/null
+++ b/lib/CodeGen/ShrinkWrap.cpp
@@ -0,0 +1,388 @@
+//===-- ShrinkWrap.cpp - Compute safe point for prolog/epilog insertion ---===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This pass looks for safe point where the prologue and epilogue can be
+// inserted.
+// The safe point for the prologue (resp. epilogue) is called Save
+// (resp. Restore).
+// A point is safe for prologue (resp. epilogue) if and only if
+// it 1) dominates (resp. post-dominates) all the frame related operations and
+// between 2) two executions of the Save (resp. Restore) point there is an
+// execution of the Restore (resp. Save) point.
+//
+// For instance, the following points are safe:
+// for (int i = 0; i < 10; ++i) {
+//   Save
+//   ...
+//   Restore
+// }
+// Indeed, the execution looks like Save -> Restore -> Save -> Restore ...
+// And the following points are not:
+// for (int i = 0; i < 10; ++i) {
+//   Save
+//   ...
+// }
+// for (int i = 0; i < 10; ++i) {
+//   ...
+//   Restore
+// }
+// Indeed, the execution looks like Save -> Save -> ... -> Restore -> Restore.
+//
+// This pass also ensures that the safe points are 3) cheaper than the regular
+// entry and exits blocks.
+//
+// Property #1 is ensured via the use of MachineDominatorTree and
+// MachinePostDominatorTree.
+// Property #2 is ensured via property #1 and MachineLoopInfo, i.e., both
+// points must be in the same loop.
+// Property #3 is ensured via the MachineBlockFrequencyInfo.
+//
+// If this pass found points matching all this properties, then
+// MachineFrameInfo is updated this that information.
+//===----------------------------------------------------------------------===//
+#include "llvm/ADT/Statistic.h"
+// To check for profitability.
+#include "llvm/CodeGen/MachineBlockFrequencyInfo.h"
+// For property #1 for Save.
+#include "llvm/CodeGen/MachineDominators.h"
+#include "llvm/CodeGen/MachineFunctionPass.h"
+// To record the result of the analysis.
+#include "llvm/CodeGen/MachineFrameInfo.h"
+// For property #2.
+#include "llvm/CodeGen/MachineLoopInfo.h"
+// For property #1 for Restore.
+#include "llvm/CodeGen/MachinePostDominators.h"
+#include "llvm/CodeGen/Passes.h"
+// To know about callee-saved.
+#include "llvm/CodeGen/RegisterClassInfo.h"
+#include "llvm/Support/Debug.h"
+// To query the target about frame lowering.
+#include "llvm/Target/TargetFrameLowering.h"
+// To know about frame setup operation.
+#include "llvm/Target/TargetInstrInfo.h"
+// To access TargetInstrInfo.
+#include "llvm/Target/TargetSubtargetInfo.h"
+
+#define DEBUG_TYPE "shrink-wrap"
+
+using namespace llvm;
+
+STATISTIC(NumFunc, "Number of functions");
+STATISTIC(NumCandidates, "Number of shrink-wrapping candidates");
+STATISTIC(NumCandidatesDropped,
+          "Number of shrink-wrapping candidates dropped because of frequency");
+
+namespace {
+/// \brief Class to determine where the safe point to insert the
+/// prologue and epilogue are.
+/// Unlike the paper from Fred C. Chow, PLDI'88, that introduces the
+/// shrink-wrapping term for prologue/epilogue placement, this pass
+/// does not rely on expensive data-flow analysis. Instead we use the
+/// dominance properties and loop information to decide which point
+/// are safe for such insertion.
+class ShrinkWrap : public MachineFunctionPass {
+  /// Hold callee-saved information.
+  RegisterClassInfo RCI;
+  MachineDominatorTree *MDT;
+  MachinePostDominatorTree *MPDT;
+  /// Current safe point found for the prologue.
+  /// The prologue will be inserted before the first instruction
+  /// in this basic block.
+  MachineBasicBlock *Save;
+  /// Current safe point found for the epilogue.
+  /// The epilogue will be inserted before the first terminator instruction
+  /// in this basic block.
+  MachineBasicBlock *Restore;
+  /// Hold the information of the basic block frequency.
+  /// Use to check the profitability of the new points.
+  MachineBlockFrequencyInfo *MBFI;
+  /// Hold the loop information. Used to determine if Save and Restore
+  /// are in the same loop.
+  MachineLoopInfo *MLI;
+  /// Frequency of the Entry block.
+  uint64_t EntryFreq;
+  /// Current opcode for frame setup.
+  unsigned FrameSetupOpcode;
+  /// Current opcode for frame destroy.
+  unsigned FrameDestroyOpcode;
+  /// Entry block.
+  const MachineBasicBlock *Entry;
+
+  /// \brief Check if \p MI uses or defines a callee-saved register or
+  /// a frame index. If this is the case, this means \p MI must happen
+  /// after Save and before Restore.
+  bool useOrDefCSROrFI(const MachineInstr &MI) const;
+
+  /// \brief Update the Save and Restore points such that \p MBB is in
+  /// the region that is dominated by Save and post-dominated by Restore
+  /// and Save and Restore still match the safe point definition.
+  /// Such point may not exist and Save and/or Restore may be null after
+  /// this call.
+  void updateSaveRestorePoints(MachineBasicBlock &MBB);
+
+  /// \brief Initialize the pass for \p MF.
+  void init(MachineFunction &MF) {
+    RCI.runOnMachineFunction(MF);
+    MDT = &getAnalysis<MachineDominatorTree>();
+    MPDT = &getAnalysis<MachinePostDominatorTree>();
+    Save = nullptr;
+    Restore = nullptr;
+    MBFI = &getAnalysis<MachineBlockFrequencyInfo>();
+    MLI = &getAnalysis<MachineLoopInfo>();
+    EntryFreq = MBFI->getEntryFreq();
+    const TargetInstrInfo &TII = *MF.getSubtarget().getInstrInfo();
+    FrameSetupOpcode = TII.getCallFrameSetupOpcode();
+    FrameDestroyOpcode = TII.getCallFrameDestroyOpcode();
+    Entry = &MF.front();
+
+    ++NumFunc;
+  }
+
+  /// Check whether or not Save and Restore points are still interesting for
+  /// shrink-wrapping.
+  bool ArePointsInteresting() const { return Save != Entry && Save && Restore; }
+
+public:
+  static char ID;
+
+  ShrinkWrap() : MachineFunctionPass(ID) {
+    initializeShrinkWrapPass(*PassRegistry::getPassRegistry());
+  }
+
+  void getAnalysisUsage(AnalysisUsage &AU) const override {
+    AU.setPreservesAll();
+    AU.addRequired<MachineBlockFrequencyInfo>();
+    AU.addRequired<MachineDominatorTree>();
+    AU.addRequired<MachinePostDominatorTree>();
+    AU.addRequired<MachineLoopInfo>();
+    MachineFunctionPass::getAnalysisUsage(AU);
+  }
+
+  const char *getPassName() const override {
+    return "Shrink Wrapping analysis";
+  }
+
+  /// \brief Perform the shrink-wrapping analysis and update
+  /// the MachineFrameInfo attached to \p MF with the results.
+  bool runOnMachineFunction(MachineFunction &MF) override;
+};
+} // End anonymous namespace.
+
+char ShrinkWrap::ID = 0;
+char &llvm::ShrinkWrapID = ShrinkWrap::ID;
+
+INITIALIZE_PASS_BEGIN(ShrinkWrap, "shrink-wrap", "Shrink Wrap Pass", false,
+                      false)
+INITIALIZE_PASS_DEPENDENCY(MachineBlockFrequencyInfo)
+INITIALIZE_PASS_DEPENDENCY(MachineDominatorTree)
+INITIALIZE_PASS_DEPENDENCY(MachinePostDominatorTree)
+INITIALIZE_PASS_DEPENDENCY(MachineLoopInfo)
+INITIALIZE_PASS_END(ShrinkWrap, "shrink-wrap", "Shrink Wrap Pass", false, false)
+
+bool ShrinkWrap::useOrDefCSROrFI(const MachineInstr &MI) const {
+  if (MI.getOpcode() == FrameSetupOpcode ||
+      MI.getOpcode() == FrameDestroyOpcode) {
+    DEBUG(dbgs() << "Frame instruction: " << MI << '\n');
+    return true;
+  }
+  for (const MachineOperand &MO : MI.operands()) {
+    bool UseCSR = false;
+    if (MO.isReg()) {
+      unsigned PhysReg = MO.getReg();
+      if (!PhysReg)
+        continue;
+      assert(TargetRegisterInfo::isPhysicalRegister(PhysReg) &&
+             "Unallocated register?!");
+      UseCSR = RCI.getLastCalleeSavedAlias(PhysReg);
+    }
+    // TODO: Handle regmask more accurately.
+    // For now, be conservative about them.
+    if (UseCSR || MO.isFI() || MO.isRegMask()) {
+      DEBUG(dbgs() << "Use or define CSR(" << UseCSR << ") or FI(" << MO.isFI()
+                   << "): " << MI << '\n');
+      return true;
+    }
+  }
+  return false;
+}
+
+/// \brief Helper function to find the immediate (post) dominator.
+template <typename ListOfBBs, typename DominanceAnalysis>
+MachineBasicBlock *FindIDom(MachineBasicBlock &Block, ListOfBBs BBs,
+                            DominanceAnalysis &Dom) {
+  MachineBasicBlock *IDom = &Block;
+  for (MachineBasicBlock *BB : BBs) {
+    IDom = Dom.findNearestCommonDominator(IDom, BB);
+    if (!IDom)
+      break;
+  }
+  return IDom;
+}
+
+void ShrinkWrap::updateSaveRestorePoints(MachineBasicBlock &MBB) {
+  // Get rid of the easy cases first.
+  if (!Save)
+    Save = &MBB;
+  else
+    Save = MDT->findNearestCommonDominator(Save, &MBB);
+
+  if (!Save) {
+    DEBUG(dbgs() << "Found a block that is not reachable from Entry\n");
+    return;
+  }
+
+  if (!Restore)
+    Restore = &MBB;
+  else
+    Restore = MPDT->findNearestCommonDominator(Restore, &MBB);
+
+  // Make sure we would be able to insert the restore code before the
+  // terminator.
+  if (Restore == &MBB) {
+    for (const MachineInstr &Terminator : MBB.terminators()) {
+      if (!useOrDefCSROrFI(Terminator))
+        continue;
+      // One of the terminator needs to happen before the restore point.
+      if (MBB.succ_empty()) {
+        Restore = nullptr;
+        break;
+      }
+      // Look for a restore point that post-dominates all the successors.
+      // The immediate post-dominator is what we are looking for.
+      Restore = FindIDom<>(*Restore, Restore->successors(), *MPDT);
+      break;
+    }
+  }
+
+  if (!Restore) {
+    DEBUG(dbgs() << "Restore point needs to be spanned on several blocks\n");
+    return;
+  }
+
+  // Make sure Save and Restore are suitable for shrink-wrapping:
+  // 1. all path from Save needs to lead to Restore before exiting.
+  // 2. all path to Restore needs to go through Save from Entry.
+  // We achieve that by making sure that:
+  // A. Save dominates Restore.
+  // B. Restore post-dominates Save.
+  // C. Save and Restore are in the same loop.
+  bool SaveDominatesRestore = false;
+  bool RestorePostDominatesSave = false;
+  while (Save && Restore &&
+         (!(SaveDominatesRestore = MDT->dominates(Save, Restore)) ||
+          !(RestorePostDominatesSave = MPDT->dominates(Restore, Save)) ||
+          MLI->getLoopFor(Save) != MLI->getLoopFor(Restore))) {
+    // Fix (A).
+    if (!SaveDominatesRestore) {
+      Save = MDT->findNearestCommonDominator(Save, Restore);
+      continue;
+    }
+    // Fix (B).
+    if (!RestorePostDominatesSave)
+      Restore = MPDT->findNearestCommonDominator(Restore, Save);
+
+    // Fix (C).
+    if (Save && Restore && Save != Restore &&
+        MLI->getLoopFor(Save) != MLI->getLoopFor(Restore)) {
+      if (MLI->getLoopDepth(Save) > MLI->getLoopDepth(Restore))
+        // Push Save outside of this loop.
+        Save = FindIDom<>(*Save, Save->predecessors(), *MDT);
+      else
+        // Push Restore outside of this loop.
+        Restore = FindIDom<>(*Restore, Restore->successors(), *MPDT);
+    }
+  }
+}
+
+bool ShrinkWrap::runOnMachineFunction(MachineFunction &MF) {
+  if (MF.empty())
+    return false;
+  DEBUG(dbgs() << "**** Analysing " << MF.getName() << '\n');
+
+  init(MF);
+
+  for (MachineBasicBlock &MBB : MF) {
+    DEBUG(dbgs() << "Look into: " << MBB.getNumber() << ' ' << MBB.getName()
+                 << '\n');
+
+    for (const MachineInstr &MI : MBB) {
+      if (!useOrDefCSROrFI(MI))
+        continue;
+      // Save (resp. restore) point must dominate (resp. post dominate)
+      // MI. Look for the proper basic block for those.
+      updateSaveRestorePoints(MBB);
+      // If we are at a point where we cannot improve the placement of
+      // save/restore instructions, just give up.
+      if (!ArePointsInteresting()) {
+        DEBUG(dbgs() << "No Shrink wrap candidate found\n");
+        return false;
+      }
+      // No need to look for other instructions, this basic block
+      // will already be part of the handled region.
+      break;
+    }
+  }
+  if (!ArePointsInteresting()) {
+    // If the points are not interesting at this point, then they must be null
+    // because it means we did not encounter any frame/CSR related code.
+    // Otherwise, we would have returned from the previous loop.
+    assert(!Save && !Restore && "We miss a shrink-wrap opportunity?!");
+    DEBUG(dbgs() << "Nothing to shrink-wrap\n");
+    return false;
+  }
+
+  DEBUG(dbgs() << "\n ** Results **\nFrequency of the Entry: " << EntryFreq
+               << '\n');
+
+  const TargetFrameLowering *TFI = MF.getSubtarget().getFrameLowering();
+  do {
+    DEBUG(dbgs() << "Shrink wrap candidates (#, Name, Freq):\nSave: "
+                 << Save->getNumber() << ' ' << Save->getName() << ' '
+                 << MBFI->getBlockFreq(Save).getFrequency() << "\nRestore: "
+                 << Restore->getNumber() << ' ' << Restore->getName() << ' '
+                 << MBFI->getBlockFreq(Restore).getFrequency() << '\n');
+
+    bool IsSaveCheap, TargetCanUseSaveAsPrologue = false;
+    if (((IsSaveCheap = EntryFreq >= MBFI->getBlockFreq(Save).getFrequency()) &&
+         EntryFreq >= MBFI->getBlockFreq(Restore).getFrequency()) &&
+        ((TargetCanUseSaveAsPrologue = TFI->canUseAsPrologue(*Save)) &&
+         TFI->canUseAsEpilogue(*Restore)))
+      break;
+    DEBUG(dbgs() << "New points are too expensive or invalid for the target\n");
+    MachineBasicBlock *NewBB;
+    if (!IsSaveCheap || !TargetCanUseSaveAsPrologue) {
+      Save = FindIDom<>(*Save, Save->predecessors(), *MDT);
+      if (!Save)
+        break;
+      NewBB = Save;
+    } else {
+      // Restore is expensive.
+      Restore = FindIDom<>(*Restore, Restore->successors(), *MPDT);
+      if (!Restore)
+        break;
+      NewBB = Restore;
+    }
+    updateSaveRestorePoints(*NewBB);
+  } while (Save && Restore);
+
+  if (!ArePointsInteresting()) {
+    ++NumCandidatesDropped;
+    return false;
+  }
+
+  DEBUG(dbgs() << "Final shrink wrap candidates:\nSave: " << Save->getNumber()
+               << ' ' << Save->getName() << "\nRestore: "
+               << Restore->getNumber() << ' ' << Restore->getName() << '\n');
+
+  MachineFrameInfo *MFI = MF.getFrameInfo();
+  MFI->setSavePoint(Save);
+  MFI->setRestorePoint(Restore);
+  ++NumCandidates;
+  return false;
+}
diff --git a/lib/CodeGen/SjLjEHPrepare.cpp b/lib/CodeGen/SjLjEHPrepare.cpp
index 0635173681e..2a16b649395 100644
--- a/lib/CodeGen/SjLjEHPrepare.cpp
+++ b/lib/CodeGen/SjLjEHPrepare.cpp
@@ -45,13 +45,12 @@ STATISTIC(NumSpilled, "Number of registers live across unwind edges");
 
 namespace {
 class SjLjEHPrepare : public FunctionPass {
-  const TargetMachine *TM;
   Type *doubleUnderDataTy;
   Type *doubleUnderJBufTy;
   Type *FunctionContextTy;
   Constant *RegisterFn;
   Constant *UnregisterFn;
-  Constant *BuiltinSetjmpFn;
+  Constant *BuiltinSetupDispatchFn;
   Constant *FrameAddrFn;
   Constant *StackAddrFn;
   Constant *StackRestoreFn;
@@ -63,7 +62,7 @@ class SjLjEHPrepare : public FunctionPass {
 
 public:
   static char ID; // Pass identification, replacement for typeid
-  explicit SjLjEHPrepare(const TargetMachine *TM) : FunctionPass(ID), TM(TM) {}
+  explicit SjLjEHPrepare() : FunctionPass(ID) {}
   bool doInitialization(Module &M) override;
   bool runOnFunction(Function &F) override;
 
@@ -83,11 +82,11 @@ class SjLjEHPrepare : public FunctionPass {
 } // end anonymous namespace
 
 char SjLjEHPrepare::ID = 0;
+INITIALIZE_PASS(SjLjEHPrepare, "sjljehprepare", "Prepare SjLj exceptions",
+                false, false)
 
 // Public Interface To the SjLjEHPrepare pass.
-FunctionPass *llvm::createSjLjEHPreparePass(const TargetMachine *TM) {
-  return new SjLjEHPrepare(TM);
-}
+FunctionPass *llvm::createSjLjEHPreparePass() { return new SjLjEHPrepare(); }
 // doInitialization - Set up decalarations and types needed to process
 // exceptions.
 bool SjLjEHPrepare::doInitialization(Module &M) {
@@ -113,7 +112,8 @@ bool SjLjEHPrepare::doInitialization(Module &M) {
   FrameAddrFn = Intrinsic::getDeclaration(&M, Intrinsic::frameaddress);
   StackAddrFn = Intrinsic::getDeclaration(&M, Intrinsic::stacksave);
   StackRestoreFn = Intrinsic::getDeclaration(&M, Intrinsic::stackrestore);
-  BuiltinSetjmpFn = Intrinsic::getDeclaration(&M, Intrinsic::eh_sjlj_setjmp);
+  BuiltinSetupDispatchFn =
+    Intrinsic::getDeclaration(&M, Intrinsic::eh_sjlj_setup_dispatch);
   LSDAAddrFn = Intrinsic::getDeclaration(&M, Intrinsic::eh_sjlj_lsda);
   CallSiteFn = Intrinsic::getDeclaration(&M, Intrinsic::eh_sjlj_callsite);
   FuncCtxFn = Intrinsic::getDeclaration(&M, Intrinsic::eh_sjlj_functioncontext);
@@ -196,9 +196,8 @@ Value *SjLjEHPrepare::setupFunctionContext(Function &F,
   // Create an alloca for the incoming jump buffer ptr and the new jump buffer
   // that needs to be restored on all exits from the function. This is an alloca
   // because the value needs to be added to the global context list.
-  const TargetLowering *TLI = TM->getSubtargetImpl(F)->getTargetLowering();
-  unsigned Align =
-      TLI->getDataLayout()->getPrefTypeAlignment(FunctionContextTy);
+  auto &DL = F.getParent()->getDataLayout();
+  unsigned Align = DL.getPrefTypeAlignment(FunctionContextTy);
   FuncCtx = new AllocaInst(FunctionContextTy, nullptr, Align, "fn_context",
                            EntryBB->begin());
 
@@ -227,7 +226,7 @@ Value *SjLjEHPrepare::setupFunctionContext(Function &F,
   // Personality function
   IRBuilder<> Builder(EntryBB->getTerminator());
   if (!PersonalityFn)
-    PersonalityFn = LPads[0]->getPersonalityFn();
+    PersonalityFn = F.getPersonalityFn();
   Value *PersonalityFieldPtr = Builder.CreateConstGEP2_32(
       FunctionContextTy, FuncCtx, 0, 3, "pers_fn_gep");
   Builder.CreateStore(
@@ -235,7 +234,7 @@ Value *SjLjEHPrepare::setupFunctionContext(Function &F,
       PersonalityFieldPtr, /*isVolatile=*/true);
 
   // LSDA address
-  Value *LSDA = Builder.CreateCall(LSDAAddrFn, "lsda_addr");
+  Value *LSDA = Builder.CreateCall(LSDAAddrFn, {}, "lsda_addr");
   Value *LSDAFieldPtr =
       Builder.CreateConstGEP2_32(FunctionContextTy, FuncCtx, 0, 4, "lsda_gep");
   Builder.CreateStore(LSDA, LSDAFieldPtr, /*isVolatile=*/true);
@@ -420,12 +419,11 @@ bool SjLjEHPrepare::setupEntryBlockAndCallSites(Function &F) {
   Value *StackPtr = Builder.CreateConstGEP2_32(doubleUnderJBufTy, JBufPtr, 0, 2,
                                                "jbuf_sp_gep");
 
-  Val = Builder.CreateCall(StackAddrFn, "sp");
+  Val = Builder.CreateCall(StackAddrFn, {}, "sp");
   Builder.CreateStore(Val, StackPtr, /*isVolatile=*/true);
 
-  // Call the setjmp instrinsic. It fills in the rest of the jmpbuf.
-  Value *SetjmpArg = Builder.CreateBitCast(JBufPtr, Builder.getInt8PtrTy());
-  Builder.CreateCall(BuiltinSetjmpFn, SetjmpArg);
+  // Call the setup_dispatch instrinsic. It fills in the rest of the jmpbuf.
+  Builder.CreateCall(BuiltinSetupDispatchFn, {});
 
   // Store a pointer to the function context so that the back-end will know
   // where to look for it.
diff --git a/lib/CodeGen/SplitKit.h b/lib/CodeGen/SplitKit.h
index 2e60c1445b8..69c65ff3f61 100644
--- a/lib/CodeGen/SplitKit.h
+++ b/lib/CodeGen/SplitKit.h
@@ -39,7 +39,7 @@ class raw_ostream;
 
 /// SplitAnalysis - Analyze a LiveInterval, looking for live range splitting
 /// opportunities.
-class SplitAnalysis {
+class LLVM_LIBRARY_VISIBILITY SplitAnalysis {
 public:
   const MachineFunction &MF;
   const VirtRegMap &VRM;
@@ -208,7 +208,7 @@ class SplitAnalysis {
 /// - Finish the current interval with closeIntv and repeat from 2.
 /// - Rewrite instructions with finish().
 ///
-class SplitEditor {
+class LLVM_LIBRARY_VISIBILITY SplitEditor {
   SplitAnalysis &SA;
   LiveIntervals &LIS;
   VirtRegMap &VRM;
@@ -419,7 +419,7 @@ class SplitEditor {
   ///              There may be extra indices created by dead code elimination.
   void finish(SmallVectorImpl<unsigned> *LRMap = nullptr);
 
-  /// dump - print the current interval maping to dbgs().
+  /// dump - print the current interval mapping to dbgs().
   void dump() const;
 
   // ===--- High level methods ---===
diff --git a/lib/CodeGen/StackColoring.cpp b/lib/CodeGen/StackColoring.cpp
index 2bf2d642e1c..3541b33a844 100644
--- a/lib/CodeGen/StackColoring.cpp
+++ b/lib/CodeGen/StackColoring.cpp
@@ -48,7 +48,6 @@
 #include "llvm/IR/Function.h"
 #include "llvm/IR/Instructions.h"
 #include "llvm/IR/Module.h"
-#include "llvm/MC/MCInstrItineraries.h"
 #include "llvm/Support/CommandLine.h"
 #include "llvm/Support/Debug.h"
 #include "llvm/Support/raw_ostream.h"
@@ -464,7 +463,7 @@ void StackColoring::remapInstructions(DenseMap<int, int> &SlotRemap) {
       continue;
     if (SlotRemap.count(VI.Slot)) {
       DEBUG(dbgs() << "Remapping debug info for ["
-                   << cast<MDLocalVariable>(VI.Var)->getName() << "].\n");
+                   << cast<DILocalVariable>(VI.Var)->getName() << "].\n");
       VI.Slot = SlotRemap[VI.Slot];
       FixedDbg++;
     }
diff --git a/lib/CodeGen/StackMapLivenessAnalysis.cpp b/lib/CodeGen/StackMapLivenessAnalysis.cpp
index d88be575d56..855058358fe 100644
--- a/lib/CodeGen/StackMapLivenessAnalysis.cpp
+++ b/lib/CodeGen/StackMapLivenessAnalysis.cpp
@@ -49,7 +49,6 @@ namespace {
 /// information provided by this pass is optional and not required by the
 /// aformentioned intrinsic to function.
 class StackMapLiveness : public MachineFunctionPass {
-  MachineFunction *MF;
   const TargetRegisterInfo *TRI;
   LivePhysRegs LiveRegs;
 
@@ -68,14 +67,14 @@ class StackMapLiveness : public MachineFunctionPass {
 
 private:
   /// \brief Performs the actual liveness calculation for the function.
-  bool calculateLiveness();
+  bool calculateLiveness(MachineFunction &MF);
 
   /// \brief Add the current register live set to the instruction.
-  void addLiveOutSetToMI(MachineInstr &MI);
+  void addLiveOutSetToMI(MachineFunction &MF, MachineInstr &MI);
 
   /// \brief Create a register mask and initialize it with the registers from
   /// the register live set.
-  uint32_t *createRegisterMask() const;
+  uint32_t *createRegisterMask(MachineFunction &MF) const;
 };
 } // namespace
 
@@ -95,8 +94,7 @@ void StackMapLiveness::getAnalysisUsage(AnalysisUsage &AU) const {
   // We preserve all information.
   AU.setPreservesAll();
   AU.setPreservesCFG();
-  // Default dependencie for all MachineFunction passes.
-  AU.addRequired<MachineFunctionAnalysis>();
+  MachineFunctionPass::getAnalysisUsage(AU);
 }
 
 /// Calculate the liveness information for the given machine function.
@@ -106,7 +104,6 @@ bool StackMapLiveness::runOnMachineFunction(MachineFunction &MF) {
 
   DEBUG(dbgs() << "********** COMPUTING STACKMAP LIVENESS: " << MF.getName()
                << " **********\n");
-  this->MF = &MF;
   TRI = MF.getSubtarget().getRegisterInfo();
   ++NumStackMapFuncVisited;
 
@@ -115,25 +112,23 @@ bool StackMapLiveness::runOnMachineFunction(MachineFunction &MF) {
     ++NumStackMapFuncSkipped;
     return false;
   }
-  return calculateLiveness();
+  return calculateLiveness(MF);
 }
 
 /// Performs the actual liveness calculation for the function.
-bool StackMapLiveness::calculateLiveness() {
+bool StackMapLiveness::calculateLiveness(MachineFunction &MF) {
   bool HasChanged = false;
   // For all basic blocks in the function.
-  for (MachineFunction::iterator MBBI = MF->begin(), MBBE = MF->end();
-       MBBI != MBBE; ++MBBI) {
-    DEBUG(dbgs() << "****** BB " << MBBI->getName() << " ******\n");
+  for (auto &MBB : MF) {
+    DEBUG(dbgs() << "****** BB " << MBB.getName() << " ******\n");
     LiveRegs.init(TRI);
-    LiveRegs.addLiveOuts(MBBI);
+    LiveRegs.addLiveOuts(&MBB);
     bool HasStackMap = false;
     // Reverse iterate over all instructions and add the current live register
     // set to an instruction if we encounter a patchpoint instruction.
-    for (MachineBasicBlock::reverse_iterator I = MBBI->rbegin(),
-         E = MBBI->rend(); I != E; ++I) {
+    for (auto I = MBB.rbegin(), E = MBB.rend(); I != E; ++I) {
       if (I->getOpcode() == TargetOpcode::PATCHPOINT) {
-        addLiveOutSetToMI(*I);
+        addLiveOutSetToMI(MF, *I);
         HasChanged = true;
         HasStackMap = true;
         ++NumStackMaps;
@@ -149,21 +144,23 @@ bool StackMapLiveness::calculateLiveness() {
 }
 
 /// Add the current register live set to the instruction.
-void StackMapLiveness::addLiveOutSetToMI(MachineInstr &MI) {
-  uint32_t *Mask = createRegisterMask();
+void StackMapLiveness::addLiveOutSetToMI(MachineFunction &MF,
+                                         MachineInstr &MI) {
+  uint32_t *Mask = createRegisterMask(MF);
   MachineOperand MO = MachineOperand::CreateRegLiveOut(Mask);
-  MI.addOperand(*MF, MO);
+  MI.addOperand(MF, MO);
 }
 
 /// Create a register mask and initialize it with the registers from the
 /// register live set.
-uint32_t *StackMapLiveness::createRegisterMask() const {
+uint32_t *StackMapLiveness::createRegisterMask(MachineFunction &MF) const {
   // The mask is owned and cleaned up by the Machine Function.
-  uint32_t *Mask = MF->allocateRegisterMask(TRI->getNumRegs());
-  for (LivePhysRegs::const_iterator RI = LiveRegs.begin(), RE = LiveRegs.end();
-       RI != RE; ++RI)
-    Mask[*RI / 32] |= 1U << (*RI % 32);
+  uint32_t *Mask = MF.allocateRegisterMask(TRI->getNumRegs());
+  for (auto Reg : LiveRegs)
+    Mask[Reg / 32] |= 1U << (Reg % 32);
 
+  // Give the target a chance to adjust the mask.
   TRI->adjustStackMapLiveOutMask(Mask);
+
   return Mask;
 }
diff --git a/lib/CodeGen/StackMaps.cpp b/lib/CodeGen/StackMaps.cpp
index aa18dea12a4..b3cd8b3d80b 100644
--- a/lib/CodeGen/StackMaps.cpp
+++ b/lib/CodeGen/StackMaps.cpp
@@ -29,17 +29,17 @@ using namespace llvm;
 
 #define DEBUG_TYPE "stackmaps"
 
-static cl::opt<int> StackMapVersion("stackmap-version", cl::init(1),
-  cl::desc("Specify the stackmap encoding version (default = 1)"));
+static cl::opt<int> StackMapVersion(
+    "stackmap-version", cl::init(1),
+    cl::desc("Specify the stackmap encoding version (default = 1)"));
 
 const char *StackMaps::WSMP = "Stack Maps: ";
 
 PatchPointOpers::PatchPointOpers(const MachineInstr *MI)
-  : MI(MI),
-    HasDef(MI->getOperand(0).isReg() && MI->getOperand(0).isDef() &&
-           !MI->getOperand(0).isImplicit()),
-    IsAnyReg(MI->getOperand(getMetaIdx(CCPos)).getImm() == CallingConv::AnyReg)
-{
+    : MI(MI), HasDef(MI->getOperand(0).isReg() && MI->getOperand(0).isDef() &&
+                     !MI->getOperand(0).isImplicit()),
+      IsAnyReg(MI->getOperand(getMetaIdx(CCPos)).getImm() ==
+               CallingConv::AnyReg) {
 #ifndef NDEBUG
   unsigned CheckStartIdx = 0, e = MI->getNumOperands();
   while (CheckStartIdx < e && MI->getOperand(CheckStartIdx).isReg() &&
@@ -76,30 +76,33 @@ StackMaps::StackMaps(AsmPrinter &AP) : AP(AP) {
 
 /// Go up the super-register chain until we hit a valid dwarf register number.
 static unsigned getDwarfRegNum(unsigned Reg, const TargetRegisterInfo *TRI) {
-  int RegNo = TRI->getDwarfRegNum(Reg, false);
-  for (MCSuperRegIterator SR(Reg, TRI); SR.isValid() && RegNo < 0; ++SR)
-    RegNo = TRI->getDwarfRegNum(*SR, false);
+  int RegNum = TRI->getDwarfRegNum(Reg, false);
+  for (MCSuperRegIterator SR(Reg, TRI); SR.isValid() && RegNum < 0; ++SR)
+    RegNum = TRI->getDwarfRegNum(*SR, false);
 
-  assert(RegNo >= 0 && "Invalid Dwarf register number.");
-  return (unsigned) RegNo;
+  assert(RegNum >= 0 && "Invalid Dwarf register number.");
+  return (unsigned)RegNum;
 }
 
 MachineInstr::const_mop_iterator
 StackMaps::parseOperand(MachineInstr::const_mop_iterator MOI,
-                        MachineInstr::const_mop_iterator MOE,
-                        LocationVec &Locs, LiveOutVec &LiveOuts) const {
+                        MachineInstr::const_mop_iterator MOE, LocationVec &Locs,
+                        LiveOutVec &LiveOuts) const {
   const TargetRegisterInfo *TRI = AP.MF->getSubtarget().getRegisterInfo();
   if (MOI->isImm()) {
     switch (MOI->getImm()) {
-    default: llvm_unreachable("Unrecognized operand type.");
+    default:
+      llvm_unreachable("Unrecognized operand type.");
     case StackMaps::DirectMemRefOp: {
-      unsigned Size = AP.TM.getDataLayout()->getPointerSizeInBits();
+      auto &DL = AP.MF->getDataLayout();
+
+      unsigned Size = DL.getPointerSizeInBits();
       assert((Size % 8) == 0 && "Need pointer size in bytes.");
       Size /= 8;
       unsigned Reg = (++MOI)->getReg();
       int64_t Imm = (++MOI)->getImm();
-      Locs.push_back(Location(StackMaps::Location::Direct, Size,
-                              getDwarfRegNum(Reg, TRI), Imm));
+      Locs.emplace_back(StackMaps::Location::Direct, Size,
+                        getDwarfRegNum(Reg, TRI), Imm);
       break;
     }
     case StackMaps::IndirectMemRefOp: {
@@ -107,15 +110,15 @@ StackMaps::parseOperand(MachineInstr::const_mop_iterator MOI,
       assert(Size > 0 && "Need a valid size for indirect memory locations.");
       unsigned Reg = (++MOI)->getReg();
       int64_t Imm = (++MOI)->getImm();
-      Locs.push_back(Location(StackMaps::Location::Indirect, Size,
-                              getDwarfRegNum(Reg, TRI), Imm));
+      Locs.emplace_back(StackMaps::Location::Indirect, Size,
+                        getDwarfRegNum(Reg, TRI), Imm);
       break;
     }
     case StackMaps::ConstantOp: {
       ++MOI;
       assert(MOI->isImm() && "Expected constant operand.");
       int64_t Imm = MOI->getImm();
-      Locs.push_back(Location(Location::Constant, sizeof(int64_t), 0, Imm));
+      Locs.emplace_back(Location::Constant, sizeof(int64_t), 0, Imm);
       break;
     }
     }
@@ -137,14 +140,13 @@ StackMaps::parseOperand(MachineInstr::const_mop_iterator MOI,
     assert(!MOI->getSubReg() && "Physical subreg still around.");
 
     unsigned Offset = 0;
-    unsigned RegNo = getDwarfRegNum(MOI->getReg(), TRI);
-    unsigned LLVMRegNo = TRI->getLLVMRegNum(RegNo, false);
-    unsigned SubRegIdx = TRI->getSubRegIndex(LLVMRegNo, MOI->getReg());
+    unsigned DwarfRegNum = getDwarfRegNum(MOI->getReg(), TRI);
+    unsigned LLVMRegNum = TRI->getLLVMRegNum(DwarfRegNum, false);
+    unsigned SubRegIdx = TRI->getSubRegIndex(LLVMRegNum, MOI->getReg());
     if (SubRegIdx)
       Offset = TRI->getSubRegIdxOffset(SubRegIdx);
 
-    Locs.push_back(
-      Location(Location::Register, RC->getSize(), RegNo, Offset));
+    Locs.emplace_back(Location::Register, RC->getSize(), DwarfRegNum, Offset);
     return ++MOI;
   }
 
@@ -165,19 +167,19 @@ void StackMaps::print(raw_ostream &OS) {
     OS << WSMP << "callsite " << CSI.ID << "\n";
     OS << WSMP << "  has " << CSLocs.size() << " locations\n";
 
-    unsigned OperIdx = 0;
+    unsigned Idx = 0;
     for (const auto &Loc : CSLocs) {
-      OS << WSMP << "  Loc " << OperIdx << ": ";
-      switch (Loc.LocType) {
+      OS << WSMP << "\t\tLoc " << Idx << ": ";
+      switch (Loc.Type) {
       case Location::Unprocessed:
         OS << "<Unprocessed operand>";
         break;
       case Location::Register:
         OS << "Register ";
-	if (TRI)
-	  OS << TRI->getName(Loc.Reg);
-	else
-	  OS << Loc.Reg;
+        if (TRI)
+          OS << TRI->getName(Loc.Reg);
+        else
+          OS << Loc.Reg;
         break;
       case Location::Direct:
         OS << "Direct ";
@@ -203,23 +205,23 @@ void StackMaps::print(raw_ostream &OS) {
         OS << "Constant Index " << Loc.Offset;
         break;
       }
-      OS << "     [encoding: .byte " << Loc.LocType << ", .byte " << Loc.Size
+      OS << "\t[encoding: .byte " << Loc.Type << ", .byte " << Loc.Size
          << ", .short " << Loc.Reg << ", .int " << Loc.Offset << "]\n";
-      OperIdx++;
+      Idx++;
     }
 
-    OS << WSMP << "  has " << LiveOuts.size() << " live-out registers\n";
+    OS << WSMP << "\thas " << LiveOuts.size() << " live-out registers\n";
 
-    OperIdx = 0;
+    Idx = 0;
     for (const auto &LO : LiveOuts) {
-      OS << WSMP << "  LO " << OperIdx << ": ";
+      OS << WSMP << "\t\tLO " << Idx << ": ";
       if (TRI)
         OS << TRI->getName(LO.Reg);
       else
         OS << LO.Reg;
-      OS << "      [encoding: .short " << LO.RegNo << ", .byte 0, .byte "
+      OS << "\t[encoding: .short " << LO.DwarfRegNum << ", .byte 0, .byte "
          << LO.Size << "]\n";
-      OperIdx++;
+      Idx++;
     }
   }
 }
@@ -227,9 +229,9 @@ void StackMaps::print(raw_ostream &OS) {
 /// Create a live-out register record for the given register Reg.
 StackMaps::LiveOutReg
 StackMaps::createLiveOutReg(unsigned Reg, const TargetRegisterInfo *TRI) const {
-  unsigned RegNo = getDwarfRegNum(Reg, TRI);
+  unsigned DwarfRegNum = getDwarfRegNum(Reg, TRI);
   unsigned Size = TRI->getMinimalPhysRegClass(Reg)->getSize();
-  return LiveOutReg(Reg, RegNo, Size);
+  return LiveOutReg(Reg, DwarfRegNum, Size);
 }
 
 /// Parse the register live-out mask and return a vector of live-out registers
@@ -248,11 +250,16 @@ StackMaps::parseRegisterLiveOutMask(const uint32_t *Mask) const {
   // We don't need to keep track of a register if its super-register is already
   // in the list. Merge entries that refer to the same dwarf register and use
   // the maximum size that needs to be spilled.
-  std::sort(LiveOuts.begin(), LiveOuts.end());
-  for (LiveOutVec::iterator I = LiveOuts.begin(), E = LiveOuts.end();
-       I != E; ++I) {
-    for (LiveOutVec::iterator II = std::next(I); II != E; ++II) {
-      if (I->RegNo != II->RegNo) {
+
+  std::sort(LiveOuts.begin(), LiveOuts.end(),
+            [](const LiveOutReg &LHS, const LiveOutReg &RHS) {
+              // Only sort by the dwarf register number.
+              return LHS.DwarfRegNum < RHS.DwarfRegNum;
+            });
+
+  for (auto I = LiveOuts.begin(), E = LiveOuts.end(); I != E; ++I) {
+    for (auto II = std::next(I); II != E; ++II) {
+      if (I->DwarfRegNum != II->DwarfRegNum) {
         // Skip all the now invalid entries.
         I = --II;
         break;
@@ -260,11 +267,15 @@ StackMaps::parseRegisterLiveOutMask(const uint32_t *Mask) const {
       I->Size = std::max(I->Size, II->Size);
       if (TRI->isSuperRegister(I->Reg, II->Reg))
         I->Reg = II->Reg;
-      II->MarkInvalid();
+      II->Reg = 0; // mark for deletion.
     }
   }
-  LiveOuts.erase(std::remove_if(LiveOuts.begin(), LiveOuts.end(),
-                                LiveOutReg::IsInvalid), LiveOuts.end());
+
+  LiveOuts.erase(
+      std::remove_if(LiveOuts.begin(), LiveOuts.end(),
+                     [](const LiveOutReg &LO) { return LO.Reg == 0; }),
+      LiveOuts.end());
+
   return LiveOuts;
 }
 
@@ -273,17 +284,17 @@ void StackMaps::recordStackMapOpers(const MachineInstr &MI, uint64_t ID,
                                     MachineInstr::const_mop_iterator MOE,
                                     bool recordResult) {
 
-  MCContext &OutContext = AP.OutStreamer.getContext();
-  MCSymbol *MILabel = OutContext.CreateTempSymbol();
-  AP.OutStreamer.EmitLabel(MILabel);
+  MCContext &OutContext = AP.OutStreamer->getContext();
+  MCSymbol *MILabel = OutContext.createTempSymbol();
+  AP.OutStreamer->EmitLabel(MILabel);
 
   LocationVec Locations;
   LiveOutVec LiveOuts;
 
   if (recordResult) {
     assert(PatchPointOpers(&MI).hasDef() && "Stackmap has no return value.");
-    parseOperand(MI.operands_begin(), std::next(MI.operands_begin()),
-                 Locations, LiveOuts);
+    parseOperand(MI.operands_begin(), std::next(MI.operands_begin()), Locations,
+                 LiveOuts);
   }
 
   // Parse operands.
@@ -292,33 +303,31 @@ void StackMaps::recordStackMapOpers(const MachineInstr &MI, uint64_t ID,
   }
 
   // Move large constants into the constant pool.
-  for (LocationVec::iterator I = Locations.begin(), E = Locations.end();
-       I != E; ++I) {
+  for (auto &Loc : Locations) {
     // Constants are encoded as sign-extended integers.
     // -1 is directly encoded as .long 0xFFFFFFFF with no constant pool.
-    if (I->LocType == Location::Constant && !isInt<32>(I->Offset)) {
-      I->LocType = Location::ConstantIndex;
+    if (Loc.Type == Location::Constant && !isInt<32>(Loc.Offset)) {
+      Loc.Type = Location::ConstantIndex;
       // ConstPool is intentionally a MapVector of 'uint64_t's (as
       // opposed to 'int64_t's).  We should never be in a situation
       // where we have to insert either the tombstone or the empty
       // keys into a map, and for a DenseMap<uint64_t, T> these are
       // (uint64_t)0 and (uint64_t)-1.  They can be and are
       // represented using 32 bit integers.
-
-      assert((uint64_t)I->Offset != DenseMapInfo<uint64_t>::getEmptyKey() &&
-             (uint64_t)I->Offset != DenseMapInfo<uint64_t>::getTombstoneKey() &&
+      assert((uint64_t)Loc.Offset != DenseMapInfo<uint64_t>::getEmptyKey() &&
+             (uint64_t)Loc.Offset !=
+                 DenseMapInfo<uint64_t>::getTombstoneKey() &&
              "empty and tombstone keys should fit in 32 bits!");
-      auto Result = ConstPool.insert(std::make_pair(I->Offset, I->Offset));
-      I->Offset = Result.first - ConstPool.begin();
+      auto Result = ConstPool.insert(std::make_pair(Loc.Offset, Loc.Offset));
+      Loc.Offset = Result.first - ConstPool.begin();
     }
   }
 
   // Create an expression to calculate the offset of the callsite from function
   // entry.
-  const MCExpr *CSOffsetExpr = MCBinaryExpr::CreateSub(
-    MCSymbolRefExpr::Create(MILabel, OutContext),
-    MCSymbolRefExpr::Create(AP.CurrentFnSymForSize, OutContext),
-    OutContext);
+  const MCExpr *CSOffsetExpr = MCBinaryExpr::createSub(
+      MCSymbolRefExpr::create(MILabel, OutContext),
+      MCSymbolRefExpr::create(AP.CurrentFnSymForSize, OutContext), OutContext);
 
   CSInfos.emplace_back(CSOffsetExpr, ID, std::move(Locations),
                        std::move(LiveOuts));
@@ -326,10 +335,10 @@ void StackMaps::recordStackMapOpers(const MachineInstr &MI, uint64_t ID,
   // Record the stack size of the current function.
   const MachineFrameInfo *MFI = AP.MF->getFrameInfo();
   const TargetRegisterInfo *RegInfo = AP.MF->getSubtarget().getRegisterInfo();
-  const bool DynamicFrameSize = MFI->hasVarSizedObjects() ||
-    RegInfo->needsStackRealignment(*(AP.MF));
+  bool HasDynamicFrameSize =
+      MFI->hasVarSizedObjects() || RegInfo->needsStackRealignment(*(AP.MF));
   FnStackSize[AP.CurrentFnSym] =
-    DynamicFrameSize ? UINT64_MAX : MFI->getStackSize();
+      HasDynamicFrameSize ? UINT64_MAX : MFI->getStackSize();
 }
 
 void StackMaps::recordStackMap(const MachineInstr &MI) {
@@ -346,33 +355,30 @@ void StackMaps::recordPatchPoint(const MachineInstr &MI) {
   PatchPointOpers opers(&MI);
   int64_t ID = opers.getMetaOper(PatchPointOpers::IDPos).getImm();
 
-  MachineInstr::const_mop_iterator MOI =
-    std::next(MI.operands_begin(), opers.getStackMapStartIdx());
+  auto MOI = std::next(MI.operands_begin(), opers.getStackMapStartIdx());
   recordStackMapOpers(MI, ID, MOI, MI.operands_end(),
                       opers.isAnyReg() && opers.hasDef());
 
 #ifndef NDEBUG
   // verify anyregcc
-  LocationVec &Locations = CSInfos.back().Locations;
+  auto &Locations = CSInfos.back().Locations;
   if (opers.isAnyReg()) {
     unsigned NArgs = opers.getMetaOper(PatchPointOpers::NArgPos).getImm();
-    for (unsigned i = 0, e = (opers.hasDef() ? NArgs+1 : NArgs); i != e; ++i)
-      assert(Locations[i].LocType == Location::Register &&
+    for (unsigned i = 0, e = (opers.hasDef() ? NArgs + 1 : NArgs); i != e; ++i)
+      assert(Locations[i].Type == Location::Register &&
              "anyreg arg must be in reg.");
   }
 #endif
 }
 void StackMaps::recordStatepoint(const MachineInstr &MI) {
-  assert(MI.getOpcode() == TargetOpcode::STATEPOINT &&
-         "expected statepoint");
+  assert(MI.getOpcode() == TargetOpcode::STATEPOINT && "expected statepoint");
 
   StatepointOpers opers(&MI);
   // Record all the deopt and gc operands (they're contiguous and run from the
   // initial index to the end of the operand list)
   const unsigned StartIdx = opers.getVarIdx();
-  recordStackMapOpers(MI, 0xABCDEF00,
-                      MI.operands_begin() + StartIdx, MI.operands_end(),
-                      false);
+  recordStackMapOpers(MI, opers.getID(), MI.operands_begin() + StartIdx,
+                      MI.operands_end(), false);
 }
 
 /// Emit the stackmap header.
@@ -388,8 +394,8 @@ void StackMaps::recordStatepoint(const MachineInstr &MI) {
 void StackMaps::emitStackmapHeader(MCStreamer &OS) {
   // Header.
   OS.EmitIntValue(StackMapVersion, 1); // Version.
-  OS.EmitIntValue(0, 1); // Reserved.
-  OS.EmitIntValue(0, 2); // Reserved.
+  OS.EmitIntValue(0, 1);               // Reserved.
+  OS.EmitIntValue(0, 2);               // Reserved.
 
   // Num functions.
   DEBUG(dbgs() << WSMP << "#functions = " << FnStackSize.size() << '\n');
@@ -413,7 +419,7 @@ void StackMaps::emitFunctionFrameRecords(MCStreamer &OS) {
   DEBUG(dbgs() << WSMP << "functions:\n");
   for (auto const &FR : FnStackSize) {
     DEBUG(dbgs() << WSMP << "function addr: " << FR.first
-                         << " frame size: " << FR.second);
+                 << " frame size: " << FR.second);
     OS.EmitSymbolValue(FR.first, 8);
     OS.EmitIntValue(FR.second, 8);
   }
@@ -425,7 +431,7 @@ void StackMaps::emitFunctionFrameRecords(MCStreamer &OS) {
 void StackMaps::emitConstantPoolEntries(MCStreamer &OS) {
   // Constant pool entries.
   DEBUG(dbgs() << WSMP << "constants:\n");
-  for (auto ConstEntry : ConstPool) {
+  for (const auto &ConstEntry : ConstPool) {
     DEBUG(dbgs() << WSMP << ConstEntry.second << '\n');
     OS.EmitIntValue(ConstEntry.second, 8);
   }
@@ -490,7 +496,7 @@ void StackMaps::emitCallsiteEntries(MCStreamer &OS) {
     OS.EmitIntValue(CSLocs.size(), 2);
 
     for (const auto &Loc : CSLocs) {
-      OS.EmitIntValue(Loc.LocType, 1);
+      OS.EmitIntValue(Loc.Type, 1);
       OS.EmitIntValue(Loc.Size, 1);
       OS.EmitIntValue(Loc.Reg, 2);
       OS.EmitIntValue(Loc.Offset, 4);
@@ -501,7 +507,7 @@ void StackMaps::emitCallsiteEntries(MCStreamer &OS) {
     OS.EmitIntValue(LiveOuts.size(), 2);
 
     for (const auto &LO : LiveOuts) {
-      OS.EmitIntValue(LO.RegNo, 2);
+      OS.EmitIntValue(LO.DwarfRegNum, 2);
       OS.EmitIntValue(0, 1);
       OS.EmitIntValue(LO.Size, 1);
     }
@@ -512,7 +518,7 @@ void StackMaps::emitCallsiteEntries(MCStreamer &OS) {
 
 /// Serialize the stackmap data.
 void StackMaps::serializeToStackMapSection() {
-  (void) WSMP;
+  (void)WSMP;
   // Bail out if there's no stack map data.
   assert((!CSInfos.empty() || (CSInfos.empty() && ConstPool.empty())) &&
          "Expected empty constant pool too!");
@@ -521,16 +527,16 @@ void StackMaps::serializeToStackMapSection() {
   if (CSInfos.empty())
     return;
 
-  MCContext &OutContext = AP.OutStreamer.getContext();
-  MCStreamer &OS = AP.OutStreamer;
+  MCContext &OutContext = AP.OutStreamer->getContext();
+  MCStreamer &OS = *AP.OutStreamer;
 
   // Create the section.
-  const MCSection *StackMapSection =
-    OutContext.getObjectFileInfo()->getStackMapSection();
+  MCSection *StackMapSection =
+      OutContext.getObjectFileInfo()->getStackMapSection();
   OS.SwitchSection(StackMapSection);
 
   // Emit a dummy symbol to force section inclusion.
-  OS.EmitLabel(OutContext.GetOrCreateSymbol(Twine("__LLVM_StackMaps")));
+  OS.EmitLabel(OutContext.getOrCreateSymbol(Twine("__LLVM_StackMaps")));
 
   // Serialize data.
   DEBUG(dbgs() << "********** Stack Map Output **********\n");
diff --git a/lib/CodeGen/StackProtector.cpp b/lib/CodeGen/StackProtector.cpp
index d1fae9558b2..bcea37a3aaf 100644
--- a/lib/CodeGen/StackProtector.cpp
+++ b/lib/CodeGen/StackProtector.cpp
@@ -122,7 +122,7 @@ bool StackProtector::ContainsProtectableArray(Type *Ty, bool &IsLarge,
 
     // If an array has more than SSPBufferSize bytes of allocated space, then we
     // emit stack protectors.
-    if (SSPBufferSize <= TLI->getDataLayout()->getTypeAllocSize(AT)) {
+    if (SSPBufferSize <= M->getDataLayout().getTypeAllocSize(AT)) {
       IsLarge = true;
       return true;
     }
@@ -353,8 +353,8 @@ static bool CreatePrologue(Function *F, Module *M, ReturnInst *RI,
   IRBuilder<> B(&F->getEntryBlock().front());
   AI = B.CreateAlloca(PtrTy, nullptr, "StackGuardSlot");
   LoadInst *LI = B.CreateLoad(StackGuardVar, "StackGuard");
-  B.CreateCall2(Intrinsic::getDeclaration(M, Intrinsic::stackprotector), LI,
-                AI);
+  B.CreateCall(Intrinsic::getDeclaration(M, Intrinsic::stackprotector),
+               {LI, AI});
 
   return SupportsSelectionDAGSP;
 }
@@ -488,7 +488,7 @@ BasicBlock *StackProtector::CreateFailBB() {
     Constant *StackChkFail =
         M->getOrInsertFunction("__stack_chk_fail", Type::getVoidTy(Context),
                                nullptr);
-    B.CreateCall(StackChkFail);
+    B.CreateCall(StackChkFail, {});
   }
   B.CreateUnreachable();
   return FailBB;
diff --git a/lib/CodeGen/TailDuplication.cpp b/lib/CodeGen/TailDuplication.cpp
index 04b39922627..237460cd905 100644
--- a/lib/CodeGen/TailDuplication.cpp
+++ b/lib/CodeGen/TailDuplication.cpp
@@ -449,6 +449,9 @@ void TailDuplicatePass::DuplicateInstruction(MachineInstr *MI,
       DenseMap<unsigned, unsigned>::iterator VI = LocalVRMap.find(Reg);
       if (VI != LocalVRMap.end()) {
         MO.setReg(VI->second);
+        // Clear any kill flags from this operand.  The new register could have
+        // uses after this one, so kills are not valid here.
+        MO.setIsKill(false);
         MRI->constrainRegClass(VI->second, MRI->getRegClass(Reg));
       }
     }
@@ -624,11 +627,8 @@ TailDuplicatePass::isSimpleBB(MachineBasicBlock *TailBB) {
     return false;
   if (TailBB->pred_empty())
     return false;
-  MachineBasicBlock::iterator I = TailBB->begin();
-  MachineBasicBlock::iterator E = TailBB->end();
-  while (I != E && I->isDebugValue())
-    ++I;
-  if (I == E)
+  MachineBasicBlock::iterator I = TailBB->getFirstNonDebugInstr();
+  if (I == TailBB->end())
     return true;
   return I->isUnconditionalBranch();
 }
diff --git a/lib/CodeGen/TargetFrameLoweringImpl.cpp b/lib/CodeGen/TargetFrameLoweringImpl.cpp
index e3f01912b87..f3cccd82a5c 100644
--- a/lib/CodeGen/TargetFrameLoweringImpl.cpp
+++ b/lib/CodeGen/TargetFrameLoweringImpl.cpp
@@ -11,9 +11,13 @@
 //
 //===----------------------------------------------------------------------===//
 
+#include "llvm/ADT/BitVector.h"
 #include "llvm/Target/TargetFrameLowering.h"
 #include "llvm/CodeGen/MachineFrameInfo.h"
 #include "llvm/CodeGen/MachineFunction.h"
+#include "llvm/CodeGen/MachineModuleInfo.h"
+#include "llvm/CodeGen/MachineRegisterInfo.h"
+#include "llvm/IR/Function.h"
 #include "llvm/Target/TargetRegisterInfo.h"
 #include "llvm/Target/TargetSubtargetInfo.h"
 #include <cstdlib>
@@ -22,6 +26,12 @@ using namespace llvm;
 TargetFrameLowering::~TargetFrameLowering() {
 }
 
+/// The default implementation just looks at attribute "no-frame-pointer-elim".
+bool TargetFrameLowering::noFramePointerElim(const MachineFunction &MF) const {
+  auto Attr = MF.getFunction()->getFnAttribute("no-frame-pointer-elim");
+  return Attr.getValueAsString() == "true";
+}
+
 /// getFrameIndexOffset - Returns the displacement from the frame register to
 /// the stack frame of the specified index. This is the default implementation
 /// which is overridden for some targets.
@@ -47,3 +57,30 @@ bool TargetFrameLowering::needsFrameIndexResolution(
     const MachineFunction &MF) const {
   return MF.getFrameInfo()->hasStackObjects();
 }
+
+void TargetFrameLowering::determineCalleeSaves(MachineFunction &MF,
+                                               BitVector &SavedRegs,
+                                               RegScavenger *RS) const {
+  // Get the callee saved register list...
+  const TargetRegisterInfo &TRI = *MF.getSubtarget().getRegisterInfo();
+  const MCPhysReg *CSRegs = TRI.getCalleeSavedRegs(&MF);
+
+  // Early exit if there are no callee saved registers.
+  if (!CSRegs || CSRegs[0] == 0)
+    return;
+
+  SavedRegs.resize(TRI.getNumRegs());
+
+  // In Naked functions we aren't going to save any registers.
+  if (MF.getFunction()->hasFnAttribute(Attribute::Naked))
+    return;
+
+  // Functions which call __builtin_unwind_init get all their registers saved.
+  bool CallsUnwindInit = MF.getMMI().callsUnwindInit();
+  const MachineRegisterInfo &MRI = MF.getRegInfo();
+  for (unsigned i = 0; CSRegs[i]; ++i) {
+    unsigned Reg = CSRegs[i];
+    if (CallsUnwindInit || MRI.isPhysRegModified(Reg))
+      SavedRegs.set(Reg);
+  }
+}
diff --git a/lib/CodeGen/TargetInstrInfo.cpp b/lib/CodeGen/TargetInstrInfo.cpp
index 38725b53b37..69def27f7b3 100644
--- a/lib/CodeGen/TargetInstrInfo.cpp
+++ b/lib/CodeGen/TargetInstrInfo.cpp
@@ -19,6 +19,7 @@
 #include "llvm/CodeGen/PseudoSourceValue.h"
 #include "llvm/CodeGen/ScoreboardHazardRecognizer.h"
 #include "llvm/CodeGen/StackMaps.h"
+#include "llvm/CodeGen/TargetSchedule.h"
 #include "llvm/IR/DataLayout.h"
 #include "llvm/MC/MCAsmInfo.h"
 #include "llvm/MC/MCInstrItineraries.h"
@@ -142,6 +143,10 @@ MachineInstr *TargetInstrInfo::commuteInstruction(MachineInstr *MI,
   unsigned SubReg2 = MI->getOperand(Idx2).getSubReg();
   bool Reg1IsKill = MI->getOperand(Idx1).isKill();
   bool Reg2IsKill = MI->getOperand(Idx2).isKill();
+  bool Reg1IsUndef = MI->getOperand(Idx1).isUndef();
+  bool Reg2IsUndef = MI->getOperand(Idx2).isUndef();
+  bool Reg1IsInternal = MI->getOperand(Idx1).isInternalRead();
+  bool Reg2IsInternal = MI->getOperand(Idx2).isInternalRead();
   // If destination is tied to either of the commuted source register, then
   // it must be updated.
   if (HasDef && Reg0 == Reg1 &&
@@ -172,6 +177,10 @@ MachineInstr *TargetInstrInfo::commuteInstruction(MachineInstr *MI,
   MI->getOperand(Idx1).setSubReg(SubReg2);
   MI->getOperand(Idx2).setIsKill(Reg1IsKill);
   MI->getOperand(Idx1).setIsKill(Reg2IsKill);
+  MI->getOperand(Idx2).setIsUndef(Reg1IsUndef);
+  MI->getOperand(Idx1).setIsUndef(Reg2IsUndef);
+  MI->getOperand(Idx2).setIsInternalRead(Reg1IsInternal);
+  MI->getOperand(Idx1).setIsInternalRead(Reg2IsInternal);
   return MI;
 }
 
@@ -211,9 +220,8 @@ TargetInstrInfo::isUnpredicatedTerminator(const MachineInstr *MI) const {
   return !isPredicated(MI);
 }
 
-
-bool TargetInstrInfo::PredicateInstruction(MachineInstr *MI,
-                            const SmallVectorImpl<MachineOperand> &Pred) const {
+bool TargetInstrInfo::PredicateInstruction(
+    MachineInstr *MI, ArrayRef<MachineOperand> Pred) const {
   bool MadeChange = false;
 
   assert(!MI->isBundle() &&
@@ -307,7 +315,7 @@ bool TargetInstrInfo::getStackSlotRange(const TargetRegisterClass *RC,
 
   assert(RC->getSize() >= (Offset + Size) && "bad subregister range");
 
-  if (!MF.getTarget().getDataLayout()->isLittleEndian()) {
+  if (!MF.getDataLayout().isLittleEndian()) {
     Offset = RC->getSize() - (Offset + Size);
   }
   return true;
@@ -376,11 +384,6 @@ void TargetInstrInfo::getNoopForMachoTarget(MCInst &NopInst) const {
   llvm_unreachable("Not a MachO target");
 }
 
-bool TargetInstrInfo::canFoldMemoryOperand(const MachineInstr *MI,
-                                           ArrayRef<unsigned> Ops) const {
-  return MI->isCopy() && Ops.size() == 1 && canFoldCopy(MI, Ops[0]);
-}
-
 static MachineInstr *foldPatchpoint(MachineFunction &MF, MachineInstr *MI,
                                     ArrayRef<unsigned> Ops, int FrameIndex,
                                     const TargetInstrInfo &TII) {
@@ -463,11 +466,13 @@ MachineInstr *TargetInstrInfo::foldMemoryOperand(MachineBasicBlock::iterator MI,
       MI->getOpcode() == TargetOpcode::PATCHPOINT) {
     // Fold stackmap/patchpoint.
     NewMI = foldPatchpoint(MF, MI, Ops, FI, *this);
+    if (NewMI)
+      MBB->insert(MI, NewMI);
   } else {
     // Ask the target to do the actual folding.
-    NewMI =foldMemoryOperandImpl(MF, MI, Ops, FI);
+    NewMI = foldMemoryOperandImpl(MF, MI, Ops, MI, FI);
   }
- 
+
   if (NewMI) {
     NewMI->setMemRefs(MI->memoperands_begin(), MI->memoperands_end());
     // Add a memory operand, foldMemoryOperandImpl doesn't do that.
@@ -485,8 +490,7 @@ MachineInstr *TargetInstrInfo::foldMemoryOperand(MachineBasicBlock::iterator MI,
                               MFI.getObjectAlignment(FI));
     NewMI->addMemOperand(MF, MMO);
 
-    // FIXME: change foldMemoryOperandImpl semantics to also insert NewMI.
-    return MBB->insert(MI, NewMI);
+    return NewMI;
   }
 
   // Straight COPY may fold as load/store.
@@ -531,15 +535,15 @@ MachineInstr *TargetInstrInfo::foldMemoryOperand(MachineBasicBlock::iterator MI,
       isLoadFromStackSlot(LoadMI, FrameIndex)) {
     // Fold stackmap/patchpoint.
     NewMI = foldPatchpoint(MF, MI, Ops, FrameIndex, *this);
+    if (NewMI)
+      NewMI = MBB.insert(MI, NewMI);
   } else {
     // Ask the target to do the actual folding.
-    NewMI = foldMemoryOperandImpl(MF, MI, Ops, LoadMI);
+    NewMI = foldMemoryOperandImpl(MF, MI, Ops, MI, LoadMI);
   }
 
   if (!NewMI) return nullptr;
 
-  NewMI = MBB.insert(MI, NewMI);
-
   // Copy the memoperands from the load to the folded instruction.
   if (MI->memoperands_empty()) {
     NewMI->setMemRefs(LoadMI->memoperands_begin(),
@@ -644,8 +648,8 @@ int TargetInstrInfo::getSPAdjust(const MachineInstr *MI) const {
   bool StackGrowsDown =
     TFI->getStackGrowthDirection() == TargetFrameLowering::StackGrowsDown;
 
-  int FrameSetupOpcode = getCallFrameSetupOpcode();
-  int FrameDestroyOpcode = getCallFrameDestroyOpcode();
+  unsigned FrameSetupOpcode = getCallFrameSetupOpcode();
+  unsigned FrameDestroyOpcode = getCallFrameDestroyOpcode();
 
   if (MI->getOpcode() != FrameSetupOpcode &&
       MI->getOpcode() != FrameDestroyOpcode)
@@ -793,9 +797,10 @@ getInstrLatency(const InstrItineraryData *ItinData,
   return ItinData->getStageLatency(MI->getDesc().getSchedClass());
 }
 
-bool TargetInstrInfo::hasLowDefLatency(const InstrItineraryData *ItinData,
+bool TargetInstrInfo::hasLowDefLatency(const TargetSchedModel &SchedModel,
                                        const MachineInstr *DefMI,
                                        unsigned DefIdx) const {
+  const InstrItineraryData *ItinData = SchedModel.getInstrItineraries();
   if (!ItinData || ItinData->isEmpty())
     return false;
 
diff --git a/lib/CodeGen/TargetLoweringBase.cpp b/lib/CodeGen/TargetLoweringBase.cpp
index 2162a513b43..0e10b2f2e11 100644
--- a/lib/CodeGen/TargetLoweringBase.cpp
+++ b/lib/CodeGen/TargetLoweringBase.cpp
@@ -38,6 +38,11 @@
 #include <cctype>
 using namespace llvm;
 
+static cl::opt<bool> JumpIsExpensiveOverride(
+    "jump-is-expensive", cl::init(false),
+    cl::desc("Do not create extra branches to split comparison logic."),
+    cl::Hidden);
+
 /// InitLibcallNames - Set default libcall names.
 ///
 static void InitLibcallNames(const char **Names, const Triple &TT) {
@@ -420,6 +425,14 @@ static void InitLibcallNames(const char **Names, const Triple &TT) {
     // These are generally not available.
     Names[RTLIB::STACKPROTECTOR_CHECK_FAIL] = nullptr;
   }
+
+  // For f16/f32 conversions, Darwin uses the standard naming scheme, instead
+  // of the gnueabi-style __gnu_*_ieee.
+  // FIXME: What about other targets?
+  if (TT.isOSDarwin()) {
+    Names[RTLIB::FPEXT_F16_F32] = "__extendhfsf2";
+    Names[RTLIB::FPROUND_F32_F16] = "__truncsfhf2";
+  }
 }
 
 /// InitLibcallCallingConvs - Set default libcall CallingConvs.
@@ -737,7 +750,6 @@ TargetLoweringBase::TargetLoweringBase(const TargetMachine &tm) : TM(tm) {
   initActions();
 
   // Perform these initializations only once.
-  IsLittleEndian = getDataLayout()->isLittleEndian();
   MaxStoresPerMemset = MaxStoresPerMemcpy = MaxStoresPerMemmove = 8;
   MaxStoresPerMemsetOptSize = MaxStoresPerMemcpyOptSize
     = MaxStoresPerMemmoveOptSize = 4;
@@ -749,7 +761,7 @@ TargetLoweringBase::TargetLoweringBase(const TargetMachine &tm) : TM(tm) {
   IntDivIsCheap = false;
   FsqrtIsCheap = false;
   Pow2SDivIsCheap = false;
-  JumpIsExpensive = false;
+  JumpIsExpensive = JumpIsExpensiveOverride;
   PredictableSelectIsExpensive = false;
   MaskAndBranchFoldingIsLegal = false;
   EnableExtLdPromotion = false;
@@ -770,7 +782,7 @@ TargetLoweringBase::TargetLoweringBase(const TargetMachine &tm) : TM(tm) {
   InsertFencesForAtomic = false;
   MinimumJumpTableEntries = 4;
 
-  InitLibcallNames(LibcallRoutineNames, Triple(TM.getTargetTriple()));
+  InitLibcallNames(LibcallRoutineNames, TM.getTargetTriple());
   InitCmpLibcallCCs(CmpLibcallCCs);
   InitLibcallCallingConvs(LibcallCallingConvs);
 }
@@ -803,6 +815,20 @@ void TargetLoweringBase::initActions() {
     setOperationAction(ISD::FMINNUM, VT, Expand);
     setOperationAction(ISD::FMAXNUM, VT, Expand);
     setOperationAction(ISD::FMAD, VT, Expand);
+    setOperationAction(ISD::SMIN, VT, Expand);
+    setOperationAction(ISD::SMAX, VT, Expand);
+    setOperationAction(ISD::UMIN, VT, Expand);
+    setOperationAction(ISD::UMAX, VT, Expand);
+
+    // Overflow operations default to expand
+    setOperationAction(ISD::SADDO, VT, Expand);
+    setOperationAction(ISD::SSUBO, VT, Expand);
+    setOperationAction(ISD::UADDO, VT, Expand);
+    setOperationAction(ISD::USUBO, VT, Expand);
+    setOperationAction(ISD::SMULO, VT, Expand);
+    setOperationAction(ISD::UMULO, VT, Expand);
+    setOperationAction(ISD::UABSDIFF, VT, Expand);
+    setOperationAction(ISD::SABSDIFF, VT, Expand);
 
     // These library functions default to expand.
     setOperationAction(ISD::FROUND, VT, Expand);
@@ -854,28 +880,17 @@ void TargetLoweringBase::initActions() {
   setOperationAction(ISD::DEBUGTRAP, MVT::Other, Expand);
 }
 
-MVT TargetLoweringBase::getPointerTy(uint32_t AS) const {
-  return MVT::getIntegerVT(getPointerSizeInBits(AS));
-}
-
-unsigned TargetLoweringBase::getPointerSizeInBits(uint32_t AS) const {
-  return getDataLayout()->getPointerSizeInBits(AS);
+MVT TargetLoweringBase::getScalarShiftAmountTy(const DataLayout &DL,
+                                               EVT) const {
+  return MVT::getIntegerVT(8 * DL.getPointerSize(0));
 }
 
-unsigned TargetLoweringBase::getPointerTypeSizeInBits(Type *Ty) const {
-  assert(Ty->isPointerTy());
-  return getPointerSizeInBits(Ty->getPointerAddressSpace());
-}
-
-MVT TargetLoweringBase::getScalarShiftAmountTy(EVT LHSTy) const {
-  return MVT::getIntegerVT(8 * getDataLayout()->getPointerSize(0));
-}
-
-EVT TargetLoweringBase::getShiftAmountTy(EVT LHSTy) const {
+EVT TargetLoweringBase::getShiftAmountTy(EVT LHSTy,
+                                         const DataLayout &DL) const {
   assert(LHSTy.isInteger() && "Shift amount is not an integer type!");
   if (LHSTy.isVector())
     return LHSTy;
-  return getScalarShiftAmountTy(LHSTy);
+  return getScalarShiftAmountTy(DL, LHSTy);
 }
 
 /// canOpTrap - Returns true if the operation can trap for the value type.
@@ -895,6 +910,12 @@ bool TargetLoweringBase::canOpTrap(unsigned Op, EVT VT) const {
   }
 }
 
+void TargetLoweringBase::setJumpIsExpensive(bool isExpensive) {
+  // If the command-line option was specified, ignore this request.
+  if (!JumpIsExpensiveOverride.getNumOccurrences())
+    JumpIsExpensive = isExpensive;
+}
+
 TargetLoweringBase::LegalizeKind
 TargetLoweringBase::getTypeConversion(LLVMContext &Context, EVT VT) const {
   // If this is a simple type, use the ComputeRegisterProp mechanism.
@@ -1130,7 +1151,7 @@ TargetLoweringBase::emitPatchPoint(MachineInstr *MI,
     }
     MachineMemOperand *MMO = MF.getMachineMemOperand(
         MachinePointerInfo::getFixedStack(FI), Flags,
-        TM.getDataLayout()->getPointerSize(), MFI.getObjectAlignment(FI));
+        MF.getDataLayout().getPointerSize(), MFI.getObjectAlignment(FI));
     MIB->addMemOperand(MF, MMO);
 
     // Replace the instruction and update the operand index.
@@ -1256,10 +1277,19 @@ void TargetLoweringBase::computeRegisterProperties(
   }
 
   if (!isTypeLegal(MVT::f16)) {
-    NumRegistersForVT[MVT::f16] = NumRegistersForVT[MVT::i16];
-    RegisterTypeForVT[MVT::f16] = RegisterTypeForVT[MVT::i16];
-    TransformToType[MVT::f16] = MVT::i16;
-    ValueTypeActions.setTypeAction(MVT::f16, TypeSoftenFloat);
+    // If the target has native f32 support, promote f16 operations to f32.  If
+    // f32 is not supported, generate soft float library calls.
+    if (isTypeLegal(MVT::f32)) {
+      NumRegistersForVT[MVT::f16] = NumRegistersForVT[MVT::f32];
+      RegisterTypeForVT[MVT::f16] = RegisterTypeForVT[MVT::f32];
+      TransformToType[MVT::f16] = MVT::f32;
+      ValueTypeActions.setTypeAction(MVT::f16, TypePromoteFloat);
+    } else {
+      NumRegistersForVT[MVT::f16] = NumRegistersForVT[MVT::i16];
+      RegisterTypeForVT[MVT::f16] = RegisterTypeForVT[MVT::i16];
+      TransformToType[MVT::f16] = MVT::i16;
+      ValueTypeActions.setTypeAction(MVT::f16, TypeSoftenFloat);
+    }
   }
 
   // Loop over all of the vector value types to see which need transformations.
@@ -1358,9 +1388,10 @@ void TargetLoweringBase::computeRegisterProperties(
   }
 }
 
-EVT TargetLoweringBase::getSetCCResultType(LLVMContext &, EVT VT) const {
+EVT TargetLoweringBase::getSetCCResultType(const DataLayout &DL, LLVMContext &,
+                                           EVT VT) const {
   assert(!VT.isVector() && "No default SetCC type for vectors!");
-  return getPointerTy(0).SimpleTy;
+  return getPointerTy(DL).SimpleTy;
 }
 
 MVT::SimpleValueType TargetLoweringBase::getCmpLibcallReturnType() const {
@@ -1445,11 +1476,11 @@ unsigned TargetLoweringBase::getVectorTypeBreakdown(LLVMContext &Context, EVT VT
 /// type of the given function.  This does not require a DAG or a return value,
 /// and is suitable for use before any DAGs for the function are constructed.
 /// TODO: Move this out of TargetLowering.cpp.
-void llvm::GetReturnInfo(Type* ReturnType, AttributeSet attr,
+void llvm::GetReturnInfo(Type *ReturnType, AttributeSet attr,
                          SmallVectorImpl<ISD::OutputArg> &Outs,
-                         const TargetLowering &TLI) {
+                         const TargetLowering &TLI, const DataLayout &DL) {
   SmallVector<EVT, 4> ValueVTs;
-  ComputeValueVTs(TLI, ReturnType, ValueVTs);
+  ComputeValueVTs(TLI, DL, ReturnType, ValueVTs);
   unsigned NumValues = ValueVTs.size();
   if (NumValues == 0) return;
 
@@ -1494,10 +1525,34 @@ void llvm::GetReturnInfo(Type* ReturnType, AttributeSet attr,
 /// getByValTypeAlignment - Return the desired alignment for ByVal aggregate
 /// function arguments in the caller parameter area.  This is the actual
 /// alignment, not its logarithm.
-unsigned TargetLoweringBase::getByValTypeAlignment(Type *Ty) const {
-  return getDataLayout()->getABITypeAlignment(Ty);
+unsigned TargetLoweringBase::getByValTypeAlignment(Type *Ty,
+                                                   const DataLayout &DL) const {
+  return DL.getABITypeAlignment(Ty);
 }
 
+bool TargetLoweringBase::allowsMemoryAccess(LLVMContext &Context,
+                                            const DataLayout &DL, EVT VT,
+                                            unsigned AddrSpace,
+                                            unsigned Alignment,
+                                            bool *Fast) const {
+  // Check if the specified alignment is sufficient based on the data layout.
+  // TODO: While using the data layout works in practice, a better solution
+  // would be to implement this check directly (make this a virtual function).
+  // For example, the ABI alignment may change based on software platform while
+  // this function should only be affected by hardware implementation.
+  Type *Ty = VT.getTypeForEVT(Context);
+  if (Alignment >= DL.getABITypeAlignment(Ty)) {
+    // Assume that an access that meets the ABI-specified alignment is fast.
+    if (Fast != nullptr)
+      *Fast = true;
+    return true;
+  }
+  
+  // This is a misaligned access.
+  return allowsMisalignedMemoryAccesses(VT, AddrSpace, Alignment, Fast);
+}
+
+
 //===----------------------------------------------------------------------===//
 //  TargetTransformInfo Helpers
 //===----------------------------------------------------------------------===//
@@ -1516,6 +1571,12 @@ int TargetLoweringBase::InstructionOpcodeToISD(unsigned Opcode) const {
   case Invoke:         return 0;
   case Resume:         return 0;
   case Unreachable:    return 0;
+  case CleanupRet:     return 0;
+  case CatchEndPad:  return 0;
+  case CatchRet:       return 0;
+  case CatchPad:     return 0;
+  case TerminatePad: return 0;
+  case CleanupPad:   return 0;
   case Add:            return ISD::ADD;
   case FAdd:           return ISD::FADD;
   case Sub:            return ISD::SUB;
@@ -1574,9 +1635,10 @@ int TargetLoweringBase::InstructionOpcodeToISD(unsigned Opcode) const {
 }
 
 std::pair<unsigned, MVT>
-TargetLoweringBase::getTypeLegalizationCost(Type *Ty) const {
+TargetLoweringBase::getTypeLegalizationCost(const DataLayout &DL,
+                                            Type *Ty) const {
   LLVMContext &C = Ty->getContext();
-  EVT MTy = getValueType(Ty);
+  EVT MTy = getValueType(DL, Ty);
 
   unsigned Cost = 1;
   // We keep legalizing the type until we find a legal kind. We assume that
@@ -1602,8 +1664,9 @@ TargetLoweringBase::getTypeLegalizationCost(Type *Ty) const {
 
 /// isLegalAddressingMode - Return true if the addressing mode represented
 /// by AM is legal for this target, for a load/store of the specified type.
-bool TargetLoweringBase::isLegalAddressingMode(const AddrMode &AM,
-                                           Type *Ty) const {
+bool TargetLoweringBase::isLegalAddressingMode(const DataLayout &DL,
+                                               const AddrMode &AM, Type *Ty,
+                                               unsigned AS) const {
   // The default implementation of this implements a conservative RISCy, r+r and
   // r+i addr mode.
 
diff --git a/lib/CodeGen/TargetLoweringObjectFileImpl.cpp b/lib/CodeGen/TargetLoweringObjectFileImpl.cpp
index 5b795e4b37b..b4383cca1b9 100644
--- a/lib/CodeGen/TargetLoweringObjectFileImpl.cpp
+++ b/lib/CodeGen/TargetLoweringObjectFileImpl.cpp
@@ -30,7 +30,7 @@
 #include "llvm/MC/MCSectionELF.h"
 #include "llvm/MC/MCSectionMachO.h"
 #include "llvm/MC/MCStreamer.h"
-#include "llvm/MC/MCSymbol.h"
+#include "llvm/MC/MCSymbolELF.h"
 #include "llvm/MC/MCValue.h"
 #include "llvm/Support/Dwarf.h"
 #include "llvm/Support/ELF.h"
@@ -51,34 +51,32 @@ MCSymbol *TargetLoweringObjectFileELF::getCFIPersonalitySymbol(
     MachineModuleInfo *MMI) const {
   unsigned Encoding = getPersonalityEncoding();
   if ((Encoding & 0x80) == dwarf::DW_EH_PE_indirect)
-    return getContext().GetOrCreateSymbol(StringRef("DW.ref.") +
+    return getContext().getOrCreateSymbol(StringRef("DW.ref.") +
                                           TM.getSymbol(GV, Mang)->getName());
   if ((Encoding & 0x70) == dwarf::DW_EH_PE_absptr)
     return TM.getSymbol(GV, Mang);
   report_fatal_error("We do not support this DWARF encoding yet!");
 }
 
-void TargetLoweringObjectFileELF::emitPersonalityValue(MCStreamer &Streamer,
-                                                       const TargetMachine &TM,
-                                                       const MCSymbol *Sym) const {
+void TargetLoweringObjectFileELF::emitPersonalityValue(
+    MCStreamer &Streamer, const DataLayout &DL, const MCSymbol *Sym) const {
   SmallString<64> NameData("DW.ref.");
   NameData += Sym->getName();
-  MCSymbol *Label = getContext().GetOrCreateSymbol(NameData);
+  MCSymbolELF *Label =
+      cast<MCSymbolELF>(getContext().getOrCreateSymbol(NameData));
   Streamer.EmitSymbolAttribute(Label, MCSA_Hidden);
   Streamer.EmitSymbolAttribute(Label, MCSA_Weak);
   StringRef Prefix = ".data.";
   NameData.insert(NameData.begin(), Prefix.begin(), Prefix.end());
   unsigned Flags = ELF::SHF_ALLOC | ELF::SHF_WRITE | ELF::SHF_GROUP;
-  const MCSection *Sec = getContext().getELFSection(NameData,
-                                                    ELF::SHT_PROGBITS,
-                                                    Flags,
-                                                    0, Label->getName());
-  unsigned Size = TM.getDataLayout()->getPointerSize();
+  MCSection *Sec = getContext().getELFSection(NameData, ELF::SHT_PROGBITS,
+                                              Flags, 0, Label->getName());
+  unsigned Size = DL.getPointerSize();
   Streamer.SwitchSection(Sec);
-  Streamer.EmitValueToAlignment(TM.getDataLayout()->getPointerABIAlignment());
+  Streamer.EmitValueToAlignment(DL.getPointerABIAlignment());
   Streamer.EmitSymbolAttribute(Label, MCSA_ELF_TypeObject);
-  const MCExpr *E = MCConstantExpr::Create(Size, getContext());
-  Streamer.EmitELFSize(Label, E);
+  const MCExpr *E = MCConstantExpr::create(Size, getContext());
+  Streamer.emitELFSize(Label, E);
   Streamer.EmitLabel(Label);
 
   Streamer.EmitSymbolValue(Sym, Size);
@@ -103,7 +101,7 @@ const MCExpr *TargetLoweringObjectFileELF::getTTypeGlobalReference(
     }
 
     return TargetLoweringObjectFile::
-      getTTypeReference(MCSymbolRefExpr::Create(SSym, getContext()),
+      getTTypeReference(MCSymbolRefExpr::create(SSym, getContext()),
                         Encoding & ~dwarf::DW_EH_PE_indirect, Streamer);
   }
 
@@ -201,7 +199,7 @@ static const Comdat *getELFComdat(const GlobalValue *GV) {
   return C;
 }
 
-const MCSection *TargetLoweringObjectFileELF::getExplicitSectionGlobal(
+MCSection *TargetLoweringObjectFileELF::getExplicitSectionGlobal(
     const GlobalValue *GV, SectionKind Kind, Mangler &Mang,
     const TargetMachine &TM) const {
   StringRef SectionName = GV->getSection();
@@ -245,7 +243,7 @@ static StringRef getSectionPrefixForGlobal(SectionKind Kind) {
   return ".data.rel.ro";
 }
 
-static const MCSectionELF *
+static MCSectionELF *
 selectELFSectionForGlobal(MCContext &Ctx, const GlobalValue *GV,
                           SectionKind Kind, Mangler &Mang,
                           const TargetMachine &TM, bool EmitUniqueSection,
@@ -283,8 +281,8 @@ selectELFSectionForGlobal(MCContext &Ctx, const GlobalValue *GV,
     // We also need alignment here.
     // FIXME: this is getting the alignment of the character, not the
     // alignment of the global!
-    unsigned Align =
-        TM.getDataLayout()->getPreferredAlignment(cast<GlobalVariable>(GV));
+    unsigned Align = GV->getParent()->getDataLayout().getPreferredAlignment(
+        cast<GlobalVariable>(GV));
 
     std::string SizeSpec = ".rodata.str" + utostr(EntrySize) + ".";
     Name = SizeSpec + utostr(Align);
@@ -308,7 +306,7 @@ selectELFSectionForGlobal(MCContext &Ctx, const GlobalValue *GV,
                            EntrySize, Group, UniqueID);
 }
 
-const MCSection *TargetLoweringObjectFileELF::SelectSectionForGlobal(
+MCSection *TargetLoweringObjectFileELF::SelectSectionForGlobal(
     const GlobalValue *GV, SectionKind Kind, Mangler &Mang,
     const TargetMachine &TM) const {
   unsigned Flags = getELFSectionFlags(Kind);
@@ -328,7 +326,7 @@ const MCSection *TargetLoweringObjectFileELF::SelectSectionForGlobal(
                                    EmitUniqueSection, Flags, &NextUniqueID);
 }
 
-const MCSection *TargetLoweringObjectFileELF::getSectionForJumpTable(
+MCSection *TargetLoweringObjectFileELF::getSectionForJumpTable(
     const Function &F, Mangler &Mang, const TargetMachine &TM) const {
   // If the function can be removed, produce a unique section so that
   // the table doesn't prevent the removal.
@@ -349,12 +347,10 @@ bool TargetLoweringObjectFileELF::shouldPutJumpTableInFunctionSection(
   return false;
 }
 
-/// getSectionForConstant - Given a mergeable constant with the
-/// specified size and relocation information, return a section that it
-/// should be placed in.
-const MCSection *
-TargetLoweringObjectFileELF::getSectionForConstant(SectionKind Kind,
-                                                   const Constant *C) const {
+/// Given a mergeable constant with the specified size and relocation
+/// information, return a section that it should be placed in.
+MCSection *TargetLoweringObjectFileELF::getSectionForConstant(
+    const DataLayout &DL, SectionKind Kind, const Constant *C) const {
   if (Kind.isMergeableConst4() && MergeableConst4Section)
     return MergeableConst4Section;
   if (Kind.isMergeableConst8() && MergeableConst8Section)
@@ -369,11 +365,9 @@ TargetLoweringObjectFileELF::getSectionForConstant(SectionKind Kind,
   return DataRelROSection;
 }
 
-static const MCSectionELF *getStaticStructorSection(MCContext &Ctx,
-                                                    bool UseInitArray,
-                                                    bool IsCtor,
-                                                    unsigned Priority,
-                                                    const MCSymbol *KeySym) {
+static MCSectionELF *getStaticStructorSection(MCContext &Ctx, bool UseInitArray,
+                                              bool IsCtor, unsigned Priority,
+                                              const MCSymbol *KeySym) {
   std::string Name;
   unsigned Type;
   unsigned Flags = ELF::SHF_ALLOC | ELF::SHF_WRITE;
@@ -411,13 +405,13 @@ static const MCSectionELF *getStaticStructorSection(MCContext &Ctx,
   return Ctx.getELFSection(Name, Type, Flags, 0, COMDAT);
 }
 
-const MCSection *TargetLoweringObjectFileELF::getStaticCtorSection(
+MCSection *TargetLoweringObjectFileELF::getStaticCtorSection(
     unsigned Priority, const MCSymbol *KeySym) const {
   return getStaticStructorSection(getContext(), UseInitArray, true, Priority,
                                   KeySym);
 }
 
-const MCSection *TargetLoweringObjectFileELF::getStaticDtorSection(
+MCSection *TargetLoweringObjectFileELF::getStaticDtorSection(
     unsigned Priority, const MCSymbol *KeySym) const {
   return getStaticStructorSection(getContext(), UseInitArray, false, Priority,
                                   KeySym);
@@ -444,16 +438,6 @@ TargetLoweringObjectFileMachO::TargetLoweringObjectFileMachO()
   SupportIndirectSymViaGOTPCRel = true;
 }
 
-/// getDepLibFromLinkerOpt - Extract the dependent library name from a linker
-/// option string. Returns StringRef() if the option does not specify a library.
-StringRef TargetLoweringObjectFileMachO::
-getDepLibFromLinkerOpt(StringRef LinkerOption) const {
-  const char *LibCmd = "-l";
-  if (LinkerOption.startswith(LibCmd))
-    return LinkerOption.substr(strlen(LibCmd));
-  return StringRef();
-}
-
 /// emitModuleFlags - Perform code emission for module flags.
 void TargetLoweringObjectFileMachO::
 emitModuleFlags(MCStreamer &Streamer,
@@ -520,12 +504,11 @@ emitModuleFlags(MCStreamer &Streamer,
                        ErrorCode + ".");
 
   // Get the section.
-  const MCSectionMachO *S =
-    getContext().getMachOSection(Segment, Section, TAA, StubSize,
-                                 SectionKind::getDataNoRel());
+  MCSectionMachO *S = getContext().getMachOSection(
+      Segment, Section, TAA, StubSize, SectionKind::getDataNoRel());
   Streamer.SwitchSection(S);
   Streamer.EmitLabel(getContext().
-                     GetOrCreateSymbol(StringRef("L_OBJC_IMAGE_INFO")));
+                     getOrCreateSymbol(StringRef("L_OBJC_IMAGE_INFO")));
   Streamer.EmitIntValue(VersionVal, 4);
   Streamer.EmitIntValue(ImageInfoFlags, 4);
   Streamer.AddBlankLine();
@@ -540,7 +523,7 @@ static void checkMachOComdat(const GlobalValue *GV) {
                      "' cannot be lowered.");
 }
 
-const MCSection *TargetLoweringObjectFileMachO::getExplicitSectionGlobal(
+MCSection *TargetLoweringObjectFileMachO::getExplicitSectionGlobal(
     const GlobalValue *GV, SectionKind Kind, Mangler &Mang,
     const TargetMachine &TM) const {
   // Parse the section specifier and create it if valid.
@@ -561,8 +544,8 @@ const MCSection *TargetLoweringObjectFileMachO::getExplicitSectionGlobal(
   }
 
   // Get the section.
-  const MCSectionMachO *S =
-    getContext().getMachOSection(Segment, Section, TAA, StubSize, Kind);
+  MCSectionMachO *S =
+      getContext().getMachOSection(Segment, Section, TAA, StubSize, Kind);
 
   // If TAA wasn't set by ParseSectionSpecifier() above,
   // use the value returned by getMachOSection() as a default.
@@ -582,9 +565,9 @@ const MCSection *TargetLoweringObjectFileMachO::getExplicitSectionGlobal(
   return S;
 }
 
-const MCSection *TargetLoweringObjectFileMachO::
-SelectSectionForGlobal(const GlobalValue *GV, SectionKind Kind,
-                       Mangler &Mang, const TargetMachine &TM) const {
+MCSection *TargetLoweringObjectFileMachO::SelectSectionForGlobal(
+    const GlobalValue *GV, SectionKind Kind, Mangler &Mang,
+    const TargetMachine &TM) const {
   checkMachOComdat(GV);
 
   // Handle thread local data.
@@ -604,14 +587,16 @@ SelectSectionForGlobal(const GlobalValue *GV, SectionKind Kind,
 
   // FIXME: Alignment check should be handled by section classifier.
   if (Kind.isMergeable1ByteCString() &&
-      TM.getDataLayout()->getPreferredAlignment(cast<GlobalVariable>(GV)) < 32)
+      GV->getParent()->getDataLayout().getPreferredAlignment(
+          cast<GlobalVariable>(GV)) < 32)
     return CStringSection;
 
   // Do not put 16-bit arrays in the UString section if they have an
   // externally visible label, this runs into issues with certain linker
   // versions.
   if (Kind.isMergeable2ByteCString() && !GV->hasExternalLinkage() &&
-      TM.getDataLayout()->getPreferredAlignment(cast<GlobalVariable>(GV)) < 32)
+      GV->getParent()->getDataLayout().getPreferredAlignment(
+          cast<GlobalVariable>(GV)) < 32)
     return UStringSection;
 
   // With MachO only variables whose corresponding symbol starts with 'l' or
@@ -649,9 +634,8 @@ SelectSectionForGlobal(const GlobalValue *GV, SectionKind Kind,
   return DataSection;
 }
 
-const MCSection *
-TargetLoweringObjectFileMachO::getSectionForConstant(SectionKind Kind,
-                                                     const Constant *C) const {
+MCSection *TargetLoweringObjectFileMachO::getSectionForConstant(
+    const DataLayout &DL, SectionKind Kind, const Constant *C) const {
   // If this constant requires a relocation, we have to put it in the data
   // segment, not in the text segment.
   if (Kind.isDataRel() || Kind.isReadOnlyWithRel())
@@ -690,7 +674,7 @@ const MCExpr *TargetLoweringObjectFileMachO::getTTypeGlobalReference(
     }
 
     return TargetLoweringObjectFile::
-      getTTypeReference(MCSymbolRefExpr::Create(SSym, getContext()),
+      getTTypeReference(MCSymbolRefExpr::create(SSym, getContext()),
                         Encoding & ~dwarf::DW_EH_PE_indirect, Streamer);
   }
 
@@ -755,10 +739,10 @@ const MCExpr *TargetLoweringObjectFileMachO::getIndirectSymViaGOTPCRel(
   // non_lazy_ptr stubs.
   SmallString<128> Name;
   StringRef Suffix = "$non_lazy_ptr";
-  Name += DL->getPrivateGlobalPrefix();
+  Name += MMI->getModule()->getDataLayout().getPrivateGlobalPrefix();
   Name += Sym->getName();
   Name += Suffix;
-  MCSymbol *Stub = Ctx.GetOrCreateSymbol(Name);
+  MCSymbol *Stub = Ctx.getOrCreateSymbol(Name);
 
   MachineModuleInfoImpl::StubValueTy &StubSym = MachOMMI.getGVStubEntry(Stub);
   if (!StubSym.getPointer())
@@ -766,16 +750,16 @@ const MCExpr *TargetLoweringObjectFileMachO::getIndirectSymViaGOTPCRel(
       StubValueTy(const_cast<MCSymbol *>(Sym), true /* access indirectly */);
 
   const MCExpr *BSymExpr =
-    MCSymbolRefExpr::Create(BaseSym, MCSymbolRefExpr::VK_None, Ctx);
+    MCSymbolRefExpr::create(BaseSym, MCSymbolRefExpr::VK_None, Ctx);
   const MCExpr *LHS =
-    MCSymbolRefExpr::Create(Stub, MCSymbolRefExpr::VK_None, Ctx);
+    MCSymbolRefExpr::create(Stub, MCSymbolRefExpr::VK_None, Ctx);
 
   if (!Offset)
-    return MCBinaryExpr::CreateSub(LHS, BSymExpr, Ctx);
+    return MCBinaryExpr::createSub(LHS, BSymExpr, Ctx);
 
   const MCExpr *RHS =
-    MCBinaryExpr::CreateAdd(BSymExpr, MCConstantExpr::Create(Offset, Ctx), Ctx);
-  return MCBinaryExpr::CreateSub(LHS, RHS, Ctx);
+    MCBinaryExpr::createAdd(BSymExpr, MCConstantExpr::create(Offset, Ctx), Ctx);
+  return MCBinaryExpr::createSub(LHS, RHS, Ctx);
 }
 
 //===----------------------------------------------------------------------===//
@@ -855,13 +839,11 @@ static int getSelectionForCOFF(const GlobalValue *GV) {
     } else {
       return COFF::IMAGE_COMDAT_SELECT_ASSOCIATIVE;
     }
-  } else if (GV->isWeakForLinker()) {
-    return COFF::IMAGE_COMDAT_SELECT_ANY;
   }
   return 0;
 }
 
-const MCSection *TargetLoweringObjectFileCOFF::getExplicitSectionGlobal(
+MCSection *TargetLoweringObjectFileCOFF::getExplicitSectionGlobal(
     const GlobalValue *GV, SectionKind Kind, Mangler &Mang,
     const TargetMachine &TM) const {
   int Selection = 0;
@@ -903,10 +885,9 @@ static const char *getCOFFSectionNameForUniqueGlobal(SectionKind Kind) {
   return ".data";
 }
 
-
-const MCSection *TargetLoweringObjectFileCOFF::
-SelectSectionForGlobal(const GlobalValue *GV, SectionKind Kind,
-                       Mangler &Mang, const TargetMachine &TM) const {
+MCSection *TargetLoweringObjectFileCOFF::SelectSectionForGlobal(
+    const GlobalValue *GV, SectionKind Kind, Mangler &Mang,
+    const TargetMachine &TM) const {
   // If we have -ffunction-sections then we should emit the global value to a
   // uniqued section specifically for it.
   bool EmitUniquedSection;
@@ -971,7 +952,7 @@ void TargetLoweringObjectFileCOFF::getNameWithPrefix(
   Mang.getNameWithPrefix(OutName, GV, CannotUsePrivateLabel);
 }
 
-const MCSection *TargetLoweringObjectFileCOFF::getSectionForJumpTable(
+MCSection *TargetLoweringObjectFileCOFF::getSectionForJumpTable(
     const Function &F, Mangler &Mang, const TargetMachine &TM) const {
   // If the function can be removed, produce a unique section so that
   // the table doesn't prevent the removal.
@@ -996,14 +977,6 @@ const MCSection *TargetLoweringObjectFileCOFF::getSectionForJumpTable(
                                      COFF::IMAGE_COMDAT_SELECT_ASSOCIATIVE);
 }
 
-StringRef TargetLoweringObjectFileCOFF::
-getDepLibFromLinkerOpt(StringRef LinkerOption) const {
-  const char *LibCmd = "/DEFAULTLIB:";
-  if (LinkerOption.startswith(LibCmd))
-    return LinkerOption.substr(strlen(LibCmd));
-  return StringRef();
-}
-
 void TargetLoweringObjectFileCOFF::
 emitModuleFlags(MCStreamer &Streamer,
                 ArrayRef<Module::ModuleFlagEntry> ModuleFlags,
@@ -1026,7 +999,7 @@ emitModuleFlags(MCStreamer &Streamer,
 
   // Emit the linker options to the linker .drectve section.  According to the
   // spec, this section is a space-separated string containing flags for linker.
-  const MCSection *Sec = getDrectveSection();
+  MCSection *Sec = getDrectveSection();
   Streamer.SwitchSection(Sec);
   for (unsigned i = 0, e = LinkerOptions->getNumOperands(); i != e; ++i) {
     MDNode *MDOptions = cast<MDNode>(LinkerOptions->getOperand(i));
@@ -1040,14 +1013,47 @@ emitModuleFlags(MCStreamer &Streamer,
   }
 }
 
-const MCSection *TargetLoweringObjectFileCOFF::getStaticCtorSection(
+MCSection *TargetLoweringObjectFileCOFF::getStaticCtorSection(
     unsigned Priority, const MCSymbol *KeySym) const {
   return getContext().getAssociativeCOFFSection(
       cast<MCSectionCOFF>(StaticCtorSection), KeySym);
 }
 
-const MCSection *TargetLoweringObjectFileCOFF::getStaticDtorSection(
+MCSection *TargetLoweringObjectFileCOFF::getStaticDtorSection(
     unsigned Priority, const MCSymbol *KeySym) const {
   return getContext().getAssociativeCOFFSection(
       cast<MCSectionCOFF>(StaticDtorSection), KeySym);
 }
+
+void TargetLoweringObjectFileCOFF::emitLinkerFlagsForGlobal(
+    raw_ostream &OS, const GlobalValue *GV, const Mangler &Mang) const {
+  if (!GV->hasDLLExportStorageClass() || GV->isDeclaration())
+    return;
+
+  const Triple &TT = getTargetTriple();
+
+  if (TT.isKnownWindowsMSVCEnvironment())
+    OS << " /EXPORT:";
+  else
+    OS << " -export:";
+
+  if (TT.isWindowsGNUEnvironment() || TT.isWindowsCygwinEnvironment()) {
+    std::string Flag;
+    raw_string_ostream FlagOS(Flag);
+    Mang.getNameWithPrefix(FlagOS, GV, false);
+    FlagOS.flush();
+    if (Flag[0] == GV->getParent()->getDataLayout().getGlobalPrefix())
+      OS << Flag.substr(1);
+    else
+      OS << Flag;
+  } else {
+    Mang.getNameWithPrefix(OS, GV, false);
+  }
+
+  if (!GV->getValueType()->isFunctionTy()) {
+    if (TT.isKnownWindowsMSVCEnvironment())
+      OS << ",DATA";
+    else
+      OS << ",data";
+  }
+}
diff --git a/lib/CodeGen/TargetOptionsImpl.cpp b/lib/CodeGen/TargetOptionsImpl.cpp
index 618d903a090..8d2048fa047 100644
--- a/lib/CodeGen/TargetOptionsImpl.cpp
+++ b/lib/CodeGen/TargetOptionsImpl.cpp
@@ -12,23 +12,26 @@
 //===----------------------------------------------------------------------===//
 
 #include "llvm/IR/Function.h"
+#include "llvm/IR/Module.h"
 #include "llvm/CodeGen/MachineFrameInfo.h"
 #include "llvm/CodeGen/MachineFunction.h"
+#include "llvm/Target/TargetFrameLowering.h"
 #include "llvm/Target/TargetOptions.h"
+#include "llvm/Target/TargetSubtargetInfo.h"
 using namespace llvm;
 
 /// DisableFramePointerElim - This returns true if frame pointer elimination
 /// optimization should be disabled for the given machine function.
 bool TargetOptions::DisableFramePointerElim(const MachineFunction &MF) const {
-  // Check to see if we should eliminate non-leaf frame pointers and then
-  // check to see if we should eliminate all frame pointers.
-  if (MF.getFunction()->hasFnAttribute("no-frame-pointer-elim-non-leaf") &&
-      !NoFramePointerElim) {
-    const MachineFrameInfo *MFI = MF.getFrameInfo();
-    return MFI->hasCalls();
-  }
+  // Check to see if we should eliminate all frame pointers.
+  if (MF.getSubtarget().getFrameLowering()->noFramePointerElim(MF))
+    return true;
 
-  return NoFramePointerElim;
+  // Check to see if we should eliminate non-leaf frame pointers.
+  if (MF.getFunction()->hasFnAttribute("no-frame-pointer-elim-non-leaf"))
+    return MF.getFrameInfo()->hasCalls();
+
+  return false;
 }
 
 /// LessPreciseFPMAD - This flag return true when -enable-fp-mad option
@@ -44,17 +47,3 @@ bool TargetOptions::LessPreciseFPMAD() const {
 bool TargetOptions::HonorSignDependentRoundingFPMath() const {
   return !UnsafeFPMath && HonorSignDependentRoundingFPMathOption;
 }
-
-/// getTrapFunctionName - If this returns a non-empty string, this means isel
-/// should lower Intrinsic::trap to a call to the specified function name
-/// instead of an ISD::TRAP node.
-StringRef TargetOptions::getTrapFunctionName() const {
-  return TrapFuncName;
-}
-
-/// getCFIFuncName - If this returns a non-empty string, then it is the name of
-/// the function that gets called on CFI violations in CFI non-enforcing mode
-/// (!TargetOptions::CFIEnforcing).
-StringRef TargetOptions::getCFIFuncName() const {
-  return CFIFuncName;
-}
diff --git a/lib/CodeGen/TargetRegisterInfo.cpp b/lib/CodeGen/TargetRegisterInfo.cpp
index 61a66b62392..12e9e8cf1b6 100644
--- a/lib/CodeGen/TargetRegisterInfo.cpp
+++ b/lib/CodeGen/TargetRegisterInfo.cpp
@@ -11,13 +11,26 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "llvm/Target/TargetRegisterInfo.h"
 #include "llvm/ADT/BitVector.h"
+#include "llvm/CodeGen/MachineFrameInfo.h"
 #include "llvm/CodeGen/MachineFunction.h"
 #include "llvm/CodeGen/MachineRegisterInfo.h"
 #include "llvm/CodeGen/VirtRegMap.h"
+#include "llvm/IR/Function.h"
 #include "llvm/Support/Debug.h"
 #include "llvm/Support/raw_ostream.h"
+#include "llvm/Target/TargetFrameLowering.h"
+#include "llvm/Target/TargetRegisterInfo.h"
+
+#define DEBUG_TYPE "target-reg-info"
+
+namespace llvm {
+cl::opt<bool>
+    ForceStackAlign("force-align-stack",
+                    cl::desc("Force align the stack to the minimum alignment"
+                             " needed for the function."),
+                    cl::init(false), cl::Hidden);
+} // end namespace llvm
 
 using namespace llvm;
 
@@ -266,7 +279,8 @@ TargetRegisterInfo::getRegAllocationHints(unsigned VirtReg,
                                           ArrayRef<MCPhysReg> Order,
                                           SmallVectorImpl<MCPhysReg> &Hints,
                                           const MachineFunction &MF,
-                                          const VirtRegMap *VRM) const {
+                                          const VirtRegMap *VRM,
+                                          const LiveRegMatrix *Matrix) const {
   const MachineRegisterInfo &MRI = MF.getRegInfo();
   std::pair<unsigned, unsigned> Hint = MRI.getRegAllocationHint(VirtReg);
 
@@ -295,6 +309,26 @@ TargetRegisterInfo::getRegAllocationHints(unsigned VirtReg,
   Hints.push_back(Phys);
 }
 
+bool TargetRegisterInfo::canRealignStack(const MachineFunction &MF) const {
+  return !MF.getFunction()->hasFnAttribute("no-realign-stack");
+}
+
+bool TargetRegisterInfo::needsStackRealignment(
+    const MachineFunction &MF) const {
+  const MachineFrameInfo *MFI = MF.getFrameInfo();
+  const TargetFrameLowering *TFI = MF.getSubtarget().getFrameLowering();
+  const Function *F = MF.getFunction();
+  unsigned StackAlign = TFI->getStackAlignment();
+  bool requiresRealignment = ((MFI->getMaxAlignment() > StackAlign) ||
+                              F->hasFnAttribute(Attribute::StackAlignment));
+  if (ForceStackAlign || requiresRealignment) {
+    if (canRealignStack(MF))
+      return true;
+    DEBUG(dbgs() << "Can't realign function's stack: " << F->getName() << "\n");
+  }
+  return false;
+}
+
 #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
 void
 TargetRegisterInfo::dumpReg(unsigned Reg, unsigned SubRegIndex,
diff --git a/lib/CodeGen/TargetSchedule.cpp b/lib/CodeGen/TargetSchedule.cpp
index ef2dab1287f..fc656396ade 100644
--- a/lib/CodeGen/TargetSchedule.cpp
+++ b/lib/CodeGen/TargetSchedule.cpp
@@ -211,11 +211,9 @@ unsigned TargetSchedModel::computeOperandLatency(
   if (SCDesc->isValid() && !DefMI->getOperand(DefOperIdx).isImplicit()
       && !DefMI->getDesc().OpInfo[DefOperIdx].isOptionalDef()
       && SchedModel.isComplete()) {
-    std::string Err;
-    raw_string_ostream ss(Err);
-    ss << "DefIdx " << DefIdx << " exceeds machine model writes for "
-       << *DefMI;
-    report_fatal_error(ss.str());
+    errs() << "DefIdx " << DefIdx << " exceeds machine model writes for "
+           << *DefMI;
+    llvm_unreachable("incomplete machine model");
   }
 #endif
   // FIXME: Automatically giving all implicit defs defaultDefLatency is
@@ -224,26 +222,29 @@ unsigned TargetSchedModel::computeOperandLatency(
   return DefMI->isTransient() ? 0 : TII->defaultDefLatency(SchedModel, DefMI);
 }
 
+unsigned
+TargetSchedModel::computeInstrLatency(const MCSchedClassDesc &SCDesc) const {
+  unsigned Latency = 0;
+  for (unsigned DefIdx = 0, DefEnd = SCDesc.NumWriteLatencyEntries;
+       DefIdx != DefEnd; ++DefIdx) {
+    // Lookup the definition's write latency in SubtargetInfo.
+    const MCWriteLatencyEntry *WLEntry =
+      STI->getWriteLatencyEntry(&SCDesc, DefIdx);
+    Latency = std::max(Latency, capLatency(WLEntry->Cycles));
+  }
+  return Latency;
+}
+
 unsigned TargetSchedModel::computeInstrLatency(unsigned Opcode) const {
   assert(hasInstrSchedModel() && "Only call this function with a SchedModel");
 
   unsigned SCIdx = TII->get(Opcode).getSchedClass();
   const MCSchedClassDesc *SCDesc = SchedModel.getSchedClassDesc(SCIdx);
-  unsigned Latency = 0;
 
-  if (SCDesc->isValid() && !SCDesc->isVariant()) {
-    for (unsigned DefIdx = 0, DefEnd = SCDesc->NumWriteLatencyEntries;
-         DefIdx != DefEnd; ++DefIdx) {
-      // Lookup the definition's write latency in SubtargetInfo.
-      const MCWriteLatencyEntry *WLEntry =
-          STI->getWriteLatencyEntry(SCDesc, DefIdx);
-      Latency = std::max(Latency, capLatency(WLEntry->Cycles));
-    }
-    return Latency;
-  }
+  if (SCDesc->isValid() && !SCDesc->isVariant())
+    return computeInstrLatency(*SCDesc);
 
-  assert(Latency && "No MI sched latency");
-  return 0;
+  llvm_unreachable("No MI sched latency");
 }
 
 unsigned
@@ -257,17 +258,8 @@ TargetSchedModel::computeInstrLatency(const MachineInstr *MI,
 
   if (hasInstrSchedModel()) {
     const MCSchedClassDesc *SCDesc = resolveSchedClass(MI);
-    if (SCDesc->isValid()) {
-      unsigned Latency = 0;
-      for (unsigned DefIdx = 0, DefEnd = SCDesc->NumWriteLatencyEntries;
-           DefIdx != DefEnd; ++DefIdx) {
-        // Lookup the definition's write latency in SubtargetInfo.
-        const MCWriteLatencyEntry *WLEntry =
-          STI->getWriteLatencyEntry(SCDesc, DefIdx);
-        Latency = std::max(Latency, capLatency(WLEntry->Cycles));
-      }
-      return Latency;
-    }
+    if (SCDesc->isValid())
+      return computeInstrLatency(*SCDesc);
   }
   return TII->defaultDefLatency(SchedModel, MI);
 }
diff --git a/lib/CodeGen/TwoAddressInstructionPass.cpp b/lib/CodeGen/TwoAddressInstructionPass.cpp
index 57daeab3ffb..1e30821dc74 100644
--- a/lib/CodeGen/TwoAddressInstructionPass.cpp
+++ b/lib/CodeGen/TwoAddressInstructionPass.cpp
@@ -189,7 +189,7 @@ sink3AddrInstruction(MachineInstr *MI, unsigned SavedReg,
 
   // Check if it's safe to move this instruction.
   bool SeenStore = true; // Be conservative.
-  if (!MI->isSafeToMove(TII, AA, SeenStore))
+  if (!MI->isSafeToMove(AA, SeenStore))
     return false;
 
   unsigned DefReg = 0;
@@ -861,7 +861,7 @@ rescheduleMIBelowKill(MachineBasicBlock::iterator &mi,
     return false;
 
   bool SeenStore = true;
-  if (!MI->isSafeToMove(TII, AA, SeenStore))
+  if (!MI->isSafeToMove(AA, SeenStore))
     return false;
 
   if (TII->getInstrLatency(InstrItins, MI) > 1)
@@ -1048,7 +1048,7 @@ rescheduleKillAboveMI(MachineBasicBlock::iterator &mi,
     return false;
 
   bool SeenStore = true;
-  if (!KillMI->isSafeToMove(TII, AA, SeenStore))
+  if (!KillMI->isSafeToMove(AA, SeenStore))
     return false;
 
   SmallSet<unsigned, 2> Uses;
@@ -1207,12 +1207,24 @@ tryInstructionTransform(MachineBasicBlock::iterator &mi,
     }
   }
 
+  // If the instruction is convertible to 3 Addr, instead
+  // of returning try 3 Addr transformation aggresively and
+  // use this variable to check later. Because it might be better.
+  // For example, we can just use `leal (%rsi,%rdi), %eax` and `ret`
+  // instead of the following code.
+  //   addl	%esi, %edi
+  //   movl	%edi, %eax
+  //   ret
+  bool Commuted = false;
+
   // If it's profitable to commute, try to do so.
   if (TryCommute && commuteInstruction(mi, regB, regC, Dist)) {
+    Commuted = true;
     ++NumCommuted;
     if (AggressiveCommute)
       ++NumAggrCommuted;
-    return false;
+    if (!MI.isConvertibleTo3Addr())
+      return false;
   }
 
   if (shouldOnlyCommute)
@@ -1220,7 +1232,7 @@ tryInstructionTransform(MachineBasicBlock::iterator &mi,
 
   // If there is one more use of regB later in the same MBB, consider
   // re-schedule this MI below it.
-  if (EnableRescheduling && rescheduleMIBelowKill(mi, nmi, regB)) {
+  if (!Commuted && EnableRescheduling && rescheduleMIBelowKill(mi, nmi, regB)) {
     ++NumReSchedDowns;
     return true;
   }
@@ -1237,6 +1249,10 @@ tryInstructionTransform(MachineBasicBlock::iterator &mi,
     }
   }
 
+  // Return if it is commuted but 3 addr conversion is failed.
+  if (Commuted)
+    return false;
+
   // If there is one more use of regB later in the same MBB, consider
   // re-schedule it before this MI if it's legal.
   if (EnableRescheduling && rescheduleKillAboveMI(mi, nmi, regB)) {
diff --git a/lib/CodeGen/VirtRegMap.cpp b/lib/CodeGen/VirtRegMap.cpp
index d9adfdf7b2e..02341b4d66b 100644
--- a/lib/CodeGen/VirtRegMap.cpp
+++ b/lib/CodeGen/VirtRegMap.cpp
@@ -163,10 +163,10 @@ class VirtRegRewriter : public MachineFunctionPass {
   SlotIndexes *Indexes;
   LiveIntervals *LIS;
   VirtRegMap *VRM;
-  SparseSet<unsigned> PhysRegs;
 
   void rewrite();
   void addMBBLiveIns();
+  bool readsUndefSubreg(const MachineOperand &MO) const;
 public:
   static char ID;
   VirtRegRewriter() : MachineFunctionPass(ID) {}
@@ -264,8 +264,7 @@ void VirtRegRewriter::addMBBLiveIns() {
             if ((SubRegLaneMask & S.LaneMask) == 0)
               continue;
             for (unsigned i = 0, e = LiveIn.size(); i != e; ++i) {
-              if (!LiveIn[i]->isLiveIn(SubReg))
-                LiveIn[i]->addLiveIn(SubReg);
+              LiveIn[i]->addLiveIn(SubReg);
             }
           }
           LiveIn.clear();
@@ -277,12 +276,41 @@ void VirtRegRewriter::addMBBLiveIns() {
         if (!Indexes->findLiveInMBBs(Seg.start, Seg.end, LiveIn))
           continue;
         for (unsigned i = 0, e = LiveIn.size(); i != e; ++i)
-          if (!LiveIn[i]->isLiveIn(PhysReg))
-            LiveIn[i]->addLiveIn(PhysReg);
+          LiveIn[i]->addLiveIn(PhysReg);
         LiveIn.clear();
       }
     }
   }
+
+  // Sort and unique MBB LiveIns as we've not checked if SubReg/PhysReg were in
+  // each MBB's LiveIns set before calling addLiveIn on them.
+  for (MachineBasicBlock &MBB : *MF)
+    MBB.sortUniqueLiveIns();
+}
+
+/// Returns true if the given machine operand \p MO only reads undefined lanes.
+/// The function only works for use operands with a subregister set.
+bool VirtRegRewriter::readsUndefSubreg(const MachineOperand &MO) const {
+  // Shortcut if the operand is already marked undef.
+  if (MO.isUndef())
+    return true;
+
+  unsigned Reg = MO.getReg();
+  const LiveInterval &LI = LIS->getInterval(Reg);
+  const MachineInstr &MI = *MO.getParent();
+  SlotIndex BaseIndex = LIS->getInstructionIndex(&MI);
+  // This code is only meant to handle reading undefined subregisters which
+  // we couldn't properly detect before.
+  assert(LI.liveAt(BaseIndex) &&
+         "Reads of completely dead register should be marked undef already");
+  unsigned SubRegIdx = MO.getSubReg();
+  unsigned UseMask = TRI->getSubRegIndexLaneMask(SubRegIdx);
+  // See if any of the relevant subregister liveranges is defined at this point.
+  for (const LiveInterval::SubRange &SR : LI.subranges()) {
+    if ((SR.LaneMask & UseMask) != 0 && SR.liveAt(BaseIndex))
+      return false;
+  }
+  return true;
 }
 
 void VirtRegRewriter::rewrite() {
@@ -290,54 +318,15 @@ void VirtRegRewriter::rewrite() {
   SmallVector<unsigned, 8> SuperDeads;
   SmallVector<unsigned, 8> SuperDefs;
   SmallVector<unsigned, 8> SuperKills;
-  SmallPtrSet<const MachineInstr *, 4> NoReturnInsts;
-
-  // Here we have a SparseSet to hold which PhysRegs are actually encountered
-  // in the MF we are about to iterate over so that later when we call
-  // setPhysRegUsed, we are only doing it for physRegs that were actually found
-  // in the program and not for all of the possible physRegs for the given
-  // target architecture. If the target has a lot of physRegs, then for a small
-  // program there will be a significant compile time reduction here.
-  PhysRegs.clear();
-  PhysRegs.setUniverse(TRI->getNumRegs());
-
-  // The function with uwtable should guarantee that the stack unwinder
-  // can unwind the stack to the previous frame.  Thus, we can't apply the
-  // noreturn optimization if the caller function has uwtable attribute.
-  bool HasUWTable = MF->getFunction()->hasFnAttribute(Attribute::UWTable);
 
   for (MachineFunction::iterator MBBI = MF->begin(), MBBE = MF->end();
        MBBI != MBBE; ++MBBI) {
     DEBUG(MBBI->print(dbgs(), Indexes));
-    bool IsExitBB = MBBI->succ_empty();
     for (MachineBasicBlock::instr_iterator
            MII = MBBI->instr_begin(), MIE = MBBI->instr_end(); MII != MIE;) {
       MachineInstr *MI = MII;
       ++MII;
 
-      // Check if this instruction is a call to a noreturn function.  If this
-      // is a call to noreturn function and we don't need the stack unwinding
-      // functionality (i.e. this function does not have uwtable attribute and
-      // the callee function has the nounwind attribute), then we can ignore
-      // the definitions set by this instruction.
-      if (!HasUWTable && IsExitBB && MI->isCall()) {
-        for (MachineInstr::mop_iterator MOI = MI->operands_begin(),
-               MOE = MI->operands_end(); MOI != MOE; ++MOI) {
-          MachineOperand &MO = *MOI;
-          if (!MO.isGlobal())
-            continue;
-          const Function *Func = dyn_cast<Function>(MO.getGlobal());
-          if (!Func || !Func->hasFnAttribute(Attribute::NoReturn) ||
-              // We need to keep correct unwind information
-              // even if the function will not return, since the
-              // runtime may need it.
-              !Func->hasFnAttribute(Attribute::NoUnwind))
-            continue;
-          NoReturnInsts.insert(MI);
-          break;
-        }
-      }
-
       for (MachineInstr::mop_iterator MOI = MI->operands_begin(),
            MOE = MI->operands_end(); MOI != MOE; ++MOI) {
         MachineOperand &MO = *MOI;
@@ -346,15 +335,6 @@ void VirtRegRewriter::rewrite() {
         if (MO.isRegMask())
           MRI->addPhysRegsUsedFromRegMask(MO.getRegMask());
 
-        // If we encounter a VirtReg or PhysReg then get at the PhysReg and add
-        // it to the physreg bitset.  Later we use only the PhysRegs that were
-        // actually encountered in the MF to populate the MRI's used physregs.
-        if (MO.isReg() && MO.getReg())
-          PhysRegs.insert(
-              TargetRegisterInfo::isVirtualRegister(MO.getReg()) ?
-              VRM->getPhys(MO.getReg()) :
-              MO.getReg());
-
         if (!MO.isReg() || !TargetRegisterInfo::isVirtualRegister(MO.getReg()))
           continue;
         unsigned VirtReg = MO.getReg();
@@ -364,32 +344,51 @@ void VirtRegRewriter::rewrite() {
         assert(!MRI->isReserved(PhysReg) && "Reserved register assignment");
 
         // Preserve semantics of sub-register operands.
-        if (MO.getSubReg()) {
-          // A virtual register kill refers to the whole register, so we may
-          // have to add <imp-use,kill> operands for the super-register.  A
-          // partial redef always kills and redefines the super-register.
-          if (NoSubRegLiveness && MO.readsReg()
-              && (MO.isDef() || MO.isKill()))
-            SuperKills.push_back(PhysReg);
-
-          if (MO.isDef()) {
-            // The <def,undef> flag only makes sense for sub-register defs, and
-            // we are substituting a full physreg.  An <imp-use,kill> operand
-            // from the SuperKills list will represent the partial read of the
-            // super-register.
-            MO.setIsUndef(false);
-
-            // Also add implicit defs for the super-register.
-            if (NoSubRegLiveness) {
+        unsigned SubReg = MO.getSubReg();
+        if (SubReg != 0) {
+          if (NoSubRegLiveness) {
+            // A virtual register kill refers to the whole register, so we may
+            // have to add <imp-use,kill> operands for the super-register.  A
+            // partial redef always kills and redefines the super-register.
+            if (MO.readsReg() && (MO.isDef() || MO.isKill()))
+              SuperKills.push_back(PhysReg);
+
+            if (MO.isDef()) {
+              // Also add implicit defs for the super-register.
               if (MO.isDead())
                 SuperDeads.push_back(PhysReg);
               else
                 SuperDefs.push_back(PhysReg);
             }
+          } else {
+            if (MO.isUse()) {
+              if (readsUndefSubreg(MO))
+                // We need to add an <undef> flag if the subregister is
+                // completely undefined (and we are not adding super-register
+                // defs).
+                MO.setIsUndef(true);
+            } else if (!MO.isDead()) {
+              assert(MO.isDef());
+              // Things get tricky when we ran out of lane mask bits and
+              // merged multiple lanes into the overflow bit: In this case
+              // our subregister liveness tracking isn't precise and we can't
+              // know what subregister parts are undefined, fall back to the
+              // implicit super-register def then.
+              unsigned LaneMask = TRI->getSubRegIndexLaneMask(SubReg);
+              if (TargetRegisterInfo::isImpreciseLaneMask(LaneMask))
+                SuperDefs.push_back(PhysReg);
+            }
           }
 
+          // The <def,undef> flag only makes sense for sub-register defs, and
+          // we are substituting a full physreg.  An <imp-use,kill> operand
+          // from the SuperKills list will represent the partial read of the
+          // super-register.
+          if (MO.isDef())
+            MO.setIsUndef(false);
+
           // PhysReg operands cannot have subregister indexes.
-          PhysReg = TRI->getSubReg(PhysReg, MO.getSubReg());
+          PhysReg = TRI->getSubReg(PhysReg, SubReg);
           assert(PhysReg && "Invalid SubReg for physical register");
           MO.setSubReg(0);
         }
@@ -414,41 +413,11 @@ void VirtRegRewriter::rewrite() {
       // Finally, remove any identity copies.
       if (MI->isIdentityCopy()) {
         ++NumIdCopies;
-        if (MI->getNumOperands() == 2) {
-          DEBUG(dbgs() << "Deleting identity copy.\n");
-          if (Indexes)
-            Indexes->removeMachineInstrFromMaps(MI);
-          // It's safe to erase MI because MII has already been incremented.
-          MI->eraseFromParent();
-        } else {
-          // Transform identity copy to a KILL to deal with subregisters.
-          MI->setDesc(TII->get(TargetOpcode::KILL));
-          DEBUG(dbgs() << "Identity copy: " << *MI);
-        }
-      }
-    }
-  }
-
-  // Tell MRI about physical registers in use.
-  if (NoReturnInsts.empty()) {
-    for (SparseSet<unsigned>::iterator
-        RegI = PhysRegs.begin(), E = PhysRegs.end(); RegI != E; ++RegI)
-      if (!MRI->reg_nodbg_empty(*RegI))
-        MRI->setPhysRegUsed(*RegI);
-  } else {
-    for (SparseSet<unsigned>::iterator
-        I = PhysRegs.begin(), E = PhysRegs.end(); I != E; ++I) {
-      unsigned Reg = *I;
-      if (MRI->reg_nodbg_empty(Reg))
-        continue;
-      // Check if this register has a use that will impact the rest of the
-      // code. Uses in debug and noreturn instructions do not impact the
-      // generated code.
-      for (MachineInstr &It : MRI->reg_nodbg_instructions(Reg)) {
-        if (!NoReturnInsts.count(&It)) {
-          MRI->setPhysRegUsed(Reg);
-          break;
-        }
+        DEBUG(dbgs() << "Deleting identity copy.\n");
+        if (Indexes)
+          Indexes->removeMachineInstrFromMaps(MI);
+        // It's safe to erase MI because MII has already been incremented.
+        MI->eraseFromParent();
       }
     }
   }
diff --git a/lib/CodeGen/WinEHPrepare.cpp b/lib/CodeGen/WinEHPrepare.cpp
index 35b944ea309..0d26ed333ca 100644
--- a/lib/CodeGen/WinEHPrepare.cpp
+++ b/lib/CodeGen/WinEHPrepare.cpp
@@ -8,9 +8,11 @@
 //===----------------------------------------------------------------------===//
 //
 // This pass lowers LLVM IR exception handling into something closer to what the
-// backend wants. It snifs the personality function to see which kind of
-// preparation is necessary. If the personality function uses the Itanium LSDA,
-// this pass delegates to the DWARF EH preparation pass.
+// backend wants for functions using a personality function from a runtime
+// provided by MSVC. Functions with other personality functions are left alone
+// and may be prepared by other passes. In particular, all supported MSVC
+// personality functions require cleanup code to be outlined, and the C++
+// personality requires catch handler code to be outlined.
 //
 //===----------------------------------------------------------------------===//
 
@@ -18,8 +20,11 @@
 #include "llvm/ADT/MapVector.h"
 #include "llvm/ADT/STLExtras.h"
 #include "llvm/ADT/SmallSet.h"
+#include "llvm/ADT/SetVector.h"
+#include "llvm/ADT/Triple.h"
 #include "llvm/ADT/TinyPtrVector.h"
 #include "llvm/Analysis/LibCallSemantics.h"
+#include "llvm/Analysis/TargetLibraryInfo.h"
 #include "llvm/CodeGen/WinEHFuncInfo.h"
 #include "llvm/IR/Dominators.h"
 #include "llvm/IR/Function.h"
@@ -29,7 +34,6 @@
 #include "llvm/IR/Module.h"
 #include "llvm/IR/PatternMatch.h"
 #include "llvm/Pass.h"
-#include "llvm/Support/CommandLine.h"
 #include "llvm/Support/Debug.h"
 #include "llvm/Support/raw_ostream.h"
 #include "llvm/Transforms/Utils/BasicBlockUtils.h"
@@ -70,7 +74,10 @@ class WinEHPrepare : public FunctionPass {
 public:
   static char ID; // Pass identification, replacement for typeid.
   WinEHPrepare(const TargetMachine *TM = nullptr)
-      : FunctionPass(ID), DT(nullptr) {}
+      : FunctionPass(ID) {
+    if (TM)
+      TheTriple = TM->getTargetTriple();
+  }
 
   bool runOnFunction(Function &Fn) override;
 
@@ -85,15 +92,26 @@ class WinEHPrepare : public FunctionPass {
 private:
   bool prepareExceptionHandlers(Function &F,
                                 SmallVectorImpl<LandingPadInst *> &LPads);
+  void identifyEHBlocks(Function &F, SmallVectorImpl<LandingPadInst *> &LPads);
   void promoteLandingPadValues(LandingPadInst *LPad);
+  void demoteValuesLiveAcrossHandlers(Function &F,
+                                      SmallVectorImpl<LandingPadInst *> &LPads);
+  void findSEHEHReturnPoints(Function &F,
+                             SetVector<BasicBlock *> &EHReturnBlocks);
+  void findCXXEHReturnPoints(Function &F,
+                             SetVector<BasicBlock *> &EHReturnBlocks);
+  void getPossibleReturnTargets(Function *ParentF, Function *HandlerF,
+                                SetVector<BasicBlock*> &Targets);
   void completeNestedLandingPad(Function *ParentFn,
                                 LandingPadInst *OutlinedLPad,
                                 const LandingPadInst *OriginalLPad,
                                 FrameVarInfoMap &VarInfo);
+  Function *createHandlerFunc(Function *ParentFn, Type *RetTy,
+                              const Twine &Name, Module *M, Value *&ParentFP);
   bool outlineHandler(ActionHandler *Action, Function *SrcFn,
                       LandingPadInst *LPad, BasicBlock *StartBB,
                       FrameVarInfoMap &VarInfo);
-  void addStubInvokeToHandlerIfNeeded(Function *Handler, Value *PersonalityFn);
+  void addStubInvokeToHandlerIfNeeded(Function *Handler);
 
   void mapLandingPadBlocks(LandingPadInst *LPad, LandingPadActions &Actions);
   CatchHandler *findCatchHandler(BasicBlock *BB, BasicBlock *&NextBB,
@@ -103,12 +121,18 @@ class WinEHPrepare : public FunctionPass {
 
   void processSEHCatchHandler(CatchHandler *Handler, BasicBlock *StartBB);
 
+  Triple TheTriple;
+
   // All fields are reset by runOnFunction.
-  DominatorTree *DT;
-  EHPersonality Personality;
+  DominatorTree *DT = nullptr;
+  const TargetLibraryInfo *LibInfo = nullptr;
+  EHPersonality Personality = EHPersonality::Unknown;
   CatchHandlerMapTy CatchHandlerMap;
   CleanupHandlerMapTy CleanupHandlerMap;
   DenseMap<const LandingPadInst *, LandingPadMap> LPadMaps;
+  SmallPtrSet<BasicBlock *, 4> NormalBlocks;
+  SmallPtrSet<BasicBlock *, 4> EHBlocks;
+  SetVector<BasicBlock *> EHReturnBlocks;
 
   // This maps landing pad instructions found in outlined handlers to
   // the landing pad instruction in the parent function from which they
@@ -130,11 +154,17 @@ class WinEHPrepare : public FunctionPass {
   // outlined into a handler.  This is done after all handlers have been
   // outlined but before the outlined code is pruned from the parent function.
   DenseMap<const BasicBlock *, BasicBlock *> LPadTargetBlocks;
+
+  // Map from outlined handler to call to parent local address. Only used for
+  // 32-bit EH.
+  DenseMap<Function *, Value *> HandlerToParentFP;
+
+  AllocaInst *SEHExceptionCodeSlot = nullptr;
 };
 
 class WinEHFrameVariableMaterializer : public ValueMaterializer {
 public:
-  WinEHFrameVariableMaterializer(Function *OutlinedFn,
+  WinEHFrameVariableMaterializer(Function *OutlinedFn, Value *ParentFP,
                                  FrameVarInfoMap &FrameVarInfo);
   ~WinEHFrameVariableMaterializer() override {}
 
@@ -170,12 +200,12 @@ class LandingPadMap {
 
 class WinEHCloningDirectorBase : public CloningDirector {
 public:
-  WinEHCloningDirectorBase(Function *HandlerFn, FrameVarInfoMap &VarInfo,
-                           LandingPadMap &LPadMap)
-      : Materializer(HandlerFn, VarInfo),
+  WinEHCloningDirectorBase(Function *HandlerFn, Value *ParentFP,
+                           FrameVarInfoMap &VarInfo, LandingPadMap &LPadMap)
+      : Materializer(HandlerFn, ParentFP, VarInfo),
         SelectorIDType(Type::getInt32Ty(HandlerFn->getContext())),
         Int8PtrType(Type::getInt8PtrTy(HandlerFn->getContext())),
-        LPadMap(LPadMap) {}
+        LPadMap(LPadMap), ParentFP(ParentFP) {}
 
   CloningAction handleInstruction(ValueToValueMapTy &VMap,
                                   const Instruction *Inst,
@@ -190,12 +220,18 @@ class WinEHCloningDirectorBase : public CloningDirector {
   virtual CloningAction handleTypeIdFor(ValueToValueMapTy &VMap,
                                         const Instruction *Inst,
                                         BasicBlock *NewBB) = 0;
+  virtual CloningAction handleIndirectBr(ValueToValueMapTy &VMap,
+                                         const IndirectBrInst *IBr,
+                                         BasicBlock *NewBB) = 0;
   virtual CloningAction handleInvoke(ValueToValueMapTy &VMap,
                                      const InvokeInst *Invoke,
                                      BasicBlock *NewBB) = 0;
   virtual CloningAction handleResume(ValueToValueMapTy &VMap,
                                      const ResumeInst *Resume,
                                      BasicBlock *NewBB) = 0;
+  virtual CloningAction handleCompare(ValueToValueMapTy &VMap,
+                                      const CmpInst *Compare,
+                                      BasicBlock *NewBB) = 0;
   virtual CloningAction handleLandingPad(ValueToValueMapTy &VMap,
                                          const LandingPadInst *LPad,
                                          BasicBlock *NewBB) = 0;
@@ -207,17 +243,22 @@ class WinEHCloningDirectorBase : public CloningDirector {
   Type *SelectorIDType;
   Type *Int8PtrType;
   LandingPadMap &LPadMap;
+
+  /// The value representing the parent frame pointer.
+  Value *ParentFP;
 };
 
 class WinEHCatchDirector : public WinEHCloningDirectorBase {
 public:
   WinEHCatchDirector(
-      Function *CatchFn, Value *Selector, FrameVarInfoMap &VarInfo,
-      LandingPadMap &LPadMap,
-      DenseMap<LandingPadInst *, const LandingPadInst *> &NestedLPads)
-      : WinEHCloningDirectorBase(CatchFn, VarInfo, LPadMap),
+      Function *CatchFn, Value *ParentFP, Value *Selector,
+      FrameVarInfoMap &VarInfo, LandingPadMap &LPadMap,
+      DenseMap<LandingPadInst *, const LandingPadInst *> &NestedLPads,
+      DominatorTree *DT, SmallPtrSetImpl<BasicBlock *> &EHBlocks)
+      : WinEHCloningDirectorBase(CatchFn, ParentFP, VarInfo, LPadMap),
         CurrentSelector(Selector->stripPointerCasts()),
-        ExceptionObjectVar(nullptr), NestedLPtoOriginalLP(NestedLPads) {}
+        ExceptionObjectVar(nullptr), NestedLPtoOriginalLP(NestedLPads),
+        DT(DT), EHBlocks(EHBlocks) {}
 
   CloningAction handleBeginCatch(ValueToValueMapTy &VMap,
                                  const Instruction *Inst,
@@ -227,10 +268,15 @@ class WinEHCatchDirector : public WinEHCloningDirectorBase {
   CloningAction handleTypeIdFor(ValueToValueMapTy &VMap,
                                 const Instruction *Inst,
                                 BasicBlock *NewBB) override;
+  CloningAction handleIndirectBr(ValueToValueMapTy &VMap,
+                                 const IndirectBrInst *IBr,
+                                 BasicBlock *NewBB) override;
   CloningAction handleInvoke(ValueToValueMapTy &VMap, const InvokeInst *Invoke,
                              BasicBlock *NewBB) override;
   CloningAction handleResume(ValueToValueMapTy &VMap, const ResumeInst *Resume,
                              BasicBlock *NewBB) override;
+  CloningAction handleCompare(ValueToValueMapTy &VMap, const CmpInst *Compare,
+                              BasicBlock *NewBB) override;
   CloningAction handleLandingPad(ValueToValueMapTy &VMap,
                                  const LandingPadInst *LPad,
                                  BasicBlock *NewBB) override;
@@ -247,13 +293,16 @@ class WinEHCatchDirector : public WinEHCloningDirectorBase {
   // This will be a reference to the field of the same name in the WinEHPrepare
   // object which instantiates this WinEHCatchDirector object.
   DenseMap<LandingPadInst *, const LandingPadInst *> &NestedLPtoOriginalLP;
+  DominatorTree *DT;
+  SmallPtrSetImpl<BasicBlock *> &EHBlocks;
 };
 
 class WinEHCleanupDirector : public WinEHCloningDirectorBase {
 public:
-  WinEHCleanupDirector(Function *CleanupFn, FrameVarInfoMap &VarInfo,
-                       LandingPadMap &LPadMap)
-      : WinEHCloningDirectorBase(CleanupFn, VarInfo, LPadMap) {}
+  WinEHCleanupDirector(Function *CleanupFn, Value *ParentFP,
+                       FrameVarInfoMap &VarInfo, LandingPadMap &LPadMap)
+      : WinEHCloningDirectorBase(CleanupFn, ParentFP, VarInfo,
+                                 LPadMap) {}
 
   CloningAction handleBeginCatch(ValueToValueMapTy &VMap,
                                  const Instruction *Inst,
@@ -263,10 +312,15 @@ class WinEHCleanupDirector : public WinEHCloningDirectorBase {
   CloningAction handleTypeIdFor(ValueToValueMapTy &VMap,
                                 const Instruction *Inst,
                                 BasicBlock *NewBB) override;
+  CloningAction handleIndirectBr(ValueToValueMapTy &VMap,
+                                 const IndirectBrInst *IBr,
+                                 BasicBlock *NewBB) override;
   CloningAction handleInvoke(ValueToValueMapTy &VMap, const InvokeInst *Invoke,
                              BasicBlock *NewBB) override;
   CloningAction handleResume(ValueToValueMapTy &VMap, const ResumeInst *Resume,
                              BasicBlock *NewBB) override;
+  CloningAction handleCompare(ValueToValueMapTy &VMap, const CmpInst *Compare,
+                              BasicBlock *NewBB) override;
   CloningAction handleLandingPad(ValueToValueMapTy &VMap,
                                  const LandingPadInst *LPad,
                                  BasicBlock *NewBB) override;
@@ -306,12 +360,11 @@ FunctionPass *llvm::createWinEHPass(const TargetMachine *TM) {
   return new WinEHPrepare(TM);
 }
 
-// FIXME: Remove this once the backend can handle the prepared IR.
-static cl::opt<bool>
-    SEHPrepare("sehprepare", cl::Hidden,
-               cl::desc("Prepare functions with SEH personalities"));
-
 bool WinEHPrepare::runOnFunction(Function &Fn) {
+  // No need to prepare outlined handlers.
+  if (Fn.hasFnAttribute("wineh-parent"))
+    return false;
+
   SmallVector<LandingPadInst *, 4> LPads;
   SmallVector<ResumeInst *, 4> Resumes;
   for (BasicBlock &BB : Fn) {
@@ -326,23 +379,14 @@ bool WinEHPrepare::runOnFunction(Function &Fn) {
     return false;
 
   // Classify the personality to see what kind of preparation we need.
-  Personality = classifyEHPersonality(LPads.back()->getPersonalityFn());
+  Personality = classifyEHPersonality(Fn.getPersonalityFn());
 
   // Do nothing if this is not an MSVC personality.
   if (!isMSVCEHPersonality(Personality))
     return false;
 
   DT = &getAnalysis<DominatorTreeWrapperPass>().getDomTree();
-
-  if (isAsynchronousEHPersonality(Personality) && !SEHPrepare) {
-    // Replace all resume instructions with unreachable.
-    // FIXME: Remove this once the backend can handle the prepared IR.
-    for (ResumeInst *Resume : Resumes) {
-      IRBuilder<>(Resume).CreateUnreachable();
-      Resume->eraseFromParent();
-    }
-    return true;
-  }
+  LibInfo = &getAnalysis<TargetLibraryInfoWrapperPass>().getTLI();
 
   // If there were any landing pads, prepareExceptionHandlers will make changes.
   prepareExceptionHandlers(Fn, LPads);
@@ -353,10 +397,342 @@ bool WinEHPrepare::doFinalization(Module &M) { return false; }
 
 void WinEHPrepare::getAnalysisUsage(AnalysisUsage &AU) const {
   AU.addRequired<DominatorTreeWrapperPass>();
+  AU.addRequired<TargetLibraryInfoWrapperPass>();
+}
+
+static bool isSelectorDispatch(BasicBlock *BB, BasicBlock *&CatchHandler,
+                               Constant *&Selector, BasicBlock *&NextBB);
+
+// Finds blocks reachable from the starting set Worklist. Does not follow unwind
+// edges or blocks listed in StopPoints.
+static void findReachableBlocks(SmallPtrSetImpl<BasicBlock *> &ReachableBBs,
+                                SetVector<BasicBlock *> &Worklist,
+                                const SetVector<BasicBlock *> *StopPoints) {
+  while (!Worklist.empty()) {
+    BasicBlock *BB = Worklist.pop_back_val();
+
+    // Don't cross blocks that we should stop at.
+    if (StopPoints && StopPoints->count(BB))
+      continue;
+
+    if (!ReachableBBs.insert(BB).second)
+      continue; // Already visited.
+
+    // Don't follow unwind edges of invokes.
+    if (auto *II = dyn_cast<InvokeInst>(BB->getTerminator())) {
+      Worklist.insert(II->getNormalDest());
+      continue;
+    }
+
+    // Otherwise, follow all successors.
+    Worklist.insert(succ_begin(BB), succ_end(BB));
+  }
+}
+
+// Attempt to find an instruction where a block can be split before
+// a call to llvm.eh.begincatch and its operands.  If the block
+// begins with the begincatch call or one of its adjacent operands
+// the block will not be split.
+static Instruction *findBeginCatchSplitPoint(BasicBlock *BB,
+                                             IntrinsicInst *II) {
+  // If the begincatch call is already the first instruction in the block,
+  // don't split.
+  Instruction *FirstNonPHI = BB->getFirstNonPHI();
+  if (II == FirstNonPHI)
+    return nullptr;
+
+  // If either operand is in the same basic block as the instruction and
+  // isn't used by another instruction before the begincatch call, include it
+  // in the split block.
+  auto *Op0 = dyn_cast<Instruction>(II->getOperand(0));
+  auto *Op1 = dyn_cast<Instruction>(II->getOperand(1));
+
+  Instruction *I = II->getPrevNode();
+  Instruction *LastI = II;
+
+  while (I == Op0 || I == Op1) {
+    // If the block begins with one of the operands and there are no other
+    // instructions between the operand and the begincatch call, don't split.
+    if (I == FirstNonPHI)
+      return nullptr;
+
+    LastI = I;
+    I = I->getPrevNode();
+  }
+
+  // If there is at least one instruction in the block before the begincatch
+  // call and its operands, split the block at either the begincatch or
+  // its operand.
+  return LastI;
+}
+
+/// Find all points where exceptional control rejoins normal control flow via
+/// llvm.eh.endcatch. Add them to the normal bb reachability worklist.
+void WinEHPrepare::findCXXEHReturnPoints(
+    Function &F, SetVector<BasicBlock *> &EHReturnBlocks) {
+  for (auto BBI = F.begin(), BBE = F.end(); BBI != BBE; ++BBI) {
+    BasicBlock *BB = BBI;
+    for (Instruction &I : *BB) {
+      if (match(&I, m_Intrinsic<Intrinsic::eh_begincatch>())) {
+        Instruction *SplitPt =
+            findBeginCatchSplitPoint(BB, cast<IntrinsicInst>(&I));
+        if (SplitPt) {
+          // Split the block before the llvm.eh.begincatch call to allow
+          // cleanup and catch code to be distinguished later.
+          // Do not update BBI because we still need to process the
+          // portion of the block that we are splitting off.
+          SplitBlock(BB, SplitPt, DT);
+          break;
+        }
+      }
+      if (match(&I, m_Intrinsic<Intrinsic::eh_endcatch>())) {
+        // Split the block after the call to llvm.eh.endcatch if there is
+        // anything other than an unconditional branch, or if the successor
+        // starts with a phi.
+        auto *Br = dyn_cast<BranchInst>(I.getNextNode());
+        if (!Br || !Br->isUnconditional() ||
+            isa<PHINode>(Br->getSuccessor(0)->begin())) {
+          DEBUG(dbgs() << "splitting block " << BB->getName()
+                       << " with llvm.eh.endcatch\n");
+          BBI = SplitBlock(BB, I.getNextNode(), DT);
+        }
+        // The next BB is normal control flow.
+        EHReturnBlocks.insert(BB->getTerminator()->getSuccessor(0));
+        break;
+      }
+    }
+  }
+}
+
+static bool isCatchAllLandingPad(const BasicBlock *BB) {
+  const LandingPadInst *LP = BB->getLandingPadInst();
+  if (!LP)
+    return false;
+  unsigned N = LP->getNumClauses();
+  return (N > 0 && LP->isCatch(N - 1) &&
+          isa<ConstantPointerNull>(LP->getClause(N - 1)));
+}
+
+/// Find all points where exceptions control rejoins normal control flow via
+/// selector dispatch.
+void WinEHPrepare::findSEHEHReturnPoints(
+    Function &F, SetVector<BasicBlock *> &EHReturnBlocks) {
+  for (auto BBI = F.begin(), BBE = F.end(); BBI != BBE; ++BBI) {
+    BasicBlock *BB = BBI;
+    // If the landingpad is a catch-all, treat the whole lpad as if it is
+    // reachable from normal control flow.
+    // FIXME: This is imprecise. We need a better way of identifying where a
+    // catch-all starts and cleanups stop. As far as LLVM is concerned, there
+    // is no difference.
+    if (isCatchAllLandingPad(BB)) {
+      EHReturnBlocks.insert(BB);
+      continue;
+    }
+
+    BasicBlock *CatchHandler;
+    BasicBlock *NextBB;
+    Constant *Selector;
+    if (isSelectorDispatch(BB, CatchHandler, Selector, NextBB)) {
+      // Split the edge if there are multiple predecessors. This creates a place
+      // where we can insert EH recovery code.
+      if (!CatchHandler->getSinglePredecessor()) {
+        DEBUG(dbgs() << "splitting EH return edge from " << BB->getName()
+                     << " to " << CatchHandler->getName() << '\n');
+        BBI = CatchHandler = SplitCriticalEdge(
+            BB, std::find(succ_begin(BB), succ_end(BB), CatchHandler));
+      }
+      EHReturnBlocks.insert(CatchHandler);
+    }
+  }
+}
+
+void WinEHPrepare::identifyEHBlocks(Function &F, 
+                                    SmallVectorImpl<LandingPadInst *> &LPads) {
+  DEBUG(dbgs() << "Demoting values live across exception handlers in function "
+               << F.getName() << '\n');
+
+  // Build a set of all non-exceptional blocks and exceptional blocks.
+  // - Non-exceptional blocks are blocks reachable from the entry block while
+  //   not following invoke unwind edges.
+  // - Exceptional blocks are blocks reachable from landingpads. Analysis does
+  //   not follow llvm.eh.endcatch blocks, which mark a transition from
+  //   exceptional to normal control.
+
+  if (Personality == EHPersonality::MSVC_CXX)
+    findCXXEHReturnPoints(F, EHReturnBlocks);
+  else
+    findSEHEHReturnPoints(F, EHReturnBlocks);
+
+  DEBUG({
+    dbgs() << "identified the following blocks as EH return points:\n";
+    for (BasicBlock *BB : EHReturnBlocks)
+      dbgs() << "  " << BB->getName() << '\n';
+  });
+
+// Join points should not have phis at this point, unless they are a
+// landingpad, in which case we will demote their phis later.
+#ifndef NDEBUG
+  for (BasicBlock *BB : EHReturnBlocks)
+    assert((BB->isLandingPad() || !isa<PHINode>(BB->begin())) &&
+           "non-lpad EH return block has phi");
+#endif
+
+  // Normal blocks are the blocks reachable from the entry block and all EH
+  // return points.
+  SetVector<BasicBlock *> Worklist;
+  Worklist = EHReturnBlocks;
+  Worklist.insert(&F.getEntryBlock());
+  findReachableBlocks(NormalBlocks, Worklist, nullptr);
+  DEBUG({
+    dbgs() << "marked the following blocks as normal:\n";
+    for (BasicBlock *BB : NormalBlocks)
+      dbgs() << "  " << BB->getName() << '\n';
+  });
+
+  // Exceptional blocks are the blocks reachable from landingpads that don't
+  // cross EH return points.
+  Worklist.clear();
+  for (auto *LPI : LPads)
+    Worklist.insert(LPI->getParent());
+  findReachableBlocks(EHBlocks, Worklist, &EHReturnBlocks);
+  DEBUG({
+    dbgs() << "marked the following blocks as exceptional:\n";
+    for (BasicBlock *BB : EHBlocks)
+      dbgs() << "  " << BB->getName() << '\n';
+  });
+
+}
+
+/// Ensure that all values live into and out of exception handlers are stored
+/// in memory.
+/// FIXME: This falls down when values are defined in one handler and live into
+/// another handler. For example, a cleanup defines a value used only by a
+/// catch handler.
+void WinEHPrepare::demoteValuesLiveAcrossHandlers(
+    Function &F, SmallVectorImpl<LandingPadInst *> &LPads) {
+  DEBUG(dbgs() << "Demoting values live across exception handlers in function "
+               << F.getName() << '\n');
+
+  // identifyEHBlocks() should have been called before this function.
+  assert(!NormalBlocks.empty());
+
+  // Try to avoid demoting EH pointer and selector values. They get in the way
+  // of our pattern matching.
+  SmallPtrSet<Instruction *, 10> EHVals;
+  for (BasicBlock &BB : F) {
+    LandingPadInst *LP = BB.getLandingPadInst();
+    if (!LP)
+      continue;
+    EHVals.insert(LP);
+    for (User *U : LP->users()) {
+      auto *EI = dyn_cast<ExtractValueInst>(U);
+      if (!EI)
+        continue;
+      EHVals.insert(EI);
+      for (User *U2 : EI->users()) {
+        if (auto *PN = dyn_cast<PHINode>(U2))
+          EHVals.insert(PN);
+      }
+    }
+  }
+
+  SetVector<Argument *> ArgsToDemote;
+  SetVector<Instruction *> InstrsToDemote;
+  for (BasicBlock &BB : F) {
+    bool IsNormalBB = NormalBlocks.count(&BB);
+    bool IsEHBB = EHBlocks.count(&BB);
+    if (!IsNormalBB && !IsEHBB)
+      continue; // Blocks that are neither normal nor EH are unreachable.
+    for (Instruction &I : BB) {
+      for (Value *Op : I.operands()) {
+        // Don't demote static allocas, constants, and labels.
+        if (isa<Constant>(Op) || isa<BasicBlock>(Op) || isa<InlineAsm>(Op))
+          continue;
+        auto *AI = dyn_cast<AllocaInst>(Op);
+        if (AI && AI->isStaticAlloca())
+          continue;
+
+        if (auto *Arg = dyn_cast<Argument>(Op)) {
+          if (IsEHBB) {
+            DEBUG(dbgs() << "Demoting argument " << *Arg
+                         << " used by EH instr: " << I << "\n");
+            ArgsToDemote.insert(Arg);
+          }
+          continue;
+        }
+
+        // Don't demote EH values.
+        auto *OpI = cast<Instruction>(Op);
+        if (EHVals.count(OpI))
+          continue;
+
+        BasicBlock *OpBB = OpI->getParent();
+        // If a value is produced and consumed in the same BB, we don't need to
+        // demote it.
+        if (OpBB == &BB)
+          continue;
+        bool IsOpNormalBB = NormalBlocks.count(OpBB);
+        bool IsOpEHBB = EHBlocks.count(OpBB);
+        if (IsNormalBB != IsOpNormalBB || IsEHBB != IsOpEHBB) {
+          DEBUG({
+            dbgs() << "Demoting instruction live in-out from EH:\n";
+            dbgs() << "Instr: " << *OpI << '\n';
+            dbgs() << "User: " << I << '\n';
+          });
+          InstrsToDemote.insert(OpI);
+        }
+      }
+    }
+  }
+
+  // Demote values live into and out of handlers.
+  // FIXME: This demotion is inefficient. We should insert spills at the point
+  // of definition, insert one reload in each handler that uses the value, and
+  // insert reloads in the BB used to rejoin normal control flow.
+  Instruction *AllocaInsertPt = F.getEntryBlock().getFirstInsertionPt();
+  for (Instruction *I : InstrsToDemote)
+    DemoteRegToStack(*I, false, AllocaInsertPt);
+
+  // Demote arguments separately, and only for uses in EH blocks.
+  for (Argument *Arg : ArgsToDemote) {
+    auto *Slot = new AllocaInst(Arg->getType(), nullptr,
+                                Arg->getName() + ".reg2mem", AllocaInsertPt);
+    SmallVector<User *, 4> Users(Arg->user_begin(), Arg->user_end());
+    for (User *U : Users) {
+      auto *I = dyn_cast<Instruction>(U);
+      if (I && EHBlocks.count(I->getParent())) {
+        auto *Reload = new LoadInst(Slot, Arg->getName() + ".reload", false, I);
+        U->replaceUsesOfWith(Arg, Reload);
+      }
+    }
+    new StoreInst(Arg, Slot, AllocaInsertPt);
+  }
+
+  // Demote landingpad phis, as the landingpad will be removed from the machine
+  // CFG.
+  for (LandingPadInst *LPI : LPads) {
+    BasicBlock *BB = LPI->getParent();
+    while (auto *Phi = dyn_cast<PHINode>(BB->begin()))
+      DemotePHIToStack(Phi, AllocaInsertPt);
+  }
+
+  DEBUG(dbgs() << "Demoted " << InstrsToDemote.size() << " instructions and "
+               << ArgsToDemote.size() << " arguments for WinEHPrepare\n\n");
 }
 
 bool WinEHPrepare::prepareExceptionHandlers(
     Function &F, SmallVectorImpl<LandingPadInst *> &LPads) {
+  // Don't run on functions that are already prepared.
+  for (LandingPadInst *LPad : LPads) {
+    BasicBlock *LPadBB = LPad->getParent();
+    for (Instruction &Inst : *LPadBB)
+      if (match(&Inst, m_Intrinsic<Intrinsic::eh_actions>()))
+        return false;
+  }
+
+  identifyEHBlocks(F, LPads);
+  demoteValuesLiveAcrossHandlers(F, LPads);
+
   // These containers are used to re-map frame variables that are used in
   // outlined catch and cleanup handlers.  They will be populated as the
   // handlers are outlined.
@@ -372,20 +748,37 @@ bool WinEHPrepare::prepareExceptionHandlers(
   Type *Int32Type = Type::getInt32Ty(Context);
   Function *ActionIntrin = Intrinsic::getDeclaration(M, Intrinsic::eh_actions);
 
+  if (isAsynchronousEHPersonality(Personality)) {
+    // FIXME: Switch the ehptr type to i32 and then switch this.
+    SEHExceptionCodeSlot =
+        new AllocaInst(Int8PtrType, nullptr, "seh_exception_code",
+                       F.getEntryBlock().getFirstInsertionPt());
+  }
+
+  // In order to handle the case where one outlined catch handler returns
+  // to a block within another outlined catch handler that would otherwise
+  // be unreachable, we need to outline the nested landing pad before we
+  // outline the landing pad which encloses it.
+  if (!isAsynchronousEHPersonality(Personality))
+    std::sort(LPads.begin(), LPads.end(),
+              [this](LandingPadInst *const &L, LandingPadInst *const &R) {
+                return DT->properlyDominates(R->getParent(), L->getParent());
+              });
+
+  // This container stores the llvm.eh.recover and IndirectBr instructions
+  // that make up the body of each landing pad after it has been outlined.
+  // We need to defer the population of the target list for the indirectbr
+  // until all landing pads have been outlined so that we can handle the
+  // case of blocks in the target that are reached only from nested
+  // landing pads.
+  SmallVector<std::pair<CallInst*, IndirectBrInst *>, 4> LPadImpls;
+
   for (LandingPadInst *LPad : LPads) {
     // Look for evidence that this landingpad has already been processed.
     bool LPadHasActionList = false;
     BasicBlock *LPadBB = LPad->getParent();
     for (Instruction &Inst : *LPadBB) {
-      if (auto *IntrinCall = dyn_cast<IntrinsicInst>(&Inst)) {
-        if (IntrinCall->getIntrinsicID() == Intrinsic::eh_actions) {
-          LPadHasActionList = true;
-          break;
-        }
-      }
-      // FIXME: This is here to help with the development of nested landing pad
-      //        outlining.  It should be removed when that is finished.
-      if (isa<UnreachableInst>(Inst)) {
+      if (match(&Inst, m_Intrinsic<Intrinsic::eh_actions>())) {
         LPadHasActionList = true;
         break;
       }
@@ -422,36 +815,55 @@ bool WinEHPrepare::prepareExceptionHandlers(
       outlineHandler(Action, &F, LPad, StartBB, FrameVarInfo);
     }
 
-    // Replace the landing pad with a new llvm.eh.action based landing pad.
-    BasicBlock *NewLPadBB = BasicBlock::Create(Context, "lpad", &F, LPadBB);
-    assert(!isa<PHINode>(LPadBB->begin()));
-    auto *NewLPad = cast<LandingPadInst>(LPad->clone());
-    NewLPadBB->getInstList().push_back(NewLPad);
-    while (!pred_empty(LPadBB)) {
-      auto *pred = *pred_begin(LPadBB);
-      InvokeInst *Invoke = cast<InvokeInst>(pred->getTerminator());
-      Invoke->setUnwindDest(NewLPadBB);
-    }
+    // Split the block after the landingpad instruction so that it is just a
+    // call to llvm.eh.actions followed by indirectbr.
+    assert(!isa<PHINode>(LPadBB->begin()) && "lpad phi not removed");
+    SplitBlock(LPadBB, LPad->getNextNode(), DT);
+    // Erase the branch inserted by the split so we can insert indirectbr.
+    LPadBB->getTerminator()->eraseFromParent();
 
-    // If anyone is still using the old landingpad value, just give them undef
-    // instead. The eh pointer and selector values are not real.
+    // Replace all extracted values with undef and ultimately replace the
+    // landingpad with undef.
+    SmallVector<Instruction *, 4> SEHCodeUses;
+    SmallVector<Instruction *, 4> EHUndefs;
+    for (User *U : LPad->users()) {
+      auto *E = dyn_cast<ExtractValueInst>(U);
+      if (!E)
+        continue;
+      assert(E->getNumIndices() == 1 &&
+             "Unexpected operation: extracting both landing pad values");
+      unsigned Idx = *E->idx_begin();
+      assert((Idx == 0 || Idx == 1) && "unexpected index");
+      if (Idx == 0 && isAsynchronousEHPersonality(Personality))
+        SEHCodeUses.push_back(E);
+      else
+        EHUndefs.push_back(E);
+    }
+    for (Instruction *E : EHUndefs) {
+      E->replaceAllUsesWith(UndefValue::get(E->getType()));
+      E->eraseFromParent();
+    }
     LPad->replaceAllUsesWith(UndefValue::get(LPad->getType()));
 
-    // Replace the mapping of any nested landing pad that previously mapped
-    // to this landing pad with a referenced to the cloned version.
-    for (auto &LPadPair : NestedLPtoOriginalLP) {
-      const LandingPadInst *OriginalLPad = LPadPair.second;
-      if (OriginalLPad == LPad) {
-        LPadPair.second = NewLPad;
+    // Rewrite uses of the exception pointer to loads of an alloca.
+    while (!SEHCodeUses.empty()) {
+      Instruction *E = SEHCodeUses.pop_back_val();
+      SmallVector<Use *, 4> Uses;
+      for (Use &U : E->uses())
+        Uses.push_back(&U);
+      for (Use *U : Uses) {
+        auto *I = cast<Instruction>(U->getUser());
+        if (isa<ResumeInst>(I))
+          continue;
+        if (auto *Phi = dyn_cast<PHINode>(I))
+          SEHCodeUses.push_back(Phi);
+        else
+          U->set(new LoadInst(SEHExceptionCodeSlot, "sehcode", false, I));
       }
+      E->replaceAllUsesWith(UndefValue::get(E->getType()));
+      E->eraseFromParent();
     }
 
-    // Replace uses of the old lpad in phis with this block and delete the old
-    // block.
-    LPadBB->replaceSuccessorsPhiUsesWith(NewLPadBB);
-    LPadBB->getTerminator()->eraseFromParent();
-    new UnreachableInst(LPadBB->getContext(), LPadBB);
-
     // Add a call to describe the actions for this landing pad.
     std::vector<Value *> ActionArgs;
     for (ActionHandler *Action : Actions) {
@@ -476,17 +888,26 @@ bool WinEHPrepare::prepareExceptionHandlers(
       ActionArgs.push_back(Action->getHandlerBlockOrFunc());
     }
     CallInst *Recover =
-        CallInst::Create(ActionIntrin, ActionArgs, "recover", NewLPadBB);
+        CallInst::Create(ActionIntrin, ActionArgs, "recover", LPadBB);
 
-    // Add an indirect branch listing possible successors of the catch handlers.
-    IndirectBrInst *Branch = IndirectBrInst::Create(Recover, 0, NewLPadBB);
+    SetVector<BasicBlock *> ReturnTargets;
     for (ActionHandler *Action : Actions) {
       if (auto *CatchAction = dyn_cast<CatchHandler>(Action)) {
-        for (auto *Target : CatchAction->getReturnTargets()) {
-          Branch->addDestination(Target);
-        }
+        const auto &CatchTargets = CatchAction->getReturnTargets();
+        ReturnTargets.insert(CatchTargets.begin(), CatchTargets.end());
       }
     }
+    IndirectBrInst *Branch =
+        IndirectBrInst::Create(Recover, ReturnTargets.size(), LPadBB);
+    for (BasicBlock *Target : ReturnTargets)
+      Branch->addDestination(Target);
+
+    if (!isAsynchronousEHPersonality(Personality)) {
+      // C++ EH must repopulate the targets later to handle the case of
+      // targets that are reached indirectly through nested landing pads.
+      LPadImpls.push_back(std::make_pair(Recover, Branch));
+    }
+
   } // End for each landingpad
 
   // If nothing got outlined, there is no more processing to be done.
@@ -500,6 +921,50 @@ bool WinEHPrepare::prepareExceptionHandlers(
     completeNestedLandingPad(&F, LPadPair.first, LPadPair.second, FrameVarInfo);
   NestedLPtoOriginalLP.clear();
 
+  // Update the indirectbr instructions' target lists if necessary.
+  SetVector<BasicBlock*> CheckedTargets;
+  SmallVector<std::unique_ptr<ActionHandler>, 4> ActionList;
+  for (auto &LPadImplPair : LPadImpls) {
+    IntrinsicInst *Recover = cast<IntrinsicInst>(LPadImplPair.first);
+    IndirectBrInst *Branch = LPadImplPair.second;
+
+    // Get a list of handlers called by 
+    parseEHActions(Recover, ActionList);
+
+    // Add an indirect branch listing possible successors of the catch handlers.
+    SetVector<BasicBlock *> ReturnTargets;
+    for (const auto &Action : ActionList) {
+      if (auto *CA = dyn_cast<CatchHandler>(Action.get())) {
+        Function *Handler = cast<Function>(CA->getHandlerBlockOrFunc());
+        getPossibleReturnTargets(&F, Handler, ReturnTargets);
+      }
+    }
+    ActionList.clear();
+    // Clear any targets we already knew about.
+    for (unsigned int I = 0, E = Branch->getNumDestinations(); I < E; ++I) {
+      BasicBlock *KnownTarget = Branch->getDestination(I);
+      if (ReturnTargets.count(KnownTarget))
+        ReturnTargets.remove(KnownTarget);
+    }
+    for (BasicBlock *Target : ReturnTargets) {
+      Branch->addDestination(Target);
+      // The target may be a block that we excepted to get pruned.
+      // If it is, it may contain a call to llvm.eh.endcatch.
+      if (CheckedTargets.insert(Target)) {
+        // Earlier preparations guarantee that all calls to llvm.eh.endcatch
+        // will be followed by an unconditional branch.
+        auto *Br = dyn_cast<BranchInst>(Target->getTerminator());
+        if (Br && Br->isUnconditional() &&
+            Br != Target->getFirstNonPHIOrDbgOrLifetime()) {
+          Instruction *Prev = Br->getPrevNode();
+          if (match(cast<Value>(Prev), m_Intrinsic<Intrinsic::eh_endcatch>()))
+            Prev->eraseFromParent();
+        }
+      }
+    }
+  }
+  LPadImpls.clear();
+
   F.addFnAttr("wineh-parent", F.getName());
 
   // Delete any blocks that were only used by handlers that were outlined above.
@@ -510,58 +975,29 @@ bool WinEHPrepare::prepareExceptionHandlers(
   Builder.SetInsertPoint(Entry->getFirstInsertionPt());
 
   Function *FrameEscapeFn =
-      Intrinsic::getDeclaration(M, Intrinsic::frameescape);
+      Intrinsic::getDeclaration(M, Intrinsic::localescape);
   Function *RecoverFrameFn =
-      Intrinsic::getDeclaration(M, Intrinsic::framerecover);
+      Intrinsic::getDeclaration(M, Intrinsic::localrecover);
+  SmallVector<Value *, 8> AllocasToEscape;
+
+  // Scan the entry block for an existing call to llvm.localescape. We need to
+  // keep escaping those objects.
+  for (Instruction &I : F.front()) {
+    auto *II = dyn_cast<IntrinsicInst>(&I);
+    if (II && II->getIntrinsicID() == Intrinsic::localescape) {
+      auto Args = II->arg_operands();
+      AllocasToEscape.append(Args.begin(), Args.end());
+      II->eraseFromParent();
+      break;
+    }
+  }
 
   // Finally, replace all of the temporary allocas for frame variables used in
-  // the outlined handlers with calls to llvm.framerecover.
-  BasicBlock::iterator II = Entry->getFirstInsertionPt();
-  Instruction *AllocaInsertPt = II;
-  SmallVector<Value *, 8> AllocasToEscape;
+  // the outlined handlers with calls to llvm.localrecover.
   for (auto &VarInfoEntry : FrameVarInfo) {
     Value *ParentVal = VarInfoEntry.first;
     TinyPtrVector<AllocaInst *> &Allocas = VarInfoEntry.second;
-
-    // If the mapped value isn't already an alloca, we need to spill it if it
-    // is a computed value or copy it if it is an argument.
-    AllocaInst *ParentAlloca = dyn_cast<AllocaInst>(ParentVal);
-    if (!ParentAlloca) {
-      if (auto *Arg = dyn_cast<Argument>(ParentVal)) {
-        // Lower this argument to a copy and then demote that to the stack.
-        // We can't just use the argument location because the handler needs
-        // it to be in the frame allocation block.
-        // Use 'select i8 true, %arg, undef' to simulate a 'no-op' instruction.
-        Value *TrueValue = ConstantInt::getTrue(Context);
-        Value *UndefValue = UndefValue::get(Arg->getType());
-        Instruction *SI =
-            SelectInst::Create(TrueValue, Arg, UndefValue,
-                               Arg->getName() + ".tmp", AllocaInsertPt);
-        Arg->replaceAllUsesWith(SI);
-        // Reset the select operand, because it was clobbered by the RAUW above.
-        SI->setOperand(1, Arg);
-        ParentAlloca = DemoteRegToStack(*SI, true, SI);
-      } else if (auto *PN = dyn_cast<PHINode>(ParentVal)) {
-        ParentAlloca = DemotePHIToStack(PN, AllocaInsertPt);
-      } else {
-        Instruction *ParentInst = cast<Instruction>(ParentVal);
-        // FIXME: This is a work-around to temporarily handle the case where an
-        //        instruction that is only used in handlers is not sunk.
-        //        Without uses, DemoteRegToStack would just eliminate the value.
-        //        This will fail if ParentInst is an invoke.
-        if (ParentInst->getNumUses() == 0) {
-          BasicBlock::iterator InsertPt = ParentInst;
-          ++InsertPt;
-          ParentAlloca =
-              new AllocaInst(ParentInst->getType(), nullptr,
-                             ParentInst->getName() + ".reg2mem",
-                             AllocaInsertPt);
-          new StoreInst(ParentInst, ParentAlloca, InsertPt);
-        } else {
-          ParentAlloca = DemoteRegToStack(*ParentInst, true, AllocaInsertPt);
-        }
-      }
-    }
+    AllocaInst *ParentAlloca = cast<AllocaInst>(ParentVal);
 
     // FIXME: We should try to sink unescaped allocas from the parent frame into
     // the child frame. If the alloca is escaped, we have to use the lifetime
@@ -575,19 +1011,23 @@ bool WinEHPrepare::prepareExceptionHandlers(
       if (TempAlloca == getCatchObjectSentinel())
         continue; // Skip catch parameter sentinels.
       Function *HandlerFn = TempAlloca->getParent()->getParent();
-      // FIXME: Sink this GEP into the blocks where it is used.
+      llvm::Value *FP = HandlerToParentFP[HandlerFn];
+      assert(FP);
+
+      // FIXME: Sink this localrecover into the blocks where it is used.
       Builder.SetInsertPoint(TempAlloca);
       Builder.SetCurrentDebugLocation(TempAlloca->getDebugLoc());
       Value *RecoverArgs[] = {
-          Builder.CreateBitCast(&F, Int8PtrType, ""),
-          &(HandlerFn->getArgumentList().back()),
+          Builder.CreateBitCast(&F, Int8PtrType, ""), FP,
           llvm::ConstantInt::get(Int32Type, AllocasToEscape.size() - 1)};
-      Value *RecoveredAlloca = Builder.CreateCall(RecoverFrameFn, RecoverArgs);
+      Instruction *RecoveredAlloca =
+          Builder.CreateCall(RecoverFrameFn, RecoverArgs);
+
       // Add a pointer bitcast if the alloca wasn't an i8.
       if (RecoveredAlloca->getType() != TempAlloca->getType()) {
         RecoveredAlloca->setName(Twine(TempAlloca->getName()) + ".i8");
-        RecoveredAlloca =
-            Builder.CreateBitCast(RecoveredAlloca, TempAlloca->getType());
+        RecoveredAlloca = cast<Instruction>(
+            Builder.CreateBitCast(RecoveredAlloca, TempAlloca->getType()));
       }
       TempAlloca->replaceAllUsesWith(RecoveredAlloca);
       TempAlloca->removeFromParent();
@@ -596,16 +1036,37 @@ bool WinEHPrepare::prepareExceptionHandlers(
     }
   } // End for each FrameVarInfo entry.
 
-  // Insert 'call void (...)* @llvm.frameescape(...)' at the end of the entry
+  // Insert 'call void (...)* @llvm.localescape(...)' at the end of the entry
   // block.
   Builder.SetInsertPoint(&F.getEntryBlock().back());
   Builder.CreateCall(FrameEscapeFn, AllocasToEscape);
 
+  if (SEHExceptionCodeSlot) {
+    if (isAllocaPromotable(SEHExceptionCodeSlot)) {
+      SmallPtrSet<BasicBlock *, 4> UserBlocks;
+      for (User *U : SEHExceptionCodeSlot->users()) {
+        if (auto *Inst = dyn_cast<Instruction>(U))
+          UserBlocks.insert(Inst->getParent());
+      }
+      PromoteMemToReg(SEHExceptionCodeSlot, *DT);
+      // After the promotion, kill off dead instructions.
+      for (BasicBlock *BB : UserBlocks)
+        SimplifyInstructionsInBlock(BB, LibInfo);
+    }
+  }
+
   // Clean up the handler action maps we created for this function
   DeleteContainerSeconds(CatchHandlerMap);
   CatchHandlerMap.clear();
   DeleteContainerSeconds(CleanupHandlerMap);
   CleanupHandlerMap.clear();
+  HandlerToParentFP.clear();
+  DT = nullptr;
+  LibInfo = nullptr;
+  SEHExceptionCodeSlot = nullptr;
+  EHBlocks.clear();
+  NormalBlocks.clear();
+  EHReturnBlocks.clear();
 
   return HandlersOutlined;
 }
@@ -647,6 +1108,42 @@ void WinEHPrepare::promoteLandingPadValues(LandingPadInst *LPad) {
     RecursivelyDeleteTriviallyDeadInstructions(U);
 }
 
+void WinEHPrepare::getPossibleReturnTargets(Function *ParentF,
+                                            Function *HandlerF,
+                                            SetVector<BasicBlock*> &Targets) {
+  for (BasicBlock &BB : *HandlerF) {
+    // If the handler contains landing pads, check for any
+    // handlers that may return directly to a block in the
+    // parent function.
+    if (auto *LPI = BB.getLandingPadInst()) {
+      IntrinsicInst *Recover = cast<IntrinsicInst>(LPI->getNextNode());
+      SmallVector<std::unique_ptr<ActionHandler>, 4> ActionList;
+      parseEHActions(Recover, ActionList);
+      for (const auto &Action : ActionList) {
+        if (auto *CH = dyn_cast<CatchHandler>(Action.get())) {
+          Function *NestedF = cast<Function>(CH->getHandlerBlockOrFunc());
+          getPossibleReturnTargets(ParentF, NestedF, Targets);
+        }
+      }
+    }
+
+    auto *Ret = dyn_cast<ReturnInst>(BB.getTerminator());
+    if (!Ret)
+      continue;
+
+    // Handler functions must always return a block address.
+    BlockAddress *BA = cast<BlockAddress>(Ret->getReturnValue());
+
+    // If this is the handler for a nested landing pad, the
+    // return address may have been remapped to a block in the
+    // parent handler.  We're not interested in those.
+    if (BA->getFunction() != ParentF)
+      continue;
+
+    Targets.insert(BA->getBasicBlock());
+  }
+}
+
 void WinEHPrepare::completeNestedLandingPad(Function *ParentFn,
                                             LandingPadInst *OutlinedLPad,
                                             const LandingPadInst *OriginalLPad,
@@ -655,10 +1152,19 @@ void WinEHPrepare::completeNestedLandingPad(Function *ParentFn,
   // temporarily inserted as its terminator.
   LLVMContext &Context = ParentFn->getContext();
   BasicBlock *OutlinedBB = OutlinedLPad->getParent();
-  assert(isa<UnreachableInst>(OutlinedBB->getTerminator()));
-  OutlinedBB->getTerminator()->eraseFromParent();
-  // That should leave OutlinedLPad as the last instruction in its block.
-  assert(&OutlinedBB->back() == OutlinedLPad);
+  // If the nested landing pad was outlined before the landing pad that enclosed
+  // it, it will already be in outlined form.  In that case, we just need to see
+  // if the returns and the enclosing branch instruction need to be updated.
+  IndirectBrInst *Branch =
+      dyn_cast<IndirectBrInst>(OutlinedBB->getTerminator());
+  if (!Branch) {
+    // If the landing pad wasn't in outlined form, it should be a stub with
+    // an unreachable terminator.
+    assert(isa<UnreachableInst>(OutlinedBB->getTerminator()));
+    OutlinedBB->getTerminator()->eraseFromParent();
+    // That should leave OutlinedLPad as the last instruction in its block.
+    assert(&OutlinedBB->back() == OutlinedLPad);
+  }
 
   // The original landing pad will have already had its action intrinsic
   // built by the outlining loop.  We need to clone that into the outlined
@@ -671,16 +1177,14 @@ void WinEHPrepare::completeNestedLandingPad(Function *ParentFn,
   ++II;
   // The instruction after the landing pad should now be a call to eh.actions.
   const Instruction *Recover = II;
-  assert(match(Recover, m_Intrinsic<Intrinsic::eh_actions>()));
-  IntrinsicInst *EHActions = cast<IntrinsicInst>(Recover->clone());
+  const IntrinsicInst *EHActions = cast<IntrinsicInst>(Recover);
 
-  // Remap the exception variables into the outlined function.
-  WinEHFrameVariableMaterializer Materializer(OutlinedHandlerFn, FrameVarInfo);
+  // Remap the return target in the nested handler.
   SmallVector<BlockAddress *, 4> ActionTargets;
-  SmallVector<ActionHandler *, 4> ActionList;
+  SmallVector<std::unique_ptr<ActionHandler>, 4> ActionList;
   parseEHActions(EHActions, ActionList);
-  for (auto *Action : ActionList) {
-    auto *Catch = dyn_cast<CatchHandler>(Action);
+  for (const auto &Action : ActionList) {
+    auto *Catch = dyn_cast<CatchHandler>(Action.get());
     if (!Catch)
       continue;
     // The dyn_cast to function here selects C++ catch handlers and skips
@@ -702,7 +1206,7 @@ void WinEHPrepare::completeNestedLandingPad(Function *ParentFn,
       // should be a block that was outlined into OutlinedHandlerFn.
       assert(BA->getFunction() == ParentFn);
 
-      // Ignore targets that aren't part of OutlinedHandlerFn.
+      // Ignore targets that aren't part of an outlined handler function.
       if (!LPadTargetBlocks.count(BA->getBasicBlock()))
         continue;
 
@@ -718,15 +1222,26 @@ void WinEHPrepare::completeNestedLandingPad(Function *ParentFn,
       ActionTargets.push_back(NewBA);
     }
   }
-  DeleteContainerPointers(ActionList);
   ActionList.clear();
-  OutlinedBB->getInstList().push_back(EHActions);
 
-  // Insert an indirect branch into the outlined landing pad BB.
-  IndirectBrInst *IBr = IndirectBrInst::Create(EHActions, 0, OutlinedBB);
-  // Add the previously collected action targets.
-  for (auto *Target : ActionTargets)
-    IBr->addDestination(Target->getBasicBlock());
+  if (Branch) {
+    // If the landing pad was already in outlined form, just update its targets.
+    for (unsigned int I = Branch->getNumDestinations(); I > 0; --I)
+      Branch->removeDestination(I);
+    // Add the previously collected action targets.
+    for (auto *Target : ActionTargets)
+      Branch->addDestination(Target->getBasicBlock());
+  } else {
+    // If the landing pad was previously stubbed out, fill in its outlined form.
+    IntrinsicInst *NewEHActions = cast<IntrinsicInst>(EHActions->clone());
+    OutlinedBB->getInstList().push_back(NewEHActions);
+
+    // Insert an indirect branch into the outlined landing pad BB.
+    IndirectBrInst *IBr = IndirectBrInst::Create(NewEHActions, 0, OutlinedBB);
+    // Add the previously collected action targets.
+    for (auto *Target : ActionTargets)
+      IBr->addDestination(Target->getBasicBlock());
+  }
 }
 
 // This function examines a block to determine whether the block ends with a
@@ -763,8 +1278,16 @@ static bool isSelectorDispatch(BasicBlock *BB, BasicBlock *&CatchHandler,
   return false;
 }
 
-static BasicBlock *createStubLandingPad(Function *Handler,
-                                        Value *PersonalityFn) {
+static bool isCatchBlock(BasicBlock *BB) {
+  for (BasicBlock::iterator II = BB->getFirstNonPHIOrDbg(), IE = BB->end();
+       II != IE; ++II) {
+    if (match(cast<Value>(II), m_Intrinsic<Intrinsic::eh_begincatch>()))
+      return true;
+  }
+  return false;
+}
+
+static BasicBlock *createStubLandingPad(Function *Handler) {
   // FIXME: Finish this!
   LLVMContext &Context = Handler->getContext();
   BasicBlock *StubBB = BasicBlock::Create(Context, "stub");
@@ -773,7 +1296,11 @@ static BasicBlock *createStubLandingPad(Function *Handler,
   LandingPadInst *LPad = Builder.CreateLandingPad(
       llvm::StructType::get(Type::getInt8PtrTy(Context),
                             Type::getInt32Ty(Context), nullptr),
-      PersonalityFn, 0);
+      0);
+  // Insert a call to llvm.eh.actions so that we don't try to outline this lpad.
+  Function *ActionIntrin =
+      Intrinsic::getDeclaration(Handler->getParent(), Intrinsic::eh_actions);
+  Builder.CreateCall(ActionIntrin, {}, "recover");
   LPad->setCleanup(true);
   Builder.CreateUnreachable();
   return StubBB;
@@ -784,9 +1311,9 @@ static BasicBlock *createStubLandingPad(Function *Handler,
 // landing pad if none is found.  The code that generates the .xdata tables for
 // the handler needs at least one landing pad to identify the parent function's
 // personality.
-void WinEHPrepare::addStubInvokeToHandlerIfNeeded(Function *Handler,
-                                                  Value *PersonalityFn) {
+void WinEHPrepare::addStubInvokeToHandlerIfNeeded(Function *Handler) {
   ReturnInst *Ret = nullptr;
+  UnreachableInst *Unreached = nullptr;
   for (BasicBlock &BB : *Handler) {
     TerminatorInst *Terminator = BB.getTerminator();
     // If we find an invoke, there is nothing to be done.
@@ -794,57 +1321,96 @@ void WinEHPrepare::addStubInvokeToHandlerIfNeeded(Function *Handler,
     if (II)
       return;
     // If we've already recorded a return instruction, keep looking for invokes.
-    if (Ret)
-      continue;
-    // If we haven't recorded a return instruction yet, try this terminator.
-    Ret = dyn_cast<ReturnInst>(Terminator);
+    if (!Ret)
+      Ret = dyn_cast<ReturnInst>(Terminator);
+    // If we haven't recorded an unreachable instruction, try this terminator.
+    if (!Unreached)
+      Unreached = dyn_cast<UnreachableInst>(Terminator);
   }
 
   // If we got this far, the handler contains no invokes.  We should have seen
-  // at least one return.  We'll insert an invoke of llvm.donothing ahead of
-  // that return.
-  assert(Ret);
-  BasicBlock *OldRetBB = Ret->getParent();
-  BasicBlock *NewRetBB = SplitBlock(OldRetBB, Ret);
+  // at least one return or unreachable instruction.  We'll insert an invoke of
+  // llvm.donothing ahead of that instruction.
+  assert(Ret || Unreached);
+  TerminatorInst *Term;
+  if (Ret)
+    Term = Ret;
+  else
+    Term = Unreached;
+  BasicBlock *OldRetBB = Term->getParent();
+  BasicBlock *NewRetBB = SplitBlock(OldRetBB, Term, DT);
   // SplitBlock adds an unconditional branch instruction at the end of the
   // parent block.  We want to replace that with an invoke call, so we can
   // erase it now.
   OldRetBB->getTerminator()->eraseFromParent();
-  BasicBlock *StubLandingPad = createStubLandingPad(Handler, PersonalityFn);
+  BasicBlock *StubLandingPad = createStubLandingPad(Handler);
   Function *F =
       Intrinsic::getDeclaration(Handler->getParent(), Intrinsic::donothing);
   InvokeInst::Create(F, NewRetBB, StubLandingPad, None, "", OldRetBB);
 }
 
+// FIXME: Consider sinking this into lib/Target/X86 somehow. TargetLowering
+// usually doesn't build LLVM IR, so that's probably the wrong place.
+Function *WinEHPrepare::createHandlerFunc(Function *ParentFn, Type *RetTy,
+                                          const Twine &Name, Module *M,
+                                          Value *&ParentFP) {
+  // x64 uses a two-argument prototype where the parent FP is the second
+  // argument. x86 uses no arguments, just the incoming EBP value.
+  LLVMContext &Context = M->getContext();
+  Type *Int8PtrType = Type::getInt8PtrTy(Context);
+  FunctionType *FnType;
+  if (TheTriple.getArch() == Triple::x86_64) {
+    Type *ArgTys[2] = {Int8PtrType, Int8PtrType};
+    FnType = FunctionType::get(RetTy, ArgTys, false);
+  } else {
+    FnType = FunctionType::get(RetTy, None, false);
+  }
+
+  Function *Handler =
+      Function::Create(FnType, GlobalVariable::InternalLinkage, Name, M);
+  BasicBlock *Entry = BasicBlock::Create(Context, "entry");
+  Handler->getBasicBlockList().push_front(Entry);
+  if (TheTriple.getArch() == Triple::x86_64) {
+    ParentFP = &(Handler->getArgumentList().back());
+  } else {
+    assert(M);
+    Function *FrameAddressFn =
+        Intrinsic::getDeclaration(M, Intrinsic::frameaddress);
+    Function *RecoverFPFn =
+        Intrinsic::getDeclaration(M, Intrinsic::x86_seh_recoverfp);
+    IRBuilder<> Builder(&Handler->getEntryBlock());
+    Value *EBP =
+        Builder.CreateCall(FrameAddressFn, {Builder.getInt32(1)}, "ebp");
+    Value *ParentI8Fn = Builder.CreateBitCast(ParentFn, Int8PtrType);
+    ParentFP = Builder.CreateCall(RecoverFPFn, {ParentI8Fn, EBP});
+  }
+  return Handler;
+}
+
 bool WinEHPrepare::outlineHandler(ActionHandler *Action, Function *SrcFn,
                                   LandingPadInst *LPad, BasicBlock *StartBB,
                                   FrameVarInfoMap &VarInfo) {
   Module *M = SrcFn->getParent();
   LLVMContext &Context = M->getContext();
+  Type *Int8PtrType = Type::getInt8PtrTy(Context);
 
   // Create a new function to receive the handler contents.
-  Type *Int8PtrType = Type::getInt8PtrTy(Context);
-  std::vector<Type *> ArgTys;
-  ArgTys.push_back(Int8PtrType);
-  ArgTys.push_back(Int8PtrType);
+  Value *ParentFP;
   Function *Handler;
   if (Action->getType() == Catch) {
-    FunctionType *FnType = FunctionType::get(Int8PtrType, ArgTys, false);
-    Handler = Function::Create(FnType, GlobalVariable::InternalLinkage,
-                               SrcFn->getName() + ".catch", M);
+    Handler = createHandlerFunc(SrcFn, Int8PtrType, SrcFn->getName() + ".catch", M,
+                                ParentFP);
   } else {
-    FunctionType *FnType =
-        FunctionType::get(Type::getVoidTy(Context), ArgTys, false);
-    Handler = Function::Create(FnType, GlobalVariable::InternalLinkage,
-                               SrcFn->getName() + ".cleanup", M);
+    Handler = createHandlerFunc(SrcFn, Type::getVoidTy(Context),
+                                SrcFn->getName() + ".cleanup", M, ParentFP);
   }
-
+  Handler->setPersonalityFn(SrcFn->getPersonalityFn());
+  HandlerToParentFP[Handler] = ParentFP;
   Handler->addFnAttr("wineh-parent", SrcFn->getName());
+  BasicBlock *Entry = &Handler->getEntryBlock();
 
   // Generate a standard prolog to setup the frame recovery structure.
   IRBuilder<> Builder(Context);
-  BasicBlock *Entry = BasicBlock::Create(Context, "entry");
-  Handler->getBasicBlockList().push_front(Entry);
   Builder.SetInsertPoint(Entry);
   Builder.SetCurrentDebugLocation(LPad->getDebugLoc());
 
@@ -857,12 +1423,14 @@ bool WinEHPrepare::outlineHandler(ActionHandler *Action, Function *SrcFn,
     LPadMap.mapLandingPad(LPad);
   if (auto *CatchAction = dyn_cast<CatchHandler>(Action)) {
     Constant *Sel = CatchAction->getSelector();
-    Director.reset(new WinEHCatchDirector(Handler, Sel, VarInfo, LPadMap,
-                                          NestedLPtoOriginalLP));
+    Director.reset(new WinEHCatchDirector(Handler, ParentFP, Sel, VarInfo,
+                                          LPadMap, NestedLPtoOriginalLP, DT,
+                                          EHBlocks));
     LPadMap.remapEHValues(VMap, UndefValue::get(Int8PtrType),
                           ConstantInt::get(Type::getInt32Ty(Context), 1));
   } else {
-    Director.reset(new WinEHCleanupDirector(Handler, VarInfo, LPadMap));
+    Director.reset(
+        new WinEHCleanupDirector(Handler, ParentFP, VarInfo, LPadMap));
     LPadMap.remapEHValues(VMap, UndefValue::get(Int8PtrType),
                           UndefValue::get(Type::getInt32Ty(Context)));
   }
@@ -888,6 +1456,10 @@ bool WinEHPrepare::outlineHandler(ActionHandler *Action, Function *SrcFn,
     ++II;
   }
 
+  // The landing pad value may be used by PHI nodes.  It will ultimately be
+  // eliminated, but we need it in the map for intermediate handling.
+  VMap[LPad] = UndefValue::get(LPad->getType());
+
   // Skip over PHIs and, if applicable, landingpad instructions.
   II = StartBB->getFirstInsertionPt();
 
@@ -895,13 +1467,22 @@ bool WinEHPrepare::outlineHandler(ActionHandler *Action, Function *SrcFn,
                             /*ModuleLevelChanges=*/false, Returns, "",
                             &OutlinedFunctionInfo, Director.get());
 
-  // Move all the instructions in the first cloned block into our entry block.
-  BasicBlock *FirstClonedBB = std::next(Function::iterator(Entry));
-  Entry->getInstList().splice(Entry->end(), FirstClonedBB->getInstList());
-  FirstClonedBB->eraseFromParent();
+  // Move all the instructions in the cloned "entry" block into our entry block.
+  // Depending on how the parent function was laid out, the block that will
+  // correspond to the outlined entry block may not be the first block in the
+  // list.  We can recognize it, however, as the cloned block which has no
+  // predecessors.  Any other block wouldn't have been cloned if it didn't
+  // have a predecessor which was also cloned.
+  Function::iterator ClonedIt = std::next(Function::iterator(Entry));
+  while (!pred_empty(ClonedIt))
+    ++ClonedIt;
+  BasicBlock *ClonedEntryBB = ClonedIt;
+  assert(ClonedEntryBB);
+  Entry->getInstList().splice(Entry->end(), ClonedEntryBB->getInstList());
+  ClonedEntryBB->eraseFromParent();
 
   // Make sure we can identify the handler's personality later.
-  addStubInvokeToHandlerIfNeeded(Handler, LPad->getPersonalityFn());
+  addStubInvokeToHandlerIfNeeded(Handler);
 
   if (auto *CatchAction = dyn_cast<CatchHandler>(Action)) {
     WinEHCatchDirector *CatchDirector =
@@ -964,9 +1545,14 @@ void WinEHPrepare::processSEHCatchHandler(CatchHandler *CatchAction,
   } else {
     // This must be a catch-all. Split the block after the landingpad.
     assert(CatchAction->getSelector()->isNullValue() && "expected catch-all");
-    HandlerBB =
-        StartBB->splitBasicBlock(StartBB->getFirstInsertionPt(), "catch.all");
+    HandlerBB = SplitBlock(StartBB, StartBB->getFirstInsertionPt(), DT);
   }
+  IRBuilder<> Builder(HandlerBB->getFirstInsertionPt());
+  Function *EHCodeFn = Intrinsic::getDeclaration(
+      StartBB->getParent()->getParent(), Intrinsic::eh_exceptioncode);
+  Value *Code = Builder.CreateCall(EHCodeFn, {}, "sehcode");
+  Code = Builder.CreateIntToPtr(Code, SEHExceptionCodeSlot->getAllocatedType());
+  Builder.CreateStore(Code, SEHExceptionCodeSlot);
   CatchAction->setHandlerBlockOrFunc(BlockAddress::get(HandlerBB));
   TinyPtrVector<BasicBlock *> Targets(HandlerBB);
   CatchAction->setReturnTargets(Targets);
@@ -1031,6 +1617,10 @@ void LandingPadMap::remapEHValues(ValueToValueMapTy &VMap, Value *EHPtrValue,
     VMap[Extract] = SelectorValue;
 }
 
+static bool isLocalAddressCall(const Value *V) {
+  return match(const_cast<Value *>(V), m_Intrinsic<Intrinsic::localaddress>());
+}
+
 CloningDirector::CloningAction WinEHCloningDirectorBase::handleInstruction(
     ValueToValueMapTy &VMap, const Instruction *Inst, BasicBlock *NewBB) {
   // If this is one of the boilerplate landing pad instructions, skip it.
@@ -1038,21 +1628,31 @@ CloningDirector::CloningAction WinEHCloningDirectorBase::handleInstruction(
   if (LPadMap.isLandingPadSpecificInst(Inst))
     return CloningDirector::SkipInstruction;
 
-  // Nested landing pads will be cloned as stubs, with just the
-  // landingpad instruction and an unreachable instruction. When
-  // all landingpads have been outlined, we'll replace this with the
-  // llvm.eh.actions call and indirect branch created when the
-  // landing pad was outlined.
+  // Nested landing pads that have not already been outlined will be cloned as
+  // stubs, with just the landingpad instruction and an unreachable instruction.
+  // When all landingpads have been outlined, we'll replace this with the
+  // llvm.eh.actions call and indirect branch created when the landing pad was
+  // outlined.
   if (auto *LPad = dyn_cast<LandingPadInst>(Inst)) {
     return handleLandingPad(VMap, LPad, NewBB);
   }
 
+  // Nested landing pads that have already been outlined will be cloned in their
+  // outlined form, but we need to intercept the ibr instruction to filter out
+  // targets that do not return to the handler we are outlining.
+  if (auto *IBr = dyn_cast<IndirectBrInst>(Inst)) {
+    return handleIndirectBr(VMap, IBr, NewBB);
+  }
+
   if (auto *Invoke = dyn_cast<InvokeInst>(Inst))
     return handleInvoke(VMap, Invoke, NewBB);
 
   if (auto *Resume = dyn_cast<ResumeInst>(Inst))
     return handleResume(VMap, Resume, NewBB);
 
+  if (auto *Cmp = dyn_cast<CmpInst>(Inst))
+    return handleCompare(VMap, Cmp, NewBB);
+
   if (match(Inst, m_Intrinsic<Intrinsic::eh_begincatch>()))
     return handleBeginCatch(VMap, Inst, NewBB);
   if (match(Inst, m_Intrinsic<Intrinsic::eh_endcatch>()))
@@ -1060,12 +1660,33 @@ CloningDirector::CloningAction WinEHCloningDirectorBase::handleInstruction(
   if (match(Inst, m_Intrinsic<Intrinsic::eh_typeid_for>()))
     return handleTypeIdFor(VMap, Inst, NewBB);
 
+  // When outlining llvm.localaddress(), remap that to the second argument,
+  // which is the FP of the parent.
+  if (isLocalAddressCall(Inst)) {
+    VMap[Inst] = ParentFP;
+    return CloningDirector::SkipInstruction;
+  }
+
   // Continue with the default cloning behavior.
   return CloningDirector::CloneInstruction;
 }
 
 CloningDirector::CloningAction WinEHCatchDirector::handleLandingPad(
     ValueToValueMapTy &VMap, const LandingPadInst *LPad, BasicBlock *NewBB) {
+  // If the instruction after the landing pad is a call to llvm.eh.actions
+  // the landing pad has already been outlined.  In this case, we should
+  // clone it because it may return to a block in the handler we are
+  // outlining now that would otherwise be unreachable.  The landing pads
+  // are sorted before outlining begins to enable this case to work
+  // properly.
+  const Instruction *NextI = LPad->getNextNode();
+  if (match(NextI, m_Intrinsic<Intrinsic::eh_actions>()))
+    return CloningDirector::CloneInstruction;
+
+  // If the landing pad hasn't been outlined yet, the landing pad we are
+  // outlining now does not dominate it and so it cannot return to a block
+  // in this handler.  In that case, we can just insert a stub landing
+  // pad now and patch it up later.
   Instruction *NewInst = LPad->clone();
   if (LPad->hasName())
     NewInst->setName(LPad->getName());
@@ -1157,6 +1778,48 @@ CloningDirector::CloningAction WinEHCatchDirector::handleTypeIdFor(
   return CloningDirector::SkipInstruction;
 }
 
+CloningDirector::CloningAction WinEHCatchDirector::handleIndirectBr(
+    ValueToValueMapTy &VMap,
+    const IndirectBrInst *IBr,
+    BasicBlock *NewBB) {
+  // If this indirect branch is not part of a landing pad block, just clone it.
+  const BasicBlock *ParentBB = IBr->getParent();
+  if (!ParentBB->isLandingPad())
+    return CloningDirector::CloneInstruction;
+
+  // If it is part of a landing pad, we want to filter out target blocks
+  // that are not part of the handler we are outlining.
+  const LandingPadInst *LPad = ParentBB->getLandingPadInst();
+
+  // Save this correlation for later processing.
+  NestedLPtoOriginalLP[cast<LandingPadInst>(VMap[LPad])] = LPad;
+
+  // We should only get here for landing pads that have already been outlined.
+  assert(match(LPad->getNextNode(), m_Intrinsic<Intrinsic::eh_actions>()));
+
+  // Copy the indirectbr, but only include targets that were previously
+  // identified as EH blocks and are dominated by the nested landing pad.
+  SetVector<const BasicBlock *> ReturnTargets;
+  for (int I = 0, E = IBr->getNumDestinations(); I < E; ++I) {
+    auto *TargetBB = IBr->getDestination(I);
+    if (EHBlocks.count(const_cast<BasicBlock*>(TargetBB)) &&
+        DT->dominates(ParentBB, TargetBB)) {
+      DEBUG(dbgs() << "  Adding destination " << TargetBB->getName() << "\n");
+      ReturnTargets.insert(TargetBB);
+    }
+  }
+  IndirectBrInst *NewBranch = 
+        IndirectBrInst::Create(const_cast<Value *>(IBr->getAddress()),
+                               ReturnTargets.size(), NewBB);
+  for (auto *Target : ReturnTargets)
+    NewBranch->addDestination(const_cast<BasicBlock*>(Target));
+
+  // The operands and targets of the branch instruction are remapped later
+  // because it is a terminator.  Tell the cloning code to clone the
+  // blocks we just added to the target list.
+  return CloningDirector::CloneSuccessors;
+}
+
 CloningDirector::CloningAction
 WinEHCatchDirector::handleInvoke(ValueToValueMapTy &VMap,
                                  const InvokeInst *Invoke, BasicBlock *NewBB) {
@@ -1173,6 +1836,30 @@ WinEHCatchDirector::handleResume(ValueToValueMapTy &VMap,
   return CloningDirector::StopCloningBB;
 }
 
+CloningDirector::CloningAction
+WinEHCatchDirector::handleCompare(ValueToValueMapTy &VMap,
+                                  const CmpInst *Compare, BasicBlock *NewBB) {
+  const IntrinsicInst *IntrinCall = nullptr;
+  if (match(Compare->getOperand(0), m_Intrinsic<Intrinsic::eh_typeid_for>())) {
+    IntrinCall = dyn_cast<IntrinsicInst>(Compare->getOperand(0));
+  } else if (match(Compare->getOperand(1),
+                   m_Intrinsic<Intrinsic::eh_typeid_for>())) {
+    IntrinCall = dyn_cast<IntrinsicInst>(Compare->getOperand(1));
+  }
+  if (IntrinCall) {
+    Value *Selector = IntrinCall->getArgOperand(0)->stripPointerCasts();
+    // This causes a replacement that will collapse the landing pad CFG based
+    // on the filter function we intend to match.
+    if (Selector == CurrentSelector->stripPointerCasts()) {
+      VMap[Compare] = ConstantInt::get(SelectorIDType, 1);
+    } else {
+      VMap[Compare] = ConstantInt::get(SelectorIDType, 0);
+    }
+    return CloningDirector::SkipInstruction;
+  }
+  return CloningDirector::CloneInstruction;
+}
+
 CloningDirector::CloningAction WinEHCleanupDirector::handleLandingPad(
     ValueToValueMapTy &VMap, const LandingPadInst *LPad, BasicBlock *NewBB) {
   // The MS runtime will terminate the process if an exception occurs in a
@@ -1189,11 +1876,11 @@ CloningDirector::CloningAction WinEHCleanupDirector::handleLandingPad(
 
 CloningDirector::CloningAction WinEHCleanupDirector::handleBeginCatch(
     ValueToValueMapTy &VMap, const Instruction *Inst, BasicBlock *NewBB) {
-  // Catch blocks within cleanup handlers will always be unreachable.
-  // We'll insert an unreachable instruction now, but it will be pruned
-  // before the cloning process is complete.
-  BasicBlock::InstListType &InstList = NewBB->getInstList();
-  InstList.push_back(new UnreachableInst(NewBB->getContext()));
+  // Cleanup code may flow into catch blocks or the catch block may be part
+  // of a branch that will be optimized away.  We'll insert a return
+  // instruction now, but it may be pruned before the cloning process is
+  // complete.
+  ReturnInst::Create(NewBB->getContext(), nullptr, NewBB);
   return CloningDirector::StopCloningBB;
 }
 
@@ -1222,6 +1909,14 @@ CloningDirector::CloningAction WinEHCleanupDirector::handleTypeIdFor(
   return CloningDirector::SkipInstruction;
 }
 
+CloningDirector::CloningAction WinEHCleanupDirector::handleIndirectBr(
+    ValueToValueMapTy &VMap,
+    const IndirectBrInst *IBr,
+    BasicBlock *NewBB) {
+  // No special handling is required for cleanup cloning.
+  return CloningDirector::CloneInstruction;
+}
+
 CloningDirector::CloningAction WinEHCleanupDirector::handleInvoke(
     ValueToValueMapTy &VMap, const InvokeInst *Invoke, BasicBlock *NewBB) {
   // All invokes in cleanup handlers can be replaced with calls.
@@ -1259,39 +1954,53 @@ CloningDirector::CloningAction WinEHCleanupDirector::handleResume(
   return CloningDirector::StopCloningBB;
 }
 
+CloningDirector::CloningAction
+WinEHCleanupDirector::handleCompare(ValueToValueMapTy &VMap,
+                                    const CmpInst *Compare, BasicBlock *NewBB) {
+  if (match(Compare->getOperand(0), m_Intrinsic<Intrinsic::eh_typeid_for>()) ||
+      match(Compare->getOperand(1), m_Intrinsic<Intrinsic::eh_typeid_for>())) {
+    VMap[Compare] = ConstantInt::get(SelectorIDType, 1);
+    return CloningDirector::SkipInstruction;
+  }
+  return CloningDirector::CloneInstruction;
+}
+
 WinEHFrameVariableMaterializer::WinEHFrameVariableMaterializer(
-    Function *OutlinedFn, FrameVarInfoMap &FrameVarInfo)
+    Function *OutlinedFn, Value *ParentFP, FrameVarInfoMap &FrameVarInfo)
     : FrameVarInfo(FrameVarInfo), Builder(OutlinedFn->getContext()) {
   BasicBlock *EntryBB = &OutlinedFn->getEntryBlock();
-  Builder.SetInsertPoint(EntryBB, EntryBB->getFirstInsertionPt());
+
+  // New allocas should be inserted in the entry block, but after the parent FP
+  // is established if it is an instruction.
+  Instruction *InsertPoint = EntryBB->getFirstInsertionPt();
+  if (auto *FPInst = dyn_cast<Instruction>(ParentFP))
+    InsertPoint = FPInst->getNextNode();
+  Builder.SetInsertPoint(EntryBB, InsertPoint);
 }
 
 Value *WinEHFrameVariableMaterializer::materializeValueFor(Value *V) {
-  // If we're asked to materialize a value that is an instruction, we
-  // temporarily create an alloca in the outlined function and add this
-  // to the FrameVarInfo map.  When all the outlining is complete, we'll
-  // collect these into a structure, spilling non-alloca values in the
-  // parent frame as necessary, and replace these temporary allocas with
-  // GEPs referencing the frame allocation block.
-
-  // If the value is an alloca, the mapping is direct.
+  // If we're asked to materialize a static alloca, we temporarily create an
+  // alloca in the outlined function and add this to the FrameVarInfo map.  When
+  // all the outlining is complete, we'll replace these temporary allocas with
+  // calls to llvm.localrecover.
   if (auto *AV = dyn_cast<AllocaInst>(V)) {
+    assert(AV->isStaticAlloca() &&
+           "cannot materialize un-demoted dynamic alloca");
     AllocaInst *NewAlloca = dyn_cast<AllocaInst>(AV->clone());
     Builder.Insert(NewAlloca, AV->getName());
     FrameVarInfo[AV].push_back(NewAlloca);
     return NewAlloca;
   }
 
-  // For other types of instructions or arguments, we need an alloca based on
-  // the value's type and a load of the alloca.  The alloca will be replaced
-  // by a GEP, but the load will stay.  In the parent function, the value will
-  // be spilled to a location in the frame allocation block.
   if (isa<Instruction>(V) || isa<Argument>(V)) {
-    AllocaInst *NewAlloca =
-        Builder.CreateAlloca(V->getType(), nullptr, "eh.temp.alloca");
-    FrameVarInfo[V].push_back(NewAlloca);
-    LoadInst *NewLoad = Builder.CreateLoad(NewAlloca, V->getName() + ".reload");
-    return NewLoad;
+    Function *Parent = isa<Instruction>(V)
+                           ? cast<Instruction>(V)->getParent()->getParent()
+                           : cast<Argument>(V)->getParent();
+    errs()
+        << "Failed to demote instruction used in exception handler of function "
+        << GlobalValue::getRealLinkageName(Parent->getName()) << ":\n";
+    errs() << "  " << *V << '\n';
+    report_fatal_error("WinEHPrepare failed to demote instruction");
   }
 
   // Don't materialize other values.
@@ -1303,7 +2012,7 @@ void WinEHFrameVariableMaterializer::escapeCatchObject(Value *V) {
   // of a catch parameter, add a sentinel to the multimap to indicate that it's
   // used from another handler. This will prevent us from trying to sink the
   // alloca into the handler and ensure that the catch parameter is present in
-  // the call to llvm.frameescape.
+  // the call to llvm.localescape.
   FrameVarInfo[V].push_back(getCatchObjectSentinel());
 }
 
@@ -1353,24 +2062,61 @@ void WinEHPrepare::mapLandingPadBlocks(LandingPadInst *LPad,
   while (HandlersFound != NumClauses) {
     BasicBlock *NextBB = nullptr;
 
+    // Skip over filter clauses.
+    if (LPad->isFilter(HandlersFound)) {
+      ++HandlersFound;
+      continue;
+    }
+
     // See if the clause we're looking for is a catch-all.
     // If so, the catch begins immediately.
-    if (isa<ConstantPointerNull>(LPad->getClause(HandlersFound))) {
+    Constant *ExpectedSelector =
+        LPad->getClause(HandlersFound)->stripPointerCasts();
+    if (isa<ConstantPointerNull>(ExpectedSelector)) {
       // The catch all must occur last.
       assert(HandlersFound == NumClauses - 1);
 
-      // For C++ EH, check if there is any interesting cleanup code before we
-      // begin the catch. This is important because cleanups cannot rethrow
-      // exceptions but code called from catches can. For SEH, it isn't
-      // important if some finally code before a catch-all is executed out of
-      // line or after recovering from the exception.
-      if (Personality == EHPersonality::MSVC_CXX)
-        findCleanupHandlers(Actions, BB, BB);
+      // There can be additional selector dispatches in the call chain that we
+      // need to ignore.
+      BasicBlock *CatchBlock = nullptr;
+      Constant *Selector;
+      while (BB && isSelectorDispatch(BB, CatchBlock, Selector, NextBB)) {
+        DEBUG(dbgs() << "  Found extra catch dispatch in block "
+                     << CatchBlock->getName() << "\n");
+        BB = NextBB;
+      }
 
       // Add the catch handler to the action list.
-      CatchHandler *Action =
-          new CatchHandler(BB, LPad->getClause(HandlersFound), nullptr);
-      CatchHandlerMap[BB] = Action;
+      CatchHandler *Action = nullptr;
+      if (CatchHandlerMap.count(BB) && CatchHandlerMap[BB] != nullptr) {
+        // If the CatchHandlerMap already has an entry for this BB, re-use it.
+        Action = CatchHandlerMap[BB];
+        assert(Action->getSelector() == ExpectedSelector);
+      } else {
+        // We don't expect a selector dispatch, but there may be a call to
+        // llvm.eh.begincatch, which separates catch handling code from
+        // cleanup code in the same control flow.  This call looks for the
+        // begincatch intrinsic.
+        Action = findCatchHandler(BB, NextBB, VisitedBlocks);
+        if (Action) {
+          // For C++ EH, check if there is any interesting cleanup code before
+          // we begin the catch. This is important because cleanups cannot
+          // rethrow exceptions but code called from catches can. For SEH, it
+          // isn't important if some finally code before a catch-all is executed
+          // out of line or after recovering from the exception.
+          if (Personality == EHPersonality::MSVC_CXX)
+            findCleanupHandlers(Actions, BB, BB);
+        } else {
+          // If an action was not found, it means that the control flows
+          // directly into the catch-all handler and there is no cleanup code.
+          // That's an expected situation and we must create a catch action.
+          // Since this is a catch-all handler, the selector won't actually
+          // appear in the code anywhere.  ExpectedSelector here is the constant
+          // null ptr that we got from the landing pad instruction.
+          Action = new CatchHandler(BB, ExpectedSelector, nullptr);
+          CatchHandlerMap[BB] = Action;
+        }
+      }
       Actions.insertCatchHandler(Action);
       DEBUG(dbgs() << "  Catch all handler at block " << BB->getName() << "\n");
       ++HandlersFound;
@@ -1381,16 +2127,40 @@ void WinEHPrepare::mapLandingPadBlocks(LandingPadInst *LPad,
     }
 
     CatchHandler *CatchAction = findCatchHandler(BB, NextBB, VisitedBlocks);
+    assert(CatchAction);
+
     // See if there is any interesting code executed before the dispatch.
     findCleanupHandlers(Actions, BB, CatchAction->getStartBlock());
 
-    assert(CatchAction);
-    ++HandlersFound;
+    // When the source program contains multiple nested try blocks the catch
+    // handlers can get strung together in such a way that we can encounter
+    // a dispatch for a selector that we've already had a handler for.
+    if (CatchAction->getSelector()->stripPointerCasts() == ExpectedSelector) {
+      ++HandlersFound;
 
-    // Add the catch handler to the action list.
-    Actions.insertCatchHandler(CatchAction);
-    DEBUG(dbgs() << "  Found catch dispatch in block "
-                 << CatchAction->getStartBlock()->getName() << "\n");
+      // Add the catch handler to the action list.
+      DEBUG(dbgs() << "  Found catch dispatch in block "
+                   << CatchAction->getStartBlock()->getName() << "\n");
+      Actions.insertCatchHandler(CatchAction);
+    } else {
+      // Under some circumstances optimized IR will flow unconditionally into a
+      // handler block without checking the selector.  This can only happen if
+      // the landing pad has a catch-all handler and the handler for the
+      // preceeding catch clause is identical to the catch-call handler
+      // (typically an empty catch).  In this case, the handler must be shared
+      // by all remaining clauses.
+      if (isa<ConstantPointerNull>(
+              CatchAction->getSelector()->stripPointerCasts())) {
+        DEBUG(dbgs() << "  Applying early catch-all handler in block "
+                     << CatchAction->getStartBlock()->getName()
+                     << "  to all remaining clauses.\n");
+        Actions.insertCatchHandler(CatchAction);
+        return;
+      }
+
+      DEBUG(dbgs() << "  Found extra catch dispatch in block "
+                   << CatchAction->getStartBlock()->getName() << "\n");
+    }
 
     // Move on to the block after the catch handler.
     BB = NextBB;
@@ -1442,6 +2212,18 @@ CatchHandler *WinEHPrepare::findCatchHandler(BasicBlock *BB,
       CatchHandlerMap[BB] = Action;
       return Action;
     }
+    // If we encounter a block containing an llvm.eh.begincatch before we
+    // find a selector dispatch block, the handler is assumed to be
+    // reached unconditionally.  This happens for catch-all blocks, but
+    // it can also happen for other catch handlers that have been combined
+    // with the catch-all handler during optimization.
+    if (isCatchBlock(BB)) {
+      PointerType *Int8PtrTy = Type::getInt8PtrTy(BB->getContext());
+      Constant *NullSelector = ConstantPointerNull::get(Int8PtrTy);
+      CatchHandler *Action = new CatchHandler(BB, NullSelector, nullptr);
+      CatchHandlerMap[BB] = Action;
+      return Action;
+    }
   }
 
   // Visit each successor, looking for the dispatch.
@@ -1469,23 +2251,19 @@ static void createCleanupHandler(LandingPadActions &Actions,
                << Action->getStartBlock()->getName() << "\n");
 }
 
-static bool isFrameAddressCall(Value *V) {
-  return match(V, m_Intrinsic<Intrinsic::frameaddress>(m_SpecificInt(0)));
-}
-
 static CallSite matchOutlinedFinallyCall(BasicBlock *BB,
                                          Instruction *MaybeCall) {
   // Look for finally blocks that Clang has already outlined for us.
-  //   %fp = call i8* @llvm.frameaddress(i32 0)
+  //   %fp = call i8* @llvm.localaddress()
   //   call void @"fin$parent"(iN 1, i8* %fp)
-  if (isFrameAddressCall(MaybeCall) && MaybeCall != BB->getTerminator())
+  if (isLocalAddressCall(MaybeCall) && MaybeCall != BB->getTerminator())
     MaybeCall = MaybeCall->getNextNode();
   CallSite FinallyCall(MaybeCall);
   if (!FinallyCall || FinallyCall.arg_size() != 2)
     return CallSite();
   if (!match(FinallyCall.getArgument(0), m_SpecificInt(1)))
     return CallSite();
-  if (!isFrameAddressCall(FinallyCall.getArgument(1)))
+  if (!isLocalAddressCall(FinallyCall.getArgument(1)))
     return CallSite();
   return FinallyCall;
 }
@@ -1535,7 +2313,7 @@ void WinEHPrepare::findCleanupHandlers(LandingPadActions &Actions,
       if (auto *Action = CleanupHandlerMap[BB]) {
         Actions.insertCleanupHandler(Action);
         DEBUG(dbgs() << "  Found cleanup code in block "
-              << Action->getStartBlock()->getName() << "\n");
+                     << Action->getStartBlock()->getName() << "\n");
         // FIXME: This cleanup might chain into another, and we need to discover
         // that.
         return;
@@ -1544,7 +2322,7 @@ void WinEHPrepare::findCleanupHandlers(LandingPadActions &Actions,
         // value for this block but the value is a nullptr.  This means that
         // we have previously analyzed the block and determined that it did
         // not contain any cleanup code.  Based on the earlier analysis, we
-        // know the the block must end in either an unconditional branch, a
+        // know the block must end in either an unconditional branch, a
         // resume or a conditional branch that is predicated on a comparison
         // with a selector.  Either the resume or the selector dispatch
         // would terminate the search for cleanup code, so the unconditional
@@ -1643,36 +2421,43 @@ void WinEHPrepare::findCleanupHandlers(LandingPadActions &Actions,
           MaybeCall = MaybeCall->getNextNode();
       }
 
-      // Look for outlined finally calls.
-      if (CallSite FinallyCall = matchOutlinedFinallyCall(BB, MaybeCall)) {
-        Function *Fin = FinallyCall.getCalledFunction();
-        assert(Fin && "outlined finally call should be direct");
-        auto *Action = new CleanupHandler(BB);
-        Action->setHandlerBlockOrFunc(Fin);
-        Actions.insertCleanupHandler(Action);
-        CleanupHandlerMap[BB] = Action;
-        DEBUG(dbgs() << "  Found frontend-outlined finally call to "
-                     << Fin->getName() << " in block "
-                     << Action->getStartBlock()->getName() << "\n");
-
-        // Split the block if there were more interesting instructions and look
-        // for finally calls in the normal successor block.
-        BasicBlock *SuccBB = BB;
-        if (FinallyCall.getInstruction() != BB->getTerminator() &&
-            FinallyCall.getInstruction()->getNextNode() != BB->getTerminator()) {
-          SuccBB = BB->splitBasicBlock(FinallyCall.getInstruction()->getNextNode());
-        } else {
-          if (FinallyCall.isInvoke()) {
-            SuccBB = cast<InvokeInst>(FinallyCall.getInstruction())->getNormalDest();
+      // Look for outlined finally calls on x64, since those happen to match the
+      // prototype provided by the runtime.
+      if (TheTriple.getArch() == Triple::x86_64) {
+        if (CallSite FinallyCall = matchOutlinedFinallyCall(BB, MaybeCall)) {
+          Function *Fin = FinallyCall.getCalledFunction();
+          assert(Fin && "outlined finally call should be direct");
+          auto *Action = new CleanupHandler(BB);
+          Action->setHandlerBlockOrFunc(Fin);
+          Actions.insertCleanupHandler(Action);
+          CleanupHandlerMap[BB] = Action;
+          DEBUG(dbgs() << "  Found frontend-outlined finally call to "
+                       << Fin->getName() << " in block "
+                       << Action->getStartBlock()->getName() << "\n");
+
+          // Split the block if there were more interesting instructions and
+          // look for finally calls in the normal successor block.
+          BasicBlock *SuccBB = BB;
+          if (FinallyCall.getInstruction() != BB->getTerminator() &&
+              FinallyCall.getInstruction()->getNextNode() !=
+                  BB->getTerminator()) {
+            SuccBB =
+                SplitBlock(BB, FinallyCall.getInstruction()->getNextNode(), DT);
           } else {
-            SuccBB = BB->getUniqueSuccessor();
-            assert(SuccBB && "splitOutlinedFinallyCalls didn't insert a branch");
+            if (FinallyCall.isInvoke()) {
+              SuccBB = cast<InvokeInst>(FinallyCall.getInstruction())
+                           ->getNormalDest();
+            } else {
+              SuccBB = BB->getUniqueSuccessor();
+              assert(SuccBB &&
+                     "splitOutlinedFinallyCalls didn't insert a branch");
+            }
           }
+          BB = SuccBB;
+          if (BB == EndBB)
+            return;
+          continue;
         }
-        BB = SuccBB;
-        if (BB == EndBB)
-          return;
-        continue;
       }
     }
 
@@ -1705,8 +2490,11 @@ void WinEHPrepare::findCleanupHandlers(LandingPadActions &Actions,
 
 // This is a public function, declared in WinEHFuncInfo.h and is also
 // referenced by WinEHNumbering in FunctionLoweringInfo.cpp.
-void llvm::parseEHActions(const IntrinsicInst *II,
-                          SmallVectorImpl<ActionHandler *> &Actions) {
+void llvm::parseEHActions(
+    const IntrinsicInst *II,
+    SmallVectorImpl<std::unique_ptr<ActionHandler>> &Actions) {
+  assert(II->getIntrinsicID() == Intrinsic::eh_actions &&
+         "attempted to parse non eh.actions intrinsic");
   for (unsigned I = 0, E = II->getNumArgOperands(); I != E;) {
     uint64_t ActionKind =
         cast<ConstantInt>(II->getArgOperand(I))->getZExtValue();
@@ -1716,19 +2504,393 @@ void llvm::parseEHActions(const IntrinsicInst *II,
       int64_t EHObjIndexVal = EHObjIndex->getSExtValue();
       Constant *Handler = cast<Constant>(II->getArgOperand(I + 3));
       I += 4;
-      auto *CH = new CatchHandler(/*BB=*/nullptr, Selector, /*NextBB=*/nullptr);
+      auto CH = make_unique<CatchHandler>(/*BB=*/nullptr, Selector,
+                                          /*NextBB=*/nullptr);
       CH->setHandlerBlockOrFunc(Handler);
       CH->setExceptionVarIndex(EHObjIndexVal);
-      Actions.push_back(CH);
+      Actions.push_back(std::move(CH));
     } else if (ActionKind == 0) {
       Constant *Handler = cast<Constant>(II->getArgOperand(I + 1));
       I += 2;
-      auto *CH = new CleanupHandler(/*BB=*/nullptr);
+      auto CH = make_unique<CleanupHandler>(/*BB=*/nullptr);
       CH->setHandlerBlockOrFunc(Handler);
-      Actions.push_back(CH);
+      Actions.push_back(std::move(CH));
     } else {
       llvm_unreachable("Expected either a catch or cleanup handler!");
     }
   }
   std::reverse(Actions.begin(), Actions.end());
 }
+
+namespace {
+struct WinEHNumbering {
+  WinEHNumbering(WinEHFuncInfo &FuncInfo) : FuncInfo(FuncInfo),
+      CurrentBaseState(-1), NextState(0) {}
+
+  WinEHFuncInfo &FuncInfo;
+  int CurrentBaseState;
+  int NextState;
+
+  SmallVector<std::unique_ptr<ActionHandler>, 4> HandlerStack;
+  SmallPtrSet<const Function *, 4> VisitedHandlers;
+
+  int currentEHNumber() const {
+    return HandlerStack.empty() ? CurrentBaseState : HandlerStack.back()->getEHState();
+  }
+
+  void createUnwindMapEntry(int ToState, ActionHandler *AH);
+  void createTryBlockMapEntry(int TryLow, int TryHigh,
+                              ArrayRef<CatchHandler *> Handlers);
+  void processCallSite(MutableArrayRef<std::unique_ptr<ActionHandler>> Actions,
+                       ImmutableCallSite CS);
+  void popUnmatchedActions(int FirstMismatch);
+  void calculateStateNumbers(const Function &F);
+  void findActionRootLPads(const Function &F);
+};
+}
+
+void WinEHNumbering::createUnwindMapEntry(int ToState, ActionHandler *AH) {
+  WinEHUnwindMapEntry UME;
+  UME.ToState = ToState;
+  if (auto *CH = dyn_cast_or_null<CleanupHandler>(AH))
+    UME.Cleanup = cast<Function>(CH->getHandlerBlockOrFunc());
+  else
+    UME.Cleanup = nullptr;
+  FuncInfo.UnwindMap.push_back(UME);
+}
+
+void WinEHNumbering::createTryBlockMapEntry(int TryLow, int TryHigh,
+                                            ArrayRef<CatchHandler *> Handlers) {
+  // See if we already have an entry for this set of handlers.
+  // This is using iterators rather than a range-based for loop because
+  // if we find the entry we're looking for we'll need the iterator to erase it.
+  int NumHandlers = Handlers.size();
+  auto I = FuncInfo.TryBlockMap.begin();
+  auto E = FuncInfo.TryBlockMap.end();
+  for ( ; I != E; ++I) {
+    auto &Entry = *I;
+    if (Entry.HandlerArray.size() != (size_t)NumHandlers)
+      continue;
+    int N;
+    for (N = 0; N < NumHandlers; ++N) {
+      if (Entry.HandlerArray[N].Handler != Handlers[N]->getHandlerBlockOrFunc())
+        break; // breaks out of inner loop
+    }
+    // If all the handlers match, this is what we were looking for.
+    if (N == NumHandlers) {
+      break;
+    }
+  }
+
+  // If we found an existing entry for this set of handlers, extend the range
+  // but move the entry to the end of the map vector.  The order of entries
+  // in the map is critical to the way that the runtime finds handlers.
+  // FIXME: Depending on what has happened with block ordering, this may
+  //        incorrectly combine entries that should remain separate.
+  if (I != E) {
+    // Copy the existing entry.
+    WinEHTryBlockMapEntry Entry = *I;
+    Entry.TryLow = std::min(TryLow, Entry.TryLow);
+    Entry.TryHigh = std::max(TryHigh, Entry.TryHigh);
+    assert(Entry.TryLow <= Entry.TryHigh);
+    // Erase the old entry and add this one to the back.
+    FuncInfo.TryBlockMap.erase(I);
+    FuncInfo.TryBlockMap.push_back(Entry);
+    return;
+  }
+
+  // If we didn't find an entry, create a new one.
+  WinEHTryBlockMapEntry TBME;
+  TBME.TryLow = TryLow;
+  TBME.TryHigh = TryHigh;
+  assert(TBME.TryLow <= TBME.TryHigh);
+  for (CatchHandler *CH : Handlers) {
+    WinEHHandlerType HT;
+    if (CH->getSelector()->isNullValue()) {
+      HT.Adjectives = 0x40;
+      HT.TypeDescriptor = nullptr;
+    } else {
+      auto *GV = cast<GlobalVariable>(CH->getSelector()->stripPointerCasts());
+      // Selectors are always pointers to GlobalVariables with 'struct' type.
+      // The struct has two fields, adjectives and a type descriptor.
+      auto *CS = cast<ConstantStruct>(GV->getInitializer());
+      HT.Adjectives =
+          cast<ConstantInt>(CS->getAggregateElement(0U))->getZExtValue();
+      HT.TypeDescriptor =
+          cast<GlobalVariable>(CS->getAggregateElement(1)->stripPointerCasts());
+    }
+    HT.Handler = cast<Function>(CH->getHandlerBlockOrFunc());
+    HT.CatchObjRecoverIdx = CH->getExceptionVarIndex();
+    TBME.HandlerArray.push_back(HT);
+  }
+  FuncInfo.TryBlockMap.push_back(TBME);
+}
+
+static void print_name(const Value *V) {
+#ifndef NDEBUG
+  if (!V) {
+    DEBUG(dbgs() << "null");
+    return;
+  }
+
+  if (const auto *F = dyn_cast<Function>(V))
+    DEBUG(dbgs() << F->getName());
+  else
+    DEBUG(V->dump());
+#endif
+}
+
+void WinEHNumbering::processCallSite(
+    MutableArrayRef<std::unique_ptr<ActionHandler>> Actions,
+    ImmutableCallSite CS) {
+  DEBUG(dbgs() << "processCallSite (EH state = " << currentEHNumber()
+               << ") for: ");
+  print_name(CS ? CS.getCalledValue() : nullptr);
+  DEBUG(dbgs() << '\n');
+
+  DEBUG(dbgs() << "HandlerStack: \n");
+  for (int I = 0, E = HandlerStack.size(); I < E; ++I) {
+    DEBUG(dbgs() << "  ");
+    print_name(HandlerStack[I]->getHandlerBlockOrFunc());
+    DEBUG(dbgs() << '\n');
+  }
+  DEBUG(dbgs() << "Actions: \n");
+  for (int I = 0, E = Actions.size(); I < E; ++I) {
+    DEBUG(dbgs() << "  ");
+    print_name(Actions[I]->getHandlerBlockOrFunc());
+    DEBUG(dbgs() << '\n');
+  }
+  int FirstMismatch = 0;
+  for (int E = std::min(HandlerStack.size(), Actions.size()); FirstMismatch < E;
+       ++FirstMismatch) {
+    if (HandlerStack[FirstMismatch]->getHandlerBlockOrFunc() !=
+        Actions[FirstMismatch]->getHandlerBlockOrFunc())
+      break;
+  }
+
+  // Remove unmatched actions from the stack and process their EH states.
+  popUnmatchedActions(FirstMismatch);
+
+  DEBUG(dbgs() << "Pushing actions for CallSite: ");
+  print_name(CS ? CS.getCalledValue() : nullptr);
+  DEBUG(dbgs() << '\n');
+
+  bool LastActionWasCatch = false;
+  const LandingPadInst *LastRootLPad = nullptr;
+  for (size_t I = FirstMismatch; I != Actions.size(); ++I) {
+    // We can reuse eh states when pushing two catches for the same invoke.
+    bool CurrActionIsCatch = isa<CatchHandler>(Actions[I].get());
+    auto *Handler = cast<Function>(Actions[I]->getHandlerBlockOrFunc());
+    // Various conditions can lead to a handler being popped from the
+    // stack and re-pushed later.  That shouldn't create a new state.
+    // FIXME: Can code optimization lead to re-used handlers?
+    if (FuncInfo.HandlerEnclosedState.count(Handler)) {
+      // If we already assigned the state enclosed by this handler re-use it.
+      Actions[I]->setEHState(FuncInfo.HandlerEnclosedState[Handler]);
+      continue;
+    }
+    const LandingPadInst* RootLPad = FuncInfo.RootLPad[Handler];
+    if (CurrActionIsCatch && LastActionWasCatch && RootLPad == LastRootLPad) {
+      DEBUG(dbgs() << "setEHState for handler to " << currentEHNumber() << "\n");
+      Actions[I]->setEHState(currentEHNumber());
+    } else {
+      DEBUG(dbgs() << "createUnwindMapEntry(" << currentEHNumber() << ", ");
+      print_name(Actions[I]->getHandlerBlockOrFunc());
+      DEBUG(dbgs() << ") with EH state " << NextState << "\n");
+      createUnwindMapEntry(currentEHNumber(), Actions[I].get());
+      DEBUG(dbgs() << "setEHState for handler to " << NextState << "\n");
+      Actions[I]->setEHState(NextState);
+      NextState++;
+    }
+    HandlerStack.push_back(std::move(Actions[I]));
+    LastActionWasCatch = CurrActionIsCatch;
+    LastRootLPad = RootLPad;
+  }
+
+  // This is used to defer numbering states for a handler until after the
+  // last time it appears in an invoke action list.
+  if (CS.isInvoke()) {
+    for (int I = 0, E = HandlerStack.size(); I < E; ++I) {
+      auto *Handler = cast<Function>(HandlerStack[I]->getHandlerBlockOrFunc());
+      if (FuncInfo.LastInvoke[Handler] != cast<InvokeInst>(CS.getInstruction()))
+        continue;
+      FuncInfo.LastInvokeVisited[Handler] = true;
+      DEBUG(dbgs() << "Last invoke of ");
+      print_name(Handler);
+      DEBUG(dbgs() << " has been visited.\n");
+    }
+  }
+
+  DEBUG(dbgs() << "In EHState " << currentEHNumber() << " for CallSite: ");
+  print_name(CS ? CS.getCalledValue() : nullptr);
+  DEBUG(dbgs() << '\n');
+}
+
+void WinEHNumbering::popUnmatchedActions(int FirstMismatch) {
+  // Don't recurse while we are looping over the handler stack.  Instead, defer
+  // the numbering of the catch handlers until we are done popping.
+  SmallVector<CatchHandler *, 4> PoppedCatches;
+  for (int I = HandlerStack.size() - 1; I >= FirstMismatch; --I) {
+    std::unique_ptr<ActionHandler> Handler = HandlerStack.pop_back_val();
+    if (isa<CatchHandler>(Handler.get()))
+      PoppedCatches.push_back(cast<CatchHandler>(Handler.release()));
+  }
+
+  int TryHigh = NextState - 1;
+  int LastTryLowIdx = 0;
+  for (int I = 0, E = PoppedCatches.size(); I != E; ++I) {
+    CatchHandler *CH = PoppedCatches[I];
+    DEBUG(dbgs() << "Popped handler with state " << CH->getEHState() << "\n");
+    if (I + 1 == E || CH->getEHState() != PoppedCatches[I + 1]->getEHState()) {
+      int TryLow = CH->getEHState();
+      auto Handlers =
+          makeArrayRef(&PoppedCatches[LastTryLowIdx], I - LastTryLowIdx + 1);
+      DEBUG(dbgs() << "createTryBlockMapEntry(" << TryLow << ", " << TryHigh);
+      for (size_t J = 0; J < Handlers.size(); ++J) {
+        DEBUG(dbgs() << ", ");
+        print_name(Handlers[J]->getHandlerBlockOrFunc());
+      }
+      DEBUG(dbgs() << ")\n");
+      createTryBlockMapEntry(TryLow, TryHigh, Handlers);
+      LastTryLowIdx = I + 1;
+    }
+  }
+
+  for (CatchHandler *CH : PoppedCatches) {
+    if (auto *F = dyn_cast<Function>(CH->getHandlerBlockOrFunc())) {
+      if (FuncInfo.LastInvokeVisited[F]) {
+        DEBUG(dbgs() << "Assigning base state " << NextState << " to ");
+        print_name(F);
+        DEBUG(dbgs() << '\n');
+        FuncInfo.HandlerBaseState[F] = NextState;
+        DEBUG(dbgs() << "createUnwindMapEntry(" << currentEHNumber()
+                     << ", null)\n");
+        createUnwindMapEntry(currentEHNumber(), nullptr);
+        ++NextState;
+        calculateStateNumbers(*F);
+      }
+      else {
+        DEBUG(dbgs() << "Deferring handling of ");
+        print_name(F);
+        DEBUG(dbgs() << " until last invoke visited.\n");
+      }
+    }
+    delete CH;
+  }
+}
+
+void WinEHNumbering::calculateStateNumbers(const Function &F) {
+  auto I = VisitedHandlers.insert(&F);
+  if (!I.second)
+    return; // We've already visited this handler, don't renumber it.
+
+  int OldBaseState = CurrentBaseState;
+  if (FuncInfo.HandlerBaseState.count(&F)) {
+    CurrentBaseState = FuncInfo.HandlerBaseState[&F];
+  }
+
+  size_t SavedHandlerStackSize = HandlerStack.size();
+
+  DEBUG(dbgs() << "Calculating state numbers for: " << F.getName() << '\n');
+  SmallVector<std::unique_ptr<ActionHandler>, 4> ActionList;
+  for (const BasicBlock &BB : F) {
+    for (const Instruction &I : BB) {
+      const auto *CI = dyn_cast<CallInst>(&I);
+      if (!CI || CI->doesNotThrow())
+        continue;
+      processCallSite(None, CI);
+    }
+    const auto *II = dyn_cast<InvokeInst>(BB.getTerminator());
+    if (!II)
+      continue;
+    const LandingPadInst *LPI = II->getLandingPadInst();
+    auto *ActionsCall = dyn_cast<IntrinsicInst>(LPI->getNextNode());
+    if (!ActionsCall)
+      continue;
+    parseEHActions(ActionsCall, ActionList);
+    if (ActionList.empty())
+      continue;
+    processCallSite(ActionList, II);
+    ActionList.clear();
+    FuncInfo.LandingPadStateMap[LPI] = currentEHNumber();
+    DEBUG(dbgs() << "Assigning state " << currentEHNumber()
+                  << " to landing pad at " << LPI->getParent()->getName()
+                  << '\n');
+  }
+
+  // Pop any actions that were pushed on the stack for this function.
+  popUnmatchedActions(SavedHandlerStackSize);
+
+  DEBUG(dbgs() << "Assigning max state " << NextState - 1
+               << " to " << F.getName() << '\n');
+  FuncInfo.CatchHandlerMaxState[&F] = NextState - 1;
+
+  CurrentBaseState = OldBaseState;
+}
+
+// This function follows the same basic traversal as calculateStateNumbers
+// but it is necessary to identify the root landing pad associated
+// with each action before we start assigning state numbers.
+void WinEHNumbering::findActionRootLPads(const Function &F) {
+  auto I = VisitedHandlers.insert(&F);
+  if (!I.second)
+    return; // We've already visited this handler, don't revisit it.
+
+  SmallVector<std::unique_ptr<ActionHandler>, 4> ActionList;
+  for (const BasicBlock &BB : F) {
+    const auto *II = dyn_cast<InvokeInst>(BB.getTerminator());
+    if (!II)
+      continue;
+    const LandingPadInst *LPI = II->getLandingPadInst();
+    auto *ActionsCall = dyn_cast<IntrinsicInst>(LPI->getNextNode());
+    if (!ActionsCall)
+      continue;
+
+    assert(ActionsCall->getIntrinsicID() == Intrinsic::eh_actions);
+    parseEHActions(ActionsCall, ActionList);
+    if (ActionList.empty())
+      continue;
+    for (int I = 0, E = ActionList.size(); I < E; ++I) {
+      if (auto *Handler
+              = dyn_cast<Function>(ActionList[I]->getHandlerBlockOrFunc())) {
+        FuncInfo.LastInvoke[Handler] = II;
+        // Don't replace the root landing pad if we previously saw this
+        // handler in a different function.
+        if (FuncInfo.RootLPad.count(Handler) &&
+            FuncInfo.RootLPad[Handler]->getParent()->getParent() != &F)
+          continue;
+        DEBUG(dbgs() << "Setting root lpad for ");
+        print_name(Handler);
+        DEBUG(dbgs() << " to " << LPI->getParent()->getName() << '\n');
+        FuncInfo.RootLPad[Handler] = LPI;
+      }
+    }
+    // Walk the actions again and look for nested handlers.  This has to
+    // happen after all of the actions have been processed in the current
+    // function.
+    for (int I = 0, E = ActionList.size(); I < E; ++I)
+      if (auto *Handler
+              = dyn_cast<Function>(ActionList[I]->getHandlerBlockOrFunc()))
+        findActionRootLPads(*Handler);
+    ActionList.clear();
+  }
+}
+
+void llvm::calculateWinCXXEHStateNumbers(const Function *ParentFn,
+                                         WinEHFuncInfo &FuncInfo) {
+  // Return if it's already been done.
+  if (!FuncInfo.LandingPadStateMap.empty())
+    return;
+
+  WinEHNumbering Num(FuncInfo);
+  Num.findActionRootLPads(*ParentFn);
+  // The VisitedHandlers list is used by both findActionRootLPads and
+  // calculateStateNumbers, but both functions need to visit all handlers.
+  Num.VisitedHandlers.clear();
+  Num.calculateStateNumbers(*ParentFn);
+  // Pop everything on the handler stack.
+  // It may be necessary to call this more than once because a handler can
+  // be pushed on the stack as a result of clearing the stack.
+  while (!Num.HandlerStack.empty())
+    Num.processCallSite(None, ImmutableCallSite());
+}
diff --git a/lib/DebugInfo/DWARF/CMakeLists.txt b/lib/DebugInfo/DWARF/CMakeLists.txt
index 8c6d495228c..d5f8a6f24ee 100644
--- a/lib/DebugInfo/DWARF/CMakeLists.txt
+++ b/lib/DebugInfo/DWARF/CMakeLists.txt
@@ -1,5 +1,4 @@
 add_llvm_library(LLVMDebugInfoDWARF
-  DIContext.cpp
   DWARFAbbreviationDeclaration.cpp
   DWARFAcceleratorTable.cpp
   DWARFCompileUnit.cpp
@@ -19,4 +18,5 @@ add_llvm_library(LLVMDebugInfoDWARF
 
   ADDITIONAL_HEADER_DIRS
   ${LLVM_MAIN_INCLUDE_DIR}/llvm/DebugInfo/DWARF
+  ${LLVM_MAIN_INCLUDE_DIR}/llvm/DebugInfo
   )
diff --git a/lib/DebugInfo/DWARF/DIContext.cpp b/lib/DebugInfo/DWARF/DIContext.cpp
deleted file mode 100644
index a1c6ca4bd74..00000000000
--- a/lib/DebugInfo/DWARF/DIContext.cpp
+++ /dev/null
@@ -1,18 +0,0 @@
-//===-- DIContext.cpp -----------------------------------------------------===//
-//
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
-//
-//===----------------------------------------------------------------------===//
-
-#include "llvm/DebugInfo/DWARF/DIContext.h"
-#include "llvm/DebugInfo/DWARF/DWARFContext.h"
-using namespace llvm;
-
-DIContext::~DIContext() {}
-
-DIContext *DIContext::getDWARFContext(const object::ObjectFile &Obj) {
-  return new DWARFContextInMemory(Obj);
-}
diff --git a/lib/DebugInfo/DWARF/DWARFCompileUnit.cpp b/lib/DebugInfo/DWARF/DWARFCompileUnit.cpp
index 01e724799ab..39a7c772dc7 100644
--- a/lib/DebugInfo/DWARF/DWARFCompileUnit.cpp
+++ b/lib/DebugInfo/DWARF/DWARFCompileUnit.cpp
@@ -22,9 +22,10 @@ void DWARFCompileUnit::dump(raw_ostream &OS) {
      << " (next unit at " << format("0x%08x", getNextUnitOffset())
      << ")\n";
 
-  const DWARFDebugInfoEntryMinimal *CU = getCompileUnitDIE(false);
-  assert(CU && "Null Compile Unit?");
-  CU->dump(OS, this, -1U);
+  if (const DWARFDebugInfoEntryMinimal *CU = getUnitDIE(false))
+    CU->dump(OS, this, -1U);
+  else
+    OS << "<compile unit can't be parsed!>\n\n";
 }
 
 // VTable anchor.
diff --git a/lib/DebugInfo/DWARF/DWARFContext.cpp b/lib/DebugInfo/DWARF/DWARFContext.cpp
index 3b427007e59..4858f3633d0 100644
--- a/lib/DebugInfo/DWARF/DWARFContext.cpp
+++ b/lib/DebugInfo/DWARF/DWARFContext.cpp
@@ -140,9 +140,11 @@ void DWARFContext::dump(raw_ostream &OS, DIDumpType DumpType) {
     OS << "\n.debug_line contents:\n";
     for (const auto &CU : compile_units()) {
       savedAddressByteSize = CU->getAddressByteSize();
-      unsigned stmtOffset =
-          CU->getCompileUnitDIE()->getAttributeValueAsSectionOffset(
-              CU.get(), DW_AT_stmt_list, -1U);
+      const auto *CUDIE = CU->getUnitDIE();
+      if (CUDIE == nullptr)
+        continue;
+      unsigned stmtOffset = CUDIE->getAttributeValueAsSectionOffset(
+          CU.get(), DW_AT_stmt_list, -1U);
       if (stmtOffset != -1U) {
         DataExtractor lineData(getLineSection().Data, isLittleEndian(),
                                savedAddressByteSize);
@@ -321,13 +323,14 @@ const DWARFDebugFrame *DWARFContext::getDebugFrame() {
 }
 
 const DWARFLineTable *
-DWARFContext::getLineTableForUnit(DWARFUnit *cu) {
+DWARFContext::getLineTableForUnit(DWARFUnit *U) {
   if (!Line)
     Line.reset(new DWARFDebugLine(&getLineSection().Relocs));
-
+  const auto *UnitDIE = U->getUnitDIE();
+  if (UnitDIE == nullptr)
+    return nullptr;
   unsigned stmtOffset =
-      cu->getCompileUnitDIE()->getAttributeValueAsSectionOffset(
-          cu, DW_AT_stmt_list, -1U);
+      UnitDIE->getAttributeValueAsSectionOffset(U, DW_AT_stmt_list, -1U);
   if (stmtOffset == -1U)
     return nullptr; // No line table for this compile unit.
 
@@ -337,7 +340,7 @@ DWARFContext::getLineTableForUnit(DWARFUnit *cu) {
 
   // We have to parse it first.
   DataExtractor lineData(getLineSection().Data, isLittleEndian(),
-                         cu->getAddressByteSize());
+                         U->getAddressByteSize());
   return Line->getOrParseLineTable(lineData, stmtOffset);
 }
 
@@ -349,7 +352,7 @@ void DWARFContext::parseTypeUnits() {
   if (!TUs.empty())
     return;
   for (const auto &I : getTypesSections()) {
-    TUs.push_back(DWARFUnitSection<DWARFTypeUnit>());
+    TUs.emplace_back();
     TUs.back().parse(*this, I.second);
   }
 }
@@ -362,7 +365,7 @@ void DWARFContext::parseDWOTypeUnits() {
   if (!DWOTUs.empty())
     return;
   for (const auto &I : getTypesDWOSections()) {
-    DWOTUs.push_back(DWARFUnitSection<DWARFTypeUnit>());
+    DWOTUs.emplace_back();
     DWOTUs.back().parseDWO(*this, I.second);
   }
 }
@@ -537,7 +540,8 @@ static bool consumeCompressedDebugSectionHeader(StringRef &data,
   return true;
 }
 
-DWARFContextInMemory::DWARFContextInMemory(const object::ObjectFile &Obj)
+DWARFContextInMemory::DWARFContextInMemory(const object::ObjectFile &Obj,
+    const LoadedObjectInfo *L)
     : IsLittleEndian(Obj.isLittleEndian()),
       AddressSize(Obj.getBytesInAddress()) {
   for (const SectionRef &Section : Obj.sections()) {
@@ -551,7 +555,13 @@ DWARFContextInMemory::DWARFContextInMemory(const object::ObjectFile &Obj)
     if (IsVirtual)
       continue;
     StringRef data;
-    Section.getContents(data);
+
+    section_iterator RelocatedSection = Section.getRelocatedSection();
+    // Try to obtain an already relocated version of this section.
+    // Else use the unrelocated section from the object file. We'll have to
+    // apply relocations ourselves later.
+    if (!L || !L->getLoadedSectionContents(*RelocatedSection,data))
+      Section.getContents(data);
 
     name = name.substr(name.find_first_not_of("._")); // Skip . and _ prefixes.
 
@@ -614,12 +624,19 @@ DWARFContextInMemory::DWARFContextInMemory(const object::ObjectFile &Obj)
       TypesDWOSections[Section].Data = data;
     }
 
-    section_iterator RelocatedSection = Section.getRelocatedSection();
     if (RelocatedSection == Obj.section_end())
       continue;
 
     StringRef RelSecName;
+    StringRef RelSecData;
     RelocatedSection->getName(RelSecName);
+
+    // If the section we're relocating was relocated already by the JIT,
+    // then we used the relocated version above, so we do not need to process
+    // relocations for it now.
+    if (L && L->getLoadedSectionContents(*RelocatedSection,RelSecData))
+      continue;
+
     RelSecName = RelSecName.substr(
         RelSecName.find_first_not_of("._")); // Skip . and _ prefixes.
 
@@ -650,23 +667,56 @@ DWARFContextInMemory::DWARFContextInMemory(const object::ObjectFile &Obj)
     if (Section.relocation_begin() != Section.relocation_end()) {
       uint64_t SectionSize = RelocatedSection->getSize();
       for (const RelocationRef &Reloc : Section.relocations()) {
-        uint64_t Address;
-        Reloc.getOffset(Address);
-        uint64_t Type;
-        Reloc.getType(Type);
+        uint64_t Address = Reloc.getOffset();
+        uint64_t Type = Reloc.getType();
         uint64_t SymAddr = 0;
+        uint64_t SectionLoadAddress = 0;
         object::symbol_iterator Sym = Reloc.getSymbol();
-        if (Sym != Obj.symbol_end())
-          Sym->getAddress(SymAddr);
+        object::section_iterator RSec = Obj.section_end();
+
+        // First calculate the address of the symbol or section as it appears
+        // in the objct file
+        if (Sym != Obj.symbol_end()) {
+          ErrorOr<uint64_t> SymAddrOrErr = Sym->getAddress();
+          if (std::error_code EC = SymAddrOrErr.getError()) {
+            errs() << "error: failed to compute symbol address: "
+                   << EC.message() << '\n';
+            continue;
+          }
+          SymAddr = *SymAddrOrErr;
+          // Also remember what section this symbol is in for later
+          Sym->getSection(RSec);
+        } else if (auto *MObj = dyn_cast<MachOObjectFile>(&Obj)) {
+          // MachO also has relocations that point to sections and
+          // scattered relocations.
+          // FIXME: We are not handling scattered relocations, do we have to?
+          RSec = MObj->getRelocationSection(Reloc.getRawDataRefImpl());
+          SymAddr = RSec->getAddress();
+        }
+
+        // If we are given load addresses for the sections, we need to adjust:
+        // SymAddr = (Address of Symbol Or Section in File) -
+        //           (Address of Section in File) +
+        //           (Load Address of Section)
+        if (L != nullptr && RSec != Obj.section_end()) {
+          // RSec is now either the section being targetted or the section
+          // containing the symbol being targetted. In either case,
+          // we need to perform the same computation.
+          StringRef SecName;
+          RSec->getName(SecName);
+//           llvm::dbgs() << "Name: '" << SecName
+//                        << "', RSec: " << RSec->getRawDataRefImpl()
+//                        << ", Section: " << Section.getRawDataRefImpl() << "\n";
+          SectionLoadAddress = L->getSectionLoadAddress(*RSec);
+          if (SectionLoadAddress != 0)
+            SymAddr += SectionLoadAddress - RSec->getAddress();
+        }
 
         object::RelocVisitor V(Obj);
         object::RelocToApply R(V.visit(Type, Reloc, SymAddr));
         if (V.error()) {
           SmallString<32> Name;
-          std::error_code ec(Reloc.getTypeName(Name));
-          if (ec) {
-            errs() << "Aaaaaa! Nameless relocation! Aaaaaa!\n";
-          }
+          Reloc.getTypeName(Name);
           errs() << "error: failed to compute relocation: "
                  << Name << "\n";
           continue;
diff --git a/lib/DebugInfo/DWARF/DWARFDebugFrame.cpp b/lib/DebugInfo/DWARF/DWARFDebugFrame.cpp
index 784652927ec..1aa31be71fe 100644
--- a/lib/DebugInfo/DWARF/DWARFDebugFrame.cpp
+++ b/lib/DebugInfo/DWARF/DWARFDebugFrame.cpp
@@ -189,10 +189,13 @@ class CIE : public FrameEntry {
   // CIEs (and FDEs) are simply container classes, so the only sensible way to
   // create them is by providing the full parsed contents in the constructor.
   CIE(uint64_t Offset, uint64_t Length, uint8_t Version,
-      SmallString<8> Augmentation, uint64_t CodeAlignmentFactor,
+      SmallString<8> Augmentation, uint8_t AddressSize,
+      uint8_t SegmentDescriptorSize, uint64_t CodeAlignmentFactor,
       int64_t DataAlignmentFactor, uint64_t ReturnAddressRegister)
       : FrameEntry(FK_CIE, Offset, Length), Version(Version),
         Augmentation(std::move(Augmentation)),
+        AddressSize(AddressSize),
+        SegmentDescriptorSize(SegmentDescriptorSize),
         CodeAlignmentFactor(CodeAlignmentFactor),
         DataAlignmentFactor(DataAlignmentFactor),
         ReturnAddressRegister(ReturnAddressRegister) {}
@@ -208,6 +211,12 @@ class CIE : public FrameEntry {
        << "\n";
     OS << format("  Version:               %d\n", Version);
     OS << "  Augmentation:          \"" << Augmentation << "\"\n";
+    if (Version >= 4) {
+      OS << format("  Address size:          %u\n",
+                   (uint32_t)AddressSize);
+      OS << format("  Segment desc size:     %u\n",
+                   (uint32_t)SegmentDescriptorSize);
+    }
     OS << format("  Code alignment factor: %u\n",
                  (uint32_t)CodeAlignmentFactor);
     OS << format("  Data alignment factor: %d\n",
@@ -222,9 +231,11 @@ class CIE : public FrameEntry {
   }
 
 private:
-  /// The following fields are defined in section 6.4.1 of the DWARF standard v3
+  /// The following fields are defined in section 6.4.1 of the DWARF standard v4
   uint8_t Version;
   SmallString<8> Augmentation;
+  uint8_t AddressSize;
+  uint8_t SegmentDescriptorSize;
   uint64_t CodeAlignmentFactor;
   int64_t DataAlignmentFactor;
   uint64_t ReturnAddressRegister;
@@ -461,18 +472,18 @@ void DWARFDebugFrame::parse(DataExtractor Data) {
     bool IsCIE = ((IsDWARF64 && Id == DW64_CIE_ID) || Id == DW_CIE_ID);
 
     if (IsCIE) {
-      // Note: this is specifically DWARFv3 CIE header structure. It was
-      // changed in DWARFv4. We currently don't support reading DWARFv4
-      // here because LLVM itself does not emit it (and LLDB doesn't
-      // support it either).
       uint8_t Version = Data.getU8(&Offset);
       const char *Augmentation = Data.getCStr(&Offset);
+      uint8_t AddressSize = Version < 4 ? Data.getAddressSize() : Data.getU8(&Offset);
+      Data.setAddressSize(AddressSize);
+      uint8_t SegmentDescriptorSize = Version < 4 ? 0 : Data.getU8(&Offset);
       uint64_t CodeAlignmentFactor = Data.getULEB128(&Offset);
       int64_t DataAlignmentFactor = Data.getSLEB128(&Offset);
       uint64_t ReturnAddressRegister = Data.getULEB128(&Offset);
 
       auto Cie = make_unique<CIE>(StartOffset, Length, Version,
-                                  StringRef(Augmentation), CodeAlignmentFactor,
+                                  StringRef(Augmentation), AddressSize,
+                                  SegmentDescriptorSize, CodeAlignmentFactor,
                                   DataAlignmentFactor, ReturnAddressRegister);
       CIEs[StartOffset] = Cie.get();
       Entries.emplace_back(std::move(Cie));
diff --git a/lib/DebugInfo/DWARF/DWARFDebugInfoEntry.cpp b/lib/DebugInfo/DWARF/DWARFDebugInfoEntry.cpp
index e963b7ca6e6..5abbde4ac0f 100644
--- a/lib/DebugInfo/DWARF/DWARFDebugInfoEntry.cpp
+++ b/lib/DebugInfo/DWARF/DWARFDebugInfoEntry.cpp
@@ -161,14 +161,15 @@ void DWARFDebugInfoEntryMinimal::dumpAttribute(raw_ostream &OS,
   // We have dumped the attribute raw value. For some attributes
   // having both the raw value and the pretty-printed value is
   // interesting. These attributes are handled below.
-  if ((attr == DW_AT_specification || attr == DW_AT_abstract_origin) &&
-      // The signature references aren't handled.
-      formValue.getForm() != DW_FORM_ref_sig8) {
-    uint32_t Ref = formValue.getAsReference(u).getValue();
-    DWARFDebugInfoEntryMinimal DIE;
-    if (const DWARFUnit *RefU = findUnitAndExtractFast(DIE, u, &Ref))
-      if (const char *Ref = DIE.getName(RefU, DINameKind::LinkageName))
-        OS << " \"" << Ref << '\"';
+  if (attr == DW_AT_specification || attr == DW_AT_abstract_origin) {
+    Optional<uint64_t> Ref = formValue.getAsReference(u);
+    if (Ref.hasValue()) {
+      uint32_t RefOffset = Ref.getValue();
+      DWARFDebugInfoEntryMinimal DIE;
+      if (const DWARFUnit *RefU = findUnitAndExtractFast(DIE, u, &RefOffset))
+        if (const char *Name = DIE.getName(RefU, DINameKind::LinkageName))
+          OS << " \"" << Name << '\"';
+    }
   } else if (attr == DW_AT_APPLE_property_attribute) {
     if (Optional<uint64_t> OptVal = formValue.getAsUnsignedConstant())
       dumpApplePropertyAttribute(OS, *OptVal);
diff --git a/lib/DebugInfo/DWARF/DWARFDebugLine.cpp b/lib/DebugInfo/DWARF/DWARFDebugLine.cpp
index b63af6a64ee..a0bee0da176 100644
--- a/lib/DebugInfo/DWARF/DWARFDebugLine.cpp
+++ b/lib/DebugInfo/DWARF/DWARFDebugLine.cpp
@@ -25,6 +25,7 @@ void DWARFDebugLine::Prologue::clear() {
   TotalLength = Version = PrologueLength = 0;
   MinInstLength = MaxOpsPerInst = DefaultIsStmt = LineBase = LineRange = 0;
   OpcodeBase = 0;
+  IsDWARF64 = false;
   StandardOpcodeLengths.clear();
   IncludeDirectories.clear();
   FileNames.clear();
@@ -32,9 +33,9 @@ void DWARFDebugLine::Prologue::clear() {
 
 void DWARFDebugLine::Prologue::dump(raw_ostream &OS) const {
   OS << "Line table prologue:\n"
-     << format("    total_length: 0x%8.8x\n", TotalLength)
+     << format("    total_length: 0x%8.8" PRIx64 "\n", TotalLength)
      << format("         version: %u\n", Version)
-     << format(" prologue_length: 0x%8.8x\n", PrologueLength)
+     << format(" prologue_length: 0x%8.8" PRIx64 "\n", PrologueLength)
      << format(" min_inst_length: %u\n", MinInstLength)
      << format(Version >= 4 ? "max_ops_per_inst: %u\n" : "", MaxOpsPerInst)
      << format(" default_is_stmt: %u\n", DefaultIsStmt)
@@ -67,16 +68,23 @@ void DWARFDebugLine::Prologue::dump(raw_ostream &OS) const {
 
 bool DWARFDebugLine::Prologue::parse(DataExtractor debug_line_data,
                                      uint32_t *offset_ptr) {
-  const uint32_t prologue_offset = *offset_ptr;
+  const uint64_t prologue_offset = *offset_ptr;
 
   clear();
   TotalLength = debug_line_data.getU32(offset_ptr);
+  if (TotalLength == UINT32_MAX) {
+    IsDWARF64 = true;
+    TotalLength = debug_line_data.getU64(offset_ptr);
+  } else if (TotalLength > 0xffffff00) {
+    return false;
+  }
   Version = debug_line_data.getU16(offset_ptr);
   if (Version < 2)
     return false;
 
-  PrologueLength = debug_line_data.getU32(offset_ptr);
-  const uint32_t end_prologue_offset = PrologueLength + *offset_ptr;
+  PrologueLength = debug_line_data.getUnsigned(offset_ptr,
+                                               sizeofPrologueLength());
+  const uint64_t end_prologue_offset = PrologueLength + *offset_ptr;
   MinInstLength = debug_line_data.getU8(offset_ptr);
   if (Version >= 4)
     MaxOpsPerInst = debug_line_data.getU8(offset_ptr);
@@ -114,9 +122,10 @@ bool DWARFDebugLine::Prologue::parse(DataExtractor debug_line_data,
   }
 
   if (*offset_ptr != end_prologue_offset) {
-    fprintf(stderr, "warning: parsing line table prologue at 0x%8.8x should"
-                    " have ended at 0x%8.8x but it ended at 0x%8.8x\n",
-            prologue_offset, end_prologue_offset, *offset_ptr);
+    fprintf(stderr, "warning: parsing line table prologue at 0x%8.8" PRIx64
+                    " should have ended at 0x%8.8" PRIx64
+                    " but it ended at 0x%8.8" PRIx64 "\n",
+            prologue_offset, end_prologue_offset, (uint64_t)*offset_ptr);
     return false;
   }
   return true;
@@ -258,7 +267,7 @@ bool DWARFDebugLine::LineTable::parse(DataExtractor debug_line_data,
   }
 
   const uint32_t end_offset = debug_line_offset + Prologue.TotalLength +
-                              sizeof(Prologue.TotalLength);
+                              Prologue.sizeofTotalLength();
 
   ParsingState State(this);
 
@@ -522,10 +531,36 @@ bool DWARFDebugLine::LineTable::parse(DataExtractor debug_line_data,
   return end_offset;
 }
 
+uint32_t
+DWARFDebugLine::LineTable::findRowInSeq(const DWARFDebugLine::Sequence &seq,
+                                        uint64_t address) const {
+  if (!seq.containsPC(address))
+    return UnknownRowIndex;
+  // Search for instruction address in the rows describing the sequence.
+  // Rows are stored in a vector, so we may use arithmetical operations with
+  // iterators.
+  DWARFDebugLine::Row row;
+  row.Address = address;
+  RowIter first_row = Rows.begin() + seq.FirstRowIndex;
+  RowIter last_row = Rows.begin() + seq.LastRowIndex;
+  LineTable::RowIter row_pos = std::lower_bound(
+      first_row, last_row, row, DWARFDebugLine::Row::orderByAddress);
+  if (row_pos == last_row) {
+    return seq.LastRowIndex - 1;
+  }
+  uint32_t index = seq.FirstRowIndex + (row_pos - first_row);
+  if (row_pos->Address > address) {
+    if (row_pos == first_row)
+      return UnknownRowIndex;
+    else
+      index--;
+  }
+  return index;
+}
+
 uint32_t DWARFDebugLine::LineTable::lookupAddress(uint64_t address) const {
-  uint32_t unknown_index = UINT32_MAX;
   if (Sequences.empty())
-    return unknown_index;
+    return UnknownRowIndex;
   // First, find an instruction sequence containing the given address.
   DWARFDebugLine::Sequence sequence;
   sequence.LowPC = address;
@@ -540,31 +575,10 @@ uint32_t DWARFDebugLine::LineTable::lookupAddress(uint64_t address) const {
     found_seq = *seq_pos;
   } else {
     if (seq_pos == first_seq)
-      return unknown_index;
+      return UnknownRowIndex;
     found_seq = *(seq_pos - 1);
   }
-  if (!found_seq.containsPC(address))
-    return unknown_index;
-  // Search for instruction address in the rows describing the sequence.
-  // Rows are stored in a vector, so we may use arithmetical operations with
-  // iterators.
-  DWARFDebugLine::Row row;
-  row.Address = address;
-  RowIter first_row = Rows.begin() + found_seq.FirstRowIndex;
-  RowIter last_row = Rows.begin() + found_seq.LastRowIndex;
-  RowIter row_pos = std::lower_bound(first_row, last_row, row,
-      DWARFDebugLine::Row::orderByAddress);
-  if (row_pos == last_row) {
-    return found_seq.LastRowIndex - 1;
-  }
-  uint32_t index = found_seq.FirstRowIndex + (row_pos - first_row);
-  if (row_pos->Address > address) {
-    if (row_pos == first_row)
-      return unknown_index;
-    else
-      index--;
-  }
-  return index;
+  return findRowInSeq(found_seq, address);
 }
 
 bool DWARFDebugLine::LineTable::lookupAddressRange(
@@ -593,45 +607,21 @@ bool DWARFDebugLine::LineTable::lookupAddressRange(
   // index we just calculated
 
   while (seq_pos != last_seq && seq_pos->LowPC < end_addr) {
-    DWARFDebugLine::Sequence cur_seq = *seq_pos;
-    uint32_t first_row_index;
-    uint32_t last_row_index;
-    if (seq_pos == start_pos) {
-      // For the first sequence, we need to find which row in the sequence is the
-      // first in our range. Rows are stored in a vector, so we may use
-      // arithmetical operations with iterators.
-      DWARFDebugLine::Row row;
-      row.Address = address;
-      RowIter first_row = Rows.begin() + cur_seq.FirstRowIndex;
-      RowIter last_row = Rows.begin() + cur_seq.LastRowIndex;
-      RowIter row_pos = std::upper_bound(first_row, last_row, row,
-                                         DWARFDebugLine::Row::orderByAddress);
-      // The 'row_pos' iterator references the first row that is greater than
-      // our start address. Unless that's the first row, we want to start at
-      // the row before that.
-      first_row_index = cur_seq.FirstRowIndex + (row_pos - first_row);
-      if (row_pos != first_row)
-        --first_row_index;
-    } else
-      first_row_index = cur_seq.FirstRowIndex;
-
-    // For the last sequence in our range, we need to figure out the last row in
-    // range.  For all other sequences we can go to the end of the sequence.
-    if (cur_seq.HighPC > end_addr) {
-      DWARFDebugLine::Row row;
-      row.Address = end_addr;
-      RowIter first_row = Rows.begin() + cur_seq.FirstRowIndex;
-      RowIter last_row = Rows.begin() + cur_seq.LastRowIndex;
-      RowIter row_pos = std::upper_bound(first_row, last_row, row,
-                                         DWARFDebugLine::Row::orderByAddress);
-      // The 'row_pos' iterator references the first row that is greater than
-      // our end address.  The row before that is the last row we want.
-      last_row_index = cur_seq.FirstRowIndex + (row_pos - first_row) - 1;
-    } else
-      // Contrary to what you might expect, DWARFDebugLine::SequenceLastRowIndex
-      // isn't a valid index within the current sequence.  It's that plus one.
+    const DWARFDebugLine::Sequence &cur_seq = *seq_pos;
+    // For the first sequence, we need to find which row in the sequence is the
+    // first in our range.
+    uint32_t first_row_index = cur_seq.FirstRowIndex;
+    if (seq_pos == start_pos)
+      first_row_index = findRowInSeq(cur_seq, address);
+
+    // Figure out the last row in the range.
+    uint32_t last_row_index = findRowInSeq(cur_seq, end_addr - 1);
+    if (last_row_index == UnknownRowIndex)
       last_row_index = cur_seq.LastRowIndex - 1;
 
+    assert(first_row_index != UnknownRowIndex);
+    assert(last_row_index != UnknownRowIndex);
+
     for (uint32_t i = first_row_index; i <= last_row_index; ++i) {
       result.push_back(i);
     }
diff --git a/lib/DebugInfo/DWARF/DWARFFormValue.cpp b/lib/DebugInfo/DWARF/DWARFFormValue.cpp
index 6946f833424..53a676efaf3 100644
--- a/lib/DebugInfo/DWARF/DWARFFormValue.cpp
+++ b/lib/DebugInfo/DWARF/DWARFFormValue.cpp
@@ -113,21 +113,22 @@ bool DWARFFormValue::isFormClass(DWARFFormValue::FormClass FC) const {
   if (Form < ArrayRef<FormClass>(DWARF4FormClasses).size() &&
       DWARF4FormClasses[Form] == FC)
     return true;
-  // Check DW_FORM_ref_sig8 from DWARF4.
-  if (Form == DW_FORM_ref_sig8)
+  // Check more forms from DWARF4 and DWARF5 proposals.
+  switch (Form) {
+  case DW_FORM_ref_sig8:
+  case DW_FORM_GNU_ref_alt:
     return (FC == FC_Reference);
-  // Check for some DWARF5 forms.
-  if (Form == DW_FORM_GNU_addr_index)
+  case DW_FORM_GNU_addr_index:
     return (FC == FC_Address);
-  if (Form == DW_FORM_GNU_str_index)
+  case DW_FORM_GNU_str_index:
+  case DW_FORM_GNU_strp_alt:
     return (FC == FC_String);
+  }
   // In DWARF3 DW_FORM_data4 and DW_FORM_data8 served also as a section offset.
   // Don't check for DWARF version here, as some producers may still do this
   // by mistake.
-  if ((Form == DW_FORM_data4 || Form == DW_FORM_data8) &&
-      FC == FC_SectionOffset)
-    return true;
-  return false;
+  return (Form == DW_FORM_data4 || Form == DW_FORM_data8) &&
+         FC == FC_SectionOffset;
 }
 
 bool DWARFFormValue::extractValue(DataExtractor data, uint32_t *offset_ptr,
@@ -199,15 +200,6 @@ bool DWARFFormValue::extractValue(DataExtractor data, uint32_t *offset_ptr,
     case DW_FORM_sdata:
       Value.sval = data.getSLEB128(offset_ptr);
       break;
-    case DW_FORM_strp: {
-      Value.uval = data.getU32(offset_ptr);
-      if (!cu)
-        break;
-      RelocAddrMap::const_iterator AI = cu->getRelocMap()->find(*offset_ptr-4);
-      if (AI != cu->getRelocMap()->end())
-        Value.uval += AI->second.second;
-      break;
-    }
     case DW_FORM_udata:
     case DW_FORM_ref_udata:
       Value.uval = data.getULEB128(offset_ptr);
@@ -219,14 +211,18 @@ bool DWARFFormValue::extractValue(DataExtractor data, uint32_t *offset_ptr,
       Form = data.getULEB128(offset_ptr);
       indirect = true;
       break;
-    case DW_FORM_sec_offset: {
+    case DW_FORM_sec_offset:
+    case DW_FORM_strp:
+    case DW_FORM_GNU_ref_alt:
+    case DW_FORM_GNU_strp_alt: {
       // FIXME: This is 64-bit for DWARF64.
       Value.uval = data.getU32(offset_ptr);
       if (!cu)
         break;
-      RelocAddrMap::const_iterator AI = cu->getRelocMap()->find(*offset_ptr-4);
+      RelocAddrMap::const_iterator AI =
+          cu->getRelocMap()->find(*offset_ptr - 4);
       if (AI != cu->getRelocMap()->end())
-        Value.uval +=  AI->second.second;
+        Value.uval += AI->second.second;
       break;
     }
     case DW_FORM_flag_present:
@@ -323,7 +319,6 @@ DWARFFormValue::skipValue(uint16_t form, DataExtractor debug_info_data,
       return true;
 
     // 4 byte values
-    case DW_FORM_strp:
     case DW_FORM_data4:
     case DW_FORM_ref4:
       *offset_ptr += 4;
@@ -353,6 +348,9 @@ DWARFFormValue::skipValue(uint16_t form, DataExtractor debug_info_data,
 
     // FIXME: 4 for DWARF32, 8 for DWARF64.
     case DW_FORM_sec_offset:
+    case DW_FORM_strp:
+    case DW_FORM_GNU_ref_alt:
+    case DW_FORM_GNU_strp_alt:
       *offset_ptr += 4;
       return true;
 
@@ -424,24 +422,17 @@ DWARFFormValue::dump(raw_ostream &OS, const DWARFUnit *cu) const {
   case DW_FORM_udata:     OS << Value.uval; break;
   case DW_FORM_strp: {
     OS << format(" .debug_str[0x%8.8x] = ", (uint32_t)uvalue);
-    Optional<const char *> DbgStr = getAsCString(cu);
-    if (DbgStr.hasValue()) {
-      raw_ostream &COS = WithColor(OS, syntax::String);
-      COS << '"';
-      COS.write_escaped(DbgStr.getValue());
-      COS << '"';
-    }
+    dumpString(OS, cu);
     break;
   }
   case DW_FORM_GNU_str_index: {
     OS << format(" indexed (%8.8x) string = ", (uint32_t)uvalue);
-    Optional<const char *> DbgStr = getAsCString(cu);
-    if (DbgStr.hasValue()) {
-      raw_ostream &COS = WithColor(OS, syntax::String);
-      COS << '"';
-      COS.write_escaped(DbgStr.getValue());
-      COS << '"';
-    }
+    dumpString(OS, cu);
+    break;
+  }
+  case DW_FORM_GNU_strp_alt: {
+    OS << format("alt indirect string, offset: 0x%" PRIx64 "", uvalue);
+    dumpString(OS, cu);
     break;
   }
   case DW_FORM_ref_addr:
@@ -467,6 +458,9 @@ DWARFFormValue::dump(raw_ostream &OS, const DWARFUnit *cu) const {
     cu_relative_offset = true;
     OS << format("cu + 0x%" PRIx64, uvalue);
     break;
+  case DW_FORM_GNU_ref_alt:
+    OS << format("<alt 0x%" PRIx64 ">", uvalue);
+    break;
 
     // All DW_FORM_indirect attributes should be resolved prior to calling
     // this function
@@ -492,12 +486,23 @@ DWARFFormValue::dump(raw_ostream &OS, const DWARFUnit *cu) const {
   }
 }
 
+void DWARFFormValue::dumpString(raw_ostream &OS, const DWARFUnit *U) const {
+  Optional<const char *> DbgStr = getAsCString(U);
+  if (DbgStr.hasValue()) {
+    raw_ostream &COS = WithColor(OS, syntax::String);
+    COS << '"';
+    COS.write_escaped(DbgStr.getValue());
+    COS << '"';
+  }
+}
+
 Optional<const char *> DWARFFormValue::getAsCString(const DWARFUnit *U) const {
   if (!isFormClass(FC_String))
     return None;
   if (Form == DW_FORM_string)
     return Value.cstr;
-  if (!U)
+  // FIXME: Add support for DW_FORM_GNU_strp_alt
+  if (Form == DW_FORM_GNU_strp_alt || U == nullptr)
     return None;
   uint32_t Offset = Value.uval;
   if (Form == DW_FORM_GNU_str_index) {
@@ -539,9 +544,9 @@ Optional<uint64_t> DWARFFormValue::getAsReference(const DWARFUnit *U) const {
     return Value.uval + U->getOffset();
   case DW_FORM_ref_addr:
     return Value.uval;
-  // FIXME: Add proper support for DW_FORM_ref_sig8
+  // FIXME: Add proper support for DW_FORM_ref_sig8 and DW_FORM_GNU_ref_alt.
   default:
-    return Value.uval;
+    return None;
   }
 }
 
diff --git a/lib/DebugInfo/DWARF/DWARFTypeUnit.cpp b/lib/DebugInfo/DWARF/DWARFTypeUnit.cpp
index 65c7bff0990..766e8ac16f0 100644
--- a/lib/DebugInfo/DWARF/DWARFTypeUnit.cpp
+++ b/lib/DebugInfo/DWARF/DWARFTypeUnit.cpp
@@ -33,7 +33,8 @@ void DWARFTypeUnit::dump(raw_ostream &OS) {
      << " (next unit at " << format("0x%08x", getNextUnitOffset())
      << ")\n";
 
-  const DWARFDebugInfoEntryMinimal *CU = getCompileUnitDIE(false);
-  assert(CU && "Null Compile Unit?");
-  CU->dump(OS, this, -1U);
+  if (const DWARFDebugInfoEntryMinimal *TU = getUnitDIE(false))
+    TU->dump(OS, this, -1U);
+  else
+    OS << "<type unit can't be parsed!>\n\n";
 }
diff --git a/lib/DebugInfo/DWARF/DWARFUnit.cpp b/lib/DebugInfo/DWARF/DWARFUnit.cpp
index d4ecd69e0ae..348476d72b6 100644
--- a/lib/DebugInfo/DWARF/DWARFUnit.cpp
+++ b/lib/DebugInfo/DWARF/DWARFUnit.cpp
@@ -79,10 +79,7 @@ bool DWARFUnit::extractImpl(DataExtractor debug_info, uint32_t *offset_ptr) {
     return false;
 
   Abbrevs = Abbrev->getAbbreviationDeclarationSet(AbbrOffset);
-  if (Abbrevs == nullptr)
-    return false;
-
-  return true;
+  return Abbrevs != nullptr;
 }
 
 bool DWARFUnit::extract(DataExtractor debug_info, uint32_t *offset_ptr) {
@@ -256,7 +253,7 @@ DWARFUnit::DWOHolder::DWOHolder(StringRef DWOPath)
     return;
   DWOFile = std::move(Obj.get());
   DWOContext.reset(
-      cast<DWARFContext>(DIContext::getDWARFContext(*DWOFile.getBinary())));
+      cast<DWARFContext>(new DWARFContextInMemory(*DWOFile.getBinary())));
   if (DWOContext->getNumDWOCompileUnits() > 0)
     DWOU = DWOContext->getDWOCompileUnitAtIndex(0);
 }
@@ -310,8 +307,11 @@ void DWARFUnit::clearDIEs(bool KeepCUDie) {
 }
 
 void DWARFUnit::collectAddressRanges(DWARFAddressRangesVector &CURanges) {
-  // First, check if CU DIE describes address ranges for the unit.
-  const auto &CUDIERanges = getCompileUnitDIE()->getAddressRanges(this);
+  const auto *U = getUnitDIE();
+  if (U == nullptr)
+    return;
+  // First, check if unit DIE describes address ranges for the whole unit.
+  const auto &CUDIERanges = U->getAddressRanges(this);
   if (!CUDIERanges.empty()) {
     CURanges.insert(CURanges.end(), CUDIERanges.begin(), CUDIERanges.end());
     return;
diff --git a/lib/DebugInfo/PDB/CMakeLists.txt b/lib/DebugInfo/PDB/CMakeLists.txt
index ed8c67411d9..1645a95aac3 100644
--- a/lib/DebugInfo/PDB/CMakeLists.txt
+++ b/lib/DebugInfo/PDB/CMakeLists.txt
@@ -9,7 +9,7 @@ if(HAVE_DIA_SDK)
   if (CMAKE_SIZEOF_VOID_P EQUAL 8)
     set(LIBPDB_LINK_FOLDERS "${LIBPDB_LINK_FOLDERS}\\amd64")
   endif()
-  set(LIBPDB_ADDITIONAL_LIBRARIES "${LIBPDB_LINK_FOLDERS}\\diaguids.lib")
+  file(TO_CMAKE_PATH "${LIBPDB_LINK_FOLDERS}\\diaguids.lib" LIBPDB_ADDITIONAL_LIBRARIES)
 
   add_pdb_impl_folder(DIA
     DIA/DIADataStream.cpp
@@ -32,6 +32,7 @@ list(APPEND LIBPDB_ADDITIONAL_HEADER_DIRS "${LLVM_MAIN_INCLUDE_DIR}/llvm/DebugIn
 add_llvm_library(LLVMDebugInfoPDB
   IPDBSourceFile.cpp
   PDB.cpp
+  PDBContext.cpp
   PDBExtras.cpp
   PDBInterfaceAnchors.cpp
   PDBSymbol.cpp
diff --git a/lib/DebugInfo/PDB/DIA/DIASession.cpp b/lib/DebugInfo/PDB/DIA/DIASession.cpp
index 4966bea96f6..99fe750ebac 100644
--- a/lib/DebugInfo/PDB/DIA/DIASession.cpp
+++ b/lib/DebugInfo/PDB/DIA/DIASession.cpp
@@ -9,6 +9,7 @@
 
 #include "llvm/ADT/STLExtras.h"
 #include "llvm/DebugInfo/PDB/DIA/DIAEnumDebugStreams.h"
+#include "llvm/DebugInfo/PDB/DIA/DIAEnumLineNumbers.h"
 #include "llvm/DebugInfo/PDB/DIA/DIAEnumSourceFiles.h"
 #include "llvm/DebugInfo/PDB/DIA/DIARawSymbol.h"
 #include "llvm/DebugInfo/PDB/DIA/DIASession.h"
@@ -64,6 +65,50 @@ PDB_ErrorCode DIASession::createFromPdb(StringRef Path,
   return PDB_ErrorCode::Success;
 }
 
+PDB_ErrorCode DIASession::createFromExe(StringRef Path,
+                                        std::unique_ptr<IPDBSession> &Session) {
+  CComPtr<IDiaDataSource> DiaDataSource;
+  CComPtr<IDiaSession> DiaSession;
+
+  // We assume that CoInitializeEx has already been called by the executable.
+  HRESULT Result = ::CoCreateInstance(
+      CLSID_DiaSource, nullptr, CLSCTX_INPROC_SERVER, IID_IDiaDataSource,
+      reinterpret_cast<LPVOID *>(&DiaDataSource));
+  if (FAILED(Result))
+    return PDB_ErrorCode::NoPdbImpl;
+
+  llvm::SmallVector<UTF16, 128> Path16;
+  if (!llvm::convertUTF8ToUTF16String(Path, Path16))
+    return PDB_ErrorCode::InvalidPath;
+
+  const wchar_t *Path16Str = reinterpret_cast<const wchar_t *>(Path16.data());
+  if (FAILED(Result =
+                 DiaDataSource->loadDataForExe(Path16Str, nullptr, nullptr))) {
+    if (Result == E_PDB_NOT_FOUND)
+      return PDB_ErrorCode::InvalidPath;
+    else if (Result == E_PDB_FORMAT)
+      return PDB_ErrorCode::InvalidFileFormat;
+    else if (Result == E_PDB_INVALID_SIG || Result == E_PDB_INVALID_AGE)
+      return PDB_ErrorCode::DebugInfoMismatch;
+    else if (Result == E_INVALIDARG)
+      return PDB_ErrorCode::InvalidParameter;
+    else if (Result == E_UNEXPECTED)
+      return PDB_ErrorCode::AlreadyLoaded;
+    else
+      return PDB_ErrorCode::UnknownError;
+  }
+
+  if (FAILED(Result = DiaDataSource->openSession(&DiaSession))) {
+    if (Result == E_OUTOFMEMORY)
+      return PDB_ErrorCode::NoMemory;
+    else
+      return PDB_ErrorCode::UnknownError;
+  }
+
+  Session.reset(new DIASession(DiaSession));
+  return PDB_ErrorCode::Success;
+}
+
 uint64_t DIASession::getLoadAddress() const {
   uint64_t LoadAddress;
   bool success = (S_OK == Session->get_loadAddress(&LoadAddress));
@@ -95,6 +140,32 @@ std::unique_ptr<PDBSymbol> DIASession::getSymbolById(uint32_t SymbolId) const {
   return PDBSymbol::create(*this, std::move(RawSymbol));
 }
 
+std::unique_ptr<PDBSymbol>
+DIASession::findSymbolByAddress(uint64_t Address, PDB_SymType Type) const {
+  enum SymTagEnum EnumVal = static_cast<enum SymTagEnum>(Type);
+
+  CComPtr<IDiaSymbol> Symbol;
+  if (S_OK != Session->findSymbolByVA(Address, EnumVal, &Symbol)) {
+    ULONGLONG LoadAddr = 0;
+    if (S_OK != Session->get_loadAddress(&LoadAddr))
+      return nullptr;
+    DWORD RVA = static_cast<DWORD>(Address - LoadAddr);
+    if (S_OK != Session->findSymbolByRVA(RVA, EnumVal, &Symbol))
+      return nullptr;
+  }
+  auto RawSymbol = llvm::make_unique<DIARawSymbol>(*this, Symbol);
+  return PDBSymbol::create(*this, std::move(RawSymbol));
+}
+
+std::unique_ptr<IPDBEnumLineNumbers>
+DIASession::findLineNumbersByAddress(uint64_t Address, uint32_t Length) const {
+  CComPtr<IDiaEnumLineNumbers> LineNumbers;
+  if (S_OK != Session->findLinesByVA(Address, Length, &LineNumbers))
+    return nullptr;
+
+  return llvm::make_unique<DIAEnumLineNumbers>(LineNumbers);
+}
+
 std::unique_ptr<IPDBEnumSourceFiles> DIASession::getAllSourceFiles() const {
   CComPtr<IDiaEnumSourceFiles> Files;
   if (S_OK != Session->findFile(nullptr, nullptr, nsNone, &Files))
diff --git a/lib/DebugInfo/PDB/LLVMBuild.txt b/lib/DebugInfo/PDB/LLVMBuild.txt
index 690598ade5c..9ee9f4067e9 100644
--- a/lib/DebugInfo/PDB/LLVMBuild.txt
+++ b/lib/DebugInfo/PDB/LLVMBuild.txt
@@ -19,5 +19,5 @@
 type = Library
 name = DebugInfoPDB
 parent = DebugInfo
-required_libraries = Support
+required_libraries = Object Support
 
diff --git a/lib/DebugInfo/PDB/PDB.cpp b/lib/DebugInfo/PDB/PDB.cpp
index a07396d1a17..13201bbaa64 100644
--- a/lib/DebugInfo/PDB/PDB.cpp
+++ b/lib/DebugInfo/PDB/PDB.cpp
@@ -20,11 +20,20 @@
 
 using namespace llvm;
 
-PDB_ErrorCode llvm::createPDBReader(PDB_ReaderType Type, StringRef Path,
-                                    std::unique_ptr<IPDBSession> &Session) {
+PDB_ErrorCode llvm::loadDataForPDB(PDB_ReaderType Type, StringRef Path,
+                                   std::unique_ptr<IPDBSession> &Session) {
   // Create the correct concrete instance type based on the value of Type.
 #if HAVE_DIA_SDK
   return DIASession::createFromPdb(Path, Session);
 #endif
   return PDB_ErrorCode::NoPdbImpl;
 }
+
+PDB_ErrorCode llvm::loadDataForEXE(PDB_ReaderType Type, StringRef Path,
+                                   std::unique_ptr<IPDBSession> &Session) {
+// Create the correct concrete instance type based on the value of Type.
+#if HAVE_DIA_SDK
+  return DIASession::createFromExe(Path, Session);
+#endif
+  return PDB_ErrorCode::NoPdbImpl;
+}
diff --git a/lib/DebugInfo/PDB/PDBContext.cpp b/lib/DebugInfo/PDB/PDBContext.cpp
new file mode 100644
index 00000000000..83f27c7fa3d
--- /dev/null
+++ b/lib/DebugInfo/PDB/PDBContext.cpp
@@ -0,0 +1,133 @@
+//===-- PDBContext.cpp ------------------------------------------*- C++ -*-===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===/
+
+#include "llvm/DebugInfo/PDB/PDBContext.h"
+#include "llvm/DebugInfo/PDB/IPDBEnumChildren.h"
+#include "llvm/DebugInfo/PDB/IPDBLineNumber.h"
+#include "llvm/DebugInfo/PDB/IPDBSourceFile.h"
+#include "llvm/DebugInfo/PDB/PDBSymbol.h"
+#include "llvm/DebugInfo/PDB/PDBSymbolFunc.h"
+#include "llvm/DebugInfo/PDB/PDBSymbolData.h"
+#include "llvm/DebugInfo/PDB/PDBSymbolPublicSymbol.h"
+#include "llvm/Object/COFF.h"
+
+using namespace llvm;
+using namespace llvm::object;
+
+PDBContext::PDBContext(const COFFObjectFile &Object,
+                       std::unique_ptr<IPDBSession> PDBSession,
+                       bool RelativeAddress)
+    : DIContext(CK_PDB), Session(std::move(PDBSession)) {
+  if (!RelativeAddress) {
+    uint64_t ImageBase = 0;
+    if (Object.is64()) {
+      const pe32plus_header *Header = nullptr;
+      Object.getPE32PlusHeader(Header);
+      if (Header)
+        ImageBase = Header->ImageBase;
+    } else {
+      const pe32_header *Header = nullptr;
+      Object.getPE32Header(Header);
+      if (Header)
+        ImageBase = static_cast<uint64_t>(Header->ImageBase);
+    }
+    Session->setLoadAddress(ImageBase);
+  }
+}
+
+void PDBContext::dump(raw_ostream &OS, DIDumpType DumpType) {}
+
+DILineInfo PDBContext::getLineInfoForAddress(uint64_t Address,
+                                             DILineInfoSpecifier Specifier) {
+  DILineInfo Result;
+  Result.FunctionName = getFunctionName(Address, Specifier.FNKind);
+
+  uint32_t Length = 1;
+  std::unique_ptr<PDBSymbol> Symbol =
+      Session->findSymbolByAddress(Address, PDB_SymType::None);
+  if (auto Func = dyn_cast_or_null<PDBSymbolFunc>(Symbol.get())) {
+    Length = Func->getLength();
+  } else if (auto Data = dyn_cast_or_null<PDBSymbolData>(Symbol.get())) {
+    Length = Data->getLength();
+  }
+
+  // If we couldn't find a symbol, then just assume 1 byte, so that we get
+  // only the line number of the first instruction.
+  auto LineNumbers = Session->findLineNumbersByAddress(Address, Length);
+  if (!LineNumbers || LineNumbers->getChildCount() == 0)
+    return Result;
+
+  auto LineInfo = LineNumbers->getNext();
+  assert(LineInfo);
+  auto SourceFile = Session->getSourceFileById(LineInfo->getSourceFileId());
+
+  if (SourceFile &&
+      Specifier.FLIKind != DILineInfoSpecifier::FileLineInfoKind::None)
+    Result.FileName = SourceFile->getFileName();
+  Result.Column = LineInfo->getColumnNumber();
+  Result.Line = LineInfo->getLineNumber();
+  return Result;
+}
+
+DILineInfoTable
+PDBContext::getLineInfoForAddressRange(uint64_t Address, uint64_t Size,
+                                       DILineInfoSpecifier Specifier) {
+  if (Size == 0)
+    return DILineInfoTable();
+
+  DILineInfoTable Table;
+  auto LineNumbers = Session->findLineNumbersByAddress(Address, Size);
+  if (!LineNumbers || LineNumbers->getChildCount() == 0)
+    return Table;
+
+  while (auto LineInfo = LineNumbers->getNext()) {
+    DILineInfo LineEntry =
+        getLineInfoForAddress(LineInfo->getVirtualAddress(), Specifier);
+    Table.push_back(std::make_pair(LineInfo->getVirtualAddress(), LineEntry));
+  }
+  return Table;
+}
+
+DIInliningInfo
+PDBContext::getInliningInfoForAddress(uint64_t Address,
+                                      DILineInfoSpecifier Specifier) {
+  DIInliningInfo InlineInfo;
+  DILineInfo Frame = getLineInfoForAddress(Address, Specifier);
+  InlineInfo.addFrame(Frame);
+  return InlineInfo;
+}
+
+std::string PDBContext::getFunctionName(uint64_t Address,
+                                        DINameKind NameKind) const {
+  if (NameKind == DINameKind::None)
+    return std::string();
+
+  if (NameKind == DINameKind::LinkageName) {
+    // It is not possible to get the mangled linkage name through a
+    // PDBSymbolFunc.  For that we have to specifically request a
+    // PDBSymbolPublicSymbol.
+    auto PublicSym =
+        Session->findSymbolByAddress(Address, PDB_SymType::PublicSymbol);
+    if (auto PS = dyn_cast_or_null<PDBSymbolPublicSymbol>(PublicSym.get()))
+      return PS->getName();
+  }
+
+  auto FuncSymbol =
+      Session->findSymbolByAddress(Address, PDB_SymType::Function);
+
+  // This could happen either if there was no public symbol (e.g. not
+  // external) or the user requested the short name.  In the former case,
+  // although they technically requested the linkage name, if the linkage
+  // name is not available we fallback to at least returning a non-empty
+  // string.
+  if (auto Func = dyn_cast_or_null<PDBSymbolFunc>(FuncSymbol.get()))
+      return Func->getName();
+
+  return std::string();
+}
diff --git a/lib/DebugInfo/PDB/PDBSymbolCustom.cpp b/lib/DebugInfo/PDB/PDBSymbolCustom.cpp
index 45faa0bf7be..bd7d9cda218 100644
--- a/lib/DebugInfo/PDB/PDBSymbolCustom.cpp
+++ b/lib/DebugInfo/PDB/PDBSymbolCustom.cpp
@@ -25,4 +25,4 @@ void PDBSymbolCustom::getDataBytes(llvm::SmallVector<uint8_t, 32> &bytes) {
   RawSymbol->getDataBytes(bytes);
 }
 
-void PDBSymbolCustom::dump(PDBSymDumper &Dumper) const { Dumper.dump(*this); }
\ No newline at end of file
+void PDBSymbolCustom::dump(PDBSymDumper &Dumper) const { Dumper.dump(*this); }
diff --git a/lib/DebugInfo/PDB/PDBSymbolData.cpp b/lib/DebugInfo/PDB/PDBSymbolData.cpp
index 60dcbc110a9..a948c2d08af 100644
--- a/lib/DebugInfo/PDB/PDBSymbolData.cpp
+++ b/lib/DebugInfo/PDB/PDBSymbolData.cpp
@@ -24,4 +24,4 @@ std::unique_ptr<PDBSymbol> PDBSymbolData::getType() const {
   return Session.getSymbolById(getTypeId());
 }
 
-void PDBSymbolData::dump(PDBSymDumper &Dumper) const { Dumper.dump(*this); }
\ No newline at end of file
+void PDBSymbolData::dump(PDBSymDumper &Dumper) const { Dumper.dump(*this); }
diff --git a/lib/ExecutionEngine/CMakeLists.txt b/lib/ExecutionEngine/CMakeLists.txt
index e8a18d3e5af..2d9337bbefd 100644
--- a/lib/ExecutionEngine/CMakeLists.txt
+++ b/lib/ExecutionEngine/CMakeLists.txt
@@ -9,6 +9,9 @@ add_llvm_library(LLVMExecutionEngine
 
   ADDITIONAL_HEADER_DIRS
   ${LLVM_MAIN_INCLUDE_DIR}/llvm/ExecutionEngine
+
+  DEPENDS
+  intrinsics_gen
   )
 
 add_subdirectory(Interpreter)
diff --git a/lib/ExecutionEngine/ExecutionEngine.cpp b/lib/ExecutionEngine/ExecutionEngine.cpp
index d7038fd2c9a..1fc6bb5fe16 100644
--- a/lib/ExecutionEngine/ExecutionEngine.cpp
+++ b/lib/ExecutionEngine/ExecutionEngine.cpp
@@ -61,8 +61,7 @@ ExecutionEngine *(*ExecutionEngine::InterpCtor)(std::unique_ptr<Module> M,
 
 void JITEventListener::anchor() {}
 
-ExecutionEngine::ExecutionEngine(std::unique_ptr<Module> M)
-  : LazyFunctionCreator(nullptr) {
+void ExecutionEngine::Init(std::unique_ptr<Module> M) {
   CompilingLazily         = false;
   GVCompilationDisabled   = false;
   SymbolSearchingDisabled = false;
@@ -79,6 +78,16 @@ ExecutionEngine::ExecutionEngine(std::unique_ptr<Module> M)
   Modules.push_back(std::move(M));
 }
 
+ExecutionEngine::ExecutionEngine(std::unique_ptr<Module> M)
+    : DL(M->getDataLayout()), LazyFunctionCreator(nullptr) {
+  Init(std::move(M));
+}
+
+ExecutionEngine::ExecutionEngine(DataLayout DL, std::unique_ptr<Module> M)
+    : DL(std::move(DL)), LazyFunctionCreator(nullptr) {
+  Init(std::move(M));
+}
+
 ExecutionEngine::~ExecutionEngine() {
   clearAllGlobalMappings();
 }
@@ -115,7 +124,7 @@ class GVMemoryBlock : public CallbackVH {
 }  // anonymous namespace
 
 char *ExecutionEngine::getMemoryForGV(const GlobalVariable *GV) {
-  return GVMemoryBlock::Create(GV, *getDataLayout());
+  return GVMemoryBlock::Create(GV, getDataLayout());
 }
 
 void ExecutionEngine::addObjectFile(std::unique_ptr<object::ObjectFile> O) {
@@ -153,6 +162,14 @@ Function *ExecutionEngine::FindFunctionNamed(const char *FnName) {
   return nullptr;
 }
 
+GlobalVariable *ExecutionEngine::FindGlobalVariableNamed(const char *Name, bool AllowInternal) {
+  for (unsigned i = 0, e = Modules.size(); i != e; ++i) {
+    GlobalVariable *GV = Modules[i]->getGlobalVariable(Name,AllowInternal);
+    if (GV && !GV->isDeclaration())
+      return GV;
+  }
+  return nullptr;
+}
 
 uint64_t ExecutionEngineState::RemoveMapping(StringRef Name) {
   GlobalAddressMapTy::iterator I = GlobalAddressMap.find(Name);
@@ -172,10 +189,17 @@ uint64_t ExecutionEngineState::RemoveMapping(StringRef Name) {
 }
 
 std::string ExecutionEngine::getMangledName(const GlobalValue *GV) {
+  assert(GV->hasName() && "Global must have name.");
+
   MutexGuard locked(lock);
-  Mangler Mang(DL);
   SmallString<128> FullName;
-  Mang.getNameWithPrefix(FullName, GV->getName());
+
+  const DataLayout &DL =
+    GV->getParent()->getDataLayout().isDefault()
+      ? getDataLayout()
+      : GV->getParent()->getDataLayout();
+
+  Mangler::getNameWithPrefix(FullName, GV->getName(), DL);
   return FullName.str();
 }
 
@@ -318,7 +342,7 @@ void *ArgvArray::reset(LLVMContext &C, ExecutionEngine *EE,
                        const std::vector<std::string> &InputArgv) {
   Values.clear();  // Free the old contents.
   Values.reserve(InputArgv.size());
-  unsigned PtrSize = EE->getDataLayout()->getPointerSize();
+  unsigned PtrSize = EE->getDataLayout().getPointerSize();
   Array = make_unique<char[]>((InputArgv.size()+1)*PtrSize);
 
   DEBUG(dbgs() << "JIT: ARGV = " << (void*)Array.get() << "\n");
@@ -376,7 +400,7 @@ void ExecutionEngine::runStaticConstructorsDestructors(Module &module,
 
     // Execute the ctor/dtor function!
     if (Function *F = dyn_cast<Function>(FP))
-      runFunction(F, std::vector<GenericValue>());
+      runFunction(F, None);
 
     // FIXME: It is marginally lame that we just do nothing here if we see an
     // entry we don't recognize. It might not be unreasonable for the verifier
@@ -393,7 +417,7 @@ void ExecutionEngine::runStaticConstructorsDestructors(bool isDtors) {
 #ifndef NDEBUG
 /// isTargetNullPtr - Return whether the target pointer stored at Loc is null.
 static bool isTargetNullPtr(ExecutionEngine *EE, void *Loc) {
-  unsigned PtrSize = EE->getDataLayout()->getPointerSize();
+  unsigned PtrSize = EE->getDataLayout().getPointerSize();
   for (unsigned i = 0; i < PtrSize; ++i)
     if (*(i + (uint8_t*)Loc))
       return false;
@@ -438,7 +462,7 @@ int ExecutionEngine::runFunctionAsMain(Function *Fn,
       if (NumArgs > 2) {
         std::vector<std::string> EnvVars;
         for (unsigned i = 0; envp[i]; ++i)
-          EnvVars.push_back(envp[i]);
+          EnvVars.emplace_back(envp[i]);
         // Arg #2 = envp.
         GVArgs.push_back(PTOGV(CEnv.reset(Fn->getContext(), this, EnvVars)));
       }
@@ -626,8 +650,8 @@ GenericValue ExecutionEngine::getConstantValue(const Constant *C) {
     case Instruction::GetElementPtr: {
       // Compute the index
       GenericValue Result = getConstantValue(Op0);
-      APInt Offset(DL->getPointerSizeInBits(), 0);
-      cast<GEPOperator>(CE)->accumulateConstantOffset(*DL, Offset);
+      APInt Offset(DL.getPointerSizeInBits(), 0);
+      cast<GEPOperator>(CE)->accumulateConstantOffset(DL, Offset);
 
       char* tmp = (char*) Result.PointerVal;
       Result = PTOGV(tmp + Offset.getSExtValue());
@@ -714,16 +738,16 @@ GenericValue ExecutionEngine::getConstantValue(const Constant *C) {
     }
     case Instruction::PtrToInt: {
       GenericValue GV = getConstantValue(Op0);
-      uint32_t PtrWidth = DL->getTypeSizeInBits(Op0->getType());
+      uint32_t PtrWidth = DL.getTypeSizeInBits(Op0->getType());
       assert(PtrWidth <= 64 && "Bad pointer width");
       GV.IntVal = APInt(PtrWidth, uintptr_t(GV.PointerVal));
-      uint32_t IntWidth = DL->getTypeSizeInBits(CE->getType());
+      uint32_t IntWidth = DL.getTypeSizeInBits(CE->getType());
       GV.IntVal = GV.IntVal.zextOrTrunc(IntWidth);
       return GV;
     }
     case Instruction::IntToPtr: {
       GenericValue GV = getConstantValue(Op0);
-      uint32_t PtrWidth = DL->getTypeSizeInBits(CE->getType());
+      uint32_t PtrWidth = DL.getTypeSizeInBits(CE->getType());
       GV.IntVal = GV.IntVal.zextOrTrunc(PtrWidth);
       assert(GV.IntVal.getBitWidth() <= 64 && "Bad pointer width");
       GV.PointerVal = PointerTy(uintptr_t(GV.IntVal.getZExtValue()));
@@ -1025,7 +1049,7 @@ static void StoreIntToMemory(const APInt &IntVal, uint8_t *Dst,
 
 void ExecutionEngine::StoreValueToMemory(const GenericValue &Val,
                                          GenericValue *Ptr, Type *Ty) {
-  const unsigned StoreBytes = getDataLayout()->getTypeStoreSize(Ty);
+  const unsigned StoreBytes = getDataLayout().getTypeStoreSize(Ty);
 
   switch (Ty->getTypeID()) {
   default:
@@ -1065,7 +1089,7 @@ void ExecutionEngine::StoreValueToMemory(const GenericValue &Val,
     break;
   }
 
-  if (sys::IsLittleEndianHost != getDataLayout()->isLittleEndian())
+  if (sys::IsLittleEndianHost != getDataLayout().isLittleEndian())
     // Host and target are different endian - reverse the stored bytes.
     std::reverse((uint8_t*)Ptr, StoreBytes + (uint8_t*)Ptr);
 }
@@ -1102,7 +1126,7 @@ static void LoadIntFromMemory(APInt &IntVal, uint8_t *Src, unsigned LoadBytes) {
 void ExecutionEngine::LoadValueFromMemory(GenericValue &Result,
                                           GenericValue *Ptr,
                                           Type *Ty) {
-  const unsigned LoadBytes = getDataLayout()->getTypeStoreSize(Ty);
+  const unsigned LoadBytes = getDataLayout().getTypeStoreSize(Ty);
 
   switch (Ty->getTypeID()) {
   case Type::IntegerTyID:
@@ -1168,20 +1192,20 @@ void ExecutionEngine::InitializeMemory(const Constant *Init, void *Addr) {
   
   if (const ConstantVector *CP = dyn_cast<ConstantVector>(Init)) {
     unsigned ElementSize =
-      getDataLayout()->getTypeAllocSize(CP->getType()->getElementType());
+        getDataLayout().getTypeAllocSize(CP->getType()->getElementType());
     for (unsigned i = 0, e = CP->getNumOperands(); i != e; ++i)
       InitializeMemory(CP->getOperand(i), (char*)Addr+i*ElementSize);
     return;
   }
   
   if (isa<ConstantAggregateZero>(Init)) {
-    memset(Addr, 0, (size_t)getDataLayout()->getTypeAllocSize(Init->getType()));
+    memset(Addr, 0, (size_t)getDataLayout().getTypeAllocSize(Init->getType()));
     return;
   }
   
   if (const ConstantArray *CPA = dyn_cast<ConstantArray>(Init)) {
     unsigned ElementSize =
-      getDataLayout()->getTypeAllocSize(CPA->getType()->getElementType());
+        getDataLayout().getTypeAllocSize(CPA->getType()->getElementType());
     for (unsigned i = 0, e = CPA->getNumOperands(); i != e; ++i)
       InitializeMemory(CPA->getOperand(i), (char*)Addr+i*ElementSize);
     return;
@@ -1189,7 +1213,7 @@ void ExecutionEngine::InitializeMemory(const Constant *Init, void *Addr) {
   
   if (const ConstantStruct *CPS = dyn_cast<ConstantStruct>(Init)) {
     const StructLayout *SL =
-      getDataLayout()->getStructLayout(cast<StructType>(CPS->getType()));
+        getDataLayout().getStructLayout(cast<StructType>(CPS->getType()));
     for (unsigned i = 0, e = CPS->getNumOperands(); i != e; ++i)
       InitializeMemory(CPS->getOperand(i), (char*)Addr+SL->getElementOffset(i));
     return;
@@ -1334,7 +1358,7 @@ void ExecutionEngine::EmitGlobalVariable(const GlobalVariable *GV) {
     InitializeMemory(GV->getInitializer(), GA);
 
   Type *ElTy = GV->getType()->getElementType();
-  size_t GVSize = (size_t)getDataLayout()->getTypeAllocSize(ElTy);
+  size_t GVSize = (size_t)getDataLayout().getTypeAllocSize(ElTy);
   NumInitBytes += (unsigned)GVSize;
   ++NumGlobals;
 }
diff --git a/lib/ExecutionEngine/ExecutionEngineBindings.cpp b/lib/ExecutionEngine/ExecutionEngineBindings.cpp
index 22ff3114da0..f7aba4cec04 100644
--- a/lib/ExecutionEngine/ExecutionEngineBindings.cpp
+++ b/lib/ExecutionEngine/ExecutionEngineBindings.cpp
@@ -18,6 +18,7 @@
 #include "llvm/IR/DerivedTypes.h"
 #include "llvm/IR/Module.h"
 #include "llvm/Support/ErrorHandling.h"
+#include "llvm/Target/TargetOptions.h"
 #include <cstring>
 
 using namespace llvm;
@@ -28,7 +29,7 @@ using namespace llvm;
 DEFINE_SIMPLE_CONVERSION_FUNCTIONS(GenericValue, LLVMGenericValueRef)
 
 
-inline LLVMTargetMachineRef wrap(const TargetMachine *P) {
+static LLVMTargetMachineRef wrap(const TargetMachine *P) {
   return
   reinterpret_cast<LLVMTargetMachineRef>(const_cast<TargetMachine*>(P));
 }
@@ -177,11 +178,22 @@ LLVMBool LLVMCreateMCJITCompilerForModule(
   memcpy(&options, PassedOptions, SizeOfPassedOptions);
   
   TargetOptions targetOptions;
-  targetOptions.NoFramePointerElim = options.NoFramePointerElim;
   targetOptions.EnableFastISel = options.EnableFastISel;
+  std::unique_ptr<Module> Mod(unwrap(M));
+
+  if (Mod)
+    // Set function attribute "no-frame-pointer-elim" based on
+    // NoFramePointerElim.
+    for (auto &F : *Mod) {
+      auto Attrs = F.getAttributes();
+      auto Value = options.NoFramePointerElim ? "true" : "false";
+      Attrs = Attrs.addAttribute(F.getContext(), AttributeSet::FunctionIndex,
+                                 "no-frame-pointer-elim", Value);
+      F.setAttributes(Attrs);
+    }
 
   std::string Error;
-  EngineBuilder builder(std::unique_ptr<Module>(unwrap(M)));
+  EngineBuilder builder(std::move(Mod));
   builder.setEngineKind(EngineKind::JIT)
          .setErrorStr(&Error)
          .setOptLevel((CodeGenOpt::Level)options.OptLevel)
@@ -243,11 +255,8 @@ int LLVMRunFunctionAsMain(LLVMExecutionEngineRef EE, LLVMValueRef F,
                           unsigned ArgC, const char * const *ArgV,
                           const char * const *EnvP) {
   unwrap(EE)->finalizeObject();
-  
-  std::vector<std::string> ArgVec;
-  for (unsigned I = 0; I != ArgC; ++I)
-    ArgVec.push_back(ArgV[I]);
-  
+
+  std::vector<std::string> ArgVec(ArgV, ArgV + ArgC);
   return unwrap(EE)->runFunctionAsMain(unwrap<Function>(F), ArgVec, EnvP);
 }
 
@@ -309,7 +318,7 @@ void *LLVMRecompileAndRelinkFunction(LLVMExecutionEngineRef EE,
 }
 
 LLVMTargetDataRef LLVMGetExecutionEngineTargetData(LLVMExecutionEngineRef EE) {
-  return wrap(unwrap(EE)->getDataLayout());
+  return wrap(&unwrap(EE)->getDataLayout());
 }
 
 LLVMTargetMachineRef
diff --git a/lib/ExecutionEngine/IntelJITEvents/CMakeLists.txt b/lib/ExecutionEngine/IntelJITEvents/CMakeLists.txt
index 348308897dc..331d2141b0e 100644
--- a/lib/ExecutionEngine/IntelJITEvents/CMakeLists.txt
+++ b/lib/ExecutionEngine/IntelJITEvents/CMakeLists.txt
@@ -3,4 +3,6 @@ include_directories( ${CMAKE_CURRENT_SOURCE_DIR}/.. )
 add_llvm_library(LLVMIntelJITEvents
   IntelJITEventListener.cpp
   jitprofiling.c
-  )
+
+  LINK_LIBS pthread ${CMAKE_DL_LIBS}
+)
diff --git a/lib/ExecutionEngine/IntelJITEvents/IntelJITEventListener.cpp b/lib/ExecutionEngine/IntelJITEvents/IntelJITEventListener.cpp
index 41359000fb0..a131763193c 100644
--- a/lib/ExecutionEngine/IntelJITEvents/IntelJITEventListener.cpp
+++ b/lib/ExecutionEngine/IntelJITEvents/IntelJITEventListener.cpp
@@ -16,13 +16,15 @@
 #include "IntelJITEventsWrapper.h"
 #include "llvm/ADT/DenseMap.h"
 #include "llvm/CodeGen/MachineFunction.h"
-#include "llvm/DebugInfo/DWARF/DIContext.h"
+#include "llvm/DebugInfo/DIContext.h"
+#include "llvm/DebugInfo/DWARF/DWARFContext.h"
 #include "llvm/ExecutionEngine/JITEventListener.h"
 #include "llvm/IR/DebugInfo.h"
 #include "llvm/IR/Function.h"
 #include "llvm/IR/Metadata.h"
 #include "llvm/IR/ValueHandle.h"
 #include "llvm/Object/ObjectFile.h"
+#include "llvm/Object/SymbolSize.h"
 #include "llvm/Support/Debug.h"
 #include "llvm/Support/Errno.h"
 #include "llvm/Support/raw_ostream.h"
@@ -102,76 +104,68 @@ void IntelJITEventListener::NotifyObjectEmitted(
 
   // Get the address of the object image for use as a unique identifier
   const void* ObjData = DebugObj.getData().data();
-  DIContext* Context = DIContext::getDWARFContext(DebugObj);
+  DIContext* Context = new DWARFContextInMemory(DebugObj);
   MethodAddressVector Functions;
 
   // Use symbol info to iterate functions in the object.
-  for (symbol_iterator I = DebugObj.symbol_begin(),
-                       E = DebugObj.symbol_end();
-                        I != E;
-                        ++I) {
+  for (const std::pair<SymbolRef, uint64_t> &P : computeSymbolSizes(DebugObj)) {
+    SymbolRef Sym = P.first;
     std::vector<LineNumberInfo> LineInfo;
     std::string SourceFileName;
 
-    SymbolRef::Type SymType;
-    if (I->getType(SymType)) continue;
-    if (SymType == SymbolRef::ST_Function) {
-      StringRef  Name;
-      uint64_t   Addr;
-      uint64_t   Size;
-      if (I->getName(Name)) continue;
-      if (I->getAddress(Addr)) continue;
-      if (I->getSize(Size)) continue;
-
-      // Record this address in a local vector
-      Functions.push_back((void*)Addr);
-
-      // Build the function loaded notification message
-      iJIT_Method_Load FunctionMessage = FunctionDescToIntelJITFormat(*Wrapper,
-                                           Name.data(),
-                                           Addr,
-                                           Size);
-      if (Context) {
-        DILineInfoTable  Lines = Context->getLineInfoForAddressRange(Addr, Size);
-        DILineInfoTable::iterator  Begin = Lines.begin();
-        DILineInfoTable::iterator  End = Lines.end();
-        for (DILineInfoTable::iterator It = Begin; It != End; ++It) {
-          LineInfo.push_back(DILineInfoToIntelJITFormat((uintptr_t)Addr,
-                                                        It->first,
-                                                        It->second));
-        }
-        if (LineInfo.size() == 0) {
-          FunctionMessage.source_file_name = 0;
-          FunctionMessage.line_number_size = 0;
-          FunctionMessage.line_number_table = 0;
-        } else {
-          // Source line information for the address range is provided as 
-          // a code offset for the start of the corresponding sub-range and
-          // a source line. JIT API treats offsets in LineNumberInfo structures
-          // as the end of the corresponding code region. The start of the code
-          // is taken from the previous element. Need to shift the elements.
-
-          LineNumberInfo last = LineInfo.back();
-          last.Offset = FunctionMessage.method_size;
-          LineInfo.push_back(last);
-          for (size_t i = LineInfo.size() - 2; i > 0; --i)
-            LineInfo[i].LineNumber = LineInfo[i - 1].LineNumber;
-
-          SourceFileName = Lines.front().second.FileName;
-          FunctionMessage.source_file_name = const_cast<char *>(SourceFileName.c_str());
-          FunctionMessage.line_number_size = LineInfo.size();
-          FunctionMessage.line_number_table = &*LineInfo.begin();
-        }
-      } else {
-        FunctionMessage.source_file_name = 0;
-        FunctionMessage.line_number_size = 0;
-        FunctionMessage.line_number_table = 0;
-      }
-
-      Wrapper->iJIT_NotifyEvent(iJVM_EVENT_TYPE_METHOD_LOAD_FINISHED,
-                                &FunctionMessage);
-      MethodIDs[(void*)Addr] = FunctionMessage.method_id;
+    if (Sym.getType() != SymbolRef::ST_Function)
+      continue;
+
+    ErrorOr<StringRef> Name = Sym.getName();
+    if (!Name)
+      continue;
+
+    ErrorOr<uint64_t> AddrOrErr = Sym.getAddress();
+    if (AddrOrErr.getError())
+      continue;
+    uint64_t Addr = *AddrOrErr;
+    uint64_t Size = P.second;
+
+    // Record this address in a local vector
+    Functions.push_back((void*)Addr);
+
+    // Build the function loaded notification message
+    iJIT_Method_Load FunctionMessage =
+      FunctionDescToIntelJITFormat(*Wrapper, Name->data(), Addr, Size);
+    DILineInfoTable Lines = Context->getLineInfoForAddressRange(Addr, Size);
+    DILineInfoTable::iterator Begin = Lines.begin();
+    DILineInfoTable::iterator End = Lines.end();
+    for (DILineInfoTable::iterator It = Begin; It != End; ++It) {
+      LineInfo.push_back(
+          DILineInfoToIntelJITFormat((uintptr_t)Addr, It->first, It->second));
     }
+    if (LineInfo.size() == 0) {
+      FunctionMessage.source_file_name = 0;
+      FunctionMessage.line_number_size = 0;
+      FunctionMessage.line_number_table = 0;
+    } else {
+      // Source line information for the address range is provided as
+      // a code offset for the start of the corresponding sub-range and
+      // a source line. JIT API treats offsets in LineNumberInfo structures
+      // as the end of the corresponding code region. The start of the code
+      // is taken from the previous element. Need to shift the elements.
+
+      LineNumberInfo last = LineInfo.back();
+      last.Offset = FunctionMessage.method_size;
+      LineInfo.push_back(last);
+      for (size_t i = LineInfo.size() - 2; i > 0; --i)
+        LineInfo[i].LineNumber = LineInfo[i - 1].LineNumber;
+
+      SourceFileName = Lines.front().second.FileName;
+      FunctionMessage.source_file_name =
+        const_cast<char *>(SourceFileName.c_str());
+      FunctionMessage.line_number_size = LineInfo.size();
+      FunctionMessage.line_number_table = &*LineInfo.begin();
+    }
+
+    Wrapper->iJIT_NotifyEvent(iJVM_EVENT_TYPE_METHOD_LOAD_FINISHED,
+                              &FunctionMessage);
+    MethodIDs[(void*)Addr] = FunctionMessage.method_id;
   }
 
   // To support object unload notification, we need to keep a list of
diff --git a/lib/ExecutionEngine/IntelJITEvents/LLVMBuild.txt b/lib/ExecutionEngine/IntelJITEvents/LLVMBuild.txt
index 1247cbd9493..afea3ecccda 100644
--- a/lib/ExecutionEngine/IntelJITEvents/LLVMBuild.txt
+++ b/lib/ExecutionEngine/IntelJITEvents/LLVMBuild.txt
@@ -21,4 +21,4 @@
 type = OptionalLibrary
 name = IntelJITEvents
 parent = ExecutionEngine
-required_libraries = Core DebugInfoDWARF Support
+required_libraries = Core DebugInfoDWARF Support Object ExecutionEngine
diff --git a/lib/ExecutionEngine/Interpreter/Execution.cpp b/lib/ExecutionEngine/Interpreter/Execution.cpp
index a26740b0da8..53beed87a41 100644
--- a/lib/ExecutionEngine/Interpreter/Execution.cpp
+++ b/lib/ExecutionEngine/Interpreter/Execution.cpp
@@ -968,7 +968,7 @@ void Interpreter::visitAllocaInst(AllocaInst &I) {
   unsigned NumElements = 
     getOperandValue(I.getOperand(0), SF).IntVal.getZExtValue();
 
-  unsigned TypeSize = (size_t)TD.getTypeAllocSize(Ty);
+  unsigned TypeSize = (size_t)getDataLayout().getTypeAllocSize(Ty);
 
   // Avoid malloc-ing zero bytes, use max()...
   unsigned MemToAlloc = std::max(1U, NumElements * TypeSize);
@@ -1000,7 +1000,7 @@ GenericValue Interpreter::executeGEPOperation(Value *Ptr, gep_type_iterator I,
 
   for (; I != E; ++I) {
     if (StructType *STy = dyn_cast<StructType>(*I)) {
-      const StructLayout *SLO = TD.getStructLayout(STy);
+      const StructLayout *SLO = getDataLayout().getStructLayout(STy);
 
       const ConstantInt *CPU = cast<ConstantInt>(I.getOperand());
       unsigned Index = unsigned(CPU->getZExtValue());
@@ -1020,7 +1020,7 @@ GenericValue Interpreter::executeGEPOperation(Value *Ptr, gep_type_iterator I,
         assert(BitWidth == 64 && "Invalid index type for getelementptr");
         Idx = (int64_t)IdxGV.IntVal.getZExtValue();
       }
-      Total += TD.getTypeAllocSize(ST->getElementType())*Idx;
+      Total += getDataLayout().getTypeAllocSize(ST->getElementType()) * Idx;
     }
   }
 
@@ -1477,7 +1477,7 @@ GenericValue Interpreter::executeIntToPtrInst(Value *SrcVal, Type *DstTy,
   GenericValue Dest, Src = getOperandValue(SrcVal, SF);
   assert(DstTy->isPointerTy() && "Invalid PtrToInt instruction");
 
-  uint32_t PtrSize = TD.getPointerSizeInBits();
+  uint32_t PtrSize = getDataLayout().getPointerSizeInBits();
   if (PtrSize != Src.IntVal.getBitWidth())
     Src.IntVal = Src.IntVal.zextOrTrunc(PtrSize);
 
@@ -1497,7 +1497,7 @@ GenericValue Interpreter::executeBitCastInst(Value *SrcVal, Type *DstTy,
       (DstTy->getTypeID() == Type::VectorTyID)) {
     // vector src bitcast to vector dst or vector src bitcast to scalar dst or
     // scalar src bitcast to vector dst
-    bool isLittleEndian = TD.isLittleEndian();
+    bool isLittleEndian = getDataLayout().isLittleEndian();
     GenericValue TempDst, TempSrc, SrcVec;
     const Type *SrcElemTy;
     const Type *DstElemTy;
@@ -2073,13 +2073,12 @@ GenericValue Interpreter::getOperandValue(Value *V, ExecutionContext &SF) {
 //===----------------------------------------------------------------------===//
 // callFunction - Execute the specified function...
 //
-void Interpreter::callFunction(Function *F,
-                               const std::vector<GenericValue> &ArgVals) {
+void Interpreter::callFunction(Function *F, ArrayRef<GenericValue> ArgVals) {
   assert((ECStack.empty() || !ECStack.back().Caller.getInstruction() ||
           ECStack.back().Caller.arg_size() == ArgVals.size()) &&
          "Incorrect number of arguments passed into function call!");
   // Make a new stack frame... and fill it in.
-  ECStack.push_back(ExecutionContext());
+  ECStack.emplace_back();
   ExecutionContext &StackFrame = ECStack.back();
   StackFrame.CurFunction = F;
 
diff --git a/lib/ExecutionEngine/Interpreter/ExternalFunctions.cpp b/lib/ExecutionEngine/Interpreter/ExternalFunctions.cpp
index e2fe0651c7e..441f0eb8572 100644
--- a/lib/ExecutionEngine/Interpreter/ExternalFunctions.cpp
+++ b/lib/ExecutionEngine/Interpreter/ExternalFunctions.cpp
@@ -49,8 +49,7 @@ using namespace llvm;
 
 static ManagedStatic<sys::Mutex> FunctionsLock;
 
-typedef GenericValue (*ExFunc)(FunctionType *,
-                               const std::vector<GenericValue> &);
+typedef GenericValue (*ExFunc)(FunctionType *, ArrayRef<GenericValue>);
 static ManagedStatic<std::map<const Function *, ExFunc> > ExportedFunctions;
 static ManagedStatic<std::map<std::string, ExFunc> > FuncNames;
 
@@ -178,9 +177,8 @@ static void *ffiValueFor(Type *Ty, const GenericValue &AV,
   return NULL;
 }
 
-static bool ffiInvoke(RawFunc Fn, Function *F,
-                      const std::vector<GenericValue> &ArgVals,
-                      const DataLayout *TD, GenericValue &Result) {
+static bool ffiInvoke(RawFunc Fn, Function *F, ArrayRef<GenericValue> ArgVals,
+                      const DataLayout &TD, GenericValue &Result) {
   ffi_cif cif;
   FunctionType *FTy = F->getFunctionType();
   const unsigned NumArgs = F->arg_size();
@@ -200,7 +198,7 @@ static bool ffiInvoke(RawFunc Fn, Function *F,
     const unsigned ArgNo = A->getArgNo();
     Type *ArgTy = FTy->getParamType(ArgNo);
     args[ArgNo] = ffiTypeFor(ArgTy);
-    ArgBytes += TD->getTypeStoreSize(ArgTy);
+    ArgBytes += TD.getTypeStoreSize(ArgTy);
   }
 
   SmallVector<uint8_t, 128> ArgData;
@@ -212,7 +210,7 @@ static bool ffiInvoke(RawFunc Fn, Function *F,
     const unsigned ArgNo = A->getArgNo();
     Type *ArgTy = FTy->getParamType(ArgNo);
     values[ArgNo] = ffiValueFor(ArgTy, ArgVals[ArgNo], ArgDataPtr);
-    ArgDataPtr += TD->getTypeStoreSize(ArgTy);
+    ArgDataPtr += TD.getTypeStoreSize(ArgTy);
   }
 
   Type *RetTy = FTy->getReturnType();
@@ -221,7 +219,7 @@ static bool ffiInvoke(RawFunc Fn, Function *F,
   if (ffi_prep_cif(&cif, FFI_DEFAULT_ABI, NumArgs, rtype, &args[0]) == FFI_OK) {
     SmallVector<uint8_t, 128> ret;
     if (RetTy->getTypeID() != Type::VoidTyID)
-      ret.resize(TD->getTypeStoreSize(RetTy));
+      ret.resize(TD.getTypeStoreSize(RetTy));
     ffi_call(&cif, Fn, ret.data(), values.data());
     switch (RetTy->getTypeID()) {
       case Type::IntegerTyID:
@@ -245,7 +243,7 @@ static bool ffiInvoke(RawFunc Fn, Function *F,
 #endif // USE_LIBFFI
 
 GenericValue Interpreter::callExternalFunction(Function *F,
-                                     const std::vector<GenericValue> &ArgVals) {
+                                               ArrayRef<GenericValue> ArgVals) {
   TheInterpreter = this;
 
   unique_lock<sys::Mutex> Guard(*FunctionsLock);
@@ -298,9 +296,8 @@ GenericValue Interpreter::callExternalFunction(Function *F,
 //
 
 // void atexit(Function*)
-static
-GenericValue lle_X_atexit(FunctionType *FT,
-                          const std::vector<GenericValue> &Args) {
+static GenericValue lle_X_atexit(FunctionType *FT,
+                                 ArrayRef<GenericValue> Args) {
   assert(Args.size() == 1);
   TheInterpreter->addAtExitHandler((Function*)GVTOP(Args[0]));
   GenericValue GV;
@@ -309,17 +306,13 @@ GenericValue lle_X_atexit(FunctionType *FT,
 }
 
 // void exit(int)
-static
-GenericValue lle_X_exit(FunctionType *FT,
-                        const std::vector<GenericValue> &Args) {
+static GenericValue lle_X_exit(FunctionType *FT, ArrayRef<GenericValue> Args) {
   TheInterpreter->exitCalled(Args[0]);
   return GenericValue();
 }
 
 // void abort(void)
-static
-GenericValue lle_X_abort(FunctionType *FT,
-                         const std::vector<GenericValue> &Args) {
+static GenericValue lle_X_abort(FunctionType *FT, ArrayRef<GenericValue> Args) {
   //FIXME: should we report or raise here?
   //report_fatal_error("Interpreted program raised SIGABRT");
   raise (SIGABRT);
@@ -328,9 +321,8 @@ GenericValue lle_X_abort(FunctionType *FT,
 
 // int sprintf(char *, const char *, ...) - a very rough implementation to make
 // output useful.
-static
-GenericValue lle_X_sprintf(FunctionType *FT,
-                           const std::vector<GenericValue> &Args) {
+static GenericValue lle_X_sprintf(FunctionType *FT,
+                                  ArrayRef<GenericValue> Args) {
   char *OutputBuffer = (char *)GVTOP(Args[0]);
   const char *FmtStr = (const char *)GVTOP(Args[1]);
   unsigned ArgNo = 2;
@@ -376,7 +368,7 @@ GenericValue lle_X_sprintf(FunctionType *FT,
       case 'x': case 'X':
         if (HowLong >= 1) {
           if (HowLong == 1 &&
-              TheInterpreter->getDataLayout()->getPointerSizeInBits() == 64 &&
+              TheInterpreter->getDataLayout().getPointerSizeInBits() == 64 &&
               sizeof(long) < sizeof(int64_t)) {
             // Make sure we use %lld with a 64 bit argument because we might be
             // compiling LLI on a 32 bit compiler.
@@ -411,9 +403,8 @@ GenericValue lle_X_sprintf(FunctionType *FT,
 
 // int printf(const char *, ...) - a very rough implementation to make output
 // useful.
-static
-GenericValue lle_X_printf(FunctionType *FT,
-                          const std::vector<GenericValue> &Args) {
+static GenericValue lle_X_printf(FunctionType *FT,
+                                 ArrayRef<GenericValue> Args) {
   char Buffer[10000];
   std::vector<GenericValue> NewArgs;
   NewArgs.push_back(PTOGV((void*)&Buffer[0]));
@@ -424,9 +415,8 @@ GenericValue lle_X_printf(FunctionType *FT,
 }
 
 // int sscanf(const char *format, ...);
-static
-GenericValue lle_X_sscanf(FunctionType *FT,
-                          const std::vector<GenericValue> &args) {
+static GenericValue lle_X_sscanf(FunctionType *FT,
+                                 ArrayRef<GenericValue> args) {
   assert(args.size() < 10 && "Only handle up to 10 args to sscanf right now!");
 
   char *Args[10];
@@ -440,9 +430,7 @@ GenericValue lle_X_sscanf(FunctionType *FT,
 }
 
 // int scanf(const char *format, ...);
-static
-GenericValue lle_X_scanf(FunctionType *FT,
-                         const std::vector<GenericValue> &args) {
+static GenericValue lle_X_scanf(FunctionType *FT, ArrayRef<GenericValue> args) {
   assert(args.size() < 10 && "Only handle up to 10 args to scanf right now!");
 
   char *Args[10];
@@ -457,9 +445,8 @@ GenericValue lle_X_scanf(FunctionType *FT,
 
 // int fprintf(FILE *, const char *, ...) - a very rough implementation to make
 // output useful.
-static
-GenericValue lle_X_fprintf(FunctionType *FT,
-                           const std::vector<GenericValue> &Args) {
+static GenericValue lle_X_fprintf(FunctionType *FT,
+                                  ArrayRef<GenericValue> Args) {
   assert(Args.size() >= 2);
   char Buffer[10000];
   std::vector<GenericValue> NewArgs;
@@ -472,7 +459,7 @@ GenericValue lle_X_fprintf(FunctionType *FT,
 }
 
 static GenericValue lle_X_memset(FunctionType *FT,
-                                 const std::vector<GenericValue> &Args) {
+                                 ArrayRef<GenericValue> Args) {
   int val = (int)Args[1].IntVal.getSExtValue();
   size_t len = (size_t)Args[2].IntVal.getZExtValue();
   memset((void *)GVTOP(Args[0]), val, len);
@@ -484,7 +471,7 @@ static GenericValue lle_X_memset(FunctionType *FT,
 }
 
 static GenericValue lle_X_memcpy(FunctionType *FT,
-                                 const std::vector<GenericValue> &Args) {
+                                 ArrayRef<GenericValue> Args) {
   memcpy(GVTOP(Args[0]), GVTOP(Args[1]),
          (size_t)(Args[2].IntVal.getLimitedValue()));
 
diff --git a/lib/ExecutionEngine/Interpreter/Interpreter.cpp b/lib/ExecutionEngine/Interpreter/Interpreter.cpp
index 8562981b629..8cb9d45bb68 100644
--- a/lib/ExecutionEngine/Interpreter/Interpreter.cpp
+++ b/lib/ExecutionEngine/Interpreter/Interpreter.cpp
@@ -49,16 +49,15 @@ ExecutionEngine *Interpreter::create(std::unique_ptr<Module> M,
 // Interpreter ctor - Initialize stuff
 //
 Interpreter::Interpreter(std::unique_ptr<Module> M)
-  : ExecutionEngine(std::move(M)), TD(Modules.back().get()) {
+    : ExecutionEngine(std::move(M)) {
 
   memset(&ExitValue.Untyped, 0, sizeof(ExitValue.Untyped));
-  setDataLayout(&TD);
   // Initialize the "backend"
   initializeExecutionEngine();
   initializeExternalFunctions();
   emitGlobals();
 
-  IL = new IntrinsicLowering(TD);
+  IL = new IntrinsicLowering(getDataLayout());
 }
 
 Interpreter::~Interpreter() {
@@ -67,7 +66,7 @@ Interpreter::~Interpreter() {
 
 void Interpreter::runAtExitHandlers () {
   while (!AtExitHandlers.empty()) {
-    callFunction(AtExitHandlers.back(), std::vector<GenericValue>());
+    callFunction(AtExitHandlers.back(), None);
     AtExitHandlers.pop_back();
     run();
   }
@@ -75,9 +74,8 @@ void Interpreter::runAtExitHandlers () {
 
 /// run - Start execution with the specified function and arguments.
 ///
-GenericValue
-Interpreter::runFunction(Function *F,
-                         const std::vector<GenericValue> &ArgValues) {
+GenericValue Interpreter::runFunction(Function *F,
+                                      ArrayRef<GenericValue> ArgValues) {
   assert (F && "Function *F was null at entry to run()");
 
   // Try extra hard not to pass extra args to a function that isn't
@@ -87,10 +85,9 @@ Interpreter::runFunction(Function *F,
   // parameters than it is declared to take. This does not attempt to
   // take into account gratuitous differences in declared types,
   // though.
-  std::vector<GenericValue> ActualArgs;
-  const unsigned ArgCount = F->getFunctionType()->getNumParams();
-  for (unsigned i = 0; i < ArgCount; ++i)
-    ActualArgs.push_back(ArgValues[i]);
+  const size_t ArgCount = F->getFunctionType()->getNumParams();
+  ArrayRef<GenericValue> ActualArgs =
+      ArgValues.slice(0, std::min(ArgValues.size(), ArgCount));
 
   // Set up the function call.
   callFunction(F, ActualArgs);
diff --git a/lib/ExecutionEngine/Interpreter/Interpreter.h b/lib/ExecutionEngine/Interpreter/Interpreter.h
index 0dc0463903d..bd813ac4900 100644
--- a/lib/ExecutionEngine/Interpreter/Interpreter.h
+++ b/lib/ExecutionEngine/Interpreter/Interpreter.h
@@ -95,7 +95,6 @@ struct ExecutionContext {
 //
 class Interpreter : public ExecutionEngine, public InstVisitor<Interpreter> {
   GenericValue ExitValue;          // The return value of the called function
-  DataLayout TD;
   IntrinsicLowering *IL;
 
   // The runtime stack of executing code.  The top of the stack is the current
@@ -127,7 +126,7 @@ class Interpreter : public ExecutionEngine, public InstVisitor<Interpreter> {
   /// run - Start execution with the specified function and arguments.
   ///
   GenericValue runFunction(Function *F,
-                           const std::vector<GenericValue> &ArgValues) override;
+                           ArrayRef<GenericValue> ArgValues) override;
 
   void *getPointerToNamedFunction(StringRef Name,
                                   bool AbortOnFailure = true) override {
@@ -137,7 +136,7 @@ class Interpreter : public ExecutionEngine, public InstVisitor<Interpreter> {
 
   // Methods used to execute code:
   // Place a call on the stack
-  void callFunction(Function *F, const std::vector<GenericValue> &ArgVals);
+  void callFunction(Function *F, ArrayRef<GenericValue> ArgVals);
   void run();                // Execute instructions until nothing left to do
 
   // Opcode Implementations
@@ -194,7 +193,7 @@ class Interpreter : public ExecutionEngine, public InstVisitor<Interpreter> {
   }
 
   GenericValue callExternalFunction(Function *F,
-                                    const std::vector<GenericValue> &ArgVals);
+                                    ArrayRef<GenericValue> ArgVals);
   void exitCalled(GenericValue GV);
 
   void addAtExitHandler(Function *F) {
diff --git a/lib/ExecutionEngine/LLVMBuild.txt b/lib/ExecutionEngine/LLVMBuild.txt
index 8fdda9a6a39..9d29a41f504 100644
--- a/lib/ExecutionEngine/LLVMBuild.txt
+++ b/lib/ExecutionEngine/LLVMBuild.txt
@@ -22,4 +22,4 @@ subdirectories = Interpreter MCJIT RuntimeDyld IntelJITEvents OProfileJIT Orc
 type = Library
 name = ExecutionEngine
 parent = Libraries
-required_libraries = Core MC Object Support RuntimeDyld
+required_libraries = Core MC Object RuntimeDyld Support Target
diff --git a/lib/ExecutionEngine/MCJIT/CMakeLists.txt b/lib/ExecutionEngine/MCJIT/CMakeLists.txt
index 2911a507722..b1e2bc3d635 100644
--- a/lib/ExecutionEngine/MCJIT/CMakeLists.txt
+++ b/lib/ExecutionEngine/MCJIT/CMakeLists.txt
@@ -1,3 +1,6 @@
 add_llvm_library(LLVMMCJIT
   MCJIT.cpp
+
+  DEPENDS
+  intrinsics_gen
   )
diff --git a/lib/ExecutionEngine/MCJIT/MCJIT.cpp b/lib/ExecutionEngine/MCJIT/MCJIT.cpp
index 7e37afe2056..c201f39a375 100644
--- a/lib/ExecutionEngine/MCJIT/MCJIT.cpp
+++ b/lib/ExecutionEngine/MCJIT/MCJIT.cpp
@@ -65,12 +65,13 @@ MCJIT::createJIT(std::unique_ptr<Module> M,
                    std::move(Resolver));
 }
 
-MCJIT::MCJIT(std::unique_ptr<Module> M, std::unique_ptr<TargetMachine> tm,
+MCJIT::MCJIT(std::unique_ptr<Module> M, std::unique_ptr<TargetMachine> TM,
              std::shared_ptr<MCJITMemoryManager> MemMgr,
              std::shared_ptr<RuntimeDyld::SymbolResolver> Resolver)
-    : ExecutionEngine(std::move(M)), TM(std::move(tm)), Ctx(nullptr),
-      MemMgr(std::move(MemMgr)), Resolver(*this, std::move(Resolver)),
-      Dyld(*this->MemMgr, this->Resolver), ObjCache(nullptr) {
+    : ExecutionEngine(TM->createDataLayout(), std::move(M)), TM(std::move(TM)),
+      Ctx(nullptr), MemMgr(std::move(MemMgr)),
+      Resolver(*this, std::move(Resolver)), Dyld(*this->MemMgr, this->Resolver),
+      ObjCache(nullptr) {
   // FIXME: We are managing our modules, so we do not want the base class
   // ExecutionEngine to manage them as well. To avoid double destruction
   // of the first (and only) module added in ExecutionEngine constructor
@@ -85,7 +86,6 @@ MCJIT::MCJIT(std::unique_ptr<Module> M, std::unique_ptr<TargetMachine> tm,
   Modules.clear();
 
   OwnedModules.addModule(std::move(First));
-  setDataLayout(TM->getDataLayout());
   RegisterJITEventListener(JITEventListener::createGDBRegistrationListener());
 }
 
@@ -147,8 +147,6 @@ std::unique_ptr<MemoryBuffer> MCJIT::emitObject(Module *M) {
 
   legacy::PassManager PM;
 
-  M->setDataLayout(*TM->getDataLayout());
-
   // The RuntimeDyld will take ownership of this shortly
   SmallVector<char, 4096> ObjBufferSV;
   raw_svector_ostream ObjStream(ObjBufferSV);
@@ -195,6 +193,12 @@ void MCJIT::generateCodeForModule(Module *M) {
   if (ObjCache)
     ObjectToLoad = ObjCache->getObject(M);
 
+  if (M->getDataLayout().isDefault()) {
+    M->setDataLayout(getDataLayout());
+  } else {
+    assert(M->getDataLayout() == getDataLayout() && "DataLayout Mismatch");
+  }
+
   // If the cache did not contain a suitable object, compile the object
   if (!ObjectToLoad) {
     ObjectToLoad = emitObject(M);
@@ -264,9 +268,14 @@ void MCJIT::finalizeModule(Module *M) {
 }
 
 RuntimeDyld::SymbolInfo MCJIT::findExistingSymbol(const std::string &Name) {
-  Mangler Mang(TM->getDataLayout());
   SmallString<128> FullName;
-  Mang.getNameWithPrefix(FullName, Name);
+  Mangler::getNameWithPrefix(FullName, Name, getDataLayout());
+
+  if (void *Addr = getPointerToGlobalIfAvailable(FullName))
+    return RuntimeDyld::SymbolInfo(static_cast<uint64_t>(
+                                     reinterpret_cast<uintptr_t>(Addr)),
+                                   JITSymbolFlags::Exported);
+
   return Dyld.getSymbol(FullName);
 }
 
@@ -369,7 +378,7 @@ uint64_t MCJIT::getFunctionAddress(const std::string &Name) {
 void *MCJIT::getPointerToFunction(Function *F) {
   MutexGuard locked(lock);
 
-  Mangler Mang(TM->getDataLayout());
+  Mangler Mang;
   SmallString<128> Name;
   TM->getNameWithPrefix(Name, F, Mang);
 
@@ -429,6 +438,19 @@ Function *MCJIT::FindFunctionNamedInModulePtrSet(const char *FnName,
   return nullptr;
 }
 
+GlobalVariable *MCJIT::FindGlobalVariableNamedInModulePtrSet(const char *Name,
+                                                             bool AllowInternal,
+                                                             ModulePtrSet::iterator I,
+                                                             ModulePtrSet::iterator E) {
+  for (; I != E; ++I) {
+    GlobalVariable *GV = (*I)->getGlobalVariable(Name, AllowInternal);
+    if (GV && !GV->isDeclaration())
+      return GV;
+  }
+  return nullptr;
+}
+
+
 Function *MCJIT::FindFunctionNamed(const char *FnName) {
   Function *F = FindFunctionNamedInModulePtrSet(
       FnName, OwnedModules.begin_added(), OwnedModules.end_added());
@@ -441,8 +463,19 @@ Function *MCJIT::FindFunctionNamed(const char *FnName) {
   return F;
 }
 
-GenericValue MCJIT::runFunction(Function *F,
-                                const std::vector<GenericValue> &ArgValues) {
+GlobalVariable *MCJIT::FindGlobalVariableNamed(const char *Name, bool AllowInternal) {
+  GlobalVariable *GV = FindGlobalVariableNamedInModulePtrSet(
+      Name, AllowInternal, OwnedModules.begin_added(), OwnedModules.end_added());
+  if (!GV)
+    GV = FindGlobalVariableNamedInModulePtrSet(Name, AllowInternal, OwnedModules.begin_loaded(),
+                                        OwnedModules.end_loaded());
+  if (!GV)
+    GV = FindGlobalVariableNamedInModulePtrSet(Name, AllowInternal, OwnedModules.begin_finalized(),
+                                        OwnedModules.end_finalized());
+  return GV;
+}
+
+GenericValue MCJIT::runFunction(Function *F, ArrayRef<GenericValue> ArgValues) {
   assert(F && "Function *F was null at entry to run()");
 
   void *FPtr = getPointerToFunction(F);
diff --git a/lib/ExecutionEngine/MCJIT/MCJIT.h b/lib/ExecutionEngine/MCJIT/MCJIT.h
index 59e99498f9a..a45173c2da8 100644
--- a/lib/ExecutionEngine/MCJIT/MCJIT.h
+++ b/lib/ExecutionEngine/MCJIT/MCJIT.h
@@ -200,6 +200,11 @@ class MCJIT : public ExecutionEngine {
                                             ModulePtrSet::iterator I,
                                             ModulePtrSet::iterator E);
 
+  GlobalVariable *FindGlobalVariableNamedInModulePtrSet(const char *Name,
+                                                        bool AllowInternal,
+                                                        ModulePtrSet::iterator I,
+                                                        ModulePtrSet::iterator E);
+
   void runStaticConstructorsDestructorsInModulePtrSet(bool isDtors,
                                                       ModulePtrSet::iterator I,
                                                       ModulePtrSet::iterator E);
@@ -215,10 +220,15 @@ class MCJIT : public ExecutionEngine {
   void addArchive(object::OwningBinary<object::Archive> O) override;
   bool removeModule(Module *M) override;
 
-  /// FindFunctionNamed - Search all of the active modules to find the one that
+  /// FindFunctionNamed - Search all of the active modules to find the function that
   /// defines FnName.  This is very slow operation and shouldn't be used for
   /// general code.
-  Function *FindFunctionNamed(const char *FnName) override;
+  virtual Function *FindFunctionNamed(const char *FnName) override;
+
+  /// FindGlobalVariableNamed - Search all of the active modules to find the global variable
+  /// that defines Name.  This is very slow operation and shouldn't be used for
+  /// general code.
+  virtual GlobalVariable *FindGlobalVariableNamed(const char *Name, bool AllowInternal = false) override;
 
   /// Sets the object manager that MCJIT should use to avoid compilation.
   void setObjectCache(ObjectCache *manager) override;
@@ -251,7 +261,7 @@ class MCJIT : public ExecutionEngine {
   void *getPointerToFunction(Function *F) override;
 
   GenericValue runFunction(Function *F,
-                           const std::vector<GenericValue> &ArgValues) override;
+                           ArrayRef<GenericValue> ArgValues) override;
 
   /// getPointerToNamedFunction - This method returns the address of the
   /// specified function by using the dlsym function call.  As such it is only
diff --git a/lib/ExecutionEngine/OProfileJIT/LLVMBuild.txt b/lib/ExecutionEngine/OProfileJIT/LLVMBuild.txt
index e30516eb3b0..7d5550046a5 100644
--- a/lib/ExecutionEngine/OProfileJIT/LLVMBuild.txt
+++ b/lib/ExecutionEngine/OProfileJIT/LLVMBuild.txt
@@ -21,3 +21,4 @@
 type = OptionalLibrary
 name = OProfileJIT
 parent = ExecutionEngine
+required_libraries = Support Object ExecutionEngine
diff --git a/lib/ExecutionEngine/OProfileJIT/OProfileJITEventListener.cpp b/lib/ExecutionEngine/OProfileJIT/OProfileJITEventListener.cpp
index 23e766206e2..324d0711870 100644
--- a/lib/ExecutionEngine/OProfileJIT/OProfileJITEventListener.cpp
+++ b/lib/ExecutionEngine/OProfileJIT/OProfileJITEventListener.cpp
@@ -20,6 +20,7 @@
 #include "llvm/IR/DebugInfo.h"
 #include "llvm/IR/Function.h"
 #include "llvm/Object/ObjectFile.h"
+#include "llvm/Object/SymbolSize.h"
 #include "llvm/Support/Debug.h"
 #include "llvm/Support/Errno.h"
 #include "llvm/Support/raw_ostream.h"
@@ -85,27 +86,29 @@ void OProfileJITEventListener::NotifyObjectEmitted(
   const ObjectFile &DebugObj = *DebugObjOwner.getBinary();
 
   // Use symbol info to iterate functions in the object.
-  for (symbol_iterator I = DebugObj.symbol_begin(), E = DebugObj.symbol_end();
-       I != E; ++I) {
-    SymbolRef::Type SymType;
-    if (I->getType(SymType)) continue;
-    if (SymType == SymbolRef::ST_Function) {
-      StringRef  Name;
-      uint64_t   Addr;
-      uint64_t   Size;
-      if (I->getName(Name)) continue;
-      if (I->getAddress(Addr)) continue;
-      if (I->getSize(Size)) continue;
-
-      if (Wrapper->op_write_native_code(Name.data(), Addr, (void*)Addr, Size)
-                        == -1) {
-        DEBUG(dbgs() << "Failed to tell OProfile about native function "
-          << Name << " at ["
-          << (void*)Addr << "-" << ((char*)Addr + Size) << "]\n");
-        continue;
-      }
-      // TODO: support line number info (similar to IntelJITEventListener.cpp)
+  for (const std::pair<SymbolRef, uint64_t> &P : computeSymbolSizes(DebugObj)) {
+    SymbolRef Sym = P.first;
+    if (Sym.getType() != SymbolRef::ST_Function)
+      continue;
+
+    ErrorOr<StringRef> NameOrErr = Sym.getName();
+    if (NameOrErr.getError())
+      continue;
+    StringRef Name = *NameOrErr;
+    ErrorOr<uint64_t> AddrOrErr = Sym.getAddress();
+    if (AddrOrErr.getError())
+      continue;
+    uint64_t Addr = *AddrOrErr;
+    uint64_t Size = P.second;
+
+    if (Wrapper->op_write_native_code(Name.data(), Addr, (void *)Addr, Size) ==
+        -1) {
+      DEBUG(dbgs() << "Failed to tell OProfile about native function " << Name
+                   << " at [" << (void *)Addr << "-" << ((char *)Addr + Size)
+                   << "]\n");
+      continue;
     }
+    // TODO: support line number info (similar to IntelJITEventListener.cpp)
   }
 
   DebugObjects[Obj.getData().data()] = std::move(DebugObjOwner);
@@ -125,11 +128,11 @@ void OProfileJITEventListener::NotifyFreeingObject(const ObjectFile &Obj) {
     for (symbol_iterator I = DebugObj.symbol_begin(),
                          E = DebugObj.symbol_end();
          I != E; ++I) {
-      SymbolRef::Type SymType;
-      if (I->getType(SymType)) continue;
-      if (SymType == SymbolRef::ST_Function) {
-        uint64_t   Addr;
-        if (I->getAddress(Addr)) continue;
+      if (I->getType() == SymbolRef::ST_Function) {
+        ErrorOr<uint64_t> AddrOrErr = I->getAddress();
+        if (AddrOrErr.getError())
+          continue;
+        uint64_t Addr = *AddrOrErr;
 
         if (Wrapper->op_unload_native_code(Addr) == -1) {
           DEBUG(dbgs()
diff --git a/lib/ExecutionEngine/Orc/CMakeLists.txt b/lib/ExecutionEngine/Orc/CMakeLists.txt
index b38b4594057..99fe22c001d 100644
--- a/lib/ExecutionEngine/Orc/CMakeLists.txt
+++ b/lib/ExecutionEngine/Orc/CMakeLists.txt
@@ -1,10 +1,13 @@
 add_llvm_library(LLVMOrcJIT
-  CloneSubModule.cpp
   ExecutionUtils.cpp
   IndirectionUtils.cpp
+  NullResolver.cpp
   OrcMCJITReplacement.cpp
   OrcTargetSupport.cpp
 
   ADDITIONAL_HEADER_DIRS
   ${LLVM_MAIN_INCLUDE_DIR}/llvm/ExecutionEngine/Orc
+
+  DEPENDS
+  intrinsics_gen
   )
diff --git a/lib/ExecutionEngine/Orc/CloneSubModule.cpp b/lib/ExecutionEngine/Orc/CloneSubModule.cpp
deleted file mode 100644
index a3196ad2f78..00000000000
--- a/lib/ExecutionEngine/Orc/CloneSubModule.cpp
+++ /dev/null
@@ -1,108 +0,0 @@
-#include "llvm/ExecutionEngine/Orc/CloneSubModule.h"
-#include "llvm/IR/Function.h"
-#include "llvm/IR/GlobalVariable.h"
-#include "llvm/IR/Module.h"
-#include "llvm/Transforms/Utils/Cloning.h"
-
-namespace llvm {
-namespace orc {
-
-void copyGVInitializer(GlobalVariable &New, const GlobalVariable &Orig,
-                             ValueToValueMapTy &VMap) {
-  if (Orig.hasInitializer())
-    New.setInitializer(MapValue(Orig.getInitializer(), VMap));
-}
-
-void copyFunctionBody(Function &New, const Function &Orig,
-                            ValueToValueMapTy &VMap) {
-  if (!Orig.isDeclaration()) {
-    Function::arg_iterator DestI = New.arg_begin();
-    for (Function::const_arg_iterator J = Orig.arg_begin(); J != Orig.arg_end();
-         ++J) {
-      DestI->setName(J->getName());
-      VMap[J] = DestI++;
-    }
-
-    SmallVector<ReturnInst *, 8> Returns; // Ignore returns cloned.
-    CloneFunctionInto(&New, &Orig, VMap, /*ModuleLevelChanges=*/true, Returns);
-  }
-}
-
-void CloneSubModule(llvm::Module &Dst, const Module &Src,
-                    HandleGlobalVariableFtor HandleGlobalVariable,
-                    HandleFunctionFtor HandleFunction, bool CloneInlineAsm) {
-
-  ValueToValueMapTy VMap;
-
-  if (CloneInlineAsm)
-    Dst.appendModuleInlineAsm(Src.getModuleInlineAsm());
-
-  // Copy global variables (but not initializers, yet).
-  for (Module::const_global_iterator I = Src.global_begin(), E = Src.global_end();
-       I != E; ++I) {
-    GlobalVariable *GV = new GlobalVariable(
-        Dst, I->getType()->getElementType(), I->isConstant(), I->getLinkage(),
-        (Constant *)nullptr, I->getName(), (GlobalVariable *)nullptr,
-        I->getThreadLocalMode(), I->getType()->getAddressSpace());
-    GV->copyAttributesFrom(I);
-    VMap[I] = GV;
-  }
-
-  // Loop over the functions in the module, making external functions as before
-  for (Module::const_iterator I = Src.begin(), E = Src.end(); I != E; ++I) {
-    Function *NF =
-        Function::Create(cast<FunctionType>(I->getType()->getElementType()),
-                         I->getLinkage(), I->getName(), &Dst);
-    NF->copyAttributesFrom(I);
-    VMap[I] = NF;
-  }
-
-  // Loop over the aliases in the module
-  for (Module::const_alias_iterator I = Src.alias_begin(), E = Src.alias_end();
-       I != E; ++I) {
-    auto *PTy = cast<PointerType>(I->getType());
-    auto *GA =
-        GlobalAlias::create(PTy->getElementType(), PTy->getAddressSpace(),
-                            I->getLinkage(), I->getName(), &Dst);
-    GA->copyAttributesFrom(I);
-    VMap[I] = GA;
-  }
-
-  // Now that all of the things that global variable initializer can refer to
-  // have been created, loop through and copy the global variable referrers
-  // over...  We also set the attributes on the global now.
-  for (Module::const_global_iterator I = Src.global_begin(), E = Src.global_end();
-       I != E; ++I) {
-    GlobalVariable &GV = *cast<GlobalVariable>(VMap[I]);
-    HandleGlobalVariable(GV, *I, VMap);
-  }
-
-  // Similarly, copy over function bodies now...
-  //
-  for (Module::const_iterator I = Src.begin(), E = Src.end(); I != E; ++I) {
-    Function &F = *cast<Function>(VMap[I]);
-    HandleFunction(F, *I, VMap);
-  }
-
-  // And aliases
-  for (Module::const_alias_iterator I = Src.alias_begin(), E = Src.alias_end();
-       I != E; ++I) {
-    GlobalAlias *GA = cast<GlobalAlias>(VMap[I]);
-    if (const Constant *C = I->getAliasee())
-      GA->setAliasee(MapValue(C, VMap));
-  }
-
-  // And named metadata....
-  for (Module::const_named_metadata_iterator I = Src.named_metadata_begin(),
-                                             E = Src.named_metadata_end();
-       I != E; ++I) {
-    const NamedMDNode &NMD = *I;
-    NamedMDNode *NewNMD = Dst.getOrInsertNamedMetadata(NMD.getName());
-    for (unsigned i = 0, e = NMD.getNumOperands(); i != e; ++i)
-      NewNMD->addOperand(MapMetadata(NMD.getOperand(i), VMap));
-  }
-
-}
-
-} // End namespace orc.
-} // End namespace llvm.
diff --git a/lib/ExecutionEngine/Orc/IndirectionUtils.cpp b/lib/ExecutionEngine/Orc/IndirectionUtils.cpp
index ebeedef7eae..b439810ed33 100644
--- a/lib/ExecutionEngine/Orc/IndirectionUtils.cpp
+++ b/lib/ExecutionEngine/Orc/IndirectionUtils.cpp
@@ -9,10 +9,10 @@
 
 #include "llvm/ADT/STLExtras.h"
 #include "llvm/ADT/Triple.h"
-#include "llvm/ExecutionEngine/Orc/CloneSubModule.h"
 #include "llvm/ExecutionEngine/Orc/IndirectionUtils.h"
 #include "llvm/IR/CallSite.h"
 #include "llvm/IR/IRBuilder.h"
+#include "llvm/Transforms/Utils/Cloning.h"
 #include <set>
 #include <sstream>
 
@@ -30,11 +30,11 @@ Constant* createIRTypedAddress(FunctionType &FT, TargetAddress Addr) {
 
 GlobalVariable* createImplPointer(PointerType &PT, Module &M,
                                   const Twine &Name, Constant *Initializer) {
-  if (!Initializer)
-    Initializer = Constant::getNullValue(&PT);
-  return new GlobalVariable(M, &PT, false, GlobalValue::ExternalLinkage,
-                            Initializer, Name, nullptr,
-                            GlobalValue::NotThreadLocal, 0, true);
+  auto IP = new GlobalVariable(M, &PT, false, GlobalValue::ExternalLinkage,
+                               Initializer, Name, nullptr,
+                               GlobalValue::NotThreadLocal, 0, true);
+  IP->setVisibility(GlobalValue::HiddenVisibility);
+  return IP;
 }
 
 void makeStub(Function &F, GlobalVariable &ImplPointer) {
@@ -49,7 +49,11 @@ void makeStub(Function &F, GlobalVariable &ImplPointer) {
     CallArgs.push_back(&A);
   CallInst *Call = Builder.CreateCall(ImplAddr, CallArgs);
   Call->setTailCall();
-  Builder.CreateRet(Call);
+  Call->setAttributes(F.getAttributes());
+  if (F.getReturnType()->isVoidTy())
+    Builder.CreateRetVoid();
+  else
+    Builder.CreateRet(Call);
 }
 
 // Utility class for renaming global values and functions during partitioning.
@@ -83,83 +87,94 @@ class GlobalRenamer {
   DenseMap<const Value*, std::string> Names;
 };
 
-void partition(Module &M, const ModulePartitionMap &PMap) {
-
-  GlobalRenamer Renamer;
-
-  for (auto &KVPair : PMap) {
-
-    auto ExtractGlobalVars =
-      [&](GlobalVariable &New, const GlobalVariable &Orig,
-          ValueToValueMapTy &VMap) {
-        if (KVPair.second.count(&Orig)) {
-          copyGVInitializer(New, Orig, VMap);
-        }
-        if (New.hasLocalLinkage()) {
-          if (Renamer.needsRenaming(New))
-            New.setName(Renamer.getRename(Orig));
-          New.setLinkage(GlobalValue::ExternalLinkage);
-          New.setVisibility(GlobalValue::HiddenVisibility);
-        }
-        assert(!Renamer.needsRenaming(New) && "Invalid global name.");
-      };
-
-    auto ExtractFunctions =
-      [&](Function &New, const Function &Orig, ValueToValueMapTy &VMap) {
-        if (KVPair.second.count(&Orig))
-          copyFunctionBody(New, Orig, VMap);
-        if (New.hasLocalLinkage()) {
-          if (Renamer.needsRenaming(New))
-            New.setName(Renamer.getRename(Orig));
-          New.setLinkage(GlobalValue::ExternalLinkage);
-          New.setVisibility(GlobalValue::HiddenVisibility);
-        }
-        assert(!Renamer.needsRenaming(New) && "Invalid function name.");
-      };
-
-    CloneSubModule(*KVPair.first, M, ExtractGlobalVars, ExtractFunctions,
-                   false);
+static void raiseVisibilityOnValue(GlobalValue &V, GlobalRenamer &R) {
+  if (V.hasLocalLinkage()) {
+    if (R.needsRenaming(V))
+      V.setName(R.getRename(V));
+    V.setLinkage(GlobalValue::ExternalLinkage);
+    V.setVisibility(GlobalValue::HiddenVisibility);
   }
+  V.setUnnamedAddr(false);
+  assert(!R.needsRenaming(V) && "Invalid global name.");
 }
 
-FullyPartitionedModule fullyPartition(Module &M) {
-  FullyPartitionedModule MP;
-
-  ModulePartitionMap PMap;
+void makeAllSymbolsExternallyAccessible(Module &M) {
+  GlobalRenamer Renamer;
 
-  for (auto &F : M) {
+  for (auto &F : M)
+    raiseVisibilityOnValue(F, Renamer);
 
-    if (F.isDeclaration())
-      continue;
+  for (auto &GV : M.globals())
+    raiseVisibilityOnValue(GV, Renamer);
+}
 
-    std::string NewModuleName = (M.getName() + "." + F.getName()).str();
-    MP.Functions.push_back(
-      llvm::make_unique<Module>(NewModuleName, M.getContext()));
-    MP.Functions.back()->setDataLayout(M.getDataLayout());
-    PMap[MP.Functions.back().get()].insert(&F);
+Function* cloneFunctionDecl(Module &Dst, const Function &F,
+                            ValueToValueMapTy *VMap) {
+  assert(F.getParent() != &Dst && "Can't copy decl over existing function.");
+  Function *NewF =
+    Function::Create(cast<FunctionType>(F.getType()->getElementType()),
+                     F.getLinkage(), F.getName(), &Dst);
+  NewF->copyAttributesFrom(&F);
+
+  if (VMap) {
+    (*VMap)[&F] = NewF;
+    auto NewArgI = NewF->arg_begin();
+    for (auto ArgI = F.arg_begin(), ArgE = F.arg_end(); ArgI != ArgE;
+         ++ArgI, ++NewArgI)
+      (*VMap)[ArgI] = NewArgI;
   }
 
-  MP.GlobalVars =
-    llvm::make_unique<Module>((M.getName() + ".globals_and_stubs").str(),
-                              M.getContext());
-  MP.GlobalVars->setDataLayout(M.getDataLayout());
-
-  MP.Commons =
-    llvm::make_unique<Module>((M.getName() + ".commons").str(), M.getContext());
-  MP.Commons->setDataLayout(M.getDataLayout());
+  return NewF;
+}
 
-  // Make sure there's at least an empty set for the stubs map or we'll fail
-  // to clone anything for it (including the decls).
-  PMap[MP.GlobalVars.get()] = ModulePartitionMap::mapped_type();
-  for (auto &GV : M.globals())
-    if (GV.getLinkage() == GlobalValue::CommonLinkage)
-      PMap[MP.Commons.get()].insert(&GV);
-    else
-      PMap[MP.GlobalVars.get()].insert(&GV);
+void moveFunctionBody(Function &OrigF, ValueToValueMapTy &VMap,
+                      ValueMaterializer *Materializer,
+                      Function *NewF) {
+  assert(!OrigF.isDeclaration() && "Nothing to move");
+  if (!NewF)
+    NewF = cast<Function>(VMap[&OrigF]);
+  else
+    assert(VMap[&OrigF] == NewF && "Incorrect function mapping in VMap.");
+  assert(NewF && "Function mapping missing from VMap.");
+  assert(NewF->getParent() != OrigF.getParent() &&
+         "moveFunctionBody should only be used to move bodies between "
+         "modules.");
+
+  SmallVector<ReturnInst *, 8> Returns; // Ignore returns cloned.
+  CloneFunctionInto(NewF, &OrigF, VMap, /*ModuleLevelChanges=*/true, Returns,
+                    "", nullptr, nullptr, Materializer);
+  OrigF.deleteBody();
+}
 
-  partition(M, PMap);
+GlobalVariable* cloneGlobalVariableDecl(Module &Dst, const GlobalVariable &GV,
+                                        ValueToValueMapTy *VMap) {
+  assert(GV.getParent() != &Dst && "Can't copy decl over existing global var.");
+  GlobalVariable *NewGV = new GlobalVariable(
+      Dst, GV.getType()->getElementType(), GV.isConstant(),
+      GV.getLinkage(), nullptr, GV.getName(), nullptr,
+      GV.getThreadLocalMode(), GV.getType()->getAddressSpace());
+  NewGV->copyAttributesFrom(&GV);
+  if (VMap)
+    (*VMap)[&GV] = NewGV;
+  return NewGV;
+}
 
-  return MP;
+void moveGlobalVariableInitializer(GlobalVariable &OrigGV,
+                                   ValueToValueMapTy &VMap,
+                                   ValueMaterializer *Materializer,
+                                   GlobalVariable *NewGV) {
+  assert(OrigGV.hasInitializer() && "Nothing to move");
+  if (!NewGV)
+    NewGV = cast<GlobalVariable>(VMap[&OrigGV]);
+  else
+    assert(VMap[&OrigGV] == NewGV &&
+           "Incorrect global variable mapping in VMap.");
+  assert(NewGV->getParent() != OrigGV.getParent() &&
+         "moveGlobalVariable should only be used to move initializers between "
+         "modules");
+
+  NewGV->setInitializer(MapValue(OrigGV.getInitializer(), VMap, RF_None,
+                                 nullptr, Materializer));
 }
 
 } // End namespace orc.
diff --git a/lib/ExecutionEngine/Orc/NullResolver.cpp b/lib/ExecutionEngine/Orc/NullResolver.cpp
new file mode 100644
index 00000000000..57666a9dada
--- /dev/null
+++ b/lib/ExecutionEngine/Orc/NullResolver.cpp
@@ -0,0 +1,27 @@
+//===---------- NullResolver.cpp - Reject symbol lookup requests ----------===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/ExecutionEngine/Orc/NullResolver.h"
+
+#include "llvm/Support/ErrorHandling.h"
+
+namespace llvm {
+namespace orc {
+
+RuntimeDyld::SymbolInfo NullResolver::findSymbol(const std::string &Name) {
+  llvm_unreachable("Unexpected cross-object symbol reference");
+}
+
+RuntimeDyld::SymbolInfo
+NullResolver::findSymbolInLogicalDylib(const std::string &Name) {
+  llvm_unreachable("Unexpected cross-object symbol reference");
+}
+
+} // End namespace orc.
+} // End namespace llvm.
diff --git a/lib/ExecutionEngine/Orc/OrcMCJITReplacement.cpp b/lib/ExecutionEngine/Orc/OrcMCJITReplacement.cpp
index 48fd31e51a6..b7a68e041c1 100644
--- a/lib/ExecutionEngine/Orc/OrcMCJITReplacement.cpp
+++ b/lib/ExecutionEngine/Orc/OrcMCJITReplacement.cpp
@@ -25,7 +25,7 @@ namespace orc {
 
 GenericValue
 OrcMCJITReplacement::runFunction(Function *F,
-                                 const std::vector<GenericValue> &ArgValues) {
+                                 ArrayRef<GenericValue> ArgValues) {
   assert(F && "Function *F was null at entry to run()");
 
   void *FPtr = getPointerToFunction(F);
diff --git a/lib/ExecutionEngine/Orc/OrcMCJITReplacement.h b/lib/ExecutionEngine/Orc/OrcMCJITReplacement.h
index 4023344d2f3..951993f75e4 100644
--- a/lib/ExecutionEngine/Orc/OrcMCJITReplacement.h
+++ b/lib/ExecutionEngine/Orc/OrcMCJITReplacement.h
@@ -137,26 +137,26 @@ class OrcMCJITReplacement : public ExecutionEngine {
   }
 
   OrcMCJITReplacement(
-                    std::shared_ptr<MCJITMemoryManager> MemMgr,
-                    std::shared_ptr<RuntimeDyld::SymbolResolver> ClientResolver,
-                    std::unique_ptr<TargetMachine> TM)
-      : TM(std::move(TM)), MemMgr(*this, std::move(MemMgr)),
-        Resolver(*this), ClientResolver(std::move(ClientResolver)),
-        Mang(this->TM->getDataLayout()),
-        NotifyObjectLoaded(*this), NotifyFinalized(*this),
+      std::shared_ptr<MCJITMemoryManager> MemMgr,
+      std::shared_ptr<RuntimeDyld::SymbolResolver> ClientResolver,
+      std::unique_ptr<TargetMachine> TM)
+      : ExecutionEngine(TM->createDataLayout()), TM(std::move(TM)),
+        MemMgr(*this, std::move(MemMgr)), Resolver(*this),
+        ClientResolver(std::move(ClientResolver)), NotifyObjectLoaded(*this),
+        NotifyFinalized(*this),
         ObjectLayer(NotifyObjectLoaded, NotifyFinalized),
         CompileLayer(ObjectLayer, SimpleCompiler(*this->TM)),
-        LazyEmitLayer(CompileLayer) {
-    setDataLayout(this->TM->getDataLayout());
-  }
+        LazyEmitLayer(CompileLayer) {}
 
   void addModule(std::unique_ptr<Module> M) override {
 
     // If this module doesn't have a DataLayout attached then attach the
     // default.
-    if (M->getDataLayout().isDefault())
-      M->setDataLayout(*getDataLayout());
-
+    if (M->getDataLayout().isDefault()) {
+      M->setDataLayout(getDataLayout());
+    } else {
+      assert(M->getDataLayout() == getDataLayout() && "DataLayout Mismatch");
+    }
     Modules.push_back(std::move(M));
     std::vector<Module *> Ms;
     Ms.push_back(&*Modules.back());
@@ -229,7 +229,7 @@ class OrcMCJITReplacement : public ExecutionEngine {
   }
 
   GenericValue runFunction(Function *F,
-                           const std::vector<GenericValue> &ArgValues) override;
+                           ArrayRef<GenericValue> ArgValues) override;
 
   void setObjectCache(ObjectCache *NewCache) override {
     CompileLayer.setObjectCache(NewCache);
@@ -290,7 +290,7 @@ class OrcMCJITReplacement : public ExecutionEngine {
              "Incorrect number of Infos for Objects.");
       for (unsigned I = 0; I < Objects.size(); ++I)
         M.MemMgr.notifyObjectLoaded(&M, *Objects[I]);
-    };
+    }
 
   private:
     OrcMCJITReplacement &M;
@@ -311,7 +311,7 @@ class OrcMCJITReplacement : public ExecutionEngine {
     std::string MangledName;
     {
       raw_string_ostream MangledNameStream(MangledName);
-      Mang.getNameWithPrefix(MangledNameStream, Name);
+      Mang.getNameWithPrefix(MangledNameStream, Name, getDataLayout());
     }
     return MangledName;
   }
diff --git a/lib/ExecutionEngine/Orc/OrcTargetSupport.cpp b/lib/ExecutionEngine/Orc/OrcTargetSupport.cpp
index fc56e67b85c..258868aa64f 100644
--- a/lib/ExecutionEngine/Orc/OrcTargetSupport.cpp
+++ b/lib/ExecutionEngine/Orc/OrcTargetSupport.cpp
@@ -56,12 +56,12 @@ void OrcX86_64::insertResolverBlock(
 
   // Start the resolver function.
   AsmStream << ResolverBlockName << ":\n"
-            << "  pushq   %rbp\n"
-            << "  movq    %rsp, %rbp\n";
+            << "  pushq     %rbp\n"
+            << "  movq      %rsp, %rbp\n";
 
   // Store the GPRs.
   for (const auto &GPR : GPRs)
-    AsmStream << "  pushq   %" << GPR << "\n";
+    AsmStream << "  pushq     %" << GPR << "\n";
 
   // Store floating-point state with FXSAVE.
   // Note: We need to keep the stack 16-byte aligned, so if we've emitted an odd
@@ -69,30 +69,30 @@ void OrcX86_64::insertResolverBlock(
   //       an extra 64 bits of padding to the FXSave area.
   unsigned Padding = (GPRs.size() + 1) % 2 ? 8 : 0;
   unsigned FXSaveSize = 512 + Padding;
-  AsmStream << "  subq    $" << FXSaveSize << ", %rsp\n"
-            << "  fxsave  (%rsp)\n"
+  AsmStream << "  subq      $" << FXSaveSize << ", %rsp\n"
+            << "  fxsave64  (%rsp)\n"
 
   // Load callback manager address, compute trampoline address, call JIT.
-            << "  lea     jit_callback_manager_addr(%rip), %rdi\n"
-            << "  movq    (%rdi), %rdi\n"
-            << "  movq    0x8(%rbp), %rsi\n"
-            << "  subq    $" << X86_64_TrampolineLength << ", %rsi\n"
-            << "  movabsq $" << CallbackAddr << ", %rax\n"
-            << "  callq   *%rax\n"
+            << "  lea       jit_callback_manager_addr(%rip), %rdi\n"
+            << "  movq      (%rdi), %rdi\n"
+            << "  movq      0x8(%rbp), %rsi\n"
+            << "  subq      $" << X86_64_TrampolineLength << ", %rsi\n"
+            << "  movabsq   $" << CallbackAddr << ", %rax\n"
+            << "  callq     *%rax\n"
 
   // Replace the return to the trampoline with the return address of the
   // compiled function body.
-            << "  movq    %rax, 0x8(%rbp)\n"
+            << "  movq      %rax, 0x8(%rbp)\n"
 
   // Restore the floating point state.
-            << "  fxrstor (%rsp)\n"
-            << "  addq    $" << FXSaveSize << ", %rsp\n";
+            << "  fxrstor64 (%rsp)\n"
+            << "  addq      $" << FXSaveSize << ", %rsp\n";
 
   for (const auto &GPR : make_range(GPRs.rbegin(), GPRs.rend()))
-    AsmStream << "  popq    %" << GPR << "\n";
+    AsmStream << "  popq      %" << GPR << "\n";
 
   // Restore original RBP and return to compiled function body.
-  AsmStream << "  popq    %rbp\n"
+  AsmStream << "  popq      %rbp\n"
             << "  retq\n";
 
   M.appendModuleInlineAsm(AsmStream.str());
diff --git a/lib/ExecutionEngine/RuntimeDyld/CMakeLists.txt b/lib/ExecutionEngine/RuntimeDyld/CMakeLists.txt
index e78408a3b6a..182f98200fc 100644
--- a/lib/ExecutionEngine/RuntimeDyld/CMakeLists.txt
+++ b/lib/ExecutionEngine/RuntimeDyld/CMakeLists.txt
@@ -5,4 +5,7 @@ add_llvm_library(LLVMRuntimeDyld
   RuntimeDyldCOFF.cpp
   RuntimeDyldELF.cpp
   RuntimeDyldMachO.cpp
+
+  DEPENDS
+  intrinsics_gen
   )
diff --git a/lib/ExecutionEngine/RuntimeDyld/RTDyldMemoryManager.cpp b/lib/ExecutionEngine/RuntimeDyld/RTDyldMemoryManager.cpp
index 2a5e4f83228..044eee43c9e 100644
--- a/lib/ExecutionEngine/RuntimeDyld/RTDyldMemoryManager.cpp
+++ b/lib/ExecutionEngine/RuntimeDyld/RTDyldMemoryManager.cpp
@@ -41,8 +41,8 @@ RTDyldMemoryManager::~RTDyldMemoryManager() {}
 #endif
 
 #if HAVE_EHTABLE_SUPPORT
-extern "C" void __register_frame(void*);
-extern "C" void __deregister_frame(void*);
+extern "C" void __register_frame(void *);
+extern "C" void __deregister_frame(void *);
 #else
 // The building compiler does not have __(de)register_frame but
 // it may be found at runtime in a dynamically-loaded library.
@@ -50,28 +50,28 @@ extern "C" void __deregister_frame(void*);
 // but using the MingW runtime.
 void __register_frame(void *p) {
   static bool Searched = false;
-  static void *rf = 0;
+  static void((*rf)(void *)) = 0;
 
   if (!Searched) {
     Searched = true;
-    rf = llvm::sys::DynamicLibrary::SearchForAddressOfSymbol(
-                                                      "__register_frame");
+    *(void **)&rf =
+        llvm::sys::DynamicLibrary::SearchForAddressOfSymbol("__register_frame");
   }
   if (rf)
-    ((void (*)(void *))rf)(p);
+    rf(p);
 }
 
 void __deregister_frame(void *p) {
   static bool Searched = false;
-  static void *df = 0;
+  static void((*df)(void *)) = 0;
 
   if (!Searched) {
     Searched = true;
-    df = llvm::sys::DynamicLibrary::SearchForAddressOfSymbol(
-                                                      "__deregister_frame");
+    *(void **)&df = llvm::sys::DynamicLibrary::SearchForAddressOfSymbol(
+        "__deregister_frame");
   }
   if (df)
-    ((void (*)(void *))df)(p);
+    df(p);
 }
 #endif
 
diff --git a/lib/ExecutionEngine/RuntimeDyld/RuntimeDyld.cpp b/lib/ExecutionEngine/RuntimeDyld/RuntimeDyld.cpp
index a13ecb7a834..24495b21d45 100644
--- a/lib/ExecutionEngine/RuntimeDyld/RuntimeDyld.cpp
+++ b/lib/ExecutionEngine/RuntimeDyld/RuntimeDyld.cpp
@@ -113,42 +113,23 @@ void RuntimeDyldImpl::mapSectionAddress(const void *LocalAddress,
   llvm_unreachable("Attempting to remap address of unknown section!");
 }
 
-static std::error_code getOffset(const SymbolRef &Sym, uint64_t &Result) {
-  uint64_t Address;
-  if (std::error_code EC = Sym.getAddress(Address))
+static std::error_code getOffset(const SymbolRef &Sym, SectionRef Sec,
+                                 uint64_t &Result) {
+  ErrorOr<uint64_t> AddressOrErr = Sym.getAddress();
+  if (std::error_code EC = AddressOrErr.getError())
     return EC;
-
-  if (Address == UnknownAddressOrSize) {
-    Result = UnknownAddressOrSize;
-    return object_error::success;
-  }
-
-  const ObjectFile *Obj = Sym.getObject();
-  section_iterator SecI(Obj->section_begin());
-  if (std::error_code EC = Sym.getSection(SecI))
-    return EC;
-
-  if (SecI == Obj->section_end()) {
-    Result = UnknownAddressOrSize;
-    return object_error::success;
-  }
-
-  uint64_t SectionAddress = SecI->getAddress();
-  Result = Address - SectionAddress;
-  return object_error::success;
+  Result = *AddressOrErr - Sec.getAddress();
+  return std::error_code();
 }
 
-std::pair<unsigned, unsigned>
+RuntimeDyldImpl::ObjSectionToIDMap
 RuntimeDyldImpl::loadObjectImpl(const object::ObjectFile &Obj) {
   MutexGuard locked(lock);
 
-  // Grab the first Section ID. We'll use this later to construct the underlying
-  // range for the returned LoadedObjectInfo.
-  unsigned SectionsAddedBeginIdx = Sections.size();
-
   // Save information about our target
   Arch = (Triple::ArchType)Obj.getArch();
   IsTargetLittleEndian = Obj.isLittleEndian();
+  setMipsABI(Obj);
 
   // Compute the memory size required to load all sections to be loaded
   // and pass this information to the memory manager
@@ -174,21 +155,21 @@ RuntimeDyldImpl::loadObjectImpl(const object::ObjectFile &Obj) {
     if (IsCommon)
       CommonSymbols.push_back(*I);
     else {
-      object::SymbolRef::Type SymType;
-      Check(I->getType(SymType));
+      object::SymbolRef::Type SymType = I->getType();
 
       if (SymType == object::SymbolRef::ST_Function ||
           SymType == object::SymbolRef::ST_Data ||
           SymType == object::SymbolRef::ST_Unknown) {
 
-        StringRef Name;
-        uint64_t SectOffset;
-        Check(I->getName(Name));
-        Check(getOffset(*I, SectOffset));
+        ErrorOr<StringRef> NameOrErr = I->getName();
+        Check(NameOrErr.getError());
+        StringRef Name = *NameOrErr;
         section_iterator SI = Obj.section_end();
         Check(I->getSection(SI));
         if (SI == Obj.section_end())
           continue;
+        uint64_t SectOffset;
+        Check(getOffset(*I, *SI, SectOffset));
         StringRef SectionData;
         Check(SI->getContents(SectionData));
         bool IsCode = SI->isText();
@@ -246,9 +227,10 @@ RuntimeDyldImpl::loadObjectImpl(const object::ObjectFile &Obj) {
   // Give the subclasses a chance to tie-up any loose ends.
   finalizeLoad(Obj, LocalSections);
 
-  unsigned SectionsAddedEndIdx = Sections.size();
+//   for (auto E : LocalSections)
+//     llvm::dbgs() << "Added: " << E.first.getRawDataRefImpl() << " -> " << E.second << "\n";
 
-  return std::make_pair(SectionsAddedBeginIdx, SectionsAddedEndIdx);
+  return LocalSections;
 }
 
 // A helper method for computeTotalAllocSize.
@@ -266,10 +248,10 @@ computeAllocationSizeForSections(std::vector<uint64_t> &SectionSizes,
   return TotalSize;
 }
 
-static bool isRequiredForExecution(const SectionRef &Section) {
+static bool isRequiredForExecution(const SectionRef Section) {
   const ObjectFile *Obj = Section.getObject();
-  if (auto *ELFObj = dyn_cast<object::ELFObjectFileBase>(Obj))
-    return ELFObj->getSectionFlags(Section) & ELF::SHF_ALLOC;
+  if (isa<object::ELFObjectFileBase>(Obj))
+    return ELFSectionRef(Section).getFlags() & ELF::SHF_ALLOC;
   if (auto *COFFObj = dyn_cast<object::COFFObjectFile>(Obj)) {
     const coff_section *CoffSection = COFFObj->getCOFFSection(Section);
     // Avoid loading zero-sized COFF sections.
@@ -286,12 +268,12 @@ static bool isRequiredForExecution(const SectionRef &Section) {
   
   assert(isa<MachOObjectFile>(Obj));
   return true;
- }
+}
 
-static bool isReadOnlyData(const SectionRef &Section) {
+static bool isReadOnlyData(const SectionRef Section) {
   const ObjectFile *Obj = Section.getObject();
-  if (auto *ELFObj = dyn_cast<object::ELFObjectFileBase>(Obj))
-    return !(ELFObj->getSectionFlags(Section) &
+  if (isa<object::ELFObjectFileBase>(Obj))
+    return !(ELFSectionRef(Section).getFlags() &
              (ELF::SHF_WRITE | ELF::SHF_EXECINSTR));
   if (auto *COFFObj = dyn_cast<object::COFFObjectFile>(Obj))
     return ((COFFObj->getCOFFSection(Section)->Characteristics &
@@ -306,10 +288,10 @@ static bool isReadOnlyData(const SectionRef &Section) {
   return false;
 }
 
-static bool isZeroInit(const SectionRef &Section) {
+static bool isZeroInit(const SectionRef Section) {
   const ObjectFile *Obj = Section.getObject();
-  if (auto *ELFObj = dyn_cast<object::ELFObjectFileBase>(Obj))
-    return ELFObj->getSectionType(Section) == ELF::SHT_NOBITS;
+  if (isa<object::ELFObjectFileBase>(Obj))
+    return ELFSectionRef(Section).getType() == ELF::SHT_NOBITS;
   if (auto *COFFObj = dyn_cast<object::COFFObjectFile>(Obj))
     return COFFObj->getCOFFSection(Section)->Characteristics &
             COFF::IMAGE_SCN_CNT_UNINITIALIZED_DATA;
@@ -386,8 +368,7 @@ void RuntimeDyldImpl::computeTotalAllocSize(const ObjectFile &Obj,
     uint32_t Flags = I->getFlags();
     if (Flags & SymbolRef::SF_Common) {
       // Add the common symbols to a list.  We'll allocate them all below.
-      uint64_t Size = 0;
-      Check(I->getSize(Size));
+      uint64_t Size = I->getCommonSize();
       CommonSize += Size;
     }
   }
@@ -482,8 +463,9 @@ void RuntimeDyldImpl::emitCommonSymbols(const ObjectFile &Obj,
   DEBUG(dbgs() << "Processing common symbols...\n");
 
   for (const auto &Sym : CommonSymbols) {
-    StringRef Name;
-    Check(Sym.getName(Name));
+    ErrorOr<StringRef> NameOrErr = Sym.getName();
+    Check(NameOrErr.getError());
+    StringRef Name = *NameOrErr;
 
     // Skip common symbols already elsewhere.
     if (GlobalSymbolTable.count(Name) ||
@@ -493,10 +475,8 @@ void RuntimeDyldImpl::emitCommonSymbols(const ObjectFile &Obj,
       continue;
     }
 
-    uint32_t Align = 0;
-    uint64_t Size = 0;
-    Check(Sym.getAlignment(Align));
-    Check(Sym.getSize(Size));
+    uint32_t Align = Sym.getAlignment();
+    uint64_t Size = Sym.getCommonSize();
 
     CommonSize += Align + Size;
     SymbolsToAllocate.push_back(Sym);
@@ -517,12 +497,11 @@ void RuntimeDyldImpl::emitCommonSymbols(const ObjectFile &Obj,
 
   // Assign the address of each symbol
   for (auto &Sym : SymbolsToAllocate) {
-    uint32_t Align;
-    uint64_t Size;
-    StringRef Name;
-    Check(Sym.getAlignment(Align));
-    Check(Sym.getSize(Size));
-    Check(Sym.getName(Name));
+    uint32_t Align = Sym.getAlignment();
+    uint64_t Size = Sym.getCommonSize();
+    ErrorOr<StringRef> NameOrErr = Sym.getName();
+    Check(NameOrErr.getError());
+    StringRef Name = *NameOrErr;
     if (Align) {
       // This symbol has an alignment requirement.
       uint64_t AlignOffset = OffsetToAlignment((uint64_t)Addr, Align);
@@ -574,10 +553,16 @@ unsigned RuntimeDyldImpl::emitSection(const ObjectFile &Obj,
   uint8_t *Addr;
   const char *pData = nullptr;
 
+  // In either case, set the location of the unrelocated section in memory,
+  // since we still process relocations for it even if we're not applying them.
+  Check(Section.getContents(data));
+  // Virtual sections have no data in the object image, so leave pData = 0
+  if (!IsVirtual)
+    pData = data.data();
+
   // Some sections, such as debug info, don't need to be loaded for execution.
   // Leave those where they are.
   if (IsRequired) {
-    Check(Section.getContents(data));
     Allocate = DataSize + PaddingSize + StubBufSize;
     if (!Allocate)
       Allocate = 1;
@@ -588,10 +573,6 @@ unsigned RuntimeDyldImpl::emitSection(const ObjectFile &Obj,
     if (!Addr)
       report_fatal_error("Unable to allocate section memory!");
 
-    // Virtual sections have no data in the object image, so leave pData = 0
-    if (!IsVirtual)
-      pData = data.data();
-
     // Zero-initialize or copy the data from the image
     if (IsZeroInit || IsVirtual)
       memset(Addr, 0, DataSize);
@@ -687,7 +668,7 @@ uint8_t *RuntimeDyldImpl::createStubFunction(uint8_t *Addr,
     // and stubs for branches Thumb - ARM and ARM - Thumb.
     writeBytesUnaligned(0xe51ff004, Addr, 4); // ldr pc,<label>
     return Addr + 4;
-  } else if (Arch == Triple::mipsel || Arch == Triple::mips) {
+  } else if (IsMipsO32ABI) {
     // 0:   3c190000        lui     t9,%hi(addr).
     // 4:   27390000        addiu   t9,t9,%lo(addr).
     // 8:   03200008        jr      t9.
@@ -814,12 +795,16 @@ void RuntimeDyldImpl::resolveExternalSymbols() {
         report_fatal_error("Program used external function '" + Name +
                            "' which could not be resolved!");
 
-      DEBUG(dbgs() << "Resolving relocations Name: " << Name << "\t"
-                   << format("0x%lx", Addr) << "\n");
-      // This list may have been updated when we called getSymbolAddress, so
-      // don't change this code to get the list earlier.
-      RelocationList &Relocs = i->second;
-      resolveRelocationList(Relocs, Addr);
+      // If Resolver returned UINT64_MAX, the client wants to handle this symbol
+      // manually and we shouldn't resolve its relocations.
+      if (Addr != UINT64_MAX) {
+        DEBUG(dbgs() << "Resolving relocations Name: " << Name << "\t"
+                     << format("0x%lx", Addr) << "\n");
+        // This list may have been updated when we called getSymbolAddress, so
+        // don't change this code to get the list earlier.
+        RelocationList &Relocs = i->second;
+        resolveRelocationList(Relocs, Addr);
+      }
     }
 
     ExternalSymbolRelocations.erase(i);
@@ -830,10 +815,19 @@ void RuntimeDyldImpl::resolveExternalSymbols() {
 // RuntimeDyld class implementation
 
 uint64_t RuntimeDyld::LoadedObjectInfo::getSectionLoadAddress(
-                                                  StringRef SectionName) const {
-  for (unsigned I = BeginIdx; I != EndIdx; ++I)
-    if (RTDyld.Sections[I].Name == SectionName)
-      return RTDyld.Sections[I].LoadAddress;
+                                          const object::SectionRef &Sec) const {
+
+//   llvm::dbgs() << "Searching for " << Sec.getRawDataRefImpl() << " in:\n";
+//   for (auto E : ObjSecToIDMap)
+//     llvm::dbgs() << "Added: " << E.first.getRawDataRefImpl() << " -> " << E.second << "\n";
+
+  auto I = ObjSecToIDMap.find(Sec);
+  if (I != ObjSecToIDMap.end()) {
+//    llvm::dbgs() << "Found ID " << I->second << " for Sec: " << Sec.getRawDataRefImpl() << ", LoadAddress = " << RTDyld.Sections[I->second].LoadAddress << "\n";
+    return RTDyld.Sections[I->second].LoadAddress;
+  } else {
+//    llvm::dbgs() << "Not found.\n";
+  }
 
   return 0;
 }
diff --git a/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldCOFF.cpp b/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldCOFF.cpp
index 8055d550006..e2979a6140f 100644
--- a/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldCOFF.cpp
+++ b/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldCOFF.cpp
@@ -24,11 +24,11 @@ using namespace llvm::object;
 
 namespace {
 
-class LoadedCOFFObjectInfo : public RuntimeDyld::LoadedObjectInfo {
+class LoadedCOFFObjectInfo
+    : public RuntimeDyld::LoadedObjectInfoHelper<LoadedCOFFObjectInfo> {
 public:
-  LoadedCOFFObjectInfo(RuntimeDyldImpl &RTDyld, unsigned BeginIdx,
-                       unsigned EndIdx)
-      : RuntimeDyld::LoadedObjectInfo(RTDyld, BeginIdx, EndIdx) {}
+  LoadedCOFFObjectInfo(RuntimeDyldImpl &RTDyld, ObjSectionToIDMap ObjSecToIDMap)
+      : LoadedObjectInfoHelper(RTDyld, std::move(ObjSecToIDMap)) {}
 
   OwningBinary<ObjectFile>
   getObjectForDebug(const ObjectFile &Obj) const override {
@@ -54,30 +54,12 @@ llvm::RuntimeDyldCOFF::create(Triple::ArchType Arch,
 
 std::unique_ptr<RuntimeDyld::LoadedObjectInfo>
 RuntimeDyldCOFF::loadObject(const object::ObjectFile &O) {
-  unsigned SectionStartIdx, SectionEndIdx;
-  std::tie(SectionStartIdx, SectionEndIdx) = loadObjectImpl(O);
-  return llvm::make_unique<LoadedCOFFObjectInfo>(*this, SectionStartIdx,
-                                                 SectionEndIdx);
+  return llvm::make_unique<LoadedCOFFObjectInfo>(*this, loadObjectImpl(O));
 }
 
 uint64_t RuntimeDyldCOFF::getSymbolOffset(const SymbolRef &Sym) {
-  uint64_t Address;
-  if (Sym.getAddress(Address))
-    return UnknownAddressOrSize;
-
-  if (Address == UnknownAddressOrSize)
-    return UnknownAddressOrSize;
-
-  const ObjectFile *Obj = Sym.getObject();
-  section_iterator SecI(Obj->section_end());
-  if (Sym.getSection(SecI))
-    return UnknownAddressOrSize;
-
-  if (SecI == Obj->section_end())
-    return UnknownAddressOrSize;
-
-  uint64_t SectionAddress = SecI->getAddress();
-  return Address - SectionAddress;
+  // The value in a relocatable COFF object is the offset.
+  return Sym.getValue();
 }
 
 bool RuntimeDyldCOFF::isCompatibleFile(const object::ObjectFile &Obj) const {
diff --git a/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldChecker.cpp b/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldChecker.cpp
index 957571b092d..ae199b72022 100644
--- a/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldChecker.cpp
+++ b/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldChecker.cpp
@@ -727,7 +727,9 @@ bool RuntimeDyldCheckerImpl::checkAllRulesInBuffer(StringRef RulePrefix,
 }
 
 bool RuntimeDyldCheckerImpl::isSymbolValid(StringRef Symbol) const {
-  return getRTDyld().getSymbolLocalAddress(Symbol) != nullptr;
+  if (getRTDyld().getSymbolLocalAddress(Symbol))
+    return true;
+  return !!getRTDyld().Resolver.findSymbol(Symbol);
 }
 
 uint64_t RuntimeDyldCheckerImpl::getSymbolLocalAddr(StringRef Symbol) const {
diff --git a/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldELF.cpp b/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldELF.cpp
index bbffdfb3941..38890aea8b3 100644
--- a/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldELF.cpp
+++ b/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldELF.cpp
@@ -104,11 +104,11 @@ void DyldELFObject<ELFT>::updateSymbolAddress(const SymbolRef &SymRef,
   sym->st_value = static_cast<addr_type>(Addr);
 }
 
-class LoadedELFObjectInfo : public RuntimeDyld::LoadedObjectInfo {
+class LoadedELFObjectInfo
+    : public RuntimeDyld::LoadedObjectInfoHelper<LoadedELFObjectInfo> {
 public:
-  LoadedELFObjectInfo(RuntimeDyldImpl &RTDyld, unsigned BeginIdx,
-                      unsigned EndIdx)
-    : RuntimeDyld::LoadedObjectInfo(RTDyld, BeginIdx, EndIdx) {}
+  LoadedELFObjectInfo(RuntimeDyldImpl &RTDyld, ObjSectionToIDMap ObjSecToIDMap)
+      : LoadedObjectInfoHelper(RTDyld, std::move(ObjSecToIDMap)) {}
 
   OwningBinary<ObjectFile>
   getObjectForDebug(const ObjectFile &Obj) const override;
@@ -117,6 +117,7 @@ class LoadedELFObjectInfo : public RuntimeDyld::LoadedObjectInfo {
 template <typename ELFT>
 std::unique_ptr<DyldELFObject<ELFT>>
 createRTDyldELFObject(MemoryBufferRef Buffer,
+                      const ObjectFile &SourceObject,
                       const LoadedELFObjectInfo &L,
                       std::error_code &ec) {
   typedef typename ELFFile<ELFT>::Elf_Shdr Elf_Shdr;
@@ -126,6 +127,7 @@ createRTDyldELFObject(MemoryBufferRef Buffer,
     llvm::make_unique<DyldELFObject<ELFT>>(Buffer, ec);
 
   // Iterate over all sections in the object.
+  auto SI = SourceObject.section_begin();
   for (const auto &Sec : Obj->sections()) {
     StringRef SectionName;
     Sec.getName(SectionName);
@@ -134,12 +136,13 @@ createRTDyldELFObject(MemoryBufferRef Buffer,
       Elf_Shdr *shdr = const_cast<Elf_Shdr *>(
           reinterpret_cast<const Elf_Shdr *>(ShdrRef.p));
 
-      if (uint64_t SecLoadAddr = L.getSectionLoadAddress(SectionName)) {
+      if (uint64_t SecLoadAddr = L.getSectionLoadAddress(*SI)) {
         // This assumes that the address passed in matches the target address
         // bitness. The template-based type cast handles everything else.
         shdr->sh_addr = static_cast<addr_type>(SecLoadAddr);
       }
     }
+    ++SI;
   }
 
   return Obj;
@@ -156,17 +159,21 @@ OwningBinary<ObjectFile> createELFDebugObject(const ObjectFile &Obj,
 
   std::unique_ptr<ObjectFile> DebugObj;
   if (Obj.getBytesInAddress() == 4 && Obj.isLittleEndian()) {
-    typedef ELFType<support::little, 2, false> ELF32LE;
-    DebugObj = createRTDyldELFObject<ELF32LE>(Buffer->getMemBufferRef(), L, ec);
+    typedef ELFType<support::little, false> ELF32LE;
+    DebugObj = createRTDyldELFObject<ELF32LE>(Buffer->getMemBufferRef(), Obj, L,
+                                              ec);
   } else if (Obj.getBytesInAddress() == 4 && !Obj.isLittleEndian()) {
-    typedef ELFType<support::big, 2, false> ELF32BE;
-    DebugObj = createRTDyldELFObject<ELF32BE>(Buffer->getMemBufferRef(), L, ec);
+    typedef ELFType<support::big, false> ELF32BE;
+    DebugObj = createRTDyldELFObject<ELF32BE>(Buffer->getMemBufferRef(), Obj, L,
+                                              ec);
   } else if (Obj.getBytesInAddress() == 8 && !Obj.isLittleEndian()) {
-    typedef ELFType<support::big, 2, true> ELF64BE;
-    DebugObj = createRTDyldELFObject<ELF64BE>(Buffer->getMemBufferRef(), L, ec);
+    typedef ELFType<support::big, true> ELF64BE;
+    DebugObj = createRTDyldELFObject<ELF64BE>(Buffer->getMemBufferRef(), Obj, L,
+                                              ec);
   } else if (Obj.getBytesInAddress() == 8 && Obj.isLittleEndian()) {
-    typedef ELFType<support::little, 2, true> ELF64LE;
-    DebugObj = createRTDyldELFObject<ELF64LE>(Buffer->getMemBufferRef(), L, ec);
+    typedef ELFType<support::little, true> ELF64LE;
+    DebugObj = createRTDyldELFObject<ELF64LE>(Buffer->getMemBufferRef(), Obj, L,
+                                              ec);
   } else
     llvm_unreachable("Unexpected ELF format");
 
@@ -214,10 +221,7 @@ void RuntimeDyldELF::deregisterEHFrames() {
 
 std::unique_ptr<RuntimeDyld::LoadedObjectInfo>
 RuntimeDyldELF::loadObject(const object::ObjectFile &O) {
-  unsigned SectionStartIdx, SectionEndIdx;
-  std::tie(SectionStartIdx, SectionEndIdx) = loadObjectImpl(O);
-  return llvm::make_unique<LoadedELFObjectInfo>(*this, SectionStartIdx,
-                                                SectionEndIdx);
+  return llvm::make_unique<LoadedELFObjectInfo>(*this, loadObjectImpl(O));
 }
 
 void RuntimeDyldELF::resolveX86_64Relocation(const SectionEntry &Section,
@@ -247,29 +251,16 @@ void RuntimeDyldELF::resolveX86_64Relocation(const SectionEntry &Section,
     break;
   }
   case ELF::R_X86_64_PC32: {
-    // Get the placeholder value from the generated object since
-    // a previous relocation attempt may have overwritten the loaded version
-    support::ulittle32_t::ref Placeholder(
-        (void *)(Section.ObjAddress + Offset));
     uint64_t FinalAddress = Section.LoadAddress + Offset;
     int64_t RealOffset = Value + Addend - FinalAddress;
-    // Don't add the placeholder if this is a stub
-    if (Offset < Section.Size)
-      RealOffset += Placeholder;
-    assert(RealOffset <= INT32_MAX && RealOffset >= INT32_MIN);
+    assert(isInt<32>(RealOffset));
     int32_t TruncOffset = (RealOffset & 0xFFFFFFFF);
     support::ulittle32_t::ref(Section.Address + Offset) = TruncOffset;
     break;
   }
   case ELF::R_X86_64_PC64: {
-    // Get the placeholder value from the generated object since
-    // a previous relocation attempt may have overwritten the loaded version
-    support::ulittle64_t::ref Placeholder(
-        (void *)(Section.ObjAddress + Offset));
     uint64_t FinalAddress = Section.LoadAddress + Offset;
     int64_t RealOffset = Value + Addend - FinalAddress;
-    if (Offset < Section.Size)
-      RealOffset += Placeholder;
     support::ulittle64_t::ref(Section.Address + Offset) = RealOffset;
     break;
   }
@@ -281,21 +272,12 @@ void RuntimeDyldELF::resolveX86Relocation(const SectionEntry &Section,
                                           uint32_t Type, int32_t Addend) {
   switch (Type) {
   case ELF::R_386_32: {
-    // Get the placeholder value from the generated object since
-    // a previous relocation attempt may have overwritten the loaded version
-    support::ulittle32_t::ref Placeholder(
-        (void *)(Section.ObjAddress + Offset));
-    support::ulittle32_t::ref(Section.Address + Offset) =
-        Placeholder + Value + Addend;
+    support::ulittle32_t::ref(Section.Address + Offset) = Value + Addend;
     break;
   }
   case ELF::R_386_PC32: {
-    // Get the placeholder value from the generated object since
-    // a previous relocation attempt may have overwritten the loaded version
-    support::ulittle32_t::ref Placeholder(
-        (void *)(Section.ObjAddress + Offset));
     uint32_t FinalAddress = ((Section.LoadAddress + Offset) & 0xFFFFFFFF);
-    uint32_t RealOffset = Placeholder + Value + Addend - FinalAddress;
+    uint32_t RealOffset = Value + Addend - FinalAddress;
     support::ulittle32_t::ref(Section.Address + Offset) = RealOffset;
     break;
   }
@@ -344,8 +326,7 @@ void RuntimeDyldELF::resolveAArch64Relocation(const SectionEntry &Section,
     uint64_t BranchImm = Value + Addend - FinalAddress;
 
     // "Check that -2^27 <= result < 2^27".
-    assert(-(1LL << 27) <= static_cast<int64_t>(BranchImm) &&
-           static_cast<int64_t>(BranchImm) < (1LL << 27));
+    assert(isInt<28>(BranchImm));
 
     // AArch64 code is emitted with .rela relocations. The data already in any
     // bits affected by the relocation on entry is garbage.
@@ -408,9 +389,7 @@ void RuntimeDyldELF::resolveAArch64Relocation(const SectionEntry &Section,
         ((Value + Addend) & ~0xfffULL) - (FinalAddress & ~0xfffULL);
 
     // Check that -2^32 <= X < 2^32
-    assert(static_cast<int64_t>(Result) >= (-1LL << 32) &&
-           static_cast<int64_t>(Result) < (1LL << 32) &&
-           "overflow check failed for relocation");
+    assert(isInt<33>(Result) && "overflow check failed for relocation");
 
     // AArch64 code is emitted with .rela relocations. The data already in any
     // bits affected by the relocation on entry is garbage.
@@ -452,8 +431,6 @@ void RuntimeDyldELF::resolveARMRelocation(const SectionEntry &Section,
                                           uint64_t Offset, uint32_t Value,
                                           uint32_t Type, int32_t Addend) {
   // TODO: Add Thumb relocations.
-  uint32_t *Placeholder =
-      reinterpret_cast<uint32_t *>(Section.ObjAddress + Offset);
   uint32_t *TargetPtr = (uint32_t *)(Section.Address + Offset);
   uint32_t FinalAddress = ((Section.LoadAddress + Offset) & 0xFFFFFFFF);
   Value += Addend;
@@ -470,39 +447,27 @@ void RuntimeDyldELF::resolveARMRelocation(const SectionEntry &Section,
 
   case ELF::R_ARM_NONE:
     break;
-  // Write a 32bit value to relocation address, taking into account the
-  // implicit addend encoded in the target.
   case ELF::R_ARM_PREL31:
   case ELF::R_ARM_TARGET1:
   case ELF::R_ARM_ABS32:
-    *TargetPtr = *Placeholder + Value;
+    *TargetPtr = Value;
     break;
-  // Write first 16 bit of 32 bit value to the mov instruction.
-  // Last 4 bit should be shifted.
+    // Write first 16 bit of 32 bit value to the mov instruction.
+    // Last 4 bit should be shifted.
   case ELF::R_ARM_MOVW_ABS_NC:
-    // We are not expecting any other addend in the relocation address.
-    // Using 0x000F0FFF because MOVW has its 16 bit immediate split into 2
-    // non-contiguous fields.
-    assert((*Placeholder & 0x000F0FFF) == 0);
-    Value = Value & 0xFFFF;
-    *TargetPtr = *Placeholder | (Value & 0xFFF);
-    *TargetPtr |= ((Value >> 12) & 0xF) << 16;
-    break;
-  // Write last 16 bit of 32 bit value to the mov instruction.
-  // Last 4 bit should be shifted.
   case ELF::R_ARM_MOVT_ABS:
-    // We are not expecting any other addend in the relocation address.
-    // Use 0x000F0FFF for the same reason as R_ARM_MOVW_ABS_NC.
-    assert((*Placeholder & 0x000F0FFF) == 0);
-
-    Value = (Value >> 16) & 0xFFFF;
-    *TargetPtr = *Placeholder | (Value & 0xFFF);
+    if (Type == ELF::R_ARM_MOVW_ABS_NC)
+      Value = Value & 0xFFFF;
+    else if (Type == ELF::R_ARM_MOVT_ABS)
+      Value = (Value >> 16) & 0xFFFF;
+    *TargetPtr &= ~0x000F0FFF;
+    *TargetPtr |= Value & 0xFFF;
     *TargetPtr |= ((Value >> 12) & 0xF) << 16;
     break;
-  // Write 24 bit relative value to the branch instruction.
+    // Write 24 bit relative value to the branch instruction.
   case ELF::R_ARM_PC24: // Fall through.
   case ELF::R_ARM_CALL: // Fall through.
-  case ELF::R_ARM_JUMP24: {
+  case ELF::R_ARM_JUMP24:
     int32_t RelValue = static_cast<int32_t>(Value - FinalAddress - 8);
     RelValue = (RelValue & 0x03FFFFFC) >> 2;
     assert((*TargetPtr & 0xFFFFFF) == 0xFFFFFE);
@@ -510,66 +475,296 @@ void RuntimeDyldELF::resolveARMRelocation(const SectionEntry &Section,
     *TargetPtr |= RelValue;
     break;
   }
-  case ELF::R_ARM_PRIVATE_0:
-    // This relocation is reserved by the ARM ELF ABI for internal use. We
-    // appropriate it here to act as an R_ARM_ABS32 without any addend for use
-    // in the stubs created during JIT (which can't put an addend into the
-    // original object file).
-    *TargetPtr = Value;
-    break;
-  }
 }
 
 void RuntimeDyldELF::resolveMIPSRelocation(const SectionEntry &Section,
                                            uint64_t Offset, uint32_t Value,
                                            uint32_t Type, int32_t Addend) {
-  uint32_t *Placeholder =
-      reinterpret_cast<uint32_t *>(Section.ObjAddress + Offset);
-  uint32_t *TargetPtr = (uint32_t *)(Section.Address + Offset);
+  uint8_t *TargetPtr = Section.Address + Offset;
   Value += Addend;
 
-  DEBUG(dbgs() << "resolveMipselocation, LocalAddress: "
+  DEBUG(dbgs() << "resolveMIPSRelocation, LocalAddress: "
                << Section.Address + Offset << " FinalAddress: "
                << format("%p", Section.LoadAddress + Offset) << " Value: "
                << format("%x", Value) << " Type: " << format("%x", Type)
                << " Addend: " << format("%x", Addend) << "\n");
 
+  uint32_t Insn = readBytesUnaligned(TargetPtr, 4);
+
   switch (Type) {
   default:
     llvm_unreachable("Not implemented relocation type!");
     break;
   case ELF::R_MIPS_32:
-    *TargetPtr = Value + (*Placeholder);
+    writeBytesUnaligned(Value, TargetPtr, 4);
     break;
   case ELF::R_MIPS_26:
-    *TargetPtr = ((*Placeholder) & 0xfc000000) | ((Value & 0x0fffffff) >> 2);
+    Insn &= 0xfc000000;
+    Insn |= (Value & 0x0fffffff) >> 2;
+    writeBytesUnaligned(Insn, TargetPtr, 4);
     break;
   case ELF::R_MIPS_HI16:
     // Get the higher 16-bits. Also add 1 if bit 15 is 1.
-    Value += ((*Placeholder) & 0x0000ffff) << 16;
-    *TargetPtr =
-        ((*Placeholder) & 0xffff0000) | (((Value + 0x8000) >> 16) & 0xffff);
+    Insn &= 0xffff0000;
+    Insn |= ((Value + 0x8000) >> 16) & 0xffff;
+    writeBytesUnaligned(Insn, TargetPtr, 4);
     break;
   case ELF::R_MIPS_LO16:
-    Value += ((*Placeholder) & 0x0000ffff);
-    *TargetPtr = ((*Placeholder) & 0xffff0000) | (Value & 0xffff);
+    Insn &= 0xffff0000;
+    Insn |= Value & 0xffff;
+    writeBytesUnaligned(Insn, TargetPtr, 4);
+    break;
+  case ELF::R_MIPS_PC32: {
+    uint32_t FinalAddress = (Section.LoadAddress + Offset);
+    writeBytesUnaligned(Value - FinalAddress, (uint8_t *)TargetPtr, 4);
     break;
-  case ELF::R_MIPS_UNUSED1:
-    // Similar to ELF::R_ARM_PRIVATE_0, R_MIPS_UNUSED1 and R_MIPS_UNUSED2
-    // are used for internal JIT purpose. These relocations are similar to
-    // R_MIPS_HI16 and R_MIPS_LO16, but they do not take any addend into
-    // account.
-    *TargetPtr =
-        ((*TargetPtr) & 0xffff0000) | (((Value + 0x8000) >> 16) & 0xffff);
+  }
+  case ELF::R_MIPS_PC16: {
+    uint32_t FinalAddress = (Section.LoadAddress + Offset);
+    Insn &= 0xffff0000;
+    Insn |= ((Value - FinalAddress) >> 2) & 0xffff;
+    writeBytesUnaligned(Insn, TargetPtr, 4);
     break;
-  case ELF::R_MIPS_UNUSED2:
-    *TargetPtr = ((*TargetPtr) & 0xffff0000) | (Value & 0xffff);
+  }
+  case ELF::R_MIPS_PC19_S2: {
+    uint32_t FinalAddress = (Section.LoadAddress + Offset);
+    Insn &= 0xfff80000;
+    Insn |= ((Value - (FinalAddress & ~0x3)) >> 2) & 0x7ffff;
+    writeBytesUnaligned(Insn, TargetPtr, 4);
     break;
   }
+  case ELF::R_MIPS_PC21_S2: {
+    uint32_t FinalAddress = (Section.LoadAddress + Offset);
+    Insn &= 0xffe00000;
+    Insn |= ((Value - FinalAddress) >> 2) & 0x1fffff;
+    writeBytesUnaligned(Insn, TargetPtr, 4);
+    break;
+  }
+  case ELF::R_MIPS_PC26_S2: {
+    uint32_t FinalAddress = (Section.LoadAddress + Offset);
+    Insn &= 0xfc000000;
+    Insn |= ((Value - FinalAddress) >> 2) & 0x3ffffff;
+    writeBytesUnaligned(Insn, TargetPtr, 4);
+    break;
+  }
+  case ELF::R_MIPS_PCHI16: {
+    uint32_t FinalAddress = (Section.LoadAddress + Offset);
+    Insn &= 0xffff0000;
+    Insn |= ((Value - FinalAddress + 0x8000) >> 16) & 0xffff;
+    writeBytesUnaligned(Insn, TargetPtr, 4);
+    break;
+  }
+  case ELF::R_MIPS_PCLO16: {
+    uint32_t FinalAddress = (Section.LoadAddress + Offset);
+    Insn &= 0xffff0000;
+    Insn |= (Value - FinalAddress) & 0xffff;
+    writeBytesUnaligned(Insn, TargetPtr, 4);
+    break;
+  }
+  }
+}
+
+void RuntimeDyldELF::setMipsABI(const ObjectFile &Obj) {
+  if (Arch == Triple::UnknownArch ||
+      !StringRef(Triple::getArchTypePrefix(Arch)).equals("mips")) {
+    IsMipsO32ABI = false;
+    IsMipsN64ABI = false;
+    return;
+  }
+  unsigned AbiVariant;
+  Obj.getPlatformFlags(AbiVariant);
+  IsMipsO32ABI = AbiVariant & ELF::EF_MIPS_ABI_O32;
+  IsMipsN64ABI = Obj.getFileFormatName().equals("ELF64-mips");
+  if (AbiVariant & ELF::EF_MIPS_ABI2)
+    llvm_unreachable("Mips N32 ABI is not supported yet");
+}
+
+void RuntimeDyldELF::resolveMIPS64Relocation(const SectionEntry &Section,
+                                             uint64_t Offset, uint64_t Value,
+                                             uint32_t Type, int64_t Addend,
+                                             uint64_t SymOffset,
+                                             SID SectionID) {
+  uint32_t r_type = Type & 0xff;
+  uint32_t r_type2 = (Type >> 8) & 0xff;
+  uint32_t r_type3 = (Type >> 16) & 0xff;
+
+  // RelType is used to keep information for which relocation type we are
+  // applying relocation.
+  uint32_t RelType = r_type;
+  int64_t CalculatedValue = evaluateMIPS64Relocation(Section, Offset, Value,
+                                                     RelType, Addend,
+                                                     SymOffset, SectionID);
+  if (r_type2 != ELF::R_MIPS_NONE) {
+    RelType = r_type2;
+    CalculatedValue = evaluateMIPS64Relocation(Section, Offset, 0, RelType,
+                                               CalculatedValue, SymOffset,
+                                               SectionID);
+  }
+  if (r_type3 != ELF::R_MIPS_NONE) {
+    RelType = r_type3;
+    CalculatedValue = evaluateMIPS64Relocation(Section, Offset, 0, RelType,
+                                               CalculatedValue, SymOffset,
+                                               SectionID);
+  }
+  applyMIPS64Relocation(Section.Address + Offset, CalculatedValue, RelType);
+}
+
+int64_t
+RuntimeDyldELF::evaluateMIPS64Relocation(const SectionEntry &Section,
+                                         uint64_t Offset, uint64_t Value,
+                                         uint32_t Type, int64_t Addend,
+                                         uint64_t SymOffset, SID SectionID) {
+
+  DEBUG(dbgs() << "evaluateMIPS64Relocation, LocalAddress: 0x"
+               << format("%llx", Section.Address + Offset)
+               << " FinalAddress: 0x"
+               << format("%llx", Section.LoadAddress + Offset)
+               << " Value: 0x" << format("%llx", Value) << " Type: 0x"
+               << format("%x", Type) << " Addend: 0x" << format("%llx", Addend)
+               << " SymOffset: " << format("%x", SymOffset)
+               << "\n");
+
+  switch (Type) {
+  default:
+    llvm_unreachable("Not implemented relocation type!");
+    break;
+  case ELF::R_MIPS_JALR:
+  case ELF::R_MIPS_NONE:
+    break;
+  case ELF::R_MIPS_32:
+  case ELF::R_MIPS_64:
+    return Value + Addend;
+  case ELF::R_MIPS_26:
+    return ((Value + Addend) >> 2) & 0x3ffffff;
+  case ELF::R_MIPS_GPREL16: {
+    uint64_t GOTAddr = getSectionLoadAddress(SectionToGOTMap[SectionID]);
+    return Value + Addend - (GOTAddr + 0x7ff0);
+  }
+  case ELF::R_MIPS_SUB:
+    return Value - Addend;
+  case ELF::R_MIPS_HI16:
+    // Get the higher 16-bits. Also add 1 if bit 15 is 1.
+    return ((Value + Addend + 0x8000) >> 16) & 0xffff;
+  case ELF::R_MIPS_LO16:
+    return (Value + Addend) & 0xffff;
+  case ELF::R_MIPS_CALL16:
+  case ELF::R_MIPS_GOT_DISP:
+  case ELF::R_MIPS_GOT_PAGE: {
+    uint8_t *LocalGOTAddr =
+        getSectionAddress(SectionToGOTMap[SectionID]) + SymOffset;
+    uint64_t GOTEntry = readBytesUnaligned(LocalGOTAddr, 8);
+
+    Value += Addend;
+    if (Type == ELF::R_MIPS_GOT_PAGE)
+      Value = (Value + 0x8000) & ~0xffff;
+
+    if (GOTEntry)
+      assert(GOTEntry == Value &&
+                   "GOT entry has two different addresses.");
+    else
+      writeBytesUnaligned(Value, LocalGOTAddr, 8);
+
+    return (SymOffset - 0x7ff0) & 0xffff;
+  }
+  case ELF::R_MIPS_GOT_OFST: {
+    int64_t page = (Value + Addend + 0x8000) & ~0xffff;
+    return (Value + Addend - page) & 0xffff;
+  }
+  case ELF::R_MIPS_GPREL32: {
+    uint64_t GOTAddr = getSectionLoadAddress(SectionToGOTMap[SectionID]);
+    return Value + Addend - (GOTAddr + 0x7ff0);
+  }
+  case ELF::R_MIPS_PC16: {
+    uint64_t FinalAddress = (Section.LoadAddress + Offset);
+    return ((Value + Addend - FinalAddress) >> 2) & 0xffff;
+  }
+  case ELF::R_MIPS_PC32: {
+    uint64_t FinalAddress = (Section.LoadAddress + Offset);
+    return Value + Addend - FinalAddress;
+  }
+  case ELF::R_MIPS_PC18_S3: {
+    uint64_t FinalAddress = (Section.LoadAddress + Offset);
+    return ((Value + Addend - ((FinalAddress | 7) ^ 7)) >> 3) & 0x3ffff;
+  }
+  case ELF::R_MIPS_PC19_S2: {
+    uint64_t FinalAddress = (Section.LoadAddress + Offset);
+    return ((Value + Addend - FinalAddress) >> 2) & 0x7ffff;
+  }
+  case ELF::R_MIPS_PC21_S2: {
+    uint64_t FinalAddress = (Section.LoadAddress + Offset);
+    return ((Value + Addend - FinalAddress) >> 2) & 0x1fffff;
+  }
+  case ELF::R_MIPS_PC26_S2: {
+    uint64_t FinalAddress = (Section.LoadAddress + Offset);
+    return ((Value + Addend - FinalAddress) >> 2) & 0x3ffffff;
+  }
+  case ELF::R_MIPS_PCHI16: {
+    uint64_t FinalAddress = (Section.LoadAddress + Offset);
+    return ((Value + Addend - FinalAddress + 0x8000) >> 16) & 0xffff;
+  }
+  case ELF::R_MIPS_PCLO16: {
+    uint64_t FinalAddress = (Section.LoadAddress + Offset);
+    return (Value + Addend - FinalAddress) & 0xffff;
+  }
+  }
+  return 0;
+}
+
+void RuntimeDyldELF::applyMIPS64Relocation(uint8_t *TargetPtr,
+                                           int64_t CalculatedValue,
+                                           uint32_t Type) {
+  uint32_t Insn = readBytesUnaligned(TargetPtr, 4);
+
+  switch (Type) {
+    default:
+      break;
+    case ELF::R_MIPS_32:
+    case ELF::R_MIPS_GPREL32:
+    case ELF::R_MIPS_PC32:
+      writeBytesUnaligned(CalculatedValue & 0xffffffff, TargetPtr, 4);
+      break;
+    case ELF::R_MIPS_64:
+    case ELF::R_MIPS_SUB:
+      writeBytesUnaligned(CalculatedValue, TargetPtr, 8);
+      break;
+    case ELF::R_MIPS_26:
+    case ELF::R_MIPS_PC26_S2:
+      Insn = (Insn & 0xfc000000) | CalculatedValue;
+      writeBytesUnaligned(Insn, TargetPtr, 4);
+      break;
+    case ELF::R_MIPS_GPREL16:
+      Insn = (Insn & 0xffff0000) | (CalculatedValue & 0xffff);
+      writeBytesUnaligned(Insn, TargetPtr, 4);
+      break;
+    case ELF::R_MIPS_HI16:
+    case ELF::R_MIPS_LO16:
+    case ELF::R_MIPS_PCHI16:
+    case ELF::R_MIPS_PCLO16:
+    case ELF::R_MIPS_PC16:
+    case ELF::R_MIPS_CALL16:
+    case ELF::R_MIPS_GOT_DISP:
+    case ELF::R_MIPS_GOT_PAGE:
+    case ELF::R_MIPS_GOT_OFST:
+      Insn = (Insn & 0xffff0000) | CalculatedValue;
+      writeBytesUnaligned(Insn, TargetPtr, 4);
+      break;
+    case ELF::R_MIPS_PC18_S3:
+      Insn = (Insn & 0xfffc0000) | CalculatedValue;
+      writeBytesUnaligned(Insn, TargetPtr, 4);
+      break;
+    case ELF::R_MIPS_PC19_S2:
+      Insn = (Insn & 0xfff80000) | CalculatedValue;
+      writeBytesUnaligned(Insn, TargetPtr, 4);
+      break;
+    case ELF::R_MIPS_PC21_S2:
+      Insn = (Insn & 0xffe00000) | CalculatedValue;
+      writeBytesUnaligned(Insn, TargetPtr, 4);
+      break;
+    }
 }
 
 // Return the .TOC. section and offset.
-void RuntimeDyldELF::findPPC64TOCSection(const ObjectFile &Obj,
+void RuntimeDyldELF::findPPC64TOCSection(const ELFObjectFileBase &Obj,
                                          ObjSectionToIDMap &LocalSections,
                                          RelocationValueRef &Rel) {
   // Set a default SectionID in case we do not find a TOC section below.
@@ -582,17 +777,15 @@ void RuntimeDyldELF::findPPC64TOCSection(const ObjectFile &Obj,
 
   // The TOC consists of sections .got, .toc, .tocbss, .plt in that
   // order. The TOC starts where the first of these sections starts.
-  for (section_iterator si = Obj.section_begin(), se = Obj.section_end();
-       si != se; ++si) {
-
+  for (auto &Section: Obj.sections()) {
     StringRef SectionName;
-    check(si->getName(SectionName));
+    check(Section.getName(SectionName));
 
     if (SectionName == ".got"
         || SectionName == ".toc"
         || SectionName == ".tocbss"
         || SectionName == ".plt") {
-      Rel.SectionID = findOrEmitSection(Obj, *si, false, LocalSections);
+      Rel.SectionID = findOrEmitSection(Obj, Section, false, LocalSections);
       break;
     }
   }
@@ -604,7 +797,7 @@ void RuntimeDyldELF::findPPC64TOCSection(const ObjectFile &Obj,
 
 // Returns the sections and offset associated with the ODP entry referenced
 // by Symbol.
-void RuntimeDyldELF::findOPDEntrySection(const ObjectFile &Obj,
+void RuntimeDyldELF::findOPDEntrySection(const ELFObjectFileBase &Obj,
                                          ObjSectionToIDMap &LocalSections,
                                          RelocationValueRef &Rel) {
   // Get the ELF symbol value (st_value) to compare with Relocation offset in
@@ -620,31 +813,29 @@ void RuntimeDyldELF::findOPDEntrySection(const ObjectFile &Obj,
     if (RelSectionName != ".opd")
       continue;
 
-    for (relocation_iterator i = si->relocation_begin(),
-                             e = si->relocation_end();
+    for (elf_relocation_iterator i = si->relocation_begin(),
+                                 e = si->relocation_end();
          i != e;) {
       // The R_PPC64_ADDR64 relocation indicates the first field
       // of a .opd entry
-      uint64_t TypeFunc;
-      check(i->getType(TypeFunc));
+      uint64_t TypeFunc = i->getType();
       if (TypeFunc != ELF::R_PPC64_ADDR64) {
         ++i;
         continue;
       }
 
-      uint64_t TargetSymbolOffset;
+      uint64_t TargetSymbolOffset = i->getOffset();
       symbol_iterator TargetSymbol = i->getSymbol();
-      check(i->getOffset(TargetSymbolOffset));
-      int64_t Addend;
-      check(getELFRelocationAddend(*i, Addend));
+      ErrorOr<int64_t> AddendOrErr = i->getAddend();
+      Check(AddendOrErr.getError());
+      int64_t Addend = *AddendOrErr;
 
       ++i;
       if (i == e)
         break;
 
       // Just check if following relocation is a R_PPC64_TOC
-      uint64_t TypeTOC;
-      check(i->getType(TypeTOC));
+      uint64_t TypeTOC = i->getType();
       if (TypeTOC != ELF::R_PPC64_TOC)
         continue;
 
@@ -845,13 +1036,13 @@ void RuntimeDyldELF::resolveRelocation(const RelocationEntry &RE,
                                        uint64_t Value) {
   const SectionEntry &Section = Sections[RE.SectionID];
   return resolveRelocation(Section, RE.Offset, Value, RE.RelType, RE.Addend,
-                           RE.SymOffset);
+                           RE.SymOffset, RE.SectionID);
 }
 
 void RuntimeDyldELF::resolveRelocation(const SectionEntry &Section,
                                        uint64_t Offset, uint64_t Value,
                                        uint32_t Type, int64_t Addend,
-                                       uint64_t SymOffset) {
+                                       uint64_t SymOffset, SID SectionID) {
   switch (Arch) {
   case Triple::x86_64:
     resolveX86_64Relocation(Section, Offset, Value, Type, Addend, SymOffset);
@@ -873,8 +1064,16 @@ void RuntimeDyldELF::resolveRelocation(const SectionEntry &Section,
     break;
   case Triple::mips: // Fall through.
   case Triple::mipsel:
-    resolveMIPSRelocation(Section, Offset, (uint32_t)(Value & 0xffffffffL),
-                          Type, (uint32_t)(Addend & 0xffffffffL));
+  case Triple::mips64:
+  case Triple::mips64el:
+    if (IsMipsO32ABI)
+      resolveMIPSRelocation(Section, Offset, (uint32_t)(Value & 0xffffffffL),
+                            Type, (uint32_t)(Addend & 0xffffffffL));
+    else if (IsMipsN64ABI)
+      resolveMIPS64Relocation(Section, Offset, Value, Type, Addend, SymOffset,
+                              SectionID);
+    else
+      llvm_unreachable("Mips ABI not handled");
     break;
   case Triple::ppc64: // Fall through.
   case Triple::ppc64le:
@@ -888,21 +1087,35 @@ void RuntimeDyldELF::resolveRelocation(const SectionEntry &Section,
   }
 }
 
+void *RuntimeDyldELF::computePlaceholderAddress(unsigned SectionID, uint64_t Offset) const {
+  return (void*)(Sections[SectionID].ObjAddress + Offset);
+}
+
+void RuntimeDyldELF::processSimpleRelocation(unsigned SectionID, uint64_t Offset, unsigned RelType, RelocationValueRef Value) {
+  RelocationEntry RE(SectionID, Offset, RelType, Value.Addend, Value.Offset);
+  if (Value.SymbolName)
+    addRelocationForSymbol(RE, Value.SymbolName);
+  else
+    addRelocationForSection(RE, Value.SectionID);
+}
+
 relocation_iterator RuntimeDyldELF::processRelocationRef(
-    unsigned SectionID, relocation_iterator RelI,
-    const ObjectFile &Obj,
-    ObjSectionToIDMap &ObjSectionToID,
-    StubMap &Stubs) {
-  uint64_t RelType;
-  Check(RelI->getType(RelType));
-  int64_t Addend;
-  Check(getELFRelocationAddend(*RelI, Addend));
-  symbol_iterator Symbol = RelI->getSymbol();
+    unsigned SectionID, relocation_iterator RelI, const ObjectFile &O,
+    ObjSectionToIDMap &ObjSectionToID, StubMap &Stubs) {
+  const auto &Obj = cast<ELFObjectFileBase>(O);
+  uint64_t RelType = RelI->getType();
+  ErrorOr<int64_t> AddendOrErr = ELFRelocationRef(*RelI).getAddend();
+  int64_t Addend = AddendOrErr ? *AddendOrErr : 0;
+  elf_symbol_iterator Symbol = RelI->getSymbol();
 
   // Obtain the symbol name which is referenced in the relocation
   StringRef TargetName;
-  if (Symbol != Obj.symbol_end())
-    Symbol->getName(TargetName);
+  if (Symbol != Obj.symbol_end()) {
+    ErrorOr<StringRef> TargetNameOrErr = Symbol->getName();
+    if (std::error_code EC = TargetNameOrErr.getError())
+      report_fatal_error(EC.message());
+    TargetName = *TargetNameOrErr;
+  }
   DEBUG(dbgs() << "\t\tRelType: " << RelType << " Addend: " << Addend
                << " TargetName: " << TargetName << "\n");
   RelocationValueRef Value;
@@ -913,7 +1126,7 @@ relocation_iterator RuntimeDyldELF::processRelocationRef(
   RTDyldSymbolTable::const_iterator gsi = GlobalSymbolTable.end();
   if (Symbol != Obj.symbol_end()) {
     gsi = GlobalSymbolTable.find(TargetName.data());
-    Symbol->getType(SymType);
+    SymType = Symbol->getType();
   }
   if (gsi != GlobalSymbolTable.end()) {
     const auto &SymInfo = gsi->second;
@@ -955,8 +1168,7 @@ relocation_iterator RuntimeDyldELF::processRelocationRef(
     }
   }
 
-  uint64_t Offset;
-  Check(RelI->getOffset(Offset));
+  uint64_t Offset = RelI->getOffset();
 
   DEBUG(dbgs() << "\t\tSectionID: " << SectionID << " Offset: " << Offset
                << "\n");
@@ -1005,81 +1217,135 @@ relocation_iterator RuntimeDyldELF::processRelocationRef(
                         0);
       Section.StubOffset += getMaxStubSize();
     }
-  } else if (Arch == Triple::arm &&
-             (RelType == ELF::R_ARM_PC24 || RelType == ELF::R_ARM_CALL ||
-              RelType == ELF::R_ARM_JUMP24)) {
-    // This is an ARM branch relocation, need to use a stub function.
-    DEBUG(dbgs() << "\t\tThis is an ARM branch relocation.");
-    SectionEntry &Section = Sections[SectionID];
+  } else if (Arch == Triple::arm) {
+    if (RelType == ELF::R_ARM_PC24 || RelType == ELF::R_ARM_CALL ||
+      RelType == ELF::R_ARM_JUMP24) {
+      // This is an ARM branch relocation, need to use a stub function.
+      DEBUG(dbgs() << "\t\tThis is an ARM branch relocation.");
+      SectionEntry &Section = Sections[SectionID];
 
-    // Look for an existing stub.
-    StubMap::const_iterator i = Stubs.find(Value);
-    if (i != Stubs.end()) {
-      resolveRelocation(Section, Offset, (uint64_t)Section.Address + i->second,
-                        RelType, 0);
-      DEBUG(dbgs() << " Stub function found\n");
-    } else {
-      // Create a new stub function.
-      DEBUG(dbgs() << " Create a new stub function\n");
-      Stubs[Value] = Section.StubOffset;
-      uint8_t *StubTargetAddr =
+      // Look for an existing stub.
+      StubMap::const_iterator i = Stubs.find(Value);
+      if (i != Stubs.end()) {
+        resolveRelocation(Section, Offset, (uint64_t)Section.Address + i->second,
+          RelType, 0);
+        DEBUG(dbgs() << " Stub function found\n");
+      } else {
+        // Create a new stub function.
+        DEBUG(dbgs() << " Create a new stub function\n");
+        Stubs[Value] = Section.StubOffset;
+        uint8_t *StubTargetAddr =
           createStubFunction(Section.Address + Section.StubOffset);
-      RelocationEntry RE(SectionID, StubTargetAddr - Section.Address,
-                         ELF::R_ARM_PRIVATE_0, Value.Addend);
-      if (Value.SymbolName)
-        addRelocationForSymbol(RE, Value.SymbolName);
-      else
-        addRelocationForSection(RE, Value.SectionID);
-
-      resolveRelocation(Section, Offset,
-                        (uint64_t)Section.Address + Section.StubOffset, RelType,
-                        0);
-      Section.StubOffset += getMaxStubSize();
+        RelocationEntry RE(SectionID, StubTargetAddr - Section.Address,
+          ELF::R_ARM_ABS32, Value.Addend);
+        if (Value.SymbolName)
+          addRelocationForSymbol(RE, Value.SymbolName);
+        else
+          addRelocationForSection(RE, Value.SectionID);
+
+        resolveRelocation(Section, Offset,
+          (uint64_t)Section.Address + Section.StubOffset, RelType,
+          0);
+        Section.StubOffset += getMaxStubSize();
+      }
+    } else {
+      uint32_t *Placeholder =
+        reinterpret_cast<uint32_t*>(computePlaceholderAddress(SectionID, Offset));
+      if (RelType == ELF::R_ARM_PREL31 || RelType == ELF::R_ARM_TARGET1 ||
+          RelType == ELF::R_ARM_ABS32) {
+        Value.Addend += *Placeholder;
+      } else if (RelType == ELF::R_ARM_MOVW_ABS_NC || RelType == ELF::R_ARM_MOVT_ABS) {
+        // See ELF for ARM documentation
+        Value.Addend += (int16_t)((*Placeholder & 0xFFF) | (((*Placeholder >> 16) & 0xF) << 12));
+      }
+      processSimpleRelocation(SectionID, Offset, RelType, Value);
     }
-  } else if ((Arch == Triple::mipsel || Arch == Triple::mips) &&
-             RelType == ELF::R_MIPS_26) {
-    // This is an Mips branch relocation, need to use a stub function.
-    DEBUG(dbgs() << "\t\tThis is a Mips branch relocation.");
-    SectionEntry &Section = Sections[SectionID];
-    uint8_t *Target = Section.Address + Offset;
-    uint32_t *TargetAddress = (uint32_t *)Target;
+  } else if (IsMipsO32ABI) {
+    uint8_t *Placeholder = reinterpret_cast<uint8_t *>(
+        computePlaceholderAddress(SectionID, Offset));
+    uint32_t Opcode = readBytesUnaligned(Placeholder, 4);
+    if (RelType == ELF::R_MIPS_26) {
+      // This is an Mips branch relocation, need to use a stub function.
+      DEBUG(dbgs() << "\t\tThis is a Mips branch relocation.");
+      SectionEntry &Section = Sections[SectionID];
 
-    // Extract the addend from the instruction.
-    uint32_t Addend = ((*TargetAddress) & 0x03ffffff) << 2;
+      // Extract the addend from the instruction.
+      // We shift up by two since the Value will be down shifted again
+      // when applying the relocation.
+      uint32_t Addend = (Opcode & 0x03ffffff) << 2;
 
-    Value.Addend += Addend;
+      Value.Addend += Addend;
 
-    //  Look up for existing stub.
-    StubMap::const_iterator i = Stubs.find(Value);
-    if (i != Stubs.end()) {
-      RelocationEntry RE(SectionID, Offset, RelType, i->second);
-      addRelocationForSection(RE, SectionID);
-      DEBUG(dbgs() << " Stub function found\n");
-    } else {
-      // Create a new stub function.
-      DEBUG(dbgs() << " Create a new stub function\n");
-      Stubs[Value] = Section.StubOffset;
-      uint8_t *StubTargetAddr =
+      //  Look up for existing stub.
+      StubMap::const_iterator i = Stubs.find(Value);
+      if (i != Stubs.end()) {
+        RelocationEntry RE(SectionID, Offset, RelType, i->second);
+        addRelocationForSection(RE, SectionID);
+        DEBUG(dbgs() << " Stub function found\n");
+      } else {
+        // Create a new stub function.
+        DEBUG(dbgs() << " Create a new stub function\n");
+        Stubs[Value] = Section.StubOffset;
+        uint8_t *StubTargetAddr =
           createStubFunction(Section.Address + Section.StubOffset);
 
-      // Creating Hi and Lo relocations for the filled stub instructions.
-      RelocationEntry REHi(SectionID, StubTargetAddr - Section.Address,
-                           ELF::R_MIPS_UNUSED1, Value.Addend);
-      RelocationEntry RELo(SectionID, StubTargetAddr - Section.Address + 4,
-                           ELF::R_MIPS_UNUSED2, Value.Addend);
+        // Creating Hi and Lo relocations for the filled stub instructions.
+        RelocationEntry REHi(SectionID, StubTargetAddr - Section.Address,
+          ELF::R_MIPS_HI16, Value.Addend);
+        RelocationEntry RELo(SectionID, StubTargetAddr - Section.Address + 4,
+          ELF::R_MIPS_LO16, Value.Addend);
 
-      if (Value.SymbolName) {
-        addRelocationForSymbol(REHi, Value.SymbolName);
-        addRelocationForSymbol(RELo, Value.SymbolName);
-      } else {
-        addRelocationForSection(REHi, Value.SectionID);
-        addRelocationForSection(RELo, Value.SectionID);
-      }
+        if (Value.SymbolName) {
+          addRelocationForSymbol(REHi, Value.SymbolName);
+          addRelocationForSymbol(RELo, Value.SymbolName);
+        }
+        else {
+          addRelocationForSection(REHi, Value.SectionID);
+          addRelocationForSection(RELo, Value.SectionID);
+        }
 
-      RelocationEntry RE(SectionID, Offset, RelType, Section.StubOffset);
-      addRelocationForSection(RE, SectionID);
-      Section.StubOffset += getMaxStubSize();
+        RelocationEntry RE(SectionID, Offset, RelType, Section.StubOffset);
+        addRelocationForSection(RE, SectionID);
+        Section.StubOffset += getMaxStubSize();
+      }
+    } else {
+      // FIXME: Calculate correct addends for R_MIPS_HI16, R_MIPS_LO16,
+      // R_MIPS_PCHI16 and R_MIPS_PCLO16 relocations.
+      if (RelType == ELF::R_MIPS_HI16 || RelType == ELF::R_MIPS_PCHI16)
+        Value.Addend += (Opcode & 0x0000ffff) << 16;
+      else if (RelType == ELF::R_MIPS_LO16)
+        Value.Addend += (Opcode & 0x0000ffff);
+      else if (RelType == ELF::R_MIPS_32)
+        Value.Addend += Opcode;
+      else if (RelType == ELF::R_MIPS_PCLO16)
+        Value.Addend += SignExtend32<16>((Opcode & 0x0000ffff));
+      else if (RelType == ELF::R_MIPS_PC16)
+        Value.Addend += SignExtend32<18>((Opcode & 0x0000ffff) << 2);
+      else if (RelType == ELF::R_MIPS_PC19_S2)
+        Value.Addend += SignExtend32<21>((Opcode & 0x0007ffff) << 2);
+      else if (RelType == ELF::R_MIPS_PC21_S2)
+        Value.Addend += SignExtend32<23>((Opcode & 0x001fffff) << 2);
+      else if (RelType == ELF::R_MIPS_PC26_S2)
+        Value.Addend += SignExtend32<28>((Opcode & 0x03ffffff) << 2);
+      processSimpleRelocation(SectionID, Offset, RelType, Value);
     }
+  } else if (IsMipsN64ABI) {
+    uint32_t r_type = RelType & 0xff;
+    RelocationEntry RE(SectionID, Offset, RelType, Value.Addend);
+    if (r_type == ELF::R_MIPS_CALL16 || r_type == ELF::R_MIPS_GOT_PAGE
+        || r_type == ELF::R_MIPS_GOT_DISP) {
+      StringMap<uint64_t>::iterator i = GOTSymbolOffsets.find(TargetName);
+      if (i != GOTSymbolOffsets.end())
+        RE.SymOffset = i->second;
+      else {
+        RE.SymOffset = allocateGOTEntries(SectionID, 1);
+        GOTSymbolOffsets[TargetName] = RE.SymOffset;
+      }
+    }
+    if (Value.SymbolName)
+      addRelocationForSymbol(RE, Value.SymbolName);
+    else
+      addRelocationForSection(RE, Value.SectionID);
   } else if (Arch == Triple::ppc64 || Arch == Triple::ppc64le) {
     if (RelType == ELF::R_PPC64_REL24) {
       // Determine ABI variant in use for this object.
@@ -1101,8 +1367,7 @@ relocation_iterator RuntimeDyldELF::processRelocationRef(
         } else {
           // In the ELFv2 ABI, a function symbol may provide a local entry
           // point, which must be used for direct calls.
-          uint8_t SymOther;
-          Symbol->getOther(SymOther);
+          uint8_t SymOther = Symbol->getOther();
           Value.Addend += ELF::decodePPC64LocalEntryOffset(SymOther);
         }
         uint8_t *RelocTarget = Sections[Value.SectionID].Address + Value.Addend;
@@ -1277,40 +1542,41 @@ relocation_iterator RuntimeDyldELF::processRelocationRef(
                         Addend);
     else
       resolveRelocation(Section, Offset, StubAddress, RelType, Addend);
-  } else if (Arch == Triple::x86_64 && RelType == ELF::R_X86_64_PLT32) {
-    // The way the PLT relocations normally work is that the linker allocates
-    // the
-    // PLT and this relocation makes a PC-relative call into the PLT.  The PLT
-    // entry will then jump to an address provided by the GOT.  On first call,
-    // the
-    // GOT address will point back into PLT code that resolves the symbol. After
-    // the first call, the GOT entry points to the actual function.
-    //
-    // For local functions we're ignoring all of that here and just replacing
-    // the PLT32 relocation type with PC32, which will translate the relocation
-    // into a PC-relative call directly to the function. For external symbols we
-    // can't be sure the function will be within 2^32 bytes of the call site, so
-    // we need to create a stub, which calls into the GOT.  This case is
-    // equivalent to the usual PLT implementation except that we use the stub
-    // mechanism in RuntimeDyld (which puts stubs at the end of the section)
-    // rather than allocating a PLT section.
-    if (Value.SymbolName) {
-      // This is a call to an external function.
-      // Look for an existing stub.
-      SectionEntry &Section = Sections[SectionID];
-      StubMap::const_iterator i = Stubs.find(Value);
-      uintptr_t StubAddress;
-      if (i != Stubs.end()) {
+  } else if (Arch == Triple::x86_64) {
+    if (RelType == ELF::R_X86_64_PLT32) {
+      // The way the PLT relocations normally work is that the linker allocates
+      // the
+      // PLT and this relocation makes a PC-relative call into the PLT.  The PLT
+      // entry will then jump to an address provided by the GOT.  On first call,
+      // the
+      // GOT address will point back into PLT code that resolves the symbol. After
+      // the first call, the GOT entry points to the actual function.
+      //
+      // For local functions we're ignoring all of that here and just replacing
+      // the PLT32 relocation type with PC32, which will translate the relocation
+      // into a PC-relative call directly to the function. For external symbols we
+      // can't be sure the function will be within 2^32 bytes of the call site, so
+      // we need to create a stub, which calls into the GOT.  This case is
+      // equivalent to the usual PLT implementation except that we use the stub
+      // mechanism in RuntimeDyld (which puts stubs at the end of the section)
+      // rather than allocating a PLT section.
+      if (Value.SymbolName) {
+        // This is a call to an external function.
+        // Look for an existing stub.
+        SectionEntry &Section = Sections[SectionID];
+        StubMap::const_iterator i = Stubs.find(Value);
+        uintptr_t StubAddress;
+        if (i != Stubs.end()) {
         StubAddress = uintptr_t(Section.Address) + i->second;
         DEBUG(dbgs() << " Stub function found\n");
-      } else {
+        } else {
         // Create a new stub function (equivalent to a PLT entry).
         DEBUG(dbgs() << " Create a new stub function\n");
 
         uintptr_t BaseAddress = uintptr_t(Section.Address);
         uintptr_t StubAlignment = getStubAlignment();
         StubAddress = (BaseAddress + Section.StubOffset + StubAlignment - 1) &
-                      -StubAlignment;
+                -StubAlignment;
         unsigned StubOffset = StubAddress - BaseAddress;
         Stubs[Value] = StubOffset;
         createStubFunction((uint8_t *)StubAddress);
@@ -1326,33 +1592,41 @@ relocation_iterator RuntimeDyldELF::processRelocationRef(
 
         // Fill in the value of the symbol we're targeting into the GOT
         addRelocationForSymbol(computeGOTOffsetRE(SectionID,GOTOffset,0,ELF::R_X86_64_64),
-            Value.SymbolName);
+          Value.SymbolName);
+        }
+
+        // Make the target call a call into the stub table.
+        resolveRelocation(Section, Offset, StubAddress, ELF::R_X86_64_PC32,
+                Addend);
+      } else {
+        RelocationEntry RE(SectionID, Offset, ELF::R_X86_64_PC32, Value.Addend,
+                  Value.Offset);
+        addRelocationForSection(RE, Value.SectionID);
       }
+    } else if (RelType == ELF::R_X86_64_GOTPCREL) {
+      uint64_t GOTOffset = allocateGOTEntries(SectionID, 1);
+      resolveGOTOffsetRelocation(SectionID, Offset, GOTOffset + Addend);
 
-      // Make the target call a call into the stub table.
-      resolveRelocation(Section, Offset, StubAddress, ELF::R_X86_64_PC32,
-                        Addend);
+      // Fill in the value of the symbol we're targeting into the GOT
+      RelocationEntry RE = computeGOTOffsetRE(SectionID, GOTOffset, Value.Offset, ELF::R_X86_64_64);
+      if (Value.SymbolName)
+        addRelocationForSymbol(RE, Value.SymbolName);
+      else
+        addRelocationForSection(RE, Value.SectionID);
+    } else if (RelType == ELF::R_X86_64_PC32) {
+      Value.Addend += support::ulittle32_t::ref(computePlaceholderAddress(SectionID, Offset));
+      processSimpleRelocation(SectionID, Offset, RelType, Value);
+    } else if (RelType == ELF::R_X86_64_PC64) {
+      Value.Addend += support::ulittle64_t::ref(computePlaceholderAddress(SectionID, Offset));
+      processSimpleRelocation(SectionID, Offset, RelType, Value);
     } else {
-      RelocationEntry RE(SectionID, Offset, ELF::R_X86_64_PC32, Value.Addend,
-                         Value.Offset);
-      addRelocationForSection(RE, Value.SectionID);
+      processSimpleRelocation(SectionID, Offset, RelType, Value);
     }
-  } else if (Arch == Triple::x86_64 && RelType == ELF::R_X86_64_GOTPCREL) {
-    uint64_t GOTOffset = allocateGOTEntries(SectionID, 1);
-    resolveGOTOffsetRelocation(SectionID, Offset, GOTOffset + Addend);
-
-    // Fill in the value of the symbol we're targeting into the GOT
-    RelocationEntry RE = computeGOTOffsetRE(SectionID, GOTOffset, Value.Offset, ELF::R_X86_64_64);
-    if (Value.SymbolName)
-      addRelocationForSymbol(RE, Value.SymbolName);
-    else
-      addRelocationForSection(RE, Value.SectionID);
   } else {
-    RelocationEntry RE(SectionID, Offset, RelType, Value.Addend, Value.Offset);
-    if (Value.SymbolName)
-      addRelocationForSymbol(RE, Value.SymbolName);
-    else
-      addRelocationForSection(RE, Value.SectionID);
+    if (Arch == Triple::x86) {
+      Value.Addend += support::ulittle32_t::ref(computePlaceholderAddress(SectionID, Offset));
+    }
+    processSimpleRelocation(SectionID, Offset, RelType, Value);
   }
   return ++RelI;
 }
@@ -1373,9 +1647,18 @@ size_t RuntimeDyldELF::getGOTEntrySize() {
   case Triple::x86:
   case Triple::arm:
   case Triple::thumb:
+    Result = sizeof(uint32_t);
+    break;
   case Triple::mips:
   case Triple::mipsel:
-    Result = sizeof(uint32_t);
+  case Triple::mips64:
+  case Triple::mips64el:
+    if (IsMipsO32ABI)
+      Result = sizeof(uint32_t);
+    else if (IsMipsN64ABI)
+      Result = sizeof(uint64_t);
+    else
+      llvm_unreachable("Mips ABI not handled");
     break;
   default:
     llvm_unreachable("Unsupported CPU type!");
@@ -1430,6 +1713,20 @@ void RuntimeDyldELF::finalizeLoad(const ObjectFile &Obj,
     // For now, initialize all GOT entries to zero.  We'll fill them in as
     // needed when GOT-based relocations are applied.
     memset(Addr, 0, TotalSize);
+    if (IsMipsN64ABI) {
+      // To correctly resolve Mips GOT relocations, we need a mapping from
+      // object's sections to GOTs.
+      for (section_iterator SI = Obj.section_begin(), SE = Obj.section_end();
+           SI != SE; ++SI) {
+        if (SI->relocation_begin() != SI->relocation_end()) {
+          section_iterator RelocatedSection = SI->getRelocatedSection();
+          ObjSectionToIDMap::iterator i = SectionMap.find(*RelocatedSection);
+          assert (i != SectionMap.end());
+          SectionToGOTMap[i->second] = GOTSectionID;
+        }
+      }
+      GOTSymbolOffsets.clear();
+    }
   }
 
   // Look for and record the EH frame section.
diff --git a/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldELF.h b/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldELF.h
index 590d26a3fb8..1a2552deed9 100644
--- a/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldELF.h
+++ b/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldELF.h
@@ -25,7 +25,7 @@ class RuntimeDyldELF : public RuntimeDyldImpl {
 
   void resolveRelocation(const SectionEntry &Section, uint64_t Offset,
                          uint64_t Value, uint32_t Type, int64_t Addend,
-                         uint64_t SymOffset = 0);
+                         uint64_t SymOffset = 0, SID SectionID = 0);
 
   void resolveX86_64Relocation(const SectionEntry &Section, uint64_t Offset,
                                uint64_t Value, uint32_t Type, int64_t Addend,
@@ -49,12 +49,24 @@ class RuntimeDyldELF : public RuntimeDyldImpl {
   void resolveSystemZRelocation(const SectionEntry &Section, uint64_t Offset,
                                 uint64_t Value, uint32_t Type, int64_t Addend);
 
+  void resolveMIPS64Relocation(const SectionEntry &Section, uint64_t Offset,
+                               uint64_t Value, uint32_t Type, int64_t Addend,
+                               uint64_t SymOffset, SID SectionID);
+
+  int64_t evaluateMIPS64Relocation(const SectionEntry &Section,
+                                   uint64_t Offset, uint64_t Value,
+                                   uint32_t Type,  int64_t Addend,
+                                   uint64_t SymOffset, SID SectionID);
+
+  void applyMIPS64Relocation(uint8_t *TargetPtr, int64_t CalculatedValue,
+                             uint32_t Type);
+
   unsigned getMaxStubSize() override {
     if (Arch == Triple::aarch64 || Arch == Triple::aarch64_be)
       return 20; // movz; movk; movk; movk; br
     if (Arch == Triple::arm || Arch == Triple::thumb)
       return 8; // 32-bit instruction and 32-bit address
-    else if (Arch == Triple::mipsel || Arch == Triple::mips)
+    else if (IsMipsO32ABI)
       return 16;
     else if (Arch == Triple::ppc64 || Arch == Triple::ppc64le)
       return 44;
@@ -73,10 +85,12 @@ class RuntimeDyldELF : public RuntimeDyldImpl {
       return 1;
   }
 
-  void findPPC64TOCSection(const ObjectFile &Obj,
+  void setMipsABI(const ObjectFile &Obj) override;
+
+  void findPPC64TOCSection(const ELFObjectFileBase &Obj,
                            ObjSectionToIDMap &LocalSections,
                            RelocationValueRef &Rel);
-  void findOPDEntrySection(const ObjectFile &Obj,
+  void findOPDEntrySection(const ELFObjectFileBase &Obj,
                            ObjSectionToIDMap &LocalSections,
                            RelocationValueRef &Rel);
 
@@ -99,6 +113,13 @@ class RuntimeDyldELF : public RuntimeDyldImpl {
                                      uint64_t SymbolOffset,
                                      unsigned Type);
 
+  // Compute the address in memory where we can find the placeholder
+  void *computePlaceholderAddress(unsigned SectionID, uint64_t Offset) const;
+
+  // Split out common case for createing the RelocationEntry for when the relocation requires
+  // no particular advanced processing.
+  void processSimpleRelocation(unsigned SectionID, uint64_t Offset, unsigned RelType, RelocationValueRef Value);
+
   // The tentative ID for the GOT section
   unsigned GOTSectionID;
 
@@ -107,6 +128,13 @@ class RuntimeDyldELF : public RuntimeDyldImpl {
   // that consume more than one slot)
   unsigned CurrentGOTIndex;
 
+  // A map from section to a GOT section that has entries for section's GOT
+  // relocations. (Mips64 specific)
+  DenseMap<SID, SID> SectionToGOTMap;
+
+  // A map to avoid duplicate got entries (Mips64 specific)
+  StringMap<uint64_t> GOTSymbolOffsets;
+
   // When a module is loaded we save the SectionID of the EH frame section
   // in a table until we receive a request to register all unregistered
   // EH frame sections with the memory manager.
diff --git a/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldImpl.h b/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldImpl.h
index ee51a754ead..518df12b67e 100644
--- a/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldImpl.h
+++ b/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldImpl.h
@@ -203,7 +203,7 @@ class RuntimeDyldImpl {
   SectionList Sections;
 
   typedef unsigned SID; // Type for SectionIDs
-#define RTDYLD_INVALID_SECTION_ID ((SID)(-1))
+#define RTDYLD_INVALID_SECTION_ID ((RuntimeDyldImpl::SID)(-1))
 
   // Keep a map of sections from object file to the SectionID which
   // references it.
@@ -236,6 +236,8 @@ class RuntimeDyldImpl {
 
   Triple::ArchType Arch;
   bool IsTargetLittleEndian;
+  bool IsMipsO32ABI;
+  bool IsMipsN64ABI;
 
   // True if all sections should be passed to the memory manager, false if only
   // sections containing relocations should be. Defaults to 'false'.
@@ -303,6 +305,11 @@ class RuntimeDyldImpl {
     *(Addr + 7) = Value & 0xFF;
   }
 
+  virtual void setMipsABI(const ObjectFile &Obj) {
+    IsMipsO32ABI = false;
+    IsMipsN64ABI = false;
+  }
+
   /// Endian-aware read Read the least significant Size bytes from Src.
   uint64_t readBytesUnaligned(uint8_t *Src, unsigned Size) const;
 
@@ -371,7 +378,7 @@ class RuntimeDyldImpl {
                                      const SectionRef &Section);
 
   // \brief Implementation of the generic part of the loadObject algorithm.
-  std::pair<unsigned, unsigned> loadObjectImpl(const object::ObjectFile &Obj);
+  ObjSectionToIDMap loadObjectImpl(const object::ObjectFile &Obj);
 
 public:
   RuntimeDyldImpl(RuntimeDyld::MemoryManager &MemMgr,
diff --git a/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldMachO.cpp b/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldMachO.cpp
index 675063c81d6..c9bd11a1e1d 100644
--- a/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldMachO.cpp
+++ b/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldMachO.cpp
@@ -26,11 +26,12 @@ using namespace llvm::object;
 
 namespace {
 
-class LoadedMachOObjectInfo : public RuntimeDyld::LoadedObjectInfo {
+class LoadedMachOObjectInfo
+    : public RuntimeDyld::LoadedObjectInfoHelper<LoadedMachOObjectInfo> {
 public:
-  LoadedMachOObjectInfo(RuntimeDyldImpl &RTDyld, unsigned BeginIdx,
-                        unsigned EndIdx)
-    : RuntimeDyld::LoadedObjectInfo(RTDyld, BeginIdx, EndIdx) {}
+  LoadedMachOObjectInfo(RuntimeDyldImpl &RTDyld,
+                        ObjSectionToIDMap ObjSecToIDMap)
+      : LoadedObjectInfoHelper(RTDyld, std::move(ObjSecToIDMap)) {}
 
   OwningBinary<ObjectFile>
   getObjectForDebug(const ObjectFile &Obj) const override {
@@ -49,6 +50,42 @@ int64_t RuntimeDyldMachO::memcpyAddend(const RelocationEntry &RE) const {
   return static_cast<int64_t>(readBytesUnaligned(Src, NumBytes));
 }
 
+relocation_iterator RuntimeDyldMachO::processScatteredVANILLA(
+                          unsigned SectionID, relocation_iterator RelI,
+                          const ObjectFile &BaseObjT,
+                          RuntimeDyldMachO::ObjSectionToIDMap &ObjSectionToID) {
+  const MachOObjectFile &Obj =
+    static_cast<const MachOObjectFile&>(BaseObjT);
+  MachO::any_relocation_info RE =
+    Obj.getRelocation(RelI->getRawDataRefImpl());
+
+  SectionEntry &Section = Sections[SectionID];
+  uint32_t RelocType = Obj.getAnyRelocationType(RE);
+  bool IsPCRel = Obj.getAnyRelocationPCRel(RE);
+  unsigned Size = Obj.getAnyRelocationLength(RE);
+  uint64_t Offset = RelI->getOffset();
+  uint8_t *LocalAddress = Section.Address + Offset;
+  unsigned NumBytes = 1 << Size;
+  int64_t Addend = readBytesUnaligned(LocalAddress, NumBytes);
+
+  unsigned SymbolBaseAddr = Obj.getScatteredRelocationValue(RE);
+  section_iterator TargetSI = getSectionByAddress(Obj, SymbolBaseAddr);
+  assert(TargetSI != Obj.section_end() && "Can't find section for symbol");
+  uint64_t SectionBaseAddr = TargetSI->getAddress();
+  SectionRef TargetSection = *TargetSI;
+  bool IsCode = TargetSection.isText();
+  uint32_t TargetSectionID =
+    findOrEmitSection(Obj, TargetSection, IsCode, ObjSectionToID);
+
+  Addend -= SectionBaseAddr;
+  RelocationEntry R(SectionID, Offset, RelocType, Addend, IsPCRel, Size);
+
+  addRelocationForSection(R, TargetSectionID);
+
+  return ++RelI;
+}
+
+
 RelocationValueRef RuntimeDyldMachO::getRelocationValueRef(
     const ObjectFile &BaseTObj, const relocation_iterator &RI,
     const RelocationEntry &RE, ObjSectionToIDMap &ObjSectionToID) {
@@ -62,8 +99,10 @@ RelocationValueRef RuntimeDyldMachO::getRelocationValueRef(
   bool IsExternal = Obj.getPlainRelocationExternal(RelInfo);
   if (IsExternal) {
     symbol_iterator Symbol = RI->getSymbol();
-    StringRef TargetName;
-    Symbol->getName(TargetName);
+    ErrorOr<StringRef> TargetNameOrErr = Symbol->getName();
+    if (std::error_code EC = TargetNameOrErr.getError())
+      report_fatal_error(EC.message());
+    StringRef TargetName = *TargetNameOrErr;
     RTDyldSymbolTable::const_iterator SI =
       GlobalSymbolTable.find(TargetName.data());
     if (SI != GlobalSymbolTable.end()) {
@@ -75,7 +114,7 @@ RelocationValueRef RuntimeDyldMachO::getRelocationValueRef(
       Value.Offset = RE.Addend;
     }
   } else {
-    SectionRef Sec = Obj.getRelocationSection(RelInfo);
+    SectionRef Sec = Obj.getAnyRelocationSection(RelInfo);
     bool IsCode = Sec.isText();
     Value.SectionID = findOrEmitSection(Obj, Sec, IsCode, ObjSectionToID);
     uint64_t Addr = Sec.getAddress();
@@ -86,20 +125,11 @@ RelocationValueRef RuntimeDyldMachO::getRelocationValueRef(
 }
 
 void RuntimeDyldMachO::makeValueAddendPCRel(RelocationValueRef &Value,
-                                            const ObjectFile &BaseTObj,
                                             const relocation_iterator &RI,
                                             unsigned OffsetToNextPC) {
-  const MachOObjectFile &Obj =
-      static_cast<const MachOObjectFile &>(BaseTObj);
-  MachO::any_relocation_info RelInfo =
-      Obj.getRelocation(RI->getRawDataRefImpl());
-
-  bool IsPCRel = Obj.getAnyRelocationPCRel(RelInfo);
-  if (IsPCRel) {
-    uint64_t RelocAddr = 0;
-    RI->getAddress(RelocAddr);
-    Value.Offset += RelocAddr + OffsetToNextPC;
-  }
+  auto &O = *cast<MachOObjectFile>(RI->getObject());
+  section_iterator SecI = O.getRelocationRelocatedSection(RI);
+  Value.Offset += RI->getOffset() + OffsetToNextPC + SecI->getAddress();
 }
 
 void RuntimeDyldMachO::dumpRelocationToResolve(const RelocationEntry &RE,
@@ -162,8 +192,10 @@ void RuntimeDyldMachO::populateIndirectSymbolPointersSection(
     unsigned SymbolIndex =
       Obj.getIndirectSymbolTableEntry(DySymTabCmd, FirstIndirectSymbol + i);
     symbol_iterator SI = Obj.getSymbolByIndex(SymbolIndex);
-    StringRef IndirectSymbolName;
-    SI->getName(IndirectSymbolName);
+    ErrorOr<StringRef> IndirectSymbolNameOrErr = SI->getName();
+    if (std::error_code EC = IndirectSymbolNameOrErr.getError())
+      report_fatal_error(EC.message());
+    StringRef IndirectSymbolName = *IndirectSymbolNameOrErr;
     DEBUG(dbgs() << "  " << IndirectSymbolName << ": index " << SymbolIndex
           << ", PT offset: " << PTEntryOffset << "\n");
     RelocationEntry RE(PTSectionID, PTEntryOffset,
@@ -302,10 +334,7 @@ RuntimeDyldMachO::create(Triple::ArchType Arch,
 
 std::unique_ptr<RuntimeDyld::LoadedObjectInfo>
 RuntimeDyldMachO::loadObject(const object::ObjectFile &O) {
-  unsigned SectionStartIdx, SectionEndIdx;
-  std::tie(SectionStartIdx, SectionEndIdx) = loadObjectImpl(O);
-  return llvm::make_unique<LoadedMachOObjectInfo>(*this, SectionStartIdx,
-                                                  SectionEndIdx);
+  return llvm::make_unique<LoadedMachOObjectInfo>(*this, loadObjectImpl(O));
 }
 
 } // end namespace llvm
diff --git a/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldMachO.h b/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldMachO.h
index 45a94ba928f..2642ed2bd92 100644
--- a/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldMachO.h
+++ b/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldMachO.h
@@ -72,14 +72,19 @@ class RuntimeDyldMachO : public RuntimeDyldImpl {
 
     bool IsPCRel = Obj.getAnyRelocationPCRel(RelInfo);
     unsigned Size = Obj.getAnyRelocationLength(RelInfo);
-    uint64_t Offset;
-    RI->getOffset(Offset);
+    uint64_t Offset = RI->getOffset();
     MachO::RelocationInfoType RelType =
       static_cast<MachO::RelocationInfoType>(Obj.getAnyRelocationType(RelInfo));
 
     return RelocationEntry(SectionID, Offset, RelType, 0, IsPCRel, Size);
   }
 
+  /// Process a scattered vanilla relocation.
+  relocation_iterator processScatteredVANILLA(
+                           unsigned SectionID, relocation_iterator RelI,
+                           const ObjectFile &BaseObjT,
+                           RuntimeDyldMachO::ObjSectionToIDMap &ObjSectionToID);
+
   /// Construct a RelocationValueRef representing the relocation target.
   /// For Symbols in known sections, this will return a RelocationValueRef
   /// representing a (SectionID, Offset) pair.
@@ -96,7 +101,6 @@ class RuntimeDyldMachO : public RuntimeDyldImpl {
 
   /// Make the RelocationValueRef addend PC-relative.
   void makeValueAddendPCRel(RelocationValueRef &Value,
-                            const ObjectFile &BaseTObj,
                             const relocation_iterator &RI,
                             unsigned OffsetToNextPC);
 
diff --git a/lib/ExecutionEngine/RuntimeDyld/Targets/RuntimeDyldCOFFX86_64.h b/lib/ExecutionEngine/RuntimeDyld/Targets/RuntimeDyldCOFFX86_64.h
index cd534a12549..408227eb0f2 100644
--- a/lib/ExecutionEngine/RuntimeDyld/Targets/RuntimeDyldCOFFX86_64.h
+++ b/lib/ExecutionEngine/RuntimeDyld/Targets/RuntimeDyldCOFFX86_64.h
@@ -114,29 +114,19 @@ class RuntimeDyldCOFFX86_64 : public RuntimeDyldCOFF {
                                            const ObjectFile &Obj,
                                            ObjSectionToIDMap &ObjSectionToID,
                                            StubMap &Stubs) override {
-    // Find the symbol referred to in the relocation, and
-    // get its section and offset.
-    //
-    // Insist for now that all symbols be resolvable within
-    // the scope of this object file.
+    // If possible, find the symbol referred to in the relocation,
+    // and the section that contains it.
     symbol_iterator Symbol = RelI->getSymbol();
     if (Symbol == Obj.symbol_end())
       report_fatal_error("Unknown symbol in relocation");
-    unsigned TargetSectionID = 0;
-    uint64_t TargetOffset = UnknownAddressOrSize;
     section_iterator SecI(Obj.section_end());
     Symbol->getSection(SecI);
-    if (SecI == Obj.section_end())
-      report_fatal_error("Unknown section in relocation");
-    bool IsCode = SecI->isText();
-    TargetSectionID = findOrEmitSection(Obj, *SecI, IsCode, ObjSectionToID);
-    TargetOffset = getSymbolOffset(*Symbol);
+    // If there is no section, this must be an external reference.
+    const bool IsExtern = SecI == Obj.section_end();
 
     // Determine the Addend used to adjust the relocation value.
-    uint64_t RelType;
-    Check(RelI->getType(RelType));
-    uint64_t Offset;
-    Check(RelI->getOffset(Offset));
+    uint64_t RelType = RelI->getType();
+    uint64_t Offset = RelI->getOffset();
     uint64_t Addend = 0;
     SectionEntry &Section = Sections[SectionID];
     uintptr_t ObjTarget = Section.ObjAddress + Offset;
@@ -165,14 +155,26 @@ class RuntimeDyldCOFFX86_64 : public RuntimeDyldCOFF {
       break;
     }
 
-    StringRef TargetName;
-    Symbol->getName(TargetName);
+    ErrorOr<StringRef> TargetNameOrErr = Symbol->getName();
+    if (std::error_code EC = TargetNameOrErr.getError())
+      report_fatal_error(EC.message());
+    StringRef TargetName = *TargetNameOrErr;
+
     DEBUG(dbgs() << "\t\tIn Section " << SectionID << " Offset " << Offset
                  << " RelType: " << RelType << " TargetName: " << TargetName
                  << " Addend " << Addend << "\n");
 
-    RelocationEntry RE(SectionID, Offset, RelType, TargetOffset + Addend);
-    addRelocationForSection(RE, TargetSectionID);
+    if (IsExtern) {
+      RelocationEntry RE(SectionID, Offset, RelType, Addend);
+      addRelocationForSymbol(RE, TargetName);
+    } else {
+      bool IsCode = SecI->isText();
+      unsigned TargetSectionID =
+          findOrEmitSection(Obj, *SecI, IsCode, ObjSectionToID);
+      uint64_t TargetOffset = getSymbolOffset(*Symbol);
+      RelocationEntry RE(SectionID, Offset, RelType, TargetOffset + Addend);
+      addRelocationForSection(RE, TargetSectionID);
+    }
 
     return ++RelI;
   }
diff --git a/lib/ExecutionEngine/RuntimeDyld/Targets/RuntimeDyldMachOAArch64.h b/lib/ExecutionEngine/RuntimeDyld/Targets/RuntimeDyldMachOAArch64.h
index 99fd6e333b4..7bf764114ba 100644
--- a/lib/ExecutionEngine/RuntimeDyld/Targets/RuntimeDyldMachOAArch64.h
+++ b/lib/ExecutionEngine/RuntimeDyld/Targets/RuntimeDyldMachOAArch64.h
@@ -284,7 +284,7 @@ class RuntimeDyldMachOAArch64
 
     bool IsExtern = Obj.getPlainRelocationExternal(RelInfo);
     if (!IsExtern && RE.IsPCRel)
-      makeValueAddendPCRel(Value, Obj, RelI, 1 << RE.Size);
+      makeValueAddendPCRel(Value, RelI, 1 << RE.Size);
 
     RE.Addend = Value.Offset;
 
diff --git a/lib/ExecutionEngine/RuntimeDyld/Targets/RuntimeDyldMachOARM.h b/lib/ExecutionEngine/RuntimeDyld/Targets/RuntimeDyldMachOARM.h
index 09e51f27da4..64b57484945 100644
--- a/lib/ExecutionEngine/RuntimeDyld/Targets/RuntimeDyldMachOARM.h
+++ b/lib/ExecutionEngine/RuntimeDyld/Targets/RuntimeDyldMachOARM.h
@@ -64,8 +64,10 @@ class RuntimeDyldMachOARM
       if (RelType == MachO::ARM_RELOC_HALF_SECTDIFF)
         return processHALFSECTDIFFRelocation(SectionID, RelI, Obj,
                                              ObjSectionToID);
+      else if (RelType == MachO::GENERIC_RELOC_VANILLA)
+        return processScatteredVANILLA(SectionID, RelI, Obj, ObjSectionToID);
       else
-        return ++++RelI;
+        return ++RelI;
     }
 
     RelocationEntry RE(getRelocationEntry(SectionID, Obj, RelI));
@@ -74,7 +76,7 @@ class RuntimeDyldMachOARM
         getRelocationValueRef(Obj, RelI, RE, ObjSectionToID));
 
     if (RE.IsPCRel)
-      makeValueAddendPCRel(Value, Obj, RelI, 8);
+      makeValueAddendPCRel(Value, RelI, 8);
 
     if ((RE.RelType & 0xf) == MachO::ARM_RELOC_BR24)
       processBranchRelocation(RE, Value, Stubs);
@@ -220,8 +222,7 @@ class RuntimeDyldMachOARM
     SectionEntry &Section = Sections[SectionID];
     uint32_t RelocType = MachO.getAnyRelocationType(RE);
     bool IsPCRel = MachO.getAnyRelocationPCRel(RE);
-    uint64_t Offset;
-    RelI->getOffset(Offset);
+    uint64_t Offset = RelI->getOffset();
     uint8_t *LocalAddress = Section.Address + Offset;
     int64_t Immediate = readBytesUnaligned(LocalAddress, 4); // Copy the whole instruction out.
     Immediate = ((Immediate >> 4) & 0xf000) | (Immediate & 0xfff);
diff --git a/lib/ExecutionEngine/RuntimeDyld/Targets/RuntimeDyldMachOI386.h b/lib/ExecutionEngine/RuntimeDyld/Targets/RuntimeDyldMachOI386.h
index 053f90ce05a..87a248bdaa0 100644
--- a/lib/ExecutionEngine/RuntimeDyld/Targets/RuntimeDyldMachOI386.h
+++ b/lib/ExecutionEngine/RuntimeDyld/Targets/RuntimeDyldMachOI386.h
@@ -47,8 +47,7 @@ class RuntimeDyldMachOI386
         return processSECTDIFFRelocation(SectionID, RelI, Obj,
                                          ObjSectionToID);
       else if (RelType == MachO::GENERIC_RELOC_VANILLA)
-        return processI386ScatteredVANILLA(SectionID, RelI, Obj,
-                                           ObjSectionToID);
+        return processScatteredVANILLA(SectionID, RelI, Obj, ObjSectionToID);
       llvm_unreachable("Unhandled scattered relocation.");
     }
 
@@ -68,7 +67,7 @@ class RuntimeDyldMachOI386
     //   Value.Addend += RelocAddr + 4;
     // }
     if (RE.IsPCRel)
-      makeValueAddendPCRel(Value, Obj, RelI, 1 << RE.Size);
+      makeValueAddendPCRel(Value, RelI, 1 << RE.Size);
 
     RE.Addend = Value.Offset;
 
@@ -138,8 +137,7 @@ class RuntimeDyldMachOI386
     uint32_t RelocType = Obj.getAnyRelocationType(RE);
     bool IsPCRel = Obj.getAnyRelocationPCRel(RE);
     unsigned Size = Obj.getAnyRelocationLength(RE);
-    uint64_t Offset;
-    RelI->getOffset(Offset);
+    uint64_t Offset = RelI->getOffset();
     uint8_t *LocalAddress = Section.Address + Offset;
     unsigned NumBytes = 1 << Size;
     uint64_t Addend = readBytesUnaligned(LocalAddress, NumBytes);
@@ -167,56 +165,19 @@ class RuntimeDyldMachOI386
     uint32_t SectionBID =
         findOrEmitSection(Obj, SectionB, IsCode, ObjSectionToID);
 
-    if (Addend != AddrA - AddrB)
-      Error("Unexpected SECTDIFF relocation addend.");
+    // Compute the addend 'C' from the original expression 'A - B + C'.
+    Addend -= AddrA - AddrB;
 
     DEBUG(dbgs() << "Found SECTDIFF: AddrA: " << AddrA << ", AddrB: " << AddrB
                  << ", Addend: " << Addend << ", SectionA ID: " << SectionAID
                  << ", SectionAOffset: " << SectionAOffset
                  << ", SectionB ID: " << SectionBID
                  << ", SectionBOffset: " << SectionBOffset << "\n");
-    RelocationEntry R(SectionID, Offset, RelocType, 0, SectionAID,
-                      SectionAOffset, SectionBID, SectionBOffset, IsPCRel,
-                      Size);
+    RelocationEntry R(SectionID, Offset, RelocType, Addend, SectionAID,
+                      SectionAOffset, SectionBID, SectionBOffset,
+                      IsPCRel, Size);
 
     addRelocationForSection(R, SectionAID);
-    addRelocationForSection(R, SectionBID);
-
-    return ++RelI;
-  }
-
-  relocation_iterator processI386ScatteredVANILLA(
-      unsigned SectionID, relocation_iterator RelI,
-      const ObjectFile &BaseObjT,
-      RuntimeDyldMachO::ObjSectionToIDMap &ObjSectionToID) {
-    const MachOObjectFile &Obj =
-        static_cast<const MachOObjectFile&>(BaseObjT);
-    MachO::any_relocation_info RE =
-        Obj.getRelocation(RelI->getRawDataRefImpl());
-
-    SectionEntry &Section = Sections[SectionID];
-    uint32_t RelocType = Obj.getAnyRelocationType(RE);
-    bool IsPCRel = Obj.getAnyRelocationPCRel(RE);
-    unsigned Size = Obj.getAnyRelocationLength(RE);
-    uint64_t Offset;
-    RelI->getOffset(Offset);
-    uint8_t *LocalAddress = Section.Address + Offset;
-    unsigned NumBytes = 1 << Size;
-    int64_t Addend = readBytesUnaligned(LocalAddress, NumBytes);
-
-    unsigned SymbolBaseAddr = Obj.getScatteredRelocationValue(RE);
-    section_iterator TargetSI = getSectionByAddress(Obj, SymbolBaseAddr);
-    assert(TargetSI != Obj.section_end() && "Can't find section for symbol");
-    uint64_t SectionBaseAddr = TargetSI->getAddress();
-    SectionRef TargetSection = *TargetSI;
-    bool IsCode = TargetSection.isText();
-    uint32_t TargetSectionID =
-        findOrEmitSection(Obj, TargetSection, IsCode, ObjSectionToID);
-
-    Addend -= SectionBaseAddr;
-    RelocationEntry R(SectionID, Offset, RelocType, Addend, IsPCRel, Size);
-
-    addRelocationForSection(R, TargetSectionID);
 
     return ++RelI;
   }
@@ -243,13 +204,14 @@ class RuntimeDyldMachOI386
       unsigned SymbolIndex =
           Obj.getIndirectSymbolTableEntry(DySymTabCmd, FirstIndirectSymbol + i);
       symbol_iterator SI = Obj.getSymbolByIndex(SymbolIndex);
-      StringRef IndirectSymbolName;
-      SI->getName(IndirectSymbolName);
+      ErrorOr<StringRef> IndirectSymbolName = SI->getName();
+      if (std::error_code EC = IndirectSymbolName.getError())
+        report_fatal_error(EC.message());
       uint8_t *JTEntryAddr = JTSectionAddr + JTEntryOffset;
       createStubFunction(JTEntryAddr);
       RelocationEntry RE(JTSectionID, JTEntryOffset + 1,
                          MachO::GENERIC_RELOC_VANILLA, 0, true, 2);
-      addRelocationForSymbol(RE, IndirectSymbolName);
+      addRelocationForSymbol(RE, *IndirectSymbolName);
       JTEntryOffset += JTEntrySize;
     }
   }
diff --git a/lib/ExecutionEngine/RuntimeDyld/Targets/RuntimeDyldMachOX86_64.h b/lib/ExecutionEngine/RuntimeDyld/Targets/RuntimeDyldMachOX86_64.h
index 4b3b01ba3c9..dd56e72f914 100644
--- a/lib/ExecutionEngine/RuntimeDyld/Targets/RuntimeDyldMachOX86_64.h
+++ b/lib/ExecutionEngine/RuntimeDyld/Targets/RuntimeDyldMachOX86_64.h
@@ -50,7 +50,7 @@ class RuntimeDyldMachOX86_64
 
     bool IsExtern = Obj.getPlainRelocationExternal(RelInfo);
     if (!IsExtern && RE.IsPCRel)
-      makeValueAddendPCRel(Value, Obj, RelI, 1 << RE.Size);
+      makeValueAddendPCRel(Value, RelI, 1 << RE.Size);
 
     if (RE.RelType == MachO::X86_64_RELOC_GOT ||
         RE.RelType == MachO::X86_64_RELOC_GOT_LOAD)
diff --git a/lib/ExecutionEngine/TargetSelect.cpp b/lib/ExecutionEngine/TargetSelect.cpp
index e6679cfb7f7..57f6e080414 100644
--- a/lib/ExecutionEngine/TargetSelect.cpp
+++ b/lib/ExecutionEngine/TargetSelect.cpp
@@ -49,21 +49,19 @@ TargetMachine *EngineBuilder::selectTarget(const Triple &TargetTriple,
   // Adjust the triple to match what the user requested.
   const Target *TheTarget = nullptr;
   if (!MArch.empty()) {
-    for (TargetRegistry::iterator it = TargetRegistry::begin(),
-           ie = TargetRegistry::end(); it != ie; ++it) {
-      if (MArch == it->getName()) {
-        TheTarget = &*it;
-        break;
-      }
-    }
+    auto I = std::find_if(
+        TargetRegistry::targets().begin(), TargetRegistry::targets().end(),
+        [&](const Target &T) { return MArch == T.getName(); });
 
-    if (!TheTarget) {
+    if (I == TargetRegistry::targets().end()) {
       if (ErrorStr)
         *ErrorStr = "No available targets are compatible with this -march, "
                     "see -version for the available targets.\n";
       return nullptr;
     }
 
+    TheTarget = &*I;
+
     // Adjust the triple to match (if known), otherwise stick with the
     // requested/host triple.
     Triple::ArchType Type = Triple::getArchTypeForLLVMName(MArch);
diff --git a/lib/Fuzzer/CMakeLists.txt b/lib/Fuzzer/CMakeLists.txt
index bfd87ecc8ec..8b4d61905d0 100644
--- a/lib/Fuzzer/CMakeLists.txt
+++ b/lib/Fuzzer/CMakeLists.txt
@@ -2,19 +2,24 @@ set(LIBFUZZER_FLAGS_BASE "${CMAKE_CXX_FLAGS_RELEASE}")
 # Disable the coverage and sanitizer instrumentation for the fuzzer itself.
 set(CMAKE_CXX_FLAGS_RELEASE "${LIBFUZZER_FLAGS_BASE} -O2 -fno-sanitize=all")
 if( LLVM_USE_SANITIZE_COVERAGE )
-  add_library(LLVMFuzzerNoMain OBJECT
+  add_library(LLVMFuzzerNoMainObjects OBJECT
     FuzzerCrossOver.cpp
-    FuzzerDFSan.cpp
+    FuzzerInterface.cpp
+    FuzzerTraceState.cpp
     FuzzerDriver.cpp
     FuzzerIO.cpp
     FuzzerLoop.cpp
     FuzzerMutate.cpp
     FuzzerSanitizerOptions.cpp
+    FuzzerSHA1.cpp
     FuzzerUtil.cpp
     )
+  add_library(LLVMFuzzerNoMain STATIC
+    $<TARGET_OBJECTS:LLVMFuzzerNoMainObjects>
+    )
   add_library(LLVMFuzzer STATIC
     FuzzerMain.cpp
-    $<TARGET_OBJECTS:LLVMFuzzerNoMain>
+    $<TARGET_OBJECTS:LLVMFuzzerNoMainObjects>
     )
 
   if( LLVM_INCLUDE_TESTS )
diff --git a/lib/Fuzzer/FuzzerCrossOver.cpp b/lib/Fuzzer/FuzzerCrossOver.cpp
index 94af6d547ed..8b13698b5ae 100644
--- a/lib/Fuzzer/FuzzerCrossOver.cpp
+++ b/lib/Fuzzer/FuzzerCrossOver.cpp
@@ -9,39 +9,43 @@
 // Cross over test inputs.
 //===----------------------------------------------------------------------===//
 
+#include <cstring>
+
 #include "FuzzerInternal.h"
-#include <algorithm>
 
 namespace fuzzer {
 
-// Cross A and B, store the result (ap to MaxLen bytes) in U.
-void CrossOver(const Unit &A, const Unit &B, Unit *U, size_t MaxLen) {
-  size_t Size = rand() % MaxLen + 1;
-  U->clear();
-  const Unit *V = &A;
-  size_t PosA = 0;
-  size_t PosB = 0;
-  size_t *Pos = &PosA;
-  while (U->size() < Size && (PosA < A.size() || PosB < B.size())) {
-    // Merge a part of V into U.
-    size_t SizeLeftU = Size - U->size();
-    if (*Pos < V->size()) {
-      size_t SizeLeftV = V->size() - *Pos;
-      size_t MaxExtraSize = std::min(SizeLeftU, SizeLeftV);
-      size_t ExtraSize = rand() % MaxExtraSize + 1;
-      U->insert(U->end(), V->begin() + *Pos, V->begin() + *Pos + ExtraSize);
-      (*Pos) += ExtraSize;
-    }
-
-    // Use the other Unit on the next iteration.
-    if (Pos == &PosA) {
-      Pos = &PosB;
-      V = &B;
-    } else {
-      Pos = &PosA;
-      V = &A;
+// Cross Data1 and Data2, store the result (up to MaxOutSize bytes) in Out.
+size_t CrossOver(const uint8_t *Data1, size_t Size1,
+                 const uint8_t *Data2, size_t Size2,
+                 uint8_t *Out, size_t MaxOutSize, FuzzerRandomBase &Rand) {
+  assert(Size1 || Size2);
+  MaxOutSize = Rand(MaxOutSize) + 1;
+  size_t OutPos = 0;
+  size_t Pos1 = 0;
+  size_t Pos2 = 0;
+  size_t *InPos = &Pos1;
+  size_t InSize = Size1;
+  const uint8_t *Data = Data1;
+  bool CurrentlyUsingFirstData = true;
+  while (OutPos < MaxOutSize && (Pos1 < Size1 || Pos2 < Size2)) {
+    // Merge a part of Data into Out.
+    size_t OutSizeLeft = MaxOutSize - OutPos;
+    if (*InPos < InSize) {
+      size_t InSizeLeft = InSize - *InPos;
+      size_t MaxExtraSize = std::min(OutSizeLeft, InSizeLeft);
+      size_t ExtraSize = Rand(MaxExtraSize) + 1;
+      memcpy(Out + OutPos, Data + *InPos, ExtraSize);
+      OutPos += ExtraSize;
+      (*InPos) += ExtraSize;
     }
+    // Use the other input data on the next iteration.
+    InPos  = CurrentlyUsingFirstData ? &Pos2 : &Pos1;
+    InSize = CurrentlyUsingFirstData ? Size2 : Size1;
+    Data   = CurrentlyUsingFirstData ? Data2 : Data1;
+    CurrentlyUsingFirstData = !CurrentlyUsingFirstData;
   }
+  return OutPos;
 }
 
 }  // namespace fuzzer
diff --git a/lib/Fuzzer/FuzzerDFSan.cpp b/lib/Fuzzer/FuzzerDFSan.cpp
deleted file mode 100644
index 16f8c0fa3d7..00000000000
--- a/lib/Fuzzer/FuzzerDFSan.cpp
+++ /dev/null
@@ -1,275 +0,0 @@
-//===- FuzzerDFSan.cpp - DFSan-based fuzzer mutator -----------------------===//
-//
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
-//
-//===----------------------------------------------------------------------===//
-// DataFlowSanitizer (DFSan) is a tool for
-// generalised dynamic data flow (taint) analysis:
-// http://clang.llvm.org/docs/DataFlowSanitizer.html .
-//
-// This file implements a mutation algorithm based on taint
-// analysis feedback from DFSan.
-//
-// The approach has some similarity to "Taint-based Directed Whitebox Fuzzing"
-// by Vijay Ganesh & Tim Leek & Martin Rinard:
-// http://dspace.mit.edu/openaccess-disseminate/1721.1/59320,
-// but it uses a full blown LLVM IR taint analysis and separate instrumentation
-// to analyze all of the "attack points" at once.
-//
-// Workflow:
-//   * lib/Fuzzer/Fuzzer*.cpp is compiled w/o any instrumentation.
-//   * The code under test is compiled with DFSan *and* with special extra hooks
-//     that are inserted before dfsan. Currently supported hooks:
-//     - __sanitizer_cov_trace_cmp: inserted before every ICMP instruction,
-//       receives the type, size and arguments of ICMP.
-//   * Every call to HOOK(a,b) is replaced by DFSan with
-//     __dfsw_HOOK(a, b, label(a), label(b)) so that __dfsw_HOOK
-//     gets all the taint labels for the arguments.
-//   * At the Fuzzer startup we assign a unique DFSan label
-//     to every byte of the input string (Fuzzer::CurrentUnit) so that for any
-//     chunk of data we know which input bytes it has derived from.
-//   * The __dfsw_* functions (implemented in this file) record the
-//     parameters (i.e. the application data and the corresponding taint labels)
-//     in a global state.
-//   * Fuzzer::MutateWithDFSan() tries to use the data recorded by __dfsw_*
-//     hooks to guide the fuzzing towards new application states.
-//     For example if 4 bytes of data that derive from input bytes {4,5,6,7}
-//     are compared with a constant 12345 and the comparison always yields
-//     the same result, we try to insert 12345, 12344, 12346 into bytes
-//     {4,5,6,7} of the next fuzzed inputs.
-//
-// This code does not function when DFSan is not linked in.
-// Instead of using ifdefs and thus requiring a separate build of lib/Fuzzer
-// we redeclare the dfsan_* interface functions as weak and check if they
-// are nullptr before calling.
-// If this approach proves to be useful we may add attribute(weak) to the
-// dfsan declarations in dfsan_interface.h
-//
-// This module is in the "proof of concept" stage.
-// It is capable of solving only the simplest puzzles
-// like test/dfsan/DFSanSimpleCmpTest.cpp.
-//===----------------------------------------------------------------------===//
-
-/* Example of manual usage:
-(
-  cd $LLVM/lib/Fuzzer/
-  clang  -fPIC -c -g -O2 -std=c++11 Fuzzer*.cpp
-  clang++ -O0 -std=c++11 -fsanitize-coverage=3  \
-    -mllvm -sanitizer-coverage-experimental-trace-compares=1 \
-    -fsanitize=dataflow -fsanitize-blacklist=./dfsan_fuzzer_abi.list  \
-    test/dfsan/DFSanSimpleCmpTest.cpp Fuzzer*.o
-  ./a.out
-)
-*/
-
-#include "FuzzerInternal.h"
-#include <sanitizer/dfsan_interface.h>
-
-#include <cstring>
-#include <iostream>
-#include <unordered_map>
-
-extern "C" {
-__attribute__((weak))
-dfsan_label dfsan_create_label(const char *desc, void *userdata);
-__attribute__((weak))
-void dfsan_set_label(dfsan_label label, void *addr, size_t size);
-__attribute__((weak))
-void dfsan_add_label(dfsan_label label, void *addr, size_t size);
-__attribute__((weak))
-const struct dfsan_label_info *dfsan_get_label_info(dfsan_label label);
-}  // extern "C"
-
-namespace {
-
-// These values are copied from include/llvm/IR/InstrTypes.h.
-// We do not include the LLVM headers here to remain independent.
-// If these values ever change, an assertion in ComputeCmp will fail.
-enum Predicate {
-  ICMP_EQ = 32,  ///< equal
-  ICMP_NE = 33,  ///< not equal
-  ICMP_UGT = 34, ///< unsigned greater than
-  ICMP_UGE = 35, ///< unsigned greater or equal
-  ICMP_ULT = 36, ///< unsigned less than
-  ICMP_ULE = 37, ///< unsigned less or equal
-  ICMP_SGT = 38, ///< signed greater than
-  ICMP_SGE = 39, ///< signed greater or equal
-  ICMP_SLT = 40, ///< signed less than
-  ICMP_SLE = 41, ///< signed less or equal
-};
-
-template <class U, class S>
-bool ComputeCmp(size_t CmpType, U Arg1, U Arg2) {
-  switch(CmpType) {
-    case ICMP_EQ : return Arg1 == Arg2;
-    case ICMP_NE : return Arg1 != Arg2;
-    case ICMP_UGT: return Arg1 > Arg2;
-    case ICMP_UGE: return Arg1 >= Arg2;
-    case ICMP_ULT: return Arg1 < Arg2;
-    case ICMP_ULE: return Arg1 <= Arg2;
-    case ICMP_SGT: return (S)Arg1 > (S)Arg2;
-    case ICMP_SGE: return (S)Arg1 >= (S)Arg2;
-    case ICMP_SLT: return (S)Arg1 < (S)Arg2;
-    case ICMP_SLE: return (S)Arg1 <= (S)Arg2;
-    default: assert(0 && "unsupported CmpType");
-  }
-  return false;
-}
-
-static bool ComputeCmp(size_t CmpSize, size_t CmpType, uint64_t Arg1,
-                       uint64_t Arg2) {
-  if (CmpSize == 8) return ComputeCmp<uint64_t, int64_t>(CmpType, Arg1, Arg2);
-  if (CmpSize == 4) return ComputeCmp<uint32_t, int32_t>(CmpType, Arg1, Arg2);
-  if (CmpSize == 2) return ComputeCmp<uint16_t, int16_t>(CmpType, Arg1, Arg2);
-  if (CmpSize == 1) return ComputeCmp<uint8_t, int8_t>(CmpType, Arg1, Arg2);
-  assert(0 && "unsupported type size");
-  return true;
-}
-
-// As a simplification we use the range of input bytes instead of a set of input
-// bytes.
-struct LabelRange {
-  uint16_t Beg, End;  // Range is [Beg, End), thus Beg==End is an empty range.
-
-  LabelRange(uint16_t Beg = 0, uint16_t End = 0) : Beg(Beg), End(End) {}
-
-  static LabelRange Join(LabelRange LR1, LabelRange LR2) {
-    if (LR1.Beg == LR1.End) return LR2;
-    if (LR2.Beg == LR2.End) return LR1;
-    return {std::min(LR1.Beg, LR2.Beg), std::max(LR1.End, LR2.End)};
-  }
-  LabelRange &Join(LabelRange LR) {
-    return *this = Join(*this, LR);
-  }
-  static LabelRange Singleton(const dfsan_label_info *LI) {
-    uint16_t Idx = (uint16_t)(uintptr_t)LI->userdata;
-    assert(Idx > 0);
-    return {(uint16_t)(Idx - 1), Idx};
-  }
-};
-
-std::ostream &operator<<(std::ostream &os, const LabelRange &LR) {
-  return os << "[" << LR.Beg << "," << LR.End << ")";
-}
-
-class DFSanState {
- public:
-   DFSanState(const fuzzer::Fuzzer::FuzzingOptions &Options)
-       : Options(Options) {}
-
-  struct CmpSiteInfo {
-    size_t ResCounters[2] = {0, 0};
-    size_t CmpSize = 0;
-    LabelRange LR;
-    std::unordered_map<uint64_t, size_t> CountedConstants;
-  };
-
-  LabelRange GetLabelRange(dfsan_label L);
-  void DFSanCmpCallback(uintptr_t PC, size_t CmpSize, size_t CmpType,
-                        uint64_t Arg1, uint64_t Arg2, dfsan_label L1,
-                        dfsan_label L2);
-  bool Mutate(fuzzer::Unit *U);
-
- private:
-  std::unordered_map<uintptr_t, CmpSiteInfo> PcToCmpSiteInfoMap;
-  LabelRange LabelRanges[1 << (sizeof(dfsan_label) * 8)] = {};
-  const fuzzer::Fuzzer::FuzzingOptions &Options;
-};
-
-LabelRange DFSanState::GetLabelRange(dfsan_label L) {
-  LabelRange &LR = LabelRanges[L];
-  if (LR.Beg < LR.End || L == 0)
-    return LR;
-  const dfsan_label_info *LI = dfsan_get_label_info(L);
-  if (LI->l1 || LI->l2)
-    return LR = LabelRange::Join(GetLabelRange(LI->l1), GetLabelRange(LI->l2));
-  return LR = LabelRange::Singleton(LI);
-}
-
-void DFSanState::DFSanCmpCallback(uintptr_t PC, size_t CmpSize, size_t CmpType,
-                                  uint64_t Arg1, uint64_t Arg2, dfsan_label L1,
-                                  dfsan_label L2) {
-  if (L1 == 0 && L2 == 0)
-    return;  // Not actionable.
-  if (L1 != 0 && L2 != 0)
-    return;  // Probably still actionable.
-  bool Res = ComputeCmp(CmpSize, CmpType, Arg1, Arg2);
-  CmpSiteInfo &CSI = PcToCmpSiteInfoMap[PC];
-  CSI.CmpSize = CmpSize;
-  CSI.LR.Join(GetLabelRange(L1)).Join(GetLabelRange(L2));
-  if (!L1) CSI.CountedConstants[Arg1]++;
-  if (!L2) CSI.CountedConstants[Arg2]++;
-  size_t Counter = CSI.ResCounters[Res]++;
-
-  if (Options.Verbosity >= 2  &&
-      (Counter & (Counter - 1)) == 0 &&
-      CSI.ResCounters[!Res] == 0)
-    std::cerr << "DFSAN:"
-              << " PC " << std::hex << PC << std::dec
-              << " S " << CmpSize
-              << " T " << CmpType
-              << " A1 " << Arg1 << " A2 " << Arg2 << " R " << Res
-              << " L" << L1 << GetLabelRange(L1)
-              << " L" << L2 << GetLabelRange(L2)
-              << " LR " << CSI.LR
-              << "\n";
-}
-
-bool DFSanState::Mutate(fuzzer::Unit *U) {
-  for (auto &PCToCmp : PcToCmpSiteInfoMap) {
-    auto &CSI = PCToCmp.second;
-    if (CSI.ResCounters[0] * CSI.ResCounters[1] != 0) continue;
-    if (CSI.ResCounters[0] + CSI.ResCounters[1] < 1000) continue;
-    if (CSI.CountedConstants.size() != 1) continue;
-    uintptr_t C = CSI.CountedConstants.begin()->first;
-    if (U->size() >= CSI.CmpSize) {
-      size_t RangeSize = CSI.LR.End - CSI.LR.Beg;
-      size_t Idx = CSI.LR.Beg + rand() % RangeSize;
-      if (Idx + CSI.CmpSize > U->size()) continue;
-      C += rand() % 5 - 2;
-      memcpy(U->data() + Idx, &C, CSI.CmpSize);
-      return true;
-    }
-  }
-  return false;
-}
-
-static DFSanState *DFSan;
-
-}  // namespace
-
-namespace fuzzer {
-
-bool Fuzzer::MutateWithDFSan(Unit *U) {
-  if (!&dfsan_create_label || !DFSan) return false;
-  return DFSan->Mutate(U);
-}
-
-void Fuzzer::InitializeDFSan() {
-  if (!&dfsan_create_label || !Options.UseDFSan) return;
-  DFSan = new DFSanState(Options);
-  CurrentUnit.resize(Options.MaxLen);
-  for (size_t i = 0; i < static_cast<size_t>(Options.MaxLen); i++) {
-    dfsan_label L = dfsan_create_label("input", (void*)(i + 1));
-    // We assume that no one else has called dfsan_create_label before.
-    assert(L == i + 1);
-    dfsan_set_label(L, &CurrentUnit[i], 1);
-  }
-}
-
-}  // namespace fuzzer
-
-extern "C" {
-void __dfsw___sanitizer_cov_trace_cmp(uint64_t SizeAndType, uint64_t Arg1,
-                                      uint64_t Arg2, dfsan_label L0,
-                                      dfsan_label L1, dfsan_label L2) {
-  assert(L0 == 0);
-  uintptr_t PC = reinterpret_cast<uintptr_t>(__builtin_return_address(0));
-  uint64_t CmpSize = (SizeAndType >> 32) / 8;
-  uint64_t Type = (SizeAndType << 32) >> 32;
-  DFSan->DFSanCmpCallback(PC, CmpSize, Type, Arg1, Arg2, L1, L2);
-}
-}  // extern "C"
diff --git a/lib/Fuzzer/FuzzerDriver.cpp b/lib/Fuzzer/FuzzerDriver.cpp
index 05a699e5068..c0f8220ad65 100644
--- a/lib/Fuzzer/FuzzerDriver.cpp
+++ b/lib/Fuzzer/FuzzerDriver.cpp
@@ -13,8 +13,8 @@
 #include "FuzzerInternal.h"
 
 #include <cstring>
+#include <chrono>
 #include <unistd.h>
-#include <iostream>
 #include <thread>
 #include <atomic>
 #include <mutex>
@@ -59,21 +59,23 @@ static std::vector<std::string> inputs;
 static const char *ProgName;
 
 static void PrintHelp() {
-  std::cerr << "Usage: " << ProgName
-            << " [-flag1=val1 [-flag2=val2 ...] ] [dir1 [dir2 ...] ]\n";
-  std::cerr << "\nFlags: (strictly in form -flag=value)\n";
+  Printf("Usage: %s [-flag1=val1 [-flag2=val2 ...] ] [dir1 [dir2 ...] ]\n",
+         ProgName);
+  Printf("\nFlags: (strictly in form -flag=value)\n");
   size_t MaxFlagLen = 0;
   for (size_t F = 0; F < kNumFlags; F++)
     MaxFlagLen = std::max(strlen(FlagDescriptions[F].Name), MaxFlagLen);
 
   for (size_t F = 0; F < kNumFlags; F++) {
     const auto &D = FlagDescriptions[F];
-    std::cerr << "  " << D.Name;
+    Printf(" %s", D.Name);
     for (size_t i = 0, n = MaxFlagLen - strlen(D.Name); i < n; i++)
-      std::cerr << " ";
-    std::cerr << "\t";
-    std::cerr << D.Default << "\t" << D.Description << "\n";
+      Printf(" ");
+    Printf("\t");
+    Printf("%d\t%s\n", D.Default, D.Description);
   }
+  Printf("\nFlags starting with '--' will be ignored and "
+            "will be passed verbatim to subprocesses.\n");
 }
 
 static const char *FlagValue(const char *Param, const char *Name) {
@@ -86,6 +88,14 @@ static const char *FlagValue(const char *Param, const char *Name) {
 
 static bool ParseOneFlag(const char *Param) {
   if (Param[0] != '-') return false;
+  if (Param[1] == '-') {
+    static bool PrintedWarning = false;
+    if (!PrintedWarning) {
+      PrintedWarning = true;
+      Printf("WARNING: libFuzzer ignores flags that start with '--'\n");
+    }
+    return true;
+  }
   for (size_t F = 0; F < kNumFlags; F++) {
     const char *Name = FlagDescriptions[F].Name;
     const char *Str = FlagValue(Param, Name);
@@ -94,12 +104,12 @@ static bool ParseOneFlag(const char *Param) {
         int Val = std::stol(Str);
         *FlagDescriptions[F].IntFlag = Val;
         if (Flags.verbosity >= 2)
-          std::cerr << "Flag: " << Name << " " << Val << "\n";
+          Printf("Flag: %s %d\n", Name, Val);;
         return true;
       } else if (FlagDescriptions[F].StrFlag) {
         *FlagDescriptions[F].StrFlag = Str;
         if (Flags.verbosity >= 2)
-          std::cerr << "Flag: " << Name << " " << Str << "\n";
+          Printf("Flag: %s %s\n", Name, Str);
         return true;
       }
     }
@@ -122,23 +132,31 @@ static void ParseFlags(int argc, char **argv) {
   }
 }
 
+static std::mutex Mu;
+
+static void PulseThread() {
+  while (true) {
+    std::this_thread::sleep_for(std::chrono::seconds(600));
+    std::lock_guard<std::mutex> Lock(Mu);
+    Printf("pulse...\n");
+  }
+}
+
 static void WorkerThread(const std::string &Cmd, std::atomic<int> *Counter,
                         int NumJobs, std::atomic<bool> *HasErrors) {
-  static std::mutex CerrMutex;
   while (true) {
     int C = (*Counter)++;
     if (C >= NumJobs) break;
     std::string Log = "fuzz-" + std::to_string(C) + ".log";
     std::string ToRun = Cmd + " > " + Log + " 2>&1\n";
     if (Flags.verbosity)
-      std::cerr << ToRun;
+      Printf("%s", ToRun.c_str());
     int ExitCode = system(ToRun.c_str());
     if (ExitCode != 0)
       *HasErrors = true;
-    std::lock_guard<std::mutex> Lock(CerrMutex);
-    std::cerr << "================== Job " << C
-              << " exited with exit code " << ExitCode
-              << " =================\n";
+    std::lock_guard<std::mutex> Lock(Mu);
+    Printf("================== Job %d exited with exit code %d ============\n",
+           C, ExitCode);
     fuzzer::CopyFileToErr(Log);
   }
 }
@@ -154,6 +172,8 @@ static int RunInMultipleProcesses(int argc, char **argv, int NumWorkers,
     Cmd += " ";
   }
   std::vector<std::thread> V;
+  std::thread Pulse(PulseThread);
+  Pulse.detach();
   for (int i = 0; i < NumWorkers; i++)
     V.push_back(std::thread(WorkerThread, Cmd, &Counter, NumJobs, &HasErrors));
   for (auto &T : V)
@@ -177,11 +197,17 @@ int ApplyTokens(const Fuzzer &F, const char *InputFilePath) {
   Unit U = FileToVector(InputFilePath);
   auto T = F.SubstituteTokens(U);
   T.push_back(0);
-  std::cout << T.data();
+  Printf("%s", T.data());
   return 0;
 }
 
 int FuzzerDriver(int argc, char **argv, UserCallback Callback) {
+  FuzzerRandomLibc Rand(0);
+  SimpleUserSuppliedFuzzer SUSF(&Rand, Callback);
+  return FuzzerDriver(argc, argv, SUSF);
+}
+
+int FuzzerDriver(int argc, char **argv, UserSuppliedFuzzer &USF) {
   using namespace fuzzer;
 
   ProgName = argv[0];
@@ -191,52 +217,64 @@ int FuzzerDriver(int argc, char **argv, UserCallback Callback) {
     return 0;
   }
 
+  if (Flags.jobs > 0 && Flags.workers == 0) {
+    Flags.workers = std::min(NumberOfCpuCores() / 2, Flags.jobs);
+    if (Flags.workers > 1)
+      Printf("Running %d workers\n", Flags.workers);
+  }
+
   if (Flags.workers > 0 && Flags.jobs > 0)
     return RunInMultipleProcesses(argc, argv, Flags.workers, Flags.jobs);
 
   Fuzzer::FuzzingOptions Options;
   Options.Verbosity = Flags.verbosity;
   Options.MaxLen = Flags.max_len;
+  Options.UnitTimeoutSec = Flags.timeout;
   Options.DoCrossOver = Flags.cross_over;
   Options.MutateDepth = Flags.mutate_depth;
   Options.ExitOnFirst = Flags.exit_on_first;
   Options.UseCounters = Flags.use_counters;
+  Options.UseTraces = Flags.use_traces;
   Options.UseFullCoverageSet = Flags.use_full_coverage_set;
-  Options.UseCoveragePairs = Flags.use_coverage_pairs;
-  Options.UseDFSan = Flags.dfsan;
   Options.PreferSmallDuringInitialShuffle =
       Flags.prefer_small_during_initial_shuffle;
   Options.Tokens = ReadTokensFile(Flags.tokens);
+  Options.Reload = Flags.reload;
   if (Flags.runs >= 0)
     Options.MaxNumberOfRuns = Flags.runs;
   if (!inputs.empty())
     Options.OutputCorpus = inputs[0];
-  Fuzzer F(Callback, Options);
+  if (Flags.sync_command)
+    Options.SyncCommand = Flags.sync_command;
+  Options.SyncTimeout = Flags.sync_timeout;
+  Fuzzer F(USF, Options);
+
+  if (Flags.apply_tokens)
+    return ApplyTokens(F, Flags.apply_tokens);
 
-  unsigned seed = Flags.seed;
-  // Initialize seed.
-  if (seed == 0)
-    seed = time(0) * 10000 + getpid();
+  unsigned Seed = Flags.seed;
+  // Initialize Seed.
+  if (Seed == 0)
+    Seed = time(0) * 10000 + getpid();
   if (Flags.verbosity)
-    std::cerr << "Seed: " << seed << "\n";
-  srand(seed);
+    Printf("Seed: %u\n", Seed);
+  USF.GetRand().ResetSeed(Seed);
 
   // Timer
   if (Flags.timeout > 0)
-    SetTimer(Flags.timeout);
+    SetTimer(Flags.timeout / 2 + 1);
 
   if (Flags.verbosity >= 2) {
-    std::cerr << "Tokens: {";
+    Printf("Tokens: {");
     for (auto &T : Options.Tokens)
-      std::cerr << T << ",";
-    std::cerr << "}\n";
+      Printf("%s,", T.c_str());
+    Printf("}\n");
   }
 
-  if (Flags.apply_tokens)
-    return ApplyTokens(F, Flags.apply_tokens);
-
+  F.RereadOutputCorpus();
   for (auto &inp : inputs)
-    F.ReadDir(inp);
+    if (inp != Options.OutputCorpus)
+      F.ReadDir(inp, nullptr);
 
   if (F.CorpusSize() == 0)
     F.AddToCorpus(Unit());  // Can't fuzz empty corpus, so add an empty input.
@@ -245,9 +283,9 @@ int FuzzerDriver(int argc, char **argv, UserCallback Callback) {
     F.SaveCorpus();
   F.Loop(Flags.iterations < 0 ? INT_MAX : Flags.iterations);
   if (Flags.verbosity)
-    std::cerr << "Done " << F.getTotalNumberOfRuns()
-              << " runs in " << F.secondsSinceProcessStartUp()
-              << " seconds\n";
+    Printf("Done %d runs in %zd second(s)\n", F.getTotalNumberOfRuns(),
+           F.secondsSinceProcessStartUp());
+
   return 0;
 }
 
diff --git a/lib/Fuzzer/FuzzerFlags.def b/lib/Fuzzer/FuzzerFlags.def
index dbaf75db696..742f672e201 100644
--- a/lib/Fuzzer/FuzzerFlags.def
+++ b/lib/Fuzzer/FuzzerFlags.def
@@ -17,7 +17,7 @@ FUZZER_FLAG_INT(iterations, -1,
             " (-1 for infinite iterations).")
 FUZZER_FLAG_INT(runs, -1,
             "Number of individual test runs (-1 for infinite runs).")
-FUZZER_FLAG_INT(max_len, 64, "Maximal length of the test input.")
+FUZZER_FLAG_INT(max_len, 64, "Maximum length of the test input.")
 FUZZER_FLAG_INT(cross_over, 1, "If 1, cross over inputs.")
 FUZZER_FLAG_INT(mutate_depth, 5,
             "Apply this number of consecutive mutations to each input.")
@@ -27,27 +27,34 @@ FUZZER_FLAG_INT(
     " If 0, never do that. If -1, do it sometimes.")
 FUZZER_FLAG_INT(exit_on_first, 0,
             "If 1, exit after the first new interesting input is found.")
-FUZZER_FLAG_INT(timeout, -1, "Timeout in seconds (if positive).")
+FUZZER_FLAG_INT(
+    timeout, 1200,
+    "Timeout in seconds (if positive). "
+    "If one unit runs more than this number of seconds the process will abort.")
 FUZZER_FLAG_INT(help, 0, "Print help.")
 FUZZER_FLAG_INT(
     save_minimized_corpus, 0,
     "If 1, the minimized corpus is saved into the first input directory")
-FUZZER_FLAG_INT(use_counters, 0, "Use coverage counters")
+FUZZER_FLAG_INT(use_counters, 1, "Use coverage counters")
+FUZZER_FLAG_INT(use_traces, 0, "Experimental: use instruction traces")
 FUZZER_FLAG_INT(use_full_coverage_set, 0,
             "Experimental: Maximize the number of different full"
             " coverage sets as opposed to maximizing the total coverage."
             " This is potentially MUCH slower, but may discover more paths.")
-FUZZER_FLAG_INT(use_coverage_pairs, 0,
-            "Experimental: Maximize the number of different coverage pairs.")
 FUZZER_FLAG_INT(jobs, 0, "Number of jobs to run. If jobs >= 1 we spawn"
                           " this number of jobs in separate worker processes"
                           " with stdout/stderr redirected to fuzz-JOB.log.")
 FUZZER_FLAG_INT(workers, 0,
-            "Number of simultaneous worker processes to run the jobs.")
-FUZZER_FLAG_INT(dfsan, 1, "Use DFSan for taint-guided mutations. No-op unless "
-                           "the DFSan instrumentation was compiled in.")
-
+            "Number of simultaneous worker processes to run the jobs."
+            " If zero, \"min(jobs,NumberOfCpuCores()/2)\" is used.")
+FUZZER_FLAG_INT(reload, 1,
+                "Reload the main corpus periodically to get new units"
+                "discovered by other processes.")
 FUZZER_FLAG_STRING(tokens, "Use the file with tokens (one token per line) to"
                            " fuzz a token based input language.")
 FUZZER_FLAG_STRING(apply_tokens, "Read the given input file, substitute bytes "
                                  " with tokens and write the result to stdout.")
+FUZZER_FLAG_STRING(sync_command, "Execute an external command "
+                                 "\"<sync_command> <test_corpus>\" "
+                                 "to synchronize the test corpus.")
+FUZZER_FLAG_INT(sync_timeout, 600, "Minimum timeout between syncs.")
diff --git a/lib/Fuzzer/FuzzerIO.cpp b/lib/Fuzzer/FuzzerIO.cpp
index ef23d42ea53..1d25389770f 100644
--- a/lib/Fuzzer/FuzzerIO.cpp
+++ b/lib/Fuzzer/FuzzerIO.cpp
@@ -9,16 +9,38 @@
 // IO functions.
 //===----------------------------------------------------------------------===//
 #include "FuzzerInternal.h"
-#include <iostream>
 #include <iterator>
 #include <fstream>
 #include <dirent.h>
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <unistd.h>
+#include <cstdio>
+
 namespace fuzzer {
 
-static std::vector<std::string> ListFilesInDir(const std::string &Dir) {
+static long GetEpoch(const std::string &Path) {
+  struct stat St;
+  if (stat(Path.c_str(), &St)) {
+    Printf("Can not stat: %s; exiting\n", Path.c_str());
+    exit(1);
+  }
+  return St.st_mtime;
+}
+
+static std::vector<std::string> ListFilesInDir(const std::string &Dir,
+                                               long *Epoch) {
   std::vector<std::string> V;
+  if (Epoch) {
+    auto E = GetEpoch(Dir.c_str());
+    if (*Epoch >= E) return V;
+    *Epoch = E;
+  }
   DIR *D = opendir(Dir.c_str());
-  if (!D) return V;
+  if (!D) {
+    Printf("No such directory: %s; exiting\n", Dir.c_str());
+    exit(1);
+  }
   while (auto E = readdir(D)) {
     if (E->d_type == DT_REG || E->d_type == DT_LNK)
       V.push_back(E->d_name);
@@ -40,9 +62,7 @@ std::string FileToString(const std::string &Path) {
 }
 
 void CopyFileToErr(const std::string &Path) {
-  std::ifstream T(Path);
-  std::copy(std::istreambuf_iterator<char>(T), std::istreambuf_iterator<char>(),
-            std::ostream_iterator<char>(std::cerr, ""));
+  Printf("%s", FileToString(Path).c_str());
 }
 
 void WriteToFile(const Unit &U, const std::string &Path) {
@@ -50,9 +70,14 @@ void WriteToFile(const Unit &U, const std::string &Path) {
   OF.write((const char*)U.data(), U.size());
 }
 
-void ReadDirToVectorOfUnits(const char *Path, std::vector<Unit> *V) {
-  for (auto &X : ListFilesInDir(Path))
-    V->push_back(FileToVector(DirPlusFile(Path, X)));
+void ReadDirToVectorOfUnits(const char *Path, std::vector<Unit> *V,
+                            long *Epoch) {
+  long E = Epoch ? *Epoch : 0;
+  for (auto &X : ListFilesInDir(Path, Epoch)) {
+    auto FilePath = DirPlusFile(Path, X);
+    if (Epoch && GetEpoch(FilePath) < E) continue;
+    V->push_back(FileToVector(FilePath));
+  }
 }
 
 std::string DirPlusFile(const std::string &DirPath,
@@ -60,4 +85,16 @@ std::string DirPlusFile(const std::string &DirPath,
   return DirPath + "/" + FileName;
 }
 
+void PrintFileAsBase64(const std::string &Path) {
+  std::string Cmd = "base64 -w 0 < " + Path + "; echo";
+  ExecuteCommand(Cmd);
+}
+
+void Printf(const char *Fmt, ...) {
+  va_list ap;
+  va_start(ap, Fmt);
+  vfprintf(stderr, Fmt, ap);
+  va_end(ap);
+}
+
 }  // namespace fuzzer
diff --git a/lib/Fuzzer/FuzzerInterface.cpp b/lib/Fuzzer/FuzzerInterface.cpp
new file mode 100644
index 00000000000..c7553f421fb
--- /dev/null
+++ b/lib/Fuzzer/FuzzerInterface.cpp
@@ -0,0 +1,43 @@
+//===- FuzzerInterface.cpp - Mutate a test input --------------------------===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+// Parts of public interface for libFuzzer.
+//===----------------------------------------------------------------------===//
+
+
+#include "FuzzerInterface.h"
+#include "FuzzerInternal.h"
+
+namespace fuzzer {
+
+void FuzzerRandomLibc::ResetSeed(int seed) { srand(seed); }
+
+size_t FuzzerRandomLibc::Rand() { return rand(); }
+
+UserSuppliedFuzzer::UserSuppliedFuzzer()
+    : OwnRand(true), Rand(new FuzzerRandomLibc(0)) {}
+
+UserSuppliedFuzzer::UserSuppliedFuzzer(FuzzerRandomBase *Rand) : Rand(Rand) {}
+
+UserSuppliedFuzzer::~UserSuppliedFuzzer() {
+  if (OwnRand)
+    delete Rand;
+}
+
+size_t UserSuppliedFuzzer::BasicMutate(uint8_t *Data, size_t Size,
+                                       size_t MaxSize) {
+  return ::fuzzer::Mutate(Data, Size, MaxSize, *Rand);
+}
+size_t UserSuppliedFuzzer::BasicCrossOver(const uint8_t *Data1, size_t Size1,
+                                          const uint8_t *Data2, size_t Size2,
+                                          uint8_t *Out, size_t MaxOutSize) {
+  return ::fuzzer::CrossOver(Data1, Size1, Data2, Size2, Out, MaxOutSize,
+                             *Rand);
+}
+
+}  // namespace fuzzer.
diff --git a/lib/Fuzzer/FuzzerInterface.h b/lib/Fuzzer/FuzzerInterface.h
index 49d8c0f6145..3be4b05e66c 100644
--- a/lib/Fuzzer/FuzzerInterface.h
+++ b/lib/Fuzzer/FuzzerInterface.h
@@ -9,6 +9,10 @@
 // Define the interface between the Fuzzer and the library being tested.
 //===----------------------------------------------------------------------===//
 
+// WARNING: keep the interface free of STL or any other header-based C++ lib,
+// to avoid bad interactions between the code used in the fuzzer and
+// the code used in the target function.
+
 #ifndef LLVM_FUZZER_INTERFACE_H
 #define LLVM_FUZZER_INTERFACE_H
 
@@ -17,9 +21,106 @@
 
 namespace fuzzer {
 
-typedef void (*UserCallback)(const uint8_t *data, size_t size);
+typedef void (*UserCallback)(const uint8_t *Data, size_t Size);
+/** Simple C-like interface with a single user-supplied callback.
+
+Usage:
+
+#\code
+#include "FuzzerInterface.h"
+
+void LLVMFuzzerTestOneInput(const uint8_t *Data, size_t Size) {
+  DoStuffWithData(Data, Size);
+}
+
+// Implement your own main() or use the one from FuzzerMain.cpp.
+int main(int argc, char **argv) {
+  InitializeMeIfNeeded();
+  return fuzzer::FuzzerDriver(argc, argv, LLVMFuzzerTestOneInput);
+}
+#\endcode
+*/
 int FuzzerDriver(int argc, char **argv, UserCallback Callback);
 
+class FuzzerRandomBase {
+ public:
+  FuzzerRandomBase(){}
+  virtual ~FuzzerRandomBase(){};
+  virtual void ResetSeed(int seed) = 0;
+  // Return a random number.
+  virtual size_t Rand() = 0;
+  // Return a random number in range [0,n).
+  size_t operator()(size_t n) { return Rand() % n; }
+  bool RandBool() { return Rand() % 2; }
+};
+
+class FuzzerRandomLibc : public FuzzerRandomBase {
+ public:
+  FuzzerRandomLibc(int seed) { ResetSeed(seed); }
+  void ResetSeed(int seed) override;
+  ~FuzzerRandomLibc() override {}
+  size_t Rand() override;
+};
+
+/** An abstract class that allows to use user-supplied mutators with libFuzzer.
+
+Usage:
+
+#\code
+#include "FuzzerInterface.h"
+class MyFuzzer : public fuzzer::UserSuppliedFuzzer {
+ public:
+  MyFuzzer(fuzzer::FuzzerRandomBase *Rand);
+  // Must define the target function.
+  void TargetFunction(...) { ... }
+  // Optionally define the mutator.
+  size_t Mutate(...) { ... }
+  // Optionally define the CrossOver method.
+  size_t CrossOver(...) { ... }
+};
+
+int main(int argc, char **argv) {
+  MyFuzzer F;
+  fuzzer::FuzzerDriver(argc, argv, F);
+}
+#\endcode
+*/
+class UserSuppliedFuzzer {
+ public:
+  UserSuppliedFuzzer();  // Deprecated, don't use.
+  UserSuppliedFuzzer(FuzzerRandomBase *Rand);
+  /// Executes the target function on 'Size' bytes of 'Data'.
+  virtual void TargetFunction(const uint8_t *Data, size_t Size) = 0;
+  /// Mutates 'Size' bytes of data in 'Data' inplace into up to 'MaxSize' bytes,
+  /// returns the new size of the data, which should be positive.
+  virtual size_t Mutate(uint8_t *Data, size_t Size, size_t MaxSize) {
+    return BasicMutate(Data, Size, MaxSize);
+  }
+  /// Crosses 'Data1' and 'Data2', writes up to 'MaxOutSize' bytes into Out,
+  /// returns the number of bytes written, which should be positive.
+  virtual size_t CrossOver(const uint8_t *Data1, size_t Size1,
+                           const uint8_t *Data2, size_t Size2,
+                           uint8_t *Out, size_t MaxOutSize) {
+    return BasicCrossOver(Data1, Size1, Data2, Size2, Out, MaxOutSize);
+  }
+  virtual ~UserSuppliedFuzzer();
+
+  FuzzerRandomBase &GetRand() { return *Rand; }
+
+ protected:
+  /// These can be called internally by Mutate and CrossOver.
+  size_t BasicMutate(uint8_t *Data, size_t Size, size_t MaxSize);
+  size_t BasicCrossOver(const uint8_t *Data1, size_t Size1,
+                        const uint8_t *Data2, size_t Size2,
+                        uint8_t *Out, size_t MaxOutSize);
+ private:
+  bool OwnRand = false;
+  FuzzerRandomBase *Rand;
+};
+
+/// Runs the fuzzing with the UserSuppliedFuzzer.
+int FuzzerDriver(int argc, char **argv, UserSuppliedFuzzer &USF);
+
 }  // namespace fuzzer
 
 #endif  // LLVM_FUZZER_INTERFACE_H
diff --git a/lib/Fuzzer/FuzzerInternal.h b/lib/Fuzzer/FuzzerInternal.h
index 77871097bd7..fe7869871f0 100644
--- a/lib/Fuzzer/FuzzerInternal.h
+++ b/lib/Fuzzer/FuzzerInternal.h
@@ -25,48 +25,66 @@ using namespace std::chrono;
 
 std::string FileToString(const std::string &Path);
 Unit FileToVector(const std::string &Path);
-void ReadDirToVectorOfUnits(const char *Path, std::vector<Unit> *V);
+void ReadDirToVectorOfUnits(const char *Path, std::vector<Unit> *V,
+                            long *Epoch);
 void WriteToFile(const Unit &U, const std::string &Path);
 void CopyFileToErr(const std::string &Path);
 // Returns "Dir/FileName" or equivalent for the current OS.
 std::string DirPlusFile(const std::string &DirPath,
                         const std::string &FileName);
 
-void Mutate(Unit *U, size_t MaxLen);
+size_t Mutate(uint8_t *Data, size_t Size, size_t MaxSize,
+              FuzzerRandomBase &Rand);
 
-void CrossOver(const Unit &A, const Unit &B, Unit *U, size_t MaxLen);
+size_t CrossOver(const uint8_t *Data1, size_t Size1, const uint8_t *Data2,
+                 size_t Size2, uint8_t *Out, size_t MaxOutSize,
+                 FuzzerRandomBase &Rand);
 
+void Printf(const char *Fmt, ...);
 void Print(const Unit &U, const char *PrintAfter = "");
 void PrintASCII(const Unit &U, const char *PrintAfter = "");
 std::string Hash(const Unit &U);
 void SetTimer(int Seconds);
+void PrintFileAsBase64(const std::string &Path);
+void ExecuteCommand(const std::string &Command);
+
+// Private copy of SHA1 implementation.
+static const int kSHA1NumBytes = 20;
+// Computes SHA1 hash of 'Len' bytes in 'Data', writes kSHA1NumBytes to 'Out'.
+void ComputeSHA1(const uint8_t *Data, size_t Len, uint8_t *Out);
+
+int NumberOfCpuCores();
 
 class Fuzzer {
  public:
   struct FuzzingOptions {
     int Verbosity = 1;
     int MaxLen = 0;
+    int UnitTimeoutSec = 300;
     bool DoCrossOver = true;
     int  MutateDepth = 5;
     bool ExitOnFirst = false;
     bool UseCounters = false;
+    bool UseTraces = false;
     bool UseFullCoverageSet  = false;
-    bool UseCoveragePairs = false;
-    bool UseDFSan = false;
+    bool Reload = true;
     int PreferSmallDuringInitialShuffle = -1;
     size_t MaxNumberOfRuns = ULONG_MAX;
+    int SyncTimeout = 600;
     std::string OutputCorpus;
+    std::string SyncCommand;
     std::vector<std::string> Tokens;
   };
-  Fuzzer(UserCallback Callback, FuzzingOptions Options);
+  Fuzzer(UserSuppliedFuzzer &USF, FuzzingOptions Options);
   void AddToCorpus(const Unit &U) { Corpus.push_back(U); }
-  size_t Loop(size_t NumIterations);
+  void Loop(size_t NumIterations);
   void ShuffleAndMinimize();
-  void InitializeDFSan();
+  void InitializeTraceState();
   size_t CorpusSize() const { return Corpus.size(); }
-  void ReadDir(const std::string &Path) {
-    ReadDirToVectorOfUnits(Path.c_str(), &Corpus);
+  void ReadDir(const std::string &Path, long *Epoch) {
+    ReadDirToVectorOfUnits(Path.c_str(), &Corpus, Epoch);
   }
+  void RereadOutputCorpus();
   // Save the current corpus to OutputCorpus.
   void SaveCorpus();
 
@@ -84,17 +102,31 @@ class Fuzzer {
  private:
   void AlarmCallback();
   void ExecuteCallback(const Unit &U);
-  size_t MutateAndTestOne(Unit *U);
+  void MutateAndTestOne(Unit *U);
+  void ReportNewCoverage(size_t NewCoverage, const Unit &U);
   size_t RunOne(const Unit &U);
+  void RunOneAndUpdateCorpus(const Unit &U);
   size_t RunOneMaximizeTotalCoverage(const Unit &U);
   size_t RunOneMaximizeFullCoverageSet(const Unit &U);
   size_t RunOneMaximizeCoveragePairs(const Unit &U);
   void WriteToOutputCorpus(const Unit &U);
-  void WriteToCrash(const Unit &U, const char *Prefix);
-  bool MutateWithDFSan(Unit *U);
+  void WriteUnitToFileWithPrefix(const Unit &U, const char *Prefix);
   void PrintStats(const char *Where, size_t Cov, const char *End = "\n");
   void PrintUnitInASCIIOrTokens(const Unit &U, const char *PrintAfter = "");
 
+  void SyncCorpus();
+
+  // Trace-based fuzzing: we run a unit with some kind of tracing
+  // enabled and record potentially useful mutations. Then
+  // We apply these mutations one by one to the unit and run it again.
+
+  // Start tracing; forget all previously proposed mutations.
+  void StartTraceRecording();
+  // Stop tracing and return the number of proposed mutations.
+  size_t StopTraceRecording();
+  // Apply Idx-th trace-based mutation to U.
+  void ApplyTraceBasedMutation(size_t Idx, Unit *U);
+
   void SetDeathCallback();
   static void StaticDeathCallback();
   void DeathCallback();
@@ -103,8 +135,8 @@ class Fuzzer {
   size_t TotalNumberOfRuns = 0;
 
   std::vector<Unit> Corpus;
+  std::unordered_set<std::string> UnitHashesAddedToCorpus;
   std::unordered_set<uintptr_t> FullCoverageSets;
-  std::unordered_set<uint64_t>  CoveragePairs;
 
   // For UseCounters
   std::vector<uint8_t> CounterBitmap;
@@ -114,11 +146,25 @@ class Fuzzer {
     return Res;
   }
 
-  UserCallback Callback;
+  UserSuppliedFuzzer &USF;
   FuzzingOptions Options;
   system_clock::time_point ProcessStartTime = system_clock::now();
+  system_clock::time_point LastExternalSync = system_clock::now();
   system_clock::time_point UnitStartTime;
   long TimeOfLongestUnitInSeconds = 0;
+  long EpochOfLastReadOfOutputCorpus = 0;
+};
+
+class SimpleUserSuppliedFuzzer: public UserSuppliedFuzzer {
+ public:
+  SimpleUserSuppliedFuzzer(FuzzerRandomBase *Rand, UserCallback Callback)
+      : UserSuppliedFuzzer(Rand), Callback(Callback) {}
+  virtual void TargetFunction(const uint8_t *Data, size_t Size) {
+    return Callback(Data, Size);
+  }
+
+ private:
+  UserCallback Callback;
 };
 
 };  // namespace fuzzer
diff --git a/lib/Fuzzer/FuzzerLoop.cpp b/lib/Fuzzer/FuzzerLoop.cpp
index 9dfe30b6d79..e659223fe1a 100644
--- a/lib/Fuzzer/FuzzerLoop.cpp
+++ b/lib/Fuzzer/FuzzerLoop.cpp
@@ -12,17 +12,16 @@
 #include "FuzzerInternal.h"
 #include <sanitizer/coverage_interface.h>
 #include <algorithm>
-#include <iostream>
 
 namespace fuzzer {
 
 // Only one Fuzzer per process.
 static Fuzzer *F;
 
-Fuzzer::Fuzzer(UserCallback Callback, FuzzingOptions Options)
-    : Callback(Callback), Options(Options) {
+Fuzzer::Fuzzer(UserSuppliedFuzzer &USF, FuzzingOptions Options)
+    : USF(USF), Options(Options) {
   SetDeathCallback();
-  InitializeDFSan();
+  InitializeTraceState();
   assert(!F);
   F = this;
 }
@@ -37,8 +36,7 @@ void Fuzzer::PrintUnitInASCIIOrTokens(const Unit &U, const char *PrintAfter) {
   } else {
     auto T = SubstituteTokens(U);
     T.push_back(0);
-    std::cerr << T.data();
-    std::cerr << PrintAfter;
+    Printf("%s%s", T.data(), PrintAfter);
   }
 }
 
@@ -48,10 +46,10 @@ void Fuzzer::StaticDeathCallback() {
 }
 
 void Fuzzer::DeathCallback() {
-  std::cerr << "DEATH: " <<  std::endl;
+  Printf("DEATH:\n");
   Print(CurrentUnit, "\n");
   PrintUnitInASCIIOrTokens(CurrentUnit, "\n");
-  WriteToCrash(CurrentUnit, "crash-");
+  WriteUnitToFileWithPrefix(CurrentUnit, "crash-");
 }
 
 void Fuzzer::StaticAlarmCallback() {
@@ -60,42 +58,69 @@ void Fuzzer::StaticAlarmCallback() {
 }
 
 void Fuzzer::AlarmCallback() {
+  assert(Options.UnitTimeoutSec > 0);
   size_t Seconds =
       duration_cast<seconds>(system_clock::now() - UnitStartTime).count();
-  std::cerr << "ALARM: working on the last Unit for " << Seconds << " seconds"
-            << std::endl;
-  if (Seconds >= 3) {
+  if (Seconds == 0) return;
+  if (Options.Verbosity >= 2)
+    Printf("AlarmCallback %zd\n", Seconds);
+  if (Seconds >= (size_t)Options.UnitTimeoutSec) {
+    Printf("ALARM: working on the last Unit for %zd seconds\n", Seconds);
+    Printf("       and the timeout value is %d (use -timeout=N to change)\n",
+           Options.UnitTimeoutSec);
     Print(CurrentUnit, "\n");
     PrintUnitInASCIIOrTokens(CurrentUnit, "\n");
-    WriteToCrash(CurrentUnit, "timeout-");
+    WriteUnitToFileWithPrefix(CurrentUnit, "timeout-");
+    exit(1);
   }
-  exit(1);
 }
 
 void Fuzzer::PrintStats(const char *Where, size_t Cov, const char *End) {
   if (!Options.Verbosity) return;
   size_t Seconds = secondsSinceProcessStartUp();
   size_t ExecPerSec = (Seconds ? TotalNumberOfRuns / Seconds : 0);
-  std::cerr
-      << "#" << TotalNumberOfRuns
-      << "\t" << Where
-      << " cov " << Cov
-      << " bits " << TotalBits()
-      << " units " << Corpus.size()
-      << " exec/s " << ExecPerSec
-      << End;
+  Printf("#%zd\t%s cov %zd bits %zd units %zd exec/s %zd %s", TotalNumberOfRuns,
+         Where, Cov, TotalBits(), Corpus.size(), ExecPerSec, End);
+}
+
+void Fuzzer::RereadOutputCorpus() {
+  if (Options.OutputCorpus.empty()) return;
+  std::vector<Unit> AdditionalCorpus;
+  ReadDirToVectorOfUnits(Options.OutputCorpus.c_str(), &AdditionalCorpus,
+                         &EpochOfLastReadOfOutputCorpus);
+  if (Corpus.empty()) {
+    Corpus = AdditionalCorpus;
+    return;
+  }
+  if (!Options.Reload) return;
+  if (Options.Verbosity >= 2)
+    Printf("Reload: read %zd new units.\n",  AdditionalCorpus.size());
+  for (auto &X : AdditionalCorpus) {
+    if (X.size() > (size_t)Options.MaxLen)
+      X.resize(Options.MaxLen);
+    if (UnitHashesAddedToCorpus.insert(Hash(X)).second) {
+      CurrentUnit.clear();
+      CurrentUnit.insert(CurrentUnit.begin(), X.begin(), X.end());
+      size_t NewCoverage = RunOne(CurrentUnit);
+      if (NewCoverage) {
+        Corpus.push_back(X);
+        if (Options.Verbosity >= 1)
+          PrintStats("RELOAD", NewCoverage);
+      }
+    }
+  }
 }
 
 void Fuzzer::ShuffleAndMinimize() {
   size_t MaxCov = 0;
-  bool PreferSmall =
-      (Options.PreferSmallDuringInitialShuffle == 1 ||
-       (Options.PreferSmallDuringInitialShuffle == -1 && rand() % 2));
+  bool PreferSmall = (Options.PreferSmallDuringInitialShuffle == 1 ||
+                      (Options.PreferSmallDuringInitialShuffle == -1 &&
+                       USF.GetRand().RandBool()));
   if (Options.Verbosity)
-    std::cerr << "PreferSmall: " << PreferSmall << "\n";
+    Printf("PreferSmall: %d\n", PreferSmall);
   PrintStats("READ  ", 0);
   std::vector<Unit> NewCorpus;
-  std::random_shuffle(Corpus.begin(), Corpus.end());
+  std::random_shuffle(Corpus.begin(), Corpus.end(), USF.GetRand());
   if (PreferSmall)
     std::stable_sort(
         Corpus.begin(), Corpus.end(),
@@ -111,13 +136,13 @@ void Fuzzer::ShuffleAndMinimize() {
         MaxCov = NewCoverage;
         NewCorpus.push_back(U);
         if (Options.Verbosity >= 2)
-          std::cerr << "NEW0: " << NewCoverage
-                    << " L " << U.size()
-                    << "\n";
+          Printf("NEW0: %zd L %zd\n", NewCoverage, U.size());
       }
     }
   }
   Corpus = NewCorpus;
+  for (auto &X : Corpus)
+    UnitHashesAddedToCorpus.insert(Hash(X));
   PrintStats("INITED", MaxCov);
 }
 
@@ -127,8 +152,6 @@ size_t Fuzzer::RunOne(const Unit &U) {
   size_t Res = 0;
   if (Options.UseFullCoverageSet)
     Res = RunOneMaximizeFullCoverageSet(U);
-  else if (Options.UseCoveragePairs)
-    Res = RunOneMaximizeCoveragePairs(U);
   else
     Res = RunOneMaximizeTotalCoverage(U);
   auto UnitStopTime = system_clock::now();
@@ -136,13 +159,19 @@ size_t Fuzzer::RunOne(const Unit &U) {
       duration_cast<seconds>(UnitStopTime - UnitStartTime).count();
   if (TimeOfUnit > TimeOfLongestUnitInSeconds) {
     TimeOfLongestUnitInSeconds = TimeOfUnit;
-    std::cerr << "Longest unit: " << TimeOfLongestUnitInSeconds
-              << " s:\n";
+    Printf("Longest unit: %zd s:\n", TimeOfLongestUnitInSeconds);
     Print(U, "\n");
+    WriteUnitToFileWithPrefix(U, "long-running-unit-");
   }
   return Res;
 }
 
+void Fuzzer::RunOneAndUpdateCorpus(const Unit &U) {
+  if (TotalNumberOfRuns >= Options.MaxNumberOfRuns)
+    return;
+  ReportNewCoverage(RunOne(U), U);
+}
+
 static uintptr_t HashOfArrayOfPCs(uintptr_t *PCs, uintptr_t NumPCs) {
   uintptr_t Res = 0;
   for (uintptr_t i = 0; i < NumPCs; i++) {
@@ -167,33 +196,11 @@ Unit Fuzzer::SubstituteTokens(const Unit &U) const {
 
 void Fuzzer::ExecuteCallback(const Unit &U) {
   if (Options.Tokens.empty()) {
-    Callback(U.data(), U.size());
+    USF.TargetFunction(U.data(), U.size());
   } else {
     auto T = SubstituteTokens(U);
-    Callback(T.data(), T.size());
-  }
-}
-
-// Experimental. Does not yet scale.
-// Fuly reset the current coverage state, run a single unit,
-// collect all coverage pairs and return non-zero if a new pair is observed.
-size_t Fuzzer::RunOneMaximizeCoveragePairs(const Unit &U) {
-  __sanitizer_reset_coverage();
-  ExecuteCallback(U);
-  uintptr_t *PCs;
-  uintptr_t NumPCs = __sanitizer_get_coverage_guards(&PCs);
-  bool HasNewPairs = false;
-  for (uintptr_t i = 0; i < NumPCs; i++) {
-    if (!PCs[i]) continue;
-    for (uintptr_t j = 0; j < NumPCs; j++) {
-      if (!PCs[j]) continue;
-      uint64_t Pair = (i << 32) | j;
-      HasNewPairs |= CoveragePairs.insert(Pair).second;
-    }
+    USF.TargetFunction(T.data(), T.size());
   }
-  if (HasNewPairs)
-    return CoveragePairs.size();
-  return 0;
 }
 
 // Experimental.
@@ -239,13 +246,14 @@ void Fuzzer::WriteToOutputCorpus(const Unit &U) {
   std::string Path = DirPlusFile(Options.OutputCorpus, Hash(U));
   WriteToFile(U, Path);
   if (Options.Verbosity >= 2)
-    std::cerr << "Written to " << Path << std::endl;
+    Printf("Written to %s\n", Path.c_str());
 }
 
-void Fuzzer::WriteToCrash(const Unit &U, const char *Prefix) {
+void Fuzzer::WriteUnitToFileWithPrefix(const Unit &U, const char *Prefix) {
   std::string Path = Prefix + Hash(U);
   WriteToFile(U, Path);
-  std::cerr << "CRASHED; file written to " << Path << std::endl;
+  Printf("Test unit written to %s\nBase64: ", Path.c_str());
+  PrintFileAsBase64(Path);
 }
 
 void Fuzzer::SaveCorpus() {
@@ -253,59 +261,84 @@ void Fuzzer::SaveCorpus() {
   for (const auto &U : Corpus)
     WriteToFile(U, DirPlusFile(Options.OutputCorpus, Hash(U)));
   if (Options.Verbosity)
-    std::cerr << "Written corpus of " << Corpus.size() << " files to "
-              << Options.OutputCorpus << "\n";
+    Printf("Written corpus of %zd files to %s\n", Corpus.size(),
+           Options.OutputCorpus.c_str());
 }
 
-size_t Fuzzer::MutateAndTestOne(Unit *U) {
-  size_t NewUnits = 0;
+void Fuzzer::ReportNewCoverage(size_t NewCoverage, const Unit &U) {
+  if (!NewCoverage) return;
+  Corpus.push_back(U);
+  UnitHashesAddedToCorpus.insert(Hash(U));
+  PrintStats("NEW   ", NewCoverage, "");
+  if (Options.Verbosity) {
+    Printf(" L: %zd", U.size());
+    if (U.size() < 30) {
+      Printf(" ");
+      PrintUnitInASCIIOrTokens(U, "\t");
+      Print(U);
+    }
+    Printf("\n");
+  }
+  WriteToOutputCorpus(U);
+  if (Options.ExitOnFirst)
+    exit(0);
+}
+
+void Fuzzer::MutateAndTestOne(Unit *U) {
   for (int i = 0; i < Options.MutateDepth; i++) {
-    if (TotalNumberOfRuns >= Options.MaxNumberOfRuns)
-      return NewUnits;
-    MutateWithDFSan(U);
-    Mutate(U, Options.MaxLen);
-    size_t NewCoverage = RunOne(*U);
-    if (NewCoverage) {
-      Corpus.push_back(*U);
-      NewUnits++;
-      PrintStats("NEW   ", NewCoverage, "");
-      if (Options.Verbosity) {
-        std::cerr << " L: " << U->size();
-        if (U->size() < 30) {
-          std::cerr << " ";
-          PrintUnitInASCIIOrTokens(*U, "\t");
-          Print(*U);
-        }
-        std::cerr << "\n";
-      }
-      WriteToOutputCorpus(*U);
-      if (Options.ExitOnFirst)
-        exit(0);
+    StartTraceRecording();
+    size_t Size = U->size();
+    U->resize(Options.MaxLen);
+    size_t NewSize = USF.Mutate(U->data(), Size, U->size());
+    assert(NewSize > 0 && "Mutator returned empty unit");
+    assert(NewSize <= (size_t)Options.MaxLen &&
+           "Mutator return overisized unit");
+    U->resize(NewSize);
+    RunOneAndUpdateCorpus(*U);
+    size_t NumTraceBasedMutations = StopTraceRecording();
+    for (size_t j = 0; j < NumTraceBasedMutations; j++) {
+      ApplyTraceBasedMutation(j, U);
+      RunOneAndUpdateCorpus(*U);
     }
   }
-  return NewUnits;
 }
 
-size_t Fuzzer::Loop(size_t NumIterations) {
-  size_t NewUnits = 0;
+void Fuzzer::Loop(size_t NumIterations) {
   for (size_t i = 1; i <= NumIterations; i++) {
     for (size_t J1 = 0; J1 < Corpus.size(); J1++) {
+      SyncCorpus();
+      RereadOutputCorpus();
       if (TotalNumberOfRuns >= Options.MaxNumberOfRuns)
-        return NewUnits;
+        return;
       // First, simply mutate the unit w/o doing crosses.
       CurrentUnit = Corpus[J1];
-      NewUnits += MutateAndTestOne(&CurrentUnit);
+      MutateAndTestOne(&CurrentUnit);
       // Now, cross with others.
-      if (Options.DoCrossOver) {
+      if (Options.DoCrossOver && !Corpus[J1].empty()) {
         for (size_t J2 = 0; J2 < Corpus.size(); J2++) {
-          CurrentUnit.clear();
-          CrossOver(Corpus[J1], Corpus[J2], &CurrentUnit, Options.MaxLen);
-          NewUnits += MutateAndTestOne(&CurrentUnit);
+          CurrentUnit.resize(Options.MaxLen);
+          size_t NewSize = USF.CrossOver(
+              Corpus[J1].data(), Corpus[J1].size(), Corpus[J2].data(),
+              Corpus[J2].size(), CurrentUnit.data(), CurrentUnit.size());
+          assert(NewSize > 0 && "CrossOver returned empty unit");
+          assert(NewSize <= (size_t)Options.MaxLen &&
+                 "CrossOver return overisized unit");
+          CurrentUnit.resize(NewSize);
+          MutateAndTestOne(&CurrentUnit);
         }
       }
     }
   }
-  return NewUnits;
+}
+
+void Fuzzer::SyncCorpus() {
+  if (Options.SyncCommand.empty() || Options.OutputCorpus.empty()) return;
+  auto Now = system_clock::now();
+  if (duration_cast<seconds>(Now - LastExternalSync).count() <
+      Options.SyncTimeout)
+    return;
+  LastExternalSync = Now;
+  ExecuteCommand(Options.SyncCommand + " " + Options.OutputCorpus);
 }
 
 }  // namespace fuzzer
diff --git a/lib/Fuzzer/FuzzerMain.cpp b/lib/Fuzzer/FuzzerMain.cpp
index d0c3df3b6b5..c4dffb45d16 100644
--- a/lib/Fuzzer/FuzzerMain.cpp
+++ b/lib/Fuzzer/FuzzerMain.cpp
@@ -13,8 +13,8 @@
 #include "FuzzerInternal.h"
 
 // This function should be defined by the user.
-extern "C" void TestOneInput(const uint8_t *Data, size_t Size);
+extern "C" void LLVMFuzzerTestOneInput(const uint8_t *Data, size_t Size);
 
 int main(int argc, char **argv) {
-  return fuzzer::FuzzerDriver(argc, argv, TestOneInput);
+  return fuzzer::FuzzerDriver(argc, argv, LLVMFuzzerTestOneInput);
 }
diff --git a/lib/Fuzzer/FuzzerMutate.cpp b/lib/Fuzzer/FuzzerMutate.cpp
index b28264ac8c1..66df98a66f5 100644
--- a/lib/Fuzzer/FuzzerMutate.cpp
+++ b/lib/Fuzzer/FuzzerMutate.cpp
@@ -9,12 +9,14 @@
 // Mutate a test input.
 //===----------------------------------------------------------------------===//
 
+#include <cstring>
+
 #include "FuzzerInternal.h"
 
 namespace fuzzer {
 
-static char FlipRandomBit(char X) {
-  int Bit = rand() % 8;
+static char FlipRandomBit(char X, FuzzerRandomBase &Rand) {
+  int Bit = Rand(8);
   char Mask = 1 << Bit;
   char R;
   if (X & (1 << Bit))
@@ -25,46 +27,46 @@ static char FlipRandomBit(char X) {
   return R;
 }
 
-static char RandCh() {
-  if (rand() % 2) return rand();
+static char RandCh(FuzzerRandomBase &Rand) {
+  if (Rand.RandBool()) return Rand(256);
   const char *Special = "!*'();:@&=+$,/?%#[]123ABCxyz-`~.";
-  return Special[rand() % (sizeof(Special) - 1)];
+  return Special[Rand(sizeof(Special) - 1)];
 }
 
-// Mutate U in place.
-void Mutate(Unit *U, size_t MaxLen) {
-  assert(MaxLen > 0);
-  assert(U->size() <= MaxLen);
-  if (U->empty()) {
-    for (size_t i = 0; i < MaxLen; i++)
-      U->push_back(RandCh());
-    return;
+// Mutates Data in place, returns new size.
+size_t Mutate(uint8_t *Data, size_t Size, size_t MaxSize,
+              FuzzerRandomBase &Rand) {
+  assert(MaxSize > 0);
+  assert(Size <= MaxSize);
+  if (Size == 0) {
+    for (size_t i = 0; i < MaxSize; i++)
+      Data[i] = RandCh(Rand);
+    return MaxSize;
   }
-  assert(!U->empty());
-  switch (rand() % 3) {
+  assert(Size > 0);
+  size_t Idx = Rand(Size);
+  switch (Rand(3)) {
   case 0:
-    if (U->size() > 1) {
-      U->erase(U->begin() + rand() % U->size());
-      break;
+    if (Size > 1) {
+      // Erase Data[Idx].
+      memmove(Data + Idx, Data + Idx + 1, Size - Idx - 1);
+      Size = Size - 1;
     }
     [[clang::fallthrough]];
   case 1:
-    if (U->size() < MaxLen) {
-      U->insert(U->begin() + rand() % U->size(), RandCh());
-    } else { // At MaxLen.
-      uint8_t Ch = RandCh();
-      size_t Idx = rand() % U->size();
-      (*U)[Idx] = Ch;
+    if (Size < MaxSize) {
+      // Insert new value at Data[Idx].
+      memmove(Data + Idx + 1, Data + Idx, Size - Idx);
+      Data[Idx] = RandCh(Rand);
     }
+    Data[Idx] = RandCh(Rand);
     break;
-  default:
-    {
-      size_t Idx = rand() % U->size();
-      (*U)[Idx] = FlipRandomBit((*U)[Idx]);
-    }
+  case 2:
+    Data[Idx] = FlipRandomBit(Data[Idx], Rand);
     break;
   }
-  assert(!U->empty());
+  assert(Size > 0);
+  return Size;
 }
 
 }  // namespace fuzzer
diff --git a/lib/Fuzzer/FuzzerSHA1.cpp b/lib/Fuzzer/FuzzerSHA1.cpp
new file mode 100644
index 00000000000..b42a04854cd
--- /dev/null
+++ b/lib/Fuzzer/FuzzerSHA1.cpp
@@ -0,0 +1,202 @@
+//===- FuzzerSHA1.h - Private copy of the SHA1 implementation ---*- C++ -* ===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+// This code is taken from public domain
+// (http://oauth.googlecode.com/svn/code/c/liboauth/src/sha1.c)
+// and modified by adding anonymous namespace, adding an interface
+// function fuzzer::ComputeSHA1() and removing unnecessary code.
+//
+// lib/Fuzzer can not use SHA1 implementation from openssl because
+// openssl may not be available and because we may be fuzzing openssl itself.
+// For the same reason we do not want to depend on SHA1 from LLVM tree.
+//===----------------------------------------------------------------------===//
+
+#include "FuzzerInternal.h"
+
+/* This code is public-domain - it is based on libcrypt
+ * placed in the public domain by Wei Dai and other contributors.
+ */
+
+#include <stdint.h>
+#include <string.h>
+
+namespace {  // Added for LibFuzzer
+
+#ifdef __BIG_ENDIAN__
+# define SHA_BIG_ENDIAN
+#elif defined __LITTLE_ENDIAN__
+/* override */
+#elif defined __BYTE_ORDER
+# if __BYTE_ORDER__ ==  __ORDER_BIG_ENDIAN__
+# define SHA_BIG_ENDIAN
+# endif
+#else // ! defined __LITTLE_ENDIAN__
+# include <endian.h> // machine/endian.h
+# if __BYTE_ORDER__ ==  __ORDER_BIG_ENDIAN__
+#  define SHA_BIG_ENDIAN
+# endif
+#endif
+
+
+/* header */
+
+#define HASH_LENGTH 20
+#define BLOCK_LENGTH 64
+
+typedef struct sha1nfo {
+	uint32_t buffer[BLOCK_LENGTH/4];
+	uint32_t state[HASH_LENGTH/4];
+	uint32_t byteCount;
+	uint8_t bufferOffset;
+	uint8_t keyBuffer[BLOCK_LENGTH];
+	uint8_t innerHash[HASH_LENGTH];
+} sha1nfo;
+
+/* public API - prototypes - TODO: doxygen*/
+
+/**
+ */
+void sha1_init(sha1nfo *s);
+/**
+ */
+void sha1_writebyte(sha1nfo *s, uint8_t data);
+/**
+ */
+void sha1_write(sha1nfo *s, const char *data, size_t len);
+/**
+ */
+uint8_t* sha1_result(sha1nfo *s);
+
+
+/* code */
+#define SHA1_K0  0x5a827999
+#define SHA1_K20 0x6ed9eba1
+#define SHA1_K40 0x8f1bbcdc
+#define SHA1_K60 0xca62c1d6
+
+void sha1_init(sha1nfo *s) {
+	s->state[0] = 0x67452301;
+	s->state[1] = 0xefcdab89;
+	s->state[2] = 0x98badcfe;
+	s->state[3] = 0x10325476;
+	s->state[4] = 0xc3d2e1f0;
+	s->byteCount = 0;
+	s->bufferOffset = 0;
+}
+
+uint32_t sha1_rol32(uint32_t number, uint8_t bits) {
+	return ((number << bits) | (number >> (32-bits)));
+}
+
+void sha1_hashBlock(sha1nfo *s) {
+	uint8_t i;
+	uint32_t a,b,c,d,e,t;
+
+	a=s->state[0];
+	b=s->state[1];
+	c=s->state[2];
+	d=s->state[3];
+	e=s->state[4];
+	for (i=0; i<80; i++) {
+		if (i>=16) {
+			t = s->buffer[(i+13)&15] ^ s->buffer[(i+8)&15] ^ s->buffer[(i+2)&15] ^ s->buffer[i&15];
+			s->buffer[i&15] = sha1_rol32(t,1);
+		}
+		if (i<20) {
+			t = (d ^ (b & (c ^ d))) + SHA1_K0;
+		} else if (i<40) {
+			t = (b ^ c ^ d) + SHA1_K20;
+		} else if (i<60) {
+			t = ((b & c) | (d & (b | c))) + SHA1_K40;
+		} else {
+			t = (b ^ c ^ d) + SHA1_K60;
+		}
+		t+=sha1_rol32(a,5) + e + s->buffer[i&15];
+		e=d;
+		d=c;
+		c=sha1_rol32(b,30);
+		b=a;
+		a=t;
+	}
+	s->state[0] += a;
+	s->state[1] += b;
+	s->state[2] += c;
+	s->state[3] += d;
+	s->state[4] += e;
+}
+
+void sha1_addUncounted(sha1nfo *s, uint8_t data) {
+	uint8_t * const b = (uint8_t*) s->buffer;
+#ifdef SHA_BIG_ENDIAN
+	b[s->bufferOffset] = data;
+#else
+	b[s->bufferOffset ^ 3] = data;
+#endif
+	s->bufferOffset++;
+	if (s->bufferOffset == BLOCK_LENGTH) {
+		sha1_hashBlock(s);
+		s->bufferOffset = 0;
+	}
+}
+
+void sha1_writebyte(sha1nfo *s, uint8_t data) {
+	++s->byteCount;
+	sha1_addUncounted(s, data);
+}
+
+void sha1_write(sha1nfo *s, const char *data, size_t len) {
+	for (;len--;) sha1_writebyte(s, (uint8_t) *data++);
+}
+
+void sha1_pad(sha1nfo *s) {
+	// Implement SHA-1 padding (fips180-2 §5.1.1)
+
+	// Pad with 0x80 followed by 0x00 until the end of the block
+	sha1_addUncounted(s, 0x80);
+	while (s->bufferOffset != 56) sha1_addUncounted(s, 0x00);
+
+	// Append length in the last 8 bytes
+	sha1_addUncounted(s, 0); // We're only using 32 bit lengths
+	sha1_addUncounted(s, 0); // But SHA-1 supports 64 bit lengths
+	sha1_addUncounted(s, 0); // So zero pad the top bits
+	sha1_addUncounted(s, s->byteCount >> 29); // Shifting to multiply by 8
+	sha1_addUncounted(s, s->byteCount >> 21); // as SHA-1 supports bitstreams as well as
+	sha1_addUncounted(s, s->byteCount >> 13); // byte.
+	sha1_addUncounted(s, s->byteCount >> 5);
+	sha1_addUncounted(s, s->byteCount << 3);
+}
+
+uint8_t* sha1_result(sha1nfo *s) {
+	// Pad to complete the last block
+	sha1_pad(s);
+
+#ifndef SHA_BIG_ENDIAN
+	// Swap byte order back
+	int i;
+	for (i=0; i<5; i++) {
+		s->state[i]=
+			  (((s->state[i])<<24)& 0xff000000)
+			| (((s->state[i])<<8) & 0x00ff0000)
+			| (((s->state[i])>>8) & 0x0000ff00)
+			| (((s->state[i])>>24)& 0x000000ff);
+	}
+#endif
+
+	// Return pointer to hash (20 characters)
+	return (uint8_t*) s->state;
+}
+
+}  // namespace; Added for LibFuzzer
+
+// The rest is added for LibFuzzer
+void fuzzer::ComputeSHA1(const uint8_t *Data, size_t Len, uint8_t *Out) {
+  sha1nfo s;
+  sha1_init(&s);
+  sha1_write(&s, (const char*)Data, Len);
+  memcpy(Out, sha1_result(&s), HASH_LENGTH);
+}
diff --git a/lib/Fuzzer/FuzzerSanitizerOptions.cpp b/lib/Fuzzer/FuzzerSanitizerOptions.cpp
index 1c58f3a0694..b2f20dd4ddf 100644
--- a/lib/Fuzzer/FuzzerSanitizerOptions.cpp
+++ b/lib/Fuzzer/FuzzerSanitizerOptions.cpp
@@ -12,7 +12,8 @@
 // ASAN options:
 //   * don't dump the coverage to disk.
 //   * enable coverage by default.
+//   * enable handle_abort.
 //===----------------------------------------------------------------------===//
 extern "C" const char *__asan_default_options() {
-  return "coverage_pcs=0:coverage=1";
+  return "coverage_pcs=0:coverage=1:handle_abort=1";
 }
diff --git a/lib/Fuzzer/FuzzerTraceState.cpp b/lib/Fuzzer/FuzzerTraceState.cpp
new file mode 100644
index 00000000000..5eae2ec8882
--- /dev/null
+++ b/lib/Fuzzer/FuzzerTraceState.cpp
@@ -0,0 +1,504 @@
+//===- FuzzerTraceState.cpp - Trace-based fuzzer mutator ------------------===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+// This file implements a mutation algorithm based on instruction traces and
+// on taint analysis feedback from DFSan.
+//
+// Instruction traces are special hooks inserted by the compiler around
+// interesting instructions. Currently supported traces:
+//   * __sanitizer_cov_trace_cmp -- inserted before every ICMP instruction,
+//    receives the type, size and arguments of ICMP.
+//
+// Every time a traced event is intercepted we analyse the data involved
+// in the event and suggest a mutation for future executions.
+// For example if 4 bytes of data that derive from input bytes {4,5,6,7}
+// are compared with a constant 12345,
+// we try to insert 12345, 12344, 12346 into bytes
+// {4,5,6,7} of the next fuzzed inputs.
+//
+// The fuzzer can work only with the traces, or with both traces and DFSan.
+//
+// DataFlowSanitizer (DFSan) is a tool for
+// generalised dynamic data flow (taint) analysis:
+// http://clang.llvm.org/docs/DataFlowSanitizer.html .
+//
+// The approach with DFSan-based fuzzing has some similarity to
+// "Taint-based Directed Whitebox Fuzzing"
+// by Vijay Ganesh & Tim Leek & Martin Rinard:
+// http://dspace.mit.edu/openaccess-disseminate/1721.1/59320,
+// but it uses a full blown LLVM IR taint analysis and separate instrumentation
+// to analyze all of the "attack points" at once.
+//
+// Workflow with DFSan:
+//   * lib/Fuzzer/Fuzzer*.cpp is compiled w/o any instrumentation.
+//   * The code under test is compiled with DFSan *and* with instruction traces.
+//   * Every call to HOOK(a,b) is replaced by DFSan with
+//     __dfsw_HOOK(a, b, label(a), label(b)) so that __dfsw_HOOK
+//     gets all the taint labels for the arguments.
+//   * At the Fuzzer startup we assign a unique DFSan label
+//     to every byte of the input string (Fuzzer::CurrentUnit) so that for any
+//     chunk of data we know which input bytes it has derived from.
+//   * The __dfsw_* functions (implemented in this file) record the
+//     parameters (i.e. the application data and the corresponding taint labels)
+//     in a global state.
+//   * Fuzzer::ApplyTraceBasedMutation() tries to use the data recorded
+//     by __dfsw_* hooks to guide the fuzzing towards new application states.
+//
+// Parts of this code will not function when DFSan is not linked in.
+// Instead of using ifdefs and thus requiring a separate build of lib/Fuzzer
+// we redeclare the dfsan_* interface functions as weak and check if they
+// are nullptr before calling.
+// If this approach proves to be useful we may add attribute(weak) to the
+// dfsan declarations in dfsan_interface.h
+//
+// This module is in the "proof of concept" stage.
+// It is capable of solving only the simplest puzzles
+// like test/dfsan/DFSanSimpleCmpTest.cpp.
+//===----------------------------------------------------------------------===//
+
+/* Example of manual usage (-fsanitize=dataflow is optional):
+(
+  cd $LLVM/lib/Fuzzer/
+  clang  -fPIC -c -g -O2 -std=c++11 Fuzzer*.cpp
+  clang++ -O0 -std=c++11 -fsanitize-coverage=edge,trace-cmp \
+    -fsanitize=dataflow \
+    test/dfsan/DFSanSimpleCmpTest.cpp Fuzzer*.o
+  ./a.out
+)
+*/
+
+#include "FuzzerInternal.h"
+#include <sanitizer/dfsan_interface.h>
+
+#include <algorithm>
+#include <cstring>
+#include <unordered_map>
+
+extern "C" {
+__attribute__((weak))
+dfsan_label dfsan_create_label(const char *desc, void *userdata);
+__attribute__((weak))
+void dfsan_set_label(dfsan_label label, void *addr, size_t size);
+__attribute__((weak))
+void dfsan_add_label(dfsan_label label, void *addr, size_t size);
+__attribute__((weak))
+const struct dfsan_label_info *dfsan_get_label_info(dfsan_label label);
+__attribute__((weak))
+dfsan_label dfsan_read_label(const void *addr, size_t size);
+}  // extern "C"
+
+namespace fuzzer {
+
+static bool ReallyHaveDFSan() {
+  return &dfsan_create_label != nullptr;
+}
+
+// These values are copied from include/llvm/IR/InstrTypes.h.
+// We do not include the LLVM headers here to remain independent.
+// If these values ever change, an assertion in ComputeCmp will fail.
+enum Predicate {
+  ICMP_EQ = 32,  ///< equal
+  ICMP_NE = 33,  ///< not equal
+  ICMP_UGT = 34, ///< unsigned greater than
+  ICMP_UGE = 35, ///< unsigned greater or equal
+  ICMP_ULT = 36, ///< unsigned less than
+  ICMP_ULE = 37, ///< unsigned less or equal
+  ICMP_SGT = 38, ///< signed greater than
+  ICMP_SGE = 39, ///< signed greater or equal
+  ICMP_SLT = 40, ///< signed less than
+  ICMP_SLE = 41, ///< signed less or equal
+};
+
+template <class U, class S>
+bool ComputeCmp(size_t CmpType, U Arg1, U Arg2) {
+  switch(CmpType) {
+    case ICMP_EQ : return Arg1 == Arg2;
+    case ICMP_NE : return Arg1 != Arg2;
+    case ICMP_UGT: return Arg1 > Arg2;
+    case ICMP_UGE: return Arg1 >= Arg2;
+    case ICMP_ULT: return Arg1 < Arg2;
+    case ICMP_ULE: return Arg1 <= Arg2;
+    case ICMP_SGT: return (S)Arg1 > (S)Arg2;
+    case ICMP_SGE: return (S)Arg1 >= (S)Arg2;
+    case ICMP_SLT: return (S)Arg1 < (S)Arg2;
+    case ICMP_SLE: return (S)Arg1 <= (S)Arg2;
+    default: assert(0 && "unsupported CmpType");
+  }
+  return false;
+}
+
+static bool ComputeCmp(size_t CmpSize, size_t CmpType, uint64_t Arg1,
+                       uint64_t Arg2) {
+  if (CmpSize == 8) return ComputeCmp<uint64_t, int64_t>(CmpType, Arg1, Arg2);
+  if (CmpSize == 4) return ComputeCmp<uint32_t, int32_t>(CmpType, Arg1, Arg2);
+  if (CmpSize == 2) return ComputeCmp<uint16_t, int16_t>(CmpType, Arg1, Arg2);
+  if (CmpSize == 1) return ComputeCmp<uint8_t, int8_t>(CmpType, Arg1, Arg2);
+  // Other size, ==
+  if (CmpType == ICMP_EQ) return Arg1 == Arg2;
+  assert(0 && "unsupported cmp and type size combination");
+  return true;
+}
+
+// As a simplification we use the range of input bytes instead of a set of input
+// bytes.
+struct LabelRange {
+  uint16_t Beg, End;  // Range is [Beg, End), thus Beg==End is an empty range.
+
+  LabelRange(uint16_t Beg = 0, uint16_t End = 0) : Beg(Beg), End(End) {}
+
+  static LabelRange Join(LabelRange LR1, LabelRange LR2) {
+    if (LR1.Beg == LR1.End) return LR2;
+    if (LR2.Beg == LR2.End) return LR1;
+    return {std::min(LR1.Beg, LR2.Beg), std::max(LR1.End, LR2.End)};
+  }
+  LabelRange &Join(LabelRange LR) {
+    return *this = Join(*this, LR);
+  }
+  static LabelRange Singleton(const dfsan_label_info *LI) {
+    uint16_t Idx = (uint16_t)(uintptr_t)LI->userdata;
+    assert(Idx > 0);
+    return {(uint16_t)(Idx - 1), Idx};
+  }
+};
+
+// A passport for a CMP site. We want to keep track of where the given CMP is
+// and how many times it is evaluated to true or false.
+struct CmpSitePassport {
+  uintptr_t PC;
+  size_t Counter[2];
+
+  bool IsInterestingCmpTarget() {
+    static const size_t kRareEnough = 50;
+    size_t C0 = Counter[0];
+    size_t C1 = Counter[1];
+    return C0 > kRareEnough * (C1 + 1) || C1 > kRareEnough * (C0 + 1);
+  }
+};
+
+// For now, just keep a simple imprecise hash table PC => CmpSitePassport.
+// Potentially, will need to have a compiler support to have a precise mapping
+// and also thread-safety.
+struct CmpSitePassportTable {
+  static const size_t kSize = 99991;  // Prime.
+  CmpSitePassport Passports[kSize];
+
+  CmpSitePassport *GetPassport(uintptr_t PC) {
+    uintptr_t Idx = PC & kSize;
+    CmpSitePassport *Res = &Passports[Idx];
+    if (Res->PC == 0)  // Not thread safe.
+      Res->PC = PC;
+    return Res->PC == PC ? Res : nullptr;
+  }
+};
+
+static CmpSitePassportTable CSPTable;  // Zero initialized.
+
+// For now, very simple: put Size bytes of Data at position Pos.
+struct TraceBasedMutation {
+  size_t Pos;
+  size_t Size;
+  uint64_t Data;
+};
+
+class TraceState {
+ public:
+   TraceState(const Fuzzer::FuzzingOptions &Options, const Unit &CurrentUnit)
+       : Options(Options), CurrentUnit(CurrentUnit) {}
+
+  LabelRange GetLabelRange(dfsan_label L);
+  void DFSanCmpCallback(uintptr_t PC, size_t CmpSize, size_t CmpType,
+                        uint64_t Arg1, uint64_t Arg2, dfsan_label L1,
+                        dfsan_label L2);
+  void DFSanSwitchCallback(uint64_t PC, size_t ValSizeInBits, uint64_t Val,
+                           size_t NumCases, uint64_t *Cases, dfsan_label L);
+  void TraceCmpCallback(uintptr_t PC, size_t CmpSize, size_t CmpType, uint64_t Arg1,
+                        uint64_t Arg2);
+
+  void TraceSwitchCallback(uintptr_t PC, size_t ValSizeInBits, uint64_t Val,
+                           size_t NumCases, uint64_t *Cases);
+  int TryToAddDesiredData(uint64_t PresentData, uint64_t DesiredData,
+                           size_t DataSize);
+
+  void StartTraceRecording() {
+    if (!Options.UseTraces) return;
+    RecordingTraces = true;
+    Mutations.clear();
+  }
+
+  size_t StopTraceRecording(FuzzerRandomBase &Rand) {
+    RecordingTraces = false;
+    std::random_shuffle(Mutations.begin(), Mutations.end(), Rand);
+    return Mutations.size();
+  }
+
+  void ApplyTraceBasedMutation(size_t Idx, fuzzer::Unit *U);
+
+ private:
+  bool IsTwoByteData(uint64_t Data) {
+    int64_t Signed = static_cast<int64_t>(Data);
+    Signed >>= 16;
+    return Signed == 0 || Signed == -1L;
+  }
+  bool RecordingTraces = false;
+  std::vector<TraceBasedMutation> Mutations;
+  LabelRange LabelRanges[1 << (sizeof(dfsan_label) * 8)] = {};
+  const Fuzzer::FuzzingOptions &Options;
+  const Unit &CurrentUnit;
+};
+
+LabelRange TraceState::GetLabelRange(dfsan_label L) {
+  LabelRange &LR = LabelRanges[L];
+  if (LR.Beg < LR.End || L == 0)
+    return LR;
+  const dfsan_label_info *LI = dfsan_get_label_info(L);
+  if (LI->l1 || LI->l2)
+    return LR = LabelRange::Join(GetLabelRange(LI->l1), GetLabelRange(LI->l2));
+  return LR = LabelRange::Singleton(LI);
+}
+
+void TraceState::ApplyTraceBasedMutation(size_t Idx, fuzzer::Unit *U) {
+  assert(Idx < Mutations.size());
+  auto &M = Mutations[Idx];
+  if (Options.Verbosity >= 3)
+    Printf("TBM %zd %zd %zd\n", M.Pos, M.Size, M.Data);
+  if (M.Pos + M.Size > U->size()) return;
+  memcpy(U->data() + M.Pos, &M.Data, M.Size);
+}
+
+void TraceState::DFSanCmpCallback(uintptr_t PC, size_t CmpSize, size_t CmpType,
+                                  uint64_t Arg1, uint64_t Arg2, dfsan_label L1,
+                                  dfsan_label L2) {
+  assert(ReallyHaveDFSan());
+  if (!RecordingTraces) return;
+  if (L1 == 0 && L2 == 0)
+    return;  // Not actionable.
+  if (L1 != 0 && L2 != 0)
+    return;  // Probably still actionable.
+  bool Res = ComputeCmp(CmpSize, CmpType, Arg1, Arg2);
+  uint64_t Data = L1 ? Arg2 : Arg1;
+  LabelRange LR = L1 ? GetLabelRange(L1) : GetLabelRange(L2);
+
+  for (size_t Pos = LR.Beg; Pos + CmpSize <= LR.End; Pos++) {
+    Mutations.push_back({Pos, CmpSize, Data});
+    Mutations.push_back({Pos, CmpSize, Data + 1});
+    Mutations.push_back({Pos, CmpSize, Data - 1});
+  }
+
+  if (CmpSize > LR.End - LR.Beg)
+    Mutations.push_back({LR.Beg, (unsigned)(LR.End - LR.Beg), Data});
+
+
+  if (Options.Verbosity >= 3)
+    Printf("DFSanCmpCallback: PC %lx S %zd T %zd A1 %llx A2 %llx R %d L1 %d L2 "
+           "%d MU %zd\n",
+           PC, CmpSize, CmpType, Arg1, Arg2, Res, L1, L2, Mutations.size());
+}
+
+void TraceState::DFSanSwitchCallback(uint64_t PC, size_t ValSizeInBits,
+                                     uint64_t Val, size_t NumCases,
+                                     uint64_t *Cases, dfsan_label L) {
+  assert(ReallyHaveDFSan());
+  if (!RecordingTraces) return;
+  if (!L) return;  // Not actionable.
+  LabelRange LR = GetLabelRange(L);
+  size_t ValSize = ValSizeInBits / 8;
+  for (size_t Pos = LR.Beg; Pos + ValSize <= LR.End; Pos++) {
+    for (size_t i = 0; i < NumCases; i++) {
+      Mutations.push_back({Pos, ValSize, Cases[i]});
+      Mutations.push_back({Pos, ValSize, Cases[i] + 1});
+      Mutations.push_back({Pos, ValSize, Cases[i] - 1});
+    }
+  }
+  if (Options.Verbosity >= 3)
+    Printf("DFSanSwitchCallback: PC %lx Val %zd # %zd L %d\n", PC, Val,
+           NumCases, L);
+}
+
+int TraceState::TryToAddDesiredData(uint64_t PresentData, uint64_t DesiredData,
+                                    size_t DataSize) {
+  int Res = 0;
+  const uint8_t *Beg = CurrentUnit.data();
+  const uint8_t *End = Beg + CurrentUnit.size();
+  for (const uint8_t *Cur = Beg; Cur < End; Cur += DataSize) {
+    Cur = (uint8_t *)memmem(Cur, End - Cur, &PresentData, DataSize);
+    if (!Cur)
+      break;
+    size_t Pos = Cur - Beg;
+    assert(Pos < CurrentUnit.size());
+    if (Mutations.size() > 100000U) return Res;  // Just in case.
+    Mutations.push_back({Pos, DataSize, DesiredData});
+    Mutations.push_back({Pos, DataSize, DesiredData + 1});
+    Mutations.push_back({Pos, DataSize, DesiredData - 1});
+    Cur += DataSize;
+    Res++;
+  }
+  return Res;
+}
+
+void TraceState::TraceCmpCallback(uintptr_t PC, size_t CmpSize, size_t CmpType, uint64_t Arg1,
+                        uint64_t Arg2) {
+  if (!RecordingTraces) return;
+  int Added = 0;
+  CmpSitePassport *CSP = CSPTable.GetPassport(PC);
+  if (!CSP) return;
+  CSP->Counter[ComputeCmp(CmpSize, CmpType, Arg1, Arg2)]++;
+  size_t C0 = CSP->Counter[0];
+  size_t C1 = CSP->Counter[1];
+  if (!CSP->IsInterestingCmpTarget())
+    return;
+  if (Options.Verbosity >= 3)
+    Printf("TraceCmp: %p %zd/%zd; %zd %zd\n", CSP->PC, C0, C1, Arg1, Arg2);
+  Added += TryToAddDesiredData(Arg1, Arg2, CmpSize);
+  Added += TryToAddDesiredData(Arg2, Arg1, CmpSize);
+  if (!Added && CmpSize == 4 && IsTwoByteData(Arg1) && IsTwoByteData(Arg2)) {
+    Added += TryToAddDesiredData(Arg1, Arg2, 2);
+    Added += TryToAddDesiredData(Arg2, Arg1, 2);
+  }
+}
+
+void TraceState::TraceSwitchCallback(uintptr_t PC, size_t ValSizeInBits,
+                                     uint64_t Val, size_t NumCases,
+                                     uint64_t *Cases) {
+  if (!RecordingTraces) return;
+  for (size_t i = 0; i < NumCases; i++)
+    TryToAddDesiredData(Val, Cases[i], ValSizeInBits / 8);
+}
+
+static TraceState *TS;
+
+void Fuzzer::StartTraceRecording() {
+  if (!TS) return;
+  TS->StartTraceRecording();
+}
+
+size_t Fuzzer::StopTraceRecording() {
+  if (!TS) return 0;
+  return TS->StopTraceRecording(USF.GetRand());
+}
+
+void Fuzzer::ApplyTraceBasedMutation(size_t Idx, Unit *U) {
+  assert(TS);
+  TS->ApplyTraceBasedMutation(Idx, U);
+}
+
+void Fuzzer::InitializeTraceState() {
+  if (!Options.UseTraces) return;
+  TS = new TraceState(Options, CurrentUnit);
+  CurrentUnit.resize(Options.MaxLen);
+  // The rest really requires DFSan.
+  if (!ReallyHaveDFSan()) return;
+  for (size_t i = 0; i < static_cast<size_t>(Options.MaxLen); i++) {
+    dfsan_label L = dfsan_create_label("input", (void*)(i + 1));
+    // We assume that no one else has called dfsan_create_label before.
+    assert(L == i + 1);
+    dfsan_set_label(L, &CurrentUnit[i], 1);
+  }
+}
+
+static size_t InternalStrnlen(const char *S, size_t MaxLen) {
+  size_t Len = 0;
+  for (; Len < MaxLen && S[Len]; Len++) {}
+  return Len;
+}
+
+}  // namespace fuzzer
+
+using fuzzer::TS;
+
+extern "C" {
+void __dfsw___sanitizer_cov_trace_cmp(uint64_t SizeAndType, uint64_t Arg1,
+                                      uint64_t Arg2, dfsan_label L0,
+                                      dfsan_label L1, dfsan_label L2) {
+  if (!TS) return;
+  assert(L0 == 0);
+  uintptr_t PC = reinterpret_cast<uintptr_t>(__builtin_return_address(0));
+  uint64_t CmpSize = (SizeAndType >> 32) / 8;
+  uint64_t Type = (SizeAndType << 32) >> 32;
+  TS->DFSanCmpCallback(PC, CmpSize, Type, Arg1, Arg2, L1, L2);
+}
+
+void __dfsw___sanitizer_cov_trace_switch(uint64_t Val, uint64_t *Cases,
+                                         dfsan_label L1, dfsan_label L2) {
+  if (!TS) return;
+  uintptr_t PC = reinterpret_cast<uintptr_t>(__builtin_return_address(0));
+  TS->DFSanSwitchCallback(PC, Cases[1], Val, Cases[0], Cases+2, L1);
+}
+
+void dfsan_weak_hook_memcmp(void *caller_pc, const void *s1, const void *s2,
+                            size_t n, dfsan_label s1_label,
+                            dfsan_label s2_label, dfsan_label n_label) {
+  if (!TS) return;
+  uintptr_t PC = reinterpret_cast<uintptr_t>(caller_pc);
+  uint64_t S1 = 0, S2 = 0;
+  // Simplification: handle only first 8 bytes.
+  memcpy(&S1, s1, std::min(n, sizeof(S1)));
+  memcpy(&S2, s2, std::min(n, sizeof(S2)));
+  dfsan_label L1 = dfsan_read_label(s1, n);
+  dfsan_label L2 = dfsan_read_label(s2, n);
+  TS->DFSanCmpCallback(PC, n, fuzzer::ICMP_EQ, S1, S2, L1, L2);
+}
+
+void dfsan_weak_hook_strncmp(void *caller_pc, const char *s1, const char *s2,
+                             size_t n, dfsan_label s1_label,
+                             dfsan_label s2_label, dfsan_label n_label) {
+  if (!TS) return;
+  uintptr_t PC = reinterpret_cast<uintptr_t>(caller_pc);
+  uint64_t S1 = 0, S2 = 0;
+  n = std::min(n, fuzzer::InternalStrnlen(s1, n));
+  n = std::min(n, fuzzer::InternalStrnlen(s2, n));
+  // Simplification: handle only first 8 bytes.
+  memcpy(&S1, s1, std::min(n, sizeof(S1)));
+  memcpy(&S2, s2, std::min(n, sizeof(S2)));
+  dfsan_label L1 = dfsan_read_label(s1, n);
+  dfsan_label L2 = dfsan_read_label(s2, n);
+  TS->DFSanCmpCallback(PC, n, fuzzer::ICMP_EQ, S1, S2, L1, L2);
+}
+
+void __sanitizer_weak_hook_memcmp(void *caller_pc, const void *s1,
+                                  const void *s2, size_t n) {
+  if (!TS) return;
+  uintptr_t PC = reinterpret_cast<uintptr_t>(caller_pc);
+  uint64_t S1 = 0, S2 = 0;
+  // Simplification: handle only first 8 bytes.
+  memcpy(&S1, s1, std::min(n, sizeof(S1)));
+  memcpy(&S2, s2, std::min(n, sizeof(S2)));
+  TS->TraceCmpCallback(PC, n, fuzzer::ICMP_EQ, S1, S2);
+}
+
+void __sanitizer_weak_hook_strncmp(void *caller_pc, const char *s1,
+                                   const char *s2, size_t n) {
+  if (!TS) return;
+  uintptr_t PC = reinterpret_cast<uintptr_t>(caller_pc);
+  uint64_t S1 = 0, S2 = 0;
+  size_t Len1 = fuzzer::InternalStrnlen(s1, n);
+  size_t Len2 = fuzzer::InternalStrnlen(s2, n);
+  n = std::min(n, Len1);
+  n = std::min(n, Len2);
+  if (n <= 1) return;  // Not interesting.
+  // Simplification: handle only first 8 bytes.
+  memcpy(&S1, s1, std::min(n, sizeof(S1)));
+  memcpy(&S2, s2, std::min(n, sizeof(S2)));
+  TS->TraceCmpCallback(PC, n, fuzzer::ICMP_EQ, S1, S2);
+}
+
+void __sanitizer_cov_trace_cmp(uint64_t SizeAndType, uint64_t Arg1,
+                               uint64_t Arg2) {
+  if (!TS) return;
+  uintptr_t PC = reinterpret_cast<uintptr_t>(__builtin_return_address(0));
+  uint64_t CmpSize = (SizeAndType >> 32) / 8;
+  uint64_t Type = (SizeAndType << 32) >> 32;
+  TS->TraceCmpCallback(PC, CmpSize, Type, Arg1, Arg2);
+}
+
+void __sanitizer_cov_trace_switch(uint64_t Val, uint64_t *Cases) {
+  if (!TS) return;
+  uintptr_t PC = reinterpret_cast<uintptr_t>(__builtin_return_address(0));
+  TS->TraceSwitchCallback(PC, Cases[1], Val, Cases[0], Cases + 2);
+}
+
+}  // extern "C"
diff --git a/lib/Fuzzer/FuzzerUtil.cpp b/lib/Fuzzer/FuzzerUtil.cpp
index 3635f39a10d..e381c040632 100644
--- a/lib/Fuzzer/FuzzerUtil.cpp
+++ b/lib/Fuzzer/FuzzerUtil.cpp
@@ -10,39 +10,39 @@
 //===----------------------------------------------------------------------===//
 
 #include "FuzzerInternal.h"
-#include <iostream>
+#include <sstream>
+#include <iomanip>
 #include <sys/time.h>
 #include <cassert>
 #include <cstring>
 #include <signal.h>
+#include <unistd.h>
 
 namespace fuzzer {
 
 void Print(const Unit &v, const char *PrintAfter) {
   for (auto x : v)
-    std::cerr << "0x" << std::hex << (unsigned) x << std::dec << ",";
-  std::cerr << PrintAfter;
+    Printf("0x%x,", (unsigned) x);
+  Printf("%s", PrintAfter);
 }
 
 void PrintASCII(const Unit &U, const char *PrintAfter) {
   for (auto X : U) {
     if (isprint(X))
-      std::cerr << X;
+      Printf("%c", X);
     else
-      std::cerr << "\\x" << std::hex << (int)(unsigned)X << std::dec;
+      Printf("\\x%x", (unsigned)X);
   }
-  std::cerr << PrintAfter;
+  Printf("%s", PrintAfter);
 }
 
-std::string Hash(const Unit &in) {
-  size_t h1 = 0, h2 = 0;
-  for (auto x : in) {
-    h1 += x;
-    h1 *= 5;
-    h2 += x;
-    h2 *= 7;
-  }
-  return std::to_string(h1) + std::to_string(h2);
+std::string Hash(const Unit &U) {
+  uint8_t Hash[kSHA1NumBytes];
+  ComputeSHA1(U.data(), U.size(), Hash);
+  std::stringstream SS;
+  for (int i = 0; i < kSHA1NumBytes; i++)
+    SS << std::hex << std::setfill('0') << std::setw(2) << (unsigned)Hash[i];
+  return SS.str();
 }
 
 static void AlarmHandler(int, siginfo_t *, void *) {
@@ -51,7 +51,7 @@ static void AlarmHandler(int, siginfo_t *, void *) {
 
 void SetTimer(int Seconds) {
   struct itimerval T {{Seconds, 0}, {Seconds, 0}};
-  std::cerr << "SetTimer " << Seconds << "\n";
+  Printf("SetTimer %d\n", Seconds);
   int Res = setitimer(ITIMER_REAL, &T, nullptr);
   assert(Res == 0);
   struct sigaction sigact;
@@ -61,4 +61,16 @@ void SetTimer(int Seconds) {
   assert(Res == 0);
 }
 
+int NumberOfCpuCores() {
+  FILE *F = popen("nproc", "r");
+  int N = 0;
+  fscanf(F, "%d", &N);
+  fclose(F);
+  return N;
+}
+
+void ExecuteCommand(const std::string &Command) {
+  system(Command.c_str());
+}
+
 }  // namespace fuzzer
diff --git a/lib/Fuzzer/dfsan_fuzzer_abi.list b/lib/Fuzzer/dfsan_fuzzer_abi.list
deleted file mode 100644
index 7da752278f7..00000000000
--- a/lib/Fuzzer/dfsan_fuzzer_abi.list
+++ /dev/null
@@ -1,12 +0,0 @@
-# Replaces __sanitizer_cov_trace_cmp with __dfsw___sanitizer_cov_trace_cmp
-fun:__sanitizer_cov_trace_cmp=custom
-fun:__sanitizer_cov_trace_cmp=uninstrumented
-
-# Ignores coverage callbacks.
-fun:__sanitizer_cov=uninstrumented
-fun:__sanitizer_cov=discard
-fun:__sanitizer_cov_module_init=uninstrumented
-fun:__sanitizer_cov_module_init=discard
-
-# Don't add extra parameters to the Fuzzer callback.
-fun:TestOneInput=uninstrumented
diff --git a/lib/Fuzzer/pull_and_push_fuzz_corpus.sh b/lib/Fuzzer/pull_and_push_fuzz_corpus.sh
new file mode 100755
index 00000000000..05c322c6e5b
--- /dev/null
+++ b/lib/Fuzzer/pull_and_push_fuzz_corpus.sh
@@ -0,0 +1,17 @@
+#!/bin/bash
+# A simple script to synchronise a fuzz test corpus
+# with an external git repository.
+# Usage:
+#   pull_and_push_fuzz_corpus.sh DIR
+# It assumes that DIR is inside a git repo and push
+# can be done w/o typing a password.
+cd $1
+git add *
+git commit -m "fuzz test corpus"
+git pull --rebase --no-edit
+for((attempt=0; attempt<5; attempt++)); do
+  echo GIT PUSH $1 ATTEMPT $attempt
+  if $(git push); then break; fi
+  git pull --rebase --no-edit
+done
+
diff --git a/lib/Fuzzer/test/CMakeLists.txt b/lib/Fuzzer/test/CMakeLists.txt
index fb3bf20ef80..b8c1f3228df 100644
--- a/lib/Fuzzer/test/CMakeLists.txt
+++ b/lib/Fuzzer/test/CMakeLists.txt
@@ -2,7 +2,14 @@
 # basic blocks and we'll fail to discover the targets.
 # Also enable the coverage instrumentation back (it is disabled
 # for the Fuzzer lib)
-set(CMAKE_CXX_FLAGS_RELEASE "${LIBFUZZER_FLAGS_BASE} -O0 -fsanitize-coverage=4")
+set(CMAKE_CXX_FLAGS_RELEASE "${LIBFUZZER_FLAGS_BASE} -O0 -fsanitize-coverage=edge,indirect-calls")
+
+set(DFSanTests
+  MemcmpTest
+  SimpleCmpTest
+  StrncmpTest
+  SwitchTest
+  )
 
 set(Tests
   CounterTest
@@ -10,15 +17,20 @@ set(Tests
   FourIndependentBranchesTest
   FullCoverageSetTest
   InfiniteTest
+  MemcmpTest
   NullDerefTest
+  SimpleCmpTest
   SimpleTest
+  StrncmpTest
+  SwitchTest
   TimeoutTest
   )
 
-set(DFSanTests
-  DFSanSimpleCmpTest
+set(CustomMainTests
+  UserSuppliedFuzzerTest
   )
 
+
 set(TestBinaries)
 
 foreach(Test ${Tests})
@@ -31,6 +43,17 @@ foreach(Test ${Tests})
   set(TestBinaries ${TestBinaries} LLVMFuzzer-${Test})
 endforeach()
 
+foreach(Test ${CustomMainTests})
+  add_executable(LLVMFuzzer-${Test}
+    ${Test}.cpp
+    )
+  target_link_libraries(LLVMFuzzer-${Test}
+    LLVMFuzzerNoMain
+    )
+  set(TestBinaries ${TestBinaries} LLVMFuzzer-${Test})
+endforeach()
+
+
 configure_lit_site_cfg(
   ${CMAKE_CURRENT_SOURCE_DIR}/lit.site.cfg.in
   ${CMAKE_CURRENT_BINARY_DIR}/lit.site.cfg
@@ -46,7 +69,7 @@ include_directories(${LLVM_MAIN_SRC_DIR}/utils/unittest/googletest/include)
 
 add_executable(LLVMFuzzer-Unittest
   FuzzerUnittest.cpp
-  $<TARGET_OBJECTS:LLVMFuzzerNoMain>
+  $<TARGET_OBJECTS:LLVMFuzzerNoMainObjects>
   )
 
 target_link_libraries(LLVMFuzzer-Unittest
@@ -59,7 +82,7 @@ set(TestBinaries ${TestBinaries} LLVMFuzzer-Unittest)
 add_subdirectory(dfsan)
 
 foreach(Test ${DFSanTests})
-  set(TestBinaries ${TestBinaries} LLVMFuzzer-${Test})
+  set(TestBinaries ${TestBinaries} LLVMFuzzer-${Test}-DFSan)
 endforeach()
 
 
diff --git a/lib/Fuzzer/test/CounterTest.cpp b/lib/Fuzzer/test/CounterTest.cpp
index 332ccfe7b02..29ddb02ebae 100644
--- a/lib/Fuzzer/test/CounterTest.cpp
+++ b/lib/Fuzzer/test/CounterTest.cpp
@@ -2,7 +2,7 @@
 // executed many times.
 #include <iostream>
 
-extern "C" void TestOneInput(const uint8_t *Data, size_t Size) {
+extern "C" void LLVMFuzzerTestOneInput(const uint8_t *Data, size_t Size) {
   int Num = 0;
   for (size_t i = 0; i < Size; i++)
     if (Data[i] == 'A' + i)
diff --git a/lib/Fuzzer/test/CxxTokensTest.cpp b/lib/Fuzzer/test/CxxTokensTest.cpp
index 1addccb4bf9..77d08b3d105 100644
--- a/lib/Fuzzer/test/CxxTokensTest.cpp
+++ b/lib/Fuzzer/test/CxxTokensTest.cpp
@@ -6,11 +6,11 @@
 #include <iostream>
 
 static void Found() {
-  std::cout << "Found the target, exiting\n";
+  std::cout << "BINGO; Found the target, exiting\n";
   exit(1);
 }
 
-extern "C" void TestOneInput(const uint8_t *Data, size_t Size) {
+extern "C" void LLVMFuzzerTestOneInput(const uint8_t *Data, size_t Size) {
   // looking for "thread_local unsigned A;"
   if (Size < 24) return;
   if (0 == memcmp(&Data[0], "thread_local", 12))
diff --git a/lib/Fuzzer/test/FourIndependentBranchesTest.cpp b/lib/Fuzzer/test/FourIndependentBranchesTest.cpp
index 171668bf764..e0b7509b8d6 100644
--- a/lib/Fuzzer/test/FourIndependentBranchesTest.cpp
+++ b/lib/Fuzzer/test/FourIndependentBranchesTest.cpp
@@ -4,7 +4,7 @@
 #include <cstddef>
 #include <iostream>
 
-extern "C" void TestOneInput(const uint8_t *Data, size_t Size) {
+extern "C" void LLVMFuzzerTestOneInput(const uint8_t *Data, size_t Size) {
   int bits = 0;
   if (Size > 0 && Data[0] == 'F') bits |= 1;
   if (Size > 1 && Data[1] == 'U') bits |= 2;
diff --git a/lib/Fuzzer/test/FullCoverageSetTest.cpp b/lib/Fuzzer/test/FullCoverageSetTest.cpp
index d4f8c115abc..2c6ff98db00 100644
--- a/lib/Fuzzer/test/FullCoverageSetTest.cpp
+++ b/lib/Fuzzer/test/FullCoverageSetTest.cpp
@@ -4,7 +4,7 @@
 #include <cstddef>
 #include <iostream>
 
-extern "C" void TestOneInput(const uint8_t *Data, size_t Size) {
+extern "C" void LLVMFuzzerTestOneInput(const uint8_t *Data, size_t Size) {
   int bits = 0;
   if (Size > 0 && Data[0] == 'F') bits |= 1;
   if (Size > 1 && Data[1] == 'U') bits |= 2;
diff --git a/lib/Fuzzer/test/FuzzerUnittest.cpp b/lib/Fuzzer/test/FuzzerUnittest.cpp
index 368a0f2bf00..ed52a55a507 100644
--- a/lib/Fuzzer/test/FuzzerUnittest.cpp
+++ b/lib/Fuzzer/test/FuzzerUnittest.cpp
@@ -2,14 +2,15 @@
 #include "gtest/gtest.h"
 #include <set>
 
-// For now, have TestOneInput just to make it link.
-// Later we may want to make unittests that actually call TestOneInput.
-extern "C" void TestOneInput(const uint8_t *Data, size_t Size) {
+// For now, have LLVMFuzzerTestOneInput just to make it link.
+// Later we may want to make unittests that actually call LLVMFuzzerTestOneInput.
+extern "C" void LLVMFuzzerTestOneInput(const uint8_t *Data, size_t Size) {
   abort();
 }
 
 TEST(Fuzzer, CrossOver) {
   using namespace fuzzer;
+  FuzzerRandomLibc Rand(0);
   Unit A({0, 1, 2}), B({5, 6, 7});
   Unit C;
   Unit Expected[] = {
@@ -51,7 +52,10 @@ TEST(Fuzzer, CrossOver) {
   for (size_t Len = 1; Len < 8; Len++) {
     std::set<Unit> FoundUnits, ExpectedUnitsWitThisLength;
     for (int Iter = 0; Iter < 3000; Iter++) {
-      CrossOver(A, B, &C, Len);
+      C.resize(Len);
+      size_t NewSize = CrossOver(A.data(), A.size(), B.data(), B.size(),
+                                 C.data(), C.size(), Rand);
+      C.resize(NewSize);
       FoundUnits.insert(C);
     }
     for (const Unit &U : Expected)
@@ -60,3 +64,11 @@ TEST(Fuzzer, CrossOver) {
     EXPECT_EQ(ExpectedUnitsWitThisLength, FoundUnits);
   }
 }
+
+TEST(Fuzzer, Hash) {
+  uint8_t A[] = {'a', 'b', 'c'};
+  fuzzer::Unit U(A, A + sizeof(A));
+  EXPECT_EQ("a9993e364706816aba3e25717850c26c9cd0d89d", fuzzer::Hash(U));
+  U.push_back('d');
+  EXPECT_EQ("81fe8bfe87576c3ecb22426f8e57847382917acf", fuzzer::Hash(U));
+}
diff --git a/lib/Fuzzer/test/InfiniteTest.cpp b/lib/Fuzzer/test/InfiniteTest.cpp
index dcb3030413b..b6d174ffdc9 100644
--- a/lib/Fuzzer/test/InfiniteTest.cpp
+++ b/lib/Fuzzer/test/InfiniteTest.cpp
@@ -6,13 +6,17 @@
 
 static volatile int Sink;
 
-extern "C" void TestOneInput(const uint8_t *Data, size_t Size) {
+static volatile int One = 1;
+
+extern "C" void LLVMFuzzerTestOneInput(const uint8_t *Data, size_t Size) {
   if (Size > 0 && Data[0] == 'H') {
     Sink = 1;
     if (Size > 1 && Data[1] == 'i') {
       Sink = 2;
       if (Size > 2 && Data[2] == '!') {
         Sink = 2;
+        while (One)
+          ;
       }
     }
   }
diff --git a/lib/Fuzzer/test/MemcmpTest.cpp b/lib/Fuzzer/test/MemcmpTest.cpp
new file mode 100644
index 00000000000..2954b6c7d48
--- /dev/null
+++ b/lib/Fuzzer/test/MemcmpTest.cpp
@@ -0,0 +1,19 @@
+// Simple test for a fuzzer. The fuzzer must find a particular string.
+#include <cstring>
+#include <cstdint>
+#include <cstdio>
+#include <cstdlib>
+
+extern "C" void LLVMFuzzerTestOneInput(const uint8_t *Data, size_t Size) {
+  // TODO: check other sizes.
+  if (Size >= 8 && memcmp(Data, "01234567", 8) == 0) {
+    if (Size >= 12 && memcmp(Data + 8, "ABCD", 4) == 0) {
+      if (Size >= 14 && memcmp(Data + 12, "XY", 2) == 0) {
+        if (Size >= 16 && memcmp(Data + 14, "KLM", 3) == 0) {
+          fprintf(stderr, "BINGO\n");
+          exit(1);
+        }
+      }
+    }
+  }
+}
diff --git a/lib/Fuzzer/test/NullDerefTest.cpp b/lib/Fuzzer/test/NullDerefTest.cpp
index 8811e386f9d..0cff6617a31 100644
--- a/lib/Fuzzer/test/NullDerefTest.cpp
+++ b/lib/Fuzzer/test/NullDerefTest.cpp
@@ -7,7 +7,7 @@
 static volatile int Sink;
 static volatile int *Null = 0;
 
-extern "C" void TestOneInput(const uint8_t *Data, size_t Size) {
+extern "C" void LLVMFuzzerTestOneInput(const uint8_t *Data, size_t Size) {
   if (Size > 0 && Data[0] == 'H') {
     Sink = 1;
     if (Size > 1 && Data[1] == 'i') {
diff --git a/lib/Fuzzer/test/SimpleCmpTest.cpp b/lib/Fuzzer/test/SimpleCmpTest.cpp
new file mode 100644
index 00000000000..ee378146dae
--- /dev/null
+++ b/lib/Fuzzer/test/SimpleCmpTest.cpp
@@ -0,0 +1,30 @@
+// Simple test for a fuzzer. The fuzzer must find several narrow ranges.
+#include <cstdint>
+#include <cstdlib>
+#include <cstring>
+#include <cstdio>
+
+extern "C" void LLVMFuzzerTestOneInput(const uint8_t *Data, size_t Size) {
+  if (Size < 14) return;
+  uint64_t x = 0;
+  int64_t  y = 0;
+  int z = 0;
+  unsigned short a = 0;
+  memcpy(&x, Data, 8);
+  memcpy(&y, Data + Size - 8, 8);
+  memcpy(&z, Data + Size / 2, sizeof(z));
+  memcpy(&a, Data + Size / 2 + 4, sizeof(a));
+
+  if (x > 1234567890 &&
+      x < 1234567895 &&
+      y >= 987654321 &&
+      y <= 987654325 &&
+      z < -10000 &&
+      z >= -10005 &&
+      z != -10003 &&
+      a == 4242) {
+    fprintf(stderr, "BINGO; Found the target: size %zd (%zd, %zd, %d, %d), exiting.\n",
+            Size, x, y, z, a);
+    exit(1);
+  }
+}
diff --git a/lib/Fuzzer/test/SimpleTest.cpp b/lib/Fuzzer/test/SimpleTest.cpp
index adb90cebe86..a891635a7f1 100644
--- a/lib/Fuzzer/test/SimpleTest.cpp
+++ b/lib/Fuzzer/test/SimpleTest.cpp
@@ -6,13 +6,13 @@
 
 static volatile int Sink;
 
-extern "C" void TestOneInput(const uint8_t *Data, size_t Size) {
+extern "C" void LLVMFuzzerTestOneInput(const uint8_t *Data, size_t Size) {
   if (Size > 0 && Data[0] == 'H') {
     Sink = 1;
     if (Size > 1 && Data[1] == 'i') {
       Sink = 2;
       if (Size > 2 && Data[2] == '!') {
-        std::cout << "Found the target, exiting\n";
+        std::cout << "BINGO; Found the target, exiting\n";
         exit(0);
       }
     }
diff --git a/lib/Fuzzer/test/StrncmpTest.cpp b/lib/Fuzzer/test/StrncmpTest.cpp
new file mode 100644
index 00000000000..187a2fd66ba
--- /dev/null
+++ b/lib/Fuzzer/test/StrncmpTest.cpp
@@ -0,0 +1,24 @@
+// Simple test for a fuzzer. The fuzzer must find a particular string.
+#include <cstring>
+#include <cstdint>
+#include <cstdio>
+#include <cstdlib>
+
+static volatile int sink;
+
+extern "C" void LLVMFuzzerTestOneInput(const uint8_t *Data, size_t Size) {
+  // TODO: check other sizes.
+  char *S = (char*)Data;
+  if (Size >= 8 && strncmp(S, "123", 8))
+    sink = 1;
+  if (Size >= 8 && strncmp(S, "01234567", 8) == 0) {
+    if (Size >= 12 && strncmp(S + 8, "ABCD", 4) == 0) {
+      if (Size >= 14 && strncmp(S + 12, "XY", 2) == 0) {
+        if (Size >= 16 && strncmp(S + 14, "KLM", 3) == 0) {
+          fprintf(stderr, "BINGO\n");
+          exit(1);
+        }
+      }
+    }
+  }
+}
diff --git a/lib/Fuzzer/test/SwitchTest.cpp b/lib/Fuzzer/test/SwitchTest.cpp
new file mode 100644
index 00000000000..0f2319b2cc9
--- /dev/null
+++ b/lib/Fuzzer/test/SwitchTest.cpp
@@ -0,0 +1,35 @@
+// Simple test for a fuzzer. The fuzzer must find the interesting switch value.
+#include <cstdint>
+#include <cstdlib>
+#include <cstdio>
+#include <cstring>
+#include <cstddef>
+
+static volatile int Sink;
+
+template<class T>
+bool Switch(const uint8_t *Data, size_t Size) {
+  T X;
+  if (Size < sizeof(X)) return false;
+  memcpy(&X, Data, sizeof(X));
+  switch (X) {
+    case 1: Sink = __LINE__; break;
+    case 101: Sink = __LINE__; break;
+    case 1001: Sink = __LINE__; break;
+    case 10001: Sink = __LINE__; break;
+    case 100001: Sink = __LINE__; break;
+    case 1000001: Sink = __LINE__; break;
+    case 10000001: Sink = __LINE__; break;
+    case 100000001: return true;
+  }
+  return false;
+}
+
+extern "C" void LLVMFuzzerTestOneInput(const uint8_t *Data, size_t Size) {
+  if (Switch<int>(Data, Size) && Size >= 12 &&
+      Switch<uint64_t>(Data + 4, Size - 4)) {
+    fprintf(stderr, "BINGO; Found the target, exiting\n");
+    exit(1);
+  }
+}
+
diff --git a/lib/Fuzzer/test/TimeoutTest.cpp b/lib/Fuzzer/test/TimeoutTest.cpp
index 23683ce866c..d541c058b64 100644
--- a/lib/Fuzzer/test/TimeoutTest.cpp
+++ b/lib/Fuzzer/test/TimeoutTest.cpp
@@ -6,7 +6,7 @@
 
 static volatile int Sink;
 
-extern "C" void TestOneInput(const uint8_t *Data, size_t Size) {
+extern "C" void LLVMFuzzerTestOneInput(const uint8_t *Data, size_t Size) {
   if (Size > 0 && Data[0] == 'H') {
     Sink = 1;
     if (Size > 1 && Data[1] == 'i') {
diff --git a/lib/Fuzzer/test/UserSuppliedFuzzerTest.cpp b/lib/Fuzzer/test/UserSuppliedFuzzerTest.cpp
new file mode 100644
index 00000000000..8ebe1575ad0
--- /dev/null
+++ b/lib/Fuzzer/test/UserSuppliedFuzzerTest.cpp
@@ -0,0 +1,50 @@
+// Simple test for a fuzzer.
+// The fuzzer must find the string "Hi!" preceded by a magic value.
+// Uses UserSuppliedFuzzer which ensures that the magic is present.
+#include <cstdint>
+#include <cassert>
+#include <cstdlib>
+#include <cstddef>
+#include <cstring>
+#include <iostream>
+
+#include "FuzzerInterface.h"
+
+static const uint64_t kMagic = 8860221463604ULL;
+
+class MyFuzzer : public fuzzer::UserSuppliedFuzzer {
+ public:
+  MyFuzzer(fuzzer::FuzzerRandomBase *Rand)
+      : fuzzer::UserSuppliedFuzzer(Rand) {}
+  void TargetFunction(const uint8_t *Data, size_t Size) {
+    if (Size <= 10) return;
+    if (memcmp(Data, &kMagic, sizeof(kMagic))) return;
+    // It's hard to get here w/o advanced fuzzing techniques (e.g. cmp tracing).
+    // So, we simply 'fix' the data in the custom mutator.
+    if (Data[8] == 'H') {
+      if (Data[9] == 'i') {
+        if (Data[10] == '!') {
+          std::cout << "BINGO; Found the target, exiting\n";
+          exit(1);
+        }
+      }
+    }
+  }
+  // Custom mutator.
+  virtual size_t Mutate(uint8_t *Data, size_t Size, size_t MaxSize) {
+    assert(MaxSize > sizeof(kMagic));
+    if (Size < sizeof(kMagic))
+      Size = sizeof(kMagic);
+    // "Fix" the data, then mutate.
+    memcpy(Data, &kMagic, std::min(MaxSize, sizeof(kMagic)));
+    return BasicMutate(Data + sizeof(kMagic), Size - sizeof(kMagic),
+                       MaxSize - sizeof(kMagic));
+  }
+  // No need to redefine CrossOver() here.
+};
+
+int main(int argc, char **argv) {
+  fuzzer::FuzzerRandomLibc Rand(0);
+  MyFuzzer F(&Rand);
+  fuzzer::FuzzerDriver(argc, argv, F);
+}
diff --git a/lib/Fuzzer/test/dfsan/CMakeLists.txt b/lib/Fuzzer/test/dfsan/CMakeLists.txt
index b5b874fb1ce..2b49831fcdb 100644
--- a/lib/Fuzzer/test/dfsan/CMakeLists.txt
+++ b/lib/Fuzzer/test/dfsan/CMakeLists.txt
@@ -1,16 +1,13 @@
 # These tests depend on both coverage and dfsan instrumentation.
 
-set(DFSAN_FUZZER_ABI_LIST "${CMAKE_CURRENT_SOURCE_DIR}/../../dfsan_fuzzer_abi.list")
-
 set(CMAKE_CXX_FLAGS_RELEASE
-  "${LIBFUZZER_FLAGS_BASE} -O0 -fno-sanitize=all -fsanitize=dataflow -mllvm -sanitizer-coverage-experimental-trace-compares=1 -fsanitize-blacklist=${DFSAN_FUZZER_ABI_LIST}")
+  "${LIBFUZZER_FLAGS_BASE} -O0 -fno-sanitize=all -fsanitize=dataflow")
 
 foreach(Test ${DFSanTests})
-  set_source_files_properties(${Test}.cpp PROPERTIES OBJECT_DEPENDS ${DFSAN_FUZZER_ABI_LIST})
-  add_executable(LLVMFuzzer-${Test}
-    ${Test}.cpp
+  add_executable(LLVMFuzzer-${Test}-DFSan
+    ../${Test}.cpp
     )
-  target_link_libraries(LLVMFuzzer-${Test}
+  target_link_libraries(LLVMFuzzer-${Test}-DFSan
     LLVMFuzzer
     )
 endforeach()
diff --git a/lib/Fuzzer/test/dfsan/DFSanSimpleCmpTest.cpp b/lib/Fuzzer/test/dfsan/DFSanSimpleCmpTest.cpp
deleted file mode 100644
index 11620929a42..00000000000
--- a/lib/Fuzzer/test/dfsan/DFSanSimpleCmpTest.cpp
+++ /dev/null
@@ -1,30 +0,0 @@
-// Simple test for a fuzzer. The fuzzer must find several narrow ranges.
-#include <cstdint>
-#include <cstdlib>
-#include <cstring>
-#include <cstdio>
-
-extern "C" void TestOneInput(const uint8_t *Data, size_t Size) {
-  if (Size < 14) return;
-  uint64_t x = 0;
-  int64_t  y = 0;
-  int z = 0;
-  unsigned short a = 0;
-  memcpy(&x, Data, 8);
-  memcpy(&y, Data + Size - 8, 8);
-  memcpy(&z, Data + Size / 2, sizeof(z));
-  memcpy(&a, Data + Size / 2 + 4, sizeof(a));
-
-  if (x > 1234567890 &&
-      x < 1234567895 &&
-      y >= 987654321 &&
-      y <= 987654325 &&
-      z < -10000 &&
-      z >= -10005 &&
-      z != -10003 &&
-      a == 4242) {
-    fprintf(stderr, "Found the target: size %zd (%zd, %zd, %d, %d), exiting.\n",
-            Size, x, y, z, a);
-    exit(1);
-  }
-}
diff --git a/lib/Fuzzer/test/fuzzer-dfsan.test b/lib/Fuzzer/test/fuzzer-dfsan.test
new file mode 100644
index 00000000000..c9d00f1d950
--- /dev/null
+++ b/lib/Fuzzer/test/fuzzer-dfsan.test
@@ -0,0 +1,15 @@
+CHECK: BINGO
+CHECK_DFSanCmpCallback: DFSanCmpCallback: PC
+CHECK_DFSanSwitchCallback: DFSanSwitchCallback: PC
+
+RUN: not LLVMFuzzer-SimpleCmpTest-DFSan -use_traces=1 -seed=1 -runs=1000000 -timeout=5 2>&1 | FileCheck %s
+RUN: LLVMFuzzer-SimpleCmpTest-DFSan -use_traces=1 -seed=1 -runs=100 -timeout=5 -verbosity=3 2>&1 | FileCheck %s  -check-prefix=CHECK_DFSanCmpCallback
+
+RUN: not LLVMFuzzer-MemcmpTest-DFSan -use_traces=1 -seed=1 -runs=1000 -timeout=5 2>&1 | FileCheck %s
+RUN: LLVMFuzzer-MemcmpTest-DFSan -use_traces=1 -seed=1 -runs=2 -timeout=5 -verbosity=3 2>&1 | FileCheck %s  -check-prefix=CHECK_DFSanCmpCallback
+
+RUN: not LLVMFuzzer-StrncmpTest-DFSan -use_traces=1 -seed=1 -runs=10000 -timeout=5 2>&1 | FileCheck %s
+RUN: LLVMFuzzer-StrncmpTest-DFSan -use_traces=1 -seed=1 -runs=2 -timeout=5 -verbosity=3 2>&1 | FileCheck %s  -check-prefix=CHECK_DFSanCmpCallback
+
+RUN: not LLVMFuzzer-SwitchTest-DFSan -use_traces=1 -seed=1 -runs=10000 -timeout=5              2>&1 | FileCheck %s
+RUN:     LLVMFuzzer-SwitchTest-DFSan -use_traces=1 -seed=1 -runs=2     -timeout=5 -verbosity=3 2>&1 | FileCheck %s  -check-prefix=CHECK_DFSanSwitchCallback
diff --git a/lib/Fuzzer/test/fuzzer.test b/lib/Fuzzer/test/fuzzer.test
index 2a0e95fbea2..63cb9573efa 100644
--- a/lib/Fuzzer/test/fuzzer.test
+++ b/lib/Fuzzer/test/fuzzer.test
@@ -1,28 +1,36 @@
-RUN: ./LLVMFuzzer-SimpleTest 2>&1 | FileCheck %s --check-prefix=SimpleTest
-SimpleTest: Found the target, exiting
+CHECK: BINGO
 
-RUN: not ./LLVMFuzzer-InfiniteTest -timeout=2 2>&1 | FileCheck %s --check-prefix=InfiniteTest
+RUN: LLVMFuzzer-SimpleTest 2>&1 | FileCheck %s
+
+RUN: not LLVMFuzzer-InfiniteTest -timeout=2 2>&1 | FileCheck %s --check-prefix=InfiniteTest
 InfiniteTest: ALARM: working on the last Unit for
-InfiniteTest-NOT: CRASHED; file written to timeout
+InfiniteTest: Test unit written to timeout-
 
-RUN: not ./LLVMFuzzer-TimeoutTest -timeout=5 2>&1 | FileCheck %s --check-prefix=TimeoutTest
+RUN: not LLVMFuzzer-TimeoutTest -timeout=5 2>&1 | FileCheck %s --check-prefix=TimeoutTest
 TimeoutTest: ALARM: working on the last Unit for
-TimeoutTest: CRASHED; file written to timeout
+TimeoutTest: Test unit written to timeout-
+
+RUN: not LLVMFuzzer-NullDerefTest 2>&1 | FileCheck %s --check-prefix=NullDerefTest
+NullDerefTest: Test unit written to crash-
+
+RUN: not LLVMFuzzer-FullCoverageSetTest -timeout=15 -seed=1 -mutate_depth=2 -use_full_coverage_set=1 2>&1 | FileCheck %s
+
+RUN: not LLVMFuzzer-FourIndependentBranchesTest -timeout=15 -seed=1 -use_full_coverage_set=1 2>&1 | FileCheck %s
+
+RUN: not LLVMFuzzer-CounterTest -use_counters=1 -max_len=6 -seed=1 -timeout=15 2>&1 | FileCheck %s
 
-RUN: not ./LLVMFuzzer-NullDerefTest 2>&1 | FileCheck %s --check-prefix=NullDerefTest
-NullDerefTest: CRASHED; file written to crash-
+RUN: not LLVMFuzzer-SimpleCmpTest -use_traces=1 -seed=1 -runs=1000000 -timeout=5 2>&1 | FileCheck %s
 
-RUN: not ./LLVMFuzzer-FullCoverageSetTest -timeout=15 -seed=1 -mutate_depth=2 -use_full_coverage_set=1 2>&1 | FileCheck %s --check-prefix=FullCoverageSetTest
-FullCoverageSetTest: BINGO
+RUN: not LLVMFuzzer-CxxTokensTest -seed=1 -timeout=15 -tokens=%S/../cxx_fuzzer_tokens.txt 2>&1 | FileCheck %s
 
-RUN: not ./LLVMFuzzer-FourIndependentBranchesTest -timeout=15 -seed=1 -use_coverage_pairs=1 2>&1 | FileCheck %s --check-prefix=FourIndependentBranchesTest
-FourIndependentBranchesTest: BINGO
+RUN: not LLVMFuzzer-UserSuppliedFuzzerTest -seed=1 -timeout=15 2>&1 | FileCheck %s
 
-RUN: not ./LLVMFuzzer-CounterTest -use_counters=1 -max_len=6 -seed=1 -timeout=15 2>&1 | FileCheck %s --check-prefix=CounterTest
-CounterTest: BINGO
+RUN: not LLVMFuzzer-MemcmpTest -use_traces=1 -seed=1 -runs=10000    2>&1 | FileCheck %s
+RUN:     LLVMFuzzer-MemcmpTest               -seed=1 -runs=1000000  2>&1 | FileCheck %s --check-prefix=Done1000000
+Done1000000: Done 1000000 runs in
 
-RUN: not ./LLVMFuzzer-DFSanSimpleCmpTest -seed=1 -timeout=15 2>&1 | FileCheck %s --check-prefix=DFSanSimpleCmpTest
-DFSanSimpleCmpTest: Found the target:
+RUN: not LLVMFuzzer-StrncmpTest -use_traces=1 -seed=1 -runs=10000    2>&1 | FileCheck %s
+RUN:     LLVMFuzzer-StrncmpTest               -seed=1 -runs=1000000  2>&1 | FileCheck %s --check-prefix=Done1000000
 
-RUN: not ./LLVMFuzzer-CxxTokensTest -seed=1 -timeout=15 -tokens=%S/../cxx_fuzzer_tokens.txt 2>&1 | FileCheck %s --check-prefix=CxxTokensTest
-CxxTokensTest: Found the target, exiting
+RUN: not LLVMFuzzer-SwitchTest -use_traces=1 -seed=1 -runs=100000    2>&1 | FileCheck %s
+RUN:     LLVMFuzzer-SwitchTest               -seed=1 -runs=1000000  2>&1 | FileCheck %s --check-prefix=Done1000000
diff --git a/lib/Fuzzer/test/lit.cfg b/lib/Fuzzer/test/lit.cfg
index 834a16aefe7..2140a97668b 100644
--- a/lib/Fuzzer/test/lit.cfg
+++ b/lib/Fuzzer/test/lit.cfg
@@ -5,10 +5,11 @@ config.test_format = lit.formats.ShTest(True)
 config.suffixes = ['.test']
 config.test_source_root = os.path.dirname(__file__)
 
-# Tweak PATH to include llvm tools dir.
+# Tweak PATH to include llvm tools dir and current exec dir.
 llvm_tools_dir = getattr(config, 'llvm_tools_dir', None)
 if (not llvm_tools_dir) or (not os.path.exists(llvm_tools_dir)):
   lit_config.fatal("Invalid llvm_tools_dir config attribute: %r" % llvm_tools_dir)
-path = os.path.pathsep.join((llvm_tools_dir, config.environment['PATH']))
+path = os.path.pathsep.join((llvm_tools_dir, config.test_exec_root,
+                             config.environment['PATH']))
 config.environment['PATH'] = path
 
diff --git a/lib/IR/AsmWriter.cpp b/lib/IR/AsmWriter.cpp
index 48737b5796e..e3905de6597 100644
--- a/lib/IR/AsmWriter.cpp
+++ b/lib/IR/AsmWriter.cpp
@@ -30,7 +30,9 @@
 #include "llvm/IR/IntrinsicInst.h"
 #include "llvm/IR/LLVMContext.h"
 #include "llvm/IR/Module.h"
+#include "llvm/IR/ModuleSlotTracker.h"
 #include "llvm/IR/Operator.h"
+#include "llvm/IR/Statepoint.h"
 #include "llvm/IR/TypeFinder.h"
 #include "llvm/IR/UseListOrder.h"
 #include "llvm/IR/ValueSymbolTable.h"
@@ -108,6 +110,10 @@ static OrderMap orderModule(const Module *M) {
       if (!isa<GlobalValue>(F.getPrologueData()))
         orderValue(F.getPrologueData(), OM);
 
+    if (F.hasPersonalityFn())
+      if (!isa<GlobalValue>(F.getPersonalityFn()))
+        orderValue(F.getPersonalityFn(), OM);
+
     orderValue(&F, OM);
 
     if (F.isDeclaration())
@@ -226,8 +232,7 @@ static UseListOrderStack predictUseListOrder(const Module *M) {
   // We want to visit the functions backward now so we can list function-local
   // constants in the last Function they're used in.  Module-level constants
   // have already been visited above.
-  for (auto I = M->rbegin(), E = M->rend(); I != E; ++I) {
-    const Function &F = *I;
+  for (const Function &F : make_range(M->rbegin(), M->rend())) {
     if (F.isDeclaration())
       continue;
     for (const BasicBlock &BB : F)
@@ -337,18 +342,8 @@ enum PrefixType {
   NoPrefix
 };
 
-/// PrintLLVMName - Turn the specified name into an 'LLVM name', which is either
-/// prefixed with % (if the string only contains simple characters) or is
-/// surrounded with ""'s (if it has special chars in it).  Print it out.
-static void PrintLLVMName(raw_ostream &OS, StringRef Name, PrefixType Prefix) {
+void llvm::printLLVMNameWithoutPrefix(raw_ostream &OS, StringRef Name) {
   assert(!Name.empty() && "Cannot get empty name!");
-  switch (Prefix) {
-  case NoPrefix: break;
-  case GlobalPrefix: OS << '@'; break;
-  case ComdatPrefix: OS << '$'; break;
-  case LabelPrefix:  break;
-  case LocalPrefix:  OS << '%'; break;
-  }
 
   // Scan the name to see if it needs quotes first.
   bool NeedsQuotes = isdigit(static_cast<unsigned char>(Name[0]));
@@ -380,9 +375,31 @@ static void PrintLLVMName(raw_ostream &OS, StringRef Name, PrefixType Prefix) {
   OS << '"';
 }
 
-/// PrintLLVMName - Turn the specified name into an 'LLVM name', which is either
-/// prefixed with % (if the string only contains simple characters) or is
-/// surrounded with ""'s (if it has special chars in it).  Print it out.
+/// Turn the specified name into an 'LLVM name', which is either prefixed with %
+/// (if the string only contains simple characters) or is surrounded with ""'s
+/// (if it has special chars in it). Print it out.
+static void PrintLLVMName(raw_ostream &OS, StringRef Name, PrefixType Prefix) {
+  switch (Prefix) {
+  case NoPrefix:
+    break;
+  case GlobalPrefix:
+    OS << '@';
+    break;
+  case ComdatPrefix:
+    OS << '$';
+    break;
+  case LabelPrefix:
+    break;
+  case LocalPrefix:
+    OS << '%';
+    break;
+  }
+  printLLVMNameWithoutPrefix(OS, Name);
+}
+
+/// Turn the specified name into an 'LLVM name', which is either prefixed with %
+/// (if the string only contains simple characters) or is surrounded with ""'s
+/// (if it has special chars in it). Print it out.
 static void PrintLLVMName(raw_ostream &OS, const Value *V) {
   PrintLLVMName(OS, V->getName(),
                 isa<GlobalValue>(V) ? GlobalPrefix : LocalPrefix);
@@ -539,7 +556,7 @@ void TypePrinting::printStructBody(StructType *STy, raw_ostream &OS) {
     OS << '>';
 }
 
-namespace {
+namespace llvm {
 //===----------------------------------------------------------------------===//
 // SlotTracker Class: Enumerate slot numbers for unnamed values
 //===----------------------------------------------------------------------===//
@@ -658,7 +675,37 @@ class SlotTracker {
   SlotTracker(const SlotTracker &) = delete;
   void operator=(const SlotTracker &) = delete;
 };
-} // namespace
+} // namespace llvm
+
+ModuleSlotTracker::ModuleSlotTracker(SlotTracker &Machine, const Module *M,
+                                     const Function *F)
+    : M(M), F(F), Machine(&Machine) {}
+
+ModuleSlotTracker::ModuleSlotTracker(const Module *M,
+                                     bool ShouldInitializeAllMetadata)
+    : MachineStorage(M ? new SlotTracker(M, ShouldInitializeAllMetadata)
+                       : nullptr),
+      M(M), Machine(MachineStorage.get()) {}
+
+ModuleSlotTracker::~ModuleSlotTracker() {}
+
+void ModuleSlotTracker::incorporateFunction(const Function &F) {
+  if (!Machine)
+    return;
+
+  // Nothing to do if this is the right function already.
+  if (this->F == &F)
+    return;
+  if (this->F)
+    Machine->purgeFunction();
+  Machine->incorporateFunction(&F);
+  this->F = &F;
+}
+
+int ModuleSlotTracker::getLocalSlot(const Value *V) {
+  assert(F && "No function incorporated");
+  return Machine->getLocalSlot(V);
+}
 
 static SlotTracker *createSlotTracker(const Module *M) {
   return new SlotTracker(M);
@@ -724,33 +771,33 @@ void SlotTracker::processModule() {
   ST_DEBUG("begin processModule!\n");
 
   // Add all of the unnamed global variables to the value table.
-  for (Module::const_global_iterator I = TheModule->global_begin(),
-         E = TheModule->global_end(); I != E; ++I) {
-    if (!I->hasName())
-      CreateModuleSlot(I);
+  for (const GlobalVariable &Var : TheModule->globals()) {
+    if (!Var.hasName())
+      CreateModuleSlot(&Var);
+  }
+
+  for (const GlobalAlias &A : TheModule->aliases()) {
+    if (!A.hasName())
+      CreateModuleSlot(&A);
   }
 
   // Add metadata used by named metadata.
-  for (Module::const_named_metadata_iterator
-         I = TheModule->named_metadata_begin(),
-         E = TheModule->named_metadata_end(); I != E; ++I) {
-    const NamedMDNode *NMD = I;
-    for (unsigned i = 0, e = NMD->getNumOperands(); i != e; ++i)
-      CreateMetadataSlot(NMD->getOperand(i));
+  for (const NamedMDNode &NMD : TheModule->named_metadata()) {
+    for (unsigned i = 0, e = NMD.getNumOperands(); i != e; ++i)
+      CreateMetadataSlot(NMD.getOperand(i));
   }
 
-  for (Module::const_iterator I = TheModule->begin(), E = TheModule->end();
-       I != E; ++I) {
-    if (!I->hasName())
+  for (const Function &F : *TheModule) {
+    if (!F.hasName())
       // Add all the unnamed functions to the table.
-      CreateModuleSlot(I);
+      CreateModuleSlot(&F);
 
     if (ShouldInitializeAllMetadata)
-      processFunctionMetadata(*I);
+      processFunctionMetadata(F);
 
     // Add all the function attributes to the table.
     // FIXME: Add attributes of other objects?
-    AttributeSet FnAttrs = I->getAttributes().getFnAttributes();
+    AttributeSet FnAttrs = F.getAttributes().getFnAttributes();
     if (FnAttrs.hasAttributes(AttributeSet::FunctionIndex))
       CreateAttributeSetSlot(FnAttrs);
   }
@@ -776,12 +823,12 @@ void SlotTracker::processFunction() {
     if (!BB.hasName())
       CreateFunctionSlot(&BB);
 
+    processFunctionMetadata(*TheFunction);
+
     for (auto &I : BB) {
       if (!I.getType()->isVoidTy() && !I.hasName())
         CreateFunctionSlot(&I);
 
-      processInstructionMetadata(I);
-
       // We allow direct calls to any llvm.foo function here, because the
       // target may not be linked into the optimizer.
       if (const CallInst *CI = dyn_cast<CallInst>(&I)) {
@@ -804,9 +851,15 @@ void SlotTracker::processFunction() {
 }
 
 void SlotTracker::processFunctionMetadata(const Function &F) {
-  for (auto &BB : F)
+  SmallVector<std::pair<unsigned, MDNode *>, 4> MDs;
+  for (auto &BB : F) {
+    F.getAllMetadata(MDs);
+    for (auto &MD : MDs)
+      CreateMetadataSlot(MD.second);
+
     for (auto &I : BB)
       processInstructionMetadata(I);
+  }
 }
 
 void SlotTracker::processInstructionMetadata(const Instruction &I) {
@@ -1287,10 +1340,7 @@ static void WriteConstantInternal(raw_ostream &Out, const Constant *CV,
     Out << " (";
 
     if (const GEPOperator *GEP = dyn_cast<GEPOperator>(CE)) {
-      TypePrinter.print(
-          cast<PointerType>(GEP->getPointerOperandType()->getScalarType())
-              ->getElementType(),
-          Out);
+      TypePrinter.print(GEP->getSourceElementType(), Out);
       Out << ", ";
     }
 
@@ -1369,7 +1419,7 @@ struct MDFieldPrinter {
                  SlotTracker *Machine, const Module *Context)
       : Out(Out), TypePrinter(TypePrinter), Machine(Machine), Context(Context) {
   }
-  void printTag(const DebugNode *N);
+  void printTag(const DINode *N);
   void printString(StringRef Name, StringRef Value,
                    bool ShouldSkipEmpty = true);
   void printMetadata(StringRef Name, const Metadata *MD,
@@ -1384,7 +1434,7 @@ struct MDFieldPrinter {
 };
 } // end namespace
 
-void MDFieldPrinter::printTag(const DebugNode *N) {
+void MDFieldPrinter::printTag(const DINode *N) {
   Out << FS << "tag: ";
   if (const char *Tag = dwarf::TagString(N->getTag()))
     Out << Tag;
@@ -1441,11 +1491,11 @@ void MDFieldPrinter::printDIFlags(StringRef Name, unsigned Flags) {
   Out << FS << Name << ": ";
 
   SmallVector<unsigned, 8> SplitFlags;
-  unsigned Extra = DebugNode::splitFlags(Flags, SplitFlags);
+  unsigned Extra = DINode::splitFlags(Flags, SplitFlags);
 
   FieldSeparator FlagsFS(" | ");
   for (unsigned F : SplitFlags) {
-    const char *StringF = DebugNode::getFlagString(F);
+    const char *StringF = DINode::getFlagString(F);
     assert(StringF && "Expected valid flag");
     Out << FlagsFS << StringF;
   }
@@ -1466,10 +1516,10 @@ void MDFieldPrinter::printDwarfEnum(StringRef Name, IntTy Value,
     Out << Value;
 }
 
-static void writeGenericDebugNode(raw_ostream &Out, const GenericDebugNode *N,
-                                  TypePrinting *TypePrinter,
-                                  SlotTracker *Machine, const Module *Context) {
-  Out << "!GenericDebugNode(";
+static void writeGenericDINode(raw_ostream &Out, const GenericDINode *N,
+                               TypePrinting *TypePrinter, SlotTracker *Machine,
+                               const Module *Context) {
+  Out << "!GenericDINode(";
   MDFieldPrinter Printer(Out, TypePrinter, Machine, Context);
   Printer.printTag(N);
   Printer.printString("header", N->getHeader());
@@ -1485,10 +1535,10 @@ static void writeGenericDebugNode(raw_ostream &Out, const GenericDebugNode *N,
   Out << ")";
 }
 
-static void writeMDLocation(raw_ostream &Out, const MDLocation *DL,
+static void writeDILocation(raw_ostream &Out, const DILocation *DL,
                             TypePrinting *TypePrinter, SlotTracker *Machine,
                             const Module *Context) {
-  Out << "!MDLocation(";
+  Out << "!DILocation(";
   MDFieldPrinter Printer(Out, TypePrinter, Machine, Context);
   // Always output the line, since 0 is a relevant and important value for it.
   Printer.printInt("line", DL->getLine(), /* ShouldSkipZero */ false);
@@ -1498,27 +1548,27 @@ static void writeMDLocation(raw_ostream &Out, const MDLocation *DL,
   Out << ")";
 }
 
-static void writeMDSubrange(raw_ostream &Out, const MDSubrange *N,
+static void writeDISubrange(raw_ostream &Out, const DISubrange *N,
                             TypePrinting *, SlotTracker *, const Module *) {
-  Out << "!MDSubrange(";
+  Out << "!DISubrange(";
   MDFieldPrinter Printer(Out);
   Printer.printInt("count", N->getCount(), /* ShouldSkipZero */ false);
   Printer.printInt("lowerBound", N->getLowerBound());
   Out << ")";
 }
 
-static void writeMDEnumerator(raw_ostream &Out, const MDEnumerator *N,
+static void writeDIEnumerator(raw_ostream &Out, const DIEnumerator *N,
                               TypePrinting *, SlotTracker *, const Module *) {
-  Out << "!MDEnumerator(";
+  Out << "!DIEnumerator(";
   MDFieldPrinter Printer(Out);
   Printer.printString("name", N->getName(), /* ShouldSkipEmpty */ false);
   Printer.printInt("value", N->getValue(), /* ShouldSkipZero */ false);
   Out << ")";
 }
 
-static void writeMDBasicType(raw_ostream &Out, const MDBasicType *N,
+static void writeDIBasicType(raw_ostream &Out, const DIBasicType *N,
                              TypePrinting *, SlotTracker *, const Module *) {
-  Out << "!MDBasicType(";
+  Out << "!DIBasicType(";
   MDFieldPrinter Printer(Out);
   if (N->getTag() != dwarf::DW_TAG_base_type)
     Printer.printTag(N);
@@ -1530,10 +1580,10 @@ static void writeMDBasicType(raw_ostream &Out, const MDBasicType *N,
   Out << ")";
 }
 
-static void writeMDDerivedType(raw_ostream &Out, const MDDerivedType *N,
+static void writeDIDerivedType(raw_ostream &Out, const DIDerivedType *N,
                                TypePrinting *TypePrinter, SlotTracker *Machine,
                                const Module *Context) {
-  Out << "!MDDerivedType(";
+  Out << "!DIDerivedType(";
   MDFieldPrinter Printer(Out, TypePrinter, Machine, Context);
   Printer.printTag(N);
   Printer.printString("name", N->getName());
@@ -1550,10 +1600,10 @@ static void writeMDDerivedType(raw_ostream &Out, const MDDerivedType *N,
   Out << ")";
 }
 
-static void writeMDCompositeType(raw_ostream &Out, const MDCompositeType *N,
+static void writeDICompositeType(raw_ostream &Out, const DICompositeType *N,
                                  TypePrinting *TypePrinter,
                                  SlotTracker *Machine, const Module *Context) {
-  Out << "!MDCompositeType(";
+  Out << "!DICompositeType(";
   MDFieldPrinter Printer(Out, TypePrinter, Machine, Context);
   Printer.printTag(N);
   Printer.printString("name", N->getName());
@@ -1574,10 +1624,10 @@ static void writeMDCompositeType(raw_ostream &Out, const MDCompositeType *N,
   Out << ")";
 }
 
-static void writeMDSubroutineType(raw_ostream &Out, const MDSubroutineType *N,
+static void writeDISubroutineType(raw_ostream &Out, const DISubroutineType *N,
                                   TypePrinting *TypePrinter,
                                   SlotTracker *Machine, const Module *Context) {
-  Out << "!MDSubroutineType(";
+  Out << "!DISubroutineType(";
   MDFieldPrinter Printer(Out, TypePrinter, Machine, Context);
   Printer.printDIFlags("flags", N->getFlags());
   Printer.printMetadata("types", N->getRawTypeArray(),
@@ -1585,9 +1635,9 @@ static void writeMDSubroutineType(raw_ostream &Out, const MDSubroutineType *N,
   Out << ")";
 }
 
-static void writeMDFile(raw_ostream &Out, const MDFile *N, TypePrinting *,
+static void writeDIFile(raw_ostream &Out, const DIFile *N, TypePrinting *,
                         SlotTracker *, const Module *) {
-  Out << "!MDFile(";
+  Out << "!DIFile(";
   MDFieldPrinter Printer(Out);
   Printer.printString("filename", N->getFilename(),
                       /* ShouldSkipEmpty */ false);
@@ -1596,10 +1646,10 @@ static void writeMDFile(raw_ostream &Out, const MDFile *N, TypePrinting *,
   Out << ")";
 }
 
-static void writeMDCompileUnit(raw_ostream &Out, const MDCompileUnit *N,
+static void writeDICompileUnit(raw_ostream &Out, const DICompileUnit *N,
                                TypePrinting *TypePrinter, SlotTracker *Machine,
                                const Module *Context) {
-  Out << "!MDCompileUnit(";
+  Out << "!DICompileUnit(";
   MDFieldPrinter Printer(Out, TypePrinter, Machine, Context);
   Printer.printDwarfEnum("language", N->getSourceLanguage(),
                          dwarf::LanguageString, /* ShouldSkipZero */ false);
@@ -1617,13 +1667,14 @@ static void writeMDCompileUnit(raw_ostream &Out, const MDCompileUnit *N,
   Printer.printMetadata("subprograms", N->getRawSubprograms());
   Printer.printMetadata("globals", N->getRawGlobalVariables());
   Printer.printMetadata("imports", N->getRawImportedEntities());
+  Printer.printInt("dwoId", N->getDWOId());
   Out << ")";
 }
 
-static void writeMDSubprogram(raw_ostream &Out, const MDSubprogram *N,
+static void writeDISubprogram(raw_ostream &Out, const DISubprogram *N,
                               TypePrinting *TypePrinter, SlotTracker *Machine,
                               const Module *Context) {
-  Out << "!MDSubprogram(";
+  Out << "!DISubprogram(";
   MDFieldPrinter Printer(Out, TypePrinter, Machine, Context);
   Printer.printString("name", N->getName());
   Printer.printString("linkageName", N->getLinkageName());
@@ -1647,10 +1698,10 @@ static void writeMDSubprogram(raw_ostream &Out, const MDSubprogram *N,
   Out << ")";
 }
 
-static void writeMDLexicalBlock(raw_ostream &Out, const MDLexicalBlock *N,
-                              TypePrinting *TypePrinter, SlotTracker *Machine,
-                              const Module *Context) {
-  Out << "!MDLexicalBlock(";
+static void writeDILexicalBlock(raw_ostream &Out, const DILexicalBlock *N,
+                                TypePrinting *TypePrinter, SlotTracker *Machine,
+                                const Module *Context) {
+  Out << "!DILexicalBlock(";
   MDFieldPrinter Printer(Out, TypePrinter, Machine, Context);
   Printer.printMetadata("scope", N->getRawScope(), /* ShouldSkipNull */ false);
   Printer.printMetadata("file", N->getRawFile());
@@ -1659,12 +1710,12 @@ static void writeMDLexicalBlock(raw_ostream &Out, const MDLexicalBlock *N,
   Out << ")";
 }
 
-static void writeMDLexicalBlockFile(raw_ostream &Out,
-                                    const MDLexicalBlockFile *N,
+static void writeDILexicalBlockFile(raw_ostream &Out,
+                                    const DILexicalBlockFile *N,
                                     TypePrinting *TypePrinter,
                                     SlotTracker *Machine,
                                     const Module *Context) {
-  Out << "!MDLexicalBlockFile(";
+  Out << "!DILexicalBlockFile(";
   MDFieldPrinter Printer(Out, TypePrinter, Machine, Context);
   Printer.printMetadata("scope", N->getRawScope(), /* ShouldSkipNull */ false);
   Printer.printMetadata("file", N->getRawFile());
@@ -1673,10 +1724,10 @@ static void writeMDLexicalBlockFile(raw_ostream &Out,
   Out << ")";
 }
 
-static void writeMDNamespace(raw_ostream &Out, const MDNamespace *N,
+static void writeDINamespace(raw_ostream &Out, const DINamespace *N,
                              TypePrinting *TypePrinter, SlotTracker *Machine,
                              const Module *Context) {
-  Out << "!MDNamespace(";
+  Out << "!DINamespace(";
   MDFieldPrinter Printer(Out, TypePrinter, Machine, Context);
   Printer.printString("name", N->getName());
   Printer.printMetadata("scope", N->getRawScope(), /* ShouldSkipNull */ false);
@@ -1685,24 +1736,38 @@ static void writeMDNamespace(raw_ostream &Out, const MDNamespace *N,
   Out << ")";
 }
 
-static void writeMDTemplateTypeParameter(raw_ostream &Out,
-                                         const MDTemplateTypeParameter *N,
+static void writeDIModule(raw_ostream &Out, const DIModule *N,
+                          TypePrinting *TypePrinter, SlotTracker *Machine,
+                          const Module *Context) {
+  Out << "!DIModule(";
+  MDFieldPrinter Printer(Out, TypePrinter, Machine, Context);
+  Printer.printMetadata("scope", N->getRawScope(), /* ShouldSkipNull */ false);
+  Printer.printString("name", N->getName());
+  Printer.printString("configMacros", N->getConfigurationMacros());
+  Printer.printString("includePath", N->getIncludePath());
+  Printer.printString("isysroot", N->getISysRoot());
+  Out << ")";
+}
+
+
+static void writeDITemplateTypeParameter(raw_ostream &Out,
+                                         const DITemplateTypeParameter *N,
                                          TypePrinting *TypePrinter,
                                          SlotTracker *Machine,
                                          const Module *Context) {
-  Out << "!MDTemplateTypeParameter(";
+  Out << "!DITemplateTypeParameter(";
   MDFieldPrinter Printer(Out, TypePrinter, Machine, Context);
   Printer.printString("name", N->getName());
   Printer.printMetadata("type", N->getRawType(), /* ShouldSkipNull */ false);
   Out << ")";
 }
 
-static void writeMDTemplateValueParameter(raw_ostream &Out,
-                                          const MDTemplateValueParameter *N,
+static void writeDITemplateValueParameter(raw_ostream &Out,
+                                          const DITemplateValueParameter *N,
                                           TypePrinting *TypePrinter,
                                           SlotTracker *Machine,
                                           const Module *Context) {
-  Out << "!MDTemplateValueParameter(";
+  Out << "!DITemplateValueParameter(";
   MDFieldPrinter Printer(Out, TypePrinter, Machine, Context);
   if (N->getTag() != dwarf::DW_TAG_template_value_parameter)
     Printer.printTag(N);
@@ -1712,10 +1777,10 @@ static void writeMDTemplateValueParameter(raw_ostream &Out,
   Out << ")";
 }
 
-static void writeMDGlobalVariable(raw_ostream &Out, const MDGlobalVariable *N,
+static void writeDIGlobalVariable(raw_ostream &Out, const DIGlobalVariable *N,
                                   TypePrinting *TypePrinter,
                                   SlotTracker *Machine, const Module *Context) {
-  Out << "!MDGlobalVariable(";
+  Out << "!DIGlobalVariable(";
   MDFieldPrinter Printer(Out, TypePrinter, Machine, Context);
   Printer.printString("name", N->getName());
   Printer.printString("linkageName", N->getLinkageName());
@@ -1730,10 +1795,10 @@ static void writeMDGlobalVariable(raw_ostream &Out, const MDGlobalVariable *N,
   Out << ")";
 }
 
-static void writeMDLocalVariable(raw_ostream &Out, const MDLocalVariable *N,
+static void writeDILocalVariable(raw_ostream &Out, const DILocalVariable *N,
                                  TypePrinting *TypePrinter,
                                  SlotTracker *Machine, const Module *Context) {
-  Out << "!MDLocalVariable(";
+  Out << "!DILocalVariable(";
   MDFieldPrinter Printer(Out, TypePrinter, Machine, Context);
   Printer.printTag(N);
   Printer.printString("name", N->getName());
@@ -1748,10 +1813,10 @@ static void writeMDLocalVariable(raw_ostream &Out, const MDLocalVariable *N,
   Out << ")";
 }
 
-static void writeMDExpression(raw_ostream &Out, const MDExpression *N,
+static void writeDIExpression(raw_ostream &Out, const DIExpression *N,
                               TypePrinting *TypePrinter, SlotTracker *Machine,
                               const Module *Context) {
-  Out << "!MDExpression(";
+  Out << "!DIExpression(";
   FieldSeparator FS;
   if (N->isValid()) {
     for (auto I = N->expr_op_begin(), E = N->expr_op_end(); I != E; ++I) {
@@ -1769,10 +1834,10 @@ static void writeMDExpression(raw_ostream &Out, const MDExpression *N,
   Out << ")";
 }
 
-static void writeMDObjCProperty(raw_ostream &Out, const MDObjCProperty *N,
+static void writeDIObjCProperty(raw_ostream &Out, const DIObjCProperty *N,
                                 TypePrinting *TypePrinter, SlotTracker *Machine,
                                 const Module *Context) {
-  Out << "!MDObjCProperty(";
+  Out << "!DIObjCProperty(";
   MDFieldPrinter Printer(Out, TypePrinter, Machine, Context);
   Printer.printString("name", N->getName());
   Printer.printMetadata("file", N->getRawFile());
@@ -1784,10 +1849,10 @@ static void writeMDObjCProperty(raw_ostream &Out, const MDObjCProperty *N,
   Out << ")";
 }
 
-static void writeMDImportedEntity(raw_ostream &Out, const MDImportedEntity *N,
+static void writeDIImportedEntity(raw_ostream &Out, const DIImportedEntity *N,
                                   TypePrinting *TypePrinter,
                                   SlotTracker *Machine, const Module *Context) {
-  Out << "!MDImportedEntity(";
+  Out << "!DIImportedEntity(";
   MDFieldPrinter Printer(Out, TypePrinter, Machine, Context);
   Printer.printTag(N);
   Printer.printString("name", N->getName());
@@ -1903,8 +1968,11 @@ static void WriteAsOperandInternal(raw_ostream &Out, const Metadata *MD,
                                    SlotTracker *Machine, const Module *Context,
                                    bool FromValue) {
   if (const MDNode *N = dyn_cast<MDNode>(MD)) {
-    if (!Machine)
-      Machine = new SlotTracker(Context);
+    std::unique_ptr<SlotTracker> MachineStorage;
+    if (!Machine) {
+      MachineStorage = make_unique<SlotTracker>(Context);
+      Machine = MachineStorage.get();
+    }
     int Slot = Machine->getMetadataSlot(N);
     if (Slot == -1)
       // Give the pointer value instead of "badref", since this comes up all
@@ -1936,13 +2004,14 @@ namespace {
 class AssemblyWriter {
   formatted_raw_ostream &Out;
   const Module *TheModule;
-  std::unique_ptr<SlotTracker> ModuleSlotTracker;
+  std::unique_ptr<SlotTracker> SlotTrackerStorage;
   SlotTracker &Machine;
   TypePrinting TypePrinter;
   AssemblyAnnotationWriter *AnnotationWriter;
   SetVector<const Comdat *> Comdats;
   bool ShouldPreserveUseListOrder;
   UseListOrderStack UseListOrders;
+  SmallVector<StringRef, 8> MDNames;
 
 public:
   /// Construct an AssemblyWriter with an external SlotTracker
@@ -1987,9 +2056,18 @@ class AssemblyWriter {
 private:
   void init();
 
+  /// \brief Print out metadata attachments.
+  void printMetadataAttachments(
+      const SmallVectorImpl<std::pair<unsigned, MDNode *>> &MDs,
+      StringRef Separator);
+
   // printInfoComment - Print a little comment after the instruction indicating
   // which slot it occupies.
   void printInfoComment(const Value &V);
+
+  // printGCRelocateComment - print comment after call to the gc.relocate
+  // intrinsic indicating base and derived pointer names.
+  void printGCRelocateComment(const Value &V);
 };
 } // namespace
 
@@ -2016,8 +2094,8 @@ AssemblyWriter::AssemblyWriter(formatted_raw_ostream &o, SlotTracker &Mac,
 AssemblyWriter::AssemblyWriter(formatted_raw_ostream &o, const Module *M,
                                AssemblyAnnotationWriter *AAW,
                                bool ShouldPreserveUseListOrder)
-    : Out(o), TheModule(M), ModuleSlotTracker(createSlotTracker(M)),
-      Machine(*ModuleSlotTracker), AnnotationWriter(AAW),
+    : Out(o), TheModule(M), SlotTrackerStorage(createSlotTracker(M)),
+      Machine(*SlotTrackerStorage), AnnotationWriter(AAW),
       ShouldPreserveUseListOrder(ShouldPreserveUseListOrder) {
   init();
 }
@@ -2122,27 +2200,20 @@ void AssemblyWriter::printModule(const Module *M) {
     Out << "target triple = \"" << M->getTargetTriple() << "\"\n";
 
   if (!M->getModuleInlineAsm().empty()) {
-    // Split the string into lines, to make it easier to read the .ll file.
-    std::string Asm = M->getModuleInlineAsm();
-    size_t CurPos = 0;
-    size_t NewLine = Asm.find_first_of('\n', CurPos);
     Out << '\n';
-    while (NewLine != std::string::npos) {
+
+    // Split the string into lines, to make it easier to read the .ll file.
+    StringRef Asm = M->getModuleInlineAsm();
+    do {
+      StringRef Front;
+      std::tie(Front, Asm) = Asm.split('\n');
+
       // We found a newline, print the portion of the asm string from the
       // last newline up to this newline.
       Out << "module asm \"";
-      PrintEscapedString(std::string(Asm.begin()+CurPos, Asm.begin()+NewLine),
-                         Out);
-      Out << "\"\n";
-      CurPos = NewLine+1;
-      NewLine = Asm.find_first_of('\n', CurPos);
-    }
-    std::string rest(Asm.begin()+CurPos, Asm.end());
-    if (!rest.empty()) {
-      Out << "module asm \"";
-      PrintEscapedString(rest, Out);
+      PrintEscapedString(Front, Out);
       Out << "\"\n";
-    }
+    } while (!Asm.empty());
   }
 
   printTypeIdentities();
@@ -2158,23 +2229,21 @@ void AssemblyWriter::printModule(const Module *M) {
 
   // Output all globals.
   if (!M->global_empty()) Out << '\n';
-  for (Module::const_global_iterator I = M->global_begin(), E = M->global_end();
-       I != E; ++I) {
-    printGlobal(I); Out << '\n';
+  for (const GlobalVariable &GV : M->globals()) {
+    printGlobal(&GV); Out << '\n';
   }
 
   // Output all aliases.
   if (!M->alias_empty()) Out << "\n";
-  for (Module::const_alias_iterator I = M->alias_begin(), E = M->alias_end();
-       I != E; ++I)
-    printAlias(I);
+  for (const GlobalAlias &GA : M->aliases())
+    printAlias(&GA);
 
   // Output global use-lists.
   printUseLists(nullptr);
 
   // Output all of the functions.
-  for (Module::const_iterator I = M->begin(), E = M->end(); I != E; ++I)
-    printFunction(I);
+  for (const Function &F : *M)
+    printFunction(&F);
   assert(UseListOrders.empty() && "All use-lists should have been consumed");
 
   // Output all attribute groups.
@@ -2186,9 +2255,8 @@ void AssemblyWriter::printModule(const Module *M) {
   // Output named metadata.
   if (!M->named_metadata_empty()) Out << '\n';
 
-  for (Module::const_named_metadata_iterator I = M->named_metadata_begin(),
-       E = M->named_metadata_end(); I != E; ++I)
-    printNamedMDNode(I);
+  for (const NamedMDNode &Node : M->named_metadata())
+    printNamedMDNode(&Node);
 
   // Output metadata.
   if (!Machine.mdn_empty()) {
@@ -2197,15 +2265,13 @@ void AssemblyWriter::printModule(const Module *M) {
   }
 }
 
-void AssemblyWriter::printNamedMDNode(const NamedMDNode *NMD) {
-  Out << '!';
-  StringRef Name = NMD->getName();
+static void printMetadataIdentifier(StringRef Name,
+                                    formatted_raw_ostream &Out) {
   if (Name.empty()) {
     Out << "<empty name> ";
   } else {
-    if (isalpha(static_cast<unsigned char>(Name[0])) ||
-        Name[0] == '-' || Name[0] == '$' ||
-        Name[0] == '.' || Name[0] == '_')
+    if (isalpha(static_cast<unsigned char>(Name[0])) || Name[0] == '-' ||
+        Name[0] == '$' || Name[0] == '.' || Name[0] == '_')
       Out << Name[0];
     else
       Out << '\\' << hexdigit(Name[0] >> 4) << hexdigit(Name[0] & 0x0F);
@@ -2218,9 +2284,15 @@ void AssemblyWriter::printNamedMDNode(const NamedMDNode *NMD) {
         Out << '\\' << hexdigit(C >> 4) << hexdigit(C & 0x0F);
     }
   }
+}
+
+void AssemblyWriter::printNamedMDNode(const NamedMDNode *NMD) {
+  Out << '!';
+  printMetadataIdentifier(NMD->getName(), Out);
   Out << " = !{";
   for (unsigned i = 0, e = NMD->getNumOperands(); i != e; ++i) {
-    if (i) Out << ", ";
+    if (i)
+      Out << ", ";
     int Slot = Machine.getMetadataSlot(NMD->getOperand(i));
     if (Slot == -1)
       Out << "<badref>";
@@ -2230,7 +2302,6 @@ void AssemblyWriter::printNamedMDNode(const NamedMDNode *NMD) {
   Out << "}\n";
 }
 
-
 static void PrintLinkage(GlobalValue::LinkageTypes LT,
                          formatted_raw_ostream &Out) {
   switch (LT) {
@@ -2250,7 +2321,6 @@ static void PrintLinkage(GlobalValue::LinkageTypes LT,
   }
 }
 
-
 static void PrintVisibility(GlobalValue::VisibilityTypes Vis,
                             formatted_raw_ostream &Out) {
   switch (Vis) {
@@ -2351,13 +2421,9 @@ void AssemblyWriter::printAlias(const GlobalAlias *GA) {
   if (GA->isMaterializable())
     Out << "; Materializable\n";
 
-  // Don't crash when dumping partially built GA
-  if (!GA->hasName())
-    Out << "<<nameless>> = ";
-  else {
-    PrintLLVMName(Out, GA);
-    Out << " = ";
-  }
+  WriteAsOperandInternal(Out, GA, &TypePrinter, &Machine, GA->getParent());
+  Out << " = ";
+
   PrintLinkage(GA->getLinkage(), Out);
   PrintVisibility(GA->getVisibility(), Out);
   PrintDLLStorageClass(GA->getDLLStorageClass(), Out);
@@ -2534,6 +2600,14 @@ void AssemblyWriter::printFunction(const Function *F) {
     Out << " prologue ";
     writeOperand(F->getPrologueData(), true);
   }
+  if (F->hasPersonalityFn()) {
+    Out << " personality ";
+    writeOperand(F->getPersonalityFn(), /*PrintType=*/true);
+  }
+
+  SmallVector<std::pair<unsigned, MDNode *>, 4> MDs;
+  F->getAllMetadata(MDs);
+  printMetadataAttachments(MDs, " ");
 
   if (F->isDeclaration()) {
     Out << '\n';
@@ -2626,10 +2700,26 @@ void AssemblyWriter::printInstructionLine(const Instruction &I) {
   Out << '\n';
 }
 
+/// printGCRelocateComment - print comment after call to the gc.relocate
+/// intrinsic indicating base and derived pointer names.
+void AssemblyWriter::printGCRelocateComment(const Value &V) {
+  assert(isGCRelocate(&V));
+  GCRelocateOperands GCOps(cast<Instruction>(&V));
+
+  Out << " ; (";
+  writeOperand(GCOps.getBasePtr(), false);
+  Out << ", ";
+  writeOperand(GCOps.getDerivedPtr(), false);
+  Out << ")";
+}
+
 /// printInfoComment - Print a little comment after the instruction indicating
 /// which slot it occupies.
 ///
 void AssemblyWriter::printInfoComment(const Value &V) {
+  if (isGCRelocate(&V))
+    printGCRelocateComment(V);
+
   if (AnnotationWriter)
     AnnotationWriter->printInfoComment(V, Out);
 }
@@ -2756,8 +2846,8 @@ void AssemblyWriter::printInstruction(const Instruction &I) {
   } else if (const LandingPadInst *LPI = dyn_cast<LandingPadInst>(&I)) {
     Out << ' ';
     TypePrinter.print(I.getType(), Out);
-    Out << " personality ";
-    writeOperand(I.getOperand(0), true); Out << '\n';
+    if (LPI->isCleanup() || LPI->getNumClauses() != 0)
+      Out << '\n';
 
     if (LPI->isCleanup())
       Out << "          cleanup";
@@ -2771,8 +2861,66 @@ void AssemblyWriter::printInstruction(const Instruction &I) {
 
       writeOperand(LPI->getClause(i), true);
     }
+  } else if (const auto *CPI = dyn_cast<CatchPadInst>(&I)) {
+    Out << ' ';
+    TypePrinter.print(I.getType(), Out);
+
+    Out << " [";
+    for (unsigned Op = 0, NumOps = CPI->getNumArgOperands(); Op < NumOps;
+         ++Op) {
+      if (Op > 0)
+        Out << ", ";
+      writeOperand(CPI->getArgOperand(Op), /*PrintType=*/true);
+    }
+    Out << "] to ";
+    writeOperand(CPI->getNormalDest(), /*PrintType=*/true);
+    Out << " unwind ";
+    writeOperand(CPI->getUnwindDest(), /*PrintType=*/true);
+  } else if (const auto *TPI = dyn_cast<TerminatePadInst>(&I)) {
+    Out << " [";
+    for (unsigned Op = 0, NumOps = TPI->getNumArgOperands(); Op < NumOps;
+         ++Op) {
+      if (Op > 0)
+        Out << ", ";
+      writeOperand(TPI->getArgOperand(Op), /*PrintType=*/true);
+    }
+    Out << "] unwind ";
+    if (TPI->hasUnwindDest())
+      writeOperand(TPI->getUnwindDest(), /*PrintType=*/true);
+    else
+      Out << "to caller";
+  } else if (const auto *CPI = dyn_cast<CleanupPadInst>(&I)) {
+    Out << ' ';
+    TypePrinter.print(I.getType(), Out);
+
+    Out << " [";
+    for (unsigned Op = 0, NumOps = CPI->getNumOperands(); Op < NumOps; ++Op) {
+      if (Op > 0)
+        Out << ", ";
+      writeOperand(CPI->getOperand(Op), /*PrintType=*/true);
+    }
+    Out << "]";
   } else if (isa<ReturnInst>(I) && !Operand) {
     Out << " void";
+  } else if (const auto *CRI = dyn_cast<CleanupReturnInst>(&I)) {
+    if (CRI->hasReturnValue()) {
+      Out << ' ';
+      writeOperand(CRI->getReturnValue(), /*PrintType=*/true);
+    } else {
+      Out << " void";
+    }
+
+    Out << " unwind ";
+    if (CRI->hasUnwindDest())
+      writeOperand(CRI->getUnwindDest(), /*PrintType=*/true);
+    else
+      Out << "to caller";
+  } else if (const auto *CEPI = dyn_cast<CatchEndPadInst>(&I)) {
+    Out << " unwind ";
+    if (CEPI->hasUnwindDest())
+      writeOperand(CEPI->getUnwindDest(), /*PrintType=*/true);
+    else
+      Out << "to caller";
   } else if (const CallInst *CI = dyn_cast<CallInst>(&I)) {
     // Print the calling convention being used.
     if (CI->getCallingConv() != CallingConv::C) {
@@ -2815,8 +2963,7 @@ void AssemblyWriter::printInstruction(const Instruction &I) {
       Out << " #" << Machine.getAttributeGroupSlot(PAL.getFnAttributes());
   } else if (const InvokeInst *II = dyn_cast<InvokeInst>(&I)) {
     Operand = II->getCalledValue();
-    PointerType *PTy = cast<PointerType>(Operand->getType());
-    FunctionType *FTy = cast<FunctionType>(PTy->getElementType());
+    FunctionType *FTy = cast<FunctionType>(II->getFunctionType());
     Type *RetTy = FTy->getReturnType();
     const AttributeSet &PAL = II->getAttributes();
 
@@ -2834,15 +2981,9 @@ void AssemblyWriter::printInstruction(const Instruction &I) {
     // and if the return type is not a pointer to a function.
     //
     Out << ' ';
-    if (!FTy->isVarArg() &&
-        (!RetTy->isPointerTy() ||
-         !cast<PointerType>(RetTy)->getElementType()->isFunctionTy())) {
-      TypePrinter.print(RetTy, Out);
-      Out << ' ';
-      writeOperand(Operand, false);
-    } else {
-      writeOperand(Operand, true);
-    }
+    TypePrinter.print(FTy->isVarArg() ? FTy : RetTy, Out);
+    Out << ' ';
+    writeOperand(Operand, false);
     Out << '(';
     for (unsigned op = 0, Eop = II->getNumArgOperands(); op < Eop; ++op) {
       if (op)
@@ -2959,24 +3100,34 @@ void AssemblyWriter::printInstruction(const Instruction &I) {
   // Print Metadata info.
   SmallVector<std::pair<unsigned, MDNode *>, 4> InstMD;
   I.getAllMetadata(InstMD);
-  if (!InstMD.empty()) {
-    SmallVector<StringRef, 8> MDNames;
-    I.getType()->getContext().getMDKindNames(MDNames);
-    for (unsigned i = 0, e = InstMD.size(); i != e; ++i) {
-      unsigned Kind = InstMD[i].first;
-       if (Kind < MDNames.size()) {
-         Out << ", !" << MDNames[Kind];
-       } else {
-         Out << ", !<unknown kind #" << Kind << ">";
-       }
-      Out << ' ';
-      WriteAsOperandInternal(Out, InstMD[i].second, &TypePrinter, &Machine,
-                             TheModule);
-    }
-  }
+  printMetadataAttachments(InstMD, ", ");
+
+  // Print a nice comment.
   printInfoComment(I);
 }
 
+void AssemblyWriter::printMetadataAttachments(
+    const SmallVectorImpl<std::pair<unsigned, MDNode *>> &MDs,
+    StringRef Separator) {
+  if (MDs.empty())
+    return;
+
+  if (MDNames.empty())
+    TheModule->getMDKindNames(MDNames);
+
+  for (const auto &I : MDs) {
+    unsigned Kind = I.first;
+    Out << Separator;
+    if (Kind < MDNames.size()) {
+      Out << "!";
+      printMetadataIdentifier(MDNames[Kind], Out);
+    } else
+      Out << "!<unknown kind #" << Kind << ">";
+    Out << ' ';
+    WriteAsOperandInternal(Out, I.second, &TypePrinter, &Machine, TheModule);
+  }
+}
+
 void AssemblyWriter::writeMDNode(unsigned Slot, const MDNode *Node) {
   Out << '!' << Slot << " = ";
   printMDNodeBody(Node);
@@ -3127,21 +3278,35 @@ static bool isReferencingMDNode(const Instruction &I) {
 }
 
 void Value::print(raw_ostream &ROS) const {
+  bool ShouldInitializeAllMetadata = false;
+  if (auto *I = dyn_cast<Instruction>(this))
+    ShouldInitializeAllMetadata = isReferencingMDNode(*I);
+  else if (isa<Function>(this) || isa<MetadataAsValue>(this))
+    ShouldInitializeAllMetadata = true;
+
+  ModuleSlotTracker MST(getModuleFromVal(this), ShouldInitializeAllMetadata);
+  print(ROS, MST);
+}
+
+void Value::print(raw_ostream &ROS, ModuleSlotTracker &MST) const {
   formatted_raw_ostream OS(ROS);
+  SlotTracker EmptySlotTable(static_cast<const Module *>(nullptr));
+  SlotTracker &SlotTable =
+      MST.getMachine() ? *MST.getMachine() : EmptySlotTable;
+  auto incorporateFunction = [&](const Function *F) {
+    if (F)
+      MST.incorporateFunction(*F);
+  };
+
   if (const Instruction *I = dyn_cast<Instruction>(this)) {
-    const Function *F = I->getParent() ? I->getParent()->getParent() : nullptr;
-    SlotTracker SlotTable(
-        F,
-        /* ShouldInitializeAllMetadata */ isReferencingMDNode(*I));
+    incorporateFunction(I->getParent() ? I->getParent()->getParent() : nullptr);
     AssemblyWriter W(OS, SlotTable, getModuleFromVal(I), nullptr);
     W.printInstruction(*I);
   } else if (const BasicBlock *BB = dyn_cast<BasicBlock>(this)) {
-    SlotTracker SlotTable(BB->getParent());
+    incorporateFunction(BB->getParent());
     AssemblyWriter W(OS, SlotTable, getModuleFromVal(BB), nullptr);
     W.printBasicBlock(BB);
   } else if (const GlobalValue *GV = dyn_cast<GlobalValue>(this)) {
-    SlotTracker SlotTable(GV->getParent(),
-                          /* ShouldInitializeAllMetadata */ isa<Function>(GV));
     AssemblyWriter W(OS, SlotTable, GV->getParent(), nullptr);
     if (const GlobalVariable *V = dyn_cast<GlobalVariable>(GV))
       W.printGlobal(V);
@@ -3150,69 +3315,108 @@ void Value::print(raw_ostream &ROS) const {
     else
       W.printAlias(cast<GlobalAlias>(GV));
   } else if (const MetadataAsValue *V = dyn_cast<MetadataAsValue>(this)) {
-    V->getMetadata()->print(ROS, getModuleFromVal(V));
+    V->getMetadata()->print(ROS, MST, getModuleFromVal(V));
   } else if (const Constant *C = dyn_cast<Constant>(this)) {
     TypePrinting TypePrinter;
     TypePrinter.print(C->getType(), OS);
     OS << ' ';
-    WriteConstantInternal(OS, C, TypePrinter, nullptr, nullptr);
+    WriteConstantInternal(OS, C, TypePrinter, MST.getMachine(), nullptr);
   } else if (isa<InlineAsm>(this) || isa<Argument>(this)) {
-    this->printAsOperand(OS);
+    this->printAsOperand(OS, /* PrintType */ true, MST);
   } else {
     llvm_unreachable("Unknown value to print out!");
   }
 }
 
-void Value::printAsOperand(raw_ostream &O, bool PrintType, const Module *M) const {
-  // Fast path: Don't construct and populate a TypePrinting object if we
-  // won't be needing any types printed.
-  bool IsMetadata = isa<MetadataAsValue>(this);
-  if (!PrintType && ((!isa<Constant>(this) && !IsMetadata) || hasName() ||
-                     isa<GlobalValue>(this))) {
-    WriteAsOperandInternal(O, this, nullptr, nullptr, M);
-    return;
+/// Print without a type, skipping the TypePrinting object.
+///
+/// \return \c true iff printing was succesful.
+static bool printWithoutType(const Value &V, raw_ostream &O,
+                             SlotTracker *Machine, const Module *M) {
+  if (V.hasName() || isa<GlobalValue>(V) ||
+      (!isa<Constant>(V) && !isa<MetadataAsValue>(V))) {
+    WriteAsOperandInternal(O, &V, nullptr, Machine, M);
+    return true;
   }
+  return false;
+}
 
-  if (!M)
-    M = getModuleFromVal(this);
-
+static void printAsOperandImpl(const Value &V, raw_ostream &O, bool PrintType,
+                               ModuleSlotTracker &MST) {
   TypePrinting TypePrinter;
-  if (M)
+  if (const Module *M = MST.getModule())
     TypePrinter.incorporateTypes(*M);
   if (PrintType) {
-    TypePrinter.print(getType(), O);
+    TypePrinter.print(V.getType(), O);
     O << ' ';
   }
 
-  SlotTracker Machine(M, /* ShouldInitializeAllMetadata */ IsMetadata);
-  WriteAsOperandInternal(O, this, &TypePrinter, &Machine, M);
+  WriteAsOperandInternal(O, &V, &TypePrinter, MST.getMachine(),
+                         MST.getModule());
+}
+
+void Value::printAsOperand(raw_ostream &O, bool PrintType,
+                           const Module *M) const {
+  if (!M)
+    M = getModuleFromVal(this);
+
+  if (!PrintType)
+    if (printWithoutType(*this, O, nullptr, M))
+      return;
+
+  SlotTracker Machine(
+      M, /* ShouldInitializeAllMetadata */ isa<MetadataAsValue>(this));
+  ModuleSlotTracker MST(Machine, M);
+  printAsOperandImpl(*this, O, PrintType, MST);
+}
+
+void Value::printAsOperand(raw_ostream &O, bool PrintType,
+                           ModuleSlotTracker &MST) const {
+  if (!PrintType)
+    if (printWithoutType(*this, O, MST.getMachine(), MST.getModule()))
+      return;
+
+  printAsOperandImpl(*this, O, PrintType, MST);
 }
 
 static void printMetadataImpl(raw_ostream &ROS, const Metadata &MD,
-                              const Module *M, bool OnlyAsOperand) {
+                              ModuleSlotTracker &MST, const Module *M,
+                              bool OnlyAsOperand) {
   formatted_raw_ostream OS(ROS);
 
-  auto *N = dyn_cast<MDNode>(&MD);
   TypePrinting TypePrinter;
-  SlotTracker Machine(M, /* ShouldInitializeAllMetadata */ N);
   if (M)
     TypePrinter.incorporateTypes(*M);
 
-  WriteAsOperandInternal(OS, &MD, &TypePrinter, &Machine, M,
+  WriteAsOperandInternal(OS, &MD, &TypePrinter, MST.getMachine(), M,
                          /* FromValue */ true);
+
+  auto *N = dyn_cast<MDNode>(&MD);
   if (OnlyAsOperand || !N)
     return;
 
   OS << " = ";
-  WriteMDNodeBodyInternal(OS, N, &TypePrinter, &Machine, M);
+  WriteMDNodeBodyInternal(OS, N, &TypePrinter, MST.getMachine(), M);
 }
 
 void Metadata::printAsOperand(raw_ostream &OS, const Module *M) const {
-  printMetadataImpl(OS, *this, M, /* OnlyAsOperand */ true);
+  ModuleSlotTracker MST(M, isa<MDNode>(this));
+  printMetadataImpl(OS, *this, MST, M, /* OnlyAsOperand */ true);
+}
+
+void Metadata::printAsOperand(raw_ostream &OS, ModuleSlotTracker &MST,
+                              const Module *M) const {
+  printMetadataImpl(OS, *this, MST, M, /* OnlyAsOperand */ true);
 }
 
 void Metadata::print(raw_ostream &OS, const Module *M) const {
-  printMetadataImpl(OS, *this, M, /* OnlyAsOperand */ false);
+  ModuleSlotTracker MST(M, isa<MDNode>(this));
+  printMetadataImpl(OS, *this, MST, M, /* OnlyAsOperand */ false);
+}
+
+void Metadata::print(raw_ostream &OS, ModuleSlotTracker &MST,
+                     const Module *M) const {
+  printMetadataImpl(OS, *this, MST, M, /* OnlyAsOperand */ false);
 }
 
 // Value::dump - allow easy printing of Values from the debugger.
diff --git a/lib/IR/AttributeImpl.h b/lib/IR/AttributeImpl.h
index d544689abec..6f338ae835f 100644
--- a/lib/IR/AttributeImpl.h
+++ b/lib/IR/AttributeImpl.h
@@ -166,6 +166,7 @@ class AttributeSetNode : public FoldingSetNode {
   unsigned getAlignment() const;
   unsigned getStackAlignment() const;
   uint64_t getDereferenceableBytes() const;
+  uint64_t getDereferenceableOrNullBytes() const;
   std::string getAsString(bool InAttrGrp) const;
 
   typedef const Attribute *iterator;
@@ -180,6 +181,9 @@ class AttributeSetNode : public FoldingSetNode {
       AttrList[I].Profile(ID);
   }
 };
+static_assert(
+    AlignOf<AttributeSetNode>::Alignment >= AlignOf<Attribute>::Alignment,
+    "Alignment is insufficient for objects appended to AttributeSetNode");
 
 //===----------------------------------------------------------------------===//
 /// \class
@@ -188,9 +192,11 @@ class AttributeSetNode : public FoldingSetNode {
 class AttributeSetImpl : public FoldingSetNode {
   friend class AttributeSet;
 
-  LLVMContext &Context;
-
+public:
   typedef std::pair<unsigned, AttributeSetNode*> IndexAttrPair;
+
+private:
+  LLVMContext &Context;
   unsigned NumAttrs; ///< Number of entries in this set.
 
   /// \brief Return a pointer to the IndexAttrPair for the specified slot.
@@ -205,6 +211,7 @@ class AttributeSetImpl : public FoldingSetNode {
   AttributeSetImpl(LLVMContext &C,
                    ArrayRef<std::pair<unsigned, AttributeSetNode *> > Attrs)
       : Context(C), NumAttrs(Attrs.size()) {
+
 #ifndef NDEBUG
     if (Attrs.size() >= 2) {
       for (const std::pair<unsigned, AttributeSetNode *> *i = Attrs.begin() + 1,
@@ -266,6 +273,10 @@ class AttributeSetImpl : public FoldingSetNode {
 
   void dump() const;
 };
+static_assert(
+    AlignOf<AttributeSetImpl>::Alignment >=
+        AlignOf<AttributeSetImpl::IndexAttrPair>::Alignment,
+    "Alignment is insufficient for objects appended to AttributeSetImpl");
 
 } // end llvm namespace
 
diff --git a/lib/IR/Attributes.cpp b/lib/IR/Attributes.cpp
index be5b74f59a2..546a98670a2 100644
--- a/lib/IR/Attributes.cpp
+++ b/lib/IR/Attributes.cpp
@@ -190,10 +190,14 @@ std::string Attribute::getAsString(bool InAttrGrp) const {
     return "sanitize_address";
   if (hasAttribute(Attribute::AlwaysInline))
     return "alwaysinline";
+  if (hasAttribute(Attribute::ArgMemOnly))
+    return "argmemonly";
   if (hasAttribute(Attribute::Builtin))
     return "builtin";
   if (hasAttribute(Attribute::ByVal))
     return "byval";
+  if (hasAttribute(Attribute::Convergent))
+    return "convergent";
   if (hasAttribute(Attribute::InAlloca))
     return "inalloca";
   if (hasAttribute(Attribute::InlineHint))
@@ -250,6 +254,8 @@ std::string Attribute::getAsString(bool InAttrGrp) const {
     return "sspreq";
   if (hasAttribute(Attribute::StackProtectStrong))
     return "sspstrong";
+  if (hasAttribute(Attribute::SafeStack))
+    return "safestack";
   if (hasAttribute(Attribute::StructRet))
     return "sret";
   if (hasAttribute(Attribute::SanitizeThread))
@@ -434,6 +440,8 @@ uint64_t AttributeImpl::getAttrMask(Attribute::AttrKind Val) {
   case Attribute::InAlloca:        return 1ULL << 43;
   case Attribute::NonNull:         return 1ULL << 44;
   case Attribute::JumpTable:       return 1ULL << 45;
+  case Attribute::Convergent:      return 1ULL << 46;
+  case Attribute::SafeStack:       return 1ULL << 47;
   case Attribute::Dereferenceable:
     llvm_unreachable("dereferenceable attribute not supported in raw format");
     break;
@@ -441,6 +449,9 @@ uint64_t AttributeImpl::getAttrMask(Attribute::AttrKind Val) {
     llvm_unreachable("dereferenceable_or_null attribute not supported in raw "
                      "format");
     break;
+  case Attribute::ArgMemOnly:
+    llvm_unreachable("argmemonly attribute not supported in raw format");
+    break;
   }
   llvm_unreachable("Unsupported attribute type");
 }
@@ -532,6 +543,13 @@ uint64_t AttributeSetNode::getDereferenceableBytes() const {
   return 0;
 }
 
+uint64_t AttributeSetNode::getDereferenceableOrNullBytes() const {
+  for (iterator I = begin(), E = end(); I != E; ++I)
+    if (I->hasAttribute(Attribute::DereferenceableOrNull))
+      return I->getDereferenceableOrNullBytes();
+  return 0;
+}
+
 std::string AttributeSetNode::getAsString(bool InAttrGrp) const {
   std::string Str;
   for (iterator I = begin(), E = end(); I != E; ++I) {
@@ -812,12 +830,10 @@ AttributeSet AttributeSet::removeAttributes(LLVMContext &C, unsigned Index,
   if (!pImpl) return AttributeSet();
   if (!Attrs.pImpl) return *this;
 
-#ifndef NDEBUG
   // FIXME it is not obvious how this should work for alignment.
   // For now, say we can't pass in alignment, which no current use does.
   assert(!Attrs.hasAttribute(Index, Attribute::Alignment) &&
          "Attempt to change alignment!");
-#endif
 
   // Add the attribute slots before the one we're trying to add.
   SmallVector<AttributeSet, 4> AttrSet;
@@ -852,6 +868,42 @@ AttributeSet AttributeSet::removeAttributes(LLVMContext &C, unsigned Index,
   return get(C, AttrSet);
 }
 
+AttributeSet AttributeSet::removeAttributes(LLVMContext &C, unsigned Index,
+                                            const AttrBuilder &Attrs) const {
+  if (!pImpl) return AttributeSet();
+
+  // FIXME it is not obvious how this should work for alignment.
+  // For now, say we can't pass in alignment, which no current use does.
+  assert(!Attrs.hasAlignmentAttr() && "Attempt to change alignment!");
+
+  // Add the attribute slots before the one we're trying to add.
+  SmallVector<AttributeSet, 4> AttrSet;
+  uint64_t NumAttrs = pImpl->getNumAttributes();
+  AttributeSet AS;
+  uint64_t LastIndex = 0;
+  for (unsigned I = 0, E = NumAttrs; I != E; ++I) {
+    if (getSlotIndex(I) >= Index) {
+      if (getSlotIndex(I) == Index) AS = getSlotAttributes(LastIndex++);
+      break;
+    }
+    LastIndex = I + 1;
+    AttrSet.push_back(getSlotAttributes(I));
+  }
+
+  // Now remove the attribute from the correct slot. There may already be an
+  // AttributeSet there.
+  AttrBuilder B(AS, Index);
+  B.remove(Attrs);
+
+  AttrSet.push_back(AttributeSet::get(C, Index, B));
+
+  // Add the remaining attribute slots.
+  for (unsigned I = LastIndex, E = NumAttrs; I < E; ++I)
+    AttrSet.push_back(getSlotAttributes(I));
+
+  return get(C, AttrSet);
+}
+
 AttributeSet AttributeSet::addDereferenceableAttr(LLVMContext &C, unsigned Index,
                                                   uint64_t Bytes) const {
   llvm::AttrBuilder B;
@@ -957,6 +1009,11 @@ uint64_t AttributeSet::getDereferenceableBytes(unsigned Index) const {
   return ASN ? ASN->getDereferenceableBytes() : 0;
 }
 
+uint64_t AttributeSet::getDereferenceableOrNullBytes(unsigned Index) const {
+  AttributeSetNode *ASN = getAttributes(Index);
+  return ASN ? ASN->getDereferenceableOrNullBytes() : 0;
+}
+
 std::string AttributeSet::getAsString(unsigned Index,
                                       bool InAttrGrp) const {
   AttributeSetNode *ASN = getAttributes(Index);
@@ -1201,15 +1258,52 @@ AttrBuilder &AttrBuilder::merge(const AttrBuilder &B) {
   if (!DerefBytes)
     DerefBytes = B.DerefBytes;
 
+  if (!DerefOrNullBytes)
+    DerefOrNullBytes = B.DerefOrNullBytes;
+
   Attrs |= B.Attrs;
 
-  for (td_const_iterator I = B.TargetDepAttrs.begin(),
-         E = B.TargetDepAttrs.end(); I != E; ++I)
-    TargetDepAttrs[I->first] = I->second;
+  for (auto I : B.td_attrs())
+    TargetDepAttrs[I.first] = I.second;
+
+  return *this;
+}
+
+AttrBuilder &AttrBuilder::remove(const AttrBuilder &B) {
+  // FIXME: What if both have alignments, but they don't match?!
+  if (B.Alignment)
+    Alignment = 0;
+
+  if (B.StackAlignment)
+    StackAlignment = 0;
+
+  if (B.DerefBytes)
+    DerefBytes = 0;
+
+  if (B.DerefOrNullBytes)
+    DerefOrNullBytes = 0;
+
+  Attrs &= ~B.Attrs;
+
+  for (auto I : B.td_attrs())
+    TargetDepAttrs.erase(I.first);
 
   return *this;
 }
 
+bool AttrBuilder::overlaps(const AttrBuilder &B) const {
+  // First check if any of the target independent attributes overlap.
+  if ((Attrs & B.Attrs).any())
+    return true;
+
+  // Then check if any target dependent ones do.
+  for (auto I : td_attrs())
+    if (B.contains(I.first))
+      return true;
+
+  return false;
+}
+
 bool AttrBuilder::contains(StringRef A) const {
   return TargetDepAttrs.find(A) != TargetDepAttrs.end();
 }
@@ -1267,7 +1361,8 @@ AttrBuilder &AttrBuilder::addRawValue(uint64_t Val) {
   for (Attribute::AttrKind I = Attribute::None; I != Attribute::EndAttrKinds;
        I = Attribute::AttrKind(I + 1)) {
     if (I == Attribute::Dereferenceable ||
-        I == Attribute::DereferenceableOrNull)
+        I == Attribute::DereferenceableOrNull ||
+        I == Attribute::ArgMemOnly)
       continue;
     if (uint64_t A = (Val & AttributeImpl::getAttrMask(I))) {
       Attrs[I] = true;
@@ -1287,7 +1382,7 @@ AttrBuilder &AttrBuilder::addRawValue(uint64_t Val) {
 //===----------------------------------------------------------------------===//
 
 /// \brief Which attributes cannot be applied to a type.
-AttributeSet AttributeFuncs::typeIncompatible(Type *Ty, uint64_t Index) {
+AttrBuilder AttributeFuncs::typeIncompatible(const Type *Ty) {
   AttrBuilder Incompatible;
 
   if (!Ty->isIntegerTy())
@@ -1309,5 +1404,5 @@ AttributeSet AttributeFuncs::typeIncompatible(Type *Ty, uint64_t Index) {
       .addAttribute(Attribute::StructRet)
       .addAttribute(Attribute::InAlloca);
 
-  return AttributeSet::get(Ty->getContext(), Index, Incompatible);
+  return Incompatible;
 }
diff --git a/lib/IR/AutoUpgrade.cpp b/lib/IR/AutoUpgrade.cpp
index bb23d2c9715..f1c6ebd4846 100644
--- a/lib/IR/AutoUpgrade.cpp
+++ b/lib/IR/AutoUpgrade.cpp
@@ -62,21 +62,6 @@ static bool UpgradeX86IntrinsicsWith8BitMask(Function *F, Intrinsic::ID IID,
   return true;
 }
 
-// Upgrade the declarations of AVX-512 cmp intrinsic functions whose 8-bit
-// immediates have changed their type from i32 to i8.
-static bool UpgradeAVX512CmpIntrinsic(Function *F, Intrinsic::ID IID,
-                                      Function *&NewFn) {
-  // Check that the last argument is an i32.
-  Type *LastArgType = F->getFunctionType()->getParamType(2);
-  if (!LastArgType->isIntegerTy(32))
-    return false;
-
-  // Move this function aside and map down.
-  F->setName(F->getName() + ".old");
-  NewFn = Intrinsic::getDeclaration(F->getParent(), IID);
-  return true;
-}
-
 static bool UpgradeIntrinsicFunction1(Function *F, Function *&NewFn) {
   assert(F && "Illegal to upgrade a non-existent Function.");
 
@@ -211,88 +196,6 @@ static bool UpgradeIntrinsicFunction1(Function *F, Function *&NewFn) {
       return UpgradeX86IntrinsicsWith8BitMask(F, Intrinsic::x86_avx2_mpsadbw,
                                               NewFn);
 
-    if (Name == "x86.avx512.mask.cmp.ps.512")
-      return UpgradeAVX512CmpIntrinsic(F, Intrinsic::x86_avx512_mask_cmp_ps_512,
-                                       NewFn);
-    if (Name == "x86.avx512.mask.cmp.pd.512")
-      return UpgradeAVX512CmpIntrinsic(F, Intrinsic::x86_avx512_mask_cmp_pd_512,
-                                       NewFn);
-
-    if (Name == "x86.avx512.mask.cmp.b.512")
-      return UpgradeAVX512CmpIntrinsic(F, Intrinsic::x86_avx512_mask_cmp_b_512,
-                                       NewFn);
-    if (Name == "x86.avx512.mask.cmp.w.512")
-      return UpgradeAVX512CmpIntrinsic(F, Intrinsic::x86_avx512_mask_cmp_w_512,
-                                       NewFn);
-    if (Name == "x86.avx512.mask.cmp.d.512")
-      return UpgradeAVX512CmpIntrinsic(F, Intrinsic::x86_avx512_mask_cmp_d_512,
-                                       NewFn);
-    if (Name == "x86.avx512.mask.cmp.q.512")
-      return UpgradeAVX512CmpIntrinsic(F, Intrinsic::x86_avx512_mask_cmp_q_512,
-                                       NewFn);
-    if (Name == "x86.avx512.mask.ucmp.b.512")
-      return UpgradeAVX512CmpIntrinsic(F, Intrinsic::x86_avx512_mask_ucmp_b_512,
-                                       NewFn);
-    if (Name == "x86.avx512.mask.ucmp.w.512")
-      return UpgradeAVX512CmpIntrinsic(F, Intrinsic::x86_avx512_mask_ucmp_w_512,
-                                       NewFn);
-    if (Name == "x86.avx512.mask.ucmp.d.512")
-      return UpgradeAVX512CmpIntrinsic(F, Intrinsic::x86_avx512_mask_ucmp_d_512,
-                                       NewFn);
-    if (Name == "x86.avx512.mask.ucmp.q.512")
-      return UpgradeAVX512CmpIntrinsic(F, Intrinsic::x86_avx512_mask_ucmp_q_512,
-                                       NewFn);
-
-    if (Name == "x86.avx512.mask.cmp.b.256")
-      return UpgradeAVX512CmpIntrinsic(F, Intrinsic::x86_avx512_mask_cmp_b_256,
-                                       NewFn);
-    if (Name == "x86.avx512.mask.cmp.w.256")
-      return UpgradeAVX512CmpIntrinsic(F, Intrinsic::x86_avx512_mask_cmp_w_256,
-                                       NewFn);
-    if (Name == "x86.avx512.mask.cmp.d.256")
-      return UpgradeAVX512CmpIntrinsic(F, Intrinsic::x86_avx512_mask_cmp_d_256,
-                                       NewFn);
-    if (Name == "x86.avx512.mask.cmp.q.256")
-      return UpgradeAVX512CmpIntrinsic(F, Intrinsic::x86_avx512_mask_cmp_q_256,
-                                       NewFn);
-    if (Name == "x86.avx512.mask.ucmp.b.256")
-      return UpgradeAVX512CmpIntrinsic(F, Intrinsic::x86_avx512_mask_ucmp_b_256,
-                                       NewFn);
-    if (Name == "x86.avx512.mask.ucmp.w.256")
-      return UpgradeAVX512CmpIntrinsic(F, Intrinsic::x86_avx512_mask_ucmp_w_256,
-                                       NewFn);
-    if (Name == "x86.avx512.mask.ucmp.d.256")
-      return UpgradeAVX512CmpIntrinsic(F, Intrinsic::x86_avx512_mask_ucmp_d_256,
-                                       NewFn);
-    if (Name == "x86.avx512.mask.ucmp.q.256")
-      return UpgradeAVX512CmpIntrinsic(F, Intrinsic::x86_avx512_mask_ucmp_q_256,
-                                       NewFn);
-
-    if (Name == "x86.avx512.mask.cmp.b.128")
-      return UpgradeAVX512CmpIntrinsic(F, Intrinsic::x86_avx512_mask_cmp_b_128,
-                                       NewFn);
-    if (Name == "x86.avx512.mask.cmp.w.128")
-      return UpgradeAVX512CmpIntrinsic(F, Intrinsic::x86_avx512_mask_cmp_w_128,
-                                       NewFn);
-    if (Name == "x86.avx512.mask.cmp.d.128")
-      return UpgradeAVX512CmpIntrinsic(F, Intrinsic::x86_avx512_mask_cmp_d_128,
-                                       NewFn);
-    if (Name == "x86.avx512.mask.cmp.q.128")
-      return UpgradeAVX512CmpIntrinsic(F, Intrinsic::x86_avx512_mask_cmp_q_128,
-                                       NewFn);
-    if (Name == "x86.avx512.mask.ucmp.b.128")
-      return UpgradeAVX512CmpIntrinsic(F, Intrinsic::x86_avx512_mask_ucmp_b_128,
-                                       NewFn);
-    if (Name == "x86.avx512.mask.ucmp.w.128")
-      return UpgradeAVX512CmpIntrinsic(F, Intrinsic::x86_avx512_mask_ucmp_w_128,
-                                       NewFn);
-    if (Name == "x86.avx512.mask.ucmp.d.128")
-      return UpgradeAVX512CmpIntrinsic(F, Intrinsic::x86_avx512_mask_ucmp_d_128,
-                                       NewFn);
-    if (Name == "x86.avx512.mask.ucmp.q.128")
-      return UpgradeAVX512CmpIntrinsic(F, Intrinsic::x86_avx512_mask_ucmp_q_128,
-                                       NewFn);
-
     // frcz.ss/sd may need to have an argument dropped
     if (Name.startswith("x86.xop.vfrcz.ss") && F->arg_size() == 2) {
       F->setName(Name + ".old");
@@ -326,13 +229,13 @@ static bool UpgradeIntrinsicFunction1(Function *F, Function *&NewFn) {
 bool llvm::UpgradeIntrinsicFunction(Function *F, Function *&NewFn) {
   NewFn = nullptr;
   bool Upgraded = UpgradeIntrinsicFunction1(F, NewFn);
+  assert(F != NewFn && "Intrinsic function upgraded to the same function");
 
   // Upgrade intrinsic attributes.  This does not change the function.
   if (NewFn)
     F = NewFn;
-  if (unsigned id = F->getIntrinsicID())
-    F->setAttributes(Intrinsic::getAttributes(F->getContext(),
-                                              (Intrinsic::ID)id));
+  if (Intrinsic::ID id = F->getIntrinsicID())
+    F->setAttributes(Intrinsic::getAttributes(F->getContext(), id));
   return Upgraded;
 }
 
@@ -513,13 +416,14 @@ void llvm::UpgradeIntrinsicCall(CallInst *CI, Function *NewFn) {
         llvm_unreachable("Unknown condition");
 
       Function *VPCOM = Intrinsic::getDeclaration(F->getParent(), intID);
-      Rep = Builder.CreateCall3(VPCOM, CI->getArgOperand(0),
-                                CI->getArgOperand(1), Builder.getInt8(Imm));
+      Rep =
+          Builder.CreateCall(VPCOM, {CI->getArgOperand(0), CI->getArgOperand(1),
+                                     Builder.getInt8(Imm)});
     } else if (Name == "llvm.x86.sse42.crc32.64.8") {
       Function *CRC32 = Intrinsic::getDeclaration(F->getParent(),
                                                Intrinsic::x86_sse42_crc32_32_8);
       Value *Trunc0 = Builder.CreateTrunc(CI->getArgOperand(0), Type::getInt32Ty(C));
-      Rep = Builder.CreateCall2(CRC32, Trunc0, CI->getArgOperand(1));
+      Rep = Builder.CreateCall(CRC32, {Trunc0, CI->getArgOperand(1)});
       Rep = Builder.CreateZExt(Rep, CI->getType(), "");
     } else if (Name.startswith("llvm.x86.avx.vbroadcast")) {
       // Replace broadcasts with a series of insertelements.
@@ -528,7 +432,7 @@ void llvm::UpgradeIntrinsicCall(CallInst *CI, Function *NewFn) {
       unsigned EltNum = VecTy->getVectorNumElements();
       Value *Cast = Builder.CreateBitCast(CI->getArgOperand(0),
                                           EltTy->getPointerTo());
-      Value *Load = Builder.CreateLoad(Cast);
+      Value *Load = Builder.CreateLoad(EltTy, Cast);
       Type *I32Ty = Type::getInt32Ty(C);
       Rep = UndefValue::get(VecTy);
       for (unsigned I = 0; I < EltNum; ++I)
@@ -536,10 +440,10 @@ void llvm::UpgradeIntrinsicCall(CallInst *CI, Function *NewFn) {
                                           ConstantInt::get(I32Ty, I));
     } else if (Name == "llvm.x86.avx2.vbroadcasti128") {
       // Replace vbroadcasts with a vector shuffle.
-      Value *Op = Builder.CreatePointerCast(
-          CI->getArgOperand(0),
-          PointerType::getUnqual(VectorType::get(Type::getInt64Ty(C), 2)));
-      Value *Load = Builder.CreateLoad(Op);
+      Type *VT = VectorType::get(Type::getInt64Ty(C), 2);
+      Value *Op = Builder.CreatePointerCast(CI->getArgOperand(0),
+                                            PointerType::getUnqual(VT));
+      Value *Load = Builder.CreateLoad(VT, Op);
       const int Idxs[4] = { 0, 1, 0, 1 };
       Rep = Builder.CreateShuffleVector(Load, UndefValue::get(Load->getType()),
                                         Idxs);
@@ -727,29 +631,27 @@ void llvm::UpgradeIntrinsicCall(CallInst *CI, Function *NewFn) {
   case Intrinsic::cttz:
     assert(CI->getNumArgOperands() == 1 &&
            "Mismatch between function args and call args");
-    CI->replaceAllUsesWith(Builder.CreateCall2(NewFn, CI->getArgOperand(0),
-                                               Builder.getFalse(), Name));
+    CI->replaceAllUsesWith(Builder.CreateCall(
+        NewFn, {CI->getArgOperand(0), Builder.getFalse()}, Name));
     CI->eraseFromParent();
     return;
 
   case Intrinsic::objectsize:
-    CI->replaceAllUsesWith(Builder.CreateCall2(NewFn,
-                                               CI->getArgOperand(0),
-                                               CI->getArgOperand(1),
-                                               Name));
+    CI->replaceAllUsesWith(Builder.CreateCall(
+        NewFn, {CI->getArgOperand(0), CI->getArgOperand(1)}, Name));
     CI->eraseFromParent();
     return;
 
   case Intrinsic::ctpop: {
-    CI->replaceAllUsesWith(Builder.CreateCall(NewFn, CI->getArgOperand(0)));
+    CI->replaceAllUsesWith(Builder.CreateCall(NewFn, {CI->getArgOperand(0)}));
     CI->eraseFromParent();
     return;
   }
 
   case Intrinsic::x86_xop_vfrcz_ss:
   case Intrinsic::x86_xop_vfrcz_sd:
-    CI->replaceAllUsesWith(Builder.CreateCall(NewFn, CI->getArgOperand(1),
-                                              Name));
+    CI->replaceAllUsesWith(
+        Builder.CreateCall(NewFn, {CI->getArgOperand(1)}, Name));
     CI->eraseFromParent();
     return;
 
@@ -767,16 +669,12 @@ void llvm::UpgradeIntrinsicCall(CallInst *CI, Function *NewFn) {
     // Old intrinsic, add bitcasts
     Value *Arg1 = CI->getArgOperand(1);
 
-    Value *BC0 =
-      Builder.CreateBitCast(Arg0,
-                            VectorType::get(Type::getInt64Ty(C), 2),
-                            "cast");
-    Value *BC1 =
-      Builder.CreateBitCast(Arg1,
-                            VectorType::get(Type::getInt64Ty(C), 2),
-                            "cast");
+    Type *NewVecTy = VectorType::get(Type::getInt64Ty(C), 2);
+
+    Value *BC0 = Builder.CreateBitCast(Arg0, NewVecTy, "cast");
+    Value *BC1 = Builder.CreateBitCast(Arg1, NewVecTy, "cast");
 
-    CallInst* NewCall = Builder.CreateCall2(NewFn, BC0, BC1, Name);
+    CallInst *NewCall = Builder.CreateCall(NewFn, {BC0, BC1}, Name);
     CI->replaceAllUsesWith(NewCall);
     CI->eraseFromParent();
     return;
@@ -801,21 +699,6 @@ void llvm::UpgradeIntrinsicCall(CallInst *CI, Function *NewFn) {
     CI->eraseFromParent();
     return;
   }
-  case Intrinsic::x86_avx512_mask_cmp_ps_512:
-  case Intrinsic::x86_avx512_mask_cmp_pd_512: {
-    // Need to truncate the last argument from i32 to i8 -- this argument models
-    // an inherently 8-bit immediate operand to these x86 instructions.
-    SmallVector<Value *, 5> Args(CI->arg_operands().begin(),
-                                 CI->arg_operands().end());
-
-    // Replace the last argument with a trunc.
-    Args[2] = Builder.CreateTrunc(Args[2], Type::getInt8Ty(C), "trunc");
-
-    CallInst *NewCall = Builder.CreateCall(NewFn, Args);
-    CI->replaceAllUsesWith(NewCall);
-    CI->eraseFromParent();
-    return;
-  }
   }
 }
 
@@ -828,16 +711,14 @@ void llvm::UpgradeCallsToIntrinsic(Function* F) {
   // Upgrade the function and check if it is a totaly new function.
   Function *NewFn;
   if (UpgradeIntrinsicFunction(F, NewFn)) {
-    if (NewFn != F) {
-      // Replace all uses to the old function with the new one if necessary.
-      for (Value::user_iterator UI = F->user_begin(), UE = F->user_end();
-           UI != UE; ) {
-        if (CallInst *CI = dyn_cast<CallInst>(*UI++))
-          UpgradeIntrinsicCall(CI, NewFn);
-      }
-      // Remove old function, no longer used, from the module.
-      F->eraseFromParent();
+    // Replace all uses to the old function with the new one if necessary.
+    for (Value::user_iterator UI = F->user_begin(), UE = F->user_end();
+         UI != UE;) {
+      if (CallInst *CI = dyn_cast<CallInst>(*UI++))
+        UpgradeIntrinsicCall(CI, NewFn);
     }
+    // Remove old function, no longer used, from the module.
+    F->eraseFromParent();
   }
 }
 
diff --git a/lib/IR/BasicBlock.cpp b/lib/IR/BasicBlock.cpp
index 23ec7052954..b5671a66f04 100644
--- a/lib/IR/BasicBlock.cpp
+++ b/lib/IR/BasicBlock.cpp
@@ -117,6 +117,10 @@ const Module *BasicBlock::getModule() const {
   return getParent()->getParent();
 }
 
+Module *BasicBlock::getModule() {
+  return getParent()->getParent();
+}
+
 TerminatorInst *BasicBlock::getTerminator() {
   if (InstList.empty()) return nullptr;
   return dyn_cast<TerminatorInst>(&InstList.back());
@@ -159,48 +163,41 @@ CallInst *BasicBlock::getTerminatingMustTailCall() {
 }
 
 Instruction* BasicBlock::getFirstNonPHI() {
-  BasicBlock::iterator i = begin();
-  // All valid basic blocks should have a terminator,
-  // which is not a PHINode. If we have an invalid basic
-  // block we'll get an assertion failure when dereferencing
-  // a past-the-end iterator.
-  while (isa<PHINode>(i)) ++i;
-  return &*i;
+  for (Instruction &I : *this)
+    if (!isa<PHINode>(I))
+      return &I;
+  return nullptr;
 }
 
 Instruction* BasicBlock::getFirstNonPHIOrDbg() {
-  BasicBlock::iterator i = begin();
-  // All valid basic blocks should have a terminator,
-  // which is not a PHINode. If we have an invalid basic
-  // block we'll get an assertion failure when dereferencing
-  // a past-the-end iterator.
-  while (isa<PHINode>(i) || isa<DbgInfoIntrinsic>(i)) ++i;
-  return &*i;
+  for (Instruction &I : *this)
+    if (!isa<PHINode>(I) && !isa<DbgInfoIntrinsic>(I))
+      return &I;
+  return nullptr;
 }
 
 Instruction* BasicBlock::getFirstNonPHIOrDbgOrLifetime() {
-  // All valid basic blocks should have a terminator,
-  // which is not a PHINode. If we have an invalid basic
-  // block we'll get an assertion failure when dereferencing
-  // a past-the-end iterator.
-  BasicBlock::iterator i = begin();
-  for (;; ++i) {
-    if (isa<PHINode>(i) || isa<DbgInfoIntrinsic>(i))
+  for (Instruction &I : *this) {
+    if (isa<PHINode>(I) || isa<DbgInfoIntrinsic>(I))
       continue;
 
-    const IntrinsicInst *II = dyn_cast<IntrinsicInst>(i);
-    if (!II)
-      break;
-    if (II->getIntrinsicID() != Intrinsic::lifetime_start &&
-        II->getIntrinsicID() != Intrinsic::lifetime_end)
-      break;
+    if (auto *II = dyn_cast<IntrinsicInst>(&I))
+      if (II->getIntrinsicID() == Intrinsic::lifetime_start ||
+          II->getIntrinsicID() == Intrinsic::lifetime_end)
+        continue;
+
+    return &I;
   }
-  return &*i;
+  return nullptr;
 }
 
 BasicBlock::iterator BasicBlock::getFirstInsertionPt() {
-  iterator InsertPt = getFirstNonPHI();
-  if (isa<LandingPadInst>(InsertPt)) ++InsertPt;
+  Instruction *FirstNonPHI = getFirstNonPHI();
+  if (!FirstNonPHI)
+    return end();
+
+  iterator InsertPt = FirstNonPHI;
+  if (InsertPt->isEHPad()) ++InsertPt;
   return InsertPt;
 }
 
@@ -238,6 +235,14 @@ BasicBlock *BasicBlock::getUniquePredecessor() {
   return PredBB;
 }
 
+BasicBlock *BasicBlock::getSingleSuccessor() {
+  succ_iterator SI = succ_begin(this), E = succ_end(this);
+  if (SI == E) return nullptr; // no successors
+  BasicBlock *TheSucc = *SI;
+  ++SI;
+  return (SI == E) ? TheSucc : nullptr /* multiple successors */;
+}
+
 BasicBlock *BasicBlock::getUniqueSuccessor() {
   succ_iterator SI = succ_begin(this), E = succ_end(this);
   if (SI == E) return NULL; // No successors
@@ -328,6 +333,17 @@ void BasicBlock::removePredecessor(BasicBlock *Pred,
   }
 }
 
+bool BasicBlock::canSplitPredecessors() const {
+  const Instruction *FirstNonPHI = getFirstNonPHI();
+  if (isa<LandingPadInst>(FirstNonPHI))
+    return true;
+  // This is perhaps a little conservative because constructs like
+  // CleanupBlockInst are pretty easy to split.  However, SplitBlockPredecessors
+  // cannot handle such things just yet.
+  if (FirstNonPHI->isEHPad())
+    return false;
+  return true;
+}
 
 /// This splits a basic block into two at the specified
 /// instruction.  Note that all instructions BEFORE the specified iterator stay
@@ -350,12 +366,15 @@ BasicBlock *BasicBlock::splitBasicBlock(iterator I, const Twine &BBName) {
   BasicBlock *New = BasicBlock::Create(getContext(), BBName,
                                        getParent(), InsertBefore);
 
+  // Save DebugLoc of split point before invalidating iterator.
+  DebugLoc Loc = I->getDebugLoc();
   // Move all of the specified instructions from the original basic block into
   // the new basic block.
   New->getInstList().splice(New->end(), this->getInstList(), I, end());
 
   // Add a branch instruction to the newly formed basic block.
-  BranchInst::Create(New, this);
+  BranchInst *BI = BranchInst::Create(New, this);
+  BI->setDebugLoc(Loc);
 
   // Now we must loop through all of the successors of the New block (which
   // _were_ the successors of the 'this' block), and update any PHI nodes in
diff --git a/lib/IR/CMakeLists.txt b/lib/IR/CMakeLists.txt
index 213c8c11d8d..d2dd5396c9a 100644
--- a/lib/IR/CMakeLists.txt
+++ b/lib/IR/CMakeLists.txt
@@ -35,6 +35,7 @@ add_llvm_library(LLVMCore
   MetadataTracking.cpp
   Module.cpp
   NaClAtomicIntrinsics.cpp
+  Operator.cpp
   Pass.cpp
   PassManager.cpp
   PassRegistry.cpp
diff --git a/lib/IR/ConstantFold.cpp b/lib/IR/ConstantFold.cpp
index d3caf04489d..46bb20e0d1b 100644
--- a/lib/IR/ConstantFold.cpp
+++ b/lib/IR/ConstantFold.cpp
@@ -632,8 +632,8 @@ Constant *llvm::ConstantFoldCastInstruction(unsigned opc, Constant *V,
     if (ConstantExpr *CE = dyn_cast<ConstantExpr>(V))
       if (CE->getOpcode() == Instruction::GetElementPtr &&
           CE->getOperand(0)->isNullValue()) {
-        Type *Ty =
-          cast<PointerType>(CE->getOperand(0)->getType())->getElementType();
+        GEPOperator *GEPO = cast<GEPOperator>(CE);
+        Type *Ty = GEPO->getSourceElementType();
         if (CE->getNumOperands() == 2) {
           // Handle a sizeof-like expression.
           Constant *Idx = CE->getOperand(1);
@@ -789,11 +789,10 @@ Constant *llvm::ConstantFoldExtractElementInstruction(Constant *Val,
     return UndefValue::get(Val->getType()->getVectorElementType());
 
   if (ConstantInt *CIdx = dyn_cast<ConstantInt>(Idx)) {
-    uint64_t Index = CIdx->getZExtValue();
     // ee({w,x,y,z}, wrong_value) -> undef
-    if (Index >= Val->getType()->getVectorNumElements())
+    if (CIdx->uge(Val->getType()->getVectorNumElements()))
       return UndefValue::get(Val->getType()->getVectorElementType());
-    return Val->getAggregateElement(Index);
+    return Val->getAggregateElement(CIdx->getZExtValue());
   }
   return nullptr;
 }
@@ -801,23 +800,30 @@ Constant *llvm::ConstantFoldExtractElementInstruction(Constant *Val,
 Constant *llvm::ConstantFoldInsertElementInstruction(Constant *Val,
                                                      Constant *Elt,
                                                      Constant *Idx) {
+  if (isa<UndefValue>(Idx))
+    return UndefValue::get(Val->getType());
+
   ConstantInt *CIdx = dyn_cast<ConstantInt>(Idx);
   if (!CIdx) return nullptr;
-  const APInt &IdxVal = CIdx->getValue();
-  
+
+  unsigned NumElts = Val->getType()->getVectorNumElements();
+  if (CIdx->uge(NumElts))
+    return UndefValue::get(Val->getType());
+
   SmallVector<Constant*, 16> Result;
-  Type *Ty = IntegerType::get(Val->getContext(), 32);
-  for (unsigned i = 0, e = Val->getType()->getVectorNumElements(); i != e; ++i){
+  Result.reserve(NumElts);
+  auto *Ty = Type::getInt32Ty(Val->getContext());
+  uint64_t IdxVal = CIdx->getZExtValue();
+  for (unsigned i = 0; i != NumElts; ++i) {    
     if (i == IdxVal) {
       Result.push_back(Elt);
       continue;
     }
     
-    Constant *C =
-      ConstantExpr::getExtractElement(Val, ConstantInt::get(Ty, i));
+    Constant *C = ConstantExpr::getExtractElement(Val, ConstantInt::get(Ty, i));
     Result.push_back(C);
   }
-  
+
   return ConstantVector::get(Result);
 }
 
@@ -1379,7 +1385,7 @@ static ICmpInst::Predicate areGlobalsPotentiallyEqual(const GlobalValue *GV1,
     if (GV->hasExternalWeakLinkage() || GV->hasWeakAnyLinkage())
       return true;
     if (const auto *GVar = dyn_cast<GlobalVariable>(GV)) {
-      Type *Ty = GVar->getType()->getPointerElementType();
+      Type *Ty = GVar->getValueType();
       // A global with opaque type might end up being zero sized.
       if (!Ty->isSized())
         return true;
@@ -2022,7 +2028,7 @@ static bool isIndexInRangeOfSequentialType(const SequentialType *STy,
 }
 
 template<typename IndexTy>
-static Constant *ConstantFoldGetElementPtrImpl(Constant *C,
+static Constant *ConstantFoldGetElementPtrImpl(Type *PointeeTy, Constant *C,
                                                bool inBounds,
                                                ArrayRef<IndexTy> Idxs) {
   if (Idxs.empty()) return C;
@@ -2157,11 +2163,11 @@ static Constant *ConstantFoldGetElementPtrImpl(Constant *C,
   // Check to see if any array indices are not within the corresponding
   // notional array or vector bounds. If so, try to determine if they can be
   // factored out into preceding dimensions.
-  bool Unknown = false;
   SmallVector<Constant *, 8> NewIdxs;
-  Type *Ty = C->getType();
-  Type *Prev = nullptr;
-  for (unsigned i = 0, e = Idxs.size(); i != e;
+  Type *Ty = PointeeTy;
+  Type *Prev = C->getType();
+  bool Unknown = !isa<ConstantInt>(Idxs[0]);
+  for (unsigned i = 1, e = Idxs.size(); i != e;
        Prev = Ty, Ty = cast<CompositeType>(Ty)->getTypeAtIndex(Idxs[i]), ++i) {
     if (ConstantInt *CI = dyn_cast<ConstantInt>(Idxs[i])) {
       if (isa<ArrayType>(Ty) || isa<VectorType>(Ty))
@@ -2215,7 +2221,7 @@ static Constant *ConstantFoldGetElementPtrImpl(Constant *C,
   if (!NewIdxs.empty()) {
     for (unsigned i = 0, e = Idxs.size(); i != e; ++i)
       if (!NewIdxs[i]) NewIdxs[i] = cast<Constant>(Idxs[i]);
-    return ConstantExpr::getGetElementPtr(nullptr, C, NewIdxs, inBounds);
+    return ConstantExpr::getGetElementPtr(PointeeTy, C, NewIdxs, inBounds);
   }
 
   // If all indices are known integers and normalized, we can do a simple
@@ -2223,7 +2229,7 @@ static Constant *ConstantFoldGetElementPtrImpl(Constant *C,
   if (!Unknown && !inBounds)
     if (auto *GV = dyn_cast<GlobalVariable>(C))
       if (!GV->hasExternalWeakLinkage() && isInBoundsIndices(Idxs))
-        return ConstantExpr::getInBoundsGetElementPtr(nullptr, C, Idxs);
+        return ConstantExpr::getInBoundsGetElementPtr(PointeeTy, C, Idxs);
 
   return nullptr;
 }
@@ -2231,11 +2237,27 @@ static Constant *ConstantFoldGetElementPtrImpl(Constant *C,
 Constant *llvm::ConstantFoldGetElementPtr(Constant *C,
                                           bool inBounds,
                                           ArrayRef<Constant *> Idxs) {
-  return ConstantFoldGetElementPtrImpl(C, inBounds, Idxs);
+  return ConstantFoldGetElementPtrImpl(
+      cast<PointerType>(C->getType()->getScalarType())->getElementType(), C,
+      inBounds, Idxs);
 }
 
 Constant *llvm::ConstantFoldGetElementPtr(Constant *C,
                                           bool inBounds,
                                           ArrayRef<Value *> Idxs) {
-  return ConstantFoldGetElementPtrImpl(C, inBounds, Idxs);
+  return ConstantFoldGetElementPtrImpl(
+      cast<PointerType>(C->getType()->getScalarType())->getElementType(), C,
+      inBounds, Idxs);
+}
+
+Constant *llvm::ConstantFoldGetElementPtr(Type *Ty, Constant *C,
+                                          bool inBounds,
+                                          ArrayRef<Constant *> Idxs) {
+  return ConstantFoldGetElementPtrImpl(Ty, C, inBounds, Idxs);
+}
+
+Constant *llvm::ConstantFoldGetElementPtr(Type *Ty, Constant *C,
+                                          bool inBounds,
+                                          ArrayRef<Value *> Idxs) {
+  return ConstantFoldGetElementPtrImpl(Ty, C, inBounds, Idxs);
 }
diff --git a/lib/IR/ConstantFold.h b/lib/IR/ConstantFold.h
index a516abe024e..42a9c6ba908 100644
--- a/lib/IR/ConstantFold.h
+++ b/lib/IR/ConstantFold.h
@@ -51,6 +51,10 @@ namespace llvm {
                                       ArrayRef<Constant *> Idxs);
   Constant *ConstantFoldGetElementPtr(Constant *C, bool inBounds,
                                       ArrayRef<Value *> Idxs);
+  Constant *ConstantFoldGetElementPtr(Type *Ty, Constant *C, bool inBounds,
+                                      ArrayRef<Constant *> Idxs);
+  Constant *ConstantFoldGetElementPtr(Type *Ty, Constant *C, bool inBounds,
+                                      ArrayRef<Value *> Idxs);
 } // End llvm namespace
 
 #endif
diff --git a/lib/IR/Constants.cpp b/lib/IR/Constants.cpp
index 3f8d1f1ca6a..308e6bde3d1 100644
--- a/lib/IR/Constants.cpp
+++ b/lib/IR/Constants.cpp
@@ -276,8 +276,19 @@ Constant *Constant::getAggregateElement(Constant *Elt) const {
   return nullptr;
 }
 
+void Constant::destroyConstant() {
+  /// First call destroyConstantImpl on the subclass.  This gives the subclass
+  /// a chance to remove the constant from any maps/pools it's contained in.
+  switch (getValueID()) {
+  default:
+    llvm_unreachable("Not a constant!");
+#define HANDLE_CONSTANT(Name)                                                  \
+  case Value::Name##Val:                                                       \
+    cast<Name>(this)->destroyConstantImpl();                                   \
+    break;
+#include "llvm/IR/Value.def"
+  }
 
-void Constant::destroyConstantImpl() {
   // When a Constant is destroyed, there may be lingering
   // references to the constant by other constants in the constant pool.  These
   // constants are implicitly dependent on the module that is being deleted,
@@ -287,11 +298,11 @@ void Constant::destroyConstantImpl() {
   //
   while (!use_empty()) {
     Value *V = user_back();
-#ifndef NDEBUG      // Only in -g mode...
+#ifndef NDEBUG // Only in -g mode...
     if (!isa<Constant>(V)) {
       dbgs() << "While deleting: " << *this
-             << "\n\nUse still stuck around after Def is destroyed: "
-             << *V << "\n\n";
+             << "\n\nUse still stuck around after Def is destroyed: " << *V
+             << "\n\n";
     }
 #endif
     assert(isa<Constant>(V) && "References remain to Constant being destroyed");
@@ -608,6 +619,11 @@ ConstantInt *ConstantInt::get(IntegerType* Ty, StringRef Str,
   return get(Ty->getContext(), APInt(Ty->getBitWidth(), Str, radix));
 }
 
+/// Remove the constant from the constant table.
+void ConstantInt::destroyConstantImpl() {
+  llvm_unreachable("You can't ConstantInt->destroyConstantImpl()!");
+}
+
 //===----------------------------------------------------------------------===//
 //                                ConstantFP
 //===----------------------------------------------------------------------===//
@@ -663,6 +679,17 @@ Constant *ConstantFP::get(Type *Ty, StringRef Str) {
   return C; 
 }
 
+Constant *ConstantFP::getNaN(Type *Ty, bool Negative, unsigned Type) {
+  const fltSemantics &Semantics = *TypeToFloatSemantics(Ty->getScalarType());
+  APFloat NaN = APFloat::getNaN(Semantics, Negative, Type);
+  Constant *C = get(Ty->getContext(), NaN);
+
+  if (VectorType *VTy = dyn_cast<VectorType>(Ty))
+    return ConstantVector::getSplat(VTy->getNumElements(), C);
+
+  return C;
+}
+
 Constant *ConstantFP::getNegativeZero(Type *Ty) {
   const fltSemantics &Semantics = *TypeToFloatSemantics(Ty->getScalarType());
   APFloat NegZero = APFloat::getZero(Semantics, /*Negative=*/true);
@@ -732,6 +759,11 @@ bool ConstantFP::isExactlyValue(const APFloat &V) const {
   return Val.bitwiseIsEqual(V);
 }
 
+/// Remove the constant from the constant table.
+void ConstantFP::destroyConstantImpl() {
+  llvm_unreachable("You can't ConstantInt->destroyConstantImpl()!");
+}
+
 //===----------------------------------------------------------------------===//
 //                   ConstantAggregateZero Implementation
 //===----------------------------------------------------------------------===//
@@ -1355,16 +1387,14 @@ ConstantAggregateZero *ConstantAggregateZero::get(Type *Ty) {
 
 /// destroyConstant - Remove the constant from the constant table.
 ///
-void ConstantAggregateZero::destroyConstant() {
+void ConstantAggregateZero::destroyConstantImpl() {
   getContext().pImpl->CAZConstants.erase(getType());
-  destroyConstantImpl();
 }
 
 /// destroyConstant - Remove the constant from the constant table...
 ///
-void ConstantArray::destroyConstant() {
+void ConstantArray::destroyConstantImpl() {
   getType()->getContext().pImpl->ArrayConstants.remove(this);
-  destroyConstantImpl();
 }
 
 
@@ -1373,16 +1403,14 @@ void ConstantArray::destroyConstant() {
 
 // destroyConstant - Remove the constant from the constant table...
 //
-void ConstantStruct::destroyConstant() {
+void ConstantStruct::destroyConstantImpl() {
   getType()->getContext().pImpl->StructConstants.remove(this);
-  destroyConstantImpl();
 }
 
 // destroyConstant - Remove the constant from the constant table...
 //
-void ConstantVector::destroyConstant() {
+void ConstantVector::destroyConstantImpl() {
   getType()->getContext().pImpl->VectorConstants.remove(this);
-  destroyConstantImpl();
 }
 
 /// getSplatValue - If this is a splat vector constant, meaning that all of
@@ -1421,7 +1449,6 @@ const APInt &Constant::getUniqueInteger() const {
   return cast<ConstantInt>(C)->getValue();
 }
 
-
 //---- ConstantPointerNull::get() implementation.
 //
 
@@ -1435,10 +1462,8 @@ ConstantPointerNull *ConstantPointerNull::get(PointerType *Ty) {
 
 // destroyConstant - Remove the constant from the constant table...
 //
-void ConstantPointerNull::destroyConstant() {
+void ConstantPointerNull::destroyConstantImpl() {
   getContext().pImpl->CPNConstants.erase(getType());
-  // Free the constant and any dangling references to it.
-  destroyConstantImpl();
 }
 
 
@@ -1455,10 +1480,9 @@ UndefValue *UndefValue::get(Type *Ty) {
 
 // destroyConstant - Remove the constant from the constant table.
 //
-void UndefValue::destroyConstant() {
+void UndefValue::destroyConstantImpl() {
   // Free the constant and any dangling references to it.
   getContext().pImpl->UVConstants.erase(getType());
-  destroyConstantImpl();
 }
 
 //---- BlockAddress::get() implementation.
@@ -1501,14 +1525,13 @@ BlockAddress *BlockAddress::lookup(const BasicBlock *BB) {
 
 // destroyConstant - Remove the constant from the constant table.
 //
-void BlockAddress::destroyConstant() {
+void BlockAddress::destroyConstantImpl() {
   getFunction()->getType()->getContext().pImpl
     ->BlockAddresses.erase(std::make_pair(getFunction(), getBasicBlock()));
   getBasicBlock()->AdjustBlockAddressRefCount(-1);
-  destroyConstantImpl();
 }
 
-void BlockAddress::replaceUsesOfWithOnConstant(Value *From, Value *To, Use *U) {
+Value *BlockAddress::handleOperandChangeImpl(Value *From, Value *To, Use *U) {
   // This could be replacing either the Basic Block or the Function.  In either
   // case, we have to remove the map entry.
   Function *NewF = getFunction();
@@ -1523,10 +1546,8 @@ void BlockAddress::replaceUsesOfWithOnConstant(Value *From, Value *To, Use *U) {
   // and return early.
   BlockAddress *&NewBA =
     getContext().pImpl->BlockAddresses[std::make_pair(NewF, NewBB)];
-  if (NewBA) {
-    replaceUsesOfWithOnConstantImpl(NewBA);
-    return;
-  }
+  if (NewBA)
+    return NewBA;
 
   getBasicBlock()->AdjustBlockAddressRefCount(-1);
 
@@ -1538,6 +1559,10 @@ void BlockAddress::replaceUsesOfWithOnConstant(Value *From, Value *To, Use *U) {
   setOperand(0, NewF);
   setOperand(1, NewBB);
   getBasicBlock()->AdjustBlockAddressRefCount(1);
+
+  // If we just want to keep the existing value, then return null.
+  // Callers know that this means we shouldn't delete this value.
+  return nullptr;
 }
 
 //---- ConstantExpr::get() implementations.
@@ -2004,14 +2029,16 @@ Constant *ConstantExpr::getSelect(Constant *C, Constant *V1, Constant *V2,
 Constant *ConstantExpr::getGetElementPtr(Type *Ty, Constant *C,
                                          ArrayRef<Value *> Idxs, bool InBounds,
                                          Type *OnlyIfReducedTy) {
-  if (Constant *FC = ConstantFoldGetElementPtr(C, InBounds, Idxs))
-    return FC;          // Fold a few common cases.
-
   if (!Ty)
     Ty = cast<PointerType>(C->getType()->getScalarType())->getElementType();
   else
-    assert(Ty ==
-           cast<PointerType>(C->getType()->getScalarType())->getElementType());
+    assert(
+        Ty ==
+        cast<PointerType>(C->getType()->getScalarType())->getContainedType(0u));
+
+  if (Constant *FC = ConstantFoldGetElementPtr(Ty, C, InBounds, Idxs))
+    return FC;          // Fold a few common cases.
+
   // Get the result type of the getelementptr!
   Type *DestTy = GetElementPtrInst::getIndexedType(Ty, Idxs);
   assert(DestTy && "GEP indices invalid!");
@@ -2037,7 +2064,8 @@ Constant *ConstantExpr::getGetElementPtr(Type *Ty, Constant *C,
     ArgVec.push_back(cast<Constant>(Idxs[i]));
   }
   const ConstantExprKeyType Key(Instruction::GetElementPtr, ArgVec, 0,
-                                InBounds ? GEPOperator::IsInBounds : 0);
+                                InBounds ? GEPOperator::IsInBounds : 0, None,
+                                Ty);
 
   LLVMContextImpl *pImpl = C->getContext().pImpl;
   return pImpl->ExprConstants.getOrCreate(ReqTy, Key);
@@ -2358,28 +2386,31 @@ Constant *ConstantExpr::getBinOpAbsorber(unsigned Opcode, Type *Ty) {
 
 // destroyConstant - Remove the constant from the constant table...
 //
-void ConstantExpr::destroyConstant() {
+void ConstantExpr::destroyConstantImpl() {
   getType()->getContext().pImpl->ExprConstants.remove(this);
-  destroyConstantImpl();
 }
 
 const char *ConstantExpr::getOpcodeName() const {
   return Instruction::getOpcodeName(getOpcode());
 }
 
-
-
-GetElementPtrConstantExpr::
-GetElementPtrConstantExpr(Constant *C, ArrayRef<Constant*> IdxList,
-                          Type *DestTy)
-  : ConstantExpr(DestTy, Instruction::GetElementPtr,
-                 OperandTraits<GetElementPtrConstantExpr>::op_end(this)
-                 - (IdxList.size()+1), IdxList.size()+1) {
-  OperandList[0] = C;
+GetElementPtrConstantExpr::GetElementPtrConstantExpr(
+    Type *SrcElementTy, Constant *C, ArrayRef<Constant *> IdxList, Type *DestTy)
+    : ConstantExpr(DestTy, Instruction::GetElementPtr,
+                   OperandTraits<GetElementPtrConstantExpr>::op_end(this) -
+                       (IdxList.size() + 1),
+                   IdxList.size() + 1),
+      SrcElementTy(SrcElementTy) {
+  Op<0>() = C;
+  Use *OperandList = getOperandList();
   for (unsigned i = 0, E = IdxList.size(); i != E; ++i)
     OperandList[i+1] = IdxList[i];
 }
 
+Type *GetElementPtrConstantExpr::getSourceElementType() const {
+  return SrcElementTy;
+}
+
 //===----------------------------------------------------------------------===//
 //                       ConstantData* implementations
 
@@ -2478,7 +2509,7 @@ Constant *ConstantDataSequential::getImpl(StringRef Elements, Type *Ty) {
   return *Entry = new ConstantDataVector(Ty, Slot.first().data());
 }
 
-void ConstantDataSequential::destroyConstant() {
+void ConstantDataSequential::destroyConstantImpl() {
   // Remove the constant from the StringMap.
   StringMap<ConstantDataSequential*> &CDSConstants = 
     getType()->getContext().pImpl->CDSConstants;
@@ -2513,9 +2544,6 @@ void ConstantDataSequential::destroyConstant() {
   // If we were part of a list, make sure that we don't delete the list that is
   // still owned by the uniquing map.
   Next = nullptr;
-
-  // Finally, actually delete it.
-  destroyConstantImpl();
 }
 
 /// get() constructors - Return a constant with array type with an element
@@ -2796,20 +2824,36 @@ Constant *ConstantDataVector::getSplatValue() const {
 }
 
 //===----------------------------------------------------------------------===//
-//                replaceUsesOfWithOnConstant implementations
+//                handleOperandChange implementations
 
-/// replaceUsesOfWithOnConstant - Update this constant array to change uses of
+/// Update this constant array to change uses of
 /// 'From' to be uses of 'To'.  This must update the uniquing data structures
 /// etc.
 ///
 /// Note that we intentionally replace all uses of From with To here.  Consider
 /// a large array that uses 'From' 1000 times.  By handling this case all here,
-/// ConstantArray::replaceUsesOfWithOnConstant is only invoked once, and that
+/// ConstantArray::handleOperandChange is only invoked once, and that
 /// single invocation handles all 1000 uses.  Handling them one at a time would
 /// work, but would be really slow because it would have to unique each updated
 /// array instance.
 ///
-void Constant::replaceUsesOfWithOnConstantImpl(Constant *Replacement) {
+void Constant::handleOperandChange(Value *From, Value *To, Use *U) {
+  Value *Replacement = nullptr;
+  switch (getValueID()) {
+  default:
+    llvm_unreachable("Not a constant!");
+#define HANDLE_CONSTANT(Name)                                                  \
+  case Value::Name##Val:                                                       \
+    Replacement = cast<Name>(this)->handleOperandChangeImpl(From, To, U);      \
+    break;
+#include "llvm/IR/Value.def"
+  }
+
+  // If handleOperandChangeImpl returned nullptr, then it handled
+  // replacing itself and we don't want to delete or replace anything else here.
+  if (!Replacement)
+    return;
+
   // I do need to replace this with an existing value.
   assert(Replacement != this && "I didn't contain From!");
 
@@ -2820,8 +2864,34 @@ void Constant::replaceUsesOfWithOnConstantImpl(Constant *Replacement) {
   destroyConstant();
 }
 
-void ConstantArray::replaceUsesOfWithOnConstant(Value *From, Value *To,
-                                                Use *U) {
+Value *ConstantInt::handleOperandChangeImpl(Value *From, Value *To, Use *U) {
+  llvm_unreachable("Unsupported class for handleOperandChange()!");
+}
+
+Value *ConstantFP::handleOperandChangeImpl(Value *From, Value *To, Use *U) {
+  llvm_unreachable("Unsupported class for handleOperandChange()!");
+}
+
+Value *UndefValue::handleOperandChangeImpl(Value *From, Value *To, Use *U) {
+  llvm_unreachable("Unsupported class for handleOperandChange()!");
+}
+
+Value *ConstantPointerNull::handleOperandChangeImpl(Value *From, Value *To,
+                                                    Use *U) {
+  llvm_unreachable("Unsupported class for handleOperandChange()!");
+}
+
+Value *ConstantAggregateZero::handleOperandChangeImpl(Value *From, Value *To,
+                                                      Use *U) {
+  llvm_unreachable("Unsupported class for handleOperandChange()!");
+}
+
+Value *ConstantDataSequential::handleOperandChangeImpl(Value *From, Value *To,
+                                                       Use *U) {
+  llvm_unreachable("Unsupported class for handleOperandChange()!");
+}
+
+Value *ConstantArray::handleOperandChangeImpl(Value *From, Value *To, Use *U) {
   assert(isa<Constant>(To) && "Cannot make Constant refer to non-constant!");
   Constant *ToC = cast<Constant>(To);
 
@@ -2834,6 +2904,7 @@ void ConstantArray::replaceUsesOfWithOnConstant(Value *From, Value *To,
 
   // Keep track of whether all the values in the array are "ToC".
   bool AllSame = true;
+  Use *OperandList = getOperandList();
   for (Use *O = OperandList, *E = OperandList+getNumOperands(); O != E; ++O) {
     Constant *Val = cast<Constant>(O->get());
     if (Val == From) {
@@ -2844,32 +2915,26 @@ void ConstantArray::replaceUsesOfWithOnConstant(Value *From, Value *To,
     AllSame &= Val == ToC;
   }
 
-  if (AllSame && ToC->isNullValue()) {
-    replaceUsesOfWithOnConstantImpl(ConstantAggregateZero::get(getType()));
-    return;
-  }
-  if (AllSame && isa<UndefValue>(ToC)) {
-    replaceUsesOfWithOnConstantImpl(UndefValue::get(getType()));
-    return;
-  }
+  if (AllSame && ToC->isNullValue())
+    return ConstantAggregateZero::get(getType());
+
+  if (AllSame && isa<UndefValue>(ToC))
+    return UndefValue::get(getType());
 
   // Check for any other type of constant-folding.
-  if (Constant *C = getImpl(getType(), Values)) {
-    replaceUsesOfWithOnConstantImpl(C);
-    return;
-  }
+  if (Constant *C = getImpl(getType(), Values))
+    return C;
 
   // Update to the new value.
-  if (Constant *C = getContext().pImpl->ArrayConstants.replaceOperandsInPlace(
-          Values, this, From, ToC, NumUpdated, U - OperandList))
-    replaceUsesOfWithOnConstantImpl(C);
+  return getContext().pImpl->ArrayConstants.replaceOperandsInPlace(
+      Values, this, From, ToC, NumUpdated, U - OperandList);
 }
 
-void ConstantStruct::replaceUsesOfWithOnConstant(Value *From, Value *To,
-                                                 Use *U) {
+Value *ConstantStruct::handleOperandChangeImpl(Value *From, Value *To, Use *U) {
   assert(isa<Constant>(To) && "Cannot make Constant refer to non-constant!");
   Constant *ToC = cast<Constant>(To);
 
+  Use *OperandList = getOperandList();
   unsigned OperandToUpdate = U-OperandList;
   assert(getOperand(OperandToUpdate) == From && "ReplaceAllUsesWith broken!");
 
@@ -2900,23 +2965,18 @@ void ConstantStruct::replaceUsesOfWithOnConstant(Value *From, Value *To,
   }
   Values[OperandToUpdate] = ToC;
 
-  if (isAllZeros) {
-    replaceUsesOfWithOnConstantImpl(ConstantAggregateZero::get(getType()));
-    return;
-  }
-  if (isAllUndef) {
-    replaceUsesOfWithOnConstantImpl(UndefValue::get(getType()));
-    return;
-  }
+  if (isAllZeros)
+    return ConstantAggregateZero::get(getType());
+
+  if (isAllUndef)
+    return UndefValue::get(getType());
 
   // Update to the new value.
-  if (Constant *C = getContext().pImpl->StructConstants.replaceOperandsInPlace(
-          Values, this, From, ToC))
-    replaceUsesOfWithOnConstantImpl(C);
+  return getContext().pImpl->StructConstants.replaceOperandsInPlace(
+      Values, this, From, ToC);
 }
 
-void ConstantVector::replaceUsesOfWithOnConstant(Value *From, Value *To,
-                                                 Use *U) {
+Value *ConstantVector::handleOperandChangeImpl(Value *From, Value *To, Use *U) {
   assert(isa<Constant>(To) && "Cannot make Constant refer to non-constant!");
   Constant *ToC = cast<Constant>(To);
 
@@ -2932,19 +2992,16 @@ void ConstantVector::replaceUsesOfWithOnConstant(Value *From, Value *To,
     Values.push_back(Val);
   }
 
-  if (Constant *C = getImpl(Values)) {
-    replaceUsesOfWithOnConstantImpl(C);
-    return;
-  }
+  if (Constant *C = getImpl(Values))
+    return C;
 
   // Update to the new value.
-  if (Constant *C = getContext().pImpl->VectorConstants.replaceOperandsInPlace(
-          Values, this, From, ToC, NumUpdated, U - OperandList))
-    replaceUsesOfWithOnConstantImpl(C);
+  Use *OperandList = getOperandList();
+  return getContext().pImpl->VectorConstants.replaceOperandsInPlace(
+      Values, this, From, ToC, NumUpdated, U - OperandList);
 }
 
-void ConstantExpr::replaceUsesOfWithOnConstant(Value *From, Value *ToV,
-                                               Use *U) {
+Value *ConstantExpr::handleOperandChangeImpl(Value *From, Value *ToV, Use *U) {
   assert(isa<Constant>(ToV) && "Cannot make Constant refer to non-constant!");
   Constant *To = cast<Constant>(ToV);
 
@@ -2960,15 +3017,13 @@ void ConstantExpr::replaceUsesOfWithOnConstant(Value *From, Value *ToV,
   }
   assert(NumUpdated && "I didn't contain From!");
 
-  if (Constant *C = getWithOperands(NewOps, getType(), true)) {
-    replaceUsesOfWithOnConstantImpl(C);
-    return;
-  }
+  if (Constant *C = getWithOperands(NewOps, getType(), true))
+    return C;
 
   // Update to the new value.
-  if (Constant *C = getContext().pImpl->ExprConstants.replaceOperandsInPlace(
-          NewOps, this, From, To, NumUpdated, U - OperandList))
-    replaceUsesOfWithOnConstantImpl(C);
+  Use *OperandList = getOperandList();
+  return getContext().pImpl->ExprConstants.replaceOperandsInPlace(
+      NewOps, this, From, To, NumUpdated, U - OperandList);
 }
 
 Instruction *ConstantExpr::getAsInstruction() {
diff --git a/lib/IR/ConstantsContext.h b/lib/IR/ConstantsContext.h
index c1dfcf13b2d..33e89515b26 100644
--- a/lib/IR/ConstantsContext.h
+++ b/lib/IR/ConstantsContext.h
@@ -100,7 +100,7 @@ class ExtractElementConstantExpr : public ConstantExpr {
     return User::operator new(s, 2);
   }
   ExtractElementConstantExpr(Constant *C1, Constant *C2)
-    : ConstantExpr(cast<VectorType>(C1->getType())->getElementType(), 
+    : ConstantExpr(cast<VectorType>(C1->getType())->getElementType(),
                    Instruction::ExtractElement, &Op<0>(), 2) {
     Op<0>() = C1;
     Op<1>() = C2;
@@ -121,7 +121,7 @@ class InsertElementConstantExpr : public ConstantExpr {
     return User::operator new(s, 3);
   }
   InsertElementConstantExpr(Constant *C1, Constant *C2, Constant *C3)
-    : ConstantExpr(C1->getType(), Instruction::InsertElement, 
+    : ConstantExpr(C1->getType(), Instruction::InsertElement,
                    &Op<0>(), 3) {
     Op<0>() = C1;
     Op<1>() = C2;
@@ -146,7 +146,7 @@ class ShuffleVectorConstantExpr : public ConstantExpr {
   : ConstantExpr(VectorType::get(
                    cast<VectorType>(C1->getType())->getElementType(),
                    cast<VectorType>(C3->getType())->getNumElements()),
-                 Instruction::ShuffleVector, 
+                 Instruction::ShuffleVector,
                  &Op<0>(), 3) {
     Op<0>() = C1;
     Op<1>() = C2;
@@ -207,23 +207,32 @@ class InsertValueConstantExpr : public ConstantExpr {
   DECLARE_TRANSPARENT_OPERAND_ACCESSORS(Value);
 };
 
-
 /// GetElementPtrConstantExpr - This class is private to Constants.cpp, and is
 /// used behind the scenes to implement getelementpr constant exprs.
 class GetElementPtrConstantExpr : public ConstantExpr {
+  Type *SrcElementTy;
   void anchor() override;
-  GetElementPtrConstantExpr(Constant *C, ArrayRef<Constant*> IdxList,
-                            Type *DestTy);
+  GetElementPtrConstantExpr(Type *SrcElementTy, Constant *C,
+                            ArrayRef<Constant *> IdxList, Type *DestTy);
+
 public:
   static GetElementPtrConstantExpr *Create(Constant *C,
                                            ArrayRef<Constant*> IdxList,
                                            Type *DestTy,
                                            unsigned Flags) {
-    GetElementPtrConstantExpr *Result =
-      new(IdxList.size() + 1) GetElementPtrConstantExpr(C, IdxList, DestTy);
+    return Create(
+        cast<PointerType>(C->getType()->getScalarType())->getElementType(), C,
+        IdxList, DestTy, Flags);
+  }
+  static GetElementPtrConstantExpr *Create(Type *SrcElementTy, Constant *C,
+                                           ArrayRef<Constant *> IdxList,
+                                           Type *DestTy, unsigned Flags) {
+    GetElementPtrConstantExpr *Result = new (IdxList.size() + 1)
+        GetElementPtrConstantExpr(SrcElementTy, C, IdxList, DestTy);
     Result->SubclassOptionalData = Flags;
     return Result;
   }
+  Type *getSourceElementType() const;
   /// Transparently provide more efficient getOperand methods.
   DECLARE_TRANSPARENT_OPERAND_ACCESSORS(Value);
 };
@@ -251,65 +260,54 @@ class CompareConstantExpr : public ConstantExpr {
 };
 
 template <>
-struct OperandTraits<UnaryConstantExpr> :
-  public FixedNumOperandTraits<UnaryConstantExpr, 1> {
-};
+struct OperandTraits<UnaryConstantExpr>
+    : public FixedNumOperandTraits<UnaryConstantExpr, 1> {};
 DEFINE_TRANSPARENT_OPERAND_ACCESSORS(UnaryConstantExpr, Value)
 
 template <>
-struct OperandTraits<BinaryConstantExpr> :
-  public FixedNumOperandTraits<BinaryConstantExpr, 2> {
-};
+struct OperandTraits<BinaryConstantExpr>
+    : public FixedNumOperandTraits<BinaryConstantExpr, 2> {};
 DEFINE_TRANSPARENT_OPERAND_ACCESSORS(BinaryConstantExpr, Value)
 
 template <>
-struct OperandTraits<SelectConstantExpr> :
-  public FixedNumOperandTraits<SelectConstantExpr, 3> {
-};
+struct OperandTraits<SelectConstantExpr>
+    : public FixedNumOperandTraits<SelectConstantExpr, 3> {};
 DEFINE_TRANSPARENT_OPERAND_ACCESSORS(SelectConstantExpr, Value)
 
 template <>
-struct OperandTraits<ExtractElementConstantExpr> :
-  public FixedNumOperandTraits<ExtractElementConstantExpr, 2> {
-};
+struct OperandTraits<ExtractElementConstantExpr>
+    : public FixedNumOperandTraits<ExtractElementConstantExpr, 2> {};
 DEFINE_TRANSPARENT_OPERAND_ACCESSORS(ExtractElementConstantExpr, Value)
 
 template <>
-struct OperandTraits<InsertElementConstantExpr> :
-  public FixedNumOperandTraits<InsertElementConstantExpr, 3> {
-};
+struct OperandTraits<InsertElementConstantExpr>
+    : public FixedNumOperandTraits<InsertElementConstantExpr, 3> {};
 DEFINE_TRANSPARENT_OPERAND_ACCESSORS(InsertElementConstantExpr, Value)
 
 template <>
-struct OperandTraits<ShuffleVectorConstantExpr> :
-    public FixedNumOperandTraits<ShuffleVectorConstantExpr, 3> {
-};
+struct OperandTraits<ShuffleVectorConstantExpr>
+    : public FixedNumOperandTraits<ShuffleVectorConstantExpr, 3> {};
 DEFINE_TRANSPARENT_OPERAND_ACCESSORS(ShuffleVectorConstantExpr, Value)
 
 template <>
-struct OperandTraits<ExtractValueConstantExpr> :
-  public FixedNumOperandTraits<ExtractValueConstantExpr, 1> {
-};
+struct OperandTraits<ExtractValueConstantExpr>
+    : public FixedNumOperandTraits<ExtractValueConstantExpr, 1> {};
 DEFINE_TRANSPARENT_OPERAND_ACCESSORS(ExtractValueConstantExpr, Value)
 
 template <>
-struct OperandTraits<InsertValueConstantExpr> :
-  public FixedNumOperandTraits<InsertValueConstantExpr, 2> {
-};
+struct OperandTraits<InsertValueConstantExpr>
+    : public FixedNumOperandTraits<InsertValueConstantExpr, 2> {};
 DEFINE_TRANSPARENT_OPERAND_ACCESSORS(InsertValueConstantExpr, Value)
 
 template <>
-struct OperandTraits<GetElementPtrConstantExpr> :
-  public VariadicOperandTraits<GetElementPtrConstantExpr, 1> {
-};
+struct OperandTraits<GetElementPtrConstantExpr>
+    : public VariadicOperandTraits<GetElementPtrConstantExpr, 1> {};
 
 DEFINE_TRANSPARENT_OPERAND_ACCESSORS(GetElementPtrConstantExpr, Value)
 
-
 template <>
-struct OperandTraits<CompareConstantExpr> :
-  public FixedNumOperandTraits<CompareConstantExpr, 2> {
-};
+struct OperandTraits<CompareConstantExpr>
+    : public FixedNumOperandTraits<CompareConstantExpr, 2> {};
 DEFINE_TRANSPARENT_OPERAND_ACCESSORS(CompareConstantExpr, Value)
 
 template <class ConstantClass> struct ConstantAggrKeyType;
@@ -375,41 +373,45 @@ template <class ConstantClass> struct ConstantAggrKeyType {
 struct InlineAsmKeyType {
   StringRef AsmString;
   StringRef Constraints;
+  FunctionType *FTy;
   bool HasSideEffects;
   bool IsAlignStack;
   InlineAsm::AsmDialect AsmDialect;
 
   InlineAsmKeyType(StringRef AsmString, StringRef Constraints,
-                   bool HasSideEffects, bool IsAlignStack,
+                   FunctionType *FTy, bool HasSideEffects, bool IsAlignStack,
                    InlineAsm::AsmDialect AsmDialect)
-      : AsmString(AsmString), Constraints(Constraints),
+      : AsmString(AsmString), Constraints(Constraints), FTy(FTy),
         HasSideEffects(HasSideEffects), IsAlignStack(IsAlignStack),
         AsmDialect(AsmDialect) {}
   InlineAsmKeyType(const InlineAsm *Asm, SmallVectorImpl<Constant *> &)
       : AsmString(Asm->getAsmString()), Constraints(Asm->getConstraintString()),
-        HasSideEffects(Asm->hasSideEffects()),
+        FTy(Asm->getFunctionType()), HasSideEffects(Asm->hasSideEffects()),
         IsAlignStack(Asm->isAlignStack()), AsmDialect(Asm->getDialect()) {}
 
   bool operator==(const InlineAsmKeyType &X) const {
     return HasSideEffects == X.HasSideEffects &&
            IsAlignStack == X.IsAlignStack && AsmDialect == X.AsmDialect &&
-           AsmString == X.AsmString && Constraints == X.Constraints;
+           AsmString == X.AsmString && Constraints == X.Constraints &&
+           FTy == X.FTy;
   }
   bool operator==(const InlineAsm *Asm) const {
     return HasSideEffects == Asm->hasSideEffects() &&
            IsAlignStack == Asm->isAlignStack() &&
            AsmDialect == Asm->getDialect() &&
            AsmString == Asm->getAsmString() &&
-           Constraints == Asm->getConstraintString();
+           Constraints == Asm->getConstraintString() &&
+           FTy == Asm->getFunctionType();
   }
   unsigned getHash() const {
     return hash_combine(AsmString, Constraints, HasSideEffects, IsAlignStack,
-                        AsmDialect);
+                        AsmDialect, FTy);
   }
 
   typedef ConstantInfo<InlineAsm>::TypeClass TypeClass;
   InlineAsm *create(TypeClass *Ty) const {
-    return new InlineAsm(Ty, AsmString, Constraints, HasSideEffects,
+    assert(PointerType::getUnqual(FTy) == Ty);
+    return new InlineAsm(FTy, AsmString, Constraints, HasSideEffects,
                          IsAlignStack, AsmDialect);
   }
 };
@@ -420,13 +422,16 @@ struct ConstantExprKeyType {
   uint16_t SubclassData;
   ArrayRef<Constant *> Ops;
   ArrayRef<unsigned> Indexes;
+  Type *ExplicitTy;
 
   ConstantExprKeyType(unsigned Opcode, ArrayRef<Constant *> Ops,
                       unsigned short SubclassData = 0,
                       unsigned short SubclassOptionalData = 0,
-                      ArrayRef<unsigned> Indexes = None)
+                      ArrayRef<unsigned> Indexes = None,
+                      Type *ExplicitTy = nullptr)
       : Opcode(Opcode), SubclassOptionalData(SubclassOptionalData),
-        SubclassData(SubclassData), Ops(Ops), Indexes(Indexes) {}
+        SubclassData(SubclassData), Ops(Ops), Indexes(Indexes),
+        ExplicitTy(ExplicitTy) {}
   ConstantExprKeyType(ArrayRef<Constant *> Operands, const ConstantExpr *CE)
       : Opcode(CE->getOpcode()),
         SubclassOptionalData(CE->getRawSubclassOptionalData()),
@@ -497,8 +502,11 @@ struct ConstantExprKeyType {
     case Instruction::ExtractValue:
       return new ExtractValueConstantExpr(Ops[0], Indexes, Ty);
     case Instruction::GetElementPtr:
-      return GetElementPtrConstantExpr::Create(Ops[0], Ops.slice(1), Ty,
-                                               SubclassOptionalData);
+      return GetElementPtrConstantExpr::Create(
+          ExplicitTy ? ExplicitTy
+                     : cast<PointerType>(Ops[0]->getType()->getScalarType())
+                           ->getElementType(),
+          Ops[0], Ops.slice(1), Ty, SubclassOptionalData);
     case Instruction::ICmp:
       return new CompareConstantExpr(Ty, Instruction::ICmp, SubclassData,
                                      Ops[0], Ops[1]);
diff --git a/lib/IR/Core.cpp b/lib/IR/Core.cpp
index 7fe7beb40c6..0eb88a96757 100644
--- a/lib/IR/Core.cpp
+++ b/lib/IR/Core.cpp
@@ -310,6 +310,9 @@ LLVMTypeRef LLVMInt32TypeInContext(LLVMContextRef C) {
 LLVMTypeRef LLVMInt64TypeInContext(LLVMContextRef C) {
   return (LLVMTypeRef) Type::getInt64Ty(*unwrap(C));
 }
+LLVMTypeRef LLVMInt128TypeInContext(LLVMContextRef C) {
+  return (LLVMTypeRef) Type::getInt128Ty(*unwrap(C));
+}
 LLVMTypeRef LLVMIntTypeInContext(LLVMContextRef C, unsigned NumBits) {
   return wrap(IntegerType::get(*unwrap(C), NumBits));
 }
@@ -329,6 +332,9 @@ LLVMTypeRef LLVMInt32Type(void) {
 LLVMTypeRef LLVMInt64Type(void) {
   return LLVMInt64TypeInContext(LLVMGetGlobalContext());
 }
+LLVMTypeRef LLVMInt128Type(void) {
+  return LLVMInt128TypeInContext(LLVMGetGlobalContext());
+}
 LLVMTypeRef LLVMIntType(unsigned NumBits) {
   return LLVMIntTypeInContext(LLVMGetGlobalContext(), NumBits);
 }
@@ -455,6 +461,11 @@ void LLVMGetStructElementTypes(LLVMTypeRef StructTy, LLVMTypeRef *Dest) {
     *Dest++ = wrap(*I);
 }
 
+LLVMTypeRef LLVMStructGetTypeAtIndex(LLVMTypeRef StructTy, unsigned i) {
+  StructType *Ty = unwrap<StructType>(StructTy);
+  return wrap(Ty->getTypeAtIndex(i));
+}
+
 LLVMBool LLVMIsPackedStruct(LLVMTypeRef StructTy) {
   return unwrap<StructType>(StructTy)->isPacked();
 }
@@ -1628,8 +1639,7 @@ void LLVMSetExternallyInitialized(LLVMValueRef GlobalVar, LLVMBool IsExtInit) {
 LLVMValueRef LLVMAddAlias(LLVMModuleRef M, LLVMTypeRef Ty, LLVMValueRef Aliasee,
                           const char *Name) {
   auto *PTy = cast<PointerType>(unwrap(Ty));
-  return wrap(GlobalAlias::create(PTy->getElementType(), PTy->getAddressSpace(),
-                                  GlobalValue::ExternalLinkage, Name,
+  return wrap(GlobalAlias::create(PTy, GlobalValue::ExternalLinkage, Name,
                                   unwrap<Constant>(Aliasee), unwrap(M)));
 }
 
@@ -1681,6 +1691,14 @@ void LLVMDeleteFunction(LLVMValueRef Fn) {
   unwrap<Function>(Fn)->eraseFromParent();
 }
 
+LLVMValueRef LLVMGetPersonalityFn(LLVMValueRef Fn) {
+  return wrap(unwrap<Function>(Fn)->getPersonalityFn());
+}
+
+void LLVMSetPersonalityFn(LLVMValueRef Fn, LLVMValueRef PersonalityFn) {
+  unwrap<Function>(Fn)->setPersonalityFn(unwrap<Constant>(PersonalityFn));
+}
+
 unsigned LLVMGetIntrinsicID(LLVMValueRef Fn) {
   if (Function *F = dyn_cast<Function>(unwrap(Fn)))
     return F->getIntrinsicID();
@@ -2241,9 +2259,13 @@ LLVMValueRef LLVMBuildInvoke(LLVMBuilderRef B, LLVMValueRef Fn,
 LLVMValueRef LLVMBuildLandingPad(LLVMBuilderRef B, LLVMTypeRef Ty,
                                  LLVMValueRef PersFn, unsigned NumClauses,
                                  const char *Name) {
-  return wrap(unwrap(B)->CreateLandingPad(unwrap(Ty),
-                                          cast<Function>(unwrap(PersFn)),
-                                          NumClauses, Name));
+  // The personality used to live on the landingpad instruction, but now it
+  // lives on the parent function. For compatibility, take the provided
+  // personality and put it on the parent function.
+  if (PersFn)
+    unwrap(B)->GetInsertBlock()->getParent()->setPersonalityFn(
+        cast<Function>(unwrap(PersFn)));
+  return wrap(unwrap(B)->CreateLandingPad(unwrap(Ty), NumClauses, Name));
 }
 
 LLVMValueRef LLVMBuildResume(LLVMBuilderRef B, LLVMValueRef Exn) {
diff --git a/lib/IR/DIBuilder.cpp b/lib/IR/DIBuilder.cpp
index 891fb861ac9..7a4c5203965 100644
--- a/lib/IR/DIBuilder.cpp
+++ b/lib/IR/DIBuilder.cpp
@@ -58,8 +58,7 @@ class HeaderBuilder {
 }
 
 DIBuilder::DIBuilder(Module &m, bool AllowUnresolvedNodes)
-    : M(m), VMContext(M.getContext()), TempEnumTypes(nullptr),
-      TempRetainTypes(nullptr), TempSubprograms(nullptr), TempGVs(nullptr),
+  : M(m), VMContext(M.getContext()), CUNode(nullptr),
       DeclareFn(nullptr), ValueFn(nullptr),
       AllowUnresolvedNodes(AllowUnresolvedNodes) {}
 
@@ -74,7 +73,13 @@ void DIBuilder::trackIfUnresolved(MDNode *N) {
 }
 
 void DIBuilder::finalize() {
-  TempEnumTypes->replaceAllUsesWith(MDTuple::get(VMContext, AllEnumTypes));
+  if (!CUNode) {
+    assert(!AllowUnresolvedNodes &&
+           "creating type nodes without a CU is not supported");
+    return;
+  }
+
+  CUNode->replaceEnumTypes(MDTuple::get(VMContext, AllEnumTypes));
 
   SmallVector<Metadata *, 16> RetainValues;
   // Declarations and definitions of the same type may be retained. Some
@@ -85,24 +90,30 @@ void DIBuilder::finalize() {
   for (unsigned I = 0, E = AllRetainTypes.size(); I < E; I++)
     if (RetainSet.insert(AllRetainTypes[I]).second)
       RetainValues.push_back(AllRetainTypes[I]);
-  TempRetainTypes->replaceAllUsesWith(MDTuple::get(VMContext, RetainValues));
 
-  MDSubprogramArray SPs = MDTuple::get(VMContext, AllSubprograms);
-  TempSubprograms->replaceAllUsesWith(SPs.get());
+  if (!RetainValues.empty())
+    CUNode->replaceRetainedTypes(MDTuple::get(VMContext, RetainValues));
+
+  DISubprogramArray SPs = MDTuple::get(VMContext, AllSubprograms);
+  if (!AllSubprograms.empty())
+    CUNode->replaceSubprograms(SPs.get());
+
   for (auto *SP : SPs) {
     if (MDTuple *Temp = SP->getVariables().get()) {
       const auto &PV = PreservedVariables.lookup(SP);
       SmallVector<Metadata *, 4> Variables(PV.begin(), PV.end());
-      DIArray AV = getOrCreateArray(Variables);
+      DINodeArray AV = getOrCreateArray(Variables);
       TempMDTuple(Temp)->replaceAllUsesWith(AV.get());
     }
   }
 
-  TempGVs->replaceAllUsesWith(MDTuple::get(VMContext, AllGVs));
+  if (!AllGVs.empty())
+    CUNode->replaceGlobalVariables(MDTuple::get(VMContext, AllGVs));
 
-  TempImportedModules->replaceAllUsesWith(MDTuple::get(
-      VMContext, SmallVector<Metadata *, 16>(AllImportedModules.begin(),
-                                             AllImportedModules.end())));
+  if (!AllImportedModules.empty())
+    CUNode->replaceImportedEntities(MDTuple::get(
+        VMContext, SmallVector<Metadata *, 16>(AllImportedModules.begin(),
+                                               AllImportedModules.end())));
 
   // Now that all temp nodes have been replaced or deleted, resolve remaining
   // cycles.
@@ -116,16 +127,16 @@ void DIBuilder::finalize() {
 }
 
 /// If N is compile unit return NULL otherwise return N.
-static MDScope *getNonCompileUnitScope(MDScope *N) {
-  if (!N || isa<MDCompileUnit>(N))
+static DIScope *getNonCompileUnitScope(DIScope *N) {
+  if (!N || isa<DICompileUnit>(N))
     return nullptr;
-  return cast<MDScope>(N);
+  return cast<DIScope>(N);
 }
 
-MDCompileUnit *DIBuilder::createCompileUnit(
+DICompileUnit *DIBuilder::createCompileUnit(
     unsigned Lang, StringRef Filename, StringRef Directory, StringRef Producer,
     bool isOptimized, StringRef Flags, unsigned RunTimeVer, StringRef SplitName,
-    DebugEmissionKind Kind, bool EmitDebugInfo) {
+    DebugEmissionKind Kind, uint64_t DWOId, bool EmitDebugInfo) {
 
   assert(((Lang <= dwarf::DW_LANG_Fortran08 && Lang >= dwarf::DW_LANG_C89) ||
           (Lang <= dwarf::DW_LANG_hi_user && Lang >= dwarf::DW_LANG_lo_user)) &&
@@ -133,21 +144,11 @@ MDCompileUnit *DIBuilder::createCompileUnit(
   assert(!Filename.empty() &&
          "Unable to create compile unit without filename");
 
-  // TODO: Once we make MDCompileUnit distinct, stop using temporaries here
-  // (just start with operands assigned to nullptr).
-  TempEnumTypes = MDTuple::getTemporary(VMContext, None);
-  TempRetainTypes = MDTuple::getTemporary(VMContext, None);
-  TempSubprograms = MDTuple::getTemporary(VMContext, None);
-  TempGVs = MDTuple::getTemporary(VMContext, None);
-  TempImportedModules = MDTuple::getTemporary(VMContext, None);
-
-  // TODO: Switch to getDistinct().  We never want to merge compile units based
-  // on contents.
-  MDCompileUnit *CUNode = MDCompileUnit::get(
-      VMContext, Lang, MDFile::get(VMContext, Filename, Directory), Producer,
-      isOptimized, Flags, RunTimeVer, SplitName, Kind, TempEnumTypes.get(),
-      TempRetainTypes.get(), TempSubprograms.get(), TempGVs.get(),
-      TempImportedModules.get());
+  assert(!CUNode && "Can only make one compile unit per DIBuilder instance");
+  CUNode = DICompileUnit::getDistinct(
+      VMContext, Lang, DIFile::get(VMContext, Filename, Directory), Producer,
+      isOptimized, Flags, RunTimeVer, SplitName, Kind, nullptr,
+      nullptr, nullptr, nullptr, nullptr, DWOId);
 
   // Create a named metadata so that it is easier to find cu in a module.
   // Note that we only generate this when the caller wants to actually
@@ -163,132 +164,138 @@ MDCompileUnit *DIBuilder::createCompileUnit(
   return CUNode;
 }
 
-static MDImportedEntity*
-createImportedModule(LLVMContext &C, dwarf::Tag Tag, MDScope* Context,
+static DIImportedEntity *
+createImportedModule(LLVMContext &C, dwarf::Tag Tag, DIScope *Context,
                      Metadata *NS, unsigned Line, StringRef Name,
                      SmallVectorImpl<TrackingMDNodeRef> &AllImportedModules) {
-  auto *M =
-      MDImportedEntity::get(C, Tag, Context, DebugNodeRef(NS), Line, Name);
+  auto *M = DIImportedEntity::get(C, Tag, Context, DINodeRef(NS), Line, Name);
   AllImportedModules.emplace_back(M);
   return M;
 }
 
-MDImportedEntity* DIBuilder::createImportedModule(MDScope* Context,
-                                                 MDNamespace* NS,
-                                                 unsigned Line) {
+DIImportedEntity *DIBuilder::createImportedModule(DIScope *Context,
+                                                  DINamespace *NS,
+                                                  unsigned Line) {
   return ::createImportedModule(VMContext, dwarf::DW_TAG_imported_module,
                                 Context, NS, Line, StringRef(), AllImportedModules);
 }
 
-MDImportedEntity* DIBuilder::createImportedModule(MDScope* Context,
-                                                 MDImportedEntity* NS,
-                                                 unsigned Line) {
+DIImportedEntity *DIBuilder::createImportedModule(DIScope *Context,
+                                                  DIImportedEntity *NS,
+                                                  unsigned Line) {
   return ::createImportedModule(VMContext, dwarf::DW_TAG_imported_module,
                                 Context, NS, Line, StringRef(), AllImportedModules);
 }
 
-MDImportedEntity *DIBuilder::createImportedDeclaration(MDScope *Context,
-                                                       DebugNode *Decl,
+DIImportedEntity *DIBuilder::createImportedModule(DIScope *Context, DIModule *M,
+                                                  unsigned Line) {
+  return ::createImportedModule(VMContext, dwarf::DW_TAG_imported_module,
+                                Context, M, Line, StringRef(), AllImportedModules);
+}
+
+DIImportedEntity *DIBuilder::createImportedDeclaration(DIScope *Context,
+                                                       DINode *Decl,
                                                        unsigned Line,
                                                        StringRef Name) {
   // Make sure to use the unique identifier based metadata reference for
   // types that have one.
   return ::createImportedModule(VMContext, dwarf::DW_TAG_imported_declaration,
-                                Context, DebugNodeRef::get(Decl), Line, Name,
+                                Context, DINodeRef::get(Decl), Line, Name,
                                 AllImportedModules);
 }
 
-MDFile* DIBuilder::createFile(StringRef Filename, StringRef Directory) {
-  return MDFile::get(VMContext, Filename, Directory);
+DIFile *DIBuilder::createFile(StringRef Filename, StringRef Directory) {
+  return DIFile::get(VMContext, Filename, Directory);
 }
 
-MDEnumerator *DIBuilder::createEnumerator(StringRef Name, int64_t Val) {
+DIEnumerator *DIBuilder::createEnumerator(StringRef Name, int64_t Val) {
   assert(!Name.empty() && "Unable to create enumerator without name");
-  return MDEnumerator::get(VMContext, Val, Name);
+  return DIEnumerator::get(VMContext, Val, Name);
 }
 
-MDBasicType *DIBuilder::createUnspecifiedType(StringRef Name) {
+DIBasicType *DIBuilder::createUnspecifiedType(StringRef Name) {
   assert(!Name.empty() && "Unable to create type without name");
-  return MDBasicType::get(VMContext, dwarf::DW_TAG_unspecified_type, Name);
+  return DIBasicType::get(VMContext, dwarf::DW_TAG_unspecified_type, Name);
 }
 
-MDBasicType *DIBuilder::createNullPtrType() {
+DIBasicType *DIBuilder::createNullPtrType() {
   return createUnspecifiedType("decltype(nullptr)");
 }
 
-MDBasicType *DIBuilder::createBasicType(StringRef Name, uint64_t SizeInBits,
+DIBasicType *DIBuilder::createBasicType(StringRef Name, uint64_t SizeInBits,
                                         uint64_t AlignInBits,
                                         unsigned Encoding) {
   assert(!Name.empty() && "Unable to create type without name");
-  return MDBasicType::get(VMContext, dwarf::DW_TAG_base_type, Name, SizeInBits,
+  return DIBasicType::get(VMContext, dwarf::DW_TAG_base_type, Name, SizeInBits,
                           AlignInBits, Encoding);
 }
 
-MDDerivedType *DIBuilder::createQualifiedType(unsigned Tag, MDType *FromTy) {
-  return MDDerivedType::get(VMContext, Tag, "", nullptr, 0, nullptr,
-                            MDTypeRef::get(FromTy), 0, 0, 0, 0);
+DIDerivedType *DIBuilder::createQualifiedType(unsigned Tag, DIType *FromTy) {
+  return DIDerivedType::get(VMContext, Tag, "", nullptr, 0, nullptr,
+                            DITypeRef::get(FromTy), 0, 0, 0, 0);
 }
 
-MDDerivedType *DIBuilder::createPointerType(MDType *PointeeTy,
+DIDerivedType *DIBuilder::createPointerType(DIType *PointeeTy,
                                             uint64_t SizeInBits,
                                             uint64_t AlignInBits,
                                             StringRef Name) {
   // FIXME: Why is there a name here?
-  return MDDerivedType::get(VMContext, dwarf::DW_TAG_pointer_type, Name,
-                            nullptr, 0, nullptr, MDTypeRef::get(PointeeTy),
+  return DIDerivedType::get(VMContext, dwarf::DW_TAG_pointer_type, Name,
+                            nullptr, 0, nullptr, DITypeRef::get(PointeeTy),
                             SizeInBits, AlignInBits, 0, 0);
 }
 
-MDDerivedType *DIBuilder::createMemberPointerType(MDType *PointeeTy,
-                                                  MDType *Base,
+DIDerivedType *DIBuilder::createMemberPointerType(DIType *PointeeTy,
+                                                  DIType *Base,
                                                   uint64_t SizeInBits,
                                                   uint64_t AlignInBits) {
-  return MDDerivedType::get(VMContext, dwarf::DW_TAG_ptr_to_member_type, "",
-                            nullptr, 0, nullptr, MDTypeRef::get(PointeeTy),
-                            SizeInBits, AlignInBits, 0, 0, MDTypeRef::get(Base));
+  return DIDerivedType::get(VMContext, dwarf::DW_TAG_ptr_to_member_type, "",
+                            nullptr, 0, nullptr, DITypeRef::get(PointeeTy),
+                            SizeInBits, AlignInBits, 0, 0,
+                            DITypeRef::get(Base));
 }
 
-MDDerivedType *DIBuilder::createReferenceType(unsigned Tag, MDType *RTy) {
+DIDerivedType *DIBuilder::createReferenceType(unsigned Tag, DIType *RTy) {
   assert(RTy && "Unable to create reference type");
-  return MDDerivedType::get(VMContext, Tag, "", nullptr, 0, nullptr,
-                            MDTypeRef::get(RTy), 0, 0, 0, 0);
+  return DIDerivedType::get(VMContext, Tag, "", nullptr, 0, nullptr,
+                            DITypeRef::get(RTy), 0, 0, 0, 0);
 }
 
-MDDerivedType *DIBuilder::createTypedef(MDType *Ty, StringRef Name,
-                                        MDFile *File, unsigned LineNo,
-                                        MDScope *Context) {
-  return MDDerivedType::get(VMContext, dwarf::DW_TAG_typedef, Name, File,
+DIDerivedType *DIBuilder::createTypedef(DIType *Ty, StringRef Name,
+                                        DIFile *File, unsigned LineNo,
+                                        DIScope *Context) {
+  return DIDerivedType::get(VMContext, dwarf::DW_TAG_typedef, Name, File,
                             LineNo,
-                            MDScopeRef::get(getNonCompileUnitScope(Context)),
-                            MDTypeRef::get(Ty), 0, 0, 0, 0);
+                            DIScopeRef::get(getNonCompileUnitScope(Context)),
+                            DITypeRef::get(Ty), 0, 0, 0, 0);
 }
 
-MDDerivedType *DIBuilder::createFriend(MDType *Ty, MDType *FriendTy) {
+DIDerivedType *DIBuilder::createFriend(DIType *Ty, DIType *FriendTy) {
   assert(Ty && "Invalid type!");
   assert(FriendTy && "Invalid friend type!");
-  return MDDerivedType::get(VMContext, dwarf::DW_TAG_friend, "", nullptr, 0,
-                            MDTypeRef::get(Ty), MDTypeRef::get(FriendTy), 0, 0,
+  return DIDerivedType::get(VMContext, dwarf::DW_TAG_friend, "", nullptr, 0,
+                            DITypeRef::get(Ty), DITypeRef::get(FriendTy), 0, 0,
                             0, 0);
 }
 
-MDDerivedType *DIBuilder::createInheritance(MDType *Ty, MDType *BaseTy,
+DIDerivedType *DIBuilder::createInheritance(DIType *Ty, DIType *BaseTy,
                                             uint64_t BaseOffset,
                                             unsigned Flags) {
   assert(Ty && "Unable to create inheritance");
-  return MDDerivedType::get(VMContext, dwarf::DW_TAG_inheritance, "", nullptr,
-                            0, MDTypeRef::get(Ty), MDTypeRef::get(BaseTy), 0, 0,
+  return DIDerivedType::get(VMContext, dwarf::DW_TAG_inheritance, "", nullptr,
+                            0, DITypeRef::get(Ty), DITypeRef::get(BaseTy), 0, 0,
                             BaseOffset, Flags);
 }
 
-MDDerivedType *DIBuilder::createMemberType(MDScope *Scope, StringRef Name,
-                                           MDFile *File, unsigned LineNumber,
+DIDerivedType *DIBuilder::createMemberType(DIScope *Scope, StringRef Name,
+                                           DIFile *File, unsigned LineNumber,
                                            uint64_t SizeInBits,
                                            uint64_t AlignInBits,
                                            uint64_t OffsetInBits,
-                                           unsigned Flags, MDType *Ty) {
-  return MDDerivedType::get(
+                                           unsigned Flags, DIType *Ty) {
+  return DIDerivedType::get(
       VMContext, dwarf::DW_TAG_member, Name, File, LineNumber,
-      MDScopeRef::get(getNonCompileUnitScope(Scope)), MDTypeRef::get(Ty),
+      DIScopeRef::get(getNonCompileUnitScope(Scope)), DITypeRef::get(Ty),
       SizeInBits, AlignInBits, OffsetInBits, Flags);
 }
 
@@ -298,91 +305,92 @@ static ConstantAsMetadata *getConstantOrNull(Constant *C) {
   return nullptr;
 }
 
-MDDerivedType *DIBuilder::createStaticMemberType(MDScope *Scope, StringRef Name,
-                                                 MDFile *File,
+DIDerivedType *DIBuilder::createStaticMemberType(DIScope *Scope, StringRef Name,
+                                                 DIFile *File,
                                                  unsigned LineNumber,
-                                                 MDType *Ty, unsigned Flags,
+                                                 DIType *Ty, unsigned Flags,
                                                  llvm::Constant *Val) {
-  Flags |= DebugNode::FlagStaticMember;
-  return MDDerivedType::get(
+  Flags |= DINode::FlagStaticMember;
+  return DIDerivedType::get(
       VMContext, dwarf::DW_TAG_member, Name, File, LineNumber,
-      MDScopeRef::get(getNonCompileUnitScope(Scope)), MDTypeRef::get(Ty), 0, 0,
+      DIScopeRef::get(getNonCompileUnitScope(Scope)), DITypeRef::get(Ty), 0, 0,
       0, Flags, getConstantOrNull(Val));
 }
 
-MDDerivedType *DIBuilder::createObjCIVar(StringRef Name, MDFile *File,
+DIDerivedType *DIBuilder::createObjCIVar(StringRef Name, DIFile *File,
                                          unsigned LineNumber,
                                          uint64_t SizeInBits,
                                          uint64_t AlignInBits,
                                          uint64_t OffsetInBits, unsigned Flags,
-                                         MDType *Ty, MDNode *PropertyNode) {
-  return MDDerivedType::get(
+                                         DIType *Ty, MDNode *PropertyNode) {
+  return DIDerivedType::get(
       VMContext, dwarf::DW_TAG_member, Name, File, LineNumber,
-      MDScopeRef::get(getNonCompileUnitScope(File)), MDTypeRef::get(Ty),
+      DIScopeRef::get(getNonCompileUnitScope(File)), DITypeRef::get(Ty),
       SizeInBits, AlignInBits, OffsetInBits, Flags, PropertyNode);
 }
 
-MDObjCProperty *
-DIBuilder::createObjCProperty(StringRef Name, MDFile *File, unsigned LineNumber,
+DIObjCProperty *
+DIBuilder::createObjCProperty(StringRef Name, DIFile *File, unsigned LineNumber,
                               StringRef GetterName, StringRef SetterName,
-                              unsigned PropertyAttributes, MDType *Ty) {
-  return MDObjCProperty::get(VMContext, Name, File, LineNumber, GetterName,
-                             SetterName, PropertyAttributes, Ty);
+                              unsigned PropertyAttributes, DIType *Ty) {
+  return DIObjCProperty::get(VMContext, Name, File, LineNumber, GetterName,
+                             SetterName, PropertyAttributes,
+                             DITypeRef::get(Ty));
 }
 
-MDTemplateTypeParameter *
-DIBuilder::createTemplateTypeParameter(MDScope *Context, StringRef Name,
-                                       MDType *Ty) {
-  assert((!Context || isa<MDCompileUnit>(Context)) && "Expected compile unit");
-  return MDTemplateTypeParameter::get(VMContext, Name, MDTypeRef::get(Ty));
+DITemplateTypeParameter *
+DIBuilder::createTemplateTypeParameter(DIScope *Context, StringRef Name,
+                                       DIType *Ty) {
+  assert((!Context || isa<DICompileUnit>(Context)) && "Expected compile unit");
+  return DITemplateTypeParameter::get(VMContext, Name, DITypeRef::get(Ty));
 }
 
-static MDTemplateValueParameter *
+static DITemplateValueParameter *
 createTemplateValueParameterHelper(LLVMContext &VMContext, unsigned Tag,
-                                   MDScope *Context, StringRef Name, MDType *Ty,
+                                   DIScope *Context, StringRef Name, DIType *Ty,
                                    Metadata *MD) {
-  assert((!Context || isa<MDCompileUnit>(Context)) && "Expected compile unit");
-  return MDTemplateValueParameter::get(VMContext, Tag, Name, MDTypeRef::get(Ty),
+  assert((!Context || isa<DICompileUnit>(Context)) && "Expected compile unit");
+  return DITemplateValueParameter::get(VMContext, Tag, Name, DITypeRef::get(Ty),
                                        MD);
 }
 
-MDTemplateValueParameter *
-DIBuilder::createTemplateValueParameter(MDScope *Context, StringRef Name,
-                                        MDType *Ty, Constant *Val) {
+DITemplateValueParameter *
+DIBuilder::createTemplateValueParameter(DIScope *Context, StringRef Name,
+                                        DIType *Ty, Constant *Val) {
   return createTemplateValueParameterHelper(
       VMContext, dwarf::DW_TAG_template_value_parameter, Context, Name, Ty,
       getConstantOrNull(Val));
 }
 
-MDTemplateValueParameter *
-DIBuilder::createTemplateTemplateParameter(MDScope *Context, StringRef Name,
-                                           MDType *Ty, StringRef Val) {
+DITemplateValueParameter *
+DIBuilder::createTemplateTemplateParameter(DIScope *Context, StringRef Name,
+                                           DIType *Ty, StringRef Val) {
   return createTemplateValueParameterHelper(
       VMContext, dwarf::DW_TAG_GNU_template_template_param, Context, Name, Ty,
       MDString::get(VMContext, Val));
 }
 
-MDTemplateValueParameter *
-DIBuilder::createTemplateParameterPack(MDScope *Context, StringRef Name,
-                                       MDType *Ty, DIArray Val) {
+DITemplateValueParameter *
+DIBuilder::createTemplateParameterPack(DIScope *Context, StringRef Name,
+                                       DIType *Ty, DINodeArray Val) {
   return createTemplateValueParameterHelper(
       VMContext, dwarf::DW_TAG_GNU_template_parameter_pack, Context, Name, Ty,
       Val.get());
 }
 
-MDCompositeType *DIBuilder::createClassType(
-    MDScope *Context, StringRef Name, MDFile *File, unsigned LineNumber,
+DICompositeType *DIBuilder::createClassType(
+    DIScope *Context, StringRef Name, DIFile *File, unsigned LineNumber,
     uint64_t SizeInBits, uint64_t AlignInBits, uint64_t OffsetInBits,
-    unsigned Flags, MDType *DerivedFrom, DIArray Elements, MDType *VTableHolder,
-    MDNode *TemplateParams, StringRef UniqueIdentifier) {
-  assert((!Context || isa<MDScope>(Context)) &&
+    unsigned Flags, DIType *DerivedFrom, DINodeArray Elements,
+    DIType *VTableHolder, MDNode *TemplateParams, StringRef UniqueIdentifier) {
+  assert((!Context || isa<DIScope>(Context)) &&
          "createClassType should be called with a valid Context");
 
-  auto *R = MDCompositeType::get(
+  auto *R = DICompositeType::get(
       VMContext, dwarf::DW_TAG_structure_type, Name, File, LineNumber,
-      MDScopeRef::get(getNonCompileUnitScope(Context)),
-      MDTypeRef::get(DerivedFrom), SizeInBits, AlignInBits, OffsetInBits, Flags,
-      Elements, 0, MDTypeRef::get(VTableHolder),
+      DIScopeRef::get(getNonCompileUnitScope(Context)),
+      DITypeRef::get(DerivedFrom), SizeInBits, AlignInBits, OffsetInBits, Flags,
+      Elements, 0, DITypeRef::get(VTableHolder),
       cast_or_null<MDTuple>(TemplateParams), UniqueIdentifier);
   if (!UniqueIdentifier.empty())
     retainType(R);
@@ -390,32 +398,29 @@ MDCompositeType *DIBuilder::createClassType(
   return R;
 }
 
-MDCompositeType *DIBuilder::createStructType(
-    MDScope *Context, StringRef Name, MDFile *File, unsigned LineNumber,
+DICompositeType *DIBuilder::createStructType(
+    DIScope *Context, StringRef Name, DIFile *File, unsigned LineNumber,
     uint64_t SizeInBits, uint64_t AlignInBits, unsigned Flags,
-    MDType *DerivedFrom, DIArray Elements, unsigned RunTimeLang,
-    MDType *VTableHolder, StringRef UniqueIdentifier) {
-  auto *R = MDCompositeType::get(
+    DIType *DerivedFrom, DINodeArray Elements, unsigned RunTimeLang,
+    DIType *VTableHolder, StringRef UniqueIdentifier) {
+  auto *R = DICompositeType::get(
       VMContext, dwarf::DW_TAG_structure_type, Name, File, LineNumber,
-      MDScopeRef::get(getNonCompileUnitScope(Context)),
-      MDTypeRef::get(DerivedFrom), SizeInBits, AlignInBits, 0, Flags, Elements,
-      RunTimeLang, MDTypeRef::get(VTableHolder), nullptr, UniqueIdentifier);
+      DIScopeRef::get(getNonCompileUnitScope(Context)),
+      DITypeRef::get(DerivedFrom), SizeInBits, AlignInBits, 0, Flags, Elements,
+      RunTimeLang, DITypeRef::get(VTableHolder), nullptr, UniqueIdentifier);
   if (!UniqueIdentifier.empty())
     retainType(R);
   trackIfUnresolved(R);
   return R;
 }
 
-MDCompositeType* DIBuilder::createUnionType(MDScope * Scope, StringRef Name,
-                                           MDFile* File, unsigned LineNumber,
-                                           uint64_t SizeInBits,
-                                           uint64_t AlignInBits, unsigned Flags,
-                                           DIArray Elements,
-                                           unsigned RunTimeLang,
-                                           StringRef UniqueIdentifier) {
-  auto *R = MDCompositeType::get(
+DICompositeType *DIBuilder::createUnionType(
+    DIScope *Scope, StringRef Name, DIFile *File, unsigned LineNumber,
+    uint64_t SizeInBits, uint64_t AlignInBits, unsigned Flags,
+    DINodeArray Elements, unsigned RunTimeLang, StringRef UniqueIdentifier) {
+  auto *R = DICompositeType::get(
       VMContext, dwarf::DW_TAG_union_type, Name, File, LineNumber,
-      MDScopeRef::get(getNonCompileUnitScope(Scope)), nullptr, SizeInBits,
+      DIScopeRef::get(getNonCompileUnitScope(Scope)), nullptr, SizeInBits,
       AlignInBits, 0, Flags, Elements, RunTimeLang, nullptr, nullptr,
       UniqueIdentifier);
   if (!UniqueIdentifier.empty())
@@ -424,20 +429,32 @@ MDCompositeType* DIBuilder::createUnionType(MDScope * Scope, StringRef Name,
   return R;
 }
 
-MDSubroutineType *DIBuilder::createSubroutineType(MDFile *File,
-                                                  DITypeArray ParameterTypes,
+DISubroutineType *DIBuilder::createSubroutineType(DIFile *File,
+                                                  DITypeRefArray ParameterTypes,
                                                   unsigned Flags) {
-  return MDSubroutineType::get(VMContext, Flags, ParameterTypes);
+  return DISubroutineType::get(VMContext, Flags, ParameterTypes);
+}
+
+DICompositeType *DIBuilder::createExternalTypeRef(unsigned Tag, DIFile *File,
+                                                  StringRef UniqueIdentifier) {
+  assert(!UniqueIdentifier.empty() && "external type ref without uid");
+  auto *CTy =
+      DICompositeType::get(VMContext, Tag, "", nullptr, 0, nullptr, nullptr, 0,
+                           0, 0, DINode::FlagExternalTypeRef, nullptr, 0,
+                           nullptr, nullptr, UniqueIdentifier);
+  // Types with unique IDs need to be in the type map.
+  retainType(CTy);
+  return CTy;
 }
 
-MDCompositeType *DIBuilder::createEnumerationType(
-    MDScope *Scope, StringRef Name, MDFile *File, unsigned LineNumber,
-    uint64_t SizeInBits, uint64_t AlignInBits, DIArray Elements,
-    MDType *UnderlyingType, StringRef UniqueIdentifier) {
-  auto *CTy = MDCompositeType::get(
+DICompositeType *DIBuilder::createEnumerationType(
+    DIScope *Scope, StringRef Name, DIFile *File, unsigned LineNumber,
+    uint64_t SizeInBits, uint64_t AlignInBits, DINodeArray Elements,
+    DIType *UnderlyingType, StringRef UniqueIdentifier) {
+  auto *CTy = DICompositeType::get(
       VMContext, dwarf::DW_TAG_enumeration_type, Name, File, LineNumber,
-      MDScopeRef::get(getNonCompileUnitScope(Scope)),
-      MDTypeRef::get(UnderlyingType), SizeInBits, AlignInBits, 0, 0, Elements,
+      DIScopeRef::get(getNonCompileUnitScope(Scope)),
+      DITypeRef::get(UnderlyingType), SizeInBits, AlignInBits, 0, 0, Elements,
       0, nullptr, nullptr, UniqueIdentifier);
   AllEnumTypes.push_back(CTy);
   if (!UniqueIdentifier.empty())
@@ -446,66 +463,67 @@ MDCompositeType *DIBuilder::createEnumerationType(
   return CTy;
 }
 
-MDCompositeType *DIBuilder::createArrayType(uint64_t Size, uint64_t AlignInBits,
-                                            MDType *Ty, DIArray Subscripts) {
-  auto *R = MDCompositeType::get(VMContext, dwarf::DW_TAG_array_type, "",
-                                 nullptr, 0, nullptr, MDTypeRef::get(Ty), Size,
+DICompositeType *DIBuilder::createArrayType(uint64_t Size, uint64_t AlignInBits,
+                                            DIType *Ty,
+                                            DINodeArray Subscripts) {
+  auto *R = DICompositeType::get(VMContext, dwarf::DW_TAG_array_type, "",
+                                 nullptr, 0, nullptr, DITypeRef::get(Ty), Size,
                                  AlignInBits, 0, 0, Subscripts, 0, nullptr);
   trackIfUnresolved(R);
   return R;
 }
 
-MDCompositeType *DIBuilder::createVectorType(uint64_t Size,
-                                             uint64_t AlignInBits, MDType *Ty,
-                                             DIArray Subscripts) {
+DICompositeType *DIBuilder::createVectorType(uint64_t Size,
+                                             uint64_t AlignInBits, DIType *Ty,
+                                             DINodeArray Subscripts) {
   auto *R =
-      MDCompositeType::get(VMContext, dwarf::DW_TAG_array_type, "", nullptr, 0,
-                           nullptr, MDTypeRef::get(Ty), Size, AlignInBits, 0,
-                           DebugNode::FlagVector, Subscripts, 0, nullptr);
+      DICompositeType::get(VMContext, dwarf::DW_TAG_array_type, "", nullptr, 0,
+                           nullptr, DITypeRef::get(Ty), Size, AlignInBits, 0,
+                           DINode::FlagVector, Subscripts, 0, nullptr);
   trackIfUnresolved(R);
   return R;
 }
 
-static MDType *createTypeWithFlags(LLVMContext &Context, MDType *Ty,
+static DIType *createTypeWithFlags(LLVMContext &Context, DIType *Ty,
                                    unsigned FlagsToSet) {
   auto NewTy = Ty->clone();
   NewTy->setFlags(NewTy->getFlags() | FlagsToSet);
   return MDNode::replaceWithUniqued(std::move(NewTy));
 }
 
-MDType *DIBuilder::createArtificialType(MDType *Ty) {
+DIType *DIBuilder::createArtificialType(DIType *Ty) {
   // FIXME: Restrict this to the nodes where it's valid.
   if (Ty->isArtificial())
     return Ty;
-  return createTypeWithFlags(VMContext, Ty, DebugNode::FlagArtificial);
+  return createTypeWithFlags(VMContext, Ty, DINode::FlagArtificial);
 }
 
-MDType *DIBuilder::createObjectPointerType(MDType *Ty) {
+DIType *DIBuilder::createObjectPointerType(DIType *Ty) {
   // FIXME: Restrict this to the nodes where it's valid.
   if (Ty->isObjectPointer())
     return Ty;
-  unsigned Flags = DebugNode::FlagObjectPointer | DebugNode::FlagArtificial;
+  unsigned Flags = DINode::FlagObjectPointer | DINode::FlagArtificial;
   return createTypeWithFlags(VMContext, Ty, Flags);
 }
 
-void DIBuilder::retainType(MDType *T) {
+void DIBuilder::retainType(DIType *T) {
   assert(T && "Expected non-null type");
   AllRetainTypes.emplace_back(T);
 }
 
-MDBasicType *DIBuilder::createUnspecifiedParameter() { return nullptr; }
+DIBasicType *DIBuilder::createUnspecifiedParameter() { return nullptr; }
 
-MDCompositeType*
-DIBuilder::createForwardDecl(unsigned Tag, StringRef Name, MDScope * Scope,
-                             MDFile* F, unsigned Line, unsigned RuntimeLang,
+DICompositeType *
+DIBuilder::createForwardDecl(unsigned Tag, StringRef Name, DIScope *Scope,
+                             DIFile *F, unsigned Line, unsigned RuntimeLang,
                              uint64_t SizeInBits, uint64_t AlignInBits,
                              StringRef UniqueIdentifier) {
   // FIXME: Define in terms of createReplaceableForwardDecl() by calling
   // replaceWithUniqued().
-  auto *RetTy = MDCompositeType::get(
+  auto *RetTy = DICompositeType::get(
       VMContext, Tag, Name, F, Line,
-      MDScopeRef::get(getNonCompileUnitScope(Scope)), nullptr, SizeInBits,
-      AlignInBits, 0, DebugNode::FlagFwdDecl, nullptr, RuntimeLang, nullptr,
+      DIScopeRef::get(getNonCompileUnitScope(Scope)), nullptr, SizeInBits,
+      AlignInBits, 0, DINode::FlagFwdDecl, nullptr, RuntimeLang, nullptr,
       nullptr, UniqueIdentifier);
   if (!UniqueIdentifier.empty())
     retainType(RetTy);
@@ -513,122 +531,143 @@ DIBuilder::createForwardDecl(unsigned Tag, StringRef Name, MDScope * Scope,
   return RetTy;
 }
 
-MDCompositeType* DIBuilder::createReplaceableCompositeType(
-    unsigned Tag, StringRef Name, MDScope * Scope, MDFile* F, unsigned Line,
+DICompositeType *DIBuilder::createReplaceableCompositeType(
+    unsigned Tag, StringRef Name, DIScope *Scope, DIFile *F, unsigned Line,
     unsigned RuntimeLang, uint64_t SizeInBits, uint64_t AlignInBits,
     unsigned Flags, StringRef UniqueIdentifier) {
-  auto *RetTy = MDCompositeType::getTemporary(
+  auto *RetTy = DICompositeType::getTemporary(
                     VMContext, Tag, Name, F, Line,
-                    MDScopeRef::get(getNonCompileUnitScope(Scope)), nullptr,
+                    DIScopeRef::get(getNonCompileUnitScope(Scope)), nullptr,
                     SizeInBits, AlignInBits, 0, Flags, nullptr, RuntimeLang,
-                    nullptr, nullptr, UniqueIdentifier).release();
+                    nullptr, nullptr, UniqueIdentifier)
+                    .release();
   if (!UniqueIdentifier.empty())
     retainType(RetTy);
   trackIfUnresolved(RetTy);
   return RetTy;
 }
 
-DIArray DIBuilder::getOrCreateArray(ArrayRef<Metadata *> Elements) {
+DINodeArray DIBuilder::getOrCreateArray(ArrayRef<Metadata *> Elements) {
   return MDTuple::get(VMContext, Elements);
 }
 
-DITypeArray DIBuilder::getOrCreateTypeArray(ArrayRef<Metadata *> Elements) {
+DITypeRefArray DIBuilder::getOrCreateTypeArray(ArrayRef<Metadata *> Elements) {
   SmallVector<llvm::Metadata *, 16> Elts;
   for (unsigned i = 0, e = Elements.size(); i != e; ++i) {
     if (Elements[i] && isa<MDNode>(Elements[i]))
-      Elts.push_back(MDTypeRef::get(cast<MDType>(Elements[i])));
+      Elts.push_back(DITypeRef::get(cast<DIType>(Elements[i])));
     else
       Elts.push_back(Elements[i]);
   }
-  return DITypeArray(MDNode::get(VMContext, Elts));
+  return DITypeRefArray(MDNode::get(VMContext, Elts));
 }
 
-MDSubrange *DIBuilder::getOrCreateSubrange(int64_t Lo, int64_t Count) {
-  return MDSubrange::get(VMContext, Count, Lo);
+DISubrange *DIBuilder::getOrCreateSubrange(int64_t Lo, int64_t Count) {
+  return DISubrange::get(VMContext, Count, Lo);
 }
 
-static void checkGlobalVariableScope(MDScope * Context) {
+static void checkGlobalVariableScope(DIScope *Context) {
 #ifndef NDEBUG
   if (auto *CT =
-          dyn_cast_or_null<MDCompositeType>(getNonCompileUnitScope(Context)))
+          dyn_cast_or_null<DICompositeType>(getNonCompileUnitScope(Context)))
     assert(CT->getIdentifier().empty() &&
            "Context of a global variable should not be a type with identifier");
 #endif
 }
 
-MDGlobalVariable *DIBuilder::createGlobalVariable(
-    MDScope *Context, StringRef Name, StringRef LinkageName, MDFile *F,
-    unsigned LineNumber, MDType *Ty, bool isLocalToUnit, Constant *Val,
+DIGlobalVariable *DIBuilder::createGlobalVariable(
+    DIScope *Context, StringRef Name, StringRef LinkageName, DIFile *F,
+    unsigned LineNumber, DIType *Ty, bool isLocalToUnit, Constant *Val,
     MDNode *Decl) {
   checkGlobalVariableScope(Context);
 
-  auto *N = MDGlobalVariable::get(VMContext, cast_or_null<MDScope>(Context),
+  auto *N = DIGlobalVariable::get(VMContext, cast_or_null<DIScope>(Context),
                                   Name, LinkageName, F, LineNumber,
-                                  MDTypeRef::get(Ty), isLocalToUnit, true, Val,
-                                  cast_or_null<MDDerivedType>(Decl));
+                                  DITypeRef::get(Ty), isLocalToUnit, true, Val,
+                                  cast_or_null<DIDerivedType>(Decl));
   AllGVs.push_back(N);
   return N;
 }
 
-MDGlobalVariable *DIBuilder::createTempGlobalVariableFwdDecl(
-    MDScope *Context, StringRef Name, StringRef LinkageName, MDFile *F,
-    unsigned LineNumber, MDType *Ty, bool isLocalToUnit, Constant *Val,
+DIGlobalVariable *DIBuilder::createTempGlobalVariableFwdDecl(
+    DIScope *Context, StringRef Name, StringRef LinkageName, DIFile *F,
+    unsigned LineNumber, DIType *Ty, bool isLocalToUnit, Constant *Val,
     MDNode *Decl) {
   checkGlobalVariableScope(Context);
 
-  return MDGlobalVariable::getTemporary(
-             VMContext, cast_or_null<MDScope>(Context), Name, LinkageName, F,
-             LineNumber, MDTypeRef::get(Ty), isLocalToUnit, false, Val,
-             cast_or_null<MDDerivedType>(Decl))
+  return DIGlobalVariable::getTemporary(
+             VMContext, cast_or_null<DIScope>(Context), Name, LinkageName, F,
+             LineNumber, DITypeRef::get(Ty), isLocalToUnit, false, Val,
+             cast_or_null<DIDerivedType>(Decl))
       .release();
 }
 
-MDLocalVariable *DIBuilder::createLocalVariable(
-    unsigned Tag, MDScope *Scope, StringRef Name, MDFile *File, unsigned LineNo,
-    MDType *Ty, bool AlwaysPreserve, unsigned Flags, unsigned ArgNo) {
+static DILocalVariable *createLocalVariable(
+    LLVMContext &VMContext,
+    DenseMap<MDNode *, std::vector<TrackingMDNodeRef>> &PreservedVariables,
+    DIScope *Scope, StringRef Name, unsigned ArgNo, DIFile *File,
+    unsigned LineNo, DIType *Ty, bool AlwaysPreserve, unsigned Flags) {
   // FIXME: Why getNonCompileUnitScope()?
   // FIXME: Why is "!Context" okay here?
-  // FIXME: WHy doesn't this check for a subprogram or lexical block (AFAICT
+  // FIXME: Why doesn't this check for a subprogram or lexical block (AFAICT
   // the only valid scopes)?
-  MDScope* Context = getNonCompileUnitScope(Scope);
+  DIScope *Context = getNonCompileUnitScope(Scope);
 
-  auto *Node = MDLocalVariable::get(
-      VMContext, Tag, cast_or_null<MDLocalScope>(Context), Name, File, LineNo,
-      MDTypeRef::get(Ty), ArgNo, Flags);
+  dwarf::Tag Tag = ArgNo ? dwarf::DW_TAG_arg_variable : dwarf::DW_TAG_auto_variable;
+  auto *Node = DILocalVariable::get(
+      VMContext, Tag, cast_or_null<DILocalScope>(Context), Name, File, LineNo,
+      DITypeRef::get(Ty), ArgNo, Flags);
   if (AlwaysPreserve) {
-    // The optimizer may remove local variable. If there is an interest
+    // The optimizer may remove local variables. If there is an interest
     // to preserve variable info in such situation then stash it in a
     // named mdnode.
-    MDSubprogram *Fn = getDISubprogram(Scope);
+    DISubprogram *Fn = getDISubprogram(Scope);
     assert(Fn && "Missing subprogram for local variable");
     PreservedVariables[Fn].emplace_back(Node);
   }
   return Node;
 }
 
-MDExpression* DIBuilder::createExpression(ArrayRef<uint64_t> Addr) {
-  return MDExpression::get(VMContext, Addr);
+DILocalVariable *DIBuilder::createAutoVariable(DIScope *Scope, StringRef Name,
+                                               DIFile *File, unsigned LineNo,
+                                               DIType *Ty, bool AlwaysPreserve,
+                                               unsigned Flags) {
+  return createLocalVariable(VMContext, PreservedVariables, Scope, Name,
+                             /* ArgNo */ 0, File, LineNo, Ty, AlwaysPreserve,
+                             Flags);
+}
+
+DILocalVariable *DIBuilder::createParameterVariable(
+    DIScope *Scope, StringRef Name, unsigned ArgNo, DIFile *File,
+    unsigned LineNo, DIType *Ty, bool AlwaysPreserve, unsigned Flags) {
+  assert(ArgNo && "Expected non-zero argument number for parameter");
+  return createLocalVariable(VMContext, PreservedVariables, Scope, Name, ArgNo,
+                             File, LineNo, Ty, AlwaysPreserve, Flags);
+}
+
+DIExpression *DIBuilder::createExpression(ArrayRef<uint64_t> Addr) {
+  return DIExpression::get(VMContext, Addr);
 }
 
-MDExpression* DIBuilder::createExpression(ArrayRef<int64_t> Signed) {
+DIExpression *DIBuilder::createExpression(ArrayRef<int64_t> Signed) {
   // TODO: Remove the callers of this signed version and delete.
   SmallVector<uint64_t, 8> Addr(Signed.begin(), Signed.end());
   return createExpression(Addr);
 }
 
-MDExpression* DIBuilder::createBitPieceExpression(unsigned OffsetInBytes,
-                                                 unsigned SizeInBytes) {
+DIExpression *DIBuilder::createBitPieceExpression(unsigned OffsetInBytes,
+                                                  unsigned SizeInBytes) {
   uint64_t Addr[] = {dwarf::DW_OP_bit_piece, OffsetInBytes, SizeInBytes};
-  return MDExpression::get(VMContext, Addr);
+  return DIExpression::get(VMContext, Addr);
 }
 
-MDSubprogram* DIBuilder::createFunction(DIScopeRef Context, StringRef Name,
-                                       StringRef LinkageName, MDFile* File,
-                                       unsigned LineNo, MDSubroutineType* Ty,
-                                       bool isLocalToUnit, bool isDefinition,
-                                       unsigned ScopeLine, unsigned Flags,
-                                       bool isOptimized, Function *Fn,
-                                       MDNode *TParams, MDNode *Decl) {
+DISubprogram *DIBuilder::createFunction(DIScopeRef Context, StringRef Name,
+                                        StringRef LinkageName, DIFile *File,
+                                        unsigned LineNo, DISubroutineType *Ty,
+                                        bool isLocalToUnit, bool isDefinition,
+                                        unsigned ScopeLine, unsigned Flags,
+                                        bool isOptimized, Function *Fn,
+                                        MDNode *TParams, MDNode *Decl) {
   // dragonegg does not generate identifier for types, so using an empty map
   // to resolve the context should be fine.
   DITypeIdentifierMap EmptyMap;
@@ -637,20 +676,20 @@ MDSubprogram* DIBuilder::createFunction(DIScopeRef Context, StringRef Name,
                         Flags, isOptimized, Fn, TParams, Decl);
 }
 
-MDSubprogram* DIBuilder::createFunction(MDScope * Context, StringRef Name,
-                                       StringRef LinkageName, MDFile* File,
-                                       unsigned LineNo, MDSubroutineType* Ty,
-                                       bool isLocalToUnit, bool isDefinition,
-                                       unsigned ScopeLine, unsigned Flags,
-                                       bool isOptimized, Function *Fn,
-                                       MDNode *TParams, MDNode *Decl) {
+DISubprogram *DIBuilder::createFunction(DIScope *Context, StringRef Name,
+                                        StringRef LinkageName, DIFile *File,
+                                        unsigned LineNo, DISubroutineType *Ty,
+                                        bool isLocalToUnit, bool isDefinition,
+                                        unsigned ScopeLine, unsigned Flags,
+                                        bool isOptimized, Function *Fn,
+                                        MDNode *TParams, MDNode *Decl) {
   assert(Ty->getTag() == dwarf::DW_TAG_subroutine_type &&
          "function types should be subroutines");
-  auto *Node = MDSubprogram::get(
-      VMContext, MDScopeRef::get(getNonCompileUnitScope(Context)), Name,
-      LinkageName, File, LineNo, Ty,
-      isLocalToUnit, isDefinition, ScopeLine, nullptr, 0, 0, Flags, isOptimized,
-      Fn, cast_or_null<MDTuple>(TParams), cast_or_null<MDSubprogram>(Decl),
+  auto *Node = DISubprogram::get(
+      VMContext, DIScopeRef::get(getNonCompileUnitScope(Context)), Name,
+      LinkageName, File, LineNo, Ty, isLocalToUnit, isDefinition, ScopeLine,
+      nullptr, 0, 0, Flags, isOptimized, Fn, cast_or_null<MDTuple>(TParams),
+      cast_or_null<DISubprogram>(Decl),
       MDTuple::getTemporary(VMContext, None).release());
 
   if (isDefinition)
@@ -659,27 +698,25 @@ MDSubprogram* DIBuilder::createFunction(MDScope * Context, StringRef Name,
   return Node;
 }
 
-MDSubprogram*
-DIBuilder::createTempFunctionFwdDecl(MDScope * Context, StringRef Name,
-                                     StringRef LinkageName, MDFile* File,
-                                     unsigned LineNo, MDSubroutineType* Ty,
-                                     bool isLocalToUnit, bool isDefinition,
-                                     unsigned ScopeLine, unsigned Flags,
-                                     bool isOptimized, Function *Fn,
-                                     MDNode *TParams, MDNode *Decl) {
-  return MDSubprogram::getTemporary(
-             VMContext, MDScopeRef::get(getNonCompileUnitScope(Context)), Name,
-             LinkageName, File, LineNo, Ty,
-             isLocalToUnit, isDefinition, ScopeLine, nullptr, 0, 0, Flags,
-             isOptimized, Fn, cast_or_null<MDTuple>(TParams),
-             cast_or_null<MDSubprogram>(Decl), nullptr).release();
-}
-
-MDSubprogram *
-DIBuilder::createMethod(MDScope *Context, StringRef Name, StringRef LinkageName,
-                        MDFile *F, unsigned LineNo, MDSubroutineType *Ty,
+DISubprogram *DIBuilder::createTempFunctionFwdDecl(
+    DIScope *Context, StringRef Name, StringRef LinkageName, DIFile *File,
+    unsigned LineNo, DISubroutineType *Ty, bool isLocalToUnit,
+    bool isDefinition, unsigned ScopeLine, unsigned Flags, bool isOptimized,
+    Function *Fn, MDNode *TParams, MDNode *Decl) {
+  return DISubprogram::getTemporary(
+             VMContext, DIScopeRef::get(getNonCompileUnitScope(Context)), Name,
+             LinkageName, File, LineNo, Ty, isLocalToUnit, isDefinition,
+             ScopeLine, nullptr, 0, 0, Flags, isOptimized, Fn,
+             cast_or_null<MDTuple>(TParams), cast_or_null<DISubprogram>(Decl),
+             nullptr)
+      .release();
+}
+
+DISubprogram *
+DIBuilder::createMethod(DIScope *Context, StringRef Name, StringRef LinkageName,
+                        DIFile *F, unsigned LineNo, DISubroutineType *Ty,
                         bool isLocalToUnit, bool isDefinition, unsigned VK,
-                        unsigned VIndex, MDType *VTableHolder, unsigned Flags,
+                        unsigned VIndex, DIType *VTableHolder, unsigned Flags,
                         bool isOptimized, Function *Fn, MDNode *TParam) {
   assert(Ty->getTag() == dwarf::DW_TAG_subroutine_type &&
          "function types should be subroutines");
@@ -687,10 +724,10 @@ DIBuilder::createMethod(MDScope *Context, StringRef Name, StringRef LinkageName,
          "Methods should have both a Context and a context that isn't "
          "the compile unit.");
   // FIXME: Do we want to use different scope/lines?
-  auto *SP = MDSubprogram::get(
-      VMContext, MDScopeRef::get(cast<MDScope>(Context)), Name, LinkageName, F,
+  auto *SP = DISubprogram::get(
+      VMContext, DIScopeRef::get(cast<DIScope>(Context)), Name, LinkageName, F,
       LineNo, Ty, isLocalToUnit, isDefinition, LineNo,
-      MDTypeRef::get(VTableHolder), VK, VIndex, Flags, isOptimized, Fn,
+      DITypeRef::get(VTableHolder), VK, VIndex, Flags, isOptimized, Fn,
       cast_or_null<MDTuple>(TParam), nullptr, nullptr);
 
   if (isDefinition)
@@ -699,23 +736,31 @@ DIBuilder::createMethod(MDScope *Context, StringRef Name, StringRef LinkageName,
   return SP;
 }
 
-MDNamespace* DIBuilder::createNameSpace(MDScope * Scope, StringRef Name,
-                                       MDFile* File, unsigned LineNo) {
-  return MDNamespace::get(VMContext, getNonCompileUnitScope(Scope), File, Name,
+DINamespace *DIBuilder::createNameSpace(DIScope *Scope, StringRef Name,
+                                        DIFile *File, unsigned LineNo) {
+  return DINamespace::get(VMContext, getNonCompileUnitScope(Scope), File, Name,
                           LineNo);
 }
 
-MDLexicalBlockFile* DIBuilder::createLexicalBlockFile(MDScope * Scope,
-                                                     MDFile* File,
-                                                     unsigned Discriminator) {
-  return MDLexicalBlockFile::get(VMContext, Scope, File, Discriminator);
+DIModule *DIBuilder::createModule(DIScope *Scope, StringRef Name,
+                                  StringRef ConfigurationMacros,
+                                  StringRef IncludePath,
+                                  StringRef ISysRoot) {
+ return DIModule::get(VMContext, getNonCompileUnitScope(Scope), Name,
+                      ConfigurationMacros, IncludePath, ISysRoot);
+}
+
+DILexicalBlockFile *DIBuilder::createLexicalBlockFile(DIScope *Scope,
+                                                      DIFile *File,
+                                                      unsigned Discriminator) {
+  return DILexicalBlockFile::get(VMContext, Scope, File, Discriminator);
 }
 
-MDLexicalBlock* DIBuilder::createLexicalBlock(MDScope * Scope, MDFile* File,
-                                             unsigned Line, unsigned Col) {
+DILexicalBlock *DIBuilder::createLexicalBlock(DIScope *Scope, DIFile *File,
+                                              unsigned Line, unsigned Col) {
   // Make these distinct, to avoid merging two lexical blocks on the same
   // file/line/column.
-  return MDLexicalBlock::getDistinct(VMContext, getNonCompileUnitScope(Scope),
+  return DILexicalBlock::getDistinct(VMContext, getNonCompileUnitScope(Scope),
                                      File, Line, Col);
 }
 
@@ -724,15 +769,15 @@ static Value *getDbgIntrinsicValueImpl(LLVMContext &VMContext, Value *V) {
   return MetadataAsValue::get(VMContext, ValueAsMetadata::get(V));
 }
 
-static Instruction *withDebugLoc(Instruction *I, const MDLocation *DL) {
-  I->setDebugLoc(const_cast<MDLocation *>(DL));
+static Instruction *withDebugLoc(Instruction *I, const DILocation *DL) {
+  I->setDebugLoc(const_cast<DILocation *>(DL));
   return I;
 }
 
-Instruction *DIBuilder::insertDeclare(Value *Storage, MDLocalVariable* VarInfo,
-                                      MDExpression* Expr, const MDLocation *DL,
+Instruction *DIBuilder::insertDeclare(Value *Storage, DILocalVariable *VarInfo,
+                                      DIExpression *Expr, const DILocation *DL,
                                       Instruction *InsertBefore) {
-  assert(VarInfo && "empty or invalid MDLocalVariable* passed to dbg.declare");
+  assert(VarInfo && "empty or invalid DILocalVariable* passed to dbg.declare");
   assert(DL && "Expected debug loc");
   assert(DL->getScope()->getSubprogram() ==
              VarInfo->getScope()->getSubprogram() &&
@@ -748,10 +793,10 @@ Instruction *DIBuilder::insertDeclare(Value *Storage, MDLocalVariable* VarInfo,
   return withDebugLoc(CallInst::Create(DeclareFn, Args, "", InsertBefore), DL);
 }
 
-Instruction *DIBuilder::insertDeclare(Value *Storage, MDLocalVariable* VarInfo,
-                                      MDExpression* Expr, const MDLocation *DL,
+Instruction *DIBuilder::insertDeclare(Value *Storage, DILocalVariable *VarInfo,
+                                      DIExpression *Expr, const DILocation *DL,
                                       BasicBlock *InsertAtEnd) {
-  assert(VarInfo && "empty or invalid MDLocalVariable* passed to dbg.declare");
+  assert(VarInfo && "empty or invalid DILocalVariable* passed to dbg.declare");
   assert(DL && "Expected debug loc");
   assert(DL->getScope()->getSubprogram() ==
              VarInfo->getScope()->getSubprogram() &&
@@ -773,12 +818,12 @@ Instruction *DIBuilder::insertDeclare(Value *Storage, MDLocalVariable* VarInfo,
 }
 
 Instruction *DIBuilder::insertDbgValueIntrinsic(Value *V, uint64_t Offset,
-                                                MDLocalVariable* VarInfo,
-                                                MDExpression* Expr,
-                                                const MDLocation *DL,
+                                                DILocalVariable *VarInfo,
+                                                DIExpression *Expr,
+                                                const DILocation *DL,
                                                 Instruction *InsertBefore) {
   assert(V && "no value passed to dbg.value");
-  assert(VarInfo && "empty or invalid MDLocalVariable* passed to dbg.value");
+  assert(VarInfo && "empty or invalid DILocalVariable* passed to dbg.value");
   assert(DL && "Expected debug loc");
   assert(DL->getScope()->getSubprogram() ==
              VarInfo->getScope()->getSubprogram() &&
@@ -796,12 +841,12 @@ Instruction *DIBuilder::insertDbgValueIntrinsic(Value *V, uint64_t Offset,
 }
 
 Instruction *DIBuilder::insertDbgValueIntrinsic(Value *V, uint64_t Offset,
-                                                MDLocalVariable* VarInfo,
-                                                MDExpression* Expr,
-                                                const MDLocation *DL,
+                                                DILocalVariable *VarInfo,
+                                                DIExpression *Expr,
+                                                const DILocation *DL,
                                                 BasicBlock *InsertAtEnd) {
   assert(V && "no value passed to dbg.value");
-  assert(VarInfo && "empty or invalid MDLocalVariable* passed to dbg.value");
+  assert(VarInfo && "empty or invalid DILocalVariable* passed to dbg.value");
   assert(DL && "Expected debug loc");
   assert(DL->getScope()->getSubprogram() ==
              VarInfo->getScope()->getSubprogram() &&
@@ -819,10 +864,11 @@ Instruction *DIBuilder::insertDbgValueIntrinsic(Value *V, uint64_t Offset,
   return withDebugLoc(CallInst::Create(ValueFn, Args, "", InsertAtEnd), DL);
 }
 
-void DIBuilder::replaceVTableHolder(MDCompositeType* &T, MDCompositeType* VTableHolder) {
+void DIBuilder::replaceVTableHolder(DICompositeType *&T,
+                                    DICompositeType *VTableHolder) {
   {
-    TypedTrackingMDRef<MDCompositeType> N(T);
-    N->replaceVTableHolder(MDTypeRef::get(VTableHolder));
+    TypedTrackingMDRef<DICompositeType> N(T);
+    N->replaceVTableHolder(DITypeRef::get(VTableHolder));
     T = N.get();
   }
 
@@ -838,14 +884,14 @@ void DIBuilder::replaceVTableHolder(MDCompositeType* &T, MDCompositeType* VTable
         trackIfUnresolved(N);
 }
 
-void DIBuilder::replaceArrays(MDCompositeType* &T, DIArray Elements,
-                              DIArray TParams) {
+void DIBuilder::replaceArrays(DICompositeType *&T, DINodeArray Elements,
+                              DINodeArray TParams) {
   {
-    TypedTrackingMDRef<MDCompositeType> N(T);
+    TypedTrackingMDRef<DICompositeType> N(T);
     if (Elements)
       N->replaceElements(Elements);
     if (TParams)
-      N->replaceTemplateParams(MDTemplateParameterArray(TParams));
+      N->replaceTemplateParams(DITemplateParameterArray(TParams));
     T = N.get();
   }
 
@@ -853,7 +899,7 @@ void DIBuilder::replaceArrays(MDCompositeType* &T, DIArray Elements,
   if (!T->isResolved())
     return;
 
-  // If "T" is resolved, it may be due to a self-reference cycle.  Track the
+  // If T is resolved, it may be due to a self-reference cycle.  Track the
   // arrays explicitly if they're unresolved, or else the cycles will be
   // orphaned.
   if (Elements)
diff --git a/lib/IR/DataLayout.cpp b/lib/IR/DataLayout.cpp
index 4d867efe1b3..80a59e05bc5 100644
--- a/lib/IR/DataLayout.cpp
+++ b/lib/IR/DataLayout.cpp
@@ -461,8 +461,8 @@ unsigned DataLayout::getAlignmentInfo(AlignTypeEnum AlignType,
       return ABIInfo ? Alignments[i].ABIAlign : Alignments[i].PrefAlign;
 
     // The best match so far depends on what we're looking for.
-     if (AlignType == INTEGER_ALIGN &&
-         Alignments[i].AlignType == INTEGER_ALIGN) {
+    if (AlignType == INTEGER_ALIGN &&
+        Alignments[i].AlignType == INTEGER_ALIGN) {
       // The "best match" for integers is the smallest size that is larger than
       // the BitWidth requested.
       if (Alignments[i].TypeBitWidth > BitWidth && (BestMatchIdx == -1 ||
diff --git a/lib/IR/DebugInfo.cpp b/lib/IR/DebugInfo.cpp
index 719c28b547f..55e46b36a14 100644
--- a/lib/IR/DebugInfo.cpp
+++ b/lib/IR/DebugInfo.cpp
@@ -33,13 +33,13 @@
 using namespace llvm;
 using namespace llvm::dwarf;
 
-DISubprogram llvm::getDISubprogram(const MDNode *Scope) {
-  if (auto *LocalScope = dyn_cast_or_null<MDLocalScope>(Scope))
+DISubprogram *llvm::getDISubprogram(const MDNode *Scope) {
+  if (auto *LocalScope = dyn_cast_or_null<DILocalScope>(Scope))
     return LocalScope->getSubprogram();
   return nullptr;
 }
 
-DISubprogram llvm::getDISubprogram(const Function *F) {
+DISubprogram *llvm::getDISubprogram(const Function *F) {
   // We look for the first instr that has a debug annotation leading back to F.
   for (auto &BB : *F) {
     auto Inst = std::find_if(BB.begin(), BB.end(), [](const Instruction &Inst) {
@@ -49,23 +49,8 @@ DISubprogram llvm::getDISubprogram(const Function *F) {
       continue;
     DebugLoc DLoc = Inst->getDebugLoc();
     const MDNode *Scope = DLoc.getInlinedAtScope();
-    DISubprogram Subprogram = getDISubprogram(Scope);
-    return Subprogram->describes(F) ? Subprogram : DISubprogram();
-  }
-
-  return DISubprogram();
-}
-
-DICompositeType llvm::getDICompositeType(DIType T) {
-  if (auto *C = dyn_cast_or_null<MDCompositeTypeBase>(T))
-    return C;
-
-  if (auto *D = dyn_cast_or_null<MDDerivedTypeBase>(T)) {
-    // This function is currently used by dragonegg and dragonegg does
-    // not generate identifier for types, so using an empty map to resolve
-    // DerivedFrom should be fine.
-    DITypeIdentifierMap EmptyMap;
-    return getDICompositeType(D->getBaseType().resolve(EmptyMap));
+    auto *Subprogram = getDISubprogram(Scope);
+    return Subprogram->describes(F) ? Subprogram : nullptr;
   }
 
   return nullptr;
@@ -75,12 +60,12 @@ DITypeIdentifierMap
 llvm::generateDITypeIdentifierMap(const NamedMDNode *CU_Nodes) {
   DITypeIdentifierMap Map;
   for (unsigned CUi = 0, CUe = CU_Nodes->getNumOperands(); CUi != CUe; ++CUi) {
-    auto *CU = cast<MDCompileUnit>(CU_Nodes->getOperand(CUi));
-    DIArray Retain = CU->getRetainedTypes();
+    auto *CU = cast<DICompileUnit>(CU_Nodes->getOperand(CUi));
+    DINodeArray Retain = CU->getRetainedTypes();
     for (unsigned Ti = 0, Te = Retain.size(); Ti != Te; ++Ti) {
-      if (!isa<MDCompositeType>(Retain[Ti]))
+      if (!isa<DICompositeType>(Retain[Ti]))
         continue;
-      auto *Ty = cast<MDCompositeType>(Retain[Ti]);
+      auto *Ty = cast<DICompositeType>(Retain[Ti]);
       if (MDString *TypeId = Ty->getRawIdentifier()) {
         // Definition has priority over declaration.
         // Try to insert (TypeId, Ty) to Map.
@@ -123,9 +108,9 @@ void DebugInfoFinder::processModule(const Module &M) {
   InitializeTypeMap(M);
   if (NamedMDNode *CU_Nodes = M.getNamedMetadata("llvm.dbg.cu")) {
     for (unsigned i = 0, e = CU_Nodes->getNumOperands(); i != e; ++i) {
-      DICompileUnit CU = cast<MDCompileUnit>(CU_Nodes->getOperand(i));
+      auto *CU = cast<DICompileUnit>(CU_Nodes->getOperand(i));
       addCompileUnit(CU);
-      for (DIGlobalVariable DIG : CU->getGlobalVariables()) {
+      for (auto *DIG : CU->getGlobalVariables()) {
         if (addGlobalVariable(DIG)) {
           processScope(DIG->getScope());
           processType(DIG->getType().resolve(TypeIdentifierMap));
@@ -137,20 +122,22 @@ void DebugInfoFinder::processModule(const Module &M) {
         processType(ET);
       for (auto *RT : CU->getRetainedTypes())
         processType(RT);
-      for (DIImportedEntity Import : CU->getImportedEntities()) {
+      for (auto *Import : CU->getImportedEntities()) {
         auto *Entity = Import->getEntity().resolve(TypeIdentifierMap);
-        if (auto *T = dyn_cast<MDType>(Entity))
+        if (auto *T = dyn_cast<DIType>(Entity))
           processType(T);
-        else if (auto *SP = dyn_cast<MDSubprogram>(Entity))
+        else if (auto *SP = dyn_cast<DISubprogram>(Entity))
           processSubprogram(SP);
-        else if (auto *NS = dyn_cast<MDNamespace>(Entity))
+        else if (auto *NS = dyn_cast<DINamespace>(Entity))
           processScope(NS->getScope());
+        else if (auto *M = dyn_cast<DIModule>(Entity))
+          processScope(M->getScope());
       }
     }
   }
 }
 
-void DebugInfoFinder::processLocation(const Module &M, DILocation Loc) {
+void DebugInfoFinder::processLocation(const Module &M, const DILocation *Loc) {
   if (!Loc)
     return;
   InitializeTypeMap(M);
@@ -158,61 +145,65 @@ void DebugInfoFinder::processLocation(const Module &M, DILocation Loc) {
   processLocation(M, Loc->getInlinedAt());
 }
 
-void DebugInfoFinder::processType(DIType DT) {
+void DebugInfoFinder::processType(DIType *DT) {
   if (!addType(DT))
     return;
   processScope(DT->getScope().resolve(TypeIdentifierMap));
-  if (auto *DCT = dyn_cast<MDCompositeTypeBase>(DT)) {
+  if (auto *ST = dyn_cast<DISubroutineType>(DT)) {
+    for (DITypeRef Ref : ST->getTypeArray())
+      processType(Ref.resolve(TypeIdentifierMap));
+    return;
+  }
+  if (auto *DCT = dyn_cast<DICompositeType>(DT)) {
     processType(DCT->getBaseType().resolve(TypeIdentifierMap));
-    if (auto *ST = dyn_cast<MDSubroutineType>(DCT)) {
-      for (MDTypeRef Ref : ST->getTypeArray())
-        processType(Ref.resolve(TypeIdentifierMap));
-      return;
-    }
     for (Metadata *D : DCT->getElements()) {
-      if (auto *T = dyn_cast<MDType>(D))
+      if (auto *T = dyn_cast<DIType>(D))
         processType(T);
-      else if (DISubprogram SP = dyn_cast<MDSubprogram>(D))
+      else if (auto *SP = dyn_cast<DISubprogram>(D))
         processSubprogram(SP);
     }
-  } else if (auto *DDT = dyn_cast<MDDerivedTypeBase>(DT)) {
+    return;
+  }
+  if (auto *DDT = dyn_cast<DIDerivedType>(DT)) {
     processType(DDT->getBaseType().resolve(TypeIdentifierMap));
   }
 }
 
-void DebugInfoFinder::processScope(DIScope Scope) {
+void DebugInfoFinder::processScope(DIScope *Scope) {
   if (!Scope)
     return;
-  if (DIType Ty = dyn_cast<MDType>(Scope)) {
+  if (auto *Ty = dyn_cast<DIType>(Scope)) {
     processType(Ty);
     return;
   }
-  if (DICompileUnit CU = dyn_cast<MDCompileUnit>(Scope)) {
+  if (auto *CU = dyn_cast<DICompileUnit>(Scope)) {
     addCompileUnit(CU);
     return;
   }
-  if (DISubprogram SP = dyn_cast<MDSubprogram>(Scope)) {
+  if (auto *SP = dyn_cast<DISubprogram>(Scope)) {
     processSubprogram(SP);
     return;
   }
   if (!addScope(Scope))
     return;
-  if (auto *LB = dyn_cast<MDLexicalBlockBase>(Scope)) {
+  if (auto *LB = dyn_cast<DILexicalBlockBase>(Scope)) {
     processScope(LB->getScope());
-  } else if (auto *NS = dyn_cast<MDNamespace>(Scope)) {
+  } else if (auto *NS = dyn_cast<DINamespace>(Scope)) {
     processScope(NS->getScope());
+  } else if (auto *M = dyn_cast<DIModule>(Scope)) {
+    processScope(M->getScope());
   }
 }
 
-void DebugInfoFinder::processSubprogram(DISubprogram SP) {
+void DebugInfoFinder::processSubprogram(DISubprogram *SP) {
   if (!addSubprogram(SP))
     return;
   processScope(SP->getScope().resolve(TypeIdentifierMap));
   processType(SP->getType());
   for (auto *Element : SP->getTemplateParams()) {
-    if (auto *TType = dyn_cast<MDTemplateTypeParameter>(Element)) {
+    if (auto *TType = dyn_cast<DITemplateTypeParameter>(Element)) {
       processType(TType->getType().resolve(TypeIdentifierMap));
-    } else if (auto *TVal = dyn_cast<MDTemplateValueParameter>(Element)) {
+    } else if (auto *TVal = dyn_cast<DITemplateValueParameter>(Element)) {
       processType(TVal->getType().resolve(TypeIdentifierMap));
     }
   }
@@ -220,12 +211,12 @@ void DebugInfoFinder::processSubprogram(DISubprogram SP) {
 
 void DebugInfoFinder::processDeclare(const Module &M,
                                      const DbgDeclareInst *DDI) {
-  MDNode *N = dyn_cast<MDNode>(DDI->getVariable());
+  auto *N = dyn_cast<MDNode>(DDI->getVariable());
   if (!N)
     return;
   InitializeTypeMap(M);
 
-  DIVariable DV = dyn_cast<MDLocalVariable>(N);
+  auto *DV = dyn_cast<DILocalVariable>(N);
   if (!DV)
     return;
 
@@ -236,12 +227,12 @@ void DebugInfoFinder::processDeclare(const Module &M,
 }
 
 void DebugInfoFinder::processValue(const Module &M, const DbgValueInst *DVI) {
-  MDNode *N = dyn_cast<MDNode>(DVI->getVariable());
+  auto *N = dyn_cast<MDNode>(DVI->getVariable());
   if (!N)
     return;
   InitializeTypeMap(M);
 
-  DIVariable DV = dyn_cast<MDLocalVariable>(N);
+  auto *DV = dyn_cast<DILocalVariable>(N);
   if (!DV)
     return;
 
@@ -251,18 +242,18 @@ void DebugInfoFinder::processValue(const Module &M, const DbgValueInst *DVI) {
   processType(DV->getType().resolve(TypeIdentifierMap));
 }
 
-bool DebugInfoFinder::addType(DIType DT) {
+bool DebugInfoFinder::addType(DIType *DT) {
   if (!DT)
     return false;
 
   if (!NodesSeen.insert(DT).second)
     return false;
 
-  TYs.push_back(DT);
+  TYs.push_back(const_cast<DIType *>(DT));
   return true;
 }
 
-bool DebugInfoFinder::addCompileUnit(DICompileUnit CU) {
+bool DebugInfoFinder::addCompileUnit(DICompileUnit *CU) {
   if (!CU)
     return false;
   if (!NodesSeen.insert(CU).second)
@@ -272,7 +263,7 @@ bool DebugInfoFinder::addCompileUnit(DICompileUnit CU) {
   return true;
 }
 
-bool DebugInfoFinder::addGlobalVariable(DIGlobalVariable DIG) {
+bool DebugInfoFinder::addGlobalVariable(DIGlobalVariable *DIG) {
   if (!DIG)
     return false;
 
@@ -283,7 +274,7 @@ bool DebugInfoFinder::addGlobalVariable(DIGlobalVariable DIG) {
   return true;
 }
 
-bool DebugInfoFinder::addSubprogram(DISubprogram SP) {
+bool DebugInfoFinder::addSubprogram(DISubprogram *SP) {
   if (!SP)
     return false;
 
@@ -294,7 +285,7 @@ bool DebugInfoFinder::addSubprogram(DISubprogram SP) {
   return true;
 }
 
-bool DebugInfoFinder::addScope(DIScope Scope) {
+bool DebugInfoFinder::addScope(DIScope *Scope) {
   if (!Scope)
     return false;
   // FIXME: Ocaml binding generates a scope with no content, we treat it
@@ -369,17 +360,17 @@ unsigned llvm::getDebugMetadataVersionFromModule(const Module &M) {
   return 0;
 }
 
-llvm::DenseMap<const llvm::Function *, llvm::DISubprogram>
+DenseMap<const llvm::Function *, DISubprogram *>
 llvm::makeSubprogramMap(const Module &M) {
-  DenseMap<const Function *, DISubprogram> R;
+  DenseMap<const Function *, DISubprogram *> R;
 
   NamedMDNode *CU_Nodes = M.getNamedMetadata("llvm.dbg.cu");
   if (!CU_Nodes)
     return R;
 
   for (MDNode *N : CU_Nodes->operands()) {
-    DICompileUnit CUNode = cast<MDCompileUnit>(N);
-    for (DISubprogram SP : CUNode->getSubprograms()) {
+    auto *CUNode = cast<DICompileUnit>(N);
+    for (auto *SP : CUNode->getSubprograms()) {
       if (Function *F = SP->getFunction())
         R.insert(std::make_pair(F, SP));
     }
diff --git a/lib/IR/DebugInfoMetadata.cpp b/lib/IR/DebugInfoMetadata.cpp
index f6f2ff2d20c..db6da05cfd8 100644
--- a/lib/IR/DebugInfoMetadata.cpp
+++ b/lib/IR/DebugInfoMetadata.cpp
@@ -19,9 +19,9 @@
 
 using namespace llvm;
 
-MDLocation::MDLocation(LLVMContext &C, StorageType Storage, unsigned Line,
+DILocation::DILocation(LLVMContext &C, StorageType Storage, unsigned Line,
                        unsigned Column, ArrayRef<Metadata *> MDs)
-    : MDNode(C, MDLocationKind, Storage, MDs) {
+    : MDNode(C, DILocationKind, Storage, MDs) {
   assert((MDs.size() == 1 || MDs.size() == 2) &&
          "Expected a scope and optional inlined-at");
 
@@ -38,7 +38,7 @@ static void adjustColumn(unsigned &Column) {
     Column = 0;
 }
 
-MDLocation *MDLocation::getImpl(LLVMContext &Context, unsigned Line,
+DILocation *DILocation::getImpl(LLVMContext &Context, unsigned Line,
                                 unsigned Column, Metadata *Scope,
                                 Metadata *InlinedAt, StorageType Storage,
                                 bool ShouldCreate) {
@@ -48,8 +48,8 @@ MDLocation *MDLocation::getImpl(LLVMContext &Context, unsigned Line,
   assert(Scope && "Expected scope");
   if (Storage == Uniqued) {
     if (auto *N =
-            getUniqued(Context.pImpl->MDLocations,
-                       MDLocationInfo::KeyTy(Line, Column, Scope, InlinedAt)))
+            getUniqued(Context.pImpl->DILocations,
+                       DILocationInfo::KeyTy(Line, Column, Scope, InlinedAt)))
       return N;
     if (!ShouldCreate)
       return nullptr;
@@ -62,11 +62,11 @@ MDLocation *MDLocation::getImpl(LLVMContext &Context, unsigned Line,
   if (InlinedAt)
     Ops.push_back(InlinedAt);
   return storeImpl(new (Ops.size())
-                       MDLocation(Context, Storage, Line, Column, Ops),
-                   Storage, Context.pImpl->MDLocations);
+                       DILocation(Context, Storage, Line, Column, Ops),
+                   Storage, Context.pImpl->DILocations);
 }
 
-unsigned MDLocation::computeNewDiscriminator() const {
+unsigned DILocation::computeNewDiscriminator() const {
   // FIXME: This seems completely wrong.
   //
   //  1. If two modules are generated in the same context, then the second
@@ -83,14 +83,14 @@ unsigned MDLocation::computeNewDiscriminator() const {
   return ++getContext().pImpl->DiscriminatorTable[Key];
 }
 
-unsigned DebugNode::getFlag(StringRef Flag) {
+unsigned DINode::getFlag(StringRef Flag) {
   return StringSwitch<unsigned>(Flag)
 #define HANDLE_DI_FLAG(ID, NAME) .Case("DIFlag" #NAME, Flag##NAME)
 #include "llvm/IR/DebugInfoFlags.def"
       .Default(0);
 }
 
-const char *DebugNode::getFlagString(unsigned Flag) {
+const char *DINode::getFlagString(unsigned Flag) {
   switch (Flag) {
   default:
     return "";
@@ -101,8 +101,8 @@ const char *DebugNode::getFlagString(unsigned Flag) {
   }
 }
 
-unsigned DebugNode::splitFlags(unsigned Flags,
-                               SmallVectorImpl<unsigned> &SplitFlags) {
+unsigned DINode::splitFlags(unsigned Flags,
+                            SmallVectorImpl<unsigned> &SplitFlags) {
   // Accessibility flags need to be specially handled, since they're packed
   // together.
   if (unsigned A = Flags & FlagAccessibility) {
@@ -125,33 +125,38 @@ unsigned DebugNode::splitFlags(unsigned Flags,
   return Flags;
 }
 
-MDScopeRef MDScope::getScope() const {
-  if (auto *T = dyn_cast<MDType>(this))
+DIScopeRef DIScope::getScope() const {
+  if (auto *T = dyn_cast<DIType>(this))
     return T->getScope();
 
-  if (auto *SP = dyn_cast<MDSubprogram>(this))
+  if (auto *SP = dyn_cast<DISubprogram>(this))
     return SP->getScope();
 
-  if (auto *LB = dyn_cast<MDLexicalBlockBase>(this))
-    return MDScopeRef(LB->getScope());
+  if (auto *LB = dyn_cast<DILexicalBlockBase>(this))
+    return DIScopeRef(LB->getScope());
 
-  if (auto *NS = dyn_cast<MDNamespace>(this))
-    return MDScopeRef(NS->getScope());
+  if (auto *NS = dyn_cast<DINamespace>(this))
+    return DIScopeRef(NS->getScope());
 
-  assert((isa<MDFile>(this) || isa<MDCompileUnit>(this)) &&
+  if (auto *M = dyn_cast<DIModule>(this))
+    return DIScopeRef(M->getScope());
+
+  assert((isa<DIFile>(this) || isa<DICompileUnit>(this)) &&
          "Unhandled type of scope.");
   return nullptr;
 }
 
-StringRef MDScope::getName() const {
-  if (auto *T = dyn_cast<MDType>(this))
+StringRef DIScope::getName() const {
+  if (auto *T = dyn_cast<DIType>(this))
     return T->getName();
-  if (auto *SP = dyn_cast<MDSubprogram>(this))
+  if (auto *SP = dyn_cast<DISubprogram>(this))
     return SP->getName();
-  if (auto *NS = dyn_cast<MDNamespace>(this))
+  if (auto *NS = dyn_cast<DINamespace>(this))
     return NS->getName();
-  assert((isa<MDLexicalBlockBase>(this) || isa<MDFile>(this) ||
-          isa<MDCompileUnit>(this)) &&
+  if (auto *M = dyn_cast<DIModule>(this))
+    return M->getName();
+  assert((isa<DILexicalBlockBase>(this) || isa<DIFile>(this) ||
+          isa<DICompileUnit>(this)) &&
          "Unhandled type of scope.");
   return "";
 }
@@ -168,15 +173,14 @@ static bool isCanonical(const MDString *S) {
 }
 #endif
 
-GenericDebugNode *GenericDebugNode::getImpl(LLVMContext &Context, unsigned Tag,
-                                            MDString *Header,
-                                            ArrayRef<Metadata *> DwarfOps,
-                                            StorageType Storage,
-                                            bool ShouldCreate) {
+GenericDINode *GenericDINode::getImpl(LLVMContext &Context, unsigned Tag,
+                                      MDString *Header,
+                                      ArrayRef<Metadata *> DwarfOps,
+                                      StorageType Storage, bool ShouldCreate) {
   unsigned Hash = 0;
   if (Storage == Uniqued) {
-    GenericDebugNodeInfo::KeyTy Key(Tag, getString(Header), DwarfOps);
-    if (auto *N = getUniqued(Context.pImpl->GenericDebugNodes, Key))
+    GenericDINodeInfo::KeyTy Key(Tag, getString(Header), DwarfOps);
+    if (auto *N = getUniqued(Context.pImpl->GenericDINodes, Key))
       return N;
     if (!ShouldCreate)
       return nullptr;
@@ -188,13 +192,13 @@ GenericDebugNode *GenericDebugNode::getImpl(LLVMContext &Context, unsigned Tag,
   // Use a nullptr for empty headers.
   assert(isCanonical(Header) && "Expected canonical MDString");
   Metadata *PreOps[] = {Header};
-  return storeImpl(new (DwarfOps.size() + 1) GenericDebugNode(
+  return storeImpl(new (DwarfOps.size() + 1) GenericDINode(
                        Context, Storage, Hash, Tag, PreOps, DwarfOps),
-                   Storage, Context.pImpl->GenericDebugNodes);
+                   Storage, Context.pImpl->GenericDINodes);
 }
 
-void GenericDebugNode::recalculateHash() {
-  setHash(GenericDebugNodeInfo::KeyTy::calculateHash(this));
+void GenericDINode::recalculateHash() {
+  setHash(GenericDINodeInfo::KeyTy::calculateHash(this));
 }
 
 #define UNWRAP_ARGS_IMPL(...) __VA_ARGS__
@@ -224,49 +228,49 @@ void GenericDebugNode::recalculateHash() {
                        CLASS(Context, Storage, OPS),                           \
                    Storage, Context.pImpl->CLASS##s)
 
-MDSubrange *MDSubrange::getImpl(LLVMContext &Context, int64_t Count, int64_t Lo,
+DISubrange *DISubrange::getImpl(LLVMContext &Context, int64_t Count, int64_t Lo,
                                 StorageType Storage, bool ShouldCreate) {
-  DEFINE_GETIMPL_LOOKUP(MDSubrange, (Count, Lo));
-  DEFINE_GETIMPL_STORE_NO_OPS(MDSubrange, (Count, Lo));
+  DEFINE_GETIMPL_LOOKUP(DISubrange, (Count, Lo));
+  DEFINE_GETIMPL_STORE_NO_OPS(DISubrange, (Count, Lo));
 }
 
-MDEnumerator *MDEnumerator::getImpl(LLVMContext &Context, int64_t Value,
+DIEnumerator *DIEnumerator::getImpl(LLVMContext &Context, int64_t Value,
                                     MDString *Name, StorageType Storage,
                                     bool ShouldCreate) {
   assert(isCanonical(Name) && "Expected canonical MDString");
-  DEFINE_GETIMPL_LOOKUP(MDEnumerator, (Value, getString(Name)));
+  DEFINE_GETIMPL_LOOKUP(DIEnumerator, (Value, getString(Name)));
   Metadata *Ops[] = {Name};
-  DEFINE_GETIMPL_STORE(MDEnumerator, (Value), Ops);
+  DEFINE_GETIMPL_STORE(DIEnumerator, (Value), Ops);
 }
 
-MDBasicType *MDBasicType::getImpl(LLVMContext &Context, unsigned Tag,
+DIBasicType *DIBasicType::getImpl(LLVMContext &Context, unsigned Tag,
                                   MDString *Name, uint64_t SizeInBits,
                                   uint64_t AlignInBits, unsigned Encoding,
                                   StorageType Storage, bool ShouldCreate) {
   assert(isCanonical(Name) && "Expected canonical MDString");
   DEFINE_GETIMPL_LOOKUP(
-      MDBasicType, (Tag, getString(Name), SizeInBits, AlignInBits, Encoding));
+      DIBasicType, (Tag, getString(Name), SizeInBits, AlignInBits, Encoding));
   Metadata *Ops[] = {nullptr, nullptr, Name};
-  DEFINE_GETIMPL_STORE(MDBasicType, (Tag, SizeInBits, AlignInBits, Encoding),
+  DEFINE_GETIMPL_STORE(DIBasicType, (Tag, SizeInBits, AlignInBits, Encoding),
                        Ops);
 }
 
-MDDerivedType *MDDerivedType::getImpl(
+DIDerivedType *DIDerivedType::getImpl(
     LLVMContext &Context, unsigned Tag, MDString *Name, Metadata *File,
     unsigned Line, Metadata *Scope, Metadata *BaseType, uint64_t SizeInBits,
     uint64_t AlignInBits, uint64_t OffsetInBits, unsigned Flags,
     Metadata *ExtraData, StorageType Storage, bool ShouldCreate) {
   assert(isCanonical(Name) && "Expected canonical MDString");
-  DEFINE_GETIMPL_LOOKUP(MDDerivedType, (Tag, getString(Name), File, Line, Scope,
+  DEFINE_GETIMPL_LOOKUP(DIDerivedType, (Tag, getString(Name), File, Line, Scope,
                                         BaseType, SizeInBits, AlignInBits,
                                         OffsetInBits, Flags, ExtraData));
   Metadata *Ops[] = {File, Scope, Name, BaseType, ExtraData};
   DEFINE_GETIMPL_STORE(
-      MDDerivedType, (Tag, Line, SizeInBits, AlignInBits, OffsetInBits, Flags),
+      DIDerivedType, (Tag, Line, SizeInBits, AlignInBits, OffsetInBits, Flags),
       Ops);
 }
 
-MDCompositeType *MDCompositeType::getImpl(
+DICompositeType *DICompositeType::getImpl(
     LLVMContext &Context, unsigned Tag, MDString *Name, Metadata *File,
     unsigned Line, Metadata *Scope, Metadata *BaseType, uint64_t SizeInBits,
     uint64_t AlignInBits, uint64_t OffsetInBits, unsigned Flags,
@@ -274,68 +278,68 @@ MDCompositeType *MDCompositeType::getImpl(
     Metadata *TemplateParams, MDString *Identifier, StorageType Storage,
     bool ShouldCreate) {
   assert(isCanonical(Name) && "Expected canonical MDString");
-  DEFINE_GETIMPL_LOOKUP(MDCompositeType,
+  DEFINE_GETIMPL_LOOKUP(DICompositeType,
                         (Tag, getString(Name), File, Line, Scope, BaseType,
                          SizeInBits, AlignInBits, OffsetInBits, Flags, Elements,
                          RuntimeLang, VTableHolder, TemplateParams,
                          getString(Identifier)));
   Metadata *Ops[] = {File,     Scope,        Name,           BaseType,
                      Elements, VTableHolder, TemplateParams, Identifier};
-  DEFINE_GETIMPL_STORE(MDCompositeType, (Tag, Line, RuntimeLang, SizeInBits,
+  DEFINE_GETIMPL_STORE(DICompositeType, (Tag, Line, RuntimeLang, SizeInBits,
                                          AlignInBits, OffsetInBits, Flags),
                        Ops);
 }
 
-MDSubroutineType *MDSubroutineType::getImpl(LLVMContext &Context,
+DISubroutineType *DISubroutineType::getImpl(LLVMContext &Context,
                                             unsigned Flags, Metadata *TypeArray,
                                             StorageType Storage,
                                             bool ShouldCreate) {
-  DEFINE_GETIMPL_LOOKUP(MDSubroutineType, (Flags, TypeArray));
-  Metadata *Ops[] = {nullptr,   nullptr, nullptr, nullptr,
-                     TypeArray, nullptr, nullptr, nullptr};
-  DEFINE_GETIMPL_STORE(MDSubroutineType, (Flags), Ops);
+  DEFINE_GETIMPL_LOOKUP(DISubroutineType, (Flags, TypeArray));
+  Metadata *Ops[] = {nullptr, nullptr, nullptr, TypeArray};
+  DEFINE_GETIMPL_STORE(DISubroutineType, (Flags), Ops);
 }
 
-MDFile *MDFile::getImpl(LLVMContext &Context, MDString *Filename,
+DIFile *DIFile::getImpl(LLVMContext &Context, MDString *Filename,
                         MDString *Directory, StorageType Storage,
                         bool ShouldCreate) {
   assert(isCanonical(Filename) && "Expected canonical MDString");
   assert(isCanonical(Directory) && "Expected canonical MDString");
-  DEFINE_GETIMPL_LOOKUP(MDFile, (getString(Filename), getString(Directory)));
+  DEFINE_GETIMPL_LOOKUP(DIFile, (getString(Filename), getString(Directory)));
   Metadata *Ops[] = {Filename, Directory};
-  DEFINE_GETIMPL_STORE_NO_CONSTRUCTOR_ARGS(MDFile, Ops);
+  DEFINE_GETIMPL_STORE_NO_CONSTRUCTOR_ARGS(DIFile, Ops);
 }
 
-MDCompileUnit *MDCompileUnit::getImpl(
+DICompileUnit *DICompileUnit::getImpl(
     LLVMContext &Context, unsigned SourceLanguage, Metadata *File,
     MDString *Producer, bool IsOptimized, MDString *Flags,
     unsigned RuntimeVersion, MDString *SplitDebugFilename,
     unsigned EmissionKind, Metadata *EnumTypes, Metadata *RetainedTypes,
     Metadata *Subprograms, Metadata *GlobalVariables,
-    Metadata *ImportedEntities, StorageType Storage, bool ShouldCreate) {
+    Metadata *ImportedEntities, uint64_t DWOId,
+    StorageType Storage, bool ShouldCreate) {
   assert(isCanonical(Producer) && "Expected canonical MDString");
   assert(isCanonical(Flags) && "Expected canonical MDString");
   assert(isCanonical(SplitDebugFilename) && "Expected canonical MDString");
   DEFINE_GETIMPL_LOOKUP(
-      MDCompileUnit,
+      DICompileUnit,
       (SourceLanguage, File, getString(Producer), IsOptimized, getString(Flags),
        RuntimeVersion, getString(SplitDebugFilename), EmissionKind, EnumTypes,
-       RetainedTypes, Subprograms, GlobalVariables, ImportedEntities));
+       RetainedTypes, Subprograms, GlobalVariables, ImportedEntities, DWOId));
   Metadata *Ops[] = {File, Producer, Flags, SplitDebugFilename, EnumTypes,
                      RetainedTypes, Subprograms, GlobalVariables,
                      ImportedEntities};
   DEFINE_GETIMPL_STORE(
-      MDCompileUnit,
-      (SourceLanguage, IsOptimized, RuntimeVersion, EmissionKind), Ops);
+      DICompileUnit,
+      (SourceLanguage, IsOptimized, RuntimeVersion, EmissionKind, DWOId), Ops);
 }
 
-MDSubprogram *MDLocalScope::getSubprogram() const {
-  if (auto *Block = dyn_cast<MDLexicalBlockBase>(this))
+DISubprogram *DILocalScope::getSubprogram() const {
+  if (auto *Block = dyn_cast<DILexicalBlockBase>(this))
     return Block->getScope()->getSubprogram();
-  return const_cast<MDSubprogram *>(cast<MDSubprogram>(this));
+  return const_cast<DISubprogram *>(cast<DISubprogram>(this));
 }
 
-MDSubprogram *MDSubprogram::getImpl(
+DISubprogram *DISubprogram::getImpl(
     LLVMContext &Context, Metadata *Scope, MDString *Name,
     MDString *LinkageName, Metadata *File, unsigned Line, Metadata *Type,
     bool IsLocalToUnit, bool IsDefinition, unsigned ScopeLine,
@@ -345,7 +349,7 @@ MDSubprogram *MDSubprogram::getImpl(
     StorageType Storage, bool ShouldCreate) {
   assert(isCanonical(Name) && "Expected canonical MDString");
   assert(isCanonical(LinkageName) && "Expected canonical MDString");
-  DEFINE_GETIMPL_LOOKUP(MDSubprogram,
+  DEFINE_GETIMPL_LOOKUP(DISubprogram,
                         (Scope, getString(Name), getString(LinkageName), File,
                          Line, Type, IsLocalToUnit, IsDefinition, ScopeLine,
                          ContainingType, Virtuality, VirtualIndex, Flags,
@@ -354,18 +358,18 @@ MDSubprogram *MDSubprogram::getImpl(
   Metadata *Ops[] = {File,           Scope,       Name,           Name,
                      LinkageName,    Type,        ContainingType, Function,
                      TemplateParams, Declaration, Variables};
-  DEFINE_GETIMPL_STORE(MDSubprogram,
+  DEFINE_GETIMPL_STORE(DISubprogram,
                        (Line, ScopeLine, Virtuality, VirtualIndex, Flags,
                         IsLocalToUnit, IsDefinition, IsOptimized),
                        Ops);
 }
 
-Function *MDSubprogram::getFunction() const {
+Function *DISubprogram::getFunction() const {
   // FIXME: Should this be looking through bitcasts?
   return dyn_cast_or_null<Function>(getFunctionConstant());
 }
 
-bool MDSubprogram::describes(const Function *F) const {
+bool DISubprogram::describes(const Function *F) const {
   assert(F && "Invalid function");
   if (F == getFunction())
     return true;
@@ -375,64 +379,76 @@ bool MDSubprogram::describes(const Function *F) const {
   return F->getName() == Name;
 }
 
-void MDSubprogram::replaceFunction(Function *F) {
+void DISubprogram::replaceFunction(Function *F) {
   replaceFunction(F ? ConstantAsMetadata::get(F)
                     : static_cast<ConstantAsMetadata *>(nullptr));
 }
 
-MDLexicalBlock *MDLexicalBlock::getImpl(LLVMContext &Context, Metadata *Scope,
+DILexicalBlock *DILexicalBlock::getImpl(LLVMContext &Context, Metadata *Scope,
                                         Metadata *File, unsigned Line,
                                         unsigned Column, StorageType Storage,
                                         bool ShouldCreate) {
   assert(Scope && "Expected scope");
-  DEFINE_GETIMPL_LOOKUP(MDLexicalBlock, (Scope, File, Line, Column));
+  DEFINE_GETIMPL_LOOKUP(DILexicalBlock, (Scope, File, Line, Column));
   Metadata *Ops[] = {File, Scope};
-  DEFINE_GETIMPL_STORE(MDLexicalBlock, (Line, Column), Ops);
+  DEFINE_GETIMPL_STORE(DILexicalBlock, (Line, Column), Ops);
 }
 
-MDLexicalBlockFile *MDLexicalBlockFile::getImpl(LLVMContext &Context,
+DILexicalBlockFile *DILexicalBlockFile::getImpl(LLVMContext &Context,
                                                 Metadata *Scope, Metadata *File,
                                                 unsigned Discriminator,
                                                 StorageType Storage,
                                                 bool ShouldCreate) {
   assert(Scope && "Expected scope");
-  DEFINE_GETIMPL_LOOKUP(MDLexicalBlockFile, (Scope, File, Discriminator));
+  DEFINE_GETIMPL_LOOKUP(DILexicalBlockFile, (Scope, File, Discriminator));
   Metadata *Ops[] = {File, Scope};
-  DEFINE_GETIMPL_STORE(MDLexicalBlockFile, (Discriminator), Ops);
+  DEFINE_GETIMPL_STORE(DILexicalBlockFile, (Discriminator), Ops);
 }
 
-MDNamespace *MDNamespace::getImpl(LLVMContext &Context, Metadata *Scope,
+DINamespace *DINamespace::getImpl(LLVMContext &Context, Metadata *Scope,
                                   Metadata *File, MDString *Name, unsigned Line,
                                   StorageType Storage, bool ShouldCreate) {
   assert(isCanonical(Name) && "Expected canonical MDString");
-  DEFINE_GETIMPL_LOOKUP(MDNamespace, (Scope, File, getString(Name), Line));
+  DEFINE_GETIMPL_LOOKUP(DINamespace, (Scope, File, getString(Name), Line));
   Metadata *Ops[] = {File, Scope, Name};
-  DEFINE_GETIMPL_STORE(MDNamespace, (Line), Ops);
+  DEFINE_GETIMPL_STORE(DINamespace, (Line), Ops);
 }
 
-MDTemplateTypeParameter *MDTemplateTypeParameter::getImpl(LLVMContext &Context,
+DIModule *DIModule::getImpl(LLVMContext &Context, Metadata *Scope,
+                            MDString *Name, MDString *ConfigurationMacros,
+                            MDString *IncludePath, MDString *ISysRoot,
+                            StorageType Storage, bool ShouldCreate) {
+  assert(isCanonical(Name) && "Expected canonical MDString");
+  DEFINE_GETIMPL_LOOKUP(DIModule,
+    (Scope, getString(Name), getString(ConfigurationMacros),
+     getString(IncludePath), getString(ISysRoot)));
+  Metadata *Ops[] = {Scope, Name, ConfigurationMacros, IncludePath, ISysRoot};
+  DEFINE_GETIMPL_STORE_NO_CONSTRUCTOR_ARGS(DIModule, Ops);
+}
+
+DITemplateTypeParameter *DITemplateTypeParameter::getImpl(LLVMContext &Context,
                                                           MDString *Name,
                                                           Metadata *Type,
                                                           StorageType Storage,
                                                           bool ShouldCreate) {
   assert(isCanonical(Name) && "Expected canonical MDString");
-  DEFINE_GETIMPL_LOOKUP(MDTemplateTypeParameter, (getString(Name), Type));
+  DEFINE_GETIMPL_LOOKUP(DITemplateTypeParameter, (getString(Name), Type));
   Metadata *Ops[] = {Name, Type};
-  DEFINE_GETIMPL_STORE_NO_CONSTRUCTOR_ARGS(MDTemplateTypeParameter, Ops);
+  DEFINE_GETIMPL_STORE_NO_CONSTRUCTOR_ARGS(DITemplateTypeParameter, Ops);
 }
 
-MDTemplateValueParameter *MDTemplateValueParameter::getImpl(
+DITemplateValueParameter *DITemplateValueParameter::getImpl(
     LLVMContext &Context, unsigned Tag, MDString *Name, Metadata *Type,
     Metadata *Value, StorageType Storage, bool ShouldCreate) {
   assert(isCanonical(Name) && "Expected canonical MDString");
-  DEFINE_GETIMPL_LOOKUP(MDTemplateValueParameter,
+  DEFINE_GETIMPL_LOOKUP(DITemplateValueParameter,
                         (Tag, getString(Name), Type, Value));
   Metadata *Ops[] = {Name, Type, Value};
-  DEFINE_GETIMPL_STORE(MDTemplateValueParameter, (Tag), Ops);
+  DEFINE_GETIMPL_STORE(DITemplateValueParameter, (Tag), Ops);
 }
 
-MDGlobalVariable *
-MDGlobalVariable::getImpl(LLVMContext &Context, Metadata *Scope, MDString *Name,
+DIGlobalVariable *
+DIGlobalVariable::getImpl(LLVMContext &Context, Metadata *Scope, MDString *Name,
                           MDString *LinkageName, Metadata *File, unsigned Line,
                           Metadata *Type, bool IsLocalToUnit, bool IsDefinition,
                           Metadata *Variable,
@@ -440,44 +456,41 @@ MDGlobalVariable::getImpl(LLVMContext &Context, Metadata *Scope, MDString *Name,
                           StorageType Storage, bool ShouldCreate) {
   assert(isCanonical(Name) && "Expected canonical MDString");
   assert(isCanonical(LinkageName) && "Expected canonical MDString");
-  DEFINE_GETIMPL_LOOKUP(MDGlobalVariable,
+  DEFINE_GETIMPL_LOOKUP(DIGlobalVariable,
                         (Scope, getString(Name), getString(LinkageName), File,
                          Line, Type, IsLocalToUnit, IsDefinition, Variable,
                          StaticDataMemberDeclaration));
   Metadata *Ops[] = {Scope, Name,        File,     Type,
                      Name,  LinkageName, Variable, StaticDataMemberDeclaration};
-  DEFINE_GETIMPL_STORE(MDGlobalVariable, (Line, IsLocalToUnit, IsDefinition),
+  DEFINE_GETIMPL_STORE(DIGlobalVariable, (Line, IsLocalToUnit, IsDefinition),
                        Ops);
 }
 
-MDLocalVariable *MDLocalVariable::getImpl(LLVMContext &Context, unsigned Tag,
+DILocalVariable *DILocalVariable::getImpl(LLVMContext &Context, unsigned Tag,
                                           Metadata *Scope, MDString *Name,
                                           Metadata *File, unsigned Line,
                                           Metadata *Type, unsigned Arg,
                                           unsigned Flags, StorageType Storage,
                                           bool ShouldCreate) {
-  // Truncate Arg to 8 bits.
-  //
-  // FIXME: This is gross (and should be changed to an assert or removed), but
-  // it matches historical behaviour for now.
-  Arg &= (1u << 8) - 1;
+  // 64K ought to be enough for any frontend.
+  assert(Arg <= UINT16_MAX && "Expected argument number to fit in 16-bits");
 
   assert(Scope && "Expected scope");
   assert(isCanonical(Name) && "Expected canonical MDString");
-  DEFINE_GETIMPL_LOOKUP(MDLocalVariable, (Tag, Scope, getString(Name), File,
+  DEFINE_GETIMPL_LOOKUP(DILocalVariable, (Tag, Scope, getString(Name), File,
                                           Line, Type, Arg, Flags));
   Metadata *Ops[] = {Scope, Name, File, Type};
-  DEFINE_GETIMPL_STORE(MDLocalVariable, (Tag, Line, Arg, Flags), Ops);
+  DEFINE_GETIMPL_STORE(DILocalVariable, (Tag, Line, Arg, Flags), Ops);
 }
 
-MDExpression *MDExpression::getImpl(LLVMContext &Context,
+DIExpression *DIExpression::getImpl(LLVMContext &Context,
                                     ArrayRef<uint64_t> Elements,
                                     StorageType Storage, bool ShouldCreate) {
-  DEFINE_GETIMPL_LOOKUP(MDExpression, (Elements));
-  DEFINE_GETIMPL_STORE_NO_OPS(MDExpression, (Elements));
+  DEFINE_GETIMPL_LOOKUP(DIExpression, (Elements));
+  DEFINE_GETIMPL_STORE_NO_OPS(DIExpression, (Elements));
 }
 
-unsigned MDExpression::ExprOperand::getSize() const {
+unsigned DIExpression::ExprOperand::getSize() const {
   switch (getOp()) {
   case dwarf::DW_OP_bit_piece:
     return 3;
@@ -488,7 +501,7 @@ unsigned MDExpression::ExprOperand::getSize() const {
   }
 }
 
-bool MDExpression::isValid() const {
+bool DIExpression::isValid() const {
   for (auto I = expr_op_begin(), E = expr_op_end(); I != E; ++I) {
     // Check that there's space for the operand.
     if (I->get() + I->getSize() > E->get())
@@ -509,7 +522,7 @@ bool MDExpression::isValid() const {
   return true;
 }
 
-bool MDExpression::isBitPiece() const {
+bool DIExpression::isBitPiece() const {
   assert(isValid() && "Expected valid expression");
   if (unsigned N = getNumElements())
     if (N >= 3)
@@ -517,38 +530,38 @@ bool MDExpression::isBitPiece() const {
   return false;
 }
 
-uint64_t MDExpression::getBitPieceOffset() const {
+uint64_t DIExpression::getBitPieceOffset() const {
   assert(isBitPiece() && "Expected bit piece");
   return getElement(getNumElements() - 2);
 }
 
-uint64_t MDExpression::getBitPieceSize() const {
+uint64_t DIExpression::getBitPieceSize() const {
   assert(isBitPiece() && "Expected bit piece");
   return getElement(getNumElements() - 1);
 }
 
-MDObjCProperty *MDObjCProperty::getImpl(
+DIObjCProperty *DIObjCProperty::getImpl(
     LLVMContext &Context, MDString *Name, Metadata *File, unsigned Line,
     MDString *GetterName, MDString *SetterName, unsigned Attributes,
     Metadata *Type, StorageType Storage, bool ShouldCreate) {
   assert(isCanonical(Name) && "Expected canonical MDString");
   assert(isCanonical(GetterName) && "Expected canonical MDString");
   assert(isCanonical(SetterName) && "Expected canonical MDString");
-  DEFINE_GETIMPL_LOOKUP(MDObjCProperty,
+  DEFINE_GETIMPL_LOOKUP(DIObjCProperty,
                         (getString(Name), File, Line, getString(GetterName),
                          getString(SetterName), Attributes, Type));
   Metadata *Ops[] = {Name, File, GetterName, SetterName, Type};
-  DEFINE_GETIMPL_STORE(MDObjCProperty, (Line, Attributes), Ops);
+  DEFINE_GETIMPL_STORE(DIObjCProperty, (Line, Attributes), Ops);
 }
 
-MDImportedEntity *MDImportedEntity::getImpl(LLVMContext &Context, unsigned Tag,
+DIImportedEntity *DIImportedEntity::getImpl(LLVMContext &Context, unsigned Tag,
                                             Metadata *Scope, Metadata *Entity,
                                             unsigned Line, MDString *Name,
                                             StorageType Storage,
                                             bool ShouldCreate) {
   assert(isCanonical(Name) && "Expected canonical MDString");
-  DEFINE_GETIMPL_LOOKUP(MDImportedEntity,
+  DEFINE_GETIMPL_LOOKUP(DIImportedEntity,
                         (Tag, Scope, Entity, Line, getString(Name)));
   Metadata *Ops[] = {Scope, Entity, Name};
-  DEFINE_GETIMPL_STORE(MDImportedEntity, (Tag, Line), Ops);
+  DEFINE_GETIMPL_STORE(DIImportedEntity, (Tag, Line), Ops);
 }
diff --git a/lib/IR/DebugLoc.cpp b/lib/IR/DebugLoc.cpp
index 4cf7e9e34e1..72d5c0e6188 100644
--- a/lib/IR/DebugLoc.cpp
+++ b/lib/IR/DebugLoc.cpp
@@ -16,11 +16,11 @@ using namespace llvm;
 //===----------------------------------------------------------------------===//
 // DebugLoc Implementation
 //===----------------------------------------------------------------------===//
-DebugLoc::DebugLoc(const MDLocation *L) : Loc(const_cast<MDLocation *>(L)) {}
+DebugLoc::DebugLoc(const DILocation *L) : Loc(const_cast<DILocation *>(L)) {}
 DebugLoc::DebugLoc(const MDNode *L) : Loc(const_cast<MDNode *>(L)) {}
 
-MDLocation *DebugLoc::get() const {
-  return cast_or_null<MDLocation>(Loc.get());
+DILocation *DebugLoc::get() const {
+  return cast_or_null<DILocation>(Loc.get());
 }
 
 unsigned DebugLoc::getLine() const {
@@ -38,19 +38,19 @@ MDNode *DebugLoc::getScope() const {
   return get()->getScope();
 }
 
-MDLocation *DebugLoc::getInlinedAt() const {
+DILocation *DebugLoc::getInlinedAt() const {
   assert(get() && "Expected valid DebugLoc");
   return get()->getInlinedAt();
 }
 
 MDNode *DebugLoc::getInlinedAtScope() const {
-  return cast<MDLocation>(Loc)->getInlinedAtScope();
+  return cast<DILocation>(Loc)->getInlinedAtScope();
 }
 
 DebugLoc DebugLoc::getFnDebugLoc() const {
-  // FIXME: Add a method on \a MDLocation that does this work.
+  // FIXME: Add a method on \a DILocation that does this work.
   const MDNode *Scope = getInlinedAtScope();
-  if (DISubprogram SP = getDISubprogram(Scope))
+  if (auto *SP = getDISubprogram(Scope))
     return DebugLoc::get(SP->getScopeLine(), 0, SP);
 
   return DebugLoc();
@@ -62,7 +62,7 @@ DebugLoc DebugLoc::get(unsigned Line, unsigned Col, const MDNode *Scope,
   if (!Scope)
     return DebugLoc();
 
-  return MDLocation::get(Scope->getContext(), Line, Col,
+  return DILocation::get(Scope->getContext(), Line, Col,
                          const_cast<MDNode *>(Scope),
                          const_cast<MDNode *>(InlinedAt));
 }
@@ -88,7 +88,7 @@ void DebugLoc::print(raw_ostream &OS) const {
     return;
 
   // Print source line info.
-  auto *Scope = cast<MDScope>(getScope());
+  auto *Scope = cast<DIScope>(getScope());
   OS << Scope->getFilename();
   OS << ':' << getLine();
   if (getCol() != 0)
diff --git a/lib/IR/DiagnosticInfo.cpp b/lib/IR/DiagnosticInfo.cpp
index 91635f67959..6612681d689 100644
--- a/lib/IR/DiagnosticInfo.cpp
+++ b/lib/IR/DiagnosticInfo.cpp
@@ -22,9 +22,9 @@
 #include "llvm/IR/Instruction.h"
 #include "llvm/IR/Metadata.h"
 #include "llvm/IR/Module.h"
-#include "llvm/Support/Atomic.h"
 #include "llvm/Support/CommandLine.h"
 #include "llvm/Support/Regex.h"
+#include <atomic>
 #include <string>
 
 using namespace llvm;
@@ -49,7 +49,7 @@ struct PassRemarksOpt {
                                "' in -pass-remarks: " + RegexError,
                            false);
     }
-  };
+  }
 };
 
 static PassRemarksOpt PassRemarksOptLoc;
@@ -87,8 +87,8 @@ PassRemarksAnalysis(
 }
 
 int llvm::getNextAvailablePluginDiagnosticKind() {
-  static sys::cas_flag PluginKindID = DK_FirstPluginKind;
-  return (int)sys::AtomicIncrement(&PluginKindID);
+  static std::atomic<int> PluginKindID(DK_FirstPluginKind);
+  return ++PluginKindID;
 }
 
 DiagnosticInfoInlineAsm::DiagnosticInfoInlineAsm(const Instruction &I,
@@ -135,7 +135,8 @@ bool DiagnosticInfoOptimizationBase::isLocationAvailable() const {
 void DiagnosticInfoOptimizationBase::getLocation(StringRef *Filename,
                                                  unsigned *Line,
                                                  unsigned *Column) const {
-  MDLocation *L = getDebugLoc();
+  DILocation *L = getDebugLoc();
+  assert(L != nullptr && "debug location is invalid");
   *Filename = L->getFilename();
   *Line = L->getLine();
   *Column = L->getColumn();
@@ -169,6 +170,10 @@ bool DiagnosticInfoOptimizationRemarkAnalysis::isEnabled() const {
          PassRemarksAnalysisOptLoc.Pattern->match(getPassName());
 }
 
+void DiagnosticInfoMIRParser::print(DiagnosticPrinter &DP) const {
+  DP << Diagnostic;
+}
+
 void llvm::emitOptimizationRemark(LLVMContext &Ctx, const char *PassName,
                                   const Function &Fn, const DebugLoc &DLoc,
                                   const Twine &Msg) {
diff --git a/lib/IR/DiagnosticPrinter.cpp b/lib/IR/DiagnosticPrinter.cpp
index f25fc20a197..659ff49d623 100644
--- a/lib/IR/DiagnosticPrinter.cpp
+++ b/lib/IR/DiagnosticPrinter.cpp
@@ -16,6 +16,7 @@
 #include "llvm/IR/Module.h"
 #include "llvm/IR/Value.h"
 #include "llvm/Support/raw_ostream.h"
+#include "llvm/Support/SourceMgr.h"
 
 using namespace llvm;
 
@@ -105,3 +106,12 @@ DiagnosticPrinter &DiagnosticPrinterRawOStream::operator<<(const Module &M) {
   Stream << M.getModuleIdentifier();
   return *this;
 }
+
+// Other types.
+DiagnosticPrinter &DiagnosticPrinterRawOStream::
+operator<<(const SMDiagnostic &Diag) {
+  // We don't have to print the SMDiagnostic kind, as the diagnostic severity
+  // is printed by the diagnostic handler.
+  Diag.print("", Stream, /*ShowColors=*/true, /*ShowKindLabel=*/false);
+  return *this;
+}
diff --git a/lib/IR/Dominators.cpp b/lib/IR/Dominators.cpp
index 9b6ff1eb136..b6a8bbcbe5f 100644
--- a/lib/IR/Dominators.cpp
+++ b/lib/IR/Dominators.cpp
@@ -62,18 +62,14 @@ bool BasicBlockEdge::isSingleEdge() const {
 //
 //===----------------------------------------------------------------------===//
 
-TEMPLATE_INSTANTIATION(class llvm::DomTreeNodeBase<BasicBlock>);
-TEMPLATE_INSTANTIATION(class llvm::DominatorTreeBase<BasicBlock>);
-
-#define LLVM_COMMA ,
-TEMPLATE_INSTANTIATION(void llvm::Calculate<Function LLVM_COMMA BasicBlock *>(
-    DominatorTreeBase<GraphTraits<BasicBlock *>::NodeType> &DT LLVM_COMMA
-        Function &F));
-TEMPLATE_INSTANTIATION(
-    void llvm::Calculate<Function LLVM_COMMA Inverse<BasicBlock *> >(
-        DominatorTreeBase<GraphTraits<Inverse<BasicBlock *> >::NodeType> &DT
-            LLVM_COMMA Function &F));
-#undef LLVM_COMMA
+template class llvm::DomTreeNodeBase<BasicBlock>;
+template class llvm::DominatorTreeBase<BasicBlock>;
+
+template void llvm::Calculate<Function, BasicBlock *>(
+    DominatorTreeBase<GraphTraits<BasicBlock *>::NodeType> &DT, Function &F);
+template void llvm::Calculate<Function, Inverse<BasicBlock *>>(
+    DominatorTreeBase<GraphTraits<Inverse<BasicBlock *>>::NodeType> &DT,
+    Function &F);
 
 // dominates - Return true if Def dominates a use in User. This performs
 // the special checks necessary if Def and User are in the same basic block.
@@ -282,9 +278,6 @@ bool DominatorTree::isReachableFromEntry(const Use &U) const {
 }
 
 void DominatorTree::verifyDomTree() const {
-  if (!VerifyDomInfo)
-    return;
-
   Function &F = *getRoot()->getParent();
 
   DominatorTree OtherDT;
@@ -350,7 +343,10 @@ bool DominatorTreeWrapperPass::runOnFunction(Function &F) {
   return false;
 }
 
-void DominatorTreeWrapperPass::verifyAnalysis() const { DT.verifyDomTree(); }
+void DominatorTreeWrapperPass::verifyAnalysis() const {
+    if (VerifyDomInfo)
+      DT.verifyDomTree();
+}
 
 void DominatorTreeWrapperPass::print(raw_ostream &OS, const Module *) const {
   DT.print(OS);
diff --git a/lib/IR/Function.cpp b/lib/IR/Function.cpp
index d3a09348220..b50ad1262c6 100644
--- a/lib/IR/Function.cpp
+++ b/lib/IR/Function.cpp
@@ -19,10 +19,13 @@
 #include "llvm/ADT/StringExtras.h"
 #include "llvm/CodeGen/ValueTypes.h"
 #include "llvm/IR/CallSite.h"
+#include "llvm/IR/Constants.h"
 #include "llvm/IR/DerivedTypes.h"
 #include "llvm/IR/InstIterator.h"
 #include "llvm/IR/IntrinsicInst.h"
 #include "llvm/IR/LLVMContext.h"
+#include "llvm/IR/MDBuilder.h"
+#include "llvm/IR/Metadata.h"
 #include "llvm/IR/Module.h"
 #include "llvm/Support/ManagedStatic.h"
 #include "llvm/Support/RWMutex.h"
@@ -117,6 +120,12 @@ uint64_t Argument::getDereferenceableBytes() const {
   return getParent()->getDereferenceableBytes(getArgNo()+1);
 }
 
+uint64_t Argument::getDereferenceableOrNullBytes() const {
+  assert(getType()->isPointerTy() &&
+         "Only pointers have dereferenceable bytes");
+  return getParent()->getDereferenceableOrNullBytes(getArgNo()+1);
+}
+
 /// hasNestAttr - Return true if this argument has the nest attribute on
 /// it in its containing function.
 bool Argument::hasNestAttr() const {
@@ -145,10 +154,8 @@ bool Argument::hasNoCaptureAttr() const {
 /// it in its containing function.
 bool Argument::hasStructRetAttr() const {
   if (!getType()->isPointerTy()) return false;
-  if (this != getParent()->arg_begin())
-    return false; // StructRet param must be first param
   return getParent()->getAttributes().
-    hasAttribute(1, Attribute::StructRet);
+    hasAttribute(getArgNo()+1, Attribute::StructRet);
 }
 
 /// hasReturnedAttr - Return true if this argument has the returned attribute on
@@ -206,10 +213,12 @@ void Argument::removeAttr(AttributeSet AS) {
 //===----------------------------------------------------------------------===//
 
 bool Function::isMaterializable() const {
-  return getGlobalObjectSubClassData();
+  return getGlobalObjectSubClassData() & IsMaterializableBit;
 }
 
-void Function::setIsMaterializable(bool V) { setGlobalObjectSubClassData(V); }
+void Function::setIsMaterializable(bool V) {
+  setGlobalObjectBit(IsMaterializableBit, V);
+}
 
 LLVMContext &Function::getContext() const {
   return getType()->getContext();
@@ -239,12 +248,12 @@ void Function::eraseFromParent() {
 
 Function::Function(FunctionType *Ty, LinkageTypes Linkage, const Twine &name,
                    Module *ParentModule)
-    : GlobalObject(PointerType::getUnqual(Ty), Value::FunctionVal, nullptr, 0,
-                   Linkage, name),
+    : GlobalObject(PointerType::getUnqual(Ty), Value::FunctionVal,
+                   OperandTraits<Function>::op_begin(this), 0, Linkage, name),
       Ty(Ty) {
   assert(FunctionType::isValidReturnType(getReturnType()) &&
          "invalid return type");
-  setIsMaterializable(false);
+  setGlobalObjectSubClassData(0);
   SymTab = new ValueSymbolTable();
 
   // If the function has arguments, mark them as lazily built.
@@ -255,9 +264,10 @@ Function::Function(FunctionType *Ty, LinkageTypes Linkage, const Twine &name,
     ParentModule->getFunctionList().push_back(this);
 
   // Ensure intrinsics have the right parameter attributes.
-  if (unsigned IID = getIntrinsicID())
-    setAttributes(Intrinsic::getAttributes(getContext(), Intrinsic::ID(IID)));
-
+  // Note, the IntID field will have been set in Value::setName if this function
+  // name is a valid intrinsic ID.
+  if (IntID)
+    setAttributes(Intrinsic::getAttributes(getContext(), IntID));
 }
 
 Function::~Function() {
@@ -270,9 +280,8 @@ Function::~Function() {
   // Remove the function from the on-the-side GC table.
   clearGC();
 
-  // Remove the intrinsicID from the Cache.
-  if (getValueName() && isIntrinsic())
-    getContext().pImpl->IntrinsicIDCache.erase(this);
+  // FIXME: needed by operator delete
+  setFunctionNumOperands(1);
 }
 
 void Function::BuildLazyArguments() const {
@@ -322,6 +331,11 @@ void Function::dropAllReferences() {
   // Prefix and prologue data are stored in a side table.
   setPrefixData(nullptr);
   setPrologueData(nullptr);
+
+  // Metadata is stored in a side-table.
+  clearMetadata();
+
+  setPersonalityFn(nullptr);
 }
 
 void Function::addAttribute(unsigned i, Attribute::AttrKind attr) {
@@ -417,35 +431,15 @@ void Function::copyAttributesFrom(const GlobalValue *Src) {
     setPrologueData(SrcF->getPrologueData());
   else
     setPrologueData(nullptr);
+  if (SrcF->hasPersonalityFn())
+    setPersonalityFn(SrcF->getPersonalityFn());
+  else
+    setPersonalityFn(nullptr);
 }
 
-/// getIntrinsicID - This method returns the ID number of the specified
-/// function, or Intrinsic::not_intrinsic if the function is not an
-/// intrinsic, or if the pointer is null.  This value is always defined to be
-/// zero to allow easy checking for whether a function is intrinsic or not.  The
-/// particular intrinsic functions which correspond to this value are defined in
-/// llvm/Intrinsics.h.  Results are cached in the LLVM context, subsequent
-/// requests for the same ID return results much faster from the cache.
-///
-unsigned Function::getIntrinsicID() const {
-  const ValueName *ValName = this->getValueName();
-  if (!ValName || !isIntrinsic())
-    return 0;
-
-  LLVMContextImpl::IntrinsicIDCacheTy &IntrinsicIDCache =
-    getContext().pImpl->IntrinsicIDCache;
-  if (!IntrinsicIDCache.count(this)) {
-    unsigned Id = lookupIntrinsicID();
-    IntrinsicIDCache[this]=Id;
-    return Id;
-  }
-  return IntrinsicIDCache[this];
-}
-
-/// This private method does the actual lookup of an intrinsic ID when the query
-/// could not be answered from the cache.
-unsigned Function::lookupIntrinsicID() const {
-  const ValueName *ValName = this->getValueName();
+/// \brief This does the actual lookup of an intrinsic ID which
+/// matches the given function name.
+static Intrinsic::ID lookupIntrinsicID(const ValueName *ValName) {
   unsigned Len = ValName->getKeyLength();
   const char *Name = ValName->getKeyData();
 
@@ -453,7 +447,16 @@ unsigned Function::lookupIntrinsicID() const {
 #include "llvm/IR/Intrinsics.gen"
 #undef GET_FUNCTION_RECOGNIZER
 
-  return 0;
+  return Intrinsic::not_intrinsic;
+}
+
+void Function::recalculateIntrinsicID() {
+  const ValueName *ValName = this->getValueName();
+  if (!ValName || !isIntrinsic()) {
+    IntID = Intrinsic::not_intrinsic;
+    return;
+  }
+  IntID = lookupIntrinsicID(ValName);
 }
 
 /// Returns a stable mangling for the type specified for use in the name
@@ -479,10 +482,8 @@ static std::string getMangledTypeStr(Type* Ty) {
     Result += "a" + llvm::utostr(ATyp->getNumElements()) +
       getMangledTypeStr(ATyp->getElementType());
   } else if (StructType* STyp = dyn_cast<StructType>(Ty)) {
-    if (!STyp->isLiteral())
-      Result += STyp->getName();
-    else
-      llvm_unreachable("TODO: implement literal types");
+    assert(!STyp->isLiteral() && "TODO: implement literal types");
+    Result += STyp->getName();
   } else if (FunctionType* FT = dyn_cast<FunctionType>(Ty)) {
     Result += "f_" + getMangledTypeStr(FT->getReturnType());
     for (size_t i = 0; i < FT->getNumParams(); i++)
@@ -554,7 +555,8 @@ enum IIT_Info {
   IIT_HALF_VEC_ARG = 29,
   IIT_SAME_VEC_WIDTH_ARG = 30,
   IIT_PTR_TO_ARG = 31,
-  IIT_VEC_OF_PTRS_TO_ELT = 32
+  IIT_VEC_OF_PTRS_TO_ELT = 32,
+  IIT_I128 = 33
 };
 
 
@@ -601,6 +603,9 @@ static void DecodeIITType(unsigned &NextElt, ArrayRef<unsigned char> Infos,
   case IIT_I64:
     OutputTable.push_back(IITDescriptor::get(IITDescriptor::Integer, 64));
     return;
+  case IIT_I128:
+    OutputTable.push_back(IITDescriptor::get(IITDescriptor::Integer, 128));
+    return;
   case IIT_V1:
     OutputTable.push_back(IITDescriptor::get(IITDescriptor::Vector, 1));
     DecodeIITType(NextElt, Infos, OutputTable);
@@ -841,6 +846,18 @@ bool Intrinsic::isOverloaded(ID id) {
 #undef GET_INTRINSIC_OVERLOAD_TABLE
 }
 
+bool Intrinsic::isLeaf(ID id) {
+  switch (id) {
+  default:
+    return true;
+
+  case Intrinsic::experimental_gc_statepoint:
+  case Intrinsic::experimental_patchpoint_void:
+  case Intrinsic::experimental_patchpoint_i64:
+    return false;
+  }
+}
+
 /// This defines the "Intrinsic::getAttributes(ID id)" method.
 #define GET_INTRINSIC_ATTRIBUTES
 #include "llvm/IR/Intrinsics.gen"
@@ -964,3 +981,38 @@ void Function::setPrologueData(Constant *PrologueData) {
   }
   setValueSubclassData(PDData);
 }
+
+void Function::setEntryCount(uint64_t Count) {
+  MDBuilder MDB(getContext());
+  setMetadata(LLVMContext::MD_prof, MDB.createFunctionEntryCount(Count));
+}
+
+Optional<uint64_t> Function::getEntryCount() const {
+  MDNode *MD = getMetadata(LLVMContext::MD_prof);
+  if (MD && MD->getOperand(0))
+    if (MDString *MDS = dyn_cast<MDString>(MD->getOperand(0)))
+      if (MDS->getString().equals("function_entry_count")) {
+        ConstantInt *CI = mdconst::extract<ConstantInt>(MD->getOperand(1));
+        return CI->getValue().getZExtValue();
+      }
+  return None;
+}
+
+void Function::setPersonalityFn(Constant *C) {
+  if (!C) {
+    if (hasPersonalityFn()) {
+      // Note, the num operands is used to compute the offset of the operand, so
+      // the order here matters.  Clearing the operand then clearing the num
+      // operands ensures we have the correct offset to the operand.
+      Op<0>().set(nullptr);
+      setFunctionNumOperands(0);
+    }
+  } else {
+    // Note, the num operands is used to compute the offset of the operand, so
+    // the order here matters.  We need to set num operands to 1 first so that
+    // we get the correct offset to the first operand when we set it.
+    if (!hasPersonalityFn())
+      setFunctionNumOperands(1);
+    Op<0>().set(C);
+  }
+}
diff --git a/lib/IR/Globals.cpp b/lib/IR/Globals.cpp
index 5a6adb3f304..1d0282677bf 100644
--- a/lib/IR/Globals.cpp
+++ b/lib/IR/Globals.cpp
@@ -38,14 +38,18 @@ bool GlobalValue::isDematerializable() const {
 std::error_code GlobalValue::materialize() {
   return getParent()->materialize(this);
 }
-void GlobalValue::Dematerialize() {
-  getParent()->Dematerialize(this);
+void GlobalValue::dematerialize() {
+  getParent()->dematerialize(this);
 }
 
-/// Override destroyConstant to make sure it doesn't get called on
+/// Override destroyConstantImpl to make sure it doesn't get called on
 /// GlobalValue's because they shouldn't be treated like other constants.
-void GlobalValue::destroyConstant() {
-  llvm_unreachable("You can't GV->destroyConstant()!");
+void GlobalValue::destroyConstantImpl() {
+  llvm_unreachable("You can't GV->destroyConstantImpl()!");
+}
+
+Value *GlobalValue::handleOperandChangeImpl(Value *From, Value *To, Use *U) {
+  llvm_unreachable("Unsupported class for handleOperandChange()!");
 }
 
 /// copyAttributesFrom - copy all additional attributes (those not needed to
@@ -191,37 +195,23 @@ void GlobalVariable::eraseFromParent() {
   getParent()->getGlobalList().erase(this);
 }
 
-void GlobalVariable::replaceUsesOfWithOnConstant(Value *From, Value *To,
-                                                 Use *U) {
-  // If you call this, then you better know this GVar has a constant
-  // initializer worth replacing. Enforce that here.
-  assert(getNumOperands() == 1 &&
-         "Attempt to replace uses of Constants on a GVar with no initializer");
-
-  // And, since you know it has an initializer, the From value better be
-  // the initializer :)
-  assert(getOperand(0) == From &&
-         "Attempt to replace wrong constant initializer in GVar");
-
-  // And, you better have a constant for the replacement value
-  assert(isa<Constant>(To) &&
-         "Attempt to replace GVar initializer with non-constant");
-
-  // Okay, preconditions out of the way, replace the constant initializer.
-  this->setOperand(0, cast<Constant>(To));
-}
-
 void GlobalVariable::setInitializer(Constant *InitVal) {
   if (!InitVal) {
     if (hasInitializer()) {
+      // Note, the num operands is used to compute the offset of the operand, so
+      // the order here matters.  Clearing the operand then clearing the num
+      // operands ensures we have the correct offset to the operand.
       Op<0>().set(nullptr);
-      NumOperands = 0;
+      setGlobalVariableNumOperands(0);
     }
   } else {
     assert(InitVal->getType() == getType()->getElementType() &&
            "Initializer type must match GlobalVariable type");
+    // Note, the num operands is used to compute the offset of the operand, so
+    // the order here matters.  We need to set num operands to 1 first so that
+    // we get the correct offset to the first operand when we set it.
     if (!hasInitializer())
-      NumOperands = 1;
+      setGlobalVariableNumOperands(1);
     Op<0>().set(InitVal);
   }
 }
@@ -241,40 +231,35 @@ void GlobalVariable::copyAttributesFrom(const GlobalValue *Src) {
 // GlobalAlias Implementation
 //===----------------------------------------------------------------------===//
 
-GlobalAlias::GlobalAlias(Type *Ty, unsigned AddressSpace, LinkageTypes Link,
-                         const Twine &Name, Constant *Aliasee,
-                         Module *ParentModule)
-    : GlobalValue(PointerType::get(Ty, AddressSpace), Value::GlobalAliasVal,
-                  &Op<0>(), 1, Link, Name) {
+GlobalAlias::GlobalAlias(PointerType *Ty, LinkageTypes Link, const Twine &Name,
+                         Constant *Aliasee, Module *ParentModule)
+    : GlobalValue(Ty, Value::GlobalAliasVal, &Op<0>(), 1, Link, Name) {
   Op<0>() = Aliasee;
 
   if (ParentModule)
     ParentModule->getAliasList().push_back(this);
 }
 
-GlobalAlias *GlobalAlias::create(Type *Ty, unsigned AddressSpace,
-                                 LinkageTypes Link, const Twine &Name,
-                                 Constant *Aliasee, Module *ParentModule) {
-  return new GlobalAlias(Ty, AddressSpace, Link, Name, Aliasee, ParentModule);
+GlobalAlias *GlobalAlias::create(PointerType *Ty, LinkageTypes Link,
+                                 const Twine &Name, Constant *Aliasee,
+                                 Module *ParentModule) {
+  return new GlobalAlias(Ty, Link, Name, Aliasee, ParentModule);
 }
 
-GlobalAlias *GlobalAlias::create(Type *Ty, unsigned AddressSpace,
-                                 LinkageTypes Linkage, const Twine &Name,
-                                 Module *Parent) {
-  return create(Ty, AddressSpace, Linkage, Name, nullptr, Parent);
+GlobalAlias *GlobalAlias::create(PointerType *Ty, LinkageTypes Linkage,
+                                 const Twine &Name, Module *Parent) {
+  return create(Ty, Linkage, Name, nullptr, Parent);
 }
 
-GlobalAlias *GlobalAlias::create(Type *Ty, unsigned AddressSpace,
-                                 LinkageTypes Linkage, const Twine &Name,
-                                 GlobalValue *Aliasee) {
-  return create(Ty, AddressSpace, Linkage, Name, Aliasee, Aliasee->getParent());
+GlobalAlias *GlobalAlias::create(PointerType *Ty, LinkageTypes Linkage,
+                                 const Twine &Name, GlobalValue *Aliasee) {
+  return create(Ty, Linkage, Name, Aliasee, Aliasee->getParent());
 }
 
 GlobalAlias *GlobalAlias::create(LinkageTypes Link, const Twine &Name,
                                  GlobalValue *Aliasee) {
   PointerType *PTy = Aliasee->getType();
-  return create(PTy->getElementType(), PTy->getAddressSpace(), Link, Name,
-                Aliasee);
+  return create(PTy, Link, Name, Aliasee);
 }
 
 GlobalAlias *GlobalAlias::create(const Twine &Name, GlobalValue *Aliasee) {
diff --git a/lib/IR/IRBuilder.cpp b/lib/IR/IRBuilder.cpp
index 06f54c7e752..bddb278dee7 100644
--- a/lib/IR/IRBuilder.cpp
+++ b/lib/IR/IRBuilder.cpp
@@ -17,6 +17,7 @@
 #include "llvm/IR/IRBuilder.h"
 #include "llvm/IR/Intrinsics.h"
 #include "llvm/IR/LLVMContext.h"
+#include "llvm/IR/Statepoint.h"
 using namespace llvm;
 
 /// CreateGlobalString - Make a new global variable with an initializer that
@@ -24,13 +25,15 @@ using namespace llvm;
 /// specified.  If Name is specified, it is the name of the global variable
 /// created.
 GlobalVariable *IRBuilderBase::CreateGlobalString(StringRef Str,
-                                                  const Twine &Name) {
+                                                  const Twine &Name,
+                                                  unsigned AddressSpace) {
   Constant *StrConstant = ConstantDataArray::getString(Context, Str);
   Module &M = *BB->getParent()->getParent();
   GlobalVariable *GV = new GlobalVariable(M, StrConstant->getType(),
                                           true, GlobalValue::PrivateLinkage,
-                                          StrConstant);
-  GV->setName(Name);
+                                          StrConstant, Name, nullptr,
+                                          GlobalVariable::NotThreadLocal,
+                                          AddressSpace);
   GV->setUnnamedAddr(true);
   return GV;
 }
@@ -62,6 +65,19 @@ static CallInst *createCallHelper(Value *Callee, ArrayRef<Value *> Ops,
   return CI;  
 }
 
+static InvokeInst *createInvokeHelper(Value *Invokee, BasicBlock *NormalDest,
+                                      BasicBlock *UnwindDest,
+                                      ArrayRef<Value *> Ops,
+                                      IRBuilderBase *Builder,
+                                      const Twine &Name = "") {
+  InvokeInst *II =
+      InvokeInst::Create(Invokee, NormalDest, UnwindDest, Ops, Name);
+  Builder->GetInsertBlock()->getInstList().insert(Builder->GetInsertPoint(),
+                                                  II);
+  Builder->SetInstDebugLocation(II);
+  return II;
+}
+
 CallInst *IRBuilderBase::
 CreateMemSet(Value *Ptr, Value *Val, Value *Size, unsigned Align,
              bool isVolatile, MDNode *TBAATag, MDNode *ScopeTag,
@@ -221,55 +237,98 @@ CallInst *IRBuilderBase::CreateMaskedStore(Value *Val, Value *Ptr,
 
 /// Create a call to a Masked intrinsic, with given intrinsic Id,
 /// an array of operands - Ops, and one overloaded type - DataTy
-CallInst *IRBuilderBase::CreateMaskedIntrinsic(unsigned Id,
+CallInst *IRBuilderBase::CreateMaskedIntrinsic(Intrinsic::ID Id,
                                                ArrayRef<Value *> Ops,
                                                Type *DataTy,
                                                const Twine &Name) {
   Module *M = BB->getParent()->getParent();
   Type *OverloadedTypes[] = { DataTy };
-  Value *TheFn = Intrinsic::getDeclaration(M, (Intrinsic::ID)Id, OverloadedTypes);
+  Value *TheFn = Intrinsic::getDeclaration(M, Id, OverloadedTypes);
   return createCallHelper(TheFn, Ops, this, Name);
 }
 
-CallInst *IRBuilderBase::CreateGCStatepoint(Value *ActualCallee,
-                                            ArrayRef<Value *> CallArgs,
-                                            ArrayRef<Value *> DeoptArgs,
-                                            ArrayRef<Value *> GCArgs,
-                                            const Twine &Name) {
- // Extract out the type of the callee.
- PointerType *FuncPtrType = cast<PointerType>(ActualCallee->getType());
- assert(isa<FunctionType>(FuncPtrType->getElementType()) &&
-        "actual callee must be a callable value");
+static std::vector<Value *>
+getStatepointArgs(IRBuilderBase &B, uint64_t ID, uint32_t NumPatchBytes,
+                  Value *ActualCallee, ArrayRef<Value *> CallArgs,
+                  ArrayRef<Value *> DeoptArgs, ArrayRef<Value *> GCArgs) {
+  std::vector<Value *> Args;
+  Args.push_back(B.getInt64(ID));
+  Args.push_back(B.getInt32(NumPatchBytes));
+  Args.push_back(ActualCallee);
+  Args.push_back(B.getInt32(CallArgs.size()));
+  Args.push_back(B.getInt32((unsigned)StatepointFlags::None));
+  Args.insert(Args.end(), CallArgs.begin(), CallArgs.end());
+  Args.push_back(B.getInt32(0 /* no transition args */));
+  Args.push_back(B.getInt32(DeoptArgs.size()));
+  Args.insert(Args.end(), DeoptArgs.begin(), DeoptArgs.end());
+  Args.insert(Args.end(), GCArgs.begin(), GCArgs.end());
 
- 
- Module *M = BB->getParent()->getParent();
- // Fill in the one generic type'd argument (the function is also vararg)
- Type *ArgTypes[] = { FuncPtrType };
- Function *FnStatepoint =
-   Intrinsic::getDeclaration(M, Intrinsic::experimental_gc_statepoint,
-                             ArgTypes);
-
- std::vector<llvm::Value *> args;
- args.push_back(ActualCallee);
- args.push_back(getInt32(CallArgs.size()));
- args.push_back(getInt32(0 /*unused*/));
- args.insert(args.end(), CallArgs.begin(), CallArgs.end());
- args.push_back(getInt32(DeoptArgs.size()));
- args.insert(args.end(), DeoptArgs.begin(), DeoptArgs.end());
- args.insert(args.end(), GCArgs.begin(), GCArgs.end());
-
- return createCallHelper(FnStatepoint, args, this, Name);
+  return Args;
+}
+
+CallInst *IRBuilderBase::CreateGCStatepointCall(
+    uint64_t ID, uint32_t NumPatchBytes, Value *ActualCallee,
+    ArrayRef<Value *> CallArgs, ArrayRef<Value *> DeoptArgs,
+    ArrayRef<Value *> GCArgs, const Twine &Name) {
+  // Extract out the type of the callee.
+  PointerType *FuncPtrType = cast<PointerType>(ActualCallee->getType());
+  assert(isa<FunctionType>(FuncPtrType->getElementType()) &&
+         "actual callee must be a callable value");
+
+  Module *M = BB->getParent()->getParent();
+  // Fill in the one generic type'd argument (the function is also vararg)
+  Type *ArgTypes[] = { FuncPtrType };
+  Function *FnStatepoint =
+    Intrinsic::getDeclaration(M, Intrinsic::experimental_gc_statepoint,
+                              ArgTypes);
+
+  std::vector<llvm::Value *> Args = getStatepointArgs(
+      *this, ID, NumPatchBytes, ActualCallee, CallArgs, DeoptArgs, GCArgs);
+  return createCallHelper(FnStatepoint, Args, this, Name);
 }
 
-CallInst *IRBuilderBase::CreateGCStatepoint(Value *ActualCallee,
-                                            ArrayRef<Use> CallArgs,
-                                            ArrayRef<Value *> DeoptArgs,
-                                            ArrayRef<Value *> GCArgs,
-                                            const Twine &Name) {
+CallInst *IRBuilderBase::CreateGCStatepointCall(
+    uint64_t ID, uint32_t NumPatchBytes, Value *ActualCallee,
+    ArrayRef<Use> CallArgs, ArrayRef<Value *> DeoptArgs,
+    ArrayRef<Value *> GCArgs, const Twine &Name) {
   std::vector<Value *> VCallArgs;
   for (auto &U : CallArgs)
     VCallArgs.push_back(U.get());
-  return CreateGCStatepoint(ActualCallee, VCallArgs, DeoptArgs, GCArgs, Name);
+  return CreateGCStatepointCall(ID, NumPatchBytes, ActualCallee, VCallArgs,
+                                DeoptArgs, GCArgs, Name);
+}
+
+InvokeInst *IRBuilderBase::CreateGCStatepointInvoke(
+    uint64_t ID, uint32_t NumPatchBytes, Value *ActualInvokee,
+    BasicBlock *NormalDest, BasicBlock *UnwindDest,
+    ArrayRef<Value *> InvokeArgs, ArrayRef<Value *> DeoptArgs,
+    ArrayRef<Value *> GCArgs, const Twine &Name) {
+  // Extract out the type of the callee.
+  PointerType *FuncPtrType = cast<PointerType>(ActualInvokee->getType());
+  assert(isa<FunctionType>(FuncPtrType->getElementType()) &&
+         "actual callee must be a callable value");
+
+  Module *M = BB->getParent()->getParent();
+  // Fill in the one generic type'd argument (the function is also vararg)
+  Function *FnStatepoint = Intrinsic::getDeclaration(
+      M, Intrinsic::experimental_gc_statepoint, {FuncPtrType});
+
+  std::vector<llvm::Value *> Args = getStatepointArgs(
+      *this, ID, NumPatchBytes, ActualInvokee, InvokeArgs, DeoptArgs, GCArgs);
+  return createInvokeHelper(FnStatepoint, NormalDest, UnwindDest, Args, this,
+                            Name);
+}
+
+InvokeInst *IRBuilderBase::CreateGCStatepointInvoke(
+    uint64_t ID, uint32_t NumPatchBytes, Value *ActualInvokee,
+    BasicBlock *NormalDest, BasicBlock *UnwindDest, ArrayRef<Use> InvokeArgs,
+    ArrayRef<Value *> DeoptArgs, ArrayRef<Value *> GCArgs, const Twine &Name) {
+  std::vector<Value *> VCallArgs;
+  for (auto &U : InvokeArgs)
+    VCallArgs.push_back(U.get());
+  return CreateGCStatepointInvoke(ID, NumPatchBytes, ActualInvokee, NormalDest,
+                                  UnwindDest, VCallArgs, DeoptArgs, GCArgs,
+                                  Name);
 }
 
 CallInst *IRBuilderBase::CreateGCResult(Instruction *Statepoint,
diff --git a/lib/IR/InlineAsm.cpp b/lib/IR/InlineAsm.cpp
index c9ae96c3158..a181c5e834d 100644
--- a/lib/IR/InlineAsm.cpp
+++ b/lib/IR/InlineAsm.cpp
@@ -24,23 +24,22 @@ using namespace llvm;
 InlineAsm::~InlineAsm() {
 }
 
-
-InlineAsm *InlineAsm::get(FunctionType *Ty, StringRef AsmString,
+InlineAsm *InlineAsm::get(FunctionType *FTy, StringRef AsmString,
                           StringRef Constraints, bool hasSideEffects,
                           bool isAlignStack, AsmDialect asmDialect) {
-  InlineAsmKeyType Key(AsmString, Constraints, hasSideEffects, isAlignStack,
-                       asmDialect);
-  LLVMContextImpl *pImpl = Ty->getContext().pImpl;
-  return pImpl->InlineAsms.getOrCreate(PointerType::getUnqual(Ty), Key);
+  InlineAsmKeyType Key(AsmString, Constraints, FTy, hasSideEffects,
+                       isAlignStack, asmDialect);
+  LLVMContextImpl *pImpl = FTy->getContext().pImpl;
+  return pImpl->InlineAsms.getOrCreate(PointerType::getUnqual(FTy), Key);
 }
 
-InlineAsm::InlineAsm(PointerType *Ty, const std::string &asmString,
+InlineAsm::InlineAsm(FunctionType *FTy, const std::string &asmString,
                      const std::string &constraints, bool hasSideEffects,
                      bool isAlignStack, AsmDialect asmDialect)
-  : Value(Ty, Value::InlineAsmVal),
-    AsmString(asmString), Constraints(constraints),
-    HasSideEffects(hasSideEffects), IsAlignStack(isAlignStack),
-    Dialect(asmDialect) {
+    : Value(PointerType::getUnqual(FTy), Value::InlineAsmVal),
+      AsmString(asmString), Constraints(constraints), FTy(FTy),
+      HasSideEffects(hasSideEffects), IsAlignStack(isAlignStack),
+      Dialect(asmDialect) {
 
   // Do various checks on the constraint string and type.
   assert(Verify(getFunctionType(), constraints) &&
@@ -53,7 +52,7 @@ void InlineAsm::destroyConstant() {
 }
 
 FunctionType *InlineAsm::getFunctionType() const {
-  return cast<FunctionType>(getType()->getElementType());
+  return FTy;
 }
     
 ///Default constructor.
@@ -292,18 +291,3 @@ bool InlineAsm::Verify(FunctionType *Ty, StringRef ConstStr) {
   return true;
 }
 
-// @LOCALMOD-START
-bool InlineAsm::isAsmMemory() const {
-  bool retVoid = getFunctionType()->getReturnType()->isVoidTy();
-  bool noArgs = getFunctionType()->getNumParams() == 0 &&
-      !getFunctionType()->isVarArg();
-  bool isEmptyAsm = AsmString.empty();
-  // Different triples will encode "touch everything" differently, e.g.:
-  //  - le32-unknown-nacl has "~{memory}".
-  //  - x86 "~{memory},~{dirflag},~{fpsr},~{flags}".
-  // The following code therefore only searches for memory.
-  bool touchesMemory = Constraints.find("~{memory}") != std::string::npos;
-
-  return retVoid && noArgs && hasSideEffects() && isEmptyAsm && touchesMemory;
-}
-// @LOCALMOD-END
diff --git a/lib/IR/Instruction.cpp b/lib/IR/Instruction.cpp
index 57c143cca2c..85173d96562 100644
--- a/lib/IR/Instruction.cpp
+++ b/lib/IR/Instruction.cpp
@@ -26,9 +26,9 @@ Instruction::Instruction(Type *ty, unsigned it, Use *Ops, unsigned NumOps,
 
   // If requested, insert this instruction into a basic block...
   if (InsertBefore) {
-    assert(InsertBefore->getParent() &&
-           "Instruction to insert before is not in a basic block!");
-    InsertBefore->getParent()->getInstList().insert(InsertBefore, this);
+    BasicBlock *BB = InsertBefore->getParent();
+    assert(BB && "Instruction to insert before is not in a basic block!");
+    BB->getInstList().insert(InsertBefore, this);
   }
 }
 
@@ -58,6 +58,11 @@ const Module *Instruction::getModule() const {
   return getParent()->getModule();
 }
 
+Module *Instruction::getModule() {
+  return getParent()->getModule();
+}
+
+
 void Instruction::removeFromParent() {
   getParent()->getInstList().remove(this);
 }
@@ -191,6 +196,11 @@ const char *Instruction::getOpcodeName(unsigned OpCode) {
   case Invoke: return "invoke";
   case Resume: return "resume";
   case Unreachable: return "unreachable";
+  case CleanupRet: return "cleanupret";
+  case CatchEndPad: return "catchendpad";
+  case CatchRet: return "catchret";
+  case CatchPad: return "catchpad";
+  case TerminatePad: return "terminatepad";
 
   // Standard binary operators...
   case Add: return "add";
@@ -251,6 +261,7 @@ const char *Instruction::getOpcodeName(unsigned OpCode) {
   case ExtractValue:   return "extractvalue";
   case InsertValue:    return "insertvalue";
   case LandingPad:     return "landingpad";
+  case CleanupPad:   return "cleanuppad";
 
   default: return "<Invalid operator> ";
   }
@@ -402,6 +413,8 @@ bool Instruction::mayReadFromMemory() const {
   case Instruction::Fence: // FIXME: refine definition of mayReadFromMemory
   case Instruction::AtomicCmpXchg:
   case Instruction::AtomicRMW:
+  case Instruction::CatchRet:
+  case Instruction::TerminatePad:
     return true;
   case Instruction::Call:
     return !cast<CallInst>(this)->doesNotAccessMemory();
@@ -422,6 +435,8 @@ bool Instruction::mayWriteToMemory() const {
   case Instruction::VAArg:
   case Instruction::AtomicCmpXchg:
   case Instruction::AtomicRMW:
+  case Instruction::CatchRet:
+  case Instruction::TerminatePad:
     return true;
   case Instruction::Call:
     return !cast<CallInst>(this)->onlyReadsMemory();
@@ -450,6 +465,12 @@ bool Instruction::isAtomic() const {
 bool Instruction::mayThrow() const {
   if (const CallInst *CI = dyn_cast<CallInst>(this))
     return !CI->doesNotThrow();
+  if (const auto *CRI = dyn_cast<CleanupReturnInst>(this))
+    return CRI->unwindsToCaller();
+  if (const auto *CEPI = dyn_cast<CatchEndPadInst>(this))
+    return CEPI->unwindsToCaller();
+  if (const auto *TPI = dyn_cast<TerminatePadInst>(this))
+    return TPI->unwindsToCaller();
   return isa<ResumeInst>(this);
 }
 
@@ -529,8 +550,23 @@ bool Instruction::isNilpotent(unsigned Opcode) {
   return Opcode == Xor;
 }
 
+Instruction *Instruction::cloneImpl() const {
+  llvm_unreachable("Subclass of Instruction failed to implement cloneImpl");
+}
+
 Instruction *Instruction::clone() const {
-  Instruction *New = clone_impl();
+  Instruction *New = nullptr;
+  switch (getOpcode()) {
+  default:
+    llvm_unreachable("Unhandled Opcode.");
+#define HANDLE_INST(num, opc, clas)                                            \
+  case Instruction::opc:                                                       \
+    New = cast<clas>(this)->cloneImpl();                                       \
+    break;
+#include "llvm/IR/Instruction.def"
+#undef HANDLE_INST
+  }
+
   New->SubclassOptionalData = SubclassOptionalData;
   if (!hasMetadata())
     return New;
diff --git a/lib/IR/Instructions.cpp b/lib/IR/Instructions.cpp
index b1a2b9eced3..701f92eb6ea 100644
--- a/lib/IR/Instructions.cpp
+++ b/lib/IR/Instructions.cpp
@@ -85,30 +85,14 @@ const char *SelectInst::areInvalidOperands(Value *Op0, Value *Op1, Value *Op2) {
 //===----------------------------------------------------------------------===//
 
 PHINode::PHINode(const PHINode &PN)
-  : Instruction(PN.getType(), Instruction::PHI,
-                allocHungoffUses(PN.getNumOperands()), PN.getNumOperands()),
-    ReservedSpace(PN.getNumOperands()) {
+    : Instruction(PN.getType(), Instruction::PHI, nullptr, PN.getNumOperands()),
+      ReservedSpace(PN.getNumOperands()) {
+  allocHungoffUses(PN.getNumOperands());
   std::copy(PN.op_begin(), PN.op_end(), op_begin());
   std::copy(PN.block_begin(), PN.block_end(), block_begin());
   SubclassOptionalData = PN.SubclassOptionalData;
 }
 
-PHINode::~PHINode() {
-  dropHungoffUses();
-}
-
-Use *PHINode::allocHungoffUses(unsigned N) const {
-  // Allocate the array of Uses of the incoming values, followed by a pointer
-  // (with bottom bit set) to the User, followed by the array of pointers to
-  // the incoming basic blocks.
-  size_t size = N * sizeof(Use) + sizeof(Use::UserRef)
-    + N * sizeof(BasicBlock*);
-  Use *Begin = static_cast<Use*>(::operator new(size));
-  Use *End = Begin + N;
-  (void) new(End) Use::UserRef(const_cast<PHINode*>(this), 1);
-  return Use::initTags(Begin, End);
-}
-
 // removeIncomingValue - Remove an incoming value.  This is useful if a
 // predecessor basic block is deleted.
 Value *PHINode::removeIncomingValue(unsigned Idx, bool DeletePHIIfEmpty) {
@@ -124,7 +108,7 @@ Value *PHINode::removeIncomingValue(unsigned Idx, bool DeletePHIIfEmpty) {
 
   // Nuke the last value.
   Op<-1>().set(nullptr);
-  --NumOperands;
+  setNumHungOffUseOperands(getNumOperands() - 1);
 
   // If the PHI node is dead, because it has zero entries, nuke it now.
   if (getNumOperands() == 0 && DeletePHIIfEmpty) {
@@ -144,16 +128,8 @@ void PHINode::growOperands() {
   unsigned NumOps = e + e / 2;
   if (NumOps < 2) NumOps = 2;      // 2 op PHI nodes are VERY common.
 
-  Use *OldOps = op_begin();
-  BasicBlock **OldBlocks = block_begin();
-
   ReservedSpace = NumOps;
-  OperandList = allocHungoffUses(ReservedSpace);
-
-  std::copy(OldOps, OldOps + e, op_begin());
-  std::copy(OldBlocks, OldBlocks + e, block_begin());
-
-  Use::zap(OldOps, OldOps + e, true);
+  growHungoffUses(ReservedSpace, /* IsPhi */ true);
 }
 
 /// hasConstantValue - If the specified PHI node always merges together the same
@@ -177,57 +153,47 @@ Value *PHINode::hasConstantValue() const {
 //                       LandingPadInst Implementation
 //===----------------------------------------------------------------------===//
 
-LandingPadInst::LandingPadInst(Type *RetTy, Value *PersonalityFn,
-                               unsigned NumReservedValues, const Twine &NameStr,
-                               Instruction *InsertBefore)
-  : Instruction(RetTy, Instruction::LandingPad, nullptr, 0, InsertBefore) {
-  init(PersonalityFn, 1 + NumReservedValues, NameStr);
+LandingPadInst::LandingPadInst(Type *RetTy, unsigned NumReservedValues,
+                               const Twine &NameStr, Instruction *InsertBefore)
+    : Instruction(RetTy, Instruction::LandingPad, nullptr, 0, InsertBefore) {
+  init(NumReservedValues, NameStr);
 }
 
-LandingPadInst::LandingPadInst(Type *RetTy, Value *PersonalityFn,
-                               unsigned NumReservedValues, const Twine &NameStr,
-                               BasicBlock *InsertAtEnd)
-  : Instruction(RetTy, Instruction::LandingPad, nullptr, 0, InsertAtEnd) {
-  init(PersonalityFn, 1 + NumReservedValues, NameStr);
+LandingPadInst::LandingPadInst(Type *RetTy, unsigned NumReservedValues,
+                               const Twine &NameStr, BasicBlock *InsertAtEnd)
+    : Instruction(RetTy, Instruction::LandingPad, nullptr, 0, InsertAtEnd) {
+  init(NumReservedValues, NameStr);
 }
 
 LandingPadInst::LandingPadInst(const LandingPadInst &LP)
-  : Instruction(LP.getType(), Instruction::LandingPad,
-                allocHungoffUses(LP.getNumOperands()), LP.getNumOperands()),
-    ReservedSpace(LP.getNumOperands()) {
-  Use *OL = OperandList, *InOL = LP.OperandList;
+    : Instruction(LP.getType(), Instruction::LandingPad, nullptr,
+                  LP.getNumOperands()),
+      ReservedSpace(LP.getNumOperands()) {
+  allocHungoffUses(LP.getNumOperands());
+  Use *OL = getOperandList();
+  const Use *InOL = LP.getOperandList();
   for (unsigned I = 0, E = ReservedSpace; I != E; ++I)
     OL[I] = InOL[I];
 
   setCleanup(LP.isCleanup());
 }
 
-LandingPadInst::~LandingPadInst() {
-  dropHungoffUses();
-}
-
-LandingPadInst *LandingPadInst::Create(Type *RetTy, Value *PersonalityFn,
-                                       unsigned NumReservedClauses,
+LandingPadInst *LandingPadInst::Create(Type *RetTy, unsigned NumReservedClauses,
                                        const Twine &NameStr,
                                        Instruction *InsertBefore) {
-  return new LandingPadInst(RetTy, PersonalityFn, NumReservedClauses, NameStr,
-                            InsertBefore);
+  return new LandingPadInst(RetTy, NumReservedClauses, NameStr, InsertBefore);
 }
 
-LandingPadInst *LandingPadInst::Create(Type *RetTy, Value *PersonalityFn,
-                                       unsigned NumReservedClauses,
+LandingPadInst *LandingPadInst::Create(Type *RetTy, unsigned NumReservedClauses,
                                        const Twine &NameStr,
                                        BasicBlock *InsertAtEnd) {
-  return new LandingPadInst(RetTy, PersonalityFn, NumReservedClauses, NameStr,
-                            InsertAtEnd);
+  return new LandingPadInst(RetTy, NumReservedClauses, NameStr, InsertAtEnd);
 }
 
-void LandingPadInst::init(Value *PersFn, unsigned NumReservedValues,
-                          const Twine &NameStr) {
+void LandingPadInst::init(unsigned NumReservedValues, const Twine &NameStr) {
   ReservedSpace = NumReservedValues;
-  NumOperands = 1;
-  OperandList = allocHungoffUses(ReservedSpace);
-  OperandList[0] = PersFn;
+  setNumHungOffUseOperands(0);
+  allocHungoffUses(ReservedSpace);
   setName(NameStr);
   setCleanup(false);
 }
@@ -237,23 +203,16 @@ void LandingPadInst::init(Value *PersFn, unsigned NumReservedValues,
 void LandingPadInst::growOperands(unsigned Size) {
   unsigned e = getNumOperands();
   if (ReservedSpace >= e + Size) return;
-  ReservedSpace = (e + Size / 2) * 2;
-
-  Use *NewOps = allocHungoffUses(ReservedSpace);
-  Use *OldOps = OperandList;
-  for (unsigned i = 0; i != e; ++i)
-      NewOps[i] = OldOps[i];
-
-  OperandList = NewOps;
-  Use::zap(OldOps, OldOps + e, true);
+  ReservedSpace = (std::max(e, 1U) + Size / 2) * 2;
+  growHungoffUses(ReservedSpace);
 }
 
 void LandingPadInst::addClause(Constant *Val) {
   unsigned OpNo = getNumOperands();
   growOperands(1);
   assert(OpNo < ReservedSpace && "Growing didn't work!");
-  ++NumOperands;
-  OperandList[OpNo] = Val;
+  setNumHungOffUseOperands(getNumOperands() + 1);
+  getOperandList()[OpNo] = Val;
 }
 
 //===----------------------------------------------------------------------===//
@@ -263,14 +222,13 @@ void LandingPadInst::addClause(Constant *Val) {
 CallInst::~CallInst() {
 }
 
-void CallInst::init(Value *Func, ArrayRef<Value *> Args, const Twine &NameStr) {
-  assert(NumOperands == Args.size() + 1 && "NumOperands not set up?");
+void CallInst::init(FunctionType *FTy, Value *Func, ArrayRef<Value *> Args,
+                    const Twine &NameStr) {
+  this->FTy = FTy;
+  assert(getNumOperands() == Args.size() + 1 && "NumOperands not set up?");
   Op<-1>() = Func;
 
 #ifndef NDEBUG
-  FunctionType *FTy =
-    cast<FunctionType>(cast<PointerType>(Func->getType())->getElementType());
-
   assert((Args.size() == FTy->getNumParams() ||
           (FTy->isVarArg() && Args.size() > FTy->getNumParams())) &&
          "Calling a function with bad signature!");
@@ -286,15 +244,12 @@ void CallInst::init(Value *Func, ArrayRef<Value *> Args, const Twine &NameStr) {
 }
 
 void CallInst::init(Value *Func, const Twine &NameStr) {
-  assert(NumOperands == 1 && "NumOperands not set up?");
+  FTy =
+      cast<FunctionType>(cast<PointerType>(Func->getType())->getElementType());
+  assert(getNumOperands() == 1 && "NumOperands not set up?");
   Op<-1>() = Func;
 
-#ifndef NDEBUG
-  FunctionType *FTy =
-    cast<FunctionType>(cast<PointerType>(Func->getType())->getElementType());
-
   assert(FTy->getNumParams() == 0 && "Calling a function with bad signature");
-#endif
 
   setName(NameStr);
 }
@@ -320,10 +275,10 @@ CallInst::CallInst(Value *Func, const Twine &Name,
 }
 
 CallInst::CallInst(const CallInst &CI)
-  : Instruction(CI.getType(), Instruction::Call,
-                OperandTraits<CallInst>::op_end(this) - CI.getNumOperands(),
-                CI.getNumOperands()) {
-  setAttributes(CI.getAttributes());
+    : Instruction(CI.getType(), Instruction::Call,
+                  OperandTraits<CallInst>::op_end(this) - CI.getNumOperands(),
+                  CI.getNumOperands()),
+      AttributeList(CI.AttributeList), FTy(CI.FTy) {
   setTailCallKind(CI.getTailCallKind());
   setCallingConv(CI.getCallingConv());
     
@@ -337,6 +292,12 @@ void CallInst::addAttribute(unsigned i, Attribute::AttrKind attr) {
   setAttributes(PAL);
 }
 
+void CallInst::addAttribute(unsigned i, StringRef Kind, StringRef Value) {
+  AttributeSet PAL = getAttributes();
+  PAL = PAL.addAttribute(getContext(), i, Kind, Value);
+  setAttributes(PAL);
+}
+
 void CallInst::removeAttribute(unsigned i, Attribute attr) {
   AttributeSet PAL = getAttributes();
   AttrBuilder B(attr);
@@ -358,14 +319,6 @@ void CallInst::addDereferenceableOrNullAttr(unsigned i, uint64_t Bytes) {
   setAttributes(PAL);
 }
 
-bool CallInst::hasFnAttrImpl(Attribute::AttrKind A) const {
-  if (AttributeList.hasAttribute(AttributeSet::FunctionIndex, A))
-    return true;
-  if (const Function *F = getCalledFunction())
-    return F->getAttributes().hasAttribute(AttributeSet::FunctionIndex, A);
-  return false;
-}
-
 bool CallInst::paramHasAttr(unsigned i, Attribute::AttrKind A) const {
   if (AttributeList.hasAttribute(i, A))
     return true;
@@ -541,17 +494,17 @@ Instruction* CallInst::CreateFree(Value* Source, BasicBlock *InsertAtEnd) {
 //                        InvokeInst Implementation
 //===----------------------------------------------------------------------===//
 
-void InvokeInst::init(Value *Fn, BasicBlock *IfNormal, BasicBlock *IfException,
-                      ArrayRef<Value *> Args, const Twine &NameStr) {
-  assert(NumOperands == 3 + Args.size() && "NumOperands not set up?");
+void InvokeInst::init(FunctionType *FTy, Value *Fn, BasicBlock *IfNormal,
+                      BasicBlock *IfException, ArrayRef<Value *> Args,
+                      const Twine &NameStr) {
+  this->FTy = FTy;
+
+  assert(getNumOperands() == 3 + Args.size() && "NumOperands not set up?");
   Op<-3>() = Fn;
   Op<-2>() = IfNormal;
   Op<-1>() = IfException;
 
 #ifndef NDEBUG
-  FunctionType *FTy =
-    cast<FunctionType>(cast<PointerType>(Fn->getType())->getElementType());
-
   assert(((Args.size() == FTy->getNumParams()) ||
           (FTy->isVarArg() && Args.size() > FTy->getNumParams())) &&
          "Invoking a function with bad signature");
@@ -567,11 +520,11 @@ void InvokeInst::init(Value *Fn, BasicBlock *IfNormal, BasicBlock *IfException,
 }
 
 InvokeInst::InvokeInst(const InvokeInst &II)
-  : TerminatorInst(II.getType(), Instruction::Invoke,
-                   OperandTraits<InvokeInst>::op_end(this)
-                   - II.getNumOperands(),
-                   II.getNumOperands()) {
-  setAttributes(II.getAttributes());
+    : TerminatorInst(II.getType(), Instruction::Invoke,
+                     OperandTraits<InvokeInst>::op_end(this) -
+                         II.getNumOperands(),
+                     II.getNumOperands()),
+      AttributeList(II.AttributeList), FTy(II.FTy) {
   setCallingConv(II.getCallingConv());
   std::copy(II.op_begin(), II.op_end(), op_begin());
   SubclassOptionalData = II.SubclassOptionalData;
@@ -717,6 +670,303 @@ BasicBlock *ResumeInst::getSuccessorV(unsigned idx) const {
   llvm_unreachable("ResumeInst has no successors!");
 }
 
+//===----------------------------------------------------------------------===//
+//                        CleanupReturnInst Implementation
+//===----------------------------------------------------------------------===//
+
+CleanupReturnInst::CleanupReturnInst(const CleanupReturnInst &CRI)
+    : TerminatorInst(CRI.getType(), Instruction::CleanupRet,
+                     OperandTraits<CleanupReturnInst>::op_end(this) -
+                         CRI.getNumOperands(),
+                     CRI.getNumOperands()) {
+  SubclassOptionalData = CRI.SubclassOptionalData;
+  if (Value *RetVal = CRI.getReturnValue())
+    setReturnValue(RetVal);
+  if (BasicBlock *UnwindDest = CRI.getUnwindDest())
+    setUnwindDest(UnwindDest);
+}
+
+void CleanupReturnInst::init(Value *RetVal, BasicBlock *UnwindBB) {
+  SubclassOptionalData = 0;
+  if (UnwindBB)
+    setInstructionSubclassData(getSubclassDataFromInstruction() | 1);
+  if (RetVal)
+    setInstructionSubclassData(getSubclassDataFromInstruction() | 2);
+
+  if (UnwindBB)
+    setUnwindDest(UnwindBB);
+  if (RetVal)
+    setReturnValue(RetVal);
+}
+
+CleanupReturnInst::CleanupReturnInst(LLVMContext &C, Value *RetVal,
+                                     BasicBlock *UnwindBB, unsigned Values,
+                                     Instruction *InsertBefore)
+    : TerminatorInst(Type::getVoidTy(C), Instruction::CleanupRet,
+                     OperandTraits<CleanupReturnInst>::op_end(this) - Values,
+                     Values, InsertBefore) {
+  init(RetVal, UnwindBB);
+}
+
+CleanupReturnInst::CleanupReturnInst(LLVMContext &C, Value *RetVal,
+                                     BasicBlock *UnwindBB, unsigned Values,
+                                     BasicBlock *InsertAtEnd)
+    : TerminatorInst(Type::getVoidTy(C), Instruction::CleanupRet,
+                     OperandTraits<CleanupReturnInst>::op_end(this) - Values,
+                     Values, InsertAtEnd) {
+  init(RetVal, UnwindBB);
+}
+
+BasicBlock *CleanupReturnInst::getUnwindDest() const {
+  if (hasUnwindDest())
+    return cast<BasicBlock>(getOperand(getUnwindLabelOpIdx()));
+  return nullptr;
+}
+void CleanupReturnInst::setUnwindDest(BasicBlock *NewDest) {
+  assert(NewDest);
+  setOperand(getUnwindLabelOpIdx(), NewDest);
+}
+
+BasicBlock *CleanupReturnInst::getSuccessorV(unsigned Idx) const {
+  assert(Idx == 0);
+  return getUnwindDest();
+}
+unsigned CleanupReturnInst::getNumSuccessorsV() const {
+  return getNumSuccessors();
+}
+void CleanupReturnInst::setSuccessorV(unsigned Idx, BasicBlock *B) {
+  assert(Idx == 0);
+  setUnwindDest(B);
+}
+
+//===----------------------------------------------------------------------===//
+//                        CatchEndPadInst Implementation
+//===----------------------------------------------------------------------===//
+
+CatchEndPadInst::CatchEndPadInst(const CatchEndPadInst &CRI)
+    : TerminatorInst(CRI.getType(), Instruction::CatchEndPad,
+                     OperandTraits<CatchEndPadInst>::op_end(this) -
+                         CRI.getNumOperands(),
+                     CRI.getNumOperands()) {
+  SubclassOptionalData = CRI.SubclassOptionalData;
+  if (BasicBlock *UnwindDest = CRI.getUnwindDest())
+    setUnwindDest(UnwindDest);
+}
+
+void CatchEndPadInst::init(BasicBlock *UnwindBB) {
+  SubclassOptionalData = 0;
+  if (UnwindBB) {
+    setInstructionSubclassData(getSubclassDataFromInstruction() | 1);
+    setUnwindDest(UnwindBB);
+  }
+}
+
+CatchEndPadInst::CatchEndPadInst(LLVMContext &C, BasicBlock *UnwindBB,
+                                     unsigned Values, Instruction *InsertBefore)
+    : TerminatorInst(Type::getVoidTy(C), Instruction::CatchEndPad,
+                     OperandTraits<CatchEndPadInst>::op_end(this) - Values,
+                     Values, InsertBefore) {
+  init(UnwindBB);
+}
+
+CatchEndPadInst::CatchEndPadInst(LLVMContext &C, BasicBlock *UnwindBB,
+                                     unsigned Values, BasicBlock *InsertAtEnd)
+    : TerminatorInst(Type::getVoidTy(C), Instruction::CatchEndPad,
+                     OperandTraits<CatchEndPadInst>::op_end(this) - Values,
+                     Values, InsertAtEnd) {
+  init(UnwindBB);
+}
+
+BasicBlock *CatchEndPadInst::getSuccessorV(unsigned Idx) const {
+  assert(Idx == 0);
+  return getUnwindDest();
+}
+unsigned CatchEndPadInst::getNumSuccessorsV() const {
+  return getNumSuccessors();
+}
+void CatchEndPadInst::setSuccessorV(unsigned Idx, BasicBlock *B) {
+  assert(Idx == 0);
+  setUnwindDest(B);
+}
+
+//===----------------------------------------------------------------------===//
+//                        CatchReturnInst Implementation
+//===----------------------------------------------------------------------===//
+
+CatchReturnInst::CatchReturnInst(const CatchReturnInst &CRI)
+    : TerminatorInst(Type::getVoidTy(CRI.getContext()), Instruction::CatchRet,
+                     OperandTraits<CatchReturnInst>::op_end(this) -
+                         CRI.getNumOperands(),
+                     CRI.getNumOperands()) {
+  Op<0>() = CRI.Op<0>();
+}
+
+CatchReturnInst::CatchReturnInst(BasicBlock *BB, Instruction *InsertBefore)
+    : TerminatorInst(Type::getVoidTy(BB->getContext()), Instruction::CatchRet,
+                     OperandTraits<CatchReturnInst>::op_begin(this), 1,
+                     InsertBefore) {
+  Op<0>() = BB;
+}
+
+CatchReturnInst::CatchReturnInst(BasicBlock *BB, BasicBlock *InsertAtEnd)
+    : TerminatorInst(Type::getVoidTy(BB->getContext()), Instruction::CatchRet,
+                     OperandTraits<CatchReturnInst>::op_begin(this), 1,
+                     InsertAtEnd) {
+  Op<0>() = BB;
+}
+
+BasicBlock *CatchReturnInst::getSuccessorV(unsigned Idx) const {
+  assert(Idx == 0);
+  return getSuccessor();
+}
+unsigned CatchReturnInst::getNumSuccessorsV() const {
+  return getNumSuccessors();
+}
+void CatchReturnInst::setSuccessorV(unsigned Idx, BasicBlock *B) {
+  assert(Idx == 0);
+  setSuccessor(B);
+}
+
+//===----------------------------------------------------------------------===//
+//                        CatchPadInst Implementation
+//===----------------------------------------------------------------------===//
+void CatchPadInst::init(BasicBlock *IfNormal, BasicBlock *IfException,
+                          ArrayRef<Value *> Args, const Twine &NameStr) {
+  assert(getNumOperands() == 2 + Args.size() && "NumOperands not set up?");
+  Op<-2>() = IfNormal;
+  Op<-1>() = IfException;
+  std::copy(Args.begin(), Args.end(), op_begin());
+  setName(NameStr);
+}
+
+CatchPadInst::CatchPadInst(const CatchPadInst &CPI)
+    : TerminatorInst(CPI.getType(), Instruction::CatchPad,
+                     OperandTraits<CatchPadInst>::op_end(this) -
+                         CPI.getNumOperands(),
+                     CPI.getNumOperands()) {
+  std::copy(CPI.op_begin(), CPI.op_end(), op_begin());
+}
+
+CatchPadInst::CatchPadInst(Type *RetTy, BasicBlock *IfNormal,
+                               BasicBlock *IfException, ArrayRef<Value *> Args,
+                               unsigned Values, const Twine &NameStr,
+                               Instruction *InsertBefore)
+    : TerminatorInst(RetTy, Instruction::CatchPad,
+                     OperandTraits<CatchPadInst>::op_end(this) - Values,
+                     Values, InsertBefore) {
+  init(IfNormal, IfException, Args, NameStr);
+}
+
+CatchPadInst::CatchPadInst(Type *RetTy, BasicBlock *IfNormal,
+                               BasicBlock *IfException, ArrayRef<Value *> Args,
+                               unsigned Values, const Twine &NameStr,
+                               BasicBlock *InsertAtEnd)
+    : TerminatorInst(RetTy, Instruction::CatchPad,
+                     OperandTraits<CatchPadInst>::op_end(this) - Values,
+                     Values, InsertAtEnd) {
+  init(IfNormal, IfException, Args, NameStr);
+}
+
+BasicBlock *CatchPadInst::getSuccessorV(unsigned Idx) const {
+  return getSuccessor(Idx);
+}
+unsigned CatchPadInst::getNumSuccessorsV() const {
+  return getNumSuccessors();
+}
+void CatchPadInst::setSuccessorV(unsigned Idx, BasicBlock *B) {
+  return setSuccessor(Idx, B);
+}
+
+//===----------------------------------------------------------------------===//
+//                        TerminatePadInst Implementation
+//===----------------------------------------------------------------------===//
+void TerminatePadInst::init(BasicBlock *BB, ArrayRef<Value *> Args,
+                              const Twine &NameStr) {
+  SubclassOptionalData = 0;
+  if (BB)
+    setInstructionSubclassData(getSubclassDataFromInstruction() | 1);
+  if (BB)
+    Op<-1>() = BB;
+  std::copy(Args.begin(), Args.end(), op_begin());
+  setName(NameStr);
+}
+
+TerminatePadInst::TerminatePadInst(const TerminatePadInst &TPI)
+    : TerminatorInst(TPI.getType(), Instruction::TerminatePad,
+                     OperandTraits<TerminatePadInst>::op_end(this) -
+                         TPI.getNumOperands(),
+                     TPI.getNumOperands()) {
+  SubclassOptionalData = TPI.SubclassOptionalData;
+  std::copy(TPI.op_begin(), TPI.op_end(), op_begin());
+}
+
+TerminatePadInst::TerminatePadInst(LLVMContext &C, BasicBlock *BB,
+                                       ArrayRef<Value *> Args, unsigned Values,
+                                       const Twine &NameStr,
+                                       Instruction *InsertBefore)
+    : TerminatorInst(Type::getVoidTy(C), Instruction::TerminatePad,
+                     OperandTraits<TerminatePadInst>::op_end(this) - Values,
+                     Values, InsertBefore) {
+  init(BB, Args, NameStr);
+}
+
+TerminatePadInst::TerminatePadInst(LLVMContext &C, BasicBlock *BB,
+                                       ArrayRef<Value *> Args, unsigned Values,
+                                       const Twine &NameStr,
+                                       BasicBlock *InsertAtEnd)
+    : TerminatorInst(Type::getVoidTy(C), Instruction::TerminatePad,
+                     OperandTraits<TerminatePadInst>::op_end(this) - Values,
+                     Values, InsertAtEnd) {
+  init(BB, Args, NameStr);
+}
+
+BasicBlock *TerminatePadInst::getSuccessorV(unsigned Idx) const {
+  assert(Idx == 0);
+  return getUnwindDest();
+}
+unsigned TerminatePadInst::getNumSuccessorsV() const {
+  return getNumSuccessors();
+}
+void TerminatePadInst::setSuccessorV(unsigned Idx, BasicBlock *B) {
+  assert(Idx == 0);
+  return setUnwindDest(B);
+}
+
+//===----------------------------------------------------------------------===//
+//                        CleanupPadInst Implementation
+//===----------------------------------------------------------------------===//
+void CleanupPadInst::init(ArrayRef<Value *> Args, const Twine &NameStr) {
+  assert(getNumOperands() == Args.size() && "NumOperands not set up?");
+  std::copy(Args.begin(), Args.end(), op_begin());
+  setName(NameStr);
+}
+
+CleanupPadInst::CleanupPadInst(const CleanupPadInst &CPI)
+    : Instruction(CPI.getType(), Instruction::CleanupPad,
+                  OperandTraits<CleanupPadInst>::op_end(this) -
+                      CPI.getNumOperands(),
+                  CPI.getNumOperands()) {
+  std::copy(CPI.op_begin(), CPI.op_end(), op_begin());
+}
+
+CleanupPadInst::CleanupPadInst(Type *RetTy, ArrayRef<Value *> Args,
+                                   const Twine &NameStr,
+                                   Instruction *InsertBefore)
+    : Instruction(RetTy, Instruction::CleanupPad,
+                  OperandTraits<CleanupPadInst>::op_end(this) - Args.size(),
+                  Args.size(), InsertBefore) {
+  init(Args, NameStr);
+}
+
+CleanupPadInst::CleanupPadInst(Type *RetTy, ArrayRef<Value *> Args,
+                                   const Twine &NameStr,
+                                   BasicBlock *InsertAtEnd)
+    : Instruction(RetTy, Instruction::CleanupPad,
+                  OperandTraits<CleanupPadInst>::op_end(this) - Args.size(),
+                  Args.size(), InsertAtEnd) {
+  init(Args, NameStr);
+}
+
 //===----------------------------------------------------------------------===//
 //                      UnreachableInst Implementation
 //===----------------------------------------------------------------------===//
@@ -869,8 +1119,9 @@ AllocaInst::AllocaInst(Type *Ty, Value *ArraySize, const Twine &Name,
 
 AllocaInst::AllocaInst(Type *Ty, Value *ArraySize, unsigned Align,
                        const Twine &Name, Instruction *InsertBefore)
-  : UnaryInstruction(PointerType::getUnqual(Ty), Alloca,
-                     getAISize(Ty->getContext(), ArraySize), InsertBefore) {
+    : UnaryInstruction(PointerType::getUnqual(Ty), Alloca,
+                       getAISize(Ty->getContext(), ArraySize), InsertBefore),
+      AllocatedType(Ty) {
   setAlignment(Align);
   assert(!Ty->isVoidTy() && "Cannot allocate void!");
   setName(Name);
@@ -878,8 +1129,9 @@ AllocaInst::AllocaInst(Type *Ty, Value *ArraySize, unsigned Align,
 
 AllocaInst::AllocaInst(Type *Ty, Value *ArraySize, unsigned Align,
                        const Twine &Name, BasicBlock *InsertAtEnd)
-  : UnaryInstruction(PointerType::getUnqual(Ty), Alloca,
-                     getAISize(Ty->getContext(), ArraySize), InsertAtEnd) {
+    : UnaryInstruction(PointerType::getUnqual(Ty), Alloca,
+                       getAISize(Ty->getContext(), ArraySize), InsertAtEnd),
+      AllocatedType(Ty) {
   setAlignment(Align);
   assert(!Ty->isVoidTy() && "Cannot allocate void!");
   setName(Name);
@@ -904,10 +1156,6 @@ bool AllocaInst::isArrayAllocation() const {
   return true;
 }
 
-Type *AllocaInst::getAllocatedType() const {
-  return getType()->getElementType();
-}
-
 /// isStaticAlloca - Return true if this alloca is in the entry block of the
 /// function and is a constant size.  If so, the code generator will fold it
 /// into the prolog/epilog code, so it is basically free.
@@ -937,17 +1185,17 @@ LoadInst::LoadInst(Value *Ptr, const Twine &Name, Instruction *InsertBef)
 LoadInst::LoadInst(Value *Ptr, const Twine &Name, BasicBlock *InsertAE)
     : LoadInst(Ptr, Name, /*isVolatile=*/false, InsertAE) {}
 
-LoadInst::LoadInst(Value *Ptr, const Twine &Name, bool isVolatile,
+LoadInst::LoadInst(Type *Ty, Value *Ptr, const Twine &Name, bool isVolatile,
                    Instruction *InsertBef)
-    : LoadInst(Ptr, Name, isVolatile, /*Align=*/0, InsertBef) {}
+    : LoadInst(Ty, Ptr, Name, isVolatile, /*Align=*/0, InsertBef) {}
 
 LoadInst::LoadInst(Value *Ptr, const Twine &Name, bool isVolatile,
                    BasicBlock *InsertAE)
     : LoadInst(Ptr, Name, isVolatile, /*Align=*/0, InsertAE) {}
 
-LoadInst::LoadInst(Value *Ptr, const Twine &Name, bool isVolatile,
+LoadInst::LoadInst(Type *Ty, Value *Ptr, const Twine &Name, bool isVolatile,
                    unsigned Align, Instruction *InsertBef)
-    : LoadInst(Ptr, Name, isVolatile, Align, NotAtomic, CrossThread,
+    : LoadInst(Ty, Ptr, Name, isVolatile, Align, NotAtomic, CrossThread,
                InsertBef) {}
 
 LoadInst::LoadInst(Value *Ptr, const Twine &Name, bool isVolatile,
@@ -959,6 +1207,7 @@ LoadInst::LoadInst(Type *Ty, Value *Ptr, const Twine &Name, bool isVolatile,
                    unsigned Align, AtomicOrdering Order,
                    SynchronizationScope SynchScope, Instruction *InsertBef)
     : UnaryInstruction(Ty, Load, Ptr, InsertBef) {
+  assert(Ty == cast<PointerType>(Ptr->getType())->getElementType());
   setVolatile(isVolatile);
   setAlignment(Align);
   setAtomic(Order, SynchScope);
@@ -999,10 +1248,10 @@ LoadInst::LoadInst(Value *Ptr, const char *Name, BasicBlock *InsertAE)
   if (Name && Name[0]) setName(Name);
 }
 
-LoadInst::LoadInst(Value *Ptr, const char *Name, bool isVolatile,
+LoadInst::LoadInst(Type *Ty, Value *Ptr, const char *Name, bool isVolatile,
                    Instruction *InsertBef)
-: UnaryInstruction(cast<PointerType>(Ptr->getType())->getElementType(),
-                   Load, Ptr, InsertBef) {
+    : UnaryInstruction(Ty, Load, Ptr, InsertBef) {
+  assert(Ty == cast<PointerType>(Ptr->getType())->getElementType());
   setVolatile(isVolatile);
   setAlignment(0);
   setAtomic(NotAtomic);
@@ -1243,17 +1492,20 @@ FenceInst::FenceInst(LLVMContext &C, AtomicOrdering Ordering,
 
 void GetElementPtrInst::init(Value *Ptr, ArrayRef<Value *> IdxList,
                              const Twine &Name) {
-  assert(NumOperands == 1 + IdxList.size() && "NumOperands not initialized?");
-  OperandList[0] = Ptr;
+  assert(getNumOperands() == 1 + IdxList.size() &&
+         "NumOperands not initialized?");
+  Op<0>() = Ptr;
   std::copy(IdxList.begin(), IdxList.end(), op_begin() + 1);
   setName(Name);
 }
 
 GetElementPtrInst::GetElementPtrInst(const GetElementPtrInst &GEPI)
-  : Instruction(GEPI.getType(), GetElementPtr,
-                OperandTraits<GetElementPtrInst>::op_end(this)
-                - GEPI.getNumOperands(),
-                GEPI.getNumOperands()) {
+    : Instruction(GEPI.getType(), GetElementPtr,
+                  OperandTraits<GetElementPtrInst>::op_end(this) -
+                      GEPI.getNumOperands(),
+                  GEPI.getNumOperands()),
+      SourceElementType(GEPI.SourceElementType),
+      ResultElementType(GEPI.ResultElementType) {
   std::copy(GEPI.op_begin(), GEPI.op_end(), op_begin());
   SubclassOptionalData = GEPI.SubclassOptionalData;
 }
@@ -1554,7 +1806,7 @@ void ShuffleVectorInst::getShuffleMask(Constant *Mask,
 
 void InsertValueInst::init(Value *Agg, Value *Val, ArrayRef<unsigned> Idxs, 
                            const Twine &Name) {
-  assert(NumOperands == 2 && "NumOperands not initialized?");
+  assert(getNumOperands() == 2 && "NumOperands not initialized?");
 
   // There's no fundamental reason why we require at least one index
   // (other than weirdness with &*IdxBegin being invalid; see
@@ -1585,7 +1837,7 @@ InsertValueInst::InsertValueInst(const InsertValueInst &IVI)
 //===----------------------------------------------------------------------===//
 
 void ExtractValueInst::init(ArrayRef<unsigned> Idxs, const Twine &Name) {
-  assert(NumOperands == 1 && "NumOperands not initialized?");
+  assert(getNumOperands() == 1 && "NumOperands not initialized?");
 
   // There's no fundamental reason why we require at least one index.
   // But there's no present need to support it.
@@ -2118,13 +2370,13 @@ unsigned CastInst::isEliminableCastPair(
     // N  X  X  U  S  F  F  N  X  N  2  V  V   |
     // C  T  T  I  I  P  P  C  T  T  P  T  T  -+
     {  1, 0, 0,99,99, 0, 0,99,99,99, 0, 3, 0}, // Trunc         -+
-    {  8, 1, 9,99,99, 2, 0,99,99,99, 2, 3, 0}, // ZExt           |
+    {  8, 1, 9,99,99, 2,17,99,99,99, 2, 3, 0}, // ZExt           |
     {  8, 0, 1,99,99, 0, 2,99,99,99, 0, 3, 0}, // SExt           |
     {  0, 0, 0,99,99, 0, 0,99,99,99, 0, 3, 0}, // FPToUI         |
     {  0, 0, 0,99,99, 0, 0,99,99,99, 0, 3, 0}, // FPToSI         |
     { 99,99,99, 0, 0,99,99, 0, 0,99,99, 4, 0}, // UIToFP         +- firstOp
     { 99,99,99, 0, 0,99,99, 0, 0,99,99, 4, 0}, // SIToFP         |
-    { 99,99,99, 0, 0,99,99, 1, 0,99,99, 4, 0}, // FPTrunc        |
+    { 99,99,99, 0, 0,99,99, 0, 0,99,99, 4, 0}, // FPTrunc        |
     { 99,99,99, 2, 2,99,99,10, 2,99,99, 4, 0}, // FPExt          |
     {  1, 0, 0,99,99, 0, 0,99,99,99, 7, 3, 0}, // PtrToInt       |
     { 99,99,99,99,99,99,99,99,99,11,99,15, 0}, // IntToPtr       |
@@ -2285,6 +2537,9 @@ unsigned CastInst::isEliminableCastPair(
         "Illegal bitcast, ptrtoint sequence!");
       // Allowed, use second cast's opcode
       return secondOp;
+    case 17:
+      // (sitofp (zext x)) -> (uitofp x)
+      return Instruction::UIToFP;
     case 99: 
       // Cast combination can't happen (error in input). This is for all cases
       // where the MidTy is not the same for the two cast instructions.
@@ -2732,9 +2987,11 @@ CastInst::getCastOpcode(
 /// it in one place and to eliminate the redundant code for getting the sizes
 /// of the types involved.
 bool 
-// @LOCALMOD-BEGIN
-CastInst::castIsValid(Instruction::CastOps op, Type *SrcTy, Type *DstTy) {
-// @LOCALMOD-END
+CastInst::castIsValid(Instruction::CastOps op, Value *S, Type *DstTy) {
+
+  // Check for type sanity on the arguments
+  Type *SrcTy = S->getType();
+
   if (!SrcTy->isFirstClassType() || !DstTy->isFirstClassType() ||
       SrcTy->isAggregateType() || DstTy->isAggregateType())
     return false;
@@ -3294,11 +3551,11 @@ bool CmpInst::isFalseWhenEqual(unsigned short predicate) {
 void SwitchInst::init(Value *Value, BasicBlock *Default, unsigned NumReserved) {
   assert(Value && Default && NumReserved);
   ReservedSpace = NumReserved;
-  NumOperands = 2;
-  OperandList = allocHungoffUses(ReservedSpace);
+  setNumHungOffUseOperands(2);
+  allocHungoffUses(ReservedSpace);
 
-  OperandList[0] = Value;
-  OperandList[1] = Default;
+  Op<0>() = Value;
+  Op<1>() = Default;
 }
 
 /// SwitchInst ctor - Create a new switch instruction, specifying a value to
@@ -3326,8 +3583,9 @@ SwitchInst::SwitchInst(Value *Value, BasicBlock *Default, unsigned NumCases,
 SwitchInst::SwitchInst(const SwitchInst &SI)
   : TerminatorInst(SI.getType(), Instruction::Switch, nullptr, 0) {
   init(SI.getCondition(), SI.getDefaultDest(), SI.getNumOperands());
-  NumOperands = SI.getNumOperands();
-  Use *OL = OperandList, *InOL = SI.OperandList;
+  setNumHungOffUseOperands(SI.getNumOperands());
+  Use *OL = getOperandList();
+  const Use *InOL = SI.getOperandList();
   for (unsigned i = 2, E = SI.getNumOperands(); i != E; i += 2) {
     OL[i] = InOL[i];
     OL[i+1] = InOL[i+1];
@@ -3335,21 +3593,17 @@ SwitchInst::SwitchInst(const SwitchInst &SI)
   SubclassOptionalData = SI.SubclassOptionalData;
 }
 
-SwitchInst::~SwitchInst() {
-  dropHungoffUses();
-}
-
 
 /// addCase - Add an entry to the switch instruction...
 ///
 void SwitchInst::addCase(ConstantInt *OnVal, BasicBlock *Dest) {
-  unsigned NewCaseIdx = getNumCases(); 
-  unsigned OpNo = NumOperands;
+  unsigned NewCaseIdx = getNumCases();
+  unsigned OpNo = getNumOperands();
   if (OpNo+2 > ReservedSpace)
     growOperands();  // Get more space!
   // Initialize some new operands.
   assert(OpNo+1 < ReservedSpace && "Growing didn't work!");
-  NumOperands = OpNo+2;
+  setNumHungOffUseOperands(OpNo+2);
   CaseIt Case(this, NewCaseIdx);
   Case.setValue(OnVal);
   Case.setSuccessor(Dest);
@@ -3363,7 +3617,7 @@ void SwitchInst::removeCase(CaseIt i) {
   assert(2 + idx*2 < getNumOperands() && "Case index out of range!!!");
 
   unsigned NumOps = getNumOperands();
-  Use *OL = OperandList;
+  Use *OL = getOperandList();
 
   // Overwrite this case with the end of the list.
   if (2 + (idx + 1) * 2 != NumOps) {
@@ -3374,7 +3628,7 @@ void SwitchInst::removeCase(CaseIt i) {
   // Nuke the last value.
   OL[NumOps-2].set(nullptr);
   OL[NumOps-2+1].set(nullptr);
-  NumOperands = NumOps-2;
+  setNumHungOffUseOperands(NumOps-2);
 }
 
 /// growOperands - grow operands - This grows the operand list in response
@@ -3385,13 +3639,7 @@ void SwitchInst::growOperands() {
   unsigned NumOps = e*3;
 
   ReservedSpace = NumOps;
-  Use *NewOps = allocHungoffUses(NumOps);
-  Use *OldOps = OperandList;
-  for (unsigned i = 0; i != e; ++i) {
-      NewOps[i] = OldOps[i];
-  }
-  OperandList = NewOps;
-  Use::zap(OldOps, OldOps + e, true);
+  growHungoffUses(ReservedSpace);
 }
 
 
@@ -3413,10 +3661,10 @@ void IndirectBrInst::init(Value *Address, unsigned NumDests) {
   assert(Address && Address->getType()->isPointerTy() &&
          "Address of indirectbr must be a pointer");
   ReservedSpace = 1+NumDests;
-  NumOperands = 1;
-  OperandList = allocHungoffUses(ReservedSpace);
-  
-  OperandList[0] = Address;
+  setNumHungOffUseOperands(1);
+  allocHungoffUses(ReservedSpace);
+
+  Op<0>() = Address;
 }
 
 
@@ -3428,12 +3676,7 @@ void IndirectBrInst::growOperands() {
   unsigned NumOps = e*2;
   
   ReservedSpace = NumOps;
-  Use *NewOps = allocHungoffUses(NumOps);
-  Use *OldOps = OperandList;
-  for (unsigned i = 0; i != e; ++i)
-    NewOps[i] = OldOps[i];
-  OperandList = NewOps;
-  Use::zap(OldOps, OldOps + e, true);
+  growHungoffUses(ReservedSpace);
 }
 
 IndirectBrInst::IndirectBrInst(Value *Address, unsigned NumCases,
@@ -3451,29 +3694,26 @@ IndirectBrInst::IndirectBrInst(Value *Address, unsigned NumCases,
 }
 
 IndirectBrInst::IndirectBrInst(const IndirectBrInst &IBI)
-  : TerminatorInst(Type::getVoidTy(IBI.getContext()), Instruction::IndirectBr,
-                   allocHungoffUses(IBI.getNumOperands()),
-                   IBI.getNumOperands()) {
-  Use *OL = OperandList, *InOL = IBI.OperandList;
+    : TerminatorInst(Type::getVoidTy(IBI.getContext()), Instruction::IndirectBr,
+                     nullptr, IBI.getNumOperands()) {
+  allocHungoffUses(IBI.getNumOperands());
+  Use *OL = getOperandList();
+  const Use *InOL = IBI.getOperandList();
   for (unsigned i = 0, E = IBI.getNumOperands(); i != E; ++i)
     OL[i] = InOL[i];
   SubclassOptionalData = IBI.SubclassOptionalData;
 }
 
-IndirectBrInst::~IndirectBrInst() {
-  dropHungoffUses();
-}
-
 /// addDestination - Add a destination.
 ///
 void IndirectBrInst::addDestination(BasicBlock *DestBB) {
-  unsigned OpNo = NumOperands;
+  unsigned OpNo = getNumOperands();
   if (OpNo+1 > ReservedSpace)
     growOperands();  // Get more space!
   // Initialize some new operands.
   assert(OpNo < ReservedSpace && "Growing didn't work!");
-  NumOperands = OpNo+1;
-  OperandList[OpNo] = DestBB;
+  setNumHungOffUseOperands(OpNo+1);
+  getOperandList()[OpNo] = DestBB;
 }
 
 /// removeDestination - This method removes the specified successor from the
@@ -3482,14 +3722,14 @@ void IndirectBrInst::removeDestination(unsigned idx) {
   assert(idx < getNumOperands()-1 && "Successor index out of range!");
   
   unsigned NumOps = getNumOperands();
-  Use *OL = OperandList;
+  Use *OL = getOperandList();
 
   // Replace this value with the last one.
   OL[idx+1] = OL[NumOps-1];
   
   // Nuke the last value.
   OL[NumOps-1].set(nullptr);
-  NumOperands = NumOps-1;
+  setNumHungOffUseOperands(NumOps-1);
 }
 
 BasicBlock *IndirectBrInst::getSuccessorV(unsigned idx) const {
@@ -3503,55 +3743,55 @@ void IndirectBrInst::setSuccessorV(unsigned idx, BasicBlock *B) {
 }
 
 //===----------------------------------------------------------------------===//
-//                           clone_impl() implementations
+//                           cloneImpl() implementations
 //===----------------------------------------------------------------------===//
 
 // Define these methods here so vtables don't get emitted into every translation
 // unit that uses these classes.
 
-GetElementPtrInst *GetElementPtrInst::clone_impl() const {
+GetElementPtrInst *GetElementPtrInst::cloneImpl() const {
   return new (getNumOperands()) GetElementPtrInst(*this);
 }
 
-BinaryOperator *BinaryOperator::clone_impl() const {
+BinaryOperator *BinaryOperator::cloneImpl() const {
   return Create(getOpcode(), Op<0>(), Op<1>());
 }
 
-FCmpInst* FCmpInst::clone_impl() const {
+FCmpInst *FCmpInst::cloneImpl() const {
   return new FCmpInst(getPredicate(), Op<0>(), Op<1>());
 }
 
-ICmpInst* ICmpInst::clone_impl() const {
+ICmpInst *ICmpInst::cloneImpl() const {
   return new ICmpInst(getPredicate(), Op<0>(), Op<1>());
 }
 
-ExtractValueInst *ExtractValueInst::clone_impl() const {
+ExtractValueInst *ExtractValueInst::cloneImpl() const {
   return new ExtractValueInst(*this);
 }
 
-InsertValueInst *InsertValueInst::clone_impl() const {
+InsertValueInst *InsertValueInst::cloneImpl() const {
   return new InsertValueInst(*this);
 }
 
-AllocaInst *AllocaInst::clone_impl() const {
+AllocaInst *AllocaInst::cloneImpl() const {
   AllocaInst *Result = new AllocaInst(getAllocatedType(),
                                       (Value *)getOperand(0), getAlignment());
   Result->setUsedWithInAlloca(isUsedWithInAlloca());
   return Result;
 }
 
-LoadInst *LoadInst::clone_impl() const {
+LoadInst *LoadInst::cloneImpl() const {
   return new LoadInst(getOperand(0), Twine(), isVolatile(),
                       getAlignment(), getOrdering(), getSynchScope());
 }
 
-StoreInst *StoreInst::clone_impl() const {
+StoreInst *StoreInst::cloneImpl() const {
   return new StoreInst(getOperand(0), getOperand(1), isVolatile(),
                        getAlignment(), getOrdering(), getSynchScope());
   
 }
 
-AtomicCmpXchgInst *AtomicCmpXchgInst::clone_impl() const {
+AtomicCmpXchgInst *AtomicCmpXchgInst::cloneImpl() const {
   AtomicCmpXchgInst *Result =
     new AtomicCmpXchgInst(getOperand(0), getOperand(1), getOperand(2),
                           getSuccessOrdering(), getFailureOrdering(),
@@ -3561,7 +3801,7 @@ AtomicCmpXchgInst *AtomicCmpXchgInst::clone_impl() const {
   return Result;
 }
 
-AtomicRMWInst *AtomicRMWInst::clone_impl() const {
+AtomicRMWInst *AtomicRMWInst::cloneImpl() const {
   AtomicRMWInst *Result =
     new AtomicRMWInst(getOperation(),getOperand(0), getOperand(1),
                       getOrdering(), getSynchScope());
@@ -3569,120 +3809,137 @@ AtomicRMWInst *AtomicRMWInst::clone_impl() const {
   return Result;
 }
 
-FenceInst *FenceInst::clone_impl() const {
+FenceInst *FenceInst::cloneImpl() const {
   return new FenceInst(getContext(), getOrdering(), getSynchScope());
 }
 
-TruncInst *TruncInst::clone_impl() const {
+TruncInst *TruncInst::cloneImpl() const {
   return new TruncInst(getOperand(0), getType());
 }
 
-ZExtInst *ZExtInst::clone_impl() const {
+ZExtInst *ZExtInst::cloneImpl() const {
   return new ZExtInst(getOperand(0), getType());
 }
 
-SExtInst *SExtInst::clone_impl() const {
+SExtInst *SExtInst::cloneImpl() const {
   return new SExtInst(getOperand(0), getType());
 }
 
-FPTruncInst *FPTruncInst::clone_impl() const {
+FPTruncInst *FPTruncInst::cloneImpl() const {
   return new FPTruncInst(getOperand(0), getType());
 }
 
-FPExtInst *FPExtInst::clone_impl() const {
+FPExtInst *FPExtInst::cloneImpl() const {
   return new FPExtInst(getOperand(0), getType());
 }
 
-UIToFPInst *UIToFPInst::clone_impl() const {
+UIToFPInst *UIToFPInst::cloneImpl() const {
   return new UIToFPInst(getOperand(0), getType());
 }
 
-SIToFPInst *SIToFPInst::clone_impl() const {
+SIToFPInst *SIToFPInst::cloneImpl() const {
   return new SIToFPInst(getOperand(0), getType());
 }
 
-FPToUIInst *FPToUIInst::clone_impl() const {
+FPToUIInst *FPToUIInst::cloneImpl() const {
   return new FPToUIInst(getOperand(0), getType());
 }
 
-FPToSIInst *FPToSIInst::clone_impl() const {
+FPToSIInst *FPToSIInst::cloneImpl() const {
   return new FPToSIInst(getOperand(0), getType());
 }
 
-PtrToIntInst *PtrToIntInst::clone_impl() const {
+PtrToIntInst *PtrToIntInst::cloneImpl() const {
   return new PtrToIntInst(getOperand(0), getType());
 }
 
-IntToPtrInst *IntToPtrInst::clone_impl() const {
+IntToPtrInst *IntToPtrInst::cloneImpl() const {
   return new IntToPtrInst(getOperand(0), getType());
 }
 
-BitCastInst *BitCastInst::clone_impl() const {
+BitCastInst *BitCastInst::cloneImpl() const {
   return new BitCastInst(getOperand(0), getType());
 }
 
-AddrSpaceCastInst *AddrSpaceCastInst::clone_impl() const {
+AddrSpaceCastInst *AddrSpaceCastInst::cloneImpl() const {
   return new AddrSpaceCastInst(getOperand(0), getType());
 }
 
-CallInst *CallInst::clone_impl() const {
+CallInst *CallInst::cloneImpl() const {
   return  new(getNumOperands()) CallInst(*this);
 }
 
-SelectInst *SelectInst::clone_impl() const {
+SelectInst *SelectInst::cloneImpl() const {
   return SelectInst::Create(getOperand(0), getOperand(1), getOperand(2));
 }
 
-VAArgInst *VAArgInst::clone_impl() const {
+VAArgInst *VAArgInst::cloneImpl() const {
   return new VAArgInst(getOperand(0), getType());
 }
 
-ExtractElementInst *ExtractElementInst::clone_impl() const {
+ExtractElementInst *ExtractElementInst::cloneImpl() const {
   return ExtractElementInst::Create(getOperand(0), getOperand(1));
 }
 
-InsertElementInst *InsertElementInst::clone_impl() const {
+InsertElementInst *InsertElementInst::cloneImpl() const {
   return InsertElementInst::Create(getOperand(0), getOperand(1), getOperand(2));
 }
 
-ShuffleVectorInst *ShuffleVectorInst::clone_impl() const {
+ShuffleVectorInst *ShuffleVectorInst::cloneImpl() const {
   return new ShuffleVectorInst(getOperand(0), getOperand(1), getOperand(2));
 }
 
-PHINode *PHINode::clone_impl() const {
-  return new PHINode(*this);
-}
+PHINode *PHINode::cloneImpl() const { return new PHINode(*this); }
 
-LandingPadInst *LandingPadInst::clone_impl() const {
+LandingPadInst *LandingPadInst::cloneImpl() const {
   return new LandingPadInst(*this);
 }
 
-ReturnInst *ReturnInst::clone_impl() const {
+ReturnInst *ReturnInst::cloneImpl() const {
   return new(getNumOperands()) ReturnInst(*this);
 }
 
-BranchInst *BranchInst::clone_impl() const {
+BranchInst *BranchInst::cloneImpl() const {
   return new(getNumOperands()) BranchInst(*this);
 }
 
-SwitchInst *SwitchInst::clone_impl() const {
-  return new SwitchInst(*this);
-}
+SwitchInst *SwitchInst::cloneImpl() const { return new SwitchInst(*this); }
 
-IndirectBrInst *IndirectBrInst::clone_impl() const {
+IndirectBrInst *IndirectBrInst::cloneImpl() const {
   return new IndirectBrInst(*this);
 }
 
-
-InvokeInst *InvokeInst::clone_impl() const {
+InvokeInst *InvokeInst::cloneImpl() const {
   return new(getNumOperands()) InvokeInst(*this);
 }
 
-ResumeInst *ResumeInst::clone_impl() const {
-  return new(1) ResumeInst(*this);
+ResumeInst *ResumeInst::cloneImpl() const { return new (1) ResumeInst(*this); }
+
+CleanupReturnInst *CleanupReturnInst::cloneImpl() const {
+  return new (getNumOperands()) CleanupReturnInst(*this);
+}
+
+CatchEndPadInst *CatchEndPadInst::cloneImpl() const {
+  return new (getNumOperands()) CatchEndPadInst(*this);
+}
+
+CatchReturnInst *CatchReturnInst::cloneImpl() const {
+  return new (1) CatchReturnInst(*this);
+}
+
+CatchPadInst *CatchPadInst::cloneImpl() const {
+  return new (getNumOperands()) CatchPadInst(*this);
+}
+
+TerminatePadInst *TerminatePadInst::cloneImpl() const {
+  return new (getNumOperands()) TerminatePadInst(*this);
+}
+
+CleanupPadInst *CleanupPadInst::cloneImpl() const {
+  return new (getNumOperands()) CleanupPadInst(*this);
 }
 
-UnreachableInst *UnreachableInst::clone_impl() const {
+UnreachableInst *UnreachableInst::cloneImpl() const {
   LLVMContext &Context = getContext();
   return new UnreachableInst(Context);
 }
diff --git a/lib/IR/LLVMContext.cpp b/lib/IR/LLVMContext.cpp
index b6d95c4fcf3..6d799e4b965 100644
--- a/lib/IR/LLVMContext.cpp
+++ b/lib/IR/LLVMContext.cpp
@@ -88,11 +88,22 @@ LLVMContext::LLVMContext() : pImpl(new LLVMContextImpl(*this)) {
          "mem_parallel_loop_access kind id drifted");
   (void)MemParallelLoopAccessID;
 
-
   // Create the 'nonnull' metadata kind.
   unsigned NonNullID = getMDKindID("nonnull");
   assert(NonNullID == MD_nonnull && "nonnull kind id drifted");
   (void)NonNullID;
+  
+  // Create the 'dereferenceable' metadata kind.
+  unsigned DereferenceableID = getMDKindID("dereferenceable");
+  assert(DereferenceableID == MD_dereferenceable && 
+         "dereferenceable kind id drifted");
+  (void)DereferenceableID;
+  
+  // Create the 'dereferenceable_or_null' metadata kind.
+  unsigned DereferenceableOrNullID = getMDKindID("dereferenceable_or_null");
+  assert(DereferenceableOrNullID == MD_dereferenceable_or_null && 
+         "dereferenceable_or_null kind id drifted");
+  (void)DereferenceableOrNullID;
 }
 LLVMContext::~LLVMContext() { delete pImpl; }
 
@@ -188,6 +199,20 @@ static bool isDiagnosticEnabled(const DiagnosticInfo &DI) {
   return true;
 }
 
+static const char *getDiagnosticMessagePrefix(DiagnosticSeverity Severity) {
+  switch (Severity) {
+  case DS_Error:
+    return "error";
+  case DS_Warning:
+    return "warning";
+  case DS_Remark:
+    return "remark";
+  case DS_Note:
+    return "note";
+  }
+  llvm_unreachable("Unknown DiagnosticSeverity");
+}
+
 void LLVMContext::diagnose(const DiagnosticInfo &DI) {
   // If there is a report handler, use it.
   if (pImpl->DiagnosticHandler) {
@@ -200,25 +225,12 @@ void LLVMContext::diagnose(const DiagnosticInfo &DI) {
     return;
 
   // Otherwise, print the message with a prefix based on the severity.
-  std::string MsgStorage;
-  raw_string_ostream Stream(MsgStorage);
-  DiagnosticPrinterRawOStream DP(Stream);
+  DiagnosticPrinterRawOStream DP(errs());
+  errs() << getDiagnosticMessagePrefix(DI.getSeverity()) << ": ";
   DI.print(DP);
-  Stream.flush();
-  switch (DI.getSeverity()) {
-  case DS_Error:
-    errs() << "error: " << MsgStorage << "\n";
+  errs() << "\n";
+  if (DI.getSeverity() == DS_Error)
     exit(1);
-  case DS_Warning:
-    errs() << "warning: " << MsgStorage << "\n";
-    break;
-  case DS_Remark:
-    errs() << "remark: " << MsgStorage << "\n";
-    break;
-  case DS_Note:
-    errs() << "note: " << MsgStorage << "\n";
-    break;
-  }
 }
 
 void LLVMContext::emitError(unsigned LocCookie, const Twine &ErrorStr) {
@@ -229,15 +241,12 @@ void LLVMContext::emitError(unsigned LocCookie, const Twine &ErrorStr) {
 // Metadata Kind Uniquing
 //===----------------------------------------------------------------------===//
 
-/// getMDKindID - Return a unique non-zero ID for the specified metadata kind.
+/// Return a unique non-zero ID for the specified metadata kind.
 unsigned LLVMContext::getMDKindID(StringRef Name) const {
-  assert(!std::isdigit(Name.front()) &&
-         "Named metadata may not start with a digit");
-
   // If this is new, assign it its ID.
-  return pImpl->CustomMDKindNames.insert(std::make_pair(
-                                             Name,
-                                             pImpl->CustomMDKindNames.size()))
+  return pImpl->CustomMDKindNames.insert(
+                                     std::make_pair(
+                                         Name, pImpl->CustomMDKindNames.size()))
       .first->second;
 }
 
diff --git a/lib/IR/LLVMContextImpl.cpp b/lib/IR/LLVMContextImpl.cpp
index d717b92d9fe..1e2080770fc 100644
--- a/lib/IR/LLVMContextImpl.cpp
+++ b/lib/IR/LLVMContextImpl.cpp
@@ -35,7 +35,8 @@ LLVMContextImpl::LLVMContextImpl(LLVMContext &C)
     Int8Ty(C, 8),
     Int16Ty(C, 16),
     Int32Ty(C, 32),
-    Int64Ty(C, 64) {
+    Int64Ty(C, 64),
+    Int128Ty(C, 128) {
   InlineAsmDiagHandler = nullptr;
   InlineAsmDiagContext = nullptr;
   DiagnosticHandler = nullptr;
diff --git a/lib/IR/LLVMContextImpl.h b/lib/IR/LLVMContextImpl.h
index c096a831b20..cbbf11e334c 100644
--- a/lib/IR/LLVMContextImpl.h
+++ b/lib/IR/LLVMContextImpl.h
@@ -228,8 +228,8 @@ template <> struct MDNodeKeyImpl<MDTuple> : MDNodeOpsKey {
   }
 };
 
-/// \brief DenseMapInfo for MDLocation.
-template <> struct MDNodeKeyImpl<MDLocation> {
+/// \brief DenseMapInfo for DILocation.
+template <> struct MDNodeKeyImpl<DILocation> {
   unsigned Line;
   unsigned Column;
   Metadata *Scope;
@@ -239,11 +239,11 @@ template <> struct MDNodeKeyImpl<MDLocation> {
                 Metadata *InlinedAt)
       : Line(Line), Column(Column), Scope(Scope), InlinedAt(InlinedAt) {}
 
-  MDNodeKeyImpl(const MDLocation *L)
+  MDNodeKeyImpl(const DILocation *L)
       : Line(L->getLine()), Column(L->getColumn()), Scope(L->getRawScope()),
         InlinedAt(L->getRawInlinedAt()) {}
 
-  bool isKeyOf(const MDLocation *RHS) const {
+  bool isKeyOf(const DILocation *RHS) const {
     return Line == RHS->getLine() && Column == RHS->getColumn() &&
            Scope == RHS->getRawScope() && InlinedAt == RHS->getRawInlinedAt();
   }
@@ -252,57 +252,57 @@ template <> struct MDNodeKeyImpl<MDLocation> {
   }
 };
 
-/// \brief DenseMapInfo for GenericDebugNode.
-template <> struct MDNodeKeyImpl<GenericDebugNode> : MDNodeOpsKey {
+/// \brief DenseMapInfo for GenericDINode.
+template <> struct MDNodeKeyImpl<GenericDINode> : MDNodeOpsKey {
   unsigned Tag;
   StringRef Header;
   MDNodeKeyImpl(unsigned Tag, StringRef Header, ArrayRef<Metadata *> DwarfOps)
       : MDNodeOpsKey(DwarfOps), Tag(Tag), Header(Header) {}
-  MDNodeKeyImpl(const GenericDebugNode *N)
+  MDNodeKeyImpl(const GenericDINode *N)
       : MDNodeOpsKey(N, 1), Tag(N->getTag()), Header(N->getHeader()) {}
 
-  bool isKeyOf(const GenericDebugNode *RHS) const {
+  bool isKeyOf(const GenericDINode *RHS) const {
     return Tag == RHS->getTag() && Header == RHS->getHeader() &&
            compareOps(RHS, 1);
   }
 
   unsigned getHashValue() const { return hash_combine(getHash(), Tag, Header); }
 
-  static unsigned calculateHash(GenericDebugNode *N) {
+  static unsigned calculateHash(GenericDINode *N) {
     return MDNodeOpsKey::calculateHash(N, 1);
   }
 };
 
-template <> struct MDNodeKeyImpl<MDSubrange> {
+template <> struct MDNodeKeyImpl<DISubrange> {
   int64_t Count;
   int64_t LowerBound;
 
   MDNodeKeyImpl(int64_t Count, int64_t LowerBound)
       : Count(Count), LowerBound(LowerBound) {}
-  MDNodeKeyImpl(const MDSubrange *N)
+  MDNodeKeyImpl(const DISubrange *N)
       : Count(N->getCount()), LowerBound(N->getLowerBound()) {}
 
-  bool isKeyOf(const MDSubrange *RHS) const {
+  bool isKeyOf(const DISubrange *RHS) const {
     return Count == RHS->getCount() && LowerBound == RHS->getLowerBound();
   }
   unsigned getHashValue() const { return hash_combine(Count, LowerBound); }
 };
 
-template <> struct MDNodeKeyImpl<MDEnumerator> {
+template <> struct MDNodeKeyImpl<DIEnumerator> {
   int64_t Value;
   StringRef Name;
 
   MDNodeKeyImpl(int64_t Value, StringRef Name) : Value(Value), Name(Name) {}
-  MDNodeKeyImpl(const MDEnumerator *N)
+  MDNodeKeyImpl(const DIEnumerator *N)
       : Value(N->getValue()), Name(N->getName()) {}
 
-  bool isKeyOf(const MDEnumerator *RHS) const {
+  bool isKeyOf(const DIEnumerator *RHS) const {
     return Value == RHS->getValue() && Name == RHS->getName();
   }
   unsigned getHashValue() const { return hash_combine(Value, Name); }
 };
 
-template <> struct MDNodeKeyImpl<MDBasicType> {
+template <> struct MDNodeKeyImpl<DIBasicType> {
   unsigned Tag;
   StringRef Name;
   uint64_t SizeInBits;
@@ -313,11 +313,11 @@ template <> struct MDNodeKeyImpl<MDBasicType> {
                 uint64_t AlignInBits, unsigned Encoding)
       : Tag(Tag), Name(Name), SizeInBits(SizeInBits), AlignInBits(AlignInBits),
         Encoding(Encoding) {}
-  MDNodeKeyImpl(const MDBasicType *N)
+  MDNodeKeyImpl(const DIBasicType *N)
       : Tag(N->getTag()), Name(N->getName()), SizeInBits(N->getSizeInBits()),
         AlignInBits(N->getAlignInBits()), Encoding(N->getEncoding()) {}
 
-  bool isKeyOf(const MDBasicType *RHS) const {
+  bool isKeyOf(const DIBasicType *RHS) const {
     return Tag == RHS->getTag() && Name == RHS->getName() &&
            SizeInBits == RHS->getSizeInBits() &&
            AlignInBits == RHS->getAlignInBits() &&
@@ -328,7 +328,7 @@ template <> struct MDNodeKeyImpl<MDBasicType> {
   }
 };
 
-template <> struct MDNodeKeyImpl<MDDerivedType> {
+template <> struct MDNodeKeyImpl<DIDerivedType> {
   unsigned Tag;
   StringRef Name;
   Metadata *File;
@@ -348,14 +348,14 @@ template <> struct MDNodeKeyImpl<MDDerivedType> {
       : Tag(Tag), Name(Name), File(File), Line(Line), Scope(Scope),
         BaseType(BaseType), SizeInBits(SizeInBits), AlignInBits(AlignInBits),
         OffsetInBits(OffsetInBits), Flags(Flags), ExtraData(ExtraData) {}
-  MDNodeKeyImpl(const MDDerivedType *N)
+  MDNodeKeyImpl(const DIDerivedType *N)
       : Tag(N->getTag()), Name(N->getName()), File(N->getRawFile()),
         Line(N->getLine()), Scope(N->getRawScope()),
         BaseType(N->getRawBaseType()), SizeInBits(N->getSizeInBits()),
         AlignInBits(N->getAlignInBits()), OffsetInBits(N->getOffsetInBits()),
         Flags(N->getFlags()), ExtraData(N->getRawExtraData()) {}
 
-  bool isKeyOf(const MDDerivedType *RHS) const {
+  bool isKeyOf(const DIDerivedType *RHS) const {
     return Tag == RHS->getTag() && Name == RHS->getName() &&
            File == RHS->getRawFile() && Line == RHS->getLine() &&
            Scope == RHS->getRawScope() && BaseType == RHS->getRawBaseType() &&
@@ -370,7 +370,7 @@ template <> struct MDNodeKeyImpl<MDDerivedType> {
   }
 };
 
-template <> struct MDNodeKeyImpl<MDCompositeType> {
+template <> struct MDNodeKeyImpl<DICompositeType> {
   unsigned Tag;
   StringRef Name;
   Metadata *File;
@@ -398,7 +398,7 @@ template <> struct MDNodeKeyImpl<MDCompositeType> {
         OffsetInBits(OffsetInBits), Flags(Flags), Elements(Elements),
         RuntimeLang(RuntimeLang), VTableHolder(VTableHolder),
         TemplateParams(TemplateParams), Identifier(Identifier) {}
-  MDNodeKeyImpl(const MDCompositeType *N)
+  MDNodeKeyImpl(const DICompositeType *N)
       : Tag(N->getTag()), Name(N->getName()), File(N->getRawFile()),
         Line(N->getLine()), Scope(N->getRawScope()),
         BaseType(N->getRawBaseType()), SizeInBits(N->getSizeInBits()),
@@ -408,7 +408,7 @@ template <> struct MDNodeKeyImpl<MDCompositeType> {
         TemplateParams(N->getRawTemplateParams()),
         Identifier(N->getIdentifier()) {}
 
-  bool isKeyOf(const MDCompositeType *RHS) const {
+  bool isKeyOf(const DICompositeType *RHS) const {
     return Tag == RHS->getTag() && Name == RHS->getName() &&
            File == RHS->getRawFile() && Line == RHS->getLine() &&
            Scope == RHS->getRawScope() && BaseType == RHS->getRawBaseType() &&
@@ -428,37 +428,37 @@ template <> struct MDNodeKeyImpl<MDCompositeType> {
   }
 };
 
-template <> struct MDNodeKeyImpl<MDSubroutineType> {
+template <> struct MDNodeKeyImpl<DISubroutineType> {
   unsigned Flags;
   Metadata *TypeArray;
 
   MDNodeKeyImpl(int64_t Flags, Metadata *TypeArray)
       : Flags(Flags), TypeArray(TypeArray) {}
-  MDNodeKeyImpl(const MDSubroutineType *N)
+  MDNodeKeyImpl(const DISubroutineType *N)
       : Flags(N->getFlags()), TypeArray(N->getRawTypeArray()) {}
 
-  bool isKeyOf(const MDSubroutineType *RHS) const {
+  bool isKeyOf(const DISubroutineType *RHS) const {
     return Flags == RHS->getFlags() && TypeArray == RHS->getRawTypeArray();
   }
   unsigned getHashValue() const { return hash_combine(Flags, TypeArray); }
 };
 
-template <> struct MDNodeKeyImpl<MDFile> {
+template <> struct MDNodeKeyImpl<DIFile> {
   StringRef Filename;
   StringRef Directory;
 
   MDNodeKeyImpl(StringRef Filename, StringRef Directory)
       : Filename(Filename), Directory(Directory) {}
-  MDNodeKeyImpl(const MDFile *N)
+  MDNodeKeyImpl(const DIFile *N)
       : Filename(N->getFilename()), Directory(N->getDirectory()) {}
 
-  bool isKeyOf(const MDFile *RHS) const {
+  bool isKeyOf(const DIFile *RHS) const {
     return Filename == RHS->getFilename() && Directory == RHS->getDirectory();
   }
   unsigned getHashValue() const { return hash_combine(Filename, Directory); }
 };
 
-template <> struct MDNodeKeyImpl<MDCompileUnit> {
+template <> struct MDNodeKeyImpl<DICompileUnit> {
   unsigned SourceLanguage;
   Metadata *File;
   StringRef Producer;
@@ -472,20 +472,21 @@ template <> struct MDNodeKeyImpl<MDCompileUnit> {
   Metadata *Subprograms;
   Metadata *GlobalVariables;
   Metadata *ImportedEntities;
+  uint64_t DWOId;
 
   MDNodeKeyImpl(unsigned SourceLanguage, Metadata *File, StringRef Producer,
                 bool IsOptimized, StringRef Flags, unsigned RuntimeVersion,
                 StringRef SplitDebugFilename, unsigned EmissionKind,
                 Metadata *EnumTypes, Metadata *RetainedTypes,
                 Metadata *Subprograms, Metadata *GlobalVariables,
-                Metadata *ImportedEntities)
+                Metadata *ImportedEntities, uint64_t DWOId)
       : SourceLanguage(SourceLanguage), File(File), Producer(Producer),
         IsOptimized(IsOptimized), Flags(Flags), RuntimeVersion(RuntimeVersion),
         SplitDebugFilename(SplitDebugFilename), EmissionKind(EmissionKind),
         EnumTypes(EnumTypes), RetainedTypes(RetainedTypes),
         Subprograms(Subprograms), GlobalVariables(GlobalVariables),
-        ImportedEntities(ImportedEntities) {}
-  MDNodeKeyImpl(const MDCompileUnit *N)
+        ImportedEntities(ImportedEntities), DWOId(DWOId) {}
+  MDNodeKeyImpl(const DICompileUnit *N)
       : SourceLanguage(N->getSourceLanguage()), File(N->getRawFile()),
         Producer(N->getProducer()), IsOptimized(N->isOptimized()),
         Flags(N->getFlags()), RuntimeVersion(N->getRuntimeVersion()),
@@ -494,9 +495,9 @@ template <> struct MDNodeKeyImpl<MDCompileUnit> {
         RetainedTypes(N->getRawRetainedTypes()),
         Subprograms(N->getRawSubprograms()),
         GlobalVariables(N->getRawGlobalVariables()),
-        ImportedEntities(N->getRawImportedEntities()) {}
+        ImportedEntities(N->getRawImportedEntities()), DWOId(N->getDWOId()) {}
 
-  bool isKeyOf(const MDCompileUnit *RHS) const {
+  bool isKeyOf(const DICompileUnit *RHS) const {
     return SourceLanguage == RHS->getSourceLanguage() &&
            File == RHS->getRawFile() && Producer == RHS->getProducer() &&
            IsOptimized == RHS->isOptimized() && Flags == RHS->getFlags() &&
@@ -507,17 +508,18 @@ template <> struct MDNodeKeyImpl<MDCompileUnit> {
            RetainedTypes == RHS->getRawRetainedTypes() &&
            Subprograms == RHS->getRawSubprograms() &&
            GlobalVariables == RHS->getRawGlobalVariables() &&
-           ImportedEntities == RHS->getRawImportedEntities();
+           ImportedEntities == RHS->getRawImportedEntities() &&
+           DWOId == RHS->getDWOId();
   }
   unsigned getHashValue() const {
     return hash_combine(SourceLanguage, File, Producer, IsOptimized, Flags,
                         RuntimeVersion, SplitDebugFilename, EmissionKind,
                         EnumTypes, RetainedTypes, Subprograms, GlobalVariables,
-                        ImportedEntities);
+                        ImportedEntities, DWOId);
   }
 };
 
-template <> struct MDNodeKeyImpl<MDSubprogram> {
+template <> struct MDNodeKeyImpl<DISubprogram> {
   Metadata *Scope;
   StringRef Name;
   StringRef LinkageName;
@@ -551,7 +553,7 @@ template <> struct MDNodeKeyImpl<MDSubprogram> {
         VirtualIndex(VirtualIndex), Flags(Flags), IsOptimized(IsOptimized),
         Function(Function), TemplateParams(TemplateParams),
         Declaration(Declaration), Variables(Variables) {}
-  MDNodeKeyImpl(const MDSubprogram *N)
+  MDNodeKeyImpl(const DISubprogram *N)
       : Scope(N->getRawScope()), Name(N->getName()),
         LinkageName(N->getLinkageName()), File(N->getRawFile()),
         Line(N->getLine()), Type(N->getRawType()),
@@ -563,7 +565,7 @@ template <> struct MDNodeKeyImpl<MDSubprogram> {
         TemplateParams(N->getRawTemplateParams()),
         Declaration(N->getRawDeclaration()), Variables(N->getRawVariables()) {}
 
-  bool isKeyOf(const MDSubprogram *RHS) const {
+  bool isKeyOf(const DISubprogram *RHS) const {
     return Scope == RHS->getRawScope() && Name == RHS->getName() &&
            LinkageName == RHS->getLinkageName() && File == RHS->getRawFile() &&
            Line == RHS->getLine() && Type == RHS->getRawType() &&
@@ -587,7 +589,7 @@ template <> struct MDNodeKeyImpl<MDSubprogram> {
   }
 };
 
-template <> struct MDNodeKeyImpl<MDLexicalBlock> {
+template <> struct MDNodeKeyImpl<DILexicalBlock> {
   Metadata *Scope;
   Metadata *File;
   unsigned Line;
@@ -595,11 +597,11 @@ template <> struct MDNodeKeyImpl<MDLexicalBlock> {
 
   MDNodeKeyImpl(Metadata *Scope, Metadata *File, unsigned Line, unsigned Column)
       : Scope(Scope), File(File), Line(Line), Column(Column) {}
-  MDNodeKeyImpl(const MDLexicalBlock *N)
+  MDNodeKeyImpl(const DILexicalBlock *N)
       : Scope(N->getRawScope()), File(N->getRawFile()), Line(N->getLine()),
         Column(N->getColumn()) {}
 
-  bool isKeyOf(const MDLexicalBlock *RHS) const {
+  bool isKeyOf(const DILexicalBlock *RHS) const {
     return Scope == RHS->getRawScope() && File == RHS->getRawFile() &&
            Line == RHS->getLine() && Column == RHS->getColumn();
   }
@@ -608,18 +610,18 @@ template <> struct MDNodeKeyImpl<MDLexicalBlock> {
   }
 };
 
-template <> struct MDNodeKeyImpl<MDLexicalBlockFile> {
+template <> struct MDNodeKeyImpl<DILexicalBlockFile> {
   Metadata *Scope;
   Metadata *File;
   unsigned Discriminator;
 
   MDNodeKeyImpl(Metadata *Scope, Metadata *File, unsigned Discriminator)
       : Scope(Scope), File(File), Discriminator(Discriminator) {}
-  MDNodeKeyImpl(const MDLexicalBlockFile *N)
+  MDNodeKeyImpl(const DILexicalBlockFile *N)
       : Scope(N->getRawScope()), File(N->getRawFile()),
         Discriminator(N->getDiscriminator()) {}
 
-  bool isKeyOf(const MDLexicalBlockFile *RHS) const {
+  bool isKeyOf(const DILexicalBlockFile *RHS) const {
     return Scope == RHS->getRawScope() && File == RHS->getRawFile() &&
            Discriminator == RHS->getDiscriminator();
   }
@@ -628,7 +630,7 @@ template <> struct MDNodeKeyImpl<MDLexicalBlockFile> {
   }
 };
 
-template <> struct MDNodeKeyImpl<MDNamespace> {
+template <> struct MDNodeKeyImpl<DINamespace> {
   Metadata *Scope;
   Metadata *File;
   StringRef Name;
@@ -636,11 +638,11 @@ template <> struct MDNodeKeyImpl<MDNamespace> {
 
   MDNodeKeyImpl(Metadata *Scope, Metadata *File, StringRef Name, unsigned Line)
       : Scope(Scope), File(File), Name(Name), Line(Line) {}
-  MDNodeKeyImpl(const MDNamespace *N)
+  MDNodeKeyImpl(const DINamespace *N)
       : Scope(N->getRawScope()), File(N->getRawFile()), Name(N->getName()),
         Line(N->getLine()) {}
 
-  bool isKeyOf(const MDNamespace *RHS) const {
+  bool isKeyOf(const DINamespace *RHS) const {
     return Scope == RHS->getRawScope() && File == RHS->getRawFile() &&
            Name == RHS->getName() && Line == RHS->getLine();
   }
@@ -649,21 +651,50 @@ template <> struct MDNodeKeyImpl<MDNamespace> {
   }
 };
 
-template <> struct MDNodeKeyImpl<MDTemplateTypeParameter> {
+template <> struct MDNodeKeyImpl<DIModule> {
+  Metadata *Scope;
+  StringRef Name;
+  StringRef ConfigurationMacros;
+  StringRef IncludePath;
+  StringRef ISysRoot;
+  MDNodeKeyImpl(Metadata *Scope, StringRef Name,
+                StringRef ConfigurationMacros,
+                StringRef IncludePath,
+                StringRef ISysRoot)
+    : Scope(Scope), Name(Name), ConfigurationMacros(ConfigurationMacros),
+      IncludePath(IncludePath), ISysRoot(ISysRoot) {}
+  MDNodeKeyImpl(const DIModule *N)
+    : Scope(N->getRawScope()), Name(N->getName()),
+      ConfigurationMacros(N->getConfigurationMacros()),
+      IncludePath(N->getIncludePath()), ISysRoot(N->getISysRoot()) {}
+
+  bool isKeyOf(const DIModule *RHS) const {
+    return Scope == RHS->getRawScope() && Name == RHS->getName() &&
+           ConfigurationMacros == RHS->getConfigurationMacros() &&
+           IncludePath == RHS->getIncludePath() &&
+           ISysRoot == RHS->getISysRoot();
+  }
+  unsigned getHashValue() const {
+    return hash_combine(Scope, Name,
+                        ConfigurationMacros, IncludePath, ISysRoot);
+  }
+};
+
+template <> struct MDNodeKeyImpl<DITemplateTypeParameter> {
   StringRef Name;
   Metadata *Type;
 
   MDNodeKeyImpl(StringRef Name, Metadata *Type) : Name(Name), Type(Type) {}
-  MDNodeKeyImpl(const MDTemplateTypeParameter *N)
+  MDNodeKeyImpl(const DITemplateTypeParameter *N)
       : Name(N->getName()), Type(N->getRawType()) {}
 
-  bool isKeyOf(const MDTemplateTypeParameter *RHS) const {
+  bool isKeyOf(const DITemplateTypeParameter *RHS) const {
     return Name == RHS->getName() && Type == RHS->getRawType();
   }
   unsigned getHashValue() const { return hash_combine(Name, Type); }
 };
 
-template <> struct MDNodeKeyImpl<MDTemplateValueParameter> {
+template <> struct MDNodeKeyImpl<DITemplateValueParameter> {
   unsigned Tag;
   StringRef Name;
   Metadata *Type;
@@ -671,18 +702,18 @@ template <> struct MDNodeKeyImpl<MDTemplateValueParameter> {
 
   MDNodeKeyImpl(unsigned Tag, StringRef Name, Metadata *Type, Metadata *Value)
       : Tag(Tag), Name(Name), Type(Type), Value(Value) {}
-  MDNodeKeyImpl(const MDTemplateValueParameter *N)
+  MDNodeKeyImpl(const DITemplateValueParameter *N)
       : Tag(N->getTag()), Name(N->getName()), Type(N->getRawType()),
         Value(N->getValue()) {}
 
-  bool isKeyOf(const MDTemplateValueParameter *RHS) const {
+  bool isKeyOf(const DITemplateValueParameter *RHS) const {
     return Tag == RHS->getTag() && Name == RHS->getName() &&
            Type == RHS->getRawType() && Value == RHS->getValue();
   }
   unsigned getHashValue() const { return hash_combine(Tag, Name, Type, Value); }
 };
 
-template <> struct MDNodeKeyImpl<MDGlobalVariable> {
+template <> struct MDNodeKeyImpl<DIGlobalVariable> {
   Metadata *Scope;
   StringRef Name;
   StringRef LinkageName;
@@ -702,7 +733,7 @@ template <> struct MDNodeKeyImpl<MDGlobalVariable> {
         Line(Line), Type(Type), IsLocalToUnit(IsLocalToUnit),
         IsDefinition(IsDefinition), Variable(Variable),
         StaticDataMemberDeclaration(StaticDataMemberDeclaration) {}
-  MDNodeKeyImpl(const MDGlobalVariable *N)
+  MDNodeKeyImpl(const DIGlobalVariable *N)
       : Scope(N->getRawScope()), Name(N->getName()),
         LinkageName(N->getLinkageName()), File(N->getRawFile()),
         Line(N->getLine()), Type(N->getRawType()),
@@ -710,7 +741,7 @@ template <> struct MDNodeKeyImpl<MDGlobalVariable> {
         Variable(N->getRawVariable()),
         StaticDataMemberDeclaration(N->getRawStaticDataMemberDeclaration()) {}
 
-  bool isKeyOf(const MDGlobalVariable *RHS) const {
+  bool isKeyOf(const DIGlobalVariable *RHS) const {
     return Scope == RHS->getRawScope() && Name == RHS->getName() &&
            LinkageName == RHS->getLinkageName() && File == RHS->getRawFile() &&
            Line == RHS->getLine() && Type == RHS->getRawType() &&
@@ -727,7 +758,7 @@ template <> struct MDNodeKeyImpl<MDGlobalVariable> {
   }
 };
 
-template <> struct MDNodeKeyImpl<MDLocalVariable> {
+template <> struct MDNodeKeyImpl<DILocalVariable> {
   unsigned Tag;
   Metadata *Scope;
   StringRef Name;
@@ -741,12 +772,12 @@ template <> struct MDNodeKeyImpl<MDLocalVariable> {
                 unsigned Line, Metadata *Type, unsigned Arg, unsigned Flags)
       : Tag(Tag), Scope(Scope), Name(Name), File(File), Line(Line), Type(Type),
         Arg(Arg), Flags(Flags) {}
-  MDNodeKeyImpl(const MDLocalVariable *N)
+  MDNodeKeyImpl(const DILocalVariable *N)
       : Tag(N->getTag()), Scope(N->getRawScope()), Name(N->getName()),
         File(N->getRawFile()), Line(N->getLine()), Type(N->getRawType()),
         Arg(N->getArg()), Flags(N->getFlags()) {}
 
-  bool isKeyOf(const MDLocalVariable *RHS) const {
+  bool isKeyOf(const DILocalVariable *RHS) const {
     return Tag == RHS->getTag() && Scope == RHS->getRawScope() &&
            Name == RHS->getName() && File == RHS->getRawFile() &&
            Line == RHS->getLine() && Type == RHS->getRawType() &&
@@ -757,13 +788,13 @@ template <> struct MDNodeKeyImpl<MDLocalVariable> {
   }
 };
 
-template <> struct MDNodeKeyImpl<MDExpression> {
+template <> struct MDNodeKeyImpl<DIExpression> {
   ArrayRef<uint64_t> Elements;
 
   MDNodeKeyImpl(ArrayRef<uint64_t> Elements) : Elements(Elements) {}
-  MDNodeKeyImpl(const MDExpression *N) : Elements(N->getElements()) {}
+  MDNodeKeyImpl(const DIExpression *N) : Elements(N->getElements()) {}
 
-  bool isKeyOf(const MDExpression *RHS) const {
+  bool isKeyOf(const DIExpression *RHS) const {
     return Elements == RHS->getElements();
   }
   unsigned getHashValue() const {
@@ -771,7 +802,7 @@ template <> struct MDNodeKeyImpl<MDExpression> {
   }
 };
 
-template <> struct MDNodeKeyImpl<MDObjCProperty> {
+template <> struct MDNodeKeyImpl<DIObjCProperty> {
   StringRef Name;
   Metadata *File;
   unsigned Line;
@@ -785,12 +816,12 @@ template <> struct MDNodeKeyImpl<MDObjCProperty> {
                 Metadata *Type)
       : Name(Name), File(File), Line(Line), GetterName(GetterName),
         SetterName(SetterName), Attributes(Attributes), Type(Type) {}
-  MDNodeKeyImpl(const MDObjCProperty *N)
+  MDNodeKeyImpl(const DIObjCProperty *N)
       : Name(N->getName()), File(N->getRawFile()), Line(N->getLine()),
         GetterName(N->getGetterName()), SetterName(N->getSetterName()),
         Attributes(N->getAttributes()), Type(N->getRawType()) {}
 
-  bool isKeyOf(const MDObjCProperty *RHS) const {
+  bool isKeyOf(const DIObjCProperty *RHS) const {
     return Name == RHS->getName() && File == RHS->getRawFile() &&
            Line == RHS->getLine() && GetterName == RHS->getGetterName() &&
            SetterName == RHS->getSetterName() &&
@@ -802,7 +833,7 @@ template <> struct MDNodeKeyImpl<MDObjCProperty> {
   }
 };
 
-template <> struct MDNodeKeyImpl<MDImportedEntity> {
+template <> struct MDNodeKeyImpl<DIImportedEntity> {
   unsigned Tag;
   Metadata *Scope;
   Metadata *Entity;
@@ -812,11 +843,11 @@ template <> struct MDNodeKeyImpl<MDImportedEntity> {
   MDNodeKeyImpl(unsigned Tag, Metadata *Scope, Metadata *Entity, unsigned Line,
                 StringRef Name)
       : Tag(Tag), Scope(Scope), Entity(Entity), Line(Line), Name(Name) {}
-  MDNodeKeyImpl(const MDImportedEntity *N)
+  MDNodeKeyImpl(const DIImportedEntity *N)
       : Tag(N->getTag()), Scope(N->getRawScope()), Entity(N->getRawEntity()),
         Line(N->getLine()), Name(N->getName()) {}
 
-  bool isKeyOf(const MDImportedEntity *RHS) const {
+  bool isKeyOf(const DIImportedEntity *RHS) const {
     return Tag == RHS->getTag() && Scope == RHS->getRawScope() &&
            Entity == RHS->getRawEntity() && Line == RHS->getLine() &&
            Name == RHS->getName();
@@ -852,6 +883,44 @@ template <class NodeTy> struct MDNodeInfo {
 #define HANDLE_MDNODE_LEAF(CLASS) typedef MDNodeInfo<CLASS> CLASS##Info;
 #include "llvm/IR/Metadata.def"
 
+/// \brief Map-like storage for metadata attachments.
+class MDAttachmentMap {
+  SmallVector<std::pair<unsigned, TrackingMDNodeRef>, 2> Attachments;
+
+public:
+  bool empty() const { return Attachments.empty(); }
+  size_t size() const { return Attachments.size(); }
+
+  /// \brief Get a particular attachment (if any).
+  MDNode *lookup(unsigned ID) const;
+
+  /// \brief Set an attachment to a particular node.
+  ///
+  /// Set the \c ID attachment to \c MD, replacing the current attachment at \c
+  /// ID (if anyway).
+  void set(unsigned ID, MDNode &MD);
+
+  /// \brief Remove an attachment.
+  ///
+  /// Remove the attachment at \c ID, if any.
+  void erase(unsigned ID);
+
+  /// \brief Copy out all the attachments.
+  ///
+  /// Copies all the current attachments into \c Result, sorting by attachment
+  /// ID.  This function does \em not clear \c Result.
+  void getAll(SmallVectorImpl<std::pair<unsigned, MDNode *>> &Result) const;
+
+  /// \brief Erase matching attachments.
+  ///
+  /// Erases all attachments matching the \c shouldRemove predicate.
+  template <class PredTy> void remove_if(PredTy shouldRemove) {
+    Attachments.erase(
+        std::remove_if(Attachments.begin(), Attachments.end(), shouldRemove),
+        Attachments.end());
+  }
+};
+
 class LLVMContextImpl {
 public:
   /// OwnedModules - The set of modules instantiated in this context, and which
@@ -882,6 +951,8 @@ class LLVMContextImpl {
   DenseMap<Value *, ValueAsMetadata *> ValuesAsMetadata;
   DenseMap<Metadata *, MetadataAsValue *> MetadataAsValues;
 
+  DenseMap<const Value*, ValueName*> ValueNames;
+
 #define HANDLE_MDNODE_LEAF(CLASS) DenseSet<CLASS *, CLASS##Info> CLASS##s;
 #include "llvm/IR/Metadata.def"
 
@@ -920,7 +991,7 @@ class LLVMContextImpl {
   // Basic type instances.
   Type VoidTy, LabelTy, HalfTy, FloatTy, DoubleTy, MetadataTy;
   Type X86_FP80Ty, FP128Ty, PPC_FP128Ty, X86_MMXTy;
-  IntegerType Int1Ty, Int8Ty, Int16Ty, Int32Ty, Int64Ty;
+  IntegerType Int1Ty, Int8Ty, Int16Ty, Int32Ty, Int64Ty, Int128Ty;
 
   
   /// TypeAllocator - All dynamically allocated types are allocated from this.
@@ -951,23 +1022,17 @@ class LLVMContextImpl {
   /// CustomMDKindNames - Map to hold the metadata string to ID mapping.
   StringMap<unsigned> CustomMDKindNames;
 
-  typedef std::pair<unsigned, TrackingMDNodeRef> MDPairTy;
-  typedef SmallVector<MDPairTy, 2> MDMapTy;
+  /// Collection of per-instruction metadata used in this context.
+  DenseMap<const Instruction *, MDAttachmentMap> InstructionMetadata;
+
+  /// Collection of per-function metadata used in this context.
+  DenseMap<const Function *, MDAttachmentMap> FunctionMetadata;
 
-  /// MetadataStore - Collection of per-instruction metadata used in this
-  /// context.
-  DenseMap<const Instruction *, MDMapTy> MetadataStore;
-  
   /// DiscriminatorTable - This table maps file:line locations to an
   /// integer representing the next DWARF path discriminator to assign to
   /// instructions in different blocks at the same location.
   DenseMap<std::pair<const char *, unsigned>, unsigned> DiscriminatorTable;
 
-  /// IntrinsicIDCache - Cache of intrinsic name (string) to numeric ID mappings
-  /// requested in this context
-  typedef DenseMap<const Function*, unsigned> IntrinsicIDCacheTy;
-  IntrinsicIDCacheTy IntrinsicIDCache;
-
   /// \brief Mapping from a function to its prefix data, which is stored as the
   /// operand of an unparented ReturnInst so that the prefix data has a Use.
   typedef DenseMap<const Function *, ReturnInst *> PrefixDataMapTy;
diff --git a/lib/IR/LegacyPassManager.cpp b/lib/IR/LegacyPassManager.cpp
index 687003234b6..27d98a279fe 100644
--- a/lib/IR/LegacyPassManager.cpp
+++ b/lib/IR/LegacyPassManager.cpp
@@ -88,8 +88,7 @@ PrintAfterAll("print-after-all",
 
 static bool ShouldPrintBeforeOrAfterPass(const PassInfo *PI,
                                          PassOptionList &PassesToPrint) {
-  for (unsigned i = 0, ie = PassesToPrint.size(); i < ie; ++i) {
-    const llvm::PassInfo *PassInf = PassesToPrint[i];
+  for (auto *PassInf : PassesToPrint) {
     if (PassInf)
       if (PassInf->getPassArgument() == PI->getPassArgument()) {
         return true;
@@ -294,10 +293,8 @@ class MPPassManager : public Pass, public PMDataManager {
 
   // Delete on the fly managers.
   ~MPPassManager() override {
-    for (std::map<Pass *, FunctionPassManagerImpl *>::iterator
-           I = OnTheFlyManagers.begin(), E = OnTheFlyManagers.end();
-         I != E; ++I) {
-      FunctionPassManagerImpl *FPP = I->second;
+    for (auto &OnTheFlyManager : OnTheFlyManagers) {
+      FunctionPassManagerImpl *FPP = OnTheFlyManager.second;
       delete FPP;
     }
   }
@@ -466,9 +463,8 @@ class TimingInfo {
   ~TimingInfo() {
     // Delete all of the timers, which accumulate their info into the
     // TimerGroup.
-    for (DenseMap<Pass*, Timer*>::iterator I = TimingData.begin(),
-         E = TimingData.end(); I != E; ++I)
-      delete I->second;
+    for (auto &I : TimingData)
+      delete I.second;
     // TimerGroup is deleted next, printing the report.
   }
 
@@ -511,9 +507,7 @@ PMTopLevelManager::setLastUser(ArrayRef<Pass*> AnalysisPasses, Pass *P) {
   if (P->getResolver())
     PDepth = P->getResolver()->getPMDataManager().getDepth();
 
-  for (SmallVectorImpl<Pass *>::const_iterator I = AnalysisPasses.begin(),
-         E = AnalysisPasses.end(); I != E; ++I) {
-    Pass *AP = *I;
+  for (Pass *AP : AnalysisPasses) {
     LastUser[AP] = P;
 
     if (P == AP)
@@ -694,22 +688,19 @@ void PMTopLevelManager::schedulePass(Pass *P) {
 Pass *PMTopLevelManager::findAnalysisPass(AnalysisID AID) {
 
   // Check pass managers
-  for (SmallVectorImpl<PMDataManager *>::iterator I = PassManagers.begin(),
-         E = PassManagers.end(); I != E; ++I)
-    if (Pass *P = (*I)->findAnalysisPass(AID, false))
+  for (PMDataManager *PassManager : PassManagers)
+    if (Pass *P = PassManager->findAnalysisPass(AID, false))
       return P;
 
   // Check other pass managers
-  for (SmallVectorImpl<PMDataManager *>::iterator
-         I = IndirectPassManagers.begin(),
-         E = IndirectPassManagers.end(); I != E; ++I)
-    if (Pass *P = (*I)->findAnalysisPass(AID, false))
+  for (PMDataManager *IndirectPassManager : IndirectPassManagers)
+    if (Pass *P = IndirectPassManager->findAnalysisPass(AID, false))
       return P;
 
   // Check the immutable passes. Iterate in reverse order so that we find
   // the most recently registered passes first.
-  for (SmallVectorImpl<ImmutablePass *>::reverse_iterator I =
-       ImmutablePasses.rbegin(), E = ImmutablePasses.rend(); I != E; ++I) {
+  for (auto I = ImmutablePasses.rbegin(), E = ImmutablePasses.rend(); I != E;
+       ++I) {
     AnalysisID PI = (*I)->getPassID();
     if (PI == AID)
       return *I;
@@ -719,11 +710,9 @@ Pass *PMTopLevelManager::findAnalysisPass(AnalysisID AID) {
     assert(PassInf && "Expected all immutable passes to be initialized");
     const std::vector<const PassInfo*> &ImmPI =
       PassInf->getInterfacesImplemented();
-    for (std::vector<const PassInfo*>::const_iterator II = ImmPI.begin(),
-         EE = ImmPI.end(); II != EE; ++II) {
-      if ((*II)->getTypeInfo() == AID)
+    for (const PassInfo *PI : ImmPI)
+      if (PI->getTypeInfo() == AID)
         return *I;
-    }
   }
 
   return nullptr;
@@ -755,9 +744,8 @@ void PMTopLevelManager::dumpPasses() const {
   // (sometimes indirectly), but there's no inheritance relationship
   // between PMDataManager and Pass, so we have to getAsPass to get
   // from a PMDataManager* to a Pass*.
-  for (SmallVectorImpl<PMDataManager *>::const_iterator I =
-       PassManagers.begin(), E = PassManagers.end(); I != E; ++I)
-    (*I)->getAsPass()->dumpPassStructure(1);
+  for (PMDataManager *Manager : PassManagers)
+    Manager->getAsPass()->dumpPassStructure(1);
 }
 
 void PMTopLevelManager::dumpArguments() const {
@@ -1427,11 +1415,8 @@ bool FunctionPassManagerImpl::doInitialization(Module &M) {
   dumpArguments();
   dumpPasses();
 
-  SmallVectorImpl<ImmutablePass *>& IPV = getImmutablePasses();
-  for (SmallVectorImpl<ImmutablePass *>::const_iterator I = IPV.begin(),
-       E = IPV.end(); I != E; ++I) {
-    Changed |= (*I)->doInitialization(M);
-  }
+  for (ImmutablePass *ImPass : getImmutablePasses())
+    Changed |= ImPass->doInitialization(M);
 
   for (unsigned Index = 0; Index < getNumContainedManagers(); ++Index)
     Changed |= getContainedManager(Index)->doInitialization(M);
@@ -1445,11 +1430,8 @@ bool FunctionPassManagerImpl::doFinalization(Module &M) {
   for (int Index = getNumContainedManagers() - 1; Index >= 0; --Index)
     Changed |= getContainedManager(Index)->doFinalization(M);
 
-  SmallVectorImpl<ImmutablePass *>& IPV = getImmutablePasses();
-  for (SmallVectorImpl<ImmutablePass *>::const_iterator I = IPV.begin(),
-       E = IPV.end(); I != E; ++I) {
-    Changed |= (*I)->doFinalization(M);
-  }
+  for (ImmutablePass *ImPass : getImmutablePasses())
+    Changed |= ImPass->doFinalization(M);
 
   return Changed;
 }
@@ -1554,8 +1536,8 @@ bool FPPassManager::runOnFunction(Function &F) {
 bool FPPassManager::runOnModule(Module &M) {
   bool Changed = false;
 
-  for (Module::iterator I = M.begin(), E = M.end(); I != E; ++I)
-    Changed |= runOnFunction(*I);
+  for (Function &F : M)
+    Changed |= runOnFunction(F);
 
   return Changed;
 }
@@ -1589,10 +1571,8 @@ MPPassManager::runOnModule(Module &M) {
   bool Changed = false;
 
   // Initialize on-the-fly passes
-  for (std::map<Pass *, FunctionPassManagerImpl *>::iterator
-       I = OnTheFlyManagers.begin(), E = OnTheFlyManagers.end();
-       I != E; ++I) {
-    FunctionPassManagerImpl *FPP = I->second;
+  for (auto &OnTheFlyManager : OnTheFlyManagers) {
+    FunctionPassManagerImpl *FPP = OnTheFlyManager.second;
     Changed |= FPP->doInitialization(M);
   }
 
@@ -1633,10 +1613,8 @@ MPPassManager::runOnModule(Module &M) {
     Changed |= getContainedPass(Index)->doFinalization(M);
 
   // Finalize on-the-fly passes
-  for (std::map<Pass *, FunctionPassManagerImpl *>::iterator
-       I = OnTheFlyManagers.begin(), E = OnTheFlyManagers.end();
-       I != E; ++I) {
-    FunctionPassManagerImpl *FPP = I->second;
+  for (auto &OnTheFlyManager : OnTheFlyManagers) {
+    FunctionPassManagerImpl *FPP = OnTheFlyManager.second;
     // We don't know when is the last time an on-the-fly pass is run,
     // so we need to releaseMemory / finalize here
     FPP->releaseMemoryOnTheFly();
@@ -1712,11 +1690,8 @@ bool PassManagerImpl::run(Module &M) {
   dumpArguments();
   dumpPasses();
 
-  SmallVectorImpl<ImmutablePass *>& IPV = getImmutablePasses();
-  for (SmallVectorImpl<ImmutablePass *>::const_iterator I = IPV.begin(),
-       E = IPV.end(); I != E; ++I) {
-    Changed |= (*I)->doInitialization(M);
-  }
+  for (ImmutablePass *ImPass : getImmutablePasses())
+    Changed |= ImPass->doInitialization(M);
 
   initializeAllAnalysisInfo();
   for (unsigned Index = 0; Index < getNumContainedManagers(); ++Index) {
@@ -1724,10 +1699,8 @@ bool PassManagerImpl::run(Module &M) {
     M.getContext().yield();
   }
 
-  for (SmallVectorImpl<ImmutablePass *>::const_iterator I = IPV.begin(),
-       E = IPV.end(); I != E; ++I) {
-    Changed |= (*I)->doFinalization(M);
-  }
+  for (ImmutablePass *ImPass : getImmutablePasses())
+    Changed |= ImPass->doFinalization(M);
 
   return Changed;
 }
@@ -1823,9 +1796,8 @@ void PMStack::push(PMDataManager *PM) {
 
 // Dump content of the pass manager stack.
 void PMStack::dump() const {
-  for (std::vector<PMDataManager *>::const_iterator I = S.begin(),
-         E = S.end(); I != E; ++I)
-    dbgs() << (*I)->getAsPass()->getPassName() << ' ';
+  for (PMDataManager *Manager : S)
+    dbgs() << Manager->getAsPass()->getPassName() << ' ';
 
   if (!S.empty())
     dbgs() << '\n';
diff --git a/lib/IR/MDBuilder.cpp b/lib/IR/MDBuilder.cpp
index a9010114044..b4c5ca7c6a1 100644
--- a/lib/IR/MDBuilder.cpp
+++ b/lib/IR/MDBuilder.cpp
@@ -53,6 +53,16 @@ MDNode *MDBuilder::createBranchWeights(ArrayRef<uint32_t> Weights) {
   return MDNode::get(Context, Vals);
 }
 
+MDNode *MDBuilder::createFunctionEntryCount(uint64_t Count) {
+  SmallVector<Metadata *, 2> Vals(2);
+  Vals[0] = createString("function_entry_count");
+
+  Type *Int64Ty = Type::getInt64Ty(Context);
+  Vals[1] = createConstant(ConstantInt::get(Int64Ty, Count));
+
+  return MDNode::get(Context, Vals);
+}
+
 MDNode *MDBuilder::createRange(const APInt &Lo, const APInt &Hi) {
   assert(Lo.getBitWidth() == Hi.getBitWidth() && "Mismatched bitwidths!");
 
@@ -158,9 +168,16 @@ MDNode *MDBuilder::createTBAAScalarTypeNode(StringRef Name, MDNode *Parent,
 /// \brief Return metadata for a TBAA tag node with the given
 /// base type, access type and offset relative to the base type.
 MDNode *MDBuilder::createTBAAStructTagNode(MDNode *BaseType, MDNode *AccessType,
-                                           uint64_t Offset) {
+                                           uint64_t Offset, bool IsConstant) {
   Type *Int64 = Type::getInt64Ty(Context);
-  Metadata *Ops[3] = {BaseType, AccessType,
-                      createConstant(ConstantInt::get(Int64, Offset))};
-  return MDNode::get(Context, Ops);
+  if (IsConstant) {
+    Metadata *Ops[4] = {BaseType, AccessType,
+                        createConstant(ConstantInt::get(Int64, Offset)),
+                        createConstant(ConstantInt::get(Int64, 1))};
+    return MDNode::get(Context, Ops);
+  } else {
+    Metadata *Ops[3] = {BaseType, AccessType,
+                        createConstant(ConstantInt::get(Int64, Offset))};
+    return MDNode::get(Context, Ops);
+  }
 }
diff --git a/lib/IR/Mangler.cpp b/lib/IR/Mangler.cpp
index a0e1b25056c..016cb9eb689 100644
--- a/lib/IR/Mangler.cpp
+++ b/lib/IR/Mangler.cpp
@@ -17,12 +17,21 @@
 #include "llvm/IR/DataLayout.h"
 #include "llvm/IR/DerivedTypes.h"
 #include "llvm/IR/Function.h"
+#include "llvm/IR/Module.h"
 #include "llvm/Support/raw_ostream.h"
 using namespace llvm;
 
-static void getNameWithPrefixx(raw_ostream &OS, const Twine &GVName,
-                              Mangler::ManglerPrefixTy PrefixTy,
-                              const DataLayout &DL, char Prefix) {
+namespace {
+enum ManglerPrefixTy {
+  Default,      ///< Emit default string before each symbol.
+  Private,      ///< Emit "private" prefix before each symbol.
+  LinkerPrivate ///< Emit "linker private" prefix before each symbol.
+};
+}
+
+static void getNameWithPrefixImpl(raw_ostream &OS, const Twine &GVName,
+                                  ManglerPrefixTy PrefixTy,
+                                  const DataLayout &DL, char Prefix) {
   SmallString<256> TmpData;
   StringRef Name = GVName.toStringRef(TmpData);
   assert(!Name.empty() && "getNameWithPrefix requires non-empty name");
@@ -34,9 +43,9 @@ static void getNameWithPrefixx(raw_ostream &OS, const Twine &GVName,
     return;
   }
 
-  if (PrefixTy == Mangler::Private)
+  if (PrefixTy == Private)
     OS << DL.getPrivateGlobalPrefix();
-  else if (PrefixTy == Mangler::LinkerPrivate)
+  else if (PrefixTy == LinkerPrivate)
     OS << DL.getLinkerPrivateGlobalPrefix();
 
   if (Prefix != '\0')
@@ -46,17 +55,23 @@ static void getNameWithPrefixx(raw_ostream &OS, const Twine &GVName,
   OS << Name;
 }
 
+static void getNameWithPrefixImpl(raw_ostream &OS, const Twine &GVName,
+                                  const DataLayout &DL,
+                                  ManglerPrefixTy PrefixTy) {
+  char Prefix = DL.getGlobalPrefix();
+  return getNameWithPrefixImpl(OS, GVName, PrefixTy, DL, Prefix);
+}
+
 void Mangler::getNameWithPrefix(raw_ostream &OS, const Twine &GVName,
-                                ManglerPrefixTy PrefixTy) const {
-  char Prefix = DL->getGlobalPrefix();
-  return getNameWithPrefixx(OS, GVName, PrefixTy, *DL, Prefix);
+                                const DataLayout &DL) {
+  return getNameWithPrefixImpl(OS, GVName, DL, Default);
 }
 
 void Mangler::getNameWithPrefix(SmallVectorImpl<char> &OutName,
-                                const Twine &GVName,
-                                ManglerPrefixTy PrefixTy) const {
+                                const Twine &GVName, const DataLayout &DL) {
   raw_svector_ostream OS(OutName);
-  return getNameWithPrefix(OS, GVName, PrefixTy);
+  char Prefix = DL.getGlobalPrefix();
+  return getNameWithPrefixImpl(OS, GVName, Default, DL, Prefix);
 }
 
 static bool hasByteCountSuffix(CallingConv::ID CC) {
@@ -92,14 +107,15 @@ static void addByteCountSuffix(raw_ostream &OS, const Function *F,
 
 void Mangler::getNameWithPrefix(raw_ostream &OS, const GlobalValue *GV,
                                 bool CannotUsePrivateLabel) const {
-  ManglerPrefixTy PrefixTy = Mangler::Default;
+  ManglerPrefixTy PrefixTy = Default;
   if (GV->hasPrivateLinkage()) {
     if (CannotUsePrivateLabel)
-      PrefixTy = Mangler::LinkerPrivate;
+      PrefixTy = LinkerPrivate;
     else
-      PrefixTy = Mangler::Private;
+      PrefixTy = Private;
   }
 
+  const DataLayout &DL = GV->getParent()->getDataLayout();
   if (!GV->hasName()) {
     // Get the ID for the global, assigning a new one if we haven't got one
     // already.
@@ -108,12 +124,12 @@ void Mangler::getNameWithPrefix(raw_ostream &OS, const GlobalValue *GV,
       ID = NextAnonGlobalID++;
 
     // Must mangle the global into a unique ID.
-    getNameWithPrefix(OS, "__unnamed_" + Twine(ID), PrefixTy);
+    getNameWithPrefixImpl(OS, "__unnamed_" + Twine(ID), DL, PrefixTy);
     return;
   }
 
   StringRef Name = GV->getName();
-  char Prefix = DL->getGlobalPrefix();
+  char Prefix = DL.getGlobalPrefix();
 
   // Mangle functions with Microsoft calling conventions specially.  Only do
   // this mangling for x86_64 vectorcall and 32-bit x86.
@@ -122,7 +138,7 @@ void Mangler::getNameWithPrefix(raw_ostream &OS, const GlobalValue *GV,
     MSFunc = nullptr; // Don't mangle when \01 is present.
   CallingConv::ID CC =
       MSFunc ? MSFunc->getCallingConv() : (unsigned)CallingConv::C;
-  if (!DL->hasMicrosoftFastStdCallMangling() &&
+  if (!DL.hasMicrosoftFastStdCallMangling() &&
       CC != CallingConv::X86_VectorCall)
     MSFunc = nullptr;
   if (MSFunc) {
@@ -132,7 +148,7 @@ void Mangler::getNameWithPrefix(raw_ostream &OS, const GlobalValue *GV,
       Prefix = '\0'; // vectorcall functions have no prefix.
   }
 
-  getNameWithPrefixx(OS, Name, PrefixTy, *DL, Prefix);
+  getNameWithPrefixImpl(OS, Name, PrefixTy, DL, Prefix);
 
   if (!MSFunc)
     return;
@@ -147,7 +163,7 @@ void Mangler::getNameWithPrefix(raw_ostream &OS, const GlobalValue *GV,
       // "Pure" variadic functions do not receive @0 suffix.
       (!FT->isVarArg() || FT->getNumParams() == 0 ||
        (FT->getNumParams() == 1 && MSFunc->hasStructRetAttr())))
-    addByteCountSuffix(OS, MSFunc, *DL);
+    addByteCountSuffix(OS, MSFunc, DL);
 }
 
 void Mangler::getNameWithPrefix(SmallVectorImpl<char> &OutName,
diff --git a/lib/IR/Metadata.cpp b/lib/IR/Metadata.cpp
index 93098b9da7e..1abcf0d18c9 100644
--- a/lib/IR/Metadata.cpp
+++ b/lib/IR/Metadata.cpp
@@ -256,9 +256,9 @@ ValueAsMetadata *ValueAsMetadata::get(Value *V) {
   if (!Entry) {
     assert((isa<Constant>(V) || isa<Argument>(V) || isa<Instruction>(V)) &&
            "Expected constant or function-local value");
-    assert(!V->NameAndIsUsedByMD.getInt() &&
+    assert(!V->IsUsedByMD &&
            "Expected this to be the only metadata use");
-    V->NameAndIsUsedByMD.setInt(true);
+    V->IsUsedByMD = true;
     if (auto *C = dyn_cast<Constant>(V))
       Entry = new ConstantAsMetadata(C);
     else
@@ -302,15 +302,15 @@ void ValueAsMetadata::handleRAUW(Value *From, Value *To) {
   auto &Store = Context.pImpl->ValuesAsMetadata;
   auto I = Store.find(From);
   if (I == Store.end()) {
-    assert(!From->NameAndIsUsedByMD.getInt() &&
+    assert(!From->IsUsedByMD &&
            "Expected From not to be used by metadata");
     return;
   }
 
   // Remove old entry from the map.
-  assert(From->NameAndIsUsedByMD.getInt() &&
+  assert(From->IsUsedByMD &&
          "Expected From to be used by metadata");
-  From->NameAndIsUsedByMD.setInt(false);
+  From->IsUsedByMD = false;
   ValueAsMetadata *MD = I->second;
   assert(MD && "Expected valid metadata");
   assert(MD->getValue() == From && "Expected valid mapping");
@@ -346,9 +346,9 @@ void ValueAsMetadata::handleRAUW(Value *From, Value *To) {
   }
 
   // Update MD in place (and update the map entry).
-  assert(!To->NameAndIsUsedByMD.getInt() &&
+  assert(!To->IsUsedByMD &&
          "Expected this to be the only metadata use");
-  To->NameAndIsUsedByMD.setInt(true);
+  To->IsUsedByMD = true;
   MD->V = To;
   Entry = MD;
 }
@@ -381,20 +381,35 @@ StringRef MDString::getString() const {
 // MDNode implementation.
 //
 
+// Assert that the MDNode types will not be unaligned by the objects
+// prepended to them.
+#define HANDLE_MDNODE_LEAF(CLASS)                                              \
+  static_assert(                                                               \
+      llvm::AlignOf<uint64_t>::Alignment >= llvm::AlignOf<CLASS>::Alignment,   \
+      "Alignment is insufficient after objects prepended to " #CLASS);
+#include "llvm/IR/Metadata.def"
+
 void *MDNode::operator new(size_t Size, unsigned NumOps) {
-  void *Ptr = ::operator new(Size + NumOps * sizeof(MDOperand));
+  size_t OpSize = NumOps * sizeof(MDOperand);
+  // uint64_t is the most aligned type we need support (ensured by static_assert
+  // above)
+  OpSize = RoundUpToAlignment(OpSize, llvm::alignOf<uint64_t>());
+  void *Ptr = reinterpret_cast<char *>(::operator new(OpSize + Size)) + OpSize;
   MDOperand *O = static_cast<MDOperand *>(Ptr);
-  for (MDOperand *E = O + NumOps; O != E; ++O)
-    (void)new (O) MDOperand;
-  return O;
+  for (MDOperand *E = O - NumOps; O != E; --O)
+    (void)new (O - 1) MDOperand;
+  return Ptr;
 }
 
 void MDNode::operator delete(void *Mem) {
   MDNode *N = static_cast<MDNode *>(Mem);
+  size_t OpSize = N->NumOperands * sizeof(MDOperand);
+  OpSize = RoundUpToAlignment(OpSize, llvm::alignOf<uint64_t>());
+
   MDOperand *O = static_cast<MDOperand *>(Mem);
   for (MDOperand *E = O - N->NumOperands; O != E; --O)
     (O - 1)->~MDOperand();
-  ::operator delete(O);
+  ::operator delete(reinterpret_cast<char *>(Mem) - OpSize);
 }
 
 MDNode::MDNode(LLVMContext &Context, unsigned ID, StorageType Storage,
@@ -975,6 +990,50 @@ StringRef NamedMDNode::getName() const {
 //===----------------------------------------------------------------------===//
 // Instruction Metadata method implementations.
 //
+void MDAttachmentMap::set(unsigned ID, MDNode &MD) {
+  for (auto &I : Attachments)
+    if (I.first == ID) {
+      I.second.reset(&MD);
+      return;
+    }
+  Attachments.emplace_back(std::piecewise_construct, std::make_tuple(ID),
+                           std::make_tuple(&MD));
+}
+
+void MDAttachmentMap::erase(unsigned ID) {
+  if (empty())
+    return;
+
+  // Common case is one/last value.
+  if (Attachments.back().first == ID) {
+    Attachments.pop_back();
+    return;
+  }
+
+  for (auto I = Attachments.begin(), E = std::prev(Attachments.end()); I != E;
+       ++I)
+    if (I->first == ID) {
+      *I = std::move(Attachments.back());
+      Attachments.pop_back();
+      return;
+    }
+}
+
+MDNode *MDAttachmentMap::lookup(unsigned ID) const {
+  for (const auto &I : Attachments)
+    if (I.first == ID)
+      return I.second;
+  return nullptr;
+}
+
+void MDAttachmentMap::getAll(
+    SmallVectorImpl<std::pair<unsigned, MDNode *>> &Result) const {
+  Result.append(Attachments.begin(), Attachments.end());
+
+  // Sort the resulting array so it is stable.
+  if (Result.size() > 1)
+    array_pod_sort(Result.begin(), Result.end());
+}
 
 void Instruction::setMetadata(StringRef Kind, MDNode *Node) {
   if (!Node && !hasMetadata())
@@ -997,35 +1056,23 @@ void Instruction::dropUnknownMetadata(ArrayRef<unsigned> KnownIDs) {
   if (!hasMetadataHashEntry())
     return; // Nothing to remove!
 
-  DenseMap<const Instruction *, LLVMContextImpl::MDMapTy> &MetadataStore =
-      getContext().pImpl->MetadataStore;
+  auto &InstructionMetadata = getContext().pImpl->InstructionMetadata;
 
   if (KnownSet.empty()) {
     // Just drop our entry at the store.
-    MetadataStore.erase(this);
+    InstructionMetadata.erase(this);
     setHasMetadataHashEntry(false);
     return;
   }
 
-  LLVMContextImpl::MDMapTy &Info = MetadataStore[this];
-  unsigned I;
-  unsigned E;
-  // Walk the array and drop any metadata we don't know.
-  for (I = 0, E = Info.size(); I != E;) {
-    if (KnownSet.count(Info[I].first)) {
-      ++I;
-      continue;
-    }
-
-    Info[I] = std::move(Info.back());
-    Info.pop_back();
-    --E;
-  }
-  assert(E == Info.size());
+  auto &Info = InstructionMetadata[this];
+  Info.remove_if([&KnownSet](const std::pair<unsigned, TrackingMDNodeRef> &I) {
+    return !KnownSet.count(I.first);
+  });
 
-  if (E == 0) {
+  if (Info.empty()) {
     // Drop our entry at the store.
-    MetadataStore.erase(this);
+    InstructionMetadata.erase(this);
     setHasMetadataHashEntry(false);
   }
 }
@@ -1045,50 +1092,31 @@ void Instruction::setMetadata(unsigned KindID, MDNode *Node) {
   
   // Handle the case when we're adding/updating metadata on an instruction.
   if (Node) {
-    LLVMContextImpl::MDMapTy &Info = getContext().pImpl->MetadataStore[this];
+    auto &Info = getContext().pImpl->InstructionMetadata[this];
     assert(!Info.empty() == hasMetadataHashEntry() &&
            "HasMetadata bit is wonked");
-    if (Info.empty()) {
+    if (Info.empty())
       setHasMetadataHashEntry(true);
-    } else {
-      // Handle replacement of an existing value.
-      for (auto &P : Info)
-        if (P.first == KindID) {
-          P.second.reset(Node);
-          return;
-        }
-    }
-
-    // No replacement, just add it to the list.
-    Info.emplace_back(std::piecewise_construct, std::make_tuple(KindID),
-                      std::make_tuple(Node));
+    Info.set(KindID, *Node);
     return;
   }
 
   // Otherwise, we're removing metadata from an instruction.
   assert((hasMetadataHashEntry() ==
-          (getContext().pImpl->MetadataStore.count(this) > 0)) &&
+          (getContext().pImpl->InstructionMetadata.count(this) > 0)) &&
          "HasMetadata bit out of date!");
   if (!hasMetadataHashEntry())
     return;  // Nothing to remove!
-  LLVMContextImpl::MDMapTy &Info = getContext().pImpl->MetadataStore[this];
+  auto &Info = getContext().pImpl->InstructionMetadata[this];
 
-  // Common case is removing the only entry.
-  if (Info.size() == 1 && Info[0].first == KindID) {
-    getContext().pImpl->MetadataStore.erase(this);
-    setHasMetadataHashEntry(false);
+  // Handle removal of an existing value.
+  Info.erase(KindID);
+
+  if (!Info.empty())
     return;
-  }
 
-  // Handle removal of an existing value.
-  for (unsigned i = 0, e = Info.size(); i != e; ++i)
-    if (Info[i].first == KindID) {
-      Info[i] = std::move(Info.back());
-      Info.pop_back();
-      assert(!Info.empty() && "Removing last entry should be handled above");
-      return;
-    }
-  // Otherwise, removing an entry that doesn't exist on the instruction.
+  getContext().pImpl->InstructionMetadata.erase(this);
+  setHasMetadataHashEntry(false);
 }
 
 void Instruction::setAAMetadata(const AAMDNodes &N) {
@@ -1102,15 +1130,12 @@ MDNode *Instruction::getMetadataImpl(unsigned KindID) const {
   if (KindID == LLVMContext::MD_dbg)
     return DbgLoc.getAsMDNode();
 
-  if (!hasMetadataHashEntry()) return nullptr;
-  
-  LLVMContextImpl::MDMapTy &Info = getContext().pImpl->MetadataStore[this];
+  if (!hasMetadataHashEntry())
+    return nullptr;
+  auto &Info = getContext().pImpl->InstructionMetadata[this];
   assert(!Info.empty() && "bit out of sync with hash table");
 
-  for (const auto &I : Info)
-    if (I.first == KindID)
-      return I.second;
-  return nullptr;
+  return Info.lookup(KindID);
 }
 
 void Instruction::getAllMetadataImpl(
@@ -1123,45 +1148,106 @@ void Instruction::getAllMetadataImpl(
         std::make_pair((unsigned)LLVMContext::MD_dbg, DbgLoc.getAsMDNode()));
     if (!hasMetadataHashEntry()) return;
   }
-  
+
   assert(hasMetadataHashEntry() &&
-         getContext().pImpl->MetadataStore.count(this) &&
+         getContext().pImpl->InstructionMetadata.count(this) &&
          "Shouldn't have called this");
-  const LLVMContextImpl::MDMapTy &Info =
-    getContext().pImpl->MetadataStore.find(this)->second;
+  const auto &Info = getContext().pImpl->InstructionMetadata.find(this)->second;
   assert(!Info.empty() && "Shouldn't have called this");
-
-  Result.reserve(Result.size() + Info.size());
-  for (auto &I : Info)
-    Result.push_back(std::make_pair(I.first, cast<MDNode>(I.second.get())));
-
-  // Sort the resulting array so it is stable.
-  if (Result.size() > 1)
-    array_pod_sort(Result.begin(), Result.end());
+  Info.getAll(Result);
 }
 
 void Instruction::getAllMetadataOtherThanDebugLocImpl(
     SmallVectorImpl<std::pair<unsigned, MDNode *>> &Result) const {
   Result.clear();
   assert(hasMetadataHashEntry() &&
-         getContext().pImpl->MetadataStore.count(this) &&
+         getContext().pImpl->InstructionMetadata.count(this) &&
          "Shouldn't have called this");
-  const LLVMContextImpl::MDMapTy &Info =
-    getContext().pImpl->MetadataStore.find(this)->second;
+  const auto &Info = getContext().pImpl->InstructionMetadata.find(this)->second;
   assert(!Info.empty() && "Shouldn't have called this");
-  Result.reserve(Result.size() + Info.size());
-  for (auto &I : Info)
-    Result.push_back(std::make_pair(I.first, cast<MDNode>(I.second.get())));
-
-  // Sort the resulting array so it is stable.
-  if (Result.size() > 1)
-    array_pod_sort(Result.begin(), Result.end());
+  Info.getAll(Result);
 }
 
 /// clearMetadataHashEntries - Clear all hashtable-based metadata from
 /// this instruction.
 void Instruction::clearMetadataHashEntries() {
   assert(hasMetadataHashEntry() && "Caller should check");
-  getContext().pImpl->MetadataStore.erase(this);
+  getContext().pImpl->InstructionMetadata.erase(this);
+  setHasMetadataHashEntry(false);
+}
+
+MDNode *Function::getMetadata(unsigned KindID) const {
+  if (!hasMetadata())
+    return nullptr;
+  return getContext().pImpl->FunctionMetadata[this].lookup(KindID);
+}
+
+MDNode *Function::getMetadata(StringRef Kind) const {
+  if (!hasMetadata())
+    return nullptr;
+  return getMetadata(getContext().getMDKindID(Kind));
+}
+
+void Function::setMetadata(unsigned KindID, MDNode *MD) {
+  if (MD) {
+    if (!hasMetadata())
+      setHasMetadataHashEntry(true);
+
+    getContext().pImpl->FunctionMetadata[this].set(KindID, *MD);
+    return;
+  }
+
+  // Nothing to unset.
+  if (!hasMetadata())
+    return;
+
+  auto &Store = getContext().pImpl->FunctionMetadata[this];
+  Store.erase(KindID);
+  if (Store.empty())
+    clearMetadata();
+}
+
+void Function::setMetadata(StringRef Kind, MDNode *MD) {
+  if (!MD && !hasMetadata())
+    return;
+  setMetadata(getContext().getMDKindID(Kind), MD);
+}
+
+void Function::getAllMetadata(
+    SmallVectorImpl<std::pair<unsigned, MDNode *>> &MDs) const {
+  MDs.clear();
+
+  if (!hasMetadata())
+    return;
+
+  getContext().pImpl->FunctionMetadata[this].getAll(MDs);
+}
+
+void Function::dropUnknownMetadata(ArrayRef<unsigned> KnownIDs) {
+  if (!hasMetadata())
+    return;
+  if (KnownIDs.empty()) {
+    clearMetadata();
+    return;
+  }
+
+  SmallSet<unsigned, 5> KnownSet;
+  KnownSet.insert(KnownIDs.begin(), KnownIDs.end());
+
+  auto &Store = getContext().pImpl->FunctionMetadata[this];
+  assert(!Store.empty());
+
+  Store.remove_if([&KnownSet](const std::pair<unsigned, TrackingMDNodeRef> &I) {
+    return !KnownSet.count(I.first);
+  });
+
+  if (Store.empty())
+    clearMetadata();
+}
+
+void Function::clearMetadata() {
+  if (!hasMetadata())
+    return;
+  getContext().pImpl->FunctionMetadata.erase(this);
   setHasMetadataHashEntry(false);
 }
diff --git a/lib/IR/Module.cpp b/lib/IR/Module.cpp
index 3e8f91fee74..043f74e12da 100644
--- a/lib/IR/Module.cpp
+++ b/lib/IR/Module.cpp
@@ -394,15 +394,15 @@ std::error_code Module::materialize(GlobalValue *GV) {
   return Materializer->materialize(GV);
 }
 
-void Module::Dematerialize(GlobalValue *GV) {
+void Module::dematerialize(GlobalValue *GV) {
   if (Materializer)
-    return Materializer->Dematerialize(GV);
+    return Materializer->dematerialize(GV);
 }
 
 std::error_code Module::materializeAll() {
   if (!Materializer)
     return std::error_code();
-  return Materializer->MaterializeModule(this);
+  return Materializer->materializeModule(this);
 }
 
 std::error_code Module::materializeAllPermanently() {
diff --git a/lib/IR/Operator.cpp b/lib/IR/Operator.cpp
new file mode 100644
index 00000000000..77dc680af11
--- /dev/null
+++ b/lib/IR/Operator.cpp
@@ -0,0 +1,44 @@
+#include "llvm/IR/Operator.h"
+#include "llvm/IR/GetElementPtrTypeIterator.h"
+#include "llvm/IR/Instructions.h"
+#include "llvm/IR/Type.h"
+
+#include "ConstantsContext.h"
+
+namespace llvm {
+Type *GEPOperator::getSourceElementType() const {
+  if (auto *I = dyn_cast<GetElementPtrInst>(this))
+    return I->getSourceElementType();
+  return cast<GetElementPtrConstantExpr>(this)->getSourceElementType();
+}
+
+bool GEPOperator::accumulateConstantOffset(const DataLayout &DL,
+                                           APInt &Offset) const {
+  assert(Offset.getBitWidth() ==
+             DL.getPointerSizeInBits(getPointerAddressSpace()) &&
+         "The offset must have exactly as many bits as our pointer.");
+
+  for (gep_type_iterator GTI = gep_type_begin(this), GTE = gep_type_end(this);
+       GTI != GTE; ++GTI) {
+    ConstantInt *OpC = dyn_cast<ConstantInt>(GTI.getOperand());
+    if (!OpC)
+      return false;
+    if (OpC->isZero())
+      continue;
+
+    // Handle a struct index, which adds its field offset to the pointer.
+    if (StructType *STy = dyn_cast<StructType>(*GTI)) {
+      unsigned ElementIdx = OpC->getZExtValue();
+      const StructLayout *SL = DL.getStructLayout(STy);
+      Offset += APInt(Offset.getBitWidth(), SL->getElementOffset(ElementIdx));
+      continue;
+    }
+
+    // For array or vector indices, scale the index by the size of the type.
+    APInt Index = OpC->getValue().sextOrTrunc(Offset.getBitWidth());
+    Offset += Index * APInt(Offset.getBitWidth(),
+                            DL.getTypeAllocSize(GTI.getIndexedType()));
+  }
+  return true;
+}
+}
diff --git a/lib/IR/Type.cpp b/lib/IR/Type.cpp
index 65060dc39d2..0891508d7b1 100644
--- a/lib/IR/Type.cpp
+++ b/lib/IR/Type.cpp
@@ -238,6 +238,7 @@ IntegerType *Type::getInt8Ty(LLVMContext &C) { return &C.pImpl->Int8Ty; }
 IntegerType *Type::getInt16Ty(LLVMContext &C) { return &C.pImpl->Int16Ty; }
 IntegerType *Type::getInt32Ty(LLVMContext &C) { return &C.pImpl->Int32Ty; }
 IntegerType *Type::getInt64Ty(LLVMContext &C) { return &C.pImpl->Int64Ty; }
+IntegerType *Type::getInt128Ty(LLVMContext &C) { return &C.pImpl->Int128Ty; }
 
 IntegerType *Type::getIntNTy(LLVMContext &C, unsigned N) {
   return IntegerType::get(C, N);
@@ -306,12 +307,13 @@ IntegerType *IntegerType::get(LLVMContext &C, unsigned NumBits) {
   
   // Check for the built-in integer types
   switch (NumBits) {
-  case  1: return cast<IntegerType>(Type::getInt1Ty(C));
-  case  8: return cast<IntegerType>(Type::getInt8Ty(C));
-  case 16: return cast<IntegerType>(Type::getInt16Ty(C));
-  case 32: return cast<IntegerType>(Type::getInt32Ty(C));
-  case 64: return cast<IntegerType>(Type::getInt64Ty(C));
-  default: 
+  case   1: return cast<IntegerType>(Type::getInt1Ty(C));
+  case   8: return cast<IntegerType>(Type::getInt8Ty(C));
+  case  16: return cast<IntegerType>(Type::getInt16Ty(C));
+  case  32: return cast<IntegerType>(Type::getInt32Ty(C));
+  case  64: return cast<IntegerType>(Type::getInt64Ty(C));
+  case 128: return cast<IntegerType>(Type::getInt128Ty(C));
+  default:
     break;
   }
   
@@ -343,7 +345,7 @@ FunctionType::FunctionType(Type *Result, ArrayRef<Type*> Params,
   assert(isValidReturnType(Result) && "invalid return type for function");
   setSubclassData(IsVarArgs);
 
-  SubTys[0] = const_cast<Type*>(Result);
+  SubTys[0] = Result;
 
   for (unsigned i = 0, e = Params.size(); i != e; ++i) {
     assert(isValidArgumentType(Params[i]) &&
@@ -678,10 +680,9 @@ ArrayType::ArrayType(Type *ElType, uint64_t NumEl)
   NumElements = NumEl;
 }
 
-ArrayType *ArrayType::get(Type *elementType, uint64_t NumElements) {
-  Type *ElementType = const_cast<Type*>(elementType);
+ArrayType *ArrayType::get(Type *ElementType, uint64_t NumElements) {
   assert(isValidElementType(ElementType) && "Invalid type for array element!");
-    
+
   LLVMContextImpl *pImpl = ElementType->getContext().pImpl;
   ArrayType *&Entry = 
     pImpl->ArrayTypes[std::make_pair(ElementType, NumElements)];
@@ -705,8 +706,7 @@ VectorType::VectorType(Type *ElType, unsigned NumEl)
   NumElements = NumEl;
 }
 
-VectorType *VectorType::get(Type *elementType, unsigned NumElements) {
-  Type *ElementType = const_cast<Type*>(elementType);
+VectorType *VectorType::get(Type *ElementType, unsigned NumElements) {
   assert(NumElements > 0 && "#Elements of a VectorType must be greater than 0");
   assert(isValidElementType(ElementType) && "Element type of a VectorType must "
                                             "be an integer, floating point, or "
@@ -764,3 +764,7 @@ bool PointerType::isValidElementType(Type *ElemTy) {
   return !ElemTy->isVoidTy() && !ElemTy->isLabelTy() &&
          !ElemTy->isMetadataTy();
 }
+
+bool PointerType::isLoadableOrStorableType(Type *ElemTy) {
+  return isValidElementType(ElemTy) && !ElemTy->isFunctionTy();
+}
diff --git a/lib/IR/TypeFinder.cpp b/lib/IR/TypeFinder.cpp
index 1d2b808d650..7accc5bef53 100644
--- a/lib/IR/TypeFinder.cpp
+++ b/lib/IR/TypeFinder.cpp
@@ -50,6 +50,9 @@ void TypeFinder::run(const Module &M, bool onlyNamed) {
     if (FI->hasPrologueData())
       incorporateValue(FI->getPrologueData());
 
+    if (FI->hasPersonalityFn())
+      incorporateValue(FI->getPersonalityFn());
+
     // First incorporate the arguments.
     for (Function::const_arg_iterator AI = FI->arg_begin(),
            AE = FI->arg_end(); AI != AE; ++AI)
diff --git a/lib/IR/User.cpp b/lib/IR/User.cpp
index ee83eacf2b2..522722d701b 100644
--- a/lib/IR/User.cpp
+++ b/lib/IR/User.cpp
@@ -13,6 +13,7 @@
 #include "llvm/IR/Operator.h"
 
 namespace llvm {
+class BasicBlock;
 
 //===----------------------------------------------------------------------===//
 //                                 User Class
@@ -39,41 +40,100 @@ void User::replaceUsesOfWith(Value *From, Value *To) {
 //                         User allocHungoffUses Implementation
 //===----------------------------------------------------------------------===//
 
-Use *User::allocHungoffUses(unsigned N) const {
+void User::allocHungoffUses(unsigned N, bool IsPhi) {
+  assert(HasHungOffUses && "alloc must have hung off uses");
+
+  static_assert(AlignOf<Use>::Alignment >= AlignOf<Use::UserRef>::Alignment,
+                "Alignment is insufficient for 'hung-off-uses' pieces");
+  static_assert(AlignOf<Use::UserRef>::Alignment >=
+                    AlignOf<BasicBlock *>::Alignment,
+                "Alignment is insufficient for 'hung-off-uses' pieces");
+
   // Allocate the array of Uses, followed by a pointer (with bottom bit set) to
   // the User.
   size_t size = N * sizeof(Use) + sizeof(Use::UserRef);
+  if (IsPhi)
+    size += N * sizeof(BasicBlock *);
   Use *Begin = static_cast<Use*>(::operator new(size));
   Use *End = Begin + N;
   (void) new(End) Use::UserRef(const_cast<User*>(this), 1);
-  return Use::initTags(Begin, End);
+  setOperandList(Use::initTags(Begin, End));
+}
+
+void User::growHungoffUses(unsigned NewNumUses, bool IsPhi) {
+  assert(HasHungOffUses && "realloc must have hung off uses");
+
+  unsigned OldNumUses = getNumOperands();
+
+  // We don't support shrinking the number of uses.  We wouldn't have enough
+  // space to copy the old uses in to the new space.
+  assert(NewNumUses > OldNumUses && "realloc must grow num uses");
+
+  Use *OldOps = getOperandList();
+  allocHungoffUses(NewNumUses, IsPhi);
+  Use *NewOps = getOperandList();
+
+  // Now copy from the old operands list to the new one.
+  std::copy(OldOps, OldOps + OldNumUses, NewOps);
+
+  // If this is a Phi, then we need to copy the BB pointers too.
+  if (IsPhi) {
+    auto *OldPtr =
+        reinterpret_cast<char *>(OldOps + OldNumUses) + sizeof(Use::UserRef);
+    auto *NewPtr =
+        reinterpret_cast<char *>(NewOps + NewNumUses) + sizeof(Use::UserRef);
+    std::copy(OldPtr, OldPtr + (OldNumUses * sizeof(BasicBlock *)), NewPtr);
+  }
+  Use::zap(OldOps, OldOps + OldNumUses, true);
 }
 
 //===----------------------------------------------------------------------===//
 //                         User operator new Implementations
 //===----------------------------------------------------------------------===//
 
-void *User::operator new(size_t s, unsigned Us) {
-  void *Storage = ::operator new(s + sizeof(Use) * Us);
+void *User::operator new(size_t Size, unsigned Us) {
+  assert(Us < (1u << NumUserOperandsBits) && "Too many operands");
+  void *Storage = ::operator new(Size + sizeof(Use) * Us);
   Use *Start = static_cast<Use*>(Storage);
   Use *End = Start + Us;
   User *Obj = reinterpret_cast<User*>(End);
-  Obj->OperandList = Start;
-  Obj->NumOperands = Us;
+  Obj->NumUserOperands = Us;
+  Obj->HasHungOffUses = false;
   Use::initTags(Start, End);
   return Obj;
 }
 
+void *User::operator new(size_t Size) {
+  // Allocate space for a single Use*
+  void *Storage = ::operator new(Size + sizeof(Use *));
+  Use **HungOffOperandList = static_cast<Use **>(Storage);
+  User *Obj = reinterpret_cast<User *>(HungOffOperandList + 1);
+  Obj->NumUserOperands = 0;
+  Obj->HasHungOffUses = true;
+  *HungOffOperandList = nullptr;
+  return Obj;
+}
+
 //===----------------------------------------------------------------------===//
 //                         User operator delete Implementation
 //===----------------------------------------------------------------------===//
 
 void User::operator delete(void *Usr) {
-  User *Start = static_cast<User*>(Usr);
-  Use *Storage = static_cast<Use*>(Usr) - Start->NumOperands;
-  // If there were hung-off uses, they will have been freed already and
-  // NumOperands reset to 0, so here we just free the User itself.
-  ::operator delete(Storage);
+  // Hung off uses use a single Use* before the User, while other subclasses
+  // use a Use[] allocated prior to the user.
+  User *Obj = static_cast<User *>(Usr);
+  if (Obj->HasHungOffUses) {
+    Use **HungOffOperandList = static_cast<Use **>(Usr) - 1;
+    // drop the hung off uses.
+    Use::zap(*HungOffOperandList, *HungOffOperandList + Obj->NumUserOperands,
+             /* Delete */ true);
+    ::operator delete(HungOffOperandList);
+  } else {
+    Use *Storage = static_cast<Use *>(Usr) - Obj->NumUserOperands;
+    Use::zap(Storage, Storage + Obj->NumUserOperands,
+             /* Delete */ false);
+    ::operator delete(Storage);
+  }
 }
 
 //===----------------------------------------------------------------------===//
diff --git a/lib/IR/Value.cpp b/lib/IR/Value.cpp
index f6eb4273a53..f554d590284 100644
--- a/lib/IR/Value.cpp
+++ b/lib/IR/Value.cpp
@@ -39,15 +39,15 @@ using namespace llvm;
 //===----------------------------------------------------------------------===//
 //                                Value Class
 //===----------------------------------------------------------------------===//
-
 static inline Type *checkType(Type *Ty) {
   assert(Ty && "Value defined with a null type: Error!");
   return Ty;
 }
 
 Value::Value(Type *ty, unsigned scid)
-    : VTy(checkType(ty)), UseList(nullptr), SubclassID(scid), HasValueHandle(0),
-      SubclassOptionalData(0), SubclassData(0), NumOperands(0) {
+    : VTy(checkType(ty)), UseList(nullptr), SubclassID(scid),
+      HasValueHandle(0), SubclassOptionalData(0), SubclassData(0),
+      NumUserOperands(0), IsUsedByMD(false), HasName(false) {
   // FIXME: Why isn't this in the subclass gunk??
   // Note, we cannot call isa<CallInst> before the CallInst has been
   // constructed.
@@ -157,16 +157,44 @@ static bool getSymTab(Value *V, ValueSymbolTable *&ST) {
   return false;
 }
 
+ValueName *Value::getValueName() const {
+  if (!HasName) return nullptr;
+
+  LLVMContext &Ctx = getContext();
+  auto I = Ctx.pImpl->ValueNames.find(this);
+  assert(I != Ctx.pImpl->ValueNames.end() &&
+         "No name entry found!");
+
+  return I->second;
+}
+
+void Value::setValueName(ValueName *VN) {
+  LLVMContext &Ctx = getContext();
+
+  assert(HasName == Ctx.pImpl->ValueNames.count(this) &&
+         "HasName bit out of sync!");
+
+  if (!VN) {
+    if (HasName)
+      Ctx.pImpl->ValueNames.erase(this);
+    HasName = false;
+    return;
+  }
+
+  HasName = true;
+  Ctx.pImpl->ValueNames[this] = VN;
+}
+
 StringRef Value::getName() const {
   // Make sure the empty string is still a C string. For historical reasons,
   // some clients want to call .data() on the result and expect it to be null
   // terminated.
-  if (!getValueName())
+  if (!hasName())
     return StringRef("", 0);
   return getValueName()->getKey();
 }
 
-void Value::setName(const Twine &NewName) {
+void Value::setNameImpl(const Twine &NewName) {
   // Fast path for common IRBuilder case of setName("") when there is no name.
   if (NewName.isTriviallyEmpty() && !hasName())
     return;
@@ -187,9 +215,6 @@ void Value::setName(const Twine &NewName) {
   if (getSymTab(this, ST))
     return;  // Cannot set a name on this value (e.g. constant).
 
-  if (Function *F = dyn_cast<Function>(this))
-    getContext().pImpl->IntrinsicIDCache.erase(F);
-
   if (!ST) { // No symbol table to update?  Just do the change.
     if (NameRef.empty()) {
       // Free the name for this value.
@@ -222,6 +247,12 @@ void Value::setName(const Twine &NewName) {
   setValueName(ST->createValueName(NameRef, this));
 }
 
+void Value::setName(const Twine &NewName) {
+  setNameImpl(NewName);
+  if (Function *F = dyn_cast<Function>(this))
+    F->recalculateIntrinsicID();
+}
+
 void Value::takeName(Value *V) {
   ValueSymbolTable *ST = nullptr;
   // If this value has a name, drop it.
@@ -336,7 +367,7 @@ void Value::replaceAllUsesWith(Value *New) {
     // constant because they are uniqued.
     if (auto *C = dyn_cast<Constant>(U.getUser())) {
       if (!isa<GlobalValue>(C)) {
-        C->replaceUsesOfWithOnConstant(this, New, &U);
+        C->handleOperandChange(this, New, &U);
         continue;
       }
     }
@@ -477,137 +508,6 @@ Value *Value::stripInBoundsOffsets() {
   return stripPointerCastsAndOffsets<PSK_InBounds>(this);
 }
 
-/// \brief Check if Value is always a dereferenceable pointer.
-///
-/// Test if V is always a pointer to allocated and suitably aligned memory for
-/// a simple load or store.
-static bool isDereferenceablePointer(const Value *V, const DataLayout &DL,
-                                     SmallPtrSetImpl<const Value *> &Visited) {
-  // Note that it is not safe to speculate into a malloc'd region because
-  // malloc may return null.
-
-  // These are obviously ok.
-  if (isa<AllocaInst>(V)) return true;
-
-  // It's not always safe to follow a bitcast, for example:
-  //   bitcast i8* (alloca i8) to i32*
-  // would result in a 4-byte load from a 1-byte alloca. However,
-  // if we're casting from a pointer from a type of larger size
-  // to a type of smaller size (or the same size), and the alignment
-  // is at least as large as for the resulting pointer type, then
-  // we can look through the bitcast.
-  if (const BitCastOperator *BC = dyn_cast<BitCastOperator>(V)) {
-    Type *STy = BC->getSrcTy()->getPointerElementType(),
-         *DTy = BC->getDestTy()->getPointerElementType();
-    if (STy->isSized() && DTy->isSized() &&
-        (DL.getTypeStoreSize(STy) >= DL.getTypeStoreSize(DTy)) &&
-        (DL.getABITypeAlignment(STy) >= DL.getABITypeAlignment(DTy)))
-      return isDereferenceablePointer(BC->getOperand(0), DL, Visited);
-  }
-
-  // Global variables which can't collapse to null are ok.
-  if (const GlobalVariable *GV = dyn_cast<GlobalVariable>(V))
-    return !GV->hasExternalWeakLinkage();
-
-  // byval arguments are okay. Arguments specifically marked as
-  // dereferenceable are okay too.
-  if (const Argument *A = dyn_cast<Argument>(V)) {
-    if (A->hasByValAttr())
-      return true;
-    else if (uint64_t Bytes = A->getDereferenceableBytes()) {
-      Type *Ty = V->getType()->getPointerElementType();
-      if (Ty->isSized() && DL.getTypeStoreSize(Ty) <= Bytes)
-        return true;
-    }
-
-    return false;
-  }
-
-  // Return values from call sites specifically marked as dereferenceable are
-  // also okay.
-  if (auto CS = ImmutableCallSite(V)) {
-    if (uint64_t Bytes = CS.getDereferenceableBytes(0)) {
-      Type *Ty = V->getType()->getPointerElementType();
-      if (Ty->isSized() && DL.getTypeStoreSize(Ty) <= Bytes)
-        return true;
-    }
-  }
-
-  // For GEPs, determine if the indexing lands within the allocated object.
-  if (const GEPOperator *GEP = dyn_cast<GEPOperator>(V)) {
-    // Conservatively require that the base pointer be fully dereferenceable.
-    if (!Visited.insert(GEP->getOperand(0)).second)
-      return false;
-    if (!isDereferenceablePointer(GEP->getOperand(0), DL, Visited))
-      return false;
-    // Check the indices.
-    gep_type_iterator GTI = gep_type_begin(GEP);
-    for (User::const_op_iterator I = GEP->op_begin()+1,
-         E = GEP->op_end(); I != E; ++I) {
-      Value *Index = *I;
-      Type *Ty = *GTI++;
-      // Struct indices can't be out of bounds.
-      if (isa<StructType>(Ty))
-        continue;
-      ConstantInt *CI = dyn_cast<ConstantInt>(Index);
-      if (!CI)
-        return false;
-      // Zero is always ok.
-      if (CI->isZero())
-        continue;
-      // Check to see that it's within the bounds of an array.
-      ArrayType *ATy = dyn_cast<ArrayType>(Ty);
-      if (!ATy)
-        return false;
-      if (CI->getValue().getActiveBits() > 64)
-        return false;
-      if (CI->getZExtValue() >= ATy->getNumElements())
-        return false;
-    }
-    // Indices check out; this is dereferenceable.
-    return true;
-  }
-
-  // For gc.relocate, look through relocations
-  if (const IntrinsicInst *I = dyn_cast<IntrinsicInst>(V))
-    if (I->getIntrinsicID() == Intrinsic::experimental_gc_relocate) {
-      GCRelocateOperands RelocateInst(I);
-      return isDereferenceablePointer(RelocateInst.derivedPtr(), DL, Visited);
-    }
-
-  if (const AddrSpaceCastInst *ASC = dyn_cast<AddrSpaceCastInst>(V))
-    return isDereferenceablePointer(ASC->getOperand(0), DL, Visited);
-
-  // If we don't know, assume the worst.
-  return false;
-}
-
-bool Value::isDereferenceablePointer(const DataLayout &DL) const {
-  // When dereferenceability information is provided by a dereferenceable
-  // attribute, we know exactly how many bytes are dereferenceable. If we can
-  // determine the exact offset to the attributed variable, we can use that
-  // information here.
-  Type *Ty = getType()->getPointerElementType();
-  if (Ty->isSized()) {
-    APInt Offset(DL.getTypeStoreSizeInBits(getType()), 0);
-    const Value *BV = stripAndAccumulateInBoundsConstantOffsets(DL, Offset);
-
-    APInt DerefBytes(Offset.getBitWidth(), 0);
-    if (const Argument *A = dyn_cast<Argument>(BV))
-      DerefBytes = A->getDereferenceableBytes();
-    else if (auto CS = ImmutableCallSite(BV))
-      DerefBytes = CS.getDereferenceableBytes(0);
-
-    if (DerefBytes.getBoolValue() && Offset.isNonNegative()) {
-      if (DerefBytes.uge(Offset + DL.getTypeStoreSize(Ty)))
-        return true;
-    }
-  }
-
-  SmallPtrSet<const Value *, 32> Visited;
-  return ::isDereferenceablePointer(this, DL, Visited);
-}
-
 Value *Value::DoPHITranslation(const BasicBlock *CurBB,
                                const BasicBlock *PredBB) {
   PHINode *PN = dyn_cast<PHINode>(this);
diff --git a/lib/IR/ValueSymbolTable.cpp b/lib/IR/ValueSymbolTable.cpp
index 4f078f09c22..e10142de823 100644
--- a/lib/IR/ValueSymbolTable.cpp
+++ b/lib/IR/ValueSymbolTable.cpp
@@ -53,7 +53,7 @@ void ValueSymbolTable::reinsertValue(Value* V) {
   while (1) {
     // Trim any suffix off and append the next number.
     UniqueName.resize(BaseSize);
-    raw_svector_ostream(UniqueName) << ++LastUnique;
+    raw_svector_ostream(UniqueName) << "." << ++LastUnique;
 
     // Try insert the vmap entry with this suffix.
     auto IterBool = vmap.insert(std::make_pair(UniqueName, V));
diff --git a/lib/IR/ValueTypes.cpp b/lib/IR/ValueTypes.cpp
index 2d4da95f0b3..d95de3989df 100644
--- a/lib/IR/ValueTypes.cpp
+++ b/lib/IR/ValueTypes.cpp
@@ -157,6 +157,7 @@ std::string EVT::getEVTString() const {
   case MVT::v4i64:   return "v4i64";
   case MVT::v8i64:   return "v8i64";
   case MVT::v16i64:  return "v16i64";
+  case MVT::v1i128:  return "v1i128";
   case MVT::v1f32:   return "v1f32";
   case MVT::v2f32:   return "v2f32";
   case MVT::v2f16:   return "v2f16";
@@ -225,6 +226,7 @@ Type *EVT::getTypeForEVT(LLVMContext &Context) const {
   case MVT::v4i64:   return VectorType::get(Type::getInt64Ty(Context), 4);
   case MVT::v8i64:   return VectorType::get(Type::getInt64Ty(Context), 8);
   case MVT::v16i64:  return VectorType::get(Type::getInt64Ty(Context), 16);
+  case MVT::v1i128:  return VectorType::get(Type::getInt128Ty(Context), 1);
   case MVT::v2f16:   return VectorType::get(Type::getHalfTy(Context), 2);
   case MVT::v4f16:   return VectorType::get(Type::getHalfTy(Context), 4);
   case MVT::v8f16:   return VectorType::get(Type::getHalfTy(Context), 8);
diff --git a/lib/IR/Verifier.cpp b/lib/IR/Verifier.cpp
index b4c0d92f41f..a4fb6eb6a69 100644
--- a/lib/IR/Verifier.cpp
+++ b/lib/IR/Verifier.cpp
@@ -102,6 +102,9 @@ struct VerifierSupport {
       OS << '\n';
     }
   }
+  void Write(ImmutableCallSite CS) {
+    Write(CS.getInstruction());
+  }
 
   void Write(const Metadata *MD) {
     if (!MD)
@@ -181,22 +184,27 @@ class Verifier : public InstVisitor<Verifier>, VerifierSupport {
   /// \brief Track unresolved string-based type references.
   SmallDenseMap<const MDString *, const MDNode *, 32> UnresolvedTypeRefs;
 
-  /// \brief The personality function referenced by the LandingPadInsts.
-  /// All LandingPadInsts within the same function must use the same
-  /// personality function.
-  const Value *PersonalityFn;
+  /// \brief The result type for a catchpad.
+  Type *CatchPadResultTy;
+
+  /// \brief The result type for a cleanuppad.
+  Type *CleanupPadResultTy;
 
-  /// \brief Whether we've seen a call to @llvm.frameescape in this function
+  /// \brief The result type for a landingpad.
+  Type *LandingPadResultTy;
+
+  /// \brief Whether we've seen a call to @llvm.localescape in this function
   /// already.
   bool SawFrameEscape;
 
-  /// Stores the count of how many objects were passed to llvm.frameescape for a
-  /// given function and the largest index passed to llvm.framerecover.
+  /// Stores the count of how many objects were passed to llvm.localescape for a
+  /// given function and the largest index passed to llvm.localrecover.
   DenseMap<Function *, std::pair<unsigned, unsigned>> FrameEscapeInfo;
 
 public:
   explicit Verifier(raw_ostream &OS)
-      : VerifierSupport(OS), Context(nullptr), PersonalityFn(nullptr),
+      : VerifierSupport(OS), Context(nullptr), CatchPadResultTy(nullptr),
+        CleanupPadResultTy(nullptr), LandingPadResultTy(nullptr),
         SawFrameEscape(false) {}
 
   bool verify(const Function &F) {
@@ -231,7 +239,9 @@ class Verifier : public InstVisitor<Verifier>, VerifierSupport {
     // FIXME: We strip const here because the inst visitor strips const.
     visit(const_cast<Function &>(F));
     InstsInThisBlock.clear();
-    PersonalityFn = nullptr;
+    CatchPadResultTy = nullptr;
+    CleanupPadResultTy = nullptr;
+    LandingPadResultTy = nullptr;
     SawFrameEscape = false;
 
     return !Broken;
@@ -305,11 +315,10 @@ class Verifier : public InstVisitor<Verifier>, VerifierSupport {
   template <class Ty> bool isValidMetadataArray(const MDTuple &N);
 #define HANDLE_SPECIALIZED_MDNODE_LEAF(CLASS) void visit##CLASS(const CLASS &N);
 #include "llvm/IR/Metadata.def"
-  void visitMDScope(const MDScope &N);
-  void visitMDDerivedTypeBase(const MDDerivedTypeBase &N);
-  void visitMDVariable(const MDVariable &N);
-  void visitMDLexicalBlockBase(const MDLexicalBlockBase &N);
-  void visitMDTemplateParameter(const MDTemplateParameter &N);
+  void visitDIScope(const DIScope &N);
+  void visitDIVariable(const DIVariable &N);
+  void visitDILexicalBlockBase(const DILexicalBlockBase &N);
+  void visitDITemplateParameter(const DITemplateParameter &N);
 
   void visitTemplateParams(const MDNode &N, const Metadata &RawParams);
 
@@ -321,17 +330,17 @@ class Verifier : public InstVisitor<Verifier>, VerifierSupport {
 
   /// \brief Check for a valid type reference.
   ///
-  /// Checks for subclasses of \a MDType, or \a isValidUUID().
+  /// Checks for subclasses of \a DIType, or \a isValidUUID().
   bool isTypeRef(const MDNode &N, const Metadata *MD);
 
   /// \brief Check for a valid scope reference.
   ///
-  /// Checks for subclasses of \a MDScope, or \a isValidUUID().
+  /// Checks for subclasses of \a DIScope, or \a isValidUUID().
   bool isScopeRef(const MDNode &N, const Metadata *MD);
 
   /// \brief Check for a valid debug info reference.
   ///
-  /// Checks for subclasses of \a DebugNode, or \a isValidUUID().
+  /// Checks for subclasses of \a DINode, or \a isValidUUID().
   bool isDIRef(const MDNode &N, const Metadata *MD);
 
   // InstVisitor overrides...
@@ -374,7 +383,7 @@ class Verifier : public InstVisitor<Verifier>, VerifierSupport {
   void visitSelectInst(SelectInst &SI);
   void visitUserOp1(Instruction &I);
   void visitUserOp2(Instruction &I) { visitUserOp1(I); }
-  void visitIntrinsicFunctionCall(Intrinsic::ID ID, CallInst &CI);
+  void visitIntrinsicCallSite(Intrinsic::ID ID, CallSite CS);
   template <class DbgIntrinsicTy>
   void visitDbgIntrinsic(StringRef Kind, DbgIntrinsicTy &DII);
   void visitAtomicCmpXchgInst(AtomicCmpXchgInst &CXI);
@@ -384,6 +393,11 @@ class Verifier : public InstVisitor<Verifier>, VerifierSupport {
   void visitExtractValueInst(ExtractValueInst &EVI);
   void visitInsertValueInst(InsertValueInst &IVI);
   void visitLandingPadInst(LandingPadInst &LPI);
+  void visitCatchPadInst(CatchPadInst &CPI);
+  void visitCatchEndPadInst(CatchEndPadInst &CEPI);
+  void visitCleanupPadInst(CleanupPadInst &CPI);
+  void visitCleanupReturnInst(CleanupReturnInst &CRI);
+  void visitTerminatePadInst(TerminatePadInst &TPI);
 
   void VerifyCallSite(CallSite CS);
   void verifyMustTailCall(CallInst &CI);
@@ -400,6 +414,8 @@ class Verifier : public InstVisitor<Verifier>, VerifierSupport {
                             bool isReturnValue, const Value *V);
   void VerifyFunctionAttrs(FunctionType *FT, AttributeSet Attrs,
                            const Value *V);
+  void VerifyFunctionMetadata(
+      const SmallVector<std::pair<unsigned, MDNode *>, 4> MDs);
 
   void VerifyConstantExprBitcastType(const ConstantExpr *CE);
   void VerifyStatepoint(ImmutableCallSite CS);
@@ -437,9 +453,12 @@ void Verifier::visitGlobalValue(const GlobalValue &GV) {
 
   if (GV.hasAppendingLinkage()) {
     const GlobalVariable *GVar = dyn_cast<GlobalVariable>(&GV);
-    Assert(GVar && GVar->getType()->getElementType()->isArrayTy(),
+    Assert(GVar && GVar->getValueType()->isArrayTy(),
            "Only global arrays can have appending linkage!", GVar);
   }
+
+  if (GV.isDeclarationForLinker())
+    Assert(!GV.hasComdat(), "Declaration may not be in a Comdat!", &GV);
 }
 
 void Verifier::visitGlobalVariable(const GlobalVariable &GV) {
@@ -469,7 +488,7 @@ void Verifier::visitGlobalVariable(const GlobalVariable &GV) {
            "invalid linkage for intrinsic global variable", &GV);
     // Don't worry about emitting an error for it not being an array,
     // visitGlobalValue will complain on appending non-array.
-    if (ArrayType *ATy = dyn_cast<ArrayType>(GV.getType()->getElementType())) {
+    if (ArrayType *ATy = dyn_cast<ArrayType>(GV.getValueType())) {
       StructType *STy = dyn_cast<StructType>(ATy->getElementType());
       PointerType *FuncPtrTy =
           FunctionType::get(Type::getVoidTy(*Context), false)->getPointerTo();
@@ -492,7 +511,7 @@ void Verifier::visitGlobalVariable(const GlobalVariable &GV) {
                        GV.getName() == "llvm.compiler.used")) {
     Assert(!GV.hasInitializer() || GV.hasAppendingLinkage(),
            "invalid linkage for intrinsic global variable", &GV);
-    Type *GVType = GV.getType()->getElementType();
+    Type *GVType = GV.getValueType();
     if (ArrayType *ATy = dyn_cast<ArrayType>(GVType)) {
       PointerType *PTy = dyn_cast<PointerType>(ATy->getElementType());
       Assert(PTy, "wrong type for intrinsic global variable", &GV);
@@ -582,7 +601,6 @@ void Verifier::visitAliaseeSubExpr(SmallPtrSetImpl<const GlobalAlias*> &Visited,
 }
 
 void Verifier::visitGlobalAlias(const GlobalAlias &GA) {
-  Assert(!GA.getName().empty(), "Alias name cannot be empty!", &GA);
   Assert(GlobalAlias::isValidLinkage(GA.getLinkage()),
          "Alias should have private, internal, linkonce, weak, linkonce_odr, "
          "weak_odr, or external linkage!",
@@ -605,7 +623,7 @@ void Verifier::visitNamedMDNode(const NamedMDNode &NMD) {
     MDNode *MD = NMD.getOperand(i);
 
     if (NMD.getName() == "llvm.dbg.cu") {
-      Assert(MD && isa<MDCompileUnit>(MD), "invalid compile unit", &NMD, MD);
+      Assert(MD && isa<DICompileUnit>(MD), "invalid compile unit", &NMD, MD);
     }
 
     if (!MD)
@@ -711,17 +729,17 @@ bool Verifier::isValidUUID(const MDNode &N, const Metadata *MD) {
 
 /// \brief Check if a value can be a reference to a type.
 bool Verifier::isTypeRef(const MDNode &N, const Metadata *MD) {
-  return !MD || isValidUUID(N, MD) || isa<MDType>(MD);
+  return !MD || isValidUUID(N, MD) || isa<DIType>(MD);
 }
 
 /// \brief Check if a value can be a ScopeRef.
 bool Verifier::isScopeRef(const MDNode &N, const Metadata *MD) {
-  return !MD || isValidUUID(N, MD) || isa<MDScope>(MD);
+  return !MD || isValidUUID(N, MD) || isa<DIScope>(MD);
 }
 
 /// \brief Check if a value can be a debug info ref.
 bool Verifier::isDIRef(const MDNode &N, const Metadata *MD) {
-  return !MD || isValidUUID(N, MD) || isa<DebugNode>(MD);
+  return !MD || isValidUUID(N, MD) || isa<DINode>(MD);
 }
 
 template <class Ty>
@@ -748,69 +766,40 @@ bool isValidMetadataNullArray(const MDTuple &N) {
   return isValidMetadataArrayImpl<Ty>(N, /* AllowNull */ true);
 }
 
-void Verifier::visitMDLocation(const MDLocation &N) {
-  Assert(N.getRawScope() && isa<MDLocalScope>(N.getRawScope()),
+void Verifier::visitDILocation(const DILocation &N) {
+  Assert(N.getRawScope() && isa<DILocalScope>(N.getRawScope()),
          "location requires a valid scope", &N, N.getRawScope());
   if (auto *IA = N.getRawInlinedAt())
-    Assert(isa<MDLocation>(IA), "inlined-at should be a location", &N, IA);
+    Assert(isa<DILocation>(IA), "inlined-at should be a location", &N, IA);
 }
 
-void Verifier::visitGenericDebugNode(const GenericDebugNode &N) {
+void Verifier::visitGenericDINode(const GenericDINode &N) {
   Assert(N.getTag(), "invalid tag", &N);
 }
 
-void Verifier::visitMDScope(const MDScope &N) {
+void Verifier::visitDIScope(const DIScope &N) {
   if (auto *F = N.getRawFile())
-    Assert(isa<MDFile>(F), "invalid file", &N, F);
+    Assert(isa<DIFile>(F), "invalid file", &N, F);
 }
 
-void Verifier::visitMDSubrange(const MDSubrange &N) {
+void Verifier::visitDISubrange(const DISubrange &N) {
   Assert(N.getTag() == dwarf::DW_TAG_subrange_type, "invalid tag", &N);
   Assert(N.getCount() >= -1, "invalid subrange count", &N);
 }
 
-void Verifier::visitMDEnumerator(const MDEnumerator &N) {
+void Verifier::visitDIEnumerator(const DIEnumerator &N) {
   Assert(N.getTag() == dwarf::DW_TAG_enumerator, "invalid tag", &N);
 }
 
-void Verifier::visitMDBasicType(const MDBasicType &N) {
+void Verifier::visitDIBasicType(const DIBasicType &N) {
   Assert(N.getTag() == dwarf::DW_TAG_base_type ||
              N.getTag() == dwarf::DW_TAG_unspecified_type,
          "invalid tag", &N);
 }
 
-void Verifier::visitMDDerivedTypeBase(const MDDerivedTypeBase &N) {
+void Verifier::visitDIDerivedType(const DIDerivedType &N) {
   // Common scope checks.
-  visitMDScope(N);
-
-  Assert(isScopeRef(N, N.getScope()), "invalid scope", &N, N.getScope());
-  Assert(isTypeRef(N, N.getBaseType()), "invalid base type", &N,
-         N.getBaseType());
-
-  // FIXME: Sink this into the subclass verifies.
-  if (!N.getFile() || N.getFile()->getFilename().empty()) {
-    // Check whether the filename is allowed to be empty.
-    uint16_t Tag = N.getTag();
-    Assert(
-        Tag == dwarf::DW_TAG_const_type || Tag == dwarf::DW_TAG_volatile_type ||
-            Tag == dwarf::DW_TAG_pointer_type ||
-            Tag == dwarf::DW_TAG_ptr_to_member_type ||
-            Tag == dwarf::DW_TAG_reference_type ||
-            Tag == dwarf::DW_TAG_rvalue_reference_type ||
-            Tag == dwarf::DW_TAG_restrict_type ||
-            Tag == dwarf::DW_TAG_array_type ||
-            Tag == dwarf::DW_TAG_enumeration_type ||
-            Tag == dwarf::DW_TAG_subroutine_type ||
-            Tag == dwarf::DW_TAG_inheritance || Tag == dwarf::DW_TAG_friend ||
-            Tag == dwarf::DW_TAG_structure_type ||
-            Tag == dwarf::DW_TAG_member || Tag == dwarf::DW_TAG_typedef,
-        "derived/composite type requires a filename", &N, N.getFile());
-  }
-}
-
-void Verifier::visitMDDerivedType(const MDDerivedType &N) {
-  // Common derived type checks.
-  visitMDDerivedTypeBase(N);
+  visitDIScope(N);
 
   Assert(N.getTag() == dwarf::DW_TAG_typedef ||
              N.getTag() == dwarf::DW_TAG_pointer_type ||
@@ -828,34 +817,41 @@ void Verifier::visitMDDerivedType(const MDDerivedType &N) {
     Assert(isTypeRef(N, N.getExtraData()), "invalid pointer to member type", &N,
            N.getExtraData());
   }
+
+  Assert(isScopeRef(N, N.getScope()), "invalid scope", &N, N.getScope());
+  Assert(isTypeRef(N, N.getBaseType()), "invalid base type", &N,
+         N.getBaseType());
 }
 
 static bool hasConflictingReferenceFlags(unsigned Flags) {
-  return (Flags & DebugNode::FlagLValueReference) &&
-         (Flags & DebugNode::FlagRValueReference);
+  return (Flags & DINode::FlagLValueReference) &&
+         (Flags & DINode::FlagRValueReference);
 }
 
 void Verifier::visitTemplateParams(const MDNode &N, const Metadata &RawParams) {
   auto *Params = dyn_cast<MDTuple>(&RawParams);
   Assert(Params, "invalid template params", &N, &RawParams);
   for (Metadata *Op : Params->operands()) {
-    Assert(Op && isa<MDTemplateParameter>(Op), "invalid template parameter", &N,
+    Assert(Op && isa<DITemplateParameter>(Op), "invalid template parameter", &N,
            Params, Op);
   }
 }
 
-void Verifier::visitMDCompositeType(const MDCompositeType &N) {
-  // Common derived type checks.
-  visitMDDerivedTypeBase(N);
+void Verifier::visitDICompositeType(const DICompositeType &N) {
+  // Common scope checks.
+  visitDIScope(N);
 
   Assert(N.getTag() == dwarf::DW_TAG_array_type ||
              N.getTag() == dwarf::DW_TAG_structure_type ||
              N.getTag() == dwarf::DW_TAG_union_type ||
              N.getTag() == dwarf::DW_TAG_enumeration_type ||
-             N.getTag() == dwarf::DW_TAG_subroutine_type ||
              N.getTag() == dwarf::DW_TAG_class_type,
          "invalid tag", &N);
 
+  Assert(isScopeRef(N, N.getScope()), "invalid scope", &N, N.getScope());
+  Assert(isTypeRef(N, N.getBaseType()), "invalid base type", &N,
+         N.getBaseType());
+
   Assert(!N.getRawElements() || isa<MDTuple>(N.getRawElements()),
          "invalid composite elements", &N, N.getRawElements());
   Assert(isTypeRef(N, N.getRawVTableHolder()), "invalid vtable holder", &N,
@@ -866,9 +862,15 @@ void Verifier::visitMDCompositeType(const MDCompositeType &N) {
          &N);
   if (auto *Params = N.getRawTemplateParams())
     visitTemplateParams(N, *Params);
+
+  if (N.getTag() == dwarf::DW_TAG_class_type ||
+      N.getTag() == dwarf::DW_TAG_union_type) {
+    Assert(N.getFile() && !N.getFile()->getFilename().empty(),
+           "class/union requires a filename", &N, N.getFile());
+  }
 }
 
-void Verifier::visitMDSubroutineType(const MDSubroutineType &N) {
+void Verifier::visitDISubroutineType(const DISubroutineType &N) {
   Assert(N.getTag() == dwarf::DW_TAG_subroutine_type, "invalid tag", &N);
   if (auto *Types = N.getRawTypeArray()) {
     Assert(isa<MDTuple>(Types), "invalid composite elements", &N, Types);
@@ -880,24 +882,24 @@ void Verifier::visitMDSubroutineType(const MDSubroutineType &N) {
          &N);
 }
 
-void Verifier::visitMDFile(const MDFile &N) {
+void Verifier::visitDIFile(const DIFile &N) {
   Assert(N.getTag() == dwarf::DW_TAG_file_type, "invalid tag", &N);
 }
 
-void Verifier::visitMDCompileUnit(const MDCompileUnit &N) {
+void Verifier::visitDICompileUnit(const DICompileUnit &N) {
   Assert(N.getTag() == dwarf::DW_TAG_compile_unit, "invalid tag", &N);
 
   // Don't bother verifying the compilation directory or producer string
   // as those could be empty.
-  Assert(N.getRawFile() && isa<MDFile>(N.getRawFile()),
-         "invalid file", &N, N.getRawFile());
+  Assert(N.getRawFile() && isa<DIFile>(N.getRawFile()), "invalid file", &N,
+         N.getRawFile());
   Assert(!N.getFile()->getFilename().empty(), "invalid filename", &N,
          N.getFile());
 
   if (auto *Array = N.getRawEnumTypes()) {
     Assert(isa<MDTuple>(Array), "invalid enum list", &N, Array);
     for (Metadata *Op : N.getEnumTypes()->operands()) {
-      auto *Enum = dyn_cast_or_null<MDCompositeType>(Op);
+      auto *Enum = dyn_cast_or_null<DICompositeType>(Op);
       Assert(Enum && Enum->getTag() == dwarf::DW_TAG_enumeration_type,
              "invalid enum type", &N, N.getEnumTypes(), Op);
     }
@@ -905,36 +907,36 @@ void Verifier::visitMDCompileUnit(const MDCompileUnit &N) {
   if (auto *Array = N.getRawRetainedTypes()) {
     Assert(isa<MDTuple>(Array), "invalid retained type list", &N, Array);
     for (Metadata *Op : N.getRetainedTypes()->operands()) {
-      Assert(Op && isa<MDType>(Op), "invalid retained type", &N, Op);
+      Assert(Op && isa<DIType>(Op), "invalid retained type", &N, Op);
     }
   }
   if (auto *Array = N.getRawSubprograms()) {
     Assert(isa<MDTuple>(Array), "invalid subprogram list", &N, Array);
     for (Metadata *Op : N.getSubprograms()->operands()) {
-      Assert(Op && isa<MDSubprogram>(Op), "invalid subprogram ref", &N, Op);
+      Assert(Op && isa<DISubprogram>(Op), "invalid subprogram ref", &N, Op);
     }
   }
   if (auto *Array = N.getRawGlobalVariables()) {
     Assert(isa<MDTuple>(Array), "invalid global variable list", &N, Array);
     for (Metadata *Op : N.getGlobalVariables()->operands()) {
-      Assert(Op && isa<MDGlobalVariable>(Op), "invalid global variable ref", &N,
+      Assert(Op && isa<DIGlobalVariable>(Op), "invalid global variable ref", &N,
              Op);
     }
   }
   if (auto *Array = N.getRawImportedEntities()) {
     Assert(isa<MDTuple>(Array), "invalid imported entity list", &N, Array);
     for (Metadata *Op : N.getImportedEntities()->operands()) {
-      Assert(Op && isa<MDImportedEntity>(Op), "invalid imported entity ref", &N,
+      Assert(Op && isa<DIImportedEntity>(Op), "invalid imported entity ref", &N,
              Op);
     }
   }
 }
 
-void Verifier::visitMDSubprogram(const MDSubprogram &N) {
+void Verifier::visitDISubprogram(const DISubprogram &N) {
   Assert(N.getTag() == dwarf::DW_TAG_subprogram, "invalid tag", &N);
   Assert(isScopeRef(N, N.getRawScope()), "invalid scope", &N, N.getRawScope());
   if (auto *T = N.getRawType())
-    Assert(isa<MDSubroutineType>(T), "invalid subroutine type", &N, T);
+    Assert(isa<DISubroutineType>(T), "invalid subroutine type", &N, T);
   Assert(isTypeRef(N, N.getRawContainingType()), "invalid containing type", &N,
          N.getRawContainingType());
   if (auto *RawF = N.getRawFunction()) {
@@ -947,14 +949,14 @@ void Verifier::visitMDSubprogram(const MDSubprogram &N) {
   if (auto *Params = N.getRawTemplateParams())
     visitTemplateParams(N, *Params);
   if (auto *S = N.getRawDeclaration()) {
-    Assert(isa<MDSubprogram>(S) && !cast<MDSubprogram>(S)->isDefinition(),
+    Assert(isa<DISubprogram>(S) && !cast<DISubprogram>(S)->isDefinition(),
            "invalid subprogram declaration", &N, S);
   }
   if (auto *RawVars = N.getRawVariables()) {
     auto *Vars = dyn_cast<MDTuple>(RawVars);
     Assert(Vars, "invalid variable list", &N, RawVars);
     for (Metadata *Op : Vars->operands()) {
-      Assert(Op && isa<MDLocalVariable>(Op), "invalid local variable", &N, Vars,
+      Assert(Op && isa<DILocalVariable>(Op), "invalid local variable", &N, Vars,
              Op);
     }
   }
@@ -973,20 +975,20 @@ void Verifier::visitMDSubprogram(const MDSubprogram &N) {
   SmallPtrSet<const MDNode *, 32> Seen;
   for (auto &BB : *F)
     for (auto &I : BB) {
-      // Be careful about using MDLocation here since we might be dealing with
+      // Be careful about using DILocation here since we might be dealing with
       // broken code (this is the Verifier after all).
-      MDLocation *DL =
-          dyn_cast_or_null<MDLocation>(I.getDebugLoc().getAsMDNode());
+      DILocation *DL =
+          dyn_cast_or_null<DILocation>(I.getDebugLoc().getAsMDNode());
       if (!DL)
         continue;
       if (!Seen.insert(DL).second)
         continue;
 
-      MDLocalScope *Scope = DL->getInlinedAtScope();
+      DILocalScope *Scope = DL->getInlinedAtScope();
       if (Scope && !Seen.insert(Scope).second)
         continue;
 
-      MDSubprogram *SP = Scope ? Scope->getSubprogram() : nullptr;
+      DISubprogram *SP = Scope ? Scope->getSubprogram() : nullptr;
       if (SP && !Seen.insert(SP).second)
         continue;
 
@@ -997,43 +999,48 @@ void Verifier::visitMDSubprogram(const MDSubprogram &N) {
     }
 }
 
-void Verifier::visitMDLexicalBlockBase(const MDLexicalBlockBase &N) {
+void Verifier::visitDILexicalBlockBase(const DILexicalBlockBase &N) {
   Assert(N.getTag() == dwarf::DW_TAG_lexical_block, "invalid tag", &N);
-  Assert(N.getRawScope() && isa<MDLocalScope>(N.getRawScope()),
+  Assert(N.getRawScope() && isa<DILocalScope>(N.getRawScope()),
          "invalid local scope", &N, N.getRawScope());
 }
 
-void Verifier::visitMDLexicalBlock(const MDLexicalBlock &N) {
-  visitMDLexicalBlockBase(N);
+void Verifier::visitDILexicalBlock(const DILexicalBlock &N) {
+  visitDILexicalBlockBase(N);
 
   Assert(N.getLine() || !N.getColumn(),
          "cannot have column info without line info", &N);
 }
 
-void Verifier::visitMDLexicalBlockFile(const MDLexicalBlockFile &N) {
-  visitMDLexicalBlockBase(N);
+void Verifier::visitDILexicalBlockFile(const DILexicalBlockFile &N) {
+  visitDILexicalBlockBase(N);
 }
 
-void Verifier::visitMDNamespace(const MDNamespace &N) {
+void Verifier::visitDINamespace(const DINamespace &N) {
   Assert(N.getTag() == dwarf::DW_TAG_namespace, "invalid tag", &N);
   if (auto *S = N.getRawScope())
-    Assert(isa<MDScope>(S), "invalid scope ref", &N, S);
+    Assert(isa<DIScope>(S), "invalid scope ref", &N, S);
+}
+
+void Verifier::visitDIModule(const DIModule &N) {
+  Assert(N.getTag() == dwarf::DW_TAG_module, "invalid tag", &N);
+  Assert(!N.getName().empty(), "anonymous module", &N);
 }
 
-void Verifier::visitMDTemplateParameter(const MDTemplateParameter &N) {
+void Verifier::visitDITemplateParameter(const DITemplateParameter &N) {
   Assert(isTypeRef(N, N.getType()), "invalid type ref", &N, N.getType());
 }
 
-void Verifier::visitMDTemplateTypeParameter(const MDTemplateTypeParameter &N) {
-  visitMDTemplateParameter(N);
+void Verifier::visitDITemplateTypeParameter(const DITemplateTypeParameter &N) {
+  visitDITemplateParameter(N);
 
   Assert(N.getTag() == dwarf::DW_TAG_template_type_parameter, "invalid tag",
          &N);
 }
 
-void Verifier::visitMDTemplateValueParameter(
-    const MDTemplateValueParameter &N) {
-  visitMDTemplateParameter(N);
+void Verifier::visitDITemplateValueParameter(
+    const DITemplateValueParameter &N) {
+  visitDITemplateParameter(N);
 
   Assert(N.getTag() == dwarf::DW_TAG_template_value_parameter ||
              N.getTag() == dwarf::DW_TAG_GNU_template_template_param ||
@@ -1041,17 +1048,17 @@ void Verifier::visitMDTemplateValueParameter(
          "invalid tag", &N);
 }
 
-void Verifier::visitMDVariable(const MDVariable &N) {
+void Verifier::visitDIVariable(const DIVariable &N) {
   if (auto *S = N.getRawScope())
-    Assert(isa<MDScope>(S), "invalid scope", &N, S);
+    Assert(isa<DIScope>(S), "invalid scope", &N, S);
   Assert(isTypeRef(N, N.getRawType()), "invalid type ref", &N, N.getRawType());
   if (auto *F = N.getRawFile())
-    Assert(isa<MDFile>(F), "invalid file", &N, F);
+    Assert(isa<DIFile>(F), "invalid file", &N, F);
 }
 
-void Verifier::visitMDGlobalVariable(const MDGlobalVariable &N) {
+void Verifier::visitDIGlobalVariable(const DIGlobalVariable &N) {
   // Checks common to all variables.
-  visitMDVariable(N);
+  visitDIVariable(N);
 
   Assert(N.getTag() == dwarf::DW_TAG_variable, "invalid tag", &N);
   Assert(!N.getName().empty(), "missing global variable name", &N);
@@ -1061,40 +1068,42 @@ void Verifier::visitMDGlobalVariable(const MDGlobalVariable &N) {
            "invalid global varaible ref", &N, V);
   }
   if (auto *Member = N.getRawStaticDataMemberDeclaration()) {
-    Assert(isa<MDDerivedType>(Member), "invalid static data member declaration",
+    Assert(isa<DIDerivedType>(Member), "invalid static data member declaration",
            &N, Member);
   }
 }
 
-void Verifier::visitMDLocalVariable(const MDLocalVariable &N) {
+void Verifier::visitDILocalVariable(const DILocalVariable &N) {
   // Checks common to all variables.
-  visitMDVariable(N);
+  visitDIVariable(N);
 
   Assert(N.getTag() == dwarf::DW_TAG_auto_variable ||
              N.getTag() == dwarf::DW_TAG_arg_variable,
          "invalid tag", &N);
-  Assert(N.getRawScope() && isa<MDLocalScope>(N.getRawScope()),
+  Assert(N.getRawScope() && isa<DILocalScope>(N.getRawScope()),
          "local variable requires a valid scope", &N, N.getRawScope());
+  Assert(bool(N.getArg()) == (N.getTag() == dwarf::DW_TAG_arg_variable),
+         "local variable should have arg iff it's a DW_TAG_arg_variable", &N);
 }
 
-void Verifier::visitMDExpression(const MDExpression &N) {
+void Verifier::visitDIExpression(const DIExpression &N) {
   Assert(N.isValid(), "invalid expression", &N);
 }
 
-void Verifier::visitMDObjCProperty(const MDObjCProperty &N) {
+void Verifier::visitDIObjCProperty(const DIObjCProperty &N) {
   Assert(N.getTag() == dwarf::DW_TAG_APPLE_property, "invalid tag", &N);
   if (auto *T = N.getRawType())
-    Assert(isa<MDType>(T), "invalid type ref", &N, T);
+    Assert(isTypeRef(N, T), "invalid type ref", &N, T);
   if (auto *F = N.getRawFile())
-    Assert(isa<MDFile>(F), "invalid file", &N, F);
+    Assert(isa<DIFile>(F), "invalid file", &N, F);
 }
 
-void Verifier::visitMDImportedEntity(const MDImportedEntity &N) {
+void Verifier::visitDIImportedEntity(const DIImportedEntity &N) {
   Assert(N.getTag() == dwarf::DW_TAG_imported_module ||
              N.getTag() == dwarf::DW_TAG_imported_declaration,
          "invalid tag", &N);
   if (auto *S = N.getRawScope())
-    Assert(isa<MDScope>(S), "invalid scope for imported entity", &N, S);
+    Assert(isa<DIScope>(S), "invalid scope for imported entity", &N, S);
   Assert(isDIRef(N, N.getEntity()), "invalid imported entity", &N,
          N.getEntity());
 }
@@ -1249,6 +1258,7 @@ void Verifier::VerifyAttributeTypes(AttributeSet Attrs, unsigned Idx,
         I->getKindAsEnum() == Attribute::StackProtect ||
         I->getKindAsEnum() == Attribute::StackProtectReq ||
         I->getKindAsEnum() == Attribute::StackProtectStrong ||
+        I->getKindAsEnum() == Attribute::SafeStack ||
         I->getKindAsEnum() == Attribute::NoRedZone ||
         I->getKindAsEnum() == Attribute::NoImplicitFloat ||
         I->getKindAsEnum() == Attribute::Naked ||
@@ -1266,7 +1276,9 @@ void Verifier::VerifyAttributeTypes(AttributeSet Attrs, unsigned Idx,
         I->getKindAsEnum() == Attribute::NoBuiltin ||
         I->getKindAsEnum() == Attribute::Cold ||
         I->getKindAsEnum() == Attribute::OptimizeNone ||
-        I->getKindAsEnum() == Attribute::JumpTable) {
+        I->getKindAsEnum() == Attribute::JumpTable ||
+        I->getKindAsEnum() == Attribute::Convergent ||
+        I->getKindAsEnum() == Attribute::ArgMemOnly) {
       if (!isFunction) {
         CheckFailed("Attribute '" + I->getAsString() +
                     "' only applies to functions!", V);
@@ -1350,9 +1362,10 @@ void Verifier::VerifyParameterAttrs(AttributeSet Attrs, unsigned Idx, Type *Ty,
          V);
 
   Assert(!AttrBuilder(Attrs, Idx)
-              .hasAttributes(AttributeFuncs::typeIncompatible(Ty, Idx), Idx),
+              .overlaps(AttributeFuncs::typeIncompatible(Ty)),
          "Wrong types for attribute: " +
-             AttributeFuncs::typeIncompatible(Ty, Idx).getAsString(Idx),
+         AttributeSet::get(*Context, Idx,
+                        AttributeFuncs::typeIncompatible(Ty)).getAsString(Idx),
          V);
 
   if (PointerType *PTy = dyn_cast<PointerType>(Ty)) {
@@ -1462,6 +1475,36 @@ void Verifier::VerifyFunctionAttrs(FunctionType *FT, AttributeSet Attrs,
   }
 }
 
+void Verifier::VerifyFunctionMetadata(
+    const SmallVector<std::pair<unsigned, MDNode *>, 4> MDs) {
+  if (MDs.empty())
+    return;
+
+  for (unsigned i = 0; i < MDs.size(); i++) {
+    if (MDs[i].first == LLVMContext::MD_prof) {
+      MDNode *MD = MDs[i].second;
+      Assert(MD->getNumOperands() == 2,
+             "!prof annotations should have exactly 2 operands", MD);
+
+      // Check first operand.
+      Assert(MD->getOperand(0) != nullptr, "first operand should not be null",
+             MD);
+      Assert(isa<MDString>(MD->getOperand(0)),
+             "expected string with name of the !prof annotation", MD);
+      MDString *MDS = cast<MDString>(MD->getOperand(0));
+      StringRef ProfName = MDS->getString();
+      Assert(ProfName.equals("function_entry_count"),
+             "first operand should be 'function_entry_count'", MD);
+
+      // Check second operand.
+      Assert(MD->getOperand(1) != nullptr, "second operand should not be null",
+             MD);
+      Assert(isa<ConstantAsMetadata>(MD->getOperand(1)),
+             "expected integer argument to function_entry_count", MD);
+    }
+  }
+}
+
 void Verifier::VerifyConstantExprBitcastType(const ConstantExpr *CE) {
   if (CE->getOpcode() != Instruction::BitCast)
     return;
@@ -1493,18 +1536,34 @@ void Verifier::VerifyStatepoint(ImmutableCallSite CS) {
 
   const Instruction &CI = *CS.getInstruction();
 
-  Assert(!CS.doesNotAccessMemory() && !CS.onlyReadsMemory(),
-         "gc.statepoint must read and write memory to preserve "
+  Assert(!CS.doesNotAccessMemory() && !CS.onlyReadsMemory() &&
+         !CS.onlyAccessesArgMemory(),
+         "gc.statepoint must read and write all memory to preserve "
          "reordering restrictions required by safepoint semantics",
          &CI);
 
-  const Value *Target = CS.getArgument(0);
+  const Value *IDV = CS.getArgument(0);
+  Assert(isa<ConstantInt>(IDV), "gc.statepoint ID must be a constant integer",
+         &CI);
+
+  const Value *NumPatchBytesV = CS.getArgument(1);
+  Assert(isa<ConstantInt>(NumPatchBytesV),
+         "gc.statepoint number of patchable bytes must be a constant integer",
+         &CI);
+  const int64_t NumPatchBytes =
+      cast<ConstantInt>(NumPatchBytesV)->getSExtValue();
+  assert(isInt<32>(NumPatchBytes) && "NumPatchBytesV is an i32!");
+  Assert(NumPatchBytes >= 0, "gc.statepoint number of patchable bytes must be "
+                             "positive",
+         &CI);
+
+  const Value *Target = CS.getArgument(2);
   const PointerType *PT = dyn_cast<PointerType>(Target->getType());
   Assert(PT && PT->getElementType()->isFunctionTy(),
          "gc.statepoint callee must be of function pointer type", &CI, Target);
   FunctionType *TargetFuncType = cast<FunctionType>(PT->getElementType());
 
-  const Value *NumCallArgsV = CS.getArgument(1);
+  const Value *NumCallArgsV = CS.getArgument(3);
   Assert(isa<ConstantInt>(NumCallArgsV),
          "gc.statepoint number of arguments to underlying call "
          "must be constant integer",
@@ -1528,22 +1587,38 @@ void Verifier::VerifyStatepoint(ImmutableCallSite CS) {
     Assert(NumCallArgs == NumParams,
            "gc.statepoint mismatch in number of call args", &CI);
 
-  const Value *Unused = CS.getArgument(2);
-  Assert(isa<ConstantInt>(Unused) && cast<ConstantInt>(Unused)->isNullValue(),
-         "gc.statepoint parameter #3 must be zero", &CI);
+  const Value *FlagsV = CS.getArgument(4);
+  Assert(isa<ConstantInt>(FlagsV),
+         "gc.statepoint flags must be constant integer", &CI);
+  const uint64_t Flags = cast<ConstantInt>(FlagsV)->getZExtValue();
+  Assert((Flags & ~(uint64_t)StatepointFlags::MaskAll) == 0,
+         "unknown flag used in gc.statepoint flags argument", &CI);
 
   // Verify that the types of the call parameter arguments match
   // the type of the wrapped callee.
   for (int i = 0; i < NumParams; i++) {
     Type *ParamType = TargetFuncType->getParamType(i);
-    Type *ArgType = CS.getArgument(3+i)->getType();
+    Type *ArgType = CS.getArgument(5 + i)->getType();
     Assert(ArgType == ParamType,
            "gc.statepoint call argument does not match wrapped "
            "function type",
            &CI);
   }
-  const int EndCallArgsInx = 2+NumCallArgs;
-  const Value *NumDeoptArgsV = CS.getArgument(EndCallArgsInx+1);
+
+  const int EndCallArgsInx = 4 + NumCallArgs;
+
+  const Value *NumTransitionArgsV = CS.getArgument(EndCallArgsInx+1);
+  Assert(isa<ConstantInt>(NumTransitionArgsV),
+         "gc.statepoint number of transition arguments "
+         "must be constant integer",
+         &CI);
+  const int NumTransitionArgs =
+      cast<ConstantInt>(NumTransitionArgsV)->getZExtValue();
+  Assert(NumTransitionArgs >= 0,
+         "gc.statepoint number of transition arguments must be positive", &CI);
+  const int EndTransitionArgsInx = EndCallArgsInx + 1 + NumTransitionArgs;
+
+  const Value *NumDeoptArgsV = CS.getArgument(EndTransitionArgsInx+1);
   Assert(isa<ConstantInt>(NumDeoptArgsV),
          "gc.statepoint number of deoptimization arguments "
          "must be constant integer",
@@ -1553,7 +1628,9 @@ void Verifier::VerifyStatepoint(ImmutableCallSite CS) {
                             "must be positive",
          &CI);
 
-  Assert(4 + NumCallArgs + NumDeoptArgs <= (int)CS.arg_size(),
+  const int ExpectedNumArgs =
+      7 + NumCallArgs + NumTransitionArgs + NumDeoptArgs;
+  Assert(ExpectedNumArgs <= (int)CS.arg_size(),
          "gc.statepoint too few arguments according to length fields", &CI);
 
   // Check that the only uses of this gc.statepoint are gc.result or 
@@ -1592,8 +1669,8 @@ void Verifier::verifyFrameRecoverIndices() {
     unsigned EscapedObjectCount = Counts.second.first;
     unsigned MaxRecoveredIndex = Counts.second.second;
     Assert(MaxRecoveredIndex <= EscapedObjectCount,
-           "all indices passed to llvm.framerecover must be less than the "
-           "number of arguments passed ot llvm.frameescape in the parent "
+           "all indices passed to llvm.localrecover must be less than the "
+           "number of arguments passed ot llvm.localescape in the parent "
            "function",
            F);
   }
@@ -1669,11 +1746,23 @@ void Verifier::visitFunction(const Function &F) {
              "Function takes metadata but isn't an intrinsic", I, &F);
   }
 
+  // Get the function metadata attachments.
+  SmallVector<std::pair<unsigned, MDNode *>, 4> MDs;
+  F.getAllMetadata(MDs);
+  assert(F.hasMetadata() != MDs.empty() && "Bit out-of-sync");
+  VerifyFunctionMetadata(MDs);
+
   if (F.isMaterializable()) {
     // Function has a body somewhere we can't see.
+    Assert(MDs.empty(), "unmaterialized function cannot have metadata", &F,
+           MDs.empty() ? nullptr : MDs.front().second);
   } else if (F.isDeclaration()) {
     Assert(F.hasExternalLinkage() || F.hasExternalWeakLinkage(),
            "invalid linkage type for function declaration", &F);
+    Assert(MDs.empty(), "function without a body cannot have metadata", &F,
+           MDs.empty() ? nullptr : MDs.front().second);
+    Assert(!F.hasPersonalityFn(),
+           "Function declaration shouldn't have a personality routine", &F);
   } else {
     // Verify that this function (which has a body) is not named "llvm.*".  It
     // is not legal to define intrinsics.
@@ -1689,6 +1778,10 @@ void Verifier::visitFunction(const Function &F) {
       Assert(!BlockAddress::lookup(Entry)->isConstantUsed(),
              "blockaddress may not be used with the entry block!", Entry);
     }
+
+    // Visit metadata attachments.
+    for (const auto &I : MDs)
+      visitMDNode(*I.second);
   }
 
   // If this function is actually an intrinsic, verify that it is only used in
@@ -2101,8 +2194,8 @@ void Verifier::visitPHINode(PHINode &PN) {
 
   // Check that all of the values of the PHI node have the same type as the
   // result, and that the incoming blocks are really basic blocks.
-  for (unsigned i = 0, e = PN.getNumIncomingValues(); i != e; ++i) {
-    Assert(PN.getType() == PN.getIncomingValue(i)->getType(),
+  for (Value *IncValue : PN.incoming_values()) {
+    Assert(PN.getType() == IncValue->getType(),
            "PHI node operands are not the same type as the result!", &PN);
   }
 
@@ -2120,7 +2213,11 @@ void Verifier::VerifyCallSite(CallSite CS) {
 
   Assert(FPTy->getElementType()->isFunctionTy(),
          "Called function is not pointer to function type!", I);
-  FunctionType *FTy = cast<FunctionType>(FPTy->getElementType());
+
+  Assert(FPTy->getElementType() == CS.getFunctionType(),
+         "Called function is not the same type as the call!", I);
+
+  FunctionType *FTy = CS.getFunctionType();
 
   // Verify that the correct number of arguments are being passed
   if (FTy->isVarArg())
@@ -2203,6 +2300,10 @@ void Verifier::VerifyCallSite(CallSite CS) {
              "Function has metadata parameter but isn't an intrinsic", I);
   }
 
+  if (Function *F = CS.getCalledFunction())
+    if (Intrinsic::ID ID = (Intrinsic::ID)F->getIntrinsicID())
+      visitIntrinsicCallSite(ID, CS);
+
   visitInstruction(*I);
 }
 
@@ -2239,12 +2340,8 @@ void Verifier::verifyMustTailCall(CallInst &CI) {
   //   parameters or return types may differ in pointee type, but not
   //   address space.
   Function *F = CI.getParent()->getParent();
-  auto GetFnTy = [](Value *V) {
-    return cast<FunctionType>(
-        cast<PointerType>(V->getType())->getElementType());
-  };
-  FunctionType *CallerTy = GetFnTy(F);
-  FunctionType *CalleeTy = GetFnTy(CI.getCalledValue());
+  FunctionType *CallerTy = F->getFunctionType();
+  FunctionType *CalleeTy = CI.getFunctionType();
   Assert(CallerTy->getNumParams() == CalleeTy->getNumParams(),
          "cannot guarantee tail call due to mismatched parameter counts", &CI);
   Assert(CallerTy->isVarArg() == CalleeTy->isVarArg(),
@@ -2302,26 +2399,17 @@ void Verifier::visitCallInst(CallInst &CI) {
 
   if (CI.isMustTailCall())
     verifyMustTailCall(CI);
-
-  if (Function *F = CI.getCalledFunction())
-    if (Intrinsic::ID ID = (Intrinsic::ID)F->getIntrinsicID())
-      visitIntrinsicFunctionCall(ID, CI);
 }
 
 void Verifier::visitInvokeInst(InvokeInst &II) {
   VerifyCallSite(&II);
 
-  // Verify that there is a landingpad instruction as the first non-PHI
-  // instruction of the 'unwind' destination.
-  Assert(II.getUnwindDest()->isLandingPad(),
-         "The unwind destination does not have a landingpad instruction!", &II);
-
-  if (Function *F = II.getCalledFunction())
-    // TODO: Ideally we should use visitIntrinsicFunction here. But it uses
-    //       CallInst as an input parameter. It not woth updating this whole
-    //       function only to support statepoint verification.
-    if (F->getIntrinsicID() == Intrinsic::experimental_gc_statepoint)
-      VerifyStatepoint(ImmutableCallSite(&II));
+  // Verify that the first non-PHI instruction of the unwind destination is an
+  // exception handling instruction.
+  Assert(
+      II.getUnwindDest()->isEHPad(),
+      "The unwind destination does not have an exception handling instruction!",
+      &II);
 
   visitTerminatorInst(II);
 }
@@ -2451,33 +2539,30 @@ void Verifier::visitGetElementPtrInst(GetElementPtrInst &GEP) {
 
   Assert(isa<PointerType>(TargetTy),
          "GEP base pointer is not a vector or a vector of pointers", &GEP);
-  Assert(cast<PointerType>(TargetTy)->getElementType()->isSized(),
-         "GEP into unsized type!", &GEP);
-  Assert(GEP.getPointerOperandType()->isVectorTy() ==
-             GEP.getType()->isVectorTy(),
-         "Vector GEP must return a vector value", &GEP);
-
+  Assert(GEP.getSourceElementType()->isSized(), "GEP into unsized type!", &GEP);
   SmallVector<Value*, 16> Idxs(GEP.idx_begin(), GEP.idx_end());
   Type *ElTy =
       GetElementPtrInst::getIndexedType(GEP.getSourceElementType(), Idxs);
   Assert(ElTy, "Invalid indices for GEP pointer type!", &GEP);
 
   Assert(GEP.getType()->getScalarType()->isPointerTy() &&
-             cast<PointerType>(GEP.getType()->getScalarType())
-                     ->getElementType() == ElTy,
+             GEP.getResultElementType() == ElTy,
          "GEP is not of right type for indices!", &GEP, ElTy);
 
-  if (GEP.getPointerOperandType()->isVectorTy()) {
+  if (GEP.getType()->isVectorTy()) {
     // Additional checks for vector GEPs.
-    unsigned GepWidth = GEP.getPointerOperandType()->getVectorNumElements();
-    Assert(GepWidth == GEP.getType()->getVectorNumElements(),
-           "Vector GEP result width doesn't match operand's", &GEP);
+    unsigned GEPWidth = GEP.getType()->getVectorNumElements();
+    if (GEP.getPointerOperandType()->isVectorTy())
+      Assert(GEPWidth == GEP.getPointerOperandType()->getVectorNumElements(),
+             "Vector GEP result width doesn't match operand's", &GEP);
     for (unsigned i = 0, e = Idxs.size(); i != e; ++i) {
       Type *IndexTy = Idxs[i]->getType();
-      Assert(IndexTy->isVectorTy(), "Vector GEP must have vector indices!",
-             &GEP);
-      unsigned IndexWidth = IndexTy->getVectorNumElements();
-      Assert(IndexWidth == GepWidth, "Invalid GEP index vector width", &GEP);
+      if (IndexTy->isVectorTy()) {
+        unsigned IndexWidth = IndexTy->getVectorNumElements();
+        Assert(IndexWidth == GEPWidth, "Invalid GEP index vector width", &GEP);
+      }
+      Assert(IndexTy->getScalarType()->isIntegerTy(),
+             "All GEP indices should be of integer type");
     }
   }
   visitInstruction(GEP);
@@ -2598,7 +2683,7 @@ void Verifier::visitAllocaInst(AllocaInst &AI) {
   Assert(PTy->getAddressSpace() == 0,
          "Allocation instruction pointer not in the generic address space!",
          &AI);
-  Assert(PTy->getElementType()->isSized(&Visited),
+  Assert(AI.getAllocatedType()->isSized(&Visited),
          "Cannot allocate unsized type", &AI);
   Assert(AI.getArraySize()->getType()->isIntegerTy(),
          "Alloca array size must have integer type", &AI);
@@ -2710,22 +2795,24 @@ void Verifier::visitLandingPadInst(LandingPadInst &LPI) {
            &LPI);
   }
 
+  if (!LandingPadResultTy)
+    LandingPadResultTy = LPI.getType();
+  else
+    Assert(LandingPadResultTy == LPI.getType(),
+           "The landingpad instruction should have a consistent result type "
+           "inside a function.",
+           &LPI);
+
+  Function *F = LPI.getParent()->getParent();
+  Assert(F->hasPersonalityFn(),
+         "LandingPadInst needs to be in a function with a personality.", &LPI);
+
   // The landingpad instruction must be the first non-PHI instruction in the
   // block.
   Assert(LPI.getParent()->getLandingPadInst() == &LPI,
          "LandingPadInst not the first non-PHI instruction in the block.",
          &LPI);
 
-  // The personality functions for all landingpad instructions within the same
-  // function should match.
-  if (PersonalityFn)
-    Assert(LPI.getPersonalityFn() == PersonalityFn,
-           "Personality function doesn't match others in function", &LPI);
-  PersonalityFn = LPI.getPersonalityFn();
-
-  // All operands must be constants.
-  Assert(isa<Constant>(PersonalityFn), "Personality function is not constant!",
-         &LPI);
   for (unsigned i = 0, e = LPI.getNumClauses(); i < e; ++i) {
     Constant *Clause = LPI.getClause(i);
     if (LPI.isCatch(i)) {
@@ -2741,6 +2828,132 @@ void Verifier::visitLandingPadInst(LandingPadInst &LPI) {
   visitInstruction(LPI);
 }
 
+void Verifier::visitCatchPadInst(CatchPadInst &CPI) {
+  BasicBlock *BB = CPI.getParent();
+
+  if (!CatchPadResultTy)
+    CatchPadResultTy = CPI.getType();
+  else
+    Assert(CatchPadResultTy == CPI.getType(),
+           "The catchpad instruction should have a consistent result type "
+           "inside a function.",
+           &CPI);
+
+  Function *F = BB->getParent();
+  Assert(F->hasPersonalityFn(),
+         "CatchPadInst needs to be in a function with a personality.", &CPI);
+
+  // The catchpad instruction must be the first non-PHI instruction in the
+  // block.
+  Assert(BB->getFirstNonPHI() == &CPI,
+         "CatchPadInst not the first non-PHI instruction in the block.",
+         &CPI);
+
+  BasicBlock *UnwindDest = CPI.getUnwindDest();
+  Instruction *I = UnwindDest->getFirstNonPHI();
+  Assert(
+      isa<CatchPadInst>(I) || isa<CatchEndPadInst>(I),
+      "CatchPadInst must unwind to a CatchPadInst or a CatchEndPadInst.",
+      &CPI);
+
+  visitTerminatorInst(CPI);
+}
+
+void Verifier::visitCatchEndPadInst(CatchEndPadInst &CEPI) {
+  BasicBlock *BB = CEPI.getParent();
+
+  Function *F = BB->getParent();
+  Assert(F->hasPersonalityFn(),
+         "CatchEndPadInst needs to be in a function with a personality.",
+         &CEPI);
+
+  // The catchendpad instruction must be the first non-PHI instruction in the
+  // block.
+  Assert(BB->getFirstNonPHI() == &CEPI,
+         "CatchEndPadInst not the first non-PHI instruction in the block.",
+         &CEPI);
+
+  unsigned CatchPadsSeen = 0;
+  for (BasicBlock *PredBB : predecessors(BB))
+    if (isa<CatchPadInst>(PredBB->getTerminator()))
+      ++CatchPadsSeen;
+
+  Assert(CatchPadsSeen <= 1, "CatchEndPadInst must have no more than one "
+                               "CatchPadInst predecessor.",
+         &CEPI);
+
+  if (BasicBlock *UnwindDest = CEPI.getUnwindDest()) {
+    Instruction *I = UnwindDest->getFirstNonPHI();
+    Assert(
+        I->isEHPad() && !isa<LandingPadInst>(I),
+        "CatchEndPad must unwind to an EH block which is not a landingpad.",
+        &CEPI);
+  }
+
+  visitTerminatorInst(CEPI);
+}
+
+void Verifier::visitCleanupPadInst(CleanupPadInst &CPI) {
+  BasicBlock *BB = CPI.getParent();
+
+  if (!CleanupPadResultTy)
+    CleanupPadResultTy = CPI.getType();
+  else
+    Assert(CleanupPadResultTy == CPI.getType(),
+           "The cleanuppad instruction should have a consistent result type "
+           "inside a function.",
+           &CPI);
+
+  Function *F = BB->getParent();
+  Assert(F->hasPersonalityFn(),
+         "CleanupPadInst needs to be in a function with a personality.", &CPI);
+
+  // The cleanuppad instruction must be the first non-PHI instruction in the
+  // block.
+  Assert(BB->getFirstNonPHI() == &CPI,
+         "CleanupPadInst not the first non-PHI instruction in the block.",
+         &CPI);
+
+  visitInstruction(CPI);
+}
+
+void Verifier::visitCleanupReturnInst(CleanupReturnInst &CRI) {
+  if (BasicBlock *UnwindDest = CRI.getUnwindDest()) {
+    Instruction *I = UnwindDest->getFirstNonPHI();
+    Assert(I->isEHPad() && !isa<LandingPadInst>(I),
+           "CleanupReturnInst must unwind to an EH block which is not a "
+           "landingpad.",
+           &CRI);
+  }
+
+  visitTerminatorInst(CRI);
+}
+
+void Verifier::visitTerminatePadInst(TerminatePadInst &TPI) {
+  BasicBlock *BB = TPI.getParent();
+
+  Function *F = BB->getParent();
+  Assert(F->hasPersonalityFn(),
+         "TerminatePadInst needs to be in a function with a personality.",
+         &TPI);
+
+  // The terminatepad instruction must be the first non-PHI instruction in the
+  // block.
+  Assert(BB->getFirstNonPHI() == &TPI,
+         "TerminatePadInst not the first non-PHI instruction in the block.",
+         &TPI);
+
+  if (BasicBlock *UnwindDest = TPI.getUnwindDest()) {
+    Instruction *I = UnwindDest->getFirstNonPHI();
+    Assert(I->isEHPad() && !isa<LandingPadInst>(I),
+           "TerminatePadInst must unwind to an EH block which is not a "
+           "landingpad.",
+           &TPI);
+  }
+
+  visitTerminatorInst(TPI);
+}
+
 void Verifier::verifyDominatesUse(Instruction &I, unsigned i) {
   Instruction *Op = cast<Instruction>(I.getOperand(i));
   // If the we have an invalid invoke, don't try to compute the dominance.
@@ -2893,7 +3106,7 @@ void Verifier::visitInstruction(Instruction &I) {
   }
 
   if (MDNode *N = I.getDebugLoc().getAsMDNode()) {
-    Assert(isa<MDLocation>(N), "invalid !dbg metadata attachment", &I, N);
+    Assert(isa<DILocation>(N), "invalid !dbg metadata attachment", &I, N);
     visitMDNode(*N);
   }
 
@@ -3036,8 +3249,8 @@ bool Verifier::VerifyIntrinsicType(Type *Ty,
       dyn_cast<PointerType>(ThisArgVecTy->getVectorElementType());
     if (!ThisArgEltTy)
       return true;
-    return (!(ThisArgEltTy->getElementType() ==
-            ReferenceType->getVectorElementType()));
+    return ThisArgEltTy->getElementType() !=
+           ReferenceType->getVectorElementType();
   }
   }
   llvm_unreachable("unhandled");
@@ -3070,10 +3283,9 @@ Verifier::VerifyIntrinsicIsVarArg(bool isVarArg,
   return true;
 }
 
-/// visitIntrinsicFunction - Allow intrinsics to be verified in different ways.
-///
-void Verifier::visitIntrinsicFunctionCall(Intrinsic::ID ID, CallInst &CI) {
-  Function *IF = CI.getCalledFunction();
+/// Allow intrinsics to be verified in different ways.
+void Verifier::visitIntrinsicCallSite(Intrinsic::ID ID, CallSite CS) {
+  Function *IF = CS.getCalledFunction();
   Assert(IF->isDeclaration(), "Intrinsic functions should never be defined!",
          IF);
 
@@ -3117,41 +3329,41 @@ void Verifier::visitIntrinsicFunctionCall(Intrinsic::ID ID, CallInst &CI) {
 
   // If the intrinsic takes MDNode arguments, verify that they are either global
   // or are local to *this* function.
-  for (unsigned i = 0, e = CI.getNumArgOperands(); i != e; ++i)
-    if (auto *MD = dyn_cast<MetadataAsValue>(CI.getArgOperand(i)))
-      visitMetadataAsValue(*MD, CI.getParent()->getParent());
+  for (Value *V : CS.args()) 
+    if (auto *MD = dyn_cast<MetadataAsValue>(V))
+      visitMetadataAsValue(*MD, CS.getCaller());
 
   switch (ID) {
   default:
     break;
   case Intrinsic::ctlz:  // llvm.ctlz
   case Intrinsic::cttz:  // llvm.cttz
-    Assert(isa<ConstantInt>(CI.getArgOperand(1)),
+    Assert(isa<ConstantInt>(CS.getArgOperand(1)),
            "is_zero_undef argument of bit counting intrinsics must be a "
            "constant int",
-           &CI);
+           CS);
     break;
   case Intrinsic::dbg_declare: // llvm.dbg.declare
-    Assert(isa<MetadataAsValue>(CI.getArgOperand(0)),
-           "invalid llvm.dbg.declare intrinsic call 1", &CI);
-    visitDbgIntrinsic("declare", cast<DbgDeclareInst>(CI));
+    Assert(isa<MetadataAsValue>(CS.getArgOperand(0)),
+           "invalid llvm.dbg.declare intrinsic call 1", CS);
+    visitDbgIntrinsic("declare", cast<DbgDeclareInst>(*CS.getInstruction()));
     break;
   case Intrinsic::dbg_value: // llvm.dbg.value
-    visitDbgIntrinsic("value", cast<DbgValueInst>(CI));
+    visitDbgIntrinsic("value", cast<DbgValueInst>(*CS.getInstruction()));
     break;
   case Intrinsic::memcpy:
   case Intrinsic::memmove:
   case Intrinsic::memset: {
-    ConstantInt *AlignCI = dyn_cast<ConstantInt>(CI.getArgOperand(3));
+    ConstantInt *AlignCI = dyn_cast<ConstantInt>(CS.getArgOperand(3));
     Assert(AlignCI,
            "alignment argument of memory intrinsics must be a constant int",
-           &CI);
+           CS);
     const APInt &AlignVal = AlignCI->getValue();
     Assert(AlignCI->isZero() || AlignVal.isPowerOf2(),
-           "alignment argument of memory intrinsics must be a power of 2", &CI);
-    Assert(isa<ConstantInt>(CI.getArgOperand(4)),
+           "alignment argument of memory intrinsics must be a power of 2", CS);
+    Assert(isa<ConstantInt>(CS.getArgOperand(4)),
            "isvolatile argument of memory intrinsics must be a constant int",
-           &CI);
+           CS);
     break;
   }
   case Intrinsic::gcroot:
@@ -3159,76 +3371,76 @@ void Verifier::visitIntrinsicFunctionCall(Intrinsic::ID ID, CallInst &CI) {
   case Intrinsic::gcread:
     if (ID == Intrinsic::gcroot) {
       AllocaInst *AI =
-        dyn_cast<AllocaInst>(CI.getArgOperand(0)->stripPointerCasts());
-      Assert(AI, "llvm.gcroot parameter #1 must be an alloca.", &CI);
-      Assert(isa<Constant>(CI.getArgOperand(1)),
-             "llvm.gcroot parameter #2 must be a constant.", &CI);
-      if (!AI->getType()->getElementType()->isPointerTy()) {
-        Assert(!isa<ConstantPointerNull>(CI.getArgOperand(1)),
+        dyn_cast<AllocaInst>(CS.getArgOperand(0)->stripPointerCasts());
+      Assert(AI, "llvm.gcroot parameter #1 must be an alloca.", CS);
+      Assert(isa<Constant>(CS.getArgOperand(1)),
+             "llvm.gcroot parameter #2 must be a constant.", CS);
+      if (!AI->getAllocatedType()->isPointerTy()) {
+        Assert(!isa<ConstantPointerNull>(CS.getArgOperand(1)),
                "llvm.gcroot parameter #1 must either be a pointer alloca, "
                "or argument #2 must be a non-null constant.",
-               &CI);
+               CS);
       }
     }
 
-    Assert(CI.getParent()->getParent()->hasGC(),
-           "Enclosing function does not use GC.", &CI);
+    Assert(CS.getParent()->getParent()->hasGC(),
+           "Enclosing function does not use GC.", CS);
     break;
   case Intrinsic::init_trampoline:
-    Assert(isa<Function>(CI.getArgOperand(1)->stripPointerCasts()),
+    Assert(isa<Function>(CS.getArgOperand(1)->stripPointerCasts()),
            "llvm.init_trampoline parameter #2 must resolve to a function.",
-           &CI);
+           CS);
     break;
   case Intrinsic::prefetch:
-    Assert(isa<ConstantInt>(CI.getArgOperand(1)) &&
-               isa<ConstantInt>(CI.getArgOperand(2)) &&
-               cast<ConstantInt>(CI.getArgOperand(1))->getZExtValue() < 2 &&
-               cast<ConstantInt>(CI.getArgOperand(2))->getZExtValue() < 4,
-           "invalid arguments to llvm.prefetch", &CI);
+    Assert(isa<ConstantInt>(CS.getArgOperand(1)) &&
+               isa<ConstantInt>(CS.getArgOperand(2)) &&
+               cast<ConstantInt>(CS.getArgOperand(1))->getZExtValue() < 2 &&
+               cast<ConstantInt>(CS.getArgOperand(2))->getZExtValue() < 4,
+           "invalid arguments to llvm.prefetch", CS);
     break;
   case Intrinsic::stackprotector:
-    Assert(isa<AllocaInst>(CI.getArgOperand(1)->stripPointerCasts()),
-           "llvm.stackprotector parameter #2 must resolve to an alloca.", &CI);
+    Assert(isa<AllocaInst>(CS.getArgOperand(1)->stripPointerCasts()),
+           "llvm.stackprotector parameter #2 must resolve to an alloca.", CS);
     break;
   case Intrinsic::lifetime_start:
   case Intrinsic::lifetime_end:
   case Intrinsic::invariant_start:
-    Assert(isa<ConstantInt>(CI.getArgOperand(0)),
+    Assert(isa<ConstantInt>(CS.getArgOperand(0)),
            "size argument of memory use markers must be a constant integer",
-           &CI);
+           CS);
     break;
   case Intrinsic::invariant_end:
-    Assert(isa<ConstantInt>(CI.getArgOperand(1)),
-           "llvm.invariant.end parameter #2 must be a constant integer", &CI);
+    Assert(isa<ConstantInt>(CS.getArgOperand(1)),
+           "llvm.invariant.end parameter #2 must be a constant integer", CS);
     break;
 
-  case Intrinsic::frameescape: {
-    BasicBlock *BB = CI.getParent();
+  case Intrinsic::localescape: {
+    BasicBlock *BB = CS.getParent();
     Assert(BB == &BB->getParent()->front(),
-           "llvm.frameescape used outside of entry block", &CI);
+           "llvm.localescape used outside of entry block", CS);
     Assert(!SawFrameEscape,
-           "multiple calls to llvm.frameescape in one function", &CI);
-    for (Value *Arg : CI.arg_operands()) {
+           "multiple calls to llvm.localescape in one function", CS);
+    for (Value *Arg : CS.args()) {
       if (isa<ConstantPointerNull>(Arg))
         continue; // Null values are allowed as placeholders.
       auto *AI = dyn_cast<AllocaInst>(Arg->stripPointerCasts());
       Assert(AI && AI->isStaticAlloca(),
-             "llvm.frameescape only accepts static allocas", &CI);
+             "llvm.localescape only accepts static allocas", CS);
     }
-    FrameEscapeInfo[BB->getParent()].first = CI.getNumArgOperands();
+    FrameEscapeInfo[BB->getParent()].first = CS.getNumArgOperands();
     SawFrameEscape = true;
     break;
   }
-  case Intrinsic::framerecover: {
-    Value *FnArg = CI.getArgOperand(0)->stripPointerCasts();
+  case Intrinsic::localrecover: {
+    Value *FnArg = CS.getArgOperand(0)->stripPointerCasts();
     Function *Fn = dyn_cast<Function>(FnArg);
     Assert(Fn && !Fn->isDeclaration(),
-           "llvm.framerecover first "
+           "llvm.localrecover first "
            "argument must be function defined in this module",
-           &CI);
-    auto *IdxArg = dyn_cast<ConstantInt>(CI.getArgOperand(2));
-    Assert(IdxArg, "idx argument of llvm.framerecover must be a constant int",
-           &CI);
+           CS);
+    auto *IdxArg = dyn_cast<ConstantInt>(CS.getArgOperand(2));
+    Assert(IdxArg, "idx argument of llvm.localrecover must be a constant int",
+           CS);
     auto &Entry = FrameEscapeInfo[Fn];
     Entry.second = unsigned(
         std::max(uint64_t(Entry.second), IdxArg->getLimitedValue(~0U) + 1));
@@ -3236,123 +3448,138 @@ void Verifier::visitIntrinsicFunctionCall(Intrinsic::ID ID, CallInst &CI) {
   }
 
   case Intrinsic::experimental_gc_statepoint:
-    Assert(!CI.isInlineAsm(),
-           "gc.statepoint support for inline assembly unimplemented", &CI);
-    Assert(CI.getParent()->getParent()->hasGC(),
-           "Enclosing function does not use GC.", &CI);
+    Assert(!CS.isInlineAsm(),
+           "gc.statepoint support for inline assembly unimplemented", CS);
+    Assert(CS.getParent()->getParent()->hasGC(),
+           "Enclosing function does not use GC.", CS);
 
-    VerifyStatepoint(ImmutableCallSite(&CI));
+    VerifyStatepoint(CS);
     break;
   case Intrinsic::experimental_gc_result_int:
   case Intrinsic::experimental_gc_result_float:
   case Intrinsic::experimental_gc_result_ptr:
   case Intrinsic::experimental_gc_result: {
-    Assert(CI.getParent()->getParent()->hasGC(),
-           "Enclosing function does not use GC.", &CI);
+    Assert(CS.getParent()->getParent()->hasGC(),
+           "Enclosing function does not use GC.", CS);
     // Are we tied to a statepoint properly?
-    CallSite StatepointCS(CI.getArgOperand(0));
+    CallSite StatepointCS(CS.getArgOperand(0));
     const Function *StatepointFn =
       StatepointCS.getInstruction() ? StatepointCS.getCalledFunction() : nullptr;
     Assert(StatepointFn && StatepointFn->isDeclaration() &&
                StatepointFn->getIntrinsicID() ==
                    Intrinsic::experimental_gc_statepoint,
-           "gc.result operand #1 must be from a statepoint", &CI,
-           CI.getArgOperand(0));
+           "gc.result operand #1 must be from a statepoint", CS,
+           CS.getArgOperand(0));
 
     // Assert that result type matches wrapped callee.
-    const Value *Target = StatepointCS.getArgument(0);
+    const Value *Target = StatepointCS.getArgument(2);
     const PointerType *PT = cast<PointerType>(Target->getType());
     const FunctionType *TargetFuncType =
       cast<FunctionType>(PT->getElementType());
-    Assert(CI.getType() == TargetFuncType->getReturnType(),
-           "gc.result result type does not match wrapped callee", &CI);
+    Assert(CS.getType() == TargetFuncType->getReturnType(),
+           "gc.result result type does not match wrapped callee", CS);
     break;
   }
   case Intrinsic::experimental_gc_relocate: {
-    Assert(CI.getNumArgOperands() == 3, "wrong number of arguments", &CI);
+    Assert(CS.getNumArgOperands() == 3, "wrong number of arguments", CS);
 
     // Check that this relocate is correctly tied to the statepoint
 
     // This is case for relocate on the unwinding path of an invoke statepoint
     if (ExtractValueInst *ExtractValue =
-          dyn_cast<ExtractValueInst>(CI.getArgOperand(0))) {
+          dyn_cast<ExtractValueInst>(CS.getArgOperand(0))) {
       Assert(isa<LandingPadInst>(ExtractValue->getAggregateOperand()),
              "gc relocate on unwind path incorrectly linked to the statepoint",
-             &CI);
+             CS);
 
-      const BasicBlock *invokeBB =
+      const BasicBlock *InvokeBB =
         ExtractValue->getParent()->getUniquePredecessor();
 
       // Landingpad relocates should have only one predecessor with invoke
       // statepoint terminator
-      Assert(invokeBB, "safepoints should have unique landingpads",
+      Assert(InvokeBB, "safepoints should have unique landingpads",
              ExtractValue->getParent());
-      Assert(invokeBB->getTerminator(), "safepoint block should be well formed",
-             invokeBB);
-      Assert(isStatepoint(invokeBB->getTerminator()),
-             "gc relocate should be linked to a statepoint", invokeBB);
+      Assert(InvokeBB->getTerminator(), "safepoint block should be well formed",
+             InvokeBB);
+      Assert(isStatepoint(InvokeBB->getTerminator()),
+             "gc relocate should be linked to a statepoint", InvokeBB);
     }
     else {
       // In all other cases relocate should be tied to the statepoint directly.
       // This covers relocates on a normal return path of invoke statepoint and
       // relocates of a call statepoint
-      auto Token = CI.getArgOperand(0);
+      auto Token = CS.getArgOperand(0);
       Assert(isa<Instruction>(Token) && isStatepoint(cast<Instruction>(Token)),
-             "gc relocate is incorrectly tied to the statepoint", &CI, Token);
+             "gc relocate is incorrectly tied to the statepoint", CS, Token);
     }
 
     // Verify rest of the relocate arguments
 
-    GCRelocateOperands ops(&CI);
-    ImmutableCallSite StatepointCS(ops.statepoint());
+    GCRelocateOperands Ops(CS);
+    ImmutableCallSite StatepointCS(Ops.getStatepoint());
 
     // Both the base and derived must be piped through the safepoint
-    Value* Base = CI.getArgOperand(1);
+    Value* Base = CS.getArgOperand(1);
     Assert(isa<ConstantInt>(Base),
-           "gc.relocate operand #2 must be integer offset", &CI);
+           "gc.relocate operand #2 must be integer offset", CS);
 
-    Value* Derived = CI.getArgOperand(2);
+    Value* Derived = CS.getArgOperand(2);
     Assert(isa<ConstantInt>(Derived),
-           "gc.relocate operand #3 must be integer offset", &CI);
+           "gc.relocate operand #3 must be integer offset", CS);
 
     const int BaseIndex = cast<ConstantInt>(Base)->getZExtValue();
     const int DerivedIndex = cast<ConstantInt>(Derived)->getZExtValue();
     // Check the bounds
     Assert(0 <= BaseIndex && BaseIndex < (int)StatepointCS.arg_size(),
-           "gc.relocate: statepoint base index out of bounds", &CI);
+           "gc.relocate: statepoint base index out of bounds", CS);
     Assert(0 <= DerivedIndex && DerivedIndex < (int)StatepointCS.arg_size(),
-           "gc.relocate: statepoint derived index out of bounds", &CI);
+           "gc.relocate: statepoint derived index out of bounds", CS);
 
     // Check that BaseIndex and DerivedIndex fall within the 'gc parameters'
     // section of the statepoint's argument
     Assert(StatepointCS.arg_size() > 0,
            "gc.statepoint: insufficient arguments");
-    Assert(isa<ConstantInt>(StatepointCS.getArgument(1)),
+    Assert(isa<ConstantInt>(StatepointCS.getArgument(3)),
            "gc.statement: number of call arguments must be constant integer");
     const unsigned NumCallArgs =
-      cast<ConstantInt>(StatepointCS.getArgument(1))->getZExtValue();
-    Assert(StatepointCS.arg_size() > NumCallArgs+3,
+        cast<ConstantInt>(StatepointCS.getArgument(3))->getZExtValue();
+    Assert(StatepointCS.arg_size() > NumCallArgs + 5,
            "gc.statepoint: mismatch in number of call arguments");
-    Assert(isa<ConstantInt>(StatepointCS.getArgument(NumCallArgs+3)),
+    Assert(isa<ConstantInt>(StatepointCS.getArgument(NumCallArgs + 5)),
+           "gc.statepoint: number of transition arguments must be "
+           "a constant integer");
+    const int NumTransitionArgs =
+        cast<ConstantInt>(StatepointCS.getArgument(NumCallArgs + 5))
+            ->getZExtValue();
+    const int DeoptArgsStart = 4 + NumCallArgs + 1 + NumTransitionArgs + 1;
+    Assert(isa<ConstantInt>(StatepointCS.getArgument(DeoptArgsStart)),
            "gc.statepoint: number of deoptimization arguments must be "
            "a constant integer");
     const int NumDeoptArgs =
-      cast<ConstantInt>(StatepointCS.getArgument(NumCallArgs + 3))->getZExtValue();
-    const int GCParamArgsStart = NumCallArgs + NumDeoptArgs + 4;
+      cast<ConstantInt>(StatepointCS.getArgument(DeoptArgsStart))->getZExtValue();
+    const int GCParamArgsStart = DeoptArgsStart + 1 + NumDeoptArgs;
     const int GCParamArgsEnd = StatepointCS.arg_size();
     Assert(GCParamArgsStart <= BaseIndex && BaseIndex < GCParamArgsEnd,
            "gc.relocate: statepoint base index doesn't fall within the "
            "'gc parameters' section of the statepoint call",
-           &CI);
+           CS);
     Assert(GCParamArgsStart <= DerivedIndex && DerivedIndex < GCParamArgsEnd,
            "gc.relocate: statepoint derived index doesn't fall within the "
            "'gc parameters' section of the statepoint call",
-           &CI);
-
-    // Assert that the result type matches the type of the relocated pointer
-    GCRelocateOperands Operands(&CI);
-    Assert(Operands.derivedPtr()->getType() == CI.getType(),
-           "gc.relocate: relocating a pointer shouldn't change its type", &CI);
+           CS);
+
+    // Relocated value must be a pointer type, but gc_relocate does not need to return the
+    // same pointer type as the relocated pointer. It can be casted to the correct type later
+    // if it's desired. However, they must have the same address space.
+    GCRelocateOperands Operands(CS);
+    Assert(Operands.getDerivedPtr()->getType()->isPointerTy(),
+           "gc.relocate: relocated value must be a gc pointer", CS);
+
+    // gc_relocate return type must be a pointer type, and is verified earlier in
+    // VerifyIntrinsicType().
+    Assert(cast<PointerType>(CS.getType())->getAddressSpace() ==
+           cast<PointerType>(Operands.getDerivedPtr()->getType())->getAddressSpace(),
+           "gc.relocate: relocating a pointer shouldn't change its address space", CS);
     break;
   }
   };
@@ -3362,18 +3589,18 @@ void Verifier::visitIntrinsicFunctionCall(Intrinsic::ID ID, CallInst &CI) {
 ///
 /// This carefully grabs the subprogram from a local scope, avoiding the
 /// built-in assertions that would typically fire.
-static MDSubprogram *getSubprogram(Metadata *LocalScope) {
+static DISubprogram *getSubprogram(Metadata *LocalScope) {
   if (!LocalScope)
     return nullptr;
 
-  if (auto *SP = dyn_cast<MDSubprogram>(LocalScope))
+  if (auto *SP = dyn_cast<DISubprogram>(LocalScope))
     return SP;
 
-  if (auto *LB = dyn_cast<MDLexicalBlockBase>(LocalScope))
+  if (auto *LB = dyn_cast<DILexicalBlockBase>(LocalScope))
     return getSubprogram(LB->getRawScope());
 
   // Just return null; broken scope chains are checked elsewhere.
-  assert(!isa<MDLocalScope>(LocalScope) && "Unknown type of local scope");
+  assert(!isa<DILocalScope>(LocalScope) && "Unknown type of local scope");
   return nullptr;
 }
 
@@ -3383,29 +3610,29 @@ void Verifier::visitDbgIntrinsic(StringRef Kind, DbgIntrinsicTy &DII) {
   Assert(isa<ValueAsMetadata>(MD) ||
              (isa<MDNode>(MD) && !cast<MDNode>(MD)->getNumOperands()),
          "invalid llvm.dbg." + Kind + " intrinsic address/value", &DII, MD);
-  Assert(isa<MDLocalVariable>(DII.getRawVariable()),
+  Assert(isa<DILocalVariable>(DII.getRawVariable()),
          "invalid llvm.dbg." + Kind + " intrinsic variable", &DII,
          DII.getRawVariable());
-  Assert(isa<MDExpression>(DII.getRawExpression()),
+  Assert(isa<DIExpression>(DII.getRawExpression()),
          "invalid llvm.dbg." + Kind + " intrinsic expression", &DII,
          DII.getRawExpression());
 
   // Ignore broken !dbg attachments; they're checked elsewhere.
   if (MDNode *N = DII.getDebugLoc().getAsMDNode())
-    if (!isa<MDLocation>(N))
+    if (!isa<DILocation>(N))
       return;
 
   BasicBlock *BB = DII.getParent();
   Function *F = BB ? BB->getParent() : nullptr;
 
   // The scopes for variables and !dbg attachments must agree.
-  MDLocalVariable *Var = DII.getVariable();
-  MDLocation *Loc = DII.getDebugLoc();
+  DILocalVariable *Var = DII.getVariable();
+  DILocation *Loc = DII.getDebugLoc();
   Assert(Loc, "llvm.dbg." + Kind + " intrinsic requires a !dbg attachment",
          &DII, BB, F);
 
-  MDSubprogram *VarSP = getSubprogram(Var->getRawScope());
-  MDSubprogram *LocSP = getSubprogram(Loc->getRawScope());
+  DISubprogram *VarSP = getSubprogram(Var->getRawScope());
+  DISubprogram *LocSP = getSubprogram(Loc->getRawScope());
   if (!VarSP || !LocSP)
     return; // Broken scope chains are checked elsewhere.
 
@@ -3416,16 +3643,16 @@ void Verifier::visitDbgIntrinsic(StringRef Kind, DbgIntrinsicTy &DII) {
 }
 
 template <class MapTy>
-static uint64_t getVariableSize(const MDLocalVariable &V, const MapTy &Map) {
+static uint64_t getVariableSize(const DILocalVariable &V, const MapTy &Map) {
   // Be careful of broken types (checked elsewhere).
   const Metadata *RawType = V.getRawType();
   while (RawType) {
     // Try to get the size directly.
-    if (auto *T = dyn_cast<MDType>(RawType))
+    if (auto *T = dyn_cast<DIType>(RawType))
       if (uint64_t Size = T->getSizeInBits())
         return Size;
 
-    if (auto *DT = dyn_cast<MDDerivedType>(RawType)) {
+    if (auto *DT = dyn_cast<DIDerivedType>(RawType)) {
       // Look at the base type.
       RawType = DT->getRawBaseType();
       continue;
@@ -3448,15 +3675,15 @@ static uint64_t getVariableSize(const MDLocalVariable &V, const MapTy &Map) {
 template <class MapTy>
 void Verifier::verifyBitPieceExpression(const DbgInfoIntrinsic &I,
                                         const MapTy &TypeRefs) {
-  MDLocalVariable *V;
-  MDExpression *E;
+  DILocalVariable *V;
+  DIExpression *E;
   if (auto *DVI = dyn_cast<DbgValueInst>(&I)) {
-    V = dyn_cast_or_null<MDLocalVariable>(DVI->getRawVariable());
-    E = dyn_cast_or_null<MDExpression>(DVI->getRawExpression());
+    V = dyn_cast_or_null<DILocalVariable>(DVI->getRawVariable());
+    E = dyn_cast_or_null<DIExpression>(DVI->getRawExpression());
   } else {
     auto *DDI = cast<DbgDeclareInst>(&I);
-    V = dyn_cast_or_null<MDLocalVariable>(DDI->getRawVariable());
-    E = dyn_cast_or_null<MDExpression>(DDI->getRawExpression());
+    V = dyn_cast_or_null<DILocalVariable>(DDI->getRawVariable());
+    E = dyn_cast_or_null<DIExpression>(DDI->getRawExpression());
   }
 
   // We don't know whether this intrinsic verified correctly.
@@ -3467,6 +3694,15 @@ void Verifier::verifyBitPieceExpression(const DbgInfoIntrinsic &I,
   if (!E->isBitPiece())
     return;
 
+  // The frontend helps out GDB by emitting the members of local anonymous
+  // unions as artificial local variables with shared storage. When SROA splits
+  // the storage for artificial local variables that are smaller than the entire
+  // union, the overhang piece will be outside of the allotted space for the
+  // variable and this check fails.
+  // FIXME: Remove this check as soon as clang stops doing this; it hides bugs.
+  if (V->isArtificial())
+    return;
+
   // If there's no size, the type is broken, but that should be checked
   // elsewhere.
   uint64_t VarSize = getVariableSize(*V, TypeRefs);
@@ -3492,11 +3728,11 @@ void Verifier::verifyTypeRefs() {
     return;
 
   // Visit all the compile units again to map the type references.
-  SmallDenseMap<const MDString *, const MDType *, 32> TypeRefs;
+  SmallDenseMap<const MDString *, const DIType *, 32> TypeRefs;
   for (auto *CU : CUs->operands())
-    if (auto Ts = cast<MDCompileUnit>(CU)->getRetainedTypes())
-      for (MDType *Op : Ts)
-        if (auto *T = dyn_cast<MDCompositeType>(Op))
+    if (auto Ts = cast<DICompileUnit>(CU)->getRetainedTypes())
+      for (DIType *Op : Ts)
+        if (auto *T = dyn_cast<DICompositeType>(Op))
           if (auto *S = T->getRawIdentifier()) {
             UnresolvedTypeRefs.erase(S);
             TypeRefs.insert(std::make_pair(S, T));
diff --git a/lib/IRReader/CMakeLists.txt b/lib/IRReader/CMakeLists.txt
index 2c0e61b65fb..87ea88039ef 100644
--- a/lib/IRReader/CMakeLists.txt
+++ b/lib/IRReader/CMakeLists.txt
@@ -3,4 +3,7 @@ add_llvm_library(LLVMIRReader
 
   ADDITIONAL_HEADER_DIRS
   ${LLVM_MAIN_INCLUDE_DIR}/llvm/IRReader
+
+  DEPENDS
+  intrinsics_gen
   )
diff --git a/lib/IRReader/IRReader.cpp b/lib/IRReader/IRReader.cpp
index fe62715317c..43fee65db7f 100644
--- a/lib/IRReader/IRReader.cpp
+++ b/lib/IRReader/IRReader.cpp
@@ -11,7 +11,6 @@
 #include "llvm-c/Core.h"
 #include "llvm-c/IRReader.h"
 #include "llvm/AsmParser/Parser.h"
-#include "llvm/Bitcode/NaCl/NaClReaderWriter.h"
 #include "llvm/Bitcode/ReaderWriter.h"
 #include "llvm/IR/LLVMContext.h"
 #include "llvm/IR/Module.h"
@@ -35,14 +34,14 @@ getLazyIRModule(std::unique_ptr<MemoryBuffer> Buffer, SMDiagnostic &Err,
                 LLVMContext &Context) {
   if (isBitcode((const unsigned char *)Buffer->getBufferStart(),
                 (const unsigned char *)Buffer->getBufferEnd())) {
-    ErrorOr<Module *> ModuleOrErr =
+    ErrorOr<std::unique_ptr<Module>> ModuleOrErr =
         getLazyBitcodeModule(std::move(Buffer), Context);
     if (std::error_code EC = ModuleOrErr.getError()) {
       Err = SMDiagnostic(Buffer->getBufferIdentifier(), SourceMgr::DK_Error,
                          EC.message());
       return nullptr;
     }
-    return std::unique_ptr<Module>(ModuleOrErr.get());
+    return std::move(ModuleOrErr.get());
   }
 
   return parseAssembly(Buffer->getMemBufferRef(), Err, Context);
@@ -68,13 +67,14 @@ std::unique_ptr<Module> llvm::parseIR(MemoryBufferRef Buffer, SMDiagnostic &Err,
                      TimePassesIsEnabled);
   if (isBitcode((const unsigned char *)Buffer.getBufferStart(),
                 (const unsigned char *)Buffer.getBufferEnd())) {
-    ErrorOr<Module *> ModuleOrErr = parseBitcodeFile(Buffer, Context);
+    ErrorOr<std::unique_ptr<Module>> ModuleOrErr =
+        parseBitcodeFile(Buffer, Context);
     if (std::error_code EC = ModuleOrErr.getError()) {
       Err = SMDiagnostic(Buffer.getBufferIdentifier(), SourceMgr::DK_Error,
                          EC.message());
       return nullptr;
     }
-    return std::unique_ptr<Module>(ModuleOrErr.get());
+    return std::move(ModuleOrErr.get());
   }
 
   return parseAssembly(Buffer, Err, Context);
@@ -93,77 +93,6 @@ std::unique_ptr<Module> llvm::parseIRFile(StringRef Filename, SMDiagnostic &Err,
   return parseIR(FileOrErr.get()->getMemBufferRef(), Err, Context);
 }
 
-// @LOCALMOD-BEGIN
-static NaClFileFormat NaClDoAutodetectFileFormat(
-    NaClFileFormat Format, const char unsigned *StartBuffer,
-    const char unsigned *EndBuffer) {
-  if (Format != AutodetectFileFormat)
-    return Format;
-  if (isNaClBitcode(StartBuffer, EndBuffer))
-    return PNaClFormat;
-  return LLVMFormat;
-}
-
-std::unique_ptr<Module> llvm::NaClParseIR(MemoryBufferRef Buffer,
-                                          NaClFileFormat Format,
-                                          SMDiagnostic &Err,
-                                          raw_ostream *Verbose,
-                                          LLVMContext &Context) {
-  NamedRegionTimer T(TimeIRParsingName, TimeIRParsingGroupName,
-                     TimePassesIsEnabled);
-  Format = NaClDoAutodetectFileFormat(
-      Format, (const unsigned char *)Buffer.getBufferStart(),
-      (const unsigned char *)Buffer.getBufferEnd());
-  if ((Format == PNaClFormat) &&
-      isNaClBitcode((const unsigned char *)Buffer.getBufferStart(),
-                    (const unsigned char *)Buffer.getBufferEnd())) {
-    ErrorOr<Module *> ModuleOrErr =
-        NaClParseBitcodeFile(Buffer, Context, Verbose);
-    if (std::error_code EC = ModuleOrErr.getError()) {
-      Err = SMDiagnostic(Buffer.getBufferIdentifier(), SourceMgr::DK_Error,
-                         EC.message());
-      return nullptr;
-    }
-    return std::unique_ptr<Module>(ModuleOrErr.get());
-  } else if (Format == LLVMFormat) {
-    if (isBitcode((const unsigned char *)Buffer.getBufferStart(),
-                  (const unsigned char *)Buffer.getBufferEnd())) {
-      ErrorOr<Module *> ModuleOrErr = parseBitcodeFile(Buffer, Context);
-      if (std::error_code EC = ModuleOrErr.getError()) {
-        Err = SMDiagnostic(Buffer.getBufferIdentifier(), SourceMgr::DK_Error,
-                           EC.message());
-        return nullptr;
-      }
-      return std::unique_ptr<Module>(ModuleOrErr.get());
-    }
-
-    return parseAssembly(Buffer, Err, Context);
-  } else {
-    Err = SMDiagnostic(Buffer.getBufferIdentifier(), SourceMgr::DK_Error,
-                       "Did not specify correct format for file");
-    return nullptr;
-  }
-}
-
-std::unique_ptr<Module> llvm::NaClParseIRFile(StringRef Filename,
-                                              NaClFileFormat Format,
-                                              SMDiagnostic &Err,
-                                              raw_ostream *Verbose,
-                                              LLVMContext &Context) {
-  ErrorOr<std::unique_ptr<MemoryBuffer>> FileOrErr =
-      MemoryBuffer::getFileOrSTDIN(Filename);
-  if (std::error_code EC = FileOrErr.getError()) {
-    Err = SMDiagnostic(Filename, SourceMgr::DK_Error,
-                       "Could not open input file: " + EC.message());
-    return nullptr;
-  }
-
-  return NaClParseIR(FileOrErr.get()->getMemBufferRef(), Format, Err, Verbose,
-                     Context);
-}
-
-// @LOCALMOD-END
-
 //===----------------------------------------------------------------------===//
 // C API.
 //===----------------------------------------------------------------------===//
diff --git a/lib/IRReader/LLVMBuild.txt b/lib/IRReader/LLVMBuild.txt
index 2fea2e1dc4a..b7bc74d6164 100644
--- a/lib/IRReader/LLVMBuild.txt
+++ b/lib/IRReader/LLVMBuild.txt
@@ -1,4 +1,4 @@
-;===- ./lib/IRReader/LLVMBuild.txt -----------------------*- Conf -*--===;
+;===- ./lib/IRReader/LLVMBuild.txt -----------------------------*- Conf -*--===;
 ;
 ;                     The LLVM Compiler Infrastructure
 ;
@@ -19,4 +19,4 @@
 type = Library
 name = IRReader
 parent = Libraries
-required_libraries = AsmParser BitReader NaClBitReader Core Support
+required_libraries = AsmParser BitReader Core Support
diff --git a/lib/LLVMBuild.txt b/lib/LLVMBuild.txt
index bc2448da0ed..2edb66ae3ae 100644
--- a/lib/LLVMBuild.txt
+++ b/lib/LLVMBuild.txt
@@ -16,7 +16,28 @@
 ;===------------------------------------------------------------------------===;
 
 [common]
-subdirectories = Analysis AsmParser Bitcode CodeGen DebugInfo ExecutionEngine LineEditor Linker IR IRReader LTO MC Object Option Passes ProfileData Support TableGen Target Transforms
+subdirectories =
+ Analysis
+ AsmParser
+ Bitcode
+ CodeGen
+ DebugInfo
+ ExecutionEngine
+ LibDriver
+ LineEditor
+ Linker
+ IR
+ IRReader
+ LTO
+ MC
+ Object
+ Option
+ Passes
+ ProfileData
+ Support
+ TableGen
+ Target
+ Transforms
 
 [component_0]
 type = Group
diff --git a/lib/LTO/LLVMBuild.txt b/lib/LTO/LLVMBuild.txt
index dfd424f3e40..ea79d5e6a83 100644
--- a/lib/LTO/LLVMBuild.txt
+++ b/lib/LTO/LLVMBuild.txt
@@ -19,4 +19,19 @@
 type = Library
 name = LTO
 parent = Libraries
-required_libraries = Analysis BitReader BitWriter CodeGen Core IPA IPO InstCombine Linker MC ObjCARC Object Scalar Support Target
+required_libraries =
+ Analysis
+ BitReader
+ BitWriter
+ CodeGen
+ Core
+ IPA
+ IPO
+ InstCombine
+ Linker
+ MC
+ ObjCARC
+ Object
+ Scalar
+ Support
+ Target
diff --git a/lib/LTO/LTOCodeGenerator.cpp b/lib/LTO/LTOCodeGenerator.cpp
index 058e0e89381..daaf2232922 100644
--- a/lib/LTO/LTOCodeGenerator.cpp
+++ b/lib/LTO/LTOCodeGenerator.cpp
@@ -65,24 +65,12 @@ const char* LTOCodeGenerator::getVersionString() {
 
 LTOCodeGenerator::LTOCodeGenerator()
     : Context(getGlobalContext()), IRLinker(new Module("ld-temp.o", Context)) {
-  initialize();
+  initializeLTOPasses();
 }
 
 LTOCodeGenerator::LTOCodeGenerator(std::unique_ptr<LLVMContext> Context)
     : OwnedContext(std::move(Context)), Context(*OwnedContext),
-      IRLinker(new Module("ld-temp.o", *OwnedContext)), OptLevel(2) {
-  initialize();
-}
-
-void LTOCodeGenerator::initialize() {
-  TargetMach = nullptr;
-  EmitDwarfDebugInfo = false;
-  ScopeRestrictionsDone = false;
-  CodeModel = LTO_CODEGEN_PIC_MODEL_DEFAULT;
-  DiagHandler = nullptr;
-  DiagContext = nullptr;
-  OwnedModule = nullptr;
-
+      IRLinker(new Module("ld-temp.o", *OwnedContext)) {
   initializeLTOPasses();
 }
 
@@ -214,8 +202,7 @@ bool LTOCodeGenerator::writeMergedModules(const char *path,
   }
 
   // write bitcode to it
-  WriteBitcodeToFile(IRLinker.getModule(), Out.os(),
-                     /* ShouldPreserveUseListOrder */ true);
+  WriteBitcodeToFile(IRLinker.getModule(), Out.os(), ShouldEmbedUselists);
   Out.os().close();
 
   if (Out.os().has_error()) {
@@ -263,8 +250,8 @@ bool LTOCodeGenerator::compileOptimizedToFile(const char **name,
   return true;
 }
 
-const void *LTOCodeGenerator::compileOptimized(size_t *length,
-                                               std::string &errMsg) {
+std::unique_ptr<MemoryBuffer>
+LTOCodeGenerator::compileOptimized(std::string &errMsg) {
   const char *name;
   if (!compileOptimizedToFile(&name, errMsg))
     return nullptr;
@@ -277,16 +264,11 @@ const void *LTOCodeGenerator::compileOptimized(size_t *length,
     sys::fs::remove(NativeObjectPath);
     return nullptr;
   }
-  NativeObjectFile = std::move(*BufferOrErr);
 
   // remove temp files
   sys::fs::remove(NativeObjectPath);
 
-  // return buffer, unless error
-  if (!NativeObjectFile)
-    return nullptr;
-  *length = NativeObjectFile->getBufferSize();
-  return NativeObjectFile->getBufferStart();
+  return std::move(*BufferOrErr);
 }
 
 
@@ -302,16 +284,14 @@ bool LTOCodeGenerator::compile_to_file(const char **name,
   return compileOptimizedToFile(name, errMsg);
 }
 
-const void* LTOCodeGenerator::compile(size_t *length,
-                                      bool disableInline,
-                                      bool disableGVNLoadPRE,
-                                      bool disableVectorization,
-                                      std::string &errMsg) {
+std::unique_ptr<MemoryBuffer>
+LTOCodeGenerator::compile(bool disableInline, bool disableGVNLoadPRE,
+                          bool disableVectorization, std::string &errMsg) {
   if (!optimize(disableInline, disableGVNLoadPRE,
                 disableVectorization, errMsg))
     return nullptr;
 
-  return compileOptimized(length, errMsg);
+  return compileOptimized(errMsg);
 }
 
 bool LTOCodeGenerator::determineTarget(std::string &errMsg) {
@@ -319,15 +299,6 @@ bool LTOCodeGenerator::determineTarget(std::string &errMsg) {
     return true;
 
   std::string TripleStr = IRLinker.getModule()->getTargetTriple();
-
-  // @LOCALMOD-BEGIN
-  // Pretend that we are ARM for name mangling and assembly conventions.
-  // https://code.google.com/p/nativeclient/issues/detail?id=2554
-  if (TripleStr == "le32-unknown-nacl") {
-    TripleStr = "armv7a-none-nacl-gnueabi";
-  }
-  // @LOCALMOD-END
-
   if (TripleStr.empty())
     TripleStr = sys::getDefaultTargetTriple();
   llvm::Triple Triple(TripleStr);
@@ -473,7 +444,7 @@ static void accumulateAndSortLibcalls(std::vector<StringRef> &Libcalls,
 }
 
 void LTOCodeGenerator::applyScopeRestrictions() {
-  if (ScopeRestrictionsDone)
+  if (ScopeRestrictionsDone || !ShouldInternalize)
     return;
   Module *mergedModule = IRLinker.getModule();
 
@@ -482,7 +453,7 @@ void LTOCodeGenerator::applyScopeRestrictions() {
   passes.add(createVerifierPass());
 
   // mark which symbols can not be internalized
-  Mangler Mangler(TargetMach->getDataLayout());
+  Mangler Mangler;
   std::vector<const char*> MustPreserveList;
   SmallPtrSet<GlobalValue*, 8> AsmUsed;
   std::vector<StringRef> Libcalls;
@@ -550,7 +521,7 @@ bool LTOCodeGenerator::optimize(bool DisableInline,
   legacy::PassManager passes;
 
   // Add an appropriate DataLayout instance for this module...
-  mergedModule->setDataLayout(*TargetMach->getDataLayout());
+  mergedModule->setDataLayout(TargetMach->createDataLayout());
 
   passes.add(
       createTargetTransformInfoWrapperPass(TargetMach->getTargetIRAnalysis()));
diff --git a/lib/LTO/LTOModule.cpp b/lib/LTO/LTOModule.cpp
index d41306fbc61..ec8991ed236 100644
--- a/lib/LTO/LTOModule.cpp
+++ b/lib/LTO/LTOModule.cpp
@@ -19,6 +19,7 @@
 #include "llvm/IR/Constants.h"
 #include "llvm/IR/DiagnosticPrinter.h"
 #include "llvm/IR/LLVMContext.h"
+#include "llvm/IR/Mangler.h"
 #include "llvm/IR/Metadata.h"
 #include "llvm/IR/Module.h"
 #include "llvm/MC/MCExpr.h"
@@ -33,7 +34,6 @@
 #include "llvm/Object/IRObjectFile.h"
 #include "llvm/Object/ObjectFile.h"
 #include "llvm/Support/CommandLine.h"
-#include "llvm/Support/ErrorHandling.h" // @LOCALMOD
 #include "llvm/Support/FileSystem.h"
 #include "llvm/Support/Host.h"
 #include "llvm/Support/MemoryBuffer.h"
@@ -148,9 +148,10 @@ LTOModule *LTOModule::createInContext(const void *mem, size_t length,
   return makeLTOModule(Buffer, options, errMsg, Context);
 }
 
-static Module *parseBitcodeFileImpl(MemoryBufferRef Buffer,
-                                    LLVMContext &Context, bool ShouldBeLazy,
-                                    std::string &ErrMsg) {
+static std::unique_ptr<Module> parseBitcodeFileImpl(MemoryBufferRef Buffer,
+                                                    LLVMContext &Context,
+                                                    bool ShouldBeLazy,
+                                                    std::string &ErrMsg) {
 
   // Find the buffer.
   ErrorOr<MemoryBufferRef> MBOrErr =
@@ -169,22 +170,22 @@ static Module *parseBitcodeFileImpl(MemoryBufferRef Buffer,
 
   if (!ShouldBeLazy) {
     // Parse the full file.
-    ErrorOr<Module *> M =
+    ErrorOr<std::unique_ptr<Module>> M =
         parseBitcodeFile(*MBOrErr, Context, DiagnosticHandler);
     if (!M)
       return nullptr;
-    return *M;
+    return std::move(*M);
   }
 
   // Parse lazily.
   std::unique_ptr<MemoryBuffer> LightweightBuf =
       MemoryBuffer::getMemBuffer(*MBOrErr, false);
-  ErrorOr<Module *> M = getLazyBitcodeModule(std::move(LightweightBuf), Context,
-                                             DiagnosticHandler,
-                                             true/*ShouldLazyLoadMetadata*/);
+  ErrorOr<std::unique_ptr<Module>> M =
+      getLazyBitcodeModule(std::move(LightweightBuf), Context,
+                           DiagnosticHandler, true /*ShouldLazyLoadMetadata*/);
   if (!M)
     return nullptr;
-  return *M;
+  return std::move(*M);
 }
 
 LTOModule *LTOModule::makeLTOModule(MemoryBufferRef Buffer,
@@ -198,20 +199,13 @@ LTOModule *LTOModule::makeLTOModule(MemoryBufferRef Buffer,
 
   // If we own a context, we know this is being used only for symbol
   // extraction, not linking.  Be lazy in that case.
-  std::unique_ptr<Module> M(parseBitcodeFileImpl(
+  std::unique_ptr<Module> M = parseBitcodeFileImpl(
       Buffer, *Context,
-      /* ShouldBeLazy */ static_cast<bool>(OwnedContext), errMsg));
+      /* ShouldBeLazy */ static_cast<bool>(OwnedContext), errMsg);
   if (!M)
     return nullptr;
 
   std::string TripleStr = M->getTargetTriple();
-  // @LOCALMOD-BEGIN
-  // Pretend that we are ARM for name mangling and assembly conventions.
-  // https://code.google.com/p/nativeclient/issues/detail?id=2554
-  if (TripleStr == "le32-unknown-nacl") {
-    TripleStr = "armv7a-none-nacl-gnueabi";
-  }
-  // @LOCALMOD-END
   if (TripleStr.empty())
     TripleStr = sys::getDefaultTargetTriple();
   llvm::Triple Triple(TripleStr);
@@ -238,7 +232,7 @@ LTOModule *LTOModule::makeLTOModule(MemoryBufferRef Buffer,
 
   TargetMachine *target = march->createTargetMachine(TripleStr, CPU, FeatureStr,
                                                      options);
-  M->setDataLayout(*target->getDataLayout());
+  M->setDataLayout(target->createDataLayout());
 
   std::unique_ptr<object::IRObjectFile> IRObj(
       new object::IRObjectFile(Buffer, std::move(M)));
@@ -476,6 +470,12 @@ void LTOModule::addDefinedSymbol(const char *Name, const GlobalValue *def,
   else
     attr |= LTO_SYMBOL_SCOPE_DEFAULT;
 
+  if (def->hasComdat())
+    attr |= LTO_SYMBOL_COMDAT;
+
+  if (isa<GlobalAlias>(def))
+    attr |= LTO_SYMBOL_ALIAS;
+
   auto Iter = _defines.insert(Name).first;
 
   // fill information structure
@@ -613,14 +613,6 @@ bool LTOModule::parseSymbols(std::string &errMsg) {
 
     auto *F = dyn_cast<Function>(GV);
     if (IsUndefined) {
-      // @LOCALMOD-BEGIN
-      // Bitcode modules may have declarations for functions or globals
-      // which are unused. Ignore them here so that gold does not mistake
-      // them for undefined symbols.
-      if (GV->use_empty())
-        continue;
-      // @LOCALMOD-END
-
       addPotentialUndefinedSymbol(Sym, F != nullptr);
       continue;
     }
@@ -654,6 +646,8 @@ bool LTOModule::parseSymbols(std::string &errMsg) {
 
 /// parseMetadata - Parse metadata from the module
 void LTOModule::parseMetadata() {
+  raw_string_ostream OS(LinkerOpts);
+
   // Linker Options
   if (Metadata *Val = getModule().getModuleFlag("Linker Options")) {
     MDNode *LinkerOptions = cast<MDNode>(Val);
@@ -661,20 +655,19 @@ void LTOModule::parseMetadata() {
       MDNode *MDOptions = cast<MDNode>(LinkerOptions->getOperand(i));
       for (unsigned ii = 0, ie = MDOptions->getNumOperands(); ii != ie; ++ii) {
         MDString *MDOption = cast<MDString>(MDOptions->getOperand(ii));
-        // FIXME: Make StringSet::insert match Self-Associative Container
-        // requirements, returning <iter,bool> rather than bool, and use that
-        // here.
-        StringRef Op =
-            _linkeropt_strings.insert(MDOption->getString()).first->first();
-        StringRef DepLibName =
-            _target->getObjFileLowering()->getDepLibFromLinkerOpt(Op);
-        if (!DepLibName.empty())
-          _deplibs.push_back(DepLibName.data());
-        else if (!Op.empty())
-          _linkeropts.push_back(Op.data());
+        OS << " " << MDOption->getString();
       }
     }
   }
 
+  // Globals
+  Mangler Mang;
+  for (const NameAndAttributes &Sym : _symbols) {
+    if (!Sym.symbol)
+      continue;
+    _target->getObjFileLowering()->emitLinkerFlagsForGlobal(OS, Sym.symbol,
+                                                            Mang);
+  }
+
   // Add other interesting metadata here.
 }
diff --git a/lib/LibDriver/CMakeLists.txt b/lib/LibDriver/CMakeLists.txt
new file mode 100644
index 00000000000..ab53a684344
--- /dev/null
+++ b/lib/LibDriver/CMakeLists.txt
@@ -0,0 +1,8 @@
+set(LLVM_TARGET_DEFINITIONS Options.td)
+tablegen(LLVM Options.inc -gen-opt-parser-defs)
+add_public_tablegen_target(LibOptionsTableGen)
+
+add_llvm_library(LLVMLibDriver
+  LibDriver.cpp
+  )
+add_dependencies(LLVMLibDriver LibOptionsTableGen)
diff --git a/lib/LibDriver/LLVMBuild.txt b/lib/LibDriver/LLVMBuild.txt
new file mode 100644
index 00000000000..799dc997c0b
--- /dev/null
+++ b/lib/LibDriver/LLVMBuild.txt
@@ -0,0 +1,22 @@
+;===- ./lib/LibDriver/LLVMBuild.txt ----------------------------*- Conf -*--===;
+;
+;                     The LLVM Compiler Infrastructure
+;
+; This file is distributed under the University of Illinois Open Source
+; License. See LICENSE.TXT for details.
+;
+;===------------------------------------------------------------------------===;
+;
+; This is an LLVMBuild description file for the components in this subdirectory.
+;
+; For more information on the LLVMBuild system, please see:
+;
+;   http://llvm.org/docs/LLVMBuild.html
+;
+;===------------------------------------------------------------------------===;
+
+[component_0]
+type = Library
+name = LibDriver
+parent = Libraries
+required_libraries = Object Option Support
diff --git a/lib/LibDriver/LibDriver.cpp b/lib/LibDriver/LibDriver.cpp
new file mode 100644
index 00000000000..b0e704d6bad
--- /dev/null
+++ b/lib/LibDriver/LibDriver.cpp
@@ -0,0 +1,154 @@
+//===- LibDriver.cpp - lib.exe-compatible driver --------------------------===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// Defines an interface to a lib.exe-compatible driver that also understands
+// bitcode files. Used by llvm-lib and lld-link2 /lib.
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/LibDriver/LibDriver.h"
+#include "llvm/ADT/STLExtras.h"
+#include "llvm/Object/ArchiveWriter.h"
+#include "llvm/Option/Arg.h"
+#include "llvm/Option/ArgList.h"
+#include "llvm/Option/Option.h"
+#include "llvm/Support/CommandLine.h"
+#include "llvm/Support/StringSaver.h"
+#include "llvm/Support/Path.h"
+#include "llvm/Support/Process.h"
+#include "llvm/Support/raw_ostream.h"
+
+using namespace llvm;
+
+namespace {
+
+enum {
+  OPT_INVALID = 0,
+#define OPTION(_1, _2, ID, _4, _5, _6, _7, _8, _9, _10, _11) OPT_##ID,
+#include "Options.inc"
+#undef OPTION
+};
+
+#define PREFIX(NAME, VALUE) const char *const NAME[] = VALUE;
+#include "Options.inc"
+#undef PREFIX
+
+static const llvm::opt::OptTable::Info infoTable[] = {
+#define OPTION(X1, X2, ID, KIND, GROUP, ALIAS, X6, X7, X8, X9, X10)    \
+  {                                                                    \
+    X1, X2, X9, X10, OPT_##ID, llvm::opt::Option::KIND##Class, X8, X7, \
+    OPT_##GROUP, OPT_##ALIAS, X6                                       \
+  },
+#include "Options.inc"
+#undef OPTION
+};
+
+class LibOptTable : public llvm::opt::OptTable {
+public:
+  LibOptTable() : OptTable(infoTable, llvm::array_lengthof(infoTable), true) {}
+};
+
+}
+
+static std::string getOutputPath(llvm::opt::InputArgList *Args,
+                                 const llvm::NewArchiveIterator &FirstMember) {
+  if (auto *Arg = Args->getLastArg(OPT_out))
+    return Arg->getValue();
+  SmallString<128> Val = FirstMember.getNew();
+  llvm::sys::path::replace_extension(Val, ".lib");
+  return Val.str();
+}
+
+static std::vector<StringRef> getSearchPaths(llvm::opt::InputArgList *Args,
+                                             StringSaver &Saver) {
+  std::vector<StringRef> Ret;
+  // Add current directory as first item of the search path.
+  Ret.push_back("");
+
+  // Add /libpath flags.
+  for (auto *Arg : Args->filtered(OPT_libpath))
+    Ret.push_back(Arg->getValue());
+
+  // Add $LIB.
+  Optional<std::string> EnvOpt = sys::Process::GetEnv("LIB");
+  if (!EnvOpt.hasValue())
+    return Ret;
+  StringRef Env = Saver.save(*EnvOpt);
+  while (!Env.empty()) {
+    StringRef Path;
+    std::tie(Path, Env) = Env.split(';');
+    Ret.push_back(Path);
+  }
+  return Ret;
+}
+
+static Optional<std::string> findInputFile(StringRef File,
+                                           ArrayRef<StringRef> Paths) {
+  for (auto Dir : Paths) {
+    SmallString<128> Path = Dir;
+    sys::path::append(Path, File);
+    if (sys::fs::exists(Path))
+      return Path.str().str();
+  }
+  return Optional<std::string>();
+}
+
+int llvm::libDriverMain(llvm::ArrayRef<const char*> ArgsArr) {
+  SmallVector<const char *, 20> NewArgs(ArgsArr.begin(), ArgsArr.end());
+  BumpPtrAllocator Alloc;
+  BumpPtrStringSaver Saver(Alloc);
+  cl::ExpandResponseFiles(Saver, cl::TokenizeWindowsCommandLine, NewArgs);
+  ArgsArr = NewArgs;
+
+  LibOptTable Table;
+  unsigned MissingIndex;
+  unsigned MissingCount;
+  llvm::opt::InputArgList Args =
+      Table.ParseArgs(ArgsArr.slice(1), MissingIndex, MissingCount);
+  if (MissingCount) {
+    llvm::errs() << "missing arg value for \""
+                 << Args.getArgString(MissingIndex) << "\", expected "
+                 << MissingCount
+                 << (MissingCount == 1 ? " argument.\n" : " arguments.\n");
+    return 1;
+  }
+  for (auto *Arg : Args.filtered(OPT_UNKNOWN))
+    llvm::errs() << "ignoring unknown argument: " << Arg->getSpelling() << "\n";
+
+  if (Args.filtered_begin(OPT_INPUT) == Args.filtered_end()) {
+    llvm::errs() << "no input files.\n";
+    return 1;
+  }
+
+  std::vector<StringRef> SearchPaths = getSearchPaths(&Args, Saver);
+
+  std::vector<llvm::NewArchiveIterator> Members;
+  for (auto *Arg : Args.filtered(OPT_INPUT)) {
+    Optional<std::string> Path = findInputFile(Arg->getValue(), SearchPaths);
+    if (!Path.hasValue()) {
+      llvm::errs() << Arg->getValue() << ": no such file or directory\n";
+      return 1;
+    }
+    Members.emplace_back(Saver.save(*Path));
+  }
+
+  std::pair<StringRef, std::error_code> Result =
+      llvm::writeArchive(getOutputPath(&Args, Members[0]), Members,
+                         /*WriteSymtab=*/true, object::Archive::K_GNU,
+                         /*Deterministic*/ true, Args.hasArg(OPT_llvmlibthin));
+
+  if (Result.second) {
+    if (Result.first.empty())
+      Result.first = ArgsArr[0];
+    llvm::errs() << Result.first << ": " << Result.second.message() << "\n";
+    return 1;
+  }
+
+  return 0;
+}
diff --git a/lib/LibDriver/Makefile b/lib/LibDriver/Makefile
new file mode 100644
index 00000000000..1c62eac9093
--- /dev/null
+++ b/lib/LibDriver/Makefile
@@ -0,0 +1,20 @@
+##===- lib/LibDriver/Makefile ------------------------------*- Makefile -*-===##
+#
+#                     The LLVM Compiler Infrastructure
+#
+# This file is distributed under the University of Illinois Open Source
+# License. See LICENSE.TXT for details.
+#
+##===----------------------------------------------------------------------===##
+
+LEVEL = ../..
+LIBRARYNAME = LLVMLibDriver
+BUILD_ARCHIVE := 1
+BUILT_SOURCES = Options.inc
+TABLEGEN_INC_FILES_COMMON = 1
+
+include $(LEVEL)/Makefile.common
+
+$(ObjDir)/Options.inc.tmp : Options.td $(LLVM_TBLGEN) $(ObjDir)/.dir
+	$(Echo) "Building lib Driver Option tables with tblgen"
+	$(Verb) $(LLVMTableGen) -gen-opt-parser-defs -o $(call SYSPATH, $@) $<
diff --git a/lib/LibDriver/Options.td b/lib/LibDriver/Options.td
new file mode 100644
index 00000000000..5a56ef7468d
--- /dev/null
+++ b/lib/LibDriver/Options.td
@@ -0,0 +1,25 @@
+include "llvm/Option/OptParser.td"
+
+// lib.exe accepts options starting with either a dash or a slash.
+
+// Flag that takes no arguments.
+class F<string name> : Flag<["/", "-", "-?"], name>;
+
+// Flag that takes one argument after ":".
+class P<string name, string help> :
+      Joined<["/", "-", "-?"], name#":">, HelpText<help>;
+
+def libpath: P<"libpath", "Object file search path">;
+def out    : P<"out", "Path to file to write output">;
+
+def llvmlibthin : F<"llvmlibthin">;
+
+//==============================================================================
+// The flags below do nothing. They are defined only for lib.exe compatibility.
+//==============================================================================
+
+class QF<string name> : Joined<["/", "-", "-?"], name#":">;
+
+def ignore : QF<"ignore">;
+def machine: QF<"machine">;
+def nologo : F<"nologo">;
diff --git a/lib/Linker/CMakeLists.txt b/lib/Linker/CMakeLists.txt
index 5a1f31a97ee..f9d8e0925ae 100644
--- a/lib/Linker/CMakeLists.txt
+++ b/lib/Linker/CMakeLists.txt
@@ -3,4 +3,7 @@ add_llvm_library(LLVMLinker
 
   ADDITIONAL_HEADER_DIRS
   ${LLVM_MAIN_INCLUDE_DIR}/llvm/Linker
+
+  DEPENDS
+  intrinsics_gen
   )
diff --git a/lib/Linker/LinkModules.cpp b/lib/Linker/LinkModules.cpp
index 03ab9fbd83c..f0906809ee4 100644
--- a/lib/Linker/LinkModules.cpp
+++ b/lib/Linker/LinkModules.cpp
@@ -424,12 +424,17 @@ class ModuleLinker {
 
   DiagnosticHandlerFunction DiagnosticHandler;
 
+  /// For symbol clashes, prefer those from Src.
+  bool OverrideFromSrc;
+
 public:
   ModuleLinker(Module *dstM, Linker::IdentifiedStructTypeSet &Set, Module *srcM,
-               DiagnosticHandlerFunction DiagnosticHandler)
+               DiagnosticHandlerFunction DiagnosticHandler,
+               bool OverrideFromSrc)
       : DstM(dstM), SrcM(srcM), TypeMap(Set),
         ValMaterializer(TypeMap, DstM, LazilyLinkGlobalValues),
-        DiagnosticHandler(DiagnosticHandler) {}
+        DiagnosticHandler(DiagnosticHandler), OverrideFromSrc(OverrideFromSrc) {
+  }
 
   bool run();
 
@@ -577,8 +582,7 @@ static GlobalAlias *copyGlobalAliasProto(TypeMapTy &TypeMap, Module &DstM,
   // If there is no linkage to be performed or we're linking from the source,
   // bring over SGA.
   auto *PTy = cast<PointerType>(TypeMap.get(SGA->getType()));
-  return GlobalAlias::create(PTy->getElementType(), PTy->getAddressSpace(),
-                             SGA->getLinkage(), SGA->getName(), &DstM);
+  return GlobalAlias::create(PTy, SGA->getLinkage(), SGA->getName(), &DstM);
 }
 
 static GlobalValue *copyGlobalValueProto(TypeMapTy &TypeMap, Module &DstM,
@@ -725,6 +729,12 @@ bool ModuleLinker::getComdatResult(const Comdat *SrcC,
 bool ModuleLinker::shouldLinkFromSource(bool &LinkFromSrc,
                                         const GlobalValue &Dest,
                                         const GlobalValue &Src) {
+  // Should we unconditionally use the Src?
+  if (OverrideFromSrc) {
+    LinkFromSrc = true;
+    return false;
+  }
+
   // We always have to add Src if it has appending linkage.
   if (Src.hasAppendingLinkage()) {
     LinkFromSrc = true;
@@ -1071,8 +1081,9 @@ bool ModuleLinker::linkGlobalValueProto(GlobalValue *SGV) {
   } else {
     // If the GV is to be lazily linked, don't create it just yet.
     // The ValueMaterializerTy will deal with creating it if it's used.
-    if (!DGV && (SGV->hasLocalLinkage() || SGV->hasLinkOnceLinkage() ||
-                 SGV->hasAvailableExternallyLinkage())) {
+    if (!DGV && !OverrideFromSrc &&
+        (SGV->hasLocalLinkage() || SGV->hasLinkOnceLinkage() ||
+         SGV->hasAvailableExternallyLinkage())) {
       DoNotLinkFromSource.insert(SGV);
       return false;
     }
@@ -1183,6 +1194,11 @@ bool ModuleLinker::linkFunctionBody(Function &Dst, Function &Src) {
     Dst.setPrologueData(MapValue(Src.getPrologueData(), ValueMap, RF_None,
                                  &TypeMap, &ValMaterializer));
 
+  // Link in the personality function.
+  if (Src.hasPersonalityFn())
+    Dst.setPersonalityFn(MapValue(Src.getPersonalityFn(), ValueMap, RF_None,
+                                  &TypeMap, &ValMaterializer));
+
   // Go through and convert function arguments over, remembering the mapping.
   Function::arg_iterator DI = Dst.arg_begin();
   for (Argument &Arg : Src.args()) {
@@ -1193,6 +1209,13 @@ bool ModuleLinker::linkFunctionBody(Function &Dst, Function &Src) {
     ++DI;
   }
 
+  // Copy over the metadata attachments.
+  SmallVector<std::pair<unsigned, MDNode *>, 8> MDs;
+  Src.getAllMetadata(MDs);
+  for (const auto &I : MDs)
+    Dst.setMetadata(I.first, MapMetadata(I.second, ValueMap, RF_None, &TypeMap,
+                                         &ValMaterializer));
+
   // Splice the body of the source function into the dest function.
   Dst.getBasicBlockList().splice(Dst.end(), Src.getBasicBlockList());
 
@@ -1209,7 +1232,7 @@ bool ModuleLinker::linkFunctionBody(Function &Dst, Function &Src) {
   for (Argument &Arg : Src.args())
     ValueMap.erase(&Arg);
 
-  Src.Dematerialize();
+  Src.dematerialize();
   return false;
 }
 
@@ -1236,15 +1259,15 @@ bool ModuleLinker::linkGlobalValueBody(GlobalValue &Src) {
 /// Insert all of the named MDNodes in Src into the Dest module.
 void ModuleLinker::linkNamedMDNodes() {
   const NamedMDNode *SrcModFlags = SrcM->getModuleFlagsMetadata();
-  for (Module::const_named_metadata_iterator I = SrcM->named_metadata_begin(),
-       E = SrcM->named_metadata_end(); I != E; ++I) {
+  for (const NamedMDNode &NMD : SrcM->named_metadata()) {
     // Don't link module flags here. Do them separately.
-    if (&*I == SrcModFlags) continue;
-    NamedMDNode *DestNMD = DstM->getOrInsertNamedMetadata(I->getName());
+    if (&NMD == SrcModFlags)
+      continue;
+    NamedMDNode *DestNMD = DstM->getOrInsertNamedMetadata(NMD.getName());
     // Add Src elements into Dest node.
-    for (unsigned i = 0, e = I->getNumOperands(); i != e; ++i)
-      DestNMD->addOperand(MapMetadata(I->getOperand(i), ValueMap, RF_None,
-                                      &TypeMap, &ValMaterializer));
+    for (const MDNode *op : NMD.operands())
+      DestNMD->addOperand(
+          MapMetadata(op, ValueMap, RF_None, &TypeMap, &ValMaterializer));
   }
 }
 
@@ -1269,10 +1292,10 @@ void ModuleLinker::stripReplacedSubprograms() {
   if (!CompileUnits)
     return;
   for (unsigned I = 0, E = CompileUnits->getNumOperands(); I != E; ++I) {
-    DICompileUnit CU = cast<MDCompileUnit>(CompileUnits->getOperand(I));
+    auto *CU = cast<DICompileUnit>(CompileUnits->getOperand(I));
     assert(CU && "Expected valid compile unit");
 
-    for (MDSubprogram *SP : CU->getSubprograms()) {
+    for (DISubprogram *SP : CU->getSubprograms()) {
       if (!SP || !SP->getFunction() || !Functions.count(SP->getFunction()))
         continue;
 
@@ -1524,9 +1547,8 @@ bool ModuleLinker::run() {
 
   // Insert all of the globals in src into the DstM module... without linking
   // initializers (which could refer to functions not yet mapped over).
-  for (Module::global_iterator I = SrcM->global_begin(),
-       E = SrcM->global_end(); I != E; ++I)
-    if (linkGlobalValueProto(I))
+  for (GlobalVariable &GV : SrcM->globals())
+    if (linkGlobalValueProto(&GV))
       return true;
 
   // Link the functions together between the two modules, without doing function
@@ -1534,26 +1556,25 @@ bool ModuleLinker::run() {
   // function...  We do this so that when we begin processing function bodies,
   // all of the global values that may be referenced are available in our
   // ValueMap.
-  for (Module::iterator I = SrcM->begin(), E = SrcM->end(); I != E; ++I)
-    if (linkGlobalValueProto(I))
+  for (Function &F :*SrcM)
+    if (linkGlobalValueProto(&F))
       return true;
 
   // If there were any aliases, link them now.
-  for (Module::alias_iterator I = SrcM->alias_begin(),
-       E = SrcM->alias_end(); I != E; ++I)
-    if (linkGlobalValueProto(I))
+  for (GlobalAlias &GA : SrcM->aliases())
+    if (linkGlobalValueProto(&GA))
       return true;
 
-  for (unsigned i = 0, e = AppendingVars.size(); i != e; ++i)
-    linkAppendingVarInit(AppendingVars[i]);
+  for (const AppendingVarInfo &AppendingVar : AppendingVars)
+    linkAppendingVarInit(AppendingVar);
 
   for (const auto &Entry : DstM->getComdatSymbolTable()) {
     const Comdat &C = Entry.getValue();
     if (C.getSelectionKind() == Comdat::Any)
       continue;
     const GlobalValue *GV = SrcM->getNamedValue(C.getName());
-    assert(GV);
-    MapValue(GV, ValueMap, RF_None, &TypeMap, &ValMaterializer);
+    if (GV)
+      MapValue(GV, ValueMap, RF_None, &TypeMap, &ValMaterializer);
   }
 
   // Strip replaced subprograms before mapping any metadata -- so that we're
@@ -1738,9 +1759,9 @@ void Linker::deleteModule() {
   Composite = nullptr;
 }
 
-bool Linker::linkInModule(Module *Src) {
+bool Linker::linkInModule(Module *Src, bool OverrideSymbols) {
   ModuleLinker TheLinker(Composite, IdentifiedStructTypes, Src,
-                         DiagnosticHandler);
+                         DiagnosticHandler, OverrideSymbols);
   bool RetCode = TheLinker.run();
   Composite->dropTriviallyDeadConstantArrays();
   return RetCode;
@@ -1784,7 +1805,9 @@ LLVMBool LLVMLinkModules(LLVMModuleRef Dest, LLVMModuleRef Src,
   LLVMBool Result = Linker::LinkModules(
       D, unwrap(Src), [&](const DiagnosticInfo &DI) { DI.print(DP); });
 
-  if (OutMessages && Result)
+  if (OutMessages && Result) {
+    Stream.flush();
     *OutMessages = strdup(Message.c_str());
+  }
   return Result;
 }
diff --git a/lib/MC/CMakeLists.txt b/lib/MC/CMakeLists.txt
index 17c0b52f162..6554d6a9e60 100644
--- a/lib/MC/CMakeLists.txt
+++ b/lib/MC/CMakeLists.txt
@@ -12,24 +12,23 @@ add_llvm_library(LLVMMC
   MCCodeGenInfo.cpp
   MCContext.cpp
   MCDwarf.cpp
-  MCELF.cpp
   MCELFObjectTargetWriter.cpp
   MCELFStreamer.cpp
   MCExpr.cpp
   MCInst.cpp
   MCInstPrinter.cpp
   MCInstrAnalysis.cpp
+  MCInstrDesc.cpp
   MCLabel.cpp
   MCLinkerOptimizationHint.cpp
   MCMachOStreamer.cpp
   MCMachObjectTargetWriter.cpp
-  MCNaCl.cpp
-  MCNaClExpander.cpp # LOCALMOD
   MCNullStreamer.cpp
   MCObjectFileInfo.cpp
   MCObjectStreamer.cpp
   MCObjectWriter.cpp
   MCRegisterInfo.cpp
+  MCSchedule.cpp
   MCSection.cpp
   MCSectionCOFF.cpp
   MCSectionELF.cpp
@@ -37,6 +36,7 @@ add_llvm_library(LLVMMC
   MCStreamer.cpp
   MCSubtargetInfo.cpp
   MCSymbol.cpp
+  MCSymbolELF.cpp
   MCSymbolizer.cpp
   MCTargetOptions.cpp
   MCValue.cpp
diff --git a/lib/MC/ConstantPools.cpp b/lib/MC/ConstantPools.cpp
index a239a8f3107..f7649fba6e8 100644
--- a/lib/MC/ConstantPools.cpp
+++ b/lib/MC/ConstantPools.cpp
@@ -37,10 +37,10 @@ void ConstantPool::emitEntries(MCStreamer &Streamer) {
 
 const MCExpr *ConstantPool::addEntry(const MCExpr *Value, MCContext &Context,
                                      unsigned Size) {
-  MCSymbol *CPEntryLabel = Context.CreateTempSymbol();
+  MCSymbol *CPEntryLabel = Context.createTempSymbol();
 
   Entries.push_back(ConstantPoolEntry(CPEntryLabel, Value, Size));
-  return MCSymbolRefExpr::Create(CPEntryLabel, Context);
+  return MCSymbolRefExpr::create(CPEntryLabel, Context);
 }
 
 bool ConstantPool::empty() { return Entries.empty(); }
@@ -48,8 +48,7 @@ bool ConstantPool::empty() { return Entries.empty(); }
 //
 // AssemblerConstantPools implementation
 //
-ConstantPool *
-AssemblerConstantPools::getConstantPool(const MCSection *Section) {
+ConstantPool *AssemblerConstantPools::getConstantPool(MCSection *Section) {
   ConstantPoolMapTy::iterator CP = ConstantPools.find(Section);
   if (CP == ConstantPools.end())
     return nullptr;
@@ -58,11 +57,11 @@ AssemblerConstantPools::getConstantPool(const MCSection *Section) {
 }
 
 ConstantPool &
-AssemblerConstantPools::getOrCreateConstantPool(const MCSection *Section) {
+AssemblerConstantPools::getOrCreateConstantPool(MCSection *Section) {
   return ConstantPools[Section];
 }
 
-static void emitConstantPool(MCStreamer &Streamer, const MCSection *Section,
+static void emitConstantPool(MCStreamer &Streamer, MCSection *Section,
                              ConstantPool &CP) {
   if (!CP.empty()) {
     Streamer.SwitchSection(Section);
@@ -75,7 +74,7 @@ void AssemblerConstantPools::emitAll(MCStreamer &Streamer) {
   for (ConstantPoolMapTy::iterator CPI = ConstantPools.begin(),
                                    CPE = ConstantPools.end();
        CPI != CPE; ++CPI) {
-    const MCSection *Section = CPI->first;
+    MCSection *Section = CPI->first;
     ConstantPool &CP = CPI->second;
 
     emitConstantPool(Streamer, Section, CP);
@@ -83,7 +82,7 @@ void AssemblerConstantPools::emitAll(MCStreamer &Streamer) {
 }
 
 void AssemblerConstantPools::emitForCurrentSection(MCStreamer &Streamer) {
-  const MCSection *Section = Streamer.getCurrentSection().first;
+  MCSection *Section = Streamer.getCurrentSection().first;
   if (ConstantPool *CP = getConstantPool(Section)) {
     emitConstantPool(Streamer, Section, *CP);
   }
@@ -92,7 +91,7 @@ void AssemblerConstantPools::emitForCurrentSection(MCStreamer &Streamer) {
 const MCExpr *AssemblerConstantPools::addEntry(MCStreamer &Streamer,
                                                const MCExpr *Expr,
                                                unsigned Size) {
-  const MCSection *Section = Streamer.getCurrentSection().first;
+  MCSection *Section = Streamer.getCurrentSection().first;
   return getOrCreateConstantPool(Section).addEntry(Expr, Streamer.getContext(),
                                                    Size);
 }
diff --git a/lib/MC/ELFObjectWriter.cpp b/lib/MC/ELFObjectWriter.cpp
index 8cb01c43edd..e925bc272dc 100644
--- a/lib/MC/ELFObjectWriter.cpp
+++ b/lib/MC/ELFObjectWriter.cpp
@@ -21,12 +21,11 @@
 #include "llvm/MC/MCAsmLayout.h"
 #include "llvm/MC/MCAssembler.h"
 #include "llvm/MC/MCContext.h"
-#include "llvm/MC/MCELF.h"
-#include "llvm/MC/MCELFSymbolFlags.h"
 #include "llvm/MC/MCExpr.h"
 #include "llvm/MC/MCFixupKindInfo.h"
 #include "llvm/MC/MCObjectWriter.h"
 #include "llvm/MC/MCSectionELF.h"
+#include "llvm/MC/MCSymbolELF.h"
 #include "llvm/MC/MCValue.h"
 #include "llvm/MC/StringTableBuilder.h"
 #include "llvm/Support/Compression.h"
@@ -41,74 +40,50 @@ using namespace llvm;
 #define DEBUG_TYPE "reloc-info"
 
 namespace {
-class FragmentWriter {
-  bool IsLittleEndian;
-
-public:
-  FragmentWriter(bool IsLittleEndian);
-  template <typename T> void write(MCDataFragment &F, T Val);
-};
 
 typedef DenseMap<const MCSectionELF *, uint32_t> SectionIndexMapTy;
 
+class ELFObjectWriter;
+
 class SymbolTableWriter {
-  MCAssembler &Asm;
-  FragmentWriter &FWriter;
+  ELFObjectWriter &EWriter;
   bool Is64Bit;
-  SectionIndexMapTy &SectionIndexMap;
 
-  // The symbol .symtab fragment we are writting to.
-  MCDataFragment *SymtabF;
-
-  // .symtab_shndx fragment we are writting to.
-  MCDataFragment *ShndxF;
+  // indexes we are going to write to .symtab_shndx.
+  std::vector<uint32_t> ShndxIndexes;
 
   // The numbel of symbols written so far.
   unsigned NumWritten;
 
   void createSymtabShndx();
 
-  template <typename T> void write(MCDataFragment &F, T Value);
+  template <typename T> void write(T Value);
 
 public:
-  SymbolTableWriter(MCAssembler &Asm, FragmentWriter &FWriter, bool Is64Bit,
-                    SectionIndexMapTy &SectionIndexMap,
-                    MCDataFragment *SymtabF);
+  SymbolTableWriter(ELFObjectWriter &EWriter, bool Is64Bit);
 
   void writeSymbol(uint32_t name, uint8_t info, uint64_t value, uint64_t size,
                    uint8_t other, uint32_t shndx, bool Reserved);
+
+  ArrayRef<uint32_t> getShndxIndexes() const { return ShndxIndexes; }
 };
 
 class ELFObjectWriter : public MCObjectWriter {
-  FragmentWriter FWriter;
-
-  protected:
-
     static bool isFixupKindPCRel(const MCAssembler &Asm, unsigned Kind);
-    static bool RelocNeedsGOT(MCSymbolRefExpr::VariantKind Variant);
-    static uint64_t SymbolValue(MCSymbolData &Data, const MCAsmLayout &Layout);
-    static bool isInSymtab(const MCAsmLayout &Layout, const MCSymbolData &Data,
+    static uint64_t SymbolValue(const MCSymbol &Sym, const MCAsmLayout &Layout);
+    static bool isInSymtab(const MCAsmLayout &Layout, const MCSymbolELF &Symbol,
                            bool Used, bool Renamed);
-    static bool isLocal(const MCSymbolData &Data, bool isUsedInReloc);
-    static bool IsELFMetaDataSection(const MCSectionData &SD);
-    static uint64_t DataSectionSize(const MCSectionData &SD);
-    static uint64_t GetSectionAddressSize(const MCAsmLayout &Layout,
-                                          const MCSectionData &SD);
-
-    void writeDataSectionData(MCAssembler &Asm, const MCAsmLayout &Layout,
-                              const MCSectionData &SD);
 
     /// Helper struct for containing some precomputed information on symbols.
     struct ELFSymbolData {
-      MCSymbolData *SymbolData;
-      uint64_t StringIndex;
+      const MCSymbolELF *Symbol;
       uint32_t SectionIndex;
       StringRef Name;
 
       // Support lexicographic sorting.
       bool operator<(const ELFSymbolData &RHS) const {
-        unsigned LHSType = MCELF::GetType(*SymbolData);
-        unsigned RHSType = MCELF::GetType(*RHS.SymbolData);
+        unsigned LHSType = Symbol->getType();
+        unsigned RHSType = RHS.Symbol->getType();
         if (LHSType == ELF::STT_SECTION && RHSType != ELF::STT_SECTION)
           return false;
         if (LHSType != ELF::STT_SECTION && RHSType == ELF::STT_SECTION)
@@ -122,28 +97,19 @@ class ELFObjectWriter : public MCObjectWriter {
     /// The target specific ELF writer instance.
     std::unique_ptr<MCELFObjectTargetWriter> TargetObjectWriter;
 
-    SmallPtrSet<const MCSymbol *, 16> UsedInReloc;
-    SmallPtrSet<const MCSymbol *, 16> WeakrefUsedInReloc;
-    DenseMap<const MCSymbol *, const MCSymbol *> Renames;
+    DenseMap<const MCSymbolELF *, const MCSymbolELF *> Renames;
 
-    llvm::DenseMap<const MCSectionData *, std::vector<ELFRelocationEntry>>
-    Relocations;
-    StringTableBuilder ShStrTabBuilder;
+    llvm::DenseMap<const MCSectionELF *, std::vector<ELFRelocationEntry>>
+        Relocations;
 
     /// @}
     /// @name Symbol Table Data
     /// @{
 
     StringTableBuilder StrTabBuilder;
-    std::vector<uint64_t> FileSymbolData;
-    std::vector<ELFSymbolData> LocalSymbolData;
-    std::vector<ELFSymbolData> ExternalSymbolData;
-    std::vector<ELFSymbolData> UndefinedSymbolData;
 
     /// @}
 
-    bool NeedsGOT;
-
     // This holds the symbol table index of the last local symbol.
     unsigned LastLocalSymbolIndex;
     // This holds the .strtab section index.
@@ -151,8 +117,9 @@ class ELFObjectWriter : public MCObjectWriter {
     // This holds the .symtab section index.
     unsigned SymbolTableIndex;
 
-    unsigned ShstrtabIndex;
-
+    // Sections in the order they are to be output in the section table.
+    std::vector<const MCSectionELF *> SectionTable;
+    unsigned addToSectionTable(const MCSectionELF *Sec);
 
     // TargetObjectWriter wrappers.
     bool is64Bit() const { return TargetObjectWriter->is64Bit(); }
@@ -164,23 +131,18 @@ class ELFObjectWriter : public MCObjectWriter {
       return TargetObjectWriter->GetRelocType(Target, Fixup, IsPCRel);
     }
 
+    void align(unsigned Alignment);
+
   public:
     ELFObjectWriter(MCELFObjectTargetWriter *MOTW, raw_pwrite_stream &OS,
                     bool IsLittleEndian)
-        : MCObjectWriter(OS, IsLittleEndian), FWriter(IsLittleEndian),
-          TargetObjectWriter(MOTW), NeedsGOT(false) {}
+        : MCObjectWriter(OS, IsLittleEndian), TargetObjectWriter(MOTW) {}
 
     void reset() override {
-      UsedInReloc.clear();
-      WeakrefUsedInReloc.clear();
       Renames.clear();
       Relocations.clear();
-      ShStrTabBuilder.clear();
       StrTabBuilder.clear();
-      FileSymbolData.clear();
-      LocalSymbolData.clear();
-      ExternalSymbolData.clear();
-      UndefinedSymbolData.clear();
+      SectionTable.clear();
       MCObjectWriter::reset();
     }
 
@@ -188,44 +150,39 @@ class ELFObjectWriter : public MCObjectWriter {
 
     void WriteWord(uint64_t W) {
       if (is64Bit())
-        Write64(W);
+        write64(W);
       else
-        Write32(W);
+        write32(W);
     }
 
-    template <typename T> void write(MCDataFragment &F, T Value) {
-      FWriter.write(F, Value);
+    template <typename T> void write(T Val) {
+      if (IsLittleEndian)
+        support::endian::Writer<support::little>(OS).write(Val);
+      else
+        support::endian::Writer<support::big>(OS).write(Val);
     }
 
-    void WriteHeader(const MCAssembler &Asm,
-                     unsigned NumberOfSections);
+    void writeHeader(const MCAssembler &Asm);
 
-    void WriteSymbol(SymbolTableWriter &Writer, ELFSymbolData &MSD,
-                     const MCAsmLayout &Layout);
+    void writeSymbol(SymbolTableWriter &Writer, uint32_t StringIndex,
+                     ELFSymbolData &MSD, const MCAsmLayout &Layout);
 
-    void WriteSymbolTable(MCDataFragment *SymtabF, MCAssembler &Asm,
-                          const MCAsmLayout &Layout,
-                          SectionIndexMapTy &SectionIndexMap);
+    // Start and end offset of each section
+    typedef std::map<const MCSectionELF *, std::pair<uint64_t, uint64_t>>
+        SectionOffsetsTy;
 
     bool shouldRelocateWithSymbol(const MCAssembler &Asm,
                                   const MCSymbolRefExpr *RefA,
-                                  const MCSymbolData *SD, uint64_t C,
+                                  const MCSymbol *Sym, uint64_t C,
                                   unsigned Type) const;
 
-    void RecordRelocation(MCAssembler &Asm, const MCAsmLayout &Layout,
+    void recordRelocation(MCAssembler &Asm, const MCAsmLayout &Layout,
                           const MCFragment *Fragment, const MCFixup &Fixup,
                           MCValue Target, bool &IsPCRel,
                           uint64_t &FixedValue) override;
 
-    uint64_t getSymbolIndexInSymbolTable(const MCAssembler &Asm,
-                                         const MCSymbol *S);
-
-    // Map from a group section to the signature symbol
-    typedef DenseMap<const MCSectionELF*, const MCSymbol*> GroupMapTy;
-    // Map from a signature symbol to the group section
-    typedef DenseMap<const MCSymbol*, const MCSectionELF*> RevGroupMapTy;
-    // Map from a section to its offset
-    typedef DenseMap<const MCSectionELF*, uint64_t> SectionOffsetMapTy;
+    // Map from a signature symbol to the group section index
+    typedef DenseMap<const MCSymbol *, unsigned> RevGroupMapTy;
 
     /// Compute the symbol table data
     ///
@@ -234,105 +191,70 @@ class ELFObjectWriter : public MCObjectWriter {
     /// \param RevGroupMap - Maps a signature symbol to the group section.
     void computeSymbolTable(MCAssembler &Asm, const MCAsmLayout &Layout,
                             const SectionIndexMapTy &SectionIndexMap,
-                            const RevGroupMapTy &RevGroupMap);
+                            const RevGroupMapTy &RevGroupMap,
+                            SectionOffsetsTy &SectionOffsets);
 
-    void computeIndexMap(MCAssembler &Asm, SectionIndexMapTy &SectionIndexMap);
+    MCSectionELF *createRelocationSection(MCContext &Ctx,
+                                          const MCSectionELF &Sec);
 
-    MCSectionData *createRelocationSection(MCAssembler &Asm,
-                                           const MCSectionData &SD);
+    const MCSectionELF *createStringTable(MCContext &Ctx);
 
-    void CompressDebugSections(MCAssembler &Asm, MCAsmLayout &Layout);
+    void executePostLayoutBinding(MCAssembler &Asm,
+                                  const MCAsmLayout &Layout) override;
 
-    void WriteRelocations(MCAssembler &Asm, MCAsmLayout &Layout);
+    void writeSectionHeader(const MCAsmLayout &Layout,
+                            const SectionIndexMapTy &SectionIndexMap,
+                            const SectionOffsetsTy &SectionOffsets);
 
-    void CreateMetadataSections(MCAssembler &Asm, MCAsmLayout &Layout,
-                                SectionIndexMapTy &SectionIndexMap);
+    void writeSectionData(const MCAssembler &Asm, MCSection &Sec,
+                          const MCAsmLayout &Layout);
 
-    // Create the sections that show up in the symbol table. Currently
-    // those are the .note.GNU-stack section and the group sections.
-    void createIndexedSections(MCAssembler &Asm, MCAsmLayout &Layout,
-                               GroupMapTy &GroupMap, RevGroupMapTy &RevGroupMap,
-                               SectionIndexMapTy &SectionIndexMap);
+    void WriteSecHdrEntry(uint32_t Name, uint32_t Type, uint64_t Flags,
+                          uint64_t Address, uint64_t Offset, uint64_t Size,
+                          uint32_t Link, uint32_t Info, uint64_t Alignment,
+                          uint64_t EntrySize);
 
-    void ExecutePostLayoutBinding(MCAssembler &Asm,
-                                  const MCAsmLayout &Layout) override;
+    void writeRelocations(const MCAssembler &Asm, const MCSectionELF &Sec);
 
-    void writeSectionHeader(ArrayRef<const MCSectionELF *> Sections,
-                            MCAssembler &Asm, const GroupMapTy &GroupMap,
-                            const MCAsmLayout &Layout,
-                            const SectionIndexMapTy &SectionIndexMap,
-                            const SectionOffsetMapTy &SectionOffsetMap);
+    bool isSymbolRefDifferenceFullyResolvedImpl(const MCAssembler &Asm,
+                                                const MCSymbol &SymA,
+                                                const MCFragment &FB,
+                                                bool InSet,
+                                                bool IsPCRel) const override;
 
-    void WriteSecHdrEntry(uint32_t Name, uint32_t Type, uint64_t Flags,
-                          uint64_t Address, uint64_t Offset,
-                          uint64_t Size, uint32_t Link, uint32_t Info,
-                          uint64_t Alignment, uint64_t EntrySize);
-
-    void WriteRelocationsFragment(const MCAssembler &Asm,
-                                  MCDataFragment *F,
-                                  const MCSectionData *SD);
-
-    bool
-    IsSymbolRefDifferenceFullyResolvedImpl(const MCAssembler &Asm,
-                                           const MCSymbolData &DataA,
-                                           const MCSymbolData *DataB,
-                                           const MCFragment &FB,
-                                           bool InSet,
-                                           bool IsPCRel) const override;
-
-    bool isWeak(const MCSymbolData &SD) const override;
-
-    void WriteObject(MCAssembler &Asm, const MCAsmLayout &Layout) override;
-    void writeSection(MCAssembler &Asm,
-                      const SectionIndexMapTy &SectionIndexMap,
-                      uint32_t GroupSymbolIndex,
-                      uint64_t Offset, uint64_t Size, uint64_t Alignment,
+    bool isWeak(const MCSymbol &Sym) const override;
+
+    void writeObject(MCAssembler &Asm, const MCAsmLayout &Layout) override;
+    void writeSection(const SectionIndexMapTy &SectionIndexMap,
+                      uint32_t GroupSymbolIndex, uint64_t Offset, uint64_t Size,
                       const MCSectionELF &Section);
   };
 }
 
-FragmentWriter::FragmentWriter(bool IsLittleEndian)
-    : IsLittleEndian(IsLittleEndian) {}
+void ELFObjectWriter::align(unsigned Alignment) {
+  uint64_t Padding = OffsetToAlignment(OS.tell(), Alignment);
+  WriteZeros(Padding);
+}
 
-template <typename T> void FragmentWriter::write(MCDataFragment &F, T Val) {
-  if (IsLittleEndian)
-    Val = support::endian::byte_swap<T, support::little>(Val);
-  else
-    Val = support::endian::byte_swap<T, support::big>(Val);
-  const char *Start = (const char *)&Val;
-  F.getContents().append(Start, Start + sizeof(T));
+unsigned ELFObjectWriter::addToSectionTable(const MCSectionELF *Sec) {
+  SectionTable.push_back(Sec);
+  StrTabBuilder.add(Sec->getSectionName());
+  return SectionTable.size();
 }
 
 void SymbolTableWriter::createSymtabShndx() {
-  if (ShndxF)
+  if (!ShndxIndexes.empty())
     return;
 
-  MCContext &Ctx = Asm.getContext();
-  const MCSectionELF *SymtabShndxSection =
-      Ctx.getELFSection(".symtab_shndxr", ELF::SHT_SYMTAB_SHNDX, 0, 4, "");
-  MCSectionData *SymtabShndxSD =
-      &Asm.getOrCreateSectionData(*SymtabShndxSection);
-  SymtabShndxSD->setAlignment(4);
-  ShndxF = new MCDataFragment(SymtabShndxSD);
-  unsigned Index = SectionIndexMap.size() + 1;
-  SectionIndexMap[SymtabShndxSection] = Index;
-
-  for (unsigned I = 0; I < NumWritten; ++I)
-    write(*ShndxF, uint32_t(0));
+  ShndxIndexes.resize(NumWritten);
 }
 
-template <typename T>
-void SymbolTableWriter::write(MCDataFragment &F, T Value) {
-  FWriter.write(F, Value);
+template <typename T> void SymbolTableWriter::write(T Value) {
+  EWriter.write(Value);
 }
 
-SymbolTableWriter::SymbolTableWriter(MCAssembler &Asm, FragmentWriter &FWriter,
-                                     bool Is64Bit,
-                                     SectionIndexMapTy &SectionIndexMap,
-                                     MCDataFragment *SymtabF)
-    : Asm(Asm), FWriter(FWriter), Is64Bit(Is64Bit),
-      SectionIndexMap(SectionIndexMap), SymtabF(SymtabF), ShndxF(nullptr),
-      NumWritten(0) {}
+SymbolTableWriter::SymbolTableWriter(ELFObjectWriter &EWriter, bool Is64Bit)
+    : EWriter(EWriter), Is64Bit(Is64Bit), NumWritten(0) {}
 
 void SymbolTableWriter::writeSymbol(uint32_t name, uint8_t info, uint64_t value,
                                     uint64_t size, uint8_t other,
@@ -342,29 +264,29 @@ void SymbolTableWriter::writeSymbol(uint32_t name, uint8_t info, uint64_t value,
   if (LargeIndex)
     createSymtabShndx();
 
-  if (ShndxF) {
+  if (!ShndxIndexes.empty()) {
     if (LargeIndex)
-      write(*ShndxF, shndx);
+      ShndxIndexes.push_back(shndx);
     else
-      write(*ShndxF, uint32_t(0));
+      ShndxIndexes.push_back(0);
   }
 
   uint16_t Index = LargeIndex ? uint16_t(ELF::SHN_XINDEX) : shndx;
 
   if (Is64Bit) {
-    write(*SymtabF, name);  // st_name
-    write(*SymtabF, info);  // st_info
-    write(*SymtabF, other); // st_other
-    write(*SymtabF, Index); // st_shndx
-    write(*SymtabF, value); // st_value
-    write(*SymtabF, size);  // st_size
+    write(name);  // st_name
+    write(info);  // st_info
+    write(other); // st_other
+    write(Index); // st_shndx
+    write(value); // st_value
+    write(size);  // st_size
   } else {
-    write(*SymtabF, name);            // st_name
-    write(*SymtabF, uint32_t(value)); // st_value
-    write(*SymtabF, uint32_t(size));  // st_size
-    write(*SymtabF, info);            // st_info
-    write(*SymtabF, other);           // st_other
-    write(*SymtabF, Index);           // st_shndx
+    write(name);            // st_name
+    write(uint32_t(value)); // st_value
+    write(uint32_t(size));  // st_size
+    write(info);            // st_info
+    write(other);           // st_other
+    write(Index);           // st_shndx
   }
 
   ++NumWritten;
@@ -377,33 +299,11 @@ bool ELFObjectWriter::isFixupKindPCRel(const MCAssembler &Asm, unsigned Kind) {
   return FKI.Flags & MCFixupKindInfo::FKF_IsPCRel;
 }
 
-bool ELFObjectWriter::RelocNeedsGOT(MCSymbolRefExpr::VariantKind Variant) {
-  switch (Variant) {
-  default:
-    return false;
-  case MCSymbolRefExpr::VK_GOT:
-  case MCSymbolRefExpr::VK_PLT:
-  case MCSymbolRefExpr::VK_GOTPCREL:
-  case MCSymbolRefExpr::VK_GOTOFF:
-  case MCSymbolRefExpr::VK_TPOFF:
-  case MCSymbolRefExpr::VK_TLSGD:
-  case MCSymbolRefExpr::VK_GOTTPOFF:
-  case MCSymbolRefExpr::VK_INDNTPOFF:
-  case MCSymbolRefExpr::VK_NTPOFF:
-  case MCSymbolRefExpr::VK_GOTNTPOFF:
-  case MCSymbolRefExpr::VK_TLSLDM:
-  case MCSymbolRefExpr::VK_DTPOFF:
-  case MCSymbolRefExpr::VK_TLSLD:
-    return true;
-  }
-}
-
 ELFObjectWriter::~ELFObjectWriter()
 {}
 
 // Emit the ELF header.
-void ELFObjectWriter::WriteHeader(const MCAssembler &Asm,
-                                  unsigned NumberOfSections) {
+void ELFObjectWriter::writeHeader(const MCAssembler &Asm) {
   // ELF Header
   // ----------
   //
@@ -412,88 +312,78 @@ void ELFObjectWriter::WriteHeader(const MCAssembler &Asm,
   // emitWord method behaves differently for ELF32 and ELF64, writing
   // 4 bytes in the former and 8 in the latter.
 
-  Write8(0x7f); // e_ident[EI_MAG0]
-  Write8('E');  // e_ident[EI_MAG1]
-  Write8('L');  // e_ident[EI_MAG2]
-  Write8('F');  // e_ident[EI_MAG3]
+  writeBytes(ELF::ElfMagic); // e_ident[EI_MAG0] to e_ident[EI_MAG3]
 
-  Write8(is64Bit() ? ELF::ELFCLASS64 : ELF::ELFCLASS32); // e_ident[EI_CLASS]
+  write8(is64Bit() ? ELF::ELFCLASS64 : ELF::ELFCLASS32); // e_ident[EI_CLASS]
 
   // e_ident[EI_DATA]
-  Write8(isLittleEndian() ? ELF::ELFDATA2LSB : ELF::ELFDATA2MSB);
+  write8(isLittleEndian() ? ELF::ELFDATA2LSB : ELF::ELFDATA2MSB);
 
-  Write8(ELF::EV_CURRENT);        // e_ident[EI_VERSION]
+  write8(ELF::EV_CURRENT);        // e_ident[EI_VERSION]
   // e_ident[EI_OSABI]
-  Write8(TargetObjectWriter->getOSABI());
-  Write8(0);                  // e_ident[EI_ABIVERSION]
+  write8(TargetObjectWriter->getOSABI());
+  write8(0);                  // e_ident[EI_ABIVERSION]
 
   WriteZeros(ELF::EI_NIDENT - ELF::EI_PAD);
 
-  Write16(ELF::ET_REL);             // e_type
+  write16(ELF::ET_REL);             // e_type
 
-  Write16(TargetObjectWriter->getEMachine()); // e_machine = target
+  write16(TargetObjectWriter->getEMachine()); // e_machine = target
 
-  Write32(ELF::EV_CURRENT);         // e_version
+  write32(ELF::EV_CURRENT);         // e_version
   WriteWord(0);                    // e_entry, no entry point in .o file
   WriteWord(0);                    // e_phoff, no program header for .o
   WriteWord(0);                     // e_shoff = sec hdr table off in bytes
 
   // e_flags = whatever the target wants
-  Write32(Asm.getELFHeaderEFlags());
+  write32(Asm.getELFHeaderEFlags());
 
   // e_ehsize = ELF header size
-  Write16(is64Bit() ? sizeof(ELF::Elf64_Ehdr) : sizeof(ELF::Elf32_Ehdr));
+  write16(is64Bit() ? sizeof(ELF::Elf64_Ehdr) : sizeof(ELF::Elf32_Ehdr));
 
-  Write16(0);                  // e_phentsize = prog header entry size
-  Write16(0);                  // e_phnum = # prog header entries = 0
+  write16(0);                  // e_phentsize = prog header entry size
+  write16(0);                  // e_phnum = # prog header entries = 0
 
   // e_shentsize = Section header entry size
-  Write16(is64Bit() ? sizeof(ELF::Elf64_Shdr) : sizeof(ELF::Elf32_Shdr));
+  write16(is64Bit() ? sizeof(ELF::Elf64_Shdr) : sizeof(ELF::Elf32_Shdr));
 
   // e_shnum     = # of section header ents
-  if (NumberOfSections >= ELF::SHN_LORESERVE)
-    Write16(ELF::SHN_UNDEF);
-  else
-    Write16(NumberOfSections);
+  write16(0);
 
   // e_shstrndx  = Section # of '.shstrtab'
-  if (ShstrtabIndex >= ELF::SHN_LORESERVE)
-    Write16(ELF::SHN_XINDEX);
-  else
-    Write16(ShstrtabIndex);
+  assert(StringTableIndex < ELF::SHN_LORESERVE);
+  write16(StringTableIndex);
 }
 
-uint64_t ELFObjectWriter::SymbolValue(MCSymbolData &Data,
+uint64_t ELFObjectWriter::SymbolValue(const MCSymbol &Sym,
                                       const MCAsmLayout &Layout) {
-  if (Data.isCommon() && Data.isExternal())
-    return Data.getCommonAlignment();
+  if (Sym.isCommon() && Sym.isExternal())
+    return Sym.getCommonAlignment();
 
   uint64_t Res;
-  if (!Layout.getSymbolOffset(&Data, Res))
+  if (!Layout.getSymbolOffset(Sym, Res))
     return 0;
 
-  if (Layout.getAssembler().isThumbFunc(&Data.getSymbol()))
+  if (Layout.getAssembler().isThumbFunc(&Sym))
     Res |= 1;
 
   return Res;
 }
 
-void ELFObjectWriter::ExecutePostLayoutBinding(MCAssembler &Asm,
+void ELFObjectWriter::executePostLayoutBinding(MCAssembler &Asm,
                                                const MCAsmLayout &Layout) {
   // The presence of symbol versions causes undefined symbols and
   // versions declared with @@@ to be renamed.
 
-  for (MCSymbolData &OriginalData : Asm.symbols()) {
-    const MCSymbol &Alias = OriginalData.getSymbol();
-
+  for (const MCSymbol &A : Asm.symbols()) {
+    const auto &Alias = cast<MCSymbolELF>(A);
     // Not an alias.
     if (!Alias.isVariable())
       continue;
     auto *Ref = dyn_cast<MCSymbolRefExpr>(Alias.getVariableValue());
     if (!Ref)
       continue;
-    const MCSymbol &Symbol = Ref->getSymbol();
-    MCSymbolData &SD = Asm.getSymbolData(Symbol);
+    const auto &Symbol = cast<MCSymbolELF>(Ref->getSymbol());
 
     StringRef AliasName = Alias.getName();
     size_t Pos = AliasName.find('@');
@@ -502,8 +392,8 @@ void ELFObjectWriter::ExecutePostLayoutBinding(MCAssembler &Asm,
 
     // Aliases defined with .symvar copy the binding from the symbol they alias.
     // This is the first place we are able to copy this information.
-    OriginalData.setExternal(SD.isExternal());
-    MCELF::SetBinding(OriginalData, MCELF::GetBinding(SD));
+    Alias.setExternal(Symbol.isExternal());
+    Alias.setBinding(Symbol.getBinding());
 
     StringRef Rest = AliasName.substr(Pos);
     if (!Symbol.isUndefined() && !Rest.startswith("@@@"))
@@ -553,41 +443,39 @@ static uint8_t mergeTypeForSet(uint8_t origType, uint8_t newType) {
   return Type;
 }
 
-void ELFObjectWriter::WriteSymbol(SymbolTableWriter &Writer, ELFSymbolData &MSD,
+void ELFObjectWriter::writeSymbol(SymbolTableWriter &Writer,
+                                  uint32_t StringIndex, ELFSymbolData &MSD,
                                   const MCAsmLayout &Layout) {
-  MCSymbolData &OrigData = *MSD.SymbolData;
-  assert((!OrigData.getFragment() ||
-          (&OrigData.getFragment()->getParent()->getSection() ==
-           &OrigData.getSymbol().getSection())) &&
+  const auto &Symbol = cast<MCSymbolELF>(*MSD.Symbol);
+  assert((!Symbol.getFragment() ||
+          (Symbol.getFragment()->getParent() == &Symbol.getSection())) &&
          "The symbol's section doesn't match the fragment's symbol");
-  const MCSymbol *Base = Layout.getBaseSymbol(OrigData.getSymbol());
+  const MCSymbolELF *Base =
+      cast_or_null<MCSymbolELF>(Layout.getBaseSymbol(Symbol));
 
   // This has to be in sync with when computeSymbolTable uses SHN_ABS or
   // SHN_COMMON.
-  bool IsReserved = !Base || OrigData.isCommon();
+  bool IsReserved = !Base || Symbol.isCommon();
 
   // Binding and Type share the same byte as upper and lower nibbles
-  uint8_t Binding = MCELF::GetBinding(OrigData);
-  uint8_t Type = MCELF::GetType(OrigData);
-  MCSymbolData *BaseSD = nullptr;
+  uint8_t Binding = Symbol.getBinding();
+  uint8_t Type = Symbol.getType();
   if (Base) {
-    BaseSD = &Layout.getAssembler().getSymbolData(*Base);
-    Type = mergeTypeForSet(Type, MCELF::GetType(*BaseSD));
+    Type = mergeTypeForSet(Type, Base->getType());
   }
-  uint8_t Info = (Binding << ELF_STB_Shift) | (Type << ELF_STT_Shift);
+  uint8_t Info = (Binding << 4) | Type;
 
   // Other and Visibility share the same byte with Visibility using the lower
   // 2 bits
-  uint8_t Visibility = MCELF::GetVisibility(OrigData);
-  uint8_t Other = MCELF::getOther(OrigData) << (ELF_STO_Shift - ELF_STV_Shift);
-  Other |= Visibility;
+  uint8_t Visibility = Symbol.getVisibility();
+  uint8_t Other = Symbol.getOther() | Visibility;
 
-  uint64_t Value = SymbolValue(OrigData, Layout);
+  uint64_t Value = SymbolValue(*MSD.Symbol, Layout);
   uint64_t Size = 0;
 
-  const MCExpr *ESize = OrigData.getSize();
+  const MCExpr *ESize = MSD.Symbol->getSize();
   if (!ESize && Base)
-    ESize = BaseSD->getSize();
+    ESize = Base->getSize();
 
   if (ESize) {
     int64_t Res;
@@ -597,55 +485,8 @@ void ELFObjectWriter::WriteSymbol(SymbolTableWriter &Writer, ELFSymbolData &MSD,
   }
 
   // Write out the symbol table entry
-  Writer.writeSymbol(MSD.StringIndex, Info, Value, Size, Other,
-                     MSD.SectionIndex, IsReserved);
-}
-
-void ELFObjectWriter::WriteSymbolTable(MCDataFragment *SymtabF,
-                                       MCAssembler &Asm,
-                                       const MCAsmLayout &Layout,
-                                       SectionIndexMapTy &SectionIndexMap) {
-  // The string table must be emitted first because we need the index
-  // into the string table for all the symbol names.
-
-  // FIXME: Make sure the start of the symbol table is aligned.
-
-  SymbolTableWriter Writer(Asm, FWriter, is64Bit(), SectionIndexMap, SymtabF);
-
-  // The first entry is the undefined symbol entry.
-  Writer.writeSymbol(0, 0, 0, 0, 0, 0, false);
-
-  for (unsigned i = 0, e = FileSymbolData.size(); i != e; ++i) {
-    Writer.writeSymbol(FileSymbolData[i], ELF::STT_FILE | ELF::STB_LOCAL, 0, 0,
-                       ELF::STV_DEFAULT, ELF::SHN_ABS, true);
-  }
-
-  // Write the symbol table entries.
-  LastLocalSymbolIndex = FileSymbolData.size() + LocalSymbolData.size() + 1;
-
-  for (unsigned i = 0, e = LocalSymbolData.size(); i != e; ++i) {
-    ELFSymbolData &MSD = LocalSymbolData[i];
-    WriteSymbol(Writer, MSD, Layout);
-  }
-
-  for (unsigned i = 0, e = ExternalSymbolData.size(); i != e; ++i) {
-    ELFSymbolData &MSD = ExternalSymbolData[i];
-    MCSymbolData &Data = *MSD.SymbolData;
-    assert(((Data.getFlags() & ELF_STB_Global) ||
-            (Data.getFlags() & ELF_STB_Weak)) &&
-           "External symbol requires STB_GLOBAL or STB_WEAK flag");
-    WriteSymbol(Writer, MSD, Layout);
-    if (MCELF::GetBinding(Data) == ELF::STB_LOCAL)
-      LastLocalSymbolIndex++;
-  }
-
-  for (unsigned i = 0, e = UndefinedSymbolData.size(); i != e; ++i) {
-    ELFSymbolData &MSD = UndefinedSymbolData[i];
-    MCSymbolData &Data = *MSD.SymbolData;
-    WriteSymbol(Writer, MSD, Layout);
-    if (MCELF::GetBinding(Data) == ELF::STB_LOCAL)
-      LastLocalSymbolIndex++;
-  }
+  Writer.writeSymbol(StringIndex, Info, Value, Size, Other, MSD.SectionIndex,
+                     IsReserved);
 }
 
 // It is always valid to create a relocation with a symbol. It is preferable
@@ -653,9 +494,9 @@ void ELFObjectWriter::WriteSymbolTable(MCDataFragment *SymtabF,
 // allows us to omit some local symbols from the symbol table.
 bool ELFObjectWriter::shouldRelocateWithSymbol(const MCAssembler &Asm,
                                                const MCSymbolRefExpr *RefA,
-                                               const MCSymbolData *SD,
-                                               uint64_t C,
+                                               const MCSymbol *S, uint64_t C,
                                                unsigned Type) const {
+  const auto *Sym = cast_or_null<MCSymbolELF>(S);
   // A PCRel relocation to an absolute value has no symbol (or section). We
   // represent that with a relocation to a null section.
   if (!RefA)
@@ -690,11 +531,11 @@ bool ELFObjectWriter::shouldRelocateWithSymbol(const MCAssembler &Asm,
 
   // An undefined symbol is not in any section, so the relocation has to point
   // to the symbol itself.
-  const MCSymbol &Sym = SD->getSymbol();
-  if (Sym.isUndefined())
+  assert(Sym && "Expected a symbol");
+  if (Sym->isUndefined())
     return true;
 
-  unsigned Binding = MCELF::GetBinding(*SD);
+  unsigned Binding = Sym->getBinding();
   switch(Binding) {
   default:
     llvm_unreachable("Invalid Binding");
@@ -718,7 +559,7 @@ bool ELFObjectWriter::shouldRelocateWithSymbol(const MCAssembler &Asm,
   // If we change such a relocation to use the section, the linker would think
   // that it pointed to another string and subtracting 42 at runtime will
   // produce the wrong value.
-  auto &Sec = cast<MCSectionELF>(Sym.getSection());
+  auto &Sec = cast<MCSectionELF>(Sym->getSection());
   unsigned Flags = Sec.getFlags();
   if (Flags & ELF::SHF_MERGE) {
     if (C != 0)
@@ -741,43 +582,40 @@ bool ELFObjectWriter::shouldRelocateWithSymbol(const MCAssembler &Asm,
   // bit. With a symbol that is done by just having the symbol have that bit
   // set, so we would lose the bit if we relocated with the section.
   // FIXME: We could use the section but add the bit to the relocation value.
-  if (Asm.isThumbFunc(&Sym))
+  if (Asm.isThumbFunc(Sym))
     return true;
 
-  if (TargetObjectWriter->needsRelocateWithSymbol(*SD, Type))
+  if (TargetObjectWriter->needsRelocateWithSymbol(*Sym, Type))
     return true;
   return false;
 }
 
-static const MCSymbol *getWeakRef(const MCSymbolRefExpr &Ref) {
-  const MCSymbol &Sym = Ref.getSymbol();
-
-  if (Ref.getKind() == MCSymbolRefExpr::VK_WEAKREF)
-    return &Sym;
-
-  if (!Sym.isVariable())
-    return nullptr;
-
-  const MCExpr *Expr = Sym.getVariableValue();
-  const auto *Inner = dyn_cast<MCSymbolRefExpr>(Expr);
-  if (!Inner)
-    return nullptr;
-
-  if (Inner->getKind() == MCSymbolRefExpr::VK_WEAKREF)
-    return &Inner->getSymbol();
-  return nullptr;
-}
+// True if the assembler knows nothing about the final value of the symbol.
+// This doesn't cover the comdat issues, since in those cases the assembler
+// can at least know that all symbols in the section will move together.
+static bool isWeak(const MCSymbolELF &Sym) {
+  if (Sym.getType() == ELF::STT_GNU_IFUNC)
+    return true;
 
-static bool isWeak(const MCSymbolData &D) {
-  return D.getFlags() & ELF_STB_Weak || MCELF::GetType(D) == ELF::STT_GNU_IFUNC;
+  switch (Sym.getBinding()) {
+  default:
+    llvm_unreachable("Unknown binding");
+  case ELF::STB_LOCAL:
+    return false;
+  case ELF::STB_GLOBAL:
+    return false;
+  case ELF::STB_WEAK:
+  case ELF::STB_GNU_UNIQUE:
+    return true;
+  }
 }
 
-void ELFObjectWriter::RecordRelocation(MCAssembler &Asm,
+void ELFObjectWriter::recordRelocation(MCAssembler &Asm,
                                        const MCAsmLayout &Layout,
                                        const MCFragment *Fragment,
                                        const MCFixup &Fixup, MCValue Target,
                                        bool &IsPCRel, uint64_t &FixedValue) {
-  const MCSectionData *FixupSection = Fragment->getParent();
+  const MCSectionELF &FixupSection = cast<MCSectionELF>(*Fragment->getParent());
   uint64_t C = Target.getConstant();
   uint64_t FixupOffset = Layout.getFragmentOffset(Fragment) + Fixup.getOffset();
 
@@ -793,30 +631,29 @@ void ELFObjectWriter::RecordRelocation(MCAssembler &Asm,
     // or (A + C - R). If B = R + K and the relocation is not pcrel, we can
     // replace B to implement it: (A - R - K + C)
     if (IsPCRel)
-      Asm.getContext().FatalError(
+      Asm.getContext().reportFatalError(
           Fixup.getLoc(),
           "No relocation available to represent this relative expression");
 
-    const MCSymbol &SymB = RefB->getSymbol();
+    const auto &SymB = cast<MCSymbolELF>(RefB->getSymbol());
 
     if (SymB.isUndefined())
-      Asm.getContext().FatalError(
+      Asm.getContext().reportFatalError(
           Fixup.getLoc(),
           Twine("symbol '") + SymB.getName() +
               "' can not be undefined in a subtraction expression");
 
     assert(!SymB.isAbsolute() && "Should have been folded");
     const MCSection &SecB = SymB.getSection();
-    if (&SecB != &FixupSection->getSection())
-      Asm.getContext().FatalError(
+    if (&SecB != &FixupSection)
+      Asm.getContext().reportFatalError(
           Fixup.getLoc(), "Cannot represent a difference across sections");
 
-    const MCSymbolData &SymBD = Asm.getSymbolData(SymB);
-    if (::isWeak(SymBD))
-      Asm.getContext().FatalError(
+    if (::isWeak(SymB))
+      Asm.getContext().reportFatalError(
           Fixup.getLoc(), "Cannot represent a subtraction with a weak symbol");
 
-    uint64_t SymBOffset = Layout.getSymbolOffset(&SymBD);
+    uint64_t SymBOffset = Layout.getSymbolOffset(SymB);
     uint64_t K = SymBOffset - FixupOffset;
     IsPCRel = true;
     C -= K;
@@ -824,13 +661,23 @@ void ELFObjectWriter::RecordRelocation(MCAssembler &Asm,
 
   // We either rejected the fixup or folded B into C at this point.
   const MCSymbolRefExpr *RefA = Target.getSymA();
-  const MCSymbol *SymA = RefA ? &RefA->getSymbol() : nullptr;
-  const MCSymbolData *SymAD = SymA ? &Asm.getSymbolData(*SymA) : nullptr;
+  const auto *SymA = RefA ? cast<MCSymbolELF>(&RefA->getSymbol()) : nullptr;
+
+  bool ViaWeakRef = false;
+  if (SymA && SymA->isVariable()) {
+    const MCExpr *Expr = SymA->getVariableValue();
+    if (const auto *Inner = dyn_cast<MCSymbolRefExpr>(Expr)) {
+      if (Inner->getKind() == MCSymbolRefExpr::VK_WEAKREF) {
+        SymA = cast<MCSymbolELF>(&Inner->getSymbol());
+        ViaWeakRef = true;
+      }
+    }
+  }
 
   unsigned Type = GetRelocType(Target, Fixup, IsPCRel);
-  bool RelocateWithSymbol = shouldRelocateWithSymbol(Asm, RefA, SymAD, C, Type);
+  bool RelocateWithSymbol = shouldRelocateWithSymbol(Asm, RefA, SymA, C, Type);
   if (!RelocateWithSymbol && SymA && !SymA->isUndefined())
-    C += Layout.getSymbolOffset(SymAD);
+    C += Layout.getSymbolOffset(*SymA);
 
   uint64_t Addend = 0;
   if (hasRelocationAddend()) {
@@ -840,50 +687,36 @@ void ELFObjectWriter::RecordRelocation(MCAssembler &Asm,
 
   FixedValue = C;
 
-  // FIXME: What is this!?!?
-  MCSymbolRefExpr::VariantKind Modifier =
-      RefA ? RefA->getKind() : MCSymbolRefExpr::VK_None;
-  if (RelocNeedsGOT(Modifier))
-    NeedsGOT = true;
-
   if (!RelocateWithSymbol) {
     const MCSection *SecA =
         (SymA && !SymA->isUndefined()) ? &SymA->getSection() : nullptr;
     auto *ELFSec = cast_or_null<MCSectionELF>(SecA);
-    MCSymbol *SectionSymbol =
-        ELFSec ? Asm.getContext().getOrCreateSectionSymbol(*ELFSec)
-               : nullptr;
+    const auto *SectionSymbol =
+        ELFSec ? cast<MCSymbolELF>(ELFSec->getBeginSymbol()) : nullptr;
+    if (SectionSymbol)
+      SectionSymbol->setUsedInReloc();
     ELFRelocationEntry Rec(FixupOffset, SectionSymbol, Type, Addend);
-    Relocations[FixupSection].push_back(Rec);
+    Relocations[&FixupSection].push_back(Rec);
     return;
   }
 
   if (SymA) {
-    if (const MCSymbol *R = Renames.lookup(SymA))
+    if (const MCSymbolELF *R = Renames.lookup(SymA))
       SymA = R;
 
-    if (const MCSymbol *WeakRef = getWeakRef(*RefA))
-      WeakrefUsedInReloc.insert(WeakRef);
+    if (ViaWeakRef)
+      SymA->setIsWeakrefUsedInReloc();
     else
-      UsedInReloc.insert(SymA);
+      SymA->setUsedInReloc();
   }
   ELFRelocationEntry Rec(FixupOffset, SymA, Type, Addend);
-  Relocations[FixupSection].push_back(Rec);
+  Relocations[&FixupSection].push_back(Rec);
   return;
 }
 
-
-uint64_t
-ELFObjectWriter::getSymbolIndexInSymbolTable(const MCAssembler &Asm,
-                                             const MCSymbol *S) {
-  const MCSymbolData &SD = Asm.getSymbolData(*S);
-  return SD.getIndex();
-}
-
 bool ELFObjectWriter::isInSymtab(const MCAsmLayout &Layout,
-                                 const MCSymbolData &Data, bool Used,
+                                 const MCSymbolELF &Symbol, bool Used,
                                  bool Renamed) {
-  const MCSymbol &Symbol = Data.getSymbol();
   if (Symbol.isVariable()) {
     const MCExpr *Expr = Symbol.getVariableValue();
     if (const MCSymbolRefExpr *Ref = dyn_cast<MCSymbolRefExpr>(Expr)) {
@@ -898,127 +731,88 @@ bool ELFObjectWriter::isInSymtab(const MCAsmLayout &Layout,
   if (Renamed)
     return false;
 
-  if (Symbol.getName() == "_GLOBAL_OFFSET_TABLE_")
-    return true;
-
-  if (Symbol.isVariable()) {
-    const MCSymbol *Base = Layout.getBaseSymbol(Symbol);
-    if (Base && Base->isUndefined())
-      return false;
+  if (Symbol.isVariable() && Symbol.isUndefined()) {
+    // FIXME: this is here just to diagnose the case of a var = commmon_sym.
+    Layout.getBaseSymbol(Symbol);
+    return false;
   }
 
-  bool IsGlobal = MCELF::GetBinding(Data) == ELF::STB_GLOBAL;
-  if (!Symbol.isVariable() && Symbol.isUndefined() && !IsGlobal)
+  if (Symbol.isUndefined() && !Symbol.isBindingSet())
     return false;
 
   if (Symbol.isTemporary())
     return false;
 
-  return true;
-}
-
-bool ELFObjectWriter::isLocal(const MCSymbolData &Data, bool isUsedInReloc) {
-  if (Data.isExternal())
-    return false;
-
-  const MCSymbol &Symbol = Data.getSymbol();
-  if (Symbol.isDefined())
-    return true;
-
-  if (isUsedInReloc)
+  if (Symbol.getType() == ELF::STT_SECTION)
     return false;
 
   return true;
 }
 
-void ELFObjectWriter::computeIndexMap(MCAssembler &Asm,
-                                      SectionIndexMapTy &SectionIndexMap) {
-  unsigned Index = 1;
-  for (MCAssembler::iterator it = Asm.begin(),
-         ie = Asm.end(); it != ie; ++it) {
-    const MCSectionELF &Section =
-      static_cast<const MCSectionELF &>(it->getSection());
-    if (Section.getType() != ELF::SHT_GROUP)
-      continue;
-    SectionIndexMap[&Section] = Index++;
-  }
-
-  for (MCAssembler::iterator it = Asm.begin(),
-         ie = Asm.end(); it != ie; ++it) {
-    const MCSectionData &SD = *it;
-    const MCSectionELF &Section =
-      static_cast<const MCSectionELF &>(SD.getSection());
-    if (Section.getType() == ELF::SHT_GROUP ||
-        Section.getType() == ELF::SHT_REL ||
-        Section.getType() == ELF::SHT_RELA)
-      continue;
-    SectionIndexMap[&Section] = Index++;
-    if (MCSectionData *RelSD = createRelocationSection(Asm, SD)) {
-      const MCSectionELF *RelSection =
-          static_cast<const MCSectionELF *>(&RelSD->getSection());
-      SectionIndexMap[RelSection] = Index++;
-    }
-  }
-}
-
 void ELFObjectWriter::computeSymbolTable(
     MCAssembler &Asm, const MCAsmLayout &Layout,
-    const SectionIndexMapTy &SectionIndexMap,
-    const RevGroupMapTy &RevGroupMap) {
-  // FIXME: Is this the correct place to do this?
-  // FIXME: Why is an undefined reference to _GLOBAL_OFFSET_TABLE_ needed?
-  if (NeedsGOT) {
-    StringRef Name = "_GLOBAL_OFFSET_TABLE_";
-    MCSymbol *Sym = Asm.getContext().GetOrCreateSymbol(Name);
-    MCSymbolData &Data = Asm.getOrCreateSymbolData(*Sym);
-    Data.setExternal(true);
-    MCELF::SetBinding(Data, ELF::STB_GLOBAL);
-  }
+    const SectionIndexMapTy &SectionIndexMap, const RevGroupMapTy &RevGroupMap,
+    SectionOffsetsTy &SectionOffsets) {
+  MCContext &Ctx = Asm.getContext();
+  SymbolTableWriter Writer(*this, is64Bit());
 
-  // Add the data for the symbols.
-  for (MCSymbolData &SD : Asm.symbols()) {
-    const MCSymbol &Symbol = SD.getSymbol();
+  // Symbol table
+  unsigned EntrySize = is64Bit() ? ELF::SYMENTRY_SIZE64 : ELF::SYMENTRY_SIZE32;
+  MCSectionELF *SymtabSection =
+      Ctx.getELFSection(".symtab", ELF::SHT_SYMTAB, 0, EntrySize, "");
+  SymtabSection->setAlignment(is64Bit() ? 8 : 4);
+  SymbolTableIndex = addToSectionTable(SymtabSection);
+
+  align(SymtabSection->getAlignment());
+  uint64_t SecStart = OS.tell();
+
+  // The first entry is the undefined symbol entry.
+  Writer.writeSymbol(0, 0, 0, 0, 0, 0, false);
 
-    bool Used = UsedInReloc.count(&Symbol);
-    bool WeakrefUsed = WeakrefUsedInReloc.count(&Symbol);
-    bool isSignature = RevGroupMap.count(&Symbol);
+  std::vector<ELFSymbolData> LocalSymbolData;
+  std::vector<ELFSymbolData> ExternalSymbolData;
 
-    if (!isInSymtab(Layout, SD,
-                    Used || WeakrefUsed || isSignature,
+  // Add the data for the symbols.
+  bool HasLargeSectionIndex = false;
+  for (const MCSymbol &S : Asm.symbols()) {
+    const auto &Symbol = cast<MCSymbolELF>(S);
+    bool Used = Symbol.isUsedInReloc();
+    bool WeakrefUsed = Symbol.isWeakrefUsedInReloc();
+    bool isSignature = Symbol.isSignature();
+
+    if (!isInSymtab(Layout, Symbol, Used || WeakrefUsed || isSignature,
                     Renames.count(&Symbol)))
       continue;
 
+    if (Symbol.isTemporary() && Symbol.isUndefined())
+      Ctx.reportFatalError(SMLoc(), "Undefined temporary");
+
     ELFSymbolData MSD;
-    MSD.SymbolData = &SD;
-    const MCSymbol *BaseSymbol = Layout.getBaseSymbol(Symbol);
-
-    // Undefined symbols are global, but this is the first place we
-    // are able to set it.
-    bool Local = isLocal(SD, Used);
-    if (!Local && MCELF::GetBinding(SD) == ELF::STB_LOCAL) {
-      assert(BaseSymbol);
-      MCSymbolData &BaseData = Asm.getSymbolData(*BaseSymbol);
-      MCELF::SetBinding(SD, ELF::STB_GLOBAL);
-      MCELF::SetBinding(BaseData, ELF::STB_GLOBAL);
-    }
+    MSD.Symbol = cast<MCSymbolELF>(&Symbol);
 
-    if (!BaseSymbol) {
+    bool Local = Symbol.getBinding() == ELF::STB_LOCAL;
+    assert(Local || !Symbol.isTemporary());
+
+    if (Symbol.isAbsolute()) {
       MSD.SectionIndex = ELF::SHN_ABS;
-    } else if (SD.isCommon()) {
+    } else if (Symbol.isCommon()) {
       assert(!Local);
       MSD.SectionIndex = ELF::SHN_COMMON;
-    } else if (BaseSymbol->isUndefined()) {
-      if (isSignature && !Used)
-        MSD.SectionIndex = SectionIndexMap.lookup(RevGroupMap.lookup(&Symbol));
-      else
+    } else if (Symbol.isUndefined()) {
+      if (isSignature && !Used) {
+        MSD.SectionIndex = RevGroupMap.lookup(&Symbol);
+        if (MSD.SectionIndex >= ELF::SHN_LORESERVE)
+          HasLargeSectionIndex = true;
+      } else {
         MSD.SectionIndex = ELF::SHN_UNDEF;
-      if (!Used && WeakrefUsed)
-        MCELF::SetBinding(SD, ELF::STB_WEAK);
+      }
     } else {
       const MCSectionELF &Section =
-        static_cast<const MCSectionELF&>(BaseSymbol->getSection());
+          static_cast<const MCSectionELF &>(Symbol.getSection());
       MSD.SectionIndex = SectionIndexMap.lookup(&Section);
       assert(MSD.SectionIndex && "Invalid section index!");
+      if (MSD.SectionIndex >= ELF::SHN_LORESERVE)
+        HasLargeSectionIndex = true;
     }
 
     // The @@@ in symbol version is replaced with @ in undefined symbols and @@
@@ -1045,12 +839,12 @@ void ELFObjectWriter::computeSymbolTable(
     // seems that this information is not easily accessible from the
     // ELFObjectWriter.
     StringRef Name = Symbol.getName();
+    SmallString<32> Buf;
     if (!Name.startswith("?") && !Name.startswith("@?") &&
         !Name.startswith("__imp_?") && !Name.startswith("__imp_@?")) {
       // This symbol isn't following the MSVC C++ name mangling convention. We
       // can thus safely interpret the @@@ in symbol names as specifying symbol
       // versioning.
-      SmallString<32> Buf;
       size_t Pos = Name.find("@@@");
       if (Pos != StringRef::npos) {
         Buf += Name.substr(0, Pos);
@@ -1061,62 +855,88 @@ void ELFObjectWriter::computeSymbolTable(
     }
 
     // Sections have their own string table
-    if (MCELF::GetType(SD) != ELF::STT_SECTION)
+    if (Symbol.getType() != ELF::STT_SECTION)
       MSD.Name = StrTabBuilder.add(Name);
 
-    if (MSD.SectionIndex == ELF::SHN_UNDEF)
-      UndefinedSymbolData.push_back(MSD);
-    else if (Local)
+    if (Local)
       LocalSymbolData.push_back(MSD);
     else
       ExternalSymbolData.push_back(MSD);
   }
 
-  for (auto i = Asm.file_names_begin(), e = Asm.file_names_end(); i != e; ++i)
-    StrTabBuilder.add(*i);
+  // This holds the .symtab_shndx section index.
+  unsigned SymtabShndxSectionIndex = 0;
 
-  StrTabBuilder.finalize(StringTableBuilder::ELF);
+  if (HasLargeSectionIndex) {
+    MCSectionELF *SymtabShndxSection =
+        Ctx.getELFSection(".symtab_shndxr", ELF::SHT_SYMTAB_SHNDX, 0, 4, "");
+    SymtabShndxSectionIndex = addToSectionTable(SymtabShndxSection);
+    SymtabShndxSection->setAlignment(4);
+  }
 
-  for (auto i = Asm.file_names_begin(), e = Asm.file_names_end(); i != e; ++i)
-    FileSymbolData.push_back(StrTabBuilder.getOffset(*i));
+  ArrayRef<std::string> FileNames = Asm.getFileNames();
+  for (const std::string &Name : FileNames)
+    StrTabBuilder.add(Name);
+
+  StrTabBuilder.finalize(StringTableBuilder::ELF);
 
-  for (ELFSymbolData &MSD : LocalSymbolData)
-    MSD.StringIndex = MCELF::GetType(*MSD.SymbolData) == ELF::STT_SECTION
-                          ? 0
-                          : StrTabBuilder.getOffset(MSD.Name);
-  for (ELFSymbolData &MSD : ExternalSymbolData)
-    MSD.StringIndex = StrTabBuilder.getOffset(MSD.Name);
-  for (ELFSymbolData& MSD : UndefinedSymbolData)
-    MSD.StringIndex = StrTabBuilder.getOffset(MSD.Name);
+  for (const std::string &Name : FileNames)
+    Writer.writeSymbol(StrTabBuilder.getOffset(Name),
+                       ELF::STT_FILE | ELF::STB_LOCAL, 0, 0, ELF::STV_DEFAULT,
+                       ELF::SHN_ABS, true);
 
   // Symbols are required to be in lexicographic order.
   array_pod_sort(LocalSymbolData.begin(), LocalSymbolData.end());
   array_pod_sort(ExternalSymbolData.begin(), ExternalSymbolData.end());
-  array_pod_sort(UndefinedSymbolData.begin(), UndefinedSymbolData.end());
 
   // Set the symbol indices. Local symbols must come before all other
   // symbols with non-local bindings.
-  unsigned Index = FileSymbolData.size() + 1;
-  for (unsigned i = 0, e = LocalSymbolData.size(); i != e; ++i)
-    LocalSymbolData[i].SymbolData->setIndex(Index++);
-
-  for (unsigned i = 0, e = ExternalSymbolData.size(); i != e; ++i)
-    ExternalSymbolData[i].SymbolData->setIndex(Index++);
-  for (unsigned i = 0, e = UndefinedSymbolData.size(); i != e; ++i)
-    UndefinedSymbolData[i].SymbolData->setIndex(Index++);
+  unsigned Index = FileNames.size() + 1;
+
+  for (ELFSymbolData &MSD : LocalSymbolData) {
+    unsigned StringIndex = MSD.Symbol->getType() == ELF::STT_SECTION
+                               ? 0
+                               : StrTabBuilder.getOffset(MSD.Name);
+    MSD.Symbol->setIndex(Index++);
+    writeSymbol(Writer, StringIndex, MSD, Layout);
+  }
+
+  // Write the symbol table entries.
+  LastLocalSymbolIndex = Index;
+
+  for (ELFSymbolData &MSD : ExternalSymbolData) {
+    unsigned StringIndex = StrTabBuilder.getOffset(MSD.Name);
+    MSD.Symbol->setIndex(Index++);
+    writeSymbol(Writer, StringIndex, MSD, Layout);
+    assert(MSD.Symbol->getBinding() != ELF::STB_LOCAL);
+  }
+
+  uint64_t SecEnd = OS.tell();
+  SectionOffsets[SymtabSection] = std::make_pair(SecStart, SecEnd);
+
+  ArrayRef<uint32_t> ShndxIndexes = Writer.getShndxIndexes();
+  if (ShndxIndexes.empty()) {
+    assert(SymtabShndxSectionIndex == 0);
+    return;
+  }
+  assert(SymtabShndxSectionIndex != 0);
+
+  SecStart = OS.tell();
+  const MCSectionELF *SymtabShndxSection =
+      SectionTable[SymtabShndxSectionIndex - 1];
+  for (uint32_t Index : ShndxIndexes)
+    write(Index);
+  SecEnd = OS.tell();
+  SectionOffsets[SymtabShndxSection] = std::make_pair(SecStart, SecEnd);
 }
 
-MCSectionData *
-ELFObjectWriter::createRelocationSection(MCAssembler &Asm,
-                                         const MCSectionData &SD) {
-  if (Relocations[&SD].empty())
+MCSectionELF *
+ELFObjectWriter::createRelocationSection(MCContext &Ctx,
+                                         const MCSectionELF &Sec) {
+  if (Relocations[&Sec].empty())
     return nullptr;
 
-  MCContext &Ctx = Asm.getContext();
-  const MCSectionELF &Section =
-      static_cast<const MCSectionELF &>(SD.getSection());
-
-  const StringRef SectionName = Section.getSectionName();
+  const StringRef SectionName = Sec.getSectionName();
   std::string RelaSectionName = hasRelocationAddend() ? ".rela" : ".rel";
   RelaSectionName += SectionName;
 
@@ -1127,18 +947,19 @@ ELFObjectWriter::createRelocationSection(MCAssembler &Asm,
     EntrySize = is64Bit() ? sizeof(ELF::Elf64_Rel) : sizeof(ELF::Elf32_Rel);
 
   unsigned Flags = 0;
-  if (Section.getFlags() & ELF::SHF_GROUP)
+  if (Sec.getFlags() & ELF::SHF_GROUP)
     Flags = ELF::SHF_GROUP;
 
-  const MCSectionELF *RelaSection = Ctx.createELFRelSection(
+  MCSectionELF *RelaSection = Ctx.createELFRelSection(
       RelaSectionName, hasRelocationAddend() ? ELF::SHT_RELA : ELF::SHT_REL,
-      Flags, EntrySize, Section.getGroup(), &Section);
-  return &Asm.getOrCreateSectionData(*RelaSection);
+      Flags, EntrySize, Sec.getGroup(), &Sec);
+  RelaSection->setAlignment(is64Bit() ? 8 : 4);
+  return RelaSection;
 }
 
 static SmallVector<char, 128>
-getUncompressedData(MCAsmLayout &Layout,
-                    MCSectionData::FragmentListType &Fragments) {
+getUncompressedData(const MCAsmLayout &Layout,
+                    const MCSection::FragmentListType &Fragments) {
   SmallVector<char, 128> UncompressedData;
   for (const MCFragment &F : Fragments) {
     const SmallVectorImpl<char> *Contents;
@@ -1181,123 +1002,41 @@ prependCompressionHeader(uint64_t Size,
   return true;
 }
 
-// Return a single fragment containing the compressed contents of the whole
-// section. Null if the section was not compressed for any reason.
-static std::unique_ptr<MCDataFragment>
-getCompressedFragment(MCAsmLayout &Layout,
-                      MCSectionData::FragmentListType &Fragments) {
-  std::unique_ptr<MCDataFragment> CompressedFragment(new MCDataFragment());
+void ELFObjectWriter::writeSectionData(const MCAssembler &Asm, MCSection &Sec,
+                                       const MCAsmLayout &Layout) {
+  MCSectionELF &Section = static_cast<MCSectionELF &>(Sec);
+  StringRef SectionName = Section.getSectionName();
+
+  // Compressing debug_frame requires handling alignment fragments which is
+  // more work (possibly generalizing MCAssembler.cpp:writeFragment to allow
+  // for writing to arbitrary buffers) for little benefit.
+  if (!Asm.getContext().getAsmInfo()->compressDebugSections() ||
+      !SectionName.startswith(".debug_") || SectionName == ".debug_frame") {
+    Asm.writeSectionData(&Section, Layout);
+    return;
+  }
 
-  // Gather the uncompressed data from all the fragments, recording the
-  // alignment fragment, if seen, and any fixups.
+  // Gather the uncompressed data from all the fragments.
+  const MCSection::FragmentListType &Fragments = Section.getFragmentList();
   SmallVector<char, 128> UncompressedData =
       getUncompressedData(Layout, Fragments);
 
-  SmallVectorImpl<char> &CompressedContents = CompressedFragment->getContents();
-
+  SmallVector<char, 128> CompressedContents;
   zlib::Status Success = zlib::compress(
       StringRef(UncompressedData.data(), UncompressedData.size()),
       CompressedContents);
-  if (Success != zlib::StatusOK)
-    return nullptr;
-
-  if (!prependCompressionHeader(UncompressedData.size(), CompressedContents))
-    return nullptr;
-
-  return CompressedFragment;
-}
-
-typedef DenseMap<const MCSectionData *, std::vector<MCSymbolData *>>
-DefiningSymbolMap;
-
-static void UpdateSymbols(const MCAsmLayout &Layout,
-                          const std::vector<MCSymbolData *> &Symbols,
-                          MCFragment &NewFragment) {
-  for (MCSymbolData *Sym : Symbols) {
-    Sym->setOffset(Sym->getOffset() +
-                   Layout.getFragmentOffset(Sym->getFragment()));
-    Sym->setFragment(&NewFragment);
+  if (Success != zlib::StatusOK) {
+    Asm.writeSectionData(&Section, Layout);
+    return;
   }
-}
 
-static void CompressDebugSection(MCAssembler &Asm, MCAsmLayout &Layout,
-                                 const DefiningSymbolMap &DefiningSymbols,
-                                 const MCSectionELF &Section,
-                                 MCSectionData &SD) {
-  StringRef SectionName = Section.getSectionName();
-  MCSectionData::FragmentListType &Fragments = SD.getFragmentList();
-
-  std::unique_ptr<MCDataFragment> CompressedFragment =
-      getCompressedFragment(Layout, Fragments);
-
-  // Leave the section as-is if the fragments could not be compressed.
-  if (!CompressedFragment)
+  if (!prependCompressionHeader(UncompressedData.size(), CompressedContents)) {
+    Asm.writeSectionData(&Section, Layout);
     return;
-
-  // Update the fragment+offsets of any symbols referring to fragments in this
-  // section to refer to the new fragment.
-  auto I = DefiningSymbols.find(&SD);
-  if (I != DefiningSymbols.end())
-    UpdateSymbols(Layout, I->second, *CompressedFragment);
-
-  // Invalidate the layout for the whole section since it will have new and
-  // different fragments now.
-  Layout.invalidateFragmentsFrom(&Fragments.front());
-  Fragments.clear();
-
-  // Complete the initialization of the new fragment
-  CompressedFragment->setParent(&SD);
-  CompressedFragment->setLayoutOrder(0);
-  Fragments.push_back(CompressedFragment.release());
-
-  // Rename from .debug_* to .zdebug_*
+  }
   Asm.getContext().renameELFSection(&Section,
                                     (".z" + SectionName.drop_front(1)).str());
-}
-
-void ELFObjectWriter::CompressDebugSections(MCAssembler &Asm,
-                                            MCAsmLayout &Layout) {
-  if (!Asm.getContext().getAsmInfo()->compressDebugSections())
-    return;
-
-  DefiningSymbolMap DefiningSymbols;
-
-  for (MCSymbolData &SD : Asm.symbols())
-    if (MCFragment *F = SD.getFragment())
-      DefiningSymbols[F->getParent()].push_back(&SD);
-
-  for (MCSectionData &SD : Asm) {
-    const MCSectionELF &Section =
-        static_cast<const MCSectionELF &>(SD.getSection());
-    StringRef SectionName = Section.getSectionName();
-
-    // Compressing debug_frame requires handling alignment fragments which is
-    // more work (possibly generalizing MCAssembler.cpp:writeFragment to allow
-    // for writing to arbitrary buffers) for little benefit.
-    if (!SectionName.startswith(".debug_") || SectionName == ".debug_frame")
-      continue;
-
-    CompressDebugSection(Asm, Layout, DefiningSymbols, Section, SD);
-  }
-}
-
-void ELFObjectWriter::WriteRelocations(MCAssembler &Asm, MCAsmLayout &Layout) {
-  for (MCAssembler::iterator it = Asm.begin(), ie = Asm.end(); it != ie; ++it) {
-    MCSectionData &RelSD = *it;
-    const MCSectionELF &RelSection =
-        static_cast<const MCSectionELF &>(RelSD.getSection());
-
-    unsigned Type = RelSection.getType();
-    if (Type != ELF::SHT_REL && Type != ELF::SHT_RELA)
-      continue;
-
-    const MCSectionELF *Section = RelSection.getAssociatedSection();
-    MCSectionData &SD = Asm.getOrCreateSectionData(*Section);
-    RelSD.setAlignment(is64Bit() ? 8 : 4);
-
-    MCDataFragment *F = new MCDataFragment(&RelSD);
-    WriteRelocationsFragment(Asm, F, &SD);
-  }
+  OS << CompressedContents;
 }
 
 void ELFObjectWriter::WriteSecHdrEntry(uint32_t Name, uint32_t Type,
@@ -1306,22 +1045,21 @@ void ELFObjectWriter::WriteSecHdrEntry(uint32_t Name, uint32_t Type,
                                        uint32_t Link, uint32_t Info,
                                        uint64_t Alignment,
                                        uint64_t EntrySize) {
-  Write32(Name);        // sh_name: index into string table
-  Write32(Type);        // sh_type
+  write32(Name);        // sh_name: index into string table
+  write32(Type);        // sh_type
   WriteWord(Flags);     // sh_flags
   WriteWord(Address);   // sh_addr
   WriteWord(Offset);    // sh_offset
   WriteWord(Size);      // sh_size
-  Write32(Link);        // sh_link
-  Write32(Info);        // sh_info
+  write32(Link);        // sh_link
+  write32(Info);        // sh_info
   WriteWord(Alignment); // sh_addralign
   WriteWord(EntrySize); // sh_entsize
 }
 
-void ELFObjectWriter::WriteRelocationsFragment(const MCAssembler &Asm,
-                                               MCDataFragment *F,
-                                               const MCSectionData *SD) {
-  std::vector<ELFRelocationEntry> &Relocs = Relocations[SD];
+void ELFObjectWriter::writeRelocations(const MCAssembler &Asm,
+                                       const MCSectionELF &Sec) {
+  std::vector<ELFRelocationEntry> &Relocs = Relocations[&Sec];
 
   // Sort the relocation entries. Most targets just sort by Offset, but some
   // (e.g., MIPS) have additional constraints.
@@ -1329,139 +1067,46 @@ void ELFObjectWriter::WriteRelocationsFragment(const MCAssembler &Asm,
 
   for (unsigned i = 0, e = Relocs.size(); i != e; ++i) {
     const ELFRelocationEntry &Entry = Relocs[e - i - 1];
-    unsigned Index =
-        Entry.Symbol ? getSymbolIndexInSymbolTable(Asm, Entry.Symbol) : 0;
+    unsigned Index = Entry.Symbol ? Entry.Symbol->getIndex() : 0;
 
     if (is64Bit()) {
-      write(*F, Entry.Offset);
+      write(Entry.Offset);
       if (TargetObjectWriter->isN64()) {
-        write(*F, uint32_t(Index));
+        write(uint32_t(Index));
 
-        write(*F, TargetObjectWriter->getRSsym(Entry.Type));
-        write(*F, TargetObjectWriter->getRType3(Entry.Type));
-        write(*F, TargetObjectWriter->getRType2(Entry.Type));
-        write(*F, TargetObjectWriter->getRType(Entry.Type));
+        write(TargetObjectWriter->getRSsym(Entry.Type));
+        write(TargetObjectWriter->getRType3(Entry.Type));
+        write(TargetObjectWriter->getRType2(Entry.Type));
+        write(TargetObjectWriter->getRType(Entry.Type));
       } else {
         struct ELF::Elf64_Rela ERE64;
         ERE64.setSymbolAndType(Index, Entry.Type);
-        write(*F, ERE64.r_info);
+        write(ERE64.r_info);
       }
       if (hasRelocationAddend())
-        write(*F, Entry.Addend);
+        write(Entry.Addend);
     } else {
-      write(*F, uint32_t(Entry.Offset));
+      write(uint32_t(Entry.Offset));
 
       struct ELF::Elf32_Rela ERE32;
       ERE32.setSymbolAndType(Index, Entry.Type);
-      write(*F, ERE32.r_info);
+      write(ERE32.r_info);
 
       if (hasRelocationAddend())
-        write(*F, uint32_t(Entry.Addend));
+        write(uint32_t(Entry.Addend));
     }
   }
 }
 
-void ELFObjectWriter::CreateMetadataSections(
-    MCAssembler &Asm, MCAsmLayout &Layout, SectionIndexMapTy &SectionIndexMap) {
-  MCContext &Ctx = Asm.getContext();
-  MCDataFragment *F;
-
-  unsigned EntrySize = is64Bit() ? ELF::SYMENTRY_SIZE64 : ELF::SYMENTRY_SIZE32;
-
-  // We construct .shstrtab, .symtab and .strtab in this order to match gnu as.
-  const MCSectionELF *ShstrtabSection =
-      Ctx.getELFSection(".shstrtab", ELF::SHT_STRTAB, 0);
-  MCSectionData &ShstrtabSD = Asm.getOrCreateSectionData(*ShstrtabSection);
-  ShstrtabSD.setAlignment(1);
-  ShstrtabIndex = SectionIndexMap.size() + 1;
-  SectionIndexMap[ShstrtabSection] = ShstrtabIndex;
-
-  const MCSectionELF *SymtabSection =
-    Ctx.getELFSection(".symtab", ELF::SHT_SYMTAB, 0,
-                      EntrySize, "");
-  MCSectionData &SymtabSD = Asm.getOrCreateSectionData(*SymtabSection);
-  SymtabSD.setAlignment(is64Bit() ? 8 : 4);
-  SymbolTableIndex = SectionIndexMap.size() + 1;
-  SectionIndexMap[SymtabSection] = SymbolTableIndex;
-
-  const MCSectionELF *StrtabSection;
-  StrtabSection = Ctx.getELFSection(".strtab", ELF::SHT_STRTAB, 0);
-  MCSectionData &StrtabSD = Asm.getOrCreateSectionData(*StrtabSection);
-  StrtabSD.setAlignment(1);
-  StringTableIndex = SectionIndexMap.size() + 1;
-  SectionIndexMap[StrtabSection] = StringTableIndex;
-
-  // Symbol table
-  F = new MCDataFragment(&SymtabSD);
-  WriteSymbolTable(F, Asm, Layout, SectionIndexMap);
-
-  F = new MCDataFragment(&StrtabSD);
-  F->getContents().append(StrTabBuilder.data().begin(),
-                          StrTabBuilder.data().end());
-
-  F = new MCDataFragment(&ShstrtabSD);
-
-  // Section header string table.
-  for (auto it = Asm.begin(), ie = Asm.end(); it != ie; ++it) {
-    const MCSectionELF &Section =
-      static_cast<const MCSectionELF&>(it->getSection());
-    ShStrTabBuilder.add(Section.getSectionName());
-  }
-  ShStrTabBuilder.finalize(StringTableBuilder::ELF);
-  F->getContents().append(ShStrTabBuilder.data().begin(),
-                          ShStrTabBuilder.data().end());
+const MCSectionELF *ELFObjectWriter::createStringTable(MCContext &Ctx) {
+  const MCSectionELF *StrtabSection = SectionTable[StringTableIndex - 1];
+  OS << StrTabBuilder.data();
+  return StrtabSection;
 }
 
-void ELFObjectWriter::createIndexedSections(
-    MCAssembler &Asm, MCAsmLayout &Layout, GroupMapTy &GroupMap,
-    RevGroupMapTy &RevGroupMap, SectionIndexMapTy &SectionIndexMap) {
-  MCContext &Ctx = Asm.getContext();
-
-  // Build the groups
-  for (MCAssembler::const_iterator it = Asm.begin(), ie = Asm.end();
-       it != ie; ++it) {
-    const MCSectionELF &Section =
-      static_cast<const MCSectionELF&>(it->getSection());
-    if (!(Section.getFlags() & ELF::SHF_GROUP))
-      continue;
-
-    const MCSymbol *SignatureSymbol = Section.getGroup();
-    Asm.getOrCreateSymbolData(*SignatureSymbol);
-    const MCSectionELF *&Group = RevGroupMap[SignatureSymbol];
-    if (!Group) {
-      Group = Ctx.CreateELFGroupSection();
-      MCSectionData &Data = Asm.getOrCreateSectionData(*Group);
-      Data.setAlignment(4);
-      MCDataFragment *F = new MCDataFragment(&Data);
-      write(*F, uint32_t(ELF::GRP_COMDAT));
-    }
-    GroupMap[Group] = SignatureSymbol;
-  }
-
-  computeIndexMap(Asm, SectionIndexMap);
-
-  // Add sections to the groups
-  for (MCAssembler::const_iterator it = Asm.begin(), ie = Asm.end();
-       it != ie; ++it) {
-    const MCSectionELF &Section =
-      static_cast<const MCSectionELF&>(it->getSection());
-    if (!(Section.getFlags() & ELF::SHF_GROUP))
-      continue;
-    const MCSectionELF *Group = RevGroupMap[Section.getGroup()];
-    MCSectionData &Data = Asm.getOrCreateSectionData(*Group);
-    // FIXME: we could use the previous fragment
-    MCDataFragment *F = new MCDataFragment(&Data);
-    uint32_t Index = SectionIndexMap.lookup(&Section);
-    write(*F, Index);
-  }
-}
-
-void ELFObjectWriter::writeSection(MCAssembler &Asm,
-                                   const SectionIndexMapTy &SectionIndexMap,
-                                   uint32_t GroupSymbolIndex,
-                                   uint64_t Offset, uint64_t Size,
-                                   uint64_t Alignment,
-                                   const MCSectionELF &Section) {
+void ELFObjectWriter::writeSection(const SectionIndexMapTy &SectionIndexMap,
+                                   uint32_t GroupSymbolIndex, uint64_t Offset,
+                                   uint64_t Size, const MCSectionELF &Section) {
   uint64_t sh_link = 0;
   uint64_t sh_info = 0;
 
@@ -1471,8 +1116,7 @@ void ELFObjectWriter::writeSection(MCAssembler &Asm,
     break;
 
   case ELF::SHT_DYNAMIC:
-    sh_link = ShStrTabBuilder.getOffset(Section.getSectionName());
-    break;
+    llvm_unreachable("SHT_DYNAMIC in a relocatable object");
 
   case ELF::SHT_REL:
   case ELF::SHT_RELA: {
@@ -1503,135 +1147,155 @@ void ELFObjectWriter::writeSection(MCAssembler &Asm,
       Section.getType() == ELF::SHT_ARM_EXIDX)
     sh_link = SectionIndexMap.lookup(Section.getAssociatedSection());
 
-  WriteSecHdrEntry(ShStrTabBuilder.getOffset(Section.getSectionName()),
-                   Section.getType(),
-                   Section.getFlags(), 0, Offset, Size, sh_link, sh_info,
-                   Alignment, Section.getEntrySize());
-}
-
-bool ELFObjectWriter::IsELFMetaDataSection(const MCSectionData &SD) {
-  return SD.getOrdinal() == ~UINT32_C(0) &&
-    !SD.getSection().isVirtualSection();
-}
-
-uint64_t ELFObjectWriter::DataSectionSize(const MCSectionData &SD) {
-  uint64_t Ret = 0;
-  for (MCSectionData::const_iterator i = SD.begin(), e = SD.end(); i != e;
-       ++i) {
-    const MCFragment &F = *i;
-    assert(F.getKind() == MCFragment::FT_Data);
-    Ret += cast<MCDataFragment>(F).getContents().size();
-  }
-  return Ret;
-}
-
-uint64_t ELFObjectWriter::GetSectionAddressSize(const MCAsmLayout &Layout,
-                                                const MCSectionData &SD) {
-  if (IsELFMetaDataSection(SD))
-    return DataSectionSize(SD);
-  return Layout.getSectionAddressSize(&SD);
-}
-
-void ELFObjectWriter::writeDataSectionData(MCAssembler &Asm,
-                                           const MCAsmLayout &Layout,
-                                           const MCSectionData &SD) {
-  if (IsELFMetaDataSection(SD)) {
-    for (MCSectionData::const_iterator i = SD.begin(), e = SD.end(); i != e;
-         ++i) {
-      const MCFragment &F = *i;
-      assert(F.getKind() == MCFragment::FT_Data);
-      WriteBytes(cast<MCDataFragment>(F).getContents());
-    }
-  } else {
-    Asm.writeSectionData(&SD, Layout);
-  }
+  WriteSecHdrEntry(StrTabBuilder.getOffset(Section.getSectionName()),
+                   Section.getType(), Section.getFlags(), 0, Offset, Size,
+                   sh_link, sh_info, Section.getAlignment(),
+                   Section.getEntrySize());
 }
 
 void ELFObjectWriter::writeSectionHeader(
-    ArrayRef<const MCSectionELF *> Sections, MCAssembler &Asm,
-    const GroupMapTy &GroupMap, const MCAsmLayout &Layout,
-    const SectionIndexMapTy &SectionIndexMap,
-    const SectionOffsetMapTy &SectionOffsetMap) {
-  const unsigned NumSections = Asm.size();
+    const MCAsmLayout &Layout, const SectionIndexMapTy &SectionIndexMap,
+    const SectionOffsetsTy &SectionOffsets) {
+  const unsigned NumSections = SectionTable.size();
 
   // Null section first.
   uint64_t FirstSectionSize =
       (NumSections + 1) >= ELF::SHN_LORESERVE ? NumSections + 1 : 0;
-  uint32_t FirstSectionLink =
-    ShstrtabIndex >= ELF::SHN_LORESERVE ? ShstrtabIndex : 0;
-  WriteSecHdrEntry(0, 0, 0, 0, 0, FirstSectionSize, FirstSectionLink, 0, 0, 0);
+  WriteSecHdrEntry(0, 0, 0, 0, 0, FirstSectionSize, 0, 0, 0, 0);
 
-  for (unsigned i = 0; i < NumSections; ++i) {
-    const MCSectionELF &Section = *Sections[i];
-    const MCSectionData &SD = Asm.getOrCreateSectionData(Section);
+  for (const MCSectionELF *Section : SectionTable) {
     uint32_t GroupSymbolIndex;
-    if (Section.getType() != ELF::SHT_GROUP)
+    unsigned Type = Section->getType();
+    if (Type != ELF::SHT_GROUP)
       GroupSymbolIndex = 0;
     else
-      GroupSymbolIndex = getSymbolIndexInSymbolTable(Asm,
-                                                     GroupMap.lookup(&Section));
+      GroupSymbolIndex = Section->getGroup()->getIndex();
 
-    uint64_t Size = GetSectionAddressSize(Layout, SD);
+    const std::pair<uint64_t, uint64_t> &Offsets =
+        SectionOffsets.find(Section)->second;
+    uint64_t Size;
+    if (Type == ELF::SHT_NOBITS)
+      Size = Layout.getSectionAddressSize(Section);
+    else
+      Size = Offsets.second - Offsets.first;
 
-    writeSection(Asm, SectionIndexMap, GroupSymbolIndex,
-                 SectionOffsetMap.lookup(&Section), Size, SD.getAlignment(),
-                 Section);
+    writeSection(SectionIndexMap, GroupSymbolIndex, Offsets.first, Size,
+                 *Section);
   }
 }
 
-void ELFObjectWriter::WriteObject(MCAssembler &Asm,
+void ELFObjectWriter::writeObject(MCAssembler &Asm,
                                   const MCAsmLayout &Layout) {
-  GroupMapTy GroupMap;
+  MCContext &Ctx = Asm.getContext();
+  MCSectionELF *StrtabSection =
+      Ctx.getELFSection(".strtab", ELF::SHT_STRTAB, 0);
+  StringTableIndex = addToSectionTable(StrtabSection);
+
   RevGroupMapTy RevGroupMap;
   SectionIndexMapTy SectionIndexMap;
 
-  CompressDebugSections(Asm, const_cast<MCAsmLayout &>(Layout));
-  createIndexedSections(Asm, const_cast<MCAsmLayout &>(Layout), GroupMap,
-                        RevGroupMap, SectionIndexMap);
+  std::map<const MCSymbol *, std::vector<const MCSectionELF *>> GroupMembers;
 
-  // Compute symbol table information.
-  computeSymbolTable(Asm, Layout, SectionIndexMap, RevGroupMap);
+  // Write out the ELF header ...
+  writeHeader(Asm);
+
+  // ... then the sections ...
+  SectionOffsetsTy SectionOffsets;
+  std::vector<MCSectionELF *> Groups;
+  std::vector<MCSectionELF *> Relocations;
+  for (MCSection &Sec : Asm) {
+    MCSectionELF &Section = static_cast<MCSectionELF &>(Sec);
 
-  WriteRelocations(Asm, const_cast<MCAsmLayout &>(Layout));
+    align(Section.getAlignment());
 
-  CreateMetadataSections(const_cast<MCAssembler&>(Asm),
-                         const_cast<MCAsmLayout&>(Layout),
-                         SectionIndexMap);
+    // Remember the offset into the file for this section.
+    uint64_t SecStart = OS.tell();
 
-  unsigned NumSections = Asm.size();
-  std::vector<const MCSectionELF*> Sections;
-  Sections.resize(NumSections);
+    const MCSymbolELF *SignatureSymbol = Section.getGroup();
+    writeSectionData(Asm, Section, Layout);
 
-  for (auto &Pair : SectionIndexMap)
-    Sections[Pair.second - 1] = Pair.first;
+    uint64_t SecEnd = OS.tell();
+    SectionOffsets[&Section] = std::make_pair(SecStart, SecEnd);
 
-  SectionOffsetMapTy SectionOffsetMap;
+    MCSectionELF *RelSection = createRelocationSection(Ctx, Section);
 
-  // Write out the ELF header ...
-  WriteHeader(Asm, NumSections + 1);
+    if (SignatureSymbol) {
+      Asm.registerSymbol(*SignatureSymbol);
+      unsigned &GroupIdx = RevGroupMap[SignatureSymbol];
+      if (!GroupIdx) {
+        MCSectionELF *Group = Ctx.createELFGroupSection(SignatureSymbol);
+        GroupIdx = addToSectionTable(Group);
+        Group->setAlignment(4);
+        Groups.push_back(Group);
+      }
+      std::vector<const MCSectionELF *> &Members =
+          GroupMembers[SignatureSymbol];
+      Members.push_back(&Section);
+      if (RelSection)
+        Members.push_back(RelSection);
+    }
 
-  // ... then the sections ...
-  for (unsigned i = 0; i < NumSections; ++i) {
-    const MCSectionELF &Section = *Sections[i];
-    const MCSectionData &SD = Asm.getOrCreateSectionData(Section);
-    uint64_t Padding = OffsetToAlignment(OS.tell(), SD.getAlignment());
-    WriteZeros(Padding);
+    SectionIndexMap[&Section] = addToSectionTable(&Section);
+    if (RelSection) {
+      SectionIndexMap[RelSection] = addToSectionTable(RelSection);
+      Relocations.push_back(RelSection);
+    }
+  }
+
+  for (MCSectionELF *Group : Groups) {
+    align(Group->getAlignment());
+
+    // Remember the offset into the file for this section.
+    uint64_t SecStart = OS.tell();
+
+    const MCSymbol *SignatureSymbol = Group->getGroup();
+    assert(SignatureSymbol);
+    write(uint32_t(ELF::GRP_COMDAT));
+    for (const MCSectionELF *Member : GroupMembers[SignatureSymbol]) {
+      uint32_t SecIndex = SectionIndexMap.lookup(Member);
+      write(SecIndex);
+    }
+
+    uint64_t SecEnd = OS.tell();
+    SectionOffsets[Group] = std::make_pair(SecStart, SecEnd);
+  }
+
+  // Compute symbol table information.
+  computeSymbolTable(Asm, Layout, SectionIndexMap, RevGroupMap, SectionOffsets);
+
+  for (MCSectionELF *RelSection : Relocations) {
+    align(RelSection->getAlignment());
 
     // Remember the offset into the file for this section.
-    SectionOffsetMap[&Section] = OS.tell();
+    uint64_t SecStart = OS.tell();
+
+    writeRelocations(Asm, *RelSection->getAssociatedSection());
+
+    uint64_t SecEnd = OS.tell();
+    SectionOffsets[RelSection] = std::make_pair(SecStart, SecEnd);
+  }
 
-    writeDataSectionData(Asm, Layout, SD);
+  {
+    uint64_t SecStart = OS.tell();
+    const MCSectionELF *Sec = createStringTable(Ctx);
+    uint64_t SecEnd = OS.tell();
+    SectionOffsets[Sec] = std::make_pair(SecStart, SecEnd);
   }
 
   uint64_t NaturalAlignment = is64Bit() ? 8 : 4;
-  uint64_t Padding = OffsetToAlignment(OS.tell(), NaturalAlignment);
-  WriteZeros(Padding);
+  align(NaturalAlignment);
 
   const unsigned SectionHeaderOffset = OS.tell();
 
   // ... then the section header table ...
-  writeSectionHeader(Sections, Asm, GroupMap, Layout, SectionIndexMap,
-                     SectionOffsetMap);
+  writeSectionHeader(Layout, SectionIndexMap, SectionOffsets);
+
+  uint16_t NumSections = (SectionTable.size() + 1 >= ELF::SHN_LORESERVE)
+                             ? (uint16_t)ELF::SHN_UNDEF
+                             : SectionTable.size() + 1;
+  if (sys::IsLittleEndianHost != IsLittleEndian)
+    sys::swapByteOrder(NumSections);
+  unsigned NumSectionsOffset;
 
   if (is64Bit()) {
     uint64_t Val = SectionHeaderOffset;
@@ -1639,27 +1303,51 @@ void ELFObjectWriter::WriteObject(MCAssembler &Asm,
       sys::swapByteOrder(Val);
     OS.pwrite(reinterpret_cast<char *>(&Val), sizeof(Val),
               offsetof(ELF::Elf64_Ehdr, e_shoff));
+    NumSectionsOffset = offsetof(ELF::Elf64_Ehdr, e_shnum);
   } else {
     uint32_t Val = SectionHeaderOffset;
     if (sys::IsLittleEndianHost != IsLittleEndian)
       sys::swapByteOrder(Val);
     OS.pwrite(reinterpret_cast<char *>(&Val), sizeof(Val),
               offsetof(ELF::Elf32_Ehdr, e_shoff));
+    NumSectionsOffset = offsetof(ELF::Elf32_Ehdr, e_shnum);
   }
+  OS.pwrite(reinterpret_cast<char *>(&NumSections), sizeof(NumSections),
+            NumSectionsOffset);
 }
 
-bool ELFObjectWriter::IsSymbolRefDifferenceFullyResolvedImpl(
-    const MCAssembler &Asm, const MCSymbolData &DataA,
-    const MCSymbolData *DataB, const MCFragment &FB, bool InSet,
-    bool IsPCRel) const {
-  if (!InSet && (::isWeak(DataA) || (DataB && ::isWeak(*DataB))))
-    return false;
-  return MCObjectWriter::IsSymbolRefDifferenceFullyResolvedImpl(
-      Asm, DataA, DataB, FB, InSet, IsPCRel);
+bool ELFObjectWriter::isSymbolRefDifferenceFullyResolvedImpl(
+    const MCAssembler &Asm, const MCSymbol &SA, const MCFragment &FB,
+    bool InSet, bool IsPCRel) const {
+  const auto &SymA = cast<MCSymbolELF>(SA);
+  if (IsPCRel) {
+    assert(!InSet);
+    if (::isWeak(SymA))
+      return false;
+  }
+  return MCObjectWriter::isSymbolRefDifferenceFullyResolvedImpl(Asm, SymA, FB,
+                                                                InSet, IsPCRel);
 }
 
-bool ELFObjectWriter::isWeak(const MCSymbolData &SD) const {
-  return ::isWeak(SD);
+bool ELFObjectWriter::isWeak(const MCSymbol &S) const {
+  const auto &Sym = cast<MCSymbolELF>(S);
+  if (::isWeak(Sym))
+    return true;
+
+  // It is invalid to replace a reference to a global in a comdat
+  // with a reference to a local since out of comdat references
+  // to a local are forbidden.
+  // We could try to return false for more cases, like the reference
+  // being in the same comdat or Sym being an alias to another global,
+  // but it is not clear if it is worth the effort.
+  if (Sym.getBinding() != ELF::STB_GLOBAL)
+    return false;
+
+  if (!Sym.isInSection())
+    return false;
+
+  const auto &Sec = cast<MCSectionELF>(Sym.getSection());
+  return Sec.getGroup();
 }
 
 MCObjectWriter *llvm::createELFObjectWriter(MCELFObjectTargetWriter *MOTW,
diff --git a/lib/MC/MCAsmBackend.cpp b/lib/MC/MCAsmBackend.cpp
index c42757b70c8..36c65b7bcd4 100644
--- a/lib/MC/MCAsmBackend.cpp
+++ b/lib/MC/MCAsmBackend.cpp
@@ -16,27 +16,33 @@ MCAsmBackend::MCAsmBackend() : HasDataInCodeSupport(false) {}
 
 MCAsmBackend::~MCAsmBackend() {}
 
-const MCFixupKindInfo &
-MCAsmBackend::getFixupKindInfo(MCFixupKind Kind) const {
+const MCFixupKindInfo &MCAsmBackend::getFixupKindInfo(MCFixupKind Kind) const {
   static const MCFixupKindInfo Builtins[] = {
-    { "FK_Data_1",  0,  8, 0 },
-    { "FK_Data_2",  0, 16, 0 },
-    { "FK_Data_4",  0, 32, 0 },
-    { "FK_Data_8",  0, 64, 0 },
-    { "FK_PCRel_1", 0,  8, MCFixupKindInfo::FKF_IsPCRel },
-    { "FK_PCRel_2", 0, 16, MCFixupKindInfo::FKF_IsPCRel },
-    { "FK_PCRel_4", 0, 32, MCFixupKindInfo::FKF_IsPCRel },
-    { "FK_PCRel_8", 0, 64, MCFixupKindInfo::FKF_IsPCRel },
-    { "FK_GPRel_1", 0,  8, 0 },
-    { "FK_GPRel_2", 0, 16, 0 },
-    { "FK_GPRel_4", 0, 32, 0 },
-    { "FK_GPRel_8", 0, 64, 0 },
-    { "FK_SecRel_1", 0,  8, 0 },
-    { "FK_SecRel_2", 0, 16, 0 },
-    { "FK_SecRel_4", 0, 32, 0 },
-    { "FK_SecRel_8", 0, 64, 0 }
-  };
+      {"FK_Data_1", 0, 8, 0},
+      {"FK_Data_2", 0, 16, 0},
+      {"FK_Data_4", 0, 32, 0},
+      {"FK_Data_8", 0, 64, 0},
+      {"FK_PCRel_1", 0, 8, MCFixupKindInfo::FKF_IsPCRel},
+      {"FK_PCRel_2", 0, 16, MCFixupKindInfo::FKF_IsPCRel},
+      {"FK_PCRel_4", 0, 32, MCFixupKindInfo::FKF_IsPCRel},
+      {"FK_PCRel_8", 0, 64, MCFixupKindInfo::FKF_IsPCRel},
+      {"FK_GPRel_1", 0, 8, 0},
+      {"FK_GPRel_2", 0, 16, 0},
+      {"FK_GPRel_4", 0, 32, 0},
+      {"FK_GPRel_8", 0, 64, 0},
+      {"FK_SecRel_1", 0, 8, 0},
+      {"FK_SecRel_2", 0, 16, 0},
+      {"FK_SecRel_4", 0, 32, 0},
+      {"FK_SecRel_8", 0, 64, 0}};
 
   assert((size_t)Kind <= array_lengthof(Builtins) && "Unknown fixup kind");
   return Builtins[Kind];
 }
+
+bool MCAsmBackend::fixupNeedsRelaxationAdvanced(
+    const MCFixup &Fixup, bool Resolved, uint64_t Value,
+    const MCRelaxableFragment *DF, const MCAsmLayout &Layout) const {
+  if (!Resolved)
+    return true;
+  return fixupNeedsRelaxation(Fixup, Value, DF, Layout);
+}
diff --git a/lib/MC/MCAsmInfo.cpp b/lib/MC/MCAsmInfo.cpp
index bad257a961b..100dc7c3dc6 100644
--- a/lib/MC/MCAsmInfo.cpp
+++ b/lib/MC/MCAsmInfo.cpp
@@ -50,6 +50,7 @@ MCAsmInfo::MCAsmInfo() {
   Code64Directive = ".code64";
   AssemblerDialect = 0;
   AllowAtInName = false;
+  SupportsQuotedNames = true;
   UseDataRegionDirectives = false;
   ZeroDirective = "\t.zero\t";
   AsciiDirective = "\t.ascii\t";
@@ -90,6 +91,7 @@ MCAsmInfo::MCAsmInfo() {
   DwarfRegNumForCFI = false;
   NeedsDwarfSectionOffsetDirective = false;
   UseParensForSymbolVariant = false;
+  UseLogicalShr = true;
 
   // FIXME: Clang's logic should be synced with the logic used to initialize
   //        this member and the two implementations should be merged.
@@ -127,12 +129,31 @@ MCAsmInfo::getExprForFDESymbol(const MCSymbol *Sym,
                                unsigned Encoding,
                                MCStreamer &Streamer) const {
   if (!(Encoding & dwarf::DW_EH_PE_pcrel))
-    return MCSymbolRefExpr::Create(Sym, Streamer.getContext());
+    return MCSymbolRefExpr::create(Sym, Streamer.getContext());
 
   MCContext &Context = Streamer.getContext();
-  const MCExpr *Res = MCSymbolRefExpr::Create(Sym, Context);
-  MCSymbol *PCSym = Context.CreateTempSymbol();
+  const MCExpr *Res = MCSymbolRefExpr::create(Sym, Context);
+  MCSymbol *PCSym = Context.createTempSymbol();
   Streamer.EmitLabel(PCSym);
-  const MCExpr *PC = MCSymbolRefExpr::Create(PCSym, Context);
-  return MCBinaryExpr::CreateSub(Res, PC, Context);
+  const MCExpr *PC = MCSymbolRefExpr::create(PCSym, Context);
+  return MCBinaryExpr::createSub(Res, PC, Context);
+}
+
+static bool isAcceptableChar(char C) {
+  return (C >= 'a' && C <= 'z') || (C >= 'A' && C <= 'Z') ||
+         (C >= '0' && C <= '9') || C == '_' || C == '$' || C == '.' || C == '@';
+}
+
+bool MCAsmInfo::isValidUnquotedName(StringRef Name) const {
+  if (Name.empty())
+    return false;
+
+  // If any of the characters in the string is an unacceptable character, force
+  // quotes.
+  for (char C : Name) {
+    if (!isAcceptableChar(C))
+      return false;
+  }
+
+  return true;
 }
diff --git a/lib/MC/MCAsmInfoCOFF.cpp b/lib/MC/MCAsmInfoCOFF.cpp
index bb3f0d3a429..97fc76a9adb 100644
--- a/lib/MC/MCAsmInfoCOFF.cpp
+++ b/lib/MC/MCAsmInfoCOFF.cpp
@@ -36,6 +36,10 @@ MCAsmInfoCOFF::MCAsmInfoCOFF() {
   NeedsDwarfSectionOffsetDirective = true;
 
   UseIntegratedAssembler = true;
+
+  // FIXME: For now keep the previous behavior, AShr. Need to double-check
+  // other COFF-targeting assemblers and change this if necessary.
+  UseLogicalShr = false;
 }
 
 void MCAsmInfoMicrosoft::anchor() { }
diff --git a/lib/MC/MCAsmInfoDarwin.cpp b/lib/MC/MCAsmInfoDarwin.cpp
index ae9486d3db4..bb90ff2c350 100644
--- a/lib/MC/MCAsmInfoDarwin.cpp
+++ b/lib/MC/MCAsmInfoDarwin.cpp
@@ -93,4 +93,9 @@ MCAsmInfoDarwin::MCAsmInfoDarwin() {
 
   UseIntegratedAssembler = true;
   SetDirectiveSuppressesReloc = true;
+
+  // FIXME: For now keep the previous behavior, AShr, matching the previous
+  // behavior of as(1) (both -q and -Q: resp. LLVM and gas v1.38).
+  // If/when this changes, the AArch64 Darwin special case can go away.
+  UseLogicalShr = false;
 }
diff --git a/lib/MC/MCAsmInfoELF.cpp b/lib/MC/MCAsmInfoELF.cpp
index cd61a43afc4..2bff6e05966 100644
--- a/lib/MC/MCAsmInfoELF.cpp
+++ b/lib/MC/MCAsmInfoELF.cpp
@@ -20,8 +20,7 @@ using namespace llvm;
 
 void MCAsmInfoELF::anchor() { }
 
-const MCSection *
-MCAsmInfoELF::getNonexecutableStackSection(MCContext &Ctx) const {
+MCSection *MCAsmInfoELF::getNonexecutableStackSection(MCContext &Ctx) const {
   return Ctx.getELFSection(".note.GNU-stack", ELF::SHT_PROGBITS, 0);
 }
 
diff --git a/lib/MC/MCAsmStreamer.cpp b/lib/MC/MCAsmStreamer.cpp
index b9acf1611d5..6fdc08670f1 100644
--- a/lib/MC/MCAsmStreamer.cpp
+++ b/lib/MC/MCAsmStreamer.cpp
@@ -14,7 +14,6 @@
 #include "llvm/ADT/Twine.h"
 #include "llvm/MC/MCAsmBackend.h"
 #include "llvm/MC/MCAsmInfo.h"
-#include "llvm/MC/MCNaClExpander.h" // @LOCALMOD
 #include "llvm/MC/MCCodeEmitter.h"
 #include "llvm/MC/MCContext.h"
 #include "llvm/MC/MCExpr.h"
@@ -25,7 +24,7 @@
 #include "llvm/MC/MCRegisterInfo.h"
 #include "llvm/MC/MCSectionCOFF.h"
 #include "llvm/MC/MCSectionMachO.h"
-#include "llvm/MC/MCSymbol.h"
+#include "llvm/MC/MCSymbolELF.h"
 #include "llvm/Support/CommandLine.h"
 #include "llvm/Support/ErrorHandling.h"
 #include "llvm/Support/Format.h"
@@ -51,9 +50,6 @@ class MCAsmStreamer final : public MCStreamer {
   unsigned IsVerboseAsm : 1;
   unsigned ShowInst : 1;
   unsigned UseDwarfDirectory : 1;
-  // @LOCALMOD: we don't have an MCAssembler object here, so we can't ask it
-  // if bundle alignment is enabled. Instead, just track the alignment here.
-  unsigned BundleAlignmentEnabled : 1;
 
   void EmitRegisterName(int64_t Register);
   void EmitCFIStartProcImpl(MCDwarfFrameInfo &Frame) override;
@@ -68,10 +64,10 @@ class MCAsmStreamer final : public MCStreamer {
         MAI(Context.getAsmInfo()), InstPrinter(printer), Emitter(emitter),
         AsmBackend(asmbackend), CommentStream(CommentToEmit),
         IsVerboseAsm(isVerboseAsm), ShowInst(showInst),
-        UseDwarfDirectory(useDwarfDirectory),
-        BundleAlignmentEnabled(0) {
-    if (InstPrinter && IsVerboseAsm)
-      InstPrinter->setCommentStream(CommentStream);
+        UseDwarfDirectory(useDwarfDirectory) {
+    assert(InstPrinter);
+    if (IsVerboseAsm)
+        InstPrinter->setCommentStream(CommentStream);
   }
 
   inline void EmitEOL() {
@@ -82,6 +78,9 @@ class MCAsmStreamer final : public MCStreamer {
     }
     EmitCommentsAndEOL();
   }
+
+  void EmitSyntaxDirective() override;
+
   void EmitCommentsAndEOL();
 
   /// isVerboseAsm - Return true if this streamer supports verbose assembly at
@@ -119,8 +118,7 @@ class MCAsmStreamer final : public MCStreamer {
   /// @name MCStreamer Interface
   /// @{
 
-  void ChangeSection(const MCSection *Section,
-                     const MCExpr *Subsection) override;
+  void ChangeSection(MCSection *Section, const MCExpr *Subsection) override;
 
   void EmitLOHDirective(MCLOHType Kind, const MCLOHArgs &Args) override;
   void EmitLabel(MCSymbol *Symbol) override;
@@ -141,9 +139,10 @@ class MCAsmStreamer final : public MCStreamer {
   void EmitCOFFSymbolStorageClass(int StorageClass) override;
   void EmitCOFFSymbolType(int Type) override;
   void EndCOFFSymbolDef() override;
+  void EmitCOFFSafeSEH(MCSymbol const *Symbol) override;
   void EmitCOFFSectionIndex(MCSymbol const *Symbol) override;
   void EmitCOFFSecRel32(MCSymbol const *Symbol) override;
-  void EmitELFSize(MCSymbol *Symbol, const MCExpr *Value) override;
+  void emitELFSize(MCSymbolELF *Symbol, const MCExpr *Value) override;
   void EmitCommonSymbol(MCSymbol *Symbol, uint64_t Size,
                         unsigned ByteAlignment) override;
 
@@ -155,11 +154,11 @@ class MCAsmStreamer final : public MCStreamer {
   void EmitLocalCommonSymbol(MCSymbol *Symbol, uint64_t Size,
                              unsigned ByteAlignment) override;
 
-  void EmitZerofill(const MCSection *Section, MCSymbol *Symbol = nullptr,
+  void EmitZerofill(MCSection *Section, MCSymbol *Symbol = nullptr,
                     uint64_t Size = 0, unsigned ByteAlignment = 0) override;
 
-  void EmitTBSSSymbol (const MCSection *Section, MCSymbol *Symbol,
-                       uint64_t Size, unsigned ByteAlignment = 0) override;
+  void EmitTBSSSymbol(MCSection *Section, MCSymbol *Symbol, uint64_t Size,
+                      unsigned ByteAlignment = 0) override;
 
   void EmitBytes(StringRef Data) override;
 
@@ -302,7 +301,7 @@ void MCAsmStreamer::emitRawComment(const Twine &T, bool TabPrefix) {
   EmitEOL();
 }
 
-void MCAsmStreamer::ChangeSection(const MCSection *Section,
+void MCAsmStreamer::ChangeSection(MCSection *Section,
                                   const MCExpr *Subsection) {
   assert(Section && "Cannot switch to a null section!");
   Section->PrintSwitchToSection(*MAI, OS, Subsection);
@@ -312,7 +311,9 @@ void MCAsmStreamer::EmitLabel(MCSymbol *Symbol) {
   assert(Symbol->isUndefined() && "Cannot define a symbol twice!");
   MCStreamer::EmitLabel(Symbol);
 
-  OS << *Symbol << MAI->getLabelSuffix();
+  Symbol->print(OS, MAI);
+  OS << MAI->getLabelSuffix();
+
   EmitEOL();
 }
 
@@ -332,7 +333,7 @@ void MCAsmStreamer::EmitLOHDirective(MCLOHType Kind, const MCLOHArgs &Args) {
     if (!IsFirst)
       OS << ", ";
     IsFirst = false;
-    OS << **It;
+    (*It)->print(OS, MAI);
   }
   EmitEOL();
 }
@@ -388,20 +389,28 @@ void MCAsmStreamer::EmitThumbFunc(MCSymbol *Func) {
   // MCSymbols when they have spaces in them.
   OS << "\t.thumb_func";
   // Only Mach-O hasSubsectionsViaSymbols()
-  if (MAI->hasSubsectionsViaSymbols())
-    OS << '\t' << *Func;
+  if (MAI->hasSubsectionsViaSymbols()) {
+    OS << '\t';
+    Func->print(OS, MAI);
+  }
   EmitEOL();
 }
 
 void MCAsmStreamer::EmitAssignment(MCSymbol *Symbol, const MCExpr *Value) {
-  OS << *Symbol << " = " << *Value;
+  Symbol->print(OS, MAI);
+  OS << " = ";
+  Value->print(OS, MAI);
+
   EmitEOL();
 
   MCStreamer::EmitAssignment(Symbol, Value);
 }
 
 void MCAsmStreamer::EmitWeakReference(MCSymbol *Alias, const MCSymbol *Symbol) {
-  OS << ".weakref " << *Alias << ", " << *Symbol;
+  OS << ".weakref ";
+  Alias->print(OS, MAI);
+  OS << ", ";
+  Symbol->print(OS, MAI);
   EmitEOL();
 }
 
@@ -418,8 +427,9 @@ bool MCAsmStreamer::EmitSymbolAttribute(MCSymbol *Symbol,
   case MCSA_ELF_TypeGnuUniqueObject:  /// .type _foo, @gnu_unique_object
     if (!MAI->hasDotTypeDotSizeDirective())
       return false; // Symbol attribute not supported
-    OS << "\t.type\t" << *Symbol << ','
-       << ((MAI->getCommentString()[0] != '@') ? '@' : '%');
+    OS << "\t.type\t";
+    Symbol->print(OS, MAI);
+    OS << ',' << ((MAI->getCommentString()[0] != '@') ? '@' : '%');
     switch (Attribute) {
     default: return false;
     case MCSA_ELF_TypeFunction:    OS << "function"; break;
@@ -460,19 +470,31 @@ bool MCAsmStreamer::EmitSymbolAttribute(MCSymbol *Symbol,
   case MCSA_WeakDefAutoPrivate: OS << "\t.weak_def_can_be_hidden\t"; break;
   }
 
-  OS << *Symbol;
+  Symbol->print(OS, MAI);
   EmitEOL();
 
   return true;
 }
 
 void MCAsmStreamer::EmitSymbolDesc(MCSymbol *Symbol, unsigned DescValue) {
-  OS << ".desc" << ' ' << *Symbol << ',' << DescValue;
+  OS << ".desc" << ' ';
+  Symbol->print(OS, MAI);
+  OS << ',' << DescValue;
   EmitEOL();
 }
 
+void MCAsmStreamer::EmitSyntaxDirective() {
+  if (MAI->getAssemblerDialect() == 1)
+    OS << "\t.intel_syntax noprefix\n";
+  // FIXME: Currently emit unprefix'ed registers.
+  // The intel_syntax directive has one optional argument 
+  // with may have a value of prefix or noprefix.
+}
+
 void MCAsmStreamer::BeginCOFFSymbolDef(const MCSymbol *Symbol) {
-  OS << "\t.def\t " << *Symbol << ';';
+  OS << "\t.def\t ";
+  Symbol->print(OS, MAI);
+  OS << ';';
   EmitEOL();
 }
 
@@ -491,19 +513,31 @@ void MCAsmStreamer::EndCOFFSymbolDef() {
   EmitEOL();
 }
 
+void MCAsmStreamer::EmitCOFFSafeSEH(MCSymbol const *Symbol) {
+  OS << "\t.safeseh\t";
+  Symbol->print(OS, MAI);
+  EmitEOL();
+}
+
 void MCAsmStreamer::EmitCOFFSectionIndex(MCSymbol const *Symbol) {
-  OS << "\t.secidx\t" << *Symbol;
+  OS << "\t.secidx\t";
+  Symbol->print(OS, MAI);
   EmitEOL();
 }
 
 void MCAsmStreamer::EmitCOFFSecRel32(MCSymbol const *Symbol) {
-  OS << "\t.secrel32\t" << *Symbol;
+  OS << "\t.secrel32\t";
+  Symbol->print(OS, MAI);
   EmitEOL();
 }
 
-void MCAsmStreamer::EmitELFSize(MCSymbol *Symbol, const MCExpr *Value) {
+void MCAsmStreamer::emitELFSize(MCSymbolELF *Symbol, const MCExpr *Value) {
   assert(MAI->hasDotTypeDotSizeDirective());
-  OS << "\t.size\t" << *Symbol << ", " << *Value << '\n';
+  OS << "\t.size\t";
+  Symbol->print(OS, MAI);
+  OS << ", ";
+  Value->print(OS, MAI);
+  OS << '\n';
 }
 
 void MCAsmStreamer::EmitCommonSymbol(MCSymbol *Symbol, uint64_t Size,
@@ -511,7 +545,10 @@ void MCAsmStreamer::EmitCommonSymbol(MCSymbol *Symbol, uint64_t Size,
   // Common symbols do not belong to any actual section.
   AssignSection(Symbol, nullptr);
 
-  OS << "\t.comm\t" << *Symbol << ',' << Size;
+  OS << "\t.comm\t";
+  Symbol->print(OS, MAI);
+  OS << ',' << Size;
+
   if (ByteAlignment != 0) {
     if (MAI->getCOMMDirectiveAlignmentIsInBytes())
       OS << ',' << ByteAlignment;
@@ -530,7 +567,10 @@ void MCAsmStreamer::EmitLocalCommonSymbol(MCSymbol *Symbol, uint64_t Size,
   // Common symbols do not belong to any actual section.
   AssignSection(Symbol, nullptr);
 
-  OS << "\t.lcomm\t" << *Symbol << ',' << Size;
+  OS << "\t.lcomm\t";
+  Symbol->print(OS, MAI);
+  OS << ',' << Size;
+
   if (ByteAlign > 1) {
     switch (MAI->getLCOMMDirectiveAlignmentType()) {
     case LCOMM::NoAlignment:
@@ -547,7 +587,7 @@ void MCAsmStreamer::EmitLocalCommonSymbol(MCSymbol *Symbol, uint64_t Size,
   EmitEOL();
 }
 
-void MCAsmStreamer::EmitZerofill(const MCSection *Section, MCSymbol *Symbol,
+void MCAsmStreamer::EmitZerofill(MCSection *Section, MCSymbol *Symbol,
                                  uint64_t Size, unsigned ByteAlignment) {
   if (Symbol)
     AssignSection(Symbol, Section);
@@ -560,7 +600,9 @@ void MCAsmStreamer::EmitZerofill(const MCSection *Section, MCSymbol *Symbol,
   OS << MOSection->getSegmentName() << "," << MOSection->getSectionName();
 
   if (Symbol) {
-    OS << ',' << *Symbol << ',' << Size;
+    OS << ',';
+    Symbol->print(OS, MAI);
+    OS << ',' << Size;
     if (ByteAlignment != 0)
       OS << ',' << Log2_32(ByteAlignment);
   }
@@ -570,14 +612,16 @@ void MCAsmStreamer::EmitZerofill(const MCSection *Section, MCSymbol *Symbol,
 // .tbss sym, size, align
 // This depends that the symbol has already been mangled from the original,
 // e.g. _a.
-void MCAsmStreamer::EmitTBSSSymbol(const MCSection *Section, MCSymbol *Symbol,
+void MCAsmStreamer::EmitTBSSSymbol(MCSection *Section, MCSymbol *Symbol,
                                    uint64_t Size, unsigned ByteAlignment) {
   AssignSection(Symbol, Section);
 
   assert(Symbol && "Symbol shouldn't be NULL!");
   // Instead of using the Section we'll just use the shortcut.
   // This is a mach-o specific directive and section.
-  OS << ".tbss " << *Symbol << ", " << Size;
+  OS << ".tbss ";
+  Symbol->print(OS, MAI);
+  OS << ", " << Size;
 
   // Output align if we have it.  We default to 1 so don't bother printing
   // that.
@@ -648,7 +692,7 @@ void MCAsmStreamer::EmitBytes(StringRef Data) {
 }
 
 void MCAsmStreamer::EmitIntValue(uint64_t Value, unsigned Size) {
-  EmitValue(MCConstantExpr::Create(Value, getContext()), Size);
+  EmitValue(MCConstantExpr::create(Value, getContext()), Size);
 }
 
 void MCAsmStreamer::EmitValueImpl(const MCExpr *Value, unsigned Size,
@@ -667,7 +711,7 @@ void MCAsmStreamer::EmitValueImpl(const MCExpr *Value, unsigned Size,
 
   if (!Directive) {
     int64_t IntValue;
-    if (!Value->EvaluateAsAbsolute(IntValue))
+    if (!Value->evaluateAsAbsolute(IntValue))
       report_fatal_error("Don't know how to emit this value.");
 
     // We couldn't handle the requested integer size so we fallback by breaking
@@ -702,39 +746,44 @@ void MCAsmStreamer::EmitValueImpl(const MCExpr *Value, unsigned Size,
   }
 
   assert(Directive && "Invalid size for machine code value!");
-  OS << Directive << *Value;
+  OS << Directive;
+  Value->print(OS, MAI);
   EmitEOL();
 }
 
 void MCAsmStreamer::EmitULEB128Value(const MCExpr *Value) {
   int64_t IntValue;
-  if (Value->EvaluateAsAbsolute(IntValue)) {
+  if (Value->evaluateAsAbsolute(IntValue)) {
     EmitULEB128IntValue(IntValue);
     return;
   }
-  OS << ".uleb128 " << *Value;
+  OS << ".uleb128 ";
+  Value->print(OS, MAI);
   EmitEOL();
 }
 
 void MCAsmStreamer::EmitSLEB128Value(const MCExpr *Value) {
   int64_t IntValue;
-  if (Value->EvaluateAsAbsolute(IntValue)) {
+  if (Value->evaluateAsAbsolute(IntValue)) {
     EmitSLEB128IntValue(IntValue);
     return;
   }
-  OS << ".sleb128 " << *Value;
+  OS << ".sleb128 ";
+  Value->print(OS, MAI);
   EmitEOL();
 }
 
 void MCAsmStreamer::EmitGPRel64Value(const MCExpr *Value) {
   assert(MAI->getGPRel64Directive() != nullptr);
-  OS << MAI->getGPRel64Directive() << *Value;
+  OS << MAI->getGPRel64Directive();
+  Value->print(OS, MAI);
   EmitEOL();
 }
 
 void MCAsmStreamer::EmitGPRel32Value(const MCExpr *Value) {
   assert(MAI->getGPRel32Directive() != nullptr);
-  OS << MAI->getGPRel32Directive() << *Value;
+  OS << MAI->getGPRel32Directive();
+  Value->print(OS, MAI);
   EmitEOL();
 }
 
@@ -821,7 +870,9 @@ void MCAsmStreamer::EmitCodeAlignment(unsigned ByteAlignment,
 bool MCAsmStreamer::EmitValueToOffset(const MCExpr *Offset,
                                       unsigned char Value) {
   // FIXME: Verify that Offset is associated with the current section.
-  OS << ".org " << *Offset << ", " << (unsigned) Value;
+  OS << ".org ";
+  Offset->print(OS, MAI);
+  OS << ", " << (unsigned)Value;
   EmitEOL();
   return false;
 }
@@ -951,7 +1002,7 @@ void MCAsmStreamer::EmitCFIEndProcImpl(MCDwarfFrameInfo &Frame) {
 }
 
 void MCAsmStreamer::EmitRegisterName(int64_t Register) {
-  if (InstPrinter && !MAI->useDwarfRegNumForCFI()) {
+  if (!MAI->useDwarfRegNumForCFI()) {
     const MCRegisterInfo *MRI = getContext().getRegisterInfo();
     unsigned LLVMRegister = MRI->getLLVMRegNum(Register, true);
     InstPrinter->printRegName(OS, LLVMRegister);
@@ -992,13 +1043,15 @@ void MCAsmStreamer::EmitCFIOffset(int64_t Register, int64_t Offset) {
 void MCAsmStreamer::EmitCFIPersonality(const MCSymbol *Sym,
                                        unsigned Encoding) {
   MCStreamer::EmitCFIPersonality(Sym, Encoding);
-  OS << "\t.cfi_personality " << Encoding << ", " << *Sym;
+  OS << "\t.cfi_personality " << Encoding << ", ";
+  Sym->print(OS, MAI);
   EmitEOL();
 }
 
 void MCAsmStreamer::EmitCFILsda(const MCSymbol *Sym, unsigned Encoding) {
   MCStreamer::EmitCFILsda(Sym, Encoding);
-  OS << "\t.cfi_lsda " << Encoding << ", " << *Sym;
+  OS << "\t.cfi_lsda " << Encoding << ", ";
+  Sym->print(OS, MAI);
   EmitEOL();
 }
 
@@ -1062,7 +1115,8 @@ void MCAsmStreamer::EmitCFIWindowSave() {
 void MCAsmStreamer::EmitWinCFIStartProc(const MCSymbol *Symbol) {
   MCStreamer::EmitWinCFIStartProc(Symbol);
 
-  OS << ".seh_proc " << *Symbol;
+  OS << ".seh_proc ";
+  Symbol->print(OS, MAI);
   EmitEOL();
 }
 
@@ -1091,7 +1145,8 @@ void MCAsmStreamer::EmitWinEHHandler(const MCSymbol *Sym, bool Unwind,
                                       bool Except) {
   MCStreamer::EmitWinEHHandler(Sym, Unwind, Except);
 
-  OS << "\t.seh_handler " << *Sym;
+  OS << "\t.seh_handler ";
+  Sym->print(OS, MAI);
   if (Unwind)
     OS << ", @unwind";
   if (Except)
@@ -1107,9 +1162,9 @@ void MCAsmStreamer::EmitWinEHHandlerData() {
   // We only do this so the section switch that terminates the handler
   // data block is visible.
   WinEH::FrameInfo *CurFrame = getCurrentWinFrameInfo();
-  if (const MCSection *XData = WinEH::UnwindEmitter::getXDataSection(
-          CurFrame->Function, getContext()))
-    SwitchSectionNoChange(XData);
+  MCSection *XData =
+      WinEH::UnwindEmitter::getXDataSection(CurFrame->Function, getContext());
+  SwitchSectionNoChange(XData);
 
   OS << "\t.seh_handlerdata";
   EmitEOL();
@@ -1172,7 +1227,7 @@ void MCAsmStreamer::AddEncodingComment(const MCInst &Inst,
   SmallString<256> Code;
   SmallVector<MCFixup, 4> Fixups;
   raw_svector_ostream VecOS(Code);
-  Emitter->EncodeInstruction(Inst, VecOS, Fixups, STI);
+  Emitter->encodeInstruction(Inst, VecOS, Fixups, STI);
   VecOS.flush();
 
   // If we are showing fixups, create symbolic markers in the encoded
@@ -1255,15 +1310,6 @@ void MCAsmStreamer::EmitInstruction(const MCInst &Inst, const MCSubtargetInfo &S
   assert(getCurrentSection().first &&
          "Cannot emit contents before setting section!");
 
-  // @LOCALMOD-START
-  if (NaClExpander && NaClExpander->expandInst(Inst, *this, STI))
-    return;
-
-  if (BundleAlignmentEnabled && AsmBackend &&
-      AsmBackend->CustomExpandInst(Inst, *this)) {
-    return;
-  }
-  // @LOCALMOD-END
   // Show the encoding in a comment if we have a code emitter.
   if (Emitter)
     AddEncodingComment(Inst, STI);
@@ -1274,17 +1320,16 @@ void MCAsmStreamer::EmitInstruction(const MCInst &Inst, const MCSubtargetInfo &S
     GetCommentOS() << "\n";
   }
 
-  // If we have an AsmPrinter, use that to print, otherwise print the MCInst.
-  if (InstPrinter)
-    InstPrinter->printInst(&Inst, OS, "", STI);
+  if(getTargetStreamer())
+    getTargetStreamer()->prettyPrintAsm(*InstPrinter, OS, Inst, STI);
   else
-    Inst.print(OS);
+    InstPrinter->printInst(&Inst, OS, "", STI);
+
   EmitEOL();
 }
 
 void MCAsmStreamer::EmitBundleAlignMode(unsigned AlignPow2) {
   OS << "\t.bundle_align_mode " << AlignPow2;
-  BundleAlignmentEnabled = AlignPow2 > 0; // @LOCALMOD
   EmitEOL();
 }
 
diff --git a/lib/MC/MCAssembler.cpp b/lib/MC/MCAssembler.cpp
index 1edd51d8392..f53b589e1ae 100644
--- a/lib/MC/MCAssembler.cpp
+++ b/lib/MC/MCAssembler.cpp
@@ -18,7 +18,6 @@
 #include "llvm/MC/MCContext.h"
 #include "llvm/MC/MCDwarf.h"
 #include "llvm/MC/MCExpr.h"
-#include "llvm/MC/MCSectionELF.h"
 #include "llvm/MC/MCFixupKindInfo.h"
 #include "llvm/MC/MCObjectWriter.h"
 #include "llvm/MC/MCSection.h"
@@ -70,19 +69,19 @@ MCAsmLayout::MCAsmLayout(MCAssembler &Asm)
  {
   // Compute the section layout order. Virtual sections must go last.
   for (MCAssembler::iterator it = Asm.begin(), ie = Asm.end(); it != ie; ++it)
-    if (!it->getSection().isVirtualSection())
+    if (!it->isVirtualSection())
       SectionOrder.push_back(&*it);
   for (MCAssembler::iterator it = Asm.begin(), ie = Asm.end(); it != ie; ++it)
-    if (it->getSection().isVirtualSection())
+    if (it->isVirtualSection())
       SectionOrder.push_back(&*it);
 }
 
 bool MCAsmLayout::isFragmentValid(const MCFragment *F) const {
-  const MCSectionData &SD = *F->getParent();
-  const MCFragment *LastValid = LastValidFragment.lookup(&SD);
+  const MCSection *Sec = F->getParent();
+  const MCFragment *LastValid = LastValidFragment.lookup(Sec);
   if (!LastValid)
     return false;
-  assert(LastValid->getParent() == F->getParent());
+  assert(LastValid->getParent() == Sec);
   return F->getLayoutOrder() <= LastValid->getLayoutOrder();
 }
 
@@ -93,16 +92,14 @@ void MCAsmLayout::invalidateFragmentsFrom(MCFragment *F) {
 
   // Otherwise, reset the last valid fragment to the previous fragment
   // (if this is the first fragment, it will be NULL).
-  const MCSectionData &SD = *F->getParent();
-  LastValidFragment[&SD] = F->getPrevNode();
+  LastValidFragment[F->getParent()] = F->getPrevNode();
 }
 
 void MCAsmLayout::ensureValid(const MCFragment *F) const {
-  MCSectionData &SD = *F->getParent();
-
-  MCFragment *Cur = LastValidFragment[&SD];
+  MCSection *Sec = F->getParent();
+  MCFragment *Cur = LastValidFragment[Sec];
   if (!Cur)
-    Cur = &*SD.begin();
+    Cur = Sec->begin();
   else
     Cur = Cur->getNextNode();
 
@@ -121,41 +118,35 @@ uint64_t MCAsmLayout::getFragmentOffset(const MCFragment *F) const {
 }
 
 // Simple getSymbolOffset helper for the non-varibale case.
-static bool getLabelOffset(const MCAsmLayout &Layout, const MCSymbolData &SD,
+static bool getLabelOffset(const MCAsmLayout &Layout, const MCSymbol &S,
                            bool ReportError, uint64_t &Val) {
-  if (!SD.getFragment()) {
+  if (!S.getFragment()) {
     if (ReportError)
       report_fatal_error("unable to evaluate offset to undefined symbol '" +
-                         SD.getSymbol().getName() + "'");
+                         S.getName() + "'");
     return false;
   }
-  Val = Layout.getFragmentOffset(SD.getFragment()) + SD.getOffset();
+  Val = Layout.getFragmentOffset(S.getFragment()) + S.getOffset();
   return true;
 }
 
-static bool getSymbolOffsetImpl(const MCAsmLayout &Layout,
-                                const MCSymbolData *SD, bool ReportError,
-                                uint64_t &Val) {
-  const MCSymbol &S = SD->getSymbol();
-
+static bool getSymbolOffsetImpl(const MCAsmLayout &Layout, const MCSymbol &S,
+                                bool ReportError, uint64_t &Val) {
   if (!S.isVariable())
-    return getLabelOffset(Layout, *SD, ReportError, Val);
+    return getLabelOffset(Layout, S, ReportError, Val);
 
   // If SD is a variable, evaluate it.
   MCValue Target;
-  if (!S.getVariableValue()->EvaluateAsRelocatable(Target, &Layout, nullptr))
+  if (!S.getVariableValue()->evaluateAsRelocatable(Target, &Layout, nullptr))
     report_fatal_error("unable to evaluate offset for variable '" +
                        S.getName() + "'");
 
   uint64_t Offset = Target.getConstant();
 
-  const MCAssembler &Asm = Layout.getAssembler();
-
   const MCSymbolRefExpr *A = Target.getSymA();
   if (A) {
     uint64_t ValA;
-    if (!getLabelOffset(Layout, Asm.getSymbolData(A->getSymbol()), ReportError,
-                        ValA))
+    if (!getLabelOffset(Layout, A->getSymbol(), ReportError, ValA))
       return false;
     Offset += ValA;
   }
@@ -163,8 +154,7 @@ static bool getSymbolOffsetImpl(const MCAsmLayout &Layout,
   const MCSymbolRefExpr *B = Target.getSymB();
   if (B) {
     uint64_t ValB;
-    if (!getLabelOffset(Layout, Asm.getSymbolData(B->getSymbol()), ReportError,
-                        ValB))
+    if (!getLabelOffset(Layout, B->getSymbol(), ReportError, ValB))
       return false;
     Offset -= ValB;
   }
@@ -173,13 +163,13 @@ static bool getSymbolOffsetImpl(const MCAsmLayout &Layout,
   return true;
 }
 
-bool MCAsmLayout::getSymbolOffset(const MCSymbolData *SD, uint64_t &Val) const {
-  return getSymbolOffsetImpl(*this, SD, false, Val);
+bool MCAsmLayout::getSymbolOffset(const MCSymbol &S, uint64_t &Val) const {
+  return getSymbolOffsetImpl(*this, S, false, Val);
 }
 
-uint64_t MCAsmLayout::getSymbolOffset(const MCSymbolData *SD) const {
+uint64_t MCAsmLayout::getSymbolOffset(const MCSymbol &S) const {
   uint64_t Val;
-  getSymbolOffsetImpl(*this, SD, true, Val);
+  getSymbolOffsetImpl(*this, S, true, Val);
   return Val;
 }
 
@@ -194,7 +184,7 @@ const MCSymbol *MCAsmLayout::getBaseSymbol(const MCSymbol &Symbol) const {
 
   const MCSymbolRefExpr *RefB = Value.getSymB();
   if (RefB)
-    Assembler.getContext().FatalError(
+    Assembler.getContext().reportFatalError(
         SMLoc(), Twine("symbol '") + RefB->getSymbol().getName() +
                      "' could not be evaluated in a subtraction expression");
 
@@ -204,10 +194,9 @@ const MCSymbol *MCAsmLayout::getBaseSymbol(const MCSymbol &Symbol) const {
 
   const MCSymbol &ASym = A->getSymbol();
   const MCAssembler &Asm = getAssembler();
-  const MCSymbolData &ASD = Asm.getSymbolData(ASym);
-  if (ASD.isCommon()) {
+  if (ASym.isCommon()) {
     // FIXME: we should probably add a SMLoc to MCExpr.
-    Asm.getContext().FatalError(SMLoc(),
+    Asm.getContext().reportFatalError(SMLoc(),
                                 "Common symbol " + ASym.getName() +
                                     " cannot be used in assignment expr");
   }
@@ -215,19 +204,19 @@ const MCSymbol *MCAsmLayout::getBaseSymbol(const MCSymbol &Symbol) const {
   return &ASym;
 }
 
-uint64_t MCAsmLayout::getSectionAddressSize(const MCSectionData *SD) const {
+uint64_t MCAsmLayout::getSectionAddressSize(const MCSection *Sec) const {
   // The size is the last fragment's end offset.
-  const MCFragment &F = SD->getFragmentList().back();
+  const MCFragment &F = Sec->getFragmentList().back();
   return getFragmentOffset(&F) + getAssembler().computeFragmentSize(*this, F);
 }
 
-uint64_t MCAsmLayout::getSectionFileSize(const MCSectionData *SD) const {
+uint64_t MCAsmLayout::getSectionFileSize(const MCSection *Sec) const {
   // Virtual sections have no file size.
-  if (SD->getSection().isVirtualSection())
+  if (Sec->isVirtualSection())
     return 0;
 
   // Otherwise, the file size is the same as the address space size.
-  return getSectionAddressSize(SD);
+  return getSectionAddressSize(Sec);
 }
 
 uint64_t llvm::computeBundlePadding(const MCAssembler &Assembler,
@@ -273,108 +262,64 @@ uint64_t llvm::computeBundlePadding(const MCAssembler &Assembler,
 
 /* *** */
 
-MCFragment::MCFragment() : Kind(FragmentType(~0)) {
+void ilist_node_traits<MCFragment>::deleteNode(MCFragment *V) {
+  V->destroy();
 }
 
-MCFragment::~MCFragment() {
+MCFragment::MCFragment() : Kind(FragmentType(~0)), HasInstructions(false),
+                           AlignToBundleEnd(false), BundlePadding(0) {
 }
 
-MCFragment::MCFragment(FragmentType Kind, MCSectionData *Parent)
-    : Kind(Kind), Parent(Parent), Atom(nullptr), Offset(~UINT64_C(0)) {
+MCFragment::~MCFragment() { }
+
+MCFragment::MCFragment(FragmentType Kind, bool HasInstructions,
+                       uint8_t BundlePadding, MCSection *Parent)
+    : Kind(Kind), HasInstructions(HasInstructions), AlignToBundleEnd(false),
+      BundlePadding(BundlePadding), Parent(Parent), Atom(nullptr),
+      Offset(~UINT64_C(0)) {
   if (Parent)
     Parent->getFragmentList().push_back(this);
 }
 
-/* *** */
-
-MCEncodedFragment::~MCEncodedFragment() {
-}
-
-/* *** */
-
-MCEncodedFragmentWithFixups::~MCEncodedFragmentWithFixups() {
-}
-
-/* *** */
-
-MCSectionData::MCSectionData() : Section(nullptr) {}
-
-MCSectionData::MCSectionData(const MCSection &Section, MCAssembler *A)
-    : Section(&Section), Ordinal(~UINT32_C(0)), Alignment(1),
-      BundleLockState(NotBundleLocked), BundleLockNestingDepth(0),
-      BundleGroupBeforeFirstInst(false), HasInstructions(false) {
-  // @LOCALMOD-BEGIN
-  if (A) {
-    // Necessary for IRT building because the IRT loader expects the end of
-    // the section to be bundle-aligned. Padding happens with 0's though,
-    // so it's not really ideal. TODO(dschuff) figure out how to do it right.
-    A->getSectionList().push_back(this);
-    if (A->isBundlingEnabled() && Section.UseCodeAlign())
-      setAlignment(A->getBundleAlignSize());
-  }
-  // @LOCALMOD-END
-}
-
-MCSectionData::iterator
-MCSectionData::getSubsectionInsertionPoint(unsigned Subsection) {
-  if (Subsection == 0 && SubsectionFragmentMap.empty())
-    return end();
-
-  SmallVectorImpl<std::pair<unsigned, MCFragment *> >::iterator MI =
-    std::lower_bound(SubsectionFragmentMap.begin(), SubsectionFragmentMap.end(),
-                     std::make_pair(Subsection, (MCFragment *)nullptr));
-  bool ExactMatch = false;
-  if (MI != SubsectionFragmentMap.end()) {
-    ExactMatch = MI->first == Subsection;
-    if (ExactMatch)
-      ++MI;
-  }
-  iterator IP;
-  if (MI == SubsectionFragmentMap.end())
-    IP = end();
-  else
-    IP = MI->second;
-  if (!ExactMatch && Subsection != 0) {
-    // The GNU as documentation claims that subsections have an alignment of 4,
-    // although this appears not to be the case.
-    MCFragment *F = new MCDataFragment();
-    SubsectionFragmentMap.insert(MI, std::make_pair(Subsection, F));
-    getFragmentList().insert(IP, F);
-    F->setParent(this);
-  }
-
-  return IP;
-}
-
-void MCSectionData::setBundleLockState(BundleLockStateType NewState) {
-  if (NewState == NotBundleLocked) {
-    if (BundleLockNestingDepth == 0) {
-      report_fatal_error("Mismatched bundle_lock/unlock directives");
-    }
-    if (--BundleLockNestingDepth == 0) {
-      BundleLockState = NotBundleLocked;
-    }
+void MCFragment::destroy() {
+  // First check if we are the sentinal.
+  if (Kind == FragmentType(~0)) {
+    delete this;
     return;
   }
 
-  // If any of the directives is an align_to_end directive, the whole nested
-  // group is align_to_end. So don't downgrade from align_to_end to just locked.
-  if (BundleLockState != BundleLockedAlignToEnd) {
-    BundleLockState = NewState;
+  switch (Kind) {
+    case FT_Align:
+      delete cast<MCAlignFragment>(this);
+      return;
+    case FT_Data:
+      delete cast<MCDataFragment>(this);
+      return;
+    case FT_CompactEncodedInst:
+      delete cast<MCCompactEncodedInstFragment>(this);
+      return;
+    case FT_Fill:
+      delete cast<MCFillFragment>(this);
+      return;
+    case FT_Relaxable:
+      delete cast<MCRelaxableFragment>(this);
+      return;
+    case FT_Org:
+      delete cast<MCOrgFragment>(this);
+      return;
+    case FT_Dwarf:
+      delete cast<MCDwarfLineAddrFragment>(this);
+      return;
+    case FT_DwarfFrame:
+      delete cast<MCDwarfCallFrameFragment>(this);
+      return;
+    case FT_LEB:
+      delete cast<MCLEBFragment>(this);
+      return;
+    case FT_SafeSEH:
+      delete cast<MCSafeSEHFragment>(this);
+      return;
   }
-  ++BundleLockNestingDepth;
-}
-
-/* *** */
-
-MCSymbolData::MCSymbolData() : Symbol(nullptr) {}
-
-MCSymbolData::MCSymbolData(const MCSymbol &Symbol, MCFragment *Fragment,
-                           uint64_t Offset, MCAssembler *A)
-    : Symbol(&Symbol), Fragment(Fragment), Offset(Offset), SymbolSize(nullptr),
-      CommonAlign(-1U), Flags(0), Index(0) {
-  if (A)
-    A->getSymbolList().push_back(this);
 }
 
 /* *** */
@@ -394,8 +339,6 @@ MCAssembler::~MCAssembler() {
 void MCAssembler::reset() {
   Sections.clear();
   Symbols.clear();
-  SectionMap.clear();
-  SymbolMap.clear();
   IndirectSymbols.clear();
   DataRegions.clear();
   LinkerOptions.clear();
@@ -440,16 +383,6 @@ bool MCAssembler::isThumbFunc(const MCSymbol *Symbol) const {
   return true;
 }
 
-void MCAssembler::addLocalUsedInReloc(const MCSymbol &Sym) {
-  assert(Sym.isTemporary());
-  LocalsUsedInReloc.insert(&Sym);
-}
-
-bool MCAssembler::isLocalUsedInReloc(const MCSymbol &Sym) const {
-  assert(Sym.isTemporary());
-  return LocalsUsedInReloc.count(&Sym);
-}
-
 bool MCAssembler::isSymbolLinkerVisible(const MCSymbol &Symbol) const {
   // Non-temporary labels should always be visible to the linker.
   if (!Symbol.isTemporary())
@@ -459,29 +392,29 @@ bool MCAssembler::isSymbolLinkerVisible(const MCSymbol &Symbol) const {
   if (!Symbol.isInSection())
     return false;
 
-  if (isLocalUsedInReloc(Symbol))
+  if (Symbol.isUsedInReloc())
     return true;
 
   return false;
 }
 
-const MCSymbolData *MCAssembler::getAtom(const MCSymbolData *SD) const {
+const MCSymbol *MCAssembler::getAtom(const MCSymbol &S) const {
   // Linker visible symbols define atoms.
-  if (isSymbolLinkerVisible(SD->getSymbol()))
-    return SD;
+  if (isSymbolLinkerVisible(S))
+    return &S;
 
   // Absolute and undefined symbols have no defining atom.
-  if (!SD->getFragment())
+  if (!S.getFragment())
     return nullptr;
 
   // Non-linker visible symbols in sections which can't be atomized have no
   // defining atom.
   if (!getContext().getAsmInfo()->isSectionAtomizableBySymbols(
-          SD->getFragment()->getParent()->getSection()))
+          *S.getFragment()->getParent()))
     return nullptr;
 
   // Otherwise, return the atom for the containing fragment.
-  return SD->getFragment()->getAtom();
+  return S.getFragment()->getAtom();
 }
 
 bool MCAssembler::evaluateFixup(const MCAsmLayout &Layout,
@@ -489,12 +422,12 @@ bool MCAssembler::evaluateFixup(const MCAsmLayout &Layout,
                                 MCValue &Target, uint64_t &Value) const {
   ++stats::evaluateFixup;
 
-  // FIXME: This code has some duplication with RecordRelocation. We should
+  // FIXME: This code has some duplication with recordRelocation. We should
   // probably merge the two into a single callback that tries to evaluate a
   // fixup and records a relocation if one is needed.
   const MCExpr *Expr = Fixup.getValue();
-  if (!Expr->EvaluateAsRelocatable(Target, &Layout, &Fixup))
-    getContext().FatalError(Fixup.getLoc(), "expected relocatable expression");
+  if (!Expr->evaluateAsRelocatable(Target, &Layout, &Fixup))
+    getContext().reportFatalError(Fixup.getLoc(), "expected relocatable expression");
 
   bool IsPCRel = Backend.getFixupKindInfo(
     Fixup.getKind()).Flags & MCFixupKindInfo::FKF_IsPCRel;
@@ -511,9 +444,8 @@ bool MCAssembler::evaluateFixup(const MCAsmLayout &Layout,
       if (A->getKind() != MCSymbolRefExpr::VK_None || SA.isUndefined()) {
         IsResolved = false;
       } else {
-        const MCSymbolData &DataA = getSymbolData(SA);
-        IsResolved = getWriter().IsSymbolRefDifferenceFullyResolvedImpl(
-            *this, DataA, nullptr, *DF, false, true);
+        IsResolved = getWriter().isSymbolRefDifferenceFullyResolvedImpl(
+            *this, SA, *DF, false, true);
       }
     }
   } else {
@@ -525,12 +457,12 @@ bool MCAssembler::evaluateFixup(const MCAsmLayout &Layout,
   if (const MCSymbolRefExpr *A = Target.getSymA()) {
     const MCSymbol &Sym = A->getSymbol();
     if (Sym.isDefined())
-      Value += Layout.getSymbolOffset(&getSymbolData(Sym));
+      Value += Layout.getSymbolOffset(Sym);
   }
   if (const MCSymbolRefExpr *B = Target.getSymB()) {
     const MCSymbol &Sym = B->getSymbol();
     if (Sym.isDefined())
-      Value -= Layout.getSymbolOffset(&getSymbolData(Sym));
+      Value -= Layout.getSymbolOffset(Sym);
   }
 
 
@@ -560,15 +492,20 @@ uint64_t MCAssembler::computeFragmentSize(const MCAsmLayout &Layout,
                                           const MCFragment &F) const {
   switch (F.getKind()) {
   case MCFragment::FT_Data:
+    return cast<MCDataFragment>(F).getContents().size();
   case MCFragment::FT_Relaxable:
+    return cast<MCRelaxableFragment>(F).getContents().size();
   case MCFragment::FT_CompactEncodedInst:
-    return cast<MCEncodedFragment>(F).getContents().size();
+    return cast<MCCompactEncodedInstFragment>(F).getContents().size();
   case MCFragment::FT_Fill:
     return cast<MCFillFragment>(F).getSize();
 
   case MCFragment::FT_LEB:
     return cast<MCLEBFragment>(F).getContents().size();
 
+  case MCFragment::FT_SafeSEH:
+    return 4;
+
   case MCFragment::FT_Align: {
     const MCAlignFragment &AF = cast<MCAlignFragment>(F);
     unsigned Offset = Layout.getFragmentOffset(&AF);
@@ -587,7 +524,7 @@ uint64_t MCAssembler::computeFragmentSize(const MCAsmLayout &Layout,
   case MCFragment::FT_Org: {
     const MCOrgFragment &OF = cast<MCOrgFragment>(F);
     int64_t TargetLocation;
-    if (!OF.getOffset().EvaluateAsAbsolute(TargetLocation, Layout))
+    if (!OF.getOffset().evaluateAsAbsolute(TargetLocation, Layout))
       report_fatal_error("expected assembly-time absolute expression");
 
     // FIXME: We need a way to communicate this error.
@@ -671,11 +608,14 @@ void MCAsmLayout::layoutFragment(MCFragment *F) {
   }
 }
 
-/// \brief Write the contents of a fragment to the given object writer. Expects
-///        a MCEncodedFragment.
-static void writeFragmentContents(const MCFragment &F, MCObjectWriter *OW) {
-  const MCEncodedFragment &EF = cast<MCEncodedFragment>(F);
-  OW->WriteBytes(EF.getContents());
+void MCAssembler::registerSymbol(const MCSymbol &Symbol, bool *Created) {
+  bool New = !Symbol.isRegistered();
+  if (Created)
+    *Created = New;
+  if (New) {
+    Symbol.setIsRegistered(true);
+    Symbols.push_back(&Symbol);
+  }
 }
 
 void MCAssembler::writeFragmentPadding(const MCFragment &F, uint64_t FSize,
@@ -759,10 +699,10 @@ static void writeFragment(const MCAssembler &Asm, const MCAsmLayout &Layout,
     for (uint64_t i = 0; i != Count; ++i) {
       switch (AF.getValueSize()) {
       default: llvm_unreachable("Invalid size!");
-      case 1: OW->Write8 (uint8_t (AF.getValue())); break;
-      case 2: OW->Write16(uint16_t(AF.getValue())); break;
-      case 4: OW->Write32(uint32_t(AF.getValue())); break;
-      case 8: OW->Write64(uint64_t(AF.getValue())); break;
+      case 1: OW->write8 (uint8_t (AF.getValue())); break;
+      case 2: OW->write16(uint16_t(AF.getValue())); break;
+      case 4: OW->write32(uint32_t(AF.getValue())); break;
+      case 8: OW->write64(uint64_t(AF.getValue())); break;
       }
     }
     break;
@@ -770,17 +710,17 @@ static void writeFragment(const MCAssembler &Asm, const MCAsmLayout &Layout,
 
   case MCFragment::FT_Data: 
     ++stats::EmittedDataFragments;
-    writeFragmentContents(F, OW);
+    OW->writeBytes(cast<MCDataFragment>(F).getContents());
     break;
 
   case MCFragment::FT_Relaxable:
     ++stats::EmittedRelaxableFragments;
-    writeFragmentContents(F, OW);
+    OW->writeBytes(cast<MCRelaxableFragment>(F).getContents());
     break;
 
   case MCFragment::FT_CompactEncodedInst:
     ++stats::EmittedCompactEncodedInstFragments;
-    writeFragmentContents(F, OW);
+    OW->writeBytes(cast<MCCompactEncodedInstFragment>(F).getContents());
     break;
 
   case MCFragment::FT_Fill: {
@@ -792,10 +732,10 @@ static void writeFragment(const MCAssembler &Asm, const MCAsmLayout &Layout,
     for (uint64_t i = 0, e = FF.getSize() / FF.getValueSize(); i != e; ++i) {
       switch (FF.getValueSize()) {
       default: llvm_unreachable("Invalid size!");
-      case 1: OW->Write8 (uint8_t (FF.getValue())); break;
-      case 2: OW->Write16(uint16_t(FF.getValue())); break;
-      case 4: OW->Write32(uint32_t(FF.getValue())); break;
-      case 8: OW->Write64(uint64_t(FF.getValue())); break;
+      case 1: OW->write8 (uint8_t (FF.getValue())); break;
+      case 2: OW->write16(uint16_t(FF.getValue())); break;
+      case 4: OW->write32(uint32_t(FF.getValue())); break;
+      case 8: OW->write64(uint64_t(FF.getValue())); break;
       }
     }
     break;
@@ -803,7 +743,13 @@ static void writeFragment(const MCAssembler &Asm, const MCAsmLayout &Layout,
 
   case MCFragment::FT_LEB: {
     const MCLEBFragment &LF = cast<MCLEBFragment>(F);
-    OW->WriteBytes(LF.getContents());
+    OW->writeBytes(LF.getContents());
+    break;
+  }
+
+  case MCFragment::FT_SafeSEH: {
+    const MCSafeSEHFragment &SF = cast<MCSafeSEHFragment>(F);
+    OW->write32(SF.getSymbol()->getIndex());
     break;
   }
 
@@ -812,19 +758,19 @@ static void writeFragment(const MCAssembler &Asm, const MCAsmLayout &Layout,
     const MCOrgFragment &OF = cast<MCOrgFragment>(F);
 
     for (uint64_t i = 0, e = FragmentSize; i != e; ++i)
-      OW->Write8(uint8_t(OF.getValue()));
+      OW->write8(uint8_t(OF.getValue()));
 
     break;
   }
 
   case MCFragment::FT_Dwarf: {
     const MCDwarfLineAddrFragment &OF = cast<MCDwarfLineAddrFragment>(F);
-    OW->WriteBytes(OF.getContents());
+    OW->writeBytes(OF.getContents());
     break;
   }
   case MCFragment::FT_DwarfFrame: {
     const MCDwarfCallFrameFragment &CF = cast<MCDwarfCallFrameFragment>(F);
-    OW->WriteBytes(CF.getContents());
+    OW->writeBytes(CF.getContents());
     break;
   }
   }
@@ -833,15 +779,15 @@ static void writeFragment(const MCAssembler &Asm, const MCAsmLayout &Layout,
          "The stream should advance by fragment size");
 }
 
-void MCAssembler::writeSectionData(const MCSectionData *SD,
+void MCAssembler::writeSectionData(const MCSection *Sec,
                                    const MCAsmLayout &Layout) const {
   // Ignore virtual sections.
-  if (SD->getSection().isVirtualSection()) {
-    assert(Layout.getSectionFileSize(SD) == 0 && "Invalid size for section!");
+  if (Sec->isVirtualSection()) {
+    assert(Layout.getSectionFileSize(Sec) == 0 && "Invalid size for section!");
 
     // Check that contents are only things legal inside a virtual section.
-    for (MCSectionData::const_iterator it = SD->begin(),
-           ie = SD->end(); it != ie; ++it) {
+    for (MCSection::const_iterator it = Sec->begin(), ie = Sec->end(); it != ie;
+         ++it) {
       switch (it->getKind()) {
       default: llvm_unreachable("Invalid fragment in virtual section!");
       case MCFragment::FT_Data: {
@@ -853,7 +799,7 @@ void MCAssembler::writeSectionData(const MCSectionData *SD,
                "Cannot have fixups in virtual section!");
         for (unsigned i = 0, e = DF.getContents().size(); i != e; ++i)
           if (DF.getContents()[i]) {
-            if (auto *ELFSec = dyn_cast<const MCSectionELF>(&SD->getSection()))
+            if (auto *ELFSec = dyn_cast<const MCSectionELF>(Sec))
               report_fatal_error("non-zero initializer found in section '" +
                   ELFSec->getSectionName() + "'");
             else
@@ -882,12 +828,12 @@ void MCAssembler::writeSectionData(const MCSectionData *SD,
   uint64_t Start = getWriter().getStream().tell();
   (void)Start;
 
-  for (MCSectionData::const_iterator it = SD->begin(), ie = SD->end();
-       it != ie; ++it)
+  for (MCSection::const_iterator it = Sec->begin(), ie = Sec->end(); it != ie;
+       ++it)
     writeFragment(*this, Layout, *it);
 
   assert(getWriter().getStream().tell() - Start ==
-         Layout.getSectionAddressSize(SD));
+         Layout.getSectionAddressSize(Sec));
 }
 
 std::pair<uint64_t, bool> MCAssembler::handleFixup(const MCAsmLayout &Layout,
@@ -902,7 +848,7 @@ std::pair<uint64_t, bool> MCAssembler::handleFixup(const MCAsmLayout &Layout,
     // The fixup was unresolved, we need a relocation. Inform the object
     // writer of the relocation, and give it an opportunity to adjust the
     // fixup value if need be.
-    getWriter().RecordRelocation(*this, Layout, &F, Fixup, Target, IsPCRel,
+    getWriter().recordRelocation(*this, Layout, &F, Fixup, Target, IsPCRel,
                                  FixedValue);
   }
   return std::make_pair(FixedValue, IsPCRel);
@@ -922,18 +868,18 @@ void MCAssembler::Finish() {
     // Create dummy fragments to eliminate any empty sections, this simplifies
     // layout.
     if (it->getFragmentList().empty())
-      new MCDataFragment(it);
+      new MCDataFragment(&*it);
 
     it->setOrdinal(SectionIndex++);
   }
 
   // Assign layout order indices to sections and fragments.
   for (unsigned i = 0, e = Layout.getSectionOrder().size(); i != e; ++i) {
-    MCSectionData *SD = Layout.getSectionOrder()[i];
-    SD->setLayoutOrder(i);
+    MCSection *Sec = Layout.getSectionOrder()[i];
+    Sec->setLayoutOrder(i);
 
     unsigned FragmentIndex = 0;
-    for (MCSectionData::iterator iFrag = SD->begin(), iFragEnd = SD->end();
+    for (MCSection::iterator iFrag = Sec->begin(), iFragEnd = Sec->end();
          iFrag != iFragEnd; ++iFrag)
       iFrag->setLayoutOrder(FragmentIndex++);
   }
@@ -957,30 +903,41 @@ void MCAssembler::Finish() {
 
   // Allow the object writer a chance to perform post-layout binding (for
   // example, to set the index fields in the symbol data).
-  getWriter().ExecutePostLayoutBinding(*this, Layout);
+  getWriter().executePostLayoutBinding(*this, Layout);
 
   // Evaluate and apply the fixups, generating relocation entries as necessary.
   for (MCAssembler::iterator it = begin(), ie = end(); it != ie; ++it) {
-    for (MCSectionData::iterator it2 = it->begin(),
-           ie2 = it->end(); it2 != ie2; ++it2) {
-      MCEncodedFragmentWithFixups *F =
-        dyn_cast<MCEncodedFragmentWithFixups>(it2);
-      if (F) {
-        for (MCEncodedFragmentWithFixups::fixup_iterator it3 = F->fixup_begin(),
-             ie3 = F->fixup_end(); it3 != ie3; ++it3) {
-          MCFixup &Fixup = *it3;
-          uint64_t FixedValue;
-          bool IsPCRel;
-          std::tie(FixedValue, IsPCRel) = handleFixup(Layout, *F, Fixup);
-          getBackend().applyFixup(Fixup, F->getContents().data(),
-                                  F->getContents().size(), FixedValue, IsPCRel);
-        }
+    for (MCSection::iterator it2 = it->begin(), ie2 = it->end(); it2 != ie2;
+         ++it2) {
+      MCEncodedFragment *F = dyn_cast<MCEncodedFragment>(it2);
+      // Data and relaxable fragments both have fixups.  So only process
+      // those here.
+      // FIXME: Is there a better way to do this?  MCEncodedFragmentWithFixups
+      // being templated makes this tricky.
+      if (!F || isa<MCCompactEncodedInstFragment>(F))
+        continue;
+      ArrayRef<MCFixup> Fixups;
+      MutableArrayRef<char> Contents;
+      if (auto *FragWithFixups = dyn_cast<MCDataFragment>(F)) {
+        Fixups = FragWithFixups->getFixups();
+        Contents = FragWithFixups->getContents();
+      } else if (auto *FragWithFixups = dyn_cast<MCRelaxableFragment>(F)) {
+        Fixups = FragWithFixups->getFixups();
+        Contents = FragWithFixups->getContents();
+      } else
+        llvm_unreachable("Unknown fragment with fixups!");
+      for (const MCFixup &Fixup : Fixups) {
+        uint64_t FixedValue;
+        bool IsPCRel;
+        std::tie(FixedValue, IsPCRel) = handleFixup(Layout, *F, Fixup);
+        getBackend().applyFixup(Fixup, Contents.data(),
+                                Contents.size(), FixedValue, IsPCRel);
       }
     }
   }
 
   // Write the object file.
-  getWriter().WriteObject(*this, Layout);
+  getWriter().writeObject(*this, Layout);
 
   stats::ObjectBytes += OS.tell() - StartOffset;
 }
@@ -988,13 +945,11 @@ void MCAssembler::Finish() {
 bool MCAssembler::fixupNeedsRelaxation(const MCFixup &Fixup,
                                        const MCRelaxableFragment *DF,
                                        const MCAsmLayout &Layout) const {
-  // If we cannot resolve the fixup value, it requires relaxation.
   MCValue Target;
   uint64_t Value;
-  if (!evaluateFixup(Layout, Fixup, DF, Target, Value))
-    return true;
-
-  return getBackend().fixupNeedsRelaxation(Fixup, Value, DF, Layout);
+  bool Resolved = evaluateFixup(Layout, Fixup, DF, Target, Value);
+  return getBackend().fixupNeedsRelaxationAdvanced(Fixup, Resolved, Value, DF,
+                                                   Layout);
 }
 
 bool MCAssembler::fragmentNeedsRelaxation(const MCRelaxableFragment *F,
@@ -1035,7 +990,7 @@ bool MCAssembler::relaxInstruction(MCAsmLayout &Layout,
   SmallVector<MCFixup, 4> Fixups;
   SmallString<256> Code;
   raw_svector_ostream VecOS(Code);
-  getEmitter().EncodeInstruction(Relaxed, VecOS, Fixups, F.getSubtargetInfo());
+  getEmitter().encodeInstruction(Relaxed, VecOS, Fixups, F.getSubtargetInfo());
   VecOS.flush();
 
   // Update the fragment.
@@ -1097,7 +1052,7 @@ bool MCAssembler::relaxDwarfCallFrameFragment(MCAsmLayout &Layout,
   return OldSize != Data.size();
 }
 
-bool MCAssembler::layoutSectionOnce(MCAsmLayout &Layout, MCSectionData &SD) {
+bool MCAssembler::layoutSectionOnce(MCAsmLayout &Layout, MCSection &Sec) {
   // Holds the first fragment which needed relaxing during this layout. It will
   // remain NULL if none were relaxed.
   // When a fragment is relaxed, all the fragments following it should get
@@ -1105,7 +1060,7 @@ bool MCAssembler::layoutSectionOnce(MCAsmLayout &Layout, MCSectionData &SD) {
   MCFragment *FirstRelaxedFragment = nullptr;
 
   // Attempt to relax all the fragments in the section.
-  for (MCSectionData::iterator I = SD.begin(), IE = SD.end(); I != IE; ++I) {
+  for (MCSection::iterator I = Sec.begin(), IE = Sec.end(); I != IE; ++I) {
     // Check if this is a fragment that needs relaxation.
     bool RelaxedFrag = false;
     switch(I->getKind()) {
@@ -1144,8 +1099,8 @@ bool MCAssembler::layoutOnce(MCAsmLayout &Layout) {
 
   bool WasRelaxed = false;
   for (iterator it = begin(), ie = end(); it != ie; ++it) {
-    MCSectionData &SD = *it;
-    while (layoutSectionOnce(Layout, SD))
+    MCSection &Sec = *it;
+    while (layoutSectionOnce(Layout, Sec))
       WasRelaxed = true;
   }
 
@@ -1188,6 +1143,7 @@ void MCFragment::dump() {
   case MCFragment::FT_Dwarf: OS << "MCDwarfFragment"; break;
   case MCFragment::FT_DwarfFrame: OS << "MCDwarfCallFrameFragment"; break;
   case MCFragment::FT_LEB:   OS << "MCLEBFragment"; break;
+  case MCFragment::FT_SafeSEH:    OS << "MCSafeSEHFragment"; break;
   }
 
   OS << "<MCFragment " << (void*) this << " LayoutOrder:" << LayoutOrder
@@ -1280,38 +1236,13 @@ void MCFragment::dump() {
     OS << " Value:" << LF->getValue() << " Signed:" << LF->isSigned();
     break;
   }
+  case MCFragment::FT_SafeSEH: {
+    const MCSafeSEHFragment *F = cast<MCSafeSEHFragment>(this);
+    OS << "\n       ";
+    OS << " Sym:" << F->getSymbol();
+    break;
   }
-  OS << ">";
-}
-
-void MCSectionData::dump() {
-  raw_ostream &OS = llvm::errs();
-
-  OS << "<MCSectionData";
-  OS << " Alignment:" << getAlignment()
-     << " Fragments:[\n      ";
-  for (iterator it = begin(), ie = end(); it != ie; ++it) {
-    if (it != begin()) OS << ",\n      ";
-    it->dump();
   }
-  OS << "]>";
-}
-
-void MCSymbolData::dump() const {
-  raw_ostream &OS = llvm::errs();
-
-  OS << "<MCSymbolData Symbol:" << getSymbol()
-     << " Fragment:" << getFragment();
-  if (!isCommon())
-    OS << " Offset:" << getOffset();
-  OS << " Flags:" << getFlags() << " Index:" << getIndex();
-  if (isCommon())
-    OS << " (common, size:" << getCommonSize()
-       << " align: " << getCommonAlignment() << ")";
-  if (isExternal())
-    OS << " (external)";
-  if (isPrivateExtern())
-    OS << " (private extern)";
   OS << ">";
 }
 
@@ -1329,21 +1260,11 @@ void MCAssembler::dump() {
 
   for (symbol_iterator it = symbol_begin(), ie = symbol_end(); it != ie; ++it) {
     if (it != symbol_begin()) OS << ",\n           ";
+    OS << "(";
     it->dump();
+    OS << ", Index:" << it->getIndex() << ", ";
+    OS << ")";
   }
   OS << "]>\n";
 }
 #endif
-
-// anchors for MC*Fragment vtables
-void MCEncodedFragment::anchor() { }
-void MCEncodedFragmentWithFixups::anchor() { }
-void MCDataFragment::anchor() { }
-void MCCompactEncodedInstFragment::anchor() { }
-void MCRelaxableFragment::anchor() { }
-void MCAlignFragment::anchor() { }
-void MCFillFragment::anchor() { }
-void MCOrgFragment::anchor() { }
-void MCLEBFragment::anchor() { }
-void MCDwarfLineAddrFragment::anchor() { }
-void MCDwarfCallFrameFragment::anchor() { }
diff --git a/lib/MC/MCCodeGenInfo.cpp b/lib/MC/MCCodeGenInfo.cpp
index d9dcfd0614b..347ec2cd01e 100644
--- a/lib/MC/MCCodeGenInfo.cpp
+++ b/lib/MC/MCCodeGenInfo.cpp
@@ -15,7 +15,7 @@
 #include "llvm/MC/MCCodeGenInfo.h"
 using namespace llvm;
 
-void MCCodeGenInfo::InitMCCodeGenInfo(Reloc::Model RM, CodeModel::Model CM,
+void MCCodeGenInfo::initMCCodeGenInfo(Reloc::Model RM, CodeModel::Model CM,
                                       CodeGenOpt::Level OL) {
   RelocationModel = RM;
   CMModel = CM;
diff --git a/lib/MC/MCContext.cpp b/lib/MC/MCContext.cpp
index 5f8e3c11de3..c601c56f395 100644
--- a/lib/MC/MCContext.cpp
+++ b/lib/MC/MCContext.cpp
@@ -10,6 +10,7 @@
 #include "llvm/MC/MCContext.h"
 #include "llvm/ADT/SmallString.h"
 #include "llvm/ADT/Twine.h"
+#include "llvm/MC/MCAssembler.h"
 #include "llvm/MC/MCAsmInfo.h"
 #include "llvm/MC/MCDwarf.h"
 #include "llvm/MC/MCLabel.h"
@@ -19,7 +20,9 @@
 #include "llvm/MC/MCSectionELF.h"
 #include "llvm/MC/MCSectionMachO.h"
 #include "llvm/MC/MCStreamer.h"
-#include "llvm/MC/MCSymbol.h"
+#include "llvm/MC/MCSymbolCOFF.h"
+#include "llvm/MC/MCSymbolELF.h"
+#include "llvm/MC/MCSymbolMachO.h"
 #include "llvm/Support/ELF.h"
 #include "llvm/Support/ErrorHandling.h"
 #include "llvm/Support/FileSystem.h"
@@ -54,7 +57,6 @@ MCContext::MCContext(const MCAsmInfo *mai, const MCRegisterInfo *mri,
 }
 
 MCContext::~MCContext() {
-
   if (AutoReset)
     reset();
 
@@ -62,7 +64,7 @@ MCContext::~MCContext() {
   // we don't need to free them here.
 
   // If the stream for the .secure_log_unique directive was created free it.
-  delete (raw_ostream*)SecureLog;
+  delete (raw_ostream *)SecureLog;
 }
 
 //===----------------------------------------------------------------------===//
@@ -70,6 +72,14 @@ MCContext::~MCContext() {
 //===----------------------------------------------------------------------===//
 
 void MCContext::reset() {
+  // Call the destructors so the fragments are freed
+  for (auto &I : ELFUniquingMap)
+    I.second->~MCSectionELF();
+  for (auto &I : COFFUniquingMap)
+    I.second->~MCSectionCOFF();
+  for (auto &I : MachOUniquingMap)
+    I.second->~MCSectionMachO();
+
   UsedNames.clear();
   Symbols.clear();
   Allocator.Reset();
@@ -77,11 +87,11 @@ void MCContext::reset() {
   CompilationDir.clear();
   MainFileName.clear();
   MCDwarfLineTablesCUMap.clear();
-  SectionStartEndSyms.clear();
+  SectionsForRanges.clear();
   MCGenDwarfLabelEntries.clear();
   DwarfDebugFlags = StringRef();
   DwarfCompileUnitID = 0;
-  CurrentDwarfLoc = MCDwarfLoc(0,0,0,DWARF2_FLAG_IS_STMT,0,0);
+  CurrentDwarfLoc = MCDwarfLoc(0, 0, 0, DWARF2_FLAG_IS_STMT, 0, 0);
 
   MachOUniquingMap.clear();
   ELFUniquingMap.clear();
@@ -98,7 +108,7 @@ void MCContext::reset() {
 // Symbol Manipulation
 //===----------------------------------------------------------------------===//
 
-MCSymbol *MCContext::GetOrCreateSymbol(const Twine &Name) {
+MCSymbol *MCContext::getOrCreateSymbol(const Twine &Name) {
   SmallString<128> NameSV;
   StringRef NameRef = Name.toStringRef(NameSV);
 
@@ -106,13 +116,13 @@ MCSymbol *MCContext::GetOrCreateSymbol(const Twine &Name) {
 
   MCSymbol *&Sym = Symbols[NameRef];
   if (!Sym)
-    Sym = CreateSymbol(NameRef, false);
+    Sym = createSymbol(NameRef, false, false);
 
   return Sym;
 }
 
-MCSymbol *MCContext::getOrCreateSectionSymbol(const MCSectionELF &Section) {
-  MCSymbol *&Sym = SectionSymbols[&Section];
+MCSymbolELF *MCContext::getOrCreateSectionSymbol(const MCSectionELF &Section) {
+  MCSymbolELF *&Sym = SectionSymbols[&Section];
   if (Sym)
     return Sym;
 
@@ -120,12 +130,12 @@ MCSymbol *MCContext::getOrCreateSectionSymbol(const MCSectionELF &Section) {
 
   MCSymbol *&OldSym = Symbols[Name];
   if (OldSym && OldSym->isUndefined()) {
-    Sym = OldSym;
-    return OldSym;
+    Sym = cast<MCSymbolELF>(OldSym);
+    return Sym;
   }
 
   auto NameIter = UsedNames.insert(std::make_pair(Name, true)).first;
-  Sym = new (*this) MCSymbol(NameIter->getKey(), /*isTemporary*/ false);
+  Sym = new (&*NameIter, *this) MCSymbolELF(&*NameIter, /*isTemporary*/ false);
 
   if (!OldSym)
     OldSym = Sym;
@@ -135,19 +145,45 @@ MCSymbol *MCContext::getOrCreateSectionSymbol(const MCSectionELF &Section) {
 
 MCSymbol *MCContext::getOrCreateFrameAllocSymbol(StringRef FuncName,
                                                  unsigned Idx) {
-  return GetOrCreateSymbol(Twine(MAI->getPrivateGlobalPrefix()) + FuncName +
+  return getOrCreateSymbol(Twine(MAI->getPrivateGlobalPrefix()) + FuncName +
                            "$frame_escape_" + Twine(Idx));
 }
 
 MCSymbol *MCContext::getOrCreateParentFrameOffsetSymbol(StringRef FuncName) {
-  return GetOrCreateSymbol(Twine(MAI->getPrivateGlobalPrefix()) + FuncName +
+  return getOrCreateSymbol(Twine(MAI->getPrivateGlobalPrefix()) + FuncName +
                            "$parent_frame_offset");
 }
 
-MCSymbol *MCContext::CreateSymbol(StringRef Name, bool AlwaysAddSuffix) {
-  // Determine whether this is an assembler temporary or normal label, if used.
-  bool IsTemporary = false;
-  if (AllowTemporaryLabels)
+MCSymbol *MCContext::getOrCreateLSDASymbol(StringRef FuncName) {
+  return getOrCreateSymbol(Twine(MAI->getPrivateGlobalPrefix()) + "__ehtable$" +
+                           FuncName);
+}
+
+MCSymbol *MCContext::createSymbolImpl(const StringMapEntry<bool> *Name,
+                                      bool IsTemporary) {
+  if (MOFI) {
+    switch (MOFI->getObjectFileType()) {
+    case MCObjectFileInfo::IsCOFF:
+      return new (Name, *this) MCSymbolCOFF(Name, IsTemporary);
+    case MCObjectFileInfo::IsELF:
+      return new (Name, *this) MCSymbolELF(Name, IsTemporary);
+    case MCObjectFileInfo::IsMachO:
+      return new (Name, *this) MCSymbolMachO(Name, IsTemporary);
+    }
+  }
+  return new (Name, *this) MCSymbol(MCSymbol::SymbolKindUnset, Name,
+                                    IsTemporary);
+}
+
+MCSymbol *MCContext::createSymbol(StringRef Name, bool AlwaysAddSuffix,
+                                  bool CanBeUnnamed) {
+  if (CanBeUnnamed && !UseNamesOnTempLabels)
+    return createSymbolImpl(nullptr, true);
+
+  // Determine whether this is an user writter assembler temporary or normal
+  // label, if used.
+  bool IsTemporary = CanBeUnnamed;
+  if (AllowTemporaryLabels && !IsTemporary)
     IsTemporary = Name.startswith(MAI->getPrivateGlobalPrefix());
 
   SmallString<128> NewName = Name;
@@ -162,9 +198,7 @@ MCSymbol *MCContext::CreateSymbol(StringRef Name, bool AlwaysAddSuffix) {
     if (NameEntry.second) {
       // Ok, we found a name. Have the MCSymbol object itself refer to the copy
       // of the string that is embedded in the UsedNames entry.
-      MCSymbol *Result =
-          new (*this) MCSymbol(NameEntry.first->getKey(), IsTemporary);
-      return Result;
+      return createSymbolImpl(&*NameEntry.first, IsTemporary);
     }
     assert(IsTemporary && "Cannot rename non-temporary symbols");
     AddSuffix = true;
@@ -172,20 +206,21 @@ MCSymbol *MCContext::CreateSymbol(StringRef Name, bool AlwaysAddSuffix) {
   llvm_unreachable("Infinite loop");
 }
 
-MCSymbol *MCContext::createTempSymbol(const Twine &Name, bool AlwaysAddSuffix) {
+MCSymbol *MCContext::createTempSymbol(const Twine &Name, bool AlwaysAddSuffix,
+                                      bool CanBeUnnamed) {
   SmallString<128> NameSV;
   raw_svector_ostream(NameSV) << MAI->getPrivateGlobalPrefix() << Name;
-  return CreateSymbol(NameSV, AlwaysAddSuffix);
+  return createSymbol(NameSV, AlwaysAddSuffix, CanBeUnnamed);
 }
 
-MCSymbol *MCContext::CreateLinkerPrivateTempSymbol() {
+MCSymbol *MCContext::createLinkerPrivateTempSymbol() {
   SmallString<128> NameSV;
   raw_svector_ostream(NameSV) << MAI->getLinkerPrivateGlobalPrefix() << "tmp";
-  return CreateSymbol(NameSV, true);
+  return createSymbol(NameSV, true, false);
 }
 
-MCSymbol *MCContext::CreateTempSymbol() {
-  return createTempSymbol("tmp", true);
+MCSymbol *MCContext::createTempSymbol(bool CanBeUnnamed) {
+  return createTempSymbol("tmp", true, CanBeUnnamed);
 }
 
 unsigned MCContext::NextInstance(unsigned LocalLabelVal) {
@@ -206,16 +241,16 @@ MCSymbol *MCContext::getOrCreateDirectionalLocalSymbol(unsigned LocalLabelVal,
                                                        unsigned Instance) {
   MCSymbol *&Sym = LocalSymbols[std::make_pair(LocalLabelVal, Instance)];
   if (!Sym)
-    Sym = CreateTempSymbol();
+    Sym = createTempSymbol(false);
   return Sym;
 }
 
-MCSymbol *MCContext::CreateDirectionalLocalSymbol(unsigned LocalLabelVal) {
+MCSymbol *MCContext::createDirectionalLocalSymbol(unsigned LocalLabelVal) {
   unsigned Instance = NextInstance(LocalLabelVal);
   return getOrCreateDirectionalLocalSymbol(LocalLabelVal, Instance);
 }
 
-MCSymbol *MCContext::GetDirectionalLocalSymbol(unsigned LocalLabelVal,
+MCSymbol *MCContext::getDirectionalLocalSymbol(unsigned LocalLabelVal,
                                                bool Before) {
   unsigned Instance = GetInstance(LocalLabelVal);
   if (!Before)
@@ -223,7 +258,7 @@ MCSymbol *MCContext::GetDirectionalLocalSymbol(unsigned LocalLabelVal,
   return getOrCreateDirectionalLocalSymbol(LocalLabelVal, Instance);
 }
 
-MCSymbol *MCContext::LookupSymbol(const Twine &Name) const {
+MCSymbol *MCContext::lookupSymbol(const Twine &Name) const {
   SmallString<128> NameSV;
   StringRef NameRef = Name.toStringRef(NameSV);
   return Symbols.lookup(NameRef);
@@ -233,10 +268,10 @@ MCSymbol *MCContext::LookupSymbol(const Twine &Name) const {
 // Section Management
 //===----------------------------------------------------------------------===//
 
-const MCSectionMachO *
-MCContext::getMachOSection(StringRef Segment, StringRef Section,
-                           unsigned TypeAndAttributes, unsigned Reserved2,
-                           SectionKind Kind, const char *BeginSymName) {
+MCSectionMachO *MCContext::getMachOSection(StringRef Segment, StringRef Section,
+                                           unsigned TypeAndAttributes,
+                                           unsigned Reserved2, SectionKind Kind,
+                                           const char *BeginSymName) {
 
   // We unique sections by their segment/section pair.  The returned section
   // may not have the same flags as the requested section, if so this should be
@@ -249,7 +284,7 @@ MCContext::getMachOSection(StringRef Segment, StringRef Section,
   Name += Section;
 
   // Do the lookup, if we have a hit, return it.
-  const MCSectionMachO *&Entry = MachOUniquingMap[Name];
+  MCSectionMachO *&Entry = MachOUniquingMap[Name];
   if (Entry)
     return Entry;
 
@@ -262,7 +297,7 @@ MCContext::getMachOSection(StringRef Segment, StringRef Section,
                                             Reserved2, Kind, Begin);
 }
 
-void MCContext::renameELFSection(const MCSectionELF *Section, StringRef Name) {
+void MCContext::renameELFSection(MCSectionELF *Section, StringRef Name) {
   StringRef GroupName;
   if (const MCSymbol *Group = Section->getGroup())
     GroupName = Group->getName();
@@ -272,15 +307,16 @@ void MCContext::renameELFSection(const MCSectionELF *Section, StringRef Name) {
       ELFSectionKey{Section->getSectionName(), GroupName, UniqueID});
   auto I = ELFUniquingMap.insert(std::make_pair(
                                      ELFSectionKey{Name, GroupName, UniqueID},
-                                     Section)).first;
+                                     Section))
+               .first;
   StringRef CachedName = I->first.SectionName;
-  const_cast<MCSectionELF*>(Section)->setSectionName(CachedName);
+  const_cast<MCSectionELF *>(Section)->setSectionName(CachedName);
 }
 
-const MCSectionELF *
-MCContext::createELFRelSection(StringRef Name, unsigned Type, unsigned Flags,
-                               unsigned EntrySize, const MCSymbol *Group,
-                               const MCSectionELF *Associated) {
+MCSectionELF *MCContext::createELFRelSection(StringRef Name, unsigned Type,
+                                             unsigned Flags, unsigned EntrySize,
+                                             const MCSymbolELF *Group,
+                                             const MCSectionELF *Associated) {
   StringMap<bool>::iterator I;
   bool Inserted;
   std::tie(I, Inserted) = ELFRelSecNames.insert(std::make_pair(Name, true));
@@ -290,24 +326,24 @@ MCContext::createELFRelSection(StringRef Name, unsigned Type, unsigned Flags,
                    EntrySize, Group, true, nullptr, Associated);
 }
 
-const MCSectionELF *MCContext::getELFSection(StringRef Section, unsigned Type,
-                                             unsigned Flags, unsigned EntrySize,
-                                             StringRef Group, unsigned UniqueID,
-                                             const char *BeginSymName) {
-  MCSymbol *GroupSym = nullptr;
+MCSectionELF *MCContext::getELFSection(StringRef Section, unsigned Type,
+                                       unsigned Flags, unsigned EntrySize,
+                                       StringRef Group, unsigned UniqueID,
+                                       const char *BeginSymName) {
+  MCSymbolELF *GroupSym = nullptr;
   if (!Group.empty())
-    GroupSym = GetOrCreateSymbol(Group);
+    GroupSym = cast<MCSymbolELF>(getOrCreateSymbol(Group));
 
   return getELFSection(Section, Type, Flags, EntrySize, GroupSym, UniqueID,
                        BeginSymName, nullptr);
 }
 
-const MCSectionELF *MCContext::getELFSection(StringRef Section, unsigned Type,
-                                             unsigned Flags, unsigned EntrySize,
-                                             const MCSymbol *GroupSym,
-                                             unsigned UniqueID,
-                                             const char *BeginSymName,
-                                             const MCSectionELF *Associated) {
+MCSectionELF *MCContext::getELFSection(StringRef Section, unsigned Type,
+                                       unsigned Flags, unsigned EntrySize,
+                                       const MCSymbolELF *GroupSym,
+                                       unsigned UniqueID,
+                                       const char *BeginSymName,
+                                       const MCSectionELF *Associated) {
   StringRef Group = "";
   if (GroupSym)
     Group = GroupSym->getName();
@@ -337,20 +373,21 @@ const MCSectionELF *MCContext::getELFSection(StringRef Section, unsigned Type,
   return Result;
 }
 
-const MCSectionELF *MCContext::CreateELFGroupSection() {
+MCSectionELF *MCContext::createELFGroupSection(const MCSymbolELF *Group) {
   MCSectionELF *Result = new (*this)
       MCSectionELF(".group", ELF::SHT_GROUP, 0, SectionKind::getReadOnly(), 4,
-                   nullptr, ~0, nullptr, nullptr);
+                   Group, ~0, nullptr, nullptr);
   return Result;
 }
 
-const MCSectionCOFF *
-MCContext::getCOFFSection(StringRef Section, unsigned Characteristics,
-                          SectionKind Kind, StringRef COMDATSymName,
-                          int Selection, const char *BeginSymName) {
+MCSectionCOFF *MCContext::getCOFFSection(StringRef Section,
+                                         unsigned Characteristics,
+                                         SectionKind Kind,
+                                         StringRef COMDATSymName, int Selection,
+                                         const char *BeginSymName) {
   MCSymbol *COMDATSymbol = nullptr;
   if (!COMDATSymName.empty()) {
-    COMDATSymbol = GetOrCreateSymbol(COMDATSymName);
+    COMDATSymbol = getOrCreateSymbol(COMDATSymName);
     COMDATSymName = COMDATSymbol->getName();
   }
 
@@ -373,14 +410,14 @@ MCContext::getCOFFSection(StringRef Section, unsigned Characteristics,
   return Result;
 }
 
-const MCSectionCOFF *MCContext::getCOFFSection(StringRef Section,
-                                               unsigned Characteristics,
-                                               SectionKind Kind,
-                                               const char *BeginSymName) {
+MCSectionCOFF *MCContext::getCOFFSection(StringRef Section,
+                                         unsigned Characteristics,
+                                         SectionKind Kind,
+                                         const char *BeginSymName) {
   return getCOFFSection(Section, Characteristics, Kind, "", 0, BeginSymName);
 }
 
-const MCSectionCOFF *MCContext::getCOFFSection(StringRef Section) {
+MCSectionCOFF *MCContext::getCOFFSection(StringRef Section) {
   COFFSectionKey T{Section, "", 0};
   auto Iter = COFFUniquingMap.find(T);
   if (Iter == COFFUniquingMap.end())
@@ -388,9 +425,8 @@ const MCSectionCOFF *MCContext::getCOFFSection(StringRef Section) {
   return Iter->second;
 }
 
-const MCSectionCOFF *
-MCContext::getAssociativeCOFFSection(const MCSectionCOFF *Sec,
-                                     const MCSymbol *KeySym) {
+MCSectionCOFF *MCContext::getAssociativeCOFFSection(MCSectionCOFF *Sec,
+                                                    const MCSymbol *KeySym) {
   // Return the normal section if we don't have to be associative.
   if (!KeySym)
     return Sec;
@@ -408,11 +444,11 @@ MCContext::getAssociativeCOFFSection(const MCSectionCOFF *Sec,
 // Dwarf Management
 //===----------------------------------------------------------------------===//
 
-/// GetDwarfFile - takes a file name an number to place in the dwarf file and
+/// getDwarfFile - takes a file name an number to place in the dwarf file and
 /// directory tables.  If the file number has already been allocated it is an
 /// error and zero is returned and the client reports the error, else the
 /// allocated file number is returned.  The file numbers may be in any order.
-unsigned MCContext::GetDwarfFile(StringRef Directory, StringRef FileName,
+unsigned MCContext::getDwarfFile(StringRef Directory, StringRef FileName,
                                  unsigned FileNumber, unsigned CUID) {
   MCDwarfLineTable &Table = MCDwarfLineTablesCUMap[CUID];
   return Table.getFile(Directory, FileName, FileNumber);
@@ -421,37 +457,21 @@ unsigned MCContext::GetDwarfFile(StringRef Directory, StringRef FileName,
 /// isValidDwarfFileNumber - takes a dwarf file number and returns true if it
 /// currently is assigned and false otherwise.
 bool MCContext::isValidDwarfFileNumber(unsigned FileNumber, unsigned CUID) {
-  const SmallVectorImpl<MCDwarfFile>& MCDwarfFiles = getMCDwarfFiles(CUID);
-  if(FileNumber == 0 || FileNumber >= MCDwarfFiles.size())
+  const SmallVectorImpl<MCDwarfFile> &MCDwarfFiles = getMCDwarfFiles(CUID);
+  if (FileNumber == 0 || FileNumber >= MCDwarfFiles.size())
     return false;
 
   return !MCDwarfFiles[FileNumber].Name.empty();
 }
 
-/// finalizeDwarfSections - Emit end symbols for each non-empty code section.
-/// Also remove empty sections from SectionStartEndSyms, to avoid generating
+/// Remove empty sections from SectionStartEndSyms, to avoid generating
 /// useless debug info for them.
 void MCContext::finalizeDwarfSections(MCStreamer &MCOS) {
-  MCContext &context = MCOS.getContext();
-
-  auto sec = SectionStartEndSyms.begin();
-  while (sec != SectionStartEndSyms.end()) {
-    assert(sec->second.first && "Start symbol must be set by now");
-    MCOS.SwitchSection(sec->first);
-    if (MCOS.mayHaveInstructions()) {
-      MCSymbol *SectionEndSym = context.CreateTempSymbol();
-      MCOS.EmitLabel(SectionEndSym);
-      sec->second.second = SectionEndSym;
-      ++sec;
-    } else {
-      MapVector<const MCSection *, std::pair<MCSymbol *, MCSymbol *> >::iterator
-        to_erase = sec;
-      sec = SectionStartEndSyms.erase(to_erase);
-    }
-  }
+  SectionsForRanges.remove_if(
+      [&](MCSection *Sec) { return !MCOS.mayHaveInstructions(*Sec); });
 }
 
-void MCContext::FatalError(SMLoc Loc, const Twine &Msg) const {
+void MCContext::reportFatalError(SMLoc Loc, const Twine &Msg) const {
   // If we have a source manager and a location, use it. Otherwise just
   // use the generic report_fatal_error().
   if (!SrcMgr || Loc == SMLoc())
diff --git a/lib/MC/MCDisassembler/MCExternalSymbolizer.cpp b/lib/MC/MCDisassembler/MCExternalSymbolizer.cpp
index f306e4e4ef7..5fc2ca44f5d 100644
--- a/lib/MC/MCDisassembler/MCExternalSymbolizer.cpp
+++ b/lib/MC/MCDisassembler/MCExternalSymbolizer.cpp
@@ -16,6 +16,10 @@
 
 using namespace llvm;
 
+namespace llvm {
+class Triple;
+}
+
 // This function tries to add a symbolic operand in place of the immediate
 // Value in the MCInst. The immediate Value has had any PC adjustment made by
 // the caller. If the instruction is a branch instruction then IsBranch is true,
@@ -87,10 +91,10 @@ bool MCExternalSymbolizer::tryAddingSymbolicOperand(MCInst &MI,
   if (SymbolicOp.AddSymbol.Present) {
     if (SymbolicOp.AddSymbol.Name) {
       StringRef Name(SymbolicOp.AddSymbol.Name);
-      MCSymbol *Sym = Ctx.GetOrCreateSymbol(Name);
-      Add = MCSymbolRefExpr::Create(Sym, Ctx);
+      MCSymbol *Sym = Ctx.getOrCreateSymbol(Name);
+      Add = MCSymbolRefExpr::create(Sym, Ctx);
     } else {
-      Add = MCConstantExpr::Create((int)SymbolicOp.AddSymbol.Value, Ctx);
+      Add = MCConstantExpr::create((int)SymbolicOp.AddSymbol.Value, Ctx);
     }
   }
 
@@ -98,45 +102,45 @@ bool MCExternalSymbolizer::tryAddingSymbolicOperand(MCInst &MI,
   if (SymbolicOp.SubtractSymbol.Present) {
       if (SymbolicOp.SubtractSymbol.Name) {
       StringRef Name(SymbolicOp.SubtractSymbol.Name);
-      MCSymbol *Sym = Ctx.GetOrCreateSymbol(Name);
-      Sub = MCSymbolRefExpr::Create(Sym, Ctx);
+      MCSymbol *Sym = Ctx.getOrCreateSymbol(Name);
+      Sub = MCSymbolRefExpr::create(Sym, Ctx);
     } else {
-      Sub = MCConstantExpr::Create((int)SymbolicOp.SubtractSymbol.Value, Ctx);
+      Sub = MCConstantExpr::create((int)SymbolicOp.SubtractSymbol.Value, Ctx);
     }
   }
 
   const MCExpr *Off = nullptr;
   if (SymbolicOp.Value != 0)
-    Off = MCConstantExpr::Create(SymbolicOp.Value, Ctx);
+    Off = MCConstantExpr::create(SymbolicOp.Value, Ctx);
 
   const MCExpr *Expr;
   if (Sub) {
     const MCExpr *LHS;
     if (Add)
-      LHS = MCBinaryExpr::CreateSub(Add, Sub, Ctx);
+      LHS = MCBinaryExpr::createSub(Add, Sub, Ctx);
     else
-      LHS = MCUnaryExpr::CreateMinus(Sub, Ctx);
+      LHS = MCUnaryExpr::createMinus(Sub, Ctx);
     if (Off)
-      Expr = MCBinaryExpr::CreateAdd(LHS, Off, Ctx);
+      Expr = MCBinaryExpr::createAdd(LHS, Off, Ctx);
     else
       Expr = LHS;
   } else if (Add) {
     if (Off)
-      Expr = MCBinaryExpr::CreateAdd(Add, Off, Ctx);
+      Expr = MCBinaryExpr::createAdd(Add, Off, Ctx);
     else
       Expr = Add;
   } else {
     if (Off)
       Expr = Off;
     else
-      Expr = MCConstantExpr::Create(0, Ctx);
+      Expr = MCConstantExpr::create(0, Ctx);
   }
 
   Expr = RelInfo->createExprForCAPIVariantKind(Expr, SymbolicOp.VariantKind);
   if (!Expr)
     return false;
 
-  MI.addOperand(MCOperand::CreateExpr(Expr));
+  MI.addOperand(MCOperand::createExpr(Expr));
   return true;
 }
 
@@ -184,7 +188,7 @@ void MCExternalSymbolizer::tryAddingPcLoadReferenceComment(raw_ostream &cStream,
 }
 
 namespace llvm {
-MCSymbolizer *createMCSymbolizer(StringRef TT, LLVMOpInfoCallback GetOpInfo,
+MCSymbolizer *createMCSymbolizer(const Triple &TT, LLVMOpInfoCallback GetOpInfo,
                                  LLVMSymbolLookupCallback SymbolLookUp,
                                  void *DisInfo, MCContext *Ctx,
                                  std::unique_ptr<MCRelocationInfo> &&RelInfo) {
diff --git a/lib/MC/MCDisassembler/MCRelocationInfo.cpp b/lib/MC/MCDisassembler/MCRelocationInfo.cpp
index ff0c27f5faf..43005e7c740 100644
--- a/lib/MC/MCDisassembler/MCRelocationInfo.cpp
+++ b/lib/MC/MCDisassembler/MCRelocationInfo.cpp
@@ -34,6 +34,7 @@ MCRelocationInfo::createExprForCAPIVariantKind(const MCExpr *SubExpr,
   return SubExpr;
 }
 
-MCRelocationInfo *llvm::createMCRelocationInfo(StringRef TT, MCContext &Ctx) {
+MCRelocationInfo *llvm::createMCRelocationInfo(const Triple &TT,
+                                               MCContext &Ctx) {
   return new MCRelocationInfo(Ctx);
 }
diff --git a/lib/MC/MCDwarf.cpp b/lib/MC/MCDwarf.cpp
index e9f685eb88f..c84c4865f51 100644
--- a/lib/MC/MCDwarf.cpp
+++ b/lib/MC/MCDwarf.cpp
@@ -64,12 +64,12 @@ static inline uint64_t ScaleAddrDelta(MCContext &Context, uint64_t AddrDelta) {
 // and if there is information from the last .loc directive that has yet to have
 // a line entry made for it is made.
 //
-void MCLineEntry::Make(MCObjectStreamer *MCOS, const MCSection *Section) {
+void MCLineEntry::Make(MCObjectStreamer *MCOS, MCSection *Section) {
   if (!MCOS->getContext().getDwarfLocSeen())
     return;
 
   // Create a symbol at in the current section for use in the line entry.
-  MCSymbol *LineSym = MCOS->getContext().CreateTempSymbol();
+  MCSymbol *LineSym = MCOS->getContext().createTempSymbol();
   // Set the value of the symbol to use for the MCLineEntry.
   MCOS->EmitLabel(LineSym);
 
@@ -80,7 +80,7 @@ void MCLineEntry::Make(MCObjectStreamer *MCOS, const MCSection *Section) {
   MCLineEntry LineEntry(LineSym, DwarfLoc);
 
   // clear DwarfLocSeen saying the current .loc info is now used.
-  MCOS->getContext().ClearDwarfLocSeen();
+  MCOS->getContext().clearDwarfLocSeen();
 
   // Add the line entry to this section's entries.
   MCOS->getContext()
@@ -98,15 +98,15 @@ static inline const MCExpr *MakeStartMinusEndExpr(const MCStreamer &MCOS,
                                                   int IntVal) {
   MCSymbolRefExpr::VariantKind Variant = MCSymbolRefExpr::VK_None;
   const MCExpr *Res =
-    MCSymbolRefExpr::Create(&End, Variant, MCOS.getContext());
+    MCSymbolRefExpr::create(&End, Variant, MCOS.getContext());
   const MCExpr *RHS =
-    MCSymbolRefExpr::Create(&Start, Variant, MCOS.getContext());
+    MCSymbolRefExpr::create(&Start, Variant, MCOS.getContext());
   const MCExpr *Res1 =
-    MCBinaryExpr::Create(MCBinaryExpr::Sub, Res, RHS, MCOS.getContext());
+    MCBinaryExpr::create(MCBinaryExpr::Sub, Res, RHS, MCOS.getContext());
   const MCExpr *Res2 =
-    MCConstantExpr::Create(IntVal, MCOS.getContext());
+    MCConstantExpr::create(IntVal, MCOS.getContext());
   const MCExpr *Res3 =
-    MCBinaryExpr::Create(MCBinaryExpr::Sub, Res1, Res2, MCOS.getContext());
+    MCBinaryExpr::create(MCBinaryExpr::Sub, Res1, Res2, MCOS.getContext());
   return Res3;
 }
 
@@ -115,7 +115,7 @@ static inline const MCExpr *MakeStartMinusEndExpr(const MCStreamer &MCOS,
 // in the LineSection.
 //
 static inline void
-EmitDwarfLineTable(MCObjectStreamer *MCOS, const MCSection *Section,
+EmitDwarfLineTable(MCObjectStreamer *MCOS, MCSection *Section,
                    const MCLineSection::MCLineEntryCollection &LineEntries) {
   unsigned FileNum = 1;
   unsigned LastLine = 1;
@@ -245,9 +245,9 @@ static const MCExpr *forceExpAbs(MCStreamer &OS, const MCExpr* Expr) {
   if (Context.getAsmInfo()->hasAggressiveSymbolFolding())
     return Expr;
 
-  MCSymbol *ABS = Context.CreateTempSymbol();
+  MCSymbol *ABS = Context.createTempSymbol();
   OS.EmitAssignment(ABS, Expr);
-  return MCSymbolRefExpr::Create(ABS, Context);
+  return MCSymbolRefExpr::create(ABS, Context);
 }
 
 static void emitAbsValue(MCStreamer &OS, const MCExpr *Value, unsigned Size) {
@@ -264,12 +264,12 @@ MCDwarfLineTableHeader::Emit(MCStreamer *MCOS,
   // Create a symbol at the beginning of the line table.
   MCSymbol *LineStartSym = Label;
   if (!LineStartSym)
-    LineStartSym = context.CreateTempSymbol();
+    LineStartSym = context.createTempSymbol();
   // Set the value of the symbol, as we are at the start of the line table.
   MCOS->EmitLabel(LineStartSym);
 
   // Create a symbol for the end of the section (to be set when we get there).
-  MCSymbol *LineEndSym = context.CreateTempSymbol();
+  MCSymbol *LineEndSym = context.createTempSymbol();
 
   // The first 4 bytes is the total length of the information for this
   // compilation unit (not including these 4 bytes for the length).
@@ -280,7 +280,7 @@ MCDwarfLineTableHeader::Emit(MCStreamer *MCOS,
   MCOS->EmitIntValue(2, 2);
 
   // Create a symbol for the end of the prologue (to be set when we get there).
-  MCSymbol *ProEndSym = context.CreateTempSymbol(); // Lprologue_end
+  MCSymbol *ProEndSym = context.createTempSymbol(); // Lprologue_end
 
   // Length of the prologue, is the next 4 bytes.  Which is the start of the
   // section to the end of the prologue.  Not including the 4 bytes for the
@@ -360,8 +360,10 @@ unsigned MCDwarfLineTableHeader::getFile(StringRef &Directory,
     FileNumber = SourceIdMap.size() + 1;
     assert((MCDwarfFiles.empty() || FileNumber == MCDwarfFiles.size()) &&
            "Don't mix autonumbered and explicit numbered line table usage");
+    SmallString<256> Buffer;
     auto IterBool = SourceIdMap.insert(
-        std::make_pair((Directory + Twine('\0') + FileName).str(), FileNumber));
+        std::make_pair((Directory + Twine('\0') + FileName).toStringRef(Buffer),
+                       FileNumber));
     if (!IterBool.second)
       return IterBool.first->second;
   }
@@ -608,13 +610,13 @@ static void EmitGenDwarfAranges(MCStreamer *MCOS,
 
   // Now emit the table of pairs of PointerSize'ed values for the section
   // addresses and sizes.
-  for (const auto &sec : Sections) {
-    MCSymbol *StartSymbol = sec.second.first;
-    MCSymbol *EndSymbol = sec.second.second;
+  for (MCSection *Sec : Sections) {
+    const MCSymbol *StartSymbol = Sec->getBeginSymbol();
+    MCSymbol *EndSymbol = Sec->getEndSymbol(context);
     assert(StartSymbol && "StartSymbol must not be NULL");
     assert(EndSymbol && "EndSymbol must not be NULL");
 
-    const MCExpr *Addr = MCSymbolRefExpr::Create(
+    const MCExpr *Addr = MCSymbolRefExpr::create(
       StartSymbol, MCSymbolRefExpr::VK_None, context);
     const MCExpr *Size = MakeStartMinusEndExpr(*MCOS,
       *StartSymbol, *EndSymbol, 0);
@@ -640,9 +642,9 @@ static void EmitGenDwarfInfo(MCStreamer *MCOS,
 
   // Create a symbol at the start and end of this section used in here for the
   // expression to calculate the length in the header.
-  MCSymbol *InfoStart = context.CreateTempSymbol();
+  MCSymbol *InfoStart = context.createTempSymbol();
   MCOS->EmitLabel(InfoStart);
-  MCSymbol *InfoEnd = context.CreateTempSymbol();
+  MCSymbol *InfoEnd = context.createTempSymbol();
 
   // First part: the header.
 
@@ -697,18 +699,18 @@ static void EmitGenDwarfInfo(MCStreamer *MCOS,
     const auto TextSection = Sections.begin();
     assert(TextSection != Sections.end() && "No text section found");
 
-    MCSymbol *StartSymbol = TextSection->second.first;
-    MCSymbol *EndSymbol = TextSection->second.second;
+    MCSymbol *StartSymbol = (*TextSection)->getBeginSymbol();
+    MCSymbol *EndSymbol = (*TextSection)->getEndSymbol(context);
     assert(StartSymbol && "StartSymbol must not be NULL");
     assert(EndSymbol && "EndSymbol must not be NULL");
 
     // AT_low_pc, the first address of the default .text section.
-    const MCExpr *Start = MCSymbolRefExpr::Create(
+    const MCExpr *Start = MCSymbolRefExpr::create(
         StartSymbol, MCSymbolRefExpr::VK_None, context);
     MCOS->EmitValue(Start, AddrSize);
 
     // AT_high_pc, the last address of the default .text section.
-    const MCExpr *End = MCSymbolRefExpr::Create(
+    const MCExpr *End = MCSymbolRefExpr::create(
       EndSymbol, MCSymbolRefExpr::VK_None, context);
     MCOS->EmitValue(End, AddrSize);
   }
@@ -770,7 +772,7 @@ static void EmitGenDwarfInfo(MCStreamer *MCOS,
     MCOS->EmitIntValue(Entry.getLineNumber(), 4);
 
     // AT_low_pc, start address of the label.
-    const MCExpr *AT_low_pc = MCSymbolRefExpr::Create(Entry.getLabel(),
+    const MCExpr *AT_low_pc = MCSymbolRefExpr::create(Entry.getLabel(),
                                              MCSymbolRefExpr::VK_None, context);
     MCOS->EmitValue(AT_low_pc, AddrSize);
 
@@ -803,15 +805,14 @@ static void EmitGenDwarfRanges(MCStreamer *MCOS) {
 
   MCOS->SwitchSection(context.getObjectFileInfo()->getDwarfRangesSection());
 
-  for (const auto &sec : Sections) {
-
-    MCSymbol *StartSymbol = sec.second.first;
-    MCSymbol *EndSymbol = sec.second.second;
+  for (MCSection *Sec : Sections) {
+    const MCSymbol *StartSymbol = Sec->getBeginSymbol();
+    MCSymbol *EndSymbol = Sec->getEndSymbol(context);
     assert(StartSymbol && "StartSymbol must not be NULL");
     assert(EndSymbol && "EndSymbol must not be NULL");
 
     // Emit a base address selection entry for the start of this section
-    const MCExpr *SectionStartAddr = MCSymbolRefExpr::Create(
+    const MCExpr *SectionStartAddr = MCSymbolRefExpr::create(
       StartSymbol, MCSymbolRefExpr::VK_None, context);
     MCOS->EmitFill(AddrSize, 0xFF);
     MCOS->EmitValue(SectionStartAddr, AddrSize);
@@ -863,18 +864,18 @@ void MCGenDwarfInfo::Emit(MCStreamer *MCOS) {
 
   MCOS->SwitchSection(context.getObjectFileInfo()->getDwarfInfoSection());
   if (CreateDwarfSectionSymbols) {
-    InfoSectionSymbol = context.CreateTempSymbol();
+    InfoSectionSymbol = context.createTempSymbol();
     MCOS->EmitLabel(InfoSectionSymbol);
   }
   MCOS->SwitchSection(context.getObjectFileInfo()->getDwarfAbbrevSection());
   if (CreateDwarfSectionSymbols) {
-    AbbrevSectionSymbol = context.CreateTempSymbol();
+    AbbrevSectionSymbol = context.createTempSymbol();
     MCOS->EmitLabel(AbbrevSectionSymbol);
   }
   if (UseRangesSection) {
     MCOS->SwitchSection(context.getObjectFileInfo()->getDwarfRangesSection());
     if (CreateDwarfSectionSymbols) {
-      RangesSectionSymbol = context.CreateTempSymbol();
+      RangesSectionSymbol = context.createTempSymbol();
       MCOS->EmitLabel(RangesSectionSymbol);
     }
   }
@@ -932,7 +933,7 @@ void MCGenDwarfLabelEntry::Make(MCSymbol *Symbol, MCStreamer *MCOS,
   // values so that they don't have things like an ARM thumb bit from the
   // original symbol. So when used they won't get a low bit set after
   // relocation.
-  MCSymbol *Label = context.CreateTempSymbol();
+  MCSymbol *Label = context.createTempSymbol();
   MCOS->EmitLabel(Label);
 
   // Create and entry for the info and add it to the other entries.
@@ -1244,6 +1245,20 @@ void FrameEmitterImpl::EmitCompactUnwind(MCObjectStreamer &Streamer,
     Streamer.EmitIntValue(0, Size); // No LSDA
 }
 
+static unsigned getCIEVersion(bool IsEH, unsigned DwarfVersion) {
+  if (IsEH)
+    return 1;
+  switch (DwarfVersion) {
+  case 2:
+    return 1;
+  case 3:
+    return 3;
+  case 4:
+    return 4;
+  }
+  llvm_unreachable("Unknown version");
+}
+
 const MCSymbol &FrameEmitterImpl::EmitCIE(MCObjectStreamer &streamer,
                                           const MCSymbol *personality,
                                           unsigned personalityEncoding,
@@ -1255,10 +1270,10 @@ const MCSymbol &FrameEmitterImpl::EmitCIE(MCObjectStreamer &streamer,
   const MCRegisterInfo *MRI = context.getRegisterInfo();
   const MCObjectFileInfo *MOFI = context.getObjectFileInfo();
 
-  MCSymbol *sectionStart = context.CreateTempSymbol();
+  MCSymbol *sectionStart = context.createTempSymbol();
   streamer.EmitLabel(sectionStart);
 
-  MCSymbol *sectionEnd = context.CreateTempSymbol();
+  MCSymbol *sectionEnd = context.createTempSymbol();
 
   // Length
   const MCExpr *Length = MakeStartMinusEndExpr(streamer, *sectionStart,
@@ -1270,9 +1285,7 @@ const MCSymbol &FrameEmitterImpl::EmitCIE(MCObjectStreamer &streamer,
   streamer.EmitIntValue(CIE_ID, 4);
 
   // Version
-  // For DWARF2, we use CIE version 1
-  // For DWARF3+, we use CIE version 3
-  uint8_t CIEVersion = context.getDwarfVersion() <= 2 ? 1 : 3;
+  uint8_t CIEVersion = getCIEVersion(IsEH, context.getDwarfVersion());
   streamer.EmitIntValue(CIEVersion, 1);
 
   // Augmentation String
@@ -1290,6 +1303,14 @@ const MCSymbol &FrameEmitterImpl::EmitCIE(MCObjectStreamer &streamer,
   }
   streamer.EmitIntValue(0, 1);
 
+  if (CIEVersion >= 4) {
+    // Address Size
+    streamer.EmitIntValue(context.getAsmInfo()->getPointerSize(), 1);
+
+    // Segment Descriptor Size
+    streamer.EmitIntValue(0, 1);
+  }
+
   // Code Alignment Factor
   streamer.EmitULEB128IntValue(context.getAsmInfo()->getMinInstAlignment());
 
@@ -1360,8 +1381,8 @@ MCSymbol *FrameEmitterImpl::EmitFDE(MCObjectStreamer &streamer,
                                     const MCSymbol &cieStart,
                                     const MCDwarfFrameInfo &frame) {
   MCContext &context = streamer.getContext();
-  MCSymbol *fdeStart = context.CreateTempSymbol();
-  MCSymbol *fdeEnd = context.CreateTempSymbol();
+  MCSymbol *fdeStart = context.createTempSymbol();
+  MCSymbol *fdeEnd = context.createTempSymbol();
   const MCObjectFileInfo *MOFI = context.getObjectFileInfo();
 
   CFAOffset = InitialCFAOffset;
@@ -1499,12 +1520,12 @@ void MCDwarfFrameEmitter::Emit(MCObjectStreamer &Streamer, MCAsmBackend *MAB,
 
   if (!NeedsEHFrameSection) return;
 
-  const MCSection &Section =
-    IsEH ? *const_cast<MCObjectFileInfo*>(MOFI)->getEHFrameSection() :
-           *MOFI->getDwarfFrameSection();
+  MCSection &Section =
+      IsEH ? *const_cast<MCObjectFileInfo *>(MOFI)->getEHFrameSection()
+           : *MOFI->getDwarfFrameSection();
 
   Streamer.SwitchSection(&Section);
-  MCSymbol *SectionStart = Context.CreateTempSymbol();
+  MCSymbol *SectionStart = Context.createTempSymbol();
   Streamer.EmitLabel(SectionStart);
   Emitter.setSectionStart(SectionStart);
 
@@ -1569,18 +1590,17 @@ void MCDwarfFrameEmitter::EncodeAdvanceLoc(MCContext &Context,
     OS << uint8_t(dwarf::DW_CFA_advance_loc1);
     OS << uint8_t(AddrDelta);
   } else if (isUInt<16>(AddrDelta)) {
-    // FIXME: check what is the correct behavior on a big endian machine.
     OS << uint8_t(dwarf::DW_CFA_advance_loc2);
-    OS << uint8_t( AddrDelta       & 0xff);
-    OS << uint8_t((AddrDelta >> 8) & 0xff);
+    if (Context.getAsmInfo()->isLittleEndian())
+      support::endian::Writer<support::little>(OS).write<uint16_t>(AddrDelta);
+    else
+      support::endian::Writer<support::big>(OS).write<uint16_t>(AddrDelta);
   } else {
-    // FIXME: check what is the correct behavior on a big endian machine.
     assert(isUInt<32>(AddrDelta));
     OS << uint8_t(dwarf::DW_CFA_advance_loc4);
-    OS << uint8_t( AddrDelta        & 0xff);
-    OS << uint8_t((AddrDelta >> 8)  & 0xff);
-    OS << uint8_t((AddrDelta >> 16) & 0xff);
-    OS << uint8_t((AddrDelta >> 24) & 0xff);
-
+    if (Context.getAsmInfo()->isLittleEndian())
+      support::endian::Writer<support::little>(OS).write<uint32_t>(AddrDelta);
+    else
+      support::endian::Writer<support::big>(OS).write<uint32_t>(AddrDelta);
   }
 }
diff --git a/lib/MC/MCELF.cpp b/lib/MC/MCELF.cpp
deleted file mode 100644
index 369063431fe..00000000000
--- a/lib/MC/MCELF.cpp
+++ /dev/null
@@ -1,85 +0,0 @@
-//===- lib/MC/MCELF.cpp - MC ELF ------------------------------------------===//
-//
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
-//
-//===----------------------------------------------------------------------===//
-//
-// This file implements ELF object file writer information.
-//
-//===----------------------------------------------------------------------===//
-
-#include "llvm/MC/MCELF.h"
-#include "llvm/MC/MCAssembler.h"
-#include "llvm/MC/MCELFSymbolFlags.h"
-#include "llvm/MC/MCFixupKindInfo.h"
-#include "llvm/Support/ELF.h"
-
-namespace llvm {
-
-void MCELF::SetBinding(MCSymbolData &SD, unsigned Binding) {
-  assert(Binding == ELF::STB_LOCAL || Binding == ELF::STB_GLOBAL ||
-         Binding == ELF::STB_WEAK || Binding == ELF::STB_GNU_UNIQUE);
-  uint32_t OtherFlags = SD.getFlags() & ~(0xf << ELF_STB_Shift);
-  SD.setFlags(OtherFlags | (Binding << ELF_STB_Shift));
-}
-
-unsigned MCELF::GetBinding(const MCSymbolData &SD) {
-  uint32_t Binding = (SD.getFlags() & (0xf << ELF_STB_Shift)) >> ELF_STB_Shift;
-  assert(Binding == ELF::STB_LOCAL || Binding == ELF::STB_GLOBAL ||
-         Binding == ELF::STB_WEAK || Binding == ELF::STB_GNU_UNIQUE);
-  return Binding;
-}
-
-void MCELF::SetType(MCSymbolData &SD, unsigned Type) {
-  assert(Type == ELF::STT_NOTYPE || Type == ELF::STT_OBJECT ||
-         Type == ELF::STT_FUNC || Type == ELF::STT_SECTION ||
-         Type == ELF::STT_COMMON || Type == ELF::STT_TLS ||
-         Type == ELF::STT_GNU_IFUNC);
-
-  uint32_t OtherFlags = SD.getFlags() & ~(0xf << ELF_STT_Shift);
-  SD.setFlags(OtherFlags | (Type << ELF_STT_Shift));
-}
-
-unsigned MCELF::GetType(const MCSymbolData &SD) {
-  uint32_t Type = (SD.getFlags() & (0xf << ELF_STT_Shift)) >> ELF_STT_Shift;
-  assert(Type == ELF::STT_NOTYPE || Type == ELF::STT_OBJECT ||
-         Type == ELF::STT_FUNC || Type == ELF::STT_SECTION ||
-         Type == ELF::STT_COMMON || Type == ELF::STT_TLS || Type == ELF::STT_GNU_IFUNC);
-  return Type;
-}
-
-// Visibility is stored in the first two bits of st_other
-// st_other values are stored in the second byte of get/setFlags
-void MCELF::SetVisibility(MCSymbolData &SD, unsigned Visibility) {
-  assert(Visibility == ELF::STV_DEFAULT || Visibility == ELF::STV_INTERNAL ||
-         Visibility == ELF::STV_HIDDEN || Visibility == ELF::STV_PROTECTED);
-
-  uint32_t OtherFlags = SD.getFlags() & ~(0x3 << ELF_STV_Shift);
-  SD.setFlags(OtherFlags | (Visibility << ELF_STV_Shift));
-}
-
-unsigned MCELF::GetVisibility(const MCSymbolData &SD) {
-  unsigned Visibility =
-    (SD.getFlags() & (0x3 << ELF_STV_Shift)) >> ELF_STV_Shift;
-  assert(Visibility == ELF::STV_DEFAULT || Visibility == ELF::STV_INTERNAL ||
-         Visibility == ELF::STV_HIDDEN || Visibility == ELF::STV_PROTECTED);
-  return Visibility;
-}
-
-// Other is stored in the last six bits of st_other
-// st_other values are stored in the second byte of get/setFlags
-void MCELF::setOther(MCSymbolData &SD, unsigned Other) {
-  uint32_t OtherFlags = SD.getFlags() & ~(0x3f << ELF_STO_Shift);
-  SD.setFlags(OtherFlags | (Other << ELF_STO_Shift));
-}
-
-unsigned MCELF::getOther(const MCSymbolData &SD) {
-  unsigned Other =
-    (SD.getFlags() & (0x3f << ELF_STO_Shift)) >> ELF_STO_Shift;
-  return Other;
-}
-
-}
diff --git a/lib/MC/MCELFObjectTargetWriter.cpp b/lib/MC/MCELFObjectTargetWriter.cpp
index dc3d6c35489..bc0ba85a8ff 100644
--- a/lib/MC/MCELFObjectTargetWriter.cpp
+++ b/lib/MC/MCELFObjectTargetWriter.cpp
@@ -24,7 +24,7 @@ MCELFObjectTargetWriter::MCELFObjectTargetWriter(bool Is64Bit_,
     IsN64(IsN64_){
 }
 
-bool MCELFObjectTargetWriter::needsRelocateWithSymbol(const MCSymbolData &SD,
+bool MCELFObjectTargetWriter::needsRelocateWithSymbol(const MCSymbol &Sym,
                                                       unsigned Type) const {
   return false;
 }
diff --git a/lib/MC/MCELFStreamer.cpp b/lib/MC/MCELFStreamer.cpp
index 24b5681e55a..fe9ac21e17f 100644
--- a/lib/MC/MCELFStreamer.cpp
+++ b/lib/MC/MCELFStreamer.cpp
@@ -20,14 +20,13 @@
 #include "llvm/MC/MCAssembler.h"
 #include "llvm/MC/MCCodeEmitter.h"
 #include "llvm/MC/MCContext.h"
-#include "llvm/MC/MCELF.h"
-#include "llvm/MC/MCELFSymbolFlags.h"
 #include "llvm/MC/MCExpr.h"
 #include "llvm/MC/MCInst.h"
 #include "llvm/MC/MCObjectFileInfo.h"
 #include "llvm/MC/MCObjectStreamer.h"
 #include "llvm/MC/MCSection.h"
 #include "llvm/MC/MCSectionELF.h"
+#include "llvm/MC/MCSymbolELF.h"
 #include "llvm/MC/MCSymbol.h"
 #include "llvm/MC/MCValue.h"
 #include "llvm/Support/Debug.h"
@@ -38,11 +37,15 @@
 
 using namespace llvm;
 
+bool MCELFStreamer::isBundleLocked() const {
+  return getCurrentSectionOnly()->isBundleLocked();
+}
+
 MCELFStreamer::~MCELFStreamer() {
 }
 
 void MCELFStreamer::mergeFragment(MCDataFragment *DF,
-                                  MCEncodedFragmentWithFixups *EF) {
+                                  MCDataFragment *EF) {
   MCAssembler &Assembler = getAssembler();
 
   if (Assembler.isBundlingEnabled() && Assembler.getRelaxAll()) {
@@ -102,16 +105,16 @@ void MCELFStreamer::InitSections(bool NoExecStack) {
     SwitchSection(Ctx.getAsmInfo()->getNonexecutableStackSection(Ctx));
 }
 
-void MCELFStreamer::EmitLabel(MCSymbol *Symbol) {
+void MCELFStreamer::EmitLabel(MCSymbol *S) {
+  auto *Symbol = cast<MCSymbolELF>(S);
   assert(Symbol->isUndefined() && "Cannot define a symbol twice!");
 
   MCObjectStreamer::EmitLabel(Symbol);
 
   const MCSectionELF &Section =
     static_cast<const MCSectionELF&>(Symbol->getSection());
-  MCSymbolData &SD = getAssembler().getSymbolData(*Symbol);
   if (Section.getFlags() & ELF::SHF_TLS)
-    MCELF::SetType(SD, ELF::STT_TLS);
+    Symbol->setType(ELF::STT_TLS);
 }
 
 void MCELFStreamer::EmitAssemblerFlag(MCAssemblerFlag Flag) {
@@ -133,18 +136,17 @@ void MCELFStreamer::EmitAssemblerFlag(MCAssemblerFlag Flag) {
 
 // If bundle aligment is used and there are any instructions in the section, it
 // needs to be aligned to at least the bundle size.
-static void setSectionAlignmentForBundling(
-    const MCAssembler &Assembler, MCSectionData *Section) {
-  if (Assembler.isBundlingEnabled() && Section &&
-      Section->hasInstructions() &&
+static void setSectionAlignmentForBundling(const MCAssembler &Assembler,
+                                           MCSection *Section) {
+  if (Section && Assembler.isBundlingEnabled() && Section->hasInstructions() &&
       Section->getAlignment() < Assembler.getBundleAlignSize())
     Section->setAlignment(Assembler.getBundleAlignSize());
 }
 
-void MCELFStreamer::ChangeSection(const MCSection *Section,
+void MCELFStreamer::ChangeSection(MCSection *Section,
                                   const MCExpr *Subsection) {
-  MCSectionData *CurSection = getCurrentSectionData();
-  if (CurSection && CurSection->isBundleLocked())
+  MCSection *CurSection = getCurrentSectionOnly();
+  if (CurSection && isBundleLocked())
     report_fatal_error("Unterminated .bundle_lock when changing a section");
 
   MCAssembler &Asm = getAssembler();
@@ -153,19 +155,24 @@ void MCELFStreamer::ChangeSection(const MCSection *Section,
   auto *SectionELF = static_cast<const MCSectionELF *>(Section);
   const MCSymbol *Grp = SectionELF->getGroup();
   if (Grp)
-    Asm.getOrCreateSymbolData(*Grp);
+    Asm.registerSymbol(*Grp);
 
   this->MCObjectStreamer::ChangeSection(Section, Subsection);
-  MCSymbol *SectionSymbol = getContext().getOrCreateSectionSymbol(*SectionELF);
-  if (SectionSymbol->isUndefined()) {
-    EmitLabel(SectionSymbol);
-    MCELF::SetType(Asm.getSymbolData(*SectionSymbol), ELF::STT_SECTION);
+  MCContext &Ctx = getContext();
+  auto *Begin = cast_or_null<MCSymbolELF>(Section->getBeginSymbol());
+  if (!Begin) {
+    Begin = Ctx.getOrCreateSectionSymbol(*SectionELF);
+    Section->setBeginSymbol(Begin);
+  }
+  if (Begin->isUndefined()) {
+    Asm.registerSymbol(*Begin);
+    Begin->setType(ELF::STT_SECTION);
   }
 }
 
 void MCELFStreamer::EmitWeakReference(MCSymbol *Alias, const MCSymbol *Symbol) {
-  getAssembler().getOrCreateSymbolData(*Symbol);
-  const MCExpr *Value = MCSymbolRefExpr::Create(
+  getAssembler().registerSymbol(*Symbol);
+  const MCExpr *Value = MCSymbolRefExpr::create(
       Symbol, MCSymbolRefExpr::VK_WEAKREF, getContext());
   Alias->setVariableValue(Value);
 }
@@ -189,8 +196,8 @@ static unsigned CombineSymbolTypes(unsigned T1, unsigned T2) {
   return T2;
 }
 
-bool MCELFStreamer::EmitSymbolAttribute(MCSymbol *Symbol,
-                                        MCSymbolAttr Attribute) {
+bool MCELFStreamer::EmitSymbolAttribute(MCSymbol *S, MCSymbolAttr Attribute) {
+  auto *Symbol = cast<MCSymbolELF>(S);
   // Indirect symbols are handled differently, to match how 'as' handles
   // them. This makes writing matching .o files easier.
   if (Attribute == MCSA_IndirectSymbol) {
@@ -198,15 +205,15 @@ bool MCELFStreamer::EmitSymbolAttribute(MCSymbol *Symbol,
     // important for matching the string table that 'as' generates.
     IndirectSymbolData ISD;
     ISD.Symbol = Symbol;
-    ISD.SectionData = getCurrentSectionData();
+    ISD.Section = getCurrentSectionOnly();
     getAssembler().getIndirectSymbols().push_back(ISD);
     return true;
   }
 
   // Adding a symbol attribute always introduces the symbol, note that an
-  // important side effect of calling getOrCreateSymbolData here is to register
+  // important side effect of calling registerSymbol here is to register
   // the symbol with the assembler.
-  MCSymbolData &SD = getAssembler().getOrCreateSymbolData(*Symbol);
+  getAssembler().registerSymbol(*Symbol);
 
   // The implementation of symbol attributes is designed to match 'as', but it
   // leaves much to desired. It doesn't really make sense to arbitrarily add and
@@ -230,122 +237,115 @@ bool MCELFStreamer::EmitSymbolAttribute(MCSymbol *Symbol,
     break;
 
   case MCSA_ELF_TypeGnuUniqueObject:
-    MCELF::SetType(SD, CombineSymbolTypes(MCELF::GetType(SD), ELF::STT_OBJECT));
-    MCELF::SetBinding(SD, ELF::STB_GNU_UNIQUE);
-    SD.setExternal(true);
-    BindingExplicitlySet.insert(Symbol);
+    Symbol->setType(CombineSymbolTypes(Symbol->getType(), ELF::STT_OBJECT));
+    Symbol->setBinding(ELF::STB_GNU_UNIQUE);
+    Symbol->setExternal(true);
     break;
 
   case MCSA_Global:
-    MCELF::SetBinding(SD, ELF::STB_GLOBAL);
-    SD.setExternal(true);
-    BindingExplicitlySet.insert(Symbol);
+    Symbol->setBinding(ELF::STB_GLOBAL);
+    Symbol->setExternal(true);
     break;
 
   case MCSA_WeakReference:
   case MCSA_Weak:
-    MCELF::SetBinding(SD, ELF::STB_WEAK);
-    SD.setExternal(true);
-    BindingExplicitlySet.insert(Symbol);
+    Symbol->setBinding(ELF::STB_WEAK);
+    Symbol->setExternal(true);
     break;
 
   case MCSA_Local:
-    MCELF::SetBinding(SD, ELF::STB_LOCAL);
-    SD.setExternal(false);
-    BindingExplicitlySet.insert(Symbol);
+    Symbol->setBinding(ELF::STB_LOCAL);
+    Symbol->setExternal(false);
     break;
 
   case MCSA_ELF_TypeFunction:
-    MCELF::SetType(SD, CombineSymbolTypes(MCELF::GetType(SD),
-                                          ELF::STT_FUNC));
+    Symbol->setType(CombineSymbolTypes(Symbol->getType(), ELF::STT_FUNC));
     break;
 
   case MCSA_ELF_TypeIndFunction:
-    MCELF::SetType(SD, CombineSymbolTypes(MCELF::GetType(SD),
-                                          ELF::STT_GNU_IFUNC));
+    Symbol->setType(CombineSymbolTypes(Symbol->getType(), ELF::STT_GNU_IFUNC));
     break;
 
   case MCSA_ELF_TypeObject:
-    MCELF::SetType(SD, CombineSymbolTypes(MCELF::GetType(SD),
-                                          ELF::STT_OBJECT));
+    Symbol->setType(CombineSymbolTypes(Symbol->getType(), ELF::STT_OBJECT));
     break;
 
   case MCSA_ELF_TypeTLS:
-    MCELF::SetType(SD, CombineSymbolTypes(MCELF::GetType(SD),
-                                          ELF::STT_TLS));
+    Symbol->setType(CombineSymbolTypes(Symbol->getType(), ELF::STT_TLS));
     break;
 
   case MCSA_ELF_TypeCommon:
     // TODO: Emit these as a common symbol.
-    MCELF::SetType(SD, CombineSymbolTypes(MCELF::GetType(SD),
-                                          ELF::STT_OBJECT));
+    Symbol->setType(CombineSymbolTypes(Symbol->getType(), ELF::STT_OBJECT));
     break;
 
   case MCSA_ELF_TypeNoType:
-    MCELF::SetType(SD, CombineSymbolTypes(MCELF::GetType(SD),
-                                          ELF::STT_NOTYPE));
+    Symbol->setType(CombineSymbolTypes(Symbol->getType(), ELF::STT_NOTYPE));
     break;
 
   case MCSA_Protected:
-    MCELF::SetVisibility(SD, ELF::STV_PROTECTED);
+    Symbol->setVisibility(ELF::STV_PROTECTED);
     break;
 
   case MCSA_Hidden:
-    MCELF::SetVisibility(SD, ELF::STV_HIDDEN);
+    Symbol->setVisibility(ELF::STV_HIDDEN);
     break;
 
   case MCSA_Internal:
-    MCELF::SetVisibility(SD, ELF::STV_INTERNAL);
+    Symbol->setVisibility(ELF::STV_INTERNAL);
     break;
   }
 
   return true;
 }
 
-void MCELFStreamer::EmitCommonSymbol(MCSymbol *Symbol, uint64_t Size,
-                                       unsigned ByteAlignment) {
-  MCSymbolData &SD = getAssembler().getOrCreateSymbolData(*Symbol);
+void MCELFStreamer::EmitCommonSymbol(MCSymbol *S, uint64_t Size,
+                                     unsigned ByteAlignment) {
+  auto *Symbol = cast<MCSymbolELF>(S);
+  getAssembler().registerSymbol(*Symbol);
 
-  if (!BindingExplicitlySet.count(Symbol)) {
-    MCELF::SetBinding(SD, ELF::STB_GLOBAL);
-    SD.setExternal(true);
+  if (!Symbol->isBindingSet()) {
+    Symbol->setBinding(ELF::STB_GLOBAL);
+    Symbol->setExternal(true);
   }
 
-  MCELF::SetType(SD, ELF::STT_OBJECT);
+  Symbol->setType(ELF::STT_OBJECT);
 
-  if (MCELF::GetBinding(SD) == ELF_STB_Local) {
-    const MCSection *Section = getAssembler().getContext().getELFSection(
+  if (Symbol->getBinding() == ELF::STB_LOCAL) {
+    MCSection *Section = getAssembler().getContext().getELFSection(
         ".bss", ELF::SHT_NOBITS, ELF::SHF_WRITE | ELF::SHF_ALLOC);
 
     AssignSection(Symbol, Section);
 
-    struct LocalCommon L = {&SD, Size, ByteAlignment};
+    struct LocalCommon L = {Symbol, Size, ByteAlignment};
     LocalCommons.push_back(L);
   } else {
-    SD.setCommon(Size, ByteAlignment);
+    if(Symbol->declareCommon(Size, ByteAlignment))
+      report_fatal_error("Symbol: " + Symbol->getName() +
+                         " redeclared as different type");
   }
 
-  SD.setSize(MCConstantExpr::Create(Size, getContext()));
+  cast<MCSymbolELF>(Symbol)
+      ->setSize(MCConstantExpr::create(Size, getContext()));
 }
 
-void MCELFStreamer::EmitELFSize(MCSymbol *Symbol, const MCExpr *Value) {
-  MCSymbolData &SD = getAssembler().getOrCreateSymbolData(*Symbol);
-  SD.setSize(Value);
+void MCELFStreamer::emitELFSize(MCSymbolELF *Symbol, const MCExpr *Value) {
+  Symbol->setSize(Value);
 }
 
-void MCELFStreamer::EmitLocalCommonSymbol(MCSymbol *Symbol, uint64_t Size,
+void MCELFStreamer::EmitLocalCommonSymbol(MCSymbol *S, uint64_t Size,
                                           unsigned ByteAlignment) {
+  auto *Symbol = cast<MCSymbolELF>(S);
   // FIXME: Should this be caught and done earlier?
-  MCSymbolData &SD = getAssembler().getOrCreateSymbolData(*Symbol);
-  MCELF::SetBinding(SD, ELF::STB_LOCAL);
-  SD.setExternal(false);
-  BindingExplicitlySet.insert(Symbol);
+  getAssembler().registerSymbol(*Symbol);
+  Symbol->setBinding(ELF::STB_LOCAL);
+  Symbol->setExternal(false);
   EmitCommonSymbol(Symbol, Size, ByteAlignment);
 }
 
 void MCELFStreamer::EmitValueImpl(const MCExpr *Value, unsigned Size,
                                   const SMLoc &Loc) {
-  if (getCurrentSectionData()->isBundleLocked())
+  if (isBundleLocked())
     report_fatal_error("Emitting values inside a locked bundle is forbidden");
   fixSymbolsInTLSFixups(Value);
   MCObjectStreamer::EmitValueImpl(Value, Size, Loc);
@@ -355,7 +355,7 @@ void MCELFStreamer::EmitValueToAlignment(unsigned ByteAlignment,
                                          int64_t Value,
                                          unsigned ValueSize,
                                          unsigned MaxBytesToEmit) {
-  if (getCurrentSectionData()->isBundleLocked())
+  if (isBundleLocked())
     report_fatal_error("Emitting values inside a locked bundle is forbidden");
   MCObjectStreamer::EmitValueToAlignment(ByteAlignment, Value,
                                          ValueSize, MaxBytesToEmit);
@@ -369,7 +369,7 @@ void MCELFStreamer::EmitFileDirective(StringRef Filename) {
 }
 
 void MCELFStreamer::EmitIdent(StringRef IdentString) {
-  const MCSection *Comment = getAssembler().getContext().getELFSection(
+  MCSection *Comment = getAssembler().getContext().getELFSection(
       ".comment", ELF::SHT_PROGBITS, ELF::SHF_MERGE | ELF::SHF_STRINGS, 1, "");
   PushSection();
   SwitchSection(Comment);
@@ -453,8 +453,8 @@ void MCELFStreamer::fixSymbolsInTLSFixups(const MCExpr *expr) {
     case MCSymbolRefExpr::VK_PPC_TLSLD:
       break;
     }
-    MCSymbolData &SD = getAssembler().getOrCreateSymbolData(symRef.getSymbol());
-    MCELF::SetType(SD, ELF::STT_TLS);
+    getAssembler().registerSymbol(symRef.getSymbol());
+    cast<MCSymbolELF>(symRef.getSymbol()).setType(ELF::STT_TLS);
     break;
   }
 
@@ -479,7 +479,7 @@ void MCELFStreamer::EmitInstToData(const MCInst &Inst,
   SmallVector<MCFixup, 4> Fixups;
   SmallString<256> Code;
   raw_svector_ostream VecOS(Code);
-  Assembler.getEmitter().EncodeInstruction(Inst, VecOS, Fixups, STI);
+  Assembler.getEmitter().encodeInstruction(Inst, VecOS, Fixups, STI);
   VecOS.flush();
 
   for (unsigned i = 0, e = Fixups.size(); i != e; ++i)
@@ -503,21 +503,21 @@ void MCELFStreamer::EmitInstToData(const MCInst &Inst,
   MCDataFragment *DF;
 
   if (Assembler.isBundlingEnabled()) {
-    MCSectionData *SD = getCurrentSectionData();
-    if (Assembler.getRelaxAll() && SD->isBundleLocked())
+    MCSection &Sec = *getCurrentSectionOnly();
+    if (Assembler.getRelaxAll() && isBundleLocked())
       // If the -mc-relax-all flag is used and we are bundle-locked, we re-use
       // the current bundle group.
       DF = BundleGroups.back();
-    else if (Assembler.getRelaxAll() && !SD->isBundleLocked())
+    else if (Assembler.getRelaxAll() && !isBundleLocked())
       // When not in a bundle-locked group and the -mc-relax-all flag is used,
       // we create a new temporary fragment which will be later merged into
       // the current fragment.
       DF = new MCDataFragment();
-    else if (SD->isBundleLocked() && !SD->isBundleGroupBeforeFirstInst())
+    else if (isBundleLocked() && !Sec.isBundleGroupBeforeFirstInst())
       // If we are bundle-locked, we re-use the current fragment.
       // The bundle-locking directive ensures this is a new data fragment.
       DF = cast<MCDataFragment>(getCurrentFragment());
-    else if (!SD->isBundleLocked() && Fixups.size() == 0) {
+    else if (!isBundleLocked() && Fixups.size() == 0) {
       // Optimize memory usage by emitting the instruction to a
       // MCCompactEncodedInstFragment when not in a bundle-locked group and
       // there are no fixups registered.
@@ -529,7 +529,7 @@ void MCELFStreamer::EmitInstToData(const MCInst &Inst,
       DF = new MCDataFragment();
       insert(DF);
     }
-    if (SD->getBundleLockState() == MCSectionData::BundleLockedAlignToEnd) {
+    if (Sec.getBundleLockState() == MCSection::BundleLockedAlignToEnd) {
       // If this fragment is for a group marked "align_to_end", set a flag
       // in the fragment. This can happen after the fragment has already been
       // created if there are nested bundle_align groups and an inner one
@@ -539,7 +539,7 @@ void MCELFStreamer::EmitInstToData(const MCInst &Inst,
 
     // We're now emitting an instruction in a bundle group, so this flag has
     // to be turned off.
-    SD->setBundleGroupBeforeFirstInst(false);
+    Sec.setBundleGroupBeforeFirstInst(false);
   } else {
     DF = getOrCreateDataFragment();
   }
@@ -553,8 +553,7 @@ void MCELFStreamer::EmitInstToData(const MCInst &Inst,
   DF->getContents().append(Code.begin(), Code.end());
 
   if (Assembler.isBundlingEnabled() && Assembler.getRelaxAll()) {
-    MCSectionData *SD = getCurrentSectionData();
-    if (!SD->isBundleLocked()) {
+    if (!isBundleLocked()) {
       mergeFragment(getOrCreateDataFragment(), DF);
       delete DF;
     }
@@ -572,79 +571,78 @@ void MCELFStreamer::EmitBundleAlignMode(unsigned AlignPow2) {
 }
 
 void MCELFStreamer::EmitBundleLock(bool AlignToEnd) {
-  MCSectionData *SD = getCurrentSectionData();
+  MCSection &Sec = *getCurrentSectionOnly();
 
   // Sanity checks
   //
   if (!getAssembler().isBundlingEnabled())
     report_fatal_error(".bundle_lock forbidden when bundling is disabled");
 
-  if (!SD->isBundleLocked())
-    SD->setBundleGroupBeforeFirstInst(true);
+  if (!isBundleLocked())
+    Sec.setBundleGroupBeforeFirstInst(true);
 
-  if (getAssembler().getRelaxAll() && !SD->isBundleLocked()) {
+  if (getAssembler().getRelaxAll() && !isBundleLocked()) {
     // TODO: drop the lock state and set directly in the fragment
     MCDataFragment *DF = new MCDataFragment();
     BundleGroups.push_back(DF);
   }
 
-  SD->setBundleLockState(AlignToEnd ? MCSectionData::BundleLockedAlignToEnd :
-                                      MCSectionData::BundleLocked);
+  Sec.setBundleLockState(AlignToEnd ? MCSection::BundleLockedAlignToEnd
+                                    : MCSection::BundleLocked);
 }
 
 void MCELFStreamer::EmitBundleUnlock() {
-  MCSectionData *SD = getCurrentSectionData();
+  MCSection &Sec = *getCurrentSectionOnly();
 
   // Sanity checks
   if (!getAssembler().isBundlingEnabled())
     report_fatal_error(".bundle_unlock forbidden when bundling is disabled");
-  else if (!SD->isBundleLocked())
+  else if (!isBundleLocked())
     report_fatal_error(".bundle_unlock without matching lock");
-  else if (SD->isBundleGroupBeforeFirstInst())
+  else if (Sec.isBundleGroupBeforeFirstInst())
     report_fatal_error("Empty bundle-locked group is forbidden");
 
   // When the -mc-relax-all flag is used, we emit instructions to fragments
-  // stored on a stack. When the bundle unlock is emited, we pop a fragment 
+  // stored on a stack. When the bundle unlock is emited, we pop a fragment
   // from the stack a merge it to the one below.
   if (getAssembler().getRelaxAll()) {
     assert(!BundleGroups.empty() && "There are no bundle groups");
     MCDataFragment *DF = BundleGroups.back();
 
     // FIXME: Use BundleGroups to track the lock state instead.
-    SD->setBundleLockState(MCSectionData::NotBundleLocked);
+    Sec.setBundleLockState(MCSection::NotBundleLocked);
 
-    // FIXME: Use more separate fragments for nested groups. 
-    if (!SD->isBundleLocked()) {
+    // FIXME: Use more separate fragments for nested groups.
+    if (!isBundleLocked()) {
       mergeFragment(getOrCreateDataFragment(), DF);
       BundleGroups.pop_back();
       delete DF;
     }
 
-    if (SD->getBundleLockState() != MCSectionData::BundleLockedAlignToEnd)
+    if (Sec.getBundleLockState() != MCSection::BundleLockedAlignToEnd)
       getOrCreateDataFragment()->setAlignToBundleEnd(false);
   } else
-    SD->setBundleLockState(MCSectionData::NotBundleLocked);
+    Sec.setBundleLockState(MCSection::NotBundleLocked);
 }
 
 void MCELFStreamer::Flush() {
   for (std::vector<LocalCommon>::const_iterator i = LocalCommons.begin(),
                                                 e = LocalCommons.end();
        i != e; ++i) {
-    MCSymbolData *SD = i->SD;
+    const MCSymbol &Symbol = *i->Symbol;
     uint64_t Size = i->Size;
     unsigned ByteAlignment = i->ByteAlignment;
-    const MCSymbol &Symbol = SD->getSymbol();
-    const MCSection &Section = Symbol.getSection();
+    MCSection &Section = Symbol.getSection();
 
-    MCSectionData &SectData = getAssembler().getOrCreateSectionData(Section);
-    new MCAlignFragment(ByteAlignment, 0, 1, ByteAlignment, &SectData);
+    getAssembler().registerSection(Section);
+    new MCAlignFragment(ByteAlignment, 0, 1, ByteAlignment, &Section);
 
-    MCFragment *F = new MCFillFragment(0, 0, Size, &SectData);
-    SD->setFragment(F);
+    MCFragment *F = new MCFillFragment(0, 0, Size, &Section);
+    Symbol.setFragment(F);
 
     // Update the maximum alignment of the section if necessary.
-    if (ByteAlignment > SectData.getAlignment())
-      SectData.setAlignment(ByteAlignment);
+    if (ByteAlignment > Section.getAlignment())
+      Section.setAlignment(ByteAlignment);
   }
 
   LocalCommons.clear();
@@ -652,7 +650,8 @@ void MCELFStreamer::Flush() {
 
 void MCELFStreamer::FinishImpl() {
   // Ensure the last section gets aligned if necessary.
-  setSectionAlignmentForBundling(getAssembler(), getCurrentSectionData());
+  MCSection *CurSection = getCurrentSectionOnly();
+  setSectionAlignmentForBundling(getAssembler(), CurSection);
 
   EmitFrames(nullptr);
 
@@ -694,12 +693,12 @@ void MCELFStreamer::EndCOFFSymbolDef() {
   llvm_unreachable("ELF doesn't support this directive");
 }
 
-void MCELFStreamer::EmitZerofill(const MCSection *Section, MCSymbol *Symbol,
+void MCELFStreamer::EmitZerofill(MCSection *Section, MCSymbol *Symbol,
                                  uint64_t Size, unsigned ByteAlignment) {
   llvm_unreachable("ELF doesn't support this directive");
 }
 
-void MCELFStreamer::EmitTBSSSymbol(const MCSection *Section, MCSymbol *Symbol,
+void MCELFStreamer::EmitTBSSSymbol(MCSection *Section, MCSymbol *Symbol,
                                    uint64_t Size, unsigned ByteAlignment) {
   llvm_unreachable("ELF doesn't support this directive");
 }
diff --git a/lib/MC/MCExpr.cpp b/lib/MC/MCExpr.cpp
index 0702539218c..a30ceecc952 100644
--- a/lib/MC/MCExpr.cpp
+++ b/lib/MC/MCExpr.cpp
@@ -30,10 +30,10 @@ STATISTIC(MCExprEvaluate, "Number of MCExpr evaluations");
 }
 }
 
-void MCExpr::print(raw_ostream &OS) const {
+void MCExpr::print(raw_ostream &OS, const MCAsmInfo *MAI) const {
   switch (getKind()) {
   case MCExpr::Target:
-    return cast<MCTargetExpr>(this)->PrintImpl(OS);
+    return cast<MCTargetExpr>(this)->printImpl(OS, MAI);
   case MCExpr::Constant:
     OS << cast<MCConstantExpr>(*this).getValue();
     return;
@@ -44,10 +44,12 @@ void MCExpr::print(raw_ostream &OS) const {
     // Parenthesize names that start with $ so that they don't look like
     // absolute names.
     bool UseParens = Sym.getName()[0] == '$';
-    if (UseParens)
-      OS << '(' << Sym << ')';
-    else
-      OS << Sym;
+    if (UseParens) {
+      OS << '(';
+      Sym.print(OS, MAI);
+      OS << ')';
+    } else
+      Sym.print(OS, MAI);
 
     if (SRE.getKind() != MCSymbolRefExpr::VK_None)
       SRE.printVariantKind(OS);
@@ -63,7 +65,7 @@ void MCExpr::print(raw_ostream &OS) const {
     case MCUnaryExpr::Not:   OS << '~'; break;
     case MCUnaryExpr::Plus:  OS << '+'; break;
     }
-    OS << *UE.getSubExpr();
+    UE.getSubExpr()->print(OS, MAI);
     return;
   }
 
@@ -72,9 +74,11 @@ void MCExpr::print(raw_ostream &OS) const {
 
     // Only print parens around the LHS if it is non-trivial.
     if (isa<MCConstantExpr>(BE.getLHS()) || isa<MCSymbolRefExpr>(BE.getLHS())) {
-      OS << *BE.getLHS();
+      BE.getLHS()->print(OS, MAI);
     } else {
-      OS << '(' << *BE.getLHS() << ')';
+      OS << '(';
+      BE.getLHS()->print(OS, MAI);
+      OS << ')';
     }
 
     switch (BE.getOpcode()) {
@@ -89,6 +93,7 @@ void MCExpr::print(raw_ostream &OS) const {
 
       OS <<  '+';
       break;
+    case MCBinaryExpr::AShr: OS << ">>"; break;
     case MCBinaryExpr::And:  OS <<  '&'; break;
     case MCBinaryExpr::Div:  OS <<  '/'; break;
     case MCBinaryExpr::EQ:   OS << "=="; break;
@@ -96,6 +101,7 @@ void MCExpr::print(raw_ostream &OS) const {
     case MCBinaryExpr::GTE:  OS << ">="; break;
     case MCBinaryExpr::LAnd: OS << "&&"; break;
     case MCBinaryExpr::LOr:  OS << "||"; break;
+    case MCBinaryExpr::LShr: OS << ">>"; break;
     case MCBinaryExpr::LT:   OS <<  '<'; break;
     case MCBinaryExpr::LTE:  OS << "<="; break;
     case MCBinaryExpr::Mod:  OS <<  '%'; break;
@@ -103,16 +109,17 @@ void MCExpr::print(raw_ostream &OS) const {
     case MCBinaryExpr::NE:   OS << "!="; break;
     case MCBinaryExpr::Or:   OS <<  '|'; break;
     case MCBinaryExpr::Shl:  OS << "<<"; break;
-    case MCBinaryExpr::Shr:  OS << ">>"; break;
     case MCBinaryExpr::Sub:  OS <<  '-'; break;
     case MCBinaryExpr::Xor:  OS <<  '^'; break;
     }
 
     // Only print parens around the LHS if it is non-trivial.
     if (isa<MCConstantExpr>(BE.getRHS()) || isa<MCSymbolRefExpr>(BE.getRHS())) {
-      OS << *BE.getRHS();
+      BE.getRHS()->print(OS, MAI);
     } else {
-      OS << '(' << *BE.getRHS() << ')';
+      OS << '(';
+      BE.getRHS()->print(OS, MAI);
+      OS << ')';
     }
     return;
   }
@@ -123,24 +130,24 @@ void MCExpr::print(raw_ostream &OS) const {
 
 #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
 void MCExpr::dump() const {
-  print(dbgs());
+  dbgs() << *this;
   dbgs() << '\n';
 }
 #endif
 
 /* *** */
 
-const MCBinaryExpr *MCBinaryExpr::Create(Opcode Opc, const MCExpr *LHS,
+const MCBinaryExpr *MCBinaryExpr::create(Opcode Opc, const MCExpr *LHS,
                                          const MCExpr *RHS, MCContext &Ctx) {
   return new (Ctx) MCBinaryExpr(Opc, LHS, RHS);
 }
 
-const MCUnaryExpr *MCUnaryExpr::Create(Opcode Opc, const MCExpr *Expr,
+const MCUnaryExpr *MCUnaryExpr::create(Opcode Opc, const MCExpr *Expr,
                                        MCContext &Ctx) {
   return new (Ctx) MCUnaryExpr(Opc, Expr);
 }
 
-const MCConstantExpr *MCConstantExpr::Create(int64_t Value, MCContext &Ctx) {
+const MCConstantExpr *MCConstantExpr::create(int64_t Value, MCContext &Ctx) {
   return new (Ctx) MCConstantExpr(Value);
 }
 
@@ -155,15 +162,15 @@ MCSymbolRefExpr::MCSymbolRefExpr(const MCSymbol *Symbol, VariantKind Kind,
   assert(Symbol);
 }
 
-const MCSymbolRefExpr *MCSymbolRefExpr::Create(const MCSymbol *Sym,
+const MCSymbolRefExpr *MCSymbolRefExpr::create(const MCSymbol *Sym,
                                                VariantKind Kind,
                                                MCContext &Ctx) {
   return new (Ctx) MCSymbolRefExpr(Sym, Kind, Ctx.getAsmInfo());
 }
 
-const MCSymbolRefExpr *MCSymbolRefExpr::Create(StringRef Name, VariantKind Kind,
+const MCSymbolRefExpr *MCSymbolRefExpr::create(StringRef Name, VariantKind Kind,
                                                MCContext &Ctx) {
-  return Create(Ctx.GetOrCreateSymbol(Name), Kind, Ctx);
+  return create(Ctx.getOrCreateSymbol(Name), Kind, Ctx);
 }
 
 StringRef MCSymbolRefExpr::getVariantKindName(VariantKind Kind) {
@@ -282,6 +289,18 @@ StringRef MCSymbolRefExpr::getVariantKindName(VariantKind Kind) {
   case VK_Mips_PCREL_HI16: return "PCREL_HI16";
   case VK_Mips_PCREL_LO16: return "PCREL_LO16";
   case VK_COFF_IMGREL32: return "IMGREL";
+  case VK_Hexagon_PCREL: return "PCREL";
+  case VK_Hexagon_LO16: return "LO16";
+  case VK_Hexagon_HI16: return "HI16";
+  case VK_Hexagon_GPREL: return "GPREL";
+  case VK_Hexagon_GD_GOT: return "GDGOT";
+  case VK_Hexagon_LD_GOT: return "LDGOT";
+  case VK_Hexagon_GD_PLT: return "GDPLT";
+  case VK_Hexagon_LD_PLT: return "LDPLT";
+  case VK_Hexagon_IE: return "IE";
+  case VK_Hexagon_IE_GOT: return "IEGOT";
+  case VK_TPREL: return "tprel";
+  case VK_DTPREL: return "dtprel";
   }
   llvm_unreachable("Invalid variant kind");
 }
@@ -387,23 +406,23 @@ void MCTargetExpr::anchor() {}
 
 /* *** */
 
-bool MCExpr::EvaluateAsAbsolute(int64_t &Res) const {
-  return EvaluateAsAbsolute(Res, nullptr, nullptr, nullptr);
+bool MCExpr::evaluateAsAbsolute(int64_t &Res) const {
+  return evaluateAsAbsolute(Res, nullptr, nullptr, nullptr);
 }
 
-bool MCExpr::EvaluateAsAbsolute(int64_t &Res,
+bool MCExpr::evaluateAsAbsolute(int64_t &Res,
                                 const MCAsmLayout &Layout) const {
-  return EvaluateAsAbsolute(Res, &Layout.getAssembler(), &Layout, nullptr);
+  return evaluateAsAbsolute(Res, &Layout.getAssembler(), &Layout, nullptr);
 }
 
-bool MCExpr::EvaluateAsAbsolute(int64_t &Res,
+bool MCExpr::evaluateAsAbsolute(int64_t &Res,
                                 const MCAsmLayout &Layout,
                                 const SectionAddrMap &Addrs) const {
-  return EvaluateAsAbsolute(Res, &Layout.getAssembler(), &Layout, &Addrs);
+  return evaluateAsAbsolute(Res, &Layout.getAssembler(), &Layout, &Addrs);
 }
 
-bool MCExpr::EvaluateAsAbsolute(int64_t &Res, const MCAssembler &Asm) const {
-  return EvaluateAsAbsolute(Res, &Asm, nullptr, nullptr);
+bool MCExpr::evaluateAsAbsolute(int64_t &Res, const MCAssembler &Asm) const {
+  return evaluateAsAbsolute(Res, &Asm, nullptr, nullptr);
 }
 
 bool MCExpr::evaluateKnownAbsolute(int64_t &Res,
@@ -412,7 +431,7 @@ bool MCExpr::evaluateKnownAbsolute(int64_t &Res,
                             true);
 }
 
-bool MCExpr::EvaluateAsAbsolute(int64_t &Res, const MCAssembler *Asm,
+bool MCExpr::evaluateAsAbsolute(int64_t &Res, const MCAssembler *Asm,
                                 const MCAsmLayout *Layout,
                                 const SectionAddrMap *Addrs) const {
   // FIXME: The use if InSet = Addrs is a hack. Setting InSet causes us
@@ -433,7 +452,7 @@ bool MCExpr::evaluateAsAbsolute(int64_t &Res, const MCAssembler *Asm,
   }
 
   bool IsRelocatable =
-      EvaluateAsRelocatableImpl(Value, Asm, Layout, nullptr, Addrs, InSet);
+      evaluateAsRelocatableImpl(Value, Asm, Layout, nullptr, Addrs, InSet);
 
   // Record the current value.
   Res = Value.getConstant();
@@ -455,14 +474,11 @@ static void AttemptToFoldSymbolOffsetDifference(
   if (SA.isUndefined() || SB.isUndefined())
     return;
 
-  if (!Asm->getWriter().IsSymbolRefDifferenceFullyResolved(*Asm, A, B, InSet))
+  if (!Asm->getWriter().isSymbolRefDifferenceFullyResolved(*Asm, A, B, InSet))
     return;
 
-  const MCSymbolData &AD = Asm->getSymbolData(SA);
-  const MCSymbolData &BD = Asm->getSymbolData(SB);
-
-  if (AD.getFragment() == BD.getFragment()) {
-    Addend += (AD.getOffset() - BD.getOffset());
+  if (SA.getFragment() == SB.getFragment()) {
+    Addend += (SA.getOffset() - SB.getOffset());
 
     // Pointers to Thumb symbols need to have their low-bit set to allow
     // for interworking.
@@ -478,15 +494,15 @@ static void AttemptToFoldSymbolOffsetDifference(
   if (!Layout)
     return;
 
-  const MCSectionData &SecA = *AD.getFragment()->getParent();
-  const MCSectionData &SecB = *BD.getFragment()->getParent();
+  const MCSection &SecA = *SA.getFragment()->getParent();
+  const MCSection &SecB = *SB.getFragment()->getParent();
 
   if ((&SecA != &SecB) && !Addrs)
     return;
 
   // Eagerly evaluate.
-  Addend += (Layout->getSymbolOffset(&Asm->getSymbolData(A->getSymbol())) -
-             Layout->getSymbolOffset(&Asm->getSymbolData(B->getSymbol())));
+  Addend += Layout->getSymbolOffset(A->getSymbol()) -
+            Layout->getSymbolOffset(B->getSymbol());
   if (Addrs && (&SecA != &SecB))
     Addend += (Addrs->lookup(&SecA) - Addrs->lookup(&SecB));
 
@@ -576,30 +592,36 @@ EvaluateSymbolicAdd(const MCAssembler *Asm, const MCAsmLayout *Layout,
   return true;
 }
 
-bool MCExpr::EvaluateAsRelocatable(MCValue &Res,
+bool MCExpr::evaluateAsRelocatable(MCValue &Res,
                                    const MCAsmLayout *Layout,
                                    const MCFixup *Fixup) const {
   MCAssembler *Assembler = Layout ? &Layout->getAssembler() : nullptr;
-  return EvaluateAsRelocatableImpl(Res, Assembler, Layout, Fixup, nullptr,
+  return evaluateAsRelocatableImpl(Res, Assembler, Layout, Fixup, nullptr,
                                    false);
 }
 
 bool MCExpr::evaluateAsValue(MCValue &Res, const MCAsmLayout &Layout) const {
   MCAssembler *Assembler = &Layout.getAssembler();
-  return EvaluateAsRelocatableImpl(Res, Assembler, &Layout, nullptr, nullptr,
+  return evaluateAsRelocatableImpl(Res, Assembler, &Layout, nullptr, nullptr,
                                    true);
 }
 
 static bool canExpand(const MCSymbol &Sym, const MCAssembler *Asm, bool InSet) {
+  const MCExpr *Expr = Sym.getVariableValue();
+  const auto *Inner = dyn_cast<MCSymbolRefExpr>(Expr);
+  if (Inner) {
+    if (Inner->getKind() == MCSymbolRefExpr::VK_WEAKREF)
+      return false;
+  }
+
   if (InSet)
     return true;
   if (!Asm)
     return false;
-  const MCSymbolData &SD = Asm->getSymbolData(Sym);
-  return !Asm->getWriter().isWeak(SD);
+  return !Asm->getWriter().isWeak(Sym);
 }
 
-bool MCExpr::EvaluateAsRelocatableImpl(MCValue &Res, const MCAssembler *Asm,
+bool MCExpr::evaluateAsRelocatableImpl(MCValue &Res, const MCAssembler *Asm,
                                        const MCAsmLayout *Layout,
                                        const MCFixup *Fixup,
                                        const SectionAddrMap *Addrs,
@@ -608,7 +630,7 @@ bool MCExpr::EvaluateAsRelocatableImpl(MCValue &Res, const MCAssembler *Asm,
 
   switch (getKind()) {
   case Target:
-    return cast<MCTargetExpr>(this)->EvaluateAsRelocatableImpl(Res, Layout,
+    return cast<MCTargetExpr>(this)->evaluateAsRelocatableImpl(Res, Layout,
                                                                Fixup);
 
   case Constant:
@@ -623,7 +645,7 @@ bool MCExpr::EvaluateAsRelocatableImpl(MCValue &Res, const MCAssembler *Asm,
     if (Sym.isVariable() && SRE->getKind() == MCSymbolRefExpr::VK_None &&
         canExpand(Sym, Asm, InSet)) {
       bool IsMachO = SRE->hasSubsectionsViaSymbols();
-      if (Sym.getVariableValue()->EvaluateAsRelocatableImpl(
+      if (Sym.getVariableValue()->evaluateAsRelocatableImpl(
               Res, Asm, Layout, Fixup, Addrs, InSet || IsMachO)) {
         if (!IsMachO)
           return true;
@@ -649,7 +671,7 @@ bool MCExpr::EvaluateAsRelocatableImpl(MCValue &Res, const MCAssembler *Asm,
     const MCUnaryExpr *AUE = cast<MCUnaryExpr>(this);
     MCValue Value;
 
-    if (!AUE->getSubExpr()->EvaluateAsRelocatableImpl(Value, Asm, Layout, Fixup,
+    if (!AUE->getSubExpr()->evaluateAsRelocatableImpl(Value, Asm, Layout, Fixup,
                                                       Addrs, InSet))
       return false;
 
@@ -683,9 +705,9 @@ bool MCExpr::EvaluateAsRelocatableImpl(MCValue &Res, const MCAssembler *Asm,
     const MCBinaryExpr *ABE = cast<MCBinaryExpr>(this);
     MCValue LHSValue, RHSValue;
 
-    if (!ABE->getLHS()->EvaluateAsRelocatableImpl(LHSValue, Asm, Layout, Fixup,
+    if (!ABE->getLHS()->evaluateAsRelocatableImpl(LHSValue, Asm, Layout, Fixup,
                                                   Addrs, InSet) ||
-        !ABE->getRHS()->EvaluateAsRelocatableImpl(RHSValue, Asm, Layout, Fixup,
+        !ABE->getRHS()->evaluateAsRelocatableImpl(RHSValue, Asm, Layout, Fixup,
                                                   Addrs, InSet))
       return false;
 
@@ -709,11 +731,12 @@ bool MCExpr::EvaluateAsRelocatableImpl(MCValue &Res, const MCAssembler *Asm,
     }
 
     // FIXME: We need target hooks for the evaluation. It may be limited in
-    // width, and gas defines the result of comparisons and right shifts
-    // differently from Apple as.
+    // width, and gas defines the result of comparisons differently from
+    // Apple as.
     int64_t LHS = LHSValue.getConstant(), RHS = RHSValue.getConstant();
     int64_t Result = 0;
     switch (ABE->getOpcode()) {
+    case MCBinaryExpr::AShr: Result = LHS >> RHS; break;
     case MCBinaryExpr::Add:  Result = LHS + RHS; break;
     case MCBinaryExpr::And:  Result = LHS & RHS; break;
     case MCBinaryExpr::Div:  Result = LHS / RHS; break;
@@ -722,14 +745,14 @@ bool MCExpr::EvaluateAsRelocatableImpl(MCValue &Res, const MCAssembler *Asm,
     case MCBinaryExpr::GTE:  Result = LHS >= RHS; break;
     case MCBinaryExpr::LAnd: Result = LHS && RHS; break;
     case MCBinaryExpr::LOr:  Result = LHS || RHS; break;
+    case MCBinaryExpr::LShr: Result = uint64_t(LHS) >> uint64_t(RHS); break;
     case MCBinaryExpr::LT:   Result = LHS < RHS; break;
     case MCBinaryExpr::LTE:  Result = LHS <= RHS; break;
     case MCBinaryExpr::Mod:  Result = LHS % RHS; break;
     case MCBinaryExpr::Mul:  Result = LHS * RHS; break;
     case MCBinaryExpr::NE:   Result = LHS != RHS; break;
     case MCBinaryExpr::Or:   Result = LHS | RHS; break;
-    case MCBinaryExpr::Shl:  Result = LHS << RHS; break;
-    case MCBinaryExpr::Shr:  Result = LHS >> RHS; break;
+    case MCBinaryExpr::Shl:  Result = uint64_t(LHS) << uint64_t(RHS); break;
     case MCBinaryExpr::Sub:  Result = LHS - RHS; break;
     case MCBinaryExpr::Xor:  Result = LHS ^ RHS; break;
     }
@@ -742,11 +765,11 @@ bool MCExpr::EvaluateAsRelocatableImpl(MCValue &Res, const MCAssembler *Asm,
   llvm_unreachable("Invalid assembly expression kind!");
 }
 
-const MCSection *MCExpr::FindAssociatedSection() const {
+MCSection *MCExpr::findAssociatedSection() const {
   switch (getKind()) {
   case Target:
     // We never look through target specific expressions.
-    return cast<MCTargetExpr>(this)->FindAssociatedSection();
+    return cast<MCTargetExpr>(this)->findAssociatedSection();
 
   case Constant:
     return MCSymbol::AbsolutePseudoSection;
@@ -762,12 +785,12 @@ const MCSection *MCExpr::FindAssociatedSection() const {
   }
 
   case Unary:
-    return cast<MCUnaryExpr>(this)->getSubExpr()->FindAssociatedSection();
+    return cast<MCUnaryExpr>(this)->getSubExpr()->findAssociatedSection();
 
   case Binary: {
     const MCBinaryExpr *BE = cast<MCBinaryExpr>(this);
-    const MCSection *LHS_S = BE->getLHS()->FindAssociatedSection();
-    const MCSection *RHS_S = BE->getRHS()->FindAssociatedSection();
+    MCSection *LHS_S = BE->getLHS()->findAssociatedSection();
+    MCSection *RHS_S = BE->getRHS()->findAssociatedSection();
 
     // If either section is absolute, return the other.
     if (LHS_S == MCSymbol::AbsolutePseudoSection)
diff --git a/lib/MC/MCInstPrinter.cpp b/lib/MC/MCInstPrinter.cpp
index 0dc31215ce9..23afe805484 100644
--- a/lib/MC/MCInstPrinter.cpp
+++ b/lib/MC/MCInstPrinter.cpp
@@ -16,6 +16,15 @@
 #include "llvm/Support/raw_ostream.h"
 using namespace llvm;
 
+void llvm::dumpBytes(ArrayRef<uint8_t> bytes, raw_ostream &OS) {
+  static const char hex_rep[] = "0123456789abcdef";
+  for (char i: bytes) {
+    OS << hex_rep[(i & 0xF0) >> 4];
+    OS << hex_rep[i & 0xF];
+    OS << ' ';
+  }
+}
+
 MCInstPrinter::~MCInstPrinter() {
 }
 
diff --git a/lib/MC/MCInstrDesc.cpp b/lib/MC/MCInstrDesc.cpp
new file mode 100644
index 00000000000..5be2fa1b30b
--- /dev/null
+++ b/lib/MC/MCInstrDesc.cpp
@@ -0,0 +1,70 @@
+//===------ llvm/MC/MCInstrDesc.cpp- Instruction Descriptors --------------===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This file defines methods on the MCOperandInfo and MCInstrDesc classes, which
+// are used to describe target instructions and their operands.
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/MC/MCInstrDesc.h"
+#include "llvm/MC/MCInst.h"
+#include "llvm/MC/MCRegisterInfo.h"
+#include "llvm/MC/MCSubtargetInfo.h"
+
+using namespace llvm;
+
+bool MCInstrDesc::getDeprecatedInfo(MCInst &MI, const MCSubtargetInfo &STI,
+                                    std::string &Info) const {
+  if (ComplexDeprecationInfo)
+    return ComplexDeprecationInfo(MI, STI, Info);
+  if (DeprecatedFeature != -1 && STI.getFeatureBits()[DeprecatedFeature]) {
+    // FIXME: it would be nice to include the subtarget feature here.
+    Info = "deprecated";
+    return true;
+  }
+  return false;
+}
+bool MCInstrDesc::mayAffectControlFlow(const MCInst &MI,
+                                       const MCRegisterInfo &RI) const {
+  if (isBranch() || isCall() || isReturn() || isIndirectBranch())
+    return true;
+  unsigned PC = RI.getProgramCounter();
+  if (PC == 0)
+    return false;
+  if (hasDefOfPhysReg(MI, PC, RI))
+    return true;
+  // A variadic instruction may define PC in the variable operand list.
+  // There's currently no indication of which entries in a variable
+  // list are defs and which are uses. While that's the case, this function
+  // needs to assume they're defs in order to be conservatively correct.
+  for (int i = NumOperands, e = MI.getNumOperands(); i != e; ++i) {
+    if (MI.getOperand(i).isReg() &&
+        RI.isSubRegisterEq(PC, MI.getOperand(i).getReg()))
+      return true;
+  }
+  return false;
+}
+
+bool MCInstrDesc::hasImplicitDefOfPhysReg(unsigned Reg,
+                                          const MCRegisterInfo *MRI) const {
+  if (const uint16_t *ImpDefs = ImplicitDefs)
+    for (; *ImpDefs; ++ImpDefs)
+      if (*ImpDefs == Reg || (MRI && MRI->isSubRegister(Reg, *ImpDefs)))
+        return true;
+  return false;
+}
+
+bool MCInstrDesc::hasDefOfPhysReg(const MCInst &MI, unsigned Reg,
+                                  const MCRegisterInfo &RI) const {
+  for (int i = 0, e = NumDefs; i != e; ++i)
+    if (MI.getOperand(i).isReg() &&
+        RI.isSubRegisterEq(Reg, MI.getOperand(i).getReg()))
+      return true;
+  return hasImplicitDefOfPhysReg(Reg, &RI);
+}
diff --git a/lib/MC/MCLinkerOptimizationHint.cpp b/lib/MC/MCLinkerOptimizationHint.cpp
index 8db22dc56e1..5f6a57980ad 100644
--- a/lib/MC/MCLinkerOptimizationHint.cpp
+++ b/lib/MC/MCLinkerOptimizationHint.cpp
@@ -22,14 +22,12 @@ using namespace llvm;
 // - Its argN.
 // <arg1> to <argN> are absolute addresses in the object file, i.e.,
 // relative addresses from the beginning of the object file.
-void MCLOHDirective::Emit_impl(raw_ostream &OutStream,
+void MCLOHDirective::emit_impl(raw_ostream &OutStream,
                                const MachObjectWriter &ObjWriter,
                                const MCAsmLayout &Layout) const {
-  const MCAssembler &Asm = Layout.getAssembler();
   encodeULEB128(Kind, OutStream);
   encodeULEB128(Args.size(), OutStream);
   for (LOHArgs::const_iterator It = Args.begin(), EndIt = Args.end();
        It != EndIt; ++It)
-    encodeULEB128(ObjWriter.getSymbolAddress(&Asm.getSymbolData(**It), Layout),
-                  OutStream);
+    encodeULEB128(ObjWriter.getSymbolAddress(**It, Layout), OutStream);
 }
diff --git a/lib/MC/MCMachOStreamer.cpp b/lib/MC/MCMachOStreamer.cpp
index 5c78f5fc69b..53cd1317a3d 100644
--- a/lib/MC/MCMachOStreamer.cpp
+++ b/lib/MC/MCMachOStreamer.cpp
@@ -18,12 +18,11 @@
 #include "llvm/MC/MCExpr.h"
 #include "llvm/MC/MCInst.h"
 #include "llvm/MC/MCLinkerOptimizationHint.h"
-#include "llvm/MC/MCMachOSymbolFlags.h"
 #include "llvm/MC/MCObjectFileInfo.h"
 #include "llvm/MC/MCObjectStreamer.h"
 #include "llvm/MC/MCSection.h"
 #include "llvm/MC/MCSectionMachO.h"
-#include "llvm/MC/MCSymbol.h"
+#include "llvm/MC/MCSymbolMachO.h"
 #include "llvm/Support/Dwarf.h"
 #include "llvm/Support/ErrorHandling.h"
 #include "llvm/Support/TargetRegistry.h"
@@ -68,7 +67,7 @@ class MCMachOStreamer : public MCObjectStreamer {
   /// @name MCStreamer Interface
   /// @{
 
-  void ChangeSection(const MCSection *Sect, const MCExpr *Subsect) override;
+  void ChangeSection(MCSection *Sect, const MCExpr *Subsect) override;
   void EmitLabel(MCSymbol *Symbol) override;
   void EmitEHSymAttributes(const MCSymbol *Symbol, MCSymbol *EHSymbol) override;
   void EmitAssemblerFlag(MCAssemblerFlag Flag) override;
@@ -93,14 +92,11 @@ class MCMachOStreamer : public MCObjectStreamer {
   void EndCOFFSymbolDef() override {
     llvm_unreachable("macho doesn't support this directive");
   }
-  void EmitELFSize(MCSymbol *Symbol, const MCExpr *Value) override {
-    llvm_unreachable("macho doesn't support this directive");
-  }
   void EmitLocalCommonSymbol(MCSymbol *Symbol, uint64_t Size,
                              unsigned ByteAlignment) override;
-  void EmitZerofill(const MCSection *Section, MCSymbol *Symbol = nullptr,
+  void EmitZerofill(MCSection *Section, MCSymbol *Symbol = nullptr,
                     uint64_t Size = 0, unsigned ByteAlignment = 0) override;
-  void EmitTBSSSymbol(const MCSection *Section, MCSymbol *Symbol, uint64_t Size,
+  void EmitTBSSSymbol(MCSection *Section, MCSymbol *Symbol, uint64_t Size,
                       unsigned ByteAlignment = 0) override;
 
   void EmitFileDirective(StringRef Filename) override {
@@ -149,7 +145,7 @@ static bool canGoAfterDWARF(const MCSectionMachO &MSec) {
   return false;
 }
 
-void MCMachOStreamer::ChangeSection(const MCSection *Section,
+void MCMachOStreamer::ChangeSection(MCSection *Section,
                                     const MCExpr *Subsection) {
   // Change the section normally.
   bool Created = MCObjectStreamer::changeSectionImpl(Section, Subsection);
@@ -162,22 +158,22 @@ void MCMachOStreamer::ChangeSection(const MCSection *Section,
 
   // Output a linker-local symbol so we don't need section-relative local
   // relocations. The linker hates us when we do that.
-  if (LabelSections && !HasSectionLabel[Section]) {
-    MCSymbol *Label = getContext().CreateLinkerPrivateTempSymbol();
-    EmitLabel(Label);
+  if (LabelSections && !HasSectionLabel[Section] &&
+      !Section->getBeginSymbol()) {
+    MCSymbol *Label = getContext().createLinkerPrivateTempSymbol();
+    Section->setBeginSymbol(Label);
     HasSectionLabel[Section] = true;
   }
 }
 
 void MCMachOStreamer::EmitEHSymAttributes(const MCSymbol *Symbol,
                                           MCSymbol *EHSymbol) {
-  MCSymbolData &SD =
-    getAssembler().getOrCreateSymbolData(*Symbol);
-  if (SD.isExternal())
+  getAssembler().registerSymbol(*Symbol);
+  if (Symbol->isExternal())
     EmitSymbolAttribute(EHSymbol, MCSA_Global);
-  if (SD.getFlags() & SF_WeakDefinition)
+  if (cast<MCSymbolMachO>(Symbol)->isWeakDefinition())
     EmitSymbolAttribute(EHSymbol, MCSA_WeakDefinition);
-  if (SD.isPrivateExtern())
+  if (Symbol->isPrivateExtern())
     EmitSymbolAttribute(EHSymbol, MCSA_PrivateExtern);
 }
 
@@ -193,7 +189,6 @@ void MCMachOStreamer::EmitLabel(MCSymbol *Symbol) {
 
   MCObjectStreamer::EmitLabel(Symbol);
 
-  MCSymbolData &SD = getAssembler().getSymbolData(*Symbol);
   // This causes the reference type flag to be cleared. Darwin 'as' was "trying"
   // to clear the weak reference and weak definition bits too, but the
   // implementation was buggy. For now we just try to match 'as', for
@@ -201,14 +196,14 @@ void MCMachOStreamer::EmitLabel(MCSymbol *Symbol) {
   //
   // FIXME: Cleanup this code, these bits should be emitted based on semantic
   // properties, not on the order of definition, etc.
-  SD.setFlags(SD.getFlags() & ~SF_ReferenceTypeMask);
+  cast<MCSymbolMachO>(Symbol)->clearReferenceType();
 }
 
 void MCMachOStreamer::EmitDataRegion(DataRegionData::KindTy Kind) {
   if (!getAssembler().getBackend().hasDataInCodeSupport())
     return;
   // Create a temporary label to mark the start of the data region.
-  MCSymbol *Start = getContext().CreateTempSymbol();
+  MCSymbol *Start = getContext().createTempSymbol();
   EmitLabel(Start);
   // Record the region for the object writer to use.
   DataRegionData Data = { Kind, Start, nullptr };
@@ -224,7 +219,7 @@ void MCMachOStreamer::EmitDataRegionEnd() {
   DataRegionData &Data = Regions.back();
   assert(!Data.End && "Mismatched .end_data_region!");
   // Create a temporary label to mark the end of the data region.
-  Data.End = getContext().CreateTempSymbol();
+  Data.End = getContext().createTempSymbol();
   EmitLabel(Data.End);
 }
 
@@ -276,10 +271,13 @@ void MCMachOStreamer::EmitThumbFunc(MCSymbol *Symbol) {
   // Remember that the function is a thumb function. Fixup and relocation
   // values will need adjusted.
   getAssembler().setIsThumbFunc(Symbol);
+  cast<MCSymbolMachO>(Symbol)->setThumbFunc();
 }
 
-bool MCMachOStreamer::EmitSymbolAttribute(MCSymbol *Symbol,
+bool MCMachOStreamer::EmitSymbolAttribute(MCSymbol *Sym,
                                           MCSymbolAttr Attribute) {
+  MCSymbolMachO *Symbol = cast<MCSymbolMachO>(Sym);
+
   // Indirect symbols are handled differently, to match how 'as' handles
   // them. This makes writing matching .o files easier.
   if (Attribute == MCSA_IndirectSymbol) {
@@ -287,15 +285,15 @@ bool MCMachOStreamer::EmitSymbolAttribute(MCSymbol *Symbol,
     // important for matching the string table that 'as' generates.
     IndirectSymbolData ISD;
     ISD.Symbol = Symbol;
-    ISD.SectionData = getCurrentSectionData();
+    ISD.Section = getCurrentSectionOnly();
     getAssembler().getIndirectSymbols().push_back(ISD);
     return true;
   }
 
   // Adding a symbol attribute always introduces the symbol, note that an
-  // important side effect of calling getOrCreateSymbolData here is to register
+  // important side effect of calling registerSymbol here is to register
   // the symbol with the assembler.
-  MCSymbolData &SD = getAssembler().getOrCreateSymbolData(*Symbol);
+  getAssembler().registerSymbol(*Symbol);
 
   // The implementation of symbol attributes is designed to match 'as', but it
   // leaves much to desired. It doesn't really make sense to arbitrarily add and
@@ -321,53 +319,54 @@ bool MCMachOStreamer::EmitSymbolAttribute(MCSymbol *Symbol,
     return false;
 
   case MCSA_Global:
-    SD.setExternal(true);
+    Symbol->setExternal(true);
     // This effectively clears the undefined lazy bit, in Darwin 'as', although
     // it isn't very consistent because it implements this as part of symbol
     // lookup.
     //
     // FIXME: Cleanup this code, these bits should be emitted based on semantic
     // properties, not on the order of definition, etc.
-    SD.setFlags(SD.getFlags() & ~SF_ReferenceTypeUndefinedLazy);
+    Symbol->setReferenceTypeUndefinedLazy(false);
     break;
 
   case MCSA_LazyReference:
     // FIXME: This requires -dynamic.
-    SD.setFlags(SD.getFlags() | SF_NoDeadStrip);
+    Symbol->setNoDeadStrip();
     if (Symbol->isUndefined())
-      SD.setFlags(SD.getFlags() | SF_ReferenceTypeUndefinedLazy);
+      Symbol->setReferenceTypeUndefinedLazy(true);
     break;
 
     // Since .reference sets the no dead strip bit, it is equivalent to
     // .no_dead_strip in practice.
   case MCSA_Reference:
   case MCSA_NoDeadStrip:
-    SD.setFlags(SD.getFlags() | SF_NoDeadStrip);
+    Symbol->setNoDeadStrip();
     break;
 
   case MCSA_SymbolResolver:
-    SD.setFlags(SD.getFlags() | SF_SymbolResolver);
+    Symbol->setSymbolResolver();
     break;
 
   case MCSA_PrivateExtern:
-    SD.setExternal(true);
-    SD.setPrivateExtern(true);
+    Symbol->setExternal(true);
+    Symbol->setPrivateExtern(true);
     break;
 
   case MCSA_WeakReference:
     // FIXME: This requires -dynamic.
     if (Symbol->isUndefined())
-      SD.setFlags(SD.getFlags() | SF_WeakReference);
+      Symbol->setWeakReference();
     break;
 
   case MCSA_WeakDefinition:
     // FIXME: 'as' enforces that this is defined and global. The manual claims
     // it has to be in a coalesced section, but this isn't enforced.
-    SD.setFlags(SD.getFlags() | SF_WeakDefinition);
+    Symbol->setWeakDefinition();
     break;
 
   case MCSA_WeakDefAutoPrivate:
-    SD.setFlags(SD.getFlags() | SF_WeakDefinition | SF_WeakReference);
+    Symbol->setWeakDefinition();
+    Symbol->setWeakReference();
     break;
   }
 
@@ -376,10 +375,8 @@ bool MCMachOStreamer::EmitSymbolAttribute(MCSymbol *Symbol,
 
 void MCMachOStreamer::EmitSymbolDesc(MCSymbol *Symbol, unsigned DescValue) {
   // Encode the 'desc' value into the lowest implementation defined bits.
-  assert(DescValue == (DescValue & SF_DescFlagsMask) &&
-         "Invalid .desc value!");
-  getAssembler().getOrCreateSymbolData(*Symbol).setFlags(
-    DescValue & SF_DescFlagsMask);
+  getAssembler().registerSymbol(*Symbol);
+  cast<MCSymbolMachO>(Symbol)->setDesc(DescValue);
 }
 
 void MCMachOStreamer::EmitCommonSymbol(MCSymbol *Symbol, uint64_t Size,
@@ -389,9 +386,9 @@ void MCMachOStreamer::EmitCommonSymbol(MCSymbol *Symbol, uint64_t Size,
 
   AssignSection(Symbol, nullptr);
 
-  MCSymbolData &SD = getAssembler().getOrCreateSymbolData(*Symbol);
-  SD.setExternal(true);
-  SD.setCommon(Size, ByteAlignment);
+  getAssembler().registerSymbol(*Symbol);
+  Symbol->setExternal(true);
+  Symbol->setCommon(Size, ByteAlignment);
 }
 
 void MCMachOStreamer::EmitLocalCommonSymbol(MCSymbol *Symbol, uint64_t Size,
@@ -401,9 +398,9 @@ void MCMachOStreamer::EmitLocalCommonSymbol(MCSymbol *Symbol, uint64_t Size,
                       Symbol, Size, ByteAlignment);
 }
 
-void MCMachOStreamer::EmitZerofill(const MCSection *Section, MCSymbol *Symbol,
+void MCMachOStreamer::EmitZerofill(MCSection *Section, MCSymbol *Symbol,
                                    uint64_t Size, unsigned ByteAlignment) {
-  MCSectionData &SectData = getAssembler().getOrCreateSectionData(*Section);
+  getAssembler().registerSection(*Section);
 
   // The symbol may not be present, which only creates the section.
   if (!Symbol)
@@ -414,25 +411,25 @@ void MCMachOStreamer::EmitZerofill(const MCSection *Section, MCSymbol *Symbol,
 
   assert(Symbol->isUndefined() && "Cannot define a symbol twice!");
 
-  MCSymbolData &SD = getAssembler().getOrCreateSymbolData(*Symbol);
+  getAssembler().registerSymbol(*Symbol);
 
   // Emit an align fragment if necessary.
   if (ByteAlignment != 1)
-    new MCAlignFragment(ByteAlignment, 0, 0, ByteAlignment, &SectData);
-
-  MCFragment *F = new MCFillFragment(0, 0, Size, &SectData);
-  SD.setFragment(F);
+    new MCAlignFragment(ByteAlignment, 0, 0, ByteAlignment, Section);
 
   AssignSection(Symbol, Section);
 
+  MCFragment *F = new MCFillFragment(0, 0, Size, Section);
+  Symbol->setFragment(F);
+
   // Update the maximum alignment on the zero fill section if necessary.
-  if (ByteAlignment > SectData.getAlignment())
-    SectData.setAlignment(ByteAlignment);
+  if (ByteAlignment > Section->getAlignment())
+    Section->setAlignment(ByteAlignment);
 }
 
 // This should always be called with the thread local bss section.  Like the
 // .zerofill directive this doesn't actually switch sections on us.
-void MCMachOStreamer::EmitTBSSSymbol(const MCSection *Section, MCSymbol *Symbol,
+void MCMachOStreamer::EmitTBSSSymbol(MCSection *Section, MCSymbol *Symbol,
                                      uint64_t Size, unsigned ByteAlignment) {
   EmitZerofill(Section, Symbol, Size, ByteAlignment);
   return;
@@ -445,7 +442,7 @@ void MCMachOStreamer::EmitInstToData(const MCInst &Inst,
   SmallVector<MCFixup, 4> Fixups;
   SmallString<256> Code;
   raw_svector_ostream VecOS(Code);
-  getAssembler().getEmitter().EncodeInstruction(Inst, VecOS, Fixups, STI);
+  getAssembler().getEmitter().encodeInstruction(Inst, VecOS, Fixups, STI);
   VecOS.flush();
 
   // Add the fixups and data.
@@ -464,13 +461,13 @@ void MCMachOStreamer::FinishImpl() {
 
   // First, scan the symbol table to build a lookup table from fragments to
   // defining symbols.
-  DenseMap<const MCFragment*, MCSymbolData*> DefiningSymbolMap;
-  for (MCSymbolData &SD : getAssembler().symbols()) {
-    if (getAssembler().isSymbolLinkerVisible(SD.getSymbol()) &&
-        SD.getFragment()) {
+  DenseMap<const MCFragment *, const MCSymbol *> DefiningSymbolMap;
+  for (const MCSymbol &Symbol : getAssembler().symbols()) {
+    if (getAssembler().isSymbolLinkerVisible(Symbol) && Symbol.getFragment()) {
       // An atom defining symbol should never be internal to a fragment.
-      assert(SD.getOffset() == 0 && "Invalid offset in atom defining symbol!");
-      DefiningSymbolMap[SD.getFragment()] = &SD;
+      assert(Symbol.getOffset() == 0 &&
+             "Invalid offset in atom defining symbol!");
+      DefiningSymbolMap[Symbol.getFragment()] = &Symbol;
     }
   }
 
@@ -478,11 +475,11 @@ void MCMachOStreamer::FinishImpl() {
   // symbol.
   for (MCAssembler::iterator it = getAssembler().begin(),
          ie = getAssembler().end(); it != ie; ++it) {
-    MCSymbolData *CurrentAtom = nullptr;
-    for (MCSectionData::iterator it2 = it->begin(),
-           ie2 = it->end(); it2 != ie2; ++it2) {
-      if (MCSymbolData *SD = DefiningSymbolMap.lookup(it2))
-        CurrentAtom = SD;
+    const MCSymbol *CurrentAtom = nullptr;
+    for (MCSection::iterator it2 = it->begin(), ie2 = it->end(); it2 != ie2;
+         ++it2) {
+      if (const MCSymbol *Symbol = DefiningSymbolMap.lookup(it2))
+        CurrentAtom = Symbol;
       it2->setAtom(CurrentAtom);
     }
   }
diff --git a/lib/MC/MCMachObjectTargetWriter.cpp b/lib/MC/MCMachObjectTargetWriter.cpp
index 146cebf01a3..4ffd6a78a61 100644
--- a/lib/MC/MCMachObjectTargetWriter.cpp
+++ b/lib/MC/MCMachObjectTargetWriter.cpp
@@ -11,12 +11,9 @@
 
 using namespace llvm;
 
-MCMachObjectTargetWriter::MCMachObjectTargetWriter(
-  bool Is64Bit_, uint32_t CPUType_, uint32_t CPUSubtype_,
-  bool UseAggressiveSymbolFolding_)
-  : Is64Bit(Is64Bit_), CPUType(CPUType_), CPUSubtype(CPUSubtype_),
-    UseAggressiveSymbolFolding(UseAggressiveSymbolFolding_) {
-}
+MCMachObjectTargetWriter::MCMachObjectTargetWriter(bool Is64Bit_,
+                                                   uint32_t CPUType_,
+                                                   uint32_t CPUSubtype_)
+    : Is64Bit(Is64Bit_), CPUType(CPUType_), CPUSubtype(CPUSubtype_) {}
 
-MCMachObjectTargetWriter::~MCMachObjectTargetWriter() {
-}
+MCMachObjectTargetWriter::~MCMachObjectTargetWriter() {}
diff --git a/lib/MC/MCNaCl.cpp b/lib/MC/MCNaCl.cpp
deleted file mode 100644
index 3c36d15f8f0..00000000000
--- a/lib/MC/MCNaCl.cpp
+++ /dev/null
@@ -1,94 +0,0 @@
-//===- lib/MC/MCNaCl.cpp - NaCl-specific MC implementation ----------------===//
-//
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
-//
-//===----------------------------------------------------------------------===//
-
-#include "llvm/MC/MCNaCl.h"
-#include "llvm/ADT/Triple.h"
-#include "llvm/MC/MCContext.h"
-#include "llvm/MC/MCNaClExpander.h"
-#include "llvm/MC/MCSectionELF.h"
-#include "llvm/MC/MCStreamer.h"
-#include "llvm/Support/CommandLine.h"
-#include "llvm/Support/TargetRegistry.h"
-#include "llvm/Support/ELF.h"
-
-static const char NoteNamespace[] = "NaCl";
-
-namespace llvm {
-
-void initializeNaClMCStreamer(MCStreamer &Streamer, MCContext &Ctx,
-                              const Triple &TheTriple) {
-  assert(TheTriple.isOSNaCl());
-  const char *NoteName;
-  const char *NoteArch;
-  unsigned BundleAlign;
-  switch (TheTriple.getArch()) {
-    case Triple::arm:
-      NoteName = ".note.NaCl.ABI.arm";
-      NoteArch = "arm";
-      BundleAlign = 4;
-      break;
-    case Triple::mipsel:
-      NoteName = ".note.NaCl.ABI.mipsel";
-      NoteArch = "mipsel";
-      BundleAlign = 4;
-      break;
-    case Triple::x86:
-      NoteName = ".note.NaCl.ABI.x86-32";
-      NoteArch = "x86-32";
-      BundleAlign = 5;
-      break;
-    case Triple::x86_64:
-      NoteName = ".note.NaCl.ABI.x86-64";
-      NoteArch = "x86-64";
-      BundleAlign = 5;
-      break;
-    default:
-      report_fatal_error("Unsupported architecture for NaCl");
-  }
-
-  std::string Error; //empty
-  const Target *TheTarget = 
-    TargetRegistry::lookupTarget(TheTriple.getTriple(), Error);
-
-  // Create the Target specific MCNaClExpander
-  assert(TheTarget != nullptr);
-  TheTarget->createMCNaClExpander(
-      Streamer, std::unique_ptr<MCRegisterInfo>(
-                    TheTarget->createMCRegInfo(TheTriple.getTriple())),
-      std::unique_ptr<MCInstrInfo>(TheTarget->createMCInstrInfo()));
-
-  // Set bundle-alignment as required by the NaCl ABI for the target.
-  Streamer.EmitBundleAlignMode(BundleAlign);
-
-  // For gas, override the size of DWARF address values generated by .loc
-  // directives.
-  if (TheTriple.getArch() == Triple::x86_64 &&
-      Streamer.hasRawTextSupport()) {
-    Streamer.EmitRawText("\t.dwarf_addr_size 4\n");
-  }
-  // Emit an ELF Note section in its own COMDAT group which identifies NaCl
-  // object files to the gold linker, so it can use the NaCl layout.
-  const MCSection *Note = Ctx.getELFSection(
-      NoteName, ELF::SHT_NOTE, ELF::SHF_ALLOC | ELF::SHF_GROUP, 0, NoteName);
-
-  Streamer.PushSection();
-  Streamer.SwitchSection(Note);
-  Streamer.EmitIntValue(strlen(NoteNamespace) + 1, 4);
-  Streamer.EmitIntValue(strlen(NoteArch) + 1, 4);
-  Streamer.EmitIntValue(ELF::NT_VERSION, 4);
-  Streamer.EmitBytes(NoteNamespace);
-  Streamer.EmitIntValue(0, 1); // NUL terminator
-  Streamer.EmitValueToAlignment(4);
-  Streamer.EmitBytes(NoteArch);
-  Streamer.EmitIntValue(0, 1); // NUL terminator
-  Streamer.EmitValueToAlignment(4);
-  Streamer.PopSection();
-}
-
-} // namespace llvm
diff --git a/lib/MC/MCNaClExpander.cpp b/lib/MC/MCNaClExpander.cpp
deleted file mode 100644
index c6bdfd5d5ca..00000000000
--- a/lib/MC/MCNaClExpander.cpp
+++ /dev/null
@@ -1,51 +0,0 @@
-//===- MCNaClExpander.cpp ---------------------------------------*- C++ -*-===//
-//
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
-//
-//===----------------------------------------------------------------------===//
-//
-// This file implements the MCNaClExpander class. This is a base
-// class that encapsulates the expansion logic for MCInsts, and holds
-// state such as available scratch registers.
-//
-//===----------------------------------------------------------------------===//
-
-#include "llvm/MC/MCNaClExpander.h"
-#include "llvm/MC/MCInst.h"
-#include "llvm/MC/MCInstrInfo.h"
-#include "llvm/MC/MCRegisterInfo.h"
-#include "llvm/Support/SourceMgr.h"
-
-namespace llvm {
-
-void MCNaClExpander::Error(const MCInst &Inst, const char msg[]) {
-  if (SrcMgr)
-    SrcMgr->PrintMessage(Inst.getLoc(), SourceMgr::DK_Error, msg);
-  else
-    report_fatal_error(msg, true);
-}
-
-void MCNaClExpander::pushScratchReg(unsigned Reg) {
-  ScratchRegs.push_back(Reg);
-}
-
-unsigned MCNaClExpander::popScratchReg() {
-  assert(!ScratchRegs.empty() &&
-         "Trying to pop an empty scratch register stack");
-
-  unsigned Reg = ScratchRegs.back();
-  ScratchRegs.pop_back();
-  return Reg;
-}
-
-unsigned MCNaClExpander::getScratchReg(int index) {
-  int len = numScratchRegs();
-  return ScratchRegs[len - index - 1];
-}
-
-unsigned MCNaClExpander::numScratchRegs() { return ScratchRegs.size(); }
-
-}
diff --git a/lib/MC/MCNullStreamer.cpp b/lib/MC/MCNullStreamer.cpp
index fc56728f635..eb2d91254b3 100644
--- a/lib/MC/MCNullStreamer.cpp
+++ b/lib/MC/MCNullStreamer.cpp
@@ -31,7 +31,7 @@ namespace {
 
     void EmitCommonSymbol(MCSymbol *Symbol, uint64_t Size,
                           unsigned ByteAlignment) override {}
-    void EmitZerofill(const MCSection *Section, MCSymbol *Symbol = nullptr,
+    void EmitZerofill(MCSection *Section, MCSymbol *Symbol = nullptr,
                       uint64_t Size = 0, unsigned ByteAlignment = 0) override {}
     void EmitGPRel32Value(const MCExpr *Value) override {}
   };
diff --git a/lib/MC/MCObjectFileInfo.cpp b/lib/MC/MCObjectFileInfo.cpp
index cd7036249ad..576827a72d5 100644
--- a/lib/MC/MCObjectFileInfo.cpp
+++ b/lib/MC/MCObjectFileInfo.cpp
@@ -39,7 +39,7 @@ static bool useCompactUnwind(const Triple &T) {
   return false;
 }
 
-void MCObjectFileInfo::InitMachOMCObjectFileInfo(Triple T) {
+void MCObjectFileInfo::initMachOMCObjectFileInfo(Triple T) {
   // MachO
   SupportsWeakOmittedEHFrame = false;
 
@@ -238,10 +238,13 @@ void MCObjectFileInfo::InitMachOMCObjectFileInfo(Triple T) {
   StackMapSection = Ctx->getMachOSection("__LLVM_STACKMAPS", "__llvm_stackmaps",
                                          0, SectionKind::getMetadata());
 
+  FaultMapSection = Ctx->getMachOSection("__LLVM_FAULTMAPS", "__llvm_faultmaps",
+                                         0, SectionKind::getMetadata());
+
   TLSExtraDataSection = TLSTLVSection;
 }
 
-void MCObjectFileInfo::InitELFMCObjectFileInfo(Triple T) {
+void MCObjectFileInfo::initELFMCObjectFileInfo(Triple T) {
   switch (T.getArch()) {
   case Triple::mips:
   case Triple::mipsel:
@@ -324,10 +327,16 @@ void MCObjectFileInfo::InitELFMCObjectFileInfo(Triple T) {
   case Triple::mipsel:
   case Triple::mips64:
   case Triple::mips64el:
-    // MIPS uses indirect pointer to refer personality functions, so that the
-    // eh_frame section can be read-only.  DW.ref.personality will be generated
-    // for relocation.
+    // MIPS uses indirect pointer to refer personality functions and types, so
+    // that the eh_frame section can be read-only. DW.ref.personality will be
+    // generated for relocation.
     PersonalityEncoding = dwarf::DW_EH_PE_indirect;
+    // FIXME: The N64 ABI probably ought to use DW_EH_PE_sdata8 but we can't
+    //        identify N64 from just a triple.
+    TTypeEncoding = dwarf::DW_EH_PE_indirect | dwarf::DW_EH_PE_pcrel |
+                    dwarf::DW_EH_PE_sdata4;
+    // We don't support PC-relative LSDA references in GAS so we use the default
+    // DW_EH_PE_absptr for those.
     break;
   case Triple::ppc64:
   case Triple::ppc64le:
@@ -337,6 +346,7 @@ void MCObjectFileInfo::InitELFMCObjectFileInfo(Triple T) {
     TTypeEncoding = dwarf::DW_EH_PE_indirect | dwarf::DW_EH_PE_pcrel |
       dwarf::DW_EH_PE_udata8;
     break;
+  case Triple::sparcel:
   case Triple::sparc:
     if (RelocM == Reloc::PIC_) {
       LSDAEncoding = dwarf::DW_EH_PE_pcrel | dwarf::DW_EH_PE_sdata4;
@@ -511,9 +521,12 @@ void MCObjectFileInfo::InitELFMCObjectFileInfo(Triple T) {
 
   StackMapSection =
       Ctx->getELFSection(".llvm_stackmaps", ELF::SHT_PROGBITS, ELF::SHF_ALLOC);
+
+  FaultMapSection =
+      Ctx->getELFSection(".llvm_faultmaps", ELF::SHT_PROGBITS, ELF::SHF_ALLOC);
 }
 
-void MCObjectFileInfo::InitCOFFMCObjectFileInfo(Triple T) {
+void MCObjectFileInfo::initCOFFMCObjectFileInfo(Triple T) {
   bool IsWoA = T.getArch() == Triple::arm || T.getArch() == Triple::thumb;
 
   CommDirectiveSupportsAlignment = true;
@@ -713,13 +726,22 @@ void MCObjectFileInfo::InitCOFFMCObjectFileInfo(Triple T) {
       ".xdata", COFF::IMAGE_SCN_CNT_INITIALIZED_DATA | COFF::IMAGE_SCN_MEM_READ,
       SectionKind::getDataRel());
 
+  SXDataSection = Ctx->getCOFFSection(".sxdata", COFF::IMAGE_SCN_LNK_INFO,
+                                      SectionKind::getMetadata());
+
   TLSDataSection = Ctx->getCOFFSection(
       ".tls$", COFF::IMAGE_SCN_CNT_INITIALIZED_DATA | COFF::IMAGE_SCN_MEM_READ |
                    COFF::IMAGE_SCN_MEM_WRITE,
       SectionKind::getDataRel());
+	  
+  StackMapSection = Ctx->getCOFFSection(".llvm_stackmaps",
+                                        COFF::IMAGE_SCN_CNT_INITIALIZED_DATA |
+                                            COFF::IMAGE_SCN_MEM_READ,
+                                        SectionKind::getReadOnly());	
 }
 
-void MCObjectFileInfo::InitMCObjectFileInfo(StringRef T, Reloc::Model relocm,
+void MCObjectFileInfo::InitMCObjectFileInfo(const Triple &TheTriple,
+                                            Reloc::Model relocm,
                                             CodeModel::Model cm,
                                             MCContext &ctx) {
   RelocM = relocm;
@@ -743,7 +765,7 @@ void MCObjectFileInfo::InitMCObjectFileInfo(StringRef T, Reloc::Model relocm,
   DwarfAccelNamespaceSection = nullptr; // Used only by selected targets.
   DwarfAccelTypesSection = nullptr;     // Used only by selected targets.
 
-  TT = Triple(T);
+  TT = TheTriple;
 
   Triple::ArchType Arch = TT.getArch();
   // FIXME: Checking for Arch here to filter out bogus triples such as
@@ -753,21 +775,27 @@ void MCObjectFileInfo::InitMCObjectFileInfo(StringRef T, Reloc::Model relocm,
        Arch == Triple::aarch64 ||
        Arch == Triple::ppc || Arch == Triple::ppc64 ||
        Arch == Triple::UnknownArch) &&
-      (TT.isOSDarwin() || TT.isOSBinFormatMachO())) {
+      TT.isOSBinFormatMachO()) {
     Env = IsMachO;
-    InitMachOMCObjectFileInfo(TT);
+    initMachOMCObjectFileInfo(TT);
   } else if ((Arch == Triple::x86 || Arch == Triple::x86_64 ||
               Arch == Triple::arm || Arch == Triple::thumb) &&
              (TT.isOSWindows() && TT.getObjectFormat() == Triple::COFF)) {
     Env = IsCOFF;
-    InitCOFFMCObjectFileInfo(TT);
+    initCOFFMCObjectFileInfo(TT);
   } else {
     Env = IsELF;
-    InitELFMCObjectFileInfo(TT);
+    initELFMCObjectFileInfo(TT);
   }
 }
 
-const MCSection *MCObjectFileInfo::getDwarfTypesSection(uint64_t Hash) const {
+void MCObjectFileInfo::InitMCObjectFileInfo(StringRef TT, Reloc::Model RM,
+                                            CodeModel::Model CM,
+                                            MCContext &ctx) {
+  InitMCObjectFileInfo(Triple(TT), RM, CM, ctx);
+}
+
+MCSection *MCObjectFileInfo::getDwarfTypesSection(uint64_t Hash) const {
   return Ctx->getELFSection(".debug_types", ELF::SHT_PROGBITS, ELF::SHF_GROUP,
                             0, utostr(Hash));
 }
diff --git a/lib/MC/MCObjectStreamer.cpp b/lib/MC/MCObjectStreamer.cpp
index b0a3d498a97..0a637775d4e 100644
--- a/lib/MC/MCObjectStreamer.cpp
+++ b/lib/MC/MCObjectStreamer.cpp
@@ -13,7 +13,6 @@
 #include "llvm/MC/MCAsmInfo.h"
 #include "llvm/MC/MCAssembler.h"
 #include "llvm/MC/MCCodeEmitter.h"
-#include "llvm/MC/MCNaClExpander.h" // @LOCALMOD
 #include "llvm/MC/MCContext.h"
 #include "llvm/MC/MCDwarf.h"
 #include "llvm/MC/MCExpr.h"
@@ -30,7 +29,7 @@ MCObjectStreamer::MCObjectStreamer(MCContext &Context, MCAsmBackend &TAB,
     : MCStreamer(Context),
       Assembler(new MCAssembler(Context, TAB, *Emitter_,
                                 *TAB.createObjectWriter(OS), OS)),
-      CurSectionData(nullptr), EmitEHFrame(true), EmitDebugFrame(false) {}
+      EmitEHFrame(true), EmitDebugFrame(false) {}
 
 MCObjectStreamer::~MCObjectStreamer() {
   delete &Assembler->getBackend();
@@ -43,22 +42,36 @@ void MCObjectStreamer::flushPendingLabels(MCFragment *F, uint64_t FOffset) {
   if (PendingLabels.size()) {
     if (!F) {
       F = new MCDataFragment();
-      CurSectionData->getFragmentList().insert(CurInsertionPoint, F);
-      F->setParent(CurSectionData);
+      MCSection *CurSection = getCurrentSectionOnly();
+      CurSection->getFragmentList().insert(CurInsertionPoint, F);
+      F->setParent(CurSection);
     }
-    for (MCSymbolData *SD : PendingLabels) {
-      SD->setFragment(F);
-      SD->setOffset(FOffset);
+    for (MCSymbol *Sym : PendingLabels) {
+      Sym->setFragment(F);
+      Sym->setOffset(FOffset);
     }
     PendingLabels.clear();
   }
 }
 
+void MCObjectStreamer::emitAbsoluteSymbolDiff(const MCSymbol *Hi,
+                                              const MCSymbol *Lo,
+                                              unsigned Size) {
+  // If not assigned to the same (valid) fragment, fallback.
+  if (!Hi->getFragment() || Hi->getFragment() != Lo->getFragment()) {
+    MCStreamer::emitAbsoluteSymbolDiff(Hi, Lo, Size);
+    return;
+  }
+
+  assert(Hi->getOffset() >= Lo->getOffset() &&
+         "Expected Hi to be greater than Lo");
+  EmitIntValue(Hi->getOffset() - Lo->getOffset(), Size);
+}
+
 void MCObjectStreamer::reset() {
   if (Assembler)
     Assembler->reset();
-  CurSectionData = nullptr;
-  CurInsertionPoint = MCSectionData::iterator();
+  CurInsertionPoint = MCSection::iterator();
   EmitEHFrame = true;
   EmitDebugFrame = false;
   PendingLabels.clear();
@@ -77,9 +90,9 @@ void MCObjectStreamer::EmitFrames(MCAsmBackend *MAB) {
 }
 
 MCFragment *MCObjectStreamer::getCurrentFragment() const {
-  assert(getCurrentSectionData() && "No current section!");
+  assert(getCurrentSectionOnly() && "No current section!");
 
-  if (CurInsertionPoint != getCurrentSectionData()->getFragmentList().begin())
+  if (CurInsertionPoint != getCurrentSectionOnly()->getFragmentList().begin())
     return std::prev(CurInsertionPoint);
 
   return nullptr;
@@ -98,7 +111,7 @@ MCDataFragment *MCObjectStreamer::getOrCreateDataFragment() {
 }
 
 void MCObjectStreamer::visitUsedSymbol(const MCSymbol &Sym) {
-  Assembler->getOrCreateSymbolData(Sym);
+  Assembler->registerSymbol(Sym);
 }
 
 void MCObjectStreamer::EmitCFISections(bool EH, bool Debug) {
@@ -117,32 +130,32 @@ void MCObjectStreamer::EmitValueImpl(const MCExpr *Value, unsigned Size,
 
   // Avoid fixups when possible.
   int64_t AbsValue;
-  if (Value->EvaluateAsAbsolute(AbsValue, getAssembler())) {
+  if (Value->evaluateAsAbsolute(AbsValue, getAssembler())) {
     EmitIntValue(AbsValue, Size);
     return;
   }
   DF->getFixups().push_back(
-      MCFixup::Create(DF->getContents().size(), Value,
+      MCFixup::create(DF->getContents().size(), Value,
                       MCFixup::getKindForSize(Size, false), Loc));
   DF->getContents().resize(DF->getContents().size() + Size, 0);
 }
 
 void MCObjectStreamer::EmitCFIStartProcImpl(MCDwarfFrameInfo &Frame) {
   // We need to create a local symbol to avoid relocations.
-  Frame.Begin = getContext().CreateTempSymbol();
+  Frame.Begin = getContext().createTempSymbol();
   EmitLabel(Frame.Begin);
 }
 
 void MCObjectStreamer::EmitCFIEndProcImpl(MCDwarfFrameInfo &Frame) {
-  Frame.End = getContext().CreateTempSymbol();
+  Frame.End = getContext().createTempSymbol();
   EmitLabel(Frame.End);
 }
 
 void MCObjectStreamer::EmitLabel(MCSymbol *Symbol) {
   MCStreamer::EmitLabel(Symbol);
 
-  MCSymbolData &SD = getAssembler().getOrCreateSymbolData(*Symbol);
-  assert(!SD.getFragment() && "Unexpected fragment on symbol data!");
+  getAssembler().registerSymbol(*Symbol);
+  assert(!Symbol->getFragment() && "Unexpected fragment on symbol data!");
 
   // If there is a current fragment, mark the symbol as pointing into it.
   // Otherwise queue the label and set its fragment pointer when we emit the
@@ -150,16 +163,16 @@ void MCObjectStreamer::EmitLabel(MCSymbol *Symbol) {
   auto *F = dyn_cast_or_null<MCDataFragment>(getCurrentFragment());
   if (F && !(getAssembler().isBundlingEnabled() &&
              getAssembler().getRelaxAll())) {
-    SD.setFragment(F);
-    SD.setOffset(F->getContents().size());
+    Symbol->setFragment(F);
+    Symbol->setOffset(F->getContents().size());
   } else {
-    PendingLabels.push_back(&SD);
+    PendingLabels.push_back(Symbol);
   }
 }
 
 void MCObjectStreamer::EmitULEB128Value(const MCExpr *Value) {
   int64_t IntValue;
-  if (Value->EvaluateAsAbsolute(IntValue, getAssembler())) {
+  if (Value->evaluateAsAbsolute(IntValue, getAssembler())) {
     EmitULEB128IntValue(IntValue);
     return;
   }
@@ -168,7 +181,7 @@ void MCObjectStreamer::EmitULEB128Value(const MCExpr *Value) {
 
 void MCObjectStreamer::EmitSLEB128Value(const MCExpr *Value) {
   int64_t IntValue;
-  if (Value->EvaluateAsAbsolute(IntValue, getAssembler())) {
+  if (Value->evaluateAsAbsolute(IntValue, getAssembler())) {
     EmitSLEB128IntValue(IntValue);
     return;
   }
@@ -180,52 +193,44 @@ void MCObjectStreamer::EmitWeakReference(MCSymbol *Alias,
   report_fatal_error("This file format doesn't support weak aliases.");
 }
 
-void MCObjectStreamer::ChangeSection(const MCSection *Section,
+void MCObjectStreamer::ChangeSection(MCSection *Section,
                                      const MCExpr *Subsection) {
   changeSectionImpl(Section, Subsection);
 }
 
-bool MCObjectStreamer::changeSectionImpl(const MCSection *Section,
+bool MCObjectStreamer::changeSectionImpl(MCSection *Section,
                                          const MCExpr *Subsection) {
   assert(Section && "Cannot switch to a null section!");
   flushPendingLabels(nullptr);
 
-  bool Created;
-  CurSectionData = &getAssembler().getOrCreateSectionData(*Section, &Created);
+  bool Created = getAssembler().registerSection(*Section);
 
   int64_t IntSubsection = 0;
   if (Subsection &&
-      !Subsection->EvaluateAsAbsolute(IntSubsection, getAssembler()))
+      !Subsection->evaluateAsAbsolute(IntSubsection, getAssembler()))
     report_fatal_error("Cannot evaluate subsection number");
   if (IntSubsection < 0 || IntSubsection > 8192)
     report_fatal_error("Subsection number out of range");
   CurInsertionPoint =
-    CurSectionData->getSubsectionInsertionPoint(unsigned(IntSubsection));
+      Section->getSubsectionInsertionPoint(unsigned(IntSubsection));
   return Created;
 }
 
 void MCObjectStreamer::EmitAssignment(MCSymbol *Symbol, const MCExpr *Value) {
-  getAssembler().getOrCreateSymbolData(*Symbol);
+  getAssembler().registerSymbol(*Symbol);
   MCStreamer::EmitAssignment(Symbol, Value);
 }
 
+bool MCObjectStreamer::mayHaveInstructions(MCSection &Sec) const {
+  return Sec.hasInstructions();
+}
+
 void MCObjectStreamer::EmitInstruction(const MCInst &Inst,
                                        const MCSubtargetInfo &STI) {
-
-  // @LOCALMOD-START
-  if (NaClExpander && NaClExpander->expandInst(Inst, *this, STI))
-    return;
-
-  if (getAssembler().isBundlingEnabled() &&
-      getAssembler().getBackend().CustomExpandInst(Inst, *this)) {
-    return;
-  }
-  // @LOCALMOD-END
-
   MCStreamer::EmitInstruction(Inst, STI);
 
-  MCSectionData *SD = getCurrentSectionData();
-  SD->setHasInstructions(true);
+  MCSection *Sec = getCurrentSectionOnly();
+  Sec->setHasInstructions(true);
 
   // Now that a machine instruction has been assembled into this section, make
   // a line entry for any .loc directive that has been seen.
@@ -244,7 +249,7 @@ void MCObjectStreamer::EmitInstruction(const MCInst &Inst,
   //   group. We want to emit all such instructions into the same data
   //   fragment.
   if (Assembler.getRelaxAll() ||
-      (Assembler.isBundlingEnabled() && SD->isBundleLocked())) {
+      (Assembler.isBundlingEnabled() && Sec->isBundleLocked())) {
     MCInst Relaxed;
     getAssembler().getBackend().relaxInstruction(Inst, Relaxed);
     while (getAssembler().getBackend().mayNeedRelaxation(Relaxed))
@@ -269,7 +274,7 @@ void MCObjectStreamer::EmitInstToFragment(const MCInst &Inst,
 
   SmallString<128> Code;
   raw_svector_ostream VecOS(Code);
-  getAssembler().getEmitter().EncodeInstruction(Inst, VecOS, IF->getFixups(),
+  getAssembler().getEmitter().encodeInstruction(Inst, VecOS, IF->getFixups(),
                                                 STI);
   VecOS.flush();
   IF->getContents().append(Code.begin(), Code.end());
@@ -309,10 +314,10 @@ static const MCExpr *buildSymbolDiff(MCObjectStreamer &OS, const MCSymbol *A,
                                      const MCSymbol *B) {
   MCContext &Context = OS.getContext();
   MCSymbolRefExpr::VariantKind Variant = MCSymbolRefExpr::VK_None;
-  const MCExpr *ARef = MCSymbolRefExpr::Create(A, Variant, Context);
-  const MCExpr *BRef = MCSymbolRefExpr::Create(B, Variant, Context);
+  const MCExpr *ARef = MCSymbolRefExpr::create(A, Variant, Context);
+  const MCExpr *BRef = MCSymbolRefExpr::create(B, Variant, Context);
   const MCExpr *AddrDelta =
-      MCBinaryExpr::Create(MCBinaryExpr::Sub, ARef, BRef, Context);
+      MCBinaryExpr::create(MCBinaryExpr::Sub, ARef, BRef, Context);
   return AddrDelta;
 }
 
@@ -338,7 +343,7 @@ void MCObjectStreamer::EmitDwarfAdvanceLineAddr(int64_t LineDelta,
   }
   const MCExpr *AddrDelta = buildSymbolDiff(*this, Label, LastLabel);
   int64_t Res;
-  if (AddrDelta->EvaluateAsAbsolute(Res, getAssembler())) {
+  if (AddrDelta->evaluateAsAbsolute(Res, getAssembler())) {
     MCDwarfLineAddr::Emit(this, LineDelta, Res);
     return;
   }
@@ -349,7 +354,7 @@ void MCObjectStreamer::EmitDwarfAdvanceFrameAddr(const MCSymbol *LastLabel,
                                                  const MCSymbol *Label) {
   const MCExpr *AddrDelta = buildSymbolDiff(*this, Label, LastLabel);
   int64_t Res;
-  if (AddrDelta->EvaluateAsAbsolute(Res, getAssembler())) {
+  if (AddrDelta->evaluateAsAbsolute(Res, getAssembler())) {
     MCDwarfFrameEmitter::EmitAdvanceLoc(*this, Res);
     return;
   }
@@ -372,8 +377,9 @@ void MCObjectStreamer::EmitValueToAlignment(unsigned ByteAlignment,
   insert(new MCAlignFragment(ByteAlignment, Value, ValueSize, MaxBytesToEmit));
 
   // Update the maximum alignment on the current section if necessary.
-  if (ByteAlignment > getCurrentSectionData()->getAlignment())
-    getCurrentSectionData()->setAlignment(ByteAlignment);
+  MCSection *CurSec = getCurrentSection().first;
+  if (ByteAlignment > CurSec->getAlignment())
+    CurSec->setAlignment(ByteAlignment);
 }
 
 void MCObjectStreamer::EmitCodeAlignment(unsigned ByteAlignment,
@@ -385,20 +391,20 @@ void MCObjectStreamer::EmitCodeAlignment(unsigned ByteAlignment,
 bool MCObjectStreamer::EmitValueToOffset(const MCExpr *Offset,
                                          unsigned char Value) {
   int64_t Res;
-  if (Offset->EvaluateAsAbsolute(Res, getAssembler())) {
+  if (Offset->evaluateAsAbsolute(Res, getAssembler())) {
     insert(new MCOrgFragment(*Offset, Value));
     return false;
   }
 
-  MCSymbol *CurrentPos = getContext().CreateTempSymbol();
+  MCSymbol *CurrentPos = getContext().createTempSymbol();
   EmitLabel(CurrentPos);
   MCSymbolRefExpr::VariantKind Variant = MCSymbolRefExpr::VK_None;
   const MCExpr *Ref =
-    MCSymbolRefExpr::Create(CurrentPos, Variant, getContext());
+    MCSymbolRefExpr::create(CurrentPos, Variant, getContext());
   const MCExpr *Delta =
-    MCBinaryExpr::Create(MCBinaryExpr::Sub, Offset, Ref, getContext());
+    MCBinaryExpr::create(MCBinaryExpr::Sub, Offset, Ref, getContext());
 
-  if (!Delta->EvaluateAsAbsolute(Res, getAssembler()))
+  if (!Delta->evaluateAsAbsolute(Res, getAssembler()))
     return true;
   EmitFill(Res, Value);
   return false;
@@ -409,7 +415,7 @@ void MCObjectStreamer::EmitGPRel32Value(const MCExpr *Value) {
   MCDataFragment *DF = getOrCreateDataFragment();
   flushPendingLabels(DF, DF->getContents().size());
 
-  DF->getFixups().push_back(MCFixup::Create(DF->getContents().size(), 
+  DF->getFixups().push_back(MCFixup::create(DF->getContents().size(), 
                                             Value, FK_GPRel_4));
   DF->getContents().resize(DF->getContents().size() + 4, 0);
 }
@@ -419,7 +425,7 @@ void MCObjectStreamer::EmitGPRel64Value(const MCExpr *Value) {
   MCDataFragment *DF = getOrCreateDataFragment();
   flushPendingLabels(DF, DF->getContents().size());
 
-  DF->getFixups().push_back(MCFixup::Create(DF->getContents().size(), 
+  DF->getFixups().push_back(MCFixup::create(DF->getContents().size(), 
                                             Value, FK_GPRel_4));
   DF->getContents().resize(DF->getContents().size() + 8, 0);
 }
diff --git a/lib/MC/MCObjectWriter.cpp b/lib/MC/MCObjectWriter.cpp
index e40c07d5527..34790340873 100644
--- a/lib/MC/MCObjectWriter.cpp
+++ b/lib/MC/MCObjectWriter.cpp
@@ -17,7 +17,7 @@ using namespace llvm;
 MCObjectWriter::~MCObjectWriter() {
 }
 
-bool MCObjectWriter::IsSymbolRefDifferenceFullyResolved(
+bool MCObjectWriter::isSymbolRefDifferenceFullyResolved(
     const MCAssembler &Asm, const MCSymbolRefExpr *A, const MCSymbolRefExpr *B,
     bool InSet) const {
   // Modified symbol references cannot be resolved.
@@ -30,23 +30,20 @@ bool MCObjectWriter::IsSymbolRefDifferenceFullyResolved(
   if (SA.isUndefined() || SB.isUndefined())
     return false;
 
-  const MCSymbolData &DataA = Asm.getSymbolData(SA);
-  const MCSymbolData &DataB = Asm.getSymbolData(SB);
-  if(!DataA.getFragment() || !DataB.getFragment())
+  if (!SA.getFragment() || !SB.getFragment())
     return false;
 
-  return IsSymbolRefDifferenceFullyResolvedImpl(
-      Asm, DataA, &DataB, *DataB.getFragment(), InSet, false);
+  return isSymbolRefDifferenceFullyResolvedImpl(Asm, SA, *SB.getFragment(),
+                                                InSet, false);
 }
 
-bool MCObjectWriter::IsSymbolRefDifferenceFullyResolvedImpl(
-    const MCAssembler &Asm, const MCSymbolData &DataA,
-    const MCSymbolData *DataB, const MCFragment &FB, bool InSet,
-    bool IsPCRel) const {
-  const MCSection &SecA = DataA.getSymbol().getSection();
-  const MCSection &SecB = FB.getParent()->getSection();
+bool MCObjectWriter::isSymbolRefDifferenceFullyResolvedImpl(
+    const MCAssembler &Asm, const MCSymbol &SymA, const MCFragment &FB,
+    bool InSet, bool IsPCRel) const {
+  const MCSection &SecA = SymA.getSection();
+  const MCSection &SecB = *FB.getParent();
   // On ELF and COFF  A - B is absolute if A and B are in the same section.
   return &SecA == &SecB;
 }
 
-bool MCObjectWriter::isWeak(const MCSymbolData &SD) const { return false; }
+bool MCObjectWriter::isWeak(const MCSymbol &) const { return false; }
diff --git a/lib/MC/MCParser/AsmParser.cpp b/lib/MC/MCParser/AsmParser.cpp
index 551ddb036aa..3f45b3d85a3 100644
--- a/lib/MC/MCParser/AsmParser.cpp
+++ b/lib/MC/MCParser/AsmParser.cpp
@@ -26,6 +26,7 @@
 #include "llvm/MC/MCParser/AsmCond.h"
 #include "llvm/MC/MCParser/AsmLexer.h"
 #include "llvm/MC/MCParser/MCAsmParser.h"
+#include "llvm/MC/MCParser/MCAsmParserUtils.h"
 #include "llvm/MC/MCParser/MCParsedAsmOperand.h"
 #include "llvm/MC/MCRegisterInfo.h"
 #include "llvm/MC/MCSectionMachO.h"
@@ -122,7 +123,6 @@ class AsmParser : public MCAsmParser {
   SourceMgr::DiagHandlerTy SavedDiagHandler;
   void *SavedDiagContext;
   std::unique_ptr<MCAsmParserExtension> PlatformParser;
-  std::unique_ptr<MCAsmParserExtension> NaClParser; // @LOCALMOD
 
   /// This is the current buffer index we're lexing from as managed by the
   /// SourceMgr object.
@@ -148,6 +148,9 @@ class AsmParser : public MCAsmParser {
   /// Boolean tracking whether macro substitution is enabled.
   unsigned MacrosEnabledFlag : 1;
 
+  /// \brief Keeps track of how many .macro's have been instantiated.
+  unsigned NumOfMacroInstantiations;
+
   /// Flag tracking whether any errors have been encountered.
   unsigned HadError : 1;
 
@@ -185,6 +188,10 @@ class AsmParser : public MCAsmParser {
     ExtensionDirectiveMap[Directive] = Handler;
   }
 
+  void addAliasForDirective(StringRef Directive, StringRef Alias) override {
+    DirectiveKindMap[Directive] = DirectiveKindMap[Alias];
+  }
+
 public:
   /// @name MCAsmParser Interface
   /// {
@@ -227,6 +234,8 @@ class AsmParser : public MCAsmParser {
   bool parseExpression(const MCExpr *&Res, SMLoc &EndLoc) override;
   bool parsePrimaryExpr(const MCExpr *&Res, SMLoc &EndLoc) override;
   bool parseParenExpression(const MCExpr *&Res, SMLoc &EndLoc) override;
+  bool parseParenExprOfDepth(unsigned ParenDepth, const MCExpr *&Res,
+                             SMLoc &EndLoc) override;
   bool parseAbsoluteExpression(int64_t &Res) override;
 
   /// \brief Parse an identifier or string (as a quoted identifier)
@@ -248,7 +257,7 @@ class AsmParser : public MCAsmParser {
                         ArrayRef<MCAsmMacroParameter> Parameters);
   bool expandMacro(raw_svector_ostream &OS, StringRef Body,
                    ArrayRef<MCAsmMacroParameter> Parameters,
-                   ArrayRef<MCAsmMacroArgument> A,
+                   ArrayRef<MCAsmMacroArgument> A, bool EnableAtPseudoVariable,
                    const SMLoc &L);
 
   /// \brief Are macros enabled in the parser?
@@ -320,6 +329,9 @@ class AsmParser : public MCAsmParser {
   bool parseAssignment(StringRef Name, bool allow_redef,
                        bool NoDeadStrip = false);
 
+  unsigned getBinOpPrecedence(AsmToken::TokenKind K,
+                              MCBinaryExpr::Opcode &Kind);
+
   bool parseBinOpRHS(unsigned Precedence, const MCExpr *&Res, SMLoc &EndLoc);
   bool parseParenExpr(const MCExpr *&Res, SMLoc &EndLoc);
   bool parseBracketExpr(const MCExpr *&Res, SMLoc &EndLoc);
@@ -482,7 +494,6 @@ namespace llvm {
 extern MCAsmParserExtension *createDarwinAsmParser();
 extern MCAsmParserExtension *createELFAsmParser();
 extern MCAsmParserExtension *createCOFFAsmParser();
-extern MCAsmParserExtension *createNaClAsmParser(MCNaClExpander *Exp); // @LOCALMOD
 
 }
 
@@ -491,8 +502,7 @@ enum { DEFAULT_ADDRSPACE = 0 };
 AsmParser::AsmParser(SourceMgr &SM, MCContext &Ctx, MCStreamer &Out,
                      const MCAsmInfo &MAI)
     : Lexer(MAI), Ctx(Ctx), Out(Out), MAI(MAI), SrcMgr(SM),
-      PlatformParser(nullptr), NaClParser(nullptr), // @LOCALMOD
-      CurBuffer(SM.getMainFileID()),
+      PlatformParser(nullptr), CurBuffer(SM.getMainFileID()),
       MacrosEnabledFlag(true), HadError(false), CppHashLineNumber(0),
       AssemblerDialect(~0U), IsDarwin(false), ParsingInlineAsm(false) {
   // Save the old handler.
@@ -517,13 +527,9 @@ AsmParser::AsmParser(SourceMgr &SM, MCContext &Ctx, MCStreamer &Out,
   }
 
   PlatformParser->Initialize(*this);
-  // @LOCALMOD-START
-  if (Out.getNaClExpander()) {
-    NaClParser.reset(createNaClAsmParser(Out.getNaClExpander()));
-    NaClParser->Initialize(*this);
-  }
-  // @LOCALMOD-END
   initializeDirectiveKindMap();
+
+  NumOfMacroInstantiations = 0;
 }
 
 AsmParser::~AsmParser() {
@@ -547,6 +553,8 @@ void AsmParser::Note(SMLoc L, const Twine &Msg, ArrayRef<SMRange> Ranges) {
 }
 
 bool AsmParser::Warning(SMLoc L, const Twine &Msg, ArrayRef<SMRange> Ranges) {
+  if(getTargetParser().getTargetOptions().MCNoWarn)
+    return false;
   if (getTargetParser().getTargetOptions().MCFatalWarnings)
     return Error(L, Msg, Ranges);
   printMessage(L, SourceMgr::DK_Warning, Msg, Ranges);
@@ -627,12 +635,15 @@ bool AsmParser::Run(bool NoInitialTextSection, bool NoFinalize) {
   // If we are generating dwarf for assembly source files save the initial text
   // section and generate a .file directive.
   if (getContext().getGenDwarfForAssembly()) {
-    MCSymbol *SectionStartSym = getContext().CreateTempSymbol();
-    getStreamer().EmitLabel(SectionStartSym);
-    auto InsertResult = getContext().addGenDwarfSection(
-        getStreamer().getCurrentSection().first);
-    assert(InsertResult.second && ".text section should not have debug info yet");
-    InsertResult.first->second.first = SectionStartSym;
+    MCSection *Sec = getStreamer().getCurrentSection().first;
+    if (!Sec->getBeginSymbol()) {
+      MCSymbol *SectionStartSym = getContext().createTempSymbol();
+      getStreamer().EmitLabel(SectionStartSym);
+      Sec->setBeginSymbol(SectionStartSym);
+    }
+    bool InsertResult = getContext().addGenDwarfSection(Sec);
+    assert(InsertResult && ".text section should not have debug info yet");
+    (void)InsertResult;
     getContext().setGenDwarfFileNumber(getStreamer().EmitDwarfFileDirective(
         0, StringRef(), getContext().getMainFileName()));
   }
@@ -783,7 +794,7 @@ bool AsmParser::parsePrimaryExpr(const MCExpr *&Res, SMLoc &EndLoc) {
     Lex(); // Eat the operator.
     if (parsePrimaryExpr(Res, EndLoc))
       return true;
-    Res = MCUnaryExpr::CreateLNot(Res, getContext());
+    Res = MCUnaryExpr::createLNot(Res, getContext());
     return false;
   case AsmToken::Dollar:
   case AsmToken::At:
@@ -795,9 +806,9 @@ bool AsmParser::parsePrimaryExpr(const MCExpr *&Res, SMLoc &EndLoc) {
         if (Lexer.getMAI().getDollarIsPC()) {
           // This is a '$' reference, which references the current PC.  Emit a
           // temporary label to the streamer and refer to it.
-          MCSymbol *Sym = Ctx.CreateTempSymbol();
+          MCSymbol *Sym = Ctx.createTempSymbol();
           Out.EmitLabel(Sym);
-          Res = MCSymbolRefExpr::Create(Sym, MCSymbolRefExpr::VK_None,
+          Res = MCSymbolRefExpr::create(Sym, MCSymbolRefExpr::VK_None,
                                         getContext());
           EndLoc = FirstTokenLoc;
           return false;
@@ -852,7 +863,7 @@ bool AsmParser::parsePrimaryExpr(const MCExpr *&Res, SMLoc &EndLoc) {
       }
     }
 
-    MCSymbol *Sym = getContext().GetOrCreateSymbol(SymbolName);
+    MCSymbol *Sym = getContext().getOrCreateSymbol(SymbolName);
 
     // If this is an absolute variable reference, substitute it now to preserve
     // semantics in the face of reassignment.
@@ -865,7 +876,7 @@ bool AsmParser::parsePrimaryExpr(const MCExpr *&Res, SMLoc &EndLoc) {
     }
 
     // Otherwise create a symbol ref.
-    Res = MCSymbolRefExpr::Create(Sym, Variant, getContext());
+    Res = MCSymbolRefExpr::create(Sym, Variant, getContext());
     return false;
   }
   case AsmToken::BigNum:
@@ -873,7 +884,7 @@ bool AsmParser::parsePrimaryExpr(const MCExpr *&Res, SMLoc &EndLoc) {
   case AsmToken::Integer: {
     SMLoc Loc = getTok().getLoc();
     int64_t IntVal = getTok().getIntVal();
-    Res = MCConstantExpr::Create(IntVal, getContext());
+    Res = MCConstantExpr::create(IntVal, getContext());
     EndLoc = Lexer.getTok().getEndLoc();
     Lex(); // Eat token.
     // Look for 'b' or 'f' following an Integer as a directional label
@@ -890,8 +901,8 @@ bool AsmParser::parsePrimaryExpr(const MCExpr *&Res, SMLoc &EndLoc) {
       }
       if (IDVal == "f" || IDVal == "b") {
         MCSymbol *Sym =
-            Ctx.GetDirectionalLocalSymbol(IntVal, IDVal == "b");
-        Res = MCSymbolRefExpr::Create(Sym, Variant, getContext());
+            Ctx.getDirectionalLocalSymbol(IntVal, IDVal == "b");
+        Res = MCSymbolRefExpr::create(Sym, Variant, getContext());
         if (IDVal == "b" && Sym->isUndefined())
           return Error(Loc, "invalid reference to undefined symbol");
         EndLoc = Lexer.getTok().getEndLoc();
@@ -903,7 +914,7 @@ bool AsmParser::parsePrimaryExpr(const MCExpr *&Res, SMLoc &EndLoc) {
   case AsmToken::Real: {
     APFloat RealVal(APFloat::IEEEdouble, getTok().getString());
     uint64_t IntVal = RealVal.bitcastToAPInt().getZExtValue();
-    Res = MCConstantExpr::Create(IntVal, getContext());
+    Res = MCConstantExpr::create(IntVal, getContext());
     EndLoc = Lexer.getTok().getEndLoc();
     Lex(); // Eat token.
     return false;
@@ -911,9 +922,9 @@ bool AsmParser::parsePrimaryExpr(const MCExpr *&Res, SMLoc &EndLoc) {
   case AsmToken::Dot: {
     // This is a '.' reference, which references the current PC.  Emit a
     // temporary label to the streamer and refer to it.
-    MCSymbol *Sym = Ctx.CreateTempSymbol();
+    MCSymbol *Sym = Ctx.createTempSymbol();
     Out.EmitLabel(Sym);
-    Res = MCSymbolRefExpr::Create(Sym, MCSymbolRefExpr::VK_None, getContext());
+    Res = MCSymbolRefExpr::create(Sym, MCSymbolRefExpr::VK_None, getContext());
     EndLoc = Lexer.getTok().getEndLoc();
     Lex(); // Eat identifier.
     return false;
@@ -930,19 +941,19 @@ bool AsmParser::parsePrimaryExpr(const MCExpr *&Res, SMLoc &EndLoc) {
     Lex(); // Eat the operator.
     if (parsePrimaryExpr(Res, EndLoc))
       return true;
-    Res = MCUnaryExpr::CreateMinus(Res, getContext());
+    Res = MCUnaryExpr::createMinus(Res, getContext());
     return false;
   case AsmToken::Plus:
     Lex(); // Eat the operator.
     if (parsePrimaryExpr(Res, EndLoc))
       return true;
-    Res = MCUnaryExpr::CreatePlus(Res, getContext());
+    Res = MCUnaryExpr::createPlus(Res, getContext());
     return false;
   case AsmToken::Tilde:
     Lex(); // Eat the operator.
     if (parsePrimaryExpr(Res, EndLoc))
       return true;
-    Res = MCUnaryExpr::CreateNot(Res, getContext());
+    Res = MCUnaryExpr::createNot(Res, getContext());
     return false;
   }
 }
@@ -975,7 +986,7 @@ AsmParser::applyModifierToExpr(const MCExpr *E,
       return E;
     }
 
-    return MCSymbolRefExpr::Create(&SRE->getSymbol(), Variant, getContext());
+    return MCSymbolRefExpr::create(&SRE->getSymbol(), Variant, getContext());
   }
 
   case MCExpr::Unary: {
@@ -983,7 +994,7 @@ AsmParser::applyModifierToExpr(const MCExpr *E,
     const MCExpr *Sub = applyModifierToExpr(UE->getSubExpr(), Variant);
     if (!Sub)
       return nullptr;
-    return MCUnaryExpr::Create(UE->getOpcode(), Sub, getContext());
+    return MCUnaryExpr::create(UE->getOpcode(), Sub, getContext());
   }
 
   case MCExpr::Binary: {
@@ -999,7 +1010,7 @@ AsmParser::applyModifierToExpr(const MCExpr *E,
     if (!RHS)
       RHS = BE->getRHS();
 
-    return MCBinaryExpr::Create(BE->getOpcode(), LHS, RHS, getContext());
+    return MCBinaryExpr::create(BE->getOpcode(), LHS, RHS, getContext());
   }
   }
 
@@ -1048,8 +1059,8 @@ bool AsmParser::parseExpression(const MCExpr *&Res, SMLoc &EndLoc) {
 
   // Try to constant fold it up front, if possible.
   int64_t Value;
-  if (Res->EvaluateAsAbsolute(Value))
-    Res = MCConstantExpr::Create(Value, getContext());
+  if (Res->evaluateAsAbsolute(Value))
+    Res = MCConstantExpr::create(Value, getContext());
 
   return false;
 }
@@ -1059,6 +1070,27 @@ bool AsmParser::parseParenExpression(const MCExpr *&Res, SMLoc &EndLoc) {
   return parseParenExpr(Res, EndLoc) || parseBinOpRHS(1, Res, EndLoc);
 }
 
+bool AsmParser::parseParenExprOfDepth(unsigned ParenDepth, const MCExpr *&Res,
+                                      SMLoc &EndLoc) {
+  if (parseParenExpr(Res, EndLoc))
+    return true;
+
+  for (; ParenDepth > 0; --ParenDepth) {
+    if (parseBinOpRHS(1, Res, EndLoc))
+      return true;
+
+    // We don't Lex() the last RParen.
+    // This is the same behavior as parseParenExpression().
+    if (ParenDepth - 1 > 0) {
+      if (Lexer.isNot(AsmToken::RParen))
+        return TokError("expected ')' in parentheses expression");
+      EndLoc = Lexer.getTok().getEndLoc();
+      Lex();
+    }
+  }
+  return false;
+}
+
 bool AsmParser::parseAbsoluteExpression(int64_t &Res) {
   const MCExpr *Expr;
 
@@ -1066,14 +1098,14 @@ bool AsmParser::parseAbsoluteExpression(int64_t &Res) {
   if (parseExpression(Expr))
     return true;
 
-  if (!Expr->EvaluateAsAbsolute(Res))
+  if (!Expr->evaluateAsAbsolute(Res))
     return Error(StartLoc, "expected absolute expression");
 
   return false;
 }
 
-static unsigned getBinOpPrecedence(AsmToken::TokenKind K,
-                                   MCBinaryExpr::Opcode &Kind) {
+unsigned AsmParser::getBinOpPrecedence(AsmToken::TokenKind K,
+                                       MCBinaryExpr::Opcode &Kind) {
   switch (K) {
   default:
     return 0; // not a binop.
@@ -1125,7 +1157,7 @@ static unsigned getBinOpPrecedence(AsmToken::TokenKind K,
     Kind = MCBinaryExpr::Shl;
     return 4;
   case AsmToken::GreaterGreater:
-    Kind = MCBinaryExpr::Shr;
+    Kind = MAI.shouldUseLogicalShr() ? MCBinaryExpr::LShr : MCBinaryExpr::AShr;
     return 4;
 
   // High Intermediate Precedence: +, -
@@ -1177,7 +1209,7 @@ bool AsmParser::parseBinOpRHS(unsigned Precedence, const MCExpr *&Res,
       return true;
 
     // Merge LHS and RHS according to operator.
-    Res = MCBinaryExpr::Create(Kind, Res, RHS, getContext());
+    Res = MCBinaryExpr::create(Kind, Res, RHS, getContext());
   }
 }
 
@@ -1300,15 +1332,17 @@ bool AsmParser::parseStatement(ParseStatementInfo &Info,
     MCSymbol *Sym;
     if (LocalLabelVal == -1) {
       if (ParsingInlineAsm && SI) {
-        StringRef RewrittenLabel = SI->LookupInlineAsmLabel(IDVal, getSourceManager(), IDLoc, true);
-        assert(RewrittenLabel.size() && "We should have an internal name here.");
+        StringRef RewrittenLabel =
+            SI->LookupInlineAsmLabel(IDVal, getSourceManager(), IDLoc, true);
+        assert(RewrittenLabel.size() &&
+               "We should have an internal name here.");
         Info.AsmRewrites->push_back(AsmRewrite(AOK_Label, IDLoc,
                                                IDVal.size(), RewrittenLabel));
         IDVal = RewrittenLabel;
       }
-      Sym = getContext().GetOrCreateSymbol(IDVal);
+      Sym = getContext().getOrCreateSymbol(IDVal);
     } else
-      Sym = Ctx.CreateDirectionalLocalSymbol(LocalLabelVal);
+      Sym = Ctx.createDirectionalLocalSymbol(LocalLabelVal);
 
     Sym->redefineIfPossible();
 
@@ -1614,8 +1648,8 @@ bool AsmParser::parseStatement(ParseStatementInfo &Info,
     if (ActiveMacros.empty())
       Line = SrcMgr.FindLineNumber(IDLoc, CurBuffer);
     else
-      Line = SrcMgr.FindLineNumber(ActiveMacros.back()->InstantiationLoc,
-                                   ActiveMacros.back()->ExitBuffer);
+      Line = SrcMgr.FindLineNumber(ActiveMacros.front()->InstantiationLoc,
+                                   ActiveMacros.front()->ExitBuffer);
 
     // If we previously parsed a cpp hash file line comment then make sure the
     // current Dwarf File is for the CppHashFilename if not then emit the
@@ -1769,7 +1803,8 @@ static bool isIdentifierChar(char c) {
 
 bool AsmParser::expandMacro(raw_svector_ostream &OS, StringRef Body,
                             ArrayRef<MCAsmMacroParameter> Parameters,
-                            ArrayRef<MCAsmMacroArgument> A, const SMLoc &L) {
+                            ArrayRef<MCAsmMacroArgument> A,
+                            bool EnableAtPseudoVariable, const SMLoc &L) {
   unsigned NParameters = Parameters.size();
   bool HasVararg = NParameters ? Parameters.back().Vararg : false;
   if ((!IsDarwin || NParameters != 0) && NParameters != A.size())
@@ -1835,36 +1870,47 @@ bool AsmParser::expandMacro(raw_svector_ostream &OS, StringRef Body,
       Pos += 2;
     } else {
       unsigned I = Pos + 1;
-      while (isIdentifierChar(Body[I]) && I + 1 != End)
+
+      // Check for the \@ pseudo-variable.
+      if (EnableAtPseudoVariable && Body[I] == '@' && I + 1 != End)
         ++I;
+      else
+        while (isIdentifierChar(Body[I]) && I + 1 != End)
+          ++I;
 
       const char *Begin = Body.data() + Pos + 1;
       StringRef Argument(Begin, I - (Pos + 1));
       unsigned Index = 0;
-      for (; Index < NParameters; ++Index)
-        if (Parameters[Index].Name == Argument)
-          break;
 
-      if (Index == NParameters) {
-        if (Body[Pos + 1] == '(' && Body[Pos + 2] == ')')
-          Pos += 3;
-        else {
-          OS << '\\' << Argument;
-          Pos = I;
-        }
+      if (Argument == "@") {
+        OS << NumOfMacroInstantiations;
+        Pos += 2;
       } else {
-        bool VarargParameter = HasVararg && Index == (NParameters - 1);
-        for (MCAsmMacroArgument::const_iterator it = A[Index].begin(),
-                                                ie = A[Index].end();
-             it != ie; ++it)
-          // We expect no quotes around the string's contents when
-          // parsing for varargs.
-          if (it->getKind() != AsmToken::String || VarargParameter)
-            OS << it->getString();
-          else
-            OS << it->getStringContents();
-
-        Pos += 1 + Argument.size();
+        for (; Index < NParameters; ++Index)
+          if (Parameters[Index].Name == Argument)
+            break;
+
+        if (Index == NParameters) {
+          if (Body[Pos + 1] == '(' && Body[Pos + 2] == ')')
+            Pos += 3;
+          else {
+            OS << '\\' << Argument;
+            Pos = I;
+          }
+        } else {
+          bool VarargParameter = HasVararg && Index == (NParameters - 1);
+          for (MCAsmMacroArgument::const_iterator it = A[Index].begin(),
+                                                  ie = A[Index].end();
+               it != ie; ++it)
+            // We expect no quotes around the string's contents when
+            // parsing for varargs.
+            if (it->getKind() != AsmToken::String || VarargParameter)
+              OS << it->getString();
+            else
+              OS << it->getStringContents();
+
+          Pos += 1 + Argument.size();
+        }
       }
     }
     // Update the scan point.
@@ -1931,7 +1977,7 @@ bool AsmParser::parseMacroArgument(MCAsmMacroArgument &MA, bool Vararg) {
   if (Vararg) {
     if (Lexer.isNot(AsmToken::EndOfStatement)) {
       StringRef Str = parseStringToEndOfStatement();
-      MA.push_back(AsmToken(AsmToken::String, Str));
+      MA.emplace_back(AsmToken::String, Str);
     }
     return false;
   }
@@ -2122,7 +2168,7 @@ bool AsmParser::handleMacroEntry(const MCAsmMacro *M, SMLoc NameLoc) {
   StringRef Body = M->Body;
   raw_svector_ostream OS(Buf);
 
-  if (expandMacro(OS, Body, M->Parameters, A, getTok().getLoc()))
+  if (expandMacro(OS, Body, M->Parameters, A, true, getTok().getLoc()))
     return true;
 
   // We include the .endmacro in the buffer as our cue to exit the macro
@@ -2138,6 +2184,8 @@ bool AsmParser::handleMacroEntry(const MCAsmMacro *M, SMLoc NameLoc) {
       NameLoc, CurBuffer, getTok().getLoc(), TheCondStack.size());
   ActiveMacros.push_back(MI);
 
+  ++NumOfMacroInstantiations;
+
   // Jump to the macro instantiation and prime the lexer.
   CurBuffer = SrcMgr.AddNewSourceBuffer(std::move(Instantiation), SMLoc());
   Lexer.setBuffer(SrcMgr.getMemoryBuffer(CurBuffer)->getBuffer());
@@ -2156,82 +2204,20 @@ void AsmParser::handleMacroExit() {
   ActiveMacros.pop_back();
 }
 
-static bool isUsedIn(const MCSymbol *Sym, const MCExpr *Value) {
-  switch (Value->getKind()) {
-  case MCExpr::Binary: {
-    const MCBinaryExpr *BE = static_cast<const MCBinaryExpr *>(Value);
-    return isUsedIn(Sym, BE->getLHS()) || isUsedIn(Sym, BE->getRHS());
-  }
-  case MCExpr::Target:
-  case MCExpr::Constant:
-    return false;
-  case MCExpr::SymbolRef: {
-    const MCSymbol &S =
-        static_cast<const MCSymbolRefExpr *>(Value)->getSymbol();
-    if (S.isVariable())
-      return isUsedIn(Sym, S.getVariableValue());
-    return &S == Sym;
-  }
-  case MCExpr::Unary:
-    return isUsedIn(Sym, static_cast<const MCUnaryExpr *>(Value)->getSubExpr());
-  }
-
-  llvm_unreachable("Unknown expr kind!");
-}
-
 bool AsmParser::parseAssignment(StringRef Name, bool allow_redef,
                                 bool NoDeadStrip) {
-  // FIXME: Use better location, we should use proper tokens.
-  SMLoc EqualLoc = Lexer.getLoc();
-
+  MCSymbol *Sym;
   const MCExpr *Value;
-  if (parseExpression(Value))
+  if (MCParserUtils::parseAssignmentExpression(Name, allow_redef, *this, Sym,
+                                               Value))
     return true;
 
-  // Note: we don't count b as used in "a = b". This is to allow
-  // a = b
-  // b = c
-
-  if (Lexer.isNot(AsmToken::EndOfStatement))
-    return TokError("unexpected token in assignment");
-
-  // Eat the end of statement marker.
-  Lex();
-
-  // Validate that the LHS is allowed to be a variable (either it has not been
-  // used as a symbol, or it is an absolute symbol).
-  MCSymbol *Sym = getContext().LookupSymbol(Name);
-  if (Sym) {
-    // Diagnose assignment to a label.
-    //
-    // FIXME: Diagnostics. Note the location of the definition as a label.
-    // FIXME: Diagnose assignment to protected identifier (e.g., register name).
-    if (isUsedIn(Sym, Value))
-      return Error(EqualLoc, "Recursive use of '" + Name + "'");
-    else if (Sym->isUndefined() && !Sym->isUsed() && !Sym->isVariable())
-      ; // Allow redefinitions of undefined symbols only used in directives.
-    else if (Sym->isVariable() && !Sym->isUsed() && allow_redef)
-      ; // Allow redefinitions of variables that haven't yet been used.
-    else if (!Sym->isUndefined() && (!Sym->isVariable() || !allow_redef))
-      return Error(EqualLoc, "redefinition of '" + Name + "'");
-    else if (!Sym->isVariable())
-      return Error(EqualLoc, "invalid assignment to '" + Name + "'");
-    else if (!isa<MCConstantExpr>(Sym->getVariableValue()))
-      return Error(EqualLoc, "invalid reassignment of non-absolute variable '" +
-                                 Name + "'");
-
-    // Don't count these checks as uses.
-    Sym->setUsed(false);
-  } else if (Name == ".") {
-    if (Out.EmitValueToOffset(Value, 0)) {
-      Error(EqualLoc, "expected absolute expression");
-      eatToEndOfStatement();
-    }
+  if (!Sym) {
+    // In the case where we parse an expression starting with a '.', we will
+    // not generate an error, nor will we create a symbol.  In this case we
+    // should just return out.
     return false;
-  } else
-    Sym = getContext().GetOrCreateSymbol(Name);
-
-  Sym->setRedefinable(allow_redef);
+  }
 
   // Do the assignment.
   Out.EmitAssignment(Sym, Value);
@@ -3169,7 +3155,7 @@ bool AsmParser::parseDirectiveCFIPersonalityOrLsda(bool IsPersonality) {
   if (parseIdentifier(Name))
     return TokError("expected identifier in directive");
 
-  MCSymbol *Sym = getContext().GetOrCreateSymbol(Name);
+  MCSymbol *Sym = getContext().getOrCreateSymbol(Name);
 
   if (IsPersonality)
     getStreamer().EmitCFIPersonality(Sym, Encoding);
@@ -3683,7 +3669,7 @@ bool AsmParser::parseDirectiveSymbolAttribute(MCSymbolAttr Attr) {
       if (parseIdentifier(Name))
         return Error(Loc, "expected identifier in directive");
 
-      MCSymbol *Sym = getContext().GetOrCreateSymbol(Name);
+      MCSymbol *Sym = getContext().getOrCreateSymbol(Name);
 
       // Assembler local symbols don't make any sense here. Complain loudly.
       if (Sym->isTemporary())
@@ -3716,7 +3702,7 @@ bool AsmParser::parseDirectiveComm(bool IsLocal) {
     return TokError("expected identifier in directive");
 
   // Handle the identifier as the key symbol.
-  MCSymbol *Sym = getContext().GetOrCreateSymbol(Name);
+  MCSymbol *Sym = getContext().getOrCreateSymbol(Name);
 
   if (getLexer().isNot(AsmToken::Comma))
     return TokError("unexpected token in directive");
@@ -4013,7 +3999,7 @@ bool AsmParser::parseDirectiveIfdef(SMLoc DirectiveLoc, bool expect_defined) {
 
     Lex();
 
-    MCSymbol *Sym = getContext().LookupSymbol(Name);
+    MCSymbol *Sym = getContext().lookupSymbol(Name);
 
     if (expect_defined)
       TheCondState.CondMet = (Sym && !Sym->isUndefined());
@@ -4326,8 +4312,7 @@ MCAsmMacro *AsmParser::parseMacroLikeBody(SMLoc DirectiveLoc) {
   StringRef Body = StringRef(BodyStart, BodyEnd - BodyStart);
 
   // We Are Anonymous.
-  MacroLikeBodies.push_back(
-      MCAsmMacro(StringRef(), Body, MCAsmMacroParameters()));
+  MacroLikeBodies.emplace_back(StringRef(), Body, MCAsmMacroParameters());
   return &MacroLikeBodies.back();
 }
 
@@ -4359,7 +4344,7 @@ bool AsmParser::parseDirectiveRept(SMLoc DirectiveLoc, StringRef Dir) {
     return true;
 
   int64_t Count;
-  if (!CountExpr->EvaluateAsAbsolute(Count)) {
+  if (!CountExpr->evaluateAsAbsolute(Count)) {
     eatToEndOfStatement();
     return Error(CountLoc, "unexpected token in '" + Dir + "' directive");
   }
@@ -4383,7 +4368,8 @@ bool AsmParser::parseDirectiveRept(SMLoc DirectiveLoc, StringRef Dir) {
   SmallString<256> Buf;
   raw_svector_ostream OS(Buf);
   while (Count--) {
-    if (expandMacro(OS, M->Body, None, None, getTok().getLoc()))
+    // Note that the AtPseudoVariable is disabled for instantiations of .rep(t).
+    if (expandMacro(OS, M->Body, None, None, false, getTok().getLoc()))
       return true;
   }
   instantiateMacroLikeBody(M, DirectiveLoc, OS);
@@ -4422,7 +4408,9 @@ bool AsmParser::parseDirectiveIrp(SMLoc DirectiveLoc) {
   raw_svector_ostream OS(Buf);
 
   for (MCAsmMacroArguments::iterator i = A.begin(), e = A.end(); i != e; ++i) {
-    if (expandMacro(OS, M->Body, Parameter, *i, getTok().getLoc()))
+    // Note that the AtPseudoVariable is enabled for instantiations of .irp.
+    // This is undocumented, but GAS seems to support it.
+    if (expandMacro(OS, M->Body, Parameter, *i, true, getTok().getLoc()))
       return true;
   }
 
@@ -4467,9 +4455,11 @@ bool AsmParser::parseDirectiveIrpc(SMLoc DirectiveLoc) {
   StringRef Values = A.front().front().getString();
   for (std::size_t I = 0, End = Values.size(); I != End; ++I) {
     MCAsmMacroArgument Arg;
-    Arg.push_back(AsmToken(AsmToken::Identifier, Values.slice(I, I + 1)));
+    Arg.emplace_back(AsmToken::Identifier, Values.slice(I, I + 1));
 
-    if (expandMacro(OS, M->Body, Parameter, Arg, getTok().getLoc()))
+    // Note that the AtPseudoVariable is enabled for instantiations of .irpc.
+    // This is undocumented, but GAS seems to support it.
+    if (expandMacro(OS, M->Body, Parameter, Arg, true, getTok().getLoc()))
       return true;
   }
 
@@ -4751,6 +4741,103 @@ bool AsmParser::parseMSInlineAsm(
   return false;
 }
 
+namespace llvm {
+namespace MCParserUtils {
+
+/// Returns whether the given symbol is used anywhere in the given expression,
+/// or subexpressions.
+static bool isSymbolUsedInExpression(const MCSymbol *Sym, const MCExpr *Value) {
+  switch (Value->getKind()) {
+  case MCExpr::Binary: {
+    const MCBinaryExpr *BE = static_cast<const MCBinaryExpr *>(Value);
+    return isSymbolUsedInExpression(Sym, BE->getLHS()) ||
+           isSymbolUsedInExpression(Sym, BE->getRHS());
+  }
+  case MCExpr::Target:
+  case MCExpr::Constant:
+    return false;
+  case MCExpr::SymbolRef: {
+    const MCSymbol &S =
+        static_cast<const MCSymbolRefExpr *>(Value)->getSymbol();
+    if (S.isVariable())
+      return isSymbolUsedInExpression(Sym, S.getVariableValue());
+    return &S == Sym;
+  }
+  case MCExpr::Unary:
+    return isSymbolUsedInExpression(
+        Sym, static_cast<const MCUnaryExpr *>(Value)->getSubExpr());
+  }
+
+  llvm_unreachable("Unknown expr kind!");
+}
+
+bool parseAssignmentExpression(StringRef Name, bool allow_redef,
+                               MCAsmParser &Parser, MCSymbol *&Sym,
+                               const MCExpr *&Value) {
+  MCAsmLexer &Lexer = Parser.getLexer();
+
+  // FIXME: Use better location, we should use proper tokens.
+  SMLoc EqualLoc = Lexer.getLoc();
+
+  if (Parser.parseExpression(Value)) {
+    Parser.TokError("missing expression");
+    Parser.eatToEndOfStatement();
+    return true;
+  }
+
+  // Note: we don't count b as used in "a = b". This is to allow
+  // a = b
+  // b = c
+
+  if (Lexer.isNot(AsmToken::EndOfStatement))
+    return Parser.TokError("unexpected token in assignment");
+
+  // Eat the end of statement marker.
+  Parser.Lex();
+
+  // Validate that the LHS is allowed to be a variable (either it has not been
+  // used as a symbol, or it is an absolute symbol).
+  Sym = Parser.getContext().lookupSymbol(Name);
+  if (Sym) {
+    // Diagnose assignment to a label.
+    //
+    // FIXME: Diagnostics. Note the location of the definition as a label.
+    // FIXME: Diagnose assignment to protected identifier (e.g., register name).
+    if (isSymbolUsedInExpression(Sym, Value))
+      return Parser.Error(EqualLoc, "Recursive use of '" + Name + "'");
+    else if (Sym->isUndefined() && !Sym->isUsed() && !Sym->isVariable())
+      ; // Allow redefinitions of undefined symbols only used in directives.
+    else if (Sym->isVariable() && !Sym->isUsed() && allow_redef)
+      ; // Allow redefinitions of variables that haven't yet been used.
+    else if (!Sym->isUndefined() && (!Sym->isVariable() || !allow_redef))
+      return Parser.Error(EqualLoc, "redefinition of '" + Name + "'");
+    else if (!Sym->isVariable())
+      return Parser.Error(EqualLoc, "invalid assignment to '" + Name + "'");
+    else if (!isa<MCConstantExpr>(Sym->getVariableValue()))
+      return Parser.Error(EqualLoc,
+                          "invalid reassignment of non-absolute variable '" +
+                              Name + "'");
+
+    // Don't count these checks as uses.
+    Sym->setUsed(false);
+  } else if (Name == ".") {
+    if (Parser.getStreamer().EmitValueToOffset(Value, 0)) {
+      Parser.Error(EqualLoc, "expected absolute expression");
+      Parser.eatToEndOfStatement();
+      return true;
+    }
+    return false;
+  } else
+    Sym = Parser.getContext().getOrCreateSymbol(Name);
+
+  Sym->setRedefinable(allow_redef);
+
+  return false;
+}
+
+} // namespace MCParserUtils
+} // namespace llvm
+
 /// \brief Create an MCAsmParser instance.
 MCAsmParser *llvm::createMCAsmParser(SourceMgr &SM, MCContext &C,
                                      MCStreamer &Out, const MCAsmInfo &MAI) {
diff --git a/lib/MC/MCParser/CMakeLists.txt b/lib/MC/MCParser/CMakeLists.txt
index 361f6928bd1..99fdd016799 100644
--- a/lib/MC/MCParser/CMakeLists.txt
+++ b/lib/MC/MCParser/CMakeLists.txt
@@ -8,8 +8,7 @@ add_llvm_library(LLVMMCParser
   MCAsmParser.cpp
   MCAsmParserExtension.cpp
   MCTargetAsmParser.cpp
-  NaClAsmParser.cpp
 
   ADDITIONAL_HEADER_DIRS
-  ${LLVM_MAIN_INCLUDE_DIR}/llvm/MCParser
+  ${LLVM_MAIN_INCLUDE_DIR}/llvm/MC/MCParser
   )
diff --git a/lib/MC/MCParser/COFFAsmParser.cpp b/lib/MC/MCParser/COFFAsmParser.cpp
index 18bdb03336a..f09bce005d6 100644
--- a/lib/MC/MCParser/COFFAsmParser.cpp
+++ b/lib/MC/MCParser/COFFAsmParser.cpp
@@ -57,6 +57,7 @@ class COFFAsmParser : public MCAsmParserExtension {
     addDirectiveHandler<&COFFAsmParser::ParseDirectiveEndef>(".endef");
     addDirectiveHandler<&COFFAsmParser::ParseDirectiveSecRel32>(".secrel32");
     addDirectiveHandler<&COFFAsmParser::ParseDirectiveSecIdx>(".secidx");
+    addDirectiveHandler<&COFFAsmParser::ParseDirectiveSafeSEH>(".safeseh");
     addDirectiveHandler<&COFFAsmParser::ParseDirectiveLinkOnce>(".linkonce");
 
     // Win64 EH directives.
@@ -118,6 +119,7 @@ class COFFAsmParser : public MCAsmParserExtension {
   bool ParseDirectiveEndef(StringRef, SMLoc);
   bool ParseDirectiveSecRel32(StringRef, SMLoc);
   bool ParseDirectiveSecIdx(StringRef, SMLoc);
+  bool ParseDirectiveSafeSEH(StringRef, SMLoc);
   bool parseCOMDATType(COFF::COMDATType &Type);
   bool ParseDirectiveLinkOnce(StringRef, SMLoc);
 
@@ -272,7 +274,7 @@ bool COFFAsmParser::ParseDirectiveSymbolAttribute(StringRef Directive, SMLoc) {
       if (getParser().parseIdentifier(Name))
         return TokError("expected identifier in directive");
 
-      MCSymbol *Sym = getContext().GetOrCreateSymbol(Name);
+      MCSymbol *Sym = getContext().getOrCreateSymbol(Name);
 
       getStreamer().EmitSymbolAttribute(Sym, Attr);
 
@@ -359,7 +361,7 @@ bool COFFAsmParser::ParseDirectiveSection(StringRef, SMLoc) {
   COFF::COMDATType Type = (COFF::COMDATType)0;
   StringRef COMDATSymName;
   if (getLexer().is(AsmToken::Comma)) {
-    Type = COFF::IMAGE_COMDAT_SELECT_ANY;;
+    Type = COFF::IMAGE_COMDAT_SELECT_ANY;
     Lex();
 
     Flags |= COFF::IMAGE_SCN_LNK_COMDAT;
@@ -398,7 +400,7 @@ bool COFFAsmParser::ParseDirectiveDef(StringRef, SMLoc) {
   if (getParser().parseIdentifier(SymbolName))
     return TokError("expected identifier in directive");
 
-  MCSymbol *Sym = getContext().GetOrCreateSymbol(SymbolName);
+  MCSymbol *Sym = getContext().getOrCreateSymbol(SymbolName);
 
   getStreamer().BeginCOFFSymbolDef(Sym);
 
@@ -446,13 +448,28 @@ bool COFFAsmParser::ParseDirectiveSecRel32(StringRef, SMLoc) {
   if (getLexer().isNot(AsmToken::EndOfStatement))
     return TokError("unexpected token in directive");
 
-  MCSymbol *Symbol = getContext().GetOrCreateSymbol(SymbolID);
+  MCSymbol *Symbol = getContext().getOrCreateSymbol(SymbolID);
 
   Lex();
   getStreamer().EmitCOFFSecRel32(Symbol);
   return false;
 }
 
+bool COFFAsmParser::ParseDirectiveSafeSEH(StringRef, SMLoc) {
+  StringRef SymbolID;
+  if (getParser().parseIdentifier(SymbolID))
+    return TokError("expected identifier in directive");
+
+  if (getLexer().isNot(AsmToken::EndOfStatement))
+    return TokError("unexpected token in directive");
+
+  MCSymbol *Symbol = getContext().getOrCreateSymbol(SymbolID);
+
+  Lex();
+  getStreamer().EmitCOFFSafeSEH(Symbol);
+  return false;
+}
+
 bool COFFAsmParser::ParseDirectiveSecIdx(StringRef, SMLoc) {
   StringRef SymbolID;
   if (getParser().parseIdentifier(SymbolID))
@@ -461,7 +478,7 @@ bool COFFAsmParser::ParseDirectiveSecIdx(StringRef, SMLoc) {
   if (getLexer().isNot(AsmToken::EndOfStatement))
     return TokError("unexpected token in directive");
 
-  MCSymbol *Symbol = getContext().GetOrCreateSymbol(SymbolID);
+  MCSymbol *Symbol = getContext().getOrCreateSymbol(SymbolID);
 
   Lex();
   getStreamer().EmitCOFFSectionIndex(Symbol);
@@ -524,7 +541,7 @@ bool COFFAsmParser::ParseSEHDirectiveStartProc(StringRef, SMLoc) {
   if (getLexer().isNot(AsmToken::EndOfStatement))
     return TokError("unexpected token in directive");
 
-  MCSymbol *Symbol = getContext().GetOrCreateSymbol(SymbolID);
+  MCSymbol *Symbol = getContext().getOrCreateSymbol(SymbolID);
 
   Lex();
   getStreamer().EmitWinCFIStartProc(Symbol);
@@ -568,7 +585,7 @@ bool COFFAsmParser::ParseSEHDirectiveHandler(StringRef, SMLoc) {
   if (getLexer().isNot(AsmToken::EndOfStatement))
     return TokError("unexpected token in directive");
 
-  MCSymbol *handler = getContext().GetOrCreateSymbol(SymbolID);
+  MCSymbol *handler = getContext().getOrCreateSymbol(SymbolID);
 
   Lex();
   getStreamer().EmitWinEHHandler(handler, unwind, except);
diff --git a/lib/MC/MCParser/DarwinAsmParser.cpp b/lib/MC/MCParser/DarwinAsmParser.cpp
index 9102dc39756..dc664e8a8f6 100644
--- a/lib/MC/MCParser/DarwinAsmParser.cpp
+++ b/lib/MC/MCParser/DarwinAsmParser.cpp
@@ -407,7 +407,7 @@ bool DarwinAsmParser::parseDirectiveDesc(StringRef, SMLoc) {
     return TokError("expected identifier in directive");
 
   // Handle the identifier as the key symbol.
-  MCSymbol *Sym = getContext().GetOrCreateSymbol(Name);
+  MCSymbol *Sym = getContext().getOrCreateSymbol(Name);
 
   if (getLexer().isNot(AsmToken::Comma))
     return TokError("unexpected token in '.desc' directive");
@@ -444,7 +444,7 @@ bool DarwinAsmParser::parseDirectiveIndirectSymbol(StringRef, SMLoc Loc) {
   if (getParser().parseIdentifier(Name))
     return TokError("expected identifier in .indirect_symbol directive");
 
-  MCSymbol *Sym = getContext().GetOrCreateSymbol(Name);
+  MCSymbol *Sym = getContext().getOrCreateSymbol(Name);
 
   // Assembler local symbols don't make any sense here. Complain loudly.
   if (Sym->isTemporary())
@@ -519,7 +519,7 @@ bool DarwinAsmParser::parseDirectiveLsym(StringRef, SMLoc) {
     return TokError("expected identifier in directive");
 
   // Handle the identifier as the key symbol.
-  MCSymbol *Sym = getContext().GetOrCreateSymbol(Name);
+  MCSymbol *Sym = getContext().getOrCreateSymbol(Name);
 
   if (getLexer().isNot(AsmToken::Comma))
     return TokError("unexpected token in '.lsym' directive");
@@ -695,7 +695,7 @@ bool DarwinAsmParser::parseDirectiveTBSS(StringRef, SMLoc) {
     return TokError("expected identifier in directive");
 
   // Handle the identifier as the key symbol.
-  MCSymbol *Sym = getContext().GetOrCreateSymbol(Name);
+  MCSymbol *Sym = getContext().getOrCreateSymbol(Name);
 
   if (getLexer().isNot(AsmToken::Comma))
     return TokError("unexpected token in directive");
@@ -778,7 +778,7 @@ bool DarwinAsmParser::parseDirectiveZerofill(StringRef, SMLoc) {
     return TokError("expected identifier in directive");
 
   // handle the identifier as the key symbol.
-  MCSymbol *Sym = getContext().GetOrCreateSymbol(IDStr);
+  MCSymbol *Sym = getContext().getOrCreateSymbol(IDStr);
 
   if (getLexer().isNot(AsmToken::Comma))
     return TokError("unexpected token in directive");
diff --git a/lib/MC/MCParser/ELFAsmParser.cpp b/lib/MC/MCParser/ELFAsmParser.cpp
index a19339da3ed..5f8a6039afd 100644
--- a/lib/MC/MCParser/ELFAsmParser.cpp
+++ b/lib/MC/MCParser/ELFAsmParser.cpp
@@ -16,7 +16,7 @@
 #include "llvm/MC/MCParser/MCAsmLexer.h"
 #include "llvm/MC/MCSectionELF.h"
 #include "llvm/MC/MCStreamer.h"
-#include "llvm/MC/MCSymbol.h"
+#include "llvm/MC/MCSymbolELF.h"
 #include "llvm/Support/ELF.h"
 using namespace llvm;
 
@@ -174,7 +174,7 @@ bool ELFAsmParser::ParseDirectiveSymbolAttribute(StringRef Directive, SMLoc) {
       if (getParser().parseIdentifier(Name))
         return TokError("expected identifier in directive");
 
-      MCSymbol *Sym = getContext().GetOrCreateSymbol(Name);
+      MCSymbol *Sym = getContext().getOrCreateSymbol(Name);
 
       getStreamer().EmitSymbolAttribute(Sym, Attr);
 
@@ -209,7 +209,7 @@ bool ELFAsmParser::ParseDirectiveSize(StringRef, SMLoc) {
   StringRef Name;
   if (getParser().parseIdentifier(Name))
     return TokError("expected identifier in directive");
-  MCSymbol *Sym = getContext().GetOrCreateSymbol(Name);
+  MCSymbolELF *Sym = cast<MCSymbolELF>(getContext().getOrCreateSymbol(Name));
 
   if (getLexer().isNot(AsmToken::Comma))
     return TokError("unexpected token in directive");
@@ -222,7 +222,7 @@ bool ELFAsmParser::ParseDirectiveSize(StringRef, SMLoc) {
   if (getLexer().isNot(AsmToken::EndOfStatement))
     return TokError("unexpected token in directive");
 
-  getStreamer().EmitELFSize(Sym, Expr);
+  getStreamer().emitELFSize(Sym, Expr);
   return false;
 }
 
@@ -527,21 +527,21 @@ bool ELFAsmParser::ParseSectionArguments(bool IsPush, SMLoc loc) {
       }
   }
 
-  const MCSection *ELFSection = getContext().getELFSection(
-      SectionName, Type, Flags, Size, GroupName, UniqueID);
+  MCSection *ELFSection = getContext().getELFSection(SectionName, Type, Flags,
+                                                     Size, GroupName, UniqueID);
   getStreamer().SwitchSection(ELFSection, Subsection);
 
   if (getContext().getGenDwarfForAssembly()) {
-    auto &Sections = getContext().getGenDwarfSectionSyms();
-    auto InsertResult = Sections.insert(
-        std::make_pair(ELFSection, std::make_pair(nullptr, nullptr)));
-    if (InsertResult.second) {
+    bool InsertResult = getContext().addGenDwarfSection(ELFSection);
+    if (InsertResult) {
       if (getContext().getDwarfVersion() <= 2)
         Warning(loc, "DWARF2 only supports one section per compilation unit");
 
-      MCSymbol *SectionStartSymbol = getContext().CreateTempSymbol();
-      getStreamer().EmitLabel(SectionStartSymbol);
-      InsertResult.first->second.first = SectionStartSymbol;
+      if (!ELFSection->getBeginSymbol()) {
+        MCSymbol *SectionStartSymbol = getContext().createTempSymbol();
+        getStreamer().EmitLabel(SectionStartSymbol);
+        ELFSection->setBeginSymbol(SectionStartSymbol);
+      }
     }
   }
 
@@ -582,7 +582,7 @@ bool ELFAsmParser::ParseDirectiveType(StringRef, SMLoc) {
     return TokError("expected identifier in directive");
 
   // Handle the identifier as the key symbol.
-  MCSymbol *Sym = getContext().GetOrCreateSymbol(Name);
+  MCSymbol *Sym = getContext().getOrCreateSymbol(Name);
 
   // NOTE the comma is optional in all cases.  It is only documented as being
   // optional in the first case, however, GAS will silently treat the comma as
@@ -593,10 +593,16 @@ bool ELFAsmParser::ParseDirectiveType(StringRef, SMLoc) {
     Lex();
 
   if (getLexer().isNot(AsmToken::Identifier) &&
-      getLexer().isNot(AsmToken::Hash) && getLexer().isNot(AsmToken::At) &&
-      getLexer().isNot(AsmToken::Percent) && getLexer().isNot(AsmToken::String))
-    return TokError("expected STT_<TYPE_IN_UPPER_CASE>, '#<type>', '@<type>', "
-                    "'%<type>' or \"<type>\"");
+      getLexer().isNot(AsmToken::Hash) &&
+      getLexer().isNot(AsmToken::Percent) &&
+      getLexer().isNot(AsmToken::String)) {
+    if (!getLexer().getAllowAtInIdentifier())
+      return TokError("expected STT_<TYPE_IN_UPPER_CASE>, '#<type>', "
+                      "'%<type>' or \"<type>\"");
+    else if (getLexer().isNot(AsmToken::At))
+      return TokError("expected STT_<TYPE_IN_UPPER_CASE>, '#<type>', '@<type>', "
+                      "'%<type>' or \"<type>\"");
+  }
 
   if (getLexer().isNot(AsmToken::String) &&
       getLexer().isNot(AsmToken::Identifier))
@@ -661,9 +667,9 @@ bool ELFAsmParser::ParseDirectiveSymver(StringRef, SMLoc) {
   if (AliasName.find('@') == StringRef::npos)
     return TokError("expected a '@' in the name");
 
-  MCSymbol *Alias = getContext().GetOrCreateSymbol(AliasName);
-  MCSymbol *Sym = getContext().GetOrCreateSymbol(Name);
-  const MCExpr *Value = MCSymbolRefExpr::Create(Sym, getContext());
+  MCSymbol *Alias = getContext().getOrCreateSymbol(AliasName);
+  MCSymbol *Sym = getContext().getOrCreateSymbol(Name);
+  const MCExpr *Value = MCSymbolRefExpr::create(Sym, getContext());
 
   getStreamer().EmitAssignment(Alias, Value);
   return false;
@@ -679,7 +685,7 @@ bool ELFAsmParser::ParseDirectiveVersion(StringRef, SMLoc) {
 
   Lex();
 
-  const MCSection *Note = getContext().getELFSection(".note", ELF::SHT_NOTE, 0);
+  MCSection *Note = getContext().getELFSection(".note", ELF::SHT_NOTE, 0);
 
   getStreamer().PushSection();
   getStreamer().SwitchSection(Note);
@@ -711,9 +717,9 @@ bool ELFAsmParser::ParseDirectiveWeakref(StringRef, SMLoc) {
   if (getParser().parseIdentifier(Name))
     return TokError("expected identifier in directive");
 
-  MCSymbol *Alias = getContext().GetOrCreateSymbol(AliasName);
+  MCSymbol *Alias = getContext().getOrCreateSymbol(AliasName);
 
-  MCSymbol *Sym = getContext().GetOrCreateSymbol(Name);
+  MCSymbol *Sym = getContext().getOrCreateSymbol(Name);
 
   getStreamer().EmitWeakReference(Alias, Sym);
   return false;
diff --git a/lib/MC/MCParser/MCTargetAsmParser.cpp b/lib/MC/MCParser/MCTargetAsmParser.cpp
index 60a3a3b59a3..081044968df 100644
--- a/lib/MC/MCParser/MCTargetAsmParser.cpp
+++ b/lib/MC/MCParser/MCTargetAsmParser.cpp
@@ -10,8 +10,8 @@
 #include "llvm/MC/MCTargetAsmParser.h"
 using namespace llvm;
 
-MCTargetAsmParser::MCTargetAsmParser()
-  : AvailableFeatures(0), ParsingInlineAsm(false)
+MCTargetAsmParser::MCTargetAsmParser(MCTargetOptions const &MCOptions)
+  : AvailableFeatures(0), ParsingInlineAsm(false), MCOptions(MCOptions)
 {
 }
 
diff --git a/lib/MC/MCParser/NaClAsmParser.cpp b/lib/MC/MCParser/NaClAsmParser.cpp
deleted file mode 100644
index fa06aa24676..00000000000
--- a/lib/MC/MCParser/NaClAsmParser.cpp
+++ /dev/null
@@ -1,80 +0,0 @@
-//===- NaClAsmParser.cpp - NaCl Assembly Parser ---------------------------===//
-//
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
-//
-//===----------------------------------------------------------------------===//
-
-#include "llvm/MC/MCParser/MCAsmParserExtension.h"
-#include "llvm/MC/MCNaClExpander.h"
-#include "llvm/MC/MCStreamer.h"
-#include "llvm/MC/MCTargetAsmParser.h"
-#include "llvm/Support/SourceMgr.h"
-#include "llvm/Support/raw_ostream.h"
-
-using namespace llvm;
-
-class NaClAsmParser : public MCAsmParserExtension {
-  MCNaClExpander *Expander;
-  template<bool (NaClAsmParser::*HandlerMethod)(StringRef, SMLoc)>
-  void addDirectiveHandler(StringRef Directive) {
-    MCAsmParser::ExtensionDirectiveHandler Handler = std::make_pair(
-        this, HandleDirective<NaClAsmParser, HandlerMethod>);
-
-    getParser().addDirectiveHandler(Directive, Handler);
-  }
-
- public:
-  NaClAsmParser(MCNaClExpander *Exp) : Expander(Exp) {}
-  void Initialize(MCAsmParser &Parser) override {
-    // Call the base implementation.
-    MCAsmParserExtension::Initialize(Parser);
-    addDirectiveHandler<&NaClAsmParser::ParseScratch>(".scratch");
-    addDirectiveHandler<&NaClAsmParser::ParseUnscratch>(".unscratch");
-  }
-
-  /// ::= {.scratch} reg
-  bool ParseScratch(StringRef Directive, SMLoc Loc) {
-    getParser().checkForValidSection();
-    unsigned RegNo;
-    const char *kInvalidOptionError =
-        "expected register name after '.scratch' directive";
-
-    if (getLexer().isNot(AsmToken::EndOfStatement)) {
-      if (getParser().getTargetParser().ParseRegister(RegNo, Loc, Loc))
-        return Error(Loc, kInvalidOptionError);
-
-      else if (getLexer().isNot(AsmToken::EndOfStatement))
-        return Error(Loc, kInvalidOptionError);
-    }
-    else {
-      return Error(Loc, kInvalidOptionError);
-    }
-    Lex();
-
-    Expander->pushScratchReg(RegNo);
-    return false;
-  }
-
-  /// ::= {.unscratch}
-  bool ParseUnscratch(StringRef Directive, SMLoc Loc) {
-    getParser().checkForValidSection();
-    if (getLexer().isNot(AsmToken::EndOfStatement))
-      return TokError("unexpected token in '.unscratch' directive");
-    Lex();
-
-    if (Expander->numScratchRegs() == 0)
-      return Error(Loc, "No scratch registers specified");
-    Expander->popScratchReg();
-
-    return false;
-  }
-};
-
-namespace llvm {
-MCAsmParserExtension *createNaClAsmParser(MCNaClExpander *Exp) {
-  return new NaClAsmParser(Exp);
-}
-}
diff --git a/lib/MC/MCSchedule.cpp b/lib/MC/MCSchedule.cpp
new file mode 100644
index 00000000000..f3919427bf0
--- /dev/null
+++ b/lib/MC/MCSchedule.cpp
@@ -0,0 +1,34 @@
+//===- MCSchedule.cpp - Scheduling ------------------------------*- C++ -*-===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This file defines the default scheduling model.
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/MC/MCSchedule.h"
+#include <type_traits>
+
+using namespace llvm;
+
+static_assert(std::is_pod<MCSchedModel>::value,
+              "We shouldn't have a static constructor here");
+const MCSchedModel MCSchedModel::Default = {DefaultIssueWidth,
+                                            DefaultMicroOpBufferSize,
+                                            DefaultLoopMicroOpBufferSize,
+                                            DefaultLoadLatency,
+                                            DefaultHighLatency,
+                                            DefaultMispredictPenalty,
+                                            false,
+                                            true,
+                                            0,
+                                            nullptr,
+                                            nullptr,
+                                            0,
+                                            0,
+                                            nullptr};
diff --git a/lib/MC/MCSection.cpp b/lib/MC/MCSection.cpp
index 7889f837b11..9152f2b42a4 100644
--- a/lib/MC/MCSection.cpp
+++ b/lib/MC/MCSection.cpp
@@ -8,6 +8,7 @@
 //===----------------------------------------------------------------------===//
 
 #include "llvm/MC/MCSection.h"
+#include "llvm/MC/MCAssembler.h"
 #include "llvm/MC/MCAsmInfo.h"
 #include "llvm/MC/MCContext.h"
 #include "llvm/MC/MCSymbol.h"
@@ -18,7 +19,11 @@ using namespace llvm;
 // MCSection
 //===----------------------------------------------------------------------===//
 
-MCSymbol *MCSection::getEndSymbol(MCContext &Ctx) const {
+MCSection::MCSection(SectionVariant V, SectionKind K, MCSymbol *Begin)
+    : Begin(Begin), BundleGroupBeforeFirstInst(false), HasInstructions(false),
+      IsRegistered(false), Variant(V), Kind(K) {}
+
+MCSymbol *MCSection::getEndSymbol(MCContext &Ctx) {
   if (!End)
     End = Ctx.createTempSymbol("sec_end", true);
   return End;
@@ -29,3 +34,76 @@ bool MCSection::hasEnded() const { return End && End->isInSection(); }
 MCSection::~MCSection() {
 }
 
+void MCSection::setBundleLockState(BundleLockStateType NewState) {
+  if (NewState == NotBundleLocked) {
+    if (BundleLockNestingDepth == 0) {
+      report_fatal_error("Mismatched bundle_lock/unlock directives");
+    }
+    if (--BundleLockNestingDepth == 0) {
+      BundleLockState = NotBundleLocked;
+    }
+    return;
+  }
+
+  // If any of the directives is an align_to_end directive, the whole nested
+  // group is align_to_end. So don't downgrade from align_to_end to just locked.
+  if (BundleLockState != BundleLockedAlignToEnd) {
+    BundleLockState = NewState;
+  }
+  ++BundleLockNestingDepth;
+}
+
+MCSection::iterator
+MCSection::getSubsectionInsertionPoint(unsigned Subsection) {
+  if (Subsection == 0 && SubsectionFragmentMap.empty())
+    return end();
+
+  SmallVectorImpl<std::pair<unsigned, MCFragment *>>::iterator MI =
+      std::lower_bound(SubsectionFragmentMap.begin(),
+                       SubsectionFragmentMap.end(),
+                       std::make_pair(Subsection, (MCFragment *)nullptr));
+  bool ExactMatch = false;
+  if (MI != SubsectionFragmentMap.end()) {
+    ExactMatch = MI->first == Subsection;
+    if (ExactMatch)
+      ++MI;
+  }
+  iterator IP;
+  if (MI == SubsectionFragmentMap.end())
+    IP = end();
+  else
+    IP = MI->second;
+  if (!ExactMatch && Subsection != 0) {
+    // The GNU as documentation claims that subsections have an alignment of 4,
+    // although this appears not to be the case.
+    MCFragment *F = new MCDataFragment();
+    SubsectionFragmentMap.insert(MI, std::make_pair(Subsection, F));
+    getFragmentList().insert(IP, F);
+    F->setParent(this);
+  }
+
+  return IP;
+}
+
+#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
+void MCSection::dump() {
+  raw_ostream &OS = llvm::errs();
+
+  OS << "<MCSection";
+  OS << " Fragments:[\n      ";
+  for (auto it = begin(), ie = end(); it != ie; ++it) {
+    if (it != begin())
+      OS << ",\n      ";
+    it->dump();
+  }
+  OS << "]>";
+}
+#endif
+
+MCSection::iterator MCSection::begin() { return Fragments.begin(); }
+
+MCSection::iterator MCSection::end() { return Fragments.end(); }
+
+MCSection::reverse_iterator MCSection::rbegin() { return Fragments.rbegin(); }
+
+MCSection::reverse_iterator MCSection::rend() { return Fragments.rend(); }
diff --git a/lib/MC/MCSectionCOFF.cpp b/lib/MC/MCSectionCOFF.cpp
index 4d6298c542e..ce0b4f5fb41 100644
--- a/lib/MC/MCSectionCOFF.cpp
+++ b/lib/MC/MCSectionCOFF.cpp
@@ -94,7 +94,7 @@ void MCSectionCOFF::PrintSwitchToSection(const MCAsmInfo &MAI,
         break;
     }
     assert(COMDATSymbol);
-    OS << *COMDATSymbol;
+    COMDATSymbol->print(OS, &MAI);
   }
   OS << '\n';
 }
diff --git a/lib/MC/MCSectionELF.cpp b/lib/MC/MCSectionELF.cpp
index 3cd84538425..b4448d79a2d 100644
--- a/lib/MC/MCSectionELF.cpp
+++ b/lib/MC/MCSectionELF.cpp
@@ -64,8 +64,10 @@ void MCSectionELF::PrintSwitchToSection(const MCAsmInfo &MAI,
 
   if (ShouldOmitSectionDirective(SectionName, MAI)) {
     OS << '\t' << getSectionName();
-    if (Subsection)
-      OS << '\t' << *Subsection;
+    if (Subsection) {
+      OS << '\t';
+      Subsection->print(OS, &MAI);
+    }
     OS << '\n';
     return;
   }
@@ -153,8 +155,11 @@ void MCSectionELF::PrintSwitchToSection(const MCAsmInfo &MAI,
 
   OS << '\n';
 
-  if (Subsection)
-    OS << "\t.subsection\t" << *Subsection << '\n';
+  if (Subsection) {
+    OS << "\t.subsection\t";
+    Subsection->print(OS, &MAI);
+    OS << '\n';
+  }
 }
 
 bool MCSectionELF::UseCodeAlign() const {
diff --git a/lib/MC/MCStreamer.cpp b/lib/MC/MCStreamer.cpp
index 27d0355bb11..a25cdfa971e 100644
--- a/lib/MC/MCStreamer.cpp
+++ b/lib/MC/MCStreamer.cpp
@@ -14,6 +14,8 @@
 #include "llvm/MC/MCAsmInfo.h"
 #include "llvm/MC/MCContext.h"
 #include "llvm/MC/MCExpr.h"
+#include "llvm/MC/MCInst.h"
+#include "llvm/MC/MCInstPrinter.h"
 #include "llvm/MC/MCObjectFileInfo.h"
 #include "llvm/MC/MCObjectWriter.h"
 #include "llvm/MC/MCSection.h"
@@ -116,7 +118,7 @@ void MCStreamer::EmitSymbolValue(const MCSymbol *Sym, unsigned Size,
          "SectionRelative value requires 4-bytes");
 
   if (!IsSectionRelative)
-    EmitValueImpl(MCSymbolRefExpr::Create(Sym, getContext()), Size);
+    EmitValueImpl(MCSymbolRefExpr::create(Sym, getContext()), Size);
   else
     EmitCOFFSecRel32(Sym);
 }
@@ -132,7 +134,7 @@ void MCStreamer::EmitGPRel32Value(const MCExpr *Value) {
 /// EmitFill - Emit NumBytes bytes worth of the value specified by
 /// FillValue.  This implements directives such as '.space'.
 void MCStreamer::EmitFill(uint64_t NumBytes, uint8_t FillValue) {
-  const MCExpr *E = MCConstantExpr::Create(FillValue, getContext());
+  const MCExpr *E = MCConstantExpr::create(FillValue, getContext());
   for (uint64_t i = 0, e = NumBytes; i != e; ++i)
     EmitValue(E, 1);
 }
@@ -145,7 +147,7 @@ void MCStreamer::EmitZeros(uint64_t NumBytes) {
 unsigned MCStreamer::EmitDwarfFileDirective(unsigned FileNo,
                                             StringRef Directory,
                                             StringRef Filename, unsigned CUID) {
-  return getContext().GetDwarfFile(Directory, Filename, FileNo, CUID);
+  return getContext().getDwarfFile(Directory, Filename, FileNo, CUID);
 }
 
 void MCStreamer::EmitDwarfLocDirective(unsigned FileNo, unsigned Line,
@@ -162,7 +164,7 @@ MCSymbol *MCStreamer::getDwarfLineTableSymbol(unsigned CUID) {
   if (!Table.getLabel()) {
     StringRef Prefix = Context.getAsmInfo()->getPrivateGlobalPrefix();
     Table.setLabel(
-        Context.GetOrCreateSymbol(Prefix + "line_table_start" + Twine(CUID)));
+        Context.getOrCreateSymbol(Prefix + "line_table_start" + Twine(CUID)));
   }
   return Table.getLabel();
 }
@@ -187,7 +189,7 @@ void MCStreamer::InitSections(bool NoExecStack) {
   SwitchSection(getContext().getObjectFileInfo()->getTextSection());
 }
 
-void MCStreamer::AssignSection(MCSymbol *Symbol, const MCSection *Section) {
+void MCStreamer::AssignSection(MCSymbol *Symbol, MCSection *Section) {
   if (Section)
     Symbol->setSection(*Section);
   else
@@ -251,7 +253,7 @@ void MCStreamer::EmitCFIEndProcImpl(MCDwarfFrameInfo &Frame) {
 
 MCSymbol *MCStreamer::EmitCFICommon() {
   EnsureValidDwarfFrame();
-  MCSymbol *Label = getContext().CreateTempSymbol();
+  MCSymbol *Label = getContext().createTempSymbol();
   EmitLabel(Label);
   return Label;
 }
@@ -390,15 +392,21 @@ void MCStreamer::EmitCFIWindowSave() {
 }
 
 void MCStreamer::EnsureValidWinFrameInfo() {
+  const MCAsmInfo *MAI = Context.getAsmInfo();
+  if (!MAI->usesWindowsCFI())
+    report_fatal_error(".seh_* directives are not supported on this target");
   if (!CurrentWinFrameInfo || CurrentWinFrameInfo->End)
     report_fatal_error("No open Win64 EH frame function!");
 }
 
 void MCStreamer::EmitWinCFIStartProc(const MCSymbol *Symbol) {
+  const MCAsmInfo *MAI = Context.getAsmInfo();
+  if (!MAI->usesWindowsCFI())
+    report_fatal_error(".seh_* directives are not supported on this target");
   if (CurrentWinFrameInfo && !CurrentWinFrameInfo->End)
     report_fatal_error("Starting a function before ending the previous one!");
 
-  MCSymbol *StartProc = getContext().CreateTempSymbol();
+  MCSymbol *StartProc = getContext().createTempSymbol();
   EmitLabel(StartProc);
 
   WinFrameInfos.push_back(new WinEH::FrameInfo(Symbol, StartProc));
@@ -410,7 +418,7 @@ void MCStreamer::EmitWinCFIEndProc() {
   if (CurrentWinFrameInfo->ChainedParent)
     report_fatal_error("Not all chained regions terminated!");
 
-  MCSymbol *Label = getContext().CreateTempSymbol();
+  MCSymbol *Label = getContext().createTempSymbol();
   EmitLabel(Label);
   CurrentWinFrameInfo->End = Label;
 }
@@ -418,7 +426,7 @@ void MCStreamer::EmitWinCFIEndProc() {
 void MCStreamer::EmitWinCFIStartChained() {
   EnsureValidWinFrameInfo();
 
-  MCSymbol *StartProc = getContext().CreateTempSymbol();
+  MCSymbol *StartProc = getContext().createTempSymbol();
   EmitLabel(StartProc);
 
   WinFrameInfos.push_back(new WinEH::FrameInfo(CurrentWinFrameInfo->Function,
@@ -431,7 +439,7 @@ void MCStreamer::EmitWinCFIEndChained() {
   if (!CurrentWinFrameInfo->ChainedParent)
     report_fatal_error("End of a chained region outside a chained region!");
 
-  MCSymbol *Label = getContext().CreateTempSymbol();
+  MCSymbol *Label = getContext().createTempSymbol();
   EmitLabel(Label);
 
   CurrentWinFrameInfo->End = Label;
@@ -459,10 +467,12 @@ void MCStreamer::EmitWinEHHandlerData() {
     report_fatal_error("Chained unwind areas can't have handlers!");
 }
 
+void MCStreamer::EmitSyntaxDirective() {}
+
 void MCStreamer::EmitWinCFIPushReg(unsigned Register) {
   EnsureValidWinFrameInfo();
 
-  MCSymbol *Label = getContext().CreateTempSymbol();
+  MCSymbol *Label = getContext().createTempSymbol();
   EmitLabel(Label);
 
   WinEH::Instruction Inst = Win64EH::Instruction::PushNonVol(Label, Register);
@@ -478,7 +488,7 @@ void MCStreamer::EmitWinCFISetFrame(unsigned Register, unsigned Offset) {
   if (Offset > 240)
     report_fatal_error("Frame offset must be less than or equal to 240!");
 
-  MCSymbol *Label = getContext().CreateTempSymbol();
+  MCSymbol *Label = getContext().createTempSymbol();
   EmitLabel(Label);
 
   WinEH::Instruction Inst =
@@ -494,7 +504,7 @@ void MCStreamer::EmitWinCFIAllocStack(unsigned Size) {
   if (Size & 7)
     report_fatal_error("Misaligned stack allocation!");
 
-  MCSymbol *Label = getContext().CreateTempSymbol();
+  MCSymbol *Label = getContext().createTempSymbol();
   EmitLabel(Label);
 
   WinEH::Instruction Inst = Win64EH::Instruction::Alloc(Label, Size);
@@ -506,7 +516,7 @@ void MCStreamer::EmitWinCFISaveReg(unsigned Register, unsigned Offset) {
   if (Offset & 7)
     report_fatal_error("Misaligned saved register offset!");
 
-  MCSymbol *Label = getContext().CreateTempSymbol();
+  MCSymbol *Label = getContext().createTempSymbol();
   EmitLabel(Label);
 
   WinEH::Instruction Inst =
@@ -519,7 +529,7 @@ void MCStreamer::EmitWinCFISaveXMM(unsigned Register, unsigned Offset) {
   if (Offset & 0x0F)
     report_fatal_error("Misaligned saved vector register offset!");
 
-  MCSymbol *Label = getContext().CreateTempSymbol();
+  MCSymbol *Label = getContext().createTempSymbol();
   EmitLabel(Label);
 
   WinEH::Instruction Inst =
@@ -532,7 +542,7 @@ void MCStreamer::EmitWinCFIPushFrame(bool Code) {
   if (CurrentWinFrameInfo->Instructions.size() > 0)
     report_fatal_error("If present, PushMachFrame must be the first UOP");
 
-  MCSymbol *Label = getContext().CreateTempSymbol();
+  MCSymbol *Label = getContext().createTempSymbol();
   EmitLabel(Label);
 
   WinEH::Instruction Inst = Win64EH::Instruction::PushMachFrame(Label, Code);
@@ -542,12 +552,15 @@ void MCStreamer::EmitWinCFIPushFrame(bool Code) {
 void MCStreamer::EmitWinCFIEndProlog() {
   EnsureValidWinFrameInfo();
 
-  MCSymbol *Label = getContext().CreateTempSymbol();
+  MCSymbol *Label = getContext().createTempSymbol();
   EmitLabel(Label);
 
   CurrentWinFrameInfo->PrologEnd = Label;
 }
 
+void MCStreamer::EmitCOFFSafeSEH(MCSymbol const *Symbol) {
+}
+
 void MCStreamer::EmitCOFFSectionIndex(MCSymbol const *Symbol) {
 }
 
@@ -591,6 +604,11 @@ void MCStreamer::EmitAssignment(MCSymbol *Symbol, const MCExpr *Value) {
     TS->emitAssignment(Symbol, Value);
 }
 
+void MCTargetStreamer::prettyPrintAsm(MCInstPrinter &InstPrinter, raw_ostream &OS,
+                              const MCInst &Inst, const MCSubtargetInfo &STI) {
+  InstPrinter.printInst(&Inst, OS, "", STI);
+}
+
 void MCStreamer::visitUsedSymbol(const MCSymbol &Sym) {
 }
 
@@ -628,6 +646,25 @@ void MCStreamer::EmitInstruction(const MCInst &Inst,
       visitUsedExpr(*Inst.getOperand(i).getExpr());
 }
 
+void MCStreamer::emitAbsoluteSymbolDiff(const MCSymbol *Hi, const MCSymbol *Lo,
+                                        unsigned Size) {
+  // Get the Hi-Lo expression.
+  const MCExpr *Diff =
+      MCBinaryExpr::createSub(MCSymbolRefExpr::create(Hi, Context),
+                              MCSymbolRefExpr::create(Lo, Context), Context);
+
+  const MCAsmInfo *MAI = Context.getAsmInfo();
+  if (!MAI->doesSetDirectiveSuppressesReloc()) {
+    EmitValue(Diff, Size);
+    return;
+  }
+
+  // Otherwise, emit with .set (aka assignment).
+  MCSymbol *SetLabel = Context.createTempSymbol("set", true);
+  EmitAssignment(SetLabel, Diff);
+  EmitSymbolValue(SetLabel, Size);
+}
+
 void MCStreamer::EmitAssemblerFlag(MCAssemblerFlag Flag) {}
 void MCStreamer::EmitThumbFunc(MCSymbol *Func) {}
 void MCStreamer::EmitSymbolDesc(MCSymbol *Symbol, unsigned DescValue) {}
@@ -636,12 +673,12 @@ void MCStreamer::EndCOFFSymbolDef() {}
 void MCStreamer::EmitFileDirective(StringRef Filename) {}
 void MCStreamer::EmitCOFFSymbolStorageClass(int StorageClass) {}
 void MCStreamer::EmitCOFFSymbolType(int Type) {}
-void MCStreamer::EmitELFSize(MCSymbol *Symbol, const MCExpr *Value) {}
+void MCStreamer::emitELFSize(MCSymbolELF *Symbol, const MCExpr *Value) {}
 void MCStreamer::EmitLocalCommonSymbol(MCSymbol *Symbol, uint64_t Size,
                                        unsigned ByteAlignment) {}
-void MCStreamer::EmitTBSSSymbol(const MCSection *Section, MCSymbol *Symbol,
+void MCStreamer::EmitTBSSSymbol(MCSection *Section, MCSymbol *Symbol,
                                 uint64_t Size, unsigned ByteAlignment) {}
-void MCStreamer::ChangeSection(const MCSection *, const MCExpr *) {}
+void MCStreamer::ChangeSection(MCSection *, const MCExpr *) {}
 void MCStreamer::EmitWeakReference(MCSymbol *Alias, const MCSymbol *Symbol) {}
 void MCStreamer::EmitBytes(StringRef Data) {}
 void MCStreamer::EmitValueImpl(const MCExpr *Value, unsigned Size,
@@ -663,22 +700,21 @@ void MCStreamer::EmitBundleLock(bool AlignToEnd) {}
 void MCStreamer::FinishImpl() {}
 void MCStreamer::EmitBundleUnlock() {}
 
-void MCStreamer::SwitchSection(const MCSection *Section,
-                               const MCExpr *Subsection) {
+void MCStreamer::SwitchSection(MCSection *Section, const MCExpr *Subsection) {
   assert(Section && "Cannot switch to a null section!");
   MCSectionSubPair curSection = SectionStack.back().first;
   SectionStack.back().second = curSection;
   if (MCSectionSubPair(Section, Subsection) != curSection) {
+    ChangeSection(Section, Subsection);
     SectionStack.back().first = MCSectionSubPair(Section, Subsection);
     assert(!Section->hasEnded() && "Section already ended");
-    ChangeSection(Section, Subsection);
     MCSymbol *Sym = Section->getBeginSymbol();
     if (Sym && !Sym->isInSection())
       EmitLabel(Sym);
   }
 }
 
-MCSymbol *MCStreamer::endSection(const MCSection *Section) {
+MCSymbol *MCStreamer::endSection(MCSection *Section) {
   // TODO: keep track of the last subsection so that this symbol appears in the
   // correct place.
   MCSymbol *Sym = Section->getEndSymbol(Context);
diff --git a/lib/MC/MCSubtargetInfo.cpp b/lib/MC/MCSubtargetInfo.cpp
index daba321c58e..9210cf544b1 100644
--- a/lib/MC/MCSubtargetInfo.cpp
+++ b/lib/MC/MCSubtargetInfo.cpp
@@ -17,68 +17,64 @@
 
 using namespace llvm;
 
-/// InitMCProcessorInfo - Set or change the CPU (optionally supplemented
-/// with feature string). Recompute feature bits and scheduling model.
-void
-MCSubtargetInfo::InitMCProcessorInfo(StringRef CPU, StringRef FS) {
+static FeatureBitset getFeatures(StringRef CPU, StringRef FS,
+                                 ArrayRef<SubtargetFeatureKV> ProcDesc,
+                                 ArrayRef<SubtargetFeatureKV> ProcFeatures) {
   SubtargetFeatures Features(FS);
-  FeatureBits = Features.getFeatureBits(CPU, ProcDesc, ProcFeatures);
-  InitCPUSchedModel(CPU);
+  return Features.getFeatureBits(CPU, ProcDesc, ProcFeatures);
 }
 
-void
-MCSubtargetInfo::InitCPUSchedModel(StringRef CPU) {
+void MCSubtargetInfo::InitMCProcessorInfo(StringRef CPU, StringRef FS) {
+  FeatureBits = getFeatures(CPU, FS, ProcDesc, ProcFeatures);
   if (!CPU.empty())
-    CPUSchedModel = getSchedModelForCPU(CPU);
+    CPUSchedModel = &getSchedModelForCPU(CPU);
   else
-    CPUSchedModel = MCSchedModel::GetDefaultSchedModel();
+    CPUSchedModel = &MCSchedModel::GetDefaultSchedModel();
 }
 
-void
-MCSubtargetInfo::InitMCSubtargetInfo(StringRef TT, StringRef C, StringRef FS,
-                                     ArrayRef<SubtargetFeatureKV> PF,
-                                     ArrayRef<SubtargetFeatureKV> PD,
-                                     const SubtargetInfoKV *ProcSched,
-                                     const MCWriteProcResEntry *WPR,
-                                     const MCWriteLatencyEntry *WL,
-                                     const MCReadAdvanceEntry *RA,
-                                     const InstrStage *IS,
-                                     const unsigned *OC,
-                                     const unsigned *FP) {
-  TargetTriple = TT;
-  CPU = C;
-  ProcFeatures = PF;
-  ProcDesc = PD;
-  ProcSchedModels = ProcSched;
-  WriteProcResTable = WPR;
-  WriteLatencyTable = WL;
-  ReadAdvanceTable = RA;
-
-  Stages = IS;
-  OperandCycles = OC;
-  ForwardingPaths = FP;
+void MCSubtargetInfo::setDefaultFeatures(StringRef CPU) {
+  FeatureBits = getFeatures(CPU, "", ProcDesc, ProcFeatures);
+}
 
+MCSubtargetInfo::MCSubtargetInfo(
+    const Triple &TT, StringRef C, StringRef FS,
+    ArrayRef<SubtargetFeatureKV> PF, ArrayRef<SubtargetFeatureKV> PD,
+    const SubtargetInfoKV *ProcSched, const MCWriteProcResEntry *WPR,
+    const MCWriteLatencyEntry *WL, const MCReadAdvanceEntry *RA,
+    const InstrStage *IS, const unsigned *OC, const unsigned *FP)
+    : TargetTriple(TT), CPU(C), ProcFeatures(PF), ProcDesc(PD),
+      ProcSchedModels(ProcSched), WriteProcResTable(WPR), WriteLatencyTable(WL),
+      ReadAdvanceTable(RA), Stages(IS), OperandCycles(OC), ForwardingPaths(FP) {
   InitMCProcessorInfo(CPU, FS);
 }
 
 /// ToggleFeature - Toggle a feature and returns the re-computed feature
 /// bits. This version does not change the implied bits.
-uint64_t MCSubtargetInfo::ToggleFeature(uint64_t FB) {
+FeatureBitset MCSubtargetInfo::ToggleFeature(uint64_t FB) {
+  FeatureBits.flip(FB);
+  return FeatureBits;
+}
+
+FeatureBitset MCSubtargetInfo::ToggleFeature(const FeatureBitset &FB) {
   FeatureBits ^= FB;
   return FeatureBits;
 }
 
 /// ToggleFeature - Toggle a feature and returns the re-computed feature
 /// bits. This version will also change all implied bits.
-uint64_t MCSubtargetInfo::ToggleFeature(StringRef FS) {
+FeatureBitset MCSubtargetInfo::ToggleFeature(StringRef FS) {
   SubtargetFeatures Features;
   FeatureBits = Features.ToggleFeature(FeatureBits, FS, ProcFeatures);
   return FeatureBits;
 }
 
+FeatureBitset MCSubtargetInfo::ApplyFeatureFlag(StringRef FS) {
+  SubtargetFeatures Features;
+  FeatureBits = Features.ApplyFeatureFlag(FeatureBits, FS, ProcFeatures);
+  return FeatureBits;
+}
 
-MCSchedModel
-MCSubtargetInfo::getSchedModelForCPU(StringRef CPU) const {
+const MCSchedModel &MCSubtargetInfo::getSchedModelForCPU(StringRef CPU) const {
   assert(ProcSchedModels && "Processor machine model not available!");
 
   unsigned NumProcs = ProcDesc.size();
@@ -111,6 +107,6 @@ MCSubtargetInfo::getInstrItineraryForCPU(StringRef CPU) const {
 
 /// Initialize an InstrItineraryData instance.
 void MCSubtargetInfo::initInstrItins(InstrItineraryData &InstrItins) const {
-  InstrItins =
-    InstrItineraryData(CPUSchedModel, Stages, OperandCycles, ForwardingPaths);
+  InstrItins = InstrItineraryData(getSchedModel(), Stages, OperandCycles,
+                                  ForwardingPaths);
 }
diff --git a/lib/MC/MCSymbol.cpp b/lib/MC/MCSymbol.cpp
index 6582574ae94..125380a9d14 100644
--- a/lib/MC/MCSymbol.cpp
+++ b/lib/MC/MCSymbol.cpp
@@ -8,69 +8,61 @@
 //===----------------------------------------------------------------------===//
 
 #include "llvm/MC/MCSymbol.h"
+#include "llvm/MC/MCAsmInfo.h"
+#include "llvm/MC/MCContext.h"
 #include "llvm/MC/MCExpr.h"
 #include "llvm/Support/Debug.h"
+#include "llvm/Support/ErrorHandling.h"
 #include "llvm/Support/raw_ostream.h"
 using namespace llvm;
 
 // Sentinel value for the absolute pseudo section.
-const MCSection *MCSymbol::AbsolutePseudoSection =
-  reinterpret_cast<const MCSection *>(1);
+MCSection *MCSymbol::AbsolutePseudoSection = reinterpret_cast<MCSection *>(1);
 
-static bool isAcceptableChar(char C) {
-  if ((C < 'a' || C > 'z') &&
-      (C < 'A' || C > 'Z') &&
-      (C < '0' || C > '9') &&
-      C != '_' && C != '$' && C != '.' && C != '@')
-    return false;
-  return true;
-}
-
-/// NameNeedsQuoting - Return true if the identifier \p Str needs quotes to be
-/// syntactically correct.
-static bool NameNeedsQuoting(StringRef Str) {
-  assert(!Str.empty() && "Cannot create an empty MCSymbol");
+void *MCSymbol::operator new(size_t s, const StringMapEntry<bool> *Name,
+                             MCContext &Ctx) {
+  // We may need more space for a Name to account for alignment.  So allocate
+  // space for the storage type and not the name pointer.
+  size_t Size = s + (Name ? sizeof(NameEntryStorageTy) : 0);
 
-  // If any of the characters in the string is an unacceptable character, force
-  // quotes.
-  for (unsigned i = 0, e = Str.size(); i != e; ++i)
-    if (!isAcceptableChar(Str[i]))
-      return true;
-  return false;
-}
-
-const MCSymbol &MCSymbol::AliasedSymbol() const {
-  const MCSymbol *S = this;
-  while (S->isVariable()) {
-    const MCExpr *Value = S->getVariableValue();
-    if (Value->getKind() != MCExpr::SymbolRef)
-      return *S;
-    const MCSymbolRefExpr *Ref = static_cast<const MCSymbolRefExpr*>(Value);
-    S = &Ref->getSymbol();
-  }
-  return *S;
+  // For safety, ensure that the alignment of a pointer is enough for an
+  // MCSymbol.  This also ensures we don't need padding between the name and
+  // symbol.
+  static_assert((unsigned)AlignOf<MCSymbol>::Alignment <=
+                AlignOf<NameEntryStorageTy>::Alignment,
+                "Bad alignment of MCSymbol");
+  void *Storage = Ctx.allocate(Size, alignOf<NameEntryStorageTy>());
+  NameEntryStorageTy *Start = static_cast<NameEntryStorageTy*>(Storage);
+  NameEntryStorageTy *End = Start + (Name ? 1 : 0);
+  return End;
 }
 
 void MCSymbol::setVariableValue(const MCExpr *Value) {
   assert(!IsUsed && "Cannot set a variable that has already been used.");
   assert(Value && "Invalid variable value!");
+  assert((SymbolContents == SymContentsUnset ||
+          SymbolContents == SymContentsVariable) &&
+         "Cannot give common/offset symbol a variable value");
   this->Value = Value;
-  this->Section = nullptr;
+  SymbolContents = SymContentsVariable;
+  setUndefined();
 }
 
-void MCSymbol::print(raw_ostream &OS) const {
+void MCSymbol::print(raw_ostream &OS, const MCAsmInfo *MAI) const {
   // The name for this MCSymbol is required to be a valid target name.  However,
   // some targets support quoting names with funny characters.  If the name
   // contains a funny character, then print it quoted.
   StringRef Name = getName();
-  if (!NameNeedsQuoting(Name)) {
+  if (!MAI || MAI->isValidUnquotedName(Name)) {
     OS << Name;
     return;
   }
 
+  if (MAI && !MAI->supportsNameQuoting())
+    report_fatal_error("Symbol name with unsupported characters");
+
   OS << '"';
-  for (unsigned I = 0, E = Name.size(); I != E; ++I) {
-    char C = Name[I];
+  for (char C : Name) {
     if (C == '\n')
       OS << "\\n";
     else if (C == '"')
@@ -82,7 +74,5 @@ void MCSymbol::print(raw_ostream &OS) const {
 }
 
 #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
-void MCSymbol::dump() const {
-  print(dbgs());
-}
+void MCSymbol::dump() const { dbgs() << *this; }
 #endif
diff --git a/lib/MC/MCSymbolELF.cpp b/lib/MC/MCSymbolELF.cpp
new file mode 100644
index 00000000000..ec7ef447ff8
--- /dev/null
+++ b/lib/MC/MCSymbolELF.cpp
@@ -0,0 +1,201 @@
+//===- lib/MC/MCSymbolELF.cpp ---------------------------------------------===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/MC/MCAssembler.h"
+#include "llvm/MC/MCSymbolELF.h"
+#include "llvm/MC/MCFixupKindInfo.h"
+#include "llvm/Support/ELF.h"
+
+namespace llvm {
+
+namespace {
+enum {
+  // Shift value for STT_* flags. 7 possible values. 3 bits.
+  ELF_STT_Shift = 0,
+
+  // Shift value for STB_* flags. 4 possible values, 2 bits.
+  ELF_STB_Shift = 3,
+
+  // Shift value for STV_* flags. 4 possible values, 2 bits.
+  ELF_STV_Shift = 5,
+
+  // Shift value for STO_* flags. 3 bits. All the values are between 0x20 and
+  // 0xe0, so we shift right by 5 before storing.
+  ELF_STO_Shift = 7,
+
+  // One bit.
+  ELF_IsSignature_Shift = 10,
+
+  // One bit.
+  ELF_WeakrefUsedInReloc_Shift = 11,
+
+  // One bit.
+  ELF_BindingSet_Shift = 12
+};
+}
+
+void MCSymbolELF::setBinding(unsigned Binding) const {
+  setIsBindingSet();
+  unsigned Val;
+  switch (Binding) {
+  default:
+    llvm_unreachable("Unsupported Binding");
+  case ELF::STB_LOCAL:
+    Val = 0;
+    break;
+  case ELF::STB_GLOBAL:
+    Val = 1;
+    break;
+  case ELF::STB_WEAK:
+    Val = 2;
+    break;
+  case ELF::STB_GNU_UNIQUE:
+    Val = 3;
+    break;
+  }
+  uint32_t OtherFlags = getFlags() & ~(0x3 << ELF_STB_Shift);
+  setFlags(OtherFlags | (Val << ELF_STB_Shift));
+}
+
+unsigned MCSymbolELF::getBinding() const {
+  if (isBindingSet()) {
+    uint32_t Val = (getFlags() & (0x3 << ELF_STB_Shift)) >> ELF_STB_Shift;
+    switch (Val) {
+    default:
+      llvm_unreachable("Invalid value");
+    case 0:
+      return ELF::STB_LOCAL;
+    case 1:
+      return ELF::STB_GLOBAL;
+    case 2:
+      return ELF::STB_WEAK;
+    case 3:
+      return ELF::STB_GNU_UNIQUE;
+    }
+  }
+
+  if (isDefined())
+    return ELF::STB_LOCAL;
+  if (isUsedInReloc())
+    return ELF::STB_GLOBAL;
+  if (isWeakrefUsedInReloc())
+    return ELF::STB_WEAK;
+  if (isSignature())
+    return ELF::STB_LOCAL;
+  return ELF::STB_GLOBAL;
+}
+
+void MCSymbolELF::setType(unsigned Type) const {
+  unsigned Val;
+  switch (Type) {
+  default:
+    llvm_unreachable("Unsupported Binding");
+  case ELF::STT_NOTYPE:
+    Val = 0;
+    break;
+  case ELF::STT_OBJECT:
+    Val = 1;
+    break;
+  case ELF::STT_FUNC:
+    Val = 2;
+    break;
+  case ELF::STT_SECTION:
+    Val = 3;
+    break;
+  case ELF::STT_COMMON:
+    Val = 4;
+    break;
+  case ELF::STT_TLS:
+    Val = 5;
+    break;
+  case ELF::STT_GNU_IFUNC:
+    Val = 6;
+    break;
+  }
+  uint32_t OtherFlags = getFlags() & ~(0x7 << ELF_STT_Shift);
+  setFlags(OtherFlags | (Val << ELF_STT_Shift));
+}
+
+unsigned MCSymbolELF::getType() const {
+  uint32_t Val = (getFlags() & (0x7 << ELF_STT_Shift)) >> ELF_STT_Shift;
+  switch (Val) {
+  default:
+    llvm_unreachable("Invalid value");
+  case 0:
+    return ELF::STT_NOTYPE;
+  case 1:
+    return ELF::STT_OBJECT;
+  case 2:
+    return ELF::STT_FUNC;
+  case 3:
+    return ELF::STT_SECTION;
+  case 4:
+    return ELF::STT_COMMON;
+  case 5:
+    return ELF::STT_TLS;
+  case 6:
+    return ELF::STT_GNU_IFUNC;
+  }
+}
+
+void MCSymbolELF::setVisibility(unsigned Visibility) {
+  assert(Visibility == ELF::STV_DEFAULT || Visibility == ELF::STV_INTERNAL ||
+         Visibility == ELF::STV_HIDDEN || Visibility == ELF::STV_PROTECTED);
+
+  uint32_t OtherFlags = getFlags() & ~(0x3 << ELF_STV_Shift);
+  setFlags(OtherFlags | (Visibility << ELF_STV_Shift));
+}
+
+unsigned MCSymbolELF::getVisibility() const {
+  unsigned Visibility = (getFlags() & (0x3 << ELF_STV_Shift)) >> ELF_STV_Shift;
+  assert(Visibility == ELF::STV_DEFAULT || Visibility == ELF::STV_INTERNAL ||
+         Visibility == ELF::STV_HIDDEN || Visibility == ELF::STV_PROTECTED);
+  return Visibility;
+}
+
+void MCSymbolELF::setOther(unsigned Other) {
+  assert((Other & 0x1f) == 0);
+  Other >>= 5;
+  assert(Other <= 0x7);
+  uint32_t OtherFlags = getFlags() & ~(0x7 << ELF_STO_Shift);
+  setFlags(OtherFlags | (Other << ELF_STO_Shift));
+}
+
+unsigned MCSymbolELF::getOther() const {
+  unsigned Other = (getFlags() & (0x7 << ELF_STO_Shift)) >> ELF_STO_Shift;
+  return Other << 5;
+}
+
+void MCSymbolELF::setIsWeakrefUsedInReloc() const {
+  uint32_t OtherFlags = getFlags() & ~(0x1 << ELF_WeakrefUsedInReloc_Shift);
+  setFlags(OtherFlags | (1 << ELF_WeakrefUsedInReloc_Shift));
+}
+
+bool MCSymbolELF::isWeakrefUsedInReloc() const {
+  return getFlags() & (0x1 << ELF_WeakrefUsedInReloc_Shift);
+}
+
+void MCSymbolELF::setIsSignature() const {
+  uint32_t OtherFlags = getFlags() & ~(0x1 << ELF_IsSignature_Shift);
+  setFlags(OtherFlags | (1 << ELF_IsSignature_Shift));
+}
+
+bool MCSymbolELF::isSignature() const {
+  return getFlags() & (0x1 << ELF_IsSignature_Shift);
+}
+
+void MCSymbolELF::setIsBindingSet() const {
+  uint32_t OtherFlags = getFlags() & ~(0x1 << ELF_BindingSet_Shift);
+  setFlags(OtherFlags | (1 << ELF_BindingSet_Shift));
+}
+
+bool MCSymbolELF::isBindingSet() const {
+  return getFlags() & (0x1 << ELF_BindingSet_Shift);
+}
+}
diff --git a/lib/MC/MCTargetOptions.cpp b/lib/MC/MCTargetOptions.cpp
index 1258d9e29f2..64796af10c8 100644
--- a/lib/MC/MCTargetOptions.cpp
+++ b/lib/MC/MCTargetOptions.cpp
@@ -14,7 +14,7 @@ namespace llvm {
 
 MCTargetOptions::MCTargetOptions()
     : SanitizeAddress(false), MCRelaxAll(false), MCNoExecStack(false),
-      MCFatalWarnings(false), MCSaveTempLabels(false),
+      MCFatalWarnings(false), MCNoWarn(false), MCSaveTempLabels(false),
       MCUseDwarfDirectory(false), ShowMCEncoding(false), ShowMCInst(false),
       AsmVerbose(false), DwarfVersion(0), ABIName() {}
 
diff --git a/lib/MC/MCValue.cpp b/lib/MC/MCValue.cpp
index 5512e031b61..495a2b6ea5b 100644
--- a/lib/MC/MCValue.cpp
+++ b/lib/MC/MCValue.cpp
@@ -26,11 +26,11 @@ void MCValue::print(raw_ostream &OS) const {
   if (getRefKind())
     OS << ':' << getRefKind() <<  ':';
 
-  getSymA()->print(OS);
+  OS << *getSymA();
 
   if (getSymB()) {
     OS << " - ";
-    getSymB()->print(OS);
+    OS << *getSymB();
   }
 
   if (getConstant())
diff --git a/lib/MC/MCWin64EH.cpp b/lib/MC/MCWin64EH.cpp
index dfadb3cc42b..1b73b7afb6a 100644
--- a/lib/MC/MCWin64EH.cpp
+++ b/lib/MC/MCWin64EH.cpp
@@ -51,8 +51,8 @@ static void EmitAbsDifference(MCStreamer &Streamer, const MCSymbol *LHS,
                               const MCSymbol *RHS) {
   MCContext &Context = Streamer.getContext();
   const MCExpr *Diff =
-      MCBinaryExpr::CreateSub(MCSymbolRefExpr::Create(LHS, Context),
-                              MCSymbolRefExpr::Create(RHS, Context), Context);
+      MCBinaryExpr::createSub(MCSymbolRefExpr::create(LHS, Context),
+                              MCSymbolRefExpr::create(RHS, Context), Context);
   Streamer.EmitValue(Diff, 1);
 }
 
@@ -126,13 +126,13 @@ static void EmitSymbolRefWithOfs(MCStreamer &streamer,
                                  const MCSymbol *Base,
                                  const MCSymbol *Other) {
   MCContext &Context = streamer.getContext();
-  const MCSymbolRefExpr *BaseRef = MCSymbolRefExpr::Create(Base, Context);
-  const MCSymbolRefExpr *OtherRef = MCSymbolRefExpr::Create(Other, Context);
-  const MCExpr *Ofs = MCBinaryExpr::CreateSub(OtherRef, BaseRef, Context);
-  const MCSymbolRefExpr *BaseRefRel = MCSymbolRefExpr::Create(Base,
+  const MCSymbolRefExpr *BaseRef = MCSymbolRefExpr::create(Base, Context);
+  const MCSymbolRefExpr *OtherRef = MCSymbolRefExpr::create(Other, Context);
+  const MCExpr *Ofs = MCBinaryExpr::createSub(OtherRef, BaseRef, Context);
+  const MCSymbolRefExpr *BaseRefRel = MCSymbolRefExpr::create(Base,
                                               MCSymbolRefExpr::VK_COFF_IMGREL32,
                                               Context);
-  streamer.EmitValue(MCBinaryExpr::CreateAdd(BaseRefRel, Ofs, Context), 4);
+  streamer.EmitValue(MCBinaryExpr::createAdd(BaseRefRel, Ofs, Context), 4);
 }
 
 static void EmitRuntimeFunction(MCStreamer &streamer,
@@ -142,7 +142,7 @@ static void EmitRuntimeFunction(MCStreamer &streamer,
   streamer.EmitValueToAlignment(4);
   EmitSymbolRefWithOfs(streamer, info->Function, info->Begin);
   EmitSymbolRefWithOfs(streamer, info->Function, info->End);
-  streamer.EmitValue(MCSymbolRefExpr::Create(info->Symbol,
+  streamer.EmitValue(MCSymbolRefExpr::create(info->Symbol,
                                              MCSymbolRefExpr::VK_COFF_IMGREL32,
                                              context), 4);
 }
@@ -153,7 +153,7 @@ static void EmitUnwindInfo(MCStreamer &streamer, WinEH::FrameInfo *info) {
     return;
 
   MCContext &context = streamer.getContext();
-  MCSymbol *Label = context.CreateTempSymbol();
+  MCSymbol *Label = context.createTempSymbol();
 
   streamer.EmitValueToAlignment(4);
   streamer.EmitLabel(Label);
@@ -207,7 +207,7 @@ static void EmitUnwindInfo(MCStreamer &streamer, WinEH::FrameInfo *info) {
     EmitRuntimeFunction(streamer, info->ChainedParent);
   else if (flags &
            ((Win64EH::UNW_TerminateHandler|Win64EH::UNW_ExceptionHandler) << 3))
-    streamer.EmitValue(MCSymbolRefExpr::Create(info->ExceptionHandler,
+    streamer.EmitValue(MCSymbolRefExpr::create(info->ExceptionHandler,
                                               MCSymbolRefExpr::VK_COFF_IMGREL32,
                                               context), 4);
   else if (numCodes == 0) {
@@ -224,16 +224,14 @@ void UnwindEmitter::Emit(MCStreamer &Streamer) const {
 
   // Emit the unwind info structs first.
   for (const auto &CFI : Streamer.getWinFrameInfos()) {
-    const MCSection *XData =
-        getXDataSection(CFI->Function, Context);
+    MCSection *XData = getXDataSection(CFI->Function, Context);
     Streamer.SwitchSection(XData);
     EmitUnwindInfo(Streamer, CFI);
   }
 
   // Now emit RUNTIME_FUNCTION entries.
   for (const auto &CFI : Streamer.getWinFrameInfos()) {
-    const MCSection *PData =
-        getPDataSection(CFI->Function, Context);
+    MCSection *PData = getPDataSection(CFI->Function, Context);
     Streamer.SwitchSection(PData);
     EmitRuntimeFunction(Streamer, CFI);
   }
@@ -244,8 +242,7 @@ void UnwindEmitter::EmitUnwindInfo(MCStreamer &Streamer,
   // Switch sections (the static function above is meant to be called from
   // here and from Emit().
   MCContext &context = Streamer.getContext();
-  const MCSection *xdataSect =
-    getXDataSection(info->Function, context);
+  MCSection *xdataSect = getXDataSection(info->Function, context);
   Streamer.SwitchSection(xdataSect);
 
   llvm::EmitUnwindInfo(Streamer, info);
diff --git a/lib/MC/MCWinEH.cpp b/lib/MC/MCWinEH.cpp
index b1c95f800cf..d5d9eadf39a 100644
--- a/lib/MC/MCWinEH.cpp
+++ b/lib/MC/MCWinEH.cpp
@@ -25,9 +25,10 @@ namespace WinEH {
 /// associated with that comdat. If the code described is not in the main .text
 /// section, make a new section for it. Otherwise use the main unwind info
 /// section.
-static const MCSection *getUnwindInfoSection(
-    StringRef SecName, const MCSectionCOFF *UnwindSec, const MCSymbol *Function,
-    MCContext &Context) {
+static MCSection *getUnwindInfoSection(StringRef SecName,
+                                       MCSectionCOFF *UnwindSec,
+                                       const MCSymbol *Function,
+                                       MCContext &Context) {
   if (Function && Function->isInSection()) {
     // If Function is in a COMDAT, get or create an unwind info section in that
     // COMDAT group.
@@ -59,16 +60,16 @@ static const MCSection *getUnwindInfoSection(
 
 }
 
-const MCSection *UnwindEmitter::getPDataSection(const MCSymbol *Function,
-                                                MCContext &Context) {
-  const MCSectionCOFF *PData =
+MCSection *UnwindEmitter::getPDataSection(const MCSymbol *Function,
+                                          MCContext &Context) {
+  MCSectionCOFF *PData =
       cast<MCSectionCOFF>(Context.getObjectFileInfo()->getPDataSection());
   return getUnwindInfoSection(".pdata", PData, Function, Context);
 }
 
-const MCSection *UnwindEmitter::getXDataSection(const MCSymbol *Function,
-                                                MCContext &Context) {
-  const MCSectionCOFF *XData =
+MCSection *UnwindEmitter::getXDataSection(const MCSymbol *Function,
+                                          MCContext &Context) {
+  MCSectionCOFF *XData =
       cast<MCSectionCOFF>(Context.getObjectFileInfo()->getXDataSection());
   return getUnwindInfoSection(".xdata", XData, Function, Context);
 }
diff --git a/lib/MC/MachObjectWriter.cpp b/lib/MC/MachObjectWriter.cpp
index 837f5857b66..8ce6127e386 100644
--- a/lib/MC/MachObjectWriter.cpp
+++ b/lib/MC/MachObjectWriter.cpp
@@ -15,10 +15,9 @@
 #include "llvm/MC/MCAssembler.h"
 #include "llvm/MC/MCExpr.h"
 #include "llvm/MC/MCFixupKindInfo.h"
-#include "llvm/MC/MCMachOSymbolFlags.h"
 #include "llvm/MC/MCObjectWriter.h"
 #include "llvm/MC/MCSectionMachO.h"
-#include "llvm/MC/MCSymbol.h"
+#include "llvm/MC/MCSymbolMachO.h"
 #include "llvm/MC/MCValue.h"
 #include "llvm/Support/Debug.h"
 #include "llvm/Support/ErrorHandling.h"
@@ -39,15 +38,14 @@ void MachObjectWriter::reset() {
   MCObjectWriter::reset();
 }
 
-bool MachObjectWriter::
-doesSymbolRequireExternRelocation(const MCSymbolData *SD) {
+bool MachObjectWriter::doesSymbolRequireExternRelocation(const MCSymbol &S) {
   // Undefined symbols are always extern.
-  if (SD->getSymbol().isUndefined())
+  if (S.isUndefined())
     return true;
 
   // References to weak definitions require external relocation entries; the
   // definition may not always be the one in the same object file.
-  if (SD->getFlags() & SF_WeakDefinition)
+  if (cast<MCSymbolMachO>(S).isWeakDefinition())
     return true;
 
   // Otherwise, we can use an internal relocation.
@@ -56,8 +54,7 @@ doesSymbolRequireExternRelocation(const MCSymbolData *SD) {
 
 bool MachObjectWriter::
 MachSymbolData::operator<(const MachSymbolData &RHS) const {
-  return SymbolData->getSymbol().getName() <
-    RHS.SymbolData->getSymbol().getName();
+  return Symbol->getName() < RHS.Symbol->getName();
 }
 
 bool MachObjectWriter::isFixupKindPCRel(const MCAssembler &Asm, unsigned Kind) {
@@ -70,13 +67,11 @@ bool MachObjectWriter::isFixupKindPCRel(const MCAssembler &Asm, unsigned Kind) {
 uint64_t MachObjectWriter::getFragmentAddress(const MCFragment *Fragment,
                                               const MCAsmLayout &Layout) const {
   return getSectionAddress(Fragment->getParent()) +
-    Layout.getFragmentOffset(Fragment);
+         Layout.getFragmentOffset(Fragment);
 }
 
-uint64_t MachObjectWriter::getSymbolAddress(const MCSymbolData* SD,
+uint64_t MachObjectWriter::getSymbolAddress(const MCSymbol &S,
                                             const MCAsmLayout &Layout) const {
-  const MCSymbol &S = SD->getSymbol();
-
   // If this is a variable, then recursively evaluate now.
   if (S.isVariable()) {
     if (const MCConstantExpr *C =
@@ -85,7 +80,7 @@ uint64_t MachObjectWriter::getSymbolAddress(const MCSymbolData* SD,
 
 
     MCValue Target;
-    if (!S.getVariableValue()->EvaluateAsRelocatable(Target, &Layout, nullptr))
+    if (!S.getVariableValue()->evaluateAsRelocatable(Target, &Layout, nullptr))
       report_fatal_error("unable to evaluate offset for variable '" +
                          S.getName() + "'");
 
@@ -99,32 +94,30 @@ uint64_t MachObjectWriter::getSymbolAddress(const MCSymbolData* SD,
 
     uint64_t Address = Target.getConstant();
     if (Target.getSymA())
-      Address += getSymbolAddress(&Layout.getAssembler().getSymbolData(
-                                    Target.getSymA()->getSymbol()), Layout);
+      Address += getSymbolAddress(Target.getSymA()->getSymbol(), Layout);
     if (Target.getSymB())
-      Address += getSymbolAddress(&Layout.getAssembler().getSymbolData(
-                                    Target.getSymB()->getSymbol()), Layout);
+      Address += getSymbolAddress(Target.getSymB()->getSymbol(), Layout);
     return Address;
   }
 
-  return getSectionAddress(SD->getFragment()->getParent()) +
-    Layout.getSymbolOffset(SD);
+  return getSectionAddress(S.getFragment()->getParent()) +
+         Layout.getSymbolOffset(S);
 }
 
-uint64_t MachObjectWriter::getPaddingSize(const MCSectionData *SD,
+uint64_t MachObjectWriter::getPaddingSize(const MCSection *Sec,
                                           const MCAsmLayout &Layout) const {
-  uint64_t EndAddr = getSectionAddress(SD) + Layout.getSectionAddressSize(SD);
-  unsigned Next = SD->getLayoutOrder() + 1;
+  uint64_t EndAddr = getSectionAddress(Sec) + Layout.getSectionAddressSize(Sec);
+  unsigned Next = Sec->getLayoutOrder() + 1;
   if (Next >= Layout.getSectionOrder().size())
     return 0;
 
-  const MCSectionData &NextSD = *Layout.getSectionOrder()[Next];
-  if (NextSD.getSection().isVirtualSection())
+  const MCSection &NextSec = *Layout.getSectionOrder()[Next];
+  if (NextSec.isVirtualSection())
     return 0;
-  return OffsetToAlignment(EndAddr, NextSD.getAlignment());
+  return OffsetToAlignment(EndAddr, NextSec.getAlignment());
 }
 
-void MachObjectWriter::WriteHeader(unsigned NumLoadCommands,
+void MachObjectWriter::writeHeader(unsigned NumLoadCommands,
                                    unsigned LoadCommandsSize,
                                    bool SubsectionsViaSymbols) {
   uint32_t Flags = 0;
@@ -138,27 +131,27 @@ void MachObjectWriter::WriteHeader(unsigned NumLoadCommands,
   uint64_t Start = OS.tell();
   (void) Start;
 
-  Write32(is64Bit() ? MachO::MH_MAGIC_64 : MachO::MH_MAGIC);
+  write32(is64Bit() ? MachO::MH_MAGIC_64 : MachO::MH_MAGIC);
 
-  Write32(TargetObjectWriter->getCPUType());
-  Write32(TargetObjectWriter->getCPUSubtype());
+  write32(TargetObjectWriter->getCPUType());
+  write32(TargetObjectWriter->getCPUSubtype());
 
-  Write32(MachO::MH_OBJECT);
-  Write32(NumLoadCommands);
-  Write32(LoadCommandsSize);
-  Write32(Flags);
+  write32(MachO::MH_OBJECT);
+  write32(NumLoadCommands);
+  write32(LoadCommandsSize);
+  write32(Flags);
   if (is64Bit())
-    Write32(0); // reserved
+    write32(0); // reserved
 
   assert(OS.tell() - Start ==
          (is64Bit()?sizeof(MachO::mach_header_64): sizeof(MachO::mach_header)));
 }
 
-/// WriteSegmentLoadCommand - Write a segment load command.
+/// writeSegmentLoadCommand - Write a segment load command.
 ///
 /// \param NumSections The number of sections in this segment.
 /// \param SectionDataSize The total size of the sections.
-void MachObjectWriter::WriteSegmentLoadCommand(unsigned NumSections,
+void MachObjectWriter::writeSegmentLoadCommand(unsigned NumSections,
                                                uint64_t VMSize,
                                                uint64_t SectionDataStartOffset,
                                                uint64_t SectionDataSize) {
@@ -171,44 +164,44 @@ void MachObjectWriter::WriteSegmentLoadCommand(unsigned NumSections,
   unsigned SegmentLoadCommandSize =
     is64Bit() ? sizeof(MachO::segment_command_64):
     sizeof(MachO::segment_command);
-  Write32(is64Bit() ? MachO::LC_SEGMENT_64 : MachO::LC_SEGMENT);
-  Write32(SegmentLoadCommandSize +
+  write32(is64Bit() ? MachO::LC_SEGMENT_64 : MachO::LC_SEGMENT);
+  write32(SegmentLoadCommandSize +
           NumSections * (is64Bit() ? sizeof(MachO::section_64) :
                          sizeof(MachO::section)));
 
-  WriteBytes("", 16);
+  writeBytes("", 16);
   if (is64Bit()) {
-    Write64(0); // vmaddr
-    Write64(VMSize); // vmsize
-    Write64(SectionDataStartOffset); // file offset
-    Write64(SectionDataSize); // file size
+    write64(0); // vmaddr
+    write64(VMSize); // vmsize
+    write64(SectionDataStartOffset); // file offset
+    write64(SectionDataSize); // file size
   } else {
-    Write32(0); // vmaddr
-    Write32(VMSize); // vmsize
-    Write32(SectionDataStartOffset); // file offset
-    Write32(SectionDataSize); // file size
+    write32(0); // vmaddr
+    write32(VMSize); // vmsize
+    write32(SectionDataStartOffset); // file offset
+    write32(SectionDataSize); // file size
   }
   // maxprot
-  Write32(MachO::VM_PROT_READ | MachO::VM_PROT_WRITE | MachO::VM_PROT_EXECUTE); 
+  write32(MachO::VM_PROT_READ | MachO::VM_PROT_WRITE | MachO::VM_PROT_EXECUTE);
   // initprot
-  Write32(MachO::VM_PROT_READ | MachO::VM_PROT_WRITE | MachO::VM_PROT_EXECUTE); 
-  Write32(NumSections);
-  Write32(0); // flags
+  write32(MachO::VM_PROT_READ | MachO::VM_PROT_WRITE | MachO::VM_PROT_EXECUTE);
+  write32(NumSections);
+  write32(0); // flags
 
   assert(OS.tell() - Start == SegmentLoadCommandSize);
 }
 
-void MachObjectWriter::WriteSection(const MCAssembler &Asm,
+void MachObjectWriter::writeSection(const MCAssembler &Asm,
                                     const MCAsmLayout &Layout,
-                                    const MCSectionData &SD,
-                                    uint64_t FileOffset,
+                                    const MCSection &Sec, uint64_t FileOffset,
                                     uint64_t RelocationsStart,
                                     unsigned NumRelocations) {
-  uint64_t SectionSize = Layout.getSectionAddressSize(&SD);
+  uint64_t SectionSize = Layout.getSectionAddressSize(&Sec);
+  const MCSectionMachO &Section = cast<MCSectionMachO>(Sec);
 
   // The offset is unused for virtual sections.
-  if (SD.getSection().isVirtualSection()) {
-    assert(Layout.getSectionFileSize(&SD) == 0 && "Invalid file size!");
+  if (Section.isVirtualSection()) {
+    assert(Layout.getSectionFileSize(&Sec) == 0 && "Invalid file size!");
     FileOffset = 0;
   }
 
@@ -218,37 +211,36 @@ void MachObjectWriter::WriteSection(const MCAssembler &Asm,
   uint64_t Start = OS.tell();
   (void) Start;
 
-  const MCSectionMachO &Section = cast<MCSectionMachO>(SD.getSection());
-  WriteBytes(Section.getSectionName(), 16);
-  WriteBytes(Section.getSegmentName(), 16);
+  writeBytes(Section.getSectionName(), 16);
+  writeBytes(Section.getSegmentName(), 16);
   if (is64Bit()) {
-    Write64(getSectionAddress(&SD)); // address
-    Write64(SectionSize); // size
+    write64(getSectionAddress(&Sec)); // address
+    write64(SectionSize); // size
   } else {
-    Write32(getSectionAddress(&SD)); // address
-    Write32(SectionSize); // size
+    write32(getSectionAddress(&Sec)); // address
+    write32(SectionSize); // size
   }
-  Write32(FileOffset);
+  write32(FileOffset);
 
   unsigned Flags = Section.getTypeAndAttributes();
-  if (SD.hasInstructions())
+  if (Section.hasInstructions())
     Flags |= MachO::S_ATTR_SOME_INSTRUCTIONS;
 
-  assert(isPowerOf2_32(SD.getAlignment()) && "Invalid alignment!");
-  Write32(Log2_32(SD.getAlignment()));
-  Write32(NumRelocations ? RelocationsStart : 0);
-  Write32(NumRelocations);
-  Write32(Flags);
-  Write32(IndirectSymBase.lookup(&SD)); // reserved1
-  Write32(Section.getStubSize()); // reserved2
+  assert(isPowerOf2_32(Section.getAlignment()) && "Invalid alignment!");
+  write32(Log2_32(Section.getAlignment()));
+  write32(NumRelocations ? RelocationsStart : 0);
+  write32(NumRelocations);
+  write32(Flags);
+  write32(IndirectSymBase.lookup(&Sec)); // reserved1
+  write32(Section.getStubSize()); // reserved2
   if (is64Bit())
-    Write32(0); // reserved3
+    write32(0); // reserved3
 
   assert(OS.tell() - Start == (is64Bit() ? sizeof(MachO::section_64) :
                                sizeof(MachO::section)));
 }
 
-void MachObjectWriter::WriteSymtabLoadCommand(uint32_t SymbolOffset,
+void MachObjectWriter::writeSymtabLoadCommand(uint32_t SymbolOffset,
                                               uint32_t NumSymbols,
                                               uint32_t StringTableOffset,
                                               uint32_t StringTableSize) {
@@ -257,17 +249,17 @@ void MachObjectWriter::WriteSymtabLoadCommand(uint32_t SymbolOffset,
   uint64_t Start = OS.tell();
   (void) Start;
 
-  Write32(MachO::LC_SYMTAB);
-  Write32(sizeof(MachO::symtab_command));
-  Write32(SymbolOffset);
-  Write32(NumSymbols);
-  Write32(StringTableOffset);
-  Write32(StringTableSize);
+  write32(MachO::LC_SYMTAB);
+  write32(sizeof(MachO::symtab_command));
+  write32(SymbolOffset);
+  write32(NumSymbols);
+  write32(StringTableOffset);
+  write32(StringTableSize);
 
   assert(OS.tell() - Start == sizeof(MachO::symtab_command));
 }
 
-void MachObjectWriter::WriteDysymtabLoadCommand(uint32_t FirstLocalSymbol,
+void MachObjectWriter::writeDysymtabLoadCommand(uint32_t FirstLocalSymbol,
                                                 uint32_t NumLocalSymbols,
                                                 uint32_t FirstExternalSymbol,
                                                 uint32_t NumExternalSymbols,
@@ -280,64 +272,71 @@ void MachObjectWriter::WriteDysymtabLoadCommand(uint32_t FirstLocalSymbol,
   uint64_t Start = OS.tell();
   (void) Start;
 
-  Write32(MachO::LC_DYSYMTAB);
-  Write32(sizeof(MachO::dysymtab_command));
-  Write32(FirstLocalSymbol);
-  Write32(NumLocalSymbols);
-  Write32(FirstExternalSymbol);
-  Write32(NumExternalSymbols);
-  Write32(FirstUndefinedSymbol);
-  Write32(NumUndefinedSymbols);
-  Write32(0); // tocoff
-  Write32(0); // ntoc
-  Write32(0); // modtaboff
-  Write32(0); // nmodtab
-  Write32(0); // extrefsymoff
-  Write32(0); // nextrefsyms
-  Write32(IndirectSymbolOffset);
-  Write32(NumIndirectSymbols);
-  Write32(0); // extreloff
-  Write32(0); // nextrel
-  Write32(0); // locreloff
-  Write32(0); // nlocrel
+  write32(MachO::LC_DYSYMTAB);
+  write32(sizeof(MachO::dysymtab_command));
+  write32(FirstLocalSymbol);
+  write32(NumLocalSymbols);
+  write32(FirstExternalSymbol);
+  write32(NumExternalSymbols);
+  write32(FirstUndefinedSymbol);
+  write32(NumUndefinedSymbols);
+  write32(0); // tocoff
+  write32(0); // ntoc
+  write32(0); // modtaboff
+  write32(0); // nmodtab
+  write32(0); // extrefsymoff
+  write32(0); // nextrefsyms
+  write32(IndirectSymbolOffset);
+  write32(NumIndirectSymbols);
+  write32(0); // extreloff
+  write32(0); // nextrel
+  write32(0); // locreloff
+  write32(0); // nlocrel
 
   assert(OS.tell() - Start == sizeof(MachO::dysymtab_command));
 }
 
 MachObjectWriter::MachSymbolData *
 MachObjectWriter::findSymbolData(const MCSymbol &Sym) {
-  for (auto &Entry : LocalSymbolData)
-    if (&Entry.SymbolData->getSymbol() == &Sym)
-      return &Entry;
-
-  for (auto &Entry : ExternalSymbolData)
-    if (&Entry.SymbolData->getSymbol() == &Sym)
-      return &Entry;
-
-  for (auto &Entry : UndefinedSymbolData)
-    if (&Entry.SymbolData->getSymbol() == &Sym)
-      return &Entry;
+  for (auto *SymbolData :
+       {&LocalSymbolData, &ExternalSymbolData, &UndefinedSymbolData})
+    for (MachSymbolData &Entry : *SymbolData)
+      if (Entry.Symbol == &Sym)
+        return &Entry;
 
   return nullptr;
 }
 
-void MachObjectWriter::WriteNlist(MachSymbolData &MSD,
+const MCSymbol &MachObjectWriter::findAliasedSymbol(const MCSymbol &Sym) const {
+  const MCSymbol *S = &Sym;
+  while (S->isVariable()) {
+    const MCExpr *Value = S->getVariableValue();
+    const auto *Ref = dyn_cast<MCSymbolRefExpr>(Value);
+    if (!Ref)
+      return *S;
+    S = &Ref->getSymbol();
+  }
+  return *S;
+}
+
+void MachObjectWriter::writeNlist(MachSymbolData &MSD,
                                   const MCAsmLayout &Layout) {
-  MCSymbolData &Data = *MSD.SymbolData;
-  const MCSymbol *Symbol = &Data.getSymbol();
-  const MCSymbol *AliasedSymbol = &Symbol->AliasedSymbol();
+  const MCSymbol *Symbol = MSD.Symbol;
+  const MCSymbol &Data = *Symbol;
+  const MCSymbol *AliasedSymbol = &findAliasedSymbol(*Symbol);
   uint8_t SectionIndex = MSD.SectionIndex;
   uint8_t Type = 0;
-  uint16_t Flags = Data.getFlags();
   uint64_t Address = 0;
   bool IsAlias = Symbol != AliasedSymbol;
 
+  const MCSymbol &OrigSymbol = *Symbol;
   MachSymbolData *AliaseeInfo;
   if (IsAlias) {
     AliaseeInfo = findSymbolData(*AliasedSymbol);
     if (AliaseeInfo)
       SectionIndex = AliaseeInfo->SectionIndex;
     Symbol = AliasedSymbol;
+    // FIXME: Should this update Data as well?  Do we need OrigSymbol at all?
   }
 
   // Set the N_TYPE bits. See <mach-o/nlist.h>.
@@ -365,53 +364,38 @@ void MachObjectWriter::WriteNlist(MachSymbolData &MSD,
   if (IsAlias && Symbol->isUndefined())
     Address = AliaseeInfo->StringIndex;
   else if (Symbol->isDefined())
-    Address = getSymbolAddress(&Data, Layout);
-  else if (Data.isCommon()) {
+    Address = getSymbolAddress(OrigSymbol, Layout);
+  else if (Symbol->isCommon()) {
     // Common symbols are encoded with the size in the address
     // field, and their alignment in the flags.
-    Address = Data.getCommonSize();
-
-    // Common alignment is packed into the 'desc' bits.
-    if (unsigned Align = Data.getCommonAlignment()) {
-      unsigned Log2Size = Log2_32(Align);
-      assert((1U << Log2Size) == Align && "Invalid 'common' alignment!");
-      if (Log2Size > 15)
-        report_fatal_error("invalid 'common' alignment '" +
-                           Twine(Align) + "' for '" + Symbol->getName() + "'",
-                           false);
-      // FIXME: Keep this mask with the SymbolFlags enumeration.
-      Flags = (Flags & 0xF0FF) | (Log2Size << 8);
-    }
+    Address = Symbol->getCommonSize();
   }
 
-  if (Layout.getAssembler().isThumbFunc(Symbol))
-    Flags |= SF_ThumbFunc;
-
   // struct nlist (12 bytes)
 
-  Write32(MSD.StringIndex);
-  Write8(Type);
-  Write8(SectionIndex);
+  write32(MSD.StringIndex);
+  write8(Type);
+  write8(SectionIndex);
 
   // The Mach-O streamer uses the lowest 16-bits of the flags for the 'desc'
   // value.
-  Write16(Flags);
+  write16(cast<MCSymbolMachO>(Symbol)->getEncodedFlags());
   if (is64Bit())
-    Write64(Address);
+    write64(Address);
   else
-    Write32(Address);
+    write32(Address);
 }
 
-void MachObjectWriter::WriteLinkeditLoadCommand(uint32_t Type,
+void MachObjectWriter::writeLinkeditLoadCommand(uint32_t Type,
                                                 uint32_t DataOffset,
                                                 uint32_t DataSize) {
   uint64_t Start = OS.tell();
   (void) Start;
 
-  Write32(Type);
-  Write32(sizeof(MachO::linkedit_data_command));
-  Write32(DataOffset);
-  Write32(DataSize);
+  write32(Type);
+  write32(sizeof(MachO::linkedit_data_command));
+  write32(DataOffset);
+  write32(DataSize);
 
   assert(OS.tell() - Start == sizeof(MachO::linkedit_data_command));
 }
@@ -420,45 +404,44 @@ static unsigned ComputeLinkerOptionsLoadCommandSize(
   const std::vector<std::string> &Options, bool is64Bit)
 {
   unsigned Size = sizeof(MachO::linker_option_command);
-  for (unsigned i = 0, e = Options.size(); i != e; ++i)
-    Size += Options[i].size() + 1;
+  for (const std::string &Option : Options)
+    Size += Option.size() + 1;
   return RoundUpToAlignment(Size, is64Bit ? 8 : 4);
 }
 
-void MachObjectWriter::WriteLinkerOptionsLoadCommand(
+void MachObjectWriter::writeLinkerOptionsLoadCommand(
   const std::vector<std::string> &Options)
 {
   unsigned Size = ComputeLinkerOptionsLoadCommandSize(Options, is64Bit());
   uint64_t Start = OS.tell();
   (void) Start;
 
-  Write32(MachO::LC_LINKER_OPTION);
-  Write32(Size);
-  Write32(Options.size());
+  write32(MachO::LC_LINKER_OPTION);
+  write32(Size);
+  write32(Options.size());
   uint64_t BytesWritten = sizeof(MachO::linker_option_command);
-  for (unsigned i = 0, e = Options.size(); i != e; ++i) {
+  for (const std::string &Option : Options) {
     // Write each string, including the null byte.
-    const std::string &Option = Options[i];
-    WriteBytes(Option.c_str(), Option.size() + 1);
+    writeBytes(Option.c_str(), Option.size() + 1);
     BytesWritten += Option.size() + 1;
   }
 
   // Pad to a multiple of the pointer size.
-  WriteBytes("", OffsetToAlignment(BytesWritten, is64Bit() ? 8 : 4));
+  writeBytes("", OffsetToAlignment(BytesWritten, is64Bit() ? 8 : 4));
 
   assert(OS.tell() - Start == Size);
 }
 
-void MachObjectWriter::RecordRelocation(MCAssembler &Asm,
+void MachObjectWriter::recordRelocation(MCAssembler &Asm,
                                         const MCAsmLayout &Layout,
                                         const MCFragment *Fragment,
                                         const MCFixup &Fixup, MCValue Target,
                                         bool &IsPCRel, uint64_t &FixedValue) {
-  TargetObjectWriter->RecordRelocation(this, Asm, Layout, Fragment, Fixup,
+  TargetObjectWriter->recordRelocation(this, Asm, Layout, Fragment, Fixup,
                                        Target, FixedValue);
 }
 
-void MachObjectWriter::BindIndirectSymbols(MCAssembler &Asm) {
+void MachObjectWriter::bindIndirectSymbols(MCAssembler &Asm) {
   // This is the point where 'as' creates actual symbols for indirect symbols
   // (in the following two passes). It would be easier for us to do this sooner
   // when we see the attribute, but that makes getting the order in the symbol
@@ -470,8 +453,7 @@ void MachObjectWriter::BindIndirectSymbols(MCAssembler &Asm) {
   // or stub section.
   for (MCAssembler::indirect_symbol_iterator it = Asm.indirect_symbol_begin(),
          ie = Asm.indirect_symbol_end(); it != ie; ++it) {
-    const MCSectionMachO &Section =
-      cast<MCSectionMachO>(it->SectionData->getSection());
+    const MCSectionMachO &Section = cast<MCSectionMachO>(*it->Section);
 
     if (Section.getType() != MachO::S_NON_LAZY_SYMBOL_POINTERS &&
         Section.getType() != MachO::S_LAZY_SYMBOL_POINTERS &&
@@ -486,44 +468,42 @@ void MachObjectWriter::BindIndirectSymbols(MCAssembler &Asm) {
   unsigned IndirectIndex = 0;
   for (MCAssembler::indirect_symbol_iterator it = Asm.indirect_symbol_begin(),
          ie = Asm.indirect_symbol_end(); it != ie; ++it, ++IndirectIndex) {
-    const MCSectionMachO &Section =
-      cast<MCSectionMachO>(it->SectionData->getSection());
+    const MCSectionMachO &Section = cast<MCSectionMachO>(*it->Section);
 
     if (Section.getType() != MachO::S_NON_LAZY_SYMBOL_POINTERS)
       continue;
 
     // Initialize the section indirect symbol base, if necessary.
-    IndirectSymBase.insert(std::make_pair(it->SectionData, IndirectIndex));
+    IndirectSymBase.insert(std::make_pair(it->Section, IndirectIndex));
 
-    Asm.getOrCreateSymbolData(*it->Symbol);
+    Asm.registerSymbol(*it->Symbol);
   }
 
   // Then lazy symbol pointers and symbol stubs.
   IndirectIndex = 0;
   for (MCAssembler::indirect_symbol_iterator it = Asm.indirect_symbol_begin(),
          ie = Asm.indirect_symbol_end(); it != ie; ++it, ++IndirectIndex) {
-    const MCSectionMachO &Section =
-      cast<MCSectionMachO>(it->SectionData->getSection());
+    const MCSectionMachO &Section = cast<MCSectionMachO>(*it->Section);
 
     if (Section.getType() != MachO::S_LAZY_SYMBOL_POINTERS &&
         Section.getType() != MachO::S_SYMBOL_STUBS)
       continue;
 
     // Initialize the section indirect symbol base, if necessary.
-    IndirectSymBase.insert(std::make_pair(it->SectionData, IndirectIndex));
+    IndirectSymBase.insert(std::make_pair(it->Section, IndirectIndex));
 
     // Set the symbol type to undefined lazy, but only on construction.
     //
     // FIXME: Do not hardcode.
     bool Created;
-    MCSymbolData &Entry = Asm.getOrCreateSymbolData(*it->Symbol, &Created);
+    Asm.registerSymbol(*it->Symbol, &Created);
     if (Created)
-      Entry.setFlags(Entry.getFlags() | 0x0001);
+      cast<MCSymbolMachO>(it->Symbol)->setReferenceTypeUndefinedLazy(true);
   }
 }
 
-/// ComputeSymbolTable - Compute the symbol table data
-void MachObjectWriter::ComputeSymbolTable(
+/// computeSymbolTable - Compute the symbol table data
+void MachObjectWriter::computeSymbolTable(
     MCAssembler &Asm, std::vector<MachSymbolData> &LocalSymbolData,
     std::vector<MachSymbolData> &ExternalSymbolData,
     std::vector<MachSymbolData> &UndefinedSymbolData) {
@@ -532,12 +512,11 @@ void MachObjectWriter::ComputeSymbolTable(
   unsigned Index = 1;
   for (MCAssembler::iterator it = Asm.begin(),
          ie = Asm.end(); it != ie; ++it, ++Index)
-    SectionIndexMap[&it->getSection()] = Index;
+    SectionIndexMap[&*it] = Index;
   assert(Index <= 256 && "Too many sections!");
 
   // Build the string table.
-  for (MCSymbolData &SD : Asm.symbols()) {
-    const MCSymbol &Symbol = SD.getSymbol();
+  for (const MCSymbol &Symbol : Asm.symbols()) {
     if (!Asm.isSymbolLinkerVisible(Symbol))
       continue;
 
@@ -550,18 +529,16 @@ void MachObjectWriter::ComputeSymbolTable(
   // The particular order that we collect and then sort the symbols is chosen to
   // match 'as'. Even though it doesn't matter for correctness, this is
   // important for letting us diff .o files.
-  for (MCSymbolData &SD : Asm.symbols()) {
-    const MCSymbol &Symbol = SD.getSymbol();
-
+  for (const MCSymbol &Symbol : Asm.symbols()) {
     // Ignore non-linker visible symbols.
     if (!Asm.isSymbolLinkerVisible(Symbol))
       continue;
 
-    if (!SD.isExternal() && !Symbol.isUndefined())
+    if (!Symbol.isExternal() && !Symbol.isUndefined())
       continue;
 
     MachSymbolData MSD;
-    MSD.SymbolData = &SD;
+    MSD.Symbol = &Symbol;
     MSD.StringIndex = StringTable.getOffset(Symbol.getName());
 
     if (Symbol.isUndefined()) {
@@ -578,18 +555,16 @@ void MachObjectWriter::ComputeSymbolTable(
   }
 
   // Now add the data for local symbols.
-  for (MCSymbolData &SD : Asm.symbols()) {
-    const MCSymbol &Symbol = SD.getSymbol();
-
+  for (const MCSymbol &Symbol : Asm.symbols()) {
     // Ignore non-linker visible symbols.
     if (!Asm.isSymbolLinkerVisible(Symbol))
       continue;
 
-    if (SD.isExternal() || Symbol.isUndefined())
+    if (Symbol.isExternal() || Symbol.isUndefined())
       continue;
 
     MachSymbolData MSD;
-    MSD.SymbolData = &SD;
+    MSD.Symbol = &Symbol;
     MSD.StringIndex = StringTable.getOffset(Symbol.getName());
 
     if (Symbol.isAbsolute()) {
@@ -608,16 +583,13 @@ void MachObjectWriter::ComputeSymbolTable(
 
   // Set the symbol indices.
   Index = 0;
-  for (unsigned i = 0, e = LocalSymbolData.size(); i != e; ++i)
-    LocalSymbolData[i].SymbolData->setIndex(Index++);
-  for (unsigned i = 0, e = ExternalSymbolData.size(); i != e; ++i)
-    ExternalSymbolData[i].SymbolData->setIndex(Index++);
-  for (unsigned i = 0, e = UndefinedSymbolData.size(); i != e; ++i)
-    UndefinedSymbolData[i].SymbolData->setIndex(Index++);
-
-  for (const MCSectionData &SD : Asm) {
-    std::vector<RelAndSymbol> &Relocs = Relocations[&SD];
-    for (RelAndSymbol &Rel : Relocs) {
+  for (auto *SymbolData :
+       {&LocalSymbolData, &ExternalSymbolData, &UndefinedSymbolData})
+    for (MachSymbolData &Entry : *SymbolData)
+      Entry.Symbol->setIndex(Index++);
+
+  for (const MCSection &Section : Asm) {
+    for (RelAndSymbol &Rel : Relocations[&Section]) {
       if (!Rel.Sym)
         continue;
 
@@ -625,7 +597,7 @@ void MachObjectWriter::ComputeSymbolTable(
       unsigned Index = Rel.Sym->getIndex();
       assert(isInt<24>(Index));
       if (IsLittleEndian)
-        Rel.MRE.r_word1 = (Rel.MRE.r_word1 & (-1 << 24)) | Index | (1 << 27);
+        Rel.MRE.r_word1 = (Rel.MRE.r_word1 & (~0U << 24)) | Index | (1 << 27);
       else
         Rel.MRE.r_word1 = (Rel.MRE.r_word1 & 0xff) | Index << 8 | (1 << 4);
     }
@@ -635,35 +607,29 @@ void MachObjectWriter::ComputeSymbolTable(
 void MachObjectWriter::computeSectionAddresses(const MCAssembler &Asm,
                                                const MCAsmLayout &Layout) {
   uint64_t StartAddress = 0;
-  const SmallVectorImpl<MCSectionData*> &Order = Layout.getSectionOrder();
-  for (int i = 0, n = Order.size(); i != n ; ++i) {
-    const MCSectionData *SD = Order[i];
-    StartAddress = RoundUpToAlignment(StartAddress, SD->getAlignment());
-    SectionAddress[SD] = StartAddress;
-    StartAddress += Layout.getSectionAddressSize(SD);
+  for (const MCSection *Sec : Layout.getSectionOrder()) {
+    StartAddress = RoundUpToAlignment(StartAddress, Sec->getAlignment());
+    SectionAddress[Sec] = StartAddress;
+    StartAddress += Layout.getSectionAddressSize(Sec);
 
     // Explicitly pad the section to match the alignment requirements of the
     // following one. This is for 'gas' compatibility, it shouldn't
     /// strictly be necessary.
-    StartAddress += getPaddingSize(SD, Layout);
+    StartAddress += getPaddingSize(Sec, Layout);
   }
 }
 
-void MachObjectWriter::ExecutePostLayoutBinding(MCAssembler &Asm,
+void MachObjectWriter::executePostLayoutBinding(MCAssembler &Asm,
                                                 const MCAsmLayout &Layout) {
   computeSectionAddresses(Asm, Layout);
 
   // Create symbol data for any indirect symbols.
-  BindIndirectSymbols(Asm);
+  bindIndirectSymbols(Asm);
 }
 
-bool MachObjectWriter::
-IsSymbolRefDifferenceFullyResolvedImpl(const MCAssembler &Asm,
-                                       const MCSymbolData &DataA,
-                                       const MCSymbolData *DataB,
-                                       const MCFragment &FB,
-                                       bool InSet,
-                                       bool IsPCRel) const {
+bool MachObjectWriter::isSymbolRefDifferenceFullyResolvedImpl(
+    const MCAssembler &Asm, const MCSymbol &SymA, const MCFragment &FB,
+    bool InSet, bool IsPCRel) const {
   if (InSet)
     return true;
 
@@ -672,11 +638,9 @@ IsSymbolRefDifferenceFullyResolvedImpl(const MCAssembler &Asm,
   //   - addr(atom(B)) - offset(B)
   // and the offsets are not relocatable, so the fixup is fully resolved when
   //  addr(atom(A)) - addr(atom(B)) == 0.
-  const MCSymbolData *A_Base = nullptr, *B_Base = nullptr;
-
-  const MCSymbol &SA = DataA.getSymbol().AliasedSymbol();
+  const MCSymbol &SA = findAliasedSymbol(SymA);
   const MCSection &SecA = SA.getSection();
-  const MCSection &SecB = FB.getParent()->getSection();
+  const MCSection &SecB = *FB.getParent();
 
   if (IsPCRel) {
     // The simple (Darwin, except on x86_64) way of dealing with this was to
@@ -695,8 +659,7 @@ IsSymbolRefDifferenceFullyResolvedImpl(const MCAssembler &Asm,
     bool hasReliableSymbolDifference = isX86_64();
     if (!hasReliableSymbolDifference) {
       if (!SA.isInSection() || &SecA != &SecB ||
-          (!SA.isTemporary() &&
-           FB.getAtom() != Asm.getSymbolData(SA).getFragment()->getAtom() &&
+          (!SA.isTemporary() && FB.getAtom() != SA.getFragment()->getAtom() &&
            Asm.getSubsectionsViaSymbols()))
         return false;
       return true;
@@ -711,36 +674,30 @@ IsSymbolRefDifferenceFullyResolvedImpl(const MCAssembler &Asm,
             SA.isTemporary() && SA.isInSection() && &SecA == &SecB){
       return true;
     }
-  } else {
-    if (!TargetObjectWriter->useAggressiveSymbolFolding())
-      return false;
   }
 
   // If they are not in the same section, we can't compute the diff.
   if (&SecA != &SecB)
     return false;
 
-  const MCFragment *FA = Asm.getSymbolData(SA).getFragment();
+  const MCFragment *FA = SA.getFragment();
 
   // Bail if the symbol has no fragment.
   if (!FA)
     return false;
 
-  A_Base = FA->getAtom();
-  B_Base = FB.getAtom();
-
   // If the atoms are the same, they are guaranteed to have the same address.
-  if (A_Base == B_Base)
+  if (FA->getAtom() == FB.getAtom())
     return true;
 
   // Otherwise, we can't prove this is fully resolved.
   return false;
 }
 
-void MachObjectWriter::WriteObject(MCAssembler &Asm,
+void MachObjectWriter::writeObject(MCAssembler &Asm,
                                    const MCAsmLayout &Layout) {
   // Compute symbol table information and bind symbol indices.
-  ComputeSymbolTable(Asm, LocalSymbolData, ExternalSymbolData,
+  computeSymbolTable(Asm, LocalSymbolData, ExternalSymbolData,
                      UndefinedSymbolData);
 
   unsigned NumSections = Asm.size();
@@ -785,12 +742,9 @@ void MachObjectWriter::WriteObject(MCAssembler &Asm,
   }
 
   // Add the linker option load commands sizes.
-  const std::vector<std::vector<std::string> > &LinkerOptions =
-    Asm.getLinkerOptions();
-  for (unsigned i = 0, e = LinkerOptions.size(); i != e; ++i) {
+  for (const auto &Option : Asm.getLinkerOptions()) {
     ++NumLoadCommands;
-    LoadCommandsSize += ComputeLinkerOptionsLoadCommandSize(LinkerOptions[i],
-                                                            is64Bit());
+    LoadCommandsSize += ComputeLinkerOptionsLoadCommandSize(Option, is64Bit());
   }
   
   // Compute the total size of the section data, as well as its file size and vm
@@ -800,17 +754,15 @@ void MachObjectWriter::WriteObject(MCAssembler &Asm,
   uint64_t SectionDataSize = 0;
   uint64_t SectionDataFileSize = 0;
   uint64_t VMSize = 0;
-  for (MCAssembler::const_iterator it = Asm.begin(),
-         ie = Asm.end(); it != ie; ++it) {
-    const MCSectionData &SD = *it;
-    uint64_t Address = getSectionAddress(&SD);
-    uint64_t Size = Layout.getSectionAddressSize(&SD);
-    uint64_t FileSize = Layout.getSectionFileSize(&SD);
-    FileSize += getPaddingSize(&SD, Layout);
+  for (const MCSection &Sec : Asm) {
+    uint64_t Address = getSectionAddress(&Sec);
+    uint64_t Size = Layout.getSectionAddressSize(&Sec);
+    uint64_t FileSize = Layout.getSectionFileSize(&Sec);
+    FileSize += getPaddingSize(&Sec, Layout);
 
     VMSize = std::max(VMSize, Address + Size);
 
-    if (SD.getSection().isVirtualSection())
+    if (Sec.isVirtualSection())
       continue;
 
     SectionDataSize = std::max(SectionDataSize, Address + Size);
@@ -824,19 +776,18 @@ void MachObjectWriter::WriteObject(MCAssembler &Asm,
   SectionDataFileSize += SectionDataPadding;
 
   // Write the prolog, starting with the header and load command...
-  WriteHeader(NumLoadCommands, LoadCommandsSize,
+  writeHeader(NumLoadCommands, LoadCommandsSize,
               Asm.getSubsectionsViaSymbols());
-  WriteSegmentLoadCommand(NumSections, VMSize,
+  writeSegmentLoadCommand(NumSections, VMSize,
                           SectionDataStart, SectionDataSize);
 
   // ... and then the section headers.
   uint64_t RelocTableEnd = SectionDataStart + SectionDataFileSize;
-  for (MCAssembler::const_iterator it = Asm.begin(),
-         ie = Asm.end(); it != ie; ++it) {
-    std::vector<RelAndSymbol> &Relocs = Relocations[it];
+  for (const MCSection &Sec : Asm) {
+    std::vector<RelAndSymbol> &Relocs = Relocations[&Sec];
     unsigned NumRelocs = Relocs.size();
-    uint64_t SectionStart = SectionDataStart + getSectionAddress(it);
-    WriteSection(Asm, Layout, *it, SectionStart, RelocTableEnd, NumRelocs);
+    uint64_t SectionStart = SectionDataStart + getSectionAddress(&Sec);
+    writeSection(Asm, Layout, Sec, SectionStart, RelocTableEnd, NumRelocs);
     RelocTableEnd += NumRelocs * sizeof(MachO::any_relocation_info);
   }
 
@@ -847,11 +798,11 @@ void MachObjectWriter::WriteObject(MCAssembler &Asm,
     assert(VersionInfo.Major < 65536 && "unencodable major target version");
     uint32_t EncodedVersion = VersionInfo.Update | (VersionInfo.Minor << 8) |
       (VersionInfo.Major << 16);
-    Write32(VersionInfo.Kind == MCVM_OSXVersionMin ? MachO::LC_VERSION_MIN_MACOSX :
+    write32(VersionInfo.Kind == MCVM_OSXVersionMin ? MachO::LC_VERSION_MIN_MACOSX :
             MachO::LC_VERSION_MIN_IPHONEOS);
-    Write32(sizeof(MachO::version_min_command));
-    Write32(EncodedVersion);
-    Write32(0);         // reserved.
+    write32(sizeof(MachO::version_min_command));
+    write32(EncodedVersion);
+    write32(0);         // reserved.
   }
 
   // Write the data-in-code load command, if used.
@@ -859,14 +810,14 @@ void MachObjectWriter::WriteObject(MCAssembler &Asm,
   if (NumDataRegions) {
     uint64_t DataRegionsOffset = RelocTableEnd;
     uint64_t DataRegionsSize = NumDataRegions * 8;
-    WriteLinkeditLoadCommand(MachO::LC_DATA_IN_CODE, DataRegionsOffset,
+    writeLinkeditLoadCommand(MachO::LC_DATA_IN_CODE, DataRegionsOffset,
                              DataRegionsSize);
   }
 
   // Write the loh load command, if used.
   uint64_t LOHTableEnd = DataInCodeTableEnd + LOHSize;
   if (LOHSize)
-    WriteLinkeditLoadCommand(MachO::LC_LINKER_OPTIMIZATION_HINT,
+    writeLinkeditLoadCommand(MachO::LC_LINKER_OPTIMIZATION_HINT,
                              DataInCodeTableEnd, LOHSize);
 
   // Write the symbol table load command, if used.
@@ -895,42 +846,38 @@ void MachObjectWriter::WriteObject(MCAssembler &Asm,
       SymbolTableOffset + NumSymTabSymbols * (is64Bit() ?
                                               sizeof(MachO::nlist_64) :
                                               sizeof(MachO::nlist));
-    WriteSymtabLoadCommand(SymbolTableOffset, NumSymTabSymbols,
+    writeSymtabLoadCommand(SymbolTableOffset, NumSymTabSymbols,
                            StringTableOffset, StringTable.data().size());
 
-    WriteDysymtabLoadCommand(FirstLocalSymbol, NumLocalSymbols,
+    writeDysymtabLoadCommand(FirstLocalSymbol, NumLocalSymbols,
                              FirstExternalSymbol, NumExternalSymbols,
                              FirstUndefinedSymbol, NumUndefinedSymbols,
                              IndirectSymbolOffset, NumIndirectSymbols);
   }
 
   // Write the linker options load commands.
-  for (unsigned i = 0, e = LinkerOptions.size(); i != e; ++i) {
-    WriteLinkerOptionsLoadCommand(LinkerOptions[i]);
-  }
+  for (const auto &Option : Asm.getLinkerOptions())
+    writeLinkerOptionsLoadCommand(Option);
 
   // Write the actual section data.
-  for (MCAssembler::const_iterator it = Asm.begin(),
-         ie = Asm.end(); it != ie; ++it) {
-    Asm.writeSectionData(it, Layout);
+  for (const MCSection &Sec : Asm) {
+    Asm.writeSectionData(&Sec, Layout);
 
-    uint64_t Pad = getPaddingSize(it, Layout);
-    for (unsigned int i = 0; i < Pad; ++i)
-      Write8(0);
+    uint64_t Pad = getPaddingSize(&Sec, Layout);
+    WriteZeros(Pad);
   }
 
   // Write the extra padding.
   WriteZeros(SectionDataPadding);
 
   // Write the relocation entries.
-  for (MCAssembler::const_iterator it = Asm.begin(),
-         ie = Asm.end(); it != ie; ++it) {
+  for (const MCSection &Sec : Asm) {
     // Write the section relocation entries, in reverse order to match 'as'
     // (approximately, the exact algorithm is more complicated than this).
-    std::vector<RelAndSymbol> &Relocs = Relocations[it];
-    for (unsigned i = 0, e = Relocs.size(); i != e; ++i) {
-      Write32(Relocs[e - i - 1].MRE.r_word0);
-      Write32(Relocs[e - i - 1].MRE.r_word1);
+    std::vector<RelAndSymbol> &Relocs = Relocations[&Sec];
+    for (const RelAndSymbol &Rel : make_range(Relocs.rbegin(), Relocs.rend())) {
+      write32(Rel.MRE.r_word0);
+      write32(Rel.MRE.r_word1);
     }
   }
 
@@ -939,20 +886,16 @@ void MachObjectWriter::WriteObject(MCAssembler &Asm,
          it = Asm.data_region_begin(), ie = Asm.data_region_end();
          it != ie; ++it) {
     const DataRegionData *Data = &(*it);
-    uint64_t Start =
-      getSymbolAddress(&Layout.getAssembler().getSymbolData(*Data->Start),
-                       Layout);
-    uint64_t End =
-      getSymbolAddress(&Layout.getAssembler().getSymbolData(*Data->End),
-                       Layout);
+    uint64_t Start = getSymbolAddress(*Data->Start, Layout);
+    uint64_t End = getSymbolAddress(*Data->End, Layout);
     DEBUG(dbgs() << "data in code region-- kind: " << Data->Kind
                  << "  start: " << Start << "(" << Data->Start->getName() << ")"
                  << "  end: " << End << "(" << Data->End->getName() << ")"
                  << "  size: " << End - Start
                  << "\n");
-    Write32(Start);
-    Write16(End - Start);
-    Write16(Data->Kind);
+    write32(Start);
+    write16(End - Start);
+    write16(Data->Kind);
   }
 
   // Write out the loh commands, if there is one.
@@ -960,9 +903,9 @@ void MachObjectWriter::WriteObject(MCAssembler &Asm,
 #ifndef NDEBUG
     unsigned Start = OS.tell();
 #endif
-    Asm.getLOHContainer().Emit(*this, Layout);
+    Asm.getLOHContainer().emit(*this, Layout);
     // Pad to a multiple of the pointer size.
-    WriteBytes("", OffsetToAlignment(LOHRawSize, is64Bit() ? 8 : 4));
+    writeBytes("", OffsetToAlignment(LOHRawSize, is64Bit() ? 8 : 4));
     assert(OS.tell() - Start == LOHSize);
   }
 
@@ -975,31 +918,28 @@ void MachObjectWriter::WriteObject(MCAssembler &Asm,
       // Indirect symbols in the non-lazy symbol pointer section have some
       // special handling.
       const MCSectionMachO &Section =
-        static_cast<const MCSectionMachO&>(it->SectionData->getSection());
+          static_cast<const MCSectionMachO &>(*it->Section);
       if (Section.getType() == MachO::S_NON_LAZY_SYMBOL_POINTERS) {
         // If this symbol is defined and internal, mark it as such.
-        if (it->Symbol->isDefined() &&
-            !Asm.getSymbolData(*it->Symbol).isExternal()) {
+        if (it->Symbol->isDefined() && !it->Symbol->isExternal()) {
           uint32_t Flags = MachO::INDIRECT_SYMBOL_LOCAL;
           if (it->Symbol->isAbsolute())
             Flags |= MachO::INDIRECT_SYMBOL_ABS;
-          Write32(Flags);
+          write32(Flags);
           continue;
         }
       }
 
-      Write32(Asm.getSymbolData(*it->Symbol).getIndex());
+      write32(it->Symbol->getIndex());
     }
 
     // FIXME: Check that offsets match computed ones.
 
     // Write the symbol table entries.
-    for (unsigned i = 0, e = LocalSymbolData.size(); i != e; ++i)
-      WriteNlist(LocalSymbolData[i], Layout);
-    for (unsigned i = 0, e = ExternalSymbolData.size(); i != e; ++i)
-      WriteNlist(ExternalSymbolData[i], Layout);
-    for (unsigned i = 0, e = UndefinedSymbolData.size(); i != e; ++i)
-      WriteNlist(UndefinedSymbolData[i], Layout);
+    for (auto *SymbolData :
+         {&LocalSymbolData, &ExternalSymbolData, &UndefinedSymbolData})
+      for (MachSymbolData &Entry : *SymbolData)
+        writeNlist(Entry, Layout);
 
     // Write the string table.
     OS << StringTable.data();
diff --git a/lib/MC/SubtargetFeature.cpp b/lib/MC/SubtargetFeature.cpp
index 3880d883389..76574e987cb 100644
--- a/lib/MC/SubtargetFeature.cpp
+++ b/lib/MC/SubtargetFeature.cpp
@@ -12,6 +12,7 @@
 //===----------------------------------------------------------------------===//
 
 #include "llvm/MC/SubtargetFeature.h"
+#include "llvm/ADT/StringExtras.h"
 #include "llvm/Support/Debug.h"
 #include "llvm/Support/Format.h"
 #include "llvm/Support/raw_ostream.h"
@@ -59,27 +60,6 @@ static void Split(std::vector<std::string> &V, StringRef S) {
   V.assign(Tmp.begin(), Tmp.end());
 }
 
-/// Join a vector of strings to a string with a comma separating each element.
-///
-static std::string Join(const std::vector<std::string> &V) {
-  // Start with empty string.
-  std::string Result;
-  // If the vector is not empty
-  if (!V.empty()) {
-    // Start with the first feature
-    Result = V[0];
-    // For each successive feature
-    for (size_t i = 1; i < V.size(); i++) {
-      // Add a comma
-      Result += ",";
-      // Add the feature
-      Result += V[i];
-    }
-  }
-  // Return the features string
-  return Result;
-}
-
 /// Adding features.
 void SubtargetFeatures::AddFeature(StringRef String, bool Enable) {
   // Don't add empty features.
@@ -144,19 +124,19 @@ SubtargetFeatures::SubtargetFeatures(StringRef Initial) {
 
 
 std::string SubtargetFeatures::getString() const {
-  return Join(Features);
+  return join(Features.begin(), Features.end(), ",");
 }
 
 /// SetImpliedBits - For each feature that is (transitively) implied by this
 /// feature, set it.
 ///
 static
-void SetImpliedBits(uint64_t &Bits, const SubtargetFeatureKV *FeatureEntry,
+void SetImpliedBits(FeatureBitset &Bits, const SubtargetFeatureKV *FeatureEntry,
                     ArrayRef<SubtargetFeatureKV> FeatureTable) {
   for (auto &FE : FeatureTable) {
     if (FeatureEntry->Value == FE.Value) continue;
 
-    if (FeatureEntry->Implies & FE.Value) {
+    if ((FeatureEntry->Implies & FE.Value).any()) {
       Bits |= FE.Value;
       SetImpliedBits(Bits, &FE, FeatureTable);
     }
@@ -167,12 +147,13 @@ void SetImpliedBits(uint64_t &Bits, const SubtargetFeatureKV *FeatureEntry,
 /// feature, clear it.
 ///
 static
-void ClearImpliedBits(uint64_t &Bits, const SubtargetFeatureKV *FeatureEntry,
+void ClearImpliedBits(FeatureBitset &Bits, 
+                      const SubtargetFeatureKV *FeatureEntry,
                       ArrayRef<SubtargetFeatureKV> FeatureTable) {
   for (auto &FE : FeatureTable) {
     if (FeatureEntry->Value == FE.Value) continue;
 
-    if (FE.Implies & FeatureEntry->Value) {
+    if ((FE.Implies & FeatureEntry->Value).any()) {
       Bits &= ~FE.Value;
       ClearImpliedBits(Bits, &FE, FeatureTable);
     }
@@ -181,8 +162,8 @@ void ClearImpliedBits(uint64_t &Bits, const SubtargetFeatureKV *FeatureEntry,
 
 /// ToggleFeature - Toggle a feature and returns the newly updated feature
 /// bits.
-uint64_t
-SubtargetFeatures::ToggleFeature(uint64_t Bits, StringRef Feature,
+FeatureBitset
+SubtargetFeatures::ToggleFeature(FeatureBitset Bits, StringRef Feature,
                                  ArrayRef<SubtargetFeatureKV> FeatureTable) {
 
   // Find feature in table.
@@ -192,7 +173,6 @@ SubtargetFeatures::ToggleFeature(uint64_t Bits, StringRef Feature,
   if (FeatureEntry) {
     if ((Bits & FeatureEntry->Value) == FeatureEntry->Value) {
       Bits &= ~FeatureEntry->Value;
-
       // For each feature that implies this, clear it.
       ClearImpliedBits(Bits, FeatureEntry, FeatureTable);
     } else {
@@ -210,16 +190,48 @@ SubtargetFeatures::ToggleFeature(uint64_t Bits, StringRef Feature,
   return Bits;
 }
 
+FeatureBitset
+SubtargetFeatures::ApplyFeatureFlag(FeatureBitset Bits, StringRef Feature,
+                                    ArrayRef<SubtargetFeatureKV> FeatureTable) {
+
+  assert(hasFlag(Feature));
+
+  // Find feature in table.
+  const SubtargetFeatureKV *FeatureEntry =
+      Find(StripFlag(Feature), FeatureTable);
+  // If there is a match
+  if (FeatureEntry) {
+    // Enable/disable feature in bits
+    if (isEnabled(Feature)) {
+      Bits |=  FeatureEntry->Value;
+
+      // For each feature that this implies, set it.
+      SetImpliedBits(Bits, FeatureEntry, FeatureTable);
+    } else {
+      Bits &= ~FeatureEntry->Value;
+
+      // For each feature that implies this, clear it.
+      ClearImpliedBits(Bits, FeatureEntry, FeatureTable);
+    }
+  } else {
+    errs() << "'" << Feature
+           << "' is not a recognized feature for this target"
+           << " (ignoring feature)\n";
+  }
+
+  return Bits;
+}
+
 
 /// getFeatureBits - Get feature bits a CPU.
 ///
-uint64_t
+FeatureBitset
 SubtargetFeatures::getFeatureBits(StringRef CPU,
                                   ArrayRef<SubtargetFeatureKV> CPUTable,
                                   ArrayRef<SubtargetFeatureKV> FeatureTable) {
 
   if (CPUTable.empty() || FeatureTable.empty())
-    return 0;
+    return FeatureBitset();
 
 #ifndef NDEBUG
   for (size_t i = 1, e = CPUTable.size(); i != e; ++i) {
@@ -231,7 +243,8 @@ SubtargetFeatures::getFeatureBits(StringRef CPU,
           "CPU features table is not sorted");
   }
 #endif
-  uint64_t Bits = 0;                    // Resulting bits
+  // Resulting bits
+  FeatureBitset Bits;
 
   // Check if help is needed
   if (CPU == "help")
@@ -248,7 +261,7 @@ SubtargetFeatures::getFeatureBits(StringRef CPU,
 
       // Set the feature implied by this CPU feature, if any.
       for (auto &FE : FeatureTable) {
-        if (CPUEntry->Value & FE.Value)
+        if ((CPUEntry->Value & FE.Value).any())
           SetImpliedBits(Bits, &FE, FeatureTable);
       }
     } else {
@@ -264,28 +277,7 @@ SubtargetFeatures::getFeatureBits(StringRef CPU,
     if (Feature == "+help")
       Help(CPUTable, FeatureTable);
 
-    // Find feature in table.
-    const SubtargetFeatureKV *FeatureEntry =
-        Find(StripFlag(Feature), FeatureTable);
-    // If there is a match
-    if (FeatureEntry) {
-      // Enable/disable feature in bits
-      if (isEnabled(Feature)) {
-        Bits |=  FeatureEntry->Value;
-
-        // For each feature that this implies, set it.
-        SetImpliedBits(Bits, FeatureEntry, FeatureTable);
-      } else {
-        Bits &= ~FeatureEntry->Value;
-
-        // For each feature that implies this, clear it.
-        ClearImpliedBits(Bits, FeatureEntry, FeatureTable);
-      }
-    } else {
-      errs() << "'" << Feature
-             << "' is not a recognized feature for this target"
-             << " (ignoring feature)\n";
-    }
+    Bits = ApplyFeatureFlag(Bits, Feature, FeatureTable);
   }
 
   return Bits;
diff --git a/lib/MC/WinCOFFObjectWriter.cpp b/lib/MC/WinCOFFObjectWriter.cpp
index 38bb88335dd..56ef1c7a273 100644
--- a/lib/MC/WinCOFFObjectWriter.cpp
+++ b/lib/MC/WinCOFFObjectWriter.cpp
@@ -21,10 +21,11 @@
 #include "llvm/MC/MCAssembler.h"
 #include "llvm/MC/MCContext.h"
 #include "llvm/MC/MCExpr.h"
+#include "llvm/MC/MCObjectFileInfo.h"
 #include "llvm/MC/MCObjectWriter.h"
 #include "llvm/MC/MCSection.h"
 #include "llvm/MC/MCSectionCOFF.h"
-#include "llvm/MC/MCSymbol.h"
+#include "llvm/MC/MCSymbolCOFF.h"
 #include "llvm/MC/MCValue.h"
 #include "llvm/MC/StringTableBuilder.h"
 #include "llvm/Support/COFF.h"
@@ -50,7 +51,7 @@ enum AuxiliaryType {
 };
 
 struct AuxSymbol {
-  AuxiliaryType   AuxType;
+  AuxiliaryType AuxType;
   COFF::Auxiliary Aux;
 };
 
@@ -63,25 +64,32 @@ class COFFSymbol {
 
   typedef SmallVector<AuxSymbol, 1> AuxiliarySymbols;
 
-  name             Name;
-  int              Index;
+  name Name;
+  int Index;
   AuxiliarySymbols Aux;
-  COFFSymbol      *Other;
-  COFFSection     *Section;
-  int              Relocations;
+  COFFSymbol *Other;
+  COFFSection *Section;
+  int Relocations;
 
-  MCSymbolData const *MCData;
+  const MCSymbol *MC;
 
   COFFSymbol(StringRef name);
   void set_name_offset(uint32_t Offset);
 
   bool should_keep() const;
+
+  int64_t getIndex() const { return Index; }
+  void setIndex(int Value) {
+    Index = Value;
+    if (MC)
+      MC->setIndex(static_cast<uint32_t>(Value));
+  }
 };
 
 // This class contains staging data for a COFF relocation entry.
 struct COFFRelocation {
   COFF::relocation Data;
-  COFFSymbol          *Symb;
+  COFFSymbol *Symb;
 
   COFFRelocation() : Symb(nullptr) {}
   static size_t size() { return COFF::RelocationSize; }
@@ -93,11 +101,11 @@ class COFFSection {
 public:
   COFF::section Header;
 
-  std::string          Name;
-  int                  Number;
-  MCSectionData const *MCData;
-  COFFSymbol          *Symbol;
-  relocations          Relocations;
+  std::string Name;
+  int Number;
+  MCSectionCOFF const *MCSection;
+  COFFSymbol *Symbol;
+  relocations Relocations;
 
   COFFSection(StringRef name);
   static size_t size();
@@ -105,24 +113,23 @@ class COFFSection {
 
 class WinCOFFObjectWriter : public MCObjectWriter {
 public:
-
-  typedef std::vector<std::unique_ptr<COFFSymbol>>  symbols;
+  typedef std::vector<std::unique_ptr<COFFSymbol>> symbols;
   typedef std::vector<std::unique_ptr<COFFSection>> sections;
 
-  typedef DenseMap<MCSymbol  const *, COFFSymbol *>   symbol_map;
+  typedef DenseMap<MCSymbol const *, COFFSymbol *> symbol_map;
   typedef DenseMap<MCSection const *, COFFSection *> section_map;
 
   std::unique_ptr<MCWinCOFFObjectTargetWriter> TargetObjectWriter;
 
   // Root level file contents.
   COFF::header Header;
-  sections     Sections;
-  symbols      Symbols;
+  sections Sections;
+  symbols Symbols;
   StringTableBuilder Strings;
 
   // Maps used during object file creation.
   section_map SectionMap;
-  symbol_map  SymbolMap;
+  symbol_map SymbolMap;
 
   bool UseBigObj;
 
@@ -140,14 +147,14 @@ class WinCOFFObjectWriter : public MCObjectWriter {
   }
 
   COFFSymbol *createSymbol(StringRef Name);
-  COFFSymbol *GetOrCreateCOFFSymbol(const MCSymbol * Symbol);
+  COFFSymbol *GetOrCreateCOFFSymbol(const MCSymbol *Symbol);
   COFFSection *createSection(StringRef Name);
 
   template <typename object_t, typename list_t>
   object_t *createCOFFEntity(StringRef Name, list_t &List);
 
-  void DefineSection(MCSectionData const &SectionData);
-  void DefineSymbol(MCSymbolData const &SymbolData, MCAssembler &Assembler,
+  void defineSection(MCSectionCOFF const &Sec);
+  void DefineSymbol(const MCSymbol &Symbol, MCAssembler &Assembler,
                     const MCAsmLayout &Layout);
 
   void SetSymbolName(COFFSymbol &S);
@@ -162,28 +169,27 @@ class WinCOFFObjectWriter : public MCObjectWriter {
   void WriteFileHeader(const COFF::header &Header);
   void WriteSymbol(const COFFSymbol &S);
   void WriteAuxiliarySymbols(const COFFSymbol::AuxiliarySymbols &S);
-  void WriteSectionHeader(const COFF::section &S);
+  void writeSectionHeader(const COFF::section &S);
   void WriteRelocation(const COFF::relocation &R);
 
   // MCObjectWriter interface implementation.
 
-  void ExecutePostLayoutBinding(MCAssembler &Asm,
+  void executePostLayoutBinding(MCAssembler &Asm,
                                 const MCAsmLayout &Layout) override;
 
-  bool IsSymbolRefDifferenceFullyResolvedImpl(const MCAssembler &Asm,
-                                              const MCSymbolData &DataA,
-                                              const MCSymbolData *DataB,
+  bool isSymbolRefDifferenceFullyResolvedImpl(const MCAssembler &Asm,
+                                              const MCSymbol &SymA,
                                               const MCFragment &FB, bool InSet,
                                               bool IsPCRel) const override;
 
-  bool isWeak(const MCSymbolData &SD) const override;
+  bool isWeak(const MCSymbol &Sym) const override;
 
-  void RecordRelocation(MCAssembler &Asm, const MCAsmLayout &Layout,
+  void recordRelocation(MCAssembler &Asm, const MCAsmLayout &Layout,
                         const MCFragment *Fragment, const MCFixup &Fixup,
                         MCValue Target, bool &IsPCRel,
                         uint64_t &FixedValue) override;
 
-  void WriteObject(MCAssembler &Asm, const MCAsmLayout &Layout) override;
+  void writeObject(MCAssembler &Asm, const MCAsmLayout &Layout) override;
 };
 }
 
@@ -196,11 +202,8 @@ static inline void write_uint32_le(void *Data, uint32_t Value) {
 // Symbol class implementation
 
 COFFSymbol::COFFSymbol(StringRef name)
-  : Name(name.begin(), name.end())
-  , Other(nullptr)
-  , Section(nullptr)
-  , Relocations(0)
-  , MCData(nullptr) {
+    : Name(name.begin(), name.end()), Other(nullptr), Section(nullptr),
+      Relocations(0), MC(nullptr) {
   memset(&Data, 0, sizeof(Data));
 }
 
@@ -219,22 +222,26 @@ bool COFFSymbol::should_keep() const {
     return true;
 
   // if it has relocations pointing at it, keep it
-  if (Relocations > 0)   {
+  if (Relocations > 0) {
     assert(Section->Number != -1 && "Sections with relocations must be real!");
     return true;
   }
 
+  // if this is a safeseh handler, keep it
+  if (MC && (cast<MCSymbolCOFF>(MC)->isSafeSEH()))
+    return true;
+
   // if the section its in is being droped, drop it
   if (Section->Number == -1)
-      return false;
+    return false;
 
   // if it is the section symbol, keep it
   if (Section->Symbol == this)
     return true;
 
   // if its temporary, drop it
-  if (MCData && MCData->getSymbol().isTemporary())
-      return false;
+  if (MC && MC->isTemporary())
+    return false;
 
   // otherwise, keep it
   return true;
@@ -244,15 +251,11 @@ bool COFFSymbol::should_keep() const {
 // Section class implementation
 
 COFFSection::COFFSection(StringRef name)
-  : Name(name)
-  , MCData(nullptr)
-  , Symbol(nullptr) {
+    : Name(name), MCSection(nullptr), Symbol(nullptr) {
   memset(&Header, 0, sizeof(Header));
 }
 
-size_t COFFSection::size() {
-  return COFF::SectionSize;
-}
+size_t COFFSection::size() { return COFF::SectionSize; }
 
 //------------------------------------------------------------------------------
 // WinCOFFObjectWriter class implementation
@@ -286,8 +289,7 @@ COFFSection *WinCOFFObjectWriter::createSection(StringRef Name) {
 /// A template used to lookup or create a symbol/section, and initialize it if
 /// needed.
 template <typename object_t, typename list_t>
-object_t *WinCOFFObjectWriter::createCOFFEntity(StringRef Name,
-                                                list_t &List) {
+object_t *WinCOFFObjectWriter::createCOFFEntity(StringRef Name, list_t &List) {
   List.push_back(make_unique<object_t>(Name));
 
   return List.back().get();
@@ -295,15 +297,9 @@ object_t *WinCOFFObjectWriter::createCOFFEntity(StringRef Name,
 
 /// This function takes a section data object from the assembler
 /// and creates the associated COFF section staging object.
-void WinCOFFObjectWriter::DefineSection(MCSectionData const &SectionData) {
-  assert(SectionData.getSection().getVariant() == MCSection::SV_COFF
-    && "Got non-COFF section in the COFF backend!");
-  // FIXME: Not sure how to verify this (at least in a debug build).
-  MCSectionCOFF const &Sec =
-    static_cast<MCSectionCOFF const &>(SectionData.getSection());
-
+void WinCOFFObjectWriter::defineSection(MCSectionCOFF const &Sec) {
   COFFSection *coff_section = createSection(Sec.getSectionName());
-  COFFSymbol  *coff_symbol = createSymbol(Sec.getSectionName());
+  COFFSymbol *coff_symbol = createSymbol(Sec.getSectionName());
   if (Sec.getSelection() != COFF::IMAGE_COMDAT_SELECT_ASSOCIATIVE) {
     if (const MCSymbol *S = Sec.getCOMDATSymbol()) {
       COFFSymbol *COMDATSymbol = GetOrCreateCOFFSymbol(S);
@@ -326,37 +322,65 @@ void WinCOFFObjectWriter::DefineSection(MCSectionData const &SectionData) {
   coff_section->Header.Characteristics = Sec.getCharacteristics();
 
   uint32_t &Characteristics = coff_section->Header.Characteristics;
-  switch (SectionData.getAlignment()) {
-  case 1:    Characteristics |= COFF::IMAGE_SCN_ALIGN_1BYTES;    break;
-  case 2:    Characteristics |= COFF::IMAGE_SCN_ALIGN_2BYTES;    break;
-  case 4:    Characteristics |= COFF::IMAGE_SCN_ALIGN_4BYTES;    break;
-  case 8:    Characteristics |= COFF::IMAGE_SCN_ALIGN_8BYTES;    break;
-  case 16:   Characteristics |= COFF::IMAGE_SCN_ALIGN_16BYTES;   break;
-  case 32:   Characteristics |= COFF::IMAGE_SCN_ALIGN_32BYTES;   break;
-  case 64:   Characteristics |= COFF::IMAGE_SCN_ALIGN_64BYTES;   break;
-  case 128:  Characteristics |= COFF::IMAGE_SCN_ALIGN_128BYTES;  break;
-  case 256:  Characteristics |= COFF::IMAGE_SCN_ALIGN_256BYTES;  break;
-  case 512:  Characteristics |= COFF::IMAGE_SCN_ALIGN_512BYTES;  break;
-  case 1024: Characteristics |= COFF::IMAGE_SCN_ALIGN_1024BYTES; break;
-  case 2048: Characteristics |= COFF::IMAGE_SCN_ALIGN_2048BYTES; break;
-  case 4096: Characteristics |= COFF::IMAGE_SCN_ALIGN_4096BYTES; break;
-  case 8192: Characteristics |= COFF::IMAGE_SCN_ALIGN_8192BYTES; break;
+  switch (Sec.getAlignment()) {
+  case 1:
+    Characteristics |= COFF::IMAGE_SCN_ALIGN_1BYTES;
+    break;
+  case 2:
+    Characteristics |= COFF::IMAGE_SCN_ALIGN_2BYTES;
+    break;
+  case 4:
+    Characteristics |= COFF::IMAGE_SCN_ALIGN_4BYTES;
+    break;
+  case 8:
+    Characteristics |= COFF::IMAGE_SCN_ALIGN_8BYTES;
+    break;
+  case 16:
+    Characteristics |= COFF::IMAGE_SCN_ALIGN_16BYTES;
+    break;
+  case 32:
+    Characteristics |= COFF::IMAGE_SCN_ALIGN_32BYTES;
+    break;
+  case 64:
+    Characteristics |= COFF::IMAGE_SCN_ALIGN_64BYTES;
+    break;
+  case 128:
+    Characteristics |= COFF::IMAGE_SCN_ALIGN_128BYTES;
+    break;
+  case 256:
+    Characteristics |= COFF::IMAGE_SCN_ALIGN_256BYTES;
+    break;
+  case 512:
+    Characteristics |= COFF::IMAGE_SCN_ALIGN_512BYTES;
+    break;
+  case 1024:
+    Characteristics |= COFF::IMAGE_SCN_ALIGN_1024BYTES;
+    break;
+  case 2048:
+    Characteristics |= COFF::IMAGE_SCN_ALIGN_2048BYTES;
+    break;
+  case 4096:
+    Characteristics |= COFF::IMAGE_SCN_ALIGN_4096BYTES;
+    break;
+  case 8192:
+    Characteristics |= COFF::IMAGE_SCN_ALIGN_8192BYTES;
+    break;
   default:
     llvm_unreachable("unsupported section alignment");
   }
 
   // Bind internal COFF section to MC section.
-  coff_section->MCData = &SectionData;
-  SectionMap[&SectionData.getSection()] = coff_section;
+  coff_section->MCSection = &Sec;
+  SectionMap[&Sec] = coff_section;
 }
 
-static uint64_t getSymbolValue(const MCSymbolData &Data,
+static uint64_t getSymbolValue(const MCSymbol &Symbol,
                                const MCAsmLayout &Layout) {
-  if (Data.isCommon() && Data.isExternal())
-    return Data.getCommonSize();
+  if (Symbol.isCommon() && Symbol.isExternal())
+    return Symbol.getCommonSize();
 
   uint64_t Res;
-  if (!Layout.getSymbolOffset(&Data, Res))
+  if (!Layout.getSymbolOffset(Symbol, Res))
     return 0;
 
   return Res;
@@ -364,19 +388,18 @@ static uint64_t getSymbolValue(const MCSymbolData &Data,
 
 /// This function takes a symbol data object from the assembler
 /// and creates the associated COFF symbol staging object.
-void WinCOFFObjectWriter::DefineSymbol(MCSymbolData const &SymbolData,
+void WinCOFFObjectWriter::DefineSymbol(const MCSymbol &Symbol,
                                        MCAssembler &Assembler,
                                        const MCAsmLayout &Layout) {
-  MCSymbol const &Symbol = SymbolData.getSymbol();
   COFFSymbol *coff_symbol = GetOrCreateCOFFSymbol(&Symbol);
   SymbolMap[&Symbol] = coff_symbol;
 
-  if (SymbolData.getFlags() & COFF::SF_WeakExternal) {
+  if (cast<MCSymbolCOFF>(Symbol).isWeakExternal()) {
     coff_symbol->Data.StorageClass = COFF::IMAGE_SYM_CLASS_WEAK_EXTERNAL;
 
     if (Symbol.isVariable()) {
       const MCSymbolRefExpr *SymRef =
-        dyn_cast<MCSymbolRefExpr>(Symbol.getVariableValue());
+          dyn_cast<MCSymbolRefExpr>(Symbol.getVariableValue());
 
       if (!SymRef)
         report_fatal_error("Weak externals may only alias symbols");
@@ -386,9 +409,9 @@ void WinCOFFObjectWriter::DefineSymbol(MCSymbolData const &SymbolData,
       std::string WeakName = (".weak." + Symbol.getName() + ".default").str();
       COFFSymbol *WeakDefault = createSymbol(WeakName);
       WeakDefault->Data.SectionNumber = COFF::IMAGE_SYM_ABSOLUTE;
-      WeakDefault->Data.StorageClass  = COFF::IMAGE_SYM_CLASS_EXTERNAL;
-      WeakDefault->Data.Type          = 0;
-      WeakDefault->Data.Value         = 0;
+      WeakDefault->Data.StorageClass = COFF::IMAGE_SYM_CLASS_EXTERNAL;
+      WeakDefault->Data.Type = 0;
+      WeakDefault->Data.Value = 0;
       coff_symbol->Other = WeakDefault;
     }
 
@@ -398,22 +421,21 @@ void WinCOFFObjectWriter::DefineSymbol(MCSymbolData const &SymbolData,
     coff_symbol->Aux[0].AuxType = ATWeakExternal;
     coff_symbol->Aux[0].Aux.WeakExternal.TagIndex = 0;
     coff_symbol->Aux[0].Aux.WeakExternal.Characteristics =
-      COFF::IMAGE_WEAK_EXTERN_SEARCH_LIBRARY;
+        COFF::IMAGE_WEAK_EXTERN_SEARCH_LIBRARY;
 
-    coff_symbol->MCData = &SymbolData;
+    coff_symbol->MC = &Symbol;
   } else {
-    const MCSymbolData &ResSymData = Assembler.getSymbolData(Symbol);
     const MCSymbol *Base = Layout.getBaseSymbol(Symbol);
-    coff_symbol->Data.Value = getSymbolValue(ResSymData, Layout);
+    coff_symbol->Data.Value = getSymbolValue(Symbol, Layout);
 
-    coff_symbol->Data.Type         = (ResSymData.getFlags() & 0x0000FFFF) >>  0;
-    coff_symbol->Data.StorageClass = (ResSymData.getFlags() & 0x00FF0000) >> 16;
+    const MCSymbolCOFF &SymbolCOFF = cast<MCSymbolCOFF>(Symbol);
+    coff_symbol->Data.Type = SymbolCOFF.getType();
+    coff_symbol->Data.StorageClass = SymbolCOFF.getClass();
 
     // If no storage class was specified in the streamer, define it here.
-    if (coff_symbol->Data.StorageClass == 0) {
-      bool IsExternal =
-          ResSymData.isExternal() ||
-          (!ResSymData.getFragment() && !ResSymData.getSymbol().isVariable());
+    if (coff_symbol->Data.StorageClass == COFF::IMAGE_SYM_CLASS_NULL) {
+      bool IsExternal = Symbol.isExternal() ||
+                        (!Symbol.getFragment() && !Symbol.isVariable());
 
       coff_symbol->Data.StorageClass = IsExternal
                                            ? COFF::IMAGE_SYM_CLASS_EXTERNAL
@@ -423,10 +445,8 @@ void WinCOFFObjectWriter::DefineSymbol(MCSymbolData const &SymbolData,
     if (!Base) {
       coff_symbol->Data.SectionNumber = COFF::IMAGE_SYM_ABSOLUTE;
     } else {
-      const MCSymbolData &BaseData = Assembler.getSymbolData(*Base);
-      if (BaseData.getFragment()) {
-        COFFSection *Sec =
-            SectionMap[&BaseData.getFragment()->getParent()->getSection()];
+      if (Base->getFragment()) {
+        COFFSection *Sec = SectionMap[Base->getFragment()->getParent()];
 
         if (coff_symbol->Section && coff_symbol->Section != Sec)
           report_fatal_error("conflicting sections for symbol");
@@ -435,7 +455,7 @@ void WinCOFFObjectWriter::DefineSymbol(MCSymbolData const &SymbolData,
       }
     }
 
-    coff_symbol->MCData = &ResSymData;
+    coff_symbol->MC = &Symbol;
   }
 }
 
@@ -447,7 +467,7 @@ static const uint64_t MaxBase64Offset = 0xFFFFFFFFFULL; // 64^6, including 0
 // Encode a string table entry offset in base 64, padded to 6 chars, and
 // prefixed with a double slash: '//AAAAAA', '//AAAAAB', ...
 // Buffer must be at least 8 bytes large. No terminating null appended.
-static void encodeBase64StringEntry(char* Buffer, uint64_t Value) {
+static void encodeBase64StringEntry(char *Buffer, uint64_t Value) {
   assert(Value > Max7DecimalOffset && Value <= MaxBase64Offset &&
          "Illegal section name encoding for value");
 
@@ -458,7 +478,7 @@ static void encodeBase64StringEntry(char* Buffer, uint64_t Value) {
   Buffer[0] = '/';
   Buffer[1] = '/';
 
-  char* Ptr = Buffer + 7;
+  char *Ptr = Buffer + 7;
   for (unsigned i = 0; i < 6; ++i) {
     unsigned Rem = Value % 64;
     Value /= 64;
@@ -475,7 +495,7 @@ void WinCOFFObjectWriter::SetSectionName(COFFSection &S) {
     } else if (StringTableEntry <= Max7DecimalOffset) {
       // With seven digits, we have to skip the terminating null. Because
       // sprintf always appends it, we use a larger temporary buffer.
-      char buffer[9] = { };
+      char buffer[9] = {};
       std::sprintf(buffer, "/%d", unsigned(StringTableEntry));
       std::memcpy(S.Header.Name, buffer, 8);
     } else if (StringTableEntry <= MaxBase64Offset) {
@@ -506,18 +526,17 @@ bool WinCOFFObjectWriter::ExportSymbol(const MCSymbol &Symbol,
   if (!Symbol.isTemporary())
     return true;
 
-  // Absolute temporary labels are never visible.
-  if (!Symbol.isInSection())
+  // Temporary variable symbols are invisible.
+  if (Symbol.isVariable())
     return false;
 
-  // For now, all non-variable symbols are exported,
-  // the linker will sort the rest out for us.
-  return !Symbol.isVariable();
+  // Absolute temporary labels are never visible.
+  return !Symbol.isAbsolute();
 }
 
 bool WinCOFFObjectWriter::IsPhysicalSection(COFFSection *S) {
-  return (S->Header.Characteristics
-         & COFF::IMAGE_SCN_CNT_UNINITIALIZED_DATA) == 0;
+  return (S->Header.Characteristics & COFF::IMAGE_SCN_CNT_UNINITIALIZED_DATA) ==
+         0;
 }
 
 //------------------------------------------------------------------------------
@@ -525,88 +544,87 @@ bool WinCOFFObjectWriter::IsPhysicalSection(COFFSection *S) {
 
 void WinCOFFObjectWriter::WriteFileHeader(const COFF::header &Header) {
   if (UseBigObj) {
-    WriteLE16(COFF::IMAGE_FILE_MACHINE_UNKNOWN);
-    WriteLE16(0xFFFF);
-    WriteLE16(COFF::BigObjHeader::MinBigObjectVersion);
-    WriteLE16(Header.Machine);
-    WriteLE32(Header.TimeDateStamp);
-    for (uint8_t MagicChar : COFF::BigObjMagic)
-      Write8(MagicChar);
-    WriteLE32(0);
-    WriteLE32(0);
-    WriteLE32(0);
-    WriteLE32(0);
-    WriteLE32(Header.NumberOfSections);
-    WriteLE32(Header.PointerToSymbolTable);
-    WriteLE32(Header.NumberOfSymbols);
+    writeLE16(COFF::IMAGE_FILE_MACHINE_UNKNOWN);
+    writeLE16(0xFFFF);
+    writeLE16(COFF::BigObjHeader::MinBigObjectVersion);
+    writeLE16(Header.Machine);
+    writeLE32(Header.TimeDateStamp);
+    writeBytes(StringRef(COFF::BigObjMagic, sizeof(COFF::BigObjMagic)));
+    writeLE32(0);
+    writeLE32(0);
+    writeLE32(0);
+    writeLE32(0);
+    writeLE32(Header.NumberOfSections);
+    writeLE32(Header.PointerToSymbolTable);
+    writeLE32(Header.NumberOfSymbols);
   } else {
-    WriteLE16(Header.Machine);
-    WriteLE16(static_cast<int16_t>(Header.NumberOfSections));
-    WriteLE32(Header.TimeDateStamp);
-    WriteLE32(Header.PointerToSymbolTable);
-    WriteLE32(Header.NumberOfSymbols);
-    WriteLE16(Header.SizeOfOptionalHeader);
-    WriteLE16(Header.Characteristics);
+    writeLE16(Header.Machine);
+    writeLE16(static_cast<int16_t>(Header.NumberOfSections));
+    writeLE32(Header.TimeDateStamp);
+    writeLE32(Header.PointerToSymbolTable);
+    writeLE32(Header.NumberOfSymbols);
+    writeLE16(Header.SizeOfOptionalHeader);
+    writeLE16(Header.Characteristics);
   }
 }
 
 void WinCOFFObjectWriter::WriteSymbol(const COFFSymbol &S) {
-  WriteBytes(StringRef(S.Data.Name, COFF::NameSize));
-  WriteLE32(S.Data.Value);
+  writeBytes(StringRef(S.Data.Name, COFF::NameSize));
+  writeLE32(S.Data.Value);
   if (UseBigObj)
-    WriteLE32(S.Data.SectionNumber);
+    writeLE32(S.Data.SectionNumber);
   else
-    WriteLE16(static_cast<int16_t>(S.Data.SectionNumber));
-  WriteLE16(S.Data.Type);
-  Write8(S.Data.StorageClass);
-  Write8(S.Data.NumberOfAuxSymbols);
+    writeLE16(static_cast<int16_t>(S.Data.SectionNumber));
+  writeLE16(S.Data.Type);
+  write8(S.Data.StorageClass);
+  write8(S.Data.NumberOfAuxSymbols);
   WriteAuxiliarySymbols(S.Aux);
 }
 
 void WinCOFFObjectWriter::WriteAuxiliarySymbols(
-                                        const COFFSymbol::AuxiliarySymbols &S) {
-  for(COFFSymbol::AuxiliarySymbols::const_iterator i = S.begin(), e = S.end();
-      i != e; ++i) {
-    switch(i->AuxType) {
+    const COFFSymbol::AuxiliarySymbols &S) {
+  for (COFFSymbol::AuxiliarySymbols::const_iterator i = S.begin(), e = S.end();
+       i != e; ++i) {
+    switch (i->AuxType) {
     case ATFunctionDefinition:
-      WriteLE32(i->Aux.FunctionDefinition.TagIndex);
-      WriteLE32(i->Aux.FunctionDefinition.TotalSize);
-      WriteLE32(i->Aux.FunctionDefinition.PointerToLinenumber);
-      WriteLE32(i->Aux.FunctionDefinition.PointerToNextFunction);
+      writeLE32(i->Aux.FunctionDefinition.TagIndex);
+      writeLE32(i->Aux.FunctionDefinition.TotalSize);
+      writeLE32(i->Aux.FunctionDefinition.PointerToLinenumber);
+      writeLE32(i->Aux.FunctionDefinition.PointerToNextFunction);
       WriteZeros(sizeof(i->Aux.FunctionDefinition.unused));
       if (UseBigObj)
         WriteZeros(COFF::Symbol32Size - COFF::Symbol16Size);
       break;
     case ATbfAndefSymbol:
       WriteZeros(sizeof(i->Aux.bfAndefSymbol.unused1));
-      WriteLE16(i->Aux.bfAndefSymbol.Linenumber);
+      writeLE16(i->Aux.bfAndefSymbol.Linenumber);
       WriteZeros(sizeof(i->Aux.bfAndefSymbol.unused2));
-      WriteLE32(i->Aux.bfAndefSymbol.PointerToNextFunction);
+      writeLE32(i->Aux.bfAndefSymbol.PointerToNextFunction);
       WriteZeros(sizeof(i->Aux.bfAndefSymbol.unused3));
       if (UseBigObj)
         WriteZeros(COFF::Symbol32Size - COFF::Symbol16Size);
       break;
     case ATWeakExternal:
-      WriteLE32(i->Aux.WeakExternal.TagIndex);
-      WriteLE32(i->Aux.WeakExternal.Characteristics);
+      writeLE32(i->Aux.WeakExternal.TagIndex);
+      writeLE32(i->Aux.WeakExternal.Characteristics);
       WriteZeros(sizeof(i->Aux.WeakExternal.unused));
       if (UseBigObj)
         WriteZeros(COFF::Symbol32Size - COFF::Symbol16Size);
       break;
     case ATFile:
-      WriteBytes(
+      writeBytes(
           StringRef(reinterpret_cast<const char *>(&i->Aux),
                     UseBigObj ? COFF::Symbol32Size : COFF::Symbol16Size));
       break;
     case ATSectionDefinition:
-      WriteLE32(i->Aux.SectionDefinition.Length);
-      WriteLE16(i->Aux.SectionDefinition.NumberOfRelocations);
-      WriteLE16(i->Aux.SectionDefinition.NumberOfLinenumbers);
-      WriteLE32(i->Aux.SectionDefinition.CheckSum);
-      WriteLE16(static_cast<int16_t>(i->Aux.SectionDefinition.Number));
-      Write8(i->Aux.SectionDefinition.Selection);
+      writeLE32(i->Aux.SectionDefinition.Length);
+      writeLE16(i->Aux.SectionDefinition.NumberOfRelocations);
+      writeLE16(i->Aux.SectionDefinition.NumberOfLinenumbers);
+      writeLE32(i->Aux.SectionDefinition.CheckSum);
+      writeLE16(static_cast<int16_t>(i->Aux.SectionDefinition.Number));
+      write8(i->Aux.SectionDefinition.Selection);
       WriteZeros(sizeof(i->Aux.SectionDefinition.unused));
-      WriteLE16(static_cast<int16_t>(i->Aux.SectionDefinition.Number >> 16));
+      writeLE16(static_cast<int16_t>(i->Aux.SectionDefinition.Number >> 16));
       if (UseBigObj)
         WriteZeros(COFF::Symbol32Size - COFF::Symbol16Size);
       break;
@@ -614,99 +632,106 @@ void WinCOFFObjectWriter::WriteAuxiliarySymbols(
   }
 }
 
-void WinCOFFObjectWriter::WriteSectionHeader(const COFF::section &S) {
-  WriteBytes(StringRef(S.Name, COFF::NameSize));
-
-  WriteLE32(S.VirtualSize);
-  WriteLE32(S.VirtualAddress);
-  WriteLE32(S.SizeOfRawData);
-  WriteLE32(S.PointerToRawData);
-  WriteLE32(S.PointerToRelocations);
-  WriteLE32(S.PointerToLineNumbers);
-  WriteLE16(S.NumberOfRelocations);
-  WriteLE16(S.NumberOfLineNumbers);
-  WriteLE32(S.Characteristics);
+void WinCOFFObjectWriter::writeSectionHeader(const COFF::section &S) {
+  writeBytes(StringRef(S.Name, COFF::NameSize));
+
+  writeLE32(S.VirtualSize);
+  writeLE32(S.VirtualAddress);
+  writeLE32(S.SizeOfRawData);
+  writeLE32(S.PointerToRawData);
+  writeLE32(S.PointerToRelocations);
+  writeLE32(S.PointerToLineNumbers);
+  writeLE16(S.NumberOfRelocations);
+  writeLE16(S.NumberOfLineNumbers);
+  writeLE32(S.Characteristics);
 }
 
 void WinCOFFObjectWriter::WriteRelocation(const COFF::relocation &R) {
-  WriteLE32(R.VirtualAddress);
-  WriteLE32(R.SymbolTableIndex);
-  WriteLE16(R.Type);
+  writeLE32(R.VirtualAddress);
+  writeLE32(R.SymbolTableIndex);
+  writeLE16(R.Type);
 }
 
 ////////////////////////////////////////////////////////////////////////////////
 // MCObjectWriter interface implementations
 
-void WinCOFFObjectWriter::ExecutePostLayoutBinding(MCAssembler &Asm,
+void WinCOFFObjectWriter::executePostLayoutBinding(MCAssembler &Asm,
                                                    const MCAsmLayout &Layout) {
   // "Define" each section & symbol. This creates section & symbol
   // entries in the staging area.
   for (const auto &Section : Asm)
-    DefineSection(Section);
+    defineSection(static_cast<const MCSectionCOFF &>(Section));
 
-  for (MCSymbolData &SD : Asm.symbols())
-    if (ExportSymbol(SD.getSymbol(), Asm))
-      DefineSymbol(SD, Asm, Layout);
+  for (const MCSymbol &Symbol : Asm.symbols())
+    if (ExportSymbol(Symbol, Asm))
+      DefineSymbol(Symbol, Asm, Layout);
 }
 
-bool WinCOFFObjectWriter::IsSymbolRefDifferenceFullyResolvedImpl(
-    const MCAssembler &Asm, const MCSymbolData &DataA,
-    const MCSymbolData *DataB, const MCFragment &FB, bool InSet,
-    bool IsPCRel) const {
+bool WinCOFFObjectWriter::isSymbolRefDifferenceFullyResolvedImpl(
+    const MCAssembler &Asm, const MCSymbol &SymA, const MCFragment &FB,
+    bool InSet, bool IsPCRel) const {
   // MS LINK expects to be able to replace all references to a function with a
   // thunk to implement their /INCREMENTAL feature.  Make sure we don't optimize
   // away any relocations to functions.
-  if ((((DataA.getFlags() & COFF::SF_TypeMask) >> COFF::SF_TypeShift) >>
-       COFF::SCT_COMPLEX_TYPE_SHIFT) == COFF::IMAGE_SYM_DTYPE_FUNCTION)
+  uint16_t Type = cast<MCSymbolCOFF>(SymA).getType();
+  if ((Type >> COFF::SCT_COMPLEX_TYPE_SHIFT) == COFF::IMAGE_SYM_DTYPE_FUNCTION)
     return false;
-  return MCObjectWriter::IsSymbolRefDifferenceFullyResolvedImpl(
-      Asm, DataA, DataB, FB, InSet, IsPCRel);
+  return MCObjectWriter::isSymbolRefDifferenceFullyResolvedImpl(Asm, SymA, FB,
+                                                                InSet, IsPCRel);
 }
 
-bool WinCOFFObjectWriter::isWeak(const MCSymbolData &SD) const {
-  // FIXME: this is for PR23025. Write a good description on
-  // why this is needed.
-  return SD.isExternal();
+bool WinCOFFObjectWriter::isWeak(const MCSymbol &Sym) const {
+  if (!Sym.isExternal())
+    return false;
+
+  if (!Sym.isInSection())
+    return false;
+
+  const auto &Sec = cast<MCSectionCOFF>(Sym.getSection());
+  if (!Sec.getCOMDATSymbol())
+    return false;
+
+  // It looks like for COFF it is invalid to replace a reference to a global
+  // in a comdat with a reference to a local.
+  // FIXME: Add a specification reference if available.
+  return true;
 }
 
-void WinCOFFObjectWriter::RecordRelocation(
+void WinCOFFObjectWriter::recordRelocation(
     MCAssembler &Asm, const MCAsmLayout &Layout, const MCFragment *Fragment,
     const MCFixup &Fixup, MCValue Target, bool &IsPCRel, uint64_t &FixedValue) {
   assert(Target.getSymA() && "Relocation must reference a symbol!");
 
   const MCSymbol &Symbol = Target.getSymA()->getSymbol();
-  const MCSymbol &A = Symbol.AliasedSymbol();
-  if (!Asm.hasSymbolData(A))
-    Asm.getContext().FatalError(
-        Fixup.getLoc(),
-        Twine("symbol '") + A.getName() + "' can not be undefined");
+  const MCSymbol &A = Symbol;
+  if (!A.isRegistered())
+    Asm.getContext().reportFatalError(Fixup.getLoc(),
+                                      Twine("symbol '") + A.getName() +
+                                          "' can not be undefined");
 
-  const MCSymbolData &A_SD = Asm.getSymbolData(A);
-
-  MCSectionData const *SectionData = Fragment->getParent();
+  MCSection *Section = Fragment->getParent();
 
   // Mark this symbol as requiring an entry in the symbol table.
-  assert(SectionMap.find(&SectionData->getSection()) != SectionMap.end() &&
-         "Section must already have been defined in ExecutePostLayoutBinding!");
-  assert(SymbolMap.find(&A_SD.getSymbol()) != SymbolMap.end() &&
-         "Symbol must already have been defined in ExecutePostLayoutBinding!");
+  assert(SectionMap.find(Section) != SectionMap.end() &&
+         "Section must already have been defined in executePostLayoutBinding!");
+  assert(SymbolMap.find(&A) != SymbolMap.end() &&
+         "Symbol must already have been defined in executePostLayoutBinding!");
 
-  COFFSection *coff_section = SectionMap[&SectionData->getSection()];
-  COFFSymbol *coff_symbol = SymbolMap[&A_SD.getSymbol()];
+  COFFSection *coff_section = SectionMap[Section];
+  COFFSymbol *coff_symbol = SymbolMap[&A];
   const MCSymbolRefExpr *SymB = Target.getSymB();
   bool CrossSection = false;
 
   if (SymB) {
     const MCSymbol *B = &SymB->getSymbol();
-    const MCSymbolData &B_SD = Asm.getSymbolData(*B);
-    if (!B_SD.getFragment())
-      Asm.getContext().FatalError(
+    if (!B->getFragment())
+      Asm.getContext().reportFatalError(
           Fixup.getLoc(),
           Twine("symbol '") + B->getName() +
               "' can not be undefined in a subtraction expression");
 
-    if (!A_SD.getFragment())
-      Asm.getContext().FatalError(
+    if (!A.getFragment())
+      Asm.getContext().reportFatalError(
           Fixup.getLoc(),
           Twine("symbol '") + Symbol.getName() +
               "' can not be undefined in a subtraction expression");
@@ -714,13 +739,13 @@ void WinCOFFObjectWriter::RecordRelocation(
     CrossSection = &Symbol.getSection() != &B->getSection();
 
     // Offset of the symbol in the section
-    int64_t OffsetOfB = Layout.getSymbolOffset(&B_SD);
+    int64_t OffsetOfB = Layout.getSymbolOffset(*B);
 
     // In the case where we have SymbA and SymB, we just need to store the delta
     // between the two symbols.  Update FixedValue to account for the delta, and
     // skip recording the relocation.
     if (!CrossSection) {
-      int64_t OffsetOfA = Layout.getSymbolOffset(&A_SD);
+      int64_t OffsetOfA = Layout.getSymbolOffset(A);
       FixedValue = (OffsetOfA - OffsetOfB) + Target.getConstant();
       return;
     }
@@ -729,7 +754,7 @@ void WinCOFFObjectWriter::RecordRelocation(
     int64_t OffsetOfRelocation =
         Layout.getFragmentOffset(Fragment) + Fixup.getOffset();
 
-    FixedValue = OffsetOfRelocation - OffsetOfB;
+    FixedValue = (OffsetOfRelocation - OffsetOfB) + Target.getConstant();
   } else {
     FixedValue = Target.getConstant();
   }
@@ -740,19 +765,18 @@ void WinCOFFObjectWriter::RecordRelocation(
   Reloc.Data.VirtualAddress = Layout.getFragmentOffset(Fragment);
 
   // Turn relocations for temporary symbols into section relocations.
-  if (coff_symbol->MCData->getSymbol().isTemporary() || CrossSection) {
+  if (coff_symbol->MC->isTemporary() || CrossSection) {
     Reloc.Symb = coff_symbol->Section->Symbol;
-    FixedValue += Layout.getFragmentOffset(coff_symbol->MCData->getFragment()) +
-                  coff_symbol->MCData->getOffset();
+    FixedValue += Layout.getFragmentOffset(coff_symbol->MC->getFragment()) +
+                  coff_symbol->MC->getOffset();
   } else
     Reloc.Symb = coff_symbol;
 
   ++Reloc.Symb->Relocations;
 
   Reloc.Data.VirtualAddress += Fixup.getOffset();
-  Reloc.Data.Type =
-      TargetObjectWriter->getRelocType(Target, Fixup, CrossSection,
-                                       Asm.getBackend());
+  Reloc.Data.Type = TargetObjectWriter->getRelocType(
+      Target, Fixup, CrossSection, Asm.getBackend());
 
   // FIXME: Can anyone explain what this does other than adjust for the size
   // of the offset?
@@ -773,9 +797,9 @@ void WinCOFFObjectWriter::RecordRelocation(
       break;
     case COFF::IMAGE_REL_ARM_BRANCH11:
     case COFF::IMAGE_REL_ARM_BLX11:
-      // IMAGE_REL_ARM_BRANCH11 and IMAGE_REL_ARM_BLX11 are only used for
-      // pre-ARMv7, which implicitly rules it out of ARMNT (it would be valid
-      // for Windows CE).
+    // IMAGE_REL_ARM_BRANCH11 and IMAGE_REL_ARM_BLX11 are only used for
+    // pre-ARMv7, which implicitly rules it out of ARMNT (it would be valid
+    // for Windows CE).
     case COFF::IMAGE_REL_ARM_BRANCH24:
     case COFF::IMAGE_REL_ARM_BLX24:
     case COFF::IMAGE_REL_ARM_MOV32A:
@@ -804,7 +828,7 @@ void WinCOFFObjectWriter::RecordRelocation(
     coff_section->Relocations.push_back(Reloc);
 }
 
-void WinCOFFObjectWriter::WriteObject(MCAssembler &Asm,
+void WinCOFFObjectWriter::writeObject(MCAssembler &Asm,
                                       const MCAsmLayout &Layout) {
   size_t SectionsSize = Sections.size();
   if (SectionsSize > static_cast<size_t>(INT32_MAX))
@@ -816,13 +840,9 @@ void WinCOFFObjectWriter::WriteObject(MCAssembler &Asm,
 
   UseBigObj = NumberOfSections > COFF::MaxNumberOfSections16;
 
-  DenseMap<COFFSection *, int32_t> SectionIndices(
-      NextPowerOf2(NumberOfSections));
-
   // Assign section numbers.
   size_t Number = 1;
   for (const auto &Section : Sections) {
-    SectionIndices[Section.get()] = Number;
     Section->Number = Number;
     Section->Symbol->Data.SectionNumber = Number;
     Section->Symbol->Aux[0].Aux.SectionDefinition.Number = Number;
@@ -832,11 +852,10 @@ void WinCOFFObjectWriter::WriteObject(MCAssembler &Asm,
   Header.NumberOfSections = NumberOfSections;
   Header.NumberOfSymbols = 0;
 
-  for (auto FI = Asm.file_names_begin(), FE = Asm.file_names_end();
-       FI != FE; ++FI) {
+  for (const std::string &Name : Asm.getFileNames()) {
     // round up to calculate the number of auxiliary symbols required
     unsigned SymbolSize = UseBigObj ? COFF::Symbol32Size : COFF::Symbol16Size;
-    unsigned Count = (FI->size() + SymbolSize - 1) / SymbolSize;
+    unsigned Count = (Name.size() + SymbolSize - 1) / SymbolSize;
 
     COFFSymbol *file = createSymbol(".file");
     file->Data.SectionNumber = COFF::IMAGE_SYM_DEBUG;
@@ -844,15 +863,15 @@ void WinCOFFObjectWriter::WriteObject(MCAssembler &Asm,
     file->Aux.resize(Count);
 
     unsigned Offset = 0;
-    unsigned Length = FI->size();
+    unsigned Length = Name.size();
     for (auto &Aux : file->Aux) {
       Aux.AuxType = ATFile;
 
       if (Length > SymbolSize) {
-        memcpy(&Aux.Aux, FI->c_str() + Offset, SymbolSize);
+        memcpy(&Aux.Aux, Name.c_str() + Offset, SymbolSize);
         Length = Length - SymbolSize;
       } else {
-        memcpy(&Aux.Aux, FI->c_str() + Offset, Length);
+        memcpy(&Aux.Aux, Name.c_str() + Offset, Length);
         memset((char *)&Aux.Aux + Length, 0, SymbolSize - Length);
         break;
       }
@@ -866,12 +885,13 @@ void WinCOFFObjectWriter::WriteObject(MCAssembler &Asm,
     if (Symbol->Section)
       Symbol->Data.SectionNumber = Symbol->Section->Number;
     if (Symbol->should_keep()) {
-      Symbol->Index = Header.NumberOfSymbols++;
+      Symbol->setIndex(Header.NumberOfSymbols++);
       // Update auxiliary symbol info.
       Symbol->Data.NumberOfAuxSymbols = Symbol->Aux.size();
       Header.NumberOfSymbols += Symbol->Data.NumberOfAuxSymbols;
-    } else
-      Symbol->Index = -1;
+    } else {
+      Symbol->setIndex(-1);
+    }
   }
 
   // Build string table.
@@ -893,11 +913,11 @@ void WinCOFFObjectWriter::WriteObject(MCAssembler &Asm,
   // Fixup weak external references.
   for (auto &Symbol : Symbols) {
     if (Symbol->Other) {
-      assert(Symbol->Index != -1);
+      assert(Symbol->getIndex() != -1);
       assert(Symbol->Aux.size() == 1 && "Symbol must contain one aux symbol!");
       assert(Symbol->Aux[0].AuxType == ATWeakExternal &&
              "Symbol's aux symbol must be a Weak External!");
-      Symbol->Aux[0].Aux.WeakExternal.TagIndex = Symbol->Other->Index;
+      Symbol->Aux[0].Aux.WeakExternal.TagIndex = Symbol->Other->getIndex();
     }
   }
 
@@ -907,8 +927,7 @@ void WinCOFFObjectWriter::WriteObject(MCAssembler &Asm,
         COFF::IMAGE_COMDAT_SELECT_ASSOCIATIVE)
       continue;
 
-    const MCSectionCOFF &MCSec =
-      static_cast<const MCSectionCOFF &>(Section->MCData->getSection());
+    const MCSectionCOFF &MCSec = *Section->MCSection;
 
     const MCSymbol *COMDAT = MCSec.getCOMDATSymbol();
     assert(COMDAT);
@@ -924,10 +943,9 @@ void WinCOFFObjectWriter::WriteObject(MCAssembler &Asm,
     if (Assoc->Number == -1)
       continue;
 
-    Section->Symbol->Aux[0].Aux.SectionDefinition.Number = SectionIndices[Assoc];
+    Section->Symbol->Aux[0].Aux.SectionDefinition.Number = Assoc->Number;
   }
 
-
   // Assign file offsets to COFF object file structures.
 
   unsigned offset = 0;
@@ -939,7 +957,7 @@ void WinCOFFObjectWriter::WriteObject(MCAssembler &Asm,
   offset += COFF::SectionSize * Header.NumberOfSections;
 
   for (const auto &Section : Asm) {
-    COFFSection *Sec = SectionMap[&Section.getSection()];
+    COFFSection *Sec = SectionMap[&Section];
 
     if (Sec->Number == -1)
       continue;
@@ -974,8 +992,8 @@ void WinCOFFObjectWriter::WriteObject(MCAssembler &Asm,
       offset += COFF::RelocationSize * Sec->Relocations.size();
 
       for (auto &Relocation : Sec->Relocations) {
-        assert(Relocation.Symb->Index != -1);
-        Relocation.Data.SymbolTableIndex = Relocation.Symb->Index;
+        assert(Relocation.Symb->getIndex() != -1);
+        Relocation.Data.SymbolTableIndex = Relocation.Symb->getIndex();
       }
     }
 
@@ -986,9 +1004,9 @@ void WinCOFFObjectWriter::WriteObject(MCAssembler &Asm,
            "Section's symbol's aux symbol must be a Section Definition!");
     Aux.Aux.SectionDefinition.Length = Sec->Header.SizeOfRawData;
     Aux.Aux.SectionDefinition.NumberOfRelocations =
-                                                Sec->Header.NumberOfRelocations;
+        Sec->Header.NumberOfRelocations;
     Aux.Aux.SectionDefinition.NumberOfLinenumbers =
-                                                Sec->Header.NumberOfLineNumbers;
+        Sec->Header.NumberOfLineNumbers;
   }
 
   Header.PointerToSymbolTable = offset;
@@ -1001,18 +1019,18 @@ void WinCOFFObjectWriter::WriteObject(MCAssembler &Asm,
 
   {
     sections::iterator i, ie;
-    MCAssembler::const_iterator j, je;
+    MCAssembler::iterator j, je;
 
     for (auto &Section : Sections) {
       if (Section->Number != -1) {
         if (Section->Relocations.size() >= 0xffff)
           Section->Header.Characteristics |= COFF::IMAGE_SCN_LNK_NRELOC_OVFL;
-        WriteSectionHeader(Section->Header);
+        writeSectionHeader(Section->Header);
       }
     }
 
-    for (i = Sections.begin(), ie = Sections.end(),
-         j = Asm.begin(), je = Asm.end();
+    for (i = Sections.begin(), ie = Sections.end(), j = Asm.begin(),
+        je = Asm.end();
          (i != ie) && (j != je); ++i, ++j) {
 
       if ((*i)->Number == -1)
@@ -1028,7 +1046,7 @@ void WinCOFFObjectWriter::WriteObject(MCAssembler &Asm,
 
         WriteZeros(SectionDataPadding);
 
-        Asm.writeSectionData(j, Layout);
+        Asm.writeSectionData(&*j, Layout);
       }
 
       if ((*i)->Relocations.size() > 0) {
@@ -1057,15 +1075,14 @@ void WinCOFFObjectWriter::WriteObject(MCAssembler &Asm,
          "Header::PointerToSymbolTable is insane!");
 
   for (auto &Symbol : Symbols)
-    if (Symbol->Index != -1)
+    if (Symbol->getIndex() != -1)
       WriteSymbol(*Symbol);
 
   OS.write(Strings.data().data(), Strings.data().size());
 }
 
-MCWinCOFFObjectTargetWriter::MCWinCOFFObjectTargetWriter(unsigned Machine_) :
-  Machine(Machine_) {
-}
+MCWinCOFFObjectTargetWriter::MCWinCOFFObjectTargetWriter(unsigned Machine_)
+    : Machine(Machine_) {}
 
 // Pin the vtable to this file.
 void MCWinCOFFObjectTargetWriter::anchor() {}
diff --git a/lib/MC/WinCOFFStreamer.cpp b/lib/MC/WinCOFFStreamer.cpp
index 8f9aacbb196..36dd691f07b 100644
--- a/lib/MC/WinCOFFStreamer.cpp
+++ b/lib/MC/WinCOFFStreamer.cpp
@@ -11,7 +11,6 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "llvm/ADT/StringExtras.h"
 #include "llvm/MC/MCAsmBackend.h"
 #include "llvm/MC/MCAsmLayout.h"
 #include "llvm/MC/MCAssembler.h"
@@ -23,7 +22,7 @@
 #include "llvm/MC/MCSection.h"
 #include "llvm/MC/MCSectionCOFF.h"
 #include "llvm/MC/MCStreamer.h"
-#include "llvm/MC/MCSymbol.h"
+#include "llvm/MC/MCSymbolCOFF.h"
 #include "llvm/MC/MCValue.h"
 #include "llvm/MC/MCWinCOFFStreamer.h"
 #include "llvm/Support/COFF.h"
@@ -49,7 +48,7 @@ void MCWinCOFFStreamer::EmitInstToData(const MCInst &Inst,
   SmallVector<MCFixup, 4> Fixups;
   SmallString<256> Code;
   raw_svector_ostream VecOS(Code);
-  getAssembler().getEmitter().EncodeInstruction(Inst, VecOS, Fixups, STI);
+  getAssembler().getEmitter().encodeInstruction(Inst, VecOS, Fixups, STI);
   VecOS.flush();
 
   // Add the fixups and data.
@@ -97,17 +96,17 @@ bool MCWinCOFFStreamer::EmitSymbolAttribute(MCSymbol *Symbol,
           Symbol->getSection().getVariant() == MCSection::SV_COFF) &&
          "Got non-COFF section in the COFF backend!");
 
-  MCSymbolData &SD = getAssembler().getOrCreateSymbolData(*Symbol);
+  getAssembler().registerSymbol(*Symbol);
 
   switch (Attribute) {
   default: return false;
   case MCSA_WeakReference:
   case MCSA_Weak:
-    SD.modifyFlags(COFF::SF_WeakExternal, COFF::SF_WeakExternal);
-    SD.setExternal(true);
+    cast<MCSymbolCOFF>(Symbol)->setIsWeakExternal();
+    Symbol->setExternal(true);
     break;
   case MCSA_Global:
-    SD.setExternal(true);
+    Symbol->setExternal(true);
     break;
   }
 
@@ -134,11 +133,11 @@ void MCWinCOFFStreamer::EmitCOFFSymbolStorageClass(int StorageClass) {
     FatalError("storage class specified outside of symbol definition");
 
   if (StorageClass & ~COFF::SSC_Invalid)
-    FatalError(Twine("storage class value '") + itostr(StorageClass) +
+    FatalError("storage class value '" + Twine(StorageClass) +
                "' out of range");
 
-  MCSymbolData &SD = getAssembler().getOrCreateSymbolData(*CurSymbol);
-  SD.modifyFlags(StorageClass << COFF::SF_ClassShift, COFF::SF_ClassMask);
+  getAssembler().registerSymbol(*CurSymbol);
+  cast<MCSymbolCOFF>(CurSymbol)->setClass((uint16_t)StorageClass);
 }
 
 void MCWinCOFFStreamer::EmitCOFFSymbolType(int Type) {
@@ -146,10 +145,10 @@ void MCWinCOFFStreamer::EmitCOFFSymbolType(int Type) {
     FatalError("symbol type specified outside of a symbol definition");
 
   if (Type & ~0xffff)
-    FatalError(Twine("type value '") + itostr(Type) + "' out of range");
+    FatalError("type value '" + Twine(Type) + "' out of range");
 
-  MCSymbolData &SD = getAssembler().getOrCreateSymbolData(*CurSymbol);
-  SD.modifyFlags(Type << COFF::SF_TypeShift, COFF::SF_TypeMask);
+  getAssembler().registerSymbol(*CurSymbol);
+  cast<MCSymbolCOFF>(CurSymbol)->setType((uint16_t)Type);
 }
 
 void MCWinCOFFStreamer::EndCOFFSymbolDef() {
@@ -158,26 +157,49 @@ void MCWinCOFFStreamer::EndCOFFSymbolDef() {
   CurSymbol = nullptr;
 }
 
+void MCWinCOFFStreamer::EmitCOFFSafeSEH(MCSymbol const *Symbol) {
+  // SafeSEH is a feature specific to 32-bit x86.  It does not exist (and is
+  // unnecessary) on all platforms which use table-based exception dispatch.
+  if (getContext().getObjectFileInfo()->getTargetTriple().getArch() !=
+      Triple::x86)
+    return;
+
+  const MCSymbolCOFF *CSymbol = cast<MCSymbolCOFF>(Symbol);
+  if (CSymbol->isSafeSEH())
+    return;
+
+  MCSection *SXData = getContext().getObjectFileInfo()->getSXDataSection();
+  getAssembler().registerSection(*SXData);
+  if (SXData->getAlignment() < 4)
+    SXData->setAlignment(4);
+
+  new MCSafeSEHFragment(Symbol, SXData);
+
+  getAssembler().registerSymbol(*Symbol);
+  CSymbol->setIsSafeSEH();
+
+  // The Microsoft linker requires that the symbol type of a handler be
+  // function. Go ahead and oblige it here.
+  CSymbol->setType(COFF::IMAGE_SYM_DTYPE_FUNCTION
+                   << COFF::SCT_COMPLEX_TYPE_SHIFT);
+}
+
 void MCWinCOFFStreamer::EmitCOFFSectionIndex(MCSymbol const *Symbol) {
   MCDataFragment *DF = getOrCreateDataFragment();
-  const MCSymbolRefExpr *SRE = MCSymbolRefExpr::Create(Symbol, getContext());
-  MCFixup Fixup = MCFixup::Create(DF->getContents().size(), SRE, FK_SecRel_2);
+  const MCSymbolRefExpr *SRE = MCSymbolRefExpr::create(Symbol, getContext());
+  MCFixup Fixup = MCFixup::create(DF->getContents().size(), SRE, FK_SecRel_2);
   DF->getFixups().push_back(Fixup);
   DF->getContents().resize(DF->getContents().size() + 2, 0);
 }
 
 void MCWinCOFFStreamer::EmitCOFFSecRel32(MCSymbol const *Symbol) {
   MCDataFragment *DF = getOrCreateDataFragment();
-  const MCSymbolRefExpr *SRE = MCSymbolRefExpr::Create(Symbol, getContext());
-  MCFixup Fixup = MCFixup::Create(DF->getContents().size(), SRE, FK_SecRel_4);
+  const MCSymbolRefExpr *SRE = MCSymbolRefExpr::create(Symbol, getContext());
+  MCFixup Fixup = MCFixup::create(DF->getContents().size(), SRE, FK_SecRel_4);
   DF->getFixups().push_back(Fixup);
   DF->getContents().resize(DF->getContents().size() + 4, 0);
 }
 
-void MCWinCOFFStreamer::EmitELFSize(MCSymbol *Symbol, const MCExpr *Value) {
-  llvm_unreachable("not supported");
-}
-
 void MCWinCOFFStreamer::EmitCommonSymbol(MCSymbol *Symbol, uint64_t Size,
                                          unsigned ByteAlignment) {
   assert((!Symbol->isInSection() ||
@@ -195,9 +217,9 @@ void MCWinCOFFStreamer::EmitCommonSymbol(MCSymbol *Symbol, uint64_t Size,
 
   AssignSection(Symbol, nullptr);
 
-  MCSymbolData &SD = getAssembler().getOrCreateSymbolData(*Symbol);
-  SD.setExternal(true);
-  SD.setCommon(Size, ByteAlignment);
+  getAssembler().registerSymbol(*Symbol);
+  Symbol->setExternal(true);
+  Symbol->setCommon(Size, ByteAlignment);
 
   if (!T.isKnownWindowsMSVCEnvironment() && ByteAlignment > 1) {
     SmallString<128> Directive;
@@ -219,34 +241,32 @@ void MCWinCOFFStreamer::EmitLocalCommonSymbol(MCSymbol *Symbol, uint64_t Size,
                                               unsigned ByteAlignment) {
   assert(!Symbol->isInSection() && "Symbol must not already have a section!");
 
-  const MCSection *Section = getContext().getObjectFileInfo()->getBSSSection();
-  MCSectionData &SectionData = getAssembler().getOrCreateSectionData(*Section);
-  if (SectionData.getAlignment() < ByteAlignment)
-    SectionData.setAlignment(ByteAlignment);
+  MCSection *Section = getContext().getObjectFileInfo()->getBSSSection();
+  getAssembler().registerSection(*Section);
+  if (Section->getAlignment() < ByteAlignment)
+    Section->setAlignment(ByteAlignment);
 
-  MCSymbolData &SD = getAssembler().getOrCreateSymbolData(*Symbol);
-  SD.setExternal(false);
+  getAssembler().registerSymbol(*Symbol);
+  Symbol->setExternal(false);
 
   AssignSection(Symbol, Section);
 
   if (ByteAlignment != 1)
     new MCAlignFragment(ByteAlignment, /*Value=*/0, /*ValueSize=*/0,
-                        ByteAlignment, &SectionData);
+                        ByteAlignment, Section);
 
-  MCFillFragment *Fragment =
-      new MCFillFragment(/*Value=*/0, /*ValueSize=*/0, Size, &SectionData);
-  SD.setFragment(Fragment);
+  MCFillFragment *Fragment = new MCFillFragment(
+      /*Value=*/0, /*ValueSize=*/0, Size, Section);
+  Symbol->setFragment(Fragment);
 }
 
-void MCWinCOFFStreamer::EmitZerofill(const MCSection *Section,
-                                     MCSymbol *Symbol, uint64_t Size,
-                                     unsigned ByteAlignment) {
+void MCWinCOFFStreamer::EmitZerofill(MCSection *Section, MCSymbol *Symbol,
+                                     uint64_t Size, unsigned ByteAlignment) {
   llvm_unreachable("not implemented");
 }
 
-void MCWinCOFFStreamer::EmitTBSSSymbol(const MCSection *Section,
-                                       MCSymbol *Symbol, uint64_t Size,
-                                       unsigned ByteAlignment) {
+void MCWinCOFFStreamer::EmitTBSSSymbol(MCSection *Section, MCSymbol *Symbol,
+                                       uint64_t Size, unsigned ByteAlignment) {
   llvm_unreachable("not implemented");
 }
 
@@ -269,7 +289,7 @@ void MCWinCOFFStreamer::FinishImpl() {
 
 LLVM_ATTRIBUTE_NORETURN
 void MCWinCOFFStreamer::FatalError(const Twine &Msg) const {
-  getContext().FatalError(SMLoc(), Msg);
+  getContext().reportFatalError(SMLoc(), Msg);
 }
 }
 
diff --git a/lib/Makefile b/lib/Makefile
index 96b46c73c5e..9b76126b80a 100644
--- a/lib/Makefile
+++ b/lib/Makefile
@@ -12,13 +12,6 @@ include $(LEVEL)/Makefile.config
 
 PARALLEL_DIRS := IR AsmParser Bitcode Analysis Transforms CodeGen Target      \
                  ExecutionEngine Linker LTO MC Object Option DebugInfo        \
-                 IRReader LineEditor ProfileData Passes
-
-# @LOCALMOD-BEGIN
-ifeq ($(PNACL_BROWSER_TRANSLATOR),1)
-  PARALLEL_DIRS := $(filter-out Archive Linker, \
-                $(PARALLEL_DIRS))
-endif
-# @LOCALMOD-END
+                 IRReader LineEditor ProfileData Passes LibDriver
 
 include $(LEVEL)/Makefile.common
diff --git a/lib/Object/Archive.cpp b/lib/Object/Archive.cpp
index 7d43daaf0f4..941f72bb6a6 100644
--- a/lib/Object/Archive.cpp
+++ b/lib/Object/Archive.cpp
@@ -17,6 +17,7 @@
 #include "llvm/ADT/Twine.h"
 #include "llvm/Support/Endian.h"
 #include "llvm/Support/MemoryBuffer.h"
+#include "llvm/Support/Path.h"
 
 using namespace llvm;
 using namespace object;
@@ -86,17 +87,17 @@ Archive::Child::Child(const Archive *Parent, const char *Start)
   if (!Start)
     return;
 
-  const ArchiveMemberHeader *Header =
-      reinterpret_cast<const ArchiveMemberHeader *>(Start);
   uint64_t Size = sizeof(ArchiveMemberHeader);
-  if (!Parent->IsThin || Header->getName() == "/" || Header->getName() == "//")
-    Size += Header->getSize();
   Data = StringRef(Start, Size);
+  if (!isThinMember()) {
+    Size += getRawSize();
+    Data = StringRef(Start, Size);
+  }
 
   // Setup StartOfFile and PaddingBytes.
   StartOfFile = sizeof(ArchiveMemberHeader);
   // Don't include attached name.
-  StringRef Name = Header->getName();
+  StringRef Name = getRawName();
   if (Name.startswith("#1/")) {
     uint64_t NameSize;
     if (Name.substr(3).rtrim(" ").getAsInteger(10, NameSize))
@@ -115,6 +116,27 @@ uint64_t Archive::Child::getRawSize() const {
   return getHeader()->getSize();
 }
 
+bool Archive::Child::isThinMember() const {
+  StringRef Name = getHeader()->getName();
+  return Parent->IsThin && Name != "/" && Name != "//";
+}
+
+ErrorOr<StringRef> Archive::Child::getBuffer() const {
+  if (!isThinMember())
+    return StringRef(Data.data() + StartOfFile, getSize());
+  ErrorOr<StringRef> Name = getName();
+  if (std::error_code EC = Name.getError())
+    return EC;
+  SmallString<128> FullName = sys::path::parent_path(
+      Parent->getMemoryBufferRef().getBufferIdentifier());
+  sys::path::append(FullName, *Name);
+  ErrorOr<std::unique_ptr<MemoryBuffer>> Buf = MemoryBuffer::getFile(FullName);
+  if (std::error_code EC = Buf.getError())
+    return EC;
+  Parent->ThinBuffers.push_back(std::move(*Buf));
+  return Parent->ThinBuffers.back()->getBuffer();
+}
+
 Archive::Child Archive::Child::getNext() const {
   size_t SpaceToSkip = Data.size();
   // If it's odd, add 1 to make it even.
@@ -162,10 +184,10 @@ ErrorOr<StringRef> Archive::Child::getName() const {
                    + Parent->StringTable->getSize()))
       return object_error::parse_failed;
 
-    // GNU long file names end with a /.
+    // GNU long file names end with a "/\n".
     if (Parent->kind() == K_GNU || Parent->kind() == K_MIPS64) {
-      StringRef::size_type End = StringRef(addr).find('/');
-      return StringRef(addr, End);
+      StringRef::size_type End = StringRef(addr).find('\n');
+      return StringRef(addr, End - 1);
     }
     return StringRef(addr);
   } else if (name.startswith("#1/")) {
@@ -186,7 +208,10 @@ ErrorOr<MemoryBufferRef> Archive::Child::getMemoryBufferRef() const {
   if (std::error_code EC = NameOrErr.getError())
     return EC;
   StringRef Name = NameOrErr.get();
-  return MemoryBufferRef(getBuffer(), Name);
+  ErrorOr<StringRef> Buf = getBuffer();
+  if (std::error_code EC = Buf.getError())
+    return EC;
+  return MemoryBufferRef(*Buf, Name);
 }
 
 ErrorOr<std::unique_ptr<Binary>>
@@ -207,7 +232,8 @@ ErrorOr<std::unique_ptr<Archive>> Archive::create(MemoryBufferRef Source) {
 }
 
 Archive::Archive(MemoryBufferRef Source, std::error_code &ec)
-    : Binary(Binary::ID_Archive, Source), SymbolTable(child_end()) {
+    : Binary(Binary::ID_Archive, Source), SymbolTable(child_end()),
+      StringTable(child_end()), FirstRegular(child_end()) {
   StringRef Buffer = Data.getBuffer();
   // Check for sufficient magic.
   if (Buffer.startswith(ThinMagic)) {
@@ -224,7 +250,7 @@ Archive::Archive(MemoryBufferRef Source, std::error_code &ec)
   child_iterator e = child_end();
 
   if (i == e) {
-    ec = object_error::success;
+    ec = std::error_code();
     return;
   }
 
@@ -254,7 +280,7 @@ Archive::Archive(MemoryBufferRef Source, std::error_code &ec)
     SymbolTable = i;
     ++i;
     FirstRegular = i;
-    ec = object_error::success;
+    ec = std::error_code();
     return;
   }
 
@@ -287,7 +313,7 @@ Archive::Archive(MemoryBufferRef Source, std::error_code &ec)
 
     ++i;
     if (i == e) {
-      ec = object_error::parse_failed;
+      ec = std::error_code();
       return;
     }
     Name = i->getRawName();
@@ -298,14 +324,14 @@ Archive::Archive(MemoryBufferRef Source, std::error_code &ec)
     StringTable = i;
     ++i;
     FirstRegular = i;
-    ec = object_error::success;
+    ec = std::error_code();
     return;
   }
 
   if (Name[0] != '/') {
     Format = has64SymTable ? K_MIPS64 : K_GNU;
     FirstRegular = i;
-    ec = object_error::success;
+    ec = std::error_code();
     return;
   }
 
@@ -320,7 +346,7 @@ Archive::Archive(MemoryBufferRef Source, std::error_code &ec)
   ++i;
   if (i == e) {
     FirstRegular = i;
-    ec = object_error::success;
+    ec = std::error_code();
     return;
   }
 
@@ -332,7 +358,7 @@ Archive::Archive(MemoryBufferRef Source, std::error_code &ec)
   }
 
   FirstRegular = i;
-  ec = object_error::success;
+  ec = std::error_code();
 }
 
 Archive::child_iterator Archive::child_begin(bool SkipInternal) const {
@@ -352,11 +378,11 @@ Archive::child_iterator Archive::child_end() const {
 }
 
 StringRef Archive::Symbol::getName() const {
-  return Parent->SymbolTable->getBuffer().begin() + StringIndex;
+  return Parent->getSymbolTable().begin() + StringIndex;
 }
 
 ErrorOr<Archive::child_iterator> Archive::Symbol::getMember() const {
-  const char *Buf = Parent->SymbolTable->getBuffer().begin();
+  const char *Buf = Parent->getSymbolTable().begin();
   const char *Offsets = Buf;
   if (Parent->kind() == K_MIPS64)
     Offsets += sizeof(uint64_t);
@@ -420,7 +446,7 @@ Archive::Symbol Archive::Symbol::getNext() const {
     // and the second being the offset into the archive of the member that
     // define the symbol. After that the next uint32_t is the byte count of
     // the string table followed by the string table.
-    const char *Buf = Parent->SymbolTable->getBuffer().begin();
+    const char *Buf = Parent->getSymbolTable().begin();
     uint32_t RanlibCount = 0;
     RanlibCount = read32le(Buf) / 8;
     // If t.SymbolIndex + 1 will be past the count of symbols (the RanlibCount)
@@ -437,8 +463,7 @@ Archive::Symbol Archive::Symbol::getNext() const {
     }
   } else {
     // Go to one past next null.
-    t.StringIndex =
-        Parent->SymbolTable->getBuffer().find('\0', t.StringIndex) + 1;
+    t.StringIndex = Parent->getSymbolTable().find('\0', t.StringIndex) + 1;
   }
   ++t.SymbolIndex;
   return t;
@@ -448,7 +473,7 @@ Archive::symbol_iterator Archive::symbol_begin() const {
   if (!hasSymbolTable())
     return symbol_iterator(Symbol(this, 0, 0));
 
-  const char *buf = SymbolTable->getBuffer().begin();
+  const char *buf = getSymbolTable().begin();
   if (kind() == K_GNU) {
     uint32_t symbol_count = 0;
     symbol_count = read32be(buf);
@@ -480,29 +505,28 @@ Archive::symbol_iterator Archive::symbol_begin() const {
     symbol_count = read32le(buf);
     buf += 4 + (symbol_count * 2); // Skip indices.
   }
-  uint32_t string_start_offset = buf - SymbolTable->getBuffer().begin();
+  uint32_t string_start_offset = buf - getSymbolTable().begin();
   return symbol_iterator(Symbol(this, 0, string_start_offset));
 }
 
 Archive::symbol_iterator Archive::symbol_end() const {
   if (!hasSymbolTable())
     return symbol_iterator(Symbol(this, 0, 0));
+  return symbol_iterator(Symbol(this, getNumberOfSymbols(), 0));
+}
 
-  const char *buf = SymbolTable->getBuffer().begin();
-  uint32_t symbol_count = 0;
-  if (kind() == K_GNU) {
-    symbol_count = read32be(buf);
-  } else if (kind() == K_MIPS64) {
-    symbol_count = read64be(buf);
-  } else if (kind() == K_BSD) {
-    symbol_count = read32le(buf) / 8;
-  } else {
-    uint32_t member_count = 0;
-    member_count = read32le(buf);
-    buf += 4 + (member_count * 4); // Skip offsets.
-    symbol_count = read32le(buf);
-  }
-  return symbol_iterator(Symbol(this, symbol_count, 0));
+uint32_t Archive::getNumberOfSymbols() const {
+  const char *buf = getSymbolTable().begin();
+  if (kind() == K_GNU)
+    return read32be(buf);
+  if (kind() == K_MIPS64)
+    return read64be(buf);
+  if (kind() == K_BSD)
+    return read32le(buf) / 8;
+  uint32_t member_count = 0;
+  member_count = read32le(buf);
+  buf += 4 + (member_count * 4); // Skip offsets.
+  return read32le(buf);
 }
 
 Archive::child_iterator Archive::findSym(StringRef name) const {
diff --git a/lib/Object/ArchiveWriter.cpp b/lib/Object/ArchiveWriter.cpp
new file mode 100644
index 00000000000..274ff2ddc56
--- /dev/null
+++ b/lib/Object/ArchiveWriter.cpp
@@ -0,0 +1,439 @@
+//===- ArchiveWriter.cpp - ar File Format implementation --------*- C++ -*-===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This file defines the writeArchive function.
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/Object/ArchiveWriter.h"
+#include "llvm/ADT/ArrayRef.h"
+#include "llvm/ADT/StringRef.h"
+#include "llvm/IR/LLVMContext.h"
+#include "llvm/Object/Archive.h"
+#include "llvm/Object/ObjectFile.h"
+#include "llvm/Object/SymbolicFile.h"
+#include "llvm/Support/EndianStream.h"
+#include "llvm/Support/Errc.h"
+#include "llvm/Support/ErrorHandling.h"
+#include "llvm/Support/Format.h"
+#include "llvm/Support/Path.h"
+#include "llvm/Support/ToolOutputFile.h"
+#include "llvm/Support/raw_ostream.h"
+
+#if !defined(_MSC_VER) && !defined(__MINGW32__)
+#include <unistd.h>
+#else
+#include <io.h>
+#endif
+
+using namespace llvm;
+
+NewArchiveIterator::NewArchiveIterator(object::Archive::child_iterator I,
+                                       StringRef Name)
+    : IsNewMember(false), Name(Name), OldI(I) {}
+
+NewArchiveIterator::NewArchiveIterator(StringRef FileName)
+    : IsNewMember(true), Name(FileName) {}
+
+StringRef NewArchiveIterator::getName() const { return Name; }
+
+bool NewArchiveIterator::isNewMember() const { return IsNewMember; }
+
+object::Archive::child_iterator NewArchiveIterator::getOld() const {
+  assert(!IsNewMember);
+  return OldI;
+}
+
+StringRef NewArchiveIterator::getNew() const {
+  assert(IsNewMember);
+  return Name;
+}
+
+llvm::ErrorOr<int>
+NewArchiveIterator::getFD(sys::fs::file_status &NewStatus) const {
+  assert(IsNewMember);
+  int NewFD;
+  if (auto EC = sys::fs::openFileForRead(Name, NewFD))
+    return EC;
+  assert(NewFD != -1);
+
+  if (auto EC = sys::fs::status(NewFD, NewStatus))
+    return EC;
+
+  // Opening a directory doesn't make sense. Let it fail.
+  // Linux cannot open directories with open(2), although
+  // cygwin and *bsd can.
+  if (NewStatus.type() == sys::fs::file_type::directory_file)
+    return make_error_code(errc::is_a_directory);
+
+  return NewFD;
+}
+
+template <typename T>
+static void printWithSpacePadding(raw_fd_ostream &OS, T Data, unsigned Size,
+				  bool MayTruncate = false) {
+  uint64_t OldPos = OS.tell();
+  OS << Data;
+  unsigned SizeSoFar = OS.tell() - OldPos;
+  if (Size > SizeSoFar) {
+    OS.indent(Size - SizeSoFar);
+  } else if (Size < SizeSoFar) {
+    assert(MayTruncate && "Data doesn't fit in Size");
+    // Some of the data this is used for (like UID) can be larger than the
+    // space available in the archive format. Truncate in that case.
+    OS.seek(OldPos + Size);
+  }
+}
+
+static void print32(raw_ostream &Out, object::Archive::Kind Kind,
+                    uint32_t Val) {
+  if (Kind == object::Archive::K_GNU)
+    support::endian::Writer<support::big>(Out).write(Val);
+  else
+    support::endian::Writer<support::little>(Out).write(Val);
+}
+
+static void printRestOfMemberHeader(raw_fd_ostream &Out,
+                                    const sys::TimeValue &ModTime, unsigned UID,
+                                    unsigned GID, unsigned Perms,
+                                    unsigned Size) {
+  printWithSpacePadding(Out, ModTime.toEpochTime(), 12);
+  printWithSpacePadding(Out, UID, 6, true);
+  printWithSpacePadding(Out, GID, 6, true);
+  printWithSpacePadding(Out, format("%o", Perms), 8);
+  printWithSpacePadding(Out, Size, 10);
+  Out << "`\n";
+}
+
+static void printGNUSmallMemberHeader(raw_fd_ostream &Out, StringRef Name,
+                                      const sys::TimeValue &ModTime,
+                                      unsigned UID, unsigned GID,
+                                      unsigned Perms, unsigned Size) {
+  printWithSpacePadding(Out, Twine(Name) + "/", 16);
+  printRestOfMemberHeader(Out, ModTime, UID, GID, Perms, Size);
+}
+
+static void printBSDMemberHeader(raw_fd_ostream &Out, StringRef Name,
+                                 const sys::TimeValue &ModTime, unsigned UID,
+                                 unsigned GID, unsigned Perms, unsigned Size) {
+  uint64_t PosAfterHeader = Out.tell() + 60 + Name.size();
+  // Pad so that even 64 bit object files are aligned.
+  unsigned Pad = OffsetToAlignment(PosAfterHeader, 8);
+  unsigned NameWithPadding = Name.size() + Pad;
+  printWithSpacePadding(Out, Twine("#1/") + Twine(NameWithPadding), 16);
+  printRestOfMemberHeader(Out, ModTime, UID, GID, Perms,
+                          NameWithPadding + Size);
+  Out << Name;
+  assert(PosAfterHeader == Out.tell());
+  while (Pad--)
+    Out.write(uint8_t(0));
+}
+
+static bool useStringTable(bool Thin, StringRef Name) {
+  return Thin || Name.size() >= 16;
+}
+
+static void
+printMemberHeader(raw_fd_ostream &Out, object::Archive::Kind Kind, bool Thin,
+                  StringRef Name,
+                  std::vector<unsigned>::iterator &StringMapIndexIter,
+                  const sys::TimeValue &ModTime, unsigned UID, unsigned GID,
+                  unsigned Perms, unsigned Size) {
+  if (Kind == object::Archive::K_BSD)
+    return printBSDMemberHeader(Out, Name, ModTime, UID, GID, Perms, Size);
+  if (!useStringTable(Thin, Name))
+    return printGNUSmallMemberHeader(Out, Name, ModTime, UID, GID, Perms, Size);
+  Out << '/';
+  printWithSpacePadding(Out, *StringMapIndexIter++, 15);
+  printRestOfMemberHeader(Out, ModTime, UID, GID, Perms, Size);
+}
+
+// Compute the relative path from From to To.
+static std::string computeRelativePath(StringRef From, StringRef To) {
+  if (sys::path::is_absolute(From) || sys::path::is_absolute(To))
+    return To;
+
+  StringRef DirFrom = sys::path::parent_path(From);
+  auto FromI = sys::path::begin(DirFrom);
+  auto ToI = sys::path::begin(To);
+  while (*FromI == *ToI) {
+    ++FromI;
+    ++ToI;
+  }
+
+  SmallString<128> Relative;
+  for (auto FromE = sys::path::end(DirFrom); FromI != FromE; ++FromI)
+    sys::path::append(Relative, "..");
+
+  for (auto ToE = sys::path::end(To); ToI != ToE; ++ToI)
+    sys::path::append(Relative, *ToI);
+
+  return Relative.str();
+}
+
+static void writeStringTable(raw_fd_ostream &Out, StringRef ArcName,
+                             ArrayRef<NewArchiveIterator> Members,
+                             std::vector<unsigned> &StringMapIndexes,
+                             bool Thin) {
+  unsigned StartOffset = 0;
+  for (const NewArchiveIterator &I : Members) {
+    StringRef Name = sys::path::filename(I.getName());
+    if (!useStringTable(Thin, Name))
+      continue;
+    if (StartOffset == 0) {
+      printWithSpacePadding(Out, "//", 58);
+      Out << "`\n";
+      StartOffset = Out.tell();
+    }
+    StringMapIndexes.push_back(Out.tell() - StartOffset);
+
+    if (Thin)
+      Out << computeRelativePath(ArcName, I.getName());
+    else
+      Out << Name;
+
+    Out << "/\n";
+  }
+  if (StartOffset == 0)
+    return;
+  if (Out.tell() % 2)
+    Out << '\n';
+  int Pos = Out.tell();
+  Out.seek(StartOffset - 12);
+  printWithSpacePadding(Out, Pos - StartOffset, 10);
+  Out.seek(Pos);
+}
+
+static sys::TimeValue now(bool Deterministic) {
+  if (!Deterministic)
+    return sys::TimeValue::now();
+  sys::TimeValue TV;
+  TV.fromEpochTime(0);
+  return TV;
+}
+
+// Returns the offset of the first reference to a member offset.
+static ErrorOr<unsigned>
+writeSymbolTable(raw_fd_ostream &Out, object::Archive::Kind Kind,
+                 ArrayRef<NewArchiveIterator> Members,
+                 ArrayRef<MemoryBufferRef> Buffers,
+                 std::vector<unsigned> &MemberOffsetRefs, bool Deterministic) {
+  unsigned HeaderStartOffset = 0;
+  unsigned BodyStartOffset = 0;
+  SmallString<128> NameBuf;
+  raw_svector_ostream NameOS(NameBuf);
+  LLVMContext Context;
+  for (unsigned MemberNum = 0, N = Members.size(); MemberNum < N; ++MemberNum) {
+    MemoryBufferRef MemberBuffer = Buffers[MemberNum];
+    ErrorOr<std::unique_ptr<object::SymbolicFile>> ObjOrErr =
+        object::SymbolicFile::createSymbolicFile(
+            MemberBuffer, sys::fs::file_magic::unknown, &Context);
+    if (!ObjOrErr)
+      continue;  // FIXME: check only for "not an object file" errors.
+    object::SymbolicFile &Obj = *ObjOrErr.get();
+
+    if (!HeaderStartOffset) {
+      HeaderStartOffset = Out.tell();
+      if (Kind == object::Archive::K_GNU)
+        printGNUSmallMemberHeader(Out, "", now(Deterministic), 0, 0, 0, 0);
+      else
+        printBSDMemberHeader(Out, "__.SYMDEF", now(Deterministic), 0, 0, 0, 0);
+      BodyStartOffset = Out.tell();
+      print32(Out, Kind, 0); // number of entries or bytes
+    }
+
+    for (const object::BasicSymbolRef &S : Obj.symbols()) {
+      uint32_t Symflags = S.getFlags();
+      if (Symflags & object::SymbolRef::SF_FormatSpecific)
+        continue;
+      if (!(Symflags & object::SymbolRef::SF_Global))
+        continue;
+      if (Symflags & object::SymbolRef::SF_Undefined)
+        continue;
+
+      unsigned NameOffset = NameOS.tell();
+      if (auto EC = S.printName(NameOS))
+        return EC;
+      NameOS << '\0';
+      MemberOffsetRefs.push_back(MemberNum);
+      if (Kind == object::Archive::K_BSD)
+        print32(Out, Kind, NameOffset);
+      print32(Out, Kind, 0); // member offset
+    }
+  }
+
+  if (HeaderStartOffset == 0)
+    return 0;
+
+  StringRef StringTable = NameOS.str();
+  if (Kind == object::Archive::K_BSD)
+    print32(Out, Kind, StringTable.size()); // byte count of the string table
+  Out << StringTable;
+
+  // ld64 requires the next member header to start at an offset that is
+  // 4 bytes aligned.
+  unsigned Pad = OffsetToAlignment(Out.tell(), 4);
+  while (Pad--)
+    Out.write(uint8_t(0));
+
+  // Patch up the size of the symbol table now that we know how big it is.
+  unsigned Pos = Out.tell();
+  const unsigned MemberHeaderSize = 60;
+  Out.seek(HeaderStartOffset + 48); // offset of the size field.
+  printWithSpacePadding(Out, Pos - MemberHeaderSize - HeaderStartOffset, 10);
+
+  // Patch up the number of symbols.
+  Out.seek(BodyStartOffset);
+  unsigned NumSyms = MemberOffsetRefs.size();
+  if (Kind == object::Archive::K_GNU)
+    print32(Out, Kind, NumSyms);
+  else
+    print32(Out, Kind, NumSyms * 8);
+
+  Out.seek(Pos);
+  return BodyStartOffset + 4;
+}
+
+std::pair<StringRef, std::error_code>
+llvm::writeArchive(StringRef ArcName,
+                   std::vector<NewArchiveIterator> &NewMembers,
+                   bool WriteSymtab, object::Archive::Kind Kind,
+                   bool Deterministic, bool Thin) {
+  SmallString<128> TmpArchive;
+  int TmpArchiveFD;
+  if (auto EC = sys::fs::createUniqueFile(ArcName + ".temp-archive-%%%%%%%.a",
+                                          TmpArchiveFD, TmpArchive))
+    return std::make_pair(ArcName, EC);
+
+  tool_output_file Output(TmpArchive, TmpArchiveFD);
+  raw_fd_ostream &Out = Output.os();
+  if (Thin)
+    Out << "!<thin>\n";
+  else
+    Out << "!<arch>\n";
+
+  std::vector<unsigned> MemberOffsetRefs;
+
+  std::vector<std::unique_ptr<MemoryBuffer>> Buffers;
+  std::vector<MemoryBufferRef> Members;
+  std::vector<sys::fs::file_status> NewMemberStatus;
+
+  for (unsigned I = 0, N = NewMembers.size(); I < N; ++I) {
+    NewArchiveIterator &Member = NewMembers[I];
+    MemoryBufferRef MemberRef;
+
+    if (Member.isNewMember()) {
+      StringRef Filename = Member.getNew();
+      NewMemberStatus.resize(NewMemberStatus.size() + 1);
+      sys::fs::file_status &Status = NewMemberStatus.back();
+      ErrorOr<int> FD = Member.getFD(Status);
+      if (auto EC = FD.getError())
+        return std::make_pair(Filename, EC);
+      ErrorOr<std::unique_ptr<MemoryBuffer>> MemberBufferOrErr =
+          MemoryBuffer::getOpenFile(FD.get(), Filename, Status.getSize(),
+                                    false);
+      if (auto EC = MemberBufferOrErr.getError())
+        return std::make_pair(Filename, EC);
+      if (close(FD.get()) != 0)
+        return std::make_pair(Filename,
+                              std::error_code(errno, std::generic_category()));
+      Buffers.push_back(std::move(MemberBufferOrErr.get()));
+      MemberRef = Buffers.back()->getMemBufferRef();
+    } else {
+      object::Archive::child_iterator OldMember = Member.getOld();
+      assert((!Thin || OldMember->getParent()->isThin()) &&
+             "Thin archives cannot refers to member of other archives");
+      ErrorOr<MemoryBufferRef> MemberBufferOrErr =
+          OldMember->getMemoryBufferRef();
+      if (auto EC = MemberBufferOrErr.getError())
+        return std::make_pair("", EC);
+      MemberRef = MemberBufferOrErr.get();
+    }
+    Members.push_back(MemberRef);
+  }
+
+  unsigned MemberReferenceOffset = 0;
+  if (WriteSymtab) {
+    ErrorOr<unsigned> MemberReferenceOffsetOrErr = writeSymbolTable(
+        Out, Kind, NewMembers, Members, MemberOffsetRefs, Deterministic);
+    if (auto EC = MemberReferenceOffsetOrErr.getError())
+      return std::make_pair(ArcName, EC);
+    MemberReferenceOffset = MemberReferenceOffsetOrErr.get();
+  }
+
+  std::vector<unsigned> StringMapIndexes;
+  if (Kind != object::Archive::K_BSD)
+    writeStringTable(Out, ArcName, NewMembers, StringMapIndexes, Thin);
+
+  unsigned MemberNum = 0;
+  unsigned NewMemberNum = 0;
+  std::vector<unsigned>::iterator StringMapIndexIter = StringMapIndexes.begin();
+  std::vector<unsigned> MemberOffset;
+  for (const NewArchiveIterator &I : NewMembers) {
+    MemoryBufferRef File = Members[MemberNum++];
+
+    unsigned Pos = Out.tell();
+    MemberOffset.push_back(Pos);
+
+    sys::TimeValue ModTime;
+    unsigned UID;
+    unsigned GID;
+    unsigned Perms;
+    if (Deterministic) {
+      ModTime.fromEpochTime(0);
+      UID = 0;
+      GID = 0;
+      Perms = 0644;
+    } else if (I.isNewMember()) {
+      const sys::fs::file_status &Status = NewMemberStatus[NewMemberNum];
+      ModTime = Status.getLastModificationTime();
+      UID = Status.getUser();
+      GID = Status.getGroup();
+      Perms = Status.permissions();
+    } else {
+      object::Archive::child_iterator OldMember = I.getOld();
+      ModTime = OldMember->getLastModified();
+      UID = OldMember->getUID();
+      GID = OldMember->getGID();
+      Perms = OldMember->getAccessMode();
+    }
+
+    if (I.isNewMember()) {
+      StringRef FileName = I.getNew();
+      const sys::fs::file_status &Status = NewMemberStatus[NewMemberNum++];
+      printMemberHeader(Out, Kind, Thin, sys::path::filename(FileName),
+                        StringMapIndexIter, ModTime, UID, GID, Perms,
+                        Status.getSize());
+    } else {
+      object::Archive::child_iterator OldMember = I.getOld();
+      printMemberHeader(Out, Kind, Thin, I.getName(), StringMapIndexIter,
+                        ModTime, UID, GID, Perms, OldMember->getSize());
+    }
+
+    if (!Thin)
+      Out << File.getBuffer();
+
+    if (Out.tell() % 2)
+      Out << '\n';
+  }
+
+  if (MemberReferenceOffset) {
+    Out.seek(MemberReferenceOffset);
+    for (unsigned MemberNum : MemberOffsetRefs) {
+      if (Kind == object::Archive::K_BSD)
+        Out.seek(Out.tell() + 4); // skip over the string offset
+      print32(Out, Kind, MemberOffset[MemberNum]);
+    }
+  }
+
+  Output.keep();
+  Out.close();
+  sys::fs::rename(TmpArchive, ArcName);
+  return std::make_pair("", std::error_code());
+}
diff --git a/lib/Object/CMakeLists.txt b/lib/Object/CMakeLists.txt
index 37add22fac5..8f10143fccc 100644
--- a/lib/Object/CMakeLists.txt
+++ b/lib/Object/CMakeLists.txt
@@ -1,5 +1,6 @@
 add_llvm_library(LLVMObject
   Archive.cpp
+  ArchiveWriter.cpp
   Binary.cpp
   COFFObjectFile.cpp
   COFFYAML.cpp
@@ -14,7 +15,11 @@ add_llvm_library(LLVMObject
   ObjectFile.cpp
   RecordStreamer.cpp
   SymbolicFile.cpp
+  SymbolSize.cpp
 
   ADDITIONAL_HEADER_DIRS
   ${LLVM_MAIN_INCLUDE_DIR}/llvm/Object
+
+  DEPENDS
+  intrinsics_gen
   )
diff --git a/lib/Object/COFFObjectFile.cpp b/lib/Object/COFFObjectFile.cpp
index 4c38b8fd2b0..d110b9293ba 100644
--- a/lib/Object/COFFObjectFile.cpp
+++ b/lib/Object/COFFObjectFile.cpp
@@ -16,6 +16,7 @@
 #include "llvm/ADT/SmallString.h"
 #include "llvm/ADT/StringSwitch.h"
 #include "llvm/ADT/Triple.h"
+#include "llvm/ADT/iterator_range.h"
 #include "llvm/Support/COFF.h"
 #include "llvm/Support/Debug.h"
 #include "llvm/Support/raw_ostream.h"
@@ -46,7 +47,7 @@ static std::error_code checkOffset(MemoryBufferRef M, uintptr_t Addr,
       Addr < uintptr_t(M.getBufferStart())) {
     return object_error::unexpected_eof;
   }
-  return object_error::success;
+  return std::error_code();
 }
 
 // Sets Obj unless any bytes in [addr, addr + size) fall outsize of m.
@@ -59,7 +60,7 @@ static std::error_code getObject(const T *&Obj, MemoryBufferRef M,
   if (std::error_code EC = checkOffset(M, Addr, Size))
     return EC;
   Obj = reinterpret_cast<const T *>(Addr);
-  return object_error::success;
+  return std::error_code();
 }
 
 // Decode a string table entry in base 64 (//AAAAAA). Expects \arg Str without
@@ -144,68 +145,64 @@ void COFFObjectFile::moveSymbolNext(DataRefImpl &Ref) const {
   }
 }
 
-std::error_code COFFObjectFile::getSymbolName(DataRefImpl Ref,
-                                              StringRef &Result) const {
+ErrorOr<StringRef> COFFObjectFile::getSymbolName(DataRefImpl Ref) const {
   COFFSymbolRef Symb = getCOFFSymbol(Ref);
-  return getSymbolName(Symb, Result);
+  StringRef Result;
+  std::error_code EC = getSymbolName(Symb, Result);
+  if (EC)
+    return EC;
+  return Result;
 }
 
-std::error_code COFFObjectFile::getSymbolAddress(DataRefImpl Ref,
-                                                 uint64_t &Result) const {
-  COFFSymbolRef Symb = getCOFFSymbol(Ref);
+uint64_t COFFObjectFile::getSymbolValueImpl(DataRefImpl Ref) const {
+  return getCOFFSymbol(Ref).getValue();
+}
 
-  if (Symb.isAnyUndefined()) {
-    Result = UnknownAddressOrSize;
-    return object_error::success;
-  }
-  if (Symb.isCommon()) {
-    Result = UnknownAddressOrSize;
-    return object_error::success;
-  }
+ErrorOr<uint64_t> COFFObjectFile::getSymbolAddress(DataRefImpl Ref) const {
+  uint64_t Result = getSymbolValue(Ref);
+  COFFSymbolRef Symb = getCOFFSymbol(Ref);
   int32_t SectionNumber = Symb.getSectionNumber();
-  if (!COFF::isReservedSectionNumber(SectionNumber)) {
-    const coff_section *Section = nullptr;
-    if (std::error_code EC = getSection(SectionNumber, Section))
-      return EC;
 
-    Result = Section->VirtualAddress + Symb.getValue();
-    return object_error::success;
-  }
+  if (Symb.isAnyUndefined() || Symb.isCommon() ||
+      COFF::isReservedSectionNumber(SectionNumber))
+    return Result;
+
+  const coff_section *Section = nullptr;
+  if (std::error_code EC = getSection(SectionNumber, Section))
+    return EC;
+  Result += Section->VirtualAddress;
+
+  // The section VirtualAddress does not include ImageBase, and we want to
+  // return virtual addresses.
+  if (PE32Header)
+    Result += PE32Header->ImageBase;
+  else if (PE32PlusHeader)
+    Result += PE32PlusHeader->ImageBase;
 
-  Result = Symb.getValue();
-  return object_error::success;
+  return Result;
 }
 
-std::error_code COFFObjectFile::getSymbolType(DataRefImpl Ref,
-                                              SymbolRef::Type &Result) const {
+SymbolRef::Type COFFObjectFile::getSymbolType(DataRefImpl Ref) const {
   COFFSymbolRef Symb = getCOFFSymbol(Ref);
   int32_t SectionNumber = Symb.getSectionNumber();
-  Result = SymbolRef::ST_Other;
-
-  if (Symb.isAnyUndefined()) {
-    Result = SymbolRef::ST_Unknown;
-  } else if (Symb.isFunctionDefinition()) {
-    Result = SymbolRef::ST_Function;
-  } else if (Symb.isCommon()) {
-    Result = SymbolRef::ST_Data;
-  } else if (Symb.isFileRecord()) {
-    Result = SymbolRef::ST_File;
-  } else if (SectionNumber == COFF::IMAGE_SYM_DEBUG ||
-             Symb.isSectionDefinition()) {
-    // TODO: perhaps we need a new symbol type ST_Section.
-    Result = SymbolRef::ST_Debug;
-  } else if (!COFF::isReservedSectionNumber(SectionNumber)) {
-    const coff_section *Section = nullptr;
-    if (std::error_code EC = getSection(SectionNumber, Section))
-      return EC;
-    uint32_t Characteristics = Section->Characteristics;
-    if (Characteristics & COFF::IMAGE_SCN_CNT_CODE)
-      Result = SymbolRef::ST_Function;
-    else if (Characteristics & (COFF::IMAGE_SCN_CNT_INITIALIZED_DATA |
-                                COFF::IMAGE_SCN_CNT_UNINITIALIZED_DATA))
-      Result = SymbolRef::ST_Data;
-  }
-  return object_error::success;
+
+  if (Symb.isAnyUndefined())
+    return SymbolRef::ST_Unknown;
+  if (Symb.isFunctionDefinition())
+    return SymbolRef::ST_Function;
+  if (Symb.isCommon())
+    return SymbolRef::ST_Data;
+  if (Symb.isFileRecord())
+    return SymbolRef::ST_File;
+
+  // TODO: perhaps we need a new symbol type ST_Section.
+  if (SectionNumber == COFF::IMAGE_SYM_DEBUG || Symb.isSectionDefinition())
+    return SymbolRef::ST_Debug;
+
+  if (!COFF::isReservedSectionNumber(SectionNumber))
+    return SymbolRef::ST_Data;
+
+  return SymbolRef::ST_Other;
 }
 
 uint32_t COFFObjectFile::getSymbolFlags(DataRefImpl Ref) const {
@@ -236,65 +233,9 @@ uint32_t COFFObjectFile::getSymbolFlags(DataRefImpl Ref) const {
   return Result;
 }
 
-std::error_code COFFObjectFile::getSymbolSize(DataRefImpl Ref,
-                                              uint64_t &Result) const {
+uint64_t COFFObjectFile::getCommonSymbolSizeImpl(DataRefImpl Ref) const {
   COFFSymbolRef Symb = getCOFFSymbol(Ref);
-
-  if (Symb.isAnyUndefined()) {
-    Result = UnknownAddressOrSize;
-    return object_error::success;
-  }
-  if (Symb.isCommon()) {
-    Result = Symb.getValue();
-    return object_error::success;
-  }
-
-  // Let's attempt to get the size of the symbol by looking at the address of
-  // the symbol after the symbol in question.
-  uint64_t SymbAddr;
-  if (std::error_code EC = getSymbolAddress(Ref, SymbAddr))
-    return EC;
-  int32_t SectionNumber = Symb.getSectionNumber();
-  if (COFF::isReservedSectionNumber(SectionNumber)) {
-    // Absolute and debug symbols aren't sorted in any interesting way.
-    Result = 0;
-    return object_error::success;
-  }
-  const section_iterator SecEnd = section_end();
-  uint64_t AfterAddr = UnknownAddressOrSize;
-  for (const symbol_iterator SymbI : symbols()) {
-    section_iterator SecI = SecEnd;
-    if (std::error_code EC = SymbI->getSection(SecI))
-      return EC;
-    // Check the symbol's section, skip it if it's in the wrong section.
-    // First, make sure it is in any section.
-    if (SecI == SecEnd)
-      continue;
-    // Second, make sure it is in the same section as the symbol in question.
-    if (!sectionContainsSymbol(SecI->getRawDataRefImpl(), Ref))
-      continue;
-    uint64_t Addr;
-    if (std::error_code EC = SymbI->getAddress(Addr))
-      return EC;
-    // We want to compare our symbol in question with the closest possible
-    // symbol that comes after.
-    if (AfterAddr > Addr && Addr > SymbAddr)
-      AfterAddr = Addr;
-  }
-  if (AfterAddr == UnknownAddressOrSize) {
-    // No symbol comes after this one, assume that everything after our symbol
-    // is part of it.
-    const coff_section *Section = nullptr;
-    if (std::error_code EC = getSection(SectionNumber, Section))
-      return EC;
-    Result = Section->SizeOfRawData - Symb.getValue();
-  } else {
-    // Take the difference between our symbol and the symbol that comes after
-    // our symbol.
-    Result = AfterAddr - SymbAddr;
-  }
-
-  return object_error::success;
+  return Symb.getValue();
 }
 
 std::error_code
@@ -311,7 +252,12 @@ COFFObjectFile::getSymbolSection(DataRefImpl Ref,
     Ref.p = reinterpret_cast<uintptr_t>(Sec);
     Result = section_iterator(SectionRef(Ref, this));
   }
-  return object_error::success;
+  return std::error_code();
+}
+
+unsigned COFFObjectFile::getSymbolSectionID(SymbolRef Sym) const {
+  COFFSymbolRef Symb = getCOFFSymbol(Sym.getRawDataRefImpl());
+  return Symb.getSectionNumber();
 }
 
 void COFFObjectFile::moveSectionNext(DataRefImpl &Ref) const {
@@ -328,7 +274,15 @@ std::error_code COFFObjectFile::getSectionName(DataRefImpl Ref,
 
 uint64_t COFFObjectFile::getSectionAddress(DataRefImpl Ref) const {
   const coff_section *Sec = toSec(Ref);
-  return Sec->VirtualAddress;
+  uint64_t Result = Sec->VirtualAddress;
+
+  // The section VirtualAddress does not include ImageBase, and we want to
+  // return virtual addresses.
+  if (PE32Header)
+    Result += PE32Header->ImageBase;
+  else if (PE32PlusHeader)
+    Result += PE32PlusHeader->ImageBase;
+  return Result;
 }
 
 uint64_t COFFObjectFile::getSectionSize(DataRefImpl Ref) const {
@@ -367,6 +321,13 @@ bool COFFObjectFile::isSectionBSS(DataRefImpl Ref) const {
   return (Sec->Characteristics & BssFlags) == BssFlags;
 }
 
+unsigned COFFObjectFile::getSectionID(SectionRef Sec) const {
+  uintptr_t Offset =
+      uintptr_t(Sec.getRawDataRefImpl().p) - uintptr_t(SectionTable);
+  assert((Offset % sizeof(coff_section)) == 0);
+  return (Offset / sizeof(coff_section)) + 1;
+}
+
 bool COFFObjectFile::isSectionVirtual(DataRefImpl Ref) const {
   const coff_section *Sec = toSec(Ref);
   // In COFF, a virtual section won't have any in-file 
@@ -374,14 +335,6 @@ bool COFFObjectFile::isSectionVirtual(DataRefImpl Ref) const {
   return Sec->PointerToRawData == 0;
 }
 
-bool COFFObjectFile::sectionContainsSymbol(DataRefImpl SecRef,
-                                           DataRefImpl SymbRef) const {
-  const coff_section *Sec = toSec(SecRef);
-  COFFSymbolRef Symb = getCOFFSymbol(SymbRef);
-  int32_t SecNumber = (Sec - SectionTable) + 1;
-  return SecNumber == Symb.getSectionNumber();
-}
-
 static uint32_t getNumberOfRelocations(const coff_section *Sec,
                                        MemoryBufferRef M, const uint8_t *base) {
   // The field for the number of relocations in COFF section table is only
@@ -419,6 +372,8 @@ getFirstReloc(const coff_section *Sec, MemoryBufferRef M, const uint8_t *Base) {
 relocation_iterator COFFObjectFile::section_rel_begin(DataRefImpl Ref) const {
   const coff_section *Sec = toSec(Ref);
   const coff_relocation *begin = getFirstReloc(Sec, Data, base());
+  if (begin && Sec->VirtualAddress != 0)
+    report_fatal_error("Sections with relocations should have an address of 0");
   DataRefImpl Ret;
   Ret.p = reinterpret_cast<uintptr_t>(begin);
   return relocation_iterator(RelocationRef(Ret, this));
@@ -470,7 +425,7 @@ std::error_code COFFObjectFile::initSymbolTablePtr() {
   // Check that the string table is null terminated if has any in it.
   if (StringTableSize > 4 && StringTable[StringTableSize - 1] != 0)
     return  object_error::parse_failed;
-  return object_error::success;
+  return std::error_code();
 }
 
 // Returns the file offset for the given VA.
@@ -491,7 +446,7 @@ std::error_code COFFObjectFile::getRvaPtr(uint32_t Addr, uintptr_t &Res) const {
     if (SectionStart <= Addr && Addr < SectionEnd) {
       uint32_t Offset = Addr - SectionStart;
       Res = uintptr_t(base()) + Section->PointerToRawData + Offset;
-      return object_error::success;
+      return std::error_code();
     }
   }
   return object_error::parse_failed;
@@ -507,7 +462,7 @@ std::error_code COFFObjectFile::getHintName(uint32_t Rva, uint16_t &Hint,
   const uint8_t *Ptr = reinterpret_cast<const uint8_t *>(IntPtr);
   Hint = *reinterpret_cast<const ulittle16_t *>(Ptr);
   Name = StringRef(reinterpret_cast<const char *>(Ptr + 2));
-  return object_error::success;
+  return std::error_code();
 }
 
 // Find the import table.
@@ -516,11 +471,11 @@ std::error_code COFFObjectFile::initImportTablePtr() {
   // the import table, do nothing.
   const data_directory *DataEntry;
   if (getDataDirectory(COFF::IMPORT_TABLE, DataEntry))
-    return object_error::success;
+    return std::error_code();
 
   // Do nothing if the pointer to import table is NULL.
   if (DataEntry->RelativeVirtualAddress == 0)
-    return object_error::success;
+    return std::error_code();
 
   uint32_t ImportTableRva = DataEntry->RelativeVirtualAddress;
   // -1 because the last entry is the null entry.
@@ -534,16 +489,16 @@ std::error_code COFFObjectFile::initImportTablePtr() {
     return EC;
   ImportDirectory = reinterpret_cast<
       const import_directory_table_entry *>(IntPtr);
-  return object_error::success;
+  return std::error_code();
 }
 
 // Initializes DelayImportDirectory and NumberOfDelayImportDirectory.
 std::error_code COFFObjectFile::initDelayImportTablePtr() {
   const data_directory *DataEntry;
   if (getDataDirectory(COFF::DELAY_IMPORT_DESCRIPTOR, DataEntry))
-    return object_error::success;
+    return std::error_code();
   if (DataEntry->RelativeVirtualAddress == 0)
-    return object_error::success;
+    return std::error_code();
 
   uint32_t RVA = DataEntry->RelativeVirtualAddress;
   NumberOfDelayImportDirectory = DataEntry->Size /
@@ -554,7 +509,7 @@ std::error_code COFFObjectFile::initDelayImportTablePtr() {
     return EC;
   DelayImportDirectory = reinterpret_cast<
       const delay_import_directory_table_entry *>(IntPtr);
-  return object_error::success;
+  return std::error_code();
 }
 
 // Find the export table.
@@ -563,11 +518,11 @@ std::error_code COFFObjectFile::initExportTablePtr() {
   // the export table, do nothing.
   const data_directory *DataEntry;
   if (getDataDirectory(COFF::EXPORT_TABLE, DataEntry))
-    return object_error::success;
+    return std::error_code();
 
   // Do nothing if the pointer to export table is NULL.
   if (DataEntry->RelativeVirtualAddress == 0)
-    return object_error::success;
+    return std::error_code();
 
   uint32_t ExportTableRva = DataEntry->RelativeVirtualAddress;
   uintptr_t IntPtr = 0;
@@ -575,15 +530,15 @@ std::error_code COFFObjectFile::initExportTablePtr() {
     return EC;
   ExportDirectory =
       reinterpret_cast<const export_directory_table_entry *>(IntPtr);
-  return object_error::success;
+  return std::error_code();
 }
 
 std::error_code COFFObjectFile::initBaseRelocPtr() {
   const data_directory *DataEntry;
   if (getDataDirectory(COFF::BASE_RELOCATION_TABLE, DataEntry))
-    return object_error::success;
+    return std::error_code();
   if (DataEntry->RelativeVirtualAddress == 0)
-    return object_error::success;
+    return std::error_code();
 
   uintptr_t IntPtr = 0;
   if (std::error_code EC = getRvaPtr(DataEntry->RelativeVirtualAddress, IntPtr))
@@ -592,7 +547,7 @@ std::error_code COFFObjectFile::initBaseRelocPtr() {
       IntPtr);
   BaseRelocEnd = reinterpret_cast<coff_base_reloc_block_header *>(
       IntPtr + DataEntry->Size);
-  return object_error::success;
+  return std::error_code();
 }
 
 COFFObjectFile::COFFObjectFile(MemoryBufferRef Object, std::error_code &EC)
@@ -657,7 +612,7 @@ COFFObjectFile::COFFObjectFile(MemoryBufferRef Object, std::error_code &EC)
   if (COFFHeader) {
     // The prior checkSize call may have failed.  This isn't a hard error
     // because we were just trying to sniff out bigobj.
-    EC = object_error::success;
+    EC = std::error_code();
     CurPtr += sizeof(coff_file_header);
 
     if (COFFHeader->isImportLibrary())
@@ -719,7 +674,7 @@ COFFObjectFile::COFFObjectFile(MemoryBufferRef Object, std::error_code &EC)
   if ((EC = initBaseRelocPtr()))
     return;
 
-  EC = object_error::success;
+  EC = std::error_code();
 }
 
 basic_symbol_iterator COFFObjectFile::symbol_begin_impl() const {
@@ -805,6 +760,8 @@ StringRef COFFObjectFile::getFileFormatName() const {
     return "COFF-x86-64";
   case COFF::IMAGE_FILE_MACHINE_ARMNT:
     return "COFF-ARM";
+  case COFF::IMAGE_FILE_MACHINE_ARM64:
+    return "COFF-ARM64";
   default:
     return "COFF-<unknown arch>";
   }
@@ -818,6 +775,8 @@ unsigned COFFObjectFile::getArch() const {
     return Triple::x86_64;
   case COFF::IMAGE_FILE_MACHINE_ARMNT:
     return Triple::thumb;
+  case COFF::IMAGE_FILE_MACHINE_ARM64:
+    return Triple::aarch64;
   default:
     return Triple::UnknownArch;
   }
@@ -845,13 +804,13 @@ iterator_range<base_reloc_iterator> COFFObjectFile::base_relocs() const {
 
 std::error_code COFFObjectFile::getPE32Header(const pe32_header *&Res) const {
   Res = PE32Header;
-  return object_error::success;
+  return std::error_code();
 }
 
 std::error_code
 COFFObjectFile::getPE32PlusHeader(const pe32plus_header *&Res) const {
   Res = PE32PlusHeader;
-  return object_error::success;
+  return std::error_code();
 }
 
 std::error_code
@@ -870,18 +829,18 @@ COFFObjectFile::getDataDirectory(uint32_t Index,
     return object_error::parse_failed;
   }
   Res = &DataDirectory[Index];
-  return object_error::success;
+  return std::error_code();
 }
 
 std::error_code COFFObjectFile::getSection(int32_t Index,
                                            const coff_section *&Result) const {
   Result = nullptr;
   if (COFF::isReservedSectionNumber(Index))
-    return object_error::success;
+    return std::error_code();
   if (static_cast<uint32_t>(Index) <= getNumberOfSections()) {
     // We already verified the section table data, so no need to check again.
     Result = SectionTable + (Index - 1);
-    return object_error::success;
+    return std::error_code();
   }
   return object_error::parse_failed;
 }
@@ -894,26 +853,30 @@ std::error_code COFFObjectFile::getString(uint32_t Offset,
   if (Offset >= StringTableSize)
     return object_error::unexpected_eof;
   Result = StringRef(StringTable + Offset);
-  return object_error::success;
+  return std::error_code();
 }
 
 std::error_code COFFObjectFile::getSymbolName(COFFSymbolRef Symbol,
                                               StringRef &Res) const {
+  return getSymbolName(Symbol.getGeneric(), Res);
+}
+
+std::error_code COFFObjectFile::getSymbolName(const coff_symbol_generic *Symbol,
+                                              StringRef &Res) const {
   // Check for string table entry. First 4 bytes are 0.
-  if (Symbol.getStringTableOffset().Zeroes == 0) {
-    uint32_t Offset = Symbol.getStringTableOffset().Offset;
-    if (std::error_code EC = getString(Offset, Res))
+  if (Symbol->Name.Offset.Zeroes == 0) {
+    if (std::error_code EC = getString(Symbol->Name.Offset.Offset, Res))
       return EC;
-    return object_error::success;
+    return std::error_code();
   }
 
-  if (Symbol.getShortName()[COFF::NameSize - 1] == 0)
+  if (Symbol->Name.ShortName[COFF::NameSize - 1] == 0)
     // Null terminated, let ::strlen figure out the length.
-    Res = StringRef(Symbol.getShortName());
+    Res = StringRef(Symbol->Name.ShortName);
   else
     // Not null terminated, use all 8 bytes.
-    Res = StringRef(Symbol.getShortName(), COFF::NameSize);
-  return object_error::success;
+    Res = StringRef(Symbol->Name.ShortName, COFF::NameSize);
+  return std::error_code();
 }
 
 ArrayRef<uint8_t>
@@ -964,7 +927,7 @@ std::error_code COFFObjectFile::getSectionName(const coff_section *Sec,
   }
 
   Res = Name;
-  return object_error::success;
+  return std::error_code();
 }
 
 uint64_t COFFObjectFile::getSectionSize(const coff_section *Sec) const {
@@ -972,19 +935,15 @@ uint64_t COFFObjectFile::getSectionSize(const coff_section *Sec) const {
   // whether or not we have an executable image.
   //
   // For object files, SizeOfRawData contains the size of section's data;
-  // VirtualSize is always zero.
+  // VirtualSize should be zero but isn't due to buggy COFF writers.
   //
   // For executables, SizeOfRawData *must* be a multiple of FileAlignment; the
   // actual section size is in VirtualSize.  It is possible for VirtualSize to
   // be greater than SizeOfRawData; the contents past that point should be
   // considered to be zero.
-  uint32_t SectionSize;
-  if (Sec->VirtualSize)
-    SectionSize = std::min(Sec->VirtualSize, Sec->SizeOfRawData);
-  else
-    SectionSize = Sec->SizeOfRawData;
-
-  return SectionSize;
+  if (getDOSHeader())
+    return std::min(Sec->VirtualSize, Sec->SizeOfRawData);
+  return Sec->SizeOfRawData;
 }
 
 std::error_code
@@ -1002,7 +961,7 @@ COFFObjectFile::getSectionContents(const coff_section *Sec,
   if (checkOffset(Data, ConStart, SectionSize))
     return object_error::parse_failed;
   Res = makeArrayRef(reinterpret_cast<const uint8_t *>(ConStart), SectionSize);
-  return object_error::success;
+  return std::error_code();
 }
 
 const coff_relocation *COFFObjectFile::toRel(DataRefImpl Rel) const {
@@ -1014,20 +973,9 @@ void COFFObjectFile::moveRelocationNext(DataRefImpl &Rel) const {
             reinterpret_cast<const coff_relocation*>(Rel.p) + 1);
 }
 
-std::error_code COFFObjectFile::getRelocationAddress(DataRefImpl Rel,
-                                                     uint64_t &Res) const {
-  report_fatal_error("getRelocationAddress not implemented in COFFObjectFile");
-}
-
-std::error_code COFFObjectFile::getRelocationOffset(DataRefImpl Rel,
-                                                    uint64_t &Res) const {
+uint64_t COFFObjectFile::getRelocationOffset(DataRefImpl Rel) const {
   const coff_relocation *R = toRel(Rel);
-  const support::ulittle32_t *VirtualAddressPtr;
-  if (std::error_code EC =
-          getObject(VirtualAddressPtr, Data, &R->VirtualAddress))
-    return EC;
-  Res = *VirtualAddressPtr;
-  return object_error::success;
+  return R->VirtualAddress;
 }
 
 symbol_iterator COFFObjectFile::getRelocationSymbol(DataRefImpl Rel) const {
@@ -1044,11 +992,9 @@ symbol_iterator COFFObjectFile::getRelocationSymbol(DataRefImpl Rel) const {
   return symbol_iterator(SymbolRef(Ref, this));
 }
 
-std::error_code COFFObjectFile::getRelocationType(DataRefImpl Rel,
-                                                  uint64_t &Res) const {
+uint64_t COFFObjectFile::getRelocationType(DataRefImpl Rel) const {
   const coff_relocation* R = toRel(Rel);
-  Res = R->Type;
-  return object_error::success;
+  return R->Type;
 }
 
 const coff_section *
@@ -1073,14 +1019,22 @@ COFFObjectFile::getCOFFRelocation(const RelocationRef &Reloc) const {
   return toRel(Reloc.getRawDataRefImpl());
 }
 
+iterator_range<const coff_relocation *>
+COFFObjectFile::getRelocations(const coff_section *Sec) const {
+  const coff_relocation *I = getFirstReloc(Sec, Data, base());
+  const coff_relocation *E = I;
+  if (I)
+    E += getNumberOfRelocations(Sec, Data, base());
+  return make_range(I, E);
+}
+
 #define LLVM_COFF_SWITCH_RELOC_TYPE_NAME(reloc_type)                           \
   case COFF::reloc_type:                                                       \
     Res = #reloc_type;                                                         \
     break;
 
-std::error_code
-COFFObjectFile::getRelocationTypeName(DataRefImpl Rel,
-                                      SmallVectorImpl<char> &Result) const {
+void COFFObjectFile::getRelocationTypeName(
+    DataRefImpl Rel, SmallVectorImpl<char> &Result) const {
   const coff_relocation *Reloc = toRel(Rel);
   StringRef Res;
   switch (getMachine()) {
@@ -1149,27 +1103,10 @@ COFFObjectFile::getRelocationTypeName(DataRefImpl Rel,
     Res = "Unknown";
   }
   Result.append(Res.begin(), Res.end());
-  return object_error::success;
 }
 
 #undef LLVM_COFF_SWITCH_RELOC_TYPE_NAME
 
-std::error_code
-COFFObjectFile::getRelocationValueString(DataRefImpl Rel,
-                                         SmallVectorImpl<char> &Result) const {
-  const coff_relocation *Reloc = toRel(Rel);
-  DataRefImpl Sym;
-  ErrorOr<COFFSymbolRef> Symb = getSymbol(Reloc->SymbolTableIndex);
-  if (std::error_code EC = Symb.getError())
-    return EC;
-  Sym.p = reinterpret_cast<uintptr_t>(Symb->getRawPtr());
-  StringRef SymName;
-  if (std::error_code EC = getSymbolName(Sym, SymName))
-    return EC;
-  Result.append(SymName.begin(), SymName.end());
-  return object_error::success;
-}
-
 bool COFFObjectFile::isRelocatableObject() const {
   return !DataDirectory;
 }
@@ -1186,7 +1123,7 @@ void ImportDirectoryEntryRef::moveNext() {
 std::error_code ImportDirectoryEntryRef::getImportTableEntry(
     const import_directory_table_entry *&Result) const {
   Result = ImportTable + Index;
-  return object_error::success;
+  return std::error_code();
 }
 
 static imported_symbol_iterator
@@ -1248,19 +1185,19 @@ std::error_code ImportDirectoryEntryRef::getName(StringRef &Result) const {
           OwningObject->getRvaPtr(ImportTable[Index].NameRVA, IntPtr))
     return EC;
   Result = StringRef(reinterpret_cast<const char *>(IntPtr));
-  return object_error::success;
+  return std::error_code();
 }
 
 std::error_code
 ImportDirectoryEntryRef::getImportLookupTableRVA(uint32_t  &Result) const {
   Result = ImportTable[Index].ImportLookupTableRVA;
-  return object_error::success;
+  return std::error_code();
 }
 
 std::error_code
 ImportDirectoryEntryRef::getImportAddressTableRVA(uint32_t &Result) const {
   Result = ImportTable[Index].ImportAddressTableRVA;
-  return object_error::success;
+  return std::error_code();
 }
 
 std::error_code ImportDirectoryEntryRef::getImportLookupEntry(
@@ -1270,7 +1207,7 @@ std::error_code ImportDirectoryEntryRef::getImportLookupEntry(
   if (std::error_code EC = OwningObject->getRvaPtr(RVA, IntPtr))
     return EC;
   Result = reinterpret_cast<const import_lookup_table_entry32 *>(IntPtr);
-  return object_error::success;
+  return std::error_code();
 }
 
 bool DelayImportDirectoryEntryRef::
@@ -1304,13 +1241,13 @@ std::error_code DelayImportDirectoryEntryRef::getName(StringRef &Result) const {
   if (std::error_code EC = OwningObject->getRvaPtr(Table[Index].Name, IntPtr))
     return EC;
   Result = StringRef(reinterpret_cast<const char *>(IntPtr));
-  return object_error::success;
+  return std::error_code();
 }
 
 std::error_code DelayImportDirectoryEntryRef::
 getDelayImportTable(const delay_import_directory_table_entry *&Result) const {
   Result = Table;
-  return object_error::success;
+  return std::error_code();
 }
 
 std::error_code DelayImportDirectoryEntryRef::
@@ -1324,7 +1261,7 @@ getImportAddress(int AddrIndex, uint64_t &Result) const {
     Result = *reinterpret_cast<const ulittle64_t *>(IntPtr);
   else
     Result = *reinterpret_cast<const ulittle32_t *>(IntPtr);
-  return object_error::success;
+  return std::error_code();
 }
 
 bool ExportDirectoryEntryRef::
@@ -1344,20 +1281,20 @@ std::error_code ExportDirectoryEntryRef::getDllName(StringRef &Result) const {
           OwningObject->getRvaPtr(ExportTable->NameRVA, IntPtr))
     return EC;
   Result = StringRef(reinterpret_cast<const char *>(IntPtr));
-  return object_error::success;
+  return std::error_code();
 }
 
 // Returns the starting ordinal number.
 std::error_code
 ExportDirectoryEntryRef::getOrdinalBase(uint32_t &Result) const {
   Result = ExportTable->OrdinalBase;
-  return object_error::success;
+  return std::error_code();
 }
 
 // Returns the export ordinal of the current export symbol.
 std::error_code ExportDirectoryEntryRef::getOrdinal(uint32_t &Result) const {
   Result = ExportTable->OrdinalBase + Index;
-  return object_error::success;
+  return std::error_code();
 }
 
 // Returns the address of the current export symbol.
@@ -1369,7 +1306,7 @@ std::error_code ExportDirectoryEntryRef::getExportRVA(uint32_t &Result) const {
   const export_address_table_entry *entry =
       reinterpret_cast<const export_address_table_entry *>(IntPtr);
   Result = entry[Index].ExportRVA;
-  return object_error::success;
+  return std::error_code();
 }
 
 // Returns the name of the current export symbol. If the symbol is exported only
@@ -1395,10 +1332,10 @@ ExportDirectoryEntryRef::getSymbolName(StringRef &Result) const {
     if (std::error_code EC = OwningObject->getRvaPtr(NamePtr[Offset], IntPtr))
       return EC;
     Result = StringRef(reinterpret_cast<const char *>(IntPtr));
-    return object_error::success;
+    return std::error_code();
   }
   Result = "";
-  return object_error::success;
+  return std::error_code();
 }
 
 bool ImportedSymbolRef::
@@ -1417,11 +1354,11 @@ ImportedSymbolRef::getSymbolName(StringRef &Result) const {
   if (Entry32) {
     // If a symbol is imported only by ordinal, it has no name.
     if (Entry32[Index].isOrdinal())
-      return object_error::success;
+      return std::error_code();
     RVA = Entry32[Index].getHintNameRVA();
   } else {
     if (Entry64[Index].isOrdinal())
-      return object_error::success;
+      return std::error_code();
     RVA = Entry64[Index].getHintNameRVA();
   }
   uintptr_t IntPtr = 0;
@@ -1429,7 +1366,7 @@ ImportedSymbolRef::getSymbolName(StringRef &Result) const {
     return EC;
   // +2 because the first two bytes is hint.
   Result = StringRef(reinterpret_cast<const char *>(IntPtr + 2));
-  return object_error::success;
+  return std::error_code();
 }
 
 std::error_code ImportedSymbolRef::getOrdinal(uint16_t &Result) const {
@@ -1437,13 +1374,13 @@ std::error_code ImportedSymbolRef::getOrdinal(uint16_t &Result) const {
   if (Entry32) {
     if (Entry32[Index].isOrdinal()) {
       Result = Entry32[Index].getOrdinal();
-      return object_error::success;
+      return std::error_code();
     }
     RVA = Entry32[Index].getHintNameRVA();
   } else {
     if (Entry64[Index].isOrdinal()) {
       Result = Entry64[Index].getOrdinal();
-      return object_error::success;
+      return std::error_code();
     }
     RVA = Entry64[Index].getHintNameRVA();
   }
@@ -1451,7 +1388,7 @@ std::error_code ImportedSymbolRef::getOrdinal(uint16_t &Result) const {
   if (std::error_code EC = OwningObject->getRvaPtr(RVA, IntPtr))
     return EC;
   Result = *reinterpret_cast<const ulittle16_t *>(IntPtr);
-  return object_error::success;
+  return std::error_code();
 }
 
 ErrorOr<std::unique_ptr<COFFObjectFile>>
@@ -1488,11 +1425,11 @@ void BaseRelocRef::moveNext() {
 std::error_code BaseRelocRef::getType(uint8_t &Type) const {
   auto *Entry = reinterpret_cast<const coff_base_reloc_block_entry *>(Header + 1);
   Type = Entry[Index].getType();
-  return object_error::success;
+  return std::error_code();
 }
 
 std::error_code BaseRelocRef::getRVA(uint32_t &Result) const {
   auto *Entry = reinterpret_cast<const coff_base_reloc_block_entry *>(Header + 1);
   Result = Header->PageRVA + Entry[Index].getOffset();
-  return object_error::success;
+  return std::error_code();
 }
diff --git a/lib/Object/COFFYAML.cpp b/lib/Object/COFFYAML.cpp
index 9a24b531da9..4c1fca19bf1 100644
--- a/lib/Object/COFFYAML.cpp
+++ b/lib/Object/COFFYAML.cpp
@@ -56,6 +56,7 @@ void ScalarEnumerationTraits<COFF::MachineTypes>::enumeration(
   ECase(IMAGE_FILE_MACHINE_AMD64);
   ECase(IMAGE_FILE_MACHINE_ARM);
   ECase(IMAGE_FILE_MACHINE_ARMNT);
+  ECase(IMAGE_FILE_MACHINE_ARM64);
   ECase(IMAGE_FILE_MACHINE_EBC);
   ECase(IMAGE_FILE_MACHINE_I386);
   ECase(IMAGE_FILE_MACHINE_IA64);
@@ -210,6 +211,7 @@ void ScalarBitSetTraits<COFF::Characteristics>::bitset(
 
 void ScalarBitSetTraits<COFF::SectionCharacteristics>::bitset(
     IO &IO, COFF::SectionCharacteristics &Value) {
+  BCase(IMAGE_SCN_TYPE_NOLOAD);
   BCase(IMAGE_SCN_TYPE_NO_PAD);
   BCase(IMAGE_SCN_CNT_CODE);
   BCase(IMAGE_SCN_CNT_INITIALIZED_DATA);
diff --git a/lib/Object/ELFObjectFile.cpp b/lib/Object/ELFObjectFile.cpp
index 8ccb2538ac7..c7df30a5903 100644
--- a/lib/Object/ELFObjectFile.cpp
+++ b/lib/Object/ELFObjectFile.cpp
@@ -27,51 +27,28 @@ ObjectFile::createELFObjectFile(MemoryBufferRef Obj) {
   std::size_t MaxAlignment =
       1ULL << countTrailingZeros(uintptr_t(Obj.getBufferStart()));
 
+  if (MaxAlignment < 2)
+    return object_error::parse_failed;
+
   std::error_code EC;
   std::unique_ptr<ObjectFile> R;
-  if (Ident.first == ELF::ELFCLASS32 && Ident.second == ELF::ELFDATA2LSB)
-#if !LLVM_IS_UNALIGNED_ACCESS_FAST
-    if (MaxAlignment >= 4)
-      R.reset(new ELFObjectFile<ELFType<support::little, 4, false>>(Obj, EC));
-    else
-#endif
-    if (MaxAlignment >= 2)
-      R.reset(new ELFObjectFile<ELFType<support::little, 2, false>>(Obj, EC));
-    else
-      return object_error::parse_failed;
-  else if (Ident.first == ELF::ELFCLASS32 && Ident.second == ELF::ELFDATA2MSB)
-#if !LLVM_IS_UNALIGNED_ACCESS_FAST
-    if (MaxAlignment >= 4)
-      R.reset(new ELFObjectFile<ELFType<support::big, 4, false>>(Obj, EC));
-    else
-#endif
-    if (MaxAlignment >= 2)
-      R.reset(new ELFObjectFile<ELFType<support::big, 2, false>>(Obj, EC));
+  if (Ident.first == ELF::ELFCLASS32) {
+    if (Ident.second == ELF::ELFDATA2LSB)
+      R.reset(new ELFObjectFile<ELFType<support::little, false>>(Obj, EC));
+    else if (Ident.second == ELF::ELFDATA2MSB)
+      R.reset(new ELFObjectFile<ELFType<support::big, false>>(Obj, EC));
     else
       return object_error::parse_failed;
-  else if (Ident.first == ELF::ELFCLASS64 && Ident.second == ELF::ELFDATA2MSB)
-#if !LLVM_IS_UNALIGNED_ACCESS_FAST
-    if (MaxAlignment >= 8)
-      R.reset(new ELFObjectFile<ELFType<support::big, 8, true>>(Obj, EC));
-    else
-#endif
-    if (MaxAlignment >= 2)
-      R.reset(new ELFObjectFile<ELFType<support::big, 2, true>>(Obj, EC));
-    else
-      return object_error::parse_failed;
-  else if (Ident.first == ELF::ELFCLASS64 && Ident.second == ELF::ELFDATA2LSB) {
-#if !LLVM_IS_UNALIGNED_ACCESS_FAST
-    if (MaxAlignment >= 8)
-      R.reset(new ELFObjectFile<ELFType<support::little, 8, true>>(Obj, EC));
-    else
-#endif
-    if (MaxAlignment >= 2)
-      R.reset(new ELFObjectFile<ELFType<support::little, 2, true>>(Obj, EC));
+  } else if (Ident.first == ELF::ELFCLASS64) {
+    if (Ident.second == ELF::ELFDATA2LSB)
+      R.reset(new ELFObjectFile<ELFType<support::little, true>>(Obj, EC));
+    else if (Ident.second == ELF::ELFDATA2MSB)
+      R.reset(new ELFObjectFile<ELFType<support::big, true>>(Obj, EC));
     else
       return object_error::parse_failed;
+  } else {
+    return object_error::parse_failed;
   }
-  else
-    llvm_unreachable("Buffer is not an ELF object file!");
 
   if (EC)
     return EC;
diff --git a/lib/Object/ELFYAML.cpp b/lib/Object/ELFYAML.cpp
index 19527e2d040..7a9cf6fcf9f 100644
--- a/lib/Object/ELFYAML.cpp
+++ b/lib/Object/ELFYAML.cpp
@@ -13,6 +13,7 @@
 
 #include "llvm/Object/ELFYAML.h"
 #include "llvm/Support/Casting.h"
+#include "llvm/Support/MipsABIFlags.h"
 
 namespace llvm {
 
@@ -43,7 +44,7 @@ ScalarEnumerationTraits<ELFYAML::ELF_EM>::enumeration(IO &IO,
   ECase(EM_386)
   ECase(EM_68K)
   ECase(EM_88K)
-  ECase(EM_486)
+  ECase(EM_IAMCU)
   ECase(EM_860)
   ECase(EM_MIPS)
   ECase(EM_S370)
@@ -192,6 +193,7 @@ ScalarEnumerationTraits<ELFYAML::ELF_EM>::enumeration(IO &IO,
   ECase(EM_VIDEOCORE5)
   ECase(EM_78KOR)
   ECase(EM_56800EX)
+  ECase(EM_AMDGPU)
 #undef ECase
 }
 
@@ -266,10 +268,33 @@ void ScalarBitSetTraits<ELFYAML::ELF_EF>::bitset(IO &IO,
     BCase(EF_MIPS_CPIC)
     BCase(EF_MIPS_ABI2)
     BCase(EF_MIPS_32BITMODE)
+    BCase(EF_MIPS_FP64)
     BCase(EF_MIPS_NAN2008)
-    BCase(EF_MIPS_ABI_O32)
     BCase(EF_MIPS_MICROMIPS)
     BCase(EF_MIPS_ARCH_ASE_M16)
+    BCase(EF_MIPS_ARCH_ASE_MDMX)
+    BCaseMask(EF_MIPS_ABI_O32, EF_MIPS_ABI)
+    BCaseMask(EF_MIPS_ABI_O64, EF_MIPS_ABI)
+    BCaseMask(EF_MIPS_ABI_EABI32, EF_MIPS_ABI)
+    BCaseMask(EF_MIPS_ABI_EABI64, EF_MIPS_ABI)
+    BCaseMask(EF_MIPS_MACH_3900, EF_MIPS_MACH)
+    BCaseMask(EF_MIPS_MACH_4010, EF_MIPS_MACH)
+    BCaseMask(EF_MIPS_MACH_4100, EF_MIPS_MACH)
+    BCaseMask(EF_MIPS_MACH_4650, EF_MIPS_MACH)
+    BCaseMask(EF_MIPS_MACH_4120, EF_MIPS_MACH)
+    BCaseMask(EF_MIPS_MACH_4111, EF_MIPS_MACH)
+    BCaseMask(EF_MIPS_MACH_SB1, EF_MIPS_MACH)
+    BCaseMask(EF_MIPS_MACH_OCTEON, EF_MIPS_MACH)
+    BCaseMask(EF_MIPS_MACH_XLR, EF_MIPS_MACH)
+    BCaseMask(EF_MIPS_MACH_OCTEON2, EF_MIPS_MACH)
+    BCaseMask(EF_MIPS_MACH_OCTEON3, EF_MIPS_MACH)
+    BCaseMask(EF_MIPS_MACH_5400, EF_MIPS_MACH)
+    BCaseMask(EF_MIPS_MACH_5900, EF_MIPS_MACH)
+    BCaseMask(EF_MIPS_MACH_5500, EF_MIPS_MACH)
+    BCaseMask(EF_MIPS_MACH_9000, EF_MIPS_MACH)
+    BCaseMask(EF_MIPS_MACH_LS2E, EF_MIPS_MACH)
+    BCaseMask(EF_MIPS_MACH_LS2F, EF_MIPS_MACH)
+    BCaseMask(EF_MIPS_MACH_LS3A, EF_MIPS_MACH)
     BCaseMask(EF_MIPS_ARCH_1, EF_MIPS_ARCH)
     BCaseMask(EF_MIPS_ARCH_2, EF_MIPS_ARCH)
     BCaseMask(EF_MIPS_ARCH_3, EF_MIPS_ARCH)
@@ -456,6 +481,93 @@ void ScalarEnumerationTraits<ELFYAML::ELF_REL>::enumeration(
 #undef ELF_RELOC
 }
 
+void ScalarEnumerationTraits<ELFYAML::MIPS_AFL_REG>::enumeration(
+    IO &IO, ELFYAML::MIPS_AFL_REG &Value) {
+#define ECase(X) IO.enumCase(Value, #X, Mips::AFL_##X);
+  ECase(REG_NONE)
+  ECase(REG_32)
+  ECase(REG_64)
+  ECase(REG_128)
+#undef ECase
+}
+
+void ScalarEnumerationTraits<ELFYAML::MIPS_ABI_FP>::enumeration(
+    IO &IO, ELFYAML::MIPS_ABI_FP &Value) {
+#define ECase(X) IO.enumCase(Value, #X, Mips::Val_GNU_MIPS_ABI_##X);
+  ECase(FP_ANY)
+  ECase(FP_DOUBLE)
+  ECase(FP_SINGLE)
+  ECase(FP_SOFT)
+  ECase(FP_OLD_64)
+  ECase(FP_XX)
+  ECase(FP_64)
+  ECase(FP_64A)
+#undef ECase
+}
+
+void ScalarEnumerationTraits<ELFYAML::MIPS_AFL_EXT>::enumeration(
+    IO &IO, ELFYAML::MIPS_AFL_EXT &Value) {
+#define ECase(X) IO.enumCase(Value, #X, Mips::AFL_##X);
+  ECase(EXT_NONE)
+  ECase(EXT_XLR)
+  ECase(EXT_OCTEON2)
+  ECase(EXT_OCTEONP)
+  ECase(EXT_LOONGSON_3A)
+  ECase(EXT_OCTEON)
+  ECase(EXT_5900)
+  ECase(EXT_4650)
+  ECase(EXT_4010)
+  ECase(EXT_4100)
+  ECase(EXT_3900)
+  ECase(EXT_10000)
+  ECase(EXT_SB1)
+  ECase(EXT_4111)
+  ECase(EXT_4120)
+  ECase(EXT_5400)
+  ECase(EXT_5500)
+  ECase(EXT_LOONGSON_2E)
+  ECase(EXT_LOONGSON_2F)
+  ECase(EXT_OCTEON3)
+#undef ECase
+}
+
+void ScalarEnumerationTraits<ELFYAML::MIPS_ISA>::enumeration(
+    IO &IO, ELFYAML::MIPS_ISA &Value) {
+  IO.enumCase(Value, "MIPS1", 1);
+  IO.enumCase(Value, "MIPS2", 2);
+  IO.enumCase(Value, "MIPS3", 3);
+  IO.enumCase(Value, "MIPS4", 4);
+  IO.enumCase(Value, "MIPS5", 5);
+  IO.enumCase(Value, "MIPS32", 32);
+  IO.enumCase(Value, "MIPS64", 64);
+}
+
+void ScalarBitSetTraits<ELFYAML::MIPS_AFL_ASE>::bitset(
+    IO &IO, ELFYAML::MIPS_AFL_ASE &Value) {
+#define BCase(X) IO.bitSetCase(Value, #X, Mips::AFL_ASE_##X);
+  BCase(DSP)
+  BCase(DSPR2)
+  BCase(EVA)
+  BCase(MCU)
+  BCase(MDMX)
+  BCase(MIPS3D)
+  BCase(MT)
+  BCase(SMARTMIPS)
+  BCase(VIRT)
+  BCase(MSA)
+  BCase(MIPS16)
+  BCase(MICROMIPS)
+  BCase(XPA)
+#undef BCase
+}
+
+void ScalarBitSetTraits<ELFYAML::MIPS_AFL_FLAGS1>::bitset(
+    IO &IO, ELFYAML::MIPS_AFL_FLAGS1 &Value) {
+#define BCase(X) IO.bitSetCase(Value, #X, Mips::AFL_FLAGS1_##X);
+  BCase(ODDSPREG)
+#undef BCase
+}
+
 void MappingTraits<ELFYAML::FileHeader>::mapping(IO &IO,
                                                  ELFYAML::FileHeader &FileHdr) {
   IO.mapRequired("Class", FileHdr.Class);
@@ -516,6 +628,11 @@ static void sectionMapping(IO &IO, ELFYAML::RawContentSection &Section) {
   IO.mapOptional("Size", Section.Size, Hex64(Section.Content.binary_size()));
 }
 
+static void sectionMapping(IO &IO, ELFYAML::NoBitsSection &Section) {
+  commonSectionMapping(IO, Section);
+  IO.mapOptional("Size", Section.Size, Hex64(0));
+}
+
 static void sectionMapping(IO &IO, ELFYAML::RelocationSection &Section) {
   commonSectionMapping(IO, Section);
   IO.mapOptional("Relocations", Section.Relocations);
@@ -531,6 +648,26 @@ void MappingTraits<ELFYAML::SectionOrType>::mapping(
   IO.mapRequired("SectionOrType", sectionOrType.sectionNameOrType);
 }
 
+static void sectionMapping(IO &IO, ELFYAML::MipsABIFlags &Section) {
+  commonSectionMapping(IO, Section);
+  IO.mapOptional("Version", Section.Version, Hex16(0));
+  IO.mapRequired("ISA", Section.ISALevel);
+  IO.mapOptional("ISARevision", Section.ISARevision, Hex8(0));
+  IO.mapOptional("ISAExtension", Section.ISAExtension,
+                 ELFYAML::MIPS_AFL_EXT(Mips::AFL_EXT_NONE));
+  IO.mapOptional("ASEs", Section.ASEs, ELFYAML::MIPS_AFL_ASE(0));
+  IO.mapOptional("FpABI", Section.FpABI,
+                 ELFYAML::MIPS_ABI_FP(Mips::Val_GNU_MIPS_ABI_FP_ANY));
+  IO.mapOptional("GPRSize", Section.GPRSize,
+                 ELFYAML::MIPS_AFL_REG(Mips::AFL_REG_NONE));
+  IO.mapOptional("CPR1Size", Section.CPR1Size,
+                 ELFYAML::MIPS_AFL_REG(Mips::AFL_REG_NONE));
+  IO.mapOptional("CPR2Size", Section.CPR2Size,
+                 ELFYAML::MIPS_AFL_REG(Mips::AFL_REG_NONE));
+  IO.mapOptional("Flags1", Section.Flags1, ELFYAML::MIPS_AFL_FLAGS1(0));
+  IO.mapOptional("Flags2", Section.Flags2, Hex32(0));
+}
+
 void MappingTraits<std::unique_ptr<ELFYAML::Section>>::mapping(
     IO &IO, std::unique_ptr<ELFYAML::Section> &Section) {
   ELFYAML::ELF_SHT sectionType;
@@ -551,6 +688,16 @@ void MappingTraits<std::unique_ptr<ELFYAML::Section>>::mapping(
       Section.reset(new ELFYAML::Group());
     groupSectionMapping(IO, *cast<ELFYAML::Group>(Section.get()));
     break;
+  case ELF::SHT_NOBITS:
+    if (!IO.outputting())
+      Section.reset(new ELFYAML::NoBitsSection());
+    sectionMapping(IO, *cast<ELFYAML::NoBitsSection>(Section.get()));
+    break;
+  case ELF::SHT_MIPS_ABIFLAGS:
+    if (!IO.outputting())
+      Section.reset(new ELFYAML::MipsABIFlags());
+    sectionMapping(IO, *cast<ELFYAML::MipsABIFlags>(Section.get()));
+    break;
   default:
     if (!IO.outputting())
       Section.reset(new ELFYAML::RawContentSection());
@@ -620,5 +767,11 @@ void MappingTraits<ELFYAML::Object>::mapping(IO &IO, ELFYAML::Object &Object) {
   IO.setContext(nullptr);
 }
 
+LLVM_YAML_STRONG_TYPEDEF(uint8_t, MIPS_AFL_REG)
+LLVM_YAML_STRONG_TYPEDEF(uint8_t, MIPS_ABI_FP)
+LLVM_YAML_STRONG_TYPEDEF(uint32_t, MIPS_AFL_EXT)
+LLVM_YAML_STRONG_TYPEDEF(uint32_t, MIPS_AFL_ASE)
+LLVM_YAML_STRONG_TYPEDEF(uint32_t, MIPS_AFL_FLAGS1)
+
 } // end namespace yaml
 } // end namespace llvm
diff --git a/lib/Object/Error.cpp b/lib/Object/Error.cpp
index d2daab72d58..7ca2f12f092 100644
--- a/lib/Object/Error.cpp
+++ b/lib/Object/Error.cpp
@@ -33,7 +33,6 @@ const char *_object_error_category::name() const LLVM_NOEXCEPT {
 std::string _object_error_category::message(int EV) const {
   object_error E = static_cast<object_error>(EV);
   switch (E) {
-  case object_error::success: return "Success";
   case object_error::arch_not_found:
     return "No object file for requested architecture";
   case object_error::invalid_file_type:
@@ -42,8 +41,18 @@ std::string _object_error_category::message(int EV) const {
     return "Invalid data was encountered while parsing the file";
   case object_error::unexpected_eof:
     return "The end of the file was unexpectedly encountered";
+  case object_error::string_table_non_null_end:
+    return "String table must end with a null terminator";
+  case object_error::invalid_section_index:
+    return "Invalid section index";
   case object_error::bitcode_section_not_found:
     return "Bitcode section not found in object file";
+  case object_error::macho_small_load_command:
+    return "Mach-O load command with size < 8 bytes";
+  case object_error::macho_load_segment_too_many_sections:
+    return "Mach-O segment load command contains too many sections";
+  case object_error::macho_load_segment_too_small:
+    return "Mach-O segment load command size is too small";
   }
   llvm_unreachable("An enumerator of object_error does not have a message "
                    "defined.");
diff --git a/lib/Object/IRObjectFile.cpp b/lib/Object/IRObjectFile.cpp
index 58c4ca312e8..9f5132e9062 100644
--- a/lib/Object/IRObjectFile.cpp
+++ b/lib/Object/IRObjectFile.cpp
@@ -25,6 +25,7 @@
 #include "llvm/MC/MCObjectFileInfo.h"
 #include "llvm/MC/MCParser/MCAsmParser.h"
 #include "llvm/MC/MCRegisterInfo.h"
+#include "llvm/MC/MCSubtargetInfo.h"
 #include "llvm/MC/MCTargetAsmParser.h"
 #include "llvm/Object/ObjectFile.h"
 #include "llvm/Support/MemoryBuffer.h"
@@ -36,30 +37,28 @@ using namespace object;
 
 IRObjectFile::IRObjectFile(MemoryBufferRef Object, std::unique_ptr<Module> Mod)
     : SymbolicFile(Binary::ID_IR, Object), M(std::move(Mod)) {
-  // Setup a mangler with the DataLayout.
-  const DataLayout &DL = M->getDataLayout();
-  Mang.reset(new Mangler(&DL));
+  Mang.reset(new Mangler());
 
   const std::string &InlineAsm = M->getModuleInlineAsm();
   if (InlineAsm.empty())
     return;
 
-  StringRef Triple = M->getTargetTriple();
+  Triple TT(M->getTargetTriple());
   std::string Err;
-  const Target *T = TargetRegistry::lookupTarget(Triple, Err);
+  const Target *T = TargetRegistry::lookupTarget(TT.str(), Err);
   if (!T)
     return;
 
-  std::unique_ptr<MCRegisterInfo> MRI(T->createMCRegInfo(Triple));
+  std::unique_ptr<MCRegisterInfo> MRI(T->createMCRegInfo(TT.str()));
   if (!MRI)
     return;
 
-  std::unique_ptr<MCAsmInfo> MAI(T->createMCAsmInfo(*MRI, Triple));
+  std::unique_ptr<MCAsmInfo> MAI(T->createMCAsmInfo(*MRI, TT.str()));
   if (!MAI)
     return;
 
   std::unique_ptr<MCSubtargetInfo> STI(
-      T->createMCSubtargetInfo(Triple, "", ""));
+      T->createMCSubtargetInfo(TT.str(), "", ""));
   if (!STI)
     return;
 
@@ -69,7 +68,7 @@ IRObjectFile::IRObjectFile(MemoryBufferRef Object, std::unique_ptr<Module> Mod)
 
   MCObjectFileInfo MOFI;
   MCContext MCCtx(MAI.get(), MRI.get(), &MOFI);
-  MOFI.InitMCObjectFileInfo(Triple, Reloc::Default, CodeModel::Default, MCCtx);
+  MOFI.InitMCObjectFileInfo(TT, Reloc::Default, CodeModel::Default, MCCtx);
   std::unique_ptr<RecordStreamer> Streamer(new RecordStreamer(MCCtx));
   T->createNullTargetStreamer(*Streamer);
 
@@ -194,15 +193,18 @@ std::error_code IRObjectFile::printSymbolName(raw_ostream &OS,
     unsigned Index = getAsmSymIndex(Symb);
     assert(Index <= AsmSymbols.size());
     OS << AsmSymbols[Index].first;
-    return object_error::success;;
+    return std::error_code();
   }
 
+  if (GV->hasDLLImportStorageClass())
+    OS << "__imp_";
+
   if (Mang)
     Mang->getNameWithPrefix(OS, GV, false);
   else
     OS << GV->getName();
 
-  return object_error::success;
+  return std::error_code();
 }
 
 uint32_t IRObjectFile::getSymbolFlags(DataRefImpl Symb) const {
@@ -300,12 +302,12 @@ llvm::object::IRObjectFile::create(MemoryBufferRef Object,
   std::unique_ptr<MemoryBuffer> Buff(
       MemoryBuffer::getMemBuffer(BCOrErr.get(), false));
 
-  ErrorOr<Module *> MOrErr =
+  ErrorOr<std::unique_ptr<Module>> MOrErr =
       getLazyBitcodeModule(std::move(Buff), Context, nullptr,
                            /*ShouldLazyLoadMetadata*/ true);
   if (std::error_code EC = MOrErr.getError())
     return EC;
 
-  std::unique_ptr<Module> M(MOrErr.get());
+  std::unique_ptr<Module> &M = MOrErr.get();
   return llvm::make_unique<IRObjectFile>(Object, std::move(M));
 }
diff --git a/lib/Object/MachOObjectFile.cpp b/lib/Object/MachOObjectFile.cpp
index 7129aa324ab..05900630c75 100644
--- a/lib/Object/MachOObjectFile.cpp
+++ b/lib/Object/MachOObjectFile.cpp
@@ -38,6 +38,7 @@ namespace {
   };
 }
 
+// FIXME: Replace all uses of this function with getStructOrErr.
 template <typename T>
 static T getStruct(const MachOObjectFile *O, const char *P) {
   // Don't read before the beginning or past the end of the file
@@ -51,39 +52,19 @@ static T getStruct(const MachOObjectFile *O, const char *P) {
   return Cmd;
 }
 
-template <typename SegmentCmd>
-static uint32_t getSegmentLoadCommandNumSections(const SegmentCmd &S,
-                                                 uint32_t Cmdsize) {
-  const unsigned SectionSize = sizeof(SegmentCmd);
-  if (S.nsects > std::numeric_limits<uint32_t>::max() / SectionSize ||
-      S.nsects * SectionSize > Cmdsize - sizeof(S))
-    report_fatal_error(
-        "Number of sections too large for size of load command.");
-  return S.nsects;
-}
-
-static uint32_t
-getSegmentLoadCommandNumSections(const MachOObjectFile *O,
-                                 const MachOObjectFile::LoadCommandInfo &L) {
-  if (O->is64Bit())
-    return getSegmentLoadCommandNumSections(O->getSegment64LoadCommand(L),
-                                            L.C.cmdsize);
-
-  return getSegmentLoadCommandNumSections(O->getSegmentLoadCommand(L),
-                                          L.C.cmdsize);
-}
+template <typename T>
+static ErrorOr<T> getStructOrErr(const MachOObjectFile *O, const char *P) {
+  // Don't read before the beginning or past the end of the file
+  if (P < O->getData().begin() || P + sizeof(T) > O->getData().end())
+    return object_error::parse_failed;
 
-static bool isPageZeroSegment(const MachOObjectFile *O,
-                              const MachOObjectFile::LoadCommandInfo &L) {
-  if (O->is64Bit()) {
-    MachO::segment_command_64 S = O->getSegment64LoadCommand(L);
-    return StringRef("__PAGEZERO").equals(S.segname);
-  }
-  MachO::segment_command S = O->getSegmentLoadCommand(L);
-  return StringRef("__PAGEZERO").equals(S.segname);
+  T Cmd;
+  memcpy(&Cmd, P, sizeof(T));
+  if (O->isLittleEndian() != sys::IsLittleEndianHost)
+    MachO::swapStruct(Cmd);
+  return Cmd;
 }
 
-
 static const char *
 getSectionPtr(const MachOObjectFile *O, MachOObjectFile::LoadCommandInfo L,
               unsigned Sec) {
@@ -128,70 +109,6 @@ static unsigned getCPUType(const MachOObjectFile *O) {
   return O->getHeader().cputype;
 }
 
-static void printRelocationTargetName(const MachOObjectFile *O,
-                                      const MachO::any_relocation_info &RE,
-                                      raw_string_ostream &fmt) {
-  bool IsScattered = O->isRelocationScattered(RE);
-
-  // Target of a scattered relocation is an address.  In the interest of
-  // generating pretty output, scan through the symbol table looking for a
-  // symbol that aligns with that address.  If we find one, print it.
-  // Otherwise, we just print the hex address of the target.
-  if (IsScattered) {
-    uint32_t Val = O->getPlainRelocationSymbolNum(RE);
-
-    for (const SymbolRef &Symbol : O->symbols()) {
-      std::error_code ec;
-      uint64_t Addr;
-      StringRef Name;
-
-      if ((ec = Symbol.getAddress(Addr)))
-        report_fatal_error(ec.message());
-      if (Addr != Val)
-        continue;
-      if ((ec = Symbol.getName(Name)))
-        report_fatal_error(ec.message());
-      fmt << Name;
-      return;
-    }
-
-    // If we couldn't find a symbol that this relocation refers to, try
-    // to find a section beginning instead.
-    for (const SectionRef &Section : O->sections()) {
-      std::error_code ec;
-
-      StringRef Name;
-      uint64_t Addr = Section.getAddress();
-      if (Addr != Val)
-        continue;
-      if ((ec = Section.getName(Name)))
-        report_fatal_error(ec.message());
-      fmt << Name;
-      return;
-    }
-
-    fmt << format("0x%x", Val);
-    return;
-  }
-
-  StringRef S;
-  bool isExtern = O->getPlainRelocationExternal(RE);
-  uint64_t Val = O->getPlainRelocationSymbolNum(RE);
-
-  if (isExtern) {
-    symbol_iterator SI = O->symbol_begin();
-    advance(SI, Val);
-    SI->getName(S);
-  } else {
-    section_iterator SI = O->section_begin();
-    // Adjust for the fact that sections are 1-indexed.
-    advance(SI, Val - 1);
-    SI->getName(S);
-  }
-
-  fmt << S;
-}
-
 static uint32_t
 getPlainRelocationAddress(const MachO::any_relocation_info &RE) {
   return RE.r_word0;
@@ -244,6 +161,69 @@ static uint32_t getSectionFlags(const MachOObjectFile *O,
   return Sect.flags;
 }
 
+static ErrorOr<MachOObjectFile::LoadCommandInfo>
+getLoadCommandInfo(const MachOObjectFile *Obj, const char *Ptr) {
+  auto CmdOrErr = getStructOrErr<MachO::load_command>(Obj, Ptr);
+  if (!CmdOrErr)
+    return CmdOrErr.getError();
+  if (CmdOrErr->cmdsize < 8)
+    return object_error::macho_small_load_command;
+  MachOObjectFile::LoadCommandInfo Load;
+  Load.Ptr = Ptr;
+  Load.C = CmdOrErr.get();
+  return Load;
+}
+
+static ErrorOr<MachOObjectFile::LoadCommandInfo>
+getFirstLoadCommandInfo(const MachOObjectFile *Obj) {
+  unsigned HeaderSize = Obj->is64Bit() ? sizeof(MachO::mach_header_64)
+                                       : sizeof(MachO::mach_header);
+  return getLoadCommandInfo(Obj, getPtr(Obj, HeaderSize));
+}
+
+static ErrorOr<MachOObjectFile::LoadCommandInfo>
+getNextLoadCommandInfo(const MachOObjectFile *Obj,
+                       const MachOObjectFile::LoadCommandInfo &L) {
+  return getLoadCommandInfo(Obj, L.Ptr + L.C.cmdsize);
+}
+
+template <typename T>
+static void parseHeader(const MachOObjectFile *Obj, T &Header,
+                        std::error_code &EC) {
+  auto HeaderOrErr = getStructOrErr<T>(Obj, getPtr(Obj, 0));
+  if (HeaderOrErr)
+    Header = HeaderOrErr.get();
+  else
+    EC = HeaderOrErr.getError();
+}
+
+// Parses LC_SEGMENT or LC_SEGMENT_64 load command, adds addresses of all
+// sections to \param Sections, and optionally sets
+// \param IsPageZeroSegment to true.
+template <typename SegmentCmd>
+static std::error_code parseSegmentLoadCommand(
+    const MachOObjectFile *Obj, const MachOObjectFile::LoadCommandInfo &Load,
+    SmallVectorImpl<const char *> &Sections, bool &IsPageZeroSegment) {
+  const unsigned SegmentLoadSize = sizeof(SegmentCmd);
+  if (Load.C.cmdsize < SegmentLoadSize)
+    return object_error::macho_load_segment_too_small;
+  auto SegOrErr = getStructOrErr<SegmentCmd>(Obj, Load.Ptr);
+  if (!SegOrErr)
+    return SegOrErr.getError();
+  SegmentCmd S = SegOrErr.get();
+  const unsigned SectionSize =
+      Obj->is64Bit() ? sizeof(MachO::section_64) : sizeof(MachO::section);
+  if (S.nsects > std::numeric_limits<uint32_t>::max() / SectionSize ||
+      S.nsects * SectionSize > Load.C.cmdsize - SegmentLoadSize)
+    return object_error::macho_load_segment_too_many_sections;
+  for (unsigned J = 0; J < S.nsects; ++J) {
+    const char *Sec = getSectionPtr(Obj, Load, J);
+    Sections.push_back(Sec);
+  }
+  IsPageZeroSegment |= StringRef("__PAGEZERO").equals(S.segname);
+  return std::error_code();
+}
+
 MachOObjectFile::MachOObjectFile(MemoryBufferRef Object, bool IsLittleEndian,
                                  bool Is64bits, std::error_code &EC)
     : ObjectFile(getMachOType(IsLittleEndian, Is64bits), Object),
@@ -251,15 +231,25 @@ MachOObjectFile::MachOObjectFile(MemoryBufferRef Object, bool IsLittleEndian,
       DataInCodeLoadCmd(nullptr), LinkOptHintsLoadCmd(nullptr),
       DyldInfoLoadCmd(nullptr), UuidLoadCmd(nullptr),
       HasPageZeroSegment(false) {
-  uint32_t LoadCommandCount = this->getHeader().ncmds;
-  if (LoadCommandCount == 0)
+  if (is64Bit())
+    parseHeader(this, Header64, EC);
+  else
+    parseHeader(this, Header, EC);
+  if (EC)
     return;
 
-  MachO::LoadCommandType SegmentLoadType = is64Bit() ?
-    MachO::LC_SEGMENT_64 : MachO::LC_SEGMENT;
+  uint32_t LoadCommandCount = getHeader().ncmds;
+  if (LoadCommandCount == 0)
+    return;
 
-  MachOObjectFile::LoadCommandInfo Load = getFirstLoadCommandInfo();
-  for (unsigned I = 0; ; ++I) {
+  auto LoadOrErr = getFirstLoadCommandInfo(this);
+  if (!LoadOrErr) {
+    EC = LoadOrErr.getError();
+    return;
+  }
+  LoadCommandInfo Load = LoadOrErr.get();
+  for (unsigned I = 0; I < LoadCommandCount; ++I) {
+    LoadCommands.push_back(Load);
     if (Load.C.cmd == MachO::LC_SYMTAB) {
       // Multiple symbol tables
       if (SymtabLoadCmd) {
@@ -303,20 +293,14 @@ MachOObjectFile::MachOObjectFile(MemoryBufferRef Object, bool IsLittleEndian,
         return;
       }
       UuidLoadCmd = Load.Ptr;
-    } else if (Load.C.cmd == SegmentLoadType) {
-      const unsigned SegmentLoadSize = this->is64Bit()
-                                           ? sizeof(MachO::segment_command_64)
-                                           : sizeof(MachO::segment_command);
-      if (Load.C.cmdsize < SegmentLoadSize)
-        report_fatal_error("Segment load command size is too small.");
-
-      uint32_t NumSections = getSegmentLoadCommandNumSections(this, Load);
-      for (unsigned J = 0; J < NumSections; ++J) {
-        const char *Sec = getSectionPtr(this, Load, J);
-        Sections.push_back(Sec);
-      }
-      if (isPageZeroSegment(this, Load))
-        HasPageZeroSegment = true;
+    } else if (Load.C.cmd == MachO::LC_SEGMENT_64) {
+      if ((EC = parseSegmentLoadCommand<MachO::segment_command_64>(
+               this, Load, Sections, HasPageZeroSegment)))
+        return;
+    } else if (Load.C.cmd == MachO::LC_SEGMENT) {
+      if ((EC = parseSegmentLoadCommand<MachO::segment_command>(
+               this, Load, Sections, HasPageZeroSegment)))
+        return;
     } else if (Load.C.cmd == MachO::LC_LOAD_DYLIB ||
                Load.C.cmd == MachO::LC_LOAD_WEAK_DYLIB ||
                Load.C.cmd == MachO::LC_LAZY_LOAD_DYLIB ||
@@ -324,12 +308,16 @@ MachOObjectFile::MachOObjectFile(MemoryBufferRef Object, bool IsLittleEndian,
                Load.C.cmd == MachO::LC_LOAD_UPWARD_DYLIB) {
       Libraries.push_back(Load.Ptr);
     }
-
-    if (I == LoadCommandCount - 1)
-      break;
-    else
-      Load = getNextLoadCommandInfo(Load);
+    if (I < LoadCommandCount - 1) {
+      auto LoadOrErr = getNextLoadCommandInfo(this, Load);
+      if (!LoadOrErr) {
+        EC = LoadOrErr.getError();
+        return;
+      }
+      Load = LoadOrErr.get();
+    }
   }
+  assert(LoadCommands.size() == LoadCommandCount);
 }
 
 void MachOObjectFile::moveSymbolNext(DataRefImpl &Symb) const {
@@ -339,16 +327,14 @@ void MachOObjectFile::moveSymbolNext(DataRefImpl &Symb) const {
   Symb.p += SymbolTableEntrySize;
 }
 
-std::error_code MachOObjectFile::getSymbolName(DataRefImpl Symb,
-                                               StringRef &Res) const {
+ErrorOr<StringRef> MachOObjectFile::getSymbolName(DataRefImpl Symb) const {
   StringRef StringTable = getStringTableData();
   MachO::nlist_base Entry = getSymbolTableEntryBase(this, Symb);
   const char *Start = &StringTable.data()[Entry.n_strx];
   if (Start < getData().begin() || Start >= getData().end())
     report_fatal_error(
         "Symbol name entry points before beginning or past end of file.");
-  Res = StringRef(Start);
-  return object_error::success;
+  return StringRef(Start);
 }
 
 unsigned MachOObjectFile::getSectionType(SectionRef Sec) const {
@@ -357,132 +343,67 @@ unsigned MachOObjectFile::getSectionType(SectionRef Sec) const {
   return Flags & MachO::SECTION_TYPE;
 }
 
+uint64_t MachOObjectFile::getNValue(DataRefImpl Sym) const {
+  if (is64Bit()) {
+    MachO::nlist_64 Entry = getSymbol64TableEntry(Sym);
+    return Entry.n_value;
+  }
+  MachO::nlist Entry = getSymbolTableEntry(Sym);
+  return Entry.n_value;
+}
+
 // getIndirectName() returns the name of the alias'ed symbol who's string table
 // index is in the n_value field.
 std::error_code MachOObjectFile::getIndirectName(DataRefImpl Symb,
                                                  StringRef &Res) const {
   StringRef StringTable = getStringTableData();
-  uint64_t NValue;
-  if (is64Bit()) {
-    MachO::nlist_64 Entry = getSymbol64TableEntry(Symb);
-    NValue = Entry.n_value;
-    if ((Entry.n_type & MachO::N_TYPE) != MachO::N_INDR)
-      return object_error::parse_failed;
-  } else {
-    MachO::nlist Entry = getSymbolTableEntry(Symb);
-    NValue = Entry.n_value;
-    if ((Entry.n_type & MachO::N_TYPE) != MachO::N_INDR)
-      return object_error::parse_failed;
-  }
+  MachO::nlist_base Entry = getSymbolTableEntryBase(this, Symb);
+  if ((Entry.n_type & MachO::N_TYPE) != MachO::N_INDR)
+    return object_error::parse_failed;
+  uint64_t NValue = getNValue(Symb);
   if (NValue >= StringTable.size())
     return object_error::parse_failed;
   const char *Start = &StringTable.data()[NValue];
   Res = StringRef(Start);
-  return object_error::success;
+  return std::error_code();
 }
 
-std::error_code MachOObjectFile::getSymbolAddress(DataRefImpl Symb,
-                                                  uint64_t &Res) const {
-  if (is64Bit()) {
-    MachO::nlist_64 Entry = getSymbol64TableEntry(Symb);
-    if ((Entry.n_type & MachO::N_TYPE) == MachO::N_UNDF &&
-        Entry.n_value == 0)
-      Res = UnknownAddressOrSize;
-    else
-      Res = Entry.n_value;
-  } else {
-    MachO::nlist Entry = getSymbolTableEntry(Symb);
-    if ((Entry.n_type & MachO::N_TYPE) == MachO::N_UNDF &&
-        Entry.n_value == 0)
-      Res = UnknownAddressOrSize;
-    else
-      Res = Entry.n_value;
-  }
-  return object_error::success;
+uint64_t MachOObjectFile::getSymbolValueImpl(DataRefImpl Sym) const {
+  return getNValue(Sym);
+}
+
+ErrorOr<uint64_t> MachOObjectFile::getSymbolAddress(DataRefImpl Sym) const {
+  return getSymbolValue(Sym);
 }
 
-std::error_code MachOObjectFile::getSymbolAlignment(DataRefImpl DRI,
-                                                    uint32_t &Result) const {
+uint32_t MachOObjectFile::getSymbolAlignment(DataRefImpl DRI) const {
   uint32_t flags = getSymbolFlags(DRI);
   if (flags & SymbolRef::SF_Common) {
     MachO::nlist_base Entry = getSymbolTableEntryBase(this, DRI);
-    Result = 1 << MachO::GET_COMM_ALIGN(Entry.n_desc);
-  } else {
-    Result = 0;
+    return 1 << MachO::GET_COMM_ALIGN(Entry.n_desc);
   }
-  return object_error::success;
+  return 0;
 }
 
-std::error_code MachOObjectFile::getSymbolSize(DataRefImpl DRI,
-                                               uint64_t &Result) const {
-  uint64_t BeginOffset;
-  uint64_t EndOffset = 0;
-  uint8_t SectionIndex;
-
-  MachO::nlist_base Entry = getSymbolTableEntryBase(this, DRI);
-  uint64_t Value;
-  getSymbolAddress(DRI, Value);
-  if (Value == UnknownAddressOrSize) {
-    Result = UnknownAddressOrSize;
-    return object_error::success;
-  }
-
-  BeginOffset = Value;
-
-  SectionIndex = Entry.n_sect;
-  if (!SectionIndex) {
-    uint32_t flags = getSymbolFlags(DRI);
-    if (flags & SymbolRef::SF_Common)
-      Result = Value;
-    else
-      Result = UnknownAddressOrSize;
-    return object_error::success;
-  }
-  // Unfortunately symbols are unsorted so we need to touch all
-  // symbols from load command
-  for (const SymbolRef &Symbol : symbols()) {
-    DataRefImpl DRI = Symbol.getRawDataRefImpl();
-    Entry = getSymbolTableEntryBase(this, DRI);
-    getSymbolAddress(DRI, Value);
-    if (Value == UnknownAddressOrSize)
-      continue;
-    if (Entry.n_sect == SectionIndex && Value > BeginOffset)
-      if (!EndOffset || Value < EndOffset)
-        EndOffset = Value;
-  }
-  if (!EndOffset) {
-    DataRefImpl Sec;
-    Sec.d.a = SectionIndex-1;
-    uint64_t Size = getSectionSize(Sec);
-    EndOffset = getSectionAddress(Sec);
-    EndOffset += Size;
-  }
-  Result = EndOffset - BeginOffset;
-  return object_error::success;
+uint64_t MachOObjectFile::getCommonSymbolSizeImpl(DataRefImpl DRI) const {
+  return getNValue(DRI);
 }
 
-std::error_code MachOObjectFile::getSymbolType(DataRefImpl Symb,
-                                               SymbolRef::Type &Res) const {
+SymbolRef::Type MachOObjectFile::getSymbolType(DataRefImpl Symb) const {
   MachO::nlist_base Entry = getSymbolTableEntryBase(this, Symb);
   uint8_t n_type = Entry.n_type;
 
-  Res = SymbolRef::ST_Other;
-
   // If this is a STAB debugging symbol, we can do nothing more.
-  if (n_type & MachO::N_STAB) {
-    Res = SymbolRef::ST_Debug;
-    return object_error::success;
-  }
+  if (n_type & MachO::N_STAB)
+    return SymbolRef::ST_Debug;
 
   switch (n_type & MachO::N_TYPE) {
     case MachO::N_UNDF :
-      Res = SymbolRef::ST_Unknown;
-      break;
+      return SymbolRef::ST_Unknown;
     case MachO::N_SECT :
-      Res = SymbolRef::ST_Function;
-      break;
+      return SymbolRef::ST_Function;
   }
-  return object_error::success;
+  return SymbolRef::ST_Other;
 }
 
 uint32_t MachOObjectFile::getSymbolFlags(DataRefImpl DRI) const {
@@ -493,9 +414,6 @@ uint32_t MachOObjectFile::getSymbolFlags(DataRefImpl DRI) const {
 
   uint32_t Result = SymbolRef::SF_None;
 
-  if ((MachOType & MachO::N_TYPE) == MachO::N_UNDF)
-    Result |= SymbolRef::SF_Undefined;
-
   if ((MachOType & MachO::N_TYPE) == MachO::N_INDR)
     Result |= SymbolRef::SF_Indirect;
 
@@ -505,10 +423,10 @@ uint32_t MachOObjectFile::getSymbolFlags(DataRefImpl DRI) const {
   if (MachOType & MachO::N_EXT) {
     Result |= SymbolRef::SF_Global;
     if ((MachOType & MachO::N_TYPE) == MachO::N_UNDF) {
-      uint64_t Value;
-      getSymbolAddress(DRI, Value);
-      if (Value && Value != UnknownAddressOrSize)
+      if (getNValue(DRI))
         Result |= SymbolRef::SF_Common;
+      else
+        Result |= SymbolRef::SF_Undefined;
     }
 
     if (!(MachOType & MachO::N_PEXT))
@@ -537,10 +455,18 @@ std::error_code MachOObjectFile::getSymbolSection(DataRefImpl Symb,
   } else {
     DataRefImpl DRI;
     DRI.d.a = index - 1;
+    if (DRI.d.a >= Sections.size())
+      report_fatal_error("getSymbolSection: Invalid section index.");
     Res = section_iterator(SectionRef(DRI, this));
   }
 
-  return object_error::success;
+  return std::error_code();
+}
+
+unsigned MachOObjectFile::getSymbolSectionID(SymbolRef Sym) const {
+  MachO::nlist_base Entry =
+      getSymbolTableEntryBase(this, Sym.getRawDataRefImpl());
+  return Entry.n_sect - 1;
 }
 
 void MachOObjectFile::moveSectionNext(DataRefImpl &Sec) const {
@@ -551,7 +477,7 @@ std::error_code MachOObjectFile::getSectionName(DataRefImpl Sec,
                                                 StringRef &Result) const {
   ArrayRef<char> Raw = getSectionRawName(Sec);
   Result = parseSegmentOrSectionName(Raw.data());
-  return object_error::success;
+  return std::error_code();
 }
 
 uint64_t MachOObjectFile::getSectionAddress(DataRefImpl Sec) const {
@@ -582,7 +508,7 @@ std::error_code MachOObjectFile::getSectionContents(DataRefImpl Sec,
   }
 
   Res = this->getData().substr(Offset, Size);
-  return object_error::success;
+  return std::error_code();
 }
 
 uint64_t MachOObjectFile::getSectionAlignment(DataRefImpl Sec) const {
@@ -619,27 +545,15 @@ bool MachOObjectFile::isSectionBSS(DataRefImpl Sec) const {
           SectionType == MachO::S_GB_ZEROFILL);
 }
 
+unsigned MachOObjectFile::getSectionID(SectionRef Sec) const {
+  return Sec.getRawDataRefImpl().d.a;
+}
+
 bool MachOObjectFile::isSectionVirtual(DataRefImpl Sec) const {
   // FIXME: Unimplemented.
   return false;
 }
 
-bool MachOObjectFile::sectionContainsSymbol(DataRefImpl Sec,
-                                            DataRefImpl Symb) const {
-  SymbolRef::Type ST;
-  this->getSymbolType(Symb, ST);
-  if (ST == SymbolRef::ST_Unknown)
-    return false;
-
-  uint64_t SectBegin = getSectionAddress(Sec);
-  uint64_t SectEnd = getSectionSize(Sec);
-  SectEnd += SectBegin;
-
-  uint64_t SymAddr;
-  getSymbolAddress(Symb, SymAddr);
-  return (SymAddr >= SectBegin) && (SymAddr < SectEnd);
-}
-
 relocation_iterator MachOObjectFile::section_rel_begin(DataRefImpl Sec) const {
   DataRefImpl Ret;
   Ret.d.a = Sec.d.a;
@@ -668,25 +582,11 @@ void MachOObjectFile::moveRelocationNext(DataRefImpl &Rel) const {
   ++Rel.d.b;
 }
 
-std::error_code MachOObjectFile::getRelocationAddress(DataRefImpl Rel,
-                                                      uint64_t &Res) const {
-  uint64_t Offset;
-  getRelocationOffset(Rel, Offset);
-
-  DataRefImpl Sec;
-  Sec.d.a = Rel.d.a;
-  uint64_t SecAddress = getSectionAddress(Sec);
-  Res = SecAddress + Offset;
-  return object_error::success;
-}
-
-std::error_code MachOObjectFile::getRelocationOffset(DataRefImpl Rel,
-                                                     uint64_t &Res) const {
+uint64_t MachOObjectFile::getRelocationOffset(DataRefImpl Rel) const {
   assert(getHeader().filetype == MachO::MH_OBJECT &&
          "Only implemented for MH_OBJECT");
   MachO::any_relocation_info RE = getRelocation(Rel);
-  Res = getAnyRelocationAddress(RE);
-  return object_error::success;
+  return getAnyRelocationAddress(RE);
 }
 
 symbol_iterator
@@ -710,19 +610,20 @@ MachOObjectFile::getRelocationSymbol(DataRefImpl Rel) const {
   return symbol_iterator(SymbolRef(Sym, this));
 }
 
-std::error_code MachOObjectFile::getRelocationType(DataRefImpl Rel,
-                                                   uint64_t &Res) const {
+section_iterator
+MachOObjectFile::getRelocationSection(DataRefImpl Rel) const {
+  return section_iterator(getAnyRelocationSection(getRelocation(Rel)));
+}
+
+uint64_t MachOObjectFile::getRelocationType(DataRefImpl Rel) const {
   MachO::any_relocation_info RE = getRelocation(Rel);
-  Res = getAnyRelocationType(RE);
-  return object_error::success;
+  return getAnyRelocationType(RE);
 }
 
-std::error_code
-MachOObjectFile::getRelocationTypeName(DataRefImpl Rel,
-                                       SmallVectorImpl<char> &Result) const {
+void MachOObjectFile::getRelocationTypeName(
+    DataRefImpl Rel, SmallVectorImpl<char> &Result) const {
   StringRef res;
-  uint64_t RType;
-  getRelocationType(Rel, RType);
+  uint64_t RType = getRelocationType(Rel);
 
   unsigned Arch = this->getArch();
 
@@ -826,211 +727,11 @@ MachOObjectFile::getRelocationTypeName(DataRefImpl Rel,
       break;
   }
   Result.append(res.begin(), res.end());
-  return object_error::success;
 }
 
-std::error_code
-MachOObjectFile::getRelocationValueString(DataRefImpl Rel,
-                                          SmallVectorImpl<char> &Result) const {
+uint8_t MachOObjectFile::getRelocationLength(DataRefImpl Rel) const {
   MachO::any_relocation_info RE = getRelocation(Rel);
-
-  unsigned Arch = this->getArch();
-
-  std::string fmtbuf;
-  raw_string_ostream fmt(fmtbuf);
-  unsigned Type = this->getAnyRelocationType(RE);
-  bool IsPCRel = this->getAnyRelocationPCRel(RE);
-
-  // Determine any addends that should be displayed with the relocation.
-  // These require decoding the relocation type, which is triple-specific.
-
-  // X86_64 has entirely custom relocation types.
-  if (Arch == Triple::x86_64) {
-    bool isPCRel = getAnyRelocationPCRel(RE);
-
-    switch (Type) {
-      case MachO::X86_64_RELOC_GOT_LOAD:
-      case MachO::X86_64_RELOC_GOT: {
-        printRelocationTargetName(this, RE, fmt);
-        fmt << "@GOT";
-        if (isPCRel) fmt << "PCREL";
-        break;
-      }
-      case MachO::X86_64_RELOC_SUBTRACTOR: {
-        DataRefImpl RelNext = Rel;
-        moveRelocationNext(RelNext);
-        MachO::any_relocation_info RENext = getRelocation(RelNext);
-
-        // X86_64_RELOC_SUBTRACTOR must be followed by a relocation of type
-        // X86_64_RELOC_UNSIGNED.
-        // NOTE: Scattered relocations don't exist on x86_64.
-        unsigned RType = getAnyRelocationType(RENext);
-        if (RType != MachO::X86_64_RELOC_UNSIGNED)
-          report_fatal_error("Expected X86_64_RELOC_UNSIGNED after "
-                             "X86_64_RELOC_SUBTRACTOR.");
-
-        // The X86_64_RELOC_UNSIGNED contains the minuend symbol;
-        // X86_64_RELOC_SUBTRACTOR contains the subtrahend.
-        printRelocationTargetName(this, RENext, fmt);
-        fmt << "-";
-        printRelocationTargetName(this, RE, fmt);
-        break;
-      }
-      case MachO::X86_64_RELOC_TLV:
-        printRelocationTargetName(this, RE, fmt);
-        fmt << "@TLV";
-        if (isPCRel) fmt << "P";
-        break;
-      case MachO::X86_64_RELOC_SIGNED_1:
-        printRelocationTargetName(this, RE, fmt);
-        fmt << "-1";
-        break;
-      case MachO::X86_64_RELOC_SIGNED_2:
-        printRelocationTargetName(this, RE, fmt);
-        fmt << "-2";
-        break;
-      case MachO::X86_64_RELOC_SIGNED_4:
-        printRelocationTargetName(this, RE, fmt);
-        fmt << "-4";
-        break;
-      default:
-        printRelocationTargetName(this, RE, fmt);
-        break;
-    }
-  // X86 and ARM share some relocation types in common.
-  } else if (Arch == Triple::x86 || Arch == Triple::arm ||
-             Arch == Triple::ppc) {
-    // Generic relocation types...
-    switch (Type) {
-      case MachO::GENERIC_RELOC_PAIR: // prints no info
-        return object_error::success;
-      case MachO::GENERIC_RELOC_SECTDIFF: {
-        DataRefImpl RelNext = Rel;
-        moveRelocationNext(RelNext);
-        MachO::any_relocation_info RENext = getRelocation(RelNext);
-
-        // X86 sect diff's must be followed by a relocation of type
-        // GENERIC_RELOC_PAIR.
-        unsigned RType = getAnyRelocationType(RENext);
-
-        if (RType != MachO::GENERIC_RELOC_PAIR)
-          report_fatal_error("Expected GENERIC_RELOC_PAIR after "
-                             "GENERIC_RELOC_SECTDIFF.");
-
-        printRelocationTargetName(this, RE, fmt);
-        fmt << "-";
-        printRelocationTargetName(this, RENext, fmt);
-        break;
-      }
-    }
-
-    if (Arch == Triple::x86 || Arch == Triple::ppc) {
-      switch (Type) {
-        case MachO::GENERIC_RELOC_LOCAL_SECTDIFF: {
-          DataRefImpl RelNext = Rel;
-          moveRelocationNext(RelNext);
-          MachO::any_relocation_info RENext = getRelocation(RelNext);
-
-          // X86 sect diff's must be followed by a relocation of type
-          // GENERIC_RELOC_PAIR.
-          unsigned RType = getAnyRelocationType(RENext);
-          if (RType != MachO::GENERIC_RELOC_PAIR)
-            report_fatal_error("Expected GENERIC_RELOC_PAIR after "
-                               "GENERIC_RELOC_LOCAL_SECTDIFF.");
-
-          printRelocationTargetName(this, RE, fmt);
-          fmt << "-";
-          printRelocationTargetName(this, RENext, fmt);
-          break;
-        }
-        case MachO::GENERIC_RELOC_TLV: {
-          printRelocationTargetName(this, RE, fmt);
-          fmt << "@TLV";
-          if (IsPCRel) fmt << "P";
-          break;
-        }
-        default:
-          printRelocationTargetName(this, RE, fmt);
-      }
-    } else { // ARM-specific relocations
-      switch (Type) {
-        case MachO::ARM_RELOC_HALF:
-        case MachO::ARM_RELOC_HALF_SECTDIFF: {
-          // Half relocations steal a bit from the length field to encode
-          // whether this is an upper16 or a lower16 relocation.
-          bool isUpper = getAnyRelocationLength(RE) >> 1;
-
-          if (isUpper)
-            fmt << ":upper16:(";
-          else
-            fmt << ":lower16:(";
-          printRelocationTargetName(this, RE, fmt);
-
-          DataRefImpl RelNext = Rel;
-          moveRelocationNext(RelNext);
-          MachO::any_relocation_info RENext = getRelocation(RelNext);
-
-          // ARM half relocs must be followed by a relocation of type
-          // ARM_RELOC_PAIR.
-          unsigned RType = getAnyRelocationType(RENext);
-          if (RType != MachO::ARM_RELOC_PAIR)
-            report_fatal_error("Expected ARM_RELOC_PAIR after "
-                               "ARM_RELOC_HALF");
-
-          // NOTE: The half of the target virtual address is stashed in the
-          // address field of the secondary relocation, but we can't reverse
-          // engineer the constant offset from it without decoding the movw/movt
-          // instruction to find the other half in its immediate field.
-
-          // ARM_RELOC_HALF_SECTDIFF encodes the second section in the
-          // symbol/section pointer of the follow-on relocation.
-          if (Type == MachO::ARM_RELOC_HALF_SECTDIFF) {
-            fmt << "-";
-            printRelocationTargetName(this, RENext, fmt);
-          }
-
-          fmt << ")";
-          break;
-        }
-        default: {
-          printRelocationTargetName(this, RE, fmt);
-        }
-      }
-    }
-  } else
-    printRelocationTargetName(this, RE, fmt);
-
-  fmt.flush();
-  Result.append(fmtbuf.begin(), fmtbuf.end());
-  return object_error::success;
-}
-
-std::error_code MachOObjectFile::getRelocationHidden(DataRefImpl Rel,
-                                                     bool &Result) const {
-  unsigned Arch = getArch();
-  uint64_t Type;
-  getRelocationType(Rel, Type);
-
-  Result = false;
-
-  // On arches that use the generic relocations, GENERIC_RELOC_PAIR
-  // is always hidden.
-  if (Arch == Triple::x86 || Arch == Triple::arm || Arch == Triple::ppc) {
-    if (Type == MachO::GENERIC_RELOC_PAIR) Result = true;
-  } else if (Arch == Triple::x86_64) {
-    // On x86_64, X86_64_RELOC_UNSIGNED is hidden only when it follows
-    // an X86_64_RELOC_SUBTRACTOR.
-    if (Type == MachO::X86_64_RELOC_UNSIGNED && Rel.d.a > 0) {
-      DataRefImpl RelPrev = Rel;
-      RelPrev.d.a--;
-      uint64_t PrevType;
-      getRelocationType(RelPrev, PrevType);
-      if (PrevType == MachO::X86_64_RELOC_SUBTRACTOR)
-        Result = true;
-    }
-  }
-
-  return object_error::success;
+  return getAnyRelocationLength(RE);
 }
 
 //
@@ -1208,7 +909,14 @@ std::error_code MachOObjectFile::getLibraryShortNameByIndex(unsigned Index,
   }
 
   Res = LibrariesShortNames[Index];
-  return object_error::success;
+  return std::error_code();
+}
+
+section_iterator
+MachOObjectFile::getRelocationRelocatedSection(relocation_iterator Rel) const {
+  DataRefImpl Sec;
+  Sec.d.a = Rel->getRawDataRefImpl().d.a;
+  return section_iterator(SectionRef(Sec, this));
 }
 
 basic_symbol_iterator MachOObjectFile::symbol_begin_impl() const {
@@ -1450,6 +1158,7 @@ bool MachOObjectFile::isValidArch(StringRef ArchFlag) {
       .Case("armv5e", true)
       .Case("armv6", true)
       .Case("armv6m", true)
+      .Case("armv7", true)
       .Case("armv7em", true)
       .Case("armv7k", true)
       .Case("armv7m", true)
@@ -1466,21 +1175,8 @@ unsigned MachOObjectFile::getArch() const {
 
 Triple MachOObjectFile::getArch(const char **McpuDefault,
                                 Triple *ThumbTriple) const {
-  Triple T;
-  if (is64Bit()) {
-    MachO::mach_header_64 H_64;
-    H_64 = getHeader64();
-    T = MachOObjectFile::getArch(H_64.cputype, H_64.cpusubtype, McpuDefault);
-    *ThumbTriple = MachOObjectFile::getThumbArch(H_64.cputype, H_64.cpusubtype,
-                                                 McpuDefault);
-  } else {
-    MachO::mach_header H;
-    H = getHeader();
-    T = MachOObjectFile::getArch(H.cputype, H.cpusubtype, McpuDefault);
-    *ThumbTriple = MachOObjectFile::getThumbArch(H.cputype, H.cpusubtype,
-                                                 McpuDefault);
-  }
-  return T;
+  *ThumbTriple = getThumbArch(Header.cputype, Header.cpusubtype, McpuDefault);
+  return getArch(Header.cputype, Header.cpusubtype, McpuDefault);
 }
 
 relocation_iterator MachOObjectFile::section_rel_begin(unsigned Index) const {
@@ -2133,6 +1829,22 @@ iterator_range<bind_iterator> MachOObjectFile::weakBindTable() const {
                    MachOBindEntry::Kind::Weak);
 }
 
+MachOObjectFile::load_command_iterator
+MachOObjectFile::begin_load_commands() const {
+  return LoadCommands.begin();
+}
+
+MachOObjectFile::load_command_iterator
+MachOObjectFile::end_load_commands() const {
+  return LoadCommands.end();
+}
+
+iterator_range<MachOObjectFile::load_command_iterator>
+MachOObjectFile::load_commands() const {
+  return iterator_range<load_command_iterator>(begin_load_commands(),
+                                               end_load_commands());
+}
+
 StringRef
 MachOObjectFile::getSectionFinalSegmentName(DataRefImpl Sec) const {
   ArrayRef<char> Raw = getSectionRawFinalSegmentName(Sec);
@@ -2141,8 +1853,7 @@ MachOObjectFile::getSectionFinalSegmentName(DataRefImpl Sec) const {
 
 ArrayRef<char>
 MachOObjectFile::getSectionRawName(DataRefImpl Sec) const {
-  if (Sec.d.a >= Sections.size())
-    report_fatal_error("getSectionRawName: Invalid section index");
+  assert(Sec.d.a < Sections.size() && "Should have detected this earlier");
   const section_base *Base =
     reinterpret_cast<const section_base *>(Sections[Sec.d.a]);
   return makeArrayRef(Base->sectname);
@@ -2150,8 +1861,7 @@ MachOObjectFile::getSectionRawName(DataRefImpl Sec) const {
 
 ArrayRef<char>
 MachOObjectFile::getSectionRawFinalSegmentName(DataRefImpl Sec) const {
-  if (Sec.d.a >= Sections.size())
-    report_fatal_error("getSectionRawFinalSegmentName: Invalid section index");
+  assert(Sec.d.a < Sections.size() && "Should have detected this earlier");
   const section_base *Base =
     reinterpret_cast<const section_base *>(Sections[Sec.d.a]);
   return makeArrayRef(Base->segname);
@@ -2224,50 +1934,25 @@ MachOObjectFile::getAnyRelocationType(
 }
 
 SectionRef
-MachOObjectFile::getRelocationSection(
+MachOObjectFile::getAnyRelocationSection(
                                    const MachO::any_relocation_info &RE) const {
   if (isRelocationScattered(RE) || getPlainRelocationExternal(RE))
     return *section_end();
-  unsigned SecNum = getPlainRelocationSymbolNum(RE) - 1;
+  unsigned SecNum = getPlainRelocationSymbolNum(RE);
+  if (SecNum == MachO::R_ABS || SecNum > Sections.size())
+    return *section_end();
   DataRefImpl DRI;
-  DRI.d.a = SecNum;
+  DRI.d.a = SecNum - 1;
   return SectionRef(DRI, this);
 }
 
-MachOObjectFile::LoadCommandInfo
-MachOObjectFile::getFirstLoadCommandInfo() const {
-  MachOObjectFile::LoadCommandInfo Load;
-
-  unsigned HeaderSize = is64Bit() ? sizeof(MachO::mach_header_64) :
-                                    sizeof(MachO::mach_header);
-  Load.Ptr = getPtr(this, HeaderSize);
-  Load.C = getStruct<MachO::load_command>(this, Load.Ptr);
-  if (Load.C.cmdsize < 8)
-    report_fatal_error("Load command with size < 8 bytes.");
-  return Load;
-}
-
-MachOObjectFile::LoadCommandInfo
-MachOObjectFile::getNextLoadCommandInfo(const LoadCommandInfo &L) const {
-  MachOObjectFile::LoadCommandInfo Next;
-  Next.Ptr = L.Ptr + L.C.cmdsize;
-  Next.C = getStruct<MachO::load_command>(this, Next.Ptr);
-  if (Next.C.cmdsize < 8)
-    report_fatal_error("Load command with size < 8 bytes.");
-  return Next;
-}
-
 MachO::section MachOObjectFile::getSection(DataRefImpl DRI) const {
-  // TODO: What if Sections.size() == 0?
-  if (DRI.d.a >= Sections.size())
-    report_fatal_error("getSection: Invalid section index.");
+  assert(DRI.d.a < Sections.size() && "Should have detected this earlier");
   return getStruct<MachO::section>(this, Sections[DRI.d.a]);
 }
 
 MachO::section_64 MachOObjectFile::getSection64(DataRefImpl DRI) const {
-  // TODO: What if Sections.size() == 0?
-  if (DRI.d.a >= Sections.size())
-    report_fatal_error("getSection64: Invalid section index.");
+  assert(DRI.d.a < Sections.size() && "Should have detected this earlier");
   return getStruct<MachO::section_64>(this, Sections[DRI.d.a]);
 }
 
@@ -2425,12 +2110,13 @@ MachOObjectFile::getDice(DataRefImpl Rel) const {
   return getStruct<MachO::data_in_code_entry>(this, P);
 }
 
-MachO::mach_header MachOObjectFile::getHeader() const {
-  return getStruct<MachO::mach_header>(this, getPtr(this, 0));
+const MachO::mach_header &MachOObjectFile::getHeader() const {
+  return Header;
 }
 
-MachO::mach_header_64 MachOObjectFile::getHeader64() const {
-  return getStruct<MachO::mach_header_64>(this, getPtr(this, 0));
+const MachO::mach_header_64 &MachOObjectFile::getHeader64() const {
+  assert(is64Bit());
+  return Header64;
 }
 
 uint32_t MachOObjectFile::getIndirectSymbolTableEntry(
diff --git a/lib/Object/MachOUniversal.cpp b/lib/Object/MachOUniversal.cpp
index a01c83873e0..1d0e69e4622 100644
--- a/lib/Object/MachOUniversal.cpp
+++ b/lib/Object/MachOUniversal.cpp
@@ -120,28 +120,16 @@ MachOUniversalBinary::MachOUniversalBinary(MemoryBufferRef Source,
     ec = object_error::parse_failed;
     return;
   }
-  ec = object_error::success;
-}
-
-static bool getCTMForArch(Triple::ArchType Arch, MachO::CPUType &CTM) {
-  switch (Arch) {
-    case Triple::x86:    CTM = MachO::CPU_TYPE_I386; return true;
-    case Triple::x86_64: CTM = MachO::CPU_TYPE_X86_64; return true;
-    case Triple::arm:    CTM = MachO::CPU_TYPE_ARM; return true;
-    case Triple::sparc:  CTM = MachO::CPU_TYPE_SPARC; return true;
-    case Triple::ppc:    CTM = MachO::CPU_TYPE_POWERPC; return true;
-    case Triple::ppc64:  CTM = MachO::CPU_TYPE_POWERPC64; return true;
-    default: return false;
-  }
+  ec = std::error_code();
 }
 
 ErrorOr<std::unique_ptr<MachOObjectFile>>
-MachOUniversalBinary::getObjectForArch(Triple::ArchType Arch) const {
-  MachO::CPUType CTM;
-  if (!getCTMForArch(Arch, CTM))
+MachOUniversalBinary::getObjectForArch(StringRef ArchName) const {
+  if (Triple(ArchName).getArch() == Triple::ArchType::UnknownArch)
     return object_error::arch_not_found;
+
   for (object_iterator I = begin_objects(), E = end_objects(); I != E; ++I) {
-    if (I->getCPUType() == static_cast<uint32_t>(CTM))
+    if (I->getArchTypeName() == ArchName)
       return I->getAsObjectFile();
   }
   return object_error::arch_not_found;
diff --git a/lib/Object/Object.cpp b/lib/Object/Object.cpp
index 84a5df089cb..5c4b7a67b2a 100644
--- a/lib/Object/Object.cpp
+++ b/lib/Object/Object.cpp
@@ -173,39 +173,26 @@ void LLVMMoveToNextRelocation(LLVMRelocationIteratorRef SI) {
 
 // SymbolRef accessors
 const char *LLVMGetSymbolName(LLVMSymbolIteratorRef SI) {
-  StringRef ret;
-  if (std::error_code ec = (*unwrap(SI))->getName(ret))
-    report_fatal_error(ec.message());
-  return ret.data();
+  ErrorOr<StringRef> Ret = (*unwrap(SI))->getName();
+  if (std::error_code EC = Ret.getError())
+    report_fatal_error(EC.message());
+  return Ret->data();
 }
 
 uint64_t LLVMGetSymbolAddress(LLVMSymbolIteratorRef SI) {
-  uint64_t ret;
-  if (std::error_code ec = (*unwrap(SI))->getAddress(ret))
-    report_fatal_error(ec.message());
-  return ret;
+  ErrorOr<uint64_t> Ret = (*unwrap(SI))->getAddress();
+  if (std::error_code EC = Ret.getError())
+    report_fatal_error(EC.message());
+  return *Ret;
 }
 
 uint64_t LLVMGetSymbolSize(LLVMSymbolIteratorRef SI) {
-  uint64_t ret;
-  if (std::error_code ec = (*unwrap(SI))->getSize(ret))
-    report_fatal_error(ec.message());
-  return ret;
+  return (*unwrap(SI))->getCommonSize();
 }
 
 // RelocationRef accessors
-uint64_t LLVMGetRelocationAddress(LLVMRelocationIteratorRef RI) {
-  uint64_t ret;
-  if (std::error_code ec = (*unwrap(RI))->getAddress(ret))
-    report_fatal_error(ec.message());
-  return ret;
-}
-
 uint64_t LLVMGetRelocationOffset(LLVMRelocationIteratorRef RI) {
-  uint64_t ret;
-  if (std::error_code ec = (*unwrap(RI))->getOffset(ret))
-    report_fatal_error(ec.message());
-  return ret;
+  return (*unwrap(RI))->getOffset();
 }
 
 LLVMSymbolIteratorRef LLVMGetRelocationSymbol(LLVMRelocationIteratorRef RI) {
@@ -214,18 +201,13 @@ LLVMSymbolIteratorRef LLVMGetRelocationSymbol(LLVMRelocationIteratorRef RI) {
 }
 
 uint64_t LLVMGetRelocationType(LLVMRelocationIteratorRef RI) {
-  uint64_t ret;
-  if (std::error_code ec = (*unwrap(RI))->getType(ret))
-    report_fatal_error(ec.message());
-  return ret;
+  return (*unwrap(RI))->getType();
 }
 
 // NOTE: Caller takes ownership of returned string.
 const char *LLVMGetRelocationTypeName(LLVMRelocationIteratorRef RI) {
   SmallVector<char, 0> ret;
-  if (std::error_code ec = (*unwrap(RI))->getTypeName(ret))
-    report_fatal_error(ec.message());
-
+  (*unwrap(RI))->getTypeName(ret);
   char *str = static_cast<char*>(malloc(ret.size()));
   std::copy(ret.begin(), ret.end(), str);
   return str;
@@ -233,12 +215,6 @@ const char *LLVMGetRelocationTypeName(LLVMRelocationIteratorRef RI) {
 
 // NOTE: Caller takes ownership of returned string.
 const char *LLVMGetRelocationValueString(LLVMRelocationIteratorRef RI) {
-  SmallVector<char, 0> ret;
-  if (std::error_code ec = (*unwrap(RI))->getValueString(ret))
-    report_fatal_error(ec.message());
-
-  char *str = static_cast<char*>(malloc(ret.size()));
-  std::copy(ret.begin(), ret.end(), str);
-  return str;
+  return strdup("");
 }
 
diff --git a/lib/Object/ObjectFile.cpp b/lib/Object/ObjectFile.cpp
index 01b76543fa2..f82edae89bc 100644
--- a/lib/Object/ObjectFile.cpp
+++ b/lib/Object/ObjectFile.cpp
@@ -28,20 +28,32 @@ void ObjectFile::anchor() { }
 ObjectFile::ObjectFile(unsigned int Type, MemoryBufferRef Source)
     : SymbolicFile(Type, Source) {}
 
+bool SectionRef::containsSymbol(SymbolRef S) const {
+  section_iterator SymSec = getObject()->section_end();
+  if (S.getSection(SymSec))
+    return false;
+  return *this == *SymSec;
+}
+
+uint64_t ObjectFile::getSymbolValue(DataRefImpl Ref) const {
+  uint32_t Flags = getSymbolFlags(Ref);
+  if (Flags & SymbolRef::SF_Undefined)
+    return 0;
+  if (Flags & SymbolRef::SF_Common)
+    return getCommonSymbolSize(Ref);
+  return getSymbolValueImpl(Ref);
+}
+
 std::error_code ObjectFile::printSymbolName(raw_ostream &OS,
                                             DataRefImpl Symb) const {
-  StringRef Name;
-  if (std::error_code EC = getSymbolName(Symb, Name))
+  ErrorOr<StringRef> Name = getSymbolName(Symb);
+  if (std::error_code EC = Name.getError())
     return EC;
-  OS << Name;
-  return object_error::success;
+  OS << *Name;
+  return std::error_code();
 }
 
-std::error_code ObjectFile::getSymbolAlignment(DataRefImpl DRI,
-                                               uint32_t &Result) const {
-  Result = 0;
-  return object_error::success;
-}
+uint32_t ObjectFile::getSymbolAlignment(DataRefImpl DRI) const { return 0; }
 
 section_iterator ObjectFile::getRelocatedSection(DataRefImpl Sec) const {
   return section_iterator(SectionRef(Sec, this));
diff --git a/lib/Object/RecordStreamer.cpp b/lib/Object/RecordStreamer.cpp
index 081faddc214..42dbd3e0c2d 100644
--- a/lib/Object/RecordStreamer.cpp
+++ b/lib/Object/RecordStreamer.cpp
@@ -89,7 +89,7 @@ bool RecordStreamer::EmitSymbolAttribute(MCSymbol *Symbol,
   return true;
 }
 
-void RecordStreamer::EmitZerofill(const MCSection *Section, MCSymbol *Symbol,
+void RecordStreamer::EmitZerofill(MCSection *Section, MCSymbol *Symbol,
                                   uint64_t Size, unsigned ByteAlignment) {
   markDefined(*Symbol);
 }
diff --git a/lib/Object/RecordStreamer.h b/lib/Object/RecordStreamer.h
index 7dacbdfbfd6..d8610610c33 100644
--- a/lib/Object/RecordStreamer.h
+++ b/lib/Object/RecordStreamer.h
@@ -33,7 +33,7 @@ class RecordStreamer : public MCStreamer {
   void EmitLabel(MCSymbol *Symbol) override;
   void EmitAssignment(MCSymbol *Symbol, const MCExpr *Value) override;
   bool EmitSymbolAttribute(MCSymbol *Symbol, MCSymbolAttr Attribute) override;
-  void EmitZerofill(const MCSection *Section, MCSymbol *Symbol, uint64_t Size,
+  void EmitZerofill(MCSection *Section, MCSymbol *Symbol, uint64_t Size,
                     unsigned ByteAlignment) override;
   void EmitCommonSymbol(MCSymbol *Symbol, uint64_t Size,
                         unsigned ByteAlignment) override;
diff --git a/lib/Object/SymbolSize.cpp b/lib/Object/SymbolSize.cpp
new file mode 100644
index 00000000000..1d5cd78e6d9
--- /dev/null
+++ b/lib/Object/SymbolSize.cpp
@@ -0,0 +1,100 @@
+//===- SymbolSize.cpp -----------------------------------------------------===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/Object/SymbolSize.h"
+#include "llvm/ADT/STLExtras.h"
+#include "llvm/Object/COFF.h"
+#include "llvm/Object/ELFObjectFile.h"
+#include "llvm/Object/MachO.h"
+
+using namespace llvm;
+using namespace object;
+
+namespace {
+struct SymEntry {
+  symbol_iterator I;
+  uint64_t Address;
+  unsigned Number;
+  unsigned SectionID;
+};
+}
+
+static int compareAddress(const SymEntry *A, const SymEntry *B) {
+  if (A->SectionID != B->SectionID)
+    return A->SectionID - B->SectionID;
+  return A->Address - B->Address;
+}
+
+static unsigned getSectionID(const ObjectFile &O, SectionRef Sec) {
+  if (auto *M = dyn_cast<MachOObjectFile>(&O))
+    return M->getSectionID(Sec);
+  return cast<COFFObjectFile>(O).getSectionID(Sec);
+}
+
+static unsigned getSymbolSectionID(const ObjectFile &O, SymbolRef Sym) {
+  if (auto *M = dyn_cast<MachOObjectFile>(&O))
+    return M->getSymbolSectionID(Sym);
+  return cast<COFFObjectFile>(O).getSymbolSectionID(Sym);
+}
+
+std::vector<std::pair<SymbolRef, uint64_t>>
+llvm::object::computeSymbolSizes(const ObjectFile &O) {
+  std::vector<std::pair<SymbolRef, uint64_t>> Ret;
+
+  if (const auto *E = dyn_cast<ELFObjectFileBase>(&O)) {
+    auto Syms = E->symbols();
+    if (Syms.begin() == Syms.end())
+      Syms = E->getDynamicSymbolIterators();
+    for (ELFSymbolRef Sym : Syms)
+      Ret.push_back({Sym, Sym.getSize()});
+    return Ret;
+  }
+
+  // Collect sorted symbol addresses. Include dummy addresses for the end
+  // of each section.
+  std::vector<SymEntry> Addresses;
+  unsigned SymNum = 0;
+  for (symbol_iterator I = O.symbol_begin(), E = O.symbol_end(); I != E; ++I) {
+    SymbolRef Sym = *I;
+    uint64_t Value = Sym.getValue();
+    Addresses.push_back({I, Value, SymNum, getSymbolSectionID(O, Sym)});
+    ++SymNum;
+  }
+  for (SectionRef Sec : O.sections()) {
+    uint64_t Address = Sec.getAddress();
+    uint64_t Size = Sec.getSize();
+    Addresses.push_back(
+        {O.symbol_end(), Address + Size, 0, getSectionID(O, Sec)});
+  }
+  array_pod_sort(Addresses.begin(), Addresses.end(), compareAddress);
+
+  // Compute the size as the gap to the next symbol
+  for (unsigned I = 0, N = Addresses.size() - 1; I < N; ++I) {
+    auto &P = Addresses[I];
+    if (P.I == O.symbol_end())
+      continue;
+
+    // If multiple symbol have the same address, give both the same size.
+    unsigned NextI = I + 1;
+    while (NextI < N && Addresses[NextI].Address == P.Address)
+      ++NextI;
+
+    uint64_t Size = Addresses[NextI].Address - P.Address;
+    P.Address = Size;
+  }
+
+  // Assign the sorted symbols in the original order.
+  Ret.resize(SymNum);
+  for (SymEntry &P : Addresses) {
+    if (P.I == O.symbol_end())
+      continue;
+    Ret[P.Number] = {*P.I, P.Address};
+  }
+  return Ret;
+}
diff --git a/lib/Option/ArgList.cpp b/lib/Option/ArgList.cpp
index b771a18a106..a37f443e56f 100644
--- a/lib/Option/ArgList.cpp
+++ b/lib/Option/ArgList.cpp
@@ -33,9 +33,6 @@ void arg_iterator::SkipToNextArg() {
   }
 }
 
-ArgList::~ArgList() {
-}
-
 void ArgList::append(Arg *A) {
   Args.push_back(A);
 }
@@ -261,6 +258,21 @@ void ArgList::AddLastArg(ArgStringList &Output, OptSpecifier Id0,
   }
 }
 
+void ArgList::AddAllArgs(ArgStringList &Output,
+                         ArrayRef<OptSpecifier> Ids) const {
+  for (const Arg *Arg : Args) {
+    for (OptSpecifier Id : Ids) {
+      if (Arg->getOption().matches(Id)) {
+        Arg->claim();
+        Arg->render(*this, Output);
+        break;
+      }
+    }
+  }
+}
+
+/// This 3-opt variant of AddAllArgs could be eliminated in favor of one
+/// that accepts a single specifier, given the above which accepts any number.
 void ArgList::AddAllArgs(ArgStringList &Output, OptSpecifier Id0,
                          OptSpecifier Id1, OptSpecifier Id2) const {
   for (auto Arg: filtered(Id0, Id1, Id2)) {
@@ -318,18 +330,18 @@ const char *ArgList::GetOrMakeJoinedArgString(unsigned Index,
 
 //
 
+void InputArgList::releaseMemory() {
+  // An InputArgList always owns its arguments.
+  for (Arg *A : *this)
+    delete A;
+}
+
 InputArgList::InputArgList(const char* const *ArgBegin,
                            const char* const *ArgEnd)
   : NumInputArgStrings(ArgEnd - ArgBegin) {
   ArgStrings.append(ArgBegin, ArgEnd);
 }
 
-InputArgList::~InputArgList() {
-  // An InputArgList always owns its arguments.
-  for (iterator it = begin(), ie = end(); it != ie; ++it)
-    delete *it;
-}
-
 unsigned InputArgList::MakeIndex(StringRef String0) const {
   unsigned Index = ArgStrings.size();
 
@@ -358,8 +370,6 @@ const char *InputArgList::MakeArgStringRef(StringRef Str) const {
 DerivedArgList::DerivedArgList(const InputArgList &BaseArgs)
     : BaseArgs(BaseArgs) {}
 
-DerivedArgList::~DerivedArgList() {}
-
 const char *DerivedArgList::MakeArgStringRef(StringRef Str) const {
   return BaseArgs.MakeArgString(Str);
 }
diff --git a/lib/Option/OptTable.cpp b/lib/Option/OptTable.cpp
index 96ba1836f4c..e83536f2b57 100644
--- a/lib/Option/OptTable.cpp
+++ b/lib/Option/OptTable.cpp
@@ -247,33 +247,32 @@ Arg *OptTable::ParseOneArg(const ArgList &Args, unsigned &Index,
   return new Arg(getOption(TheUnknownOptionID), Str, Index++, Str);
 }
 
-InputArgList *OptTable::ParseArgs(const char *const *ArgBegin,
-                                  const char *const *ArgEnd,
-                                  unsigned &MissingArgIndex,
-                                  unsigned &MissingArgCount,
-                                  unsigned FlagsToInclude,
-                                  unsigned FlagsToExclude) const {
-  InputArgList *Args = new InputArgList(ArgBegin, ArgEnd);
+InputArgList OptTable::ParseArgs(ArrayRef<const char *> ArgArr,
+                                 unsigned &MissingArgIndex,
+                                 unsigned &MissingArgCount,
+                                 unsigned FlagsToInclude,
+                                 unsigned FlagsToExclude) const {
+  InputArgList Args(ArgArr.begin(), ArgArr.end());
 
   // FIXME: Handle '@' args (or at least error on them).
 
   MissingArgIndex = MissingArgCount = 0;
-  unsigned Index = 0, End = ArgEnd - ArgBegin;
+  unsigned Index = 0, End = ArgArr.size();
   while (Index < End) {
     // Ingore nullptrs, they are response file's EOL markers
-    if (Args->getArgString(Index) == nullptr) {
+    if (Args.getArgString(Index) == nullptr) {
       ++Index;
       continue;
     }
     // Ignore empty arguments (other things may still take them as arguments).
-    StringRef Str = Args->getArgString(Index);
+    StringRef Str = Args.getArgString(Index);
     if (Str == "") {
       ++Index;
       continue;
     }
 
     unsigned Prev = Index;
-    Arg *A = ParseOneArg(*Args, Index, FlagsToInclude, FlagsToExclude);
+    Arg *A = ParseOneArg(Args, Index, FlagsToInclude, FlagsToExclude);
     assert(Index > Prev && "Parser failed to consume argument.");
 
     // Check for missing argument error.
@@ -285,7 +284,7 @@ InputArgList *OptTable::ParseArgs(const char *const *ArgBegin,
       break;
     }
 
-    Args->append(A);
+    Args.append(A);
   }
 
   return Args;
diff --git a/lib/Option/Option.cpp b/lib/Option/Option.cpp
index e29d6491c7f..221414d79e7 100644
--- a/lib/Option/Option.cpp
+++ b/lib/Option/Option.cpp
@@ -125,6 +125,11 @@ Arg *Option::accept(const ArgList &Args,
         Val += strlen(Val) + 1;
       }
     }
+
+    if (UnaliasedOption.getKind() == JoinedClass && !getAliasArgs())
+      // A Flag alias for a Joined option must provide an argument.
+      A->getValues().push_back("");
+
     return A;
   }
   case JoinedClass: {
diff --git a/lib/ProfileData/CMakeLists.txt b/lib/ProfileData/CMakeLists.txt
index 282760f0e66..22cca4b44df 100644
--- a/lib/ProfileData/CMakeLists.txt
+++ b/lib/ProfileData/CMakeLists.txt
@@ -11,4 +11,7 @@ add_llvm_library(LLVMProfileData
 
   ADDITIONAL_HEADER_DIRS
   ${LLVM_MAIN_INCLUDE_DIR}/llvm/ProfileData
+
+  DEPENDS
+  intrinsics_gen
   )
diff --git a/lib/ProfileData/CoverageMapping.cpp b/lib/ProfileData/CoverageMapping.cpp
index 46d494b82db..cf04fea8491 100644
--- a/lib/ProfileData/CoverageMapping.cpp
+++ b/lib/ProfileData/CoverageMapping.cpp
@@ -19,7 +19,10 @@
 #include "llvm/ProfileData/CoverageMappingReader.h"
 #include "llvm/ProfileData/InstrProfReader.h"
 #include "llvm/Support/Debug.h"
+#include "llvm/Support/Errc.h"
 #include "llvm/Support/ErrorHandling.h"
+#include "llvm/Support/ManagedStatic.h"
+#include "llvm/Support/Path.h"
 #include "llvm/Support/raw_ostream.h"
 
 using namespace llvm;
@@ -152,11 +155,11 @@ ErrorOr<int64_t> CounterMappingContext::evaluate(const Counter &C) const {
     return 0;
   case Counter::CounterValueReference:
     if (C.getCounterID() >= CounterValues.size())
-      return std::make_error_code(std::errc::argument_out_of_domain);
+      return make_error_code(errc::argument_out_of_domain);
     return CounterValues[C.getCounterID()];
   case Counter::Expression: {
     if (C.getExpressionID() >= Expressions.size())
-      return std::make_error_code(std::errc::argument_out_of_domain);
+      return make_error_code(errc::argument_out_of_domain);
     const auto &E = Expressions[C.getExpressionID()];
     ErrorOr<int64_t> LHS = evaluate(E.LHS);
     if (!LHS)
@@ -178,6 +181,18 @@ void FunctionRecordIterator::skipOtherFiles() {
     *this = FunctionRecordIterator();
 }
 
+/// Get the function name from the record, removing the filename prefix if
+/// necessary.
+static StringRef getFuncNameWithoutPrefix(const CoverageMappingRecord &Record) {
+  StringRef FunctionName = Record.FunctionName;
+  if (Record.Filenames.empty())
+    return FunctionName;
+  StringRef Filename = sys::path::filename(Record.Filenames[0]);
+  if (FunctionName.startswith(Filename))
+    FunctionName = FunctionName.drop_front(Filename.size() + 1);
+  return FunctionName;
+}
+
 ErrorOr<std::unique_ptr<CoverageMapping>>
 CoverageMapping::load(CoverageMappingReader &CoverageReader,
                       IndexedInstrProfReader &ProfileReader) {
@@ -195,11 +210,13 @@ CoverageMapping::load(CoverageMappingReader &CoverageReader,
         continue;
       } else if (EC != instrprof_error::unknown_function)
         return EC;
-    } else
-      Ctx.setCounts(Counts);
+      Counts.assign(Record.MappingRegions.size(), 0);
+    }
+    Ctx.setCounts(Counts);
 
     assert(!Record.MappingRegions.empty() && "Function has no regions");
-    FunctionRecord Function(Record.FunctionName, Record.Filenames);
+
+    FunctionRecord Function(getFuncNameWithoutPrefix(Record), Record.Filenames);
     for (const auto &Region : Record.MappingRegions) {
       ErrorOr<int64_t> ExecutionCount = Ctx.evaluate(Region.Count);
       if (!ExecutionCount)
@@ -219,7 +236,7 @@ CoverageMapping::load(CoverageMappingReader &CoverageReader,
 
 ErrorOr<std::unique_ptr<CoverageMapping>>
 CoverageMapping::load(StringRef ObjectFilename, StringRef ProfileFilename,
-                      Triple::ArchType Arch) {
+                      StringRef Arch) {
   auto CounterMappingBuff = MemoryBuffer::getFileOrSTDIN(ObjectFilename);
   if (std::error_code EC = CounterMappingBuff.getError())
     return EC;
@@ -481,3 +498,33 @@ CoverageMapping::getCoverageForExpansion(const ExpansionRecord &Expansion) {
 
   return ExpansionCoverage;
 }
+
+namespace {
+class CoverageMappingErrorCategoryType : public std::error_category {
+  const char *name() const LLVM_NOEXCEPT override { return "llvm.coveragemap"; }
+  std::string message(int IE) const override {
+    auto E = static_cast<coveragemap_error>(IE);
+    switch (E) {
+    case coveragemap_error::success:
+      return "Success";
+    case coveragemap_error::eof:
+      return "End of File";
+    case coveragemap_error::no_data_found:
+      return "No coverage data found";
+    case coveragemap_error::unsupported_version:
+      return "Unsupported coverage format version";
+    case coveragemap_error::truncated:
+      return "Truncated coverage data";
+    case coveragemap_error::malformed:
+      return "Malformed coverage data";
+    }
+    llvm_unreachable("A value of coveragemap_error has no message.");
+  }
+};
+}
+
+static ManagedStatic<CoverageMappingErrorCategoryType> ErrorCategory;
+
+const std::error_category &llvm::coveragemap_category() {
+  return *ErrorCategory;
+}
diff --git a/lib/ProfileData/CoverageMappingReader.cpp b/lib/ProfileData/CoverageMappingReader.cpp
index 12e9e888d36..334a3f51ec9 100644
--- a/lib/ProfileData/CoverageMappingReader.cpp
+++ b/lib/ProfileData/CoverageMappingReader.cpp
@@ -19,6 +19,7 @@
 #include "llvm/Support/Debug.h"
 #include "llvm/Support/Endian.h"
 #include "llvm/Support/LEB128.h"
+#include "llvm/Support/MathExtras.h"
 #include "llvm/Support/raw_ostream.h"
 
 using namespace llvm;
@@ -36,13 +37,13 @@ void CoverageMappingIterator::increment() {
 
 std::error_code RawCoverageReader::readULEB128(uint64_t &Result) {
   if (Data.size() < 1)
-    return error(instrprof_error::truncated);
+    return coveragemap_error::truncated;
   unsigned N = 0;
   Result = decodeULEB128(reinterpret_cast<const uint8_t *>(Data.data()), &N);
   if (N > Data.size())
-    return error(instrprof_error::malformed);
+    return coveragemap_error::malformed;
   Data = Data.substr(N);
-  return success();
+  return std::error_code();
 }
 
 std::error_code RawCoverageReader::readIntMax(uint64_t &Result,
@@ -50,8 +51,8 @@ std::error_code RawCoverageReader::readIntMax(uint64_t &Result,
   if (auto Err = readULEB128(Result))
     return Err;
   if (Result >= MaxPlus1)
-    return error(instrprof_error::malformed);
-  return success();
+    return coveragemap_error::malformed;
+  return std::error_code();
 }
 
 std::error_code RawCoverageReader::readSize(uint64_t &Result) {
@@ -59,8 +60,8 @@ std::error_code RawCoverageReader::readSize(uint64_t &Result) {
     return Err;
   // Sanity check the number.
   if (Result > Data.size())
-    return error(instrprof_error::malformed);
-  return success();
+    return coveragemap_error::malformed;
+  return std::error_code();
 }
 
 std::error_code RawCoverageReader::readString(StringRef &Result) {
@@ -69,7 +70,7 @@ std::error_code RawCoverageReader::readString(StringRef &Result) {
     return Err;
   Result = Data.substr(0, Length);
   Data = Data.substr(Length);
-  return success();
+  return std::error_code();
 }
 
 std::error_code RawCoverageFilenamesReader::read() {
@@ -82,7 +83,7 @@ std::error_code RawCoverageFilenamesReader::read() {
       return Err;
     Filenames.push_back(Filename);
   }
-  return success();
+  return std::error_code();
 }
 
 std::error_code RawCoverageMappingReader::decodeCounter(unsigned Value,
@@ -91,10 +92,10 @@ std::error_code RawCoverageMappingReader::decodeCounter(unsigned Value,
   switch (Tag) {
   case Counter::Zero:
     C = Counter::getZero();
-    return success();
+    return std::error_code();
   case Counter::CounterValueReference:
     C = Counter::getCounter(Value >> Counter::EncodingTagBits);
-    return success();
+    return std::error_code();
   default:
     break;
   }
@@ -104,15 +105,15 @@ std::error_code RawCoverageMappingReader::decodeCounter(unsigned Value,
   case CounterExpression::Add: {
     auto ID = Value >> Counter::EncodingTagBits;
     if (ID >= Expressions.size())
-      return error(instrprof_error::malformed);
+      return coveragemap_error::malformed;
     Expressions[ID].Kind = CounterExpression::ExprKind(Tag);
     C = Counter::getExpression(ID);
     break;
   }
   default:
-    return error(instrprof_error::malformed);
+    return coveragemap_error::malformed;
   }
-  return success();
+  return std::error_code();
 }
 
 std::error_code RawCoverageMappingReader::readCounter(Counter &C) {
@@ -122,7 +123,7 @@ std::error_code RawCoverageMappingReader::readCounter(Counter &C) {
     return Err;
   if (auto Err = decodeCounter(EncodedCounter, C))
     return Err;
-  return success();
+  return std::error_code();
 }
 
 static const unsigned EncodingExpansionRegionBit = 1
@@ -159,7 +160,7 @@ std::error_code RawCoverageMappingReader::readMappingRegionsSubArray(
         ExpandedFileID = EncodedCounterAndRegion >>
                          Counter::EncodingCounterTagAndExpansionRegionTagBits;
         if (ExpandedFileID >= NumFileIDs)
-          return error(instrprof_error::malformed);
+          return coveragemap_error::malformed;
       } else {
         switch (EncodedCounterAndRegion >>
                 Counter::EncodingCounterTagAndExpansionRegionTagBits) {
@@ -170,7 +171,7 @@ std::error_code RawCoverageMappingReader::readMappingRegionsSubArray(
           Kind = CounterMappingRegion::SkippedRegion;
           break;
         default:
-          return error(instrprof_error::malformed);
+          return coveragemap_error::malformed;
         }
       }
     }
@@ -183,7 +184,7 @@ std::error_code RawCoverageMappingReader::readMappingRegionsSubArray(
     if (auto Err = readULEB128(ColumnStart))
       return Err;
     if (ColumnStart > std::numeric_limits<unsigned>::max())
-      return error(instrprof_error::malformed);
+      return coveragemap_error::malformed;
     if (auto Err = readIntMax(NumLines, std::numeric_limits<unsigned>::max()))
       return Err;
     if (auto Err = readIntMax(ColumnEnd, std::numeric_limits<unsigned>::max()))
@@ -217,7 +218,7 @@ std::error_code RawCoverageMappingReader::readMappingRegionsSubArray(
         C, InferredFileID, ExpandedFileID, LineStart, ColumnStart,
         LineStart + NumLines, ColumnEnd, Kind));
   }
-  return success();
+  return std::error_code();
 }
 
 std::error_code RawCoverageMappingReader::read() {
@@ -286,7 +287,7 @@ std::error_code RawCoverageMappingReader::read() {
     }
   }
 
-  return success();
+  return std::error_code();
 }
 
 namespace {
@@ -301,17 +302,17 @@ struct SectionData {
     if (auto Err = Section.getContents(Data))
       return Err;
     Address = Section.getAddress();
-    return instrprof_error::success;
+    return std::error_code();
   }
 
   std::error_code get(uint64_t Pointer, size_t Size, StringRef &Result) {
     if (Pointer < Address)
-      return instrprof_error::malformed;
+      return coveragemap_error::malformed;
     auto Offset = Pointer - Address;
     if (Offset + Size > Data.size())
-      return instrprof_error::malformed;
+      return coveragemap_error::malformed;
     Result = Data.substr(Pointer - Address, Size);
-    return instrprof_error::success;
+    return std::error_code();
   }
 };
 }
@@ -327,7 +328,7 @@ std::error_code readCoverageMappingData(
   // Read the records in the coverage data section.
   for (const char *Buf = Data.data(), *End = Buf + Data.size(); Buf < End;) {
     if (Buf + 4 * sizeof(uint32_t) > End)
-      return instrprof_error::malformed;
+      return coveragemap_error::malformed;
     uint32_t NRecords = endian::readNext<uint32_t, Endian, unaligned>(Buf);
     uint32_t FilenamesSize = endian::readNext<uint32_t, Endian, unaligned>(Buf);
     uint32_t CoverageSize = endian::readNext<uint32_t, Endian, unaligned>(Buf);
@@ -337,7 +338,7 @@ std::error_code readCoverageMappingData(
     case CoverageMappingVersion1:
       break;
     default:
-      return instrprof_error::unsupported_version;
+      return coveragemap_error::unsupported_version;
     }
 
     // Skip past the function records, saving the start and end for later.
@@ -347,7 +348,7 @@ std::error_code readCoverageMappingData(
 
     // Get the filenames.
     if (Buf + FilenamesSize > End)
-      return instrprof_error::malformed;
+      return coveragemap_error::malformed;
     size_t FilenamesBegin = Filenames.size();
     RawCoverageFilenamesReader Reader(StringRef(Buf, FilenamesSize), Filenames);
     if (auto Err = Reader.read())
@@ -358,8 +359,12 @@ std::error_code readCoverageMappingData(
     const char *CovBuf = Buf;
     Buf += CoverageSize;
     const char *CovEnd = Buf;
+
     if (Buf > End)
-      return instrprof_error::malformed;
+      return coveragemap_error::malformed;
+    // Each coverage map has an alignment of 8, so we need to adjust alignment
+    // before reading the next map.
+    Buf += alignmentAdjustment(Buf, 8);
 
     while (FunBuf < FunEnd) {
       // Read the function information
@@ -370,7 +375,7 @@ std::error_code readCoverageMappingData(
 
       // Now use that to read the coverage data.
       if (CovBuf + DataSize > CovEnd)
-        return instrprof_error::malformed;
+        return coveragemap_error::malformed;
       auto Mapping = StringRef(CovBuf, DataSize);
       CovBuf += DataSize;
 
@@ -390,7 +395,7 @@ std::error_code readCoverageMappingData(
     }
   }
 
-  return instrprof_error::success;
+  return std::error_code();
 }
 
 static const char *TestingFormatMagic = "llvmcovmtestdata";
@@ -405,26 +410,37 @@ static std::error_code loadTestingFormat(StringRef Data,
 
   Data = Data.substr(StringRef(TestingFormatMagic).size());
   if (Data.size() < 1)
-    return instrprof_error::truncated;
+    return coveragemap_error::truncated;
   unsigned N = 0;
   auto ProfileNamesSize =
       decodeULEB128(reinterpret_cast<const uint8_t *>(Data.data()), &N);
   if (N > Data.size())
-    return instrprof_error::malformed;
+    return coveragemap_error::malformed;
   Data = Data.substr(N);
   if (Data.size() < 1)
-    return instrprof_error::truncated;
+    return coveragemap_error::truncated;
   N = 0;
   ProfileNames.Address =
       decodeULEB128(reinterpret_cast<const uint8_t *>(Data.data()), &N);
   if (N > Data.size())
-    return instrprof_error::malformed;
+    return coveragemap_error::malformed;
   Data = Data.substr(N);
   if (Data.size() < ProfileNamesSize)
-    return instrprof_error::malformed;
+    return coveragemap_error::malformed;
   ProfileNames.Data = Data.substr(0, ProfileNamesSize);
   CoverageMapping = Data.substr(ProfileNamesSize);
-  return instrprof_error::success;
+  return std::error_code();
+}
+
+static ErrorOr<SectionRef> lookupSection(ObjectFile &OF, StringRef Name) {
+  StringRef FoundName;
+  for (const auto &Section : OF.sections()) {
+    if (auto EC = Section.getName(FoundName))
+      return EC;
+    if (FoundName == Name)
+      return Section;
+  }
+  return coveragemap_error::no_data_found;
 }
 
 static std::error_code loadBinaryFormat(MemoryBufferRef ObjectBuffer,
@@ -432,7 +448,7 @@ static std::error_code loadBinaryFormat(MemoryBufferRef ObjectBuffer,
                                         StringRef &CoverageMapping,
                                         uint8_t &BytesInAddress,
                                         support::endianness &Endian,
-                                        Triple::ArchType Arch) {
+                                        StringRef Arch) {
   auto BinOrErr = object::createBinary(ObjectBuffer);
   if (std::error_code EC = BinOrErr.getError())
     return EC;
@@ -449,11 +465,11 @@ static std::error_code loadBinaryFormat(MemoryBufferRef ObjectBuffer,
     // For any other object file, upcast and take ownership.
     OF.reset(cast<object::ObjectFile>(Bin.release()));
     // If we've asked for a particular arch, make sure they match.
-    if (Arch != Triple::ArchType::UnknownArch && OF->getArch() != Arch)
+    if (!Arch.empty() && OF->getArch() != Triple(Arch).getArch())
       return object_error::arch_not_found;
   } else
     // We can only handle object files.
-    return instrprof_error::malformed;
+    return coveragemap_error::malformed;
 
   // The coverage uses native pointer sizes for the object it's written in.
   BytesInAddress = OF->getBytesInAddress();
@@ -461,27 +477,17 @@ static std::error_code loadBinaryFormat(MemoryBufferRef ObjectBuffer,
                                 : support::endianness::big;
 
   // Look for the sections that we are interested in.
-  int FoundSectionCount = 0;
-  SectionRef NamesSection, CoverageSection;
-  for (const auto &Section : OF->sections()) {
-    StringRef Name;
-    if (auto Err = Section.getName(Name))
-      return Err;
-    if (Name == "__llvm_prf_names") {
-      NamesSection = Section;
-    } else if (Name == "__llvm_covmap") {
-      CoverageSection = Section;
-    } else
-      continue;
-    ++FoundSectionCount;
-  }
-  if (FoundSectionCount != 2)
-    return instrprof_error::bad_header;
+  auto NamesSection = lookupSection(*OF, "__llvm_prf_names");
+  if (auto EC = NamesSection.getError())
+    return EC;
+  auto CoverageSection = lookupSection(*OF, "__llvm_covmap");
+  if (auto EC = CoverageSection.getError())
+    return EC;
 
   // Get the contents of the given sections.
-  if (std::error_code EC = CoverageSection.getContents(CoverageMapping))
+  if (std::error_code EC = CoverageSection->getContents(CoverageMapping))
     return EC;
-  if (std::error_code EC = ProfileNames.load(NamesSection))
+  if (std::error_code EC = ProfileNames.load(*NamesSection))
     return EC;
 
   return std::error_code();
@@ -489,7 +495,7 @@ static std::error_code loadBinaryFormat(MemoryBufferRef ObjectBuffer,
 
 ErrorOr<std::unique_ptr<BinaryCoverageReader>>
 BinaryCoverageReader::create(std::unique_ptr<MemoryBuffer> &ObjectBuffer,
-                             Triple::ArchType Arch) {
+                             StringRef Arch) {
   std::unique_ptr<BinaryCoverageReader> Reader(new BinaryCoverageReader());
 
   SectionData Profile;
@@ -520,7 +526,7 @@ BinaryCoverageReader::create(std::unique_ptr<MemoryBuffer> &ObjectBuffer,
     EC = readCoverageMappingData<uint64_t, support::endianness::big>(
         Profile, Coverage, Reader->MappingRecords, Reader->Filenames);
   else
-    return instrprof_error::malformed;
+    return coveragemap_error::malformed;
   if (EC)
     return EC;
   return std::move(Reader);
@@ -529,7 +535,7 @@ BinaryCoverageReader::create(std::unique_ptr<MemoryBuffer> &ObjectBuffer,
 std::error_code
 BinaryCoverageReader::readNextRecord(CoverageMappingRecord &Record) {
   if (CurrentRecord >= MappingRecords.size())
-    return instrprof_error::eof;
+    return coveragemap_error::eof;
 
   FunctionsFilenames.clear();
   Expressions.clear();
diff --git a/lib/ProfileData/InstrProf.cpp b/lib/ProfileData/InstrProf.cpp
index 900dff96739..92822a71402 100644
--- a/lib/ProfileData/InstrProf.cpp
+++ b/lib/ProfileData/InstrProf.cpp
@@ -29,13 +29,13 @@ class InstrProfErrorCategoryType : public std::error_category {
     case instrprof_error::eof:
       return "End of File";
     case instrprof_error::bad_magic:
-      return "Invalid file format (bad magic)";
+      return "Invalid profile data (bad magic)";
     case instrprof_error::bad_header:
-      return "Invalid header";
+      return "Invalid profile data (file header is corrupt)";
     case instrprof_error::unsupported_version:
-      return "Unsupported format version";
+      return "Unsupported profiling format version";
     case instrprof_error::unsupported_hash_type:
-      return "Unsupported hash function";
+      return "Unsupported profiling hash";
     case instrprof_error::too_large:
       return "Too much profile data";
     case instrprof_error::truncated:
diff --git a/lib/ProfileData/InstrProfReader.cpp b/lib/ProfileData/InstrProfReader.cpp
index 3a5b266016c..8a529a000c5 100644
--- a/lib/ProfileData/InstrProfReader.cpp
+++ b/lib/ProfileData/InstrProfReader.cpp
@@ -15,7 +15,6 @@
 #include "llvm/ProfileData/InstrProfReader.h"
 #include "InstrProfIndexed.h"
 #include "llvm/ADT/STLExtras.h"
-#include "llvm/ProfileData/InstrProf.h"
 #include <cassert>
 
 using namespace llvm;
@@ -126,18 +125,16 @@ std::error_code TextInstrProfReader::readNextRecord(InstrProfRecord &Record) {
     return error(instrprof_error::malformed);
 
   // Read each counter and fill our internal storage with the values.
-  Counts.clear();
-  Counts.reserve(NumCounters);
+  Record.Counts.clear();
+  Record.Counts.reserve(NumCounters);
   for (uint64_t I = 0; I < NumCounters; ++I) {
     if (Line.is_at_end())
       return error(instrprof_error::truncated);
     uint64_t Count;
     if ((Line++)->getAsInteger(10, Count))
       return error(instrprof_error::malformed);
-    Counts.push_back(Count);
+    Record.Counts.push_back(Count);
   }
-  // Give the record a reference to our internal counter storage.
-  Record.Counts = Counts;
 
   return success();
 }
@@ -280,11 +277,10 @@ RawInstrProfReader<IntPtrT>::readNextRecord(InstrProfRecord &Record) {
   Record.Hash = swap(Data->FuncHash);
   Record.Name = RawName;
   if (ShouldSwapBytes) {
-    Counts.clear();
-    Counts.reserve(RawCounts.size());
+    Record.Counts.clear();
+    Record.Counts.reserve(RawCounts.size());
     for (uint64_t Count : RawCounts)
-      Counts.push_back(swap(Count));
-    Record.Counts = Counts;
+      Record.Counts.push_back(swap(Count));
   } else
     Record.Counts = RawCounts;
 
@@ -303,6 +299,49 @@ InstrProfLookupTrait::ComputeHash(StringRef K) {
   return IndexedInstrProf::ComputeHash(HashType, K);
 }
 
+typedef InstrProfLookupTrait::data_type data_type;
+typedef InstrProfLookupTrait::offset_type offset_type;
+
+data_type InstrProfLookupTrait::ReadData(StringRef K, const unsigned char *D,
+                                         offset_type N) {
+
+  // Check if the data is corrupt. If so, don't try to read it.
+  if (N % sizeof(uint64_t))
+    return data_type();
+
+  DataBuffer.clear();
+  uint64_t NumCounts;
+  uint64_t NumEntries = N / sizeof(uint64_t);
+  std::vector<uint64_t> CounterBuffer;
+  for (uint64_t I = 0; I < NumEntries; I += NumCounts) {
+    using namespace support;
+    // The function hash comes first.
+    uint64_t Hash = endian::readNext<uint64_t, little, unaligned>(D);
+
+    if (++I >= NumEntries)
+      return data_type();
+
+    // In v1, we have at least one count.
+    // Later, we have the number of counts.
+    NumCounts = (1 == FormatVersion)
+                    ? NumEntries - I
+                    : endian::readNext<uint64_t, little, unaligned>(D);
+    if (1 != FormatVersion)
+      ++I;
+
+    // If we have more counts than data, this is bogus.
+    if (I + NumCounts > NumEntries)
+      return data_type();
+
+    CounterBuffer.clear();
+    for (unsigned J = 0; J < NumCounts; ++J)
+      CounterBuffer.push_back(endian::readNext<uint64_t, little, unaligned>(D));
+
+    DataBuffer.push_back(InstrProfRecord(K, Hash, std::move(CounterBuffer)));
+  }
+  return DataBuffer;
+}
+
 bool IndexedInstrProfReader::hasFormat(const MemoryBuffer &DataBuffer) {
   if (DataBuffer.getBufferSize() < 8)
     return false;
@@ -342,8 +381,9 @@ std::error_code IndexedInstrProfReader::readHeader() {
   uint64_t HashOffset = endian::readNext<uint64_t, little, unaligned>(Cur);
 
   // The rest of the file is an on disk hash table.
-  Index.reset(InstrProfReaderIndex::Create(Start + HashOffset, Cur, Start,
-                                           InstrProfLookupTrait(HashType)));
+  Index.reset(InstrProfReaderIndex::Create(
+      Start + HashOffset, Cur, Start,
+      InstrProfLookupTrait(HashType, FormatVersion)));
   // Set up our iterator for readNextRecord.
   RecordIterator = Index->data_begin();
 
@@ -357,21 +397,14 @@ std::error_code IndexedInstrProfReader::getFunctionCounts(
     return error(instrprof_error::unknown_function);
 
   // Found it. Look for counters with the right hash.
-  ArrayRef<uint64_t> Data = (*Iter).Data;
-  uint64_t NumCounts;
-  for (uint64_t I = 0, E = Data.size(); I != E; I += NumCounts) {
-    // The function hash comes first.
-    uint64_t FoundHash = Data[I++];
-    // In v1, we have at least one count. Later, we have the number of counts.
-    if (I == E)
-      return error(instrprof_error::malformed);
-    NumCounts = FormatVersion == 1 ? E - I : Data[I++];
-    // If we have more counts than data, this is bogus.
-    if (I + NumCounts > E)
-      return error(instrprof_error::malformed);
+  ArrayRef<InstrProfRecord> Data = (*Iter);
+  if (Data.empty())
+    return error(instrprof_error::malformed);
+
+  for (unsigned I = 0, E = Data.size(); I < E; ++I) {
     // Check for a match and fill the vector if there is one.
-    if (FoundHash == FuncHash) {
-      Counts = Data.slice(I, NumCounts);
+    if (Data[I].Hash == FuncHash) {
+      Counts = Data[I].Counts;
       return success();
     }
   }
@@ -384,30 +417,15 @@ IndexedInstrProfReader::readNextRecord(InstrProfRecord &Record) {
   if (RecordIterator == Index->data_end())
     return error(instrprof_error::eof);
 
-  // Record the current function name.
-  Record.Name = (*RecordIterator).Name;
-
-  ArrayRef<uint64_t> Data = (*RecordIterator).Data;
-  // Valid data starts with a hash and either a count or the number of counts.
-  if (CurrentOffset + 1 > Data.size())
-    return error(instrprof_error::malformed);
-  // First we have a function hash.
-  Record.Hash = Data[CurrentOffset++];
-  // In version 1 we knew the number of counters implicitly, but in newer
-  // versions we store the number of counters next.
-  uint64_t NumCounts =
-      FormatVersion == 1 ? Data.size() - CurrentOffset : Data[CurrentOffset++];
-  if (CurrentOffset + NumCounts > Data.size())
+  if ((*RecordIterator).empty())
     return error(instrprof_error::malformed);
-  // And finally the counts themselves.
-  Record.Counts = Data.slice(CurrentOffset, NumCounts);
 
-  // If we've exhausted this function's data, increment the record.
-  CurrentOffset += NumCounts;
-  if (CurrentOffset == Data.size()) {
+  static unsigned RecordIndex = 0;
+  ArrayRef<InstrProfRecord> Data = (*RecordIterator);
+  Record = Data[RecordIndex++];
+  if (RecordIndex >= Data.size()) {
     ++RecordIterator;
-    CurrentOffset = 0;
+    RecordIndex = 0;
   }
-
   return success();
 }
diff --git a/lib/Support/APFloat.cpp b/lib/Support/APFloat.cpp
index 5a402bb2804..5d31225396d 100644
--- a/lib/Support/APFloat.cpp
+++ b/lib/Support/APFloat.cpp
@@ -52,14 +52,17 @@ namespace llvm {
     /* Number of bits in the significand.  This includes the integer
        bit.  */
     unsigned int precision;
+
+    /* Number of bits actually used in the semantics. */
+    unsigned int sizeInBits;
   };
 
-  const fltSemantics APFloat::IEEEhalf = { 15, -14, 11 };
-  const fltSemantics APFloat::IEEEsingle = { 127, -126, 24 };
-  const fltSemantics APFloat::IEEEdouble = { 1023, -1022, 53 };
-  const fltSemantics APFloat::IEEEquad = { 16383, -16382, 113 };
-  const fltSemantics APFloat::x87DoubleExtended = { 16383, -16382, 64 };
-  const fltSemantics APFloat::Bogus = { 0, 0, 0 };
+  const fltSemantics APFloat::IEEEhalf = { 15, -14, 11, 16 };
+  const fltSemantics APFloat::IEEEsingle = { 127, -126, 24, 32 };
+  const fltSemantics APFloat::IEEEdouble = { 1023, -1022, 53, 64 };
+  const fltSemantics APFloat::IEEEquad = { 16383, -16382, 113, 128 };
+  const fltSemantics APFloat::x87DoubleExtended = { 16383, -16382, 64, 80 };
+  const fltSemantics APFloat::Bogus = { 0, 0, 0, 0 };
 
   /* The PowerPC format consists of two doubles.  It does not map cleanly
      onto the usual format above.  It is approximated using twice the
@@ -72,7 +75,7 @@ namespace llvm {
      to represent all possible values held by a PPC double-double number,
      for example: (long double) 1.0 + (long double) 0x1p-106
      Should this be replaced by a full emulation of PPC double-double?  */
-  const fltSemantics APFloat::PPCDoubleDouble = { 1023, -1022 + 53, 53 + 53 };
+  const fltSemantics APFloat::PPCDoubleDouble = { 1023, -1022 + 53, 53 + 53, 128 };
 
   /* A tight upper bound on number of parts required to hold the value
      pow(5, power) is
@@ -2416,7 +2419,7 @@ APFloat::roundSignificandWithExponent(const integerPart *decSigParts,
                                       roundingMode rounding_mode)
 {
   unsigned int parts, pow5PartCount;
-  fltSemantics calcSemantics = { 32767, -32767, 0 };
+  fltSemantics calcSemantics = { 32767, -32767, 0, 0 };
   integerPart pow5Parts[maxPowerOfFiveParts];
   bool isNearest;
 
@@ -3368,6 +3371,10 @@ APFloat::getAllOnesValue(unsigned BitWidth, bool isIEEE)
   }
 }
 
+unsigned APFloat::getSizeInBits(const fltSemantics &Sem) {
+  return Sem.sizeInBits;
+}
+
 /// Make this number the largest magnitude normal number in the given
 /// semantics.
 void APFloat::makeLargest(bool Negative) {
@@ -3920,7 +3927,7 @@ APFloat::makeZero(bool Negative) {
 
 APFloat llvm::scalbn(APFloat X, int Exp) {
   if (X.isInfinity() || X.isZero() || X.isNaN())
-    return std::move(X);
+    return X;
 
   auto MaxExp = X.getSemantics().maxExponent;
   auto MinExp = X.getSemantics().minExponent;
@@ -3932,5 +3939,5 @@ APFloat llvm::scalbn(APFloat X, int Exp) {
     return APFloat::getZero(X.getSemantics(), X.isNegative());
 
   X.exponent += Exp;
-  return std::move(X);
+  return X;
 }
diff --git a/lib/Support/APInt.cpp b/lib/Support/APInt.cpp
index 228f75e58cd..23f89bb66f9 100644
--- a/lib/Support/APInt.cpp
+++ b/lib/Support/APInt.cpp
@@ -1586,28 +1586,18 @@ static void KnuthDiv(unsigned *u, unsigned *v, unsigned *q, unsigned* r,
     // this step is actually negative, (u[j+n]...u[j]) should be left as the
     // true value plus b**(n+1), namely as the b's complement of
     // the true value, and a "borrow" to the left should be remembered.
-    bool isNeg = false;
+    int64_t borrow = 0;
     for (unsigned i = 0; i < n; ++i) {
-      uint64_t u_tmp = (uint64_t(u[j+i+1]) << 32) | uint64_t(u[j+i]);
-      uint64_t subtrahend = uint64_t(qp) * uint64_t(v[i]);
-      bool borrow = subtrahend > u_tmp;
-      DEBUG(dbgs() << "KnuthDiv: u_tmp = " << u_tmp
-                   << ", subtrahend = " << subtrahend
+      uint64_t p = uint64_t(qp) * uint64_t(v[i]);
+      int64_t subres = int64_t(u[j+i]) - borrow - (unsigned)p;
+      u[j+i] = (unsigned)subres;
+      borrow = (p >> 32) - (subres >> 32);
+      DEBUG(dbgs() << "KnuthDiv: u[j+i] = " << u[j+i]
                    << ", borrow = " << borrow << '\n');
-
-      uint64_t result = u_tmp - subtrahend;
-      unsigned k = j + i;
-      u[k++] = (unsigned)result;         // subtraction low word
-      u[k++] = (unsigned)(result >> 32); // subtraction high word
-      while (borrow && k <= m+n) {       // deal with borrow to the left
-        borrow = u[k] == 0;
-        u[k]--;
-        k++;
-      }
-      isNeg |= borrow;
-      DEBUG(dbgs() << "KnuthDiv: u[j+i] = " << u[j+i] 
-                   << ", u[j+i+1] = " << u[j+i+1] << '\n');
     }
+    bool isNeg = u[j+n] < borrow;
+    u[j+n] -= (unsigned)borrow;
+
     DEBUG(dbgs() << "KnuthDiv: after subtraction:");
     DEBUG(for (int i = m+n; i >=0; i--) dbgs() << " " << u[i]);
     DEBUG(dbgs() << '\n');
diff --git a/lib/Support/APSInt.cpp b/lib/Support/APSInt.cpp
index 73acafa690c..975457c070a 100644
--- a/lib/Support/APSInt.cpp
+++ b/lib/Support/APSInt.cpp
@@ -17,6 +17,25 @@
 
 using namespace llvm;
 
+APSInt::APSInt(StringRef Str) {
+  assert(!Str.empty() && "Invalid string length");
+
+  // (Over-)estimate the required number of bits.
+  unsigned NumBits = ((Str.size() * 64) / 19) + 2;
+  APInt Tmp(NumBits, Str, /*Radix=*/10);
+  if (Str[0] == '-') {
+    unsigned MinBits = Tmp.getMinSignedBits();
+    if (MinBits > 0 && MinBits < NumBits)
+      Tmp = Tmp.trunc(MinBits);
+    *this = APSInt(Tmp, /*IsUnsigned=*/false);
+    return;
+  }
+  unsigned ActiveBits = Tmp.getActiveBits();
+  if (ActiveBits > 0 && ActiveBits < NumBits)
+    Tmp = Tmp.trunc(ActiveBits);
+  *this = APSInt(Tmp, /*IsUnsigned=*/true);
+}
+
 void APSInt::Profile(FoldingSetNodeID& ID) const {
   ID.AddInteger((unsigned) (IsUnsigned ? 1 : 0));
   APInt::Profile(ID);
diff --git a/lib/Support/CMakeLists.txt b/lib/Support/CMakeLists.txt
index 7c804ddce83..a8a4df51661 100644
--- a/lib/Support/CMakeLists.txt
+++ b/lib/Support/CMakeLists.txt
@@ -1,7 +1,7 @@
 set(system_libs)
 if( NOT MSVC )
   if( MINGW )
-    set(system_libs ${system_libs} imagehlp psapi shell32)
+    set(system_libs ${system_libs} psapi shell32 ole32)
   elseif( CMAKE_HOST_UNIX )
     if( HAVE_LIBRT )
       set(system_libs ${system_libs} rt)
@@ -37,6 +37,7 @@ add_llvm_library(LLVMSupport
   BlockFrequency.cpp
   BranchProbability.cpp
   circular_raw_ostream.cpp
+  COM.cpp
   CommandLine.cpp
   Compression.cpp
   ConvertUTF.c
@@ -70,8 +71,6 @@ add_llvm_library(LLVMSupport
   Options.cpp
   PluginLoader.cpp
   PrettyStackTrace.cpp
-  # @LOCALMOD
-  QueueStreamer.cpp
   RandomNumberGenerator.cpp
   Regex.cpp
   ScaledNumber.cpp
@@ -84,8 +83,10 @@ add_llvm_library(LLVMSupport
   StringExtras.cpp
   StringMap.cpp
   StringPool.cpp
+  StringSaver.cpp
   StringRef.cpp
   SystemUtils.cpp
+  TargetParser.cpp
   Timer.cpp
   ToolOutputFile.cpp
   Triple.cpp
diff --git a/lib/Support/COM.cpp b/lib/Support/COM.cpp
new file mode 100644
index 00000000000..cf3a133fd9b
--- /dev/null
+++ b/lib/Support/COM.cpp
@@ -0,0 +1,23 @@
+//===-- COM.cpp - Implement COM utility classes -----------------*- C++ -*-===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+//  This file implements utility classes related to COM.
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/Support/COM.h"
+
+#include "llvm/Config/config.h"
+
+// Include the platform-specific parts of this class.
+#ifdef LLVM_ON_UNIX
+#include "Unix/COM.inc"
+#elif LLVM_ON_WIN32
+#include "Windows/COM.inc"
+#endif
diff --git a/lib/Support/CommandLine.cpp b/lib/Support/CommandLine.cpp
index 3cabc54a73a..17fba95ebb2 100644
--- a/lib/Support/CommandLine.cpp
+++ b/lib/Support/CommandLine.cpp
@@ -32,6 +32,7 @@
 #include "llvm/Support/ManagedStatic.h"
 #include "llvm/Support/MemoryBuffer.h"
 #include "llvm/Support/Path.h"
+#include "llvm/Support/StringSaver.h"
 #include "llvm/Support/raw_ostream.h"
 #include <cstdlib>
 #include <map>
@@ -45,21 +46,21 @@ using namespace cl;
 //
 namespace llvm {
 namespace cl {
-TEMPLATE_INSTANTIATION(class basic_parser<bool>);
-TEMPLATE_INSTANTIATION(class basic_parser<boolOrDefault>);
-TEMPLATE_INSTANTIATION(class basic_parser<int>);
-TEMPLATE_INSTANTIATION(class basic_parser<unsigned>);
-TEMPLATE_INSTANTIATION(class basic_parser<unsigned long long>);
-TEMPLATE_INSTANTIATION(class basic_parser<double>);
-TEMPLATE_INSTANTIATION(class basic_parser<float>);
-TEMPLATE_INSTANTIATION(class basic_parser<std::string>);
-TEMPLATE_INSTANTIATION(class basic_parser<char>);
-
-TEMPLATE_INSTANTIATION(class opt<unsigned>);
-TEMPLATE_INSTANTIATION(class opt<int>);
-TEMPLATE_INSTANTIATION(class opt<std::string>);
-TEMPLATE_INSTANTIATION(class opt<char>);
-TEMPLATE_INSTANTIATION(class opt<bool>);
+template class basic_parser<bool>;
+template class basic_parser<boolOrDefault>;
+template class basic_parser<int>;
+template class basic_parser<unsigned>;
+template class basic_parser<unsigned long long>;
+template class basic_parser<double>;
+template class basic_parser<float>;
+template class basic_parser<std::string>;
+template class basic_parser<char>;
+
+template class opt<unsigned>;
+template class opt<int>;
+template class opt<std::string>;
+template class opt<char>;
+template class opt<bool>;
 }
 } // end namespace llvm::cl
 
@@ -78,7 +79,6 @@ void parser<double>::anchor() {}
 void parser<float>::anchor() {}
 void parser<std::string>::anchor() {}
 void parser<char>::anchor() {}
-void StringSaver::anchor() {}
 
 //===----------------------------------------------------------------------===//
 
@@ -564,7 +564,7 @@ void cl::TokenizeGNUCommandLine(StringRef Src, StringSaver &Saver,
     // End the token if this is whitespace.
     if (isWhitespace(Src[I])) {
       if (!Token.empty())
-        NewArgv.push_back(Saver.SaveString(Token.c_str()));
+        NewArgv.push_back(Saver.save(Token.c_str()));
       Token.clear();
       continue;
     }
@@ -575,7 +575,7 @@ void cl::TokenizeGNUCommandLine(StringRef Src, StringSaver &Saver,
 
   // Append the last token after hitting EOF with no whitespace.
   if (!Token.empty())
-    NewArgv.push_back(Saver.SaveString(Token.c_str()));
+    NewArgv.push_back(Saver.save(Token.c_str()));
   // Mark the end of response files
   if (MarkEOLs)
     NewArgv.push_back(nullptr);
@@ -656,7 +656,7 @@ void cl::TokenizeWindowsCommandLine(StringRef Src, StringSaver &Saver,
     if (State == UNQUOTED) {
       // Whitespace means the end of the token.
       if (isWhitespace(Src[I])) {
-        NewArgv.push_back(Saver.SaveString(Token.c_str()));
+        NewArgv.push_back(Saver.save(Token.c_str()));
         Token.clear();
         State = INIT;
         // Mark the end of lines in response files
@@ -691,7 +691,7 @@ void cl::TokenizeWindowsCommandLine(StringRef Src, StringSaver &Saver,
   }
   // Append the last token after hitting EOF with no whitespace.
   if (!Token.empty())
-    NewArgv.push_back(Saver.SaveString(Token.c_str()));
+    NewArgv.push_back(Saver.save(Token.c_str()));
   // Mark the end of response files
   if (MarkEOLs)
     NewArgv.push_back(nullptr);
@@ -779,26 +779,6 @@ bool cl::ExpandResponseFiles(StringSaver &Saver, TokenizerCallback Tokenizer,
   return AllExpanded;
 }
 
-namespace {
-class StrDupSaver : public StringSaver {
-  std::vector<char *> Dups;
-
-public:
-  ~StrDupSaver() override {
-    for (std::vector<char *>::iterator I = Dups.begin(), E = Dups.end(); I != E;
-         ++I) {
-      char *Dup = *I;
-      free(Dup);
-    }
-  }
-  const char *SaveString(const char *Str) override {
-    char *Dup = strdup(Str);
-    Dups.push_back(Dup);
-    return Dup;
-  }
-};
-}
-
 /// ParseEnvironmentOptions - An alternative entry point to the
 /// CommandLine library, which allows you to read the program's name
 /// from the caller (as PROGNAME) and its command-line arguments from
@@ -818,8 +798,9 @@ void cl::ParseEnvironmentOptions(const char *progName, const char *envVar,
   // Get program's "name", which we wouldn't know without the caller
   // telling us.
   SmallVector<const char *, 20> newArgv;
-  StrDupSaver Saver;
-  newArgv.push_back(Saver.SaveString(progName));
+  BumpPtrAllocator A;
+  BumpPtrStringSaver Saver(A);
+  newArgv.push_back(Saver.save(progName));
 
   // Parse the value of the environment variable into a "command line"
   // and hand it off to ParseCommandLineOptions().
@@ -840,7 +821,8 @@ void CommandLineParser::ParseCommandLineOptions(int argc,
 
   // Expand response files.
   SmallVector<const char *, 20> newArgv(argv, argv + argc);
-  StrDupSaver Saver;
+  BumpPtrAllocator A;
+  BumpPtrStringSaver Saver(A);
   ExpandResponseFiles(Saver, TokenizeGNUCommandLine, newArgv);
   argv = &newArgv[0];
   argc = static_cast<int>(newArgv.size());
diff --git a/lib/Support/CrashRecoveryContext.cpp b/lib/Support/CrashRecoveryContext.cpp
index a158b761a09..aba0f1ddeee 100644
--- a/lib/Support/CrashRecoveryContext.cpp
+++ b/lib/Support/CrashRecoveryContext.cpp
@@ -275,7 +275,6 @@ void CrashRecoveryContext::Enable() {
 
   gCrashRecoveryEnabled = true;
 
-#if !defined(__native_client__) // @LOCALMOD
   // Setup the signal handler.
   struct sigaction Handler;
   Handler.sa_handler = CrashRecoverySignalHandler;
@@ -285,11 +284,6 @@ void CrashRecoveryContext::Enable() {
   for (unsigned i = 0; i != NumSignals; ++i) {
     sigaction(Signals[i], &Handler, &PrevActions[i]);
   }
-// @LOCALMOD-START
-#else
-#warning Cannot setup the signal handler on this machine
-#endif
-// @LOCALMOD-END
 }
 
 void CrashRecoveryContext::Disable() {
@@ -300,11 +294,9 @@ void CrashRecoveryContext::Disable() {
 
   gCrashRecoveryEnabled = false;
 
-#if !defined(__native_client__) // @LOCALMOD
   // Restore the previous signal handlers.
   for (unsigned i = 0; i != NumSignals; ++i)
     sigaction(Signals[i], &PrevActions[i], nullptr);
-#endif // @LOCALMOD
 }
 
 #endif
diff --git a/lib/Support/DataStream.cpp b/lib/Support/DataStream.cpp
index c24315526cf..3b10fc5eeca 100644
--- a/lib/Support/DataStream.cpp
+++ b/lib/Support/DataStream.cpp
@@ -16,6 +16,7 @@
 
 #include "llvm/Support/DataStream.h"
 #include "llvm/ADT/Statistic.h"
+#include "llvm/ADT/STLExtras.h"
 #include "llvm/Support/FileSystem.h"
 #include "llvm/Support/Program.h"
 #include <string>
@@ -73,16 +74,13 @@ class DataFileStreamer : public DataStreamer {
 
 }
 
-namespace llvm {
-DataStreamer *getDataFileStreamer(const std::string &Filename,
-                                  std::string *StrError) {
-  DataFileStreamer *s = new DataFileStreamer();
+std::unique_ptr<DataStreamer>
+llvm::getDataFileStreamer(const std::string &Filename, std::string *StrError) {
+  std::unique_ptr<DataFileStreamer> s = make_unique<DataFileStreamer>();
   if (std::error_code e = s->OpenFile(Filename)) {
     *StrError = std::string("Could not open ") + Filename + ": " +
         e.message() + "\n";
     return nullptr;
   }
-  return s;
-}
-
+  return std::move(s);
 }
diff --git a/lib/Support/Debug.cpp b/lib/Support/Debug.cpp
index eb99242914c..47751fce3fc 100644
--- a/lib/Support/Debug.cpp
+++ b/lib/Support/Debug.cpp
@@ -49,9 +49,9 @@ static ManagedStatic<std::vector<std::string>> CurrentDebugType;
 bool isCurrentDebugType(const char *DebugType) {
   if (CurrentDebugType->empty())
     return true;
-  // see if DebugType is in list. Note: do not use find() as that forces us to
+  // See if DebugType is in list. Note: do not use find() as that forces us to
   // unnecessarily create an std::string instance.
-  for (auto d : *CurrentDebugType) {
+  for (auto &d : *CurrentDebugType) {
     if (d == DebugType)
       return true;
   }
diff --git a/lib/Support/Dwarf.cpp b/lib/Support/Dwarf.cpp
index 95c4bc32701..13a41557a8d 100644
--- a/lib/Support/Dwarf.cpp
+++ b/lib/Support/Dwarf.cpp
@@ -190,6 +190,9 @@ const char *llvm::dwarf::AttributeString(unsigned Attribute) {
   case DW_AT_APPLE_property_attribute:   return "DW_AT_APPLE_property_attribute";
   case DW_AT_APPLE_property:             return "DW_AT_APPLE_property";
   case DW_AT_APPLE_objc_complete_type:   return "DW_AT_APPLE_objc_complete_type";
+  case DW_AT_LLVM_include_path:          return "DW_AT_LLVM_include_path";
+  case DW_AT_LLVM_config_macros:         return "DW_AT_LLVM_config_macros";
+  case DW_AT_LLVM_isysroot:              return "DW_AT_LLVM_isysroot";
 
     // DWARF5 Fission Extension Attribute
   case DW_AT_GNU_dwo_name:               return "DW_AT_GNU_dwo_name";
@@ -233,6 +236,10 @@ const char *llvm::dwarf::FormEncodingString(unsigned Encoding) {
     // DWARF5 Fission Extension Forms
   case DW_FORM_GNU_addr_index:           return "DW_FORM_GNU_addr_index";
   case DW_FORM_GNU_str_index:            return "DW_FORM_GNU_str_index";
+
+  // Alternate debug sections proposal (output of "dwz" tool).
+  case DW_FORM_GNU_ref_alt:              return "DW_FORM_GNU_ref_alt";
+  case DW_FORM_GNU_strp_alt:             return "DW_FORM_GNU_strp_alt";
   }
   return nullptr;
 }
diff --git a/lib/Support/DynamicLibrary.cpp b/lib/Support/DynamicLibrary.cpp
index d2b551e8a0a..9a7aeb50a21 100644
--- a/lib/Support/DynamicLibrary.cpp
+++ b/lib/Support/DynamicLibrary.cpp
@@ -178,3 +178,12 @@ void* DynamicLibrary::SearchForAddressOfSymbol(const char *symbolName) {
 LLVMBool LLVMLoadLibraryPermanently(const char* Filename) {
   return llvm::sys::DynamicLibrary::LoadLibraryPermanently(Filename);
 }
+
+void *LLVMSearchForAddressOfSymbol(const char *symbolName) {
+  return llvm::sys::DynamicLibrary::SearchForAddressOfSymbol(symbolName);
+}
+
+void LLVMAddSymbol(const char *symbolName, void *symbolValue) {
+  return llvm::sys::DynamicLibrary::AddSymbol(symbolName, symbolValue);
+}
+
diff --git a/lib/Support/GraphWriter.cpp b/lib/Support/GraphWriter.cpp
index 97aedc88473..a9b02204146 100644
--- a/lib/Support/GraphWriter.cpp
+++ b/lib/Support/GraphWriter.cpp
@@ -135,12 +135,12 @@ static const char *getProgramName(GraphProgram::Name program) {
 bool llvm::DisplayGraph(StringRef FilenameRef, bool wait,
                         GraphProgram::Name program) {
   std::string Filename = FilenameRef;
-  wait &= !ViewBackground;
   std::string ErrMsg;
   std::string ViewerPath;
   GraphSession S;
 
 #ifdef __APPLE__
+  wait &= !ViewBackground;
   if (S.TryFindProgram("open", ViewerPath)) {
     std::vector<const char *> args;
     args.push_back(ViewerPath.c_str());
diff --git a/lib/Support/Host.cpp b/lib/Support/Host.cpp
index 726961a2321..1bd1fe2bea0 100644
--- a/lib/Support/Host.cpp
+++ b/lib/Support/Host.cpp
@@ -682,6 +682,28 @@ StringRef sys::getHostCPUName() {
   StringRef Str(buffer, CPUInfoSize);
   SmallVector<StringRef, 32> Lines;
   Str.split(Lines, "\n");
+
+  // Look for the CPU features.
+  SmallVector<StringRef, 32> CPUFeatures;
+  for (unsigned I = 0, E = Lines.size(); I != E; ++I)
+    if (Lines[I].startswith("features")) {
+      size_t Pos = Lines[I].find(":");
+      if (Pos != StringRef::npos) {
+        Lines[I].drop_front(Pos + 1).split(CPUFeatures, " ");
+        break;
+      }
+    }
+
+  // We need to check for the presence of vector support independently of
+  // the machine type, since we may only use the vector register set when
+  // supported by the kernel (and hypervisor).
+  bool HaveVectorSupport = false;
+  for (unsigned I = 0, E = CPUFeatures.size(); I != E; ++I) {
+    if (CPUFeatures[I] == "vx")
+      HaveVectorSupport = true;
+  }
+
+  // Now check the processor machine type.
   for (unsigned I = 0, E = Lines.size(); I != E; ++I) {
     if (Lines[I].startswith("processor ")) {
       size_t Pos = Lines[I].find("machine = ");
@@ -689,6 +711,8 @@ StringRef sys::getHostCPUName() {
         Pos += sizeof("machine = ") - 1;
         unsigned int Id;
         if (!Lines[I].drop_front(Pos).getAsInteger(10, Id)) {
+          if (Id >= 2964 && HaveVectorSupport)
+            return "z13";
           if (Id >= 2827)
             return "zEC12";
           if (Id >= 2817)
diff --git a/lib/Support/Locale.cpp b/lib/Support/Locale.cpp
index 35ddf7f11bf..d5cb72b5db3 100644
--- a/lib/Support/Locale.cpp
+++ b/lib/Support/Locale.cpp
@@ -15,7 +15,7 @@ int columnWidth(StringRef Text) {
 
 bool isPrint(int UCS) {
 #if LLVM_ON_WIN32
-  // Restrict characters that we'll try to print to the the lower part of ASCII
+  // Restrict characters that we'll try to print to the lower part of ASCII
   // except for the control characters (0x20 - 0x7E). In general one can not
   // reliably output code points U+0080 and higher using narrow character C/C++
   // output functions in Windows, because the meaning of the upper 128 codes is
diff --git a/lib/Support/LockFileManager.cpp b/lib/Support/LockFileManager.cpp
index 1ec9748ae47..fb81d60f183 100644
--- a/lib/Support/LockFileManager.cpp
+++ b/lib/Support/LockFileManager.cpp
@@ -12,6 +12,7 @@
 #include "llvm/Support/FileSystem.h"
 #include "llvm/Support/MemoryBuffer.h"
 #include "llvm/Support/raw_ostream.h"
+#include "llvm/Support/Signals.h"
 #include <sys/stat.h>
 #include <sys/types.h>
 #if LLVM_ON_WIN32
@@ -20,11 +21,17 @@
 #if LLVM_ON_UNIX
 #include <unistd.h>
 #endif
-using namespace llvm;
 
-// @LOCALMOD-BEGIN
-#if !defined(PNACL_BROWSER_TRANSLATOR)
-// @LOCALMOD-END
+#if defined(__APPLE__) && defined(__MAC_OS_X_VERSION_MIN_REQUIRED) && (__MAC_OS_X_VERSION_MIN_REQUIRED > 1050)
+#define USE_OSX_GETHOSTUUID 1
+#else
+#define USE_OSX_GETHOSTUUID 0
+#endif
+
+#if USE_OSX_GETHOSTUUID
+#include <uuid/uuid.h>
+#endif
+using namespace llvm;
 
 /// \brief Attempt to read the lock file with the given name, if it exists.
 ///
@@ -59,22 +66,80 @@ LockFileManager::readLockFile(StringRef LockFileName) {
   return None;
 }
 
-bool LockFileManager::processStillExecuting(StringRef Hostname, int PID) {
-// @LOCALMOD-BEGIN
-#if LLVM_ON_UNIX && !defined(__ANDROID__) && !defined(__native_client__)
-// @LOCALMOD-END
-  char MyHostname[256];
-  MyHostname[255] = 0;
-  MyHostname[0] = 0;
-  gethostname(MyHostname, 255);
+static std::error_code getHostID(SmallVectorImpl<char> &HostID) {
+  HostID.clear();
+
+#if USE_OSX_GETHOSTUUID
+  // On OS X, use the more stable hardware UUID instead of hostname.
+  struct timespec wait = {1, 0}; // 1 second.
+  uuid_t uuid;
+  if (gethostuuid(uuid, &wait) != 0)
+    return std::error_code(errno, std::system_category());
+
+  uuid_string_t UUIDStr;
+  uuid_unparse(uuid, UUIDStr);
+  StringRef UUIDRef(UUIDStr);
+  HostID.append(UUIDRef.begin(), UUIDRef.end());
+
+#elif LLVM_ON_UNIX
+  char HostName[256];
+  HostName[255] = 0;
+  HostName[0] = 0;
+  gethostname(HostName, 255);
+  StringRef HostNameRef(HostName);
+  HostID.append(HostNameRef.begin(), HostNameRef.end());
+
+#else
+  StringRef Dummy("localhost");
+  HostID.append(Dummy.begin(), Dummy.end());
+#endif
+
+  return std::error_code();
+}
+
+bool LockFileManager::processStillExecuting(StringRef HostID, int PID) {
+#if LLVM_ON_UNIX && !defined(__ANDROID__)
+  SmallString<256> StoredHostID;
+  if (getHostID(StoredHostID))
+    return true; // Conservatively assume it's executing on error.
+
   // Check whether the process is dead. If so, we're done.
-  if (MyHostname == Hostname && getsid(PID) == -1 && errno == ESRCH)
+  if (StoredHostID == HostID && getsid(PID) == -1 && errno == ESRCH)
     return false;
 #endif
 
   return true;
 }
 
+namespace {
+/// An RAII helper object ensure that the unique lock file is removed.
+///
+/// Ensures that if there is an error or a signal before we finish acquiring the
+/// lock, the unique file will be removed. And if we successfully take the lock,
+/// the signal handler is left in place so that signals while the lock is held
+/// will remove the unique lock file. The caller should ensure there is a
+/// matching call to sys::DontRemoveFileOnSignal when the lock is released.
+class RemoveUniqueLockFileOnSignal {
+  StringRef Filename;
+  bool RemoveImmediately;
+public:
+  RemoveUniqueLockFileOnSignal(StringRef Name)
+  : Filename(Name), RemoveImmediately(true) {
+    sys::RemoveFileOnSignal(Filename, nullptr);
+  }
+  ~RemoveUniqueLockFileOnSignal() {
+    if (!RemoveImmediately) {
+      // Leave the signal handler enabled. It will be removed when the lock is
+      // released.
+      return;
+    }
+    sys::fs::remove(Filename);
+    sys::DontRemoveFileOnSignal(Filename);
+  }
+  void lockAcquired() { RemoveImmediately = false; }
+};
+} // end anonymous namespace
+
 LockFileManager::LockFileManager(StringRef FileName)
 {
   this->FileName = FileName;
@@ -102,17 +167,18 @@ LockFileManager::LockFileManager(StringRef FileName)
 
   // Write our process ID to our unique lock file.
   {
-    raw_fd_ostream Out(UniqueLockFileID, /*shouldClose=*/true);
+    SmallString<256> HostID;
+    if (auto EC = getHostID(HostID)) {
+      Error = EC;
+      return;
+    }
 
+    raw_fd_ostream Out(UniqueLockFileID, /*shouldClose=*/true);
+    Out << HostID << ' ';
 #if LLVM_ON_UNIX
-    // FIXME: move getpid() call into LLVM
-    char hostname[256];
-    hostname[255] = 0;
-    hostname[0] = 0;
-    gethostname(hostname, 255);
-    Out << hostname << ' ' << getpid();
+    Out << getpid();
 #else
-    Out << "localhost 1";
+    Out << "1";
 #endif
     Out.close();
 
@@ -125,12 +191,18 @@ LockFileManager::LockFileManager(StringRef FileName)
     }
   }
 
+  // Clean up the unique file on signal, which also releases the lock if it is
+  // held since the .lock symlink will point to a nonexistent file.
+  RemoveUniqueLockFileOnSignal RemoveUniqueFile(UniqueLockFileName);
+
   while (1) {
     // Create a link from the lock file name. If this succeeds, we're done.
     std::error_code EC =
         sys::fs::create_link(UniqueLockFileName, LockFileName);
-    if (!EC)
+    if (!EC) {
+      RemoveUniqueFile.lockAcquired();
       return;
+    }
 
     if (EC != errc::file_exists) {
       Error = EC;
@@ -177,6 +249,9 @@ LockFileManager::~LockFileManager() {
   // Since we own the lock, remove the lock file and our own unique lock file.
   sys::fs::remove(LockFileName);
   sys::fs::remove(UniqueLockFileName);
+  // The unique file is now gone, so remove it from the signal handler. This
+  // matches a sys::RemoveFileOnSignal() in LockFileManager().
+  sys::DontRemoveFileOnSignal(UniqueLockFileName);
 }
 
 LockFileManager::WaitForUnlockResult LockFileManager::waitForUnlock() {
@@ -242,5 +317,3 @@ LockFileManager::WaitForUnlockResult LockFileManager::waitForUnlock() {
 std::error_code LockFileManager::unsafeRemoveLockFile() {
   return sys::fs::remove(LockFileName);
 }
-
-#endif // __native_client__ @LOCALMOD
diff --git a/lib/Support/MemoryBuffer.cpp b/lib/Support/MemoryBuffer.cpp
index 09a7bdba008..98862e96b74 100644
--- a/lib/Support/MemoryBuffer.cpp
+++ b/lib/Support/MemoryBuffer.cpp
@@ -272,7 +272,7 @@ getFileAux(const Twine &Filename, int64_t FileSize, uint64_t MapSize,
 static bool shouldUseMmap(int FD,
                           size_t FileSize,
                           size_t MapSize,
-                          int64_t Offset,   // @LOCALMOD (?)
+                          off_t Offset,
                           bool RequiresNullTerminator,
                           int PageSize,
                           bool IsVolatileSize) {
diff --git a/lib/Support/Path.cpp b/lib/Support/Path.cpp
index cf467381db8..985cdbf7432 100644
--- a/lib/Support/Path.cpp
+++ b/lib/Support/Path.cpp
@@ -889,8 +889,7 @@ std::error_code is_other(const Twine &Path, bool &Result) {
 }
 
 void directory_entry::replace_filename(const Twine &filename, file_status st) {
-  SmallString<128> path(Path.begin(), Path.end());
-  path::remove_filename(path);
+  SmallString<128> path = path::parent_path(Path);
   path::append(path, filename);
   Path = path.str();
   Status = st;
@@ -940,7 +939,8 @@ file_magic identify_magic(StringRef Magic) {
       break;
     case '!':
       if (Magic.size() >= 8)
-        if (memcmp(Magic.data(),"!<arch>\n",8) == 0)
+        if (memcmp(Magic.data(), "!<arch>\n", 8) == 0 ||
+            memcmp(Magic.data(), "!<thin>\n", 8) == 0)
           return file_magic::archive;
       break;
 
diff --git a/lib/Support/Process.cpp b/lib/Support/Process.cpp
index 3571cd3132e..6dcbb47e87d 100644
--- a/lib/Support/Process.cpp
+++ b/lib/Support/Process.cpp
@@ -29,6 +29,7 @@ using namespace sys;
 Optional<std::string> Process::FindInEnvPath(const std::string& EnvName,
                                              const std::string& FileName)
 {
+  assert(!path::is_absolute(FileName));
   Optional<std::string> FoundPath;
   Optional<std::string> OptPath = Process::GetEnv(EnvName);
   if (!OptPath.hasValue())
diff --git a/lib/Support/QueueStreamer.cpp b/lib/Support/QueueStreamer.cpp
deleted file mode 100644
index 0ddc47a0ad4..00000000000
--- a/lib/Support/QueueStreamer.cpp
+++ /dev/null
@@ -1,114 +0,0 @@
-//===--- llvm/Support/QueueStreamer.cpp - Producer/consumer data streamer -===//
-//
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
-//
-//===----------------------------------------------------------------------===//
-//
-// This file implements QueueStreamer.
-//
-//===----------------------------------------------------------------------===//
-
-#include "llvm/Support/QueueStreamer.h"
-
-#include <cassert>
-#include <cstring>
-
-#include "llvm/Support/Debug.h"
-#include "llvm/Support/raw_ostream.h"
-
-#define DEBUG_TYPE "queue-streamer"
-
-namespace llvm {
-
-size_t QueueStreamer::GetBytes(unsigned char *Buf, size_t Len) {
-  size_t TotalCopied = 0;
-  std::unique_lock<LockType> L(Mutex);
-  while (!Done && queueSize() < Len - TotalCopied) {
-    size_t Size = queueSize();
-    DEBUG(dbgs() << "QueueStreamer::GetBytes Len " << Len << " size " <<
-          Size << " << waiting\n");
-    queueGet(Buf + TotalCopied, Size);
-    TotalCopied += Size;
-    Cond.notify_one();
-    Cond.wait(L);
-  }
-  // If this is the last partial chunk, adjust Len such that the amount we
-  // fetch will be just the remaining bytes.
-  if (Done && queueSize() < Len - TotalCopied) {
-    Len = queueSize() + TotalCopied;
-  }
-  queueGet(Buf + TotalCopied, Len - TotalCopied);
-  Cond.notify_one();
-  return Len;
-}
-
-size_t QueueStreamer::PutBytes(unsigned char *Buf, size_t Len) {
-  size_t TotalCopied = 0;
-  std::unique_lock<LockType> L(Mutex);
-  while (capacityRemaining() < Len - TotalCopied) {
-    if (Bytes.size() * 2 > QueueStreamer::MaxSize) {
-      size_t Space = capacityRemaining();
-      queuePut(Buf + TotalCopied, Space);
-      TotalCopied += Space;
-      Cond.notify_one();
-      Cond.wait(L);
-    } else {
-      queueResize();
-    }
-  }
-  queuePut(Buf + TotalCopied, Len - TotalCopied);
-  Cond.notify_one();
-  return Len;
-}
-
-void QueueStreamer::SetDone() {
-  // Still need the lock to avoid signaling between the check and
-  // the wait in GetBytes.
-  std::unique_lock<LockType> L(Mutex);
-  Done = true;
-  Cond.notify_one();
-}
-
-// Double the size of the queue. Called with Mutex to protect Cons/Prod/Bytes.
-void QueueStreamer::queueResize() {
-  int Leftover = Bytes.size() - Cons;
-  DEBUG(dbgs() << "resizing to " << Bytes.size() * 2 << " " << Leftover << " "
-        << Prod << " " << Cons << "\n");
-  Bytes.resize(Bytes.size() * 2);
-  if (Cons > Prod) {
-    // There are unread bytes left between Cons and the previous end of the
-    // buffer. Move them to the new end of the buffer.
-    memmove(&Bytes[Bytes.size() - Leftover], &Bytes[Cons], Leftover);
-    Cons = Bytes.size() - Leftover;
-  }
-}
-
-// Called with Mutex held to protect Cons, Prod, and Bytes
-void QueueStreamer::queuePut(unsigned char *Buf, size_t Len) {
-  size_t EndSpace = std::min(Len, Bytes.size() - Prod);
-  DEBUG(dbgs() << "put, Len " << Len << " Endspace " << EndSpace << " p " <<
-        Prod << " c " << Cons << "\n");
-  // Copy up to the end of the buffer
-  memcpy(&Bytes[Prod], Buf, EndSpace);
-  // Wrap around if necessary
-  memcpy(&Bytes[0], Buf + EndSpace, Len - EndSpace);
-  Prod = (Prod + Len) % Bytes.size();
-}
-
-// Called with Mutex held to protect Cons, Prod, and Bytes
-void QueueStreamer::queueGet(unsigned char *Buf, size_t Len) {
-  assert(Len <= queueSize());
-  size_t EndSpace = std::min(Len, Bytes.size() - Cons);
-  DEBUG(dbgs() << "get, Len " << Len << " Endspace " << EndSpace << " p " <<
-        Prod << " c " << Cons << "\n");
-  // Copy up to the end of the buffer
-  memcpy(Buf, &Bytes[Cons], EndSpace);
-  // Wrap around if necessary
-  memcpy(Buf + EndSpace, &Bytes[0], Len - EndSpace);
-  Cons = (Cons + Len) % Bytes.size();
-}
-
-} // end of namespace llvm
diff --git a/lib/Support/Signals.cpp b/lib/Support/Signals.cpp
index a11789372d9..2c3560c3be9 100644
--- a/lib/Support/Signals.cpp
+++ b/lib/Support/Signals.cpp
@@ -12,8 +12,11 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "llvm/Support/Signals.h"
 #include "llvm/Config/config.h"
+#include "llvm/Support/ManagedStatic.h"
+#include "llvm/Support/Signals.h"
+
+#include <vector>
 
 namespace llvm {
 using namespace sys;
@@ -23,6 +26,15 @@ using namespace sys;
 //===          independent code.
 //===----------------------------------------------------------------------===//
 
+static ManagedStatic<std::vector<std::pair<void (*)(void *), void *>>>
+    CallBacksToRun;
+void sys::RunSignalHandlers() {
+  if (!CallBacksToRun.isConstructed())
+    return;
+  for (auto &I : *CallBacksToRun)
+    I.first(I.second);
+  CallBacksToRun->clear();
+}
 }
 
 // Include the platform-specific parts of this class.
diff --git a/lib/Support/SmallVector.cpp b/lib/Support/SmallVector.cpp
index f9c0e78270c..b931505bd6a 100644
--- a/lib/Support/SmallVector.cpp
+++ b/lib/Support/SmallVector.cpp
@@ -33,6 +33,7 @@ void SmallVectorBase::grow_pod(void *FirstEl, size_t MinSizeInBytes,
     // If this wasn't grown from the inline copy, grow the allocated space.
     NewElts = realloc(this->BeginX, NewCapacityInBytes);
   }
+  assert(NewElts && "Out of memory");
 
   this->EndX = (char*)NewElts+CurSizeBytes;
   this->BeginX = NewElts;
diff --git a/lib/Support/SourceMgr.cpp b/lib/Support/SourceMgr.cpp
index d5e3157b064..6d44a4d51f6 100644
--- a/lib/Support/SourceMgr.cpp
+++ b/lib/Support/SourceMgr.cpp
@@ -332,8 +332,8 @@ static bool isNonASCII(char c) {
   return c & 0x80;
 }
 
-void SMDiagnostic::print(const char *ProgName, raw_ostream &S,
-                         bool ShowColors) const {
+void SMDiagnostic::print(const char *ProgName, raw_ostream &S, bool ShowColors,
+                         bool ShowKindLabel) const {
   // Display colors only if OS supports colors.
   ShowColors &= S.has_colors();
 
@@ -357,27 +357,29 @@ void SMDiagnostic::print(const char *ProgName, raw_ostream &S,
     S << ": ";
   }
 
-  switch (Kind) {
-  case SourceMgr::DK_Error:
-    if (ShowColors)
-      S.changeColor(raw_ostream::RED, true);
-    S << "error: ";
-    break;
-  case SourceMgr::DK_Warning:
-    if (ShowColors)
-      S.changeColor(raw_ostream::MAGENTA, true);
-    S << "warning: ";
-    break;
-  case SourceMgr::DK_Note:
-    if (ShowColors)
-      S.changeColor(raw_ostream::BLACK, true);
-    S << "note: ";
-    break;
-  }
+  if (ShowKindLabel) {
+    switch (Kind) {
+    case SourceMgr::DK_Error:
+      if (ShowColors)
+        S.changeColor(raw_ostream::RED, true);
+      S << "error: ";
+      break;
+    case SourceMgr::DK_Warning:
+      if (ShowColors)
+        S.changeColor(raw_ostream::MAGENTA, true);
+      S << "warning: ";
+      break;
+    case SourceMgr::DK_Note:
+      if (ShowColors)
+        S.changeColor(raw_ostream::BLACK, true);
+      S << "note: ";
+      break;
+    }
 
-  if (ShowColors) {
-    S.resetColor();
-    S.changeColor(raw_ostream::SAVEDCOLOR, true);
+    if (ShowColors) {
+      S.resetColor();
+      S.changeColor(raw_ostream::SAVEDCOLOR, true);
+    }
   }
 
   S << Message << '\n';
diff --git a/lib/Support/StreamingMemoryObject.cpp b/lib/Support/StreamingMemoryObject.cpp
index f1563665982..5a44e624eb8 100644
--- a/lib/Support/StreamingMemoryObject.cpp
+++ b/lib/Support/StreamingMemoryObject.cpp
@@ -71,14 +71,12 @@ const uint8_t *RawMemoryObject::getPointer(uint64_t address,
 namespace llvm {
 // If the bitcode has a header, then its size is known, and we don't have to
 // block until we actually want to read it.
-// @LOCALMOD -- separated into Impl (revisit after merge)
-bool StreamingMemoryObjectImpl::isValidAddress(uint64_t address) const {
+bool StreamingMemoryObject::isValidAddress(uint64_t address) const {
   if (ObjectSize && address < ObjectSize) return true;
-    return fetchToPos(address);
+  return fetchToPos(address);
 }
 
-// @LOCALMOD -- separated into Impl
-uint64_t StreamingMemoryObjectImpl::getExtent() const {
+uint64_t StreamingMemoryObject::getExtent() const {
   if (ObjectSize) return ObjectSize;
   size_t pos = BytesRead + kChunkSize;
   // keep fetching until we run out of bytes
@@ -86,34 +84,38 @@ uint64_t StreamingMemoryObjectImpl::getExtent() const {
   return ObjectSize;
 }
 
-// @LOCALMOD --separated into Impl
-uint64_t StreamingMemoryObjectImpl::readBytes(uint8_t *Buf, uint64_t Size,
+uint64_t StreamingMemoryObject::readBytes(uint8_t *Buf, uint64_t Size,
                                           uint64_t Address) const {
   fetchToPos(Address + Size - 1);
-  if (Address >= BytesRead)
+  // Note: For wrapped bitcode files will set ObjectSize after the
+  // first call to fetchToPos. In such cases, ObjectSize can be
+  // smaller than BytesRead.
+  size_t MaxAddress =
+      (ObjectSize && ObjectSize < BytesRead) ? ObjectSize : BytesRead;
+  if (Address >= MaxAddress)
     return 0;
 
   uint64_t End = Address + Size;
-  if (End > BytesRead)
-    End = BytesRead;
-  assert(static_cast<int64_t>(End - Address) >= 0);
+  if (End > MaxAddress)
+    End = MaxAddress;
+  assert(End >= Address);
   Size = End - Address;
   memcpy(Buf, &Bytes[Address + BytesSkipped], Size);
   return Size;
 }
 
-// @LOCALMOD -- separated into Impl
-bool StreamingMemoryObjectImpl::dropLeadingBytes(size_t s) {
+bool StreamingMemoryObject::dropLeadingBytes(size_t s) {
   if (BytesRead < s) return true;
   BytesSkipped = s;
   BytesRead -= s;
   return false;
 }
 
-// @LOCALMOD -- separated into Impl
-void StreamingMemoryObjectImpl::setKnownObjectSize(size_t size) {
+void StreamingMemoryObject::setKnownObjectSize(size_t size) {
   ObjectSize = size;
   Bytes.reserve(size);
+  if (ObjectSize <= BytesRead)
+    EOFReached = true;
 }
 
 MemoryObject *getNonStreamedMemoryObject(const unsigned char *Start,
@@ -121,11 +123,10 @@ MemoryObject *getNonStreamedMemoryObject(const unsigned char *Start,
   return new RawMemoryObject(Start, End);
 }
 
-// @LOCALMOD -- separated into Impl
-StreamingMemoryObjectImpl::StreamingMemoryObjectImpl(
-    DataStreamer *streamer) :
-  Bytes(kChunkSize), Streamer(streamer), BytesRead(0), BytesSkipped(0),
-  ObjectSize(0), EOFReached(false) {
-  BytesRead = streamer->GetBytes(&Bytes[0], kChunkSize);
+StreamingMemoryObject::StreamingMemoryObject(
+    std::unique_ptr<DataStreamer> Streamer)
+    : Bytes(kChunkSize), Streamer(std::move(Streamer)), BytesRead(0),
+      BytesSkipped(0), ObjectSize(0), EOFReached(false) {
+  BytesRead = this->Streamer->GetBytes(&Bytes[0], kChunkSize);
 }
 }
diff --git a/lib/Support/StringSaver.cpp b/lib/Support/StringSaver.cpp
new file mode 100644
index 00000000000..d6b84e53dcc
--- /dev/null
+++ b/lib/Support/StringSaver.cpp
@@ -0,0 +1,19 @@
+//===-- StringSaver.cpp ---------------------------------------------------===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/Support/StringSaver.h"
+
+using namespace llvm;
+
+const char *StringSaver::saveImpl(StringRef S) {
+  char *P = Alloc.Allocate<char>(S.size() + 1);
+  memcpy(P, S.data(), S.size());
+  P[S.size()] = '\0';
+  return P;
+}
diff --git a/lib/Support/TargetParser.cpp b/lib/Support/TargetParser.cpp
new file mode 100644
index 00000000000..6d43f252eaf
--- /dev/null
+++ b/lib/Support/TargetParser.cpp
@@ -0,0 +1,671 @@
+//===-- TargetParser - Parser for target features ---------------*- C++ -*-===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This file implements a target parser to recognise hardware features such as
+// FPU/CPU/ARCH names as well as specific support such as HDIV, etc.
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/Support/ARMBuildAttributes.h"
+#include "llvm/Support/TargetParser.h"
+#include "llvm/ADT/StringExtras.h"
+#include "llvm/ADT/StringSwitch.h"
+#include <cctype>
+
+using namespace llvm;
+
+namespace {
+
+// List of canonical FPU names (use getFPUSynonym) and which architectural
+// features they correspond to (use getFPUFeatures).
+// FIXME: TableGen this.
+// The entries must appear in the order listed in ARM::FPUKind for correct indexing
+struct {
+  const char * Name;
+  ARM::FPUKind ID;
+  ARM::FPUVersion FPUVersion;
+  ARM::NeonSupportLevel NeonSupport;
+  ARM::FPURestriction Restriction;
+} FPUNames[] = {
+  { "invalid",        ARM::FK_INVALID,        ARM::FV_NONE,       ARM::NS_None,   ARM::FR_None},
+  { "none",           ARM::FK_NONE,           ARM::FV_NONE,       ARM::NS_None,   ARM::FR_None},
+  { "vfp",            ARM::FK_VFP,            ARM::FV_VFPV2,      ARM::NS_None,   ARM::FR_None},
+  { "vfpv2",          ARM::FK_VFPV2,          ARM::FV_VFPV2,      ARM::NS_None,   ARM::FR_None},
+  { "vfpv3",          ARM::FK_VFPV3,          ARM::FV_VFPV3,      ARM::NS_None,   ARM::FR_None},
+  { "vfpv3-fp16",     ARM::FK_VFPV3_FP16,     ARM::FV_VFPV3_FP16, ARM::NS_None,   ARM::FR_None},
+  { "vfpv3-d16",      ARM::FK_VFPV3_D16,      ARM::FV_VFPV3,      ARM::NS_None,   ARM::FR_D16},
+  { "vfpv3-d16-fp16", ARM::FK_VFPV3_D16_FP16, ARM::FV_VFPV3_FP16, ARM::NS_None,   ARM::FR_D16},
+  { "vfpv3xd",        ARM::FK_VFPV3XD,        ARM::FV_VFPV3,      ARM::NS_None,   ARM::FR_SP_D16},
+  { "vfpv3xd-fp16",   ARM::FK_VFPV3XD_FP16,   ARM::FV_VFPV3_FP16, ARM::NS_None,   ARM::FR_SP_D16},
+  { "vfpv4",          ARM::FK_VFPV4,          ARM::FV_VFPV4,      ARM::NS_None,   ARM::FR_None},
+  { "vfpv4-d16",      ARM::FK_VFPV4_D16,      ARM::FV_VFPV4,      ARM::NS_None,   ARM::FR_D16},
+  { "fpv4-sp-d16",    ARM::FK_FPV4_SP_D16,    ARM::FV_VFPV4,      ARM::NS_None,   ARM::FR_SP_D16},
+  { "fpv5-d16",       ARM::FK_FPV5_D16,       ARM::FV_VFPV5,      ARM::NS_None,   ARM::FR_D16},
+  { "fpv5-sp-d16",    ARM::FK_FPV5_SP_D16,    ARM::FV_VFPV5,      ARM::NS_None,   ARM::FR_SP_D16},
+  { "fp-armv8",       ARM::FK_FP_ARMV8,       ARM::FV_VFPV5,      ARM::NS_None,   ARM::FR_None},
+  { "neon",           ARM::FK_NEON,           ARM::FV_VFPV3,      ARM::NS_Neon,   ARM::FR_None},
+  { "neon-fp16",      ARM::FK_NEON_FP16,      ARM::FV_VFPV3_FP16, ARM::NS_Neon,   ARM::FR_None},
+  { "neon-vfpv4",     ARM::FK_NEON_VFPV4,     ARM::FV_VFPV4,      ARM::NS_Neon,   ARM::FR_None},
+  { "neon-fp-armv8",  ARM::FK_NEON_FP_ARMV8,  ARM::FV_VFPV5,      ARM::NS_Neon,   ARM::FR_None},
+  { "crypto-neon-fp-armv8",
+               ARM::FK_CRYPTO_NEON_FP_ARMV8,  ARM::FV_VFPV5,      ARM::NS_Crypto, ARM::FR_None},
+  { "softvfp",        ARM::FK_SOFTVFP,        ARM::FV_NONE,       ARM::NS_None,   ARM::FR_None},
+};
+
+// List of canonical arch names (use getArchSynonym).
+// This table also provides the build attribute fields for CPU arch
+// and Arch ID, according to the Addenda to the ARM ABI, chapters
+// 2.4 and 2.3.5.2 respectively.
+// FIXME: SubArch values were simplified to fit into the expectations
+// of the triples and are not conforming with their official names.
+// Check to see if the expectation should be changed.
+// FIXME: TableGen this.
+struct {
+  const char *Name;
+  ARM::ArchKind ID;
+  const char *CPUAttr; // CPU class in build attributes.
+  const char *SubArch; // Sub-Arch name.
+  ARMBuildAttrs::CPUArch ArchAttr; // Arch ID in build attributes.
+} ARCHNames[] = {
+  { "invalid",   ARM::AK_INVALID,  nullptr,   nullptr, ARMBuildAttrs::CPUArch::Pre_v4 },
+  { "armv2",     ARM::AK_ARMV2,    "2",       "v2",    ARMBuildAttrs::CPUArch::Pre_v4 },
+  { "armv2a",    ARM::AK_ARMV2A,   "2A",      "v2a",   ARMBuildAttrs::CPUArch::Pre_v4 },
+  { "armv3",     ARM::AK_ARMV3,    "3",       "v3",    ARMBuildAttrs::CPUArch::Pre_v4 },
+  { "armv3m",    ARM::AK_ARMV3M,   "3M",      "v3m",   ARMBuildAttrs::CPUArch::Pre_v4 },
+  { "armv4",     ARM::AK_ARMV4,    "4",       "v4",    ARMBuildAttrs::CPUArch::v4 },
+  { "armv4t",    ARM::AK_ARMV4T,   "4T",      "v4t",   ARMBuildAttrs::CPUArch::v4T },
+  { "armv5t",    ARM::AK_ARMV5T,   "5T",      "v5",    ARMBuildAttrs::CPUArch::v5T },
+  { "armv5te",   ARM::AK_ARMV5TE,  "5TE",     "v5e",   ARMBuildAttrs::CPUArch::v5TE },
+  { "armv5tej",  ARM::AK_ARMV5TEJ, "5TEJ",    "v5e",   ARMBuildAttrs::CPUArch::v5TEJ },
+  { "armv6",     ARM::AK_ARMV6,    "6",       "v6",    ARMBuildAttrs::CPUArch::v6 },
+  { "armv6k",    ARM::AK_ARMV6K,   "6K",      "v6k",   ARMBuildAttrs::CPUArch::v6K },
+  { "armv6t2",   ARM::AK_ARMV6T2,  "6T2",     "v6t2",  ARMBuildAttrs::CPUArch::v6T2 },
+  { "armv6z",    ARM::AK_ARMV6Z,   "6Z",      "v6z",   ARMBuildAttrs::CPUArch::v6KZ },
+  { "armv6zk",   ARM::AK_ARMV6ZK,  "6ZK",     "v6zk",  ARMBuildAttrs::CPUArch::v6KZ },
+  { "armv6-m",   ARM::AK_ARMV6M,   "6-M",     "v6m",   ARMBuildAttrs::CPUArch::v6_M },
+  { "armv6s-m",  ARM::AK_ARMV6SM,  "6S-M",    "v6sm",  ARMBuildAttrs::CPUArch::v6S_M },
+  { "armv7-a",   ARM::AK_ARMV7A,   "7-A",     "v7",    ARMBuildAttrs::CPUArch::v7 },
+  { "armv7-r",   ARM::AK_ARMV7R,   "7-R",     "v7r",   ARMBuildAttrs::CPUArch::v7 },
+  { "armv7-m",   ARM::AK_ARMV7M,   "7-M",     "v7m",   ARMBuildAttrs::CPUArch::v7 },
+  { "armv7e-m",  ARM::AK_ARMV7EM,  "7E-M",    "v7em",  ARMBuildAttrs::CPUArch::v7E_M },
+  { "armv8-a",   ARM::AK_ARMV8A,   "8-A",     "v8",    ARMBuildAttrs::CPUArch::v8 },
+  { "armv8.1-a", ARM::AK_ARMV8_1A, "8.1-A",   "v8.1a", ARMBuildAttrs::CPUArch::v8 },
+  // Non-standard Arch names.
+  { "iwmmxt",    ARM::AK_IWMMXT,   "iwmmxt",  "",      ARMBuildAttrs::CPUArch::v5TE },
+  { "iwmmxt2",   ARM::AK_IWMMXT2,  "iwmmxt2", "",      ARMBuildAttrs::CPUArch::v5TE },
+  { "xscale",    ARM::AK_XSCALE,   "xscale",  "",      ARMBuildAttrs::CPUArch::v5TE },
+  { "armv5",     ARM::AK_ARMV5,    "5T",      "v5",    ARMBuildAttrs::CPUArch::v5T },
+  { "armv5e",    ARM::AK_ARMV5E,   "5TE",     "v5e",   ARMBuildAttrs::CPUArch::v5TE },
+  { "armv6j",    ARM::AK_ARMV6J,   "6J",      "v6",    ARMBuildAttrs::CPUArch::v6 },
+  { "armv6hl",   ARM::AK_ARMV6HL,  "6-M",     "v6hl",  ARMBuildAttrs::CPUArch::v6_M },
+  { "armv7",     ARM::AK_ARMV7,    "7",       "v7",    ARMBuildAttrs::CPUArch::v7 },
+  { "armv7l",    ARM::AK_ARMV7L,   "7-L",     "v7l",   ARMBuildAttrs::CPUArch::v7 },
+  { "armv7hl",   ARM::AK_ARMV7HL,  "7-L",     "v7hl",  ARMBuildAttrs::CPUArch::v7 },
+  { "armv7s",    ARM::AK_ARMV7S,   "7-S",     "v7s",   ARMBuildAttrs::CPUArch::v7 }
+};
+// List of Arch Extension names.
+// FIXME: TableGen this.
+struct {
+  const char *Name;
+  unsigned ID;
+} ARCHExtNames[] = {
+  { "invalid",   ARM::AEK_INVALID },
+  { "none",      ARM::AEK_NONE },
+  { "crc",       ARM::AEK_CRC },
+  { "crypto",    ARM::AEK_CRYPTO },
+  { "fp",        ARM::AEK_FP },
+  { "idiv",     (ARM::AEK_HWDIVARM | ARM::AEK_HWDIV) },
+  { "mp",        ARM::AEK_MP },
+  { "simd",      ARM::AEK_SIMD },
+  { "sec",       ARM::AEK_SEC },
+  { "virt",      ARM::AEK_VIRT },
+  { "os",        ARM::AEK_OS },
+  { "iwmmxt",    ARM::AEK_IWMMXT },
+  { "iwmmxt2",   ARM::AEK_IWMMXT2 },
+  { "maverick",  ARM::AEK_MAVERICK },
+  { "xscale",    ARM::AEK_XSCALE }
+};
+// List of HWDiv names (use getHWDivSynonym) and which architectural
+// features they correspond to (use getHWDivFeatures).
+// FIXME: TableGen this.
+struct {
+  const char *Name;
+  unsigned ID;
+} HWDivNames[] = {
+  { "invalid",    ARM::AEK_INVALID },
+  { "none",       ARM::AEK_NONE },
+  { "thumb",      ARM::AEK_HWDIV },
+  { "arm",        ARM::AEK_HWDIVARM },
+  { "arm,thumb", (ARM::AEK_HWDIVARM | ARM::AEK_HWDIV) }
+};
+// List of CPU names and their arches.
+// The same CPU can have multiple arches and can be default on multiple arches.
+// When finding the Arch for a CPU, first-found prevails. Sort them accordingly.
+// When this becomes table-generated, we'd probably need two tables.
+// FIXME: TableGen this.
+struct {
+  const char *Name;
+  ARM::ArchKind ArchID;
+  ARM::FPUKind DefaultFPU;
+  bool Default; // is $Name the default CPU for $ArchID ?
+} CPUNames[] = {
+  { "arm2",          ARM::AK_ARMV2,    ARM::FK_NONE,       true },
+  { "arm3",          ARM::AK_ARMV2A,   ARM::FK_NONE,       true },
+  { "arm6",          ARM::AK_ARMV3,    ARM::FK_NONE,       true },
+  { "arm7m",         ARM::AK_ARMV3M,   ARM::FK_NONE,       true },
+  { "arm8",          ARM::AK_ARMV4,    ARM::FK_NONE,       false },
+  { "arm810",        ARM::AK_ARMV4,    ARM::FK_NONE,       false },
+  { "strongarm",     ARM::AK_ARMV4,    ARM::FK_NONE,       true },
+  { "strongarm110",  ARM::AK_ARMV4,    ARM::FK_NONE,       false },
+  { "strongarm1100", ARM::AK_ARMV4,    ARM::FK_NONE,       false },
+  { "strongarm1110", ARM::AK_ARMV4,    ARM::FK_NONE,       false },
+  { "arm7tdmi",      ARM::AK_ARMV4T,   ARM::FK_NONE,       true },
+  { "arm7tdmi-s",    ARM::AK_ARMV4T,   ARM::FK_NONE,       false },
+  { "arm710t",       ARM::AK_ARMV4T,   ARM::FK_NONE,       false },
+  { "arm720t",       ARM::AK_ARMV4T,   ARM::FK_NONE,       false },
+  { "arm9",          ARM::AK_ARMV4T,   ARM::FK_NONE,       false },
+  { "arm9tdmi",      ARM::AK_ARMV4T,   ARM::FK_NONE,       false },
+  { "arm920",        ARM::AK_ARMV4T,   ARM::FK_NONE,       false },
+  { "arm920t",       ARM::AK_ARMV4T,   ARM::FK_NONE,       false },
+  { "arm922t",       ARM::AK_ARMV4T,   ARM::FK_NONE,       false },
+  { "arm9312",       ARM::AK_ARMV4T,   ARM::FK_NONE,       false },
+  { "arm940t",       ARM::AK_ARMV4T,   ARM::FK_NONE,       false },
+  { "ep9312",        ARM::AK_ARMV4T,   ARM::FK_NONE,       false },
+  { "arm10tdmi",     ARM::AK_ARMV5T,   ARM::FK_NONE,       true },
+  { "arm1020t",      ARM::AK_ARMV5T,   ARM::FK_NONE,       false },
+  { "arm9e",         ARM::AK_ARMV5TE,  ARM::FK_NONE,       false },
+  { "arm946e-s",     ARM::AK_ARMV5TE,  ARM::FK_NONE,       false },
+  { "arm966e-s",     ARM::AK_ARMV5TE,  ARM::FK_NONE,       false },
+  { "arm968e-s",     ARM::AK_ARMV5TE,  ARM::FK_NONE,       false },
+  { "arm10e",        ARM::AK_ARMV5TE,  ARM::FK_NONE,       false },
+  { "arm1020e",      ARM::AK_ARMV5TE,  ARM::FK_NONE,       false },
+  { "arm1022e",      ARM::AK_ARMV5TE,  ARM::FK_NONE,       true },
+  { "iwmmxt",        ARM::AK_ARMV5TE,  ARM::FK_NONE,       false },
+  { "xscale",        ARM::AK_ARMV5TE,  ARM::FK_NONE,       false },
+  { "arm926ej-s",    ARM::AK_ARMV5TEJ, ARM::FK_NONE,       true },
+  { "arm1136jf-s",   ARM::AK_ARMV6,    ARM::FK_VFPV2,      true },
+  { "arm1176j-s",    ARM::AK_ARMV6K,   ARM::FK_NONE,       false },
+  { "arm1176jz-s",   ARM::AK_ARMV6K,   ARM::FK_NONE,       false },
+  { "mpcore",        ARM::AK_ARMV6K,   ARM::FK_VFPV2,      false },
+  { "mpcorenovfp",   ARM::AK_ARMV6K,   ARM::FK_NONE,       false },
+  { "arm1176jzf-s",  ARM::AK_ARMV6K,   ARM::FK_VFPV2,      true },
+  { "arm1176jzf-s",  ARM::AK_ARMV6Z,   ARM::FK_VFPV2,      true },
+  { "arm1176jzf-s",  ARM::AK_ARMV6ZK,  ARM::FK_VFPV2,      true },
+  { "arm1156t2-s",   ARM::AK_ARMV6T2,  ARM::FK_NONE,       true },
+  { "arm1156t2f-s",  ARM::AK_ARMV6T2,  ARM::FK_VFPV2,      false },
+  { "cortex-m0",     ARM::AK_ARMV6M,   ARM::FK_NONE,       true },
+  { "cortex-m0plus", ARM::AK_ARMV6M,   ARM::FK_NONE,       false },
+  { "cortex-m1",     ARM::AK_ARMV6M,   ARM::FK_NONE,       false },
+  { "sc000",         ARM::AK_ARMV6M,   ARM::FK_NONE,       false },
+  { "cortex-a5",     ARM::AK_ARMV7A,   ARM::FK_NEON_VFPV4, false },
+  { "cortex-a7",     ARM::AK_ARMV7A,   ARM::FK_NEON_VFPV4, false },
+  { "cortex-a8",     ARM::AK_ARMV7A,   ARM::FK_NEON,       true },
+  { "cortex-a9",     ARM::AK_ARMV7A,   ARM::FK_NEON_FP16,  false },
+  { "cortex-a12",    ARM::AK_ARMV7A,   ARM::FK_NEON_VFPV4, false },
+  { "cortex-a15",    ARM::AK_ARMV7A,   ARM::FK_NEON_VFPV4, false },
+  { "cortex-a17",    ARM::AK_ARMV7A,   ARM::FK_NEON_VFPV4, false },
+  { "krait",         ARM::AK_ARMV7A,   ARM::FK_NEON_VFPV4, false },
+  { "cortex-r4",     ARM::AK_ARMV7R,   ARM::FK_NONE,       true },
+  { "cortex-r4f",    ARM::AK_ARMV7R,   ARM::FK_VFPV3_D16,  false },
+  { "cortex-r5",     ARM::AK_ARMV7R,   ARM::FK_VFPV3_D16,      false },
+  { "cortex-r7",     ARM::AK_ARMV7R,   ARM::FK_VFPV3_D16_FP16, false },
+  { "sc300",         ARM::AK_ARMV7M,   ARM::FK_NONE,           false },
+  { "cortex-m3",     ARM::AK_ARMV7M,   ARM::FK_NONE,           true },
+  { "cortex-m4",     ARM::AK_ARMV7EM,  ARM::FK_FPV4_SP_D16,    true },
+  { "cortex-m7",     ARM::AK_ARMV7EM,  ARM::FK_FPV5_D16,             false },
+  { "cortex-a53",    ARM::AK_ARMV8A,   ARM::FK_CRYPTO_NEON_FP_ARMV8, true },
+  { "cortex-a57",    ARM::AK_ARMV8A,   ARM::FK_CRYPTO_NEON_FP_ARMV8, false },
+  { "cortex-a72",    ARM::AK_ARMV8A,   ARM::FK_CRYPTO_NEON_FP_ARMV8, false },
+  { "cyclone",       ARM::AK_ARMV8A,   ARM::FK_CRYPTO_NEON_FP_ARMV8, false },
+  { "generic",       ARM::AK_ARMV8_1A, ARM::FK_NEON_FP_ARMV8,        true },
+  // Non-standard Arch names.
+  { "iwmmxt",        ARM::AK_IWMMXT,   ARM::FK_NONE,       true },
+  { "xscale",        ARM::AK_XSCALE,   ARM::FK_NONE,       true },
+  { "arm10tdmi",     ARM::AK_ARMV5,    ARM::FK_NONE,       true },
+  { "arm1022e",      ARM::AK_ARMV5E,   ARM::FK_NONE,       true },
+  { "arm1136j-s",    ARM::AK_ARMV6J,   ARM::FK_NONE,       true },
+  { "arm1136jz-s",   ARM::AK_ARMV6J,   ARM::FK_NONE,       false },
+  { "cortex-m0",     ARM::AK_ARMV6SM,  ARM::FK_NONE,       true },
+  { "arm1176jzf-s",  ARM::AK_ARMV6HL,  ARM::FK_VFPV2,      true },
+  { "cortex-a8",     ARM::AK_ARMV7,    ARM::FK_NEON,       true },
+  { "cortex-a8",     ARM::AK_ARMV7L,   ARM::FK_NEON,       true },
+  { "cortex-a8",     ARM::AK_ARMV7HL,  ARM::FK_NEON,       true },
+  { "cortex-m4",     ARM::AK_ARMV7EM,  ARM::FK_NONE,       true },
+  { "swift",         ARM::AK_ARMV7S,   ARM::FK_NEON_VFPV4, true },
+  // Invalid CPU
+  { "invalid",       ARM::AK_INVALID,  ARM::FK_INVALID,    true }
+};
+
+} // namespace
+
+// ======================================================= //
+// Information by ID
+// ======================================================= //
+
+const char *ARMTargetParser::getFPUName(unsigned FPUKind) {
+  if (FPUKind >= ARM::FK_LAST)
+    return nullptr;
+  return FPUNames[FPUKind].Name;
+}
+
+unsigned ARMTargetParser::getFPUVersion(unsigned FPUKind) {
+  if (FPUKind >= ARM::FK_LAST)
+    return 0;
+  return FPUNames[FPUKind].FPUVersion;
+}
+
+unsigned ARMTargetParser::getFPUNeonSupportLevel(unsigned FPUKind) {
+  if (FPUKind >= ARM::FK_LAST)
+    return 0;
+  return FPUNames[FPUKind].NeonSupport;
+}
+
+unsigned ARMTargetParser::getFPURestriction(unsigned FPUKind) {
+  if (FPUKind >= ARM::FK_LAST)
+    return 0;
+  return FPUNames[FPUKind].Restriction;
+}
+
+unsigned ARMTargetParser::getDefaultFPU(StringRef CPU) {
+  for (const auto C : CPUNames) {
+    if (CPU == C.Name)
+      return C.DefaultFPU;
+  }
+  return ARM::FK_INVALID;
+}
+
+bool ARMTargetParser::getHWDivFeatures(unsigned HWDivKind,
+                                       std::vector<const char *> &Features) {
+
+  if (HWDivKind == ARM::AEK_INVALID)
+    return false;
+
+   if (HWDivKind & ARM::AEK_HWDIVARM)
+    Features.push_back("+hwdiv-arm");
+  else
+    Features.push_back("-hwdiv-arm");
+
+  if (HWDivKind & ARM::AEK_HWDIV)
+    Features.push_back("+hwdiv");
+  else
+    Features.push_back("-hwdiv");
+
+  return true;
+}
+
+bool ARMTargetParser::getFPUFeatures(unsigned FPUKind,
+                                     std::vector<const char *> &Features) {
+
+  if (FPUKind >= ARM::FK_LAST || FPUKind == ARM::FK_INVALID)
+    return false;
+
+  // fp-only-sp and d16 subtarget features are independent of each other, so we
+  // must enable/disable both.
+  switch (FPUNames[FPUKind].Restriction) {
+  case ARM::FR_SP_D16:
+    Features.push_back("+fp-only-sp");
+    Features.push_back("+d16");
+    break;
+  case ARM::FR_D16:
+    Features.push_back("-fp-only-sp");
+    Features.push_back("+d16");
+    break;
+  case ARM::FR_None:
+    Features.push_back("-fp-only-sp");
+    Features.push_back("-d16");
+    break;
+  }
+
+  // FPU version subtarget features are inclusive of lower-numbered ones, so
+  // enable the one corresponding to this version and disable all that are
+  // higher. We also have to make sure to disable fp16 when vfp4 is disabled,
+  // as +vfp4 implies +fp16 but -vfp4 does not imply -fp16.
+  switch (FPUNames[FPUKind].FPUVersion) {
+  case ARM::FV_VFPV5:
+    Features.push_back("+fp-armv8");
+    break;
+  case ARM::FV_VFPV4:
+    Features.push_back("+vfp4");
+    Features.push_back("-fp-armv8");
+    break;
+  case ARM::FV_VFPV3_FP16:
+    Features.push_back("+vfp3");
+    Features.push_back("+fp16");
+    Features.push_back("-vfp4");
+    Features.push_back("-fp-armv8");
+    break;
+  case ARM::FV_VFPV3:
+    Features.push_back("+vfp3");
+    Features.push_back("-fp16");
+    Features.push_back("-vfp4");
+    Features.push_back("-fp-armv8");
+    break;
+  case ARM::FV_VFPV2:
+    Features.push_back("+vfp2");
+    Features.push_back("-vfp3");
+    Features.push_back("-fp16");
+    Features.push_back("-vfp4");
+    Features.push_back("-fp-armv8");
+    break;
+  case ARM::FV_NONE:
+    Features.push_back("-vfp2");
+    Features.push_back("-vfp3");
+    Features.push_back("-fp16");
+    Features.push_back("-vfp4");
+    Features.push_back("-fp-armv8");
+    break;
+  }
+
+  // crypto includes neon, so we handle this similarly to FPU version.
+  switch (FPUNames[FPUKind].NeonSupport) {
+  case ARM::NS_Crypto:
+    Features.push_back("+crypto");
+    break;
+  case ARM::NS_Neon:
+    Features.push_back("+neon");
+    Features.push_back("-crypto");
+    break;
+  case ARM::NS_None:
+    Features.push_back("-neon");
+    Features.push_back("-crypto");
+    break;
+  }
+
+  return true;
+}
+
+const char *ARMTargetParser::getArchName(unsigned ArchKind) {
+  if (ArchKind >= ARM::AK_LAST)
+    return nullptr;
+  return ARCHNames[ArchKind].Name;
+}
+
+const char *ARMTargetParser::getCPUAttr(unsigned ArchKind) {
+  if (ArchKind >= ARM::AK_LAST)
+    return nullptr;
+  return ARCHNames[ArchKind].CPUAttr;
+}
+
+const char *ARMTargetParser::getSubArch(unsigned ArchKind) {
+  if (ArchKind >= ARM::AK_LAST)
+    return nullptr;
+  return ARCHNames[ArchKind].SubArch;
+}
+
+unsigned ARMTargetParser::getArchAttr(unsigned ArchKind) {
+  if (ArchKind >= ARM::AK_LAST)
+    return ARMBuildAttrs::CPUArch::Pre_v4;
+  return ARCHNames[ArchKind].ArchAttr;
+}
+
+const char *ARMTargetParser::getArchExtName(unsigned ArchExtKind) {
+  for (const auto AE : ARCHExtNames) {
+    if (ArchExtKind == AE.ID)
+      return AE.Name;
+  }
+  return nullptr;
+}
+
+const char *ARMTargetParser::getHWDivName(unsigned HWDivKind) {
+  for (const auto D : HWDivNames) {
+    if (HWDivKind == D.ID)
+      return D.Name;
+  }
+  return nullptr;
+}
+
+const char *ARMTargetParser::getDefaultCPU(StringRef Arch) {
+  unsigned AK = parseArch(Arch);
+  if (AK == ARM::AK_INVALID)
+    return nullptr;
+
+  // Look for multiple AKs to find the default for pair AK+Name.
+  for (const auto CPU : CPUNames) {
+    if (CPU.ArchID == AK && CPU.Default)
+      return CPU.Name;
+  }
+  return nullptr;
+}
+
+// ======================================================= //
+// Parsers
+// ======================================================= //
+
+StringRef ARMTargetParser::getHWDivSynonym(StringRef HWDiv) {
+  return StringSwitch<StringRef>(HWDiv)
+    .Case("thumb,arm", "arm,thumb")
+    .Default(HWDiv);
+}
+
+StringRef ARMTargetParser::getFPUSynonym(StringRef FPU) {
+  return StringSwitch<StringRef>(FPU)
+    .Cases("fpa", "fpe2", "fpe3", "maverick", "invalid") // Unsupported
+    .Case("vfp2", "vfpv2")
+    .Case("vfp3", "vfpv3")
+    .Case("vfp4", "vfpv4")
+    .Case("vfp3-d16", "vfpv3-d16")
+    .Case("vfp4-d16", "vfpv4-d16")
+    .Cases("fp4-sp-d16", "vfpv4-sp-d16", "fpv4-sp-d16")
+    .Cases("fp4-dp-d16", "fpv4-dp-d16", "vfpv4-d16")
+    .Case("fp5-sp-d16", "fpv5-sp-d16")
+    .Cases("fp5-dp-d16", "fpv5-dp-d16", "fpv5-d16")
+    // FIXME: Clang uses it, but it's bogus, since neon defaults to vfpv3.
+    .Case("neon-vfpv3", "neon")
+    .Default(FPU);
+}
+
+StringRef ARMTargetParser::getArchSynonym(StringRef Arch) {
+  return StringSwitch<StringRef>(Arch)
+    .Case("v6sm", "v6s-m")
+    .Case("v6m", "v6-m")
+    .Case("v7a", "v7-a")
+    .Case("v7r", "v7-r")
+    .Case("v7m", "v7-m")
+    .Case("v7em", "v7e-m")
+    .Cases("v8", "v8a", "aarch64", "arm64", "v8-a")
+    .Case("v8.1a", "v8.1-a")
+    .Default(Arch);
+}
+
+// MArch is expected to be of the form (arm|thumb)?(eb)?(v.+)?(eb)?, but
+// (iwmmxt|xscale)(eb)? is also permitted. If the former, return
+// "v.+", if the latter, return unmodified string, minus 'eb'.
+// If invalid, return empty string.
+StringRef ARMTargetParser::getCanonicalArchName(StringRef Arch) {
+  size_t offset = StringRef::npos;
+  StringRef A = Arch;
+  StringRef Error = "";
+
+  // Begins with "arm" / "thumb", move past it.
+  if (A.startswith("arm64"))
+    offset = 5;
+  else if (A.startswith("arm"))
+    offset = 3;
+  else if (A.startswith("thumb"))
+    offset = 5;
+  else if (A.startswith("aarch64")) {
+    offset = 7;
+    // AArch64 uses "_be", not "eb" suffix.
+    if (A.find("eb") != StringRef::npos)
+      return Error;
+    if (A.substr(offset,3) == "_be")
+      offset += 3;
+  }
+
+  // Ex. "armebv7", move past the "eb".
+  if (offset != StringRef::npos && A.substr(offset, 2) == "eb")
+    offset += 2;
+  // Or, if it ends with eb ("armv7eb"), chop it off.
+  else if (A.endswith("eb"))
+    A = A.substr(0, A.size() - 2);
+  // Trim the head
+  if (offset != StringRef::npos)
+    A = A.substr(offset);
+
+  // Empty string means offset reached the end, which means it's valid.
+  if (A.empty())
+    return Arch;
+
+  // Only match non-marketing names
+  if (offset != StringRef::npos) {
+  // Must start with 'vN'.
+    if (A[0] != 'v' || !std::isdigit(A[1]))
+      return Error;
+    // Can't have an extra 'eb'.
+    if (A.find("eb") != StringRef::npos)
+      return Error;
+  }
+
+  // Arch will either be a 'v' name (v7a) or a marketing name (xscale).
+  return A;
+}
+
+unsigned ARMTargetParser::parseHWDiv(StringRef HWDiv) {
+  StringRef Syn = getHWDivSynonym(HWDiv);
+  for (const auto D : HWDivNames) {
+    if (Syn == D.Name)
+      return D.ID;
+  }
+  return ARM::AEK_INVALID;
+}
+
+unsigned ARMTargetParser::parseFPU(StringRef FPU) {
+  StringRef Syn = getFPUSynonym(FPU);
+  for (const auto F : FPUNames) {
+    if (Syn == F.Name)
+      return F.ID;
+  }
+  return ARM::FK_INVALID;
+}
+
+// Allows partial match, ex. "v7a" matches "armv7a".
+unsigned ARMTargetParser::parseArch(StringRef Arch) {
+  Arch = getCanonicalArchName(Arch);
+  StringRef Syn = getArchSynonym(Arch);
+  for (const auto A : ARCHNames) {
+    if (StringRef(A.Name).endswith(Syn))
+      return A.ID;
+  }
+  return ARM::AK_INVALID;
+}
+
+unsigned ARMTargetParser::parseArchExt(StringRef ArchExt) {
+  for (const auto A : ARCHExtNames) {
+    if (ArchExt == A.Name)
+      return A.ID;
+  }
+  return ARM::AEK_INVALID;
+}
+
+unsigned ARMTargetParser::parseCPUArch(StringRef CPU) {
+  for (const auto C : CPUNames) {
+    if (CPU == C.Name)
+      return C.ArchID;
+  }
+  return ARM::AK_INVALID;
+}
+
+// ARM, Thumb, AArch64
+unsigned ARMTargetParser::parseArchISA(StringRef Arch) {
+  return StringSwitch<unsigned>(Arch)
+      .StartsWith("aarch64", ARM::IK_AARCH64)
+      .StartsWith("arm64",   ARM::IK_AARCH64)
+      .StartsWith("thumb",   ARM::IK_THUMB)
+      .StartsWith("arm",     ARM::IK_ARM)
+      .Default(ARM::EK_INVALID);
+}
+
+// Little/Big endian
+unsigned ARMTargetParser::parseArchEndian(StringRef Arch) {
+  if (Arch.startswith("armeb") ||
+      Arch.startswith("thumbeb") ||
+      Arch.startswith("aarch64_be"))
+    return ARM::EK_BIG;
+
+  if (Arch.startswith("arm") || Arch.startswith("thumb")) {
+    if (Arch.endswith("eb"))
+      return ARM::EK_BIG;
+    else
+      return ARM::EK_LITTLE;
+  }
+
+  if (Arch.startswith("aarch64"))
+    return ARM::EK_LITTLE;
+
+  return ARM::EK_INVALID;
+}
+
+// Profile A/R/M
+unsigned ARMTargetParser::parseArchProfile(StringRef Arch) {
+  Arch = getCanonicalArchName(Arch);
+  switch(parseArch(Arch)) {
+  case ARM::AK_ARMV6M:
+  case ARM::AK_ARMV7M:
+  case ARM::AK_ARMV6SM:
+  case ARM::AK_ARMV7EM:
+    return ARM::PK_M;
+  case ARM::AK_ARMV7R:
+    return ARM::PK_R;
+  case ARM::AK_ARMV7:
+  case ARM::AK_ARMV7A:
+  case ARM::AK_ARMV7L:
+  case ARM::AK_ARMV8A:
+  case ARM::AK_ARMV8_1A:
+    return ARM::PK_A;
+  }
+  return ARM::PK_INVALID;
+}
+
+// Version number (ex. v7 = 7).
+unsigned ARMTargetParser::parseArchVersion(StringRef Arch) {
+  Arch = getCanonicalArchName(Arch);
+  switch(parseArch(Arch)) {
+  case ARM::AK_ARMV2:
+  case ARM::AK_ARMV2A:
+    return 2;
+  case ARM::AK_ARMV3:
+  case ARM::AK_ARMV3M:
+    return 3;
+  case ARM::AK_ARMV4:
+  case ARM::AK_ARMV4T:
+    return 4;
+  case ARM::AK_ARMV5:
+  case ARM::AK_ARMV5T:
+  case ARM::AK_ARMV5TE:
+  case ARM::AK_IWMMXT:
+  case ARM::AK_IWMMXT2:
+  case ARM::AK_XSCALE:
+  case ARM::AK_ARMV5E:
+  case ARM::AK_ARMV5TEJ:
+    return 5;
+  case ARM::AK_ARMV6:
+  case ARM::AK_ARMV6J:
+  case ARM::AK_ARMV6K:
+  case ARM::AK_ARMV6T2:
+  case ARM::AK_ARMV6Z:
+  case ARM::AK_ARMV6ZK:
+  case ARM::AK_ARMV6M:
+  case ARM::AK_ARMV6SM:
+  case ARM::AK_ARMV6HL:
+    return 6;
+  case ARM::AK_ARMV7:
+  case ARM::AK_ARMV7A:
+  case ARM::AK_ARMV7R:
+  case ARM::AK_ARMV7M:
+  case ARM::AK_ARMV7L:
+  case ARM::AK_ARMV7HL:
+  case ARM::AK_ARMV7S:
+  case ARM::AK_ARMV7EM:
+    return 7;
+  case ARM::AK_ARMV8A:
+  case ARM::AK_ARMV8_1A:
+    return 8;
+  }
+  return 0;
+}
diff --git a/lib/Support/TargetRegistry.cpp b/lib/Support/TargetRegistry.cpp
index 3ca85728985..eefef8ad8ea 100644
--- a/lib/Support/TargetRegistry.cpp
+++ b/lib/Support/TargetRegistry.cpp
@@ -18,8 +18,8 @@ using namespace llvm;
 // Clients are responsible for avoid race conditions in registration.
 static Target *FirstTarget = nullptr;
 
-TargetRegistry::iterator TargetRegistry::begin() {
-  return iterator(FirstTarget);
+iterator_range<TargetRegistry::iterator> TargetRegistry::targets() {
+  return make_range(iterator(FirstTarget), iterator());
 }
 
 const Target *TargetRegistry::lookupTarget(const std::string &ArchName,
@@ -30,19 +30,17 @@ const Target *TargetRegistry::lookupTarget(const std::string &ArchName,
   // name, because it might be a backend that has no mapping to a target triple.
   const Target *TheTarget = nullptr;
   if (!ArchName.empty()) {
-    for (TargetRegistry::iterator it = TargetRegistry::begin(),
-           ie = TargetRegistry::end(); it != ie; ++it) {
-      if (ArchName == it->getName()) {
-        TheTarget = &*it;
-        break;
-      }
-    }
+    auto I =
+        std::find_if(targets().begin(), targets().end(),
+                     [&](const Target &T) { return ArchName == T.getName(); });
 
-    if (!TheTarget) {
+    if (I == targets().end()) {
       Error = "error: invalid target '" + ArchName + "'.\n";
       return nullptr;
     }
 
+    TheTarget = &*I;
+
     // Adjust the triple to match (if known), otherwise stick with the
     // given triple.
     Triple::ArchType Type = Triple::getArchTypeForLLVMName(ArchName);
@@ -66,30 +64,28 @@ const Target *TargetRegistry::lookupTarget(const std::string &ArchName,
 const Target *TargetRegistry::lookupTarget(const std::string &TT,
                                            std::string &Error) {
   // Provide special warning when no targets are initialized.
-  if (begin() == end()) {
+  if (targets().begin() == targets().end()) {
     Error = "Unable to find target for this triple (no targets are registered)";
     return nullptr;
   }
-  const Target *Matching = nullptr;
-  Triple::ArchType Arch =  Triple(TT).getArch();
-  for (iterator it = begin(), ie = end(); it != ie; ++it) {
-    if (it->ArchMatchFn(Arch)) {
-      if (Matching) {
-        Error = std::string("Cannot choose between targets \"") +
-          Matching->Name  + "\" and \"" + it->Name + "\"";
-        return nullptr;
-      }
-      Matching = &*it;
-    }
-  }
+  Triple::ArchType Arch = Triple(TT).getArch();
+  auto ArchMatch = [&](const Target &T) { return T.ArchMatchFn(Arch); };
+  auto I = std::find_if(targets().begin(), targets().end(), ArchMatch);
 
-  if (!Matching) {
+  if (I == targets().end()) {
     Error = "No available targets are compatible with this triple, "
       "see -version for the available targets.";
     return nullptr;
   }
 
-  return Matching;
+  auto J = std::find_if(std::next(I), targets().end(), ArchMatch);
+  if (J != targets().end()) {
+    Error = std::string("Cannot choose between targets \"") + I->Name +
+            "\" and \"" + J->Name + "\"";
+    return nullptr;
+  }
+
+  return &*I;
 }
 
 void TargetRegistry::RegisterTarget(Target &T,
@@ -123,10 +119,8 @@ static int TargetArraySortFn(const std::pair<StringRef, const Target *> *LHS,
 void TargetRegistry::printRegisteredTargetsForVersion() {
   std::vector<std::pair<StringRef, const Target*> > Targets;
   size_t Width = 0;
-  for (TargetRegistry::iterator I = TargetRegistry::begin(),
-       E = TargetRegistry::end();
-       I != E; ++I) {
-    Targets.push_back(std::make_pair(I->getName(), &*I));
+  for (const auto &T : TargetRegistry::targets()) {
+    Targets.push_back(std::make_pair(T.getName(), &T));
     Width = std::max(Width, Targets.back().first.size());
   }
   array_pod_sort(Targets.begin(), Targets.end(), TargetArraySortFn);
diff --git a/lib/Support/Triple.cpp b/lib/Support/Triple.cpp
index 034638c8c5b..bb3bb130619 100644
--- a/lib/Support/Triple.cpp
+++ b/lib/Support/Triple.cpp
@@ -12,6 +12,8 @@
 #include "llvm/ADT/SmallString.h"
 #include "llvm/ADT/StringSwitch.h"
 #include "llvm/Support/ErrorHandling.h"
+#include "llvm/Support/TargetParser.h"
+#include "llvm/Support/Host.h"
 #include <cstring>
 using namespace llvm;
 
@@ -23,7 +25,8 @@ const char *Triple::getArchTypeName(ArchType Kind) {
   case aarch64_be:  return "aarch64_be";
   case arm:         return "arm";
   case armeb:       return "armeb";
-  case bpf:         return "bpf";
+  case bpfel:       return "bpfel";
+  case bpfeb:       return "bpfeb";
   case hexagon:     return "hexagon";
   case mips:        return "mips";
   case mipsel:      return "mipsel";
@@ -37,6 +40,7 @@ const char *Triple::getArchTypeName(ArchType Kind) {
   case amdgcn:      return "amdgcn";
   case sparc:       return "sparc";
   case sparcv9:     return "sparcv9";
+  case sparcel:     return "sparcel";
   case systemz:     return "s390x";
   case tce:         return "tce";
   case thumb:       return "thumb";
@@ -56,6 +60,9 @@ const char *Triple::getArchTypeName(ArchType Kind) {
   case spir:        return "spir";
   case spir64:      return "spir64";
   case kalimba:     return "kalimba";
+  case shave:       return "shave";
+  case wasm32:      return "wasm32";
+  case wasm64:      return "wasm64";
   }
 
   llvm_unreachable("Invalid ArchType!");
@@ -88,9 +95,11 @@ const char *Triple::getArchTypePrefix(ArchType Kind) {
   case amdgcn:
   case r600:        return "amdgpu";
 
-  case bpf:         return "bpf";
+  case bpfel:
+  case bpfeb:       return "bpf";
 
   case sparcv9:
+  case sparcel:
   case sparc:       return "sparc";
 
   case systemz:     return "s390";
@@ -117,6 +126,9 @@ const char *Triple::getArchTypePrefix(ArchType Kind) {
   case spir:
   case spir64:      return "spir";
   case kalimba:     return "kalimba";
+  case shave:       return "shave";
+  case wasm32:      return "wasm32";
+  case wasm64:      return "wasm64";
   }
 }
 
@@ -188,19 +200,36 @@ const char *Triple::getEnvironmentTypeName(EnvironmentType Kind) {
   case MSVC: return "msvc";
   case Itanium: return "itanium";
   case Cygnus: return "cygnus";
+  case AMDOpenCL: return "amdopencl";
   }
 
   llvm_unreachable("Invalid EnvironmentType!");
 }
 
+static Triple::ArchType parseBPFArch(StringRef ArchName) {
+  if (ArchName.equals("bpf")) {
+    if (sys::IsLittleEndianHost)
+      return Triple::bpfel;
+    else
+      return Triple::bpfeb;
+  } else if (ArchName.equals("bpf_be") || ArchName.equals("bpfeb")) {
+    return Triple::bpfeb;
+  } else if (ArchName.equals("bpf_le") || ArchName.equals("bpfel")) {
+    return Triple::bpfel;
+  } else {
+    return Triple::UnknownArch;
+  }
+}
+
 Triple::ArchType Triple::getArchTypeForLLVMName(StringRef Name) {
+  Triple::ArchType BPFArch(parseBPFArch(Name));
   return StringSwitch<Triple::ArchType>(Name)
     .Case("aarch64", aarch64)
     .Case("aarch64_be", aarch64_be)
     .Case("arm64", aarch64) // "arm64" is an alias for "aarch64"
     .Case("arm", arm)
     .Case("armeb", armeb)
-    .Case("bpf", bpf)
+    .StartsWith("bpf", BPFArch)
     .Case("mips", mips)
     .Case("mipsel", mipsel)
     .Case("mips64", mips64)
@@ -214,6 +243,7 @@ Triple::ArchType Triple::getArchTypeForLLVMName(StringRef Name) {
     .Case("amdgcn", amdgcn)
     .Case("hexagon", hexagon)
     .Case("sparc", sparc)
+    .Case("sparcel", sparcel)
     .Case("sparcv9", sparcv9)
     .Case("systemz", systemz)
     .Case("tce", tce)
@@ -234,64 +264,73 @@ Triple::ArchType Triple::getArchTypeForLLVMName(StringRef Name) {
     .Case("spir", spir)
     .Case("spir64", spir64)
     .Case("kalimba", kalimba)
+    .Case("shave", shave)
+    .Case("wasm32", wasm32)
+    .Case("wasm64", wasm64)
     .Default(UnknownArch);
 }
 
 static Triple::ArchType parseARMArch(StringRef ArchName) {
-  size_t offset = StringRef::npos;
+  unsigned ISA = ARMTargetParser::parseArchISA(ArchName);
+  unsigned ENDIAN = ARMTargetParser::parseArchEndian(ArchName);
+
   Triple::ArchType arch = Triple::UnknownArch;
-  bool isThumb = ArchName.startswith("thumb");
-
-  if (ArchName.equals("arm"))
-    return Triple::arm;
-  if (ArchName.equals("armeb"))
-    return Triple::armeb;
-  if (ArchName.equals("thumb"))
-    return Triple::thumb;
-  if (ArchName.equals("thumbeb"))
-    return Triple::thumbeb;
-  if (ArchName.equals("arm64") || ArchName.equals("aarch64"))
-    return Triple::aarch64;
-  if (ArchName.equals("aarch64_be"))
-    return Triple::aarch64_be;
-
-  if (ArchName.startswith("armv")) {
-    offset = 3;
-    if (ArchName.endswith("eb")) {
-      arch = Triple::armeb;
-      ArchName = ArchName.substr(0, ArchName.size() - 2);
-    } else
+  switch (ENDIAN) {
+  case ARM::EK_LITTLE: {
+    switch (ISA) {
+    case ARM::IK_ARM:
       arch = Triple::arm;
-  } else if (ArchName.startswith("armebv")) {
-    offset = 5;
-    arch = Triple::armeb;
-  } else if (ArchName.startswith("thumbv")) {
-    offset = 5;
-    if (ArchName.endswith("eb")) {
-      arch = Triple::thumbeb;
-      ArchName = ArchName.substr(0, ArchName.size() - 2);
-    } else
+      break;
+    case ARM::IK_THUMB:
       arch = Triple::thumb;
-  } else if (ArchName.startswith("thumbebv")) {
-    offset = 7;
-    arch = Triple::thumbeb;
+      break;
+    case ARM::IK_AARCH64:
+      arch = Triple::aarch64;
+      break;
+    }
+    break;
   }
-  return StringSwitch<Triple::ArchType>(ArchName.substr(offset))
-    .Cases("v2", "v2a", isThumb ? Triple::UnknownArch : arch)
-    .Cases("v3", "v3m", isThumb ? Triple::UnknownArch : arch)
-    .Cases("v4", "v4t", arch)
-    .Cases("v5", "v5e", "v5t", "v5te", "v5tej", arch)
-    .Cases("v6", "v6j", "v6k", "v6m", "v6sm", arch)
-    .Cases("v6t2", "v6z", "v6zk", arch)
-    .Cases("v7", "v7a", "v7em", "v7l", arch)
-    .Cases("v7m", "v7r", "v7s", arch)
-    .Cases("v8", "v8a", arch)
-    .Cases("v8.1", "v8.1a", arch)
-    .Default(Triple::UnknownArch);
+  case ARM::EK_BIG: {
+    switch (ISA) {
+    case ARM::IK_ARM:
+      arch = Triple::armeb;
+      break;
+    case ARM::IK_THUMB:
+      arch = Triple::thumbeb;
+      break;
+    case ARM::IK_AARCH64:
+      arch = Triple::aarch64_be;
+      break;
+    }
+    break;
+  }
+  }
+
+  ArchName = ARMTargetParser::getCanonicalArchName(ArchName);
+  if (ArchName.empty())
+    return Triple::UnknownArch;
+
+  // Thumb only exists in v4+
+  if (ISA == ARM::IK_THUMB &&
+      (ArchName.startswith("v2") || ArchName.startswith("v3")))
+    return Triple::UnknownArch;
+
+  // Thumb only for v6m
+  unsigned Profile = ARMTargetParser::parseArchProfile(ArchName);
+  unsigned Version = ARMTargetParser::parseArchVersion(ArchName);
+  if (Profile == ARM::PK_M && Version == 6) {
+    if (ENDIAN == ARM::EK_BIG)
+      return Triple::thumbeb;
+    else
+      return Triple::thumb;
+  }
+
+  return arch;
 }
 
 static Triple::ArchType parseArch(StringRef ArchName) {
   Triple::ArchType ARMArch(parseARMArch(ArchName));
+  Triple::ArchType BPFArch(parseBPFArch(ArchName));
 
   return StringSwitch<Triple::ArchType>(ArchName)
     .Cases("i386", "i486", "i586", "i686", Triple::x86)
@@ -313,10 +352,11 @@ static Triple::ArchType parseArch(StringRef ArchName) {
     .Case("mips64el", Triple::mips64el)
     .Case("r600", Triple::r600)
     .Case("amdgcn", Triple::amdgcn)
-    .Case("bpf", Triple::bpf)
+    .StartsWith("bpf", BPFArch)
     .Case("hexagon", Triple::hexagon)
     .Case("s390x", Triple::systemz)
     .Case("sparc", Triple::sparc)
+    .Case("sparcel", Triple::sparcel)
     .Cases("sparcv9", "sparc64", Triple::sparcv9)
     .Case("tce", Triple::tce)
     .Case("xcore", Triple::xcore)
@@ -332,6 +372,9 @@ static Triple::ArchType parseArch(StringRef ArchName) {
     .Case("spir", Triple::spir)
     .Case("spir64", Triple::spir64)
     .StartsWith("kalimba", Triple::kalimba)
+    .Case("shave", Triple::shave)
+    .Case("wasm32", Triple::wasm32)
+    .Case("wasm64", Triple::wasm64)
     .Default(Triple::UnknownArch);
 }
 
@@ -395,6 +438,7 @@ static Triple::EnvironmentType parseEnvironment(StringRef EnvironmentName) {
     .StartsWith("msvc", Triple::MSVC)
     .StartsWith("itanium", Triple::Itanium)
     .StartsWith("cygnus", Triple::Cygnus)
+    .StartsWith("amdopencl", Triple::AMDOpenCL)
     .Default(Triple::UnknownEnvironment);
 }
 
@@ -407,34 +451,64 @@ static Triple::ObjectFormatType parseFormat(StringRef EnvironmentName) {
 }
 
 static Triple::SubArchType parseSubArch(StringRef SubArchName) {
-  if (SubArchName.endswith("eb"))
-    SubArchName = SubArchName.substr(0, SubArchName.size() - 2);
-
-  return StringSwitch<Triple::SubArchType>(SubArchName)
-    .EndsWith("v8.1a", Triple::ARMSubArch_v8_1a)
-    .EndsWith("v8", Triple::ARMSubArch_v8)
-    .EndsWith("v8a", Triple::ARMSubArch_v8)
-    .EndsWith("v7", Triple::ARMSubArch_v7)
-    .EndsWith("v7a", Triple::ARMSubArch_v7)
-    .EndsWith("v7em", Triple::ARMSubArch_v7em)
-    .EndsWith("v7l", Triple::ARMSubArch_v7)
-    .EndsWith("v7m", Triple::ARMSubArch_v7m)
-    .EndsWith("v7r", Triple::ARMSubArch_v7)
-    .EndsWith("v7s", Triple::ARMSubArch_v7s)
-    .EndsWith("v6", Triple::ARMSubArch_v6)
-    .EndsWith("v6m", Triple::ARMSubArch_v6m)
-    .EndsWith("v6sm", Triple::ARMSubArch_v6m)
-    .EndsWith("v6k", Triple::ARMSubArch_v6k)
-    .EndsWith("v6t2", Triple::ARMSubArch_v6t2)
-    .EndsWith("v5", Triple::ARMSubArch_v5)
-    .EndsWith("v5e", Triple::ARMSubArch_v5)
-    .EndsWith("v5t", Triple::ARMSubArch_v5)
-    .EndsWith("v5te", Triple::ARMSubArch_v5te)
-    .EndsWith("v4t", Triple::ARMSubArch_v4t)
-    .EndsWith("kalimba3", Triple::KalimbaSubArch_v3)
-    .EndsWith("kalimba4", Triple::KalimbaSubArch_v4)
-    .EndsWith("kalimba5", Triple::KalimbaSubArch_v5)
-    .Default(Triple::NoSubArch);
+  StringRef ARMSubArch = ARMTargetParser::getCanonicalArchName(SubArchName);
+
+  // For now, this is the small part. Early return.
+  if (ARMSubArch.empty())
+    return StringSwitch<Triple::SubArchType>(SubArchName)
+      .EndsWith("kalimba3", Triple::KalimbaSubArch_v3)
+      .EndsWith("kalimba4", Triple::KalimbaSubArch_v4)
+      .EndsWith("kalimba5", Triple::KalimbaSubArch_v5)
+      .Default(Triple::NoSubArch);
+
+  // ARM sub arch.
+  switch(ARMTargetParser::parseArch(ARMSubArch)) {
+  case ARM::AK_ARMV4:
+    return Triple::NoSubArch;
+  case ARM::AK_ARMV4T:
+    return Triple::ARMSubArch_v4t;
+  case ARM::AK_ARMV5:
+  case ARM::AK_ARMV5T:
+  case ARM::AK_ARMV5E:
+    return Triple::ARMSubArch_v5;
+  case ARM::AK_ARMV5TE:
+  case ARM::AK_IWMMXT:
+  case ARM::AK_IWMMXT2:
+  case ARM::AK_XSCALE:
+  case ARM::AK_ARMV5TEJ:
+    return Triple::ARMSubArch_v5te;
+  case ARM::AK_ARMV6:
+  case ARM::AK_ARMV6J:
+  case ARM::AK_ARMV6Z:
+    return Triple::ARMSubArch_v6;
+  case ARM::AK_ARMV6K:
+  case ARM::AK_ARMV6ZK:
+  case ARM::AK_ARMV6HL:
+    return Triple::ARMSubArch_v6k;
+  case ARM::AK_ARMV6T2:
+    return Triple::ARMSubArch_v6t2;
+  case ARM::AK_ARMV6M:
+  case ARM::AK_ARMV6SM:
+    return Triple::ARMSubArch_v6m;
+  case ARM::AK_ARMV7:
+  case ARM::AK_ARMV7A:
+  case ARM::AK_ARMV7R:
+  case ARM::AK_ARMV7L:
+  case ARM::AK_ARMV7HL:
+    return Triple::ARMSubArch_v7;
+  case ARM::AK_ARMV7M:
+    return Triple::ARMSubArch_v7m;
+  case ARM::AK_ARMV7S:
+    return Triple::ARMSubArch_v7s;
+  case ARM::AK_ARMV7EM:
+    return Triple::ARMSubArch_v7em;
+  case ARM::AK_ARMV8A:
+    return Triple::ARMSubArch_v8;
+  case ARM::AK_ARMV8_1A:
+    return Triple::ARMSubArch_v8_1a;
+  default:
+    return Triple::NoSubArch;
+  }
 }
 
 static const char *getObjectFormatTypeName(Triple::ObjectFormatType Kind) {
@@ -669,6 +743,16 @@ std::string Triple::normalize(StringRef Str) {
 
   // Special case logic goes here.  At this point Arch, Vendor and OS have the
   // correct values for the computed components.
+  std::string NormalizedEnvironment;
+  if (Environment == Triple::Android && Components[3].startswith("androideabi")) {
+    StringRef AndroidVersion = Components[3].drop_front(strlen("androideabi"));
+    if (AndroidVersion.empty()) {
+      Components[3] = "android";
+    } else {
+      NormalizedEnvironment = Twine("android", AndroidVersion).str();
+      Components[3] = NormalizedEnvironment;
+    }
+  }
 
   if (OS == Triple::Win32) {
     Components.resize(4);
@@ -746,41 +830,47 @@ static unsigned EatNumber(StringRef &Str) {
   return Result;
 }
 
-void Triple::getOSVersion(unsigned &Major, unsigned &Minor,
-                          unsigned &Micro) const {
-  StringRef OSName = getOSName();
-
-  // For Android, we care about the Android version rather than the Linux
-  // version.
-  if (getEnvironment() == Android) {
-    OSName = getEnvironmentName().substr(strlen("android"));
-    if (OSName.startswith("eabi"))
-      OSName = OSName.substr(strlen("eabi"));
-  }
-
-  // Assume that the OS portion of the triple starts with the canonical name.
-  StringRef OSTypeName = getOSTypeName(getOS());
-  if (OSName.startswith(OSTypeName))
-    OSName = OSName.substr(OSTypeName.size());
-
+static void parseVersionFromName(StringRef Name, unsigned &Major,
+                                 unsigned &Minor, unsigned &Micro) {
   // Any unset version defaults to 0.
   Major = Minor = Micro = 0;
 
   // Parse up to three components.
-  unsigned *Components[3] = { &Major, &Minor, &Micro };
+  unsigned *Components[3] = {&Major, &Minor, &Micro};
   for (unsigned i = 0; i != 3; ++i) {
-    if (OSName.empty() || OSName[0] < '0' || OSName[0] > '9')
+    if (Name.empty() || Name[0] < '0' || Name[0] > '9')
       break;
 
     // Consume the leading number.
-    *Components[i] = EatNumber(OSName);
+    *Components[i] = EatNumber(Name);
 
     // Consume the separator, if present.
-    if (OSName.startswith("."))
-      OSName = OSName.substr(1);
+    if (Name.startswith("."))
+      Name = Name.substr(1);
   }
 }
 
+void Triple::getEnvironmentVersion(unsigned &Major, unsigned &Minor,
+                                   unsigned &Micro) const {
+  StringRef EnvironmentName = getEnvironmentName();
+  StringRef EnvironmentTypeName = getEnvironmentTypeName(getEnvironment());
+  if (EnvironmentName.startswith(EnvironmentTypeName))
+    EnvironmentName = EnvironmentName.substr(EnvironmentTypeName.size());
+
+  parseVersionFromName(EnvironmentName, Major, Minor, Micro);
+}
+
+void Triple::getOSVersion(unsigned &Major, unsigned &Minor,
+                          unsigned &Micro) const {
+  StringRef OSName = getOSName();
+  // Assume that the OS portion of the triple starts with the canonical name.
+  StringRef OSTypeName = getOSTypeName(getOS());
+  if (OSName.startswith(OSTypeName))
+    OSName = OSName.substr(OSTypeName.size());
+
+  parseVersionFromName(OSName, Major, Minor, Micro);
+}
+
 bool Triple::getMacOSXVersion(unsigned &Major, unsigned &Minor,
                               unsigned &Micro) const {
   getOSVersion(Major, Minor, Micro);
@@ -926,6 +1016,7 @@ static unsigned getArchPointerBitWidth(llvm::Triple::ArchType Arch) {
   case llvm::Triple::ppc:
   case llvm::Triple::r600:
   case llvm::Triple::sparc:
+  case llvm::Triple::sparcel:
   case llvm::Triple::tce:
   case llvm::Triple::thumb:
   case llvm::Triple::thumbeb:
@@ -935,12 +1026,15 @@ static unsigned getArchPointerBitWidth(llvm::Triple::ArchType Arch) {
   case llvm::Triple::hsail:
   case llvm::Triple::spir:
   case llvm::Triple::kalimba:
+  case llvm::Triple::shave:
+  case llvm::Triple::wasm32:
     return 32;
 
   case llvm::Triple::aarch64:
   case llvm::Triple::aarch64_be:
   case llvm::Triple::amdgcn:
-  case llvm::Triple::bpf:
+  case llvm::Triple::bpfel:
+  case llvm::Triple::bpfeb:
   case llvm::Triple::le64:
   case llvm::Triple::mips64:
   case llvm::Triple::mips64el:
@@ -953,6 +1047,7 @@ static unsigned getArchPointerBitWidth(llvm::Triple::ArchType Arch) {
   case llvm::Triple::amdil64:
   case llvm::Triple::hsail64:
   case llvm::Triple::spir64:
+  case llvm::Triple::wasm64:
     return 64;
   }
   llvm_unreachable("Invalid architecture value");
@@ -977,7 +1072,8 @@ Triple Triple::get32BitArchVariant() const {
   case Triple::aarch64:
   case Triple::aarch64_be:
   case Triple::amdgcn:
-  case Triple::bpf:
+  case Triple::bpfel:
+  case Triple::bpfeb:
   case Triple::msp430:
   case Triple::systemz:
   case Triple::ppc64le:
@@ -999,11 +1095,14 @@ Triple Triple::get32BitArchVariant() const {
   case Triple::ppc:
   case Triple::r600:
   case Triple::sparc:
+  case Triple::sparcel:
   case Triple::tce:
   case Triple::thumb:
   case Triple::thumbeb:
   case Triple::x86:
   case Triple::xcore:
+  case Triple::shave:
+  case Triple::wasm32:
     // Already 32-bit.
     break;
 
@@ -1017,6 +1116,7 @@ Triple Triple::get32BitArchVariant() const {
   case Triple::amdil64:   T.setArch(Triple::amdil);   break;
   case Triple::hsail64:   T.setArch(Triple::hsail);   break;
   case Triple::spir64:    T.setArch(Triple::spir);    break;
+  case Triple::wasm64:    T.setArch(Triple::wasm32);  break;
   }
   return T;
 }
@@ -1036,12 +1136,15 @@ Triple Triple::get64BitArchVariant() const {
   case Triple::thumbeb:
   case Triple::xcore:
   case Triple::asmjs: // @LOCALMOD Emscripten
+  case Triple::sparcel:
+  case Triple::shave:
     T.setArch(UnknownArch);
     break;
 
   case Triple::aarch64:
   case Triple::aarch64_be:
-  case Triple::bpf:
+  case Triple::bpfel:
+  case Triple::bpfeb:
   case Triple::le64:
   case Triple::amdil64:
   case Triple::amdgcn:
@@ -1055,6 +1158,7 @@ Triple Triple::get64BitArchVariant() const {
   case Triple::sparcv9:
   case Triple::systemz:
   case Triple::x86_64:
+  case Triple::wasm64:
     // Already 64-bit.
     break;
 
@@ -1068,19 +1172,139 @@ Triple Triple::get64BitArchVariant() const {
   case Triple::amdil:   T.setArch(Triple::amdil64);   break;
   case Triple::hsail:   T.setArch(Triple::hsail64);   break;
   case Triple::spir:    T.setArch(Triple::spir64);    break;
+  case Triple::wasm32:  T.setArch(Triple::wasm64);    break;
+  }
+  return T;
+}
+
+Triple Triple::getBigEndianArchVariant() const {
+  Triple T(*this);
+  switch (getArch()) {
+  case Triple::UnknownArch:
+  case Triple::amdgcn:
+  case Triple::amdil64:
+  case Triple::amdil:
+  case Triple::asmjs:
+  case Triple::hexagon:
+  case Triple::hsail64:
+  case Triple::hsail:
+  case Triple::kalimba:
+  case Triple::le32:
+  case Triple::le64:
+  case Triple::msp430:
+  case Triple::nvptx64:
+  case Triple::nvptx:
+  case Triple::r600:
+  case Triple::shave:
+  case Triple::spir64:
+  case Triple::spir:
+  case Triple::wasm32:
+  case Triple::wasm64:
+  case Triple::x86:
+  case Triple::x86_64:
+  case Triple::xcore:
+
+  // ARM is intentionally unsupported here, changing the architecture would
+  // drop any arch suffixes.
+  case Triple::arm:
+  case Triple::thumb:
+    T.setArch(UnknownArch);
+    break;
+
+  case Triple::aarch64_be:
+  case Triple::armeb:
+  case Triple::bpfeb:
+  case Triple::mips64:
+  case Triple::mips:
+  case Triple::ppc64:
+  case Triple::ppc:
+  case Triple::sparc:
+  case Triple::sparcv9:
+  case Triple::systemz:
+  case Triple::tce:
+  case Triple::thumbeb:
+    // Already big endian.
+    break;
+
+  case Triple::aarch64: T.setArch(Triple::aarch64_be); break;
+  case Triple::bpfel:   T.setArch(Triple::bpfeb);      break;
+  case Triple::mips64el:T.setArch(Triple::mips64);     break;
+  case Triple::mipsel:  T.setArch(Triple::mips);       break;
+  case Triple::ppc64le: T.setArch(Triple::ppc64);      break;
+  case Triple::sparcel: T.setArch(Triple::sparc);      break;
+  }
+  return T;
+}
+
+Triple Triple::getLittleEndianArchVariant() const {
+  Triple T(*this);
+  switch (getArch()) {
+  case Triple::UnknownArch:
+  case Triple::ppc:
+  case Triple::sparcv9:
+  case Triple::systemz:
+  case Triple::tce:
+
+  // ARM is intentionally unsupported here, changing the architecture would
+  // drop any arch suffixes.
+  case Triple::armeb:
+  case Triple::thumbeb:
+    T.setArch(UnknownArch);
+    break;
+
+  case Triple::aarch64:
+  case Triple::amdgcn:
+  case Triple::amdil64:
+  case Triple::amdil:
+  case Triple::arm:
+  case Triple::asmjs:
+  case Triple::bpfel:
+  case Triple::hexagon:
+  case Triple::hsail64:
+  case Triple::hsail:
+  case Triple::kalimba:
+  case Triple::le32:
+  case Triple::le64:
+  case Triple::mips64el:
+  case Triple::mipsel:
+  case Triple::msp430:
+  case Triple::nvptx64:
+  case Triple::nvptx:
+  case Triple::ppc64le:
+  case Triple::r600:
+  case Triple::shave:
+  case Triple::sparcel:
+  case Triple::spir64:
+  case Triple::spir:
+  case Triple::thumb:
+  case Triple::wasm32:
+  case Triple::wasm64:
+  case Triple::x86:
+  case Triple::x86_64:
+  case Triple::xcore:
+    // Already little endian.
+    break;
+
+  case Triple::aarch64_be: T.setArch(Triple::aarch64);  break;
+  case Triple::bpfeb:      T.setArch(Triple::bpfel);    break;
+  case Triple::mips64:     T.setArch(Triple::mips64el); break;
+  case Triple::mips:       T.setArch(Triple::mipsel);   break;
+  case Triple::ppc64:      T.setArch(Triple::ppc64le);  break;
+  case Triple::sparc:      T.setArch(Triple::sparcel);  break;
   }
   return T;
 }
 
-// FIXME: tblgen this.
 const char *Triple::getARMCPUForArch(StringRef MArch) const {
   if (MArch.empty())
     MArch = getArchName();
+  MArch = ARMTargetParser::getCanonicalArchName(MArch);
 
+  // Some defaults are forced.
   switch (getOS()) {
   case llvm::Triple::FreeBSD:
   case llvm::Triple::NetBSD:
-    if (MArch == "armv6")
+    if (!MArch.empty() && MArch == "v6")
       return "arm1176jzf-s";
     break;
   case llvm::Triple::Win32:
@@ -1090,52 +1314,15 @@ const char *Triple::getARMCPUForArch(StringRef MArch) const {
     break;
   }
 
-  const char *result = nullptr;
-  size_t offset = StringRef::npos;
-  if (MArch.startswith("arm"))
-    offset = 3;
-  if (MArch.startswith("thumb"))
-    offset = 5;
-  if (offset != StringRef::npos && MArch.substr(offset, 2) == "eb")
-    offset += 2;
-  if (MArch.endswith("eb"))
-    MArch = MArch.substr(0, MArch.size() - 2);
-  if (offset != StringRef::npos)
-    result = llvm::StringSwitch<const char *>(MArch.substr(offset))
-      .Cases("v2", "v2a", "arm2")
-      .Case("v3", "arm6")
-      .Case("v3m", "arm7m")
-      .Case("v4", "strongarm")
-      .Case("v4t", "arm7tdmi")
-      .Cases("v5", "v5t", "arm10tdmi")
-      .Cases("v5e", "v5te", "arm1022e")
-      .Case("v5tej", "arm926ej-s")
-      .Case("v6", "arm1136jf-s")
-      .Case("v6j", "arm1136j-s")
-      .Cases("v6k", "v6z", "v6zk", "arm1176jzf-s")
-      .Case("v6t2", "arm1156t2-s")
-      .Cases("v6m", "v6-m", "v6sm", "v6s-m", "cortex-m0")
-      .Cases("v7", "v7a", "v7-a", "v7l", "v7-l", "cortex-a8")
-      .Cases("v7s", "v7-s", "swift")
-      .Cases("v7r", "v7-r", "cortex-r4")
-      .Cases("v7m", "v7-m", "cortex-m3")
-      .Cases("v7em", "v7e-m", "cortex-m4")
-      .Cases("v8", "v8a", "v8-a", "cortex-a53")
-      .Cases("v8.1a", "v8.1-a", "generic")
-      .Default(nullptr);
-  else
-    result = llvm::StringSwitch<const char *>(MArch)
-      .Case("ep9312", "ep9312")
-      .Case("iwmmxt", "iwmmxt")
-      .Case("xscale", "xscale")
-      .Default(nullptr);
-
-  if (result)
-    return result;
-
-  // If all else failed, return the most base CPU with thumb interworking
-  // supported by LLVM.
-  // FIXME: Should warn once that we're falling back.
+  if (MArch.empty())
+    return nullptr;
+
+  const char *CPU = ARMTargetParser::getDefaultCPU(MArch);
+  if (CPU)
+    return CPU;
+
+  // If no specific architecture version is requested, return the minimum CPU
+  // required by the OS and environment.
   switch (getOS()) {
   case llvm::Triple::NetBSD:
     switch (getEnvironment()) {
@@ -1158,4 +1345,6 @@ const char *Triple::getARMCPUForArch(StringRef MArch) const {
       return "arm7tdmi";
     }
   }
+
+  llvm_unreachable("invalid arch name");
 }
diff --git a/lib/Support/Twine.cpp b/lib/Support/Twine.cpp
index d2cc75b8c60..020dd9596d9 100644
--- a/lib/Support/Twine.cpp
+++ b/lib/Support/Twine.cpp
@@ -118,10 +118,9 @@ void Twine::printOneChildRepr(raw_ostream &OS, Child Ptr,
     OS << "stringref:\""
        << Ptr.stringRef << "\"";
     break;
-  case Twine::SmallStringKind:
-    OS << "smallstring:\""
-       << *Ptr.smallString << "\"";
-    break;
+  case Twine::SmallStringKind:
+    OS << "smallstring:\"" << *Ptr.smallString << "\"";
+    break;
   case Twine::CharKind:
     OS << "char:\"" << Ptr.character << "\"";
     break;
diff --git a/lib/Support/Unix/COM.inc b/lib/Support/Unix/COM.inc
new file mode 100644
index 00000000000..5b71de74ebf
--- /dev/null
+++ b/lib/Support/Unix/COM.inc
@@ -0,0 +1,27 @@
+//===- llvm/Support/Unix/COM.inc - Unix COM Implementation -----*- C++ -*-===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This file implements the Unix portion of COM support.
+//
+//===----------------------------------------------------------------------===//
+
+//===----------------------------------------------------------------------===//
+//=== WARNING: Implementation here must contain only generic UNIX code that
+//===          is guaranteed to work on *all* UNIX variants.
+//===----------------------------------------------------------------------===//
+
+namespace llvm {
+namespace sys {
+
+InitializeCOMRAII::InitializeCOMRAII(COMThreadingMode Threading,
+                                     bool SpeedOverMemory) {}
+
+InitializeCOMRAII::~InitializeCOMRAII() {}
+}
+}
diff --git a/lib/Support/Unix/Host.inc b/lib/Support/Unix/Host.inc
index a7ba87196c1..457217125a2 100644
--- a/lib/Support/Unix/Host.inc
+++ b/lib/Support/Unix/Host.inc
@@ -21,23 +21,17 @@
 #include "llvm/Config/config.h"
 #include <cctype>
 #include <string>
-#if !defined(__native_client__)
 #include <sys/utsname.h>
-#endif
 
 using namespace llvm;
 
 static std::string getOSVersion() {
-#if !defined(__native_client__)
   struct utsname info;
 
   if (uname(&info))
     return "";
 
   return info.release;
-#else // (__native_client__)
-  return "";
-#endif // (__native_client__)
 }
 
 std::string sys::getDefaultTargetTriple() {
diff --git a/lib/Support/Unix/Memory.inc b/lib/Support/Unix/Memory.inc
index 2760d02bd17..c421ee84c2b 100644
--- a/lib/Support/Unix/Memory.inc
+++ b/lib/Support/Unix/Memory.inc
@@ -333,10 +333,6 @@ void Memory::InvalidateInstructionCache(const void *Addr,
   for (intptr_t Line = StartLine; Line < EndLine; Line += LineSize)
     asm volatile("icbi 0, %0" : : "r"(Line));
   asm volatile("isync");
-  // @LOCALMOD-START
-#  elif defined(__native_client__)
-  exit(1); // Native Client disallows JIT-compilation.
-  // @LOCALMOD-END
 #  elif (defined(__arm__) || defined(__aarch64__) || defined(__mips__)) && \
         defined(__GNUC__)
   // FIXME: Can we safely always call this for __GNUC__ everywhere?
diff --git a/lib/Support/Unix/Path.inc b/lib/Support/Unix/Path.inc
index d6af2f8ba8f..973d010dcac 100644
--- a/lib/Support/Unix/Path.inc
+++ b/lib/Support/Unix/Path.inc
@@ -168,10 +168,6 @@ std::string getMainExecutable(const char *argv0, void *MainAddr) {
   char link_path[MAXPATHLEN];
   if (realpath(DLInfo.dli_fname, link_path))
     return link_path;
-  // @LOCALMOD-BEGIN
-#elif defined(__native_client__)
-  // Nothing.
-  // @LOCALMOD-END
 #else
 #error GetMainExecutable is not implemented on this host yet.
 #endif
@@ -190,16 +186,7 @@ UniqueID file_status::getUniqueID() const {
 
 std::error_code current_path(SmallVectorImpl<char> &result) {
   result.clear();
-  // @LOCALMOD-START NaCl doesn't have paths, and the translator returns an
-  //                 error for getcwd below. Return a dummy path instead.
-#if defined(__native_client__)
-  result.reserve(2);
-  result.set_size(2);
-  result[0] = '/';
-  result[1] = '\0';
-  return std::error_code();
-#else // !defined(__native_client__)
-  // @LOCALMOD-END
+
   const char *pwd = ::getenv("PWD");
   llvm::sys::fs::file_status PWDStatus, DotStatus;
   if (pwd && llvm::sys::path::is_absolute(pwd) &&
@@ -230,9 +217,6 @@ std::error_code current_path(SmallVectorImpl<char> &result) {
 
   result.set_size(strlen(result.data()));
   return std::error_code();
-  // @LOCALMOD-START
-#endif // defined(__native_client__)
-  // @LOCALMOD-END
 }
 
 std::error_code create_directory(const Twine &path, bool IgnoreExisting) {
diff --git a/lib/Support/Unix/Process.inc b/lib/Support/Unix/Process.inc
index e92c2761f4f..df13bd22173 100644
--- a/lib/Support/Unix/Process.inc
+++ b/lib/Support/Unix/Process.inc
@@ -245,16 +245,6 @@ std::error_code Process::FixupStandardFileDescriptors() {
 }
 
 std::error_code Process::SafelyCloseFileDescriptor(int FD) {
-  // @LOCALMOD-BEGIN: For NaCl, don't use pthread_sigmask, etc.
-  // They are stubs and fail with ENOSYS, which make this return
-  // an error.
-#ifdef __native_client__
-  int ErrnoFromClose = 0;
-  if (::close(FD) < 0)
-    ErrnoFromClose = errno;
-  return std::error_code(ErrnoFromClose, std::generic_category());
-#else
-
   // Create a signal set filled with *all* signals.
   sigset_t FullSet;
   if (sigfillset(&FullSet) < 0)
@@ -287,7 +277,6 @@ std::error_code Process::SafelyCloseFileDescriptor(int FD) {
   if (ErrnoFromClose)
     return std::error_code(ErrnoFromClose, std::generic_category());
   return std::error_code(EC, std::generic_category());
-#endif  // @LOCALMOD-END
 }
 
 bool Process::StandardInIsUserInput() {
diff --git a/lib/Support/Unix/Program.inc b/lib/Support/Unix/Program.inc
index 16d4b6f12a5..8947b62e4dc 100644
--- a/lib/Support/Unix/Program.inc
+++ b/lib/Support/Unix/Program.inc
@@ -20,6 +20,7 @@
 #include "llvm/ADT/StringExtras.h"
 #include "llvm/Config/config.h"
 #include "llvm/Support/Compiler.h"
+#include "llvm/Support/Errc.h"
 #include "llvm/Support/FileSystem.h"
 #include "llvm/Support/Path.h"
 #include "llvm/Support/raw_ostream.h"
@@ -92,7 +93,7 @@ ErrorOr<std::string> sys::findProgramByName(StringRef Name,
     if (sys::fs::can_execute(FilePath.c_str()))
       return std::string(FilePath.str()); // Found the executable!
   }
-  return std::errc::no_such_file_or_directory;
+  return errc::no_such_file_or_directory;
 }
 
 static bool RedirectIO(const StringRef *Path, int FD, std::string* ErrMsg) {
@@ -322,9 +323,7 @@ namespace llvm {
 
 ProcessInfo sys::Wait(const ProcessInfo &PI, unsigned SecondsToWait,
                       bool WaitUntilTerminates, std::string *ErrMsg) {
-  // @LOCALMOD-BEGIN
-#if defined(HAVE_SYS_WAIT_H) && !defined(PNACL_BROWSER_TRANSLATOR)
-  // @LOCALMOD-END
+#ifdef HAVE_SYS_WAIT_H
   struct sigaction Act, Old;
   assert(PI.Pid && "invalid pid to wait on, process not started?");
 
@@ -449,17 +448,13 @@ llvm::sys::writeFileWithEncoding(StringRef FileName, StringRef Contents,
   OS << Contents;
 
   if (OS.has_error())
-    return std::make_error_code(std::errc::io_error);
+    return make_error_code(errc::io_error);
 
   return EC;
 }
 
 bool llvm::sys::argumentsFitWithinSystemLimits(ArrayRef<const char*> Args) {
-#if defined(__native_client__)
-  static long ArgMax = -1;
-#else // !__native_client__
   static long ArgMax = sysconf(_SC_ARG_MAX);
-#endif // __native_client__
 
   // System says no practical limit.
   if (ArgMax == -1)
diff --git a/lib/Support/Unix/Signals.inc b/lib/Support/Unix/Signals.inc
index a80b2ac1d27..94336305d84 100644
--- a/lib/Support/Unix/Signals.inc
+++ b/lib/Support/Unix/Signals.inc
@@ -17,7 +17,6 @@
 #include "llvm/Support/Format.h"
 #include "llvm/Support/FileSystem.h"
 #include "llvm/Support/FileUtilities.h"
-#include "llvm/Support/ManagedStatic.h"
 #include "llvm/Support/MemoryBuffer.h"
 #include "llvm/Support/Mutex.h"
 #include "llvm/Support/Program.h"
@@ -25,7 +24,6 @@
 #include "llvm/Support/raw_ostream.h"
 #include <algorithm>
 #include <string>
-#include <vector>
 #if HAVE_EXECINFO_H
 # include <execinfo.h>         // For backtrace().
 #endif
@@ -58,8 +56,6 @@ static ManagedStatic<SmartMutex<true> > SignalsMutex;
 static void (*InterruptFunction)() = nullptr;
 
 static ManagedStatic<std::vector<std::string>> FilesToRemove;
-static ManagedStatic<std::vector<std::pair<void (*)(void *), void *>>>
-    CallBacksToRun;
 
 // IntSigs - Signals that represent requested termination. There's no bug
 // or failure, or if there is, it's not our direct responsibility. For whatever
@@ -94,7 +90,6 @@ static struct {
 
 
 static void RegisterHandler(int Signal) {
-#if !defined(__native_client__)
   assert(NumRegisteredSignals <
          sizeof(RegisteredSignalInfo)/sizeof(RegisteredSignalInfo[0]) &&
          "Out of space for signal handlers!");
@@ -110,10 +105,15 @@ static void RegisterHandler(int Signal) {
             &RegisteredSignalInfo[NumRegisteredSignals].SA);
   RegisteredSignalInfo[NumRegisteredSignals].SigNo = Signal;
   ++NumRegisteredSignals;
-#endif // (__native_client__)
 }
 
 static void RegisterHandlers() {
+  // We need to dereference the signals mutex during handler registration so
+  // that we force its construction. This is to prevent the first use being
+  // during handling an actual signal because you can't safely call new in a
+  // signal handler.
+  *SignalsMutex;
+  
   // If the handlers are already registered, we're done.
   if (NumRegisteredSignals != 0) return;
 
@@ -122,13 +122,11 @@ static void RegisterHandlers() {
 }
 
 static void UnregisterHandlers() {
-#if !defined(__native_client__)
   // Restore all of the signal handlers to how they were before we showed up.
   for (unsigned i = 0, e = NumRegisteredSignals; i != e; ++i)
     sigaction(RegisteredSignalInfo[i].SigNo,
               &RegisteredSignalInfo[i].SA, nullptr);
   NumRegisteredSignals = 0;
-#endif // (__native_client__)
 }
 
 
@@ -137,13 +135,15 @@ static void UnregisterHandlers() {
 /// NB: This must be an async signal safe function. It cannot allocate or free
 /// memory, even in debug builds.
 static void RemoveFilesToRemove() {
+  // Avoid constructing ManagedStatic in the signal handler.
+  // If FilesToRemove is not constructed, there are no files to remove.
+  if (!FilesToRemove.isConstructed())
+    return;
+
   // We avoid iterators in case of debug iterators that allocate or release
   // memory.
   std::vector<std::string>& FilesToRemoveRef = *FilesToRemove;
   for (unsigned i = 0, e = FilesToRemoveRef.size(); i != e; ++i) {
-    // We rely on a std::string implementation for which repeated calls to
-    // 'c_str()' don't allocate memory. We pre-call 'c_str()' on all of these
-    // strings to try to ensure this is safe.
     const char *path = FilesToRemoveRef[i].c_str();
 
     // Get the status so we can determine if it's a file or directory. If we
@@ -172,12 +172,10 @@ static RETSIGTYPE SignalHandler(int Sig) {
   // instead of recursing in the signal handler.
   UnregisterHandlers();
 
-#if !defined(__native_client__)
   // Unmask all potentially blocked kill signals.
   sigset_t SigMask;
   sigfillset(&SigMask);
   sigprocmask(SIG_UNBLOCK, &SigMask, nullptr);
-#endif
 
   {
     unique_lock<SmartMutex<true>> Guard(*SignalsMutex);
@@ -200,10 +198,7 @@ static RETSIGTYPE SignalHandler(int Sig) {
   }
 
   // Otherwise if it is a fault (like SEGV) run any handler.
-  std::vector<std::pair<void (*)(void *), void *>>& CallBacksToRunRef =
-      *CallBacksToRun;
-  for (unsigned i = 0, e = CallBacksToRun->size(); i != e; ++i)
-    CallBacksToRunRef[i].first(CallBacksToRunRef[i].second);
+  llvm::sys::RunSignalHandlers();
 
 #ifdef __s390__
   // On S/390, certain signals are delivered with PSW Address pointing to
@@ -233,21 +228,7 @@ bool llvm::sys::RemoveFileOnSignal(StringRef Filename,
                                    std::string* ErrMsg) {
   {
     sys::SmartScopedLock<true> Guard(*SignalsMutex);
-    std::vector<std::string>& FilesToRemoveRef = *FilesToRemove;
-    std::string *OldPtr =
-        FilesToRemoveRef.empty() ? nullptr : &FilesToRemoveRef[0];
-    FilesToRemoveRef.push_back(Filename);
-
-    // We want to call 'c_str()' on every std::string in this vector so that if
-    // the underlying implementation requires a re-allocation, it happens here
-    // rather than inside of the signal handler. If we see the vector grow, we
-    // have to call it on every entry. If it remains in place, we only need to
-    // call it on the latest one.
-    if (OldPtr == &FilesToRemoveRef[0])
-      FilesToRemoveRef.back().c_str();
-    else
-      for (unsigned i = 0, e = FilesToRemoveRef.size(); i != e; ++i)
-        FilesToRemoveRef[i].c_str();
+    FilesToRemove->push_back(Filename);
   }
 
   RegisterHandlers();
@@ -262,13 +243,6 @@ void llvm::sys::DontRemoveFileOnSignal(StringRef Filename) {
   std::vector<std::string>::iterator I = FilesToRemove->end();
   if (RI != FilesToRemove->rend())
     I = FilesToRemove->erase(RI.base()-1);
-
-  // We need to call c_str() on every element which would have been moved by
-  // the erase. These elements, in a C++98 implementation where c_str()
-  // requires a reallocation on the first call may have had the call to c_str()
-  // made on insertion become invalid by being copied down an element.
-  for (std::vector<std::string>::iterator E = FilesToRemove->end(); I != E; ++I)
-    I->c_str();
 }
 
 /// AddSignalHandler - Add a function to be called when a signal is delivered
diff --git a/lib/Support/Unix/TimeValue.inc b/lib/Support/Unix/TimeValue.inc
index 0b99eb74207..042e0dacc34 100644
--- a/lib/Support/Unix/TimeValue.inc
+++ b/lib/Support/Unix/TimeValue.inc
@@ -18,13 +18,6 @@
 
 #include "Unix.h"
 
-// @LOCALMOD-START
-#ifndef timerclear
-// Newlib does not have the timer{clear,add,sub} macros
-#define timerclear(tvp)  ((tvp)->tv_sec = (tvp)->tv_usec = 0)
-#endif
-// @LOCALMOD-END
-
 namespace llvm {
   using namespace sys;
 
diff --git a/lib/Support/Unix/Watchdog.inc b/lib/Support/Unix/Watchdog.inc
index 9133d01183a..5d89c0e51b1 100644
--- a/lib/Support/Unix/Watchdog.inc
+++ b/lib/Support/Unix/Watchdog.inc
@@ -19,17 +19,13 @@ namespace llvm {
   namespace sys {
     Watchdog::Watchdog(unsigned int seconds) {
 #ifdef HAVE_UNISTD_H
-  #ifndef __native_client__
       alarm(seconds);
-  #endif // __native_client__
 #endif
     }
 
     Watchdog::~Watchdog() {
 #ifdef HAVE_UNISTD_H
-  #ifndef __native_client__
       alarm(0);
-  #endif //__native_client__
 #endif
     }
   }
diff --git a/lib/Support/Windows/COM.inc b/lib/Support/Windows/COM.inc
new file mode 100644
index 00000000000..0c50d6f74ea
--- /dev/null
+++ b/lib/Support/Windows/COM.inc
@@ -0,0 +1,37 @@
+//===- llvm/Support/Windows/COM.inc - Windows COM Implementation *- C++ -*-===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This file implements the Windows portion of COM support.
+//
+//===----------------------------------------------------------------------===//
+
+//===----------------------------------------------------------------------===//
+//=== WARNING: Implementation here must contain only Windows code.
+//===----------------------------------------------------------------------===//
+
+#include <objbase.h>
+
+namespace llvm {
+namespace sys {
+
+InitializeCOMRAII::InitializeCOMRAII(COMThreadingMode Threading,
+                                     bool SpeedOverMemory) {
+  DWORD Coinit = 0;
+  if (Threading == COMThreadingMode::SingleThreaded)
+    Coinit |= COINIT_APARTMENTTHREADED;
+  else
+    Coinit |= COINIT_MULTITHREADED;
+  if (SpeedOverMemory)
+    Coinit |= COINIT_SPEED_OVER_MEMORY;
+  ::CoInitializeEx(nullptr, Coinit);
+}
+
+InitializeCOMRAII::~InitializeCOMRAII() { ::CoUninitialize(); }
+}
+}
diff --git a/lib/Support/Windows/DynamicLibrary.inc b/lib/Support/Windows/DynamicLibrary.inc
index 79d5f79afa9..d38f1974911 100644
--- a/lib/Support/Windows/DynamicLibrary.inc
+++ b/lib/Support/Windows/DynamicLibrary.inc
@@ -23,14 +23,6 @@
  #include <ntverp.h>
 #endif
 
-#ifdef __MINGW32__
- #if (HAVE_LIBIMAGEHLP != 1)
-  #error "libimagehlp.a should be present"
- #endif
-#else
- #pragma comment(lib, "dbghelp.lib")
-#endif
-
 namespace llvm {
 using namespace sys;
 
@@ -39,10 +31,21 @@ using namespace sys;
 //===          and must not be UNIX code.
 //===----------------------------------------------------------------------===//
 
+typedef BOOL (WINAPI *fpEnumerateLoadedModules)(HANDLE,PENUMLOADED_MODULES_CALLBACK64,PVOID);
+static fpEnumerateLoadedModules fEnumerateLoadedModules;
 static DenseSet<HMODULE> *OpenedHandles;
 
+static bool loadDebugHelp(void) {
+  HMODULE hLib = ::LoadLibraryW(L"Dbghelp.dll");
+  if (hLib) {
+    fEnumerateLoadedModules = (fpEnumerateLoadedModules)
+      ::GetProcAddress(hLib, "EnumerateLoadedModules64");
+  }
+  return fEnumerateLoadedModules != 0;
+}
+
 static BOOL CALLBACK
-ELM_Callback(WIN32_ELMCB_PCSTR ModuleName, ULONG_PTR ModuleBase,
+ELM_Callback(WIN32_ELMCB_PCSTR ModuleName, DWORD64 ModuleBase,
              ULONG ModuleSize, PVOID UserContext) {
   OpenedHandles->insert((HMODULE)ModuleBase);
   return TRUE;
@@ -57,7 +60,14 @@ DynamicLibrary DynamicLibrary::getPermanentLibrary(const char *filename,
     if (OpenedHandles == 0)
       OpenedHandles = new DenseSet<HMODULE>();
 
-    EnumerateLoadedModules(GetCurrentProcess(), ELM_Callback, 0);
+    if (!fEnumerateLoadedModules) {
+      if (!loadDebugHelp()) {
+        assert(false && "These APIs should always be available");
+        return DynamicLibrary();
+      }
+    }
+
+    fEnumerateLoadedModules(GetCurrentProcess(), ELM_Callback, 0);
     // Dummy library that represents "search all handles".
     // This is mostly to ensure that the return value still shows up as "valid".
     return DynamicLibrary(&OpenedHandles);
diff --git a/lib/Support/Windows/Memory.inc b/lib/Support/Windows/Memory.inc
index ae8371abf5b..4b2ff2e2d32 100644
--- a/lib/Support/Windows/Memory.inc
+++ b/lib/Support/Windows/Memory.inc
@@ -78,7 +78,15 @@ MemoryBlock Memory::allocateMappedMemory(size_t NumBytes,
   // While we'd be happy to allocate single pages, the Windows allocation
   // granularity may be larger than a single page (in practice, it is 64K)
   // so mapping less than that will create an unreachable fragment of memory.
-  static const size_t Granularity = getAllocationGranularity();
+  // Avoid using one-time initialization of static locals here, since they
+  // aren't thread safe with MSVC.
+  static volatile size_t GranularityCached;
+  size_t Granularity = GranularityCached;
+  if (Granularity == 0) {
+    Granularity = getAllocationGranularity();
+    GranularityCached = Granularity;
+  }
+
   const size_t NumBlocks = (NumBytes+Granularity-1)/Granularity;
 
   uintptr_t Start = NearBlock ? reinterpret_cast<uintptr_t>(NearBlock->base()) +
diff --git a/lib/Support/Windows/Path.inc b/lib/Support/Windows/Path.inc
index b5523aaf438..72da7c5fec3 100644
--- a/lib/Support/Windows/Path.inc
+++ b/lib/Support/Windows/Path.inc
@@ -46,10 +46,6 @@ using llvm::sys::windows::UTF8ToUTF16;
 using llvm::sys::windows::UTF16ToUTF8;
 using llvm::sys::path::widenPath;
 
-static std::error_code windows_error(DWORD E) {
-  return mapWindowsError(E);
-}
-
 static bool is_separator(const wchar_t value) {
   switch (value) {
   case L'\\':
@@ -83,7 +79,7 @@ std::error_code widenPath(const Twine &Path8,
   else {
     CurPathLen = ::GetCurrentDirectoryW(0, NULL);
     if (CurPathLen == 0)
-      return windows_error(::GetLastError());
+      return mapWindowsError(::GetLastError());
   }
 
   // Would the absolute path be longer than our limit?
@@ -174,7 +170,7 @@ std::error_code current_path(SmallVectorImpl<char> &result) {
 
     // A zero return value indicates a failure other than insufficient space.
     if (len == 0)
-      return windows_error(::GetLastError());
+      return mapWindowsError(::GetLastError());
 
     // If there's insufficient space, the len returned is larger than the len
     // given.
@@ -195,7 +191,7 @@ std::error_code create_directory(const Twine &path, bool IgnoreExisting) {
   if (!::CreateDirectoryW(path_utf16.begin(), NULL)) {
     DWORD LastError = ::GetLastError();
     if (LastError != ERROR_ALREADY_EXISTS || !IgnoreExisting)
-      return windows_error(LastError);
+      return mapWindowsError(LastError);
   }
 
   return std::error_code();
@@ -212,7 +208,7 @@ std::error_code create_link(const Twine &to, const Twine &from) {
     return ec;
 
   if (!::CreateHardLinkW(wide_from.begin(), wide_to.begin(), NULL))
-    return windows_error(::GetLastError());
+    return mapWindowsError(::GetLastError());
 
   return std::error_code();
 }
@@ -232,14 +228,14 @@ std::error_code remove(const Twine &path, bool IgnoreNonExisting) {
 
   if (ST.type() == file_type::directory_file) {
     if (!::RemoveDirectoryW(c_str(path_utf16))) {
-      std::error_code EC = windows_error(::GetLastError());
+      std::error_code EC = mapWindowsError(::GetLastError());
       if (EC != errc::no_such_file_or_directory || !IgnoreNonExisting)
         return EC;
     }
     return std::error_code();
   }
   if (!::DeleteFileW(c_str(path_utf16))) {
-    std::error_code EC = windows_error(::GetLastError());
+    std::error_code EC = mapWindowsError(::GetLastError());
     if (EC != errc::no_such_file_or_directory || !IgnoreNonExisting)
       return EC;
   }
@@ -261,7 +257,7 @@ std::error_code rename(const Twine &from, const Twine &to) {
                       MOVEFILE_COPY_ALLOWED | MOVEFILE_REPLACE_EXISTING))
       return std::error_code();
     DWORD LastError = ::GetLastError();
-    ec = windows_error(LastError);
+    ec = mapWindowsError(LastError);
     if (LastError != ERROR_ACCESS_DENIED)
       break;
     // Retry MoveFile() at ACCESS_DENIED.
@@ -295,7 +291,7 @@ std::error_code access(const Twine &Path, AccessMode Mode) {
     DWORD LastError = ::GetLastError();
     if (LastError != ERROR_FILE_NOT_FOUND &&
         LastError != ERROR_PATH_NOT_FOUND)
-      return windows_error(LastError);
+      return mapWindowsError(LastError);
     return errc::no_such_file_or_directory;
   }
 
@@ -359,7 +355,7 @@ static std::error_code getStatus(HANDLE FileHandle, file_status &Result) {
   case FILE_TYPE_UNKNOWN: {
     DWORD Err = ::GetLastError();
     if (Err != NO_ERROR)
-      return windows_error(Err);
+      return mapWindowsError(Err);
     Result = file_status(file_type::type_unknown);
     return std::error_code();
   }
@@ -398,7 +394,7 @@ handle_status_error:
     Result = file_status(file_type::type_unknown);
   else
     Result = file_status(file_type::status_error);
-  return windows_error(LastError);
+  return mapWindowsError(LastError);
 }
 
 std::error_code status(const Twine &path, file_status &result) {
@@ -455,7 +451,7 @@ std::error_code setLastModificationAndAccessTime(int FD, TimeValue Time) {
   FT.dwHighDateTime = UI.HighPart;
   HANDLE FileHandle = reinterpret_cast<HANDLE>(_get_osfhandle(FD));
   if (!SetFileTime(FileHandle, NULL, &FT, &FT))
-    return windows_error(::GetLastError());
+    return mapWindowsError(::GetLastError());
   return std::error_code();
 }
 
@@ -482,7 +478,7 @@ std::error_code mapped_file_region::init(int FD, uint64_t Offset,
                            (Offset + Size) & 0xffffffff,
                            0);
   if (FileMappingHandle == NULL) {
-    std::error_code ec = windows_error(GetLastError());
+    std::error_code ec = mapWindowsError(GetLastError());
     return ec;
   }
 
@@ -498,7 +494,7 @@ std::error_code mapped_file_region::init(int FD, uint64_t Offset,
                             Offset & 0xffffffff,
                             Size);
   if (Mapping == NULL) {
-    std::error_code ec = windows_error(GetLastError());
+    std::error_code ec = mapWindowsError(GetLastError());
     ::CloseHandle(FileMappingHandle);
     return ec;
   }
@@ -507,7 +503,7 @@ std::error_code mapped_file_region::init(int FD, uint64_t Offset,
     MEMORY_BASIC_INFORMATION mbi;
     SIZE_T Result = VirtualQuery(Mapping, &mbi, sizeof(mbi));
     if (Result == 0) {
-      std::error_code ec = windows_error(GetLastError());
+      std::error_code ec = mapWindowsError(GetLastError());
       ::UnmapViewOfFile(Mapping);
       ::CloseHandle(FileMappingHandle);
       return ec;
@@ -576,7 +572,7 @@ std::error_code detail::directory_iterator_construct(detail::DirIterState &it,
   WIN32_FIND_DATAW FirstFind;
   ScopedFindHandle FindHandle(::FindFirstFileW(c_str(path_utf16), &FirstFind));
   if (!FindHandle)
-    return windows_error(::GetLastError());
+    return mapWindowsError(::GetLastError());
 
   size_t FilenameLen = ::wcslen(FirstFind.cFileName);
   while ((FilenameLen == 1 && FirstFind.cFileName[0] == L'.') ||
@@ -587,7 +583,7 @@ std::error_code detail::directory_iterator_construct(detail::DirIterState &it,
       // Check for end.
       if (LastError == ERROR_NO_MORE_FILES)
         return detail::directory_iterator_destruct(it);
-      return windows_error(LastError);
+      return mapWindowsError(LastError);
     } else
       FilenameLen = ::wcslen(FirstFind.cFileName);
 
@@ -622,7 +618,7 @@ std::error_code detail::directory_iterator_increment(detail::DirIterState &it) {
     // Check for end.
     if (LastError == ERROR_NO_MORE_FILES)
       return detail::directory_iterator_destruct(it);
-    return windows_error(LastError);
+    return mapWindowsError(LastError);
   }
 
   size_t FilenameLen = ::wcslen(FindData.cFileName);
@@ -652,7 +648,7 @@ std::error_code openFileForRead(const Twine &Name, int &ResultFD) {
                            OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL, NULL);
   if (H == INVALID_HANDLE_VALUE) {
     DWORD LastError = ::GetLastError();
-    std::error_code EC = windows_error(LastError);
+    std::error_code EC = mapWindowsError(LastError);
     // Provide a better error message when trying to open directories.
     // This only runs if we failed to open the file, so there is probably
     // no performances issues.
@@ -666,7 +662,7 @@ std::error_code openFileForRead(const Twine &Name, int &ResultFD) {
   int FD = ::_open_osfhandle(intptr_t(H), 0);
   if (FD == -1) {
     ::CloseHandle(H);
-    return windows_error(ERROR_INVALID_HANDLE);
+    return mapWindowsError(ERROR_INVALID_HANDLE);
   }
 
   ResultFD = FD;
@@ -702,7 +698,7 @@ std::error_code openFileForWrite(const Twine &Name, int &ResultFD,
 
   if (H == INVALID_HANDLE_VALUE) {
     DWORD LastError = ::GetLastError();
-    std::error_code EC = windows_error(LastError);
+    std::error_code EC = mapWindowsError(LastError);
     // Provide a better error message when trying to open directories.
     // This only runs if we failed to open the file, so there is probably
     // no performances issues.
@@ -723,7 +719,7 @@ std::error_code openFileForWrite(const Twine &Name, int &ResultFD,
   int FD = ::_open_osfhandle(intptr_t(H), OpenFlags);
   if (FD == -1) {
     ::CloseHandle(H);
-    return windows_error(ERROR_INVALID_HANDLE);
+    return mapWindowsError(ERROR_INVALID_HANDLE);
   }
 
   ResultFD = FD;
@@ -800,7 +796,7 @@ std::error_code UTF8ToUTF16(llvm::StringRef utf8,
                                     utf8.size(), utf16.begin(), 0);
 
     if (len == 0)
-      return windows_error(::GetLastError());
+      return mapWindowsError(::GetLastError());
 
     utf16.reserve(len + 1);
     utf16.set_size(len);
@@ -809,7 +805,7 @@ std::error_code UTF8ToUTF16(llvm::StringRef utf8,
                                 utf8.size(), utf16.begin(), utf16.size());
 
     if (len == 0)
-      return windows_error(::GetLastError());
+      return mapWindowsError(::GetLastError());
   }
 
   // Make utf16 null terminated.
@@ -829,7 +825,7 @@ std::error_code UTF16ToCodePage(unsigned codepage, const wchar_t *utf16,
                                     0, NULL, NULL);
 
     if (len == 0)
-      return windows_error(::GetLastError());
+      return mapWindowsError(::GetLastError());
 
     utf8.reserve(len);
     utf8.set_size(len);
@@ -839,7 +835,7 @@ std::error_code UTF16ToCodePage(unsigned codepage, const wchar_t *utf16,
                                 utf8.size(), NULL, NULL);
 
     if (len == 0)
-      return windows_error(::GetLastError());
+      return mapWindowsError(::GetLastError());
   }
 
   // Make utf8 null terminated.
diff --git a/lib/Support/Windows/Process.inc b/lib/Support/Windows/Process.inc
index 5f9ce7f3f6d..8164956d151 100644
--- a/lib/Support/Windows/Process.inc
+++ b/lib/Support/Windows/Process.inc
@@ -156,10 +156,6 @@ Optional<std::string> Process::GetEnv(StringRef Name) {
   return std::string(Res.data());
 }
 
-static std::error_code windows_error(DWORD E) {
-  return mapWindowsError(E);
-}
-
 static void AllocateAndPush(const SmallVectorImpl<char> &S,
                             SmallVectorImpl<const char *> &Vector,
                             SpecificBumpPtrAllocator<char> &Allocator) {
@@ -235,7 +231,7 @@ Process::GetArgumentVector(SmallVectorImpl<const char *> &Args,
   wchar_t **UnicodeCommandLine =
       CommandLineToArgvW(GetCommandLineW(), &ArgCount);
   if (!UnicodeCommandLine)
-    return windows_error(::GetLastError());
+    return mapWindowsError(::GetLastError());
 
   Args.reserve(ArgCount);
   std::error_code ec;
diff --git a/lib/Support/Windows/Program.inc b/lib/Support/Windows/Program.inc
index c3700774514..c29d8729b1d 100644
--- a/lib/Support/Windows/Program.inc
+++ b/lib/Support/Windows/Program.inc
@@ -14,6 +14,7 @@
 #include "WindowsSupport.h"
 #include "llvm/ADT/StringExtras.h"
 #include "llvm/Support/ConvertUTF.h"
+#include "llvm/Support/Errc.h"
 #include "llvm/Support/FileSystem.h"
 #include "llvm/Support/WindowsError.h"
 #include "llvm/Support/raw_ostream.h"
@@ -434,7 +435,8 @@ ProcessInfo sys::Wait(const ProcessInfo &PI, unsigned SecondsToWait,
   DWORD status;
   BOOL rc = GetExitCodeProcess(PI.ProcessHandle, &status);
   DWORD err = GetLastError();
-  CloseHandle(PI.ProcessHandle);
+  if (err != ERROR_INVALID_HANDLE)
+    CloseHandle(PI.ProcessHandle);
 
   if (!rc) {
     SetLastError(err);
@@ -513,7 +515,7 @@ llvm::sys::writeFileWithEncoding(StringRef FileName, StringRef Contents,
   }
 
   if (OS.has_error())
-    return std::make_error_code(std::errc::io_error);
+    return make_error_code(errc::io_error);
 
   return EC;
 }
diff --git a/lib/Support/Windows/Signals.inc b/lib/Support/Windows/Signals.inc
index f070111a0e7..824edb3c39b 100644
--- a/lib/Support/Windows/Signals.inc
+++ b/lib/Support/Windows/Signals.inc
@@ -14,7 +14,6 @@
 #include <algorithm>
 #include <signal.h>
 #include <stdio.h>
-#include <vector>
 
 #include "llvm/Support/Format.h"
 #include "llvm/Support/raw_ostream.h"
@@ -31,10 +30,9 @@
 
 #ifdef _MSC_VER
  #pragma comment(lib, "psapi.lib")
- #pragma comment(lib, "dbghelp.lib")
 #elif __MINGW32__
- #if ((HAVE_LIBIMAGEHLP != 1) || (HAVE_LIBPSAPI != 1))
-  #error "libimagehlp.a & libpsapi.a should be present"
+ #if (HAVE_LIBPSAPI != 1)
+  #error "libpsapi.a should be present"
  #endif
  // The version of g++ that comes with MinGW does *not* properly understand
  // the ll format specifier for printf. However, MinGW passes the format
@@ -103,6 +101,8 @@
    DWORD64     Reserved[3];
    KDHELP64    KdHelp;
  } STACKFRAME64, *LPSTACKFRAME64;
+ #endif // !defined(__MINGW64_VERSION_MAJOR)
+#endif // __MINGW32__
 
 typedef BOOL (__stdcall *PREAD_PROCESS_MEMORY_ROUTINE64)(HANDLE hProcess,
                       DWORD64 qwBaseAddress, PVOID lpBuffer, DWORD nSize,
@@ -122,40 +122,48 @@ typedef BOOL (WINAPI *fpStackWalk64)(DWORD, HANDLE, HANDLE, LPSTACKFRAME64,
                       PFUNCTION_TABLE_ACCESS_ROUTINE64,
                       PGET_MODULE_BASE_ROUTINE64,
                       PTRANSLATE_ADDRESS_ROUTINE64);
-static fpStackWalk64 StackWalk64;
+static fpStackWalk64 fStackWalk64;
 
 typedef DWORD64 (WINAPI *fpSymGetModuleBase64)(HANDLE, DWORD64);
-static fpSymGetModuleBase64 SymGetModuleBase64;
+static fpSymGetModuleBase64 fSymGetModuleBase64;
 
 typedef BOOL (WINAPI *fpSymGetSymFromAddr64)(HANDLE, DWORD64,
                       PDWORD64, PIMAGEHLP_SYMBOL64);
-static fpSymGetSymFromAddr64 SymGetSymFromAddr64;
+static fpSymGetSymFromAddr64 fSymGetSymFromAddr64;
 
 typedef BOOL (WINAPI *fpSymGetLineFromAddr64)(HANDLE, DWORD64,
                       PDWORD, PIMAGEHLP_LINE64);
-static fpSymGetLineFromAddr64 SymGetLineFromAddr64;
+static fpSymGetLineFromAddr64 fSymGetLineFromAddr64;
 
 typedef PVOID (WINAPI *fpSymFunctionTableAccess64)(HANDLE, DWORD64);
-static fpSymFunctionTableAccess64 SymFunctionTableAccess64;
+static fpSymFunctionTableAccess64 fSymFunctionTableAccess64;
+
+typedef DWORD (WINAPI *fpSymSetOptions)(DWORD);
+static fpSymSetOptions fSymSetOptions;
+
+typedef BOOL (WINAPI *fpSymInitialize)(HANDLE, PCSTR, BOOL);
+static fpSymInitialize fSymInitialize;
 
 static bool load64BitDebugHelp(void) {
   HMODULE hLib = ::LoadLibraryW(L"Dbghelp.dll");
   if (hLib) {
-    StackWalk64 = (fpStackWalk64)
+    fStackWalk64 = (fpStackWalk64)
                       ::GetProcAddress(hLib, "StackWalk64");
-    SymGetModuleBase64 = (fpSymGetModuleBase64)
+    fSymGetModuleBase64 = (fpSymGetModuleBase64)
                       ::GetProcAddress(hLib, "SymGetModuleBase64");
-    SymGetSymFromAddr64 = (fpSymGetSymFromAddr64)
+    fSymGetSymFromAddr64 = (fpSymGetSymFromAddr64)
                       ::GetProcAddress(hLib, "SymGetSymFromAddr64");
-    SymGetLineFromAddr64 = (fpSymGetLineFromAddr64)
+    fSymGetLineFromAddr64 = (fpSymGetLineFromAddr64)
                       ::GetProcAddress(hLib, "SymGetLineFromAddr64");
-    SymFunctionTableAccess64 = (fpSymFunctionTableAccess64)
+    fSymFunctionTableAccess64 = (fpSymFunctionTableAccess64)
                      ::GetProcAddress(hLib, "SymFunctionTableAccess64");
+    fSymSetOptions = (fpSymSetOptions)::GetProcAddress(hLib, "SymSetOptions");
+    fSymInitialize = (fpSymInitialize)::GetProcAddress(hLib, "SymInitialize");
   }
-  return StackWalk64 != NULL;
+  return fStackWalk64 && fSymInitialize && fSymSetOptions;
 }
- #endif // !defined(__MINGW64_VERSION_MAJOR)
-#endif // __MINGW32__
+
+using namespace llvm;
 
 // Forward declare.
 static LONG WINAPI LLVMUnhandledExceptionFilter(LPEXCEPTION_POINTERS ep);
@@ -165,7 +173,6 @@ static BOOL WINAPI LLVMConsoleCtrlHandler(DWORD dwCtrlType);
 static void (*InterruptFunction)() = 0;
 
 static std::vector<std::string> *FilesToRemove = NULL;
-static std::vector<std::pair<void(*)(void*), void*> > *CallBacksToRun = 0;
 static bool RegisteredUnhandledExceptionFilter = false;
 static bool CleanupExecuted = false;
 static PTOP_LEVEL_EXCEPTION_FILTER OldFilter = NULL;
@@ -187,12 +194,12 @@ static void PrintStackTraceForThread(llvm::raw_ostream &OS, HANDLE hProcess,
 #endif
 
   // Initialize the symbol handler.
-  SymSetOptions(SYMOPT_DEFERRED_LOADS | SYMOPT_LOAD_LINES);
-  SymInitialize(hProcess, NULL, TRUE);
+  fSymSetOptions(SYMOPT_DEFERRED_LOADS | SYMOPT_LOAD_LINES);
+  fSymInitialize(hProcess, NULL, TRUE);
 
   while (true) {
-    if (!StackWalk64(machineType, hProcess, hThread, &StackFrame, Context, NULL,
-                     SymFunctionTableAccess64, SymGetModuleBase64, NULL)) {
+    if (!fStackWalk64(machineType, hProcess, hThread, &StackFrame, Context, 0,
+                      fSymFunctionTableAccess64, fSymGetModuleBase64, 0)) {
       break;
     }
 
@@ -221,7 +228,7 @@ static void PrintStackTraceForThread(llvm::raw_ostream &OS, HANDLE hProcess,
             static_cast<DWORD>(StackFrame.Params[3]));
 #endif
     // Verify the PC belongs to a module in this process.
-    if (!SymGetModuleBase64(hProcess, PC)) {
+    if (!fSymGetModuleBase64(hProcess, PC)) {
       OS << " <unknown module>\n";
       continue;
     }
@@ -234,7 +241,7 @@ static void PrintStackTraceForThread(llvm::raw_ostream &OS, HANDLE hProcess,
     symbol->MaxNameLength = 512 - sizeof(IMAGEHLP_SYMBOL64);
 
     DWORD64 dwDisp;
-    if (!SymGetSymFromAddr64(hProcess, PC, &dwDisp, symbol)) {
+    if (!fSymGetSymFromAddr64(hProcess, PC, &dwDisp, symbol)) {
       OS << '\n';
       continue;
     }
@@ -247,11 +254,10 @@ static void PrintStackTraceForThread(llvm::raw_ostream &OS, HANDLE hProcess,
       OS << format(", %s", (const char*)symbol->Name);
 
     // Print the source file and line number information.
-    IMAGEHLP_LINE64 line;
+    IMAGEHLP_LINE64 line = {};
     DWORD dwLineDisp;
-    memset(&line, 0, sizeof(line));
     line.SizeOfStruct = sizeof(line);
-    if (SymGetLineFromAddr64(hProcess, PC, &dwLineDisp, &line)) {
+    if (fSymGetLineFromAddr64(hProcess, PC, &dwLineDisp, &line)) {
       OS << format(", %s, line %lu", line.FileName, line.LineNumber);
       if (dwLineDisp > 0)
         OS << format(" + 0x%lX byte(s)", dwLineDisp);
@@ -302,17 +308,13 @@ static void InitializeThreading() {
 }
 
 static void RegisterHandler() {
-#if __MINGW32__ && !defined(__MINGW64_VERSION_MAJOR)
-  // On MinGW.org, we need to load up the symbols explicitly, because the
-  // Win32 framework they include does not have support for the 64-bit
-  // versions of the APIs we need.  If we cannot load up the APIs (which
-  // would be unexpected as they should exist on every version of Windows
-  // we support), we will bail out since there would be nothing to report.
+  // If we cannot load up the APIs (which would be unexpected as they should
+  // exist on every version of Windows we support), we will bail out since
+  // there would be nothing to report.
   if (!load64BitDebugHelp()) {
     assert(false && "These APIs should always be available");
     return;
   }
-#endif
 
   if (RegisteredUnhandledExceptionFilter) {
     EnterCriticalSection(&CriticalSection);
@@ -359,7 +361,6 @@ void sys::DontRemoveFileOnSignal(StringRef Filename) {
 
   RegisterHandler();
 
-  FilesToRemove->push_back(Filename);
   std::vector<std::string>::reverse_iterator I =
   std::find(FilesToRemove->rbegin(), FilesToRemove->rend(), Filename);
   if (I != FilesToRemove->rend())
@@ -405,7 +406,7 @@ extern "C" VOID WINAPI RtlCaptureContext(PCONTEXT ContextRecord);
 void llvm::sys::PrintStackTrace(raw_ostream &OS) {
 
   STACKFRAME64 StackFrame = {};
-  CONTEXT Context = {0};
+  CONTEXT Context = {};
   ::RtlCaptureContext(&Context);
 #if defined(_M_X64)
   StackFrame.AddrPC.Offset = Context.Rip;
@@ -435,14 +436,15 @@ void llvm::sys::SetInterruptFunction(void (*IF)()) {
 /// to the process.  The handler can have a cookie passed to it to identify
 /// what instance of the handler it is.
 void llvm::sys::AddSignalHandler(void (*FnPtr)(void *), void *Cookie) {
-  if (CallBacksToRun == 0)
-    CallBacksToRun = new std::vector<std::pair<void(*)(void*), void*> >();
   CallBacksToRun->push_back(std::make_pair(FnPtr, Cookie));
   RegisterHandler();
   LeaveCriticalSection(&CriticalSection);
 }
 
 static void Cleanup() {
+  if (CleanupExecuted)
+    return;
+
   EnterCriticalSection(&CriticalSection);
 
   // Prevent other thread from registering new files and directories for
@@ -450,17 +452,12 @@ static void Cleanup() {
   CleanupExecuted = true;
 
   // FIXME: open files cannot be deleted.
-
   if (FilesToRemove != NULL)
     while (!FilesToRemove->empty()) {
       llvm::sys::fs::remove(FilesToRemove->back());
       FilesToRemove->pop_back();
     }
-
-  if (CallBacksToRun)
-    for (unsigned i = 0, e = CallBacksToRun->size(); i != e; ++i)
-      (*CallBacksToRun)[i].first((*CallBacksToRun)[i].second);
-
+  llvm::sys::RunSignalHandlers();
   LeaveCriticalSection(&CriticalSection);
 }
 
@@ -477,8 +474,7 @@ static LONG WINAPI LLVMUnhandledExceptionFilter(LPEXCEPTION_POINTERS ep) {
   Cleanup();
 
   // Initialize the STACKFRAME structure.
-  STACKFRAME64 StackFrame;
-  memset(&StackFrame, 0, sizeof(StackFrame));
+  STACKFRAME64 StackFrame = {};
 
 #if defined(_M_X64)
   StackFrame.AddrPC.Offset = ep->ContextRecord->Rip;
diff --git a/lib/Support/YAMLParser.cpp b/lib/Support/YAMLParser.cpp
index 93aec7c0438..d55da5ef1e4 100644
--- a/lib/Support/YAMLParser.cpp
+++ b/lib/Support/YAMLParser.cpp
@@ -101,6 +101,7 @@ namespace yaml {
 void Node::anchor() {}
 void NullNode::anchor() {}
 void ScalarNode::anchor() {}
+void BlockScalarNode::anchor() {}
 void KeyValueNode::anchor() {}
 void MappingNode::anchor() {}
 void SequenceNode::anchor() {}
@@ -128,6 +129,7 @@ struct Token : ilist_node<Token> {
     TK_Key,
     TK_Value,
     TK_Scalar,
+    TK_BlockScalar,
     TK_Alias,
     TK_Anchor,
     TK_Tag
@@ -137,6 +139,9 @@ struct Token : ilist_node<Token> {
   /// of the token in the input.
   StringRef Range;
 
+  /// The value of a block scalar node.
+  std::string Value;
+
   Token() : Kind(TK_Error) {}
 };
 }
@@ -163,7 +168,7 @@ struct ilist_node_traits<Token> {
   Token *createNode(const Token &V) {
     return new (Alloc.Allocate<Token>()) Token(V);
   }
-  static void deleteNode(Token *V) {}
+  static void deleteNode(Token *V) { V->~Token(); }
 
   void addNodeToList(Token *) {}
   void removeNodeFromList(Token *) {}
@@ -260,8 +265,8 @@ namespace yaml {
 /// @brief Scans YAML tokens from a MemoryBuffer.
 class Scanner {
 public:
-  Scanner(StringRef Input, SourceMgr &SM);
-  Scanner(MemoryBufferRef Buffer, SourceMgr &SM_);
+  Scanner(StringRef Input, SourceMgr &SM, bool ShowColors = true);
+  Scanner(MemoryBufferRef Buffer, SourceMgr &SM_, bool ShowColors = true);
 
   /// @brief Parse the next token and return it without popping it.
   Token &peekNext();
@@ -271,7 +276,7 @@ class Scanner {
 
   void printError(SMLoc Loc, SourceMgr::DiagKind Kind, const Twine &Message,
                   ArrayRef<SMRange> Ranges = None) {
-    SM.PrintMessage(Loc, Kind, Message, Ranges);
+    SM.PrintMessage(Loc, Kind, Message, Ranges, /* FixIts= */ None, ShowColors);
   }
 
   void setError(const Twine &Message, StringRef::iterator Position) {
@@ -348,6 +353,14 @@ class Scanner {
   ///          b-break.
   StringRef::iterator skip_b_break(StringRef::iterator Position);
 
+  /// Skip a single s-space[31] starting at Position.
+  ///
+  /// An s-space is 0x20
+  ///
+  /// @returns The code unit after the s-space, or Position if it's not a
+  ///          s-space.
+  StringRef::iterator skip_s_space(StringRef::iterator Position);
+
   /// @brief Skip a single s-white[33] starting at Position.
   ///
   /// A s-white is 0x20 | 0x9
@@ -373,6 +386,10 @@ class Scanner {
   StringRef::iterator skip_while( SkipWhileFunc Func
                                 , StringRef::iterator Position);
 
+  /// Skip minimal well-formed code unit subsequences until Func returns its
+  /// input.
+  void advanceWhile(SkipWhileFunc Func);
+
   /// @brief Scan ns-uri-char[39]s starting at Cur.
   ///
   /// This updates Cur and Column while scanning.
@@ -393,6 +410,11 @@ class Scanner {
   ///        Pos is whitespace or a new line
   bool isBlankOrBreak(StringRef::iterator Position);
 
+  /// Consume a single b-break[28] if it's present at the current position.
+  ///
+  /// Return false if the code unit at the current position isn't a line break.
+  bool consumeLineBreakIfPresent();
+
   /// @brief If IsSimpleKeyAllowed, create and push_back a new SimpleKey.
   void saveSimpleKeyCandidate( TokenQueueT::iterator Tok
                              , unsigned AtColumn
@@ -417,6 +439,10 @@ class Scanner {
                  , Token::TokenKind Kind
                  , TokenQueueT::iterator InsertPoint);
 
+  /// @brief Skip a single-line comment when the comment starts at the current
+  /// position of the scanner.
+  void skipComment();
+
   /// @brief Skip whitespace and comments until the start of the next token.
   void scanToNextToken();
 
@@ -462,6 +488,30 @@ class Scanner {
   /// @brief Scan a block scalar starting with | or >.
   bool scanBlockScalar(bool IsLiteral);
 
+  /// Scan a chomping indicator in a block scalar header.
+  char scanBlockChompingIndicator();
+
+  /// Scan an indentation indicator in a block scalar header.
+  unsigned scanBlockIndentationIndicator();
+
+  /// Scan a block scalar header.
+  ///
+  /// Return false if an error occurred.
+  bool scanBlockScalarHeader(char &ChompingIndicator, unsigned &IndentIndicator,
+                             bool &IsDone);
+
+  /// Look for the indentation level of a block scalar.
+  ///
+  /// Return false if an error occurred.
+  bool findBlockScalarIndent(unsigned &BlockIndent, unsigned BlockExitIndent,
+                             unsigned &LineBreaks, bool &IsDone);
+
+  /// Scan the indentation of a text line in a block scalar.
+  ///
+  /// Return false if an error occurred.
+  bool scanBlockScalarIndent(unsigned BlockIndent, unsigned BlockExitIndent,
+                             bool &IsDone);
+
   /// @brief Scan a tag of the form !stuff.
   bool scanTag();
 
@@ -501,6 +551,9 @@ class Scanner {
   /// @brief True if an error has occurred.
   bool Failed;
 
+  /// @brief Should colors be used when printing out the diagnostic messages?
+  bool ShowColors;
+
   /// @brief Queue of tokens. This is required to queue up tokens while looking
   ///        for the end of a simple key. And for cases where a single character
   ///        can produce multiple tokens (e.g. BlockEnd).
@@ -605,6 +658,9 @@ bool yaml::dumpTokens(StringRef Input, raw_ostream &OS) {
     case Token::TK_Scalar:
       OS << "Scalar: ";
       break;
+    case Token::TK_BlockScalar:
+      OS << "Block Scalar: ";
+      break;
     case Token::TK_Alias:
       OS << "Alias: ";
       break;
@@ -702,11 +758,13 @@ std::string yaml::escape(StringRef Input) {
   return EscapedInput;
 }
 
-Scanner::Scanner(StringRef Input, SourceMgr &sm) : SM(sm) {
+Scanner::Scanner(StringRef Input, SourceMgr &sm, bool ShowColors)
+    : SM(sm), ShowColors(ShowColors) {
   init(MemoryBufferRef(Input, "YAML"));
 }
 
-Scanner::Scanner(MemoryBufferRef Buffer, SourceMgr &SM_) : SM(SM_) {
+Scanner::Scanner(MemoryBufferRef Buffer, SourceMgr &SM_, bool ShowColors)
+    : SM(SM_), ShowColors(ShowColors) {
   init(Buffer);
 }
 
@@ -807,6 +865,13 @@ StringRef::iterator Scanner::skip_b_break(StringRef::iterator Position) {
   return Position;
 }
 
+StringRef::iterator Scanner::skip_s_space(StringRef::iterator Position) {
+  if (Position == End)
+    return Position;
+  if (*Position == ' ')
+    return Position + 1;
+  return Position;
+}
 
 StringRef::iterator Scanner::skip_s_white(StringRef::iterator Position) {
   if (Position == End)
@@ -835,6 +900,12 @@ StringRef::iterator Scanner::skip_while( SkipWhileFunc Func
   return Position;
 }
 
+void Scanner::advanceWhile(SkipWhileFunc Func) {
+  auto Final = skip_while(Func, Current);
+  Column += Final - Current;
+  Current = Final;
+}
+
 static bool is_ns_hex_digit(const char C) {
   return    (C >= '0' && C <= '9')
          || (C >= 'a' && C <= 'z')
@@ -897,6 +968,16 @@ bool Scanner::isBlankOrBreak(StringRef::iterator Position) {
   return false;
 }
 
+bool Scanner::consumeLineBreakIfPresent() {
+  auto Next = skip_b_break(Current);
+  if (Next == Current)
+    return false;
+  Column = 0;
+  ++Line;
+  Current = Next;
+  return true;
+}
+
 void Scanner::saveSimpleKeyCandidate( TokenQueueT::iterator Tok
                                     , unsigned AtColumn
                                     , bool IsRequired) {
@@ -962,24 +1043,27 @@ bool Scanner::rollIndent( int ToColumn
   return true;
 }
 
+void Scanner::skipComment() {
+  if (*Current != '#')
+    return;
+  while (true) {
+    // This may skip more than one byte, thus Column is only incremented
+    // for code points.
+    StringRef::iterator I = skip_nb_char(Current);
+    if (I == Current)
+      break;
+    Current = I;
+    ++Column;
+  }
+}
+
 void Scanner::scanToNextToken() {
   while (true) {
     while (*Current == ' ' || *Current == '\t') {
       skip(1);
     }
 
-    // Skip comment.
-    if (*Current == '#') {
-      while (true) {
-        // This may skip more than one byte, thus Column is only incremented
-        // for code points.
-        StringRef::iterator i = skip_nb_char(Current);
-        if (i == Current)
-          break;
-        Current = i;
-        ++Column;
-      }
-    }
+    skipComment();
 
     // Skip EOL.
     StringRef::iterator i = skip_b_break(Current);
@@ -1362,38 +1446,204 @@ bool Scanner::scanAliasOrAnchor(bool IsAlias) {
   return true;
 }
 
-bool Scanner::scanBlockScalar(bool IsLiteral) {
-  StringRef::iterator Start = Current;
-  skip(1); // Eat | or >
-  while(true) {
-    StringRef::iterator i = skip_nb_char(Current);
-    if (i == Current) {
-      if (Column == 0)
-        break;
-      i = skip_b_break(Current);
-      if (i != Current) {
-        // We got a line break.
-        Column = 0;
-        ++Line;
-        Current = i;
-        continue;
-      } else {
-        // There was an error, which should already have been printed out.
+char Scanner::scanBlockChompingIndicator() {
+  char Indicator = ' ';
+  if (Current != End && (*Current == '+' || *Current == '-')) {
+    Indicator = *Current;
+    skip(1);
+  }
+  return Indicator;
+}
+
+/// Get the number of line breaks after chomping.
+///
+/// Return the number of trailing line breaks to emit, depending on
+/// \p ChompingIndicator.
+static unsigned getChompedLineBreaks(char ChompingIndicator,
+                                     unsigned LineBreaks, StringRef Str) {
+  if (ChompingIndicator == '-') // Strip all line breaks.
+    return 0;
+  if (ChompingIndicator == '+') // Keep all line breaks.
+    return LineBreaks;
+  // Clip trailing lines.
+  return Str.empty() ? 0 : 1;
+}
+
+unsigned Scanner::scanBlockIndentationIndicator() {
+  unsigned Indent = 0;
+  if (Current != End && (*Current >= '1' && *Current <= '9')) {
+    Indent = unsigned(*Current - '0');
+    skip(1);
+  }
+  return Indent;
+}
+
+bool Scanner::scanBlockScalarHeader(char &ChompingIndicator,
+                                    unsigned &IndentIndicator, bool &IsDone) {
+  auto Start = Current;
+
+  ChompingIndicator = scanBlockChompingIndicator();
+  IndentIndicator = scanBlockIndentationIndicator();
+  // Check for the chomping indicator once again.
+  if (ChompingIndicator == ' ')
+    ChompingIndicator = scanBlockChompingIndicator();
+  Current = skip_while(&Scanner::skip_s_white, Current);
+  skipComment();
+
+  if (Current == End) { // EOF, we have an empty scalar.
+    Token T;
+    T.Kind = Token::TK_BlockScalar;
+    T.Range = StringRef(Start, Current - Start);
+    TokenQueue.push_back(T);
+    IsDone = true;
+    return true;
+  }
+
+  if (!consumeLineBreakIfPresent()) {
+    setError("Expected a line break after block scalar header", Current);
+    return false;
+  }
+  return true;
+}
+
+bool Scanner::findBlockScalarIndent(unsigned &BlockIndent,
+                                    unsigned BlockExitIndent,
+                                    unsigned &LineBreaks, bool &IsDone) {
+  unsigned MaxAllSpaceLineCharacters = 0;
+  StringRef::iterator LongestAllSpaceLine;
+
+  while (true) {
+    advanceWhile(&Scanner::skip_s_space);
+    if (skip_nb_char(Current) != Current) {
+      // This line isn't empty, so try and find the indentation.
+      if (Column <= BlockExitIndent) { // End of the block literal.
+        IsDone = true;
+        return true;
+      }
+      // We found the block's indentation.
+      BlockIndent = Column;
+      if (MaxAllSpaceLineCharacters > BlockIndent) {
+        setError(
+            "Leading all-spaces line must be smaller than the block indent",
+            LongestAllSpaceLine);
         return false;
       }
+      return true;
     }
-    Current = i;
+    if (skip_b_break(Current) != Current &&
+        Column > MaxAllSpaceLineCharacters) {
+      // Record the longest all-space line in case it's longer than the
+      // discovered block indent.
+      MaxAllSpaceLineCharacters = Column;
+      LongestAllSpaceLine = Current;
+    }
+
+    // Check for EOF.
+    if (Current == End) {
+      IsDone = true;
+      return true;
+    }
+
+    if (!consumeLineBreakIfPresent()) {
+      IsDone = true;
+      return true;
+    }
+    ++LineBreaks;
+  }
+  return true;
+}
+
+bool Scanner::scanBlockScalarIndent(unsigned BlockIndent,
+                                    unsigned BlockExitIndent, bool &IsDone) {
+  // Skip the indentation.
+  while (Column < BlockIndent) {
+    auto I = skip_s_space(Current);
+    if (I == Current)
+      break;
+    Current = I;
     ++Column;
   }
 
-  if (Start == Current) {
-    setError("Got empty block scalar", Start);
+  if (skip_nb_char(Current) == Current)
+    return true;
+
+  if (Column <= BlockExitIndent) { // End of the block literal.
+    IsDone = true;
+    return true;
+  }
+
+  if (Column < BlockIndent) {
+    if (Current != End && *Current == '#') { // Trailing comment.
+      IsDone = true;
+      return true;
+    }
+    setError("A text line is less indented than the block scalar", Current);
+    return false;
+  }
+  return true; // A normal text line.
+}
+
+bool Scanner::scanBlockScalar(bool IsLiteral) {
+  // Eat '|' or '>'
+  assert(*Current == '|' || *Current == '>');
+  skip(1);
+
+  char ChompingIndicator;
+  unsigned BlockIndent;
+  bool IsDone = false;
+  if (!scanBlockScalarHeader(ChompingIndicator, BlockIndent, IsDone))
     return false;
+  if (IsDone)
+    return true;
+
+  auto Start = Current;
+  unsigned BlockExitIndent = Indent < 0 ? 0 : (unsigned)Indent;
+  unsigned LineBreaks = 0;
+  if (BlockIndent == 0) {
+    if (!findBlockScalarIndent(BlockIndent, BlockExitIndent, LineBreaks,
+                               IsDone))
+      return false;
   }
 
+  // Scan the block's scalars body.
+  SmallString<256> Str;
+  while (!IsDone) {
+    if (!scanBlockScalarIndent(BlockIndent, BlockExitIndent, IsDone))
+      return false;
+    if (IsDone)
+      break;
+
+    // Parse the current line.
+    auto LineStart = Current;
+    advanceWhile(&Scanner::skip_nb_char);
+    if (LineStart != Current) {
+      Str.append(LineBreaks, '\n');
+      Str.append(StringRef(LineStart, Current - LineStart));
+      LineBreaks = 0;
+    }
+
+    // Check for EOF.
+    if (Current == End)
+      break;
+
+    if (!consumeLineBreakIfPresent())
+      break;
+    ++LineBreaks;
+  }
+
+  if (Current == End && !LineBreaks)
+    // Ensure that there is at least one line break before the end of file.
+    LineBreaks = 1;
+  Str.append(getChompedLineBreaks(ChompingIndicator, LineBreaks, Str), '\n');
+
+  // New lines may start a simple key.
+  if (!FlowLevel)
+    IsSimpleKeyAllowed = true;
+
   Token T;
-  T.Kind = Token::TK_Scalar;
+  T.Kind = Token::TK_BlockScalar;
   T.Range = StringRef(Start, Current - Start);
+  T.Value = Str.str().str();
   TokenQueue.push_back(T);
   return true;
 }
@@ -1518,11 +1768,11 @@ bool Scanner::fetchMoreTokens() {
   return false;
 }
 
-Stream::Stream(StringRef Input, SourceMgr &SM)
-    : scanner(new Scanner(Input, SM)), CurrentDoc() {}
+Stream::Stream(StringRef Input, SourceMgr &SM, bool ShowColors)
+    : scanner(new Scanner(Input, SM, ShowColors)), CurrentDoc() {}
 
-Stream::Stream(MemoryBufferRef InputBuffer, SourceMgr &SM)
-    : scanner(new Scanner(InputBuffer, SM)), CurrentDoc() {}
+Stream::Stream(MemoryBufferRef InputBuffer, SourceMgr &SM, bool ShowColors)
+    : scanner(new Scanner(InputBuffer, SM, ShowColors)), CurrentDoc() {}
 
 Stream::~Stream() {}
 
@@ -1595,6 +1845,7 @@ std::string Node::getVerbatimTag() const {
   case NK_Null:
     return "tag:yaml.org,2002:null";
   case NK_Scalar:
+  case NK_BlockScalar:
     // TODO: Tag resolution.
     return "tag:yaml.org,2002:str";
   case NK_Mapping:
@@ -2126,6 +2377,14 @@ Node *Document::parseBlockNode() {
                 , AnchorInfo.Range.substr(1)
                 , TagInfo.Range
                 , T.Range);
+  case Token::TK_BlockScalar: {
+    getNext();
+    StringRef NullTerminatedStr(T.Value.c_str(), T.Value.length() + 1);
+    StringRef StrCopy = NullTerminatedStr.copy(NodeAllocator).drop_back();
+    return new (NodeAllocator)
+        BlockScalarNode(stream.CurrentDoc, AnchorInfo.Range.substr(1),
+                        TagInfo.Range, StrCopy, T.Range);
+  }
   case Token::TK_Key:
     // Don't eat the TK_Key, KeyValueNode expects it.
     return new (NodeAllocator)
diff --git a/lib/Support/YAMLTraits.cpp b/lib/Support/YAMLTraits.cpp
index 74e541449f2..6b59a16514b 100644
--- a/lib/Support/YAMLTraits.cpp
+++ b/lib/Support/YAMLTraits.cpp
@@ -14,6 +14,7 @@
 #include "llvm/Support/Errc.h"
 #include "llvm/Support/ErrorHandling.h"
 #include "llvm/Support/Format.h"
+#include "llvm/Support/LineIterator.h"
 #include "llvm/Support/YAMLParser.h"
 #include "llvm/Support/raw_ostream.h"
 #include <cctype>
@@ -96,6 +97,10 @@ bool Input::nextDocument() {
   return ++DocIterator != Strm->end();
 }
 
+const Node *Input::getCurrentNode() const {
+  return CurrentNode ? CurrentNode->_node : nullptr;
+}
+
 bool Input::mapTag(StringRef Tag, bool Default) {
   std::string foundTag = CurrentNode->_node->getVerbatimTag();
   if (foundTag.empty()) {
@@ -168,6 +173,10 @@ void Input::endMapping() {
   }
 }
 
+void Input::beginFlowMapping() { beginMapping(); }
+
+void Input::endFlowMapping() { endMapping(); }
+
 unsigned Input::beginSequence() {
   if (SequenceHNode *SQ = dyn_cast<SequenceHNode>(CurrentNode))
     return SQ->Entries.size();
@@ -305,6 +314,8 @@ void Input::scalarString(StringRef &S, bool) {
   }
 }
 
+void Input::blockScalarString(StringRef &S) { scalarString(S, false); }
+
 void Input::setError(HNode *hnode, const Twine &message) {
   assert(hnode && "HNode must not be NULL");
   this->setError(hnode->_node, message);
@@ -327,6 +338,11 @@ std::unique_ptr<Input::HNode> Input::createHNodes(Node *N) {
       KeyStr = StringRef(Buf, Len);
     }
     return llvm::make_unique<ScalarHNode>(N, KeyStr);
+  } else if (BlockScalarNode *BSN = dyn_cast<BlockScalarNode>(N)) {
+    StringRef Value = BSN->getValue();
+    char *Buf = StringAllocator.Allocate<char>(Value.size());
+    memcpy(Buf, Value.data(), Value.size());
+    return llvm::make_unique<ScalarHNode>(N, StringRef(Buf, Value.size()));
   } else if (SequenceNode *SQ = dyn_cast<SequenceNode>(N)) {
     auto SQHNode = llvm::make_unique<SequenceHNode>(N);
     for (Node &SN : *SQ) {
@@ -388,11 +404,13 @@ bool Input::canElideEmptySequence() {
 //  Output
 //===----------------------------------------------------------------------===//
 
-Output::Output(raw_ostream &yout, void *context)
+Output::Output(raw_ostream &yout, void *context, int WrapColumn)
     : IO(context),
       Out(yout),
+      WrapColumn(WrapColumn),
       Column(0),
       ColumnAtFlowStart(0),
+      ColumnAtMapFlowStart(0),
       NeedBitValueComma(false),
       NeedFlowSequenceComma(false),
       EnumerationMatchFound(false),
@@ -427,8 +445,13 @@ bool Output::preflightKey(const char *Key, bool Required, bool SameAsDefault,
                           bool &UseDefault, void *&) {
   UseDefault = false;
   if (Required || !SameAsDefault) {
-    this->newLineCheck();
-    this->paddedKey(Key);
+    auto State = StateStack.back();
+    if (State == inFlowMapFirstKey || State == inFlowMapOtherKey) {
+      flowKey(Key);
+    } else {
+      this->newLineCheck();
+      this->paddedKey(Key);
+    }
     return true;
   }
   return false;
@@ -438,9 +461,24 @@ void Output::postflightKey(void *) {
   if (StateStack.back() == inMapFirstKey) {
     StateStack.pop_back();
     StateStack.push_back(inMapOtherKey);
+  } else if (StateStack.back() == inFlowMapFirstKey) {
+    StateStack.pop_back();
+    StateStack.push_back(inFlowMapOtherKey);
   }
 }
 
+void Output::beginFlowMapping() {
+  StateStack.push_back(inFlowMapFirstKey);
+  this->newLineCheck();
+  ColumnAtMapFlowStart = Column;
+  output("{ ");
+}
+
+void Output::endFlowMapping() {
+  StateStack.pop_back();
+  this->outputUpToEndOfLine(" }");
+}
+
 void Output::beginDocuments() {
   this->outputUpToEndOfLine("---");
 }
@@ -492,7 +530,7 @@ void Output::endFlowSequence() {
 bool Output::preflightFlowElement(unsigned, void *&) {
   if (NeedFlowSequenceComma)
     output(", ");
-  if (Column > 70) {
+  if (WrapColumn && Column > WrapColumn) {
     output("\n");
     for (int i = 0; i < ColumnAtFlowStart; ++i)
       output(" ");
@@ -584,6 +622,24 @@ void Output::scalarString(StringRef &S, bool MustQuote) {
   this->outputUpToEndOfLine("'"); // Ending single quote.
 }
 
+void Output::blockScalarString(StringRef &S) {
+  if (!StateStack.empty())
+    newLineCheck();
+  output(" |");
+  outputNewLine();
+
+  unsigned Indent = StateStack.empty() ? 1 : StateStack.size();
+
+  auto Buffer = MemoryBuffer::getMemBuffer(S, "", false);
+  for (line_iterator Lines(*Buffer, false); !Lines.is_at_end(); ++Lines) {
+    for (unsigned I = 0; I < Indent; ++I) {
+      output("  ");
+    }
+    output(*Lines);
+    outputNewLine();
+  }
+}
+
 void Output::setError(const Twine &message) {
 }
 
@@ -607,7 +663,9 @@ void Output::output(StringRef s) {
 
 void Output::outputUpToEndOfLine(StringRef s) {
   this->output(s);
-  if (StateStack.empty() || StateStack.back() != inFlowSeq)
+  if (StateStack.empty() || (StateStack.back() != inFlowSeq &&
+                             StateStack.back() != inFlowMapFirstKey &&
+                             StateStack.back() != inFlowMapOtherKey))
     NeedsNewLine = true;
 }
 
@@ -633,7 +691,9 @@ void Output::newLineCheck() {
 
   if (StateStack.back() == inSeq) {
     OutputDash = true;
-  } else if ((StateStack.size() > 1) && (StateStack.back() == inMapFirstKey) &&
+  } else if ((StateStack.size() > 1) && ((StateStack.back() == inMapFirstKey) ||
+             (StateStack.back() == inFlowSeq) ||
+             (StateStack.back() == inFlowMapFirstKey)) &&
              (StateStack[StateStack.size() - 2] == inSeq)) {
     --Indent;
     OutputDash = true;
@@ -658,6 +718,20 @@ void Output::paddedKey(StringRef key) {
     output(" ");
 }
 
+void Output::flowKey(StringRef Key) {
+  if (StateStack.back() == inFlowMapOtherKey)
+    output(", ");
+  if (WrapColumn && Column > WrapColumn) {
+    output("\n");
+    for (int I = 0; I < ColumnAtMapFlowStart; ++I)
+      output(" ");
+    Column = ColumnAtMapFlowStart;
+    output("  ");
+  }
+  output(Key);
+  output(": ");
+}
+
 //===----------------------------------------------------------------------===//
 //  traits for built-in types
 //===----------------------------------------------------------------------===//
diff --git a/lib/Support/raw_ostream.cpp b/lib/Support/raw_ostream.cpp
index 6f9f910cad9..42f830bbf0f 100644
--- a/lib/Support/raw_ostream.cpp
+++ b/lib/Support/raw_ostream.cpp
@@ -630,7 +630,8 @@ uint64_t raw_fd_ostream::seek(uint64_t off) {
   return pos;
 }
 
-void raw_fd_ostream::pwrite(const char *Ptr, size_t Size, uint64_t Offset) {
+void raw_fd_ostream::pwrite_impl(const char *Ptr, size_t Size,
+                                 uint64_t Offset) {
   uint64_t Pos = tell();
   seek(Offset);
   write(Ptr, Size);
@@ -781,14 +782,9 @@ raw_svector_ostream::~raw_svector_ostream() {
   flush();
 }
 
-void raw_svector_ostream::pwrite(const char *Ptr, size_t Size,
-                                 uint64_t Offset) {
+void raw_svector_ostream::pwrite_impl(const char *Ptr, size_t Size,
+                                      uint64_t Offset) {
   flush();
-
-  uint64_t End = Offset + Size;
-  if (End > OS.size())
-    OS.resize(End);
-
   memcpy(OS.begin() + Offset, Ptr, Size);
 }
 
@@ -819,7 +815,7 @@ void raw_svector_ostream::write_impl(const char *Ptr, size_t Size) {
 }
 
 uint64_t raw_svector_ostream::current_pos() const {
-   return OS.size();
+  return OS.size();
 }
 
 StringRef raw_svector_ostream::str() {
@@ -847,4 +843,5 @@ uint64_t raw_null_ostream::current_pos() const {
   return 0;
 }
 
-void raw_null_ostream::pwrite(const char *Ptr, size_t Size, uint64_t Offset) {}
+void raw_null_ostream::pwrite_impl(const char *Ptr, size_t Size,
+                                   uint64_t Offset) {}
diff --git a/lib/TableGen/Error.cpp b/lib/TableGen/Error.cpp
index 8d9ae20dd52..fd089356625 100644
--- a/lib/TableGen/Error.cpp
+++ b/lib/TableGen/Error.cpp
@@ -14,6 +14,7 @@
 
 #include "llvm/TableGen/Error.h"
 #include "llvm/ADT/Twine.h"
+#include "llvm/Support/Signals.h"
 #include "llvm/Support/raw_ostream.h"
 #include <cstdlib>
 
@@ -64,11 +65,15 @@ void PrintError(const Twine &Msg) {
 
 void PrintFatalError(const Twine &Msg) {
   PrintError(Msg);
+  // The following call runs the file cleanup handlers.
+  sys::RunInterruptHandlers();
   std::exit(1);
 }
 
 void PrintFatalError(ArrayRef<SMLoc> ErrorLoc, const Twine &Msg) {
   PrintError(ErrorLoc, Msg);
+  // The following call runs the file cleanup handlers.
+  sys::RunInterruptHandlers();
   std::exit(1);
 }
 
diff --git a/lib/TableGen/Main.cpp b/lib/TableGen/Main.cpp
index c44045152a4..bb590c7d87f 100644
--- a/lib/TableGen/Main.cpp
+++ b/lib/TableGen/Main.cpp
@@ -15,37 +15,35 @@
 //
 //===----------------------------------------------------------------------===//
 
+#include "llvm/TableGen/Main.h"
 #include "TGParser.h"
 #include "llvm/Support/CommandLine.h"
 #include "llvm/Support/FileSystem.h"
 #include "llvm/Support/MemoryBuffer.h"
 #include "llvm/Support/ToolOutputFile.h"
 #include "llvm/TableGen/Error.h"
-#include "llvm/TableGen/Main.h"
 #include "llvm/TableGen/Record.h"
 #include <algorithm>
 #include <cstdio>
 #include <system_error>
 using namespace llvm;
 
-namespace {
-  cl::opt<std::string>
-  OutputFilename("o", cl::desc("Output filename"), cl::value_desc("filename"),
-                 cl::init("-"));
+static cl::opt<std::string>
+OutputFilename("o", cl::desc("Output filename"), cl::value_desc("filename"),
+               cl::init("-"));
 
-  cl::opt<std::string>
-  DependFilename("d",
-                 cl::desc("Dependency filename"),
-                 cl::value_desc("filename"),
-                 cl::init(""));
+static cl::opt<std::string>
+DependFilename("d",
+               cl::desc("Dependency filename"),
+               cl::value_desc("filename"),
+               cl::init(""));
 
-  cl::opt<std::string>
-  InputFilename(cl::Positional, cl::desc("<input file>"), cl::init("-"));
+static cl::opt<std::string>
+InputFilename(cl::Positional, cl::desc("<input file>"), cl::init("-"));
 
-  cl::list<std::string>
-  IncludeDirs("I", cl::desc("Directory of include files"),
-              cl::value_desc("directory"), cl::Prefix);
-}
+static cl::list<std::string>
+IncludeDirs("I", cl::desc("Directory of include files"),
+            cl::value_desc("directory"), cl::Prefix);
 
 /// \brief Create a dependency file for `-d` option.
 ///
@@ -72,9 +70,7 @@ static int createDependencyFile(const TGParser &Parser, const char *argv0) {
   return 0;
 }
 
-namespace llvm {
-
-int TableGenMain(char *argv0, TableGenMainFn *MainFn) {
+int llvm::TableGenMain(char *argv0, TableGenMainFn *MainFn) {
   RecordKeeper Records;
 
   // Parse the input file.
@@ -122,5 +118,3 @@ int TableGenMain(char *argv0, TableGenMainFn *MainFn) {
   Out.keep();
   return 0;
 }
-
-}
diff --git a/lib/TableGen/Record.cpp b/lib/TableGen/Record.cpp
index 8a8f0ee3a08..271e52567a2 100644
--- a/lib/TableGen/Record.cpp
+++ b/lib/TableGen/Record.cpp
@@ -86,262 +86,69 @@ IntRecTy IntRecTy::Shared;
 StringRecTy StringRecTy::Shared;
 DagRecTy DagRecTy::Shared;
 
-void RecTy::anchor() { }
 void RecTy::dump() const { print(errs()); }
 
 ListRecTy *RecTy::getListTy() {
   if (!ListTy)
-    ListTy = new ListRecTy(this);
-  return ListTy;
+    ListTy.reset(new ListRecTy(this));
+  return ListTy.get();
 }
 
-bool RecTy::baseClassOf(const RecTy *RHS) const{
-  assert (RHS && "NULL pointer");
+bool RecTy::typeIsConvertibleTo(const RecTy *RHS) const {
+  assert(RHS && "NULL pointer");
   return Kind == RHS->getRecTyKind();
 }
 
-Init *BitRecTy::convertValue(BitsInit *BI) {
-  if (BI->getNumBits() != 1) return nullptr; // Only accept if just one bit!
-  return BI->getBit(0);
-}
-
-Init *BitRecTy::convertValue(IntInit *II) {
-  int64_t Val = II->getValue();
-  if (Val != 0 && Val != 1) return nullptr;  // Only accept 0 or 1 for a bit!
-
-  return BitInit::get(Val != 0);
-}
-
-Init *BitRecTy::convertValue(TypedInit *VI) {
-  RecTy *Ty = VI->getType();
-  if (isa<BitRecTy>(Ty))
-    return VI;  // Accept variable if it is already of bit type!
-  if (auto *BitsTy = dyn_cast<BitsRecTy>(Ty))
-    // Accept only bits<1> expression.
-    return BitsTy->getNumBits() == 1 ? VI : nullptr;
-  // Ternary !if can be converted to bit, but only if both sides are
-  // convertible to a bit.
-  if (TernOpInit *TOI = dyn_cast<TernOpInit>(VI)) {
-    if (TOI->getOpcode() != TernOpInit::TernaryOp::IF)
-      return nullptr;
-    if (!TOI->getMHS()->convertInitializerTo(BitRecTy::get()) ||
-        !TOI->getRHS()->convertInitializerTo(BitRecTy::get()))
-      return nullptr;
-    return TOI;
-  }
-  return nullptr;
-}
-
-bool BitRecTy::baseClassOf(const RecTy *RHS) const{
-  if(RecTy::baseClassOf(RHS) || getRecTyKind() == IntRecTyKind)
+bool BitRecTy::typeIsConvertibleTo(const RecTy *RHS) const{
+  if (RecTy::typeIsConvertibleTo(RHS) || RHS->getRecTyKind() == IntRecTyKind)
     return true;
-  if(const BitsRecTy *BitsTy = dyn_cast<BitsRecTy>(RHS))
+  if (const BitsRecTy *BitsTy = dyn_cast<BitsRecTy>(RHS))
     return BitsTy->getNumBits() == 1;
   return false;
 }
 
 BitsRecTy *BitsRecTy::get(unsigned Sz) {
-  static std::vector<BitsRecTy*> Shared;
+  static std::vector<std::unique_ptr<BitsRecTy>> Shared;
   if (Sz >= Shared.size())
     Shared.resize(Sz + 1);
-  BitsRecTy *&Ty = Shared[Sz];
+  std::unique_ptr<BitsRecTy> &Ty = Shared[Sz];
   if (!Ty)
-    Ty = new BitsRecTy(Sz);
-  return Ty;
+    Ty.reset(new BitsRecTy(Sz));
+  return Ty.get();
 }
 
 std::string BitsRecTy::getAsString() const {
   return "bits<" + utostr(Size) + ">";
 }
 
-Init *BitsRecTy::convertValue(UnsetInit *UI) {
-  SmallVector<Init *, 16> NewBits(Size);
-
-  for (unsigned i = 0; i != Size; ++i)
-    NewBits[i] = UnsetInit::get();
-
-  return BitsInit::get(NewBits);
-}
-
-Init *BitsRecTy::convertValue(BitInit *UI) {
-  if (Size != 1) return nullptr;  // Can only convert single bit.
-  return BitsInit::get(UI);
-}
-
-/// canFitInBitfield - Return true if the number of bits is large enough to hold
-/// the integer value.
-static bool canFitInBitfield(int64_t Value, unsigned NumBits) {
-  // For example, with NumBits == 4, we permit Values from [-7 .. 15].
-  return (NumBits >= sizeof(Value) * 8) ||
-         (Value >> NumBits == 0) || (Value >> (NumBits-1) == -1);
-}
-
-/// convertValue from Int initializer to bits type: Split the integer up into the
-/// appropriate bits.
-///
-Init *BitsRecTy::convertValue(IntInit *II) {
-  int64_t Value = II->getValue();
-  // Make sure this bitfield is large enough to hold the integer value.
-  if (!canFitInBitfield(Value, Size))
-    return nullptr;
-
-  SmallVector<Init *, 16> NewBits(Size);
-
-  for (unsigned i = 0; i != Size; ++i)
-    NewBits[i] = BitInit::get(Value & (1LL << i));
-
-  return BitsInit::get(NewBits);
-}
-
-Init *BitsRecTy::convertValue(BitsInit *BI) {
-  // If the number of bits is right, return it.  Otherwise we need to expand or
-  // truncate.
-  if (BI->getNumBits() == Size) return BI;
-  return nullptr;
-}
-
-Init *BitsRecTy::convertValue(TypedInit *VI) {
-  if (Size == 1 && isa<BitRecTy>(VI->getType()))
-    return BitsInit::get(VI);
-
-  if (VI->getType()->typeIsConvertibleTo(this)) {
-    SmallVector<Init *, 16> NewBits(Size);
-
-    for (unsigned i = 0; i != Size; ++i)
-      NewBits[i] = VarBitInit::get(VI, i);
-    return BitsInit::get(NewBits);
-  }
-
-  return nullptr;
-}
-
-bool BitsRecTy::baseClassOf(const RecTy *RHS) const{
-  if (RecTy::baseClassOf(RHS)) //argument and the receiver are the same type
+bool BitsRecTy::typeIsConvertibleTo(const RecTy *RHS) const {
+  if (RecTy::typeIsConvertibleTo(RHS)) //argument and the sender are same type
     return cast<BitsRecTy>(RHS)->Size == Size;
   RecTyKind kind = RHS->getRecTyKind();
   return (kind == BitRecTyKind && Size == 1) || (kind == IntRecTyKind);
 }
 
-Init *IntRecTy::convertValue(BitInit *BI) {
-  return IntInit::get(BI->getValue());
-}
-
-Init *IntRecTy::convertValue(BitsInit *BI) {
-  int64_t Result = 0;
-  for (unsigned i = 0, e = BI->getNumBits(); i != e; ++i)
-    if (BitInit *Bit = dyn_cast<BitInit>(BI->getBit(i))) {
-      Result |= Bit->getValue() << i;
-    } else {
-      return nullptr;
-    }
-  return IntInit::get(Result);
-}
-
-Init *IntRecTy::convertValue(TypedInit *TI) {
-  if (TI->getType()->typeIsConvertibleTo(this))
-    return TI;  // Accept variable if already of the right type!
-  return nullptr;
-}
-
-bool IntRecTy::baseClassOf(const RecTy *RHS) const{
+bool IntRecTy::typeIsConvertibleTo(const RecTy *RHS) const {
   RecTyKind kind = RHS->getRecTyKind();
   return kind==BitRecTyKind || kind==BitsRecTyKind || kind==IntRecTyKind;
 }
 
-Init *StringRecTy::convertValue(UnOpInit *BO) {
-  if (BO->getOpcode() == UnOpInit::CAST) {
-    Init *L = BO->getOperand()->convertInitializerTo(this);
-    if (!L) return nullptr;
-    if (L != BO->getOperand())
-      return UnOpInit::get(UnOpInit::CAST, L, new StringRecTy);
-    return BO;
-  }
-
-  return convertValue((TypedInit*)BO);
-}
-
-Init *StringRecTy::convertValue(BinOpInit *BO) {
-  if (BO->getOpcode() == BinOpInit::STRCONCAT) {
-    Init *L = BO->getLHS()->convertInitializerTo(this);
-    Init *R = BO->getRHS()->convertInitializerTo(this);
-    if (!L || !R) return nullptr;
-    if (L != BO->getLHS() || R != BO->getRHS())
-      return BinOpInit::get(BinOpInit::STRCONCAT, L, R, new StringRecTy);
-    return BO;
-  }
-
-  return convertValue((TypedInit*)BO);
-}
-
-
-Init *StringRecTy::convertValue(TypedInit *TI) {
-  if (isa<StringRecTy>(TI->getType()))
-    return TI;  // Accept variable if already of the right type!
-  return nullptr;
+std::string StringRecTy::getAsString() const {
+  return "string";
 }
 
 std::string ListRecTy::getAsString() const {
   return "list<" + Ty->getAsString() + ">";
 }
 
-Init *ListRecTy::convertValue(ListInit *LI) {
-  std::vector<Init*> Elements;
-
-  // Verify that all of the elements of the list are subclasses of the
-  // appropriate class!
-  for (unsigned i = 0, e = LI->getSize(); i != e; ++i)
-    if (Init *CI = LI->getElement(i)->convertInitializerTo(Ty))
-      Elements.push_back(CI);
-    else
-      return nullptr;
-
-  if (!isa<ListRecTy>(LI->getType()))
-    return nullptr;
-
-  return ListInit::get(Elements, this);
-}
-
-Init *ListRecTy::convertValue(TypedInit *TI) {
-  // Ensure that TI is compatible with our class.
-  if (ListRecTy *LRT = dyn_cast<ListRecTy>(TI->getType()))
-    if (LRT->getElementType()->typeIsConvertibleTo(getElementType()))
-      return TI;
-  return nullptr;
-}
-
-bool ListRecTy::baseClassOf(const RecTy *RHS) const{
-  if(const ListRecTy* ListTy = dyn_cast<ListRecTy>(RHS))
-    return ListTy->getElementType()->typeIsConvertibleTo(Ty);
+bool ListRecTy::typeIsConvertibleTo(const RecTy *RHS) const {
+  if (const auto *ListTy = dyn_cast<ListRecTy>(RHS))
+    return Ty->typeIsConvertibleTo(ListTy->getElementType());
   return false;
 }
 
-Init *DagRecTy::convertValue(TypedInit *TI) {
-  if (TI->getType()->typeIsConvertibleTo(this))
-    return TI;
-  return nullptr;
-}
-
-Init *DagRecTy::convertValue(UnOpInit *BO) {
-  if (BO->getOpcode() == UnOpInit::CAST) {
-    Init *L = BO->getOperand()->convertInitializerTo(this);
-    if (!L) return nullptr;
-    if (L != BO->getOperand())
-      return UnOpInit::get(UnOpInit::CAST, L, new DagRecTy);
-    return BO;
-  }
-  return nullptr;
-}
-
-Init *DagRecTy::convertValue(BinOpInit *BO) {
-  if (BO->getOpcode() == BinOpInit::CONCAT) {
-    Init *L = BO->getLHS()->convertInitializerTo(this);
-    Init *R = BO->getRHS()->convertInitializerTo(this);
-    if (!L || !R) return nullptr;
-    if (L != BO->getLHS() || R != BO->getRHS())
-      return BinOpInit::get(BinOpInit::CONCAT, L, R, new DagRecTy);
-    return BO;
-  }
-  return nullptr;
+std::string DagRecTy::getAsString() const {
+  return "dag";
 }
 
 RecordRecTy *RecordRecTy::get(Record *R) {
@@ -352,40 +159,23 @@ std::string RecordRecTy::getAsString() const {
   return Rec->getName();
 }
 
-Init *RecordRecTy::convertValue(DefInit *DI) {
-  // Ensure that DI is a subclass of Rec.
-  if (!DI->getDef()->isSubClassOf(Rec))
-    return nullptr;
-  return DI;
-}
-
-Init *RecordRecTy::convertValue(TypedInit *TI) {
-  // Ensure that TI is compatible with Rec.
-  if (RecordRecTy *RRT = dyn_cast<RecordRecTy>(TI->getType()))
-    if (RRT->getRecord()->isSubClassOf(getRecord()) ||
-        RRT->getRecord() == getRecord())
-      return TI;
-  return nullptr;
-}
-
-bool RecordRecTy::baseClassOf(const RecTy *RHS) const{
+bool RecordRecTy::typeIsConvertibleTo(const RecTy *RHS) const {
   const RecordRecTy *RTy = dyn_cast<RecordRecTy>(RHS);
   if (!RTy)
     return false;
 
-  if (Rec == RTy->getRecord() || RTy->getRecord()->isSubClassOf(Rec))
+  if (RTy->getRecord() == Rec || Rec->isSubClassOf(RTy->getRecord()))
     return true;
 
-  const std::vector<Record*> &SC = Rec->getSuperClasses();
-  for (unsigned i = 0, e = SC.size(); i != e; ++i)
-    if (RTy->getRecord()->isSubClassOf(SC[i]))
+  for (Record *SC : RTy->getRecord()->getSuperClasses())
+    if (Rec->isSubClassOf(SC))
       return true;
 
   return false;
 }
 
 /// resolveTypes - Find a common type that T1 and T2 convert to.
-/// Return 0 if no such type exists.
+/// Return null if no such type exists.
 ///
 RecTy *llvm::resolveTypes(RecTy *T1, RecTy *T2) {
   if (T1->typeIsConvertibleTo(T2))
@@ -396,38 +186,20 @@ RecTy *llvm::resolveTypes(RecTy *T1, RecTy *T2) {
   // If one is a Record type, check superclasses
   if (RecordRecTy *RecTy1 = dyn_cast<RecordRecTy>(T1)) {
     // See if T2 inherits from a type T1 also inherits from
-    const std::vector<Record *> &T1SuperClasses =
-      RecTy1->getRecord()->getSuperClasses();
-    for(std::vector<Record *>::const_iterator i = T1SuperClasses.begin(),
-          iend = T1SuperClasses.end();
-        i != iend;
-        ++i) {
-      RecordRecTy *SuperRecTy1 = RecordRecTy::get(*i);
+    for (Record *SuperRec1 : RecTy1->getRecord()->getSuperClasses()) {
+      RecordRecTy *SuperRecTy1 = RecordRecTy::get(SuperRec1);
       RecTy *NewType1 = resolveTypes(SuperRecTy1, T2);
-      if (NewType1) {
-        if (NewType1 != SuperRecTy1) {
-          delete SuperRecTy1;
-        }
+      if (NewType1)
         return NewType1;
-      }
     }
   }
   if (RecordRecTy *RecTy2 = dyn_cast<RecordRecTy>(T2)) {
     // See if T1 inherits from a type T2 also inherits from
-    const std::vector<Record *> &T2SuperClasses =
-      RecTy2->getRecord()->getSuperClasses();
-    for (std::vector<Record *>::const_iterator i = T2SuperClasses.begin(),
-          iend = T2SuperClasses.end();
-        i != iend;
-        ++i) {
-      RecordRecTy *SuperRecTy2 = RecordRecTy::get(*i);
+    for (Record *SuperRec2 : RecTy2->getRecord()->getSuperClasses()) {
+      RecordRecTy *SuperRecTy2 = RecordRecTy::get(SuperRec2);
       RecTy *NewType2 = resolveTypes(T1, SuperRecTy2);
-      if (NewType2) {
-        if (NewType2 != SuperRecTy2) {
-          delete SuperRecTy2;
-        }
+      if (NewType2)
         return NewType2;
-      }
     }
   }
   return nullptr;
@@ -441,14 +213,24 @@ RecTy *llvm::resolveTypes(RecTy *T1, RecTy *T2) {
 void Init::anchor() { }
 void Init::dump() const { return print(errs()); }
 
-void UnsetInit::anchor() { }
-
 UnsetInit *UnsetInit::get() {
   static UnsetInit TheInit;
   return &TheInit;
 }
 
-void BitInit::anchor() { }
+Init *UnsetInit::convertInitializerTo(RecTy *Ty) const {
+  if (auto *BRT = dyn_cast<BitsRecTy>(Ty)) {
+    SmallVector<Init *, 16> NewBits(BRT->getNumBits());
+
+    for (unsigned i = 0; i != BRT->getNumBits(); ++i)
+      NewBits[i] = UnsetInit::get();
+
+    return BitsInit::get(NewBits);
+  }
+
+  // All other types can just be returned.
+  return const_cast<UnsetInit *>(this);
+}
 
 BitInit *BitInit::get(bool V) {
   static BitInit True(true);
@@ -457,20 +239,33 @@ BitInit *BitInit::get(bool V) {
   return V ? &True : &False;
 }
 
+Init *BitInit::convertInitializerTo(RecTy *Ty) const {
+  if (isa<BitRecTy>(Ty))
+    return const_cast<BitInit *>(this);
+
+  if (isa<IntRecTy>(Ty))
+    return IntInit::get(getValue());
+
+  if (auto *BRT = dyn_cast<BitsRecTy>(Ty)) {
+    // Can only convert single bit.
+    if (BRT->getNumBits() == 1)
+      return BitsInit::get(const_cast<BitInit *>(this));
+  }
+
+  return nullptr;
+}
+
 static void
 ProfileBitsInit(FoldingSetNodeID &ID, ArrayRef<Init *> Range) {
   ID.AddInteger(Range.size());
 
-  for (ArrayRef<Init *>::iterator i = Range.begin(),
-         iend = Range.end();
-       i != iend;
-       ++i)
-    ID.AddPointer(*i);
+  for (Init *I : Range)
+    ID.AddPointer(I);
 }
 
 BitsInit *BitsInit::get(ArrayRef<Init *> Range) {
-  typedef FoldingSet<BitsInit> Pool;
-  static Pool ThePool;  
+  static FoldingSet<BitsInit> ThePool;
+  static std::vector<std::unique_ptr<BitsInit>> TheActualPool;
 
   FoldingSetNodeID ID;
   ProfileBitsInit(ID, Range);
@@ -481,7 +276,7 @@ BitsInit *BitsInit::get(ArrayRef<Init *> Range) {
 
   BitsInit *I = new BitsInit(Range);
   ThePool.InsertNode(I, IP);
-
+  TheActualPool.push_back(std::unique_ptr<BitsInit>(I));
   return I;
 }
 
@@ -489,6 +284,32 @@ void BitsInit::Profile(FoldingSetNodeID &ID) const {
   ProfileBitsInit(ID, Bits);
 }
 
+Init *BitsInit::convertInitializerTo(RecTy *Ty) const {
+  if (isa<BitRecTy>(Ty)) {
+    if (getNumBits() != 1) return nullptr; // Only accept if just one bit!
+    return getBit(0);
+  }
+
+  if (auto *BRT = dyn_cast<BitsRecTy>(Ty)) {
+    // If the number of bits is right, return it.  Otherwise we need to expand
+    // or truncate.
+    if (getNumBits() != BRT->getNumBits()) return nullptr;
+    return const_cast<BitsInit *>(this);
+  }
+
+  if (isa<IntRecTy>(Ty)) {
+    int64_t Result = 0;
+    for (unsigned i = 0, e = getNumBits(); i != e; ++i)
+      if (auto *Bit = dyn_cast<BitInit>(getBit(i)))
+        Result |= static_cast<int64_t>(Bit->getValue()) << i;
+      else
+        return nullptr;
+    return IntInit::get(Result);
+  }
+
+  return nullptr;
+}
+
 Init *
 BitsInit::convertInitializerBitRange(const std::vector<unsigned> &Bits) const {
   SmallVector<Init *, 16> NewBits(Bits.size());
@@ -517,7 +338,7 @@ std::string BitsInit::getAsString() const {
 // bits initializer will resolve into VarBitInit to keep the field name and bit
 // number used in targets with fixed insn length.
 static Init *fixBitInit(const RecordVal *RV, Init *Before, Init *After) {
-  if (RV || After != UnsetInit::get())
+  if (RV || !isa<UnsetInit>(After))
     return After;
   return Before;
 }
@@ -570,33 +391,52 @@ Init *BitsInit::resolveReferences(Record &R, const RecordVal *RV) const {
   return const_cast<BitsInit *>(this);
 }
 
-namespace {
-  template<typename T>
-  class Pool : public T {
-  public:
-    ~Pool();
-  };
-  template<typename T>
-  Pool<T>::~Pool() {
-    for (typename T::iterator I = this->begin(), E = this->end(); I != E; ++I) {
-      typename T::value_type &Item = *I;
-      delete Item.second;
-    }
-  }
-}
-
 IntInit *IntInit::get(int64_t V) {
-  static Pool<DenseMap<int64_t, IntInit *> > ThePool;
+  static DenseMap<int64_t, std::unique_ptr<IntInit>> ThePool;
 
-  IntInit *&I = ThePool[V];
-  if (!I) I = new IntInit(V);
-  return I;
+  std::unique_ptr<IntInit> &I = ThePool[V];
+  if (!I) I.reset(new IntInit(V));
+  return I.get();
 }
 
 std::string IntInit::getAsString() const {
   return itostr(Value);
 }
 
+/// canFitInBitfield - Return true if the number of bits is large enough to hold
+/// the integer value.
+static bool canFitInBitfield(int64_t Value, unsigned NumBits) {
+  // For example, with NumBits == 4, we permit Values from [-7 .. 15].
+  return (NumBits >= sizeof(Value) * 8) ||
+         (Value >> NumBits == 0) || (Value >> (NumBits-1) == -1);
+}
+
+Init *IntInit::convertInitializerTo(RecTy *Ty) const {
+  if (isa<IntRecTy>(Ty))
+    return const_cast<IntInit *>(this);
+
+  if (isa<BitRecTy>(Ty)) {
+    int64_t Val = getValue();
+    if (Val != 0 && Val != 1) return nullptr;  // Only accept 0 or 1 for a bit!
+    return BitInit::get(Val != 0);
+  }
+
+  if (auto *BRT = dyn_cast<BitsRecTy>(Ty)) {
+    int64_t Value = getValue();
+    // Make sure this bitfield is large enough to hold the integer value.
+    if (!canFitInBitfield(Value, BRT->getNumBits()))
+      return nullptr;
+
+    SmallVector<Init *, 16> NewBits(BRT->getNumBits());
+    for (unsigned i = 0; i != BRT->getNumBits(); ++i)
+      NewBits[i] = BitInit::get(Value & (1LL << i));
+
+    return BitsInit::get(NewBits);
+  }
+
+  return nullptr;
+}
+
 Init *
 IntInit::convertInitializerBitRange(const std::vector<unsigned> &Bits) const {
   SmallVector<Init *, 16> NewBits(Bits.size());
@@ -610,14 +450,19 @@ IntInit::convertInitializerBitRange(const std::vector<unsigned> &Bits) const {
   return BitsInit::get(NewBits);
 }
 
-void StringInit::anchor() { }
-
 StringInit *StringInit::get(StringRef V) {
-  static Pool<StringMap<StringInit *> > ThePool;
+  static StringMap<std::unique_ptr<StringInit>> ThePool;
 
-  StringInit *&I = ThePool[V];
-  if (!I) I = new StringInit(V);
-  return I;
+  std::unique_ptr<StringInit> &I = ThePool[V];
+  if (!I) I.reset(new StringInit(V));
+  return I.get();
+}
+
+Init *StringInit::convertInitializerTo(RecTy *Ty) const {
+  if (isa<StringRecTy>(Ty))
+    return const_cast<StringInit *>(this);
+
+  return nullptr;
 }
 
 static void ProfileListInit(FoldingSetNodeID &ID,
@@ -626,16 +471,12 @@ static void ProfileListInit(FoldingSetNodeID &ID,
   ID.AddInteger(Range.size());
   ID.AddPointer(EltTy);
 
-  for (ArrayRef<Init *>::iterator i = Range.begin(),
-         iend = Range.end();
-       i != iend;
-       ++i)
-    ID.AddPointer(*i);
+  for (Init *I : Range)
+    ID.AddPointer(I);
 }
 
 ListInit *ListInit::get(ArrayRef<Init *> Range, RecTy *EltTy) {
-  typedef FoldingSet<ListInit> Pool;
-  static Pool ThePool;
+  static FoldingSet<ListInit> ThePool;
   static std::vector<std::unique_ptr<ListInit>> TheActualPool;
 
   FoldingSetNodeID ID;
@@ -652,18 +493,35 @@ ListInit *ListInit::get(ArrayRef<Init *> Range, RecTy *EltTy) {
 }
 
 void ListInit::Profile(FoldingSetNodeID &ID) const {
-  ListRecTy *ListType = dyn_cast<ListRecTy>(getType());
-  assert(ListType && "Bad type for ListInit!");
-  RecTy *EltTy = ListType->getElementType();
+  RecTy *EltTy = cast<ListRecTy>(getType())->getElementType();
 
   ProfileListInit(ID, Values, EltTy);
 }
 
+Init *ListInit::convertInitializerTo(RecTy *Ty) const {
+  if (auto *LRT = dyn_cast<ListRecTy>(Ty)) {
+    std::vector<Init*> Elements;
+
+    // Verify that all of the elements of the list are subclasses of the
+    // appropriate class!
+    for (Init *I : getValues())
+      if (Init *CI = I->convertInitializerTo(LRT->getElementType()))
+        Elements.push_back(CI);
+      else
+        return nullptr;
+
+    if (isa<ListRecTy>(getType()))
+      return ListInit::get(Elements, Ty);
+  }
+
+  return nullptr;
+}
+
 Init *
 ListInit::convertInitListSlice(const std::vector<unsigned> &Elements) const {
   std::vector<Init*> Vals;
   for (unsigned i = 0, e = Elements.size(); i != e; ++i) {
-    if (Elements[i] >= getSize())
+    if (Elements[i] >= size())
       return nullptr;
     Vals.push_back(getElement(Elements[i]));
   }
@@ -680,12 +538,11 @@ Record *ListInit::getElementAsRecord(unsigned i) const {
 
 Init *ListInit::resolveReferences(Record &R, const RecordVal *RV) const {
   std::vector<Init*> Resolved;
-  Resolved.reserve(getSize());
+  Resolved.reserve(size());
   bool Changed = false;
 
-  for (unsigned i = 0, e = getSize(); i != e; ++i) {
+  for (Init *CurElt : getValues()) {
     Init *E;
-    Init *CurElt = getElement(i);
 
     do {
       E = CurElt;
@@ -702,7 +559,7 @@ Init *ListInit::resolveReferences(Record &R, const RecordVal *RV) const {
 
 Init *ListInit::resolveListElementReference(Record &R, const RecordVal *IRV,
                                             unsigned Elt) const {
-  if (Elt >= getSize())
+  if (Elt >= size())
     return nullptr;  // Out of range reference.
   Init *E = getElement(Elt);
   // If the element is set to some value, or if we are resolving a reference
@@ -731,14 +588,10 @@ Init *OpInit::resolveListElementReference(Record &R, const RecordVal *IRV,
   }
 
   if (Resolved != this) {
-    TypedInit *Typed = dyn_cast<TypedInit>(Resolved);
-    assert(Typed && "Expected typed init for list reference");
-    if (Typed) {
-      Init *New = Typed->resolveListElementReference(R, IRV, Elt);
-      if (New)
-        return New;
-      return VarListElementInit::get(Typed, Elt);
-    }
+    TypedInit *Typed = cast<TypedInit>(Resolved);
+    if (Init *New = Typed->resolveListElementReference(R, IRV, Elt))
+      return New;
+    return VarListElementInit::get(Typed, Elt);
   }
 
   return nullptr;
@@ -752,24 +605,24 @@ Init *OpInit::getBit(unsigned Bit) const {
 
 UnOpInit *UnOpInit::get(UnaryOp opc, Init *lhs, RecTy *Type) {
   typedef std::pair<std::pair<unsigned, Init *>, RecTy *> Key;
-  static Pool<DenseMap<Key, UnOpInit *> > ThePool;
+  static DenseMap<Key, std::unique_ptr<UnOpInit>> ThePool;
 
   Key TheKey(std::make_pair(std::make_pair(opc, lhs), Type));
 
-  UnOpInit *&I = ThePool[TheKey];
-  if (!I) I = new UnOpInit(opc, lhs, Type);
-  return I;
+  std::unique_ptr<UnOpInit> &I = ThePool[TheKey];
+  if (!I) I.reset(new UnOpInit(opc, lhs, Type));
+  return I.get();
 }
 
 Init *UnOpInit::Fold(Record *CurRec, MultiClass *CurMultiClass) const {
   switch (getOpcode()) {
   case CAST: {
-    if (getType()->getAsString() == "string") {
+    if (isa<StringRecTy>(getType())) {
       if (StringInit *LHSs = dyn_cast<StringInit>(LHS))
         return LHSs;
 
       if (DefInit *LHSd = dyn_cast<DefInit>(LHS))
-        return StringInit::get(LHSd->getDef()->getName());
+        return StringInit::get(LHSd->getAsString());
 
       if (IntInit *LHSi = dyn_cast<IntInit>(LHS))
         return StringInit::get(LHSi->getAsString());
@@ -787,7 +640,7 @@ Init *UnOpInit::Fold(Record *CurRec, MultiClass *CurMultiClass) const {
 
           Init *TemplateArgName = QualifyName(*CurRec, CurMultiClass, Name,
                                               ":");
-      
+
           if (CurRec->isTemplateArg(TemplateArgName)) {
             const RecordVal *RV = CurRec->getValue(TemplateArgName);
             assert(RV && "Template arg doesn't exist??");
@@ -800,7 +653,8 @@ Init *UnOpInit::Fold(Record *CurRec, MultiClass *CurMultiClass) const {
         }
 
         if (CurMultiClass) {
-          Init *MCName = QualifyName(CurMultiClass->Rec, CurMultiClass, Name, "::");
+          Init *MCName = QualifyName(CurMultiClass->Rec, CurMultiClass, Name,
+                                     "::");
 
           if (CurMultiClass->Rec.isTemplateArg(MCName)) {
             const RecordVal *RV = CurMultiClass->Rec.getValue(MCName);
@@ -819,45 +673,38 @@ Init *UnOpInit::Fold(Record *CurRec, MultiClass *CurMultiClass) const {
         PrintFatalError(CurRec->getLoc(),
                         "Undefined reference:'" + Name + "'\n");
       }
+
+      if (isa<IntRecTy>(getType())) {
+        if (BitsInit *BI = dyn_cast<BitsInit>(LHS)) {
+          if (Init *NewInit = BI->convertInitializerTo(IntRecTy::get()))
+            return NewInit;
+          break;
+        }
+      }
     }
     break;
   }
   case HEAD: {
     if (ListInit *LHSl = dyn_cast<ListInit>(LHS)) {
-      assert(LHSl->getSize() != 0 && "Empty list in car");
+      assert(!LHSl->empty() && "Empty list in head");
       return LHSl->getElement(0);
     }
     break;
   }
   case TAIL: {
     if (ListInit *LHSl = dyn_cast<ListInit>(LHS)) {
-      assert(LHSl->getSize() != 0 && "Empty list in cdr");
+      assert(!LHSl->empty() && "Empty list in tail");
       // Note the +1.  We can't just pass the result of getValues()
       // directly.
-      ArrayRef<Init *>::iterator begin = LHSl->getValues().begin()+1;
-      ArrayRef<Init *>::iterator end   = LHSl->getValues().end();
-      ListInit *Result =
-        ListInit::get(ArrayRef<Init *>(begin, end - begin),
-                      LHSl->getType());
-      return Result;
+      return ListInit::get(LHSl->getValues().slice(1), LHSl->getType());
     }
     break;
   }
   case EMPTY: {
-    if (ListInit *LHSl = dyn_cast<ListInit>(LHS)) {
-      if (LHSl->getSize() == 0) {
-        return IntInit::get(1);
-      } else {
-        return IntInit::get(0);
-      }
-    }
-    if (StringInit *LHSs = dyn_cast<StringInit>(LHS)) {
-      if (LHSs->getValue().empty()) {
-        return IntInit::get(1);
-      } else {
-        return IntInit::get(0);
-      }
-    }
+    if (ListInit *LHSl = dyn_cast<ListInit>(LHS))
+      return IntInit::get(LHSl->empty());
+    if (StringInit *LHSs = dyn_cast<StringInit>(LHS))
+      return IntInit::get(LHSs->getValue().empty());
 
     break;
   }
@@ -891,14 +738,14 @@ BinOpInit *BinOpInit::get(BinaryOp opc, Init *lhs,
     RecTy *
     > Key;
 
-  static Pool<DenseMap<Key, BinOpInit *> > ThePool;
+  static DenseMap<Key, std::unique_ptr<BinOpInit>> ThePool;
 
   Key TheKey(std::make_pair(std::make_pair(std::make_pair(opc, lhs), rhs),
                             Type));
 
-  BinOpInit *&I = ThePool[TheKey];
-  if (!I) I = new BinOpInit(opc, lhs, rhs, Type);
-  return I;
+  std::unique_ptr<BinOpInit> &I = ThePool[TheKey];
+  if (!I) I.reset(new BinOpInit(opc, lhs, rhs, Type));
+  return I.get();
 }
 
 Init *BinOpInit::Fold(Record *CurRec, MultiClass *CurMultiClass) const {
@@ -933,7 +780,7 @@ Init *BinOpInit::Fold(Record *CurRec, MultiClass *CurMultiClass) const {
       Args.insert(Args.end(), LHSs->begin(), LHSs->end());
       Args.insert(Args.end(), RHSs->begin(), RHSs->end());
       return ListInit::get(
-          Args, static_cast<ListRecTy *>(LHSs->getType())->getElementType());
+          Args, cast<ListRecTy>(LHSs->getType())->getElementType());
     }
     break;
   }
@@ -1017,9 +864,8 @@ std::string BinOpInit::getAsString() const {
   return Result + "(" + LHS->getAsString() + ", " + RHS->getAsString() + ")";
 }
 
-TernOpInit *TernOpInit::get(TernaryOp opc, Init *lhs,
-                                  Init *mhs, Init *rhs,
-                                  RecTy *Type) {
+TernOpInit *TernOpInit::get(TernaryOp opc, Init *lhs, Init *mhs, Init *rhs,
+                            RecTy *Type) {
   typedef std::pair<
     std::pair<
       std::pair<std::pair<unsigned, RecTy *>, Init *>,
@@ -1028,8 +874,7 @@ TernOpInit *TernOpInit::get(TernaryOp opc, Init *lhs,
     Init *
     > Key;
 
-  typedef DenseMap<Key, TernOpInit *> Pool;
-  static Pool ThePool;
+  static DenseMap<Key, std::unique_ptr<TernOpInit>> ThePool;
 
   Key TheKey(std::make_pair(std::make_pair(std::make_pair(std::make_pair(opc,
                                                                          Type),
@@ -1037,9 +882,9 @@ TernOpInit *TernOpInit::get(TernaryOp opc, Init *lhs,
                                            mhs),
                             rhs));
 
-  TernOpInit *&I = ThePool[TheKey];
-  if (!I) I = new TernOpInit(opc, lhs, mhs, rhs, Type);
-  return I;
+  std::unique_ptr<TernOpInit> &I = ThePool[TheKey];
+  if (!I) I.reset(new TernOpInit(opc, lhs, mhs, rhs, Type));
+  return I.get();
 }
 
 static Init *ForeachHelper(Init *LHS, Init *MHS, Init *RHS, RecTy *Type,
@@ -1048,28 +893,19 @@ static Init *ForeachHelper(Init *LHS, Init *MHS, Init *RHS, RecTy *Type,
 static Init *EvaluateOperation(OpInit *RHSo, Init *LHS, Init *Arg,
                                RecTy *Type, Record *CurRec,
                                MultiClass *CurMultiClass) {
-  std::vector<Init *> NewOperands;
-
-  TypedInit *TArg = dyn_cast<TypedInit>(Arg);
-
   // If this is a dag, recurse
-  if (TArg && TArg->getType()->getAsString() == "dag") {
-    Init *Result = ForeachHelper(LHS, Arg, RHSo, Type,
-                                 CurRec, CurMultiClass);
-    return Result;
-  }
-
-  for (int i = 0; i < RHSo->getNumOperands(); ++i) {
-    OpInit *RHSoo = dyn_cast<OpInit>(RHSo->getOperand(i));
+  if (auto *TArg = dyn_cast<TypedInit>(Arg))
+    if (isa<DagRecTy>(TArg->getType()))
+      return ForeachHelper(LHS, Arg, RHSo, Type, CurRec, CurMultiClass);
 
-    if (RHSoo) {
-      Init *Result = EvaluateOperation(RHSoo, LHS, Arg,
-                                       Type, CurRec, CurMultiClass);
-      if (Result) {
+  std::vector<Init *> NewOperands;
+  for (unsigned i = 0; i < RHSo->getNumOperands(); ++i) {
+    if (auto *RHSoo = dyn_cast<OpInit>(RHSo->getOperand(i))) {
+      if (Init *Result = EvaluateOperation(RHSoo, LHS, Arg,
+                                           Type, CurRec, CurMultiClass))
         NewOperands.push_back(Result);
-      } else {
+      else
         NewOperands.push_back(Arg);
-      }
     } else if (LHS->getAsString() == RHSo->getOperand(i)->getAsString()) {
       NewOperands.push_back(Arg);
     } else {
@@ -1085,76 +921,63 @@ static Init *EvaluateOperation(OpInit *RHSo, Init *LHS, Init *Arg,
 
 static Init *ForeachHelper(Init *LHS, Init *MHS, Init *RHS, RecTy *Type,
                            Record *CurRec, MultiClass *CurMultiClass) {
-  DagInit *MHSd = dyn_cast<DagInit>(MHS);
-  ListInit *MHSl = dyn_cast<ListInit>(MHS);
 
   OpInit *RHSo = dyn_cast<OpInit>(RHS);
 
-  if (!RHSo) {
+  if (!RHSo)
     PrintFatalError(CurRec->getLoc(), "!foreach requires an operator\n");
-  }
 
   TypedInit *LHSt = dyn_cast<TypedInit>(LHS);
 
   if (!LHSt)
     PrintFatalError(CurRec->getLoc(), "!foreach requires typed variable\n");
 
-  if ((MHSd && isa<DagRecTy>(Type)) || (MHSl && isa<ListRecTy>(Type))) {
-    if (MHSd) {
-      Init *Val = MHSd->getOperator();
-      Init *Result = EvaluateOperation(RHSo, LHS, Val,
-                                       Type, CurRec, CurMultiClass);
-      if (Result) {
-        Val = Result;
-      }
+  DagInit *MHSd = dyn_cast<DagInit>(MHS);
+  if (MHSd && isa<DagRecTy>(Type)) {
+    Init *Val = MHSd->getOperator();
+    if (Init *Result = EvaluateOperation(RHSo, LHS, Val,
+                                         Type, CurRec, CurMultiClass))
+      Val = Result;
+
+    std::vector<std::pair<Init *, std::string> > args;
+    for (unsigned int i = 0; i < MHSd->getNumArgs(); ++i) {
+      Init *Arg = MHSd->getArg(i);
+      std::string ArgName = MHSd->getArgName(i);
+
+      // Process args
+      if (Init *Result = EvaluateOperation(RHSo, LHS, Arg, Type,
+                                           CurRec, CurMultiClass))
+        Arg = Result;
+
+      // TODO: Process arg names
+      args.push_back(std::make_pair(Arg, ArgName));
+    }
 
-      std::vector<std::pair<Init *, std::string> > args;
-      for (unsigned int i = 0; i < MHSd->getNumArgs(); ++i) {
-        Init *Arg;
-        std::string ArgName;
-        Arg = MHSd->getArg(i);
-        ArgName = MHSd->getArgName(i);
-
-        // Process args
-        Init *Result = EvaluateOperation(RHSo, LHS, Arg, Type,
-                                         CurRec, CurMultiClass);
-        if (Result) {
-          Arg = Result;
-        }
+    return DagInit::get(Val, "", args);
+  }
 
-        // TODO: Process arg names
-        args.push_back(std::make_pair(Arg, ArgName));
+  ListInit *MHSl = dyn_cast<ListInit>(MHS);
+  if (MHSl && isa<ListRecTy>(Type)) {
+    std::vector<Init *> NewOperands;
+    std::vector<Init *> NewList(MHSl->begin(), MHSl->end());
+
+    for (Init *&Item : NewList) {
+      NewOperands.clear();
+      for(unsigned i = 0; i < RHSo->getNumOperands(); ++i) {
+        // First, replace the foreach variable with the list item
+        if (LHS->getAsString() == RHSo->getOperand(i)->getAsString())
+          NewOperands.push_back(Item);
+        else
+          NewOperands.push_back(RHSo->getOperand(i));
       }
 
-      return DagInit::get(Val, "", args);
-    }
-    if (MHSl) {
-      std::vector<Init *> NewOperands;
-      std::vector<Init *> NewList(MHSl->begin(), MHSl->end());
-
-      for (std::vector<Init *>::iterator li = NewList.begin(),
-             liend = NewList.end();
-           li != liend;
-           ++li) {
-        Init *Item = *li;
-        NewOperands.clear();
-        for(int i = 0; i < RHSo->getNumOperands(); ++i) {
-          // First, replace the foreach variable with the list item
-          if (LHS->getAsString() == RHSo->getOperand(i)->getAsString()) {
-            NewOperands.push_back(Item);
-          } else {
-            NewOperands.push_back(RHSo->getOperand(i));
-          }
-        }
-
-        // Now run the operator and use its result as the new list item
-        const OpInit *NewOp = RHSo->clone(NewOperands);
-        Init *NewItem = NewOp->Fold(CurRec, CurMultiClass);
-        if (NewItem != NewOp)
-          *li = NewItem;
-      }
-      return ListInit::get(NewList, MHSl->getType());
+      // Now run the operator and use its result as the new list item
+      const OpInit *NewOp = RHSo->clone(NewOperands);
+      Init *NewItem = NewOp->Fold(CurRec, CurMultiClass);
+      if (NewItem != NewOp)
+        Item = NewItem;
     }
+    return ListInit::get(NewList, MHSl->getType());
   }
   return nullptr;
 }
@@ -1174,48 +997,40 @@ Init *TernOpInit::Fold(Record *CurRec, MultiClass *CurMultiClass) const {
     VarInit *RHSv = dyn_cast<VarInit>(RHS);
     StringInit *RHSs = dyn_cast<StringInit>(RHS);
 
-    if ((LHSd && MHSd && RHSd)
-        || (LHSv && MHSv && RHSv)
-        || (LHSs && MHSs && RHSs)) {
-      if (RHSd) {
-        Record *Val = RHSd->getDef();
-        if (LHSd->getAsString() == RHSd->getAsString()) {
-          Val = MHSd->getDef();
-        }
-        return DefInit::get(Val);
-      }
-      if (RHSv) {
-        std::string Val = RHSv->getName();
-        if (LHSv->getAsString() == RHSv->getAsString()) {
-          Val = MHSv->getName();
-        }
-        return VarInit::get(Val, getType());
+    if (LHSd && MHSd && RHSd) {
+      Record *Val = RHSd->getDef();
+      if (LHSd->getAsString() == RHSd->getAsString())
+        Val = MHSd->getDef();
+      return DefInit::get(Val);
+    }
+    if (LHSv && MHSv && RHSv) {
+      std::string Val = RHSv->getName();
+      if (LHSv->getAsString() == RHSv->getAsString())
+        Val = MHSv->getName();
+      return VarInit::get(Val, getType());
+    }
+    if (LHSs && MHSs && RHSs) {
+      std::string Val = RHSs->getValue();
+
+      std::string::size_type found;
+      std::string::size_type idx = 0;
+      while (true) {
+        found = Val.find(LHSs->getValue(), idx);
+        if (found == std::string::npos)
+          break;
+        Val.replace(found, LHSs->getValue().size(), MHSs->getValue());
+        idx = found + MHSs->getValue().size();
       }
-      if (RHSs) {
-        std::string Val = RHSs->getValue();
-
-        std::string::size_type found;
-        std::string::size_type idx = 0;
-        do {
-          found = Val.find(LHSs->getValue(), idx);
-          if (found != std::string::npos) {
-            Val.replace(found, LHSs->getValue().size(), MHSs->getValue());
-          }
-          idx = found +  MHSs->getValue().size();
-        } while (found != std::string::npos);
 
-        return StringInit::get(Val);
-      }
+      return StringInit::get(Val);
     }
     break;
   }
 
   case FOREACH: {
-    Init *Result = ForeachHelper(LHS, MHS, RHS, getType(),
-                                 CurRec, CurMultiClass);
-    if (Result) {
+    if (Init *Result = ForeachHelper(LHS, MHS, RHS, getType(),
+                                     CurRec, CurMultiClass))
       return Result;
-    }
     break;
   }
 
@@ -1224,11 +1039,9 @@ Init *TernOpInit::Fold(Record *CurRec, MultiClass *CurMultiClass) const {
     if (Init *I = LHS->convertInitializerTo(IntRecTy::get()))
       LHSi = dyn_cast<IntInit>(I);
     if (LHSi) {
-      if (LHSi->getValue()) {
+      if (LHSi->getValue())
         return MHS;
-      } else {
-        return RHS;
-      }
+      return RHS;
     }
     break;
   }
@@ -1251,11 +1064,10 @@ Init *TernOpInit::resolveReferences(Record &R,
         Init *mhs = MHS->resolveReferences(R, RV);
         return (TernOpInit::get(getOpcode(), lhs, mhs,
                                 RHS, getType()))->Fold(&R, nullptr);
-      } else {
-        Init *rhs = RHS->resolveReferences(R, RV);
-        return (TernOpInit::get(getOpcode(), lhs, MHS,
-                                rhs, getType()))->Fold(&R, nullptr);
       }
+      Init *rhs = RHS->resolveReferences(R, RV);
+      return (TernOpInit::get(getOpcode(), lhs, MHS,
+                              rhs, getType()))->Fold(&R, nullptr);
     }
   }
 
@@ -1274,9 +1086,9 @@ std::string TernOpInit::getAsString() const {
   case SUBST: Result = "!subst"; break;
   case FOREACH: Result = "!foreach"; break;
   case IF: Result = "!if"; break;
- }
-  return Result + "(" + LHS->getAsString() + ", " + MHS->getAsString() + ", "
-    + RHS->getAsString() + ")";
+  }
+  return Result + "(" + LHS->getAsString() + ", " + MHS->getAsString() + ", " +
+         RHS->getAsString() + ")";
 }
 
 RecTy *TypedInit::getFieldType(const std::string &FieldName) const {
@@ -1286,6 +1098,82 @@ RecTy *TypedInit::getFieldType(const std::string &FieldName) const {
   return nullptr;
 }
 
+Init *
+TypedInit::convertInitializerTo(RecTy *Ty) const {
+  if (isa<IntRecTy>(Ty)) {
+    if (getType()->typeIsConvertibleTo(Ty))
+      return const_cast<TypedInit *>(this);
+    return nullptr;
+  }
+
+  if (isa<StringRecTy>(Ty)) {
+    if (isa<StringRecTy>(getType()))
+      return const_cast<TypedInit *>(this);
+    return nullptr;
+  }
+
+  if (isa<BitRecTy>(Ty)) {
+    // Accept variable if it is already of bit type!
+    if (isa<BitRecTy>(getType()))
+      return const_cast<TypedInit *>(this);
+    if (auto *BitsTy = dyn_cast<BitsRecTy>(getType())) {
+      // Accept only bits<1> expression.
+      if (BitsTy->getNumBits() == 1)
+        return const_cast<TypedInit *>(this);
+      return nullptr;
+    }
+    // Ternary !if can be converted to bit, but only if both sides are
+    // convertible to a bit.
+    if (const auto *TOI = dyn_cast<TernOpInit>(this)) {
+      if (TOI->getOpcode() == TernOpInit::TernaryOp::IF &&
+          TOI->getMHS()->convertInitializerTo(BitRecTy::get()) &&
+          TOI->getRHS()->convertInitializerTo(BitRecTy::get()))
+        return const_cast<TypedInit *>(this);
+      return nullptr;
+    }
+    return nullptr;
+  }
+
+  if (auto *BRT = dyn_cast<BitsRecTy>(Ty)) {
+    if (BRT->getNumBits() == 1 && isa<BitRecTy>(getType()))
+      return BitsInit::get(const_cast<TypedInit *>(this));
+
+    if (getType()->typeIsConvertibleTo(BRT)) {
+      SmallVector<Init *, 16> NewBits(BRT->getNumBits());
+
+      for (unsigned i = 0; i != BRT->getNumBits(); ++i)
+        NewBits[i] = VarBitInit::get(const_cast<TypedInit *>(this), i);
+      return BitsInit::get(NewBits);
+    }
+
+    return nullptr;
+  }
+
+  if (auto *DLRT = dyn_cast<ListRecTy>(Ty)) {
+    if (auto *SLRT = dyn_cast<ListRecTy>(getType()))
+      if (SLRT->getElementType()->typeIsConvertibleTo(DLRT->getElementType()))
+        return const_cast<TypedInit *>(this);
+    return nullptr;
+  }
+
+  if (auto *DRT = dyn_cast<DagRecTy>(Ty)) {
+    if (getType()->typeIsConvertibleTo(DRT))
+      return const_cast<TypedInit *>(this);
+    return nullptr;
+  }
+
+  if (auto *SRRT = dyn_cast<RecordRecTy>(Ty)) {
+    // Ensure that this is compatible with Rec.
+    if (RecordRecTy *DRRT = dyn_cast<RecordRecTy>(getType()))
+      if (DRRT->getRecord()->isSubClassOf(SRRT->getRecord()) ||
+          DRRT->getRecord() == SRRT->getRecord())
+        return const_cast<TypedInit *>(this);
+    return nullptr;
+  }
+
+  return nullptr;
+}
+
 Init *
 TypedInit::convertInitializerBitRange(const std::vector<unsigned> &Bits) const {
   BitsRecTy *T = dyn_cast<BitsRecTy>(getType());
@@ -1326,18 +1214,17 @@ VarInit *VarInit::get(const std::string &VN, RecTy *T) {
 
 VarInit *VarInit::get(Init *VN, RecTy *T) {
   typedef std::pair<RecTy *, Init *> Key;
-  static Pool<DenseMap<Key, VarInit *> > ThePool;
+  static DenseMap<Key, std::unique_ptr<VarInit>> ThePool;
 
   Key TheKey(std::make_pair(T, VN));
 
-  VarInit *&I = ThePool[TheKey];
-  if (!I) I = new VarInit(VN, T);
-  return I;
+  std::unique_ptr<VarInit> &I = ThePool[TheKey];
+  if (!I) I.reset(new VarInit(VN, T));
+  return I.get();
 }
 
 const std::string &VarInit::getName() const {
-  StringInit *NameString = dyn_cast<StringInit>(getNameInit());
-  assert(NameString && "VarInit name is not a string!");
+  StringInit *NameString = cast<StringInit>(getNameInit());
   return NameString->getValue();
 }
 
@@ -1356,13 +1243,10 @@ Init *VarInit::resolveListElementReference(Record &R,
   RecordVal *RV = R.getValue(getNameInit());
   assert(RV && "Reference to a non-existent variable?");
   ListInit *LI = dyn_cast<ListInit>(RV->getValue());
-  if (!LI) {
-    TypedInit *VI = dyn_cast<TypedInit>(RV->getValue());
-    assert(VI && "Invalid list element!");
-    return VarListElementInit::get(VI, Elt);
-  }
+  if (!LI)
+    return VarListElementInit::get(cast<TypedInit>(RV->getValue()), Elt);
 
-  if (Elt >= LI->getSize())
+  if (Elt >= LI->size())
     return nullptr;  // Out of range reference.
   Init *E = LI->getElement(Elt);
   // If the element is set to some value, or if we are resolving a reference
@@ -1391,8 +1275,7 @@ Init *VarInit::getFieldInit(Record &R, const RecordVal *RV,
       assert(TheInit != this && "Infinite loop detected!");
       if (Init *I = TheInit->getFieldInit(R, RV, FieldName))
         return I;
-      else
-        return nullptr;
+      return nullptr;
     }
   return nullptr;
 }
@@ -1411,19 +1294,24 @@ Init *VarInit::resolveReferences(Record &R, const RecordVal *RV) const {
 
 VarBitInit *VarBitInit::get(TypedInit *T, unsigned B) {
   typedef std::pair<TypedInit *, unsigned> Key;
-  typedef DenseMap<Key, VarBitInit *> Pool;
-
-  static Pool ThePool;
+  static DenseMap<Key, std::unique_ptr<VarBitInit>> ThePool;
 
   Key TheKey(std::make_pair(T, B));
 
-  VarBitInit *&I = ThePool[TheKey];
-  if (!I) I = new VarBitInit(T, B);
-  return I;
+  std::unique_ptr<VarBitInit> &I = ThePool[TheKey];
+  if (!I) I.reset(new VarBitInit(T, B));
+  return I.get();
+}
+
+Init *VarBitInit::convertInitializerTo(RecTy *Ty) const {
+  if (isa<BitRecTy>(Ty))
+    return const_cast<VarBitInit *>(this);
+
+  return nullptr;
 }
 
 std::string VarBitInit::getAsString() const {
-   return TI->getAsString() + "{" + utostr(Bit) + "}";
+  return TI->getAsString() + "{" + utostr(Bit) + "}";
 }
 
 Init *VarBitInit::resolveReferences(Record &R, const RecordVal *RV) const {
@@ -1437,15 +1325,13 @@ Init *VarBitInit::resolveReferences(Record &R, const RecordVal *RV) const {
 VarListElementInit *VarListElementInit::get(TypedInit *T,
                                             unsigned E) {
   typedef std::pair<TypedInit *, unsigned> Key;
-  typedef DenseMap<Key, VarListElementInit *> Pool;
-
-  static Pool ThePool;
+  static DenseMap<Key, std::unique_ptr<VarListElementInit>> ThePool;
 
   Key TheKey(std::make_pair(T, E));
 
-  VarListElementInit *&I = ThePool[TheKey];
-  if (!I) I = new VarListElementInit(T, E);
-  return I;
+  std::unique_ptr<VarListElementInit> &I = ThePool[TheKey];
+  if (!I) I.reset(new VarListElementInit(T, E));
+  return I.get();
 }
 
 std::string VarListElementInit::getAsString() const {
@@ -1469,17 +1355,15 @@ Init *VarListElementInit::getBit(unsigned Bit) const {
 Init *VarListElementInit:: resolveListElementReference(Record &R,
                                                        const RecordVal *RV,
                                                        unsigned Elt) const {
-  Init *Result = TI->resolveListElementReference(R, RV, Element);
-  
-  if (Result) {
+  if (Init *Result = TI->resolveListElementReference(R, RV, Element)) {
     if (TypedInit *TInit = dyn_cast<TypedInit>(Result)) {
-      Init *Result2 = TInit->resolveListElementReference(R, RV, Elt);
-      if (Result2) return Result2;
-      return new VarListElementInit(TInit, Elt);
+      if (Init *Result2 = TInit->resolveListElementReference(R, RV, Elt))
+        return Result2;
+      return VarListElementInit::get(TInit, Elt);
     }
     return Result;
   }
- 
+
   return nullptr;
 }
 
@@ -1487,6 +1371,13 @@ DefInit *DefInit::get(Record *R) {
   return R->getDefInit();
 }
 
+Init *DefInit::convertInitializerTo(RecTy *Ty) const {
+  if (auto *RRT = dyn_cast<RecordRecTy>(Ty))
+    if (getDef()->isSubClassOf(RRT->getRecord()))
+      return const_cast<DefInit *>(this);
+  return nullptr;
+}
+
 RecTy *DefInit::getFieldType(const std::string &FieldName) const {
   if (const RecordVal *RV = Def->getValue(FieldName))
     return RV->getType();
@@ -1505,14 +1396,13 @@ std::string DefInit::getAsString() const {
 
 FieldInit *FieldInit::get(Init *R, const std::string &FN) {
   typedef std::pair<Init *, TableGenStringKey> Key;
-  typedef DenseMap<Key, FieldInit *> Pool;
-  static Pool ThePool;  
+  static DenseMap<Key, std::unique_ptr<FieldInit>> ThePool;
 
   Key TheKey(std::make_pair(R, FN));
 
-  FieldInit *&I = ThePool[TheKey];
-  if (!I) I = new FieldInit(R, FN);
-  return I;
+  std::unique_ptr<FieldInit> &I = ThePool[TheKey];
+  if (!I) I.reset(new FieldInit(R, FN));
+  return I.get();
 }
 
 Init *FieldInit::getBit(unsigned Bit) const {
@@ -1525,7 +1415,7 @@ Init *FieldInit::resolveListElementReference(Record &R, const RecordVal *RV,
                                              unsigned Elt) const {
   if (Init *ListVal = Rec->getFieldInit(R, RV, FieldName))
     if (ListInit *LI = dyn_cast<ListInit>(ListVal)) {
-      if (Elt >= LI->getSize()) return nullptr;
+      if (Elt >= LI->size()) return nullptr;
       Init *E = LI->getElement(Elt);
 
       // If the element is set to some value, or if we are resolving a
@@ -1540,15 +1430,13 @@ Init *FieldInit::resolveListElementReference(Record &R, const RecordVal *RV,
 Init *FieldInit::resolveReferences(Record &R, const RecordVal *RV) const {
   Init *NewRec = RV ? Rec->resolveReferences(R, RV) : Rec;
 
-  Init *BitsVal = NewRec->getFieldInit(R, RV, FieldName);
-  if (BitsVal) {
+  if (Init *BitsVal = NewRec->getFieldInit(R, RV, FieldName)) {
     Init *BVR = BitsVal->resolveReferences(R, RV);
     return BVR->isComplete() ? BVR : const_cast<FieldInit *>(this);
   }
 
-  if (NewRec != Rec) {
+  if (NewRec != Rec)
     return FieldInit::get(NewRec, FieldName);
-  }
   return const_cast<FieldInit *>(this);
 }
 
@@ -1572,8 +1460,8 @@ DagInit *
 DagInit::get(Init *V, const std::string &VN,
              ArrayRef<Init *> ArgRange,
              ArrayRef<std::string> NameRange) {
-  typedef FoldingSet<DagInit> Pool;
-  static Pool ThePool;  
+  static FoldingSet<DagInit> ThePool;
+  static std::vector<std::unique_ptr<DagInit>> TheActualPool;
 
   FoldingSetNodeID ID;
   ProfileDagInit(ID, V, VN, ArgRange, NameRange);
@@ -1584,24 +1472,19 @@ DagInit::get(Init *V, const std::string &VN,
 
   DagInit *I = new DagInit(V, VN, ArgRange, NameRange);
   ThePool.InsertNode(I, IP);
-
+  TheActualPool.push_back(std::unique_ptr<DagInit>(I));
   return I;
 }
 
 DagInit *
 DagInit::get(Init *V, const std::string &VN,
              const std::vector<std::pair<Init*, std::string> > &args) {
-  typedef std::pair<Init*, std::string> PairType;
-
   std::vector<Init *> Args;
   std::vector<std::string> Names;
 
-  for (std::vector<PairType>::const_iterator i = args.begin(),
-         iend = args.end();
-       i != iend;
-       ++i) {
-    Args.push_back(i->first);
-    Names.push_back(i->second);
+  for (const auto &Arg : args) {
+    Args.push_back(Arg.first);
+    Names.push_back(Arg.second);
   }
 
   return DagInit::get(V, VN, Args, Names);
@@ -1611,6 +1494,13 @@ void DagInit::Profile(FoldingSetNodeID &ID) const {
   ProfileDagInit(ID, Val, ValName, Args, ArgNames);
 }
 
+Init *DagInit::convertInitializerTo(RecTy *Ty) const {
+  if (isa<DagRecTy>(Ty))
+    return const_cast<DagInit *>(this);
+
+  return nullptr;
+}
+
 Init *DagInit::resolveReferences(Record &R, const RecordVal *RV) const {
   std::vector<Init*> NewArgs;
   for (unsigned i = 0, e = Args.size(); i != e; ++i)
@@ -1645,22 +1535,20 @@ std::string DagInit::getAsString() const {
 //    Other implementations
 //===----------------------------------------------------------------------===//
 
-RecordVal::RecordVal(Init *N, RecTy *T, unsigned P)
-  : Name(N), Ty(T), Prefix(P) {
-  Value = Ty->convertValue(UnsetInit::get());
+RecordVal::RecordVal(Init *N, RecTy *T, bool P)
+  : NameAndPrefix(N, P), Ty(T) {
+  Value = UnsetInit::get()->convertInitializerTo(Ty);
   assert(Value && "Cannot create unset value for current type!");
 }
 
-RecordVal::RecordVal(const std::string &N, RecTy *T, unsigned P)
-  : Name(StringInit::get(N)), Ty(T), Prefix(P) {
-  Value = Ty->convertValue(UnsetInit::get());
+RecordVal::RecordVal(const std::string &N, RecTy *T, bool P)
+  : NameAndPrefix(StringInit::get(N), P), Ty(T) {
+  Value = UnsetInit::get()->convertInitializerTo(Ty);
   assert(Value && "Cannot create unset value for current type!");
 }
 
 const std::string &RecordVal::getName() const {
-  StringInit *NameString = dyn_cast<StringInit>(Name);
-  assert(NameString && "RecordVal name is not a string!");
-  return NameString->getValue();
+  return cast<StringInit>(getNameInit())->getValue();
 }
 
 void RecordVal::dump() const { errs() << *this; }
@@ -1688,23 +1576,19 @@ void Record::init() {
 
 void Record::checkName() {
   // Ensure the record name has string type.
-  const TypedInit *TypedName = dyn_cast<const TypedInit>(Name);
-  assert(TypedName && "Record name is not typed!");
-  RecTy *Type = TypedName->getType();
-  if (!isa<StringRecTy>(Type))
+  const TypedInit *TypedName = cast<const TypedInit>(Name);
+  if (!isa<StringRecTy>(TypedName->getType()))
     PrintFatalError(getLoc(), "Record name is not a string!");
 }
 
 DefInit *Record::getDefInit() {
   if (!TheInit)
-    TheInit = new DefInit(this, new RecordRecTy(this));
-  return TheInit;
+    TheInit.reset(new DefInit(this, new RecordRecTy(this)));
+  return TheInit.get();
 }
 
 const std::string &Record::getName() const {
-  const StringInit *NameString = dyn_cast<StringInit>(Name);
-  assert(NameString && "Record name is not a string!");
-  return NameString->getValue();
+  return cast<StringInit>(Name)->getValue();
 }
 
 void Record::setName(Init *NewName) {
@@ -1736,14 +1620,13 @@ void Record::resolveReferencesTo(const RecordVal *RV) {
       continue;
     if (Init *V = Values[i].getValue())
       if (Values[i].setValue(V->resolveReferences(*this, RV)))
-        PrintFatalError(getLoc(), "Invalid value is found when setting '"
-                      + Values[i].getNameInitAsString()
-                      + "' after resolving references"
-                      + (RV ? " against '" + RV->getNameInitAsString()
-                              + "' of ("
-                              + RV->getValue()->getAsUnquotedString() + ")"
-                            : "")
-                      + "\n");
+        PrintFatalError(getLoc(), "Invalid value is found when setting '" +
+                        Values[i].getNameInitAsString() +
+                        "' after resolving references" +
+                        (RV ? " against '" + RV->getNameInitAsString() +
+                              "' of (" + RV->getValue()->getAsUnquotedString() +
+                              ")"
+                            : "") + "\n");
   }
   Init *OldName = getNameInit();
   Init *NewName = Name->resolveReferences(*this, RV);
@@ -1761,9 +1644,11 @@ raw_ostream &llvm::operator<<(raw_ostream &OS, const Record &R) {
   const std::vector<Init *> &TArgs = R.getTemplateArgs();
   if (!TArgs.empty()) {
     OS << "<";
-    for (unsigned i = 0, e = TArgs.size(); i != e; ++i) {
-      if (i) OS << ", ";
-      const RecordVal *RV = R.getValue(TArgs[i]);
+    bool NeedComma = false;
+    for (const Init *TA : TArgs) {
+      if (NeedComma) OS << ", ";
+      NeedComma = true;
+      const RecordVal *RV = R.getValue(TA);
       assert(RV && "Template argument record not found??");
       RV->print(OS, false);
     }
@@ -1771,21 +1656,20 @@ raw_ostream &llvm::operator<<(raw_ostream &OS, const Record &R) {
   }
 
   OS << " {";
-  const std::vector<Record*> &SC = R.getSuperClasses();
+  ArrayRef<Record *> SC = R.getSuperClasses();
   if (!SC.empty()) {
     OS << "\t//";
-    for (unsigned i = 0, e = SC.size(); i != e; ++i)
-      OS << " " << SC[i]->getNameInitAsString();
+    for (const Record *Super : SC)
+      OS << " " << Super->getNameInitAsString();
   }
   OS << "\n";
 
-  const std::vector<RecordVal> &Vals = R.getValues();
-  for (unsigned i = 0, e = Vals.size(); i != e; ++i)
-    if (Vals[i].getPrefix() && !R.isTemplateArg(Vals[i].getName()))
-      OS << Vals[i];
-  for (unsigned i = 0, e = Vals.size(); i != e; ++i)
-    if (!Vals[i].getPrefix() && !R.isTemplateArg(Vals[i].getName()))
-      OS << Vals[i];
+  for (const RecordVal &Val : R.getValues())
+    if (Val.getPrefix() && !R.isTemplateArg(Val.getName()))
+      OS << Val;
+  for (const RecordVal &Val : R.getValues())
+    if (!Val.getPrefix() && !R.isTemplateArg(Val.getName()))
+      OS << Val;
 
   return OS << "}\n";
 }
@@ -1858,13 +1742,12 @@ std::vector<Record*>
 Record::getValueAsListOfDefs(StringRef FieldName) const {
   ListInit *List = getValueAsListInit(FieldName);
   std::vector<Record*> Defs;
-  for (unsigned i = 0; i < List->getSize(); i++) {
-    if (DefInit *DI = dyn_cast<DefInit>(List->getElement(i))) {
+  for (Init *I : List->getValues()) {
+    if (DefInit *DI = dyn_cast<DefInit>(I))
       Defs.push_back(DI->getDef());
-    } else {
+    else
       PrintFatalError(getLoc(), "Record `" + getName() + "', field `" +
         FieldName + "' list is not entirely DefInit!");
-    }
   }
   return Defs;
 }
@@ -1893,13 +1776,12 @@ std::vector<int64_t>
 Record::getValueAsListOfInts(StringRef FieldName) const {
   ListInit *List = getValueAsListInit(FieldName);
   std::vector<int64_t> Ints;
-  for (unsigned i = 0; i < List->getSize(); i++) {
-    if (IntInit *II = dyn_cast<IntInit>(List->getElement(i))) {
+  for (Init *I : List->getValues()) {
+    if (IntInit *II = dyn_cast<IntInit>(I))
       Ints.push_back(II->getValue());
-    } else {
+    else
       PrintFatalError(getLoc(), "Record `" + getName() + "', field `" +
         FieldName + "' does not have a list of ints initializer!");
-    }
   }
   return Ints;
 }
@@ -1912,13 +1794,12 @@ std::vector<std::string>
 Record::getValueAsListOfStrings(StringRef FieldName) const {
   ListInit *List = getValueAsListInit(FieldName);
   std::vector<std::string> Strings;
-  for (unsigned i = 0; i < List->getSize(); i++) {
-    if (StringInit *II = dyn_cast<StringInit>(List->getElement(i))) {
-      Strings.push_back(II->getValue());
-    } else {
+  for (Init *I : List->getValues()) {
+    if (StringInit *SI = dyn_cast<StringInit>(I))
+      Strings.push_back(SI->getValue());
+    else
       PrintFatalError(getLoc(), "Record `" + getName() + "', field `" +
         FieldName + "' does not have a list of strings initializer!");
-    }
   }
   return Strings;
 }
@@ -1961,7 +1842,7 @@ bool Record::getValueAsBitOrUnset(StringRef FieldName, bool &Unset) const {
     PrintFatalError(getLoc(), "Record `" + getName() +
       "' does not have a field named `" + FieldName.str() + "'!\n");
 
-  if (R->getValue() == UnsetInit::get()) {
+  if (isa<UnsetInit>(R->getValue())) {
     Unset = true;
     return false;
   }
@@ -1994,12 +1875,8 @@ void MultiClass::dump() const {
   Rec.dump();
 
   errs() << "Defs:\n";
-  for (RecordVector::const_iterator r = DefPrototypes.begin(),
-         rend = DefPrototypes.end();
-       r != rend;
-       ++r) {
-    (*r)->dump();
-  }
+  for (const auto &Proto : DefPrototypes)
+    Proto->dump();
 }
 
 
@@ -2007,13 +1884,11 @@ void RecordKeeper::dump() const { errs() << *this; }
 
 raw_ostream &llvm::operator<<(raw_ostream &OS, const RecordKeeper &RK) {
   OS << "------------- Classes -----------------\n";
-  const auto &Classes = RK.getClasses();
-  for (const auto &C : Classes)
+  for (const auto &C : RK.getClasses())
     OS << "class " << *C.second;
 
   OS << "------------- Defs -----------------\n";
-  const auto &Defs = RK.getDefs();
-  for (const auto &D : Defs)
+  for (const auto &D : RK.getDefs())
     OS << "def " << *D.second;
   return OS;
 }
@@ -2043,23 +1918,23 @@ Init *llvm::QualifyName(Record &CurRec, MultiClass *CurMultiClass,
   RecTy *Type = cast<TypedInit>(Name)->getType();
 
   BinOpInit *NewName =
-    BinOpInit::get(BinOpInit::STRCONCAT, 
-                      BinOpInit::get(BinOpInit::STRCONCAT,
-                                        CurRec.getNameInit(),
-                                        StringInit::get(Scoper),
-                                        Type)->Fold(&CurRec, CurMultiClass),
-                      Name,
-                      Type);
+    BinOpInit::get(BinOpInit::STRCONCAT,
+                   BinOpInit::get(BinOpInit::STRCONCAT,
+                                  CurRec.getNameInit(),
+                                  StringInit::get(Scoper),
+                                  Type)->Fold(&CurRec, CurMultiClass),
+                   Name,
+                   Type);
 
   if (CurMultiClass && Scoper != "::") {
     NewName =
-      BinOpInit::get(BinOpInit::STRCONCAT, 
-                        BinOpInit::get(BinOpInit::STRCONCAT,
-                                          CurMultiClass->Rec.getNameInit(),
-                                          StringInit::get("::"),
-                                          Type)->Fold(&CurRec, CurMultiClass),
-                        NewName->Fold(&CurRec, CurMultiClass),
-                        Type);
+      BinOpInit::get(BinOpInit::STRCONCAT,
+                     BinOpInit::get(BinOpInit::STRCONCAT,
+                                    CurMultiClass->Rec.getNameInit(),
+                                    StringInit::get("::"),
+                                    Type)->Fold(&CurRec, CurMultiClass),
+                     NewName->Fold(&CurRec, CurMultiClass),
+                     Type);
   }
 
   return NewName->Fold(&CurRec, CurMultiClass);
diff --git a/lib/TableGen/SetTheory.cpp b/lib/TableGen/SetTheory.cpp
index c99c2bab45a..f56b17acbfb 100644
--- a/lib/TableGen/SetTheory.cpp
+++ b/lib/TableGen/SetTheory.cpp
@@ -12,10 +12,10 @@
 //
 //===----------------------------------------------------------------------===//
 
+#include "llvm/TableGen/SetTheory.h"
 #include "llvm/Support/Format.h"
 #include "llvm/TableGen/Error.h"
 #include "llvm/TableGen/Record.h"
-#include "llvm/TableGen/SetTheory.h"
 
 using namespace llvm;
 
@@ -196,7 +196,7 @@ struct SequenceOp : public SetTheory::Operator {
     if (IntInit *II = dyn_cast<IntInit>(Expr->arg_begin()[2]))
       To = II->getValue();
     else
-      PrintFatalError(Loc, "From must be an integer: " + Expr->getAsString());
+      PrintFatalError(Loc, "To must be an integer: " + Expr->getAsString());
     if (To < 0 || To >= (1 << 30))
       PrintFatalError(Loc, "To out of range");
 
@@ -245,28 +245,28 @@ void SetTheory::Expander::anchor() {}
 
 
 SetTheory::SetTheory() {
-  addOperator("add", new AddOp);
-  addOperator("sub", new SubOp);
-  addOperator("and", new AndOp);
-  addOperator("shl", new ShlOp);
-  addOperator("trunc", new TruncOp);
-  addOperator("rotl", new RotOp(false));
-  addOperator("rotr", new RotOp(true));
-  addOperator("decimate", new DecimateOp);
-  addOperator("interleave", new InterleaveOp);
-  addOperator("sequence", new SequenceOp);
+  addOperator("add", llvm::make_unique<AddOp>());
+  addOperator("sub", llvm::make_unique<SubOp>());
+  addOperator("and", llvm::make_unique<AndOp>());
+  addOperator("shl", llvm::make_unique<ShlOp>());
+  addOperator("trunc", llvm::make_unique<TruncOp>());
+  addOperator("rotl", llvm::make_unique<RotOp>(false));
+  addOperator("rotr", llvm::make_unique<RotOp>(true));
+  addOperator("decimate", llvm::make_unique<DecimateOp>());
+  addOperator("interleave", llvm::make_unique<InterleaveOp>());
+  addOperator("sequence", llvm::make_unique<SequenceOp>());
 }
 
-void SetTheory::addOperator(StringRef Name, Operator *Op) {
-  Operators[Name] = Op;
+void SetTheory::addOperator(StringRef Name, std::unique_ptr<Operator> Op) {
+  Operators[Name] = std::move(Op);
 }
 
-void SetTheory::addExpander(StringRef ClassName, Expander *E) {
-  Expanders[ClassName] = E;
+void SetTheory::addExpander(StringRef ClassName, std::unique_ptr<Expander> E) {
+  Expanders[ClassName] = std::move(E);
 }
 
 void SetTheory::addFieldExpander(StringRef ClassName, StringRef FieldName) {
-  addExpander(ClassName, new FieldExpander(FieldName));
+  addExpander(ClassName, llvm::make_unique<FieldExpander>(FieldName));
 }
 
 void SetTheory::evaluate(Init *Expr, RecSet &Elts, ArrayRef<SMLoc> Loc) {
@@ -289,10 +289,10 @@ void SetTheory::evaluate(Init *Expr, RecSet &Elts, ArrayRef<SMLoc> Loc) {
   DefInit *OpInit = dyn_cast<DefInit>(DagExpr->getOperator());
   if (!OpInit)
     PrintFatalError(Loc, "Bad set expression: " + Expr->getAsString());
-  Operator *Op = Operators.lookup(OpInit->getDef()->getName());
-  if (!Op)
+  auto I = Operators.find(OpInit->getDef()->getName());
+  if (I == Operators.end())
     PrintFatalError(Loc, "Unknown set operator: " + Expr->getAsString());
-  Op->apply(*this, DagExpr, Elts, Loc);
+  I->second->apply(*this, DagExpr, Elts, Loc);
 }
 
 const RecVec *SetTheory::expand(Record *Set) {
@@ -302,16 +302,17 @@ const RecVec *SetTheory::expand(Record *Set) {
     return &I->second;
 
   // This is the first time we see Set. Find a suitable expander.
-  const std::vector<Record*> &SC = Set->getSuperClasses();
+  ArrayRef<Record *> SC = Set->getSuperClasses();
   for (unsigned i = 0, e = SC.size(); i != e; ++i) {
     // Skip unnamed superclasses.
     if (!dyn_cast<StringInit>(SC[i]->getNameInit()))
       continue;
-    if (Expander *Exp = Expanders.lookup(SC[i]->getName())) {
+    auto I = Expanders.find(SC[i]->getName());
+    if (I != Expanders.end()) {
       // This breaks recursive definitions.
       RecVec &EltVec = Expansions[Set];
       RecSet Elts;
-      Exp->expand(*this, Set, Elts);
+      I->second->expand(*this, Set, Elts);
       EltVec.assign(Elts.begin(), Elts.end());
       return &EltVec;
     }
diff --git a/lib/TableGen/TGParser.cpp b/lib/TableGen/TGParser.cpp
index f66dfd37499..5c36fda2e1c 100644
--- a/lib/TableGen/TGParser.cpp
+++ b/lib/TableGen/TGParser.cpp
@@ -51,12 +51,8 @@ void SubMultiClassReference::dump() const {
   MC->dump();
 
   errs() << "Template args:\n";
-  for (std::vector<Init *>::const_iterator i = TemplateArgs.begin(),
-         iend = TemplateArgs.end();
-       i != iend;
-       ++i) {
-    (*i)->dump();
-  }
+  for (Init *TA : TemplateArgs)
+    TA->dump();
 }
 
 } // end namespace llvm
@@ -88,8 +84,8 @@ bool TGParser::SetValue(Record *CurRec, SMLoc Loc, Init *ValName,
 
   RecordVal *RV = CurRec->getValue(ValName);
   if (!RV)
-    return Error(Loc, "Value '" + ValName->getAsUnquotedString()
-                 + "' unknown!");
+    return Error(Loc, "Value '" + ValName->getAsUnquotedString() +
+                 "' unknown!");
 
   // Do not allow assignments like 'X = X'.  This will just cause infinite loops
   // in the resolution machinery.
@@ -105,18 +101,16 @@ bool TGParser::SetValue(Record *CurRec, SMLoc Loc, Init *ValName,
   if (!BitList.empty()) {
     BitsInit *CurVal = dyn_cast<BitsInit>(RV->getValue());
     if (!CurVal)
-      return Error(Loc, "Value '" + ValName->getAsUnquotedString()
-                   + "' is not a bits type");
+      return Error(Loc, "Value '" + ValName->getAsUnquotedString() +
+                   "' is not a bits type");
 
     // Convert the incoming value to a bits type of the appropriate size...
     Init *BI = V->convertInitializerTo(BitsRecTy::get(BitList.size()));
-    if (!BI) {
+    if (!BI)
       return Error(Loc, "Initializer is not compatible with bit range");
-    }
 
     // We should have a BitsInit type now.
-    BitsInit *BInit = dyn_cast<BitsInit>(BI);
-    assert(BInit != nullptr);
+    BitsInit *BInit = cast<BitsInit>(BI);
 
     SmallVector<Init *, 16> NewBits(CurVal->getNumBits());
 
@@ -124,7 +118,7 @@ bool TGParser::SetValue(Record *CurRec, SMLoc Loc, Init *ValName,
     for (unsigned i = 0, e = BitList.size(); i != e; ++i) {
       unsigned Bit = BitList[i];
       if (NewBits[Bit])
-        return Error(Loc, "Cannot set bit #" + utostr(Bit) + " of value '" +
+        return Error(Loc, "Cannot set bit #" + Twine(Bit) + " of value '" +
                      ValName->getAsUnquotedString() + "' more than once");
       NewBits[Bit] = BInit->getBit(i);
     }
@@ -138,15 +132,13 @@ bool TGParser::SetValue(Record *CurRec, SMLoc Loc, Init *ValName,
 
   if (RV->setValue(V)) {
     std::string InitType = "";
-    if (BitsInit *BI = dyn_cast<BitsInit>(V)) {
+    if (BitsInit *BI = dyn_cast<BitsInit>(V))
       InitType = (Twine("' of type bit initializer with length ") +
                   Twine(BI->getNumBits())).str();
-    }
-    return Error(Loc, "Value '" + ValName->getAsUnquotedString() + "' of type '"
-                 + RV->getType()->getAsString() +
-                 "' is incompatible with initializer '" + V->getAsString()
-                 + InitType
-                 + "'");
+    return Error(Loc, "Value '" + ValName->getAsUnquotedString() +
+                 "' of type '" + RV->getType()->getAsString() +
+                 "' is incompatible with initializer '" + V->getAsString() +
+                 InitType + "'");
   }
   return false;
 }
@@ -156,9 +148,8 @@ bool TGParser::SetValue(Record *CurRec, SMLoc Loc, Init *ValName,
 bool TGParser::AddSubClass(Record *CurRec, SubClassReference &SubClass) {
   Record *SC = SubClass.Rec;
   // Add all of the values in the subclass into the current class.
-  const std::vector<RecordVal> &Vals = SC->getValues();
-  for (unsigned i = 0, e = Vals.size(); i != e; ++i)
-    if (AddValue(CurRec, SubClass.RefRange.Start, Vals[i]))
+  for (const RecordVal &Val : SC->getValues())
+    if (AddValue(CurRec, SubClass.RefRange.Start, Val))
       return true;
 
   const std::vector<Init *> &TArgs = SC->getTemplateArgs();
@@ -185,15 +176,15 @@ bool TGParser::AddSubClass(Record *CurRec, SubClassReference &SubClass) {
 
     } else if (!CurRec->getValue(TArgs[i])->getValue()->isComplete()) {
       return Error(SubClass.RefRange.Start,
-                   "Value not specified for template argument #"
-                   + utostr(i) + " (" + TArgs[i]->getAsUnquotedString()
-                   + ") of subclass '" + SC->getNameInitAsString() + "'!");
+                   "Value not specified for template argument #" +
+                   Twine(i) + " (" + TArgs[i]->getAsUnquotedString() +
+                   ") of subclass '" + SC->getNameInitAsString() + "'!");
     }
   }
 
   // Since everything went well, we can now set the "superclass" list for the
   // current record.
-  const std::vector<Record*> &SCs = SC->getSuperClasses();
+  ArrayRef<Record *> SCs = SC->getSuperClasses();
   ArrayRef<SMRange> SCRanges = SC->getSuperClassRanges();
   for (unsigned i = 0, e = SCs.size(); i != e; ++i) {
     if (CurRec->isSubClassOf(SCs[i]))
@@ -217,27 +208,21 @@ bool TGParser::AddSubMultiClass(MultiClass *CurMC,
   MultiClass *SMC = SubMultiClass.MC;
   Record *CurRec = &CurMC->Rec;
 
-  const std::vector<RecordVal> &MCVals = CurRec->getValues();
-
   // Add all of the values in the subclass into the current class.
-  const std::vector<RecordVal> &SMCVals = SMC->Rec.getValues();
-  for (unsigned i = 0, e = SMCVals.size(); i != e; ++i)
-    if (AddValue(CurRec, SubMultiClass.RefRange.Start, SMCVals[i]))
+  for (const auto &SMCVal : SMC->Rec.getValues())
+    if (AddValue(CurRec, SubMultiClass.RefRange.Start, SMCVal))
       return true;
 
   unsigned newDefStart = CurMC->DefPrototypes.size();
 
   // Add all of the defs in the subclass into the current multiclass.
-  for (MultiClass::RecordVector::const_iterator i = SMC->DefPrototypes.begin(),
-         iend = SMC->DefPrototypes.end();
-       i != iend;
-       ++i) {
+  for (const std::unique_ptr<Record> &R : SMC->DefPrototypes) {
     // Clone the def and add it to the current multiclass
-    auto NewDef = make_unique<Record>(**i);
+    auto NewDef = make_unique<Record>(*R);
 
     // Add all of the values in the superclass into the current def.
-    for (unsigned i = 0, e = MCVals.size(); i != e; ++i)
-      if (AddValue(NewDef.get(), SubMultiClass.RefRange.Start, MCVals[i]))
+    for (const auto &MCVal : CurRec->getValues())
+      if (AddValue(NewDef.get(), SubMultiClass.RefRange.Start, MCVal))
         return true;
 
     CurMC->DefPrototypes.push_back(std::move(NewDef));
@@ -285,9 +270,9 @@ bool TGParser::AddSubMultiClass(MultiClass *CurMC,
       }
     } else if (!CurRec->getValue(SMCTArgs[i])->getValue()->isComplete()) {
       return Error(SubMultiClass.RefRange.Start,
-                   "Value not specified for template argument #"
-                   + utostr(i) + " (" + SMCTArgs[i]->getAsUnquotedString()
-                   + ") of subclass '" + SMC->Rec.getNameInitAsString() + "'!");
+                   "Value not specified for template argument #" +
+                   Twine(i) + " (" + SMCTArgs[i]->getAsUnquotedString() +
+                   ") of subclass '" + SMC->Rec.getNameInitAsString() + "'!");
     }
   }
 
@@ -323,7 +308,7 @@ bool TGParser::ProcessForeachDefs(Record *CurRec, SMLoc Loc, IterSet &IterVals){
     }
 
     // Process each value.
-    for (int64_t i = 0; i < List->getSize(); ++i) {
+    for (unsigned i = 0; i < List->size(); ++i) {
       Init *ItemVal = List->resolveListElementReference(*CurRec, nullptr, i);
       IterVals.push_back(IterRecord(CurLoop.IterVar, ItemVal));
       if (ProcessForeachDefs(CurRec, Loc, IterVals))
@@ -339,9 +324,9 @@ bool TGParser::ProcessForeachDefs(Record *CurRec, SMLoc Loc, IterSet &IterVals){
   auto IterRec = make_unique<Record>(*CurRec);
 
   // Set the iterator values now.
-  for (unsigned i = 0, e = IterVals.size(); i != e; ++i) {
-    VarInit *IterVar = IterVals[i].IterVar;
-    TypedInit *IVal = dyn_cast<TypedInit>(IterVals[i].IterValue);
+  for (IterRecord &IR : IterVals) {
+    VarInit *IterVar = IR.IterVar;
+    TypedInit *IVal = dyn_cast<TypedInit>(IR.IterValue);
     if (!IVal)
       return Error(Loc, "foreach iterator value is untyped");
 
@@ -591,13 +576,12 @@ bool TGParser::ParseRangePiece(std::vector<unsigned> &Ranges) {
   Lex.Lex();
 
   // Add to the range.
-  if (Start < End) {
+  if (Start < End)
     for (; Start <= End; ++Start)
       Ranges.push_back(Start);
-  } else {
+  else
     for (; Start >= End; --Start)
       Ranges.push_back(Start);
-  }
   return false;
 }
 
@@ -758,10 +742,8 @@ Init *TGParser::ParseIDValue(Record *CurRec,
   }
 
   // If this is in a foreach loop, make sure it's not a loop iterator
-  for (LoopVector::iterator i = Loops.begin(), iend = Loops.end();
-       i != iend;
-       ++i) {
-    VarInit *IterVar = dyn_cast<VarInit>(i->IterVar);
+  for (const auto &L : Loops) {
+    VarInit *IterVar = dyn_cast<VarInit>(L.IterVar);
     if (IterVar && IterVar->getName() == Name)
       return IterVar;
   }
@@ -776,7 +758,7 @@ Init *TGParser::ParseIDValue(Record *CurRec,
     Error(NameLoc, "Variable not defined: '" + Name + "'");
     return nullptr;
   }
-  
+
   return StringInit::get(Name);
 }
 
@@ -833,9 +815,9 @@ Init *TGParser::ParseOperation(Record *CurRec, RecTy *ItemType) {
     Init *LHS = ParseValue(CurRec);
     if (!LHS) return nullptr;
 
-    if (Code == UnOpInit::HEAD
-        || Code == UnOpInit::TAIL
-        || Code == UnOpInit::EMPTY) {
+    if (Code == UnOpInit::HEAD ||
+        Code == UnOpInit::TAIL ||
+        Code == UnOpInit::EMPTY) {
       ListInit *LHSl = dyn_cast<ListInit>(LHS);
       StringInit *LHSs = dyn_cast<StringInit>(LHS);
       TypedInit *LHSt = dyn_cast<TypedInit>(LHS);
@@ -852,14 +834,13 @@ Init *TGParser::ParseOperation(Record *CurRec, RecTy *ItemType) {
         }
       }
 
-      if (Code == UnOpInit::HEAD
-          || Code == UnOpInit::TAIL) {
+      if (Code == UnOpInit::HEAD || Code == UnOpInit::TAIL) {
         if (!LHSl && !LHSt) {
           TokError("expected list type argument in unary operator");
           return nullptr;
         }
 
-        if (LHSl && LHSl->getSize() == 0) {
+        if (LHSl && LHSl->empty()) {
           TokError("empty list argument in unary operator");
           return nullptr;
         }
@@ -870,11 +851,8 @@ Init *TGParser::ParseOperation(Record *CurRec, RecTy *ItemType) {
             TokError("untyped list element in unary operator");
             return nullptr;
           }
-          if (Code == UnOpInit::HEAD) {
-            Type = Itemt->getType();
-          } else {
-            Type = ListRecTy::get(Itemt->getType());
-          }
+          Type = (Code == UnOpInit::HEAD) ? Itemt->getType()
+                                          : ListRecTy::get(Itemt->getType());
         } else {
           assert(LHSt && "expected list type argument in unary operator");
           ListRecTy *LType = dyn_cast<ListRecTy>(LHSt->getType());
@@ -882,11 +860,7 @@ Init *TGParser::ParseOperation(Record *CurRec, RecTy *ItemType) {
             TokError("expected list type argument in unary operator");
             return nullptr;
           }
-          if (Code == UnOpInit::HEAD) {
-            Type = LType->getElementType();
-          } else {
-            Type = LType;
-          }
+          Type = (Code == UnOpInit::HEAD) ? LType->getElementType() : LType;
         }
       }
     }
@@ -1256,11 +1230,8 @@ Init *TGParser::ParseSimpleValue(Record *CurRec, RecTy *ItemType,
       CurMultiClass->DefPrototypes.push_back(std::move(NewRecOwner));
 
       // Copy the template arguments for the multiclass into the def.
-      const std::vector<Init *> &TArgs =
-                                  CurMultiClass->Rec.getTemplateArgs();
-
-      for (unsigned i = 0, e = TArgs.size(); i != e; ++i) {
-        const RecordVal *RV = CurMultiClass->Rec.getValue(TArgs[i]);
+      for (Init *TArg : CurMultiClass->Rec.getTemplateArgs()) {
+        const RecordVal *RV = CurMultiClass->Rec.getValue(TArg);
         assert(RV && "Template arg doesn't exist?");
         NewRec->addValue(*RV);
       }
@@ -1324,7 +1295,7 @@ Init *TGParser::ParseSimpleValue(Record *CurRec, RecTy *ItemType,
       // All other values must be convertible to just a single bit.
       Init *Bit = Vals[i]->convertInitializerTo(BitRecTy::get());
       if (!Bit) {
-        Error(BraceLoc, "Element #" + utostr(i) + " (" + Vals[i]->getAsString()+
+        Error(BraceLoc, "Element #" + Twine(i) + " (" + Vals[i]->getAsString() +
               ") is not convertable to a bit");
         return nullptr;
       }
@@ -1343,11 +1314,8 @@ Init *TGParser::ParseSimpleValue(Record *CurRec, RecTy *ItemType,
     if (ItemType) {
       ListRecTy *ListType = dyn_cast<ListRecTy>(ItemType);
       if (!ListType) {
-        std::string s;
-        raw_string_ostream ss(s);
-        ss << "Type mismatch for list, expected list type, got "
-           << ItemType->getAsString();
-        TokError(ss.str());
+        TokError(Twine("Type mismatch for list, expected list type, got ") +
+                 ItemType->getAsString());
         return nullptr;
       }
       GivenListTy = ListType;
@@ -1384,10 +1352,8 @@ Init *TGParser::ParseSimpleValue(Record *CurRec, RecTy *ItemType,
 
     // Check elements
     RecTy *EltTy = nullptr;
-    for (std::vector<Init *>::iterator i = Vals.begin(), ie = Vals.end();
-         i != ie;
-         ++i) {
-      TypedInit *TArg = dyn_cast<TypedInit>(*i);
+    for (Init *V : Vals) {
+      TypedInit *TArg = dyn_cast<TypedInit>(V);
       if (!TArg) {
         TokError("Untyped list element");
         return nullptr;
@@ -1578,7 +1544,7 @@ Init *TGParser::ParseValue(Record *CurRec, RecTy *ItemType, IDParseMode Mode) {
         Error(PasteLoc, "LHS of paste is not typed!");
         return nullptr;
       }
-  
+
       if (LHS->getType() != StringRecTy::get()) {
         LHS = UnOpInit::get(UnOpInit::CAST, LHS, StringRecTy::get());
       }
@@ -1593,7 +1559,7 @@ Init *TGParser::ParseValue(Record *CurRec, RecTy *ItemType, IDParseMode Mode) {
         // These are all of the tokens that can begin an object body.
         // Some of these can also begin values but we disallow those cases
         // because they are unlikely to be useful.
-       
+
         // Trailing paste, concat with an empty string.
         RHS = StringInit::get("");
         break;
@@ -1609,7 +1575,7 @@ Init *TGParser::ParseValue(Record *CurRec, RecTy *ItemType, IDParseMode Mode) {
         if (RHS->getType() != StringRecTy::get()) {
           RHS = UnOpInit::get(UnOpInit::CAST, RHS, StringRecTy::get());
         }
-  
+
         break;
       }
 
@@ -1634,7 +1600,7 @@ TGParser::ParseDagArgList(Record *CurRec) {
     // DagArg ::= VARNAME
     if (Lex.getCode() == tgtok::VarName) {
       // A missing value is treated like '?'.
-      Result.push_back(std::make_pair(UnsetInit::get(), Lex.getCurStrVal()));
+      Result.emplace_back(UnsetInit::get(), Lex.getCurStrVal());
       Lex.Lex();
     } else {
       // DagArg ::= Value (':' VARNAME)?
@@ -1743,11 +1709,10 @@ Init *TGParser::ParseDeclaration(Record *CurRec,
   Lex.Lex();
 
   if (ParsingTemplateArgs) {
-    if (CurRec) {
+    if (CurRec)
       DeclName = QualifyName(*CurRec, CurMultiClass, DeclName, ":");
-    } else {
+    else
       assert(CurMultiClass);
-    }
     if (CurMultiClass)
       DeclName = QualifyName(CurMultiClass->Rec, CurMultiClass, DeclName,
                              "::");
@@ -1841,8 +1806,8 @@ VarInit *TGParser::ParseForeachDeclaration(ListInit *&ForeachListValue) {
     assert(!IterType && "Type already initialized?");
     IterType = IntRecTy::get();
     std::vector<Init*> Values;
-    for (unsigned i = 0, e = Ranges.size(); i != e; ++i)
-      Values.push_back(IntInit::get(Ranges[i]));
+    for (unsigned R : Ranges)
+      Values.push_back(IntInit::get(R));
     ForeachListValue = ListInit::get(Values, IterType);
   }
 
@@ -1968,10 +1933,9 @@ bool TGParser::ParseBody(Record *CurRec) {
 /// \brief Apply the current let bindings to \a CurRec.
 /// \returns true on error, false otherwise.
 bool TGParser::ApplyLetStack(Record *CurRec) {
-  for (unsigned i = 0, e = LetStack.size(); i != e; ++i)
-    for (unsigned j = 0, e = LetStack[i].size(); j != e; ++j)
-      if (SetValue(CurRec, LetStack[i][j].Loc, LetStack[i][j].Name,
-                   LetStack[i][j].Bits, LetStack[i][j].Value))
+  for (std::vector<LetRecord> &LetInfo : LetStack)
+    for (LetRecord &LR : LetInfo)
+      if (SetValue(CurRec, LR.Loc, LR.Name, LR.Bits, LR.Value))
         return true;
   return false;
 }
@@ -2053,9 +2017,8 @@ bool TGParser::ParseDef(MultiClass *CurMultiClass) {
       return true;
 
     // Otherwise, a def inside a multiclass, add it to the multiclass.
-    for (unsigned i = 0, e = CurMultiClass->DefPrototypes.size(); i != e; ++i)
-      if (CurMultiClass->DefPrototypes[i]->getNameInit()
-          == CurRec->getNameInit())
+    for (const auto &Proto : CurMultiClass->DefPrototypes)
+      if (Proto->getNameInit() == CurRec->getNameInit())
         return Error(DefLoc, "def '" + CurRec->getNameInitAsString() +
                      "' already defined in this multiclass!");
     CurMultiClass->DefPrototypes.push_back(std::move(CurRecOwner));
@@ -2074,20 +2037,16 @@ bool TGParser::ParseDef(MultiClass *CurMultiClass) {
 
   if (CurMultiClass) {
     // Copy the template arguments for the multiclass into the def.
-    const std::vector<Init *> &TArgs =
-                                CurMultiClass->Rec.getTemplateArgs();
-
-    for (unsigned i = 0, e = TArgs.size(); i != e; ++i) {
-      const RecordVal *RV = CurMultiClass->Rec.getValue(TArgs[i]);
+    for (Init *TArg : CurMultiClass->Rec.getTemplateArgs()) {
+      const RecordVal *RV = CurMultiClass->Rec.getValue(TArg);
       assert(RV && "Template arg doesn't exist?");
       CurRec->addValue(*RV);
     }
   }
 
-  if (ProcessForeachDefs(CurRec, DefLoc)) {
+  if (ProcessForeachDefs(CurRec, DefLoc))
     return Error(DefLoc, "Could not process loops for def" +
                  CurRec->getNameInitAsString());
-  }
 
   return false;
 }
@@ -2120,8 +2079,7 @@ bool TGParser::ParseForeach(MultiClass *CurMultiClass) {
     // FOREACH Declaration IN Object
     if (ParseObject(CurMultiClass))
       return true;
-  }
-  else {
+  } else {
     SMLoc BraceLoc = Lex.getLoc();
     // Otherwise, this is a group foreach.
     Lex.Lex();  // eat the '{'.
@@ -2160,8 +2118,8 @@ bool TGParser::ParseClass() {
     if (CurRec->getValues().size() > 1 ||  // Account for NAME.
         !CurRec->getSuperClasses().empty() ||
         !CurRec->getTemplateArgs().empty())
-      return TokError("Class '" + CurRec->getNameInitAsString()
-                      + "' already defined");
+      return TokError("Class '" + CurRec->getNameInitAsString() +
+                      "' already defined");
   } else {
     // If this is the first reference to this class, create and add it.
     auto NewRec =
@@ -2214,7 +2172,7 @@ std::vector<LetRecord> TGParser::ParseLetList() {
     if (!Val) return std::vector<LetRecord>();
 
     // Now that we have everything, add the record.
-    Result.push_back(LetRecord(Name, Bits, Val, NameLoc));
+    Result.emplace_back(std::move(Name), std::move(Bits), Val, NameLoc);
 
     if (Lex.getCode() != tgtok::comma)
       return Result;
@@ -2359,7 +2317,9 @@ Record *TGParser::
 InstantiateMulticlassDef(MultiClass &MC,
                          Record *DefProto,
                          Init *&DefmPrefix,
-                         SMRange DefmPrefixRange) {
+                         SMRange DefmPrefixRange,
+                         const std::vector<Init *> &TArgs,
+                         std::vector<Init *> &TemplateVals) {
   // We need to preserve DefProto so it can be reused for later
   // instantiations, so create a new Record to inherit from it.
 
@@ -2375,7 +2335,6 @@ InstantiateMulticlassDef(MultiClass &MC,
   }
 
   Init *DefName = DefProto->getNameInit();
-
   StringInit *DefNameString = dyn_cast<StringInit>(DefName);
 
   if (DefNameString) {
@@ -2403,9 +2362,9 @@ InstantiateMulticlassDef(MultiClass &MC,
   // confused.
   if (SetValue(CurRec.get(), Ref.RefRange.Start, "NAME",
                std::vector<unsigned>(), DefmPrefix)) {
-    Error(DefmPrefixRange.Start, "Could not resolve "
-          + CurRec->getNameInitAsString() + ":NAME to '"
-          + DefmPrefix->getAsUnquotedString() + "'");
+    Error(DefmPrefixRange.Start, "Could not resolve " +
+          CurRec->getNameInitAsString() + ":NAME to '" +
+          DefmPrefix->getAsUnquotedString() + "'");
     return nullptr;
   }
 
@@ -2423,12 +2382,40 @@ InstantiateMulticlassDef(MultiClass &MC,
     RecordVal *DefNameRV = CurRec->getValue("NAME");
     CurRec->resolveReferencesTo(DefNameRV);
 
+    // Check if the name is a complex pattern.
+    // If so, resolve it.
+    DefName = CurRec->getNameInit();
+    DefNameString = dyn_cast<StringInit>(DefName);
+
+    // OK the pattern is more complex than simply using NAME.
+    // Let's use the heavy weaponery.
+    if (!DefNameString) {
+      ResolveMulticlassDefArgs(MC, CurRec.get(), DefmPrefixRange.Start,
+                               Lex.getLoc(), TArgs, TemplateVals,
+                               false/*Delete args*/);
+      DefName = CurRec->getNameInit();
+      DefNameString = dyn_cast<StringInit>(DefName);
+
+      if (!DefNameString)
+        DefName = DefName->convertInitializerTo(StringRecTy::get());
+
+      // We ran out of options here...
+      DefNameString = dyn_cast<StringInit>(DefName);
+      if (!DefNameString) {
+        PrintFatalError(CurRec->getLoc()[CurRec->getLoc().size() - 1],
+                        DefName->getAsUnquotedString() + " is not a string.");
+        return nullptr;
+      }
+
+      CurRec->setName(DefName);
+    }
+
     // Now that NAME references are resolved and we're at the top level of
     // any multiclass expansions, add the record to the RecordKeeper. If we are
     // currently in a multiclass, it means this defm appears inside a
     // multiclass and its name won't be fully resolvable until we see
-    // the top-level defm.  Therefore, we don't add this to the
-    // RecordKeeper at this point.  If we did we could get duplicate
+    // the top-level defm. Therefore, we don't add this to the
+    // RecordKeeper at this point. If we did we could get duplicate
     // defs as more than one probably refers to NAME or some other
     // common internal placeholder.
 
@@ -2466,19 +2453,19 @@ bool TGParser::ResolveMulticlassDefArgs(MultiClass &MC,
       if (SetValue(CurRec, DefmPrefixLoc, TArgs[i], std::vector<unsigned>(),
                    TemplateVals[i]))
         return true;
-        
+
       // Resolve it next.
       CurRec->resolveReferencesTo(CurRec->getValue(TArgs[i]));
 
       if (DeleteArgs)
         // Now remove it.
         CurRec->removeValue(TArgs[i]);
-        
+
     } else if (!CurRec->getValue(TArgs[i])->getValue()->isComplete()) {
-      return Error(SubClassLoc, "value not specified for template argument #"+
-                   utostr(i) + " (" + TArgs[i]->getAsUnquotedString()
-                   + ") of multiclassclass '" + MC.Rec.getNameInitAsString()
-                   + "'");
+      return Error(SubClassLoc, "value not specified for template argument #" +
+                   Twine(i) + " (" + TArgs[i]->getAsUnquotedString() +
+                   ") of multiclassclass '" + MC.Rec.getNameInitAsString() +
+                   "'");
     }
   }
   return false;
@@ -2497,20 +2484,15 @@ bool TGParser::ResolveMulticlassDef(MultiClass &MC,
   // with the new created definition.
   if (!CurMultiClass)
     return false;
-  for (unsigned i = 0, e = CurMultiClass->DefPrototypes.size();
-       i != e; ++i)
-    if (CurMultiClass->DefPrototypes[i]->getNameInit()
-        == CurRec->getNameInit())
+  for (const auto &Proto : CurMultiClass->DefPrototypes)
+    if (Proto->getNameInit() == CurRec->getNameInit())
       return Error(DefmPrefixLoc, "defm '" + CurRec->getNameInitAsString() +
                    "' already defined in this multiclass!");
   CurMultiClass->DefPrototypes.push_back(std::unique_ptr<Record>(CurRec));
 
   // Copy the template arguments for the multiclass into the new def.
-  const std::vector<Init *> &TA =
-    CurMultiClass->Rec.getTemplateArgs();
-
-  for (unsigned i = 0, e = TA.size(); i != e; ++i) {
-    const RecordVal *RV = CurMultiClass->Rec.getValue(TA[i]);
+  for (Init * TA : CurMultiClass->Rec.getTemplateArgs()) {
+    const RecordVal *RV = CurMultiClass->Rec.getValue(TA);
     assert(RV && "Template arg doesn't exist?");
     CurRec->addValue(*RV);
   }
@@ -2564,20 +2546,25 @@ bool TGParser::ParseDefm(MultiClass *CurMultiClass) {
                    "more template args specified than multiclass expects");
 
     // Loop over all the def's in the multiclass, instantiating each one.
-    for (unsigned i = 0, e = MC->DefPrototypes.size(); i != e; ++i) {
-      Record *DefProto = MC->DefPrototypes[i].get();
-
-      Record *CurRec = InstantiateMulticlassDef(*MC, DefProto, DefmPrefix,
+    for (const std::unique_ptr<Record> &DefProto : MC->DefPrototypes) {
+      // The record name construction goes as follow:
+      //  - If the def name is a string, prepend the prefix.
+      //  - If the def name is a more complex pattern, use that pattern.
+      // As a result, the record is instanciated before resolving
+      // arguments, as it would make its name a string.
+      Record *CurRec = InstantiateMulticlassDef(*MC, DefProto.get(), DefmPrefix,
                                                 SMRange(DefmLoc,
-                                                        DefmPrefixEndLoc));
+                                                        DefmPrefixEndLoc),
+                                                TArgs, TemplateVals);
       if (!CurRec)
         return true;
 
+      // Now that the record is instanciated, we can resolve arguments.
       if (ResolveMulticlassDefArgs(*MC, CurRec, DefmLoc, SubClassLoc,
                                    TArgs, TemplateVals, true/*Delete args*/))
         return Error(SubClassLoc, "could not instantiate def");
 
-      if (ResolveMulticlassDef(*MC, CurRec, DefProto, DefmLoc))
+      if (ResolveMulticlassDef(*MC, CurRec, DefProto.get(), DefmLoc))
         return Error(SubClassLoc, "could not instantiate def");
 
       // Defs that can be used by other definitions should be fully resolved
@@ -2618,9 +2605,7 @@ bool TGParser::ParseDefm(MultiClass *CurMultiClass) {
 
       // Get the expanded definition prototypes and teach them about
       // the record values the current class to inherit has
-      for (unsigned i = 0, e = NewRecDefs.size(); i != e; ++i) {
-        Record *CurRec = NewRecDefs[i];
-
+      for (Record *CurRec : NewRecDefs) {
         // Add it.
         if (AddSubClass(CurRec, SubClass))
           return true;
@@ -2636,11 +2621,11 @@ bool TGParser::ParseDefm(MultiClass *CurMultiClass) {
   }
 
   if (!CurMultiClass)
-    for (unsigned i = 0, e = NewRecDefs.size(); i != e; ++i)
+    for (Record *CurRec : NewRecDefs)
       // See Record::setName().  This resolve step will see any new
       // name for the def that might have been created when resolving
       // inheritance, values and arguments above.
-      NewRecDefs[i]->resolveReferences();
+      CurRec->resolveReferences();
 
   if (Lex.getCode() != tgtok::semi)
     return TokError("expected ';' at end of defm");
diff --git a/lib/TableGen/TGParser.h b/lib/TableGen/TGParser.h
index 22a00e59195..d69d1f4572f 100644
--- a/lib/TableGen/TGParser.h
+++ b/lib/TableGen/TGParser.h
@@ -138,7 +138,9 @@ class TGParser {
   Record *InstantiateMulticlassDef(MultiClass &MC,
                                    Record *DefProto,
                                    Init *&DefmPrefix,
-                                   SMRange DefmPrefixRange);
+                                   SMRange DefmPrefixRange,
+                                   const std::vector<Init *> &TArgs,
+                                   std::vector<Init *> &TemplateVals);
   bool ResolveMulticlassDefArgs(MultiClass &MC,
                                 Record *DefProto,
                                 SMLoc DefmPrefixLoc,
diff --git a/lib/TableGen/TableGenBackend.cpp b/lib/TableGen/TableGenBackend.cpp
index 79d567753a6..77ed8414b15 100644
--- a/lib/TableGen/TableGenBackend.cpp
+++ b/lib/TableGen/TableGenBackend.cpp
@@ -11,10 +11,9 @@
 //
 //===----------------------------------------------------------------------===//
 
+#include "llvm/TableGen/TableGenBackend.h"
 #include "llvm/ADT/Twine.h"
 #include "llvm/Support/raw_ostream.h"
-#include "llvm/TableGen/TableGenBackend.h"
-#include <algorithm>
 
 using namespace llvm;
 
@@ -23,32 +22,30 @@ const size_t MAX_LINE_LEN = 80U;
 static void printLine(raw_ostream &OS, const Twine &Prefix, char Fill,
                       StringRef Suffix) {
   size_t Pos = (size_t)OS.tell();
-  assert((MAX_LINE_LEN - Prefix.str().size() - Suffix.size() > 0) &&
-    "header line exceeds max limit");
+  assert((Prefix.str().size() + Suffix.size() <= MAX_LINE_LEN) &&
+         "header line exceeds max limit");
   OS << Prefix;
-  const size_t e = MAX_LINE_LEN - Suffix.size();
-  for (size_t i = (size_t)OS.tell() - Pos; i < e; ++i)
+  for (size_t i = (size_t)OS.tell() - Pos, e = MAX_LINE_LEN - Suffix.size();
+         i < e; ++i)
     OS << Fill;
   OS << Suffix << '\n';
 }
 
 void llvm::emitSourceFileHeader(StringRef Desc, raw_ostream &OS) {
   printLine(OS, "/*===- TableGen'erated file ", '-', "*- C++ -*-===*\\");
-  printLine(OS, "|*", ' ', "*|");
-  size_t Pos = 0U;
-  size_t PosE;
-  StringRef Prefix("|*");
+  StringRef Prefix("|* ");
   StringRef Suffix(" *|");
-  do{
-    size_t PSLen = Suffix.size() + Prefix.size();
-    PosE = Pos + ((MAX_LINE_LEN > (Desc.size() - PSLen)) ?
-      Desc.size() :
-      MAX_LINE_LEN - PSLen);
-    printLine(OS, Prefix + Desc.slice(Pos, PosE), ' ', Suffix);
-    Pos = PosE;
-  } while(Pos < Desc.size());
   printLine(OS, Prefix, ' ', Suffix);
-  printLine(OS, Prefix + " Automatically generated file, do not edit!", ' ',
+  size_t PSLen = Prefix.size() + Suffix.size();
+  assert(PSLen < MAX_LINE_LEN);
+  size_t Pos = 0U;
+  do {
+    size_t Length = std::min(Desc.size() - Pos, MAX_LINE_LEN - PSLen);
+    printLine(OS, Prefix + Desc.substr(Pos, Length), ' ', Suffix);
+    Pos += Length;
+  } while (Pos < Desc.size());
+  printLine(OS, Prefix, ' ', Suffix);
+  printLine(OS, Prefix + "Automatically generated file, do not edit!", ' ',
     Suffix);
   printLine(OS, Prefix, ' ', Suffix);
   printLine(OS, "\\*===", '-', "===*/");
diff --git a/lib/Target/AArch64/AArch64.td b/lib/Target/AArch64/AArch64.td
index 9a7d6c884db..d19b319ff7e 100644
--- a/lib/Target/AArch64/AArch64.td
+++ b/lib/Target/AArch64/AArch64.td
@@ -40,6 +40,15 @@ def FeatureZCRegMove : SubtargetFeature<"zcm", "HasZeroCycleRegMove", "true",
 def FeatureZCZeroing : SubtargetFeature<"zcz", "HasZeroCycleZeroing", "true",
                                         "Has zero-cycle zeroing instructions">;
 
+def FeatureStrictAlign : SubtargetFeature<"strict-align",
+                                          "StrictAlign", "true",
+                                          "Disallow all unaligned memory "
+                                          "access">;
+
+def FeatureReserveX18 : SubtargetFeature<"reserve-x18", "ReserveX18", "true",
+                                         "Reserve X18, making it unavailable "
+                                         "as a GPR">;
+
 //===----------------------------------------------------------------------===//
 // Architectures.
 //
diff --git a/lib/Target/AArch64/AArch64A53Fix835769.cpp b/lib/Target/AArch64/AArch64A53Fix835769.cpp
index 3bc5a54b5ae..398269433ec 100644
--- a/lib/Target/AArch64/AArch64A53Fix835769.cpp
+++ b/lib/Target/AArch64/AArch64A53Fix835769.cpp
@@ -48,7 +48,7 @@ static bool isFirstInstructionInSequence(MachineInstr *MI) {
   case AArch64::PRFUMi:
     return true;
   default:
-    return (MI->mayLoad() || MI->mayStore());
+    return MI->mayLoadOrStore();
   }
 }
 
@@ -151,10 +151,9 @@ static MachineInstr *getLastNonPseudo(MachineBasicBlock &MBB,
   // If there is no non-pseudo in the current block, loop back around and try
   // the previous block (if there is one).
   while ((FMBB = getBBFallenThrough(FMBB, TII))) {
-    for (auto I = FMBB->rbegin(), E = FMBB->rend(); I != E; ++I) {
-      if (!I->isPseudo())
-        return &*I;
-    }
+    for (MachineInstr &I : make_range(FMBB->rbegin(), FMBB->rend()))
+      if (!I.isPseudo())
+        return &I;
   }
 
   // There was no previous non-pseudo in the fallen through blocks
diff --git a/lib/Target/AArch64/AArch64A57FPLoadBalancing.cpp b/lib/Target/AArch64/AArch64A57FPLoadBalancing.cpp
index bffd9e6e8c7..79a84ad8c6c 100644
--- a/lib/Target/AArch64/AArch64A57FPLoadBalancing.cpp
+++ b/lib/Target/AArch64/AArch64A57FPLoadBalancing.cpp
@@ -510,9 +510,17 @@ int AArch64A57FPLoadBalancing::scavengeRegister(Chain *G, Color C,
       if (J.isRegMask())
         AvailableRegs.clearBitsNotInMask(J.getRegMask());
 
-      if (J.isReg() && J.isDef() && AvailableRegs[J.getReg()]) {
-        assert(J.isDead() && "Non-dead def should have been removed by now!");
-        AvailableRegs.reset(J.getReg());
+      if (J.isReg() && J.isDef()) {
+        MCRegAliasIterator AI(J.getReg(), TRI, /*IncludeSelf=*/true);
+        if (J.isDead())
+          for (; AI.isValid(); ++AI)
+            AvailableRegs.reset(*AI);
+#ifndef NDEBUG
+        else
+          for (; AI.isValid(); ++AI)
+            assert(!AvailableRegs[*AI] &&
+                   "Non-dead def should have been removed by now!");
+#endif
       }
     }
   }
@@ -585,7 +593,6 @@ bool AArch64A57FPLoadBalancing::colorChain(Chain *G, Color C,
       if (Change) {
         Substs[MO.getReg()] = Reg;
         MO.setReg(Reg);
-        MRI->setPhysRegUsed(Reg);
 
         Changed = true;
       }
diff --git a/lib/Target/AArch64/AArch64AdvSIMDScalarPass.cpp b/lib/Target/AArch64/AArch64AdvSIMDScalarPass.cpp
index f27dfc99027..18d21fd3861 100644
--- a/lib/Target/AArch64/AArch64AdvSIMDScalarPass.cpp
+++ b/lib/Target/AArch64/AArch64AdvSIMDScalarPass.cpp
@@ -158,7 +158,7 @@ static unsigned getSrcFromCopy(const MachineInstr *MI,
 // getTransformOpcode - For any opcode for which there is an AdvSIMD equivalent
 // that we're considering transforming to, return that AdvSIMD opcode. For all
 // others, return the original opcode.
-static int getTransformOpcode(unsigned Opc) {
+static unsigned getTransformOpcode(unsigned Opc) {
   switch (Opc) {
   default:
     break;
@@ -179,7 +179,7 @@ static int getTransformOpcode(unsigned Opc) {
 }
 
 static bool isTransformable(const MachineInstr *MI) {
-  int Opc = MI->getOpcode();
+  unsigned Opc = MI->getOpcode();
   return Opc != getTransformOpcode(Opc);
 }
 
@@ -286,8 +286,8 @@ void AArch64AdvSIMDScalar::transformInstruction(MachineInstr *MI) {
   DEBUG(dbgs() << "Scalar transform: " << *MI);
 
   MachineBasicBlock *MBB = MI->getParent();
-  int OldOpc = MI->getOpcode();
-  int NewOpc = getTransformOpcode(OldOpc);
+  unsigned OldOpc = MI->getOpcode();
+  unsigned NewOpc = getTransformOpcode(OldOpc);
   assert(OldOpc != NewOpc && "transform an instruction to itself?!");
 
   // Check if we need a copy for the source registers.
diff --git a/lib/Target/AArch64/AArch64AsmPrinter.cpp b/lib/Target/AArch64/AArch64AsmPrinter.cpp
index 0821cff3fc4..ada995bad37 100644
--- a/lib/Target/AArch64/AArch64AsmPrinter.cpp
+++ b/lib/Target/AArch64/AArch64AsmPrinter.cpp
@@ -121,14 +121,14 @@ class AArch64AsmPrinter : public AsmPrinter {
 //===----------------------------------------------------------------------===//
 
 void AArch64AsmPrinter::EmitEndOfAsmFile(Module &M) {
-  Triple TT(TM.getTargetTriple());
+  const Triple &TT = TM.getTargetTriple();
   if (TT.isOSBinFormatMachO()) {
     // Funny Darwin hack: This flag tells the linker that no global symbols
     // contain code that falls through to other global symbols (e.g. the obvious
     // implementation of multiple entry points).  If this doesn't occur, the
     // linker can safely perform dead code stripping.  Since LLVM never
     // generates code that does this, it is always safe to set.
-    OutStreamer.EmitAssemblerFlag(MCAF_SubsectionsViaSymbols);
+    OutStreamer->EmitAssemblerFlag(MCAF_SubsectionsViaSymbols);
     SM.serializeToStackMapSection();
   }
 }
@@ -156,7 +156,7 @@ void AArch64AsmPrinter::EmitLOHs() {
              "Label hasn't been inserted for LOH related instruction");
       MCArgs.push_back(LabelIt->second);
     }
-    OutStreamer.EmitLOHDirective(D.getKind(), MCArgs);
+    OutStreamer->EmitLOHDirective(D.getKind(), MCArgs);
     MCArgs.clear();
   }
 }
@@ -172,11 +172,11 @@ MCSymbol *AArch64AsmPrinter::GetCPISymbol(unsigned CPID) const {
   // avoid addends on the relocation?), ELF has no such concept and
   // uses a normal private symbol.
   if (getDataLayout().getLinkerPrivateGlobalPrefix()[0])
-    return OutContext.GetOrCreateSymbol(
+    return OutContext.getOrCreateSymbol(
         Twine(getDataLayout().getLinkerPrivateGlobalPrefix()) + "CPI" +
         Twine(getFunctionNumber()) + "_" + Twine(CPID));
 
-  return OutContext.GetOrCreateSymbol(
+  return OutContext.getOrCreateSymbol(
       Twine(getDataLayout().getPrivateGlobalPrefix()) + "CPI" +
       Twine(getFunctionNumber()) + "_" + Twine(CPID));
 }
@@ -206,7 +206,7 @@ void AArch64AsmPrinter::printOperand(const MachineInstr *MI, unsigned OpNum,
     // FIXME: Can we get anything other than a plain symbol here?
     assert(!MO.getTargetFlags() && "Unknown operand target flag!");
 
-    O << *Sym;
+    Sym->print(O, MAI);
     printOffset(MO.getOffset(), O);
     break;
   }
@@ -348,7 +348,7 @@ void AArch64AsmPrinter::PrintDebugValueComment(const MachineInstr *MI,
   assert(NOps == 4);
   OS << '\t' << MAI->getCommentString() << "DEBUG_VALUE: ";
   // cast away const; DIetc do not take const operands for some reason.
-  OS << cast<MDLocalVariable>(MI->getOperand(NOps - 2).getMetadata())
+  OS << cast<DILocalVariable>(MI->getOperand(NOps - 2).getMetadata())
             ->getName();
   OS << " <- ";
   // Frame address.  Currently handles register +- offset only.
@@ -436,7 +436,7 @@ void AArch64AsmPrinter::LowerPATCHPOINT(MCStreamer &OutStreamer, StackMaps &SM,
 
 void AArch64AsmPrinter::EmitInstruction(const MachineInstr *MI) {
   // Do any auto-generated pseudo lowerings.
-  if (emitPseudoExpansionLowering(OutStreamer, MI))
+  if (emitPseudoExpansionLowering(*OutStreamer, MI))
     return;
 
   if (AArch64FI->getLOHRelated().count(MI)) {
@@ -444,7 +444,7 @@ void AArch64AsmPrinter::EmitInstruction(const MachineInstr *MI) {
     MCSymbol *LOHLabel = createTempSymbol("loh");
     // Associate the instruction with the label
     LOHInstToLabel[MI] = LOHLabel;
-    OutStreamer.EmitLabel(LOHLabel);
+    OutStreamer->EmitLabel(LOHLabel);
   }
 
   // Do any manual lowerings.
@@ -452,11 +452,11 @@ void AArch64AsmPrinter::EmitInstruction(const MachineInstr *MI) {
   default:
     break;
   case AArch64::DBG_VALUE: {
-    if (isVerbose() && OutStreamer.hasRawTextSupport()) {
+    if (isVerbose() && OutStreamer->hasRawTextSupport()) {
       SmallString<128> TmpStr;
       raw_svector_ostream OS(TmpStr);
       PrintDebugValueComment(MI, OS);
-      OutStreamer.EmitRawText(StringRef(OS.str()));
+      OutStreamer->EmitRawText(StringRef(OS.str()));
     }
     return;
   }
@@ -467,8 +467,8 @@ void AArch64AsmPrinter::EmitInstruction(const MachineInstr *MI) {
   case AArch64::TCRETURNri: {
     MCInst TmpInst;
     TmpInst.setOpcode(AArch64::BR);
-    TmpInst.addOperand(MCOperand::CreateReg(MI->getOperand(0).getReg()));
-    EmitToStreamer(OutStreamer, TmpInst);
+    TmpInst.addOperand(MCOperand::createReg(MI->getOperand(0).getReg()));
+    EmitToStreamer(*OutStreamer, TmpInst);
     return;
   }
   case AArch64::TCRETURNdi: {
@@ -477,7 +477,7 @@ void AArch64AsmPrinter::EmitInstruction(const MachineInstr *MI) {
     MCInst TmpInst;
     TmpInst.setOpcode(AArch64::B);
     TmpInst.addOperand(Dest);
-    EmitToStreamer(OutStreamer, TmpInst);
+    EmitToStreamer(*OutStreamer, TmpInst);
     return;
   }
   case AArch64::TLSDESC_CALLSEQ: {
@@ -500,52 +500,52 @@ void AArch64AsmPrinter::EmitInstruction(const MachineInstr *MI) {
 
     MCInst Adrp;
     Adrp.setOpcode(AArch64::ADRP);
-    Adrp.addOperand(MCOperand::CreateReg(AArch64::X0));
+    Adrp.addOperand(MCOperand::createReg(AArch64::X0));
     Adrp.addOperand(SymTLSDesc);
-    EmitToStreamer(OutStreamer, Adrp);
+    EmitToStreamer(*OutStreamer, Adrp);
 
     MCInst Ldr;
     Ldr.setOpcode(AArch64::LDRXui);
-    Ldr.addOperand(MCOperand::CreateReg(AArch64::X1));
-    Ldr.addOperand(MCOperand::CreateReg(AArch64::X0));
+    Ldr.addOperand(MCOperand::createReg(AArch64::X1));
+    Ldr.addOperand(MCOperand::createReg(AArch64::X0));
     Ldr.addOperand(SymTLSDescLo12);
-    Ldr.addOperand(MCOperand::CreateImm(0));
-    EmitToStreamer(OutStreamer, Ldr);
+    Ldr.addOperand(MCOperand::createImm(0));
+    EmitToStreamer(*OutStreamer, Ldr);
 
     MCInst Add;
     Add.setOpcode(AArch64::ADDXri);
-    Add.addOperand(MCOperand::CreateReg(AArch64::X0));
-    Add.addOperand(MCOperand::CreateReg(AArch64::X0));
+    Add.addOperand(MCOperand::createReg(AArch64::X0));
+    Add.addOperand(MCOperand::createReg(AArch64::X0));
     Add.addOperand(SymTLSDescLo12);
-    Add.addOperand(MCOperand::CreateImm(AArch64_AM::getShiftValue(0)));
-    EmitToStreamer(OutStreamer, Add);
+    Add.addOperand(MCOperand::createImm(AArch64_AM::getShiftValue(0)));
+    EmitToStreamer(*OutStreamer, Add);
 
     // Emit a relocation-annotation. This expands to no code, but requests
     // the following instruction gets an R_AARCH64_TLSDESC_CALL.
     MCInst TLSDescCall;
     TLSDescCall.setOpcode(AArch64::TLSDESCCALL);
     TLSDescCall.addOperand(Sym);
-    EmitToStreamer(OutStreamer, TLSDescCall);
+    EmitToStreamer(*OutStreamer, TLSDescCall);
 
     MCInst Blr;
     Blr.setOpcode(AArch64::BLR);
-    Blr.addOperand(MCOperand::CreateReg(AArch64::X1));
-    EmitToStreamer(OutStreamer, Blr);
+    Blr.addOperand(MCOperand::createReg(AArch64::X1));
+    EmitToStreamer(*OutStreamer, Blr);
 
     return;
   }
 
   case TargetOpcode::STACKMAP:
-    return LowerSTACKMAP(OutStreamer, SM, *MI);
+    return LowerSTACKMAP(*OutStreamer, SM, *MI);
 
   case TargetOpcode::PATCHPOINT:
-    return LowerPATCHPOINT(OutStreamer, SM, *MI);
+    return LowerPATCHPOINT(*OutStreamer, SM, *MI);
   }
 
   // Finally, do the automated lowerings for everything else.
   MCInst TmpInst;
   MCInstLowering.Lower(MI, TmpInst);
-  EmitToStreamer(OutStreamer, TmpInst);
+  EmitToStreamer(*OutStreamer, TmpInst);
 }
 
 // Force static initialization.
diff --git a/lib/Target/AArch64/AArch64CallingConvention.h b/lib/Target/AArch64/AArch64CallingConvention.h
index 1e2d1c3b93b..68f9dcf3fc2 100644
--- a/lib/Target/AArch64/AArch64CallingConvention.h
+++ b/lib/Target/AArch64/AArch64CallingConvention.h
@@ -45,10 +45,8 @@ static bool finishStackBlock(SmallVectorImpl<CCValAssign> &PendingMembers,
                              MVT LocVT, ISD::ArgFlagsTy &ArgFlags,
                              CCState &State, unsigned SlotAlign) {
   unsigned Size = LocVT.getSizeInBits() / 8;
-  unsigned StackAlign = State.getMachineFunction()
-                            .getTarget()
-                            .getDataLayout()
-                            ->getStackAlignment();
+  unsigned StackAlign =
+      State.getMachineFunction().getDataLayout().getStackAlignment();
   unsigned Align = std::min(ArgFlags.getOrigAlign(), StackAlign);
 
   for (auto &It : PendingMembers) {
diff --git a/lib/Target/AArch64/AArch64CallingConvention.td b/lib/Target/AArch64/AArch64CallingConvention.td
index 4691e949838..948b9ddb5df 100644
--- a/lib/Target/AArch64/AArch64CallingConvention.td
+++ b/lib/Target/AArch64/AArch64CallingConvention.td
@@ -16,7 +16,7 @@ class CCIfAlign<string Align, CCAction A> :
   CCIf<!strconcat("ArgFlags.getOrigAlign() == ", Align), A>;
 /// CCIfBigEndian - Match only if we're in big endian mode.
 class CCIfBigEndian<CCAction A> :
-  CCIf<"State.getMachineFunction().getTarget().getDataLayout()->isBigEndian()", A>;
+  CCIf<"State.getMachineFunction().getDataLayout().isBigEndian()", A>;
 
 //===----------------------------------------------------------------------===//
 // ARM AAPCS64 Calling Convention
@@ -40,6 +40,11 @@ def CC_AArch64_AAPCS : CallingConv<[
   // slot is 64-bit.
   CCIfByVal<CCPassByVal<8, 8>>,
 
+  // The 'nest' parameter, if any, is passed in X18.
+  // Darwin uses X18 as the platform register and hence 'nest' isn't currently
+  // supported there.
+  CCIfNest<CCAssignToReg<[X18]>>,
+
   CCIfConsecutiveRegs<CCCustom<"CC_AArch64_Custom_Block">>,
 
   // Handle i1, i8, i16, i32, i64, f32, f64 and v2f64 by passing in registers,
diff --git a/lib/Target/AArch64/AArch64ConditionOptimizer.cpp b/lib/Target/AArch64/AArch64ConditionOptimizer.cpp
index e68571f1c24..fc27bfee73d 100644
--- a/lib/Target/AArch64/AArch64ConditionOptimizer.cpp
+++ b/lib/Target/AArch64/AArch64ConditionOptimizer.cpp
@@ -59,6 +59,7 @@
 //===----------------------------------------------------------------------===//
 
 #include "AArch64.h"
+#include "MCTargetDesc/AArch64AddressingModes.h"
 #include "llvm/ADT/DepthFirstIterator.h"
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/ADT/Statistic.h"
@@ -67,6 +68,7 @@
 #include "llvm/CodeGen/MachineFunction.h"
 #include "llvm/CodeGen/MachineFunctionPass.h"
 #include "llvm/CodeGen/MachineInstrBuilder.h"
+#include "llvm/CodeGen/MachineRegisterInfo.h"
 #include "llvm/CodeGen/Passes.h"
 #include "llvm/Support/CommandLine.h"
 #include "llvm/Support/Debug.h"
@@ -86,11 +88,12 @@ namespace {
 class AArch64ConditionOptimizer : public MachineFunctionPass {
   const TargetInstrInfo *TII;
   MachineDominatorTree *DomTree;
+  const MachineRegisterInfo *MRI;
 
 public:
   // Stores immediate, compare instruction opcode and branch condition (in this
   // order) of adjusted comparison.
-  typedef std::tuple<int, int, AArch64CC::CondCode> CmpInfo;
+  typedef std::tuple<int, unsigned, AArch64CC::CondCode> CmpInfo;
 
   static char ID;
   AArch64ConditionOptimizer() : MachineFunctionPass(ID) {}
@@ -116,7 +119,6 @@ void initializeAArch64ConditionOptimizerPass(PassRegistry &);
 INITIALIZE_PASS_BEGIN(AArch64ConditionOptimizer, "aarch64-condopt",
                       "AArch64 CondOpt Pass", false, false)
 INITIALIZE_PASS_DEPENDENCY(MachineDominatorTree)
-INITIALIZE_PASS_DEPENDENCY(LiveIntervals)
 INITIALIZE_PASS_END(AArch64ConditionOptimizer, "aarch64-condopt",
                     "AArch64 CondOpt Pass", false, false)
 
@@ -127,8 +129,6 @@ FunctionPass *llvm::createAArch64ConditionOptimizerPass() {
 void AArch64ConditionOptimizer::getAnalysisUsage(AnalysisUsage &AU) const {
   AU.addRequired<MachineDominatorTree>();
   AU.addPreserved<MachineDominatorTree>();
-  AU.addRequired<LiveIntervals>();
-  AU.addPreserved<LiveIntervals>();
   MachineFunctionPass::getAnalysisUsage(AU);
 }
 
@@ -154,13 +154,20 @@ MachineInstr *AArch64ConditionOptimizer::findSuitableCompare(
     case AArch64::SUBSXri:
     // cmn is an alias for adds with a dead destination register.
     case AArch64::ADDSWri:
-    case AArch64::ADDSXri:
-      if (I->getOperand(0).isDead())
-        return I;
-
-      DEBUG(dbgs() << "Destination of cmp is not dead, " << *I << '\n');
-      return nullptr;
-
+    case AArch64::ADDSXri: {
+      unsigned ShiftAmt = AArch64_AM::getShiftValue(I->getOperand(3).getImm());
+      if (!I->getOperand(2).isImm()) {
+        DEBUG(dbgs() << "Immediate of cmp is symbolic, " << *I << '\n');
+        return nullptr;
+      } else if (I->getOperand(2).getImm() << ShiftAmt >= 0xfff) {
+        DEBUG(dbgs() << "Immediate of cmp may be out of range, " << *I << '\n');
+        return nullptr;
+      } else if (!MRI->use_empty(I->getOperand(0).getReg())) {
+        DEBUG(dbgs() << "Destination of cmp is not dead, " << *I << '\n');
+        return nullptr;
+      }
+      return I;
+    }
     // Prevent false positive case like:
     // cmp      w19, #0
     // cinc     w0, w19, gt
@@ -216,7 +223,7 @@ static AArch64CC::CondCode getAdjustedCmp(AArch64CC::CondCode Cmp) {
 // operator and condition code.
 AArch64ConditionOptimizer::CmpInfo AArch64ConditionOptimizer::adjustCmp(
     MachineInstr *CmpMI, AArch64CC::CondCode Cmp) {
-  int Opc = CmpMI->getOpcode();
+  unsigned Opc = CmpMI->getOpcode();
 
   // CMN (compare with negative immediate) is an alias to ADDS (as
   // "operand - negative" == "operand + positive")
@@ -245,7 +252,7 @@ AArch64ConditionOptimizer::CmpInfo AArch64ConditionOptimizer::adjustCmp(
 void AArch64ConditionOptimizer::modifyCmp(MachineInstr *CmpMI,
     const CmpInfo &Info) {
   int Imm;
-  int Opc;
+  unsigned Opc;
   AArch64CC::CondCode Cmp;
   std::tie(Imm, Opc, Cmp) = Info;
 
@@ -306,6 +313,7 @@ bool AArch64ConditionOptimizer::runOnMachineFunction(MachineFunction &MF) {
                << "********** Function: " << MF.getName() << '\n');
   TII = MF.getSubtarget().getInstrInfo();
   DomTree = &getAnalysis<MachineDominatorTree>();
+  MRI = &MF.getRegInfo();
 
   bool Changed = false;
 
diff --git a/lib/Target/AArch64/AArch64ConditionalCompares.cpp b/lib/Target/AArch64/AArch64ConditionalCompares.cpp
index fccd8df55b4..2b0c92fe02d 100644
--- a/lib/Target/AArch64/AArch64ConditionalCompares.cpp
+++ b/lib/Target/AArch64/AArch64ConditionalCompares.cpp
@@ -416,7 +416,7 @@ bool SSACCmpConv::canSpeculateInstrs(MachineBasicBlock *MBB,
 
     // We never speculate stores, so an AA pointer isn't necessary.
     bool DontMoveAcrossStore = true;
-    if (!I.isSafeToMove(TII, nullptr, DontMoveAcrossStore)) {
+    if (!I.isSafeToMove(nullptr, DontMoveAcrossStore)) {
       DEBUG(dbgs() << "Can't speculate: " << I);
       return false;
     }
diff --git a/lib/Target/AArch64/AArch64FastISel.cpp b/lib/Target/AArch64/AArch64FastISel.cpp
index c3f6859f510..1f9882ed31e 100644
--- a/lib/Target/AArch64/AArch64FastISel.cpp
+++ b/lib/Target/AArch64/AArch64FastISel.cpp
@@ -36,6 +36,7 @@
 #include "llvm/IR/Instructions.h"
 #include "llvm/IR/IntrinsicInst.h"
 #include "llvm/IR/Operator.h"
+#include "llvm/MC/MCSymbol.h"
 #include "llvm/Support/CommandLine.h"
 using namespace llvm;
 
@@ -309,7 +310,7 @@ CCAssignFn *AArch64FastISel::CCAssignFnForCall(CallingConv::ID CC) const {
 }
 
 unsigned AArch64FastISel::fastMaterializeAlloca(const AllocaInst *AI) {
-  assert(TLI.getValueType(AI->getType(), true) == MVT::i64 &&
+  assert(TLI.getValueType(DL, AI->getType(), true) == MVT::i64 &&
          "Alloca should always return a pointer.");
 
   // Don't handle dynamic allocas.
@@ -419,7 +420,7 @@ unsigned AArch64FastISel::materializeGV(const GlobalValue *GV) {
 
   unsigned char OpFlags = Subtarget->ClassifyGlobalReference(GV, TM);
 
-  EVT DestEVT = TLI.getValueType(GV->getType(), true);
+  EVT DestEVT = TLI.getValueType(DL, GV->getType(), true);
   if (!DestEVT.isSimple())
     return 0;
 
@@ -458,7 +459,7 @@ unsigned AArch64FastISel::materializeGV(const GlobalValue *GV) {
 }
 
 unsigned AArch64FastISel::fastMaterializeConstant(const Constant *C) {
-  EVT CEVT = TLI.getValueType(C->getType(), true);
+  EVT CEVT = TLI.getValueType(DL, C->getType(), true);
 
   // Only handle simple types.
   if (!CEVT.isSimple())
@@ -537,13 +538,14 @@ bool AArch64FastISel::computeAddress(const Value *Obj, Address &Addr, Type *Ty)
   }
   case Instruction::IntToPtr: {
     // Look past no-op inttoptrs.
-    if (TLI.getValueType(U->getOperand(0)->getType()) == TLI.getPointerTy())
+    if (TLI.getValueType(DL, U->getOperand(0)->getType()) ==
+        TLI.getPointerTy(DL))
       return computeAddress(U->getOperand(0), Addr, Ty);
     break;
   }
   case Instruction::PtrToInt: {
     // Look past no-op ptrtoints.
-    if (TLI.getValueType(U->getType()) == TLI.getPointerTy())
+    if (TLI.getValueType(DL, U->getType()) == TLI.getPointerTy(DL))
       return computeAddress(U->getOperand(0), Addr, Ty);
     break;
   }
@@ -664,20 +666,22 @@ bool AArch64FastISel::computeAddress(const Value *Obj, Address &Addr, Type *Ty)
     Addr.setExtendType(AArch64_AM::LSL);
 
     const Value *Src = U->getOperand(0);
-    if (const auto *I = dyn_cast<Instruction>(Src))
-      if (FuncInfo.MBBMap[I->getParent()] == FuncInfo.MBB)
-        Src = I;
-
-    // Fold the zext or sext when it won't become a noop.
-    if (const auto *ZE = dyn_cast<ZExtInst>(Src)) {
-      if (!isIntExtFree(ZE) && ZE->getOperand(0)->getType()->isIntegerTy(32)) {
-          Addr.setExtendType(AArch64_AM::UXTW);
-          Src = ZE->getOperand(0);
-      }
-    } else if (const auto *SE = dyn_cast<SExtInst>(Src)) {
-      if (!isIntExtFree(SE) && SE->getOperand(0)->getType()->isIntegerTy(32)) {
-        Addr.setExtendType(AArch64_AM::SXTW);
-        Src = SE->getOperand(0);
+    if (const auto *I = dyn_cast<Instruction>(Src)) {
+      if (FuncInfo.MBBMap[I->getParent()] == FuncInfo.MBB) {
+        // Fold the zext or sext when it won't become a noop.
+        if (const auto *ZE = dyn_cast<ZExtInst>(I)) {
+          if (!isIntExtFree(ZE) &&
+              ZE->getOperand(0)->getType()->isIntegerTy(32)) {
+            Addr.setExtendType(AArch64_AM::UXTW);
+            Src = ZE->getOperand(0);
+          }
+        } else if (const auto *SE = dyn_cast<SExtInst>(I)) {
+          if (!isIntExtFree(SE) &&
+              SE->getOperand(0)->getType()->isIntegerTy(32)) {
+            Addr.setExtendType(AArch64_AM::SXTW);
+            Src = SE->getOperand(0);
+          }
+        }
       }
     }
 
@@ -746,21 +750,22 @@ bool AArch64FastISel::computeAddress(const Value *Obj, Address &Addr, Type *Ty)
     Addr.setExtendType(AArch64_AM::LSL);
 
     const Value *Src = LHS;
-    if (const auto *I = dyn_cast<Instruction>(Src))
-      if (FuncInfo.MBBMap[I->getParent()] == FuncInfo.MBB)
-        Src = I;
-
-
-    // Fold the zext or sext when it won't become a noop.
-    if (const auto *ZE = dyn_cast<ZExtInst>(Src)) {
-      if (!isIntExtFree(ZE) && ZE->getOperand(0)->getType()->isIntegerTy(32)) {
-        Addr.setExtendType(AArch64_AM::UXTW);
-        Src = ZE->getOperand(0);
-      }
-    } else if (const auto *SE = dyn_cast<SExtInst>(Src)) {
-      if (!isIntExtFree(SE) && SE->getOperand(0)->getType()->isIntegerTy(32)) {
-        Addr.setExtendType(AArch64_AM::SXTW);
-        Src = SE->getOperand(0);
+    if (const auto *I = dyn_cast<Instruction>(Src)) {
+      if (FuncInfo.MBBMap[I->getParent()] == FuncInfo.MBB) {
+        // Fold the zext or sext when it won't become a noop.
+        if (const auto *ZE = dyn_cast<ZExtInst>(I)) {
+          if (!isIntExtFree(ZE) &&
+              ZE->getOperand(0)->getType()->isIntegerTy(32)) {
+            Addr.setExtendType(AArch64_AM::UXTW);
+            Src = ZE->getOperand(0);
+          }
+        } else if (const auto *SE = dyn_cast<SExtInst>(I)) {
+          if (!isIntExtFree(SE) &&
+              SE->getOperand(0)->getType()->isIntegerTy(32)) {
+            Addr.setExtendType(AArch64_AM::SXTW);
+            Src = SE->getOperand(0);
+          }
+        }
       }
     }
 
@@ -875,13 +880,13 @@ bool AArch64FastISel::computeCallAddress(const Value *V, Address &Addr) {
   case Instruction::IntToPtr:
     // Look past no-op inttoptrs if its operand is in the same BB.
     if (InMBB &&
-        TLI.getValueType(U->getOperand(0)->getType()) == TLI.getPointerTy())
+        TLI.getValueType(DL, U->getOperand(0)->getType()) ==
+            TLI.getPointerTy(DL))
       return computeCallAddress(U->getOperand(0), Addr);
     break;
   case Instruction::PtrToInt:
     // Look past no-op ptrtoints if its operand is in the same BB.
-    if (InMBB &&
-        TLI.getValueType(U->getType()) == TLI.getPointerTy())
+    if (InMBB && TLI.getValueType(DL, U->getType()) == TLI.getPointerTy(DL))
       return computeCallAddress(U->getOperand(0), Addr);
     break;
   }
@@ -902,7 +907,7 @@ bool AArch64FastISel::computeCallAddress(const Value *V, Address &Addr) {
 
 
 bool AArch64FastISel::isTypeLegal(Type *Ty, MVT &VT) {
-  EVT evt = TLI.getValueType(Ty, true);
+  EVT evt = TLI.getValueType(DL, Ty, true);
 
   // Only handle simple types.
   if (evt == MVT::Other || !evt.isSimple())
@@ -1386,7 +1391,7 @@ unsigned AArch64FastISel::emitAddSub_rx(bool UseAdd, MVT RetVT, unsigned LHSReg,
 
 bool AArch64FastISel::emitCmp(const Value *LHS, const Value *RHS, bool IsZExt) {
   Type *Ty = LHS->getType();
-  EVT EVT = TLI.getValueType(Ty, true);
+  EVT EVT = TLI.getValueType(DL, Ty, true);
   if (!EVT.isSimple())
     return false;
   MVT VT = EVT.getSimpleVT();
@@ -1675,6 +1680,9 @@ unsigned AArch64FastISel::emitAnd_ri(MVT RetVT, unsigned LHSReg, bool LHSIsKill,
 
 unsigned AArch64FastISel::emitLoad(MVT VT, MVT RetVT, Address Addr,
                                    bool WantZExt, MachineMemOperand *MMO) {
+  if (!TLI.allowsMisalignedMemoryAccesses(VT))
+    return 0;
+
   // Simplify this down to something we can handle.
   if (!simplifyAddress(Addr, VT))
     return 0;
@@ -1959,6 +1967,9 @@ bool AArch64FastISel::selectLoad(const Instruction *I) {
 
 bool AArch64FastISel::emitStore(MVT VT, unsigned SrcReg, Address Addr,
                                 MachineMemOperand *MMO) {
+  if (!TLI.allowsMisalignedMemoryAccesses(VT))
+    return false;
+
   // Simplify this down to something we can handle.
   if (!simplifyAddress(Addr, VT))
     return false;
@@ -2573,7 +2584,7 @@ bool AArch64FastISel::optimizeSelect(const SelectInst *SI) {
     Src1Reg = emitLogicalOp_ri(ISD::XOR, MVT::i32, Src1Reg, Src1IsKill, 1);
     Src1IsKill = true;
   }
-  unsigned ResultReg = fastEmitInst_rr(Opc, &AArch64::GPR32spRegClass, Src1Reg,
+  unsigned ResultReg = fastEmitInst_rr(Opc, &AArch64::GPR32RegClass, Src1Reg,
                                        Src1IsKill, Src2Reg, Src2IsKill);
   updateValueMap(SI, ResultReg);
   return true;
@@ -2679,8 +2690,11 @@ bool AArch64FastISel::selectSelect(const Instruction *I) {
       return false;
     bool CondIsKill = hasTrivialKill(Cond);
 
+    const MCInstrDesc &II = TII.get(AArch64::ANDSWri);
+    CondReg = constrainOperandRegClass(II, CondReg, 1);
+
     // Emit a TST instruction (ANDS wzr, reg, #imm).
-    BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, TII.get(AArch64::ANDSWri),
+    BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, II,
             AArch64::WZR)
         .addReg(CondReg, getKillRegState(CondIsKill))
         .addImm(AArch64_AM::encodeLogicalImmediate(1, 32));
@@ -2748,7 +2762,7 @@ bool AArch64FastISel::selectFPToInt(const Instruction *I, bool Signed) {
   if (SrcReg == 0)
     return false;
 
-  EVT SrcVT = TLI.getValueType(I->getOperand(0)->getType(), true);
+  EVT SrcVT = TLI.getValueType(DL, I->getOperand(0)->getType(), true);
   if (SrcVT == MVT::f128)
     return false;
 
@@ -2784,7 +2798,7 @@ bool AArch64FastISel::selectIntToFP(const Instruction *I, bool Signed) {
     return false;
   bool SrcIsKill = hasTrivialKill(I->getOperand(0));
 
-  EVT SrcVT = TLI.getValueType(I->getOperand(0)->getType(), true);
+  EVT SrcVT = TLI.getValueType(DL, I->getOperand(0)->getType(), true);
 
   // Handle sign-extension.
   if (SrcVT == MVT::i16 || SrcVT == MVT::i8 || SrcVT == MVT::i1) {
@@ -2843,7 +2857,7 @@ bool AArch64FastISel::fastLowerArguments() {
     if (ArgTy->isStructTy() || ArgTy->isArrayTy())
       return false;
 
-    EVT ArgVT = TLI.getValueType(ArgTy);
+    EVT ArgVT = TLI.getValueType(DL, ArgTy);
     if (!ArgVT.isSimple())
       return false;
 
@@ -2885,7 +2899,7 @@ bool AArch64FastISel::fastLowerArguments() {
   unsigned GPRIdx = 0;
   unsigned FPRIdx = 0;
   for (auto const &Arg : F->args()) {
-    MVT VT = TLI.getSimpleValueType(Arg.getType());
+    MVT VT = TLI.getSimpleValueType(DL, Arg.getType());
     unsigned SrcReg;
     const TargetRegisterClass *RC;
     if (VT >= MVT::i1 && VT <= MVT::i32) {
@@ -3058,9 +3072,9 @@ bool AArch64FastISel::fastLowerCall(CallLoweringInfo &CLI) {
   bool IsTailCall     = CLI.IsTailCall;
   bool IsVarArg       = CLI.IsVarArg;
   const Value *Callee = CLI.Callee;
-  const char *SymName = CLI.SymName;
+  MCSymbol *Symbol = CLI.Symbol;
 
-  if (!Callee && !SymName)
+  if (!Callee && !Symbol)
     return false;
 
   // Allow SelectionDAG isel to handle tail calls.
@@ -3122,8 +3136,8 @@ bool AArch64FastISel::fastLowerCall(CallLoweringInfo &CLI) {
   if (CM == CodeModel::Small) {
     const MCInstrDesc &II = TII.get(Addr.getReg() ? AArch64::BLR : AArch64::BL);
     MIB = BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, II);
-    if (SymName)
-      MIB.addExternalSymbol(SymName, 0);
+    if (Symbol)
+      MIB.addSym(Symbol, 0);
     else if (Addr.getGlobalValue())
       MIB.addGlobalAddress(Addr.getGlobalValue(), 0, 0);
     else if (Addr.getReg()) {
@@ -3133,18 +3147,18 @@ bool AArch64FastISel::fastLowerCall(CallLoweringInfo &CLI) {
       return false;
   } else {
     unsigned CallReg = 0;
-    if (SymName) {
+    if (Symbol) {
       unsigned ADRPReg = createResultReg(&AArch64::GPR64commonRegClass);
       BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, TII.get(AArch64::ADRP),
               ADRPReg)
-        .addExternalSymbol(SymName, AArch64II::MO_GOT | AArch64II::MO_PAGE);
+          .addSym(Symbol, AArch64II::MO_GOT | AArch64II::MO_PAGE);
 
       CallReg = createResultReg(&AArch64::GPR64RegClass);
-      BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, TII.get(AArch64::LDRXui),
-              CallReg)
-        .addReg(ADRPReg)
-        .addExternalSymbol(SymName, AArch64II::MO_GOT | AArch64II::MO_PAGEOFF |
-                           AArch64II::MO_NC);
+      BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc,
+              TII.get(AArch64::LDRXui), CallReg)
+          .addReg(ADRPReg)
+          .addSym(Symbol,
+                  AArch64II::MO_GOT | AArch64II::MO_PAGEOFF | AArch64II::MO_NC);
     } else if (Addr.getGlobalValue())
       CallReg = materializeGV(Addr.getGlobalValue());
     else if (Addr.getReg())
@@ -3263,7 +3277,7 @@ bool AArch64FastISel::foldXALUIntrinsic(AArch64CC::CondCode &CC,
     std::swap(LHS, RHS);
 
   // Simplify multiplies.
-  unsigned IID = II->getIntrinsicID();
+  Intrinsic::ID IID = II->getIntrinsicID();
   switch (IID) {
   default:
     break;
@@ -3448,7 +3462,8 @@ bool AArch64FastISel::fastLowerIntrinsicCall(const IntrinsicInst *II) {
     }
 
     CallLoweringInfo CLI;
-    CLI.setCallee(TLI.getLibcallCallingConv(LC), II->getType(),
+    MCContext &Ctx = MF->getContext();
+    CLI.setCallee(DL, Ctx, TLI.getLibcallCallingConv(LC), II->getType(),
                   TLI.getLibcallName(LC), std::move(Args));
     if (!lowerCallTo(CLI))
       return false;
@@ -3531,7 +3546,7 @@ bool AArch64FastISel::fastLowerIntrinsicCall(const IntrinsicInst *II) {
       std::swap(LHS, RHS);
 
     // Simplify multiplies.
-    unsigned IID = II->getIntrinsicID();
+    Intrinsic::ID IID = II->getIntrinsicID();
     switch (IID) {
     default:
       break;
@@ -3595,7 +3610,10 @@ bool AArch64FastISel::fastLowerIntrinsicCall(const IntrinsicInst *II) {
                     AArch64_AM::ASR, 31, /*WantResult=*/false);
       } else {
         assert(VT == MVT::i64 && "Unexpected value type.");
-        MulReg = emitMul_rr(VT, LHSReg, LHSIsKill, RHSReg, RHSIsKill);
+        // LHSReg and RHSReg cannot be killed by this Mul, since they are
+        // reused in the next instruction.
+        MulReg = emitMul_rr(VT, LHSReg, /*IsKill=*/false, RHSReg,
+                            /*IsKill=*/false);
         unsigned SMULHReg = fastEmit_rr(VT, VT, ISD::MULHS, LHSReg, LHSIsKill,
                                         RHSReg, RHSIsKill);
         emitSubs_rs(VT, SMULHReg, /*IsKill=*/true, MulReg, /*IsKill=*/false,
@@ -3624,7 +3642,10 @@ bool AArch64FastISel::fastLowerIntrinsicCall(const IntrinsicInst *II) {
                                             AArch64::sub_32);
       } else {
         assert(VT == MVT::i64 && "Unexpected value type.");
-        MulReg = emitMul_rr(VT, LHSReg, LHSIsKill, RHSReg, RHSIsKill);
+        // LHSReg and RHSReg cannot be killed by this Mul, since they are
+        // reused in the next instruction.
+        MulReg = emitMul_rr(VT, LHSReg, /*IsKill=*/false, RHSReg,
+                            /*IsKill=*/false);
         unsigned UMULHReg = fastEmit_rr(VT, VT, ISD::MULHU, LHSReg, LHSIsKill,
                                         RHSReg, RHSIsKill);
         emitSubs_rr(VT, AArch64::XZR, /*IsKill=*/true, UMULHReg,
@@ -3669,7 +3690,7 @@ bool AArch64FastISel::selectRet(const Instruction *I) {
   if (Ret->getNumOperands() > 0) {
     CallingConv::ID CC = F.getCallingConv();
     SmallVector<ISD::OutputArg, 4> Outs;
-    GetReturnInfo(F.getReturnType(), F.getAttributes(), Outs, TLI);
+    GetReturnInfo(F.getReturnType(), F.getAttributes(), Outs, TLI, DL);
 
     // Analyze operands of the call, assigning locations to each operand.
     SmallVector<CCValAssign, 16> ValLocs;
@@ -3704,7 +3725,7 @@ bool AArch64FastISel::selectRet(const Instruction *I) {
     if (!MRI.getRegClass(SrcReg)->contains(DestReg))
       return false;
 
-    EVT RVEVT = TLI.getValueType(RV->getType());
+    EVT RVEVT = TLI.getValueType(DL, RV->getType());
     if (!RVEVT.isSimple())
       return false;
 
@@ -3752,8 +3773,8 @@ bool AArch64FastISel::selectTrunc(const Instruction *I) {
   Value *Op = I->getOperand(0);
   Type *SrcTy = Op->getType();
 
-  EVT SrcEVT = TLI.getValueType(SrcTy, true);
-  EVT DestEVT = TLI.getValueType(DestTy, true);
+  EVT SrcEVT = TLI.getValueType(DL, SrcTy, true);
+  EVT DestEVT = TLI.getValueType(DL, DestTy, true);
   if (!SrcEVT.isSimple())
     return false;
   if (!DestEVT.isSimple())
@@ -3774,41 +3795,34 @@ bool AArch64FastISel::selectTrunc(const Instruction *I) {
     return false;
   bool SrcIsKill = hasTrivialKill(Op);
 
-  // If we're truncating from i64 to a smaller non-legal type then generate an
-  // AND. Otherwise, we know the high bits are undefined and a truncate only
-  // generate a COPY. We cannot mark the source register also as result
-  // register, because this can incorrectly transfer the kill flag onto the
-  // source register.
-  unsigned ResultReg;
+  // If we're truncating from i64/i32 to a smaller non-legal type then generate
+  // an AND.
+  uint64_t Mask = 0;
+  switch (DestVT.SimpleTy) {
+  default:
+    // Trunc i64 to i32 is handled by the target-independent fast-isel.
+    return false;
+  case MVT::i1:
+    Mask = 0x1;
+    break;
+  case MVT::i8:
+    Mask = 0xff;
+    break;
+  case MVT::i16:
+    Mask = 0xffff;
+    break;
+  }
   if (SrcVT == MVT::i64) {
-    uint64_t Mask = 0;
-    switch (DestVT.SimpleTy) {
-    default:
-      // Trunc i64 to i32 is handled by the target-independent fast-isel.
-      return false;
-    case MVT::i1:
-      Mask = 0x1;
-      break;
-    case MVT::i8:
-      Mask = 0xff;
-      break;
-    case MVT::i16:
-      Mask = 0xffff;
-      break;
-    }
     // Issue an extract_subreg to get the lower 32-bits.
-    unsigned Reg32 = fastEmitInst_extractsubreg(MVT::i32, SrcReg, SrcIsKill,
-                                                AArch64::sub_32);
-    // Create the AND instruction which performs the actual truncation.
-    ResultReg = emitAnd_ri(MVT::i32, Reg32, /*IsKill=*/true, Mask);
-    assert(ResultReg && "Unexpected AND instruction emission failure.");
-  } else {
-    ResultReg = createResultReg(&AArch64::GPR32RegClass);
-    BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc,
-            TII.get(TargetOpcode::COPY), ResultReg)
-        .addReg(SrcReg, getKillRegState(SrcIsKill));
+    SrcReg = fastEmitInst_extractsubreg(MVT::i32, SrcReg, SrcIsKill,
+                                        AArch64::sub_32);
+    SrcIsKill = true;
   }
 
+  // Create the AND instruction which performs the actual truncation.
+  unsigned ResultReg = emitAnd_ri(MVT::i32, SrcReg, SrcIsKill, Mask);
+  assert(ResultReg && "Unexpected AND instruction emission failure.");
+
   updateValueMap(I, ResultReg);
   return true;
 }
@@ -4439,7 +4453,7 @@ bool AArch64FastISel::selectIntExt(const Instruction *I) {
 }
 
 bool AArch64FastISel::selectRem(const Instruction *I, unsigned ISDOpcode) {
-  EVT DestEVT = TLI.getValueType(I->getType(), true);
+  EVT DestEVT = TLI.getValueType(DL, I->getType(), true);
   if (!DestEVT.isSimple())
     return false;
 
@@ -4716,7 +4730,8 @@ bool AArch64FastISel::selectFRem(const Instruction *I) {
   }
 
   CallLoweringInfo CLI;
-  CLI.setCallee(TLI.getLibcallCallingConv(LC), I->getType(),
+  MCContext &Ctx = MF->getContext();
+  CLI.setCallee(DL, Ctx, TLI.getLibcallCallingConv(LC), I->getType(),
                 TLI.getLibcallName(LC), std::move(Args));
   if (!lowerCallTo(CLI))
     return false;
@@ -4804,7 +4819,7 @@ std::pair<unsigned, bool> AArch64FastISel::getRegForGEPIndex(const Value *Idx) {
   bool IdxNIsKill = hasTrivialKill(Idx);
 
   // If the index is smaller or larger than intptr_t, truncate or extend it.
-  MVT PtrVT = TLI.getPointerTy();
+  MVT PtrVT = TLI.getPointerTy(DL);
   EVT IdxVT = EVT::getEVT(Idx->getType(), /*HandleUnknown=*/false);
   if (IdxVT.bitsLT(PtrVT)) {
     IdxN = emitIntExt(IdxVT.getSimpleVT(), IdxN, PtrVT, /*IsZExt=*/false);
@@ -4828,7 +4843,7 @@ bool AArch64FastISel::selectGetElementPtr(const Instruction *I) {
   // into a single N = N + TotalOffset.
   uint64_t TotalOffs = 0;
   Type *Ty = I->getOperand(0)->getType();
-  MVT VT = TLI.getPointerTy();
+  MVT VT = TLI.getPointerTy(DL);
   for (auto OI = std::next(I->op_begin()), E = I->op_end(); OI != E; ++OI) {
     const Value *Idx = *OI;
     if (auto *StTy = dyn_cast<StructType>(Ty)) {
diff --git a/lib/Target/AArch64/AArch64FrameLowering.cpp b/lib/Target/AArch64/AArch64FrameLowering.cpp
index bd2af161210..f43fb23b937 100644
--- a/lib/Target/AArch64/AArch64FrameLowering.cpp
+++ b/lib/Target/AArch64/AArch64FrameLowering.cpp
@@ -161,7 +161,7 @@ void AArch64FrameLowering::eliminateCallFramePseudoInstr(
   const AArch64InstrInfo *TII =
       static_cast<const AArch64InstrInfo *>(MF.getSubtarget().getInstrInfo());
   DebugLoc DL = I->getDebugLoc();
-  int Opc = I->getOpcode();
+  unsigned Opc = I->getOpcode();
   bool IsDestroy = Opc == TII->getCallFrameDestroyOpcode();
   uint64_t CalleePopAmount = IsDestroy ? I->getOperand(1).getImm() : 0;
 
@@ -216,11 +216,11 @@ void AArch64FrameLowering::emitCalleeSavedFrameMoves(
   if (CSI.empty())
     return;
 
-  const DataLayout *TD = MF.getTarget().getDataLayout();
+  const DataLayout &TD = MF.getDataLayout();
   bool HasFP = hasFP(MF);
 
   // Calculate amount of bytes used for return address storing.
-  int stackGrowth = -TD->getPointerSize(0);
+  int stackGrowth = -TD.getPointerSize(0);
 
   // Calculate offsets.
   int64_t saveAreaOffset = (HasFP ? 2 : 1) * stackGrowth;
@@ -250,8 +250,33 @@ void AArch64FrameLowering::emitCalleeSavedFrameMoves(
   }
 }
 
-void AArch64FrameLowering::emitPrologue(MachineFunction &MF) const {
-  MachineBasicBlock &MBB = MF.front(); // Prologue goes in entry BB.
+/// Get FPOffset by analyzing the first instruction.
+static int getFPOffsetInPrologue(MachineInstr *MBBI) {
+  // First instruction must a) allocate the stack  and b) have an immediate
+  // that is a multiple of -2.
+  assert(((MBBI->getOpcode() == AArch64::STPXpre ||
+           MBBI->getOpcode() == AArch64::STPDpre) &&
+          MBBI->getOperand(3).getReg() == AArch64::SP &&
+          MBBI->getOperand(4).getImm() < 0 &&
+          (MBBI->getOperand(4).getImm() & 1) == 0));
+
+  // Frame pointer is fp = sp - 16. Since the  STPXpre subtracts the space
+  // required for the callee saved register area we get the frame pointer
+  // by addding that offset - 16 = -getImm()*8 - 2*8 = -(getImm() + 2) * 8.
+  int FPOffset = -(MBBI->getOperand(4).getImm() + 2) * 8;
+  assert(FPOffset >= 0 && "Bad Framepointer Offset");
+  return FPOffset;
+}
+
+static bool isCSSave(MachineInstr *MBBI) {
+  return MBBI->getOpcode() == AArch64::STPXi ||
+         MBBI->getOpcode() == AArch64::STPDi ||
+         MBBI->getOpcode() == AArch64::STPXpre ||
+         MBBI->getOpcode() == AArch64::STPDpre;
+}
+
+void AArch64FrameLowering::emitPrologue(MachineFunction &MF,
+                                        MachineBasicBlock &MBB) const {
   MachineBasicBlock::iterator MBBI = MBB.begin();
   const MachineFrameInfo *MFI = MF.getFrameInfo();
   const Function *Fn = MF.getFunction();
@@ -277,7 +302,7 @@ void AArch64FrameLowering::emitPrologue(MachineFunction &MF) const {
     AFI->setLocalStackSize(NumBytes);
 
     // Label used to tie together the PROLOG_LABEL and the MachineMoves.
-    MCSymbol *FrameLabel = MMI.getContext().CreateTempSymbol();
+    MCSymbol *FrameLabel = MMI.getContext().createTempSymbol();
 
     // REDZONE: If the stack size is less than 128 bytes, we don't need
     // to actually allocate.
@@ -300,27 +325,11 @@ void AArch64FrameLowering::emitPrologue(MachineFunction &MF) const {
 
   // Only set up FP if we actually need to.
   int FPOffset = 0;
-  if (HasFP) {
-    // First instruction must a) allocate the stack  and b) have an immediate
-    // that is a multiple of -2.
-    assert((MBBI->getOpcode() == AArch64::STPXpre ||
-            MBBI->getOpcode() == AArch64::STPDpre) &&
-           MBBI->getOperand(3).getReg() == AArch64::SP &&
-           MBBI->getOperand(4).getImm() < 0 &&
-           (MBBI->getOperand(4).getImm() & 1) == 0);
-
-    // Frame pointer is fp = sp - 16. Since the  STPXpre subtracts the space
-    // required for the callee saved register area we get the frame pointer
-    // by addding that offset - 16 = -getImm()*8 - 2*8 = -(getImm() + 2) * 8.
-    FPOffset = -(MBBI->getOperand(4).getImm() + 2) * 8;
-    assert(FPOffset >= 0 && "Bad Framepointer Offset");
-  }
+  if (HasFP)
+    FPOffset = getFPOffsetInPrologue(MBBI);
 
   // Move past the saves of the callee-saved registers.
-  while (MBBI->getOpcode() == AArch64::STPXi ||
-         MBBI->getOpcode() == AArch64::STPDi ||
-         MBBI->getOpcode() == AArch64::STPXpre ||
-         MBBI->getOpcode() == AArch64::STPDpre) {
+  while (isCSSave(MBBI)) {
     ++MBBI;
     NumBytes -= 16;
   }
@@ -340,12 +349,10 @@ void AArch64FrameLowering::emitPrologue(MachineFunction &MF) const {
   // Allocate space for the rest of the frame.
 
   const unsigned Alignment = MFI->getMaxAlignment();
-  const bool NeedsRealignment = (Alignment > 16);
+  const bool NeedsRealignment = RegInfo->needsStackRealignment(MF);
   unsigned scratchSPReg = AArch64::SP;
-  if (NeedsRealignment) {
-    // Use the first callee-saved register as a scratch register
-    assert(MF.getRegInfo().isPhysRegUsed(AArch64::X9) &&
-           "No scratch register to align SP!");
+  if (NumBytes && NeedsRealignment) {
+    // Use the first callee-saved register as a scratch register.
     scratchSPReg = AArch64::X9;
   }
 
@@ -357,9 +364,6 @@ void AArch64FrameLowering::emitPrologue(MachineFunction &MF) const {
     emitFrameOffset(MBB, MBBI, DL, scratchSPReg, AArch64::SP, -NumBytes, TII,
                     MachineInstr::FrameSetup);
 
-  assert(!(NeedsRealignment && NumBytes==0) &&
-         "NumBytes should never be 0 when realignment is needed");
-
   if (NumBytes && NeedsRealignment) {
     const unsigned NrBitsToZero = countTrailingZeros(Alignment);
     assert(NrBitsToZero > 1);
@@ -395,8 +399,8 @@ void AArch64FrameLowering::emitPrologue(MachineFunction &MF) const {
   }
 
   if (needsFrameMoves) {
-    const DataLayout *TD = MF.getTarget().getDataLayout();
-    const int StackGrowth = -TD->getPointerSize(0);
+    const DataLayout &TD = MF.getDataLayout();
+    const int StackGrowth = -TD.getPointerSize(0);
     unsigned FramePtr = RegInfo->getFrameRegister(MF);
     // An example of the prologue:
     //
@@ -530,15 +534,19 @@ static bool isCSRestore(MachineInstr *MI, const MCPhysReg *CSRegs) {
 void AArch64FrameLowering::emitEpilogue(MachineFunction &MF,
                                         MachineBasicBlock &MBB) const {
   MachineBasicBlock::iterator MBBI = MBB.getLastNonDebugInstr();
-  assert(MBBI->isReturn() && "Can only insert epilog into returning blocks");
   MachineFrameInfo *MFI = MF.getFrameInfo();
   const AArch64InstrInfo *TII =
       static_cast<const AArch64InstrInfo *>(MF.getSubtarget().getInstrInfo());
   const AArch64RegisterInfo *RegInfo = static_cast<const AArch64RegisterInfo *>(
       MF.getSubtarget().getRegisterInfo());
-  DebugLoc DL = MBBI->getDebugLoc();
-  unsigned RetOpcode = MBBI->getOpcode();
-
+  DebugLoc DL;
+  bool IsTailCallReturn = false;
+  if (MBB.end() != MBBI) {
+    DL = MBBI->getDebugLoc();
+    unsigned RetOpcode = MBBI->getOpcode();
+    IsTailCallReturn = RetOpcode == AArch64::TCRETURNdi ||
+      RetOpcode == AArch64::TCRETURNri;
+  }
   int NumBytes = MFI->getStackSize();
   const AArch64FunctionInfo *AFI = MF.getInfo<AArch64FunctionInfo>();
 
@@ -550,7 +558,7 @@ void AArch64FrameLowering::emitEpilogue(MachineFunction &MF,
   // Initial and residual are named for consistency with the prologue. Note that
   // in the epilogue, the residual adjustment is executed first.
   uint64_t ArgumentPopSize = 0;
-  if (RetOpcode == AArch64::TCRETURNdi || RetOpcode == AArch64::TCRETURNri) {
+  if (IsTailCallReturn) {
     MachineOperand &StackAdjust = MBBI->getOperand(1);
 
     // For a tail-call in a callee-pops-arguments environment, some or all of
@@ -595,7 +603,7 @@ void AArch64FrameLowering::emitEpilogue(MachineFunction &MF,
 
   unsigned NumRestores = 0;
   // Move past the restores of the callee-saved registers.
-  MachineBasicBlock::iterator LastPopI = MBBI;
+  MachineBasicBlock::iterator LastPopI = MBB.getFirstTerminator();
   const MCPhysReg *CSRegs = RegInfo->getCalleeSavedRegs(&MF);
   if (LastPopI != MBB.begin()) {
     do {
@@ -868,35 +876,41 @@ bool AArch64FrameLowering::restoreCalleeSavedRegisters(
   return true;
 }
 
-void AArch64FrameLowering::processFunctionBeforeCalleeSavedScan(
-    MachineFunction &MF, RegScavenger *RS) const {
+void AArch64FrameLowering::determineCalleeSaves(MachineFunction &MF,
+                                                BitVector &SavedRegs,
+                                                RegScavenger *RS) const {
+  // All calls are tail calls in GHC calling conv, and functions have no
+  // prologue/epilogue.
+  if (MF.getFunction()->getCallingConv() == CallingConv::GHC)
+    return;
+
+  TargetFrameLowering::determineCalleeSaves(MF, SavedRegs, RS);
   const AArch64RegisterInfo *RegInfo = static_cast<const AArch64RegisterInfo *>(
       MF.getSubtarget().getRegisterInfo());
   AArch64FunctionInfo *AFI = MF.getInfo<AArch64FunctionInfo>();
-  MachineRegisterInfo *MRI = &MF.getRegInfo();
   SmallVector<unsigned, 4> UnspilledCSGPRs;
   SmallVector<unsigned, 4> UnspilledCSFPRs;
 
   // The frame record needs to be created by saving the appropriate registers
   if (hasFP(MF)) {
-    MRI->setPhysRegUsed(AArch64::FP);
-    MRI->setPhysRegUsed(AArch64::LR);
+    SavedRegs.set(AArch64::FP);
+    SavedRegs.set(AArch64::LR);
   }
 
   // Spill the BasePtr if it's used. Do this first thing so that the
   // getCalleeSavedRegs() below will get the right answer.
   if (RegInfo->hasBasePointer(MF))
-    MRI->setPhysRegUsed(RegInfo->getBaseRegister());
+    SavedRegs.set(RegInfo->getBaseRegister());
 
   if (RegInfo->needsStackRealignment(MF) && !RegInfo->hasBasePointer(MF))
-    MRI->setPhysRegUsed(AArch64::X9);
+    SavedRegs.set(AArch64::X9);
 
   // If any callee-saved registers are used, the frame cannot be eliminated.
   unsigned NumGPRSpilled = 0;
   unsigned NumFPRSpilled = 0;
   bool ExtraCSSpill = false;
   bool CanEliminateFrame = true;
-  DEBUG(dbgs() << "*** processFunctionBeforeCalleeSavedScan\nUsed CSRs:");
+  DEBUG(dbgs() << "*** determineCalleeSaves\nUsed CSRs:");
   const MCPhysReg *CSRegs = RegInfo->getCalleeSavedRegs(&MF);
 
   // Check pairs of consecutive callee-saved registers.
@@ -911,8 +925,8 @@ void AArch64FrameLowering::processFunctionBeforeCalleeSavedScan(
                 AArch64::FPR64RegClass.contains(EvenReg)) &&
            "Register class mismatch!");
 
-    const bool OddRegUsed = MRI->isPhysRegUsed(OddReg);
-    const bool EvenRegUsed = MRI->isPhysRegUsed(EvenReg);
+    const bool OddRegUsed = SavedRegs.test(OddReg);
+    const bool EvenRegUsed = SavedRegs.test(EvenReg);
 
     // Early exit if none of the registers in the register pair is actually
     // used.
@@ -933,7 +947,7 @@ void AArch64FrameLowering::processFunctionBeforeCalleeSavedScan(
     if (OddRegUsed ^ EvenRegUsed) {
       // Find out which register is the additional spill.
       Reg = OddRegUsed ? EvenReg : OddReg;
-      MRI->setPhysRegUsed(Reg);
+      SavedRegs.set(Reg);
     }
 
     DEBUG(dbgs() << ' ' << PrintReg(OddReg, RegInfo));
@@ -988,7 +1002,7 @@ void AArch64FrameLowering::processFunctionBeforeCalleeSavedScan(
       UnspilledCSGPRs.pop_back();
       DEBUG(dbgs() << "Spilling " << PrintReg(Reg, RegInfo)
                    << " to get a scratch register.\n");
-      MRI->setPhysRegUsed(Reg);
+      SavedRegs.set(Reg);
       ExtraCSSpill = true;
       ++Count;
     }
diff --git a/lib/Target/AArch64/AArch64FrameLowering.h b/lib/Target/AArch64/AArch64FrameLowering.h
index 1439bf32085..731f031ff85 100644
--- a/lib/Target/AArch64/AArch64FrameLowering.h
+++ b/lib/Target/AArch64/AArch64FrameLowering.h
@@ -34,7 +34,7 @@ class AArch64FrameLowering : public TargetFrameLowering {
 
   /// emitProlog/emitEpilog - These methods insert prolog and epilog code into
   /// the function.
-  void emitPrologue(MachineFunction &MF) const override;
+  void emitPrologue(MachineFunction &MF, MachineBasicBlock &MBB) const override;
   void emitEpilogue(MachineFunction &MF, MachineBasicBlock &MBB) const override;
 
   int getFrameIndexOffset(const MachineFunction &MF, int FI) const override;
@@ -59,8 +59,8 @@ class AArch64FrameLowering : public TargetFrameLowering {
   bool hasFP(const MachineFunction &MF) const override;
   bool hasReservedCallFrame(const MachineFunction &MF) const override;
 
-  void processFunctionBeforeCalleeSavedScan(MachineFunction &MF,
-                                            RegScavenger *RS) const override;
+  void determineCalleeSaves(MachineFunction &MF, BitVector &SavedRegs,
+                            RegScavenger *RS) const override;
 };
 
 } // End llvm namespace
diff --git a/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp b/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp
index f75700d6d33..0b848fc7b39 100644
--- a/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp
+++ b/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp
@@ -155,6 +155,10 @@ class AArch64DAGToDAGISel : public SelectionDAGISel {
   SDNode *SelectBitfieldInsertOp(SDNode *N);
 
   SDNode *SelectLIBM(SDNode *N);
+  SDNode *SelectFPConvertWithRound(SDNode *N);
+
+  SDNode *SelectReadRegister(SDNode *N);
+  SDNode *SelectWriteRegister(SDNode *N);
 
 // Include the pieces autogenerated from the target description.
 #include "AArch64GenDAGISel.inc"
@@ -182,6 +186,9 @@ class AArch64DAGToDAGISel : public SelectionDAGISel {
   }
 
   bool SelectCVTFixedPosOperand(SDValue N, SDValue &FixedPos, unsigned Width);
+
+  SDNode *GenerateInexactFlagIfNeeded(const SDValue &In, unsigned InTyVariant,
+                                      SDLoc DL);
 };
 } // end anonymous namespace
 
@@ -252,8 +259,9 @@ bool AArch64DAGToDAGISel::SelectArithImmed(SDValue N, SDValue &Val,
     return false;
 
   unsigned ShVal = AArch64_AM::getShifterImm(AArch64_AM::LSL, ShiftAmt);
-  Val = CurDAG->getTargetConstant(Immed, MVT::i32);
-  Shift = CurDAG->getTargetConstant(ShVal, MVT::i32);
+  SDLoc dl(N);
+  Val = CurDAG->getTargetConstant(Immed, dl, MVT::i32);
+  Shift = CurDAG->getTargetConstant(ShVal, dl, MVT::i32);
   return true;
 }
 
@@ -286,7 +294,8 @@ bool AArch64DAGToDAGISel::SelectNegArithImmed(SDValue N, SDValue &Val,
     return false;
 
   Immed &= 0xFFFFFFULL;
-  return SelectArithImmed(CurDAG->getConstant(Immed, MVT::i32), Val, Shift);
+  return SelectArithImmed(CurDAG->getConstant(Immed, SDLoc(N), MVT::i32), Val,
+                          Shift);
 }
 
 /// getShiftTypeForNode - Translate a shift node to the corresponding
@@ -334,7 +343,7 @@ bool AArch64DAGToDAGISel::SelectShiftedRegister(SDValue N, bool AllowROR,
     unsigned ShVal = AArch64_AM::getShifterImm(ShType, Val);
 
     Reg = N.getOperand(0);
-    Shift = CurDAG->getTargetConstant(ShVal, MVT::i32);
+    Shift = CurDAG->getTargetConstant(ShVal, SDLoc(N), MVT::i32);
     return isWorthFolding(N);
   }
 
@@ -435,6 +444,7 @@ static bool checkV64LaneV128(SDValue Op0, SDValue Op1, SDValue &StdOp,
 /// is a lane in the upper half of a 128-bit vector.  Recognize and select this
 /// so that we don't emit unnecessary lane extracts.
 SDNode *AArch64DAGToDAGISel::SelectMLAV64LaneV128(SDNode *N) {
+  SDLoc dl(N);
   SDValue Op0 = N->getOperand(0);
   SDValue Op1 = N->getOperand(1);
   SDValue MLAOp1;   // Will hold ordinary multiplicand for MLA.
@@ -451,7 +461,7 @@ SDNode *AArch64DAGToDAGISel::SelectMLAV64LaneV128(SDNode *N) {
       return nullptr;
   }
 
-  SDValue LaneIdxVal = CurDAG->getTargetConstant(LaneIdx, MVT::i64);
+  SDValue LaneIdxVal = CurDAG->getTargetConstant(LaneIdx, dl, MVT::i64);
 
   SDValue Ops[] = { Op0, MLAOp1, MLAOp2, LaneIdxVal };
 
@@ -474,10 +484,11 @@ SDNode *AArch64DAGToDAGISel::SelectMLAV64LaneV128(SDNode *N) {
     break;
   }
 
-  return CurDAG->getMachineNode(MLAOpc, SDLoc(N), N->getValueType(0), Ops);
+  return CurDAG->getMachineNode(MLAOpc, dl, N->getValueType(0), Ops);
 }
 
 SDNode *AArch64DAGToDAGISel::SelectMULLV64LaneV128(unsigned IntNo, SDNode *N) {
+  SDLoc dl(N);
   SDValue SMULLOp0;
   SDValue SMULLOp1;
   int LaneIdx;
@@ -486,7 +497,7 @@ SDNode *AArch64DAGToDAGISel::SelectMULLV64LaneV128(unsigned IntNo, SDNode *N) {
                         LaneIdx))
     return nullptr;
 
-  SDValue LaneIdxVal = CurDAG->getTargetConstant(LaneIdx, MVT::i64);
+  SDValue LaneIdxVal = CurDAG->getTargetConstant(LaneIdx, dl, MVT::i64);
 
   SDValue Ops[] = { SMULLOp0, SMULLOp1, LaneIdxVal };
 
@@ -517,7 +528,7 @@ SDNode *AArch64DAGToDAGISel::SelectMULLV64LaneV128(unsigned IntNo, SDNode *N) {
   } else
     llvm_unreachable("Unrecognized intrinsic.");
 
-  return CurDAG->getMachineNode(SMULLOpc, SDLoc(N), N->getValueType(0), Ops);
+  return CurDAG->getMachineNode(SMULLOpc, dl, N->getValueType(0), Ops);
 }
 
 /// Instructions that accept extend modifiers like UXTW expect the register
@@ -528,9 +539,10 @@ static SDValue narrowIfNeeded(SelectionDAG *CurDAG, SDValue N) {
   if (N.getValueType() == MVT::i32)
     return N;
 
-  SDValue SubReg = CurDAG->getTargetConstant(AArch64::sub_32, MVT::i32);
+  SDLoc dl(N);
+  SDValue SubReg = CurDAG->getTargetConstant(AArch64::sub_32, dl, MVT::i32);
   MachineSDNode *Node = CurDAG->getMachineNode(TargetOpcode::EXTRACT_SUBREG,
-                                               SDLoc(N), MVT::i32, N, SubReg);
+                                               dl, MVT::i32, N, SubReg);
   return SDValue(Node, 0);
 }
 
@@ -570,7 +582,8 @@ bool AArch64DAGToDAGISel::SelectArithExtendedRegister(SDValue N, SDValue &Reg,
   // (harmlessly) synthesize one by injected an EXTRACT_SUBREG here.
   assert(Ext != AArch64_AM::UXTX && Ext != AArch64_AM::SXTX);
   Reg = narrowIfNeeded(CurDAG, Reg);
-  Shift = CurDAG->getTargetConstant(getArithExtendImm(Ext, ShiftVal), MVT::i32);
+  Shift = CurDAG->getTargetConstant(getArithExtendImm(Ext, ShiftVal), SDLoc(N),
+                                    MVT::i32);
   return isWorthFolding(N);
 }
 
@@ -600,11 +613,13 @@ static bool isWorthFoldingADDlow(SDValue N) {
 /// reference, which determines the scale.
 bool AArch64DAGToDAGISel::SelectAddrModeIndexed(SDValue N, unsigned Size,
                                               SDValue &Base, SDValue &OffImm) {
+  SDLoc dl(N);
+  const DataLayout &DL = CurDAG->getDataLayout();
   const TargetLowering *TLI = getTargetLowering();
   if (N.getOpcode() == ISD::FrameIndex) {
     int FI = cast<FrameIndexSDNode>(N)->getIndex();
-    Base = CurDAG->getTargetFrameIndex(FI, TLI->getPointerTy());
-    OffImm = CurDAG->getTargetConstant(0, MVT::i64);
+    Base = CurDAG->getTargetFrameIndex(FI, TLI->getPointerTy(DL));
+    OffImm = CurDAG->getTargetConstant(0, dl, MVT::i64);
     return true;
   }
 
@@ -618,10 +633,9 @@ bool AArch64DAGToDAGISel::SelectAddrModeIndexed(SDValue N, unsigned Size,
 
     const GlobalValue *GV = GAN->getGlobal();
     unsigned Alignment = GV->getAlignment();
-    const DataLayout *DL = TLI->getDataLayout();
     Type *Ty = GV->getType()->getElementType();
     if (Alignment == 0 && Ty->isSized())
-      Alignment = DL->getABITypeAlignment(Ty);
+      Alignment = DL.getABITypeAlignment(Ty);
 
     if (Alignment >= Size)
       return true;
@@ -635,9 +649,9 @@ bool AArch64DAGToDAGISel::SelectAddrModeIndexed(SDValue N, unsigned Size,
         Base = N.getOperand(0);
         if (Base.getOpcode() == ISD::FrameIndex) {
           int FI = cast<FrameIndexSDNode>(Base)->getIndex();
-          Base = CurDAG->getTargetFrameIndex(FI, TLI->getPointerTy());
+          Base = CurDAG->getTargetFrameIndex(FI, TLI->getPointerTy(DL));
         }
-        OffImm = CurDAG->getTargetConstant(RHSC >> Scale, MVT::i64);
+        OffImm = CurDAG->getTargetConstant(RHSC >> Scale, dl, MVT::i64);
         return true;
       }
     }
@@ -653,7 +667,7 @@ bool AArch64DAGToDAGISel::SelectAddrModeIndexed(SDValue N, unsigned Size,
   //    add x0, Xbase, #offset
   //    ldr x0, [x0]
   Base = N;
-  OffImm = CurDAG->getTargetConstant(0, MVT::i64);
+  OffImm = CurDAG->getTargetConstant(0, dl, MVT::i64);
   return true;
 }
 
@@ -678,9 +692,10 @@ bool AArch64DAGToDAGISel::SelectAddrModeUnscaled(SDValue N, unsigned Size,
       if (Base.getOpcode() == ISD::FrameIndex) {
         int FI = cast<FrameIndexSDNode>(Base)->getIndex();
         const TargetLowering *TLI = getTargetLowering();
-        Base = CurDAG->getTargetFrameIndex(FI, TLI->getPointerTy());
+        Base = CurDAG->getTargetFrameIndex(
+            FI, TLI->getPointerTy(CurDAG->getDataLayout()));
       }
-      OffImm = CurDAG->getTargetConstant(RHSC, MVT::i64);
+      OffImm = CurDAG->getTargetConstant(RHSC, SDLoc(N), MVT::i64);
       return true;
     }
   }
@@ -688,12 +703,12 @@ bool AArch64DAGToDAGISel::SelectAddrModeUnscaled(SDValue N, unsigned Size,
 }
 
 static SDValue Widen(SelectionDAG *CurDAG, SDValue N) {
-  SDValue SubReg = CurDAG->getTargetConstant(AArch64::sub_32, MVT::i32);
+  SDLoc dl(N);
+  SDValue SubReg = CurDAG->getTargetConstant(AArch64::sub_32, dl, MVT::i32);
   SDValue ImpDef = SDValue(
-      CurDAG->getMachineNode(TargetOpcode::IMPLICIT_DEF, SDLoc(N), MVT::i64),
-      0);
+      CurDAG->getMachineNode(TargetOpcode::IMPLICIT_DEF, dl, MVT::i64), 0);
   MachineSDNode *Node = CurDAG->getMachineNode(
-      TargetOpcode::INSERT_SUBREG, SDLoc(N), MVT::i64, ImpDef, N, SubReg);
+      TargetOpcode::INSERT_SUBREG, dl, MVT::i64, ImpDef, N, SubReg);
   return SDValue(Node, 0);
 }
 
@@ -707,6 +722,7 @@ bool AArch64DAGToDAGISel::SelectExtendedSHL(SDValue N, unsigned Size,
   if (!CSD || (CSD->getZExtValue() & 0x7) != CSD->getZExtValue())
     return false;
 
+  SDLoc dl(N);
   if (WantExtend) {
     AArch64_AM::ShiftExtendType Ext =
         getExtendTypeForNode(N.getOperand(0), true);
@@ -714,10 +730,11 @@ bool AArch64DAGToDAGISel::SelectExtendedSHL(SDValue N, unsigned Size,
       return false;
 
     Offset = narrowIfNeeded(CurDAG, N.getOperand(0).getOperand(0));
-    SignExtend = CurDAG->getTargetConstant(Ext == AArch64_AM::SXTW, MVT::i32);
+    SignExtend = CurDAG->getTargetConstant(Ext == AArch64_AM::SXTW, dl,
+                                           MVT::i32);
   } else {
     Offset = N.getOperand(0);
-    SignExtend = CurDAG->getTargetConstant(0, MVT::i32);
+    SignExtend = CurDAG->getTargetConstant(0, dl, MVT::i32);
   }
 
   unsigned LegalShiftVal = Log2_32(Size);
@@ -740,6 +757,7 @@ bool AArch64DAGToDAGISel::SelectAddrModeWRO(SDValue N, unsigned Size,
     return false;
   SDValue LHS = N.getOperand(0);
   SDValue RHS = N.getOperand(1);
+  SDLoc dl(N);
 
   // We don't want to match immediate adds here, because they are better lowered
   // to the register-immediate addressing modes.
@@ -762,7 +780,7 @@ bool AArch64DAGToDAGISel::SelectAddrModeWRO(SDValue N, unsigned Size,
   if (IsExtendedRegisterWorthFolding && RHS.getOpcode() == ISD::SHL &&
       SelectExtendedSHL(RHS, Size, true, Offset, SignExtend)) {
     Base = LHS;
-    DoShift = CurDAG->getTargetConstant(true, MVT::i32);
+    DoShift = CurDAG->getTargetConstant(true, dl, MVT::i32);
     return true;
   }
 
@@ -770,12 +788,12 @@ bool AArch64DAGToDAGISel::SelectAddrModeWRO(SDValue N, unsigned Size,
   if (IsExtendedRegisterWorthFolding && LHS.getOpcode() == ISD::SHL &&
       SelectExtendedSHL(LHS, Size, true, Offset, SignExtend)) {
     Base = RHS;
-    DoShift = CurDAG->getTargetConstant(true, MVT::i32);
+    DoShift = CurDAG->getTargetConstant(true, dl, MVT::i32);
     return true;
   }
 
   // There was no shift, whatever else we find.
-  DoShift = CurDAG->getTargetConstant(false, MVT::i32);
+  DoShift = CurDAG->getTargetConstant(false, dl, MVT::i32);
 
   AArch64_AM::ShiftExtendType Ext = AArch64_AM::InvalidShiftExtend;
   // Try to match an unshifted extend on the LHS.
@@ -784,7 +802,8 @@ bool AArch64DAGToDAGISel::SelectAddrModeWRO(SDValue N, unsigned Size,
           AArch64_AM::InvalidShiftExtend) {
     Base = RHS;
     Offset = narrowIfNeeded(CurDAG, LHS.getOperand(0));
-    SignExtend = CurDAG->getTargetConstant(Ext == AArch64_AM::SXTW, MVT::i32);
+    SignExtend = CurDAG->getTargetConstant(Ext == AArch64_AM::SXTW, dl,
+                                           MVT::i32);
     if (isWorthFolding(LHS))
       return true;
   }
@@ -795,7 +814,8 @@ bool AArch64DAGToDAGISel::SelectAddrModeWRO(SDValue N, unsigned Size,
           AArch64_AM::InvalidShiftExtend) {
     Base = LHS;
     Offset = narrowIfNeeded(CurDAG, RHS.getOperand(0));
-    SignExtend = CurDAG->getTargetConstant(Ext == AArch64_AM::SXTW, MVT::i32);
+    SignExtend = CurDAG->getTargetConstant(Ext == AArch64_AM::SXTW, dl,
+                                           MVT::i32);
     if (isWorthFolding(RHS))
       return true;
   }
@@ -826,6 +846,7 @@ bool AArch64DAGToDAGISel::SelectAddrModeXRO(SDValue N, unsigned Size,
     return false;
   SDValue LHS = N.getOperand(0);
   SDValue RHS = N.getOperand(1);
+  SDLoc DL(N);
 
   // Check if this particular node is reused in any non-memory related
   // operation.  If yes, do not try to fold this node into the address
@@ -857,7 +878,6 @@ bool AArch64DAGToDAGISel::SelectAddrModeXRO(SDValue N, unsigned Size,
         isPreferredADD(ImmOff) || isPreferredADD(-ImmOff))
       return false;
 
-    SDLoc DL(N.getNode());
     SDValue Ops[] = { RHS };
     SDNode *MOVI =
         CurDAG->getMachineNode(AArch64::MOVi64imm, DL, MVT::i64, Ops);
@@ -873,7 +893,7 @@ bool AArch64DAGToDAGISel::SelectAddrModeXRO(SDValue N, unsigned Size,
   if (IsExtendedRegisterWorthFolding && RHS.getOpcode() == ISD::SHL &&
       SelectExtendedSHL(RHS, Size, false, Offset, SignExtend)) {
     Base = LHS;
-    DoShift = CurDAG->getTargetConstant(true, MVT::i32);
+    DoShift = CurDAG->getTargetConstant(true, DL, MVT::i32);
     return true;
   }
 
@@ -881,15 +901,15 @@ bool AArch64DAGToDAGISel::SelectAddrModeXRO(SDValue N, unsigned Size,
   if (IsExtendedRegisterWorthFolding && LHS.getOpcode() == ISD::SHL &&
       SelectExtendedSHL(LHS, Size, false, Offset, SignExtend)) {
     Base = RHS;
-    DoShift = CurDAG->getTargetConstant(true, MVT::i32);
+    DoShift = CurDAG->getTargetConstant(true, DL, MVT::i32);
     return true;
   }
 
   // Match any non-shifted, non-extend, non-immediate add expression.
   Base = LHS;
   Offset = RHS;
-  SignExtend = CurDAG->getTargetConstant(false, MVT::i32);
-  DoShift = CurDAG->getTargetConstant(false, MVT::i32);
+  SignExtend = CurDAG->getTargetConstant(false, DL, MVT::i32);
+  DoShift = CurDAG->getTargetConstant(false, DL, MVT::i32);
   // Reg1 + Reg2 is free: no check needed.
   return true;
 }
@@ -922,18 +942,18 @@ SDValue AArch64DAGToDAGISel::createTuple(ArrayRef<SDValue> Regs,
 
   assert(Regs.size() >= 2 && Regs.size() <= 4);
 
-  SDLoc DL(Regs[0].getNode());
+  SDLoc DL(Regs[0]);
 
   SmallVector<SDValue, 4> Ops;
 
   // First operand of REG_SEQUENCE is the desired RegClass.
   Ops.push_back(
-      CurDAG->getTargetConstant(RegClassIDs[Regs.size() - 2], MVT::i32));
+      CurDAG->getTargetConstant(RegClassIDs[Regs.size() - 2], DL, MVT::i32));
 
   // Then we get pairs of source & subregister-position for the components.
   for (unsigned i = 0; i < Regs.size(); ++i) {
     Ops.push_back(Regs[i]);
-    Ops.push_back(CurDAG->getTargetConstant(SubRegs[i], MVT::i32));
+    Ops.push_back(CurDAG->getTargetConstant(SubRegs[i], DL, MVT::i32));
   }
 
   SDNode *N =
@@ -1030,19 +1050,21 @@ SDNode *AArch64DAGToDAGISel::SelectIndexedLoad(SDNode *N, bool &Done) {
   SDValue Base = LD->getBasePtr();
   ConstantSDNode *OffsetOp = cast<ConstantSDNode>(LD->getOffset());
   int OffsetVal = (int)OffsetOp->getZExtValue();
-  SDValue Offset = CurDAG->getTargetConstant(OffsetVal, MVT::i64);
+  SDLoc dl(N);
+  SDValue Offset = CurDAG->getTargetConstant(OffsetVal, dl, MVT::i64);
   SDValue Ops[] = { Base, Offset, Chain };
-  SDNode *Res = CurDAG->getMachineNode(Opcode, SDLoc(N), MVT::i64, DstVT,
+  SDNode *Res = CurDAG->getMachineNode(Opcode, dl, MVT::i64, DstVT,
                                        MVT::Other, Ops);
   // Either way, we're replacing the node, so tell the caller that.
   Done = true;
   SDValue LoadedVal = SDValue(Res, 1);
   if (InsertTo64) {
-    SDValue SubReg = CurDAG->getTargetConstant(AArch64::sub_32, MVT::i32);
+    SDValue SubReg = CurDAG->getTargetConstant(AArch64::sub_32, dl, MVT::i32);
     LoadedVal =
         SDValue(CurDAG->getMachineNode(
-                    AArch64::SUBREG_TO_REG, SDLoc(N), MVT::i64,
-                    CurDAG->getTargetConstant(0, MVT::i64), LoadedVal, SubReg),
+                    AArch64::SUBREG_TO_REG, dl, MVT::i64,
+                    CurDAG->getTargetConstant(0, dl, MVT::i64), LoadedVal,
+                    SubReg),
                 0);
   }
 
@@ -1198,7 +1220,7 @@ SDNode *AArch64DAGToDAGISel::SelectLoadLane(SDNode *N, unsigned NumVecs,
   unsigned LaneNo =
       cast<ConstantSDNode>(N->getOperand(NumVecs + 2))->getZExtValue();
 
-  SDValue Ops[] = {RegSeq, CurDAG->getTargetConstant(LaneNo, MVT::i64),
+  SDValue Ops[] = {RegSeq, CurDAG->getTargetConstant(LaneNo, dl, MVT::i64),
                    N->getOperand(NumVecs + 3), N->getOperand(0)};
   SDNode *Ld = CurDAG->getMachineNode(Opc, dl, ResTys, Ops);
   SDValue SuperReg = SDValue(Ld, 0);
@@ -1234,13 +1256,14 @@ SDNode *AArch64DAGToDAGISel::SelectPostLoadLane(SDNode *N, unsigned NumVecs,
   SDValue RegSeq = createQTuple(Regs);
 
   const EVT ResTys[] = {MVT::i64, // Type of the write back register
-                        MVT::Untyped, MVT::Other};
+                        RegSeq->getValueType(0), MVT::Other};
 
   unsigned LaneNo =
       cast<ConstantSDNode>(N->getOperand(NumVecs + 1))->getZExtValue();
 
   SDValue Ops[] = {RegSeq,
-                   CurDAG->getTargetConstant(LaneNo, MVT::i64), // Lane Number
+                   CurDAG->getTargetConstant(LaneNo, dl,
+                                             MVT::i64),         // Lane Number
                    N->getOperand(NumVecs + 2),                  // Base register
                    N->getOperand(NumVecs + 3),                  // Incremental
                    N->getOperand(0)};
@@ -1291,7 +1314,7 @@ SDNode *AArch64DAGToDAGISel::SelectStoreLane(SDNode *N, unsigned NumVecs,
   unsigned LaneNo =
       cast<ConstantSDNode>(N->getOperand(NumVecs + 2))->getZExtValue();
 
-  SDValue Ops[] = {RegSeq, CurDAG->getTargetConstant(LaneNo, MVT::i64),
+  SDValue Ops[] = {RegSeq, CurDAG->getTargetConstant(LaneNo, dl, MVT::i64),
                    N->getOperand(NumVecs + 3), N->getOperand(0)};
   SDNode *St = CurDAG->getMachineNode(Opc, dl, MVT::Other, Ops);
 
@@ -1324,7 +1347,7 @@ SDNode *AArch64DAGToDAGISel::SelectPostStoreLane(SDNode *N, unsigned NumVecs,
   unsigned LaneNo =
       cast<ConstantSDNode>(N->getOperand(NumVecs + 1))->getZExtValue();
 
-  SDValue Ops[] = {RegSeq, CurDAG->getTargetConstant(LaneNo, MVT::i64),
+  SDValue Ops[] = {RegSeq, CurDAG->getTargetConstant(LaneNo, dl, MVT::i64),
                    N->getOperand(NumVecs + 2), // Base Register
                    N->getOperand(NumVecs + 3), // Incremental
                    N->getOperand(0)};
@@ -1476,7 +1499,7 @@ static bool isSeveralBitsExtractOpFromShr(SDNode *N, unsigned &Opc,
 }
 
 static bool isBitfieldExtractOpFromShr(SDNode *N, unsigned &Opc, SDValue &Opd0,
-                                       unsigned &LSB, unsigned &MSB,
+                                       unsigned &Immr, unsigned &Imms,
                                        bool BiggerPattern) {
   assert((N->getOpcode() == ISD::SRA || N->getOpcode() == ISD::SRL) &&
          "N must be a SHR/SRA operation to call this function");
@@ -1490,7 +1513,7 @@ static bool isBitfieldExtractOpFromShr(SDNode *N, unsigned &Opc, SDValue &Opd0,
          "Type checking must have been done before calling this function");
 
   // Check for AND + SRL doing several bits extract.
-  if (isSeveralBitsExtractOpFromShr(N, Opc, Opd0, LSB, MSB))
+  if (isSeveralBitsExtractOpFromShr(N, Opc, Opd0, Immr, Imms))
     return true;
 
   // we're looking for a shift of a shift
@@ -1530,13 +1553,9 @@ static bool isBitfieldExtractOpFromShr(SDNode *N, unsigned &Opc, SDValue &Opd0,
 
   assert(Srl_imm > 0 && Srl_imm < VT.getSizeInBits() &&
          "bad amount in shift node!");
-  // Note: The width operand is encoded as width-1.
-  unsigned Width = VT.getSizeInBits() - Trunc_bits - Srl_imm - 1;
-  int sLSB = Srl_imm - Shl_imm;
-  if (sLSB < 0)
-    return false;
-  LSB = sLSB;
-  MSB = LSB + Width;
+  int immr = Srl_imm - Shl_imm;
+  Immr = immr < 0 ? immr + VT.getSizeInBits() : immr;
+  Imms = VT.getSizeInBits() - Shl_imm - Trunc_bits - 1;
   // SRA requires a signed extraction
   if (VT == MVT::i32)
     Opc = N->getOpcode() == ISD::SRA ? AArch64::SBFMWri : AArch64::UBFMWri;
@@ -1546,7 +1565,7 @@ static bool isBitfieldExtractOpFromShr(SDNode *N, unsigned &Opc, SDValue &Opd0,
 }
 
 static bool isBitfieldExtractOp(SelectionDAG *CurDAG, SDNode *N, unsigned &Opc,
-                                SDValue &Opd0, unsigned &LSB, unsigned &MSB,
+                                SDValue &Opd0, unsigned &Immr, unsigned &Imms,
                                 unsigned NumberOfIgnoredLowBits = 0,
                                 bool BiggerPattern = false) {
   if (N->getValueType(0) != MVT::i32 && N->getValueType(0) != MVT::i64)
@@ -1558,11 +1577,11 @@ static bool isBitfieldExtractOp(SelectionDAG *CurDAG, SDNode *N, unsigned &Opc,
       return false;
     break;
   case ISD::AND:
-    return isBitfieldExtractOpFromAnd(CurDAG, N, Opc, Opd0, LSB, MSB,
+    return isBitfieldExtractOpFromAnd(CurDAG, N, Opc, Opd0, Immr, Imms,
                                       NumberOfIgnoredLowBits, BiggerPattern);
   case ISD::SRL:
   case ISD::SRA:
-    return isBitfieldExtractOpFromShr(N, Opc, Opd0, LSB, MSB, BiggerPattern);
+    return isBitfieldExtractOpFromShr(N, Opc, Opd0, Immr, Imms, BiggerPattern);
   }
 
   unsigned NOpc = N->getMachineOpcode();
@@ -1575,8 +1594,8 @@ static bool isBitfieldExtractOp(SelectionDAG *CurDAG, SDNode *N, unsigned &Opc,
   case AArch64::UBFMXri:
     Opc = NOpc;
     Opd0 = N->getOperand(0);
-    LSB = cast<ConstantSDNode>(N->getOperand(1).getNode())->getZExtValue();
-    MSB = cast<ConstantSDNode>(N->getOperand(2).getNode())->getZExtValue();
+    Immr = cast<ConstantSDNode>(N->getOperand(1).getNode())->getZExtValue();
+    Imms = cast<ConstantSDNode>(N->getOperand(2).getNode())->getZExtValue();
     return true;
   }
   // Unreachable
@@ -1584,29 +1603,30 @@ static bool isBitfieldExtractOp(SelectionDAG *CurDAG, SDNode *N, unsigned &Opc,
 }
 
 SDNode *AArch64DAGToDAGISel::SelectBitfieldExtractOp(SDNode *N) {
-  unsigned Opc, LSB, MSB;
+  unsigned Opc, Immr, Imms;
   SDValue Opd0;
-  if (!isBitfieldExtractOp(CurDAG, N, Opc, Opd0, LSB, MSB))
+  if (!isBitfieldExtractOp(CurDAG, N, Opc, Opd0, Immr, Imms))
     return nullptr;
 
   EVT VT = N->getValueType(0);
+  SDLoc dl(N);
 
   // If the bit extract operation is 64bit but the original type is 32bit, we
   // need to add one EXTRACT_SUBREG.
   if ((Opc == AArch64::SBFMXri || Opc == AArch64::UBFMXri) && VT == MVT::i32) {
-    SDValue Ops64[] = {Opd0, CurDAG->getTargetConstant(LSB, MVT::i64),
-                       CurDAG->getTargetConstant(MSB, MVT::i64)};
+    SDValue Ops64[] = {Opd0, CurDAG->getTargetConstant(Immr, dl, MVT::i64),
+                       CurDAG->getTargetConstant(Imms, dl, MVT::i64)};
 
-    SDNode *BFM = CurDAG->getMachineNode(Opc, SDLoc(N), MVT::i64, Ops64);
-    SDValue SubReg = CurDAG->getTargetConstant(AArch64::sub_32, MVT::i32);
+    SDNode *BFM = CurDAG->getMachineNode(Opc, dl, MVT::i64, Ops64);
+    SDValue SubReg = CurDAG->getTargetConstant(AArch64::sub_32, dl, MVT::i32);
     MachineSDNode *Node =
-        CurDAG->getMachineNode(TargetOpcode::EXTRACT_SUBREG, SDLoc(N), MVT::i32,
+        CurDAG->getMachineNode(TargetOpcode::EXTRACT_SUBREG, dl, MVT::i32,
                                SDValue(BFM, 0), SubReg);
     return Node;
   }
 
-  SDValue Ops[] = {Opd0, CurDAG->getTargetConstant(LSB, VT),
-                   CurDAG->getTargetConstant(MSB, VT)};
+  SDValue Ops[] = {Opd0, CurDAG->getTargetConstant(Immr, dl, VT),
+                   CurDAG->getTargetConstant(Imms, dl, VT)};
   return CurDAG->SelectNodeTo(N, Opc, VT, Ops);
 }
 
@@ -1810,6 +1830,7 @@ static SDValue getLeftShift(SelectionDAG *CurDAG, SDValue Op, int ShlAmount) {
     return Op;
 
   EVT VT = Op.getValueType();
+  SDLoc dl(Op);
   unsigned BitWidth = VT.getSizeInBits();
   unsigned UBFMOpc = BitWidth == 32 ? AArch64::UBFMWri : AArch64::UBFMXri;
 
@@ -1817,16 +1838,16 @@ static SDValue getLeftShift(SelectionDAG *CurDAG, SDValue Op, int ShlAmount) {
   if (ShlAmount > 0) {
     // LSL wD, wN, #Amt == UBFM wD, wN, #32-Amt, #31-Amt
     ShiftNode = CurDAG->getMachineNode(
-        UBFMOpc, SDLoc(Op), VT, Op,
-        CurDAG->getTargetConstant(BitWidth - ShlAmount, VT),
-        CurDAG->getTargetConstant(BitWidth - 1 - ShlAmount, VT));
+        UBFMOpc, dl, VT, Op,
+        CurDAG->getTargetConstant(BitWidth - ShlAmount, dl, VT),
+        CurDAG->getTargetConstant(BitWidth - 1 - ShlAmount, dl, VT));
   } else {
     // LSR wD, wN, #Amt == UBFM wD, wN, #Amt, #32-1
     assert(ShlAmount < 0 && "expected right shift");
     int ShrAmount = -ShlAmount;
     ShiftNode = CurDAG->getMachineNode(
-        UBFMOpc, SDLoc(Op), VT, Op, CurDAG->getTargetConstant(ShrAmount, VT),
-        CurDAG->getTargetConstant(BitWidth - 1, VT));
+        UBFMOpc, dl, VT, Op, CurDAG->getTargetConstant(ShrAmount, dl, VT),
+        CurDAG->getTargetConstant(BitWidth - 1, dl, VT));
   }
 
   return SDValue(ShiftNode, 0);
@@ -1991,18 +2012,37 @@ SDNode *AArch64DAGToDAGISel::SelectBitfieldInsertOp(SDNode *N) {
     return nullptr;
 
   EVT VT = N->getValueType(0);
+  SDLoc dl(N);
   SDValue Ops[] = { Opd0,
                     Opd1,
-                    CurDAG->getTargetConstant(LSB, VT),
-                    CurDAG->getTargetConstant(MSB, VT) };
+                    CurDAG->getTargetConstant(LSB, dl, VT),
+                    CurDAG->getTargetConstant(MSB, dl, VT) };
   return CurDAG->SelectNodeTo(N, Opc, VT, Ops);
 }
 
+/// GenerateInexactFlagIfNeeded - Insert FRINTX instruction to generate inexact
+/// signal on round-to-integer operations if needed. C11 leaves it
+/// implementation-defined whether these operations trigger an inexact
+/// exception. IEEE says they don't.  Unfortunately, Darwin decided they do so
+/// we sometimes have to insert a special instruction just to set the right bit
+/// in FPSR.
+SDNode *AArch64DAGToDAGISel::GenerateInexactFlagIfNeeded(const SDValue &In,
+                                                         unsigned InTyVariant,
+                                                         SDLoc DL) {
+  if (Subtarget->isTargetDarwin() && !TM.Options.UnsafeFPMath) {
+    // Pick the right FRINTX using InTyVariant needed to set the flags.
+    // InTyVariant is 0 for 32-bit and 1 for 64-bit.
+    unsigned FRINTXOpcs[] = { AArch64::FRINTXSr, AArch64::FRINTXDr };
+    return CurDAG->getMachineNode(FRINTXOpcs[InTyVariant], DL,
+                                  In.getValueType(), MVT::Glue, In);
+  }
+  return nullptr;
+}
+
 SDNode *AArch64DAGToDAGISel::SelectLIBM(SDNode *N) {
   EVT VT = N->getValueType(0);
   unsigned Variant;
   unsigned Opc;
-  unsigned FRINTXOpcs[] = { AArch64::FRINTXSr, AArch64::FRINTXDr };
 
   if (VT == MVT::f32) {
     Variant = 0;
@@ -2011,9 +2051,6 @@ SDNode *AArch64DAGToDAGISel::SelectLIBM(SDNode *N) {
   } else
     return nullptr; // Unrecognized argument type. Fall back on default codegen.
 
-  // Pick the FRINTX variant needed to set the flags.
-  unsigned FRINTXOpc = FRINTXOpcs[Variant];
-
   switch (N->getOpcode()) {
   default:
     return nullptr; // Unrecognized libm ISD node. Fall back on default codegen.
@@ -2044,14 +2081,97 @@ SDNode *AArch64DAGToDAGISel::SelectLIBM(SDNode *N) {
   SmallVector<SDValue, 2> Ops;
   Ops.push_back(In);
 
-  if (!TM.Options.UnsafeFPMath) {
-    SDNode *FRINTX = CurDAG->getMachineNode(FRINTXOpc, dl, VT, MVT::Glue, In);
-    Ops.push_back(SDValue(FRINTX, 1));
-  }
+  if (SDNode *FRINTXNode = GenerateInexactFlagIfNeeded(In, Variant, dl))
+    Ops.push_back(SDValue(FRINTXNode, 1));
 
   return CurDAG->getMachineNode(Opc, dl, VT, Ops);
 }
 
+/// SelectFPConvertWithRound - Try to combine FP rounding and
+/// FP-INT conversion.
+SDNode *AArch64DAGToDAGISel::SelectFPConvertWithRound(SDNode *N) {
+  SDNode *Op0 = N->getOperand(0).getNode();
+
+  // Return if the round op is used by other nodes, as this would result in two
+  // FRINTX, one each for round and convert.
+  if (!Op0->hasOneUse())
+    return nullptr;
+
+  unsigned InTyVariant;
+  EVT InTy = Op0->getValueType(0);
+  if (InTy == MVT::f32)
+    InTyVariant = 0;
+  else if (InTy == MVT::f64)
+    InTyVariant = 1;
+  else
+    return nullptr;
+
+  unsigned OutTyVariant;
+  EVT OutTy = N->getValueType(0);
+  if (OutTy == MVT::i32)
+    OutTyVariant = 0;
+  else if (OutTy == MVT::i64)
+    OutTyVariant = 1;
+  else
+    return nullptr;
+
+  assert((N->getOpcode() == ISD::FP_TO_SINT
+          || N->getOpcode() == ISD::FP_TO_UINT) && "Unexpected opcode!");
+  unsigned FpConVariant = N->getOpcode() == ISD::FP_TO_SINT ? 0 : 1;
+
+  unsigned Opc;
+  switch (Op0->getOpcode()) {
+  default:
+    return nullptr;
+  case ISD::FCEIL: {
+    unsigned FCVTPOpcs[2][2][2] = {
+        { { AArch64::FCVTPSUWSr, AArch64::FCVTPSUXSr },
+          { AArch64::FCVTPSUWDr, AArch64::FCVTPSUXDr } },
+        { { AArch64::FCVTPUUWSr, AArch64::FCVTPUUXSr },
+          { AArch64::FCVTPUUWDr, AArch64::FCVTPUUXDr } } };
+    Opc = FCVTPOpcs[FpConVariant][InTyVariant][OutTyVariant];
+    break;
+  }
+  case ISD::FFLOOR: {
+    unsigned FCVTMOpcs[2][2][2] = {
+        { { AArch64::FCVTMSUWSr, AArch64::FCVTMSUXSr },
+          { AArch64::FCVTMSUWDr, AArch64::FCVTMSUXDr } },
+        { { AArch64::FCVTMUUWSr, AArch64::FCVTMUUXSr },
+          { AArch64::FCVTMUUWDr, AArch64::FCVTMUUXDr } } };
+    Opc = FCVTMOpcs[FpConVariant][InTyVariant][OutTyVariant];
+    break;
+  }
+  case ISD::FTRUNC: {
+    unsigned FCVTZOpcs[2][2][2] = {
+        { { AArch64::FCVTZSUWSr, AArch64::FCVTZSUXSr },
+          { AArch64::FCVTZSUWDr, AArch64::FCVTZSUXDr } },
+        { { AArch64::FCVTZUUWSr, AArch64::FCVTZUUXSr },
+          { AArch64::FCVTZUUWDr, AArch64::FCVTZUUXDr } } };
+    Opc = FCVTZOpcs[FpConVariant][InTyVariant][OutTyVariant];
+    break;
+  }
+  case ISD::FROUND: {
+    unsigned FCVTAOpcs[2][2][2] = {
+        { { AArch64::FCVTASUWSr, AArch64::FCVTASUXSr },
+          { AArch64::FCVTASUWDr, AArch64::FCVTASUXDr } },
+        { { AArch64::FCVTAUUWSr, AArch64::FCVTAUUXSr },
+          { AArch64::FCVTAUUWDr, AArch64::FCVTAUUXDr } } };
+    Opc = FCVTAOpcs[FpConVariant][InTyVariant][OutTyVariant];
+    break;
+  }
+  }
+
+  SDLoc DL(N);
+  SDValue In = Op0->getOperand(0);
+  SmallVector<SDValue, 2> Ops;
+  Ops.push_back(In);
+
+  if (SDNode *FRINTXNode = GenerateInexactFlagIfNeeded(In, InTyVariant, DL))
+    Ops.push_back(SDValue(FRINTXNode, 1));
+
+  return CurDAG->getMachineNode(Opc, DL, OutTy, Ops);
+}
+
 bool
 AArch64DAGToDAGISel::SelectCVTFixedPosOperand(SDValue N, SDValue &FixedPos,
                                               unsigned RegWidth) {
@@ -2092,10 +2212,124 @@ AArch64DAGToDAGISel::SelectCVTFixedPosOperand(SDValue N, SDValue &FixedPos,
   // finding FBits, but it must still be in range.
   if (FBits == 0 || FBits > RegWidth) return false;
 
-  FixedPos = CurDAG->getTargetConstant(FBits, MVT::i32);
+  FixedPos = CurDAG->getTargetConstant(FBits, SDLoc(N), MVT::i32);
   return true;
 }
 
+// Inspects a register string of the form o0:op1:CRn:CRm:op2 gets the fields
+// of the string and obtains the integer values from them and combines these
+// into a single value to be used in the MRS/MSR instruction.
+static int getIntOperandFromRegisterString(StringRef RegString) {
+  SmallVector<StringRef, 5> Fields;
+  RegString.split(Fields, ":");
+
+  if (Fields.size() == 1)
+    return -1;
+
+  assert(Fields.size() == 5
+            && "Invalid number of fields in read register string");
+
+  SmallVector<int, 5> Ops;
+  bool AllIntFields = true;
+
+  for (StringRef Field : Fields) {
+    unsigned IntField;
+    AllIntFields &= !Field.getAsInteger(10, IntField);
+    Ops.push_back(IntField);
+  }
+
+  assert(AllIntFields &&
+          "Unexpected non-integer value in special register string.");
+
+  // Need to combine the integer fields of the string into a single value
+  // based on the bit encoding of MRS/MSR instruction.
+  return (Ops[0] << 14) | (Ops[1] << 11) | (Ops[2] << 7) |
+         (Ops[3] << 3) | (Ops[4]);
+}
+
+// Lower the read_register intrinsic to an MRS instruction node if the special
+// register string argument is either of the form detailed in the ALCE (the
+// form described in getIntOperandsFromRegsterString) or is a named register
+// known by the MRS SysReg mapper.
+SDNode *AArch64DAGToDAGISel::SelectReadRegister(SDNode *N) {
+  const MDNodeSDNode *MD = dyn_cast<MDNodeSDNode>(N->getOperand(1));
+  const MDString *RegString = dyn_cast<MDString>(MD->getMD()->getOperand(0));
+  SDLoc DL(N);
+
+  int Reg = getIntOperandFromRegisterString(RegString->getString());
+  if (Reg != -1)
+    return CurDAG->getMachineNode(AArch64::MRS, DL, N->getSimpleValueType(0),
+                                  MVT::Other,
+                                  CurDAG->getTargetConstant(Reg, DL, MVT::i32),
+                                  N->getOperand(0));
+
+  // Use the sysreg mapper to map the remaining possible strings to the
+  // value for the register to be used for the instruction operand.
+  AArch64SysReg::MRSMapper mapper;
+  bool IsValidSpecialReg;
+  Reg = mapper.fromString(RegString->getString(),
+                          Subtarget->getFeatureBits(),
+                          IsValidSpecialReg);
+  if (IsValidSpecialReg)
+    return CurDAG->getMachineNode(AArch64::MRS, DL, N->getSimpleValueType(0),
+                                  MVT::Other,
+                                  CurDAG->getTargetConstant(Reg, DL, MVT::i32),
+                                  N->getOperand(0));
+
+  return nullptr;
+}
+
+// Lower the write_register intrinsic to an MSR instruction node if the special
+// register string argument is either of the form detailed in the ALCE (the
+// form described in getIntOperandsFromRegsterString) or is a named register
+// known by the MSR SysReg mapper.
+SDNode *AArch64DAGToDAGISel::SelectWriteRegister(SDNode *N) {
+  const MDNodeSDNode *MD = dyn_cast<MDNodeSDNode>(N->getOperand(1));
+  const MDString *RegString = dyn_cast<MDString>(MD->getMD()->getOperand(0));
+  SDLoc DL(N);
+
+  int Reg = getIntOperandFromRegisterString(RegString->getString());
+  if (Reg != -1)
+    return CurDAG->getMachineNode(AArch64::MSR, DL, MVT::Other,
+                                  CurDAG->getTargetConstant(Reg, DL, MVT::i32),
+                                  N->getOperand(2), N->getOperand(0));
+
+  // Check if the register was one of those allowed as the pstatefield value in
+  // the MSR (immediate) instruction. To accept the values allowed in the
+  // pstatefield for the MSR (immediate) instruction, we also require that an
+  // immediate value has been provided as an argument, we know that this is
+  // the case as it has been ensured by semantic checking.
+  AArch64PState::PStateMapper PMapper;
+  bool IsValidSpecialReg;
+  Reg = PMapper.fromString(RegString->getString(),
+                           Subtarget->getFeatureBits(),
+                           IsValidSpecialReg);
+  if (IsValidSpecialReg) {
+    assert (isa<ConstantSDNode>(N->getOperand(2))
+              && "Expected a constant integer expression.");
+    uint64_t Immed = cast<ConstantSDNode>(N->getOperand(2))->getZExtValue();
+    return CurDAG->getMachineNode(AArch64::MSRpstate, DL, MVT::Other,
+                                  CurDAG->getTargetConstant(Reg, DL, MVT::i32),
+                                  CurDAG->getTargetConstant(Immed, DL, MVT::i16),
+                                  N->getOperand(0));
+  }
+
+  // Use the sysreg mapper to attempt to map the remaining possible strings
+  // to the value for the register to be used for the MSR (register)
+  // instruction operand.
+  AArch64SysReg::MSRMapper Mapper;
+  Reg = Mapper.fromString(RegString->getString(),
+                          Subtarget->getFeatureBits(),
+                          IsValidSpecialReg);
+
+  if (IsValidSpecialReg)
+    return CurDAG->getMachineNode(AArch64::MSR, DL, MVT::Other,
+                                  CurDAG->getTargetConstant(Reg, DL, MVT::i32),
+                                  N->getOperand(2), N->getOperand(0));
+
+  return nullptr;
+}
+
 SDNode *AArch64DAGToDAGISel::Select(SDNode *Node) {
   // Dump information about the Node being selected
   DEBUG(errs() << "Selecting: ");
@@ -2117,6 +2351,16 @@ SDNode *AArch64DAGToDAGISel::Select(SDNode *Node) {
   default:
     break;
 
+  case ISD::READ_REGISTER:
+    if (SDNode *Res = SelectReadRegister(Node))
+      return Res;
+    break;
+
+  case ISD::WRITE_REGISTER:
+    if (SDNode *Res = SelectWriteRegister(Node))
+      return Res;
+    break;
+
   case ISD::ADD:
     if (SDNode *I = SelectMLAV64LaneV128(Node))
       return I;
@@ -2206,9 +2450,11 @@ SDNode *AArch64DAGToDAGISel::Select(SDNode *Node) {
     int FI = cast<FrameIndexSDNode>(Node)->getIndex();
     unsigned Shifter = AArch64_AM::getShifterImm(AArch64_AM::LSL, 0);
     const TargetLowering *TLI = getTargetLowering();
-    SDValue TFI = CurDAG->getTargetFrameIndex(FI, TLI->getPointerTy());
-    SDValue Ops[] = { TFI, CurDAG->getTargetConstant(0, MVT::i32),
-                      CurDAG->getTargetConstant(Shifter, MVT::i32) };
+    SDValue TFI = CurDAG->getTargetFrameIndex(
+        FI, TLI->getPointerTy(CurDAG->getDataLayout()));
+    SDLoc DL(Node);
+    SDValue Ops[] = { TFI, CurDAG->getTargetConstant(0, DL, MVT::i32),
+                      CurDAG->getTargetConstant(Shifter, DL, MVT::i32) };
     return CurDAG->SelectNodeTo(Node, AArch64::ADDXri, MVT::i64, Ops);
   }
   case ISD::INTRINSIC_W_CHAIN: {
@@ -3078,6 +3324,12 @@ SDNode *AArch64DAGToDAGISel::Select(SDNode *Node) {
     if (SDNode *I = SelectLIBM(Node))
       return I;
     break;
+
+  case ISD::FP_TO_SINT:
+  case ISD::FP_TO_UINT:
+    if (SDNode *I = SelectFPConvertWithRound(Node))
+      return I;
+    break;
   }
 
   // Select the default instruction
diff --git a/lib/Target/AArch64/AArch64ISelLowering.cpp b/lib/Target/AArch64/AArch64ISelLowering.cpp
index 90a5e5ef6b7..8b690b96f1d 100644
--- a/lib/Target/AArch64/AArch64ISelLowering.cpp
+++ b/lib/Target/AArch64/AArch64ISelLowering.cpp
@@ -25,6 +25,7 @@
 #include "llvm/CodeGen/MachineInstrBuilder.h"
 #include "llvm/CodeGen/MachineRegisterInfo.h"
 #include "llvm/IR/Function.h"
+#include "llvm/IR/GetElementPtrTypeIterator.h"
 #include "llvm/IR/Intrinsics.h"
 #include "llvm/IR/Type.h"
 #include "llvm/Support/CommandLine.h"
@@ -39,23 +40,6 @@ using namespace llvm;
 STATISTIC(NumTailCalls, "Number of tail calls");
 STATISTIC(NumShiftInserts, "Number of vector shift inserts");
 
-namespace {
-enum AlignMode {
-  StrictAlign,
-  NoStrictAlign
-};
-}
-
-static cl::opt<AlignMode>
-Align(cl::desc("Load/store alignment support"),
-      cl::Hidden, cl::init(NoStrictAlign),
-      cl::values(
-          clEnumValN(StrictAlign,   "aarch64-strict-align",
-                     "Disallow all unaligned memory accesses"),
-          clEnumValN(NoStrictAlign, "aarch64-no-strict-align",
-                     "Allow unaligned memory accesses"),
-          clEnumValEnd));
-
 // Place holder until extr generation is tested fully.
 static cl::opt<bool>
 EnableAArch64ExtrGeneration("aarch64-extr-generation", cl::Hidden,
@@ -75,6 +59,9 @@ cl::opt<bool> EnableAArch64ELFLocalDynamicTLSGeneration(
     cl::desc("Allow AArch64 Local Dynamic TLS code generation"),
     cl::init(false));
 
+/// Value type used for condition codes.
+static const MVT MVT_CC = MVT::i32;
+
 AArch64TargetLowering::AArch64TargetLowering(const TargetMachine &TM,
                                              const AArch64Subtarget &STI)
     : TargetLowering(TM), Subtarget(&STI) {
@@ -491,6 +478,7 @@ AArch64TargetLowering::AArch64TargetLowering(const TargetMachine &TM,
 
   setTargetDAGCombine(ISD::SELECT);
   setTargetDAGCombine(ISD::VSELECT);
+  setTargetDAGCombine(ISD::SELECT_CC);
 
   setTargetDAGCombine(ISD::INTRINSIC_VOID);
   setTargetDAGCombine(ISD::INTRINSIC_W_CHAIN);
@@ -510,10 +498,10 @@ AArch64TargetLowering::AArch64TargetLowering(const TargetMachine &TM,
 
   setMinFunctionAlignment(2);
 
-  RequireStrictAlign = (Align == StrictAlign);
-
   setHasExtractBitsInsn(true);
 
+  setOperationAction(ISD::INTRINSIC_WO_CHAIN, MVT::Other, Custom);
+
   if (Subtarget->hasNEON()) {
     // FIXME: v1f64 shouldn't be legal if we can avoid it, because it leads to
     // silliness like this:
@@ -557,11 +545,21 @@ AArch64TargetLowering::AArch64TargetLowering(const TargetMachine &TM,
     setOperationAction(ISD::SINT_TO_FP, MVT::v4i8, Promote);
     setOperationAction(ISD::UINT_TO_FP, MVT::v4i16, Promote);
     setOperationAction(ISD::SINT_TO_FP, MVT::v4i16, Promote);
+    // i8 and i16 vector elements also need promotion to i32 for v8i8 or v8i16
+    // -> v8f16 conversions.
+    setOperationAction(ISD::SINT_TO_FP, MVT::v8i8, Promote);
+    setOperationAction(ISD::UINT_TO_FP, MVT::v8i8, Promote);
+    setOperationAction(ISD::SINT_TO_FP, MVT::v8i16, Promote);
+    setOperationAction(ISD::UINT_TO_FP, MVT::v8i16, Promote);
     // Similarly, there is no direct i32 -> f64 vector conversion instruction.
     setOperationAction(ISD::SINT_TO_FP, MVT::v2i32, Custom);
     setOperationAction(ISD::UINT_TO_FP, MVT::v2i32, Custom);
     setOperationAction(ISD::SINT_TO_FP, MVT::v2i64, Custom);
     setOperationAction(ISD::UINT_TO_FP, MVT::v2i64, Custom);
+    // Or, direct i32 -> f16 vector conversion.  Set it so custom, so the
+    // conversion happens in two steps: v4i32 -> v4f32 -> v4f16
+    setOperationAction(ISD::SINT_TO_FP, MVT::v4i32, Custom);
+    setOperationAction(ISD::UINT_TO_FP, MVT::v4i32, Custom);
 
     // AArch64 doesn't have MUL.2d:
     setOperationAction(ISD::MUL, MVT::v2i64, Expand);
@@ -668,6 +666,14 @@ void AArch64TargetLowering::addTypeForNEON(EVT VT, EVT PromotedBitwiseVT) {
   setOperationAction(ISD::FP_TO_SINT, VT.getSimpleVT(), Custom);
   setOperationAction(ISD::FP_TO_UINT, VT.getSimpleVT(), Custom);
 
+  // [SU][MIN|MAX] and [SU]ABSDIFF are available for all NEON types apart from
+  // i64.
+  if (!VT.isFloatingPoint() &&
+      VT.getSimpleVT() != MVT::v2i64 && VT.getSimpleVT() != MVT::v1i64)
+    for (unsigned Opcode : {ISD::SMIN, ISD::SMAX, ISD::UMIN, ISD::UMAX,
+                            ISD::SABSDIFF, ISD::UABSDIFF})
+      setOperationAction(Opcode, VT.getSimpleVT(), Legal);
+
   if (Subtarget->isLittleEndian()) {
     for (unsigned im = (unsigned)ISD::PRE_INC;
          im != (unsigned)ISD::LAST_INDEXED_MODE; ++im) {
@@ -687,7 +693,8 @@ void AArch64TargetLowering::addQRTypeForNEON(MVT VT) {
   addTypeForNEON(VT, MVT::v4i32);
 }
 
-EVT AArch64TargetLowering::getSetCCResultType(LLVMContext &, EVT VT) const {
+EVT AArch64TargetLowering::getSetCCResultType(const DataLayout &, LLVMContext &,
+                                              EVT VT) const {
   if (!VT.isVector())
     return MVT::i32;
   return VT.changeVectorElementTypeToInteger();
@@ -756,10 +763,23 @@ void AArch64TargetLowering::computeKnownBitsForTargetNode(
   }
 }
 
-MVT AArch64TargetLowering::getScalarShiftAmountTy(EVT LHSTy) const {
+MVT AArch64TargetLowering::getScalarShiftAmountTy(const DataLayout &DL,
+                                                  EVT) const {
   return MVT::i64;
 }
 
+bool AArch64TargetLowering::allowsMisalignedMemoryAccesses(EVT VT,
+                                                           unsigned AddrSpace,
+                                                           unsigned Align,
+                                                           bool *Fast) const {
+  if (Subtarget->requiresStrictAlign())
+    return false;
+  // FIXME: True for Cyclone, but not necessary others.
+  if (Fast)
+    *Fast = true;
+  return true;
+}
+
 FastISel *
 AArch64TargetLowering::createFastISel(FunctionLoweringInfo &funcInfo,
                                       const TargetLibraryInfo *libInfo) const {
@@ -767,9 +787,8 @@ AArch64TargetLowering::createFastISel(FunctionLoweringInfo &funcInfo,
 }
 
 const char *AArch64TargetLowering::getTargetNodeName(unsigned Opcode) const {
-  switch (Opcode) {
-  default:
-    return nullptr;
+  switch ((AArch64ISD::NodeType)Opcode) {
+  case AArch64ISD::FIRST_NUMBER:      break;
   case AArch64ISD::CALL:              return "AArch64ISD::CALL";
   case AArch64ISD::ADRP:              return "AArch64ISD::ADRP";
   case AArch64ISD::ADDlow:            return "AArch64ISD::ADDlow";
@@ -790,6 +809,9 @@ const char *AArch64TargetLowering::getTargetNodeName(unsigned Opcode) const {
   case AArch64ISD::ADCS:              return "AArch64ISD::ADCS";
   case AArch64ISD::SBCS:              return "AArch64ISD::SBCS";
   case AArch64ISD::ANDS:              return "AArch64ISD::ANDS";
+  case AArch64ISD::CCMP:              return "AArch64ISD::CCMP";
+  case AArch64ISD::CCMN:              return "AArch64ISD::CCMN";
+  case AArch64ISD::FCCMP:             return "AArch64ISD::FCCMP";
   case AArch64ISD::FCMP:              return "AArch64ISD::FCMP";
   case AArch64ISD::FMIN:              return "AArch64ISD::FMIN";
   case AArch64ISD::FMAX:              return "AArch64ISD::FMAX";
@@ -854,6 +876,7 @@ const char *AArch64TargetLowering::getTargetNodeName(unsigned Opcode) const {
   case AArch64ISD::TBZ:               return "AArch64ISD::TBZ";
   case AArch64ISD::TBNZ:              return "AArch64ISD::TBNZ";
   case AArch64ISD::TC_RETURN:         return "AArch64ISD::TC_RETURN";
+  case AArch64ISD::PREFETCH:          return "AArch64ISD::PREFETCH";
   case AArch64ISD::SITOF:             return "AArch64ISD::SITOF";
   case AArch64ISD::UITOF:             return "AArch64ISD::UITOF";
   case AArch64ISD::NVCAST:            return "AArch64ISD::NVCAST";
@@ -889,6 +912,7 @@ const char *AArch64TargetLowering::getTargetNodeName(unsigned Opcode) const {
   case AArch64ISD::SMULL:             return "AArch64ISD::SMULL";
   case AArch64ISD::UMULL:             return "AArch64ISD::UMULL";
   }
+  return nullptr;
 }
 
 MachineBasicBlock *
@@ -1146,14 +1170,224 @@ static SDValue emitComparison(SDValue LHS, SDValue RHS, ISD::CondCode CC,
     LHS = LHS.getOperand(0);
   }
 
-  return DAG.getNode(Opcode, dl, DAG.getVTList(VT, MVT::i32), LHS, RHS)
+  return DAG.getNode(Opcode, dl, DAG.getVTList(VT, MVT_CC), LHS, RHS)
       .getValue(1);
 }
 
+/// \defgroup AArch64CCMP CMP;CCMP matching
+///
+/// These functions deal with the formation of CMP;CCMP;... sequences.
+/// The CCMP/CCMN/FCCMP/FCCMPE instructions allow the conditional execution of
+/// a comparison. They set the NZCV flags to a predefined value if their
+/// predicate is false. This allows to express arbitrary conjunctions, for
+/// example "cmp 0 (and (setCA (cmp A)) (setCB (cmp B))))"
+/// expressed as:
+///   cmp A
+///   ccmp B, inv(CB), CA
+///   check for CB flags
+///
+/// In general we can create code for arbitrary "... (and (and A B) C)"
+/// sequences. We can also implement some "or" expressions, because "(or A B)"
+/// is equivalent to "not (and (not A) (not B))" and we can implement some
+/// negation operations:
+/// We can negate the results of a single comparison by inverting the flags
+/// used when the predicate fails and inverting the flags tested in the next
+/// instruction; We can also negate the results of the whole previous
+/// conditional compare sequence by inverting the flags tested in the next
+/// instruction. However there is no way to negate the result of a partial
+/// sequence.
+///
+/// Therefore on encountering an "or" expression we can negate the subtree on
+/// one side and have to be able to push the negate to the leafs of the subtree
+/// on the other side (see also the comments in code). As complete example:
+/// "or (or (setCA (cmp A)) (setCB (cmp B)))
+///     (and (setCC (cmp C)) (setCD (cmp D)))"
+/// is transformed to
+/// "not (and (not (and (setCC (cmp C)) (setCC (cmp D))))
+///           (and (not (setCA (cmp A)) (not (setCB (cmp B))))))"
+/// and implemented as:
+///   cmp C
+///   ccmp D, inv(CD), CC
+///   ccmp A, CA, inv(CD)
+///   ccmp B, CB, inv(CA)
+///   check for CB flags
+/// A counterexample is "or (and A B) (and C D)" which cannot be implemented
+/// by conditional compare sequences.
+/// @{
+
+/// Create a conditional comparison; Use CCMP, CCMN or FCCMP as appropriate.
+static SDValue emitConditionalComparison(SDValue LHS, SDValue RHS,
+                                         ISD::CondCode CC, SDValue CCOp,
+                                         SDValue Condition, unsigned NZCV,
+                                         SDLoc DL, SelectionDAG &DAG) {
+  unsigned Opcode = 0;
+  if (LHS.getValueType().isFloatingPoint())
+    Opcode = AArch64ISD::FCCMP;
+  else if (RHS.getOpcode() == ISD::SUB) {
+    SDValue SubOp0 = RHS.getOperand(0);
+    if (const ConstantSDNode *SubOp0C = dyn_cast<ConstantSDNode>(SubOp0))
+      if (SubOp0C->isNullValue() && (CC == ISD::SETEQ || CC == ISD::SETNE)) {
+        // See emitComparison() on why we can only do this for SETEQ and SETNE.
+        Opcode = AArch64ISD::CCMN;
+        RHS = RHS.getOperand(1);
+      }
+  }
+  if (Opcode == 0)
+    Opcode = AArch64ISD::CCMP;
+
+  SDValue NZCVOp = DAG.getConstant(NZCV, DL, MVT::i32);
+  return DAG.getNode(Opcode, DL, MVT_CC, LHS, RHS, NZCVOp, Condition, CCOp);
+}
+
+/// Returns true if @p Val is a tree of AND/OR/SETCC operations.
+/// CanPushNegate is set to true if we can push a negate operation through
+/// the tree in a was that we are left with AND operations and negate operations
+/// at the leafs only. i.e. "not (or (or x y) z)" can be changed to
+/// "and (and (not x) (not y)) (not z)"; "not (or (and x y) z)" cannot be
+/// brought into such a form.
+static bool isConjunctionDisjunctionTree(const SDValue Val, bool &CanPushNegate,
+                                         unsigned Depth = 0) {
+  if (!Val.hasOneUse())
+    return false;
+  unsigned Opcode = Val->getOpcode();
+  if (Opcode == ISD::SETCC) {
+    CanPushNegate = true;
+    return true;
+  }
+  // Protect against stack overflow.
+  if (Depth > 15)
+    return false;
+  if (Opcode == ISD::AND || Opcode == ISD::OR) {
+    SDValue O0 = Val->getOperand(0);
+    SDValue O1 = Val->getOperand(1);
+    bool CanPushNegateL;
+    if (!isConjunctionDisjunctionTree(O0, CanPushNegateL, Depth+1))
+      return false;
+    bool CanPushNegateR;
+    if (!isConjunctionDisjunctionTree(O1, CanPushNegateR, Depth+1))
+      return false;
+    // We cannot push a negate through an AND operation (it would become an OR),
+    // we can however change a (not (or x y)) to (and (not x) (not y)) if we can
+    // push the negate through the x/y subtrees.
+    CanPushNegate = (Opcode == ISD::OR) && CanPushNegateL && CanPushNegateR;
+    return true;
+  }
+  return false;
+}
+
+/// Emit conjunction or disjunction tree with the CMP/FCMP followed by a chain
+/// of CCMP/CFCMP ops. See @ref AArch64CCMP.
+/// Tries to transform the given i1 producing node @p Val to a series compare
+/// and conditional compare operations. @returns an NZCV flags producing node
+/// and sets @p OutCC to the flags that should be tested or returns SDValue() if
+/// transformation was not possible.
+/// On recursive invocations @p PushNegate may be set to true to have negation
+/// effects pushed to the tree leafs; @p Predicate is an NZCV flag predicate
+/// for the comparisons in the current subtree; @p Depth limits the search
+/// depth to avoid stack overflow.
+static SDValue emitConjunctionDisjunctionTree(SelectionDAG &DAG, SDValue Val,
+    AArch64CC::CondCode &OutCC, bool PushNegate = false,
+    SDValue CCOp = SDValue(), AArch64CC::CondCode Predicate = AArch64CC::AL,
+    unsigned Depth = 0) {
+  // We're at a tree leaf, produce a conditional comparison operation.
+  unsigned Opcode = Val->getOpcode();
+  if (Opcode == ISD::SETCC) {
+    SDValue LHS = Val->getOperand(0);
+    SDValue RHS = Val->getOperand(1);
+    ISD::CondCode CC = cast<CondCodeSDNode>(Val->getOperand(2))->get();
+    bool isInteger = LHS.getValueType().isInteger();
+    if (PushNegate)
+      CC = getSetCCInverse(CC, isInteger);
+    SDLoc DL(Val);
+    // Determine OutCC and handle FP special case.
+    if (isInteger) {
+      OutCC = changeIntCCToAArch64CC(CC);
+    } else {
+      assert(LHS.getValueType().isFloatingPoint());
+      AArch64CC::CondCode ExtraCC;
+      changeFPCCToAArch64CC(CC, OutCC, ExtraCC);
+      // Surpisingly some floating point conditions can't be tested with a
+      // single condition code. Construct an additional comparison in this case.
+      // See comment below on how we deal with OR conditions.
+      if (ExtraCC != AArch64CC::AL) {
+        SDValue ExtraCmp;
+        if (!CCOp.getNode())
+          ExtraCmp = emitComparison(LHS, RHS, CC, DL, DAG);
+        else {
+          SDValue ConditionOp = DAG.getConstant(Predicate, DL, MVT_CC);
+          // Note that we want the inverse of ExtraCC, so NZCV is not inversed.
+          unsigned NZCV = AArch64CC::getNZCVToSatisfyCondCode(ExtraCC);
+          ExtraCmp = emitConditionalComparison(LHS, RHS, CC, CCOp, ConditionOp,
+                                               NZCV, DL, DAG);
+        }
+        CCOp = ExtraCmp;
+        Predicate = AArch64CC::getInvertedCondCode(ExtraCC);
+        OutCC = AArch64CC::getInvertedCondCode(OutCC);
+      }
+    }
+
+    // Produce a normal comparison if we are first in the chain
+    if (!CCOp.getNode())
+      return emitComparison(LHS, RHS, CC, DL, DAG);
+    // Otherwise produce a ccmp.
+    SDValue ConditionOp = DAG.getConstant(Predicate, DL, MVT_CC);
+    AArch64CC::CondCode InvOutCC = AArch64CC::getInvertedCondCode(OutCC);
+    unsigned NZCV = AArch64CC::getNZCVToSatisfyCondCode(InvOutCC);
+    return emitConditionalComparison(LHS, RHS, CC, CCOp, ConditionOp, NZCV, DL,
+                                     DAG);
+  } else if (Opcode != ISD::AND && Opcode != ISD::OR)
+    return SDValue();
+
+  assert((Opcode == ISD::OR || !PushNegate)
+         && "Can only push negate through OR operation");
+
+  // Check if both sides can be transformed.
+  SDValue LHS = Val->getOperand(0);
+  SDValue RHS = Val->getOperand(1);
+  bool CanPushNegateL;
+  if (!isConjunctionDisjunctionTree(LHS, CanPushNegateL, Depth+1))
+    return SDValue();
+  bool CanPushNegateR;
+  if (!isConjunctionDisjunctionTree(RHS, CanPushNegateR, Depth+1))
+    return SDValue();
+
+  // Do we need to negate our operands?
+  bool NegateOperands = Opcode == ISD::OR;
+  // We can negate the results of all previous operations by inverting the
+  // predicate flags giving us a free negation for one side. For the other side
+  // we need to be able to push the negation to the leafs of the tree.
+  if (NegateOperands) {
+    if (!CanPushNegateL && !CanPushNegateR)
+      return SDValue();
+    // Order the side where we can push the negate through to LHS.
+    if (!CanPushNegateL && CanPushNegateR) {
+      std::swap(LHS, RHS);
+      CanPushNegateL = true;
+    }
+  }
+
+  // Emit RHS. If we want to negate the tree we only need to push a negate
+  // through if we are already in a PushNegate case, otherwise we can negate
+  // the "flags to test" afterwards.
+  AArch64CC::CondCode RHSCC;
+  SDValue CmpR = emitConjunctionDisjunctionTree(DAG, RHS, RHSCC, PushNegate,
+                                                CCOp, Predicate, Depth+1);
+  if (NegateOperands && !PushNegate)
+    RHSCC = AArch64CC::getInvertedCondCode(RHSCC);
+  // Emit LHS. We must push the negate through if we need to negate it.
+  SDValue CmpL = emitConjunctionDisjunctionTree(DAG, LHS, OutCC, NegateOperands,
+                                                CmpR, RHSCC, Depth+1);
+  // If we transformed an OR to and AND then we have to negate the result
+  // (or absorb a PushNegate resulting in a double negation).
+  if (Opcode == ISD::OR && !PushNegate)
+    OutCC = AArch64CC::getInvertedCondCode(OutCC);
+  return CmpL;
+}
+
+/// @}
+
 static SDValue getAArch64Cmp(SDValue LHS, SDValue RHS, ISD::CondCode CC,
                              SDValue &AArch64cc, SelectionDAG &DAG, SDLoc dl) {
-  SDValue Cmp;
-  AArch64CC::CondCode AArch64CC;
   if (ConstantSDNode *RHSC = dyn_cast<ConstantSDNode>(RHS.getNode())) {
     EVT VT = RHS.getValueType();
     uint64_t C = RHSC->getZExtValue();
@@ -1170,7 +1404,7 @@ static SDValue getAArch64Cmp(SDValue LHS, SDValue RHS, ISD::CondCode CC,
              isLegalArithImmed(C - 1ULL))) {
           CC = (CC == ISD::SETLT) ? ISD::SETLE : ISD::SETGT;
           C = (VT == MVT::i32) ? (uint32_t)(C - 1) : C - 1;
-          RHS = DAG.getConstant(C, VT);
+          RHS = DAG.getConstant(C, dl, VT);
         }
         break;
       case ISD::SETULT:
@@ -1180,7 +1414,7 @@ static SDValue getAArch64Cmp(SDValue LHS, SDValue RHS, ISD::CondCode CC,
             (VT == MVT::i64 && C != 0ULL && isLegalArithImmed(C - 1ULL))) {
           CC = (CC == ISD::SETULT) ? ISD::SETULE : ISD::SETUGT;
           C = (VT == MVT::i32) ? (uint32_t)(C - 1) : C - 1;
-          RHS = DAG.getConstant(C, VT);
+          RHS = DAG.getConstant(C, dl, VT);
         }
         break;
       case ISD::SETLE:
@@ -1191,7 +1425,7 @@ static SDValue getAArch64Cmp(SDValue LHS, SDValue RHS, ISD::CondCode CC,
              isLegalArithImmed(C + 1ULL))) {
           CC = (CC == ISD::SETLE) ? ISD::SETLT : ISD::SETGE;
           C = (VT == MVT::i32) ? (uint32_t)(C + 1) : C + 1;
-          RHS = DAG.getConstant(C, VT);
+          RHS = DAG.getConstant(C, dl, VT);
         }
         break;
       case ISD::SETULE:
@@ -1202,52 +1436,62 @@ static SDValue getAArch64Cmp(SDValue LHS, SDValue RHS, ISD::CondCode CC,
              isLegalArithImmed(C + 1ULL))) {
           CC = (CC == ISD::SETULE) ? ISD::SETULT : ISD::SETUGE;
           C = (VT == MVT::i32) ? (uint32_t)(C + 1) : C + 1;
-          RHS = DAG.getConstant(C, VT);
+          RHS = DAG.getConstant(C, dl, VT);
         }
         break;
       }
     }
   }
-  // The imm operand of ADDS is an unsigned immediate, in the range 0 to 4095.
-  // For the i8 operand, the largest immediate is 255, so this can be easily
-  // encoded in the compare instruction. For the i16 operand, however, the
-  // largest immediate cannot be encoded in the compare.
-  // Therefore, use a sign extending load and cmn to avoid materializing the -1
-  // constant. For example,
-  // movz w1, #65535
-  // ldrh w0, [x0, #0]
-  // cmp w0, w1
-  // >
-  // ldrsh w0, [x0, #0]
-  // cmn w0, #1
-  // Fundamental, we're relying on the property that (zext LHS) == (zext RHS)
-  // if and only if (sext LHS) == (sext RHS). The checks are in place to ensure
-  // both the LHS and RHS are truely zero extended and to make sure the
-  // transformation is profitable.
+  SDValue Cmp;
+  AArch64CC::CondCode AArch64CC;
   if ((CC == ISD::SETEQ || CC == ISD::SETNE) && isa<ConstantSDNode>(RHS)) {
-    if ((cast<ConstantSDNode>(RHS)->getZExtValue() >> 16 == 0) &&
-        isa<LoadSDNode>(LHS)) {
-      if (cast<LoadSDNode>(LHS)->getExtensionType() == ISD::ZEXTLOAD &&
-          cast<LoadSDNode>(LHS)->getMemoryVT() == MVT::i16 &&
-          LHS.getNode()->hasNUsesOfValue(1, 0)) {
-        int16_t ValueofRHS = cast<ConstantSDNode>(RHS)->getZExtValue();
-        if (ValueofRHS < 0 && isLegalArithImmed(-ValueofRHS)) {
-          SDValue SExt =
-              DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, LHS.getValueType(), LHS,
-                          DAG.getValueType(MVT::i16));
-          Cmp = emitComparison(SExt,
-                               DAG.getConstant(ValueofRHS, RHS.getValueType()),
-                               CC, dl, DAG);
-          AArch64CC = changeIntCCToAArch64CC(CC);
-          AArch64cc = DAG.getConstant(AArch64CC, MVT::i32);
-          return Cmp;
-        }
+    const ConstantSDNode *RHSC = cast<ConstantSDNode>(RHS);
+
+    // The imm operand of ADDS is an unsigned immediate, in the range 0 to 4095.
+    // For the i8 operand, the largest immediate is 255, so this can be easily
+    // encoded in the compare instruction. For the i16 operand, however, the
+    // largest immediate cannot be encoded in the compare.
+    // Therefore, use a sign extending load and cmn to avoid materializing the
+    // -1 constant. For example,
+    // movz w1, #65535
+    // ldrh w0, [x0, #0]
+    // cmp w0, w1
+    // >
+    // ldrsh w0, [x0, #0]
+    // cmn w0, #1
+    // Fundamental, we're relying on the property that (zext LHS) == (zext RHS)
+    // if and only if (sext LHS) == (sext RHS). The checks are in place to
+    // ensure both the LHS and RHS are truely zero extended and to make sure the
+    // transformation is profitable.
+    if ((RHSC->getZExtValue() >> 16 == 0) && isa<LoadSDNode>(LHS) &&
+        cast<LoadSDNode>(LHS)->getExtensionType() == ISD::ZEXTLOAD &&
+        cast<LoadSDNode>(LHS)->getMemoryVT() == MVT::i16 &&
+        LHS.getNode()->hasNUsesOfValue(1, 0)) {
+      int16_t ValueofRHS = cast<ConstantSDNode>(RHS)->getZExtValue();
+      if (ValueofRHS < 0 && isLegalArithImmed(-ValueofRHS)) {
+        SDValue SExt =
+            DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, LHS.getValueType(), LHS,
+                        DAG.getValueType(MVT::i16));
+        Cmp = emitComparison(SExt, DAG.getConstant(ValueofRHS, dl,
+                                                   RHS.getValueType()),
+                             CC, dl, DAG);
+        AArch64CC = changeIntCCToAArch64CC(CC);
       }
     }
+
+    if (!Cmp && (RHSC->isNullValue() || RHSC->isOne())) {
+      if ((Cmp = emitConjunctionDisjunctionTree(DAG, LHS, AArch64CC))) {
+        if ((CC == ISD::SETNE) ^ RHSC->isNullValue())
+          AArch64CC = AArch64CC::getInvertedCondCode(AArch64CC);
+      }
+    }
+  }
+
+  if (!Cmp) {
+    Cmp = emitComparison(LHS, RHS, CC, dl, DAG);
+    AArch64CC = changeIntCCToAArch64CC(CC);
   }
-  Cmp = emitComparison(LHS, RHS, CC, dl, DAG);
-  AArch64CC = changeIntCCToAArch64CC(CC);
-  AArch64cc = DAG.getConstant(AArch64CC, MVT::i32);
+  AArch64cc = DAG.getConstant(AArch64CC, dl, MVT_CC);
   return Cmp;
 }
 
@@ -1294,7 +1538,7 @@ getAArch64XALUOOp(AArch64CC::CondCode &CC, SDValue Op, SelectionDAG &DAG) {
       RHS = DAG.getNode(ExtendOpc, DL, MVT::i64, RHS);
       SDValue Mul = DAG.getNode(ISD::MUL, DL, MVT::i64, LHS, RHS);
       SDValue Add = DAG.getNode(ISD::ADD, DL, MVT::i64, Mul,
-                                DAG.getConstant(0, MVT::i64));
+                                DAG.getConstant(0, DL, MVT::i64));
       // On AArch64 the upper 32 bits are always zero extended for a 32 bit
       // operation. We need to clear out the upper 32 bits, because we used a
       // widening multiply that wrote all 64 bits. In the end this should be a
@@ -1307,10 +1551,10 @@ getAArch64XALUOOp(AArch64CC::CondCode &CC, SDValue Op, SelectionDAG &DAG) {
         // check we have to arithmetic shift right the 32nd bit of the result by
         // 31 bits. Then we compare the result to the upper 32 bits.
         SDValue UpperBits = DAG.getNode(ISD::SRL, DL, MVT::i64, Add,
-                                        DAG.getConstant(32, MVT::i64));
+                                        DAG.getConstant(32, DL, MVT::i64));
         UpperBits = DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, UpperBits);
         SDValue LowerBits = DAG.getNode(ISD::SRA, DL, MVT::i32, Value,
-                                        DAG.getConstant(31, MVT::i64));
+                                        DAG.getConstant(31, DL, MVT::i64));
         // It is important that LowerBits is last, otherwise the arithmetic
         // shift will not be folded into the compare (SUBS).
         SDVTList VTs = DAG.getVTList(MVT::i32, MVT::i32);
@@ -1323,10 +1567,11 @@ getAArch64XALUOOp(AArch64CC::CondCode &CC, SDValue Op, SelectionDAG &DAG) {
         // pattern:
         // (i64 AArch64ISD::SUBS i64 0, (i64 srl i64 %Mul, i64 32)
         SDValue UpperBits = DAG.getNode(ISD::SRL, DL, MVT::i64, Mul,
-                                        DAG.getConstant(32, MVT::i64));
+                                        DAG.getConstant(32, DL, MVT::i64));
         SDVTList VTs = DAG.getVTList(MVT::i64, MVT::i32);
         Overflow =
-            DAG.getNode(AArch64ISD::SUBS, DL, VTs, DAG.getConstant(0, MVT::i64),
+            DAG.getNode(AArch64ISD::SUBS, DL, VTs,
+                        DAG.getConstant(0, DL, MVT::i64),
                         UpperBits).getValue(1);
       }
       break;
@@ -1337,7 +1582,7 @@ getAArch64XALUOOp(AArch64CC::CondCode &CC, SDValue Op, SelectionDAG &DAG) {
     if (IsSigned) {
       SDValue UpperBits = DAG.getNode(ISD::MULHS, DL, MVT::i64, LHS, RHS);
       SDValue LowerBits = DAG.getNode(ISD::SRA, DL, MVT::i64, Value,
-                                      DAG.getConstant(63, MVT::i64));
+                                      DAG.getConstant(63, DL, MVT::i64));
       // It is important that LowerBits is last, otherwise the arithmetic
       // shift will not be folded into the compare (SUBS).
       SDVTList VTs = DAG.getVTList(MVT::i64, MVT::i32);
@@ -1347,7 +1592,8 @@ getAArch64XALUOOp(AArch64CC::CondCode &CC, SDValue Op, SelectionDAG &DAG) {
       SDValue UpperBits = DAG.getNode(ISD::MULHU, DL, MVT::i64, LHS, RHS);
       SDVTList VTs = DAG.getVTList(MVT::i64, MVT::i32);
       Overflow =
-          DAG.getNode(AArch64ISD::SUBS, DL, VTs, DAG.getConstant(0, MVT::i64),
+          DAG.getNode(AArch64ISD::SUBS, DL, VTs,
+                      DAG.getConstant(0, DL, MVT::i64),
                       UpperBits).getValue(1);
     }
     break;
@@ -1402,7 +1648,7 @@ static SDValue LowerXOR(SDValue Op, SelectionDAG &DAG) {
   ConstantSDNode *CFVal = dyn_cast<ConstantSDNode>(FVal);
   ConstantSDNode *CTVal = dyn_cast<ConstantSDNode>(TVal);
 
-  // The the values aren't constants, this isn't the pattern we're looking for.
+  // The values aren't constants, this isn't the pattern we're looking for.
   if (!CFVal || !CTVal)
     return Op;
 
@@ -1421,7 +1667,7 @@ static SDValue LowerXOR(SDValue Op, SelectionDAG &DAG) {
 
     FVal = Other;
     TVal = DAG.getNode(ISD::XOR, dl, Other.getValueType(), Other,
-                       DAG.getConstant(-1ULL, Other.getValueType()));
+                       DAG.getConstant(-1ULL, dl, Other.getValueType()));
 
     return DAG.getNode(AArch64ISD::CSEL, dl, Sel.getValueType(), FVal, TVal,
                        CCVal, Cmp);
@@ -1471,24 +1717,25 @@ static SDValue LowerXALUO(SDValue Op, SelectionDAG &DAG) {
   if (!DAG.getTargetLoweringInfo().isTypeLegal(Op.getValueType()))
     return SDValue();
 
+  SDLoc dl(Op);
   AArch64CC::CondCode CC;
   // The actual operation that sets the overflow or carry flag.
   SDValue Value, Overflow;
   std::tie(Value, Overflow) = getAArch64XALUOOp(CC, Op, DAG);
 
   // We use 0 and 1 as false and true values.
-  SDValue TVal = DAG.getConstant(1, MVT::i32);
-  SDValue FVal = DAG.getConstant(0, MVT::i32);
+  SDValue TVal = DAG.getConstant(1, dl, MVT::i32);
+  SDValue FVal = DAG.getConstant(0, dl, MVT::i32);
 
   // We use an inverted condition, because the conditional select is inverted
   // too. This will allow it to be selected to a single instruction:
   // CSINC Wd, WZR, WZR, invert(cond).
-  SDValue CCVal = DAG.getConstant(getInvertedCondCode(CC), MVT::i32);
-  Overflow = DAG.getNode(AArch64ISD::CSEL, SDLoc(Op), MVT::i32, FVal, TVal,
+  SDValue CCVal = DAG.getConstant(getInvertedCondCode(CC), dl, MVT::i32);
+  Overflow = DAG.getNode(AArch64ISD::CSEL, dl, MVT::i32, FVal, TVal,
                          CCVal, Overflow);
 
   SDVTList VTs = DAG.getVTList(Op.getValueType(), MVT::i32);
-  return DAG.getNode(ISD::MERGE_VALUES, SDLoc(Op), VTs, Value, Overflow);
+  return DAG.getNode(ISD::MERGE_VALUES, dl, VTs, Value, Overflow);
 }
 
 // Prefetch operands are:
@@ -1519,7 +1766,7 @@ static SDValue LowerPREFETCH(SDValue Op, SelectionDAG &DAG) {
                    (Locality << 1) |    // Cache level bits
                    (unsigned)IsStream;  // Stream bit
   return DAG.getNode(AArch64ISD::PREFETCH, DL, MVT::Other, Op.getOperand(0),
-                     DAG.getConstant(PrfOp, MVT::i32), Op.getOperand(1));
+                     DAG.getConstant(PrfOp, DL, MVT::i32), Op.getOperand(1));
 }
 
 SDValue AArch64TargetLowering::LowerFP_EXTEND(SDValue Op,
@@ -1621,7 +1868,7 @@ static SDValue LowerVectorINT_TO_FP(SDValue Op, SelectionDAG &DAG) {
         MVT::getVectorVT(MVT::getFloatingPointVT(InVT.getScalarSizeInBits()),
                          InVT.getVectorNumElements());
     In = DAG.getNode(Op.getOpcode(), dl, CastVT, In);
-    return DAG.getNode(ISD::FP_ROUND, dl, VT, In, DAG.getIntPtrConstant(0));
+    return DAG.getNode(ISD::FP_ROUND, dl, VT, In, DAG.getIntPtrConstant(0, dl));
   }
 
   if (VT.getSizeInBits() > InVT.getSizeInBits()) {
@@ -1646,7 +1893,7 @@ SDValue AArch64TargetLowering::LowerINT_TO_FP(SDValue Op,
     return DAG.getNode(
         ISD::FP_ROUND, dl, MVT::f16,
         DAG.getNode(Op.getOpcode(), dl, MVT::f32, Op.getOperand(0)),
-        DAG.getIntPtrConstant(0));
+        DAG.getIntPtrConstant(0, dl));
   }
 
   // i128 conversions are libcalls.
@@ -1687,7 +1934,8 @@ SDValue AArch64TargetLowering::LowerFSINCOS(SDValue Op,
 
   const char *LibcallName =
       (ArgVT == MVT::f64) ? "__sincos_stret" : "__sincosf_stret";
-  SDValue Callee = DAG.getExternalSymbol(LibcallName, getPointerTy());
+  SDValue Callee =
+      DAG.getExternalSymbol(LibcallName, getPointerTy(DAG.getDataLayout()));
 
   StructType *RetTy = StructType::get(ArgTy, ArgTy, nullptr);
   TargetLowering::CallLoweringInfo CLI(DAG);
@@ -1709,7 +1957,7 @@ static SDValue LowerBITCAST(SDValue Op, SelectionDAG &DAG) {
   Op = DAG.getNode(ISD::BITCAST, DL, MVT::f32, Op);
   return SDValue(
       DAG.getMachineNode(TargetOpcode::EXTRACT_SUBREG, DL, MVT::f16, Op,
-                         DAG.getTargetConstant(AArch64::hsub, MVT::i32)),
+                         DAG.getTargetConstant(AArch64::hsub, DL, MVT::i32)),
       0);
 }
 
@@ -1754,8 +2002,7 @@ static bool isExtendedBUILD_VECTOR(SDNode *N, SelectionDAG &DAG,
   if (N->getOpcode() != ISD::BUILD_VECTOR)
     return false;
 
-  for (unsigned i = 0, e = N->getNumOperands(); i != e; ++i) {
-    SDNode *Elt = N->getOperand(i).getNode();
+  for (const SDValue &Elt : N->op_values()) {
     if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Elt)) {
       unsigned EltSize = VT.getVectorElementType().getSizeInBits();
       unsigned HalfSize = EltSize / 2;
@@ -1783,6 +2030,7 @@ static SDValue skipExtensionForVectorMULL(SDNode *N, SelectionDAG &DAG) {
 
   assert(N->getOpcode() == ISD::BUILD_VECTOR && "expected BUILD_VECTOR");
   EVT VT = N->getValueType(0);
+  SDLoc dl(N);
   unsigned EltSize = VT.getVectorElementType().getSizeInBits() / 2;
   unsigned NumElts = VT.getVectorNumElements();
   MVT TruncVT = MVT::getIntegerVT(EltSize);
@@ -1792,9 +2040,9 @@ static SDValue skipExtensionForVectorMULL(SDNode *N, SelectionDAG &DAG) {
     const APInt &CInt = C->getAPIntValue();
     // Element types smaller than 32 bits are not legal, so use i32 elements.
     // The values are implicitly truncated so sext vs. zext doesn't matter.
-    Ops.push_back(DAG.getConstant(CInt.zextOrTrunc(32), MVT::i32));
+    Ops.push_back(DAG.getConstant(CInt.zextOrTrunc(32), dl, MVT::i32));
   }
-  return DAG.getNode(ISD::BUILD_VECTOR, SDLoc(N),
+  return DAG.getNode(ISD::BUILD_VECTOR, dl,
                      MVT::getVectorVT(TruncVT, NumElts), Ops);
 }
 
@@ -1905,6 +2153,19 @@ static SDValue LowerMUL(SDValue Op, SelectionDAG &DAG) {
                                DAG.getNode(ISD::BITCAST, DL, Op1VT, N01), Op1));
 }
 
+SDValue AArch64TargetLowering::LowerINTRINSIC_WO_CHAIN(SDValue Op,
+                                                     SelectionDAG &DAG) const {
+  unsigned IntNo = cast<ConstantSDNode>(Op.getOperand(0))->getZExtValue();
+  SDLoc dl(Op);
+  switch (IntNo) {
+  default: return SDValue();    // Don't custom lower most intrinsics.
+  case Intrinsic::aarch64_thread_pointer: {
+    EVT PtrVT = getPointerTy(DAG.getDataLayout());
+    return DAG.getNode(AArch64ISD::THREAD_POINTER, dl, PtrVT);
+  }
+  }
+}
+
 SDValue AArch64TargetLowering::LowerOperation(SDValue Op,
                                               SelectionDAG &DAG) const {
   switch (Op.getOpcode()) {
@@ -2006,6 +2267,8 @@ SDValue AArch64TargetLowering::LowerOperation(SDValue Op,
     return LowerFSINCOS(Op, DAG);
   case ISD::MUL:
     return LowerMUL(Op, DAG);
+  case ISD::INTRINSIC_WO_CHAIN:
+    return LowerINTRINSIC_WO_CHAIN(Op, DAG);
   }
 }
 
@@ -2066,7 +2329,8 @@ SDValue AArch64TargetLowering::LowerFormalArguments(
       CurArgIdx = Ins[i].getOrigArgIndex();
 
       // Get type of the original argument.
-      EVT ActualVT = getValueType(CurOrigArg->getType(), /*AllowUnknown*/ true);
+      EVT ActualVT = getValueType(DAG.getDataLayout(), CurOrigArg->getType(),
+                                  /*AllowUnknown*/ true);
       MVT ActualMVT = ActualVT.isSimple() ? ActualVT.getSimpleVT() : MVT::Other;
       // If ActualMVT is i1/i8/i16, we should set LocVT to i8/i8/i16.
       if (ActualMVT == MVT::i1 || ActualMVT == MVT::i8)
@@ -2088,7 +2352,7 @@ SDValue AArch64TargetLowering::LowerFormalArguments(
     if (Ins[i].Flags.isByVal()) {
       // Byval is used for HFAs in the PCS, but the system should work in a
       // non-compliant manner for larger structs.
-      EVT PtrTy = getPointerTy();
+      EVT PtrVT = getPointerTy(DAG.getDataLayout());
       int Size = Ins[i].Flags.getByValSize();
       unsigned NumRegs = (Size + 7) / 8;
 
@@ -2096,7 +2360,7 @@ SDValue AArch64TargetLowering::LowerFormalArguments(
       // case. It should also work for fundamental types too.
       unsigned FrameIdx =
         MFI->CreateFixedObject(8 * NumRegs, VA.getLocMemOffset(), false);
-      SDValue FrameIdxN = DAG.getFrameIndex(FrameIdx, PtrTy);
+      SDValue FrameIdxN = DAG.getFrameIndex(FrameIdx, PtrVT);
       InVals.push_back(FrameIdxN);
 
       continue;
@@ -2163,7 +2427,7 @@ SDValue AArch64TargetLowering::LowerFormalArguments(
       int FI = MFI->CreateFixedObject(ArgSize, ArgOffset + BEAlign, true);
 
       // Create load nodes to retrieve arguments from the stack.
-      SDValue FIN = DAG.getFrameIndex(FI, getPointerTy());
+      SDValue FIN = DAG.getFrameIndex(FI, getPointerTy(DAG.getDataLayout()));
       SDValue ArgValue;
 
       // For NON_EXTLOAD, generic code in getLoad assert(ValVT == MemVT)
@@ -2242,6 +2506,7 @@ void AArch64TargetLowering::saveVarArgRegisters(CCState &CCInfo,
   MachineFunction &MF = DAG.getMachineFunction();
   MachineFrameInfo *MFI = MF.getFrameInfo();
   AArch64FunctionInfo *FuncInfo = MF.getInfo<AArch64FunctionInfo>();
+  auto PtrVT = getPointerTy(DAG.getDataLayout());
 
   SmallVector<SDValue, 8> MemOps;
 
@@ -2256,7 +2521,7 @@ void AArch64TargetLowering::saveVarArgRegisters(CCState &CCInfo,
   if (GPRSaveSize != 0) {
     GPRIdx = MFI->CreateStackObject(GPRSaveSize, 8, false);
 
-    SDValue FIN = DAG.getFrameIndex(GPRIdx, getPointerTy());
+    SDValue FIN = DAG.getFrameIndex(GPRIdx, PtrVT);
 
     for (unsigned i = FirstVariadicGPR; i < NumGPRArgRegs; ++i) {
       unsigned VReg = MF.addLiveIn(GPRArgRegs[i], &AArch64::GPR64RegClass);
@@ -2265,8 +2530,8 @@ void AArch64TargetLowering::saveVarArgRegisters(CCState &CCInfo,
           DAG.getStore(Val.getValue(1), DL, Val, FIN,
                        MachinePointerInfo::getStack(i * 8), false, false, 0);
       MemOps.push_back(Store);
-      FIN = DAG.getNode(ISD::ADD, DL, getPointerTy(), FIN,
-                        DAG.getConstant(8, getPointerTy()));
+      FIN =
+          DAG.getNode(ISD::ADD, DL, PtrVT, FIN, DAG.getConstant(8, DL, PtrVT));
     }
   }
   FuncInfo->setVarArgsGPRIndex(GPRIdx);
@@ -2284,7 +2549,7 @@ void AArch64TargetLowering::saveVarArgRegisters(CCState &CCInfo,
     if (FPRSaveSize != 0) {
       FPRIdx = MFI->CreateStackObject(FPRSaveSize, 16, false);
 
-      SDValue FIN = DAG.getFrameIndex(FPRIdx, getPointerTy());
+      SDValue FIN = DAG.getFrameIndex(FPRIdx, PtrVT);
 
       for (unsigned i = FirstVariadicFPR; i < NumFPRArgRegs; ++i) {
         unsigned VReg = MF.addLiveIn(FPRArgRegs[i], &AArch64::FPR128RegClass);
@@ -2294,8 +2559,8 @@ void AArch64TargetLowering::saveVarArgRegisters(CCState &CCInfo,
             DAG.getStore(Val.getValue(1), DL, Val, FIN,
                          MachinePointerInfo::getStack(i * 16), false, false, 0);
         MemOps.push_back(Store);
-        FIN = DAG.getNode(ISD::ADD, DL, getPointerTy(), FIN,
-                          DAG.getConstant(16, getPointerTy()));
+        FIN = DAG.getNode(ISD::ADD, DL, PtrVT, FIN,
+                          DAG.getConstant(16, DL, PtrVT));
       }
     }
     FuncInfo->setVarArgsFPRIndex(FPRIdx);
@@ -2398,7 +2663,7 @@ bool AArch64TargetLowering::isEligibleForTailCallOptimization(
   // cannot rely on the linker replacing the tail call with a return.
   if (GlobalAddressSDNode *G = dyn_cast<GlobalAddressSDNode>(Callee)) {
     const GlobalValue *GV = G->getGlobal();
-    const Triple TT(getTargetMachine().getTargetTriple());
+    const Triple &TT = getTargetMachine().getTargetTriple();
     if (GV->hasExternalWeakLinkage() &&
         (!TT.isOSWindows() || TT.isOSBinFormatELF() || TT.isOSBinFormatMachO()))
       return false;
@@ -2591,7 +2856,8 @@ AArch64TargetLowering::LowerCall(CallLoweringInfo &CLI,
     for (unsigned i = 0; i != NumArgs; ++i) {
       MVT ValVT = Outs[i].VT;
       // Get type of the original argument.
-      EVT ActualVT = getValueType(CLI.getArgs()[Outs[i].OrigArgIndex].Ty,
+      EVT ActualVT = getValueType(DAG.getDataLayout(),
+                                  CLI.getArgs()[Outs[i].OrigArgIndex].Ty,
                                   /*AllowUnknown*/ true);
       MVT ActualMVT = ActualVT.isSimple() ? ActualVT.getSimpleVT() : ValVT;
       ISD::ArgFlagsTy ArgFlags = Outs[i].Flags;
@@ -2647,13 +2913,16 @@ AArch64TargetLowering::LowerCall(CallLoweringInfo &CLI,
   // Adjust the stack pointer for the new arguments...
   // These operations are automatically eliminated by the prolog/epilog pass
   if (!IsSibCall)
-    Chain =
-        DAG.getCALLSEQ_START(Chain, DAG.getIntPtrConstant(NumBytes, true), DL);
+    Chain = DAG.getCALLSEQ_START(Chain, DAG.getIntPtrConstant(NumBytes, DL,
+                                                              true),
+                                 DL);
 
-  SDValue StackPtr = DAG.getCopyFromReg(Chain, DL, AArch64::SP, getPointerTy());
+  SDValue StackPtr = DAG.getCopyFromReg(Chain, DL, AArch64::SP,
+                                        getPointerTy(DAG.getDataLayout()));
 
   SmallVector<std::pair<unsigned, SDValue>, 8> RegsToPass;
   SmallVector<SDValue, 8> MemOpChains;
+  auto PtrVT = getPointerTy(DAG.getDataLayout());
 
   // Walk the register/memloc assignments, inserting copies/loads.
   for (unsigned i = 0, realArgIdx = 0, e = ArgLocs.size(); i != e;
@@ -2718,14 +2987,14 @@ AArch64TargetLowering::LowerCall(CallLoweringInfo &CLI,
       }
       unsigned LocMemOffset = VA.getLocMemOffset();
       int32_t Offset = LocMemOffset + BEAlign;
-      SDValue PtrOff = DAG.getIntPtrConstant(Offset);
-      PtrOff = DAG.getNode(ISD::ADD, DL, getPointerTy(), StackPtr, PtrOff);
+      SDValue PtrOff = DAG.getIntPtrConstant(Offset, DL);
+      PtrOff = DAG.getNode(ISD::ADD, DL, PtrVT, StackPtr, PtrOff);
 
       if (IsTailCall) {
         Offset = Offset + FPDiff;
         int FI = MF.getFrameInfo()->CreateFixedObject(OpSize, Offset, true);
 
-        DstAddr = DAG.getFrameIndex(FI, getPointerTy());
+        DstAddr = DAG.getFrameIndex(FI, PtrVT);
         DstInfo = MachinePointerInfo::getFixedStack(FI);
 
         // Make sure any stack arguments overlapping with where we're storing
@@ -2733,15 +3002,15 @@ AArch64TargetLowering::LowerCall(CallLoweringInfo &CLI,
         // clobbered.
         Chain = addTokenForArgument(Chain, DAG, MF.getFrameInfo(), FI);
       } else {
-        SDValue PtrOff = DAG.getIntPtrConstant(Offset);
+        SDValue PtrOff = DAG.getIntPtrConstant(Offset, DL);
 
-        DstAddr = DAG.getNode(ISD::ADD, DL, getPointerTy(), StackPtr, PtrOff);
+        DstAddr = DAG.getNode(ISD::ADD, DL, PtrVT, StackPtr, PtrOff);
         DstInfo = MachinePointerInfo::getStack(LocMemOffset);
       }
 
       if (Outs[i].Flags.isByVal()) {
         SDValue SizeNode =
-            DAG.getConstant(Outs[i].Flags.getByValSize(), MVT::i64);
+            DAG.getConstant(Outs[i].Flags.getByValSize(), DL, MVT::i64);
         SDValue Cpy = DAG.getMemcpy(
             Chain, DL, DstAddr, Arg, SizeNode, Outs[i].Flags.getByValAlign(),
             /*isVol = */ false, /*AlwaysInline = */ false,
@@ -2785,25 +3054,24 @@ AArch64TargetLowering::LowerCall(CallLoweringInfo &CLI,
       const GlobalValue *GV = G->getGlobal();
       bool InternalLinkage = GV->hasInternalLinkage();
       if (InternalLinkage)
-        Callee = DAG.getTargetGlobalAddress(GV, DL, getPointerTy(), 0, 0);
+        Callee = DAG.getTargetGlobalAddress(GV, DL, PtrVT, 0, 0);
       else {
-        Callee = DAG.getTargetGlobalAddress(GV, DL, getPointerTy(), 0,
-                                            AArch64II::MO_GOT);
-        Callee = DAG.getNode(AArch64ISD::LOADgot, DL, getPointerTy(), Callee);
+        Callee =
+            DAG.getTargetGlobalAddress(GV, DL, PtrVT, 0, AArch64II::MO_GOT);
+        Callee = DAG.getNode(AArch64ISD::LOADgot, DL, PtrVT, Callee);
       }
     } else if (ExternalSymbolSDNode *S =
                    dyn_cast<ExternalSymbolSDNode>(Callee)) {
       const char *Sym = S->getSymbol();
-      Callee =
-          DAG.getTargetExternalSymbol(Sym, getPointerTy(), AArch64II::MO_GOT);
-      Callee = DAG.getNode(AArch64ISD::LOADgot, DL, getPointerTy(), Callee);
+      Callee = DAG.getTargetExternalSymbol(Sym, PtrVT, AArch64II::MO_GOT);
+      Callee = DAG.getNode(AArch64ISD::LOADgot, DL, PtrVT, Callee);
     }
   } else if (GlobalAddressSDNode *G = dyn_cast<GlobalAddressSDNode>(Callee)) {
     const GlobalValue *GV = G->getGlobal();
-    Callee = DAG.getTargetGlobalAddress(GV, DL, getPointerTy(), 0, 0);
+    Callee = DAG.getTargetGlobalAddress(GV, DL, PtrVT, 0, 0);
   } else if (ExternalSymbolSDNode *S = dyn_cast<ExternalSymbolSDNode>(Callee)) {
     const char *Sym = S->getSymbol();
-    Callee = DAG.getTargetExternalSymbol(Sym, getPointerTy(), 0);
+    Callee = DAG.getTargetExternalSymbol(Sym, PtrVT, 0);
   }
 
   // We don't usually want to end the call-sequence here because we would tidy
@@ -2811,8 +3079,8 @@ AArch64TargetLowering::LowerCall(CallLoweringInfo &CLI,
   // we've carefully laid out the parameters so that when sp is reset they'll be
   // in the correct location.
   if (IsTailCall && !IsSibCall) {
-    Chain = DAG.getCALLSEQ_END(Chain, DAG.getIntPtrConstant(NumBytes, true),
-                               DAG.getIntPtrConstant(0, true), InFlag, DL);
+    Chain = DAG.getCALLSEQ_END(Chain, DAG.getIntPtrConstant(NumBytes, DL, true),
+                               DAG.getIntPtrConstant(0, DL, true), InFlag, DL);
     InFlag = Chain.getValue(1);
   }
 
@@ -2824,7 +3092,7 @@ AArch64TargetLowering::LowerCall(CallLoweringInfo &CLI,
     // Each tail call may have to adjust the stack by a different amount, so
     // this information must travel along with the operation for eventual
     // consumption by emitEpilogue.
-    Ops.push_back(DAG.getTargetConstant(FPDiff, MVT::i32));
+    Ops.push_back(DAG.getTargetConstant(FPDiff, DL, MVT::i32));
   }
 
   // Add argument registers to the end of the list so that they are known live
@@ -2856,8 +3124,10 @@ AArch64TargetLowering::LowerCall(CallLoweringInfo &CLI,
 
   // If we're doing a tall call, use a TC_RETURN here rather than an
   // actual call instruction.
-  if (IsTailCall)
+  if (IsTailCall) {
+    MF.getFrameInfo()->setHasTailCall();
     return DAG.getNode(AArch64ISD::TC_RETURN, DL, NodeTys, Ops);
+  }
 
   // Returns a chain and a flag for retval copy to use.
   Chain = DAG.getNode(AArch64ISD::CALL, DL, NodeTys, Ops);
@@ -2867,8 +3137,8 @@ AArch64TargetLowering::LowerCall(CallLoweringInfo &CLI,
                                 ? RoundUpToAlignment(NumBytes, 16)
                                 : 0;
 
-  Chain = DAG.getCALLSEQ_END(Chain, DAG.getIntPtrConstant(NumBytes, true),
-                             DAG.getIntPtrConstant(CalleePopBytes, true),
+  Chain = DAG.getCALLSEQ_END(Chain, DAG.getIntPtrConstant(NumBytes, DL, true),
+                             DAG.getIntPtrConstant(CalleePopBytes, DL, true),
                              InFlag, DL);
   if (!Ins.empty())
     InFlag = Chain.getValue(1);
@@ -2951,7 +3221,7 @@ AArch64TargetLowering::LowerReturn(SDValue Chain, CallingConv::ID CallConv,
 
 SDValue AArch64TargetLowering::LowerGlobalAddress(SDValue Op,
                                                   SelectionDAG &DAG) const {
-  EVT PtrVT = getPointerTy();
+  EVT PtrVT = getPointerTy(DAG.getDataLayout());
   SDLoc DL(Op);
   const GlobalAddressSDNode *GN = cast<GlobalAddressSDNode>(Op);
   const GlobalValue *GV = GN->getGlobal();
@@ -2984,7 +3254,7 @@ SDValue AArch64TargetLowering::LowerGlobalAddress(SDValue Op,
                                      /*isInvariant=*/ true, 8);
     if (GN->getOffset() != 0)
       return DAG.getNode(ISD::ADD, DL, PtrVT, GlobalAddr,
-                         DAG.getConstant(GN->getOffset(), PtrVT));
+                         DAG.getConstant(GN->getOffset(), DL, PtrVT));
     return GlobalAddr;
   }
 
@@ -3043,7 +3313,7 @@ AArch64TargetLowering::LowerDarwinGlobalTLSAddress(SDValue Op,
   assert(Subtarget->isTargetDarwin() && "TLS only supported on Darwin");
 
   SDLoc DL(Op);
-  MVT PtrVT = getPointerTy();
+  MVT PtrVT = getPointerTy(DAG.getDataLayout());
   const GlobalValue *GV = cast<GlobalAddressSDNode>(Op)->getGlobal();
 
   SDValue TLVPAddr =
@@ -3098,7 +3368,7 @@ AArch64TargetLowering::LowerDarwinGlobalTLSAddress(SDValue Op,
 ///  the sequence is produced as per above.
 SDValue AArch64TargetLowering::LowerELFTLSDescCallSeq(SDValue SymAddr, SDLoc DL,
                                                       SelectionDAG &DAG) const {
-  EVT PtrVT = getPointerTy();
+  EVT PtrVT = getPointerTy(DAG.getDataLayout());
 
   SDValue Chain = DAG.getEntryNode();
   SDVTList NodeTys = DAG.getVTList(MVT::Other, MVT::Glue);
@@ -3127,13 +3397,17 @@ AArch64TargetLowering::LowerELFGlobalTLSAddress(SDValue Op,
   const GlobalAddressSDNode *GA = cast<GlobalAddressSDNode>(Op);
 
   TLSModel::Model Model = getTargetMachine().getTLSModel(GA->getGlobal());
+
+  if (DAG.getTarget().Options.EmulatedTLS)
+    return LowerToTLSEmulatedModel(GA, DAG);
+
   if (!EnableAArch64ELFLocalDynamicTLSGeneration) {
     if (Model == TLSModel::LocalDynamic)
       Model = TLSModel::GeneralDynamic;
   }
 
   SDValue TPOff;
-  EVT PtrVT = getPointerTy();
+  EVT PtrVT = getPointerTy(DAG.getDataLayout());
   SDLoc DL(Op);
   const GlobalValue *GV = GA->getGlobal();
 
@@ -3148,11 +3422,13 @@ AArch64TargetLowering::LowerELFGlobalTLSAddress(SDValue Op,
 
     SDValue TPWithOff_lo =
         SDValue(DAG.getMachineNode(AArch64::ADDXri, DL, PtrVT, ThreadBase,
-                                   HiVar, DAG.getTargetConstant(0, MVT::i32)),
+                                   HiVar,
+                                   DAG.getTargetConstant(0, DL, MVT::i32)),
                 0);
     SDValue TPWithOff =
         SDValue(DAG.getMachineNode(AArch64::ADDXri, DL, PtrVT, TPWithOff_lo,
-                                   LoVar, DAG.getTargetConstant(0, MVT::i32)),
+                                   LoVar,
+                                   DAG.getTargetConstant(0, DL, MVT::i32)),
                 0);
     return TPWithOff;
   } else if (Model == TLSModel::InitialExec) {
@@ -3188,10 +3464,10 @@ AArch64TargetLowering::LowerELFGlobalTLSAddress(SDValue Op,
         AArch64II::MO_TLS | AArch64II::MO_PAGEOFF | AArch64II::MO_NC);
 
     TPOff = SDValue(DAG.getMachineNode(AArch64::ADDXri, DL, PtrVT, TPOff, HiVar,
-                                       DAG.getTargetConstant(0, MVT::i32)),
+                                       DAG.getTargetConstant(0, DL, MVT::i32)),
                     0);
     TPOff = SDValue(DAG.getMachineNode(AArch64::ADDXri, DL, PtrVT, TPOff, LoVar,
-                                       DAG.getTargetConstant(0, MVT::i32)),
+                                       DAG.getTargetConstant(0, DL, MVT::i32)),
                     0);
   } else if (Model == TLSModel::GeneralDynamic) {
     // The call needs a relocation too for linker relaxation. It doesn't make
@@ -3234,7 +3510,7 @@ SDValue AArch64TargetLowering::LowerBR_CC(SDValue Op, SelectionDAG &DAG) const {
     // If softenSetCCOperands returned a scalar, we need to compare the result
     // against zero to select between true and false values.
     if (!RHS.getNode()) {
-      RHS = DAG.getConstant(0, LHS.getValueType());
+      RHS = DAG.getConstant(0, dl, LHS.getValueType());
       CC = ISD::SETNE;
     }
   }
@@ -3259,7 +3535,7 @@ SDValue AArch64TargetLowering::LowerBR_CC(SDValue Op, SelectionDAG &DAG) const {
 
     if (CC == ISD::SETNE)
       OFCC = getInvertedCondCode(OFCC);
-    SDValue CCVal = DAG.getConstant(OFCC, MVT::i32);
+    SDValue CCVal = DAG.getConstant(OFCC, dl, MVT::i32);
 
     return DAG.getNode(AArch64ISD::BRCOND, dl, MVT::Other, Chain, Dest, CCVal,
                        Overflow);
@@ -3284,7 +3560,8 @@ SDValue AArch64TargetLowering::LowerBR_CC(SDValue Op, SelectionDAG &DAG) const {
           SDValue Test = LHS.getOperand(0);
           uint64_t Mask = LHS.getConstantOperandVal(1);
           return DAG.getNode(AArch64ISD::TBZ, dl, MVT::Other, Chain, Test,
-                             DAG.getConstant(Log2_64(Mask), MVT::i64), Dest);
+                             DAG.getConstant(Log2_64(Mask), dl, MVT::i64),
+                             Dest);
         }
 
         return DAG.getNode(AArch64ISD::CBZ, dl, MVT::Other, Chain, LHS, Dest);
@@ -3299,7 +3576,8 @@ SDValue AArch64TargetLowering::LowerBR_CC(SDValue Op, SelectionDAG &DAG) const {
           SDValue Test = LHS.getOperand(0);
           uint64_t Mask = LHS.getConstantOperandVal(1);
           return DAG.getNode(AArch64ISD::TBNZ, dl, MVT::Other, Chain, Test,
-                             DAG.getConstant(Log2_64(Mask), MVT::i64), Dest);
+                             DAG.getConstant(Log2_64(Mask), dl, MVT::i64),
+                             Dest);
         }
 
         return DAG.getNode(AArch64ISD::CBNZ, dl, MVT::Other, Chain, LHS, Dest);
@@ -3309,7 +3587,7 @@ SDValue AArch64TargetLowering::LowerBR_CC(SDValue Op, SelectionDAG &DAG) const {
         // becomes redundant.  This would also increase register pressure.
         uint64_t Mask = LHS.getValueType().getSizeInBits() - 1;
         return DAG.getNode(AArch64ISD::TBNZ, dl, MVT::Other, Chain, LHS,
-                           DAG.getConstant(Mask, MVT::i64), Dest);
+                           DAG.getConstant(Mask, dl, MVT::i64), Dest);
       }
     }
     if (RHSC && RHSC->getSExtValue() == -1 && CC == ISD::SETGT &&
@@ -3319,7 +3597,7 @@ SDValue AArch64TargetLowering::LowerBR_CC(SDValue Op, SelectionDAG &DAG) const {
       // becomes redundant.  This would also increase register pressure.
       uint64_t Mask = LHS.getValueType().getSizeInBits() - 1;
       return DAG.getNode(AArch64ISD::TBZ, dl, MVT::Other, Chain, LHS,
-                         DAG.getConstant(Mask, MVT::i64), Dest);
+                         DAG.getConstant(Mask, dl, MVT::i64), Dest);
     }
 
     SDValue CCVal;
@@ -3335,11 +3613,11 @@ SDValue AArch64TargetLowering::LowerBR_CC(SDValue Op, SelectionDAG &DAG) const {
   SDValue Cmp = emitComparison(LHS, RHS, CC, dl, DAG);
   AArch64CC::CondCode CC1, CC2;
   changeFPCCToAArch64CC(CC, CC1, CC2);
-  SDValue CC1Val = DAG.getConstant(CC1, MVT::i32);
+  SDValue CC1Val = DAG.getConstant(CC1, dl, MVT::i32);
   SDValue BR1 =
       DAG.getNode(AArch64ISD::BRCOND, dl, MVT::Other, Chain, Dest, CC1Val, Cmp);
   if (CC2 != AArch64CC::AL) {
-    SDValue CC2Val = DAG.getConstant(CC2, MVT::i32);
+    SDValue CC2Val = DAG.getConstant(CC2, dl, MVT::i32);
     return DAG.getNode(AArch64ISD::BRCOND, dl, MVT::Other, BR1, Dest, CC2Val,
                        Cmp);
   }
@@ -3359,7 +3637,8 @@ SDValue AArch64TargetLowering::LowerFCOPYSIGN(SDValue Op,
     if (SrcVT == MVT::f32 && VT == MVT::f64)
       In2 = DAG.getNode(ISD::FP_EXTEND, DL, VT, In2);
     else if (SrcVT == MVT::f64 && VT == MVT::f32)
-      In2 = DAG.getNode(ISD::FP_ROUND, DL, VT, In2, DAG.getIntPtrConstant(0));
+      In2 = DAG.getNode(ISD::FP_ROUND, DL, VT, In2,
+                        DAG.getIntPtrConstant(0, DL));
     else
       // FIXME: Src type is different, bail out for now. Can VT really be a
       // vector type?
@@ -3388,7 +3667,7 @@ SDValue AArch64TargetLowering::LowerFCOPYSIGN(SDValue Op,
     EltVT = MVT::i64;
     VecVT = MVT::v2i64;
 
-    // We want to materialize a mask with the the high bit set, but the AdvSIMD
+    // We want to materialize a mask with the high bit set, but the AdvSIMD
     // immediate moves cannot materialize that in a single instruction for
     // 64-bit elements. Instead, materialize zero and then negate it.
     EltMask = 0;
@@ -3406,7 +3685,7 @@ SDValue AArch64TargetLowering::LowerFCOPYSIGN(SDValue Op,
     llvm_unreachable("Invalid type for copysign!");
   }
 
-  SDValue BuildVec = DAG.getConstant(EltMask, VecVT);
+  SDValue BuildVec = DAG.getConstant(EltMask, DL, VecVT);
 
   // If we couldn't materialize the mask above, then the mask vector will be
   // the zero vector, and we need to negate it here.
@@ -3454,7 +3733,7 @@ SDValue AArch64TargetLowering::LowerCTPOP(SDValue Op, SelectionDAG &DAG) const {
   SDValue CtPop = DAG.getNode(ISD::CTPOP, DL, MVT::v8i8, Val);
   SDValue UaddLV = DAG.getNode(
       ISD::INTRINSIC_WO_CHAIN, DL, MVT::i32,
-      DAG.getConstant(Intrinsic::aarch64_neon_uaddlv, MVT::i32), CtPop);
+      DAG.getConstant(Intrinsic::aarch64_neon_uaddlv, DL, MVT::i32), CtPop);
 
   if (VT == MVT::i64)
     UaddLV = DAG.getNode(ISD::ZERO_EXTEND, DL, MVT::i64, UaddLV);
@@ -3473,8 +3752,8 @@ SDValue AArch64TargetLowering::LowerSETCC(SDValue Op, SelectionDAG &DAG) const {
 
   // We chose ZeroOrOneBooleanContents, so use zero and one.
   EVT VT = Op.getValueType();
-  SDValue TVal = DAG.getConstant(1, VT);
-  SDValue FVal = DAG.getConstant(0, VT);
+  SDValue TVal = DAG.getConstant(1, dl, VT);
+  SDValue FVal = DAG.getConstant(0, dl, VT);
 
   // Handle f128 first, since one possible outcome is a normal integer
   // comparison which gets picked up by the next if statement.
@@ -3511,7 +3790,7 @@ SDValue AArch64TargetLowering::LowerSETCC(SDValue Op, SelectionDAG &DAG) const {
   changeFPCCToAArch64CC(CC, CC1, CC2);
   if (CC2 == AArch64CC::AL) {
     changeFPCCToAArch64CC(ISD::getSetCCInverse(CC, false), CC1, CC2);
-    SDValue CC1Val = DAG.getConstant(CC1, MVT::i32);
+    SDValue CC1Val = DAG.getConstant(CC1, dl, MVT::i32);
 
     // Note that we inverted the condition above, so we reverse the order of
     // the true and false operands here.  This will allow the setcc to be
@@ -3524,11 +3803,11 @@ SDValue AArch64TargetLowering::LowerSETCC(SDValue Op, SelectionDAG &DAG) const {
     // of the first as the RHS.  We're effectively OR'ing the two CC's together.
 
     // FIXME: It would be nice if we could match the two CSELs to two CSINCs.
-    SDValue CC1Val = DAG.getConstant(CC1, MVT::i32);
+    SDValue CC1Val = DAG.getConstant(CC1, dl, MVT::i32);
     SDValue CS1 =
         DAG.getNode(AArch64ISD::CSEL, dl, VT, TVal, FVal, CC1Val, Cmp);
 
-    SDValue CC2Val = DAG.getConstant(CC2, MVT::i32);
+    SDValue CC2Val = DAG.getConstant(CC2, dl, MVT::i32);
     return DAG.getNode(AArch64ISD::CSEL, dl, VT, TVal, CS1, CC2Val, Cmp);
   }
 }
@@ -3543,7 +3822,8 @@ SDValue AArch64TargetLowering::LowerSETCC(SDValue Op, SelectionDAG &DAG) const {
 /// operations would *not* be semantically equivalent.
 static bool selectCCOpsAreFMaxCompatible(SDValue Cmp, SDValue Result) {
   if (Cmp == Result)
-    return true;
+    return (Cmp.getValueType() == MVT::f32 ||
+            Cmp.getValueType() == MVT::f64);
 
   ConstantFPSDNode *CCmp = dyn_cast<ConstantFPSDNode>(Cmp);
   ConstantFPSDNode *CResult = dyn_cast<ConstantFPSDNode>(Result);
@@ -3570,7 +3850,7 @@ SDValue AArch64TargetLowering::LowerSELECT_CC(ISD::CondCode CC, SDValue LHS,
     // If softenSetCCOperands returned a scalar, we need to compare the result
     // against zero to select between true and false values.
     if (!RHS.getNode()) {
-      RHS = DAG.getConstant(0, LHS.getValueType());
+      RHS = DAG.getConstant(0, dl, LHS.getValueType());
       CC = ISD::SETNE;
     }
   }
@@ -3677,59 +3957,19 @@ SDValue AArch64TargetLowering::LowerSELECT_CC(ISD::CondCode CC, SDValue LHS,
   assert(LHS.getValueType() == MVT::f32 || LHS.getValueType() == MVT::f64);
   assert(LHS.getValueType() == RHS.getValueType());
   EVT VT = TVal.getValueType();
-
-  // Try to match this select into a max/min operation, which have dedicated
-  // opcode in the instruction set.
-  // FIXME: This is not correct in the presence of NaNs, so we only enable this
-  // in no-NaNs mode.
-  if (getTargetMachine().Options.NoNaNsFPMath) {
-    SDValue MinMaxLHS = TVal, MinMaxRHS = FVal;
-    if (selectCCOpsAreFMaxCompatible(LHS, MinMaxRHS) &&
-        selectCCOpsAreFMaxCompatible(RHS, MinMaxLHS)) {
-      CC = ISD::getSetCCSwappedOperands(CC);
-      std::swap(MinMaxLHS, MinMaxRHS);
-    }
-
-    if (selectCCOpsAreFMaxCompatible(LHS, MinMaxLHS) &&
-        selectCCOpsAreFMaxCompatible(RHS, MinMaxRHS)) {
-      switch (CC) {
-      default:
-        break;
-      case ISD::SETGT:
-      case ISD::SETGE:
-      case ISD::SETUGT:
-      case ISD::SETUGE:
-      case ISD::SETOGT:
-      case ISD::SETOGE:
-        return DAG.getNode(AArch64ISD::FMAX, dl, VT, MinMaxLHS, MinMaxRHS);
-        break;
-      case ISD::SETLT:
-      case ISD::SETLE:
-      case ISD::SETULT:
-      case ISD::SETULE:
-      case ISD::SETOLT:
-      case ISD::SETOLE:
-        return DAG.getNode(AArch64ISD::FMIN, dl, VT, MinMaxLHS, MinMaxRHS);
-        break;
-      }
-    }
-  }
-
-  // If that fails, we'll need to perform an FCMP + CSEL sequence.  Go ahead
-  // and do the comparison.
   SDValue Cmp = emitComparison(LHS, RHS, CC, dl, DAG);
 
   // Unfortunately, the mapping of LLVM FP CC's onto AArch64 CC's isn't totally
   // clean.  Some of them require two CSELs to implement.
   AArch64CC::CondCode CC1, CC2;
   changeFPCCToAArch64CC(CC, CC1, CC2);
-  SDValue CC1Val = DAG.getConstant(CC1, MVT::i32);
+  SDValue CC1Val = DAG.getConstant(CC1, dl, MVT::i32);
   SDValue CS1 = DAG.getNode(AArch64ISD::CSEL, dl, VT, TVal, FVal, CC1Val, Cmp);
 
   // If we need a second CSEL, emit it, using the output of the first as the
   // RHS.  We're effectively OR'ing the two CC's together.
   if (CC2 != AArch64CC::AL) {
-    SDValue CC2Val = DAG.getConstant(CC2, MVT::i32);
+    SDValue CC2Val = DAG.getConstant(CC2, dl, MVT::i32);
     return DAG.getNode(AArch64ISD::CSEL, dl, VT, TVal, CS1, CC2Val, Cmp);
   }
 
@@ -3768,7 +4008,7 @@ SDValue AArch64TargetLowering::LowerSELECT(SDValue Op,
     AArch64CC::CondCode OFCC;
     SDValue Value, Overflow;
     std::tie(Value, Overflow) = getAArch64XALUOOp(OFCC, CCVal.getValue(0), DAG);
-    SDValue CCVal = DAG.getConstant(OFCC, MVT::i32);
+    SDValue CCVal = DAG.getConstant(OFCC, DL, MVT::i32);
 
     return DAG.getNode(AArch64ISD::CSEL, DL, Op.getValueType(), TVal, FVal,
                        CCVal, Overflow);
@@ -3783,7 +4023,7 @@ SDValue AArch64TargetLowering::LowerSELECT(SDValue Op,
     CC = cast<CondCodeSDNode>(CCVal->getOperand(2))->get();
   } else {
     LHS = CCVal;
-    RHS = DAG.getConstant(0, CCVal.getValueType());
+    RHS = DAG.getConstant(0, DL, CCVal.getValueType());
     CC = ISD::SETNE;
   }
   return LowerSELECT_CC(CC, LHS, RHS, TVal, FVal, DL, DAG);
@@ -3794,7 +4034,7 @@ SDValue AArch64TargetLowering::LowerJumpTable(SDValue Op,
   // Jump table entries as PC relative offsets. No additional tweaking
   // is necessary here. Just get the address of the jump table.
   JumpTableSDNode *JT = cast<JumpTableSDNode>(Op);
-  EVT PtrVT = getPointerTy();
+  EVT PtrVT = getPointerTy(DAG.getDataLayout());
   SDLoc DL(Op);
 
   if (getTargetMachine().getCodeModel() == CodeModel::Large &&
@@ -3820,7 +4060,7 @@ SDValue AArch64TargetLowering::LowerJumpTable(SDValue Op,
 SDValue AArch64TargetLowering::LowerConstantPool(SDValue Op,
                                                  SelectionDAG &DAG) const {
   ConstantPoolSDNode *CP = cast<ConstantPoolSDNode>(Op);
-  EVT PtrVT = getPointerTy();
+  EVT PtrVT = getPointerTy(DAG.getDataLayout());
   SDLoc DL(Op);
 
   if (getTargetMachine().getCodeModel() == CodeModel::Large) {
@@ -3861,7 +4101,7 @@ SDValue AArch64TargetLowering::LowerConstantPool(SDValue Op,
 SDValue AArch64TargetLowering::LowerBlockAddress(SDValue Op,
                                                SelectionDAG &DAG) const {
   const BlockAddress *BA = cast<BlockAddressSDNode>(Op)->getBlockAddress();
-  EVT PtrVT = getPointerTy();
+  EVT PtrVT = getPointerTy(DAG.getDataLayout());
   SDLoc DL(Op);
   if (getTargetMachine().getCodeModel() == CodeModel::Large &&
       !Subtarget->isTargetMachO()) {
@@ -3887,8 +4127,8 @@ SDValue AArch64TargetLowering::LowerDarwin_VASTART(SDValue Op,
       DAG.getMachineFunction().getInfo<AArch64FunctionInfo>();
 
   SDLoc DL(Op);
-  SDValue FR =
-      DAG.getFrameIndex(FuncInfo->getVarArgsStackIndex(), getPointerTy());
+  SDValue FR = DAG.getFrameIndex(FuncInfo->getVarArgsStackIndex(),
+                                 getPointerTy(DAG.getDataLayout()));
   const Value *SV = cast<SrcValueSDNode>(Op.getOperand(2))->getValue();
   return DAG.getStore(Op.getOperand(0), DL, FR, Op.getOperand(1),
                       MachinePointerInfo(SV), false, false, 0);
@@ -3900,6 +4140,7 @@ SDValue AArch64TargetLowering::LowerAAPCS_VASTART(SDValue Op,
   // Standard, section B.3.
   MachineFunction &MF = DAG.getMachineFunction();
   AArch64FunctionInfo *FuncInfo = MF.getInfo<AArch64FunctionInfo>();
+  auto PtrVT = getPointerTy(DAG.getDataLayout());
   SDLoc DL(Op);
 
   SDValue Chain = Op.getOperand(0);
@@ -3908,8 +4149,7 @@ SDValue AArch64TargetLowering::LowerAAPCS_VASTART(SDValue Op,
   SmallVector<SDValue, 4> MemOps;
 
   // void *__stack at offset 0
-  SDValue Stack =
-      DAG.getFrameIndex(FuncInfo->getVarArgsStackIndex(), getPointerTy());
+  SDValue Stack = DAG.getFrameIndex(FuncInfo->getVarArgsStackIndex(), PtrVT);
   MemOps.push_back(DAG.getStore(Chain, DL, Stack, VAList,
                                 MachinePointerInfo(SV), false, false, 8));
 
@@ -3918,12 +4158,12 @@ SDValue AArch64TargetLowering::LowerAAPCS_VASTART(SDValue Op,
   if (GPRSize > 0) {
     SDValue GRTop, GRTopAddr;
 
-    GRTopAddr = DAG.getNode(ISD::ADD, DL, getPointerTy(), VAList,
-                            DAG.getConstant(8, getPointerTy()));
+    GRTopAddr =
+        DAG.getNode(ISD::ADD, DL, PtrVT, VAList, DAG.getConstant(8, DL, PtrVT));
 
-    GRTop = DAG.getFrameIndex(FuncInfo->getVarArgsGPRIndex(), getPointerTy());
-    GRTop = DAG.getNode(ISD::ADD, DL, getPointerTy(), GRTop,
-                        DAG.getConstant(GPRSize, getPointerTy()));
+    GRTop = DAG.getFrameIndex(FuncInfo->getVarArgsGPRIndex(), PtrVT);
+    GRTop = DAG.getNode(ISD::ADD, DL, PtrVT, GRTop,
+                        DAG.getConstant(GPRSize, DL, PtrVT));
 
     MemOps.push_back(DAG.getStore(Chain, DL, GRTop, GRTopAddr,
                                   MachinePointerInfo(SV, 8), false, false, 8));
@@ -3933,28 +4173,30 @@ SDValue AArch64TargetLowering::LowerAAPCS_VASTART(SDValue Op,
   int FPRSize = FuncInfo->getVarArgsFPRSize();
   if (FPRSize > 0) {
     SDValue VRTop, VRTopAddr;
-    VRTopAddr = DAG.getNode(ISD::ADD, DL, getPointerTy(), VAList,
-                            DAG.getConstant(16, getPointerTy()));
+    VRTopAddr = DAG.getNode(ISD::ADD, DL, PtrVT, VAList,
+                            DAG.getConstant(16, DL, PtrVT));
 
-    VRTop = DAG.getFrameIndex(FuncInfo->getVarArgsFPRIndex(), getPointerTy());
-    VRTop = DAG.getNode(ISD::ADD, DL, getPointerTy(), VRTop,
-                        DAG.getConstant(FPRSize, getPointerTy()));
+    VRTop = DAG.getFrameIndex(FuncInfo->getVarArgsFPRIndex(), PtrVT);
+    VRTop = DAG.getNode(ISD::ADD, DL, PtrVT, VRTop,
+                        DAG.getConstant(FPRSize, DL, PtrVT));
 
     MemOps.push_back(DAG.getStore(Chain, DL, VRTop, VRTopAddr,
                                   MachinePointerInfo(SV, 16), false, false, 8));
   }
 
   // int __gr_offs at offset 24
-  SDValue GROffsAddr = DAG.getNode(ISD::ADD, DL, getPointerTy(), VAList,
-                                   DAG.getConstant(24, getPointerTy()));
-  MemOps.push_back(DAG.getStore(Chain, DL, DAG.getConstant(-GPRSize, MVT::i32),
+  SDValue GROffsAddr =
+      DAG.getNode(ISD::ADD, DL, PtrVT, VAList, DAG.getConstant(24, DL, PtrVT));
+  MemOps.push_back(DAG.getStore(Chain, DL,
+                                DAG.getConstant(-GPRSize, DL, MVT::i32),
                                 GROffsAddr, MachinePointerInfo(SV, 24), false,
                                 false, 4));
 
   // int __vr_offs at offset 28
-  SDValue VROffsAddr = DAG.getNode(ISD::ADD, DL, getPointerTy(), VAList,
-                                   DAG.getConstant(28, getPointerTy()));
-  MemOps.push_back(DAG.getStore(Chain, DL, DAG.getConstant(-FPRSize, MVT::i32),
+  SDValue VROffsAddr =
+      DAG.getNode(ISD::ADD, DL, PtrVT, VAList, DAG.getConstant(28, DL, PtrVT));
+  MemOps.push_back(DAG.getStore(Chain, DL,
+                                DAG.getConstant(-FPRSize, DL, MVT::i32),
                                 VROffsAddr, MachinePointerInfo(SV, 28), false,
                                 false, 4));
 
@@ -3971,12 +4213,14 @@ SDValue AArch64TargetLowering::LowerVACOPY(SDValue Op,
                                            SelectionDAG &DAG) const {
   // AAPCS has three pointers and two ints (= 32 bytes), Darwin has single
   // pointer.
+  SDLoc DL(Op);
   unsigned VaListSize = Subtarget->isTargetDarwin() ? 8 : 32;
   const Value *DestSV = cast<SrcValueSDNode>(Op.getOperand(3))->getValue();
   const Value *SrcSV = cast<SrcValueSDNode>(Op.getOperand(4))->getValue();
 
-  return DAG.getMemcpy(Op.getOperand(0), SDLoc(Op), Op.getOperand(1),
-                       Op.getOperand(2), DAG.getConstant(VaListSize, MVT::i32),
+  return DAG.getMemcpy(Op.getOperand(0), DL, Op.getOperand(1),
+                       Op.getOperand(2),
+                       DAG.getConstant(VaListSize, DL, MVT::i32),
                        8, false, false, false, MachinePointerInfo(DestSV),
                        MachinePointerInfo(SrcSV));
 }
@@ -3991,21 +4235,22 @@ SDValue AArch64TargetLowering::LowerVAARG(SDValue Op, SelectionDAG &DAG) const {
   SDValue Chain = Op.getOperand(0);
   SDValue Addr = Op.getOperand(1);
   unsigned Align = Op.getConstantOperandVal(3);
+  auto PtrVT = getPointerTy(DAG.getDataLayout());
 
-  SDValue VAList = DAG.getLoad(getPointerTy(), DL, Chain, Addr,
-                               MachinePointerInfo(V), false, false, false, 0);
+  SDValue VAList = DAG.getLoad(PtrVT, DL, Chain, Addr, MachinePointerInfo(V),
+                               false, false, false, 0);
   Chain = VAList.getValue(1);
 
   if (Align > 8) {
     assert(((Align & (Align - 1)) == 0) && "Expected Align to be a power of 2");
-    VAList = DAG.getNode(ISD::ADD, DL, getPointerTy(), VAList,
-                         DAG.getConstant(Align - 1, getPointerTy()));
-    VAList = DAG.getNode(ISD::AND, DL, getPointerTy(), VAList,
-                         DAG.getConstant(-(int64_t)Align, getPointerTy()));
+    VAList = DAG.getNode(ISD::ADD, DL, PtrVT, VAList,
+                         DAG.getConstant(Align - 1, DL, PtrVT));
+    VAList = DAG.getNode(ISD::AND, DL, PtrVT, VAList,
+                         DAG.getConstant(-(int64_t)Align, DL, PtrVT));
   }
 
   Type *ArgTy = VT.getTypeForEVT(*DAG.getContext());
-  uint64_t ArgSize = getDataLayout()->getTypeAllocSize(ArgTy);
+  uint64_t ArgSize = DAG.getDataLayout().getTypeAllocSize(ArgTy);
 
   // Scalar integer and FP values smaller than 64 bits are implicitly extended
   // up to 64 bits.  At the very least, we have to increase the striding of the
@@ -4020,8 +4265,8 @@ SDValue AArch64TargetLowering::LowerVAARG(SDValue Op, SelectionDAG &DAG) const {
   }
 
   // Increment the pointer, VAList, to the next vaarg
-  SDValue VANext = DAG.getNode(ISD::ADD, DL, getPointerTy(), VAList,
-                               DAG.getConstant(ArgSize, getPointerTy()));
+  SDValue VANext = DAG.getNode(ISD::ADD, DL, PtrVT, VAList,
+                               DAG.getConstant(ArgSize, DL, PtrVT));
   // Store the incremented VAList to the legalized pointer
   SDValue APStore = DAG.getStore(Chain, DL, VANext, Addr, MachinePointerInfo(V),
                                  false, false, 0);
@@ -4033,7 +4278,7 @@ SDValue AArch64TargetLowering::LowerVAARG(SDValue Op, SelectionDAG &DAG) const {
                                  MachinePointerInfo(), false, false, false, 0);
     // Round the value down to an f32.
     SDValue NarrowFP = DAG.getNode(ISD::FP_ROUND, DL, VT, WideFP.getValue(0),
-                                   DAG.getIntPtrConstant(1));
+                                   DAG.getIntPtrConstant(1, DL));
     SDValue Ops[] = { NarrowFP, WideFP.getValue(1) };
     // Merge the rounded value with the chain output of the load.
     return DAG.getMergeValues(Ops, DL);
@@ -4061,14 +4306,15 @@ SDValue AArch64TargetLowering::LowerFRAMEADDR(SDValue Op,
 
 // FIXME? Maybe this could be a TableGen attribute on some registers and
 // this table could be generated automatically from RegInfo.
-unsigned AArch64TargetLowering::getRegisterByName(const char* RegName,
-                                                  EVT VT) const {
+unsigned AArch64TargetLowering::getRegisterByName(const char* RegName, EVT VT,
+                                                  SelectionDAG &DAG) const {
   unsigned Reg = StringSwitch<unsigned>(RegName)
                        .Case("sp", AArch64::SP)
                        .Default(0);
   if (Reg)
     return Reg;
-  report_fatal_error("Invalid register name global variable");
+  report_fatal_error(Twine("Invalid register name \""
+                              + StringRef(RegName)  + "\"."));
 }
 
 SDValue AArch64TargetLowering::LowerRETURNADDR(SDValue Op,
@@ -4082,7 +4328,7 @@ SDValue AArch64TargetLowering::LowerRETURNADDR(SDValue Op,
   unsigned Depth = cast<ConstantSDNode>(Op.getOperand(0))->getZExtValue();
   if (Depth) {
     SDValue FrameAddr = LowerFRAMEADDR(Op, DAG);
-    SDValue Offset = DAG.getConstant(8, getPointerTy());
+    SDValue Offset = DAG.getConstant(8, DL, getPointerTy(DAG.getDataLayout()));
     return DAG.getLoad(VT, DL, DAG.getEntryNode(),
                        DAG.getNode(ISD::ADD, DL, VT, FrameAddr, Offset),
                        MachinePointerInfo(), false, false, false, 0);
@@ -4110,15 +4356,15 @@ SDValue AArch64TargetLowering::LowerShiftRightParts(SDValue Op,
   assert(Op.getOpcode() == ISD::SRA_PARTS || Op.getOpcode() == ISD::SRL_PARTS);
 
   SDValue RevShAmt = DAG.getNode(ISD::SUB, dl, MVT::i64,
-                                 DAG.getConstant(VTBits, MVT::i64), ShAmt);
+                                 DAG.getConstant(VTBits, dl, MVT::i64), ShAmt);
   SDValue Tmp1 = DAG.getNode(ISD::SRL, dl, VT, ShOpLo, ShAmt);
   SDValue ExtraShAmt = DAG.getNode(ISD::SUB, dl, MVT::i64, ShAmt,
-                                   DAG.getConstant(VTBits, MVT::i64));
+                                   DAG.getConstant(VTBits, dl, MVT::i64));
   SDValue Tmp2 = DAG.getNode(ISD::SHL, dl, VT, ShOpHi, RevShAmt);
 
-  SDValue Cmp = emitComparison(ExtraShAmt, DAG.getConstant(0, MVT::i64),
+  SDValue Cmp = emitComparison(ExtraShAmt, DAG.getConstant(0, dl, MVT::i64),
                                ISD::SETGE, dl, DAG);
-  SDValue CCVal = DAG.getConstant(AArch64CC::GE, MVT::i32);
+  SDValue CCVal = DAG.getConstant(AArch64CC::GE, dl, MVT::i32);
 
   SDValue FalseValLo = DAG.getNode(ISD::OR, dl, VT, Tmp1, Tmp2);
   SDValue TrueValLo = DAG.getNode(Opc, dl, VT, ShOpHi, ExtraShAmt);
@@ -4130,8 +4376,9 @@ SDValue AArch64TargetLowering::LowerShiftRightParts(SDValue Op,
   SDValue FalseValHi = DAG.getNode(Opc, dl, VT, ShOpHi, ShAmt);
   SDValue TrueValHi = Opc == ISD::SRA
                           ? DAG.getNode(Opc, dl, VT, ShOpHi,
-                                        DAG.getConstant(VTBits - 1, MVT::i64))
-                          : DAG.getConstant(0, VT);
+                                        DAG.getConstant(VTBits - 1, dl,
+                                                        MVT::i64))
+                          : DAG.getConstant(0, dl, VT);
   SDValue Hi =
       DAG.getNode(AArch64ISD::CSEL, dl, VT, TrueValHi, FalseValHi, CCVal, Cmp);
 
@@ -4154,24 +4401,24 @@ SDValue AArch64TargetLowering::LowerShiftLeftParts(SDValue Op,
 
   assert(Op.getOpcode() == ISD::SHL_PARTS);
   SDValue RevShAmt = DAG.getNode(ISD::SUB, dl, MVT::i64,
-                                 DAG.getConstant(VTBits, MVT::i64), ShAmt);
+                                 DAG.getConstant(VTBits, dl, MVT::i64), ShAmt);
   SDValue Tmp1 = DAG.getNode(ISD::SRL, dl, VT, ShOpLo, RevShAmt);
   SDValue ExtraShAmt = DAG.getNode(ISD::SUB, dl, MVT::i64, ShAmt,
-                                   DAG.getConstant(VTBits, MVT::i64));
+                                   DAG.getConstant(VTBits, dl, MVT::i64));
   SDValue Tmp2 = DAG.getNode(ISD::SHL, dl, VT, ShOpHi, ShAmt);
   SDValue Tmp3 = DAG.getNode(ISD::SHL, dl, VT, ShOpLo, ExtraShAmt);
 
   SDValue FalseVal = DAG.getNode(ISD::OR, dl, VT, Tmp1, Tmp2);
 
-  SDValue Cmp = emitComparison(ExtraShAmt, DAG.getConstant(0, MVT::i64),
+  SDValue Cmp = emitComparison(ExtraShAmt, DAG.getConstant(0, dl, MVT::i64),
                                ISD::SETGE, dl, DAG);
-  SDValue CCVal = DAG.getConstant(AArch64CC::GE, MVT::i32);
+  SDValue CCVal = DAG.getConstant(AArch64CC::GE, dl, MVT::i32);
   SDValue Hi =
       DAG.getNode(AArch64ISD::CSEL, dl, VT, Tmp3, FalseVal, CCVal, Cmp);
 
   // AArch64 shifts of larger than register sizes are wrapped rather than
   // clamped, so we can't just emit "lo << a" if a is too big.
-  SDValue TrueValLo = DAG.getConstant(0, VT);
+  SDValue TrueValLo = DAG.getConstant(0, dl, VT);
   SDValue FalseValLo = DAG.getNode(ISD::SHL, dl, VT, ShOpLo, ShAmt);
   SDValue Lo =
       DAG.getNode(AArch64ISD::CSEL, dl, VT, TrueValLo, FalseValLo, CCVal, Cmp);
@@ -4234,7 +4481,7 @@ bool AArch64TargetLowering::isFPImmLegal(const APFloat &Imm, EVT VT) const {
 /// getConstraintType - Given a constraint letter, return the type of
 /// constraint it is for this target.
 AArch64TargetLowering::ConstraintType
-AArch64TargetLowering::getConstraintType(const std::string &Constraint) const {
+AArch64TargetLowering::getConstraintType(StringRef Constraint) const {
   if (Constraint.size() == 1) {
     switch (Constraint[0]) {
     default:
@@ -4285,8 +4532,7 @@ AArch64TargetLowering::getSingleConstraintMatchWeight(
 
 std::pair<unsigned, const TargetRegisterClass *>
 AArch64TargetLowering::getRegForInlineAsmConstraint(
-    const TargetRegisterInfo *TRI, const std::string &Constraint,
-    MVT VT) const {
+    const TargetRegisterInfo *TRI, StringRef Constraint, MVT VT) const {
   if (Constraint.size() == 1) {
     switch (Constraint[0]) {
     case 'r':
@@ -4322,10 +4568,9 @@ AArch64TargetLowering::getRegForInlineAsmConstraint(
     unsigned Size = Constraint.size();
     if ((Size == 4 || Size == 5) && Constraint[0] == '{' &&
         tolower(Constraint[1]) == 'v' && Constraint[Size - 1] == '}') {
-      const std::string Reg =
-          std::string(&Constraint[2], &Constraint[Size - 1]);
-      int RegNo = atoi(Reg.c_str());
-      if (RegNo >= 0 && RegNo <= 31) {
+      int RegNo;
+      bool Failed = Constraint.slice(2, Size - 1).getAsInteger(10, RegNo);
+      if (!Failed && RegNo >= 0 && RegNo <= 31) {
         // v0 - v31 are aliases of q0 - q31.
         // By default we'll emit v0-v31 for this unless there's a modifier where
         // we'll emit the correct register as well.
@@ -4464,7 +4709,7 @@ void AArch64TargetLowering::LowerAsmOperandForConstraint(
     }
 
     // All assembler immediates are 64-bit integers.
-    Result = DAG.getTargetConstant(CVal, MVT::i64);
+    Result = DAG.getTargetConstant(CVal, SDLoc(Op), MVT::i64);
     break;
   }
 
@@ -4490,7 +4735,7 @@ static SDValue WidenVector(SDValue V64Reg, SelectionDAG &DAG) {
   SDLoc DL(V64Reg);
 
   return DAG.getNode(ISD::INSERT_SUBVECTOR, DL, WideTy, DAG.getUNDEF(WideTy),
-                     V64Reg, DAG.getConstant(0, MVT::i32));
+                     V64Reg, DAG.getConstant(0, DL, MVT::i32));
 }
 
 /// getExtFactor - Determine the adjustment factor for the position when
@@ -4622,25 +4867,26 @@ SDValue AArch64TargetLowering::ReconstructShuffle(SDValue Op,
       // The extraction can just take the second half
       Src.ShuffleVec =
           DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, DestVT, Src.ShuffleVec,
-                      DAG.getConstant(NumSrcElts, MVT::i64));
+                      DAG.getConstant(NumSrcElts, dl, MVT::i64));
       Src.WindowBase = -NumSrcElts;
     } else if (Src.MaxElt < NumSrcElts) {
       // The extraction can just take the first half
       Src.ShuffleVec =
           DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, DestVT, Src.ShuffleVec,
-                      DAG.getConstant(0, MVT::i64));
+                      DAG.getConstant(0, dl, MVT::i64));
     } else {
       // An actual VEXT is needed
       SDValue VEXTSrc1 =
           DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, DestVT, Src.ShuffleVec,
-                      DAG.getConstant(0, MVT::i64));
+                      DAG.getConstant(0, dl, MVT::i64));
       SDValue VEXTSrc2 =
           DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, DestVT, Src.ShuffleVec,
-                      DAG.getConstant(NumSrcElts, MVT::i64));
+                      DAG.getConstant(NumSrcElts, dl, MVT::i64));
       unsigned Imm = Src.MinElt * getExtFactor(VEXTSrc1);
 
       Src.ShuffleVec = DAG.getNode(AArch64ISD::EXT, dl, DestVT, VEXTSrc1,
-                                   VEXTSrc2, DAG.getConstant(Imm, MVT::i32));
+                                   VEXTSrc2,
+                                   DAG.getConstant(Imm, dl, MVT::i32));
       Src.WindowBase = -Src.MinElt;
     }
   }
@@ -4975,11 +5221,11 @@ static SDValue tryFormConcatFromShuffle(SDValue Op, SelectionDAG &DAG) {
                                 VT.getVectorNumElements() / 2);
   if (SplitV0) {
     V0 = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, CastVT, V0,
-                     DAG.getConstant(0, MVT::i64));
+                     DAG.getConstant(0, DL, MVT::i64));
   }
   if (V1.getValueType().getSizeInBits() == 128) {
     V1 = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, CastVT, V1,
-                     DAG.getConstant(0, MVT::i64));
+                     DAG.getConstant(0, DL, MVT::i64));
   }
   return DAG.getNode(ISD::CONCAT_VECTORS, DL, VT, V0, V1);
 }
@@ -5057,7 +5303,7 @@ static SDValue GeneratePerfectShuffle(unsigned PFEntry, SDValue LHS,
 
     if (VT.getSizeInBits() == 64)
       OpLHS = WidenVector(OpLHS, DAG);
-    SDValue Lane = DAG.getConstant(OpNum - OP_VDUP0, MVT::i64);
+    SDValue Lane = DAG.getConstant(OpNum - OP_VDUP0, dl, MVT::i64);
     return DAG.getNode(Opcode, dl, VT, OpLHS, Lane);
   }
   case OP_VEXT1:
@@ -5065,7 +5311,7 @@ static SDValue GeneratePerfectShuffle(unsigned PFEntry, SDValue LHS,
   case OP_VEXT3: {
     unsigned Imm = (OpNum - OP_VEXT1 + 1) * getExtFactor(OpLHS);
     return DAG.getNode(AArch64ISD::EXT, dl, VT, OpLHS, OpRHS,
-                       DAG.getConstant(Imm, MVT::i32));
+                       DAG.getConstant(Imm, dl, MVT::i32));
   }
   case OP_VUZPL:
     return DAG.getNode(AArch64ISD::UZP1, dl, DAG.getVTList(VT, VT), OpLHS,
@@ -5102,7 +5348,7 @@ static SDValue GenerateTBL(SDValue Op, ArrayRef<int> ShuffleMask,
   for (int Val : ShuffleMask) {
     for (unsigned Byte = 0; Byte < BytesPerElt; ++Byte) {
       unsigned Offset = Byte + Val * BytesPerElt;
-      TBLMask.push_back(DAG.getConstant(Offset, MVT::i32));
+      TBLMask.push_back(DAG.getConstant(Offset, DL, MVT::i32));
     }
   }
 
@@ -5122,7 +5368,7 @@ static SDValue GenerateTBL(SDValue Op, ArrayRef<int> ShuffleMask,
       V1Cst = DAG.getNode(ISD::CONCAT_VECTORS, DL, MVT::v16i8, V1Cst, V1Cst);
     Shuffle = DAG.getNode(
         ISD::INTRINSIC_WO_CHAIN, DL, IndexVT,
-        DAG.getConstant(Intrinsic::aarch64_neon_tbl1, MVT::i32), V1Cst,
+        DAG.getConstant(Intrinsic::aarch64_neon_tbl1, DL, MVT::i32), V1Cst,
         DAG.getNode(ISD::BUILD_VECTOR, DL, IndexVT,
                     makeArrayRef(TBLMask.data(), IndexLen)));
   } else {
@@ -5130,7 +5376,7 @@ static SDValue GenerateTBL(SDValue Op, ArrayRef<int> ShuffleMask,
       V1Cst = DAG.getNode(ISD::CONCAT_VECTORS, DL, MVT::v16i8, V1Cst, V2Cst);
       Shuffle = DAG.getNode(
           ISD::INTRINSIC_WO_CHAIN, DL, IndexVT,
-          DAG.getConstant(Intrinsic::aarch64_neon_tbl1, MVT::i32), V1Cst,
+          DAG.getConstant(Intrinsic::aarch64_neon_tbl1, DL, MVT::i32), V1Cst,
           DAG.getNode(ISD::BUILD_VECTOR, DL, IndexVT,
                       makeArrayRef(TBLMask.data(), IndexLen)));
     } else {
@@ -5142,7 +5388,8 @@ static SDValue GenerateTBL(SDValue Op, ArrayRef<int> ShuffleMask,
       //                               &TBLMask[0], IndexLen));
       Shuffle = DAG.getNode(
           ISD::INTRINSIC_WO_CHAIN, DL, IndexVT,
-          DAG.getConstant(Intrinsic::aarch64_neon_tbl2, MVT::i32), V1Cst, V2Cst,
+          DAG.getConstant(Intrinsic::aarch64_neon_tbl2, DL, MVT::i32),
+          V1Cst, V2Cst,
           DAG.getNode(ISD::BUILD_VECTOR, DL, IndexVT,
                       makeArrayRef(TBLMask.data(), IndexLen)));
     }
@@ -5211,7 +5458,7 @@ SDValue AArch64TargetLowering::LowerVECTOR_SHUFFLE(SDValue Op,
     } else if (VT.getSizeInBits() == 64)
       V1 = WidenVector(V1, DAG);
 
-    return DAG.getNode(Opcode, dl, VT, V1, DAG.getConstant(Lane, MVT::i64));
+    return DAG.getNode(Opcode, dl, VT, V1, DAG.getConstant(Lane, dl, MVT::i64));
   }
 
   if (isREVMask(ShuffleMask, VT, 64))
@@ -5228,12 +5475,12 @@ SDValue AArch64TargetLowering::LowerVECTOR_SHUFFLE(SDValue Op,
       std::swap(V1, V2);
     Imm *= getExtFactor(V1);
     return DAG.getNode(AArch64ISD::EXT, dl, V1.getValueType(), V1, V2,
-                       DAG.getConstant(Imm, MVT::i32));
+                       DAG.getConstant(Imm, dl, MVT::i32));
   } else if (V2->getOpcode() == ISD::UNDEF &&
              isSingletonEXTMask(ShuffleMask, VT, Imm)) {
     Imm *= getExtFactor(V1);
     return DAG.getNode(AArch64ISD::EXT, dl, V1.getValueType(), V1, V1,
-                       DAG.getConstant(Imm, MVT::i32));
+                       DAG.getConstant(Imm, dl, MVT::i32));
   }
 
   unsigned WhichResult;
@@ -5272,7 +5519,7 @@ SDValue AArch64TargetLowering::LowerVECTOR_SHUFFLE(SDValue Op,
   int NumInputElements = V1.getValueType().getVectorNumElements();
   if (isINSMask(ShuffleMask, NumInputElements, DstIsLeft, Anomaly)) {
     SDValue DstVec = DstIsLeft ? V1 : V2;
-    SDValue DstLaneV = DAG.getConstant(Anomaly, MVT::i64);
+    SDValue DstLaneV = DAG.getConstant(Anomaly, dl, MVT::i64);
 
     SDValue SrcVec = V1;
     int SrcLane = ShuffleMask[Anomaly];
@@ -5280,7 +5527,7 @@ SDValue AArch64TargetLowering::LowerVECTOR_SHUFFLE(SDValue Op,
       SrcVec = V2;
       SrcLane -= VT.getVectorNumElements();
     }
-    SDValue SrcLaneV = DAG.getConstant(SrcLane, MVT::i64);
+    SDValue SrcLaneV = DAG.getConstant(SrcLane, dl, MVT::i64);
 
     EVT ScalarVT = VT.getVectorElementType();
 
@@ -5370,8 +5617,8 @@ SDValue AArch64TargetLowering::LowerVectorAND(SDValue Op,
         CnstVal = AArch64_AM::encodeAdvSIMDModImmType1(CnstVal);
         MVT MovTy = (VT.getSizeInBits() == 128) ? MVT::v4i32 : MVT::v2i32;
         SDValue Mov = DAG.getNode(AArch64ISD::BICi, dl, MovTy, LHS,
-                                  DAG.getConstant(CnstVal, MVT::i32),
-                                  DAG.getConstant(0, MVT::i32));
+                                  DAG.getConstant(CnstVal, dl, MVT::i32),
+                                  DAG.getConstant(0, dl, MVT::i32));
         return DAG.getNode(AArch64ISD::NVCAST, dl, VT, Mov);
       }
 
@@ -5379,8 +5626,8 @@ SDValue AArch64TargetLowering::LowerVectorAND(SDValue Op,
         CnstVal = AArch64_AM::encodeAdvSIMDModImmType2(CnstVal);
         MVT MovTy = (VT.getSizeInBits() == 128) ? MVT::v4i32 : MVT::v2i32;
         SDValue Mov = DAG.getNode(AArch64ISD::BICi, dl, MovTy, LHS,
-                                  DAG.getConstant(CnstVal, MVT::i32),
-                                  DAG.getConstant(8, MVT::i32));
+                                  DAG.getConstant(CnstVal, dl, MVT::i32),
+                                  DAG.getConstant(8, dl, MVT::i32));
         return DAG.getNode(AArch64ISD::NVCAST, dl, VT, Mov);
       }
 
@@ -5388,8 +5635,8 @@ SDValue AArch64TargetLowering::LowerVectorAND(SDValue Op,
         CnstVal = AArch64_AM::encodeAdvSIMDModImmType3(CnstVal);
         MVT MovTy = (VT.getSizeInBits() == 128) ? MVT::v4i32 : MVT::v2i32;
         SDValue Mov = DAG.getNode(AArch64ISD::BICi, dl, MovTy, LHS,
-                                  DAG.getConstant(CnstVal, MVT::i32),
-                                  DAG.getConstant(16, MVT::i32));
+                                  DAG.getConstant(CnstVal, dl, MVT::i32),
+                                  DAG.getConstant(16, dl, MVT::i32));
         return DAG.getNode(AArch64ISD::NVCAST, dl, VT, Mov);
       }
 
@@ -5397,8 +5644,8 @@ SDValue AArch64TargetLowering::LowerVectorAND(SDValue Op,
         CnstVal = AArch64_AM::encodeAdvSIMDModImmType4(CnstVal);
         MVT MovTy = (VT.getSizeInBits() == 128) ? MVT::v4i32 : MVT::v2i32;
         SDValue Mov = DAG.getNode(AArch64ISD::BICi, dl, MovTy, LHS,
-                                  DAG.getConstant(CnstVal, MVT::i32),
-                                  DAG.getConstant(24, MVT::i32));
+                                  DAG.getConstant(CnstVal, dl, MVT::i32),
+                                  DAG.getConstant(24, dl, MVT::i32));
         return DAG.getNode(AArch64ISD::NVCAST, dl, VT, Mov);
       }
 
@@ -5406,8 +5653,8 @@ SDValue AArch64TargetLowering::LowerVectorAND(SDValue Op,
         CnstVal = AArch64_AM::encodeAdvSIMDModImmType5(CnstVal);
         MVT MovTy = (VT.getSizeInBits() == 128) ? MVT::v8i16 : MVT::v4i16;
         SDValue Mov = DAG.getNode(AArch64ISD::BICi, dl, MovTy, LHS,
-                                  DAG.getConstant(CnstVal, MVT::i32),
-                                  DAG.getConstant(0, MVT::i32));
+                                  DAG.getConstant(CnstVal, dl, MVT::i32),
+                                  DAG.getConstant(0, dl, MVT::i32));
         return DAG.getNode(AArch64ISD::NVCAST, dl, VT, Mov);
       }
 
@@ -5415,8 +5662,8 @@ SDValue AArch64TargetLowering::LowerVectorAND(SDValue Op,
         CnstVal = AArch64_AM::encodeAdvSIMDModImmType6(CnstVal);
         MVT MovTy = (VT.getSizeInBits() == 128) ? MVT::v8i16 : MVT::v4i16;
         SDValue Mov = DAG.getNode(AArch64ISD::BICi, dl, MovTy, LHS,
-                                  DAG.getConstant(CnstVal, MVT::i32),
-                                  DAG.getConstant(8, MVT::i32));
+                                  DAG.getConstant(CnstVal, dl, MVT::i32),
+                                  DAG.getConstant(8, dl, MVT::i32));
         return DAG.getNode(AArch64ISD::NVCAST, dl, VT, Mov);
       }
     }
@@ -5520,7 +5767,8 @@ static SDValue tryLowerToSLI(SDNode *N, SelectionDAG &DAG) {
       IsShiftRight ? Intrinsic::aarch64_neon_vsri : Intrinsic::aarch64_neon_vsli;
   SDValue ResultSLI =
       DAG.getNode(ISD::INTRINSIC_WO_CHAIN, DL, VT,
-                  DAG.getConstant(Intrin, MVT::i32), X, Y, Shift.getOperand(1));
+                  DAG.getConstant(Intrin, DL, MVT::i32), X, Y,
+                  Shift.getOperand(1));
 
   DEBUG(dbgs() << "aarch64-lower: transformed: \n");
   DEBUG(N->dump(&DAG));
@@ -5570,8 +5818,8 @@ SDValue AArch64TargetLowering::LowerVectorOR(SDValue Op,
         CnstVal = AArch64_AM::encodeAdvSIMDModImmType1(CnstVal);
         MVT MovTy = (VT.getSizeInBits() == 128) ? MVT::v4i32 : MVT::v2i32;
         SDValue Mov = DAG.getNode(AArch64ISD::ORRi, dl, MovTy, LHS,
-                                  DAG.getConstant(CnstVal, MVT::i32),
-                                  DAG.getConstant(0, MVT::i32));
+                                  DAG.getConstant(CnstVal, dl, MVT::i32),
+                                  DAG.getConstant(0, dl, MVT::i32));
         return DAG.getNode(AArch64ISD::NVCAST, dl, VT, Mov);
       }
 
@@ -5579,8 +5827,8 @@ SDValue AArch64TargetLowering::LowerVectorOR(SDValue Op,
         CnstVal = AArch64_AM::encodeAdvSIMDModImmType2(CnstVal);
         MVT MovTy = (VT.getSizeInBits() == 128) ? MVT::v4i32 : MVT::v2i32;
         SDValue Mov = DAG.getNode(AArch64ISD::ORRi, dl, MovTy, LHS,
-                                  DAG.getConstant(CnstVal, MVT::i32),
-                                  DAG.getConstant(8, MVT::i32));
+                                  DAG.getConstant(CnstVal, dl, MVT::i32),
+                                  DAG.getConstant(8, dl, MVT::i32));
         return DAG.getNode(AArch64ISD::NVCAST, dl, VT, Mov);
       }
 
@@ -5588,8 +5836,8 @@ SDValue AArch64TargetLowering::LowerVectorOR(SDValue Op,
         CnstVal = AArch64_AM::encodeAdvSIMDModImmType3(CnstVal);
         MVT MovTy = (VT.getSizeInBits() == 128) ? MVT::v4i32 : MVT::v2i32;
         SDValue Mov = DAG.getNode(AArch64ISD::ORRi, dl, MovTy, LHS,
-                                  DAG.getConstant(CnstVal, MVT::i32),
-                                  DAG.getConstant(16, MVT::i32));
+                                  DAG.getConstant(CnstVal, dl, MVT::i32),
+                                  DAG.getConstant(16, dl, MVT::i32));
         return DAG.getNode(AArch64ISD::NVCAST, dl, VT, Mov);
       }
 
@@ -5597,8 +5845,8 @@ SDValue AArch64TargetLowering::LowerVectorOR(SDValue Op,
         CnstVal = AArch64_AM::encodeAdvSIMDModImmType4(CnstVal);
         MVT MovTy = (VT.getSizeInBits() == 128) ? MVT::v4i32 : MVT::v2i32;
         SDValue Mov = DAG.getNode(AArch64ISD::ORRi, dl, MovTy, LHS,
-                                  DAG.getConstant(CnstVal, MVT::i32),
-                                  DAG.getConstant(24, MVT::i32));
+                                  DAG.getConstant(CnstVal, dl, MVT::i32),
+                                  DAG.getConstant(24, dl, MVT::i32));
         return DAG.getNode(AArch64ISD::NVCAST, dl, VT, Mov);
       }
 
@@ -5606,8 +5854,8 @@ SDValue AArch64TargetLowering::LowerVectorOR(SDValue Op,
         CnstVal = AArch64_AM::encodeAdvSIMDModImmType5(CnstVal);
         MVT MovTy = (VT.getSizeInBits() == 128) ? MVT::v8i16 : MVT::v4i16;
         SDValue Mov = DAG.getNode(AArch64ISD::ORRi, dl, MovTy, LHS,
-                                  DAG.getConstant(CnstVal, MVT::i32),
-                                  DAG.getConstant(0, MVT::i32));
+                                  DAG.getConstant(CnstVal, dl, MVT::i32),
+                                  DAG.getConstant(0, dl, MVT::i32));
         return DAG.getNode(AArch64ISD::NVCAST, dl, VT, Mov);
       }
 
@@ -5615,8 +5863,8 @@ SDValue AArch64TargetLowering::LowerVectorOR(SDValue Op,
         CnstVal = AArch64_AM::encodeAdvSIMDModImmType6(CnstVal);
         MVT MovTy = (VT.getSizeInBits() == 128) ? MVT::v8i16 : MVT::v4i16;
         SDValue Mov = DAG.getNode(AArch64ISD::ORRi, dl, MovTy, LHS,
-                                  DAG.getConstant(CnstVal, MVT::i32),
-                                  DAG.getConstant(8, MVT::i32));
+                                  DAG.getConstant(CnstVal, dl, MVT::i32),
+                                  DAG.getConstant(8, dl, MVT::i32));
         return DAG.getNode(AArch64ISD::NVCAST, dl, VT, Mov);
       }
     }
@@ -5651,7 +5899,7 @@ static SDValue NormalizeBuildVector(SDValue Op,
     if (Lane.getOpcode() == ISD::Constant) {
       APInt LowBits(EltTy.getSizeInBits(),
                     cast<ConstantSDNode>(Lane)->getZExtValue());
-      Lane = DAG.getConstant(LowBits.getZExtValue(), MVT::i32);
+      Lane = DAG.getConstant(LowBits.getZExtValue(), dl, MVT::i32);
     }
     Ops.push_back(Lane);
   }
@@ -5689,13 +5937,13 @@ SDValue AArch64TargetLowering::LowerBUILD_VECTOR(SDValue Op,
         CnstVal = AArch64_AM::encodeAdvSIMDModImmType10(CnstVal);
         if (VT.getSizeInBits() == 128) {
           SDValue Mov = DAG.getNode(AArch64ISD::MOVIedit, dl, MVT::v2i64,
-                                    DAG.getConstant(CnstVal, MVT::i32));
+                                    DAG.getConstant(CnstVal, dl, MVT::i32));
           return DAG.getNode(AArch64ISD::NVCAST, dl, VT, Mov);
         }
 
         // Support the V64 version via subregister insertion.
         SDValue Mov = DAG.getNode(AArch64ISD::MOVIedit, dl, MVT::f64,
-                                  DAG.getConstant(CnstVal, MVT::i32));
+                                  DAG.getConstant(CnstVal, dl, MVT::i32));
         return DAG.getNode(AArch64ISD::NVCAST, dl, VT, Mov);
       }
 
@@ -5703,8 +5951,8 @@ SDValue AArch64TargetLowering::LowerBUILD_VECTOR(SDValue Op,
         CnstVal = AArch64_AM::encodeAdvSIMDModImmType1(CnstVal);
         MVT MovTy = (VT.getSizeInBits() == 128) ? MVT::v4i32 : MVT::v2i32;
         SDValue Mov = DAG.getNode(AArch64ISD::MOVIshift, dl, MovTy,
-                                  DAG.getConstant(CnstVal, MVT::i32),
-                                  DAG.getConstant(0, MVT::i32));
+                                  DAG.getConstant(CnstVal, dl, MVT::i32),
+                                  DAG.getConstant(0, dl, MVT::i32));
         return DAG.getNode(AArch64ISD::NVCAST, dl, VT, Mov);
       }
 
@@ -5712,8 +5960,8 @@ SDValue AArch64TargetLowering::LowerBUILD_VECTOR(SDValue Op,
         CnstVal = AArch64_AM::encodeAdvSIMDModImmType2(CnstVal);
         MVT MovTy = (VT.getSizeInBits() == 128) ? MVT::v4i32 : MVT::v2i32;
         SDValue Mov = DAG.getNode(AArch64ISD::MOVIshift, dl, MovTy,
-                                  DAG.getConstant(CnstVal, MVT::i32),
-                                  DAG.getConstant(8, MVT::i32));
+                                  DAG.getConstant(CnstVal, dl, MVT::i32),
+                                  DAG.getConstant(8, dl, MVT::i32));
         return DAG.getNode(AArch64ISD::NVCAST, dl, VT, Mov);
       }
 
@@ -5721,8 +5969,8 @@ SDValue AArch64TargetLowering::LowerBUILD_VECTOR(SDValue Op,
         CnstVal = AArch64_AM::encodeAdvSIMDModImmType3(CnstVal);
         MVT MovTy = (VT.getSizeInBits() == 128) ? MVT::v4i32 : MVT::v2i32;
         SDValue Mov = DAG.getNode(AArch64ISD::MOVIshift, dl, MovTy,
-                                  DAG.getConstant(CnstVal, MVT::i32),
-                                  DAG.getConstant(16, MVT::i32));
+                                  DAG.getConstant(CnstVal, dl, MVT::i32),
+                                  DAG.getConstant(16, dl, MVT::i32));
         return DAG.getNode(AArch64ISD::NVCAST, dl, VT, Mov);
       }
 
@@ -5730,8 +5978,8 @@ SDValue AArch64TargetLowering::LowerBUILD_VECTOR(SDValue Op,
         CnstVal = AArch64_AM::encodeAdvSIMDModImmType4(CnstVal);
         MVT MovTy = (VT.getSizeInBits() == 128) ? MVT::v4i32 : MVT::v2i32;
         SDValue Mov = DAG.getNode(AArch64ISD::MOVIshift, dl, MovTy,
-                                  DAG.getConstant(CnstVal, MVT::i32),
-                                  DAG.getConstant(24, MVT::i32));
+                                  DAG.getConstant(CnstVal, dl, MVT::i32),
+                                  DAG.getConstant(24, dl, MVT::i32));
         return DAG.getNode(AArch64ISD::NVCAST, dl, VT, Mov);
       }
 
@@ -5739,8 +5987,8 @@ SDValue AArch64TargetLowering::LowerBUILD_VECTOR(SDValue Op,
         CnstVal = AArch64_AM::encodeAdvSIMDModImmType5(CnstVal);
         MVT MovTy = (VT.getSizeInBits() == 128) ? MVT::v8i16 : MVT::v4i16;
         SDValue Mov = DAG.getNode(AArch64ISD::MOVIshift, dl, MovTy,
-                                  DAG.getConstant(CnstVal, MVT::i32),
-                                  DAG.getConstant(0, MVT::i32));
+                                  DAG.getConstant(CnstVal, dl, MVT::i32),
+                                  DAG.getConstant(0, dl, MVT::i32));
         return DAG.getNode(AArch64ISD::NVCAST, dl, VT, Mov);
       }
 
@@ -5748,8 +5996,8 @@ SDValue AArch64TargetLowering::LowerBUILD_VECTOR(SDValue Op,
         CnstVal = AArch64_AM::encodeAdvSIMDModImmType6(CnstVal);
         MVT MovTy = (VT.getSizeInBits() == 128) ? MVT::v8i16 : MVT::v4i16;
         SDValue Mov = DAG.getNode(AArch64ISD::MOVIshift, dl, MovTy,
-                                  DAG.getConstant(CnstVal, MVT::i32),
-                                  DAG.getConstant(8, MVT::i32));
+                                  DAG.getConstant(CnstVal, dl, MVT::i32),
+                                  DAG.getConstant(8, dl, MVT::i32));
         return DAG.getNode(AArch64ISD::NVCAST, dl, VT, Mov);
       }
 
@@ -5757,8 +6005,8 @@ SDValue AArch64TargetLowering::LowerBUILD_VECTOR(SDValue Op,
         CnstVal = AArch64_AM::encodeAdvSIMDModImmType7(CnstVal);
         MVT MovTy = (VT.getSizeInBits() == 128) ? MVT::v4i32 : MVT::v2i32;
         SDValue Mov = DAG.getNode(AArch64ISD::MOVImsl, dl, MovTy,
-                                  DAG.getConstant(CnstVal, MVT::i32),
-                                  DAG.getConstant(264, MVT::i32));
+                                  DAG.getConstant(CnstVal, dl, MVT::i32),
+                                  DAG.getConstant(264, dl, MVT::i32));
         return DAG.getNode(AArch64ISD::NVCAST, dl, VT, Mov);
       }
 
@@ -5766,8 +6014,8 @@ SDValue AArch64TargetLowering::LowerBUILD_VECTOR(SDValue Op,
         CnstVal = AArch64_AM::encodeAdvSIMDModImmType8(CnstVal);
         MVT MovTy = (VT.getSizeInBits() == 128) ? MVT::v4i32 : MVT::v2i32;
         SDValue Mov = DAG.getNode(AArch64ISD::MOVImsl, dl, MovTy,
-                                  DAG.getConstant(CnstVal, MVT::i32),
-                                  DAG.getConstant(272, MVT::i32));
+                                  DAG.getConstant(CnstVal, dl, MVT::i32),
+                                  DAG.getConstant(272, dl, MVT::i32));
         return DAG.getNode(AArch64ISD::NVCAST, dl, VT, Mov);
       }
 
@@ -5775,7 +6023,7 @@ SDValue AArch64TargetLowering::LowerBUILD_VECTOR(SDValue Op,
         CnstVal = AArch64_AM::encodeAdvSIMDModImmType9(CnstVal);
         MVT MovTy = (VT.getSizeInBits() == 128) ? MVT::v16i8 : MVT::v8i8;
         SDValue Mov = DAG.getNode(AArch64ISD::MOVI, dl, MovTy,
-                                  DAG.getConstant(CnstVal, MVT::i32));
+                                  DAG.getConstant(CnstVal, dl, MVT::i32));
         return DAG.getNode(AArch64ISD::NVCAST, dl, VT, Mov);
       }
 
@@ -5784,7 +6032,7 @@ SDValue AArch64TargetLowering::LowerBUILD_VECTOR(SDValue Op,
         CnstVal = AArch64_AM::encodeAdvSIMDModImmType11(CnstVal);
         MVT MovTy = (VT.getSizeInBits() == 128) ? MVT::v4f32 : MVT::v2f32;
         SDValue Mov = DAG.getNode(AArch64ISD::FMOV, dl, MovTy,
-                                  DAG.getConstant(CnstVal, MVT::i32));
+                                  DAG.getConstant(CnstVal, dl, MVT::i32));
         return DAG.getNode(AArch64ISD::NVCAST, dl, VT, Mov);
       }
 
@@ -5792,7 +6040,7 @@ SDValue AArch64TargetLowering::LowerBUILD_VECTOR(SDValue Op,
           VT.getSizeInBits() == 128) {
         CnstVal = AArch64_AM::encodeAdvSIMDModImmType12(CnstVal);
         SDValue Mov = DAG.getNode(AArch64ISD::FMOV, dl, MVT::v2f64,
-                                  DAG.getConstant(CnstVal, MVT::i32));
+                                  DAG.getConstant(CnstVal, dl, MVT::i32));
         return DAG.getNode(AArch64ISD::NVCAST, dl, VT, Mov);
       }
 
@@ -5802,8 +6050,8 @@ SDValue AArch64TargetLowering::LowerBUILD_VECTOR(SDValue Op,
         CnstVal = AArch64_AM::encodeAdvSIMDModImmType1(CnstVal);
         MVT MovTy = (VT.getSizeInBits() == 128) ? MVT::v4i32 : MVT::v2i32;
         SDValue Mov = DAG.getNode(AArch64ISD::MVNIshift, dl, MovTy,
-                                  DAG.getConstant(CnstVal, MVT::i32),
-                                  DAG.getConstant(0, MVT::i32));
+                                  DAG.getConstant(CnstVal, dl, MVT::i32),
+                                  DAG.getConstant(0, dl, MVT::i32));
         return DAG.getNode(AArch64ISD::NVCAST, dl, VT, Mov);
       }
 
@@ -5811,8 +6059,8 @@ SDValue AArch64TargetLowering::LowerBUILD_VECTOR(SDValue Op,
         CnstVal = AArch64_AM::encodeAdvSIMDModImmType2(CnstVal);
         MVT MovTy = (VT.getSizeInBits() == 128) ? MVT::v4i32 : MVT::v2i32;
         SDValue Mov = DAG.getNode(AArch64ISD::MVNIshift, dl, MovTy,
-                                  DAG.getConstant(CnstVal, MVT::i32),
-                                  DAG.getConstant(8, MVT::i32));
+                                  DAG.getConstant(CnstVal, dl, MVT::i32),
+                                  DAG.getConstant(8, dl, MVT::i32));
         return DAG.getNode(AArch64ISD::NVCAST, dl, VT, Mov);
       }
 
@@ -5820,8 +6068,8 @@ SDValue AArch64TargetLowering::LowerBUILD_VECTOR(SDValue Op,
         CnstVal = AArch64_AM::encodeAdvSIMDModImmType3(CnstVal);
         MVT MovTy = (VT.getSizeInBits() == 128) ? MVT::v4i32 : MVT::v2i32;
         SDValue Mov = DAG.getNode(AArch64ISD::MVNIshift, dl, MovTy,
-                                  DAG.getConstant(CnstVal, MVT::i32),
-                                  DAG.getConstant(16, MVT::i32));
+                                  DAG.getConstant(CnstVal, dl, MVT::i32),
+                                  DAG.getConstant(16, dl, MVT::i32));
         return DAG.getNode(AArch64ISD::NVCAST, dl, VT, Mov);
       }
 
@@ -5829,8 +6077,8 @@ SDValue AArch64TargetLowering::LowerBUILD_VECTOR(SDValue Op,
         CnstVal = AArch64_AM::encodeAdvSIMDModImmType4(CnstVal);
         MVT MovTy = (VT.getSizeInBits() == 128) ? MVT::v4i32 : MVT::v2i32;
         SDValue Mov = DAG.getNode(AArch64ISD::MVNIshift, dl, MovTy,
-                                  DAG.getConstant(CnstVal, MVT::i32),
-                                  DAG.getConstant(24, MVT::i32));
+                                  DAG.getConstant(CnstVal, dl, MVT::i32),
+                                  DAG.getConstant(24, dl, MVT::i32));
         return DAG.getNode(AArch64ISD::NVCAST, dl, VT, Mov);
       }
 
@@ -5838,8 +6086,8 @@ SDValue AArch64TargetLowering::LowerBUILD_VECTOR(SDValue Op,
         CnstVal = AArch64_AM::encodeAdvSIMDModImmType5(CnstVal);
         MVT MovTy = (VT.getSizeInBits() == 128) ? MVT::v8i16 : MVT::v4i16;
         SDValue Mov = DAG.getNode(AArch64ISD::MVNIshift, dl, MovTy,
-                                  DAG.getConstant(CnstVal, MVT::i32),
-                                  DAG.getConstant(0, MVT::i32));
+                                  DAG.getConstant(CnstVal, dl, MVT::i32),
+                                  DAG.getConstant(0, dl, MVT::i32));
         return DAG.getNode(AArch64ISD::NVCAST, dl, VT, Mov);
       }
 
@@ -5847,8 +6095,8 @@ SDValue AArch64TargetLowering::LowerBUILD_VECTOR(SDValue Op,
         CnstVal = AArch64_AM::encodeAdvSIMDModImmType6(CnstVal);
         MVT MovTy = (VT.getSizeInBits() == 128) ? MVT::v8i16 : MVT::v4i16;
         SDValue Mov = DAG.getNode(AArch64ISD::MVNIshift, dl, MovTy,
-                                  DAG.getConstant(CnstVal, MVT::i32),
-                                  DAG.getConstant(8, MVT::i32));
+                                  DAG.getConstant(CnstVal, dl, MVT::i32),
+                                  DAG.getConstant(8, dl, MVT::i32));
         return DAG.getNode(AArch64ISD::NVCAST, dl, VT, Mov);
       }
 
@@ -5856,8 +6104,8 @@ SDValue AArch64TargetLowering::LowerBUILD_VECTOR(SDValue Op,
         CnstVal = AArch64_AM::encodeAdvSIMDModImmType7(CnstVal);
         MVT MovTy = (VT.getSizeInBits() == 128) ? MVT::v4i32 : MVT::v2i32;
         SDValue Mov = DAG.getNode(AArch64ISD::MVNImsl, dl, MovTy,
-                                  DAG.getConstant(CnstVal, MVT::i32),
-                                  DAG.getConstant(264, MVT::i32));
+                                  DAG.getConstant(CnstVal, dl, MVT::i32),
+                                  DAG.getConstant(264, dl, MVT::i32));
         return DAG.getNode(AArch64ISD::NVCAST, dl, VT, Mov);
       }
 
@@ -5865,8 +6113,8 @@ SDValue AArch64TargetLowering::LowerBUILD_VECTOR(SDValue Op,
         CnstVal = AArch64_AM::encodeAdvSIMDModImmType8(CnstVal);
         MVT MovTy = (VT.getSizeInBits() == 128) ? MVT::v4i32 : MVT::v2i32;
         SDValue Mov = DAG.getNode(AArch64ISD::MVNImsl, dl, MovTy,
-                                  DAG.getConstant(CnstVal, MVT::i32),
-                                  DAG.getConstant(272, MVT::i32));
+                                  DAG.getConstant(CnstVal, dl, MVT::i32),
+                                  DAG.getConstant(272, dl, MVT::i32));
         return DAG.getNode(AArch64ISD::NVCAST, dl, VT, Mov);
       }
     }
@@ -5972,7 +6220,7 @@ SDValue AArch64TargetLowering::LowerBUILD_VECTOR(SDValue Op,
     // Now insert the non-constant lanes.
     for (unsigned i = 0; i < NumElts; ++i) {
       SDValue V = Op.getOperand(i);
-      SDValue LaneIdx = DAG.getConstant(i, MVT::i64);
+      SDValue LaneIdx = DAG.getConstant(i, dl, MVT::i64);
       if (!isa<ConstantSDNode>(V) && !isa<ConstantFPSDNode>(V)) {
         // Note that type legalization likely mucked about with the VT of the
         // source operand, so we may have to convert it here before inserting.
@@ -6014,7 +6262,7 @@ SDValue AArch64TargetLowering::LowerBUILD_VECTOR(SDValue Op,
       unsigned SubIdx = ElemSize == 32 ? AArch64::ssub : AArch64::dsub;
       MachineSDNode *N =
           DAG.getMachineNode(TargetOpcode::INSERT_SUBREG, dl, VT, Vec, Op0,
-                             DAG.getTargetConstant(SubIdx, MVT::i32));
+                             DAG.getTargetConstant(SubIdx, dl, MVT::i32));
       Vec = SDValue(N, 0);
       ++i;
     }
@@ -6022,7 +6270,7 @@ SDValue AArch64TargetLowering::LowerBUILD_VECTOR(SDValue Op,
       SDValue V = Op.getOperand(i);
       if (V.getOpcode() == ISD::UNDEF)
         continue;
-      SDValue LaneIdx = DAG.getConstant(i, MVT::i64);
+      SDValue LaneIdx = DAG.getConstant(i, dl, MVT::i64);
       Vec = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, VT, Vec, V, LaneIdx);
     }
     return Vec;
@@ -6206,26 +6454,20 @@ static bool getVShiftImm(SDValue Op, unsigned ElementBits, int64_t &Cnt) {
 ///   0 <= Value <= ElementBits for a long left shift.
 static bool isVShiftLImm(SDValue Op, EVT VT, bool isLong, int64_t &Cnt) {
   assert(VT.isVector() && "vector shift count is not a vector type");
-  unsigned ElementBits = VT.getVectorElementType().getSizeInBits();
+  int64_t ElementBits = VT.getVectorElementType().getSizeInBits();
   if (!getVShiftImm(Op, ElementBits, Cnt))
     return false;
   return (Cnt >= 0 && (isLong ? Cnt - 1 : Cnt) < ElementBits);
 }
 
 /// isVShiftRImm - Check if this is a valid build_vector for the immediate
-/// operand of a vector shift right operation.  For a shift opcode, the value
-/// is positive, but for an intrinsic the value count must be negative. The
-/// absolute value must be in the range:
-///   1 <= |Value| <= ElementBits for a right shift; or
-///   1 <= |Value| <= ElementBits/2 for a narrow right shift.
-static bool isVShiftRImm(SDValue Op, EVT VT, bool isNarrow, bool isIntrinsic,
-                         int64_t &Cnt) {
+/// operand of a vector shift right operation. The value must be in the range:
+///   1 <= Value <= ElementBits for a right shift; or
+static bool isVShiftRImm(SDValue Op, EVT VT, bool isNarrow, int64_t &Cnt) {
   assert(VT.isVector() && "vector shift count is not a vector type");
-  unsigned ElementBits = VT.getVectorElementType().getSizeInBits();
+  int64_t ElementBits = VT.getVectorElementType().getSizeInBits();
   if (!getVShiftImm(Op, ElementBits, Cnt))
     return false;
-  if (isIntrinsic)
-    Cnt = -Cnt;
   return (Cnt >= 1 && Cnt <= (isNarrow ? ElementBits / 2 : ElementBits));
 }
 
@@ -6245,20 +6487,20 @@ SDValue AArch64TargetLowering::LowerVectorSRA_SRL_SHL(SDValue Op,
 
   case ISD::SHL:
     if (isVShiftLImm(Op.getOperand(1), VT, false, Cnt) && Cnt < EltSize)
-      return DAG.getNode(AArch64ISD::VSHL, SDLoc(Op), VT, Op.getOperand(0),
-                         DAG.getConstant(Cnt, MVT::i32));
+      return DAG.getNode(AArch64ISD::VSHL, DL, VT, Op.getOperand(0),
+                         DAG.getConstant(Cnt, DL, MVT::i32));
     return DAG.getNode(ISD::INTRINSIC_WO_CHAIN, DL, VT,
-                       DAG.getConstant(Intrinsic::aarch64_neon_ushl, MVT::i32),
+                       DAG.getConstant(Intrinsic::aarch64_neon_ushl, DL,
+                                       MVT::i32),
                        Op.getOperand(0), Op.getOperand(1));
   case ISD::SRA:
   case ISD::SRL:
     // Right shift immediate
-    if (isVShiftRImm(Op.getOperand(1), VT, false, false, Cnt) &&
-        Cnt < EltSize) {
+    if (isVShiftRImm(Op.getOperand(1), VT, false, Cnt) && Cnt < EltSize) {
       unsigned Opc =
           (Op.getOpcode() == ISD::SRA) ? AArch64ISD::VASHR : AArch64ISD::VLSHR;
-      return DAG.getNode(Opc, SDLoc(Op), VT, Op.getOperand(0),
-                         DAG.getConstant(Cnt, MVT::i32));
+      return DAG.getNode(Opc, DL, VT, Op.getOperand(0),
+                         DAG.getConstant(Cnt, DL, MVT::i32));
     }
 
     // Right shift register.  Note, there is not a shift right register
@@ -6270,7 +6512,8 @@ SDValue AArch64TargetLowering::LowerVectorSRA_SRL_SHL(SDValue Op,
     SDValue NegShift = DAG.getNode(AArch64ISD::NEG, DL, VT, Op.getOperand(1));
     SDValue NegShiftLeft =
         DAG.getNode(ISD::INTRINSIC_WO_CHAIN, DL, VT,
-                    DAG.getConstant(Opc, MVT::i32), Op.getOperand(0), NegShift);
+                    DAG.getConstant(Opc, DL, MVT::i32), Op.getOperand(0),
+                    NegShift);
     return NegShiftLeft;
   }
 
@@ -6426,6 +6669,7 @@ SDValue AArch64TargetLowering::LowerVSETCC(SDValue Op,
 bool AArch64TargetLowering::getTgtMemIntrinsic(IntrinsicInfo &Info,
                                                const CallInst &I,
                                                unsigned Intrinsic) const {
+  auto &DL = I.getModule()->getDataLayout();
   switch (Intrinsic) {
   case Intrinsic::aarch64_neon_ld2:
   case Intrinsic::aarch64_neon_ld3:
@@ -6441,7 +6685,7 @@ bool AArch64TargetLowering::getTgtMemIntrinsic(IntrinsicInfo &Info,
   case Intrinsic::aarch64_neon_ld4r: {
     Info.opc = ISD::INTRINSIC_W_CHAIN;
     // Conservatively set memVT to the entire set of vectors loaded.
-    uint64_t NumElts = getDataLayout()->getTypeAllocSize(I.getType()) / 8;
+    uint64_t NumElts = DL.getTypeAllocSize(I.getType()) / 8;
     Info.memVT = EVT::getVectorVT(I.getType()->getContext(), MVT::i64, NumElts);
     Info.ptrVal = I.getArgOperand(I.getNumArgOperands() - 1);
     Info.offset = 0;
@@ -6467,7 +6711,7 @@ bool AArch64TargetLowering::getTgtMemIntrinsic(IntrinsicInfo &Info,
       Type *ArgTy = I.getArgOperand(ArgI)->getType();
       if (!ArgTy->isVectorTy())
         break;
-      NumElts += getDataLayout()->getTypeAllocSize(ArgTy) / 8;
+      NumElts += DL.getTypeAllocSize(ArgTy) / 8;
     }
     Info.memVT = EVT::getVectorVT(I.getType()->getContext(), MVT::i64, NumElts);
     Info.ptrVal = I.getArgOperand(I.getNumArgOperands() - 1);
@@ -6485,7 +6729,7 @@ bool AArch64TargetLowering::getTgtMemIntrinsic(IntrinsicInfo &Info,
     Info.memVT = MVT::getVT(PtrTy->getElementType());
     Info.ptrVal = I.getArgOperand(0);
     Info.offset = 0;
-    Info.align = getDataLayout()->getABITypeAlignment(PtrTy->getElementType());
+    Info.align = DL.getABITypeAlignment(PtrTy->getElementType());
     Info.vol = true;
     Info.readMem = true;
     Info.writeMem = false;
@@ -6498,7 +6742,7 @@ bool AArch64TargetLowering::getTgtMemIntrinsic(IntrinsicInfo &Info,
     Info.memVT = MVT::getVT(PtrTy->getElementType());
     Info.ptrVal = I.getArgOperand(1);
     Info.offset = 0;
-    Info.align = getDataLayout()->getABITypeAlignment(PtrTy->getElementType());
+    Info.align = DL.getABITypeAlignment(PtrTy->getElementType());
     Info.vol = true;
     Info.readMem = false;
     Info.writeMem = true;
@@ -6569,7 +6813,8 @@ bool AArch64TargetLowering::isProfitableToHoist(Instruction *I) const {
     return true;
 
   const TargetOptions &Options = getTargetMachine().Options;
-  EVT VT = getValueType(User->getOperand(0)->getType());
+  const DataLayout &DL = I->getModule()->getDataLayout();
+  EVT VT = getValueType(DL, User->getOperand(0)->getType());
 
   if (isFMAFasterThanFMulAndFAdd(VT) &&
       isOperationLegalOrCustom(ISD::FMA, VT) &&
@@ -6634,6 +6879,7 @@ bool AArch64TargetLowering::isExtFreeImpl(const Instruction *Ext) const {
       break;
     case Instruction::GetElementPtr: {
       gep_type_iterator GTI = gep_type_begin(Instr);
+      auto &DL = Ext->getModule()->getDataLayout();
       std::advance(GTI, U.getOperandNo());
       Type *IdxTy = *GTI;
       // This extension will end up with a shift because of the scaling factor.
@@ -6641,7 +6887,7 @@ bool AArch64TargetLowering::isExtFreeImpl(const Instruction *Ext) const {
       // Get the shift amount based on the scaling factor:
       // log2(sizeof(IdxTy)) - log2(8).
       uint64_t ShiftAmt =
-        countTrailingZeros(getDataLayout()->getTypeStoreSizeInBits(IdxTy)) - 3;
+          countTrailingZeros(DL.getTypeStoreSizeInBits(IdxTy)) - 3;
       // Is the constant foldable in the shift of the addressing mode?
       // I.e., shift amount is between 1 and 4 inclusive.
       if (ShiftAmt == 0 || ShiftAmt > 4)
@@ -6685,6 +6931,160 @@ bool AArch64TargetLowering::hasPairedLoad(EVT LoadedType,
   return NumBits == 32 || NumBits == 64;
 }
 
+/// \brief Lower an interleaved load into a ldN intrinsic.
+///
+/// E.g. Lower an interleaved load (Factor = 2):
+///        %wide.vec = load <8 x i32>, <8 x i32>* %ptr
+///        %v0 = shuffle %wide.vec, undef, <0, 2, 4, 6>  ; Extract even elements
+///        %v1 = shuffle %wide.vec, undef, <1, 3, 5, 7>  ; Extract odd elements
+///
+///      Into:
+///        %ld2 = { <4 x i32>, <4 x i32> } call llvm.aarch64.neon.ld2(%ptr)
+///        %vec0 = extractelement { <4 x i32>, <4 x i32> } %ld2, i32 0
+///        %vec1 = extractelement { <4 x i32>, <4 x i32> } %ld2, i32 1
+bool AArch64TargetLowering::lowerInterleavedLoad(
+    LoadInst *LI, ArrayRef<ShuffleVectorInst *> Shuffles,
+    ArrayRef<unsigned> Indices, unsigned Factor) const {
+  assert(Factor >= 2 && Factor <= getMaxSupportedInterleaveFactor() &&
+         "Invalid interleave factor");
+  assert(!Shuffles.empty() && "Empty shufflevector input");
+  assert(Shuffles.size() == Indices.size() &&
+         "Unmatched number of shufflevectors and indices");
+
+  const DataLayout &DL = LI->getModule()->getDataLayout();
+
+  VectorType *VecTy = Shuffles[0]->getType();
+  unsigned VecSize = DL.getTypeAllocSizeInBits(VecTy);
+
+  // Skip illegal vector types.
+  if (VecSize != 64 && VecSize != 128)
+    return false;
+
+  // A pointer vector can not be the return type of the ldN intrinsics. Need to
+  // load integer vectors first and then convert to pointer vectors.
+  Type *EltTy = VecTy->getVectorElementType();
+  if (EltTy->isPointerTy())
+    VecTy =
+        VectorType::get(DL.getIntPtrType(EltTy), VecTy->getVectorNumElements());
+
+  Type *PtrTy = VecTy->getPointerTo(LI->getPointerAddressSpace());
+  Type *Tys[2] = {VecTy, PtrTy};
+  static const Intrinsic::ID LoadInts[3] = {Intrinsic::aarch64_neon_ld2,
+                                            Intrinsic::aarch64_neon_ld3,
+                                            Intrinsic::aarch64_neon_ld4};
+  Function *LdNFunc =
+      Intrinsic::getDeclaration(LI->getModule(), LoadInts[Factor - 2], Tys);
+
+  IRBuilder<> Builder(LI);
+  Value *Ptr = Builder.CreateBitCast(LI->getPointerOperand(), PtrTy);
+
+  CallInst *LdN = Builder.CreateCall(LdNFunc, Ptr, "ldN");
+
+  // Replace uses of each shufflevector with the corresponding vector loaded
+  // by ldN.
+  for (unsigned i = 0; i < Shuffles.size(); i++) {
+    ShuffleVectorInst *SVI = Shuffles[i];
+    unsigned Index = Indices[i];
+
+    Value *SubVec = Builder.CreateExtractValue(LdN, Index);
+
+    // Convert the integer vector to pointer vector if the element is pointer.
+    if (EltTy->isPointerTy())
+      SubVec = Builder.CreateIntToPtr(SubVec, SVI->getType());
+
+    SVI->replaceAllUsesWith(SubVec);
+  }
+
+  return true;
+}
+
+/// \brief Get a mask consisting of sequential integers starting from \p Start.
+///
+/// I.e. <Start, Start + 1, ..., Start + NumElts - 1>
+static Constant *getSequentialMask(IRBuilder<> &Builder, unsigned Start,
+                                   unsigned NumElts) {
+  SmallVector<Constant *, 16> Mask;
+  for (unsigned i = 0; i < NumElts; i++)
+    Mask.push_back(Builder.getInt32(Start + i));
+
+  return ConstantVector::get(Mask);
+}
+
+/// \brief Lower an interleaved store into a stN intrinsic.
+///
+/// E.g. Lower an interleaved store (Factor = 3):
+///        %i.vec = shuffle <8 x i32> %v0, <8 x i32> %v1,
+///                                  <0, 4, 8, 1, 5, 9, 2, 6, 10, 3, 7, 11>
+///        store <12 x i32> %i.vec, <12 x i32>* %ptr
+///
+///      Into:
+///        %sub.v0 = shuffle <8 x i32> %v0, <8 x i32> v1, <0, 1, 2, 3>
+///        %sub.v1 = shuffle <8 x i32> %v0, <8 x i32> v1, <4, 5, 6, 7>
+///        %sub.v2 = shuffle <8 x i32> %v0, <8 x i32> v1, <8, 9, 10, 11>
+///        call void llvm.aarch64.neon.st3(%sub.v0, %sub.v1, %sub.v2, %ptr)
+///
+/// Note that the new shufflevectors will be removed and we'll only generate one
+/// st3 instruction in CodeGen.
+bool AArch64TargetLowering::lowerInterleavedStore(StoreInst *SI,
+                                                  ShuffleVectorInst *SVI,
+                                                  unsigned Factor) const {
+  assert(Factor >= 2 && Factor <= getMaxSupportedInterleaveFactor() &&
+         "Invalid interleave factor");
+
+  VectorType *VecTy = SVI->getType();
+  assert(VecTy->getVectorNumElements() % Factor == 0 &&
+         "Invalid interleaved store");
+
+  unsigned NumSubElts = VecTy->getVectorNumElements() / Factor;
+  Type *EltTy = VecTy->getVectorElementType();
+  VectorType *SubVecTy = VectorType::get(EltTy, NumSubElts);
+
+  const DataLayout &DL = SI->getModule()->getDataLayout();
+  unsigned SubVecSize = DL.getTypeAllocSizeInBits(SubVecTy);
+
+  // Skip illegal vector types.
+  if (SubVecSize != 64 && SubVecSize != 128)
+    return false;
+
+  Value *Op0 = SVI->getOperand(0);
+  Value *Op1 = SVI->getOperand(1);
+  IRBuilder<> Builder(SI);
+
+  // StN intrinsics don't support pointer vectors as arguments. Convert pointer
+  // vectors to integer vectors.
+  if (EltTy->isPointerTy()) {
+    Type *IntTy = DL.getIntPtrType(EltTy);
+    unsigned NumOpElts =
+        dyn_cast<VectorType>(Op0->getType())->getVectorNumElements();
+
+    // Convert to the corresponding integer vector.
+    Type *IntVecTy = VectorType::get(IntTy, NumOpElts);
+    Op0 = Builder.CreatePtrToInt(Op0, IntVecTy);
+    Op1 = Builder.CreatePtrToInt(Op1, IntVecTy);
+
+    SubVecTy = VectorType::get(IntTy, NumSubElts);
+  }
+
+  Type *PtrTy = SubVecTy->getPointerTo(SI->getPointerAddressSpace());
+  Type *Tys[2] = {SubVecTy, PtrTy};
+  static const Intrinsic::ID StoreInts[3] = {Intrinsic::aarch64_neon_st2,
+                                             Intrinsic::aarch64_neon_st3,
+                                             Intrinsic::aarch64_neon_st4};
+  Function *StNFunc =
+      Intrinsic::getDeclaration(SI->getModule(), StoreInts[Factor - 2], Tys);
+
+  SmallVector<Value *, 5> Ops;
+
+  // Split the shufflevector operands into sub vectors for the new stN call.
+  for (unsigned i = 0; i < Factor; i++)
+    Ops.push_back(Builder.CreateShuffleVector(
+        Op0, Op1, getSequentialMask(Builder, NumSubElts * i, NumSubElts)));
+
+  Ops.push_back(Builder.CreateBitCast(SI->getPointerOperand(), PtrTy));
+  Builder.CreateCall(StNFunc, Ops);
+  return true;
+}
+
 static bool memOpAlign(unsigned DstAlign, unsigned SrcAlign,
                        unsigned AlignCheck) {
   return ((SrcAlign == 0 || SrcAlign % AlignCheck == 0) &&
@@ -6737,8 +7137,9 @@ bool AArch64TargetLowering::isLegalICmpImmediate(int64_t Immed) const {
 
 /// isLegalAddressingMode - Return true if the addressing mode represented
 /// by AM is legal for this target, for a load/store of the specified type.
-bool AArch64TargetLowering::isLegalAddressingMode(const AddrMode &AM,
-                                                  Type *Ty) const {
+bool AArch64TargetLowering::isLegalAddressingMode(const DataLayout &DL,
+                                                  const AddrMode &AM, Type *Ty,
+                                                  unsigned AS) const {
   // AArch64 has five basic addressing modes:
   //  reg
   //  reg + 9-bit signed offset
@@ -6758,7 +7159,7 @@ bool AArch64TargetLowering::isLegalAddressingMode(const AddrMode &AM,
   // i.e., reg + 0, reg + imm9, reg + SIZE_IN_BYTES * uimm12
   uint64_t NumBytes = 0;
   if (Ty->isSized()) {
-    uint64_t NumBits = getDataLayout()->getTypeSizeInBits(Ty);
+    uint64_t NumBits = DL.getTypeSizeInBits(Ty);
     NumBytes = NumBits / 8;
     if (!isPowerOf2_64(NumBits))
       NumBytes = 0;
@@ -6788,8 +7189,9 @@ bool AArch64TargetLowering::isLegalAddressingMode(const AddrMode &AM,
   return false;
 }
 
-int AArch64TargetLowering::getScalingFactorCost(const AddrMode &AM,
-                                                Type *Ty) const {
+int AArch64TargetLowering::getScalingFactorCost(const DataLayout &DL,
+                                                const AddrMode &AM, Type *Ty,
+                                                unsigned AS) const {
   // Scaling factors are not free at all.
   // Operands                     | Rt Latency
   // -------------------------------------------
@@ -6797,7 +7199,7 @@ int AArch64TargetLowering::getScalingFactorCost(const AddrMode &AM,
   // -------------------------------------------
   // Rt, [Xn, Xm, lsl #imm]       | Rn: 4 Rm: 5
   // Rt, [Xn, Wm, <extend> #imm]  |
-  if (isLegalAddressingMode(AM, Ty))
+  if (isLegalAddressingMode(DL, AM, Ty, AS))
     // Scale represents reg2 * scale, thus account for 1 if
     // it is not equal to 0 or 1.
     return AM.Scale != 0 && AM.Scale != 1;
@@ -6886,14 +7288,14 @@ static SDValue performIntegerAbsCombine(SDNode *N, SelectionDAG &DAG) {
       N1.getOpcode() == ISD::SRA && N1.getOperand(0) == N0.getOperand(0))
     if (ConstantSDNode *Y1C = dyn_cast<ConstantSDNode>(N1.getOperand(1)))
       if (Y1C->getAPIntValue() == VT.getSizeInBits() - 1) {
-        SDValue Neg = DAG.getNode(ISD::SUB, DL, VT, DAG.getConstant(0, VT),
+        SDValue Neg = DAG.getNode(ISD::SUB, DL, VT, DAG.getConstant(0, DL, VT),
                                   N0.getOperand(0));
         // Generate SUBS & CSEL.
         SDValue Cmp =
             DAG.getNode(AArch64ISD::SUBS, DL, DAG.getVTList(VT, MVT::i32),
-                        N0.getOperand(0), DAG.getConstant(0, VT));
+                        N0.getOperand(0), DAG.getConstant(0, DL, VT));
         return DAG.getNode(AArch64ISD::CSEL, DL, VT, N0.getOperand(0), Neg,
-                           DAG.getConstant(AArch64CC::PL, MVT::i32),
+                           DAG.getConstant(AArch64CC::PL, DL, MVT::i32),
                            SDValue(Cmp.getNode(), 1));
       }
   return SDValue();
@@ -6922,8 +7324,8 @@ AArch64TargetLowering::BuildSDIVPow2(SDNode *N, const APInt &Divisor,
   SDLoc DL(N);
   SDValue N0 = N->getOperand(0);
   unsigned Lg2 = Divisor.countTrailingZeros();
-  SDValue Zero = DAG.getConstant(0, VT);
-  SDValue Pow2MinusOne = DAG.getConstant((1ULL << Lg2) - 1, VT);
+  SDValue Zero = DAG.getConstant(0, DL, VT);
+  SDValue Pow2MinusOne = DAG.getConstant((1ULL << Lg2) - 1, DL, VT);
 
   // Add (N0 < 0) ? Pow2 - 1 : 0;
   SDValue CCVal;
@@ -6939,7 +7341,7 @@ AArch64TargetLowering::BuildSDIVPow2(SDNode *N, const APInt &Divisor,
 
   // Divide by pow2.
   SDValue SRA =
-      DAG.getNode(ISD::SRA, DL, VT, CSel, DAG.getConstant(Lg2, MVT::i64));
+      DAG.getNode(ISD::SRA, DL, VT, CSel, DAG.getConstant(Lg2, DL, MVT::i64));
 
   // If we're dividing by a positive value, we're done.  Otherwise, we must
   // negate the result.
@@ -6948,7 +7350,7 @@ AArch64TargetLowering::BuildSDIVPow2(SDNode *N, const APInt &Divisor,
 
   if (Created)
     Created->push_back(SRA.getNode());
-  return DAG.getNode(ISD::SUB, DL, VT, DAG.getConstant(0, VT), SRA);
+  return DAG.getNode(ISD::SUB, DL, VT, DAG.getConstant(0, DL, VT), SRA);
 }
 
 static SDValue performMulCombine(SDNode *N, SelectionDAG &DAG,
@@ -6965,23 +7367,24 @@ static SDValue performMulCombine(SDNode *N, SelectionDAG &DAG,
   if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(N->getOperand(1))) {
     APInt Value = C->getAPIntValue();
     EVT VT = N->getValueType(0);
+    SDLoc DL(N);
     if (Value.isNonNegative()) {
       // (mul x, 2^N + 1) => (add (shl x, N), x)
       APInt VM1 = Value - 1;
       if (VM1.isPowerOf2()) {
         SDValue ShiftedVal =
-            DAG.getNode(ISD::SHL, SDLoc(N), VT, N->getOperand(0),
-                        DAG.getConstant(VM1.logBase2(), MVT::i64));
-        return DAG.getNode(ISD::ADD, SDLoc(N), VT, ShiftedVal,
+            DAG.getNode(ISD::SHL, DL, VT, N->getOperand(0),
+                        DAG.getConstant(VM1.logBase2(), DL, MVT::i64));
+        return DAG.getNode(ISD::ADD, DL, VT, ShiftedVal,
                            N->getOperand(0));
       }
       // (mul x, 2^N - 1) => (sub (shl x, N), x)
       APInt VP1 = Value + 1;
       if (VP1.isPowerOf2()) {
         SDValue ShiftedVal =
-            DAG.getNode(ISD::SHL, SDLoc(N), VT, N->getOperand(0),
-                        DAG.getConstant(VP1.logBase2(), MVT::i64));
-        return DAG.getNode(ISD::SUB, SDLoc(N), VT, ShiftedVal,
+            DAG.getNode(ISD::SHL, DL, VT, N->getOperand(0),
+                        DAG.getConstant(VP1.logBase2(), DL, MVT::i64));
+        return DAG.getNode(ISD::SUB, DL, VT, ShiftedVal,
                            N->getOperand(0));
       }
     } else {
@@ -6989,20 +7392,20 @@ static SDValue performMulCombine(SDNode *N, SelectionDAG &DAG,
       APInt VNP1 = -Value + 1;
       if (VNP1.isPowerOf2()) {
         SDValue ShiftedVal =
-            DAG.getNode(ISD::SHL, SDLoc(N), VT, N->getOperand(0),
-                        DAG.getConstant(VNP1.logBase2(), MVT::i64));
-        return DAG.getNode(ISD::SUB, SDLoc(N), VT, N->getOperand(0),
+            DAG.getNode(ISD::SHL, DL, VT, N->getOperand(0),
+                        DAG.getConstant(VNP1.logBase2(), DL, MVT::i64));
+        return DAG.getNode(ISD::SUB, DL, VT, N->getOperand(0),
                            ShiftedVal);
       }
       // (mul x, -(2^N + 1)) => - (add (shl x, N), x)
       APInt VNM1 = -Value - 1;
       if (VNM1.isPowerOf2()) {
         SDValue ShiftedVal =
-            DAG.getNode(ISD::SHL, SDLoc(N), VT, N->getOperand(0),
-                        DAG.getConstant(VNM1.logBase2(), MVT::i64));
+            DAG.getNode(ISD::SHL, DL, VT, N->getOperand(0),
+                        DAG.getConstant(VNM1.logBase2(), DL, MVT::i64));
         SDValue Add =
-            DAG.getNode(ISD::ADD, SDLoc(N), VT, ShiftedVal, N->getOperand(0));
-        return DAG.getNode(ISD::SUB, SDLoc(N), VT, DAG.getConstant(0, VT), Add);
+            DAG.getNode(ISD::ADD, DL, VT, ShiftedVal, N->getOperand(0));
+        return DAG.getNode(ISD::SUB, DL, VT, DAG.getConstant(0, DL, VT), Add);
       }
     }
   }
@@ -7157,7 +7560,7 @@ static SDValue tryCombineToEXTR(SDNode *N,
   }
 
   return DAG.getNode(AArch64ISD::EXTR, DL, VT, LHS, RHS,
-                     DAG.getConstant(ShiftRHS, MVT::i64));
+                     DAG.getConstant(ShiftRHS, DL, MVT::i64));
 }
 
 static SDValue tryCombineToBSL(SDNode *N,
@@ -7285,10 +7688,10 @@ static SDValue performBitcastCombine(SDNode *N,
   SDLoc dl(N);
   unsigned NumElements = VT.getVectorNumElements();
   if (idx) {
-    SDValue HalfIdx = DAG.getConstant(NumElements, MVT::i64);
+    SDValue HalfIdx = DAG.getConstant(NumElements, dl, MVT::i64);
     return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, VT, Source, HalfIdx);
   } else {
-    SDValue SubReg = DAG.getTargetConstant(AArch64::dsub, MVT::i32);
+    SDValue SubReg = DAG.getTargetConstant(AArch64::dsub, dl, MVT::i32);
     return SDValue(DAG.getMachineNode(TargetOpcode::EXTRACT_SUBREG, dl, VT,
                                       Source, SubReg),
                    0);
@@ -7346,7 +7749,7 @@ static SDValue performConcatVectorsCombine(SDNode *N,
   if (N0 == N1 && VT.getVectorNumElements() == 2) {
     assert(VT.getVectorElementType().getSizeInBits() == 64);
     return DAG.getNode(AArch64ISD::DUPLANE64, dl, VT, WidenVector(N0, DAG),
-                       DAG.getConstant(0, MVT::i64));
+                       DAG.getConstant(0, dl, MVT::i64));
   }
 
   // Canonicalise concat_vectors so that the right-hand vector has as few
@@ -7436,21 +7839,26 @@ static SDValue tryCombineFixedPointConvert(SDNode *N,
 //
 // This routine does the actual conversion of such DUPs, once outer routines
 // have determined that everything else is in order.
+// It also supports immediate DUP-like nodes (MOVI/MVNi), which we can fold
+// similarly here.
 static SDValue tryExtendDUPToExtractHigh(SDValue N, SelectionDAG &DAG) {
-  // We can handle most types of duplicate, but the lane ones have an extra
-  // operand saying *which* lane, so we need to know.
-  bool IsDUPLANE;
   switch (N.getOpcode()) {
   case AArch64ISD::DUP:
-    IsDUPLANE = false;
-    break;
   case AArch64ISD::DUPLANE8:
   case AArch64ISD::DUPLANE16:
   case AArch64ISD::DUPLANE32:
   case AArch64ISD::DUPLANE64:
-    IsDUPLANE = true;
+  case AArch64ISD::MOVI:
+  case AArch64ISD::MOVIshift:
+  case AArch64ISD::MOVIedit:
+  case AArch64ISD::MOVImsl:
+  case AArch64ISD::MVNIshift:
+  case AArch64ISD::MVNImsl:
     break;
   default:
+    // FMOV could be supported, but isn't very useful, as it would only occur
+    // if you passed a bitcast' floating point immediate to an eligible long
+    // integer op (addl, smull, ...).
     return SDValue();
   }
 
@@ -7460,17 +7868,12 @@ static SDValue tryExtendDUPToExtractHigh(SDValue N, SelectionDAG &DAG) {
 
   MVT ElementTy = NarrowTy.getVectorElementType();
   unsigned NumElems = NarrowTy.getVectorNumElements();
-  MVT NewDUPVT = MVT::getVectorVT(ElementTy, NumElems * 2);
-
-  SDValue NewDUP;
-  if (IsDUPLANE)
-    NewDUP = DAG.getNode(N.getOpcode(), SDLoc(N), NewDUPVT, N.getOperand(0),
-                         N.getOperand(1));
-  else
-    NewDUP = DAG.getNode(AArch64ISD::DUP, SDLoc(N), NewDUPVT, N.getOperand(0));
+  MVT NewVT = MVT::getVectorVT(ElementTy, NumElems * 2);
 
-  return DAG.getNode(ISD::EXTRACT_SUBVECTOR, SDLoc(N.getNode()), NarrowTy,
-                     NewDUP, DAG.getConstant(NumElems, MVT::i64));
+  SDLoc dl(N);
+  return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, NarrowTy,
+                     DAG.getNode(N->getOpcode(), dl, NewVT, N->ops()),
+                     DAG.getConstant(NumElems, dl, MVT::i64));
 }
 
 static bool isEssentiallyExtractSubvector(SDValue N) {
@@ -7595,7 +7998,8 @@ static SDValue performSetccAddFolding(SDNode *Op, SelectionDAG &DAG) {
   SDLoc dl(Op);
   if (InfoAndKind.IsAArch64) {
     CCVal = DAG.getConstant(
-        AArch64CC::getInvertedCondCode(InfoAndKind.Info.AArch64.CC), MVT::i32);
+        AArch64CC::getInvertedCondCode(InfoAndKind.Info.AArch64.CC), dl,
+        MVT::i32);
     Cmp = *InfoAndKind.Info.AArch64.Cmp;
   } else
     Cmp = getAArch64Cmp(*InfoAndKind.Info.Generic.Opnd0,
@@ -7604,7 +8008,7 @@ static SDValue performSetccAddFolding(SDNode *Op, SelectionDAG &DAG) {
                       CCVal, DAG, dl);
 
   EVT VT = Op->getValueType(0);
-  LHS = DAG.getNode(ISD::ADD, dl, VT, RHS, DAG.getConstant(1, VT));
+  LHS = DAG.getNode(ISD::ADD, dl, VT, RHS, DAG.getConstant(1, dl, VT));
   return DAG.getNode(AArch64ISD::CSEL, dl, VT, RHS, LHS, CCVal, Cmp);
 }
 
@@ -7668,14 +8072,15 @@ static SDValue performAddSubLongCombine(SDNode *N,
 //   (aarch64_neon_umull (extract_high (v2i64 vec)))
 //                     (extract_high (v2i64 (dup128 scalar)))))
 //
-static SDValue tryCombineLongOpWithDup(unsigned IID, SDNode *N,
+static SDValue tryCombineLongOpWithDup(SDNode *N,
                                        TargetLowering::DAGCombinerInfo &DCI,
                                        SelectionDAG &DAG) {
   if (DCI.isBeforeLegalizeOps())
     return SDValue();
 
-  SDValue LHS = N->getOperand(1);
-  SDValue RHS = N->getOperand(2);
+  bool IsIntrinsic = N->getOpcode() == ISD::INTRINSIC_WO_CHAIN;
+  SDValue LHS = N->getOperand(IsIntrinsic ? 1 : 0);
+  SDValue RHS = N->getOperand(IsIntrinsic ? 2 : 1);
   assert(LHS.getValueType().is64BitVector() &&
          RHS.getValueType().is64BitVector() &&
          "unexpected shape for long operation");
@@ -7693,8 +8098,13 @@ static SDValue tryCombineLongOpWithDup(unsigned IID, SDNode *N,
       return SDValue();
   }
 
-  return DAG.getNode(ISD::INTRINSIC_WO_CHAIN, SDLoc(N), N->getValueType(0),
-                     N->getOperand(0), LHS, RHS);
+  // N could either be an intrinsic or a sabsdiff/uabsdiff node.
+  if (IsIntrinsic)
+    return DAG.getNode(ISD::INTRINSIC_WO_CHAIN, SDLoc(N), N->getValueType(0),
+                       N->getOperand(0), LHS, RHS);
+  else
+    return DAG.getNode(N->getOpcode(), SDLoc(N), N->getValueType(0),
+                       LHS, RHS);
 }
 
 static SDValue tryCombineShiftImm(unsigned IID, SDNode *N, SelectionDAG &DAG) {
@@ -7744,12 +8154,15 @@ static SDValue tryCombineShiftImm(unsigned IID, SDNode *N, SelectionDAG &DAG) {
     break;
   }
 
-  if (IsRightShift && ShiftAmount <= -1 && ShiftAmount >= -(int)ElemBits)
-    return DAG.getNode(Opcode, SDLoc(N), N->getValueType(0), N->getOperand(1),
-                       DAG.getConstant(-ShiftAmount, MVT::i32));
-  else if (!IsRightShift && ShiftAmount >= 0 && ShiftAmount < ElemBits)
-    return DAG.getNode(Opcode, SDLoc(N), N->getValueType(0), N->getOperand(1),
-                       DAG.getConstant(ShiftAmount, MVT::i32));
+  if (IsRightShift && ShiftAmount <= -1 && ShiftAmount >= -(int)ElemBits) {
+    SDLoc dl(N);
+    return DAG.getNode(Opcode, dl, N->getValueType(0), N->getOperand(1),
+                       DAG.getConstant(-ShiftAmount, dl, MVT::i32));
+  } else if (!IsRightShift && ShiftAmount >= 0 && ShiftAmount < ElemBits) {
+    SDLoc dl(N);
+    return DAG.getNode(Opcode, dl, N->getValueType(0), N->getOperand(1),
+                       DAG.getConstant(ShiftAmount, dl, MVT::i32));
+  }
 
   return SDValue();
 }
@@ -7772,11 +8185,12 @@ static SDValue tryCombineCRC32(unsigned Mask, SDNode *N, SelectionDAG &DAG) {
 
 static SDValue combineAcrossLanesIntrinsic(unsigned Opc, SDNode *N,
                                            SelectionDAG &DAG) {
-  return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SDLoc(N), N->getValueType(0),
-                     DAG.getNode(Opc, SDLoc(N),
+  SDLoc dl(N);
+  return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, N->getValueType(0),
+                     DAG.getNode(Opc, dl,
                                  N->getOperand(1).getSimpleValueType(),
                                  N->getOperand(1)),
-                     DAG.getConstant(0, MVT::i64));
+                     DAG.getConstant(0, dl, MVT::i64));
 }
 
 static SDValue performIntrinsicCombine(SDNode *N,
@@ -7809,11 +8223,17 @@ static SDValue performIntrinsicCombine(SDNode *N,
   case Intrinsic::aarch64_neon_fmin:
     return DAG.getNode(AArch64ISD::FMIN, SDLoc(N), N->getValueType(0),
                        N->getOperand(1), N->getOperand(2));
+  case Intrinsic::aarch64_neon_sabd:
+    return DAG.getNode(ISD::SABSDIFF, SDLoc(N), N->getValueType(0),
+                       N->getOperand(1), N->getOperand(2));
+  case Intrinsic::aarch64_neon_uabd:
+    return DAG.getNode(ISD::UABSDIFF, SDLoc(N), N->getValueType(0),
+                       N->getOperand(1), N->getOperand(2));
   case Intrinsic::aarch64_neon_smull:
   case Intrinsic::aarch64_neon_umull:
   case Intrinsic::aarch64_neon_pmull:
   case Intrinsic::aarch64_neon_sqdmull:
-    return tryCombineLongOpWithDup(IID, N, DCI, DAG);
+    return tryCombineLongOpWithDup(N, DCI, DAG);
   case Intrinsic::aarch64_neon_sqshl:
   case Intrinsic::aarch64_neon_uqshl:
   case Intrinsic::aarch64_neon_sqshlu:
@@ -7838,18 +8258,15 @@ static SDValue performExtendCombine(SDNode *N,
   // helps the backend to decide that an sabdl2 would be useful, saving a real
   // extract_high operation.
   if (!DCI.isBeforeLegalizeOps() && N->getOpcode() == ISD::ZERO_EXTEND &&
-      N->getOperand(0).getOpcode() == ISD::INTRINSIC_WO_CHAIN) {
+      (N->getOperand(0).getOpcode() == ISD::SABSDIFF ||
+       N->getOperand(0).getOpcode() == ISD::UABSDIFF)) {
     SDNode *ABDNode = N->getOperand(0).getNode();
-    unsigned IID = getIntrinsicID(ABDNode);
-    if (IID == Intrinsic::aarch64_neon_sabd ||
-        IID == Intrinsic::aarch64_neon_uabd) {
-      SDValue NewABD = tryCombineLongOpWithDup(IID, ABDNode, DCI, DAG);
-      if (!NewABD.getNode())
-        return SDValue();
+    SDValue NewABD = tryCombineLongOpWithDup(ABDNode, DCI, DAG);
+    if (!NewABD.getNode())
+      return SDValue();
 
-      return DAG.getNode(ISD::ZERO_EXTEND, SDLoc(N), N->getValueType(0),
-                         NewABD);
-    }
+    return DAG.getNode(ISD::ZERO_EXTEND, SDLoc(N), N->getValueType(0),
+                       NewABD);
   }
 
   // This is effectively a custom type legalization for AArch64.
@@ -7917,9 +8334,9 @@ static SDValue performExtendCombine(SDNode *N,
   EVT InNVT = EVT::getVectorVT(*DAG.getContext(), SrcVT.getVectorElementType(),
                                LoVT.getVectorNumElements());
   Lo = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, InNVT, Src,
-                   DAG.getConstant(0, MVT::i64));
+                   DAG.getConstant(0, DL, MVT::i64));
   Hi = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, InNVT, Src,
-                   DAG.getConstant(InNVT.getVectorNumElements(), MVT::i64));
+                   DAG.getConstant(InNVT.getVectorNumElements(), DL, MVT::i64));
   Lo = DAG.getNode(N->getOpcode(), DL, LoVT, Lo);
   Hi = DAG.getNode(N->getOpcode(), DL, HiVT, Hi);
 
@@ -7979,7 +8396,7 @@ static SDValue replaceSplatVectorStore(SelectionDAG &DAG, StoreSDNode *St) {
   unsigned Offset = EltOffset;
   while (--NumVecElts) {
     SDValue OffsetPtr = DAG.getNode(ISD::ADD, DL, MVT::i64, BasePtr,
-                                    DAG.getConstant(Offset, MVT::i64));
+                                    DAG.getConstant(Offset, DL, MVT::i64));
     NewST1 = DAG.getStore(NewST1.getValue(0), DL, SplatVal, OffsetPtr,
                           St->getPointerInfo(), St->isVolatile(),
                           St->isNonTemporal(), Alignment);
@@ -8040,15 +8457,15 @@ static SDValue performSTORECombine(SDNode *N,
   EVT HalfVT =
       EVT::getVectorVT(*DAG.getContext(), VT.getVectorElementType(), NumElts);
   SDValue SubVector0 = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, HalfVT, StVal,
-                                   DAG.getConstant(0, MVT::i64));
+                                   DAG.getConstant(0, DL, MVT::i64));
   SDValue SubVector1 = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, HalfVT, StVal,
-                                   DAG.getConstant(NumElts, MVT::i64));
+                                   DAG.getConstant(NumElts, DL, MVT::i64));
   SDValue BasePtr = S->getBasePtr();
   SDValue NewST1 =
       DAG.getStore(S->getChain(), DL, SubVector0, BasePtr, S->getPointerInfo(),
                    S->isVolatile(), S->isNonTemporal(), S->getAlignment());
   SDValue OffsetPtr = DAG.getNode(ISD::ADD, DL, MVT::i64, BasePtr,
-                                  DAG.getConstant(8, MVT::i64));
+                                  DAG.getConstant(8, DL, MVT::i64));
   return DAG.getStore(NewST1.getValue(0), DL, SubVector1, OffsetPtr,
                       S->getPointerInfo(), S->isVolatile(), S->isNonTemporal(),
                       S->getAlignment());
@@ -8116,6 +8533,13 @@ static SDValue performPostLD1Combine(SDNode *N,
       Inc = DAG.getRegister(AArch64::XZR, MVT::i64);
     }
 
+    // Finally, check that the vector doesn't depend on the load.
+    // Again, this would create a cycle.
+    // The load depending on the vector is fine, as that's the case for the
+    // LD1*post we'll eventually generate anyway.
+    if (LoadSDN->isPredecessorOf(Vector.getNode()))
+      continue;
+
     SmallVector<SDValue, 8> Ops;
     Ops.push_back(LD->getOperand(0));  // Chain
     if (IsLaneOp) {
@@ -8627,13 +9051,21 @@ static SDValue performVSelectCombine(SDNode *N, SelectionDAG &DAG) {
 /// the compare-mask instructions rather than going via NZCV, even if LHS and
 /// RHS are really scalar. This replaces any scalar setcc in the above pattern
 /// with a vector one followed by a DUP shuffle on the result.
-static SDValue performSelectCombine(SDNode *N, SelectionDAG &DAG) {
+static SDValue performSelectCombine(SDNode *N,
+                                    TargetLowering::DAGCombinerInfo &DCI) {
+  SelectionDAG &DAG = DCI.DAG;
   SDValue N0 = N->getOperand(0);
   EVT ResVT = N->getValueType(0);
 
-  if (N0.getOpcode() != ISD::SETCC || N0.getValueType() != MVT::i1)
+  if (N0.getOpcode() != ISD::SETCC)
     return SDValue();
 
+  // Make sure the SETCC result is either i1 (initial DAG), or i32, the lowered
+  // scalar SetCCResultType. We also don't expect vectors, because we assume
+  // that selects fed by vector SETCCs are canonicalized to VSELECT.
+  assert((N0.getValueType() == MVT::i1 || N0.getValueType() == MVT::i32) &&
+         "Scalar-SETCC feeding SELECT has unexpected result type!");
+
   // If NumMaskElts == 0, the comparison is larger than select result. The
   // largest real NEON comparison is 64-bits per lane, which means the result is
   // at most 32-bits and an illegal vector. Just bail out for now.
@@ -8651,6 +9083,16 @@ static SDValue performSelectCombine(SDNode *N, SelectionDAG &DAG) {
   SrcVT = EVT::getVectorVT(*DAG.getContext(), SrcVT, NumMaskElts);
   EVT CCVT = SrcVT.changeVectorElementTypeToInteger();
 
+  // Also bail out if the vector CCVT isn't the same size as ResVT.
+  // This can happen if the SETCC operand size doesn't divide the ResVT size
+  // (e.g., f64 vs v3f32).
+  if (CCVT.getSizeInBits() != ResVT.getSizeInBits())
+    return SDValue();
+
+  // Make sure we didn't create illegal types, if we're not supposed to.
+  assert(DCI.isBeforeLegalize() ||
+         DAG.getTargetLoweringInfo().isTypeLegal(SrcVT));
+
   // First perform a vector comparison, where lane 0 is the one we're interested
   // in.
   SDLoc DL(N0);
@@ -8669,6 +9111,83 @@ static SDValue performSelectCombine(SDNode *N, SelectionDAG &DAG) {
   return DAG.getSelect(DL, ResVT, Mask, N->getOperand(1), N->getOperand(2));
 }
 
+/// performSelectCCCombine - Target-specific DAG combining for ISD::SELECT_CC
+/// to match FMIN/FMAX patterns.
+static SDValue performSelectCCCombine(SDNode *N, SelectionDAG &DAG) {
+  // Try to use FMIN/FMAX instructions for FP selects like "x < y ? x : y".
+  // Unless the NoNaNsFPMath option is set, be careful about NaNs:
+  // vmax/vmin return NaN if either operand is a NaN;
+  // only do the transformation when it matches that behavior.
+
+  SDValue CondLHS = N->getOperand(0);
+  SDValue CondRHS = N->getOperand(1);
+  SDValue LHS = N->getOperand(2);
+  SDValue RHS = N->getOperand(3);
+  ISD::CondCode CC = cast<CondCodeSDNode>(N->getOperand(4))->get();
+
+  unsigned Opcode;
+  bool IsReversed;
+  if (selectCCOpsAreFMaxCompatible(CondLHS, LHS) &&
+      selectCCOpsAreFMaxCompatible(CondRHS, RHS)) {
+    IsReversed = false; // x CC y ? x : y
+  } else if (selectCCOpsAreFMaxCompatible(CondRHS, LHS) &&
+             selectCCOpsAreFMaxCompatible(CondLHS, RHS)) {
+    IsReversed = true ; // x CC y ? y : x
+  } else {
+    return SDValue();
+  }
+
+  bool IsUnordered = false, IsOrEqual;
+  switch (CC) {
+  default:
+    return SDValue();
+  case ISD::SETULT:
+  case ISD::SETULE:
+    IsUnordered = true;
+  case ISD::SETOLT:
+  case ISD::SETOLE:
+  case ISD::SETLT:
+  case ISD::SETLE:
+    IsOrEqual = (CC == ISD::SETLE || CC == ISD::SETOLE || CC == ISD::SETULE);
+    Opcode = IsReversed ? AArch64ISD::FMAX : AArch64ISD::FMIN;
+    break;
+
+  case ISD::SETUGT:
+  case ISD::SETUGE:
+    IsUnordered = true;
+  case ISD::SETOGT:
+  case ISD::SETOGE:
+  case ISD::SETGT:
+  case ISD::SETGE:
+    IsOrEqual = (CC == ISD::SETGE || CC == ISD::SETOGE || CC == ISD::SETUGE);
+    Opcode = IsReversed ? AArch64ISD::FMIN : AArch64ISD::FMAX;
+    break;
+  }
+
+  // If LHS is NaN, an ordered comparison will be false and the result will be
+  // the RHS, but FMIN(NaN, RHS) = FMAX(NaN, RHS) = NaN. Avoid this by checking
+  // that LHS != NaN. Likewise, for unordered comparisons, check for RHS != NaN.
+  if (!DAG.isKnownNeverNaN(IsUnordered ? RHS : LHS))
+    return SDValue();
+
+  // For xxx-or-equal comparisons, "+0 <= -0" and "-0 >= +0" will both be true,
+  // but FMIN will return -0, and FMAX will return +0. So FMIN/FMAX can only be
+  // used for unsafe math or if one of the operands is known to be nonzero.
+  if (IsOrEqual && !DAG.getTarget().Options.UnsafeFPMath &&
+      !(DAG.isKnownNeverZero(LHS) || DAG.isKnownNeverZero(RHS)))
+    return SDValue();
+
+  return DAG.getNode(Opcode, SDLoc(N), N->getValueType(0), LHS, RHS);
+}
+
+/// Get rid of unnecessary NVCASTs (that don't change the type).
+static SDValue performNVCASTCombine(SDNode *N) {
+  if (N->getValueType(0) == N->getOperand(0).getValueType())
+    return N->getOperand(0);
+
+  return SDValue();
+}
+
 SDValue AArch64TargetLowering::PerformDAGCombine(SDNode *N,
                                                  DAGCombinerInfo &DCI) const {
   SelectionDAG &DAG = DCI.DAG;
@@ -8698,9 +9217,11 @@ SDValue AArch64TargetLowering::PerformDAGCombine(SDNode *N,
   case ISD::CONCAT_VECTORS:
     return performConcatVectorsCombine(N, DCI, DAG);
   case ISD::SELECT:
-    return performSelectCombine(N, DAG);
+    return performSelectCombine(N, DCI);
   case ISD::VSELECT:
     return performVSelectCombine(N, DCI.DAG);
+  case ISD::SELECT_CC:
+    return performSelectCCCombine(N, DCI.DAG);
   case ISD::STORE:
     return performSTORECombine(N, DCI, DAG, Subtarget);
   case AArch64ISD::BRCOND:
@@ -8709,6 +9230,8 @@ SDValue AArch64TargetLowering::PerformDAGCombine(SDNode *N,
     return performCONDCombine(N, DCI, DAG, 2, 3);
   case AArch64ISD::DUP:
     return performPostLD1Combine(N, DCI, false);
+  case AArch64ISD::NVCAST:
+    return performNVCASTCombine(N);
   case ISD::INSERT_VECTOR_ELT:
     return performPostLD1Combine(N, DCI, true);
   case ISD::INTRINSIC_VOID:
@@ -8871,7 +9394,7 @@ static void ReplaceBITCASTResults(SDNode *N, SmallVectorImpl<SDValue> &Results,
   Op = SDValue(
       DAG.getMachineNode(TargetOpcode::INSERT_SUBREG, DL, MVT::f32,
                          DAG.getUNDEF(MVT::i32), Op,
-                         DAG.getTargetConstant(AArch64::hsub, MVT::i32)),
+                         DAG.getTargetConstant(AArch64::hsub, DL, MVT::i32)),
       0);
   Op = DAG.getNode(ISD::BITCAST, DL, MVT::i32, Op);
   Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, MVT::i16, Op));
@@ -8897,10 +9420,10 @@ bool AArch64TargetLowering::useLoadStackGuardNode() const {
   return true;
 }
 
-bool AArch64TargetLowering::combineRepeatedFPDivisors(unsigned NumUsers) const {
+unsigned AArch64TargetLowering::combineRepeatedFPDivisors() const {
   // Combine multiple FDIVs with the same divisor into multiple FMULs by the
   // reciprocal if there are three or more FDIVs.
-  return NumUsers > 2;
+  return 3;
 }
 
 TargetLoweringBase::LegalizeTypeAction
@@ -8996,7 +9519,7 @@ Value *AArch64TargetLowering::emitStoreConditional(IRBuilder<> &Builder,
     Value *Lo = Builder.CreateTrunc(Val, Int64Ty, "lo");
     Value *Hi = Builder.CreateTrunc(Builder.CreateLShr(Val, 64), Int64Ty, "hi");
     Addr = Builder.CreateBitCast(Addr, Type::getInt8PtrTy(M->getContext()));
-    return Builder.CreateCall3(Stxr, Lo, Hi, Addr);
+    return Builder.CreateCall(Stxr, {Lo, Hi, Addr});
   }
 
   Intrinsic::ID Int =
@@ -9004,13 +9527,18 @@ Value *AArch64TargetLowering::emitStoreConditional(IRBuilder<> &Builder,
   Type *Tys[] = { Addr->getType() };
   Function *Stxr = Intrinsic::getDeclaration(M, Int, Tys);
 
-  return Builder.CreateCall2(
-      Stxr, Builder.CreateZExtOrBitCast(
-                Val, Stxr->getFunctionType()->getParamType(0)),
-      Addr);
+  return Builder.CreateCall(Stxr,
+                            {Builder.CreateZExtOrBitCast(
+                                 Val, Stxr->getFunctionType()->getParamType(0)),
+                             Addr});
 }
 
 bool AArch64TargetLowering::functionArgumentNeedsConsecutiveRegisters(
     Type *Ty, CallingConv::ID CallConv, bool isVarArg) const {
   return Ty->isArrayTy();
 }
+
+bool AArch64TargetLowering::shouldNormalizeToSelectSequence(LLVMContext &,
+                                                            EVT) const {
+  return false;
+}
diff --git a/lib/Target/AArch64/AArch64ISelLowering.h b/lib/Target/AArch64/AArch64ISelLowering.h
index 820613b917d..6b71ae42c59 100644
--- a/lib/Target/AArch64/AArch64ISelLowering.h
+++ b/lib/Target/AArch64/AArch64ISelLowering.h
@@ -25,7 +25,7 @@ namespace llvm {
 
 namespace AArch64ISD {
 
-enum {
+enum NodeType : unsigned {
   FIRST_NUMBER = ISD::BUILTIN_OP_END,
   WrapperLarge, // 4-instruction MOVZ/MOVK sequence for 64-bit addresses.
   CALL,         // Function call.
@@ -58,6 +58,11 @@ enum {
   SBCS,
   ANDS,
 
+  // Conditional compares. Operands: left,right,falsecc,cc,flags
+  CCMP,
+  CCMN,
+  FCCMP,
+
   // Floating point comparison
   FCMP,
 
@@ -217,8 +222,6 @@ class AArch64Subtarget;
 class AArch64TargetMachine;
 
 class AArch64TargetLowering : public TargetLowering {
-  bool RequireStrictAlign;
-
 public:
   explicit AArch64TargetLowering(const TargetMachine &TM,
                                  const AArch64Subtarget &STI);
@@ -233,20 +236,13 @@ class AArch64TargetLowering : public TargetLowering {
                                      APInt &KnownOne, const SelectionDAG &DAG,
                                      unsigned Depth = 0) const override;
 
-  MVT getScalarShiftAmountTy(EVT LHSTy) const override;
+  MVT getScalarShiftAmountTy(const DataLayout &DL, EVT) const override;
 
   /// allowsMisalignedMemoryAccesses - Returns true if the target allows
   /// unaligned memory accesses of the specified type.
   bool allowsMisalignedMemoryAccesses(EVT VT, unsigned AddrSpace = 0,
                                       unsigned Align = 1,
-                                      bool *Fast = nullptr) const override {
-    if (RequireStrictAlign)
-      return false;
-    // FIXME: True for Cyclone, but not necessary others.
-    if (Fast)
-      *Fast = true;
-    return true;
-  }
+                                      bool *Fast = nullptr) const override;
 
   /// LowerOperation - Provide custom lowering hooks for some operations.
   SDValue LowerOperation(SDValue Op, SelectionDAG &DAG) const override;
@@ -278,7 +274,8 @@ class AArch64TargetLowering : public TargetLowering {
   bool isShuffleMaskLegal(const SmallVectorImpl<int> &M, EVT VT) const override;
 
   /// getSetCCResultType - Return the ISD::SETCC ValueType
-  EVT getSetCCResultType(LLVMContext &Context, EVT VT) const override;
+  EVT getSetCCResultType(const DataLayout &DL, LLVMContext &Context,
+                         EVT VT) const override;
 
   SDValue ReconstructShuffle(SDValue Op, SelectionDAG &DAG) const;
 
@@ -305,6 +302,15 @@ class AArch64TargetLowering : public TargetLowering {
                      unsigned &RequiredAligment) const override;
   bool hasPairedLoad(EVT LoadedType, unsigned &RequiredAligment) const override;
 
+  unsigned getMaxSupportedInterleaveFactor() const override { return 4; }
+
+  bool lowerInterleavedLoad(LoadInst *LI,
+                            ArrayRef<ShuffleVectorInst *> Shuffles,
+                            ArrayRef<unsigned> Indices,
+                            unsigned Factor) const override;
+  bool lowerInterleavedStore(StoreInst *SI, ShuffleVectorInst *SVI,
+                             unsigned Factor) const override;
+
   bool isLegalAddImmediate(int64_t) const override;
   bool isLegalICmpImmediate(int64_t) const override;
 
@@ -314,14 +320,16 @@ class AArch64TargetLowering : public TargetLowering {
 
   /// isLegalAddressingMode - Return true if the addressing mode represented
   /// by AM is legal for this target, for a load/store of the specified type.
-  bool isLegalAddressingMode(const AddrMode &AM, Type *Ty) const override;
+  bool isLegalAddressingMode(const DataLayout &DL, const AddrMode &AM, Type *Ty,
+                             unsigned AS) const override;
 
   /// \brief Return the cost of the scaling factor used in the addressing
   /// mode represented by AM for this target, for a load/store
   /// of the specified type.
   /// If the AM is supported, the return value must be >= 0.
   /// If the AM is not supported, it returns a negative value.
-  int getScalingFactorCost(const AddrMode &AM, Type *Ty) const override;
+  int getScalingFactorCost(const DataLayout &DL, const AddrMode &AM, Type *Ty,
+                           unsigned AS) const override;
 
   /// isFMAFasterThanFMulAndFAdd - Return true if an FMA operation is faster
   /// than a pair of fmul and fadd instructions. fmuladd intrinsics will be
@@ -380,6 +388,8 @@ class AArch64TargetLowering : public TargetLowering {
                           SelectionDAG &DAG, SmallVectorImpl<SDValue> &InVals,
                           bool isThisReturn, SDValue ThisVal) const;
 
+  SDValue LowerINTRINSIC_WO_CHAIN(SDValue Op, SelectionDAG &DAG) const;
+
   bool isEligibleForTailCallOptimization(
       SDValue Callee, CallingConv::ID CalleeCC, bool isVarArg,
       bool isCalleeStructRet, bool isCallerStructRet,
@@ -458,11 +468,11 @@ class AArch64TargetLowering : public TargetLowering {
 
   SDValue BuildSDIVPow2(SDNode *N, const APInt &Divisor, SelectionDAG &DAG,
                         std::vector<SDNode *> *Created) const override;
-  bool combineRepeatedFPDivisors(unsigned NumUsers) const override;
+  unsigned combineRepeatedFPDivisors() const override;
 
-  ConstraintType
-  getConstraintType(const std::string &Constraint) const override;
-  unsigned getRegisterByName(const char* RegName, EVT VT) const override;
+  ConstraintType getConstraintType(StringRef Constraint) const override;
+  unsigned getRegisterByName(const char* RegName, EVT VT,
+                             SelectionDAG &DAG) const override;
 
   /// Examine constraint string and operand type and determine a weight value.
   /// The operand object must already have been set up with the operand type.
@@ -472,14 +482,12 @@ class AArch64TargetLowering : public TargetLowering {
 
   std::pair<unsigned, const TargetRegisterClass *>
   getRegForInlineAsmConstraint(const TargetRegisterInfo *TRI,
-                               const std::string &Constraint,
-                               MVT VT) const override;
+                               StringRef Constraint, MVT VT) const override;
   void LowerAsmOperandForConstraint(SDValue Op, std::string &Constraint,
                                     std::vector<SDValue> &Ops,
                                     SelectionDAG &DAG) const override;
 
-  unsigned getInlineAsmMemConstraint(
-      const std::string &ConstraintCode) const override {
+  unsigned getInlineAsmMemConstraint(StringRef ConstraintCode) const override {
     if (ConstraintCode == "Q")
       return InlineAsm::Constraint_Q;
     // FIXME: clang has code for 'Ump', 'Utf', 'Usa', and 'Ush' but these are
@@ -506,6 +514,8 @@ class AArch64TargetLowering : public TargetLowering {
   bool functionArgumentNeedsConsecutiveRegisters(Type *Ty,
                                                  CallingConv::ID CallConv,
                                                  bool isVarArg) const override;
+
+  bool shouldNormalizeToSelectSequence(LLVMContext &, EVT) const override;
 };
 
 namespace AArch64 {
diff --git a/lib/Target/AArch64/AArch64InstrFormats.td b/lib/Target/AArch64/AArch64InstrFormats.td
index 0c0efaf7c1a..123a3654c93 100644
--- a/lib/Target/AArch64/AArch64InstrFormats.td
+++ b/lib/Target/AArch64/AArch64InstrFormats.td
@@ -441,11 +441,11 @@ def vecshiftL64 : Operand<i32>, ImmLeaf<i32, [{
 // instructions for splatting repeating bit patterns across the immediate.
 def logical_imm32_XFORM : SDNodeXForm<imm, [{
   uint64_t enc = AArch64_AM::encodeLogicalImmediate(N->getZExtValue(), 32);
-  return CurDAG->getTargetConstant(enc, MVT::i32);
+  return CurDAG->getTargetConstant(enc, SDLoc(N), MVT::i32);
 }]>;
 def logical_imm64_XFORM : SDNodeXForm<imm, [{
   uint64_t enc = AArch64_AM::encodeLogicalImmediate(N->getZExtValue(), 64);
-  return CurDAG->getTargetConstant(enc, MVT::i32);
+  return CurDAG->getTargetConstant(enc, SDLoc(N), MVT::i32);
 }]>;
 
 let DiagnosticType = "LogicalSecondSource" in {
@@ -525,6 +525,13 @@ def imm0_31 : Operand<i64>, ImmLeaf<i64, [{
   let ParserMatchClass = Imm0_31Operand;
 }
 
+// True if the 32-bit immediate is in the range [0,31]
+def imm32_0_31 : Operand<i32>, ImmLeaf<i32, [{
+  return ((uint64_t)Imm) < 32;
+}]> {
+  let ParserMatchClass = Imm0_31Operand;
+}
+
 // imm0_15 predicate - True if the immediate is in the range [0,15]
 def imm0_15 : Operand<i64>, ImmLeaf<i64, [{
   return ((uint64_t)Imm) < 16;
@@ -542,7 +549,9 @@ def imm0_7 : Operand<i64>, ImmLeaf<i64, [{
 // imm32_0_15 predicate - True if the 32-bit immediate is in the range [0,15]
 def imm32_0_15 : Operand<i32>, ImmLeaf<i32, [{
   return ((uint32_t)Imm) < 16;
-}]>;
+}]> {
+  let ParserMatchClass = Imm0_15Operand;
+}
 
 // An arithmetic shifter operand:
 //  {7-6} - shift type: 00 = lsl, 01 = lsr, 10 = asr
@@ -614,10 +623,15 @@ def move_vec_shift : Operand<i32> {
   let ParserMatchClass = MoveVecShifterOperand;
 }
 
-def AddSubImmOperand : AsmOperandClass {
-  let Name = "AddSubImm";
-  let ParserMethod = "tryParseAddSubImm";
-  let DiagnosticType = "AddSubSecondSource";
+let DiagnosticType = "AddSubSecondSource" in {
+  def AddSubImmOperand : AsmOperandClass {
+    let Name = "AddSubImm";
+    let ParserMethod = "tryParseAddSubImm";
+  }
+  def AddSubImmNegOperand : AsmOperandClass {
+    let Name = "AddSubImmNeg";
+    let ParserMethod = "tryParseAddSubImm";
+  }
 }
 // An ADD/SUB immediate shifter operand:
 //  second operand:
@@ -631,8 +645,17 @@ class addsub_shifted_imm<ValueType Ty>
   let MIOperandInfo = (ops i32imm, i32imm);
 }
 
+class addsub_shifted_imm_neg<ValueType Ty>
+    : Operand<Ty> {
+  let EncoderMethod = "getAddSubImmOpValue";
+  let ParserMatchClass = AddSubImmNegOperand;
+  let MIOperandInfo = (ops i32imm, i32imm);
+}
+
 def addsub_shifted_imm32 : addsub_shifted_imm<i32>;
 def addsub_shifted_imm64 : addsub_shifted_imm<i64>;
+def addsub_shifted_imm32_neg : addsub_shifted_imm_neg<i32>;
+def addsub_shifted_imm64_neg : addsub_shifted_imm_neg<i64>;
 
 class neg_addsub_shifted_imm<ValueType Ty>
     : Operand<Ty>, ComplexPattern<Ty, 2, "SelectNegArithImmed", [imm]> {
@@ -682,7 +705,7 @@ def fpimm32 : Operand<f32>,
     }], SDNodeXForm<fpimm, [{
       APFloat InVal = N->getValueAPF();
       uint32_t enc = AArch64_AM::getFP32Imm(InVal);
-      return CurDAG->getTargetConstant(enc, MVT::i32);
+      return CurDAG->getTargetConstant(enc, SDLoc(N), MVT::i32);
     }]>> {
   let ParserMatchClass = FPImmOperand;
   let PrintMethod = "printFPImmOperand";
@@ -693,7 +716,7 @@ def fpimm64 : Operand<f64>,
     }], SDNodeXForm<fpimm, [{
       APFloat InVal = N->getValueAPF();
       uint32_t enc = AArch64_AM::getFP64Imm(InVal);
-      return CurDAG->getTargetConstant(enc, MVT::i32);
+      return CurDAG->getTargetConstant(enc, SDLoc(N), MVT::i32);
     }]>> {
   let ParserMatchClass = FPImmOperand;
   let PrintMethod = "printFPImmOperand";
@@ -768,7 +791,7 @@ def simdimmtype10 : Operand<i32>,
       uint32_t enc = AArch64_AM::encodeAdvSIMDModImmType10(N->getValueAPF()
                                                            .bitcastToAPInt()
                                                            .getZExtValue());
-      return CurDAG->getTargetConstant(enc, MVT::i32);
+      return CurDAG->getTargetConstant(enc, SDLoc(N), MVT::i32);
     }]>> {
   let ParserMatchClass = SIMDImmType10Operand;
   let PrintMethod = "printSIMDType10Operand";
@@ -899,6 +922,9 @@ class MSRpstateI
   let Inst{7-5} = pstatefield{2-0};
 
   let DecoderMethod = "DecodeSystemPStateInstruction";
+  // MSRpstateI aliases with MSRI. When the MSRpstateI decoder method returns
+  // Fail the decoder should attempt to decode the instruction as MSRI.
+  let hasCompleteDecoder = 0;
 }
 
 // SYS and SYSL generic system instructions.
@@ -1633,7 +1659,7 @@ class AddSubRegAlias<string asm, Instruction inst, RegisterClass dstRegtype,
                 (inst dstRegtype:$dst, src1Regtype:$src1, src2Regtype:$src2,
                       shiftExt)>;
 
-multiclass AddSub<bit isSub, string mnemonic,
+multiclass AddSub<bit isSub, string mnemonic, string alias,
                   SDPatternOperator OpNode = null_frag> {
   let hasSideEffects = 0, isReMaterializable = 1, isAsCheapAsAMove = 1 in {
   // Add/Subtract immediate
@@ -1686,6 +1712,14 @@ multiclass AddSub<bit isSub, string mnemonic,
     let Inst{31} = 1;
   }
 
+  // add Rd, Rb, -imm -> sub Rd, Rn, imm
+  def : InstAlias<alias#" $Rd, $Rn, $imm",
+                  (!cast<Instruction>(NAME # "Wri") GPR32sp:$Rd, GPR32sp:$Rn,
+                      addsub_shifted_imm32_neg:$imm), 0>;
+  def : InstAlias<alias#" $Rd, $Rn, $imm",
+                  (!cast<Instruction>(NAME # "Xri") GPR64sp:$Rd, GPR64sp:$Rn,
+                       addsub_shifted_imm64_neg:$imm), 0>;
+
   // Register/register aliases with no shift when SP is not used.
   def : AddSubRegAlias<mnemonic, !cast<Instruction>(NAME#"Wrs"),
                        GPR32, GPR32, GPR32, 0>;
@@ -1706,7 +1740,8 @@ multiclass AddSub<bit isSub, string mnemonic,
                        GPR64sp, GPR64sponly, GPR64, 24>; // UXTX #0
 }
 
-multiclass AddSubS<bit isSub, string mnemonic, SDNode OpNode, string cmp> {
+multiclass AddSubS<bit isSub, string mnemonic, SDNode OpNode, string cmp,
+                   string alias, string cmpAlias> {
   let isCompare = 1, Defs = [NZCV] in {
   // Add/Subtract immediate
   def Wri  : BaseAddSubImm<isSub, 1, GPR32, GPR32sp, addsub_shifted_imm32,
@@ -1752,6 +1787,14 @@ multiclass AddSubS<bit isSub, string mnemonic, SDNode OpNode, string cmp> {
   }
   } // Defs = [NZCV]
 
+  // Support negative immediates, e.g. adds Rd, Rn, -imm -> subs Rd, Rn, imm
+  def : InstAlias<alias#" $Rd, $Rn, $imm",
+                  (!cast<Instruction>(NAME # "Wri") GPR32:$Rd, GPR32sp:$Rn,
+                      addsub_shifted_imm32_neg:$imm), 0>;
+  def : InstAlias<alias#" $Rd, $Rn, $imm",
+                  (!cast<Instruction>(NAME # "Xri") GPR64:$Rd, GPR64sp:$Rn,
+                       addsub_shifted_imm64_neg:$imm), 0>;
+
   // Compare aliases
   def : InstAlias<cmp#" $src, $imm", (!cast<Instruction>(NAME#"Wri")
                   WZR, GPR32sp:$src, addsub_shifted_imm32:$imm), 5>;
@@ -1768,6 +1811,12 @@ multiclass AddSubS<bit isSub, string mnemonic, SDNode OpNode, string cmp> {
   def : InstAlias<cmp#" $src1, $src2$sh", (!cast<Instruction>(NAME#"Xrs")
                   XZR, GPR64:$src1, GPR64:$src2, arith_shift64:$sh), 4>;
 
+  // Support negative immediates, e.g. cmp Rn, -imm -> cmn Rn, imm
+  def : InstAlias<cmpAlias#" $src, $imm", (!cast<Instruction>(NAME#"Wri")
+                  WZR, GPR32sp:$src, addsub_shifted_imm32_neg:$imm), 0>;
+  def : InstAlias<cmpAlias#" $src, $imm", (!cast<Instruction>(NAME#"Xri")
+                  XZR, GPR64sp:$src, addsub_shifted_imm64_neg:$imm), 0>;
+
   // Compare shorthands
   def : InstAlias<cmp#" $src1, $src2", (!cast<Instruction>(NAME#"Wrs")
                   WZR, GPR32:$src1, GPR32:$src2, 0), 5>;
@@ -2068,9 +2117,12 @@ multiclass LogicalRegS<bits<2> opc, bit N, string mnemonic,
 //---
 
 let mayLoad = 0, mayStore = 0, hasSideEffects = 0 in
-class BaseCondSetFlagsImm<bit op, RegisterClass regtype, string asm>
-    : I<(outs), (ins regtype:$Rn, imm0_31:$imm, imm0_15:$nzcv, ccode:$cond),
-         asm, "\t$Rn, $imm, $nzcv, $cond", "", []>,
+class BaseCondComparisonImm<bit op, RegisterClass regtype, ImmLeaf immtype,
+                            string mnemonic, SDNode OpNode>
+    : I<(outs), (ins regtype:$Rn, immtype:$imm, imm32_0_15:$nzcv, ccode:$cond),
+         mnemonic, "\t$Rn, $imm, $nzcv, $cond", "",
+         [(set NZCV, (OpNode regtype:$Rn, immtype:$imm, (i32 imm:$nzcv),
+                             (i32 imm:$cond), NZCV))]>,
       Sched<[WriteI, ReadI]> {
   let Uses = [NZCV];
   let Defs = [NZCV];
@@ -2090,19 +2142,13 @@ class BaseCondSetFlagsImm<bit op, RegisterClass regtype, string asm>
   let Inst{3-0}   = nzcv;
 }
 
-multiclass CondSetFlagsImm<bit op, string asm> {
-  def Wi : BaseCondSetFlagsImm<op, GPR32, asm> {
-    let Inst{31} = 0;
-  }
-  def Xi : BaseCondSetFlagsImm<op, GPR64, asm> {
-    let Inst{31} = 1;
-  }
-}
-
 let mayLoad = 0, mayStore = 0, hasSideEffects = 0 in
-class BaseCondSetFlagsReg<bit op, RegisterClass regtype, string asm>
-    : I<(outs), (ins regtype:$Rn, regtype:$Rm, imm0_15:$nzcv, ccode:$cond),
-         asm, "\t$Rn, $Rm, $nzcv, $cond", "", []>,
+class BaseCondComparisonReg<bit op, RegisterClass regtype, string mnemonic,
+                            SDNode OpNode>
+    : I<(outs), (ins regtype:$Rn, regtype:$Rm, imm32_0_15:$nzcv, ccode:$cond),
+         mnemonic, "\t$Rn, $Rm, $nzcv, $cond", "",
+         [(set NZCV, (OpNode regtype:$Rn, regtype:$Rm, (i32 imm:$nzcv),
+                             (i32 imm:$cond), NZCV))]>,
       Sched<[WriteI, ReadI, ReadI]> {
   let Uses = [NZCV];
   let Defs = [NZCV];
@@ -2122,11 +2168,19 @@ class BaseCondSetFlagsReg<bit op, RegisterClass regtype, string asm>
   let Inst{3-0}   = nzcv;
 }
 
-multiclass CondSetFlagsReg<bit op, string asm> {
-  def Wr : BaseCondSetFlagsReg<op, GPR32, asm> {
+multiclass CondComparison<bit op, string mnemonic, SDNode OpNode> {
+  // immediate operand variants
+  def Wi : BaseCondComparisonImm<op, GPR32, imm32_0_31, mnemonic, OpNode> {
+    let Inst{31} = 0;
+  }
+  def Xi : BaseCondComparisonImm<op, GPR64, imm0_31, mnemonic, OpNode> {
+    let Inst{31} = 1;
+  }
+  // register operand variants
+  def Wr : BaseCondComparisonReg<op, GPR32, mnemonic, OpNode> {
     let Inst{31} = 0;
   }
-  def Xr : BaseCondSetFlagsReg<op, GPR64, asm> {
+  def Xr : BaseCondComparisonReg<op, GPR64, mnemonic, OpNode> {
     let Inst{31} = 1;
   }
 }
@@ -2192,7 +2246,8 @@ class BaseCondSelectOp<bit op, bits<2> op2, RegisterClass regtype, string asm,
 
 def inv_cond_XFORM : SDNodeXForm<imm, [{
   AArch64CC::CondCode CC = static_cast<AArch64CC::CondCode>(N->getZExtValue());
-  return CurDAG->getTargetConstant(AArch64CC::getInvertedCondCode(CC), MVT::i32);
+  return CurDAG->getTargetConstant(AArch64CC::getInvertedCondCode(CC), SDLoc(N),
+                                   MVT::i32);
 }]>;
 
 multiclass CondSelectOp<bit op, bits<2> op2, string asm, PatFrag frag> {
@@ -3933,11 +3988,14 @@ multiclass FPComparison<bit signalAllNans, string asm,
 //---
 
 let mayLoad = 0, mayStore = 0, hasSideEffects = 0 in
-class BaseFPCondComparison<bit signalAllNans,
-                              RegisterClass regtype, string asm>
-    : I<(outs), (ins regtype:$Rn, regtype:$Rm, imm0_15:$nzcv, ccode:$cond),
-         asm, "\t$Rn, $Rm, $nzcv, $cond", "", []>,
+class BaseFPCondComparison<bit signalAllNans, RegisterClass regtype,
+                           string mnemonic, list<dag> pat>
+    : I<(outs), (ins regtype:$Rn, regtype:$Rm, imm32_0_15:$nzcv, ccode:$cond),
+         mnemonic, "\t$Rn, $Rm, $nzcv, $cond", "", pat>,
       Sched<[WriteFCmp]> {
+  let Uses = [NZCV];
+  let Defs = [NZCV];
+
   bits<5> Rn;
   bits<5> Rm;
   bits<4> nzcv;
@@ -3953,16 +4011,18 @@ class BaseFPCondComparison<bit signalAllNans,
   let Inst{3-0}   = nzcv;
 }
 
-multiclass FPCondComparison<bit signalAllNans, string asm> {
-  let Defs = [NZCV], Uses = [NZCV] in {
-  def Srr : BaseFPCondComparison<signalAllNans, FPR32, asm> {
+multiclass FPCondComparison<bit signalAllNans, string mnemonic,
+                            SDPatternOperator OpNode = null_frag> {
+  def Srr : BaseFPCondComparison<signalAllNans, FPR32, mnemonic,
+      [(set NZCV, (OpNode (f32 FPR32:$Rn), (f32 FPR32:$Rm), (i32 imm:$nzcv),
+                          (i32 imm:$cond), NZCV))]> {
     let Inst{22} = 0;
   }
-
-  def Drr : BaseFPCondComparison<signalAllNans, FPR64, asm> {
+  def Drr : BaseFPCondComparison<signalAllNans, FPR64, mnemonic,
+      [(set NZCV, (OpNode (f64 FPR64:$Rn), (f64 FPR64:$Rm), (i32 imm:$nzcv),
+                          (i32 imm:$cond), NZCV))]> {
     let Inst{22} = 1;
   }
-  } // Defs = [NZCV], Uses = [NZCV]
 }
 
 //---
@@ -8821,6 +8881,178 @@ class SHAInstSS<bits<4> opc, string asm, Intrinsic OpNode>
                [(set (i32 FPR32:$Rd), (OpNode (i32 FPR32:$Rn)))]>;
 } // end of 'let Predicates = [HasCrypto]'
 
+//----------------------------------------------------------------------------
+// v8.1 atomic instructions extension:
+// * CAS
+// * CASP
+// * SWP
+// * LDOPregister<OP>, and aliases STOPregister<OP>
+
+// Instruction encodings:
+//
+//      31 30|29  24|23|22|21|20 16|15|14  10|9 5|4 0
+// CAS  SZ   |001000|1 |A |1 |Rs   |R |11111 |Rn |Rt
+// CASP  0|SZ|001000|0 |A |1 |Rs   |R |11111 |Rn |Rt
+// SWP  SZ   |111000|A |R |1 |Rs   |1 |OPC|00|Rn |Rt
+// LD   SZ   |111000|A |R |1 |Rs   |0 |OPC|00|Rn |Rt
+// ST   SZ   |111000|A |R |1 |Rs   |0 |OPC|00|Rn |11111
+
+// Instruction syntax:
+//
+// CAS{<order>}[<size>] <Ws>, <Wt>, [<Xn|SP>]
+// CAS{<order>} <Xs>, <Xt>, [<Xn|SP>]
+// CASP{<order>} <Ws>, <W(s+1)>, <Wt>, <W(t+1)>, [<Xn|SP>]
+// CASP{<order>} <Xs>, <X(s+1)>, <Xt>, <X(t+1)>, [<Xn|SP>]
+// SWP{<order>}[<size>] <Ws>, <Wt>, [<Xn|SP>]
+// SWP{<order>} <Xs>, <Xt>, [<Xn|SP>]
+// LD<OP>{<order>}[<size>] <Ws>, <Wt>, [<Xn|SP>]
+// LD<OP>{<order>} <Xs>, <Xt>, [<Xn|SP>]
+// ST<OP>{<order>}[<size>] <Ws>, [<Xn|SP>]
+// ST<OP>{<order>} <Xs>, [<Xn|SP>]
+
+let Predicates = [HasV8_1a], mayLoad = 1, mayStore = 1, hasSideEffects = 1 in
+class BaseCASEncoding<dag oops, dag iops, string asm, string operands,
+                      string cstr, list<dag> pattern>
+      : I<oops, iops, asm, operands, cstr, pattern> {
+  bits<2> Sz;
+  bit NP;
+  bit Acq;
+  bit Rel;
+  bits<5> Rs;
+  bits<5> Rn;
+  bits<5> Rt;
+  let Inst{31-30} = Sz;
+  let Inst{29-24} = 0b001000;
+  let Inst{23} = NP;
+  let Inst{22} = Acq;
+  let Inst{21} = 0b1;
+  let Inst{20-16} = Rs;
+  let Inst{15} = Rel;
+  let Inst{14-10} = 0b11111;
+  let Inst{9-5} = Rn;
+  let Inst{4-0} = Rt;
+}
+
+class BaseCAS<string order, string size, RegisterClass RC>
+      : BaseCASEncoding<(outs RC:$out),(ins RC:$Rs, RC:$Rt, GPR64sp:$Rn),
+                        "cas" # order # size, "\t$Rs, $Rt, [$Rn]",
+                        "$out = $Rs",[]> {
+  let NP = 1;
+}
+
+multiclass CompareAndSwap<bits<1> Acq, bits<1> Rel, string order> {
+  let Sz = 0b00, Acq = Acq, Rel = Rel in def b : BaseCAS<order, "b", GPR32>;
+  let Sz = 0b01, Acq = Acq, Rel = Rel in def h : BaseCAS<order, "h", GPR32>;
+  let Sz = 0b10, Acq = Acq, Rel = Rel in def s : BaseCAS<order, "", GPR32>;
+  let Sz = 0b11, Acq = Acq, Rel = Rel in def d : BaseCAS<order, "", GPR64>;
+}
+
+class BaseCASP<string order, string size, RegisterOperand RC>
+      : BaseCASEncoding<(outs RC:$out),(ins RC:$Rs, RC:$Rt, GPR64sp:$Rn),
+                        "casp" # order # size, "\t$Rs, $Rt, [$Rn]",
+                        "$out = $Rs",[]> {
+  let NP = 0;
+}
+
+multiclass CompareAndSwapPair<bits<1> Acq, bits<1> Rel, string order> {
+  let Sz = 0b00, Acq = Acq, Rel = Rel in 
+    def s : BaseCASP<order, "", WSeqPairClassOperand>;
+  let Sz = 0b01, Acq = Acq, Rel = Rel in 
+    def d : BaseCASP<order, "", XSeqPairClassOperand>;
+}
+
+let Predicates = [HasV8_1a] in
+class BaseSWP<string order, string size, RegisterClass RC>
+      : I<(outs RC:$Rt),(ins RC:$Rs, GPR64sp:$Rn), "swp" # order # size,
+          "\t$Rs, $Rt, [$Rn]","",[]> {
+  bits<2> Sz;
+  bit Acq;
+  bit Rel;
+  bits<5> Rs;
+  bits<3> opc = 0b000;
+  bits<5> Rn;
+  bits<5> Rt;
+  let Inst{31-30} = Sz;
+  let Inst{29-24} = 0b111000;
+  let Inst{23} = Acq;
+  let Inst{22} = Rel;
+  let Inst{21} = 0b1;
+  let Inst{20-16} = Rs;
+  let Inst{15} = 0b1;
+  let Inst{14-12} = opc;
+  let Inst{11-10} = 0b00;
+  let Inst{9-5} = Rn;
+  let Inst{4-0} = Rt;
+}
+
+multiclass Swap<bits<1> Acq, bits<1> Rel, string order> {
+  let Sz = 0b00, Acq = Acq, Rel = Rel in def b : BaseSWP<order, "b", GPR32>;
+  let Sz = 0b01, Acq = Acq, Rel = Rel in def h : BaseSWP<order, "h", GPR32>;
+  let Sz = 0b10, Acq = Acq, Rel = Rel in def s : BaseSWP<order, "", GPR32>;
+  let Sz = 0b11, Acq = Acq, Rel = Rel in def d : BaseSWP<order, "", GPR64>;
+}
+
+let Predicates = [HasV8_1a], mayLoad = 1, mayStore = 1, hasSideEffects = 1 in
+class BaseLDOPregister<string op, string order, string size, RegisterClass RC>
+      : I<(outs RC:$Rt),(ins RC:$Rs, GPR64sp:$Rn), "ld" # op # order # size,
+          "\t$Rs, $Rt, [$Rn]","",[]> {
+  bits<2> Sz;
+  bit Acq;
+  bit Rel;
+  bits<5> Rs;
+  bits<3> opc;
+  bits<5> Rn;
+  bits<5> Rt;
+  let Inst{31-30} = Sz;
+  let Inst{29-24} = 0b111000;
+  let Inst{23} = Acq;
+  let Inst{22} = Rel;
+  let Inst{21} = 0b1;
+  let Inst{20-16} = Rs;
+  let Inst{15} = 0b0;
+  let Inst{14-12} = opc;
+  let Inst{11-10} = 0b00;
+  let Inst{9-5} = Rn;
+  let Inst{4-0} = Rt;
+}
+
+multiclass LDOPregister<bits<3> opc, string op, bits<1> Acq, bits<1> Rel, 
+                        string order> {
+  let Sz = 0b00, Acq = Acq, Rel = Rel, opc = opc in 
+    def b : BaseLDOPregister<op, order, "b", GPR32>;
+  let Sz = 0b01, Acq = Acq, Rel = Rel, opc = opc in 
+    def h : BaseLDOPregister<op, order, "h", GPR32>;
+  let Sz = 0b10, Acq = Acq, Rel = Rel, opc = opc in 
+    def s : BaseLDOPregister<op, order, "", GPR32>;
+  let Sz = 0b11, Acq = Acq, Rel = Rel, opc = opc in 
+    def d : BaseLDOPregister<op, order, "", GPR64>;
+}
+
+let Predicates = [HasV8_1a] in
+class BaseSTOPregister<string asm, RegisterClass OP, Register Reg,
+                        Instruction inst> :
+      InstAlias<asm # "\t$Rs, [$Rn]", (inst Reg, OP:$Rs, GPR64sp:$Rn)>;
+
+multiclass STOPregister<string asm, string instr> {
+  def : BaseSTOPregister<asm # "lb", GPR32, WZR, 
+                    !cast<Instruction>(instr # "Lb")>;
+  def : BaseSTOPregister<asm # "lh", GPR32, WZR, 
+                    !cast<Instruction>(instr # "Lh")>;
+  def : BaseSTOPregister<asm # "l",  GPR32, WZR, 
+                    !cast<Instruction>(instr # "Ls")>;
+  def : BaseSTOPregister<asm # "l",  GPR64, XZR, 
+                    !cast<Instruction>(instr # "Ld")>;
+  def : BaseSTOPregister<asm # "b",  GPR32, WZR, 
+                    !cast<Instruction>(instr # "b")>;
+  def : BaseSTOPregister<asm # "h",  GPR32, WZR, 
+                    !cast<Instruction>(instr # "h")>;
+  def : BaseSTOPregister<asm,        GPR32, WZR, 
+                    !cast<Instruction>(instr # "s")>;
+  def : BaseSTOPregister<asm,        GPR64, XZR, 
+                    !cast<Instruction>(instr # "d")>;
+}
+
+//----------------------------------------------------------------------------
 // Allow the size specifier tokens to be upper case, not just lower.
 def : TokenAlias<".8B", ".8b">;
 def : TokenAlias<".4H", ".4h">;
diff --git a/lib/Target/AArch64/AArch64InstrInfo.cpp b/lib/Target/AArch64/AArch64InstrInfo.cpp
index db231c44ae4..7fa57fe5076 100644
--- a/lib/Target/AArch64/AArch64InstrInfo.cpp
+++ b/lib/Target/AArch64/AArch64InstrInfo.cpp
@@ -96,15 +96,10 @@ bool AArch64InstrInfo::AnalyzeBranch(MachineBasicBlock &MBB,
                                    SmallVectorImpl<MachineOperand> &Cond,
                                    bool AllowModify) const {
   // If the block has no terminators, it just falls into the block after it.
-  MachineBasicBlock::iterator I = MBB.end();
-  if (I == MBB.begin())
+  MachineBasicBlock::iterator I = MBB.getLastNonDebugInstr();
+  if (I == MBB.end())
     return false;
-  --I;
-  while (I->isDebugValue()) {
-    if (I == MBB.begin())
-      return false;
-    --I;
-  }
+
   if (!isUnpredicatedTerminator(I))
     return false;
 
@@ -224,15 +219,10 @@ bool AArch64InstrInfo::ReverseBranchCondition(
 }
 
 unsigned AArch64InstrInfo::RemoveBranch(MachineBasicBlock &MBB) const {
-  MachineBasicBlock::iterator I = MBB.end();
-  if (I == MBB.begin())
+  MachineBasicBlock::iterator I = MBB.getLastNonDebugInstr();
+  if (I == MBB.end())
     return 0;
-  --I;
-  while (I->isDebugValue()) {
-    if (I == MBB.begin())
-      return 0;
-    --I;
-  }
+
   if (!isUncondBranchOpcode(I->getOpcode()) &&
       !isCondBranchOpcode(I->getOpcode()))
     return 0;
@@ -255,7 +245,7 @@ unsigned AArch64InstrInfo::RemoveBranch(MachineBasicBlock &MBB) const {
 
 void AArch64InstrInfo::instantiateCondBranch(
     MachineBasicBlock &MBB, DebugLoc DL, MachineBasicBlock *TBB,
-    const SmallVectorImpl<MachineOperand> &Cond) const {
+    ArrayRef<MachineOperand> Cond) const {
   if (Cond[0].getImm() != -1) {
     // Regular Bcc
     BuildMI(&MBB, DL, get(AArch64::Bcc)).addImm(Cond[0].getImm()).addMBB(TBB);
@@ -272,7 +262,7 @@ void AArch64InstrInfo::instantiateCondBranch(
 
 unsigned AArch64InstrInfo::InsertBranch(
     MachineBasicBlock &MBB, MachineBasicBlock *TBB, MachineBasicBlock *FBB,
-    const SmallVectorImpl<MachineOperand> &Cond, DebugLoc DL) const {
+    ArrayRef<MachineOperand> Cond, DebugLoc DL) const {
   // Shouldn't be a fall through.
   assert(TBB && "InsertBranch must not be told to insert a fallthrough");
 
@@ -369,7 +359,7 @@ static unsigned canFoldIntoCSel(const MachineRegisterInfo &MRI, unsigned VReg,
 }
 
 bool AArch64InstrInfo::canInsertSelect(
-    const MachineBasicBlock &MBB, const SmallVectorImpl<MachineOperand> &Cond,
+    const MachineBasicBlock &MBB, ArrayRef<MachineOperand> Cond,
     unsigned TrueReg, unsigned FalseReg, int &CondCycles, int &TrueCycles,
     int &FalseCycles) const {
   // Check register classes.
@@ -412,7 +402,7 @@ bool AArch64InstrInfo::canInsertSelect(
 void AArch64InstrInfo::insertSelect(MachineBasicBlock &MBB,
                                     MachineBasicBlock::iterator I, DebugLoc DL,
                                     unsigned DstReg,
-                                    const SmallVectorImpl<MachineOperand> &Cond,
+                                    ArrayRef<MachineOperand> Cond,
                                     unsigned TrueReg, unsigned FalseReg) const {
   MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
 
@@ -543,6 +533,14 @@ void AArch64InstrInfo::insertSelect(MachineBasicBlock &MBB,
       CC);
 }
 
+/// Returns true if a MOVi32imm or MOVi64imm can be expanded to an  ORRxx.
+static bool canBeExpandedToORR(const MachineInstr *MI, unsigned BitSize) {
+  uint64_t Imm = MI->getOperand(1).getImm();
+  uint64_t UImm = Imm << (64 - BitSize) >> (64 - BitSize);
+  uint64_t Encoding;
+  return AArch64_AM::processLogicalImmediate(UImm, BitSize, Encoding);
+}
+
 // FIXME: this implementation should be micro-architecture dependent, so a
 // micro-architecture target hook should be introduced here in future.
 bool AArch64InstrInfo::isAsCheapAsAMove(const MachineInstr *MI) const {
@@ -583,6 +581,12 @@ bool AArch64InstrInfo::isAsCheapAsAMove(const MachineInstr *MI) const {
   case AArch64::ORRWrr:
   case AArch64::ORRXrr:
     return true;
+  // If MOVi32imm or MOVi64imm can be expanded into ORRWri or
+  // ORRXri, it is as cheap as MOV
+  case AArch64::MOVi32imm:
+    return canBeExpandedToORR(MI, 32);
+  case AArch64::MOVi64imm:
+    return canBeExpandedToORR(MI, 64);
   }
 
   llvm_unreachable("Unknown opcode to check as cheap as a move!");
@@ -617,10 +621,8 @@ AArch64InstrInfo::areMemAccessesTriviallyDisjoint(MachineInstr *MIa,
   int OffsetA = 0, OffsetB = 0;
   int WidthA = 0, WidthB = 0;
 
-  assert(MIa && (MIa->mayLoad() || MIa->mayStore()) &&
-         "MIa must be a store or a load");
-  assert(MIb && (MIb->mayLoad() || MIb->mayStore()) &&
-         "MIb must be a store or a load");
+  assert(MIa && MIa->mayLoadOrStore() && "MIa must be a load or store.");
+  assert(MIb && MIb->mayLoadOrStore() && "MIb must be a load or store.");
 
   if (MIa->hasUnmodeledSideEffects() || MIb->hasUnmodeledSideEffects() ||
       MIa->hasOrderedMemoryRef() || MIb->hasOrderedMemoryRef())
@@ -631,8 +633,8 @@ AArch64InstrInfo::areMemAccessesTriviallyDisjoint(MachineInstr *MIa,
   // base registers are identical, and the offset of a lower memory access +
   // the width doesn't overlap the offset of a higher memory access,
   // then the memory accesses are different.
-  if (getLdStBaseRegImmOfsWidth(MIa, BaseRegA, OffsetA, WidthA, TRI) &&
-      getLdStBaseRegImmOfsWidth(MIb, BaseRegB, OffsetB, WidthB, TRI)) {
+  if (getMemOpBaseRegImmOfsWidth(MIa, BaseRegA, OffsetA, WidthA, TRI) &&
+      getMemOpBaseRegImmOfsWidth(MIb, BaseRegB, OffsetB, WidthB, TRI)) {
     if (BaseRegA == BaseRegB) {
       int LowOffset = OffsetA < OffsetB ? OffsetA : OffsetB;
       int HighOffset = OffsetA < OffsetB ? OffsetB : OffsetA;
@@ -1312,9 +1314,9 @@ void AArch64InstrInfo::suppressLdStPair(MachineInstr *MI) const {
 }
 
 bool
-AArch64InstrInfo::getLdStBaseRegImmOfs(MachineInstr *LdSt, unsigned &BaseReg,
-                                       unsigned &Offset,
-                                       const TargetRegisterInfo *TRI) const {
+AArch64InstrInfo::getMemOpBaseRegImmOfs(MachineInstr *LdSt, unsigned &BaseReg,
+                                        unsigned &Offset,
+                                        const TargetRegisterInfo *TRI) const {
   switch (LdSt->getOpcode()) {
   default:
     return false;
@@ -1338,7 +1340,7 @@ AArch64InstrInfo::getLdStBaseRegImmOfs(MachineInstr *LdSt, unsigned &BaseReg,
   };
 }
 
-bool AArch64InstrInfo::getLdStBaseRegImmOfsWidth(
+bool AArch64InstrInfo::getMemOpBaseRegImmOfsWidth(
     MachineInstr *LdSt, unsigned &BaseReg, int &Offset, int &Width,
     const TargetRegisterInfo *TRI) const {
   // Handle only loads/stores with base register followed by immediate offset.
@@ -1436,7 +1438,7 @@ bool AArch64InstrInfo::getLdStBaseRegImmOfsWidth(
 
 /// Detect opportunities for ldp/stp formation.
 ///
-/// Only called for LdSt for which getLdStBaseRegImmOfs returns true.
+/// Only called for LdSt for which getMemOpBaseRegImmOfs returns true.
 bool AArch64InstrInfo::shouldClusterLoads(MachineInstr *FirstLdSt,
                                           MachineInstr *SecondLdSt,
                                           unsigned NumLoads) const {
@@ -1445,7 +1447,7 @@ bool AArch64InstrInfo::shouldClusterLoads(MachineInstr *FirstLdSt,
     return false;
   if (FirstLdSt->getOpcode() != SecondLdSt->getOpcode())
     return false;
-  // getLdStBaseRegImmOfs guarantees that oper 2 isImm.
+  // getMemOpBaseRegImmOfs guarantees that oper 2 isImm.
   unsigned Ofs1 = FirstLdSt->getOperand(2).getImm();
   // Allow 6 bits of positive range.
   if (Ofs1 > 64)
@@ -1457,23 +1459,43 @@ bool AArch64InstrInfo::shouldClusterLoads(MachineInstr *FirstLdSt,
 
 bool AArch64InstrInfo::shouldScheduleAdjacent(MachineInstr *First,
                                               MachineInstr *Second) const {
-  // Cyclone can fuse CMN, CMP followed by Bcc.
-
-  // FIXME: B0 can also fuse:
-  // AND, BIC, ORN, ORR, or EOR (optional S) followed by Bcc or CBZ or CBNZ.
-  if (Second->getOpcode() != AArch64::Bcc)
-    return false;
-  switch (First->getOpcode()) {
-  default:
-    return false;
-  case AArch64::SUBSWri:
-  case AArch64::ADDSWri:
-  case AArch64::ANDSWri:
-  case AArch64::SUBSXri:
-  case AArch64::ADDSXri:
-  case AArch64::ANDSXri:
-    return true;
+  if (Subtarget.isCyclone()) {
+    // Cyclone can fuse CMN, CMP, TST followed by Bcc.
+    unsigned SecondOpcode = Second->getOpcode();
+    if (SecondOpcode == AArch64::Bcc) {
+      switch (First->getOpcode()) {
+      default:
+        return false;
+      case AArch64::SUBSWri:
+      case AArch64::ADDSWri:
+      case AArch64::ANDSWri:
+      case AArch64::SUBSXri:
+      case AArch64::ADDSXri:
+      case AArch64::ANDSXri:
+        return true;
+      }
+    }
+    // Cyclone B0 also supports ALU operations followed by CBZ/CBNZ.
+    if (SecondOpcode == AArch64::CBNZW || SecondOpcode == AArch64::CBNZX ||
+        SecondOpcode == AArch64::CBZW || SecondOpcode == AArch64::CBZX) {
+      switch (First->getOpcode()) {
+      default:
+        return false;
+      case AArch64::ADDWri:
+      case AArch64::ADDXri:
+      case AArch64::ANDWri:
+      case AArch64::ANDXri:
+      case AArch64::EORWri:
+      case AArch64::EORXri:
+      case AArch64::ORRWri:
+      case AArch64::ORRXri:
+      case AArch64::SUBWri:
+      case AArch64::SUBXri:
+        return true;
+      }
+    }
   }
+  return false;
 }
 
 MachineInstr *AArch64InstrInfo::emitFrameIndexDebugValue(
@@ -2068,10 +2090,9 @@ void llvm::emitFrameOffset(MachineBasicBlock &MBB,
       .setMIFlag(Flag);
 }
 
-MachineInstr *AArch64InstrInfo::foldMemoryOperandImpl(MachineFunction &MF,
-                                                      MachineInstr *MI,
-                                                      ArrayRef<unsigned> Ops,
-                                                      int FrameIndex) const {
+MachineInstr *AArch64InstrInfo::foldMemoryOperandImpl(
+    MachineFunction &MF, MachineInstr *MI, ArrayRef<unsigned> Ops,
+    MachineBasicBlock::iterator InsertPt, int FrameIndex) const {
   // This is a bit of a hack. Consider this instruction:
   //
   //   %vreg0<def> = COPY %SP; GPR64all:%vreg0
@@ -2366,7 +2387,7 @@ bool llvm::rewriteAArch64FrameIndex(MachineInstr &MI, unsigned FrameRegIdx,
 
 void AArch64InstrInfo::getNoopForMachoTarget(MCInst &NopInst) const {
   NopInst.setOpcode(AArch64::HINT);
-  NopInst.addOperand(MCOperand::CreateImm(0));
+  NopInst.addOperand(MCOperand::createImm(0));
 }
 /// useMachineCombiner - return true when a target supports MachineCombiner
 bool AArch64InstrInfo::useMachineCombiner() const {
@@ -2462,15 +2483,15 @@ static bool canCombineWithMUL(MachineBasicBlock &MBB, MachineOperand &MO,
   return true;
 }
 
-/// hasPattern - return true when there is potentially a faster code sequence
+/// Return true when there is potentially a faster code sequence
 /// for an instruction chain ending in \p Root. All potential patterns are
 /// listed
 /// in the \p Pattern vector. Pattern should be sorted in priority order since
 /// the pattern evaluator stops checking as soon as it finds a faster sequence.
 
-bool AArch64InstrInfo::hasPattern(
+bool AArch64InstrInfo::getMachineCombinerPatterns(
     MachineInstr &Root,
-    SmallVectorImpl<MachineCombinerPattern::MC_PATTERN> &Pattern) const {
+    SmallVectorImpl<MachineCombinerPattern::MC_PATTERN> &Patterns) const {
   unsigned Opc = Root.getOpcode();
   MachineBasicBlock &MBB = *Root.getParent();
   bool Found = false;
@@ -2498,76 +2519,76 @@ bool AArch64InstrInfo::hasPattern(
            "ADDWrr does not have register operands");
     if (canCombineWithMUL(MBB, Root.getOperand(1), AArch64::MADDWrrr,
                           AArch64::WZR)) {
-      Pattern.push_back(MachineCombinerPattern::MC_MULADDW_OP1);
+      Patterns.push_back(MachineCombinerPattern::MC_MULADDW_OP1);
       Found = true;
     }
     if (canCombineWithMUL(MBB, Root.getOperand(2), AArch64::MADDWrrr,
                           AArch64::WZR)) {
-      Pattern.push_back(MachineCombinerPattern::MC_MULADDW_OP2);
+      Patterns.push_back(MachineCombinerPattern::MC_MULADDW_OP2);
       Found = true;
     }
     break;
   case AArch64::ADDXrr:
     if (canCombineWithMUL(MBB, Root.getOperand(1), AArch64::MADDXrrr,
                           AArch64::XZR)) {
-      Pattern.push_back(MachineCombinerPattern::MC_MULADDX_OP1);
+      Patterns.push_back(MachineCombinerPattern::MC_MULADDX_OP1);
       Found = true;
     }
     if (canCombineWithMUL(MBB, Root.getOperand(2), AArch64::MADDXrrr,
                           AArch64::XZR)) {
-      Pattern.push_back(MachineCombinerPattern::MC_MULADDX_OP2);
+      Patterns.push_back(MachineCombinerPattern::MC_MULADDX_OP2);
       Found = true;
     }
     break;
   case AArch64::SUBWrr:
     if (canCombineWithMUL(MBB, Root.getOperand(1), AArch64::MADDWrrr,
                           AArch64::WZR)) {
-      Pattern.push_back(MachineCombinerPattern::MC_MULSUBW_OP1);
+      Patterns.push_back(MachineCombinerPattern::MC_MULSUBW_OP1);
       Found = true;
     }
     if (canCombineWithMUL(MBB, Root.getOperand(2), AArch64::MADDWrrr,
                           AArch64::WZR)) {
-      Pattern.push_back(MachineCombinerPattern::MC_MULSUBW_OP2);
+      Patterns.push_back(MachineCombinerPattern::MC_MULSUBW_OP2);
       Found = true;
     }
     break;
   case AArch64::SUBXrr:
     if (canCombineWithMUL(MBB, Root.getOperand(1), AArch64::MADDXrrr,
                           AArch64::XZR)) {
-      Pattern.push_back(MachineCombinerPattern::MC_MULSUBX_OP1);
+      Patterns.push_back(MachineCombinerPattern::MC_MULSUBX_OP1);
       Found = true;
     }
     if (canCombineWithMUL(MBB, Root.getOperand(2), AArch64::MADDXrrr,
                           AArch64::XZR)) {
-      Pattern.push_back(MachineCombinerPattern::MC_MULSUBX_OP2);
+      Patterns.push_back(MachineCombinerPattern::MC_MULSUBX_OP2);
       Found = true;
     }
     break;
   case AArch64::ADDWri:
     if (canCombineWithMUL(MBB, Root.getOperand(1), AArch64::MADDWrrr,
                           AArch64::WZR)) {
-      Pattern.push_back(MachineCombinerPattern::MC_MULADDWI_OP1);
+      Patterns.push_back(MachineCombinerPattern::MC_MULADDWI_OP1);
       Found = true;
     }
     break;
   case AArch64::ADDXri:
     if (canCombineWithMUL(MBB, Root.getOperand(1), AArch64::MADDXrrr,
                           AArch64::XZR)) {
-      Pattern.push_back(MachineCombinerPattern::MC_MULADDXI_OP1);
+      Patterns.push_back(MachineCombinerPattern::MC_MULADDXI_OP1);
       Found = true;
     }
     break;
   case AArch64::SUBWri:
     if (canCombineWithMUL(MBB, Root.getOperand(1), AArch64::MADDWrrr,
                           AArch64::WZR)) {
-      Pattern.push_back(MachineCombinerPattern::MC_MULSUBWI_OP1);
+      Patterns.push_back(MachineCombinerPattern::MC_MULSUBWI_OP1);
       Found = true;
     }
     break;
   case AArch64::SUBXri:
     if (canCombineWithMUL(MBB, Root.getOperand(1), AArch64::MADDXrrr,
                           AArch64::XZR)) {
-      Pattern.push_back(MachineCombinerPattern::MC_MULSUBXI_OP1);
+      Patterns.push_back(MachineCombinerPattern::MC_MULSUBXI_OP1);
       Found = true;
     }
     break;
@@ -2670,7 +2691,7 @@ static MachineInstr *genMaddR(MachineFunction &MF, MachineRegisterInfo &MRI,
   return MUL;
 }
 
-/// genAlternativeCodeSequence - when hasPattern() finds a pattern
+/// When getMachineCombinerPatterns() finds potential patterns,
 /// this function generates the instructions that could replace the
 /// original code sequence
 void AArch64InstrInfo::genAlternativeCodeSequence(
diff --git a/lib/Target/AArch64/AArch64InstrInfo.h b/lib/Target/AArch64/AArch64InstrInfo.h
index fa4b8b7e617..68c2a288258 100644
--- a/lib/Target/AArch64/AArch64InstrInfo.h
+++ b/lib/Target/AArch64/AArch64InstrInfo.h
@@ -90,13 +90,13 @@ class AArch64InstrInfo : public AArch64GenInstrInfo {
   /// Hint that pairing the given load or store is unprofitable.
   void suppressLdStPair(MachineInstr *MI) const;
 
-  bool getLdStBaseRegImmOfs(MachineInstr *LdSt, unsigned &BaseReg,
-                            unsigned &Offset,
-                            const TargetRegisterInfo *TRI) const override;
+  bool getMemOpBaseRegImmOfs(MachineInstr *LdSt, unsigned &BaseReg,
+                             unsigned &Offset,
+                             const TargetRegisterInfo *TRI) const override;
 
-  bool getLdStBaseRegImmOfsWidth(MachineInstr *LdSt, unsigned &BaseReg,
-                                 int &Offset, int &Width,
-                                 const TargetRegisterInfo *TRI) const;
+  bool getMemOpBaseRegImmOfsWidth(MachineInstr *LdSt, unsigned &BaseReg,
+                                  int &Offset, int &Width,
+                                  const TargetRegisterInfo *TRI) const;
 
   bool enableClusterLoads() const override { return true; }
 
@@ -131,6 +131,7 @@ class AArch64InstrInfo : public AArch64GenInstrInfo {
   using TargetInstrInfo::foldMemoryOperandImpl;
   MachineInstr *foldMemoryOperandImpl(MachineFunction &MF, MachineInstr *MI,
                                       ArrayRef<unsigned> Ops,
+                                      MachineBasicBlock::iterator InsertPt,
                                       int FrameIndex) const override;
 
   bool AnalyzeBranch(MachineBasicBlock &MBB, MachineBasicBlock *&TBB,
@@ -139,17 +140,14 @@ class AArch64InstrInfo : public AArch64GenInstrInfo {
                      bool AllowModify = false) const override;
   unsigned RemoveBranch(MachineBasicBlock &MBB) const override;
   unsigned InsertBranch(MachineBasicBlock &MBB, MachineBasicBlock *TBB,
-                        MachineBasicBlock *FBB,
-                        const SmallVectorImpl<MachineOperand> &Cond,
+                        MachineBasicBlock *FBB, ArrayRef<MachineOperand> Cond,
                         DebugLoc DL) const override;
   bool
   ReverseBranchCondition(SmallVectorImpl<MachineOperand> &Cond) const override;
-  bool canInsertSelect(const MachineBasicBlock &,
-                       const SmallVectorImpl<MachineOperand> &Cond, unsigned,
-                       unsigned, int &, int &, int &) const override;
+  bool canInsertSelect(const MachineBasicBlock &, ArrayRef<MachineOperand> Cond,
+                       unsigned, unsigned, int &, int &, int &) const override;
   void insertSelect(MachineBasicBlock &MBB, MachineBasicBlock::iterator MI,
-                    DebugLoc DL, unsigned DstReg,
-                    const SmallVectorImpl<MachineOperand> &Cond,
+                    DebugLoc DL, unsigned DstReg, ArrayRef<MachineOperand> Cond,
                     unsigned TrueReg, unsigned FalseReg) const override;
   void getNoopForMachoTarget(MCInst &NopInst) const override;
 
@@ -165,19 +163,17 @@ class AArch64InstrInfo : public AArch64GenInstrInfo {
                             unsigned SrcReg2, int CmpMask, int CmpValue,
                             const MachineRegisterInfo *MRI) const override;
   bool optimizeCondBranch(MachineInstr *MI) const override;
-  /// hasPattern - return true when there is potentially a faster code sequence
+  /// Return true when there is potentially a faster code sequence
   /// for an instruction chain ending in <Root>. All potential patterns are
-  /// listed
-  /// in the <Pattern> array.
-  bool hasPattern(MachineInstr &Root,
-                  SmallVectorImpl<MachineCombinerPattern::MC_PATTERN> &Pattern)
+  /// listed in the <Patterns> array.
+  bool getMachineCombinerPatterns(MachineInstr &Root,
+                  SmallVectorImpl<MachineCombinerPattern::MC_PATTERN> &Patterns)
       const override;
 
-  /// genAlternativeCodeSequence - when hasPattern() finds a pattern
-  /// this function generates the instructions that could replace the
-  /// original code sequence
+  /// When getMachineCombinerPatterns() finds patterns, this function generates
+  /// the instructions that could replace the original code sequence
   void genAlternativeCodeSequence(
-      MachineInstr &Root, MachineCombinerPattern::MC_PATTERN P,
+      MachineInstr &Root, MachineCombinerPattern::MC_PATTERN Pattern,
       SmallVectorImpl<MachineInstr *> &InsInstrs,
       SmallVectorImpl<MachineInstr *> &DelInstrs,
       DenseMap<unsigned, unsigned> &InstrIdxForVirtReg) const override;
@@ -188,7 +184,7 @@ class AArch64InstrInfo : public AArch64GenInstrInfo {
 private:
   void instantiateCondBranch(MachineBasicBlock &MBB, DebugLoc DL,
                              MachineBasicBlock *TBB,
-                             const SmallVectorImpl<MachineOperand> &Cond) const;
+                             ArrayRef<MachineOperand> Cond) const;
 };
 
 /// emitFrameOffset - Emit instructions as needed to set DestReg to SrcReg
diff --git a/lib/Target/AArch64/AArch64InstrInfo.td b/lib/Target/AArch64/AArch64InstrInfo.td
index f7db50a580d..7a0690dae7d 100644
--- a/lib/Target/AArch64/AArch64InstrInfo.td
+++ b/lib/Target/AArch64/AArch64InstrInfo.td
@@ -66,6 +66,20 @@ def SDT_AArch64CSel  : SDTypeProfile<1, 4,
                                     SDTCisSameAs<0, 2>,
                                     SDTCisInt<3>,
                                     SDTCisVT<4, i32>]>;
+def SDT_AArch64CCMP : SDTypeProfile<1, 5,
+                                    [SDTCisVT<0, i32>,
+                                     SDTCisInt<1>,
+                                     SDTCisSameAs<1, 2>,
+                                     SDTCisInt<3>,
+                                     SDTCisInt<4>,
+                                     SDTCisVT<5, i32>]>;
+def SDT_AArch64FCCMP : SDTypeProfile<1, 5,
+                                     [SDTCisVT<0, i32>,
+                                      SDTCisFP<1>,
+                                      SDTCisSameAs<1, 2>,
+                                      SDTCisInt<3>,
+                                      SDTCisInt<4>,
+                                      SDTCisVT<5, i32>]>;
 def SDT_AArch64FCmp   : SDTypeProfile<0, 2,
                                    [SDTCisFP<0>,
                                     SDTCisSameAs<0, 1>]>;
@@ -160,6 +174,10 @@ def AArch64and_flag  : SDNode<"AArch64ISD::ANDS",  SDTBinaryArithWithFlagsOut,
 def AArch64adc_flag  : SDNode<"AArch64ISD::ADCS",  SDTBinaryArithWithFlagsInOut>;
 def AArch64sbc_flag  : SDNode<"AArch64ISD::SBCS",  SDTBinaryArithWithFlagsInOut>;
 
+def AArch64ccmp      : SDNode<"AArch64ISD::CCMP",  SDT_AArch64CCMP>;
+def AArch64ccmn      : SDNode<"AArch64ISD::CCMN",  SDT_AArch64CCMP>;
+def AArch64fccmp     : SDNode<"AArch64ISD::FCCMP", SDT_AArch64FCCMP>;
+
 def AArch64threadpointer : SDNode<"AArch64ISD::THREAD_POINTER", SDTPtrLeaf>;
 
 def AArch64fcmp      : SDNode<"AArch64ISD::FCMP", SDT_AArch64FCmp>;
@@ -498,7 +516,7 @@ def i64imm_32bit : ImmLeaf<i64, [{
 }]>;
 
 def trunc_imm : SDNodeXForm<imm, [{
-  return CurDAG->getTargetConstant(N->getZExtValue(), MVT::i32);
+  return CurDAG->getTargetConstant(N->getZExtValue(), SDLoc(N), MVT::i32);
 }]>;
 
 def : Pat<(i64 i64imm_32bit:$src),
@@ -507,12 +525,12 @@ def : Pat<(i64 i64imm_32bit:$src),
 // Materialize FP constants via MOVi32imm/MOVi64imm (MachO large code model).
 def bitcast_fpimm_to_i32 : SDNodeXForm<fpimm, [{
 return CurDAG->getTargetConstant(
-  N->getValueAPF().bitcastToAPInt().getZExtValue(), MVT::i32);
+  N->getValueAPF().bitcastToAPInt().getZExtValue(), SDLoc(N), MVT::i32);
 }]>;
 
 def bitcast_fpimm_to_i64 : SDNodeXForm<fpimm, [{
 return CurDAG->getTargetConstant(
-  N->getValueAPF().bitcastToAPInt().getZExtValue(), MVT::i64);
+  N->getValueAPF().bitcastToAPInt().getZExtValue(), SDLoc(N), MVT::i64);
 }]>;
 
 
@@ -567,8 +585,8 @@ def : InstAlias<"ngcs $dst, $src", (SBCSWr GPR32:$dst, WZR, GPR32:$src)>;
 def : InstAlias<"ngcs $dst, $src", (SBCSXr GPR64:$dst, XZR, GPR64:$src)>;
 
 // Add/subtract
-defm ADD : AddSub<0, "add", add>;
-defm SUB : AddSub<1, "sub">;
+defm ADD : AddSub<0, "add", "sub", add>;
+defm SUB : AddSub<1, "sub", "add">;
 
 def : InstAlias<"mov $dst, $src",
                 (ADDWri GPR32sponly:$dst, GPR32sp:$src, 0, 0)>;
@@ -579,8 +597,8 @@ def : InstAlias<"mov $dst, $src",
 def : InstAlias<"mov $dst, $src",
                 (ADDXri GPR64sp:$dst, GPR64sponly:$src, 0, 0)>;
 
-defm ADDS : AddSubS<0, "adds", AArch64add_flag, "cmn">;
-defm SUBS : AddSubS<1, "subs", AArch64sub_flag, "cmp">;
+defm ADDS : AddSubS<0, "adds", AArch64add_flag, "cmn", "subs", "cmp">;
+defm SUBS : AddSubS<1, "subs", AArch64sub_flag, "cmp", "adds", "cmn">;
 
 // Use SUBS instead of SUB to enable CSE between SUBS and SUB.
 def : Pat<(sub GPR32sp:$Rn, addsub_shifted_imm32:$imm),
@@ -595,10 +613,12 @@ def : Pat<(sub GPR32:$Rn, arith_shifted_reg32:$Rm),
           (SUBSWrs GPR32:$Rn, arith_shifted_reg32:$Rm)>;
 def : Pat<(sub GPR64:$Rn, arith_shifted_reg64:$Rm),
           (SUBSXrs GPR64:$Rn, arith_shifted_reg64:$Rm)>;
+let AddedComplexity = 1 in {
 def : Pat<(sub GPR32sp:$R2, arith_extended_reg32<i32>:$R3),
           (SUBSWrx GPR32sp:$R2, arith_extended_reg32<i32>:$R3)>;
 def : Pat<(sub GPR64sp:$R2, arith_extended_reg32to64<i64>:$R3),
           (SUBSXrx GPR64sp:$R2, arith_extended_reg32to64<i64>:$R3)>;
+}
 
 // Because of the immediate format for add/sub-imm instructions, the
 // expression (add x, -1) must be transformed to (SUB{W,X}ri x, 1).
@@ -727,6 +747,74 @@ def CRC32CHrr : BaseCRC32<0, 0b01, 1, GPR32, int_aarch64_crc32ch, "crc32ch">;
 def CRC32CWrr : BaseCRC32<0, 0b10, 1, GPR32, int_aarch64_crc32cw, "crc32cw">;
 def CRC32CXrr : BaseCRC32<1, 0b11, 1, GPR64, int_aarch64_crc32cx, "crc32cx">;
 
+// v8.1 atomic CAS
+defm CAS   : CompareAndSwap<0, 0, "">;
+defm CASA  : CompareAndSwap<1, 0, "a">;
+defm CASL  : CompareAndSwap<0, 1, "l">;
+defm CASAL : CompareAndSwap<1, 1, "al">;
+
+// v8.1 atomic CASP
+defm CASP   : CompareAndSwapPair<0, 0, "">;
+defm CASPA  : CompareAndSwapPair<1, 0, "a">;
+defm CASPL  : CompareAndSwapPair<0, 1, "l">;
+defm CASPAL : CompareAndSwapPair<1, 1, "al">;
+
+// v8.1 atomic SWP
+defm SWP   : Swap<0, 0, "">;
+defm SWPA  : Swap<1, 0, "a">;
+defm SWPL  : Swap<0, 1, "l">;
+defm SWPAL : Swap<1, 1, "al">;
+
+// v8.1 atomic LD<OP>(register). Performs load and then ST<OP>(register)
+defm LDADD   : LDOPregister<0b000, "add", 0, 0, "">;
+defm LDADDA  : LDOPregister<0b000, "add", 1, 0, "a">;
+defm LDADDL  : LDOPregister<0b000, "add", 0, 1, "l">;
+defm LDADDAL : LDOPregister<0b000, "add", 1, 1, "al">;
+
+defm LDCLR   : LDOPregister<0b001, "clr", 0, 0, "">;
+defm LDCLRA  : LDOPregister<0b001, "clr", 1, 0, "a">;
+defm LDCLRL  : LDOPregister<0b001, "clr", 0, 1, "l">;
+defm LDCLRAL : LDOPregister<0b001, "clr", 1, 1, "al">;
+
+defm LDEOR   : LDOPregister<0b010, "eor", 0, 0, "">;
+defm LDEORA  : LDOPregister<0b010, "eor", 1, 0, "a">;
+defm LDEORL  : LDOPregister<0b010, "eor", 0, 1, "l">;
+defm LDEORAL : LDOPregister<0b010, "eor", 1, 1, "al">;
+
+defm LDSET   : LDOPregister<0b011, "set", 0, 0, "">;
+defm LDSETA  : LDOPregister<0b011, "set", 1, 0, "a">;
+defm LDSETL  : LDOPregister<0b011, "set", 0, 1, "l">;
+defm LDSETAL : LDOPregister<0b011, "set", 1, 1, "al">;
+
+defm LDSMAX   : LDOPregister<0b100, "smax", 0, 0, "">;
+defm LDSMAXA  : LDOPregister<0b100, "smax", 1, 0, "a">;
+defm LDSMAXL  : LDOPregister<0b100, "smax", 0, 1, "l">;
+defm LDSMAXAL : LDOPregister<0b100, "smax", 1, 1, "al">;
+
+defm LDSMIN   : LDOPregister<0b101, "smin", 0, 0, "">;
+defm LDSMINA  : LDOPregister<0b101, "smin", 1, 0, "a">;
+defm LDSMINL  : LDOPregister<0b101, "smin", 0, 1, "l">;
+defm LDSMINAL : LDOPregister<0b101, "smin", 1, 1, "al">;
+
+defm LDUMAX   : LDOPregister<0b110, "umax", 0, 0, "">;
+defm LDUMAXA  : LDOPregister<0b110, "umax", 1, 0, "a">;
+defm LDUMAXL  : LDOPregister<0b110, "umax", 0, 1, "l">;
+defm LDUMAXAL : LDOPregister<0b110, "umax", 1, 1, "al">;
+
+defm LDUMIN   : LDOPregister<0b111, "umin", 0, 0, "">;
+defm LDUMINA  : LDOPregister<0b111, "umin", 1, 0, "a">;
+defm LDUMINL  : LDOPregister<0b111, "umin", 0, 1, "l">;
+defm LDUMINAL : LDOPregister<0b111, "umin", 1, 1, "al">;
+
+// v8.1 atomic ST<OP>(register) as aliases to "LD<OP>(register) when Rt=xZR"
+defm : STOPregister<"stadd","LDADD">; // STADDx
+defm : STOPregister<"stclr","LDCLR">; // STCLRx
+defm : STOPregister<"steor","LDEOR">; // STEORx
+defm : STOPregister<"stset","LDSET">; // STSETx
+defm : STOPregister<"stsmax","LDSMAX">;// STSMAXx
+defm : STOPregister<"stsmin","LDSMIN">;// STSMINx
+defm : STOPregister<"stumax","LDUMAX">;// STUMAXx
+defm : STOPregister<"stumin","LDUMIN">;// STUMINx
 
 //===----------------------------------------------------------------------===//
 // Logical instructions.
@@ -755,7 +843,7 @@ defm AND  : LogicalReg<0b00, 0, "and", and>;
 defm BIC  : LogicalReg<0b00, 1, "bic",
                        BinOpFrag<(and node:$LHS, (not node:$RHS))>>;
 defm EON  : LogicalReg<0b10, 1, "eon",
-                       BinOpFrag<(xor node:$LHS, (not node:$RHS))>>;
+                       BinOpFrag<(not (xor node:$LHS, node:$RHS))>>;
 defm EOR  : LogicalReg<0b10, 0, "eor", xor>;
 defm ORN  : LogicalReg<0b01, 1, "orn",
                        BinOpFrag<(or node:$LHS, (not node:$RHS))>>;
@@ -826,6 +914,8 @@ def REVXr   : OneXRegData<0b011, "rev", bswap>;
 def REV32Xr : OneXRegData<0b010, "rev32",
                                  UnOpFrag<(rotr (bswap node:$LHS), (i64 32))>>;
 
+def : InstAlias<"rev64 $Rd, $Rn", (REVXr GPR64:$Rd, GPR64:$Rn), 0>;
+
 // The bswap commutes with the rotr so we want a pattern for both possible
 // orders.
 def : Pat<(bswap (rotr GPR32:$Rn, (i64 16))), (REV16Wr GPR32:$Rn)>;
@@ -857,57 +947,57 @@ defm UBFM : BitfieldImm<0b10, "ubfm">;
 
 def i32shift_a : Operand<i64>, SDNodeXForm<imm, [{
   uint64_t enc = (32 - N->getZExtValue()) & 0x1f;
-  return CurDAG->getTargetConstant(enc, MVT::i64);
+  return CurDAG->getTargetConstant(enc, SDLoc(N), MVT::i64);
 }]>;
 
 def i32shift_b : Operand<i64>, SDNodeXForm<imm, [{
   uint64_t enc = 31 - N->getZExtValue();
-  return CurDAG->getTargetConstant(enc, MVT::i64);
+  return CurDAG->getTargetConstant(enc, SDLoc(N), MVT::i64);
 }]>;
 
 // min(7, 31 - shift_amt)
 def i32shift_sext_i8 : Operand<i64>, SDNodeXForm<imm, [{
   uint64_t enc = 31 - N->getZExtValue();
   enc = enc > 7 ? 7 : enc;
-  return CurDAG->getTargetConstant(enc, MVT::i64);
+  return CurDAG->getTargetConstant(enc, SDLoc(N), MVT::i64);
 }]>;
 
 // min(15, 31 - shift_amt)
 def i32shift_sext_i16 : Operand<i64>, SDNodeXForm<imm, [{
   uint64_t enc = 31 - N->getZExtValue();
   enc = enc > 15 ? 15 : enc;
-  return CurDAG->getTargetConstant(enc, MVT::i64);
+  return CurDAG->getTargetConstant(enc, SDLoc(N), MVT::i64);
 }]>;
 
 def i64shift_a : Operand<i64>, SDNodeXForm<imm, [{
   uint64_t enc = (64 - N->getZExtValue()) & 0x3f;
-  return CurDAG->getTargetConstant(enc, MVT::i64);
+  return CurDAG->getTargetConstant(enc, SDLoc(N), MVT::i64);
 }]>;
 
 def i64shift_b : Operand<i64>, SDNodeXForm<imm, [{
   uint64_t enc = 63 - N->getZExtValue();
-  return CurDAG->getTargetConstant(enc, MVT::i64);
+  return CurDAG->getTargetConstant(enc, SDLoc(N), MVT::i64);
 }]>;
 
 // min(7, 63 - shift_amt)
 def i64shift_sext_i8 : Operand<i64>, SDNodeXForm<imm, [{
   uint64_t enc = 63 - N->getZExtValue();
   enc = enc > 7 ? 7 : enc;
-  return CurDAG->getTargetConstant(enc, MVT::i64);
+  return CurDAG->getTargetConstant(enc, SDLoc(N), MVT::i64);
 }]>;
 
 // min(15, 63 - shift_amt)
 def i64shift_sext_i16 : Operand<i64>, SDNodeXForm<imm, [{
   uint64_t enc = 63 - N->getZExtValue();
   enc = enc > 15 ? 15 : enc;
-  return CurDAG->getTargetConstant(enc, MVT::i64);
+  return CurDAG->getTargetConstant(enc, SDLoc(N), MVT::i64);
 }]>;
 
 // min(31, 63 - shift_amt)
 def i64shift_sext_i32 : Operand<i64>, SDNodeXForm<imm, [{
   uint64_t enc = 63 - N->getZExtValue();
   enc = enc > 31 ? 31 : enc;
-  return CurDAG->getTargetConstant(enc, MVT::i64);
+  return CurDAG->getTargetConstant(enc, SDLoc(N), MVT::i64);
 }]>;
 
 def : Pat<(shl GPR32:$Rn, (i64 imm0_31:$imm)),
@@ -950,13 +1040,10 @@ def : InstAlias<"uxth $dst, $src", (UBFMXri GPR64:$dst, GPR64:$src, 0, 15)>;
 def : InstAlias<"uxtw $dst, $src", (UBFMXri GPR64:$dst, GPR64:$src, 0, 31)>;
 
 //===----------------------------------------------------------------------===//
-// Conditionally set flags instructions.
+// Conditional comparison instructions.
 //===----------------------------------------------------------------------===//
-defm CCMN : CondSetFlagsImm<0, "ccmn">;
-defm CCMP : CondSetFlagsImm<1, "ccmp">;
-
-defm CCMN : CondSetFlagsReg<0, "ccmn">;
-defm CCMP : CondSetFlagsReg<1, "ccmp">;
+defm CCMN : CondComparison<0, "ccmn", AArch64ccmn>;
+defm CCMP : CondComparison<1, "ccmp", AArch64ccmp>;
 
 //===----------------------------------------------------------------------===//
 // Conditional select instructions.
@@ -2486,7 +2573,7 @@ defm FCMP  : FPComparison<0, "fcmp", AArch64fcmp>;
 //===----------------------------------------------------------------------===//
 
 defm FCCMPE : FPCondComparison<1, "fccmpe">;
-defm FCCMP  : FPCondComparison<0, "fccmp">;
+defm FCCMP  : FPCondComparison<0, "fccmp", AArch64fccmp>;
 
 //===----------------------------------------------------------------------===//
 // Floating point conditional select instruction.
@@ -2758,8 +2845,8 @@ defm MLS      : SIMDThreeSameVectorBHSTied<1, 0b10010, "mls",
 defm MUL      : SIMDThreeSameVectorBHS<0, 0b10011, "mul", mul>;
 defm PMUL     : SIMDThreeSameVectorB<1, 0b10011, "pmul", int_aarch64_neon_pmul>;
 defm SABA     : SIMDThreeSameVectorBHSTied<0, 0b01111, "saba",
-      TriOpFrag<(add node:$LHS, (int_aarch64_neon_sabd node:$MHS, node:$RHS))> >;
-defm SABD     : SIMDThreeSameVectorBHS<0,0b01110,"sabd", int_aarch64_neon_sabd>;
+      TriOpFrag<(add node:$LHS, (sabsdiff node:$MHS, node:$RHS))> >;
+defm SABD     : SIMDThreeSameVectorBHS<0,0b01110,"sabd", sabsdiff>;
 defm SHADD    : SIMDThreeSameVectorBHS<0,0b00000,"shadd", int_aarch64_neon_shadd>;
 defm SHSUB    : SIMDThreeSameVectorBHS<0,0b00100,"shsub", int_aarch64_neon_shsub>;
 defm SMAXP    : SIMDThreeSameVectorBHS<0,0b10100,"smaxp", int_aarch64_neon_smaxp>;
@@ -2777,8 +2864,8 @@ defm SRSHL    : SIMDThreeSameVector<0,0b01010,"srshl", int_aarch64_neon_srshl>;
 defm SSHL     : SIMDThreeSameVector<0,0b01000,"sshl", int_aarch64_neon_sshl>;
 defm SUB      : SIMDThreeSameVector<1,0b10000,"sub", sub>;
 defm UABA     : SIMDThreeSameVectorBHSTied<1, 0b01111, "uaba",
-      TriOpFrag<(add node:$LHS, (int_aarch64_neon_uabd node:$MHS, node:$RHS))> >;
-defm UABD     : SIMDThreeSameVectorBHS<1,0b01110,"uabd", int_aarch64_neon_uabd>;
+      TriOpFrag<(add node:$LHS, (uabsdiff node:$MHS, node:$RHS))> >;
+defm UABD     : SIMDThreeSameVectorBHS<1,0b01110,"uabd", uabsdiff>;
 defm UHADD    : SIMDThreeSameVectorBHS<1,0b00000,"uhadd", int_aarch64_neon_uhadd>;
 defm UHSUB    : SIMDThreeSameVectorBHS<1,0b00100,"uhsub", int_aarch64_neon_uhsub>;
 defm UMAXP    : SIMDThreeSameVectorBHS<1,0b10100,"umaxp", int_aarch64_neon_umaxp>;
@@ -2809,6 +2896,55 @@ defm ORN : SIMDLogicalThreeVector<0, 0b11, "orn",
                                   BinOpFrag<(or node:$LHS, (vnot node:$RHS))> >;
 defm ORR : SIMDLogicalThreeVector<0, 0b10, "orr", or>;
 
+def : Pat<(v8i8 (smin V64:$Rn, V64:$Rm)),
+          (SMINv8i8 V64:$Rn, V64:$Rm)>;
+def : Pat<(v4i16 (smin V64:$Rn, V64:$Rm)),
+          (SMINv4i16 V64:$Rn, V64:$Rm)>;
+def : Pat<(v2i32 (smin V64:$Rn, V64:$Rm)),
+          (SMINv2i32 V64:$Rn, V64:$Rm)>;
+def : Pat<(v16i8 (smin V128:$Rn, V128:$Rm)),
+          (SMINv16i8 V128:$Rn, V128:$Rm)>;
+def : Pat<(v8i16 (smin V128:$Rn, V128:$Rm)),
+          (SMINv8i16 V128:$Rn, V128:$Rm)>;
+def : Pat<(v4i32 (smin V128:$Rn, V128:$Rm)),
+          (SMINv4i32 V128:$Rn, V128:$Rm)>;
+def : Pat<(v8i8 (smax V64:$Rn, V64:$Rm)),
+          (SMAXv8i8 V64:$Rn, V64:$Rm)>;
+def : Pat<(v4i16 (smax V64:$Rn, V64:$Rm)),
+          (SMAXv4i16 V64:$Rn, V64:$Rm)>;
+def : Pat<(v2i32 (smax V64:$Rn, V64:$Rm)),
+          (SMAXv2i32 V64:$Rn, V64:$Rm)>;
+def : Pat<(v16i8 (smax V128:$Rn, V128:$Rm)),
+          (SMAXv16i8 V128:$Rn, V128:$Rm)>;
+def : Pat<(v8i16 (smax V128:$Rn, V128:$Rm)),
+          (SMAXv8i16 V128:$Rn, V128:$Rm)>;
+def : Pat<(v4i32 (smax V128:$Rn, V128:$Rm)),
+          (SMAXv4i32 V128:$Rn, V128:$Rm)>;
+def : Pat<(v8i8 (umin V64:$Rn, V64:$Rm)),
+          (UMINv8i8 V64:$Rn, V64:$Rm)>;
+def : Pat<(v4i16 (umin V64:$Rn, V64:$Rm)),
+          (UMINv4i16 V64:$Rn, V64:$Rm)>;
+def : Pat<(v2i32 (umin V64:$Rn, V64:$Rm)),
+          (UMINv2i32 V64:$Rn, V64:$Rm)>;
+def : Pat<(v16i8 (umin V128:$Rn, V128:$Rm)),
+          (UMINv16i8 V128:$Rn, V128:$Rm)>;
+def : Pat<(v8i16 (umin V128:$Rn, V128:$Rm)),
+          (UMINv8i16 V128:$Rn, V128:$Rm)>;
+def : Pat<(v4i32 (umin V128:$Rn, V128:$Rm)),
+          (UMINv4i32 V128:$Rn, V128:$Rm)>;
+def : Pat<(v8i8 (umax V64:$Rn, V64:$Rm)),
+          (UMAXv8i8 V64:$Rn, V64:$Rm)>;
+def : Pat<(v4i16 (umax V64:$Rn, V64:$Rm)),
+          (UMAXv4i16 V64:$Rn, V64:$Rm)>;
+def : Pat<(v2i32 (umax V64:$Rn, V64:$Rm)),
+          (UMAXv2i32 V64:$Rn, V64:$Rm)>;
+def : Pat<(v16i8 (umax V128:$Rn, V128:$Rm)),
+          (UMAXv16i8 V128:$Rn, V128:$Rm)>;
+def : Pat<(v8i16 (umax V128:$Rn, V128:$Rm)),
+          (UMAXv8i16 V128:$Rn, V128:$Rm)>;
+def : Pat<(v4i32 (umax V128:$Rn, V128:$Rm)),
+          (UMAXv4i32 V128:$Rn, V128:$Rm)>;
+
 def : Pat<(AArch64bsl (v8i8 V64:$Rd), V64:$Rn, V64:$Rm),
           (BSLv8i8 V64:$Rd, V64:$Rn, V64:$Rm)>;
 def : Pat<(AArch64bsl (v4i16 V64:$Rd), V64:$Rn, V64:$Rm),
@@ -3247,9 +3383,9 @@ defm RADDHN : SIMDNarrowThreeVectorBHS<1,0b0100,"raddhn",int_aarch64_neon_raddhn
 defm RSUBHN : SIMDNarrowThreeVectorBHS<1,0b0110,"rsubhn",int_aarch64_neon_rsubhn>;
 defm PMULL  : SIMDDifferentThreeVectorBD<0,0b1110,"pmull",int_aarch64_neon_pmull>;
 defm SABAL  : SIMDLongThreeVectorTiedBHSabal<0,0b0101,"sabal",
-                                             int_aarch64_neon_sabd>;
+                                             sabsdiff>;
 defm SABDL   : SIMDLongThreeVectorBHSabdl<0, 0b0111, "sabdl",
-                                          int_aarch64_neon_sabd>;
+                                          sabsdiff>;
 defm SADDL   : SIMDLongThreeVectorBHS<   0, 0b0000, "saddl",
             BinOpFrag<(add (sext node:$LHS), (sext node:$RHS))>>;
 defm SADDW   : SIMDWideThreeVectorBHS<   0, 0b0001, "saddw",
@@ -3270,9 +3406,9 @@ defm SSUBL   : SIMDLongThreeVectorBHS<0, 0b0010, "ssubl",
 defm SSUBW   : SIMDWideThreeVectorBHS<0, 0b0011, "ssubw",
                  BinOpFrag<(sub node:$LHS, (sext node:$RHS))>>;
 defm UABAL   : SIMDLongThreeVectorTiedBHSabal<1, 0b0101, "uabal",
-                                              int_aarch64_neon_uabd>;
+                                              uabsdiff>;
 defm UABDL   : SIMDLongThreeVectorBHSabdl<1, 0b0111, "uabdl",
-                                          int_aarch64_neon_uabd>;
+                                          uabsdiff>;
 defm UADDL   : SIMDLongThreeVectorBHS<1, 0b0000, "uaddl",
                  BinOpFrag<(add (zext node:$LHS), (zext node:$RHS))>>;
 defm UADDW   : SIMDWideThreeVectorBHS<1, 0b0001, "uaddw",
@@ -3563,13 +3699,13 @@ def : Pat<(v2f64 (AArch64duplane64 (v2f64 V128:$Rn), VectorIndexD:$imm)),
 // instruction even if the types don't match: we just have to remap the lane
 // carefully. N.b. this trick only applies to truncations.
 def VecIndex_x2 : SDNodeXForm<imm, [{
-  return CurDAG->getTargetConstant(2 * N->getZExtValue(), MVT::i64);
+  return CurDAG->getTargetConstant(2 * N->getZExtValue(), SDLoc(N), MVT::i64);
 }]>;
 def VecIndex_x4 : SDNodeXForm<imm, [{
-  return CurDAG->getTargetConstant(4 * N->getZExtValue(), MVT::i64);
+  return CurDAG->getTargetConstant(4 * N->getZExtValue(), SDLoc(N), MVT::i64);
 }]>;
 def VecIndex_x8 : SDNodeXForm<imm, [{
-  return CurDAG->getTargetConstant(8 * N->getZExtValue(), MVT::i64);
+  return CurDAG->getTargetConstant(8 * N->getZExtValue(), SDLoc(N), MVT::i64);
 }]>;
 
 multiclass DUPWithTruncPats<ValueType ResVT, ValueType Src64VT,
@@ -5014,10 +5150,10 @@ def : Pat<(i64 (zext def32:$src)), (SUBREG_TO_REG (i64 0), GPR32:$src, sub_32)>;
 def : Pat<(i64 (anyext GPR32:$src)),
           (INSERT_SUBREG (i64 (IMPLICIT_DEF)), GPR32:$src, sub_32)>;
 
-// When we need to explicitly zero-extend, we use an unsigned bitfield move
-// instruction (UBFM) on the enclosing super-reg.
+// When we need to explicitly zero-extend, we use a 32-bit MOV instruction and
+// then assert the extension has happened.
 def : Pat<(i64 (zext GPR32:$src)),
- (UBFMXri (INSERT_SUBREG (i64 (IMPLICIT_DEF)), GPR32:$src, sub_32), 0, 31)>;
+          (SUBREG_TO_REG (i32 0), (ORRWrs WZR, GPR32:$src, 0), sub_32)>;
 
 // To sign extend, we use a signed bitfield move instruction (SBFM) on the
 // containing super-reg.
@@ -5128,22 +5264,26 @@ def : Pat<(trap), (BRK 1)>;
 // Natural vector casts (64 bit)
 def : Pat<(v8i8 (AArch64NvCast (v2i32 FPR64:$src))), (v8i8 FPR64:$src)>;
 def : Pat<(v4i16 (AArch64NvCast (v2i32 FPR64:$src))), (v4i16 FPR64:$src)>;
+def : Pat<(v4f16 (AArch64NvCast (v2i32 FPR64:$src))), (v4f16 FPR64:$src)>;
 def : Pat<(v2i32 (AArch64NvCast (v2i32 FPR64:$src))), (v2i32 FPR64:$src)>;
 def : Pat<(v2f32 (AArch64NvCast (v2i32 FPR64:$src))), (v2f32 FPR64:$src)>;
 def : Pat<(v1i64 (AArch64NvCast (v2i32 FPR64:$src))), (v1i64 FPR64:$src)>;
 
 def : Pat<(v8i8 (AArch64NvCast (v4i16 FPR64:$src))), (v8i8 FPR64:$src)>;
 def : Pat<(v4i16 (AArch64NvCast (v4i16 FPR64:$src))), (v4i16 FPR64:$src)>;
+def : Pat<(v4f16 (AArch64NvCast (v4i16 FPR64:$src))), (v4f16 FPR64:$src)>;
 def : Pat<(v2i32 (AArch64NvCast (v4i16 FPR64:$src))), (v2i32 FPR64:$src)>;
 def : Pat<(v1i64 (AArch64NvCast (v4i16 FPR64:$src))), (v1i64 FPR64:$src)>;
 
 def : Pat<(v8i8 (AArch64NvCast (v8i8 FPR64:$src))), (v8i8 FPR64:$src)>;
 def : Pat<(v4i16 (AArch64NvCast (v8i8 FPR64:$src))), (v4i16 FPR64:$src)>;
+def : Pat<(v4f16 (AArch64NvCast (v8i8 FPR64:$src))), (v4f16 FPR64:$src)>;
 def : Pat<(v2i32 (AArch64NvCast (v8i8 FPR64:$src))), (v2i32 FPR64:$src)>;
 def : Pat<(v1i64 (AArch64NvCast (v8i8 FPR64:$src))), (v1i64 FPR64:$src)>;
 
 def : Pat<(v8i8 (AArch64NvCast (f64 FPR64:$src))), (v8i8 FPR64:$src)>;
 def : Pat<(v4i16 (AArch64NvCast (f64 FPR64:$src))), (v4i16 FPR64:$src)>;
+def : Pat<(v4f16 (AArch64NvCast (f64 FPR64:$src))), (v4f16 FPR64:$src)>;
 def : Pat<(v2i32 (AArch64NvCast (f64 FPR64:$src))), (v2i32 FPR64:$src)>;
 def : Pat<(v2f32 (AArch64NvCast (f64 FPR64:$src))), (v2f32 FPR64:$src)>;
 def : Pat<(v1i64 (AArch64NvCast (f64 FPR64:$src))), (v1i64 FPR64:$src)>;
@@ -5158,22 +5298,31 @@ def : Pat<(v1i64 (AArch64NvCast (v2f32 FPR64:$src))), (v1i64 FPR64:$src)>;
 // Natural vector casts (128 bit)
 def : Pat<(v16i8 (AArch64NvCast (v4i32 FPR128:$src))), (v16i8 FPR128:$src)>;
 def : Pat<(v8i16 (AArch64NvCast (v4i32 FPR128:$src))), (v8i16 FPR128:$src)>;
+def : Pat<(v8f16 (AArch64NvCast (v4i32 FPR128:$src))), (v8f16 FPR128:$src)>;
 def : Pat<(v4i32 (AArch64NvCast (v4i32 FPR128:$src))), (v4i32 FPR128:$src)>;
 def : Pat<(v4f32 (AArch64NvCast (v4i32 FPR128:$src))), (v4f32 FPR128:$src)>;
 def : Pat<(v2i64 (AArch64NvCast (v4i32 FPR128:$src))), (v2i64 FPR128:$src)>;
+def : Pat<(v2f64 (AArch64NvCast (v4i32 FPR128:$src))), (v2f64 FPR128:$src)>;
 
 def : Pat<(v16i8 (AArch64NvCast (v8i16 FPR128:$src))), (v16i8 FPR128:$src)>;
 def : Pat<(v8i16 (AArch64NvCast (v8i16 FPR128:$src))), (v8i16 FPR128:$src)>;
+def : Pat<(v8f16 (AArch64NvCast (v8i16 FPR128:$src))), (v8f16 FPR128:$src)>;
 def : Pat<(v4i32 (AArch64NvCast (v8i16 FPR128:$src))), (v4i32 FPR128:$src)>;
 def : Pat<(v2i64 (AArch64NvCast (v8i16 FPR128:$src))), (v2i64 FPR128:$src)>;
+def : Pat<(v4f32 (AArch64NvCast (v8i16 FPR128:$src))), (v4f32 FPR128:$src)>;
+def : Pat<(v2f64 (AArch64NvCast (v8i16 FPR128:$src))), (v2f64 FPR128:$src)>;
 
 def : Pat<(v16i8 (AArch64NvCast (v16i8 FPR128:$src))), (v16i8 FPR128:$src)>;
 def : Pat<(v8i16 (AArch64NvCast (v16i8 FPR128:$src))), (v8i16 FPR128:$src)>;
+def : Pat<(v8f16 (AArch64NvCast (v16i8 FPR128:$src))), (v8f16 FPR128:$src)>;
 def : Pat<(v4i32 (AArch64NvCast (v16i8 FPR128:$src))), (v4i32 FPR128:$src)>;
 def : Pat<(v2i64 (AArch64NvCast (v16i8 FPR128:$src))), (v2i64 FPR128:$src)>;
+def : Pat<(v4f32 (AArch64NvCast (v16i8 FPR128:$src))), (v4f32 FPR128:$src)>;
+def : Pat<(v2f64 (AArch64NvCast (v16i8 FPR128:$src))), (v2f64 FPR128:$src)>;
 
 def : Pat<(v16i8 (AArch64NvCast (v2i64 FPR128:$src))), (v16i8 FPR128:$src)>;
 def : Pat<(v8i16 (AArch64NvCast (v2i64 FPR128:$src))), (v8i16 FPR128:$src)>;
+def : Pat<(v8f16 (AArch64NvCast (v2i64 FPR128:$src))), (v8f16 FPR128:$src)>;
 def : Pat<(v4i32 (AArch64NvCast (v2i64 FPR128:$src))), (v4i32 FPR128:$src)>;
 def : Pat<(v2i64 (AArch64NvCast (v2i64 FPR128:$src))), (v2i64 FPR128:$src)>;
 def : Pat<(v4f32 (AArch64NvCast (v2i64 FPR128:$src))), (v4f32 FPR128:$src)>;
@@ -5184,12 +5333,16 @@ def : Pat<(v8i16 (AArch64NvCast (v4f32 FPR128:$src))), (v8i16 FPR128:$src)>;
 def : Pat<(v4i32 (AArch64NvCast (v4f32 FPR128:$src))), (v4i32 FPR128:$src)>;
 def : Pat<(v4f32 (AArch64NvCast (v4f32 FPR128:$src))), (v4f32 FPR128:$src)>;
 def : Pat<(v2i64 (AArch64NvCast (v4f32 FPR128:$src))), (v2i64 FPR128:$src)>;
+def : Pat<(v8f16 (AArch64NvCast (v4f32 FPR128:$src))), (v8f16 FPR128:$src)>;
+def : Pat<(v2f64 (AArch64NvCast (v4f32 FPR128:$src))), (v2f64 FPR128:$src)>;
 
 def : Pat<(v16i8 (AArch64NvCast (v2f64 FPR128:$src))), (v16i8 FPR128:$src)>;
 def : Pat<(v8i16 (AArch64NvCast (v2f64 FPR128:$src))), (v8i16 FPR128:$src)>;
 def : Pat<(v4i32 (AArch64NvCast (v2f64 FPR128:$src))), (v4i32 FPR128:$src)>;
 def : Pat<(v2i64 (AArch64NvCast (v2f64 FPR128:$src))), (v2i64 FPR128:$src)>;
 def : Pat<(v2f64 (AArch64NvCast (v2f64 FPR128:$src))), (v2f64 FPR128:$src)>;
+def : Pat<(v8f16 (AArch64NvCast (v2f64 FPR128:$src))), (v8f16 FPR128:$src)>;
+def : Pat<(v4f32 (AArch64NvCast (v2f64 FPR128:$src))), (v4f32 FPR128:$src)>;
 
 let Predicates = [IsLE] in {
 def : Pat<(v8i8  (bitconvert GPR64:$Xn)), (COPY_TO_REGCLASS GPR64:$Xn, FPR64)>;
diff --git a/lib/Target/AArch64/AArch64LoadStoreOptimizer.cpp b/lib/Target/AArch64/AArch64LoadStoreOptimizer.cpp
index b1499e2c06d..c1ff41c224c 100644
--- a/lib/Target/AArch64/AArch64LoadStoreOptimizer.cpp
+++ b/lib/Target/AArch64/AArch64LoadStoreOptimizer.cpp
@@ -16,6 +16,7 @@
 #include "AArch64Subtarget.h"
 #include "MCTargetDesc/AArch64AddressingModes.h"
 #include "llvm/ADT/BitVector.h"
+#include "llvm/ADT/SmallVector.h"
 #include "llvm/ADT/Statistic.h"
 #include "llvm/CodeGen/MachineBasicBlock.h"
 #include "llvm/CodeGen/MachineFunctionPass.h"
@@ -50,6 +51,29 @@ static cl::opt<bool> EnableAArch64UnscaledMemOp(
     cl::desc("Allow AArch64 unscaled load/store combining"), cl::init(true));
 
 namespace {
+
+typedef struct LdStPairFlags {
+  // If a matching instruction is found, MergeForward is set to true if the
+  // merge is to remove the first instruction and replace the second with
+  // a pair-wise insn, and false if the reverse is true.
+  bool MergeForward;
+
+  // SExtIdx gives the index of the result of the load pair that must be
+  // extended. The value of SExtIdx assumes that the paired load produces the
+  // value in this order: (I, returned iterator), i.e., -1 means no value has
+  // to be extended, 0 means I, and 1 means the returned iterator.
+  int SExtIdx;
+
+  LdStPairFlags() : MergeForward(false), SExtIdx(-1) {}
+
+  void setMergeForward(bool V = true) { MergeForward = V; }
+  bool getMergeForward() const { return MergeForward; }
+
+  void setSExtIdx(int V) { SExtIdx = V; }
+  int getSExtIdx() const { return SExtIdx; }
+
+} LdStPairFlags;
+
 struct AArch64LoadStoreOpt : public MachineFunctionPass {
   static char ID;
   AArch64LoadStoreOpt() : MachineFunctionPass(ID) {}
@@ -60,27 +84,17 @@ struct AArch64LoadStoreOpt : public MachineFunctionPass {
   // Scan the instructions looking for a load/store that can be combined
   // with the current instruction into a load/store pair.
   // Return the matching instruction if one is found, else MBB->end().
-  // If a matching instruction is found, MergeForward is set to true if the
-  // merge is to remove the first instruction and replace the second with
-  // a pair-wise insn, and false if the reverse is true.
-  // \p SExtIdx[out] gives the index of the result of the load pair that
-  // must be extended. The value of SExtIdx assumes that the paired load
-  // produces the value in this order: (I, returned iterator), i.e.,
-  // -1 means no value has to be extended, 0 means I, and 1 means the
-  // returned iterator.
   MachineBasicBlock::iterator findMatchingInsn(MachineBasicBlock::iterator I,
-                                               bool &MergeForward, int &SExtIdx,
+                                               LdStPairFlags &Flags,
                                                unsigned Limit);
   // Merge the two instructions indicated into a single pair-wise instruction.
   // If MergeForward is true, erase the first instruction and fold its
   // operation into the second. If false, the reverse. Return the instruction
   // following the first instruction (which may change during processing).
-  // \p SExtIdx index of the result that must be extended for a paired load.
-  // -1 means none, 0 means I, and 1 means Paired.
   MachineBasicBlock::iterator
   mergePairedInsns(MachineBasicBlock::iterator I,
-                   MachineBasicBlock::iterator Paired, bool MergeForward,
-                   int SExtIdx);
+                   MachineBasicBlock::iterator Paired,
+                   const LdStPairFlags &Flags);
 
   // Scan the instruction list to find a base register update that can
   // be combined with the current instruction (a load or store) using
@@ -124,25 +138,15 @@ static bool isUnscaledLdst(unsigned Opc) {
   default:
     return false;
   case AArch64::STURSi:
-    return true;
   case AArch64::STURDi:
-    return true;
   case AArch64::STURQi:
-    return true;
   case AArch64::STURWi:
-    return true;
   case AArch64::STURXi:
-    return true;
   case AArch64::LDURSi:
-    return true;
   case AArch64::LDURDi:
-    return true;
   case AArch64::LDURQi:
-    return true;
   case AArch64::LDURWi:
-    return true;
   case AArch64::LDURXi:
-    return true;
   case AArch64::LDURSWi:
     return true;
   }
@@ -327,7 +331,7 @@ static unsigned getPostIndexedOpcode(unsigned Opc) {
 MachineBasicBlock::iterator
 AArch64LoadStoreOpt::mergePairedInsns(MachineBasicBlock::iterator I,
                                       MachineBasicBlock::iterator Paired,
-                                      bool MergeForward, int SExtIdx) {
+                                      const LdStPairFlags &Flags) {
   MachineBasicBlock::iterator NextI = I;
   ++NextI;
   // If NextI is the second of the two instructions to be merged, we need
@@ -337,12 +341,14 @@ AArch64LoadStoreOpt::mergePairedInsns(MachineBasicBlock::iterator I,
   if (NextI == Paired)
     ++NextI;
 
+  int SExtIdx = Flags.getSExtIdx();
   unsigned Opc =
       SExtIdx == -1 ? I->getOpcode() : getMatchingNonSExtOpcode(I->getOpcode());
   bool IsUnscaled = isUnscaledLdst(Opc);
   int OffsetStride =
       IsUnscaled && EnableAArch64UnscaledMemOp ? getMemSize(I) : 1;
 
+  bool MergeForward = Flags.getMergeForward();
   unsigned NewOpc = getMatchingPairOpcode(Opc);
   // Insert our new paired instruction after whichever of the paired
   // instructions MergeForward indicates.
@@ -483,18 +489,41 @@ static int alignTo(int Num, int PowOf2) {
   return (Num + PowOf2 - 1) & ~(PowOf2 - 1);
 }
 
+static bool mayAlias(MachineInstr *MIa, MachineInstr *MIb,
+                     const AArch64InstrInfo *TII) {
+  // One of the instructions must modify memory.
+  if (!MIa->mayStore() && !MIb->mayStore())
+    return false;
+
+  // Both instructions must be memory operations.
+  if (!MIa->mayLoadOrStore() && !MIb->mayLoadOrStore())
+    return false;
+
+  return !TII->areMemAccessesTriviallyDisjoint(MIa, MIb);
+}
+
+static bool mayAlias(MachineInstr *MIa,
+                     SmallVectorImpl<MachineInstr *> &MemInsns,
+                     const AArch64InstrInfo *TII) {
+  for (auto &MIb : MemInsns)
+    if (mayAlias(MIa, MIb, TII))
+      return true;
+
+  return false;
+}
+
 /// findMatchingInsn - Scan the instructions looking for a load/store that can
 /// be combined with the current instruction into a load/store pair.
 MachineBasicBlock::iterator
 AArch64LoadStoreOpt::findMatchingInsn(MachineBasicBlock::iterator I,
-                                      bool &MergeForward, int &SExtIdx,
+                                      LdStPairFlags &Flags,
                                       unsigned Limit) {
   MachineBasicBlock::iterator E = I->getParent()->end();
   MachineBasicBlock::iterator MBBI = I;
   MachineInstr *FirstMI = I;
   ++MBBI;
 
-  int Opc = FirstMI->getOpcode();
+  unsigned Opc = FirstMI->getOpcode();
   bool MayLoad = FirstMI->mayLoad();
   bool IsUnscaled = isUnscaledLdst(Opc);
   unsigned Reg = FirstMI->getOperand(0).getReg();
@@ -518,6 +547,10 @@ AArch64LoadStoreOpt::findMatchingInsn(MachineBasicBlock::iterator I,
   BitVector ModifiedRegs, UsedRegs;
   ModifiedRegs.resize(TRI->getNumRegs());
   UsedRegs.resize(TRI->getNumRegs());
+
+  // Remember any instructions that read/write memory between FirstMI and MI.
+  SmallVector<MachineInstr *, 4> MemInsns;
+
   for (unsigned Count = 0; MBBI != E && Count < Limit; ++MBBI) {
     MachineInstr *MI = MBBI;
     // Skip DBG_VALUE instructions. Otherwise debug info can affect the
@@ -529,14 +562,14 @@ AArch64LoadStoreOpt::findMatchingInsn(MachineBasicBlock::iterator I,
     ++Count;
 
     bool CanMergeOpc = Opc == MI->getOpcode();
-    SExtIdx = -1;
+    Flags.setSExtIdx(-1);
     if (!CanMergeOpc) {
       bool IsValidLdStrOpc;
       unsigned NonSExtOpc = getMatchingNonSExtOpcode(Opc, &IsValidLdStrOpc);
       if (!IsValidLdStrOpc)
         continue;
       // Opc will be the first instruction in the pair.
-      SExtIdx = NonSExtOpc == (unsigned)Opc ? 1 : 0;
+      Flags.setSExtIdx(NonSExtOpc == (unsigned)Opc ? 1 : 0);
       CanMergeOpc = NonSExtOpc == getMatchingNonSExtOpcode(MI->getOpcode());
     }
 
@@ -566,6 +599,8 @@ AArch64LoadStoreOpt::findMatchingInsn(MachineBasicBlock::iterator I,
         bool MIIsUnscaled = isUnscaledLdst(MI->getOpcode());
         if (!inBoundsForPair(MIIsUnscaled, MinOffset, OffsetStride)) {
           trackRegDefsUses(MI, ModifiedRegs, UsedRegs, TRI);
+          if (MI->mayLoadOrStore())
+            MemInsns.push_back(MI);
           continue;
         }
         // If the alignment requirements of the paired (scaled) instruction
@@ -574,6 +609,8 @@ AArch64LoadStoreOpt::findMatchingInsn(MachineBasicBlock::iterator I,
         if (IsUnscaled && EnableAArch64UnscaledMemOp &&
             (alignTo(MinOffset, OffsetStride) != MinOffset)) {
           trackRegDefsUses(MI, ModifiedRegs, UsedRegs, TRI);
+          if (MI->mayLoadOrStore())
+            MemInsns.push_back(MI);
           continue;
         }
         // If the destination register of the loads is the same register, bail
@@ -581,23 +618,31 @@ AArch64LoadStoreOpt::findMatchingInsn(MachineBasicBlock::iterator I,
         // registers the same is UNPREDICTABLE and will result in an exception.
         if (MayLoad && Reg == MI->getOperand(0).getReg()) {
           trackRegDefsUses(MI, ModifiedRegs, UsedRegs, TRI);
+          if (MI->mayLoadOrStore())
+            MemInsns.push_back(MI);
           continue;
         }
 
         // If the Rt of the second instruction was not modified or used between
-        // the two instructions, we can combine the second into the first.
+        // the two instructions and none of the instructions between the second
+        // and first alias with the second, we can combine the second into the
+        // first.
         if (!ModifiedRegs[MI->getOperand(0).getReg()] &&
-            !UsedRegs[MI->getOperand(0).getReg()]) {
-          MergeForward = false;
+            !(MI->mayLoad() && UsedRegs[MI->getOperand(0).getReg()]) &&
+            !mayAlias(MI, MemInsns, TII)) {
+          Flags.setMergeForward(false);
           return MBBI;
         }
 
         // Likewise, if the Rt of the first instruction is not modified or used
-        // between the two instructions, we can combine the first into the
-        // second.
+        // between the two instructions and none of the instructions between the
+        // first and the second alias with the first, we can combine the first
+        // into the second.
         if (!ModifiedRegs[FirstMI->getOperand(0).getReg()] &&
-            !UsedRegs[FirstMI->getOperand(0).getReg()]) {
-          MergeForward = true;
+            !(FirstMI->mayLoad() &&
+              UsedRegs[FirstMI->getOperand(0).getReg()]) &&
+            !mayAlias(FirstMI, MemInsns, TII)) {
+          Flags.setMergeForward(true);
           return MBBI;
         }
         // Unable to combine these instructions due to interference in between.
@@ -605,21 +650,9 @@ AArch64LoadStoreOpt::findMatchingInsn(MachineBasicBlock::iterator I,
       }
     }
 
-    // If the instruction wasn't a matching load or store, but does (or can)
-    // modify memory, stop searching, as we don't have alias analysis or
-    // anything like that to tell us whether the access is tromping on the
-    // locations we care about. The big one we want to catch is calls.
-    //
-    // FIXME: Theoretically, we can do better than that for SP and FP based
-    // references since we can effectively know where those are touching. It's
-    // unclear if it's worth the extra code, though. Most paired instructions
-    // will be sequential, perhaps with a few intervening non-memory related
-    // instructions.
-    if (MI->mayStore() || MI->isCall())
-      return E;
-    // Likewise, if we're matching a store instruction, we don't want to
-    // move across a load, as it may be reading the same location.
-    if (FirstMI->mayStore() && MI->mayLoad())
+    // If the instruction wasn't a matching load or store.  Stop searching if we
+    // encounter a call instruction that might modify memory.
+    if (MI->isCall())
       return E;
 
     // Update modified / uses register lists.
@@ -629,6 +662,10 @@ AArch64LoadStoreOpt::findMatchingInsn(MachineBasicBlock::iterator I,
     // return early.
     if (ModifiedRegs[BaseReg])
       return E;
+
+    // Update list of instructions that read/write memory.
+    if (MI->mayLoadOrStore())
+      MemInsns.push_back(MI);
   }
   return E;
 }
@@ -926,15 +963,14 @@ bool AArch64LoadStoreOpt::optimizeBlock(MachineBasicBlock &MBB) {
         break;
       }
       // Look ahead up to ScanLimit instructions for a pairable instruction.
-      bool MergeForward = false;
-      int SExtIdx = -1;
+      LdStPairFlags Flags;
       MachineBasicBlock::iterator Paired =
-          findMatchingInsn(MBBI, MergeForward, SExtIdx, ScanLimit);
+          findMatchingInsn(MBBI, Flags, ScanLimit);
       if (Paired != E) {
         // Merge the loads into a pair. Keeping the iterator straight is a
         // pain, so we let the merge routine tell us what the next instruction
         // is after it's done mucking about.
-        MBBI = mergePairedInsns(MBBI, Paired, MergeForward, SExtIdx);
+        MBBI = mergePairedInsns(MBBI, Paired, Flags);
 
         Modified = true;
         ++NumPairCreated;
@@ -954,7 +990,7 @@ bool AArch64LoadStoreOpt::optimizeBlock(MachineBasicBlock &MBB) {
     MachineInstr *MI = MBBI;
     // Do update merging. It's simpler to keep this separate from the above
     // switch, though not strictly necessary.
-    int Opc = MI->getOpcode();
+    unsigned Opc = MI->getOpcode();
     switch (Opc) {
     default:
       // Just move on to the next instruction.
diff --git a/lib/Target/AArch64/AArch64MCInstLower.cpp b/lib/Target/AArch64/AArch64MCInstLower.cpp
index b82934134d9..580427ab3cc 100644
--- a/lib/Target/AArch64/AArch64MCInstLower.cpp
+++ b/lib/Target/AArch64/AArch64MCInstLower.cpp
@@ -69,11 +69,11 @@ MCOperand AArch64MCInstLower::lowerSymbolOperandDarwin(const MachineOperand &MO,
              AArch64II::MO_PAGEOFF)
       RefKind = MCSymbolRefExpr::VK_PAGEOFF;
   }
-  const MCExpr *Expr = MCSymbolRefExpr::Create(Sym, RefKind, Ctx);
+  const MCExpr *Expr = MCSymbolRefExpr::create(Sym, RefKind, Ctx);
   if (!MO.isJTI() && MO.getOffset())
-    Expr = MCBinaryExpr::CreateAdd(
-        Expr, MCConstantExpr::Create(MO.getOffset(), Ctx), Ctx);
-  return MCOperand::CreateExpr(Expr);
+    Expr = MCBinaryExpr::createAdd(
+        Expr, MCConstantExpr::create(MO.getOffset(), Ctx), Ctx);
+  return MCOperand::createExpr(Expr);
 }
 
 MCOperand AArch64MCInstLower::lowerSymbolOperandELF(const MachineOperand &MO,
@@ -139,16 +139,16 @@ MCOperand AArch64MCInstLower::lowerSymbolOperandELF(const MachineOperand &MO,
     RefFlags |= AArch64MCExpr::VK_NC;
 
   const MCExpr *Expr =
-      MCSymbolRefExpr::Create(Sym, MCSymbolRefExpr::VK_None, Ctx);
+      MCSymbolRefExpr::create(Sym, MCSymbolRefExpr::VK_None, Ctx);
   if (!MO.isJTI() && MO.getOffset())
-    Expr = MCBinaryExpr::CreateAdd(
-        Expr, MCConstantExpr::Create(MO.getOffset(), Ctx), Ctx);
+    Expr = MCBinaryExpr::createAdd(
+        Expr, MCConstantExpr::create(MO.getOffset(), Ctx), Ctx);
 
   AArch64MCExpr::VariantKind RefKind;
   RefKind = static_cast<AArch64MCExpr::VariantKind>(RefFlags);
-  Expr = AArch64MCExpr::Create(Expr, RefKind, Ctx);
+  Expr = AArch64MCExpr::create(Expr, RefKind, Ctx);
 
-  return MCOperand::CreateExpr(Expr);
+  return MCOperand::createExpr(Expr);
 }
 
 MCOperand AArch64MCInstLower::LowerSymbolOperand(const MachineOperand &MO,
@@ -169,17 +169,17 @@ bool AArch64MCInstLower::lowerOperand(const MachineOperand &MO,
     // Ignore all implicit register operands.
     if (MO.isImplicit())
       return false;
-    MCOp = MCOperand::CreateReg(MO.getReg());
+    MCOp = MCOperand::createReg(MO.getReg());
     break;
   case MachineOperand::MO_RegisterMask:
     // Regmasks are like implicit defs.
     return false;
   case MachineOperand::MO_Immediate:
-    MCOp = MCOperand::CreateImm(MO.getImm());
+    MCOp = MCOperand::createImm(MO.getImm());
     break;
   case MachineOperand::MO_MachineBasicBlock:
-    MCOp = MCOperand::CreateExpr(
-        MCSymbolRefExpr::Create(MO.getMBB()->getSymbol(), Ctx));
+    MCOp = MCOperand::createExpr(
+        MCSymbolRefExpr::create(MO.getMBB()->getSymbol(), Ctx));
     break;
   case MachineOperand::MO_GlobalAddress:
     MCOp = LowerSymbolOperand(MO, GetGlobalAddressSymbol(MO));
@@ -187,6 +187,9 @@ bool AArch64MCInstLower::lowerOperand(const MachineOperand &MO,
   case MachineOperand::MO_ExternalSymbol:
     MCOp = LowerSymbolOperand(MO, GetExternalSymbolSymbol(MO));
     break;
+  case MachineOperand::MO_MCSymbol:
+    MCOp = LowerSymbolOperand(MO, MO.getMCSymbol());
+    break;
   case MachineOperand::MO_JumpTableIndex:
     MCOp = LowerSymbolOperand(MO, Printer.GetJTISymbol(MO.getIndex()));
     break;
diff --git a/lib/Target/AArch64/AArch64MachineFunctionInfo.h b/lib/Target/AArch64/AArch64MachineFunctionInfo.h
index 536a8d0f97a..cc0ccc2771f 100644
--- a/lib/Target/AArch64/AArch64MachineFunctionInfo.h
+++ b/lib/Target/AArch64/AArch64MachineFunctionInfo.h
@@ -1,4 +1,4 @@
-//=- AArch64MachineFuctionInfo.h - AArch64 machine function info --*- C++ -*-=//
+//=- AArch64MachineFunctionInfo.h - AArch64 machine function info -*- C++ -*-=//
 //
 //                     The LLVM Compiler Infrastructure
 //
@@ -42,7 +42,7 @@ class AArch64FunctionInfo : public MachineFunctionInfo {
   unsigned ArgumentStackToRestore;
 
   /// HasStackFrame - True if this function has a stack frame. Set by
-  /// processFunctionBeforeCalleeSavedScan().
+  /// determineCalleeSaves().
   bool HasStackFrame;
 
   /// \brief Amount of stack frame size, not including callee-saved registers.
diff --git a/lib/Target/AArch64/AArch64RegisterInfo.cpp b/lib/Target/AArch64/AArch64RegisterInfo.cpp
index 1836682e386..1aef31baad2 100644
--- a/lib/Target/AArch64/AArch64RegisterInfo.cpp
+++ b/lib/Target/AArch64/AArch64RegisterInfo.cpp
@@ -34,10 +34,6 @@ using namespace llvm;
 #define GET_REGINFO_TARGET_DESC
 #include "AArch64GenRegisterInfo.inc"
 
-static cl::opt<bool>
-ReserveX18("aarch64-reserve-x18", cl::Hidden,
-          cl::desc("Reserve X18, making it unavailable as GPR"));
-
 AArch64RegisterInfo::AArch64RegisterInfo(const Triple &TT)
     : AArch64GenRegisterInfo(AArch64::LR), TT(TT) {}
 
@@ -90,7 +86,7 @@ AArch64RegisterInfo::getThisReturnPreservedMask(const MachineFunction &MF,
 
 BitVector
 AArch64RegisterInfo::getReservedRegs(const MachineFunction &MF) const {
-  const TargetFrameLowering *TFI = MF.getSubtarget().getFrameLowering();
+  const AArch64FrameLowering *TFI = getFrameLowering(MF);
 
   // FIXME: avoid re-calculating this every time.
   BitVector Reserved(getNumRegs());
@@ -104,7 +100,7 @@ AArch64RegisterInfo::getReservedRegs(const MachineFunction &MF) const {
     Reserved.set(AArch64::W29);
   }
 
-  if (TT.isOSDarwin() || ReserveX18) {
+  if (MF.getSubtarget<AArch64Subtarget>().isX18Reserved()) {
     Reserved.set(AArch64::X18); // Platform register
     Reserved.set(AArch64::W18);
   }
@@ -119,7 +115,7 @@ AArch64RegisterInfo::getReservedRegs(const MachineFunction &MF) const {
 
 bool AArch64RegisterInfo::isReservedReg(const MachineFunction &MF,
                                       unsigned Reg) const {
-  const TargetFrameLowering *TFI = MF.getSubtarget().getFrameLowering();
+  const AArch64FrameLowering *TFI = getFrameLowering(MF);
 
   switch (Reg) {
   default:
@@ -131,7 +127,7 @@ bool AArch64RegisterInfo::isReservedReg(const MachineFunction &MF,
     return true;
   case AArch64::X18:
   case AArch64::W18:
-    return TT.isOSDarwin() || ReserveX18;
+    return MF.getSubtarget<AArch64Subtarget>().isX18Reserved();
   case AArch64::FP:
   case AArch64::W29:
     return TFI->hasFP(MF) || TT.isOSDarwin();
@@ -186,35 +182,9 @@ bool AArch64RegisterInfo::hasBasePointer(const MachineFunction &MF) const {
   return false;
 }
 
-bool AArch64RegisterInfo::canRealignStack(const MachineFunction &MF) const {
-
-  if (MF.getFunction()->hasFnAttribute("no-realign-stack"))
-    return false;
-
-  return true;
-}
-
-// FIXME: share this with other backends with identical implementation?
-bool
-AArch64RegisterInfo::needsStackRealignment(const MachineFunction &MF) const {
-  const MachineFrameInfo *MFI = MF.getFrameInfo();
-  const Function *F = MF.getFunction();
-  unsigned StackAlign = MF.getTarget()
-                            .getSubtargetImpl(*MF.getFunction())
-                            ->getFrameLowering()
-                            ->getStackAlignment();
-  bool requiresRealignment =
-      ((MFI->getMaxAlignment() > StackAlign) ||
-       F->getAttributes().hasAttribute(AttributeSet::FunctionIndex,
-                                       Attribute::StackAlignment));
-
-  return requiresRealignment && canRealignStack(MF);
-}
-
 unsigned
 AArch64RegisterInfo::getFrameRegister(const MachineFunction &MF) const {
-  const TargetFrameLowering *TFI = MF.getSubtarget().getFrameLowering();
-
+  const AArch64FrameLowering *TFI = getFrameLowering(MF);
   return TFI->hasFP(MF) ? AArch64::FP : AArch64::SP;
 }
 
@@ -280,7 +250,7 @@ bool AArch64RegisterInfo::needsFrameBaseReg(MachineInstr *MI,
   // Note that the incoming offset is based on the SP value at function entry,
   // so it'll be negative.
   MachineFunction &MF = *MI->getParent()->getParent();
-  const TargetFrameLowering *TFI = MF.getSubtarget().getFrameLowering();
+  const AArch64FrameLowering *TFI = getFrameLowering(MF);
   MachineFrameInfo *MFI = MF.getFrameInfo();
 
   // Estimate an offset from the frame pointer.
@@ -376,8 +346,7 @@ void AArch64RegisterInfo::eliminateFrameIndex(MachineBasicBlock::iterator II,
   MachineFunction &MF = *MBB.getParent();
   const AArch64InstrInfo *TII =
       MF.getSubtarget<AArch64Subtarget>().getInstrInfo();
-  const AArch64FrameLowering *TFI = static_cast<const AArch64FrameLowering *>(
-      MF.getSubtarget().getFrameLowering());
+  const AArch64FrameLowering *TFI = getFrameLowering(MF);
 
   int FrameIndex = MI.getOperand(FIOperandNum).getIndex();
   unsigned FrameReg;
@@ -415,7 +384,7 @@ namespace llvm {
 
 unsigned AArch64RegisterInfo::getRegPressureLimit(const TargetRegisterClass *RC,
                                                   MachineFunction &MF) const {
-  const TargetFrameLowering *TFI = MF.getSubtarget().getFrameLowering();
+  const AArch64FrameLowering *TFI = getFrameLowering(MF);
 
   switch (RC->getID()) {
   default:
@@ -428,10 +397,11 @@ unsigned AArch64RegisterInfo::getRegPressureLimit(const TargetRegisterClass *RC,
   case AArch64::GPR64RegClassID:
   case AArch64::GPR32commonRegClassID:
   case AArch64::GPR64commonRegClassID:
-    return 32 - 1                                // XZR/SP
-           - (TFI->hasFP(MF) || TT.isOSDarwin()) // FP
-           - (TT.isOSDarwin() || ReserveX18) // X18 reserved as platform register
-           - hasBasePointer(MF);           // X19
+    return 32 - 1                                   // XZR/SP
+              - (TFI->hasFP(MF) || TT.isOSDarwin()) // FP
+              - MF.getSubtarget<AArch64Subtarget>()
+                    .isX18Reserved() // X18 reserved as platform register
+              - hasBasePointer(MF);  // X19
   case AArch64::FPR8RegClassID:
   case AArch64::FPR16RegClassID:
   case AArch64::FPR32RegClassID:
diff --git a/lib/Target/AArch64/AArch64RegisterInfo.h b/lib/Target/AArch64/AArch64RegisterInfo.h
index 8c379d92610..c01bfa5ea70 100644
--- a/lib/Target/AArch64/AArch64RegisterInfo.h
+++ b/lib/Target/AArch64/AArch64RegisterInfo.h
@@ -93,9 +93,6 @@ struct AArch64RegisterInfo : public AArch64GenRegisterInfo {
 
   unsigned getRegPressureLimit(const TargetRegisterClass *RC,
                                MachineFunction &MF) const override;
-  // Base pointer (stack realignment) support.
-  bool canRealignStack(const MachineFunction &MF) const;
-  bool needsStackRealignment(const MachineFunction &MF) const override;
 };
 
 } // end namespace llvm
diff --git a/lib/Target/AArch64/AArch64RegisterInfo.td b/lib/Target/AArch64/AArch64RegisterInfo.td
index d5ff3f1f337..b2efca02337 100644
--- a/lib/Target/AArch64/AArch64RegisterInfo.td
+++ b/lib/Target/AArch64/AArch64RegisterInfo.td
@@ -26,8 +26,12 @@ let Namespace = "AArch64" in {
   def hsub : SubRegIndex<16>;
   def ssub : SubRegIndex<32>;
   def dsub : SubRegIndex<32>;
+  def sube32 : SubRegIndex<32>;
+  def subo32 : SubRegIndex<32>;
   def qhisub : SubRegIndex<64>;
   def qsub : SubRegIndex<64>;
+  def sube64 : SubRegIndex<64>;
+  def subo64 : SubRegIndex<64>;
   // Note: Code depends on these having consecutive numbers
   def dsub0 : SubRegIndex<64>;
   def dsub1 : SubRegIndex<64>;
@@ -592,3 +596,40 @@ def FPR16Op : RegisterOperand<FPR16, "printOperand">;
 def FPR32Op : RegisterOperand<FPR32, "printOperand">;
 def FPR64Op : RegisterOperand<FPR64, "printOperand">;
 def FPR128Op : RegisterOperand<FPR128, "printOperand">;
+
+
+//===----------------------------------------------------------------------===//
+// ARMv8.1a atomic CASP register operands
+
+
+def WSeqPairs : RegisterTuples<[sube32, subo32], 
+                               [(rotl GPR32, 0), (rotl GPR32, 1)]>;
+def XSeqPairs : RegisterTuples<[sube64, subo64], 
+                               [(rotl GPR64, 0), (rotl GPR64, 1)]>;
+
+def WSeqPairsClass   : RegisterClass<"AArch64", [untyped], 32, 
+                                     (add WSeqPairs)>{
+  let Size = 64;
+}
+def XSeqPairsClass   : RegisterClass<"AArch64", [untyped], 64, 
+                                     (add XSeqPairs)>{
+  let Size = 128;
+}
+
+
+let RenderMethod = "addRegOperands", ParserMethod="tryParseGPRSeqPair" in {
+  def WSeqPairsAsmOperandClass : AsmOperandClass { let Name = "WSeqPair"; }
+  def XSeqPairsAsmOperandClass : AsmOperandClass { let Name = "XSeqPair"; }
+}
+
+def WSeqPairClassOperand :
+    RegisterOperand<WSeqPairsClass, "printGPRSeqPairsClassOperand<32>"> {
+  let ParserMatchClass = WSeqPairsAsmOperandClass;
+}
+def XSeqPairClassOperand :
+    RegisterOperand<XSeqPairsClass, "printGPRSeqPairsClassOperand<64>"> {
+  let ParserMatchClass = XSeqPairsAsmOperandClass;
+}
+
+
+//===----- END: v8.1a atomic CASP register operands -----------------------===//
diff --git a/lib/Target/AArch64/AArch64SelectionDAGInfo.cpp b/lib/Target/AArch64/AArch64SelectionDAGInfo.cpp
index b9c53998752..f40293021d7 100644
--- a/lib/Target/AArch64/AArch64SelectionDAGInfo.cpp
+++ b/lib/Target/AArch64/AArch64SelectionDAGInfo.cpp
@@ -16,11 +16,6 @@ using namespace llvm;
 
 #define DEBUG_TYPE "aarch64-selectiondag-info"
 
-AArch64SelectionDAGInfo::AArch64SelectionDAGInfo(const DataLayout *DL)
-    : TargetSelectionDAGInfo(DL) {}
-
-AArch64SelectionDAGInfo::~AArch64SelectionDAGInfo() {}
-
 SDValue AArch64SelectionDAGInfo::EmitTargetCodeForMemset(
     SelectionDAG &DAG, SDLoc dl, SDValue Chain, SDValue Dst, SDValue Src,
     SDValue Size, unsigned Align, bool isVolatile,
@@ -37,8 +32,8 @@ SDValue AArch64SelectionDAGInfo::EmitTargetCodeForMemset(
   if (bzeroEntry && (!SizeValue || SizeValue->getZExtValue() > 256)) {
     const AArch64TargetLowering &TLI = *STI.getTargetLowering();
 
-    EVT IntPtr = TLI.getPointerTy();
-    Type *IntPtrTy = getDataLayout()->getIntPtrType(*DAG.getContext());
+    EVT IntPtr = TLI.getPointerTy(DAG.getDataLayout());
+    Type *IntPtrTy = DAG.getDataLayout().getIntPtrType(*DAG.getContext());
     TargetLowering::ArgListTy Args;
     TargetLowering::ArgListEntry Entry;
     Entry.Node = Dst;
diff --git a/lib/Target/AArch64/AArch64SelectionDAGInfo.h b/lib/Target/AArch64/AArch64SelectionDAGInfo.h
index 11932d2b1c2..97421b45b12 100644
--- a/lib/Target/AArch64/AArch64SelectionDAGInfo.h
+++ b/lib/Target/AArch64/AArch64SelectionDAGInfo.h
@@ -20,8 +20,6 @@ namespace llvm {
 
 class AArch64SelectionDAGInfo : public TargetSelectionDAGInfo {
 public:
-  explicit AArch64SelectionDAGInfo(const DataLayout *DL);
-  ~AArch64SelectionDAGInfo();
 
   SDValue EmitTargetCodeForMemset(SelectionDAG &DAG, SDLoc dl, SDValue Chain,
                                   SDValue Dst, SDValue Src, SDValue Size,
diff --git a/lib/Target/AArch64/AArch64StorePairSuppress.cpp b/lib/Target/AArch64/AArch64StorePairSuppress.cpp
index 85b44a20e11..1c6b15790ea 100644
--- a/lib/Target/AArch64/AArch64StorePairSuppress.cpp
+++ b/lib/Target/AArch64/AArch64StorePairSuppress.cpp
@@ -142,7 +142,7 @@ bool AArch64StorePairSuppress::runOnMachineFunction(MachineFunction &MF) {
         continue;
       unsigned BaseReg;
       unsigned Offset;
-      if (TII->getLdStBaseRegImmOfs(&MI, BaseReg, Offset, TRI)) {
+      if (TII->getMemOpBaseRegImmOfs(&MI, BaseReg, Offset, TRI)) {
         if (PrevBaseReg == BaseReg) {
           // If this block can take STPs, skip ahead to the next block.
           if (!SuppressSTP && shouldAddSTPToBlock(MI.getParent()))
diff --git a/lib/Target/AArch64/AArch64Subtarget.cpp b/lib/Target/AArch64/AArch64Subtarget.cpp
index 0b97af80a6a..6aa3ed5cec2 100644
--- a/lib/Target/AArch64/AArch64Subtarget.cpp
+++ b/lib/Target/AArch64/AArch64Subtarget.cpp
@@ -42,24 +42,23 @@ AArch64Subtarget::initializeSubtargetDependencies(StringRef FS) {
   return *this;
 }
 
-AArch64Subtarget::AArch64Subtarget(const std::string &TT,
-                                   const std::string &CPU,
+AArch64Subtarget::AArch64Subtarget(const Triple &TT, const std::string &CPU,
                                    const std::string &FS,
                                    const TargetMachine &TM, bool LittleEndian)
     : AArch64GenSubtargetInfo(TT, CPU, FS), ARMProcFamily(Others),
-      HasV8_1aOps(false), 
-      HasFPARMv8(false), HasNEON(false), HasCrypto(false), HasCRC(false),
-      HasZeroCycleRegMove(false), HasZeroCycleZeroing(false),
-      IsLittle(LittleEndian), CPUString(CPU), TargetTriple(TT), FrameLowering(),
-      InstrInfo(initializeSubtargetDependencies(FS)),
-      TSInfo(TM.getDataLayout()), TLInfo(TM, *this) {}
+      HasV8_1aOps(false), HasFPARMv8(false), HasNEON(false), HasCrypto(false),
+      HasCRC(false), HasZeroCycleRegMove(false), HasZeroCycleZeroing(false),
+      StrictAlign(false), ReserveX18(false), IsLittle(LittleEndian),
+      CPUString(CPU), TargetTriple(TT), FrameLowering(),
+      InstrInfo(initializeSubtargetDependencies(FS)), TSInfo(),
+      TLInfo(TM, *this) {}
 
 /// ClassifyGlobalReference - Find the target operand flags that describe
 /// how a global value should be referenced for the current subtarget.
 unsigned char
 AArch64Subtarget::ClassifyGlobalReference(const GlobalValue *GV,
                                         const TargetMachine &TM) const {
-  bool isDecl = GV->isDeclarationForLinker();
+  bool isDef = GV->isStrongDefinitionForLinker();
 
   // MachO large model always goes via a GOT, simply to get a single 8-byte
   // absolute relocation on all global addresses.
@@ -68,8 +67,7 @@ AArch64Subtarget::ClassifyGlobalReference(const GlobalValue *GV,
 
   // The small code mode's direct accesses use ADRP, which cannot necessarily
   // produce the value 0 (if the code is above 4GB).
-  if (TM.getCodeModel() == CodeModel::Small &&
-      GV->isWeakForLinker() && isDecl) {
+  if (TM.getCodeModel() == CodeModel::Small && GV->hasExternalWeakLinkage()) {
     // In PIC mode use the GOT, but in absolute mode use a constant pool load.
     if (TM.getRelocationModel() == Reloc::Static)
         return AArch64II::MO_CONSTPOOL;
@@ -87,8 +85,7 @@ AArch64Subtarget::ClassifyGlobalReference(const GlobalValue *GV,
   //     defined could end up in unexpected places. Use a GOT.
   if (TM.getRelocationModel() != Reloc::Static && GV->hasDefaultVisibility()) {
     if (isTargetMachO())
-      return (isDecl || GV->isWeakForLinker()) ? AArch64II::MO_GOT
-                                               : AArch64II::MO_NO_FLAG;
+      return isDef ? AArch64II::MO_NO_FLAG : AArch64II::MO_GOT;
     else
       // No need to go through the GOT for local symbols on ELF.
       return GV->hasLocalLinkage() ? AArch64II::MO_NO_FLAG : AArch64II::MO_GOT;
diff --git a/lib/Target/AArch64/AArch64Subtarget.h b/lib/Target/AArch64/AArch64Subtarget.h
index 5454b205719..6b5ca20f033 100644
--- a/lib/Target/AArch64/AArch64Subtarget.h
+++ b/lib/Target/AArch64/AArch64Subtarget.h
@@ -29,6 +29,7 @@
 namespace llvm {
 class GlobalValue;
 class StringRef;
+class Triple;
 
 class AArch64Subtarget : public AArch64GenSubtargetInfo {
 protected:
@@ -50,6 +51,12 @@ class AArch64Subtarget : public AArch64GenSubtargetInfo {
   // HasZeroCycleZeroing - Has zero-cycle zeroing instructions.
   bool HasZeroCycleZeroing;
 
+  // StrictAlign - Disallow unaligned memory accesses.
+  bool StrictAlign;
+
+  // ReserveX18 - X18 is not available as a general purpose register.
+  bool ReserveX18;
+
   bool IsLittle;
 
   /// CPUString - String name of used CPU.
@@ -71,7 +78,7 @@ class AArch64Subtarget : public AArch64GenSubtargetInfo {
 public:
   /// This constructor initializes the data members to match that
   /// of the specified triple.
-  AArch64Subtarget(const std::string &TT, const std::string &CPU,
+  AArch64Subtarget(const Triple &TT, const std::string &CPU,
                    const std::string &FS, const TargetMachine &TM,
                    bool LittleEndian);
 
@@ -90,7 +97,7 @@ class AArch64Subtarget : public AArch64GenSubtargetInfo {
   }
   const Triple &getTargetTriple() const { return TargetTriple; }
   bool enableMachineScheduler() const override { return true; }
-  bool enablePostMachineScheduler() const override {
+  bool enablePostRAScheduler() const override {
     return isCortexA53() || isCortexA57();
   }
 
@@ -100,6 +107,9 @@ class AArch64Subtarget : public AArch64GenSubtargetInfo {
 
   bool hasZeroCycleZeroing() const { return HasZeroCycleZeroing; }
 
+  bool requiresStrictAlign() const { return StrictAlign; }
+
+  bool isX18Reserved() const { return ReserveX18; }
   bool hasFPARMv8() const { return HasFPARMv8; }
   bool hasNEON() const { return HasNEON; }
   bool hasCrypto() const { return HasCrypto; }
diff --git a/lib/Target/AArch64/AArch64TargetMachine.cpp b/lib/Target/AArch64/AArch64TargetMachine.cpp
index ab28a16515d..db6e244337a 100644
--- a/lib/Target/AArch64/AArch64TargetMachine.cpp
+++ b/lib/Target/AArch64/AArch64TargetMachine.cpp
@@ -85,7 +85,7 @@ EnableA53Fix835769("aarch64-fix-cortex-a53-835769", cl::Hidden,
 static cl::opt<bool>
 EnableGEPOpt("aarch64-gep-opt", cl::Hidden,
              cl::desc("Enable optimizations on complex GEPs"),
-             cl::init(true));
+             cl::init(false));
 
 // FIXME: Unify control over GlobalMerge.
 static cl::opt<cl::boolOrDefault>
@@ -110,9 +110,8 @@ static std::unique_ptr<TargetLoweringObjectFile> createTLOF(const Triple &TT) {
 }
 
 // Helper function to build a DataLayout string
-static std::string computeDataLayout(StringRef TT, bool LittleEndian) {
-  Triple Triple(TT);
-  if (Triple.isOSBinFormatMachO())
+static std::string computeDataLayout(const Triple &TT, bool LittleEndian) {
+  if (TT.isOSBinFormatMachO())
     return "e-m:o-i64:64-i128:128-n32:64-S128";
   if (LittleEndian)
     return "e-m:e-i64:64-i128:128-n32:64-S128";
@@ -121,7 +120,7 @@ static std::string computeDataLayout(StringRef TT, bool LittleEndian) {
 
 /// TargetMachine ctor - Create an AArch64 architecture model.
 ///
-AArch64TargetMachine::AArch64TargetMachine(const Target &T, StringRef TT,
+AArch64TargetMachine::AArch64TargetMachine(const Target &T, const Triple &TT,
                                            StringRef CPU, StringRef FS,
                                            const TargetOptions &Options,
                                            Reloc::Model RM, CodeModel::Model CM,
@@ -131,7 +130,7 @@ AArch64TargetMachine::AArch64TargetMachine(const Target &T, StringRef TT,
     // initialized before TLInfo is constructed.
     : LLVMTargetMachine(T, computeDataLayout(TT, LittleEndian), TT, CPU, FS,
                         Options, RM, CM, OL),
-      TLOF(createTLOF(Triple(getTargetTriple()))),
+      TLOF(createTLOF(getTargetTriple())),
       isLittle(LittleEndian) {
   initAsmInfo();
 }
@@ -156,28 +155,27 @@ AArch64TargetMachine::getSubtargetImpl(const Function &F) const {
     // creation will depend on the TM and the code generation flags on the
     // function that reside in TargetOptions.
     resetTargetOptions(F);
-    I = llvm::make_unique<AArch64Subtarget>(TargetTriple, CPU, FS, *this, isLittle);
+    I = llvm::make_unique<AArch64Subtarget>(TargetTriple, CPU, FS, *this,
+                                            isLittle);
   }
   return I.get();
 }
 
 void AArch64leTargetMachine::anchor() { }
 
-AArch64leTargetMachine::
-AArch64leTargetMachine(const Target &T, StringRef TT,
-                       StringRef CPU, StringRef FS, const TargetOptions &Options,
-                       Reloc::Model RM, CodeModel::Model CM,
-                       CodeGenOpt::Level OL)
-  : AArch64TargetMachine(T, TT, CPU, FS, Options, RM, CM, OL, true) {}
+AArch64leTargetMachine::AArch64leTargetMachine(
+    const Target &T, const Triple &TT, StringRef CPU, StringRef FS,
+    const TargetOptions &Options, Reloc::Model RM, CodeModel::Model CM,
+    CodeGenOpt::Level OL)
+    : AArch64TargetMachine(T, TT, CPU, FS, Options, RM, CM, OL, true) {}
 
 void AArch64beTargetMachine::anchor() { }
 
-AArch64beTargetMachine::
-AArch64beTargetMachine(const Target &T, StringRef TT,
-                       StringRef CPU, StringRef FS, const TargetOptions &Options,
-                       Reloc::Model RM, CodeModel::Model CM,
-                       CodeGenOpt::Level OL)
-  : AArch64TargetMachine(T, TT, CPU, FS, Options, RM, CM, OL, false) {}
+AArch64beTargetMachine::AArch64beTargetMachine(
+    const Target &T, const Triple &TT, StringRef CPU, StringRef FS,
+    const TargetOptions &Options, Reloc::Model RM, CodeModel::Model CM,
+    CodeGenOpt::Level OL)
+    : AArch64TargetMachine(T, TT, CPU, FS, Options, RM, CM, OL, false) {}
 
 namespace {
 /// AArch64 Code Generator Pass Configuration Options.
@@ -227,6 +225,10 @@ void AArch64PassConfig::addIRPasses() {
 
   TargetPassConfig::addIRPasses();
 
+  // Match interleaved memory accesses to ldN/stN intrinsics.
+  if (TM->getOptLevel() != CodeGenOpt::None)
+    addPass(createInterleavedAccessPass(TM));
+
   if (TM->getOptLevel() == CodeGenOpt::Aggressive && EnableGEPOpt) {
     // Call SeparateConstOffsetFromGEP pass to extract constants within indices
     // and lower a GEP with multiple indices to either arithmetic operations or
@@ -250,10 +252,14 @@ bool AArch64PassConfig::addPreISel() {
   // FIXME: On AArch64, this depends on the type.
   // Basically, the addressable offsets are up to 4095 * Ty.getSizeInBytes().
   // and the offset has to be a multiple of the related size in bytes.
-  if ((TM->getOptLevel() == CodeGenOpt::Aggressive &&
+  if ((TM->getOptLevel() != CodeGenOpt::None &&
        EnableGlobalMerge == cl::BOU_UNSET) ||
-      EnableGlobalMerge == cl::BOU_TRUE)
-    addPass(createGlobalMergePass(TM, 4095));
+      EnableGlobalMerge == cl::BOU_TRUE) {
+    bool OnlyOptimizeForSize = (TM->getOptLevel() < CodeGenOpt::Aggressive) &&
+                               (EnableGlobalMerge == cl::BOU_UNSET);
+    addPass(createGlobalMergePass(TM, 4095, OnlyOptimizeForSize));
+  }
+
   if (TM->getOptLevel() != CodeGenOpt::None)
     addPass(createAArch64AddressTypePromotionPass());
 
@@ -265,7 +271,7 @@ bool AArch64PassConfig::addInstSelector() {
 
   // For ELF, cleanup any local-dynamic TLS accesses (i.e. combine as many
   // references to _TLS_MODULE_BASE_ as possible.
-  if (Triple(TM->getTargetTriple()).isOSBinFormatELF() &&
+  if (TM->getTargetTriple().isOSBinFormatELF() &&
       getOptLevel() != CodeGenOpt::None)
     addPass(createAArch64CleanupLocalDynamicTLSPass());
 
@@ -320,6 +326,6 @@ void AArch64PassConfig::addPreEmitPass() {
   // range of their destination.
   addPass(createAArch64BranchRelaxation());
   if (TM->getOptLevel() != CodeGenOpt::None && EnableCollectLOH &&
-      Triple(TM->getTargetTriple()).isOSBinFormatMachO())
+      TM->getTargetTriple().isOSBinFormatMachO())
     addPass(createAArch64CollectLOHPass());
 }
diff --git a/lib/Target/AArch64/AArch64TargetMachine.h b/lib/Target/AArch64/AArch64TargetMachine.h
index ec34fad97c8..8d49a29386a 100644
--- a/lib/Target/AArch64/AArch64TargetMachine.h
+++ b/lib/Target/AArch64/AArch64TargetMachine.h
@@ -27,7 +27,7 @@ class AArch64TargetMachine : public LLVMTargetMachine {
   mutable StringMap<std::unique_ptr<AArch64Subtarget>> SubtargetMap;
 
 public:
-  AArch64TargetMachine(const Target &T, StringRef TT, StringRef CPU,
+  AArch64TargetMachine(const Target &T, const Triple &TT, StringRef CPU,
                        StringRef FS, const TargetOptions &Options,
                        Reloc::Model RM, CodeModel::Model CM,
                        CodeGenOpt::Level OL, bool IsLittleEndian);
@@ -54,7 +54,7 @@ class AArch64TargetMachine : public LLVMTargetMachine {
 class AArch64leTargetMachine : public AArch64TargetMachine {
   virtual void anchor();
 public:
-  AArch64leTargetMachine(const Target &T, StringRef TT, StringRef CPU,
+  AArch64leTargetMachine(const Target &T, const Triple &TT, StringRef CPU,
                          StringRef FS, const TargetOptions &Options,
                          Reloc::Model RM, CodeModel::Model CM,
                          CodeGenOpt::Level OL);
@@ -65,7 +65,7 @@ class AArch64leTargetMachine : public AArch64TargetMachine {
 class AArch64beTargetMachine : public AArch64TargetMachine {
   virtual void anchor();
 public:
-  AArch64beTargetMachine(const Target &T, StringRef TT, StringRef CPU,
+  AArch64beTargetMachine(const Target &T, const Triple &TT, StringRef CPU,
                          StringRef FS, const TargetOptions &Options,
                          Reloc::Model RM, CodeModel::Model CM,
                          CodeGenOpt::Level OL);
diff --git a/lib/Target/AArch64/AArch64TargetObjectFile.cpp b/lib/Target/AArch64/AArch64TargetObjectFile.cpp
index 8ff58e9864d..18ee4a9c72b 100644
--- a/lib/Target/AArch64/AArch64TargetObjectFile.cpp
+++ b/lib/Target/AArch64/AArch64TargetObjectFile.cpp
@@ -40,11 +40,11 @@ const MCExpr *AArch64_MachoTargetObjectFile::getTTypeGlobalReference(
   if (Encoding & (DW_EH_PE_indirect | DW_EH_PE_pcrel)) {
     const MCSymbol *Sym = TM.getSymbol(GV, Mang);
     const MCExpr *Res =
-        MCSymbolRefExpr::Create(Sym, MCSymbolRefExpr::VK_GOT, getContext());
-    MCSymbol *PCSym = getContext().CreateTempSymbol();
+        MCSymbolRefExpr::create(Sym, MCSymbolRefExpr::VK_GOT, getContext());
+    MCSymbol *PCSym = getContext().createTempSymbol();
     Streamer.EmitLabel(PCSym);
-    const MCExpr *PC = MCSymbolRefExpr::Create(PCSym, getContext());
-    return MCBinaryExpr::CreateSub(Res, PC, getContext());
+    const MCExpr *PC = MCSymbolRefExpr::create(PCSym, getContext());
+    return MCBinaryExpr::createSub(Res, PC, getContext());
   }
 
   return TargetLoweringObjectFileMachO::getTTypeGlobalReference(
@@ -65,9 +65,9 @@ const MCExpr *AArch64_MachoTargetObjectFile::getIndirectSymViaGOTPCRel(
   // On ARM64 Darwin, we can reference symbols with foo@GOT-., which
   // is an indirect pc-relative reference.
   const MCExpr *Res =
-      MCSymbolRefExpr::Create(Sym, MCSymbolRefExpr::VK_GOT, getContext());
-  MCSymbol *PCSym = getContext().CreateTempSymbol();
+      MCSymbolRefExpr::create(Sym, MCSymbolRefExpr::VK_GOT, getContext());
+  MCSymbol *PCSym = getContext().createTempSymbol();
   Streamer.EmitLabel(PCSym);
-  const MCExpr *PC = MCSymbolRefExpr::Create(PCSym, getContext());
-  return MCBinaryExpr::CreateSub(Res, PC, getContext());
+  const MCExpr *PC = MCSymbolRefExpr::create(PCSym, getContext());
+  return MCBinaryExpr::createSub(Res, PC, getContext());
 }
diff --git a/lib/Target/AArch64/AArch64TargetTransformInfo.cpp b/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
index 0533355b01d..1ffc6d960e9 100644
--- a/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
+++ b/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
@@ -181,8 +181,8 @@ unsigned AArch64TTIImpl::getCastInstrCost(unsigned Opcode, Type *Dst,
   int ISD = TLI->InstructionOpcodeToISD(Opcode);
   assert(ISD && "Invalid opcode");
 
-  EVT SrcTy = TLI->getValueType(Src);
-  EVT DstTy = TLI->getValueType(Dst);
+  EVT SrcTy = TLI->getValueType(DL, Src);
+  EVT DstTy = TLI->getValueType(DL, Dst);
 
   if (!SrcTy.isSimple() || !DstTy.isSimple())
     return BaseT::getCastInstrCost(Opcode, Dst, Src);
@@ -265,7 +265,7 @@ unsigned AArch64TTIImpl::getVectorInstrCost(unsigned Opcode, Type *Val,
 
   if (Index != -1U) {
     // Legalize the type.
-    std::pair<unsigned, MVT> LT = TLI->getTypeLegalizationCost(Val);
+    std::pair<unsigned, MVT> LT = TLI->getTypeLegalizationCost(DL, Val);
 
     // This type is legalized to a scalar type.
     if (!LT.second.isVector())
@@ -289,7 +289,7 @@ unsigned AArch64TTIImpl::getArithmeticInstrCost(
     TTI::OperandValueKind Opd2Info, TTI::OperandValueProperties Opd1PropInfo,
     TTI::OperandValueProperties Opd2PropInfo) {
   // Legalize the type.
-  std::pair<unsigned, MVT> LT = TLI->getTypeLegalizationCost(Ty);
+  std::pair<unsigned, MVT> LT = TLI->getTypeLegalizationCost(DL, Ty);
 
   int ISD = TLI->InstructionOpcodeToISD(Opcode);
 
@@ -364,8 +364,8 @@ unsigned AArch64TTIImpl::getCmpSelInstrCost(unsigned Opcode, Type *ValTy,
       { ISD::SELECT, MVT::v16i1, MVT::v16i64, 16 * AmortizationCost }
     };
 
-    EVT SelCondTy = TLI->getValueType(CondTy);
-    EVT SelValTy = TLI->getValueType(ValTy);
+    EVT SelCondTy = TLI->getValueType(DL, CondTy);
+    EVT SelValTy = TLI->getValueType(DL, ValTy);
     if (SelCondTy.isSimple() && SelValTy.isSimple()) {
       int Idx =
           ConvertCostTableLookup(VectorSelectTbl, ISD, SelCondTy.getSimpleVT(),
@@ -380,7 +380,7 @@ unsigned AArch64TTIImpl::getCmpSelInstrCost(unsigned Opcode, Type *ValTy,
 unsigned AArch64TTIImpl::getMemoryOpCost(unsigned Opcode, Type *Src,
                                          unsigned Alignment,
                                          unsigned AddressSpace) {
-  std::pair<unsigned, MVT> LT = TLI->getTypeLegalizationCost(Src);
+  std::pair<unsigned, MVT> LT = TLI->getTypeLegalizationCost(DL, Src);
 
   if (Opcode == Instruction::Store && Src->isVectorTy() && Alignment != 16 &&
       Src->getVectorElementType()->isIntegerTy(64)) {
@@ -407,6 +407,26 @@ unsigned AArch64TTIImpl::getMemoryOpCost(unsigned Opcode, Type *Src,
   return LT.first;
 }
 
+unsigned AArch64TTIImpl::getInterleavedMemoryOpCost(
+    unsigned Opcode, Type *VecTy, unsigned Factor, ArrayRef<unsigned> Indices,
+    unsigned Alignment, unsigned AddressSpace) {
+  assert(Factor >= 2 && "Invalid interleave factor");
+  assert(isa<VectorType>(VecTy) && "Expect a vector type");
+
+  if (Factor <= TLI->getMaxSupportedInterleaveFactor()) {
+    unsigned NumElts = VecTy->getVectorNumElements();
+    Type *SubVecTy = VectorType::get(VecTy->getScalarType(), NumElts / Factor);
+    unsigned SubVecSize = DL.getTypeAllocSizeInBits(SubVecTy);
+
+    // ldN/stN only support legal vector types of size 64 or 128 in bits.
+    if (NumElts % Factor == 0 && (SubVecSize == 64 || SubVecSize == 128))
+      return Factor;
+  }
+
+  return BaseT::getInterleavedMemoryOpCost(Opcode, VecTy, Factor, Indices,
+                                           Alignment, AddressSpace);
+}
+
 unsigned AArch64TTIImpl::getCostOfKeepingLiveOverCall(ArrayRef<Type *> Tys) {
   unsigned Cost = 0;
   for (auto *I : Tys) {
@@ -419,7 +439,7 @@ unsigned AArch64TTIImpl::getCostOfKeepingLiveOverCall(ArrayRef<Type *> Tys) {
   return Cost;
 }
 
-unsigned AArch64TTIImpl::getMaxInterleaveFactor() {
+unsigned AArch64TTIImpl::getMaxInterleaveFactor(unsigned VF) {
   if (ST->isCortexA57())
     return 4;
   return 2;
diff --git a/lib/Target/AArch64/AArch64TargetTransformInfo.h b/lib/Target/AArch64/AArch64TargetTransformInfo.h
index dd3fd1f5ab7..444d3ccc15e 100644
--- a/lib/Target/AArch64/AArch64TargetTransformInfo.h
+++ b/lib/Target/AArch64/AArch64TargetTransformInfo.h
@@ -31,7 +31,6 @@ class AArch64TTIImpl : public BasicTTIImplBase<AArch64TTIImpl> {
   typedef TargetTransformInfo TTI;
   friend BaseT;
 
-  const AArch64TargetMachine *TM;
   const AArch64Subtarget *ST;
   const AArch64TargetLowering *TLI;
 
@@ -50,30 +49,15 @@ class AArch64TTIImpl : public BasicTTIImplBase<AArch64TTIImpl> {
 
 public:
   explicit AArch64TTIImpl(const AArch64TargetMachine *TM, Function &F)
-      : BaseT(TM), TM(TM), ST(TM->getSubtargetImpl(F)),
+      : BaseT(TM, F.getParent()->getDataLayout()), ST(TM->getSubtargetImpl(F)),
         TLI(ST->getTargetLowering()) {}
 
   // Provide value semantics. MSVC requires that we spell all of these out.
   AArch64TTIImpl(const AArch64TTIImpl &Arg)
-      : BaseT(static_cast<const BaseT &>(Arg)), TM(Arg.TM), ST(Arg.ST),
-        TLI(Arg.TLI) {}
+      : BaseT(static_cast<const BaseT &>(Arg)), ST(Arg.ST), TLI(Arg.TLI) {}
   AArch64TTIImpl(AArch64TTIImpl &&Arg)
-      : BaseT(std::move(static_cast<BaseT &>(Arg))), TM(std::move(Arg.TM)),
-        ST(std::move(Arg.ST)), TLI(std::move(Arg.TLI)) {}
-  AArch64TTIImpl &operator=(const AArch64TTIImpl &RHS) {
-    BaseT::operator=(static_cast<const BaseT &>(RHS));
-    TM = RHS.TM;
-    ST = RHS.ST;
-    TLI = RHS.TLI;
-    return *this;
-  }
-  AArch64TTIImpl &operator=(AArch64TTIImpl &&RHS) {
-    BaseT::operator=(std::move(static_cast<BaseT &>(RHS)));
-    TM = std::move(RHS.TM);
-    ST = std::move(RHS.ST);
-    TLI = std::move(RHS.TLI);
-    return *this;
-  }
+      : BaseT(std::move(static_cast<BaseT &>(Arg))), ST(std::move(Arg.ST)),
+        TLI(std::move(Arg.TLI)) {}
 
   /// \name Scalar TTI Implementations
   /// @{
@@ -110,7 +94,7 @@ class AArch64TTIImpl : public BasicTTIImplBase<AArch64TTIImpl> {
     return 64;
   }
 
-  unsigned getMaxInterleaveFactor();
+  unsigned getMaxInterleaveFactor(unsigned VF);
 
   unsigned getCastInstrCost(unsigned Opcode, Type *Dst, Type *Src);
 
@@ -139,6 +123,11 @@ class AArch64TTIImpl : public BasicTTIImplBase<AArch64TTIImpl> {
 
   bool getTgtMemIntrinsic(IntrinsicInst *Inst, MemIntrinsicInfo &Info);
 
+  unsigned getInterleavedMemoryOpCost(unsigned Opcode, Type *VecTy,
+                                      unsigned Factor,
+                                      ArrayRef<unsigned> Indices,
+                                      unsigned Alignment,
+                                      unsigned AddressSpace);
   /// @}
 };
 
diff --git a/lib/Target/AArch64/AsmParser/AArch64AsmParser.cpp b/lib/Target/AArch64/AsmParser/AArch64AsmParser.cpp
index 063c7142564..44ed4dee029 100644
--- a/lib/Target/AArch64/AsmParser/AArch64AsmParser.cpp
+++ b/lib/Target/AArch64/AsmParser/AArch64AsmParser.cpp
@@ -9,6 +9,7 @@
 
 #include "MCTargetDesc/AArch64AddressingModes.h"
 #include "MCTargetDesc/AArch64MCExpr.h"
+#include "MCTargetDesc/AArch64TargetStreamer.h"
 #include "Utils/AArch64BaseInfo.h"
 #include "llvm/ADT/APInt.h"
 #include "llvm/ADT/STLExtras.h"
@@ -106,6 +107,7 @@ class AArch64AsmParser : public MCTargetAsmParser {
   OperandMatchResultTy tryParseAddSubImm(OperandVector &Operands);
   OperandMatchResultTy tryParseGPR64sp0Operand(OperandVector &Operands);
   bool tryParseVectorRegister(OperandVector &Operands);
+  OperandMatchResultTy tryParseGPRSeqPair(OperandVector &Operands);
 
 public:
   enum AArch64MatchResultTy {
@@ -115,7 +117,7 @@ class AArch64AsmParser : public MCTargetAsmParser {
   };
   AArch64AsmParser(MCSubtargetInfo &STI, MCAsmParser &Parser,
                    const MCInstrInfo &MII, const MCTargetOptions &Options)
-      : MCTargetAsmParser(), STI(STI) {
+      : MCTargetAsmParser(Options), STI(STI) {
     MCAsmParserExtension::Initialize(Parser);
     MCStreamer &S = getParser().getStreamer();
     if (S.getTargetStreamer() == nullptr)
@@ -697,6 +699,25 @@ class AArch64Operand : public MCParsedAsmOperand {
     const MCConstantExpr *CE = cast<MCConstantExpr>(Expr);
     return CE->getValue() >= 0 && CE->getValue() <= 0xfff;
   }
+  bool isAddSubImmNeg() const {
+    if (!isShiftedImm() && !isImm())
+      return false;
+
+    const MCExpr *Expr;
+
+    // An ADD/SUB shifter is either 'lsl #0' or 'lsl #12'.
+    if (isShiftedImm()) {
+      unsigned Shift = ShiftedImm.ShiftAmount;
+      Expr = ShiftedImm.Val;
+      if (Shift != 0 && Shift != 12)
+        return false;
+    } else
+      Expr = getImm();
+
+    // Otherwise it should be a real negative immediate in range:
+    const MCConstantExpr *CE = dyn_cast<MCConstantExpr>(Expr);
+    return CE != nullptr && CE->getValue() < 0 && -CE->getValue() <= 0xfff;
+  }
   bool isCondCode() const { return Kind == k_CondCode; }
   bool isSIMDImmType10() const {
     if (!isImm())
@@ -874,6 +895,16 @@ class AArch64Operand : public MCParsedAsmOperand {
     return Kind == k_Register && !Reg.isVector &&
       AArch64MCRegisterClasses[AArch64::GPR64RegClassID].contains(Reg.RegNum);
   }
+  bool isWSeqPair() const {
+    return Kind == k_Register && !Reg.isVector &&
+           AArch64MCRegisterClasses[AArch64::WSeqPairsClassRegClassID].contains(
+               Reg.RegNum);
+  }
+  bool isXSeqPair() const {
+    return Kind == k_Register && !Reg.isVector &&
+           AArch64MCRegisterClasses[AArch64::XSeqPairsClassRegClassID].contains(
+               Reg.RegNum);
+  }
 
   bool isGPR64sp0() const {
     return Kind == k_Register && !Reg.isVector &&
@@ -1098,16 +1129,16 @@ class AArch64Operand : public MCParsedAsmOperand {
   void addExpr(MCInst &Inst, const MCExpr *Expr) const {
     // Add as immediates when possible.  Null MCExpr = 0.
     if (!Expr)
-      Inst.addOperand(MCOperand::CreateImm(0));
+      Inst.addOperand(MCOperand::createImm(0));
     else if (const MCConstantExpr *CE = dyn_cast<MCConstantExpr>(Expr))
-      Inst.addOperand(MCOperand::CreateImm(CE->getValue()));
+      Inst.addOperand(MCOperand::createImm(CE->getValue()));
     else
-      Inst.addOperand(MCOperand::CreateExpr(Expr));
+      Inst.addOperand(MCOperand::createExpr(Expr));
   }
 
   void addRegOperands(MCInst &Inst, unsigned N) const {
     assert(N == 1 && "Invalid number of operands!");
-    Inst.addOperand(MCOperand::CreateReg(getReg()));
+    Inst.addOperand(MCOperand::createReg(getReg()));
   }
 
   void addGPR32as64Operands(MCInst &Inst, unsigned N) const {
@@ -1119,26 +1150,26 @@ class AArch64Operand : public MCParsedAsmOperand {
     uint32_t Reg = RI->getRegClass(AArch64::GPR32RegClassID).getRegister(
         RI->getEncodingValue(getReg()));
 
-    Inst.addOperand(MCOperand::CreateReg(Reg));
+    Inst.addOperand(MCOperand::createReg(Reg));
   }
 
   void addVectorReg64Operands(MCInst &Inst, unsigned N) const {
     assert(N == 1 && "Invalid number of operands!");
     assert(
         AArch64MCRegisterClasses[AArch64::FPR128RegClassID].contains(getReg()));
-    Inst.addOperand(MCOperand::CreateReg(AArch64::D0 + getReg() - AArch64::Q0));
+    Inst.addOperand(MCOperand::createReg(AArch64::D0 + getReg() - AArch64::Q0));
   }
 
   void addVectorReg128Operands(MCInst &Inst, unsigned N) const {
     assert(N == 1 && "Invalid number of operands!");
     assert(
         AArch64MCRegisterClasses[AArch64::FPR128RegClassID].contains(getReg()));
-    Inst.addOperand(MCOperand::CreateReg(getReg()));
+    Inst.addOperand(MCOperand::createReg(getReg()));
   }
 
   void addVectorRegLoOperands(MCInst &Inst, unsigned N) const {
     assert(N == 1 && "Invalid number of operands!");
-    Inst.addOperand(MCOperand::CreateReg(getReg()));
+    Inst.addOperand(MCOperand::createReg(getReg()));
   }
 
   template <unsigned NumRegs>
@@ -1149,7 +1180,7 @@ class AArch64Operand : public MCParsedAsmOperand {
     unsigned FirstReg = FirstRegs[NumRegs - 1];
 
     Inst.addOperand(
-        MCOperand::CreateReg(FirstReg + getVectorListStart() - AArch64::Q0));
+        MCOperand::createReg(FirstReg + getVectorListStart() - AArch64::Q0));
   }
 
   template <unsigned NumRegs>
@@ -1160,32 +1191,32 @@ class AArch64Operand : public MCParsedAsmOperand {
     unsigned FirstReg = FirstRegs[NumRegs - 1];
 
     Inst.addOperand(
-        MCOperand::CreateReg(FirstReg + getVectorListStart() - AArch64::Q0));
+        MCOperand::createReg(FirstReg + getVectorListStart() - AArch64::Q0));
   }
 
   void addVectorIndex1Operands(MCInst &Inst, unsigned N) const {
     assert(N == 1 && "Invalid number of operands!");
-    Inst.addOperand(MCOperand::CreateImm(getVectorIndex()));
+    Inst.addOperand(MCOperand::createImm(getVectorIndex()));
   }
 
   void addVectorIndexBOperands(MCInst &Inst, unsigned N) const {
     assert(N == 1 && "Invalid number of operands!");
-    Inst.addOperand(MCOperand::CreateImm(getVectorIndex()));
+    Inst.addOperand(MCOperand::createImm(getVectorIndex()));
   }
 
   void addVectorIndexHOperands(MCInst &Inst, unsigned N) const {
     assert(N == 1 && "Invalid number of operands!");
-    Inst.addOperand(MCOperand::CreateImm(getVectorIndex()));
+    Inst.addOperand(MCOperand::createImm(getVectorIndex()));
   }
 
   void addVectorIndexSOperands(MCInst &Inst, unsigned N) const {
     assert(N == 1 && "Invalid number of operands!");
-    Inst.addOperand(MCOperand::CreateImm(getVectorIndex()));
+    Inst.addOperand(MCOperand::createImm(getVectorIndex()));
   }
 
   void addVectorIndexDOperands(MCInst &Inst, unsigned N) const {
     assert(N == 1 && "Invalid number of operands!");
-    Inst.addOperand(MCOperand::CreateImm(getVectorIndex()));
+    Inst.addOperand(MCOperand::createImm(getVectorIndex()));
   }
 
   void addImmOperands(MCInst &Inst, unsigned N) const {
@@ -1200,16 +1231,28 @@ class AArch64Operand : public MCParsedAsmOperand {
     assert(N == 2 && "Invalid number of operands!");
     if (isShiftedImm()) {
       addExpr(Inst, getShiftedImmVal());
-      Inst.addOperand(MCOperand::CreateImm(getShiftedImmShift()));
+      Inst.addOperand(MCOperand::createImm(getShiftedImmShift()));
     } else {
       addExpr(Inst, getImm());
-      Inst.addOperand(MCOperand::CreateImm(0));
+      Inst.addOperand(MCOperand::createImm(0));
     }
   }
 
+  void addAddSubImmNegOperands(MCInst &Inst, unsigned N) const {
+    assert(N == 2 && "Invalid number of operands!");
+
+    const MCExpr *MCE = isShiftedImm() ? getShiftedImmVal() : getImm();
+    const MCConstantExpr *CE = cast<MCConstantExpr>(MCE);
+    int64_t Val = -CE->getValue();
+    unsigned ShiftAmt = isShiftedImm() ? ShiftedImm.ShiftAmount : 0;
+
+    Inst.addOperand(MCOperand::createImm(Val));
+    Inst.addOperand(MCOperand::createImm(ShiftAmt));
+  }
+
   void addCondCodeOperands(MCInst &Inst, unsigned N) const {
     assert(N == 1 && "Invalid number of operands!");
-    Inst.addOperand(MCOperand::CreateImm(getCondCode()));
+    Inst.addOperand(MCOperand::createImm(getCondCode()));
   }
 
   void addAdrpLabelOperands(MCInst &Inst, unsigned N) const {
@@ -1218,7 +1261,7 @@ class AArch64Operand : public MCParsedAsmOperand {
     if (!MCE)
       addExpr(Inst, getImm());
     else
-      Inst.addOperand(MCOperand::CreateImm(MCE->getValue() >> 12));
+      Inst.addOperand(MCOperand::createImm(MCE->getValue() >> 12));
   }
 
   void addAdrLabelOperands(MCInst &Inst, unsigned N) const {
@@ -1231,119 +1274,119 @@ class AArch64Operand : public MCParsedAsmOperand {
     const MCConstantExpr *MCE = dyn_cast<MCConstantExpr>(getImm());
 
     if (!MCE) {
-      Inst.addOperand(MCOperand::CreateExpr(getImm()));
+      Inst.addOperand(MCOperand::createExpr(getImm()));
       return;
     }
-    Inst.addOperand(MCOperand::CreateImm(MCE->getValue() / Scale));
+    Inst.addOperand(MCOperand::createImm(MCE->getValue() / Scale));
   }
 
   void addSImm9Operands(MCInst &Inst, unsigned N) const {
     assert(N == 1 && "Invalid number of operands!");
     const MCConstantExpr *MCE = cast<MCConstantExpr>(getImm());
-    Inst.addOperand(MCOperand::CreateImm(MCE->getValue()));
+    Inst.addOperand(MCOperand::createImm(MCE->getValue()));
   }
 
   void addSImm7s4Operands(MCInst &Inst, unsigned N) const {
     assert(N == 1 && "Invalid number of operands!");
     const MCConstantExpr *MCE = cast<MCConstantExpr>(getImm());
-    Inst.addOperand(MCOperand::CreateImm(MCE->getValue() / 4));
+    Inst.addOperand(MCOperand::createImm(MCE->getValue() / 4));
   }
 
   void addSImm7s8Operands(MCInst &Inst, unsigned N) const {
     assert(N == 1 && "Invalid number of operands!");
     const MCConstantExpr *MCE = cast<MCConstantExpr>(getImm());
-    Inst.addOperand(MCOperand::CreateImm(MCE->getValue() / 8));
+    Inst.addOperand(MCOperand::createImm(MCE->getValue() / 8));
   }
 
   void addSImm7s16Operands(MCInst &Inst, unsigned N) const {
     assert(N == 1 && "Invalid number of operands!");
     const MCConstantExpr *MCE = cast<MCConstantExpr>(getImm());
-    Inst.addOperand(MCOperand::CreateImm(MCE->getValue() / 16));
+    Inst.addOperand(MCOperand::createImm(MCE->getValue() / 16));
   }
 
   void addImm0_7Operands(MCInst &Inst, unsigned N) const {
     assert(N == 1 && "Invalid number of operands!");
     const MCConstantExpr *MCE = cast<MCConstantExpr>(getImm());
-    Inst.addOperand(MCOperand::CreateImm(MCE->getValue()));
+    Inst.addOperand(MCOperand::createImm(MCE->getValue()));
   }
 
   void addImm1_8Operands(MCInst &Inst, unsigned N) const {
     assert(N == 1 && "Invalid number of operands!");
     const MCConstantExpr *MCE = cast<MCConstantExpr>(getImm());
-    Inst.addOperand(MCOperand::CreateImm(MCE->getValue()));
+    Inst.addOperand(MCOperand::createImm(MCE->getValue()));
   }
 
   void addImm0_15Operands(MCInst &Inst, unsigned N) const {
     assert(N == 1 && "Invalid number of operands!");
     const MCConstantExpr *MCE = cast<MCConstantExpr>(getImm());
-    Inst.addOperand(MCOperand::CreateImm(MCE->getValue()));
+    Inst.addOperand(MCOperand::createImm(MCE->getValue()));
   }
 
   void addImm1_16Operands(MCInst &Inst, unsigned N) const {
     assert(N == 1 && "Invalid number of operands!");
     const MCConstantExpr *MCE = cast<MCConstantExpr>(getImm());
     assert(MCE && "Invalid constant immediate operand!");
-    Inst.addOperand(MCOperand::CreateImm(MCE->getValue()));
+    Inst.addOperand(MCOperand::createImm(MCE->getValue()));
   }
 
   void addImm0_31Operands(MCInst &Inst, unsigned N) const {
     assert(N == 1 && "Invalid number of operands!");
     const MCConstantExpr *MCE = cast<MCConstantExpr>(getImm());
-    Inst.addOperand(MCOperand::CreateImm(MCE->getValue()));
+    Inst.addOperand(MCOperand::createImm(MCE->getValue()));
   }
 
   void addImm1_31Operands(MCInst &Inst, unsigned N) const {
     assert(N == 1 && "Invalid number of operands!");
     const MCConstantExpr *MCE = cast<MCConstantExpr>(getImm());
-    Inst.addOperand(MCOperand::CreateImm(MCE->getValue()));
+    Inst.addOperand(MCOperand::createImm(MCE->getValue()));
   }
 
   void addImm1_32Operands(MCInst &Inst, unsigned N) const {
     assert(N == 1 && "Invalid number of operands!");
     const MCConstantExpr *MCE = cast<MCConstantExpr>(getImm());
-    Inst.addOperand(MCOperand::CreateImm(MCE->getValue()));
+    Inst.addOperand(MCOperand::createImm(MCE->getValue()));
   }
 
   void addImm0_63Operands(MCInst &Inst, unsigned N) const {
     assert(N == 1 && "Invalid number of operands!");
     const MCConstantExpr *MCE = cast<MCConstantExpr>(getImm());
-    Inst.addOperand(MCOperand::CreateImm(MCE->getValue()));
+    Inst.addOperand(MCOperand::createImm(MCE->getValue()));
   }
 
   void addImm1_63Operands(MCInst &Inst, unsigned N) const {
     assert(N == 1 && "Invalid number of operands!");
     const MCConstantExpr *MCE = cast<MCConstantExpr>(getImm());
-    Inst.addOperand(MCOperand::CreateImm(MCE->getValue()));
+    Inst.addOperand(MCOperand::createImm(MCE->getValue()));
   }
 
   void addImm1_64Operands(MCInst &Inst, unsigned N) const {
     assert(N == 1 && "Invalid number of operands!");
     const MCConstantExpr *MCE = cast<MCConstantExpr>(getImm());
-    Inst.addOperand(MCOperand::CreateImm(MCE->getValue()));
+    Inst.addOperand(MCOperand::createImm(MCE->getValue()));
   }
 
   void addImm0_127Operands(MCInst &Inst, unsigned N) const {
     assert(N == 1 && "Invalid number of operands!");
     const MCConstantExpr *MCE = cast<MCConstantExpr>(getImm());
-    Inst.addOperand(MCOperand::CreateImm(MCE->getValue()));
+    Inst.addOperand(MCOperand::createImm(MCE->getValue()));
   }
 
   void addImm0_255Operands(MCInst &Inst, unsigned N) const {
     assert(N == 1 && "Invalid number of operands!");
     const MCConstantExpr *MCE = cast<MCConstantExpr>(getImm());
-    Inst.addOperand(MCOperand::CreateImm(MCE->getValue()));
+    Inst.addOperand(MCOperand::createImm(MCE->getValue()));
   }
 
   void addImm0_65535Operands(MCInst &Inst, unsigned N) const {
     assert(N == 1 && "Invalid number of operands!");
     const MCConstantExpr *MCE = cast<MCConstantExpr>(getImm());
-    Inst.addOperand(MCOperand::CreateImm(MCE->getValue()));
+    Inst.addOperand(MCOperand::createImm(MCE->getValue()));
   }
 
   void addImm32_63Operands(MCInst &Inst, unsigned N) const {
     assert(N == 1 && "Invalid number of operands!");
     const MCConstantExpr *MCE = cast<MCConstantExpr>(getImm());
-    Inst.addOperand(MCOperand::CreateImm(MCE->getValue()));
+    Inst.addOperand(MCOperand::createImm(MCE->getValue()));
   }
 
   void addLogicalImm32Operands(MCInst &Inst, unsigned N) const {
@@ -1351,14 +1394,14 @@ class AArch64Operand : public MCParsedAsmOperand {
     const MCConstantExpr *MCE = cast<MCConstantExpr>(getImm());
     uint64_t encoding =
         AArch64_AM::encodeLogicalImmediate(MCE->getValue() & 0xFFFFFFFF, 32);
-    Inst.addOperand(MCOperand::CreateImm(encoding));
+    Inst.addOperand(MCOperand::createImm(encoding));
   }
 
   void addLogicalImm64Operands(MCInst &Inst, unsigned N) const {
     assert(N == 1 && "Invalid number of operands!");
     const MCConstantExpr *MCE = cast<MCConstantExpr>(getImm());
     uint64_t encoding = AArch64_AM::encodeLogicalImmediate(MCE->getValue(), 64);
-    Inst.addOperand(MCOperand::CreateImm(encoding));
+    Inst.addOperand(MCOperand::createImm(encoding));
   }
 
   void addLogicalImm32NotOperands(MCInst &Inst, unsigned N) const {
@@ -1366,7 +1409,7 @@ class AArch64Operand : public MCParsedAsmOperand {
     const MCConstantExpr *MCE = cast<MCConstantExpr>(getImm());
     int64_t Val = ~MCE->getValue() & 0xFFFFFFFF;
     uint64_t encoding = AArch64_AM::encodeLogicalImmediate(Val, 32);
-    Inst.addOperand(MCOperand::CreateImm(encoding));
+    Inst.addOperand(MCOperand::createImm(encoding));
   }
 
   void addLogicalImm64NotOperands(MCInst &Inst, unsigned N) const {
@@ -1374,14 +1417,14 @@ class AArch64Operand : public MCParsedAsmOperand {
     const MCConstantExpr *MCE = cast<MCConstantExpr>(getImm());
     uint64_t encoding =
         AArch64_AM::encodeLogicalImmediate(~MCE->getValue(), 64);
-    Inst.addOperand(MCOperand::CreateImm(encoding));
+    Inst.addOperand(MCOperand::createImm(encoding));
   }
 
   void addSIMDImmType10Operands(MCInst &Inst, unsigned N) const {
     assert(N == 1 && "Invalid number of operands!");
     const MCConstantExpr *MCE = cast<MCConstantExpr>(getImm());
     uint64_t encoding = AArch64_AM::encodeAdvSIMDModImmType10(MCE->getValue());
-    Inst.addOperand(MCOperand::CreateImm(encoding));
+    Inst.addOperand(MCOperand::createImm(encoding));
   }
 
   void addBranchTarget26Operands(MCInst &Inst, unsigned N) const {
@@ -1395,7 +1438,7 @@ class AArch64Operand : public MCParsedAsmOperand {
       return;
     }
     assert(MCE && "Invalid constant immediate operand!");
-    Inst.addOperand(MCOperand::CreateImm(MCE->getValue() >> 2));
+    Inst.addOperand(MCOperand::createImm(MCE->getValue() >> 2));
   }
 
   void addPCRelLabel19Operands(MCInst &Inst, unsigned N) const {
@@ -1409,7 +1452,7 @@ class AArch64Operand : public MCParsedAsmOperand {
       return;
     }
     assert(MCE && "Invalid constant immediate operand!");
-    Inst.addOperand(MCOperand::CreateImm(MCE->getValue() >> 2));
+    Inst.addOperand(MCOperand::createImm(MCE->getValue() >> 2));
   }
 
   void addBranchTarget14Operands(MCInst &Inst, unsigned N) const {
@@ -1423,52 +1466,52 @@ class AArch64Operand : public MCParsedAsmOperand {
       return;
     }
     assert(MCE && "Invalid constant immediate operand!");
-    Inst.addOperand(MCOperand::CreateImm(MCE->getValue() >> 2));
+    Inst.addOperand(MCOperand::createImm(MCE->getValue() >> 2));
   }
 
   void addFPImmOperands(MCInst &Inst, unsigned N) const {
     assert(N == 1 && "Invalid number of operands!");
-    Inst.addOperand(MCOperand::CreateImm(getFPImm()));
+    Inst.addOperand(MCOperand::createImm(getFPImm()));
   }
 
   void addBarrierOperands(MCInst &Inst, unsigned N) const {
     assert(N == 1 && "Invalid number of operands!");
-    Inst.addOperand(MCOperand::CreateImm(getBarrier()));
+    Inst.addOperand(MCOperand::createImm(getBarrier()));
   }
 
   void addMRSSystemRegisterOperands(MCInst &Inst, unsigned N) const {
     assert(N == 1 && "Invalid number of operands!");
 
-    Inst.addOperand(MCOperand::CreateImm(SysReg.MRSReg));
+    Inst.addOperand(MCOperand::createImm(SysReg.MRSReg));
   }
 
   void addMSRSystemRegisterOperands(MCInst &Inst, unsigned N) const {
     assert(N == 1 && "Invalid number of operands!");
 
-    Inst.addOperand(MCOperand::CreateImm(SysReg.MSRReg));
+    Inst.addOperand(MCOperand::createImm(SysReg.MSRReg));
   }
 
   void addSystemPStateFieldOperands(MCInst &Inst, unsigned N) const {
     assert(N == 1 && "Invalid number of operands!");
 
-    Inst.addOperand(MCOperand::CreateImm(SysReg.PStateField));
+    Inst.addOperand(MCOperand::createImm(SysReg.PStateField));
   }
 
   void addSysCROperands(MCInst &Inst, unsigned N) const {
     assert(N == 1 && "Invalid number of operands!");
-    Inst.addOperand(MCOperand::CreateImm(getSysCR()));
+    Inst.addOperand(MCOperand::createImm(getSysCR()));
   }
 
   void addPrefetchOperands(MCInst &Inst, unsigned N) const {
     assert(N == 1 && "Invalid number of operands!");
-    Inst.addOperand(MCOperand::CreateImm(getPrefetch()));
+    Inst.addOperand(MCOperand::createImm(getPrefetch()));
   }
 
   void addShifterOperands(MCInst &Inst, unsigned N) const {
     assert(N == 1 && "Invalid number of operands!");
     unsigned Imm =
         AArch64_AM::getShifterImm(getShiftExtendType(), getShiftExtendAmount());
-    Inst.addOperand(MCOperand::CreateImm(Imm));
+    Inst.addOperand(MCOperand::createImm(Imm));
   }
 
   void addExtendOperands(MCInst &Inst, unsigned N) const {
@@ -1476,7 +1519,7 @@ class AArch64Operand : public MCParsedAsmOperand {
     AArch64_AM::ShiftExtendType ET = getShiftExtendType();
     if (ET == AArch64_AM::LSL) ET = AArch64_AM::UXTW;
     unsigned Imm = AArch64_AM::getArithExtendImm(ET, getShiftExtendAmount());
-    Inst.addOperand(MCOperand::CreateImm(Imm));
+    Inst.addOperand(MCOperand::createImm(Imm));
   }
 
   void addExtend64Operands(MCInst &Inst, unsigned N) const {
@@ -1484,15 +1527,15 @@ class AArch64Operand : public MCParsedAsmOperand {
     AArch64_AM::ShiftExtendType ET = getShiftExtendType();
     if (ET == AArch64_AM::LSL) ET = AArch64_AM::UXTX;
     unsigned Imm = AArch64_AM::getArithExtendImm(ET, getShiftExtendAmount());
-    Inst.addOperand(MCOperand::CreateImm(Imm));
+    Inst.addOperand(MCOperand::createImm(Imm));
   }
 
   void addMemExtendOperands(MCInst &Inst, unsigned N) const {
     assert(N == 2 && "Invalid number of operands!");
     AArch64_AM::ShiftExtendType ET = getShiftExtendType();
     bool IsSigned = ET == AArch64_AM::SXTW || ET == AArch64_AM::SXTX;
-    Inst.addOperand(MCOperand::CreateImm(IsSigned));
-    Inst.addOperand(MCOperand::CreateImm(getShiftExtendAmount() != 0));
+    Inst.addOperand(MCOperand::createImm(IsSigned));
+    Inst.addOperand(MCOperand::createImm(getShiftExtendAmount() != 0));
   }
 
   // For 8-bit load/store instructions with a register offset, both the
@@ -1503,8 +1546,8 @@ class AArch64Operand : public MCParsedAsmOperand {
     assert(N == 2 && "Invalid number of operands!");
     AArch64_AM::ShiftExtendType ET = getShiftExtendType();
     bool IsSigned = ET == AArch64_AM::SXTW || ET == AArch64_AM::SXTX;
-    Inst.addOperand(MCOperand::CreateImm(IsSigned));
-    Inst.addOperand(MCOperand::CreateImm(hasShiftExtendAmount()));
+    Inst.addOperand(MCOperand::createImm(IsSigned));
+    Inst.addOperand(MCOperand::createImm(hasShiftExtendAmount()));
   }
 
   template<int Shift>
@@ -1513,7 +1556,7 @@ class AArch64Operand : public MCParsedAsmOperand {
 
     const MCConstantExpr *CE = cast<MCConstantExpr>(getImm());
     uint64_t Value = CE->getValue();
-    Inst.addOperand(MCOperand::CreateImm((Value >> Shift) & 0xffff));
+    Inst.addOperand(MCOperand::createImm((Value >> Shift) & 0xffff));
   }
 
   template<int Shift>
@@ -1522,7 +1565,7 @@ class AArch64Operand : public MCParsedAsmOperand {
 
     const MCConstantExpr *CE = cast<MCConstantExpr>(getImm());
     uint64_t Value = CE->getValue();
-    Inst.addOperand(MCOperand::CreateImm((~Value >> Shift) & 0xffff));
+    Inst.addOperand(MCOperand::createImm((~Value >> Shift) & 0xffff));
   }
 
   void print(raw_ostream &OS) const override;
@@ -1690,12 +1733,12 @@ void AArch64Operand::print(raw_ostream &OS) const {
     break;
   }
   case k_Immediate:
-    getImm()->print(OS);
+    OS << *getImm();
     break;
   case k_ShiftedImm: {
     unsigned Shift = getShiftedImmShift();
     OS << "<shiftedimm ";
-    getShiftedImmVal()->print(OS);
+    OS << *getShiftedImmVal();
     OS << ", lsl #" << AArch64_AM::getShiftValue(Shift) << ">";
     break;
   }
@@ -1752,7 +1795,7 @@ static unsigned MatchRegisterName(StringRef Name);
 /// }
 
 static unsigned matchVectorRegName(StringRef Name) {
-  return StringSwitch<unsigned>(Name)
+  return StringSwitch<unsigned>(Name.lower())
       .Case("v0", AArch64::Q0)
       .Case("v1", AArch64::Q1)
       .Case("v2", AArch64::Q2)
@@ -2023,7 +2066,7 @@ AArch64AsmParser::tryParseAdrpLabel(OperandVector &Operands) {
       // No modifier was specified at all; this is the syntax for an ELF basic
       // ADRP relocation (unfortunately).
       Expr =
-          AArch64MCExpr::Create(Expr, AArch64MCExpr::VK_ABS_PAGE, getContext());
+          AArch64MCExpr::create(Expr, AArch64MCExpr::VK_ABS_PAGE, getContext());
     } else if ((DarwinRefKind == MCSymbolRefExpr::VK_GOTPAGE ||
                 DarwinRefKind == MCSymbolRefExpr::VK_TLVPPAGE) &&
                Addend != 0) {
@@ -2156,7 +2199,7 @@ AArch64AsmParser::tryParseAddSubImm(OperandVector &Operands) {
     if (MCE) {
       int64_t Val = MCE->getValue();
       if (Val > 0xfff && (Val & 0xfff) == 0) {
-        Imm = MCConstantExpr::Create(Val >> 12, getContext());
+        Imm = MCConstantExpr::create(Val >> 12, getContext());
         ShiftAmount = 12;
       }
     }
@@ -2346,14 +2389,14 @@ bool AArch64AsmParser::parseSysAlias(StringRef Name, SMLoc NameLoc,
 
 #define SYS_ALIAS(op1, Cn, Cm, op2)                                            \
   do {                                                                         \
-    Expr = MCConstantExpr::Create(op1, getContext());                          \
+    Expr = MCConstantExpr::create(op1, getContext());                          \
     Operands.push_back(                                                        \
         AArch64Operand::CreateImm(Expr, S, getLoc(), getContext()));           \
     Operands.push_back(                                                        \
         AArch64Operand::CreateSysCR(Cn, S, getLoc(), getContext()));           \
     Operands.push_back(                                                        \
         AArch64Operand::CreateSysCR(Cm, S, getLoc(), getContext()));           \
-    Expr = MCConstantExpr::Create(op2, getContext());                          \
+    Expr = MCConstantExpr::create(op2, getContext());                          \
     Operands.push_back(                                                        \
         AArch64Operand::CreateImm(Expr, S, getLoc(), getContext()));           \
   } while (0)
@@ -2834,7 +2877,7 @@ bool AArch64AsmParser::parseSymbolicImmVal(const MCExpr *&ImmVal) {
     return true;
 
   if (HasELFModifier)
-    ImmVal = AArch64MCExpr::Create(ImmVal, RefKind, getContext());
+    ImmVal = AArch64MCExpr::create(ImmVal, RefKind, getContext());
 
   return false;
 }
@@ -3127,7 +3170,7 @@ bool AArch64AsmParser::parseOperand(OperandVector &Operands, bool isCondCode,
       if (ShiftAmt <= MaxShiftAmt && Imm <= 0xFFFF) {
           Operands[0] = AArch64Operand::CreateToken("movz", false, Loc, Ctx);
           Operands.push_back(AArch64Operand::CreateImm(
-                     MCConstantExpr::Create(Imm, Ctx), S, E, Ctx));
+                     MCConstantExpr::create(Imm, Ctx), S, E, Ctx));
         if (ShiftAmt)
           Operands.push_back(AArch64Operand::CreateShiftExtend(AArch64_AM::LSL,
                      ShiftAmt, true, S, E, Ctx));
@@ -3633,8 +3676,8 @@ bool AArch64AsmParser::MatchAndEmitInstruction(SMLoc IDLoc, unsigned &Opcode,
           NewOp4Val = 63 - Op3Val;
         }
 
-        const MCExpr *NewOp3 = MCConstantExpr::Create(NewOp3Val, getContext());
-        const MCExpr *NewOp4 = MCConstantExpr::Create(NewOp4Val, getContext());
+        const MCExpr *NewOp3 = MCConstantExpr::create(NewOp3Val, getContext());
+        const MCExpr *NewOp4 = MCConstantExpr::create(NewOp4Val, getContext());
 
         Operands[0] = AArch64Operand::CreateToken(
             "ubfm", false, Op.getStartLoc(), getContext());
@@ -3644,6 +3687,60 @@ bool AArch64AsmParser::MatchAndEmitInstruction(SMLoc IDLoc, unsigned &Opcode,
                                                 Op3.getEndLoc(), getContext());
       }
     }
+  } else if (NumOperands == 4 && Tok == "bfc") {
+    // FIXME: Horrible hack to handle BFC->BFM alias.
+    AArch64Operand &Op1 = static_cast<AArch64Operand &>(*Operands[1]);
+    AArch64Operand LSBOp = static_cast<AArch64Operand &>(*Operands[2]);
+    AArch64Operand WidthOp = static_cast<AArch64Operand &>(*Operands[3]);
+
+    if (Op1.isReg() && LSBOp.isImm() && WidthOp.isImm()) {
+      const MCConstantExpr *LSBCE = dyn_cast<MCConstantExpr>(LSBOp.getImm());
+      const MCConstantExpr *WidthCE = dyn_cast<MCConstantExpr>(WidthOp.getImm());
+
+      if (LSBCE && WidthCE) {
+        uint64_t LSB = LSBCE->getValue();
+        uint64_t Width = WidthCE->getValue();
+
+        uint64_t RegWidth = 0;
+        if (AArch64MCRegisterClasses[AArch64::GPR64allRegClassID].contains(
+                Op1.getReg()))
+          RegWidth = 64;
+        else
+          RegWidth = 32;
+
+        if (LSB >= RegWidth)
+          return Error(LSBOp.getStartLoc(),
+                       "expected integer in range [0, 31]");
+        if (Width < 1 || Width > RegWidth)
+          return Error(WidthOp.getStartLoc(),
+                       "expected integer in range [1, 32]");
+
+        uint64_t ImmR = 0;
+        if (RegWidth == 32)
+          ImmR = (32 - LSB) & 0x1f;
+        else
+          ImmR = (64 - LSB) & 0x3f;
+
+        uint64_t ImmS = Width - 1;
+
+        if (ImmR != 0 && ImmS >= ImmR)
+          return Error(WidthOp.getStartLoc(),
+                       "requested insert overflows register");
+
+        const MCExpr *ImmRExpr = MCConstantExpr::create(ImmR, getContext());
+        const MCExpr *ImmSExpr = MCConstantExpr::create(ImmS, getContext());
+        Operands[0] = AArch64Operand::CreateToken(
+              "bfm", false, Op.getStartLoc(), getContext());
+        Operands[2] = AArch64Operand::CreateReg(
+            RegWidth == 32 ? AArch64::WZR : AArch64::XZR, false, SMLoc(),
+            SMLoc(), getContext());
+        Operands[3] = AArch64Operand::CreateImm(
+            ImmRExpr, LSBOp.getStartLoc(), LSBOp.getEndLoc(), getContext());
+        Operands.emplace_back(
+            AArch64Operand::CreateImm(ImmSExpr, WidthOp.getStartLoc(),
+                                      WidthOp.getEndLoc(), getContext()));
+      }
+    }
   } else if (NumOperands == 5) {
     // FIXME: Horrible hack to handle the BFI -> BFM, SBFIZ->SBFM, and
     // UBFIZ -> UBFM aliases.
@@ -3675,8 +3772,7 @@ bool AArch64AsmParser::MatchAndEmitInstruction(SMLoc IDLoc, unsigned &Opcode,
                          "expected integer in range [1, 32]");
 
           uint64_t NewOp3Val = 0;
-          if (AArch64MCRegisterClasses[AArch64::GPR32allRegClassID].contains(
-                  Op1.getReg()))
+          if (RegWidth == 32)
             NewOp3Val = (32 - Op3Val) & 0x1f;
           else
             NewOp3Val = (64 - Op3Val) & 0x3f;
@@ -3688,9 +3784,9 @@ bool AArch64AsmParser::MatchAndEmitInstruction(SMLoc IDLoc, unsigned &Opcode,
                          "requested insert overflows register");
 
           const MCExpr *NewOp3 =
-              MCConstantExpr::Create(NewOp3Val, getContext());
+              MCConstantExpr::create(NewOp3Val, getContext());
           const MCExpr *NewOp4 =
-              MCConstantExpr::Create(NewOp4Val, getContext());
+              MCConstantExpr::create(NewOp4Val, getContext());
           Operands[3] = AArch64Operand::CreateImm(
               NewOp3, Op3.getStartLoc(), Op3.getEndLoc(), getContext());
           Operands[4] = AArch64Operand::CreateImm(
@@ -3746,7 +3842,7 @@ bool AArch64AsmParser::MatchAndEmitInstruction(SMLoc IDLoc, unsigned &Opcode,
                          "requested extract overflows register");
 
           const MCExpr *NewOp4 =
-              MCConstantExpr::Create(NewOp4Val, getContext());
+              MCConstantExpr::create(NewOp4Val, getContext());
           Operands[4] = AArch64Operand::CreateImm(
               NewOp4, Op4.getStartLoc(), Op4.getEndLoc(), getContext());
           if (Tok == "bfxil")
@@ -3967,7 +4063,7 @@ bool AArch64AsmParser::ParseDirective(AsmToken DirectiveID) {
   if (IDVal == ".ltorg" || IDVal == ".pool")
     return parseDirectiveLtorg(Loc);
   if (IDVal == ".unreq")
-    return parseDirectiveUnreq(DirectiveID.getLoc());
+    return parseDirectiveUnreq(Loc);
 
   if (!IsMachO && !IsCOFF) {
     if (IDVal == ".inst")
@@ -4051,13 +4147,13 @@ bool AArch64AsmParser::parseDirectiveTLSDescCall(SMLoc L) {
   if (getParser().parseIdentifier(Name))
     return Error(L, "expected symbol after directive");
 
-  MCSymbol *Sym = getContext().GetOrCreateSymbol(Name);
-  const MCExpr *Expr = MCSymbolRefExpr::Create(Sym, getContext());
-  Expr = AArch64MCExpr::Create(Expr, AArch64MCExpr::VK_TLSDESC, getContext());
+  MCSymbol *Sym = getContext().getOrCreateSymbol(Name);
+  const MCExpr *Expr = MCSymbolRefExpr::create(Sym, getContext());
+  Expr = AArch64MCExpr::create(Expr, AArch64MCExpr::VK_TLSDESC, getContext());
 
   MCInst Inst;
   Inst.setOpcode(AArch64::TLSDESCCALL);
-  Inst.addOperand(MCOperand::CreateExpr(Expr));
+  Inst.addOperand(MCOperand::createExpr(Expr));
 
   getParser().getStreamer().EmitInstruction(Inst, STI);
   return false;
@@ -4100,7 +4196,7 @@ bool AArch64AsmParser::parseDirectiveLOH(StringRef IDVal, SMLoc Loc) {
     StringRef Name;
     if (getParser().parseIdentifier(Name))
       return TokError("expected identifier in directive");
-    Args.push_back(getContext().GetOrCreateSymbol(Name));
+    Args.push_back(getContext().getOrCreateSymbol(Name));
 
     if (Idx + 1 == NbArgs)
       break;
@@ -4300,3 +4396,77 @@ unsigned AArch64AsmParser::validateTargetOperandClass(MCParsedAsmOperand &AsmOp,
     return Match_Success;
   return Match_InvalidOperand;
 }
+
+
+AArch64AsmParser::OperandMatchResultTy
+AArch64AsmParser::tryParseGPRSeqPair(OperandVector &Operands) {
+
+  SMLoc S = getLoc();
+
+  if (getParser().getTok().isNot(AsmToken::Identifier)) {
+    Error(S, "expected register");
+    return MatchOperand_ParseFail;
+  }
+
+  int FirstReg = tryParseRegister();
+  if (FirstReg == -1) {
+    return MatchOperand_ParseFail;
+  }
+  const MCRegisterClass &WRegClass =
+      AArch64MCRegisterClasses[AArch64::GPR32RegClassID];
+  const MCRegisterClass &XRegClass =
+      AArch64MCRegisterClasses[AArch64::GPR64RegClassID];
+
+  bool isXReg = XRegClass.contains(FirstReg),
+       isWReg = WRegClass.contains(FirstReg);
+  if (!isXReg && !isWReg) {
+    Error(S, "expected first even register of a "
+             "consecutive same-size even/odd register pair");
+    return MatchOperand_ParseFail;
+  }
+
+  const MCRegisterInfo *RI = getContext().getRegisterInfo();
+  unsigned FirstEncoding = RI->getEncodingValue(FirstReg);
+
+  if (FirstEncoding & 0x1) {
+    Error(S, "expected first even register of a "
+             "consecutive same-size even/odd register pair");
+    return MatchOperand_ParseFail;
+  }
+
+  SMLoc M = getLoc();
+  if (getParser().getTok().isNot(AsmToken::Comma)) {
+    Error(M, "expected comma");
+    return MatchOperand_ParseFail;
+  }
+  // Eat the comma
+  getParser().Lex();
+
+  SMLoc E = getLoc();
+  int SecondReg = tryParseRegister();
+  if (SecondReg ==-1) {
+    return MatchOperand_ParseFail;
+  }
+
+ if (RI->getEncodingValue(SecondReg) != FirstEncoding + 1 ||
+      (isXReg && !XRegClass.contains(SecondReg)) ||
+      (isWReg && !WRegClass.contains(SecondReg))) {
+    Error(E,"expected second odd register of a "
+             "consecutive same-size even/odd register pair");
+    return MatchOperand_ParseFail;
+  }
+  
+  unsigned Pair = 0;
+  if(isXReg) {
+    Pair = RI->getMatchingSuperReg(FirstReg, AArch64::sube64,
+           &AArch64MCRegisterClasses[AArch64::XSeqPairsClassRegClassID]);
+  } else {
+    Pair = RI->getMatchingSuperReg(FirstReg, AArch64::sube32,
+           &AArch64MCRegisterClasses[AArch64::WSeqPairsClassRegClassID]);
+  }
+
+  Operands.push_back(AArch64Operand::CreateReg(Pair, false, S, getLoc(),
+      getContext()));
+
+  return MatchOperand_Success;
+}
diff --git a/lib/Target/AArch64/Disassembler/AArch64Disassembler.cpp b/lib/Target/AArch64/Disassembler/AArch64Disassembler.cpp
index 1c8c0a665cc..db9fb0e775d 100644
--- a/lib/Target/AArch64/Disassembler/AArch64Disassembler.cpp
+++ b/lib/Target/AArch64/Disassembler/AArch64Disassembler.cpp
@@ -169,6 +169,14 @@ static DecodeStatus DecodeVecShiftL16Imm(llvm::MCInst &Inst, unsigned Imm,
                                          uint64_t Addr, const void *Decoder);
 static DecodeStatus DecodeVecShiftL8Imm(llvm::MCInst &Inst, unsigned Imm,
                                         uint64_t Addr, const void *Decoder);
+static DecodeStatus DecodeWSeqPairsClassRegisterClass(MCInst &Inst,
+                                                      unsigned RegNo,
+                                                      uint64_t Addr,
+                                                      const void *Decoder);
+static DecodeStatus DecodeXSeqPairsClassRegisterClass(MCInst &Inst,
+                                                      unsigned RegNo,
+                                                      uint64_t Addr,
+                                                      const void *Decoder);
 
 static bool Check(DecodeStatus &Out, DecodeStatus In) {
   switch (In) {
@@ -220,7 +228,7 @@ DecodeStatus AArch64Disassembler::getInstruction(MCInst &MI, uint64_t &Size,
 }
 
 static MCSymbolizer *
-createAArch64ExternalSymbolizer(StringRef TT, LLVMOpInfoCallback GetOpInfo,
+createAArch64ExternalSymbolizer(const Triple &TT, LLVMOpInfoCallback GetOpInfo,
                                 LLVMSymbolLookupCallback SymbolLookUp,
                                 void *DisInfo, MCContext *Ctx,
                                 std::unique_ptr<MCRelocationInfo> &&RelInfo) {
@@ -261,7 +269,7 @@ static DecodeStatus DecodeFPR128RegisterClass(MCInst &Inst, unsigned RegNo,
     return Fail;
 
   unsigned Register = FPR128DecoderTable[RegNo];
-  Inst.addOperand(MCOperand::CreateReg(Register));
+  Inst.addOperand(MCOperand::createReg(Register));
   return Success;
 }
 
@@ -290,7 +298,7 @@ static DecodeStatus DecodeFPR64RegisterClass(MCInst &Inst, unsigned RegNo,
     return Fail;
 
   unsigned Register = FPR64DecoderTable[RegNo];
-  Inst.addOperand(MCOperand::CreateReg(Register));
+  Inst.addOperand(MCOperand::createReg(Register));
   return Success;
 }
 
@@ -311,7 +319,7 @@ static DecodeStatus DecodeFPR32RegisterClass(MCInst &Inst, unsigned RegNo,
     return Fail;
 
   unsigned Register = FPR32DecoderTable[RegNo];
-  Inst.addOperand(MCOperand::CreateReg(Register));
+  Inst.addOperand(MCOperand::createReg(Register));
   return Success;
 }
 
@@ -332,7 +340,7 @@ static DecodeStatus DecodeFPR16RegisterClass(MCInst &Inst, unsigned RegNo,
     return Fail;
 
   unsigned Register = FPR16DecoderTable[RegNo];
-  Inst.addOperand(MCOperand::CreateReg(Register));
+  Inst.addOperand(MCOperand::createReg(Register));
   return Success;
 }
 
@@ -353,7 +361,7 @@ static DecodeStatus DecodeFPR8RegisterClass(MCInst &Inst, unsigned RegNo,
     return Fail;
 
   unsigned Register = FPR8DecoderTable[RegNo];
-  Inst.addOperand(MCOperand::CreateReg(Register));
+  Inst.addOperand(MCOperand::createReg(Register));
   return Success;
 }
 
@@ -374,7 +382,7 @@ static DecodeStatus DecodeGPR64RegisterClass(MCInst &Inst, unsigned RegNo,
     return Fail;
 
   unsigned Register = GPR64DecoderTable[RegNo];
-  Inst.addOperand(MCOperand::CreateReg(Register));
+  Inst.addOperand(MCOperand::createReg(Register));
   return Success;
 }
 
@@ -386,7 +394,7 @@ static DecodeStatus DecodeGPR64spRegisterClass(MCInst &Inst, unsigned RegNo,
   unsigned Register = GPR64DecoderTable[RegNo];
   if (Register == AArch64::XZR)
     Register = AArch64::SP;
-  Inst.addOperand(MCOperand::CreateReg(Register));
+  Inst.addOperand(MCOperand::createReg(Register));
   return Success;
 }
 
@@ -407,7 +415,7 @@ static DecodeStatus DecodeGPR32RegisterClass(MCInst &Inst, unsigned RegNo,
     return Fail;
 
   unsigned Register = GPR32DecoderTable[RegNo];
-  Inst.addOperand(MCOperand::CreateReg(Register));
+  Inst.addOperand(MCOperand::createReg(Register));
   return Success;
 }
 
@@ -420,7 +428,7 @@ static DecodeStatus DecodeGPR32spRegisterClass(MCInst &Inst, unsigned RegNo,
   unsigned Register = GPR32DecoderTable[RegNo];
   if (Register == AArch64::WZR)
     Register = AArch64::WSP;
-  Inst.addOperand(MCOperand::CreateReg(Register));
+  Inst.addOperand(MCOperand::createReg(Register));
   return Success;
 }
 
@@ -441,7 +449,7 @@ static DecodeStatus DecodeVectorRegisterClass(MCInst &Inst, unsigned RegNo,
     return Fail;
 
   unsigned Register = VectorDecoderTable[RegNo];
-  Inst.addOperand(MCOperand::CreateReg(Register));
+  Inst.addOperand(MCOperand::createReg(Register));
   return Success;
 }
 
@@ -461,7 +469,7 @@ static DecodeStatus DecodeQQRegisterClass(MCInst &Inst, unsigned RegNo,
   if (RegNo > 31)
     return Fail;
   unsigned Register = QQDecoderTable[RegNo];
-  Inst.addOperand(MCOperand::CreateReg(Register));
+  Inst.addOperand(MCOperand::createReg(Register));
   return Success;
 }
 
@@ -484,7 +492,7 @@ static DecodeStatus DecodeQQQRegisterClass(MCInst &Inst, unsigned RegNo,
   if (RegNo > 31)
     return Fail;
   unsigned Register = QQQDecoderTable[RegNo];
-  Inst.addOperand(MCOperand::CreateReg(Register));
+  Inst.addOperand(MCOperand::createReg(Register));
   return Success;
 }
 
@@ -508,7 +516,7 @@ static DecodeStatus DecodeQQQQRegisterClass(MCInst &Inst, unsigned RegNo,
   if (RegNo > 31)
     return Fail;
   unsigned Register = QQQQDecoderTable[RegNo];
-  Inst.addOperand(MCOperand::CreateReg(Register));
+  Inst.addOperand(MCOperand::createReg(Register));
   return Success;
 }
 
@@ -528,7 +536,7 @@ static DecodeStatus DecodeDDRegisterClass(MCInst &Inst, unsigned RegNo,
   if (RegNo > 31)
     return Fail;
   unsigned Register = DDDecoderTable[RegNo];
-  Inst.addOperand(MCOperand::CreateReg(Register));
+  Inst.addOperand(MCOperand::createReg(Register));
   return Success;
 }
 
@@ -551,7 +559,7 @@ static DecodeStatus DecodeDDDRegisterClass(MCInst &Inst, unsigned RegNo,
   if (RegNo > 31)
     return Fail;
   unsigned Register = DDDDecoderTable[RegNo];
-  Inst.addOperand(MCOperand::CreateReg(Register));
+  Inst.addOperand(MCOperand::createReg(Register));
   return Success;
 }
 
@@ -575,7 +583,7 @@ static DecodeStatus DecodeDDDDRegisterClass(MCInst &Inst, unsigned RegNo,
   if (RegNo > 31)
     return Fail;
   unsigned Register = DDDDDecoderTable[RegNo];
-  Inst.addOperand(MCOperand::CreateReg(Register));
+  Inst.addOperand(MCOperand::createReg(Register));
   return Success;
 }
 
@@ -584,14 +592,14 @@ static DecodeStatus DecodeFixedPointScaleImm32(llvm::MCInst &Inst, unsigned Imm,
                                                const void *Decoder) {
   // scale{5} is asserted as 1 in tblgen.
   Imm |= 0x20;
-  Inst.addOperand(MCOperand::CreateImm(64 - Imm));
+  Inst.addOperand(MCOperand::createImm(64 - Imm));
   return Success;
 }
 
 static DecodeStatus DecodeFixedPointScaleImm64(llvm::MCInst &Inst, unsigned Imm,
                                                uint64_t Addr,
                                                const void *Decoder) {
-  Inst.addOperand(MCOperand::CreateImm(64 - Imm));
+  Inst.addOperand(MCOperand::createImm(64 - Imm));
   return Success;
 }
 
@@ -607,21 +615,21 @@ static DecodeStatus DecodePCRelLabel19(llvm::MCInst &Inst, unsigned Imm,
 
   if (!Dis->tryAddingSymbolicOperand(Inst, ImmVal *  4, Addr,
                                      Inst.getOpcode() != AArch64::LDRXl, 0, 4))
-    Inst.addOperand(MCOperand::CreateImm(ImmVal));
+    Inst.addOperand(MCOperand::createImm(ImmVal));
   return Success;
 }
 
 static DecodeStatus DecodeMemExtend(llvm::MCInst &Inst, unsigned Imm,
                                     uint64_t Address, const void *Decoder) {
-  Inst.addOperand(MCOperand::CreateImm((Imm  >> 1) & 1));
-  Inst.addOperand(MCOperand::CreateImm(Imm & 1));
+  Inst.addOperand(MCOperand::createImm((Imm  >> 1) & 1));
+  Inst.addOperand(MCOperand::createImm(Imm & 1));
   return Success;
 }
 
 static DecodeStatus DecodeMRSSystemRegister(llvm::MCInst &Inst, unsigned Imm,
                                             uint64_t Address,
                                             const void *Decoder) {
-  Inst.addOperand(MCOperand::CreateImm(Imm));
+  Inst.addOperand(MCOperand::createImm(Imm));
 
   // Every system register in the encoding space is valid with the syntax
   // S<op0>_<op1>_<Cn>_<Cm>_<op2>, so decoding system registers always succeeds.
@@ -631,7 +639,7 @@ static DecodeStatus DecodeMRSSystemRegister(llvm::MCInst &Inst, unsigned Imm,
 static DecodeStatus DecodeMSRSystemRegister(llvm::MCInst &Inst, unsigned Imm,
                                             uint64_t Address,
                                             const void *Decoder) {
-  Inst.addOperand(MCOperand::CreateImm(Imm));
+  Inst.addOperand(MCOperand::createImm(Imm));
 
   return Success;
 }
@@ -654,20 +662,20 @@ static DecodeStatus DecodeFMOVLaneInstruction(llvm::MCInst &Inst, unsigned Insn,
   }
 
   // Add the lane
-  Inst.addOperand(MCOperand::CreateImm(1));
+  Inst.addOperand(MCOperand::createImm(1));
 
   return Success;
 }
 
 static DecodeStatus DecodeVecShiftRImm(llvm::MCInst &Inst, unsigned Imm,
                                        unsigned Add) {
-  Inst.addOperand(MCOperand::CreateImm(Add - Imm));
+  Inst.addOperand(MCOperand::createImm(Add - Imm));
   return Success;
 }
 
 static DecodeStatus DecodeVecShiftLImm(llvm::MCInst &Inst, unsigned Imm,
                                        unsigned Add) {
-  Inst.addOperand(MCOperand::CreateImm((Imm + Add) & (Add - 1)));
+  Inst.addOperand(MCOperand::createImm((Imm + Add) & (Add - 1)));
   return Success;
 }
 
@@ -787,7 +795,7 @@ static DecodeStatus DecodeThreeAddrSRegInstruction(llvm::MCInst &Inst,
     break;
   }
 
-  Inst.addOperand(MCOperand::CreateImm(shift));
+  Inst.addOperand(MCOperand::createImm(shift));
   return Success;
 }
 
@@ -819,8 +827,8 @@ static DecodeStatus DecodeMoveImmInstruction(llvm::MCInst &Inst, uint32_t insn,
       Inst.getOpcode() == AArch64::MOVKXi)
     Inst.addOperand(Inst.getOperand(0));
 
-  Inst.addOperand(MCOperand::CreateImm(imm));
-  Inst.addOperand(MCOperand::CreateImm(shift));
+  Inst.addOperand(MCOperand::createImm(imm));
+  Inst.addOperand(MCOperand::createImm(shift));
   return Success;
 }
 
@@ -838,7 +846,7 @@ static DecodeStatus DecodeUnsignedLdStInstruction(llvm::MCInst &Inst,
     return Fail;
   case AArch64::PRFMui:
     // Rt is an immediate in prefetch.
-    Inst.addOperand(MCOperand::CreateImm(Rt));
+    Inst.addOperand(MCOperand::createImm(Rt));
     break;
   case AArch64::STRBBui:
   case AArch64::LDRBBui:
@@ -881,7 +889,7 @@ static DecodeStatus DecodeUnsignedLdStInstruction(llvm::MCInst &Inst,
 
   DecodeGPR64spRegisterClass(Inst, Rn, Addr, Decoder);
   if (!Dis->tryAddingSymbolicOperand(Inst, offset, Addr, Fail, 0, 4))
-    Inst.addOperand(MCOperand::CreateImm(offset));
+    Inst.addOperand(MCOperand::createImm(offset));
   return Success;
 }
 
@@ -956,7 +964,7 @@ static DecodeStatus DecodeSignedLdStInstruction(llvm::MCInst &Inst,
     return Fail;
   case AArch64::PRFUMi:
     // Rt is an immediate in prefetch.
-    Inst.addOperand(MCOperand::CreateImm(Rt));
+    Inst.addOperand(MCOperand::createImm(Rt));
     break;
   case AArch64::STURBBi:
   case AArch64::LDURBBi:
@@ -1057,7 +1065,7 @@ static DecodeStatus DecodeSignedLdStInstruction(llvm::MCInst &Inst,
   }
 
   DecodeGPR64spRegisterClass(Inst, Rn, Addr, Decoder);
-  Inst.addOperand(MCOperand::CreateImm(offset));
+  Inst.addOperand(MCOperand::createImm(offset));
 
   bool IsLoad = fieldFromInstruction(insn, 22, 1);
   bool IsIndexed = fieldFromInstruction(insn, 10, 2) != 0;
@@ -1268,7 +1276,7 @@ static DecodeStatus DecodePairLdStInstruction(llvm::MCInst &Inst, uint32_t insn,
   }
 
   DecodeGPR64spRegisterClass(Inst, Rn, Addr, Decoder);
-  Inst.addOperand(MCOperand::CreateImm(offset));
+  Inst.addOperand(MCOperand::createImm(offset));
 
   // You shouldn't load to the same register twice in an instruction...
   if (IsLoad && Rt == Rt2)
@@ -1335,7 +1343,7 @@ static DecodeStatus DecodeAddSubERegInstruction(llvm::MCInst &Inst,
     break;
   }
 
-  Inst.addOperand(MCOperand::CreateImm(extend));
+  Inst.addOperand(MCOperand::createImm(extend));
   return Success;
 }
 
@@ -1366,7 +1374,7 @@ static DecodeStatus DecodeLogicalImmInstruction(llvm::MCInst &Inst,
     if (!AArch64_AM::isValidDecodeLogicalImmediate(imm, 32))
       return Fail;
   }
-  Inst.addOperand(MCOperand::CreateImm(imm));
+  Inst.addOperand(MCOperand::createImm(imm));
   return Success;
 }
 
@@ -1383,7 +1391,7 @@ static DecodeStatus DecodeModImmInstruction(llvm::MCInst &Inst, uint32_t insn,
   else
     DecodeVectorRegisterClass(Inst, Rd, Addr, Decoder);
 
-  Inst.addOperand(MCOperand::CreateImm(imm));
+  Inst.addOperand(MCOperand::createImm(imm));
 
   switch (Inst.getOpcode()) {
   default:
@@ -1396,13 +1404,13 @@ static DecodeStatus DecodeModImmInstruction(llvm::MCInst &Inst, uint32_t insn,
   case AArch64::MOVIv4i32:
   case AArch64::MVNIv2i32:
   case AArch64::MVNIv4i32:
-    Inst.addOperand(MCOperand::CreateImm((cmode & 6) << 2));
+    Inst.addOperand(MCOperand::createImm((cmode & 6) << 2));
     break;
   case AArch64::MOVIv2s_msl:
   case AArch64::MOVIv4s_msl:
   case AArch64::MVNIv2s_msl:
   case AArch64::MVNIv4s_msl:
-    Inst.addOperand(MCOperand::CreateImm(cmode & 1 ? 0x110 : 0x108));
+    Inst.addOperand(MCOperand::createImm(cmode & 1 ? 0x110 : 0x108));
     break;
   }
 
@@ -1421,8 +1429,8 @@ static DecodeStatus DecodeModImmTiedInstruction(llvm::MCInst &Inst,
   DecodeVectorRegisterClass(Inst, Rd, Addr, Decoder);
   DecodeVectorRegisterClass(Inst, Rd, Addr, Decoder);
 
-  Inst.addOperand(MCOperand::CreateImm(imm));
-  Inst.addOperand(MCOperand::CreateImm((cmode & 6) << 2));
+  Inst.addOperand(MCOperand::createImm(imm));
+  Inst.addOperand(MCOperand::createImm((cmode & 6) << 2));
 
   return Success;
 }
@@ -1441,7 +1449,7 @@ static DecodeStatus DecodeAdrInstruction(llvm::MCInst &Inst, uint32_t insn,
 
   DecodeGPR64RegisterClass(Inst, Rd, Addr, Decoder);
   if (!Dis->tryAddingSymbolicOperand(Inst, imm, Addr, Fail, 0, 4))
-    Inst.addOperand(MCOperand::CreateImm(imm));
+    Inst.addOperand(MCOperand::createImm(imm));
 
   return Success;
 }
@@ -1477,8 +1485,8 @@ static DecodeStatus DecodeBaseAddSubImm(llvm::MCInst &Inst, uint32_t insn,
   }
 
   if (!Dis->tryAddingSymbolicOperand(Inst, Imm, Addr, Fail, 0, 4))
-    Inst.addOperand(MCOperand::CreateImm(ImmVal));
-  Inst.addOperand(MCOperand::CreateImm(12 * ShifterVal));
+    Inst.addOperand(MCOperand::createImm(ImmVal));
+  Inst.addOperand(MCOperand::createImm(12 * ShifterVal));
   return Success;
 }
 
@@ -1494,7 +1502,7 @@ static DecodeStatus DecodeUnconditionalBranch(llvm::MCInst &Inst, uint32_t insn,
     imm |= ~((1LL << 26) - 1);
 
   if (!Dis->tryAddingSymbolicOperand(Inst, imm * 4, Addr, true, 0, 4))
-    Inst.addOperand(MCOperand::CreateImm(imm));
+    Inst.addOperand(MCOperand::createImm(imm));
 
   return Success;
 }
@@ -1508,8 +1516,8 @@ static DecodeStatus DecodeSystemPStateInstruction(llvm::MCInst &Inst,
 
   uint64_t pstate_field = (op1 << 3) | op2;
 
-  Inst.addOperand(MCOperand::CreateImm(pstate_field));
-  Inst.addOperand(MCOperand::CreateImm(crm));
+  Inst.addOperand(MCOperand::createImm(pstate_field));
+  Inst.addOperand(MCOperand::createImm(crm));
 
   bool ValidNamed;
   const AArch64Disassembler *Dis = 
@@ -1537,9 +1545,41 @@ static DecodeStatus DecodeTestAndBranch(llvm::MCInst &Inst, uint32_t insn,
     DecodeGPR32RegisterClass(Inst, Rt, Addr, Decoder);
   else
     DecodeGPR64RegisterClass(Inst, Rt, Addr, Decoder);
-  Inst.addOperand(MCOperand::CreateImm(bit));
+  Inst.addOperand(MCOperand::createImm(bit));
   if (!Dis->tryAddingSymbolicOperand(Inst, dst * 4, Addr, true, 0, 4))
-    Inst.addOperand(MCOperand::CreateImm(dst));
+    Inst.addOperand(MCOperand::createImm(dst));
 
   return Success;
 }
+
+static DecodeStatus DecodeGPRSeqPairsClassRegisterClass(MCInst &Inst,
+                                                        unsigned RegClassID,
+                                                        unsigned RegNo,
+                                                        uint64_t Addr,
+                                                        const void *Decoder) {
+  // Register number must be even (see CASP instruction)
+  if (RegNo & 0x1)
+    return Fail;
+
+  unsigned Register = AArch64MCRegisterClasses[RegClassID].getRegister(RegNo);
+  Inst.addOperand(MCOperand::createReg(Register));
+  return Success;
+}
+
+static DecodeStatus DecodeWSeqPairsClassRegisterClass(MCInst &Inst,
+                                                      unsigned RegNo,
+                                                      uint64_t Addr,
+                                                      const void *Decoder) {
+  return DecodeGPRSeqPairsClassRegisterClass(Inst, 
+                                             AArch64::WSeqPairsClassRegClassID,
+                                             RegNo, Addr, Decoder);
+}
+
+static DecodeStatus DecodeXSeqPairsClassRegisterClass(MCInst &Inst,
+                                                      unsigned RegNo,
+                                                      uint64_t Addr,
+                                                      const void *Decoder) {
+  return DecodeGPRSeqPairsClassRegisterClass(Inst, 
+                                             AArch64::XSeqPairsClassRegClassID,
+                                             RegNo, Addr, Decoder);
+}
diff --git a/lib/Target/AArch64/Disassembler/AArch64ExternalSymbolizer.cpp b/lib/Target/AArch64/Disassembler/AArch64ExternalSymbolizer.cpp
index 2057c51346a..82bc949927c 100644
--- a/lib/Target/AArch64/Disassembler/AArch64ExternalSymbolizer.cpp
+++ b/lib/Target/AArch64/Disassembler/AArch64ExternalSymbolizer.cpp
@@ -52,7 +52,7 @@ getVariant(uint64_t LLVMDisassembler_VariantKind) {
 /// returns zero and isBranch is Success then a symbol look up for
 /// Address + Value is done and if a symbol is found an MCExpr is created with
 /// that, else an MCExpr with Address + Value is created.  If GetOpInfo()
-/// returns zero and isBranch is Fail then the the Opcode of the MCInst is
+/// returns zero and isBranch is Fail then the Opcode of the MCInst is
 /// tested and for ADRP an other instructions that help to load of pointers
 /// a symbol look up is done to see it is returns a specific reference type
 /// to add to the comment stream.  This function returns Success if it adds
@@ -165,14 +165,14 @@ bool AArch64ExternalSymbolizer::tryAddingSymbolicOperand(
   if (SymbolicOp.AddSymbol.Present) {
     if (SymbolicOp.AddSymbol.Name) {
       StringRef Name(SymbolicOp.AddSymbol.Name);
-      MCSymbol *Sym = Ctx.GetOrCreateSymbol(Name);
+      MCSymbol *Sym = Ctx.getOrCreateSymbol(Name);
       MCSymbolRefExpr::VariantKind Variant = getVariant(SymbolicOp.VariantKind);
       if (Variant != MCSymbolRefExpr::VK_None)
-        Add = MCSymbolRefExpr::Create(Sym, Variant, Ctx);
+        Add = MCSymbolRefExpr::create(Sym, Variant, Ctx);
       else
-        Add = MCSymbolRefExpr::Create(Sym, Ctx);
+        Add = MCSymbolRefExpr::create(Sym, Ctx);
     } else {
-      Add = MCConstantExpr::Create(SymbolicOp.AddSymbol.Value, Ctx);
+      Add = MCConstantExpr::create(SymbolicOp.AddSymbol.Value, Ctx);
     }
   }
 
@@ -180,41 +180,41 @@ bool AArch64ExternalSymbolizer::tryAddingSymbolicOperand(
   if (SymbolicOp.SubtractSymbol.Present) {
     if (SymbolicOp.SubtractSymbol.Name) {
       StringRef Name(SymbolicOp.SubtractSymbol.Name);
-      MCSymbol *Sym = Ctx.GetOrCreateSymbol(Name);
-      Sub = MCSymbolRefExpr::Create(Sym, Ctx);
+      MCSymbol *Sym = Ctx.getOrCreateSymbol(Name);
+      Sub = MCSymbolRefExpr::create(Sym, Ctx);
     } else {
-      Sub = MCConstantExpr::Create(SymbolicOp.SubtractSymbol.Value, Ctx);
+      Sub = MCConstantExpr::create(SymbolicOp.SubtractSymbol.Value, Ctx);
     }
   }
 
   const MCExpr *Off = nullptr;
   if (SymbolicOp.Value != 0)
-    Off = MCConstantExpr::Create(SymbolicOp.Value, Ctx);
+    Off = MCConstantExpr::create(SymbolicOp.Value, Ctx);
 
   const MCExpr *Expr;
   if (Sub) {
     const MCExpr *LHS;
     if (Add)
-      LHS = MCBinaryExpr::CreateSub(Add, Sub, Ctx);
+      LHS = MCBinaryExpr::createSub(Add, Sub, Ctx);
     else
-      LHS = MCUnaryExpr::CreateMinus(Sub, Ctx);
+      LHS = MCUnaryExpr::createMinus(Sub, Ctx);
     if (Off)
-      Expr = MCBinaryExpr::CreateAdd(LHS, Off, Ctx);
+      Expr = MCBinaryExpr::createAdd(LHS, Off, Ctx);
     else
       Expr = LHS;
   } else if (Add) {
     if (Off)
-      Expr = MCBinaryExpr::CreateAdd(Add, Off, Ctx);
+      Expr = MCBinaryExpr::createAdd(Add, Off, Ctx);
     else
       Expr = Add;
   } else {
     if (Off)
       Expr = Off;
     else
-      Expr = MCConstantExpr::Create(0, Ctx);
+      Expr = MCConstantExpr::create(0, Ctx);
   }
 
-  MI.addOperand(MCOperand::CreateExpr(Expr));
+  MI.addOperand(MCOperand::createExpr(Expr));
 
   return true;
 }
diff --git a/lib/Target/AArch64/Disassembler/LLVMBuild.txt b/lib/Target/AArch64/Disassembler/LLVMBuild.txt
index 62827e8f50e..73665eb5701 100644
--- a/lib/Target/AArch64/Disassembler/LLVMBuild.txt
+++ b/lib/Target/AArch64/Disassembler/LLVMBuild.txt
@@ -19,5 +19,5 @@
 type = Library
 name = AArch64Disassembler
 parent = AArch64
-required_libraries = AArch64Info AArch64Utils MC MCDisassembler Support
+required_libraries = AArch64Desc AArch64Info AArch64Utils MC MCDisassembler Support
 add_to_library_groups = AArch64
diff --git a/lib/Target/AArch64/InstPrinter/AArch64InstPrinter.cpp b/lib/Target/AArch64/InstPrinter/AArch64InstPrinter.cpp
index febd332414d..7f56c2cf6bb 100644
--- a/lib/Target/AArch64/InstPrinter/AArch64InstPrinter.cpp
+++ b/lib/Target/AArch64/InstPrinter/AArch64InstPrinter.cpp
@@ -162,11 +162,23 @@ void AArch64InstPrinter::printInst(const MCInst *MI, raw_ostream &O,
     int ImmR = MI->getOperand(3).getImm();
     int ImmS = MI->getOperand(4).getImm();
 
-    // BFI alias
-    if (ImmS < ImmR) {
+    if ((Op2.getReg() == AArch64::WZR || Op2.getReg() == AArch64::XZR) &&
+        (ImmR == 0 || ImmS < ImmR)) {
+      // BFC takes precedence over its entire range, sligtly differently to BFI.
       int BitWidth = Opcode == AArch64::BFMXri ? 64 : 32;
       int LSB = (BitWidth - ImmR) % BitWidth;
       int Width = ImmS + 1;
+
+      O << "\tbfc\t" << getRegisterName(Op0.getReg())
+        << ", #" << LSB << ", #" << Width;
+      printAnnotation(O, Annot);
+      return;
+    } else if (ImmS < ImmR) {
+      // BFI alias
+      int BitWidth = Opcode == AArch64::BFMXri ? 64 : 32;
+      int LSB = (BitWidth - ImmR) % BitWidth;
+      int Width = ImmS + 1;
+
       O << "\tbfi\t" << getRegisterName(Op0.getReg()) << ", "
         << getRegisterName(Op2.getReg()) << ", #" << LSB << ", #" << Width;
       printAnnotation(O, Annot);
@@ -194,15 +206,15 @@ void AArch64InstPrinter::printInst(const MCInst *MI, raw_ostream &O,
     else
       O << "\tmovn\t";
 
-    O << getRegisterName(MI->getOperand(0).getReg()) << ", #"
-      << *MI->getOperand(1).getExpr();
+    O << getRegisterName(MI->getOperand(0).getReg()) << ", #";
+    MI->getOperand(1).getExpr()->print(O, &MAI);
     return;
   }
 
   if ((Opcode == AArch64::MOVKXi || Opcode == AArch64::MOVKWi) &&
       MI->getOperand(2).isExpr()) {
-    O << "\tmovk\t" << getRegisterName(MI->getOperand(0).getReg()) << ", #"
-      << *MI->getOperand(2).getExpr();
+    O << "\tmovk\t" << getRegisterName(MI->getOperand(0).getReg()) << ", #";
+    MI->getOperand(2).getExpr()->print(O, &MAI);
     return;
   }
 
@@ -896,7 +908,7 @@ void AArch64InstPrinter::printOperand(const MCInst *MI, unsigned OpNo,
     O << '#' << Op.getImm();
   } else {
     assert(Op.isExpr() && "unknown operand kind in printOperand");
-    O << *Op.getExpr();
+    Op.getExpr()->print(O, &MAI);
   }
 }
 
@@ -954,7 +966,7 @@ void AArch64InstPrinter::printAddSubImm(const MCInst *MI, unsigned OpNum,
       *CommentStream << '=' << (Val << Shift) << '\n';
   } else {
     assert(MO.isExpr() && "Unexpected operand type!");
-    O << *MO.getExpr();
+    MO.getExpr()->print(O, &MAI);
     printShifter(MI, OpNum + 1, STI, O);
   }
 }
@@ -1079,7 +1091,7 @@ void AArch64InstPrinter::printUImm12Offset(const MCInst *MI, unsigned OpNum,
     O << "#" << (MO.getImm() * Scale);
   } else {
     assert(MO.isExpr() && "Unexpected operand type!");
-    O << *MO.getExpr();
+    MO.getExpr()->print(O, &MAI);
   }
 }
 
@@ -1091,7 +1103,8 @@ void AArch64InstPrinter::printAMIndexedWB(const MCInst *MI, unsigned OpNum,
       O << ", #" << (MO1.getImm() * Scale);
   } else {
     assert(MO1.isExpr() && "Unexpected operand type!");
-    O << ", " << *MO1.getExpr();
+    O << ", ";
+    MO1.getExpr()->print(O, &MAI);
   }
   O << ']';
 }
@@ -1101,7 +1114,7 @@ void AArch64InstPrinter::printPrefetchOp(const MCInst *MI, unsigned OpNum,
                                          raw_ostream &O) {
   unsigned prfop = MI->getOperand(OpNum).getImm();
   bool Valid;
-  StringRef Name = 
+  StringRef Name =
       AArch64PRFM::PRFMMapper().toString(prfop, STI.getFeatureBits(), Valid);
   if (Valid)
     O << Name;
@@ -1165,6 +1178,23 @@ static unsigned getNextVectorRegister(unsigned Reg, unsigned Stride = 1) {
   return Reg;
 }
 
+template<unsigned size>
+void AArch64InstPrinter::printGPRSeqPairsClassOperand(const MCInst *MI,
+                                                   unsigned OpNum,
+                                                   const MCSubtargetInfo &STI,
+                                                   raw_ostream &O) {
+  static_assert(size == 64 || size == 32,
+                "Template parameter must be either 32 or 64");
+  unsigned Reg = MI->getOperand(OpNum).getReg();
+
+  unsigned Sube = (size == 32) ? AArch64::sube32 : AArch64::sube64;
+  unsigned Subo = (size == 32) ? AArch64::subo32 : AArch64::subo64;
+
+  unsigned Even = MRI.getSubReg(Reg,  Sube);
+  unsigned Odd = MRI.getSubReg(Reg,  Subo);
+  O << getRegisterName(Even) << ", " << getRegisterName(Odd);
+}
+
 void AArch64InstPrinter::printVectorList(const MCInst *MI, unsigned OpNum,
                                          const MCSubtargetInfo &STI,
                                          raw_ostream &O,
@@ -1252,12 +1282,12 @@ void AArch64InstPrinter::printAlignedLabel(const MCInst *MI, unsigned OpNum,
   const MCConstantExpr *BranchTarget =
       dyn_cast<MCConstantExpr>(MI->getOperand(OpNum).getExpr());
   int64_t Address;
-  if (BranchTarget && BranchTarget->EvaluateAsAbsolute(Address)) {
+  if (BranchTarget && BranchTarget->evaluateAsAbsolute(Address)) {
     O << "0x";
     O.write_hex(Address);
   } else {
     // Otherwise, just print the expression.
-    O << *MI->getOperand(OpNum).getExpr();
+    MI->getOperand(OpNum).getExpr()->print(O, &MAI);
   }
 }
 
@@ -1274,7 +1304,7 @@ void AArch64InstPrinter::printAdrpLabel(const MCInst *MI, unsigned OpNum,
   }
 
   // Otherwise, just print the expression.
-  O << *MI->getOperand(OpNum).getExpr();
+  MI->getOperand(OpNum).getExpr()->print(O, &MAI);
 }
 
 void AArch64InstPrinter::printBarrierOption(const MCInst *MI, unsigned OpNo,
@@ -1286,10 +1316,10 @@ void AArch64InstPrinter::printBarrierOption(const MCInst *MI, unsigned OpNo,
   bool Valid;
   StringRef Name;
   if (Opcode == AArch64::ISB)
-    Name = AArch64ISB::ISBMapper().toString(Val, STI.getFeatureBits(), 
+    Name = AArch64ISB::ISBMapper().toString(Val, STI.getFeatureBits(),
                                             Valid);
   else
-    Name = AArch64DB::DBarrierMapper().toString(Val, STI.getFeatureBits(), 
+    Name = AArch64DB::DBarrierMapper().toString(Val, STI.getFeatureBits(),
                                                 Valid);
   if (Valid)
     O << Name;
@@ -1325,10 +1355,10 @@ void AArch64InstPrinter::printSystemPStateField(const MCInst *MI, unsigned OpNo,
   unsigned Val = MI->getOperand(OpNo).getImm();
 
   bool Valid;
-  StringRef Name = 
+  StringRef Name =
       AArch64PState::PStateMapper().toString(Val, STI.getFeatureBits(), Valid);
   if (Valid)
-    O << StringRef(Name.str()).upper();
+    O << Name.upper();
   else
     O << "#" << Val;
 }
diff --git a/lib/Target/AArch64/InstPrinter/AArch64InstPrinter.h b/lib/Target/AArch64/InstPrinter/AArch64InstPrinter.h
index c2077a0fe55..15dee978e22 100644
--- a/lib/Target/AArch64/InstPrinter/AArch64InstPrinter.h
+++ b/lib/Target/AArch64/InstPrinter/AArch64InstPrinter.h
@@ -153,6 +153,10 @@ class AArch64InstPrinter : public MCInstPrinter {
                               const MCSubtargetInfo &STI, raw_ostream &O);
   void printSIMDType10Operand(const MCInst *MI, unsigned OpNum,
                               const MCSubtargetInfo &STI, raw_ostream &O);
+  template<unsigned size>
+  void printGPRSeqPairsClassOperand(const MCInst *MI, unsigned OpNum,
+                                    const MCSubtargetInfo &STI,
+                                    raw_ostream &O);
 };
 
 class AArch64AppleInstPrinter : public AArch64InstPrinter {
diff --git a/lib/Target/AArch64/LLVMBuild.txt b/lib/Target/AArch64/LLVMBuild.txt
index 573fa10561c..642c18394a6 100644
--- a/lib/Target/AArch64/LLVMBuild.txt
+++ b/lib/Target/AArch64/LLVMBuild.txt
@@ -31,5 +31,5 @@ has_jit = 1
 type = Library
 name = AArch64CodeGen
 parent = AArch64
-required_libraries = AArch64AsmPrinter AArch64Desc AArch64Info Analysis AsmPrinter CodeGen Core MC Scalar SelectionDAG Support Target
+required_libraries = AArch64AsmPrinter AArch64Desc AArch64Info AArch64Utils Analysis AsmPrinter CodeGen Core MC Scalar SelectionDAG Support Target
 add_to_library_groups = AArch64
diff --git a/lib/Target/AArch64/MCTargetDesc/AArch64AsmBackend.cpp b/lib/Target/AArch64/MCTargetDesc/AArch64AsmBackend.cpp
index e5eb90c6d04..7624c7240d6 100644
--- a/lib/Target/AArch64/MCTargetDesc/AArch64AsmBackend.cpp
+++ b/lib/Target/AArch64/MCTargetDesc/AArch64AsmBackend.cpp
@@ -247,15 +247,12 @@ bool AArch64AsmBackend::writeNopData(uint64_t Count, MCObjectWriter *OW) const {
   // If the count is not 4-byte aligned, we must be writing data into the text
   // section (otherwise we have unaligned instructions, and thus have far
   // bigger problems), so just write zeros instead.
-  if ((Count & 3) != 0) {
-    for (uint64_t i = 0, e = (Count & 3); i != e; ++i)
-      OW->Write8(0);
-  }
+  OW->WriteZeros(Count % 4);
 
   // We are properly aligned, so write NOPs as requested.
   Count /= 4;
   for (uint64_t i = 0; i != Count; ++i)
-    OW->Write32(0xd503201f);
+    OW->write32(0xd503201f);
   return true;
 }
 
@@ -499,7 +496,7 @@ void ELFAArch64AsmBackend::processFixupValue(
 // FIXME: Should be replaced with something more principled.
 static bool isByteSwappedFixup(const MCExpr *E) {
   MCValue Val;
-  if (!E->EvaluateAsRelocatable(Val, nullptr, nullptr))
+  if (!E->evaluateAsRelocatable(Val, nullptr, nullptr))
     return false;
 
   if (!Val.getSymA() || Val.getSymA()->getSymbol().isUndefined())
@@ -523,11 +520,10 @@ void ELFAArch64AsmBackend::applyFixup(const MCFixup &Fixup, char *Data,
 }
 
 MCAsmBackend *llvm::createAArch64leAsmBackend(const Target &T,
-                                            const MCRegisterInfo &MRI,
-                                            StringRef TT, StringRef CPU) {
-  Triple TheTriple(TT);
-
-  if (TheTriple.isOSDarwin())
+                                              const MCRegisterInfo &MRI,
+                                              const Triple &TheTriple,
+                                              StringRef CPU) {
+  if (TheTriple.isOSBinFormatMachO())
     return new DarwinAArch64AsmBackend(T, MRI);
 
   assert(TheTriple.isOSBinFormatELF() && "Expect either MachO or ELF target");
@@ -536,10 +532,9 @@ MCAsmBackend *llvm::createAArch64leAsmBackend(const Target &T,
 }
 
 MCAsmBackend *llvm::createAArch64beAsmBackend(const Target &T,
-                                            const MCRegisterInfo &MRI,
-                                            StringRef TT, StringRef CPU) {
-  Triple TheTriple(TT);
-
+                                              const MCRegisterInfo &MRI,
+                                              const Triple &TheTriple,
+                                              StringRef CPU) {
   assert(TheTriple.isOSBinFormatELF() &&
          "Big endian is only supported for ELF targets!");
   uint8_t OSABI = MCELFObjectTargetWriter::getOSABI(TheTriple.getOS());
diff --git a/lib/Target/AArch64/MCTargetDesc/AArch64ELFStreamer.cpp b/lib/Target/AArch64/MCTargetDesc/AArch64ELFStreamer.cpp
index 540d1fc8f5b..16d53569b23 100644
--- a/lib/Target/AArch64/MCTargetDesc/AArch64ELFStreamer.cpp
+++ b/lib/Target/AArch64/MCTargetDesc/AArch64ELFStreamer.cpp
@@ -13,6 +13,7 @@
 //
 //===----------------------------------------------------------------------===//
 
+#include "AArch64TargetStreamer.h"
 #include "llvm/MC/MCELFStreamer.h"
 #include "llvm/ADT/SmallPtrSet.h"
 #include "llvm/ADT/StringExtras.h"
@@ -22,16 +23,14 @@
 #include "llvm/MC/MCAssembler.h"
 #include "llvm/MC/MCCodeEmitter.h"
 #include "llvm/MC/MCContext.h"
-#include "llvm/MC/MCELF.h"
 #include "llvm/MC/MCELFStreamer.h"
-#include "llvm/MC/MCELFSymbolFlags.h"
 #include "llvm/MC/MCExpr.h"
 #include "llvm/MC/MCInst.h"
 #include "llvm/MC/MCObjectStreamer.h"
 #include "llvm/MC/MCSection.h"
 #include "llvm/MC/MCSectionELF.h"
 #include "llvm/MC/MCStreamer.h"
-#include "llvm/MC/MCSymbol.h"
+#include "llvm/MC/MCSymbolELF.h"
 #include "llvm/MC/MCValue.h"
 #include "llvm/Support/Debug.h"
 #include "llvm/Support/ELF.h"
@@ -59,7 +58,7 @@ AArch64TargetAsmStreamer::AArch64TargetAsmStreamer(MCStreamer &S,
   : AArch64TargetStreamer(S), OS(OS) {}
 
 void AArch64TargetAsmStreamer::emitInst(uint32_t Inst) {
-  OS << "\t.inst\t0x" << utohexstr(Inst) << "\n";
+  OS << "\t.inst\t0x" << Twine::utohexstr(Inst) << "\n";
 }
 
 class AArch64TargetELFStreamer : public AArch64TargetStreamer {
@@ -94,10 +93,7 @@ class AArch64ELFStreamer : public MCELFStreamer {
       : MCELFStreamer(Context, TAB, OS, Emitter), MappingSymbolCounter(0),
         LastEMS(EMS_None) {}
 
-  ~AArch64ELFStreamer() override {}
-
-  void ChangeSection(const MCSection *Section,
-                     const MCExpr *Subsection) override {
+  void ChangeSection(MCSection *Section, const MCExpr *Subsection) override {
     // We have to keep track of the mapping symbol state of any sections we
     // use. Each one should start off as EMS_None, which is provided as the
     // default constructor by DenseMap::lookup.
@@ -117,15 +113,8 @@ class AArch64ELFStreamer : public MCELFStreamer {
   }
 
   void emitInst(uint32_t Inst) {
-    char Buffer[4];
-    const bool LittleEndian = getContext().getAsmInfo()->isLittleEndian();
-
     EmitA64MappingSymbol();
-    for (unsigned II = 0; II != 4; ++II) {
-      const unsigned I = LittleEndian ? (4 - II - 1) : II;
-      Buffer[4 - II - 1] = uint8_t(Inst >> I * CHAR_BIT);
-    }
-    MCELFStreamer::EmitBytes(StringRef(Buffer, 4));
+    MCELFStreamer::EmitIntValue(Inst, 4);
   }
 
   /// This is one of the functions used to emit data into an ELF section, so the
@@ -167,30 +156,18 @@ class AArch64ELFStreamer : public MCELFStreamer {
   }
 
   void EmitMappingSymbol(StringRef Name) {
-    MCSymbol *Start = getContext().CreateTempSymbol();
-    EmitLabel(Start);
-
-    MCSymbol *Symbol = getContext().GetOrCreateSymbol(
-        Name + "." + Twine(MappingSymbolCounter++));
-
-    MCSymbolData &SD = getAssembler().getOrCreateSymbolData(*Symbol);
-    MCELF::SetType(SD, ELF::STT_NOTYPE);
-    MCELF::SetBinding(SD, ELF::STB_LOCAL);
-    SD.setExternal(false);
-    auto Sec = getCurrentSection().first;
-    assert(Sec && "need a section");
-    Symbol->setSection(*Sec);
-
-    const MCExpr *Value = MCSymbolRefExpr::Create(Start, getContext());
-    Symbol->setVariableValue(Value);
+    auto *Symbol = cast<MCSymbolELF>(getContext().getOrCreateSymbol(
+        Name + "." + Twine(MappingSymbolCounter++)));
+    EmitLabel(Symbol);
+    Symbol->setType(ELF::STT_NOTYPE);
+    Symbol->setBinding(ELF::STB_LOCAL);
+    Symbol->setExternal(false);
   }
 
   int64_t MappingSymbolCounter;
 
   DenseMap<const MCSection *, ElfMappingSymbol> LastMappingSymbols;
   ElfMappingSymbol LastEMS;
-
-  /// @}
 };
 } // end anonymous namespace
 
@@ -221,8 +198,8 @@ MCELFStreamer *createAArch64ELFStreamer(MCContext &Context, MCAsmBackend &TAB,
 
 MCTargetStreamer *
 createAArch64ObjectTargetStreamer(MCStreamer &S, const MCSubtargetInfo &STI) {
-  Triple TT(STI.getTargetTriple());
-  if (TT.getObjectFormat() == Triple::ELF)
+  const Triple &TT = STI.getTargetTriple();
+  if (TT.isOSBinFormatELF())
     return new AArch64TargetELFStreamer(S);
   return nullptr;
 }
diff --git a/lib/Target/AArch64/MCTargetDesc/AArch64MCAsmInfo.cpp b/lib/Target/AArch64/MCTargetDesc/AArch64MCAsmInfo.cpp
index f048474fd4a..921c4b94a72 100644
--- a/lib/Target/AArch64/MCTargetDesc/AArch64MCAsmInfo.cpp
+++ b/lib/Target/AArch64/MCTargetDesc/AArch64MCAsmInfo.cpp
@@ -48,6 +48,10 @@ AArch64MCAsmInfoDarwin::AArch64MCAsmInfoDarwin() {
   UseDataRegionDirectives = true;
 
   ExceptionsType = ExceptionHandling::DwarfCFI;
+
+  // AArch64 Darwin doesn't have the baggage of X86/ARM, so it's fine to use
+  // LShr instead of AShr.
+  UseLogicalShr = true;
 }
 
 const MCExpr *AArch64MCAsmInfoDarwin::getExprForPersonalitySymbol(
@@ -58,15 +62,14 @@ const MCExpr *AArch64MCAsmInfoDarwin::getExprForPersonalitySymbol(
   // version.
   MCContext &Context = Streamer.getContext();
   const MCExpr *Res =
-      MCSymbolRefExpr::Create(Sym, MCSymbolRefExpr::VK_GOT, Context);
-  MCSymbol *PCSym = Context.CreateTempSymbol();
+      MCSymbolRefExpr::create(Sym, MCSymbolRefExpr::VK_GOT, Context);
+  MCSymbol *PCSym = Context.createTempSymbol();
   Streamer.EmitLabel(PCSym);
-  const MCExpr *PC = MCSymbolRefExpr::Create(PCSym, Context);
-  return MCBinaryExpr::CreateSub(Res, PC, Context);
+  const MCExpr *PC = MCSymbolRefExpr::create(PCSym, Context);
+  return MCBinaryExpr::createSub(Res, PC, Context);
 }
 
-AArch64MCAsmInfoELF::AArch64MCAsmInfoELF(StringRef TT) {
-  Triple T(TT);
+AArch64MCAsmInfoELF::AArch64MCAsmInfoELF(const Triple &T) {
   if (T.getArch() == Triple::aarch64_be)
     IsLittleEndian = false;
 
diff --git a/lib/Target/AArch64/MCTargetDesc/AArch64MCAsmInfo.h b/lib/Target/AArch64/MCTargetDesc/AArch64MCAsmInfo.h
index 9b88de7dabb..253cd30f26e 100644
--- a/lib/Target/AArch64/MCTargetDesc/AArch64MCAsmInfo.h
+++ b/lib/Target/AArch64/MCTargetDesc/AArch64MCAsmInfo.h
@@ -18,9 +18,10 @@
 #include "llvm/MC/MCAsmInfoELF.h"
 
 namespace llvm {
-class Target;
-class StringRef;
 class MCStreamer;
+class Target;
+class Triple;
+
 struct AArch64MCAsmInfoDarwin : public MCAsmInfoDarwin {
   explicit AArch64MCAsmInfoDarwin();
   const MCExpr *
@@ -29,7 +30,7 @@ struct AArch64MCAsmInfoDarwin : public MCAsmInfoDarwin {
 };
 
 struct AArch64MCAsmInfoELF : public MCAsmInfoELF {
-  explicit AArch64MCAsmInfoELF(StringRef TT);
+  explicit AArch64MCAsmInfoELF(const Triple &T);
 };
 
 } // namespace llvm
diff --git a/lib/Target/AArch64/MCTargetDesc/AArch64MCCodeEmitter.cpp b/lib/Target/AArch64/MCTargetDesc/AArch64MCCodeEmitter.cpp
index fd4dc47f210..7d8e79bc63c 100644
--- a/lib/Target/AArch64/MCTargetDesc/AArch64MCCodeEmitter.cpp
+++ b/lib/Target/AArch64/MCTargetDesc/AArch64MCCodeEmitter.cpp
@@ -22,6 +22,7 @@
 #include "llvm/MC/MCInstrInfo.h"
 #include "llvm/MC/MCRegisterInfo.h"
 #include "llvm/MC/MCSubtargetInfo.h"
+#include "llvm/Support/EndianStream.h"
 #include "llvm/Support/raw_ostream.h"
 using namespace llvm;
 
@@ -174,17 +175,7 @@ class AArch64MCCodeEmitter : public MCCodeEmitter {
   unsigned fixMOVZ(const MCInst &MI, unsigned EncodedValue,
                    const MCSubtargetInfo &STI) const;
 
-  void EmitByte(unsigned char C, raw_ostream &OS) const { OS << (char)C; }
-
-  void EmitConstant(uint64_t Val, unsigned Size, raw_ostream &OS) const {
-    // Output the constant in little endian byte order.
-    for (unsigned i = 0; i != Size; ++i) {
-      EmitByte(Val & 255, OS);
-      Val >>= 8;
-    }
-  }
-
-  void EncodeInstruction(const MCInst &MI, raw_ostream &OS,
+  void encodeInstruction(const MCInst &MI, raw_ostream &OS,
                          SmallVectorImpl<MCFixup> &Fixups,
                          const MCSubtargetInfo &STI) const override;
 
@@ -232,7 +223,7 @@ AArch64MCCodeEmitter::getLdStUImm12OpValue(const MCInst &MI, unsigned OpIdx,
   else {
     assert(MO.isExpr() && "unable to encode load/store imm operand");
     MCFixupKind Kind = MCFixupKind(FixupKind);
-    Fixups.push_back(MCFixup::Create(0, MO.getExpr(), Kind, MI.getLoc()));
+    Fixups.push_back(MCFixup::create(0, MO.getExpr(), Kind, MI.getLoc()));
     ++MCNumFixups;
   }
 
@@ -256,7 +247,7 @@ AArch64MCCodeEmitter::getAdrLabelOpValue(const MCInst &MI, unsigned OpIdx,
   MCFixupKind Kind = MI.getOpcode() == AArch64::ADR
                          ? MCFixupKind(AArch64::fixup_aarch64_pcrel_adr_imm21)
                          : MCFixupKind(AArch64::fixup_aarch64_pcrel_adrp_imm21);
-  Fixups.push_back(MCFixup::Create(0, Expr, Kind, MI.getLoc()));
+  Fixups.push_back(MCFixup::create(0, Expr, Kind, MI.getLoc()));
 
   MCNumFixups += 1;
 
@@ -286,7 +277,7 @@ AArch64MCCodeEmitter::getAddSubImmOpValue(const MCInst &MI, unsigned OpIdx,
 
   // Encode the 12 bits of the fixup.
   MCFixupKind Kind = MCFixupKind(AArch64::fixup_aarch64_add_imm12);
-  Fixups.push_back(MCFixup::Create(0, Expr, Kind, MI.getLoc()));
+  Fixups.push_back(MCFixup::create(0, Expr, Kind, MI.getLoc()));
 
   ++MCNumFixups;
 
@@ -306,7 +297,7 @@ uint32_t AArch64MCCodeEmitter::getCondBranchTargetOpValue(
   assert(MO.isExpr() && "Unexpected target type!");
 
   MCFixupKind Kind = MCFixupKind(AArch64::fixup_aarch64_pcrel_branch19);
-  Fixups.push_back(MCFixup::Create(0, MO.getExpr(), Kind, MI.getLoc()));
+  Fixups.push_back(MCFixup::create(0, MO.getExpr(), Kind, MI.getLoc()));
 
   ++MCNumFixups;
 
@@ -328,7 +319,7 @@ AArch64MCCodeEmitter::getLoadLiteralOpValue(const MCInst &MI, unsigned OpIdx,
   assert(MO.isExpr() && "Unexpected target type!");
 
   MCFixupKind Kind = MCFixupKind(AArch64::fixup_aarch64_ldr_pcrel_imm19);
-  Fixups.push_back(MCFixup::Create(0, MO.getExpr(), Kind, MI.getLoc()));
+  Fixups.push_back(MCFixup::create(0, MO.getExpr(), Kind, MI.getLoc()));
 
   ++MCNumFixups;
 
@@ -355,7 +346,7 @@ AArch64MCCodeEmitter::getMoveWideImmOpValue(const MCInst &MI, unsigned OpIdx,
     return MO.getImm();
   assert(MO.isExpr() && "Unexpected movz/movk immediate");
 
-  Fixups.push_back(MCFixup::Create(
+  Fixups.push_back(MCFixup::create(
       0, MO.getExpr(), MCFixupKind(AArch64::fixup_aarch64_movw), MI.getLoc()));
 
   ++MCNumFixups;
@@ -376,7 +367,7 @@ uint32_t AArch64MCCodeEmitter::getTestBranchTargetOpValue(
   assert(MO.isExpr() && "Unexpected ADR target type!");
 
   MCFixupKind Kind = MCFixupKind(AArch64::fixup_aarch64_pcrel_branch14);
-  Fixups.push_back(MCFixup::Create(0, MO.getExpr(), Kind, MI.getLoc()));
+  Fixups.push_back(MCFixup::create(0, MO.getExpr(), Kind, MI.getLoc()));
 
   ++MCNumFixups;
 
@@ -400,7 +391,7 @@ AArch64MCCodeEmitter::getBranchTargetOpValue(const MCInst &MI, unsigned OpIdx,
   MCFixupKind Kind = MI.getOpcode() == AArch64::BL
                          ? MCFixupKind(AArch64::fixup_aarch64_pcrel_call26)
                          : MCFixupKind(AArch64::fixup_aarch64_pcrel_branch26);
-  Fixups.push_back(MCFixup::Create(0, MO.getExpr(), Kind, MI.getLoc()));
+  Fixups.push_back(MCFixup::create(0, MO.getExpr(), Kind, MI.getLoc()));
 
   ++MCNumFixups;
 
@@ -598,7 +589,7 @@ unsigned AArch64MCCodeEmitter::fixMOVZ(const MCInst &MI, unsigned EncodedValue,
   return EncodedValue & ~(1u << 30);
 }
 
-void AArch64MCCodeEmitter::EncodeInstruction(const MCInst &MI, raw_ostream &OS,
+void AArch64MCCodeEmitter::encodeInstruction(const MCInst &MI, raw_ostream &OS,
                                              SmallVectorImpl<MCFixup> &Fixups,
                                              const MCSubtargetInfo &STI) const {
   if (MI.getOpcode() == AArch64::TLSDESCCALL) {
@@ -606,12 +597,12 @@ void AArch64MCCodeEmitter::EncodeInstruction(const MCInst &MI, raw_ostream &OS,
     // following (BLR) instruction. It doesn't emit any code itself so it
     // doesn't go through the normal TableGenerated channels.
     MCFixupKind Fixup = MCFixupKind(AArch64::fixup_aarch64_tlsdesc_call);
-    Fixups.push_back(MCFixup::Create(0, MI.getOperand(0).getExpr(), Fixup));
+    Fixups.push_back(MCFixup::create(0, MI.getOperand(0).getExpr(), Fixup));
     return;
   }
 
   uint64_t Binary = getBinaryCodeForInstr(MI, Fixups, STI);
-  EmitConstant(Binary, 4, OS);
+  support::endian::Writer<support::little>(OS).write<uint32_t>(Binary);
   ++MCNumEmitted; // Keep track of the # of mi's emitted.
 }
 
diff --git a/lib/Target/AArch64/MCTargetDesc/AArch64MCExpr.cpp b/lib/Target/AArch64/MCTargetDesc/AArch64MCExpr.cpp
index 9e31508bfb0..28703419514 100644
--- a/lib/Target/AArch64/MCTargetDesc/AArch64MCExpr.cpp
+++ b/lib/Target/AArch64/MCTargetDesc/AArch64MCExpr.cpp
@@ -15,9 +15,8 @@
 #include "AArch64MCExpr.h"
 #include "llvm/MC/MCAssembler.h"
 #include "llvm/MC/MCContext.h"
-#include "llvm/MC/MCELF.h"
 #include "llvm/MC/MCStreamer.h"
-#include "llvm/MC/MCSymbol.h"
+#include "llvm/MC/MCSymbolELF.h"
 #include "llvm/MC/MCValue.h"
 #include "llvm/Object/ELF.h"
 #include "llvm/Support/ErrorHandling.h"
@@ -26,7 +25,7 @@ using namespace llvm;
 
 #define DEBUG_TYPE "aarch64symbolrefexpr"
 
-const AArch64MCExpr *AArch64MCExpr::Create(const MCExpr *Expr, VariantKind Kind,
+const AArch64MCExpr *AArch64MCExpr::create(const MCExpr *Expr, VariantKind Kind,
                                        MCContext &Ctx) {
   return new (Ctx) AArch64MCExpr(Expr, Kind);
 }
@@ -76,24 +75,24 @@ StringRef AArch64MCExpr::getVariantKindName() const {
   }
 }
 
-void AArch64MCExpr::PrintImpl(raw_ostream &OS) const {
+void AArch64MCExpr::printImpl(raw_ostream &OS, const MCAsmInfo *MAI) const {
   if (getKind() != VK_NONE)
     OS << getVariantKindName();
-  OS << *Expr;
+  Expr->print(OS, MAI);
 }
 
 void AArch64MCExpr::visitUsedExpr(MCStreamer &Streamer) const {
   Streamer.visitUsedExpr(*getSubExpr());
 }
 
-const MCSection *AArch64MCExpr::FindAssociatedSection() const {
+MCSection *AArch64MCExpr::findAssociatedSection() const {
   llvm_unreachable("FIXME: what goes here?");
 }
 
-bool AArch64MCExpr::EvaluateAsRelocatableImpl(MCValue &Res,
+bool AArch64MCExpr::evaluateAsRelocatableImpl(MCValue &Res,
                                             const MCAsmLayout *Layout,
 					    const MCFixup *Fixup) const {
-  if (!getSubExpr()->EvaluateAsRelocatable(Res, Layout, Fixup))
+  if (!getSubExpr()->evaluateAsRelocatable(Res, Layout, Fixup))
     return false;
 
   Res =
@@ -121,8 +120,7 @@ static void fixELFSymbolsInTLSFixupsImpl(const MCExpr *Expr, MCAssembler &Asm) {
     // We're known to be under a TLS fixup, so any symbol should be
     // modified. There should be only one.
     const MCSymbolRefExpr &SymRef = *cast<MCSymbolRefExpr>(Expr);
-    MCSymbolData &SD = Asm.getOrCreateSymbolData(SymRef.getSymbol());
-    MCELF::SetType(SD, ELF::STT_TLS);
+    cast<MCSymbolELF>(SymRef.getSymbol()).setType(ELF::STT_TLS);
     break;
   }
 
diff --git a/lib/Target/AArch64/MCTargetDesc/AArch64MCExpr.h b/lib/Target/AArch64/MCTargetDesc/AArch64MCExpr.h
index db48ac9f569..1165314e410 100644
--- a/lib/Target/AArch64/MCTargetDesc/AArch64MCExpr.h
+++ b/lib/Target/AArch64/MCTargetDesc/AArch64MCExpr.h
@@ -112,7 +112,7 @@ class AArch64MCExpr : public MCTargetExpr {
   /// @name Construction
   /// @{
 
-  static const AArch64MCExpr *Create(const MCExpr *Expr, VariantKind Kind,
+  static const AArch64MCExpr *create(const MCExpr *Expr, VariantKind Kind,
                                    MCContext &Ctx);
 
   /// @}
@@ -120,7 +120,7 @@ class AArch64MCExpr : public MCTargetExpr {
   /// @{
 
   /// Get the kind of this expression.
-  VariantKind getKind() const { return static_cast<VariantKind>(Kind); }
+  VariantKind getKind() const { return Kind; }
 
   /// Get the expression this modifier applies to.
   const MCExpr *getSubExpr() const { return Expr; }
@@ -145,13 +145,13 @@ class AArch64MCExpr : public MCTargetExpr {
   /// (e.g. ":got:", ":lo12:").
   StringRef getVariantKindName() const;
 
-  void PrintImpl(raw_ostream &OS) const override;
+  void printImpl(raw_ostream &OS, const MCAsmInfo *MAI) const override;
 
   void visitUsedExpr(MCStreamer &Streamer) const override;
 
-  const MCSection *FindAssociatedSection() const override;
+  MCSection *findAssociatedSection() const override;
 
-  bool EvaluateAsRelocatableImpl(MCValue &Res,
+  bool evaluateAsRelocatableImpl(MCValue &Res,
                                  const MCAsmLayout *Layout,
 				 const MCFixup *Fixup) const override;
 
diff --git a/lib/Target/AArch64/MCTargetDesc/AArch64MCTargetDesc.cpp b/lib/Target/AArch64/MCTargetDesc/AArch64MCTargetDesc.cpp
index afad674db06..9f7bed0d3b1 100644
--- a/lib/Target/AArch64/MCTargetDesc/AArch64MCTargetDesc.cpp
+++ b/lib/Target/AArch64/MCTargetDesc/AArch64MCTargetDesc.cpp
@@ -41,32 +41,27 @@ static MCInstrInfo *createAArch64MCInstrInfo() {
 }
 
 static MCSubtargetInfo *
-createAArch64MCSubtargetInfo(StringRef TT, StringRef CPU, StringRef FS) {
-  MCSubtargetInfo *X = new MCSubtargetInfo();
-
+createAArch64MCSubtargetInfo(const Triple &TT, StringRef CPU, StringRef FS) {
   if (CPU.empty())
     CPU = "generic";
 
-  InitAArch64MCSubtargetInfo(X, TT, CPU, FS);
-  return X;
+  return createAArch64MCSubtargetInfoImpl(TT, CPU, FS);
 }
 
-static MCRegisterInfo *createAArch64MCRegisterInfo(StringRef Triple) {
+static MCRegisterInfo *createAArch64MCRegisterInfo(const Triple &Triple) {
   MCRegisterInfo *X = new MCRegisterInfo();
   InitAArch64MCRegisterInfo(X, AArch64::LR);
   return X;
 }
 
 static MCAsmInfo *createAArch64MCAsmInfo(const MCRegisterInfo &MRI,
-                                         StringRef TT) {
-  Triple TheTriple(TT);
-
+                                         const Triple &TheTriple) {
   MCAsmInfo *MAI;
-  if (TheTriple.isOSDarwin())
+  if (TheTriple.isOSBinFormatMachO())
     MAI = new AArch64MCAsmInfoDarwin();
   else {
     assert(TheTriple.isOSBinFormatELF() && "Only expect Darwin or ELF");
-    MAI = new AArch64MCAsmInfoELF(TT);
+    MAI = new AArch64MCAsmInfoELF(TheTriple);
   }
 
   // Initial state of the frame pointer is SP.
@@ -77,11 +72,11 @@ static MCAsmInfo *createAArch64MCAsmInfo(const MCRegisterInfo &MRI,
   return MAI;
 }
 
-static MCCodeGenInfo *createAArch64MCCodeGenInfo(StringRef TT, Reloc::Model RM,
+static MCCodeGenInfo *createAArch64MCCodeGenInfo(const Triple &TT,
+                                                 Reloc::Model RM,
                                                  CodeModel::Model CM,
                                                  CodeGenOpt::Level OL) {
-  Triple TheTriple(TT);
-  assert((TheTriple.isOSBinFormatELF() || TheTriple.isOSBinFormatMachO()) &&
+  assert((TT.isOSBinFormatELF() || TT.isOSBinFormatMachO()) &&
          "Only expect Darwin and ELF targets");
 
   if (CM == CodeModel::Default)
@@ -96,7 +91,7 @@ static MCCodeGenInfo *createAArch64MCCodeGenInfo(StringRef TT, Reloc::Model RM,
         "Only small and large code models are allowed on AArch64");
 
   // AArch64 Darwin is always PIC.
-  if (TheTriple.isOSDarwin())
+  if (TT.isOSDarwin())
     RM = Reloc::PIC_;
   // On ELF platforms the default static relocation model has a smart enough
   // linker to cope with referencing external symbols defined in a shared
@@ -105,7 +100,7 @@ static MCCodeGenInfo *createAArch64MCCodeGenInfo(StringRef TT, Reloc::Model RM,
     RM = Reloc::Static;
 
   MCCodeGenInfo *X = new MCCodeGenInfo();
-  X->InitMCCodeGenInfo(RM, CM, OL);
+  X->initMCCodeGenInfo(RM, CM, OL);
   return X;
 }
 
diff --git a/lib/Target/AArch64/MCTargetDesc/AArch64MCTargetDesc.h b/lib/Target/AArch64/MCTargetDesc/AArch64MCTargetDesc.h
index 4705bdf546f..342384437c6 100644
--- a/lib/Target/AArch64/MCTargetDesc/AArch64MCTargetDesc.h
+++ b/lib/Target/AArch64/MCTargetDesc/AArch64MCTargetDesc.h
@@ -43,11 +43,11 @@ MCCodeEmitter *createAArch64MCCodeEmitter(const MCInstrInfo &MCII,
                                           const MCRegisterInfo &MRI,
                                           MCContext &Ctx);
 MCAsmBackend *createAArch64leAsmBackend(const Target &T,
-                                        const MCRegisterInfo &MRI, StringRef TT,
-                                        StringRef CPU);
+                                        const MCRegisterInfo &MRI,
+                                        const Triple &TT, StringRef CPU);
 MCAsmBackend *createAArch64beAsmBackend(const Target &T,
-                                        const MCRegisterInfo &MRI, StringRef TT,
-                                        StringRef CPU);
+                                        const MCRegisterInfo &MRI,
+                                        const Triple &TT, StringRef CPU);
 
 MCObjectWriter *createAArch64ELFObjectWriter(raw_pwrite_stream &OS,
                                              uint8_t OSABI,
diff --git a/lib/Target/AArch64/MCTargetDesc/AArch64MachObjectWriter.cpp b/lib/Target/AArch64/MCTargetDesc/AArch64MachObjectWriter.cpp
index 61649c44142..741b273073e 100644
--- a/lib/Target/AArch64/MCTargetDesc/AArch64MachObjectWriter.cpp
+++ b/lib/Target/AArch64/MCTargetDesc/AArch64MachObjectWriter.cpp
@@ -31,10 +31,9 @@ class AArch64MachObjectWriter : public MCMachObjectTargetWriter {
 
 public:
   AArch64MachObjectWriter(uint32_t CPUType, uint32_t CPUSubtype)
-      : MCMachObjectTargetWriter(true /* is64Bit */, CPUType, CPUSubtype,
-                                 /*UseAggressiveSymbolFolding=*/true) {}
+      : MCMachObjectTargetWriter(true /* is64Bit */, CPUType, CPUSubtype) {}
 
-  void RecordRelocation(MachObjectWriter *Writer, MCAssembler &Asm,
+  void recordRelocation(MachObjectWriter *Writer, MCAssembler &Asm,
                         const MCAsmLayout &Layout, const MCFragment *Fragment,
                         const MCFixup &Fixup, MCValue Target,
                         uint64_t &FixedValue) override;
@@ -92,7 +91,7 @@ bool AArch64MachObjectWriter::getAArch64FixupKindMachOInfo(
     // This encompasses the relocation for the whole 21-bit value.
     switch (Sym->getKind()) {
     default:
-      Asm.getContext().FatalError(Fixup.getLoc(),
+      Asm.getContext().reportFatalError(Fixup.getLoc(),
                                   "ADR/ADRP relocations must be GOT relative");
     case MCSymbolRefExpr::VK_PAGE:
       RelocType = unsigned(MachO::ARM64_RELOC_PAGE21);
@@ -130,18 +129,17 @@ static bool canUseLocalRelocation(const MCSectionMachO &Section,
   if (RefSec.getType() == MachO::S_CSTRING_LITERALS)
     return false;
 
-  if (RefSec.getSegmentName() == "__DATA" &&
-      RefSec.getSectionName() == "__cfstring")
-    return false;
-
   if (RefSec.getSegmentName() == "__DATA" &&
       RefSec.getSectionName() == "__objc_classrefs")
     return false;
 
-  return true;
+  // FIXME: ld64 currently handles internal pointer-sized relocations
+  // incorrectly (applying the addend twice). We should be able to return true
+  // unconditionally by this point when that's fixed.
+  return false;
 }
 
-void AArch64MachObjectWriter::RecordRelocation(
+void AArch64MachObjectWriter::recordRelocation(
     MachObjectWriter *Writer, MCAssembler &Asm, const MCAsmLayout &Layout,
     const MCFragment *Fragment, const MCFixup &Fixup, MCValue Target,
     uint64_t &FixedValue) {
@@ -154,7 +152,7 @@ void AArch64MachObjectWriter::RecordRelocation(
   unsigned Index = 0;
   unsigned Type = 0;
   unsigned Kind = Fixup.getKind();
-  const MCSymbolData *RelSymbol = nullptr;
+  const MCSymbol *RelSymbol = nullptr;
 
   FixupOffset += Fixup.getOffset();
 
@@ -172,7 +170,7 @@ void AArch64MachObjectWriter::RecordRelocation(
   // assembler local symbols. If we got here, that's not what we have,
   // so complain loudly.
   if (Kind == AArch64::fixup_aarch64_pcrel_branch19) {
-    Asm.getContext().FatalError(Fixup.getLoc(),
+    Asm.getContext().reportFatalError(Fixup.getLoc(),
                                 "conditional branch requires assembler-local"
                                 " label. '" +
                                     Target.getSymA()->getSymbol().getName() +
@@ -183,14 +181,14 @@ void AArch64MachObjectWriter::RecordRelocation(
   // 14-bit branch relocations should only target internal labels, and so
   // should never get here.
   if (Kind == AArch64::fixup_aarch64_pcrel_branch14) {
-    Asm.getContext().FatalError(Fixup.getLoc(),
+    Asm.getContext().reportFatalError(Fixup.getLoc(),
                                 "Invalid relocation on conditional branch!");
     return;
   }
 
   if (!getAArch64FixupKindMachOInfo(Fixup, Type, Target.getSymA(), Log2Size,
                                   Asm)) {
-    Asm.getContext().FatalError(Fixup.getLoc(), "unknown AArch64 fixup kind!");
+    Asm.getContext().reportFatalError(Fixup.getLoc(), "unknown AArch64 fixup kind!");
     return;
   }
 
@@ -202,7 +200,7 @@ void AArch64MachObjectWriter::RecordRelocation(
     Type = MachO::ARM64_RELOC_UNSIGNED;
 
     if (IsPCRel) {
-      Asm.getContext().FatalError(Fixup.getLoc(),
+      Asm.getContext().reportFatalError(Fixup.getLoc(),
                                   "PC relative absolute relocation!");
 
       // FIXME: x86_64 sets the type to a branch reloc here. Should we do
@@ -210,19 +208,17 @@ void AArch64MachObjectWriter::RecordRelocation(
     }
   } else if (Target.getSymB()) { // A - B + constant
     const MCSymbol *A = &Target.getSymA()->getSymbol();
-    const MCSymbolData &A_SD = Asm.getSymbolData(*A);
-    const MCSymbolData *A_Base = Asm.getAtom(&A_SD);
+    const MCSymbol *A_Base = Asm.getAtom(*A);
 
     const MCSymbol *B = &Target.getSymB()->getSymbol();
-    const MCSymbolData &B_SD = Asm.getSymbolData(*B);
-    const MCSymbolData *B_Base = Asm.getAtom(&B_SD);
+    const MCSymbol *B_Base = Asm.getAtom(*B);
 
     // Check for "_foo@got - .", which comes through here as:
     // Ltmp0:
     //    ... _foo@got - Ltmp0
     if (Target.getSymA()->getKind() == MCSymbolRefExpr::VK_GOT &&
         Target.getSymB()->getKind() == MCSymbolRefExpr::VK_None &&
-        Layout.getSymbolOffset(&B_SD) ==
+        Layout.getSymbolOffset(*B) ==
             Layout.getFragmentOffset(Fragment) + Fixup.getOffset()) {
       // SymB is the PC, so use a PC-rel pointer-to-GOT relocation.
       Type = MachO::ARM64_RELOC_POINTER_TO_GOT;
@@ -235,12 +231,12 @@ void AArch64MachObjectWriter::RecordRelocation(
     } else if (Target.getSymA()->getKind() != MCSymbolRefExpr::VK_None ||
                Target.getSymB()->getKind() != MCSymbolRefExpr::VK_None)
       // Otherwise, neither symbol can be modified.
-      Asm.getContext().FatalError(Fixup.getLoc(),
+      Asm.getContext().reportFatalError(Fixup.getLoc(),
                                   "unsupported relocation of modified symbol");
 
     // We don't support PCrel relocations of differences.
     if (IsPCRel)
-      Asm.getContext().FatalError(Fixup.getLoc(),
+      Asm.getContext().reportFatalError(Fixup.getLoc(),
                                   "unsupported pc-relative relocation of "
                                   "difference");
 
@@ -251,30 +247,26 @@ void AArch64MachObjectWriter::RecordRelocation(
     // FIXME: We should probably just synthesize an external symbol and use
     // that.
     if (!A_Base)
-      Asm.getContext().FatalError(
+      Asm.getContext().reportFatalError(
           Fixup.getLoc(),
           "unsupported relocation of local symbol '" + A->getName() +
               "'. Must have non-local symbol earlier in section.");
     if (!B_Base)
-      Asm.getContext().FatalError(
+      Asm.getContext().reportFatalError(
           Fixup.getLoc(),
           "unsupported relocation of local symbol '" + B->getName() +
               "'. Must have non-local symbol earlier in section.");
 
     if (A_Base == B_Base && A_Base)
-      Asm.getContext().FatalError(Fixup.getLoc(),
+      Asm.getContext().reportFatalError(Fixup.getLoc(),
                                   "unsupported relocation with identical base");
 
-    Value += (!A_SD.getFragment() ? 0
-                                  : Writer->getSymbolAddress(&A_SD, Layout)) -
-             (!A_Base || !A_Base->getFragment()
-                  ? 0
-                  : Writer->getSymbolAddress(A_Base, Layout));
-    Value -= (!B_SD.getFragment() ? 0
-                                  : Writer->getSymbolAddress(&B_SD, Layout)) -
-             (!B_Base || !B_Base->getFragment()
-                  ? 0
-                  : Writer->getSymbolAddress(B_Base, Layout));
+    Value += (!A->getFragment() ? 0 : Writer->getSymbolAddress(*A, Layout)) -
+             (!A_Base || !A_Base->getFragment() ? 0 : Writer->getSymbolAddress(
+                                                          *A_Base, Layout));
+    Value -= (!B->getFragment() ? 0 : Writer->getSymbolAddress(*B, Layout)) -
+             (!B_Base || !B_Base->getFragment() ? 0 : Writer->getSymbolAddress(
+                                                          *B_Base, Layout));
 
     Type = MachO::ARM64_RELOC_UNSIGNED;
 
@@ -287,19 +279,18 @@ void AArch64MachObjectWriter::RecordRelocation(
     Type = MachO::ARM64_RELOC_SUBTRACTOR;
   } else { // A + constant
     const MCSymbol *Symbol = &Target.getSymA()->getSymbol();
-    const MCSectionMachO &Section = static_cast<const MCSectionMachO &>(
-        Fragment->getParent()->getSection());
+    const MCSectionMachO &Section =
+        static_cast<const MCSectionMachO &>(*Fragment->getParent());
 
     bool CanUseLocalRelocation =
         canUseLocalRelocation(Section, *Symbol, Log2Size);
     if (Symbol->isTemporary() && (Value || !CanUseLocalRelocation)) {
       const MCSection &Sec = Symbol->getSection();
       if (!Asm.getContext().getAsmInfo()->isSectionAtomizableBySymbols(Sec))
-        Asm.addLocalUsedInReloc(*Symbol);
+        Symbol->setUsedInReloc();
     }
 
-    const MCSymbolData &SD = Asm.getSymbolData(*Symbol);
-    const MCSymbolData *Base = Asm.getAtom(&SD);
+    const MCSymbol *Base = Asm.getAtom(*Symbol);
 
     // If the symbol is a variable and we weren't able to get a Base for it
     // (i.e., it's not in the symbol table associated with a section) resolve
@@ -308,7 +299,7 @@ void AArch64MachObjectWriter::RecordRelocation(
       // If the evaluation is an absolute value, just use that directly
       // to keep things easy.
       int64_t Res;
-      if (SD.getSymbol().getVariableValue()->EvaluateAsAbsolute(
+      if (Symbol->getVariableValue()->evaluateAsAbsolute(
               Res, Layout, Writer->getSectionAddressMap())) {
         FixedValue = Res;
         return;
@@ -317,12 +308,12 @@ void AArch64MachObjectWriter::RecordRelocation(
       // FIXME: Will the Target we already have ever have any data in it
       // we need to preserve and merge with the new Target? How about
       // the FixedValue?
-      if (!Symbol->getVariableValue()->EvaluateAsRelocatable(Target, &Layout,
+      if (!Symbol->getVariableValue()->evaluateAsRelocatable(Target, &Layout,
                                                              &Fixup))
-        Asm.getContext().FatalError(Fixup.getLoc(),
+        Asm.getContext().reportFatalError(Fixup.getLoc(),
                                     "unable to resolve variable '" +
                                         Symbol->getName() + "'");
-      return RecordRelocation(Writer, Asm, Layout, Fragment, Fixup, Target,
+      return recordRelocation(Writer, Asm, Layout, Fragment, Fixup, Target,
                               FixedValue);
     }
 
@@ -342,35 +333,35 @@ void AArch64MachObjectWriter::RecordRelocation(
       RelSymbol = Base;
 
       // Add the local offset, if needed.
-      if (Base != &SD)
-        Value += Layout.getSymbolOffset(&SD) - Layout.getSymbolOffset(Base);
+      if (Base != Symbol)
+        Value +=
+            Layout.getSymbolOffset(*Symbol) - Layout.getSymbolOffset(*Base);
     } else if (Symbol->isInSection()) {
       if (!CanUseLocalRelocation)
-        Asm.getContext().FatalError(
+        Asm.getContext().reportFatalError(
             Fixup.getLoc(),
             "unsupported relocation of local symbol '" + Symbol->getName() +
                 "'. Must have non-local symbol earlier in section.");
       // Adjust the relocation to be section-relative.
       // The index is the section ordinal (1-based).
-      const MCSectionData &SymSD =
-          Asm.getSectionData(SD.getSymbol().getSection());
-      Index = SymSD.getOrdinal() + 1;
-      Value += Writer->getSymbolAddress(&SD, Layout);
+      const MCSection &Sec = Symbol->getSection();
+      Index = Sec.getOrdinal() + 1;
+      Value += Writer->getSymbolAddress(*Symbol, Layout);
 
       if (IsPCRel)
         Value -= Writer->getFragmentAddress(Fragment, Layout) +
                  Fixup.getOffset() + (1ULL << Log2Size);
     } else {
       // Resolve constant variables.
-      if (SD.getSymbol().isVariable()) {
+      if (Symbol->isVariable()) {
         int64_t Res;
-        if (SD.getSymbol().getVariableValue()->EvaluateAsAbsolute(
+        if (Symbol->getVariableValue()->evaluateAsAbsolute(
                 Res, Layout, Writer->getSectionAddressMap())) {
           FixedValue = Res;
           return;
         }
       }
-      Asm.getContext().FatalError(Fixup.getLoc(),
+      Asm.getContext().reportFatalError(Fixup.getLoc(),
                                   "unsupported relocation of variable '" +
                                       Symbol->getName() + "'");
     }
diff --git a/lib/Target/AArch64/MCTargetDesc/AArch64TargetStreamer.cpp b/lib/Target/AArch64/MCTargetDesc/AArch64TargetStreamer.cpp
index e3112fa76d3..52b000d15b8 100644
--- a/lib/Target/AArch64/MCTargetDesc/AArch64TargetStreamer.cpp
+++ b/lib/Target/AArch64/MCTargetDesc/AArch64TargetStreamer.cpp
@@ -1,4 +1,4 @@
-//===- AArch64TargetStreamer.cpp - AArch64TargetStreamer class --*- C++ -*---------===//
+//===- AArch64TargetStreamer.cpp - AArch64TargetStreamer class ------------===//
 //
 //                     The LLVM Compiler Infrastructure
 //
@@ -10,12 +10,9 @@
 // This file implements the AArch64TargetStreamer class.
 //
 //===----------------------------------------------------------------------===//
-#include "llvm/ADT/MapVector.h"
-#include "llvm/MC/ConstantPools.h"
-#include "llvm/MC/MCContext.h"
-#include "llvm/MC/MCExpr.h"
-#include "llvm/MC/MCStreamer.h"
 
+#include "AArch64TargetStreamer.h"
+#include "llvm/MC/ConstantPools.h"
 using namespace llvm;
 
 //
diff --git a/lib/Target/AArch64/MCTargetDesc/AArch64TargetStreamer.h b/lib/Target/AArch64/MCTargetDesc/AArch64TargetStreamer.h
new file mode 100644
index 00000000000..fcc0d053f6e
--- /dev/null
+++ b/lib/Target/AArch64/MCTargetDesc/AArch64TargetStreamer.h
@@ -0,0 +1,42 @@
+//===-- AArch64TargetStreamer.h - AArch64 Target Streamer ------*- C++ -*--===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIB_TARGET_AARCH64_MCTARGETDESC_AARCH64TARGETSTREAMER_H
+#define LLVM_LIB_TARGET_AARCH64_MCTARGETDESC_AARCH64TARGETSTREAMER_H
+
+#include "llvm/MC/MCStreamer.h"
+
+namespace llvm {
+
+class AArch64TargetStreamer : public MCTargetStreamer {
+public:
+  AArch64TargetStreamer(MCStreamer &S);
+  ~AArch64TargetStreamer() override;
+
+  void finish() override;
+
+  /// Callback used to implement the ldr= pseudo.
+  /// Add a new entry to the constant pool for the current section and return an
+  /// MCExpr that can be used to refer to the constant pool location.
+  const MCExpr *addConstantPoolEntry(const MCExpr *, unsigned Size);
+
+  /// Callback used to implemnt the .ltorg directive.
+  /// Emit contents of constant pool for the current section.
+  void emitCurrentConstantPool();
+
+  /// Callback used to implement the .inst directive.
+  virtual void emitInst(uint32_t Inst);
+
+private:
+  std::unique_ptr<AssemblerConstantPools> ConstantPools;
+};
+
+} // end namespace llvm
+
+#endif
diff --git a/lib/Target/AArch64/Utils/AArch64BaseInfo.cpp b/lib/Target/AArch64/Utils/AArch64BaseInfo.cpp
index 86961632206..ee85b65bf39 100644
--- a/lib/Target/AArch64/Utils/AArch64BaseInfo.cpp
+++ b/lib/Target/AArch64/Utils/AArch64BaseInfo.cpp
@@ -18,8 +18,8 @@
 
 using namespace llvm;
 
-StringRef AArch64NamedImmMapper::toString(uint32_t Value, uint64_t FeatureBits,
-                                          bool &Valid) const {
+StringRef AArch64NamedImmMapper::toString(uint32_t Value,
+          const FeatureBitset& FeatureBits, bool &Valid) const {
   for (unsigned i = 0; i < NumMappings; ++i) {
     if (Mappings[i].isValueEqual(Value, FeatureBits)) {
       Valid = true;
@@ -31,8 +31,8 @@ StringRef AArch64NamedImmMapper::toString(uint32_t Value, uint64_t FeatureBits,
   return StringRef();
 }
 
-uint32_t AArch64NamedImmMapper::fromString(StringRef Name, uint64_t FeatureBits,
-                                           bool &Valid) const {
+uint32_t AArch64NamedImmMapper::fromString(StringRef Name,
+         const FeatureBitset& FeatureBits, bool &Valid) const {
   std::string LowerCaseName = Name.lower();
   for (unsigned i = 0; i < NumMappings; ++i) {
     if (Mappings[i].isNameEqual(LowerCaseName, FeatureBits)) {
@@ -50,204 +50,208 @@ bool AArch64NamedImmMapper::validImm(uint32_t Value) const {
 }
 
 const AArch64NamedImmMapper::Mapping AArch64AT::ATMapper::ATMappings[] = {
-  {"s1e1r", S1E1R, 0},
-  {"s1e2r", S1E2R, 0},
-  {"s1e3r", S1E3R, 0},
-  {"s1e1w", S1E1W, 0},
-  {"s1e2w", S1E2W, 0},
-  {"s1e3w", S1E3W, 0},
-  {"s1e0r", S1E0R, 0},
-  {"s1e0w", S1E0W, 0},
-  {"s12e1r", S12E1R, 0},
-  {"s12e1w", S12E1W, 0},
-  {"s12e0r", S12E0R, 0},
-  {"s12e0w", S12E0W, 0},
+  {"s1e1r", S1E1R, {}},
+  {"s1e2r", S1E2R, {}},
+  {"s1e3r", S1E3R, {}},
+  {"s1e1w", S1E1W, {}},
+  {"s1e2w", S1E2W, {}},
+  {"s1e3w", S1E3W, {}},
+  {"s1e0r", S1E0R, {}},
+  {"s1e0w", S1E0W, {}},
+  {"s12e1r", S12E1R, {}},
+  {"s12e1w", S12E1W, {}},
+  {"s12e0r", S12E0R, {}},
+  {"s12e0w", S12E0W, {}},
 };
 
 AArch64AT::ATMapper::ATMapper()
   : AArch64NamedImmMapper(ATMappings, 0) {}
 
 const AArch64NamedImmMapper::Mapping AArch64DB::DBarrierMapper::DBarrierMappings[] = {
-  {"oshld", OSHLD, 0},
-  {"oshst", OSHST, 0},
-  {"osh", OSH, 0},
-  {"nshld", NSHLD, 0},
-  {"nshst", NSHST, 0},
-  {"nsh", NSH, 0},
-  {"ishld", ISHLD, 0},
-  {"ishst", ISHST, 0},
-  {"ish", ISH, 0},
-  {"ld", LD, 0},
-  {"st", ST, 0},
-  {"sy", SY, 0}
+  {"oshld", OSHLD, {}},
+  {"oshst", OSHST, {}},
+  {"osh", OSH, {}},
+  {"nshld", NSHLD, {}},
+  {"nshst", NSHST, {}},
+  {"nsh", NSH, {}},
+  {"ishld", ISHLD, {}},
+  {"ishst", ISHST, {}},
+  {"ish", ISH, {}},
+  {"ld", LD, {}},
+  {"st", ST, {}},
+  {"sy", SY, {}}
 };
 
 AArch64DB::DBarrierMapper::DBarrierMapper()
   : AArch64NamedImmMapper(DBarrierMappings, 16u) {}
 
 const AArch64NamedImmMapper::Mapping AArch64DC::DCMapper::DCMappings[] = {
-  {"zva", ZVA, 0},
-  {"ivac", IVAC, 0},
-  {"isw", ISW, 0},
-  {"cvac", CVAC, 0},
-  {"csw", CSW, 0},
-  {"cvau", CVAU, 0},
-  {"civac", CIVAC, 0},
-  {"cisw", CISW, 0}
+  {"zva", ZVA, {}},
+  {"ivac", IVAC, {}},
+  {"isw", ISW, {}},
+  {"cvac", CVAC, {}},
+  {"csw", CSW, {}},
+  {"cvau", CVAU, {}},
+  {"civac", CIVAC, {}},
+  {"cisw", CISW, {}}
 };
 
 AArch64DC::DCMapper::DCMapper()
   : AArch64NamedImmMapper(DCMappings, 0) {}
 
 const AArch64NamedImmMapper::Mapping AArch64IC::ICMapper::ICMappings[] = {
-  {"ialluis",  IALLUIS, 0},
-  {"iallu", IALLU, 0},
-  {"ivau", IVAU, 0}
+  {"ialluis",  IALLUIS, {}},
+  {"iallu", IALLU, {}},
+  {"ivau", IVAU, {}}
 };
 
 AArch64IC::ICMapper::ICMapper()
   : AArch64NamedImmMapper(ICMappings, 0) {}
 
 const AArch64NamedImmMapper::Mapping AArch64ISB::ISBMapper::ISBMappings[] = {
-  {"sy",  SY, 0},
+  {"sy",  SY, {}},
 };
 
 AArch64ISB::ISBMapper::ISBMapper()
   : AArch64NamedImmMapper(ISBMappings, 16) {}
 
 const AArch64NamedImmMapper::Mapping AArch64PRFM::PRFMMapper::PRFMMappings[] = {
-  {"pldl1keep", PLDL1KEEP, 0},
-  {"pldl1strm", PLDL1STRM, 0},
-  {"pldl2keep", PLDL2KEEP, 0},
-  {"pldl2strm", PLDL2STRM, 0},
-  {"pldl3keep", PLDL3KEEP, 0},
-  {"pldl3strm", PLDL3STRM, 0},
-  {"plil1keep", PLIL1KEEP, 0},
-  {"plil1strm", PLIL1STRM, 0},
-  {"plil2keep", PLIL2KEEP, 0},
-  {"plil2strm", PLIL2STRM, 0},
-  {"plil3keep", PLIL3KEEP, 0},
-  {"plil3strm", PLIL3STRM, 0},
-  {"pstl1keep", PSTL1KEEP, 0},
-  {"pstl1strm", PSTL1STRM, 0},
-  {"pstl2keep", PSTL2KEEP, 0},
-  {"pstl2strm", PSTL2STRM, 0},
-  {"pstl3keep", PSTL3KEEP, 0},
-  {"pstl3strm", PSTL3STRM, 0}
+  {"pldl1keep", PLDL1KEEP, {}},
+  {"pldl1strm", PLDL1STRM, {}},
+  {"pldl2keep", PLDL2KEEP, {}},
+  {"pldl2strm", PLDL2STRM, {}},
+  {"pldl3keep", PLDL3KEEP, {}},
+  {"pldl3strm", PLDL3STRM, {}},
+  {"plil1keep", PLIL1KEEP, {}},
+  {"plil1strm", PLIL1STRM, {}},
+  {"plil2keep", PLIL2KEEP, {}},
+  {"plil2strm", PLIL2STRM, {}},
+  {"plil3keep", PLIL3KEEP, {}},
+  {"plil3strm", PLIL3STRM, {}},
+  {"pstl1keep", PSTL1KEEP, {}},
+  {"pstl1strm", PSTL1STRM, {}},
+  {"pstl2keep", PSTL2KEEP, {}},
+  {"pstl2strm", PSTL2STRM, {}},
+  {"pstl3keep", PSTL3KEEP, {}},
+  {"pstl3strm", PSTL3STRM, {}}
 };
 
 AArch64PRFM::PRFMMapper::PRFMMapper()
   : AArch64NamedImmMapper(PRFMMappings, 32) {}
 
 const AArch64NamedImmMapper::Mapping AArch64PState::PStateMapper::PStateMappings[] = {
-  {"spsel", SPSel, 0},
-  {"daifset", DAIFSet, 0},
-  {"daifclr", DAIFClr, 0},
+  {"spsel", SPSel, {}},
+  {"daifset", DAIFSet, {}},
+  {"daifclr", DAIFClr, {}},
 
   // v8.1a "Privileged Access Never" extension-specific PStates
-  {"pan", PAN, AArch64::HasV8_1aOps},
+  {"pan", PAN, {AArch64::HasV8_1aOps}},
 };
 
 AArch64PState::PStateMapper::PStateMapper()
   : AArch64NamedImmMapper(PStateMappings, 0) {}
 
 const AArch64NamedImmMapper::Mapping AArch64SysReg::MRSMapper::MRSMappings[] = {
-  {"mdccsr_el0", MDCCSR_EL0, 0},
-  {"dbgdtrrx_el0", DBGDTRRX_EL0, 0},
-  {"mdrar_el1", MDRAR_EL1, 0},
-  {"oslsr_el1", OSLSR_EL1, 0},
-  {"dbgauthstatus_el1", DBGAUTHSTATUS_EL1, 0},
-  {"pmceid0_el0", PMCEID0_EL0, 0},
-  {"pmceid1_el0", PMCEID1_EL0, 0},
-  {"midr_el1", MIDR_EL1, 0},
-  {"ccsidr_el1", CCSIDR_EL1, 0},
-  {"clidr_el1", CLIDR_EL1, 0},
-  {"ctr_el0", CTR_EL0, 0},
-  {"mpidr_el1", MPIDR_EL1, 0},
-  {"revidr_el1", REVIDR_EL1, 0},
-  {"aidr_el1", AIDR_EL1, 0},
-  {"dczid_el0", DCZID_EL0, 0},
-  {"id_pfr0_el1", ID_PFR0_EL1, 0},
-  {"id_pfr1_el1", ID_PFR1_EL1, 0},
-  {"id_dfr0_el1", ID_DFR0_EL1, 0},
-  {"id_afr0_el1", ID_AFR0_EL1, 0},
-  {"id_mmfr0_el1", ID_MMFR0_EL1, 0},
-  {"id_mmfr1_el1", ID_MMFR1_EL1, 0},
-  {"id_mmfr2_el1", ID_MMFR2_EL1, 0},
-  {"id_mmfr3_el1", ID_MMFR3_EL1, 0},
-  {"id_isar0_el1", ID_ISAR0_EL1, 0},
-  {"id_isar1_el1", ID_ISAR1_EL1, 0},
-  {"id_isar2_el1", ID_ISAR2_EL1, 0},
-  {"id_isar3_el1", ID_ISAR3_EL1, 0},
-  {"id_isar4_el1", ID_ISAR4_EL1, 0},
-  {"id_isar5_el1", ID_ISAR5_EL1, 0},
-  {"id_aa64pfr0_el1", ID_A64PFR0_EL1, 0},
-  {"id_aa64pfr1_el1", ID_A64PFR1_EL1, 0},
-  {"id_aa64dfr0_el1", ID_A64DFR0_EL1, 0},
-  {"id_aa64dfr1_el1", ID_A64DFR1_EL1, 0},
-  {"id_aa64afr0_el1", ID_A64AFR0_EL1, 0},
-  {"id_aa64afr1_el1", ID_A64AFR1_EL1, 0},
-  {"id_aa64isar0_el1", ID_A64ISAR0_EL1, 0},
-  {"id_aa64isar1_el1", ID_A64ISAR1_EL1, 0},
-  {"id_aa64mmfr0_el1", ID_A64MMFR0_EL1, 0},
-  {"id_aa64mmfr1_el1", ID_A64MMFR1_EL1, 0},
-  {"mvfr0_el1", MVFR0_EL1, 0},
-  {"mvfr1_el1", MVFR1_EL1, 0},
-  {"mvfr2_el1", MVFR2_EL1, 0},
-  {"rvbar_el1", RVBAR_EL1, 0},
-  {"rvbar_el2", RVBAR_EL2, 0},
-  {"rvbar_el3", RVBAR_EL3, 0},
-  {"isr_el1", ISR_EL1, 0},
-  {"cntpct_el0", CNTPCT_EL0, 0},
-  {"cntvct_el0", CNTVCT_EL0, 0},
+  {"mdccsr_el0", MDCCSR_EL0, {}},
+  {"dbgdtrrx_el0", DBGDTRRX_EL0, {}},
+  {"mdrar_el1", MDRAR_EL1, {}},
+  {"oslsr_el1", OSLSR_EL1, {}},
+  {"dbgauthstatus_el1", DBGAUTHSTATUS_EL1, {}},
+  {"pmceid0_el0", PMCEID0_EL0, {}},
+  {"pmceid1_el0", PMCEID1_EL0, {}},
+  {"midr_el1", MIDR_EL1, {}},
+  {"ccsidr_el1", CCSIDR_EL1, {}},
+  {"clidr_el1", CLIDR_EL1, {}},
+  {"ctr_el0", CTR_EL0, {}},
+  {"mpidr_el1", MPIDR_EL1, {}},
+  {"revidr_el1", REVIDR_EL1, {}},
+  {"aidr_el1", AIDR_EL1, {}},
+  {"dczid_el0", DCZID_EL0, {}},
+  {"id_pfr0_el1", ID_PFR0_EL1, {}},
+  {"id_pfr1_el1", ID_PFR1_EL1, {}},
+  {"id_dfr0_el1", ID_DFR0_EL1, {}},
+  {"id_afr0_el1", ID_AFR0_EL1, {}},
+  {"id_mmfr0_el1", ID_MMFR0_EL1, {}},
+  {"id_mmfr1_el1", ID_MMFR1_EL1, {}},
+  {"id_mmfr2_el1", ID_MMFR2_EL1, {}},
+  {"id_mmfr3_el1", ID_MMFR3_EL1, {}},
+  {"id_mmfr4_el1", ID_MMFR4_EL1, {}},
+  {"id_isar0_el1", ID_ISAR0_EL1, {}},
+  {"id_isar1_el1", ID_ISAR1_EL1, {}},
+  {"id_isar2_el1", ID_ISAR2_EL1, {}},
+  {"id_isar3_el1", ID_ISAR3_EL1, {}},
+  {"id_isar4_el1", ID_ISAR4_EL1, {}},
+  {"id_isar5_el1", ID_ISAR5_EL1, {}},
+  {"id_aa64pfr0_el1", ID_A64PFR0_EL1, {}},
+  {"id_aa64pfr1_el1", ID_A64PFR1_EL1, {}},
+  {"id_aa64dfr0_el1", ID_A64DFR0_EL1, {}},
+  {"id_aa64dfr1_el1", ID_A64DFR1_EL1, {}},
+  {"id_aa64afr0_el1", ID_A64AFR0_EL1, {}},
+  {"id_aa64afr1_el1", ID_A64AFR1_EL1, {}},
+  {"id_aa64isar0_el1", ID_A64ISAR0_EL1, {}},
+  {"id_aa64isar1_el1", ID_A64ISAR1_EL1, {}},
+  {"id_aa64mmfr0_el1", ID_A64MMFR0_EL1, {}},
+  {"id_aa64mmfr1_el1", ID_A64MMFR1_EL1, {}},
+  {"mvfr0_el1", MVFR0_EL1, {}},
+  {"mvfr1_el1", MVFR1_EL1, {}},
+  {"mvfr2_el1", MVFR2_EL1, {}},
+  {"rvbar_el1", RVBAR_EL1, {}},
+  {"rvbar_el2", RVBAR_EL2, {}},
+  {"rvbar_el3", RVBAR_EL3, {}},
+  {"isr_el1", ISR_EL1, {}},
+  {"cntpct_el0", CNTPCT_EL0, {}},
+  {"cntvct_el0", CNTVCT_EL0, {}},
 
   // Trace registers
-  {"trcstatr", TRCSTATR, 0},
-  {"trcidr8", TRCIDR8, 0},
-  {"trcidr9", TRCIDR9, 0},
-  {"trcidr10", TRCIDR10, 0},
-  {"trcidr11", TRCIDR11, 0},
-  {"trcidr12", TRCIDR12, 0},
-  {"trcidr13", TRCIDR13, 0},
-  {"trcidr0", TRCIDR0, 0},
-  {"trcidr1", TRCIDR1, 0},
-  {"trcidr2", TRCIDR2, 0},
-  {"trcidr3", TRCIDR3, 0},
-  {"trcidr4", TRCIDR4, 0},
-  {"trcidr5", TRCIDR5, 0},
-  {"trcidr6", TRCIDR6, 0},
-  {"trcidr7", TRCIDR7, 0},
-  {"trcoslsr", TRCOSLSR, 0},
-  {"trcpdsr", TRCPDSR, 0},
-  {"trcdevaff0", TRCDEVAFF0, 0},
-  {"trcdevaff1", TRCDEVAFF1, 0},
-  {"trclsr", TRCLSR, 0},
-  {"trcauthstatus", TRCAUTHSTATUS, 0},
-  {"trcdevarch", TRCDEVARCH, 0},
-  {"trcdevid", TRCDEVID, 0},
-  {"trcdevtype", TRCDEVTYPE, 0},
-  {"trcpidr4", TRCPIDR4, 0},
-  {"trcpidr5", TRCPIDR5, 0},
-  {"trcpidr6", TRCPIDR6, 0},
-  {"trcpidr7", TRCPIDR7, 0},
-  {"trcpidr0", TRCPIDR0, 0},
-  {"trcpidr1", TRCPIDR1, 0},
-  {"trcpidr2", TRCPIDR2, 0},
-  {"trcpidr3", TRCPIDR3, 0},
-  {"trccidr0", TRCCIDR0, 0},
-  {"trccidr1", TRCCIDR1, 0},
-  {"trccidr2", TRCCIDR2, 0},
-  {"trccidr3", TRCCIDR3, 0},
+  {"trcstatr", TRCSTATR, {}},
+  {"trcidr8", TRCIDR8, {}},
+  {"trcidr9", TRCIDR9, {}},
+  {"trcidr10", TRCIDR10, {}},
+  {"trcidr11", TRCIDR11, {}},
+  {"trcidr12", TRCIDR12, {}},
+  {"trcidr13", TRCIDR13, {}},
+  {"trcidr0", TRCIDR0, {}},
+  {"trcidr1", TRCIDR1, {}},
+  {"trcidr2", TRCIDR2, {}},
+  {"trcidr3", TRCIDR3, {}},
+  {"trcidr4", TRCIDR4, {}},
+  {"trcidr5", TRCIDR5, {}},
+  {"trcidr6", TRCIDR6, {}},
+  {"trcidr7", TRCIDR7, {}},
+  {"trcoslsr", TRCOSLSR, {}},
+  {"trcpdsr", TRCPDSR, {}},
+  {"trcdevaff0", TRCDEVAFF0, {}},
+  {"trcdevaff1", TRCDEVAFF1, {}},
+  {"trclsr", TRCLSR, {}},
+  {"trcauthstatus", TRCAUTHSTATUS, {}},
+  {"trcdevarch", TRCDEVARCH, {}},
+  {"trcdevid", TRCDEVID, {}},
+  {"trcdevtype", TRCDEVTYPE, {}},
+  {"trcpidr4", TRCPIDR4, {}},
+  {"trcpidr5", TRCPIDR5, {}},
+  {"trcpidr6", TRCPIDR6, {}},
+  {"trcpidr7", TRCPIDR7, {}},
+  {"trcpidr0", TRCPIDR0, {}},
+  {"trcpidr1", TRCPIDR1, {}},
+  {"trcpidr2", TRCPIDR2, {}},
+  {"trcpidr3", TRCPIDR3, {}},
+  {"trccidr0", TRCCIDR0, {}},
+  {"trccidr1", TRCCIDR1, {}},
+  {"trccidr2", TRCCIDR2, {}},
+  {"trccidr3", TRCCIDR3, {}},
 
   // GICv3 registers
-  {"icc_iar1_el1", ICC_IAR1_EL1, 0},
-  {"icc_iar0_el1", ICC_IAR0_EL1, 0},
-  {"icc_hppir1_el1", ICC_HPPIR1_EL1, 0},
-  {"icc_hppir0_el1", ICC_HPPIR0_EL1, 0},
-  {"icc_rpr_el1", ICC_RPR_EL1, 0},
-  {"ich_vtr_el2", ICH_VTR_EL2, 0},
-  {"ich_eisr_el2", ICH_EISR_EL2, 0},
-  {"ich_elsr_el2", ICH_ELSR_EL2, 0}
+  {"icc_iar1_el1", ICC_IAR1_EL1, {}},
+  {"icc_iar0_el1", ICC_IAR0_EL1, {}},
+  {"icc_hppir1_el1", ICC_HPPIR1_EL1, {}},
+  {"icc_hppir0_el1", ICC_HPPIR0_EL1, {}},
+  {"icc_rpr_el1", ICC_RPR_EL1, {}},
+  {"ich_vtr_el2", ICH_VTR_EL2, {}},
+  {"ich_eisr_el2", ICH_EISR_EL2, {}},
+  {"ich_elsr_el2", ICH_ELSR_EL2, {}},
+
+  // v8.1a "Limited Ordering Regions" extension-specific system registers
+  {"lorid_el1", LORID_EL1, {AArch64::HasV8_1aOps}},
 };
 
 AArch64SysReg::MRSMapper::MRSMapper() {
@@ -256,24 +260,24 @@ AArch64SysReg::MRSMapper::MRSMapper() {
 }
 
 const AArch64NamedImmMapper::Mapping AArch64SysReg::MSRMapper::MSRMappings[] = {
-  {"dbgdtrtx_el0", DBGDTRTX_EL0, 0},
-  {"oslar_el1", OSLAR_EL1, 0},
-  {"pmswinc_el0", PMSWINC_EL0, 0},
+  {"dbgdtrtx_el0", DBGDTRTX_EL0, {}},
+  {"oslar_el1", OSLAR_EL1, {}},
+  {"pmswinc_el0", PMSWINC_EL0, {}},
 
   // Trace registers
-  {"trcoslar", TRCOSLAR, 0},
-  {"trclar", TRCLAR, 0},
+  {"trcoslar", TRCOSLAR, {}},
+  {"trclar", TRCLAR, {}},
 
   // GICv3 registers
-  {"icc_eoir1_el1", ICC_EOIR1_EL1, 0},
-  {"icc_eoir0_el1", ICC_EOIR0_EL1, 0},
-  {"icc_dir_el1", ICC_DIR_EL1, 0},
-  {"icc_sgi1r_el1", ICC_SGI1R_EL1, 0},
-  {"icc_asgi1r_el1", ICC_ASGI1R_EL1, 0},
-  {"icc_sgi0r_el1", ICC_SGI0R_EL1, 0},
+  {"icc_eoir1_el1", ICC_EOIR1_EL1, {}},
+  {"icc_eoir0_el1", ICC_EOIR0_EL1, {}},
+  {"icc_dir_el1", ICC_DIR_EL1, {}},
+  {"icc_sgi1r_el1", ICC_SGI1R_EL1, {}},
+  {"icc_asgi1r_el1", ICC_ASGI1R_EL1, {}},
+  {"icc_sgi0r_el1", ICC_SGI0R_EL1, {}},
 
   // v8.1a "Privileged Access Never" extension-specific system registers
-  {"pan", PAN, AArch64::HasV8_1aOps},
+  {"pan", PAN, {AArch64::HasV8_1aOps}},
 };
 
 AArch64SysReg::MSRMapper::MSRMapper() {
@@ -283,529 +287,528 @@ AArch64SysReg::MSRMapper::MSRMapper() {
 
 
 const AArch64NamedImmMapper::Mapping AArch64SysReg::SysRegMapper::SysRegMappings[] = {
-  {"osdtrrx_el1", OSDTRRX_EL1, 0},
-  {"osdtrtx_el1",  OSDTRTX_EL1, 0},
-  {"teecr32_el1", TEECR32_EL1, 0},
-  {"mdccint_el1", MDCCINT_EL1, 0},
-  {"mdscr_el1", MDSCR_EL1, 0},
-  {"dbgdtr_el0", DBGDTR_EL0, 0},
-  {"oseccr_el1", OSECCR_EL1, 0},
-  {"dbgvcr32_el2", DBGVCR32_EL2, 0},
-  {"dbgbvr0_el1", DBGBVR0_EL1, 0},
-  {"dbgbvr1_el1", DBGBVR1_EL1, 0},
-  {"dbgbvr2_el1", DBGBVR2_EL1, 0},
-  {"dbgbvr3_el1", DBGBVR3_EL1, 0},
-  {"dbgbvr4_el1", DBGBVR4_EL1, 0},
-  {"dbgbvr5_el1", DBGBVR5_EL1, 0},
-  {"dbgbvr6_el1", DBGBVR6_EL1, 0},
-  {"dbgbvr7_el1", DBGBVR7_EL1, 0},
-  {"dbgbvr8_el1", DBGBVR8_EL1, 0},
-  {"dbgbvr9_el1", DBGBVR9_EL1, 0},
-  {"dbgbvr10_el1", DBGBVR10_EL1, 0},
-  {"dbgbvr11_el1", DBGBVR11_EL1, 0},
-  {"dbgbvr12_el1", DBGBVR12_EL1, 0},
-  {"dbgbvr13_el1", DBGBVR13_EL1, 0},
-  {"dbgbvr14_el1", DBGBVR14_EL1, 0},
-  {"dbgbvr15_el1", DBGBVR15_EL1, 0},
-  {"dbgbcr0_el1", DBGBCR0_EL1, 0},
-  {"dbgbcr1_el1", DBGBCR1_EL1, 0},
-  {"dbgbcr2_el1", DBGBCR2_EL1, 0},
-  {"dbgbcr3_el1", DBGBCR3_EL1, 0},
-  {"dbgbcr4_el1", DBGBCR4_EL1, 0},
-  {"dbgbcr5_el1", DBGBCR5_EL1, 0},
-  {"dbgbcr6_el1", DBGBCR6_EL1, 0},
-  {"dbgbcr7_el1", DBGBCR7_EL1, 0},
-  {"dbgbcr8_el1", DBGBCR8_EL1, 0},
-  {"dbgbcr9_el1", DBGBCR9_EL1, 0},
-  {"dbgbcr10_el1", DBGBCR10_EL1, 0},
-  {"dbgbcr11_el1", DBGBCR11_EL1, 0},
-  {"dbgbcr12_el1", DBGBCR12_EL1, 0},
-  {"dbgbcr13_el1", DBGBCR13_EL1, 0},
-  {"dbgbcr14_el1", DBGBCR14_EL1, 0},
-  {"dbgbcr15_el1", DBGBCR15_EL1, 0},
-  {"dbgwvr0_el1", DBGWVR0_EL1, 0},
-  {"dbgwvr1_el1", DBGWVR1_EL1, 0},
-  {"dbgwvr2_el1", DBGWVR2_EL1, 0},
-  {"dbgwvr3_el1", DBGWVR3_EL1, 0},
-  {"dbgwvr4_el1", DBGWVR4_EL1, 0},
-  {"dbgwvr5_el1", DBGWVR5_EL1, 0},
-  {"dbgwvr6_el1", DBGWVR6_EL1, 0},
-  {"dbgwvr7_el1", DBGWVR7_EL1, 0},
-  {"dbgwvr8_el1", DBGWVR8_EL1, 0},
-  {"dbgwvr9_el1", DBGWVR9_EL1, 0},
-  {"dbgwvr10_el1", DBGWVR10_EL1, 0},
-  {"dbgwvr11_el1", DBGWVR11_EL1, 0},
-  {"dbgwvr12_el1", DBGWVR12_EL1, 0},
-  {"dbgwvr13_el1", DBGWVR13_EL1, 0},
-  {"dbgwvr14_el1", DBGWVR14_EL1, 0},
-  {"dbgwvr15_el1", DBGWVR15_EL1, 0},
-  {"dbgwcr0_el1", DBGWCR0_EL1, 0},
-  {"dbgwcr1_el1", DBGWCR1_EL1, 0},
-  {"dbgwcr2_el1", DBGWCR2_EL1, 0},
-  {"dbgwcr3_el1", DBGWCR3_EL1, 0},
-  {"dbgwcr4_el1", DBGWCR4_EL1, 0},
-  {"dbgwcr5_el1", DBGWCR5_EL1, 0},
-  {"dbgwcr6_el1", DBGWCR6_EL1, 0},
-  {"dbgwcr7_el1", DBGWCR7_EL1, 0},
-  {"dbgwcr8_el1", DBGWCR8_EL1, 0},
-  {"dbgwcr9_el1", DBGWCR9_EL1, 0},
-  {"dbgwcr10_el1", DBGWCR10_EL1, 0},
-  {"dbgwcr11_el1", DBGWCR11_EL1, 0},
-  {"dbgwcr12_el1", DBGWCR12_EL1, 0},
-  {"dbgwcr13_el1", DBGWCR13_EL1, 0},
-  {"dbgwcr14_el1", DBGWCR14_EL1, 0},
-  {"dbgwcr15_el1", DBGWCR15_EL1, 0},
-  {"teehbr32_el1", TEEHBR32_EL1, 0},
-  {"osdlr_el1", OSDLR_EL1, 0},
-  {"dbgprcr_el1", DBGPRCR_EL1, 0},
-  {"dbgclaimset_el1", DBGCLAIMSET_EL1, 0},
-  {"dbgclaimclr_el1", DBGCLAIMCLR_EL1, 0},
-  {"csselr_el1", CSSELR_EL1, 0},
-  {"vpidr_el2", VPIDR_EL2, 0},
-  {"vmpidr_el2", VMPIDR_EL2, 0},
-  {"sctlr_el1", SCTLR_EL1, 0},
-  {"sctlr_el2", SCTLR_EL2, 0},
-  {"sctlr_el3", SCTLR_EL3, 0},
-  {"actlr_el1", ACTLR_EL1, 0},
-  {"actlr_el2", ACTLR_EL2, 0},
-  {"actlr_el3", ACTLR_EL3, 0},
-  {"cpacr_el1", CPACR_EL1, 0},
-  {"hcr_el2", HCR_EL2, 0},
-  {"scr_el3", SCR_EL3, 0},
-  {"mdcr_el2", MDCR_EL2, 0},
-  {"sder32_el3", SDER32_EL3, 0},
-  {"cptr_el2", CPTR_EL2, 0},
-  {"cptr_el3", CPTR_EL3, 0},
-  {"hstr_el2", HSTR_EL2, 0},
-  {"hacr_el2", HACR_EL2, 0},
-  {"mdcr_el3", MDCR_EL3, 0},
-  {"ttbr0_el1", TTBR0_EL1, 0},
-  {"ttbr0_el2", TTBR0_EL2, 0},
-  {"ttbr0_el3", TTBR0_EL3, 0},
-  {"ttbr1_el1", TTBR1_EL1, 0},
-  {"tcr_el1", TCR_EL1, 0},
-  {"tcr_el2", TCR_EL2, 0},
-  {"tcr_el3", TCR_EL3, 0},
-  {"vttbr_el2", VTTBR_EL2, 0},
-  {"vtcr_el2", VTCR_EL2, 0},
-  {"dacr32_el2", DACR32_EL2, 0},
-  {"spsr_el1", SPSR_EL1, 0},
-  {"spsr_el2", SPSR_EL2, 0},
-  {"spsr_el3", SPSR_EL3, 0},
-  {"elr_el1", ELR_EL1, 0},
-  {"elr_el2", ELR_EL2, 0},
-  {"elr_el3", ELR_EL3, 0},
-  {"sp_el0", SP_EL0, 0},
-  {"sp_el1", SP_EL1, 0},
-  {"sp_el2", SP_EL2, 0},
-  {"spsel", SPSel, 0},
-  {"nzcv", NZCV, 0},
-  {"daif", DAIF, 0},
-  {"currentel", CurrentEL, 0},
-  {"spsr_irq", SPSR_irq, 0},
-  {"spsr_abt", SPSR_abt, 0},
-  {"spsr_und", SPSR_und, 0},
-  {"spsr_fiq", SPSR_fiq, 0},
-  {"fpcr", FPCR, 0},
-  {"fpsr", FPSR, 0},
-  {"dspsr_el0", DSPSR_EL0, 0},
-  {"dlr_el0", DLR_EL0, 0},
-  {"ifsr32_el2", IFSR32_EL2, 0},
-  {"afsr0_el1", AFSR0_EL1, 0},
-  {"afsr0_el2", AFSR0_EL2, 0},
-  {"afsr0_el3", AFSR0_EL3, 0},
-  {"afsr1_el1", AFSR1_EL1, 0},
-  {"afsr1_el2", AFSR1_EL2, 0},
-  {"afsr1_el3", AFSR1_EL3, 0},
-  {"esr_el1", ESR_EL1, 0},
-  {"esr_el2", ESR_EL2, 0},
-  {"esr_el3", ESR_EL3, 0},
-  {"fpexc32_el2", FPEXC32_EL2, 0},
-  {"far_el1", FAR_EL1, 0},
-  {"far_el2", FAR_EL2, 0},
-  {"far_el3", FAR_EL3, 0},
-  {"hpfar_el2", HPFAR_EL2, 0},
-  {"par_el1", PAR_EL1, 0},
-  {"pmcr_el0", PMCR_EL0, 0},
-  {"pmcntenset_el0", PMCNTENSET_EL0, 0},
-  {"pmcntenclr_el0", PMCNTENCLR_EL0, 0},
-  {"pmovsclr_el0", PMOVSCLR_EL0, 0},
-  {"pmselr_el0", PMSELR_EL0, 0},
-  {"pmccntr_el0", PMCCNTR_EL0, 0},
-  {"pmxevtyper_el0", PMXEVTYPER_EL0, 0},
-  {"pmxevcntr_el0", PMXEVCNTR_EL0, 0},
-  {"pmuserenr_el0", PMUSERENR_EL0, 0},
-  {"pmintenset_el1", PMINTENSET_EL1, 0},
-  {"pmintenclr_el1", PMINTENCLR_EL1, 0},
-  {"pmovsset_el0", PMOVSSET_EL0, 0},
-  {"mair_el1", MAIR_EL1, 0},
-  {"mair_el2", MAIR_EL2, 0},
-  {"mair_el3", MAIR_EL3, 0},
-  {"amair_el1", AMAIR_EL1, 0},
-  {"amair_el2", AMAIR_EL2, 0},
-  {"amair_el3", AMAIR_EL3, 0},
-  {"vbar_el1", VBAR_EL1, 0},
-  {"vbar_el2", VBAR_EL2, 0},
-  {"vbar_el3", VBAR_EL3, 0},
-  {"rmr_el1", RMR_EL1, 0},
-  {"rmr_el2", RMR_EL2, 0},
-  {"rmr_el3", RMR_EL3, 0},
-  {"contextidr_el1", CONTEXTIDR_EL1, 0},
-  {"tpidr_el0", TPIDR_EL0, 0},
-  {"tpidr_el2", TPIDR_EL2, 0},
-  {"tpidr_el3", TPIDR_EL3, 0},
-  {"tpidrro_el0", TPIDRRO_EL0, 0},
-  {"tpidr_el1", TPIDR_EL1, 0},
-  {"cntfrq_el0", CNTFRQ_EL0, 0},
-  {"cntvoff_el2", CNTVOFF_EL2, 0},
-  {"cntkctl_el1", CNTKCTL_EL1, 0},
-  {"cnthctl_el2", CNTHCTL_EL2, 0},
-  {"cntp_tval_el0", CNTP_TVAL_EL0, 0},
-  {"cnthp_tval_el2", CNTHP_TVAL_EL2, 0},
-  {"cntps_tval_el1", CNTPS_TVAL_EL1, 0},
-  {"cntp_ctl_el0", CNTP_CTL_EL0, 0},
-  {"cnthp_ctl_el2", CNTHP_CTL_EL2, 0},
-  {"cntps_ctl_el1", CNTPS_CTL_EL1, 0},
-  {"cntp_cval_el0", CNTP_CVAL_EL0, 0},
-  {"cnthp_cval_el2", CNTHP_CVAL_EL2, 0},
-  {"cntps_cval_el1", CNTPS_CVAL_EL1, 0},
-  {"cntv_tval_el0", CNTV_TVAL_EL0, 0},
-  {"cntv_ctl_el0", CNTV_CTL_EL0, 0},
-  {"cntv_cval_el0", CNTV_CVAL_EL0, 0},
-  {"pmevcntr0_el0", PMEVCNTR0_EL0, 0},
-  {"pmevcntr1_el0", PMEVCNTR1_EL0, 0},
-  {"pmevcntr2_el0", PMEVCNTR2_EL0, 0},
-  {"pmevcntr3_el0", PMEVCNTR3_EL0, 0},
-  {"pmevcntr4_el0", PMEVCNTR4_EL0, 0},
-  {"pmevcntr5_el0", PMEVCNTR5_EL0, 0},
-  {"pmevcntr6_el0", PMEVCNTR6_EL0, 0},
-  {"pmevcntr7_el0", PMEVCNTR7_EL0, 0},
-  {"pmevcntr8_el0", PMEVCNTR8_EL0, 0},
-  {"pmevcntr9_el0", PMEVCNTR9_EL0, 0},
-  {"pmevcntr10_el0", PMEVCNTR10_EL0, 0},
-  {"pmevcntr11_el0", PMEVCNTR11_EL0, 0},
-  {"pmevcntr12_el0", PMEVCNTR12_EL0, 0},
-  {"pmevcntr13_el0", PMEVCNTR13_EL0, 0},
-  {"pmevcntr14_el0", PMEVCNTR14_EL0, 0},
-  {"pmevcntr15_el0", PMEVCNTR15_EL0, 0},
-  {"pmevcntr16_el0", PMEVCNTR16_EL0, 0},
-  {"pmevcntr17_el0", PMEVCNTR17_EL0, 0},
-  {"pmevcntr18_el0", PMEVCNTR18_EL0, 0},
-  {"pmevcntr19_el0", PMEVCNTR19_EL0, 0},
-  {"pmevcntr20_el0", PMEVCNTR20_EL0, 0},
-  {"pmevcntr21_el0", PMEVCNTR21_EL0, 0},
-  {"pmevcntr22_el0", PMEVCNTR22_EL0, 0},
-  {"pmevcntr23_el0", PMEVCNTR23_EL0, 0},
-  {"pmevcntr24_el0", PMEVCNTR24_EL0, 0},
-  {"pmevcntr25_el0", PMEVCNTR25_EL0, 0},
-  {"pmevcntr26_el0", PMEVCNTR26_EL0, 0},
-  {"pmevcntr27_el0", PMEVCNTR27_EL0, 0},
-  {"pmevcntr28_el0", PMEVCNTR28_EL0, 0},
-  {"pmevcntr29_el0", PMEVCNTR29_EL0, 0},
-  {"pmevcntr30_el0", PMEVCNTR30_EL0, 0},
-  {"pmccfiltr_el0", PMCCFILTR_EL0, 0},
-  {"pmevtyper0_el0", PMEVTYPER0_EL0, 0},
-  {"pmevtyper1_el0", PMEVTYPER1_EL0, 0},
-  {"pmevtyper2_el0", PMEVTYPER2_EL0, 0},
-  {"pmevtyper3_el0", PMEVTYPER3_EL0, 0},
-  {"pmevtyper4_el0", PMEVTYPER4_EL0, 0},
-  {"pmevtyper5_el0", PMEVTYPER5_EL0, 0},
-  {"pmevtyper6_el0", PMEVTYPER6_EL0, 0},
-  {"pmevtyper7_el0", PMEVTYPER7_EL0, 0},
-  {"pmevtyper8_el0", PMEVTYPER8_EL0, 0},
-  {"pmevtyper9_el0", PMEVTYPER9_EL0, 0},
-  {"pmevtyper10_el0", PMEVTYPER10_EL0, 0},
-  {"pmevtyper11_el0", PMEVTYPER11_EL0, 0},
-  {"pmevtyper12_el0", PMEVTYPER12_EL0, 0},
-  {"pmevtyper13_el0", PMEVTYPER13_EL0, 0},
-  {"pmevtyper14_el0", PMEVTYPER14_EL0, 0},
-  {"pmevtyper15_el0", PMEVTYPER15_EL0, 0},
-  {"pmevtyper16_el0", PMEVTYPER16_EL0, 0},
-  {"pmevtyper17_el0", PMEVTYPER17_EL0, 0},
-  {"pmevtyper18_el0", PMEVTYPER18_EL0, 0},
-  {"pmevtyper19_el0", PMEVTYPER19_EL0, 0},
-  {"pmevtyper20_el0", PMEVTYPER20_EL0, 0},
-  {"pmevtyper21_el0", PMEVTYPER21_EL0, 0},
-  {"pmevtyper22_el0", PMEVTYPER22_EL0, 0},
-  {"pmevtyper23_el0", PMEVTYPER23_EL0, 0},
-  {"pmevtyper24_el0", PMEVTYPER24_EL0, 0},
-  {"pmevtyper25_el0", PMEVTYPER25_EL0, 0},
-  {"pmevtyper26_el0", PMEVTYPER26_EL0, 0},
-  {"pmevtyper27_el0", PMEVTYPER27_EL0, 0},
-  {"pmevtyper28_el0", PMEVTYPER28_EL0, 0},
-  {"pmevtyper29_el0", PMEVTYPER29_EL0, 0},
-  {"pmevtyper30_el0", PMEVTYPER30_EL0, 0},
+  {"osdtrrx_el1", OSDTRRX_EL1, {}},
+  {"osdtrtx_el1",  OSDTRTX_EL1, {}},
+  {"teecr32_el1", TEECR32_EL1, {}},
+  {"mdccint_el1", MDCCINT_EL1, {}},
+  {"mdscr_el1", MDSCR_EL1, {}},
+  {"dbgdtr_el0", DBGDTR_EL0, {}},
+  {"oseccr_el1", OSECCR_EL1, {}},
+  {"dbgvcr32_el2", DBGVCR32_EL2, {}},
+  {"dbgbvr0_el1", DBGBVR0_EL1, {}},
+  {"dbgbvr1_el1", DBGBVR1_EL1, {}},
+  {"dbgbvr2_el1", DBGBVR2_EL1, {}},
+  {"dbgbvr3_el1", DBGBVR3_EL1, {}},
+  {"dbgbvr4_el1", DBGBVR4_EL1, {}},
+  {"dbgbvr5_el1", DBGBVR5_EL1, {}},
+  {"dbgbvr6_el1", DBGBVR6_EL1, {}},
+  {"dbgbvr7_el1", DBGBVR7_EL1, {}},
+  {"dbgbvr8_el1", DBGBVR8_EL1, {}},
+  {"dbgbvr9_el1", DBGBVR9_EL1, {}},
+  {"dbgbvr10_el1", DBGBVR10_EL1, {}},
+  {"dbgbvr11_el1", DBGBVR11_EL1, {}},
+  {"dbgbvr12_el1", DBGBVR12_EL1, {}},
+  {"dbgbvr13_el1", DBGBVR13_EL1, {}},
+  {"dbgbvr14_el1", DBGBVR14_EL1, {}},
+  {"dbgbvr15_el1", DBGBVR15_EL1, {}},
+  {"dbgbcr0_el1", DBGBCR0_EL1, {}},
+  {"dbgbcr1_el1", DBGBCR1_EL1, {}},
+  {"dbgbcr2_el1", DBGBCR2_EL1, {}},
+  {"dbgbcr3_el1", DBGBCR3_EL1, {}},
+  {"dbgbcr4_el1", DBGBCR4_EL1, {}},
+  {"dbgbcr5_el1", DBGBCR5_EL1, {}},
+  {"dbgbcr6_el1", DBGBCR6_EL1, {}},
+  {"dbgbcr7_el1", DBGBCR7_EL1, {}},
+  {"dbgbcr8_el1", DBGBCR8_EL1, {}},
+  {"dbgbcr9_el1", DBGBCR9_EL1, {}},
+  {"dbgbcr10_el1", DBGBCR10_EL1, {}},
+  {"dbgbcr11_el1", DBGBCR11_EL1, {}},
+  {"dbgbcr12_el1", DBGBCR12_EL1, {}},
+  {"dbgbcr13_el1", DBGBCR13_EL1, {}},
+  {"dbgbcr14_el1", DBGBCR14_EL1, {}},
+  {"dbgbcr15_el1", DBGBCR15_EL1, {}},
+  {"dbgwvr0_el1", DBGWVR0_EL1, {}},
+  {"dbgwvr1_el1", DBGWVR1_EL1, {}},
+  {"dbgwvr2_el1", DBGWVR2_EL1, {}},
+  {"dbgwvr3_el1", DBGWVR3_EL1, {}},
+  {"dbgwvr4_el1", DBGWVR4_EL1, {}},
+  {"dbgwvr5_el1", DBGWVR5_EL1, {}},
+  {"dbgwvr6_el1", DBGWVR6_EL1, {}},
+  {"dbgwvr7_el1", DBGWVR7_EL1, {}},
+  {"dbgwvr8_el1", DBGWVR8_EL1, {}},
+  {"dbgwvr9_el1", DBGWVR9_EL1, {}},
+  {"dbgwvr10_el1", DBGWVR10_EL1, {}},
+  {"dbgwvr11_el1", DBGWVR11_EL1, {}},
+  {"dbgwvr12_el1", DBGWVR12_EL1, {}},
+  {"dbgwvr13_el1", DBGWVR13_EL1, {}},
+  {"dbgwvr14_el1", DBGWVR14_EL1, {}},
+  {"dbgwvr15_el1", DBGWVR15_EL1, {}},
+  {"dbgwcr0_el1", DBGWCR0_EL1, {}},
+  {"dbgwcr1_el1", DBGWCR1_EL1, {}},
+  {"dbgwcr2_el1", DBGWCR2_EL1, {}},
+  {"dbgwcr3_el1", DBGWCR3_EL1, {}},
+  {"dbgwcr4_el1", DBGWCR4_EL1, {}},
+  {"dbgwcr5_el1", DBGWCR5_EL1, {}},
+  {"dbgwcr6_el1", DBGWCR6_EL1, {}},
+  {"dbgwcr7_el1", DBGWCR7_EL1, {}},
+  {"dbgwcr8_el1", DBGWCR8_EL1, {}},
+  {"dbgwcr9_el1", DBGWCR9_EL1, {}},
+  {"dbgwcr10_el1", DBGWCR10_EL1, {}},
+  {"dbgwcr11_el1", DBGWCR11_EL1, {}},
+  {"dbgwcr12_el1", DBGWCR12_EL1, {}},
+  {"dbgwcr13_el1", DBGWCR13_EL1, {}},
+  {"dbgwcr14_el1", DBGWCR14_EL1, {}},
+  {"dbgwcr15_el1", DBGWCR15_EL1, {}},
+  {"teehbr32_el1", TEEHBR32_EL1, {}},
+  {"osdlr_el1", OSDLR_EL1, {}},
+  {"dbgprcr_el1", DBGPRCR_EL1, {}},
+  {"dbgclaimset_el1", DBGCLAIMSET_EL1, {}},
+  {"dbgclaimclr_el1", DBGCLAIMCLR_EL1, {}},
+  {"csselr_el1", CSSELR_EL1, {}},
+  {"vpidr_el2", VPIDR_EL2, {}},
+  {"vmpidr_el2", VMPIDR_EL2, {}},
+  {"sctlr_el1", SCTLR_EL1, {}},
+  {"sctlr_el2", SCTLR_EL2, {}},
+  {"sctlr_el3", SCTLR_EL3, {}},
+  {"actlr_el1", ACTLR_EL1, {}},
+  {"actlr_el2", ACTLR_EL2, {}},
+  {"actlr_el3", ACTLR_EL3, {}},
+  {"cpacr_el1", CPACR_EL1, {}},
+  {"hcr_el2", HCR_EL2, {}},
+  {"scr_el3", SCR_EL3, {}},
+  {"mdcr_el2", MDCR_EL2, {}},
+  {"sder32_el3", SDER32_EL3, {}},
+  {"cptr_el2", CPTR_EL2, {}},
+  {"cptr_el3", CPTR_EL3, {}},
+  {"hstr_el2", HSTR_EL2, {}},
+  {"hacr_el2", HACR_EL2, {}},
+  {"mdcr_el3", MDCR_EL3, {}},
+  {"ttbr0_el1", TTBR0_EL1, {}},
+  {"ttbr0_el2", TTBR0_EL2, {}},
+  {"ttbr0_el3", TTBR0_EL3, {}},
+  {"ttbr1_el1", TTBR1_EL1, {}},
+  {"tcr_el1", TCR_EL1, {}},
+  {"tcr_el2", TCR_EL2, {}},
+  {"tcr_el3", TCR_EL3, {}},
+  {"vttbr_el2", VTTBR_EL2, {}},
+  {"vtcr_el2", VTCR_EL2, {}},
+  {"dacr32_el2", DACR32_EL2, {}},
+  {"spsr_el1", SPSR_EL1, {}},
+  {"spsr_el2", SPSR_EL2, {}},
+  {"spsr_el3", SPSR_EL3, {}},
+  {"elr_el1", ELR_EL1, {}},
+  {"elr_el2", ELR_EL2, {}},
+  {"elr_el3", ELR_EL3, {}},
+  {"sp_el0", SP_EL0, {}},
+  {"sp_el1", SP_EL1, {}},
+  {"sp_el2", SP_EL2, {}},
+  {"spsel", SPSel, {}},
+  {"nzcv", NZCV, {}},
+  {"daif", DAIF, {}},
+  {"currentel", CurrentEL, {}},
+  {"spsr_irq", SPSR_irq, {}},
+  {"spsr_abt", SPSR_abt, {}},
+  {"spsr_und", SPSR_und, {}},
+  {"spsr_fiq", SPSR_fiq, {}},
+  {"fpcr", FPCR, {}},
+  {"fpsr", FPSR, {}},
+  {"dspsr_el0", DSPSR_EL0, {}},
+  {"dlr_el0", DLR_EL0, {}},
+  {"ifsr32_el2", IFSR32_EL2, {}},
+  {"afsr0_el1", AFSR0_EL1, {}},
+  {"afsr0_el2", AFSR0_EL2, {}},
+  {"afsr0_el3", AFSR0_EL3, {}},
+  {"afsr1_el1", AFSR1_EL1, {}},
+  {"afsr1_el2", AFSR1_EL2, {}},
+  {"afsr1_el3", AFSR1_EL3, {}},
+  {"esr_el1", ESR_EL1, {}},
+  {"esr_el2", ESR_EL2, {}},
+  {"esr_el3", ESR_EL3, {}},
+  {"fpexc32_el2", FPEXC32_EL2, {}},
+  {"far_el1", FAR_EL1, {}},
+  {"far_el2", FAR_EL2, {}},
+  {"far_el3", FAR_EL3, {}},
+  {"hpfar_el2", HPFAR_EL2, {}},
+  {"par_el1", PAR_EL1, {}},
+  {"pmcr_el0", PMCR_EL0, {}},
+  {"pmcntenset_el0", PMCNTENSET_EL0, {}},
+  {"pmcntenclr_el0", PMCNTENCLR_EL0, {}},
+  {"pmovsclr_el0", PMOVSCLR_EL0, {}},
+  {"pmselr_el0", PMSELR_EL0, {}},
+  {"pmccntr_el0", PMCCNTR_EL0, {}},
+  {"pmxevtyper_el0", PMXEVTYPER_EL0, {}},
+  {"pmxevcntr_el0", PMXEVCNTR_EL0, {}},
+  {"pmuserenr_el0", PMUSERENR_EL0, {}},
+  {"pmintenset_el1", PMINTENSET_EL1, {}},
+  {"pmintenclr_el1", PMINTENCLR_EL1, {}},
+  {"pmovsset_el0", PMOVSSET_EL0, {}},
+  {"mair_el1", MAIR_EL1, {}},
+  {"mair_el2", MAIR_EL2, {}},
+  {"mair_el3", MAIR_EL3, {}},
+  {"amair_el1", AMAIR_EL1, {}},
+  {"amair_el2", AMAIR_EL2, {}},
+  {"amair_el3", AMAIR_EL3, {}},
+  {"vbar_el1", VBAR_EL1, {}},
+  {"vbar_el2", VBAR_EL2, {}},
+  {"vbar_el3", VBAR_EL3, {}},
+  {"rmr_el1", RMR_EL1, {}},
+  {"rmr_el2", RMR_EL2, {}},
+  {"rmr_el3", RMR_EL3, {}},
+  {"contextidr_el1", CONTEXTIDR_EL1, {}},
+  {"tpidr_el0", TPIDR_EL0, {}},
+  {"tpidr_el2", TPIDR_EL2, {}},
+  {"tpidr_el3", TPIDR_EL3, {}},
+  {"tpidrro_el0", TPIDRRO_EL0, {}},
+  {"tpidr_el1", TPIDR_EL1, {}},
+  {"cntfrq_el0", CNTFRQ_EL0, {}},
+  {"cntvoff_el2", CNTVOFF_EL2, {}},
+  {"cntkctl_el1", CNTKCTL_EL1, {}},
+  {"cnthctl_el2", CNTHCTL_EL2, {}},
+  {"cntp_tval_el0", CNTP_TVAL_EL0, {}},
+  {"cnthp_tval_el2", CNTHP_TVAL_EL2, {}},
+  {"cntps_tval_el1", CNTPS_TVAL_EL1, {}},
+  {"cntp_ctl_el0", CNTP_CTL_EL0, {}},
+  {"cnthp_ctl_el2", CNTHP_CTL_EL2, {}},
+  {"cntps_ctl_el1", CNTPS_CTL_EL1, {}},
+  {"cntp_cval_el0", CNTP_CVAL_EL0, {}},
+  {"cnthp_cval_el2", CNTHP_CVAL_EL2, {}},
+  {"cntps_cval_el1", CNTPS_CVAL_EL1, {}},
+  {"cntv_tval_el0", CNTV_TVAL_EL0, {}},
+  {"cntv_ctl_el0", CNTV_CTL_EL0, {}},
+  {"cntv_cval_el0", CNTV_CVAL_EL0, {}},
+  {"pmevcntr0_el0", PMEVCNTR0_EL0, {}},
+  {"pmevcntr1_el0", PMEVCNTR1_EL0, {}},
+  {"pmevcntr2_el0", PMEVCNTR2_EL0, {}},
+  {"pmevcntr3_el0", PMEVCNTR3_EL0, {}},
+  {"pmevcntr4_el0", PMEVCNTR4_EL0, {}},
+  {"pmevcntr5_el0", PMEVCNTR5_EL0, {}},
+  {"pmevcntr6_el0", PMEVCNTR6_EL0, {}},
+  {"pmevcntr7_el0", PMEVCNTR7_EL0, {}},
+  {"pmevcntr8_el0", PMEVCNTR8_EL0, {}},
+  {"pmevcntr9_el0", PMEVCNTR9_EL0, {}},
+  {"pmevcntr10_el0", PMEVCNTR10_EL0, {}},
+  {"pmevcntr11_el0", PMEVCNTR11_EL0, {}},
+  {"pmevcntr12_el0", PMEVCNTR12_EL0, {}},
+  {"pmevcntr13_el0", PMEVCNTR13_EL0, {}},
+  {"pmevcntr14_el0", PMEVCNTR14_EL0, {}},
+  {"pmevcntr15_el0", PMEVCNTR15_EL0, {}},
+  {"pmevcntr16_el0", PMEVCNTR16_EL0, {}},
+  {"pmevcntr17_el0", PMEVCNTR17_EL0, {}},
+  {"pmevcntr18_el0", PMEVCNTR18_EL0, {}},
+  {"pmevcntr19_el0", PMEVCNTR19_EL0, {}},
+  {"pmevcntr20_el0", PMEVCNTR20_EL0, {}},
+  {"pmevcntr21_el0", PMEVCNTR21_EL0, {}},
+  {"pmevcntr22_el0", PMEVCNTR22_EL0, {}},
+  {"pmevcntr23_el0", PMEVCNTR23_EL0, {}},
+  {"pmevcntr24_el0", PMEVCNTR24_EL0, {}},
+  {"pmevcntr25_el0", PMEVCNTR25_EL0, {}},
+  {"pmevcntr26_el0", PMEVCNTR26_EL0, {}},
+  {"pmevcntr27_el0", PMEVCNTR27_EL0, {}},
+  {"pmevcntr28_el0", PMEVCNTR28_EL0, {}},
+  {"pmevcntr29_el0", PMEVCNTR29_EL0, {}},
+  {"pmevcntr30_el0", PMEVCNTR30_EL0, {}},
+  {"pmccfiltr_el0", PMCCFILTR_EL0, {}},
+  {"pmevtyper0_el0", PMEVTYPER0_EL0, {}},
+  {"pmevtyper1_el0", PMEVTYPER1_EL0, {}},
+  {"pmevtyper2_el0", PMEVTYPER2_EL0, {}},
+  {"pmevtyper3_el0", PMEVTYPER3_EL0, {}},
+  {"pmevtyper4_el0", PMEVTYPER4_EL0, {}},
+  {"pmevtyper5_el0", PMEVTYPER5_EL0, {}},
+  {"pmevtyper6_el0", PMEVTYPER6_EL0, {}},
+  {"pmevtyper7_el0", PMEVTYPER7_EL0, {}},
+  {"pmevtyper8_el0", PMEVTYPER8_EL0, {}},
+  {"pmevtyper9_el0", PMEVTYPER9_EL0, {}},
+  {"pmevtyper10_el0", PMEVTYPER10_EL0, {}},
+  {"pmevtyper11_el0", PMEVTYPER11_EL0, {}},
+  {"pmevtyper12_el0", PMEVTYPER12_EL0, {}},
+  {"pmevtyper13_el0", PMEVTYPER13_EL0, {}},
+  {"pmevtyper14_el0", PMEVTYPER14_EL0, {}},
+  {"pmevtyper15_el0", PMEVTYPER15_EL0, {}},
+  {"pmevtyper16_el0", PMEVTYPER16_EL0, {}},
+  {"pmevtyper17_el0", PMEVTYPER17_EL0, {}},
+  {"pmevtyper18_el0", PMEVTYPER18_EL0, {}},
+  {"pmevtyper19_el0", PMEVTYPER19_EL0, {}},
+  {"pmevtyper20_el0", PMEVTYPER20_EL0, {}},
+  {"pmevtyper21_el0", PMEVTYPER21_EL0, {}},
+  {"pmevtyper22_el0", PMEVTYPER22_EL0, {}},
+  {"pmevtyper23_el0", PMEVTYPER23_EL0, {}},
+  {"pmevtyper24_el0", PMEVTYPER24_EL0, {}},
+  {"pmevtyper25_el0", PMEVTYPER25_EL0, {}},
+  {"pmevtyper26_el0", PMEVTYPER26_EL0, {}},
+  {"pmevtyper27_el0", PMEVTYPER27_EL0, {}},
+  {"pmevtyper28_el0", PMEVTYPER28_EL0, {}},
+  {"pmevtyper29_el0", PMEVTYPER29_EL0, {}},
+  {"pmevtyper30_el0", PMEVTYPER30_EL0, {}},
 
   // Trace registers
-  {"trcprgctlr", TRCPRGCTLR, 0},
-  {"trcprocselr", TRCPROCSELR, 0},
-  {"trcconfigr", TRCCONFIGR, 0},
-  {"trcauxctlr", TRCAUXCTLR, 0},
-  {"trceventctl0r", TRCEVENTCTL0R, 0},
-  {"trceventctl1r", TRCEVENTCTL1R, 0},
-  {"trcstallctlr", TRCSTALLCTLR, 0},
-  {"trctsctlr", TRCTSCTLR, 0},
-  {"trcsyncpr", TRCSYNCPR, 0},
-  {"trcccctlr", TRCCCCTLR, 0},
-  {"trcbbctlr", TRCBBCTLR, 0},
-  {"trctraceidr", TRCTRACEIDR, 0},
-  {"trcqctlr", TRCQCTLR, 0},
-  {"trcvictlr", TRCVICTLR, 0},
-  {"trcviiectlr", TRCVIIECTLR, 0},
-  {"trcvissctlr", TRCVISSCTLR, 0},
-  {"trcvipcssctlr", TRCVIPCSSCTLR, 0},
-  {"trcvdctlr", TRCVDCTLR, 0},
-  {"trcvdsacctlr", TRCVDSACCTLR, 0},
-  {"trcvdarcctlr", TRCVDARCCTLR, 0},
-  {"trcseqevr0", TRCSEQEVR0, 0},
-  {"trcseqevr1", TRCSEQEVR1, 0},
-  {"trcseqevr2", TRCSEQEVR2, 0},
-  {"trcseqrstevr", TRCSEQRSTEVR, 0},
-  {"trcseqstr", TRCSEQSTR, 0},
-  {"trcextinselr", TRCEXTINSELR, 0},
-  {"trccntrldvr0", TRCCNTRLDVR0, 0},
-  {"trccntrldvr1", TRCCNTRLDVR1, 0},
-  {"trccntrldvr2", TRCCNTRLDVR2, 0},
-  {"trccntrldvr3", TRCCNTRLDVR3, 0},
-  {"trccntctlr0", TRCCNTCTLR0, 0},
-  {"trccntctlr1", TRCCNTCTLR1, 0},
-  {"trccntctlr2", TRCCNTCTLR2, 0},
-  {"trccntctlr3", TRCCNTCTLR3, 0},
-  {"trccntvr0", TRCCNTVR0, 0},
-  {"trccntvr1", TRCCNTVR1, 0},
-  {"trccntvr2", TRCCNTVR2, 0},
-  {"trccntvr3", TRCCNTVR3, 0},
-  {"trcimspec0", TRCIMSPEC0, 0},
-  {"trcimspec1", TRCIMSPEC1, 0},
-  {"trcimspec2", TRCIMSPEC2, 0},
-  {"trcimspec3", TRCIMSPEC3, 0},
-  {"trcimspec4", TRCIMSPEC4, 0},
-  {"trcimspec5", TRCIMSPEC5, 0},
-  {"trcimspec6", TRCIMSPEC6, 0},
-  {"trcimspec7", TRCIMSPEC7, 0},
-  {"trcrsctlr2", TRCRSCTLR2, 0},
-  {"trcrsctlr3", TRCRSCTLR3, 0},
-  {"trcrsctlr4", TRCRSCTLR4, 0},
-  {"trcrsctlr5", TRCRSCTLR5, 0},
-  {"trcrsctlr6", TRCRSCTLR6, 0},
-  {"trcrsctlr7", TRCRSCTLR7, 0},
-  {"trcrsctlr8", TRCRSCTLR8, 0},
-  {"trcrsctlr9", TRCRSCTLR9, 0},
-  {"trcrsctlr10", TRCRSCTLR10, 0},
-  {"trcrsctlr11", TRCRSCTLR11, 0},
-  {"trcrsctlr12", TRCRSCTLR12, 0},
-  {"trcrsctlr13", TRCRSCTLR13, 0},
-  {"trcrsctlr14", TRCRSCTLR14, 0},
-  {"trcrsctlr15", TRCRSCTLR15, 0},
-  {"trcrsctlr16", TRCRSCTLR16, 0},
-  {"trcrsctlr17", TRCRSCTLR17, 0},
-  {"trcrsctlr18", TRCRSCTLR18, 0},
-  {"trcrsctlr19", TRCRSCTLR19, 0},
-  {"trcrsctlr20", TRCRSCTLR20, 0},
-  {"trcrsctlr21", TRCRSCTLR21, 0},
-  {"trcrsctlr22", TRCRSCTLR22, 0},
-  {"trcrsctlr23", TRCRSCTLR23, 0},
-  {"trcrsctlr24", TRCRSCTLR24, 0},
-  {"trcrsctlr25", TRCRSCTLR25, 0},
-  {"trcrsctlr26", TRCRSCTLR26, 0},
-  {"trcrsctlr27", TRCRSCTLR27, 0},
-  {"trcrsctlr28", TRCRSCTLR28, 0},
-  {"trcrsctlr29", TRCRSCTLR29, 0},
-  {"trcrsctlr30", TRCRSCTLR30, 0},
-  {"trcrsctlr31", TRCRSCTLR31, 0},
-  {"trcssccr0", TRCSSCCR0, 0},
-  {"trcssccr1", TRCSSCCR1, 0},
-  {"trcssccr2", TRCSSCCR2, 0},
-  {"trcssccr3", TRCSSCCR3, 0},
-  {"trcssccr4", TRCSSCCR4, 0},
-  {"trcssccr5", TRCSSCCR5, 0},
-  {"trcssccr6", TRCSSCCR6, 0},
-  {"trcssccr7", TRCSSCCR7, 0},
-  {"trcsscsr0", TRCSSCSR0, 0},
-  {"trcsscsr1", TRCSSCSR1, 0},
-  {"trcsscsr2", TRCSSCSR2, 0},
-  {"trcsscsr3", TRCSSCSR3, 0},
-  {"trcsscsr4", TRCSSCSR4, 0},
-  {"trcsscsr5", TRCSSCSR5, 0},
-  {"trcsscsr6", TRCSSCSR6, 0},
-  {"trcsscsr7", TRCSSCSR7, 0},
-  {"trcsspcicr0", TRCSSPCICR0, 0},
-  {"trcsspcicr1", TRCSSPCICR1, 0},
-  {"trcsspcicr2", TRCSSPCICR2, 0},
-  {"trcsspcicr3", TRCSSPCICR3, 0},
-  {"trcsspcicr4", TRCSSPCICR4, 0},
-  {"trcsspcicr5", TRCSSPCICR5, 0},
-  {"trcsspcicr6", TRCSSPCICR6, 0},
-  {"trcsspcicr7", TRCSSPCICR7, 0},
-  {"trcpdcr", TRCPDCR, 0},
-  {"trcacvr0", TRCACVR0, 0},
-  {"trcacvr1", TRCACVR1, 0},
-  {"trcacvr2", TRCACVR2, 0},
-  {"trcacvr3", TRCACVR3, 0},
-  {"trcacvr4", TRCACVR4, 0},
-  {"trcacvr5", TRCACVR5, 0},
-  {"trcacvr6", TRCACVR6, 0},
-  {"trcacvr7", TRCACVR7, 0},
-  {"trcacvr8", TRCACVR8, 0},
-  {"trcacvr9", TRCACVR9, 0},
-  {"trcacvr10", TRCACVR10, 0},
-  {"trcacvr11", TRCACVR11, 0},
-  {"trcacvr12", TRCACVR12, 0},
-  {"trcacvr13", TRCACVR13, 0},
-  {"trcacvr14", TRCACVR14, 0},
-  {"trcacvr15", TRCACVR15, 0},
-  {"trcacatr0", TRCACATR0, 0},
-  {"trcacatr1", TRCACATR1, 0},
-  {"trcacatr2", TRCACATR2, 0},
-  {"trcacatr3", TRCACATR3, 0},
-  {"trcacatr4", TRCACATR4, 0},
-  {"trcacatr5", TRCACATR5, 0},
-  {"trcacatr6", TRCACATR6, 0},
-  {"trcacatr7", TRCACATR7, 0},
-  {"trcacatr8", TRCACATR8, 0},
-  {"trcacatr9", TRCACATR9, 0},
-  {"trcacatr10", TRCACATR10, 0},
-  {"trcacatr11", TRCACATR11, 0},
-  {"trcacatr12", TRCACATR12, 0},
-  {"trcacatr13", TRCACATR13, 0},
-  {"trcacatr14", TRCACATR14, 0},
-  {"trcacatr15", TRCACATR15, 0},
-  {"trcdvcvr0", TRCDVCVR0, 0},
-  {"trcdvcvr1", TRCDVCVR1, 0},
-  {"trcdvcvr2", TRCDVCVR2, 0},
-  {"trcdvcvr3", TRCDVCVR3, 0},
-  {"trcdvcvr4", TRCDVCVR4, 0},
-  {"trcdvcvr5", TRCDVCVR5, 0},
-  {"trcdvcvr6", TRCDVCVR6, 0},
-  {"trcdvcvr7", TRCDVCVR7, 0},
-  {"trcdvcmr0", TRCDVCMR0, 0},
-  {"trcdvcmr1", TRCDVCMR1, 0},
-  {"trcdvcmr2", TRCDVCMR2, 0},
-  {"trcdvcmr3", TRCDVCMR3, 0},
-  {"trcdvcmr4", TRCDVCMR4, 0},
-  {"trcdvcmr5", TRCDVCMR5, 0},
-  {"trcdvcmr6", TRCDVCMR6, 0},
-  {"trcdvcmr7", TRCDVCMR7, 0},
-  {"trccidcvr0", TRCCIDCVR0, 0},
-  {"trccidcvr1", TRCCIDCVR1, 0},
-  {"trccidcvr2", TRCCIDCVR2, 0},
-  {"trccidcvr3", TRCCIDCVR3, 0},
-  {"trccidcvr4", TRCCIDCVR4, 0},
-  {"trccidcvr5", TRCCIDCVR5, 0},
-  {"trccidcvr6", TRCCIDCVR6, 0},
-  {"trccidcvr7", TRCCIDCVR7, 0},
-  {"trcvmidcvr0", TRCVMIDCVR0, 0},
-  {"trcvmidcvr1", TRCVMIDCVR1, 0},
-  {"trcvmidcvr2", TRCVMIDCVR2, 0},
-  {"trcvmidcvr3", TRCVMIDCVR3, 0},
-  {"trcvmidcvr4", TRCVMIDCVR4, 0},
-  {"trcvmidcvr5", TRCVMIDCVR5, 0},
-  {"trcvmidcvr6", TRCVMIDCVR6, 0},
-  {"trcvmidcvr7", TRCVMIDCVR7, 0},
-  {"trccidcctlr0", TRCCIDCCTLR0, 0},
-  {"trccidcctlr1", TRCCIDCCTLR1, 0},
-  {"trcvmidcctlr0", TRCVMIDCCTLR0, 0},
-  {"trcvmidcctlr1", TRCVMIDCCTLR1, 0},
-  {"trcitctrl", TRCITCTRL, 0},
-  {"trcclaimset", TRCCLAIMSET, 0},
-  {"trcclaimclr", TRCCLAIMCLR, 0},
+  {"trcprgctlr", TRCPRGCTLR, {}},
+  {"trcprocselr", TRCPROCSELR, {}},
+  {"trcconfigr", TRCCONFIGR, {}},
+  {"trcauxctlr", TRCAUXCTLR, {}},
+  {"trceventctl0r", TRCEVENTCTL0R, {}},
+  {"trceventctl1r", TRCEVENTCTL1R, {}},
+  {"trcstallctlr", TRCSTALLCTLR, {}},
+  {"trctsctlr", TRCTSCTLR, {}},
+  {"trcsyncpr", TRCSYNCPR, {}},
+  {"trcccctlr", TRCCCCTLR, {}},
+  {"trcbbctlr", TRCBBCTLR, {}},
+  {"trctraceidr", TRCTRACEIDR, {}},
+  {"trcqctlr", TRCQCTLR, {}},
+  {"trcvictlr", TRCVICTLR, {}},
+  {"trcviiectlr", TRCVIIECTLR, {}},
+  {"trcvissctlr", TRCVISSCTLR, {}},
+  {"trcvipcssctlr", TRCVIPCSSCTLR, {}},
+  {"trcvdctlr", TRCVDCTLR, {}},
+  {"trcvdsacctlr", TRCVDSACCTLR, {}},
+  {"trcvdarcctlr", TRCVDARCCTLR, {}},
+  {"trcseqevr0", TRCSEQEVR0, {}},
+  {"trcseqevr1", TRCSEQEVR1, {}},
+  {"trcseqevr2", TRCSEQEVR2, {}},
+  {"trcseqrstevr", TRCSEQRSTEVR, {}},
+  {"trcseqstr", TRCSEQSTR, {}},
+  {"trcextinselr", TRCEXTINSELR, {}},
+  {"trccntrldvr0", TRCCNTRLDVR0, {}},
+  {"trccntrldvr1", TRCCNTRLDVR1, {}},
+  {"trccntrldvr2", TRCCNTRLDVR2, {}},
+  {"trccntrldvr3", TRCCNTRLDVR3, {}},
+  {"trccntctlr0", TRCCNTCTLR0, {}},
+  {"trccntctlr1", TRCCNTCTLR1, {}},
+  {"trccntctlr2", TRCCNTCTLR2, {}},
+  {"trccntctlr3", TRCCNTCTLR3, {}},
+  {"trccntvr0", TRCCNTVR0, {}},
+  {"trccntvr1", TRCCNTVR1, {}},
+  {"trccntvr2", TRCCNTVR2, {}},
+  {"trccntvr3", TRCCNTVR3, {}},
+  {"trcimspec0", TRCIMSPEC0, {}},
+  {"trcimspec1", TRCIMSPEC1, {}},
+  {"trcimspec2", TRCIMSPEC2, {}},
+  {"trcimspec3", TRCIMSPEC3, {}},
+  {"trcimspec4", TRCIMSPEC4, {}},
+  {"trcimspec5", TRCIMSPEC5, {}},
+  {"trcimspec6", TRCIMSPEC6, {}},
+  {"trcimspec7", TRCIMSPEC7, {}},
+  {"trcrsctlr2", TRCRSCTLR2, {}},
+  {"trcrsctlr3", TRCRSCTLR3, {}},
+  {"trcrsctlr4", TRCRSCTLR4, {}},
+  {"trcrsctlr5", TRCRSCTLR5, {}},
+  {"trcrsctlr6", TRCRSCTLR6, {}},
+  {"trcrsctlr7", TRCRSCTLR7, {}},
+  {"trcrsctlr8", TRCRSCTLR8, {}},
+  {"trcrsctlr9", TRCRSCTLR9, {}},
+  {"trcrsctlr10", TRCRSCTLR10, {}},
+  {"trcrsctlr11", TRCRSCTLR11, {}},
+  {"trcrsctlr12", TRCRSCTLR12, {}},
+  {"trcrsctlr13", TRCRSCTLR13, {}},
+  {"trcrsctlr14", TRCRSCTLR14, {}},
+  {"trcrsctlr15", TRCRSCTLR15, {}},
+  {"trcrsctlr16", TRCRSCTLR16, {}},
+  {"trcrsctlr17", TRCRSCTLR17, {}},
+  {"trcrsctlr18", TRCRSCTLR18, {}},
+  {"trcrsctlr19", TRCRSCTLR19, {}},
+  {"trcrsctlr20", TRCRSCTLR20, {}},
+  {"trcrsctlr21", TRCRSCTLR21, {}},
+  {"trcrsctlr22", TRCRSCTLR22, {}},
+  {"trcrsctlr23", TRCRSCTLR23, {}},
+  {"trcrsctlr24", TRCRSCTLR24, {}},
+  {"trcrsctlr25", TRCRSCTLR25, {}},
+  {"trcrsctlr26", TRCRSCTLR26, {}},
+  {"trcrsctlr27", TRCRSCTLR27, {}},
+  {"trcrsctlr28", TRCRSCTLR28, {}},
+  {"trcrsctlr29", TRCRSCTLR29, {}},
+  {"trcrsctlr30", TRCRSCTLR30, {}},
+  {"trcrsctlr31", TRCRSCTLR31, {}},
+  {"trcssccr0", TRCSSCCR0, {}},
+  {"trcssccr1", TRCSSCCR1, {}},
+  {"trcssccr2", TRCSSCCR2, {}},
+  {"trcssccr3", TRCSSCCR3, {}},
+  {"trcssccr4", TRCSSCCR4, {}},
+  {"trcssccr5", TRCSSCCR5, {}},
+  {"trcssccr6", TRCSSCCR6, {}},
+  {"trcssccr7", TRCSSCCR7, {}},
+  {"trcsscsr0", TRCSSCSR0, {}},
+  {"trcsscsr1", TRCSSCSR1, {}},
+  {"trcsscsr2", TRCSSCSR2, {}},
+  {"trcsscsr3", TRCSSCSR3, {}},
+  {"trcsscsr4", TRCSSCSR4, {}},
+  {"trcsscsr5", TRCSSCSR5, {}},
+  {"trcsscsr6", TRCSSCSR6, {}},
+  {"trcsscsr7", TRCSSCSR7, {}},
+  {"trcsspcicr0", TRCSSPCICR0, {}},
+  {"trcsspcicr1", TRCSSPCICR1, {}},
+  {"trcsspcicr2", TRCSSPCICR2, {}},
+  {"trcsspcicr3", TRCSSPCICR3, {}},
+  {"trcsspcicr4", TRCSSPCICR4, {}},
+  {"trcsspcicr5", TRCSSPCICR5, {}},
+  {"trcsspcicr6", TRCSSPCICR6, {}},
+  {"trcsspcicr7", TRCSSPCICR7, {}},
+  {"trcpdcr", TRCPDCR, {}},
+  {"trcacvr0", TRCACVR0, {}},
+  {"trcacvr1", TRCACVR1, {}},
+  {"trcacvr2", TRCACVR2, {}},
+  {"trcacvr3", TRCACVR3, {}},
+  {"trcacvr4", TRCACVR4, {}},
+  {"trcacvr5", TRCACVR5, {}},
+  {"trcacvr6", TRCACVR6, {}},
+  {"trcacvr7", TRCACVR7, {}},
+  {"trcacvr8", TRCACVR8, {}},
+  {"trcacvr9", TRCACVR9, {}},
+  {"trcacvr10", TRCACVR10, {}},
+  {"trcacvr11", TRCACVR11, {}},
+  {"trcacvr12", TRCACVR12, {}},
+  {"trcacvr13", TRCACVR13, {}},
+  {"trcacvr14", TRCACVR14, {}},
+  {"trcacvr15", TRCACVR15, {}},
+  {"trcacatr0", TRCACATR0, {}},
+  {"trcacatr1", TRCACATR1, {}},
+  {"trcacatr2", TRCACATR2, {}},
+  {"trcacatr3", TRCACATR3, {}},
+  {"trcacatr4", TRCACATR4, {}},
+  {"trcacatr5", TRCACATR5, {}},
+  {"trcacatr6", TRCACATR6, {}},
+  {"trcacatr7", TRCACATR7, {}},
+  {"trcacatr8", TRCACATR8, {}},
+  {"trcacatr9", TRCACATR9, {}},
+  {"trcacatr10", TRCACATR10, {}},
+  {"trcacatr11", TRCACATR11, {}},
+  {"trcacatr12", TRCACATR12, {}},
+  {"trcacatr13", TRCACATR13, {}},
+  {"trcacatr14", TRCACATR14, {}},
+  {"trcacatr15", TRCACATR15, {}},
+  {"trcdvcvr0", TRCDVCVR0, {}},
+  {"trcdvcvr1", TRCDVCVR1, {}},
+  {"trcdvcvr2", TRCDVCVR2, {}},
+  {"trcdvcvr3", TRCDVCVR3, {}},
+  {"trcdvcvr4", TRCDVCVR4, {}},
+  {"trcdvcvr5", TRCDVCVR5, {}},
+  {"trcdvcvr6", TRCDVCVR6, {}},
+  {"trcdvcvr7", TRCDVCVR7, {}},
+  {"trcdvcmr0", TRCDVCMR0, {}},
+  {"trcdvcmr1", TRCDVCMR1, {}},
+  {"trcdvcmr2", TRCDVCMR2, {}},
+  {"trcdvcmr3", TRCDVCMR3, {}},
+  {"trcdvcmr4", TRCDVCMR4, {}},
+  {"trcdvcmr5", TRCDVCMR5, {}},
+  {"trcdvcmr6", TRCDVCMR6, {}},
+  {"trcdvcmr7", TRCDVCMR7, {}},
+  {"trccidcvr0", TRCCIDCVR0, {}},
+  {"trccidcvr1", TRCCIDCVR1, {}},
+  {"trccidcvr2", TRCCIDCVR2, {}},
+  {"trccidcvr3", TRCCIDCVR3, {}},
+  {"trccidcvr4", TRCCIDCVR4, {}},
+  {"trccidcvr5", TRCCIDCVR5, {}},
+  {"trccidcvr6", TRCCIDCVR6, {}},
+  {"trccidcvr7", TRCCIDCVR7, {}},
+  {"trcvmidcvr0", TRCVMIDCVR0, {}},
+  {"trcvmidcvr1", TRCVMIDCVR1, {}},
+  {"trcvmidcvr2", TRCVMIDCVR2, {}},
+  {"trcvmidcvr3", TRCVMIDCVR3, {}},
+  {"trcvmidcvr4", TRCVMIDCVR4, {}},
+  {"trcvmidcvr5", TRCVMIDCVR5, {}},
+  {"trcvmidcvr6", TRCVMIDCVR6, {}},
+  {"trcvmidcvr7", TRCVMIDCVR7, {}},
+  {"trccidcctlr0", TRCCIDCCTLR0, {}},
+  {"trccidcctlr1", TRCCIDCCTLR1, {}},
+  {"trcvmidcctlr0", TRCVMIDCCTLR0, {}},
+  {"trcvmidcctlr1", TRCVMIDCCTLR1, {}},
+  {"trcitctrl", TRCITCTRL, {}},
+  {"trcclaimset", TRCCLAIMSET, {}},
+  {"trcclaimclr", TRCCLAIMCLR, {}},
 
   // GICv3 registers
-  {"icc_bpr1_el1", ICC_BPR1_EL1, 0},
-  {"icc_bpr0_el1", ICC_BPR0_EL1, 0},
-  {"icc_pmr_el1", ICC_PMR_EL1, 0},
-  {"icc_ctlr_el1", ICC_CTLR_EL1, 0},
-  {"icc_ctlr_el3", ICC_CTLR_EL3, 0},
-  {"icc_sre_el1", ICC_SRE_EL1, 0},
-  {"icc_sre_el2", ICC_SRE_EL2, 0},
-  {"icc_sre_el3", ICC_SRE_EL3, 0},
-  {"icc_igrpen0_el1", ICC_IGRPEN0_EL1, 0},
-  {"icc_igrpen1_el1", ICC_IGRPEN1_EL1, 0},
-  {"icc_igrpen1_el3", ICC_IGRPEN1_EL3, 0},
-  {"icc_seien_el1", ICC_SEIEN_EL1, 0},
-  {"icc_ap0r0_el1", ICC_AP0R0_EL1, 0},
-  {"icc_ap0r1_el1", ICC_AP0R1_EL1, 0},
-  {"icc_ap0r2_el1", ICC_AP0R2_EL1, 0},
-  {"icc_ap0r3_el1", ICC_AP0R3_EL1, 0},
-  {"icc_ap1r0_el1", ICC_AP1R0_EL1, 0},
-  {"icc_ap1r1_el1", ICC_AP1R1_EL1, 0},
-  {"icc_ap1r2_el1", ICC_AP1R2_EL1, 0},
-  {"icc_ap1r3_el1", ICC_AP1R3_EL1, 0},
-  {"ich_ap0r0_el2", ICH_AP0R0_EL2, 0},
-  {"ich_ap0r1_el2", ICH_AP0R1_EL2, 0},
-  {"ich_ap0r2_el2", ICH_AP0R2_EL2, 0},
-  {"ich_ap0r3_el2", ICH_AP0R3_EL2, 0},
-  {"ich_ap1r0_el2", ICH_AP1R0_EL2, 0},
-  {"ich_ap1r1_el2", ICH_AP1R1_EL2, 0},
-  {"ich_ap1r2_el2", ICH_AP1R2_EL2, 0},
-  {"ich_ap1r3_el2", ICH_AP1R3_EL2, 0},
-  {"ich_hcr_el2", ICH_HCR_EL2, 0},
-  {"ich_misr_el2", ICH_MISR_EL2, 0},
-  {"ich_vmcr_el2", ICH_VMCR_EL2, 0},
-  {"ich_vseir_el2", ICH_VSEIR_EL2, 0},
-  {"ich_lr0_el2", ICH_LR0_EL2, 0},
-  {"ich_lr1_el2", ICH_LR1_EL2, 0},
-  {"ich_lr2_el2", ICH_LR2_EL2, 0},
-  {"ich_lr3_el2", ICH_LR3_EL2, 0},
-  {"ich_lr4_el2", ICH_LR4_EL2, 0},
-  {"ich_lr5_el2", ICH_LR5_EL2, 0},
-  {"ich_lr6_el2", ICH_LR6_EL2, 0},
-  {"ich_lr7_el2", ICH_LR7_EL2, 0},
-  {"ich_lr8_el2", ICH_LR8_EL2, 0},
-  {"ich_lr9_el2", ICH_LR9_EL2, 0},
-  {"ich_lr10_el2", ICH_LR10_EL2, 0},
-  {"ich_lr11_el2", ICH_LR11_EL2, 0},
-  {"ich_lr12_el2", ICH_LR12_EL2, 0},
-  {"ich_lr13_el2", ICH_LR13_EL2, 0},
-  {"ich_lr14_el2", ICH_LR14_EL2, 0},
-  {"ich_lr15_el2", ICH_LR15_EL2, 0},
+  {"icc_bpr1_el1", ICC_BPR1_EL1, {}},
+  {"icc_bpr0_el1", ICC_BPR0_EL1, {}},
+  {"icc_pmr_el1", ICC_PMR_EL1, {}},
+  {"icc_ctlr_el1", ICC_CTLR_EL1, {}},
+  {"icc_ctlr_el3", ICC_CTLR_EL3, {}},
+  {"icc_sre_el1", ICC_SRE_EL1, {}},
+  {"icc_sre_el2", ICC_SRE_EL2, {}},
+  {"icc_sre_el3", ICC_SRE_EL3, {}},
+  {"icc_igrpen0_el1", ICC_IGRPEN0_EL1, {}},
+  {"icc_igrpen1_el1", ICC_IGRPEN1_EL1, {}},
+  {"icc_igrpen1_el3", ICC_IGRPEN1_EL3, {}},
+  {"icc_seien_el1", ICC_SEIEN_EL1, {}},
+  {"icc_ap0r0_el1", ICC_AP0R0_EL1, {}},
+  {"icc_ap0r1_el1", ICC_AP0R1_EL1, {}},
+  {"icc_ap0r2_el1", ICC_AP0R2_EL1, {}},
+  {"icc_ap0r3_el1", ICC_AP0R3_EL1, {}},
+  {"icc_ap1r0_el1", ICC_AP1R0_EL1, {}},
+  {"icc_ap1r1_el1", ICC_AP1R1_EL1, {}},
+  {"icc_ap1r2_el1", ICC_AP1R2_EL1, {}},
+  {"icc_ap1r3_el1", ICC_AP1R3_EL1, {}},
+  {"ich_ap0r0_el2", ICH_AP0R0_EL2, {}},
+  {"ich_ap0r1_el2", ICH_AP0R1_EL2, {}},
+  {"ich_ap0r2_el2", ICH_AP0R2_EL2, {}},
+  {"ich_ap0r3_el2", ICH_AP0R3_EL2, {}},
+  {"ich_ap1r0_el2", ICH_AP1R0_EL2, {}},
+  {"ich_ap1r1_el2", ICH_AP1R1_EL2, {}},
+  {"ich_ap1r2_el2", ICH_AP1R2_EL2, {}},
+  {"ich_ap1r3_el2", ICH_AP1R3_EL2, {}},
+  {"ich_hcr_el2", ICH_HCR_EL2, {}},
+  {"ich_misr_el2", ICH_MISR_EL2, {}},
+  {"ich_vmcr_el2", ICH_VMCR_EL2, {}},
+  {"ich_vseir_el2", ICH_VSEIR_EL2, {}},
+  {"ich_lr0_el2", ICH_LR0_EL2, {}},
+  {"ich_lr1_el2", ICH_LR1_EL2, {}},
+  {"ich_lr2_el2", ICH_LR2_EL2, {}},
+  {"ich_lr3_el2", ICH_LR3_EL2, {}},
+  {"ich_lr4_el2", ICH_LR4_EL2, {}},
+  {"ich_lr5_el2", ICH_LR5_EL2, {}},
+  {"ich_lr6_el2", ICH_LR6_EL2, {}},
+  {"ich_lr7_el2", ICH_LR7_EL2, {}},
+  {"ich_lr8_el2", ICH_LR8_EL2, {}},
+  {"ich_lr9_el2", ICH_LR9_EL2, {}},
+  {"ich_lr10_el2", ICH_LR10_EL2, {}},
+  {"ich_lr11_el2", ICH_LR11_EL2, {}},
+  {"ich_lr12_el2", ICH_LR12_EL2, {}},
+  {"ich_lr13_el2", ICH_LR13_EL2, {}},
+  {"ich_lr14_el2", ICH_LR14_EL2, {}},
+  {"ich_lr15_el2", ICH_LR15_EL2, {}},
 
   // Cyclone registers
-  {"cpm_ioacc_ctl_el3", CPM_IOACC_CTL_EL3, AArch64::ProcCyclone},
+  {"cpm_ioacc_ctl_el3", CPM_IOACC_CTL_EL3, {AArch64::ProcCyclone}},
 
   // v8.1a "Privileged Access Never" extension-specific system registers
-  {"pan", PAN, AArch64::HasV8_1aOps},
+  {"pan", PAN, {AArch64::HasV8_1aOps}},
 
   // v8.1a "Limited Ordering Regions" extension-specific system registers
-  {"lorsa_el1", LORSA_EL1, AArch64::HasV8_1aOps},
-  {"lorea_el1", LOREA_EL1, AArch64::HasV8_1aOps},
-  {"lorn_el1", LORN_EL1, AArch64::HasV8_1aOps},
-  {"lorc_el1", LORC_EL1, AArch64::HasV8_1aOps},
-  {"lorid_el1", LORID_EL1, AArch64::HasV8_1aOps},
+  {"lorsa_el1", LORSA_EL1, {AArch64::HasV8_1aOps}},
+  {"lorea_el1", LOREA_EL1, {AArch64::HasV8_1aOps}},
+  {"lorn_el1", LORN_EL1, {AArch64::HasV8_1aOps}},
+  {"lorc_el1", LORC_EL1, {AArch64::HasV8_1aOps}},
 
   // v8.1a "Virtualization host extensions" system registers
-  {"ttbr1_el2", TTBR1_EL2, AArch64::HasV8_1aOps},
-  {"contextidr_el2", CONTEXTIDR_EL2, AArch64::HasV8_1aOps},
-  {"cnthv_tval_el2", CNTHV_TVAL_EL2, AArch64::HasV8_1aOps},
-  {"cnthv_cval_el2", CNTHV_CVAL_EL2, AArch64::HasV8_1aOps},
-  {"cnthv_ctl_el2", CNTHV_CTL_EL2, AArch64::HasV8_1aOps},
-  {"sctlr_el12", SCTLR_EL12, AArch64::HasV8_1aOps},
-  {"cpacr_el12", CPACR_EL12, AArch64::HasV8_1aOps},
-  {"ttbr0_el12", TTBR0_EL12, AArch64::HasV8_1aOps},
-  {"ttbr1_el12", TTBR1_EL12, AArch64::HasV8_1aOps},
-  {"tcr_el12", TCR_EL12, AArch64::HasV8_1aOps},
-  {"afsr0_el12", AFSR0_EL12, AArch64::HasV8_1aOps},
-  {"afsr1_el12", AFSR1_EL12, AArch64::HasV8_1aOps},
-  {"esr_el12", ESR_EL12, AArch64::HasV8_1aOps},
-  {"far_el12", FAR_EL12, AArch64::HasV8_1aOps},
-  {"mair_el12", MAIR_EL12, AArch64::HasV8_1aOps},
-  {"amair_el12", AMAIR_EL12, AArch64::HasV8_1aOps},
-  {"vbar_el12", VBAR_EL12, AArch64::HasV8_1aOps},
-  {"contextidr_el12", CONTEXTIDR_EL12, AArch64::HasV8_1aOps},
-  {"cntkctl_el12", CNTKCTL_EL12, AArch64::HasV8_1aOps},
-  {"cntp_tval_el02", CNTP_TVAL_EL02, AArch64::HasV8_1aOps},
-  {"cntp_ctl_el02", CNTP_CTL_EL02, AArch64::HasV8_1aOps},
-  {"cntp_cval_el02", CNTP_CVAL_EL02, AArch64::HasV8_1aOps},
-  {"cntv_tval_el02", CNTV_TVAL_EL02, AArch64::HasV8_1aOps},
-  {"cntv_ctl_el02", CNTV_CTL_EL02, AArch64::HasV8_1aOps},
-  {"cntv_cval_el02", CNTV_CVAL_EL02, AArch64::HasV8_1aOps},
-  {"spsr_el12", SPSR_EL12, AArch64::HasV8_1aOps},
-  {"elr_el12", ELR_EL12, AArch64::HasV8_1aOps},
+  {"ttbr1_el2", TTBR1_EL2, {AArch64::HasV8_1aOps}},
+  {"contextidr_el2", CONTEXTIDR_EL2, {AArch64::HasV8_1aOps}},
+  {"cnthv_tval_el2", CNTHV_TVAL_EL2, {AArch64::HasV8_1aOps}},
+  {"cnthv_cval_el2", CNTHV_CVAL_EL2, {AArch64::HasV8_1aOps}},
+  {"cnthv_ctl_el2", CNTHV_CTL_EL2, {AArch64::HasV8_1aOps}},
+  {"sctlr_el12", SCTLR_EL12, {AArch64::HasV8_1aOps}},
+  {"cpacr_el12", CPACR_EL12, {AArch64::HasV8_1aOps}},
+  {"ttbr0_el12", TTBR0_EL12, {AArch64::HasV8_1aOps}},
+  {"ttbr1_el12", TTBR1_EL12, {AArch64::HasV8_1aOps}},
+  {"tcr_el12", TCR_EL12, {AArch64::HasV8_1aOps}},
+  {"afsr0_el12", AFSR0_EL12, {AArch64::HasV8_1aOps}},
+  {"afsr1_el12", AFSR1_EL12, {AArch64::HasV8_1aOps}},
+  {"esr_el12", ESR_EL12, {AArch64::HasV8_1aOps}},
+  {"far_el12", FAR_EL12, {AArch64::HasV8_1aOps}},
+  {"mair_el12", MAIR_EL12, {AArch64::HasV8_1aOps}},
+  {"amair_el12", AMAIR_EL12, {AArch64::HasV8_1aOps}},
+  {"vbar_el12", VBAR_EL12, {AArch64::HasV8_1aOps}},
+  {"contextidr_el12", CONTEXTIDR_EL12, {AArch64::HasV8_1aOps}},
+  {"cntkctl_el12", CNTKCTL_EL12, {AArch64::HasV8_1aOps}},
+  {"cntp_tval_el02", CNTP_TVAL_EL02, {AArch64::HasV8_1aOps}},
+  {"cntp_ctl_el02", CNTP_CTL_EL02, {AArch64::HasV8_1aOps}},
+  {"cntp_cval_el02", CNTP_CVAL_EL02, {AArch64::HasV8_1aOps}},
+  {"cntv_tval_el02", CNTV_TVAL_EL02, {AArch64::HasV8_1aOps}},
+  {"cntv_ctl_el02", CNTV_CTL_EL02, {AArch64::HasV8_1aOps}},
+  {"cntv_cval_el02", CNTV_CVAL_EL02, {AArch64::HasV8_1aOps}},
+  {"spsr_el12", SPSR_EL12, {AArch64::HasV8_1aOps}},
+  {"elr_el12", ELR_EL12, {AArch64::HasV8_1aOps}},
 };
 
 uint32_t
-AArch64SysReg::SysRegMapper::fromString(StringRef Name, uint64_t FeatureBits, 
-                                        bool &Valid) const {
+AArch64SysReg::SysRegMapper::fromString(StringRef Name, 
+    const FeatureBitset& FeatureBits, bool &Valid) const {
   std::string NameLower = Name.lower();
 
   // First search the registers shared by all
@@ -848,7 +851,8 @@ AArch64SysReg::SysRegMapper::fromString(StringRef Name, uint64_t FeatureBits,
 }
 
 std::string
-AArch64SysReg::SysRegMapper::toString(uint32_t Bits, uint64_t FeatureBits) const {
+AArch64SysReg::SysRegMapper::toString(uint32_t Bits, 
+                                      const FeatureBitset& FeatureBits) const {
   // First search the registers shared by all
   for (unsigned i = 0; i < array_lengthof(SysRegMappings); ++i) {
     if (SysRegMappings[i].isValueEqual(Bits, FeatureBits)) {
@@ -876,38 +880,38 @@ AArch64SysReg::SysRegMapper::toString(uint32_t Bits, uint64_t FeatureBits) const
 }
 
 const AArch64NamedImmMapper::Mapping AArch64TLBI::TLBIMapper::TLBIMappings[] = {
-  {"ipas2e1is", IPAS2E1IS, 0},
-  {"ipas2le1is", IPAS2LE1IS, 0},
-  {"vmalle1is", VMALLE1IS, 0},
-  {"alle2is", ALLE2IS, 0},
-  {"alle3is", ALLE3IS, 0},
-  {"vae1is", VAE1IS, 0},
-  {"vae2is", VAE2IS, 0},
-  {"vae3is", VAE3IS, 0},
-  {"aside1is", ASIDE1IS, 0},
-  {"vaae1is", VAAE1IS, 0},
-  {"alle1is", ALLE1IS, 0},
-  {"vale1is", VALE1IS, 0},
-  {"vale2is", VALE2IS, 0},
-  {"vale3is", VALE3IS, 0},
-  {"vmalls12e1is", VMALLS12E1IS, 0},
-  {"vaale1is", VAALE1IS, 0},
-  {"ipas2e1", IPAS2E1, 0},
-  {"ipas2le1", IPAS2LE1, 0},
-  {"vmalle1", VMALLE1, 0},
-  {"alle2", ALLE2, 0},
-  {"alle3", ALLE3, 0},
-  {"vae1", VAE1, 0},
-  {"vae2", VAE2, 0},
-  {"vae3", VAE3, 0},
-  {"aside1", ASIDE1, 0},
-  {"vaae1", VAAE1, 0},
-  {"alle1", ALLE1, 0},
-  {"vale1", VALE1, 0},
-  {"vale2", VALE2, 0},
-  {"vale3", VALE3, 0},
-  {"vmalls12e1", VMALLS12E1, 0},
-  {"vaale1", VAALE1, 0}
+  {"ipas2e1is", IPAS2E1IS, {}},
+  {"ipas2le1is", IPAS2LE1IS, {}},
+  {"vmalle1is", VMALLE1IS, {}},
+  {"alle2is", ALLE2IS, {}},
+  {"alle3is", ALLE3IS, {}},
+  {"vae1is", VAE1IS, {}},
+  {"vae2is", VAE2IS, {}},
+  {"vae3is", VAE3IS, {}},
+  {"aside1is", ASIDE1IS, {}},
+  {"vaae1is", VAAE1IS, {}},
+  {"alle1is", ALLE1IS, {}},
+  {"vale1is", VALE1IS, {}},
+  {"vale2is", VALE2IS, {}},
+  {"vale3is", VALE3IS, {}},
+  {"vmalls12e1is", VMALLS12E1IS, {}},
+  {"vaale1is", VAALE1IS, {}},
+  {"ipas2e1", IPAS2E1, {}},
+  {"ipas2le1", IPAS2LE1, {}},
+  {"vmalle1", VMALLE1, {}},
+  {"alle2", ALLE2, {}},
+  {"alle3", ALLE3, {}},
+  {"vae1", VAE1, {}},
+  {"vae2", VAE2, {}},
+  {"vae3", VAE3, {}},
+  {"aside1", ASIDE1, {}},
+  {"vaae1", VAAE1, {}},
+  {"alle1", ALLE1, {}},
+  {"vale1", VALE1, {}},
+  {"vale2", VALE2, {}},
+  {"vale3", VALE3, {}},
+  {"vmalls12e1", VMALLS12E1, {}},
+  {"vaale1", VAALE1, {}}
 };
 
 AArch64TLBI::TLBIMapper::TLBIMapper()
diff --git a/lib/Target/AArch64/Utils/AArch64BaseInfo.h b/lib/Target/AArch64/Utils/AArch64BaseInfo.h
index 659ea909d7d..7e42f8e3601 100644
--- a/lib/Target/AArch64/Utils/AArch64BaseInfo.h
+++ b/lib/Target/AArch64/Utils/AArch64BaseInfo.h
@@ -22,6 +22,7 @@
 #include "MCTargetDesc/AArch64MCTargetDesc.h" // For AArch64::X0 and friends.
 #include "llvm/ADT/STLExtras.h"
 #include "llvm/ADT/StringSwitch.h"
+#include "llvm/MC/SubtargetFeature.h"
 #include "llvm/Support/ErrorHandling.h"
 
 namespace llvm {
@@ -280,15 +281,22 @@ struct AArch64NamedImmMapper {
   struct Mapping {
     const char *Name;
     uint32_t Value;
-    uint64_t AvailableForFeatures;
-    // empty AvailableForFeatures means "always-on"
-    bool isNameEqual(std::string Other, uint64_t FeatureBits=~0ULL) const {
-      if (AvailableForFeatures && !(AvailableForFeatures & FeatureBits))
+    // Set of features this mapping is available for
+    // Zero value of FeatureBitSet means the mapping is always available
+    FeatureBitset FeatureBitSet;
+
+    bool isNameEqual(std::string Other, 
+                     const FeatureBitset& FeatureBits) const {
+      if (FeatureBitSet.any() && 
+          (FeatureBitSet & FeatureBits).none())
         return false;
       return Name == Other;
     }
-    bool isValueEqual(uint32_t Other, uint64_t FeatureBits=~0ULL) const {
-      if (AvailableForFeatures && !(AvailableForFeatures & FeatureBits))
+
+    bool isValueEqual(uint32_t Other, 
+                      const FeatureBitset& FeatureBits) const {
+      if (FeatureBitSet.any() && 
+          (FeatureBitSet & FeatureBits).none())
         return false;
       return Value == Other;
     }
@@ -298,8 +306,12 @@ struct AArch64NamedImmMapper {
   AArch64NamedImmMapper(const Mapping (&Mappings)[N], uint32_t TooBigImm)
     : Mappings(&Mappings[0]), NumMappings(N), TooBigImm(TooBigImm) {}
 
-  StringRef toString(uint32_t Value, uint64_t FeatureBits, bool &Valid) const;
-  uint32_t fromString(StringRef Name, uint64_t FeatureBits, bool &Valid) const;
+  // Maps value to string, depending on availability for FeatureBits given
+  StringRef toString(uint32_t Value, const FeatureBitset& FeatureBits,
+                     bool &Valid) const;
+  // Maps string to value, depending on availability for FeatureBits given
+  uint32_t fromString(StringRef Name, const FeatureBitset& FeatureBits, 
+                     bool &Valid) const;
 
   /// Many of the instructions allow an alternative assembly form consisting of
   /// a simple immediate. Currently the only valid forms are ranges [0, N) where
@@ -591,6 +603,7 @@ namespace AArch64SysReg {
     ISR_EL1           = 0xc608, // 11  000  1100  0001  000
     CNTPCT_EL0        = 0xdf01, // 11  011  1110  0000  001
     CNTVCT_EL0        = 0xdf02,  // 11  011  1110  0000  010
+    ID_MMFR4_EL1      = 0xc016,  // 11  000  0000  0010  110
 
     // Trace registers
     TRCSTATR          = 0x8818, // 10  001  0000  0011  000
@@ -1192,8 +1205,9 @@ namespace AArch64SysReg {
     size_t NumInstMappings;
 
     SysRegMapper() { }
-    uint32_t fromString(StringRef Name, uint64_t FeatureBits, bool &Valid) const;
-    std::string toString(uint32_t Bits, uint64_t FeatureBits) const;
+    uint32_t fromString(StringRef Name, const FeatureBitset& FeatureBits,
+                        bool &Valid) const;
+    std::string toString(uint32_t Bits, const FeatureBitset& FeatureBits) const;
   };
 
   struct MSRMapper : SysRegMapper {
diff --git a/lib/Target/R600/AMDGPU.h b/lib/Target/AMDGPU/AMDGPU.h
similarity index 93%
rename from lib/Target/R600/AMDGPU.h
rename to lib/Target/AMDGPU/AMDGPU.h
index fb87cc5c4f2..0a05d25189b 100644
--- a/lib/Target/R600/AMDGPU.h
+++ b/lib/Target/AMDGPU/AMDGPU.h
@@ -43,6 +43,7 @@ FunctionPass *createSILowerI1CopiesPass();
 FunctionPass *createSIShrinkInstructionsPass();
 FunctionPass *createSILoadStoreOptimizerPass(TargetMachine &tm);
 FunctionPass *createSILowerControlFlowPass(TargetMachine &tm);
+FunctionPass *createSIFixControlFlowLiveIntervalsPass();
 FunctionPass *createSIFixSGPRCopiesPass(TargetMachine &tm);
 FunctionPass *createSIFixSGPRLiveRangesPass();
 FunctionPass *createSICodeEmitterPass(formatted_raw_ostream &OS);
@@ -64,6 +65,9 @@ Pass *createAMDGPUStructurizeCFGPass();
 FunctionPass *createAMDGPUISelDag(TargetMachine &tm);
 ModulePass *createAMDGPUAlwaysInlinePass();
 
+void initializeSIFixControlFlowLiveIntervalsPass(PassRegistry&);
+extern char &SIFixControlFlowLiveIntervalsID;
+
 void initializeSIFixSGPRLiveRangesPass(PassRegistry&);
 extern char &SIFixSGPRLiveRangesID;
 
@@ -101,7 +105,7 @@ namespace ShaderType {
 /// a separate piece of memory that is unique from other
 /// memory locations.
 namespace AMDGPUAS {
-enum AddressSpaces {
+enum AddressSpaces : unsigned {
   PRIVATE_ADDRESS  = 0, ///< Address space for private memory.
   GLOBAL_ADDRESS   = 1, ///< Address space for global memory (RAT0, VTX0).
   CONSTANT_ADDRESS = 2, ///< Address space for constant memory
@@ -133,7 +137,10 @@ enum AddressSpaces {
   CONSTANT_BUFFER_14 = 22,
   CONSTANT_BUFFER_15 = 23,
   ADDRESS_NONE = 24, ///< Address space for unknown memory.
-  LAST_ADDRESS = ADDRESS_NONE
+  LAST_ADDRESS = ADDRESS_NONE,
+
+  // Some places use this if the address space can't be determined.
+  UNKNOWN_ADDRESS_SPACE = ~0u
 };
 
 } // namespace AMDGPUAS
diff --git a/lib/Target/AMDGPU/AMDGPU.td b/lib/Target/AMDGPU/AMDGPU.td
new file mode 100644
index 00000000000..68b50504ee4
--- /dev/null
+++ b/lib/Target/AMDGPU/AMDGPU.td
@@ -0,0 +1,294 @@
+//===-- AMDGPU.td - AMDGPU Tablegen files ------------------*- tablegen -*-===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+
+include "llvm/Target/Target.td"
+
+//===----------------------------------------------------------------------===//
+// Subtarget Features
+//===----------------------------------------------------------------------===//
+
+// Debugging Features
+
+def FeatureDumpCode : SubtargetFeature <"DumpCode",
+        "DumpCode",
+        "true",
+        "Dump MachineInstrs in the CodeEmitter">;
+
+def FeatureDumpCodeLower : SubtargetFeature <"dumpcode",
+        "DumpCode",
+        "true",
+        "Dump MachineInstrs in the CodeEmitter">;
+
+def FeatureIRStructurizer : SubtargetFeature <"disable-irstructurizer",
+        "EnableIRStructurizer",
+        "false",
+        "Disable IR Structurizer">;
+
+def FeaturePromoteAlloca : SubtargetFeature <"promote-alloca",
+        "EnablePromoteAlloca",
+        "true",
+        "Enable promote alloca pass">;
+
+// Target features
+
+def FeatureIfCvt : SubtargetFeature <"disable-ifcvt",
+        "EnableIfCvt",
+        "false",
+        "Disable the if conversion pass">;
+
+def FeatureFP64 : SubtargetFeature<"fp64",
+        "FP64",
+        "true",
+        "Enable double precision operations">;
+
+def FeatureFP64Denormals : SubtargetFeature<"fp64-denormals",
+        "FP64Denormals",
+        "true",
+        "Enable double precision denormal handling",
+        [FeatureFP64]>;
+
+def FeatureFastFMAF32 : SubtargetFeature<"fast-fmaf",
+        "FastFMAF32",
+        "true",
+        "Assuming f32 fma is at least as fast as mul + add",
+        []>;
+
+// Some instructions do not support denormals despite this flag. Using
+// fp32 denormals also causes instructions to run at the double
+// precision rate for the device.
+def FeatureFP32Denormals : SubtargetFeature<"fp32-denormals",
+        "FP32Denormals",
+        "true",
+        "Enable single precision denormal handling">;
+
+def Feature64BitPtr : SubtargetFeature<"64BitPtr",
+        "Is64bit",
+        "true",
+        "Specify if 64-bit addressing should be used">;
+
+def FeatureR600ALUInst : SubtargetFeature<"R600ALUInst",
+        "R600ALUInst",
+        "false",
+        "Older version of ALU instructions encoding">;
+
+def FeatureVertexCache : SubtargetFeature<"HasVertexCache",
+        "HasVertexCache",
+        "true",
+        "Specify use of dedicated vertex cache">;
+
+def FeatureCaymanISA : SubtargetFeature<"caymanISA",
+        "CaymanISA",
+        "true",
+        "Use Cayman ISA">;
+
+def FeatureCFALUBug : SubtargetFeature<"cfalubug",
+        "CFALUBug",
+        "true",
+        "GPU has CF_ALU bug">;
+
+// XXX - This should probably be removed once enabled by default
+def FeatureEnableLoadStoreOpt : SubtargetFeature <"load-store-opt",
+        "EnableLoadStoreOpt",
+        "true",
+        "Enable SI load/store optimizer pass">;
+
+// Performance debugging feature. Allow using DS instruction immediate
+// offsets even if the base pointer can't be proven to be base. On SI,
+// base pointer values that won't give the same result as a 16-bit add
+// are not safe to fold, but this will override the conservative test
+// for the base pointer.
+def FeatureEnableUnsafeDSOffsetFolding : SubtargetFeature <"unsafe-ds-offset-folding",
+        "EnableUnsafeDSOffsetFolding",
+        "true",
+        "Force using DS instruction immediate offsets on SI">;
+
+def FeatureFlatAddressSpace : SubtargetFeature<"flat-address-space",
+        "FlatAddressSpace",
+        "true",
+        "Support flat address space">;
+
+def FeatureVGPRSpilling : SubtargetFeature<"vgpr-spilling",
+        "EnableVGPRSpilling",
+        "true",
+        "Enable spilling of VGPRs to scratch memory">;
+
+def FeatureSGPRInitBug : SubtargetFeature<"sgpr-init-bug",
+        "SGPRInitBug",
+        "true",
+        "VI SGPR initilization bug requiring a fixed SGPR allocation size">;
+
+def FeatureEnableHugeScratchBuffer : SubtargetFeature<"huge-scratch-buffer",
+        "EnableHugeScratchBuffer",
+        "true",
+        "Enable scratch buffer sizes greater than 128 GB">;
+
+class SubtargetFeatureFetchLimit <string Value> :
+                          SubtargetFeature <"fetch"#Value,
+        "TexVTXClauseSize",
+        Value,
+        "Limit the maximum number of fetches in a clause to "#Value>;
+
+def FeatureFetchLimit8 : SubtargetFeatureFetchLimit <"8">;
+def FeatureFetchLimit16 : SubtargetFeatureFetchLimit <"16">;
+
+class SubtargetFeatureWavefrontSize <int Value> : SubtargetFeature<
+        "wavefrontsize"#Value,
+        "WavefrontSize",
+        !cast<string>(Value),
+        "The number of threads per wavefront">;
+
+def FeatureWavefrontSize16 : SubtargetFeatureWavefrontSize<16>;
+def FeatureWavefrontSize32 : SubtargetFeatureWavefrontSize<32>;
+def FeatureWavefrontSize64 : SubtargetFeatureWavefrontSize<64>;
+
+class SubtargetFeatureLDSBankCount <int Value> : SubtargetFeature <
+      "ldsbankcount"#Value,
+      "LDSBankCount",
+      !cast<string>(Value),
+      "The number of LDS banks per compute unit.">;
+
+def FeatureLDSBankCount16 : SubtargetFeatureLDSBankCount<16>;
+def FeatureLDSBankCount32 : SubtargetFeatureLDSBankCount<32>;
+
+class SubtargetFeatureISAVersion <int Major, int Minor, int Stepping>
+                                 : SubtargetFeature <
+      "isaver"#Major#"."#Minor#"."#Stepping,
+      "IsaVersion",
+      "ISAVersion"#Major#"_"#Minor#"_"#Stepping,
+      "Instruction set version number"
+>;
+
+def FeatureISAVersion7_0_0 : SubtargetFeatureISAVersion <7,0,0>;
+def FeatureISAVersion7_0_1 : SubtargetFeatureISAVersion <7,0,1>;
+def FeatureISAVersion8_0_0 : SubtargetFeatureISAVersion <8,0,0>;
+def FeatureISAVersion8_0_1 : SubtargetFeatureISAVersion <8,0,1>;
+
+class SubtargetFeatureLocalMemorySize <int Value> : SubtargetFeature<
+        "localmemorysize"#Value,
+        "LocalMemorySize",
+        !cast<string>(Value),
+        "The size of local memory in bytes">;
+
+def FeatureGCN : SubtargetFeature<"gcn",
+        "IsGCN",
+        "true",
+        "GCN or newer GPU">;
+
+def FeatureGCN1Encoding : SubtargetFeature<"gcn1-encoding",
+        "GCN1Encoding",
+        "true",
+        "Encoding format for SI and CI">;
+
+def FeatureGCN3Encoding : SubtargetFeature<"gcn3-encoding",
+        "GCN3Encoding",
+        "true",
+        "Encoding format for VI">;
+
+def FeatureCIInsts : SubtargetFeature<"ci-insts",
+        "CIInsts",
+        "true",
+        "Additional intstructions for CI+">;
+
+// Dummy feature used to disable assembler instructions.
+def FeatureDisable : SubtargetFeature<"",
+                                      "FeatureDisable","true",
+                                      "Dummy feature to disable assembler"
+                                      " instructions">;
+
+class SubtargetFeatureGeneration <string Value,
+                                  list<SubtargetFeature> Implies> :
+        SubtargetFeature <Value, "Gen", "AMDGPUSubtarget::"#Value,
+                          Value#" GPU generation", Implies>;
+
+def FeatureLocalMemorySize0 : SubtargetFeatureLocalMemorySize<0>;
+def FeatureLocalMemorySize32768 : SubtargetFeatureLocalMemorySize<32768>;
+def FeatureLocalMemorySize65536 : SubtargetFeatureLocalMemorySize<65536>;
+
+def FeatureR600 : SubtargetFeatureGeneration<"R600",
+        [FeatureR600ALUInst, FeatureFetchLimit8, FeatureLocalMemorySize0]>;
+
+def FeatureR700 : SubtargetFeatureGeneration<"R700",
+        [FeatureFetchLimit16, FeatureLocalMemorySize0]>;
+
+def FeatureEvergreen : SubtargetFeatureGeneration<"EVERGREEN",
+        [FeatureFetchLimit16, FeatureLocalMemorySize32768]>;
+
+def FeatureNorthernIslands : SubtargetFeatureGeneration<"NORTHERN_ISLANDS",
+        [FeatureFetchLimit16, FeatureWavefrontSize64,
+         FeatureLocalMemorySize32768]
+>;
+
+def FeatureSouthernIslands : SubtargetFeatureGeneration<"SOUTHERN_ISLANDS",
+        [Feature64BitPtr, FeatureFP64, FeatureLocalMemorySize32768,
+         FeatureWavefrontSize64, FeatureGCN, FeatureGCN1Encoding,
+         FeatureLDSBankCount32]>;
+
+def FeatureSeaIslands : SubtargetFeatureGeneration<"SEA_ISLANDS",
+        [Feature64BitPtr, FeatureFP64, FeatureLocalMemorySize65536,
+         FeatureWavefrontSize64, FeatureGCN, FeatureFlatAddressSpace,
+         FeatureGCN1Encoding, FeatureCIInsts]>;
+
+def FeatureVolcanicIslands : SubtargetFeatureGeneration<"VOLCANIC_ISLANDS",
+        [Feature64BitPtr, FeatureFP64, FeatureLocalMemorySize65536,
+         FeatureWavefrontSize64, FeatureFlatAddressSpace, FeatureGCN,
+         FeatureGCN3Encoding, FeatureCIInsts, FeatureLDSBankCount32]>;
+
+//===----------------------------------------------------------------------===//
+
+def AMDGPUInstrInfo : InstrInfo {
+  let guessInstructionProperties = 1;
+  let noNamedPositionallyEncodedOperands = 1;
+}
+
+def AMDGPUAsmParser : AsmParser {
+  // Some of the R600 registers have the same name, so this crashes.
+  // For example T0_XYZW and T0_XY both have the asm name T0.
+  let ShouldEmitMatchRegisterName = 0;
+}
+
+def AMDGPU : Target {
+  // Pull in Instruction Info:
+  let InstructionSet = AMDGPUInstrInfo;
+  let AssemblyParsers = [AMDGPUAsmParser];
+}
+
+// Dummy Instruction itineraries for pseudo instructions
+def ALU_NULL : FuncUnit;
+def NullALU : InstrItinClass;
+
+//===----------------------------------------------------------------------===//
+// Predicate helper class
+//===----------------------------------------------------------------------===//
+
+def TruePredicate : Predicate<"true">;
+def isSICI : Predicate<
+  "Subtarget->getGeneration() == AMDGPUSubtarget::SOUTHERN_ISLANDS ||"
+  "Subtarget->getGeneration() == AMDGPUSubtarget::SEA_ISLANDS"
+>, AssemblerPredicate<"FeatureGCN1Encoding">;
+
+class PredicateControl {
+  Predicate SubtargetPredicate;
+  Predicate SIAssemblerPredicate = isSICI;
+  list<Predicate> AssemblerPredicates = [];
+  Predicate AssemblerPredicate = TruePredicate;
+  list<Predicate> OtherPredicates = [];
+  list<Predicate> Predicates = !listconcat([SubtargetPredicate, AssemblerPredicate],
+                                            AssemblerPredicates,
+                                            OtherPredicates);
+}
+
+// Include AMDGPU TD files
+include "R600Schedule.td"
+include "SISchedule.td"
+include "Processors.td"
+include "AMDGPUInstrInfo.td"
+include "AMDGPUIntrinsics.td"
+include "AMDGPURegisterInfo.td"
+include "AMDGPUInstructions.td"
+include "AMDGPUCallingConv.td"
diff --git a/lib/Target/R600/AMDGPUAlwaysInlinePass.cpp b/lib/Target/AMDGPU/AMDGPUAlwaysInlinePass.cpp
similarity index 83%
rename from lib/Target/R600/AMDGPUAlwaysInlinePass.cpp
rename to lib/Target/AMDGPU/AMDGPUAlwaysInlinePass.cpp
index b545b456161..ad267d35085 100644
--- a/lib/Target/R600/AMDGPUAlwaysInlinePass.cpp
+++ b/lib/Target/AMDGPU/AMDGPUAlwaysInlinePass.cpp
@@ -22,7 +22,6 @@ using namespace llvm;
 namespace {
 
 class AMDGPUAlwaysInline : public ModulePass {
-
   static char ID;
 
 public:
@@ -36,11 +35,11 @@ class AMDGPUAlwaysInline : public ModulePass {
 char AMDGPUAlwaysInline::ID = 0;
 
 bool AMDGPUAlwaysInline::runOnModule(Module &M) {
+  std::vector<Function *> FuncsToClone;
 
-  std::vector<Function*> FuncsToClone;
-  for (Module::iterator I = M.begin(), E = M.end(); I != E; ++I) {
-    Function &F = *I;
-    if (!F.hasLocalLinkage() && !F.isDeclaration() && !F.use_empty())
+  for (Function &F : M) {
+    if (!F.hasLocalLinkage() && !F.isDeclaration() && !F.use_empty() &&
+        !F.hasFnAttribute(Attribute::NoInline))
       FuncsToClone.push_back(&F);
   }
 
@@ -48,13 +47,12 @@ bool AMDGPUAlwaysInline::runOnModule(Module &M) {
     ValueToValueMapTy VMap;
     Function *NewFunc = CloneFunction(F, VMap, false);
     NewFunc->setLinkage(GlobalValue::InternalLinkage);
-    F->getParent()->getFunctionList().push_back(NewFunc);
+    M.getFunctionList().push_back(NewFunc);
     F->replaceAllUsesWith(NewFunc);
   }
 
-  for (Module::iterator I = M.begin(), E = M.end(); I != E; ++I) {
-    Function &F = *I;
-    if (F.hasLocalLinkage()) {
+  for (Function &F : M) {
+    if (F.hasLocalLinkage() && !F.hasFnAttribute(Attribute::NoInline)) {
       F.addFnAttr(Attribute::AlwaysInline);
     }
   }
diff --git a/lib/Target/AMDGPU/AMDGPUAsmPrinter.cpp b/lib/Target/AMDGPU/AMDGPUAsmPrinter.cpp
new file mode 100644
index 00000000000..709d7531d38
--- /dev/null
+++ b/lib/Target/AMDGPU/AMDGPUAsmPrinter.cpp
@@ -0,0 +1,521 @@
+//===-- AMDGPUAsmPrinter.cpp - AMDGPU Assebly printer  --------------------===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+/// \file
+///
+/// The AMDGPUAsmPrinter is used to print both assembly string and also binary
+/// code.  When passed an MCAsmStreamer it prints assembly and when passed
+/// an MCObjectStreamer it outputs binary code.
+//
+//===----------------------------------------------------------------------===//
+//
+
+#include "AMDGPUAsmPrinter.h"
+#include "MCTargetDesc/AMDGPUTargetStreamer.h"
+#include "InstPrinter/AMDGPUInstPrinter.h"
+#include "Utils/AMDGPUBaseInfo.h"
+#include "AMDGPU.h"
+#include "AMDKernelCodeT.h"
+#include "AMDGPUSubtarget.h"
+#include "R600Defines.h"
+#include "R600MachineFunctionInfo.h"
+#include "R600RegisterInfo.h"
+#include "SIDefines.h"
+#include "SIMachineFunctionInfo.h"
+#include "SIRegisterInfo.h"
+#include "llvm/CodeGen/MachineFrameInfo.h"
+#include "llvm/MC/MCContext.h"
+#include "llvm/MC/MCSectionELF.h"
+#include "llvm/MC/MCStreamer.h"
+#include "llvm/Support/ELF.h"
+#include "llvm/Support/MathExtras.h"
+#include "llvm/Support/TargetRegistry.h"
+#include "llvm/Target/TargetLoweringObjectFile.h"
+
+using namespace llvm;
+
+// TODO: This should get the default rounding mode from the kernel. We just set
+// the default here, but this could change if the OpenCL rounding mode pragmas
+// are used.
+//
+// The denormal mode here should match what is reported by the OpenCL runtime
+// for the CL_FP_DENORM bit from CL_DEVICE_{HALF|SINGLE|DOUBLE}_FP_CONFIG, but
+// can also be override to flush with the -cl-denorms-are-zero compiler flag.
+//
+// AMD OpenCL only sets flush none and reports CL_FP_DENORM for double
+// precision, and leaves single precision to flush all and does not report
+// CL_FP_DENORM for CL_DEVICE_SINGLE_FP_CONFIG. Mesa's OpenCL currently reports
+// CL_FP_DENORM for both.
+//
+// FIXME: It seems some instructions do not support single precision denormals
+// regardless of the mode (exp_*_f32, rcp_*_f32, rsq_*_f32, rsq_*f32, sqrt_f32,
+// and sin_f32, cos_f32 on most parts).
+
+// We want to use these instructions, and using fp32 denormals also causes
+// instructions to run at the double precision rate for the device so it's
+// probably best to just report no single precision denormals.
+static uint32_t getFPMode(const MachineFunction &F) {
+  const AMDGPUSubtarget& ST = F.getSubtarget<AMDGPUSubtarget>();
+  // TODO: Is there any real use for the flush in only / flush out only modes?
+
+  uint32_t FP32Denormals =
+    ST.hasFP32Denormals() ? FP_DENORM_FLUSH_NONE : FP_DENORM_FLUSH_IN_FLUSH_OUT;
+
+  uint32_t FP64Denormals =
+    ST.hasFP64Denormals() ? FP_DENORM_FLUSH_NONE : FP_DENORM_FLUSH_IN_FLUSH_OUT;
+
+  return FP_ROUND_MODE_SP(FP_ROUND_ROUND_TO_NEAREST) |
+         FP_ROUND_MODE_DP(FP_ROUND_ROUND_TO_NEAREST) |
+         FP_DENORM_MODE_SP(FP32Denormals) |
+         FP_DENORM_MODE_DP(FP64Denormals);
+}
+
+static AsmPrinter *
+createAMDGPUAsmPrinterPass(TargetMachine &tm,
+                           std::unique_ptr<MCStreamer> &&Streamer) {
+  return new AMDGPUAsmPrinter(tm, std::move(Streamer));
+}
+
+extern "C" void LLVMInitializeAMDGPUAsmPrinter() {
+  TargetRegistry::RegisterAsmPrinter(TheAMDGPUTarget, createAMDGPUAsmPrinterPass);
+  TargetRegistry::RegisterAsmPrinter(TheGCNTarget, createAMDGPUAsmPrinterPass);
+}
+
+AMDGPUAsmPrinter::AMDGPUAsmPrinter(TargetMachine &TM,
+                                   std::unique_ptr<MCStreamer> Streamer)
+    : AsmPrinter(TM, std::move(Streamer)) {}
+
+void AMDGPUAsmPrinter::EmitFunctionBodyStart() {
+  const AMDGPUSubtarget &STM = MF->getSubtarget<AMDGPUSubtarget>();
+  SIProgramInfo KernelInfo;
+  if (STM.isAmdHsaOS()) {
+    getSIProgramInfo(KernelInfo, *MF);
+    EmitAmdKernelCodeT(*MF, KernelInfo);
+  }
+}
+
+void AMDGPUAsmPrinter::EmitEndOfAsmFile(Module &M) {
+
+  // This label is used to mark the end of the .text section.
+  const TargetLoweringObjectFile &TLOF = getObjFileLowering();
+  OutStreamer->SwitchSection(TLOF.getTextSection());
+  MCSymbol *EndOfTextLabel =
+      OutContext.getOrCreateSymbol(StringRef(END_OF_TEXT_LABEL_NAME));
+  OutStreamer->EmitLabel(EndOfTextLabel);
+}
+
+bool AMDGPUAsmPrinter::runOnMachineFunction(MachineFunction &MF) {
+
+  // The starting address of all shader programs must be 256 bytes aligned.
+  MF.setAlignment(8);
+
+  SetupMachineFunction(MF);
+
+  MCContext &Context = getObjFileLowering().getContext();
+  MCSectionELF *ConfigSection =
+      Context.getELFSection(".AMDGPU.config", ELF::SHT_PROGBITS, 0);
+  OutStreamer->SwitchSection(ConfigSection);
+
+  const AMDGPUSubtarget &STM = MF.getSubtarget<AMDGPUSubtarget>();
+  SIProgramInfo KernelInfo;
+  if (STM.getGeneration() >= AMDGPUSubtarget::SOUTHERN_ISLANDS) {
+    if (!STM.isAmdHsaOS()) {
+      getSIProgramInfo(KernelInfo, MF);
+      EmitProgramInfoSI(MF, KernelInfo);
+    }
+    // Emit directives
+    AMDGPUTargetStreamer *TS =
+        static_cast<AMDGPUTargetStreamer *>(OutStreamer->getTargetStreamer());
+    TS->EmitDirectiveHSACodeObjectVersion(1, 0);
+    AMDGPU::IsaVersion ISA = STM.getIsaVersion();
+    TS->EmitDirectiveHSACodeObjectISA(ISA.Major, ISA.Minor, ISA.Stepping,
+                                      "AMD", "AMDGPU");
+  } else {
+    EmitProgramInfoR600(MF);
+  }
+
+  DisasmLines.clear();
+  HexLines.clear();
+  DisasmLineMaxLen = 0;
+
+  EmitFunctionBody();
+
+  if (isVerbose()) {
+    MCSectionELF *CommentSection =
+        Context.getELFSection(".AMDGPU.csdata", ELF::SHT_PROGBITS, 0);
+    OutStreamer->SwitchSection(CommentSection);
+
+    if (STM.getGeneration() >= AMDGPUSubtarget::SOUTHERN_ISLANDS) {
+      OutStreamer->emitRawComment(" Kernel info:", false);
+      OutStreamer->emitRawComment(" codeLenInByte = " + Twine(KernelInfo.CodeLen),
+                                  false);
+      OutStreamer->emitRawComment(" NumSgprs: " + Twine(KernelInfo.NumSGPR),
+                                  false);
+      OutStreamer->emitRawComment(" NumVgprs: " + Twine(KernelInfo.NumVGPR),
+                                  false);
+      OutStreamer->emitRawComment(" FloatMode: " + Twine(KernelInfo.FloatMode),
+                                  false);
+      OutStreamer->emitRawComment(" IeeeMode: " + Twine(KernelInfo.IEEEMode),
+                                  false);
+      OutStreamer->emitRawComment(" ScratchSize: " + Twine(KernelInfo.ScratchSize),
+                                  false);
+    } else {
+      R600MachineFunctionInfo *MFI = MF.getInfo<R600MachineFunctionInfo>();
+      OutStreamer->emitRawComment(
+        Twine("SQ_PGM_RESOURCES:STACK_SIZE = " + Twine(MFI->StackSize)));
+    }
+  }
+
+  if (STM.dumpCode()) {
+
+    OutStreamer->SwitchSection(
+        Context.getELFSection(".AMDGPU.disasm", ELF::SHT_NOTE, 0));
+
+    for (size_t i = 0; i < DisasmLines.size(); ++i) {
+      std::string Comment(DisasmLineMaxLen - DisasmLines[i].size(), ' ');
+      Comment += " ; " + HexLines[i] + "\n";
+
+      OutStreamer->EmitBytes(StringRef(DisasmLines[i]));
+      OutStreamer->EmitBytes(StringRef(Comment));
+    }
+  }
+
+  return false;
+}
+
+void AMDGPUAsmPrinter::EmitProgramInfoR600(const MachineFunction &MF) {
+  unsigned MaxGPR = 0;
+  bool killPixel = false;
+  const AMDGPUSubtarget &STM = MF.getSubtarget<AMDGPUSubtarget>();
+  const R600RegisterInfo *RI =
+      static_cast<const R600RegisterInfo *>(STM.getRegisterInfo());
+  const R600MachineFunctionInfo *MFI = MF.getInfo<R600MachineFunctionInfo>();
+
+  for (const MachineBasicBlock &MBB : MF) {
+    for (const MachineInstr &MI : MBB) {
+      if (MI.getOpcode() == AMDGPU::KILLGT)
+        killPixel = true;
+      unsigned numOperands = MI.getNumOperands();
+      for (unsigned op_idx = 0; op_idx < numOperands; op_idx++) {
+        const MachineOperand &MO = MI.getOperand(op_idx);
+        if (!MO.isReg())
+          continue;
+        unsigned HWReg = RI->getEncodingValue(MO.getReg()) & 0xff;
+
+        // Register with value > 127 aren't GPR
+        if (HWReg > 127)
+          continue;
+        MaxGPR = std::max(MaxGPR, HWReg);
+      }
+    }
+  }
+
+  unsigned RsrcReg;
+  if (STM.getGeneration() >= AMDGPUSubtarget::EVERGREEN) {
+    // Evergreen / Northern Islands
+    switch (MFI->getShaderType()) {
+    default: // Fall through
+    case ShaderType::COMPUTE:  RsrcReg = R_0288D4_SQ_PGM_RESOURCES_LS; break;
+    case ShaderType::GEOMETRY: RsrcReg = R_028878_SQ_PGM_RESOURCES_GS; break;
+    case ShaderType::PIXEL:    RsrcReg = R_028844_SQ_PGM_RESOURCES_PS; break;
+    case ShaderType::VERTEX:   RsrcReg = R_028860_SQ_PGM_RESOURCES_VS; break;
+    }
+  } else {
+    // R600 / R700
+    switch (MFI->getShaderType()) {
+    default: // Fall through
+    case ShaderType::GEOMETRY: // Fall through
+    case ShaderType::COMPUTE:  // Fall through
+    case ShaderType::VERTEX:   RsrcReg = R_028868_SQ_PGM_RESOURCES_VS; break;
+    case ShaderType::PIXEL:    RsrcReg = R_028850_SQ_PGM_RESOURCES_PS; break;
+    }
+  }
+
+  OutStreamer->EmitIntValue(RsrcReg, 4);
+  OutStreamer->EmitIntValue(S_NUM_GPRS(MaxGPR + 1) |
+                           S_STACK_SIZE(MFI->StackSize), 4);
+  OutStreamer->EmitIntValue(R_02880C_DB_SHADER_CONTROL, 4);
+  OutStreamer->EmitIntValue(S_02880C_KILL_ENABLE(killPixel), 4);
+
+  if (MFI->getShaderType() == ShaderType::COMPUTE) {
+    OutStreamer->EmitIntValue(R_0288E8_SQ_LDS_ALLOC, 4);
+    OutStreamer->EmitIntValue(RoundUpToAlignment(MFI->LDSSize, 4) >> 2, 4);
+  }
+}
+
+void AMDGPUAsmPrinter::getSIProgramInfo(SIProgramInfo &ProgInfo,
+                                        const MachineFunction &MF) const {
+  const AMDGPUSubtarget &STM = MF.getSubtarget<AMDGPUSubtarget>();
+  const SIMachineFunctionInfo *MFI = MF.getInfo<SIMachineFunctionInfo>();
+  uint64_t CodeSize = 0;
+  unsigned MaxSGPR = 0;
+  unsigned MaxVGPR = 0;
+  bool VCCUsed = false;
+  bool FlatUsed = false;
+  const SIRegisterInfo *RI =
+      static_cast<const SIRegisterInfo *>(STM.getRegisterInfo());
+
+  for (const MachineBasicBlock &MBB : MF) {
+    for (const MachineInstr &MI : MBB) {
+      // TODO: CodeSize should account for multiple functions.
+      CodeSize += MI.getDesc().Size;
+
+      unsigned numOperands = MI.getNumOperands();
+      for (unsigned op_idx = 0; op_idx < numOperands; op_idx++) {
+        const MachineOperand &MO = MI.getOperand(op_idx);
+        unsigned width = 0;
+        bool isSGPR = false;
+
+        if (!MO.isReg()) {
+          continue;
+        }
+        unsigned reg = MO.getReg();
+        if (reg == AMDGPU::VCC || reg == AMDGPU::VCC_LO ||
+	    reg == AMDGPU::VCC_HI) {
+          VCCUsed = true;
+          continue;
+        } else if (reg == AMDGPU::FLAT_SCR ||
+                   reg == AMDGPU::FLAT_SCR_LO ||
+                   reg == AMDGPU::FLAT_SCR_HI) {
+          FlatUsed = true;
+          continue;
+        }
+
+        switch (reg) {
+        default: break;
+        case AMDGPU::SCC:
+        case AMDGPU::EXEC:
+        case AMDGPU::M0:
+          continue;
+        }
+
+        if (AMDGPU::SReg_32RegClass.contains(reg)) {
+          isSGPR = true;
+          width = 1;
+        } else if (AMDGPU::VGPR_32RegClass.contains(reg)) {
+          isSGPR = false;
+          width = 1;
+        } else if (AMDGPU::SReg_64RegClass.contains(reg)) {
+          isSGPR = true;
+          width = 2;
+        } else if (AMDGPU::VReg_64RegClass.contains(reg)) {
+          isSGPR = false;
+          width = 2;
+        } else if (AMDGPU::VReg_96RegClass.contains(reg)) {
+          isSGPR = false;
+          width = 3;
+        } else if (AMDGPU::SReg_128RegClass.contains(reg)) {
+          isSGPR = true;
+          width = 4;
+        } else if (AMDGPU::VReg_128RegClass.contains(reg)) {
+          isSGPR = false;
+          width = 4;
+        } else if (AMDGPU::SReg_256RegClass.contains(reg)) {
+          isSGPR = true;
+          width = 8;
+        } else if (AMDGPU::VReg_256RegClass.contains(reg)) {
+          isSGPR = false;
+          width = 8;
+        } else if (AMDGPU::SReg_512RegClass.contains(reg)) {
+          isSGPR = true;
+          width = 16;
+        } else if (AMDGPU::VReg_512RegClass.contains(reg)) {
+          isSGPR = false;
+          width = 16;
+        } else {
+          llvm_unreachable("Unknown register class");
+        }
+        unsigned hwReg = RI->getEncodingValue(reg) & 0xff;
+        unsigned maxUsed = hwReg + width - 1;
+        if (isSGPR) {
+          MaxSGPR = maxUsed > MaxSGPR ? maxUsed : MaxSGPR;
+        } else {
+          MaxVGPR = maxUsed > MaxVGPR ? maxUsed : MaxVGPR;
+        }
+      }
+    }
+  }
+
+  if (VCCUsed)
+    MaxSGPR += 2;
+
+  if (FlatUsed)
+    MaxSGPR += 2;
+
+  // We found the maximum register index. They start at 0, so add one to get the
+  // number of registers.
+  ProgInfo.NumVGPR = MaxVGPR + 1;
+  ProgInfo.NumSGPR = MaxSGPR + 1;
+
+  if (STM.hasSGPRInitBug()) {
+    if (ProgInfo.NumSGPR > AMDGPUSubtarget::FIXED_SGPR_COUNT_FOR_INIT_BUG) {
+      LLVMContext &Ctx = MF.getFunction()->getContext();
+      Ctx.emitError("too many SGPRs used with the SGPR init bug");
+    }
+
+    ProgInfo.NumSGPR = AMDGPUSubtarget::FIXED_SGPR_COUNT_FOR_INIT_BUG;
+  }
+
+  ProgInfo.VGPRBlocks = (ProgInfo.NumVGPR - 1) / 4;
+  ProgInfo.SGPRBlocks = (ProgInfo.NumSGPR - 1) / 8;
+  // Set the value to initialize FP_ROUND and FP_DENORM parts of the mode
+  // register.
+  ProgInfo.FloatMode = getFPMode(MF);
+
+  // XXX: Not quite sure what this does, but sc seems to unset this.
+  ProgInfo.IEEEMode = 0;
+
+  // Do not clamp NAN to 0.
+  ProgInfo.DX10Clamp = 0;
+
+  const MachineFrameInfo *FrameInfo = MF.getFrameInfo();
+  ProgInfo.ScratchSize = FrameInfo->estimateStackSize(MF);
+
+  ProgInfo.FlatUsed = FlatUsed;
+  ProgInfo.VCCUsed = VCCUsed;
+  ProgInfo.CodeLen = CodeSize;
+
+  unsigned LDSAlignShift;
+  if (STM.getGeneration() < AMDGPUSubtarget::SEA_ISLANDS) {
+    // LDS is allocated in 64 dword blocks.
+    LDSAlignShift = 8;
+  } else {
+    // LDS is allocated in 128 dword blocks.
+    LDSAlignShift = 9;
+  }
+
+  unsigned LDSSpillSize = MFI->LDSWaveSpillSize *
+                          MFI->getMaximumWorkGroupSize(MF);
+
+  ProgInfo.LDSSize = MFI->LDSSize + LDSSpillSize;
+  ProgInfo.LDSBlocks =
+     RoundUpToAlignment(ProgInfo.LDSSize, 1 << LDSAlignShift) >> LDSAlignShift;
+
+  // Scratch is allocated in 256 dword blocks.
+  unsigned ScratchAlignShift = 10;
+  // We need to program the hardware with the amount of scratch memory that
+  // is used by the entire wave.  ProgInfo.ScratchSize is the amount of
+  // scratch memory used per thread.
+  ProgInfo.ScratchBlocks =
+    RoundUpToAlignment(ProgInfo.ScratchSize * STM.getWavefrontSize(),
+                       1 << ScratchAlignShift) >> ScratchAlignShift;
+
+  ProgInfo.ComputePGMRSrc1 =
+      S_00B848_VGPRS(ProgInfo.VGPRBlocks) |
+      S_00B848_SGPRS(ProgInfo.SGPRBlocks) |
+      S_00B848_PRIORITY(ProgInfo.Priority) |
+      S_00B848_FLOAT_MODE(ProgInfo.FloatMode) |
+      S_00B848_PRIV(ProgInfo.Priv) |
+      S_00B848_DX10_CLAMP(ProgInfo.DX10Clamp) |
+      S_00B848_IEEE_MODE(ProgInfo.DebugMode) |
+      S_00B848_IEEE_MODE(ProgInfo.IEEEMode);
+
+  ProgInfo.ComputePGMRSrc2 =
+      S_00B84C_SCRATCH_EN(ProgInfo.ScratchBlocks > 0) |
+      S_00B84C_USER_SGPR(MFI->NumUserSGPRs) |
+      S_00B84C_TGID_X_EN(1) |
+      S_00B84C_TGID_Y_EN(1) |
+      S_00B84C_TGID_Z_EN(1) |
+      S_00B84C_TG_SIZE_EN(1) |
+      S_00B84C_TIDIG_COMP_CNT(2) |
+      S_00B84C_LDS_SIZE(ProgInfo.LDSBlocks);
+}
+
+static unsigned getRsrcReg(unsigned ShaderType) {
+  switch (ShaderType) {
+  default: // Fall through
+  case ShaderType::COMPUTE:  return R_00B848_COMPUTE_PGM_RSRC1;
+  case ShaderType::GEOMETRY: return R_00B228_SPI_SHADER_PGM_RSRC1_GS;
+  case ShaderType::PIXEL:    return R_00B028_SPI_SHADER_PGM_RSRC1_PS;
+  case ShaderType::VERTEX:   return R_00B128_SPI_SHADER_PGM_RSRC1_VS;
+  }
+}
+
+void AMDGPUAsmPrinter::EmitProgramInfoSI(const MachineFunction &MF,
+                                         const SIProgramInfo &KernelInfo) {
+  const AMDGPUSubtarget &STM = MF.getSubtarget<AMDGPUSubtarget>();
+  const SIMachineFunctionInfo *MFI = MF.getInfo<SIMachineFunctionInfo>();
+  unsigned RsrcReg = getRsrcReg(MFI->getShaderType());
+
+  if (MFI->getShaderType() == ShaderType::COMPUTE) {
+    OutStreamer->EmitIntValue(R_00B848_COMPUTE_PGM_RSRC1, 4);
+
+    OutStreamer->EmitIntValue(KernelInfo.ComputePGMRSrc1, 4);
+
+    OutStreamer->EmitIntValue(R_00B84C_COMPUTE_PGM_RSRC2, 4);
+    OutStreamer->EmitIntValue(KernelInfo.ComputePGMRSrc2, 4);
+
+    OutStreamer->EmitIntValue(R_00B860_COMPUTE_TMPRING_SIZE, 4);
+    OutStreamer->EmitIntValue(S_00B860_WAVESIZE(KernelInfo.ScratchBlocks), 4);
+
+    // TODO: Should probably note flat usage somewhere. SC emits a "FlatPtr32 =
+    // 0" comment but I don't see a corresponding field in the register spec.
+  } else {
+    OutStreamer->EmitIntValue(RsrcReg, 4);
+    OutStreamer->EmitIntValue(S_00B028_VGPRS(KernelInfo.VGPRBlocks) |
+                              S_00B028_SGPRS(KernelInfo.SGPRBlocks), 4);
+    if (STM.isVGPRSpillingEnabled(MFI)) {
+      OutStreamer->EmitIntValue(R_0286E8_SPI_TMPRING_SIZE, 4);
+      OutStreamer->EmitIntValue(S_0286E8_WAVESIZE(KernelInfo.ScratchBlocks), 4);
+    }
+  }
+
+  if (MFI->getShaderType() == ShaderType::PIXEL) {
+    OutStreamer->EmitIntValue(R_00B02C_SPI_SHADER_PGM_RSRC2_PS, 4);
+    OutStreamer->EmitIntValue(S_00B02C_EXTRA_LDS_SIZE(KernelInfo.LDSBlocks), 4);
+    OutStreamer->EmitIntValue(R_0286CC_SPI_PS_INPUT_ENA, 4);
+    OutStreamer->EmitIntValue(MFI->PSInputAddr, 4);
+  }
+}
+
+void AMDGPUAsmPrinter::EmitAmdKernelCodeT(const MachineFunction &MF,
+                                         const SIProgramInfo &KernelInfo) const {
+  const SIMachineFunctionInfo *MFI = MF.getInfo<SIMachineFunctionInfo>();
+  const AMDGPUSubtarget &STM = MF.getSubtarget<AMDGPUSubtarget>();
+  amd_kernel_code_t header;
+
+  AMDGPU::initDefaultAMDKernelCodeT(header, STM.getFeatureBits());
+
+  header.compute_pgm_resource_registers =
+      KernelInfo.ComputePGMRSrc1 |
+      (KernelInfo.ComputePGMRSrc2 << 32);
+  header.code_properties =
+      AMD_CODE_PROPERTY_ENABLE_SGPR_KERNARG_SEGMENT_PTR |
+      AMD_CODE_PROPERTY_IS_PTR64;
+
+  header.kernarg_segment_byte_size = MFI->ABIArgOffset;
+  header.wavefront_sgpr_count = KernelInfo.NumSGPR;
+  header.workitem_vgpr_count = KernelInfo.NumVGPR;
+
+
+  AMDGPUTargetStreamer *TS =
+      static_cast<AMDGPUTargetStreamer *>(OutStreamer->getTargetStreamer());
+  TS->EmitAMDKernelCodeT(header);
+}
+
+bool AMDGPUAsmPrinter::PrintAsmOperand(const MachineInstr *MI, unsigned OpNo,
+                                       unsigned AsmVariant,
+                                       const char *ExtraCode, raw_ostream &O) {
+  if (ExtraCode && ExtraCode[0]) {
+    if (ExtraCode[1] != 0)
+      return true; // Unknown modifier.
+
+    switch (ExtraCode[0]) {
+    default:
+      // See if this is a generic print operand
+      return AsmPrinter::PrintAsmOperand(MI, OpNo, AsmVariant, ExtraCode, O);
+    case 'r':
+      break;
+    }
+  }
+
+  AMDGPUInstPrinter::printRegOperand(MI->getOperand(OpNo).getReg(), O,
+                   *TM.getSubtargetImpl(*MF->getFunction())->getRegisterInfo());
+  return false;
+}
diff --git a/lib/Target/R600/AMDGPUAsmPrinter.h b/lib/Target/AMDGPU/AMDGPUAsmPrinter.h
similarity index 98%
rename from lib/Target/R600/AMDGPUAsmPrinter.h
rename to lib/Target/AMDGPU/AMDGPUAsmPrinter.h
index 1acff3a3222..345af9b85e1 100644
--- a/lib/Target/R600/AMDGPUAsmPrinter.h
+++ b/lib/Target/AMDGPU/AMDGPUAsmPrinter.h
@@ -97,6 +97,8 @@ class AMDGPUAsmPrinter : public AsmPrinter {
   /// Implemented in AMDGPUMCInstLower.cpp
   void EmitInstruction(const MachineInstr *MI) override;
 
+  void EmitFunctionBodyStart() override;
+
   void EmitEndOfAsmFile(Module &M) override;
 
   bool PrintAsmOperand(const MachineInstr *MI, unsigned OpNo,
diff --git a/lib/Target/R600/AMDGPUCallingConv.td b/lib/Target/AMDGPU/AMDGPUCallingConv.td
similarity index 100%
rename from lib/Target/R600/AMDGPUCallingConv.td
rename to lib/Target/AMDGPU/AMDGPUCallingConv.td
diff --git a/lib/Target/R600/AMDGPUFrameLowering.cpp b/lib/Target/AMDGPU/AMDGPUFrameLowering.cpp
similarity index 96%
rename from lib/Target/R600/AMDGPUFrameLowering.cpp
rename to lib/Target/AMDGPU/AMDGPUFrameLowering.cpp
index 9e8302ec0a1..8175786fb9b 100644
--- a/lib/Target/R600/AMDGPUFrameLowering.cpp
+++ b/lib/Target/AMDGPU/AMDGPUFrameLowering.cpp
@@ -99,9 +99,8 @@ AMDGPUFrameLowering::getCalleeSavedSpillSlots(unsigned &NumEntries) const {
   NumEntries = 0;
   return nullptr;
 }
-void
-AMDGPUFrameLowering::emitPrologue(MachineFunction &MF) const {
-}
+void AMDGPUFrameLowering::emitPrologue(MachineFunction &MF,
+                                       MachineBasicBlock &MBB) const {}
 void
 AMDGPUFrameLowering::emitEpilogue(MachineFunction &MF,
                                   MachineBasicBlock &MBB) const {
diff --git a/lib/Target/R600/AMDGPUFrameLowering.h b/lib/Target/AMDGPU/AMDGPUFrameLowering.h
similarity index 95%
rename from lib/Target/R600/AMDGPUFrameLowering.h
rename to lib/Target/AMDGPU/AMDGPUFrameLowering.h
index 15a6636a1ae..9f31be1af79 100644
--- a/lib/Target/R600/AMDGPUFrameLowering.h
+++ b/lib/Target/AMDGPU/AMDGPUFrameLowering.h
@@ -37,7 +37,7 @@ class AMDGPUFrameLowering : public TargetFrameLowering {
   int getFrameIndexOffset(const MachineFunction &MF, int FI) const override;
   const SpillSlot *
     getCalleeSavedSpillSlots(unsigned &NumEntries) const override;
-  void emitPrologue(MachineFunction &MF) const override;
+  void emitPrologue(MachineFunction &MF, MachineBasicBlock &MBB) const override;
   void emitEpilogue(MachineFunction &MF, MachineBasicBlock &MBB) const override;
   bool hasFP(const MachineFunction &MF) const override;
 };
diff --git a/lib/Target/R600/AMDGPUISelDAGToDAG.cpp b/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp
similarity index 82%
rename from lib/Target/R600/AMDGPUISelDAGToDAG.cpp
rename to lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp
index def252a47b2..64c54ccb31f 100644
--- a/lib/Target/R600/AMDGPUISelDAGToDAG.cpp
+++ b/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp
@@ -50,7 +50,6 @@ class AMDGPUDAGToDAGISel : public SelectionDAGISel {
 
 private:
   bool isInlineImmediate(SDNode *N) const;
-  inline SDValue getSmallIPtrImm(unsigned Imm);
   bool FoldOperand(SDValue &Src, SDValue &Sel, SDValue &Neg, SDValue &Abs,
                    const R600InstrInfo *TII);
   bool FoldOperands(unsigned, const R600InstrInfo *, std::vector<SDValue> &);
@@ -79,6 +78,8 @@ class AMDGPUDAGToDAGISel : public SelectionDAGISel {
   bool isLocalLoad(const LoadSDNode *N) const;
   bool isRegionLoad(const LoadSDNode *N) const;
 
+  SDNode *glueCopyToM0(SDNode *N) const;
+
   const TargetRegisterClass *getOperandRegClass(SDNode *N, unsigned OpNo) const;
   bool SelectGlobalValueConstantOffset(SDValue Addr, SDValue& IntPtr);
   bool SelectGlobalValueVariableOffset(SDValue Addr, SDValue &BaseReg,
@@ -109,8 +110,11 @@ class AMDGPUDAGToDAGISel : public SelectionDAGISel {
                          SDValue &Offset, SDValue &GLC) const;
   SDNode *SelectAddrSpaceCast(SDNode *N);
   bool SelectVOP3Mods(SDValue In, SDValue &Src, SDValue &SrcMods) const;
+  bool SelectVOP3NoMods(SDValue In, SDValue &Src, SDValue &SrcMods) const;
   bool SelectVOP3Mods0(SDValue In, SDValue &Src, SDValue &SrcMods,
                        SDValue &Clamp, SDValue &Omod) const;
+  bool SelectVOP3NoMods0(SDValue In, SDValue &Src, SDValue &SrcMods,
+                         SDValue &Clamp, SDValue &Omod) const;
 
   bool SelectVOP3Mods0Clamp(SDValue In, SDValue &Src, SDValue &SrcMods,
                             SDValue &Omod) const;
@@ -189,27 +193,23 @@ const TargetRegisterClass *AMDGPUDAGToDAGISel::getOperandRegClass(SDNode *N,
   }
 }
 
-SDValue AMDGPUDAGToDAGISel::getSmallIPtrImm(unsigned int Imm) {
-  return CurDAG->getTargetConstant(Imm, MVT::i32);
-}
-
 bool AMDGPUDAGToDAGISel::SelectADDRParam(
   SDValue Addr, SDValue& R1, SDValue& R2) {
 
   if (Addr.getOpcode() == ISD::FrameIndex) {
     if (FrameIndexSDNode *FIN = dyn_cast<FrameIndexSDNode>(Addr)) {
       R1 = CurDAG->getTargetFrameIndex(FIN->getIndex(), MVT::i32);
-      R2 = CurDAG->getTargetConstant(0, MVT::i32);
+      R2 = CurDAG->getTargetConstant(0, SDLoc(Addr), MVT::i32);
     } else {
       R1 = Addr;
-      R2 = CurDAG->getTargetConstant(0, MVT::i32);
+      R2 = CurDAG->getTargetConstant(0, SDLoc(Addr), MVT::i32);
     }
   } else if (Addr.getOpcode() == ISD::ADD) {
     R1 = Addr.getOperand(0);
     R2 = Addr.getOperand(1);
   } else {
     R1 = Addr;
-    R2 = CurDAG->getTargetConstant(0, MVT::i32);
+    R2 = CurDAG->getTargetConstant(0, SDLoc(Addr), MVT::i32);
   }
   return true;
 }
@@ -232,21 +232,47 @@ bool AMDGPUDAGToDAGISel::SelectADDR64(SDValue Addr, SDValue& R1, SDValue& R2) {
   if (Addr.getOpcode() == ISD::FrameIndex) {
     if (FrameIndexSDNode *FIN = dyn_cast<FrameIndexSDNode>(Addr)) {
       R1 = CurDAG->getTargetFrameIndex(FIN->getIndex(), MVT::i64);
-      R2 = CurDAG->getTargetConstant(0, MVT::i64);
+      R2 = CurDAG->getTargetConstant(0, SDLoc(Addr), MVT::i64);
     } else {
       R1 = Addr;
-      R2 = CurDAG->getTargetConstant(0, MVT::i64);
+      R2 = CurDAG->getTargetConstant(0, SDLoc(Addr), MVT::i64);
     }
   } else if (Addr.getOpcode() == ISD::ADD) {
     R1 = Addr.getOperand(0);
     R2 = Addr.getOperand(1);
   } else {
     R1 = Addr;
-    R2 = CurDAG->getTargetConstant(0, MVT::i64);
+    R2 = CurDAG->getTargetConstant(0, SDLoc(Addr), MVT::i64);
   }
   return true;
 }
 
+SDNode *AMDGPUDAGToDAGISel::glueCopyToM0(SDNode *N) const {
+  if (Subtarget->getGeneration() < AMDGPUSubtarget::SOUTHERN_ISLANDS ||
+      !checkType(cast<MemSDNode>(N)->getMemOperand()->getValue(),
+                 AMDGPUAS::LOCAL_ADDRESS))
+    return N;
+
+  const SITargetLowering& Lowering =
+      *static_cast<const SITargetLowering*>(getTargetLowering());
+
+  // Write max value to m0 before each load operation
+
+  SDValue M0 = Lowering.copyToM0(*CurDAG, CurDAG->getEntryNode(), SDLoc(N),
+                                 CurDAG->getTargetConstant(-1, SDLoc(N), MVT::i32));
+
+  SDValue Glue = M0.getValue(1);
+
+  SmallVector <SDValue, 8> Ops;
+  for (unsigned i = 0, e = N->getNumOperands(); i != e; ++i) {
+     Ops.push_back(N->getOperand(i));
+  }
+  Ops.push_back(Glue);
+  CurDAG->MorphNodeTo(N, N->getOpcode(), N->getVTList(), Ops);
+
+  return N;
+}
+
 SDNode *AMDGPUDAGToDAGISel::Select(SDNode *N) {
   unsigned int Opc = N->getOpcode();
   if (N->isMachineOpcode()) {
@@ -254,6 +280,9 @@ SDNode *AMDGPUDAGToDAGISel::Select(SDNode *N) {
     return nullptr;   // Already selected.
   }
 
+  if (isa<AtomicSDNode>(N))
+    N = glueCopyToM0(N);
+
   switch (Opc) {
   default: break;
   // We are selecting i64 ADD here instead of custom lower it during
@@ -326,7 +355,8 @@ SDNode *AMDGPUDAGToDAGISel::Select(SDNode *N) {
       }
     }
 
-    SDValue RegClass = CurDAG->getTargetConstant(RegClassID, MVT::i32);
+    SDLoc DL(N);
+    SDValue RegClass = CurDAG->getTargetConstant(RegClassID, DL, MVT::i32);
 
     if (NumVectorElts == 1) {
       return CurDAG->SelectNodeTo(N, AMDGPU::COPY_TO_REGCLASS, EltVT,
@@ -340,7 +370,7 @@ SDNode *AMDGPUDAGToDAGISel::Select(SDNode *N) {
     // 1 = Vector Register Class
     SmallVector<SDValue, 16 * 2 + 1> RegSeqArgs(NumVectorElts * 2 + 1);
 
-    RegSeqArgs[0] = CurDAG->getTargetConstant(RegClassID, MVT::i32);
+    RegSeqArgs[0] = CurDAG->getTargetConstant(RegClassID, DL, MVT::i32);
     bool IsRegSeq = true;
     unsigned NOps = N->getNumOperands();
     for (unsigned i = 0; i < NOps; i++) {
@@ -351,7 +381,8 @@ SDNode *AMDGPUDAGToDAGISel::Select(SDNode *N) {
       }
       RegSeqArgs[1 + (2 * i)] = N->getOperand(i);
       RegSeqArgs[1 + (2 * i) + 1] =
-              CurDAG->getTargetConstant(TRI->getSubRegFromChannel(i), MVT::i32);
+              CurDAG->getTargetConstant(TRI->getSubRegFromChannel(i), DL,
+                                        MVT::i32);
     }
 
     if (NOps != NumVectorElts) {
@@ -359,11 +390,11 @@ SDNode *AMDGPUDAGToDAGISel::Select(SDNode *N) {
       assert(Opc == ISD::SCALAR_TO_VECTOR && NOps < NumVectorElts);
 
       MachineSDNode *ImpDef = CurDAG->getMachineNode(TargetOpcode::IMPLICIT_DEF,
-                                                     SDLoc(N), EltVT);
+                                                     DL, EltVT);
       for (unsigned i = NOps; i < NumVectorElts; ++i) {
         RegSeqArgs[1 + (2 * i)] = SDValue(ImpDef, 0);
         RegSeqArgs[1 + (2 * i) + 1] =
-          CurDAG->getTargetConstant(TRI->getSubRegFromChannel(i), MVT::i32);
+          CurDAG->getTargetConstant(TRI->getSubRegFromChannel(i), DL, MVT::i32);
       }
     }
 
@@ -377,21 +408,22 @@ SDNode *AMDGPUDAGToDAGISel::Select(SDNode *N) {
     if (Subtarget->getGeneration() <= AMDGPUSubtarget::NORTHERN_ISLANDS) {
       break;
     }
+    SDLoc DL(N);
     if (N->getValueType(0) == MVT::i128) {
-      RC = CurDAG->getTargetConstant(AMDGPU::SReg_128RegClassID, MVT::i32);
-      SubReg0 = CurDAG->getTargetConstant(AMDGPU::sub0_sub1, MVT::i32);
-      SubReg1 = CurDAG->getTargetConstant(AMDGPU::sub2_sub3, MVT::i32);
+      RC = CurDAG->getTargetConstant(AMDGPU::SReg_128RegClassID, DL, MVT::i32);
+      SubReg0 = CurDAG->getTargetConstant(AMDGPU::sub0_sub1, DL, MVT::i32);
+      SubReg1 = CurDAG->getTargetConstant(AMDGPU::sub2_sub3, DL, MVT::i32);
     } else if (N->getValueType(0) == MVT::i64) {
-      RC = CurDAG->getTargetConstant(AMDGPU::SReg_64RegClassID, MVT::i32);
-      SubReg0 = CurDAG->getTargetConstant(AMDGPU::sub0, MVT::i32);
-      SubReg1 = CurDAG->getTargetConstant(AMDGPU::sub1, MVT::i32);
+      RC = CurDAG->getTargetConstant(AMDGPU::SReg_64RegClassID, DL, MVT::i32);
+      SubReg0 = CurDAG->getTargetConstant(AMDGPU::sub0, DL, MVT::i32);
+      SubReg1 = CurDAG->getTargetConstant(AMDGPU::sub1, DL, MVT::i32);
     } else {
       llvm_unreachable("Unhandled value type for BUILD_PAIR");
     }
     const SDValue Ops[] = { RC, N->getOperand(0), SubReg0,
                             N->getOperand(1), SubReg1 };
     return CurDAG->getMachineNode(TargetOpcode::REG_SEQUENCE,
-                                  SDLoc(N), N->getValueType(0), Ops);
+                                  DL, N->getValueType(0), Ops);
   }
 
   case ISD::Constant:
@@ -408,38 +440,46 @@ SDNode *AMDGPUDAGToDAGISel::Select(SDNode *N) {
       Imm = C->getZExtValue();
     }
 
-    SDNode *Lo = CurDAG->getMachineNode(AMDGPU::S_MOV_B32, SDLoc(N), MVT::i32,
-                                CurDAG->getConstant(Imm & 0xFFFFFFFF, MVT::i32));
-    SDNode *Hi = CurDAG->getMachineNode(AMDGPU::S_MOV_B32, SDLoc(N), MVT::i32,
-                                CurDAG->getConstant(Imm >> 32, MVT::i32));
+    SDLoc DL(N);
+    SDNode *Lo = CurDAG->getMachineNode(AMDGPU::S_MOV_B32, DL, MVT::i32,
+                                CurDAG->getConstant(Imm & 0xFFFFFFFF, DL,
+                                                    MVT::i32));
+    SDNode *Hi = CurDAG->getMachineNode(AMDGPU::S_MOV_B32, DL, MVT::i32,
+                                CurDAG->getConstant(Imm >> 32, DL, MVT::i32));
     const SDValue Ops[] = {
-      CurDAG->getTargetConstant(AMDGPU::SReg_64RegClassID, MVT::i32),
-      SDValue(Lo, 0), CurDAG->getTargetConstant(AMDGPU::sub0, MVT::i32),
-      SDValue(Hi, 0), CurDAG->getTargetConstant(AMDGPU::sub1, MVT::i32)
+      CurDAG->getTargetConstant(AMDGPU::SReg_64RegClassID, DL, MVT::i32),
+      SDValue(Lo, 0), CurDAG->getTargetConstant(AMDGPU::sub0, DL, MVT::i32),
+      SDValue(Hi, 0), CurDAG->getTargetConstant(AMDGPU::sub1, DL, MVT::i32)
     };
 
-    return CurDAG->getMachineNode(TargetOpcode::REG_SEQUENCE, SDLoc(N),
+    return CurDAG->getMachineNode(TargetOpcode::REG_SEQUENCE, DL,
                                   N->getValueType(0), Ops);
   }
 
   case ISD::LOAD: {
+    LoadSDNode *LD = cast<LoadSDNode>(N);
+    SDLoc SL(N);
+    EVT VT = N->getValueType(0);
+
+    if (VT != MVT::i64 || LD->getExtensionType() != ISD::NON_EXTLOAD) {
+      N = glueCopyToM0(N);
+      break;
+    }
+
     // To simplify the TableGen patters, we replace all i64 loads with
     // v2i32 loads.  Alternatively, we could promote i64 loads to v2i32
     // during DAG legalization, however, so places (ExpandUnalignedLoad)
     // in the DAG legalizer assume that if i64 is legal, so doing this
     // promotion early can cause problems.
-    EVT VT = N->getValueType(0);
-    LoadSDNode *LD = cast<LoadSDNode>(N);
-    if (VT != MVT::i64 || LD->getExtensionType() != ISD::NON_EXTLOAD)
-      break;
 
     SDValue NewLoad = CurDAG->getLoad(MVT::v2i32, SDLoc(N), LD->getChain(),
-                                     LD->getBasePtr(), LD->getMemOperand());
-    SDValue BitCast = CurDAG->getNode(ISD::BITCAST, SDLoc(N),
+                                      LD->getBasePtr(), LD->getMemOperand());
+    SDValue BitCast = CurDAG->getNode(ISD::BITCAST, SL,
                                       MVT::i64, NewLoad);
     CurDAG->ReplaceAllUsesOfValueWith(SDValue(N, 1), NewLoad.getValue(1));
     CurDAG->ReplaceAllUsesOfValueWith(SDValue(N, 0), BitCast);
-    SelectCode(NewLoad.getNode());
+    SDNode *Load = glueCopyToM0(NewLoad.getNode());
+    SelectCode(Load);
     N = BitCast.getNode();
     break;
   }
@@ -448,24 +488,26 @@ SDNode *AMDGPUDAGToDAGISel::Select(SDNode *N) {
     // Handle i64 stores here for the same reason mentioned above for loads.
     StoreSDNode *ST = cast<StoreSDNode>(N);
     SDValue Value = ST->getValue();
-    if (Value.getValueType() != MVT::i64 || ST->isTruncatingStore())
-      break;
+    if (Value.getValueType() == MVT::i64 && !ST->isTruncatingStore()) {
+
+      SDValue NewValue = CurDAG->getNode(ISD::BITCAST, SDLoc(N),
+                                        MVT::v2i32, Value);
+      SDValue NewStore = CurDAG->getStore(ST->getChain(), SDLoc(N), NewValue,
+                                          ST->getBasePtr(), ST->getMemOperand());
 
-    SDValue NewValue = CurDAG->getNode(ISD::BITCAST, SDLoc(N),
-                                      MVT::v2i32, Value);
-    SDValue NewStore = CurDAG->getStore(ST->getChain(), SDLoc(N), NewValue,
-                                        ST->getBasePtr(), ST->getMemOperand());
+      CurDAG->ReplaceAllUsesOfValueWith(SDValue(N, 0), NewStore);
 
-    CurDAG->ReplaceAllUsesOfValueWith(SDValue(N, 0), NewStore);
+      if (NewValue.getOpcode() == ISD::BITCAST) {
+        Select(NewStore.getNode());
+        return SelectCode(NewValue.getNode());
+      }
 
-    if (NewValue.getOpcode() == ISD::BITCAST) {
-      Select(NewStore.getNode());
-      return SelectCode(NewValue.getNode());
+      // getNode() may fold the bitcast if its input was another bitcast.  If that
+      // happens we should only select the new store.
+      N = NewStore.getNode();
     }
 
-    // getNode() may fold the bitcast if its input was another bitcast.  If that
-    // happens we should only select the new store.
-    N = NewStore.getNode();
+    N = glueCopyToM0(N);
     break;
   }
 
@@ -474,15 +516,17 @@ SDNode *AMDGPUDAGToDAGISel::Select(SDNode *N) {
       break;
     SDValue Addr, Offset;
 
+    SDLoc DL(N);
     SelectADDRIndirect(N->getOperand(1), Addr, Offset);
     const SDValue Ops[] = {
       Addr,
       Offset,
-      CurDAG->getTargetConstant(0, MVT::i32),
+      CurDAG->getTargetConstant(0, DL, MVT::i32),
       N->getOperand(0),
     };
-    return CurDAG->getMachineNode(AMDGPU::SI_RegisterLoad, SDLoc(N),
-                                  CurDAG->getVTList(MVT::i32, MVT::i64, MVT::Other),
+    return CurDAG->getMachineNode(AMDGPU::SI_RegisterLoad, DL,
+                                  CurDAG->getVTList(MVT::i32, MVT::i64,
+                                                    MVT::Other),
                                   Ops);
   }
   case AMDGPUISD::REGISTER_STORE: {
@@ -490,14 +534,15 @@ SDNode *AMDGPUDAGToDAGISel::Select(SDNode *N) {
       break;
     SDValue Addr, Offset;
     SelectADDRIndirect(N->getOperand(2), Addr, Offset);
+    SDLoc DL(N);
     const SDValue Ops[] = {
       N->getOperand(1),
       Addr,
       Offset,
-      CurDAG->getTargetConstant(0, MVT::i32),
+      CurDAG->getTargetConstant(0, DL, MVT::i32),
       N->getOperand(0),
     };
-    return CurDAG->getMachineNode(AMDGPU::SI_RegisterStorePseudo, SDLoc(N),
+    return CurDAG->getMachineNode(AMDGPU::SI_RegisterStorePseudo, DL,
                                         CurDAG->getVTList(MVT::Other),
                                         Ops);
   }
@@ -682,7 +727,8 @@ const char *AMDGPUDAGToDAGISel::getPassName() const {
 bool AMDGPUDAGToDAGISel::SelectGlobalValueConstantOffset(SDValue Addr,
                                                          SDValue& IntPtr) {
   if (ConstantSDNode *Cst = dyn_cast<ConstantSDNode>(Addr)) {
-    IntPtr = CurDAG->getIntPtrConstant(Cst->getZExtValue() / 4, true);
+    IntPtr = CurDAG->getIntPtrConstant(Cst->getZExtValue() / 4, SDLoc(Addr),
+                                       true);
     return true;
   }
   return false;
@@ -692,7 +738,7 @@ bool AMDGPUDAGToDAGISel::SelectGlobalValueVariableOffset(SDValue Addr,
     SDValue& BaseReg, SDValue &Offset) {
   if (!isa<ConstantSDNode>(Addr)) {
     BaseReg = Addr;
-    Offset = CurDAG->getIntPtrConstant(0, true);
+    Offset = CurDAG->getIntPtrConstant(0, SDLoc(Addr), true);
     return true;
   }
   return false;
@@ -707,7 +753,8 @@ bool AMDGPUDAGToDAGISel::SelectADDRVTX_READ(SDValue Addr, SDValue &Base,
       && isInt<16>(IMMOffset->getZExtValue())) {
 
       Base = Addr.getOperand(0);
-      Offset = CurDAG->getTargetConstant(IMMOffset->getZExtValue(), MVT::i32);
+      Offset = CurDAG->getTargetConstant(IMMOffset->getZExtValue(), SDLoc(Addr),
+                                         MVT::i32);
       return true;
   // If the pointer address is constant, we can move it to the offset field.
   } else if ((IMMOffset = dyn_cast<ConstantSDNode>(Addr))
@@ -715,30 +762,32 @@ bool AMDGPUDAGToDAGISel::SelectADDRVTX_READ(SDValue Addr, SDValue &Base,
     Base = CurDAG->getCopyFromReg(CurDAG->getEntryNode(),
                                   SDLoc(CurDAG->getEntryNode()),
                                   AMDGPU::ZERO, MVT::i32);
-    Offset = CurDAG->getTargetConstant(IMMOffset->getZExtValue(), MVT::i32);
+    Offset = CurDAG->getTargetConstant(IMMOffset->getZExtValue(), SDLoc(Addr),
+                                       MVT::i32);
     return true;
   }
 
   // Default case, no offset
   Base = Addr;
-  Offset = CurDAG->getTargetConstant(0, MVT::i32);
+  Offset = CurDAG->getTargetConstant(0, SDLoc(Addr), MVT::i32);
   return true;
 }
 
 bool AMDGPUDAGToDAGISel::SelectADDRIndirect(SDValue Addr, SDValue &Base,
                                             SDValue &Offset) {
   ConstantSDNode *C;
+  SDLoc DL(Addr);
 
   if ((C = dyn_cast<ConstantSDNode>(Addr))) {
     Base = CurDAG->getRegister(AMDGPU::INDIRECT_BASE_ADDR, MVT::i32);
-    Offset = CurDAG->getTargetConstant(C->getZExtValue(), MVT::i32);
+    Offset = CurDAG->getTargetConstant(C->getZExtValue(), DL, MVT::i32);
   } else if ((Addr.getOpcode() == ISD::ADD || Addr.getOpcode() == ISD::OR) &&
             (C = dyn_cast<ConstantSDNode>(Addr.getOperand(1)))) {
     Base = Addr.getOperand(0);
-    Offset = CurDAG->getTargetConstant(C->getZExtValue(), MVT::i32);
+    Offset = CurDAG->getTargetConstant(C->getZExtValue(), DL, MVT::i32);
   } else {
     Base = Addr;
-    Offset = CurDAG->getTargetConstant(0, MVT::i32);
+    Offset = CurDAG->getTargetConstant(0, DL, MVT::i32);
   }
 
   return true;
@@ -751,8 +800,8 @@ SDNode *AMDGPUDAGToDAGISel::SelectADD_SUB_I64(SDNode *N) {
 
   bool IsAdd = (N->getOpcode() == ISD::ADD);
 
-  SDValue Sub0 = CurDAG->getTargetConstant(AMDGPU::sub0, MVT::i32);
-  SDValue Sub1 = CurDAG->getTargetConstant(AMDGPU::sub1, MVT::i32);
+  SDValue Sub0 = CurDAG->getTargetConstant(AMDGPU::sub0, DL, MVT::i32);
+  SDValue Sub1 = CurDAG->getTargetConstant(AMDGPU::sub1, DL, MVT::i32);
 
   SDNode *Lo0 = CurDAG->getMachineNode(TargetOpcode::EXTRACT_SUBREG,
                                        DL, MVT::i32, LHS, Sub0);
@@ -778,7 +827,7 @@ SDNode *AMDGPUDAGToDAGISel::SelectADD_SUB_I64(SDNode *N) {
                              SDValue(Hi0, 0), SDValue(Hi1, 0), Carry);
 
   SDValue Args[5] = {
-    CurDAG->getTargetConstant(AMDGPU::SReg_64RegClassID, MVT::i32),
+    CurDAG->getTargetConstant(AMDGPU::SReg_64RegClassID, DL, MVT::i32),
     SDValue(AddLo,0),
     Sub0,
     SDValue(AddHi,0),
@@ -813,7 +862,8 @@ bool AMDGPUDAGToDAGISel::isDSOffsetLegal(const SDValue &Base, unsigned Offset,
       (OffsetBits == 8 && !isUInt<8>(Offset)))
     return false;
 
-  if (Subtarget->getGeneration() >= AMDGPUSubtarget::SEA_ISLANDS)
+  if (Subtarget->getGeneration() >= AMDGPUSubtarget::SEA_ISLANDS ||
+      Subtarget->unsafeDSOffsetFoldingEnabled())
     return true;
 
   // On Southern Islands instruction with a negative base value and an offset
@@ -835,15 +885,17 @@ bool AMDGPUDAGToDAGISel::SelectDS1Addr1Offset(SDValue Addr, SDValue &Base,
     }
   }
 
+  SDLoc DL(Addr);
+
   // If we have a constant address, prefer to put the constant into the
   // offset. This can save moves to load the constant address since multiple
   // operations can share the zero base address register, and enables merging
   // into read2 / write2 instructions.
   if (const ConstantSDNode *CAddr = dyn_cast<ConstantSDNode>(Addr)) {
     if (isUInt<16>(CAddr->getZExtValue())) {
-      SDValue Zero = CurDAG->getTargetConstant(0, MVT::i32);
+      SDValue Zero = CurDAG->getTargetConstant(0, DL, MVT::i32);
       MachineSDNode *MovZero = CurDAG->getMachineNode(AMDGPU::V_MOV_B32_e32,
-                                 SDLoc(Addr), MVT::i32, Zero);
+                                 DL, MVT::i32, Zero);
       Base = SDValue(MovZero, 0);
       Offset = Addr;
       return true;
@@ -852,13 +904,15 @@ bool AMDGPUDAGToDAGISel::SelectDS1Addr1Offset(SDValue Addr, SDValue &Base,
 
   // default case
   Base = Addr;
-  Offset = CurDAG->getTargetConstant(0, MVT::i16);
+  Offset = CurDAG->getTargetConstant(0, DL, MVT::i16);
   return true;
 }
 
 bool AMDGPUDAGToDAGISel::SelectDS64Bit4ByteAligned(SDValue Addr, SDValue &Base,
                                                    SDValue &Offset0,
                                                    SDValue &Offset1) const {
+  SDLoc DL(Addr);
+
   if (CurDAG->isBaseWithConstantOffset(Addr)) {
     SDValue N0 = Addr.getOperand(0);
     SDValue N1 = Addr.getOperand(1);
@@ -868,8 +922,8 @@ bool AMDGPUDAGToDAGISel::SelectDS64Bit4ByteAligned(SDValue Addr, SDValue &Base,
     // (add n0, c0)
     if (isDSOffsetLegal(N0, DWordOffset1, 8)) {
       Base = N0;
-      Offset0 = CurDAG->getTargetConstant(DWordOffset0, MVT::i8);
-      Offset1 = CurDAG->getTargetConstant(DWordOffset1, MVT::i8);
+      Offset0 = CurDAG->getTargetConstant(DWordOffset0, DL, MVT::i8);
+      Offset1 = CurDAG->getTargetConstant(DWordOffset1, DL, MVT::i8);
       return true;
     }
   }
@@ -880,21 +934,21 @@ bool AMDGPUDAGToDAGISel::SelectDS64Bit4ByteAligned(SDValue Addr, SDValue &Base,
     assert(4 * DWordOffset0 == CAddr->getZExtValue());
 
     if (isUInt<8>(DWordOffset0) && isUInt<8>(DWordOffset1)) {
-      SDValue Zero = CurDAG->getTargetConstant(0, MVT::i32);
+      SDValue Zero = CurDAG->getTargetConstant(0, DL, MVT::i32);
       MachineSDNode *MovZero
         = CurDAG->getMachineNode(AMDGPU::V_MOV_B32_e32,
-                                 SDLoc(Addr), MVT::i32, Zero);
+                                 DL, MVT::i32, Zero);
       Base = SDValue(MovZero, 0);
-      Offset0 = CurDAG->getTargetConstant(DWordOffset0, MVT::i8);
-      Offset1 = CurDAG->getTargetConstant(DWordOffset1, MVT::i8);
+      Offset0 = CurDAG->getTargetConstant(DWordOffset0, DL, MVT::i8);
+      Offset1 = CurDAG->getTargetConstant(DWordOffset1, DL, MVT::i8);
       return true;
     }
   }
 
   // default case
   Base = Addr;
-  Offset0 = CurDAG->getTargetConstant(0, MVT::i8);
-  Offset1 = CurDAG->getTargetConstant(1, MVT::i8);
+  Offset0 = CurDAG->getTargetConstant(0, DL, MVT::i8);
+  Offset1 = CurDAG->getTargetConstant(1, DL, MVT::i8);
   return true;
 }
 
@@ -910,14 +964,14 @@ void AMDGPUDAGToDAGISel::SelectMUBUF(SDValue Addr, SDValue &Ptr,
                                      SDValue &TFE) const {
   SDLoc DL(Addr);
 
-  GLC = CurDAG->getTargetConstant(0, MVT::i1);
-  SLC = CurDAG->getTargetConstant(0, MVT::i1);
-  TFE = CurDAG->getTargetConstant(0, MVT::i1);
+  GLC = CurDAG->getTargetConstant(0, DL, MVT::i1);
+  SLC = CurDAG->getTargetConstant(0, DL, MVT::i1);
+  TFE = CurDAG->getTargetConstant(0, DL, MVT::i1);
 
-  Idxen = CurDAG->getTargetConstant(0, MVT::i1);
-  Offen = CurDAG->getTargetConstant(0, MVT::i1);
-  Addr64 = CurDAG->getTargetConstant(0, MVT::i1);
-  SOffset = CurDAG->getTargetConstant(0, MVT::i32);
+  Idxen = CurDAG->getTargetConstant(0, DL, MVT::i1);
+  Offen = CurDAG->getTargetConstant(0, DL, MVT::i1);
+  Addr64 = CurDAG->getTargetConstant(0, DL, MVT::i1);
+  SOffset = CurDAG->getTargetConstant(0, DL, MVT::i32);
 
   if (CurDAG->isBaseWithConstantOffset(Addr)) {
     SDValue N0 = Addr.getOperand(0);
@@ -928,24 +982,25 @@ void AMDGPUDAGToDAGISel::SelectMUBUF(SDValue Addr, SDValue &Ptr,
       // (add (add N2, N3), C1) -> addr64
       SDValue N2 = N0.getOperand(0);
       SDValue N3 = N0.getOperand(1);
-      Addr64 = CurDAG->getTargetConstant(1, MVT::i1);
+      Addr64 = CurDAG->getTargetConstant(1, DL, MVT::i1);
       Ptr = N2;
       VAddr = N3;
     } else {
 
       // (add N0, C1) -> offset
-      VAddr = CurDAG->getTargetConstant(0, MVT::i32);
+      VAddr = CurDAG->getTargetConstant(0, DL, MVT::i32);
       Ptr = N0;
     }
 
     if (isLegalMUBUFImmOffset(C1)) {
-        Offset = CurDAG->getTargetConstant(C1->getZExtValue(), MVT::i16);
+        Offset = CurDAG->getTargetConstant(C1->getZExtValue(), DL, MVT::i16);
         return;
     } else if (isUInt<32>(C1->getZExtValue())) {
       // Illegal offset, store it in soffset.
-      Offset = CurDAG->getTargetConstant(0, MVT::i16);
+      Offset = CurDAG->getTargetConstant(0, DL, MVT::i16);
       SOffset = SDValue(CurDAG->getMachineNode(AMDGPU::S_MOV_B32, DL, MVT::i32,
-                   CurDAG->getTargetConstant(C1->getZExtValue(), MVT::i32)), 0);
+                   CurDAG->getTargetConstant(C1->getZExtValue(), DL, MVT::i32)),
+                        0);
       return;
     }
   }
@@ -954,17 +1009,17 @@ void AMDGPUDAGToDAGISel::SelectMUBUF(SDValue Addr, SDValue &Ptr,
     // (add N0, N1) -> addr64
     SDValue N0 = Addr.getOperand(0);
     SDValue N1 = Addr.getOperand(1);
-    Addr64 = CurDAG->getTargetConstant(1, MVT::i1);
+    Addr64 = CurDAG->getTargetConstant(1, DL, MVT::i1);
     Ptr = N0;
     VAddr = N1;
-    Offset = CurDAG->getTargetConstant(0, MVT::i16);
+    Offset = CurDAG->getTargetConstant(0, DL, MVT::i16);
     return;
   }
 
   // default case -> offset
-  VAddr = CurDAG->getTargetConstant(0, MVT::i32);
+  VAddr = CurDAG->getTargetConstant(0, DL, MVT::i32);
   Ptr = Addr;
-  Offset = CurDAG->getTargetConstant(0, MVT::i16);
+  Offset = CurDAG->getTargetConstant(0, DL, MVT::i16);
 
 }
 
@@ -974,6 +1029,10 @@ bool AMDGPUDAGToDAGISel::SelectMUBUFAddr64(SDValue Addr, SDValue &SRsrc,
                                            SDValue &SLC, SDValue &TFE) const {
   SDValue Ptr, Offen, Idxen, Addr64;
 
+  // addr64 bit was removed for volcanic islands.
+  if (Subtarget->getGeneration() >= AMDGPUSubtarget::VOLCANIC_ISLANDS)
+    return false;
+
   SelectMUBUF(Addr, Ptr, VAddr, SOffset, Offset, Offen, Idxen, Addr64,
               GLC, SLC, TFE);
 
@@ -995,7 +1054,7 @@ bool AMDGPUDAGToDAGISel::SelectMUBUFAddr64(SDValue Addr, SDValue &SRsrc,
                                            SDValue &VAddr, SDValue &SOffset,
 					   SDValue &Offset,
 					   SDValue &SLC) const {
-  SLC = CurDAG->getTargetConstant(0, MVT::i1);
+  SLC = CurDAG->getTargetConstant(0, SDLoc(Addr), MVT::i1);
   SDValue GLC, TFE;
 
   return SelectMUBUFAddr64(Addr, SRsrc, VAddr, SOffset, Offset, GLC, SLC, TFE);
@@ -1026,11 +1085,11 @@ bool AMDGPUDAGToDAGISel::SelectMUBUFScratch(SDValue Addr, SDValue &Rsrc,
       SDValue(CurDAG->getMachineNode(AMDGPU::S_MOV_B32, DL, MVT::i32, Sym1), 0);
 
   const SDValue RsrcOps[] = {
-      CurDAG->getTargetConstant(AMDGPU::SReg_64RegClassID, MVT::i32),
+      CurDAG->getTargetConstant(AMDGPU::SReg_64RegClassID, DL, MVT::i32),
       ScratchRsrcDword0,
-      CurDAG->getTargetConstant(AMDGPU::sub0, MVT::i32),
+      CurDAG->getTargetConstant(AMDGPU::sub0, DL, MVT::i32),
       ScratchRsrcDword1,
-      CurDAG->getTargetConstant(AMDGPU::sub1, MVT::i32),
+      CurDAG->getTargetConstant(AMDGPU::sub1, DL, MVT::i32),
   };
   SDValue ScratchPtr = SDValue(CurDAG->getMachineNode(AMDGPU::REG_SEQUENCE, DL,
                                               MVT::v2i32, RsrcOps), 0);
@@ -1040,19 +1099,22 @@ bool AMDGPUDAGToDAGISel::SelectMUBUFScratch(SDValue Addr, SDValue &Rsrc,
 
   // (add n0, c1)
   if (CurDAG->isBaseWithConstantOffset(Addr)) {
+    SDValue N0 = Addr.getOperand(0);
     SDValue N1 = Addr.getOperand(1);
-    ConstantSDNode *C1 = cast<ConstantSDNode>(N1);
-
-    if (isLegalMUBUFImmOffset(C1)) {
-      VAddr = Addr.getOperand(0);
-      ImmOffset = CurDAG->getTargetConstant(C1->getZExtValue(), MVT::i16);
-      return true;
+    // Offsets in vaddr must be positive.
+    if (CurDAG->SignBitIsZero(N0)) {
+      ConstantSDNode *C1 = cast<ConstantSDNode>(N1);
+      if (isLegalMUBUFImmOffset(C1)) {
+        VAddr = N0;
+        ImmOffset = CurDAG->getTargetConstant(C1->getZExtValue(), DL, MVT::i16);
+        return true;
+      }
     }
   }
 
   // (node)
   VAddr = Addr;
-  ImmOffset = CurDAG->getTargetConstant(0, MVT::i16);
+  ImmOffset = CurDAG->getTargetConstant(0, DL, MVT::i16);
   return true;
 }
 
@@ -1125,7 +1187,7 @@ SDNode *AMDGPUDAGToDAGISel::SelectAddrSpaceCast(SDNode *N) {
       DL,
       DestVT,
       Src,
-      CurDAG->getTargetConstant(AMDGPU::sub0, MVT::i32));
+      CurDAG->getTargetConstant(AMDGPU::sub0, DL, MVT::i32));
   }
 
 
@@ -1134,19 +1196,20 @@ SDNode *AMDGPUDAGToDAGISel::SelectAddrSpaceCast(SDNode *N) {
 
     // FIXME: This is probably wrong, we should never be defining
     // a register class with both VGPRs and SGPRs
-    SDValue RC = CurDAG->getTargetConstant(AMDGPU::VS_64RegClassID, MVT::i32);
+    SDValue RC = CurDAG->getTargetConstant(AMDGPU::VS_64RegClassID, DL,
+                                           MVT::i32);
 
     const SDValue Ops[] = {
       RC,
       Src,
-      CurDAG->getTargetConstant(AMDGPU::sub0, MVT::i32),
-      SDValue(CurDAG->getMachineNode(AMDGPU::S_MOV_B32, SDLoc(N), MVT::i32,
-                                     CurDAG->getConstant(0, MVT::i32)), 0),
-      CurDAG->getTargetConstant(AMDGPU::sub1, MVT::i32)
+      CurDAG->getTargetConstant(AMDGPU::sub0, DL, MVT::i32),
+      SDValue(CurDAG->getMachineNode(AMDGPU::S_MOV_B32, DL, MVT::i32,
+                                     CurDAG->getConstant(0, DL, MVT::i32)), 0),
+      CurDAG->getTargetConstant(AMDGPU::sub1, DL, MVT::i32)
     };
 
     return CurDAG->getMachineNode(TargetOpcode::REG_SEQUENCE,
-                                  SDLoc(N), N->getValueType(0), Ops);
+                                  DL, N->getValueType(0), Ops);
   }
 
   assert(SrcSize == 64 && DestSize == 64);
@@ -1159,7 +1222,7 @@ SDNode *AMDGPUDAGToDAGISel::getS_BFE(unsigned Opcode, SDLoc DL, SDValue Val,
   // the format expected by the S_BFE_I32 / S_BFE_U32. In the second
   // source, bits [5:0] contain the offset and bits [22:16] the width.
   uint32_t PackedVal = Offset | (Width << 16);
-  SDValue PackedConst = CurDAG->getTargetConstant(PackedVal, MVT::i32);
+  SDValue PackedConst = CurDAG->getTargetConstant(PackedVal, DL, MVT::i32);
 
   return CurDAG->getMachineNode(Opcode, DL, MVT::i32, Val, PackedConst);
 }
@@ -1259,26 +1322,43 @@ bool AMDGPUDAGToDAGISel::SelectVOP3Mods(SDValue In, SDValue &Src,
     Src = Src.getOperand(0);
   }
 
-  SrcMods = CurDAG->getTargetConstant(Mods, MVT::i32);
+  SrcMods = CurDAG->getTargetConstant(Mods, SDLoc(In), MVT::i32);
 
   return true;
 }
 
+bool AMDGPUDAGToDAGISel::SelectVOP3NoMods(SDValue In, SDValue &Src,
+                                         SDValue &SrcMods) const {
+  bool Res = SelectVOP3Mods(In, Src, SrcMods);
+  return Res && cast<ConstantSDNode>(SrcMods)->isNullValue();
+}
+
 bool AMDGPUDAGToDAGISel::SelectVOP3Mods0(SDValue In, SDValue &Src,
                                          SDValue &SrcMods, SDValue &Clamp,
                                          SDValue &Omod) const {
+  SDLoc DL(In);
   // FIXME: Handle Clamp and Omod
-  Clamp = CurDAG->getTargetConstant(0, MVT::i32);
-  Omod = CurDAG->getTargetConstant(0, MVT::i32);
+  Clamp = CurDAG->getTargetConstant(0, DL, MVT::i32);
+  Omod = CurDAG->getTargetConstant(0, DL, MVT::i32);
 
   return SelectVOP3Mods(In, Src, SrcMods);
 }
 
+bool AMDGPUDAGToDAGISel::SelectVOP3NoMods0(SDValue In, SDValue &Src,
+                                           SDValue &SrcMods, SDValue &Clamp,
+                                           SDValue &Omod) const {
+  bool Res = SelectVOP3Mods0(In, Src, SrcMods, Clamp, Omod);
+
+  return Res && cast<ConstantSDNode>(SrcMods)->isNullValue() &&
+                cast<ConstantSDNode>(Clamp)->isNullValue() &&
+                cast<ConstantSDNode>(Omod)->isNullValue();
+}
+
 bool AMDGPUDAGToDAGISel::SelectVOP3Mods0Clamp(SDValue In, SDValue &Src,
                                               SDValue &SrcMods,
                                               SDValue &Omod) const {
   // FIXME: Handle Omod
-  Omod = CurDAG->getTargetConstant(0, MVT::i32);
+  Omod = CurDAG->getTargetConstant(0, SDLoc(In), MVT::i32);
 
   return SelectVOP3Mods(In, Src, SrcMods);
 }
@@ -1287,7 +1367,7 @@ bool AMDGPUDAGToDAGISel::SelectVOP3Mods0Clamp0OMod(SDValue In, SDValue &Src,
                                                    SDValue &SrcMods,
                                                    SDValue &Clamp,
                                                    SDValue &Omod) const {
-  Clamp = Omod = CurDAG->getTargetConstant(0, MVT::i32);
+  Clamp = Omod = CurDAG->getTargetConstant(0, SDLoc(In), MVT::i32);
   return SelectVOP3Mods(In, Src, SrcMods);
 }
 
@@ -1298,18 +1378,14 @@ void AMDGPUDAGToDAGISel::PostprocessISelDAG() {
   do {
     IsModified = false;
     // Go over all selected nodes and try to fold them a bit more
-    for (SelectionDAG::allnodes_iterator I = CurDAG->allnodes_begin(),
-         E = CurDAG->allnodes_end(); I != E; ++I) {
-
-      SDNode *Node = I;
-
-      MachineSDNode *MachineNode = dyn_cast<MachineSDNode>(I);
+    for (SDNode &Node : CurDAG->allnodes()) {
+      MachineSDNode *MachineNode = dyn_cast<MachineSDNode>(&Node);
       if (!MachineNode)
         continue;
 
       SDNode *ResNode = Lowering.PostISelFolding(MachineNode, *CurDAG);
-      if (ResNode != Node) {
-        ReplaceUses(Node, ResNode);
+      if (ResNode != &Node) {
+        ReplaceUses(&Node, ResNode);
         IsModified = true;
       }
     }
diff --git a/lib/Target/R600/AMDGPUISelLowering.cpp b/lib/Target/AMDGPU/AMDGPUISelLowering.cpp
similarity index 88%
rename from lib/Target/R600/AMDGPUISelLowering.cpp
rename to lib/Target/AMDGPU/AMDGPUISelLowering.cpp
index 7c5235d1507..3a65f3b5614 100644
--- a/lib/Target/R600/AMDGPUISelLowering.cpp
+++ b/lib/Target/AMDGPU/AMDGPUISelLowering.cpp
@@ -257,9 +257,22 @@ AMDGPUTargetLowering::AMDGPUTargetLowering(TargetMachine &TM,
   setOperationAction(ISD::FP16_TO_FP, MVT::f64, Expand);
 
   setLoadExtAction(ISD::EXTLOAD, MVT::f32, MVT::f16, Expand);
+  setLoadExtAction(ISD::EXTLOAD, MVT::v2f32, MVT::v2f16, Expand);
+  setLoadExtAction(ISD::EXTLOAD, MVT::v4f32, MVT::v4f16, Expand);
+  setLoadExtAction(ISD::EXTLOAD, MVT::v8f32, MVT::v8f16, Expand);
+
   setLoadExtAction(ISD::EXTLOAD, MVT::f64, MVT::f16, Expand);
+  setLoadExtAction(ISD::EXTLOAD, MVT::v2f64, MVT::v2f16, Expand);
+  setLoadExtAction(ISD::EXTLOAD, MVT::v4f64, MVT::v4f16, Expand);
+  setLoadExtAction(ISD::EXTLOAD, MVT::v8f64, MVT::v8f16, Expand);
+
   setTruncStoreAction(MVT::f32, MVT::f16, Expand);
+  setTruncStoreAction(MVT::v2f32, MVT::v2f16, Expand);
+  setTruncStoreAction(MVT::v4f32, MVT::v4f16, Expand);
+  setTruncStoreAction(MVT::v8f32, MVT::v8f16, Expand);
+
   setTruncStoreAction(MVT::f64, MVT::f16, Expand);
+  setTruncStoreAction(MVT::f64, MVT::f32, Expand);
 
   const MVT ScalarIntVTs[] = { MVT::i32, MVT::i64 };
   for (MVT VT : ScalarIntVTs) {
@@ -301,6 +314,11 @@ AMDGPUTargetLowering::AMDGPUTargetLowering(TargetMachine &TM,
   setOperationAction(ISD::FP_TO_UINT, MVT::i64, Custom);
   setOperationAction(ISD::SELECT_CC, MVT::i64, Expand);
 
+  setOperationAction(ISD::SMIN, MVT::i32, Legal);
+  setOperationAction(ISD::UMIN, MVT::i32, Legal);
+  setOperationAction(ISD::SMAX, MVT::i32, Legal);
+  setOperationAction(ISD::UMAX, MVT::i32, Legal);
+
   if (!Subtarget->hasFFBH())
     setOperationAction(ISD::CTLZ_ZERO_UNDEF, MVT::i32, Expand);
 
@@ -388,6 +406,7 @@ AMDGPUTargetLowering::AMDGPUTargetLowering(TargetMachine &TM,
   setOperationAction(ISD::FNEARBYINT, MVT::f32, Custom);
   setOperationAction(ISD::FNEARBYINT, MVT::f64, Custom);
 
+  setTargetDAGCombine(ISD::SHL);
   setTargetDAGCombine(ISD::MUL);
   setTargetDAGCombine(ISD::SELECT);
   setTargetDAGCombine(ISD::SELECT_CC);
@@ -426,7 +445,7 @@ AMDGPUTargetLowering::AMDGPUTargetLowering(TargetMachine &TM,
 // Target Information
 //===----------------------------------------------------------------------===//
 
-MVT AMDGPUTargetLowering::getVectorIdxTy() const {
+MVT AMDGPUTargetLowering::getVectorIdxTy(const DataLayout &) const {
   return MVT::i32;
 }
 
@@ -509,6 +528,12 @@ bool AMDGPUTargetLowering::isFNegFree(EVT VT) const {
   return VT == MVT::f32 || VT == MVT::f64;
 }
 
+bool AMDGPUTargetLowering:: storeOfVectorConstantIsCheap(EVT MemVT,
+                                                         unsigned NumElem,
+                                                         unsigned AS) const {
+  return true;
+}
+
 bool AMDGPUTargetLowering::isTruncateFree(EVT Source, EVT Dest) const {
   // Truncate is just accessing a subregister.
   return Dest.bitsLT(Source) && (Dest.getSizeInBits() % 32 == 0);
@@ -521,9 +546,8 @@ bool AMDGPUTargetLowering::isTruncateFree(Type *Source, Type *Dest) const {
 }
 
 bool AMDGPUTargetLowering::isZExtFree(Type *Src, Type *Dest) const {
-  const DataLayout *DL = getDataLayout();
-  unsigned SrcSize = DL->getTypeSizeInBits(Src->getScalarType());
-  unsigned DestSize = DL->getTypeSizeInBits(Dest->getScalarType());
+  unsigned SrcSize = Src->getScalarSizeInBits();
+  unsigned DestSize = Dest->getScalarSizeInBits();
 
   return SrcSize == 32 && DestSize == 64;
 }
@@ -673,34 +697,34 @@ SDValue AMDGPUTargetLowering::LowerConstantInitializer(const Constant* Init,
                                                        const SDValue &InitPtr,
                                                        SDValue Chain,
                                                        SelectionDAG &DAG) const {
-  const DataLayout *TD = getDataLayout();
+  const DataLayout &TD = DAG.getDataLayout();
   SDLoc DL(InitPtr);
   Type *InitTy = Init->getType();
 
   if (const ConstantInt *CI = dyn_cast<ConstantInt>(Init)) {
     EVT VT = EVT::getEVT(InitTy);
     PointerType *PtrTy = PointerType::get(InitTy, AMDGPUAS::PRIVATE_ADDRESS);
-    return DAG.getStore(Chain, DL, DAG.getConstant(*CI, VT), InitPtr,
-                        MachinePointerInfo(UndefValue::get(PtrTy)), false, false,
-                        TD->getPrefTypeAlignment(InitTy));
+    return DAG.getStore(Chain, DL, DAG.getConstant(*CI, DL, VT), InitPtr,
+                        MachinePointerInfo(UndefValue::get(PtrTy)), false,
+                        false, TD.getPrefTypeAlignment(InitTy));
   }
 
   if (const ConstantFP *CFP = dyn_cast<ConstantFP>(Init)) {
     EVT VT = EVT::getEVT(CFP->getType());
     PointerType *PtrTy = PointerType::get(CFP->getType(), 0);
-    return DAG.getStore(Chain, DL, DAG.getConstantFP(*CFP, VT), InitPtr,
-                 MachinePointerInfo(UndefValue::get(PtrTy)), false, false,
-                 TD->getPrefTypeAlignment(CFP->getType()));
+    return DAG.getStore(Chain, DL, DAG.getConstantFP(*CFP, DL, VT), InitPtr,
+                        MachinePointerInfo(UndefValue::get(PtrTy)), false,
+                        false, TD.getPrefTypeAlignment(CFP->getType()));
   }
 
   if (StructType *ST = dyn_cast<StructType>(InitTy)) {
-    const StructLayout *SL = TD->getStructLayout(ST);
+    const StructLayout *SL = TD.getStructLayout(ST);
 
     EVT PtrVT = InitPtr.getValueType();
     SmallVector<SDValue, 8> Chains;
 
     for (unsigned I = 0, N = ST->getNumElements(); I != N; ++I) {
-      SDValue Offset = DAG.getConstant(SL->getElementOffset(I), PtrVT);
+      SDValue Offset = DAG.getConstant(SL->getElementOffset(I), DL, PtrVT);
       SDValue Ptr = DAG.getNode(ISD::ADD, DL, PtrVT, InitPtr, Offset);
 
       Constant *Elt = Init->getAggregateElement(I);
@@ -721,10 +745,10 @@ SDValue AMDGPUTargetLowering::LowerConstantInitializer(const Constant* Init,
     else
       llvm_unreachable("Unexpected type");
 
-    unsigned EltSize = TD->getTypeAllocSize(SeqTy->getElementType());
+    unsigned EltSize = TD.getTypeAllocSize(SeqTy->getElementType());
     SmallVector<SDValue, 8> Chains;
     for (unsigned i = 0; i < NumElements; ++i) {
-      SDValue Offset = DAG.getConstant(i * EltSize, PtrVT);
+      SDValue Offset = DAG.getConstant(i * EltSize, DL, PtrVT);
       SDValue Ptr = DAG.getNode(ISD::ADD, DL, PtrVT, InitPtr, Offset);
 
       Constant *Elt = Init->getAggregateElement(i);
@@ -738,8 +762,8 @@ SDValue AMDGPUTargetLowering::LowerConstantInitializer(const Constant* Init,
     EVT VT = EVT::getEVT(InitTy);
     PointerType *PtrTy = PointerType::get(InitTy, AMDGPUAS::PRIVATE_ADDRESS);
     return DAG.getStore(Chain, DL, DAG.getUNDEF(VT), InitPtr,
-                        MachinePointerInfo(UndefValue::get(PtrTy)), false, false,
-                        TD->getPrefTypeAlignment(InitTy));
+                        MachinePointerInfo(UndefValue::get(PtrTy)), false,
+                        false, TD.getPrefTypeAlignment(InitTy));
   }
 
   Init->dump();
@@ -761,7 +785,7 @@ SDValue AMDGPUTargetLowering::LowerGlobalAddress(AMDGPUMachineFunction* MFI,
                                                  SDValue Op,
                                                  SelectionDAG &DAG) const {
 
-  const DataLayout *TD = getDataLayout();
+  const DataLayout &DL = DAG.getDataLayout();
   GlobalAddressSDNode *G = cast<GlobalAddressSDNode>(Op);
   const GlobalValue *GV = G->getGlobal();
 
@@ -777,7 +801,7 @@ SDValue AMDGPUTargetLowering::LowerGlobalAddress(AMDGPUMachineFunction* MFI,
 
     unsigned Offset;
     if (MFI->LocalMemoryObjects.count(GV) == 0) {
-      uint64_t Size = TD->getTypeAllocSize(GV->getType()->getElementType());
+      uint64_t Size = DL.getTypeAllocSize(GV->getType()->getElementType());
       Offset = MFI->LDSSize;
       MFI->LocalMemoryObjects[GV] = Offset;
       // XXX: Account for alignment?
@@ -786,16 +810,17 @@ SDValue AMDGPUTargetLowering::LowerGlobalAddress(AMDGPUMachineFunction* MFI,
       Offset = MFI->LocalMemoryObjects[GV];
     }
 
-    return DAG.getConstant(Offset, getPointerTy(AMDGPUAS::LOCAL_ADDRESS));
+    return DAG.getConstant(Offset, SDLoc(Op),
+                           getPointerTy(DL, AMDGPUAS::LOCAL_ADDRESS));
   }
   case AMDGPUAS::CONSTANT_ADDRESS: {
     MachineFrameInfo *FrameInfo = DAG.getMachineFunction().getFrameInfo();
     Type *EltType = GV->getType()->getElementType();
-    unsigned Size = TD->getTypeAllocSize(EltType);
-    unsigned Alignment = TD->getPrefTypeAlignment(EltType);
+    unsigned Size = DL.getTypeAllocSize(EltType);
+    unsigned Alignment = DL.getPrefTypeAlignment(EltType);
 
-    MVT PrivPtrVT = getPointerTy(AMDGPUAS::PRIVATE_ADDRESS);
-    MVT ConstPtrVT = getPointerTy(AMDGPUAS::CONSTANT_ADDRESS);
+    MVT PrivPtrVT = getPointerTy(DL, AMDGPUAS::PRIVATE_ADDRESS);
+    MVT ConstPtrVT = getPointerTy(DL, AMDGPUAS::CONSTANT_ADDRESS);
 
     int FI = FrameInfo->CreateStackObject(Size, Alignment, false);
     SDValue InitPtr = DAG.getFrameIndex(FI, PrivPtrVT);
@@ -839,11 +864,9 @@ SDValue AMDGPUTargetLowering::LowerGlobalAddress(AMDGPUMachineFunction* MFI,
 SDValue AMDGPUTargetLowering::LowerCONCAT_VECTORS(SDValue Op,
                                                   SelectionDAG &DAG) const {
   SmallVector<SDValue, 8> Args;
-  SDValue A = Op.getOperand(0);
-  SDValue B = Op.getOperand(1);
 
-  DAG.ExtractVectorElements(A, Args);
-  DAG.ExtractVectorElements(B, Args);
+  for (const SDUse &U : Op->ops())
+    DAG.ExtractVectorElements(U.get(), Args);
 
   return DAG.getNode(ISD::BUILD_VECTOR, SDLoc(Op), Op.getValueType(), Args);
 }
@@ -870,7 +893,7 @@ SDValue AMDGPUTargetLowering::LowerFrameIndex(SDValue Op,
 
   unsigned FrameIndex = FIN->getIndex();
   unsigned Offset = TFL->getFrameIndexOffset(MF, FrameIndex);
-  return DAG.getConstant(Offset * 4 * TFL->getStackWidth(MF),
+  return DAG.getConstant(Offset * 4 * TFL->getStackWidth(MF), SDLoc(Op),
                          Op.getValueType());
 }
 
@@ -945,9 +968,9 @@ SDValue AMDGPUTargetLowering::LowerINTRINSIC_WO_CHAIN(SDValue Op,
 
         SDValue Rsq = DAG.getNode(AMDGPUISD::RSQ, DL, VT, Op.getOperand(1));
         SDValue Tmp = DAG.getNode(ISD::FMINNUM, DL, VT, Rsq,
-                                  DAG.getConstantFP(Max, VT));
+                                  DAG.getConstantFP(Max, DL, VT));
         return DAG.getNode(ISD::FMAXNUM, DL, VT, Tmp,
-                           DAG.getConstantFP(Min, VT));
+                           DAG.getConstantFP(Min, DL, VT));
       } else {
         return DAG.getNode(AMDGPUISD::RSQ_CLAMPED, DL, VT, Op.getOperand(1));
       }
@@ -957,17 +980,17 @@ SDValue AMDGPUTargetLowering::LowerINTRINSIC_WO_CHAIN(SDValue Op,
                                                    Op.getOperand(2));
 
     case AMDGPUIntrinsic::AMDGPU_imax:
-      return DAG.getNode(AMDGPUISD::SMAX, DL, VT, Op.getOperand(1),
-                                                  Op.getOperand(2));
+      return DAG.getNode(ISD::SMAX, DL, VT, Op.getOperand(1),
+                                            Op.getOperand(2));
     case AMDGPUIntrinsic::AMDGPU_umax:
-      return DAG.getNode(AMDGPUISD::UMAX, DL, VT, Op.getOperand(1),
-                                                  Op.getOperand(2));
+      return DAG.getNode(ISD::UMAX, DL, VT, Op.getOperand(1),
+                                            Op.getOperand(2));
     case AMDGPUIntrinsic::AMDGPU_imin:
-      return DAG.getNode(AMDGPUISD::SMIN, DL, VT, Op.getOperand(1),
-                                                  Op.getOperand(2));
+      return DAG.getNode(ISD::SMIN, DL, VT, Op.getOperand(1),
+                                            Op.getOperand(2));
     case AMDGPUIntrinsic::AMDGPU_umin:
-      return DAG.getNode(AMDGPUISD::UMIN, DL, VT, Op.getOperand(1),
-                                                  Op.getOperand(2));
+      return DAG.getNode(ISD::UMIN, DL, VT, Op.getOperand(1),
+                                            Op.getOperand(2));
 
     case AMDGPUIntrinsic::AMDGPU_umul24:
       return DAG.getNode(AMDGPUISD::MUL_U24, DL, VT,
@@ -1042,10 +1065,10 @@ SDValue AMDGPUTargetLowering::LowerIntrinsicIABS(SDValue Op,
                                                  SelectionDAG &DAG) const {
   SDLoc DL(Op);
   EVT VT = Op.getValueType();
-  SDValue Neg = DAG.getNode(ISD::SUB, DL, VT, DAG.getConstant(0, VT),
-                                              Op.getOperand(1));
+  SDValue Neg = DAG.getNode(ISD::SUB, DL, VT, DAG.getConstant(0, DL, VT),
+                            Op.getOperand(1));
 
-  return DAG.getNode(AMDGPUISD::SMAX, DL, VT, Neg, Op.getOperand(1));
+  return DAG.getNode(ISD::SMAX, DL, VT, Neg, Op.getOperand(1));
 }
 
 /// Linear Interpolation
@@ -1055,7 +1078,7 @@ SDValue AMDGPUTargetLowering::LowerIntrinsicLRP(SDValue Op,
   SDLoc DL(Op);
   EVT VT = Op.getValueType();
   SDValue OneSubA = DAG.getNode(ISD::FSUB, DL, VT,
-                                DAG.getConstantFP(1.0f, MVT::f32),
+                                DAG.getConstantFP(1.0f, DL, MVT::f32),
                                 Op.getOperand(1));
   SDValue OneSubAC = DAG.getNode(ISD::FMUL, DL, VT, OneSubA,
                                                     Op.getOperand(3));
@@ -1144,7 +1167,7 @@ SDValue AMDGPUTargetLowering::CombineFMinMaxLegacy(SDLoc DL,
   return SDValue();
 }
 
-/// \brief Generate Min/Max node
+// FIXME: Remove this when combines added to DAGCombiner.
 SDValue AMDGPUTargetLowering::CombineIMinMax(SDLoc DL,
                                              EVT VT,
                                              SDValue LHS,
@@ -1160,22 +1183,22 @@ SDValue AMDGPUTargetLowering::CombineIMinMax(SDLoc DL,
   switch (CCOpcode) {
   case ISD::SETULE:
   case ISD::SETULT: {
-    unsigned Opc = (LHS == True) ? AMDGPUISD::UMIN : AMDGPUISD::UMAX;
+    unsigned Opc = (LHS == True) ? ISD::UMIN : ISD::UMAX;
     return DAG.getNode(Opc, DL, VT, LHS, RHS);
   }
   case ISD::SETLE:
   case ISD::SETLT: {
-    unsigned Opc = (LHS == True) ? AMDGPUISD::SMIN : AMDGPUISD::SMAX;
+    unsigned Opc = (LHS == True) ? ISD::SMIN : ISD::SMAX;
     return DAG.getNode(Opc, DL, VT, LHS, RHS);
   }
   case ISD::SETGT:
   case ISD::SETGE: {
-    unsigned Opc = (LHS == True) ? AMDGPUISD::SMAX : AMDGPUISD::SMIN;
+    unsigned Opc = (LHS == True) ? ISD::SMAX : ISD::SMIN;
     return DAG.getNode(Opc, DL, VT, LHS, RHS);
   }
   case ISD::SETUGE:
   case ISD::SETUGT: {
-    unsigned Opc = (LHS == True) ? AMDGPUISD::UMAX : AMDGPUISD::UMIN;
+    unsigned Opc = (LHS == True) ? ISD::UMAX : ISD::UMIN;
     return DAG.getNode(Opc, DL, VT, LHS, RHS);
   }
   default:
@@ -1203,7 +1226,7 @@ SDValue AMDGPUTargetLowering::ScalarizeVectorLoad(const SDValue Op,
 
   for (unsigned i = 0; i < NumElts; ++i) {
     SDValue Ptr = DAG.getNode(ISD::ADD, SL, PtrVT, Load->getBasePtr(),
-                              DAG.getConstant(i * MemEltSize, PtrVT));
+                              DAG.getConstant(i * MemEltSize, SL, PtrVT));
 
     SDValue NewLoad
       = DAG.getExtLoad(Load->getExtensionType(), SL, EltVT,
@@ -1254,7 +1277,8 @@ SDValue AMDGPUTargetLowering::SplitVectorLoad(const SDValue Op,
                      Load->isInvariant(), Load->getAlignment());
 
   SDValue HiPtr = DAG.getNode(ISD::ADD, SL, PtrVT, BasePtr,
-                              DAG.getConstant(LoMemVT.getStoreSize(), PtrVT));
+                              DAG.getConstant(LoMemVT.getStoreSize(), SL,
+                                              PtrVT));
 
   SDValue HiLoad
     = DAG.getExtLoad(Load->getExtensionType(), SL, HiVT,
@@ -1294,18 +1318,18 @@ SDValue AMDGPUTargetLowering::MergeVectorStore(const SDValue &Op,
   unsigned MemEltBits = MemEltVT.getSizeInBits();
   unsigned MemNumElements = MemVT.getVectorNumElements();
   unsigned PackedSize = MemVT.getStoreSizeInBits();
-  SDValue Mask = DAG.getConstant((1 << MemEltBits) - 1, MVT::i32);
+  SDValue Mask = DAG.getConstant((1 << MemEltBits) - 1, DL, MVT::i32);
 
   assert(Value.getValueType().getScalarSizeInBits() >= 32);
 
   SDValue PackedValue;
   for (unsigned i = 0; i < MemNumElements; ++i) {
     SDValue Elt = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, ElemVT, Value,
-                              DAG.getConstant(i, MVT::i32));
+                              DAG.getConstant(i, DL, MVT::i32));
     Elt = DAG.getZExtOrTrunc(Elt, DL, MVT::i32);
     Elt = DAG.getNode(ISD::AND, DL, MVT::i32, Elt, Mask); // getZeroExtendInReg
 
-    SDValue Shift = DAG.getConstant(MemEltBits * i, MVT::i32);
+    SDValue Shift = DAG.getConstant(MemEltBits * i, DL, MVT::i32);
     Elt = DAG.getNode(ISD::SHL, DL, MVT::i32, Elt, Shift);
 
     if (i == 0) {
@@ -1347,9 +1371,9 @@ SDValue AMDGPUTargetLowering::ScalarizeVectorStore(SDValue Op,
   for (unsigned i = 0, e = NumElts; i != e; ++i) {
     SDValue Val = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, EltVT,
                               Store->getValue(),
-                              DAG.getConstant(i, MVT::i32));
+                              DAG.getConstant(i, SL, MVT::i32));
 
-    SDValue Offset = DAG.getConstant(i * MemEltVT.getStoreSize(), PtrVT);
+    SDValue Offset = DAG.getConstant(i * MemEltVT.getStoreSize(), SL, PtrVT);
     SDValue Ptr = DAG.getNode(ISD::ADD, SL, PtrVT, Store->getBasePtr(), Offset);
     SDValue NewStore =
       DAG.getTruncStore(Store->getChain(), SL, Val, Ptr,
@@ -1388,7 +1412,8 @@ SDValue AMDGPUTargetLowering::SplitVectorStore(SDValue Op,
 
   EVT PtrVT = BasePtr.getValueType();
   SDValue HiPtr = DAG.getNode(ISD::ADD, SL, PtrVT, BasePtr,
-                              DAG.getConstant(LoMemVT.getStoreSize(), PtrVT));
+                              DAG.getConstant(LoMemVT.getStoreSize(), SL,
+                                              PtrVT));
 
   MachinePointerInfo SrcValue(Store->getMemOperand()->getValue());
   SDValue LoStore
@@ -1444,22 +1469,34 @@ SDValue AMDGPUTargetLowering::LowerLOAD(SDValue Op, SelectionDAG &DAG) const {
       ExtType == ISD::NON_EXTLOAD || Load->getMemoryVT().bitsGE(MVT::i32))
     return SDValue();
 
+  // <SI && AS=PRIVATE && EXTLOAD && size < 32bit,
+  // register (2-)byte extract.
 
+  // Get Register holding the target.
   SDValue Ptr = DAG.getNode(ISD::SRL, DL, MVT::i32, Load->getBasePtr(),
-                            DAG.getConstant(2, MVT::i32));
+                            DAG.getConstant(2, DL, MVT::i32));
+  // Load the Register.
   SDValue Ret = DAG.getNode(AMDGPUISD::REGISTER_LOAD, DL, Op.getValueType(),
                             Load->getChain(), Ptr,
-                            DAG.getTargetConstant(0, MVT::i32),
+                            DAG.getTargetConstant(0, DL, MVT::i32),
                             Op.getOperand(2));
+
+  // Get offset within the register.
   SDValue ByteIdx = DAG.getNode(ISD::AND, DL, MVT::i32,
                                 Load->getBasePtr(),
-                                DAG.getConstant(0x3, MVT::i32));
+                                DAG.getConstant(0x3, DL, MVT::i32));
+
+  // Bit offset of target byte (byteIdx * 8).
   SDValue ShiftAmt = DAG.getNode(ISD::SHL, DL, MVT::i32, ByteIdx,
-                                 DAG.getConstant(3, MVT::i32));
+                                 DAG.getConstant(3, DL, MVT::i32));
 
+  // Shift to the right.
   Ret = DAG.getNode(ISD::SRL, DL, MVT::i32, Ret, ShiftAmt);
 
+  // Eliminate the upper bits by setting them to ...
   EVT MemEltVT = MemVT.getScalarType();
+
+  // ... ones.
   if (ExtType == ISD::SEXTLOAD) {
     SDValue MemEltVTNode = DAG.getValueType(MemEltVT);
 
@@ -1471,6 +1508,7 @@ SDValue AMDGPUTargetLowering::LowerLOAD(SDValue Op, SelectionDAG &DAG) const {
     return DAG.getMergeValues(Ops, DL);
   }
 
+  // ... or zeros.
   SDValue Ops[] = {
     DAG.getZeroExtendInReg(Ret, DL, MemEltVT),
     Load->getChain()
@@ -1505,15 +1543,16 @@ SDValue AMDGPUTargetLowering::LowerSTORE(SDValue Op, SelectionDAG &DAG) const {
     }
     SDValue BasePtr = Store->getBasePtr();
     SDValue Ptr = DAG.getNode(ISD::SRL, DL, MVT::i32, BasePtr,
-                              DAG.getConstant(2, MVT::i32));
+                              DAG.getConstant(2, DL, MVT::i32));
     SDValue Dst = DAG.getNode(AMDGPUISD::REGISTER_LOAD, DL, MVT::i32,
-                              Chain, Ptr, DAG.getTargetConstant(0, MVT::i32));
+                              Chain, Ptr,
+                              DAG.getTargetConstant(0, DL, MVT::i32));
 
     SDValue ByteIdx = DAG.getNode(ISD::AND, DL, MVT::i32, BasePtr,
-                                  DAG.getConstant(0x3, MVT::i32));
+                                  DAG.getConstant(0x3, DL, MVT::i32));
 
     SDValue ShiftAmt = DAG.getNode(ISD::SHL, DL, MVT::i32, ByteIdx,
-                                   DAG.getConstant(3, MVT::i32));
+                                   DAG.getConstant(3, DL, MVT::i32));
 
     SDValue SExtValue = DAG.getNode(ISD::SIGN_EXTEND, DL, MVT::i32,
                                     Store->getValue());
@@ -1523,15 +1562,17 @@ SDValue AMDGPUTargetLowering::LowerSTORE(SDValue Op, SelectionDAG &DAG) const {
     SDValue ShiftedValue = DAG.getNode(ISD::SHL, DL, MVT::i32,
                                        MaskedValue, ShiftAmt);
 
-    SDValue DstMask = DAG.getNode(ISD::SHL, DL, MVT::i32, DAG.getConstant(Mask, MVT::i32),
+    SDValue DstMask = DAG.getNode(ISD::SHL, DL, MVT::i32,
+                                  DAG.getConstant(Mask, DL, MVT::i32),
                                   ShiftAmt);
     DstMask = DAG.getNode(ISD::XOR, DL, MVT::i32, DstMask,
-                          DAG.getConstant(0xffffffff, MVT::i32));
+                          DAG.getConstant(0xffffffff, DL, MVT::i32));
     Dst = DAG.getNode(ISD::AND, DL, MVT::i32, Dst, DstMask);
 
     SDValue Value = DAG.getNode(ISD::OR, DL, MVT::i32, Dst, ShiftedValue);
     return DAG.getNode(AMDGPUISD::REGISTER_STORE, DL, MVT::Other,
-                       Chain, Value, Ptr, DAG.getTargetConstant(0, MVT::i32));
+                       Chain, Value, Ptr,
+                       DAG.getTargetConstant(0, DL, MVT::i32));
   }
   return SDValue();
 }
@@ -1558,17 +1599,18 @@ SDValue AMDGPUTargetLowering::LowerDIVREM24(SDValue Op, SelectionDAG &DAG, bool
 
   unsigned BitSize = VT.getScalarType().getSizeInBits();
 
-  SDValue jq = DAG.getConstant(1, IntVT);
+  SDValue jq = DAG.getConstant(1, DL, IntVT);
 
   if (sign) {
     // char|short jq = ia ^ ib;
     jq = DAG.getNode(ISD::XOR, DL, VT, LHS, RHS);
 
     // jq = jq >> (bitsize - 2)
-    jq = DAG.getNode(ISD::SRA, DL, VT, jq, DAG.getConstant(BitSize - 2, VT));
+    jq = DAG.getNode(ISD::SRA, DL, VT, jq,
+                     DAG.getConstant(BitSize - 2, DL, VT));
 
     // jq = jq | 0x1
-    jq = DAG.getNode(ISD::OR, DL, VT, jq, DAG.getConstant(1, VT));
+    jq = DAG.getNode(ISD::OR, DL, VT, jq, DAG.getConstant(1, DL, VT));
 
     // jq = (int)jq
     jq = DAG.getSExtOrTrunc(jq, DL, IntVT);
@@ -1611,13 +1653,13 @@ SDValue AMDGPUTargetLowering::LowerDIVREM24(SDValue Op, SelectionDAG &DAG, bool
   // fb = fabs(fb);
   fb = DAG.getNode(ISD::FABS, DL, FltVT, fb);
 
-  EVT SetCCVT = getSetCCResultType(*DAG.getContext(), VT);
+  EVT SetCCVT = getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT);
 
   // int cv = fr >= fb;
   SDValue cv = DAG.getSetCC(DL, SetCCVT, fr, fb, ISD::SETOGE);
 
   // jq = (cv ? jq : 0);
-  jq = DAG.getNode(ISD::SELECT, DL, VT, cv, jq, DAG.getConstant(0, VT));
+  jq = DAG.getNode(ISD::SELECT, DL, VT, cv, jq, DAG.getConstant(0, DL, VT));
 
   // dst = trunc/extend to legal type
   iq = sign ? DAG.getSExtOrTrunc(iq, DL, VT) : DAG.getZExtOrTrunc(iq, DL, VT);
@@ -1645,8 +1687,8 @@ void AMDGPUTargetLowering::LowerUDIVREM64(SDValue Op,
   EVT VT = Op.getValueType();
   EVT HalfVT = VT.getHalfSizedIntegerVT(*DAG.getContext());
 
-  SDValue one = DAG.getConstant(1, HalfVT);
-  SDValue zero = DAG.getConstant(0, HalfVT);
+  SDValue one = DAG.getConstant(1, DL, HalfVT);
+  SDValue zero = DAG.getConstant(0, DL, HalfVT);
 
   //HiLo split
   SDValue LHS = Op.getOperand(0);
@@ -1685,18 +1727,18 @@ void AMDGPUTargetLowering::LowerUDIVREM64(SDValue Op,
 
   for (unsigned i = 0; i < halfBitWidth; ++i) {
     const unsigned bitPos = halfBitWidth - i - 1;
-    SDValue POS = DAG.getConstant(bitPos, HalfVT);
+    SDValue POS = DAG.getConstant(bitPos, DL, HalfVT);
     // Get value of high bit
     SDValue HBit = DAG.getNode(ISD::SRL, DL, HalfVT, LHS_Lo, POS);
     HBit = DAG.getNode(ISD::AND, DL, HalfVT, HBit, one);
     HBit = DAG.getNode(ISD::ZERO_EXTEND, DL, VT, HBit);
 
     // Shift
-    REM = DAG.getNode(ISD::SHL, DL, VT, REM, DAG.getConstant(1, VT));
+    REM = DAG.getNode(ISD::SHL, DL, VT, REM, DAG.getConstant(1, DL, VT));
     // Add LHS high bit
     REM = DAG.getNode(ISD::OR, DL, VT, REM, HBit);
 
-    SDValue BIT = DAG.getConstant(1 << bitPos, HalfVT);
+    SDValue BIT = DAG.getConstant(1 << bitPos, DL, HalfVT);
     SDValue realBIT = DAG.getSelectCC(DL, REM, RHS, BIT, zero, ISD::SETUGE);
 
     DIV_Lo = DAG.getNode(ISD::OR, DL, HalfVT, DIV_Lo, realBIT);
@@ -1746,11 +1788,11 @@ SDValue AMDGPUTargetLowering::LowerUDIVREM(SDValue Op,
   SDValue RCP_HI = DAG.getNode(ISD::MULHU, DL, VT, RCP, Den);
 
   // NEG_RCP_LO = -RCP_LO
-  SDValue NEG_RCP_LO = DAG.getNode(ISD::SUB, DL, VT, DAG.getConstant(0, VT),
+  SDValue NEG_RCP_LO = DAG.getNode(ISD::SUB, DL, VT, DAG.getConstant(0, DL, VT),
                                                      RCP_LO);
 
   // ABS_RCP_LO = (RCP_HI == 0 ? NEG_RCP_LO : RCP_LO)
-  SDValue ABS_RCP_LO = DAG.getSelectCC(DL, RCP_HI, DAG.getConstant(0, VT),
+  SDValue ABS_RCP_LO = DAG.getSelectCC(DL, RCP_HI, DAG.getConstant(0, DL, VT),
                                            NEG_RCP_LO, RCP_LO,
                                            ISD::SETEQ);
   // Calculate the rounding error from the URECIP instruction
@@ -1764,7 +1806,7 @@ SDValue AMDGPUTargetLowering::LowerUDIVREM(SDValue Op,
   SDValue RCP_S_E = DAG.getNode(ISD::SUB, DL, VT, RCP, E);
 
   // Tmp0 = (RCP_HI == 0 ? RCP_A_E : RCP_SUB_E)
-  SDValue Tmp0 = DAG.getSelectCC(DL, RCP_HI, DAG.getConstant(0, VT),
+  SDValue Tmp0 = DAG.getSelectCC(DL, RCP_HI, DAG.getConstant(0, DL, VT),
                                      RCP_A_E, RCP_S_E,
                                      ISD::SETEQ);
   // Quotient = mulhu(Tmp0, Num)
@@ -1778,14 +1820,14 @@ SDValue AMDGPUTargetLowering::LowerUDIVREM(SDValue Op,
 
   // Remainder_GE_Den = (Remainder >= Den ? -1 : 0)
   SDValue Remainder_GE_Den = DAG.getSelectCC(DL, Remainder, Den,
-                                                 DAG.getConstant(-1, VT),
-                                                 DAG.getConstant(0, VT),
+                                                 DAG.getConstant(-1, DL, VT),
+                                                 DAG.getConstant(0, DL, VT),
                                                  ISD::SETUGE);
   // Remainder_GE_Zero = (Num >= Num_S_Remainder ? -1 : 0)
   SDValue Remainder_GE_Zero = DAG.getSelectCC(DL, Num,
                                                   Num_S_Remainder,
-                                                  DAG.getConstant(-1, VT),
-                                                  DAG.getConstant(0, VT),
+                                                  DAG.getConstant(-1, DL, VT),
+                                                  DAG.getConstant(0, DL, VT),
                                                   ISD::SETUGE);
   // Tmp1 = Remainder_GE_Den & Remainder_GE_Zero
   SDValue Tmp1 = DAG.getNode(ISD::AND, DL, VT, Remainder_GE_Den,
@@ -1795,18 +1837,18 @@ SDValue AMDGPUTargetLowering::LowerUDIVREM(SDValue Op,
 
   // Quotient_A_One = Quotient + 1
   SDValue Quotient_A_One = DAG.getNode(ISD::ADD, DL, VT, Quotient,
-                                                         DAG.getConstant(1, VT));
+                                       DAG.getConstant(1, DL, VT));
 
   // Quotient_S_One = Quotient - 1
   SDValue Quotient_S_One = DAG.getNode(ISD::SUB, DL, VT, Quotient,
-                                                         DAG.getConstant(1, VT));
+                                       DAG.getConstant(1, DL, VT));
 
   // Div = (Tmp1 == 0 ? Quotient : Quotient_A_One)
-  SDValue Div = DAG.getSelectCC(DL, Tmp1, DAG.getConstant(0, VT),
+  SDValue Div = DAG.getSelectCC(DL, Tmp1, DAG.getConstant(0, DL, VT),
                                      Quotient, Quotient_A_One, ISD::SETEQ);
 
   // Div = (Remainder_GE_Zero == 0 ? Quotient_S_One : Div)
-  Div = DAG.getSelectCC(DL, Remainder_GE_Zero, DAG.getConstant(0, VT),
+  Div = DAG.getSelectCC(DL, Remainder_GE_Zero, DAG.getConstant(0, DL, VT),
                             Quotient_S_One, Div, ISD::SETEQ);
 
   // Calculate Rem result:
@@ -1818,11 +1860,11 @@ SDValue AMDGPUTargetLowering::LowerUDIVREM(SDValue Op,
   SDValue Remainder_A_Den = DAG.getNode(ISD::ADD, DL, VT, Remainder, Den);
 
   // Rem = (Tmp1 == 0 ? Remainder : Remainder_S_Den)
-  SDValue Rem = DAG.getSelectCC(DL, Tmp1, DAG.getConstant(0, VT),
+  SDValue Rem = DAG.getSelectCC(DL, Tmp1, DAG.getConstant(0, DL, VT),
                                     Remainder, Remainder_S_Den, ISD::SETEQ);
 
   // Rem = (Remainder_GE_Zero == 0 ? Remainder_A_Den : Rem)
-  Rem = DAG.getSelectCC(DL, Remainder_GE_Zero, DAG.getConstant(0, VT),
+  Rem = DAG.getSelectCC(DL, Remainder_GE_Zero, DAG.getConstant(0, DL, VT),
                             Remainder_A_Den, Rem, ISD::SETEQ);
   SDValue Ops[2] = {
     Div,
@@ -1839,8 +1881,8 @@ SDValue AMDGPUTargetLowering::LowerSDIVREM(SDValue Op,
   SDValue LHS = Op.getOperand(0);
   SDValue RHS = Op.getOperand(1);
 
-  SDValue Zero = DAG.getConstant(0, VT);
-  SDValue NegOne = DAG.getConstant(-1, VT);
+  SDValue Zero = DAG.getConstant(0, DL, VT);
+  SDValue NegOne = DAG.getConstant(-1, DL, VT);
 
   if (VT == MVT::i32 &&
       DAG.ComputeNumSignBits(LHS) > 8 &&
@@ -1915,10 +1957,11 @@ SDValue AMDGPUTargetLowering::LowerFCEIL(SDValue Op, SelectionDAG &DAG) const {
 
   SDValue Trunc = DAG.getNode(ISD::FTRUNC, SL, MVT::f64, Src);
 
-  const SDValue Zero = DAG.getConstantFP(0.0, MVT::f64);
-  const SDValue One = DAG.getConstantFP(1.0, MVT::f64);
+  const SDValue Zero = DAG.getConstantFP(0.0, SL, MVT::f64);
+  const SDValue One = DAG.getConstantFP(1.0, SL, MVT::f64);
 
-  EVT SetCCVT = getSetCCResultType(*DAG.getContext(), MVT::f64);
+  EVT SetCCVT =
+      getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), MVT::f64);
 
   SDValue Lt0 = DAG.getSetCC(SL, SetCCVT, Src, Zero, ISD::SETOGT);
   SDValue NeTrunc = DAG.getSetCC(SL, SetCCVT, Src, Trunc, ISD::SETONE);
@@ -1934,10 +1977,10 @@ static SDValue extractF64Exponent(SDValue Hi, SDLoc SL, SelectionDAG &DAG) {
 
   SDValue ExpPart = DAG.getNode(AMDGPUISD::BFE_U32, SL, MVT::i32,
                                 Hi,
-                                DAG.getConstant(FractBits - 32, MVT::i32),
-                                DAG.getConstant(ExpBits, MVT::i32));
+                                DAG.getConstant(FractBits - 32, SL, MVT::i32),
+                                DAG.getConstant(ExpBits, SL, MVT::i32));
   SDValue Exp = DAG.getNode(ISD::SUB, SL, MVT::i32, ExpPart,
-                            DAG.getConstant(1023, MVT::i32));
+                            DAG.getConstant(1023, SL, MVT::i32));
 
   return Exp;
 }
@@ -1948,8 +1991,8 @@ SDValue AMDGPUTargetLowering::LowerFTRUNC(SDValue Op, SelectionDAG &DAG) const {
 
   assert(Op.getValueType() == MVT::f64);
 
-  const SDValue Zero = DAG.getConstant(0, MVT::i32);
-  const SDValue One = DAG.getConstant(1, MVT::i32);
+  const SDValue Zero = DAG.getConstant(0, SL, MVT::i32);
+  const SDValue One = DAG.getConstant(1, SL, MVT::i32);
 
   SDValue VecSrc = DAG.getNode(ISD::BITCAST, SL, MVT::v2i32, Src);
 
@@ -1962,7 +2005,7 @@ SDValue AMDGPUTargetLowering::LowerFTRUNC(SDValue Op, SelectionDAG &DAG) const {
   const unsigned FractBits = 52;
 
   // Extract the sign bit.
-  const SDValue SignBitMask = DAG.getConstant(UINT32_C(1) << 31, MVT::i32);
+  const SDValue SignBitMask = DAG.getConstant(UINT32_C(1) << 31, SL, MVT::i32);
   SDValue SignBit = DAG.getNode(ISD::AND, SL, MVT::i32, Hi, SignBitMask);
 
   // Extend back to to 64-bits.
@@ -1972,15 +2015,16 @@ SDValue AMDGPUTargetLowering::LowerFTRUNC(SDValue Op, SelectionDAG &DAG) const {
 
   SDValue BcInt = DAG.getNode(ISD::BITCAST, SL, MVT::i64, Src);
   const SDValue FractMask
-    = DAG.getConstant((UINT64_C(1) << FractBits) - 1, MVT::i64);
+    = DAG.getConstant((UINT64_C(1) << FractBits) - 1, SL, MVT::i64);
 
   SDValue Shr = DAG.getNode(ISD::SRA, SL, MVT::i64, FractMask, Exp);
   SDValue Not = DAG.getNOT(SL, Shr, MVT::i64);
   SDValue Tmp0 = DAG.getNode(ISD::AND, SL, MVT::i64, BcInt, Not);
 
-  EVT SetCCVT = getSetCCResultType(*DAG.getContext(), MVT::i32);
+  EVT SetCCVT =
+      getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), MVT::i32);
 
-  const SDValue FiftyOne = DAG.getConstant(FractBits - 1, MVT::i32);
+  const SDValue FiftyOne = DAG.getConstant(FractBits - 1, SL, MVT::i32);
 
   SDValue ExpLt0 = DAG.getSetCC(SL, SetCCVT, Exp, Zero, ISD::SETLT);
   SDValue ExpGt51 = DAG.getSetCC(SL, SetCCVT, Exp, FiftyOne, ISD::SETGT);
@@ -1998,7 +2042,7 @@ SDValue AMDGPUTargetLowering::LowerFRINT(SDValue Op, SelectionDAG &DAG) const {
   assert(Op.getValueType() == MVT::f64);
 
   APFloat C1Val(APFloat::IEEEdouble, "0x1.0p+52");
-  SDValue C1 = DAG.getConstantFP(C1Val, MVT::f64);
+  SDValue C1 = DAG.getConstantFP(C1Val, SL, MVT::f64);
   SDValue CopySign = DAG.getNode(ISD::FCOPYSIGN, SL, MVT::f64, C1, Src);
 
   SDValue Tmp1 = DAG.getNode(ISD::FADD, SL, MVT::f64, Src, CopySign);
@@ -2007,9 +2051,10 @@ SDValue AMDGPUTargetLowering::LowerFRINT(SDValue Op, SelectionDAG &DAG) const {
   SDValue Fabs = DAG.getNode(ISD::FABS, SL, MVT::f64, Src);
 
   APFloat C2Val(APFloat::IEEEdouble, "0x1.fffffffffffffp+51");
-  SDValue C2 = DAG.getConstantFP(C2Val, MVT::f64);
+  SDValue C2 = DAG.getConstantFP(C2Val, SL, MVT::f64);
 
-  EVT SetCCVT = getSetCCResultType(*DAG.getContext(), MVT::f64);
+  EVT SetCCVT =
+      getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), MVT::f64);
   SDValue Cond = DAG.getSetCC(SL, SetCCVT, Fabs, C2, ISD::SETOGT);
 
   return DAG.getSelect(SL, MVT::f64, Cond, Src, Tmp2);
@@ -2033,13 +2078,14 @@ SDValue AMDGPUTargetLowering::LowerFROUND32(SDValue Op, SelectionDAG &DAG) const
 
   SDValue AbsDiff = DAG.getNode(ISD::FABS, SL, MVT::f32, Diff);
 
-  const SDValue Zero = DAG.getConstantFP(0.0, MVT::f32);
-  const SDValue One = DAG.getConstantFP(1.0, MVT::f32);
-  const SDValue Half = DAG.getConstantFP(0.5, MVT::f32);
+  const SDValue Zero = DAG.getConstantFP(0.0, SL, MVT::f32);
+  const SDValue One = DAG.getConstantFP(1.0, SL, MVT::f32);
+  const SDValue Half = DAG.getConstantFP(0.5, SL, MVT::f32);
 
   SDValue SignOne = DAG.getNode(ISD::FCOPYSIGN, SL, MVT::f32, One, X);
 
-  EVT SetCCVT = getSetCCResultType(*DAG.getContext(), MVT::f32);
+  EVT SetCCVT =
+      getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), MVT::f32);
 
   SDValue Cmp = DAG.getSetCC(SL, SetCCVT, AbsDiff, Half, ISD::SETOGE);
 
@@ -2054,12 +2100,12 @@ SDValue AMDGPUTargetLowering::LowerFROUND64(SDValue Op, SelectionDAG &DAG) const
 
   SDValue L = DAG.getNode(ISD::BITCAST, SL, MVT::i64, X);
 
-  const SDValue Zero = DAG.getConstant(0, MVT::i32);
-  const SDValue One = DAG.getConstant(1, MVT::i32);
-  const SDValue NegOne = DAG.getConstant(-1, MVT::i32);
-  const SDValue FiftyOne = DAG.getConstant(51, MVT::i32);
-  EVT SetCCVT = getSetCCResultType(*DAG.getContext(), MVT::i32);
-
+  const SDValue Zero = DAG.getConstant(0, SL, MVT::i32);
+  const SDValue One = DAG.getConstant(1, SL, MVT::i32);
+  const SDValue NegOne = DAG.getConstant(-1, SL, MVT::i32);
+  const SDValue FiftyOne = DAG.getConstant(51, SL, MVT::i32);
+  EVT SetCCVT =
+      getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), MVT::i32);
 
   SDValue BC = DAG.getNode(ISD::BITCAST, SL, MVT::v2i32, X);
 
@@ -2067,20 +2113,22 @@ SDValue AMDGPUTargetLowering::LowerFROUND64(SDValue Op, SelectionDAG &DAG) const
 
   SDValue Exp = extractF64Exponent(Hi, SL, DAG);
 
-  const SDValue Mask = DAG.getConstant(INT64_C(0x000fffffffffffff), MVT::i64);
+  const SDValue Mask = DAG.getConstant(INT64_C(0x000fffffffffffff), SL,
+                                       MVT::i64);
 
   SDValue M = DAG.getNode(ISD::SRA, SL, MVT::i64, Mask, Exp);
   SDValue D = DAG.getNode(ISD::SRA, SL, MVT::i64,
-                          DAG.getConstant(INT64_C(0x0008000000000000), MVT::i64),
+                          DAG.getConstant(INT64_C(0x0008000000000000), SL,
+                                          MVT::i64),
                           Exp);
 
   SDValue Tmp0 = DAG.getNode(ISD::AND, SL, MVT::i64, L, M);
   SDValue Tmp1 = DAG.getSetCC(SL, SetCCVT,
-                              DAG.getConstant(0, MVT::i64), Tmp0,
+                              DAG.getConstant(0, SL, MVT::i64), Tmp0,
                               ISD::SETNE);
 
   SDValue Tmp2 = DAG.getNode(ISD::SELECT, SL, MVT::i64, Tmp1,
-                             D, DAG.getConstant(0, MVT::i64));
+                             D, DAG.getConstant(0, SL, MVT::i64));
   SDValue K = DAG.getNode(ISD::ADD, SL, MVT::i64, L, Tmp2);
 
   K = DAG.getNode(ISD::AND, SL, MVT::i64, K, DAG.getNOT(SL, M, MVT::i64));
@@ -2092,8 +2140,8 @@ SDValue AMDGPUTargetLowering::LowerFROUND64(SDValue Op, SelectionDAG &DAG) const
 
   SDValue Mag = DAG.getNode(ISD::SELECT, SL, MVT::f64,
                             ExpEqNegOne,
-                            DAG.getConstantFP(1.0, MVT::f64),
-                            DAG.getConstantFP(0.0, MVT::f64));
+                            DAG.getConstantFP(1.0, SL, MVT::f64),
+                            DAG.getConstantFP(0.0, SL, MVT::f64));
 
   SDValue S = DAG.getNode(ISD::FCOPYSIGN, SL, MVT::f64, Mag, X);
 
@@ -2125,10 +2173,11 @@ SDValue AMDGPUTargetLowering::LowerFFLOOR(SDValue Op, SelectionDAG &DAG) const {
 
   SDValue Trunc = DAG.getNode(ISD::FTRUNC, SL, MVT::f64, Src);
 
-  const SDValue Zero = DAG.getConstantFP(0.0, MVT::f64);
-  const SDValue NegOne = DAG.getConstantFP(-1.0, MVT::f64);
+  const SDValue Zero = DAG.getConstantFP(0.0, SL, MVT::f64);
+  const SDValue NegOne = DAG.getConstantFP(-1.0, SL, MVT::f64);
 
-  EVT SetCCVT = getSetCCResultType(*DAG.getContext(), MVT::f64);
+  EVT SetCCVT =
+      getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), MVT::f64);
 
   SDValue Lt0 = DAG.getSetCC(SL, SetCCVT, Src, Zero, ISD::SETOLT);
   SDValue NeTrunc = DAG.getSetCC(SL, SetCCVT, Src, Trunc, ISD::SETONE);
@@ -2146,9 +2195,9 @@ SDValue AMDGPUTargetLowering::LowerINT_TO_FP64(SDValue Op, SelectionDAG &DAG,
   SDValue BC = DAG.getNode(ISD::BITCAST, SL, MVT::v2i32, Src);
 
   SDValue Lo = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, MVT::i32, BC,
-                           DAG.getConstant(0, MVT::i32));
+                           DAG.getConstant(0, SL, MVT::i32));
   SDValue Hi = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, MVT::i32, BC,
-                           DAG.getConstant(1, MVT::i32));
+                           DAG.getConstant(1, SL, MVT::i32));
 
   SDValue CvtHi = DAG.getNode(Signed ? ISD::SINT_TO_FP : ISD::UINT_TO_FP,
                               SL, MVT::f64, Hi);
@@ -2156,7 +2205,7 @@ SDValue AMDGPUTargetLowering::LowerINT_TO_FP64(SDValue Op, SelectionDAG &DAG,
   SDValue CvtLo = DAG.getNode(ISD::UINT_TO_FP, SL, MVT::f64, Lo);
 
   SDValue LdExp = DAG.getNode(AMDGPUISD::LDEXP, SL, MVT::f64, CvtHi,
-                              DAG.getConstant(32, MVT::i32));
+                              DAG.getConstant(32, SL, MVT::i32));
 
   return DAG.getNode(ISD::FADD, SL, MVT::f64, LdExp, CvtLo);
 }
@@ -2177,13 +2226,13 @@ SDValue AMDGPUTargetLowering::LowerUINT_TO_FP(SDValue Op,
 
   // f32 uint_to_fp i64
   SDValue Lo = DAG.getNode(ISD::EXTRACT_ELEMENT, DL, MVT::i32, S0,
-                           DAG.getConstant(0, MVT::i32));
+                           DAG.getConstant(0, DL, MVT::i32));
   SDValue FloatLo = DAG.getNode(ISD::UINT_TO_FP, DL, MVT::f32, Lo);
   SDValue Hi = DAG.getNode(ISD::EXTRACT_ELEMENT, DL, MVT::i32, S0,
-                           DAG.getConstant(1, MVT::i32));
+                           DAG.getConstant(1, DL, MVT::i32));
   SDValue FloatHi = DAG.getNode(ISD::UINT_TO_FP, DL, MVT::f32, Hi);
   FloatHi = DAG.getNode(ISD::FMUL, DL, MVT::f32, FloatHi,
-                        DAG.getConstantFP(4294967296.0f, MVT::f32)); // 2^32
+                        DAG.getConstantFP(4294967296.0f, DL, MVT::f32)); // 2^32
   return DAG.getNode(ISD::FADD, DL, MVT::f32, FloatLo, FloatHi);
 }
 
@@ -2204,10 +2253,10 @@ SDValue AMDGPUTargetLowering::LowerFP64_TO_INT(SDValue Op, SelectionDAG &DAG,
 
   SDValue Trunc = DAG.getNode(ISD::FTRUNC, SL, MVT::f64, Src);
 
-  SDValue K0
-    = DAG.getConstantFP(BitsToDouble(UINT64_C(0x3df0000000000000)), MVT::f64);
-  SDValue K1
-    = DAG.getConstantFP(BitsToDouble(UINT64_C(0xc1f0000000000000)), MVT::f64);
+  SDValue K0 = DAG.getConstantFP(BitsToDouble(UINT64_C(0x3df0000000000000)), SL,
+                                 MVT::f64);
+  SDValue K1 = DAG.getConstantFP(BitsToDouble(UINT64_C(0xc1f0000000000000)), SL,
+                                 MVT::f64);
 
   SDValue Mul = DAG.getNode(ISD::FMUL, SL, MVT::f64, Trunc, K0);
 
@@ -2306,14 +2355,14 @@ static void simplifyI24(SDValue Op, TargetLowering::DAGCombinerInfo &DCI) {
 
 template <typename IntTy>
 static SDValue constantFoldBFE(SelectionDAG &DAG, IntTy Src0,
-                               uint32_t Offset, uint32_t Width) {
+                               uint32_t Offset, uint32_t Width, SDLoc DL) {
   if (Width + Offset < 32) {
     uint32_t Shl = static_cast<uint32_t>(Src0) << (32 - Offset - Width);
     IntTy Result = static_cast<IntTy>(Shl) >> (32 - Width);
-    return DAG.getConstant(Result, MVT::i32);
+    return DAG.getConstant(Result, DL, MVT::i32);
   }
 
-  return DAG.getConstant(Src0 >> Offset, MVT::i32);
+  return DAG.getConstant(Src0 >> Offset, DL, MVT::i32);
 }
 
 static bool usesAllNormalStores(SDNode *LoadVal) {
@@ -2367,6 +2416,33 @@ SDValue AMDGPUTargetLowering::performStoreCombine(SDNode *N,
                       SN->getBasePtr(), SN->getMemOperand());
 }
 
+SDValue AMDGPUTargetLowering::performShlCombine(SDNode *N,
+                                                DAGCombinerInfo &DCI) const {
+  if (N->getValueType(0) != MVT::i64)
+    return SDValue();
+
+  // i64 (shl x, 32) -> (build_pair 0, x)
+
+  // Doing this with moves theoretically helps MI optimizations that understand
+  // copies. 2 v_mov_b32_e32 will have the same code size / cycle count as
+  // v_lshl_b64. In the SALU case, I think this is slightly worse since it
+  // doubles the code size and I'm unsure about cycle count.
+  const ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N->getOperand(1));
+  if (!RHS || RHS->getZExtValue() != 32)
+    return SDValue();
+
+  SDValue LHS = N->getOperand(0);
+
+  SDLoc SL(N);
+  SelectionDAG &DAG = DCI.DAG;
+
+  // Extract low 32-bits.
+  SDValue Lo = DAG.getNode(ISD::TRUNCATE, SL, MVT::i32, LHS);
+
+  const SDValue Zero = DAG.getConstant(0, SL, MVT::i32);
+  return DAG.getNode(ISD::BUILD_PAIR, SL, MVT::i64, Zero, Lo);
+}
+
 SDValue AMDGPUTargetLowering::performMulCombine(SDNode *N,
                                                 DAGCombinerInfo &DCI) const {
   EVT VT = N->getValueType(0);
@@ -2404,21 +2480,27 @@ SDValue AMDGPUTargetLowering::PerformDAGCombine(SDNode *N,
   SDLoc DL(N);
 
   switch(N->getOpcode()) {
-    default: break;
-    case ISD::MUL:
-      return performMulCombine(N, DCI);
-    case AMDGPUISD::MUL_I24:
-    case AMDGPUISD::MUL_U24: {
-      SDValue N0 = N->getOperand(0);
-      SDValue N1 = N->getOperand(1);
-      simplifyI24(N0, DCI);
-      simplifyI24(N1, DCI);
-      return SDValue();
-    }
+  default:
+    break;
+  case ISD::SHL: {
+    if (DCI.getDAGCombineLevel() < AfterLegalizeDAG)
+      break;
+
+    return performShlCombine(N, DCI);
+  }
+  case ISD::MUL:
+    return performMulCombine(N, DCI);
+  case AMDGPUISD::MUL_I24:
+  case AMDGPUISD::MUL_U24: {
+    SDValue N0 = N->getOperand(0);
+    SDValue N1 = N->getOperand(1);
+    simplifyI24(N0, DCI);
+    simplifyI24(N1, DCI);
+    return SDValue();
+  }
   case ISD::SELECT: {
     SDValue Cond = N->getOperand(0);
     if (Cond.getOpcode() == ISD::SETCC && Cond.hasOneUse()) {
-      SDLoc DL(N);
       EVT VT = N->getValueType(0);
       SDValue LHS = Cond.getOperand(0);
       SDValue RHS = Cond.getOperand(1);
@@ -2449,7 +2531,7 @@ SDValue AMDGPUTargetLowering::PerformDAGCombine(SDNode *N,
 
     uint32_t WidthVal = Width->getZExtValue() & 0x1f;
     if (WidthVal == 0)
-      return DAG.getConstant(0, MVT::i32);
+      return DAG.getConstant(0, DL, MVT::i32);
 
     ConstantSDNode *Offset = dyn_cast<ConstantSDNode>(N->getOperand(1));
     if (!Offset)
@@ -2488,17 +2570,19 @@ SDValue AMDGPUTargetLowering::PerformDAGCombine(SDNode *N,
         return constantFoldBFE<int32_t>(DAG,
                                         CVal->getSExtValue(),
                                         OffsetVal,
-                                        WidthVal);
+                                        WidthVal,
+                                        DL);
       }
 
       return constantFoldBFE<uint32_t>(DAG,
                                        CVal->getZExtValue(),
                                        OffsetVal,
-                                       WidthVal);
+                                       WidthVal,
+                                       DL);
     }
 
     if ((OffsetVal + WidthVal) >= 32) {
-      SDValue ShiftVal = DAG.getConstant(OffsetVal, MVT::i32);
+      SDValue ShiftVal = DAG.getConstant(OffsetVal, DL, MVT::i32);
       return DAG.getNode(Signed ? ISD::SRA : ISD::SRL, DL, MVT::i32,
                          BitsFrom, ShiftVal);
     }
@@ -2599,11 +2683,23 @@ SDValue AMDGPUTargetLowering::CreateLiveInRegister(SelectionDAG &DAG,
   return DAG.getRegister(VirtualRegister, VT);
 }
 
+uint32_t AMDGPUTargetLowering::getImplicitParameterOffset(
+    const AMDGPUMachineFunction *MFI, const ImplicitParameter Param) const {
+  uint64_t ArgOffset = MFI->ABIArgOffset;
+  switch (Param) {
+  case GRID_DIM:
+    return ArgOffset;
+  case GRID_OFFSET:
+    return ArgOffset + 4;
+  }
+  llvm_unreachable("unexpected implicit parameter type");
+}
+
 #define NODE_NAME_CASE(node) case AMDGPUISD::node: return #node;
 
 const char* AMDGPUTargetLowering::getTargetNodeName(unsigned Opcode) const {
-  switch (Opcode) {
-  default: return nullptr;
+  switch ((AMDGPUISD::NodeType)Opcode) {
+  case AMDGPUISD::FIRST_NUMBER: break;
   // AMDIL DAG nodes
   NODE_NAME_CASE(CALL);
   NODE_NAME_CASE(UMUL);
@@ -2614,12 +2710,10 @@ const char* AMDGPUTargetLowering::getTargetNodeName(unsigned Opcode) const {
   NODE_NAME_CASE(DWORDADDR)
   NODE_NAME_CASE(FRACT)
   NODE_NAME_CASE(CLAMP)
+  NODE_NAME_CASE(COS_HW)
+  NODE_NAME_CASE(SIN_HW)
   NODE_NAME_CASE(FMAX_LEGACY)
-  NODE_NAME_CASE(SMAX)
-  NODE_NAME_CASE(UMAX)
   NODE_NAME_CASE(FMIN_LEGACY)
-  NODE_NAME_CASE(SMIN)
-  NODE_NAME_CASE(UMIN)
   NODE_NAME_CASE(FMAX3)
   NODE_NAME_CASE(SMAX3)
   NODE_NAME_CASE(UMAX3)
@@ -2638,6 +2732,8 @@ const char* AMDGPUTargetLowering::getTargetNodeName(unsigned Opcode) const {
   NODE_NAME_CASE(LDEXP)
   NODE_NAME_CASE(FP_CLASS)
   NODE_NAME_CASE(DOT4)
+  NODE_NAME_CASE(CARRY)
+  NODE_NAME_CASE(BORROW)
   NODE_NAME_CASE(BFE_U32)
   NODE_NAME_CASE(BFE_I32)
   NODE_NAME_CASE(BFI)
@@ -2647,6 +2743,7 @@ const char* AMDGPUTargetLowering::getTargetNodeName(unsigned Opcode) const {
   NODE_NAME_CASE(MUL_I24)
   NODE_NAME_CASE(MAD_U24)
   NODE_NAME_CASE(MAD_I24)
+  NODE_NAME_CASE(TEXTURE_FETCH)
   NODE_NAME_CASE(EXPORT)
   NODE_NAME_CASE(CONST_ADDRESS)
   NODE_NAME_CASE(REGISTER_LOAD)
@@ -2663,9 +2760,16 @@ const char* AMDGPUTargetLowering::getTargetNodeName(unsigned Opcode) const {
   NODE_NAME_CASE(CVT_F32_UBYTE3)
   NODE_NAME_CASE(BUILD_VERTICAL_VECTOR)
   NODE_NAME_CASE(CONST_DATA_PTR)
+  case AMDGPUISD::FIRST_MEM_OPCODE_NUMBER: break;
+  NODE_NAME_CASE(SENDMSG)
+  NODE_NAME_CASE(INTERP_MOV)
+  NODE_NAME_CASE(INTERP_P1)
+  NODE_NAME_CASE(INTERP_P2)
   NODE_NAME_CASE(STORE_MSKOR)
   NODE_NAME_CASE(TBUFFER_STORE_FORMAT)
+  case AMDGPUISD::LAST_AMDGPU_ISD_NUMBER: break;
   }
+  return nullptr;
 }
 
 SDValue AMDGPUTargetLowering::getRsqrtEstimate(SDValue Operand,
@@ -2755,13 +2859,11 @@ void AMDGPUTargetLowering::computeKnownBitsForTargetNode(
 
     break;
   }
-  case AMDGPUISD::SMAX:
-  case AMDGPUISD::UMAX:
-  case AMDGPUISD::SMIN:
-  case AMDGPUISD::UMIN:
-    computeKnownBitsForMinMax(Op.getOperand(0), Op.getOperand(1),
-                              KnownZero, KnownOne, DAG, Depth);
+  case AMDGPUISD::CARRY:
+  case AMDGPUISD::BORROW: {
+    KnownZero = APInt::getHighBitsSet(32, 31);
     break;
+  }
 
   case AMDGPUISD::BFE_I32:
   case AMDGPUISD::BFE_U32: {
@@ -2805,6 +2907,10 @@ unsigned AMDGPUTargetLowering::ComputeNumSignBitsForTargetNode(
     return Width ? 32 - (Width->getZExtValue() & 0x1f) : 1;
   }
 
+  case AMDGPUISD::CARRY:
+  case AMDGPUISD::BORROW:
+    return 31;
+
   default:
     return 1;
   }
diff --git a/lib/Target/R600/AMDGPUISelLowering.h b/lib/Target/AMDGPU/AMDGPUISelLowering.h
similarity index 94%
rename from lib/Target/R600/AMDGPUISelLowering.h
rename to lib/Target/AMDGPU/AMDGPUISelLowering.h
index 6bc6ca5bbc5..1ce391e9816 100644
--- a/lib/Target/R600/AMDGPUISelLowering.h
+++ b/lib/Target/AMDGPU/AMDGPUISelLowering.h
@@ -65,6 +65,7 @@ class AMDGPUTargetLowering : public TargetLowering {
   SDValue LowerSIGN_EXTEND_INREG(SDValue Op, SelectionDAG &DAG) const;
 
   SDValue performStoreCombine(SDNode *N, DAGCombinerInfo &DCI) const;
+  SDValue performShlCombine(SDNode *N, DAGCombinerInfo &DCI) const;
   SDValue performMulCombine(SDNode *N, DAGCombinerInfo &DCI) const;
 
 protected:
@@ -123,7 +124,7 @@ class AMDGPUTargetLowering : public TargetLowering {
 
   bool isNarrowingProfitable(EVT VT1, EVT VT2) const override;
 
-  MVT getVectorIdxTy() const override;
+  MVT getVectorIdxTy(const DataLayout &) const override;
   bool isSelectSupported(SelectSupportKind) const override;
 
   bool isFPImmLegal(const APFloat &Imm, EVT VT) const override;
@@ -133,6 +134,10 @@ class AMDGPUTargetLowering : public TargetLowering {
                              EVT ExtVT) const override;
 
   bool isLoadBitCastBeneficial(EVT, EVT) const override;
+
+  bool storeOfVectorConstantIsCheap(EVT MemVT,
+                                    unsigned NumElem,
+                                    unsigned AS) const override;
   bool isCheapToSpeculateCttz() const override;
   bool isCheapToSpeculateCtlz() const override;
 
@@ -203,11 +208,21 @@ class AMDGPUTargetLowering : public TargetLowering {
   virtual SDValue CreateLiveInRegister(SelectionDAG &DAG,
                                        const TargetRegisterClass *RC,
                                        unsigned Reg, EVT VT) const;
+
+  enum ImplicitParameter {
+    GRID_DIM,
+    GRID_OFFSET
+  };
+
+  /// \brief Helper function that returns the byte offset of the given
+  /// type of implicit parameter.
+  uint32_t getImplicitParameterOffset(const AMDGPUMachineFunction *MFI,
+                                      const ImplicitParameter Param) const;
 };
 
 namespace AMDGPUISD {
 
-enum {
+enum NodeType : unsigned {
   // AMDIL ISD Opcodes
   FIRST_NUMBER = ISD::BUILTIN_OP_END,
   CALL,        // Function call based on a single integer
@@ -224,11 +239,7 @@ enum {
   COS_HW,
   SIN_HW,
   FMAX_LEGACY,
-  SMAX,
-  UMAX,
   FMIN_LEGACY,
-  SMIN,
-  UMIN,
   FMAX3,
   SMAX3,
   UMAX3,
@@ -250,6 +261,8 @@ enum {
   LDEXP,
   FP_CLASS,
   DOT4,
+  CARRY,
+  BORROW,
   BFE_U32, // Extract range of bits with zero extension to 32-bits.
   BFE_I32, // Extract range of bits with sign extension to 32-bits.
   BFI, // (src0 & src1) | (~src0 & src2)
@@ -286,6 +299,10 @@ enum {
   BUILD_VERTICAL_VECTOR,
   /// Pointer to the start of the shader's constant data.
   CONST_DATA_PTR,
+  SENDMSG,
+  INTERP_MOV,
+  INTERP_P1,
+  INTERP_P2,
   FIRST_MEM_OPCODE_NUMBER = ISD::FIRST_TARGET_MEMORY_OPCODE,
   STORE_MSKOR,
   LOAD_CONSTANT,
diff --git a/lib/Target/R600/AMDGPUInstrInfo.cpp b/lib/Target/AMDGPU/AMDGPUInstrInfo.cpp
similarity index 91%
rename from lib/Target/R600/AMDGPUInstrInfo.cpp
rename to lib/Target/AMDGPU/AMDGPUInstrInfo.cpp
index f0f10ca5972..85e0ac5a07c 100644
--- a/lib/Target/R600/AMDGPUInstrInfo.cpp
+++ b/lib/Target/AMDGPU/AMDGPUInstrInfo.cpp
@@ -152,25 +152,18 @@ bool AMDGPUInstrInfo::expandPostRAPseudo (MachineBasicBlock::iterator MI) const
   return true;
 }
 
-MachineInstr *AMDGPUInstrInfo::foldMemoryOperandImpl(MachineFunction &MF,
-                                                     MachineInstr *MI,
-                                                     ArrayRef<unsigned> Ops,
-                                                     int FrameIndex) const {
+MachineInstr *AMDGPUInstrInfo::foldMemoryOperandImpl(
+    MachineFunction &MF, MachineInstr *MI, ArrayRef<unsigned> Ops,
+    MachineBasicBlock::iterator InsertPt, int FrameIndex) const {
 // TODO: Implement this function
   return nullptr;
 }
-MachineInstr *
-AMDGPUInstrInfo::foldMemoryOperandImpl(MachineFunction &MF, MachineInstr *MI,
-                                       ArrayRef<unsigned> Ops,
-                                       MachineInstr *LoadMI) const {
+MachineInstr *AMDGPUInstrInfo::foldMemoryOperandImpl(
+    MachineFunction &MF, MachineInstr *MI, ArrayRef<unsigned> Ops,
+    MachineBasicBlock::iterator InsertPt, MachineInstr *LoadMI) const {
   // TODO: Implement this function
   return nullptr;
 }
-bool AMDGPUInstrInfo::canFoldMemoryOperand(const MachineInstr *MI,
-                                           ArrayRef<unsigned> Ops) const {
-  // TODO: Implement this function
-  return false;
-}
 bool
 AMDGPUInstrInfo::unfoldMemoryOperand(MachineFunction &MF, MachineInstr *MI,
                                  unsigned Reg, bool UnfoldLoad,
@@ -236,10 +229,9 @@ bool AMDGPUInstrInfo::isPredicated(const MachineInstr *MI) const {
   // TODO: Implement this function
   return false;
 }
-bool
-AMDGPUInstrInfo::SubsumesPredicate(const SmallVectorImpl<MachineOperand> &Pred1,
-                                  const SmallVectorImpl<MachineOperand> &Pred2)
-  const {
+
+bool AMDGPUInstrInfo::SubsumesPredicate(ArrayRef<MachineOperand> Pred1,
+                                        ArrayRef<MachineOperand> Pred2) const {
   // TODO: Implement this function
   return false;
 }
@@ -370,3 +362,14 @@ int AMDGPUInstrInfo::pseudoToMCOpcode(int Opcode) const {
 
   return MCOp;
 }
+
+ArrayRef<std::pair<int, const char *>>
+AMDGPUInstrInfo::getSerializableTargetIndices() const {
+  static std::pair<int, const char *> TargetIndices[] = {
+      {AMDGPU::TI_CONSTDATA_START, "amdgpu-constdata-start"},
+      {AMDGPU::TI_SCRATCH_RSRC_DWORD0, "amdgpu-scratch-rsrc-dword0"},
+      {AMDGPU::TI_SCRATCH_RSRC_DWORD1, "amdgpu-scratch-rsrc-dword1"},
+      {AMDGPU::TI_SCRATCH_RSRC_DWORD2, "amdgpu-scratch-rsrc-dword2"},
+      {AMDGPU::TI_SCRATCH_RSRC_DWORD3, "amdgpu-scratch-rsrc-dword3"}};
+  return makeArrayRef(TargetIndices);
+}
diff --git a/lib/Target/R600/AMDGPUInstrInfo.h b/lib/Target/AMDGPU/AMDGPUInstrInfo.h
similarity index 95%
rename from lib/Target/R600/AMDGPUInstrInfo.h
rename to lib/Target/AMDGPU/AMDGPUInstrInfo.h
index 07042b59be7..bbfa70ecffb 100644
--- a/lib/Target/R600/AMDGPUInstrInfo.h
+++ b/lib/Target/AMDGPU/AMDGPUInstrInfo.h
@@ -87,9 +87,11 @@ class AMDGPUInstrInfo : public AMDGPUGenInstrInfo {
 protected:
   MachineInstr *foldMemoryOperandImpl(MachineFunction &MF, MachineInstr *MI,
                                       ArrayRef<unsigned> Ops,
+                                      MachineBasicBlock::iterator InsertPt,
                                       int FrameIndex) const override;
   MachineInstr *foldMemoryOperandImpl(MachineFunction &MF, MachineInstr *MI,
                                       ArrayRef<unsigned> Ops,
+                                      MachineBasicBlock::iterator InsertPt,
                                       MachineInstr *LoadMI) const override;
 
 public:
@@ -101,8 +103,6 @@ class AMDGPUInstrInfo : public AMDGPUGenInstrInfo {
   /// read or write or -1 if indirect addressing is not used by this program.
   int getIndirectIndexEnd(const MachineFunction &MF) const;
 
-  bool canFoldMemoryOperand(const MachineInstr *MI,
-                            ArrayRef<unsigned> Ops) const override;
   bool unfoldMemoryOperand(MachineFunction &MF, MachineInstr *MI,
                         unsigned Reg, bool UnfoldLoad, bool UnfoldStore,
                         SmallVectorImpl<MachineInstr *> &NewMIs) const override;
@@ -123,8 +123,8 @@ class AMDGPUInstrInfo : public AMDGPUGenInstrInfo {
   void insertNoop(MachineBasicBlock &MBB,
                   MachineBasicBlock::iterator MI) const override;
   bool isPredicated(const MachineInstr *MI) const override;
-  bool SubsumesPredicate(const SmallVectorImpl<MachineOperand> &Pred1,
-                   const SmallVectorImpl<MachineOperand> &Pred2) const override;
+  bool SubsumesPredicate(ArrayRef<MachineOperand> Pred1,
+                         ArrayRef<MachineOperand> Pred2) const override;
   bool DefinesPredicate(MachineInstr *MI,
                         std::vector<MachineOperand> &Pred) const override;
   bool isPredicable(MachineInstr *MI) const override;
@@ -145,6 +145,9 @@ class AMDGPUInstrInfo : public AMDGPUGenInstrInfo {
     return get(pseudoToMCOpcode(Opcode));
   }
 
+  ArrayRef<std::pair<int, const char *>>
+  getSerializableTargetIndices() const override;
+
 //===---------------------------------------------------------------------===//
 // Pure virtual funtions to be implemented by sub-classes.
 //===---------------------------------------------------------------------===//
diff --git a/lib/Target/R600/AMDGPUInstrInfo.td b/lib/Target/AMDGPU/AMDGPUInstrInfo.td
similarity index 90%
rename from lib/Target/R600/AMDGPUInstrInfo.td
rename to lib/Target/AMDGPU/AMDGPUInstrInfo.td
index 901eb5110f2..b413897d9d2 100644
--- a/lib/Target/R600/AMDGPUInstrInfo.td
+++ b/lib/Target/AMDGPU/AMDGPUInstrInfo.td
@@ -94,16 +94,6 @@ def AMDGPUfmin_legacy : SDNode<"AMDGPUISD::FMIN_LEGACY", SDTFPBinOp,
   []
 >;
 
-// out = min(a, b) a and b are signed ints
-def AMDGPUsmin : SDNode<"AMDGPUISD::SMIN", SDTIntBinOp,
-  [SDNPCommutative, SDNPAssociative]
->;
-
-// out = min(a, b) a and b are unsigned ints
-def AMDGPUumin : SDNode<"AMDGPUISD::UMIN", SDTIntBinOp,
-  [SDNPCommutative, SDNPAssociative]
->;
-
 // FIXME: TableGen doesn't like commutative instructions with more
 // than 2 operands.
 // out = max(a, b, c) a, b and c are floats
@@ -136,6 +126,13 @@ def AMDGPUumin3 : SDNode<"AMDGPUISD::UMIN3", AMDGPUDTIntTernaryOp,
   [/*SDNPCommutative, SDNPAssociative*/]
 >;
 
+// out = (src0 + src1 > 0xFFFFFFFF) ? 1 : 0
+def AMDGPUcarry : SDNode<"AMDGPUISD::CARRY", SDTIntBinOp, []>;
+
+// out = (src1 > src0) ? 1 : 0
+def AMDGPUborrow : SDNode<"AMDGPUISD::BORROW", SDTIntBinOp, []>;
+
+
 def AMDGPUcvt_f32_ubyte0 : SDNode<"AMDGPUISD::CVT_F32_UBYTE0",
   SDTIntToFPOp, []>;
 def AMDGPUcvt_f32_ubyte1 : SDNode<"AMDGPUISD::CVT_F32_UBYTE1",
@@ -212,6 +209,22 @@ def AMDGPUmad_i24 : SDNode<"AMDGPUISD::MAD_I24", AMDGPUDTIntTernaryOp,
   []
 >;
 
+def AMDGPUsendmsg : SDNode<"AMDGPUISD::SENDMSG",
+                    SDTypeProfile<0, 1, [SDTCisInt<0>]>,
+                    [SDNPHasChain, SDNPInGlue]>;
+
+def AMDGPUinterp_mov : SDNode<"AMDGPUISD::INTERP_MOV",
+                        SDTypeProfile<1, 3, [SDTCisFP<0>]>,
+                        [SDNPInGlue]>;
+
+def AMDGPUinterp_p1 : SDNode<"AMDGPUISD::INTERP_P1",
+                      SDTypeProfile<1, 3, [SDTCisFP<0>]>,
+                      [SDNPInGlue, SDNPOutGlue]>;
+
+def AMDGPUinterp_p2 : SDNode<"AMDGPUISD::INTERP_P2",
+                      SDTypeProfile<1, 4, [SDTCisFP<0>]>,
+                      [SDNPInGlue]>;
+
 //===----------------------------------------------------------------------===//
 // Flow Control Profile Types
 //===----------------------------------------------------------------------===//
diff --git a/lib/Target/R600/AMDGPUInstructions.td b/lib/Target/AMDGPU/AMDGPUInstructions.td
similarity index 95%
rename from lib/Target/R600/AMDGPUInstructions.td
rename to lib/Target/AMDGPU/AMDGPUInstructions.td
index eeb7f3fcde5..72cab39277c 100644
--- a/lib/Target/R600/AMDGPUInstructions.td
+++ b/lib/Target/AMDGPU/AMDGPUInstructions.td
@@ -183,12 +183,15 @@ def constant_load : PatFrag<(ops node:$ptr), (load node:$ptr), [{
     return isConstantLoad(dyn_cast<LoadSDNode>(N), -1);
 }]>;
 
-def az_extload : PatFrag<(ops node:$ptr), (unindexedload node:$ptr), [{
+class AZExtLoadBase <SDPatternOperator ld_node>: PatFrag<(ops node:$ptr),
+                                              (ld_node node:$ptr), [{
   LoadSDNode *L = cast<LoadSDNode>(N);
   return L->getExtensionType() == ISD::ZEXTLOAD ||
          L->getExtensionType() == ISD::EXTLOAD;
 }]>;
 
+def az_extload : AZExtLoadBase <unindexedload>;
+
 def az_extloadi8 : PatFrag<(ops node:$ptr), (az_extload node:$ptr), [{
   return cast<LoadSDNode>(N)->getMemoryVT() == MVT::i8;
 }]>;
@@ -361,22 +364,26 @@ def mskor_global : PatFrag<(ops node:$val, node:$ptr),
   return cast<MemSDNode>(N)->getAddressSpace() == AMDGPUAS::GLOBAL_ADDRESS;
 }]>;
 
+multiclass AtomicCmpSwapLocal <SDNode cmp_swap_node> {
 
-def atomic_cmp_swap_32_local :
-  PatFrag<(ops node:$ptr, node:$cmp, node:$swap),
-          (atomic_cmp_swap node:$ptr, node:$cmp, node:$swap), [{
-  AtomicSDNode *AN = cast<AtomicSDNode>(N);
-  return AN->getMemoryVT() == MVT::i32 &&
-         AN->getAddressSpace() == AMDGPUAS::LOCAL_ADDRESS;
-}]>;
+  def _32_local : PatFrag <
+    (ops node:$ptr, node:$cmp, node:$swap),
+    (cmp_swap_node node:$ptr, node:$cmp, node:$swap), [{
+      AtomicSDNode *AN = cast<AtomicSDNode>(N);
+      return AN->getMemoryVT() == MVT::i32 &&
+             AN->getAddressSpace() == AMDGPUAS::LOCAL_ADDRESS;
+  }]>;
 
-def atomic_cmp_swap_64_local :
-  PatFrag<(ops node:$ptr, node:$cmp, node:$swap),
-          (atomic_cmp_swap node:$ptr, node:$cmp, node:$swap), [{
-  AtomicSDNode *AN = cast<AtomicSDNode>(N);
-  return AN->getMemoryVT() == MVT::i64 &&
-         AN->getAddressSpace() == AMDGPUAS::LOCAL_ADDRESS;
-}]>;
+  def _64_local : PatFrag<
+    (ops node:$ptr, node:$cmp, node:$swap),
+    (cmp_swap_node node:$ptr, node:$cmp, node:$swap), [{
+      AtomicSDNode *AN = cast<AtomicSDNode>(N);
+      return AN->getMemoryVT() == MVT::i64 &&
+             AN->getAddressSpace() == AMDGPUAS::LOCAL_ADDRESS;
+  }]>;
+}
+
+defm atomic_cmp_swap : AtomicCmpSwapLocal <atomic_cmp_swap>;
 
 def flat_load : PatFrag<(ops node:$ptr), (load node:$ptr), [{
     return isFlatLoad(dyn_cast<LoadSDNode>(N));
@@ -583,7 +590,7 @@ def IMMZeroBasedBitfieldMask : PatLeaf <(imm), [{
 }]>;
 
 def IMMPopCount : SDNodeXForm<imm, [{
-  return CurDAG->getTargetConstant(countPopulation(N->getZExtValue()),
+  return CurDAG->getTargetConstant(countPopulation(N->getZExtValue()), SDLoc(N),
                                    MVT::i32);
 }]>;
 
diff --git a/lib/Target/R600/AMDGPUIntrinsicInfo.cpp b/lib/Target/AMDGPU/AMDGPUIntrinsicInfo.cpp
similarity index 100%
rename from lib/Target/R600/AMDGPUIntrinsicInfo.cpp
rename to lib/Target/AMDGPU/AMDGPUIntrinsicInfo.cpp
diff --git a/lib/Target/R600/AMDGPUIntrinsicInfo.h b/lib/Target/AMDGPU/AMDGPUIntrinsicInfo.h
similarity index 100%
rename from lib/Target/R600/AMDGPUIntrinsicInfo.h
rename to lib/Target/AMDGPU/AMDGPUIntrinsicInfo.h
diff --git a/lib/Target/R600/AMDGPUIntrinsics.td b/lib/Target/AMDGPU/AMDGPUIntrinsics.td
similarity index 100%
rename from lib/Target/R600/AMDGPUIntrinsics.td
rename to lib/Target/AMDGPU/AMDGPUIntrinsics.td
diff --git a/lib/Target/R600/AMDGPUMCInstLower.cpp b/lib/Target/AMDGPU/AMDGPUMCInstLower.cpp
similarity index 85%
rename from lib/Target/R600/AMDGPUMCInstLower.cpp
rename to lib/Target/AMDGPU/AMDGPUMCInstLower.cpp
index 7e274a9a5fc..20831460b93 100644
--- a/lib/Target/R600/AMDGPUMCInstLower.cpp
+++ b/lib/Target/AMDGPU/AMDGPUMCInstLower.cpp
@@ -58,32 +58,32 @@ void AMDGPUMCInstLower::lower(const MachineInstr *MI, MCInst &OutMI) const {
     default:
       llvm_unreachable("unknown operand type");
     case MachineOperand::MO_Immediate:
-      MCOp = MCOperand::CreateImm(MO.getImm());
+      MCOp = MCOperand::createImm(MO.getImm());
       break;
     case MachineOperand::MO_Register:
-      MCOp = MCOperand::CreateReg(MO.getReg());
+      MCOp = MCOperand::createReg(MO.getReg());
       break;
     case MachineOperand::MO_MachineBasicBlock:
-      MCOp = MCOperand::CreateExpr(MCSymbolRefExpr::Create(
+      MCOp = MCOperand::createExpr(MCSymbolRefExpr::create(
                                    MO.getMBB()->getSymbol(), Ctx));
       break;
     case MachineOperand::MO_GlobalAddress: {
       const GlobalValue *GV = MO.getGlobal();
-      MCSymbol *Sym = Ctx.GetOrCreateSymbol(StringRef(GV->getName()));
-      MCOp = MCOperand::CreateExpr(MCSymbolRefExpr::Create(Sym, Ctx));
+      MCSymbol *Sym = Ctx.getOrCreateSymbol(StringRef(GV->getName()));
+      MCOp = MCOperand::createExpr(MCSymbolRefExpr::create(Sym, Ctx));
       break;
     }
     case MachineOperand::MO_TargetIndex: {
       assert(MO.getIndex() == AMDGPU::TI_CONSTDATA_START);
-      MCSymbol *Sym = Ctx.GetOrCreateSymbol(StringRef(END_OF_TEXT_LABEL_NAME));
-      const MCSymbolRefExpr *Expr = MCSymbolRefExpr::Create(Sym, Ctx);
-      MCOp = MCOperand::CreateExpr(Expr);
+      MCSymbol *Sym = Ctx.getOrCreateSymbol(StringRef(END_OF_TEXT_LABEL_NAME));
+      const MCSymbolRefExpr *Expr = MCSymbolRefExpr::create(Sym, Ctx);
+      MCOp = MCOperand::createExpr(Expr);
       break;
     }
     case MachineOperand::MO_ExternalSymbol: {
-      MCSymbol *Sym = Ctx.GetOrCreateSymbol(StringRef(MO.getSymbolName()));
-      const MCSymbolRefExpr *Expr = MCSymbolRefExpr::Create(Sym, Ctx);
-      MCOp = MCOperand::CreateExpr(Expr);
+      MCSymbol *Sym = Ctx.getOrCreateSymbol(StringRef(MO.getSymbolName()));
+      const MCSymbolRefExpr *Expr = MCSymbolRefExpr::create(Sym, Ctx);
+      MCOp = MCOperand::createExpr(Expr);
       break;
     }
     }
@@ -113,7 +113,7 @@ void AMDGPUAsmPrinter::EmitInstruction(const MachineInstr *MI) {
   } else {
     MCInst TmpInst;
     MCInstLowering.lower(MI, TmpInst);
-    EmitToStreamer(OutStreamer, TmpInst);
+    EmitToStreamer(*OutStreamer, TmpInst);
 
     if (STI.dumpCode()) {
       // Disassemble instruction/operands to text.
@@ -132,9 +132,9 @@ void AMDGPUAsmPrinter::EmitInstruction(const MachineInstr *MI) {
       SmallVector<char, 16> CodeBytes;
       raw_svector_ostream CodeStream(CodeBytes);
 
-      MCObjectStreamer &ObjStreamer = (MCObjectStreamer &)OutStreamer;
+      auto &ObjStreamer = static_cast<MCObjectStreamer&>(*OutStreamer);
       MCCodeEmitter &InstEmitter = ObjStreamer.getAssembler().getEmitter();
-      InstEmitter.EncodeInstruction(TmpInst, CodeStream, Fixups,
+      InstEmitter.encodeInstruction(TmpInst, CodeStream, Fixups,
                                     MF->getSubtarget<MCSubtargetInfo>());
       CodeStream.flush();
 
diff --git a/lib/Target/R600/AMDGPUMCInstLower.h b/lib/Target/AMDGPU/AMDGPUMCInstLower.h
similarity index 100%
rename from lib/Target/R600/AMDGPUMCInstLower.h
rename to lib/Target/AMDGPU/AMDGPUMCInstLower.h
diff --git a/lib/Target/R600/AMDGPUMachineFunction.cpp b/lib/Target/AMDGPU/AMDGPUMachineFunction.cpp
similarity index 100%
rename from lib/Target/R600/AMDGPUMachineFunction.cpp
rename to lib/Target/AMDGPU/AMDGPUMachineFunction.cpp
diff --git a/lib/Target/R600/AMDGPUMachineFunction.h b/lib/Target/AMDGPU/AMDGPUMachineFunction.h
similarity index 100%
rename from lib/Target/R600/AMDGPUMachineFunction.h
rename to lib/Target/AMDGPU/AMDGPUMachineFunction.h
diff --git a/lib/Target/R600/AMDGPUPromoteAlloca.cpp b/lib/Target/AMDGPU/AMDGPUPromoteAlloca.cpp
similarity index 92%
rename from lib/Target/R600/AMDGPUPromoteAlloca.cpp
rename to lib/Target/AMDGPU/AMDGPUPromoteAlloca.cpp
index 6d5f94e165c..57b7a73bf56 100644
--- a/lib/Target/R600/AMDGPUPromoteAlloca.cpp
+++ b/lib/Target/AMDGPU/AMDGPUPromoteAlloca.cpp
@@ -134,13 +134,17 @@ static Value* GEPToVectorIndex(GetElementPtrInst *GEP) {
 //
 // TODO: Check isTriviallyVectorizable for calls and handle other
 // instructions.
-static bool canVectorizeInst(Instruction *Inst) {
+static bool canVectorizeInst(Instruction *Inst, User *User) {
   switch (Inst->getOpcode()) {
   case Instruction::Load:
-  case Instruction::Store:
   case Instruction::BitCast:
   case Instruction::AddrSpaceCast:
     return true;
+  case Instruction::Store: {
+    // Must be the stored pointer operand, not a stored value.
+    StoreInst *SI = cast<StoreInst>(Inst);
+    return SI->getPointerOperand() == User;
+  }
   default:
     return false;
   }
@@ -166,7 +170,7 @@ static bool tryPromoteAllocaToVector(AllocaInst *Alloca) {
   for (User *AllocaUser : Alloca->users()) {
     GetElementPtrInst *GEP = dyn_cast<GetElementPtrInst>(AllocaUser);
     if (!GEP) {
-      if (!canVectorizeInst(cast<Instruction>(AllocaUser)))
+      if (!canVectorizeInst(cast<Instruction>(AllocaUser), Alloca))
         return false;
 
       WorkList.push_back(AllocaUser);
@@ -184,7 +188,7 @@ static bool tryPromoteAllocaToVector(AllocaInst *Alloca) {
 
     GEPVectorIdx[GEP] = Index;
     for (User *GEPUser : AllocaUser->users()) {
-      if (!canVectorizeInst(cast<Instruction>(GEPUser)))
+      if (!canVectorizeInst(cast<Instruction>(GEPUser), AllocaUser))
         return false;
 
       WorkList.push_back(GEPUser);
@@ -240,7 +244,12 @@ static bool collectUsesWithPtrTypes(Value *Val, std::vector<Value*> &WorkList) {
   for (User *User : Val->users()) {
     if(std::find(WorkList.begin(), WorkList.end(), User) != WorkList.end())
       continue;
-    if (isa<CallInst>(User)) {
+    if (CallInst *CI = dyn_cast<CallInst>(User)) {
+      // TODO: We might be able to handle some cases where the callee is a
+      // constantexpr bitcast of a function.
+      if (!CI->getCalledFunction())
+        return false;
+
       WorkList.push_back(User);
       continue;
     }
@@ -250,6 +259,12 @@ static bool collectUsesWithPtrTypes(Value *Val, std::vector<Value*> &WorkList) {
     if (UseInst && UseInst->getOpcode() == Instruction::PtrToInt)
       return false;
 
+    if (StoreInst *SI = dyn_cast_or_null<StoreInst>(UseInst)) {
+      // Reject if the stored value is not the pointer operand.
+      if (SI->getPointerOperand() != Val)
+        return false;
+    }
+
     if (!User->getType()->isPointerTy())
       continue;
 
@@ -316,12 +331,11 @@ void AMDGPUPromoteAlloca::visitAlloca(AllocaInst &I) {
   Value *ReadTIDIGZ = Mod->getOrInsertFunction(
       "llvm.r600.read.tidig.z", FTy, AttrSet);
 
-
-  Value *TCntY = Builder.CreateCall(ReadLocalSizeY);
-  Value *TCntZ = Builder.CreateCall(ReadLocalSizeZ);
-  Value *TIdX  = Builder.CreateCall(ReadTIDIGX);
-  Value *TIdY  = Builder.CreateCall(ReadTIDIGY);
-  Value *TIdZ  = Builder.CreateCall(ReadTIDIGZ);
+  Value *TCntY = Builder.CreateCall(ReadLocalSizeY, {});
+  Value *TCntZ = Builder.CreateCall(ReadLocalSizeZ, {});
+  Value *TIdX = Builder.CreateCall(ReadTIDIGX, {});
+  Value *TIdY = Builder.CreateCall(ReadTIDIGY, {});
+  Value *TIdZ = Builder.CreateCall(ReadTIDIGZ, {});
 
   Value *Tmp0 = Builder.CreateMul(TCntY, TCntZ);
   Tmp0 = Builder.CreateMul(Tmp0, TIdX);
diff --git a/lib/Target/R600/AMDGPURegisterInfo.cpp b/lib/Target/AMDGPU/AMDGPURegisterInfo.cpp
similarity index 100%
rename from lib/Target/R600/AMDGPURegisterInfo.cpp
rename to lib/Target/AMDGPU/AMDGPURegisterInfo.cpp
diff --git a/lib/Target/R600/AMDGPURegisterInfo.h b/lib/Target/AMDGPU/AMDGPURegisterInfo.h
similarity index 100%
rename from lib/Target/R600/AMDGPURegisterInfo.h
rename to lib/Target/AMDGPU/AMDGPURegisterInfo.h
diff --git a/lib/Target/R600/AMDGPURegisterInfo.td b/lib/Target/AMDGPU/AMDGPURegisterInfo.td
similarity index 85%
rename from lib/Target/R600/AMDGPURegisterInfo.td
rename to lib/Target/AMDGPU/AMDGPURegisterInfo.td
index 835a1464395..ba0490abee8 100644
--- a/lib/Target/R600/AMDGPURegisterInfo.td
+++ b/lib/Target/AMDGPU/AMDGPURegisterInfo.td
@@ -14,8 +14,7 @@
 let Namespace = "AMDGPU" in {
 
 foreach Index = 0-15 in {
-  // Indices are used in a variety of ways here, so don't set a size/offset.
-  def sub#Index : SubRegIndex<-1, -1>;
+  def sub#Index : SubRegIndex<32, !shl(Index, 5)>;
 }
 
 def INDIRECT_BASE_ADDR : Register <"INDIRECT_BASE_ADDR">;
diff --git a/lib/Target/R600/AMDGPUSubtarget.cpp b/lib/Target/AMDGPU/AMDGPUSubtarget.cpp
similarity index 87%
rename from lib/Target/R600/AMDGPUSubtarget.cpp
rename to lib/Target/AMDGPU/AMDGPUSubtarget.cpp
index 259224a8af2..5f32a65c933 100644
--- a/lib/Target/R600/AMDGPUSubtarget.cpp
+++ b/lib/Target/AMDGPU/AMDGPUSubtarget.cpp
@@ -32,8 +32,8 @@ using namespace llvm;
 #include "AMDGPUGenSubtargetInfo.inc"
 
 AMDGPUSubtarget &
-AMDGPUSubtarget::initializeSubtargetDependencies(StringRef TT, StringRef GPU,
-                                                 StringRef FS) {
+AMDGPUSubtarget::initializeSubtargetDependencies(const Triple &TT,
+                                                 StringRef GPU, StringRef FS) {
   // Determine default and user-specified characteristics
   // On SI+, we want FP64 denormals to be on by default. FP32 denormals can be
   // enabled, but some instructions do not respect them and they run at the
@@ -46,7 +46,7 @@ AMDGPUSubtarget::initializeSubtargetDependencies(StringRef TT, StringRef GPU,
   SmallString<256> FullFS("+promote-alloca,+fp64-denormals,");
   FullFS += FS;
 
-  if (GPU == "" && Triple(TT).getArch() == Triple::amdgcn)
+  if (GPU == "" && TT.getArch() == Triple::amdgcn)
     GPU = "SI";
 
   ParseSubtargetFeatures(GPU, FullFS);
@@ -61,7 +61,7 @@ AMDGPUSubtarget::initializeSubtargetDependencies(StringRef TT, StringRef GPU,
   return *this;
 }
 
-AMDGPUSubtarget::AMDGPUSubtarget(StringRef TT, StringRef GPU, StringRef FS,
+AMDGPUSubtarget::AMDGPUSubtarget(const Triple &TT, StringRef GPU, StringRef FS,
                                  TargetMachine &TM)
     : AMDGPUGenSubtargetInfo(TT, GPU, FS), DevName(GPU), Is64bit(false),
       DumpCode(false), R600ALUInst(false), HasVertexCache(false),
@@ -69,9 +69,11 @@ AMDGPUSubtarget::AMDGPUSubtarget(StringRef TT, StringRef GPU, StringRef FS,
       FP64Denormals(false), FP32Denormals(false), FastFMAF32(false),
       CaymanISA(false), FlatAddressSpace(false), EnableIRStructurizer(true),
       EnablePromoteAlloca(false), EnableIfCvt(true), EnableLoadStoreOpt(false),
+      EnableUnsafeDSOffsetFolding(false),
       WavefrontSize(0), CFALUBug(false), LocalMemorySize(0),
-      EnableVGPRSpilling(false), SGPRInitBug(false),
-      IsGCN(false), GCN1Encoding(false), GCN3Encoding(false),
+      EnableVGPRSpilling(false), SGPRInitBug(false), IsGCN(false),
+      GCN1Encoding(false), GCN3Encoding(false), CIInsts(false), LDSBankCount(0),
+      IsaVersion(ISAVersion0_0_0), EnableHugeScratchBuffer(false),
       FrameLowering(TargetFrameLowering::StackGrowsUp,
                     64 * 16, // Maximum stack alignment (long16)
                     0),
@@ -109,6 +111,10 @@ unsigned AMDGPUSubtarget::getAmdKernelCodeChipID() const {
   }
 }
 
+AMDGPU::IsaVersion AMDGPUSubtarget::getIsaVersion() const {
+  return AMDGPU::getIsaVersion(getFeatureBits());
+}
+
 bool AMDGPUSubtarget::isVGPRSpillingEnabled(
                                        const SIMachineFunctionInfo *MFI) const {
   return MFI->getShaderType() == ShaderType::COMPUTE || EnableVGPRSpilling;
@@ -131,3 +137,4 @@ void AMDGPUSubtarget::overrideSchedPolicy(MachineSchedPolicy &Policy,
     Policy.OnlyBottomUp = false;
   }
 }
+
diff --git a/lib/Target/R600/AMDGPUSubtarget.h b/lib/Target/AMDGPU/AMDGPUSubtarget.h
similarity index 81%
rename from lib/Target/R600/AMDGPUSubtarget.h
rename to lib/Target/AMDGPU/AMDGPUSubtarget.h
index aeb0817553d..735f01dfa7c 100644
--- a/lib/Target/R600/AMDGPUSubtarget.h
+++ b/lib/Target/AMDGPU/AMDGPUSubtarget.h
@@ -20,6 +20,8 @@
 #include "AMDGPUIntrinsicInfo.h"
 #include "AMDGPUSubtarget.h"
 #include "R600ISelLowering.h"
+#include "AMDKernelCodeT.h"
+#include "Utils/AMDGPUBaseInfo.h"
 #include "llvm/ADT/StringExtras.h"
 #include "llvm/ADT/StringRef.h"
 #include "llvm/Target/TargetSubtargetInfo.h"
@@ -48,6 +50,14 @@ class AMDGPUSubtarget : public AMDGPUGenSubtargetInfo {
     FIXED_SGPR_COUNT_FOR_INIT_BUG = 80
   };
 
+  enum {
+    ISAVersion0_0_0,
+    ISAVersion7_0_0,
+    ISAVersion7_0_1,
+    ISAVersion8_0_0,
+    ISAVersion8_0_1
+  };
+
 private:
   std::string DevName;
   bool Is64bit;
@@ -66,6 +76,7 @@ class AMDGPUSubtarget : public AMDGPUGenSubtargetInfo {
   bool EnablePromoteAlloca;
   bool EnableIfCvt;
   bool EnableLoadStoreOpt;
+  bool EnableUnsafeDSOffsetFolding;
   unsigned WavefrontSize;
   bool CFALUBug;
   int LocalMemorySize;
@@ -74,6 +85,11 @@ class AMDGPUSubtarget : public AMDGPUGenSubtargetInfo {
   bool IsGCN;
   bool GCN1Encoding;
   bool GCN3Encoding;
+  bool CIInsts;
+  bool FeatureDisable;
+  int LDSBankCount;
+  unsigned IsaVersion; 
+  bool EnableHugeScratchBuffer;
 
   AMDGPUFrameLowering FrameLowering;
   std::unique_ptr<AMDGPUTargetLowering> TLInfo;
@@ -82,9 +98,10 @@ class AMDGPUSubtarget : public AMDGPUGenSubtargetInfo {
   Triple TargetTriple;
 
 public:
-  AMDGPUSubtarget(StringRef TT, StringRef CPU, StringRef FS, TargetMachine &TM);
-  AMDGPUSubtarget &initializeSubtargetDependencies(StringRef TT, StringRef GPU,
-                                                   StringRef FS);
+  AMDGPUSubtarget(const Triple &TT, StringRef CPU, StringRef FS,
+                  TargetMachine &TM);
+  AMDGPUSubtarget &initializeSubtargetDependencies(const Triple &TT,
+                                                   StringRef GPU, StringRef FS);
 
   const AMDGPUFrameLowering *getFrameLowering() const override {
     return &FrameLowering;
@@ -183,6 +200,14 @@ class AMDGPUSubtarget : public AMDGPUGenSubtargetInfo {
     return (getGeneration() >= EVERGREEN);
   }
 
+  bool hasCARRY() const {
+    return (getGeneration() >= EVERGREEN);
+  }
+
+  bool hasBORROW() const {
+    return (getGeneration() >= EVERGREEN);
+  }
+
   bool IsIRStructurizerEnabled() const {
     return EnableIRStructurizer;
   }
@@ -199,6 +224,10 @@ class AMDGPUSubtarget : public AMDGPUGenSubtargetInfo {
     return EnableLoadStoreOpt;
   }
 
+  bool unsafeDSOffsetFoldingEnabled() const {
+    return EnableUnsafeDSOffsetFolding;
+  }
+
   unsigned getWavefrontSize() const {
     return WavefrontSize;
   }
@@ -218,8 +247,14 @@ class AMDGPUSubtarget : public AMDGPUGenSubtargetInfo {
     return SGPRInitBug;
   }
 
+  int getLDSBankCount() const {
+    return LDSBankCount;
+  }
+
   unsigned getAmdKernelCodeChipID() const;
 
+  AMDGPU::IsaVersion getIsaVersion() const;
+
   bool enableMachineScheduler() const override {
     return true;
   }
@@ -237,6 +272,10 @@ class AMDGPUSubtarget : public AMDGPUGenSubtargetInfo {
     return DevName;
   }
 
+  bool enableHugeScratchBuffer() const {
+    return EnableHugeScratchBuffer;
+  }
+
   bool dumpCode() const {
     return DumpCode;
   }
@@ -257,8 +296,15 @@ class AMDGPUSubtarget : public AMDGPUGenSubtargetInfo {
   }
 
   bool enableSubRegLiveness() const override {
-    return false;
+    return true;
   }
+
+  /// \brief Returns the offset in bytes from the start of the input buffer
+  ///        of the first explicit kernel argument.
+  unsigned getExplicitKernelArgOffset() const {
+    return isAmdHsaOS() ? 0 : 36;
+  }
+
 };
 
 } // End namespace llvm
diff --git a/lib/Target/R600/AMDGPUTargetMachine.cpp b/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp
similarity index 85%
rename from lib/Target/R600/AMDGPUTargetMachine.cpp
rename to lib/Target/AMDGPU/AMDGPUTargetMachine.cpp
index cb95835d2db..2297b52b423 100644
--- a/lib/Target/R600/AMDGPUTargetMachine.cpp
+++ b/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp
@@ -37,7 +37,7 @@
 
 using namespace llvm;
 
-extern "C" void LLVMInitializeR600Target() {
+extern "C" void LLVMInitializeAMDGPUTarget() {
   // Register the target
   RegisterTargetMachine<R600TargetMachine> X(TheAMDGPUTarget);
   RegisterTargetMachine<GCNTargetMachine> Y(TheGCNTarget);
@@ -51,11 +51,10 @@ static MachineSchedRegistry
 SchedCustomRegistry("r600", "Run R600's custom scheduler",
                     createR600MachineScheduler);
 
-static std::string computeDataLayout(StringRef TT) {
-  Triple Triple(TT);
+static std::string computeDataLayout(const Triple &TT) {
   std::string Ret = "e-p:32:32";
 
-  if (Triple.getArch() == Triple::amdgcn) {
+  if (TT.getArch() == Triple::amdgcn) {
     // 32-bit private, local, and region pointers. 64-bit global and constant.
     Ret += "-p1:64:64-p2:64:64-p3:32:32-p4:64:64-p5:32:32-p24:64:64";
   }
@@ -66,7 +65,7 @@ static std::string computeDataLayout(StringRef TT) {
   return Ret;
 }
 
-AMDGPUTargetMachine::AMDGPUTargetMachine(const Target &T, StringRef TT,
+AMDGPUTargetMachine::AMDGPUTargetMachine(const Target &T, const Triple &TT,
                                          StringRef CPU, StringRef FS,
                                          TargetOptions Options, Reloc::Model RM,
                                          CodeModel::Model CM,
@@ -87,20 +86,21 @@ AMDGPUTargetMachine::~AMDGPUTargetMachine() {
 // R600 Target Machine (R600 -> Cayman)
 //===----------------------------------------------------------------------===//
 
-R600TargetMachine::R600TargetMachine(const Target &T, StringRef TT, StringRef FS,
-                    StringRef CPU, TargetOptions Options, Reloc::Model RM,
-                    CodeModel::Model CM, CodeGenOpt::Level OL) :
-    AMDGPUTargetMachine(T, TT, FS, CPU, Options, RM, CM, OL) { }
-
+R600TargetMachine::R600TargetMachine(const Target &T, const Triple &TT,
+                                     StringRef FS, StringRef CPU,
+                                     TargetOptions Options, Reloc::Model RM,
+                                     CodeModel::Model CM, CodeGenOpt::Level OL)
+    : AMDGPUTargetMachine(T, TT, FS, CPU, Options, RM, CM, OL) {}
 
 //===----------------------------------------------------------------------===//
 // GCN Target Machine (SI+)
 //===----------------------------------------------------------------------===//
 
-GCNTargetMachine::GCNTargetMachine(const Target &T, StringRef TT, StringRef FS,
-                    StringRef CPU, TargetOptions Options, Reloc::Model RM,
-                    CodeModel::Model CM, CodeGenOpt::Level OL) :
-    AMDGPUTargetMachine(T, TT, FS, CPU, Options, RM, CM, OL) { }
+GCNTargetMachine::GCNTargetMachine(const Target &T, const Triple &TT,
+                                   StringRef FS, StringRef CPU,
+                                   TargetOptions Options, Reloc::Model RM,
+                                   CodeModel::Model CM, CodeGenOpt::Level OL)
+    : AMDGPUTargetMachine(T, TT, FS, CPU, Options, RM, CM, OL) {}
 
 //===----------------------------------------------------------------------===//
 // AMDGPU Pass Setup
@@ -156,8 +156,10 @@ class GCNPassConfig : public AMDGPUPassConfig {
 } // End of anonymous namespace
 
 TargetIRAnalysis AMDGPUTargetMachine::getTargetIRAnalysis() {
-  return TargetIRAnalysis(
-      [this](Function &F) { return TargetTransformInfo(AMDGPUTTIImpl(this)); });
+  return TargetIRAnalysis([this](Function &F) {
+    return TargetTransformInfo(
+        AMDGPUTTIImpl(this, F.getParent()->getDataLayout()));
+  });
 }
 
 void AMDGPUPassConfig::addIRPasses() {
@@ -252,6 +254,15 @@ bool GCNPassConfig::addInstSelector() {
 
 void GCNPassConfig::addPreRegAlloc() {
   const AMDGPUSubtarget &ST = *getAMDGPUTargetMachine().getSubtargetImpl();
+
+  // This needs to be run directly before register allocation because
+  // earlier passes might recompute live intervals.
+  // TODO: handle CodeGenOpt::None; fast RA ignores spill weights set by the pass
+  if (getOptLevel() > CodeGenOpt::None) {
+    initializeSIFixControlFlowLiveIntervalsPass(*PassRegistry::getPassRegistry());
+    insertPass(&MachineSchedulerID, &SIFixControlFlowLiveIntervalsID);
+  }
+
   if (getOptLevel() > CodeGenOpt::None && ST.loadStoreOptEnabled()) {
     // Don't do this with no optimizations since it throws away debug info by
     // merging nonadjacent loads.
@@ -260,6 +271,7 @@ void GCNPassConfig::addPreRegAlloc() {
     // also need extra copies to the address operand to be eliminated.
     initializeSILoadStoreOptimizerPass(*PassRegistry::getPassRegistry());
     insertPass(&MachineSchedulerID, &SILoadStoreOptimizerID);
+    insertPass(&MachineSchedulerID, &RegisterCoalescerID);
   }
   addPass(createSIShrinkInstructionsPass(), false);
   addPass(createSIFixSGPRLiveRangesPass(), false);
@@ -271,10 +283,10 @@ void GCNPassConfig::addPostRegAlloc() {
 }
 
 void GCNPassConfig::addPreSched2() {
-  addPass(createSIInsertWaits(*TM), false);
 }
 
 void GCNPassConfig::addPreEmitPass() {
+  addPass(createSIInsertWaits(*TM), false);
   addPass(createSILowerControlFlowPass(*TM), false);
 }
 
diff --git a/lib/Target/R600/AMDGPUTargetMachine.h b/lib/Target/AMDGPU/AMDGPUTargetMachine.h
similarity index 88%
rename from lib/Target/R600/AMDGPUTargetMachine.h
rename to lib/Target/AMDGPU/AMDGPUTargetMachine.h
index 785c119a102..14792e347a7 100644
--- a/lib/Target/R600/AMDGPUTargetMachine.h
+++ b/lib/Target/AMDGPU/AMDGPUTargetMachine.h
@@ -37,7 +37,7 @@ class AMDGPUTargetMachine : public LLVMTargetMachine {
   AMDGPUIntrinsicInfo IntrinsicInfo;
 
 public:
-  AMDGPUTargetMachine(const Target &T, StringRef TT, StringRef FS,
+  AMDGPUTargetMachine(const Target &T, const Triple &TT, StringRef FS,
                       StringRef CPU, TargetOptions Options, Reloc::Model RM,
                       CodeModel::Model CM, CodeGenOpt::Level OL);
   ~AMDGPUTargetMachine();
@@ -63,7 +63,7 @@ class AMDGPUTargetMachine : public LLVMTargetMachine {
 class R600TargetMachine : public AMDGPUTargetMachine {
 
 public:
-  R600TargetMachine(const Target &T, StringRef TT, StringRef FS,
+  R600TargetMachine(const Target &T, const Triple &TT, StringRef FS,
                     StringRef CPU, TargetOptions Options, Reloc::Model RM,
                     CodeModel::Model CM, CodeGenOpt::Level OL);
 
@@ -77,9 +77,9 @@ class R600TargetMachine : public AMDGPUTargetMachine {
 class GCNTargetMachine : public AMDGPUTargetMachine {
 
 public:
-  GCNTargetMachine(const Target &T, StringRef TT, StringRef FS,
-                    StringRef CPU, TargetOptions Options, Reloc::Model RM,
-                    CodeModel::Model CM, CodeGenOpt::Level OL);
+  GCNTargetMachine(const Target &T, const Triple &TT, StringRef FS,
+                   StringRef CPU, TargetOptions Options, Reloc::Model RM,
+                   CodeModel::Model CM, CodeGenOpt::Level OL);
 
   TargetPassConfig *createPassConfig(PassManagerBase &PM) override;
 };
diff --git a/lib/Target/R600/AMDGPUTargetTransformInfo.cpp b/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.cpp
similarity index 97%
rename from lib/Target/R600/AMDGPUTargetTransformInfo.cpp
rename to lib/Target/AMDGPU/AMDGPUTargetTransformInfo.cpp
index 96edc417822..6dacc742b12 100644
--- a/lib/Target/R600/AMDGPUTargetTransformInfo.cpp
+++ b/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.cpp
@@ -76,7 +76,7 @@ unsigned AMDGPUTTIImpl::getNumberOfRegisters(bool Vec) {
 
 unsigned AMDGPUTTIImpl::getRegisterBitWidth(bool) { return 32; }
 
-unsigned AMDGPUTTIImpl::getMaxInterleaveFactor() {
+unsigned AMDGPUTTIImpl::getMaxInterleaveFactor(unsigned VF) {
   // Semi-arbitrary large amount.
   return 64;
 }
diff --git a/lib/Target/R600/AMDGPUTargetTransformInfo.h b/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.h
similarity index 80%
rename from lib/Target/R600/AMDGPUTargetTransformInfo.h
rename to lib/Target/AMDGPU/AMDGPUTargetTransformInfo.h
index 4abbdf20e76..dee0a69d1e6 100644
--- a/lib/Target/R600/AMDGPUTargetTransformInfo.h
+++ b/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.h
@@ -37,8 +37,9 @@ class AMDGPUTTIImpl : public BasicTTIImplBase<AMDGPUTTIImpl> {
   const AMDGPUTargetLowering *getTLI() const { return TLI; }
 
 public:
-  explicit AMDGPUTTIImpl(const AMDGPUTargetMachine *TM)
-      : BaseT(TM), ST(TM->getSubtargetImpl()), TLI(ST->getTargetLowering()) {}
+  explicit AMDGPUTTIImpl(const AMDGPUTargetMachine *TM, const DataLayout &DL)
+      : BaseT(TM, DL), ST(TM->getSubtargetImpl()),
+        TLI(ST->getTargetLowering()) {}
 
   // Provide value semantics. MSVC requires that we spell all of these out.
   AMDGPUTTIImpl(const AMDGPUTTIImpl &Arg)
@@ -46,18 +47,6 @@ class AMDGPUTTIImpl : public BasicTTIImplBase<AMDGPUTTIImpl> {
   AMDGPUTTIImpl(AMDGPUTTIImpl &&Arg)
       : BaseT(std::move(static_cast<BaseT &>(Arg))), ST(std::move(Arg.ST)),
         TLI(std::move(Arg.TLI)) {}
-  AMDGPUTTIImpl &operator=(const AMDGPUTTIImpl &RHS) {
-    BaseT::operator=(static_cast<const BaseT &>(RHS));
-    ST = RHS.ST;
-    TLI = RHS.TLI;
-    return *this;
-  }
-  AMDGPUTTIImpl &operator=(AMDGPUTTIImpl &&RHS) {
-    BaseT::operator=(std::move(static_cast<BaseT &>(RHS)));
-    ST = std::move(RHS.ST);
-    TLI = std::move(RHS.TLI);
-    return *this;
-  }
 
   bool hasBranchDivergence() { return true; }
 
@@ -70,7 +59,7 @@ class AMDGPUTTIImpl : public BasicTTIImplBase<AMDGPUTTIImpl> {
 
   unsigned getNumberOfRegisters(bool Vector);
   unsigned getRegisterBitWidth(bool Vector);
-  unsigned getMaxInterleaveFactor();
+  unsigned getMaxInterleaveFactor(unsigned VF);
 };
 
 } // end namespace llvm
diff --git a/lib/Target/R600/AMDILCFGStructurizer.cpp b/lib/Target/AMDGPU/AMDILCFGStructurizer.cpp
similarity index 99%
rename from lib/Target/R600/AMDILCFGStructurizer.cpp
rename to lib/Target/AMDGPU/AMDILCFGStructurizer.cpp
index c9b25a1a0b8..d918ac3a5b3 100644
--- a/lib/Target/R600/AMDILCFGStructurizer.cpp
+++ b/lib/Target/AMDGPU/AMDILCFGStructurizer.cpp
@@ -1719,7 +1719,6 @@ MachineBasicBlock *
 AMDGPUCFGStructurizer::normalizeInfiniteLoopExit(MachineLoop* LoopRep) {
   MachineBasicBlock *LoopHeader = LoopRep->getHeader();
   MachineBasicBlock *LoopLatch = LoopRep->getLoopLatch();
-  const TargetRegisterClass * I32RC = TRI->getCFGStructurizerRegClass(MVT::i32);
 
   if (!LoopHeader || !LoopLatch)
     return nullptr;
@@ -1732,18 +1731,9 @@ AMDGPUCFGStructurizer::normalizeInfiniteLoopExit(MachineLoop* LoopRep) {
   FuncRep->push_back(DummyExitBlk);  //insert to function
   SHOWNEWBLK(DummyExitBlk, "DummyExitBlock to normalize infiniteLoop: ");
   DEBUG(dbgs() << "Old branch instr: " << *BranchMI << "\n";);
-  MachineBasicBlock::iterator I = BranchMI;
-  unsigned ImmReg = FuncRep->getRegInfo().createVirtualRegister(I32RC);
-  llvm_unreachable("Extra register needed to handle CFG");
-  MachineInstr *NewMI = insertInstrBefore(I, AMDGPU::BRANCH_COND_i32);
-  MachineInstrBuilder MIB(*FuncRep, NewMI);
-  MIB.addMBB(LoopHeader);
-  MIB.addReg(ImmReg, false);
-  SHOWNEWINSTR(NewMI);
-  BranchMI->eraseFromParent();
-  LoopLatch->addSuccessor(DummyExitBlk);
-
-  return DummyExitBlk;
+  LLVMContext &Ctx = LoopHeader->getParent()->getFunction()->getContext();
+  Ctx.emitError("Extra register needed to handle CFG");
+  return nullptr;
 }
 
 void AMDGPUCFGStructurizer::removeUnconditionalBranch(MachineBasicBlock *MBB) {
diff --git a/lib/Target/R600/AMDKernelCodeT.h b/lib/Target/AMDGPU/AMDKernelCodeT.h
similarity index 89%
rename from lib/Target/R600/AMDKernelCodeT.h
rename to lib/Target/AMDGPU/AMDKernelCodeT.h
index 4d3041ff3db..a9ba60c8cba 100644
--- a/lib/Target/R600/AMDKernelCodeT.h
+++ b/lib/Target/AMDGPU/AMDKernelCodeT.h
@@ -12,9 +12,12 @@
 #ifndef AMDKERNELCODET_H
 #define AMDKERNELCODET_H
 
+#include "llvm/MC/SubtargetFeature.h"
+
 #include <cstddef>
 #include <cstdint>
 
+#include "llvm/Support/Debug.h"
 //---------------------------------------------------------------------------//
 // AMD Kernel Code, and its dependencies                                     //
 //---------------------------------------------------------------------------//
@@ -132,7 +135,7 @@ enum amd_code_property_mask_t {
   /// private memory do not exceed this size. For example, if the
   /// element size is 4 (32-bits or dword) and a 64-bit value must be
   /// loaded, the finalizer will generate two 32-bit loads. This
-  /// ensures that the interleaving will get the the work-item
+  /// ensures that the interleaving will get the work-item
   /// specific dword for both halves of the 64-bit value. If it just
   /// did a 64-bit load then it would get one dword which belonged to
   /// its own work-item, but the second dword would belong to the
@@ -142,7 +145,7 @@ enum amd_code_property_mask_t {
   /// the GPU flat scratch (SH_STATIC_MEM_CONFIG.ELEMENT_SIZE). This
   /// is generally DWORD.
   ///
-  /// Use values from the amd_element_byte_size_t enum.
+  /// uSE VALUES FROM THE AMD_ELEMENT_BYTE_SIZE_T ENUM.
   AMD_CODE_PROPERTY_PRIVATE_ELEMENT_SIZE_SHIFT = 11,
   AMD_CODE_PROPERTY_PRIVATE_ELEMENT_SIZE_WIDTH = 2,
   AMD_CODE_PROPERTY_PRIVATE_ELEMENT_SIZE = ((1 << AMD_CODE_PROPERTY_PRIVATE_ELEMENT_SIZE_WIDTH) - 1) << AMD_CODE_PROPERTY_PRIVATE_ELEMENT_SIZE_SHIFT,
@@ -171,7 +174,11 @@ enum amd_code_property_mask_t {
   /// Indicate if code generated has support for debugging.
   AMD_CODE_PROPERTY_IS_DEBUG_SUPPORTED_SHIFT = 15,
   AMD_CODE_PROPERTY_IS_DEBUG_SUPPORTED_WIDTH = 1,
-  AMD_CODE_PROPERTY_IS_DEBUG_SUPPORTED = ((1 << AMD_CODE_PROPERTY_IS_DEBUG_SUPPORTED_WIDTH) - 1) << AMD_CODE_PROPERTY_IS_DEBUG_SUPPORTED_SHIFT
+  AMD_CODE_PROPERTY_IS_DEBUG_SUPPORTED = ((1 << AMD_CODE_PROPERTY_IS_DEBUG_SUPPORTED_WIDTH) - 1) << AMD_CODE_PROPERTY_IS_DEBUG_SUPPORTED_SHIFT,
+
+  AMD_CODE_PROPERTY_IS_XNACK_SUPPORTED_SHIFT = 15,
+  AMD_CODE_PROPERTY_IS_XNACK_SUPPORTED_WIDTH = 1,
+  AMD_CODE_PROPERTY_IS_XNACK_SUPPORTED = ((1 << AMD_CODE_PROPERTY_IS_XNACK_SUPPORTED_WIDTH) - 1) << AMD_CODE_PROPERTY_IS_XNACK_SUPPORTED_SHIFT
 };
 
 /// @brief The hsa_ext_control_directives_t specifies the values for the HSAIL
@@ -369,7 +376,7 @@ typedef struct hsa_ext_control_directives_s {
 ///     Scratch Wave Offset must be added by the kernel code and moved to
 ///     SGPRn-4 for use as the FLAT SCRATCH BASE in flat memory instructions.
 ///
-///     The second SGPR is 32 bit byte size of a single work-item’s scratch
+///     The second SGPR is 32 bit byte size of a single work-item's scratch
 ///     memory usage. This is directly loaded from the dispatch packet Private
 ///     Segment Byte Size and rounded up to a multiple of DWORD.
 ///
@@ -385,7 +392,7 @@ typedef struct hsa_ext_control_directives_s {
 ///
 /// Private Segment Size (enable_sgpr_private_segment_size):
 ///   Number of User SGPR registers: 1. The 32 bit byte size of a single
-///   work-item’s scratch memory allocation. This is the value from the dispatch
+///   work-item's scratch memory allocation. This is the value from the dispatch
 ///   packet. Private Segment Byte Size rounded up by CP to a multiple of DWORD.
 ///
 ///   \todo [Does CP need to round this to >4 byte alignment?]
@@ -433,7 +440,7 @@ typedef struct hsa_ext_control_directives_s {
 ///   present
 ///
 /// Work-Group Info (enable_sgpr_workgroup_info):
-///   Number of System SGPR registers: 1. {first_wave, 14’b0000,
+///   Number of System SGPR registers: 1. {first_wave, 14'b0000,
 ///   ordered_append_term[10:0], threadgroup_size_in_waves[5:0]}
 ///
 /// Private Segment Wave Byte Offset
@@ -499,25 +506,14 @@ typedef struct hsa_ext_control_directives_s {
 /// Alternatively scalar loads can be used if the kernarg offset is uniform, as
 /// the kernarg segment is constant for the duration of the kernel execution.
 ///
-typedef struct amd_kernel_code_s {
-  /// The AMD major version of the Code Object. Must be the value
-  /// AMD_CODE_VERSION_MAJOR.
-  amd_code_version32_t amd_code_version_major;
 
-  /// The AMD minor version of the Code Object. Minor versions must be
-  /// backward compatible. Must be the value
-  /// AMD_CODE_VERSION_MINOR.
-  amd_code_version32_t amd_code_version_minor;
-
-  /// The byte size of this struct. Must be set to
-  /// sizeof(amd_kernel_code_t). Used for backward
-  /// compatibility.
-  uint32_t struct_byte_size;
-
-  /// The target chip instruction set for which code has been
-  /// generated. Values are from the E_SC_INSTRUCTION_SET enumeration
-  /// in sc/Interface/SCCommon.h.
-  uint32_t target_chip;
+typedef struct amd_kernel_code_s {
+  uint32_t amd_kernel_code_version_major;
+  uint32_t amd_kernel_code_version_minor;
+  uint16_t amd_machine_kind;
+  uint16_t amd_machine_version_major;
+  uint16_t amd_machine_version_minor;
+  uint16_t amd_machine_version_stepping;
 
   /// Byte offset (possibly negative) from start of amd_kernel_code_t
   /// object to kernel's entry point instruction. The actual code for
@@ -535,10 +531,6 @@ typedef struct amd_kernel_code_s {
   /// and size. The offset is from the start (possibly negative) of
   /// amd_kernel_code_t object. Set both to 0 if no prefetch
   /// information is available.
-  ///
-  /// \todo ttye 11/15/2013 Is the prefetch definition we want? Did
-  /// not make the size a uint64_t as prefetching more than 4GiB seems
-  /// excessive.
   int64_t kernel_code_prefetch_byte_offset;
   uint64_t kernel_code_prefetch_byte_size;
 
@@ -553,11 +545,11 @@ typedef struct amd_kernel_code_s {
 
   /// Shader program settings for CS. Contains COMPUTE_PGM_RSRC1 and
   /// COMPUTE_PGM_RSRC2 registers.
-  amd_compute_pgm_resource_register64_t compute_pgm_resource_registers;
+  uint64_t compute_pgm_resource_registers;
 
   /// Code properties. See amd_code_property_mask_t for a full list of
   /// properties.
-  amd_code_property32_t code_properties;
+  uint32_t code_properties;
 
   /// The amount of memory required for the combined private, spill
   /// and arg segments for a work-item in bytes. If
@@ -629,76 +621,21 @@ typedef struct amd_kernel_code_s {
   /// The maximum byte alignment of variables used by the kernel in
   /// the specified memory segment. Expressed as a power of two. Must
   /// be at least HSA_POWERTWO_16.
-  hsa_powertwo8_t kernarg_segment_alignment;
-  hsa_powertwo8_t group_segment_alignment;
-  hsa_powertwo8_t private_segment_alignment;
-
-  uint8_t reserved3;
-
-  /// Type of code object.
-  hsa_ext_code_kind32_t code_type;
-
-  /// Reserved for code properties if any are defined in the future.
-  /// There are currently no code properties so this field must be 0.
-  uint32_t reserved4;
+  uint8_t kernarg_segment_alignment;
+  uint8_t group_segment_alignment;
+  uint8_t private_segment_alignment;
 
   /// Wavefront size expressed as a power of two. Must be a power of 2
   /// in range 1..64 inclusive. Used to support runtime query that
   /// obtains wavefront size, which may be used by application to
   /// allocated dynamic group memory and set the dispatch work-group
   /// size.
-  hsa_powertwo8_t wavefront_size;
-
-  /// The optimization level specified when the kernel was
-  /// finalized.
-  uint8_t optimization_level;
-
-  /// The HSAIL profile defines which features are used. This
-  /// information is from the HSAIL version directive. If this
-  /// amd_kernel_code_t is not generated from an HSAIL compilation
-  /// unit then must be 0.
-  hsa_ext_brig_profile8_t hsail_profile;
-
-  /// The HSAIL machine model gives the address sizes used by the
-  /// code. This information is from the HSAIL version directive. If
-  /// not generated from an HSAIL compilation unit then must still
-  /// indicate for what machine mode the code is generated.
-  hsa_ext_brig_machine_model8_t hsail_machine_model;
-
-  /// The HSAIL major version. This information is from the HSAIL
-  /// version directive. If this amd_kernel_code_t is not
-  /// generated from an HSAIL compilation unit then must be 0.
-  uint32_t hsail_version_major;
-
-  /// The HSAIL minor version. This information is from the HSAIL
-  /// version directive. If this amd_kernel_code_t is not
-  /// generated from an HSAIL compilation unit then must be 0.
-  uint32_t hsail_version_minor;
-
-  /// Reserved for HSAIL target options if any are defined in the
-  /// future. There are currently no target options so this field
-  /// must be 0.
-  uint16_t reserved5;
-
-  /// Reserved. Must be 0.
-  uint16_t reserved6;
-
-  /// The values should be the actually values used by the finalizer
-  /// in generating the code. This may be the union of values
-  /// specified as finalizer arguments and explicit HSAIL control
-  /// directives. If the finalizer chooses to ignore a control
-  /// directive, and not generate constrained code, then the control
-  /// directive should not be marked as enabled even though it was
-  /// present in the HSAIL or finalizer argument. The values are
-  /// intended to reflect the constraints that the code actually
-  /// requires to correctly execute, not the values that were
-  /// actually specified at finalize time.
-  hsa_ext_control_directives_t control_directive;
-
-  /// The code can immediately follow the amd_kernel_code_t, or can
-  /// come after subsequent amd_kernel_code_t structs when there are
-  /// multiple kernels in the compilation unit.
+  uint8_t wavefront_size;
 
+  int32_t call_convention;
+  uint8_t reserved3[12];
+  uint64_t runtime_loader_kernel_symbol;
+  uint64_t control_directives[16];
 } amd_kernel_code_t;
 
 #endif // AMDKERNELCODET_H
diff --git a/lib/Target/AMDGPU/AsmParser/AMDGPUAsmParser.cpp b/lib/Target/AMDGPU/AsmParser/AMDGPUAsmParser.cpp
new file mode 100644
index 00000000000..e98dd4012cf
--- /dev/null
+++ b/lib/Target/AMDGPU/AsmParser/AMDGPUAsmParser.cpp
@@ -0,0 +1,1692 @@
+//===-- AMDGPUAsmParser.cpp - Parse SI asm to MCInst instructions ----------===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+
+#include "MCTargetDesc/AMDGPUMCTargetDesc.h"
+#include "MCTargetDesc/AMDGPUTargetStreamer.h"
+#include "Utils/AMDGPUBaseInfo.h"
+#include "AMDKernelCodeT.h"
+#include "SIDefines.h"
+#include "llvm/ADT/APFloat.h"
+#include "llvm/ADT/SmallString.h"
+#include "llvm/ADT/SmallVector.h"
+#include "llvm/ADT/STLExtras.h"
+#include "llvm/ADT/StringSwitch.h"
+#include "llvm/ADT/Twine.h"
+#include "llvm/MC/MCContext.h"
+#include "llvm/MC/MCExpr.h"
+#include "llvm/MC/MCInst.h"
+#include "llvm/MC/MCInstrInfo.h"
+#include "llvm/MC/MCParser/MCAsmLexer.h"
+#include "llvm/MC/MCParser/MCAsmParser.h"
+#include "llvm/MC/MCParser/MCParsedAsmOperand.h"
+#include "llvm/MC/MCRegisterInfo.h"
+#include "llvm/MC/MCStreamer.h"
+#include "llvm/MC/MCSubtargetInfo.h"
+#include "llvm/MC/MCTargetAsmParser.h"
+#include "llvm/Support/SourceMgr.h"
+#include "llvm/Support/TargetRegistry.h"
+#include "llvm/Support/raw_ostream.h"
+#include "llvm/Support/Debug.h"
+
+using namespace llvm;
+
+namespace {
+
+struct OptionalOperand;
+
+class AMDGPUOperand : public MCParsedAsmOperand {
+  enum KindTy {
+    Token,
+    Immediate,
+    Register,
+    Expression
+  } Kind;
+
+  SMLoc StartLoc, EndLoc;
+
+public:
+  AMDGPUOperand(enum KindTy K) : MCParsedAsmOperand(), Kind(K) {}
+
+  MCContext *Ctx;
+
+  enum ImmTy {
+    ImmTyNone,
+    ImmTyDSOffset0,
+    ImmTyDSOffset1,
+    ImmTyGDS,
+    ImmTyOffset,
+    ImmTyGLC,
+    ImmTySLC,
+    ImmTyTFE,
+    ImmTyClamp,
+    ImmTyOMod
+  };
+
+  struct TokOp {
+    const char *Data;
+    unsigned Length;
+  };
+
+  struct ImmOp {
+    bool IsFPImm;
+    ImmTy Type;
+    int64_t Val;
+  };
+
+  struct RegOp {
+    unsigned RegNo;
+    int Modifiers;
+    const MCRegisterInfo *TRI;
+    bool IsForcedVOP3;
+  };
+
+  union {
+    TokOp Tok;
+    ImmOp Imm;
+    RegOp Reg;
+    const MCExpr *Expr;
+  };
+
+  void addImmOperands(MCInst &Inst, unsigned N) const {
+    Inst.addOperand(MCOperand::createImm(getImm()));
+  }
+
+  StringRef getToken() const {
+    return StringRef(Tok.Data, Tok.Length);
+  }
+
+  void addRegOperands(MCInst &Inst, unsigned N) const {
+    Inst.addOperand(MCOperand::createReg(getReg()));
+  }
+
+  void addRegOrImmOperands(MCInst &Inst, unsigned N) const {
+    if (isReg())
+      addRegOperands(Inst, N);
+    else
+      addImmOperands(Inst, N);
+  }
+
+  void addRegWithInputModsOperands(MCInst &Inst, unsigned N) const {
+    Inst.addOperand(MCOperand::createImm(
+        Reg.Modifiers == -1 ? 0 : Reg.Modifiers));
+    addRegOperands(Inst, N);
+  }
+
+  void addSoppBrTargetOperands(MCInst &Inst, unsigned N) const {
+    if (isImm())
+      addImmOperands(Inst, N);
+    else {
+      assert(isExpr());
+      Inst.addOperand(MCOperand::createExpr(Expr));
+    }
+  }
+
+  bool defaultTokenHasSuffix() const {
+    StringRef Token(Tok.Data, Tok.Length);
+
+    return Token.endswith("_e32") || Token.endswith("_e64");
+  }
+
+  bool isToken() const override {
+    return Kind == Token;
+  }
+
+  bool isImm() const override {
+    return Kind == Immediate;
+  }
+
+  bool isInlineImm() const {
+    float F = BitsToFloat(Imm.Val);
+    // TODO: Add 0.5pi for VI
+    return isImm() && ((Imm.Val <= 64 && Imm.Val >= -16) ||
+           (F == 0.0 || F == 0.5 || F == -0.5 || F == 1.0 || F == -1.0 ||
+           F == 2.0 || F == -2.0 || F == 4.0 || F == -4.0));
+  }
+
+  bool isDSOffset0() const {
+    assert(isImm());
+    return Imm.Type == ImmTyDSOffset0;
+  }
+
+  bool isDSOffset1() const {
+    assert(isImm());
+    return Imm.Type == ImmTyDSOffset1;
+  }
+
+  int64_t getImm() const {
+    return Imm.Val;
+  }
+
+  enum ImmTy getImmTy() const {
+    assert(isImm());
+    return Imm.Type;
+  }
+
+  bool isRegKind() const {
+    return Kind == Register;
+  }
+
+  bool isReg() const override {
+    return Kind == Register && Reg.Modifiers == -1;
+  }
+
+  bool isRegWithInputMods() const {
+    return Kind == Register && (Reg.IsForcedVOP3 || Reg.Modifiers != -1);
+  }
+
+  void setModifiers(unsigned Mods) {
+    assert(isReg());
+    Reg.Modifiers = Mods;
+  }
+
+  bool hasModifiers() const {
+    assert(isRegKind());
+    return Reg.Modifiers != -1;
+  }
+
+  unsigned getReg() const override {
+    return Reg.RegNo;
+  }
+
+  bool isRegOrImm() const {
+    return isReg() || isImm();
+  }
+
+  bool isRegClass(unsigned RCID) const {
+    return Reg.TRI->getRegClass(RCID).contains(getReg());
+  }
+
+  bool isSCSrc32() const {
+    return isInlineImm() || (isReg() && isRegClass(AMDGPU::SReg_32RegClassID));
+  }
+
+  bool isSSrc32() const {
+    return isImm() || (isReg() && isRegClass(AMDGPU::SReg_32RegClassID));
+  }
+
+  bool isSSrc64() const {
+    return isImm() || isInlineImm() ||
+           (isReg() && isRegClass(AMDGPU::SReg_64RegClassID));
+  }
+
+  bool isVCSrc32() const {
+    return isInlineImm() || (isReg() && isRegClass(AMDGPU::VS_32RegClassID));
+  }
+
+  bool isVCSrc64() const {
+    return isInlineImm() || (isReg() && isRegClass(AMDGPU::VS_64RegClassID));
+  }
+
+  bool isVSrc32() const {
+    return isImm() || (isReg() && isRegClass(AMDGPU::VS_32RegClassID));
+  }
+
+  bool isVSrc64() const {
+    return isImm() || (isReg() && isRegClass(AMDGPU::VS_64RegClassID));
+  }
+
+  bool isMem() const override {
+    return false;
+  }
+
+  bool isExpr() const {
+    return Kind == Expression;
+  }
+
+  bool isSoppBrTarget() const {
+    return isExpr() || isImm();
+  }
+
+  SMLoc getStartLoc() const override {
+    return StartLoc;
+  }
+
+  SMLoc getEndLoc() const override {
+    return EndLoc;
+  }
+
+  void print(raw_ostream &OS) const override { }
+
+  static std::unique_ptr<AMDGPUOperand> CreateImm(int64_t Val, SMLoc Loc,
+                                                  enum ImmTy Type = ImmTyNone,
+                                                  bool IsFPImm = false) {
+    auto Op = llvm::make_unique<AMDGPUOperand>(Immediate);
+    Op->Imm.Val = Val;
+    Op->Imm.IsFPImm = IsFPImm;
+    Op->Imm.Type = Type;
+    Op->StartLoc = Loc;
+    Op->EndLoc = Loc;
+    return Op;
+  }
+
+  static std::unique_ptr<AMDGPUOperand> CreateToken(StringRef Str, SMLoc Loc,
+                                           bool HasExplicitEncodingSize = true) {
+    auto Res = llvm::make_unique<AMDGPUOperand>(Token);
+    Res->Tok.Data = Str.data();
+    Res->Tok.Length = Str.size();
+    Res->StartLoc = Loc;
+    Res->EndLoc = Loc;
+    return Res;
+  }
+
+  static std::unique_ptr<AMDGPUOperand> CreateReg(unsigned RegNo, SMLoc S,
+                                                  SMLoc E,
+                                                  const MCRegisterInfo *TRI,
+                                                  bool ForceVOP3) {
+    auto Op = llvm::make_unique<AMDGPUOperand>(Register);
+    Op->Reg.RegNo = RegNo;
+    Op->Reg.TRI = TRI;
+    Op->Reg.Modifiers = -1;
+    Op->Reg.IsForcedVOP3 = ForceVOP3;
+    Op->StartLoc = S;
+    Op->EndLoc = E;
+    return Op;
+  }
+
+  static std::unique_ptr<AMDGPUOperand> CreateExpr(const class MCExpr *Expr, SMLoc S) {
+    auto Op = llvm::make_unique<AMDGPUOperand>(Expression);
+    Op->Expr = Expr;
+    Op->StartLoc = S;
+    Op->EndLoc = S;
+    return Op;
+  }
+
+  bool isDSOffset() const;
+  bool isDSOffset01() const;
+  bool isSWaitCnt() const;
+  bool isMubufOffset() const;
+};
+
+class AMDGPUAsmParser : public MCTargetAsmParser {
+  MCSubtargetInfo &STI;
+  const MCInstrInfo &MII;
+  MCAsmParser &Parser;
+
+  unsigned ForcedEncodingSize;
+  /// @name Auto-generated Match Functions
+  /// {
+
+#define GET_ASSEMBLER_HEADER
+#include "AMDGPUGenAsmMatcher.inc"
+
+  /// }
+
+private:
+  bool ParseDirectiveMajorMinor(uint32_t &Major, uint32_t &Minor);
+  bool ParseDirectiveHSACodeObjectVersion();
+  bool ParseDirectiveHSACodeObjectISA();
+  bool ParseAMDKernelCodeTValue(StringRef ID, amd_kernel_code_t &Header);
+  bool ParseDirectiveAMDKernelCodeT();
+
+public:
+  AMDGPUAsmParser(MCSubtargetInfo &STI, MCAsmParser &_Parser,
+               const MCInstrInfo &MII,
+               const MCTargetOptions &Options)
+      : MCTargetAsmParser(Options), STI(STI), MII(MII), Parser(_Parser),
+        ForcedEncodingSize(0){
+
+    if (STI.getFeatureBits().none()) {
+      // Set default features.
+      STI.ToggleFeature("SOUTHERN_ISLANDS");
+    }
+
+    setAvailableFeatures(ComputeAvailableFeatures(STI.getFeatureBits()));
+  }
+
+  AMDGPUTargetStreamer &getTargetStreamer() {
+    MCTargetStreamer &TS = *getParser().getStreamer().getTargetStreamer();
+    return static_cast<AMDGPUTargetStreamer &>(TS);
+  }
+
+  unsigned getForcedEncodingSize() const {
+    return ForcedEncodingSize;
+  }
+
+  void setForcedEncodingSize(unsigned Size) {
+    ForcedEncodingSize = Size;
+  }
+
+  bool isForcedVOP3() const {
+    return ForcedEncodingSize == 64;
+  }
+
+  bool ParseRegister(unsigned &RegNo, SMLoc &StartLoc, SMLoc &EndLoc) override;
+  unsigned checkTargetMatchPredicate(MCInst &Inst) override;
+  bool MatchAndEmitInstruction(SMLoc IDLoc, unsigned &Opcode,
+                               OperandVector &Operands, MCStreamer &Out,
+                               uint64_t &ErrorInfo,
+                               bool MatchingInlineAsm) override;
+  bool ParseDirective(AsmToken DirectiveID) override;
+  OperandMatchResultTy parseOperand(OperandVector &Operands, StringRef Mnemonic);
+  bool ParseInstruction(ParseInstructionInfo &Info, StringRef Name,
+                        SMLoc NameLoc, OperandVector &Operands) override;
+
+  OperandMatchResultTy parseIntWithPrefix(const char *Prefix, int64_t &Int,
+                                          int64_t Default = 0);
+  OperandMatchResultTy parseIntWithPrefix(const char *Prefix,
+                                          OperandVector &Operands,
+                                          enum AMDGPUOperand::ImmTy ImmTy =
+                                                      AMDGPUOperand::ImmTyNone);
+  OperandMatchResultTy parseNamedBit(const char *Name, OperandVector &Operands,
+                                     enum AMDGPUOperand::ImmTy ImmTy =
+                                                      AMDGPUOperand::ImmTyNone);
+  OperandMatchResultTy parseOptionalOps(
+                                   const ArrayRef<OptionalOperand> &OptionalOps,
+                                   OperandVector &Operands);
+
+
+  void cvtDSOffset01(MCInst &Inst, const OperandVector &Operands);
+  void cvtDS(MCInst &Inst, const OperandVector &Operands);
+  OperandMatchResultTy parseDSOptionalOps(OperandVector &Operands);
+  OperandMatchResultTy parseDSOff01OptionalOps(OperandVector &Operands);
+  OperandMatchResultTy parseDSOffsetOptional(OperandVector &Operands);
+
+  bool parseCnt(int64_t &IntVal);
+  OperandMatchResultTy parseSWaitCntOps(OperandVector &Operands);
+  OperandMatchResultTy parseSOppBrTarget(OperandVector &Operands);
+
+  OperandMatchResultTy parseFlatOptionalOps(OperandVector &Operands);
+  OperandMatchResultTy parseFlatAtomicOptionalOps(OperandVector &Operands);
+  void cvtFlat(MCInst &Inst, const OperandVector &Operands);
+
+  void cvtMubuf(MCInst &Inst, const OperandVector &Operands);
+  OperandMatchResultTy parseOffset(OperandVector &Operands);
+  OperandMatchResultTy parseMubufOptionalOps(OperandVector &Operands);
+  OperandMatchResultTy parseGLC(OperandVector &Operands);
+  OperandMatchResultTy parseSLC(OperandVector &Operands);
+  OperandMatchResultTy parseTFE(OperandVector &Operands);
+
+  OperandMatchResultTy parseDMask(OperandVector &Operands);
+  OperandMatchResultTy parseUNorm(OperandVector &Operands);
+  OperandMatchResultTy parseR128(OperandVector &Operands);
+
+  void cvtVOP3(MCInst &Inst, const OperandVector &Operands);
+  OperandMatchResultTy parseVOP3OptionalOps(OperandVector &Operands);
+};
+
+struct OptionalOperand {
+  const char *Name;
+  AMDGPUOperand::ImmTy Type;
+  bool IsBit;
+  int64_t Default;
+  bool (*ConvertResult)(int64_t&);
+};
+
+}
+
+static unsigned getRegClass(bool IsVgpr, unsigned RegWidth) {
+  if (IsVgpr) {
+    switch (RegWidth) {
+      default: llvm_unreachable("Unknown register width");
+      case 1: return AMDGPU::VGPR_32RegClassID;
+      case 2: return AMDGPU::VReg_64RegClassID;
+      case 3: return AMDGPU::VReg_96RegClassID;
+      case 4: return AMDGPU::VReg_128RegClassID;
+      case 8: return AMDGPU::VReg_256RegClassID;
+      case 16: return AMDGPU::VReg_512RegClassID;
+    }
+  }
+
+  switch (RegWidth) {
+    default: llvm_unreachable("Unknown register width");
+    case 1: return AMDGPU::SGPR_32RegClassID;
+    case 2: return AMDGPU::SGPR_64RegClassID;
+    case 4: return AMDGPU::SReg_128RegClassID;
+    case 8: return AMDGPU::SReg_256RegClassID;
+    case 16: return AMDGPU::SReg_512RegClassID;
+  }
+}
+
+static unsigned getRegForName(const StringRef &RegName) {
+
+  return StringSwitch<unsigned>(RegName)
+    .Case("exec", AMDGPU::EXEC)
+    .Case("vcc", AMDGPU::VCC)
+    .Case("flat_scr", AMDGPU::FLAT_SCR)
+    .Case("m0", AMDGPU::M0)
+    .Case("scc", AMDGPU::SCC)
+    .Case("flat_scr_lo", AMDGPU::FLAT_SCR_LO)
+    .Case("flat_scr_hi", AMDGPU::FLAT_SCR_HI)
+    .Case("vcc_lo", AMDGPU::VCC_LO)
+    .Case("vcc_hi", AMDGPU::VCC_HI)
+    .Case("exec_lo", AMDGPU::EXEC_LO)
+    .Case("exec_hi", AMDGPU::EXEC_HI)
+    .Default(0);
+}
+
+bool AMDGPUAsmParser::ParseRegister(unsigned &RegNo, SMLoc &StartLoc, SMLoc &EndLoc) {
+  const AsmToken Tok = Parser.getTok();
+  StartLoc = Tok.getLoc();
+  EndLoc = Tok.getEndLoc();
+  const StringRef &RegName = Tok.getString();
+  RegNo = getRegForName(RegName);
+
+  if (RegNo) {
+    Parser.Lex();
+    return false;
+  }
+
+  // Match vgprs and sgprs
+  if (RegName[0] != 's' && RegName[0] != 'v')
+    return true;
+
+  bool IsVgpr = RegName[0] == 'v';
+  unsigned RegWidth;
+  unsigned RegIndexInClass;
+  if (RegName.size() > 1) {
+    // We have a 32-bit register
+    RegWidth = 1;
+    if (RegName.substr(1).getAsInteger(10, RegIndexInClass))
+      return true;
+    Parser.Lex();
+  } else {
+    // We have a register greater than 32-bits.
+
+    int64_t RegLo, RegHi;
+    Parser.Lex();
+    if (getLexer().isNot(AsmToken::LBrac))
+      return true;
+
+    Parser.Lex();
+    if (getParser().parseAbsoluteExpression(RegLo))
+      return true;
+
+    if (getLexer().isNot(AsmToken::Colon))
+      return true;
+
+    Parser.Lex();
+    if (getParser().parseAbsoluteExpression(RegHi))
+      return true;
+
+    if (getLexer().isNot(AsmToken::RBrac))
+      return true;
+
+    Parser.Lex();
+    RegWidth = (RegHi - RegLo) + 1;
+    if (IsVgpr) {
+      // VGPR registers aren't aligned.
+      RegIndexInClass = RegLo;
+    } else {
+      // SGPR registers are aligned.  Max alignment is 4 dwords.
+      RegIndexInClass = RegLo / std::min(RegWidth, 4u);
+    }
+  }
+
+  const MCRegisterInfo *TRC = getContext().getRegisterInfo();
+  unsigned RC = getRegClass(IsVgpr, RegWidth);
+  if (RegIndexInClass > TRC->getRegClass(RC).getNumRegs())
+    return true;
+  RegNo = TRC->getRegClass(RC).getRegister(RegIndexInClass);
+  return false;
+}
+
+unsigned AMDGPUAsmParser::checkTargetMatchPredicate(MCInst &Inst) {
+
+  uint64_t TSFlags = MII.get(Inst.getOpcode()).TSFlags;
+
+  if ((getForcedEncodingSize() == 32 && (TSFlags & SIInstrFlags::VOP3)) ||
+      (getForcedEncodingSize() == 64 && !(TSFlags & SIInstrFlags::VOP3)))
+    return Match_InvalidOperand;
+
+  return Match_Success;
+}
+
+
+bool AMDGPUAsmParser::MatchAndEmitInstruction(SMLoc IDLoc, unsigned &Opcode,
+                                              OperandVector &Operands,
+                                              MCStreamer &Out,
+                                              uint64_t &ErrorInfo,
+                                              bool MatchingInlineAsm) {
+  MCInst Inst;
+
+  switch (MatchInstructionImpl(Operands, Inst, ErrorInfo, MatchingInlineAsm)) {
+    default: break;
+    case Match_Success:
+      Inst.setLoc(IDLoc);
+      Out.EmitInstruction(Inst, STI);
+      return false;
+    case Match_MissingFeature:
+      return Error(IDLoc, "instruction not supported on this GPU");
+
+    case Match_MnemonicFail:
+      return Error(IDLoc, "unrecognized instruction mnemonic");
+
+    case Match_InvalidOperand: {
+      SMLoc ErrorLoc = IDLoc;
+      if (ErrorInfo != ~0ULL) {
+        if (ErrorInfo >= Operands.size()) {
+          if (isForcedVOP3()) {
+            // If 64-bit encoding has been forced we can end up with no
+            // clamp or omod operands if none of the registers have modifiers,
+            // so we need to add these to the operand list.
+            AMDGPUOperand &LastOp =
+                ((AMDGPUOperand &)*Operands[Operands.size() - 1]);
+            if (LastOp.isRegKind() ||
+               (LastOp.isImm() &&
+                LastOp.getImmTy() != AMDGPUOperand::ImmTyNone)) {
+              SMLoc S = Parser.getTok().getLoc();
+              Operands.push_back(AMDGPUOperand::CreateImm(0, S,
+                                 AMDGPUOperand::ImmTyClamp));
+              Operands.push_back(AMDGPUOperand::CreateImm(0, S,
+                                 AMDGPUOperand::ImmTyOMod));
+              bool Res = MatchAndEmitInstruction(IDLoc, Opcode, Operands,
+                                                 Out, ErrorInfo,
+                                                 MatchingInlineAsm);
+              if (!Res)
+                return Res;
+            }
+
+          }
+          return Error(IDLoc, "too few operands for instruction");
+        }
+
+        ErrorLoc = ((AMDGPUOperand &)*Operands[ErrorInfo]).getStartLoc();
+        if (ErrorLoc == SMLoc())
+          ErrorLoc = IDLoc;
+      }
+      return Error(ErrorLoc, "invalid operand for instruction");
+    }
+  }
+  llvm_unreachable("Implement any new match types added!");
+}
+
+bool AMDGPUAsmParser::ParseDirectiveMajorMinor(uint32_t &Major,
+                                               uint32_t &Minor) {
+  if (getLexer().isNot(AsmToken::Integer))
+    return TokError("invalid major version");
+
+  Major = getLexer().getTok().getIntVal();
+  Lex();
+
+  if (getLexer().isNot(AsmToken::Comma))
+    return TokError("minor version number required, comma expected");
+  Lex();
+
+  if (getLexer().isNot(AsmToken::Integer))
+    return TokError("invalid minor version");
+
+  Minor = getLexer().getTok().getIntVal();
+  Lex();
+
+  return false;
+}
+
+bool AMDGPUAsmParser::ParseDirectiveHSACodeObjectVersion() {
+
+  uint32_t Major;
+  uint32_t Minor;
+
+  if (ParseDirectiveMajorMinor(Major, Minor))
+    return true;
+
+  getTargetStreamer().EmitDirectiveHSACodeObjectVersion(Major, Minor);
+  return false;
+}
+
+bool AMDGPUAsmParser::ParseDirectiveHSACodeObjectISA() {
+
+  uint32_t Major;
+  uint32_t Minor;
+  uint32_t Stepping;
+  StringRef VendorName;
+  StringRef ArchName;
+
+  // If this directive has no arguments, then use the ISA version for the
+  // targeted GPU.
+  if (getLexer().is(AsmToken::EndOfStatement)) {
+    AMDGPU::IsaVersion Isa = AMDGPU::getIsaVersion(STI.getFeatureBits());
+    getTargetStreamer().EmitDirectiveHSACodeObjectISA(Isa.Major, Isa.Minor,
+                                                      Isa.Stepping,
+                                                      "AMD", "AMDGPU");
+    return false;
+  }
+
+
+  if (ParseDirectiveMajorMinor(Major, Minor))
+    return true;
+
+  if (getLexer().isNot(AsmToken::Comma))
+    return TokError("stepping version number required, comma expected");
+  Lex();
+
+  if (getLexer().isNot(AsmToken::Integer))
+    return TokError("invalid stepping version");
+
+  Stepping = getLexer().getTok().getIntVal();
+  Lex();
+
+  if (getLexer().isNot(AsmToken::Comma))
+    return TokError("vendor name required, comma expected");
+  Lex();
+
+  if (getLexer().isNot(AsmToken::String))
+    return TokError("invalid vendor name");
+
+  VendorName = getLexer().getTok().getStringContents();
+  Lex();
+
+  if (getLexer().isNot(AsmToken::Comma))
+    return TokError("arch name required, comma expected");
+  Lex();
+
+  if (getLexer().isNot(AsmToken::String))
+    return TokError("invalid arch name");
+
+  ArchName = getLexer().getTok().getStringContents();
+  Lex();
+
+  getTargetStreamer().EmitDirectiveHSACodeObjectISA(Major, Minor, Stepping,
+                                                    VendorName, ArchName);
+  return false;
+}
+
+bool AMDGPUAsmParser::ParseAMDKernelCodeTValue(StringRef ID,
+                                               amd_kernel_code_t &Header) {
+
+  if (getLexer().isNot(AsmToken::Equal))
+    return TokError("expected '='");
+  Lex();
+
+  if (getLexer().isNot(AsmToken::Integer))
+    return TokError("amd_kernel_code_t values must be integers");
+
+  uint64_t Value = getLexer().getTok().getIntVal();
+  Lex();
+
+  if (ID == "kernel_code_version_major")
+    Header.amd_kernel_code_version_major = Value;
+  else if (ID == "kernel_code_version_minor")
+    Header.amd_kernel_code_version_minor = Value;
+  else if (ID == "machine_kind")
+    Header.amd_machine_kind = Value;
+  else if (ID == "machine_version_major")
+    Header.amd_machine_version_major = Value;
+  else if (ID == "machine_version_minor")
+    Header.amd_machine_version_minor = Value;
+  else if (ID == "machine_version_stepping")
+    Header.amd_machine_version_stepping = Value;
+  else if (ID == "kernel_code_entry_byte_offset")
+    Header.kernel_code_entry_byte_offset = Value;
+  else if (ID == "kernel_code_prefetch_byte_size")
+    Header.kernel_code_prefetch_byte_size = Value;
+  else if (ID == "max_scratch_backing_memory_byte_size")
+    Header.max_scratch_backing_memory_byte_size = Value;
+  else if (ID == "compute_pgm_rsrc1_vgprs")
+    Header.compute_pgm_resource_registers |= S_00B848_VGPRS(Value);
+  else if (ID == "compute_pgm_rsrc1_sgprs")
+    Header.compute_pgm_resource_registers |= S_00B848_SGPRS(Value);
+  else if (ID == "compute_pgm_rsrc1_priority")
+    Header.compute_pgm_resource_registers |= S_00B848_PRIORITY(Value);
+  else if (ID == "compute_pgm_rsrc1_float_mode")
+    Header.compute_pgm_resource_registers |= S_00B848_FLOAT_MODE(Value);
+  else if (ID == "compute_pgm_rsrc1_priv")
+    Header.compute_pgm_resource_registers |= S_00B848_PRIV(Value);
+  else if (ID == "compute_pgm_rsrc1_dx10_clamp")
+    Header.compute_pgm_resource_registers |= S_00B848_DX10_CLAMP(Value);
+  else if (ID == "compute_pgm_rsrc1_debug_mode")
+    Header.compute_pgm_resource_registers |= S_00B848_DEBUG_MODE(Value);
+  else if (ID == "compute_pgm_rsrc1_ieee_mode")
+    Header.compute_pgm_resource_registers |= S_00B848_IEEE_MODE(Value);
+  else if (ID == "compute_pgm_rsrc2_scratch_en")
+    Header.compute_pgm_resource_registers |= (S_00B84C_SCRATCH_EN(Value) << 32);
+  else if (ID == "compute_pgm_rsrc2_user_sgpr")
+    Header.compute_pgm_resource_registers |= (S_00B84C_USER_SGPR(Value) << 32);
+  else if (ID == "compute_pgm_rsrc2_tgid_x_en")
+    Header.compute_pgm_resource_registers |= (S_00B84C_TGID_X_EN(Value) << 32);
+  else if (ID == "compute_pgm_rsrc2_tgid_y_en")
+    Header.compute_pgm_resource_registers |= (S_00B84C_TGID_Y_EN(Value) << 32);
+  else if (ID == "compute_pgm_rsrc2_tgid_z_en")
+    Header.compute_pgm_resource_registers |= (S_00B84C_TGID_Z_EN(Value) << 32);
+  else if (ID == "compute_pgm_rsrc2_tg_size_en")
+    Header.compute_pgm_resource_registers |= (S_00B84C_TG_SIZE_EN(Value) << 32);
+  else if (ID == "compute_pgm_rsrc2_tidig_comp_cnt")
+    Header.compute_pgm_resource_registers |=
+        (S_00B84C_TIDIG_COMP_CNT(Value) << 32);
+  else if (ID == "compute_pgm_rsrc2_excp_en_msb")
+    Header.compute_pgm_resource_registers |=
+        (S_00B84C_EXCP_EN_MSB(Value) << 32);
+  else if (ID == "compute_pgm_rsrc2_lds_size")
+    Header.compute_pgm_resource_registers |= (S_00B84C_LDS_SIZE(Value) << 32);
+  else if (ID == "compute_pgm_rsrc2_excp_en")
+    Header.compute_pgm_resource_registers |= (S_00B84C_EXCP_EN(Value) << 32);
+  else if (ID == "compute_pgm_resource_registers")
+    Header.compute_pgm_resource_registers = Value;
+  else if (ID == "enable_sgpr_private_segment_buffer")
+    Header.code_properties |=
+        (Value << AMD_CODE_PROPERTY_ENABLE_SGPR_PRIVATE_SEGMENT_BUFFER_SHIFT);
+  else if (ID == "enable_sgpr_dispatch_ptr")
+    Header.code_properties |=
+        (Value << AMD_CODE_PROPERTY_ENABLE_SGPR_DISPATCH_PTR_SHIFT);
+  else if (ID == "enable_sgpr_queue_ptr")
+    Header.code_properties |=
+        (Value << AMD_CODE_PROPERTY_ENABLE_SGPR_QUEUE_PTR_SHIFT);
+  else if (ID == "enable_sgpr_kernarg_segment_ptr")
+    Header.code_properties |=
+        (Value << AMD_CODE_PROPERTY_ENABLE_SGPR_KERNARG_SEGMENT_PTR_SHIFT);
+  else if (ID == "enable_sgpr_dispatch_id")
+    Header.code_properties |=
+        (Value << AMD_CODE_PROPERTY_ENABLE_SGPR_DISPATCH_ID_SHIFT);
+  else if (ID == "enable_sgpr_flat_scratch_init")
+    Header.code_properties |=
+        (Value << AMD_CODE_PROPERTY_ENABLE_SGPR_FLAT_SCRATCH_INIT_SHIFT);
+  else if (ID == "enable_sgpr_private_segment_size")
+    Header.code_properties |=
+        (Value << AMD_CODE_PROPERTY_ENABLE_SGPR_PRIVATE_SEGMENT_SIZE_SHIFT);
+  else if (ID == "enable_sgpr_grid_workgroup_count_x")
+    Header.code_properties |=
+        (Value << AMD_CODE_PROPERTY_ENABLE_SGPR_GRID_WORKGROUP_COUNT_X_SHIFT);
+  else if (ID == "enable_sgpr_grid_workgroup_count_y")
+    Header.code_properties |=
+        (Value << AMD_CODE_PROPERTY_ENABLE_SGPR_GRID_WORKGROUP_COUNT_Y_SHIFT);
+  else if (ID == "enable_sgpr_grid_workgroup_count_z")
+    Header.code_properties |=
+        (Value << AMD_CODE_PROPERTY_ENABLE_SGPR_GRID_WORKGROUP_COUNT_Z_SHIFT);
+  else if (ID == "enable_ordered_append_gds")
+    Header.code_properties |=
+        (Value << AMD_CODE_PROPERTY_ENABLE_ORDERED_APPEND_GDS_SHIFT);
+  else if (ID == "private_element_size")
+    Header.code_properties |=
+        (Value << AMD_CODE_PROPERTY_PRIVATE_ELEMENT_SIZE_SHIFT);
+  else if (ID == "is_ptr64")
+    Header.code_properties |=
+        (Value << AMD_CODE_PROPERTY_IS_PTR64_SHIFT);
+  else if (ID == "is_dynamic_callstack")
+    Header.code_properties |=
+        (Value << AMD_CODE_PROPERTY_IS_DYNAMIC_CALLSTACK_SHIFT);
+  else if (ID == "is_debug_enabled")
+    Header.code_properties |=
+        (Value << AMD_CODE_PROPERTY_IS_DEBUG_SUPPORTED_SHIFT);
+  else if (ID == "is_xnack_enabled")
+    Header.code_properties |=
+        (Value << AMD_CODE_PROPERTY_IS_XNACK_SUPPORTED_SHIFT);
+  else if (ID == "workitem_private_segment_byte_size")
+    Header.workitem_private_segment_byte_size = Value;
+  else if (ID == "workgroup_group_segment_byte_size")
+    Header.workgroup_group_segment_byte_size = Value;
+  else if (ID == "gds_segment_byte_size")
+    Header.gds_segment_byte_size = Value;
+  else if (ID == "kernarg_segment_byte_size")
+    Header.kernarg_segment_byte_size = Value;
+  else if (ID == "workgroup_fbarrier_count")
+    Header.workgroup_fbarrier_count = Value;
+  else if (ID == "wavefront_sgpr_count")
+    Header.wavefront_sgpr_count = Value;
+  else if (ID == "workitem_vgpr_count")
+    Header.workitem_vgpr_count = Value;
+  else if (ID == "reserved_vgpr_first")
+    Header.reserved_vgpr_first = Value;
+  else if (ID == "reserved_vgpr_count")
+    Header.reserved_vgpr_count = Value;
+  else if (ID == "reserved_sgpr_first")
+    Header.reserved_sgpr_first = Value;
+  else if (ID == "reserved_sgpr_count")
+    Header.reserved_sgpr_count = Value;
+  else if (ID == "debug_wavefront_private_segment_offset_sgpr")
+    Header.debug_wavefront_private_segment_offset_sgpr = Value;
+  else if (ID == "debug_private_segment_buffer_sgpr")
+    Header.debug_private_segment_buffer_sgpr = Value;
+  else if (ID == "kernarg_segment_alignment")
+    Header.kernarg_segment_alignment = Value;
+  else if (ID == "group_segment_alignment")
+    Header.group_segment_alignment = Value;
+  else if (ID == "private_segment_alignment")
+    Header.private_segment_alignment = Value;
+  else if (ID == "wavefront_size")
+    Header.wavefront_size = Value;
+  else if (ID == "call_convention")
+    Header.call_convention = Value;
+  else if (ID == "runtime_loader_kernel_symbol")
+    Header.runtime_loader_kernel_symbol = Value;
+  else
+    return TokError("amd_kernel_code_t value not recognized.");
+
+  return false;
+}
+
+bool AMDGPUAsmParser::ParseDirectiveAMDKernelCodeT() {
+
+  amd_kernel_code_t Header;
+  AMDGPU::initDefaultAMDKernelCodeT(Header, STI.getFeatureBits());
+
+  while (true) {
+
+    if (getLexer().isNot(AsmToken::EndOfStatement))
+      return TokError("amd_kernel_code_t values must begin on a new line");
+
+    // Lex EndOfStatement.  This is in a while loop, because lexing a comment
+    // will set the current token to EndOfStatement.
+    while(getLexer().is(AsmToken::EndOfStatement))
+      Lex();
+
+    if (getLexer().isNot(AsmToken::Identifier))
+      return TokError("expected value identifier or .end_amd_kernel_code_t");
+
+    StringRef ID = getLexer().getTok().getIdentifier();
+    Lex();
+
+    if (ID == ".end_amd_kernel_code_t")
+      break;
+
+    if (ParseAMDKernelCodeTValue(ID, Header))
+      return true;
+  }
+
+  getTargetStreamer().EmitAMDKernelCodeT(Header);
+
+  return false;
+}
+
+bool AMDGPUAsmParser::ParseDirective(AsmToken DirectiveID) {
+  StringRef IDVal = DirectiveID.getString();
+
+  if (IDVal == ".hsa_code_object_version")
+    return ParseDirectiveHSACodeObjectVersion();
+
+  if (IDVal == ".hsa_code_object_isa")
+    return ParseDirectiveHSACodeObjectISA();
+
+  if (IDVal == ".amd_kernel_code_t")
+    return ParseDirectiveAMDKernelCodeT();
+
+  return true;
+}
+
+static bool operandsHaveModifiers(const OperandVector &Operands) {
+
+  for (unsigned i = 0, e = Operands.size(); i != e; ++i) {
+    const AMDGPUOperand &Op = ((AMDGPUOperand&)*Operands[i]);
+    if (Op.isRegKind() && Op.hasModifiers())
+      return true;
+    if (Op.isImm() && (Op.getImmTy() == AMDGPUOperand::ImmTyOMod ||
+                       Op.getImmTy() == AMDGPUOperand::ImmTyClamp))
+      return true;
+  }
+  return false;
+}
+
+AMDGPUAsmParser::OperandMatchResultTy
+AMDGPUAsmParser::parseOperand(OperandVector &Operands, StringRef Mnemonic) {
+
+  // Try to parse with a custom parser
+  OperandMatchResultTy ResTy = MatchOperandParserImpl(Operands, Mnemonic);
+
+  // If we successfully parsed the operand or if there as an error parsing,
+  // we are done.
+  //
+  // If we are parsing after we reach EndOfStatement then this means we
+  // are appending default values to the Operands list.  This is only done
+  // by custom parser, so we shouldn't continue on to the generic parsing.
+  if (ResTy == MatchOperand_Success || ResTy == MatchOperand_ParseFail ||
+      getLexer().is(AsmToken::EndOfStatement))
+    return ResTy;
+
+  bool Negate = false, Abs = false;
+  if (getLexer().getKind()== AsmToken::Minus) {
+    Parser.Lex();
+    Negate = true;
+  }
+
+  if (getLexer().getKind() == AsmToken::Pipe) {
+    Parser.Lex();
+    Abs = true;
+  }
+
+  switch(getLexer().getKind()) {
+    case AsmToken::Integer: {
+      SMLoc S = Parser.getTok().getLoc();
+      int64_t IntVal;
+      if (getParser().parseAbsoluteExpression(IntVal))
+        return MatchOperand_ParseFail;
+      APInt IntVal32(32, IntVal);
+      if (IntVal32.getSExtValue() != IntVal) {
+        Error(S, "invalid immediate: only 32-bit values are legal");
+        return MatchOperand_ParseFail;
+      }
+
+      IntVal = IntVal32.getSExtValue();
+      if (Negate)
+        IntVal *= -1;
+      Operands.push_back(AMDGPUOperand::CreateImm(IntVal, S));
+      return MatchOperand_Success;
+    }
+    case AsmToken::Real: {
+      // FIXME: We should emit an error if a double precisions floating-point
+      // value is used.  I'm not sure the best way to detect this.
+      SMLoc S = Parser.getTok().getLoc();
+      int64_t IntVal;
+      if (getParser().parseAbsoluteExpression(IntVal))
+        return MatchOperand_ParseFail;
+
+      APFloat F((float)BitsToDouble(IntVal));
+      if (Negate)
+        F.changeSign();
+      Operands.push_back(
+          AMDGPUOperand::CreateImm(F.bitcastToAPInt().getZExtValue(), S));
+      return MatchOperand_Success;
+    }
+    case AsmToken::Identifier: {
+      SMLoc S, E;
+      unsigned RegNo;
+      if (!ParseRegister(RegNo, S, E)) {
+
+        bool HasModifiers = operandsHaveModifiers(Operands);
+        unsigned Modifiers = 0;
+
+        if (Negate)
+          Modifiers |= 0x1;
+
+        if (Abs) {
+          if (getLexer().getKind() != AsmToken::Pipe)
+            return MatchOperand_ParseFail;
+          Parser.Lex();
+          Modifiers |= 0x2;
+        }
+
+        if (Modifiers && !HasModifiers) {
+          // We are adding a modifier to src1 or src2 and previous sources
+          // don't have modifiers, so we need to go back and empty modifers
+          // for each previous source.
+          for (unsigned PrevRegIdx = Operands.size() - 1; PrevRegIdx > 1;
+               --PrevRegIdx) {
+
+            AMDGPUOperand &RegOp = ((AMDGPUOperand&)*Operands[PrevRegIdx]);
+            RegOp.setModifiers(0);
+          }
+        }
+
+
+        Operands.push_back(AMDGPUOperand::CreateReg(
+            RegNo, S, E, getContext().getRegisterInfo(),
+            isForcedVOP3()));
+
+        if (HasModifiers || Modifiers) {
+          AMDGPUOperand &RegOp = ((AMDGPUOperand&)*Operands[Operands.size() - 1]);
+          RegOp.setModifiers(Modifiers);
+
+        }
+     }  else {
+      Operands.push_back(AMDGPUOperand::CreateToken(Parser.getTok().getString(),
+                                                    S));
+      Parser.Lex();
+     }
+     return MatchOperand_Success;
+    }
+    default:
+      return MatchOperand_NoMatch;
+  }
+}
+
+bool AMDGPUAsmParser::ParseInstruction(ParseInstructionInfo &Info,
+                                       StringRef Name,
+                                       SMLoc NameLoc, OperandVector &Operands) {
+
+  // Clear any forced encodings from the previous instruction.
+  setForcedEncodingSize(0);
+
+  if (Name.endswith("_e64"))
+    setForcedEncodingSize(64);
+  else if (Name.endswith("_e32"))
+    setForcedEncodingSize(32);
+
+  // Add the instruction mnemonic
+  Operands.push_back(AMDGPUOperand::CreateToken(Name, NameLoc));
+
+  while (!getLexer().is(AsmToken::EndOfStatement)) {
+    AMDGPUAsmParser::OperandMatchResultTy Res = parseOperand(Operands, Name);
+
+    // Eat the comma or space if there is one.
+    if (getLexer().is(AsmToken::Comma))
+      Parser.Lex();
+
+    switch (Res) {
+      case MatchOperand_Success: break;
+      case MatchOperand_ParseFail: return Error(getLexer().getLoc(),
+                                                "failed parsing operand.");
+      case MatchOperand_NoMatch: return Error(getLexer().getLoc(),
+                                              "not a valid operand.");
+    }
+  }
+
+  // Once we reach end of statement, continue parsing so we can add default
+  // values for optional arguments.
+  AMDGPUAsmParser::OperandMatchResultTy Res;
+  while ((Res = parseOperand(Operands, Name)) != MatchOperand_NoMatch) {
+    if (Res != MatchOperand_Success)
+      return Error(getLexer().getLoc(), "failed parsing operand.");
+  }
+  return false;
+}
+
+//===----------------------------------------------------------------------===//
+// Utility functions
+//===----------------------------------------------------------------------===//
+
+AMDGPUAsmParser::OperandMatchResultTy
+AMDGPUAsmParser::parseIntWithPrefix(const char *Prefix, int64_t &Int,
+                                    int64_t Default) {
+
+  // We are at the end of the statement, and this is a default argument, so
+  // use a default value.
+  if (getLexer().is(AsmToken::EndOfStatement)) {
+    Int = Default;
+    return MatchOperand_Success;
+  }
+
+  switch(getLexer().getKind()) {
+    default: return MatchOperand_NoMatch;
+    case AsmToken::Identifier: {
+      StringRef OffsetName = Parser.getTok().getString();
+      if (!OffsetName.equals(Prefix))
+        return MatchOperand_NoMatch;
+
+      Parser.Lex();
+      if (getLexer().isNot(AsmToken::Colon))
+        return MatchOperand_ParseFail;
+
+      Parser.Lex();
+      if (getLexer().isNot(AsmToken::Integer))
+        return MatchOperand_ParseFail;
+
+      if (getParser().parseAbsoluteExpression(Int))
+        return MatchOperand_ParseFail;
+      break;
+    }
+  }
+  return MatchOperand_Success;
+}
+
+AMDGPUAsmParser::OperandMatchResultTy
+AMDGPUAsmParser::parseIntWithPrefix(const char *Prefix, OperandVector &Operands,
+                                    enum AMDGPUOperand::ImmTy ImmTy) {
+
+  SMLoc S = Parser.getTok().getLoc();
+  int64_t Offset = 0;
+
+  AMDGPUAsmParser::OperandMatchResultTy Res = parseIntWithPrefix(Prefix, Offset);
+  if (Res != MatchOperand_Success)
+    return Res;
+
+  Operands.push_back(AMDGPUOperand::CreateImm(Offset, S, ImmTy));
+  return MatchOperand_Success;
+}
+
+AMDGPUAsmParser::OperandMatchResultTy
+AMDGPUAsmParser::parseNamedBit(const char *Name, OperandVector &Operands,
+                               enum AMDGPUOperand::ImmTy ImmTy) {
+  int64_t Bit = 0;
+  SMLoc S = Parser.getTok().getLoc();
+
+  // We are at the end of the statement, and this is a default argument, so
+  // use a default value.
+  if (getLexer().isNot(AsmToken::EndOfStatement)) {
+    switch(getLexer().getKind()) {
+      case AsmToken::Identifier: {
+        StringRef Tok = Parser.getTok().getString();
+        if (Tok == Name) {
+          Bit = 1;
+          Parser.Lex();
+        } else if (Tok.startswith("no") && Tok.endswith(Name)) {
+          Bit = 0;
+          Parser.Lex();
+        } else {
+          return MatchOperand_NoMatch;
+        }
+        break;
+      }
+      default:
+        return MatchOperand_NoMatch;
+    }
+  }
+
+  Operands.push_back(AMDGPUOperand::CreateImm(Bit, S, ImmTy));
+  return MatchOperand_Success;
+}
+
+static bool operandsHasOptionalOp(const OperandVector &Operands,
+                                  const OptionalOperand &OOp) {
+  for (unsigned i = 0; i < Operands.size(); i++) {
+    const AMDGPUOperand &ParsedOp = ((const AMDGPUOperand &)*Operands[i]);
+    if ((ParsedOp.isImm() && ParsedOp.getImmTy() == OOp.Type) ||
+        (ParsedOp.isToken() && ParsedOp.getToken() == OOp.Name))
+      return true;
+
+  }
+  return false;
+}
+
+AMDGPUAsmParser::OperandMatchResultTy
+AMDGPUAsmParser::parseOptionalOps(const ArrayRef<OptionalOperand> &OptionalOps,
+                                   OperandVector &Operands) {
+  SMLoc S = Parser.getTok().getLoc();
+  for (const OptionalOperand &Op : OptionalOps) {
+    if (operandsHasOptionalOp(Operands, Op))
+      continue;
+    AMDGPUAsmParser::OperandMatchResultTy Res;
+    int64_t Value;
+    if (Op.IsBit) {
+      Res = parseNamedBit(Op.Name, Operands, Op.Type);
+      if (Res == MatchOperand_NoMatch)
+        continue;
+      return Res;
+    }
+
+    Res = parseIntWithPrefix(Op.Name, Value, Op.Default);
+
+    if (Res == MatchOperand_NoMatch)
+      continue;
+
+    if (Res != MatchOperand_Success)
+      return Res;
+
+    if (Op.ConvertResult && !Op.ConvertResult(Value)) {
+      return MatchOperand_ParseFail;
+    }
+
+    Operands.push_back(AMDGPUOperand::CreateImm(Value, S, Op.Type));
+    return MatchOperand_Success;
+  }
+  return MatchOperand_NoMatch;
+}
+
+//===----------------------------------------------------------------------===//
+// ds
+//===----------------------------------------------------------------------===//
+
+static const OptionalOperand DSOptionalOps [] = {
+  {"offset",  AMDGPUOperand::ImmTyOffset, false, 0, nullptr},
+  {"gds",     AMDGPUOperand::ImmTyGDS, true, 0, nullptr}
+};
+
+static const OptionalOperand DSOptionalOpsOff01 [] = {
+  {"offset0", AMDGPUOperand::ImmTyDSOffset0, false, 0, nullptr},
+  {"offset1", AMDGPUOperand::ImmTyDSOffset1, false, 0, nullptr},
+  {"gds",     AMDGPUOperand::ImmTyGDS, true, 0, nullptr}
+};
+
+AMDGPUAsmParser::OperandMatchResultTy
+AMDGPUAsmParser::parseDSOptionalOps(OperandVector &Operands) {
+  return parseOptionalOps(DSOptionalOps, Operands);
+}
+AMDGPUAsmParser::OperandMatchResultTy
+AMDGPUAsmParser::parseDSOff01OptionalOps(OperandVector &Operands) {
+  return parseOptionalOps(DSOptionalOpsOff01, Operands);
+}
+
+AMDGPUAsmParser::OperandMatchResultTy
+AMDGPUAsmParser::parseDSOffsetOptional(OperandVector &Operands) {
+  SMLoc S = Parser.getTok().getLoc();
+  AMDGPUAsmParser::OperandMatchResultTy Res =
+    parseIntWithPrefix("offset", Operands, AMDGPUOperand::ImmTyOffset);
+  if (Res == MatchOperand_NoMatch) {
+    Operands.push_back(AMDGPUOperand::CreateImm(0, S,
+                       AMDGPUOperand::ImmTyOffset));
+    Res = MatchOperand_Success;
+  }
+  return Res;
+}
+
+bool AMDGPUOperand::isDSOffset() const {
+  return isImm() && isUInt<16>(getImm());
+}
+
+bool AMDGPUOperand::isDSOffset01() const {
+  return isImm() && isUInt<8>(getImm());
+}
+
+void AMDGPUAsmParser::cvtDSOffset01(MCInst &Inst,
+                                    const OperandVector &Operands) {
+
+  std::map<enum AMDGPUOperand::ImmTy, unsigned> OptionalIdx;
+
+  for (unsigned i = 1, e = Operands.size(); i != e; ++i) {
+    AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[i]);
+
+    // Add the register arguments
+    if (Op.isReg()) {
+      Op.addRegOperands(Inst, 1);
+      continue;
+    }
+
+    // Handle optional arguments
+    OptionalIdx[Op.getImmTy()] = i;
+  }
+
+  unsigned Offset0Idx = OptionalIdx[AMDGPUOperand::ImmTyDSOffset0];
+  unsigned Offset1Idx = OptionalIdx[AMDGPUOperand::ImmTyDSOffset1];
+  unsigned GDSIdx = OptionalIdx[AMDGPUOperand::ImmTyGDS];
+
+  ((AMDGPUOperand &)*Operands[Offset0Idx]).addImmOperands(Inst, 1); // offset0
+  ((AMDGPUOperand &)*Operands[Offset1Idx]).addImmOperands(Inst, 1); // offset1
+  ((AMDGPUOperand &)*Operands[GDSIdx]).addImmOperands(Inst, 1); // gds
+  Inst.addOperand(MCOperand::createReg(AMDGPU::M0)); // m0
+}
+
+void AMDGPUAsmParser::cvtDS(MCInst &Inst, const OperandVector &Operands) {
+
+  std::map<enum AMDGPUOperand::ImmTy, unsigned> OptionalIdx;
+  bool GDSOnly = false;
+
+  for (unsigned i = 1, e = Operands.size(); i != e; ++i) {
+    AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[i]);
+
+    // Add the register arguments
+    if (Op.isReg()) {
+      Op.addRegOperands(Inst, 1);
+      continue;
+    }
+
+    if (Op.isToken() && Op.getToken() == "gds") {
+      GDSOnly = true;
+      continue;
+    }
+
+    // Handle optional arguments
+    OptionalIdx[Op.getImmTy()] = i;
+  }
+
+  unsigned OffsetIdx = OptionalIdx[AMDGPUOperand::ImmTyOffset];
+  ((AMDGPUOperand &)*Operands[OffsetIdx]).addImmOperands(Inst, 1); // offset
+
+  if (!GDSOnly) {
+    unsigned GDSIdx = OptionalIdx[AMDGPUOperand::ImmTyGDS];
+    ((AMDGPUOperand &)*Operands[GDSIdx]).addImmOperands(Inst, 1); // gds
+  }
+  Inst.addOperand(MCOperand::createReg(AMDGPU::M0)); // m0
+}
+
+
+//===----------------------------------------------------------------------===//
+// s_waitcnt
+//===----------------------------------------------------------------------===//
+
+bool AMDGPUAsmParser::parseCnt(int64_t &IntVal) {
+  StringRef CntName = Parser.getTok().getString();
+  int64_t CntVal;
+
+  Parser.Lex();
+  if (getLexer().isNot(AsmToken::LParen))
+    return true;
+
+  Parser.Lex();
+  if (getLexer().isNot(AsmToken::Integer))
+    return true;
+
+  if (getParser().parseAbsoluteExpression(CntVal))
+    return true;
+
+  if (getLexer().isNot(AsmToken::RParen))
+    return true;
+
+  Parser.Lex();
+  if (getLexer().is(AsmToken::Amp) || getLexer().is(AsmToken::Comma))
+    Parser.Lex();
+
+  int CntShift;
+  int CntMask;
+
+  if (CntName == "vmcnt") {
+    CntMask = 0xf;
+    CntShift = 0;
+  } else if (CntName == "expcnt") {
+    CntMask = 0x7;
+    CntShift = 4;
+  } else if (CntName == "lgkmcnt") {
+    CntMask = 0x7;
+    CntShift = 8;
+  } else {
+    return true;
+  }
+
+  IntVal &= ~(CntMask << CntShift);
+  IntVal |= (CntVal << CntShift);
+  return false;
+}
+
+AMDGPUAsmParser::OperandMatchResultTy
+AMDGPUAsmParser::parseSWaitCntOps(OperandVector &Operands) {
+  // Disable all counters by default.
+  // vmcnt   [3:0]
+  // expcnt  [6:4]
+  // lgkmcnt [10:8]
+  int64_t CntVal = 0x77f;
+  SMLoc S = Parser.getTok().getLoc();
+
+  switch(getLexer().getKind()) {
+    default: return MatchOperand_ParseFail;
+    case AsmToken::Integer:
+      // The operand can be an integer value.
+      if (getParser().parseAbsoluteExpression(CntVal))
+        return MatchOperand_ParseFail;
+      break;
+
+    case AsmToken::Identifier:
+      do {
+        if (parseCnt(CntVal))
+          return MatchOperand_ParseFail;
+      } while(getLexer().isNot(AsmToken::EndOfStatement));
+      break;
+  }
+  Operands.push_back(AMDGPUOperand::CreateImm(CntVal, S));
+  return MatchOperand_Success;
+}
+
+bool AMDGPUOperand::isSWaitCnt() const {
+  return isImm();
+}
+
+//===----------------------------------------------------------------------===//
+// sopp branch targets
+//===----------------------------------------------------------------------===//
+
+AMDGPUAsmParser::OperandMatchResultTy
+AMDGPUAsmParser::parseSOppBrTarget(OperandVector &Operands) {
+  SMLoc S = Parser.getTok().getLoc();
+
+  switch (getLexer().getKind()) {
+    default: return MatchOperand_ParseFail;
+    case AsmToken::Integer: {
+      int64_t Imm;
+      if (getParser().parseAbsoluteExpression(Imm))
+        return MatchOperand_ParseFail;
+      Operands.push_back(AMDGPUOperand::CreateImm(Imm, S));
+      return MatchOperand_Success;
+    }
+
+    case AsmToken::Identifier:
+      Operands.push_back(AMDGPUOperand::CreateExpr(
+          MCSymbolRefExpr::create(getContext().getOrCreateSymbol(
+                                  Parser.getTok().getString()), getContext()), S));
+      Parser.Lex();
+      return MatchOperand_Success;
+  }
+}
+
+//===----------------------------------------------------------------------===//
+// flat
+//===----------------------------------------------------------------------===//
+
+static const OptionalOperand FlatOptionalOps [] = {
+  {"glc",    AMDGPUOperand::ImmTyGLC, true, 0, nullptr},
+  {"slc",    AMDGPUOperand::ImmTySLC, true, 0, nullptr},
+  {"tfe",    AMDGPUOperand::ImmTyTFE, true, 0, nullptr}
+};
+
+static const OptionalOperand FlatAtomicOptionalOps [] = {
+  {"slc",    AMDGPUOperand::ImmTySLC, true, 0, nullptr},
+  {"tfe",    AMDGPUOperand::ImmTyTFE, true, 0, nullptr}
+};
+
+AMDGPUAsmParser::OperandMatchResultTy
+AMDGPUAsmParser::parseFlatOptionalOps(OperandVector &Operands) {
+  return parseOptionalOps(FlatOptionalOps, Operands);
+}
+
+AMDGPUAsmParser::OperandMatchResultTy
+AMDGPUAsmParser::parseFlatAtomicOptionalOps(OperandVector &Operands) {
+  return parseOptionalOps(FlatAtomicOptionalOps, Operands);
+}
+
+void AMDGPUAsmParser::cvtFlat(MCInst &Inst,
+                               const OperandVector &Operands) {
+  std::map<AMDGPUOperand::ImmTy, unsigned> OptionalIdx;
+
+  for (unsigned i = 1, e = Operands.size(); i != e; ++i) {
+    AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[i]);
+
+    // Add the register arguments
+    if (Op.isReg()) {
+      Op.addRegOperands(Inst, 1);
+      continue;
+    }
+
+    // Handle 'glc' token which is sometimes hard-coded into the
+    // asm string.  There are no MCInst operands for these.
+    if (Op.isToken())
+      continue;
+
+    // Handle optional arguments
+    OptionalIdx[Op.getImmTy()] = i;
+
+  }
+
+  // flat atomic instructions don't have a glc argument.
+  if (OptionalIdx.count(AMDGPUOperand::ImmTyGLC)) {
+    unsigned GLCIdx = OptionalIdx[AMDGPUOperand::ImmTyGLC];
+    ((AMDGPUOperand &)*Operands[GLCIdx]).addImmOperands(Inst, 1);
+  }
+
+  unsigned SLCIdx = OptionalIdx[AMDGPUOperand::ImmTySLC];
+  unsigned TFEIdx = OptionalIdx[AMDGPUOperand::ImmTyTFE];
+
+  ((AMDGPUOperand &)*Operands[SLCIdx]).addImmOperands(Inst, 1);
+  ((AMDGPUOperand &)*Operands[TFEIdx]).addImmOperands(Inst, 1);
+}
+
+//===----------------------------------------------------------------------===//
+// mubuf
+//===----------------------------------------------------------------------===//
+
+static const OptionalOperand MubufOptionalOps [] = {
+  {"offset", AMDGPUOperand::ImmTyOffset, false, 0, nullptr},
+  {"glc",    AMDGPUOperand::ImmTyGLC, true, 0, nullptr},
+  {"slc",    AMDGPUOperand::ImmTySLC, true, 0, nullptr},
+  {"tfe",    AMDGPUOperand::ImmTyTFE, true, 0, nullptr}
+};
+
+AMDGPUAsmParser::OperandMatchResultTy
+AMDGPUAsmParser::parseMubufOptionalOps(OperandVector &Operands) {
+  return parseOptionalOps(MubufOptionalOps, Operands);
+}
+
+AMDGPUAsmParser::OperandMatchResultTy
+AMDGPUAsmParser::parseOffset(OperandVector &Operands) {
+  return parseIntWithPrefix("offset", Operands);
+}
+
+AMDGPUAsmParser::OperandMatchResultTy
+AMDGPUAsmParser::parseGLC(OperandVector &Operands) {
+  return parseNamedBit("glc", Operands);
+}
+
+AMDGPUAsmParser::OperandMatchResultTy
+AMDGPUAsmParser::parseSLC(OperandVector &Operands) {
+  return parseNamedBit("slc", Operands);
+}
+
+AMDGPUAsmParser::OperandMatchResultTy
+AMDGPUAsmParser::parseTFE(OperandVector &Operands) {
+  return parseNamedBit("tfe", Operands);
+}
+
+bool AMDGPUOperand::isMubufOffset() const {
+  return isImm() && isUInt<12>(getImm());
+}
+
+void AMDGPUAsmParser::cvtMubuf(MCInst &Inst,
+                               const OperandVector &Operands) {
+  std::map<enum AMDGPUOperand::ImmTy, unsigned> OptionalIdx;
+
+  for (unsigned i = 1, e = Operands.size(); i != e; ++i) {
+    AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[i]);
+
+    // Add the register arguments
+    if (Op.isReg()) {
+      Op.addRegOperands(Inst, 1);
+      continue;
+    }
+
+    // Handle the case where soffset is an immediate
+    if (Op.isImm() && Op.getImmTy() == AMDGPUOperand::ImmTyNone) {
+      Op.addImmOperands(Inst, 1);
+      continue;
+    }
+
+    // Handle tokens like 'offen' which are sometimes hard-coded into the
+    // asm string.  There are no MCInst operands for these.
+    if (Op.isToken()) {
+      continue;
+    }
+    assert(Op.isImm());
+
+    // Handle optional arguments
+    OptionalIdx[Op.getImmTy()] = i;
+  }
+
+  assert(OptionalIdx.size() == 4);
+
+  unsigned OffsetIdx = OptionalIdx[AMDGPUOperand::ImmTyOffset];
+  unsigned GLCIdx = OptionalIdx[AMDGPUOperand::ImmTyGLC];
+  unsigned SLCIdx = OptionalIdx[AMDGPUOperand::ImmTySLC];
+  unsigned TFEIdx = OptionalIdx[AMDGPUOperand::ImmTyTFE];
+
+  ((AMDGPUOperand &)*Operands[OffsetIdx]).addImmOperands(Inst, 1);
+  ((AMDGPUOperand &)*Operands[GLCIdx]).addImmOperands(Inst, 1);
+  ((AMDGPUOperand &)*Operands[SLCIdx]).addImmOperands(Inst, 1);
+  ((AMDGPUOperand &)*Operands[TFEIdx]).addImmOperands(Inst, 1);
+}
+
+//===----------------------------------------------------------------------===//
+// mimg
+//===----------------------------------------------------------------------===//
+
+AMDGPUAsmParser::OperandMatchResultTy
+AMDGPUAsmParser::parseDMask(OperandVector &Operands) {
+  return parseIntWithPrefix("dmask", Operands);
+}
+
+AMDGPUAsmParser::OperandMatchResultTy
+AMDGPUAsmParser::parseUNorm(OperandVector &Operands) {
+  return parseNamedBit("unorm", Operands);
+}
+
+AMDGPUAsmParser::OperandMatchResultTy
+AMDGPUAsmParser::parseR128(OperandVector &Operands) {
+  return parseNamedBit("r128", Operands);
+}
+
+//===----------------------------------------------------------------------===//
+// vop3
+//===----------------------------------------------------------------------===//
+
+static bool ConvertOmodMul(int64_t &Mul) {
+  if (Mul != 1 && Mul != 2 && Mul != 4)
+    return false;
+
+  Mul >>= 1;
+  return true;
+}
+
+static bool ConvertOmodDiv(int64_t &Div) {
+  if (Div == 1) {
+    Div = 0;
+    return true;
+  }
+
+  if (Div == 2) {
+    Div = 3;
+    return true;
+  }
+
+  return false;
+}
+
+static const OptionalOperand VOP3OptionalOps [] = {
+  {"clamp", AMDGPUOperand::ImmTyClamp, true, 0, nullptr},
+  {"mul",   AMDGPUOperand::ImmTyOMod, false, 1, ConvertOmodMul},
+  {"div",   AMDGPUOperand::ImmTyOMod, false, 1, ConvertOmodDiv},
+};
+
+static bool isVOP3(OperandVector &Operands) {
+  if (operandsHaveModifiers(Operands))
+    return true;
+
+  AMDGPUOperand &DstOp = ((AMDGPUOperand&)*Operands[1]);
+
+  if (DstOp.isReg() && DstOp.isRegClass(AMDGPU::SGPR_64RegClassID))
+    return true;
+
+  if (Operands.size() >= 5)
+    return true;
+
+  if (Operands.size() > 3) {
+    AMDGPUOperand &Src1Op = ((AMDGPUOperand&)*Operands[3]);
+    if (Src1Op.getReg() && (Src1Op.isRegClass(AMDGPU::SReg_32RegClassID) ||
+                            Src1Op.isRegClass(AMDGPU::SReg_64RegClassID)))
+      return true;
+  }
+  return false;
+}
+
+AMDGPUAsmParser::OperandMatchResultTy
+AMDGPUAsmParser::parseVOP3OptionalOps(OperandVector &Operands) {
+
+  // The value returned by this function may change after parsing
+  // an operand so store the original value here.
+  bool HasModifiers = operandsHaveModifiers(Operands);
+
+  bool IsVOP3 = isVOP3(Operands);
+  if (HasModifiers || IsVOP3 ||
+      getLexer().isNot(AsmToken::EndOfStatement) ||
+      getForcedEncodingSize() == 64) {
+
+    AMDGPUAsmParser::OperandMatchResultTy Res =
+        parseOptionalOps(VOP3OptionalOps, Operands);
+
+    if (!HasModifiers && Res == MatchOperand_Success) {
+      // We have added a modifier operation, so we need to make sure all
+      // previous register operands have modifiers
+      for (unsigned i = 2, e = Operands.size(); i != e; ++i) {
+        AMDGPUOperand &Op = ((AMDGPUOperand&)*Operands[i]);
+        if (Op.isReg())
+          Op.setModifiers(0);
+      }
+    }
+    return Res;
+  }
+  return MatchOperand_NoMatch;
+}
+
+void AMDGPUAsmParser::cvtVOP3(MCInst &Inst, const OperandVector &Operands) {
+  ((AMDGPUOperand &)*Operands[1]).addRegOperands(Inst, 1);
+  unsigned i = 2;
+
+  std::map<enum AMDGPUOperand::ImmTy, unsigned> OptionalIdx;
+
+  if (operandsHaveModifiers(Operands)) {
+    for (unsigned e = Operands.size(); i != e; ++i) {
+      AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[i]);
+
+      if (Op.isRegWithInputMods()) {
+        ((AMDGPUOperand &)*Operands[i]).addRegWithInputModsOperands(Inst, 2);
+        continue;
+      }
+      OptionalIdx[Op.getImmTy()] = i;
+    }
+
+    unsigned ClampIdx = OptionalIdx[AMDGPUOperand::ImmTyClamp];
+    unsigned OModIdx = OptionalIdx[AMDGPUOperand::ImmTyOMod];
+
+    ((AMDGPUOperand &)*Operands[ClampIdx]).addImmOperands(Inst, 1);
+    ((AMDGPUOperand &)*Operands[OModIdx]).addImmOperands(Inst, 1);
+  } else {
+    for (unsigned e = Operands.size(); i != e; ++i)
+      ((AMDGPUOperand &)*Operands[i]).addRegOrImmOperands(Inst, 1);
+  }
+}
+
+/// Force static initialization.
+extern "C" void LLVMInitializeAMDGPUAsmParser() {
+  RegisterMCAsmParser<AMDGPUAsmParser> A(TheAMDGPUTarget);
+  RegisterMCAsmParser<AMDGPUAsmParser> B(TheGCNTarget);
+}
+
+#define GET_REGISTER_MATCHER
+#define GET_MATCHER_IMPLEMENTATION
+#include "AMDGPUGenAsmMatcher.inc"
+
diff --git a/lib/Target/AMDGPU/AsmParser/CMakeLists.txt b/lib/Target/AMDGPU/AsmParser/CMakeLists.txt
new file mode 100644
index 00000000000..21ddc4eb83d
--- /dev/null
+++ b/lib/Target/AMDGPU/AsmParser/CMakeLists.txt
@@ -0,0 +1,3 @@
+add_llvm_library(LLVMAMDGPUAsmParser
+  AMDGPUAsmParser.cpp
+  )
diff --git a/lib/Target/AMDGPU/AsmParser/LLVMBuild.txt b/lib/Target/AMDGPU/AsmParser/LLVMBuild.txt
new file mode 100644
index 00000000000..dab0c6f585a
--- /dev/null
+++ b/lib/Target/AMDGPU/AsmParser/LLVMBuild.txt
@@ -0,0 +1,23 @@
+;===- ./lib/Target/AMDGPU/AsmParser/LLVMBuild.txt -------------*- Conf -*--===;
+;
+;                     The LLVM Compiler Infrastructure
+;
+; This file is distributed under the University of Illinois Open Source
+; License. See LICENSE.TXT for details.
+;
+;===------------------------------------------------------------------------===;
+;
+; This is an LLVMBuild description file for the components in this subdirectory.
+;
+; For more information on the LLVMBuild system, please see:
+;
+;   http://llvm.org/docs/LLVMBuild.html
+;
+;===------------------------------------------------------------------------===;
+
+[component_0]
+type = Library
+name = AMDGPUAsmParser
+parent = AMDGPU
+required_libraries = MC MCParser AMDGPUDesc AMDGPUInfo AMDGPUUtils Support
+add_to_library_groups = AMDGPU
diff --git a/lib/Target/AMDGPU/AsmParser/Makefile b/lib/Target/AMDGPU/AsmParser/Makefile
new file mode 100644
index 00000000000..5ad21902803
--- /dev/null
+++ b/lib/Target/AMDGPU/AsmParser/Makefile
@@ -0,0 +1,15 @@
+##===- lib/Target/AMDGPU/AsmParser/Makefile ----------------*- Makefile -*-===##
+#
+#                     The LLVM Compiler Infrastructure
+#
+# This file is distributed under the University of Illinois Open Source
+# License. See LICENSE.TXT for details.
+#
+##===----------------------------------------------------------------------===##
+LEVEL = ../../../..
+LIBRARYNAME = LLVMAMDGPUAsmParser
+
+# Hack: we need to include 'main' AMDGPU target directory to grab private headers
+CPP.Flags += -I$(PROJ_OBJ_DIR)/.. -I$(PROJ_SRC_DIR)/..
+
+include $(LEVEL)/Makefile.common
diff --git a/lib/Target/AMDGPU/CIInstructions.td b/lib/Target/AMDGPU/CIInstructions.td
new file mode 100644
index 00000000000..2f5fdbe9207
--- /dev/null
+++ b/lib/Target/AMDGPU/CIInstructions.td
@@ -0,0 +1,149 @@
+//===-- CIInstructions.td - CI Instruction Defintions ---------------------===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+// Instruction definitions for CI and newer.
+//===----------------------------------------------------------------------===//
+
+
+def isCIVI : Predicate <
+  "Subtarget->getGeneration() == AMDGPUSubtarget::SEA_ISLANDS || "
+  "Subtarget->getGeneration() == AMDGPUSubtarget::VOLCANIC_ISLANDS"
+>, AssemblerPredicate<"FeatureCIInsts">;
+
+def HasFlatAddressSpace : Predicate<"Subtarget->hasFlatAddressSpace()">;
+
+//===----------------------------------------------------------------------===//
+// VOP1 Instructions
+//===----------------------------------------------------------------------===//
+
+let SubtargetPredicate = isCIVI in {
+
+defm V_TRUNC_F64 : VOP1Inst <vop1<0x17>, "v_trunc_f64",
+  VOP_F64_F64, ftrunc
+>;
+defm V_CEIL_F64 : VOP1Inst <vop1<0x18>, "v_ceil_f64",
+  VOP_F64_F64, fceil
+>;
+defm V_FLOOR_F64 : VOP1Inst <vop1<0x1A>, "v_floor_f64",
+  VOP_F64_F64, ffloor
+>;
+defm V_RNDNE_F64 : VOP1Inst <vop1<0x19>, "v_rndne_f64",
+  VOP_F64_F64, frint
+>;
+defm V_LOG_LEGACY_F32 : VOP1Inst <vop1<0x45, 0x4c>, "v_log_legacy_f32",
+  VOP_F32_F32
+>;
+defm V_EXP_LEGACY_F32 : VOP1Inst <vop1<0x46, 0x4b>, "v_exp_legacy_f32",
+  VOP_F32_F32
+>;
+
+//===----------------------------------------------------------------------===//
+// Flat Instructions
+//===----------------------------------------------------------------------===//
+
+def FLAT_LOAD_UBYTE : FLAT_Load_Helper <0x8, "flat_load_ubyte", VGPR_32>;
+def FLAT_LOAD_SBYTE : FLAT_Load_Helper <0x9, "flat_load_sbyte", VGPR_32>;
+def FLAT_LOAD_USHORT : FLAT_Load_Helper <0xa, "flat_load_ushort", VGPR_32>;
+def FLAT_LOAD_SSHORT : FLAT_Load_Helper <0xb, "flat_load_sshort", VGPR_32>;
+def FLAT_LOAD_DWORD : FLAT_Load_Helper <0xc, "flat_load_dword", VGPR_32>;
+def FLAT_LOAD_DWORDX2 : FLAT_Load_Helper <0xd, "flat_load_dwordx2", VReg_64>;
+def FLAT_LOAD_DWORDX4 : FLAT_Load_Helper <0xe, "flat_load_dwordx4", VReg_128>;
+def FLAT_LOAD_DWORDX3 : FLAT_Load_Helper <0xf, "flat_load_dwordx3", VReg_96>;
+def FLAT_STORE_BYTE : FLAT_Store_Helper <0x18, "flat_store_byte", VGPR_32>;
+def FLAT_STORE_SHORT : FLAT_Store_Helper <0x1a, "flat_store_short", VGPR_32>;
+def FLAT_STORE_DWORD : FLAT_Store_Helper <0x1c, "flat_store_dword", VGPR_32>;
+def FLAT_STORE_DWORDX2 : FLAT_Store_Helper <
+  0x1d, "flat_store_dwordx2", VReg_64
+>;
+def FLAT_STORE_DWORDX4 : FLAT_Store_Helper <
+  0x1e, "flat_store_dwordx4", VReg_128
+>;
+def FLAT_STORE_DWORDX3 : FLAT_Store_Helper <
+  0x1f, "flat_store_dwordx3", VReg_96
+>;
+defm FLAT_ATOMIC_SWAP : FLAT_ATOMIC <0x30, "flat_atomic_swap", VGPR_32>;
+defm FLAT_ATOMIC_CMPSWAP : FLAT_ATOMIC <
+  0x31, "flat_atomic_cmpswap", VGPR_32, VReg_64
+>;
+defm FLAT_ATOMIC_ADD : FLAT_ATOMIC <0x32, "flat_atomic_add", VGPR_32>;
+defm FLAT_ATOMIC_SUB : FLAT_ATOMIC <0x33, "flat_atomic_sub", VGPR_32>;
+defm FLAT_ATOMIC_RSUB : FLAT_ATOMIC <0x34, "flat_atomic_rsub", VGPR_32>;
+defm FLAT_ATOMIC_SMIN : FLAT_ATOMIC <0x35, "flat_atomic_smin", VGPR_32>;
+defm FLAT_ATOMIC_UMIN : FLAT_ATOMIC <0x36, "flat_atomic_umin", VGPR_32>;
+defm FLAT_ATOMIC_SMAX : FLAT_ATOMIC <0x37, "flat_atomic_smax", VGPR_32>;
+defm FLAT_ATOMIC_UMAX : FLAT_ATOMIC <0x38, "flat_atomic_umax", VGPR_32>;
+defm FLAT_ATOMIC_AND : FLAT_ATOMIC <0x39, "flat_atomic_and", VGPR_32>;
+defm FLAT_ATOMIC_OR : FLAT_ATOMIC <0x3a, "flat_atomic_or", VGPR_32>;
+defm FLAT_ATOMIC_XOR : FLAT_ATOMIC <0x3b, "flat_atomic_xor", VGPR_32>;
+defm FLAT_ATOMIC_INC : FLAT_ATOMIC <0x3c, "flat_atomic_inc", VGPR_32>;
+defm FLAT_ATOMIC_DEC : FLAT_ATOMIC <0x3d, "flat_atomic_dec", VGPR_32>;
+defm FLAT_ATOMIC_FCMPSWAP : FLAT_ATOMIC <
+  0x3e, "flat_atomic_fcmpswap", VGPR_32, VReg_64
+>;
+defm FLAT_ATOMIC_FMIN : FLAT_ATOMIC <0x3f, "flat_atomic_fmin", VGPR_32>;
+defm FLAT_ATOMIC_FMAX : FLAT_ATOMIC <0x40, "flat_atomic_fmax", VGPR_32>;
+defm FLAT_ATOMIC_SWAP_X2 : FLAT_ATOMIC <0x50, "flat_atomic_swap_x2", VReg_64>;
+defm FLAT_ATOMIC_CMPSWAP_X2 : FLAT_ATOMIC <
+  0x51, "flat_atomic_cmpswap_x2", VReg_64, VReg_128
+>;
+defm FLAT_ATOMIC_ADD_X2 : FLAT_ATOMIC <0x52, "flat_atomic_add_x2", VReg_64>;
+defm FLAT_ATOMIC_SUB_X2 : FLAT_ATOMIC <0x53, "flat_atomic_sub_x2", VReg_64>;
+defm FLAT_ATOMIC_RSUB_X2 : FLAT_ATOMIC <0x54, "flat_atomic_rsub_x2", VReg_64>;
+defm FLAT_ATOMIC_SMIN_X2 : FLAT_ATOMIC <0x55, "flat_atomic_smin_x2", VReg_64>;
+defm FLAT_ATOMIC_UMIN_X2 : FLAT_ATOMIC <0x56, "flat_atomic_umin_x2", VReg_64>;
+defm FLAT_ATOMIC_SMAX_X2 : FLAT_ATOMIC <0x57, "flat_atomic_smax_x2", VReg_64>;
+defm FLAT_ATOMIC_UMAX_X2 : FLAT_ATOMIC <0x58, "flat_atomic_umax_x2", VReg_64>;
+defm FLAT_ATOMIC_AND_X2 : FLAT_ATOMIC <0x59, "flat_atomic_and_x2", VReg_64>;
+defm FLAT_ATOMIC_OR_X2 : FLAT_ATOMIC <0x5a, "flat_atomic_or_x2", VReg_64>;
+defm FLAT_ATOMIC_XOR_X2 : FLAT_ATOMIC <0x5b, "flat_atomic_xor_x2", VReg_64>;
+defm FLAT_ATOMIC_INC_X2 : FLAT_ATOMIC <0x5c, "flat_atomic_inc_x2", VReg_64>;
+defm FLAT_ATOMIC_DEC_X2 : FLAT_ATOMIC <0x5d, "flat_atomic_dec_x2", VReg_64>;
+defm FLAT_ATOMIC_FCMPSWAP_X2 : FLAT_ATOMIC <
+  0x5e, "flat_atomic_fcmpswap_x2", VReg_64, VReg_128
+>;
+defm FLAT_ATOMIC_FMIN_X2 : FLAT_ATOMIC <0x5f, "flat_atomic_fmin_x2", VReg_64>;
+defm FLAT_ATOMIC_FMAX_X2 : FLAT_ATOMIC <0x60, "flat_atomic_fmax_x2", VReg_64>;
+
+} // End SubtargetPredicate = isCIVI
+
+//===----------------------------------------------------------------------===//
+// Flat Patterns
+//===----------------------------------------------------------------------===//
+
+let Predicates = [HasFlatAddressSpace] in {
+
+class FLATLoad_Pattern <FLAT Instr_ADDR64, ValueType vt,
+                             PatFrag flat_ld> :
+  Pat <(vt (flat_ld i64:$ptr)),
+       (Instr_ADDR64 $ptr, 0, 0, 0)
+>;
+
+def : FLATLoad_Pattern <FLAT_LOAD_SBYTE, i32, sextloadi8_flat>;
+def : FLATLoad_Pattern <FLAT_LOAD_UBYTE, i32, az_extloadi8_flat>;
+def : FLATLoad_Pattern <FLAT_LOAD_SSHORT, i32, sextloadi16_flat>;
+def : FLATLoad_Pattern <FLAT_LOAD_USHORT, i32, az_extloadi16_flat>;
+def : FLATLoad_Pattern <FLAT_LOAD_DWORD, i32, flat_load>;
+def : FLATLoad_Pattern <FLAT_LOAD_DWORDX2, i64, flat_load>;
+def : FLATLoad_Pattern <FLAT_LOAD_DWORDX2, i64, az_extloadi32_flat>;
+def : FLATLoad_Pattern <FLAT_LOAD_DWORDX2, v2i32, flat_load>;
+def : FLATLoad_Pattern <FLAT_LOAD_DWORDX4, v4i32, flat_load>;
+
+class FLATStore_Pattern <FLAT Instr, ValueType vt, PatFrag st> :
+  Pat <(st vt:$value, i64:$ptr),
+        (Instr $value, $ptr, 0, 0, 0)
+  >;
+
+def : FLATStore_Pattern <FLAT_STORE_BYTE, i32, truncstorei8_flat>;
+def : FLATStore_Pattern <FLAT_STORE_SHORT, i32, truncstorei16_flat>;
+def : FLATStore_Pattern <FLAT_STORE_DWORD, i32, flat_store>;
+def : FLATStore_Pattern <FLAT_STORE_DWORDX2, i64, flat_store>;
+def : FLATStore_Pattern <FLAT_STORE_DWORDX2, v2i32, flat_store>;
+def : FLATStore_Pattern <FLAT_STORE_DWORDX4, v4i32, flat_store>;
+
+} // End HasFlatAddressSpace predicate
+
diff --git a/lib/Target/AMDGPU/CMakeLists.txt b/lib/Target/AMDGPU/CMakeLists.txt
new file mode 100644
index 00000000000..9460bf6b933
--- /dev/null
+++ b/lib/Target/AMDGPU/CMakeLists.txt
@@ -0,0 +1,65 @@
+set(LLVM_TARGET_DEFINITIONS AMDGPU.td)
+
+tablegen(LLVM AMDGPUGenRegisterInfo.inc -gen-register-info)
+tablegen(LLVM AMDGPUGenInstrInfo.inc -gen-instr-info)
+tablegen(LLVM AMDGPUGenDAGISel.inc -gen-dag-isel)
+tablegen(LLVM AMDGPUGenCallingConv.inc -gen-callingconv)
+tablegen(LLVM AMDGPUGenSubtargetInfo.inc -gen-subtarget)
+tablegen(LLVM AMDGPUGenIntrinsics.inc -gen-tgt-intrinsic)
+tablegen(LLVM AMDGPUGenMCCodeEmitter.inc -gen-emitter)
+tablegen(LLVM AMDGPUGenDFAPacketizer.inc -gen-dfa-packetizer)
+tablegen(LLVM AMDGPUGenAsmWriter.inc -gen-asm-writer)
+tablegen(LLVM AMDGPUGenAsmMatcher.inc -gen-asm-matcher)
+add_public_tablegen_target(AMDGPUCommonTableGen)
+
+add_llvm_target(AMDGPUCodeGen
+  AMDILCFGStructurizer.cpp
+  AMDGPUAlwaysInlinePass.cpp
+  AMDGPUAsmPrinter.cpp
+  AMDGPUFrameLowering.cpp
+  AMDGPUIntrinsicInfo.cpp
+  AMDGPUISelDAGToDAG.cpp
+  AMDGPUMCInstLower.cpp
+  AMDGPUMachineFunction.cpp
+  AMDGPUSubtarget.cpp
+  AMDGPUTargetMachine.cpp
+  AMDGPUTargetTransformInfo.cpp
+  AMDGPUISelLowering.cpp
+  AMDGPUInstrInfo.cpp
+  AMDGPUPromoteAlloca.cpp
+  AMDGPURegisterInfo.cpp
+  R600ClauseMergePass.cpp
+  R600ControlFlowFinalizer.cpp
+  R600EmitClauseMarkers.cpp
+  R600ExpandSpecialInstrs.cpp
+  R600InstrInfo.cpp
+  R600ISelLowering.cpp
+  R600MachineFunctionInfo.cpp
+  R600MachineScheduler.cpp
+  R600OptimizeVectorRegisters.cpp
+  R600Packetizer.cpp
+  R600RegisterInfo.cpp
+  R600TextureIntrinsicsReplacer.cpp
+  SIAnnotateControlFlow.cpp
+  SIFixControlFlowLiveIntervals.cpp
+  SIFixSGPRCopies.cpp
+  SIFixSGPRLiveRanges.cpp
+  SIFoldOperands.cpp
+  SIInsertWaits.cpp
+  SIInstrInfo.cpp
+  SIISelLowering.cpp
+  SILoadStoreOptimizer.cpp
+  SILowerControlFlow.cpp
+  SILowerI1Copies.cpp
+  SIMachineFunctionInfo.cpp
+  SIPrepareScratchRegs.cpp
+  SIRegisterInfo.cpp
+  SIShrinkInstructions.cpp
+  SITypeRewriter.cpp
+  )
+
+add_subdirectory(AsmParser)
+add_subdirectory(InstPrinter)
+add_subdirectory(TargetInfo)
+add_subdirectory(MCTargetDesc)
+add_subdirectory(Utils)
diff --git a/lib/Target/R600/CaymanInstructions.td b/lib/Target/AMDGPU/CaymanInstructions.td
similarity index 100%
rename from lib/Target/R600/CaymanInstructions.td
rename to lib/Target/AMDGPU/CaymanInstructions.td
diff --git a/lib/Target/R600/EvergreenInstructions.td b/lib/Target/AMDGPU/EvergreenInstructions.td
similarity index 99%
rename from lib/Target/R600/EvergreenInstructions.td
rename to lib/Target/AMDGPU/EvergreenInstructions.td
index 55601469009..7adcd46fe19 100644
--- a/lib/Target/R600/EvergreenInstructions.td
+++ b/lib/Target/AMDGPU/EvergreenInstructions.td
@@ -335,6 +335,9 @@ defm CUBE_eg : CUBE_Common<0xC0>;
 
 def BCNT_INT : R600_1OP_Helper <0xAA, "BCNT_INT", ctpop, VecALU>;
 
+def ADDC_UINT : R600_2OP_Helper <0x52, "ADDC_UINT", AMDGPUcarry>;
+def SUBB_UINT : R600_2OP_Helper <0x53, "SUBB_UINT", AMDGPUborrow>;
+
 def FFBH_UINT : R600_1OP_Helper <0xAB, "FFBH_UINT", ctlz_zero_undef, VecALU>;
 def FFBL_INT : R600_1OP_Helper <0xAC, "FFBL_INT", cttz_zero_undef, VecALU>;
 
diff --git a/lib/Target/R600/InstPrinter/AMDGPUInstPrinter.cpp b/lib/Target/AMDGPU/InstPrinter/AMDGPUInstPrinter.cpp
similarity index 99%
rename from lib/Target/R600/InstPrinter/AMDGPUInstPrinter.cpp
rename to lib/Target/AMDGPU/InstPrinter/AMDGPUInstPrinter.cpp
index 279c3eb1912..e811d5cff22 100644
--- a/lib/Target/R600/InstPrinter/AMDGPUInstPrinter.cpp
+++ b/lib/Target/AMDGPU/InstPrinter/AMDGPUInstPrinter.cpp
@@ -337,7 +337,7 @@ void AMDGPUInstPrinter::printOperand(const MCInst *MI, unsigned OpNo,
     }
   } else if (Op.isExpr()) {
     const MCExpr *Exp = Op.getExpr();
-    Exp->print(O);
+    Exp->print(O, &MAI);
   } else {
     llvm_unreachable("unknown operand type in printOperand");
   }
@@ -424,7 +424,7 @@ void AMDGPUInstPrinter::printLiteral(const MCInst *MI, unsigned OpNo,
 
 void AMDGPUInstPrinter::printLast(const MCInst *MI, unsigned OpNo,
                                   raw_ostream &O) {
-  printIfSet(MI, OpNo, O.indent(25 - O.GetNumBytesInBuffer()), "*", " ");
+  printIfSet(MI, OpNo, O, "*", " ");
 }
 
 void AMDGPUInstPrinter::printNeg(const MCInst *MI, unsigned OpNo,
diff --git a/lib/Target/R600/InstPrinter/AMDGPUInstPrinter.h b/lib/Target/AMDGPU/InstPrinter/AMDGPUInstPrinter.h
similarity index 100%
rename from lib/Target/R600/InstPrinter/AMDGPUInstPrinter.h
rename to lib/Target/AMDGPU/InstPrinter/AMDGPUInstPrinter.h
diff --git a/lib/Target/AMDGPU/InstPrinter/CMakeLists.txt b/lib/Target/AMDGPU/InstPrinter/CMakeLists.txt
new file mode 100644
index 00000000000..ce63bd553b9
--- /dev/null
+++ b/lib/Target/AMDGPU/InstPrinter/CMakeLists.txt
@@ -0,0 +1,3 @@
+add_llvm_library(LLVMAMDGPUAsmPrinter
+  AMDGPUInstPrinter.cpp
+  )
diff --git a/lib/Target/AMDGPU/InstPrinter/LLVMBuild.txt b/lib/Target/AMDGPU/InstPrinter/LLVMBuild.txt
new file mode 100644
index 00000000000..fdb43844dc6
--- /dev/null
+++ b/lib/Target/AMDGPU/InstPrinter/LLVMBuild.txt
@@ -0,0 +1,24 @@
+;===- ./lib/Target/AMDGPU/InstPrinter/LLVMBuild.txt -----------*- Conf -*--===;
+;
+;                     The LLVM Compiler Infrastructure
+;
+; This file is distributed under the University of Illinois Open Source
+; License. See LICENSE.TXT for details.
+;
+;===------------------------------------------------------------------------===;
+;
+; This is an LLVMBuild description file for the components in this subdirectory.
+;
+; For more information on the LLVMBuild system, please see:
+;
+;   http://llvm.org/docs/LLVMBuild.html
+;
+;===------------------------------------------------------------------------===;
+
+[component_0]
+type = Library
+name = AMDGPUAsmPrinter
+parent = AMDGPU
+required_libraries = MC Support
+add_to_library_groups = AMDGPU
+
diff --git a/lib/Target/AMDGPU/InstPrinter/Makefile b/lib/Target/AMDGPU/InstPrinter/Makefile
new file mode 100644
index 00000000000..4e48ac7e28a
--- /dev/null
+++ b/lib/Target/AMDGPU/InstPrinter/Makefile
@@ -0,0 +1,15 @@
+#===- lib/Target/R600/AsmPrinter/Makefile ------------------*- Makefile -*-===##
+#
+#                     The LLVM Compiler Infrastructure
+#
+# This file is distributed under the University of Illinois Open Source
+# License. See LICENSE.TXT for details.
+#
+##===----------------------------------------------------------------------===##
+LEVEL = ../../../..
+LIBRARYNAME = LLVMAMDGPUAsmPrinter
+
+# Hack: we need to include 'main' x86 target directory to grab private headers
+CPP.Flags += -I$(PROJ_OBJ_DIR)/.. -I$(PROJ_SRC_DIR)/..
+
+include $(LEVEL)/Makefile.common
diff --git a/lib/Target/AMDGPU/LLVMBuild.txt b/lib/Target/AMDGPU/LLVMBuild.txt
new file mode 100644
index 00000000000..38c5489586f
--- /dev/null
+++ b/lib/Target/AMDGPU/LLVMBuild.txt
@@ -0,0 +1,33 @@
+;===- ./lib/Target/AMDIL/LLVMBuild.txt -------------------------*- Conf -*--===;
+;
+;                     The LLVM Compiler Infrastructure
+;
+; This file is distributed under the University of Illinois Open Source
+; License. See LICENSE.TXT for details.
+;
+;===------------------------------------------------------------------------===;
+;
+; This is an LLVMBuild description file for the components in this subdirectory.
+;
+; For more information on the LLVMBuild system, please see:
+;
+;   http://llvm.org/docs/LLVMBuild.html
+;
+;===------------------------------------------------------------------------===;
+
+[common]
+subdirectories = AsmParser InstPrinter MCTargetDesc TargetInfo Utils
+
+[component_0]
+type = TargetGroup
+name = AMDGPU
+parent = Target
+has_asmparser = 1
+has_asmprinter = 1
+
+[component_1]
+type = Library
+name = AMDGPUCodeGen
+parent = AMDGPU
+required_libraries = Analysis AsmPrinter CodeGen Core IPO MC AMDGPUAsmParser AMDGPUAsmPrinter AMDGPUDesc AMDGPUInfo AMDGPUUtils Scalar SelectionDAG Support Target TransformUtils
+add_to_library_groups = AMDGPU
diff --git a/lib/Target/AMDGPU/MCTargetDesc/AMDGPUAsmBackend.cpp b/lib/Target/AMDGPU/MCTargetDesc/AMDGPUAsmBackend.cpp
new file mode 100644
index 00000000000..4434d9b119c
--- /dev/null
+++ b/lib/Target/AMDGPU/MCTargetDesc/AMDGPUAsmBackend.cpp
@@ -0,0 +1,182 @@
+//===-- AMDGPUAsmBackend.cpp - AMDGPU Assembler Backend -------------------===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+/// \file
+//===----------------------------------------------------------------------===//
+
+#include "MCTargetDesc/AMDGPUMCTargetDesc.h"
+#include "MCTargetDesc/AMDGPUFixupKinds.h"
+#include "llvm/ADT/StringRef.h"
+#include "llvm/MC/MCAsmBackend.h"
+#include "llvm/MC/MCAssembler.h"
+#include "llvm/MC/MCFixupKindInfo.h"
+#include "llvm/MC/MCObjectWriter.h"
+#include "llvm/MC/MCValue.h"
+#include "llvm/Support/TargetRegistry.h"
+
+using namespace llvm;
+
+namespace {
+
+class AMDGPUMCObjectWriter : public MCObjectWriter {
+public:
+  AMDGPUMCObjectWriter(raw_pwrite_stream &OS) : MCObjectWriter(OS, true) {}
+  void executePostLayoutBinding(MCAssembler &Asm,
+                                const MCAsmLayout &Layout) override {
+    //XXX: Implement if necessary.
+  }
+  void recordRelocation(MCAssembler &Asm, const MCAsmLayout &Layout,
+                        const MCFragment *Fragment, const MCFixup &Fixup,
+                        MCValue Target, bool &IsPCRel,
+                        uint64_t &FixedValue) override {
+    assert(!"Not implemented");
+  }
+
+  void writeObject(MCAssembler &Asm, const MCAsmLayout &Layout) override;
+
+};
+
+class AMDGPUAsmBackend : public MCAsmBackend {
+public:
+  AMDGPUAsmBackend(const Target &T)
+    : MCAsmBackend() {}
+
+  unsigned getNumFixupKinds() const override { return AMDGPU::NumTargetFixupKinds; };
+  void applyFixup(const MCFixup &Fixup, char *Data, unsigned DataSize,
+                  uint64_t Value, bool IsPCRel) const override;
+  bool fixupNeedsRelaxation(const MCFixup &Fixup, uint64_t Value,
+                            const MCRelaxableFragment *DF,
+                            const MCAsmLayout &Layout) const override {
+    return false;
+  }
+  void relaxInstruction(const MCInst &Inst, MCInst &Res) const override {
+    assert(!"Not implemented");
+  }
+  bool mayNeedRelaxation(const MCInst &Inst) const override { return false; }
+  bool writeNopData(uint64_t Count, MCObjectWriter *OW) const override;
+
+  const MCFixupKindInfo &getFixupKindInfo(MCFixupKind Kind) const override;
+};
+
+} //End anonymous namespace
+
+void AMDGPUMCObjectWriter::writeObject(MCAssembler &Asm,
+                                       const MCAsmLayout &Layout) {
+  for (MCAssembler::iterator I = Asm.begin(), E = Asm.end(); I != E; ++I) {
+    Asm.writeSectionData(&*I, Layout);
+  }
+}
+
+static unsigned getFixupKindNumBytes(unsigned Kind) {
+  switch (Kind) {
+  case FK_Data_1:
+    return 1;
+  case FK_Data_2:
+    return 2;
+  case FK_Data_4:
+    return 4;
+  case FK_Data_8:
+    return 8;
+  default:
+    llvm_unreachable("Unknown fixup kind!");
+  }
+}
+
+void AMDGPUAsmBackend::applyFixup(const MCFixup &Fixup, char *Data,
+                                  unsigned DataSize, uint64_t Value,
+                                  bool IsPCRel) const {
+
+  switch ((unsigned)Fixup.getKind()) {
+    case AMDGPU::fixup_si_sopp_br: {
+      uint16_t *Dst = (uint16_t*)(Data + Fixup.getOffset());
+      *Dst = (Value - 4) / 4;
+      break;
+    }
+
+    case AMDGPU::fixup_si_rodata: {
+      uint32_t *Dst = (uint32_t*)(Data + Fixup.getOffset());
+      *Dst = Value;
+      break;
+    }
+
+    case AMDGPU::fixup_si_end_of_text: {
+      uint32_t *Dst = (uint32_t*)(Data + Fixup.getOffset());
+      // The value points to the last instruction in the text section, so we
+      // need to add 4 bytes to get to the start of the constants.
+      *Dst = Value + 4;
+      break;
+    }
+    default: {
+      // FIXME: Copied from AArch64
+      unsigned NumBytes = getFixupKindNumBytes(Fixup.getKind());
+      if (!Value)
+        return; // Doesn't change encoding.
+      MCFixupKindInfo Info = getFixupKindInfo(Fixup.getKind());
+
+      // Shift the value into position.
+      Value <<= Info.TargetOffset;
+
+      unsigned Offset = Fixup.getOffset();
+      assert(Offset + NumBytes <= DataSize && "Invalid fixup offset!");
+
+      // For each byte of the fragment that the fixup touches, mask in the
+      // bits from the fixup value.
+      for (unsigned i = 0; i != NumBytes; ++i)
+        Data[Offset + i] |= uint8_t((Value >> (i * 8)) & 0xff);
+    }
+  }
+}
+
+const MCFixupKindInfo &AMDGPUAsmBackend::getFixupKindInfo(
+                                                       MCFixupKind Kind) const {
+  const static MCFixupKindInfo Infos[AMDGPU::NumTargetFixupKinds] = {
+    // name                   offset bits  flags
+    { "fixup_si_sopp_br",     0,     16,   MCFixupKindInfo::FKF_IsPCRel },
+    { "fixup_si_rodata",      0,     32,   0 },
+    { "fixup_si_end_of_text", 0,     32,   MCFixupKindInfo::FKF_IsPCRel }
+  };
+
+  if (Kind < FirstTargetFixupKind)
+    return MCAsmBackend::getFixupKindInfo(Kind);
+
+  return Infos[Kind - FirstTargetFixupKind];
+}
+
+bool AMDGPUAsmBackend::writeNopData(uint64_t Count, MCObjectWriter *OW) const {
+  OW->WriteZeros(Count);
+
+  return true;
+}
+
+//===----------------------------------------------------------------------===//
+// ELFAMDGPUAsmBackend class
+//===----------------------------------------------------------------------===//
+
+namespace {
+
+class ELFAMDGPUAsmBackend : public AMDGPUAsmBackend {
+  bool Is64Bit;
+
+public:
+  ELFAMDGPUAsmBackend(const Target &T, bool Is64Bit) :
+      AMDGPUAsmBackend(T), Is64Bit(Is64Bit) { }
+
+  MCObjectWriter *createObjectWriter(raw_pwrite_stream &OS) const override {
+    return createAMDGPUELFObjectWriter(Is64Bit, OS);
+  }
+};
+
+} // end anonymous namespace
+
+MCAsmBackend *llvm::createAMDGPUAsmBackend(const Target &T,
+                                           const MCRegisterInfo &MRI,
+                                           const Triple &TT, StringRef CPU) {
+  Triple TargetTriple(TT);
+
+  // Use 64-bit ELF for amdgcn
+  return new ELFAMDGPUAsmBackend(T, TargetTriple.getArch() == Triple::amdgcn);
+}
diff --git a/lib/Target/AMDGPU/MCTargetDesc/AMDGPUELFObjectWriter.cpp b/lib/Target/AMDGPU/MCTargetDesc/AMDGPUELFObjectWriter.cpp
new file mode 100644
index 00000000000..820f17df896
--- /dev/null
+++ b/lib/Target/AMDGPU/MCTargetDesc/AMDGPUELFObjectWriter.cpp
@@ -0,0 +1,40 @@
+//===-- AMDGPUELFObjectWriter.cpp - AMDGPU ELF Writer ----------------------==//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+/// \file
+//===----------------------------------------------------------------------===//
+
+#include "AMDGPUMCTargetDesc.h"
+#include "llvm/MC/MCELFObjectWriter.h"
+#include "llvm/MC/MCFixup.h"
+
+using namespace llvm;
+
+namespace {
+
+class AMDGPUELFObjectWriter : public MCELFObjectTargetWriter {
+public:
+  AMDGPUELFObjectWriter(bool Is64Bit);
+protected:
+  unsigned GetRelocType(const MCValue &Target, const MCFixup &Fixup,
+                        bool IsPCRel) const override {
+    return Fixup.getKind();
+  }
+
+};
+
+
+} // End anonymous namespace
+
+AMDGPUELFObjectWriter::AMDGPUELFObjectWriter(bool Is64Bit)
+  : MCELFObjectTargetWriter(Is64Bit, ELF::ELFOSABI_AMDGPU_HSA,
+                            ELF::EM_AMDGPU, false) { }
+
+MCObjectWriter *llvm::createAMDGPUELFObjectWriter(bool Is64Bit, raw_pwrite_stream &OS) {
+  MCELFObjectTargetWriter *MOTW = new AMDGPUELFObjectWriter(Is64Bit);
+  return createELFObjectWriter(MOTW, OS, true);
+}
diff --git a/lib/Target/R600/MCTargetDesc/AMDGPUFixupKinds.h b/lib/Target/AMDGPU/MCTargetDesc/AMDGPUFixupKinds.h
similarity index 100%
rename from lib/Target/R600/MCTargetDesc/AMDGPUFixupKinds.h
rename to lib/Target/AMDGPU/MCTargetDesc/AMDGPUFixupKinds.h
diff --git a/lib/Target/R600/MCTargetDesc/AMDGPUMCAsmInfo.cpp b/lib/Target/AMDGPU/MCTargetDesc/AMDGPUMCAsmInfo.cpp
similarity index 95%
rename from lib/Target/R600/MCTargetDesc/AMDGPUMCAsmInfo.cpp
rename to lib/Target/AMDGPU/MCTargetDesc/AMDGPUMCAsmInfo.cpp
index 19d89fb27ca..028a86dfc7a 100644
--- a/lib/Target/R600/MCTargetDesc/AMDGPUMCAsmInfo.cpp
+++ b/lib/Target/AMDGPU/MCTargetDesc/AMDGPUMCAsmInfo.cpp
@@ -11,7 +11,7 @@
 #include "AMDGPUMCAsmInfo.h"
 
 using namespace llvm;
-AMDGPUMCAsmInfo::AMDGPUMCAsmInfo(StringRef &TT) : MCAsmInfoELF() {
+AMDGPUMCAsmInfo::AMDGPUMCAsmInfo(const Triple &TT) : MCAsmInfoELF() {
   HasSingleParameterDotFile = false;
   //===------------------------------------------------------------------===//
   MaxInstLength = 16;
diff --git a/lib/Target/R600/MCTargetDesc/AMDGPUMCAsmInfo.h b/lib/Target/AMDGPU/MCTargetDesc/AMDGPUMCAsmInfo.h
similarity index 88%
rename from lib/Target/R600/MCTargetDesc/AMDGPUMCAsmInfo.h
rename to lib/Target/AMDGPU/MCTargetDesc/AMDGPUMCAsmInfo.h
index 8f75c76c425..119dd425916 100644
--- a/lib/Target/R600/MCTargetDesc/AMDGPUMCAsmInfo.h
+++ b/lib/Target/AMDGPU/MCTargetDesc/AMDGPUMCAsmInfo.h
@@ -17,16 +17,16 @@
 #include "llvm/MC/MCAsmInfoELF.h"
 namespace llvm {
 
-class StringRef;
+class Triple;
 
 // If you need to create another MCAsmInfo class, which inherits from MCAsmInfo,
 // you will need to make sure your new class sets PrivateGlobalPrefix to
-// a prefix that won't appeary in a fuction name.  The default value
+// a prefix that won't appear in a function name.  The default value
 // for PrivateGlobalPrefix is 'L', so it will consider any function starting
 // with 'L' as a local symbol.
 class AMDGPUMCAsmInfo : public MCAsmInfoELF {
 public:
-  explicit AMDGPUMCAsmInfo(StringRef &TT);
+  explicit AMDGPUMCAsmInfo(const Triple &TT);
 };
 } // namespace llvm
 #endif
diff --git a/lib/Target/R600/MCTargetDesc/AMDGPUMCCodeEmitter.cpp b/lib/Target/AMDGPU/MCTargetDesc/AMDGPUMCCodeEmitter.cpp
similarity index 100%
rename from lib/Target/R600/MCTargetDesc/AMDGPUMCCodeEmitter.cpp
rename to lib/Target/AMDGPU/MCTargetDesc/AMDGPUMCCodeEmitter.cpp
diff --git a/lib/Target/R600/MCTargetDesc/AMDGPUMCCodeEmitter.h b/lib/Target/AMDGPU/MCTargetDesc/AMDGPUMCCodeEmitter.h
similarity index 100%
rename from lib/Target/R600/MCTargetDesc/AMDGPUMCCodeEmitter.h
rename to lib/Target/AMDGPU/MCTargetDesc/AMDGPUMCCodeEmitter.h
diff --git a/lib/Target/AMDGPU/MCTargetDesc/AMDGPUMCTargetDesc.cpp b/lib/Target/AMDGPU/MCTargetDesc/AMDGPUMCTargetDesc.cpp
new file mode 100644
index 00000000000..c709741f377
--- /dev/null
+++ b/lib/Target/AMDGPU/MCTargetDesc/AMDGPUMCTargetDesc.cpp
@@ -0,0 +1,111 @@
+//===-- AMDGPUMCTargetDesc.cpp - AMDGPU Target Descriptions ---------------===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+/// \file
+/// \brief This file provides AMDGPU specific target descriptions.
+//
+//===----------------------------------------------------------------------===//
+
+#include "AMDGPUMCTargetDesc.h"
+#include "AMDGPUMCAsmInfo.h"
+#include "AMDGPUTargetStreamer.h"
+#include "InstPrinter/AMDGPUInstPrinter.h"
+#include "SIDefines.h"
+#include "llvm/MC/MCCodeGenInfo.h"
+#include "llvm/MC/MCContext.h"
+#include "llvm/MC/MCInstrInfo.h"
+#include "llvm/MC/MCRegisterInfo.h"
+#include "llvm/MC/MCStreamer.h"
+#include "llvm/MC/MCSubtargetInfo.h"
+#include "llvm/MC/MachineLocation.h"
+#include "llvm/Support/ErrorHandling.h"
+#include "llvm/Support/TargetRegistry.h"
+
+using namespace llvm;
+
+#define GET_INSTRINFO_MC_DESC
+#include "AMDGPUGenInstrInfo.inc"
+
+#define GET_SUBTARGETINFO_MC_DESC
+#include "AMDGPUGenSubtargetInfo.inc"
+
+#define GET_REGINFO_MC_DESC
+#include "AMDGPUGenRegisterInfo.inc"
+
+static MCInstrInfo *createAMDGPUMCInstrInfo() {
+  MCInstrInfo *X = new MCInstrInfo();
+  InitAMDGPUMCInstrInfo(X);
+  return X;
+}
+
+static MCRegisterInfo *createAMDGPUMCRegisterInfo(const Triple &TT) {
+  MCRegisterInfo *X = new MCRegisterInfo();
+  InitAMDGPUMCRegisterInfo(X, 0);
+  return X;
+}
+
+static MCSubtargetInfo *
+createAMDGPUMCSubtargetInfo(const Triple &TT, StringRef CPU, StringRef FS) {
+  return createAMDGPUMCSubtargetInfoImpl(TT, CPU, FS);
+}
+
+static MCCodeGenInfo *createAMDGPUMCCodeGenInfo(const Triple &TT,
+                                                Reloc::Model RM,
+                                                CodeModel::Model CM,
+                                                CodeGenOpt::Level OL) {
+  MCCodeGenInfo *X = new MCCodeGenInfo();
+  X->initMCCodeGenInfo(RM, CM, OL);
+  return X;
+}
+
+static MCInstPrinter *createAMDGPUMCInstPrinter(const Triple &T,
+                                                unsigned SyntaxVariant,
+                                                const MCAsmInfo &MAI,
+                                                const MCInstrInfo &MII,
+                                                const MCRegisterInfo &MRI) {
+  return new AMDGPUInstPrinter(MAI, MII, MRI);
+}
+
+static MCTargetStreamer *createAMDGPUAsmTargetStreamer(MCStreamer &S,
+                                                      formatted_raw_ostream &OS,
+                                                      MCInstPrinter *InstPrint,
+                                                      bool isVerboseAsm) {
+  return new AMDGPUTargetAsmStreamer(S, OS);
+}
+
+static MCTargetStreamer * createAMDGPUObjectTargetStreamer(
+                                                   MCStreamer &S,
+                                                   const MCSubtargetInfo &STI) {
+  return new AMDGPUTargetELFStreamer(S);
+}
+
+extern "C" void LLVMInitializeAMDGPUTargetMC() {
+  for (Target *T : {&TheAMDGPUTarget, &TheGCNTarget}) {
+    RegisterMCAsmInfo<AMDGPUMCAsmInfo> X(*T);
+
+    TargetRegistry::RegisterMCCodeGenInfo(*T, createAMDGPUMCCodeGenInfo);
+    TargetRegistry::RegisterMCInstrInfo(*T, createAMDGPUMCInstrInfo);
+    TargetRegistry::RegisterMCRegInfo(*T, createAMDGPUMCRegisterInfo);
+    TargetRegistry::RegisterMCSubtargetInfo(*T, createAMDGPUMCSubtargetInfo);
+    TargetRegistry::RegisterMCInstPrinter(*T, createAMDGPUMCInstPrinter);
+    TargetRegistry::RegisterMCAsmBackend(*T, createAMDGPUAsmBackend);
+  }
+
+  // R600 specific registration
+  TargetRegistry::RegisterMCCodeEmitter(TheAMDGPUTarget,
+                                        createR600MCCodeEmitter);
+
+  // GCN specific registration
+  TargetRegistry::RegisterMCCodeEmitter(TheGCNTarget, createSIMCCodeEmitter);
+
+  TargetRegistry::RegisterAsmTargetStreamer(TheGCNTarget,
+                                            createAMDGPUAsmTargetStreamer);
+  TargetRegistry::RegisterObjectTargetStreamer(TheGCNTarget,
+                                              createAMDGPUObjectTargetStreamer);
+}
diff --git a/lib/Target/R600/MCTargetDesc/AMDGPUMCTargetDesc.h b/lib/Target/AMDGPU/MCTargetDesc/AMDGPUMCTargetDesc.h
similarity index 88%
rename from lib/Target/R600/MCTargetDesc/AMDGPUMCTargetDesc.h
rename to lib/Target/AMDGPU/MCTargetDesc/AMDGPUMCTargetDesc.h
index 9a7548e9fbf..5d1b86b8c0c 100644
--- a/lib/Target/R600/MCTargetDesc/AMDGPUMCTargetDesc.h
+++ b/lib/Target/AMDGPU/MCTargetDesc/AMDGPUMCTargetDesc.h
@@ -28,6 +28,7 @@ class MCObjectWriter;
 class MCRegisterInfo;
 class MCSubtargetInfo;
 class Target;
+class Triple;
 class raw_pwrite_stream;
 class raw_ostream;
 
@@ -43,9 +44,10 @@ MCCodeEmitter *createSIMCCodeEmitter(const MCInstrInfo &MCII,
                                      MCContext &Ctx);
 
 MCAsmBackend *createAMDGPUAsmBackend(const Target &T, const MCRegisterInfo &MRI,
-                                     StringRef TT, StringRef CPU);
+                                     const Triple &TT, StringRef CPU);
 
-MCObjectWriter *createAMDGPUELFObjectWriter(raw_pwrite_stream &OS);
+MCObjectWriter *createAMDGPUELFObjectWriter(bool Is64Bit,
+                                            raw_pwrite_stream &OS);
 } // End llvm namespace
 
 #define GET_REGINFO_ENUM
diff --git a/lib/Target/AMDGPU/MCTargetDesc/AMDGPUTargetStreamer.cpp b/lib/Target/AMDGPU/MCTargetDesc/AMDGPUTargetStreamer.cpp
new file mode 100644
index 00000000000..09e6cb1f1ff
--- /dev/null
+++ b/lib/Target/AMDGPU/MCTargetDesc/AMDGPUTargetStreamer.cpp
@@ -0,0 +1,297 @@
+//===-- AMDGPUTargetStreamer.cpp - Mips Target Streamer Methods -----------===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This file provides AMDGPU specific target streamer methods.
+//
+//===----------------------------------------------------------------------===//
+
+#include "AMDGPUTargetStreamer.h"
+#include "SIDefines.h"
+#include "llvm/ADT/Twine.h"
+#include "llvm/MC/MCContext.h"
+#include "llvm/MC/MCELFStreamer.h"
+#include "llvm/MC/MCObjectFileInfo.h"
+#include "llvm/MC/MCSectionELF.h"
+#include "llvm/Support/ELF.h"
+#include "llvm/Support/FormattedStream.h"
+
+using namespace llvm;
+
+AMDGPUTargetStreamer::AMDGPUTargetStreamer(MCStreamer &S)
+    : MCTargetStreamer(S) { }
+
+//===----------------------------------------------------------------------===//
+// AMDGPUTargetAsmStreamer
+//===----------------------------------------------------------------------===//
+
+AMDGPUTargetAsmStreamer::AMDGPUTargetAsmStreamer(MCStreamer &S,
+                                                 formatted_raw_ostream &OS)
+    : AMDGPUTargetStreamer(S), OS(OS) { }
+
+void
+AMDGPUTargetAsmStreamer::EmitDirectiveHSACodeObjectVersion(uint32_t Major,
+                                                           uint32_t Minor) {
+  OS << "\t.hsa_code_object_version " <<
+        Twine(Major) << "," << Twine(Minor) << '\n';
+}
+
+void
+AMDGPUTargetAsmStreamer::EmitDirectiveHSACodeObjectISA(uint32_t Major,
+                                                       uint32_t Minor,
+                                                       uint32_t Stepping,
+                                                       StringRef VendorName,
+                                                       StringRef ArchName) {
+  OS << "\t.hsa_code_object_isa " <<
+        Twine(Major) << "," << Twine(Minor) << "," << Twine(Stepping) <<
+        ",\"" << VendorName << "\",\"" << ArchName << "\"\n";
+
+}
+
+void
+AMDGPUTargetAsmStreamer::EmitAMDKernelCodeT(const amd_kernel_code_t &Header) {
+  uint64_t ComputePgmRsrc2 = (Header.compute_pgm_resource_registers >> 32);
+  bool EnableSGPRPrivateSegmentBuffer = (Header.code_properties &
+      AMD_CODE_PROPERTY_ENABLE_SGPR_PRIVATE_SEGMENT_BUFFER);
+  bool EnableSGPRDispatchPtr = (Header.code_properties &
+      AMD_CODE_PROPERTY_ENABLE_SGPR_DISPATCH_PTR);
+  bool EnableSGPRQueuePtr = (Header.code_properties &
+      AMD_CODE_PROPERTY_ENABLE_SGPR_QUEUE_PTR);
+  bool EnableSGPRKernargSegmentPtr = (Header.code_properties &
+      AMD_CODE_PROPERTY_ENABLE_SGPR_KERNARG_SEGMENT_PTR);
+  bool EnableSGPRDispatchID = (Header.code_properties &
+      AMD_CODE_PROPERTY_ENABLE_SGPR_DISPATCH_ID);
+  bool EnableSGPRFlatScratchInit = (Header.code_properties &
+      AMD_CODE_PROPERTY_ENABLE_SGPR_FLAT_SCRATCH_INIT);
+  bool EnableSGPRPrivateSegmentSize = (Header.code_properties &
+      AMD_CODE_PROPERTY_ENABLE_SGPR_PRIVATE_SEGMENT_SIZE);
+  bool EnableSGPRGridWorkgroupCountX = (Header.code_properties &
+      AMD_CODE_PROPERTY_ENABLE_SGPR_GRID_WORKGROUP_COUNT_X);
+  bool EnableSGPRGridWorkgroupCountY = (Header.code_properties &
+      AMD_CODE_PROPERTY_ENABLE_SGPR_GRID_WORKGROUP_COUNT_Y);
+  bool EnableSGPRGridWorkgroupCountZ = (Header.code_properties &
+      AMD_CODE_PROPERTY_ENABLE_SGPR_GRID_WORKGROUP_COUNT_Z);
+  bool EnableOrderedAppendGDS = (Header.code_properties &
+      AMD_CODE_PROPERTY_ENABLE_ORDERED_APPEND_GDS);
+  uint32_t PrivateElementSize = (Header.code_properties &
+      AMD_CODE_PROPERTY_PRIVATE_ELEMENT_SIZE) >>
+          AMD_CODE_PROPERTY_PRIVATE_ELEMENT_SIZE_SHIFT;
+  bool IsPtr64 = (Header.code_properties & AMD_CODE_PROPERTY_IS_PTR64);
+  bool IsDynamicCallstack = (Header.code_properties &
+      AMD_CODE_PROPERTY_IS_DYNAMIC_CALLSTACK);
+  bool IsDebugEnabled = (Header.code_properties &
+      AMD_CODE_PROPERTY_IS_DEBUG_SUPPORTED);
+  bool IsXNackEnabled = (Header.code_properties &
+      AMD_CODE_PROPERTY_IS_XNACK_SUPPORTED);
+
+  OS << "\t.amd_kernel_code_t\n" <<
+    "\t\tkernel_code_version_major = " <<
+        Header.amd_kernel_code_version_major << '\n' <<
+    "\t\tkernel_code_version_minor = " <<
+        Header.amd_kernel_code_version_minor << '\n' <<
+    "\t\tmachine_kind = " <<
+        Header.amd_machine_kind << '\n' <<
+    "\t\tmachine_version_major = " <<
+        Header.amd_machine_version_major << '\n' <<
+    "\t\tmachine_version_minor = " <<
+        Header.amd_machine_version_minor << '\n' <<
+    "\t\tmachine_version_stepping = " <<
+        Header.amd_machine_version_stepping << '\n' <<
+    "\t\tkernel_code_entry_byte_offset = " <<
+        Header.kernel_code_entry_byte_offset << '\n' <<
+    "\t\tkernel_code_prefetch_byte_size = " <<
+        Header.kernel_code_prefetch_byte_size << '\n' <<
+    "\t\tmax_scratch_backing_memory_byte_size = " <<
+        Header.max_scratch_backing_memory_byte_size << '\n' <<
+    "\t\tcompute_pgm_rsrc1_vgprs = " <<
+        G_00B848_VGPRS(Header.compute_pgm_resource_registers) << '\n' <<
+    "\t\tcompute_pgm_rsrc1_sgprs = " <<
+        G_00B848_SGPRS(Header.compute_pgm_resource_registers) << '\n' <<
+    "\t\tcompute_pgm_rsrc1_priority = " <<
+        G_00B848_PRIORITY(Header.compute_pgm_resource_registers) << '\n' <<
+    "\t\tcompute_pgm_rsrc1_float_mode = " <<
+        G_00B848_FLOAT_MODE(Header.compute_pgm_resource_registers) << '\n' <<
+    "\t\tcompute_pgm_rsrc1_priv = " <<
+        G_00B848_PRIV(Header.compute_pgm_resource_registers) << '\n' <<
+    "\t\tcompute_pgm_rsrc1_dx10_clamp = " <<
+        G_00B848_DX10_CLAMP(Header.compute_pgm_resource_registers) << '\n' <<
+    "\t\tcompute_pgm_rsrc1_debug_mode = " <<
+        G_00B848_DEBUG_MODE(Header.compute_pgm_resource_registers) << '\n' <<
+    "\t\tcompute_pgm_rsrc1_ieee_mode = " <<
+        G_00B848_IEEE_MODE(Header.compute_pgm_resource_registers) << '\n' <<
+    "\t\tcompute_pgm_rsrc2_scratch_en = " <<
+        G_00B84C_SCRATCH_EN(ComputePgmRsrc2) << '\n' <<
+    "\t\tcompute_pgm_rsrc2_user_sgpr = " <<
+        G_00B84C_USER_SGPR(ComputePgmRsrc2) << '\n' <<
+    "\t\tcompute_pgm_rsrc2_tgid_x_en = " <<
+        G_00B84C_TGID_X_EN(ComputePgmRsrc2) << '\n' <<
+    "\t\tcompute_pgm_rsrc2_tgid_y_en = " <<
+        G_00B84C_TGID_Y_EN(ComputePgmRsrc2) << '\n' <<
+    "\t\tcompute_pgm_rsrc2_tgid_z_en = " <<
+        G_00B84C_TGID_Z_EN(ComputePgmRsrc2) << '\n' <<
+    "\t\tcompute_pgm_rsrc2_tg_size_en = " <<
+        G_00B84C_TG_SIZE_EN(ComputePgmRsrc2) << '\n' <<
+    "\t\tcompute_pgm_rsrc2_tidig_comp_cnt = " <<
+        G_00B84C_TIDIG_COMP_CNT(ComputePgmRsrc2) << '\n' <<
+    "\t\tcompute_pgm_rsrc2_excp_en_msb = " <<
+        G_00B84C_EXCP_EN_MSB(ComputePgmRsrc2) << '\n' <<
+    "\t\tcompute_pgm_rsrc2_lds_size = " <<
+        G_00B84C_LDS_SIZE(ComputePgmRsrc2) << '\n' <<
+    "\t\tcompute_pgm_rsrc2_excp_en = " <<
+        G_00B84C_EXCP_EN(ComputePgmRsrc2) << '\n' <<
+
+    "\t\tenable_sgpr_private_segment_buffer = " <<
+        EnableSGPRPrivateSegmentBuffer << '\n' <<
+    "\t\tenable_sgpr_dispatch_ptr = " <<
+        EnableSGPRDispatchPtr << '\n' <<
+    "\t\tenable_sgpr_queue_ptr = " <<
+        EnableSGPRQueuePtr << '\n' <<
+    "\t\tenable_sgpr_kernarg_segment_ptr = " <<
+        EnableSGPRKernargSegmentPtr << '\n' <<
+    "\t\tenable_sgpr_dispatch_id = " <<
+        EnableSGPRDispatchID << '\n' <<
+    "\t\tenable_sgpr_flat_scratch_init = " <<
+        EnableSGPRFlatScratchInit << '\n' <<
+    "\t\tenable_sgpr_private_segment_size = " <<
+        EnableSGPRPrivateSegmentSize << '\n' <<
+    "\t\tenable_sgpr_grid_workgroup_count_x = " <<
+        EnableSGPRGridWorkgroupCountX << '\n' <<
+    "\t\tenable_sgpr_grid_workgroup_count_y = " <<
+        EnableSGPRGridWorkgroupCountY << '\n' <<
+    "\t\tenable_sgpr_grid_workgroup_count_z = " <<
+        EnableSGPRGridWorkgroupCountZ << '\n' <<
+    "\t\tenable_ordered_append_gds = " <<
+        EnableOrderedAppendGDS << '\n' <<
+    "\t\tprivate_element_size = " <<
+        PrivateElementSize << '\n' <<
+    "\t\tis_ptr64 = " <<
+        IsPtr64 << '\n' <<
+    "\t\tis_dynamic_callstack = " <<
+        IsDynamicCallstack << '\n' <<
+    "\t\tis_debug_enabled = " <<
+        IsDebugEnabled << '\n' <<
+    "\t\tis_xnack_enabled = " <<
+        IsXNackEnabled << '\n' <<
+    "\t\tworkitem_private_segment_byte_size = " <<
+        Header.workitem_private_segment_byte_size << '\n' <<
+    "\t\tworkgroup_group_segment_byte_size = " <<
+        Header.workgroup_group_segment_byte_size << '\n' <<
+    "\t\tgds_segment_byte_size = " <<
+        Header.gds_segment_byte_size << '\n' <<
+    "\t\tkernarg_segment_byte_size = " <<
+        Header.kernarg_segment_byte_size << '\n' <<
+    "\t\tworkgroup_fbarrier_count = " <<
+        Header.workgroup_fbarrier_count << '\n' <<
+    "\t\twavefront_sgpr_count = " <<
+        Header.wavefront_sgpr_count << '\n' <<
+    "\t\tworkitem_vgpr_count = " <<
+        Header.workitem_vgpr_count << '\n' <<
+    "\t\treserved_vgpr_first = " <<
+        Header.reserved_vgpr_first << '\n' <<
+    "\t\treserved_vgpr_count = " <<
+        Header.reserved_vgpr_count << '\n' <<
+    "\t\treserved_sgpr_first = " <<
+        Header.reserved_sgpr_first << '\n' <<
+    "\t\treserved_sgpr_count = " <<
+        Header.reserved_sgpr_count << '\n' <<
+    "\t\tdebug_wavefront_private_segment_offset_sgpr = " <<
+        Header.debug_wavefront_private_segment_offset_sgpr << '\n' <<
+    "\t\tdebug_private_segment_buffer_sgpr = " <<
+        Header.debug_private_segment_buffer_sgpr << '\n' <<
+    "\t\tkernarg_segment_alignment = " <<
+        (uint32_t)Header.kernarg_segment_alignment << '\n' <<
+    "\t\tgroup_segment_alignment = " <<
+        (uint32_t)Header.group_segment_alignment << '\n' <<
+    "\t\tprivate_segment_alignment = " <<
+        (uint32_t)Header.private_segment_alignment << '\n' <<
+    "\t\twavefront_size = " <<
+        (uint32_t)Header.wavefront_size << '\n' <<
+    "\t\tcall_convention = " <<
+        Header.call_convention << '\n' <<
+    "\t\truntime_loader_kernel_symbol = " <<
+        Header.runtime_loader_kernel_symbol << '\n' <<
+    // TODO: control_directives
+    "\t.end_amd_kernel_code_t\n";
+
+}
+
+//===----------------------------------------------------------------------===//
+// AMDGPUTargetELFStreamer
+//===----------------------------------------------------------------------===//
+
+AMDGPUTargetELFStreamer::AMDGPUTargetELFStreamer(MCStreamer &S)
+    : AMDGPUTargetStreamer(S), Streamer(S) { }
+
+MCELFStreamer &AMDGPUTargetELFStreamer::getStreamer() {
+  return static_cast<MCELFStreamer &>(Streamer);
+}
+
+void
+AMDGPUTargetELFStreamer::EmitDirectiveHSACodeObjectVersion(uint32_t Major,
+                                                           uint32_t Minor) {
+  MCStreamer &OS = getStreamer();
+  MCSectionELF *Note = OS.getContext().getELFSection(".note", ELF::SHT_NOTE, 0);
+
+  unsigned NameSZ = 4;
+
+  OS.PushSection();
+  OS.SwitchSection(Note);
+  OS.EmitIntValue(NameSZ, 4);                            // namesz
+  OS.EmitIntValue(8, 4);                                 // descz
+  OS.EmitIntValue(NT_AMDGPU_HSA_CODE_OBJECT_VERSION, 4); // type
+  OS.EmitBytes(StringRef("AMD", NameSZ));                // name
+  OS.EmitIntValue(Major, 4);                             // desc
+  OS.EmitIntValue(Minor, 4);
+  OS.EmitValueToAlignment(4);
+  OS.PopSection();
+}
+
+void
+AMDGPUTargetELFStreamer::EmitDirectiveHSACodeObjectISA(uint32_t Major,
+                                                       uint32_t Minor,
+                                                       uint32_t Stepping,
+                                                       StringRef VendorName,
+                                                       StringRef ArchName) {
+  MCStreamer &OS = getStreamer();
+  MCSectionELF *Note = OS.getContext().getELFSection(".note", ELF::SHT_NOTE, 0);
+
+  unsigned NameSZ = 4;
+  uint16_t VendorNameSize = VendorName.size() + 1;
+  uint16_t ArchNameSize = ArchName.size() + 1;
+  unsigned DescSZ = sizeof(VendorNameSize) + sizeof(ArchNameSize) +
+                    sizeof(Major) + sizeof(Minor) + sizeof(Stepping) +
+                    VendorNameSize + ArchNameSize;
+
+  OS.PushSection();
+  OS.SwitchSection(Note);
+  OS.EmitIntValue(NameSZ, 4);                            // namesz
+  OS.EmitIntValue(DescSZ, 4);                            // descsz
+  OS.EmitIntValue(NT_AMDGPU_HSA_ISA, 4);                 // type
+  OS.EmitBytes(StringRef("AMD", 4));                     // name
+  OS.EmitIntValue(VendorNameSize, 2);                    // desc
+  OS.EmitIntValue(ArchNameSize, 2);
+  OS.EmitIntValue(Major, 4);
+  OS.EmitIntValue(Minor, 4);
+  OS.EmitIntValue(Stepping, 4);
+  OS.EmitBytes(VendorName);
+  OS.EmitIntValue(0, 1); // NULL terminate VendorName
+  OS.EmitBytes(ArchName);
+  OS.EmitIntValue(0, 1); // NULL terminte ArchName
+  OS.EmitValueToAlignment(4);
+  OS.PopSection();
+}
+
+void
+AMDGPUTargetELFStreamer::EmitAMDKernelCodeT(const amd_kernel_code_t &Header) {
+
+  MCStreamer &OS = getStreamer();
+  OS.PushSection();
+  OS.SwitchSection(OS.getContext().getObjectFileInfo()->getTextSection());
+  OS.EmitBytes(StringRef((const char*)&Header, sizeof(Header)));
+  OS.PopSection();
+}
diff --git a/lib/Target/AMDGPU/MCTargetDesc/AMDGPUTargetStreamer.h b/lib/Target/AMDGPU/MCTargetDesc/AMDGPUTargetStreamer.h
new file mode 100644
index 00000000000..d37677c6b86
--- /dev/null
+++ b/lib/Target/AMDGPU/MCTargetDesc/AMDGPUTargetStreamer.h
@@ -0,0 +1,77 @@
+//===-- AMDGPUTargetStreamer.h - AMDGPU Target Streamer --------*- C++ -*--===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+
+#include "AMDKernelCodeT.h"
+#include "llvm/MC/MCStreamer.h"
+#include "llvm/MC/MCSymbol.h"
+#include "llvm/Support/Debug.h"
+namespace llvm {
+
+class MCELFStreamer;
+
+class AMDGPUTargetStreamer : public MCTargetStreamer {
+public:
+  AMDGPUTargetStreamer(MCStreamer &S);
+  virtual void EmitDirectiveHSACodeObjectVersion(uint32_t Major,
+                                                 uint32_t Minor) = 0;
+
+  virtual void EmitDirectiveHSACodeObjectISA(uint32_t Major, uint32_t Minor,
+                                             uint32_t Stepping,
+                                             StringRef VendorName,
+                                             StringRef ArchName) = 0;
+
+  virtual void EmitAMDKernelCodeT(const amd_kernel_code_t &Header) = 0;
+};
+
+class AMDGPUTargetAsmStreamer : public AMDGPUTargetStreamer {
+  formatted_raw_ostream &OS;
+public:
+  AMDGPUTargetAsmStreamer(MCStreamer &S, formatted_raw_ostream &OS);
+  void EmitDirectiveHSACodeObjectVersion(uint32_t Major,
+                                         uint32_t Minor) override;
+
+  void EmitDirectiveHSACodeObjectISA(uint32_t Major, uint32_t Minor,
+                                     uint32_t Stepping, StringRef VendorName,
+                                     StringRef ArchName) override;
+
+  void EmitAMDKernelCodeT(const amd_kernel_code_t &Header) override;
+};
+
+class AMDGPUTargetELFStreamer : public AMDGPUTargetStreamer {
+
+  enum NoteType {
+    NT_AMDGPU_HSA_CODE_OBJECT_VERSION = 1,
+    NT_AMDGPU_HSA_HSAIL = 2,
+    NT_AMDGPU_HSA_ISA = 3,
+    NT_AMDGPU_HSA_PRODUCER = 4,
+    NT_AMDGPU_HSA_PRODUCER_OPTIONS = 5,
+    NT_AMDGPU_HSA_EXTENSION = 6,
+    NT_AMDGPU_HSA_HLDEBUG_DEBUG = 101,
+    NT_AMDGPU_HSA_HLDEBUG_TARGET = 102
+  };
+
+  MCStreamer &Streamer;
+
+public:
+  AMDGPUTargetELFStreamer(MCStreamer &S);
+
+  MCELFStreamer &getStreamer();
+
+  void EmitDirectiveHSACodeObjectVersion(uint32_t Major,
+                                         uint32_t Minor) override;
+
+  void EmitDirectiveHSACodeObjectISA(uint32_t Major, uint32_t Minor,
+                                     uint32_t Stepping, StringRef VendorName,
+                                     StringRef ArchName) override;
+
+  void EmitAMDKernelCodeT(const amd_kernel_code_t &Header) override;
+
+};
+
+}
diff --git a/lib/Target/AMDGPU/MCTargetDesc/CMakeLists.txt b/lib/Target/AMDGPU/MCTargetDesc/CMakeLists.txt
new file mode 100644
index 00000000000..8306a051ff9
--- /dev/null
+++ b/lib/Target/AMDGPU/MCTargetDesc/CMakeLists.txt
@@ -0,0 +1,11 @@
+
+add_llvm_library(LLVMAMDGPUDesc
+  AMDGPUAsmBackend.cpp
+  AMDGPUELFObjectWriter.cpp
+  AMDGPUMCCodeEmitter.cpp
+  AMDGPUMCTargetDesc.cpp
+  AMDGPUMCAsmInfo.cpp
+  AMDGPUTargetStreamer.cpp
+  R600MCCodeEmitter.cpp
+  SIMCCodeEmitter.cpp
+  )
diff --git a/lib/Target/AMDGPU/MCTargetDesc/LLVMBuild.txt b/lib/Target/AMDGPU/MCTargetDesc/LLVMBuild.txt
new file mode 100644
index 00000000000..4217bb36297
--- /dev/null
+++ b/lib/Target/AMDGPU/MCTargetDesc/LLVMBuild.txt
@@ -0,0 +1,23 @@
+;===- ./lib/Target/AMDGPU/MCTargetDesc/LLVMBuild.txt -------------*- Conf -*--===;
+;
+;                     The LLVM Compiler Infrastructure
+;
+; This file is distributed under the University of Illinois Open Source
+; License. See LICENSE.TXT for details.
+;
+;===------------------------------------------------------------------------===;
+;
+; This is an LLVMBuild description file for the components in this subdirectory.
+;
+; For more information on the LLVMBuild system, please see:
+;
+;   http://llvm.org/docs/LLVMBuild.html
+;
+;===------------------------------------------------------------------------===;
+
+[component_0]
+type = Library
+name = AMDGPUDesc
+parent = AMDGPU
+required_libraries = MC AMDGPUAsmPrinter AMDGPUInfo Support
+add_to_library_groups = AMDGPU
diff --git a/lib/Target/AMDGPU/MCTargetDesc/Makefile b/lib/Target/AMDGPU/MCTargetDesc/Makefile
new file mode 100644
index 00000000000..5ad68662d98
--- /dev/null
+++ b/lib/Target/AMDGPU/MCTargetDesc/Makefile
@@ -0,0 +1,16 @@
+##===- lib/Target/AMDGPU/TargetDesc/Makefile ----------------*- Makefile -*-===##
+#
+#                     The LLVM Compiler Infrastructure
+#
+# This file is distributed under the University of Illinois Open Source
+# License. See LICENSE.TXT for details.
+#
+##===----------------------------------------------------------------------===##
+
+LEVEL = ../../../..
+LIBRARYNAME = LLVMAMDGPUDesc
+
+# Hack: we need to include 'main' target directory to grab private headers
+CPP.Flags += -I$(PROJ_OBJ_DIR)/.. -I$(PROJ_SRC_DIR)/..
+
+include $(LEVEL)/Makefile.common
diff --git a/lib/Target/R600/MCTargetDesc/R600MCCodeEmitter.cpp b/lib/Target/AMDGPU/MCTargetDesc/R600MCCodeEmitter.cpp
similarity index 92%
rename from lib/Target/R600/MCTargetDesc/R600MCCodeEmitter.cpp
rename to lib/Target/AMDGPU/MCTargetDesc/R600MCCodeEmitter.cpp
index fa25f590f23..e683498d52a 100644
--- a/lib/Target/R600/MCTargetDesc/R600MCCodeEmitter.cpp
+++ b/lib/Target/AMDGPU/MCTargetDesc/R600MCCodeEmitter.cpp
@@ -23,6 +23,7 @@
 #include "llvm/MC/MCInstrInfo.h"
 #include "llvm/MC/MCRegisterInfo.h"
 #include "llvm/MC/MCSubtargetInfo.h"
+#include "llvm/Support/EndianStream.h"
 #include "llvm/Support/raw_ostream.h"
 
 using namespace llvm;
@@ -41,7 +42,7 @@ class R600MCCodeEmitter : public AMDGPUMCCodeEmitter {
     : MCII(mcii), MRI(mri) { }
 
   /// \brief Encode the instruction and write it to the OS.
-  void EncodeInstruction(const MCInst &MI, raw_ostream &OS,
+  void encodeInstruction(const MCInst &MI, raw_ostream &OS,
                          SmallVectorImpl<MCFixup> &Fixups,
                          const MCSubtargetInfo &STI) const override;
 
@@ -86,7 +87,7 @@ MCCodeEmitter *llvm::createR600MCCodeEmitter(const MCInstrInfo &MCII,
   return new R600MCCodeEmitter(MCII, MRI);
 }
 
-void R600MCCodeEmitter::EncodeInstruction(const MCInst &MI, raw_ostream &OS,
+void R600MCCodeEmitter::encodeInstruction(const MCInst &MI, raw_ostream &OS,
                                        SmallVectorImpl<MCFixup> &Fixups,
                                        const MCSubtargetInfo &STI) const {
   const MCInstrDesc &Desc = MCII.get(MI.getOpcode());
@@ -99,7 +100,7 @@ void R600MCCodeEmitter::EncodeInstruction(const MCInst &MI, raw_ostream &OS,
   } else if (IS_VTX(Desc)) {
     uint64_t InstWord01 = getBinaryCodeForInstr(MI, Fixups, STI);
     uint32_t InstWord2 = MI.getOperand(2).getImm(); // Offset
-    if (!(STI.getFeatureBits() & AMDGPU::FeatureCaymanISA)) {
+    if (!(STI.getFeatureBits()[AMDGPU::FeatureCaymanISA])) {
       InstWord2 |= 1 << 19; // Mega-Fetch bit
     }
 
@@ -132,7 +133,7 @@ void R600MCCodeEmitter::EncodeInstruction(const MCInst &MI, raw_ostream &OS,
       Emit((uint32_t) 0, OS);
   } else {
     uint64_t Inst = getBinaryCodeForInstr(MI, Fixups, STI);
-    if ((STI.getFeatureBits() & AMDGPU::FeatureR600ALUInst) &&
+    if ((STI.getFeatureBits()[AMDGPU::FeatureR600ALUInst]) &&
        ((Desc.TSFlags & R600_InstFlag::OP1) ||
          Desc.TSFlags & R600_InstFlag::OP2)) {
       uint64_t ISAOpCode = Inst & (0x3FFULL << 39);
@@ -148,15 +149,11 @@ void R600MCCodeEmitter::EmitByte(unsigned int Byte, raw_ostream &OS) const {
 }
 
 void R600MCCodeEmitter::Emit(uint32_t Value, raw_ostream &OS) const {
-  for (unsigned i = 0; i < 4; i++) {
-    OS.write((uint8_t) ((Value >> (8 * i)) & 0xff));
-  }
+  support::endian::Writer<support::little>(OS).write(Value);
 }
 
 void R600MCCodeEmitter::Emit(uint64_t Value, raw_ostream &OS) const {
-  for (unsigned i = 0; i < 8; i++) {
-    EmitByte((Value >> (8 * i)) & 0xff, OS);
-  }
+  support::endian::Writer<support::little>(OS).write(Value);
 }
 
 unsigned R600MCCodeEmitter::getHWRegChan(unsigned reg) const {
diff --git a/lib/Target/R600/MCTargetDesc/SIMCCodeEmitter.cpp b/lib/Target/AMDGPU/MCTargetDesc/SIMCCodeEmitter.cpp
similarity index 96%
rename from lib/Target/R600/MCTargetDesc/SIMCCodeEmitter.cpp
rename to lib/Target/AMDGPU/MCTargetDesc/SIMCCodeEmitter.cpp
index 24f2b6dd69f..65a0eeba2b1 100644
--- a/lib/Target/R600/MCTargetDesc/SIMCCodeEmitter.cpp
+++ b/lib/Target/AMDGPU/MCTargetDesc/SIMCCodeEmitter.cpp
@@ -52,7 +52,7 @@ class SIMCCodeEmitter : public  AMDGPUMCCodeEmitter {
   ~SIMCCodeEmitter() override {}
 
   /// \brief Encode the instruction and write it to the OS.
-  void EncodeInstruction(const MCInst &MI, raw_ostream &OS,
+  void encodeInstruction(const MCInst &MI, raw_ostream &OS,
                          SmallVectorImpl<MCFixup> &Fixups,
                          const MCSubtargetInfo &STI) const override;
 
@@ -179,7 +179,7 @@ uint32_t SIMCCodeEmitter::getLitEncoding(const MCOperand &MO,
   return getLit64Encoding(static_cast<uint64_t>(MO.getImm()));
 }
 
-void SIMCCodeEmitter::EncodeInstruction(const MCInst &MI, raw_ostream &OS,
+void SIMCCodeEmitter::encodeInstruction(const MCInst &MI, raw_ostream &OS,
                                        SmallVectorImpl<MCFixup> &Fixups,
                                        const MCSubtargetInfo &STI) const {
 
@@ -234,7 +234,7 @@ unsigned SIMCCodeEmitter::getSOPPBrEncoding(const MCInst &MI, unsigned OpNo,
   if (MO.isExpr()) {
     const MCExpr *Expr = MO.getExpr();
     MCFixupKind Kind = (MCFixupKind)AMDGPU::fixup_si_sopp_br;
-    Fixups.push_back(MCFixup::Create(0, Expr, Kind, MI.getLoc()));
+    Fixups.push_back(MCFixup::create(0, Expr, Kind, MI.getLoc()));
     return 0;
   }
 
@@ -252,7 +252,7 @@ uint64_t SIMCCodeEmitter::getMachineOpValue(const MCInst &MI,
     const MCSymbolRefExpr *Expr = cast<MCSymbolRefExpr>(MO.getExpr());
     MCFixupKind Kind;
     const MCSymbol *Sym =
-        Ctx.GetOrCreateSymbol(StringRef(END_OF_TEXT_LABEL_NAME));
+        Ctx.getOrCreateSymbol(StringRef(END_OF_TEXT_LABEL_NAME));
 
     if (&Expr->getSymbol() == Sym) {
       // Add the offset to the beginning of the constant values.
@@ -261,7 +261,7 @@ uint64_t SIMCCodeEmitter::getMachineOpValue(const MCInst &MI,
       // This is used for constant data stored in .rodata.
      Kind = (MCFixupKind)AMDGPU::fixup_si_rodata;
     }
-    Fixups.push_back(MCFixup::Create(4, Expr, Kind, MI.getLoc()));
+    Fixups.push_back(MCFixup::create(4, Expr, Kind, MI.getLoc()));
   }
 
   // Figure out the operand number, needed for isSrcOperand check
diff --git a/lib/Target/AMDGPU/Makefile b/lib/Target/AMDGPU/Makefile
new file mode 100644
index 00000000000..219f34daa24
--- /dev/null
+++ b/lib/Target/AMDGPU/Makefile
@@ -0,0 +1,23 @@
+##===- lib/Target/R600/Makefile ---------------------------*- Makefile -*-===##
+#
+#                     The LLVM Compiler Infrastructure
+#
+# This file is distributed under the University of Illinois Open Source
+# License. See LICENSE.TXT for details.
+#
+##===----------------------------------------------------------------------===##
+
+LEVEL = ../../..
+LIBRARYNAME = LLVMAMDGPUCodeGen
+TARGET = AMDGPU
+
+# Make sure that tblgen is run, first thing.
+BUILT_SOURCES = AMDGPUGenRegisterInfo.inc AMDGPUGenInstrInfo.inc \
+		AMDGPUGenDAGISel.inc  AMDGPUGenSubtargetInfo.inc \
+		AMDGPUGenMCCodeEmitter.inc AMDGPUGenCallingConv.inc \
+		AMDGPUGenIntrinsics.inc AMDGPUGenDFAPacketizer.inc \
+		AMDGPUGenAsmWriter.inc AMDGPUGenAsmMatcher.inc
+
+DIRS = AsmParser InstPrinter TargetInfo MCTargetDesc Utils
+
+include $(LEVEL)/Makefile.common
diff --git a/lib/Target/R600/Processors.td b/lib/Target/AMDGPU/Processors.td
similarity index 84%
rename from lib/Target/R600/Processors.td
rename to lib/Target/AMDGPU/Processors.td
index 82c6d13c44f..69efb8b8bc4 100644
--- a/lib/Target/R600/Processors.td
+++ b/lib/Target/AMDGPU/Processors.td
@@ -103,28 +103,38 @@ def : ProcessorModel<"hainan",   SIQuarterSpeedModel, [FeatureSouthernIslands]>;
 // Sea Islands
 //===----------------------------------------------------------------------===//
 
-def : ProcessorModel<"bonaire",    SIQuarterSpeedModel, [FeatureSeaIslands]>;
+def : ProcessorModel<"bonaire",    SIQuarterSpeedModel,
+  [FeatureSeaIslands, FeatureLDSBankCount32, FeatureISAVersion7_0_0]
+>;
 
-def : ProcessorModel<"kabini",     SIQuarterSpeedModel, [FeatureSeaIslands]>;
+def : ProcessorModel<"kabini",     SIQuarterSpeedModel,
+  [FeatureSeaIslands, FeatureLDSBankCount16]
+>;
 
-def : ProcessorModel<"kaveri",     SIQuarterSpeedModel, [FeatureSeaIslands]>;
+def : ProcessorModel<"kaveri",     SIQuarterSpeedModel,
+  [FeatureSeaIslands, FeatureLDSBankCount32, FeatureISAVersion7_0_0]
+>;
 
 def : ProcessorModel<"hawaii", SIFullSpeedModel,
-  [FeatureSeaIslands, FeatureFastFMAF32]
+  [FeatureSeaIslands, FeatureFastFMAF32, FeatureLDSBankCount32,
+   FeatureISAVersion7_0_1]
 >;
 
-def : ProcessorModel<"mullins",    SIQuarterSpeedModel, [FeatureSeaIslands]>;
+def : ProcessorModel<"mullins",    SIQuarterSpeedModel,
+  [FeatureSeaIslands, FeatureLDSBankCount16]>;
 
 //===----------------------------------------------------------------------===//
 // Volcanic Islands
 //===----------------------------------------------------------------------===//
 
 def : ProcessorModel<"tonga",   SIQuarterSpeedModel,
-  [FeatureVolcanicIslands, FeatureSGPRInitBug]
+  [FeatureVolcanicIslands, FeatureSGPRInitBug, FeatureISAVersion8_0_0]
 >;
 
 def : ProcessorModel<"iceland", SIQuarterSpeedModel,
-  [FeatureVolcanicIslands, FeatureSGPRInitBug]
+  [FeatureVolcanicIslands, FeatureSGPRInitBug, FeatureISAVersion8_0_0]
 >;
 
-def : ProcessorModel<"carrizo", SIQuarterSpeedModel, [FeatureVolcanicIslands]>;
+def : ProcessorModel<"carrizo", SIQuarterSpeedModel,
+  [FeatureVolcanicIslands, FeatureISAVersion8_0_1]
+>;
diff --git a/lib/Target/R600/R600ClauseMergePass.cpp b/lib/Target/AMDGPU/R600ClauseMergePass.cpp
similarity index 100%
rename from lib/Target/R600/R600ClauseMergePass.cpp
rename to lib/Target/AMDGPU/R600ClauseMergePass.cpp
diff --git a/lib/Target/R600/R600ControlFlowFinalizer.cpp b/lib/Target/AMDGPU/R600ControlFlowFinalizer.cpp
similarity index 100%
rename from lib/Target/R600/R600ControlFlowFinalizer.cpp
rename to lib/Target/AMDGPU/R600ControlFlowFinalizer.cpp
diff --git a/lib/Target/R600/R600Defines.h b/lib/Target/AMDGPU/R600Defines.h
similarity index 100%
rename from lib/Target/R600/R600Defines.h
rename to lib/Target/AMDGPU/R600Defines.h
diff --git a/lib/Target/R600/R600EmitClauseMarkers.cpp b/lib/Target/AMDGPU/R600EmitClauseMarkers.cpp
similarity index 100%
rename from lib/Target/R600/R600EmitClauseMarkers.cpp
rename to lib/Target/AMDGPU/R600EmitClauseMarkers.cpp
diff --git a/lib/Target/R600/R600ExpandSpecialInstrs.cpp b/lib/Target/AMDGPU/R600ExpandSpecialInstrs.cpp
similarity index 100%
rename from lib/Target/R600/R600ExpandSpecialInstrs.cpp
rename to lib/Target/AMDGPU/R600ExpandSpecialInstrs.cpp
diff --git a/lib/Target/R600/R600ISelLowering.cpp b/lib/Target/AMDGPU/R600ISelLowering.cpp
similarity index 91%
rename from lib/Target/R600/R600ISelLowering.cpp
rename to lib/Target/AMDGPU/R600ISelLowering.cpp
index b6b7067f7e1..ecda332ed66 100644
--- a/lib/Target/R600/R600ISelLowering.cpp
+++ b/lib/Target/AMDGPU/R600ISelLowering.cpp
@@ -91,6 +91,14 @@ R600TargetLowering::R600TargetLowering(TargetMachine &TM,
   setOperationAction(ISD::SELECT, MVT::v2i32, Expand);
   setOperationAction(ISD::SELECT, MVT::v4i32, Expand);
 
+  // ADD, SUB overflow.
+  // TODO: turn these into Legal?
+  if (Subtarget->hasCARRY())
+    setOperationAction(ISD::UADDO, MVT::i32, Custom);
+
+  if (Subtarget->hasBORROW())
+    setOperationAction(ISD::USUBO, MVT::i32, Custom);
+
   // Expand sign extension of vectors
   if (!Subtarget->hasBFE())
     setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::i1, Expand);
@@ -163,15 +171,6 @@ R600TargetLowering::R600TargetLowering(TargetMachine &TM,
   setTargetDAGCombine(ISD::SELECT_CC);
   setTargetDAGCombine(ISD::INSERT_VECTOR_ELT);
 
-  setOperationAction(ISD::SUB, MVT::i64, Expand);
-
-  // These should be replaced by UDVIREM, but it does not happen automatically
-  // during Type Legalization
-  setOperationAction(ISD::UDIV, MVT::i64, Custom);
-  setOperationAction(ISD::UREM, MVT::i64, Custom);
-  setOperationAction(ISD::SDIV, MVT::i64, Custom);
-  setOperationAction(ISD::SREM, MVT::i64, Custom);
-
   // We don't have 64-bit shifts. Thus we need either SHX i64 or SHX_PARTS i32
   //  to be Legal/Custom in order to avoid library calls.
   setOperationAction(ISD::SHL_PARTS, MVT::i32, Custom);
@@ -585,6 +584,8 @@ SDValue R600TargetLowering::LowerOperation(SDValue Op, SelectionDAG &DAG) const
   case ISD::SHL_PARTS: return LowerSHLParts(Op, DAG);
   case ISD::SRA_PARTS:
   case ISD::SRL_PARTS: return LowerSRXParts(Op, DAG);
+  case ISD::UADDO: return LowerUADDSUBO(Op, DAG, ISD::ADD, AMDGPUISD::CARRY);
+  case ISD::USUBO: return LowerUADDSUBO(Op, DAG, ISD::SUB, AMDGPUISD::BORROW);
   case ISD::FCOS:
   case ISD::FSIN: return LowerTrig(Op, DAG);
   case ISD::SELECT_CC: return LowerSELECT_CC(Op, DAG);
@@ -611,17 +612,18 @@ SDValue R600TargetLowering::LowerOperation(SDValue Op, SelectionDAG &DAG) const
       return DAG.getCopyToReg(Chain, SDLoc(Op), Reg, Op.getOperand(2));
     }
     case AMDGPUIntrinsic::R600_store_swizzle: {
+      SDLoc DL(Op);
       const SDValue Args[8] = {
         Chain,
         Op.getOperand(2), // Export Value
         Op.getOperand(3), // ArrayBase
         Op.getOperand(4), // Type
-        DAG.getConstant(0, MVT::i32), // SWZ_X
-        DAG.getConstant(1, MVT::i32), // SWZ_Y
-        DAG.getConstant(2, MVT::i32), // SWZ_Z
-        DAG.getConstant(3, MVT::i32) // SWZ_W
+        DAG.getConstant(0, DL, MVT::i32), // SWZ_X
+        DAG.getConstant(1, DL, MVT::i32), // SWZ_Y
+        DAG.getConstant(2, DL, MVT::i32), // SWZ_Z
+        DAG.getConstant(3, DL, MVT::i32) // SWZ_W
       };
-      return DAG.getNode(AMDGPUISD::EXPORT, SDLoc(Op), Op.getValueType(), Args);
+      return DAG.getNode(AMDGPUISD::EXPORT, DL, Op.getValueType(), Args);
     }
 
     // default for switch(IntrinsicID)
@@ -655,7 +657,7 @@ SDValue R600TargetLowering::LowerOperation(SDValue Op, SelectionDAG &DAG) const
         const R600InstrInfo *TII =
             static_cast<const R600InstrInfo *>(Subtarget->getInstrInfo());
         interp = DAG.getMachineNode(AMDGPU::INTERP_VEC_LOAD, DL,
-            MVT::v4f32, DAG.getTargetConstant(slot / 4 , MVT::i32));
+            MVT::v4f32, DAG.getTargetConstant(slot / 4, DL, MVT::i32));
         return DAG.getTargetExtractSubreg(
             TII->getRegisterInfo().getSubRegFromChannel(slot % 4),
             DL, MVT::f32, SDValue(interp, 0));
@@ -673,11 +675,11 @@ SDValue R600TargetLowering::LowerOperation(SDValue Op, SelectionDAG &DAG) const
 
       if (slot % 4 < 2)
         interp = DAG.getMachineNode(AMDGPU::INTERP_PAIR_XY, DL,
-            MVT::f32, MVT::f32, DAG.getTargetConstant(slot / 4 , MVT::i32),
+            MVT::f32, MVT::f32, DAG.getTargetConstant(slot / 4, DL, MVT::i32),
             RegisterJNode, RegisterINode);
       else
         interp = DAG.getMachineNode(AMDGPU::INTERP_PAIR_ZW, DL,
-            MVT::f32, MVT::f32, DAG.getTargetConstant(slot / 4 , MVT::i32),
+            MVT::f32, MVT::f32, DAG.getTargetConstant(slot / 4, DL, MVT::i32),
             RegisterJNode, RegisterINode);
       return SDValue(interp, slot % 2);
     }
@@ -690,11 +692,11 @@ SDValue R600TargetLowering::LowerOperation(SDValue Op, SelectionDAG &DAG) const
 
       if (IntrinsicID == AMDGPUIntrinsic::R600_interp_xy)
         interp = DAG.getMachineNode(AMDGPU::INTERP_PAIR_XY, DL,
-            MVT::f32, MVT::f32, DAG.getTargetConstant(slot, MVT::i32),
+            MVT::f32, MVT::f32, DAG.getTargetConstant(slot, DL, MVT::i32),
             RegisterJNode, RegisterINode);
       else
         interp = DAG.getMachineNode(AMDGPU::INTERP_PAIR_ZW, DL,
-            MVT::f32, MVT::f32, DAG.getTargetConstant(slot, MVT::i32),
+            MVT::f32, MVT::f32, DAG.getTargetConstant(slot, DL, MVT::i32),
             RegisterJNode, RegisterINode);
       return DAG.getNode(ISD::BUILD_VECTOR, DL, MVT::v2f32,
           SDValue(interp, 0), SDValue(interp, 1));
@@ -750,19 +752,19 @@ SDValue R600TargetLowering::LowerOperation(SDValue Op, SelectionDAG &DAG) const
       }
 
       SDValue TexArgs[19] = {
-        DAG.getConstant(TextureOp, MVT::i32),
+        DAG.getConstant(TextureOp, DL, MVT::i32),
         Op.getOperand(1),
-        DAG.getConstant(0, MVT::i32),
-        DAG.getConstant(1, MVT::i32),
-        DAG.getConstant(2, MVT::i32),
-        DAG.getConstant(3, MVT::i32),
+        DAG.getConstant(0, DL, MVT::i32),
+        DAG.getConstant(1, DL, MVT::i32),
+        DAG.getConstant(2, DL, MVT::i32),
+        DAG.getConstant(3, DL, MVT::i32),
         Op.getOperand(2),
         Op.getOperand(3),
         Op.getOperand(4),
-        DAG.getConstant(0, MVT::i32),
-        DAG.getConstant(1, MVT::i32),
-        DAG.getConstant(2, MVT::i32),
-        DAG.getConstant(3, MVT::i32),
+        DAG.getConstant(0, DL, MVT::i32),
+        DAG.getConstant(1, DL, MVT::i32),
+        DAG.getConstant(2, DL, MVT::i32),
+        DAG.getConstant(3, DL, MVT::i32),
         Op.getOperand(5),
         Op.getOperand(6),
         Op.getOperand(7),
@@ -775,21 +777,21 @@ SDValue R600TargetLowering::LowerOperation(SDValue Op, SelectionDAG &DAG) const
     case AMDGPUIntrinsic::AMDGPU_dp4: {
       SDValue Args[8] = {
       DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, MVT::f32, Op.getOperand(1),
-          DAG.getConstant(0, MVT::i32)),
+          DAG.getConstant(0, DL, MVT::i32)),
       DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, MVT::f32, Op.getOperand(2),
-          DAG.getConstant(0, MVT::i32)),
+          DAG.getConstant(0, DL, MVT::i32)),
       DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, MVT::f32, Op.getOperand(1),
-          DAG.getConstant(1, MVT::i32)),
+          DAG.getConstant(1, DL, MVT::i32)),
       DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, MVT::f32, Op.getOperand(2),
-          DAG.getConstant(1, MVT::i32)),
+          DAG.getConstant(1, DL, MVT::i32)),
       DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, MVT::f32, Op.getOperand(1),
-          DAG.getConstant(2, MVT::i32)),
+          DAG.getConstant(2, DL, MVT::i32)),
       DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, MVT::f32, Op.getOperand(2),
-          DAG.getConstant(2, MVT::i32)),
+          DAG.getConstant(2, DL, MVT::i32)),
       DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, MVT::f32, Op.getOperand(1),
-          DAG.getConstant(3, MVT::i32)),
+          DAG.getConstant(3, DL, MVT::i32)),
       DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, MVT::f32, Op.getOperand(2),
-          DAG.getConstant(3, MVT::i32))
+          DAG.getConstant(3, DL, MVT::i32))
       };
       return DAG.getNode(AMDGPUISD::DOT4, DL, MVT::f32, Args);
     }
@@ -813,8 +815,10 @@ SDValue R600TargetLowering::LowerOperation(SDValue Op, SelectionDAG &DAG) const
     case Intrinsic::r600_read_local_size_z:
       return LowerImplicitParameter(DAG, VT, DL, 8);
 
-    case Intrinsic::AMDGPU_read_workdim:
-      return LowerImplicitParameter(DAG, VT, DL, MFI->ABIArgOffset / 4);
+    case Intrinsic::AMDGPU_read_workdim: {
+      uint32_t ByteOffset = getImplicitParameterOffset(MFI, GRID_DIM);
+      return LowerImplicitParameter(DAG, VT, DL, ByteOffset / 4);
+    }
 
     case Intrinsic::r600_read_tgid_x:
       return CreateLiveInRegister(DAG, &AMDGPU::R600_TReg32RegClass,
@@ -870,42 +874,6 @@ void R600TargetLowering::ReplaceNodeResults(SDNode *N,
       Results.push_back(Result);
     return;
   }
-  case ISD::UDIV: {
-    SDValue Op = SDValue(N, 0);
-    SDLoc DL(Op);
-    EVT VT = Op.getValueType();
-    SDValue UDIVREM = DAG.getNode(ISD::UDIVREM, DL, DAG.getVTList(VT, VT),
-      N->getOperand(0), N->getOperand(1));
-    Results.push_back(UDIVREM);
-    break;
-  }
-  case ISD::UREM: {
-    SDValue Op = SDValue(N, 0);
-    SDLoc DL(Op);
-    EVT VT = Op.getValueType();
-    SDValue UDIVREM = DAG.getNode(ISD::UDIVREM, DL, DAG.getVTList(VT, VT),
-      N->getOperand(0), N->getOperand(1));
-    Results.push_back(UDIVREM.getValue(1));
-    break;
-  }
-  case ISD::SDIV: {
-    SDValue Op = SDValue(N, 0);
-    SDLoc DL(Op);
-    EVT VT = Op.getValueType();
-    SDValue SDIVREM = DAG.getNode(ISD::SDIVREM, DL, DAG.getVTList(VT, VT),
-      N->getOperand(0), N->getOperand(1));
-    Results.push_back(SDIVREM);
-    break;
-  }
-  case ISD::SREM: {
-    SDValue Op = SDValue(N, 0);
-    SDLoc DL(Op);
-    EVT VT = Op.getValueType();
-    SDValue SDIVREM = DAG.getNode(ISD::SDIVREM, DL, DAG.getVTList(VT, VT),
-      N->getOperand(0), N->getOperand(1));
-    Results.push_back(SDIVREM.getValue(1));
-    break;
-  }
   case ISD::SDIVREM: {
     SDValue Op = SDValue(N, 1);
     SDValue RES = LowerSDIVREM(Op, DAG);
@@ -931,8 +899,9 @@ SDValue R600TargetLowering::vectorToVerticalVector(SelectionDAG &DAG,
 
   for (unsigned i = 0, e = VecVT.getVectorNumElements();
                                                            i != e; ++i) {
-    Args.push_back(DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, EltVT,
-                               Vector, DAG.getConstant(i, getVectorIdxTy())));
+    Args.push_back(DAG.getNode(
+        ISD::EXTRACT_VECTOR_ELT, DL, EltVT, Vector,
+        DAG.getConstant(i, DL, getVectorIdxTy(DAG.getDataLayout()))));
   }
 
   return DAG.getNode(AMDGPUISD::BUILD_VERTICAL_VECTOR, DL, VecVT, Args);
@@ -976,11 +945,12 @@ SDValue R600TargetLowering::LowerTrig(SDValue Op, SelectionDAG &DAG) const {
   // Thus we lower them to TRIG ( FRACT ( x / 2Pi + 0.5) - 0.5)
   EVT VT = Op.getValueType();
   SDValue Arg = Op.getOperand(0);
-  SDValue FractPart = DAG.getNode(AMDGPUISD::FRACT, SDLoc(Op), VT,
-      DAG.getNode(ISD::FADD, SDLoc(Op), VT,
-        DAG.getNode(ISD::FMUL, SDLoc(Op), VT, Arg,
-          DAG.getConstantFP(0.15915494309, MVT::f32)),
-        DAG.getConstantFP(0.5, MVT::f32)));
+  SDLoc DL(Op);
+  SDValue FractPart = DAG.getNode(AMDGPUISD::FRACT, DL, VT,
+      DAG.getNode(ISD::FADD, DL, VT,
+        DAG.getNode(ISD::FMUL, DL, VT, Arg,
+          DAG.getConstantFP(0.15915494309, DL, MVT::f32)),
+        DAG.getConstantFP(0.5, DL, MVT::f32)));
   unsigned TrigNode;
   switch (Op.getOpcode()) {
   case ISD::FCOS:
@@ -992,14 +962,14 @@ SDValue R600TargetLowering::LowerTrig(SDValue Op, SelectionDAG &DAG) const {
   default:
     llvm_unreachable("Wrong trig opcode");
   }
-  SDValue TrigVal = DAG.getNode(TrigNode, SDLoc(Op), VT,
-      DAG.getNode(ISD::FADD, SDLoc(Op), VT, FractPart,
-        DAG.getConstantFP(-0.5, MVT::f32)));
+  SDValue TrigVal = DAG.getNode(TrigNode, DL, VT,
+      DAG.getNode(ISD::FADD, DL, VT, FractPart,
+        DAG.getConstantFP(-0.5, DL, MVT::f32)));
   if (Gen >= AMDGPUSubtarget::R700)
     return TrigVal;
   // On R600 hw, COS/SIN input must be between -Pi and Pi.
-  return DAG.getNode(ISD::FMUL, SDLoc(Op), VT, TrigVal,
-      DAG.getConstantFP(3.14159265359, MVT::f32));
+  return DAG.getNode(ISD::FMUL, DL, VT, TrigVal,
+      DAG.getConstantFP(3.14159265359, DL, MVT::f32));
 }
 
 SDValue R600TargetLowering::LowerSHLParts(SDValue Op, SelectionDAG &DAG) const {
@@ -1009,11 +979,11 @@ SDValue R600TargetLowering::LowerSHLParts(SDValue Op, SelectionDAG &DAG) const {
   SDValue Lo = Op.getOperand(0);
   SDValue Hi = Op.getOperand(1);
   SDValue Shift = Op.getOperand(2);
-  SDValue Zero = DAG.getConstant(0, VT);
-  SDValue One  = DAG.getConstant(1, VT);
+  SDValue Zero = DAG.getConstant(0, DL, VT);
+  SDValue One  = DAG.getConstant(1, DL, VT);
 
-  SDValue Width  = DAG.getConstant(VT.getSizeInBits(), VT);
-  SDValue Width1 = DAG.getConstant(VT.getSizeInBits() - 1, VT);
+  SDValue Width  = DAG.getConstant(VT.getSizeInBits(), DL, VT);
+  SDValue Width1 = DAG.getConstant(VT.getSizeInBits() - 1, DL, VT);
   SDValue BigShift  = DAG.getNode(ISD::SUB, DL, VT, Shift, Width);
   SDValue CompShift = DAG.getNode(ISD::SUB, DL, VT, Width1, Shift);
 
@@ -1045,13 +1015,13 @@ SDValue R600TargetLowering::LowerSRXParts(SDValue Op, SelectionDAG &DAG) const {
   SDValue Lo = Op.getOperand(0);
   SDValue Hi = Op.getOperand(1);
   SDValue Shift = Op.getOperand(2);
-  SDValue Zero = DAG.getConstant(0, VT);
-  SDValue One  = DAG.getConstant(1, VT);
+  SDValue Zero = DAG.getConstant(0, DL, VT);
+  SDValue One  = DAG.getConstant(1, DL, VT);
 
   const bool SRA = Op.getOpcode() == ISD::SRA_PARTS;
 
-  SDValue Width  = DAG.getConstant(VT.getSizeInBits(), VT);
-  SDValue Width1 = DAG.getConstant(VT.getSizeInBits() - 1, VT);
+  SDValue Width  = DAG.getConstant(VT.getSizeInBits(), DL, VT);
+  SDValue Width1 = DAG.getConstant(VT.getSizeInBits() - 1, DL, VT);
   SDValue BigShift  = DAG.getNode(ISD::SUB, DL, VT, Shift, Width);
   SDValue CompShift = DAG.getNode(ISD::SUB, DL, VT, Width1, Shift);
 
@@ -1076,12 +1046,31 @@ SDValue R600TargetLowering::LowerSRXParts(SDValue Op, SelectionDAG &DAG) const {
   return DAG.getNode(ISD::MERGE_VALUES, DL, DAG.getVTList(VT,VT), Lo, Hi);
 }
 
+SDValue R600TargetLowering::LowerUADDSUBO(SDValue Op, SelectionDAG &DAG,
+                                          unsigned mainop, unsigned ovf) const {
+  SDLoc DL(Op);
+  EVT VT = Op.getValueType();
+
+  SDValue Lo = Op.getOperand(0);
+  SDValue Hi = Op.getOperand(1);
+
+  SDValue OVF = DAG.getNode(ovf, DL, VT, Lo, Hi);
+  // Extend sign.
+  OVF = DAG.getNode(ISD::SIGN_EXTEND_INREG, DL, VT, OVF,
+                    DAG.getValueType(MVT::i1));
+
+  SDValue Res = DAG.getNode(mainop, DL, VT, Lo, Hi);
+
+  return DAG.getNode(ISD::MERGE_VALUES, DL, DAG.getVTList(VT, VT), Res, OVF);
+}
+
 SDValue R600TargetLowering::LowerFPTOUINT(SDValue Op, SelectionDAG &DAG) const {
+  SDLoc DL(Op);
   return DAG.getNode(
       ISD::SETCC,
-      SDLoc(Op),
+      DL,
       MVT::i1,
-      Op, DAG.getConstantFP(0.0f, MVT::f32),
+      Op, DAG.getConstantFP(0.0f, DL, MVT::f32),
       DAG.getCondCode(ISD::SETNE)
       );
 }
@@ -1097,7 +1086,7 @@ SDValue R600TargetLowering::LowerImplicitParameter(SelectionDAG &DAG, EVT VT,
   assert(isInt<16>(ByteOffset));
 
   return DAG.getLoad(VT, DL, DAG.getEntryNode(),
-                     DAG.getConstant(ByteOffset, MVT::i32), // PTR
+                     DAG.getConstant(ByteOffset, DL, MVT::i32), // PTR
                      MachinePointerInfo(ConstantPointerNull::get(PtrType)),
                      false, false, false, 0);
 }
@@ -1234,11 +1223,11 @@ SDValue R600TargetLowering::LowerSELECT_CC(SDValue Op, SelectionDAG &DAG) const
   SDValue HWTrue, HWFalse;
 
   if (CompareVT == MVT::f32) {
-    HWTrue = DAG.getConstantFP(1.0f, CompareVT);
-    HWFalse = DAG.getConstantFP(0.0f, CompareVT);
+    HWTrue = DAG.getConstantFP(1.0f, DL, CompareVT);
+    HWFalse = DAG.getConstantFP(0.0f, DL, CompareVT);
   } else if (CompareVT == MVT::i32) {
-    HWTrue = DAG.getConstant(-1, CompareVT);
-    HWFalse = DAG.getConstant(0, CompareVT);
+    HWTrue = DAG.getConstant(-1, DL, CompareVT);
+    HWFalse = DAG.getConstant(0, DL, CompareVT);
   }
   else {
     llvm_unreachable("Unhandled value type in LowerSELECT_CC");
@@ -1276,8 +1265,9 @@ SDValue R600TargetLowering::stackPtrToRegIndex(SDValue Ptr,
   default: llvm_unreachable("Invalid stack width");
   }
 
-  return DAG.getNode(ISD::SRL, SDLoc(Ptr), Ptr.getValueType(), Ptr,
-                     DAG.getConstant(SRLPad, MVT::i32));
+  SDLoc DL(Ptr);
+  return DAG.getNode(ISD::SRL, DL, Ptr.getValueType(), Ptr,
+                     DAG.getConstant(SRLPad, DL, MVT::i32));
 }
 
 void R600TargetLowering::getStackAddress(unsigned StackWidth,
@@ -1328,26 +1318,26 @@ SDValue R600TargetLowering::LowerSTORE(SDValue Op, SelectionDAG &DAG) const {
       EVT MemVT = StoreNode->getMemoryVT();
       SDValue MaskConstant;
       if (MemVT == MVT::i8) {
-        MaskConstant = DAG.getConstant(0xFF, MVT::i32);
+        MaskConstant = DAG.getConstant(0xFF, DL, MVT::i32);
       } else {
         assert(MemVT == MVT::i16);
-        MaskConstant = DAG.getConstant(0xFFFF, MVT::i32);
+        MaskConstant = DAG.getConstant(0xFFFF, DL, MVT::i32);
       }
       SDValue DWordAddr = DAG.getNode(ISD::SRL, DL, VT, Ptr,
-                                      DAG.getConstant(2, MVT::i32));
+                                      DAG.getConstant(2, DL, MVT::i32));
       SDValue ByteIndex = DAG.getNode(ISD::AND, DL, Ptr.getValueType(), Ptr,
-                                      DAG.getConstant(0x00000003, VT));
+                                      DAG.getConstant(0x00000003, DL, VT));
       SDValue TruncValue = DAG.getNode(ISD::AND, DL, VT, Value, MaskConstant);
       SDValue Shift = DAG.getNode(ISD::SHL, DL, VT, ByteIndex,
-                                   DAG.getConstant(3, VT));
+                                   DAG.getConstant(3, DL, VT));
       SDValue ShiftedValue = DAG.getNode(ISD::SHL, DL, VT, TruncValue, Shift);
       SDValue Mask = DAG.getNode(ISD::SHL, DL, VT, MaskConstant, Shift);
       // XXX: If we add a 64-bit ZW register class, then we could use a 2 x i32
       // vector instead.
       SDValue Src[4] = {
         ShiftedValue,
-        DAG.getConstant(0, MVT::i32),
-        DAG.getConstant(0, MVT::i32),
+        DAG.getConstant(0, DL, MVT::i32),
+        DAG.getConstant(0, DL, MVT::i32),
         Mask
       };
       SDValue Input = DAG.getNode(ISD::BUILD_VECTOR, DL, MVT::v4i32, Src);
@@ -1360,7 +1350,7 @@ SDValue R600TargetLowering::LowerSTORE(SDValue Op, SelectionDAG &DAG) const {
       // Convert pointer from byte address to dword address.
       Ptr = DAG.getNode(AMDGPUISD::DWORDADDR, DL, Ptr.getValueType(),
                         DAG.getNode(ISD::SRL, DL, Ptr.getValueType(),
-                                    Ptr, DAG.getConstant(2, MVT::i32)));
+                                    Ptr, DAG.getConstant(2, DL, MVT::i32)));
 
       if (StoreNode->isTruncatingStore() || StoreNode->isIndexed()) {
         llvm_unreachable("Truncated and indexed stores not supported yet");
@@ -1402,13 +1392,13 @@ SDValue R600TargetLowering::LowerSTORE(SDValue Op, SelectionDAG &DAG) const {
       unsigned Channel, PtrIncr;
       getStackAddress(StackWidth, i, Channel, PtrIncr);
       Ptr = DAG.getNode(ISD::ADD, DL, MVT::i32, Ptr,
-                        DAG.getConstant(PtrIncr, MVT::i32));
+                        DAG.getConstant(PtrIncr, DL, MVT::i32));
       SDValue Elem = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, ElemVT,
-                                 Value, DAG.getConstant(i, MVT::i32));
+                                 Value, DAG.getConstant(i, DL, MVT::i32));
 
       Stores[i] = DAG.getNode(AMDGPUISD::REGISTER_STORE, DL, MVT::Other,
                               Chain, Elem, Ptr,
-                              DAG.getTargetConstant(Channel, MVT::i32));
+                              DAG.getTargetConstant(Channel, DL, MVT::i32));
     }
      Chain =  DAG.getNode(ISD::TokenFactor, DL, MVT::Other, Stores);
    } else {
@@ -1416,7 +1406,7 @@ SDValue R600TargetLowering::LowerSTORE(SDValue Op, SelectionDAG &DAG) const {
       Value = DAG.getNode(ISD::ZERO_EXTEND, DL, MVT::i32, Value);
     }
     Chain = DAG.getNode(AMDGPUISD::REGISTER_STORE, DL, MVT::Other, Chain, Value, Ptr,
-    DAG.getTargetConstant(0, MVT::i32)); // Channel
+    DAG.getTargetConstant(0, DL, MVT::i32)); // Channel
   }
 
   return Chain;
@@ -1472,27 +1462,23 @@ SDValue R600TargetLowering::LowerLOAD(SDValue Op, SelectionDAG &DAG) const
   SDValue Ptr = Op.getOperand(1);
   SDValue LoweredLoad;
 
-  SDValue Ret = AMDGPUTargetLowering::LowerLOAD(Op, DAG);
-  if (Ret.getNode()) {
-    SDValue Ops[2] = {
-      Ret,
-      Chain
-    };
-    return DAG.getMergeValues(Ops, DL);
-  }
+  if (SDValue Ret = AMDGPUTargetLowering::LowerLOAD(Op, DAG))
+    return Ret;
 
   // Lower loads constant address space global variable loads
   if (LoadNode->getAddressSpace() == AMDGPUAS::CONSTANT_ADDRESS &&
       isa<GlobalVariable>(GetUnderlyingObject(
-          LoadNode->getMemOperand()->getValue(), *getDataLayout()))) {
+          LoadNode->getMemOperand()->getValue(), DAG.getDataLayout()))) {
 
-    SDValue Ptr = DAG.getZExtOrTrunc(LoadNode->getBasePtr(), DL,
-        getPointerTy(AMDGPUAS::PRIVATE_ADDRESS));
+    SDValue Ptr = DAG.getZExtOrTrunc(
+        LoadNode->getBasePtr(), DL,
+        getPointerTy(DAG.getDataLayout(), AMDGPUAS::PRIVATE_ADDRESS));
     Ptr = DAG.getNode(ISD::SRL, DL, MVT::i32, Ptr,
-        DAG.getConstant(2, MVT::i32));
+        DAG.getConstant(2, DL, MVT::i32));
     return DAG.getNode(AMDGPUISD::REGISTER_LOAD, DL, Op->getVTList(),
                        LoadNode->getChain(), Ptr,
-                       DAG.getTargetConstant(0, MVT::i32), Op.getOperand(2));
+                       DAG.getTargetConstant(0, DL, MVT::i32),
+                       Op.getOperand(2));
   }
 
   if (LoadNode->getAddressSpace() == AMDGPUAS::LOCAL_ADDRESS && VT.isVector()) {
@@ -1519,7 +1505,7 @@ SDValue R600TargetLowering::LowerLOAD(SDValue Op, SelectionDAG &DAG) const
         // Thus we add (((512 + (kc_bank << 12)) + chan ) * 4 here and
         // then div by 4 at the ISel step
         SDValue NewPtr = DAG.getNode(ISD::ADD, DL, Ptr.getValueType(), Ptr,
-            DAG.getConstant(4 * i + ConstantBlock * 16, MVT::i32));
+            DAG.getConstant(4 * i + ConstantBlock * 16, DL, MVT::i32));
         Slots[i] = DAG.getNode(AMDGPUISD::CONST_ADDRESS, DL, MVT::i32, NewPtr);
       }
       EVT NewVT = MVT::v4i32;
@@ -1533,15 +1519,16 @@ SDValue R600TargetLowering::LowerLOAD(SDValue Op, SelectionDAG &DAG) const
     } else {
       // non-constant ptr can't be folded, keeps it as a v4f32 load
       Result = DAG.getNode(AMDGPUISD::CONST_ADDRESS, DL, MVT::v4i32,
-          DAG.getNode(ISD::SRL, DL, MVT::i32, Ptr, DAG.getConstant(4, MVT::i32)),
-          DAG.getConstant(LoadNode->getAddressSpace() -
-                          AMDGPUAS::CONSTANT_BUFFER_0, MVT::i32)
+          DAG.getNode(ISD::SRL, DL, MVT::i32, Ptr,
+                      DAG.getConstant(4, DL, MVT::i32)),
+                      DAG.getConstant(LoadNode->getAddressSpace() -
+                                      AMDGPUAS::CONSTANT_BUFFER_0, DL, MVT::i32)
           );
     }
 
     if (!VT.isVector()) {
       Result = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, MVT::i32, Result,
-          DAG.getConstant(0, MVT::i32));
+                           DAG.getConstant(0, DL, MVT::i32));
     }
 
     SDValue MergedValues[2] = {
@@ -1561,18 +1548,16 @@ SDValue R600TargetLowering::LowerLOAD(SDValue Op, SelectionDAG &DAG) const
   if (LoadNode->getExtensionType() == ISD::SEXTLOAD) {
     EVT MemVT = LoadNode->getMemoryVT();
     assert(!MemVT.isVector() && (MemVT == MVT::i16 || MemVT == MVT::i8));
-    SDValue ShiftAmount =
-          DAG.getConstant(VT.getSizeInBits() - MemVT.getSizeInBits(), MVT::i32);
     SDValue NewLoad = DAG.getExtLoad(ISD::EXTLOAD, DL, VT, Chain, Ptr,
                                   LoadNode->getPointerInfo(), MemVT,
                                   LoadNode->isVolatile(),
                                   LoadNode->isNonTemporal(),
                                   LoadNode->isInvariant(),
                                   LoadNode->getAlignment());
-    SDValue Shl = DAG.getNode(ISD::SHL, DL, VT, NewLoad, ShiftAmount);
-    SDValue Sra = DAG.getNode(ISD::SRA, DL, VT, Shl, ShiftAmount);
+    SDValue Res = DAG.getNode(ISD::SIGN_EXTEND_INREG, DL, VT, NewLoad,
+                              DAG.getValueType(MemVT));
 
-    SDValue MergedValues[2] = { Sra, Chain };
+    SDValue MergedValues[2] = { Res, Chain };
     return DAG.getMergeValues(MergedValues, DL);
   }
 
@@ -1600,10 +1585,10 @@ SDValue R600TargetLowering::LowerLOAD(SDValue Op, SelectionDAG &DAG) const
       unsigned Channel, PtrIncr;
       getStackAddress(StackWidth, i, Channel, PtrIncr);
       Ptr = DAG.getNode(ISD::ADD, DL, MVT::i32, Ptr,
-                        DAG.getConstant(PtrIncr, MVT::i32));
+                        DAG.getConstant(PtrIncr, DL, MVT::i32));
       Loads[i] = DAG.getNode(AMDGPUISD::REGISTER_LOAD, DL, ElemVT,
                              Chain, Ptr,
-                             DAG.getTargetConstant(Channel, MVT::i32),
+                             DAG.getTargetConstant(Channel, DL, MVT::i32),
                              Op.getOperand(2));
     }
     for (unsigned i = NumElemVT; i < 4; ++i) {
@@ -1614,7 +1599,7 @@ SDValue R600TargetLowering::LowerLOAD(SDValue Op, SelectionDAG &DAG) const
   } else {
     LoweredLoad = DAG.getNode(AMDGPUISD::REGISTER_LOAD, DL, VT,
                               Chain, Ptr,
-                              DAG.getTargetConstant(0, MVT::i32), // Channel
+                              DAG.getTargetConstant(0, DL, MVT::i32), // Channel
                               Op.getOperand(2));
   }
 
@@ -1703,7 +1688,7 @@ SDValue R600TargetLowering::LowerFormalArguments(
 
     MachinePointerInfo PtrInfo(UndefValue::get(PtrTy), PartOffset - ValBase);
     SDValue Arg = DAG.getLoad(ISD::UNINDEXED, Ext, VT, DL, Chain,
-                              DAG.getConstant(Offset, MVT::i32),
+                              DAG.getConstant(Offset, DL, MVT::i32),
                               DAG.getUNDEF(MVT::i32),
                               PtrInfo,
                               MemVT, false, true, true, 4);
@@ -1715,7 +1700,8 @@ SDValue R600TargetLowering::LowerFormalArguments(
   return Chain;
 }
 
-EVT R600TargetLowering::getSetCCResultType(LLVMContext &, EVT VT) const {
+EVT R600TargetLowering::getSetCCResultType(const DataLayout &DL, LLVMContext &,
+                                           EVT VT) const {
    if (!VT.isVector())
      return MVT::i32;
    return VT.changeVectorElementTypeToInteger();
@@ -1804,7 +1790,8 @@ static SDValue ReorganizeVector(SelectionDAG &DAG, SDValue VectorEntry,
 
 
 SDValue R600TargetLowering::OptimizeSwizzle(SDValue BuildVector,
-SDValue Swz[4], SelectionDAG &DAG) const {
+                                            SDValue Swz[4], SelectionDAG &DAG,
+                                            SDLoc DL) const {
   assert(BuildVector.getOpcode() == ISD::BUILD_VECTOR);
   // Old -> New swizzle values
   DenseMap<unsigned, unsigned> SwizzleRemap;
@@ -1813,7 +1800,7 @@ SDValue Swz[4], SelectionDAG &DAG) const {
   for (unsigned i = 0; i < 4; i++) {
     unsigned Idx = cast<ConstantSDNode>(Swz[i])->getZExtValue();
     if (SwizzleRemap.find(Idx) != SwizzleRemap.end())
-      Swz[i] = DAG.getConstant(SwizzleRemap[Idx], MVT::i32);
+      Swz[i] = DAG.getConstant(SwizzleRemap[Idx], DL, MVT::i32);
   }
 
   SwizzleRemap.clear();
@@ -1821,7 +1808,7 @@ SDValue Swz[4], SelectionDAG &DAG) const {
   for (unsigned i = 0; i < 4; i++) {
     unsigned Idx = cast<ConstantSDNode>(Swz[i])->getZExtValue();
     if (SwizzleRemap.find(Idx) != SwizzleRemap.end())
-      Swz[i] = DAG.getConstant(SwizzleRemap[Idx], MVT::i32);
+      Swz[i] = DAG.getConstant(SwizzleRemap[Idx], DL, MVT::i32);
   }
 
   return BuildVector;
@@ -1867,11 +1854,12 @@ SDValue R600TargetLowering::PerformDAGCombine(SDNode *N,
       return SDValue();
     }
 
-    return DAG.getNode(ISD::SELECT_CC, SDLoc(N), N->getValueType(0),
+    SDLoc dl(N);
+    return DAG.getNode(ISD::SELECT_CC, dl, N->getValueType(0),
                            SelectCC.getOperand(0), // LHS
                            SelectCC.getOperand(1), // RHS
-                           DAG.getConstant(-1, MVT::i32), // True
-                           DAG.getConstant(0, MVT::i32),  // False
+                           DAG.getConstant(-1, dl, MVT::i32), // True
+                           DAG.getConstant(0, dl, MVT::i32),  // False
                            SelectCC.getOperand(4)); // CC
 
     break;
@@ -1948,6 +1936,7 @@ SDValue R600TargetLowering::PerformDAGCombine(SDNode *N,
             Arg->getOperand(0).getOperand(Element));
       }
     }
+    break;
   }
 
   case ISD::SELECT_CC: {
@@ -2014,7 +2003,7 @@ SDValue R600TargetLowering::PerformDAGCombine(SDNode *N,
       N->getOperand(7) // SWZ_W
     };
     SDLoc DL(N);
-    NewArgs[1] = OptimizeSwizzle(N->getOperand(1), &NewArgs[4], DAG);
+    NewArgs[1] = OptimizeSwizzle(N->getOperand(1), &NewArgs[4], DAG, DL);
     return DAG.getNode(AMDGPUISD::EXPORT, DL, N->getVTList(), NewArgs);
   }
   case AMDGPUISD::TEXTURE_FETCH: {
@@ -2043,9 +2032,9 @@ SDValue R600TargetLowering::PerformDAGCombine(SDNode *N,
       N->getOperand(17),
       N->getOperand(18),
     };
-    NewArgs[1] = OptimizeSwizzle(N->getOperand(1), &NewArgs[2], DAG);
-    return DAG.getNode(AMDGPUISD::TEXTURE_FETCH, SDLoc(N), N->getVTList(),
-        NewArgs);
+    SDLoc DL(N);
+    NewArgs[1] = OptimizeSwizzle(N->getOperand(1), &NewArgs[2], DAG, DL);
+    return DAG.getNode(AMDGPUISD::TEXTURE_FETCH, DL, N->getVTList(), NewArgs);
   }
   }
 
@@ -2064,13 +2053,13 @@ FoldOperand(SDNode *ParentNode, unsigned SrcIdx, SDValue &Src, SDValue &Neg,
     if (!Neg.getNode())
       return false;
     Src = Src.getOperand(0);
-    Neg = DAG.getTargetConstant(1, MVT::i32);
+    Neg = DAG.getTargetConstant(1, SDLoc(ParentNode), MVT::i32);
     return true;
   case AMDGPU::FABS_R600:
     if (!Abs.getNode())
       return false;
     Src = Src.getOperand(0);
-    Abs = DAG.getTargetConstant(1, MVT::i32);
+    Abs = DAG.getTargetConstant(1, SDLoc(ParentNode), MVT::i32);
     return true;
   case AMDGPU::CONST_COPY: {
     unsigned Opcode = ParentNode->getMachineOpcode();
@@ -2166,7 +2155,7 @@ FoldOperand(SDNode *ParentNode, unsigned SrcIdx, SDValue &Src, SDValue &Neg,
       assert(C);
       if (C->getZExtValue())
         return false;
-      Imm = DAG.getTargetConstant(ImmValue, MVT::i32);
+      Imm = DAG.getTargetConstant(ImmValue, SDLoc(ParentNode), MVT::i32);
     }
     Src = DAG.getRegister(ImmReg, MVT::i32);
     return true;
@@ -2249,10 +2238,11 @@ SDNode *R600TargetLowering::PostISelFolding(MachineSDNode *Node,
         AMDGPU::OpName::clamp);
     if (ClampIdx < 0)
       return Node;
+    SDLoc DL(Node);
     std::vector<SDValue> Ops(Src->op_begin(), Src->op_end());
-    Ops[ClampIdx - 1] = DAG.getTargetConstant(1, MVT::i32);
-    return DAG.getMachineNode(Src.getMachineOpcode(), SDLoc(Node),
-        Node->getVTList(), Ops);
+    Ops[ClampIdx - 1] = DAG.getTargetConstant(1, DL, MVT::i32);
+    return DAG.getMachineNode(Src.getMachineOpcode(), DL,
+                              Node->getVTList(), Ops);
   } else {
     if (!TII->hasInstrModifiers(Opcode))
       return Node;
diff --git a/lib/Target/R600/R600ISelLowering.h b/lib/Target/AMDGPU/R600ISelLowering.h
similarity index 91%
rename from lib/Target/R600/R600ISelLowering.h
rename to lib/Target/AMDGPU/R600ISelLowering.h
index c54719574f9..4dbac97af2a 100644
--- a/lib/Target/R600/R600ISelLowering.h
+++ b/lib/Target/AMDGPU/R600ISelLowering.h
@@ -38,7 +38,9 @@ class R600TargetLowering : public AMDGPUTargetLowering {
                               const SmallVectorImpl<ISD::InputArg> &Ins,
                               SDLoc DL, SelectionDAG &DAG,
                               SmallVectorImpl<SDValue> &InVals) const override;
-  EVT getSetCCResultType(LLVMContext &, EVT VT) const override;
+  EVT getSetCCResultType(const DataLayout &DL, LLVMContext &,
+                         EVT VT) const override;
+
 private:
   unsigned Gen;
   /// Each OpenCL kernel has nine implicit parameters that are stored in the
@@ -50,7 +52,8 @@ class R600TargetLowering : public AMDGPUTargetLowering {
 
   void lowerImplicitParameter(MachineInstr *MI, MachineBasicBlock &BB,
       MachineRegisterInfo & MRI, unsigned dword_offset) const;
-  SDValue OptimizeSwizzle(SDValue BuildVector, SDValue Swz[], SelectionDAG &DAG) const;
+  SDValue OptimizeSwizzle(SDValue BuildVector, SDValue Swz[], SelectionDAG &DAG,
+                          SDLoc DL) const;
   SDValue vectorToVerticalVector(SelectionDAG &DAG, SDValue Vector) const;
 
   SDValue LowerEXTRACT_VECTOR_ELT(SDValue Op, SelectionDAG &DAG) const;
@@ -63,6 +66,8 @@ class R600TargetLowering : public AMDGPUTargetLowering {
   SDValue LowerTrig(SDValue Op, SelectionDAG &DAG) const;
   SDValue LowerSHLParts(SDValue Op, SelectionDAG &DAG) const;
   SDValue LowerSRXParts(SDValue Op, SelectionDAG &DAG) const;
+  SDValue LowerUADDSUBO(SDValue Op, SelectionDAG &DAG,
+                        unsigned mainop, unsigned ovf) const;
 
   SDValue stackPtrToRegIndex(SDValue Ptr, unsigned StackWidth,
                                           SelectionDAG &DAG) const;
diff --git a/lib/Target/R600/R600InstrFormats.td b/lib/Target/AMDGPU/R600InstrFormats.td
similarity index 100%
rename from lib/Target/R600/R600InstrFormats.td
rename to lib/Target/AMDGPU/R600InstrFormats.td
diff --git a/lib/Target/R600/R600InstrInfo.cpp b/lib/Target/AMDGPU/R600InstrInfo.cpp
similarity index 98%
rename from lib/Target/R600/R600InstrInfo.cpp
rename to lib/Target/AMDGPU/R600InstrInfo.cpp
index 5f0bdf34815..855fa9fe45b 100644
--- a/lib/Target/R600/R600InstrInfo.cpp
+++ b/lib/Target/AMDGPU/R600InstrInfo.cpp
@@ -354,7 +354,7 @@ R600InstrInfo::ExtractSrcs(MachineInstr *MI,
                            const DenseMap<unsigned, unsigned> &PV,
                            unsigned &ConstCount) const {
   ConstCount = 0;
-  const SmallVector<std::pair<MachineOperand *, int64_t>, 3> Srcs = getSrcs(MI);
+  ArrayRef<std::pair<MachineOperand *, int64_t>> Srcs = getSrcs(MI);
   const std::pair<int, unsigned> DummyPair(-1, 0);
   std::vector<std::pair<int, unsigned> > Result;
   unsigned i = 0;
@@ -628,8 +628,7 @@ R600InstrInfo::fitsConstReadLimitations(const std::vector<MachineInstr *> &MIs)
     if (!isALUInstr(MI->getOpcode()))
       continue;
 
-    const SmallVectorImpl<std::pair<MachineOperand *, int64_t> > &Srcs =
-        getSrcs(MI);
+    ArrayRef<std::pair<MachineOperand *, int64_t>> Srcs = getSrcs(MI);
 
     for (unsigned j = 0, e = Srcs.size(); j < e; j++) {
       std::pair<MachineOperand *, unsigned> Src = Srcs[j];
@@ -698,15 +697,10 @@ R600InstrInfo::AnalyzeBranch(MachineBasicBlock &MBB,
   // Most of the following comes from the ARM implementation of AnalyzeBranch
 
   // If the block has no terminators, it just falls into the block after it.
-  MachineBasicBlock::iterator I = MBB.end();
-  if (I == MBB.begin())
+  MachineBasicBlock::iterator I = MBB.getLastNonDebugInstr();
+  if (I == MBB.end())
     return false;
-  --I;
-  while (I->isDebugValue()) {
-    if (I == MBB.begin())
-      return false;
-    --I;
-  }
+
   // AMDGPU::BRANCH* instructions are only available after isel and are not
   // handled
   if (isBranch(I->getOpcode()))
@@ -782,7 +776,7 @@ unsigned
 R600InstrInfo::InsertBranch(MachineBasicBlock &MBB,
                             MachineBasicBlock *TBB,
                             MachineBasicBlock *FBB,
-                            const SmallVectorImpl<MachineOperand> &Cond,
+                            ArrayRef<MachineOperand> Cond,
                             DebugLoc DL) const {
   assert(TBB && "InsertBranch must not be told to insert a fallthrough");
 
@@ -1000,15 +994,15 @@ R600InstrInfo::DefinesPredicate(MachineInstr *MI,
 
 
 bool
-R600InstrInfo::SubsumesPredicate(const SmallVectorImpl<MachineOperand> &Pred1,
-                       const SmallVectorImpl<MachineOperand> &Pred2) const {
+R600InstrInfo::SubsumesPredicate(ArrayRef<MachineOperand> Pred1,
+                                 ArrayRef<MachineOperand> Pred2) const {
   return false;
 }
 
 
 bool
 R600InstrInfo::PredicateInstruction(MachineInstr *MI,
-                      const SmallVectorImpl<MachineOperand> &Pred) const {
+                                    ArrayRef<MachineOperand> Pred) const {
   int PIdx = MI->findFirstPredOperandIdx();
 
   if (MI->getOpcode() == AMDGPU::CF_ALU) {
diff --git a/lib/Target/R600/R600InstrInfo.h b/lib/Target/AMDGPU/R600InstrInfo.h
similarity index 97%
rename from lib/Target/R600/R600InstrInfo.h
rename to lib/Target/AMDGPU/R600InstrInfo.h
index d3dc0e58daa..dee4c2b9ae3 100644
--- a/lib/Target/R600/R600InstrInfo.h
+++ b/lib/Target/AMDGPU/R600InstrInfo.h
@@ -162,7 +162,9 @@ namespace llvm {
   bool AnalyzeBranch(MachineBasicBlock &MBB, MachineBasicBlock *&TBB, MachineBasicBlock *&FBB,
                      SmallVectorImpl<MachineOperand> &Cond, bool AllowModify) const override;
 
-  unsigned InsertBranch(MachineBasicBlock &MBB, MachineBasicBlock *TBB, MachineBasicBlock *FBB, const SmallVectorImpl<MachineOperand> &Cond, DebugLoc DL) const override;
+  unsigned InsertBranch(MachineBasicBlock &MBB, MachineBasicBlock *TBB,
+                        MachineBasicBlock *FBB, ArrayRef<MachineOperand> Cond,
+                        DebugLoc DL) const override;
 
   unsigned RemoveBranch(MachineBasicBlock &MBB) const override;
 
@@ -188,14 +190,14 @@ namespace llvm {
   bool DefinesPredicate(MachineInstr *MI,
                                   std::vector<MachineOperand> &Pred) const override;
 
-  bool SubsumesPredicate(const SmallVectorImpl<MachineOperand> &Pred1,
-                         const SmallVectorImpl<MachineOperand> &Pred2) const override;
+  bool SubsumesPredicate(ArrayRef<MachineOperand> Pred1,
+                         ArrayRef<MachineOperand> Pred2) const override;
 
   bool isProfitableToUnpredicate(MachineBasicBlock &TMBB,
                                           MachineBasicBlock &FMBB) const override;
 
   bool PredicateInstruction(MachineInstr *MI,
-                        const SmallVectorImpl<MachineOperand> &Pred) const override;
+                            ArrayRef<MachineOperand> Pred) const override;
 
   unsigned int getPredicationCost(const MachineInstr *) const override;
 
diff --git a/lib/Target/R600/R600Instructions.td b/lib/Target/AMDGPU/R600Instructions.td
similarity index 99%
rename from lib/Target/R600/R600Instructions.td
rename to lib/Target/AMDGPU/R600Instructions.td
index 7126c82c033..7beed092b3f 100644
--- a/lib/Target/R600/R600Instructions.td
+++ b/lib/Target/AMDGPU/R600Instructions.td
@@ -781,10 +781,10 @@ def XOR_INT : R600_2OP_Helper <0x32, "XOR_INT", xor>;
 def NOT_INT : R600_1OP_Helper <0x33, "NOT_INT", not>;
 def ADD_INT : R600_2OP_Helper <0x34, "ADD_INT", add>;
 def SUB_INT : R600_2OP_Helper <0x35, "SUB_INT", sub>;
-def MAX_INT : R600_2OP_Helper <0x36, "MAX_INT", AMDGPUsmax>;
-def MIN_INT : R600_2OP_Helper <0x37, "MIN_INT", AMDGPUsmin>;
-def MAX_UINT : R600_2OP_Helper <0x38, "MAX_UINT", AMDGPUumax>;
-def MIN_UINT : R600_2OP_Helper <0x39, "MIN_UINT", AMDGPUumin>;
+def MAX_INT : R600_2OP_Helper <0x36, "MAX_INT", smax>;
+def MIN_INT : R600_2OP_Helper <0x37, "MIN_INT", smin>;
+def MAX_UINT : R600_2OP_Helper <0x38, "MAX_UINT", umax>;
+def MIN_UINT : R600_2OP_Helper <0x39, "MIN_UINT", umin>;
 
 def SETE_INT : R600_2OP <
   0x3A, "SETE_INT",
diff --git a/lib/Target/R600/R600Intrinsics.td b/lib/Target/AMDGPU/R600Intrinsics.td
similarity index 100%
rename from lib/Target/R600/R600Intrinsics.td
rename to lib/Target/AMDGPU/R600Intrinsics.td
diff --git a/lib/Target/R600/R600MachineFunctionInfo.cpp b/lib/Target/AMDGPU/R600MachineFunctionInfo.cpp
similarity index 100%
rename from lib/Target/R600/R600MachineFunctionInfo.cpp
rename to lib/Target/AMDGPU/R600MachineFunctionInfo.cpp
diff --git a/lib/Target/R600/R600MachineFunctionInfo.h b/lib/Target/AMDGPU/R600MachineFunctionInfo.h
similarity index 100%
rename from lib/Target/R600/R600MachineFunctionInfo.h
rename to lib/Target/AMDGPU/R600MachineFunctionInfo.h
diff --git a/lib/Target/R600/R600MachineScheduler.cpp b/lib/Target/AMDGPU/R600MachineScheduler.cpp
similarity index 100%
rename from lib/Target/R600/R600MachineScheduler.cpp
rename to lib/Target/AMDGPU/R600MachineScheduler.cpp
diff --git a/lib/Target/R600/R600MachineScheduler.h b/lib/Target/AMDGPU/R600MachineScheduler.h
similarity index 100%
rename from lib/Target/R600/R600MachineScheduler.h
rename to lib/Target/AMDGPU/R600MachineScheduler.h
diff --git a/lib/Target/R600/R600OptimizeVectorRegisters.cpp b/lib/Target/AMDGPU/R600OptimizeVectorRegisters.cpp
similarity index 100%
rename from lib/Target/R600/R600OptimizeVectorRegisters.cpp
rename to lib/Target/AMDGPU/R600OptimizeVectorRegisters.cpp
diff --git a/lib/Target/R600/R600Packetizer.cpp b/lib/Target/AMDGPU/R600Packetizer.cpp
similarity index 100%
rename from lib/Target/R600/R600Packetizer.cpp
rename to lib/Target/AMDGPU/R600Packetizer.cpp
diff --git a/lib/Target/R600/R600RegisterInfo.cpp b/lib/Target/AMDGPU/R600RegisterInfo.cpp
similarity index 100%
rename from lib/Target/R600/R600RegisterInfo.cpp
rename to lib/Target/AMDGPU/R600RegisterInfo.cpp
diff --git a/lib/Target/R600/R600RegisterInfo.h b/lib/Target/AMDGPU/R600RegisterInfo.h
similarity index 100%
rename from lib/Target/R600/R600RegisterInfo.h
rename to lib/Target/AMDGPU/R600RegisterInfo.h
diff --git a/lib/Target/R600/R600RegisterInfo.td b/lib/Target/AMDGPU/R600RegisterInfo.td
similarity index 100%
rename from lib/Target/R600/R600RegisterInfo.td
rename to lib/Target/AMDGPU/R600RegisterInfo.td
diff --git a/lib/Target/R600/R600Schedule.td b/lib/Target/AMDGPU/R600Schedule.td
similarity index 100%
rename from lib/Target/R600/R600Schedule.td
rename to lib/Target/AMDGPU/R600Schedule.td
diff --git a/lib/Target/R600/R600TextureIntrinsicsReplacer.cpp b/lib/Target/AMDGPU/R600TextureIntrinsicsReplacer.cpp
similarity index 100%
rename from lib/Target/R600/R600TextureIntrinsicsReplacer.cpp
rename to lib/Target/AMDGPU/R600TextureIntrinsicsReplacer.cpp
diff --git a/lib/Target/R600/R700Instructions.td b/lib/Target/AMDGPU/R700Instructions.td
similarity index 100%
rename from lib/Target/R600/R700Instructions.td
rename to lib/Target/AMDGPU/R700Instructions.td
diff --git a/lib/Target/R600/SIAnnotateControlFlow.cpp b/lib/Target/AMDGPU/SIAnnotateControlFlow.cpp
similarity index 96%
rename from lib/Target/R600/SIAnnotateControlFlow.cpp
rename to lib/Target/AMDGPU/SIAnnotateControlFlow.cpp
index d39ab3f1850..ec7d65d5743 100644
--- a/lib/Target/R600/SIAnnotateControlFlow.cpp
+++ b/lib/Target/AMDGPU/SIAnnotateControlFlow.cpp
@@ -209,7 +209,15 @@ void SIAnnotateControlFlow::insertElse(BranchInst *Term) {
 /// \brief Recursively handle the condition leading to a loop
 Value *SIAnnotateControlFlow::handleLoopCondition(Value *Cond, PHINode *Broken,
                                                   llvm::Loop *L) {
-  if (PHINode *Phi = dyn_cast<PHINode>(Cond)) {
+
+  // Only search through PHI nodes which are inside the loop.  If we try this
+  // with PHI nodes that are outside of the loop, we end up inserting new PHI
+  // nodes outside of the loop which depend on values defined inside the loop.
+  // This will break the module with
+  // 'Instruction does not dominate all users!' errors.
+  PHINode *Phi = nullptr;
+  if ((Phi = dyn_cast<PHINode>(Cond)) && L->contains(Phi)) {
+
     BasicBlock *Parent = Phi->getParent();
     PHINode *NewPhi = PHINode::Create(Int64, 0, "", &Parent->front());
     Value *Ret = NewPhi;
@@ -304,8 +312,7 @@ void SIAnnotateControlFlow::closeControlFlow(BasicBlock *BB) {
       if (std::find(Latches.begin(), Latches.end(), *PI) == Latches.end())
         Preds.push_back(*PI);
     }
-    BB = llvm::SplitBlockPredecessors(BB, Preds, "endcf.split", nullptr, DT,
-                                      LI, false);
+    BB = llvm::SplitBlockPredecessors(BB, Preds, "endcf.split", DT, LI, false);
   }
 
   CallInst::Create(EndCf, popSaved(), "", BB->getFirstInsertionPt());
diff --git a/lib/Target/R600/SIDefines.h b/lib/Target/AMDGPU/SIDefines.h
similarity index 78%
rename from lib/Target/R600/SIDefines.h
rename to lib/Target/AMDGPU/SIDefines.h
index b54014072bf..4c3263911c4 100644
--- a/lib/Target/R600/SIDefines.h
+++ b/lib/Target/AMDGPU/SIDefines.h
@@ -36,7 +36,8 @@ enum {
   DS = 1 << 17,
   MIMG = 1 << 18,
   FLAT = 1 << 19,
-  WQM = 1 << 20
+  WQM = 1 << 20,
+  VGPRSpill = 1 << 21
 };
 }
 
@@ -99,16 +100,41 @@ namespace SIOutMods {
 #define R_00B848_COMPUTE_PGM_RSRC1                                      0x00B848
 #define   S_00B028_VGPRS(x)                                           (((x) & 0x3F) << 0)
 #define   S_00B028_SGPRS(x)                                           (((x) & 0x0F) << 6)
+
 #define R_00B84C_COMPUTE_PGM_RSRC2                                      0x00B84C
 #define   S_00B84C_SCRATCH_EN(x)                                      (((x) & 0x1) << 0)
+#define   G_00B84C_SCRATCH_EN(x)                                      (((x) >> 0) & 0x1)
+#define   C_00B84C_SCRATCH_EN                                         0xFFFFFFFE
 #define   S_00B84C_USER_SGPR(x)                                       (((x) & 0x1F) << 1)
+#define   G_00B84C_USER_SGPR(x)                                       (((x) >> 1) & 0x1F)
+#define   C_00B84C_USER_SGPR                                          0xFFFFFFC1
 #define   S_00B84C_TGID_X_EN(x)                                       (((x) & 0x1) << 7)
+#define   G_00B84C_TGID_X_EN(x)                                       (((x) >> 7) & 0x1)
+#define   C_00B84C_TGID_X_EN                                          0xFFFFFF7F
 #define   S_00B84C_TGID_Y_EN(x)                                       (((x) & 0x1) << 8)
+#define   G_00B84C_TGID_Y_EN(x)                                       (((x) >> 8) & 0x1)
+#define   C_00B84C_TGID_Y_EN                                          0xFFFFFEFF
 #define   S_00B84C_TGID_Z_EN(x)                                       (((x) & 0x1) << 9)
+#define   G_00B84C_TGID_Z_EN(x)                                       (((x) >> 9) & 0x1)
+#define   C_00B84C_TGID_Z_EN                                          0xFFFFFDFF
 #define   S_00B84C_TG_SIZE_EN(x)                                      (((x) & 0x1) << 10)
+#define   G_00B84C_TG_SIZE_EN(x)                                      (((x) >> 10) & 0x1)
+#define   C_00B84C_TG_SIZE_EN                                         0xFFFFFBFF
 #define   S_00B84C_TIDIG_COMP_CNT(x)                                  (((x) & 0x03) << 11)
-
+#define   G_00B84C_TIDIG_COMP_CNT(x)                                  (((x) >> 11) & 0x03)
+#define   C_00B84C_TIDIG_COMP_CNT                                     0xFFFFE7FF
+/* CIK */
+#define   S_00B84C_EXCP_EN_MSB(x)                                     (((x) & 0x03) << 13)
+#define   G_00B84C_EXCP_EN_MSB(x)                                     (((x) >> 13) & 0x03)
+#define   C_00B84C_EXCP_EN_MSB                                        0xFFFF9FFF
+/*     */
 #define   S_00B84C_LDS_SIZE(x)                                        (((x) & 0x1FF) << 15)
+#define   G_00B84C_LDS_SIZE(x)                                        (((x) >> 15) & 0x1FF)
+#define   C_00B84C_LDS_SIZE                                           0xFF007FFF
+#define   S_00B84C_EXCP_EN(x)                                         (((x) & 0x7F) << 24)
+#define   G_00B84C_EXCP_EN(x)                                         (((x) >> 24) & 0x7F)
+#define   C_00B84C_EXCP_EN 
+
 #define R_0286CC_SPI_PS_INPUT_ENA                                       0x0286CC
 
 
diff --git a/lib/Target/AMDGPU/SIFixControlFlowLiveIntervals.cpp b/lib/Target/AMDGPU/SIFixControlFlowLiveIntervals.cpp
new file mode 100644
index 00000000000..5fe8d19426d
--- /dev/null
+++ b/lib/Target/AMDGPU/SIFixControlFlowLiveIntervals.cpp
@@ -0,0 +1,96 @@
+//===-- SIFixControlFlowLiveIntervals.cpp - Fix CF live intervals ---------===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+/// \file
+/// \brief Spilling of EXEC masks used for control flow messes up control flow
+/// lowering, so mark all live intervals associated with CF instructions as
+/// non-spillable.
+///
+//===----------------------------------------------------------------------===//
+
+#include "AMDGPU.h"
+#include "SIInstrInfo.h"
+#include "SIRegisterInfo.h"
+#include "llvm/CodeGen/LiveIntervalAnalysis.h"
+#include "llvm/CodeGen/MachineFunctionPass.h"
+#include "llvm/CodeGen/MachineInstrBuilder.h"
+#include "llvm/CodeGen/MachinePostDominators.h"
+#include "llvm/CodeGen/MachineRegisterInfo.h"
+#include "llvm/Support/Debug.h"
+#include "llvm/Support/raw_ostream.h"
+#include "llvm/Target/TargetMachine.h"
+
+using namespace llvm;
+
+#define DEBUG_TYPE "si-fix-cf-live-intervals"
+
+namespace {
+
+class SIFixControlFlowLiveIntervals : public MachineFunctionPass {
+public:
+  static char ID;
+
+public:
+  SIFixControlFlowLiveIntervals() : MachineFunctionPass(ID) {
+    initializeSIFixControlFlowLiveIntervalsPass(*PassRegistry::getPassRegistry());
+  }
+
+  bool runOnMachineFunction(MachineFunction &MF) override;
+
+  const char *getPassName() const override {
+    return "SI Fix CF Live Intervals";
+  }
+
+  void getAnalysisUsage(AnalysisUsage &AU) const override {
+    AU.addRequired<LiveIntervals>();
+    AU.setPreservesAll();
+    MachineFunctionPass::getAnalysisUsage(AU);
+  }
+};
+
+} // End anonymous namespace.
+
+INITIALIZE_PASS_BEGIN(SIFixControlFlowLiveIntervals, DEBUG_TYPE,
+                      "SI Fix CF Live Intervals", false, false)
+INITIALIZE_PASS_DEPENDENCY(LiveIntervals)
+INITIALIZE_PASS_END(SIFixControlFlowLiveIntervals, DEBUG_TYPE,
+                    "SI Fix CF Live Intervals", false, false)
+
+char SIFixControlFlowLiveIntervals::ID = 0;
+
+char &llvm::SIFixControlFlowLiveIntervalsID = SIFixControlFlowLiveIntervals::ID;
+
+FunctionPass *llvm::createSIFixControlFlowLiveIntervalsPass() {
+  return new SIFixControlFlowLiveIntervals();
+}
+
+bool SIFixControlFlowLiveIntervals::runOnMachineFunction(MachineFunction &MF) {
+  LiveIntervals *LIS = &getAnalysis<LiveIntervals>();
+
+  for (const MachineBasicBlock &MBB : MF) {
+    for (const MachineInstr &MI : MBB) {
+      switch (MI.getOpcode()) {
+        case AMDGPU::SI_IF:
+        case AMDGPU::SI_ELSE:
+        case AMDGPU::SI_BREAK:
+        case AMDGPU::SI_IF_BREAK:
+        case AMDGPU::SI_ELSE_BREAK:
+        case AMDGPU::SI_END_CF: {
+          unsigned Reg = MI.getOperand(0).getReg();
+          LIS->getInterval(Reg).markNotSpillable();
+          break;
+        }
+        default:
+          break;
+      }
+    }
+  }
+
+  return false;
+}
diff --git a/lib/Target/R600/SIFixSGPRCopies.cpp b/lib/Target/AMDGPU/SIFixSGPRCopies.cpp
similarity index 97%
rename from lib/Target/R600/SIFixSGPRCopies.cpp
rename to lib/Target/AMDGPU/SIFixSGPRCopies.cpp
index cd1b3acc5c8..23502b45905 100644
--- a/lib/Target/R600/SIFixSGPRCopies.cpp
+++ b/lib/Target/AMDGPU/SIFixSGPRCopies.cpp
@@ -140,7 +140,7 @@ const TargetRegisterClass *SIFixSGPRCopies::inferRegClassFromUses(
   const TargetRegisterClass *RC
     = TargetRegisterInfo::isVirtualRegister(Reg) ?
     MRI.getRegClass(Reg) :
-    TRI->getRegClass(Reg);
+    TRI->getPhysRegClass(Reg);
 
   RC = TRI->getSubRegClass(RC, SubReg);
   for (MachineRegisterInfo::use_instr_iterator
@@ -183,15 +183,17 @@ bool SIFixSGPRCopies::isVGPRToSGPRCopy(const MachineInstr &Copy,
   unsigned SrcReg = Copy.getOperand(1).getReg();
   unsigned SrcSubReg = Copy.getOperand(1).getSubReg();
 
-  const TargetRegisterClass *DstRC
-    = TargetRegisterInfo::isVirtualRegister(DstReg) ?
-    MRI.getRegClass(DstReg) :
-    TRI->getRegClass(DstReg);
+  if (!TargetRegisterInfo::isVirtualRegister(DstReg)) {
+    // If the destination register is a physical register there isn't really
+    // much we can do to fix this.
+    return false;
+  }
+
+  const TargetRegisterClass *DstRC = MRI.getRegClass(DstReg);
 
   const TargetRegisterClass *SrcRC;
 
   if (!TargetRegisterInfo::isVirtualRegister(SrcReg) ||
-      DstRC == &AMDGPU::M0RegRegClass ||
       MRI.getRegClass(SrcReg) == &AMDGPU::VReg_1RegClass)
     return false;
 
diff --git a/lib/Target/R600/SIFixSGPRLiveRanges.cpp b/lib/Target/AMDGPU/SIFixSGPRLiveRanges.cpp
similarity index 100%
rename from lib/Target/R600/SIFixSGPRLiveRanges.cpp
rename to lib/Target/AMDGPU/SIFixSGPRLiveRanges.cpp
diff --git a/lib/Target/R600/SIFoldOperands.cpp b/lib/Target/AMDGPU/SIFoldOperands.cpp
similarity index 88%
rename from lib/Target/R600/SIFoldOperands.cpp
rename to lib/Target/AMDGPU/SIFoldOperands.cpp
index 7ba5a6d7c38..c2887255cc1 100644
--- a/lib/Target/R600/SIFoldOperands.cpp
+++ b/lib/Target/AMDGPU/SIFoldOperands.cpp
@@ -126,11 +126,42 @@ static bool updateOperand(FoldCandidate &Fold,
   return false;
 }
 
+static bool isUseMIInFoldList(const std::vector<FoldCandidate> &FoldList,
+                              const MachineInstr *MI) {
+  for (auto Candidate : FoldList) {
+    if (Candidate.UseMI == MI)
+      return true;
+  }
+  return false;
+}
+
 static bool tryAddToFoldList(std::vector<FoldCandidate> &FoldList,
                              MachineInstr *MI, unsigned OpNo,
                              MachineOperand *OpToFold,
                              const SIInstrInfo *TII) {
   if (!TII->isOperandLegal(MI, OpNo, OpToFold)) {
+
+    // Special case for v_mac_f32_e64 if we are trying to fold into src2
+    unsigned Opc = MI->getOpcode();
+    if (Opc == AMDGPU::V_MAC_F32_e64 &&
+        (int)OpNo == AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src2)) {
+      // Check if changing this to a v_mad_f32 instruction will allow us to
+      // fold the operand.
+      MI->setDesc(TII->get(AMDGPU::V_MAD_F32));
+      bool FoldAsMAD = tryAddToFoldList(FoldList, MI, OpNo, OpToFold, TII);
+      if (FoldAsMAD) {
+        MI->untieRegOperand(OpNo);
+        return true;
+      }
+      MI->setDesc(TII->get(Opc));
+    }
+
+    // If we are already folding into another operand of MI, then
+    // we can't commute the instruction, otherwise we risk making the
+    // other fold illegal.
+    if (isUseMIInFoldList(FoldList, MI))
+      return false;
+
     // Operand is not legal, so try to commute the instruction to
     // see if this makes it possible to fold.
     unsigned CommuteIdx0;
@@ -216,7 +247,7 @@ bool SIFoldOperands::runOnMachineFunction(MachineFunction &MF) {
           const TargetRegisterClass *UseRC
             = TargetRegisterInfo::isVirtualRegister(UseReg) ?
             MRI.getRegClass(UseReg) :
-            TRI.getRegClass(UseReg);
+            TRI.getPhysRegClass(UseReg);
 
           Imm = APInt(64, OpToFold.getImm());
 
@@ -240,7 +271,7 @@ bool SIFoldOperands::runOnMachineFunction(MachineFunction &MF) {
             const TargetRegisterClass *DestRC
               = TargetRegisterInfo::isVirtualRegister(DestReg) ?
               MRI.getRegClass(DestReg) :
-              TRI.getRegClass(DestReg);
+              TRI.getPhysRegClass(DestReg);
 
             unsigned MovOp = TII->getMovOpcode(DestRC);
             if (MovOp == AMDGPU::COPY)
diff --git a/lib/Target/R600/SIISelLowering.cpp b/lib/Target/AMDGPU/SIISelLowering.cpp
similarity index 84%
rename from lib/Target/R600/SIISelLowering.cpp
rename to lib/Target/AMDGPU/SIISelLowering.cpp
index 43507d8bf33..25798c42a8c 100644
--- a/lib/Target/R600/SIISelLowering.cpp
+++ b/lib/Target/AMDGPU/SIISelLowering.cpp
@@ -155,9 +155,9 @@ SITargetLowering::SITargetLowering(TargetMachine &TM,
   for (MVT VT : MVT::fp_valuetypes())
     setLoadExtAction(ISD::EXTLOAD, VT, MVT::f32, Expand);
 
-  setTruncStoreAction(MVT::f64, MVT::f32, Expand);
   setTruncStoreAction(MVT::i64, MVT::i32, Expand);
   setTruncStoreAction(MVT::v8i32, MVT::v8i16, Expand);
+  setTruncStoreAction(MVT::v16i32, MVT::v16i8, Expand);
   setTruncStoreAction(MVT::v16i32, MVT::v16i16, Expand);
 
   setOperationAction(ISD::LOAD, MVT::i1, Custom);
@@ -211,6 +211,10 @@ SITargetLowering::SITargetLowering(TargetMachine &TM,
   setTargetDAGCombine(ISD::FSUB);
   setTargetDAGCombine(ISD::FMINNUM);
   setTargetDAGCombine(ISD::FMAXNUM);
+  setTargetDAGCombine(ISD::SMIN);
+  setTargetDAGCombine(ISD::SMAX);
+  setTargetDAGCombine(ISD::UMIN);
+  setTargetDAGCombine(ISD::UMAX);
   setTargetDAGCombine(ISD::SELECT_CC);
   setTargetDAGCombine(ISD::SETCC);
   setTargetDAGCombine(ISD::AND);
@@ -251,47 +255,101 @@ bool SITargetLowering::isShuffleMaskLegal(const SmallVectorImpl<int> &,
   return false;
 }
 
-// FIXME: This really needs an address space argument. The immediate offset
-// size is different for different sets of memory instruction sets.
+bool SITargetLowering::isLegalFlatAddressingMode(const AddrMode &AM) const {
+  // Flat instructions do not have offsets, and only have the register
+  // address.
+  return AM.BaseOffs == 0 && (AM.Scale == 0 || AM.Scale == 1);
+}
 
-// The single offset DS instructions have a 16-bit unsigned byte offset.
-//
-// MUBUF / MTBUF have a 12-bit unsigned byte offset, and additionally can do r +
-// r + i with addr64. 32-bit has more addressing mode options. Depending on the
-// resource constant, it can also do (i64 r0) + (i32 r1) * (i14 i).
-//
-// SMRD instructions have an 8-bit, dword offset.
-//
-bool SITargetLowering::isLegalAddressingMode(const AddrMode &AM,
-                                             Type *Ty) const {
+bool SITargetLowering::isLegalAddressingMode(const DataLayout &DL,
+                                             const AddrMode &AM, Type *Ty,
+                                             unsigned AS) const {
   // No global is ever allowed as a base.
   if (AM.BaseGV)
     return false;
 
-  // Allow a 16-bit unsigned immediate field, since this is what DS instructions
-  // use.
-  if (!isUInt<16>(AM.BaseOffs))
-    return false;
+  switch (AS) {
+  case AMDGPUAS::GLOBAL_ADDRESS:
+    if (Subtarget->getGeneration() >= AMDGPUSubtarget::VOLCANIC_ISLANDS) {
+      // Assume the we will use FLAT for all global memory accesses
+      // on VI.
+      // FIXME: This assumption is currently wrong.  On VI we still use
+      // MUBUF instructions for the r + i addressing mode.  As currently
+      // implemented, the MUBUF instructions only work on buffer < 4GB.
+      // It may be possible to support > 4GB buffers with MUBUF instructions,
+      // by setting the stride value in the resource descriptor which would
+      // increase the size limit to (stride * 4GB).  However, this is risky,
+      // because it has never been validated.
+      return isLegalFlatAddressingMode(AM);
+    }
+    // fall-through
+  case AMDGPUAS::PRIVATE_ADDRESS:
+  case AMDGPUAS::CONSTANT_ADDRESS: // XXX - Should we assume SMRD instructions?
+  case AMDGPUAS::UNKNOWN_ADDRESS_SPACE: {
+    // MUBUF / MTBUF instructions have a 12-bit unsigned byte offset, and
+    // additionally can do r + r + i with addr64. 32-bit has more addressing
+    // mode options. Depending on the resource constant, it can also do
+    // (i64 r0) + (i32 r1) * (i14 i).
+    //
+    // SMRD instructions have an 8-bit, dword offset.
+    //
+    // Assume nonunifom access, since the address space isn't enough to know
+    // what instruction we will use, and since we don't know if this is a load
+    // or store and scalar stores are only available on VI.
+    //
+    // We also know if we are doing an extload, we can't do a scalar load.
+    //
+    // Private arrays end up using a scratch buffer most of the time, so also
+    // assume those use MUBUF instructions. Scratch loads / stores are currently
+    // implemented as mubuf instructions with offen bit set, so slightly
+    // different than the normal addr64.
+    if (!isUInt<12>(AM.BaseOffs))
+      return false;
 
-  // Only support r+r,
-  switch (AM.Scale) {
-  case 0:  // "r+i" or just "i", depending on HasBaseReg.
-    break;
-  case 1:
-    if (AM.HasBaseReg && AM.BaseOffs)  // "r+r+i" is not allowed.
+    // FIXME: Since we can split immediate into soffset and immediate offset,
+    // would it make sense to allow any immediate?
+
+    switch (AM.Scale) {
+    case 0: // r + i or just i, depending on HasBaseReg.
+      return true;
+    case 1:
+      return true; // We have r + r or r + i.
+    case 2:
+      if (AM.HasBaseReg) {
+        // Reject 2 * r + r.
+        return false;
+      }
+
+      // Allow 2 * r as r + r
+      // Or  2 * r + i is allowed as r + r + i.
+      return true;
+    default: // Don't allow n * r
       return false;
-    // Otherwise we have r+r or r+i.
-    break;
-  case 2:
-    if (AM.HasBaseReg || AM.BaseOffs)  // 2*r+r  or  2*r+i is not allowed.
+    }
+  }
+  case AMDGPUAS::LOCAL_ADDRESS:
+  case AMDGPUAS::REGION_ADDRESS: {
+    // Basic, single offset DS instructions allow a 16-bit unsigned immediate
+    // field.
+    // XXX - If doing a 4-byte aligned 8-byte type access, we effectively have
+    // an 8-bit dword offset but we don't know the alignment here.
+    if (!isUInt<16>(AM.BaseOffs))
       return false;
-    // Allow 2*r as r+r.
-    break;
-  default: // Don't allow n * r
+
+    if (AM.Scale == 0) // r + i or just i, depending on HasBaseReg.
+      return true;
+
+    if (AM.Scale == 1 && AM.HasBaseReg)
+      return true;
+
     return false;
   }
+  case AMDGPUAS::FLAT_ADDRESS:
+    return isLegalFlatAddressingMode(AM);
 
-  return true;
+  default:
+    llvm_unreachable("unhandled address space");
+  }
 }
 
 bool SITargetLowering::allowsMisalignedMemoryAccesses(EVT VT,
@@ -368,10 +426,16 @@ bool SITargetLowering::shouldConvertConstantLoadToIntImm(const APInt &Imm,
   return TII->isInlineConstant(Imm);
 }
 
+static EVT toIntegerVT(EVT VT) {
+  if (VT.isVector())
+    return VT.changeVectorElementTypeToInteger();
+  return MVT::getIntegerVT(VT.getSizeInBits());
+}
+
 SDValue SITargetLowering::LowerParameter(SelectionDAG &DAG, EVT VT, EVT MemVT,
                                          SDLoc SL, SDValue Chain,
                                          unsigned Offset, bool Signed) const {
-  const DataLayout *DL = getDataLayout();
+  const DataLayout &DL = DAG.getDataLayout();
   MachineFunction &MF = DAG.getMachineFunction();
   const SIRegisterInfo *TRI =
       static_cast<const SIRegisterInfo*>(Subtarget->getRegisterInfo());
@@ -380,20 +444,47 @@ SDValue SITargetLowering::LowerParameter(SelectionDAG &DAG, EVT VT, EVT MemVT,
   Type *Ty = VT.getTypeForEVT(*DAG.getContext());
 
   MachineRegisterInfo &MRI = DAG.getMachineFunction().getRegInfo();
+  MVT PtrVT = getPointerTy(DL, AMDGPUAS::CONSTANT_ADDRESS);
   PointerType *PtrTy = PointerType::get(Ty, AMDGPUAS::CONSTANT_ADDRESS);
-  SDValue BasePtr =  DAG.getCopyFromReg(Chain, SL,
-                           MRI.getLiveInVirtReg(InputPtrReg), MVT::i64);
-  SDValue Ptr = DAG.getNode(ISD::ADD, SL, MVT::i64, BasePtr,
-                                             DAG.getConstant(Offset, MVT::i64));
-  SDValue PtrOffset = DAG.getUNDEF(getPointerTy(AMDGPUAS::CONSTANT_ADDRESS));
+  SDValue BasePtr = DAG.getCopyFromReg(Chain, SL,
+                                       MRI.getLiveInVirtReg(InputPtrReg), PtrVT);
+  SDValue Ptr = DAG.getNode(ISD::ADD, SL, PtrVT, BasePtr,
+                            DAG.getConstant(Offset, SL, PtrVT));
+  SDValue PtrOffset = DAG.getUNDEF(PtrVT);
   MachinePointerInfo PtrInfo(UndefValue::get(PtrTy));
 
-  return DAG.getLoad(ISD::UNINDEXED, Signed ? ISD::SEXTLOAD : ISD::ZEXTLOAD,
+  unsigned Align = DL.getABITypeAlignment(Ty);
+
+  if (VT != MemVT && VT.isFloatingPoint()) {
+    // Do an integer load and convert.
+    // FIXME: This is mostly because load legalization after type legalization
+    // doesn't handle FP extloads.
+    assert(VT.getScalarType() == MVT::f32 &&
+           MemVT.getScalarType() == MVT::f16);
+
+    EVT IVT = toIntegerVT(VT);
+    EVT MemIVT = toIntegerVT(MemVT);
+    SDValue Load = DAG.getLoad(ISD::UNINDEXED, ISD::ZEXTLOAD,
+                               IVT, SL, Chain, Ptr, PtrOffset, PtrInfo, MemIVT,
+                               false, // isVolatile
+                               true, // isNonTemporal
+                               true, // isInvariant
+                               Align); // Alignment
+    SDValue Ops[] = {
+      DAG.getNode(ISD::FP16_TO_FP, SL, VT, Load),
+      Load.getValue(1)
+    };
+
+    return DAG.getMergeValues(Ops, SL);
+  }
+
+  ISD::LoadExtType ExtTy = Signed ? ISD::SEXTLOAD : ISD::ZEXTLOAD;
+  return DAG.getLoad(ISD::UNINDEXED, ExtTy,
                      VT, SL, Chain, Ptr, PtrOffset, PtrInfo, MemVT,
                      false, // isVolatile
                      true, // isNonTemporal
                      true, // isInvariant
-                     DL->getABITypeAlignment(Ty)); // Alignment
+                     Align); // Alignment
 }
 
 SDValue SITargetLowering::LowerFormalArguments(
@@ -502,6 +593,8 @@ SDValue SITargetLowering::LowerFormalArguments(
 
   AnalyzeFormalArguments(CCInfo, Splits);
 
+  SmallVector<SDValue, 16> Chains;
+
   for (unsigned i = 0, e = Ins.size(), ArgIdx = 0; i != e; ++i) {
 
     const ISD::InputArg &Arg = Ins[i];
@@ -516,11 +609,13 @@ SDValue SITargetLowering::LowerFormalArguments(
     if (VA.isMemLoc()) {
       VT = Ins[i].VT;
       EVT MemVT = Splits[i].VT;
-      const unsigned Offset = 36 + VA.getLocMemOffset();
+      const unsigned Offset = Subtarget->getExplicitKernelArgOffset() +
+                              VA.getLocMemOffset();
       // The first 36 bytes of the input buffer contains information about
       // thread group and global sizes.
-      SDValue Arg = LowerParameter(DAG, VT, MemVT,  DL, DAG.getRoot(),
+      SDValue Arg = LowerParameter(DAG, VT, MemVT,  DL, Chain,
                                    Offset, Ins[i].Flags.isSExt());
+      Chains.push_back(Arg.getValue(1));
 
       const PointerType *ParamTy =
         dyn_cast<PointerType>(FType->getParamType(Ins[i].getOrigArgIndex()));
@@ -546,7 +641,8 @@ SDValue SITargetLowering::LowerFormalArguments(
       Reg = TRI->getMatchingSuperReg(Reg, AMDGPU::sub0,
                                      &AMDGPU::SReg_64RegClass);
       Reg = MF.addLiveIn(Reg, &AMDGPU::SReg_64RegClass);
-      InVals.push_back(DAG.getCopyFromReg(Chain, DL, Reg, VT));
+      SDValue Copy = DAG.getCopyFromReg(Chain, DL, Reg, VT);
+      InVals.push_back(Copy);
       continue;
     }
 
@@ -566,7 +662,9 @@ SDValue SITargetLowering::LowerFormalArguments(
       for (unsigned j = 1; j != NumElements; ++j) {
         Reg = ArgLocs[ArgIdx++].getLocReg();
         Reg = MF.addLiveIn(Reg, RC);
-        Regs.push_back(DAG.getCopyFromReg(Chain, DL, Reg, VT));
+
+        SDValue Copy = DAG.getCopyFromReg(Chain, DL, Reg, VT);
+        Regs.push_back(Copy);
       }
 
       // Fill up the missing vector elements
@@ -585,7 +683,11 @@ SDValue SITargetLowering::LowerFormalArguments(
         AMDGPU::SGPR_32RegClass.begin(), AMDGPU::SGPR_32RegClass.getNumRegs()));
     Info->ScratchOffsetReg = AMDGPU::SGPR_32RegClass.getRegister(ScratchIdx);
   }
-  return Chain;
+
+  if (Chains.empty())
+    return Chain;
+
+  return DAG.getNode(ISD::TokenFactor, DL, MVT::Other, Chains);
 }
 
 MachineBasicBlock * SITargetLowering::EmitInstrWithCustomInserter(
@@ -627,14 +729,15 @@ bool SITargetLowering::enableAggressiveFMAFusion(EVT VT) const {
   return true;
 }
 
-EVT SITargetLowering::getSetCCResultType(LLVMContext &Ctx, EVT VT) const {
+EVT SITargetLowering::getSetCCResultType(const DataLayout &DL, LLVMContext &Ctx,
+                                         EVT VT) const {
   if (!VT.isVector()) {
     return MVT::i1;
   }
   return EVT::getVectorVT(Ctx, MVT::i1, VT.getVectorNumElements());
 }
 
-MVT SITargetLowering::getScalarShiftAmountTy(EVT VT) const {
+MVT SITargetLowering::getScalarShiftAmountTy(const DataLayout &, EVT) const {
   return MVT::i32;
 }
 
@@ -727,10 +830,29 @@ static SDNode *findUser(SDValue Value, unsigned Opcode) {
 
 SDValue SITargetLowering::LowerFrameIndex(SDValue Op, SelectionDAG &DAG) const {
 
+  SDLoc SL(Op);
   FrameIndexSDNode *FINode = cast<FrameIndexSDNode>(Op);
   unsigned FrameIndex = FINode->getIndex();
 
-  return DAG.getTargetFrameIndex(FrameIndex, MVT::i32);
+  // A FrameIndex node represents a 32-bit offset into scratch memory.  If
+  // the high bit of a frame index offset were to be set, this would mean
+  // that it represented an offset of ~2GB * 64 = ~128GB from the start of the
+  // scratch buffer, with 64 being the number of threads per wave.
+  //
+  // If we know the machine uses less than 128GB of scratch, then we can
+  // amrk the high bit of the FrameIndex node as known zero,
+  // which is important, because it means in most situations we can
+  // prove that values derived from FrameIndex nodes are non-negative.
+  // This enables us to take advantage of more addressing modes when
+  // accessing scratch buffers, since for scratch reads/writes, the register
+  // offset must always be positive.
+
+  SDValue TFI = DAG.getTargetFrameIndex(FrameIndex, MVT::i32);
+  if (Subtarget->enableHugeScratchBuffer())
+    return TFI;
+
+  return DAG.getNode(ISD::AssertZext, SL, MVT::i32, TFI,
+                    DAG.getValueType(EVT::getIntegerVT(*DAG.getContext(), 31)));
 }
 
 /// This transforms the control flow intrinsics to get the branch destination as
@@ -820,27 +942,45 @@ SDValue SITargetLowering::LowerGlobalAddress(AMDGPUMachineFunction *MFI,
 
   SDLoc DL(GSD);
   const GlobalValue *GV = GSD->getGlobal();
-  MVT PtrVT = getPointerTy(GSD->getAddressSpace());
+  MVT PtrVT = getPointerTy(DAG.getDataLayout(), GSD->getAddressSpace());
 
   SDValue Ptr = DAG.getNode(AMDGPUISD::CONST_DATA_PTR, DL, PtrVT);
   SDValue GA = DAG.getTargetGlobalAddress(GV, DL, MVT::i32);
 
   SDValue PtrLo = DAG.getNode(ISD::EXTRACT_ELEMENT, DL, MVT::i32, Ptr,
-                              DAG.getConstant(0, MVT::i32));
+                              DAG.getConstant(0, DL, MVT::i32));
   SDValue PtrHi = DAG.getNode(ISD::EXTRACT_ELEMENT, DL, MVT::i32, Ptr,
-                              DAG.getConstant(1, MVT::i32));
+                              DAG.getConstant(1, DL, MVT::i32));
 
   SDValue Lo = DAG.getNode(ISD::ADDC, DL, DAG.getVTList(MVT::i32, MVT::Glue),
                            PtrLo, GA);
   SDValue Hi = DAG.getNode(ISD::ADDE, DL, DAG.getVTList(MVT::i32, MVT::Glue),
-                           PtrHi, DAG.getConstant(0, MVT::i32),
+                           PtrHi, DAG.getConstant(0, DL, MVT::i32),
                            SDValue(Lo.getNode(), 1));
   return DAG.getNode(ISD::BUILD_PAIR, DL, MVT::i64, Lo, Hi);
 }
 
+SDValue SITargetLowering::copyToM0(SelectionDAG &DAG, SDValue Chain, SDLoc DL,
+                                   SDValue V) const {
+  // We can't use CopyToReg, because MachineCSE won't combine COPY instructions,
+  // so we will end up with redundant moves to m0.
+  //
+  // We can't use S_MOV_B32, because there is no way to specify m0 as the
+  // destination register.
+  //
+  // We have to use them both.  Machine cse will combine all the S_MOV_B32
+  // instructions and the register coalescer eliminate the extra copies.
+  SDNode *M0 = DAG.getMachineNode(AMDGPU::S_MOV_B32, DL, V.getValueType(), V);
+  return DAG.getCopyToReg(Chain, DL, DAG.getRegister(AMDGPU::M0, MVT::i32),
+                          SDValue(M0, 0), SDValue()); // Glue
+                                                      // A Null SDValue creates
+                                                      // a glue result.
+}
+
 SDValue SITargetLowering::LowerINTRINSIC_WO_CHAIN(SDValue Op,
                                                   SelectionDAG &DAG) const {
   MachineFunction &MF = DAG.getMachineFunction();
+  auto MFI = MF.getInfo<SIMachineFunctionInfo>();
   const SIRegisterInfo *TRI =
       static_cast<const SIRegisterInfo *>(Subtarget->getRegisterInfo());
 
@@ -879,8 +1019,7 @@ SDValue SITargetLowering::LowerINTRINSIC_WO_CHAIN(SDValue Op,
 
   case Intrinsic::AMDGPU_read_workdim:
     return LowerParameter(DAG, VT, VT, DL, DAG.getEntryNode(),
-                          MF.getInfo<SIMachineFunctionInfo>()->ABIArgOffset,
-                          false);
+                          getImplicitParameterOffset(MFI, GRID_DIM), false);
 
   case Intrinsic::r600_read_tgid_x:
     return CreateLiveInRegister(DAG, &AMDGPU::SReg_32RegClass,
@@ -931,7 +1070,28 @@ SDValue SITargetLowering::LowerINTRINSIC_WO_CHAIN(SDValue Op,
   case AMDGPUIntrinsic::AMDIL_fraction: // Legacy name.
     return DAG.getNode(ISD::FSUB, DL, VT, Op.getOperand(1),
                        DAG.getNode(ISD::FFLOOR, DL, VT, Op.getOperand(1)));
-
+  case AMDGPUIntrinsic::SI_fs_constant: {
+    SDValue M0 = copyToM0(DAG, DAG.getEntryNode(), DL, Op.getOperand(3));
+    SDValue Glue = M0.getValue(1);
+    return DAG.getNode(AMDGPUISD::INTERP_MOV, DL, MVT::f32,
+                       DAG.getConstant(2, DL, MVT::i32), // P0
+                       Op.getOperand(1), Op.getOperand(2), Glue);
+  }
+  case AMDGPUIntrinsic::SI_fs_interp: {
+    SDValue IJ = Op.getOperand(4);
+    SDValue I = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, MVT::i32, IJ,
+                            DAG.getConstant(0, DL, MVT::i32));
+    SDValue J = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, MVT::i32, IJ,
+                            DAG.getConstant(1, DL, MVT::i32));
+    SDValue M0 = copyToM0(DAG, DAG.getEntryNode(), DL, Op.getOperand(3));
+    SDValue Glue = M0.getValue(1);
+    SDValue P1 = DAG.getNode(AMDGPUISD::INTERP_P1, DL,
+                             DAG.getVTList(MVT::f32, MVT::Glue),
+                             I, Op.getOperand(1), Op.getOperand(2), Glue);
+    Glue = SDValue(P1.getNode(), 1);
+    return DAG.getNode(AMDGPUISD::INTERP_P2, DL, MVT::f32, P1, J,
+                             Op.getOperand(1), Op.getOperand(2), Glue);
+  }
   default:
     return AMDGPUTargetLowering::LowerOperation(Op, DAG);
   }
@@ -940,12 +1100,18 @@ SDValue SITargetLowering::LowerINTRINSIC_WO_CHAIN(SDValue Op,
 SDValue SITargetLowering::LowerINTRINSIC_VOID(SDValue Op,
                                               SelectionDAG &DAG) const {
   MachineFunction &MF = DAG.getMachineFunction();
+  SDLoc DL(Op);
   SDValue Chain = Op.getOperand(0);
   unsigned IntrinsicID = cast<ConstantSDNode>(Op.getOperand(1))->getZExtValue();
 
   switch (IntrinsicID) {
+  case AMDGPUIntrinsic::SI_sendmsg: {
+    Chain = copyToM0(DAG, Chain, DL, Op.getOperand(3));
+    SDValue Glue = Chain.getValue(1);
+    return DAG.getNode(AMDGPUISD::SENDMSG, DL, MVT::Other, Chain,
+                       Op.getOperand(2), Glue);
+  }
   case AMDGPUIntrinsic::SI_tbuffer_store: {
-    SDLoc DL(Op);
     SDValue Ops[] = {
       Chain,
       Op.getOperand(2),
@@ -1018,8 +1184,8 @@ SDValue SITargetLowering::LowerSELECT(SDValue Op, SelectionDAG &DAG) const {
   SDLoc DL(Op);
   SDValue Cond = Op.getOperand(0);
 
-  SDValue Zero = DAG.getConstant(0, MVT::i32);
-  SDValue One = DAG.getConstant(1, MVT::i32);
+  SDValue Zero = DAG.getConstant(0, DL, MVT::i32);
+  SDValue One = DAG.getConstant(1, DL, MVT::i32);
 
   SDValue LHS = DAG.getNode(ISD::BITCAST, DL, MVT::v2i32, Op.getOperand(1));
   SDValue RHS = DAG.getNode(ISD::BITCAST, DL, MVT::v2i32, Op.getOperand(2));
@@ -1094,14 +1260,15 @@ SDValue SITargetLowering::LowerFDIV32(SDValue Op, SelectionDAG &DAG) const {
   SDValue r1 = DAG.getNode(ISD::FABS, SL, MVT::f32, RHS);
 
   const APFloat K0Val(BitsToFloat(0x6f800000));
-  const SDValue K0 = DAG.getConstantFP(K0Val, MVT::f32);
+  const SDValue K0 = DAG.getConstantFP(K0Val, SL, MVT::f32);
 
   const APFloat K1Val(BitsToFloat(0x2f800000));
-  const SDValue K1 = DAG.getConstantFP(K1Val, MVT::f32);
+  const SDValue K1 = DAG.getConstantFP(K1Val, SL, MVT::f32);
 
-  const SDValue One = DAG.getConstantFP(1.0, MVT::f32);
+  const SDValue One = DAG.getConstantFP(1.0, SL, MVT::f32);
 
-  EVT SetCCVT = getSetCCResultType(*DAG.getContext(), MVT::f32);
+  EVT SetCCVT =
+      getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), MVT::f32);
 
   SDValue r2 = DAG.getSetCC(SL, SetCCVT, r1, K0, ISD::SETOGT);
 
@@ -1124,7 +1291,7 @@ SDValue SITargetLowering::LowerFDIV64(SDValue Op, SelectionDAG &DAG) const {
   SDValue X = Op.getOperand(0);
   SDValue Y = Op.getOperand(1);
 
-  const SDValue One = DAG.getConstantFP(1.0, MVT::f64);
+  const SDValue One = DAG.getConstantFP(1.0, SL, MVT::f64);
 
   SDVTList ScaleVT = DAG.getVTList(MVT::f64, MVT::i1);
 
@@ -1154,7 +1321,7 @@ SDValue SITargetLowering::LowerFDIV64(SDValue Op, SelectionDAG &DAG) const {
     // Workaround a hardware bug on SI where the condition output from div_scale
     // is not usable.
 
-    const SDValue Hi = DAG.getConstant(1, MVT::i32);
+    const SDValue Hi = DAG.getConstant(1, SL, MVT::i32);
 
     // Figure out if the scale to use for div_fmas.
     SDValue NumBC = DAG.getNode(ISD::BITCAST, SL, MVT::v2i32, X);
@@ -1223,11 +1390,13 @@ SDValue SITargetLowering::LowerSTORE(SDValue Op, SelectionDAG &DAG) const {
 }
 
 SDValue SITargetLowering::LowerTrig(SDValue Op, SelectionDAG &DAG) const {
+  SDLoc DL(Op);
   EVT VT = Op.getValueType();
   SDValue Arg = Op.getOperand(0);
-  SDValue FractPart = DAG.getNode(AMDGPUISD::FRACT, SDLoc(Op), VT,
-        DAG.getNode(ISD::FMUL, SDLoc(Op), VT, Arg,
-          DAG.getConstantFP(0.5 / M_PI, VT)));
+  SDValue FractPart = DAG.getNode(AMDGPUISD::FRACT, DL, VT,
+                                  DAG.getNode(ISD::FMUL, DL, VT, Arg,
+                                              DAG.getConstantFP(0.5/M_PI, DL,
+                                                                VT)));
 
   switch (Op.getOpcode()) {
   case ISD::FCOS:
@@ -1297,7 +1466,7 @@ SDValue SITargetLowering::performUCharToFloatCombine(SDNode *N,
     unsigned AS = Load->getAddressSpace();
     unsigned Align = Load->getAlignment();
     Type *Ty = LoadVT.getTypeForEVT(*DAG.getContext());
-    unsigned ABIAlignment = getDataLayout()->getABITypeAlignment(Ty);
+    unsigned ABIAlignment = DAG.getDataLayout().getABITypeAlignment(Ty);
 
     // Don't try to replace the load if we have to expand it due to alignment
     // problems. Otherwise we will end up scalarizing the load, and trying to
@@ -1417,7 +1586,7 @@ SDValue SITargetLowering::performSHLPtrCombine(SDNode *N,
   EVT VT = N->getValueType(0);
 
   SDValue ShlX = DAG.getNode(ISD::SHL, SL, VT, N0.getOperand(0), N1);
-  SDValue COffset = DAG.getConstant(Offset, MVT::i32);
+  SDValue COffset = DAG.getConstant(Offset, SL, MVT::i32);
 
   return DAG.getNode(ISD::ADD, SL, VT, ShlX, COffset);
 }
@@ -1466,8 +1635,9 @@ SDValue SITargetLowering::performAndCombine(SDNode *N,
                           SIInstrFlags::P_INFINITY)) & 0x3ff) == Mask,
                       "mask not equal");
 
-        return DAG.getNode(AMDGPUISD::FP_CLASS, SDLoc(N), MVT::i1,
-                           X, DAG.getConstant(Mask, MVT::i32));
+        SDLoc DL(N);
+        return DAG.getNode(AMDGPUISD::FP_CLASS, DL, MVT::i1,
+                           X, DAG.getConstant(Mask, DL, MVT::i32));
       }
     }
   }
@@ -1497,8 +1667,9 @@ SDValue SITargetLowering::performOrCombine(SDNode *N,
     static const uint32_t MaxMask = 0x3ff;
 
     uint32_t NewMask = (CLHS->getZExtValue() | CRHS->getZExtValue()) & MaxMask;
-    return DAG.getNode(AMDGPUISD::FP_CLASS, SDLoc(N), MVT::i1,
-                       Src, DAG.getConstant(NewMask, MVT::i32));
+    SDLoc DL(N);
+    return DAG.getNode(AMDGPUISD::FP_CLASS, DL, MVT::i1,
+                       Src, DAG.getConstant(NewMask, DL, MVT::i32));
   }
 
   return SDValue();
@@ -1512,7 +1683,7 @@ SDValue SITargetLowering::performClassCombine(SDNode *N,
   // fp_class x, 0 -> false
   if (const ConstantSDNode *CMask = dyn_cast<ConstantSDNode>(Mask)) {
     if (CMask->isNullValue())
-      return DAG.getConstant(0, MVT::i1);
+      return DAG.getConstant(0, SDLoc(N), MVT::i1);
   }
 
   return SDValue();
@@ -1522,15 +1693,15 @@ static unsigned minMaxOpcToMin3Max3Opc(unsigned Opc) {
   switch (Opc) {
   case ISD::FMAXNUM:
     return AMDGPUISD::FMAX3;
-  case AMDGPUISD::SMAX:
+  case ISD::SMAX:
     return AMDGPUISD::SMAX3;
-  case AMDGPUISD::UMAX:
+  case ISD::UMAX:
     return AMDGPUISD::UMAX3;
   case ISD::FMINNUM:
     return AMDGPUISD::FMIN3;
-  case AMDGPUISD::SMIN:
+  case ISD::SMIN:
     return AMDGPUISD::SMIN3;
-  case AMDGPUISD::UMIN:
+  case ISD::UMIN:
     return AMDGPUISD::UMIN3;
   default:
     llvm_unreachable("Not a min/max opcode");
@@ -1596,8 +1767,8 @@ SDValue SITargetLowering::performSetCCCombine(SDNode *N,
     const APFloat &APF = CRHS->getValueAPF();
     if (APF.isInfinity() && !APF.isNegative()) {
       unsigned Mask = SIInstrFlags::P_INFINITY | SIInstrFlags::N_INFINITY;
-      return DAG.getNode(AMDGPUISD::FP_CLASS, SL, MVT::i1,
-                         LHS.getOperand(0), DAG.getConstant(Mask, MVT::i32));
+      return DAG.getNode(AMDGPUISD::FP_CLASS, SL, MVT::i1, LHS.getOperand(0),
+                         DAG.getConstant(Mask, SL, MVT::i32));
     }
   }
 
@@ -1616,10 +1787,10 @@ SDValue SITargetLowering::PerformDAGCombine(SDNode *N,
     return performSetCCCombine(N, DCI);
   case ISD::FMAXNUM: // TODO: What about fmax_legacy?
   case ISD::FMINNUM:
-  case AMDGPUISD::SMAX:
-  case AMDGPUISD::SMIN:
-  case AMDGPUISD::UMAX:
-  case AMDGPUISD::UMIN: {
+  case ISD::SMAX:
+  case ISD::SMIN:
+  case ISD::UMAX:
+  case ISD::UMIN: {
     if (DCI.getDAGCombineLevel() >= AfterLegalizeDAG &&
         N->getValueType(0) != MVT::f64 &&
         getTargetMachine().getOptLevel() > CodeGenOpt::None)
@@ -1674,7 +1845,7 @@ SDValue SITargetLowering::PerformDAGCombine(SDNode *N,
     if (LHS.getOpcode() == ISD::FADD) {
       SDValue A = LHS.getOperand(0);
       if (A == LHS.getOperand(1)) {
-        const SDValue Two = DAG.getConstantFP(2.0, MVT::f32);
+        const SDValue Two = DAG.getConstantFP(2.0, DL, MVT::f32);
         return DAG.getNode(ISD::FMAD, DL, VT, Two, A, RHS);
       }
     }
@@ -1683,7 +1854,7 @@ SDValue SITargetLowering::PerformDAGCombine(SDNode *N,
     if (RHS.getOpcode() == ISD::FADD) {
       SDValue A = RHS.getOperand(0);
       if (A == RHS.getOperand(1)) {
-        const SDValue Two = DAG.getConstantFP(2.0, MVT::f32);
+        const SDValue Two = DAG.getConstantFP(2.0, DL, MVT::f32);
         return DAG.getNode(ISD::FMAD, DL, VT, Two, A, LHS);
       }
     }
@@ -1710,7 +1881,7 @@ SDValue SITargetLowering::PerformDAGCombine(SDNode *N,
 
         SDValue A = LHS.getOperand(0);
         if (A == LHS.getOperand(1)) {
-          const SDValue Two = DAG.getConstantFP(2.0, MVT::f32);
+          const SDValue Two = DAG.getConstantFP(2.0, DL, MVT::f32);
           SDValue NegRHS = DAG.getNode(ISD::FNEG, DL, VT, RHS);
 
           return DAG.getNode(ISD::FMAD, DL, VT, Two, A, NegRHS);
@@ -1722,7 +1893,7 @@ SDValue SITargetLowering::PerformDAGCombine(SDNode *N,
 
         SDValue A = RHS.getOperand(0);
         if (A == RHS.getOperand(1)) {
-          const SDValue NegTwo = DAG.getConstantFP(-2.0, MVT::f32);
+          const SDValue NegTwo = DAG.getConstantFP(-2.0, DL, MVT::f32);
           return DAG.getNode(ISD::FMAD, DL, VT, NegTwo, A, LHS);
         }
       }
@@ -1865,14 +2036,15 @@ void SITargetLowering::adjustWritemask(MachineSDNode *&Node,
 
   // Adjust the writemask in the node
   std::vector<SDValue> Ops;
-  Ops.push_back(DAG.getTargetConstant(NewDmask, MVT::i32));
+  Ops.push_back(DAG.getTargetConstant(NewDmask, SDLoc(Node), MVT::i32));
   Ops.insert(Ops.end(), Node->op_begin() + 1, Node->op_end());
   Node = (MachineSDNode*)DAG.UpdateNodeOperands(Node, Ops);
 
   // If we only got one lane, replace it with a copy
   // (if NewDmask has only one bit set...)
   if (NewDmask && (NewDmask & (NewDmask-1)) == 0) {
-    SDValue RC = DAG.getTargetConstant(AMDGPU::VGPR_32RegClassID, MVT::i32);
+    SDValue RC = DAG.getTargetConstant(AMDGPU::VGPR_32RegClassID, SDLoc(),
+                                       MVT::i32);
     SDNode *Copy = DAG.getMachineNode(TargetOpcode::COPY_TO_REGCLASS,
                                       SDLoc(), Users[Lane]->getValueType(0),
                                       SDValue(Node, 0), RC);
@@ -1887,7 +2059,7 @@ void SITargetLowering::adjustWritemask(MachineSDNode *&Node,
     if (!User)
       continue;
 
-    SDValue Op = DAG.getTargetConstant(Idx, MVT::i32);
+    SDValue Op = DAG.getTargetConstant(Idx, SDLoc(User), MVT::i32);
     DAG.UpdateNodeOperands(User, User->getOperand(0), Op);
 
     switch (Idx) {
@@ -1899,6 +2071,13 @@ void SITargetLowering::adjustWritemask(MachineSDNode *&Node,
   }
 }
 
+static bool isFrameIndexOp(SDValue Op) {
+  if (Op.getOpcode() == ISD::AssertZext)
+    Op = Op.getOperand(0);
+
+  return isa<FrameIndexSDNode>(Op);
+}
+
 /// \brief Legalize target independent instructions (e.g. INSERT_SUBREG)
 /// with frame index operands.
 /// LLVM assumes that inputs are to these instructions are registers.
@@ -1907,7 +2086,7 @@ void SITargetLowering::legalizeTargetIndependentNode(SDNode *Node,
 
   SmallVector<SDValue, 8> Ops;
   for (unsigned i = 0; i < Node->getNumOperands(); ++i) {
-    if (!isa<FrameIndexSDNode>(Node->getOperand(i))) {
+    if (!isFrameIndexOp(Node->getOperand(i))) {
       Ops.push_back(Node->getOperand(i));
       continue;
     }
@@ -1982,7 +2161,7 @@ void SITargetLowering::AdjustInstrPostInstrSelection(MachineInstr *MI,
 }
 
 static SDValue buildSMovImm32(SelectionDAG &DAG, SDLoc DL, uint64_t Val) {
-  SDValue K = DAG.getTargetConstant(Val, MVT::i32);
+  SDValue K = DAG.getTargetConstant(Val, DL, MVT::i32);
   return SDValue(DAG.getMachineNode(AMDGPU::S_MOV_B32, DL, MVT::i32, K), 0);
 }
 
@@ -1997,11 +2176,11 @@ MachineSDNode *SITargetLowering::wrapAddr64Rsrc(SelectionDAG &DAG,
 
     // Build the half of the subregister with the constants.
     const SDValue Ops0[] = {
-      DAG.getTargetConstant(AMDGPU::SGPR_64RegClassID, MVT::i32),
+      DAG.getTargetConstant(AMDGPU::SGPR_64RegClassID, DL, MVT::i32),
       buildSMovImm32(DAG, DL, 0),
-      DAG.getTargetConstant(AMDGPU::sub0, MVT::i32),
+      DAG.getTargetConstant(AMDGPU::sub0, DL, MVT::i32),
       buildSMovImm32(DAG, DL, TII->getDefaultRsrcDataFormat() >> 32),
-      DAG.getTargetConstant(AMDGPU::sub1, MVT::i32)
+      DAG.getTargetConstant(AMDGPU::sub1, DL, MVT::i32)
     };
 
     SDValue SubRegHi = SDValue(DAG.getMachineNode(AMDGPU::REG_SEQUENCE, DL,
@@ -2009,11 +2188,11 @@ MachineSDNode *SITargetLowering::wrapAddr64Rsrc(SelectionDAG &DAG,
 
     // Combine the constants and the pointer.
     const SDValue Ops1[] = {
-      DAG.getTargetConstant(AMDGPU::SReg_128RegClassID, MVT::i32),
+      DAG.getTargetConstant(AMDGPU::SReg_128RegClassID, DL, MVT::i32),
       Ptr,
-      DAG.getTargetConstant(AMDGPU::sub0_sub1, MVT::i32),
+      DAG.getTargetConstant(AMDGPU::sub0_sub1, DL, MVT::i32),
       SubRegHi,
-      DAG.getTargetConstant(AMDGPU::sub2_sub3, MVT::i32)
+      DAG.getTargetConstant(AMDGPU::sub2_sub3, DL, MVT::i32)
     };
 
     return DAG.getMachineNode(AMDGPU::REG_SEQUENCE, DL, MVT::v4i32, Ops1);
@@ -2046,7 +2225,8 @@ MachineSDNode *SITargetLowering::buildRSRC(SelectionDAG &DAG,
   SDValue PtrHi = DAG.getTargetExtractSubreg(AMDGPU::sub1, DL, MVT::i32, Ptr);
   if (RsrcDword1) {
     PtrHi = SDValue(DAG.getMachineNode(AMDGPU::S_OR_B32, DL, MVT::i32, PtrHi,
-                                     DAG.getConstant(RsrcDword1, MVT::i32)), 0);
+                                     DAG.getConstant(RsrcDword1, DL, MVT::i32)),
+                    0);
   }
 
   SDValue DataLo = buildSMovImm32(DAG, DL,
@@ -2054,15 +2234,15 @@ MachineSDNode *SITargetLowering::buildRSRC(SelectionDAG &DAG,
   SDValue DataHi = buildSMovImm32(DAG, DL, RsrcDword2And3 >> 32);
 
   const SDValue Ops[] = {
-    DAG.getTargetConstant(AMDGPU::SReg_128RegClassID, MVT::i32),
+    DAG.getTargetConstant(AMDGPU::SReg_128RegClassID, DL, MVT::i32),
     PtrLo,
-    DAG.getTargetConstant(AMDGPU::sub0, MVT::i32),
+    DAG.getTargetConstant(AMDGPU::sub0, DL, MVT::i32),
     PtrHi,
-    DAG.getTargetConstant(AMDGPU::sub1, MVT::i32),
+    DAG.getTargetConstant(AMDGPU::sub1, DL, MVT::i32),
     DataLo,
-    DAG.getTargetConstant(AMDGPU::sub2, MVT::i32),
+    DAG.getTargetConstant(AMDGPU::sub2, DL, MVT::i32),
     DataHi,
-    DAG.getTargetConstant(AMDGPU::sub3, MVT::i32)
+    DAG.getTargetConstant(AMDGPU::sub3, DL, MVT::i32)
   };
 
   return DAG.getMachineNode(AMDGPU::REG_SEQUENCE, DL, MVT::v4i32, Ops);
@@ -2094,7 +2274,7 @@ SDValue SITargetLowering::CreateLiveInRegister(SelectionDAG &DAG,
 
 std::pair<unsigned, const TargetRegisterClass *>
 SITargetLowering::getRegForInlineAsmConstraint(const TargetRegisterInfo *TRI,
-                                               const std::string &Constraint,
+                                               StringRef Constraint,
                                                MVT VT) const {
   if (Constraint == "r") {
     switch(VT.SimpleTy) {
@@ -2115,8 +2295,9 @@ SITargetLowering::getRegForInlineAsmConstraint(const TargetRegisterInfo *TRI,
     }
 
     if (RC) {
-      unsigned Idx = std::atoi(Constraint.substr(2).c_str());
-      if (Idx < RC->getNumRegs())
+      uint32_t Idx;
+      bool Failed = Constraint.substr(2).getAsInteger(10, Idx);
+      if (!Failed && Idx < RC->getNumRegs())
         return std::make_pair(RC->getRegister(Idx), RC);
     }
   }
diff --git a/lib/Target/R600/SIISelLowering.h b/lib/Target/AMDGPU/SIISelLowering.h
similarity index 88%
rename from lib/Target/R600/SIISelLowering.h
rename to lib/Target/AMDGPU/SIISelLowering.h
index a6bc7c607fa..d84c32ec009 100644
--- a/lib/Target/R600/SIISelLowering.h
+++ b/lib/Target/AMDGPU/SIISelLowering.h
@@ -56,14 +56,15 @@ class SITargetLowering : public AMDGPUTargetLowering {
   SDValue performMin3Max3Combine(SDNode *N, DAGCombinerInfo &DCI) const;
   SDValue performSetCCCombine(SDNode *N, DAGCombinerInfo &DCI) const;
 
+  bool isLegalFlatAddressingMode(const AddrMode &AM) const;
 public:
   SITargetLowering(TargetMachine &tm, const AMDGPUSubtarget &STI);
 
   bool isShuffleMaskLegal(const SmallVectorImpl<int> &/*Mask*/,
                           EVT /*VT*/) const override;
 
-  bool isLegalAddressingMode(const AddrMode &AM,
-                             Type *Ty) const override;
+  bool isLegalAddressingMode(const DataLayout &DL, const AddrMode &AM, Type *Ty,
+                             unsigned AS) const override;
 
   bool allowsMisalignedMemoryAccesses(EVT VT, unsigned AS,
                                       unsigned Align,
@@ -90,8 +91,9 @@ class SITargetLowering : public AMDGPUTargetLowering {
   MachineBasicBlock * EmitInstrWithCustomInserter(MachineInstr * MI,
                                       MachineBasicBlock * BB) const override;
   bool enableAggressiveFMAFusion(EVT VT) const override;
-  EVT getSetCCResultType(LLVMContext &Context, EVT VT) const override;
-  MVT getScalarShiftAmountTy(EVT VT) const override;
+  EVT getSetCCResultType(const DataLayout &DL, LLVMContext &Context,
+                         EVT VT) const override;
+  MVT getScalarShiftAmountTy(const DataLayout &, EVT) const override;
   bool isFMAFasterThanFMulAndFAdd(EVT VT) const override;
   SDValue LowerOperation(SDValue Op, SelectionDAG &DAG) const override;
   SDValue PerformDAGCombine(SDNode *N, DAGCombinerInfo &DCI) const override;
@@ -114,9 +116,10 @@ class SITargetLowering : public AMDGPUTargetLowering {
                                   SDLoc DL,
                                   SDValue Ptr) const;
 
-  std::pair<unsigned, const TargetRegisterClass *> getRegForInlineAsmConstraint(
-                                   const TargetRegisterInfo *TRI,
-                                   const std::string &Constraint, MVT VT) const override;
+  std::pair<unsigned, const TargetRegisterClass *>
+  getRegForInlineAsmConstraint(const TargetRegisterInfo *TRI,
+                               StringRef Constraint, MVT VT) const override;
+  SDValue copyToM0(SelectionDAG &DAG, SDValue Chain, SDLoc DL, SDValue V) const;
 };
 
 } // End namespace llvm
diff --git a/lib/Target/R600/SIInsertWaits.cpp b/lib/Target/AMDGPU/SIInsertWaits.cpp
similarity index 100%
rename from lib/Target/R600/SIInsertWaits.cpp
rename to lib/Target/AMDGPU/SIInsertWaits.cpp
diff --git a/lib/Target/R600/SIInstrFormats.td b/lib/Target/AMDGPU/SIInstrFormats.td
similarity index 99%
rename from lib/Target/R600/SIInstrFormats.td
rename to lib/Target/AMDGPU/SIInstrFormats.td
index bc693c3894f..211666a9bdb 100644
--- a/lib/Target/R600/SIInstrFormats.td
+++ b/lib/Target/AMDGPU/SIInstrFormats.td
@@ -39,6 +39,7 @@ class InstSI <dag outs, dag ins, string asm, list<dag> pattern> :
   field bits<1> MIMG = 0;
   field bits<1> FLAT = 0;
   field bits<1> WQM = 0;
+  field bits<1> VGPRSpill = 0;
 
   // These need to be kept in sync with the enum in SIInstrFlags.
   let TSFlags{0} = VM_CNT;
@@ -66,6 +67,7 @@ class InstSI <dag outs, dag ins, string asm, list<dag> pattern> :
   let TSFlags{18} = MIMG;
   let TSFlags{19} = FLAT;
   let TSFlags{20} = WQM;
+  let TSFlags{21} = VGPRSpill;
 
   // Most instructions require adjustments after selection to satisfy
   // operand requirements.
@@ -604,7 +606,7 @@ class DS <dag outs, dag ins, string asm, list<dag> pattern> :
   let LGKM_CNT = 1;
   let DS = 1;
   let UseNamedOperandTable = 1;
-  let DisableEncoding = "$m0";
+  let Uses = [M0];
 
   // Most instruction load and store data, so set this as the default.
   let mayLoad = 1;
@@ -653,6 +655,8 @@ class FLAT <bits<7> op, dag outs, dag ins, string asm, list<dag> pattern> :
 
   let UseNamedOperandTable = 1;
   let hasSideEffects = 0;
+  let AsmMatchConverter = "cvtFlat";
+  let SchedRW = [WriteVMEM];
 }
 
 class MIMG <bits<7> op, dag outs, dag ins, string asm, list<dag> pattern> :
diff --git a/lib/Target/R600/SIInstrInfo.cpp b/lib/Target/AMDGPU/SIInstrInfo.cpp
similarity index 95%
rename from lib/Target/R600/SIInstrInfo.cpp
rename to lib/Target/AMDGPU/SIInstrInfo.cpp
index ba98ad7dd70..9b391530a2f 100644
--- a/lib/Target/R600/SIInstrInfo.cpp
+++ b/lib/Target/AMDGPU/SIInstrInfo.cpp
@@ -74,6 +74,20 @@ static bool nodesHaveSameOperandValue(SDNode *N0, SDNode* N1, unsigned OpName) {
   return N0->getOperand(Op0Idx) == N1->getOperand(Op1Idx);
 }
 
+bool SIInstrInfo::isReallyTriviallyReMaterializable(const MachineInstr *MI,
+                                                    AliasAnalysis *AA) const {
+  // TODO: The generic check fails for VALU instructions that should be
+  // rematerializable due to implicit reads of exec. We really want all of the
+  // generic logic for this except for this.
+  switch (MI->getOpcode()) {
+  case AMDGPU::V_MOV_B32_e32:
+  case AMDGPU::V_MOV_B32_e64:
+    return true;
+  default:
+    return false;
+  }
+}
+
 bool SIInstrInfo::areLoadsFromSameBasePtr(SDNode *Load0, SDNode *Load1,
                                           int64_t &Offset0,
                                           int64_t &Offset1) const {
@@ -186,9 +200,9 @@ static bool isStride64(unsigned Opc) {
   }
 }
 
-bool SIInstrInfo::getLdStBaseRegImmOfs(MachineInstr *LdSt,
-                                       unsigned &BaseReg, unsigned &Offset,
-                                       const TargetRegisterInfo *TRI) const {
+bool SIInstrInfo::getMemOpBaseRegImmOfs(MachineInstr *LdSt, unsigned &BaseReg,
+                                        unsigned &Offset,
+                                        const TargetRegisterInfo *TRI) const {
   unsigned Opc = LdSt->getOpcode();
   if (isDS(Opc)) {
     const MachineOperand *OffsetImm = getNamedOperand(*LdSt,
@@ -213,9 +227,8 @@ bool SIInstrInfo::getLdStBaseRegImmOfs(MachineInstr *LdSt,
 
     uint8_t Offset0 = Offset0Imm->getImm();
     uint8_t Offset1 = Offset1Imm->getImm();
-    assert(Offset1 > Offset0);
 
-    if (Offset1 - Offset0 == 1) {
+    if (Offset1 > Offset0 && Offset1 - Offset0 == 1) {
       // Each of these offsets is in element sized units, so we need to convert
       // to bytes of the individual reads.
 
@@ -426,22 +439,22 @@ SIInstrInfo::copyPhysReg(MachineBasicBlock &MBB,
   }
 }
 
-unsigned SIInstrInfo::commuteOpcode(const MachineInstr &MI) const {
+int SIInstrInfo::commuteOpcode(const MachineInstr &MI) const {
   const unsigned Opcode = MI.getOpcode();
 
   int NewOpc;
 
   // Try to map original to commuted opcode
   NewOpc = AMDGPU::getCommuteRev(Opcode);
-  // Check if the commuted (REV) opcode exists on the target.
-  if (NewOpc != -1 && pseudoToMCOpcode(NewOpc) != -1)
-    return NewOpc;
+  if (NewOpc != -1)
+    // Check if the commuted (REV) opcode exists on the target.
+    return pseudoToMCOpcode(NewOpc) != -1 ? NewOpc : -1;
 
   // Try to map commuted to original opcode
   NewOpc = AMDGPU::getCommuteOrig(Opcode);
-  // Check if the original (non-REV) opcode exists on the target.
-  if (NewOpc != -1 && pseudoToMCOpcode(NewOpc) != -1)
-    return NewOpc;
+  if (NewOpc != -1)
+    // Check if the original (non-REV) opcode exists on the target.
+    return pseudoToMCOpcode(NewOpc) != -1 ? NewOpc : -1;
 
   return Opcode;
 }
@@ -757,6 +770,10 @@ MachineInstr *SIInstrInfo::commuteInstruction(MachineInstr *MI,
   if (MI->getNumOperands() < 3)
     return nullptr;
 
+  int CommutedOpcode = commuteOpcode(*MI);
+  if (CommutedOpcode == -1)
+    return nullptr;
+
   int Src0Idx = AMDGPU::getNamedOperandIdx(MI->getOpcode(),
                                            AMDGPU::OpName::src0);
   assert(Src0Idx != -1 && "Should always have src0 operand");
@@ -819,7 +836,7 @@ MachineInstr *SIInstrInfo::commuteInstruction(MachineInstr *MI,
   }
 
   if (MI)
-    MI->setDesc(get(commuteOpcode(*MI)));
+    MI->setDesc(get(CommutedOpcode));
 
   return MI;
 }
@@ -906,7 +923,7 @@ bool SIInstrInfo::FoldImmediate(MachineInstr *UseMI, MachineInstr *DefMI,
     return false;
 
   unsigned Opc = UseMI->getOpcode();
-  if (Opc == AMDGPU::V_MAD_F32) {
+  if (Opc == AMDGPU::V_MAD_F32 || Opc == AMDGPU::V_MAC_F32_e64) {
     // Don't fold if we are using source modifiers. The new VOP2 instructions
     // don't have them.
     if (hasModifiersSet(*UseMI, AMDGPU::OpName::src0_modifiers) ||
@@ -945,9 +962,9 @@ bool SIInstrInfo::FoldImmediate(MachineInstr *UseMI, MachineInstr *DefMI,
       // instead of having to modify in place.
 
       // Remove these first since they are at the end.
-      UseMI->RemoveOperand(AMDGPU::getNamedOperandIdx(AMDGPU::V_MAD_F32,
+      UseMI->RemoveOperand(AMDGPU::getNamedOperandIdx(Opc,
                                                       AMDGPU::OpName::omod));
-      UseMI->RemoveOperand(AMDGPU::getNamedOperandIdx(AMDGPU::V_MAD_F32,
+      UseMI->RemoveOperand(AMDGPU::getNamedOperandIdx(Opc,
                                                       AMDGPU::OpName::clamp));
 
       unsigned Src1Reg = Src1->getReg();
@@ -956,9 +973,20 @@ bool SIInstrInfo::FoldImmediate(MachineInstr *UseMI, MachineInstr *DefMI,
       unsigned Src2SubReg = Src2->getSubReg();
       Src0->setReg(Src1Reg);
       Src0->setSubReg(Src1SubReg);
+      Src0->setIsKill(Src1->isKill());
+
       Src1->setReg(Src2Reg);
       Src1->setSubReg(Src2SubReg);
+      Src1->setIsKill(Src2->isKill());
+
+      if (Opc == AMDGPU::V_MAC_F32_e64) {
+        UseMI->untieRegOperand(
+          AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src2));
+      }
 
+      UseMI->RemoveOperand(AMDGPU::getNamedOperandIdx(Opc,
+                                                      AMDGPU::OpName::src2));
+      // ChangingToImmediate adds Src2 back to the instruction.
       Src2->ChangeToImmediate(Imm);
 
       removeModOperands(*UseMI);
@@ -989,11 +1017,17 @@ bool SIInstrInfo::FoldImmediate(MachineInstr *UseMI, MachineInstr *DefMI,
       // instead of having to modify in place.
 
       // Remove these first since they are at the end.
-      UseMI->RemoveOperand(AMDGPU::getNamedOperandIdx(AMDGPU::V_MAD_F32,
+      UseMI->RemoveOperand(AMDGPU::getNamedOperandIdx(Opc,
                                                       AMDGPU::OpName::omod));
-      UseMI->RemoveOperand(AMDGPU::getNamedOperandIdx(AMDGPU::V_MAD_F32,
+      UseMI->RemoveOperand(AMDGPU::getNamedOperandIdx(Opc,
                                                       AMDGPU::OpName::clamp));
 
+      if (Opc == AMDGPU::V_MAC_F32_e64) {
+        UseMI->untieRegOperand(
+          AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src2));
+      }
+
+      // ChangingToImmediate adds Src2 back to the instruction.
       Src2->ChangeToImmediate(Imm);
 
       // These come before src2.
@@ -1011,18 +1045,6 @@ bool SIInstrInfo::FoldImmediate(MachineInstr *UseMI, MachineInstr *DefMI,
   return false;
 }
 
-bool
-SIInstrInfo::isTriviallyReMaterializable(const MachineInstr *MI,
-                                         AliasAnalysis *AA) const {
-  switch(MI->getOpcode()) {
-  default: return AMDGPUInstrInfo::isTriviallyReMaterializable(MI, AA);
-  case AMDGPU::S_MOV_B32:
-  case AMDGPU::S_MOV_B64:
-  case AMDGPU::V_MOV_B32_e32:
-    return MI->getOperand(1).isImm();
-  }
-}
-
 static bool offsetsDoNotOverlap(int WidthA, int OffsetA,
                                 int WidthB, int OffsetB) {
   int LowOffset = OffsetA < OffsetB ? OffsetA : OffsetB;
@@ -1036,8 +1058,8 @@ bool SIInstrInfo::checkInstOffsetsDoNotOverlap(MachineInstr *MIa,
   unsigned BaseReg0, Offset0;
   unsigned BaseReg1, Offset1;
 
-  if (getLdStBaseRegImmOfs(MIa, BaseReg0, Offset0, &RI) &&
-      getLdStBaseRegImmOfs(MIb, BaseReg1, Offset1, &RI)) {
+  if (getMemOpBaseRegImmOfs(MIa, BaseReg0, Offset0, &RI) &&
+      getMemOpBaseRegImmOfs(MIb, BaseReg1, Offset1, &RI)) {
     assert(MIa->hasOneMemOperand() && MIb->hasOneMemOperand() &&
            "read2 / write2 not expected here yet");
     unsigned Width0 = (*MIa->memoperands_begin())->getSize();
@@ -1105,6 +1127,38 @@ bool SIInstrInfo::areMemAccessesTriviallyDisjoint(MachineInstr *MIa,
   return false;
 }
 
+MachineInstr *SIInstrInfo::convertToThreeAddress(MachineFunction::iterator &MBB,
+                                                MachineBasicBlock::iterator &MI,
+                                                LiveVariables *LV) const {
+
+  switch (MI->getOpcode()) {
+    default: return nullptr;
+    case AMDGPU::V_MAC_F32_e64: break;
+    case AMDGPU::V_MAC_F32_e32: {
+      const MachineOperand *Src0 = getNamedOperand(*MI, AMDGPU::OpName::src0);
+      if (Src0->isImm() && !isInlineConstant(*Src0, 4))
+        return nullptr;
+      break;
+    }
+  }
+
+  const MachineOperand *Dst = getNamedOperand(*MI, AMDGPU::OpName::dst);
+  const MachineOperand *Src0 = getNamedOperand(*MI, AMDGPU::OpName::src0);
+  const MachineOperand *Src1 = getNamedOperand(*MI, AMDGPU::OpName::src1);
+  const MachineOperand *Src2 = getNamedOperand(*MI, AMDGPU::OpName::src2);
+
+  return BuildMI(*MBB, MI, MI->getDebugLoc(), get(AMDGPU::V_MAD_F32))
+                 .addOperand(*Dst)
+                 .addImm(0) // Src0 mods
+                 .addOperand(*Src0)
+                 .addImm(0) // Src1 mods
+                 .addOperand(*Src1)
+                 .addImm(0) // Src mods
+                 .addOperand(*Src2)
+                 .addImm(0)  // clamp
+                 .addImm(0); // omod
+}
+
 bool SIInstrInfo::isInlineConstant(const APInt &Imm) const {
   int64_t SVal = Imm.getSExtValue();
   if (SVal >= -16 && SVal <= 64)
@@ -1604,7 +1658,10 @@ bool SIInstrInfo::isOperandLegal(const MachineInstr *MI, unsigned OpIdx,
 
   if (MO->isReg()) {
     assert(DefinedRC);
-    const TargetRegisterClass *RC = MRI.getRegClass(MO->getReg());
+    const TargetRegisterClass *RC =
+        TargetRegisterInfo::isVirtualRegister(MO->getReg()) ?
+            MRI.getRegClass(MO->getReg()) :
+            RI.getPhysRegClass(MO->getReg());
 
     // In order to be legal, the common sub-class must be equal to the
     // class of the current operand.  For example:
@@ -1789,7 +1846,7 @@ void SIInstrInfo::legalizeOperands(MachineInstr *MI) const {
     }
 
     MachineBasicBlock &MBB = *MI->getParent();
-    // Extract the the ptr from the resource descriptor.
+    // Extract the ptr from the resource descriptor.
 
     // SRsrcPtrLo = srsrc:sub0
     unsigned SRsrcPtrLo = buildExtractSubReg(MI, MRI, *SRsrc,
@@ -1999,13 +2056,13 @@ void SIInstrInfo::splitSMRD(MachineInstr *MI,
 
 void SIInstrInfo::moveSMRDToVALU(MachineInstr *MI, MachineRegisterInfo &MRI) const {
   MachineBasicBlock *MBB = MI->getParent();
-  switch (MI->getOpcode()) {
-    case AMDGPU::S_LOAD_DWORD_IMM:
-    case AMDGPU::S_LOAD_DWORD_SGPR:
-    case AMDGPU::S_LOAD_DWORDX2_IMM:
-    case AMDGPU::S_LOAD_DWORDX2_SGPR:
-    case AMDGPU::S_LOAD_DWORDX4_IMM:
-    case AMDGPU::S_LOAD_DWORDX4_SGPR: {
+  int DstIdx = AMDGPU::getNamedOperandIdx(MI->getOpcode(), AMDGPU::OpName::dst);
+  assert(DstIdx != -1);
+  unsigned DstRCID = get(MI->getOpcode()).OpInfo[DstIdx].RegClass;
+  switch(RI.getRegClass(DstRCID)->getSize()) {
+    case 4:
+    case 8:
+    case 16: {
       unsigned NewOpcode = getVALUOp(*MI);
       unsigned RegOffset;
       unsigned ImmOffset;
@@ -2077,8 +2134,7 @@ void SIInstrInfo::moveSMRDToVALU(MachineInstr *MI, MachineRegisterInfo &MRI) con
       MRI.replaceRegWith(DstReg, NewDstReg);
       break;
     }
-    case AMDGPU::S_LOAD_DWORDX8_IMM:
-    case AMDGPU::S_LOAD_DWORDX8_SGPR: {
+    case 32: {
       MachineInstr *Lo, *Hi;
       splitSMRD(MI, &AMDGPU::SReg_128RegClass, AMDGPU::S_LOAD_DWORDX4_IMM,
                 AMDGPU::S_LOAD_DWORDX4_SGPR, Lo, Hi);
@@ -2088,8 +2144,7 @@ void SIInstrInfo::moveSMRDToVALU(MachineInstr *MI, MachineRegisterInfo &MRI) con
       break;
     }
 
-    case AMDGPU::S_LOAD_DWORDX16_IMM:
-    case AMDGPU::S_LOAD_DWORDX16_SGPR: {
+    case 64: {
       MachineInstr *Lo, *Hi;
       splitSMRD(MI, &AMDGPU::SReg_256RegClass, AMDGPU::S_LOAD_DWORDX8_IMM,
                 AMDGPU::S_LOAD_DWORDX8_SGPR, Lo, Hi);
@@ -2699,8 +2754,13 @@ MachineOperand *SIInstrInfo::getNamedOperand(MachineInstr &MI,
 
 uint64_t SIInstrInfo::getDefaultRsrcDataFormat() const {
   uint64_t RsrcDataFormat = AMDGPU::RSRC_DATA_FORMAT;
-  if (ST.isAmdHsaOS())
+  if (ST.isAmdHsaOS()) {
     RsrcDataFormat |= (1ULL << 56);
 
+  if (ST.getGeneration() >= AMDGPUSubtarget::VOLCANIC_ISLANDS)
+    // Set MTYPE = 2
+    RsrcDataFormat |= (2ULL << 59);
+  }
+
   return RsrcDataFormat;
 }
diff --git a/lib/Target/R600/SIInstrInfo.h b/lib/Target/AMDGPU/SIInstrInfo.h
similarity index 94%
rename from lib/Target/R600/SIInstrInfo.h
rename to lib/Target/AMDGPU/SIInstrInfo.h
index a9aa99fc227..4f3b0b3fd71 100644
--- a/lib/Target/R600/SIInstrInfo.h
+++ b/lib/Target/AMDGPU/SIInstrInfo.h
@@ -72,13 +72,16 @@ class SIInstrInfo : public AMDGPUInstrInfo {
     return RI;
   }
 
+  bool isReallyTriviallyReMaterializable(const MachineInstr *MI,
+                                         AliasAnalysis *AA) const override;
+
   bool areLoadsFromSameBasePtr(SDNode *Load1, SDNode *Load2,
                                int64_t &Offset1,
                                int64_t &Offset2) const override;
 
-  bool getLdStBaseRegImmOfs(MachineInstr *LdSt,
-                            unsigned &BaseReg, unsigned &Offset,
-                            const TargetRegisterInfo *TRI) const final;
+  bool getMemOpBaseRegImmOfs(MachineInstr *LdSt, unsigned &BaseReg,
+                             unsigned &Offset,
+                             const TargetRegisterInfo *TRI) const final;
 
   bool shouldClusterLoads(MachineInstr *FirstLdSt,
                           MachineInstr *SecondLdSt,
@@ -114,7 +117,7 @@ class SIInstrInfo : public AMDGPUInstrInfo {
   // register.  If there is no hardware instruction that can store to \p
   // DstRC, then AMDGPU::COPY is returned.
   unsigned getMovOpcode(const TargetRegisterClass *DstRC) const;
-  unsigned commuteOpcode(const MachineInstr &MI) const;
+  int commuteOpcode(const MachineInstr &MI) const;
 
   MachineInstr *commuteInstruction(MachineInstr *MI,
                                    bool NewMI = false) const override;
@@ -122,9 +125,6 @@ class SIInstrInfo : public AMDGPUInstrInfo {
                              unsigned &SrcOpIdx1,
                              unsigned &SrcOpIdx2) const override;
 
-  bool isTriviallyReMaterializable(const MachineInstr *MI,
-                                   AliasAnalysis *AA = nullptr) const;
-
   bool areMemAccessesTriviallyDisjoint(
     MachineInstr *MIa, MachineInstr *MIb,
     AliasAnalysis *AA = nullptr) const override;
@@ -139,6 +139,12 @@ class SIInstrInfo : public AMDGPUInstrInfo {
   bool FoldImmediate(MachineInstr *UseMI, MachineInstr *DefMI,
                      unsigned Reg, MachineRegisterInfo *MRI) const final;
 
+  unsigned getMachineCSELookAheadLimit() const override { return 500; }
+
+  MachineInstr *convertToThreeAddress(MachineFunction::iterator &MBB,
+                                      MachineBasicBlock::iterator &MI,
+                                      LiveVariables *LV) const override;
+
   bool isSALU(uint16_t Opcode) const {
     return get(Opcode).TSFlags & SIInstrFlags::SALU;
   }
@@ -211,6 +217,10 @@ class SIInstrInfo : public AMDGPUInstrInfo {
     return get(Opcode).TSFlags & SIInstrFlags::WQM;
   }
 
+  bool isVGPRSpill(uint16_t Opcode) const {
+    return get(Opcode).TSFlags & SIInstrFlags::VGPRSpill;
+  }
+
   bool isInlineConstant(const APInt &Imm) const;
   bool isInlineConstant(const MachineOperand &MO, unsigned OpSize) const;
   bool isLiteralConstant(const MachineOperand &MO, unsigned OpSize) const;
diff --git a/lib/Target/R600/SIInstrInfo.td b/lib/Target/AMDGPU/SIInstrInfo.td
similarity index 90%
rename from lib/Target/R600/SIInstrInfo.td
rename to lib/Target/AMDGPU/SIInstrInfo.td
index 076a0ce4e1b..b39a7871464 100644
--- a/lib/Target/R600/SIInstrInfo.td
+++ b/lib/Target/AMDGPU/SIInstrInfo.td
@@ -6,16 +6,14 @@
 // License. See LICENSE.TXT for details.
 //
 //===----------------------------------------------------------------------===//
-def isSICI : Predicate<
-  "Subtarget->getGeneration() == AMDGPUSubtarget::SOUTHERN_ISLANDS ||"
-  "Subtarget->getGeneration() == AMDGPUSubtarget::SEA_ISLANDS"
->, AssemblerPredicate<"FeatureGCN1Encoding">;
 def isCI : Predicate<"Subtarget->getGeneration() "
                       ">= AMDGPUSubtarget::SEA_ISLANDS">;
 def isVI : Predicate <
   "Subtarget->getGeneration() >= AMDGPUSubtarget::VOLCANIC_ISLANDS">,
   AssemblerPredicate<"FeatureGCN3Encoding">;
 
+def DisableInst : Predicate <"false">, AssemblerPredicate<"FeatureDisable">;
+
 class vop {
   field bits<9> SI3;
   field bits<10> VI3;
@@ -126,9 +124,111 @@ def SIconstdata_ptr : SDNode<
   "AMDGPUISD::CONST_DATA_PTR", SDTypeProfile <1, 0, [SDTCisVT<0, i64>]>
 >;
 
+//===----------------------------------------------------------------------===//
+// SDNodes and PatFrag for local loads and stores to enable s_mov_b32 m0, -1
+// to be glued to the memory instructions.
+//===----------------------------------------------------------------------===//
+
+def SIld_local : SDNode <"ISD::LOAD", SDTLoad,
+  [SDNPHasChain, SDNPMayLoad, SDNPMemOperand, SDNPInGlue]
+>;
+
+def si_ld_local : PatFrag <(ops node:$ptr), (SIld_local node:$ptr), [{
+  return isLocalLoad(cast<LoadSDNode>(N));
+}]>;
+
+def si_load_local : PatFrag <(ops node:$ptr), (si_ld_local node:$ptr), [{
+  return cast<LoadSDNode>(N)->getAddressingMode() == ISD::UNINDEXED &&
+         cast<LoadSDNode>(N)->getExtensionType() == ISD::NON_EXTLOAD;
+}]>;
+
+def si_load_local_align8 : Aligned8Bytes <
+  (ops node:$ptr), (si_load_local node:$ptr)
+>;
+
+def si_sextload_local : PatFrag <(ops node:$ptr), (si_ld_local node:$ptr), [{
+  return cast<LoadSDNode>(N)->getExtensionType() == ISD::SEXTLOAD;
+}]>;
+def si_az_extload_local : AZExtLoadBase <si_ld_local>;
+
+multiclass SIExtLoadLocal <PatFrag ld_node> {
+
+  def _i8 : PatFrag <(ops node:$ptr), (ld_node node:$ptr),
+                     [{return cast<LoadSDNode>(N)->getMemoryVT() == MVT::i8;}]
+  >;
+
+  def _i16 : PatFrag <(ops node:$ptr), (ld_node node:$ptr),
+                     [{return cast<LoadSDNode>(N)->getMemoryVT() == MVT::i16;}]
+  >;
+}
+
+defm si_sextload_local : SIExtLoadLocal <si_sextload_local>;
+defm si_az_extload_local : SIExtLoadLocal <si_az_extload_local>;
+
+def SIst_local : SDNode <"ISD::STORE", SDTStore,
+  [SDNPHasChain, SDNPMayStore, SDNPMemOperand, SDNPInGlue]
+>;
+
+def si_st_local : PatFrag <
+  (ops node:$val, node:$ptr), (SIst_local node:$val, node:$ptr), [{
+  return isLocalStore(cast<StoreSDNode>(N));
+}]>;
+
+def si_store_local : PatFrag <
+  (ops node:$val, node:$ptr), (si_st_local node:$val, node:$ptr), [{
+  return cast<StoreSDNode>(N)->getAddressingMode() == ISD::UNINDEXED &&
+         !cast<StoreSDNode>(N)->isTruncatingStore();
+}]>;
+
+def si_store_local_align8 : Aligned8Bytes <
+  (ops node:$val, node:$ptr), (si_store_local node:$val, node:$ptr)
+>;
+
+def si_truncstore_local : PatFrag <
+  (ops node:$val, node:$ptr), (si_st_local node:$val, node:$ptr), [{
+  return cast<StoreSDNode>(N)->isTruncatingStore();
+}]>;
+
+def si_truncstore_local_i8 : PatFrag <
+  (ops node:$val, node:$ptr), (si_truncstore_local node:$val, node:$ptr), [{
+  return cast<StoreSDNode>(N)->getMemoryVT() == MVT::i8;
+}]>;
+
+def si_truncstore_local_i16 : PatFrag <
+  (ops node:$val, node:$ptr), (si_truncstore_local node:$val, node:$ptr), [{
+  return cast<StoreSDNode>(N)->getMemoryVT() == MVT::i16;
+}]>;
+
+multiclass SIAtomicM0Glue2 <string op_name> {
+
+  def _glue : SDNode <"ISD::ATOMIC_"#op_name, SDTAtomic2,
+    [SDNPHasChain, SDNPMayStore, SDNPMayLoad, SDNPMemOperand, SDNPInGlue]
+  >;
+
+  def _local : local_binary_atomic_op <!cast<SDNode>(NAME#"_glue")>;
+}
+
+defm si_atomic_load_add : SIAtomicM0Glue2 <"LOAD_ADD">;
+defm si_atomic_load_and : SIAtomicM0Glue2 <"LOAD_AND">;
+defm si_atomic_load_min : SIAtomicM0Glue2 <"LOAD_MIN">;
+defm si_atomic_load_max : SIAtomicM0Glue2 <"LOAD_MAX">;
+defm si_atomic_load_or : SIAtomicM0Glue2 <"LOAD_OR">;
+defm si_atomic_load_sub : SIAtomicM0Glue2 <"LOAD_SUB">;
+defm si_atomic_load_xor : SIAtomicM0Glue2 <"LOAD_XOR">;
+defm si_atomic_load_umin : SIAtomicM0Glue2 <"LOAD_UMIN">;
+defm si_atomic_load_umax : SIAtomicM0Glue2 <"LOAD_UMAX">;
+defm si_atomic_swap : SIAtomicM0Glue2 <"SWAP">;
+
+def si_atomic_cmp_swap_glue : SDNode <"ISD::ATOMIC_CMP_SWAP", SDTAtomic3,
+  [SDNPHasChain, SDNPMayStore, SDNPMayLoad, SDNPMemOperand, SDNPInGlue]
+>;
+
+defm si_atomic_cmp_swap : AtomicCmpSwapLocal <si_atomic_cmp_swap_glue>;
+
 // Transformation function, extract the lower 32bit of a 64bit immediate
 def LO32 : SDNodeXForm<imm, [{
-  return CurDAG->getTargetConstant(N->getZExtValue() & 0xffffffff, MVT::i32);
+  return CurDAG->getTargetConstant(N->getZExtValue() & 0xffffffff, SDLoc(N),
+                                   MVT::i32);
 }]>;
 
 def LO32f : SDNodeXForm<fpimm, [{
@@ -138,12 +238,13 @@ def LO32f : SDNodeXForm<fpimm, [{
 
 // Transformation function, extract the upper 32bit of a 64bit immediate
 def HI32 : SDNodeXForm<imm, [{
-  return CurDAG->getTargetConstant(N->getZExtValue() >> 32, MVT::i32);
+  return CurDAG->getTargetConstant(N->getZExtValue() >> 32, SDLoc(N), MVT::i32);
 }]>;
 
 def HI32f : SDNodeXForm<fpimm, [{
   APInt V = N->getValueAPF().bitcastToAPInt().lshr(32).trunc(32);
-  return CurDAG->getTargetConstantFP(APFloat(APFloat::IEEEsingle, V), MVT::f32);
+  return CurDAG->getTargetConstantFP(APFloat(APFloat::IEEEsingle, V), SDLoc(N),
+                                     MVT::f32);
 }]>;
 
 def IMM8bitDWORD : PatLeaf <(imm),
@@ -151,39 +252,39 @@ def IMM8bitDWORD : PatLeaf <(imm),
 >;
 
 def as_dword_i32imm : SDNodeXForm<imm, [{
-  return CurDAG->getTargetConstant(N->getZExtValue() >> 2, MVT::i32);
+  return CurDAG->getTargetConstant(N->getZExtValue() >> 2, SDLoc(N), MVT::i32);
 }]>;
 
 def as_i1imm : SDNodeXForm<imm, [{
-  return CurDAG->getTargetConstant(N->getZExtValue(), MVT::i1);
+  return CurDAG->getTargetConstant(N->getZExtValue(), SDLoc(N), MVT::i1);
 }]>;
 
 def as_i8imm : SDNodeXForm<imm, [{
-  return CurDAG->getTargetConstant(N->getZExtValue(), MVT::i8);
+  return CurDAG->getTargetConstant(N->getZExtValue(), SDLoc(N), MVT::i8);
 }]>;
 
 def as_i16imm : SDNodeXForm<imm, [{
-  return CurDAG->getTargetConstant(N->getSExtValue(), MVT::i16);
+  return CurDAG->getTargetConstant(N->getSExtValue(), SDLoc(N), MVT::i16);
 }]>;
 
 def as_i32imm: SDNodeXForm<imm, [{
-  return CurDAG->getTargetConstant(N->getSExtValue(), MVT::i32);
+  return CurDAG->getTargetConstant(N->getSExtValue(), SDLoc(N), MVT::i32);
 }]>;
 
 def as_i64imm: SDNodeXForm<imm, [{
-  return CurDAG->getTargetConstant(N->getSExtValue(), MVT::i64);
+  return CurDAG->getTargetConstant(N->getSExtValue(), SDLoc(N), MVT::i64);
 }]>;
 
 // Copied from the AArch64 backend:
 def bitcast_fpimm_to_i32 : SDNodeXForm<fpimm, [{
 return CurDAG->getTargetConstant(
-  N->getValueAPF().bitcastToAPInt().getZExtValue(), MVT::i32);
+  N->getValueAPF().bitcastToAPInt().getZExtValue(), SDLoc(N), MVT::i32);
 }]>;
 
 // Copied from the AArch64 backend:
 def bitcast_fpimm_to_i64 : SDNodeXForm<fpimm, [{
 return CurDAG->getTargetConstant(
-  N->getValueAPF().bitcastToAPInt().getZExtValue(), MVT::i64);
+  N->getValueAPF().bitcastToAPInt().getZExtValue(), SDLoc(N), MVT::i64);
 }]>;
 
 def IMM8bit : PatLeaf <(imm),
@@ -289,27 +390,38 @@ class GDSBaseMatchClass <string parser> : AsmOperandClass {
 def GDSMatchClass : GDSBaseMatchClass <"parseDSOptionalOps">;
 def GDS01MatchClass : GDSBaseMatchClass <"parseDSOff01OptionalOps">;
 
-def GLCMatchClass : AsmOperandClass {
-  let Name = "GLC";
+class GLCBaseMatchClass <string parser> : AsmOperandClass {
+  let Name = "GLC"#parser;
   let PredicateMethod = "isImm";
-  let ParserMethod = "parseMubufOptionalOps";
+  let ParserMethod = parser; 
   let RenderMethod = "addImmOperands";
 }
 
-def SLCMatchClass : AsmOperandClass {
-  let Name = "SLC";
+def GLCMubufMatchClass : GLCBaseMatchClass <"parseMubufOptionalOps">;
+def GLCFlatMatchClass : GLCBaseMatchClass <"parseFlatOptionalOps">;
+
+class SLCBaseMatchClass <string parser> : AsmOperandClass {
+  let Name = "SLC"#parser;
   let PredicateMethod = "isImm";
-  let ParserMethod = "parseMubufOptionalOps";
+  let ParserMethod = parser;
   let RenderMethod = "addImmOperands";
 }
 
-def TFEMatchClass : AsmOperandClass {
-  let Name = "TFE";
+def SLCMubufMatchClass : SLCBaseMatchClass <"parseMubufOptionalOps">;
+def SLCFlatMatchClass : SLCBaseMatchClass <"parseFlatOptionalOps">;
+def SLCFlatAtomicMatchClass : SLCBaseMatchClass <"parseFlatAtomicOptionalOps">;
+
+class TFEBaseMatchClass <string parser> : AsmOperandClass {
+  let Name = "TFE"#parser;
   let PredicateMethod = "isImm";
-  let ParserMethod = "parseMubufOptionalOps";
+  let ParserMethod = parser;
   let RenderMethod = "addImmOperands";
 }
 
+def TFEMubufMatchClass : TFEBaseMatchClass <"parseMubufOptionalOps">;
+def TFEFlatMatchClass : TFEBaseMatchClass <"parseFlatOptionalOps">;
+def TFEFlatAtomicMatchClass : TFEBaseMatchClass <"parseFlatAtomicOptionalOps">;
+
 def OModMatchClass : AsmOperandClass {
   let Name = "OMod";
   let PredicateMethod = "isImm";
@@ -362,19 +474,32 @@ def gds : gds_base <GDSMatchClass>;
 
 def gds01 : gds_base <GDS01MatchClass>;
 
-def glc : Operand <i1> {
+class glc_base <AsmOperandClass mc> : Operand <i1> {
   let PrintMethod = "printGLC";
-  let ParserMatchClass = GLCMatchClass;
+  let ParserMatchClass = mc;
 }
-def slc : Operand <i1> {
+
+def glc : glc_base <GLCMubufMatchClass>;
+def glc_flat : glc_base <GLCFlatMatchClass>;
+
+class slc_base <AsmOperandClass mc> : Operand <i1> {
   let PrintMethod = "printSLC";
-  let ParserMatchClass = SLCMatchClass;
+  let ParserMatchClass = mc;
 }
-def tfe : Operand <i1> {
+
+def slc : slc_base <SLCMubufMatchClass>;
+def slc_flat : slc_base <SLCFlatMatchClass>;
+def slc_flat_atomic : slc_base <SLCFlatAtomicMatchClass>;
+
+class tfe_base <AsmOperandClass mc> : Operand <i1> {
   let PrintMethod = "printTFE";
-  let ParserMatchClass = TFEMatchClass;
+  let ParserMatchClass = mc;
 }
 
+def tfe : tfe_base <TFEMubufMatchClass>;
+def tfe_flat : tfe_base <TFEFlatMatchClass>;
+def tfe_flat_atomic : tfe_base <TFEFlatAtomicMatchClass>;
+
 def omod : Operand <i32> {
   let PrintMethod = "printOModSI";
   let ParserMatchClass = OModMatchClass;
@@ -404,9 +529,11 @@ def MUBUFOffset : ComplexPattern<i64, 6, "SelectMUBUFOffset">;
 def MUBUFOffsetAtomic : ComplexPattern<i64, 4, "SelectMUBUFOffset">;
 
 def VOP3Mods0 : ComplexPattern<untyped, 4, "SelectVOP3Mods0">;
+def VOP3NoMods0 : ComplexPattern<untyped, 4, "SelectVOP3NoMods0">;
 def VOP3Mods0Clamp : ComplexPattern<untyped, 3, "SelectVOP3Mods0Clamp">;
 def VOP3Mods0Clamp0OMod : ComplexPattern<untyped, 4, "SelectVOP3Mods0Clamp0OMod">;
 def VOP3Mods  : ComplexPattern<untyped, 2, "SelectVOP3Mods">;
+def VOP3NoMods : ComplexPattern<untyped, 2, "SelectVOP3NoMods">;
 
 //===----------------------------------------------------------------------===//
 // SI assembler operands
@@ -933,6 +1060,16 @@ class VOPProfile <list<ValueType> _ArgVT> {
   field string Asm64 = getAsm64<NumSrcArgs, HasModifiers>.ret;
 }
 
+// FIXME: I think these F16/I16 profiles will need to use f16/i16 types in order
+//        for the instruction patterns to work.
+def VOP_F16_F16 : VOPProfile <[f32, f32, untyped, untyped]>;
+def VOP_F16_I16 : VOPProfile <[f32, i32, untyped, untyped]>;
+def VOP_I16_F16 : VOPProfile <[i32, f32, untyped, untyped]>;
+
+def VOP_F16_F16_F16 : VOPProfile <[f32, f32, f32, untyped]>;
+def VOP_F16_F16_I16 : VOPProfile <[f32, f32, i32, untyped]>;
+def VOP_I16_I16_I16 : VOPProfile <[i32, i32, i32, untyped]>;
+
 def VOP_F32_F32 : VOPProfile <[f32, f32, untyped, untyped]>;
 def VOP_F32_F64 : VOPProfile <[f32, f64, untyped, untyped]>;
 def VOP_F32_I32 : VOPProfile <[f32, i32, untyped, untyped]>;
@@ -978,6 +1115,13 @@ def VOP_MADK : VOPProfile <[f32, f32, f32, f32]> {
   field dag Ins = (ins VCSrc_32:$src0, VGPR_32:$vsrc1, u32imm:$src2);
   field string Asm = "$dst, $src0, $vsrc1, $src2";
 }
+def VOP_MAC : VOPProfile <[f32, f32, f32, f32]> {
+  let Ins32 = (ins Src0RC32:$src0, Src1RC32:$src1, VGPR_32:$src2);
+  let Ins64 = getIns64<Src0RC64, Src1RC64, RegisterOperand<VGPR_32>, 3,
+                             HasModifiers>.ret;
+  let Asm32 = getAsm32<2>.ret;
+  let Asm64 = getAsm64<2, HasModifiers>.ret;
+}
 def VOP_F64_F64_F64_F64 : VOPProfile <[f64, f64, f64, f64]>;
 def VOP_I32_I32_I32_I32 : VOPProfile <[i32, i32, i32, i32]>;
 def VOP_I64_I32_I32_I64 : VOPProfile <[i64, i32, i32, i64]>;
@@ -1011,11 +1155,15 @@ class VOP1_Pseudo <dag outs, dag ins, list<dag> pattern, string opName> :
 
 class VOP1_Real_si <string opName, vop1 op, dag outs, dag ins, string asm> :
   VOP1<op.SI, outs, ins, asm, []>,
-  SIMCInstr <opName#"_e32", SISubtarget.SI>;
+  SIMCInstr <opName#"_e32", SISubtarget.SI> {
+  let AssemblerPredicate = SIAssemblerPredicate;
+}
 
 class VOP1_Real_vi <string opName, vop1 op, dag outs, dag ins, string asm> :
   VOP1<op.VI, outs, ins, asm, []>,
-  SIMCInstr <opName#"_e32", SISubtarget.VI>;
+  SIMCInstr <opName#"_e32", SISubtarget.VI> {
+  let AssemblerPredicates = [isVI];
+}
 
 multiclass VOP1_m <vop1 op, dag outs, dag ins, string asm, list<dag> pattern,
                    string opName> {
@@ -1422,12 +1570,16 @@ let isCodeGenOnly = 0 in {
   def _si : VOP2Common <VOP_MADK.Outs, VOP_MADK.Ins,
                         !strconcat(opName, VOP_MADK.Asm), []>,
             SIMCInstr <opName#"_e32", SISubtarget.SI>,
-            VOP2_MADKe <op.SI>;
+            VOP2_MADKe <op.SI> {
+            let AssemblerPredicates = [isSICI];
+            }
 
   def _vi : VOP2Common <VOP_MADK.Outs, VOP_MADK.Ins,
                         !strconcat(opName, VOP_MADK.Asm), []>,
             SIMCInstr <opName#"_e32", SISubtarget.VI>,
-            VOP2_MADKe <op.VI>;
+            VOP2_MADKe <op.VI> {
+            let AssemblerPredicates = [isVI];
+            }
 } // End isCodeGenOnly = 0
 }
 
@@ -1597,7 +1749,7 @@ multiclass VOP3_VCC_Inst <vop3 op, string opName,
        InputModsNoDefault:$src2_modifiers, P.Src2RC64:$src2,
        ClampMod:$clamp,
        omod:$omod),
-  " $dst, $src0_modifiers, $src1_modifiers, $src2_modifiers"#"$clamp"#"$omod",
+  "$dst, $src0_modifiers, $src1_modifiers, $src2_modifiers"#"$clamp"#"$omod",
   [(set P.DstVT:$dst,
             (node (P.Src0VT (VOP3Mods0 P.Src0VT:$src0, i32:$src0_modifiers,
                                        i1:$clamp, i32:$omod)),
@@ -1659,17 +1811,13 @@ class VINTRP_Real_vi <bits <2> op, string opName, dag outs, dag ins,
   VINTRPe_vi <op>,
   SIMCInstr<opName, SISubtarget.VI>;
 
-multiclass VINTRP_m <bits <2> op, string opName, dag outs, dag ins, string asm,
-                     string disableEncoding = "", string constraints = "",
+multiclass VINTRP_m <bits <2> op, dag outs, dag ins, string asm,
                      list<dag> pattern = []> {
-  let DisableEncoding = disableEncoding,
-      Constraints = constraints in {
-    def "" : VINTRP_Pseudo <opName, outs, ins, pattern>;
+  def "" : VINTRP_Pseudo <NAME, outs, ins, pattern>;
 
-    def _si : VINTRP_Real_si <op, opName, outs, ins, asm>;
+  def _si : VINTRP_Real_si <op, NAME, outs, ins, asm>;
 
-    def _vi : VINTRP_Real_vi <op, opName, outs, ins, asm>;
-  }
+  def _vi : VINTRP_Real_vi <op, NAME, outs, ins, asm>;
 }
 
 //===----------------------------------------------------------------------===//
@@ -1716,7 +1864,7 @@ class DS_Off16_Real_vi <bits<8> op, string opName, dag outs, dag ins, string asm
 
 multiclass DS_1A_RET <bits<8> op, string opName, RegisterClass rc,
   dag outs = (outs rc:$vdst),
-  dag ins = (ins VGPR_32:$addr, ds_offset:$offset, gds:$gds, M0Reg:$m0),
+  dag ins = (ins VGPR_32:$addr, ds_offset:$offset, gds:$gds),
   string asm = opName#" $vdst, $addr"#"$offset$gds"> {
 
   def "" : DS_Pseudo <opName, outs, ins, []>;
@@ -1730,7 +1878,7 @@ multiclass DS_1A_RET <bits<8> op, string opName, RegisterClass rc,
 multiclass DS_1A_Off8_RET <bits<8> op, string opName, RegisterClass rc,
   dag outs = (outs rc:$vdst),
   dag ins = (ins VGPR_32:$addr, ds_offset0:$offset0, ds_offset1:$offset1,
-                 gds01:$gds, M0Reg:$m0),
+                 gds01:$gds),
   string asm = opName#" $vdst, $addr"#"$offset0"#"$offset1$gds"> {
 
   def "" : DS_Pseudo <opName, outs, ins, []>;
@@ -1743,8 +1891,7 @@ multiclass DS_1A_Off8_RET <bits<8> op, string opName, RegisterClass rc,
 
 multiclass DS_1A1D_NORET <bits<8> op, string opName, RegisterClass rc,
   dag outs = (outs),
-  dag ins = (ins VGPR_32:$addr, rc:$data0, ds_offset:$offset, gds:$gds,
-                 M0Reg:$m0),
+  dag ins = (ins VGPR_32:$addr, rc:$data0, ds_offset:$offset, gds:$gds),
   string asm = opName#" $addr, $data0"#"$offset$gds"> {
 
   def "" : DS_Pseudo <opName, outs, ins, []>,
@@ -1759,7 +1906,7 @@ multiclass DS_1A1D_NORET <bits<8> op, string opName, RegisterClass rc,
 multiclass DS_1A1D_Off8_NORET <bits<8> op, string opName, RegisterClass rc,
   dag outs = (outs),
   dag ins = (ins VGPR_32:$addr, rc:$data0, rc:$data1,
-              ds_offset0:$offset0, ds_offset1:$offset1, gds01:$gds, M0Reg:$m0),
+              ds_offset0:$offset0, ds_offset1:$offset1, gds01:$gds),
   string asm = opName#" $addr, $data0, $data1"#"$offset0"#"$offset1"#"$gds"> {
 
   def "" : DS_Pseudo <opName, outs, ins, []>;
@@ -1773,8 +1920,7 @@ multiclass DS_1A1D_Off8_NORET <bits<8> op, string opName, RegisterClass rc,
 multiclass DS_1A1D_RET <bits<8> op, string opName, RegisterClass rc,
                         string noRetOp = "",
   dag outs = (outs rc:$vdst),
-  dag ins = (ins VGPR_32:$addr, rc:$data0, ds_offset:$offset, gds:$gds,
-                 M0Reg:$m0),
+  dag ins = (ins VGPR_32:$addr, rc:$data0, ds_offset:$offset, gds:$gds),
   string asm = opName#" $vdst, $addr, $data0"#"$offset$gds"> {
 
   def "" : DS_Pseudo <opName, outs, ins, []>,
@@ -1802,14 +1948,14 @@ multiclass DS_1A2D_RET <bits<8> op, string asm, RegisterClass rc,
                         string noRetOp = "", RegisterClass src = rc> :
   DS_1A2D_RET_m <op, asm, rc, noRetOp,
                  (ins VGPR_32:$addr, src:$data0, src:$data1,
-                      ds_offset:$offset, gds:$gds, M0Reg:$m0)
+                      ds_offset:$offset, gds:$gds)
 >;
 
 multiclass DS_1A2D_NORET <bits<8> op, string opName, RegisterClass rc,
                           string noRetOp = opName,
   dag outs = (outs),
   dag ins = (ins VGPR_32:$addr, rc:$data0, rc:$data1,
-                 ds_offset:$offset, gds:$gds, M0Reg:$m0),
+                 ds_offset:$offset, gds:$gds),
   string asm = opName#" $addr, $data0, $data1"#"$offset"#"$gds"> {
 
   def "" : DS_Pseudo <opName, outs, ins, []>,
@@ -1823,7 +1969,7 @@ multiclass DS_1A2D_NORET <bits<8> op, string opName, RegisterClass rc,
 
 multiclass DS_0A_RET <bits<8> op, string opName,
   dag outs = (outs VGPR_32:$vdst),
-  dag ins = (ins ds_offset:$offset, gds:$gds, M0Reg:$m0),
+  dag ins = (ins ds_offset:$offset, gds:$gds),
   string asm = opName#" $vdst"#"$offset"#"$gds"> {
 
   let mayLoad = 1, mayStore = 1 in {
@@ -1838,7 +1984,7 @@ multiclass DS_0A_RET <bits<8> op, string opName,
 
 multiclass DS_1A_RET_GDS <bits<8> op, string opName,
   dag outs = (outs VGPR_32:$vdst),
-  dag ins = (ins VGPR_32:$addr, ds_offset_gds:$offset, M0Reg:$m0),
+  dag ins = (ins VGPR_32:$addr, ds_offset_gds:$offset),
   string asm = opName#" $vdst, $addr"#"$offset gds"> {
 
   def "" : DS_Pseudo <opName, outs, ins, []>;
@@ -1851,7 +1997,7 @@ multiclass DS_1A_RET_GDS <bits<8> op, string opName,
 
 multiclass DS_1A_GDS <bits<8> op, string opName,
   dag outs = (outs),
-  dag ins = (ins VGPR_32:$addr, M0Reg:$m0),
+  dag ins = (ins VGPR_32:$addr),
   string asm = opName#" $addr gds"> {
 
   def "" : DS_Pseudo <opName, outs, ins, []>;
@@ -1864,7 +2010,7 @@ multiclass DS_1A_GDS <bits<8> op, string opName,
 
 multiclass DS_1A <bits<8> op, string opName,
   dag outs = (outs),
-  dag ins = (ins VGPR_32:$addr, ds_offset:$offset, M0Reg:$m0, gds:$gds),
+  dag ins = (ins VGPR_32:$addr, ds_offset:$offset, gds:$gds),
   string asm = opName#" $addr"#"$offset"#"$gds"> {
 
   let mayLoad = 1, mayStore = 1 in {
@@ -2222,30 +2368,48 @@ multiclass MUBUF_Store_Helper <mubuf op, string name, RegisterClass vdataClass,
 
 class FLAT_Load_Helper <bits<7> op, string asm, RegisterClass regClass> :
       FLAT <op, (outs regClass:$vdst),
-                (ins VReg_64:$addr),
-            asm#" $vdst, $addr, [M0, FLAT_SCRATCH]", []> {
-  let glc = 0;
-  let slc = 0;
-  let tfe = 0;
+                (ins VReg_64:$addr, glc_flat:$glc, slc_flat:$slc, tfe_flat:$tfe),
+            asm#" $vdst, $addr"#"$glc"#"$slc"#"$tfe", []> {
   let data = 0;
   let mayLoad = 1;
 }
 
 class FLAT_Store_Helper <bits<7> op, string name, RegisterClass vdataClass> :
-      FLAT <op, (outs), (ins vdataClass:$data, VReg_64:$addr),
-          name#" $data, $addr, [M0, FLAT_SCRATCH]",
+      FLAT <op, (outs), (ins vdataClass:$data, VReg_64:$addr,
+                             glc_flat:$glc, slc_flat:$slc, tfe_flat:$tfe),
+          name#" $data, $addr"#"$glc"#"$slc"#"$tfe",
          []> {
 
   let mayLoad = 0;
   let mayStore = 1;
 
   // Encoding
-  let glc = 0;
-  let slc = 0;
-  let tfe = 0;
   let vdst = 0;
 }
 
+multiclass FLAT_ATOMIC <bits<7> op, string name, RegisterClass vdst_rc,
+                        RegisterClass data_rc = vdst_rc> {
+
+  let mayLoad = 1, mayStore = 1 in {
+    def "" : FLAT <op, (outs),
+                  (ins VReg_64:$addr, data_rc:$data, slc_flat_atomic:$slc,
+                       tfe_flat_atomic:$tfe),
+                   name#" $addr, $data"#"$slc"#"$tfe", []>,
+             AtomicNoRet <NAME, 0> {
+      let glc = 0;
+      let vdst = 0;
+    }
+
+    def _RTN : FLAT <op, (outs vdst_rc:$vdst),
+                     (ins VReg_64:$addr, data_rc:$data, slc_flat_atomic:$slc,
+                          tfe_flat_atomic:$tfe),
+                     name#" $vdst, $addr, $data glc"#"$slc"#"$tfe", []>,
+               AtomicNoRet <NAME, 1> {
+      let glc = 1;
+    }
+  }
+}
+
 class MIMG_Mask <string op, int channels> {
   string Op = op;
   int Channels = channels;
diff --git a/lib/Target/R600/SIInstructions.td b/lib/Target/AMDGPU/SIInstructions.td
similarity index 91%
rename from lib/Target/R600/SIInstructions.td
rename to lib/Target/AMDGPU/SIInstructions.td
index 91e8c8c236b..019b1b4cc9a 100644
--- a/lib/Target/R600/SIInstructions.td
+++ b/lib/Target/AMDGPU/SIInstructions.td
@@ -32,7 +32,8 @@ def isGCN : Predicate<"Subtarget->getGeneration() "
 def isSI : Predicate<"Subtarget->getGeneration() "
                       "== AMDGPUSubtarget::SOUTHERN_ISLANDS">;
 
-def HasFlatAddressSpace : Predicate<"Subtarget.hasFlatAddressSpace()">;
+def has16BankLDS : Predicate<"Subtarget->getLDSBankCount() == 16">;
+def has32BankLDS : Predicate<"Subtarget->getLDSBankCount() == 32">;
 
 def SWaitMatchClass : AsmOperandClass {
   let Name = "SWaitCnt";
@@ -97,7 +98,7 @@ defm S_BUFFER_LOAD_DWORDX16 : SMRD_Helper <
 //===----------------------------------------------------------------------===//
 
 let isMoveImm = 1 in {
-  let isReMaterializable = 1 in {
+  let isReMaterializable = 1, isAsCheapAsAMove = 1 in {
     defm S_MOV_B32 : SOP1_32 <sop1<0x03, 0x00>, "s_mov_b32", []>;
     defm S_MOV_B64 : SOP1_64 <sop1<0x04, 0x01>, "s_mov_b64", []>;
   } // let isRematerializeable = 1
@@ -221,16 +222,16 @@ defm S_SUBB_U32 : SOP2_32 <sop2<0x05>, "s_subb_u32",
 } // End Uses = [SCC]
 
 defm S_MIN_I32 : SOP2_32 <sop2<0x06>, "s_min_i32",
-  [(set i32:$dst, (AMDGPUsmin i32:$src0, i32:$src1))]
+  [(set i32:$dst, (smin i32:$src0, i32:$src1))]
 >;
 defm S_MIN_U32 : SOP2_32 <sop2<0x07>, "s_min_u32",
-  [(set i32:$dst, (AMDGPUumin i32:$src0, i32:$src1))]
+  [(set i32:$dst, (umin i32:$src0, i32:$src1))]
 >;
 defm S_MAX_I32 : SOP2_32 <sop2<0x08>, "s_max_i32",
-  [(set i32:$dst, (AMDGPUsmax i32:$src0, i32:$src1))]
+  [(set i32:$dst, (smax i32:$src0, i32:$src1))]
 >;
 defm S_MAX_U32 : SOP2_32 <sop2<0x09>, "s_max_u32",
-  [(set i32:$dst, (AMDGPUumax i32:$src0, i32:$src1))]
+  [(set i32:$dst, (umax i32:$src0, i32:$src1))]
 >;
 } // End Defs = [SCC]
 
@@ -353,7 +354,7 @@ def S_CMP_LE_U32 : SOPC_32 <0x0000000b, "s_cmp_le_u32">;
 // SOPK Instructions
 //===----------------------------------------------------------------------===//
 
-let isReMaterializable = 1 in {
+let isReMaterializable = 1, isMoveImm = 1 in {
 defm S_MOVK_I32 : SOPK_32 <sopk<0x00>, "s_movk_i32", []>;
 } // End isReMaterializable = 1
 let Uses = [SCC] in {
@@ -488,13 +489,11 @@ def S_SETHALT : SOPP <0x0000000d, (ins i16imm:$simm16), "s_sethalt $simm16">;
 def S_SLEEP : SOPP <0x0000000e, (ins i16imm:$simm16), "s_sleep $simm16">;
 def S_SETPRIO : SOPP <0x0000000f, (ins i16imm:$sim16), "s_setprio $sim16">;
 
-let Uses = [EXEC] in {
-  def S_SENDMSG : SOPP <0x00000010, (ins SendMsgImm:$simm16, M0Reg:$m0), "s_sendmsg $simm16",
-      [(int_SI_sendmsg imm:$simm16, M0Reg:$m0)]
-  > {
-    let DisableEncoding = "$m0";
-  }
-} // End Uses = [EXEC]
+let Uses = [EXEC, M0] in {
+  def S_SENDMSG : SOPP <0x00000010, (ins SendMsgImm:$simm16), "s_sendmsg $simm16",
+      [(AMDGPUsendmsg (i32 imm:$simm16))]
+  >;
+} // End Uses = [EXEC, M0]
 
 def S_SENDMSGHALT : SOPP <0x00000011, (ins i16imm:$simm16), "s_sendmsghalt $simm16">;
 def S_TRAP : SOPP <0x00000012, (ins i16imm:$simm16), "s_trap $simm16">;
@@ -1152,80 +1151,6 @@ defm IMAGE_SAMPLE_C_CD_CL_O : MIMG_Sampler <0x0000006f, "image_sample_c_cd_cl_o"
 //def IMAGE_RSRC256 : MIMG_NoPattern_RSRC256 <"image_rsrc256", 0x0000007e>;
 //def IMAGE_SAMPLER : MIMG_NoPattern_ <"image_sampler", 0x0000007f>;
 
-//===----------------------------------------------------------------------===//
-// Flat Instructions
-//===----------------------------------------------------------------------===//
-
-let Predicates = [HasFlatAddressSpace] in {
-def FLAT_LOAD_UBYTE : FLAT_Load_Helper <0x00000008, "flat_load_ubyte", VGPR_32>;
-def FLAT_LOAD_SBYTE : FLAT_Load_Helper <0x00000009, "flat_load_sbyte", VGPR_32>;
-def FLAT_LOAD_USHORT : FLAT_Load_Helper <0x0000000a, "flat_load_ushort", VGPR_32>;
-def FLAT_LOAD_SSHORT : FLAT_Load_Helper <0x0000000b, "flat_load_sshort", VGPR_32>;
-def FLAT_LOAD_DWORD : FLAT_Load_Helper <0x0000000c, "flat_load_dword", VGPR_32>;
-def FLAT_LOAD_DWORDX2 : FLAT_Load_Helper <0x0000000d, "flat_load_dwordx2", VReg_64>;
-def FLAT_LOAD_DWORDX4 : FLAT_Load_Helper <0x0000000e, "flat_load_dwordx4", VReg_128>;
-def FLAT_LOAD_DWORDX3 : FLAT_Load_Helper <0x00000010, "flat_load_dwordx3", VReg_96>;
-
-def FLAT_STORE_BYTE : FLAT_Store_Helper <
-  0x00000018, "flat_store_byte", VGPR_32
->;
-
-def FLAT_STORE_SHORT : FLAT_Store_Helper <
-  0x0000001a, "flat_store_short", VGPR_32
->;
-
-def FLAT_STORE_DWORD : FLAT_Store_Helper <
-  0x0000001c, "flat_store_dword", VGPR_32
->;
-
-def FLAT_STORE_DWORDX2 : FLAT_Store_Helper <
-  0x0000001d, "flat_store_dwordx2", VReg_64
->;
-
-def FLAT_STORE_DWORDX4 : FLAT_Store_Helper <
-  0x0000001e, "flat_store_dwordx4", VReg_128
->;
-
-def FLAT_STORE_DWORDX3 : FLAT_Store_Helper <
-  0x0000001e, "flat_store_dwordx3", VReg_96
->;
-
-//def FLAT_ATOMIC_SWAP : FLAT_ <0x00000030, "flat_atomic_swap", []>;
-//def FLAT_ATOMIC_CMPSWAP : FLAT_ <0x00000031, "flat_atomic_cmpswap", []>;
-//def FLAT_ATOMIC_ADD : FLAT_ <0x00000032, "flat_atomic_add", []>;
-//def FLAT_ATOMIC_SUB : FLAT_ <0x00000033, "flat_atomic_sub", []>;
-//def FLAT_ATOMIC_RSUB : FLAT_ <0x00000034, "flat_atomic_rsub", []>;
-//def FLAT_ATOMIC_SMIN : FLAT_ <0x00000035, "flat_atomic_smin", []>;
-//def FLAT_ATOMIC_UMIN : FLAT_ <0x00000036, "flat_atomic_umin", []>;
-//def FLAT_ATOMIC_SMAX : FLAT_ <0x00000037, "flat_atomic_smax", []>;
-//def FLAT_ATOMIC_UMAX : FLAT_ <0x00000038, "flat_atomic_umax", []>;
-//def FLAT_ATOMIC_AND : FLAT_ <0x00000039, "flat_atomic_and", []>;
-//def FLAT_ATOMIC_OR : FLAT_ <0x0000003a, "flat_atomic_or", []>;
-//def FLAT_ATOMIC_XOR : FLAT_ <0x0000003b, "flat_atomic_xor", []>;
-//def FLAT_ATOMIC_INC : FLAT_ <0x0000003c, "flat_atomic_inc", []>;
-//def FLAT_ATOMIC_DEC : FLAT_ <0x0000003d, "flat_atomic_dec", []>;
-//def FLAT_ATOMIC_FCMPSWAP : FLAT_ <0x0000003e, "flat_atomic_fcmpswap", []>;
-//def FLAT_ATOMIC_FMIN : FLAT_ <0x0000003f, "flat_atomic_fmin", []>;
-//def FLAT_ATOMIC_FMAX : FLAT_ <0x00000040, "flat_atomic_fmax", []>;
-//def FLAT_ATOMIC_SWAP_X2 : FLAT_X2 <0x00000050, "flat_atomic_swap_x2", []>;
-//def FLAT_ATOMIC_CMPSWAP_X2 : FLAT_X2 <0x00000051, "flat_atomic_cmpswap_x2", []>;
-//def FLAT_ATOMIC_ADD_X2 : FLAT_X2 <0x00000052, "flat_atomic_add_x2", []>;
-//def FLAT_ATOMIC_SUB_X2 : FLAT_X2 <0x00000053, "flat_atomic_sub_x2", []>;
-//def FLAT_ATOMIC_RSUB_X2 : FLAT_X2 <0x00000054, "flat_atomic_rsub_x2", []>;
-//def FLAT_ATOMIC_SMIN_X2 : FLAT_X2 <0x00000055, "flat_atomic_smin_x2", []>;
-//def FLAT_ATOMIC_UMIN_X2 : FLAT_X2 <0x00000056, "flat_atomic_umin_x2", []>;
-//def FLAT_ATOMIC_SMAX_X2 : FLAT_X2 <0x00000057, "flat_atomic_smax_x2", []>;
-//def FLAT_ATOMIC_UMAX_X2 : FLAT_X2 <0x00000058, "flat_atomic_umax_x2", []>;
-//def FLAT_ATOMIC_AND_X2 : FLAT_X2 <0x00000059, "flat_atomic_and_x2", []>;
-//def FLAT_ATOMIC_OR_X2 : FLAT_X2 <0x0000005a, "flat_atomic_or_x2", []>;
-//def FLAT_ATOMIC_XOR_X2 : FLAT_X2 <0x0000005b, "flat_atomic_xor_x2", []>;
-//def FLAT_ATOMIC_INC_X2 : FLAT_X2 <0x0000005c, "flat_atomic_inc_x2", []>;
-//def FLAT_ATOMIC_DEC_X2 : FLAT_X2 <0x0000005d, "flat_atomic_dec_x2", []>;
-//def FLAT_ATOMIC_FCMPSWAP_X2 : FLAT_X2 <0x0000005e, "flat_atomic_fcmpswap_x2", []>;
-//def FLAT_ATOMIC_FMIN_X2 : FLAT_X2 <0x0000005f, "flat_atomic_fmin_x2", []>;
-//def FLAT_ATOMIC_FMAX_X2 : FLAT_X2 <0x00000060, "flat_atomic_fmax_x2", []>;
-
-} // End HasFlatAddressSpace predicate
 //===----------------------------------------------------------------------===//
 // VOP1 Instructions
 //===----------------------------------------------------------------------===//
@@ -1234,7 +1159,7 @@ let vdst = 0, src0 = 0 in {
 defm V_NOP : VOP1_m <vop1<0x0>, (outs), (ins), "v_nop", [], "v_nop">;
 }
 
-let isMoveImm = 1 in {
+let isMoveImm = 1, isReMaterializable = 1, isAsCheapAsAMove = 1 in {
 defm V_MOV_B32 : VOP1Inst <vop1<0x1>, "v_mov_b32", VOP_I32_I32>;
 } // End isMoveImm = 1
 
@@ -1272,7 +1197,6 @@ defm V_CVT_U32_F32 : VOP1Inst <vop1<0x7>, "v_cvt_u32_f32",
 defm V_CVT_I32_F32 : VOP1Inst <vop1<0x8>, "v_cvt_i32_f32",
   VOP_I32_F32, fp_to_sint
 >;
-defm V_MOV_FED_B32 : VOP1Inst <vop1<0x9>, "v_mov_fed_b32", VOP_I32_I32>;
 defm V_CVT_F16_F32 : VOP1Inst <vop1<0xa>, "v_cvt_f16_f32",
   VOP_I32_F32, fp_to_f16
 >;
@@ -1408,6 +1332,7 @@ let SubtargetPredicate = isSICI in {
 
 let SchedRW = [WriteQuarterRate32] in {
 
+defm V_MOV_FED_B32 : VOP1InstSI <vop1<0x9>, "v_mov_fed_b32", VOP_I32_I32>;
 defm V_LOG_CLAMP_F32 : VOP1InstSI <vop1<0x26>, "v_log_clamp_f32", VOP_F32_F32>;
 defm V_RCP_CLAMP_F32 : VOP1InstSI <vop1<0x28>, "v_rcp_clamp_f32", VOP_F32_F32>;
 defm V_RCP_LEGACY_F32 : VOP1InstSI <vop1<0x29>, "v_rcp_legacy_f32", VOP_F32_F32>;
@@ -1435,28 +1360,52 @@ defm V_RSQ_CLAMP_F64 : VOP1InstSI <vop1<0x32>, "v_rsq_clamp_f64",
 // VINTRP Instructions
 //===----------------------------------------------------------------------===//
 
+let Uses = [M0] in {
+
 // FIXME: Specify SchedRW for VINTRP insturctions.
-defm V_INTERP_P1_F32 : VINTRP_m <
-  0x00000000, "v_interp_p1_f32",
+
+multiclass V_INTERP_P1_F32_m : VINTRP_m <
+  0x00000000,
   (outs VGPR_32:$dst),
-  (ins VGPR_32:$i, i32imm:$attr_chan, i32imm:$attr, M0Reg:$m0),
-  "v_interp_p1_f32 $dst, $i, $attr_chan, $attr, [$m0]",
-  "$m0">;
+  (ins VGPR_32:$i, i32imm:$attr_chan, i32imm:$attr),
+  "v_interp_p1_f32 $dst, $i, $attr_chan, $attr, [m0]",
+  [(set f32:$dst, (AMDGPUinterp_p1 i32:$i, (i32 imm:$attr_chan),
+                                           (i32 imm:$attr)))]
+>;
+
+let OtherPredicates = [has32BankLDS] in {
+
+defm V_INTERP_P1_F32 : V_INTERP_P1_F32_m;
+
+} // End OtherPredicates = [has32BankLDS]
+
+let OtherPredicates = [has16BankLDS], Constraints = "@earlyclobber $dst" in {
+
+defm V_INTERP_P1_F32_16bank : V_INTERP_P1_F32_m;
+
+} // End OtherPredicates = [has32BankLDS], Constraints = "@earlyclobber $dst"
+
+let DisableEncoding = "$src0", Constraints = "$src0 = $dst" in {
 
 defm V_INTERP_P2_F32 : VINTRP_m <
-  0x00000001, "v_interp_p2_f32",
+  0x00000001,
   (outs VGPR_32:$dst),
-  (ins VGPR_32:$src0, VGPR_32:$j, i32imm:$attr_chan, i32imm:$attr, M0Reg:$m0),
-  "v_interp_p2_f32 $dst, [$src0], $j, $attr_chan, $attr, [$m0]",
-  "$src0,$m0",
-  "$src0 = $dst">;
+  (ins VGPR_32:$src0, VGPR_32:$j, i32imm:$attr_chan, i32imm:$attr),
+  "v_interp_p2_f32 $dst, [$src0], $j, $attr_chan, $attr, [m0]",
+  [(set f32:$dst, (AMDGPUinterp_p2 f32:$src0, i32:$j, (i32 imm:$attr_chan),
+                                                     (i32 imm:$attr)))]>;
+
+} // End DisableEncoding = "$src0", Constraints = "$src0 = $dst"
 
 defm V_INTERP_MOV_F32 : VINTRP_m <
-  0x00000002, "v_interp_mov_f32",
+  0x00000002,
   (outs VGPR_32:$dst),
-  (ins InterpSlot:$src0, i32imm:$attr_chan, i32imm:$attr, M0Reg:$m0),
-  "v_interp_mov_f32 $dst, $src0, $attr_chan, $attr, [$m0]",
-  "$m0">;
+  (ins InterpSlot:$src0, i32imm:$attr_chan, i32imm:$attr),
+  "v_interp_mov_f32 $dst, $src0, $attr_chan, $attr, [m0]",
+  [(set f32:$dst, (AMDGPUinterp_mov (i32 imm:$src0), (i32 imm:$attr_chan),
+                                    (i32 imm:$attr)))]>;
+
+} // End Uses = [M0]
 
 //===----------------------------------------------------------------------===//
 // VOP2 Instructions
@@ -1539,7 +1488,10 @@ defm V_AND_B32 : VOP2Inst <vop2<0x1b, 0x13>, "v_and_b32", VOP_I32_I32_I32>;
 defm V_OR_B32 : VOP2Inst <vop2<0x1c, 0x14>, "v_or_b32", VOP_I32_I32_I32>;
 defm V_XOR_B32 : VOP2Inst <vop2<0x1d, 0x15>, "v_xor_b32", VOP_I32_I32_I32>;
 
-defm V_MAC_F32 : VOP2Inst <vop2<0x1f, 0x16>, "v_mac_f32", VOP_F32_F32_F32>;
+let Constraints = "$dst = $src2", DisableEncoding="$src2",
+    isConvertibleToThreeAddress = 1 in {
+defm V_MAC_F32 : VOP2Inst <vop2<0x1f, 0x16>, "v_mac_f32", VOP_MAC>;
+}
 } // End isCommutable = 1
 
 defm V_MADMK_F32 : VOP2MADK <vop2<0x20, 0x17>, "v_madmk_f32">;
@@ -1896,8 +1848,8 @@ def SGPR_USE : InstSI <(outs),(ins), "", []>;
 // SI pseudo instructions. These are used by the CFG structurizer pass
 // and should be lowered to ISA instructions prior to codegen.
 
-let mayLoad = 1, mayStore = 1, hasSideEffects = 1,
-    Uses = [EXEC], Defs = [EXEC] in {
+let mayLoad = 1, mayStore = 1, hasSideEffects = 1 in {
+let Uses = [EXEC], Defs = [EXEC] in {
 
 let isBranch = 1, isTerminator = 1 in {
 
@@ -1954,15 +1906,18 @@ def SI_END_CF : InstSI <
   [(int_SI_end_cf i64:$saved)]
 >;
 
+} // End Uses = [EXEC], Defs = [EXEC]
+
+let Uses = [EXEC], Defs = [EXEC,VCC] in {
 def SI_KILL : InstSI <
   (outs),
   (ins VSrc_32:$src),
   "si_kill $src",
   [(int_AMDGPU_kill f32:$src)]
 >;
+} // End Uses = [EXEC], Defs = [EXEC,VCC]
 
 } // end mayLoad = 1, mayStore = 1, hasSideEffects = 1
-  // Uses = [EXEC], Defs = [EXEC]
 
 let Uses = [EXEC], Defs = [EXEC,VCC,M0] in {
 
@@ -2038,14 +1993,17 @@ multiclass SI_SPILL_SGPR <RegisterClass sgpr_class> {
   } // End UseNamedOperandTable = 1
 }
 
-defm SI_SPILL_S32  : SI_SPILL_SGPR <SReg_32>;
+// It's unclear whether you can use M0 as the output of v_readlane_b32
+// instructions, so use SGPR_32 register class for spills to prevent
+// this from happening.
+defm SI_SPILL_S32  : SI_SPILL_SGPR <SGPR_32>;
 defm SI_SPILL_S64  : SI_SPILL_SGPR <SReg_64>;
 defm SI_SPILL_S128 : SI_SPILL_SGPR <SReg_128>;
 defm SI_SPILL_S256 : SI_SPILL_SGPR <SReg_256>;
 defm SI_SPILL_S512 : SI_SPILL_SGPR <SReg_512>;
 
 multiclass SI_SPILL_VGPR <RegisterClass vgpr_class> {
-  let UseNamedOperandTable = 1 in {
+  let UseNamedOperandTable = 1, VGPRSpill = 1 in {
     def _SAVE : InstSI <
       (outs),
       (ins vgpr_class:$src, i32imm:$frame_idx, SReg_128:$scratch_rsrc,
@@ -2058,7 +2016,7 @@ multiclass SI_SPILL_VGPR <RegisterClass vgpr_class> {
       (ins i32imm:$frame_idx, SReg_128:$scratch_rsrc, SReg_32:$scratch_offset),
       "", []
     >;
-  } // End UseNamedOperandTable = 1
+  } // End UseNamedOperandTable = 1, VGPRSpill = 1
 }
 
 defm SI_SPILL_V32  : SI_SPILL_VGPR <VGPR_32>;
@@ -2157,7 +2115,6 @@ multiclass SMRD_Pattern_vi <SMRD Instr_IMM, SMRD Instr_SGPR, ValueType vt> {
 }
 
 let Predicates = [isSICI] in {
-defm : SMRD_Pattern <S_LOAD_DWORD_IMM, S_LOAD_DWORD_SGPR, f32>;
 defm : SMRD_Pattern <S_LOAD_DWORD_IMM, S_LOAD_DWORD_SGPR, i32>;
 defm : SMRD_Pattern <S_LOAD_DWORDX2_IMM, S_LOAD_DWORDX2_SGPR, v2i32>;
 defm : SMRD_Pattern <S_LOAD_DWORDX4_IMM, S_LOAD_DWORDX4_SGPR, v4i32>;
@@ -2251,6 +2208,15 @@ def : Pat <
   (V_CNDMASK_B32_e64 $src2, $src1, $src0)
 >;
 
+// Pattern for V_MAC_F32
+def : Pat <
+  (fmad  (VOP3NoMods0 f32:$src0, i32:$src0_modifiers, i1:$clamp, i32:$omod),
+         (VOP3NoMods f32:$src1, i32:$src1_modifiers),
+         (VOP3NoMods f32:$src2, i32:$src2_modifiers)),
+  (V_MAC_F32_e64 $src0_modifiers, $src0, $src1_modifiers, $src1,
+                 $src2_modifiers, $src2, $clamp, $omod)
+>;
+
 /********** ======================= **********/
 /********** Image sampling patterns **********/
 /********** ======================= **********/
@@ -2723,27 +2689,6 @@ def : Pat <
   (S_MOV_B64 (f64 (bitcast_fpimm_to_i64 InlineFPImm<f64>:$imm)))
 >;
 
-/********** ===================== **********/
-/********** Interpolation Paterns **********/
-/********** ===================== **********/
-
-// The value of $params is constant through out the entire kernel.
-// We need to use S_MOV_B32 $params, because CSE ignores copies, so
-// without it we end up with a lot of redundant moves.
-
-def : Pat <
-  (int_SI_fs_constant imm:$attr_chan, imm:$attr, i32:$params),
-  (V_INTERP_MOV_F32 INTERP.P0, imm:$attr_chan, imm:$attr, (S_MOV_B32 $params))
->;
-
-def : Pat <
-  (int_SI_fs_interp imm:$attr_chan, imm:$attr, i32:$params, v2i32:$ij),
-  (V_INTERP_P2_F32 (V_INTERP_P1_F32 (EXTRACT_SUBREG v2i32:$ij, sub0),
-                                    imm:$attr_chan, imm:$attr, (S_MOV_B32 $params)),
-                   (EXTRACT_SUBREG $ij, sub1),
-                   imm:$attr_chan, imm:$attr, (S_MOV_B32 $params))
->;
-
 /********** ================== **********/
 /********** Intrinsic Patterns **********/
 /********** ================== **********/
@@ -2836,52 +2781,52 @@ def : ROTRPattern <V_ALIGNBIT_B32>;
 
 class DSReadPat <DS inst, ValueType vt, PatFrag frag> : Pat <
   (vt (frag (DS1Addr1Offset i32:$ptr, i32:$offset))),
-  (inst $ptr, (as_i16imm $offset), (i1 0), (S_MOV_B32 -1))
+  (inst $ptr, (as_i16imm $offset), (i1 0))
 >;
 
-def : DSReadPat <DS_READ_I8,  i32, sextloadi8_local>;
-def : DSReadPat <DS_READ_U8,  i32, az_extloadi8_local>;
-def : DSReadPat <DS_READ_I16, i32, sextloadi16_local>;
-def : DSReadPat <DS_READ_U16, i32, az_extloadi16_local>;
-def : DSReadPat <DS_READ_B32, i32, local_load>;
+def : DSReadPat <DS_READ_I8,  i32, si_sextload_local_i8>;
+def : DSReadPat <DS_READ_U8,  i32, si_az_extload_local_i8>;
+def : DSReadPat <DS_READ_I16, i32, si_sextload_local_i16>;
+def : DSReadPat <DS_READ_U16, i32, si_az_extload_local_i16>;
+def : DSReadPat <DS_READ_B32, i32, si_load_local>;
 
 let AddedComplexity = 100 in {
 
-def : DSReadPat <DS_READ_B64, v2i32, local_load_aligned8bytes>;
+def : DSReadPat <DS_READ_B64, v2i32, si_load_local_align8>;
 
 } // End AddedComplexity = 100
 
 def : Pat <
-  (v2i32 (local_load (DS64Bit4ByteAligned i32:$ptr, i8:$offset0,
+  (v2i32 (si_load_local (DS64Bit4ByteAligned i32:$ptr, i8:$offset0,
                                                     i8:$offset1))),
-  (DS_READ2_B32 $ptr, $offset0, $offset1, (i1 0), (S_MOV_B32 -1))
+  (DS_READ2_B32 $ptr, $offset0, $offset1, (i1 0))
 >;
 
 class DSWritePat <DS inst, ValueType vt, PatFrag frag> : Pat <
   (frag vt:$value, (DS1Addr1Offset i32:$ptr, i32:$offset)),
-  (inst $ptr, $value, (as_i16imm $offset), (i1 0), (S_MOV_B32 -1))
+  (inst $ptr, $value, (as_i16imm $offset), (i1 0))
 >;
 
-def : DSWritePat <DS_WRITE_B8, i32, truncstorei8_local>;
-def : DSWritePat <DS_WRITE_B16, i32, truncstorei16_local>;
-def : DSWritePat <DS_WRITE_B32, i32, local_store>;
+def : DSWritePat <DS_WRITE_B8, i32, si_truncstore_local_i8>;
+def : DSWritePat <DS_WRITE_B16, i32, si_truncstore_local_i16>;
+def : DSWritePat <DS_WRITE_B32, i32, si_store_local>;
 
 let AddedComplexity = 100 in {
 
-def : DSWritePat <DS_WRITE_B64, v2i32, local_store_aligned8bytes>;
+def : DSWritePat <DS_WRITE_B64, v2i32, si_store_local_align8>;
 } // End AddedComplexity = 100
 
 def : Pat <
-  (local_store v2i32:$value, (DS64Bit4ByteAligned i32:$ptr, i8:$offset0,
-                                                            i8:$offset1)),
+  (si_store_local v2i32:$value, (DS64Bit4ByteAligned i32:$ptr, i8:$offset0,
+                                                               i8:$offset1)),
   (DS_WRITE2_B32 $ptr, (EXTRACT_SUBREG $value, sub0),
                        (EXTRACT_SUBREG $value, sub1), $offset0, $offset1,
-                       (i1 0), (S_MOV_B32 -1))
+                       (i1 0))
 >;
 
 class DSAtomicRetPat<DS inst, ValueType vt, PatFrag frag> : Pat <
   (frag (DS1Addr1Offset i32:$ptr, i32:$offset), vt:$value),
-  (inst $ptr, $value, (as_i16imm $offset), (i1 0), (S_MOV_B32 -1))
+  (inst $ptr, $value, (as_i16imm $offset), (i1 0))
 >;
 
 // Special case of DSAtomicRetPat for add / sub 1 -> inc / dec
@@ -2897,53 +2842,53 @@ class DSAtomicRetPat<DS inst, ValueType vt, PatFrag frag> : Pat <
 class DSAtomicIncRetPat<DS inst, ValueType vt,
                         Instruction LoadImm, PatFrag frag> : Pat <
   (frag (DS1Addr1Offset i32:$ptr, i32:$offset), (vt 1)),
-  (inst $ptr, (LoadImm (vt -1)), (as_i16imm $offset), (i1 0), (S_MOV_B32 -1))
+  (inst $ptr, (LoadImm (vt -1)), (as_i16imm $offset), (i1 0))
 >;
 
 
 class DSAtomicCmpXChg <DS inst, ValueType vt, PatFrag frag> : Pat <
   (frag (DS1Addr1Offset i32:$ptr, i32:$offset), vt:$cmp, vt:$swap),
-  (inst $ptr, $cmp, $swap, (as_i16imm $offset), (i1 0), (S_MOV_B32 -1))
+  (inst $ptr, $cmp, $swap, (as_i16imm $offset), (i1 0))
 >;
 
 
 // 32-bit atomics.
 def : DSAtomicIncRetPat<DS_INC_RTN_U32, i32,
-                        S_MOV_B32, atomic_load_add_local>;
+                        S_MOV_B32, si_atomic_load_add_local>;
 def : DSAtomicIncRetPat<DS_DEC_RTN_U32, i32,
-                        S_MOV_B32, atomic_load_sub_local>;
+                        S_MOV_B32, si_atomic_load_sub_local>;
 
-def : DSAtomicRetPat<DS_WRXCHG_RTN_B32, i32, atomic_swap_local>;
-def : DSAtomicRetPat<DS_ADD_RTN_U32, i32, atomic_load_add_local>;
-def : DSAtomicRetPat<DS_SUB_RTN_U32, i32, atomic_load_sub_local>;
-def : DSAtomicRetPat<DS_AND_RTN_B32, i32, atomic_load_and_local>;
-def : DSAtomicRetPat<DS_OR_RTN_B32, i32, atomic_load_or_local>;
-def : DSAtomicRetPat<DS_XOR_RTN_B32, i32, atomic_load_xor_local>;
-def : DSAtomicRetPat<DS_MIN_RTN_I32, i32, atomic_load_min_local>;
-def : DSAtomicRetPat<DS_MAX_RTN_I32, i32, atomic_load_max_local>;
-def : DSAtomicRetPat<DS_MIN_RTN_U32, i32, atomic_load_umin_local>;
-def : DSAtomicRetPat<DS_MAX_RTN_U32, i32, atomic_load_umax_local>;
+def : DSAtomicRetPat<DS_WRXCHG_RTN_B32, i32, si_atomic_swap_local>;
+def : DSAtomicRetPat<DS_ADD_RTN_U32, i32, si_atomic_load_add_local>;
+def : DSAtomicRetPat<DS_SUB_RTN_U32, i32, si_atomic_load_sub_local>;
+def : DSAtomicRetPat<DS_AND_RTN_B32, i32, si_atomic_load_and_local>;
+def : DSAtomicRetPat<DS_OR_RTN_B32, i32, si_atomic_load_or_local>;
+def : DSAtomicRetPat<DS_XOR_RTN_B32, i32, si_atomic_load_xor_local>;
+def : DSAtomicRetPat<DS_MIN_RTN_I32, i32, si_atomic_load_min_local>;
+def : DSAtomicRetPat<DS_MAX_RTN_I32, i32, si_atomic_load_max_local>;
+def : DSAtomicRetPat<DS_MIN_RTN_U32, i32, si_atomic_load_umin_local>;
+def : DSAtomicRetPat<DS_MAX_RTN_U32, i32, si_atomic_load_umax_local>;
 
-def : DSAtomicCmpXChg<DS_CMPST_RTN_B32, i32, atomic_cmp_swap_32_local>;
+def : DSAtomicCmpXChg<DS_CMPST_RTN_B32, i32, si_atomic_cmp_swap_32_local>;
 
 // 64-bit atomics.
 def : DSAtomicIncRetPat<DS_INC_RTN_U64, i64,
-                        S_MOV_B64, atomic_load_add_local>;
+                        S_MOV_B64, si_atomic_load_add_local>;
 def : DSAtomicIncRetPat<DS_DEC_RTN_U64, i64,
-                        S_MOV_B64, atomic_load_sub_local>;
+                        S_MOV_B64, si_atomic_load_sub_local>;
 
-def : DSAtomicRetPat<DS_WRXCHG_RTN_B64, i64, atomic_swap_local>;
-def : DSAtomicRetPat<DS_ADD_RTN_U64, i64, atomic_load_add_local>;
-def : DSAtomicRetPat<DS_SUB_RTN_U64, i64, atomic_load_sub_local>;
-def : DSAtomicRetPat<DS_AND_RTN_B64, i64, atomic_load_and_local>;
-def : DSAtomicRetPat<DS_OR_RTN_B64, i64, atomic_load_or_local>;
-def : DSAtomicRetPat<DS_XOR_RTN_B64, i64, atomic_load_xor_local>;
-def : DSAtomicRetPat<DS_MIN_RTN_I64, i64, atomic_load_min_local>;
-def : DSAtomicRetPat<DS_MAX_RTN_I64, i64, atomic_load_max_local>;
-def : DSAtomicRetPat<DS_MIN_RTN_U64, i64, atomic_load_umin_local>;
-def : DSAtomicRetPat<DS_MAX_RTN_U64, i64, atomic_load_umax_local>;
+def : DSAtomicRetPat<DS_WRXCHG_RTN_B64, i64, si_atomic_swap_local>;
+def : DSAtomicRetPat<DS_ADD_RTN_U64, i64, si_atomic_load_add_local>;
+def : DSAtomicRetPat<DS_SUB_RTN_U64, i64, si_atomic_load_sub_local>;
+def : DSAtomicRetPat<DS_AND_RTN_B64, i64, si_atomic_load_and_local>;
+def : DSAtomicRetPat<DS_OR_RTN_B64, i64, si_atomic_load_or_local>;
+def : DSAtomicRetPat<DS_XOR_RTN_B64, i64, si_atomic_load_xor_local>;
+def : DSAtomicRetPat<DS_MIN_RTN_I64, i64, si_atomic_load_min_local>;
+def : DSAtomicRetPat<DS_MAX_RTN_I64, i64, si_atomic_load_max_local>;
+def : DSAtomicRetPat<DS_MIN_RTN_U64, i64, si_atomic_load_umin_local>;
+def : DSAtomicRetPat<DS_MAX_RTN_U64, i64, si_atomic_load_umax_local>;
 
-def : DSAtomicCmpXChg<DS_CMPST_RTN_B64, i64, atomic_cmp_swap_64_local>;
+def : DSAtomicCmpXChg<DS_CMPST_RTN_B64, i64, si_atomic_cmp_swap_64_local>;
 
 
 //===----------------------------------------------------------------------===//
@@ -2964,9 +2909,6 @@ defm : MUBUFLoad_Pattern <BUFFER_LOAD_SBYTE_ADDR64, i32, sextloadi8_constant>;
 defm : MUBUFLoad_Pattern <BUFFER_LOAD_UBYTE_ADDR64, i32, az_extloadi8_constant>;
 defm : MUBUFLoad_Pattern <BUFFER_LOAD_SSHORT_ADDR64, i32, sextloadi16_constant>;
 defm : MUBUFLoad_Pattern <BUFFER_LOAD_USHORT_ADDR64, i32, az_extloadi16_constant>;
-defm : MUBUFLoad_Pattern <BUFFER_LOAD_DWORD_ADDR64, i32, constant_load>;
-defm : MUBUFLoad_Pattern <BUFFER_LOAD_DWORDX2_ADDR64, v2i32, constant_load>;
-defm : MUBUFLoad_Pattern <BUFFER_LOAD_DWORDX4_ADDR64, v4i32, constant_load>;
 } // End Predicates = [isSICI]
 
 class MUBUFScratchLoadPat <MUBUF Instr, ValueType vt, PatFrag ld> : Pat <
@@ -3120,38 +3062,6 @@ defm V_MAD_I64_I32 : VOP3Inst <vop3<0x177>, "v_mad_i64_i32",
 
 } // End isCI
 
-//===----------------------------------------------------------------------===//
-// Flat Patterns
-//===----------------------------------------------------------------------===//
-
-class FLATLoad_Pattern <FLAT Instr_ADDR64, ValueType vt,
-                             PatFrag flat_ld> :
-  Pat <(vt (flat_ld i64:$ptr)),
-       (Instr_ADDR64 $ptr)
->;
-
-def : FLATLoad_Pattern <FLAT_LOAD_SBYTE, i32, sextloadi8_flat>;
-def : FLATLoad_Pattern <FLAT_LOAD_UBYTE, i32, az_extloadi8_flat>;
-def : FLATLoad_Pattern <FLAT_LOAD_SSHORT, i32, sextloadi16_flat>;
-def : FLATLoad_Pattern <FLAT_LOAD_USHORT, i32, az_extloadi16_flat>;
-def : FLATLoad_Pattern <FLAT_LOAD_DWORD, i32, flat_load>;
-def : FLATLoad_Pattern <FLAT_LOAD_DWORDX2, i64, flat_load>;
-def : FLATLoad_Pattern <FLAT_LOAD_DWORDX2, i64, az_extloadi32_flat>;
-def : FLATLoad_Pattern <FLAT_LOAD_DWORDX2, v2i32, flat_load>;
-def : FLATLoad_Pattern <FLAT_LOAD_DWORDX4, v4i32, flat_load>;
-
-class FLATStore_Pattern <FLAT Instr, ValueType vt, PatFrag st> :
-  Pat <(st vt:$value, i64:$ptr),
-        (Instr $value, $ptr)
-  >;
-
-def : FLATStore_Pattern <FLAT_STORE_BYTE, i32, truncstorei8_flat>;
-def : FLATStore_Pattern <FLAT_STORE_SHORT, i32, truncstorei16_flat>;
-def : FLATStore_Pattern <FLAT_STORE_DWORD, i32, flat_store>;
-def : FLATStore_Pattern <FLAT_STORE_DWORDX2, i64, flat_store>;
-def : FLATStore_Pattern <FLAT_STORE_DWORDX2, v2i32, flat_store>;
-def : FLATStore_Pattern <FLAT_STORE_DWORDX4, v4i32, flat_store>;
-
 /********** ====================== **********/
 /**********   Indirect adressing   **********/
 /********** ====================== **********/
@@ -3359,13 +3269,13 @@ def : Pat <
   (f64 (fadd (f64 (VOP3Mods f64:$x, i32:$mods)),
              (f64 (fneg (f64 (ffloor (f64 (VOP3Mods f64:$x, i32:$mods)))))))),
   (V_CNDMASK_B64_PSEUDO
-      $x,
       (V_MIN_F64
           SRCMODS.NONE,
           (V_FRACT_F64_e64 $mods, $x, DSTCLAMP.NONE, DSTOMOD.NONE),
           SRCMODS.NONE,
           (V_MOV_B64_PSEUDO 0x3fefffffffffffff),
           DSTCLAMP.NONE, DSTOMOD.NONE),
+      $x,
       (V_CMP_CLASS_F64_e64 SRCMODS.NONE, $x, 3/*NaN*/))
 >;
 
@@ -3377,13 +3287,13 @@ def : Pat <
       $x,
       SRCMODS.NEG,
       (V_CNDMASK_B64_PSEUDO
-         $x,
          (V_MIN_F64
              SRCMODS.NONE,
              (V_FRACT_F64_e64 $mods, $x, DSTCLAMP.NONE, DSTOMOD.NONE),
              SRCMODS.NONE,
              (V_MOV_B64_PSEUDO 0x3fefffffffffffff),
              DSTCLAMP.NONE, DSTOMOD.NONE),
+         $x,
          (V_CMP_CLASS_F64_e64 SRCMODS.NONE, $x, 3/*NaN*/)),
       DSTCLAMP.NONE, DSTOMOD.NONE)
 >;
@@ -3414,4 +3324,12 @@ def : Pat <
 
 def : SHA256MaPattern <V_BFI_B32, V_XOR_B32_e64>;
 
+//============================================================================//
+// Assembler aliases
+//============================================================================//
+
+def : MnemonicAlias<"v_add_u32", "v_add_i32">;
+def : MnemonicAlias<"v_sub_u32", "v_sub_i32">;
+def : MnemonicAlias<"v_subrev_u32", "v_subrev_i32">;
+
 } // End isGCN predicate
diff --git a/lib/Target/R600/SIIntrinsics.td b/lib/Target/AMDGPU/SIIntrinsics.td
similarity index 100%
rename from lib/Target/R600/SIIntrinsics.td
rename to lib/Target/AMDGPU/SIIntrinsics.td
diff --git a/lib/Target/R600/SILoadStoreOptimizer.cpp b/lib/Target/AMDGPU/SILoadStoreOptimizer.cpp
similarity index 86%
rename from lib/Target/R600/SILoadStoreOptimizer.cpp
rename to lib/Target/AMDGPU/SILoadStoreOptimizer.cpp
index a927ad82ba9..1bdb1f0ee9f 100644
--- a/lib/Target/R600/SILoadStoreOptimizer.cpp
+++ b/lib/Target/AMDGPU/SILoadStoreOptimizer.cpp
@@ -213,14 +213,12 @@ MachineBasicBlock::iterator  SILoadStoreOptimizer::mergeRead2Pair(
   // Be careful, since the addresses could be subregisters themselves in weird
   // cases, like vectors of pointers.
   const MachineOperand *AddrReg = TII->getNamedOperand(*I, AMDGPU::OpName::addr);
-  const MachineOperand *M0Reg = TII->getNamedOperand(*I, AMDGPU::OpName::m0);
 
-  unsigned DestReg0 = TII->getNamedOperand(*I, AMDGPU::OpName::vdst)->getReg();
-  unsigned DestReg1
-    = TII->getNamedOperand(*Paired, AMDGPU::OpName::vdst)->getReg();
+  const MachineOperand *Dest0 = TII->getNamedOperand(*I, AMDGPU::OpName::vdst);
+  const MachineOperand *Dest1 = TII->getNamedOperand(*Paired, AMDGPU::OpName::vdst);
 
   unsigned Offset0
-          = TII->getNamedOperand(*I, AMDGPU::OpName::offset)->getImm() & 0xffff;
+    = TII->getNamedOperand(*I, AMDGPU::OpName::offset)->getImm() & 0xffff;
   unsigned Offset1
     = TII->getNamedOperand(*Paired, AMDGPU::OpName::offset)->getImm() & 0xffff;
 
@@ -254,38 +252,48 @@ MachineBasicBlock::iterator  SILoadStoreOptimizer::mergeRead2Pair(
     .addImm(NewOffset0) // offset0
     .addImm(NewOffset1) // offset1
     .addImm(0) // gds
-    .addOperand(*M0Reg) // M0
     .addMemOperand(*I->memoperands_begin())
     .addMemOperand(*Paired->memoperands_begin());
 
-  LIS->InsertMachineInstrInMaps(Read2);
-
   unsigned SubRegIdx0 = (EltSize == 4) ? AMDGPU::sub0 : AMDGPU::sub0_sub1;
   unsigned SubRegIdx1 = (EltSize == 4) ? AMDGPU::sub1 : AMDGPU::sub2_sub3;
-  updateRegDefsUses(DestReg0, DestReg, SubRegIdx0);
-  updateRegDefsUses(DestReg1, DestReg, SubRegIdx1);
 
-  LIS->RemoveMachineInstrFromMaps(I);
-  LIS->RemoveMachineInstrFromMaps(Paired);
+  const MCInstrDesc &CopyDesc = TII->get(TargetOpcode::COPY);
+
+  // Copy to the old destination registers.
+  MachineInstr *Copy0 = BuildMI(*MBB, I, DL, CopyDesc)
+    .addOperand(*Dest0) // Copy to same destination including flags and sub reg.
+    .addReg(DestReg, 0, SubRegIdx0);
+  MachineInstr *Copy1 = BuildMI(*MBB, I, DL, CopyDesc)
+    .addOperand(*Dest1)
+    .addReg(DestReg, RegState::Kill, SubRegIdx1);
+
+  LIS->InsertMachineInstrInMaps(Read2);
+
+  // repairLiveintervalsInRange() doesn't handle physical register, so we have
+  // to update the M0 range manually.
+  SlotIndex PairedIndex = LIS->getInstructionIndex(Paired);
+  LiveRange &M0Range = LIS->getRegUnit(*MCRegUnitIterator(AMDGPU::M0, TRI));
+  LiveRange::Segment *M0Segment = M0Range.getSegmentContaining(PairedIndex);
+  bool UpdateM0Range = M0Segment->end == PairedIndex.getRegSlot();
+
+  // The new write to the original destination register is now the copy. Steal
+  // the old SlotIndex.
+  LIS->ReplaceMachineInstrInMaps(I, Copy0);
+  LIS->ReplaceMachineInstrInMaps(Paired, Copy1);
+
   I->eraseFromParent();
   Paired->eraseFromParent();
 
   LiveInterval &AddrRegLI = LIS->getInterval(AddrReg->getReg());
   LIS->shrinkToUses(&AddrRegLI);
 
-  LiveInterval &M0RegLI = LIS->getInterval(M0Reg->getReg());
-  LIS->shrinkToUses(&M0RegLI);
-
-  // Currently m0 is treated as a register class with one member instead of an
-  // implicit physical register. We are using the virtual register for the first
-  // one, but we still need to update the live range of the now unused second m0
-  // virtual register to avoid verifier errors.
-  const MachineOperand *PairedM0Reg
-    = TII->getNamedOperand(*Paired, AMDGPU::OpName::m0);
-  LiveInterval &PairedM0RegLI = LIS->getInterval(PairedM0Reg->getReg());
-  LIS->shrinkToUses(&PairedM0RegLI);
+  LIS->createAndComputeVirtRegInterval(DestReg);
 
-  LIS->getInterval(DestReg); // Create new LI
+  if (UpdateM0Range) {
+    SlotIndex Read2Index = LIS->getInstructionIndex(Read2);
+    M0Segment->end = Read2Index.getRegSlot();
+  }
 
   DEBUG(dbgs() << "Inserted read2: " << *Read2 << '\n');
   return Read2.getInstr();
@@ -300,7 +308,6 @@ MachineBasicBlock::iterator SILoadStoreOptimizer::mergeWrite2Pair(
   // Be sure to use .addOperand(), and not .addReg() with these. We want to be
   // sure we preserve the subregister index and any register flags set on them.
   const MachineOperand *Addr = TII->getNamedOperand(*I, AMDGPU::OpName::addr);
-  const MachineOperand *M0Reg = TII->getNamedOperand(*I, AMDGPU::OpName::m0);
   const MachineOperand *Data0 = TII->getNamedOperand(*I, AMDGPU::OpName::data0);
   const MachineOperand *Data1
     = TII->getNamedOperand(*Paired, AMDGPU::OpName::data0);
@@ -331,6 +338,13 @@ MachineBasicBlock::iterator SILoadStoreOptimizer::mergeWrite2Pair(
   const MCInstrDesc &Write2Desc = TII->get(Opc);
   DebugLoc DL = I->getDebugLoc();
 
+  // repairLiveintervalsInRange() doesn't handle physical register, so we have
+  // to update the M0 range manually.
+  SlotIndex PairedIndex = LIS->getInstructionIndex(Paired);
+  LiveRange &M0Range = LIS->getRegUnit(*MCRegUnitIterator(AMDGPU::M0, TRI));
+  LiveRange::Segment *M0Segment = M0Range.getSegmentContaining(PairedIndex);
+  bool UpdateM0Range = M0Segment->end == PairedIndex.getRegSlot();
+
   MachineInstrBuilder Write2
     = BuildMI(*MBB, I, DL, Write2Desc)
     .addOperand(*Addr) // addr
@@ -339,21 +353,25 @@ MachineBasicBlock::iterator SILoadStoreOptimizer::mergeWrite2Pair(
     .addImm(NewOffset0) // offset0
     .addImm(NewOffset1) // offset1
     .addImm(0) // gds
-    .addOperand(*M0Reg)  // m0
     .addMemOperand(*I->memoperands_begin())
     .addMemOperand(*Paired->memoperands_begin());
 
   // XXX - How do we express subregisters here?
-  unsigned OrigRegs[] = { Data0->getReg(), Data1->getReg(), Addr->getReg(),
-                          M0Reg->getReg()};
+  unsigned OrigRegs[] = { Data0->getReg(), Data1->getReg(), Addr->getReg() };
 
   LIS->RemoveMachineInstrFromMaps(I);
   LIS->RemoveMachineInstrFromMaps(Paired);
   I->eraseFromParent();
   Paired->eraseFromParent();
 
+  // This doesn't handle physical registers like M0
   LIS->repairIntervalsInRange(MBB, Write2, Write2, OrigRegs);
 
+  if (UpdateM0Range) {
+    SlotIndex Write2Index = LIS->getInstructionIndex(Write2);
+    M0Segment->end = Write2Index.getRegSlot();
+  }
+
   DEBUG(dbgs() << "Inserted write2 inst: " << *Write2 << '\n');
   return Write2.getInstr();
 }
diff --git a/lib/Target/R600/SILowerControlFlow.cpp b/lib/Target/AMDGPU/SILowerControlFlow.cpp
similarity index 89%
rename from lib/Target/R600/SILowerControlFlow.cpp
rename to lib/Target/AMDGPU/SILowerControlFlow.cpp
index 2e08c9f5a5d..c319b32111f 100644
--- a/lib/Target/R600/SILowerControlFlow.cpp
+++ b/lib/Target/AMDGPU/SILowerControlFlow.cpp
@@ -88,7 +88,8 @@ class SILowerControlFlowPass : public MachineFunctionPass {
   void Kill(MachineInstr &MI);
   void Branch(MachineInstr &MI);
 
-  void LoadM0(MachineInstr &MI, MachineInstr *MovRel);
+  void LoadM0(MachineInstr &MI, MachineInstr *MovRel, int Offset = 0);
+  void computeIndirectRegAndOffset(unsigned VecReg, unsigned &Reg, int &Offset);
   void IndirectSrc(MachineInstr &MI);
   void IndirectDst(MachineInstr &MI);
 
@@ -323,7 +324,7 @@ void SILowerControlFlowPass::Kill(MachineInstr &MI) {
   MI.eraseFromParent();
 }
 
-void SILowerControlFlowPass::LoadM0(MachineInstr &MI, MachineInstr *MovRel) {
+void SILowerControlFlowPass::LoadM0(MachineInstr &MI, MachineInstr *MovRel, int Offset) {
 
   MachineBasicBlock &MBB = *MI.getParent();
   DebugLoc DL = MI.getDebugLoc();
@@ -333,8 +334,14 @@ void SILowerControlFlowPass::LoadM0(MachineInstr &MI, MachineInstr *MovRel) {
   unsigned Idx = MI.getOperand(3).getReg();
 
   if (AMDGPU::SReg_32RegClass.contains(Idx)) {
-    BuildMI(MBB, &MI, DL, TII->get(AMDGPU::S_MOV_B32), AMDGPU::M0)
-            .addReg(Idx);
+    if (Offset) {
+      BuildMI(MBB, &MI, DL, TII->get(AMDGPU::S_ADD_I32), AMDGPU::M0)
+              .addReg(Idx)
+              .addImm(Offset);
+    } else {
+      BuildMI(MBB, &MI, DL, TII->get(AMDGPU::S_MOV_B32), AMDGPU::M0)
+              .addReg(Idx);
+    }
     MBB.insert(I, MovRel);
   } else {
 
@@ -363,6 +370,11 @@ void SILowerControlFlowPass::LoadM0(MachineInstr &MI, MachineInstr *MovRel) {
     BuildMI(MBB, &MI, DL, TII->get(AMDGPU::S_AND_SAVEEXEC_B64), AMDGPU::VCC)
             .addReg(AMDGPU::VCC);
 
+    if (Offset) {
+      BuildMI(MBB, &MI, DL, TII->get(AMDGPU::S_ADD_I32), AMDGPU::M0)
+              .addReg(AMDGPU::M0)
+              .addImm(Offset);
+    }
     // Do the actual move
     MBB.insert(I, MovRel);
 
@@ -384,6 +396,33 @@ void SILowerControlFlowPass::LoadM0(MachineInstr &MI, MachineInstr *MovRel) {
   MI.eraseFromParent();
 }
 
+/// \param @VecReg The register which holds element zero of the vector
+///                 being addressed into.
+/// \param[out] @Reg The base register to use in the indirect addressing instruction.
+/// \param[in,out] @Offset As an input, this is the constant offset part of the
+//                         indirect Index. e.g. v0 = v[VecReg + Offset]
+//                         As an output, this is a constant value that needs
+//                         to be added to the value stored in M0.
+void SILowerControlFlowPass::computeIndirectRegAndOffset(unsigned VecReg,
+                                                         unsigned &Reg,
+                                                         int &Offset) {
+  unsigned SubReg = TRI->getSubReg(VecReg, AMDGPU::sub0);
+  if (!SubReg)
+    SubReg = VecReg;
+
+  const TargetRegisterClass *RC = TRI->getPhysRegClass(SubReg);
+  int RegIdx = TRI->getHWRegIndex(SubReg) + Offset;
+
+  if (RegIdx < 0) {
+    Offset = RegIdx;
+    RegIdx = 0;
+  } else {
+    Offset = 0;
+  }
+
+  Reg = RC->getRegister(RegIdx);
+}
+
 void SILowerControlFlowPass::IndirectSrc(MachineInstr &MI) {
 
   MachineBasicBlock &MBB = *MI.getParent();
@@ -391,18 +430,18 @@ void SILowerControlFlowPass::IndirectSrc(MachineInstr &MI) {
 
   unsigned Dst = MI.getOperand(0).getReg();
   unsigned Vec = MI.getOperand(2).getReg();
-  unsigned Off = MI.getOperand(4).getImm();
-  unsigned SubReg = TRI->getSubReg(Vec, AMDGPU::sub0);
-  if (!SubReg)
-    SubReg = Vec;
+  int Off = MI.getOperand(4).getImm();
+  unsigned Reg;
+
+  computeIndirectRegAndOffset(Vec, Reg, Off);
 
   MachineInstr *MovRel =
     BuildMI(*MBB.getParent(), DL, TII->get(AMDGPU::V_MOVRELS_B32_e32), Dst)
-            .addReg(SubReg + Off)
+            .addReg(Reg)
             .addReg(AMDGPU::M0, RegState::Implicit)
             .addReg(Vec, RegState::Implicit);
 
-  LoadM0(MI, MovRel);
+  LoadM0(MI, MovRel, Off);
 }
 
 void SILowerControlFlowPass::IndirectDst(MachineInstr &MI) {
@@ -411,20 +450,20 @@ void SILowerControlFlowPass::IndirectDst(MachineInstr &MI) {
   DebugLoc DL = MI.getDebugLoc();
 
   unsigned Dst = MI.getOperand(0).getReg();
-  unsigned Off = MI.getOperand(4).getImm();
+  int Off = MI.getOperand(4).getImm();
   unsigned Val = MI.getOperand(5).getReg();
-  unsigned SubReg = TRI->getSubReg(Dst, AMDGPU::sub0);
-  if (!SubReg)
-    SubReg = Dst;
+  unsigned Reg;
+
+  computeIndirectRegAndOffset(Dst, Reg, Off);
 
   MachineInstr *MovRel = 
     BuildMI(*MBB.getParent(), DL, TII->get(AMDGPU::V_MOVRELD_B32_e32))
-            .addReg(SubReg + Off, RegState::Define)
+            .addReg(Reg, RegState::Define)
             .addReg(Val)
             .addReg(AMDGPU::M0, RegState::Implicit)
             .addReg(Dst, RegState::Implicit);
 
-  LoadM0(MI, MovRel);
+  LoadM0(MI, MovRel, Off);
 }
 
 bool SILowerControlFlowPass::runOnMachineFunction(MachineFunction &MF) {
diff --git a/lib/Target/R600/SILowerI1Copies.cpp b/lib/Target/AMDGPU/SILowerI1Copies.cpp
similarity index 100%
rename from lib/Target/R600/SILowerI1Copies.cpp
rename to lib/Target/AMDGPU/SILowerI1Copies.cpp
diff --git a/lib/Target/R600/SIMachineFunctionInfo.cpp b/lib/Target/AMDGPU/SIMachineFunctionInfo.cpp
similarity index 98%
rename from lib/Target/R600/SIMachineFunctionInfo.cpp
rename to lib/Target/AMDGPU/SIMachineFunctionInfo.cpp
index 587ea63d679..d23b92edef3 100644
--- a/lib/Target/R600/SIMachineFunctionInfo.cpp
+++ b/lib/Target/AMDGPU/SIMachineFunctionInfo.cpp
@@ -53,7 +53,6 @@ SIMachineFunctionInfo::SpilledReg SIMachineFunctionInfo::getSpilledReg(
   if (!LaneVGPRs.count(LaneVGPRIdx)) {
     unsigned LaneVGPR = TRI->findUnusedRegister(MRI, &AMDGPU::VGPR_32RegClass);
     LaneVGPRs[LaneVGPRIdx] = LaneVGPR;
-    MRI.setPhysRegUsed(LaneVGPR);
 
     // Add this register as live-in to all blocks to avoid machine verifer
     // complaining about use of an undefined physical register.
diff --git a/lib/Target/R600/SIMachineFunctionInfo.h b/lib/Target/AMDGPU/SIMachineFunctionInfo.h
similarity index 100%
rename from lib/Target/R600/SIMachineFunctionInfo.h
rename to lib/Target/AMDGPU/SIMachineFunctionInfo.h
diff --git a/lib/Target/AMDGPU/SIPrepareScratchRegs.cpp b/lib/Target/AMDGPU/SIPrepareScratchRegs.cpp
new file mode 100644
index 00000000000..b086d2ed665
--- /dev/null
+++ b/lib/Target/AMDGPU/SIPrepareScratchRegs.cpp
@@ -0,0 +1,193 @@
+//===-- SIPrepareScratchRegs.cpp - Use predicates for control flow --------===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+/// \file
+///
+/// This pass loads scratch pointer and scratch offset into a register or a
+/// frame index which can be used anywhere in the program.  These values will
+/// be used for spilling VGPRs.
+///
+//===----------------------------------------------------------------------===//
+
+#include "AMDGPU.h"
+#include "AMDGPUSubtarget.h"
+#include "SIDefines.h"
+#include "SIInstrInfo.h"
+#include "SIMachineFunctionInfo.h"
+#include "llvm/CodeGen/MachineFrameInfo.h"
+#include "llvm/CodeGen/MachineFunction.h"
+#include "llvm/CodeGen/MachineFunctionPass.h"
+#include "llvm/CodeGen/MachineInstrBuilder.h"
+#include "llvm/CodeGen/MachineRegisterInfo.h"
+#include "llvm/CodeGen/RegisterScavenging.h"
+#include "llvm/IR/Function.h"
+#include "llvm/IR/LLVMContext.h"
+
+using namespace llvm;
+
+namespace {
+
+class SIPrepareScratchRegs : public MachineFunctionPass {
+
+private:
+  static char ID;
+
+public:
+  SIPrepareScratchRegs() : MachineFunctionPass(ID) { }
+
+  bool runOnMachineFunction(MachineFunction &MF) override;
+
+  const char *getPassName() const override {
+    return "SI prepare scratch registers";
+  }
+
+};
+
+} // End anonymous namespace
+
+char SIPrepareScratchRegs::ID = 0;
+
+FunctionPass *llvm::createSIPrepareScratchRegs() {
+  return new SIPrepareScratchRegs();
+}
+
+bool SIPrepareScratchRegs::runOnMachineFunction(MachineFunction &MF) {
+  SIMachineFunctionInfo *MFI = MF.getInfo<SIMachineFunctionInfo>();
+  const SIInstrInfo *TII =
+      static_cast<const SIInstrInfo *>(MF.getSubtarget().getInstrInfo());
+  const SIRegisterInfo *TRI = &TII->getRegisterInfo();
+  MachineRegisterInfo &MRI = MF.getRegInfo();
+  MachineFrameInfo *FrameInfo = MF.getFrameInfo();
+  MachineBasicBlock *Entry = MF.begin();
+  MachineBasicBlock::iterator I = Entry->begin();
+  DebugLoc DL = I->getDebugLoc();
+
+  // FIXME: If we don't have enough VGPRs for SGPR spilling we will need to
+  // run this pass.
+  if (!MFI->hasSpilledVGPRs())
+    return false;
+
+  unsigned ScratchPtrPreloadReg =
+      TRI->getPreloadedValue(MF, SIRegisterInfo::SCRATCH_PTR);
+  unsigned ScratchOffsetPreloadReg =
+      TRI->getPreloadedValue(MF, SIRegisterInfo::SCRATCH_WAVE_OFFSET);
+
+  if (!Entry->isLiveIn(ScratchPtrPreloadReg))
+    Entry->addLiveIn(ScratchPtrPreloadReg);
+
+  if (!Entry->isLiveIn(ScratchOffsetPreloadReg))
+    Entry->addLiveIn(ScratchOffsetPreloadReg);
+
+  // Load the scratch offset.
+  unsigned ScratchOffsetReg =
+      TRI->findUnusedRegister(MRI, &AMDGPU::SGPR_32RegClass);
+  int ScratchOffsetFI = -1;
+
+  if (ScratchOffsetReg != AMDGPU::NoRegister) {
+    // Found an SGPR to use
+    BuildMI(*Entry, I, DL, TII->get(AMDGPU::S_MOV_B32), ScratchOffsetReg)
+            .addReg(ScratchOffsetPreloadReg);
+  } else {
+    // No SGPR is available, we must spill.
+    ScratchOffsetFI = FrameInfo->CreateSpillStackObject(4,4);
+    BuildMI(*Entry, I, DL, TII->get(AMDGPU::SI_SPILL_S32_SAVE))
+            .addReg(ScratchOffsetPreloadReg)
+            .addFrameIndex(ScratchOffsetFI)
+            .addReg(AMDGPU::SGPR0_SGPR1_SGPR2_SGPR3, RegState::Undef)
+            .addReg(AMDGPU::SGPR0, RegState::Undef);
+  }
+
+
+  // Now that we have the scratch pointer and offset values, we need to
+  // add them to all the SI_SPILL_V* instructions.
+
+  RegScavenger RS;
+  unsigned ScratchRsrcFI = FrameInfo->CreateSpillStackObject(16, 4);
+  RS.addScavengingFrameIndex(ScratchRsrcFI);
+
+  for (MachineFunction::iterator BI = MF.begin(), BE = MF.end();
+       BI != BE; ++BI) {
+
+    MachineBasicBlock &MBB = *BI;
+    // Add the scratch offset reg as a live-in so that the register scavenger
+    // doesn't re-use it.
+    if (!MBB.isLiveIn(ScratchOffsetReg) &&
+        ScratchOffsetReg != AMDGPU::NoRegister)
+      MBB.addLiveIn(ScratchOffsetReg);
+    RS.enterBasicBlock(&MBB);
+
+    for (MachineBasicBlock::iterator I = MBB.begin(), E = MBB.end();
+         I != E; ++I) {
+      MachineInstr &MI = *I;
+      RS.forward(I);
+      DebugLoc DL = MI.getDebugLoc();
+      if (!TII->isVGPRSpill(MI.getOpcode()))
+        continue;
+
+      // Scratch resource
+      unsigned ScratchRsrcReg =
+          RS.scavengeRegister(&AMDGPU::SReg_128RegClass, 0);
+
+      uint64_t Rsrc = AMDGPU::RSRC_DATA_FORMAT | AMDGPU::RSRC_TID_ENABLE |
+                      0xffffffff; // Size
+
+      unsigned Rsrc0 = TRI->getSubReg(ScratchRsrcReg, AMDGPU::sub0);
+      unsigned Rsrc1 = TRI->getSubReg(ScratchRsrcReg, AMDGPU::sub1);
+      unsigned Rsrc2 = TRI->getSubReg(ScratchRsrcReg, AMDGPU::sub2);
+      unsigned Rsrc3 = TRI->getSubReg(ScratchRsrcReg, AMDGPU::sub3);
+
+      BuildMI(MBB, I, DL, TII->get(AMDGPU::S_MOV_B32), Rsrc0)
+              .addExternalSymbol("SCRATCH_RSRC_DWORD0")
+              .addReg(ScratchRsrcReg, RegState::ImplicitDefine);
+
+      BuildMI(MBB, I, DL, TII->get(AMDGPU::S_MOV_B32), Rsrc1)
+              .addExternalSymbol("SCRATCH_RSRC_DWORD1")
+              .addReg(ScratchRsrcReg, RegState::ImplicitDefine);
+
+      BuildMI(MBB, I, DL, TII->get(AMDGPU::S_MOV_B32), Rsrc2)
+              .addImm(Rsrc & 0xffffffff)
+              .addReg(ScratchRsrcReg, RegState::ImplicitDefine);
+
+      BuildMI(MBB, I, DL, TII->get(AMDGPU::S_MOV_B32), Rsrc3)
+              .addImm(Rsrc >> 32)
+              .addReg(ScratchRsrcReg, RegState::ImplicitDefine);
+
+      // Scratch Offset
+      if (ScratchOffsetReg == AMDGPU::NoRegister) {
+        ScratchOffsetReg = RS.scavengeRegister(&AMDGPU::SGPR_32RegClass, 0);
+        BuildMI(MBB, I, DL, TII->get(AMDGPU::SI_SPILL_S32_RESTORE),
+                ScratchOffsetReg)
+                .addFrameIndex(ScratchOffsetFI)
+                .addReg(AMDGPU::SGPR0_SGPR1_SGPR2_SGPR3, RegState::Undef)
+                .addReg(AMDGPU::SGPR0, RegState::Undef);
+      } else if (!MBB.isLiveIn(ScratchOffsetReg)) {
+        MBB.addLiveIn(ScratchOffsetReg);
+      }
+
+      if (ScratchRsrcReg == AMDGPU::NoRegister ||
+          ScratchOffsetReg == AMDGPU::NoRegister) {
+        LLVMContext &Ctx = MF.getFunction()->getContext();
+        Ctx.emitError("ran out of SGPRs for spilling VGPRs");
+        ScratchRsrcReg = AMDGPU::SGPR0;
+        ScratchOffsetReg = AMDGPU::SGPR0;
+      }
+      MI.getOperand(2).setReg(ScratchRsrcReg);
+      MI.getOperand(2).setIsKill(true);
+      MI.getOperand(2).setIsUndef(false);
+      MI.getOperand(3).setReg(ScratchOffsetReg);
+      MI.getOperand(3).setIsUndef(false);
+      MI.getOperand(3).setIsKill(false);
+      MI.addOperand(MachineOperand::CreateReg(Rsrc0, false, true, true));
+      MI.addOperand(MachineOperand::CreateReg(Rsrc1, false, true, true));
+      MI.addOperand(MachineOperand::CreateReg(Rsrc2, false, true, true));
+      MI.addOperand(MachineOperand::CreateReg(Rsrc3, false, true, true));
+    }
+  }
+  return true;
+}
diff --git a/lib/Target/R600/SIRegisterInfo.cpp b/lib/Target/AMDGPU/SIRegisterInfo.cpp
similarity index 98%
rename from lib/Target/R600/SIRegisterInfo.cpp
rename to lib/Target/AMDGPU/SIRegisterInfo.cpp
index 13a89743677..ce4acafac9f 100644
--- a/lib/Target/R600/SIRegisterInfo.cpp
+++ b/lib/Target/AMDGPU/SIRegisterInfo.cpp
@@ -245,7 +245,6 @@ void SIRegisterInfo::eliminateFrameIndex(MachineBasicBlock::iterator MI,
       for (unsigned i = 0, e = NumSubRegs; i < e; ++i) {
         unsigned SubReg = getPhysRegSubReg(MI->getOperand(0).getReg(),
                                            &AMDGPU::SGPR_32RegClass, i);
-        bool isM0 = SubReg == AMDGPU::M0;
         struct SIMachineFunctionInfo::SpilledReg Spill =
             MFI->getSpilledReg(MF, Index, i);
 
@@ -254,19 +253,12 @@ void SIRegisterInfo::eliminateFrameIndex(MachineBasicBlock::iterator MI,
            Ctx.emitError("Ran out of VGPRs for spilling SGPR");
         }
 
-        if (isM0)
-          SubReg = RS->scavengeRegister(&AMDGPU::SGPR_32RegClass, MI, 0);
-
         BuildMI(*MBB, MI, DL,
                 TII->getMCOpcodeFromPseudo(AMDGPU::V_READLANE_B32),
                 SubReg)
                 .addReg(Spill.VGPR)
                 .addImm(Spill.Lane)
                 .addReg(MI->getOperand(0).getReg(), RegState::ImplicitDefine);
-        if (isM0) {
-          BuildMI(*MBB, MI, DL, TII->get(AMDGPU::S_MOV_B32), AMDGPU::M0)
-                  .addReg(SubReg);
-        }
       }
 
       // TODO: only do this when it is needed
@@ -507,7 +499,7 @@ unsigned SIRegisterInfo::findUnusedRegister(const MachineRegisterInfo &MRI,
 
   for (TargetRegisterClass::iterator I = RC->begin(), E = RC->end();
        I != E; ++I) {
-    if (!MRI.isPhysRegUsed(*I))
+    if (MRI.reg_nodbg_empty(*I))
       return *I;
   }
   return AMDGPU::NoRegister;
diff --git a/lib/Target/R600/SIRegisterInfo.h b/lib/Target/AMDGPU/SIRegisterInfo.h
similarity index 100%
rename from lib/Target/R600/SIRegisterInfo.h
rename to lib/Target/AMDGPU/SIRegisterInfo.h
diff --git a/lib/Target/R600/SIRegisterInfo.td b/lib/Target/AMDGPU/SIRegisterInfo.td
similarity index 94%
rename from lib/Target/R600/SIRegisterInfo.td
rename to lib/Target/AMDGPU/SIRegisterInfo.td
index f28901437a9..1c4596ad167 100644
--- a/lib/Target/R600/SIRegisterInfo.td
+++ b/lib/Target/AMDGPU/SIRegisterInfo.td
@@ -10,10 +10,13 @@
 //===----------------------------------------------------------------------===//
 //  Declarations that describe the SI registers
 //===----------------------------------------------------------------------===//
-
-class SIReg <string n, bits<16> encoding = 0> : Register<n> {
+class SIReg <string n, bits<16> regIdx = 0> : Register<n>,
+  DwarfRegNum<[!cast<int>(HWEncoding)]> {
   let Namespace = "AMDGPU";
-  let HWEncoding = encoding;
+
+  // This is the not yet the complete register encoding. An additional
+  // bit is set for VGPRs.
+  let HWEncoding = regIdx;
 }
 
 // Special Registers
@@ -21,7 +24,8 @@ def VCC_LO : SIReg<"vcc_lo", 106>;
 def VCC_HI : SIReg<"vcc_hi", 107>;
 
 // VCC for 64-bit instructions
-def VCC : RegisterWithSubRegs<"vcc", [VCC_LO, VCC_HI]> {
+def VCC : RegisterWithSubRegs<"vcc", [VCC_LO, VCC_HI]>,
+          DwarfRegAlias<VCC_LO> {
   let Namespace = "AMDGPU";
   let SubRegIndices = [sub0, sub1];
   let HWEncoding = 106;
@@ -30,7 +34,8 @@ def VCC : RegisterWithSubRegs<"vcc", [VCC_LO, VCC_HI]> {
 def EXEC_LO : SIReg<"exec_lo", 126>;
 def EXEC_HI : SIReg<"exec_hi", 127>;
 
-def EXEC : RegisterWithSubRegs<"EXEC", [EXEC_LO, EXEC_HI]> {
+def EXEC : RegisterWithSubRegs<"EXEC", [EXEC_LO, EXEC_HI]>,
+           DwarfRegAlias<EXEC_LO> {
   let Namespace = "AMDGPU";
   let SubRegIndices = [sub0, sub1];
   let HWEncoding = 126;
@@ -43,7 +48,8 @@ def FLAT_SCR_LO : SIReg<"flat_scr_lo", 104>; // Offset in units of 256-bytes.
 def FLAT_SCR_HI : SIReg<"flat_scr_hi", 105>; // Size is the per-thread scratch size, in bytes.
 
 // Pair to indicate location of scratch space for flat accesses.
-def FLAT_SCR : RegisterWithSubRegs <"flat_scr", [FLAT_SCR_LO, FLAT_SCR_HI]> {
+def FLAT_SCR : RegisterWithSubRegs <"flat_scr", [FLAT_SCR_LO, FLAT_SCR_HI]>,
+               DwarfRegAlias<FLAT_SCR_LO> {
   let Namespace = "AMDGPU";
   let SubRegIndices = [sub0, sub1];
   let HWEncoding = 104;
@@ -65,6 +71,8 @@ foreach Index = 0-255 in {
 //  Groupings using register classes and tuples
 //===----------------------------------------------------------------------===//
 
+// TODO: Do we need to set DwarfRegAlias on register tuples?
+
 // SGPR 32-bit registers
 def SGPR_32 : RegisterClass<"AMDGPU", [i32, f32], 32,
                             (add (sequence "SGPR%u", 0, 101))>;
@@ -182,11 +190,10 @@ def SCCReg : RegisterClass<"AMDGPU", [i32, i1], 32, (add SCC)> {
 
 def VCCReg : RegisterClass<"AMDGPU", [i64, i1], 64, (add VCC)>;
 def EXECReg : RegisterClass<"AMDGPU", [i64, i1], 64, (add EXEC)>;
-def M0Reg : RegisterClass<"AMDGPU", [i32], 32, (add M0)>;
 
 // Register class for all scalar registers (SGPRs + Special Registers)
 def SReg_32 : RegisterClass<"AMDGPU", [i32, f32], 32,
-  (add SGPR_32, M0Reg, VCC_LO, VCC_HI, EXEC_LO, EXEC_HI, FLAT_SCR_LO, FLAT_SCR_HI)
+  (add SGPR_32, M0, VCC_LO, VCC_HI, EXEC_LO, EXEC_HI, FLAT_SCR_LO, FLAT_SCR_HI)
 >;
 
 def SGPR_64 : RegisterClass<"AMDGPU", [v2i32, i64, f64], 64, (add SGPR_64Regs)>;
diff --git a/lib/Target/R600/SISchedule.td b/lib/Target/AMDGPU/SISchedule.td
similarity index 100%
rename from lib/Target/R600/SISchedule.td
rename to lib/Target/AMDGPU/SISchedule.td
diff --git a/lib/Target/R600/SIShrinkInstructions.cpp b/lib/Target/AMDGPU/SIShrinkInstructions.cpp
similarity index 89%
rename from lib/Target/R600/SIShrinkInstructions.cpp
rename to lib/Target/AMDGPU/SIShrinkInstructions.cpp
index 51e72cdb5f9..5d00bdd6a9b 100644
--- a/lib/Target/R600/SIShrinkInstructions.cpp
+++ b/lib/Target/AMDGPU/SIShrinkInstructions.cpp
@@ -94,8 +94,20 @@ static bool canShrink(MachineInstr &MI, const SIInstrInfo *TII,
   // is vcc.  We should handle this the same way we handle vopc, by addding
   // a register allocation hint pre-regalloc and then do the shrining
   // post-regalloc.
-  if (Src2)
-    return false;
+  if (Src2) {
+    switch (MI.getOpcode()) {
+      default: return false;
+
+      case AMDGPU::V_MAC_F32_e64:
+        if (!isVGPR(Src2, TRI, MRI) ||
+            TII->hasModifiersSet(MI, AMDGPU::OpName::src2_modifiers))
+          return false;
+        break;
+
+      case AMDGPU::V_CNDMASK_B32_e64:
+        break;
+    }
+  }
 
   const MachineOperand *Src1 = TII->getNamedOperand(MI, AMDGPU::OpName::src1);
   const MachineOperand *Src1Mod =
@@ -149,7 +161,7 @@ static void foldImmediates(MachineInstr &MI, const SIInstrInfo *TII,
     return;
 
   // Try to fold Src0
-  if (Src0.isReg()) {
+  if (Src0.isReg() && MRI.hasOneUse(Src0.getReg())) {
     unsigned Reg = Src0.getReg();
     MachineInstr *Def = MRI.getUniqueVRegDef(Reg);
     if (Def && Def->isMoveImmediate()) {
@@ -243,6 +255,22 @@ bool SIShrinkInstructions::runOnMachineFunction(MachineFunction &MF) {
           continue;
       }
 
+      if (Op32 == AMDGPU::V_CNDMASK_B32_e32) {
+        // We shrink V_CNDMASK_B32_e64 using regalloc hints like we do for VOPC
+        // instructions.
+        const MachineOperand *Src2 =
+            TII->getNamedOperand(MI, AMDGPU::OpName::src2);
+        if (!Src2->isReg())
+          continue;
+        unsigned SReg = Src2->getReg();
+        if (TargetRegisterInfo::isVirtualRegister(SReg)) {
+          MRI.setRegAllocationHint(SReg, 0, AMDGPU::VCC);
+          continue;
+        }
+        if (SReg != AMDGPU::VCC)
+          continue;
+      }
+
       // We can shrink this instruction
       DEBUG(dbgs() << "Shrinking "; MI.dump(); dbgs() << '\n';);
 
@@ -259,6 +287,11 @@ bool SIShrinkInstructions::runOnMachineFunction(MachineFunction &MF) {
       if (Src1)
         Inst32.addOperand(*Src1);
 
+      const MachineOperand *Src2 =
+          TII->getNamedOperand(MI, AMDGPU::OpName::src2);
+      if (Src2)
+        Inst32.addOperand(*Src2);
+
       ++NumInstructionsShrunk;
       MI.eraseFromParent();
 
diff --git a/lib/Target/R600/SITypeRewriter.cpp b/lib/Target/AMDGPU/SITypeRewriter.cpp
similarity index 100%
rename from lib/Target/R600/SITypeRewriter.cpp
rename to lib/Target/AMDGPU/SITypeRewriter.cpp
diff --git a/lib/Target/R600/TargetInfo/AMDGPUTargetInfo.cpp b/lib/Target/AMDGPU/TargetInfo/AMDGPUTargetInfo.cpp
similarity index 95%
rename from lib/Target/R600/TargetInfo/AMDGPUTargetInfo.cpp
rename to lib/Target/AMDGPU/TargetInfo/AMDGPUTargetInfo.cpp
index d723d6e3e8b..2112135aa5d 100644
--- a/lib/Target/R600/TargetInfo/AMDGPUTargetInfo.cpp
+++ b/lib/Target/AMDGPU/TargetInfo/AMDGPUTargetInfo.cpp
@@ -23,7 +23,7 @@ Target llvm::TheAMDGPUTarget;
 Target llvm::TheGCNTarget;
 
 /// \brief Extern function to initialize the targets for the AMDGPU backend
-extern "C" void LLVMInitializeR600TargetInfo() {
+extern "C" void LLVMInitializeAMDGPUTargetInfo() {
   RegisterTarget<Triple::r600, false>
     R600(TheAMDGPUTarget, "r600", "AMD GPUs HD2XXX-HD6XXX");
   RegisterTarget<Triple::amdgcn, false> GCN(TheGCNTarget, "amdgcn", "AMD GCN GPUs");
diff --git a/lib/Target/AMDGPU/TargetInfo/CMakeLists.txt b/lib/Target/AMDGPU/TargetInfo/CMakeLists.txt
new file mode 100644
index 00000000000..961dc550900
--- /dev/null
+++ b/lib/Target/AMDGPU/TargetInfo/CMakeLists.txt
@@ -0,0 +1,3 @@
+add_llvm_library(LLVMAMDGPUInfo
+  AMDGPUTargetInfo.cpp
+  )
diff --git a/lib/Target/AMDGPU/TargetInfo/LLVMBuild.txt b/lib/Target/AMDGPU/TargetInfo/LLVMBuild.txt
new file mode 100644
index 00000000000..291317fa072
--- /dev/null
+++ b/lib/Target/AMDGPU/TargetInfo/LLVMBuild.txt
@@ -0,0 +1,23 @@
+;===- ./lib/Target/AMDGPU/TargetInfo/LLVMBuild.txt --------------*- Conf -*--===;
+;
+;                     The LLVM Compiler Infrastructure
+;
+; This file is distributed under the University of Illinois Open Source
+; License. See LICENSE.TXT for details.
+;
+;===------------------------------------------------------------------------===;
+;
+; This is an LLVMBuild description file for the components in this subdirectory.
+;
+; For more information on the LLVMBuild system, please see:
+;
+;   http://llvm.org/docs/LLVMBuild.html
+;
+;===------------------------------------------------------------------------===;
+
+[component_0]
+type = Library
+name = AMDGPUInfo
+parent = AMDGPU
+required_libraries = Support
+add_to_library_groups = AMDGPU
diff --git a/lib/Target/AMDGPU/TargetInfo/Makefile b/lib/Target/AMDGPU/TargetInfo/Makefile
new file mode 100644
index 00000000000..1b232871bd6
--- /dev/null
+++ b/lib/Target/AMDGPU/TargetInfo/Makefile
@@ -0,0 +1,15 @@
+##===- lib/Target/AMDGPU/TargetInfo/Makefile ----------------*- Makefile -*-===##
+#
+#                     The LLVM Compiler Infrastructure
+#
+# This file is distributed under the University of Illinois Open Source
+# License. See LICENSE.TXT for details.
+#
+##===----------------------------------------------------------------------===##
+LEVEL = ../../../..
+LIBRARYNAME = LLVMAMDGPUInfo
+
+# Hack: we need to include 'main' target directory to grab private headers
+CPPFLAGS = -I$(PROJ_OBJ_DIR)/.. -I$(PROJ_SRC_DIR)/..
+
+include $(LEVEL)/Makefile.common
diff --git a/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.cpp b/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.cpp
new file mode 100644
index 00000000000..b76b4007003
--- /dev/null
+++ b/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.cpp
@@ -0,0 +1,60 @@
+//===-- AMDGPUBaseInfo.cpp - AMDGPU Base encoding information--------------===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+#include "AMDGPUBaseInfo.h"
+#include "llvm/MC/SubtargetFeature.h"
+
+#define GET_SUBTARGETINFO_ENUM
+#include "AMDGPUGenSubtargetInfo.inc"
+#undef GET_SUBTARGETINFO_ENUM
+
+namespace llvm {
+namespace AMDGPU {
+
+IsaVersion getIsaVersion(const FeatureBitset &Features) {
+
+  if (Features.test(FeatureISAVersion7_0_0))
+    return {7, 0, 0};
+
+  if (Features.test(FeatureISAVersion7_0_1))
+    return {7, 0, 1};
+
+  if (Features.test(FeatureISAVersion8_0_0))
+    return {8, 0, 0};
+
+  if (Features.test(FeatureISAVersion8_0_1))
+    return {8, 0, 1};
+
+  return {0, 0, 0};
+}
+
+void initDefaultAMDKernelCodeT(amd_kernel_code_t &Header,
+                               const FeatureBitset &Features) {
+
+  IsaVersion ISA = getIsaVersion(Features);
+
+  memset(&Header, 0, sizeof(Header));
+
+  Header.amd_kernel_code_version_major = 1;
+  Header.amd_kernel_code_version_minor = 0;
+  Header.amd_machine_kind = 1; // AMD_MACHINE_KIND_AMDGPU
+  Header.amd_machine_version_major = ISA.Major;
+  Header.amd_machine_version_minor = ISA.Minor;
+  Header.amd_machine_version_stepping = ISA.Stepping;
+  Header.kernel_code_entry_byte_offset = sizeof(Header);
+  // wavefront_size is specified as a power of 2: 2^6 = 64 threads.
+  Header.wavefront_size = 6;
+  // These alignment values are specified in powers of two, so alignment =
+  // 2^n.  The minimum alignment is 2^4 = 16.
+  Header.kernarg_segment_alignment = 4;
+  Header.group_segment_alignment = 4;
+  Header.private_segment_alignment = 4;
+}
+
+} // End namespace AMDGPU
+} // End namespace llvm
diff --git a/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.h b/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.h
new file mode 100644
index 00000000000..f57028cc5bf
--- /dev/null
+++ b/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.h
@@ -0,0 +1,34 @@
+//===-- AMDGPUBaseInfo.h - Top level definitions for AMDGPU -----*- C++ -*-===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIB_TARGET_AMDGPU_UTILS_AMDGPUBASEINFO_H
+#define LLVM_LIB_TARGET_AMDGPU_UTILS_AMDGPUBASEINFO_H
+
+#include "AMDKernelCodeT.h"
+
+namespace llvm {
+
+class FeatureBitset;
+
+namespace AMDGPU {
+
+struct IsaVersion {
+  unsigned Major;
+  unsigned Minor;
+  unsigned Stepping;
+};
+
+IsaVersion getIsaVersion(const FeatureBitset &Features);
+void initDefaultAMDKernelCodeT(amd_kernel_code_t &Header,
+                               const FeatureBitset &Features);
+
+} // end namespace AMDGPU
+} // end namespace llvm
+
+#endif
diff --git a/lib/Target/AMDGPU/Utils/CMakeLists.txt b/lib/Target/AMDGPU/Utils/CMakeLists.txt
new file mode 100644
index 00000000000..2c07aeab7dd
--- /dev/null
+++ b/lib/Target/AMDGPU/Utils/CMakeLists.txt
@@ -0,0 +1,3 @@
+add_llvm_library(LLVMAMDGPUUtils
+  AMDGPUBaseInfo.cpp
+  )
diff --git a/lib/Target/AMDGPU/Utils/LLVMBuild.txt b/lib/Target/AMDGPU/Utils/LLVMBuild.txt
new file mode 100644
index 00000000000..dec5360e3bc
--- /dev/null
+++ b/lib/Target/AMDGPU/Utils/LLVMBuild.txt
@@ -0,0 +1,23 @@
+;===- ./lib/Target/AMDGPU/Utils/LLVMBuild.txt ------------------*- Conf -*--===;
+;
+;                     The LLVM Compiler Infrastructure
+;
+; This file is distributed under the University of Illinois Open Source
+; License. See LICENSE.TXT for details.
+;
+;===------------------------------------------------------------------------===;
+;
+; This is an LLVMBuild description file for the components in this subdirectory.
+;
+; For more information on the LLVMBuild system, please see:
+;
+;   http://llvm.org/docs/LLVMBuild.html
+;
+;===------------------------------------------------------------------------===;
+
+[component_0]
+type = Library
+name = AMDGPUUtils
+parent = AMDGPU
+required_libraries = Support
+add_to_library_groups = AMDGPU
diff --git a/lib/Target/AMDGPU/Utils/Makefile b/lib/Target/AMDGPU/Utils/Makefile
new file mode 100644
index 00000000000..1019e726d50
--- /dev/null
+++ b/lib/Target/AMDGPU/Utils/Makefile
@@ -0,0 +1,16 @@
+##===- lib/Target/AMDGPU/Utils/Makefile --------------------*- Makefile -*-===##
+#
+#                     The LLVM Compiler Infrastructure
+#
+# This file is distributed under the University of Illinois Open Source
+# License. See LICENSE.TXT for details.
+#
+##===----------------------------------------------------------------------===##
+LEVEL = ../../../..
+LIBRARYNAME = LLVMAMDGPUUtils
+
+# Hack: we need to include 'main' AMDGPU target directory to grab private
+# headers
+CPP.Flags += -I$(PROJ_OBJ_DIR)/.. -I$(PROJ_SRC_DIR)/..
+
+include $(LEVEL)/Makefile.common
diff --git a/lib/Target/R600/VIInstrFormats.td b/lib/Target/AMDGPU/VIInstrFormats.td
similarity index 100%
rename from lib/Target/R600/VIInstrFormats.td
rename to lib/Target/AMDGPU/VIInstrFormats.td
diff --git a/lib/Target/AMDGPU/VIInstructions.td b/lib/Target/AMDGPU/VIInstructions.td
new file mode 100644
index 00000000000..aca46732adb
--- /dev/null
+++ b/lib/Target/AMDGPU/VIInstructions.td
@@ -0,0 +1,148 @@
+//===-- VIInstructions.td - VI Instruction Defintions ---------------------===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+// Instruction definitions for VI and newer.
+//===----------------------------------------------------------------------===//
+
+let SIAssemblerPredicate = DisableInst, SubtargetPredicate = isVI in {
+
+//===----------------------------------------------------------------------===//
+// VOP1 Instructions
+//===----------------------------------------------------------------------===//
+
+defm V_CVT_F16_U16 : VOP1Inst <vop1<0, 0x39>, "v_cvt_f16_u16", VOP_F16_I16>;
+defm V_CVT_F16_I16 : VOP1Inst <vop1<0, 0x3a>, "v_cvt_f16_i16", VOP_F16_I16>;
+defm V_CVT_U16_F16 : VOP1Inst <vop1<0, 0x3b>, "v_cvt_u16_f16", VOP_I16_F16>;
+defm V_CVT_I16_F16 : VOP1Inst <vop1<0, 0x3c>, "v_cvt_i16_f16", VOP_I16_F16>;
+defm V_RCP_F16 : VOP1Inst <vop1<0, 0x3d>, "v_rcp_f16", VOP_F16_F16>;
+defm V_SQRT_F16 : VOP1Inst <vop1<0, 0x3e>, "v_sqrt_f16", VOP_F16_F16>;
+defm V_RSQ_F16 : VOP1Inst <vop1<0, 0x3f>, "v_rsq_f16", VOP_F16_F16>;
+defm V_LOG_F16 : VOP1Inst <vop1<0, 0x40>, "v_log_f16", VOP_F16_F16>;
+defm V_EXP_F16 : VOP1Inst <vop1<0, 0x41>, "v_exp_f16", VOP_F16_F16>;
+defm V_FREXP_MANT_F16 : VOP1Inst <vop1<0, 0x42>, "v_frexp_mant_f16",
+  VOP_F16_F16
+>;
+defm V_FREXP_EXP_I16_F16 : VOP1Inst <vop1<0, 0x43>, "v_frexp_exp_i16_f16",
+  VOP_I16_F16
+>;
+defm V_FLOOR_F16 : VOP1Inst <vop1<0, 0x44>, "v_floor_f16", VOP_F16_F16>;
+defm V_CEIL_F16 : VOP1Inst <vop1<0, 0x45>, "v_ceil_f16", VOP_F16_F16>;
+defm V_TRUNC_F16 : VOP1Inst <vop1<0, 0x46>, "v_trunc_f16", VOP_F16_F16>;
+defm V_RNDNE_F16 : VOP1Inst <vop1<0, 0x47>, "v_rndne_f16", VOP_F16_F16>;
+defm V_FRACT_F16 : VOP1Inst <vop1<0, 0x48>, "v_fract_f16", VOP_F16_F16>;
+defm V_SIN_F16 : VOP1Inst <vop1<0, 0x49>, "v_sin_f16", VOP_F16_F16>;
+defm V_COS_F16 : VOP1Inst <vop1<0, 0x4a>, "v_cos_f16", VOP_F16_F16>;
+
+//===----------------------------------------------------------------------===//
+// VOP2 Instructions
+//===----------------------------------------------------------------------===//
+
+let isCommutable = 1 in {
+
+defm V_ADD_F16 : VOP2Inst <vop2<0, 0x1f>, "v_add_f16", VOP_F16_F16_F16>;
+defm V_SUB_F16 : VOP2Inst <vop2<0, 0x20>, "v_sub_f16", VOP_F16_F16_F16>;
+defm V_SUBREV_F16 : VOP2Inst <vop2<0, 0x21>, "v_subrev_f16", VOP_F16_F16_F16,
+  null_frag, "v_sub_f16"
+>;
+defm V_MUL_F16 : VOP2Inst <vop2<0, 0x22>, "v_mul_f16", VOP_F16_F16_F16>;
+defm V_MAC_F16 : VOP2Inst <vop2<0, 0x23>, "v_mac_f16", VOP_F16_F16_F16>;
+} // End isCommutable = 1
+defm V_MADMK_F16 : VOP2MADK <vop2<0,0x24>, "v_madmk_f16">;
+let isCommutable = 1 in {
+defm V_MADAK_F16 : VOP2MADK <vop2<0,0x25>, "v_madak_f16">;
+defm V_ADD_U16 : VOP2Inst <vop2<0,0x26>, "v_add_u16", VOP_I16_I16_I16>;
+defm V_SUB_U16 : VOP2Inst <vop2<0,0x27>, "v_sub_u16" , VOP_I16_I16_I16>;
+defm V_SUBREV_U16 : VOP2Inst <vop2<0,0x28>, "v_subrev_u16", VOP_I16_I16_I16>;
+defm V_MUL_LO_U16 : VOP2Inst <vop2<0,0x29>, "v_mul_lo_u16", VOP_I16_I16_I16>;
+} // End isCommutable = 1
+defm V_LSHLREV_B16 : VOP2Inst <vop2<0,0x2a>, "v_lshlrev_b16", VOP_I16_I16_I16>;
+defm V_LSHRREV_B16 : VOP2Inst <vop2<0,0x2b>, "v_lshrrev_b16", VOP_I16_I16_I16>;
+defm V_ASHRREV_B16 : VOP2Inst <vop2<0,0x2c>, "v_ashrrev_b16", VOP_I16_I16_I16>;
+let isCommutable = 1 in {
+defm V_MAX_F16 : VOP2Inst <vop2<0,0x2d>, "v_max_f16", VOP_F16_F16_F16>;
+defm V_MIN_F16 : VOP2Inst <vop2<0,0x2e>, "v_min_f16", VOP_F16_F16_F16>;
+defm V_MAX_U16 : VOP2Inst <vop2<0,0x2f>, "v_max_u16", VOP_I16_I16_I16>;
+defm V_MAX_I16 : VOP2Inst <vop2<0,0x30>, "v_max_i16", VOP_I16_I16_I16>;
+defm V_MIN_U16 : VOP2Inst <vop2<0,0x31>, "v_min_u16", VOP_I16_I16_I16>;
+defm V_MIN_I16 : VOP2Inst <vop2<0,0x32>, "v_min_i16", VOP_I16_I16_I16>;
+} // End isCommutable = 1
+defm V_LDEXP_F16 : VOP2Inst <vop2<0,0x33>, "v_ldexp_f16", VOP_F16_F16_I16>;
+
+// Aliases to simplify matching of floating-pint instructions that are VOP2 on
+// SI and VOP3 on VI.
+
+class SI2_VI3Alias <string name, Instruction inst> : InstAlias <
+  name#" $dst, $src0, $src1",
+  (inst VGPR_32:$dst, 0, VCSrc_32:$src0, 0, VCSrc_32:$src1, 0, 0)
+>, PredicateControl {
+  let UseInstAsmMatchConverter = 0;
+}
+
+def : SI2_VI3Alias <"v_ldexp_f32", V_LDEXP_F32_e64_vi>;
+def : SI2_VI3Alias <"v_cvt_pkaccum_u8_f32", V_CVT_PKACCUM_U8_F32_e64_vi>;
+def : SI2_VI3Alias <"v_cvt_pknorm_i16_f32", V_CVT_PKNORM_I16_F32_e64_vi>;
+def : SI2_VI3Alias <"v_cvt_pknorm_u16_f32", V_CVT_PKNORM_U16_F32_e64_vi>;
+def : SI2_VI3Alias <"v_cvt_pkrtz_f16_f32", V_CVT_PKRTZ_F16_F32_e64_vi>;
+
+} // End SIAssemblerPredicate = DisableInst, SubtargetPredicate = isVI
+
+//===----------------------------------------------------------------------===//
+// SMEM Patterns
+//===----------------------------------------------------------------------===//
+
+let Predicates = [isVI] in {
+
+// 1. Offset as 20bit DWORD immediate
+def : Pat <
+  (SIload_constant v4i32:$sbase, IMM20bit:$offset),
+  (S_BUFFER_LOAD_DWORD_IMM $sbase, (as_i32imm $offset))
+>;
+
+// Patterns for global loads with no offset
+class FlatLoadPat <FLAT inst, SDPatternOperator node, ValueType vt> : Pat <
+  (vt (node i64:$addr)),
+  (inst $addr, 0, 0, 0)
+>;
+
+def : FlatLoadPat <FLAT_LOAD_UBYTE, az_extloadi8_global, i32>;
+def : FlatLoadPat <FLAT_LOAD_SBYTE, sextloadi8_global, i32>;
+def : FlatLoadPat <FLAT_LOAD_USHORT, az_extloadi16_global, i32>;
+def : FlatLoadPat <FLAT_LOAD_SSHORT, sextloadi16_global, i32>;
+def : FlatLoadPat <FLAT_LOAD_DWORD, global_load, i32>;
+def : FlatLoadPat <FLAT_LOAD_DWORDX2, global_load, v2i32>;
+def : FlatLoadPat <FLAT_LOAD_DWORDX4, global_load, v4i32>;
+
+class FlatStorePat <FLAT inst, SDPatternOperator node, ValueType vt> : Pat <
+  (node vt:$data, i64:$addr),
+  (inst $data, $addr, 0, 0, 0)
+>;
+
+def : FlatStorePat <FLAT_STORE_BYTE, truncstorei8_global, i32>;
+def : FlatStorePat <FLAT_STORE_SHORT, truncstorei16_global, i32>;
+def : FlatStorePat <FLAT_STORE_DWORD, global_store, i32>;
+def : FlatStorePat <FLAT_STORE_DWORDX2, global_store, v2i32>;
+def : FlatStorePat <FLAT_STORE_DWORDX4, global_store, v4i32>;
+
+class FlatAtomicPat <FLAT inst, SDPatternOperator node, ValueType vt> : Pat <
+  (vt (node i64:$addr, vt:$data)),
+  (inst $addr, $data, 0, 0)
+>;
+
+def : FlatAtomicPat <FLAT_ATOMIC_ADD_RTN, atomic_add_global, i32>;
+def : FlatAtomicPat <FLAT_ATOMIC_AND_RTN, atomic_and_global, i32>;
+def : FlatAtomicPat <FLAT_ATOMIC_SUB_RTN, atomic_sub_global, i32>;
+def : FlatAtomicPat <FLAT_ATOMIC_SMAX_RTN, atomic_max_global, i32>;
+def : FlatAtomicPat <FLAT_ATOMIC_UMAX_RTN, atomic_umax_global, i32>;
+def : FlatAtomicPat <FLAT_ATOMIC_SMIN_RTN, atomic_min_global, i32>;
+def : FlatAtomicPat <FLAT_ATOMIC_UMIN_RTN, atomic_umin_global, i32>;
+def : FlatAtomicPat <FLAT_ATOMIC_OR_RTN, atomic_or_global, i32>;
+def : FlatAtomicPat <FLAT_ATOMIC_SWAP_RTN, atomic_swap_global, i32>;
+def : FlatAtomicPat <FLAT_ATOMIC_XOR_RTN, atomic_xor_global, i32>;
+
+
+} // End Predicates = [isVI]
diff --git a/lib/Target/ARM/ARM.h b/lib/Target/ARM/ARM.h
index 3f84048cadf..9550a3a3cad 100644
--- a/lib/Target/ARM/ARM.h
+++ b/lib/Target/ARM/ARM.h
@@ -16,14 +16,13 @@
 #define LLVM_LIB_TARGET_ARM_ARM_H
 
 #include "llvm/Support/CodeGen.h"
-
-// @LOCALMOD (for LowerARMMachineInstrToMCInstPCRel)
-#include "llvm/MC/MCSymbol.h"
+#include <functional>
 
 namespace llvm {
 
 class ARMAsmPrinter;
 class ARMBaseTargetMachine;
+class Function;
 class FunctionPass;
 class ImmutablePass;
 class MachineInstr;
@@ -41,28 +40,12 @@ FunctionPass *createARMConstantIslandPass();
 FunctionPass *createMLxExpansionPass();
 FunctionPass *createThumb2ITBlockPass();
 FunctionPass *createARMOptimizeBarriersPass();
-FunctionPass *createThumb2SizeReductionPass();
-
-/* @LOCALMOD-START */
-FunctionPass *createARMNaClRewritePass();
-/* @LOCALMOD-END */
+FunctionPass *createThumb2SizeReductionPass(
+    std::function<bool(const Function &)> Ftor = nullptr);
 
 void LowerARMMachineInstrToMCInst(const MachineInstr *MI, MCInst &OutMI,
                                   ARMAsmPrinter &AP);
 
-/* @LOCALMOD-START */
-// Used to lower the pc-relative MOVi16PIC / MOVTi16PIC pseudo instructions
-// into the real MOVi16 / MOVTi16 instructions.
-// See comment on MOVi16PIC for more details.
-void LowerARMMachineInstrToMCInstPCRel(const MachineInstr *MI,
-                                       MCInst &OutMI,
-                                       ARMAsmPrinter &AP,
-                                       unsigned ImmIndex,
-                                       unsigned PCIndex,
-                                       MCSymbol *PCLabel,
-                                       unsigned PCAdjustment);
-/* @LOCALMOD-END */
-
 } // end namespace llvm;
 
 #endif
diff --git a/lib/Target/ARM/ARM.td b/lib/Target/ARM/ARM.td
index 9882227905c..3e0c0519d15 100644
--- a/lib/Target/ARM/ARM.td
+++ b/lib/Target/ARM/ARM.td
@@ -23,6 +23,9 @@ include "llvm/Target/Target.td"
 def ModeThumb  : SubtargetFeature<"thumb-mode", "InThumbMode", "true",
                                   "Thumb mode">;
 
+def ModeSoftFloat : SubtargetFeature<"soft-float", "UseSoftFloat", "true",
+                                     "Use software floating point features.">;
+
 //===----------------------------------------------------------------------===//
 // ARM Subtarget features.
 //
@@ -147,6 +150,23 @@ def FeatureAClass : SubtargetFeature<"aclass", "ARMProcClass", "AClass",
 def FeatureNaClTrap : SubtargetFeature<"nacl-trap", "UseNaClTrap", "true",
                                        "NaCl trap">;
 
+def FeatureStrictAlign : SubtargetFeature<"strict-align",
+                                          "StrictAlign", "true",
+                                          "Disallow all unaligned memory "
+                                          "access">;
+
+def FeatureLongCalls : SubtargetFeature<"long-calls", "GenLongCalls", "true",
+                                        "Generate calls via indirect call "
+                                        "instructions">;
+
+def FeatureReserveR9 : SubtargetFeature<"reserve-r9", "ReserveR9", "true",
+                                        "Reserve R9, making it unavailable as "
+                                        "GPR">;
+
+def FeatureNoMovt : SubtargetFeature<"no-movt", "NoMovt", "true",
+                                     "Don't use movt/movw pairs for 32-bit "
+                                     "imms">;
+
 // ARM ISAs.
 def HasV4TOps   : SubtargetFeature<"v4t", "HasV4TOps", "true",
                                    "Support ARM v4T instructions">;
@@ -407,13 +427,13 @@ def : ProcessorModel<"cortex-r4",   CortexA8Model,
 def : ProcessorModel<"cortex-r4f",  CortexA8Model,
                                     [ProcR4,
                                      FeatureSlowFPBrcc, FeatureHasSlowFPVMLx,
-                                     FeatureVFP3, FeatureVFPOnlySP, FeatureD16]>;
+                                     FeatureVFP3, FeatureD16]>;
 
 // FIXME: R5 has currently the same ProcessorModel as A8.
 def : ProcessorModel<"cortex-r5",   CortexA8Model,
                                     [ProcR5, HasV7Ops, FeatureDB,
                                      FeatureVFP3, FeatureDSPThumb2,
-                                     FeatureHasRAS, FeatureVFPOnlySP,
+                                     FeatureHasRAS,
                                      FeatureD16, FeatureRClass]>;
 
 // FIXME: R7 has currently the same ProcessorModel as A8 and is modelled as R5.
diff --git a/lib/Target/ARM/ARMArchExtName.def b/lib/Target/ARM/ARMArchExtName.def
deleted file mode 100644
index d6da50c59e5..00000000000
--- a/lib/Target/ARM/ARMArchExtName.def
+++ /dev/null
@@ -1,30 +0,0 @@
-//===-- ARMArchExtName.def - List of the ARM Extension names ----*- C++ -*-===//
-//
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
-//
-//===----------------------------------------------------------------------===//
-//
-// This file contains the list of the supported ARM Architecture Extension
-// names. These can be used to enable the extension through .arch_extension
-// attribute
-//
-//===----------------------------------------------------------------------===//
-
-// NOTE: NO INCLUDE GUARD DESIRED!
-
-#ifndef ARM_ARCHEXT_NAME
-#error "You must define ARM_ARCHEXT_NAME(NAME, ID) before including ARMArchExtName.h"
-#endif
-
-ARM_ARCHEXT_NAME("crc", CRC)
-ARM_ARCHEXT_NAME("crypto", CRYPTO)
-ARM_ARCHEXT_NAME("fp", FP)
-ARM_ARCHEXT_NAME("idiv", HWDIV)
-ARM_ARCHEXT_NAME("mp", MP)
-ARM_ARCHEXT_NAME("sec", SEC)
-ARM_ARCHEXT_NAME("virt", VIRT)
-
-#undef ARM_ARCHEXT_NAME
diff --git a/lib/Target/ARM/ARMArchExtName.h b/lib/Target/ARM/ARMArchExtName.h
deleted file mode 100644
index bc1157acdbe..00000000000
--- a/lib/Target/ARM/ARMArchExtName.h
+++ /dev/null
@@ -1,26 +0,0 @@
-//===-- ARMArchExtName.h - List of the ARM Extension names ------*- C++ -*-===//
-//
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
-//
-//===----------------------------------------------------------------------===//
-
-#ifndef LLVM_LIB_TARGET_ARM_ARMARCHEXTNAME_H
-#define LLVM_LIB_TARGET_ARM_ARMARCHEXTNAME_H
-
-namespace llvm {
-namespace ARM {
-
-enum ArchExtKind {
-  INVALID_ARCHEXT = 0
-
-#define ARM_ARCHEXT_NAME(NAME, ID) , ID
-#include "ARMArchExtName.def"
-};
-
-} // namespace ARM
-} // namespace llvm
-
-#endif
diff --git a/lib/Target/ARM/ARMAsmPrinter.cpp b/lib/Target/ARM/ARMAsmPrinter.cpp
index ab9d6d31b57..f5f12472437 100644
--- a/lib/Target/ARM/ARMAsmPrinter.cpp
+++ b/lib/Target/ARM/ARMAsmPrinter.cpp
@@ -15,8 +15,6 @@
 #include "ARMAsmPrinter.h"
 #include "ARM.h"
 #include "ARMConstantPoolValue.h"
-#include "ARMFPUName.h"
-#include "ARMArchExtName.h"
 #include "ARMMachineFunctionInfo.h"
 #include "ARMTargetMachine.h"
 #include "ARMTargetObjectFile.h"
@@ -40,12 +38,12 @@
 #include "llvm/MC/MCELFStreamer.h"
 #include "llvm/MC/MCInst.h"
 #include "llvm/MC/MCInstBuilder.h"
-#include "llvm/MC/MCNaCl.h"  // @LOCALMOD
 #include "llvm/MC/MCObjectStreamer.h"
 #include "llvm/MC/MCSectionMachO.h"
 #include "llvm/MC/MCStreamer.h"
 #include "llvm/MC/MCSymbol.h"
 #include "llvm/Support/ARMBuildAttributes.h"
+#include "llvm/Support/TargetParser.h"
 #include "llvm/Support/COFF.h"
 #include "llvm/Support/CommandLine.h"
 #include "llvm/Support/Debug.h"
@@ -59,77 +57,6 @@ using namespace llvm;
 
 #define DEBUG_TYPE "asm-printer"
 
-
-// @LOCALMOD-START
-// Make sure all jump targets are aligned and also all constant pools
-static void NaClAlignAllJumpTargetsAndConstantPools(MachineFunction &MF) {
-  // JUMP TABLE TARGETS
-  MachineJumpTableInfo *jt_info = MF.getJumpTableInfo();
-  if (jt_info) {
-    const std::vector<MachineJumpTableEntry> &JT = jt_info->getJumpTables();
-    for (unsigned i=0; i < JT.size(); ++i) {
-      std::vector<MachineBasicBlock*> MBBs = JT[i].MBBs;
-
-      for (unsigned j=0; j < MBBs.size(); ++j) {
-        if (MBBs[j]->begin()->getOpcode() == ARM::CONSTPOOL_ENTRY) {
-          continue;
-        }
-        MBBs[j]->setAlignment(4);
-      }
-    }
-  }
-
-  // FIRST ENTRY IN A ConstantPool
-  bool last_bb_was_constant_pool = false;
-  for (MachineFunction::iterator I = MF.begin(), E = MF.end();
-       I != E; ++I) {
-    if (I->isLandingPad()) {
-        I->setAlignment(4);
-    }
-
-    if (I->empty()) continue;
-
-    bool is_constant_pool = I->begin()->getOpcode() == ARM::CONSTPOOL_ENTRY;
-
-    if (last_bb_was_constant_pool != is_constant_pool) {
-      I->setAlignment(4);
-    }
-
-    last_bb_was_constant_pool = is_constant_pool;
-  }
-}
-
-bool ARMAsmPrinter::UseReadOnlyJumpTables() const {
-  if (Subtarget->isTargetNaCl())
-    return true;
-  return false;
-}
-
-unsigned ARMAsmPrinter::GetTargetBasicBlockAlign() const {
-  if (Subtarget->isTargetNaCl())
-    return 4;
-  return 0;
-}
-
-unsigned ARMAsmPrinter::GetTargetLabelAlign(const MachineInstr *MI) const {
-  if (Subtarget->isTargetNaCl()) {
-    switch (MI->getOpcode()) {
-      default: return 0;
-      // These labels may indicate an indirect entry point that is
-      // externally reachable and hence must be bundle aligned.
-      // Note: these labels appear to be always at basic block beginnings
-      // so it may be possible to simply set the MBB alignment.
-      // However, it is unclear whether this always holds.
-      case TargetOpcode::EH_LABEL:
-      case TargetOpcode::GC_LABEL:
-        return 4;
-    }
-  }
-  return 0;
-}
-// @LOCALMOD-END
-
-
 ARMAsmPrinter::ARMAsmPrinter(TargetMachine &TM,
                              std::unique_ptr<MCStreamer> Streamer)
     : AsmPrinter(TM, std::move(Streamer)), AFI(nullptr), MCP(nullptr),
@@ -141,45 +68,33 @@ void ARMAsmPrinter::EmitFunctionBodyEnd() {
   if (!InConstantPool)
     return;
   InConstantPool = false;
-  OutStreamer.EmitDataRegion(MCDR_DataRegionEnd);
+  OutStreamer->EmitDataRegion(MCDR_DataRegionEnd);
 }
 
 void ARMAsmPrinter::EmitFunctionEntryLabel() {
   if (AFI->isThumbFunction()) {
-    OutStreamer.EmitAssemblerFlag(MCAF_Code16);
-    OutStreamer.EmitThumbFunc(CurrentFnSym);
+    OutStreamer->EmitAssemblerFlag(MCAF_Code16);
+    OutStreamer->EmitThumbFunc(CurrentFnSym);
   }
 
-  // @LOCALMOD-START
-  // make sure function entry is aligned. We use  XmagicX as our basis
-  // for alignment decisions (c.f. assembler sfi macros)
-  if (Subtarget->isTargetNaCl()) {
-    EmitAlignment(std::max(MF->getAlignment(), 4u));
-
-    if (OutStreamer.hasRawTextSupport()) {
-      OutStreamer.EmitRawText(StringRef("\t.set XmagicX, .\n"));
-    }
-  }
-  // @LOCALMOD-END
- 
-  OutStreamer.EmitLabel(CurrentFnSym);
+  OutStreamer->EmitLabel(CurrentFnSym);
 }
 
-void ARMAsmPrinter::EmitXXStructor(const Constant *CV) {
-  uint64_t Size = TM.getDataLayout()->getTypeAllocSize(CV->getType());
+void ARMAsmPrinter::EmitXXStructor(const DataLayout &DL, const Constant *CV) {
+  uint64_t Size = getDataLayout().getTypeAllocSize(CV->getType());
   assert(Size && "C++ constructor pointer had zero size!");
 
   const GlobalValue *GV = dyn_cast<GlobalValue>(CV->stripPointerCasts());
   assert(GV && "C++ constructor pointer was not a GlobalValue!");
 
-  const MCExpr *E = MCSymbolRefExpr::Create(GetARMGVSymbol(GV,
+  const MCExpr *E = MCSymbolRefExpr::create(GetARMGVSymbol(GV,
                                                            ARMII::MO_NO_FLAG),
                                             (Subtarget->isTargetELF()
                                              ? MCSymbolRefExpr::VK_ARM_TARGET1
                                              : MCSymbolRefExpr::VK_None),
                                             OutContext);
 
-  OutStreamer.EmitValue(E, Size);
+  OutStreamer->EmitValue(E, Size);
 }
 
 /// runOnMachineFunction - This uses the EmitInstruction()
@@ -190,12 +105,6 @@ bool ARMAsmPrinter::runOnMachineFunction(MachineFunction &MF) {
   MCP = MF.getConstantPool();
   Subtarget = &MF.getSubtarget<ARMSubtarget>();
 
-  // @LOCALMOD-START
-  if (Subtarget->isTargetNaCl()) {
-    NaClAlignAllJumpTargetsAndConstantPools(MF);
-  }
-  // @LOCALMOD-END
-
   SetupMachineFunction(MF);
 
   if (Subtarget->isTargetCOFF()) {
@@ -204,10 +113,10 @@ bool ARMAsmPrinter::runOnMachineFunction(MachineFunction &MF) {
                                             : COFF::IMAGE_SYM_CLASS_EXTERNAL;
     int Type = COFF::IMAGE_SYM_DTYPE_FUNCTION << COFF::SCT_COMPLEX_TYPE_SHIFT;
 
-    OutStreamer.BeginCOFFSymbolDef(CurrentFnSym);
-    OutStreamer.EmitCOFFSymbolStorageClass(Scl);
-    OutStreamer.EmitCOFFSymbolType(Type);
-    OutStreamer.EndCOFFSymbolDef();
+    OutStreamer->BeginCOFFSymbolDef(CurrentFnSym);
+    OutStreamer->EmitCOFFSymbolStorageClass(Scl);
+    OutStreamer->EmitCOFFSymbolType(Type);
+    OutStreamer->EndCOFFSymbolDef();
   }
 
   // Emit the rest of the function body.
@@ -217,11 +126,11 @@ bool ARMAsmPrinter::runOnMachineFunction(MachineFunction &MF) {
   // These are created per function, rather than per TU, since it's
   // relatively easy to exceed the thumb branch range within a TU.
   if (! ThumbIndirectPads.empty()) {
-    OutStreamer.EmitAssemblerFlag(MCAF_Code16);
+    OutStreamer->EmitAssemblerFlag(MCAF_Code16);
     EmitAlignment(1);
     for (unsigned i = 0, e = ThumbIndirectPads.size(); i < e; i++) {
-      OutStreamer.EmitLabel(ThumbIndirectPads[i].second);
-      EmitToStreamer(OutStreamer, MCInstBuilder(ARM::tBX)
+      OutStreamer->EmitLabel(ThumbIndirectPads[i].second);
+      EmitToStreamer(*OutStreamer, MCInstBuilder(ARM::tBX)
         .addReg(ThumbIndirectPads[i].first)
         // Add predicate operands.
         .addImm(ARMCC::AL)
@@ -235,7 +144,7 @@ bool ARMAsmPrinter::runOnMachineFunction(MachineFunction &MF) {
 }
 
 void ARMAsmPrinter::printOperand(const MachineInstr *MI, int OpNum,
-                                 raw_ostream &O, const char *Modifier) {
+                                 raw_ostream &O) {
   const MachineOperand &MO = MI->getOperand(OpNum);
   unsigned TF = MO.getTargetFlags();
 
@@ -256,27 +165,23 @@ void ARMAsmPrinter::printOperand(const MachineInstr *MI, int OpNum,
   case MachineOperand::MO_Immediate: {
     int64_t Imm = MO.getImm();
     O << '#';
-    if ((Modifier && strcmp(Modifier, "lo16") == 0) ||
-        (TF == ARMII::MO_LO16))
+    if (TF == ARMII::MO_LO16)
       O << ":lower16:";
-    else if ((Modifier && strcmp(Modifier, "hi16") == 0) ||
-             (TF == ARMII::MO_HI16))
+    else if (TF == ARMII::MO_HI16)
       O << ":upper16:";
     O << Imm;
     break;
   }
   case MachineOperand::MO_MachineBasicBlock:
-    O << *MO.getMBB()->getSymbol();
+    MO.getMBB()->getSymbol()->print(O, MAI);
     return;
   case MachineOperand::MO_GlobalAddress: {
     const GlobalValue *GV = MO.getGlobal();
-    if ((Modifier && strcmp(Modifier, "lo16") == 0) ||
-        (TF == ARMII::MO_LO16)) // @LOCALMOD: TEMPORARY FIX (pr8252)
+    if (TF & ARMII::MO_LO16)
       O << ":lower16:";
-    else if ((Modifier && strcmp(Modifier, "hi16") == 0) ||
-             (TF == ARMII::MO_HI16)) // @LOCALMOD: TEMPORARY FIX (pr8252)
+    else if (TF & ARMII::MO_HI16)
       O << ":upper16:";
-    O << *GetARMGVSymbol(GV, TF);
+    GetARMGVSymbol(GV, TF)->print(O, MAI);
 
     printOffset(MO.getOffset(), O);
     if (TF == ARMII::MO_PLT)
@@ -284,7 +189,7 @@ void ARMAsmPrinter::printOperand(const MachineInstr *MI, int OpNum,
     break;
   }
   case MachineOperand::MO_ConstantPoolIndex:
-    O << *GetCPISymbol(MO.getIndex());
+    GetCPISymbol(MO.getIndex())->print(O, MAI);
     break;
   }
 }
@@ -292,21 +197,12 @@ void ARMAsmPrinter::printOperand(const MachineInstr *MI, int OpNum,
 //===--------------------------------------------------------------------===//
 
 MCSymbol *ARMAsmPrinter::
-GetARMJTIPICJumpTableLabel2(unsigned uid, unsigned uid2) const {
-  const DataLayout *DL = TM.getDataLayout();
+GetARMJTIPICJumpTableLabel(unsigned uid) const {
+  const DataLayout &DL = getDataLayout();
   SmallString<60> Name;
-  raw_svector_ostream(Name) << DL->getPrivateGlobalPrefix() << "JTI"
-    << getFunctionNumber() << '_' << uid << '_' << uid2;
-  return OutContext.GetOrCreateSymbol(Name);
-}
-
-
-MCSymbol *ARMAsmPrinter::GetARMSJLJEHLabel() const {
-  const DataLayout *DL = TM.getDataLayout();
-  SmallString<60> Name;
-  raw_svector_ostream(Name) << DL->getPrivateGlobalPrefix() << "SJLJEH"
-    << getFunctionNumber();
-  return OutContext.GetOrCreateSymbol(Name);
+  raw_svector_ostream(Name) << DL.getPrivateGlobalPrefix() << "JTI"
+                            << getFunctionNumber() << '_' << uid;
+  return OutContext.getOrCreateSymbol(Name);
 }
 
 bool ARMAsmPrinter::PrintAsmOperand(const MachineInstr *MI, unsigned OpNum,
@@ -509,10 +405,8 @@ bool ARMAsmPrinter::PrintAsmMemoryOperand(const MachineInstr *MI,
   return false;
 }
 
-void EmitSFIHeaders(raw_ostream &O); // @LOCALMOD
-
 static bool isThumb(const MCSubtargetInfo& STI) {
-  return (STI.getFeatureBits() & ARM::ModeThumb) != 0;
+  return STI.getFeatureBits()[ARM::ModeThumb];
 }
 
 void ARMAsmPrinter::emitInlineAsmEnd(const MCSubtargetInfo &StartInfo,
@@ -521,31 +415,19 @@ void ARMAsmPrinter::emitInlineAsmEnd(const MCSubtargetInfo &StartInfo,
   // the start mode, then restore the start mode.
   const bool WasThumb = isThumb(StartInfo);
   if (!EndInfo || WasThumb != isThumb(*EndInfo)) {
-    OutStreamer.EmitAssemblerFlag(WasThumb ? MCAF_Code16 : MCAF_Code32);
+    OutStreamer->EmitAssemblerFlag(WasThumb ? MCAF_Code16 : MCAF_Code32);
   }
 }
 
 void ARMAsmPrinter::EmitStartOfAsmFile(Module &M) {
-  Triple TT(TM.getTargetTriple());
+  const Triple &TT = TM.getTargetTriple();
   // Use unified assembler syntax.
-  OutStreamer.EmitAssemblerFlag(MCAF_SyntaxUnified);
+  OutStreamer->EmitAssemblerFlag(MCAF_SyntaxUnified);
 
   // Emit ARM Build Attributes
   if (TT.isOSBinFormatELF())
     emitAttributes();
 
-  // @LOCALMOD-BEGIN
-  if (TT.isOSNaCl()) {
-    if (OutStreamer.hasRawTextSupport()) {
-      std::string str;
-      raw_string_ostream OS(str);
-      EmitSFIHeaders(OS);
-      OutStreamer.EmitRawText(StringRef(OS.str()));
-    }
-    initializeNaClMCStreamer(OutStreamer, OutContext, TT);
-  }
-  // @LOCALMOD-END
-
   // Use the triple's architecture and subarchitecture to determine
   // if we're thumb for the purposes of the top level code16 assembler
   // flag.
@@ -554,7 +436,7 @@ void ARMAsmPrinter::EmitStartOfAsmFile(Module &M) {
                  TT.getSubArch() == Triple::ARMSubArch_v7m ||
                  TT.getSubArch() == Triple::ARMSubArch_v6m;
   if (!M.getModuleInlineAsm().empty() && isThumb)
-    OutStreamer.EmitAssemblerFlag(MCAF_Code16);
+    OutStreamer->EmitAssemblerFlag(MCAF_Code16);
 }
 
 static void
@@ -576,13 +458,13 @@ emitNonLazySymbolPointer(MCStreamer &OutStreamer, MCSymbol *StubLabel,
     // using NLPs; however, sometimes the types are local to the file.
     // We need to fill in the value for the NLP in those cases.
     OutStreamer.EmitValue(
-        MCSymbolRefExpr::Create(MCSym.getPointer(), OutStreamer.getContext()),
+        MCSymbolRefExpr::create(MCSym.getPointer(), OutStreamer.getContext()),
         4 /*size*/);
 }
 
 
 void ARMAsmPrinter::EmitEndOfAsmFile(Module &M) {
-  Triple TT(TM.getTargetTriple());
+  const Triple &TT = TM.getTargetTriple();
   if (TT.isOSBinFormatMachO()) {
     // All darwin targets use mach-o.
     const TargetLoweringObjectFileMachO &TLOFMacho =
@@ -595,26 +477,26 @@ void ARMAsmPrinter::EmitEndOfAsmFile(Module &M) {
 
     if (!Stubs.empty()) {
       // Switch with ".non_lazy_symbol_pointer" directive.
-      OutStreamer.SwitchSection(TLOFMacho.getNonLazySymbolPointerSection());
+      OutStreamer->SwitchSection(TLOFMacho.getNonLazySymbolPointerSection());
       EmitAlignment(2);
 
       for (auto &Stub : Stubs)
-        emitNonLazySymbolPointer(OutStreamer, Stub.first, Stub.second);
+        emitNonLazySymbolPointer(*OutStreamer, Stub.first, Stub.second);
 
       Stubs.clear();
-      OutStreamer.AddBlankLine();
+      OutStreamer->AddBlankLine();
     }
 
     Stubs = MMIMacho.GetHiddenGVStubList();
     if (!Stubs.empty()) {
-      OutStreamer.SwitchSection(TLOFMacho.getNonLazySymbolPointerSection());
+      OutStreamer->SwitchSection(TLOFMacho.getNonLazySymbolPointerSection());
       EmitAlignment(2);
 
       for (auto &Stub : Stubs)
-        emitNonLazySymbolPointer(OutStreamer, Stub.first, Stub.second);
+        emitNonLazySymbolPointer(*OutStreamer, Stub.first, Stub.second);
 
       Stubs.clear();
-      OutStreamer.AddBlankLine();
+      OutStreamer->AddBlankLine();
     }
 
     // Funny Darwin hack: This flag tells the linker that no global symbols
@@ -622,7 +504,7 @@ void ARMAsmPrinter::EmitEndOfAsmFile(Module &M) {
     // implementation of multiple entry points).  If this doesn't occur, the
     // linker can safely perform dead code stripping.  Since LLVM never
     // generates code that does this, it is always safe to set.
-    OutStreamer.EmitAssemblerFlag(MCAF_SubsectionsViaSymbols);
+    OutStreamer->EmitAssemblerFlag(MCAF_SubsectionsViaSymbols);
   }
 }
 
@@ -661,7 +543,7 @@ static ARMBuildAttrs::CPUArch getArchForCPU(StringRef CPU,
 }
 
 void ARMAsmPrinter::emitAttributes() {
-  MCTargetStreamer &TS = *OutStreamer.getTargetStreamer();
+  MCTargetStreamer &TS = *OutStreamer->getTargetStreamer();
   ARMTargetStreamer &ATS = static_cast<ARMTargetStreamer &>(TS);
 
   ATS.emitTextAttribute(ARMBuildAttrs::conformance, "2.09");
@@ -673,7 +555,7 @@ void ARMAsmPrinter::emitAttributes() {
   // anyhow.
   // FIXME: For ifunc related functions we could iterate over and look
   // for a feature string that doesn't match the default one.
-  StringRef TT = TM.getTargetTriple();
+  const Triple &TT = TM.getTargetTriple();
   StringRef CPU = TM.getTargetCPU();
   StringRef FS = TM.getTargetFeatureString();
   std::string ArchFS = ARM_MC::ParseARMTriple(TT, CPU);
@@ -696,7 +578,7 @@ void ARMAsmPrinter::emitAttributes() {
       // We consider krait as a "cortex-a9" + hwdiv CPU
       // Enable hwdiv through ".arch_extension idiv"
       if (STI.hasDivide() || STI.hasDivideInARMMode())
-        ATS.emitArchExtension(ARM::HWDIV);
+        ATS.emitArchExtension(ARM::AEK_HWDIV | ARM::AEK_HWDIVARM);
     } else
       ATS.emitTextAttribute(ARMBuildAttrs::CPU_name, CPUString);
   }
@@ -733,13 +615,13 @@ void ARMAsmPrinter::emitAttributes() {
      * neon/neon-fp-armv8/neon-vfpv4/vfpv3/vfpv2 for .fpu parameters */
     if (STI.hasFPARMv8()) {
       if (STI.hasCrypto())
-        ATS.emitFPU(ARM::CRYPTO_NEON_FP_ARMV8);
+        ATS.emitFPU(ARM::FK_CRYPTO_NEON_FP_ARMV8);
       else
-        ATS.emitFPU(ARM::NEON_FP_ARMV8);
+        ATS.emitFPU(ARM::FK_NEON_FP_ARMV8);
     } else if (STI.hasVFP4())
-      ATS.emitFPU(ARM::NEON_VFPV4);
+      ATS.emitFPU(ARM::FK_NEON_VFPV4);
     else
-      ATS.emitFPU(ARM::NEON);
+      ATS.emitFPU(STI.hasFP16() ? ARM::FK_NEON_FP16 : ARM::FK_NEON);
     // Emit Tag_Advanced_SIMD_arch for ARMv8 architecture
     if (STI.hasV8Ops())
       ATS.emitAttribute(ARMBuildAttrs::Advanced_SIMD_arch,
@@ -749,13 +631,23 @@ void ARMAsmPrinter::emitAttributes() {
     if (STI.hasFPARMv8())
       // FPv5 and FP-ARMv8 have the same instructions, so are modeled as one
       // FPU, but there are two different names for it depending on the CPU.
-      ATS.emitFPU(STI.hasD16() ? ARM::FPV5_D16 : ARM::FP_ARMV8);
+      ATS.emitFPU(STI.hasD16()
+                  ? (STI.isFPOnlySP() ? ARM::FK_FPV5_SP_D16 : ARM::FK_FPV5_D16)
+                  : ARM::FK_FP_ARMV8);
     else if (STI.hasVFP4())
-      ATS.emitFPU(STI.hasD16() ? ARM::VFPV4_D16 : ARM::VFPV4);
+      ATS.emitFPU(STI.hasD16()
+                  ? (STI.isFPOnlySP() ? ARM::FK_FPV4_SP_D16 : ARM::FK_VFPV4_D16)
+                  : ARM::FK_VFPV4);
     else if (STI.hasVFP3())
-      ATS.emitFPU(STI.hasD16() ? ARM::VFPV3_D16 : ARM::VFPV3);
+      ATS.emitFPU(STI.hasD16()
+                  // +d16
+                  ? (STI.isFPOnlySP()
+                     ? (STI.hasFP16() ? ARM::FK_VFPV3XD_FP16 : ARM::FK_VFPV3XD)
+                     : (STI.hasFP16() ? ARM::FK_VFPV3_D16_FP16 : ARM::FK_VFPV3_D16))
+                  // -d16
+                  : (STI.hasFP16() ? ARM::FK_VFPV3_FP16 : ARM::FK_VFPV3));
     else if (STI.hasVFP2())
-      ATS.emitFPU(ARM::VFPV2);
+      ATS.emitFPU(ARM::FK_VFPV2);
   }
 
   if (TM.getRelocationModel() == Reloc::PIC_) {
@@ -915,7 +807,7 @@ void ARMAsmPrinter::emitAttributes() {
 static MCSymbol *getPICLabel(const char *Prefix, unsigned FunctionNumber,
                              unsigned LabelId, MCContext &Ctx) {
 
-  MCSymbol *Label = Ctx.GetOrCreateSymbol(Twine(Prefix)
+  MCSymbol *Label = Ctx.getOrCreateSymbol(Twine(Prefix)
                        + "PC" + Twine(FunctionNumber) + "_" + Twine(LabelId));
   return Label;
 }
@@ -965,7 +857,7 @@ MCSymbol *ARMAsmPrinter::GetARMGVSymbol(const GlobalValue *GV,
     Name = "__imp_";
     getNameWithPrefix(Name, GV);
 
-    return OutContext.GetOrCreateSymbol(Name);
+    return OutContext.getOrCreateSymbol(Name);
   } else if (Subtarget->isTargetELF()) {
     return getSymbol(GV);
   }
@@ -974,8 +866,8 @@ MCSymbol *ARMAsmPrinter::GetARMGVSymbol(const GlobalValue *GV,
 
 void ARMAsmPrinter::
 EmitMachineConstantPoolValue(MachineConstantPoolValue *MCPV) {
-  const DataLayout *DL = TM.getDataLayout();
-  int Size = TM.getDataLayout()->getTypeAllocSize(MCPV->getType());
+  const DataLayout &DL = getDataLayout();
+  int Size = DL.getTypeAllocSize(MCPV->getType());
 
   ARMConstantPoolValue *ACPV = static_cast<ARMConstantPoolValue*>(MCPV);
 
@@ -1004,64 +896,46 @@ EmitMachineConstantPoolValue(MachineConstantPoolValue *MCPV) {
 
   // Create an MCSymbol for the reference.
   const MCExpr *Expr =
-    MCSymbolRefExpr::Create(MCSym, getModifierVariantKind(ACPV->getModifier()),
+    MCSymbolRefExpr::create(MCSym, getModifierVariantKind(ACPV->getModifier()),
                             OutContext);
 
   if (ACPV->getPCAdjustment()) {
-    MCSymbol *PCLabel = getPICLabel(DL->getPrivateGlobalPrefix(),
-                                    getFunctionNumber(),
-                                    ACPV->getLabelId(),
-                                    OutContext);
-    const MCExpr *PCRelExpr = MCSymbolRefExpr::Create(PCLabel, OutContext);
+    MCSymbol *PCLabel =
+        getPICLabel(DL.getPrivateGlobalPrefix(), getFunctionNumber(),
+                    ACPV->getLabelId(), OutContext);
+    const MCExpr *PCRelExpr = MCSymbolRefExpr::create(PCLabel, OutContext);
     PCRelExpr =
-      MCBinaryExpr::CreateAdd(PCRelExpr,
-                              MCConstantExpr::Create(ACPV->getPCAdjustment(),
+      MCBinaryExpr::createAdd(PCRelExpr,
+                              MCConstantExpr::create(ACPV->getPCAdjustment(),
                                                      OutContext),
                               OutContext);
     if (ACPV->mustAddCurrentAddress()) {
       // We want "(<expr> - .)", but MC doesn't have a concept of the '.'
       // label, so just emit a local label end reference that instead.
-      MCSymbol *DotSym = OutContext.CreateTempSymbol();
-      OutStreamer.EmitLabel(DotSym);
-      const MCExpr *DotExpr = MCSymbolRefExpr::Create(DotSym, OutContext);
-      PCRelExpr = MCBinaryExpr::CreateSub(PCRelExpr, DotExpr, OutContext);
-    }
-    Expr = MCBinaryExpr::CreateSub(Expr, PCRelExpr, OutContext);
-  } else {   // @LOCALMOD-BEGIN
-    // Check mustAddCurrentAddress() when getPCAdjustment() == 0,
-    // and make it actually *Subtract* the current address.
-    // A more appropriate name is probably "relativeToCurrentAddress",
-    // since the assembler can't actually handle "X + .", only "X - .".
-    if (ACPV->mustAddCurrentAddress()) {
-      MCSymbol *DotSym = OutContext.CreateTempSymbol();
-      OutStreamer.EmitLabel(DotSym);
-      const MCExpr *DotExpr = MCSymbolRefExpr::Create(DotSym, OutContext);
-      Expr = MCBinaryExpr::CreateSub(Expr, DotExpr, OutContext);
+      MCSymbol *DotSym = OutContext.createTempSymbol();
+      OutStreamer->EmitLabel(DotSym);
+      const MCExpr *DotExpr = MCSymbolRefExpr::create(DotSym, OutContext);
+      PCRelExpr = MCBinaryExpr::createSub(PCRelExpr, DotExpr, OutContext);
     }
+    Expr = MCBinaryExpr::createSub(Expr, PCRelExpr, OutContext);
   }
-  // @LOCALMOD-END
-
-  OutStreamer.EmitValue(Expr, Size);
+  OutStreamer->EmitValue(Expr, Size);
 }
 
-void ARMAsmPrinter::EmitJumpTable(const MachineInstr *MI) {
-  unsigned Opcode = MI->getOpcode();
-  int OpNum = 1;
-  if (Opcode == ARM::BR_JTadd)
-    OpNum = 2;
-  else if (Opcode == ARM::BR_JTm)
-    OpNum = 3;
-
-  const MachineOperand &MO1 = MI->getOperand(OpNum);
-  const MachineOperand &MO2 = MI->getOperand(OpNum+1); // Unique Id
+void ARMAsmPrinter::EmitJumpTableAddrs(const MachineInstr *MI) {
+  const MachineOperand &MO1 = MI->getOperand(1);
   unsigned JTI = MO1.getIndex();
 
+  // Make sure the Thumb jump table is 4-byte aligned. This will be a nop for
+  // ARM mode tables.
+  EmitAlignment(2);
+
   // Emit a label for the jump table.
-  MCSymbol *JTISymbol = GetARMJTIPICJumpTableLabel2(JTI, MO2.getImm());
-  OutStreamer.EmitLabel(JTISymbol);
+  MCSymbol *JTISymbol = GetARMJTIPICJumpTableLabel(JTI);
+  OutStreamer->EmitLabel(JTISymbol);
 
   // Mark the jump table as data-in-code.
-  OutStreamer.EmitDataRegion(MCDR_DataRegionJT32);
+  OutStreamer->EmitDataRegion(MCDR_DataRegionJT32);
 
   // Emit each entry of the table.
   const MachineJumpTableInfo *MJTI = MF->getJumpTableInfo();
@@ -1078,89 +952,102 @@ void ARMAsmPrinter::EmitJumpTable(const MachineInstr *MI) {
     // LJTI_0_0:
     //    .word (LBB0 - LJTI_0_0)
     //    .word (LBB1 - LJTI_0_0)
-    const MCExpr *Expr = MCSymbolRefExpr::Create(MBB->getSymbol(), OutContext);
+    const MCExpr *Expr = MCSymbolRefExpr::create(MBB->getSymbol(), OutContext);
 
     if (TM.getRelocationModel() == Reloc::PIC_)
-      Expr = MCBinaryExpr::CreateSub(Expr, MCSymbolRefExpr::Create(JTISymbol,
+      Expr = MCBinaryExpr::createSub(Expr, MCSymbolRefExpr::create(JTISymbol,
                                                                    OutContext),
                                      OutContext);
     // If we're generating a table of Thumb addresses in static relocation
     // model, we need to add one to keep interworking correctly.
     else if (AFI->isThumbFunction())
-      Expr = MCBinaryExpr::CreateAdd(Expr, MCConstantExpr::Create(1,OutContext),
+      Expr = MCBinaryExpr::createAdd(Expr, MCConstantExpr::create(1,OutContext),
                                      OutContext);
-    OutStreamer.EmitValue(Expr, 4);
+    OutStreamer->EmitValue(Expr, 4);
   }
   // Mark the end of jump table data-in-code region.
-  OutStreamer.EmitDataRegion(MCDR_DataRegionEnd);
+  OutStreamer->EmitDataRegion(MCDR_DataRegionEnd);
 }
 
-void ARMAsmPrinter::EmitJump2Table(const MachineInstr *MI) {
-  unsigned Opcode = MI->getOpcode();
-  int OpNum = (Opcode == ARM::t2BR_JT) ? 2 : 1;
-  const MachineOperand &MO1 = MI->getOperand(OpNum);
-  const MachineOperand &MO2 = MI->getOperand(OpNum+1); // Unique Id
+void ARMAsmPrinter::EmitJumpTableInsts(const MachineInstr *MI) {
+  const MachineOperand &MO1 = MI->getOperand(1);
   unsigned JTI = MO1.getIndex();
 
-  MCSymbol *JTISymbol = GetARMJTIPICJumpTableLabel2(JTI, MO2.getImm());
-  OutStreamer.EmitLabel(JTISymbol);
+  MCSymbol *JTISymbol = GetARMJTIPICJumpTableLabel(JTI);
+  OutStreamer->EmitLabel(JTISymbol);
 
   // Emit each entry of the table.
   const MachineJumpTableInfo *MJTI = MF->getJumpTableInfo();
   const std::vector<MachineJumpTableEntry> &JT = MJTI->getJumpTables();
   const std::vector<MachineBasicBlock*> &JTBBs = JT[JTI].MBBs;
-  unsigned OffsetWidth = 4;
-  if (MI->getOpcode() == ARM::t2TBB_JT) {
-    OffsetWidth = 1;
-    // Mark the jump table as data-in-code.
-    OutStreamer.EmitDataRegion(MCDR_DataRegionJT8);
-  } else if (MI->getOpcode() == ARM::t2TBH_JT) {
-    OffsetWidth = 2;
-    // Mark the jump table as data-in-code.
-    OutStreamer.EmitDataRegion(MCDR_DataRegionJT16);
-  }
 
   for (unsigned i = 0, e = JTBBs.size(); i != e; ++i) {
     MachineBasicBlock *MBB = JTBBs[i];
-    const MCExpr *MBBSymbolExpr = MCSymbolRefExpr::Create(MBB->getSymbol(),
+    const MCExpr *MBBSymbolExpr = MCSymbolRefExpr::create(MBB->getSymbol(),
                                                           OutContext);
     // If this isn't a TBB or TBH, the entries are direct branch instructions.
-    if (OffsetWidth == 4) {
-      EmitToStreamer(OutStreamer, MCInstBuilder(ARM::t2B)
+    EmitToStreamer(*OutStreamer, MCInstBuilder(ARM::t2B)
         .addExpr(MBBSymbolExpr)
         .addImm(ARMCC::AL)
         .addReg(0));
-      continue;
-    }
+  }
+}
+
+void ARMAsmPrinter::EmitJumpTableTBInst(const MachineInstr *MI,
+                                        unsigned OffsetWidth) {
+  assert((OffsetWidth == 1 || OffsetWidth == 2) && "invalid tbb/tbh width");
+  const MachineOperand &MO1 = MI->getOperand(1);
+  unsigned JTI = MO1.getIndex();
+
+  MCSymbol *JTISymbol = GetARMJTIPICJumpTableLabel(JTI);
+  OutStreamer->EmitLabel(JTISymbol);
+
+  // Emit each entry of the table.
+  const MachineJumpTableInfo *MJTI = MF->getJumpTableInfo();
+  const std::vector<MachineJumpTableEntry> &JT = MJTI->getJumpTables();
+  const std::vector<MachineBasicBlock*> &JTBBs = JT[JTI].MBBs;
+
+  // Mark the jump table as data-in-code.
+  OutStreamer->EmitDataRegion(OffsetWidth == 1 ? MCDR_DataRegionJT8
+                                               : MCDR_DataRegionJT16);
+
+  for (auto MBB : JTBBs) {
+    const MCExpr *MBBSymbolExpr = MCSymbolRefExpr::create(MBB->getSymbol(),
+                                                          OutContext);
     // Otherwise it's an offset from the dispatch instruction. Construct an
     // MCExpr for the entry. We want a value of the form:
-    // (BasicBlockAddr - TableBeginAddr) / 2
+    // (BasicBlockAddr - TBBInstAddr + 4) / 2
     //
     // For example, a TBB table with entries jumping to basic blocks BB0 and BB1
     // would look like:
     // LJTI_0_0:
-    //    .byte (LBB0 - LJTI_0_0) / 2
-    //    .byte (LBB1 - LJTI_0_0) / 2
-    const MCExpr *Expr =
-      MCBinaryExpr::CreateSub(MBBSymbolExpr,
-                              MCSymbolRefExpr::Create(JTISymbol, OutContext),
-                              OutContext);
-    Expr = MCBinaryExpr::CreateDiv(Expr, MCConstantExpr::Create(2, OutContext),
+    //    .byte (LBB0 - (LCPI0_0 + 4)) / 2
+    //    .byte (LBB1 - (LCPI0_0 + 4)) / 2
+    // where LCPI0_0 is a label defined just before the TBB instruction using
+    // this table.
+    MCSymbol *TBInstPC = GetCPISymbol(MI->getOperand(0).getImm());
+    const MCExpr *Expr = MCBinaryExpr::createAdd(
+        MCSymbolRefExpr::create(TBInstPC, OutContext),
+        MCConstantExpr::create(4, OutContext), OutContext);
+    Expr = MCBinaryExpr::createSub(MBBSymbolExpr, Expr, OutContext);
+    Expr = MCBinaryExpr::createDiv(Expr, MCConstantExpr::create(2, OutContext),
                                    OutContext);
-    OutStreamer.EmitValue(Expr, OffsetWidth);
+    OutStreamer->EmitValue(Expr, OffsetWidth);
   }
   // Mark the end of jump table data-in-code region. 32-bit offsets use
   // actual branch instructions here, so we don't mark those as a data-region
   // at all.
-  if (OffsetWidth != 4)
-    OutStreamer.EmitDataRegion(MCDR_DataRegionEnd);
+  OutStreamer->EmitDataRegion(MCDR_DataRegionEnd);
+
+  // Make sure the next instruction is 2-byte aligned.
+  EmitAlignment(1);
 }
 
 void ARMAsmPrinter::EmitUnwindingInstruction(const MachineInstr *MI) {
   assert(MI->getFlag(MachineInstr::FrameSetup) &&
       "Only instruction which are involved into frame setup code are allowed");
 
-  MCTargetStreamer &TS = *OutStreamer.getTargetStreamer();
+  MCTargetStreamer &TS = *OutStreamer->getTargetStreamer();
   ARMTargetStreamer &ATS = static_cast<ARMTargetStreamer &>(TS);
   const MachineFunction &MF = *MI->getParent()->getParent();
   const TargetRegisterInfo *RegInfo = MF.getSubtarget().getRegisterInfo();
@@ -1301,11 +1188,11 @@ void ARMAsmPrinter::EmitUnwindingInstruction(const MachineInstr *MI) {
 #include "ARMGenMCPseudoLowering.inc"
 
 void ARMAsmPrinter::EmitInstruction(const MachineInstr *MI) {
-  const DataLayout *DL = TM.getDataLayout();
+  const DataLayout &DL = getDataLayout();
 
   // If we just ended a constant pool, mark it as such.
   if (InConstantPool && MI->getOpcode() != ARM::CONSTPOOL_ENTRY) {
-    OutStreamer.EmitDataRegion(MCDR_DataRegionEnd);
+    OutStreamer->EmitDataRegion(MCDR_DataRegionEnd);
     InConstantPool = false;
   }
 
@@ -1315,7 +1202,7 @@ void ARMAsmPrinter::EmitInstruction(const MachineInstr *MI) {
     EmitUnwindingInstruction(MI);
 
   // Do any auto-generated pseudo lowerings.
-  if (emitPseudoExpansionLowering(OutStreamer, MI))
+  if (emitPseudoExpansionLowering(*OutStreamer, MI))
     return;
 
   assert(!convertAddSubFlagsOpcode(MI->getOpcode()) &&
@@ -1331,12 +1218,12 @@ void ARMAsmPrinter::EmitInstruction(const MachineInstr *MI) {
   case ARM::t2LEApcrel: {
     // FIXME: Need to also handle globals and externals
     MCSymbol *CPISymbol = GetCPISymbol(MI->getOperand(1).getIndex());
-    EmitToStreamer(OutStreamer, MCInstBuilder(MI->getOpcode() ==
-                                              ARM::t2LEApcrel ? ARM::t2ADR
+    EmitToStreamer(*OutStreamer, MCInstBuilder(MI->getOpcode() ==
+                                               ARM::t2LEApcrel ? ARM::t2ADR
                   : (MI->getOpcode() == ARM::tLEApcrel ? ARM::tADR
                      : ARM::ADR))
       .addReg(MI->getOperand(0).getReg())
-      .addExpr(MCSymbolRefExpr::Create(CPISymbol, OutContext))
+      .addExpr(MCSymbolRefExpr::create(CPISymbol, OutContext))
       // Add predicate operands.
       .addImm(MI->getOperand(2).getImm())
       .addReg(MI->getOperand(3).getReg()));
@@ -1346,23 +1233,22 @@ void ARMAsmPrinter::EmitInstruction(const MachineInstr *MI) {
   case ARM::tLEApcrelJT:
   case ARM::t2LEApcrelJT: {
     MCSymbol *JTIPICSymbol =
-      GetARMJTIPICJumpTableLabel2(MI->getOperand(1).getIndex(),
-                                  MI->getOperand(2).getImm());
-    EmitToStreamer(OutStreamer, MCInstBuilder(MI->getOpcode() ==
-                                              ARM::t2LEApcrelJT ? ARM::t2ADR
+      GetARMJTIPICJumpTableLabel(MI->getOperand(1).getIndex());
+    EmitToStreamer(*OutStreamer, MCInstBuilder(MI->getOpcode() ==
+                                               ARM::t2LEApcrelJT ? ARM::t2ADR
                   : (MI->getOpcode() == ARM::tLEApcrelJT ? ARM::tADR
                      : ARM::ADR))
       .addReg(MI->getOperand(0).getReg())
-      .addExpr(MCSymbolRefExpr::Create(JTIPICSymbol, OutContext))
+      .addExpr(MCSymbolRefExpr::create(JTIPICSymbol, OutContext))
       // Add predicate operands.
-      .addImm(MI->getOperand(3).getImm())
-      .addReg(MI->getOperand(4).getReg()));
+      .addImm(MI->getOperand(2).getImm())
+      .addReg(MI->getOperand(3).getReg()));
     return;
   }
   // Darwin call instructions are just normal call instructions with different
   // clobber semantics (they clobber R9).
   case ARM::BX_CALL: {
-    EmitToStreamer(OutStreamer, MCInstBuilder(ARM::MOVr)
+    EmitToStreamer(*OutStreamer, MCInstBuilder(ARM::MOVr)
       .addReg(ARM::LR)
       .addReg(ARM::PC)
       // Add predicate operands.
@@ -1371,7 +1257,7 @@ void ARMAsmPrinter::EmitInstruction(const MachineInstr *MI) {
       // Add 's' bit operand (always reg0 for this)
       .addReg(0));
 
-    EmitToStreamer(OutStreamer, MCInstBuilder(ARM::BX)
+    EmitToStreamer(*OutStreamer, MCInstBuilder(ARM::BX)
       .addReg(MI->getOperand(0).getReg()));
     return;
   }
@@ -1395,19 +1281,19 @@ void ARMAsmPrinter::EmitInstruction(const MachineInstr *MI) {
     }
 
     if (!TRegSym) {
-      TRegSym = OutContext.CreateTempSymbol();
+      TRegSym = OutContext.createTempSymbol();
       ThumbIndirectPads.push_back(std::make_pair(TReg, TRegSym));
     }
 
     // Create a link-saving branch to the Reg Indirect Jump Pad.
-    EmitToStreamer(OutStreamer, MCInstBuilder(ARM::tBL)
+    EmitToStreamer(*OutStreamer, MCInstBuilder(ARM::tBL)
         // Predicate comes first here.
         .addImm(ARMCC::AL).addReg(0)
-        .addExpr(MCSymbolRefExpr::Create(TRegSym, OutContext)));
+        .addExpr(MCSymbolRefExpr::create(TRegSym, OutContext)));
     return;
   }
   case ARM::BMOVPCRX_CALL: {
-    EmitToStreamer(OutStreamer, MCInstBuilder(ARM::MOVr)
+    EmitToStreamer(*OutStreamer, MCInstBuilder(ARM::MOVr)
       .addReg(ARM::LR)
       .addReg(ARM::PC)
       // Add predicate operands.
@@ -1416,7 +1302,7 @@ void ARMAsmPrinter::EmitInstruction(const MachineInstr *MI) {
       // Add 's' bit operand (always reg0 for this)
       .addReg(0));
 
-    EmitToStreamer(OutStreamer, MCInstBuilder(ARM::MOVr)
+    EmitToStreamer(*OutStreamer, MCInstBuilder(ARM::MOVr)
       .addReg(ARM::PC)
       .addReg(MI->getOperand(0).getReg())
       // Add predicate operands.
@@ -1427,7 +1313,7 @@ void ARMAsmPrinter::EmitInstruction(const MachineInstr *MI) {
     return;
   }
   case ARM::BMOVPCB_CALL: {
-    EmitToStreamer(OutStreamer, MCInstBuilder(ARM::MOVr)
+    EmitToStreamer(*OutStreamer, MCInstBuilder(ARM::MOVr)
       .addReg(ARM::LR)
       .addReg(ARM::PC)
       // Add predicate operands.
@@ -1440,8 +1326,8 @@ void ARMAsmPrinter::EmitInstruction(const MachineInstr *MI) {
     const GlobalValue *GV = Op.getGlobal();
     const unsigned TF = Op.getTargetFlags();
     MCSymbol *GVSym = GetARMGVSymbol(GV, TF);
-    const MCExpr *GVSymExpr = MCSymbolRefExpr::Create(GVSym, OutContext);
-    EmitToStreamer(OutStreamer, MCInstBuilder(ARM::Bcc)
+    const MCExpr *GVSymExpr = MCSymbolRefExpr::create(GVSym, OutContext);
+    EmitToStreamer(*OutStreamer, MCInstBuilder(ARM::Bcc)
       .addExpr(GVSymExpr)
       // Add predicate operands.
       .addImm(ARMCC::AL)
@@ -1452,31 +1338,31 @@ void ARMAsmPrinter::EmitInstruction(const MachineInstr *MI) {
   case ARM::t2MOVi16_ga_pcrel: {
     MCInst TmpInst;
     TmpInst.setOpcode(Opc == ARM::MOVi16_ga_pcrel? ARM::MOVi16 : ARM::t2MOVi16);
-    TmpInst.addOperand(MCOperand::CreateReg(MI->getOperand(0).getReg()));
+    TmpInst.addOperand(MCOperand::createReg(MI->getOperand(0).getReg()));
 
     unsigned TF = MI->getOperand(1).getTargetFlags();
     const GlobalValue *GV = MI->getOperand(1).getGlobal();
     MCSymbol *GVSym = GetARMGVSymbol(GV, TF);
-    const MCExpr *GVSymExpr = MCSymbolRefExpr::Create(GVSym, OutContext);
+    const MCExpr *GVSymExpr = MCSymbolRefExpr::create(GVSym, OutContext);
 
-    MCSymbol *LabelSym = getPICLabel(DL->getPrivateGlobalPrefix(),
-                                     getFunctionNumber(),
-                                     MI->getOperand(2).getImm(), OutContext);
-    const MCExpr *LabelSymExpr= MCSymbolRefExpr::Create(LabelSym, OutContext);
+    MCSymbol *LabelSym =
+        getPICLabel(DL.getPrivateGlobalPrefix(), getFunctionNumber(),
+                    MI->getOperand(2).getImm(), OutContext);
+    const MCExpr *LabelSymExpr= MCSymbolRefExpr::create(LabelSym, OutContext);
     unsigned PCAdj = (Opc == ARM::MOVi16_ga_pcrel) ? 8 : 4;
     const MCExpr *PCRelExpr =
-      ARMMCExpr::CreateLower16(MCBinaryExpr::CreateSub(GVSymExpr,
-                                      MCBinaryExpr::CreateAdd(LabelSymExpr,
-                                      MCConstantExpr::Create(PCAdj, OutContext),
+      ARMMCExpr::createLower16(MCBinaryExpr::createSub(GVSymExpr,
+                                      MCBinaryExpr::createAdd(LabelSymExpr,
+                                      MCConstantExpr::create(PCAdj, OutContext),
                                       OutContext), OutContext), OutContext);
-      TmpInst.addOperand(MCOperand::CreateExpr(PCRelExpr));
+      TmpInst.addOperand(MCOperand::createExpr(PCRelExpr));
 
     // Add predicate operands.
-    TmpInst.addOperand(MCOperand::CreateImm(ARMCC::AL));
-    TmpInst.addOperand(MCOperand::CreateReg(0));
+    TmpInst.addOperand(MCOperand::createImm(ARMCC::AL));
+    TmpInst.addOperand(MCOperand::createReg(0));
     // Add 's' bit operand (always reg0 for this)
-    TmpInst.addOperand(MCOperand::CreateReg(0));
-    EmitToStreamer(OutStreamer, TmpInst);
+    TmpInst.addOperand(MCOperand::createReg(0));
+    EmitToStreamer(*OutStreamer, TmpInst);
     return;
   }
   case ARM::MOVTi16_ga_pcrel:
@@ -1484,31 +1370,31 @@ void ARMAsmPrinter::EmitInstruction(const MachineInstr *MI) {
     MCInst TmpInst;
     TmpInst.setOpcode(Opc == ARM::MOVTi16_ga_pcrel
                       ? ARM::MOVTi16 : ARM::t2MOVTi16);
-    TmpInst.addOperand(MCOperand::CreateReg(MI->getOperand(0).getReg()));
-    TmpInst.addOperand(MCOperand::CreateReg(MI->getOperand(1).getReg()));
+    TmpInst.addOperand(MCOperand::createReg(MI->getOperand(0).getReg()));
+    TmpInst.addOperand(MCOperand::createReg(MI->getOperand(1).getReg()));
 
     unsigned TF = MI->getOperand(2).getTargetFlags();
     const GlobalValue *GV = MI->getOperand(2).getGlobal();
     MCSymbol *GVSym = GetARMGVSymbol(GV, TF);
-    const MCExpr *GVSymExpr = MCSymbolRefExpr::Create(GVSym, OutContext);
+    const MCExpr *GVSymExpr = MCSymbolRefExpr::create(GVSym, OutContext);
 
-    MCSymbol *LabelSym = getPICLabel(DL->getPrivateGlobalPrefix(),
-                                     getFunctionNumber(),
-                                     MI->getOperand(3).getImm(), OutContext);
-    const MCExpr *LabelSymExpr= MCSymbolRefExpr::Create(LabelSym, OutContext);
+    MCSymbol *LabelSym =
+        getPICLabel(DL.getPrivateGlobalPrefix(), getFunctionNumber(),
+                    MI->getOperand(3).getImm(), OutContext);
+    const MCExpr *LabelSymExpr= MCSymbolRefExpr::create(LabelSym, OutContext);
     unsigned PCAdj = (Opc == ARM::MOVTi16_ga_pcrel) ? 8 : 4;
     const MCExpr *PCRelExpr =
-        ARMMCExpr::CreateUpper16(MCBinaryExpr::CreateSub(GVSymExpr,
-                                   MCBinaryExpr::CreateAdd(LabelSymExpr,
-                                      MCConstantExpr::Create(PCAdj, OutContext),
+        ARMMCExpr::createUpper16(MCBinaryExpr::createSub(GVSymExpr,
+                                   MCBinaryExpr::createAdd(LabelSymExpr,
+                                      MCConstantExpr::create(PCAdj, OutContext),
                                           OutContext), OutContext), OutContext);
-      TmpInst.addOperand(MCOperand::CreateExpr(PCRelExpr));
+      TmpInst.addOperand(MCOperand::createExpr(PCRelExpr));
     // Add predicate operands.
-    TmpInst.addOperand(MCOperand::CreateImm(ARMCC::AL));
-    TmpInst.addOperand(MCOperand::CreateReg(0));
+    TmpInst.addOperand(MCOperand::createImm(ARMCC::AL));
+    TmpInst.addOperand(MCOperand::createReg(0));
     // Add 's' bit operand (always reg0 for this)
-    TmpInst.addOperand(MCOperand::CreateReg(0));
-    EmitToStreamer(OutStreamer, TmpInst);
+    TmpInst.addOperand(MCOperand::createReg(0));
+    EmitToStreamer(*OutStreamer, TmpInst);
     return;
   }
   case ARM::tPICADD: {
@@ -1518,12 +1404,12 @@ void ARMAsmPrinter::EmitInstruction(const MachineInstr *MI) {
     // This adds the address of LPC0 to r0.
 
     // Emit the label.
-    OutStreamer.EmitLabel(getPICLabel(DL->getPrivateGlobalPrefix(),
-                          getFunctionNumber(), MI->getOperand(2).getImm(),
-                          OutContext));
+    OutStreamer->EmitLabel(getPICLabel(DL.getPrivateGlobalPrefix(),
+                                       getFunctionNumber(),
+                                       MI->getOperand(2).getImm(), OutContext));
 
     // Form and emit the add.
-    EmitToStreamer(OutStreamer, MCInstBuilder(ARM::tADDhirr)
+    EmitToStreamer(*OutStreamer, MCInstBuilder(ARM::tADDhirr)
       .addReg(MI->getOperand(0).getReg())
       .addReg(MI->getOperand(0).getReg())
       .addReg(ARM::PC)
@@ -1539,12 +1425,12 @@ void ARMAsmPrinter::EmitInstruction(const MachineInstr *MI) {
     // This adds the address of LPC0 to r0.
 
     // Emit the label.
-    OutStreamer.EmitLabel(getPICLabel(DL->getPrivateGlobalPrefix(),
-                          getFunctionNumber(), MI->getOperand(2).getImm(),
-                          OutContext));
+    OutStreamer->EmitLabel(getPICLabel(DL.getPrivateGlobalPrefix(),
+                                       getFunctionNumber(),
+                                       MI->getOperand(2).getImm(), OutContext));
 
     // Form and emit the add.
-    EmitToStreamer(OutStreamer, MCInstBuilder(ARM::ADDrr)
+    EmitToStreamer(*OutStreamer, MCInstBuilder(ARM::ADDrr)
       .addReg(MI->getOperand(0).getReg())
       .addReg(ARM::PC)
       .addReg(MI->getOperand(1).getReg())
@@ -1570,9 +1456,9 @@ void ARMAsmPrinter::EmitInstruction(const MachineInstr *MI) {
     // a PC-relative address at the ldr instruction.
 
     // Emit the label.
-    OutStreamer.EmitLabel(getPICLabel(DL->getPrivateGlobalPrefix(),
-                          getFunctionNumber(), MI->getOperand(2).getImm(),
-                          OutContext));
+    OutStreamer->EmitLabel(getPICLabel(DL.getPrivateGlobalPrefix(),
+                                       getFunctionNumber(),
+                                       MI->getOperand(2).getImm(), OutContext));
 
     // Form and emit the load
     unsigned Opcode;
@@ -1588,7 +1474,7 @@ void ARMAsmPrinter::EmitInstruction(const MachineInstr *MI) {
     case ARM::PICLDRSB: Opcode = ARM::LDRSB; break;
     case ARM::PICLDRSH: Opcode = ARM::LDRSH; break;
     }
-    EmitToStreamer(OutStreamer, MCInstBuilder(Opcode)
+    EmitToStreamer(*OutStreamer, MCInstBuilder(Opcode)
       .addReg(MI->getOperand(0).getReg())
       .addReg(ARM::PC)
       .addReg(MI->getOperand(1).getReg())
@@ -1610,58 +1496,50 @@ void ARMAsmPrinter::EmitInstruction(const MachineInstr *MI) {
 
     // If this is the first entry of the pool, mark it.
     if (!InConstantPool) {
-      OutStreamer.EmitDataRegion(MCDR_DataRegion);
+      OutStreamer->EmitDataRegion(MCDR_DataRegion);
       InConstantPool = true;
     }
 
-    OutStreamer.EmitLabel(GetCPISymbol(LabelId));
+    OutStreamer->EmitLabel(GetCPISymbol(LabelId));
 
     const MachineConstantPoolEntry &MCPE = MCP->getConstants()[CPIdx];
     if (MCPE.isMachineConstantPoolEntry())
       EmitMachineConstantPoolValue(MCPE.Val.MachineCPVal);
     else
-      EmitGlobalConstant(MCPE.Val.ConstVal);
+      EmitGlobalConstant(DL, MCPE.Val.ConstVal);
     return;
   }
-  case ARM::t2BR_JT: {
-    // Lower and emit the instruction itself, then the jump table following it.
-    EmitToStreamer(OutStreamer, MCInstBuilder(ARM::tMOVr)
-      .addReg(ARM::PC)
-      .addReg(MI->getOperand(0).getReg())
-      // Add predicate operands.
-      .addImm(ARMCC::AL)
-      .addReg(0));
-
-    // Output the data for the jump table itself
-    EmitJump2Table(MI);
+  case ARM::JUMPTABLE_ADDRS:
+    EmitJumpTableAddrs(MI);
     return;
-  }
-  case ARM::t2TBB_JT: {
+  case ARM::JUMPTABLE_INSTS:
+    EmitJumpTableInsts(MI);
+    return;
+  case ARM::JUMPTABLE_TBB:
+  case ARM::JUMPTABLE_TBH:
+    EmitJumpTableTBInst(MI, MI->getOpcode() == ARM::JUMPTABLE_TBB ? 1 : 2);
+    return;
+  case ARM::t2BR_JT: {
     // Lower and emit the instruction itself, then the jump table following it.
-    EmitToStreamer(OutStreamer, MCInstBuilder(ARM::t2TBB)
+    EmitToStreamer(*OutStreamer, MCInstBuilder(ARM::tMOVr)
       .addReg(ARM::PC)
       .addReg(MI->getOperand(0).getReg())
       // Add predicate operands.
       .addImm(ARMCC::AL)
       .addReg(0));
-
-    // Output the data for the jump table itself
-    EmitJump2Table(MI);
-    // Make sure the next instruction is 2-byte aligned.
-    EmitAlignment(1);
     return;
   }
+  case ARM::t2TBB_JT:
   case ARM::t2TBH_JT: {
-    // Lower and emit the instruction itself, then the jump table following it.
-    EmitToStreamer(OutStreamer, MCInstBuilder(ARM::t2TBH)
-      .addReg(ARM::PC)
-      .addReg(MI->getOperand(0).getReg())
-      // Add predicate operands.
-      .addImm(ARMCC::AL)
-      .addReg(0));
-
-    // Output the data for the jump table itself
-    EmitJump2Table(MI);
+    unsigned Opc = MI->getOpcode() == ARM::t2TBB_JT ? ARM::t2TBB : ARM::t2TBH;
+    // Lower and emit the PC label, then the instruction itself.
+    OutStreamer->EmitLabel(GetCPISymbol(MI->getOperand(3).getImm()));
+    EmitToStreamer(*OutStreamer, MCInstBuilder(Opc)
+                                     .addReg(MI->getOperand(0).getReg())
+                                     .addReg(MI->getOperand(1).getReg())
+                                     // Add predicate operands.
+                                     .addImm(ARMCC::AL)
+                                     .addReg(0));
     return;
   }
   case ARM::tBR_JTr:
@@ -1672,22 +1550,15 @@ void ARMAsmPrinter::EmitInstruction(const MachineInstr *MI) {
     unsigned Opc = MI->getOpcode() == ARM::BR_JTr ?
       ARM::MOVr : ARM::tMOVr;
     TmpInst.setOpcode(Opc);
-    TmpInst.addOperand(MCOperand::CreateReg(ARM::PC));
-    TmpInst.addOperand(MCOperand::CreateReg(MI->getOperand(0).getReg()));
+    TmpInst.addOperand(MCOperand::createReg(ARM::PC));
+    TmpInst.addOperand(MCOperand::createReg(MI->getOperand(0).getReg()));
     // Add predicate operands.
-    TmpInst.addOperand(MCOperand::CreateImm(ARMCC::AL));
-    TmpInst.addOperand(MCOperand::CreateReg(0));
+    TmpInst.addOperand(MCOperand::createImm(ARMCC::AL));
+    TmpInst.addOperand(MCOperand::createReg(0));
     // Add 's' bit operand (always reg0 for this)
     if (Opc == ARM::MOVr)
-      TmpInst.addOperand(MCOperand::CreateReg(0));
-    EmitToStreamer(OutStreamer, TmpInst);
-
-    // Make sure the Thumb jump table is 4-byte aligned.
-    if (Opc == ARM::tMOVr)
-      EmitAlignment(2);
-
-    // Output the data for the jump table itself
-    EmitJumpTable(MI);
+      TmpInst.addOperand(MCOperand::createReg(0));
+    EmitToStreamer(*OutStreamer, TmpInst);
     return;
   }
   case ARM::BR_JTm: {
@@ -1697,29 +1568,26 @@ void ARMAsmPrinter::EmitInstruction(const MachineInstr *MI) {
     if (MI->getOperand(1).getReg() == 0) {
       // literal offset
       TmpInst.setOpcode(ARM::LDRi12);
-      TmpInst.addOperand(MCOperand::CreateReg(ARM::PC));
-      TmpInst.addOperand(MCOperand::CreateReg(MI->getOperand(0).getReg()));
-      TmpInst.addOperand(MCOperand::CreateImm(MI->getOperand(2).getImm()));
+      TmpInst.addOperand(MCOperand::createReg(ARM::PC));
+      TmpInst.addOperand(MCOperand::createReg(MI->getOperand(0).getReg()));
+      TmpInst.addOperand(MCOperand::createImm(MI->getOperand(2).getImm()));
     } else {
       TmpInst.setOpcode(ARM::LDRrs);
-      TmpInst.addOperand(MCOperand::CreateReg(ARM::PC));
-      TmpInst.addOperand(MCOperand::CreateReg(MI->getOperand(0).getReg()));
-      TmpInst.addOperand(MCOperand::CreateReg(MI->getOperand(1).getReg()));
-      TmpInst.addOperand(MCOperand::CreateImm(0));
+      TmpInst.addOperand(MCOperand::createReg(ARM::PC));
+      TmpInst.addOperand(MCOperand::createReg(MI->getOperand(0).getReg()));
+      TmpInst.addOperand(MCOperand::createReg(MI->getOperand(1).getReg()));
+      TmpInst.addOperand(MCOperand::createImm(0));
     }
     // Add predicate operands.
-    TmpInst.addOperand(MCOperand::CreateImm(ARMCC::AL));
-    TmpInst.addOperand(MCOperand::CreateReg(0));
-    EmitToStreamer(OutStreamer, TmpInst);
-
-    // Output the data for the jump table itself
-    EmitJumpTable(MI);
+    TmpInst.addOperand(MCOperand::createImm(ARMCC::AL));
+    TmpInst.addOperand(MCOperand::createReg(0));
+    EmitToStreamer(*OutStreamer, TmpInst);
     return;
   }
   case ARM::BR_JTadd: {
     // Lower and emit the instruction itself, then the jump table following it.
     // add pc, target, idx
-    EmitToStreamer(OutStreamer, MCInstBuilder(ARM::ADDrr)
+    EmitToStreamer(*OutStreamer, MCInstBuilder(ARM::ADDrr)
       .addReg(ARM::PC)
       .addReg(MI->getOperand(0).getReg())
       .addReg(MI->getOperand(1).getReg())
@@ -1728,13 +1596,10 @@ void ARMAsmPrinter::EmitInstruction(const MachineInstr *MI) {
       .addReg(0)
       // Add 's' bit operand (always reg0 for this)
       .addReg(0));
-
-    // Output the data for the jump table itself
-    EmitJumpTable(MI);
     return;
   }
   case ARM::SPACE:
-    OutStreamer.EmitZeros(MI->getOperand(1).getImm());
+    OutStreamer->EmitZeros(MI->getOperand(1).getImm());
     return;
   case ARM::TRAP: {
     // Non-Darwin binutils don't yet support the "trap" mnemonic.
@@ -1742,8 +1607,8 @@ void ARMAsmPrinter::EmitInstruction(const MachineInstr *MI) {
     if (!Subtarget->isTargetMachO()) {
       //.long 0xe7ffdefe @ trap
       uint32_t Val = 0xe7ffdefeUL;
-      OutStreamer.AddComment("trap");
-      OutStreamer.EmitIntValue(Val, 4);
+      OutStreamer->AddComment("trap");
+      OutStreamer->EmitIntValue(Val, 4);
       return;
     }
     break;
@@ -1751,8 +1616,8 @@ void ARMAsmPrinter::EmitInstruction(const MachineInstr *MI) {
   case ARM::TRAPNaCl: {
     //.long 0xe7fedef0 @ trap
     uint32_t Val = 0xe7fedef0UL;
-    OutStreamer.AddComment("trap");
-    OutStreamer.EmitIntValue(Val, 4);
+    OutStreamer->AddComment("trap");
+    OutStreamer->EmitIntValue(Val, 4);
     return;
   }
   case ARM::tTRAP: {
@@ -1761,8 +1626,8 @@ void ARMAsmPrinter::EmitInstruction(const MachineInstr *MI) {
     if (!Subtarget->isTargetMachO()) {
       //.short 57086 @ trap
       uint16_t Val = 0xdefe;
-      OutStreamer.AddComment("trap");
-      OutStreamer.EmitIntValue(Val, 2);
+      OutStreamer->AddComment("trap");
+      OutStreamer->EmitIntValue(Val, 2);
       return;
     }
     break;
@@ -1775,21 +1640,21 @@ void ARMAsmPrinter::EmitInstruction(const MachineInstr *MI) {
     // adds $val, #7
     // str $val, [$src, #4]
     // movs r0, #0
-    // b 1f
+    // b LSJLJEH
     // movs r0, #1
-    // 1:
+    // LSJLJEH:
     unsigned SrcReg = MI->getOperand(0).getReg();
     unsigned ValReg = MI->getOperand(1).getReg();
-    MCSymbol *Label = GetARMSJLJEHLabel();
-    OutStreamer.AddComment("eh_setjmp begin");
-    EmitToStreamer(OutStreamer, MCInstBuilder(ARM::tMOVr)
+    MCSymbol *Label = OutContext.createTempSymbol("SJLJEH", false, true);
+    OutStreamer->AddComment("eh_setjmp begin");
+    EmitToStreamer(*OutStreamer, MCInstBuilder(ARM::tMOVr)
       .addReg(ValReg)
       .addReg(ARM::PC)
       // Predicate.
       .addImm(ARMCC::AL)
       .addReg(0));
 
-    EmitToStreamer(OutStreamer, MCInstBuilder(ARM::tADDi3)
+    EmitToStreamer(*OutStreamer, MCInstBuilder(ARM::tADDi3)
       .addReg(ValReg)
       // 's' bit operand
       .addReg(ARM::CPSR)
@@ -1799,7 +1664,7 @@ void ARMAsmPrinter::EmitInstruction(const MachineInstr *MI) {
       .addImm(ARMCC::AL)
       .addReg(0));
 
-    EmitToStreamer(OutStreamer, MCInstBuilder(ARM::tSTRi)
+    EmitToStreamer(*OutStreamer, MCInstBuilder(ARM::tSTRi)
       .addReg(ValReg)
       .addReg(SrcReg)
       // The offset immediate is #4. The operand value is scaled by 4 for the
@@ -1809,7 +1674,7 @@ void ARMAsmPrinter::EmitInstruction(const MachineInstr *MI) {
       .addImm(ARMCC::AL)
       .addReg(0));
 
-    EmitToStreamer(OutStreamer, MCInstBuilder(ARM::tMOVi8)
+    EmitToStreamer(*OutStreamer, MCInstBuilder(ARM::tMOVi8)
       .addReg(ARM::R0)
       .addReg(ARM::CPSR)
       .addImm(0)
@@ -1817,14 +1682,14 @@ void ARMAsmPrinter::EmitInstruction(const MachineInstr *MI) {
       .addImm(ARMCC::AL)
       .addReg(0));
 
-    const MCExpr *SymbolExpr = MCSymbolRefExpr::Create(Label, OutContext);
-    EmitToStreamer(OutStreamer, MCInstBuilder(ARM::tB)
+    const MCExpr *SymbolExpr = MCSymbolRefExpr::create(Label, OutContext);
+    EmitToStreamer(*OutStreamer, MCInstBuilder(ARM::tB)
       .addExpr(SymbolExpr)
       .addImm(ARMCC::AL)
       .addReg(0));
 
-    OutStreamer.AddComment("eh_setjmp end");
-    EmitToStreamer(OutStreamer, MCInstBuilder(ARM::tMOVi8)
+    OutStreamer->AddComment("eh_setjmp end");
+    EmitToStreamer(*OutStreamer, MCInstBuilder(ARM::tMOVi8)
       .addReg(ARM::R0)
       .addReg(ARM::CPSR)
       .addImm(1)
@@ -1832,7 +1697,7 @@ void ARMAsmPrinter::EmitInstruction(const MachineInstr *MI) {
       .addImm(ARMCC::AL)
       .addReg(0));
 
-    OutStreamer.EmitLabel(Label);
+    OutStreamer->EmitLabel(Label);
     return;
   }
 
@@ -1847,8 +1712,8 @@ void ARMAsmPrinter::EmitInstruction(const MachineInstr *MI) {
     unsigned SrcReg = MI->getOperand(0).getReg();
     unsigned ValReg = MI->getOperand(1).getReg();
 
-    OutStreamer.AddComment("eh_setjmp begin");
-    EmitToStreamer(OutStreamer, MCInstBuilder(ARM::ADDri)
+    OutStreamer->AddComment("eh_setjmp begin");
+    EmitToStreamer(*OutStreamer, MCInstBuilder(ARM::ADDri)
       .addReg(ValReg)
       .addReg(ARM::PC)
       .addImm(8)
@@ -1858,7 +1723,7 @@ void ARMAsmPrinter::EmitInstruction(const MachineInstr *MI) {
       // 's' bit operand (always reg0 for this).
       .addReg(0));
 
-    EmitToStreamer(OutStreamer, MCInstBuilder(ARM::STRi12)
+    EmitToStreamer(*OutStreamer, MCInstBuilder(ARM::STRi12)
       .addReg(ValReg)
       .addReg(SrcReg)
       .addImm(4)
@@ -1866,7 +1731,7 @@ void ARMAsmPrinter::EmitInstruction(const MachineInstr *MI) {
       .addImm(ARMCC::AL)
       .addReg(0));
 
-    EmitToStreamer(OutStreamer, MCInstBuilder(ARM::MOVi)
+    EmitToStreamer(*OutStreamer, MCInstBuilder(ARM::MOVi)
       .addReg(ARM::R0)
       .addImm(0)
       // Predicate.
@@ -1875,7 +1740,7 @@ void ARMAsmPrinter::EmitInstruction(const MachineInstr *MI) {
       // 's' bit operand (always reg0 for this).
       .addReg(0));
 
-    EmitToStreamer(OutStreamer, MCInstBuilder(ARM::ADDri)
+    EmitToStreamer(*OutStreamer, MCInstBuilder(ARM::ADDri)
       .addReg(ARM::PC)
       .addReg(ARM::PC)
       .addImm(0)
@@ -1885,8 +1750,8 @@ void ARMAsmPrinter::EmitInstruction(const MachineInstr *MI) {
       // 's' bit operand (always reg0 for this).
       .addReg(0));
 
-    OutStreamer.AddComment("eh_setjmp end");
-    EmitToStreamer(OutStreamer, MCInstBuilder(ARM::MOVi)
+    OutStreamer->AddComment("eh_setjmp end");
+    EmitToStreamer(*OutStreamer, MCInstBuilder(ARM::MOVi)
       .addReg(ARM::R0)
       .addImm(1)
       // Predicate.
@@ -1903,7 +1768,7 @@ void ARMAsmPrinter::EmitInstruction(const MachineInstr *MI) {
     // bx $scratch
     unsigned SrcReg = MI->getOperand(0).getReg();
     unsigned ScratchReg = MI->getOperand(1).getReg();
-    EmitToStreamer(OutStreamer, MCInstBuilder(ARM::LDRi12)
+    EmitToStreamer(*OutStreamer, MCInstBuilder(ARM::LDRi12)
       .addReg(ARM::SP)
       .addReg(SrcReg)
       .addImm(8)
@@ -1911,7 +1776,7 @@ void ARMAsmPrinter::EmitInstruction(const MachineInstr *MI) {
       .addImm(ARMCC::AL)
       .addReg(0));
 
-    EmitToStreamer(OutStreamer, MCInstBuilder(ARM::LDRi12)
+    EmitToStreamer(*OutStreamer, MCInstBuilder(ARM::LDRi12)
       .addReg(ScratchReg)
       .addReg(SrcReg)
       .addImm(4)
@@ -1919,7 +1784,7 @@ void ARMAsmPrinter::EmitInstruction(const MachineInstr *MI) {
       .addImm(ARMCC::AL)
       .addReg(0));
 
-    EmitToStreamer(OutStreamer, MCInstBuilder(ARM::LDRi12)
+    EmitToStreamer(*OutStreamer, MCInstBuilder(ARM::LDRi12)
       .addReg(ARM::R7)
       .addReg(SrcReg)
       .addImm(0)
@@ -1927,7 +1792,7 @@ void ARMAsmPrinter::EmitInstruction(const MachineInstr *MI) {
       .addImm(ARMCC::AL)
       .addReg(0));
 
-    EmitToStreamer(OutStreamer, MCInstBuilder(ARM::BX)
+    EmitToStreamer(*OutStreamer, MCInstBuilder(ARM::BX)
       .addReg(ScratchReg)
       // Predicate.
       .addImm(ARMCC::AL)
@@ -1942,7 +1807,7 @@ void ARMAsmPrinter::EmitInstruction(const MachineInstr *MI) {
     // bx $scratch
     unsigned SrcReg = MI->getOperand(0).getReg();
     unsigned ScratchReg = MI->getOperand(1).getReg();
-    EmitToStreamer(OutStreamer, MCInstBuilder(ARM::tLDRi)
+    EmitToStreamer(*OutStreamer, MCInstBuilder(ARM::tLDRi)
       .addReg(ScratchReg)
       .addReg(SrcReg)
       // The offset immediate is #8. The operand value is scaled by 4 for the
@@ -1952,14 +1817,14 @@ void ARMAsmPrinter::EmitInstruction(const MachineInstr *MI) {
       .addImm(ARMCC::AL)
       .addReg(0));
 
-    EmitToStreamer(OutStreamer, MCInstBuilder(ARM::tMOVr)
+    EmitToStreamer(*OutStreamer, MCInstBuilder(ARM::tMOVr)
       .addReg(ARM::SP)
       .addReg(ScratchReg)
       // Predicate.
       .addImm(ARMCC::AL)
       .addReg(0));
 
-    EmitToStreamer(OutStreamer, MCInstBuilder(ARM::tLDRi)
+    EmitToStreamer(*OutStreamer, MCInstBuilder(ARM::tLDRi)
       .addReg(ScratchReg)
       .addReg(SrcReg)
       .addImm(1)
@@ -1967,7 +1832,7 @@ void ARMAsmPrinter::EmitInstruction(const MachineInstr *MI) {
       .addImm(ARMCC::AL)
       .addReg(0));
 
-    EmitToStreamer(OutStreamer, MCInstBuilder(ARM::tLDRi)
+    EmitToStreamer(*OutStreamer, MCInstBuilder(ARM::tLDRi)
       .addReg(ARM::R7)
       .addReg(SrcReg)
       .addImm(0)
@@ -1975,63 +1840,19 @@ void ARMAsmPrinter::EmitInstruction(const MachineInstr *MI) {
       .addImm(ARMCC::AL)
       .addReg(0));
 
-    EmitToStreamer(OutStreamer, MCInstBuilder(ARM::tBX)
+    EmitToStreamer(*OutStreamer, MCInstBuilder(ARM::tBX)
       .addReg(ScratchReg)
       // Predicate.
       .addImm(ARMCC::AL)
       .addReg(0));
     return;
   }
-
-  // @LOCALMOD-BEGIN
-  // These are pseudo ops for MOVW / MOVT with operands relative to a PC label.
-  // See the comments on MOVi16PIC in the .td file for more details.
-  case ARM::MOVi16PIC: {
-    MCInst TmpInst;
-    // First, build an instruction w/ the real opcode.
-    TmpInst.setOpcode(ARM::MOVi16);
-
-    unsigned ImmIndex = 1;
-    unsigned PIC_id_index = 2;
-    unsigned PCAdjustment = 8;
-    // NOTE: if getPICLabel was a method of "this", or otherwise in scope for
-    // LowerARMMachineInstrToMCInstPCRel, then we wouldn't need to create
-    // it here (as well as below).
-    MCSymbol *PCLabel = getPICLabel(MAI->getPrivateGlobalPrefix(),
-                                    getFunctionNumber(),
-                                    MI->getOperand(PIC_id_index).getImm(),
-                                    OutContext);
-    LowerARMMachineInstrToMCInstPCRel(MI, TmpInst, *this, ImmIndex,
-                                      PIC_id_index, PCLabel, PCAdjustment);
-    EmitToStreamer(OutStreamer, TmpInst);
-    return;
-  }
-  case ARM::MOVTi16PIC: {
-    MCInst TmpInst;
-    // First, build an instruction w/ the real opcode.
-    TmpInst.setOpcode(ARM::MOVTi16);
-
-    unsigned ImmIndex = 2;
-    unsigned PIC_id_index = 3;
-    unsigned PCAdjustment = 8;
-
-    MCSymbol *PCLabel = getPICLabel(MAI->getPrivateGlobalPrefix(),
-                                    getFunctionNumber(),
-                                    MI->getOperand(PIC_id_index).getImm(),
-                                    OutContext);
-
-    LowerARMMachineInstrToMCInstPCRel(MI, TmpInst, *this, ImmIndex,
-                                      PIC_id_index, PCLabel, PCAdjustment);
-    EmitToStreamer(OutStreamer, TmpInst);
-    return;
-  }
-  //@LOCALMOD-END
   }
 
   MCInst TmpInst;
   LowerARMMachineInstrToMCInst(MI, TmpInst, *this);
 
-  EmitToStreamer(OutStreamer, TmpInst);
+  EmitToStreamer(*OutStreamer, TmpInst);
 }
 
 //===----------------------------------------------------------------------===//
diff --git a/lib/Target/ARM/ARMAsmPrinter.h b/lib/Target/ARM/ARMAsmPrinter.h
index a18408cb272..fb925f162f7 100644
--- a/lib/Target/ARM/ARMAsmPrinter.h
+++ b/lib/Target/ARM/ARMAsmPrinter.h
@@ -59,8 +59,7 @@ class LLVM_LIBRARY_VISIBILITY ARMAsmPrinter : public AsmPrinter {
     return "ARM Assembly / Object Emitter";
   }
 
-  void printOperand(const MachineInstr *MI, int OpNum, raw_ostream &O,
-                    const char *Modifier = nullptr);
+  void printOperand(const MachineInstr *MI, int OpNum, raw_ostream &O);
 
   bool PrintAsmOperand(const MachineInstr *MI, unsigned OpNum,
                        unsigned AsmVariant, const char *ExtraCode,
@@ -72,39 +71,23 @@ class LLVM_LIBRARY_VISIBILITY ARMAsmPrinter : public AsmPrinter {
   void emitInlineAsmEnd(const MCSubtargetInfo &StartInfo,
                         const MCSubtargetInfo *EndInfo) const override;
 
-  void EmitJumpTable(const MachineInstr *MI);
-  void EmitJump2Table(const MachineInstr *MI);
+  void EmitJumpTableAddrs(const MachineInstr *MI);
+  void EmitJumpTableInsts(const MachineInstr *MI);
+  void EmitJumpTableTBInst(const MachineInstr *MI, unsigned OffsetWidth);
   void EmitInstruction(const MachineInstr *MI) override;
   bool runOnMachineFunction(MachineFunction &F) override;
 
-  // @LOCALMOD-START
-  // Usually this does nothing on ARM as constants pools are handled with
-  // custom code (for constant islands).
-  // When not using constant islands, fall back to the default implementation.
   void EmitConstantPool() override {
-    if (!Subtarget->useConstIslands()) AsmPrinter::EmitConstantPool();
+    // we emit constant pools customly!
   }
-  // @LOCALMOD-END
-
   void EmitFunctionBodyEnd() override;
   void EmitFunctionEntryLabel() override;
   void EmitStartOfAsmFile(Module &M) override;
   void EmitEndOfAsmFile(Module &M) override;
-  void EmitXXStructor(const Constant *CV) override;
+  void EmitXXStructor(const DataLayout &DL, const Constant *CV) override;
 
   // lowerOperand - Convert a MachineOperand into the equivalent MCOperand.
   bool lowerOperand(const MachineOperand &MO, MCOperand &MCOp);
-  
-  // @LOCALMOD-START
-  /// UseReadOnlyJumpTables - true if JumpTableInfo must be in rodata.
-  bool UseReadOnlyJumpTables() const override;
-  /// GetTargetBasicBlockAlign - Get the target alignment for basic blocks.
-  unsigned GetTargetBasicBlockAlign() const override;
-  /// GetTargetLabelAlign - Get optional alignment for TargetOpcode
-  /// labels E.g., EH_LABEL.
-  /// TODO(sehr,robertm): remove this if the labeled block has address taken.
-  unsigned GetTargetLabelAlign(const MachineInstr *MI) const override;
-  // @LOCALMOD-END
 
 private:
   // Helpers for EmitStartOfAsmFile() and EmitEndOfAsmFile()
@@ -122,7 +105,7 @@ class LLVM_LIBRARY_VISIBILITY ARMAsmPrinter : public AsmPrinter {
 public:
   unsigned getISAEncoding() override {
     // ARM/Darwin adds ISA to the DWARF info for each function.
-    Triple TT(TM.getTargetTriple());
+    const Triple &TT = TM.getTargetTriple();
     if (!TT.isOSBinFormatMachO())
       return 0;
     bool isThumb = TT.getArch() == Triple::thumb ||
@@ -134,9 +117,7 @@ class LLVM_LIBRARY_VISIBILITY ARMAsmPrinter : public AsmPrinter {
 
 private:
   MCOperand GetSymbolRef(const MachineOperand &MO, const MCSymbol *Symbol);
-  MCSymbol *GetARMJTIPICJumpTableLabel2(unsigned uid, unsigned uid2) const;
-
-  MCSymbol *GetARMSJLJEHLabel() const;
+  MCSymbol *GetARMJTIPICJumpTableLabel(unsigned uid) const;
 
   MCSymbol *GetARMGVSymbol(const GlobalValue *GV, unsigned char TargetFlags);
 
diff --git a/lib/Target/ARM/ARMBaseInstrInfo.cpp b/lib/Target/ARM/ARMBaseInstrInfo.cpp
index 7ee3cb0a3d1..9f43e732bd7 100644
--- a/lib/Target/ARM/ARMBaseInstrInfo.cpp
+++ b/lib/Target/ARM/ARMBaseInstrInfo.cpp
@@ -28,6 +28,7 @@
 #include "llvm/CodeGen/MachineMemOperand.h"
 #include "llvm/CodeGen/MachineRegisterInfo.h"
 #include "llvm/CodeGen/SelectionDAGNodes.h"
+#include "llvm/CodeGen/TargetSchedule.h"
 #include "llvm/IR/Constants.h"
 #include "llvm/IR/Function.h"
 #include "llvm/IR/GlobalValue.h"
@@ -366,14 +367,10 @@ ARMBaseInstrInfo::AnalyzeBranch(MachineBasicBlock &MBB,MachineBasicBlock *&TBB,
 
 
 unsigned ARMBaseInstrInfo::RemoveBranch(MachineBasicBlock &MBB) const {
-  MachineBasicBlock::iterator I = MBB.end();
-  if (I == MBB.begin()) return 0;
-  --I;
-  while (I->isDebugValue()) {
-    if (I == MBB.begin())
-      return 0;
-    --I;
-  }
+  MachineBasicBlock::iterator I = MBB.getLastNonDebugInstr();
+  if (I == MBB.end())
+    return 0;
+
   if (!isUncondBranchOpcode(I->getOpcode()) &&
       !isCondBranchOpcode(I->getOpcode()))
     return 0;
@@ -396,7 +393,7 @@ unsigned ARMBaseInstrInfo::RemoveBranch(MachineBasicBlock &MBB) const {
 unsigned
 ARMBaseInstrInfo::InsertBranch(MachineBasicBlock &MBB, MachineBasicBlock *TBB,
                                MachineBasicBlock *FBB,
-                               const SmallVectorImpl<MachineOperand> &Cond,
+                               ArrayRef<MachineOperand> Cond,
                                DebugLoc DL) const {
   ARMFunctionInfo *AFI = MBB.getParent()->getInfo<ARMFunctionInfo>();
   int BOpc   = !AFI->isThumbFunction()
@@ -410,6 +407,8 @@ ARMBaseInstrInfo::InsertBranch(MachineBasicBlock &MBB, MachineBasicBlock *TBB,
   assert((Cond.size() == 2 || Cond.size() == 0) &&
          "ARM branch conditions have two components!");
 
+  // For conditional branches, we use addOperand to preserve CPSR flags.
+
   if (!FBB) {
     if (Cond.empty()) { // Unconditional branch?
       if (isThumb)
@@ -418,13 +417,13 @@ ARMBaseInstrInfo::InsertBranch(MachineBasicBlock &MBB, MachineBasicBlock *TBB,
         BuildMI(&MBB, DL, get(BOpc)).addMBB(TBB);
     } else
       BuildMI(&MBB, DL, get(BccOpc)).addMBB(TBB)
-        .addImm(Cond[0].getImm()).addReg(Cond[1].getReg());
+        .addImm(Cond[0].getImm()).addOperand(Cond[1]);
     return 1;
   }
 
   // Two-way conditional branch.
   BuildMI(&MBB, DL, get(BccOpc)).addMBB(TBB)
-    .addImm(Cond[0].getImm()).addReg(Cond[1].getReg());
+    .addImm(Cond[0].getImm()).addOperand(Cond[1]);
   if (isThumb)
     BuildMI(&MBB, DL, get(BOpc)).addMBB(FBB).addImm(ARMCC::AL).addReg(0);
   else
@@ -456,8 +455,7 @@ bool ARMBaseInstrInfo::isPredicated(const MachineInstr *MI) const {
 }
 
 bool ARMBaseInstrInfo::
-PredicateInstruction(MachineInstr *MI,
-                     const SmallVectorImpl<MachineOperand> &Pred) const {
+PredicateInstruction(MachineInstr *MI, ArrayRef<MachineOperand> Pred) const {
   unsigned Opc = MI->getOpcode();
   if (isUncondBranchOpcode(Opc)) {
     MI->setDesc(get(getMatchingCondBranchOpcode(Opc)));
@@ -477,9 +475,8 @@ PredicateInstruction(MachineInstr *MI,
   return false;
 }
 
-bool ARMBaseInstrInfo::
-SubsumesPredicate(const SmallVectorImpl<MachineOperand> &Pred1,
-                  const SmallVectorImpl<MachineOperand> &Pred2) const {
+bool ARMBaseInstrInfo::SubsumesPredicate(ArrayRef<MachineOperand> Pred1,
+                                         ArrayRef<MachineOperand> Pred2) const {
   if (Pred1.size() > 2 || Pred2.size() > 2)
     return false;
 
@@ -625,6 +622,10 @@ unsigned ARMBaseInstrInfo::GetInstSizeInBytes(const MachineInstr *MI) const {
   case ARM::t2MOVi32imm:
     return 8;
   case ARM::CONSTPOOL_ENTRY:
+  case ARM::JUMPTABLE_INSTS:
+  case ARM::JUMPTABLE_ADDRS:
+  case ARM::JUMPTABLE_TBB:
+  case ARM::JUMPTABLE_TBH:
     // If this machine instr is a constant pool entry, its size is recorded as
     // operand #2.
     return MI->getOperand(2).getImm();
@@ -639,42 +640,6 @@ unsigned ARMBaseInstrInfo::GetInstSizeInBytes(const MachineInstr *MI) const {
   case ARM::t2Int_eh_sjlj_setjmp:
   case ARM::t2Int_eh_sjlj_setjmp_nofp:
     return 12;
-  case ARM::BR_JTr:
-  case ARM::BR_JTm:
-  case ARM::BR_JTadd:
-  case ARM::tBR_JTr:
-  case ARM::t2BR_JT:
-  case ARM::t2TBB_JT:
-  case ARM::t2TBH_JT: {
-    // These are jumptable branches, i.e. a branch followed by an inlined
-    // jumptable. The size is 4 + 4 * number of entries. For TBB, each
-    // entry is one byte; TBH two byte each.
-    unsigned EntrySize = (Opc == ARM::t2TBB_JT)
-      ? 1 : ((Opc == ARM::t2TBH_JT) ? 2 : 4);
-    unsigned NumOps = MCID.getNumOperands();
-    MachineOperand JTOP =
-      MI->getOperand(NumOps - (MI->isPredicable() ? 3 : 2));
-    unsigned JTI = JTOP.getIndex();
-    const MachineJumpTableInfo *MJTI = MF->getJumpTableInfo();
-    assert(MJTI != nullptr);
-    const std::vector<MachineJumpTableEntry> &JT = MJTI->getJumpTables();
-    assert(JTI < JT.size());
-    // Thumb instructions are 2 byte aligned, but JT entries are 4 byte
-    // 4 aligned. The assembler / linker may add 2 byte padding just before
-    // the JT entries.  The size does not include this padding; the
-    // constant islands pass does separate bookkeeping for it.
-    // FIXME: If we know the size of the function is less than (1 << 16) *2
-    // bytes, we can use 16-bit entries instead. Then there won't be an
-    // alignment issue.
-    unsigned InstSize = (Opc == ARM::tBR_JTr || Opc == ARM::t2BR_JT) ? 2 : 4;
-    unsigned NumEntries = JT[JTI].MBBs.size();
-    if (Opc == ARM::t2TBB_JT && (NumEntries & 1))
-      // Make sure the instruction that follows TBB is 2-byte aligned.
-      // FIXME: Constant island pass should insert an "ALIGN" instruction
-      // instead.
-      ++NumEntries;
-    return NumEntries * EntrySize + InstSize;
-  }
   case ARM::SPACE:
     return MI->getOperand(1).getImm();
   }
@@ -1265,8 +1230,7 @@ ARMBaseInstrInfo::expandPostRAPseudo(MachineBasicBlock::iterator MI) const {
   Reloc::Model RM = MF.getTarget().getRelocationModel();
 
   if (MI->getOpcode() == TargetOpcode::LOAD_STACK_GUARD) {
-    assert(getSubtarget().getTargetTriple().getObjectFormat() ==
-           Triple::MachO &&
+    assert(getSubtarget().getTargetTriple().isOSBinFormatMachO() &&
            "LOAD_STACK_GUARD currently supported only for MachO.");
     expandLoadStackGuard(MI, RM);
     MI->getParent()->erase(MI);
@@ -1432,7 +1396,7 @@ ARMBaseInstrInfo::duplicate(MachineInstr *Orig, MachineFunction &MF) const {
 bool ARMBaseInstrInfo::produceSameValue(const MachineInstr *MI0,
                                         const MachineInstr *MI1,
                                         const MachineRegisterInfo *MRI) const {
-  int Opcode = MI0->getOpcode();
+  unsigned Opcode = MI0->getOpcode();
   if (Opcode == ARM::t2LDRpci ||
       Opcode == ARM::t2LDRpci_pic ||
       Opcode == ARM::tLDRpci ||
@@ -1685,6 +1649,33 @@ isProfitableToIfCvt(MachineBasicBlock &MBB,
   if (!NumCycles)
     return false;
 
+  // If we are optimizing for size, see if the branch in the predecessor can be
+  // lowered to cbn?z by the constant island lowering pass, and return false if
+  // so. This results in a shorter instruction sequence.
+  const Function *F = MBB.getParent()->getFunction();
+  if (F->hasFnAttribute(Attribute::OptimizeForSize) ||
+      F->hasFnAttribute(Attribute::MinSize)) {
+    MachineBasicBlock *Pred = *MBB.pred_begin();
+    if (!Pred->empty()) {
+      MachineInstr *LastMI = &*Pred->rbegin();
+      if (LastMI->getOpcode() == ARM::t2Bcc) {
+        MachineBasicBlock::iterator CmpMI = LastMI;
+        if (CmpMI != Pred->begin()) {
+          --CmpMI;
+          if (CmpMI->getOpcode() == ARM::tCMPi8 ||
+              CmpMI->getOpcode() == ARM::t2CMPri) {
+            unsigned Reg = CmpMI->getOperand(0).getReg();
+            unsigned PredReg = 0;
+            ARMCC::CondCodes P = getInstrPredicate(CmpMI, PredReg);
+            if (P == ARMCC::AL && CmpMI->getOperand(1).getImm() == 0 &&
+                isARMLowRegister(Reg))
+              return false;
+          }
+        }
+      }
+    }
+  }
+
   // Attempt to estimate the relative costs of predication versus branching.
   unsigned UnpredCost = Probability.getNumerator() * NumCycles;
   UnpredCost /= Probability.getDenominator();
@@ -1742,7 +1733,7 @@ llvm::getInstrPredicate(const MachineInstr *MI, unsigned &PredReg) {
 }
 
 
-int llvm::getMatchingCondBranchOpcode(int Opc) {
+unsigned llvm::getMatchingCondBranchOpcode(unsigned Opc) {
   if (Opc == ARM::B)
     return ARM::Bcc;
   if (Opc == ARM::tB)
@@ -1810,8 +1801,7 @@ static MachineInstr *canFoldIntoMOVCC(unsigned Reg,
       return nullptr;
   }
   bool DontMoveAcrossStores = true;
-  if (!MI->isSafeToMove(TII, /* AliasAnalysis = */ nullptr,
-                        DontMoveAcrossStores))
+  if (!MI->isSafeToMove(/* AliasAnalysis = */ nullptr, DontMoveAcrossStores))
     return nullptr;
   return MI;
 }
@@ -1892,6 +1882,13 @@ ARMBaseInstrInfo::optimizeSelect(MachineInstr *MI,
   SeenMIs.insert(NewMI);
   SeenMIs.erase(DefMI);
 
+  // If MI is inside a loop, and DefMI is outside the loop, then kill flags on
+  // DefMI would be invalid when tranferred inside the loop.  Checking for a
+  // loop is expensive, but at least remove kill flags if they are in different
+  // BBs.
+  if (DefMI->getParent() != MI->getParent())
+    NewMI->clearKillInfo();
+
   // The caller will erase MI, but not DefMI.
   DefMI->eraseFromParent();
   return NewMI;
@@ -2286,16 +2283,6 @@ static bool isSuitableForMask(MachineInstr *&MI, unsigned SrcReg,
       if (SrcReg == MI->getOperand(CommonUse ? 1 : 0).getReg())
         return true;
       break;
-    case ARM::COPY: {
-      // Walk down one instruction which is potentially an 'and'.
-      const MachineInstr &Copy = *MI;
-      MachineBasicBlock::iterator AND(
-        std::next(MachineBasicBlock::iterator(MI)));
-      if (AND == MI->getParent()->end()) return false;
-      MI = AND;
-      return isSuitableForMask(MI, Copy.getOperand(0).getReg(),
-                               CmpMask, true);
-    }
   }
 
   return false;
@@ -4002,7 +3989,7 @@ int ARMBaseInstrInfo::getInstrLatency(const InstrItineraryData *ItinData,
 }
 
 bool ARMBaseInstrInfo::
-hasHighOperandLatency(const InstrItineraryData *ItinData,
+hasHighOperandLatency(const TargetSchedModel &SchedModel,
                       const MachineRegisterInfo *MRI,
                       const MachineInstr *DefMI, unsigned DefIdx,
                       const MachineInstr *UseMI, unsigned UseIdx) const {
@@ -4014,9 +4001,8 @@ hasHighOperandLatency(const InstrItineraryData *ItinData,
     return true;
 
   // Hoist VFP / NEON instructions with 4 or higher latency.
-  int Latency = computeOperandLatency(ItinData, DefMI, DefIdx, UseMI, UseIdx);
-  if (Latency < 0)
-    Latency = getInstrLatency(ItinData, DefMI);
+  unsigned Latency
+    = SchedModel.computeOperandLatency(DefMI, DefIdx, UseMI, UseIdx);
   if (Latency <= 3)
     return false;
   return DDomain == ARMII::DomainVFP || DDomain == ARMII::DomainNEON ||
@@ -4024,8 +4010,9 @@ hasHighOperandLatency(const InstrItineraryData *ItinData,
 }
 
 bool ARMBaseInstrInfo::
-hasLowDefLatency(const InstrItineraryData *ItinData,
+hasLowDefLatency(const TargetSchedModel &SchedModel,
                  const MachineInstr *DefMI, unsigned DefIdx) const {
+  const InstrItineraryData *ItinData = SchedModel.getInstrItineraries();
   if (!ItinData || ItinData->isEmpty())
     return false;
 
@@ -4513,7 +4500,7 @@ breakPartialRegDependency(MachineBasicBlock::iterator MI,
 }
 
 bool ARMBaseInstrInfo::hasNOP() const {
-  return (Subtarget.getFeatureBits() & ARM::HasV6KOps) != 0;
+  return Subtarget.getFeatureBits()[ARM::HasV6KOps];
 }
 
 bool ARMBaseInstrInfo::isSwiftFastImmShift(const MachineInstr *MI) const {
diff --git a/lib/Target/ARM/ARMBaseInstrInfo.h b/lib/Target/ARM/ARMBaseInstrInfo.h
index ecbcf5c0f96..b4706e34893 100644
--- a/lib/Target/ARM/ARMBaseInstrInfo.h
+++ b/lib/Target/ARM/ARMBaseInstrInfo.h
@@ -116,8 +116,7 @@ class ARMBaseInstrInfo : public ARMGenInstrInfo {
                      bool AllowModify = false) const override;
   unsigned RemoveBranch(MachineBasicBlock &MBB) const override;
   unsigned InsertBranch(MachineBasicBlock &MBB, MachineBasicBlock *TBB,
-                        MachineBasicBlock *FBB,
-                        const SmallVectorImpl<MachineOperand> &Cond,
+                        MachineBasicBlock *FBB, ArrayRef<MachineOperand> Cond,
                         DebugLoc DL) const override;
 
   bool
@@ -133,10 +132,10 @@ class ARMBaseInstrInfo : public ARMGenInstrInfo {
   }
 
   bool PredicateInstruction(MachineInstr *MI,
-                    const SmallVectorImpl<MachineOperand> &Pred) const override;
+                    ArrayRef<MachineOperand> Pred) const override;
 
-  bool SubsumesPredicate(const SmallVectorImpl<MachineOperand> &Pred1,
-                   const SmallVectorImpl<MachineOperand> &Pred2) const override;
+  bool SubsumesPredicate(ArrayRef<MachineOperand> Pred1,
+                         ArrayRef<MachineOperand> Pred2) const override;
 
   bool DefinesPredicate(MachineInstr *MI,
                         std::vector<MachineOperand> &Pred) const override;
@@ -328,12 +327,12 @@ class ARMBaseInstrInfo : public ARMGenInstrInfo {
   int getInstrLatency(const InstrItineraryData *ItinData,
                       SDNode *Node) const override;
 
-  bool hasHighOperandLatency(const InstrItineraryData *ItinData,
+  bool hasHighOperandLatency(const TargetSchedModel &SchedModel,
                              const MachineRegisterInfo *MRI,
                              const MachineInstr *DefMI, unsigned DefIdx,
                              const MachineInstr *UseMI,
                              unsigned UseIdx) const override;
-  bool hasLowDefLatency(const InstrItineraryData *ItinData,
+  bool hasLowDefLatency(const TargetSchedModel &SchedModel,
                         const MachineInstr *DefMI,
                         unsigned DefIdx) const override;
 
@@ -439,7 +438,7 @@ static inline bool isPushOpcode(int Opc) {
 /// register by reference.
 ARMCC::CondCodes getInstrPredicate(const MachineInstr *MI, unsigned &PredReg);
 
-int getMatchingCondBranchOpcode(int Opc);
+unsigned getMatchingCondBranchOpcode(unsigned Opc);
 
 /// Determine if MI can be folded into an ARM MOVCC instruction, and return the
 /// opcode of the SSA instruction representing the conditional MI.
diff --git a/lib/Target/ARM/ARMBaseRegisterInfo.cpp b/lib/Target/ARM/ARMBaseRegisterInfo.cpp
index b673e9909ee..419717c85a7 100644
--- a/lib/Target/ARM/ARMBaseRegisterInfo.cpp
+++ b/lib/Target/ARM/ARMBaseRegisterInfo.cpp
@@ -63,8 +63,6 @@ static unsigned getFramePointerReg(const ARMSubtarget &STI) {
 const MCPhysReg*
 ARMBaseRegisterInfo::getCalleeSavedRegs(const MachineFunction *MF) const {
   const ARMSubtarget &STI = MF->getSubtarget<ARMSubtarget>();
-  if (STI.isTargetNaCl()) return CSR_NaCl_SaveList; // @LOCALMOD
-
   const MCPhysReg *RegList =
       STI.isTargetDarwin() ? CSR_iOS_SaveList : CSR_AAPCS_SaveList;
 
@@ -96,7 +94,6 @@ const uint32_t *
 ARMBaseRegisterInfo::getCallPreservedMask(const MachineFunction &MF,
                                           CallingConv::ID CC) const {
   const ARMSubtarget &STI = MF.getSubtarget<ARMSubtarget>();
-  if (STI.isTargetNaCl()) return CSR_NaCl_RegMask; // @LOCALMOD
   if (CC == CallingConv::GHC)
     // This is academic becase all GHC calls are (supposed to be) tail calls
     return CSR_NoRegs_RegMask;
@@ -130,7 +127,7 @@ ARMBaseRegisterInfo::getThisReturnPreservedMask(const MachineFunction &MF,
 BitVector ARMBaseRegisterInfo::
 getReservedRegs(const MachineFunction &MF) const {
   const ARMSubtarget &STI = MF.getSubtarget<ARMSubtarget>();
-  const TargetFrameLowering *TFI = STI.getFrameLowering();
+  const ARMFrameLowering *TFI = getFrameLowering(MF);
 
   // FIXME: avoid re-calculating this every time.
   BitVector Reserved(getNumRegs());
@@ -197,7 +194,7 @@ unsigned
 ARMBaseRegisterInfo::getRegPressureLimit(const TargetRegisterClass *RC,
                                          MachineFunction &MF) const {
   const ARMSubtarget &STI = MF.getSubtarget<ARMSubtarget>();
-  const TargetFrameLowering *TFI = STI.getFrameLowering();
+  const ARMFrameLowering *TFI = getFrameLowering(MF);
 
   switch (RC->getID()) {
   default:
@@ -228,7 +225,8 @@ ARMBaseRegisterInfo::getRegAllocationHints(unsigned VirtReg,
                                            ArrayRef<MCPhysReg> Order,
                                            SmallVectorImpl<MCPhysReg> &Hints,
                                            const MachineFunction &MF,
-                                           const VirtRegMap *VRM) const {
+                                           const VirtRegMap *VRM,
+                                           const LiveRegMatrix *Matrix) const {
   const MachineRegisterInfo &MRI = MF.getRegInfo();
   std::pair<unsigned, unsigned> Hint = MRI.getRegAllocationHint(VirtReg);
 
@@ -305,7 +303,7 @@ ARMBaseRegisterInfo::updateRegAllocHint(unsigned Reg, unsigned NewReg,
 bool ARMBaseRegisterInfo::hasBasePointer(const MachineFunction &MF) const {
   const MachineFrameInfo *MFI = MF.getFrameInfo();
   const ARMFunctionInfo *AFI = MF.getInfo<ARMFunctionInfo>();
-  const TargetFrameLowering *TFI = MF.getSubtarget().getFrameLowering();
+  const ARMFrameLowering *TFI = getFrameLowering(MF);
 
   // When outgoing call frames are so large that we adjust the stack pointer
   // around the call, we can no longer use the stack pointer to reach the
@@ -336,11 +334,12 @@ bool ARMBaseRegisterInfo::hasBasePointer(const MachineFunction &MF) const {
 bool ARMBaseRegisterInfo::canRealignStack(const MachineFunction &MF) const {
   const MachineRegisterInfo *MRI = &MF.getRegInfo();
   const ARMFunctionInfo *AFI = MF.getInfo<ARMFunctionInfo>();
+  const ARMFrameLowering *TFI = getFrameLowering(MF);
   // We can't realign the stack if:
   // 1. Dynamic stack realignment is explicitly disabled,
   // 2. This is a Thumb1 function (it's not useful, so we don't bother), or
   // 3. There are VLAs in the function and the base pointer is disabled.
-  if (MF.getFunction()->hasFnAttribute("no-realign-stack"))
+  if (!TargetRegisterInfo::canRealignStack(MF))
     return false;
   if (AFI->isThumb1OnlyFunction())
     return false;
@@ -350,25 +349,13 @@ bool ARMBaseRegisterInfo::canRealignStack(const MachineFunction &MF) const {
     return false;
   // We may also need a base pointer if there are dynamic allocas or stack
   // pointer adjustments around calls.
-  if (MF.getSubtarget().getFrameLowering()->hasReservedCallFrame(MF))
+  if (TFI->hasReservedCallFrame(MF))
     return true;
   // A base pointer is required and allowed.  Check that it isn't too late to
   // reserve it.
   return MRI->canReserveReg(BasePtr);
 }
 
-bool ARMBaseRegisterInfo::
-needsStackRealignment(const MachineFunction &MF) const {
-  const MachineFrameInfo *MFI = MF.getFrameInfo();
-  const Function *F = MF.getFunction();
-  unsigned StackAlign =
-      MF.getSubtarget().getFrameLowering()->getStackAlignment();
-  bool requiresRealignment = ((MFI->getMaxAlignment() > StackAlign) ||
-                              F->hasFnAttribute(Attribute::StackAlignment));
-
-  return requiresRealignment && canRealignStack(MF);
-}
-
 bool ARMBaseRegisterInfo::
 cannotEliminateFrame(const MachineFunction &MF) const {
   const MachineFrameInfo *MFI = MF.getFrameInfo();
@@ -381,7 +368,7 @@ cannotEliminateFrame(const MachineFunction &MF) const {
 unsigned
 ARMBaseRegisterInfo::getFrameRegister(const MachineFunction &MF) const {
   const ARMSubtarget &STI = MF.getSubtarget<ARMSubtarget>();
-  const TargetFrameLowering *TFI = STI.getFrameLowering();
+  const ARMFrameLowering *TFI = getFrameLowering(MF);
 
   if (TFI->hasFP(MF))
     return getFramePointerReg(STI);
@@ -520,7 +507,7 @@ needsFrameBaseReg(MachineInstr *MI, int64_t Offset) const {
   // Note that the incoming offset is based on the SP value at function entry,
   // so it'll be negative.
   MachineFunction &MF = *MI->getParent()->getParent();
-  const TargetFrameLowering *TFI = MF.getSubtarget().getFrameLowering();
+  const ARMFrameLowering *TFI = getFrameLowering(MF);
   MachineFrameInfo *MFI = MF.getFrameInfo();
   ARMFunctionInfo *AFI = MF.getInfo<ARMFunctionInfo>();
 
@@ -697,8 +684,7 @@ ARMBaseRegisterInfo::eliminateFrameIndex(MachineBasicBlock::iterator II,
   MachineFunction &MF = *MBB.getParent();
   const ARMBaseInstrInfo &TII =
       *static_cast<const ARMBaseInstrInfo *>(MF.getSubtarget().getInstrInfo());
-  const ARMFrameLowering *TFI = static_cast<const ARMFrameLowering *>(
-      MF.getSubtarget().getFrameLowering());
+  const ARMFrameLowering *TFI = getFrameLowering(MF);
   ARMFunctionInfo *AFI = MF.getInfo<ARMFunctionInfo>();
   assert(!AFI->isThumb1OnlyFunction() &&
          "This eliminateFrameIndex does not support Thumb1!");
diff --git a/lib/Target/ARM/ARMBaseRegisterInfo.h b/lib/Target/ARM/ARMBaseRegisterInfo.h
index fdc1ef9432c..d4b7c7be865 100644
--- a/lib/Target/ARM/ARMBaseRegisterInfo.h
+++ b/lib/Target/ARM/ARMBaseRegisterInfo.h
@@ -126,15 +126,15 @@ class ARMBaseRegisterInfo : public ARMGenRegisterInfo {
                              ArrayRef<MCPhysReg> Order,
                              SmallVectorImpl<MCPhysReg> &Hints,
                              const MachineFunction &MF,
-                             const VirtRegMap *VRM) const override;
+                             const VirtRegMap *VRM,
+                             const LiveRegMatrix *Matrix) const override;
 
   void updateRegAllocHint(unsigned Reg, unsigned NewReg,
                           MachineFunction &MF) const override;
 
   bool hasBasePointer(const MachineFunction &MF) const;
 
-  bool canRealignStack(const MachineFunction &MF) const;
-  bool needsStackRealignment(const MachineFunction &MF) const override;
+  bool canRealignStack(const MachineFunction &MF) const override;
   int64_t getFrameIndexInstrOffset(const MachineInstr *MI,
                                    int Idx) const override;
   bool needsFrameBaseReg(MachineInstr *MI, int64_t Offset) const override;
diff --git a/lib/Target/ARM/ARMCallingConv.td b/lib/Target/ARM/ARMCallingConv.td
index 83fa0130c72..27cf06b995a 100644
--- a/lib/Target/ARM/ARMCallingConv.td
+++ b/lib/Target/ARM/ARMCallingConv.td
@@ -142,6 +142,9 @@ def CC_ARM_AAPCS : CallingConv<[
   // Handles byval parameters.
   CCIfByVal<CCPassByVal<4, 4>>,
 
+  // The 'nest' parameter, if any, is passed in R12.
+  CCIfNest<CCAssignToReg<[R12]>>,
+
   // Handle all vector types as either f64 or v2f64.
   CCIfType<[v1i64, v2i32, v4i16, v8i8, v2f32], CCBitConvertToType<f64>>,
   CCIfType<[v2i64, v4i32, v8i16, v16i8, v4f32], CCBitConvertToType<v2f64>>,
@@ -238,8 +241,4 @@ def CSR_GenericInt : CalleeSavedRegs<(add LR, (sequence "R%u", 12, 0))>;
 // registers.
 def CSR_FIQ : CalleeSavedRegs<(add LR, R11, (sequence "R%u", 7, 0))>;
 
-// @LOCALMOD-START
-// NaCl does not save R9, but otherwise uses the same order as AAPCS
-def CSR_NaCl : CalleeSavedRegs<(add LR, R11, R10, R8, R7, R6, R5, R4,
-                                     (sequence "D%u", 15, 8))>;
-// @LOCALMOD-END
+
diff --git a/lib/Target/ARM/ARMConstantIslandPass.cpp b/lib/Target/ARM/ARMConstantIslandPass.cpp
index 0d79e6c3b4c..f3a6027107c 100644
--- a/lib/Target/ARM/ARMConstantIslandPass.cpp
+++ b/lib/Target/ARM/ARMConstantIslandPass.cpp
@@ -53,11 +53,6 @@ static cl::opt<bool>
 AdjustJumpTableBlocks("arm-adjust-jump-tables", cl::Hidden, cl::init(true),
           cl::desc("Adjust basic block layout to better use TB[BH]"));
 
-// FIXME: This option should be removed once it has received sufficient testing.
-static cl::opt<bool>
-AlignConstantIslands("arm-align-constant-islands", cl::Hidden, cl::init(true),
-          cl::desc("Align constant islands in code"));
-
 /// UnknownPadding - Return the worst case padding that could result from
 /// unknown offset bits.  This does not include alignment padding caused by
 /// known offset bits.
@@ -185,9 +180,7 @@ namespace {
       MachineInstr *MI;
       MachineInstr *CPEMI;
       MachineBasicBlock *HighWaterMark;
-    private:
       unsigned MaxDisp;
-    public:
       bool NegOk;
       bool IsSoImm;
       bool KnownAlignment;
@@ -221,12 +214,24 @@ namespace {
     };
 
     /// CPEntries - Keep track of all of the constant pool entry machine
-    /// instructions. For each original constpool index (i.e. those that
-    /// existed upon entry to this pass), it keeps a vector of entries.
-    /// Original elements are cloned as we go along; the clones are
-    /// put in the vector of the original element, but have distinct CPIs.
+    /// instructions. For each original constpool index (i.e. those that existed
+    /// upon entry to this pass), it keeps a vector of entries.  Original
+    /// elements are cloned as we go along; the clones are put in the vector of
+    /// the original element, but have distinct CPIs.
+    ///
+    /// The first half of CPEntries contains generic constants, the second half
+    /// contains jump tables. Use getCombinedIndex on a generic CPEMI to look up
+    /// which vector it will be in here.
     std::vector<std::vector<CPEntry> > CPEntries;
 
+    /// Maps a JT index to the offset in CPEntries containing copies of that
+    /// table. The equivalent map for a CONSTPOOL_ENTRY is the identity.
+    DenseMap<int, int> JumpTableEntryIndices;
+
+    /// Maps a JT index to the LEA that actually uses the index to calculate its
+    /// base address.
+    DenseMap<int, int> JumpTableUserIndices;
+
     /// ImmBranch - One per immediate branch, keeping the machine instruction
     /// pointer, conditional or unconditional, the max displacement,
     /// and (if isCond is true) the corresponding unconditional branch
@@ -235,8 +240,8 @@ namespace {
       MachineInstr *MI;
       unsigned MaxDisp : 31;
       bool isCond : 1;
-      int UncondBr;
-      ImmBranch(MachineInstr *mi, unsigned maxdisp, bool cond, int ubr)
+      unsigned UncondBr;
+      ImmBranch(MachineInstr *mi, unsigned maxdisp, bool cond, unsigned ubr)
         : MI(mi), MaxDisp(maxdisp), isCond(cond), UncondBr(ubr) {}
     };
 
@@ -274,7 +279,8 @@ namespace {
     }
 
   private:
-    void doInitialPlacement(std::vector<MachineInstr*> &CPEMIs);
+    void doInitialConstPlacement(std::vector<MachineInstr *> &CPEMIs);
+    void doInitialJumpTablePlacement(std::vector<MachineInstr *> &CPEMIs);
     bool BBHasFallthrough(MachineBasicBlock *MBB);
     CPEntry *findConstPoolEntry(unsigned CPI, const MachineInstr *CPEMI);
     unsigned getCPELogAlign(const MachineInstr *CPEMI);
@@ -284,6 +290,7 @@ namespace {
     void updateForInsertedWaterBlock(MachineBasicBlock *NewBB);
     void adjustBBOffsetsAfter(MachineBasicBlock *BB);
     bool decrementCPEReferenceCount(unsigned CPI, MachineInstr* CPEMI);
+    unsigned getCombinedIndex(const MachineInstr *CPEMI);
     int findInRangeCPEntry(CPUser& U, unsigned UserOffset);
     bool findAvailableWater(CPUser&U, unsigned UserOffset,
                             water_iterator &WaterIter);
@@ -306,6 +313,9 @@ namespace {
     bool optimizeThumb2Instructions();
     bool optimizeThumb2Branches();
     bool reorderThumb2JumpTables();
+    bool preserveBaseRegister(MachineInstr *JumpMI, MachineInstr *LEAMI,
+                              unsigned &DeadSize, bool &CanDeleteLEA,
+                              bool &BaseRegKill);
     bool optimizeThumb2JumpTables();
     MachineBasicBlock *adjustJTTargetBlockForward(MachineBasicBlock *BB,
                                                   MachineBasicBlock *JTBB);
@@ -387,8 +397,6 @@ bool ARMConstantIslands::runOnMachineFunction(MachineFunction &mf) {
   TII = STI->getInstrInfo();
   AFI = MF->getInfo<ARMFunctionInfo>();
 
-  if (!STI->useConstIslands()) return false;   // @LOCALMOD
-
   isThumb = AFI->isThumbFunction();
   isThumb1 = AFI->isThumb1OnlyFunction();
   isThumb2 = AFI->isThumb2Function();
@@ -414,18 +422,14 @@ bool ARMConstantIslands::runOnMachineFunction(MachineFunction &mf) {
     MF->RenumberBlocks();
   }
 
-  // Thumb1 functions containing constant pools get 4-byte alignment.
-  // This is so we can keep exact track of where the alignment padding goes.
-
-  // ARM and Thumb2 functions need to be 4-byte aligned.
-  if (!isThumb1)
-    MF->ensureAlignment(2);  // 2 = log2(4)
-
   // Perform the initial placement of the constant pool entries.  To start with,
   // we put them all at the end of the function.
   std::vector<MachineInstr*> CPEMIs;
   if (!MCP->isEmpty())
-    doInitialPlacement(CPEMIs);
+    doInitialConstPlacement(CPEMIs);
+
+  if (MF->getJumpTableInfo())
+    doInitialJumpTablePlacement(CPEMIs);
 
   /// The next UID to take is the first unused one.
   AFI->initPICLabelUId(CPEMIs.size());
@@ -437,6 +441,10 @@ bool ARMConstantIslands::runOnMachineFunction(MachineFunction &mf) {
   CPEMIs.clear();
   DEBUG(dumpBBs());
 
+  // Functions with jump tables need an alignment of 4 because they use the ADR
+  // instruction, which aligns the PC to 4 bytes before adding an offset.
+  if (!T2JumpTables.empty())
+    MF->ensureAlignment(2);
 
   /// Remove dead constant pool entries.
   MadeChange |= removeUnusedCPEntries();
@@ -486,7 +494,8 @@ bool ARMConstantIslands::runOnMachineFunction(MachineFunction &mf) {
   for (unsigned i = 0, e = CPEntries.size(); i != e; ++i) {
     for (unsigned j = 0, je = CPEntries[i].size(); j != je; ++j) {
       const CPEntry & CPE = CPEntries[i][j];
-      AFI->recordCPEClone(i, CPE.CPI);
+      if (CPE.CPEMI && CPE.CPEMI->getOperand(1).isCPI())
+        AFI->recordCPEClone(i, CPE.CPI);
     }
   }
 
@@ -496,6 +505,8 @@ bool ARMConstantIslands::runOnMachineFunction(MachineFunction &mf) {
   WaterList.clear();
   CPUsers.clear();
   CPEntries.clear();
+  JumpTableEntryIndices.clear();
+  JumpTableUserIndices.clear();
   ImmBranches.clear();
   PushPopMIs.clear();
   T2JumpTables.clear();
@@ -503,10 +514,10 @@ bool ARMConstantIslands::runOnMachineFunction(MachineFunction &mf) {
   return MadeChange;
 }
 
-/// doInitialPlacement - Perform the initial placement of the constant pool
-/// entries.  To start with, we put them all at the end of the function.
+/// \brief Perform the initial placement of the regular constant pool entries.
+/// To start with, we put them all at the end of the function.
 void
-ARMConstantIslands::doInitialPlacement(std::vector<MachineInstr*> &CPEMIs) {
+ARMConstantIslands::doInitialConstPlacement(std::vector<MachineInstr*> &CPEMIs) {
   // Create the basic block to hold the CPE's.
   MachineBasicBlock *BB = MF->CreateMachineBasicBlock();
   MF->push_back(BB);
@@ -515,8 +526,7 @@ ARMConstantIslands::doInitialPlacement(std::vector<MachineInstr*> &CPEMIs) {
   unsigned MaxAlign = Log2_32(MCP->getConstantPoolAlignment());
 
   // Mark the basic block as required by the const-pool.
-  // If AlignConstantIslands isn't set, use 4-byte alignment for everything.
-  BB->setAlignment(AlignConstantIslands ? MaxAlign : 2);
+  BB->setAlignment(MaxAlign);
 
   // The function needs to be as aligned as the basic blocks. The linker may
   // move functions around based on their alignment.
@@ -532,7 +542,7 @@ ARMConstantIslands::doInitialPlacement(std::vector<MachineInstr*> &CPEMIs) {
   // identity mapping of CPI's to CPE's.
   const std::vector<MachineConstantPoolEntry> &CPs = MCP->getConstants();
 
-  const DataLayout &TD = *MF->getTarget().getDataLayout();
+  const DataLayout &TD = MF->getDataLayout();
   for (unsigned i = 0, e = CPs.size(); i != e; ++i) {
     unsigned Size = TD.getTypeAllocSize(CPs[i].getType());
     assert(Size >= 4 && "Too small constant pool entry");
@@ -565,6 +575,66 @@ ARMConstantIslands::doInitialPlacement(std::vector<MachineInstr*> &CPEMIs) {
   DEBUG(BB->dump());
 }
 
+/// \brief Do initial placement of the jump tables. Because Thumb2's TBB and TBH
+/// instructions can be made more efficient if the jump table immediately
+/// follows the instruction, it's best to place them immediately next to their
+/// jumps to begin with. In almost all cases they'll never be moved from that
+/// position.
+void ARMConstantIslands::doInitialJumpTablePlacement(
+    std::vector<MachineInstr *> &CPEMIs) {
+  unsigned i = CPEntries.size();
+  auto MJTI = MF->getJumpTableInfo();
+  const std::vector<MachineJumpTableEntry> &JT = MJTI->getJumpTables();
+
+  MachineBasicBlock *LastCorrectlyNumberedBB = nullptr;
+  for (MachineBasicBlock &MBB : *MF) {
+    auto MI = MBB.getLastNonDebugInstr();
+
+    unsigned JTOpcode;
+    switch (MI->getOpcode()) {
+    default:
+      continue;
+    case ARM::BR_JTadd:
+    case ARM::BR_JTr:
+    case ARM::tBR_JTr:
+    case ARM::BR_JTm:
+      JTOpcode = ARM::JUMPTABLE_ADDRS;
+      break;
+    case ARM::t2BR_JT:
+      JTOpcode = ARM::JUMPTABLE_INSTS;
+      break;
+    case ARM::t2TBB_JT:
+      JTOpcode = ARM::JUMPTABLE_TBB;
+      break;
+    case ARM::t2TBH_JT:
+      JTOpcode = ARM::JUMPTABLE_TBH;
+      break;
+    }
+
+    unsigned NumOps = MI->getDesc().getNumOperands();
+    MachineOperand JTOp =
+      MI->getOperand(NumOps - (MI->isPredicable() ? 2 : 1));
+    unsigned JTI = JTOp.getIndex();
+    unsigned Size = JT[JTI].MBBs.size() * sizeof(uint32_t);
+    MachineBasicBlock *JumpTableBB = MF->CreateMachineBasicBlock();
+    MF->insert(std::next(MachineFunction::iterator(MBB)), JumpTableBB);
+    MachineInstr *CPEMI = BuildMI(*JumpTableBB, JumpTableBB->begin(),
+                                  DebugLoc(), TII->get(JTOpcode))
+                              .addImm(i++)
+                              .addJumpTableIndex(JTI)
+                              .addImm(Size);
+    CPEMIs.push_back(CPEMI);
+    CPEntries.emplace_back(1, CPEntry(CPEMI, JTI));
+    JumpTableEntryIndices.insert(std::make_pair(JTI, CPEntries.size() - 1));
+    if (!LastCorrectlyNumberedBB)
+      LastCorrectlyNumberedBB = &MBB;
+  }
+
+  // If we did anything then we need to renumber the subsequent blocks.
+  if (LastCorrectlyNumberedBB)
+    MF->RenumberBlocks(LastCorrectlyNumberedBB);
+}
+
 /// BBHasFallthrough - Return true if the specified basic block can fallthrough
 /// into the block immediately after it.
 bool ARMConstantIslands::BBHasFallthrough(MachineBasicBlock *MBB) {
@@ -604,13 +674,21 @@ ARMConstantIslands::CPEntry
 /// getCPELogAlign - Returns the required alignment of the constant pool entry
 /// represented by CPEMI.  Alignment is measured in log2(bytes) units.
 unsigned ARMConstantIslands::getCPELogAlign(const MachineInstr *CPEMI) {
-  assert(CPEMI && CPEMI->getOpcode() == ARM::CONSTPOOL_ENTRY);
-
-  // Everything is 4-byte aligned unless AlignConstantIslands is set.
-  if (!AlignConstantIslands)
+  switch (CPEMI->getOpcode()) {
+  case ARM::CONSTPOOL_ENTRY:
+    break;
+  case ARM::JUMPTABLE_TBB:
+    return 0;
+  case ARM::JUMPTABLE_TBH:
+  case ARM::JUMPTABLE_INSTS:
+    return 1;
+  case ARM::JUMPTABLE_ADDRS:
     return 2;
+  default:
+    llvm_unreachable("unknown constpool entry kind");
+  }
 
-  unsigned CPI = CPEMI->getOperand(1).getIndex();
+  unsigned CPI = getCombinedIndex(CPEMI);
   assert(CPI < MCP->getConstants().size() && "Invalid constant pool index.");
   unsigned Align = MCP->getConstants()[CPI].getAlignment();
   assert(isPowerOf2_32(Align) && "Invalid CPE alignment");
@@ -669,7 +747,7 @@ initializeFunctionInfo(const std::vector<MachineInstr*> &CPEMIs) {
       if (I->isDebugValue())
         continue;
 
-      int Opc = I->getOpcode();
+      unsigned Opc = I->getOpcode();
       if (I->isBranch()) {
         bool isCond = false;
         unsigned Bits = 0;
@@ -719,12 +797,14 @@ initializeFunctionInfo(const std::vector<MachineInstr*> &CPEMIs) {
       if (Opc == ARM::tPUSH || Opc == ARM::tPOP_RET)
         PushPopMIs.push_back(I);
 
-      if (Opc == ARM::CONSTPOOL_ENTRY)
+      if (Opc == ARM::CONSTPOOL_ENTRY || Opc == ARM::JUMPTABLE_ADDRS ||
+          Opc == ARM::JUMPTABLE_INSTS || Opc == ARM::JUMPTABLE_TBB ||
+          Opc == ARM::JUMPTABLE_TBH)
         continue;
 
       // Scan the instructions for constant pool operands.
       for (unsigned op = 0, e = I->getNumOperands(); op != e; ++op)
-        if (I->getOperand(op).isCPI()) {
+        if (I->getOperand(op).isCPI() || I->getOperand(op).isJTI()) {
           // We found one.  The addressing mode tells us the max displacement
           // from the PC that this instruction permits.
 
@@ -740,6 +820,7 @@ initializeFunctionInfo(const std::vector<MachineInstr*> &CPEMIs) {
 
           // Taking the address of a CP entry.
           case ARM::LEApcrel:
+          case ARM::LEApcrelJT:
             // This takes a SoImm, which is 8 bit immediate rotated. We'll
             // pretend the maximum offset is 255 * 4. Since each instruction
             // 4 byte wide, this is always correct. We'll check for other
@@ -750,10 +831,12 @@ initializeFunctionInfo(const std::vector<MachineInstr*> &CPEMIs) {
             IsSoImm = true;
             break;
           case ARM::t2LEApcrel:
+          case ARM::t2LEApcrelJT:
             Bits = 12;
             NegOk = true;
             break;
           case ARM::tLEApcrel:
+          case ARM::tLEApcrelJT:
             Bits = 8;
             Scale = 4;
             break;
@@ -781,6 +864,11 @@ initializeFunctionInfo(const std::vector<MachineInstr*> &CPEMIs) {
 
           // Remember that this is a user of a CP entry.
           unsigned CPI = I->getOperand(op).getIndex();
+          if (I->getOperand(op).isJTI()) {
+            JumpTableUserIndices.insert(std::make_pair(CPI, CPUsers.size()));
+            CPI = JumpTableEntryIndices[CPI];
+          }
+
           MachineInstr *CPEMI = CPEMIs[CPI];
           unsigned MaxOffs = ((1 << Bits)-1) * Scale;
           CPUsers.push_back(CPUser(I, CPEMI, MaxOffs, NegOk, IsSoImm));
@@ -1114,6 +1202,13 @@ bool ARMConstantIslands::decrementCPEReferenceCount(unsigned CPI,
   return false;
 }
 
+unsigned ARMConstantIslands::getCombinedIndex(const MachineInstr *CPEMI) {
+  if (CPEMI->getOperand(1).isCPI())
+    return CPEMI->getOperand(1).getIndex();
+
+  return JumpTableEntryIndices[CPEMI->getOperand(1).getIndex()];
+}
+
 /// LookForCPEntryInRange - see if the currently referenced CPE is in range;
 /// if not, see if an in-range clone of the CPE is in range, and if so,
 /// change the data structures so the user references the clone.  Returns:
@@ -1133,7 +1228,7 @@ int ARMConstantIslands::findInRangeCPEntry(CPUser& U, unsigned UserOffset)
   }
 
   // No.  Look for previously created clones of the CPE that are in range.
-  unsigned CPI = CPEMI->getOperand(1).getIndex();
+  unsigned CPI = getCombinedIndex(CPEMI);
   std::vector<CPEntry> &CPEs = CPEntries[CPI];
   for (unsigned i = 0, e = CPEs.size(); i != e; ++i) {
     // We already tried this one
@@ -1378,7 +1473,7 @@ bool ARMConstantIslands::handleConstantPoolUser(unsigned CPUserIndex) {
   CPUser &U = CPUsers[CPUserIndex];
   MachineInstr *UserMI = U.MI;
   MachineInstr *CPEMI  = U.CPEMI;
-  unsigned CPI = CPEMI->getOperand(1).getIndex();
+  unsigned CPI = getCombinedIndex(CPEMI);
   unsigned Size = CPEMI->getOperand(2).getImm();
   // Compute this only once, it's expensive.
   unsigned UserOffset = getUserOffset(U);
@@ -1442,17 +1537,17 @@ bool ARMConstantIslands::handleConstantPoolUser(unsigned CPUserIndex) {
   // Update internal data structures to account for the newly inserted MBB.
   updateForInsertedWaterBlock(NewIsland);
 
-  // Decrement the old entry, and remove it if refcount becomes 0.
-  decrementCPEReferenceCount(CPI, CPEMI);
-
   // Now that we have an island to add the CPE to, clone the original CPE and
   // add it to the island.
   U.HighWaterMark = NewIsland;
-  U.CPEMI = BuildMI(NewIsland, DebugLoc(), TII->get(ARM::CONSTPOOL_ENTRY))
-                .addImm(ID).addConstantPoolIndex(CPI).addImm(Size);
+  U.CPEMI = BuildMI(NewIsland, DebugLoc(), CPEMI->getDesc())
+                .addImm(ID).addOperand(CPEMI->getOperand(1)).addImm(Size);
   CPEntries[CPI].push_back(CPEntry(U.CPEMI, ID, 1));
   ++NumCPEs;
 
+  // Decrement the old entry, and remove it if refcount becomes 0.
+  decrementCPEReferenceCount(CPI, CPEMI);
+
   // Mark the basic block as aligned as required by the const-pool entry.
   NewIsland->setAlignment(getCPELogAlign(U.CPEMI));
 
@@ -1764,8 +1859,13 @@ bool ARMConstantIslands::optimizeThumb2Instructions() {
 bool ARMConstantIslands::optimizeThumb2Branches() {
   bool MadeChange = false;
 
-  for (unsigned i = 0, e = ImmBranches.size(); i != e; ++i) {
-    ImmBranch &Br = ImmBranches[i];
+  // The order in which branches appear in ImmBranches is approximately their
+  // order within the function body. By visiting later branches first, we reduce
+  // the distance between earlier forward branches and their targets, making it
+  // more likely that the cbn?z optimization, which can only apply to forward
+  // branches, will succeed.
+  for (unsigned i = ImmBranches.size(); i != 0; --i) {
+    ImmBranch &Br = ImmBranches[i-1];
     unsigned Opcode = Br.MI->getOpcode();
     unsigned NewOpc = 0;
     unsigned Scale = 1;
@@ -1852,6 +1952,122 @@ bool ARMConstantIslands::optimizeThumb2Branches() {
   return MadeChange;
 }
 
+static bool isSimpleIndexCalc(MachineInstr &I, unsigned EntryReg,
+                              unsigned BaseReg) {
+  if (I.getOpcode() != ARM::t2ADDrs)
+    return false;
+
+  if (I.getOperand(0).getReg() != EntryReg)
+    return false;
+
+  if (I.getOperand(1).getReg() != BaseReg)
+    return false;
+
+  // FIXME: what about CC and IdxReg?
+  return true;
+}
+
+/// \brief While trying to form a TBB/TBH instruction, we may (if the table
+/// doesn't immediately follow the BR_JT) need access to the start of the
+/// jump-table. We know one instruction that produces such a register; this
+/// function works out whether that definition can be preserved to the BR_JT,
+/// possibly by removing an intervening addition (which is usually needed to
+/// calculate the actual entry to jump to).
+bool ARMConstantIslands::preserveBaseRegister(MachineInstr *JumpMI,
+                                              MachineInstr *LEAMI,
+                                              unsigned &DeadSize,
+                                              bool &CanDeleteLEA,
+                                              bool &BaseRegKill) {
+  if (JumpMI->getParent() != LEAMI->getParent())
+    return false;
+
+  // Now we hope that we have at least these instructions in the basic block:
+  //     BaseReg = t2LEA ...
+  //     [...]
+  //     EntryReg = t2ADDrs BaseReg, ...
+  //     [...]
+  //     t2BR_JT EntryReg
+  //
+  // We have to be very conservative about what we recognise here though. The
+  // main perturbing factors to watch out for are:
+  //    + Spills at any point in the chain: not direct problems but we would
+  //      expect a blocking Def of the spilled register so in practice what we
+  //      can do is limited.
+  //    + EntryReg == BaseReg: this is the one situation we should allow a Def
+  //      of BaseReg, but only if the t2ADDrs can be removed.
+  //    + Some instruction other than t2ADDrs computing the entry. Not seen in
+  //      the wild, but we should be careful.
+  unsigned EntryReg = JumpMI->getOperand(0).getReg();
+  unsigned BaseReg = LEAMI->getOperand(0).getReg();
+
+  CanDeleteLEA = true;
+  BaseRegKill = false;
+  MachineInstr *RemovableAdd = nullptr;
+  MachineBasicBlock::iterator I(LEAMI);
+  for (++I; &*I != JumpMI; ++I) {
+    if (isSimpleIndexCalc(*I, EntryReg, BaseReg)) {
+      RemovableAdd = &*I;
+      break;
+    }
+
+    for (unsigned K = 0, E = I->getNumOperands(); K != E; ++K) {
+      const MachineOperand &MO = I->getOperand(K);
+      if (!MO.isReg() || !MO.getReg())
+        continue;
+      if (MO.isDef() && MO.getReg() == BaseReg)
+        return false;
+      if (MO.isUse() && MO.getReg() == BaseReg) {
+        BaseRegKill = BaseRegKill || MO.isKill();
+        CanDeleteLEA = false;
+      }
+    }
+  }
+
+  if (!RemovableAdd)
+    return true;
+
+  // Check the add really is removable, and that nothing else in the block
+  // clobbers BaseReg.
+  for (++I; &*I != JumpMI; ++I) {
+    for (unsigned K = 0, E = I->getNumOperands(); K != E; ++K) {
+      const MachineOperand &MO = I->getOperand(K);
+      if (!MO.isReg() || !MO.getReg())
+        continue;
+      if (MO.isDef() && MO.getReg() == BaseReg)
+        return false;
+      if (MO.isUse() && MO.getReg() == EntryReg)
+        RemovableAdd = nullptr;
+    }
+  }
+
+  if (RemovableAdd) {
+    RemovableAdd->eraseFromParent();
+    DeadSize += 4;
+  } else if (BaseReg == EntryReg) {
+    // The add wasn't removable, but clobbered the base for the TBB. So we can't
+    // preserve it.
+    return false;
+  }
+
+  // We reached the end of the block without seeing another definition of
+  // BaseReg (except, possibly the t2ADDrs, which was removed). BaseReg can be
+  // used in the TBB/TBH if necessary.
+  return true;
+}
+
+/// \brief Returns whether CPEMI is the first instruction in the block
+/// immediately following JTMI (assumed to be a TBB or TBH terminator). If so,
+/// we can switch the first register to PC and usually remove the address
+/// calculation that preceeded it.
+static bool jumpTableFollowsTB(MachineInstr *JTMI, MachineInstr *CPEMI) {
+  MachineFunction::iterator MBB = JTMI->getParent();
+  MachineFunction *MF = MBB->getParent();
+  ++MBB;
+
+  return MBB != MF->end() && MBB->begin() != MBB->end() &&
+         &*MBB->begin() == CPEMI;
+}
+
 /// optimizeThumb2JumpTables - Use tbb / tbh instructions to generate smaller
 /// jumptables when it's possible.
 bool ARMConstantIslands::optimizeThumb2JumpTables() {
@@ -1867,7 +2083,7 @@ bool ARMConstantIslands::optimizeThumb2JumpTables() {
     MachineInstr *MI = T2JumpTables[i];
     const MCInstrDesc &MCID = MI->getDesc();
     unsigned NumOps = MCID.getNumOperands();
-    unsigned JTOpIdx = NumOps - (MI->isPredicable() ? 3 : 2);
+    unsigned JTOpIdx = NumOps - (MI->isPredicable() ? 2 : 1);
     MachineOperand JTOP = MI->getOperand(JTOpIdx);
     unsigned JTI = JTOP.getIndex();
     assert(JTI < JT.size());
@@ -1890,93 +2106,79 @@ bool ARMConstantIslands::optimizeThumb2JumpTables() {
         break;
     }
 
-    if (ByteOk || HalfWordOk) {
-      MachineBasicBlock *MBB = MI->getParent();
-      unsigned BaseReg = MI->getOperand(0).getReg();
-      bool BaseRegKill = MI->getOperand(0).isKill();
-      if (!BaseRegKill)
-        continue;
-      unsigned IdxReg = MI->getOperand(1).getReg();
-      bool IdxRegKill = MI->getOperand(1).isKill();
-
-      // Scan backwards to find the instruction that defines the base
-      // register. Due to post-RA scheduling, we can't count on it
-      // immediately preceding the branch instruction.
-      MachineBasicBlock::iterator PrevI = MI;
-      MachineBasicBlock::iterator B = MBB->begin();
-      while (PrevI != B && !PrevI->definesRegister(BaseReg))
-        --PrevI;
-
-      // If for some reason we didn't find it, we can't do anything, so
-      // just skip this one.
-      if (!PrevI->definesRegister(BaseReg))
-        continue;
-
-      MachineInstr *AddrMI = PrevI;
-      bool OptOk = true;
-      // Examine the instruction that calculates the jumptable entry address.
-      // Make sure it only defines the base register and kills any uses
-      // other than the index register.
-      for (unsigned k = 0, eee = AddrMI->getNumOperands(); k != eee; ++k) {
-        const MachineOperand &MO = AddrMI->getOperand(k);
-        if (!MO.isReg() || !MO.getReg())
-          continue;
-        if (MO.isDef() && MO.getReg() != BaseReg) {
-          OptOk = false;
-          break;
-        }
-        if (MO.isUse() && !MO.isKill() && MO.getReg() != IdxReg) {
-          OptOk = false;
-          break;
-        }
-      }
-      if (!OptOk)
-        continue;
+    if (!ByteOk && !HalfWordOk)
+      continue;
 
-      // Now scan back again to find the tLEApcrel or t2LEApcrelJT instruction
-      // that gave us the initial base register definition.
-      for (--PrevI; PrevI != B && !PrevI->definesRegister(BaseReg); --PrevI)
-        ;
+    MachineBasicBlock *MBB = MI->getParent();
+    if (!MI->getOperand(0).isKill()) // FIXME: needed now?
+      continue;
+    unsigned IdxReg = MI->getOperand(1).getReg();
+    bool IdxRegKill = MI->getOperand(1).isKill();
 
-      // The instruction should be a tLEApcrel or t2LEApcrelJT; we want
-      // to delete it as well.
-      MachineInstr *LeaMI = PrevI;
-      if ((LeaMI->getOpcode() != ARM::tLEApcrelJT &&
-           LeaMI->getOpcode() != ARM::t2LEApcrelJT) ||
-          LeaMI->getOperand(0).getReg() != BaseReg)
-        OptOk = false;
+    CPUser &User = CPUsers[JumpTableUserIndices[JTI]];
+    unsigned DeadSize = 0;
+    bool CanDeleteLEA = false;
+    bool BaseRegKill = false;
+    bool PreservedBaseReg =
+        preserveBaseRegister(MI, User.MI, DeadSize, CanDeleteLEA, BaseRegKill);
 
-      if (!OptOk)
-        continue;
+    if (!jumpTableFollowsTB(MI, User.CPEMI) && !PreservedBaseReg)
+      continue;
 
-      DEBUG(dbgs() << "Shrink JT: " << *MI << "     addr: " << *AddrMI
-                   << "      lea: " << *LeaMI);
-      unsigned Opc = ByteOk ? ARM::t2TBB_JT : ARM::t2TBH_JT;
-      MachineBasicBlock::iterator MI_JT = MI;
-      MachineInstr *NewJTMI =
+    DEBUG(dbgs() << "Shrink JT: " << *MI);
+    MachineInstr *CPEMI = User.CPEMI;
+    unsigned Opc = ByteOk ? ARM::t2TBB_JT : ARM::t2TBH_JT;
+    MachineBasicBlock::iterator MI_JT = MI;
+    MachineInstr *NewJTMI =
         BuildMI(*MBB, MI_JT, MI->getDebugLoc(), TII->get(Opc))
-        .addReg(IdxReg, getKillRegState(IdxRegKill))
-        .addJumpTableIndex(JTI, JTOP.getTargetFlags())
-        .addImm(MI->getOperand(JTOpIdx+1).getImm());
-      DEBUG(dbgs() << "BB#" << MBB->getNumber() << ": " << *NewJTMI);
-      // FIXME: Insert an "ALIGN" instruction to ensure the next instruction
-      // is 2-byte aligned. For now, asm printer will fix it up.
-      unsigned NewSize = TII->GetInstSizeInBytes(NewJTMI);
-      unsigned OrigSize = TII->GetInstSizeInBytes(AddrMI);
-      OrigSize += TII->GetInstSizeInBytes(LeaMI);
-      OrigSize += TII->GetInstSizeInBytes(MI);
-
-      AddrMI->eraseFromParent();
-      LeaMI->eraseFromParent();
-      MI->eraseFromParent();
+            .addReg(User.MI->getOperand(0).getReg(),
+                    getKillRegState(BaseRegKill))
+            .addReg(IdxReg, getKillRegState(IdxRegKill))
+            .addJumpTableIndex(JTI, JTOP.getTargetFlags())
+            .addImm(CPEMI->getOperand(0).getImm());
+    DEBUG(dbgs() << "BB#" << MBB->getNumber() << ": " << *NewJTMI);
+
+    unsigned JTOpc = ByteOk ? ARM::JUMPTABLE_TBB : ARM::JUMPTABLE_TBH;
+    CPEMI->setDesc(TII->get(JTOpc));
+
+    if (jumpTableFollowsTB(MI, User.CPEMI)) {
+      NewJTMI->getOperand(0).setReg(ARM::PC);
+      NewJTMI->getOperand(0).setIsKill(false);
+
+      if (CanDeleteLEA)  {
+        User.MI->eraseFromParent();
+        DeadSize += 4;
+
+        // The LEA was eliminated, the TBB instruction becomes the only new user
+        // of the jump table.
+        User.MI = NewJTMI;
+        User.MaxDisp = 4;
+        User.NegOk = false;
+        User.IsSoImm = false;
+        User.KnownAlignment = false;
+      } else {
+        // The LEA couldn't be eliminated, so we must add another CPUser to
+        // record the TBB or TBH use.
+        int CPEntryIdx = JumpTableEntryIndices[JTI];
+        auto &CPEs = CPEntries[CPEntryIdx];
+        auto Entry = std::find_if(CPEs.begin(), CPEs.end(), [&](CPEntry &E) {
+          return E.CPEMI == User.CPEMI;
+        });
+        ++Entry->RefCount;
+        CPUsers.emplace_back(CPUser(NewJTMI, User.CPEMI, 4, false, false));
+      }
+    }
 
-      int delta = OrigSize - NewSize;
-      BBInfo[MBB->getNumber()].Size -= delta;
-      adjustBBOffsetsAfter(MBB);
+    unsigned NewSize = TII->GetInstSizeInBytes(NewJTMI);
+    unsigned OrigSize = TII->GetInstSizeInBytes(MI);
+    MI->eraseFromParent();
 
-      ++NumTBs;
-      MadeChange = true;
-    }
+    int Delta = OrigSize - NewSize + DeadSize;
+    BBInfo[MBB->getNumber()].Size -= Delta;
+    adjustBBOffsetsAfter(MBB);
+
+    ++NumTBs;
+    MadeChange = true;
   }
 
   return MadeChange;
@@ -1995,7 +2197,7 @@ bool ARMConstantIslands::reorderThumb2JumpTables() {
     MachineInstr *MI = T2JumpTables[i];
     const MCInstrDesc &MCID = MI->getDesc();
     unsigned NumOps = MCID.getNumOperands();
-    unsigned JTOpIdx = NumOps - (MI->isPredicable() ? 3 : 2);
+    unsigned JTOpIdx = NumOps - (MI->isPredicable() ? 2 : 1);
     MachineOperand JTOP = MI->getOperand(JTOpIdx);
     unsigned JTI = JTOP.getIndex();
     assert(JTI < JT.size());
diff --git a/lib/Target/ARM/ARMConstantPoolValue.h b/lib/Target/ARM/ARMConstantPoolValue.h
index de9ee9640c7..36f63e239a9 100644
--- a/lib/Target/ARM/ARMConstantPoolValue.h
+++ b/lib/Target/ARM/ARMConstantPoolValue.h
@@ -102,9 +102,6 @@ class ARMConstantPoolValue : public MachineConstantPoolValue {
   bool isBlockAddress() const { return Kind == ARMCP::CPBlockAddress; }
   bool isLSDA() const { return Kind == ARMCP::CPLSDA; }
   bool isMachineBasicBlock() const{ return Kind == ARMCP::CPMachineBasicBlock; }
-  // @LOCALMOD-START
-  bool isValue() const { return Kind == ARMCP::CPValue; }
-  // @LOCALMOD-END
 
   unsigned getRelocationInfo() const override { return 2; }
 
diff --git a/lib/Target/ARM/ARMExpandPseudoInsts.cpp b/lib/Target/ARM/ARMExpandPseudoInsts.cpp
index 773d6addbfb..f090e5abd95 100644
--- a/lib/Target/ARM/ARMExpandPseudoInsts.cpp
+++ b/lib/Target/ARM/ARMExpandPseudoInsts.cpp
@@ -28,7 +28,6 @@
 #include "llvm/Support/CommandLine.h"
 #include "llvm/Support/raw_ostream.h" // FIXME: for debug only. remove!
 #include "llvm/Target/TargetFrameLowering.h"
-#include "llvm/Target/TargetOptions.h" // @LOCALMOD for llvm::TLSUseCall
 #include "llvm/Target/TargetRegisterInfo.h"
 using namespace llvm;
 
@@ -48,7 +47,6 @@ namespace {
     const TargetRegisterInfo *TRI;
     const ARMSubtarget *STI;
     ARMFunctionInfo *AFI;
-    bool IsRelocPIC; // @LOCALMOD
 
     bool runOnMachineFunction(MachineFunction &Fn) override;
 
@@ -69,16 +67,6 @@ namespace {
                     unsigned Opc, bool IsExt);
     void ExpandMOV32BitImm(MachineBasicBlock &MBB,
                            MachineBasicBlock::iterator &MBBI);
-    // @LOCALMOD-BEGIN
-    void AddPICADD_MOVi16_PICID(MachineInstr &MI,
-                                MachineBasicBlock &MBB,
-                                MachineBasicBlock::iterator &MBBI,
-                                bool NotThumb,
-                                unsigned PredReg, ARMCC::CondCodes Pred,
-                                unsigned DstReg, bool DstIsDead,
-                                MachineInstrBuilder &LO16,
-                                MachineInstrBuilder &HI16);
-    // @LOCALMOD-END
   };
   char ARMExpandPseudo::ID = 0;
 }
@@ -505,41 +493,6 @@ void ARMExpandPseudo::ExpandVST(MachineBasicBlock::iterator &MBBI) {
   MI.eraseFromParent();
 }
 
-// @LOCALMOD-BEGIN
-// AddPICADD_MOVi16_PICID - Inserts a PICADD into the given basic block,
-// and adds the PC label ID (of the PICADD) as an operand of the LO16 / HI16
-// MOVs. The ID operand will follow the "Immediate" operand (assumes that
-// operand is already added).
-void ARMExpandPseudo::AddPICADD_MOVi16_PICID(MachineInstr &MI,
-                                       MachineBasicBlock &MBB,
-                                       MachineBasicBlock::iterator &MBBI,
-                                       bool NotThumb,
-                                       unsigned PredReg, ARMCC::CondCodes Pred,
-                                       unsigned DstReg, bool DstIsDead,
-                                       MachineInstrBuilder &LO16,
-                                       MachineInstrBuilder &HI16) {
-  // Throw in a PICADD, and tack on the PC label ID to the MOVT/MOVWs
-  MachineFunction &MF = *MI.getParent()->getParent();
-  ARMFunctionInfo *AFI = MF.getInfo<ARMFunctionInfo>();
-
-  // Make a unique ID for this PC by pulling from pool of constPoolIDs
-  unsigned PC_ID = AFI->createPICLabelUId();
-  MachineInstrBuilder PicADD =
-      BuildMI(MBB, MBBI, MI.getDebugLoc(),
-              TII->get(NotThumb ? ARM::PICADD : ARM::tPICADD))
-      .addReg(DstReg, RegState::Define | getDeadRegState(DstIsDead))
-      .addReg(DstReg)
-      .addImm(PC_ID)
-      .addImm(Pred)
-      .addReg(PredReg);
-  (void)PicADD; // squelch unused warning.
-
-  // Add the PC label ID after what would have been an absolute address.
-  LO16 = LO16.addImm(PC_ID);
-  HI16 = HI16.addImm(PC_ID);
-}
-// @LOCALMOD-END
-
 /// ExpandLaneOp - Translate VLD*LN and VST*LN instructions with Q, QQ or QQQQ
 /// register operands to real instructions with D register operands.
 void ARMExpandPseudo::ExpandLaneOp(MachineBasicBlock::iterator &MBBI) {
@@ -737,9 +690,7 @@ void ARMExpandPseudo::ExpandMOV32BitImm(MachineBasicBlock &MBB,
 
   unsigned LO16Opc = 0;
   unsigned HI16Opc = 0;
-  // @LOCALMOD
-  bool isThumb2 = (Opcode == ARM::t2MOVi32imm || Opcode == ARM::t2MOVCCi32imm);
-  if (isThumb2) {
+  if (Opcode == ARM::t2MOVi32imm || Opcode == ARM::t2MOVCCi32imm) {
     LO16Opc = ARM::t2MOVi16;
     HI16Opc = ARM::t2MOVTi16;
   } else {
@@ -747,28 +698,10 @@ void ARMExpandPseudo::ExpandMOV32BitImm(MachineBasicBlock &MBB,
     HI16Opc = ARM::MOVTi16;
   }
 
-  // @LOCALMOD-BEGIN
-  // If constant pools are "disabled" (actually, moved to rodata), then
-  // many addresses (e.g., the addresses of what used to be the "pools")
-  // may not be materialized in a pc-relative manner, because MOVT / MOVW
-  // are used to materialize the addresses.
-  // We need to know if it matters that references are pc-relative
-  // (e.g., to be PIC).
-  // See the comments on MOVi16PIC / MOVTi16PIC for more details.
-  const bool ShouldUseMOV16PIC = !STI->useConstIslands() && IsRelocPIC &&
-      (MO.isCPI() || MO.isJTI() || MO.isGlobal()); // TODO check this list.
-  if (ShouldUseMOV16PIC) {
-    if (isThumb2)
-      llvm_unreachable("FIXME: add PIC versions of t2MOVi16");
-    LO16Opc = ARM::MOVi16PIC;
-    HI16Opc = ARM::MOVTi16PIC;
-  }
-  // @LOCALMOD-END
-
   LO16 = BuildMI(MBB, MBBI, MI.getDebugLoc(), TII->get(LO16Opc), DstReg);
   HI16 = BuildMI(MBB, MBBI, MI.getDebugLoc(), TII->get(HI16Opc))
     .addReg(DstReg, RegState::Define | getDeadRegState(DstIsDead))
-    .addReg(DstReg, RegState::Kill); // @LOCALMOD
+    .addReg(DstReg);
 
   switch (MO.getType()) {
   case MachineOperand::MO_Immediate: {
@@ -787,35 +720,14 @@ void ARMExpandPseudo::ExpandMOV32BitImm(MachineBasicBlock &MBB,
     break;
   }
   default: {
-  if (MO.isGlobal()) { // @LOCALMOD
     const GlobalValue *GV = MO.getGlobal();
     unsigned TF = MO.getTargetFlags();
     LO16 = LO16.addGlobalAddress(GV, MO.getOffset(), TF | ARMII::MO_LO16);
     HI16 = HI16.addGlobalAddress(GV, MO.getOffset(), TF | ARMII::MO_HI16);
-  // @LOCALMOD-START - support for jumptable addresses and CPI
-  } else if (MO.isCPI()) {
-    int i = MO.getIndex();
-    unsigned TF = MO.getTargetFlags();
-    LO16 = LO16.addConstantPoolIndex(i, MO.getOffset(), TF|ARMII::MO_LO16);
-    HI16 = HI16.addConstantPoolIndex(i, MO.getOffset(), TF|ARMII::MO_HI16);
-  } else if (MO.isJTI()){
-    unsigned TF = MO.getTargetFlags();
-    LO16 = LO16.addJumpTableIndex(MO.getIndex(), TF | ARMII::MO_LO16);
-    HI16 = HI16.addJumpTableIndex(MO.getIndex(), TF | ARMII::MO_HI16);
-  } else {
-    assert (0 && "unexpected operand");
-  }
-  // @LOCALMOD-END
     break;
   }
   }
 
-  // @LOCALMOD-BEGIN
-  if (ShouldUseMOV16PIC) {
-    AddPICADD_MOVi16_PICID(MI, MBB, MBBI, !isThumb2,
-                           PredReg, Pred, DstReg, DstIsDead, LO16, HI16);
-  }
-  // @LOCALMOD-END
   LO16->setMemRefs(MI.memoperands_begin(), MI.memoperands_end());
   HI16->setMemRefs(MI.memoperands_begin(), MI.memoperands_end());
   LO16.addImm(Pred).addReg(PredReg);
@@ -835,6 +747,55 @@ bool ARMExpandPseudo::ExpandMI(MachineBasicBlock &MBB,
   switch (Opcode) {
     default:
       return false;
+
+    case ARM::TCRETURNdi:
+    case ARM::TCRETURNri: {
+      MachineBasicBlock::iterator MBBI = MBB.getLastNonDebugInstr();
+      assert(MBBI->isReturn() &&
+             "Can only insert epilog into returning blocks");
+      unsigned RetOpcode = MBBI->getOpcode();
+      DebugLoc dl = MBBI->getDebugLoc();
+      const ARMBaseInstrInfo &TII = *static_cast<const ARMBaseInstrInfo *>(
+          MBB.getParent()->getSubtarget().getInstrInfo());
+
+      // Tail call return: adjust the stack pointer and jump to callee.
+      MBBI = MBB.getLastNonDebugInstr();
+      MachineOperand &JumpTarget = MBBI->getOperand(0);
+
+      // Jump to label or value in register.
+      if (RetOpcode == ARM::TCRETURNdi) {
+        unsigned TCOpcode =
+            STI->isThumb()
+                ? (STI->isTargetMachO() ? ARM::tTAILJMPd : ARM::tTAILJMPdND)
+                : ARM::TAILJMPd;
+        MachineInstrBuilder MIB = BuildMI(MBB, MBBI, dl, TII.get(TCOpcode));
+        if (JumpTarget.isGlobal())
+          MIB.addGlobalAddress(JumpTarget.getGlobal(), JumpTarget.getOffset(),
+                               JumpTarget.getTargetFlags());
+        else {
+          assert(JumpTarget.isSymbol());
+          MIB.addExternalSymbol(JumpTarget.getSymbolName(),
+                                JumpTarget.getTargetFlags());
+        }
+
+        // Add the default predicate in Thumb mode.
+        if (STI->isThumb())
+          MIB.addImm(ARMCC::AL).addReg(0);
+      } else if (RetOpcode == ARM::TCRETURNri) {
+        BuildMI(MBB, MBBI, dl,
+                TII.get(STI->isThumb() ? ARM::tTAILJMPr : ARM::TAILJMPr))
+            .addReg(JumpTarget.getReg(), RegState::Kill);
+      }
+
+      MachineInstr *NewMI = std::prev(MBBI);
+      for (unsigned i = 1, e = MBBI->getNumOperands(); i != e; ++i)
+        NewMI->addOperand(MBBI->getOperand(i));
+
+      // Delete the pseudo instruction TCRETURN.
+      MBB.erase(MBBI);
+      MBBI = NewMI;
+      return true;
+    }
     case ARM::VMOVScc:
     case ARM::VMOVDcc: {
       unsigned newOpc = Opcode == ARM::VMOVScc ? ARM::VMOVS : ARM::VMOVD;
@@ -1018,8 +979,6 @@ bool ARMExpandPseudo::ExpandMI(MachineBasicBlock &MBB,
     }
     case ARM::tTPsoft:
     case ARM::TPsoft: {
-      // @LOCALMOD-BEGIN
-      if (!STI->isTargetNaCl() || llvm::TLSUseCall) {
       MachineInstrBuilder MIB;
       if (Opcode == ARM::tTPsoft)
         MIB = BuildMI(MBB, MBBI, MI.getDebugLoc(),
@@ -1027,24 +986,12 @@ bool ARMExpandPseudo::ExpandMI(MachineBasicBlock &MBB,
               .addImm((unsigned)ARMCC::AL).addReg(0)
               .addExternalSymbol("__aeabi_read_tp", 0);
       else
-        MIB = BuildMI_NoImp(MBB, MBBI, MI.getDebugLoc(), // @LOCALMOD
+        MIB = BuildMI(MBB, MBBI, MI.getDebugLoc(),
                       TII->get( ARM::BL))
               .addExternalSymbol("__aeabi_read_tp", 0);
 
       MIB->setMemRefs(MI.memoperands_begin(), MI.memoperands_end());
       TransferImpOps(MI, MIB, MIB);
-      } else {
-        // Inline version for native client.
-        // See native_client/src/untrusted/nacl/aeabi_read_tp.S
-        // .nexe builds use this version, while irt builds use a call to
-        // __aeabi_read_tp.
-        // ldr r0, [r9, #0]
-        AddDefaultPred(BuildMI(MBB, MBBI, MI.getDebugLoc(),
-                               TII->get(ARM::LDRi12), ARM::R0)
-                       .addReg(ARM::R9)
-                       .addImm(0));
-      }
-      // @LOCALMOD-END
       MI.eraseFromParent();
       return true;
     }
@@ -1165,7 +1112,6 @@ bool ARMExpandPseudo::ExpandMI(MachineBasicBlock &MBB,
     }
 
     case ARM::MOVi32imm:
-    case ARM::MOVti32imm: // @LOCALMOD
     case ARM::MOVCCi32imm:
     case ARM::t2MOVi32imm:
     case ARM::t2MOVCCi32imm:
@@ -1432,62 +1378,6 @@ bool ARMExpandPseudo::ExpandMI(MachineBasicBlock &MBB,
     case ARM::VTBL4Pseudo: ExpandVTBL(MBBI, ARM::VTBL4, false); return true;
     case ARM::VTBX3Pseudo: ExpandVTBL(MBBI, ARM::VTBX3, true); return true;
     case ARM::VTBX4Pseudo: ExpandVTBL(MBBI, ARM::VTBX4, true); return true;
-
-    // @LOCALMOD-BEGIN
-    case ARM::ARMeh_return: {
-      // This pseudo instruction is generated as part of the lowering of
-      // ISD::EH_RETURN (c.f. ARMISelLowering.cpp)
-      // we convert it to a stack increment by OffsetReg and
-      // indirect jump to TargetReg
-      unsigned PredReg = 0;
-      ARMCC::CondCodes Pred = llvm::getInstrPredicate(&MI, PredReg);
-      unsigned OffsetReg = MI.getOperand(0).getReg();
-      unsigned TargetReg = MI.getOperand(1).getReg();
-      BuildMI(MBB, MBBI, MI.getDebugLoc(), TII->get(ARM::ADDrr), ARM::SP)
-          .addReg(OffsetReg)
-          .addReg(ARM::SP)
-          .addImm(Pred)
-          .addReg(PredReg)
-          .addReg(0);
-
-      BuildMI(MBB, MBBI, MI.getDebugLoc(), TII->get(ARM::BX))
-          .addReg(TargetReg);
-      MI.eraseFromParent();
-      return true;
-    }
-    case ARM::MOVGOTAddr : {
-      // Expand the pseudo-inst that requests for the GOT address
-      // to be materialized into a register. We use MOVW/MOVT for this.
-      // See ARMISelLowering.cpp for a comment on the strategy.
-      unsigned PredReg = 0;
-      ARMCC::CondCodes Pred = llvm::getInstrPredicate(&MI, PredReg);
-      unsigned DstReg = MI.getOperand(0).getReg();
-      bool DstIsDead = MI.getOperand(0).isDead();
-      MachineInstrBuilder LO16, HI16;
-
-      LO16 = BuildMI(MBB, MBBI, MI.getDebugLoc(),
-                     TII->get(ARM::MOVi16PIC),
-                     DstReg)
-        .addExternalSymbol("_GLOBAL_OFFSET_TABLE_", ARMII::MO_LO16);
-
-      HI16 = BuildMI(MBB, MBBI, MI.getDebugLoc(),
-                     TII->get(ARM::MOVTi16PIC))
-        .addReg(DstReg, RegState::Define | getDeadRegState(DstIsDead))
-        .addReg(DstReg)
-        .addExternalSymbol("_GLOBAL_OFFSET_TABLE_", ARMII::MO_HI16);
-
-      AddPICADD_MOVi16_PICID(MI, MBB, MBBI, true,
-                             PredReg, Pred, DstReg, DstIsDead, LO16, HI16);
-
-      (*LO16).setMemRefs(MI.memoperands_begin(), MI.memoperands_end());
-      (*HI16).setMemRefs(MI.memoperands_begin(), MI.memoperands_end());
-      LO16.addImm(Pred).addReg(PredReg);
-      HI16.addImm(Pred).addReg(PredReg);
-      TransferImpOps(MI, LO16, HI16);
-      MI.eraseFromParent();
-      return true;
-    }
-    // @LOCALMOD-END
   }
 }
 
@@ -1509,7 +1399,6 @@ bool ARMExpandPseudo::runOnMachineFunction(MachineFunction &MF) {
   TII = STI->getInstrInfo();
   TRI = STI->getRegisterInfo();
   AFI = MF.getInfo<ARMFunctionInfo>();
-  IsRelocPIC = MF.getTarget().getRelocationModel() == Reloc::PIC_;
 
   bool Modified = false;
   for (MachineFunction::iterator MFI = MF.begin(), E = MF.end(); MFI != E;
diff --git a/lib/Target/ARM/ARMFPUName.def b/lib/Target/ARM/ARMFPUName.def
deleted file mode 100644
index 34ce85d280e..00000000000
--- a/lib/Target/ARM/ARMFPUName.def
+++ /dev/null
@@ -1,34 +0,0 @@
-//===-- ARMFPUName.def - List of the ARM FPU names --------------*- C++ -*-===//
-//
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
-//
-//===----------------------------------------------------------------------===//
-//
-// This file contains the list of the supported ARM FPU names.
-//
-//===----------------------------------------------------------------------===//
-
-// NOTE: NO INCLUDE GUARD DESIRED!
-
-#ifndef ARM_FPU_NAME
-#error "You must define ARM_FPU_NAME(NAME, ID) before including ARMFPUName.h"
-#endif
-
-ARM_FPU_NAME("vfp", VFP)
-ARM_FPU_NAME("vfpv2", VFPV2)
-ARM_FPU_NAME("vfpv3", VFPV3)
-ARM_FPU_NAME("vfpv3-d16", VFPV3_D16)
-ARM_FPU_NAME("vfpv4", VFPV4)
-ARM_FPU_NAME("vfpv4-d16", VFPV4_D16)
-ARM_FPU_NAME("fpv5-d16", FPV5_D16)
-ARM_FPU_NAME("fp-armv8", FP_ARMV8)
-ARM_FPU_NAME("neon", NEON)
-ARM_FPU_NAME("neon-vfpv4", NEON_VFPV4)
-ARM_FPU_NAME("neon-fp-armv8", NEON_FP_ARMV8)
-ARM_FPU_NAME("crypto-neon-fp-armv8", CRYPTO_NEON_FP_ARMV8)
-ARM_FPU_NAME("softvfp", SOFTVFP)
-
-#undef ARM_FPU_NAME
diff --git a/lib/Target/ARM/ARMFPUName.h b/lib/Target/ARM/ARMFPUName.h
deleted file mode 100644
index 86acffbc8f7..00000000000
--- a/lib/Target/ARM/ARMFPUName.h
+++ /dev/null
@@ -1,26 +0,0 @@
-//===-- ARMFPUName.h - List of the ARM FPU names ----------------*- C++ -*-===//
-//
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
-//
-//===----------------------------------------------------------------------===//
-
-#ifndef LLVM_LIB_TARGET_ARM_ARMFPUNAME_H
-#define LLVM_LIB_TARGET_ARM_ARMFPUNAME_H
-
-namespace llvm {
-namespace ARM {
-
-enum FPUKind {
-  INVALID_FPU = 0
-
-#define ARM_FPU_NAME(NAME, ID) , ID
-#include "ARMFPUName.def"
-};
-
-} // namespace ARM
-} // namespace llvm
-
-#endif
diff --git a/lib/Target/ARM/ARMFastISel.cpp b/lib/Target/ARM/ARMFastISel.cpp
index 9ce40ce816a..fdd0763ea60 100644
--- a/lib/Target/ARM/ARMFastISel.cpp
+++ b/lib/Target/ARM/ARMFastISel.cpp
@@ -49,8 +49,6 @@
 #include "llvm/Target/TargetOptions.h"
 using namespace llvm;
 
-extern cl::opt<bool> EnableARMLongCalls;
-
 namespace {
 
   // All possible address modes, plus some.
@@ -490,11 +488,6 @@ unsigned ARMFastISel::ARMMaterializeFP(const ConstantFP *CFP, MVT VT) {
   // Require VFP2 for loading fp constants.
   if (!Subtarget->hasVFP2()) return false;
 
-  // @LOCALMOD-START
-  if (!Subtarget->useConstIslands())
-    return false;
-  // @LOCALMOD-END
-
   // MachineConstantPool wants an explicit alignment.
   unsigned Align = DL.getPrefTypeAlignment(CFP->getType());
   if (Align == 0) {
@@ -556,11 +549,6 @@ unsigned ARMFastISel::ARMMaterializeInt(const Constant *C, MVT VT) {
   if (ResultReg)
     return ResultReg;
 
-  // @LOCALMOD-START
-  // Should have been handled by fastEmit_i above.
-  assert(Subtarget->useConstIslands());
-  // @LOCALMOD-END
-
   // Load from constant pool.  For now 32-bit only.
   if (VT != MVT::i32)
     return 0;
@@ -623,11 +611,6 @@ unsigned ARMFastISel::ARMMaterializeGV(const GlobalValue *GV, MVT VT) {
     AddOptionalDefs(BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc,
                             TII.get(Opc), DestReg).addGlobalAddress(GV, 0, TF));
   } else {
-    // @LOCALMOD-BEGIN
-    if (!Subtarget->useConstIslands())
-      return 0;
-    // @LOCALMOD-END
-
     // MachineConstantPool wants an explicit alignment.
     unsigned Align = DL.getPrefTypeAlignment(GV->getType());
     if (Align == 0) {
@@ -700,7 +683,7 @@ unsigned ARMFastISel::ARMMaterializeGV(const GlobalValue *GV, MVT VT) {
 }
 
 unsigned ARMFastISel::fastMaterializeConstant(const Constant *C) {
-  EVT CEVT = TLI.getValueType(C->getType(), true);
+  EVT CEVT = TLI.getValueType(DL, C->getType(), true);
 
   // Only handle simple types.
   if (!CEVT.isSimple()) return 0;
@@ -747,7 +730,7 @@ unsigned ARMFastISel::fastMaterializeAlloca(const AllocaInst *AI) {
 }
 
 bool ARMFastISel::isTypeLegal(Type *Ty, MVT &VT) {
-  EVT evt = TLI.getValueType(Ty, true);
+  EVT evt = TLI.getValueType(DL, Ty, true);
 
   // Only handle simple types.
   if (evt == MVT::Other || !evt.isSimple()) return false;
@@ -801,12 +784,13 @@ bool ARMFastISel::ARMComputeAddress(const Value *Obj, Address &Addr) {
       return ARMComputeAddress(U->getOperand(0), Addr);
     case Instruction::IntToPtr:
       // Look past no-op inttoptrs.
-      if (TLI.getValueType(U->getOperand(0)->getType()) == TLI.getPointerTy())
+      if (TLI.getValueType(DL, U->getOperand(0)->getType()) ==
+          TLI.getPointerTy(DL))
         return ARMComputeAddress(U->getOperand(0), Addr);
       break;
     case Instruction::PtrToInt:
       // Look past no-op ptrtoints.
-      if (TLI.getValueType(U->getType()) == TLI.getPointerTy())
+      if (TLI.getValueType(DL, U->getType()) == TLI.getPointerTy(DL))
         return ARMComputeAddress(U->getOperand(0), Addr);
       break;
     case Instruction::GetElementPtr: {
@@ -1380,7 +1364,7 @@ bool ARMFastISel::SelectIndirectBr(const Instruction *I) {
 bool ARMFastISel::ARMEmitCmp(const Value *Src1Value, const Value *Src2Value,
                              bool isZExt) {
   Type *Ty = Src1Value->getType();
-  EVT SrcEVT = TLI.getValueType(Ty, true);
+  EVT SrcEVT = TLI.getValueType(DL, Ty, true);
   if (!SrcEVT.isSimple()) return false;
   MVT SrcVT = SrcEVT.getSimpleVT();
 
@@ -1572,7 +1556,7 @@ bool ARMFastISel::SelectIToFP(const Instruction *I, bool isSigned) {
     return false;
 
   Value *Src = I->getOperand(0);
-  EVT SrcEVT = TLI.getValueType(Src->getType(), true);
+  EVT SrcEVT = TLI.getValueType(DL, Src->getType(), true);
   if (!SrcEVT.isSimple())
     return false;
   MVT SrcVT = SrcEVT.getSimpleVT();
@@ -1765,7 +1749,7 @@ bool ARMFastISel::SelectRem(const Instruction *I, bool isSigned) {
 }
 
 bool ARMFastISel::SelectBinaryIntOp(const Instruction *I, unsigned ISDOpcode) {
-  EVT DestVT  = TLI.getValueType(I->getType(), true);
+  EVT DestVT = TLI.getValueType(DL, I->getType(), true);
 
   // We can get here in the case when we have a binary operation on a non-legal
   // type and the target independent selector doesn't know how to handle it.
@@ -1805,10 +1789,14 @@ bool ARMFastISel::SelectBinaryIntOp(const Instruction *I, unsigned ISDOpcode) {
 }
 
 bool ARMFastISel::SelectBinaryFPOp(const Instruction *I, unsigned ISDOpcode) {
-  EVT FPVT = TLI.getValueType(I->getType(), true);
+  EVT FPVT = TLI.getValueType(DL, I->getType(), true);
   if (!FPVT.isSimple()) return false;
   MVT VT = FPVT.getSimpleVT();
 
+  // FIXME: Support vector types where possible.
+  if (VT.isVector())
+    return false;
+
   // We can get here in the case when we want to use NEON for our fp
   // operations, but can't figure out how to. Just use the vfp instructions
   // if we have them.
@@ -2106,7 +2094,7 @@ bool ARMFastISel::SelectRet(const Instruction *I) {
   CallingConv::ID CC = F.getCallingConv();
   if (Ret->getNumOperands() > 0) {
     SmallVector<ISD::OutputArg, 4> Outs;
-    GetReturnInfo(F.getReturnType(), F.getAttributes(), Outs, TLI);
+    GetReturnInfo(F.getReturnType(), F.getAttributes(), Outs, TLI, DL);
 
     // Analyze operands of the call, assigning locations to each operand.
     SmallVector<CCValAssign, 16> ValLocs;
@@ -2133,7 +2121,7 @@ bool ARMFastISel::SelectRet(const Instruction *I) {
       return false;
 
     unsigned SrcReg = Reg + VA.getValNo();
-    EVT RVEVT = TLI.getValueType(RV->getType());
+    EVT RVEVT = TLI.getValueType(DL, RV->getType());
     if (!RVEVT.isSimple()) return false;
     MVT RVVT = RVEVT.getSimpleVT();
     MVT DestVT = VA.getValVT();
@@ -2184,7 +2172,7 @@ unsigned ARMFastISel::ARMSelectCallOp(bool UseReg) {
 unsigned ARMFastISel::getLibcallReg(const Twine &Name) {
   // Manually compute the global's type to avoid building it when unnecessary.
   Type *GVTy = Type::getInt32PtrTy(*Context, /*AS=*/0);
-  EVT LCREVT = TLI.getValueType(GVTy);
+  EVT LCREVT = TLI.getValueType(DL, GVTy);
   if (!LCREVT.isSimple()) return 0;
 
   GlobalValue *GV = new GlobalVariable(M, Type::getInt32Ty(*Context), false,
@@ -2257,19 +2245,19 @@ bool ARMFastISel::ARMEmitLibcall(const Instruction *I, RTLIB::Libcall Call) {
     return false;
 
   unsigned CalleeReg = 0;
-  if (EnableARMLongCalls) {
+  if (Subtarget->genLongCalls()) {
     CalleeReg = getLibcallReg(TLI.getLibcallName(Call));
     if (CalleeReg == 0) return false;
   }
 
   // Issue the call.
-  unsigned CallOpc = ARMSelectCallOp(EnableARMLongCalls);
+  unsigned CallOpc = ARMSelectCallOp(Subtarget->genLongCalls());
   MachineInstrBuilder MIB = BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt,
                                     DbgLoc, TII.get(CallOpc));
   // BL / BLX don't take a predicate, but tBL / tBLX do.
   if (isThumb2)
     AddDefaultPred(MIB);
-  if (EnableARMLongCalls)
+  if (Subtarget->genLongCalls())
     MIB.addReg(CalleeReg);
   else
     MIB.addExternalSymbol(TLI.getLibcallName(Call));
@@ -2391,7 +2379,7 @@ bool ARMFastISel::SelectCall(const Instruction *I,
 
   bool UseReg = false;
   const GlobalValue *GV = dyn_cast<GlobalValue>(Callee);
-  if (!GV || EnableARMLongCalls) UseReg = true;
+  if (!GV || Subtarget->genLongCalls()) UseReg = true;
 
   unsigned CalleeReg = 0;
   if (UseReg) {
@@ -2587,8 +2575,8 @@ bool ARMFastISel::SelectTrunc(const Instruction *I) {
   Value *Op = I->getOperand(0);
 
   EVT SrcVT, DestVT;
-  SrcVT = TLI.getValueType(Op->getType(), true);
-  DestVT = TLI.getValueType(I->getType(), true);
+  SrcVT = TLI.getValueType(DL, Op->getType(), true);
+  DestVT = TLI.getValueType(DL, I->getType(), true);
 
   if (SrcVT != MVT::i32 && SrcVT != MVT::i16 && SrcVT != MVT::i8)
     return false;
@@ -2753,8 +2741,8 @@ bool ARMFastISel::SelectIntExt(const Instruction *I) {
   if (!SrcReg) return false;
 
   EVT SrcEVT, DestEVT;
-  SrcEVT = TLI.getValueType(SrcTy, true);
-  DestEVT = TLI.getValueType(DestTy, true);
+  SrcEVT = TLI.getValueType(DL, SrcTy, true);
+  DestEVT = TLI.getValueType(DL, DestTy, true);
   if (!SrcEVT.isSimple()) return false;
   if (!DestEVT.isSimple()) return false;
 
@@ -2774,7 +2762,7 @@ bool ARMFastISel::SelectShift(const Instruction *I,
     return false;
 
   // Only handle i32 now.
-  EVT DestVT = TLI.getValueType(I->getType(), true);
+  EVT DestVT = TLI.getValueType(DL, I->getType(), true);
   if (DestVT != MVT::i32)
     return false;
 
@@ -3037,7 +3025,7 @@ bool ARMFastISel::fastLowerArguments() {
     if (ArgTy->isStructTy() || ArgTy->isArrayTy() || ArgTy->isVectorTy())
       return false;
 
-    EVT ArgVT = TLI.getValueType(ArgTy);
+    EVT ArgVT = TLI.getValueType(DL, ArgTy);
     if (!ArgVT.isSimple()) return false;
     switch (ArgVT.getSimpleVT().SimpleTy) {
     case MVT::i8:
@@ -3076,23 +3064,9 @@ bool ARMFastISel::fastLowerArguments() {
 namespace llvm {
   FastISel *ARM::createFastISel(FunctionLoweringInfo &funcInfo,
                                 const TargetLibraryInfo *libInfo) {
-    const TargetMachine &TM = funcInfo.MF->getTarget();
-    const ARMSubtarget &STI =
-        static_cast<const ARMSubtarget &>(funcInfo.MF->getSubtarget());
-    // Thumb2 support on iOS; ARM support on iOS, Linux and NaCl.
-    bool UseFastISel = false;
-    UseFastISel |= STI.isTargetMachO() && !STI.isThumb1Only();
-    UseFastISel |= STI.isTargetLinux() && !STI.isThumb();
-    UseFastISel |= STI.isTargetNaCl() && !STI.isThumb();
-
-    if (UseFastISel) {
-      // iOS always has a FP for backtracking, force other targets
-      // to keep their FP when doing FastISel. The emitted code is
-      // currently superior, and in cases like test-suite's lencod
-      // FastISel isn't quite correct when FP is eliminated.
-      TM.Options.NoFramePointerElim = true;
+    if (funcInfo.MF->getSubtarget<ARMSubtarget>().useFastISel())
       return new ARMFastISel(funcInfo, libInfo);
-    }
+
     return nullptr;
   }
 }
diff --git a/lib/Target/ARM/ARMFrameLowering.cpp b/lib/Target/ARM/ARMFrameLowering.cpp
index 8895dfa5c8e..ba0d5f25b09 100644
--- a/lib/Target/ARM/ARMFrameLowering.cpp
+++ b/lib/Target/ARM/ARMFrameLowering.cpp
@@ -43,6 +43,14 @@ ARMFrameLowering::ARMFrameLowering(const ARMSubtarget &sti)
     : TargetFrameLowering(StackGrowsDown, sti.getStackAlignment(), 0, 4),
       STI(sti) {}
 
+bool ARMFrameLowering::noFramePointerElim(const MachineFunction &MF) const {
+  // iOS always has a FP for backtracking, force other targets to keep their FP
+  // when doing FastISel. The emitted code is currently superior, and in cases
+  // like test-suite's lencod FastISel isn't quite correct when FP is eliminated.
+  return TargetFrameLowering::noFramePointerElim(MF) ||
+         MF.getSubtarget<ARMSubtarget>().useFastISel();
+}
+
 /// hasFP - Return true if the specified function should have a dedicated frame
 /// pointer register.  This is true if the function has variable sized allocas
 /// or if frame pointer elimination is disabled.
@@ -278,8 +286,8 @@ static void emitAligningInstructions(MachineFunction &MF, ARMFunctionInfo *AFI,
   }
 }
 
-void ARMFrameLowering::emitPrologue(MachineFunction &MF) const {
-  MachineBasicBlock &MBB = MF.front();
+void ARMFrameLowering::emitPrologue(MachineFunction &MF,
+                                    MachineBasicBlock &MBB) const {
   MachineBasicBlock::iterator MBBI = MBB.begin();
   MachineFrameInfo  *MFI = MF.getFrameInfo();
   ARMFunctionInfo *AFI = MF.getInfo<ARMFunctionInfo>();
@@ -680,60 +688,8 @@ void ARMFrameLowering::emitPrologue(MachineFunction &MF) const {
     AFI->setShouldRestoreSPFromFP(true);
 }
 
-// Resolve TCReturn pseudo-instruction
-void ARMFrameLowering::fixTCReturn(MachineFunction &MF,
-                                   MachineBasicBlock &MBB) const {
-  MachineBasicBlock::iterator MBBI = MBB.getLastNonDebugInstr();
-  assert(MBBI->isReturn() && "Can only insert epilog into returning blocks");
-  unsigned RetOpcode = MBBI->getOpcode();
-  DebugLoc dl = MBBI->getDebugLoc();
-  const ARMBaseInstrInfo &TII =
-      *static_cast<const ARMBaseInstrInfo *>(MF.getSubtarget().getInstrInfo());
-
-  if (!(RetOpcode == ARM::TCRETURNdi || RetOpcode == ARM::TCRETURNri))
-    return;
-
-  // Tail call return: adjust the stack pointer and jump to callee.
-  MBBI = MBB.getLastNonDebugInstr();
-  MachineOperand &JumpTarget = MBBI->getOperand(0);
-
-  // Jump to label or value in register.
-  if (RetOpcode == ARM::TCRETURNdi) {
-    unsigned TCOpcode = STI.isThumb() ?
-             (STI.isTargetMachO() ? ARM::tTAILJMPd : ARM::tTAILJMPdND) :
-             ARM::TAILJMPd;
-    MachineInstrBuilder MIB = BuildMI(MBB, MBBI, dl, TII.get(TCOpcode));
-    if (JumpTarget.isGlobal())
-      MIB.addGlobalAddress(JumpTarget.getGlobal(), JumpTarget.getOffset(),
-                           JumpTarget.getTargetFlags());
-    else {
-      assert(JumpTarget.isSymbol());
-      MIB.addExternalSymbol(JumpTarget.getSymbolName(),
-                            JumpTarget.getTargetFlags());
-    }
-
-    // Add the default predicate in Thumb mode.
-    if (STI.isThumb()) MIB.addImm(ARMCC::AL).addReg(0);
-  } else if (RetOpcode == ARM::TCRETURNri) {
-    BuildMI(MBB, MBBI, dl,
-            TII.get(STI.isThumb() ? ARM::tTAILJMPr : ARM::TAILJMPr)).
-      addReg(JumpTarget.getReg(), RegState::Kill);
-  }
-
-  MachineInstr *NewMI = std::prev(MBBI);
-  for (unsigned i = 1, e = MBBI->getNumOperands(); i != e; ++i)
-    NewMI->addOperand(MBBI->getOperand(i));
-
-  // Delete the pseudo instruction TCRETURN.
-  MBB.erase(MBBI);
-  MBBI = NewMI;
-}
-
 void ARMFrameLowering::emitEpilogue(MachineFunction &MF,
                                     MachineBasicBlock &MBB) const {
-  MachineBasicBlock::iterator MBBI = MBB.getLastNonDebugInstr();
-  assert(MBBI->isReturn() && "Can only insert epilog into returning blocks");
-  DebugLoc dl = MBBI->getDebugLoc();
   MachineFrameInfo *MFI = MF.getFrameInfo();
   ARMFunctionInfo *AFI = MF.getInfo<ARMFunctionInfo>();
   const TargetRegisterInfo *RegInfo = MF.getSubtarget().getRegisterInfo();
@@ -749,10 +705,12 @@ void ARMFrameLowering::emitEpilogue(MachineFunction &MF,
 
   // All calls are tail calls in GHC calling conv, and functions have no
   // prologue/epilogue.
-  if (MF.getFunction()->getCallingConv() == CallingConv::GHC) {
-    fixTCReturn(MF, MBB);
+  if (MF.getFunction()->getCallingConv() == CallingConv::GHC)
     return;
-  }
+
+  // First put ourselves on the first (from top) terminator instructions.
+  MachineBasicBlock::iterator MBBI = MBB.getFirstTerminator();
+  DebugLoc dl = MBBI != MBB.end() ? MBBI->getDebugLoc() : DebugLoc();
 
   if (!AFI->hasStackFrame()) {
     if (NumBytes - ArgRegsSaveSize != 0)
@@ -791,7 +749,7 @@ void ARMFrameLowering::emitEpilogue(MachineFunction &MF,
           // This is bad, if an interrupt is taken after the mov, sp is in an
           // inconsistent state.
           // Use the first callee-saved register as a scratch register.
-          assert(MF.getRegInfo().isPhysRegUsed(ARM::R4) &&
+          assert(!MFI->getPristineRegs(MF).test(ARM::R4) &&
                  "No scratch register to restore SP from FP!");
           emitT2RegPlusImmediate(MBB, MBBI, dl, ARM::R4, FramePtr, -NumBytes,
                                  ARMCC::AL, 0, TII);
@@ -831,8 +789,6 @@ void ARMFrameLowering::emitEpilogue(MachineFunction &MF,
     if (AFI->getGPRCalleeSavedArea1Size()) MBBI++;
   }
 
-  fixTCReturn(MF, MBB);
-
   if (ArgRegsSaveSize)
     emitSPUpdate(isARM, MBB, MBBI, dl, TII, ArgRegsSaveSize);
 }
@@ -999,7 +955,8 @@ void ARMFrameLowering::emitPushInst(MachineBasicBlock &MBB,
     // Put any subsequent vpush instructions before this one: they will refer to
     // higher register numbers so need to be pushed first in order to preserve
     // monotonicity.
-    --MI;
+    if (MI != MBB.begin())
+      --MI;
   }
 }
 
@@ -1013,12 +970,16 @@ void ARMFrameLowering::emitPopInst(MachineBasicBlock &MBB,
   MachineFunction &MF = *MBB.getParent();
   const TargetInstrInfo &TII = *MF.getSubtarget().getInstrInfo();
   ARMFunctionInfo *AFI = MF.getInfo<ARMFunctionInfo>();
-  DebugLoc DL = MI->getDebugLoc();
-  unsigned RetOpcode = MI->getOpcode();
-  bool isTailCall = (RetOpcode == ARM::TCRETURNdi ||
-                     RetOpcode == ARM::TCRETURNri);
-  bool isInterrupt =
-      RetOpcode == ARM::SUBS_PC_LR || RetOpcode == ARM::t2SUBS_PC_LR;
+  DebugLoc DL;
+  bool isTailCall = false;
+  bool isInterrupt = false;
+  if (MBB.end() != MI) {
+    DL = MI->getDebugLoc();
+    unsigned RetOpcode = MI->getOpcode();
+    isTailCall = (RetOpcode == ARM::TCRETURNdi || RetOpcode == ARM::TCRETURNri);
+    isInterrupt =
+        RetOpcode == ARM::SUBS_PC_LR || RetOpcode == ARM::t2SUBS_PC_LR;
+  }
 
   SmallVector<unsigned, 4> Regs;
   unsigned i = CSI.size();
@@ -1034,12 +995,14 @@ void ARMFrameLowering::emitPopInst(MachineBasicBlock &MBB,
         continue;
 
       if (Reg == ARM::LR && !isTailCall && !isVarArg && !isInterrupt &&
-          STI.hasV5TOps() &&
-          !STI.isTargetNaCl() /* @LOCALMOD */) {
-        Reg = ARM::PC;
-        LdmOpc = AFI->isThumbFunction() ? ARM::t2LDMIA_RET : ARM::LDMIA_RET;
+          STI.hasV5TOps()) {
+        if (MBB.succ_empty()) {
+          Reg = ARM::PC;
+          DeleteRet = true;
+          LdmOpc = AFI->isThumbFunction() ? ARM::t2LDMIA_RET : ARM::LDMIA_RET;
+        } else
+          LdmOpc = AFI->isThumbFunction() ? ARM::t2LDMIA_UPD : ARM::LDMIA_UPD;
         // Fold the return instruction into the LDM.
-        DeleteRet = true;
       }
 
       // If NoGap is true, pop consecutive registers and then leave the rest
@@ -1060,7 +1023,7 @@ void ARMFrameLowering::emitPopInst(MachineBasicBlock &MBB,
                        .addReg(ARM::SP));
       for (unsigned i = 0, e = Regs.size(); i < e; ++i)
         MIB.addReg(Regs[i], getDefRegState(true));
-      if (DeleteRet) {
+      if (DeleteRet && MI != MBB.end()) {
         MIB.copyImplicitOps(&*MI);
         MI->eraseFromParent();
       }
@@ -1087,7 +1050,8 @@ void ARMFrameLowering::emitPopInst(MachineBasicBlock &MBB,
 
     // Put any subsequent vpop instructions after this one: they will refer to
     // higher register numbers so need to be popped afterwards.
-    ++MI;
+    if (MI != MBB.end())
+      ++MI;
   }
 }
 
@@ -1462,7 +1426,8 @@ static unsigned estimateRSStackSizeLimit(MachineFunction &MF,
 // callee-saved vector registers after realigning the stack. The vst1 and vld1
 // instructions take alignment hints that can improve performance.
 //
-static void checkNumAlignedDPRCS2Regs(MachineFunction &MF) {
+static void
+checkNumAlignedDPRCS2Regs(MachineFunction &MF, BitVector &SavedRegs) {
   MF.getInfo<ARMFunctionInfo>()->setNumAlignedDPRCS2Regs(0);
   if (!SpillAlignedNEONRegs)
     return;
@@ -1489,10 +1454,9 @@ static void checkNumAlignedDPRCS2Regs(MachineFunction &MF) {
   // callee-saved registers in order, but it can happen that there are holes in
   // the range.  Registers above the hole will be spilled to the standard DPRCS
   // area.
-  MachineRegisterInfo &MRI = MF.getRegInfo();
   unsigned NumSpills = 0;
   for (; NumSpills < 8; ++NumSpills)
-    if (!MRI.isPhysRegUsed(ARM::D8 + NumSpills))
+    if (!SavedRegs.test(ARM::D8 + NumSpills))
       break;
 
   // Don't do this for just one d-register. It's not worth it.
@@ -1503,12 +1467,13 @@ static void checkNumAlignedDPRCS2Regs(MachineFunction &MF) {
   MF.getInfo<ARMFunctionInfo>()->setNumAlignedDPRCS2Regs(NumSpills);
 
   // A scratch register is required for the vst1 / vld1 instructions.
-  MF.getRegInfo().setPhysRegUsed(ARM::R4);
+  SavedRegs.set(ARM::R4);
 }
 
-void
-ARMFrameLowering::processFunctionBeforeCalleeSavedScan(MachineFunction &MF,
-                                                       RegScavenger *RS) const {
+void ARMFrameLowering::determineCalleeSaves(MachineFunction &MF,
+                                            BitVector &SavedRegs,
+                                            RegScavenger *RS) const {
+  TargetFrameLowering::determineCalleeSaves(MF, SavedRegs, RS);
   // This tells PEI to spill the FP as if it is any other callee-save register
   // to take advantage the eliminateFrameIndex machinery. This also ensures it
   // is spilled in the order specified by getCalleeSavedRegs() to make it easier
@@ -1535,12 +1500,12 @@ ARMFrameLowering::processFunctionBeforeCalleeSavedScan(MachineFunction &MF,
   // FIXME: It will be better just to find spare register here.
   if (AFI->isThumb2Function() &&
       (MFI->hasVarSizedObjects() || RegInfo->needsStackRealignment(MF)))
-    MRI.setPhysRegUsed(ARM::R4);
+    SavedRegs.set(ARM::R4);
 
   if (AFI->isThumb1OnlyFunction()) {
     // Spill LR if Thumb1 function uses variable length argument lists.
     if (AFI->getArgRegsSaveSize() > 0)
-      MRI.setPhysRegUsed(ARM::LR);
+      SavedRegs.set(ARM::LR);
 
     // Spill R4 if Thumb1 epilogue has to restore SP from FP. We don't know
     // for sure what the stack size will be, but for this, an estimate is good
@@ -1550,23 +1515,23 @@ ARMFrameLowering::processFunctionBeforeCalleeSavedScan(MachineFunction &MF,
     // FIXME: It will be better just to find spare register here.
     unsigned StackSize = MFI->estimateStackSize(MF);
     if (MFI->hasVarSizedObjects() || StackSize > 508)
-      MRI.setPhysRegUsed(ARM::R4);
+      SavedRegs.set(ARM::R4);
   }
 
   // See if we can spill vector registers to aligned stack.
-  checkNumAlignedDPRCS2Regs(MF);
+  checkNumAlignedDPRCS2Regs(MF, SavedRegs);
 
   // Spill the BasePtr if it's used.
   if (RegInfo->hasBasePointer(MF))
-    MRI.setPhysRegUsed(RegInfo->getBaseRegister());
+    SavedRegs.set(RegInfo->getBaseRegister());
 
   // Don't spill FP if the frame can be eliminated. This is determined
-  // by scanning the callee-save registers to see if any is used.
+  // by scanning the callee-save registers to see if any is modified.
   const MCPhysReg *CSRegs = RegInfo->getCalleeSavedRegs(&MF);
   for (unsigned i = 0; CSRegs[i]; ++i) {
     unsigned Reg = CSRegs[i];
     bool Spilled = false;
-    if (MRI.isPhysRegUsed(Reg)) {
+    if (SavedRegs.test(Reg)) {
       Spilled = true;
       CanEliminateFrame = false;
     }
@@ -1660,7 +1625,7 @@ ARMFrameLowering::processFunctionBeforeCalleeSavedScan(MachineFunction &MF,
     // If LR is not spilled, but at least one of R4, R5, R6, and R7 is spilled.
     // Spill LR as well so we can fold BX_RET to the registers restore (LDM).
     if (!LRSpilled && CS1Spilled) {
-      MRI.setPhysRegUsed(ARM::LR);
+      SavedRegs.set(ARM::LR);
       NumGPRSpills++;
       SmallVectorImpl<unsigned>::iterator LRPos;
       LRPos = std::find(UnspilledCS1GPRs.begin(), UnspilledCS1GPRs.end(),
@@ -1673,7 +1638,7 @@ ARMFrameLowering::processFunctionBeforeCalleeSavedScan(MachineFunction &MF,
     }
 
     if (hasFP(MF)) {
-      MRI.setPhysRegUsed(FramePtr);
+      SavedRegs.set(FramePtr);
       auto FPPos = std::find(UnspilledCS1GPRs.begin(), UnspilledCS1GPRs.end(),
                              FramePtr);
       if (FPPos != UnspilledCS1GPRs.end())
@@ -1689,10 +1654,10 @@ ARMFrameLowering::processFunctionBeforeCalleeSavedScan(MachineFunction &MF,
       if (CS1Spilled && !UnspilledCS1GPRs.empty()) {
         for (unsigned i = 0, e = UnspilledCS1GPRs.size(); i != e; ++i) {
           unsigned Reg = UnspilledCS1GPRs[i];
-          // Don't spill high register if the function is thumb1
-          if (!AFI->isThumb1OnlyFunction() ||
+          // Don't spill high register if the function is thumb
+          if (!AFI->isThumbFunction() ||
               isARMLowRegister(Reg) || Reg == ARM::LR) {
-            MRI.setPhysRegUsed(Reg);
+            SavedRegs.set(Reg);
             if (!MRI.isReserved(Reg))
               ExtraCSSpill = true;
             break;
@@ -1700,7 +1665,7 @@ ARMFrameLowering::processFunctionBeforeCalleeSavedScan(MachineFunction &MF,
         }
       } else if (!UnspilledCS2GPRs.empty() && !AFI->isThumb1OnlyFunction()) {
         unsigned Reg = UnspilledCS2GPRs.front();
-        MRI.setPhysRegUsed(Reg);
+        SavedRegs.set(Reg);
         if (!MRI.isReserved(Reg))
           ExtraCSSpill = true;
       }
@@ -1739,7 +1704,7 @@ ARMFrameLowering::processFunctionBeforeCalleeSavedScan(MachineFunction &MF,
       }
       if (Extras.size() && NumExtras == 0) {
         for (unsigned i = 0, e = Extras.size(); i != e; ++i) {
-          MRI.setPhysRegUsed(Extras[i]);
+          SavedRegs.set(Extras[i]);
         }
       } else if (!AFI->isThumb1OnlyFunction()) {
         // note: Thumb1 functions spill to R12, not the stack.  Reserve a slot
@@ -1753,7 +1718,7 @@ ARMFrameLowering::processFunctionBeforeCalleeSavedScan(MachineFunction &MF,
   }
 
   if (ForceLRSpill) {
-    MRI.setPhysRegUsed(ARM::LR);
+    SavedRegs.set(ARM::LR);
     AFI->setLRIsSpilledForFarJump(true);
   }
 }
@@ -1862,7 +1827,8 @@ static const uint64_t kSplitStackAvailable = 256;
 // ARM can be found at [1].
 //
 // [1] - https://github.com/mozilla/rust/blob/86efd9/src/rt/arch/arm/morestack.S
-void ARMFrameLowering::adjustForSegmentedStacks(MachineFunction &MF) const {
+void ARMFrameLowering::adjustForSegmentedStacks(
+    MachineFunction &MF, MachineBasicBlock &PrologueMBB) const {
   unsigned Opcode;
   unsigned CFIIndex;
   const ARMSubtarget *ST = &MF.getSubtarget<ARMSubtarget>();
@@ -1875,7 +1841,6 @@ void ARMFrameLowering::adjustForSegmentedStacks(MachineFunction &MF) const {
   if (!ST->isTargetAndroid() && !ST->isTargetLinux())
     report_fatal_error("Segmented stacks not supported on this platform.");
 
-  MachineBasicBlock &prologueMBB = MF.front();
   MachineFrameInfo *MFI = MF.getFrameInfo();
   MachineModuleInfo &MMI = MF.getMMI();
   MCContext &Context = MMI.getContext();
@@ -1903,21 +1868,51 @@ void ARMFrameLowering::adjustForSegmentedStacks(MachineFunction &MF) const {
   MachineBasicBlock *GetMBB = MF.CreateMachineBasicBlock();
   MachineBasicBlock *McrMBB = MF.CreateMachineBasicBlock();
 
-  for (MachineBasicBlock::livein_iterator i = prologueMBB.livein_begin(),
-                                          e = prologueMBB.livein_end();
+  // Grab everything that reaches PrologueMBB to update there liveness as well.
+  SmallPtrSet<MachineBasicBlock *, 8> BeforePrologueRegion;
+  SmallVector<MachineBasicBlock *, 2> WalkList;
+  WalkList.push_back(&PrologueMBB);
+
+  do {
+    MachineBasicBlock *CurMBB = WalkList.pop_back_val();
+    for (MachineBasicBlock *PredBB : CurMBB->predecessors()) {
+      if (BeforePrologueRegion.insert(PredBB).second)
+        WalkList.push_back(PredBB);
+    }
+  } while (!WalkList.empty());
+
+  // The order in that list is important.
+  // The blocks will all be inserted before PrologueMBB using that order.
+  // Therefore the block that should appear first in the CFG should appear
+  // first in the list.
+  MachineBasicBlock *AddedBlocks[] = {PrevStackMBB, McrMBB, GetMBB, AllocMBB,
+                                      PostStackMBB};
+  const int NbAddedBlocks = sizeof(AddedBlocks) / sizeof(AddedBlocks[0]);
+
+  for (int Idx = 0; Idx < NbAddedBlocks; ++Idx)
+    BeforePrologueRegion.insert(AddedBlocks[Idx]);
+
+  for (MachineBasicBlock::livein_iterator i = PrologueMBB.livein_begin(),
+                                          e = PrologueMBB.livein_end();
        i != e; ++i) {
-    AllocMBB->addLiveIn(*i);
-    GetMBB->addLiveIn(*i);
-    McrMBB->addLiveIn(*i);
-    PrevStackMBB->addLiveIn(*i);
-    PostStackMBB->addLiveIn(*i);
+    for (MachineBasicBlock *PredBB : BeforePrologueRegion)
+      PredBB->addLiveIn(*i);
+  }
+
+  // Remove the newly added blocks from the list, since we know
+  // we do not have to do the following updates for them.
+  for (int Idx = 0; Idx < NbAddedBlocks; ++Idx) {
+    BeforePrologueRegion.erase(AddedBlocks[Idx]);
+    MF.insert(&PrologueMBB, AddedBlocks[Idx]);
   }
 
-  MF.push_front(PostStackMBB);
-  MF.push_front(AllocMBB);
-  MF.push_front(GetMBB);
-  MF.push_front(McrMBB);
-  MF.push_front(PrevStackMBB);
+  for (MachineBasicBlock *MBB : BeforePrologueRegion) {
+    // Make sure the LiveIns are still sorted and unique.
+    MBB->sortUniqueLiveIns();
+    // Replace the edges to PrologueMBB by edges to the sequences
+    // we are about to add.
+    MBB->ReplaceUsesOfBlockWith(&PrologueMBB, AddedBlocks[0]);
+  }
 
   // The required stack size that is aligned to ARM constant criterion.
   AlignedStackSize = alignToARMConstant(StackSize);
@@ -2157,7 +2152,7 @@ void ARMFrameLowering::adjustForSegmentedStacks(MachineFunction &MF) const {
       .addCFIIndex(CFIIndex);
 
   // Organizing MBB lists
-  PostStackMBB->addSuccessor(&prologueMBB);
+  PostStackMBB->addSuccessor(&PrologueMBB);
 
   AllocMBB->addSuccessor(PostStackMBB);
 
diff --git a/lib/Target/ARM/ARMFrameLowering.h b/lib/Target/ARM/ARMFrameLowering.h
index b7be43642ad..aabdd8dbe8f 100644
--- a/lib/Target/ARM/ARMFrameLowering.h
+++ b/lib/Target/ARM/ARMFrameLowering.h
@@ -28,11 +28,9 @@ class ARMFrameLowering : public TargetFrameLowering {
 
   /// emitProlog/emitEpilog - These methods insert prolog and epilog code into
   /// the function.
-  void emitPrologue(MachineFunction &MF) const override;
+  void emitPrologue(MachineFunction &MF, MachineBasicBlock &MBB) const override;
   void emitEpilogue(MachineFunction &MF, MachineBasicBlock &MBB) const override;
 
-  void fixTCReturn(MachineFunction &MF, MachineBasicBlock &MBB) const;
-
   bool spillCalleeSavedRegisters(MachineBasicBlock &MBB,
                                  MachineBasicBlock::iterator MI,
                                  const std::vector<CalleeSavedInfo> &CSI,
@@ -43,6 +41,8 @@ class ARMFrameLowering : public TargetFrameLowering {
                                   const std::vector<CalleeSavedInfo> &CSI,
                                   const TargetRegisterInfo *TRI) const override;
 
+  bool noFramePointerElim(const MachineFunction &MF) const override;
+
   bool hasFP(const MachineFunction &MF) const override;
   bool hasReservedCallFrame(const MachineFunction &MF) const override;
   bool canSimplifyCallFramePseudos(const MachineFunction &MF) const override;
@@ -52,10 +52,11 @@ class ARMFrameLowering : public TargetFrameLowering {
                                  unsigned &FrameReg, int SPAdj) const;
   int getFrameIndexOffset(const MachineFunction &MF, int FI) const override;
 
-  void processFunctionBeforeCalleeSavedScan(MachineFunction &MF,
-                                            RegScavenger *RS) const override;
+  void determineCalleeSaves(MachineFunction &MF, BitVector &SavedRegs,
+                            RegScavenger *RS) const override;
 
-  void adjustForSegmentedStacks(MachineFunction &MF) const override;
+  void adjustForSegmentedStacks(MachineFunction &MF,
+                                MachineBasicBlock &MBB) const override;
 
  private:
   void emitPushInst(MachineBasicBlock &MBB, MachineBasicBlock::iterator MI,
diff --git a/lib/Target/ARM/ARMHazardRecognizer.cpp b/lib/Target/ARM/ARMHazardRecognizer.cpp
index a84603bf8fd..0157c0a3528 100644
--- a/lib/Target/ARM/ARMHazardRecognizer.cpp
+++ b/lib/Target/ARM/ARMHazardRecognizer.cpp
@@ -51,8 +51,7 @@ ARMHazardRecognizer::getHazardType(SUnit *SU, int Stalls) {
       // Skip over one non-VFP / NEON instruction.
       if (!LastMI->isBarrier() &&
           // On A9, AGU and NEON/FPU are muxed.
-          !(TII.getSubtarget().isLikeA9() &&
-            (LastMI->mayLoad() || LastMI->mayStore())) &&
+          !(TII.getSubtarget().isLikeA9() && LastMI->mayLoadOrStore()) &&
           (LastMCID.TSFlags & ARMII::DomainMask) == ARMII::DomainGeneral) {
         MachineBasicBlock::iterator I = LastMI;
         if (I != LastMI->getParent()->begin()) {
diff --git a/lib/Target/ARM/ARMISelDAGToDAG.cpp b/lib/Target/ARM/ARMISelDAGToDAG.cpp
index 32f4df56293..b110628a0a8 100644
--- a/lib/Target/ARM/ARMISelDAGToDAG.cpp
+++ b/lib/Target/ARM/ARMISelDAGToDAG.cpp
@@ -15,6 +15,7 @@
 #include "ARMBaseInstrInfo.h"
 #include "ARMTargetMachine.h"
 #include "MCTargetDesc/ARMAddressingModes.h"
+#include "llvm/ADT/StringSwitch.h"
 #include "llvm/CodeGen/MachineFrameInfo.h"
 #include "llvm/CodeGen/MachineFunction.h"
 #include "llvm/CodeGen/MachineInstrBuilder.h"
@@ -83,8 +84,8 @@ class ARMDAGToDAGISel : public SelectionDAGISel {
 
   /// getI32Imm - Return a target constant of type i32 with the specified
   /// value.
-  inline SDValue getI32Imm(unsigned Imm) {
-    return CurDAG->getTargetConstant(Imm, MVT::i32);
+  inline SDValue getI32Imm(unsigned Imm, SDLoc dl) {
+    return CurDAG->getTargetConstant(Imm, dl, MVT::i32);
   }
 
   SDNode *Select(SDNode *N) override;
@@ -112,24 +113,21 @@ class ARMDAGToDAGISel : public SelectionDAGISel {
   bool SelectAddrModeImm12(SDValue N, SDValue &Base, SDValue &OffImm);
   bool SelectLdStSOReg(SDValue N, SDValue &Base, SDValue &Offset, SDValue &Opc);
 
-  AddrMode2Type SelectAddrMode2Worker(SDNode *Op, SDValue N, SDValue &Base,
+  AddrMode2Type SelectAddrMode2Worker(SDValue N, SDValue &Base,
                                       SDValue &Offset, SDValue &Opc);
-  bool SelectAddrMode2Base(SDNode *Op,
-                           SDValue N, SDValue &Base, SDValue &Offset,
+  bool SelectAddrMode2Base(SDValue N, SDValue &Base, SDValue &Offset,
                            SDValue &Opc) {
-    return SelectAddrMode2Worker(Op, N, Base, Offset, Opc) == AM2_BASE;
+    return SelectAddrMode2Worker(N, Base, Offset, Opc) == AM2_BASE;
   }
 
-  bool SelectAddrMode2ShOp(SDNode *Op,
-                           SDValue N, SDValue &Base, SDValue &Offset,
+  bool SelectAddrMode2ShOp(SDValue N, SDValue &Base, SDValue &Offset,
                            SDValue &Opc) {
-    return SelectAddrMode2Worker(Op, N, Base, Offset, Opc) == AM2_SHOP;
+    return SelectAddrMode2Worker(N, Base, Offset, Opc) == AM2_SHOP;
   }
 
-  bool SelectAddrMode2(SDNode *Op, 
-                       SDValue N, SDValue &Base, SDValue &Offset,
+  bool SelectAddrMode2(SDValue N, SDValue &Base, SDValue &Offset,
                        SDValue &Opc) {
-    SelectAddrMode2Worker(Op, N, Base, Offset, Opc);
+    SelectAddrMode2Worker(N, Base, Offset, Opc);
 //    return SelectAddrMode2ShOp(N, Base, Offset, Opc);
     // This always matches one way or another.
     return true;
@@ -137,7 +135,7 @@ class ARMDAGToDAGISel : public SelectionDAGISel {
 
   bool SelectCMOVPred(SDValue N, SDValue &Pred, SDValue &Reg) {
     const ConstantSDNode *CN = cast<ConstantSDNode>(N);
-    Pred = CurDAG->getTargetConstant(CN->getZExtValue(), MVT::i32);
+    Pred = CurDAG->getTargetConstant(CN->getZExtValue(), SDLoc(N), MVT::i32);
     Reg = CurDAG->getRegister(ARM::CPSR, MVT::i32);
     return true;
   }
@@ -149,7 +147,7 @@ class ARMDAGToDAGISel : public SelectionDAGISel {
   bool SelectAddrMode2OffsetImmPre(SDNode *Op, SDValue N,
                              SDValue &Offset, SDValue &Opc);
   bool SelectAddrOffsetNone(SDValue N, SDValue &Base);
-  bool SelectAddrMode3(SDNode *Op, SDValue N, SDValue &Base,
+  bool SelectAddrMode3(SDValue N, SDValue &Base,
                        SDValue &Offset, SDValue &Opc);
   bool SelectAddrMode3Offset(SDNode *Op, SDValue N,
                              SDValue &Offset, SDValue &Opc);
@@ -254,6 +252,9 @@ class ARMDAGToDAGISel : public SelectionDAGISel {
   // Select special operations if node forms integer ABS pattern
   SDNode *SelectABSOp(SDNode *N);
 
+  SDNode *SelectReadRegister(SDNode *N);
+  SDNode *SelectWriteRegister(SDNode *N);
+
   SDNode *SelectInlineAsm(SDNode *N);
 
   SDNode *SelectConcatVector(SDNode *N);
@@ -275,7 +276,8 @@ class ARMDAGToDAGISel : public SelectionDAGISel {
   SDNode *createQuadQRegsNode(EVT VT, SDValue V0, SDValue V1, SDValue V2, SDValue V3);
 
   // Get the alignment operand for a NEON VLD or VST instruction.
-  SDValue GetVLDSTAlign(SDValue Align, unsigned NumVecs, bool is64BitVector);
+  SDValue GetVLDSTAlign(SDValue Align, SDLoc dl, unsigned NumVecs,
+                        bool is64BitVector);
 };
 }
 
@@ -397,11 +399,13 @@ void ARMDAGToDAGISel::PreprocessISelDAG() {
     // Now make the transformation.
     Srl = CurDAG->getNode(ISD::SRL, SDLoc(Srl), MVT::i32,
                           Srl.getOperand(0),
-                          CurDAG->getConstant(Srl_imm+TZ, MVT::i32));
+                          CurDAG->getConstant(Srl_imm + TZ, SDLoc(Srl),
+                                              MVT::i32));
     N1 = CurDAG->getNode(ISD::AND, SDLoc(N1), MVT::i32,
-                         Srl, CurDAG->getConstant(And_imm, MVT::i32));
+                         Srl,
+                         CurDAG->getConstant(And_imm, SDLoc(Srl), MVT::i32));
     N1 = CurDAG->getNode(ISD::SHL, SDLoc(N1), MVT::i32,
-                         N1, CurDAG->getConstant(TZ, MVT::i32));
+                         N1, CurDAG->getConstant(TZ, SDLoc(Srl), MVT::i32));
     CurDAG->UpdateNodeOperands(N, N0, N1);
   }
 }
@@ -486,7 +490,7 @@ bool ARMDAGToDAGISel::SelectImmShifterOperand(SDValue N,
   if (!RHS) return false;
   ShImmVal = RHS->getZExtValue() & 31;
   Opc = CurDAG->getTargetConstant(ARM_AM::getSORegOpc(ShOpcVal, ShImmVal),
-                                  MVT::i32);
+                                  SDLoc(N), MVT::i32);
   return true;
 }
 
@@ -513,22 +517,10 @@ bool ARMDAGToDAGISel::SelectRegShifterOperand(SDValue N,
   if (CheckProfitability && !isShifterOpProfitable(N, ShOpcVal, ShImmVal))
     return false;
   Opc = CurDAG->getTargetConstant(ARM_AM::getSORegOpc(ShOpcVal, ShImmVal),
-                                  MVT::i32);
+                                  SDLoc(N), MVT::i32);
   return true;
 }
 
-// @LOCALMOD-START
-static bool ShouldOperandBeUnwrappedForUseAsBaseAddress(
-  SDValue& N, const ARMSubtarget* Subtarget) {
-  assert (N.getOpcode() == ARMISD::Wrapper);
-  // Never use this transformation if constant island pools are disallowed
-  if (!Subtarget->useConstIslands()) return false;
-
-  // explain why we do not want to use this for TargetGlobalAddress
-  if (N.getOperand(0).getOpcode() != ISD::TargetGlobalAddress) return true;
-  return false;
-}
-// @LOCALMOD-END
 
 bool ARMDAGToDAGISel::SelectAddrModeImm12(SDValue N,
                                           SDValue &Base,
@@ -541,18 +533,18 @@ bool ARMDAGToDAGISel::SelectAddrModeImm12(SDValue N,
     if (N.getOpcode() == ISD::FrameIndex) {
       // Match frame index.
       int FI = cast<FrameIndexSDNode>(N)->getIndex();
-      Base = CurDAG->getTargetFrameIndex(FI, TLI->getPointerTy());
-      OffImm  = CurDAG->getTargetConstant(0, MVT::i32);
+      Base = CurDAG->getTargetFrameIndex(
+          FI, TLI->getPointerTy(CurDAG->getDataLayout()));
+      OffImm  = CurDAG->getTargetConstant(0, SDLoc(N), MVT::i32);
       return true;
     }
 
     if (N.getOpcode() == ARMISD::Wrapper &&
-        // @LOCALMOD
-        ShouldOperandBeUnwrappedForUseAsBaseAddress(N, Subtarget)) {
+        N.getOperand(0).getOpcode() != ISD::TargetGlobalAddress) {
       Base = N.getOperand(0);
     } else
       Base = N;
-    OffImm  = CurDAG->getTargetConstant(0, MVT::i32);
+    OffImm  = CurDAG->getTargetConstant(0, SDLoc(N), MVT::i32);
     return true;
   }
 
@@ -565,16 +557,17 @@ bool ARMDAGToDAGISel::SelectAddrModeImm12(SDValue N,
       Base   = N.getOperand(0);
       if (Base.getOpcode() == ISD::FrameIndex) {
         int FI = cast<FrameIndexSDNode>(Base)->getIndex();
-        Base = CurDAG->getTargetFrameIndex(FI, TLI->getPointerTy());
+        Base = CurDAG->getTargetFrameIndex(
+            FI, TLI->getPointerTy(CurDAG->getDataLayout()));
       }
-      OffImm = CurDAG->getTargetConstant(RHSC, MVT::i32);
+      OffImm = CurDAG->getTargetConstant(RHSC, SDLoc(N), MVT::i32);
       return true;
     }
   }
 
   // Base only.
   Base = N;
-  OffImm  = CurDAG->getTargetConstant(0, MVT::i32);
+  OffImm  = CurDAG->getTargetConstant(0, SDLoc(N), MVT::i32);
   return true;
 }
 
@@ -582,11 +575,6 @@ bool ARMDAGToDAGISel::SelectAddrModeImm12(SDValue N,
 
 bool ARMDAGToDAGISel::SelectLdStSOReg(SDValue N, SDValue &Base, SDValue &Offset,
                                       SDValue &Opc) {
-  // @LOCALMOD-BEGIN
-  // Disallow offsets of Reg + Reg (which may escape sandbox).
-  if (Subtarget->isTargetNaCl())
-    return false;
-  // @LOCALMOD-END
   if (N.getOpcode() == ISD::MUL &&
       ((!Subtarget->isLikeA9() && !Subtarget->isSwift()) || N.hasOneUse())) {
     if (ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N.getOperand(1))) {
@@ -604,7 +592,7 @@ bool ARMDAGToDAGISel::SelectLdStSOReg(SDValue N, SDValue &Base, SDValue &Offset,
           Base = Offset = N.getOperand(0);
           Opc = CurDAG->getTargetConstant(ARM_AM::getAM2Opc(AddSub, ShAmt,
                                                             ARM_AM::lsl),
-                                          MVT::i32);
+                                          SDLoc(N), MVT::i32);
           return true;
         }
       }
@@ -675,26 +663,17 @@ bool ARMDAGToDAGISel::SelectLdStSOReg(SDValue N, SDValue &Base, SDValue &Offset,
   }
 
   Opc = CurDAG->getTargetConstant(ARM_AM::getAM2Opc(AddSub, ShAmt, ShOpcVal),
-                                  MVT::i32);
+                                  SDLoc(N), MVT::i32);
   return true;
 }
 
 
 //-----
 
-AddrMode2Type ARMDAGToDAGISel::SelectAddrMode2Worker(SDNode *Op,
-                                                     SDValue N,
+AddrMode2Type ARMDAGToDAGISel::SelectAddrMode2Worker(SDValue N,
                                                      SDValue &Base,
                                                      SDValue &Offset,
                                                      SDValue &Opc) {
-  // @LOCALMOD-START
-  // Avoid two reg addressing mode for loads and stores
-  const bool restrict_addressing_modes_for_nacl =
-      Subtarget->isTargetNaCl() && isa<MemSDNode>(Op);
-  // This is neither a sandboxable load nor a sandboxable store.
-  if (!restrict_addressing_modes_for_nacl) {
-  // @LOCALMOD-END
-
   if (N.getOpcode() == ISD::MUL &&
       (!(Subtarget->isLikeA9() || Subtarget->isSwift()) || N.hasOneUse())) {
     if (ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N.getOperand(1))) {
@@ -712,13 +691,12 @@ AddrMode2Type ARMDAGToDAGISel::SelectAddrMode2Worker(SDNode *Op,
           Base = Offset = N.getOperand(0);
           Opc = CurDAG->getTargetConstant(ARM_AM::getAM2Opc(AddSub, ShAmt,
                                                             ARM_AM::lsl),
-                                          MVT::i32);
+                                          SDLoc(N), MVT::i32);
           return AM2_SHOP;
         }
       }
     }
   }
-  } // @LOCALMOD
 
   if (N.getOpcode() != ISD::ADD && N.getOpcode() != ISD::SUB &&
       // ISD::OR that is equivalent to an ADD.
@@ -726,16 +704,16 @@ AddrMode2Type ARMDAGToDAGISel::SelectAddrMode2Worker(SDNode *Op,
     Base = N;
     if (N.getOpcode() == ISD::FrameIndex) {
       int FI = cast<FrameIndexSDNode>(N)->getIndex();
-      Base = CurDAG->getTargetFrameIndex(FI, TLI->getPointerTy());
+      Base = CurDAG->getTargetFrameIndex(
+          FI, TLI->getPointerTy(CurDAG->getDataLayout()));
     } else if (N.getOpcode() == ARMISD::Wrapper &&
-               // @LOCALMOD
-               ShouldOperandBeUnwrappedForUseAsBaseAddress(N, Subtarget)) {
+               N.getOperand(0).getOpcode() != ISD::TargetGlobalAddress) {
       Base = N.getOperand(0);
     }
     Offset = CurDAG->getRegister(0, MVT::i32);
     Opc = CurDAG->getTargetConstant(ARM_AM::getAM2Opc(ARM_AM::add, 0,
                                                       ARM_AM::no_shift),
-                                    MVT::i32);
+                                    SDLoc(N), MVT::i32);
     return AM2_BASE;
   }
 
@@ -747,7 +725,8 @@ AddrMode2Type ARMDAGToDAGISel::SelectAddrMode2Worker(SDNode *Op,
       Base = N.getOperand(0);
       if (Base.getOpcode() == ISD::FrameIndex) {
         int FI = cast<FrameIndexSDNode>(Base)->getIndex();
-        Base = CurDAG->getTargetFrameIndex(FI, TLI->getPointerTy());
+        Base = CurDAG->getTargetFrameIndex(
+            FI, TLI->getPointerTy(CurDAG->getDataLayout()));
       }
       Offset = CurDAG->getRegister(0, MVT::i32);
 
@@ -758,7 +737,7 @@ AddrMode2Type ARMDAGToDAGISel::SelectAddrMode2Worker(SDNode *Op,
       }
       Opc = CurDAG->getTargetConstant(ARM_AM::getAM2Opc(AddSub, RHSC,
                                                         ARM_AM::no_shift),
-                                      MVT::i32);
+                                      SDLoc(N), MVT::i32);
       return AM2_BASE;
     }
   }
@@ -769,28 +748,9 @@ AddrMode2Type ARMDAGToDAGISel::SelectAddrMode2Worker(SDNode *Op,
     Offset = CurDAG->getRegister(0, MVT::i32);
     Opc = CurDAG->getTargetConstant(ARM_AM::getAM2Opc(ARM_AM::add, 0,
                                                       ARM_AM::no_shift),
-                                    MVT::i32);
+                                    SDLoc(N), MVT::i32);
     return AM2_BASE;
   }
-  
-  // @LOCALMOD-START
-  // Keep load and store addressing modes simple
-  if (restrict_addressing_modes_for_nacl) {
-    Base = N;
-    if (N.getOpcode() == ISD::FrameIndex) {
-      int FI = cast<FrameIndexSDNode>(N)->getIndex();
-      Base = CurDAG->getTargetFrameIndex(FI,
-                                         getTargetLowering()->getPointerTy());
-    } else if (N.getOpcode() == ARMISD::Wrapper) {
-      Base = N.getOperand(0);
-    }
-    Offset = CurDAG->getRegister(0, MVT::i32);
-    Opc = CurDAG->getTargetConstant(ARM_AM::getAM2Opc(ARM_AM::add, 0,
-                                                      ARM_AM::no_shift),
-                                    MVT::i32);
-    return AM2_BASE;
-  }
-  // @LOCALMOD-END
 
   // Otherwise this is R +/- [possibly shifted] R.
   ARM_AM::AddrOpc AddSub = N.getOpcode() != ISD::SUB ? ARM_AM::add:ARM_AM::sub;
@@ -843,7 +803,7 @@ AddrMode2Type ARMDAGToDAGISel::SelectAddrMode2Worker(SDNode *Op,
   }
 
   Opc = CurDAG->getTargetConstant(ARM_AM::getAM2Opc(AddSub, ShAmt, ShOpcVal),
-                                  MVT::i32);
+                                  SDLoc(N), MVT::i32);
   return AM2_SHOP;
 }
 
@@ -859,25 +819,13 @@ bool ARMDAGToDAGISel::SelectAddrMode2OffsetReg(SDNode *Op, SDValue N,
   if (isScaledConstantInRange(N, /*Scale=*/1, 0, 0x1000, Val))
     return false;
 
-  // @LOCALMOD-BEGIN
-  // Avoid two reg addressing mode for loads and stores
-  const bool restrict_addressing_modes_for_nacl =
-      Subtarget->isTargetNaCl() && isa<MemSDNode>(Op);
-  // @LOCALMOD-END
-
   Offset = N;
   ARM_AM::ShiftOpc ShOpcVal = ARM_AM::getShiftOpcForNode(N.getOpcode());
   unsigned ShAmt = 0;
   if (ShOpcVal != ARM_AM::no_shift) {
     // Check to see if the RHS of the shift is a constant, if not, we can't fold
     // it.
-
-    //if (ConstantSDNode *Sh = dyn_cast<ConstantSDNode>(N.getOperand(1))) {
-    ConstantSDNode *Sh = dyn_cast<ConstantSDNode>(N.getOperand(1));
-    // @LOCALMOD-BEGIN
-    // Neither a sandboxable load nor a sandboxable store.
-    if (!restrict_addressing_modes_for_nacl && Sh ) {
-    // @LOCALMOD-END
+    if (ConstantSDNode *Sh = dyn_cast<ConstantSDNode>(N.getOperand(1))) {
       ShAmt = Sh->getZExtValue();
       if (isShifterOpProfitable(N, ShOpcVal, ShAmt))
         Offset = N.getOperand(0);
@@ -891,7 +839,7 @@ bool ARMDAGToDAGISel::SelectAddrMode2OffsetReg(SDNode *Op, SDValue N,
   }
 
   Opc = CurDAG->getTargetConstant(ARM_AM::getAM2Opc(AddSub, ShAmt, ShOpcVal),
-                                  MVT::i32);
+                                  SDLoc(N), MVT::i32);
   return true;
 }
 
@@ -907,7 +855,7 @@ bool ARMDAGToDAGISel::SelectAddrMode2OffsetImmPre(SDNode *Op, SDValue N,
   if (isScaledConstantInRange(N, /*Scale=*/1, 0, 0x1000, Val)) { // 12 bits.
     if (AddSub == ARM_AM::sub) Val *= -1;
     Offset = CurDAG->getRegister(0, MVT::i32);
-    Opc = CurDAG->getTargetConstant(Val, MVT::i32);
+    Opc = CurDAG->getTargetConstant(Val, SDLoc(Op), MVT::i32);
     return true;
   }
 
@@ -928,7 +876,7 @@ bool ARMDAGToDAGISel::SelectAddrMode2OffsetImm(SDNode *Op, SDValue N,
     Offset = CurDAG->getRegister(0, MVT::i32);
     Opc = CurDAG->getTargetConstant(ARM_AM::getAM2Opc(AddSub, Val,
                                                       ARM_AM::no_shift),
-                                    MVT::i32);
+                                    SDLoc(Op), MVT::i32);
     return true;
   }
 
@@ -940,32 +888,28 @@ bool ARMDAGToDAGISel::SelectAddrOffsetNone(SDValue N, SDValue &Base) {
   return true;
 }
 
-bool ARMDAGToDAGISel::SelectAddrMode3(SDNode *Op, SDValue N,
+bool ARMDAGToDAGISel::SelectAddrMode3(SDValue N,
                                       SDValue &Base, SDValue &Offset,
                                       SDValue &Opc) {
-  // @LOCALMOD-START
-  // Avoid two reg addressing mode for loads and stores
-  const bool restrict_addressing_modes_for_nacl =
-      Subtarget->isTargetNaCl() && isa<MemSDNode>(Op);
-  if (!restrict_addressing_modes_for_nacl) {
-  // @LOCALMOD-END
   if (N.getOpcode() == ISD::SUB) {
     // X - C  is canonicalize to X + -C, no need to handle it here.
     Base = N.getOperand(0);
     Offset = N.getOperand(1);
-    Opc = CurDAG->getTargetConstant(ARM_AM::getAM3Opc(ARM_AM::sub, 0),MVT::i32);
+    Opc = CurDAG->getTargetConstant(ARM_AM::getAM3Opc(ARM_AM::sub, 0), SDLoc(N),
+                                    MVT::i32);
     return true;
   }
-  } // @LOCALMOD-END
 
   if (!CurDAG->isBaseWithConstantOffset(N)) {
     Base = N;
     if (N.getOpcode() == ISD::FrameIndex) {
       int FI = cast<FrameIndexSDNode>(N)->getIndex();
-      Base = CurDAG->getTargetFrameIndex(FI, TLI->getPointerTy());
+      Base = CurDAG->getTargetFrameIndex(
+          FI, TLI->getPointerTy(CurDAG->getDataLayout()));
     }
     Offset = CurDAG->getRegister(0, MVT::i32);
-    Opc = CurDAG->getTargetConstant(ARM_AM::getAM3Opc(ARM_AM::add, 0),MVT::i32);
+    Opc = CurDAG->getTargetConstant(ARM_AM::getAM3Opc(ARM_AM::add, 0), SDLoc(N),
+                                    MVT::i32);
     return true;
   }
 
@@ -976,7 +920,8 @@ bool ARMDAGToDAGISel::SelectAddrMode3(SDNode *Op, SDValue N,
     Base = N.getOperand(0);
     if (Base.getOpcode() == ISD::FrameIndex) {
       int FI = cast<FrameIndexSDNode>(Base)->getIndex();
-      Base = CurDAG->getTargetFrameIndex(FI, TLI->getPointerTy());
+      Base = CurDAG->getTargetFrameIndex(
+          FI, TLI->getPointerTy(CurDAG->getDataLayout()));
     }
     Offset = CurDAG->getRegister(0, MVT::i32);
 
@@ -985,23 +930,15 @@ bool ARMDAGToDAGISel::SelectAddrMode3(SDNode *Op, SDValue N,
       AddSub = ARM_AM::sub;
       RHSC = -RHSC;
     }
-    Opc = CurDAG->getTargetConstant(ARM_AM::getAM3Opc(AddSub, RHSC),MVT::i32);
-    return true;
-  }
-
-  // @LOCALMOD-START
-  // A sandboxable load or a sandboxable store.
-  if (restrict_addressing_modes_for_nacl) {
-    Base = N;
-    Offset = CurDAG->getRegister(0, MVT::i32);
-    Opc = CurDAG->getTargetConstant(ARM_AM::getAM3Opc(ARM_AM::add, 0),MVT::i32);
+    Opc = CurDAG->getTargetConstant(ARM_AM::getAM3Opc(AddSub, RHSC), SDLoc(N),
+                                    MVT::i32);
     return true;
   }
-  // @LOCALMOD-END
 
   Base = N.getOperand(0);
   Offset = N.getOperand(1);
-  Opc = CurDAG->getTargetConstant(ARM_AM::getAM3Opc(ARM_AM::add, 0), MVT::i32);
+  Opc = CurDAG->getTargetConstant(ARM_AM::getAM3Opc(ARM_AM::add, 0), SDLoc(N),
+                                  MVT::i32);
   return true;
 }
 
@@ -1016,12 +953,14 @@ bool ARMDAGToDAGISel::SelectAddrMode3Offset(SDNode *Op, SDValue N,
   int Val;
   if (isScaledConstantInRange(N, /*Scale=*/1, 0, 256, Val)) { // 12 bits.
     Offset = CurDAG->getRegister(0, MVT::i32);
-    Opc = CurDAG->getTargetConstant(ARM_AM::getAM3Opc(AddSub, Val), MVT::i32);
+    Opc = CurDAG->getTargetConstant(ARM_AM::getAM3Opc(AddSub, Val), SDLoc(Op),
+                                    MVT::i32);
     return true;
   }
 
   Offset = N;
-  Opc = CurDAG->getTargetConstant(ARM_AM::getAM3Opc(AddSub, 0), MVT::i32);
+  Opc = CurDAG->getTargetConstant(ARM_AM::getAM3Opc(AddSub, 0), SDLoc(Op),
+                                  MVT::i32);
   return true;
 }
 
@@ -1031,14 +970,14 @@ bool ARMDAGToDAGISel::SelectAddrMode5(SDValue N,
     Base = N;
     if (N.getOpcode() == ISD::FrameIndex) {
       int FI = cast<FrameIndexSDNode>(N)->getIndex();
-      Base = CurDAG->getTargetFrameIndex(FI, TLI->getPointerTy());
+      Base = CurDAG->getTargetFrameIndex(
+          FI, TLI->getPointerTy(CurDAG->getDataLayout()));
     } else if (N.getOpcode() == ARMISD::Wrapper &&
-               // @LOCALMOD
-               ShouldOperandBeUnwrappedForUseAsBaseAddress(N, Subtarget)) {
+               N.getOperand(0).getOpcode() != ISD::TargetGlobalAddress) {
       Base = N.getOperand(0);
     }
     Offset = CurDAG->getTargetConstant(ARM_AM::getAM5Opc(ARM_AM::add, 0),
-                                       MVT::i32);
+                                       SDLoc(N), MVT::i32);
     return true;
   }
 
@@ -1049,7 +988,8 @@ bool ARMDAGToDAGISel::SelectAddrMode5(SDValue N,
     Base = N.getOperand(0);
     if (Base.getOpcode() == ISD::FrameIndex) {
       int FI = cast<FrameIndexSDNode>(Base)->getIndex();
-      Base = CurDAG->getTargetFrameIndex(FI, TLI->getPointerTy());
+      Base = CurDAG->getTargetFrameIndex(
+          FI, TLI->getPointerTy(CurDAG->getDataLayout()));
     }
 
     ARM_AM::AddrOpc AddSub = ARM_AM::add;
@@ -1058,13 +998,13 @@ bool ARMDAGToDAGISel::SelectAddrMode5(SDValue N,
       RHSC = -RHSC;
     }
     Offset = CurDAG->getTargetConstant(ARM_AM::getAM5Opc(AddSub, RHSC),
-                                       MVT::i32);
+                                       SDLoc(N), MVT::i32);
     return true;
   }
 
   Base = N;
   Offset = CurDAG->getTargetConstant(ARM_AM::getAM5Opc(ARM_AM::add, 0),
-                                     MVT::i32);
+                                     SDLoc(N), MVT::i32);
   return true;
 }
 
@@ -1093,7 +1033,7 @@ bool ARMDAGToDAGISel::SelectAddrMode6(SDNode *Parent, SDValue N, SDValue &Addr,
     Alignment = MemN->getAlignment();
   }
 
-  Align = CurDAG->getTargetConstant(Alignment, MVT::i32);
+  Align = CurDAG->getTargetConstant(Alignment, SDLoc(N), MVT::i32);
   return true;
 }
 
@@ -1117,7 +1057,7 @@ bool ARMDAGToDAGISel::SelectAddrModePC(SDValue N,
     Offset = N.getOperand(0);
     SDValue N1 = N.getOperand(1);
     Label = CurDAG->getTargetConstant(cast<ConstantSDNode>(N1)->getZExtValue(),
-                                      MVT::i32);
+                                      SDLoc(N), MVT::i32);
     return true;
   }
 
@@ -1222,7 +1162,7 @@ ARMDAGToDAGISel::SelectThumbAddrModeImm5S(SDValue N, unsigned Scale,
       Base = N;
     }
 
-    OffImm = CurDAG->getTargetConstant(0, MVT::i32);
+    OffImm = CurDAG->getTargetConstant(0, SDLoc(N), MVT::i32);
     return true;
   }
 
@@ -1239,7 +1179,7 @@ ARMDAGToDAGISel::SelectThumbAddrModeImm5S(SDValue N, unsigned Scale,
     if (LHSC != 0 || RHSC != 0) return false;
 
     Base = N;
-    OffImm = CurDAG->getTargetConstant(0, MVT::i32);
+    OffImm = CurDAG->getTargetConstant(0, SDLoc(N), MVT::i32);
     return true;
   }
 
@@ -1247,12 +1187,12 @@ ARMDAGToDAGISel::SelectThumbAddrModeImm5S(SDValue N, unsigned Scale,
   int RHSC;
   if (isScaledConstantInRange(N.getOperand(1), Scale, 0, 32, RHSC)) {
     Base = N.getOperand(0);
-    OffImm = CurDAG->getTargetConstant(RHSC, MVT::i32);
+    OffImm = CurDAG->getTargetConstant(RHSC, SDLoc(N), MVT::i32);
     return true;
   }
 
   Base = N.getOperand(0);
-  OffImm = CurDAG->getTargetConstant(0, MVT::i32);
+  OffImm = CurDAG->getTargetConstant(0, SDLoc(N), MVT::i32);
   return true;
 }
 
@@ -1283,8 +1223,9 @@ bool ARMDAGToDAGISel::SelectThumbAddrModeSP(SDValue N,
     MachineFrameInfo *MFI = MF->getFrameInfo();
     if (MFI->getObjectAlignment(FI) < 4)
       MFI->setObjectAlignment(FI, 4);
-    Base = CurDAG->getTargetFrameIndex(FI, TLI->getPointerTy());
-    OffImm = CurDAG->getTargetConstant(0, MVT::i32);
+    Base = CurDAG->getTargetFrameIndex(
+        FI, TLI->getPointerTy(CurDAG->getDataLayout()));
+    OffImm = CurDAG->getTargetConstant(0, SDLoc(N), MVT::i32);
     return true;
   }
 
@@ -1305,9 +1246,10 @@ bool ARMDAGToDAGISel::SelectThumbAddrModeSP(SDValue N,
         MachineFrameInfo *MFI = MF->getFrameInfo();
         if (MFI->getObjectAlignment(FI) < 4)
           MFI->setObjectAlignment(FI, 4);
-        Base = CurDAG->getTargetFrameIndex(FI, TLI->getPointerTy());
+        Base = CurDAG->getTargetFrameIndex(
+            FI, TLI->getPointerTy(CurDAG->getDataLayout()));
       }
-      OffImm = CurDAG->getTargetConstant(RHSC, MVT::i32);
+      OffImm = CurDAG->getTargetConstant(RHSC, SDLoc(N), MVT::i32);
       return true;
     }
   }
@@ -1336,7 +1278,7 @@ bool ARMDAGToDAGISel::SelectT2ShifterOperandReg(SDValue N, SDValue &BaseReg,
   unsigned ShImmVal = 0;
   if (ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N.getOperand(1))) {
     ShImmVal = RHS->getZExtValue() & 31;
-    Opc = getI32Imm(ARM_AM::getSORegOpc(ShOpcVal, ShImmVal));
+    Opc = getI32Imm(ARM_AM::getSORegOpc(ShOpcVal, ShImmVal), SDLoc(N));
     return true;
   }
 
@@ -1353,8 +1295,9 @@ bool ARMDAGToDAGISel::SelectT2AddrModeImm12(SDValue N,
     if (N.getOpcode() == ISD::FrameIndex) {
       // Match frame index.
       int FI = cast<FrameIndexSDNode>(N)->getIndex();
-      Base = CurDAG->getTargetFrameIndex(FI, TLI->getPointerTy());
-      OffImm  = CurDAG->getTargetConstant(0, MVT::i32);
+      Base = CurDAG->getTargetFrameIndex(
+          FI, TLI->getPointerTy(CurDAG->getDataLayout()));
+      OffImm  = CurDAG->getTargetConstant(0, SDLoc(N), MVT::i32);
       return true;
     }
 
@@ -1365,7 +1308,7 @@ bool ARMDAGToDAGISel::SelectT2AddrModeImm12(SDValue N,
         return false;  // We want to select t2LDRpci instead.
     } else
       Base = N;
-    OffImm  = CurDAG->getTargetConstant(0, MVT::i32);
+    OffImm  = CurDAG->getTargetConstant(0, SDLoc(N), MVT::i32);
     return true;
   }
 
@@ -1382,16 +1325,17 @@ bool ARMDAGToDAGISel::SelectT2AddrModeImm12(SDValue N,
       Base   = N.getOperand(0);
       if (Base.getOpcode() == ISD::FrameIndex) {
         int FI = cast<FrameIndexSDNode>(Base)->getIndex();
-        Base = CurDAG->getTargetFrameIndex(FI, TLI->getPointerTy());
+        Base = CurDAG->getTargetFrameIndex(
+            FI, TLI->getPointerTy(CurDAG->getDataLayout()));
       }
-      OffImm = CurDAG->getTargetConstant(RHSC, MVT::i32);
+      OffImm = CurDAG->getTargetConstant(RHSC, SDLoc(N), MVT::i32);
       return true;
     }
   }
 
   // Base only.
   Base = N;
-  OffImm  = CurDAG->getTargetConstant(0, MVT::i32);
+  OffImm  = CurDAG->getTargetConstant(0, SDLoc(N), MVT::i32);
   return true;
 }
 
@@ -1411,9 +1355,10 @@ bool ARMDAGToDAGISel::SelectT2AddrModeImm8(SDValue N,
       Base = N.getOperand(0);
       if (Base.getOpcode() == ISD::FrameIndex) {
         int FI = cast<FrameIndexSDNode>(Base)->getIndex();
-        Base = CurDAG->getTargetFrameIndex(FI, TLI->getPointerTy());
+        Base = CurDAG->getTargetFrameIndex(
+            FI, TLI->getPointerTy(CurDAG->getDataLayout()));
       }
-      OffImm = CurDAG->getTargetConstant(RHSC, MVT::i32);
+      OffImm = CurDAG->getTargetConstant(RHSC, SDLoc(N), MVT::i32);
       return true;
     }
   }
@@ -1430,8 +1375,8 @@ bool ARMDAGToDAGISel::SelectT2AddrModeImm8Offset(SDNode *Op, SDValue N,
   int RHSC;
   if (isScaledConstantInRange(N, /*Scale=*/1, 0, 0x100, RHSC)) { // 8 bits.
     OffImm = ((AM == ISD::PRE_INC) || (AM == ISD::POST_INC))
-      ? CurDAG->getTargetConstant(RHSC, MVT::i32)
-      : CurDAG->getTargetConstant(-RHSC, MVT::i32);
+      ? CurDAG->getTargetConstant(RHSC, SDLoc(N), MVT::i32)
+      : CurDAG->getTargetConstant(-RHSC, SDLoc(N), MVT::i32);
     return true;
   }
 
@@ -1480,7 +1425,7 @@ bool ARMDAGToDAGISel::SelectT2AddrModeSoReg(SDValue N,
     }
   }
 
-  ShImm = CurDAG->getTargetConstant(ShAmt, MVT::i32);
+  ShImm = CurDAG->getTargetConstant(ShAmt, SDLoc(N), MVT::i32);
 
   return true;
 }
@@ -1490,7 +1435,7 @@ bool ARMDAGToDAGISel::SelectT2AddrModeExclusive(SDValue N, SDValue &Base,
   // This *must* succeed since it's used for the irreplaceable ldrex and strex
   // instructions.
   Base = N;
-  OffImm = CurDAG->getTargetConstant(0, MVT::i32);
+  OffImm = CurDAG->getTargetConstant(0, SDLoc(N), MVT::i32);
 
   if (N.getOpcode() != ISD::ADD || !CurDAG->isBaseWithConstantOffset(N))
     return true;
@@ -1506,18 +1451,19 @@ bool ARMDAGToDAGISel::SelectT2AddrModeExclusive(SDValue N, SDValue &Base,
   Base = N.getOperand(0);
   if (Base.getOpcode() == ISD::FrameIndex) {
     int FI = cast<FrameIndexSDNode>(Base)->getIndex();
-    Base = CurDAG->getTargetFrameIndex(FI, TLI->getPointerTy());
+    Base = CurDAG->getTargetFrameIndex(
+        FI, TLI->getPointerTy(CurDAG->getDataLayout()));
   }
 
-  OffImm = CurDAG->getTargetConstant(RHSC / 4, MVT::i32);
+  OffImm = CurDAG->getTargetConstant(RHSC/4, SDLoc(N), MVT::i32);
   return true;
 }
 
 //===--------------------------------------------------------------------===//
 
 /// getAL - Returns a ARMCC::AL immediate node.
-static inline SDValue getAL(SelectionDAG *CurDAG) {
-  return CurDAG->getTargetConstant((uint64_t)ARMCC::AL, MVT::i32);
+static inline SDValue getAL(SelectionDAG *CurDAG, SDLoc dl) {
+  return CurDAG->getTargetConstant((uint64_t)ARMCC::AL, dl, MVT::i32);
 }
 
 SDNode *ARMDAGToDAGISel::SelectARMIndexedLoad(SDNode *N) {
@@ -1576,14 +1522,14 @@ SDNode *ARMDAGToDAGISel::SelectARMIndexedLoad(SDNode *N) {
     if (Opcode == ARM::LDR_PRE_IMM || Opcode == ARM::LDRB_PRE_IMM) {
       SDValue Chain = LD->getChain();
       SDValue Base = LD->getBasePtr();
-      SDValue Ops[]= { Base, AMOpc, getAL(CurDAG),
+      SDValue Ops[]= { Base, AMOpc, getAL(CurDAG, SDLoc(N)),
                        CurDAG->getRegister(0, MVT::i32), Chain };
       return CurDAG->getMachineNode(Opcode, SDLoc(N), MVT::i32,
                                     MVT::i32, MVT::Other, Ops);
     } else {
       SDValue Chain = LD->getChain();
       SDValue Base = LD->getBasePtr();
-      SDValue Ops[]= { Base, Offset, AMOpc, getAL(CurDAG),
+      SDValue Ops[]= { Base, Offset, AMOpc, getAL(CurDAG, SDLoc(N)),
                        CurDAG->getRegister(0, MVT::i32), Chain };
       return CurDAG->getMachineNode(Opcode, SDLoc(N), MVT::i32,
                                     MVT::i32, MVT::Other, Ops);
@@ -1632,7 +1578,7 @@ SDNode *ARMDAGToDAGISel::SelectT2IndexedLoad(SDNode *N) {
   if (Match) {
     SDValue Chain = LD->getChain();
     SDValue Base = LD->getBasePtr();
-    SDValue Ops[]= { Base, Offset, getAL(CurDAG),
+    SDValue Ops[]= { Base, Offset, getAL(CurDAG, SDLoc(N)),
                      CurDAG->getRegister(0, MVT::i32), Chain };
     return CurDAG->getMachineNode(Opcode, SDLoc(N), MVT::i32, MVT::i32,
                                   MVT::Other, Ops);
@@ -1645,9 +1591,9 @@ SDNode *ARMDAGToDAGISel::SelectT2IndexedLoad(SDNode *N) {
 SDNode *ARMDAGToDAGISel::createGPRPairNode(EVT VT, SDValue V0, SDValue V1) {
   SDLoc dl(V0.getNode());
   SDValue RegClass =
-    CurDAG->getTargetConstant(ARM::GPRPairRegClassID, MVT::i32);
-  SDValue SubReg0 = CurDAG->getTargetConstant(ARM::gsub_0, MVT::i32);
-  SDValue SubReg1 = CurDAG->getTargetConstant(ARM::gsub_1, MVT::i32);
+    CurDAG->getTargetConstant(ARM::GPRPairRegClassID, dl, MVT::i32);
+  SDValue SubReg0 = CurDAG->getTargetConstant(ARM::gsub_0, dl, MVT::i32);
+  SDValue SubReg1 = CurDAG->getTargetConstant(ARM::gsub_1, dl, MVT::i32);
   const SDValue Ops[] = { RegClass, V0, SubReg0, V1, SubReg1 };
   return CurDAG->getMachineNode(TargetOpcode::REG_SEQUENCE, dl, VT, Ops);
 }
@@ -1656,9 +1602,9 @@ SDNode *ARMDAGToDAGISel::createGPRPairNode(EVT VT, SDValue V0, SDValue V1) {
 SDNode *ARMDAGToDAGISel::createSRegPairNode(EVT VT, SDValue V0, SDValue V1) {
   SDLoc dl(V0.getNode());
   SDValue RegClass =
-    CurDAG->getTargetConstant(ARM::DPR_VFP2RegClassID, MVT::i32);
-  SDValue SubReg0 = CurDAG->getTargetConstant(ARM::ssub_0, MVT::i32);
-  SDValue SubReg1 = CurDAG->getTargetConstant(ARM::ssub_1, MVT::i32);
+    CurDAG->getTargetConstant(ARM::DPR_VFP2RegClassID, dl, MVT::i32);
+  SDValue SubReg0 = CurDAG->getTargetConstant(ARM::ssub_0, dl, MVT::i32);
+  SDValue SubReg1 = CurDAG->getTargetConstant(ARM::ssub_1, dl, MVT::i32);
   const SDValue Ops[] = { RegClass, V0, SubReg0, V1, SubReg1 };
   return CurDAG->getMachineNode(TargetOpcode::REG_SEQUENCE, dl, VT, Ops);
 }
@@ -1666,9 +1612,10 @@ SDNode *ARMDAGToDAGISel::createSRegPairNode(EVT VT, SDValue V0, SDValue V1) {
 /// \brief Form a quad register from a pair of D registers.
 SDNode *ARMDAGToDAGISel::createDRegPairNode(EVT VT, SDValue V0, SDValue V1) {
   SDLoc dl(V0.getNode());
-  SDValue RegClass = CurDAG->getTargetConstant(ARM::QPRRegClassID, MVT::i32);
-  SDValue SubReg0 = CurDAG->getTargetConstant(ARM::dsub_0, MVT::i32);
-  SDValue SubReg1 = CurDAG->getTargetConstant(ARM::dsub_1, MVT::i32);
+  SDValue RegClass = CurDAG->getTargetConstant(ARM::QPRRegClassID, dl,
+                                               MVT::i32);
+  SDValue SubReg0 = CurDAG->getTargetConstant(ARM::dsub_0, dl, MVT::i32);
+  SDValue SubReg1 = CurDAG->getTargetConstant(ARM::dsub_1, dl, MVT::i32);
   const SDValue Ops[] = { RegClass, V0, SubReg0, V1, SubReg1 };
   return CurDAG->getMachineNode(TargetOpcode::REG_SEQUENCE, dl, VT, Ops);
 }
@@ -1676,9 +1623,10 @@ SDNode *ARMDAGToDAGISel::createDRegPairNode(EVT VT, SDValue V0, SDValue V1) {
 /// \brief Form 4 consecutive D registers from a pair of Q registers.
 SDNode *ARMDAGToDAGISel::createQRegPairNode(EVT VT, SDValue V0, SDValue V1) {
   SDLoc dl(V0.getNode());
-  SDValue RegClass = CurDAG->getTargetConstant(ARM::QQPRRegClassID, MVT::i32);
-  SDValue SubReg0 = CurDAG->getTargetConstant(ARM::qsub_0, MVT::i32);
-  SDValue SubReg1 = CurDAG->getTargetConstant(ARM::qsub_1, MVT::i32);
+  SDValue RegClass = CurDAG->getTargetConstant(ARM::QQPRRegClassID, dl,
+                                               MVT::i32);
+  SDValue SubReg0 = CurDAG->getTargetConstant(ARM::qsub_0, dl, MVT::i32);
+  SDValue SubReg1 = CurDAG->getTargetConstant(ARM::qsub_1, dl, MVT::i32);
   const SDValue Ops[] = { RegClass, V0, SubReg0, V1, SubReg1 };
   return CurDAG->getMachineNode(TargetOpcode::REG_SEQUENCE, dl, VT, Ops);
 }
@@ -1688,11 +1636,11 @@ SDNode *ARMDAGToDAGISel::createQuadSRegsNode(EVT VT, SDValue V0, SDValue V1,
                                    SDValue V2, SDValue V3) {
   SDLoc dl(V0.getNode());
   SDValue RegClass =
-    CurDAG->getTargetConstant(ARM::QPR_VFP2RegClassID, MVT::i32);
-  SDValue SubReg0 = CurDAG->getTargetConstant(ARM::ssub_0, MVT::i32);
-  SDValue SubReg1 = CurDAG->getTargetConstant(ARM::ssub_1, MVT::i32);
-  SDValue SubReg2 = CurDAG->getTargetConstant(ARM::ssub_2, MVT::i32);
-  SDValue SubReg3 = CurDAG->getTargetConstant(ARM::ssub_3, MVT::i32);
+    CurDAG->getTargetConstant(ARM::QPR_VFP2RegClassID, dl, MVT::i32);
+  SDValue SubReg0 = CurDAG->getTargetConstant(ARM::ssub_0, dl, MVT::i32);
+  SDValue SubReg1 = CurDAG->getTargetConstant(ARM::ssub_1, dl, MVT::i32);
+  SDValue SubReg2 = CurDAG->getTargetConstant(ARM::ssub_2, dl, MVT::i32);
+  SDValue SubReg3 = CurDAG->getTargetConstant(ARM::ssub_3, dl, MVT::i32);
   const SDValue Ops[] = { RegClass, V0, SubReg0, V1, SubReg1,
                                     V2, SubReg2, V3, SubReg3 };
   return CurDAG->getMachineNode(TargetOpcode::REG_SEQUENCE, dl, VT, Ops);
@@ -1702,11 +1650,12 @@ SDNode *ARMDAGToDAGISel::createQuadSRegsNode(EVT VT, SDValue V0, SDValue V1,
 SDNode *ARMDAGToDAGISel::createQuadDRegsNode(EVT VT, SDValue V0, SDValue V1,
                                    SDValue V2, SDValue V3) {
   SDLoc dl(V0.getNode());
-  SDValue RegClass = CurDAG->getTargetConstant(ARM::QQPRRegClassID, MVT::i32);
-  SDValue SubReg0 = CurDAG->getTargetConstant(ARM::dsub_0, MVT::i32);
-  SDValue SubReg1 = CurDAG->getTargetConstant(ARM::dsub_1, MVT::i32);
-  SDValue SubReg2 = CurDAG->getTargetConstant(ARM::dsub_2, MVT::i32);
-  SDValue SubReg3 = CurDAG->getTargetConstant(ARM::dsub_3, MVT::i32);
+  SDValue RegClass = CurDAG->getTargetConstant(ARM::QQPRRegClassID, dl,
+                                               MVT::i32);
+  SDValue SubReg0 = CurDAG->getTargetConstant(ARM::dsub_0, dl, MVT::i32);
+  SDValue SubReg1 = CurDAG->getTargetConstant(ARM::dsub_1, dl, MVT::i32);
+  SDValue SubReg2 = CurDAG->getTargetConstant(ARM::dsub_2, dl, MVT::i32);
+  SDValue SubReg3 = CurDAG->getTargetConstant(ARM::dsub_3, dl, MVT::i32);
   const SDValue Ops[] = { RegClass, V0, SubReg0, V1, SubReg1,
                                     V2, SubReg2, V3, SubReg3 };
   return CurDAG->getMachineNode(TargetOpcode::REG_SEQUENCE, dl, VT, Ops);
@@ -1716,11 +1665,12 @@ SDNode *ARMDAGToDAGISel::createQuadDRegsNode(EVT VT, SDValue V0, SDValue V1,
 SDNode *ARMDAGToDAGISel::createQuadQRegsNode(EVT VT, SDValue V0, SDValue V1,
                                    SDValue V2, SDValue V3) {
   SDLoc dl(V0.getNode());
-  SDValue RegClass = CurDAG->getTargetConstant(ARM::QQQQPRRegClassID, MVT::i32);
-  SDValue SubReg0 = CurDAG->getTargetConstant(ARM::qsub_0, MVT::i32);
-  SDValue SubReg1 = CurDAG->getTargetConstant(ARM::qsub_1, MVT::i32);
-  SDValue SubReg2 = CurDAG->getTargetConstant(ARM::qsub_2, MVT::i32);
-  SDValue SubReg3 = CurDAG->getTargetConstant(ARM::qsub_3, MVT::i32);
+  SDValue RegClass = CurDAG->getTargetConstant(ARM::QQQQPRRegClassID, dl,
+                                               MVT::i32);
+  SDValue SubReg0 = CurDAG->getTargetConstant(ARM::qsub_0, dl, MVT::i32);
+  SDValue SubReg1 = CurDAG->getTargetConstant(ARM::qsub_1, dl, MVT::i32);
+  SDValue SubReg2 = CurDAG->getTargetConstant(ARM::qsub_2, dl, MVT::i32);
+  SDValue SubReg3 = CurDAG->getTargetConstant(ARM::qsub_3, dl, MVT::i32);
   const SDValue Ops[] = { RegClass, V0, SubReg0, V1, SubReg1,
                                     V2, SubReg2, V3, SubReg3 };
   return CurDAG->getMachineNode(TargetOpcode::REG_SEQUENCE, dl, VT, Ops);
@@ -1729,8 +1679,8 @@ SDNode *ARMDAGToDAGISel::createQuadQRegsNode(EVT VT, SDValue V0, SDValue V1,
 /// GetVLDSTAlign - Get the alignment (in bytes) for the alignment operand
 /// of a NEON VLD or VST instruction.  The supported values depend on the
 /// number of registers being loaded.
-SDValue ARMDAGToDAGISel::GetVLDSTAlign(SDValue Align, unsigned NumVecs,
-                                       bool is64BitVector) {
+SDValue ARMDAGToDAGISel::GetVLDSTAlign(SDValue Align, SDLoc dl,
+                                       unsigned NumVecs, bool is64BitVector) {
   unsigned NumRegs = NumVecs;
   if (!is64BitVector && NumVecs < 3)
     NumRegs *= 2;
@@ -1745,7 +1695,7 @@ SDValue ARMDAGToDAGISel::GetVLDSTAlign(SDValue Align, unsigned NumVecs,
   else
     Alignment = 0;
 
-  return CurDAG->getTargetConstant(Alignment, MVT::i32);
+  return CurDAG->getTargetConstant(Alignment, dl, MVT::i32);
 }
 
 static bool isVLDfixed(unsigned Opc)
@@ -1865,7 +1815,7 @@ SDNode *ARMDAGToDAGISel::SelectVLD(SDNode *N, bool isUpdating, unsigned NumVecs,
   SDValue Chain = N->getOperand(0);
   EVT VT = N->getValueType(0);
   bool is64BitVector = VT.is64BitVector();
-  Align = GetVLDSTAlign(Align, NumVecs, is64BitVector);
+  Align = GetVLDSTAlign(Align, dl, NumVecs, is64BitVector);
 
   unsigned OpcodeIndex;
   switch (VT.getSimpleVT().SimpleTy) {
@@ -1902,7 +1852,7 @@ SDNode *ARMDAGToDAGISel::SelectVLD(SDNode *N, bool isUpdating, unsigned NumVecs,
     ResTys.push_back(MVT::i32);
   ResTys.push_back(MVT::Other);
 
-  SDValue Pred = getAL(CurDAG);
+  SDValue Pred = getAL(CurDAG, dl);
   SDValue Reg0 = CurDAG->getRegister(0, MVT::i32);
   SDNode *VLd;
   SmallVector<SDValue, 7> Ops;
@@ -2002,7 +1952,7 @@ SDNode *ARMDAGToDAGISel::SelectVST(SDNode *N, bool isUpdating, unsigned NumVecs,
   SDValue Chain = N->getOperand(0);
   EVT VT = N->getOperand(Vec0Idx).getValueType();
   bool is64BitVector = VT.is64BitVector();
-  Align = GetVLDSTAlign(Align, NumVecs, is64BitVector);
+  Align = GetVLDSTAlign(Align, dl, NumVecs, is64BitVector);
 
   unsigned OpcodeIndex;
   switch (VT.getSimpleVT().SimpleTy) {
@@ -2029,7 +1979,7 @@ SDNode *ARMDAGToDAGISel::SelectVST(SDNode *N, bool isUpdating, unsigned NumVecs,
     ResTys.push_back(MVT::i32);
   ResTys.push_back(MVT::Other);
 
-  SDValue Pred = getAL(CurDAG);
+  SDValue Pred = getAL(CurDAG, dl);
   SDValue Reg0 = CurDAG->getRegister(0, MVT::i32);
   SmallVector<SDValue, 7> Ops;
 
@@ -2165,7 +2115,7 @@ SDNode *ARMDAGToDAGISel::SelectVLDSTLane(SDNode *N, bool IsLoad,
     if (Alignment == 1)
       Alignment = 0;
   }
-  Align = CurDAG->getTargetConstant(Alignment, MVT::i32);
+  Align = CurDAG->getTargetConstant(Alignment, dl, MVT::i32);
 
   unsigned OpcodeIndex;
   switch (VT.getSimpleVT().SimpleTy) {
@@ -2193,7 +2143,7 @@ SDNode *ARMDAGToDAGISel::SelectVLDSTLane(SDNode *N, bool IsLoad,
     ResTys.push_back(MVT::i32);
   ResTys.push_back(MVT::Other);
 
-  SDValue Pred = getAL(CurDAG);
+  SDValue Pred = getAL(CurDAG, dl);
   SDValue Reg0 = CurDAG->getRegister(0, MVT::i32);
 
   SmallVector<SDValue, 8> Ops;
@@ -2223,7 +2173,7 @@ SDNode *ARMDAGToDAGISel::SelectVLDSTLane(SDNode *N, bool IsLoad,
       SuperReg = SDValue(createQuadQRegsNode(MVT::v8i64, V0, V1, V2, V3), 0);
   }
   Ops.push_back(SuperReg);
-  Ops.push_back(getI32Imm(Lane));
+  Ops.push_back(getI32Imm(Lane, dl));
   Ops.push_back(Pred);
   Ops.push_back(Reg0);
   Ops.push_back(Chain);
@@ -2278,7 +2228,7 @@ SDNode *ARMDAGToDAGISel::SelectVLDDup(SDNode *N, bool isUpdating,
     if (Alignment == 1)
       Alignment = 0;
   }
-  Align = CurDAG->getTargetConstant(Alignment, MVT::i32);
+  Align = CurDAG->getTargetConstant(Alignment, dl, MVT::i32);
 
   unsigned OpcodeIndex;
   switch (VT.getSimpleVT().SimpleTy) {
@@ -2289,7 +2239,7 @@ SDNode *ARMDAGToDAGISel::SelectVLDDup(SDNode *N, bool isUpdating,
   case MVT::v2i32: OpcodeIndex = 2; break;
   }
 
-  SDValue Pred = getAL(CurDAG);
+  SDValue Pred = getAL(CurDAG, dl);
   SDValue Reg0 = CurDAG->getRegister(0, MVT::i32);
   SDValue SuperReg;
   unsigned Opc = Opcodes[OpcodeIndex];
@@ -2360,7 +2310,7 @@ SDNode *ARMDAGToDAGISel::SelectVTBL(SDNode *N, bool IsExt, unsigned NumVecs,
     Ops.push_back(N->getOperand(1));
   Ops.push_back(RegSeq);
   Ops.push_back(N->getOperand(FirstTblReg + NumVecs));
-  Ops.push_back(getAL(CurDAG)); // predicate
+  Ops.push_back(getAL(CurDAG, dl)); // predicate
   Ops.push_back(CurDAG->getRegister(0, MVT::i32)); // predicate register
   return CurDAG->getMachineNode(Opc, dl, VT, Ops);
 }
@@ -2373,6 +2323,7 @@ SDNode *ARMDAGToDAGISel::SelectV6T2BitfieldExtractOp(SDNode *N,
   unsigned Opc = isSigned
     ? (Subtarget->isThumb() ? ARM::t2SBFX : ARM::SBFX)
     : (Subtarget->isThumb() ? ARM::t2UBFX : ARM::UBFX);
+  SDLoc dl(N);
 
   // For unsigned extracts, check for a shift right and mask
   unsigned And_imm = 0;
@@ -2399,25 +2350,25 @@ SDNode *ARMDAGToDAGISel::SelectV6T2BitfieldExtractOp(SDNode *N,
           if (Subtarget->isThumb()) {
             Opc = isSigned ? ARM::t2ASRri : ARM::t2LSRri;
             SDValue Ops[] = { N->getOperand(0).getOperand(0),
-                              CurDAG->getTargetConstant(LSB, MVT::i32),
-                              getAL(CurDAG), Reg0, Reg0 };
+                              CurDAG->getTargetConstant(LSB, dl, MVT::i32),
+                              getAL(CurDAG, dl), Reg0, Reg0 };
             return CurDAG->SelectNodeTo(N, Opc, MVT::i32, Ops);
           }
 
           // ARM models shift instructions as MOVsi with shifter operand.
           ARM_AM::ShiftOpc ShOpcVal = ARM_AM::getShiftOpcForNode(ISD::SRL);
           SDValue ShOpc =
-            CurDAG->getTargetConstant(ARM_AM::getSORegOpc(ShOpcVal, LSB),
+            CurDAG->getTargetConstant(ARM_AM::getSORegOpc(ShOpcVal, LSB), dl,
                                       MVT::i32);
           SDValue Ops[] = { N->getOperand(0).getOperand(0), ShOpc,
-                            getAL(CurDAG), Reg0, Reg0 };
+                            getAL(CurDAG, dl), Reg0, Reg0 };
           return CurDAG->SelectNodeTo(N, ARM::MOVsi, MVT::i32, Ops);
         }
 
         SDValue Ops[] = { N->getOperand(0).getOperand(0),
-                          CurDAG->getTargetConstant(LSB, MVT::i32),
-                          CurDAG->getTargetConstant(Width, MVT::i32),
-                          getAL(CurDAG), Reg0 };
+                          CurDAG->getTargetConstant(LSB, dl, MVT::i32),
+                          CurDAG->getTargetConstant(Width, dl, MVT::i32),
+                          getAL(CurDAG, dl), Reg0 };
         return CurDAG->SelectNodeTo(N, Opc, MVT::i32, Ops);
       }
     }
@@ -2438,9 +2389,9 @@ SDNode *ARMDAGToDAGISel::SelectV6T2BitfieldExtractOp(SDNode *N,
         return nullptr;
       SDValue Reg0 = CurDAG->getRegister(0, MVT::i32);
       SDValue Ops[] = { N->getOperand(0).getOperand(0),
-                        CurDAG->getTargetConstant(LSB, MVT::i32),
-                        CurDAG->getTargetConstant(Width, MVT::i32),
-                        getAL(CurDAG), Reg0 };
+                        CurDAG->getTargetConstant(LSB, dl, MVT::i32),
+                        CurDAG->getTargetConstant(Width, dl, MVT::i32),
+                        getAL(CurDAG, dl), Reg0 };
       return CurDAG->SelectNodeTo(N, Opc, MVT::i32, Ops);
     }
   }
@@ -2457,9 +2408,9 @@ SDNode *ARMDAGToDAGISel::SelectV6T2BitfieldExtractOp(SDNode *N,
 
     SDValue Reg0 = CurDAG->getRegister(0, MVT::i32);
     SDValue Ops[] = { N->getOperand(0).getOperand(0),
-                      CurDAG->getTargetConstant(LSB, MVT::i32),
-                      CurDAG->getTargetConstant(Width - 1, MVT::i32),
-                      getAL(CurDAG), Reg0 };
+                      CurDAG->getTargetConstant(LSB, dl, MVT::i32),
+                      CurDAG->getTargetConstant(Width - 1, dl, MVT::i32),
+                      getAL(CurDAG, dl), Reg0 };
     return CurDAG->SelectNodeTo(N, Opc, MVT::i32, Ops);
   }
 
@@ -2524,6 +2475,18 @@ SDNode *ARMDAGToDAGISel::Select(SDNode *N) {
 
   switch (N->getOpcode()) {
   default: break;
+  case ISD::WRITE_REGISTER: {
+    SDNode *ResNode = SelectWriteRegister(N);
+    if (ResNode)
+      return ResNode;
+    break;
+  }
+  case ISD::READ_REGISTER: {
+    SDNode *ResNode = SelectReadRegister(N);
+    if (ResNode)
+      return ResNode;
+    break;
+  }
   case ISD::INLINEASM: {
     SDNode *ResNode = SelectInlineAsm(N);
     if (ResNode)
@@ -2558,16 +2521,14 @@ SDNode *ARMDAGToDAGISel::Select(SDNode *N) {
                  !(Subtarget->hasV6T2Ops() && Val <= 0xffff)); // MOVW
     }
 
-    if (!Subtarget->useConstIslands()) UseCP = false; // @LOCALMOD
-
     if (UseCP) {
       SDValue CPIdx = CurDAG->getTargetConstantPool(
           ConstantInt::get(Type::getInt32Ty(*CurDAG->getContext()), Val),
-          TLI->getPointerTy());
+          TLI->getPointerTy(CurDAG->getDataLayout()));
 
       SDNode *ResNode;
       if (Subtarget->isThumb()) {
-        SDValue Pred = getAL(CurDAG);
+        SDValue Pred = getAL(CurDAG, dl);
         SDValue PredReg = CurDAG->getRegister(0, MVT::i32);
         SDValue Ops[] = { CPIdx, Pred, PredReg, CurDAG->getEntryNode() };
         ResNode = CurDAG->getMachineNode(ARM::tLDRpci, dl, MVT::i32, MVT::Other,
@@ -2575,8 +2536,8 @@ SDNode *ARMDAGToDAGISel::Select(SDNode *N) {
       } else {
         SDValue Ops[] = {
           CPIdx,
-          CurDAG->getTargetConstant(0, MVT::i32),
-          getAL(CurDAG),
+          CurDAG->getTargetConstant(0, dl, MVT::i32),
+          getAL(CurDAG, dl),
           CurDAG->getRegister(0, MVT::i32),
           CurDAG->getEntryNode()
         };
@@ -2593,7 +2554,8 @@ SDNode *ARMDAGToDAGISel::Select(SDNode *N) {
   case ISD::FrameIndex: {
     // Selects to ADDri FI, 0 which in turn will become ADDri SP, imm.
     int FI = cast<FrameIndexSDNode>(N)->getIndex();
-    SDValue TFI = CurDAG->getTargetFrameIndex(FI, TLI->getPointerTy());
+    SDValue TFI = CurDAG->getTargetFrameIndex(
+        FI, TLI->getPointerTy(CurDAG->getDataLayout()));
     if (Subtarget->isThumb1Only()) {
       // Set the alignment of the frame object to 4, to avoid having to generate
       // more than one ADD
@@ -2601,12 +2563,12 @@ SDNode *ARMDAGToDAGISel::Select(SDNode *N) {
       if (MFI->getObjectAlignment(FI) < 4)
         MFI->setObjectAlignment(FI, 4);
       return CurDAG->SelectNodeTo(N, ARM::tADDframe, MVT::i32, TFI,
-                                  CurDAG->getTargetConstant(0, MVT::i32));
+                                  CurDAG->getTargetConstant(0, dl, MVT::i32));
     } else {
       unsigned Opc = ((Subtarget->isThumb() && Subtarget->hasThumb2()) ?
                       ARM::t2ADDri : ARM::ADDri);
-      SDValue Ops[] = { TFI, CurDAG->getTargetConstant(0, MVT::i32),
-                        getAL(CurDAG), CurDAG->getRegister(0, MVT::i32),
+      SDValue Ops[] = { TFI, CurDAG->getTargetConstant(0, dl, MVT::i32),
+                        getAL(CurDAG, dl), CurDAG->getRegister(0, MVT::i32),
                         CurDAG->getRegister(0, MVT::i32) };
       return CurDAG->SelectNodeTo(N, Opc, MVT::i32, Ops);
     }
@@ -2632,13 +2594,14 @@ SDNode *ARMDAGToDAGISel::Select(SDNode *N) {
           break;
         SDValue V = N->getOperand(0);
         ShImm = ARM_AM::getSORegOpc(ARM_AM::lsl, ShImm);
-        SDValue ShImmOp = CurDAG->getTargetConstant(ShImm, MVT::i32);
+        SDValue ShImmOp = CurDAG->getTargetConstant(ShImm, dl, MVT::i32);
         SDValue Reg0 = CurDAG->getRegister(0, MVT::i32);
         if (Subtarget->isThumb()) {
-          SDValue Ops[] = { V, V, ShImmOp, getAL(CurDAG), Reg0, Reg0 };
+          SDValue Ops[] = { V, V, ShImmOp, getAL(CurDAG, dl), Reg0, Reg0 };
           return CurDAG->SelectNodeTo(N, ARM::t2ADDrs, MVT::i32, Ops);
         } else {
-          SDValue Ops[] = { V, V, Reg0, ShImmOp, getAL(CurDAG), Reg0, Reg0 };
+          SDValue Ops[] = { V, V, Reg0, ShImmOp, getAL(CurDAG, dl), Reg0,
+                            Reg0 };
           return CurDAG->SelectNodeTo(N, ARM::ADDrsi, MVT::i32, Ops);
         }
       }
@@ -2648,13 +2611,14 @@ SDNode *ARMDAGToDAGISel::Select(SDNode *N) {
           break;
         SDValue V = N->getOperand(0);
         ShImm = ARM_AM::getSORegOpc(ARM_AM::lsl, ShImm);
-        SDValue ShImmOp = CurDAG->getTargetConstant(ShImm, MVT::i32);
+        SDValue ShImmOp = CurDAG->getTargetConstant(ShImm, dl, MVT::i32);
         SDValue Reg0 = CurDAG->getRegister(0, MVT::i32);
         if (Subtarget->isThumb()) {
-          SDValue Ops[] = { V, V, ShImmOp, getAL(CurDAG), Reg0, Reg0 };
+          SDValue Ops[] = { V, V, ShImmOp, getAL(CurDAG, dl), Reg0, Reg0 };
           return CurDAG->SelectNodeTo(N, ARM::t2RSBrs, MVT::i32, Ops);
         } else {
-          SDValue Ops[] = { V, V, Reg0, ShImmOp, getAL(CurDAG), Reg0, Reg0 };
+          SDValue Ops[] = { V, V, Reg0, ShImmOp, getAL(CurDAG, dl), Reg0,
+                            Reg0 };
           return CurDAG->SelectNodeTo(N, ARM::RSBrsi, MVT::i32, Ops);
         }
       }
@@ -2693,9 +2657,9 @@ SDNode *ARMDAGToDAGISel::Select(SDNode *N) {
           (N1CVal & 0xffffU) == 0xffffU &&
           (N2CVal & 0xffffU) == 0x0U) {
         SDValue Imm16 = CurDAG->getTargetConstant((N2CVal & 0xFFFF0000U) >> 16,
-                                                  MVT::i32);
+                                                  dl, MVT::i32);
         SDValue Ops[] = { N0.getOperand(0), Imm16,
-                          getAL(CurDAG), CurDAG->getRegister(0, MVT::i32) };
+                          getAL(CurDAG, dl), CurDAG->getRegister(0, MVT::i32) };
         return CurDAG->getMachineNode(Opc, dl, VT, Ops);
       }
     }
@@ -2703,18 +2667,18 @@ SDNode *ARMDAGToDAGISel::Select(SDNode *N) {
   }
   case ARMISD::VMOVRRD:
     return CurDAG->getMachineNode(ARM::VMOVRRD, dl, MVT::i32, MVT::i32,
-                                  N->getOperand(0), getAL(CurDAG),
+                                  N->getOperand(0), getAL(CurDAG, dl),
                                   CurDAG->getRegister(0, MVT::i32));
   case ISD::UMUL_LOHI: {
     if (Subtarget->isThumb1Only())
       break;
     if (Subtarget->isThumb()) {
       SDValue Ops[] = { N->getOperand(0), N->getOperand(1),
-                        getAL(CurDAG), CurDAG->getRegister(0, MVT::i32) };
+                        getAL(CurDAG, dl), CurDAG->getRegister(0, MVT::i32) };
       return CurDAG->getMachineNode(ARM::t2UMULL, dl, MVT::i32, MVT::i32, Ops);
     } else {
       SDValue Ops[] = { N->getOperand(0), N->getOperand(1),
-                        getAL(CurDAG), CurDAG->getRegister(0, MVT::i32),
+                        getAL(CurDAG, dl), CurDAG->getRegister(0, MVT::i32),
                         CurDAG->getRegister(0, MVT::i32) };
       return CurDAG->getMachineNode(Subtarget->hasV6Ops() ?
                                     ARM::UMULL : ARM::UMULLv5,
@@ -2726,11 +2690,11 @@ SDNode *ARMDAGToDAGISel::Select(SDNode *N) {
       break;
     if (Subtarget->isThumb()) {
       SDValue Ops[] = { N->getOperand(0), N->getOperand(1),
-                        getAL(CurDAG), CurDAG->getRegister(0, MVT::i32) };
+                        getAL(CurDAG, dl), CurDAG->getRegister(0, MVT::i32) };
       return CurDAG->getMachineNode(ARM::t2SMULL, dl, MVT::i32, MVT::i32, Ops);
     } else {
       SDValue Ops[] = { N->getOperand(0), N->getOperand(1),
-                        getAL(CurDAG), CurDAG->getRegister(0, MVT::i32),
+                        getAL(CurDAG, dl), CurDAG->getRegister(0, MVT::i32),
                         CurDAG->getRegister(0, MVT::i32) };
       return CurDAG->getMachineNode(Subtarget->hasV6Ops() ?
                                     ARM::SMULL : ARM::SMULLv5,
@@ -2740,12 +2704,12 @@ SDNode *ARMDAGToDAGISel::Select(SDNode *N) {
   case ARMISD::UMLAL:{
     if (Subtarget->isThumb()) {
       SDValue Ops[] = { N->getOperand(0), N->getOperand(1), N->getOperand(2),
-                        N->getOperand(3), getAL(CurDAG),
+                        N->getOperand(3), getAL(CurDAG, dl),
                         CurDAG->getRegister(0, MVT::i32)};
       return CurDAG->getMachineNode(ARM::t2UMLAL, dl, MVT::i32, MVT::i32, Ops);
     }else{
       SDValue Ops[] = { N->getOperand(0), N->getOperand(1), N->getOperand(2),
-                        N->getOperand(3), getAL(CurDAG),
+                        N->getOperand(3), getAL(CurDAG, dl),
                         CurDAG->getRegister(0, MVT::i32),
                         CurDAG->getRegister(0, MVT::i32) };
       return CurDAG->getMachineNode(Subtarget->hasV6Ops() ?
@@ -2756,12 +2720,12 @@ SDNode *ARMDAGToDAGISel::Select(SDNode *N) {
   case ARMISD::SMLAL:{
     if (Subtarget->isThumb()) {
       SDValue Ops[] = { N->getOperand(0), N->getOperand(1), N->getOperand(2),
-                        N->getOperand(3), getAL(CurDAG),
+                        N->getOperand(3), getAL(CurDAG, dl),
                         CurDAG->getRegister(0, MVT::i32)};
       return CurDAG->getMachineNode(ARM::t2SMLAL, dl, MVT::i32, MVT::i32, Ops);
     }else{
       SDValue Ops[] = { N->getOperand(0), N->getOperand(1), N->getOperand(2),
-                        N->getOperand(3), getAL(CurDAG),
+                        N->getOperand(3), getAL(CurDAG, dl),
                         CurDAG->getRegister(0, MVT::i32),
                         CurDAG->getRegister(0, MVT::i32) };
       return CurDAG->getMachineNode(Subtarget->hasV6Ops() ?
@@ -2805,7 +2769,7 @@ SDNode *ARMDAGToDAGISel::Select(SDNode *N) {
     assert(N3.getOpcode() == ISD::Register);
 
     SDValue Tmp2 = CurDAG->getTargetConstant(((unsigned)
-                               cast<ConstantSDNode>(N2)->getZExtValue()),
+                               cast<ConstantSDNode>(N2)->getZExtValue()), dl,
                                MVT::i32);
     SDValue Ops[] = { N1, Tmp2, N3, Chain, InFlag };
     SDNode *ResNode = CurDAG->getMachineNode(Opc, dl, MVT::Other,
@@ -2834,7 +2798,7 @@ SDNode *ARMDAGToDAGISel::Select(SDNode *N) {
     case MVT::v4f32:
     case MVT::v4i32: Opc = ARM::VZIPq32; break;
     }
-    SDValue Pred = getAL(CurDAG);
+    SDValue Pred = getAL(CurDAG, dl);
     SDValue PredReg = CurDAG->getRegister(0, MVT::i32);
     SDValue Ops[] = { N->getOperand(0), N->getOperand(1), Pred, PredReg };
     return CurDAG->getMachineNode(Opc, dl, VT, VT, Ops);
@@ -2854,7 +2818,7 @@ SDNode *ARMDAGToDAGISel::Select(SDNode *N) {
     case MVT::v4f32:
     case MVT::v4i32: Opc = ARM::VUZPq32; break;
     }
-    SDValue Pred = getAL(CurDAG);
+    SDValue Pred = getAL(CurDAG, dl);
     SDValue PredReg = CurDAG->getRegister(0, MVT::i32);
     SDValue Ops[] = { N->getOperand(0), N->getOperand(1), Pred, PredReg };
     return CurDAG->getMachineNode(Opc, dl, VT, VT, Ops);
@@ -2873,7 +2837,7 @@ SDNode *ARMDAGToDAGISel::Select(SDNode *N) {
     case MVT::v4f32:
     case MVT::v4i32: Opc = ARM::VTRNq32; break;
     }
-    SDValue Pred = getAL(CurDAG);
+    SDValue Pred = getAL(CurDAG, dl);
     SDValue PredReg = CurDAG->getRegister(0, MVT::i32);
     SDValue Ops[] = { N->getOperand(0), N->getOperand(1), Pred, PredReg };
     return CurDAG->getMachineNode(Opc, dl, VT, VT, Ops);
@@ -3121,7 +3085,7 @@ SDNode *ARMDAGToDAGISel::Select(SDNode *N) {
       // Place arguments in the right order.
       SmallVector<SDValue, 7> Ops;
       Ops.push_back(MemAddr);
-      Ops.push_back(getAL(CurDAG));
+      Ops.push_back(getAL(CurDAG, dl));
       Ops.push_back(CurDAG->getRegister(0, MVT::i32));
       Ops.push_back(Chain);
       SDNode *Ld = CurDAG->getMachineNode(NewOpc, dl, ResTys, Ops);
@@ -3137,7 +3101,8 @@ SDNode *ARMDAGToDAGISel::Select(SDNode *N) {
         if (isThumb)
           Result = SDValue(Ld, 0);
         else {
-          SDValue SubRegIdx = CurDAG->getTargetConstant(ARM::gsub_0, MVT::i32);
+          SDValue SubRegIdx =
+            CurDAG->getTargetConstant(ARM::gsub_0, dl, MVT::i32);
           SDNode *ResNode = CurDAG->getMachineNode(TargetOpcode::EXTRACT_SUBREG,
               dl, MVT::i32, SDValue(Ld, 0), SubRegIdx);
           Result = SDValue(ResNode,0);
@@ -3149,7 +3114,8 @@ SDNode *ARMDAGToDAGISel::Select(SDNode *N) {
         if (isThumb)
           Result = SDValue(Ld, 1);
         else {
-          SDValue SubRegIdx = CurDAG->getTargetConstant(ARM::gsub_1, MVT::i32);
+          SDValue SubRegIdx =
+            CurDAG->getTargetConstant(ARM::gsub_1, dl, MVT::i32);
           SDNode *ResNode = CurDAG->getMachineNode(TargetOpcode::EXTRACT_SUBREG,
               dl, MVT::i32, SDValue(Ld, 0), SubRegIdx);
           Result = SDValue(ResNode,0);
@@ -3181,7 +3147,7 @@ SDNode *ARMDAGToDAGISel::Select(SDNode *N) {
         // arm_strexd uses GPRPair.
         Ops.push_back(SDValue(createGPRPairNode(MVT::Untyped, Val0, Val1), 0));
       Ops.push_back(MemAddr);
-      Ops.push_back(getAL(CurDAG));
+      Ops.push_back(getAL(CurDAG, dl));
       Ops.push_back(CurDAG->getRegister(0, MVT::i32));
       Ops.push_back(Chain);
 
@@ -3373,7 +3339,7 @@ SDNode *ARMDAGToDAGISel::Select(SDNode *N) {
 
     Ops.push_back(N->getOperand(0));
     Ops.push_back(N->getOperand(1));
-    Ops.push_back(getAL(CurDAG));                    // Predicate
+    Ops.push_back(getAL(CurDAG, dl));                // Predicate
     Ops.push_back(CurDAG->getRegister(0, MVT::i32)); // Predicate Register
     return CurDAG->getMachineNode(ARM::VTBL1, dl, VT, Ops);
   }
@@ -3389,7 +3355,7 @@ SDNode *ARMDAGToDAGISel::Select(SDNode *N) {
     SmallVector<SDValue, 6> Ops;
     Ops.push_back(RegSeq);
     Ops.push_back(N->getOperand(2));
-    Ops.push_back(getAL(CurDAG));                    // Predicate
+    Ops.push_back(getAL(CurDAG, dl));                // Predicate
     Ops.push_back(CurDAG->getRegister(0, MVT::i32)); // Predicate Register
     return CurDAG->getMachineNode(ARM::VTBL2, dl, VT, Ops);
   }
@@ -3401,6 +3367,418 @@ SDNode *ARMDAGToDAGISel::Select(SDNode *N) {
   return SelectCode(N);
 }
 
+// Inspect a register string of the form
+// cp<coprocessor>:<opc1>:c<CRn>:c<CRm>:<opc2> (32bit) or
+// cp<coprocessor>:<opc1>:c<CRm> (64bit) inspect the fields of the string
+// and obtain the integer operands from them, adding these operands to the
+// provided vector.
+static void getIntOperandsFromRegisterString(StringRef RegString,
+                                             SelectionDAG *CurDAG, SDLoc DL,
+                                             std::vector<SDValue>& Ops) {
+  SmallVector<StringRef, 5> Fields;
+  RegString.split(Fields, ":");
+
+  if (Fields.size() > 1) {
+    bool AllIntFields = true;
+
+    for (StringRef Field : Fields) {
+      // Need to trim out leading 'cp' characters and get the integer field.
+      unsigned IntField;
+      AllIntFields &= !Field.trim("CPcp").getAsInteger(10, IntField);
+      Ops.push_back(CurDAG->getTargetConstant(IntField, DL, MVT::i32));
+    }
+
+    assert(AllIntFields &&
+            "Unexpected non-integer value in special register string.");
+  }
+}
+
+// Maps a Banked Register string to its mask value. The mask value returned is
+// for use in the MRSbanked / MSRbanked instruction nodes as the Banked Register
+// mask operand, which expresses which register is to be used, e.g. r8, and in
+// which mode it is to be used, e.g. usr. Returns -1 to signify that the string
+// was invalid.
+static inline int getBankedRegisterMask(StringRef RegString) {
+  return StringSwitch<int>(RegString.lower())
+          .Case("r8_usr", 0x00)
+          .Case("r9_usr", 0x01)
+          .Case("r10_usr", 0x02)
+          .Case("r11_usr", 0x03)
+          .Case("r12_usr", 0x04)
+          .Case("sp_usr", 0x05)
+          .Case("lr_usr", 0x06)
+          .Case("r8_fiq", 0x08)
+          .Case("r9_fiq", 0x09)
+          .Case("r10_fiq", 0x0a)
+          .Case("r11_fiq", 0x0b)
+          .Case("r12_fiq", 0x0c)
+          .Case("sp_fiq", 0x0d)
+          .Case("lr_fiq", 0x0e)
+          .Case("lr_irq", 0x10)
+          .Case("sp_irq", 0x11)
+          .Case("lr_svc", 0x12)
+          .Case("sp_svc", 0x13)
+          .Case("lr_abt", 0x14)
+          .Case("sp_abt", 0x15)
+          .Case("lr_und", 0x16)
+          .Case("sp_und", 0x17)
+          .Case("lr_mon", 0x1c)
+          .Case("sp_mon", 0x1d)
+          .Case("elr_hyp", 0x1e)
+          .Case("sp_hyp", 0x1f)
+          .Case("spsr_fiq", 0x2e)
+          .Case("spsr_irq", 0x30)
+          .Case("spsr_svc", 0x32)
+          .Case("spsr_abt", 0x34)
+          .Case("spsr_und", 0x36)
+          .Case("spsr_mon", 0x3c)
+          .Case("spsr_hyp", 0x3e)
+          .Default(-1);
+}
+
+// Maps a MClass special register string to its value for use in the
+// t2MRS_M / t2MSR_M instruction nodes as the SYSm value operand.
+// Returns -1 to signify that the string was invalid.
+static inline int getMClassRegisterSYSmValueMask(StringRef RegString) {
+  return StringSwitch<int>(RegString.lower())
+          .Case("apsr", 0x0)
+          .Case("iapsr", 0x1)
+          .Case("eapsr", 0x2)
+          .Case("xpsr", 0x3)
+          .Case("ipsr", 0x5)
+          .Case("epsr", 0x6)
+          .Case("iepsr", 0x7)
+          .Case("msp", 0x8)
+          .Case("psp", 0x9)
+          .Case("primask", 0x10)
+          .Case("basepri", 0x11)
+          .Case("basepri_max", 0x12)
+          .Case("faultmask", 0x13)
+          .Case("control", 0x14)
+          .Default(-1);
+}
+
+// The flags here are common to those allowed for apsr in the A class cores and
+// those allowed for the special registers in the M class cores. Returns a
+// value representing which flags were present, -1 if invalid.
+static inline int getMClassFlagsMask(StringRef Flags) {
+  if (Flags.empty())
+    return 0x3;
+
+  return StringSwitch<int>(Flags)
+          .Case("g", 0x1)
+          .Case("nzcvq", 0x2)
+          .Case("nzcvqg", 0x3)
+          .Default(-1);
+}
+
+static int getMClassRegisterMask(StringRef Reg, StringRef Flags, bool IsRead,
+                                 const ARMSubtarget *Subtarget) {
+  // Ensure that the register (without flags) was a valid M Class special
+  // register.
+  int SYSmvalue = getMClassRegisterSYSmValueMask(Reg);
+  if (SYSmvalue == -1)
+    return -1;
+
+  // basepri, basepri_max and faultmask are only valid for V7m.
+  if (!Subtarget->hasV7Ops() && SYSmvalue >= 0x11 && SYSmvalue <= 0x13)
+    return -1;
+
+  // If it was a read then we won't be expecting flags and so at this point
+  // we can return the mask.
+  if (IsRead) {
+    assert (Flags.empty() && "Unexpected flags for reading M class register.");
+    return SYSmvalue;
+  }
+
+  // We know we are now handling a write so need to get the mask for the flags.
+  int Mask = getMClassFlagsMask(Flags);
+
+  // Only apsr, iapsr, eapsr, xpsr can have flags. The other register values
+  // shouldn't have flags present.
+  if ((SYSmvalue < 0x4 && Mask == -1) || (SYSmvalue > 0x4 && !Flags.empty()))
+    return -1;
+
+  // The _g and _nzcvqg versions are only valid if the DSP extension is
+  // available.
+  if (!Subtarget->hasThumb2DSP() && (Mask & 0x2))
+    return -1;
+
+  // The register was valid so need to put the mask in the correct place
+  // (the flags need to be in bits 11-10) and combine with the SYSmvalue to
+  // construct the operand for the instruction node.
+  if (SYSmvalue < 0x4)
+    return SYSmvalue | Mask << 10;
+
+  return SYSmvalue;
+}
+
+static int getARClassRegisterMask(StringRef Reg, StringRef Flags) {
+  // The mask operand contains the special register (R Bit) in bit 4, whether
+  // the register is spsr (R bit is 1) or one of cpsr/apsr (R bit is 0), and
+  // bits 3-0 contains the fields to be accessed in the special register, set by
+  // the flags provided with the register.
+  int Mask = 0;
+  if (Reg == "apsr") {
+    // The flags permitted for apsr are the same flags that are allowed in
+    // M class registers. We get the flag value and then shift the flags into
+    // the correct place to combine with the mask.
+    Mask = getMClassFlagsMask(Flags);
+    if (Mask == -1)
+      return -1;
+    return Mask << 2;
+  }
+
+  if (Reg != "cpsr" && Reg != "spsr") {
+    return -1;
+  }
+
+  // This is the same as if the flags were "fc"
+  if (Flags.empty() || Flags == "all")
+    return Mask | 0x9;
+
+  // Inspect the supplied flags string and set the bits in the mask for
+  // the relevant and valid flags allowed for cpsr and spsr.
+  for (char Flag : Flags) {
+    int FlagVal;
+    switch (Flag) {
+      case 'c':
+        FlagVal = 0x1;
+        break;
+      case 'x':
+        FlagVal = 0x2;
+        break;
+      case 's':
+        FlagVal = 0x4;
+        break;
+      case 'f':
+        FlagVal = 0x8;
+        break;
+      default:
+        FlagVal = 0;
+    }
+
+    // This avoids allowing strings where the same flag bit appears twice.
+    if (!FlagVal || (Mask & FlagVal))
+      return -1;
+    Mask |= FlagVal;
+  }
+
+  // If the register is spsr then we need to set the R bit.
+  if (Reg == "spsr")
+    Mask |= 0x10;
+
+  return Mask;
+}
+
+// Lower the read_register intrinsic to ARM specific DAG nodes
+// using the supplied metadata string to select the instruction node to use
+// and the registers/masks to construct as operands for the node.
+SDNode *ARMDAGToDAGISel::SelectReadRegister(SDNode *N){
+  const MDNodeSDNode *MD = dyn_cast<MDNodeSDNode>(N->getOperand(1));
+  const MDString *RegString = dyn_cast<MDString>(MD->getMD()->getOperand(0));
+  bool IsThumb2 = Subtarget->isThumb2();
+  SDLoc DL(N);
+
+  std::vector<SDValue> Ops;
+  getIntOperandsFromRegisterString(RegString->getString(), CurDAG, DL, Ops);
+
+  if (!Ops.empty()) {
+    // If the special register string was constructed of fields (as defined
+    // in the ACLE) then need to lower to MRC node (32 bit) or
+    // MRRC node(64 bit), we can make the distinction based on the number of
+    // operands we have.
+    unsigned Opcode;
+    SmallVector<EVT, 3> ResTypes;
+    if (Ops.size() == 5){
+      Opcode = IsThumb2 ? ARM::t2MRC : ARM::MRC;
+      ResTypes.append({ MVT::i32, MVT::Other });
+    } else {
+      assert(Ops.size() == 3 &&
+              "Invalid number of fields in special register string.");
+      Opcode = IsThumb2 ? ARM::t2MRRC : ARM::MRRC;
+      ResTypes.append({ MVT::i32, MVT::i32, MVT::Other });
+    }
+
+    Ops.push_back(getAL(CurDAG, DL));
+    Ops.push_back(CurDAG->getRegister(0, MVT::i32));
+    Ops.push_back(N->getOperand(0));
+    return CurDAG->getMachineNode(Opcode, DL, ResTypes, Ops);
+  }
+
+  std::string SpecialReg = RegString->getString().lower();
+
+  int BankedReg = getBankedRegisterMask(SpecialReg);
+  if (BankedReg != -1) {
+    Ops = { CurDAG->getTargetConstant(BankedReg, DL, MVT::i32),
+            getAL(CurDAG, DL), CurDAG->getRegister(0, MVT::i32),
+            N->getOperand(0) };
+    return CurDAG->getMachineNode(IsThumb2 ? ARM::t2MRSbanked : ARM::MRSbanked,
+                                  DL, MVT::i32, MVT::Other, Ops);
+  }
+
+  // The VFP registers are read by creating SelectionDAG nodes with opcodes
+  // corresponding to the register that is being read from. So we switch on the
+  // string to find which opcode we need to use.
+  unsigned Opcode = StringSwitch<unsigned>(SpecialReg)
+                    .Case("fpscr", ARM::VMRS)
+                    .Case("fpexc", ARM::VMRS_FPEXC)
+                    .Case("fpsid", ARM::VMRS_FPSID)
+                    .Case("mvfr0", ARM::VMRS_MVFR0)
+                    .Case("mvfr1", ARM::VMRS_MVFR1)
+                    .Case("mvfr2", ARM::VMRS_MVFR2)
+                    .Case("fpinst", ARM::VMRS_FPINST)
+                    .Case("fpinst2", ARM::VMRS_FPINST2)
+                    .Default(0);
+
+  // If an opcode was found then we can lower the read to a VFP instruction.
+  if (Opcode) {
+    if (!Subtarget->hasVFP2())
+      return nullptr;
+    if (Opcode == ARM::VMRS_MVFR2 && !Subtarget->hasFPARMv8())
+      return nullptr;
+
+    Ops = { getAL(CurDAG, DL), CurDAG->getRegister(0, MVT::i32),
+            N->getOperand(0) };
+    return CurDAG->getMachineNode(Opcode, DL, MVT::i32, MVT::Other, Ops);
+  }
+
+  // If the target is M Class then need to validate that the register string
+  // is an acceptable value, so check that a mask can be constructed from the
+  // string.
+  if (Subtarget->isMClass()) {
+    int SYSmValue = getMClassRegisterMask(SpecialReg, "", true, Subtarget);
+    if (SYSmValue == -1)
+      return nullptr;
+
+    SDValue Ops[] = { CurDAG->getTargetConstant(SYSmValue, DL, MVT::i32),
+                      getAL(CurDAG, DL), CurDAG->getRegister(0, MVT::i32),
+                      N->getOperand(0) };
+    return CurDAG->getMachineNode(ARM::t2MRS_M, DL, MVT::i32, MVT::Other, Ops);
+  }
+
+  // Here we know the target is not M Class so we need to check if it is one
+  // of the remaining possible values which are apsr, cpsr or spsr.
+  if (SpecialReg == "apsr" || SpecialReg == "cpsr") {
+    Ops = { getAL(CurDAG, DL), CurDAG->getRegister(0, MVT::i32),
+            N->getOperand(0) };
+    return CurDAG->getMachineNode(IsThumb2 ? ARM::t2MRS_AR : ARM::MRS, DL,
+                                  MVT::i32, MVT::Other, Ops);
+  }
+
+  if (SpecialReg == "spsr") {
+    Ops = { getAL(CurDAG, DL), CurDAG->getRegister(0, MVT::i32),
+            N->getOperand(0) };
+    return CurDAG->getMachineNode(IsThumb2 ? ARM::t2MRSsys_AR : ARM::MRSsys,
+                                  DL, MVT::i32, MVT::Other, Ops);
+  }
+
+  return nullptr;
+}
+
+// Lower the write_register intrinsic to ARM specific DAG nodes
+// using the supplied metadata string to select the instruction node to use
+// and the registers/masks to use in the nodes
+SDNode *ARMDAGToDAGISel::SelectWriteRegister(SDNode *N){
+  const MDNodeSDNode *MD = dyn_cast<MDNodeSDNode>(N->getOperand(1));
+  const MDString *RegString = dyn_cast<MDString>(MD->getMD()->getOperand(0));
+  bool IsThumb2 = Subtarget->isThumb2();
+  SDLoc DL(N);
+
+  std::vector<SDValue> Ops;
+  getIntOperandsFromRegisterString(RegString->getString(), CurDAG, DL, Ops);
+
+  if (!Ops.empty()) {
+    // If the special register string was constructed of fields (as defined
+    // in the ACLE) then need to lower to MCR node (32 bit) or
+    // MCRR node(64 bit), we can make the distinction based on the number of
+    // operands we have.
+    unsigned Opcode;
+    if (Ops.size() == 5) {
+      Opcode = IsThumb2 ? ARM::t2MCR : ARM::MCR;
+      Ops.insert(Ops.begin()+2, N->getOperand(2));
+    } else {
+      assert(Ops.size() == 3 &&
+              "Invalid number of fields in special register string.");
+      Opcode = IsThumb2 ? ARM::t2MCRR : ARM::MCRR;
+      SDValue WriteValue[] = { N->getOperand(2), N->getOperand(3) };
+      Ops.insert(Ops.begin()+2, WriteValue, WriteValue+2);
+    }
+
+    Ops.push_back(getAL(CurDAG, DL));
+    Ops.push_back(CurDAG->getRegister(0, MVT::i32));
+    Ops.push_back(N->getOperand(0));
+
+    return CurDAG->getMachineNode(Opcode, DL, MVT::Other, Ops);
+  }
+
+  std::string SpecialReg = RegString->getString().lower();
+  int BankedReg = getBankedRegisterMask(SpecialReg);
+  if (BankedReg != -1) {
+    Ops = { CurDAG->getTargetConstant(BankedReg, DL, MVT::i32), N->getOperand(2),
+            getAL(CurDAG, DL), CurDAG->getRegister(0, MVT::i32),
+            N->getOperand(0) };
+    return CurDAG->getMachineNode(IsThumb2 ? ARM::t2MSRbanked : ARM::MSRbanked,
+                                  DL, MVT::Other, Ops);
+  }
+
+  // The VFP registers are written to by creating SelectionDAG nodes with
+  // opcodes corresponding to the register that is being written. So we switch
+  // on the string to find which opcode we need to use.
+  unsigned Opcode = StringSwitch<unsigned>(SpecialReg)
+                    .Case("fpscr", ARM::VMSR)
+                    .Case("fpexc", ARM::VMSR_FPEXC)
+                    .Case("fpsid", ARM::VMSR_FPSID)
+                    .Case("fpinst", ARM::VMSR_FPINST)
+                    .Case("fpinst2", ARM::VMSR_FPINST2)
+                    .Default(0);
+
+  if (Opcode) {
+    if (!Subtarget->hasVFP2())
+      return nullptr;
+    Ops = { N->getOperand(2), getAL(CurDAG, DL),
+            CurDAG->getRegister(0, MVT::i32), N->getOperand(0) };
+    return CurDAG->getMachineNode(Opcode, DL, MVT::Other, Ops);
+  }
+
+  SmallVector<StringRef, 5> Fields;
+  StringRef(SpecialReg).split(Fields, "_", 1, false);
+  std::string Reg = Fields[0].str();
+  StringRef Flags = Fields.size() == 2 ? Fields[1] : "";
+
+  // If the target was M Class then need to validate the special register value
+  // and retrieve the mask for use in the instruction node.
+  if (Subtarget->isMClass()) {
+    // basepri_max gets split so need to correct Reg and Flags.
+    if (SpecialReg == "basepri_max") {
+      Reg = SpecialReg;
+      Flags = "";
+    }
+    int SYSmValue = getMClassRegisterMask(Reg, Flags, false, Subtarget);
+    if (SYSmValue == -1)
+      return nullptr;
+
+    SDValue Ops[] = { CurDAG->getTargetConstant(SYSmValue, DL, MVT::i32),
+                      N->getOperand(2), getAL(CurDAG, DL),
+                      CurDAG->getRegister(0, MVT::i32), N->getOperand(0) };
+    return CurDAG->getMachineNode(ARM::t2MSR_M, DL, MVT::Other, Ops);
+  }
+
+  // We then check to see if a valid mask can be constructed for one of the
+  // register string values permitted for the A and R class cores. These values
+  // are apsr, spsr and cpsr; these are also valid on older cores.
+  int Mask = getARClassRegisterMask(Reg, Flags);
+  if (Mask != -1) {
+    Ops = { CurDAG->getTargetConstant(Mask, DL, MVT::i32), N->getOperand(2),
+            getAL(CurDAG, DL), CurDAG->getRegister(0, MVT::i32),
+            N->getOperand(0) };
+    return CurDAG->getMachineNode(IsThumb2 ? ARM::t2MSR_AR : ARM::MSR,
+                                  DL, MVT::Other, Ops);
+  }
+
+  return nullptr;
+}
+
 SDNode *ARMDAGToDAGISel::SelectInlineAsm(SDNode *N){
   std::vector<SDValue> AsmNodeOperands;
   unsigned Flag, Kind;
@@ -3534,7 +3912,7 @@ SDNode *ARMDAGToDAGISel::SelectInlineAsm(SDNode *N){
         Flag = InlineAsm::getFlagWordForRegClass(Flag, ARM::GPRPairRegClassID);
       // Replace the current flag.
       AsmNodeOperands[AsmNodeOperands.size() -1] = CurDAG->getTargetConstant(
-          Flag, MVT::i32);
+          Flag, dl, MVT::i32);
       // Add the new register node and skip the original two GPRs.
       AsmNodeOperands.push_back(PairedReg);
       // Skip the next two GPRs.
@@ -3557,13 +3935,29 @@ SDNode *ARMDAGToDAGISel::SelectInlineAsm(SDNode *N){
 bool ARMDAGToDAGISel::
 SelectInlineAsmMemoryOperand(const SDValue &Op, unsigned ConstraintID,
                              std::vector<SDValue> &OutOps) {
-  assert(ConstraintID == InlineAsm::Constraint_m &&
-         "unexpected asm memory constraint");
-  // Require the address to be in a register.  That is safe for all ARM
-  // variants and it is hard to do anything much smarter without knowing
-  // how the operand is used.
-  OutOps.push_back(Op);
-  return false;
+  switch(ConstraintID) {
+  default:
+    llvm_unreachable("Unexpected asm memory constraint");
+  case InlineAsm::Constraint_i:
+    // FIXME: It seems strange that 'i' is needed here since it's supposed to
+    //        be an immediate and not a memory constraint.
+    // Fallthrough.
+  case InlineAsm::Constraint_m:
+  case InlineAsm::Constraint_Q:
+  case InlineAsm::Constraint_Um:
+  case InlineAsm::Constraint_Un:
+  case InlineAsm::Constraint_Uq:
+  case InlineAsm::Constraint_Us:
+  case InlineAsm::Constraint_Ut:
+  case InlineAsm::Constraint_Uv:
+  case InlineAsm::Constraint_Uy:
+    // Require the address to be in a register.  That is safe for all ARM
+    // variants and it is hard to do anything much smarter without knowing
+    // how the operand is used.
+    OutOps.push_back(Op);
+    return false;
+  }
+  return true;
 }
 
 /// createARMISelDag - This pass converts a legalized DAG into a
diff --git a/lib/Target/ARM/ARMISelLowering.cpp b/lib/Target/ARM/ARMISelLowering.cpp
index 3e402d7a1c8..49ebbfb1d91 100644
--- a/lib/Target/ARM/ARMISelLowering.cpp
+++ b/lib/Target/ARM/ARMISelLowering.cpp
@@ -52,7 +52,6 @@
 #include "llvm/Support/raw_ostream.h"
 #include "llvm/Target/TargetOptions.h"
 #include <utility>
-
 using namespace llvm;
 
 #define DEBUG_TYPE "arm-isel"
@@ -61,11 +60,6 @@ STATISTIC(NumTailCalls, "Number of tail calls");
 STATISTIC(NumMovwMovt, "Number of GAs materialized with movw + movt");
 STATISTIC(NumLoopByVals, "Number of loops generated for byval arguments");
 
-cl::opt<bool>
-EnableARMLongCalls("arm-long-calls", cl::Hidden,
-  cl::desc("Generate calls via indirect call instructions"),
-  cl::init(false));
-
 static cl::opt<bool>
 ARMInterworking("arm-interworking", cl::Hidden,
   cl::desc("Enable / disable ARM interworking (for debugging only)"),
@@ -148,6 +142,11 @@ void ARMTargetLowering::addTypeForNEON(MVT VT, MVT PromotedLdStVT,
   setOperationAction(ISD::SREM, VT, Expand);
   setOperationAction(ISD::UREM, VT, Expand);
   setOperationAction(ISD::FREM, VT, Expand);
+
+  if (VT.isInteger()) {
+    setOperationAction(ISD::SABSDIFF, VT, Legal);
+    setOperationAction(ISD::UABSDIFF, VT, Legal);
+  }
 }
 
 void ARMTargetLowering::addDRTypeForNEON(MVT VT) {
@@ -171,7 +170,7 @@ ARMTargetLowering::ARMTargetLowering(const TargetMachine &TM,
   if (Subtarget->isTargetMachO()) {
     // Uses VFP for Thumb libfuncs if available.
     if (Subtarget->isThumb() && Subtarget->hasVFP2() &&
-        Subtarget->hasARMOps() && !TM.Options.UseSoftFloat) {
+        Subtarget->hasARMOps() && !Subtarget->useSoftFloat()) {
       // Single-precision floating-point arithmetic.
       setLibcallName(RTLIB::ADD_F32, "__addsf3vfp");
       setLibcallName(RTLIB::SUB_F32, "__subsf3vfp");
@@ -402,7 +401,7 @@ ARMTargetLowering::ARMTargetLowering(const TargetMachine &TM,
     addRegisterClass(MVT::i32, &ARM::tGPRRegClass);
   else
     addRegisterClass(MVT::i32, &ARM::GPRRegClass);
-  if (!TM.Options.UseSoftFloat && Subtarget->hasVFP2() &&
+  if (!Subtarget->useSoftFloat() && Subtarget->hasVFP2() &&
       !Subtarget->isThumb1Only()) {
     addRegisterClass(MVT::f32, &ARM::SPRRegClass);
     addRegisterClass(MVT::f64, &ARM::DPRRegClass);
@@ -427,6 +426,9 @@ ARMTargetLowering::ARMTargetLowering(const TargetMachine &TM,
   setOperationAction(ISD::ConstantFP, MVT::f32, Custom);
   setOperationAction(ISD::ConstantFP, MVT::f64, Custom);
 
+  setOperationAction(ISD::READ_REGISTER, MVT::i64, Custom);
+  setOperationAction(ISD::WRITE_REGISTER, MVT::i64, Custom);
+
   if (Subtarget->hasNEON()) {
     addDRTypeForNEON(MVT::v2f32);
     addDRTypeForNEON(MVT::v8i8);
@@ -546,6 +548,27 @@ ARMTargetLowering::ARMTargetLowering(const TargetMachine &TM,
     setOperationAction(ISD::CTPOP,      MVT::v4i16, Custom);
     setOperationAction(ISD::CTPOP,      MVT::v8i16, Custom);
 
+    // NEON does not have single instruction CTTZ for vectors.
+    setOperationAction(ISD::CTTZ, MVT::v8i8, Custom);
+    setOperationAction(ISD::CTTZ, MVT::v4i16, Custom);
+    setOperationAction(ISD::CTTZ, MVT::v2i32, Custom);
+    setOperationAction(ISD::CTTZ, MVT::v1i64, Custom);
+
+    setOperationAction(ISD::CTTZ, MVT::v16i8, Custom);
+    setOperationAction(ISD::CTTZ, MVT::v8i16, Custom);
+    setOperationAction(ISD::CTTZ, MVT::v4i32, Custom);
+    setOperationAction(ISD::CTTZ, MVT::v2i64, Custom);
+
+    setOperationAction(ISD::CTTZ_ZERO_UNDEF, MVT::v8i8, Custom);
+    setOperationAction(ISD::CTTZ_ZERO_UNDEF, MVT::v4i16, Custom);
+    setOperationAction(ISD::CTTZ_ZERO_UNDEF, MVT::v2i32, Custom);
+    setOperationAction(ISD::CTTZ_ZERO_UNDEF, MVT::v1i64, Custom);
+
+    setOperationAction(ISD::CTTZ_ZERO_UNDEF, MVT::v16i8, Custom);
+    setOperationAction(ISD::CTTZ_ZERO_UNDEF, MVT::v8i16, Custom);
+    setOperationAction(ISD::CTTZ_ZERO_UNDEF, MVT::v4i32, Custom);
+    setOperationAction(ISD::CTTZ_ZERO_UNDEF, MVT::v2i64, Custom);
+
     // NEON only has FMA instructions as of VFP4.
     if (!Subtarget->hasVFP4()) {
       setOperationAction(ISD::FMA, MVT::v2f32, Expand);
@@ -712,11 +735,11 @@ ARMTargetLowering::ARMTargetLowering(const TargetMachine &TM,
     setOperationAction(ISD::UDIV,  MVT::i32, Expand);
   }
 
-  // FIXME: Also set divmod for SREM on EABI
+  // FIXME: Also set divmod for SREM on EABI/androideabi
   setOperationAction(ISD::SREM,  MVT::i32, Expand);
   setOperationAction(ISD::UREM,  MVT::i32, Expand);
   // Register based DivRem for AEABI (RTABI 4.2)
-  if (Subtarget->isTargetAEABI()) {
+  if (Subtarget->isTargetAEABI() || Subtarget->isTargetAndroid()) {
     setLibcallName(RTLIB::SDIVREM_I8,  "__aeabi_idivmod");
     setLibcallName(RTLIB::SDIVREM_I16, "__aeabi_idivmod");
     setLibcallName(RTLIB::SDIVREM_I32, "__aeabi_idivmod");
@@ -747,11 +770,7 @@ ARMTargetLowering::ARMTargetLowering(const TargetMachine &TM,
   setOperationAction(ISD::GLOBAL_OFFSET_TABLE, MVT::i32, Custom);
   setOperationAction(ISD::GlobalTLSAddress, MVT::i32, Custom);
   setOperationAction(ISD::BlockAddress, MVT::i32, Custom);
-  // @LOCALMOD-START
-  if (!Subtarget->useInlineJumpTables())
-    setOperationAction(ISD::JumpTable,     MVT::i32,   Custom);
-  // @LOCALMOD-END
-  
+
   setOperationAction(ISD::TRAP, MVT::Other, Legal);
 
   // Use the default implementation.
@@ -765,22 +784,8 @@ ARMTargetLowering::ARMTargetLowering(const TargetMachine &TM,
   if (!Subtarget->isTargetMachO()) {
     // Non-MachO platforms may return values in these registers via the
     // personality function.
-    // @LOCALMOD-START
-    if (Subtarget->isTargetNaCl()) {
-      // we use the first caller saved regs here
-      // c.f.: llvm-gcc/llvm-gcc-4.2/gcc/unwind-dw2.c::uw_install_context
-      // NOTE: these are related to the _Unwind_PNaClSetResult{0,1} functions
-      setExceptionPointerRegister(ARM::R4);
-      setExceptionSelectorRegister(ARM::R5);
-  
-      setOperationAction(ISD::FRAME_TO_ARGS_OFFSET, MVT::i32, Custom);
-
-      setOperationAction(ISD::EH_RETURN, MVT::Other, Custom);
-    } else {
-      setExceptionPointerRegister(ARM::R0);
-      setExceptionSelectorRegister(ARM::R1);
-    }
-    // @LOCALMOD-END
+    setExceptionPointerRegister(ARM::R0);
+    setExceptionSelectorRegister(ARM::R1);
   }
 
   if (Subtarget->getTargetTriple().isWindowsItaniumEnvironment())
@@ -839,7 +844,7 @@ ARMTargetLowering::ARMTargetLowering(const TargetMachine &TM,
   }
   setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::i1, Expand);
 
-  if (!TM.Options.UseSoftFloat && Subtarget->hasVFP2() &&
+  if (!Subtarget->useSoftFloat() && Subtarget->hasVFP2() &&
       !Subtarget->isThumb1Only()) {
     // Turn f64->i64 into VMOVRRD, i64 -> f64 to VMOVDRR
     // iff target supports vfp2.
@@ -849,11 +854,11 @@ ARMTargetLowering::ARMTargetLowering(const TargetMachine &TM,
 
   // We want to custom lower some of our intrinsics.
   setOperationAction(ISD::INTRINSIC_WO_CHAIN, MVT::Other, Custom);
-  if (Subtarget->isTargetDarwin()) {
-    setOperationAction(ISD::EH_SJLJ_SETJMP, MVT::i32, Custom);
-    setOperationAction(ISD::EH_SJLJ_LONGJMP, MVT::Other, Custom);
+  setOperationAction(ISD::EH_SJLJ_SETJMP, MVT::i32, Custom);
+  setOperationAction(ISD::EH_SJLJ_LONGJMP, MVT::Other, Custom);
+  setOperationAction(ISD::EH_SJLJ_SETUP_DISPATCH, MVT::Other, Custom);
+  if (Subtarget->isTargetDarwin())
     setLibcallName(RTLIB::UNWIND_RESUME, "_Unwind_SjLj_Resume");
-  }
 
   setOperationAction(ISD::SETCC,     MVT::i32, Expand);
   setOperationAction(ISD::SETCC,     MVT::f32, Expand);
@@ -869,12 +874,8 @@ ARMTargetLowering::ARMTargetLowering(const TargetMachine &TM,
   setOperationAction(ISD::BR_CC,     MVT::i32,   Custom);
   setOperationAction(ISD::BR_CC,     MVT::f32,   Custom);
   setOperationAction(ISD::BR_CC,     MVT::f64,   Custom);
-  // @LOCALMOD-START
-  //setOperationAction(ISD::BR_JT,     MVT::Other, Custom);
-  setOperationAction(ISD::BR_JT,     MVT::Other,
-                     Subtarget->useInlineJumpTables() ? Custom : Expand);
-  // @LOCALMOD-END
-  
+  setOperationAction(ISD::BR_JT,     MVT::Other, Custom);
+
   // We don't support sin/cos/fmod/copysign/pow
   setOperationAction(ISD::FSIN,      MVT::f64, Expand);
   setOperationAction(ISD::FSIN,      MVT::f32, Expand);
@@ -884,7 +885,7 @@ ARMTargetLowering::ARMTargetLowering(const TargetMachine &TM,
   setOperationAction(ISD::FSINCOS,   MVT::f32, Expand);
   setOperationAction(ISD::FREM,      MVT::f64, Expand);
   setOperationAction(ISD::FREM,      MVT::f32, Expand);
-  if (!TM.Options.UseSoftFloat && Subtarget->hasVFP2() &&
+  if (!Subtarget->useSoftFloat() && Subtarget->hasVFP2() &&
       !Subtarget->isThumb1Only()) {
     setOperationAction(ISD::FCOPYSIGN, MVT::f64, Custom);
     setOperationAction(ISD::FCOPYSIGN, MVT::f32, Custom);
@@ -898,7 +899,7 @@ ARMTargetLowering::ARMTargetLowering(const TargetMachine &TM,
   }
 
   // Various VFP goodness
-  if (!TM.Options.UseSoftFloat && !Subtarget->isThumb1Only()) {
+  if (!Subtarget->useSoftFloat() && !Subtarget->isThumb1Only()) {
     // FP-ARMv8 adds f64 <-> f16 conversion. Before that it should be expanded.
     if (!Subtarget->hasFPARMv8() || Subtarget->isFPOnlySP()) {
       setOperationAction(ISD::FP16_TO_FP, MVT::f64, Expand);
@@ -955,7 +956,7 @@ ARMTargetLowering::ARMTargetLowering(const TargetMachine &TM,
 
   setStackPointerRegisterToSaveRestore(ARM::SP);
 
-  if (TM.Options.UseSoftFloat || Subtarget->isThumb1Only() ||
+  if (Subtarget->useSoftFloat() || Subtarget->isThumb1Only() ||
       !Subtarget->hasVFP2())
     setSchedulingPreference(Sched::RegPressure);
   else
@@ -963,11 +964,11 @@ ARMTargetLowering::ARMTargetLowering(const TargetMachine &TM,
 
   //// temporary - rewrite interface to use type
   MaxStoresPerMemset = 8;
-  MaxStoresPerMemsetOptSize = Subtarget->isTargetDarwin() ? 8 : 4;
+  MaxStoresPerMemsetOptSize = 4;
   MaxStoresPerMemcpy = 4; // For @llvm.memcpy -> sequence of stores
-  MaxStoresPerMemcpyOptSize = Subtarget->isTargetDarwin() ? 4 : 2;
+  MaxStoresPerMemcpyOptSize = 2;
   MaxStoresPerMemmove = 4; // For @llvm.memmove -> sequence of stores
-  MaxStoresPerMemmoveOptSize = Subtarget->isTargetDarwin() ? 4 : 2;
+  MaxStoresPerMemmoveOptSize = 2;
 
   // On ARM arguments smaller than 4 bytes are extended, so all arguments
   // are at least 4 bytes aligned.
@@ -979,6 +980,10 @@ ARMTargetLowering::ARMTargetLowering(const TargetMachine &TM,
   setMinFunctionAlignment(Subtarget->isThumb() ? 1 : 2);
 }
 
+bool ARMTargetLowering::useSoftFloat() const {
+  return Subtarget->useSoftFloat();
+}
+
 // FIXME: It might make sense to define the representative register class as the
 // nearest super-register that has a non-null superset. For example, DPR_VFP2 is
 // a super-register of SPR, and DPR is a superset if DPR_VFP2. Consequently,
@@ -1028,15 +1033,12 @@ ARMTargetLowering::findRepresentativeClass(const TargetRegisterInfo *TRI,
 }
 
 const char *ARMTargetLowering::getTargetNodeName(unsigned Opcode) const {
-  switch (Opcode) {
-  default: return nullptr;
+  switch ((ARMISD::NodeType)Opcode) {
+  case ARMISD::FIRST_NUMBER:  break;
   case ARMISD::Wrapper:       return "ARMISD::Wrapper";
   case ARMISD::WrapperPIC:    return "ARMISD::WrapperPIC";
   case ARMISD::WrapperJT:     return "ARMISD::WrapperJT";
-  // @LOCALMOD-START
-  case ARMISD::WrapperJT2:    return "ARMISD::WrapperJT2"; 
-  case ARMISD::EH_RETURN:     return "ARMISD::EH_RETURN"; 
-  // @LOCALMOD-END
+  case ARMISD::COPY_STRUCT_BYVAL: return "ARMISD::COPY_STRUCT_BYVAL";
   case ARMISD::CALL:          return "ARMISD::CALL";
   case ARMISD::CALL_PRED:     return "ARMISD::CALL_PRED";
   case ARMISD::CALL_NOLINK:   return "ARMISD::CALL_NOLINK";
@@ -1072,7 +1074,8 @@ const char *ARMTargetLowering::getTargetNodeName(unsigned Opcode) const {
   case ARMISD::VMOVDRR:       return "ARMISD::VMOVDRR";
 
   case ARMISD::EH_SJLJ_SETJMP: return "ARMISD::EH_SJLJ_SETJMP";
-  case ARMISD::EH_SJLJ_LONGJMP:return "ARMISD::EH_SJLJ_LONGJMP";
+  case ARMISD::EH_SJLJ_LONGJMP: return "ARMISD::EH_SJLJ_LONGJMP";
+  case ARMISD::EH_SJLJ_SETUP_DISPATCH: return "ARMISD::EH_SJLJ_SETUP_DISPATCH";
 
   case ARMISD::TC_RETURN:     return "ARMISD::TC_RETURN";
 
@@ -1113,6 +1116,8 @@ const char *ARMTargetLowering::getTargetNodeName(unsigned Opcode) const {
   case ARMISD::VQRSHRNs:      return "ARMISD::VQRSHRNs";
   case ARMISD::VQRSHRNu:      return "ARMISD::VQRSHRNu";
   case ARMISD::VQRSHRNsu:     return "ARMISD::VQRSHRNsu";
+  case ARMISD::VSLI:          return "ARMISD::VSLI";
+  case ARMISD::VSRI:          return "ARMISD::VSRI";
   case ARMISD::VGETLANEu:     return "ARMISD::VGETLANEu";
   case ARMISD::VGETLANEs:     return "ARMISD::VGETLANEs";
   case ARMISD::VMOVIMM:       return "ARMISD::VMOVIMM";
@@ -1163,10 +1168,13 @@ const char *ARMTargetLowering::getTargetNodeName(unsigned Opcode) const {
   case ARMISD::VST3LN_UPD:    return "ARMISD::VST3LN_UPD";
   case ARMISD::VST4LN_UPD:    return "ARMISD::VST4LN_UPD";
   }
+  return nullptr;
 }
 
-EVT ARMTargetLowering::getSetCCResultType(LLVMContext &, EVT VT) const {
-  if (!VT.isVector()) return getPointerTy();
+EVT ARMTargetLowering::getSetCCResultType(const DataLayout &DL, LLVMContext &,
+                                          EVT VT) const {
+  if (!VT.isVector())
+    return getPointerTy(DL);
   return VT.changeVectorElementTypeToInteger();
 }
 
@@ -1399,7 +1407,7 @@ ARMTargetLowering::LowerCallResult(SDValue Chain, SDValue InFlag,
       if (VA.getLocVT() == MVT::v2f64) {
         SDValue Vec = DAG.getNode(ISD::UNDEF, dl, MVT::v2f64);
         Vec = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, MVT::v2f64, Vec, Val,
-                          DAG.getConstant(0, MVT::i32));
+                          DAG.getConstant(0, dl, MVT::i32));
 
         VA = RVLocs[++i]; // skip ahead to next loc
         Lo = DAG.getCopyFromReg(Chain, dl, VA.getLocReg(), MVT::i32, InFlag);
@@ -1413,7 +1421,7 @@ ARMTargetLowering::LowerCallResult(SDValue Chain, SDValue InFlag,
           std::swap (Lo, Hi);
         Val = DAG.getNode(ARMISD::VMOVDRR, dl, MVT::f64, Lo, Hi);
         Val = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, MVT::v2f64, Vec, Val,
-                          DAG.getConstant(1, MVT::i32));
+                          DAG.getConstant(1, dl, MVT::i32));
       }
     } else {
       Val = DAG.getCopyFromReg(Chain, dl, VA.getLocReg(), VA.getLocVT(),
@@ -1444,8 +1452,9 @@ ARMTargetLowering::LowerMemOpCallTo(SDValue Chain,
                                     const CCValAssign &VA,
                                     ISD::ArgFlagsTy Flags) const {
   unsigned LocMemOffset = VA.getLocMemOffset();
-  SDValue PtrOff = DAG.getIntPtrConstant(LocMemOffset);
-  PtrOff = DAG.getNode(ISD::ADD, dl, getPointerTy(), StackPtr, PtrOff);
+  SDValue PtrOff = DAG.getIntPtrConstant(LocMemOffset, dl);
+  PtrOff = DAG.getNode(ISD::ADD, dl, getPointerTy(DAG.getDataLayout()),
+                       StackPtr, PtrOff);
   return DAG.getStore(Chain, dl, Arg, PtrOff,
                       MachinePointerInfo::getStack(LocMemOffset),
                       false, false, 0);
@@ -1469,7 +1478,8 @@ void ARMTargetLowering::PassF64ArgInRegs(SDLoc dl, SelectionDAG &DAG,
   else {
     assert(NextVA.isMemLoc());
     if (!StackPtr.getNode())
-      StackPtr = DAG.getCopyFromReg(Chain, dl, ARM::SP, getPointerTy());
+      StackPtr = DAG.getCopyFromReg(Chain, dl, ARM::SP,
+                                    getPointerTy(DAG.getDataLayout()));
 
     MemOpChains.push_back(LowerMemOpCallTo(Chain, StackPtr, fmrrd.getValue(1-id),
                                            dl, DAG, NextVA,
@@ -1484,7 +1494,7 @@ SDValue
 ARMTargetLowering::LowerCall(TargetLowering::CallLoweringInfo &CLI,
                              SmallVectorImpl<SDValue> &InVals) const {
   SelectionDAG &DAG                     = CLI.DAG;
-  SDLoc &dl                          = CLI.DL;
+  SDLoc &dl                             = CLI.DL;
   SmallVectorImpl<ISD::OutputArg> &Outs = CLI.Outs;
   SmallVectorImpl<SDValue> &OutVals     = CLI.OutVals;
   SmallVectorImpl<ISD::InputArg> &Ins   = CLI.Ins;
@@ -1499,9 +1509,10 @@ ARMTargetLowering::LowerCall(TargetLowering::CallLoweringInfo &CLI,
   bool isStructRet    = (Outs.empty()) ? false : Outs[0].Flags.isSRet();
   bool isThisReturn   = false;
   bool isSibCall      = false;
+  auto Attr = MF.getFunction()->getFnAttribute("disable-tail-calls");
 
   // Disable tail calls if they're not supported.
-  if (!Subtarget->supportsTailCall() || MF.getTarget().Options.DisableTailCalls)
+  if (!Subtarget->supportsTailCall() || Attr.getValueAsString() == "true")
     isTailCall = false;
 
   if (isTailCall) {
@@ -1538,10 +1549,11 @@ ARMTargetLowering::LowerCall(TargetLowering::CallLoweringInfo &CLI,
   // Adjust the stack pointer for the new arguments...
   // These operations are automatically eliminated by the prolog/epilog pass
   if (!isSibCall)
-    Chain = DAG.getCALLSEQ_START(Chain, DAG.getIntPtrConstant(NumBytes, true),
-                                 dl);
+    Chain = DAG.getCALLSEQ_START(Chain,
+                                 DAG.getIntPtrConstant(NumBytes, dl, true), dl);
 
-  SDValue StackPtr = DAG.getCopyFromReg(Chain, dl, ARM::SP, getPointerTy());
+  SDValue StackPtr =
+      DAG.getCopyFromReg(Chain, dl, ARM::SP, getPointerTy(DAG.getDataLayout()));
 
   RegsToPassVector RegsToPass;
   SmallVector<SDValue, 8> MemOpChains;
@@ -1578,9 +1590,9 @@ ARMTargetLowering::LowerCall(TargetLowering::CallLoweringInfo &CLI,
     if (VA.needsCustom()) {
       if (VA.getLocVT() == MVT::v2f64) {
         SDValue Op0 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::f64, Arg,
-                                  DAG.getConstant(0, MVT::i32));
+                                  DAG.getConstant(0, dl, MVT::i32));
         SDValue Op1 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::f64, Arg,
-                                  DAG.getConstant(1, MVT::i32));
+                                  DAG.getConstant(1, dl, MVT::i32));
 
         PassF64ArgInRegs(dl, DAG, Chain, Op0, RegsToPass,
                          VA, ArgLocs[++i], StackPtr, MemOpChains, Flags);
@@ -1622,10 +1634,11 @@ ARMTargetLowering::LowerCall(TargetLowering::CallLoweringInfo &CLI,
         unsigned RegBegin, RegEnd;
         CCInfo.getInRegsParamInfo(CurByValIdx, RegBegin, RegEnd);
 
-        EVT PtrVT = DAG.getTargetLoweringInfo().getPointerTy();
+        EVT PtrVT =
+            DAG.getTargetLoweringInfo().getPointerTy(DAG.getDataLayout());
         unsigned int i, j;
         for (i = 0, j = RegBegin; j < RegEnd; i++, j++) {
-          SDValue Const = DAG.getConstant(4*i, MVT::i32);
+          SDValue Const = DAG.getConstant(4*i, dl, MVT::i32);
           SDValue AddArg = DAG.getNode(ISD::ADD, dl, PtrVT, Arg, Const);
           SDValue Load = DAG.getLoad(PtrVT, dl, Chain, AddArg,
                                      MachinePointerInfo(),
@@ -1643,15 +1656,16 @@ ARMTargetLowering::LowerCall(TargetLowering::CallLoweringInfo &CLI,
       }
 
       if (Flags.getByValSize() > 4*offset) {
+        auto PtrVT = getPointerTy(DAG.getDataLayout());
         unsigned LocMemOffset = VA.getLocMemOffset();
-        SDValue StkPtrOff = DAG.getIntPtrConstant(LocMemOffset);
-        SDValue Dst = DAG.getNode(ISD::ADD, dl, getPointerTy(), StackPtr,
-                                  StkPtrOff);
-        SDValue SrcOffset = DAG.getIntPtrConstant(4*offset);
-        SDValue Src = DAG.getNode(ISD::ADD, dl, getPointerTy(), Arg, SrcOffset);
-        SDValue SizeNode = DAG.getConstant(Flags.getByValSize() - 4*offset,
+        SDValue StkPtrOff = DAG.getIntPtrConstant(LocMemOffset, dl);
+        SDValue Dst = DAG.getNode(ISD::ADD, dl, PtrVT, StackPtr, StkPtrOff);
+        SDValue SrcOffset = DAG.getIntPtrConstant(4*offset, dl);
+        SDValue Src = DAG.getNode(ISD::ADD, dl, PtrVT, Arg, SrcOffset);
+        SDValue SizeNode = DAG.getConstant(Flags.getByValSize() - 4*offset, dl,
                                            MVT::i32);
-        SDValue AlignNode = DAG.getConstant(Flags.getByValAlign(), MVT::i32);
+        SDValue AlignNode = DAG.getConstant(Flags.getByValAlign(), dl,
+                                            MVT::i32);
 
         SDVTList VTs = DAG.getVTList(MVT::Other, MVT::Glue);
         SDValue Ops[] = { Chain, Dst, Src, SizeNode, AlignNode};
@@ -1707,8 +1721,9 @@ ARMTargetLowering::LowerCall(TargetLowering::CallLoweringInfo &CLI,
   bool isARMFunc = false;
   bool isLocalARMFunc = false;
   ARMFunctionInfo *AFI = MF.getInfo<ARMFunctionInfo>();
+  auto PtrVt = getPointerTy(DAG.getDataLayout());
 
-  if (EnableARMLongCalls) {
+  if (Subtarget->genLongCalls()) {
     assert((Subtarget->isTargetWindows() ||
             getTargetMachine().getRelocationModel() == Reloc::Static) &&
            "long-calls with non-static relocation model!");
@@ -1723,12 +1738,11 @@ ARMTargetLowering::LowerCall(TargetLowering::CallLoweringInfo &CLI,
         ARMConstantPoolConstant::Create(GV, ARMPCLabelIndex, ARMCP::CPValue, 0);
 
       // Get the address of the callee into a register
-      SDValue CPAddr = DAG.getTargetConstantPool(CPV, getPointerTy(), 4);
+      SDValue CPAddr = DAG.getTargetConstantPool(CPV, PtrVt, 4);
       CPAddr = DAG.getNode(ARMISD::Wrapper, dl, MVT::i32, CPAddr);
-      Callee = DAG.getLoad(getPointerTy(), dl,
-                           DAG.getEntryNode(), CPAddr,
-                           MachinePointerInfo::getConstantPool(),
-                           false, false, false, 0);
+      Callee = DAG.getLoad(PtrVt, dl, DAG.getEntryNode(), CPAddr,
+                           MachinePointerInfo::getConstantPool(), false, false,
+                           false, 0);
     } else if (ExternalSymbolSDNode *S=dyn_cast<ExternalSymbolSDNode>(Callee)) {
       const char *Sym = S->getSymbol();
 
@@ -1738,29 +1752,28 @@ ARMTargetLowering::LowerCall(TargetLowering::CallLoweringInfo &CLI,
         ARMConstantPoolSymbol::Create(*DAG.getContext(), Sym,
                                       ARMPCLabelIndex, 0);
       // Get the address of the callee into a register
-      SDValue CPAddr = DAG.getTargetConstantPool(CPV, getPointerTy(), 4);
+      SDValue CPAddr = DAG.getTargetConstantPool(CPV, PtrVt, 4);
       CPAddr = DAG.getNode(ARMISD::Wrapper, dl, MVT::i32, CPAddr);
-      Callee = DAG.getLoad(getPointerTy(), dl,
-                           DAG.getEntryNode(), CPAddr,
-                           MachinePointerInfo::getConstantPool(),
-                           false, false, false, 0);
+      Callee = DAG.getLoad(PtrVt, dl, DAG.getEntryNode(), CPAddr,
+                           MachinePointerInfo::getConstantPool(), false, false,
+                           false, 0);
     }
   } else if (GlobalAddressSDNode *G = dyn_cast<GlobalAddressSDNode>(Callee)) {
     const GlobalValue *GV = G->getGlobal();
     isDirect = true;
-    bool isExt = GV->isDeclaration() || GV->isWeakForLinker();
-    bool isStub = (isExt && Subtarget->isTargetMachO()) &&
+    bool isDef = GV->isStrongDefinitionForLinker();
+    bool isStub = (!isDef && Subtarget->isTargetMachO()) &&
                    getTargetMachine().getRelocationModel() != Reloc::Static;
     isARMFunc = !Subtarget->isThumb() || (isStub && !Subtarget->isMClass());
     // ARM call to a local ARM function is predicable.
-    isLocalARMFunc = !Subtarget->isThumb() && (!isExt || !ARMInterworking);
+    isLocalARMFunc = !Subtarget->isThumb() && (isDef || !ARMInterworking);
     // tBX takes a register source operand.
     if (isStub && Subtarget->isThumb1Only() && !Subtarget->hasV5TOps()) {
       assert(Subtarget->isTargetMachO() && "WrapperPIC use on non-MachO?");
-      Callee = DAG.getNode(ARMISD::WrapperPIC, dl, getPointerTy(),
-                           DAG.getTargetGlobalAddress(GV, dl, getPointerTy(),
-                                                      0, ARMII::MO_NONLAZY));
-      Callee = DAG.getLoad(getPointerTy(), dl, DAG.getEntryNode(), Callee,
+      Callee = DAG.getNode(
+          ARMISD::WrapperPIC, dl, PtrVt,
+          DAG.getTargetGlobalAddress(GV, dl, PtrVt, 0, ARMII::MO_NONLAZY));
+      Callee = DAG.getLoad(PtrVt, dl, DAG.getEntryNode(), Callee,
                            MachinePointerInfo::getGOT(), false, false, true, 0);
     } else if (Subtarget->isTargetCOFF()) {
       assert(Subtarget->isTargetWindows() &&
@@ -1768,20 +1781,20 @@ ARMTargetLowering::LowerCall(TargetLowering::CallLoweringInfo &CLI,
       unsigned TargetFlags = GV->hasDLLImportStorageClass()
                                  ? ARMII::MO_DLLIMPORT
                                  : ARMII::MO_NO_FLAG;
-      Callee = DAG.getTargetGlobalAddress(GV, dl, getPointerTy(), /*Offset=*/0,
-                                          TargetFlags);
+      Callee =
+          DAG.getTargetGlobalAddress(GV, dl, PtrVt, /*Offset=*/0, TargetFlags);
       if (GV->hasDLLImportStorageClass())
-        Callee = DAG.getLoad(getPointerTy(), dl, DAG.getEntryNode(),
-                             DAG.getNode(ARMISD::Wrapper, dl, getPointerTy(),
-                                         Callee), MachinePointerInfo::getGOT(),
-                             false, false, false, 0);
+        Callee =
+            DAG.getLoad(PtrVt, dl, DAG.getEntryNode(),
+                        DAG.getNode(ARMISD::Wrapper, dl, PtrVt, Callee),
+                        MachinePointerInfo::getGOT(), false, false, false, 0);
     } else {
       // On ELF targets for PIC code, direct calls should go through the PLT
       unsigned OpFlags = 0;
       if (Subtarget->isTargetELF() &&
           getTargetMachine().getRelocationModel() == Reloc::PIC_)
         OpFlags = ARMII::MO_PLT;
-      Callee = DAG.getTargetGlobalAddress(GV, dl, getPointerTy(), 0, OpFlags);
+      Callee = DAG.getTargetGlobalAddress(GV, dl, PtrVt, 0, OpFlags);
     }
   } else if (ExternalSymbolSDNode *S = dyn_cast<ExternalSymbolSDNode>(Callee)) {
     isDirect = true;
@@ -1795,22 +1808,20 @@ ARMTargetLowering::LowerCall(TargetLowering::CallLoweringInfo &CLI,
       ARMConstantPoolValue *CPV =
         ARMConstantPoolSymbol::Create(*DAG.getContext(), Sym,
                                       ARMPCLabelIndex, 4);
-      SDValue CPAddr = DAG.getTargetConstantPool(CPV, getPointerTy(), 4);
+      SDValue CPAddr = DAG.getTargetConstantPool(CPV, PtrVt, 4);
       CPAddr = DAG.getNode(ARMISD::Wrapper, dl, MVT::i32, CPAddr);
-      Callee = DAG.getLoad(getPointerTy(), dl,
-                           DAG.getEntryNode(), CPAddr,
-                           MachinePointerInfo::getConstantPool(),
-                           false, false, false, 0);
-      SDValue PICLabel = DAG.getConstant(ARMPCLabelIndex, MVT::i32);
-      Callee = DAG.getNode(ARMISD::PIC_ADD, dl,
-                           getPointerTy(), Callee, PICLabel);
+      Callee = DAG.getLoad(PtrVt, dl, DAG.getEntryNode(), CPAddr,
+                           MachinePointerInfo::getConstantPool(), false, false,
+                           false, 0);
+      SDValue PICLabel = DAG.getConstant(ARMPCLabelIndex, dl, MVT::i32);
+      Callee = DAG.getNode(ARMISD::PIC_ADD, dl, PtrVt, Callee, PICLabel);
     } else {
       unsigned OpFlags = 0;
       // On ELF targets for PIC code, direct calls should go through the PLT
       if (Subtarget->isTargetELF() &&
                   getTargetMachine().getRelocationModel() == Reloc::PIC_)
         OpFlags = ARMII::MO_PLT;
-      Callee = DAG.getTargetExternalSymbol(Sym, getPointerTy(), OpFlags);
+      Callee = DAG.getTargetExternalSymbol(Sym, PtrVt, OpFlags);
     }
   }
 
@@ -1869,15 +1880,17 @@ ARMTargetLowering::LowerCall(TargetLowering::CallLoweringInfo &CLI,
     Ops.push_back(InFlag);
 
   SDVTList NodeTys = DAG.getVTList(MVT::Other, MVT::Glue);
-  if (isTailCall)
+  if (isTailCall) {
+    MF.getFrameInfo()->setHasTailCall();
     return DAG.getNode(ARMISD::TC_RETURN, dl, NodeTys, Ops);
+  }
 
   // Returns a chain and a flag for retval copy to use.
   Chain = DAG.getNode(CallOpc, dl, NodeTys, Ops);
   InFlag = Chain.getValue(1);
 
-  Chain = DAG.getCALLSEQ_END(Chain, DAG.getIntPtrConstant(NumBytes, true),
-                             DAG.getIntPtrConstant(0, true), InFlag, dl);
+  Chain = DAG.getCALLSEQ_END(Chain, DAG.getIntPtrConstant(NumBytes, dl, true),
+                             DAG.getIntPtrConstant(0, dl, true), InFlag, dl);
   if (!Ins.empty())
     InFlag = Chain.getValue(1);
 
@@ -1946,7 +1959,6 @@ void ARMTargetLowering::HandleByVal(CCState *State, unsigned &Size,
   Size = std::max<int>(Size - Excess, 0);
 }
 
-
 /// MatchingStackOffset - Return true if the given stack call argument is
 /// already available in the same position (relatively) of the caller's
 /// incoming argument stack.
@@ -2056,7 +2068,7 @@ ARMTargetLowering::IsEligibleForTailCallOptimization(SDValue Callee,
   // cannot rely on the linker replacing the tail call with a return.
   if (GlobalAddressSDNode *G = dyn_cast<GlobalAddressSDNode>(Callee)) {
     const GlobalValue *GV = G->getGlobal();
-    const Triple TT(getTargetMachine().getTargetTriple());
+    const Triple &TT = getTargetMachine().getTargetTriple();
     if (GV->hasExternalWeakLinkage() &&
         (!TT.isOSWindows() || TT.isOSBinFormatELF() || TT.isOSBinFormatMachO()))
       return false;
@@ -2192,7 +2204,8 @@ static SDValue LowerInterruptReturn(SmallVectorImpl<SDValue> &RetOps,
     report_fatal_error("Unsupported interrupt attribute. If present, value "
                        "must be one of: IRQ, FIQ, SWI, ABORT or UNDEF");
 
-  RetOps.insert(RetOps.begin() + 1, DAG.getConstant(LROffset, MVT::i32, false));
+  RetOps.insert(RetOps.begin() + 1,
+                DAG.getConstant(LROffset, DL, MVT::i32, false));
 
   return DAG.getNode(ARMISD::INTRET_FLAG, DL, MVT::Other, RetOps);
 }
@@ -2245,7 +2258,7 @@ ARMTargetLowering::LowerReturn(SDValue Chain,
       if (VA.getLocVT() == MVT::v2f64) {
         // Extract the first half and return it in two registers.
         SDValue Half = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::f64, Arg,
-                                   DAG.getConstant(0, MVT::i32));
+                                   DAG.getConstant(0, dl, MVT::i32));
         SDValue HalfGPRs = DAG.getNode(ARMISD::VMOVRRD, dl,
                                        DAG.getVTList(MVT::i32, MVT::i32), Half);
 
@@ -2264,7 +2277,7 @@ ARMTargetLowering::LowerReturn(SDValue Chain,
 
         // Extract the 2nd half and fall through to handle it as an f64 value.
         Arg = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::f64, Arg,
-                          DAG.getConstant(1, MVT::i32));
+                          DAG.getConstant(1, dl, MVT::i32));
       }
       // Legalize ret f64 -> ret 2 x i32.  We always have fmrrd if f64 is
       // available.
@@ -2388,12 +2401,32 @@ bool ARMTargetLowering::mayBeEmittedAsTailCall(CallInst *CI) const {
   if (!Subtarget->supportsTailCall())
     return false;
 
-  if (!CI->isTailCall() || getTargetMachine().Options.DisableTailCalls)
+  auto Attr =
+      CI->getParent()->getParent()->getFnAttribute("disable-tail-calls");
+  if (!CI->isTailCall() || Attr.getValueAsString() == "true")
     return false;
 
   return !Subtarget->isThumb1Only();
 }
 
+// Trying to write a 64 bit value so need to split into two 32 bit values first,
+// and pass the lower and high parts through.
+static SDValue LowerWRITE_REGISTER(SDValue Op, SelectionDAG &DAG) {
+  SDLoc DL(Op);
+  SDValue WriteValue = Op->getOperand(2);
+
+  // This function is only supposed to be called for i64 type argument.
+  assert(WriteValue.getValueType() == MVT::i64
+          && "LowerWRITE_REGISTER called for non-i64 type argument.");
+
+  SDValue Lo = DAG.getNode(ISD::EXTRACT_ELEMENT, DL, MVT::i32, WriteValue,
+                           DAG.getConstant(0, DL, MVT::i32));
+  SDValue Hi = DAG.getNode(ISD::EXTRACT_ELEMENT, DL, MVT::i32, WriteValue,
+                           DAG.getConstant(1, DL, MVT::i32));
+  SDValue Ops[] = { Op->getOperand(0), Op->getOperand(1), Lo, Hi };
+  return DAG.getNode(ISD::WRITE_REGISTER, DL, MVT::Other, Ops);
+}
+
 // ConstantPool, JumpTable, GlobalAddress, and ExternalSymbol are lowered as
 // their target counterpart wrapped in the ARMISD::Wrapper node. Suppose N is
 // one of the above mentioned nodes. It has to be wrapped because otherwise
@@ -2416,14 +2449,7 @@ static SDValue LowerConstantPool(SDValue Op, SelectionDAG &DAG) {
 }
 
 unsigned ARMTargetLowering::getJumpTableEncoding() const {
-  // @LOCALMOD-BEGIN
-  if (Subtarget->useInlineJumpTables()) { 
-    return MachineJumpTableInfo::EK_Inline;
-  } else {
-    // TODO: Find a better way to call the super-class.
-    return TargetLowering::getJumpTableEncoding();
-  }
-  // @LOCALMOD-END
+  return MachineJumpTableInfo::EK_Inline;
 }
 
 SDValue ARMTargetLowering::LowerBlockAddress(SDValue Op,
@@ -2432,7 +2458,7 @@ SDValue ARMTargetLowering::LowerBlockAddress(SDValue Op,
   ARMFunctionInfo *AFI = MF.getInfo<ARMFunctionInfo>();
   unsigned ARMPCLabelIndex = 0;
   SDLoc DL(Op);
-  EVT PtrVT = getPointerTy();
+  EVT PtrVT = getPointerTy(DAG.getDataLayout());
   const BlockAddress *BA = cast<BlockAddressSDNode>(Op)->getBlockAddress();
   Reloc::Model RelocM = getTargetMachine().getRelocationModel();
   SDValue CPAddr;
@@ -2452,73 +2478,16 @@ SDValue ARMTargetLowering::LowerBlockAddress(SDValue Op,
                                false, false, false, 0);
   if (RelocM == Reloc::Static)
     return Result;
-  SDValue PICLabel = DAG.getConstant(ARMPCLabelIndex, MVT::i32);
+  SDValue PICLabel = DAG.getConstant(ARMPCLabelIndex, DL, MVT::i32);
   return DAG.getNode(ARMISD::PIC_ADD, DL, PtrVT, Result, PICLabel);
 }
 
-// @LOCALMOD-START
-// more conventional jumptable implementation
-SDValue ARMTargetLowering::LowerJumpTable(SDValue Op, SelectionDAG &DAG) const {
-  assert(!Subtarget->useInlineJumpTables() &&
-         "inline jump tables not custom lowered");
-  const SDLoc dl(Op);
-  EVT PTy = getPointerTy();
-  JumpTableSDNode *JT = cast<JumpTableSDNode>(Op);
-  SDValue JTI = DAG.getTargetJumpTable(JT->getIndex(), PTy);
-  return DAG.getNode(ARMISD::WrapperJT2, dl, MVT::i32, JTI);
-}
-// @LOCALMOD-END
-
 // Lower ISD::GlobalTLSAddress using the "general dynamic" model
 SDValue
 ARMTargetLowering::LowerToTLSGeneralDynamicModel(GlobalAddressSDNode *GA,
                                                  SelectionDAG &DAG) const {
   SDLoc dl(GA);
-  EVT PtrVT = getPointerTy();
-  // @LOCALMOD-BEGIN
-  SDValue Chain;
-  SDValue Argument;
-
-  if (!Subtarget->useConstIslands()) {
-    // With constant islands "disabled" (moved to rodata), this constant pool
-    // entry is no longer in text, and simultaneous PC relativeness
-    // and CP Addr relativeness is no longer expressible.
-    // So, instead of having:
-    //
-    // .LCPI12_0:
-    //   .long var(tlsgd)-((.LPC12_0+8) - .)
-    // ...
-    //    ldr r2, .LCPI12_0
-    // .LPC12_0:
-    //    add r0, pc, r2
-    //
-    // we have:
-    //
-    // .LCPI12_0:
-    //   .long var(tlsgd)
-    // ...
-    //    // get addr of .LCPI12_0 into r2
-    //    ldr r0, [r2]
-    //    add r0, r2, r0
-    // (1) No longer subtracting pc, so no longer adding that back
-    // (2) Not adding "." in the CP entry, so adding it via instructions.
-    //
-    unsigned char PCAdj = 0;
-    MachineFunction &MF = DAG.getMachineFunction();
-    ARMFunctionInfo *AFI = MF.getInfo<ARMFunctionInfo>();
-    unsigned ARMPCLabelIndex = AFI->createPICLabelUId();
-    ARMConstantPoolValue *CPV =
-        ARMConstantPoolConstant::Create(GA->getGlobal(), ARMPCLabelIndex,
-                                        ARMCP::CPValue, PCAdj, ARMCP::TLSGD,
-                                        false);
-    SDValue CPAddr = DAG.getTargetConstantPool(CPV, PtrVT, 4);
-    CPAddr = DAG.getNode(ARMISD::Wrapper, dl, MVT::i32, CPAddr);
-    Argument = DAG.getLoad(PtrVT, dl, DAG.getEntryNode(), CPAddr,
-                                   MachinePointerInfo::getConstantPool(),
-                                   false, false, false, 0);
-    Chain = Argument.getValue(1);
-    Argument = DAG.getNode(ISD::ADD, dl, PtrVT, Argument, CPAddr);
-  } else { // sort of @LOCALMOD-END
+  EVT PtrVT = getPointerTy(DAG.getDataLayout());
   unsigned char PCAdj = Subtarget->isThumb() ? 4 : 8;
   MachineFunction &MF = DAG.getMachineFunction();
   ARMFunctionInfo *AFI = MF.getInfo<ARMFunctionInfo>();
@@ -2526,16 +2495,15 @@ ARMTargetLowering::LowerToTLSGeneralDynamicModel(GlobalAddressSDNode *GA,
   ARMConstantPoolValue *CPV =
     ARMConstantPoolConstant::Create(GA->getGlobal(), ARMPCLabelIndex,
                                     ARMCP::CPValue, PCAdj, ARMCP::TLSGD, true);
-  /*SDValue*/ Argument = DAG.getTargetConstantPool(CPV, PtrVT, 4); // @LOCALMOD
+  SDValue Argument = DAG.getTargetConstantPool(CPV, PtrVT, 4);
   Argument = DAG.getNode(ARMISD::Wrapper, dl, MVT::i32, Argument);
   Argument = DAG.getLoad(PtrVT, dl, DAG.getEntryNode(), Argument,
                          MachinePointerInfo::getConstantPool(),
                          false, false, false, 0);
-  /*SDValue*/ Chain = Argument.getValue(1); // @LOCALMOD
+  SDValue Chain = Argument.getValue(1);
 
-  SDValue PICLabel = DAG.getConstant(ARMPCLabelIndex, MVT::i32);
+  SDValue PICLabel = DAG.getConstant(ARMPCLabelIndex, dl, MVT::i32);
   Argument = DAG.getNode(ARMISD::PIC_ADD, dl, PtrVT, Argument, PICLabel);
-  } // @LOCALMOD-END
 
   // call __tls_get_addr.
   ArgListTy Args;
@@ -2565,7 +2533,7 @@ ARMTargetLowering::LowerToTLSExecModels(GlobalAddressSDNode *GA,
   SDLoc dl(GA);
   SDValue Offset;
   SDValue Chain = DAG.getEntryNode();
-  EVT PtrVT = getPointerTy();
+  EVT PtrVT = getPointerTy(DAG.getDataLayout());
   // Get the Thread Pointer
   SDValue ThreadPointer = DAG.getNode(ARMISD::THREAD_POINTER, dl, PtrVT);
 
@@ -2573,49 +2541,25 @@ ARMTargetLowering::LowerToTLSExecModels(GlobalAddressSDNode *GA,
     MachineFunction &MF = DAG.getMachineFunction();
     ARMFunctionInfo *AFI = MF.getInfo<ARMFunctionInfo>();
     unsigned ARMPCLabelIndex = AFI->createPICLabelUId();
-
-    // @LOCALMOD-BEGIN
-    if (!Subtarget->useConstIslands()) {
-      // Similar to change to LowerToTLSGeneralDynamicModel, and
-      // for the same reason.
-      unsigned char PCAdj = 0;
-      ARMConstantPoolValue *CPV =
-        ARMConstantPoolConstant::Create(GA->getGlobal(), ARMPCLabelIndex,
-                                        ARMCP::CPValue, PCAdj, ARMCP::GOTTPOFF,
-                                        false);
-      SDValue CPAddr = DAG.getTargetConstantPool(CPV, PtrVT, 4);
-      CPAddr = DAG.getNode(ARMISD::Wrapper, dl, MVT::i32, CPAddr);
-      Offset = DAG.getLoad(PtrVT, dl, Chain, CPAddr,
-                           MachinePointerInfo::getConstantPool(),
-                           false, false, false, 0);
-      Chain = Offset.getValue(1);
-
-      Offset = DAG.getNode(ISD::ADD, dl, PtrVT, Offset, CPAddr);
-
-      Offset = DAG.getLoad(PtrVT, dl, Chain, Offset,
-                           MachinePointerInfo::getConstantPool(),
-                           false, false, false, 0);
-    } else { // sort of @LOCALMOD-END (indentation)
-      // Initial exec model.
-      unsigned char PCAdj = Subtarget->isThumb() ? 4 : 8;
-      ARMConstantPoolValue *CPV =
+    // Initial exec model.
+    unsigned char PCAdj = Subtarget->isThumb() ? 4 : 8;
+    ARMConstantPoolValue *CPV =
       ARMConstantPoolConstant::Create(GA->getGlobal(), ARMPCLabelIndex,
                                       ARMCP::CPValue, PCAdj, ARMCP::GOTTPOFF,
                                       true);
-      Offset = DAG.getTargetConstantPool(CPV, PtrVT, 4);
-      Offset = DAG.getNode(ARMISD::Wrapper, dl, MVT::i32, Offset);
-      Offset = DAG.getLoad(PtrVT, dl, Chain, Offset,
-                           MachinePointerInfo::getConstantPool(),
-                           false, false, false, 0);
-      Chain = Offset.getValue(1);
+    Offset = DAG.getTargetConstantPool(CPV, PtrVT, 4);
+    Offset = DAG.getNode(ARMISD::Wrapper, dl, MVT::i32, Offset);
+    Offset = DAG.getLoad(PtrVT, dl, Chain, Offset,
+                         MachinePointerInfo::getConstantPool(),
+                         false, false, false, 0);
+    Chain = Offset.getValue(1);
 
-      SDValue PICLabel = DAG.getConstant(ARMPCLabelIndex, MVT::i32);
-      Offset = DAG.getNode(ARMISD::PIC_ADD, dl, PtrVT, Offset, PICLabel);
+    SDValue PICLabel = DAG.getConstant(ARMPCLabelIndex, dl, MVT::i32);
+    Offset = DAG.getNode(ARMISD::PIC_ADD, dl, PtrVT, Offset, PICLabel);
 
-      Offset = DAG.getLoad(PtrVT, dl, Chain, Offset,
-                           MachinePointerInfo::getConstantPool(),
-                           false, false, false, 0);
-    } // @LOCALMOD-END
+    Offset = DAG.getLoad(PtrVT, dl, Chain, Offset,
+                         MachinePointerInfo::getConstantPool(),
+                         false, false, false, 0);
   } else {
     // local exec model
     assert(model == TLSModel::LocalExec);
@@ -2639,6 +2583,8 @@ ARMTargetLowering::LowerGlobalTLSAddress(SDValue Op, SelectionDAG &DAG) const {
   assert(Subtarget->isTargetELF() &&
          "TLS not implemented for non-ELF targets");
   GlobalAddressSDNode *GA = cast<GlobalAddressSDNode>(Op);
+  if (DAG.getTarget().Options.EmulatedTLS)
+    return LowerToTLSEmulatedModel(GA, DAG);
 
   TLSModel::Model model = getTargetMachine().getTLSModel(GA->getGlobal());
 
@@ -2655,7 +2601,7 @@ ARMTargetLowering::LowerGlobalTLSAddress(SDValue Op, SelectionDAG &DAG) const {
 
 SDValue ARMTargetLowering::LowerGlobalAddressELF(SDValue Op,
                                                  SelectionDAG &DAG) const {
-  EVT PtrVT = getPointerTy();
+  EVT PtrVT = getPointerTy(DAG.getDataLayout());
   SDLoc dl(Op);
   const GlobalValue *GV = cast<GlobalAddressSDNode>(Op)->getGlobal();
   if (getTargetMachine().getRelocationModel() == Reloc::PIC_) {
@@ -2698,7 +2644,7 @@ SDValue ARMTargetLowering::LowerGlobalAddressELF(SDValue Op,
 
 SDValue ARMTargetLowering::LowerGlobalAddressDarwin(SDValue Op,
                                                     SelectionDAG &DAG) const {
-  EVT PtrVT = getPointerTy();
+  EVT PtrVT = getPointerTy(DAG.getDataLayout());
   SDLoc dl(Op);
   const GlobalValue *GV = cast<GlobalAddressSDNode>(Op)->getGlobal();
   Reloc::Model RelocM = getTargetMachine().getRelocationModel();
@@ -2729,7 +2675,7 @@ SDValue ARMTargetLowering::LowerGlobalAddressWindows(SDValue Op,
   const GlobalValue *GV = cast<GlobalAddressSDNode>(Op)->getGlobal();
   const ARMII::TOF TargetFlags =
     (GV->hasDLLImportStorageClass() ? ARMII::MO_DLLIMPORT : ARMII::MO_NO_FLAG);
-  EVT PtrVT = getPointerTy();
+  EVT PtrVT = getPointerTy(DAG.getDataLayout());
   SDValue Result;
   SDLoc DL(Op);
 
@@ -2753,46 +2699,8 @@ SDValue ARMTargetLowering::LowerGLOBAL_OFFSET_TABLE(SDValue Op,
   MachineFunction &MF = DAG.getMachineFunction();
   ARMFunctionInfo *AFI = MF.getInfo<ARMFunctionInfo>();
   unsigned ARMPCLabelIndex = AFI->createPICLabelUId();
-  EVT PtrVT = getPointerTy();
+  EVT PtrVT = getPointerTy(DAG.getDataLayout());
   SDLoc dl(Op);
-
-  // @LOCALMOD-BEGIN
-  if (!Subtarget->useConstIslands()) {
-    // With constant islands "disabled" (moved to rodata), the constant pool
-    // entry is no longer in text, and the PC relativeness is
-    // no longer expressible.
-    //
-    // Instead of having:
-    //
-    // .LCPI12_0:
-    //   .long _GLOBAL_OFFSET_TABLE_-(.LPC12_0+8)
-    // ...
-    //    ldr r2, .LCPI12_0
-    // .LPC12_0:
-    //    add r0, pc, r2
-    //
-    // Things to try:
-    // (1) get the address of the GOT through a pc-relative MOVW / MOVT.
-    //
-    //    movw r0, :lower16:_GLOBAL_OFFSET_TABLE_ - (.LPC12_0 + 8)
-    //    movt r0, :upper16:_GLOBAL_OFFSET_TABLE_ - (.LPC12_0 + 8)
-    // .LPC12_0:
-    //    add r0, pc, r0
-    //
-    // (2) Make the constant pool entry relative to its own location
-    //
-    // .LCPI12_0:
-    //   .long _GLOBAL_OFFSET_TABLE_-.
-    // ...
-    //    // get address of LCPI12_0 into r0 (possibly 3 instructions for PIC)
-    //    ldr r1, [r0]
-    //    add r1, r0, r1
-    //
-    // We will try (1) for now, since (2) takes about 3 more instructions
-    // (and one of them is a load).
-    return DAG.getNode(ARMISD::WrapperGOT, dl, MVT::i32);
-  }
-  // @LOCALMOD-END
   unsigned PCAdj = Subtarget->isThumb() ? 4 : 8;
   ARMConstantPoolValue *CPV =
     ARMConstantPoolSymbol::Create(*DAG.getContext(), "_GLOBAL_OFFSET_TABLE_",
@@ -2802,14 +2710,14 @@ SDValue ARMTargetLowering::LowerGLOBAL_OFFSET_TABLE(SDValue Op,
   SDValue Result = DAG.getLoad(PtrVT, dl, DAG.getEntryNode(), CPAddr,
                                MachinePointerInfo::getConstantPool(),
                                false, false, false, 0);
-  SDValue PICLabel = DAG.getConstant(ARMPCLabelIndex, MVT::i32);
+  SDValue PICLabel = DAG.getConstant(ARMPCLabelIndex, dl, MVT::i32);
   return DAG.getNode(ARMISD::PIC_ADD, dl, PtrVT, Result, PICLabel);
 }
 
 SDValue
 ARMTargetLowering::LowerEH_SJLJ_SETJMP(SDValue Op, SelectionDAG &DAG) const {
   SDLoc dl(Op);
-  SDValue Val = DAG.getConstant(0, MVT::i32);
+  SDValue Val = DAG.getConstant(0, dl, MVT::i32);
   return DAG.getNode(ARMISD::EH_SJLJ_SETJMP, dl,
                      DAG.getVTList(MVT::i32, MVT::Other), Op.getOperand(0),
                      Op.getOperand(1), Val);
@@ -2819,7 +2727,14 @@ SDValue
 ARMTargetLowering::LowerEH_SJLJ_LONGJMP(SDValue Op, SelectionDAG &DAG) const {
   SDLoc dl(Op);
   return DAG.getNode(ARMISD::EH_SJLJ_LONGJMP, dl, MVT::Other, Op.getOperand(0),
-                     Op.getOperand(1), DAG.getConstant(0, MVT::i32));
+                     Op.getOperand(1), DAG.getConstant(0, dl, MVT::i32));
+}
+
+SDValue ARMTargetLowering::LowerEH_SJLJ_SETUP_DISPATCH(SDValue Op,
+                                                      SelectionDAG &DAG) const {
+  SDLoc dl(Op);
+  return DAG.getNode(ARMISD::EH_SJLJ_SETUP_DISPATCH, dl, MVT::Other,
+                     Op.getOperand(0));
 }
 
 SDValue
@@ -2834,16 +2749,15 @@ ARMTargetLowering::LowerINTRINSIC_WO_CHAIN(SDValue Op, SelectionDAG &DAG,
            "RBIT intrinsic must have i32 type!");
     return DAG.getNode(ARMISD::RBIT, dl, MVT::i32, Op.getOperand(1));
   }
-  case Intrinsic::nacl_read_tp:  // @LOCALMOD
   case Intrinsic::arm_thread_pointer: {
-    EVT PtrVT = DAG.getTargetLoweringInfo().getPointerTy();
+    EVT PtrVT = getPointerTy(DAG.getDataLayout());
     return DAG.getNode(ARMISD::THREAD_POINTER, dl, PtrVT);
   }
   case Intrinsic::eh_sjlj_lsda: {
     MachineFunction &MF = DAG.getMachineFunction();
     ARMFunctionInfo *AFI = MF.getInfo<ARMFunctionInfo>();
     unsigned ARMPCLabelIndex = AFI->createPICLabelUId();
-    EVT PtrVT = getPointerTy();
+    EVT PtrVT = getPointerTy(DAG.getDataLayout());
     Reloc::Model RelocM = getTargetMachine().getRelocationModel();
     SDValue CPAddr;
     unsigned PCAdj = (RelocM != Reloc::PIC_)
@@ -2859,7 +2773,7 @@ ARMTargetLowering::LowerINTRINSIC_WO_CHAIN(SDValue Op, SelectionDAG &DAG,
                   false, false, false, 0);
 
     if (RelocM == Reloc::PIC_) {
-      SDValue PICLabel = DAG.getConstant(ARMPCLabelIndex, MVT::i32);
+      SDValue PICLabel = DAG.getConstant(ARMPCLabelIndex, dl, MVT::i32);
       Result = DAG.getNode(ARMISD::PIC_ADD, dl, PtrVT, Result, PICLabel);
     }
     return Result;
@@ -2885,7 +2799,7 @@ static SDValue LowerATOMIC_FENCE(SDValue Op, SelectionDAG &DAG,
     assert(Subtarget->hasV6Ops() && !Subtarget->isThumb() &&
            "Unexpected ISD::ATOMIC_FENCE encountered. Should be libcall!");
     return DAG.getNode(ARMISD::MEMBARRIER_MCR, dl, MVT::Other, Op.getOperand(0),
-                       DAG.getConstant(0, MVT::i32));
+                       DAG.getConstant(0, dl, MVT::i32));
   }
 
   ConstantSDNode *OrdN = cast<ConstantSDNode>(Op.getOperand(1));
@@ -2902,8 +2816,8 @@ static SDValue LowerATOMIC_FENCE(SDValue Op, SelectionDAG &DAG,
   }
 
   return DAG.getNode(ISD::INTRINSIC_VOID, dl, MVT::Other, Op.getOperand(0),
-                     DAG.getConstant(Intrinsic::arm_dmb, MVT::i32),
-                     DAG.getConstant(Domain, MVT::i32));
+                     DAG.getConstant(Intrinsic::arm_dmb, dl, MVT::i32),
+                     DAG.getConstant(Domain, dl, MVT::i32));
 }
 
 static SDValue LowerPREFETCH(SDValue Op, SelectionDAG &DAG,
@@ -2929,8 +2843,8 @@ static SDValue LowerPREFETCH(SDValue Op, SelectionDAG &DAG,
   }
 
   return DAG.getNode(ARMISD::PRELOAD, dl, MVT::Other, Op.getOperand(0),
-                     Op.getOperand(1), DAG.getConstant(isRead, MVT::i32),
-                     DAG.getConstant(isData, MVT::i32));
+                     Op.getOperand(1), DAG.getConstant(isRead, dl, MVT::i32),
+                     DAG.getConstant(isData, dl, MVT::i32));
 }
 
 static SDValue LowerVASTART(SDValue Op, SelectionDAG &DAG) {
@@ -2940,7 +2854,7 @@ static SDValue LowerVASTART(SDValue Op, SelectionDAG &DAG) {
   // vastart just stores the address of the VarArgsFrameIndex slot into the
   // memory location argument.
   SDLoc dl(Op);
-  EVT PtrVT = DAG.getTargetLoweringInfo().getPointerTy();
+  EVT PtrVT = DAG.getTargetLoweringInfo().getPointerTy(DAG.getDataLayout());
   SDValue FR = DAG.getFrameIndex(FuncInfo->getVarArgsFrameIndex(), PtrVT);
   const Value *SV = cast<SrcValueSDNode>(Op.getOperand(2))->getValue();
   return DAG.getStore(Op.getOperand(0), dl, FR, Op.getOperand(1),
@@ -2970,7 +2884,7 @@ ARMTargetLowering::GetF64FormalArgument(CCValAssign &VA, CCValAssign &NextVA,
     int FI = MFI->CreateFixedObject(4, NextVA.getLocMemOffset(), true);
 
     // Create load node to retrieve arguments from the stack.
-    SDValue FIN = DAG.getFrameIndex(FI, getPointerTy());
+    SDValue FIN = DAG.getFrameIndex(FI, getPointerTy(DAG.getDataLayout()));
     ArgValue2 = DAG.getLoad(MVT::i32, dl, Root, FIN,
                             MachinePointerInfo::getFixedStack(FI),
                             false, false, false, 0);
@@ -3024,8 +2938,9 @@ ARMTargetLowering::StoreByValRegs(CCState &CCInfo, SelectionDAG &DAG,
   if (REnd != RBegin)
     ArgOffset = -4 * (ARM::R4 - RBegin);
 
+  auto PtrVT = getPointerTy(DAG.getDataLayout());
   int FrameIndex = MFI->CreateFixedObject(ArgSize, ArgOffset, false);
-  SDValue FIN = DAG.getFrameIndex(FrameIndex, getPointerTy());
+  SDValue FIN = DAG.getFrameIndex(FrameIndex, PtrVT);
 
   SmallVector<SDValue, 4> MemOps;
   const TargetRegisterClass *RC =
@@ -3038,8 +2953,7 @@ ARMTargetLowering::StoreByValRegs(CCState &CCInfo, SelectionDAG &DAG,
         DAG.getStore(Val.getValue(1), dl, Val, FIN,
                      MachinePointerInfo(OrigArg, 4 * i), false, false, 0);
     MemOps.push_back(Store);
-    FIN = DAG.getNode(ISD::ADD, dl, getPointerTy(), FIN,
-                      DAG.getConstant(4, getPointerTy()));
+    FIN = DAG.getNode(ISD::ADD, dl, PtrVT, FIN, DAG.getConstant(4, dl, PtrVT));
   }
 
   if (!MemOps.empty())
@@ -3133,6 +3047,7 @@ ARMTargetLowering::LowerFormalArguments(SDValue Chain,
 
   unsigned TotalArgRegsSaveSize = 4 * (ARM::R4 - ArgRegBegin);
   AFI->setArgRegsSaveSize(TotalArgRegsSaveSize);
+  auto PtrVT = getPointerTy(DAG.getDataLayout());
 
   for (unsigned i = 0, e = ArgLocs.size(); i != e; ++i) {
     CCValAssign &VA = ArgLocs[i];
@@ -3155,7 +3070,7 @@ ARMTargetLowering::LowerFormalArguments(SDValue Chain,
           SDValue ArgValue2;
           if (VA.isMemLoc()) {
             int FI = MFI->CreateFixedObject(8, VA.getLocMemOffset(), true);
-            SDValue FIN = DAG.getFrameIndex(FI, getPointerTy());
+            SDValue FIN = DAG.getFrameIndex(FI, PtrVT);
             ArgValue2 = DAG.getLoad(MVT::f64, dl, Chain, FIN,
                                     MachinePointerInfo::getFixedStack(FI),
                                     false, false, false, 0);
@@ -3165,9 +3080,11 @@ ARMTargetLowering::LowerFormalArguments(SDValue Chain,
           }
           ArgValue = DAG.getNode(ISD::UNDEF, dl, MVT::v2f64);
           ArgValue = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, MVT::v2f64,
-                                 ArgValue, ArgValue1, DAG.getIntPtrConstant(0));
+                                 ArgValue, ArgValue1,
+                                 DAG.getIntPtrConstant(0, dl));
           ArgValue = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, MVT::v2f64,
-                                 ArgValue, ArgValue2, DAG.getIntPtrConstant(1));
+                                 ArgValue, ArgValue2,
+                                 DAG.getIntPtrConstant(1, dl));
         } else
           ArgValue = GetF64FormalArgument(VA, ArgLocs[++i], Chain, DAG, dl);
 
@@ -3240,7 +3157,7 @@ ARMTargetLowering::LowerFormalArguments(SDValue Chain,
             int FrameIndex = StoreByValRegs(CCInfo, DAG, dl, Chain, CurOrigArg,
                                             CurByValIndex, VA.getLocMemOffset(),
                                             Flags.getByValSize());
-            InVals.push_back(DAG.getFrameIndex(FrameIndex, getPointerTy()));
+            InVals.push_back(DAG.getFrameIndex(FrameIndex, PtrVT));
             CCInfo.nextInRegsParam();
           } else {
             unsigned FIOffset = VA.getLocMemOffset();
@@ -3248,7 +3165,7 @@ ARMTargetLowering::LowerFormalArguments(SDValue Chain,
                                             FIOffset, true);
 
             // Create load nodes to retrieve arguments from the stack.
-            SDValue FIN = DAG.getFrameIndex(FI, getPointerTy());
+            SDValue FIN = DAG.getFrameIndex(FI, PtrVT);
             InVals.push_back(DAG.getLoad(VA.getValVT(), dl, Chain, FIN,
                                          MachinePointerInfo::getFixedStack(FI),
                                          false, false, false, 0));
@@ -3313,28 +3230,28 @@ ARMTargetLowering::getARMCmp(SDValue LHS, SDValue RHS, ISD::CondCode CC,
       case ISD::SETGE:
         if (C != 0x80000000 && isLegalICmpImmediate(C-1)) {
           CC = (CC == ISD::SETLT) ? ISD::SETLE : ISD::SETGT;
-          RHS = DAG.getConstant(C-1, MVT::i32);
+          RHS = DAG.getConstant(C - 1, dl, MVT::i32);
         }
         break;
       case ISD::SETULT:
       case ISD::SETUGE:
         if (C != 0 && isLegalICmpImmediate(C-1)) {
           CC = (CC == ISD::SETULT) ? ISD::SETULE : ISD::SETUGT;
-          RHS = DAG.getConstant(C-1, MVT::i32);
+          RHS = DAG.getConstant(C - 1, dl, MVT::i32);
         }
         break;
       case ISD::SETLE:
       case ISD::SETGT:
         if (C != 0x7fffffff && isLegalICmpImmediate(C+1)) {
           CC = (CC == ISD::SETLE) ? ISD::SETLT : ISD::SETGE;
-          RHS = DAG.getConstant(C+1, MVT::i32);
+          RHS = DAG.getConstant(C + 1, dl, MVT::i32);
         }
         break;
       case ISD::SETULE:
       case ISD::SETUGT:
         if (C != 0xffffffff && isLegalICmpImmediate(C+1)) {
           CC = (CC == ISD::SETULE) ? ISD::SETULT : ISD::SETUGE;
-          RHS = DAG.getConstant(C+1, MVT::i32);
+          RHS = DAG.getConstant(C + 1, dl, MVT::i32);
         }
         break;
       }
@@ -3353,7 +3270,7 @@ ARMTargetLowering::getARMCmp(SDValue LHS, SDValue RHS, ISD::CondCode CC,
     CompareType = ARMISD::CMPZ;
     break;
   }
-  ARMcc = DAG.getConstant(CondCode, MVT::i32);
+  ARMcc = DAG.getConstant(CondCode, dl, MVT::i32);
   return DAG.getNode(CompareType, dl, MVT::Glue, LHS, RHS);
 }
 
@@ -3399,7 +3316,7 @@ ARMTargetLowering::getARMXALUOOp(SDValue Op, SelectionDAG &DAG,
   SDValue Value, OverflowCmp;
   SDValue LHS = Op.getOperand(0);
   SDValue RHS = Op.getOperand(1);
-
+  SDLoc dl(Op);
 
   // FIXME: We are currently always generating CMPs because we don't support
   // generating CMN through the backend. This is not as good as the natural
@@ -3410,24 +3327,24 @@ ARMTargetLowering::getARMXALUOOp(SDValue Op, SelectionDAG &DAG,
   default:
     llvm_unreachable("Unknown overflow instruction!");
   case ISD::SADDO:
-    ARMcc = DAG.getConstant(ARMCC::VC, MVT::i32);
-    Value = DAG.getNode(ISD::ADD, SDLoc(Op), Op.getValueType(), LHS, RHS);
-    OverflowCmp = DAG.getNode(ARMISD::CMP, SDLoc(Op), MVT::Glue, Value, LHS);
+    ARMcc = DAG.getConstant(ARMCC::VC, dl, MVT::i32);
+    Value = DAG.getNode(ISD::ADD, dl, Op.getValueType(), LHS, RHS);
+    OverflowCmp = DAG.getNode(ARMISD::CMP, dl, MVT::Glue, Value, LHS);
     break;
   case ISD::UADDO:
-    ARMcc = DAG.getConstant(ARMCC::HS, MVT::i32);
-    Value = DAG.getNode(ISD::ADD, SDLoc(Op), Op.getValueType(), LHS, RHS);
-    OverflowCmp = DAG.getNode(ARMISD::CMP, SDLoc(Op), MVT::Glue, Value, LHS);
+    ARMcc = DAG.getConstant(ARMCC::HS, dl, MVT::i32);
+    Value = DAG.getNode(ISD::ADD, dl, Op.getValueType(), LHS, RHS);
+    OverflowCmp = DAG.getNode(ARMISD::CMP, dl, MVT::Glue, Value, LHS);
     break;
   case ISD::SSUBO:
-    ARMcc = DAG.getConstant(ARMCC::VC, MVT::i32);
-    Value = DAG.getNode(ISD::SUB, SDLoc(Op), Op.getValueType(), LHS, RHS);
-    OverflowCmp = DAG.getNode(ARMISD::CMP, SDLoc(Op), MVT::Glue, LHS, RHS);
+    ARMcc = DAG.getConstant(ARMCC::VC, dl, MVT::i32);
+    Value = DAG.getNode(ISD::SUB, dl, Op.getValueType(), LHS, RHS);
+    OverflowCmp = DAG.getNode(ARMISD::CMP, dl, MVT::Glue, LHS, RHS);
     break;
   case ISD::USUBO:
-    ARMcc = DAG.getConstant(ARMCC::HS, MVT::i32);
-    Value = DAG.getNode(ISD::SUB, SDLoc(Op), Op.getValueType(), LHS, RHS);
-    OverflowCmp = DAG.getNode(ARMISD::CMP, SDLoc(Op), MVT::Glue, LHS, RHS);
+    ARMcc = DAG.getConstant(ARMCC::HS, dl, MVT::i32);
+    Value = DAG.getNode(ISD::SUB, dl, Op.getValueType(), LHS, RHS);
+    OverflowCmp = DAG.getNode(ARMISD::CMP, dl, MVT::Glue, LHS, RHS);
     break;
   } // switch (...)
 
@@ -3445,16 +3362,17 @@ ARMTargetLowering::LowerXALUO(SDValue Op, SelectionDAG &DAG) const {
   SDValue ARMcc;
   std::tie(Value, OverflowCmp) = getARMXALUOOp(Op, DAG, ARMcc);
   SDValue CCR = DAG.getRegister(ARM::CPSR, MVT::i32);
+  SDLoc dl(Op);
   // We use 0 and 1 as false and true values.
-  SDValue TVal = DAG.getConstant(1, MVT::i32);
-  SDValue FVal = DAG.getConstant(0, MVT::i32);
+  SDValue TVal = DAG.getConstant(1, dl, MVT::i32);
+  SDValue FVal = DAG.getConstant(0, dl, MVT::i32);
   EVT VT = Op.getValueType();
 
-  SDValue Overflow = DAG.getNode(ARMISD::CMOV, SDLoc(Op), VT, TVal, FVal,
+  SDValue Overflow = DAG.getNode(ARMISD::CMOV, dl, VT, TVal, FVal,
                                  ARMcc, CCR, OverflowCmp);
 
   SDVTList VTs = DAG.getVTList(Op.getValueType(), MVT::i32);
-  return DAG.getNode(ISD::MERGE_VALUES, SDLoc(Op), VTs, Value, Overflow);
+  return DAG.getNode(ISD::MERGE_VALUES, dl, VTs, Value, Overflow);
 }
 
 
@@ -3477,7 +3395,7 @@ SDValue ARMTargetLowering::LowerSELECT(SDValue Op, SelectionDAG &DAG) const {
     SDValue CCR = DAG.getRegister(ARM::CPSR, MVT::i32);
     EVT VT = Op.getValueType();
 
-    return getCMOV(SDLoc(Op), VT, SelectTrue, SelectFalse, ARMcc, CCR,
+    return getCMOV(dl, VT, SelectTrue, SelectFalse, ARMcc, CCR,
                    OverflowCmp, DAG);
   }
 
@@ -3520,19 +3438,13 @@ SDValue ARMTargetLowering::LowerSELECT(SDValue Op, SelectionDAG &DAG) const {
   // ARM's BooleanContents value is UndefinedBooleanContent. Mask out the
   // undefined bits before doing a full-word comparison with zero.
   Cond = DAG.getNode(ISD::AND, dl, Cond.getValueType(), Cond,
-                     DAG.getConstant(1, Cond.getValueType()));
+                     DAG.getConstant(1, dl, Cond.getValueType()));
 
   return DAG.getSelectCC(dl, Cond,
-                         DAG.getConstant(0, Cond.getValueType()),
+                         DAG.getConstant(0, dl, Cond.getValueType()),
                          SelectTrue, SelectFalse, ISD::SETNE);
 }
 
-static ISD::CondCode getInverseCCForVSEL(ISD::CondCode CC) {
-  if (CC == ISD::SETNE)
-    return ISD::SETEQ;
-  return ISD::getSetCCInverse(CC, true);
-}
-
 static void checkVSELConstraints(ISD::CondCode CC, ARMCC::CondCodes &CondCode,
                                  bool &swpCmpOps, bool &swpVselOps) {
   // Start by selecting the GE condition code for opcodes that return true for
@@ -3624,7 +3536,7 @@ SDValue ARMTargetLowering::LowerSELECT_CC(SDValue Op, SelectionDAG &DAG) const {
     // If softenSetCCOperands only returned one value, we should compare it to
     // zero.
     if (!RHS.getNode()) {
-      RHS = DAG.getConstant(0, LHS.getValueType());
+      RHS = DAG.getConstant(0, dl, LHS.getValueType());
       CC = ISD::SETNE;
     }
   }
@@ -3645,7 +3557,7 @@ SDValue ARMTargetLowering::LowerSELECT_CC(SDValue Op, SelectionDAG &DAG) const {
       ARMCC::CondCodes CondCode = IntCCToARMCC(CC);
       if (CondCode == ARMCC::LT || CondCode == ARMCC::LE ||
           CondCode == ARMCC::VC || CondCode == ARMCC::NE) {
-        CC = getInverseCCForVSEL(CC);
+        CC = ISD::getSetCCInverse(CC, true);
         std::swap(TrueVal, FalseVal);
       }
     }
@@ -3667,27 +3579,105 @@ SDValue ARMTargetLowering::LowerSELECT_CC(SDValue Op, SelectionDAG &DAG) const {
     //   c = fcmp [?gt, ?ge, ?lt, ?le] a, b
     //   select c, a, b
     // In NoNaNsFPMath the CC will have been changed from, e.g., 'ogt' to 'gt'.
-    // We only do this transformation in UnsafeFPMath and for no-NaNs
-    // comparisons, because signed zeros and NaNs are handled differently than
-    // the original code sequence.
-    // FIXME: There are more cases that can be transformed even with NaNs,
-    // signed zeroes and safe math.  E.g. in the following, the result will be
-    // FalseVal if a is a NaN or -0./0. and that's what vmaxnm will give, too.
-    //   c = fcmp ogt, a, 0. ; select c, a, 0. => vmaxnm a, 0.
-    // FIXME: There is similar code that allows some extensions in
-    // AArch64TargetLowering::LowerSELECT_CC that should be shared with this
-    // code.
-    if (getTargetMachine().Options.UnsafeFPMath) {
-      if (LHS == TrueVal && RHS == FalseVal) {
-        if (CC == ISD::SETGT || CC == ISD::SETGE)
-          return DAG.getNode(ARMISD::VMAXNM, dl, VT, TrueVal, FalseVal);
-        if (CC == ISD::SETLT || CC == ISD::SETLE)
-          return DAG.getNode(ARMISD::VMINNM, dl, VT, TrueVal, FalseVal);
-      } else if (LHS == FalseVal && RHS == TrueVal) {
-        if (CC == ISD::SETLT || CC == ISD::SETLE)
-          return DAG.getNode(ARMISD::VMAXNM, dl, VT, TrueVal, FalseVal);
-        if (CC == ISD::SETGT || CC == ISD::SETGE)
-          return DAG.getNode(ARMISD::VMINNM, dl, VT, TrueVal, FalseVal);
+    bool swapSides = false;
+    if (!getTargetMachine().Options.NoNaNsFPMath) {
+      // transformability may depend on which way around we compare
+      switch (CC) {
+      default:
+        break;
+      case ISD::SETOGT:
+      case ISD::SETOGE:
+      case ISD::SETOLT:
+      case ISD::SETOLE:
+        // the non-NaN should be RHS
+        swapSides = DAG.isKnownNeverNaN(LHS) && !DAG.isKnownNeverNaN(RHS);
+        break;
+      case ISD::SETUGT:
+      case ISD::SETUGE:
+      case ISD::SETULT:
+      case ISD::SETULE:
+        // the non-NaN should be LHS
+        swapSides = DAG.isKnownNeverNaN(RHS) && !DAG.isKnownNeverNaN(LHS);
+        break;
+      }
+    }
+    swapSides = swapSides || (LHS == FalseVal && RHS == TrueVal);
+    if (swapSides) {
+      CC = ISD::getSetCCSwappedOperands(CC);
+      std::swap(LHS, RHS);
+    }
+    if (LHS == TrueVal && RHS == FalseVal) {
+      bool canTransform = true;
+      // FIXME: FastMathFlags::noSignedZeros() doesn't appear reachable from here
+      if (!getTargetMachine().Options.UnsafeFPMath &&
+          !DAG.isKnownNeverZero(LHS) && !DAG.isKnownNeverZero(RHS)) {
+        const ConstantFPSDNode *Zero;
+        switch (CC) {
+        default:
+          break;
+        case ISD::SETOGT:
+        case ISD::SETUGT:
+        case ISD::SETGT:
+          // RHS must not be -0
+          canTransform = (Zero = dyn_cast<ConstantFPSDNode>(RHS)) &&
+                         !Zero->isNegative();
+          break;
+        case ISD::SETOGE:
+        case ISD::SETUGE:
+        case ISD::SETGE:
+          // LHS must not be -0
+          canTransform = (Zero = dyn_cast<ConstantFPSDNode>(LHS)) &&
+                         !Zero->isNegative();
+          break;
+        case ISD::SETOLT:
+        case ISD::SETULT:
+        case ISD::SETLT:
+          // RHS must not be +0
+          canTransform = (Zero = dyn_cast<ConstantFPSDNode>(RHS)) &&
+                          Zero->isNegative();
+          break;
+        case ISD::SETOLE:
+        case ISD::SETULE:
+        case ISD::SETLE:
+          // LHS must not be +0
+          canTransform = (Zero = dyn_cast<ConstantFPSDNode>(LHS)) &&
+                          Zero->isNegative();
+          break;
+        }
+      }
+      if (canTransform) {
+        // Note: If one of the elements in a pair is a number and the other
+        // element is NaN, the corresponding result element is the number.
+        // This is consistent with the IEEE 754-2008 standard.
+        // Therefore, a > b ? a : b <=> vmax(a,b), if b is constant and a is NaN
+        switch (CC) {
+        default:
+          break;
+        case ISD::SETOGT:
+        case ISD::SETOGE:
+          if (!DAG.isKnownNeverNaN(RHS))
+            break;
+          return DAG.getNode(ARMISD::VMAXNM, dl, VT, LHS, RHS);
+        case ISD::SETUGT:
+        case ISD::SETUGE:
+          if (!DAG.isKnownNeverNaN(LHS))
+            break;
+        case ISD::SETGT:
+        case ISD::SETGE:
+          return DAG.getNode(ARMISD::VMAXNM, dl, VT, LHS, RHS);
+        case ISD::SETOLT:
+        case ISD::SETOLE:
+          if (!DAG.isKnownNeverNaN(RHS))
+            break;
+          return DAG.getNode(ARMISD::VMINNM, dl, VT, LHS, RHS);
+        case ISD::SETULT:
+        case ISD::SETULE:
+          if (!DAG.isKnownNeverNaN(LHS))
+            break;
+        case ISD::SETLT:
+        case ISD::SETLE:
+          return DAG.getNode(ARMISD::VMINNM, dl, VT, LHS, RHS);
+        }
       }
     }
 
@@ -3704,12 +3694,12 @@ SDValue ARMTargetLowering::LowerSELECT_CC(SDValue Op, SelectionDAG &DAG) const {
     }
   }
 
-  SDValue ARMcc = DAG.getConstant(CondCode, MVT::i32);
+  SDValue ARMcc = DAG.getConstant(CondCode, dl, MVT::i32);
   SDValue Cmp = getVFPCmp(LHS, RHS, DAG, dl);
   SDValue CCR = DAG.getRegister(ARM::CPSR, MVT::i32);
   SDValue Result = getCMOV(dl, VT, FalseVal, TrueVal, ARMcc, CCR, Cmp, DAG);
   if (CondCode2 != ARMCC::AL) {
-    SDValue ARMcc2 = DAG.getConstant(CondCode2, MVT::i32);
+    SDValue ARMcc2 = DAG.getConstant(CondCode2, dl, MVT::i32);
     // FIXME: Needs another CMP because flag can have but one use.
     SDValue Cmp2 = getVFPCmp(LHS, RHS, DAG, dl);
     Result = getCMOV(dl, VT, Result, TrueVal, ARMcc2, CCR, Cmp2, DAG);
@@ -3742,7 +3732,7 @@ static bool canChangeToInt(SDValue Op, bool &SeenZero,
 
 static SDValue bitcastf32Toi32(SDValue Op, SelectionDAG &DAG) {
   if (isFloatingPointZero(Op))
-    return DAG.getConstant(0, MVT::i32);
+    return DAG.getConstant(0, SDLoc(Op), MVT::i32);
 
   if (LoadSDNode *Ld = dyn_cast<LoadSDNode>(Op))
     return DAG.getLoad(MVT::i32, SDLoc(Op),
@@ -3755,15 +3745,17 @@ static SDValue bitcastf32Toi32(SDValue Op, SelectionDAG &DAG) {
 
 static void expandf64Toi32(SDValue Op, SelectionDAG &DAG,
                            SDValue &RetVal1, SDValue &RetVal2) {
+  SDLoc dl(Op);
+
   if (isFloatingPointZero(Op)) {
-    RetVal1 = DAG.getConstant(0, MVT::i32);
-    RetVal2 = DAG.getConstant(0, MVT::i32);
+    RetVal1 = DAG.getConstant(0, dl, MVT::i32);
+    RetVal2 = DAG.getConstant(0, dl, MVT::i32);
     return;
   }
 
   if (LoadSDNode *Ld = dyn_cast<LoadSDNode>(Op)) {
     SDValue Ptr = Ld->getBasePtr();
-    RetVal1 = DAG.getLoad(MVT::i32, SDLoc(Op),
+    RetVal1 = DAG.getLoad(MVT::i32, dl,
                           Ld->getChain(), Ptr,
                           Ld->getPointerInfo(),
                           Ld->isVolatile(), Ld->isNonTemporal(),
@@ -3771,9 +3763,9 @@ static void expandf64Toi32(SDValue Op, SelectionDAG &DAG,
 
     EVT PtrType = Ptr.getValueType();
     unsigned NewAlign = MinAlign(Ld->getAlignment(), 4);
-    SDValue NewPtr = DAG.getNode(ISD::ADD, SDLoc(Op),
-                                 PtrType, Ptr, DAG.getConstant(4, PtrType));
-    RetVal2 = DAG.getLoad(MVT::i32, SDLoc(Op),
+    SDValue NewPtr = DAG.getNode(ISD::ADD, dl,
+                                 PtrType, Ptr, DAG.getConstant(4, dl, PtrType));
+    RetVal2 = DAG.getLoad(MVT::i32, dl,
                           Ld->getChain(), NewPtr,
                           Ld->getPointerInfo().getWithOffset(4),
                           Ld->isVolatile(), Ld->isNonTemporal(),
@@ -3808,7 +3800,7 @@ ARMTargetLowering::OptimizeVFPBrcond(SDValue Op, SelectionDAG &DAG) const {
     else if (CC == ISD::SETUNE)
       CC = ISD::SETNE;
 
-    SDValue Mask = DAG.getConstant(0x7fffffff, MVT::i32);
+    SDValue Mask = DAG.getConstant(0x7fffffff, dl, MVT::i32);
     SDValue ARMcc;
     if (LHS.getValueType() == MVT::f32) {
       LHS = DAG.getNode(ISD::AND, dl, MVT::i32,
@@ -3828,7 +3820,7 @@ ARMTargetLowering::OptimizeVFPBrcond(SDValue Op, SelectionDAG &DAG) const {
     LHS2 = DAG.getNode(ISD::AND, dl, MVT::i32, LHS2, Mask);
     RHS2 = DAG.getNode(ISD::AND, dl, MVT::i32, RHS2, Mask);
     ARMCC::CondCodes CondCode = IntCCToARMCC(CC);
-    ARMcc = DAG.getConstant(CondCode, MVT::i32);
+    ARMcc = DAG.getConstant(CondCode, dl, MVT::i32);
     SDVTList VTList = DAG.getVTList(MVT::Other, MVT::Glue);
     SDValue Ops[] = { Chain, ARMcc, LHS1, LHS2, RHS1, RHS2, Dest };
     return DAG.getNode(ARMISD::BCC_i64, dl, VTList, Ops);
@@ -3852,7 +3844,7 @@ SDValue ARMTargetLowering::LowerBR_CC(SDValue Op, SelectionDAG &DAG) const {
     // If softenSetCCOperands only returned one value, we should compare it to
     // zero.
     if (!RHS.getNode()) {
-      RHS = DAG.getConstant(0, LHS.getValueType());
+      RHS = DAG.getConstant(0, dl, LHS.getValueType());
       CC = ISD::SETNE;
     }
   }
@@ -3878,14 +3870,14 @@ SDValue ARMTargetLowering::LowerBR_CC(SDValue Op, SelectionDAG &DAG) const {
   ARMCC::CondCodes CondCode, CondCode2;
   FPCCToARMCC(CC, CondCode, CondCode2);
 
-  SDValue ARMcc = DAG.getConstant(CondCode, MVT::i32);
+  SDValue ARMcc = DAG.getConstant(CondCode, dl, MVT::i32);
   SDValue Cmp = getVFPCmp(LHS, RHS, DAG, dl);
   SDValue CCR = DAG.getRegister(ARM::CPSR, MVT::i32);
   SDVTList VTList = DAG.getVTList(MVT::Other, MVT::Glue);
   SDValue Ops[] = { Chain, Dest, ARMcc, CCR, Cmp };
   SDValue Res = DAG.getNode(ARMISD::BRCOND, dl, VTList, Ops);
   if (CondCode2 != ARMCC::AL) {
-    ARMcc = DAG.getConstant(CondCode2, MVT::i32);
+    ARMcc = DAG.getConstant(CondCode2, dl, MVT::i32);
     SDValue Ops[] = { Res, Dest, ARMcc, CCR, Res.getValue(1) };
     Res = DAG.getNode(ARMISD::BRCOND, dl, VTList, Ops);
   }
@@ -3898,13 +3890,11 @@ SDValue ARMTargetLowering::LowerBR_JT(SDValue Op, SelectionDAG &DAG) const {
   SDValue Index = Op.getOperand(2);
   SDLoc dl(Op);
 
-  EVT PTy = getPointerTy();
+  EVT PTy = getPointerTy(DAG.getDataLayout());
   JumpTableSDNode *JT = cast<JumpTableSDNode>(Table);
-  ARMFunctionInfo *AFI = DAG.getMachineFunction().getInfo<ARMFunctionInfo>();
-  SDValue UId = DAG.getConstant(AFI->createJumpTableUId(), PTy);
   SDValue JTI = DAG.getTargetJumpTable(JT->getIndex(), PTy);
-  Table = DAG.getNode(ARMISD::WrapperJT, dl, MVT::i32, JTI, UId);
-  Index = DAG.getNode(ISD::MUL, dl, PTy, Index, DAG.getConstant(4, PTy));
+  Table = DAG.getNode(ARMISD::WrapperJT, dl, MVT::i32, JTI);
+  Index = DAG.getNode(ISD::MUL, dl, PTy, Index, DAG.getConstant(4, dl, PTy));
   SDValue Addr = DAG.getNode(ISD::ADD, dl, PTy, Index, Table);
   if (Subtarget->isThumb2()) {
     // Thumb2 uses a two-level jump. That is, it jumps into the jump table
@@ -3912,7 +3902,7 @@ SDValue ARMTargetLowering::LowerBR_JT(SDValue Op, SelectionDAG &DAG) const {
     // to translate it to TBB / TBH later.
     // FIXME: This might not work if the function is extremely large.
     return DAG.getNode(ARMISD::BR2_JT, dl, MVT::Other, Chain,
-                       Addr, Op.getOperand(2), JTI, UId);
+                       Addr, Op.getOperand(2), JTI);
   }
   if (getTargetMachine().getRelocationModel() == Reloc::PIC_) {
     Addr = DAG.getLoad((EVT)MVT::i32, dl, Chain, Addr,
@@ -3920,13 +3910,13 @@ SDValue ARMTargetLowering::LowerBR_JT(SDValue Op, SelectionDAG &DAG) const {
                        false, false, false, 0);
     Chain = Addr.getValue(1);
     Addr = DAG.getNode(ISD::ADD, dl, PTy, Addr, Table);
-    return DAG.getNode(ARMISD::BR_JT, dl, MVT::Other, Chain, Addr, JTI, UId);
+    return DAG.getNode(ARMISD::BR_JT, dl, MVT::Other, Chain, Addr, JTI);
   } else {
     Addr = DAG.getLoad(PTy, dl, Chain, Addr,
                        MachinePointerInfo::getJumpTable(),
                        false, false, false, 0);
     Chain = Addr.getValue(1);
-    return DAG.getNode(ARMISD::BR_JT, dl, MVT::Other, Chain, Addr, JTI, UId);
+    return DAG.getNode(ARMISD::BR_JT, dl, MVT::Other, Chain, Addr, JTI);
   }
 }
 
@@ -4035,12 +4025,12 @@ SDValue ARMTargetLowering::LowerFCOPYSIGN(SDValue Op, SelectionDAG &DAG) const {
     // Use VBSL to copy the sign bit.
     unsigned EncodedVal = ARM_AM::createNEONModImm(0x6, 0x80);
     SDValue Mask = DAG.getNode(ARMISD::VMOVIMM, dl, MVT::v2i32,
-                               DAG.getTargetConstant(EncodedVal, MVT::i32));
+                               DAG.getTargetConstant(EncodedVal, dl, MVT::i32));
     EVT OpVT = (VT == MVT::f32) ? MVT::v2i32 : MVT::v1i64;
     if (VT == MVT::f64)
       Mask = DAG.getNode(ARMISD::VSHL, dl, OpVT,
                          DAG.getNode(ISD::BITCAST, dl, OpVT, Mask),
-                         DAG.getConstant(32, MVT::i32));
+                         DAG.getConstant(32, dl, MVT::i32));
     else /*if (VT == MVT::f32)*/
       Tmp0 = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, MVT::v2f32, Tmp0);
     if (SrcVT == MVT::f32) {
@@ -4048,16 +4038,16 @@ SDValue ARMTargetLowering::LowerFCOPYSIGN(SDValue Op, SelectionDAG &DAG) const {
       if (VT == MVT::f64)
         Tmp1 = DAG.getNode(ARMISD::VSHL, dl, OpVT,
                            DAG.getNode(ISD::BITCAST, dl, OpVT, Tmp1),
-                           DAG.getConstant(32, MVT::i32));
+                           DAG.getConstant(32, dl, MVT::i32));
     } else if (VT == MVT::f32)
       Tmp1 = DAG.getNode(ARMISD::VSHRu, dl, MVT::v1i64,
                          DAG.getNode(ISD::BITCAST, dl, MVT::v1i64, Tmp1),
-                         DAG.getConstant(32, MVT::i32));
+                         DAG.getConstant(32, dl, MVT::i32));
     Tmp0 = DAG.getNode(ISD::BITCAST, dl, OpVT, Tmp0);
     Tmp1 = DAG.getNode(ISD::BITCAST, dl, OpVT, Tmp1);
 
     SDValue AllOnes = DAG.getTargetConstant(ARM_AM::createNEONModImm(0xe, 0xff),
-                                            MVT::i32);
+                                            dl, MVT::i32);
     AllOnes = DAG.getNode(ARMISD::VMOVIMM, dl, MVT::v8i8, AllOnes);
     SDValue MaskNot = DAG.getNode(ISD::XOR, dl, OpVT, Mask,
                                   DAG.getNode(ISD::BITCAST, dl, OpVT, AllOnes));
@@ -4068,7 +4058,7 @@ SDValue ARMTargetLowering::LowerFCOPYSIGN(SDValue Op, SelectionDAG &DAG) const {
     if (VT == MVT::f32) {
       Res = DAG.getNode(ISD::BITCAST, dl, MVT::v2f32, Res);
       Res = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::f32, Res,
-                        DAG.getConstant(0, MVT::i32));
+                        DAG.getConstant(0, dl, MVT::i32));
     } else {
       Res = DAG.getNode(ISD::BITCAST, dl, MVT::f64, Res);
     }
@@ -4083,8 +4073,8 @@ SDValue ARMTargetLowering::LowerFCOPYSIGN(SDValue Op, SelectionDAG &DAG) const {
   Tmp1 = DAG.getNode(ISD::BITCAST, dl, MVT::i32, Tmp1);
 
   // Or in the signbit with integer operations.
-  SDValue Mask1 = DAG.getConstant(0x80000000, MVT::i32);
-  SDValue Mask2 = DAG.getConstant(0x7fffffff, MVT::i32);
+  SDValue Mask1 = DAG.getConstant(0x80000000, dl, MVT::i32);
+  SDValue Mask2 = DAG.getConstant(0x7fffffff, dl, MVT::i32);
   Tmp1 = DAG.getNode(ISD::AND, dl, MVT::i32, Tmp1, Mask1);
   if (VT == MVT::f32) {
     Tmp0 = DAG.getNode(ISD::AND, dl, MVT::i32,
@@ -4115,7 +4105,7 @@ SDValue ARMTargetLowering::LowerRETURNADDR(SDValue Op, SelectionDAG &DAG) const{
   unsigned Depth = cast<ConstantSDNode>(Op.getOperand(0))->getZExtValue();
   if (Depth) {
     SDValue FrameAddr = LowerFRAMEADDR(Op, DAG);
-    SDValue Offset = DAG.getConstant(4, MVT::i32);
+    SDValue Offset = DAG.getConstant(4, dl, MVT::i32);
     return DAG.getLoad(VT, dl, DAG.getEntryNode(),
                        DAG.getNode(ISD::ADD, dl, VT, FrameAddr, Offset),
                        MachinePointerInfo(), false, false, false, 0);
@@ -4147,14 +4137,35 @@ SDValue ARMTargetLowering::LowerFRAMEADDR(SDValue Op, SelectionDAG &DAG) const {
 
 // FIXME? Maybe this could be a TableGen attribute on some registers and
 // this table could be generated automatically from RegInfo.
-unsigned ARMTargetLowering::getRegisterByName(const char* RegName,
-                                              EVT VT) const {
+unsigned ARMTargetLowering::getRegisterByName(const char* RegName, EVT VT,
+                                              SelectionDAG &DAG) const {
   unsigned Reg = StringSwitch<unsigned>(RegName)
                        .Case("sp", ARM::SP)
                        .Default(0);
   if (Reg)
     return Reg;
-  report_fatal_error("Invalid register name global variable");
+  report_fatal_error(Twine("Invalid register name \""
+                              + StringRef(RegName)  + "\"."));
+}
+
+// Result is 64 bit value so split into two 32 bit values and return as a
+// pair of values.
+static void ExpandREAD_REGISTER(SDNode *N, SmallVectorImpl<SDValue> &Results,
+                                SelectionDAG &DAG) {
+  SDLoc DL(N);
+
+  // This function is only supposed to be called for i64 type destination.
+  assert(N->getValueType(0) == MVT::i64
+          && "ExpandREAD_REGISTER called for non-i64 type result.");
+
+  SDValue Read = DAG.getNode(ISD::READ_REGISTER, DL,
+                             DAG.getVTList(MVT::i32, MVT::i32, MVT::Other),
+                             N->getOperand(0),
+                             N->getOperand(1));
+
+  Results.push_back(DAG.getNode(ISD::BUILD_PAIR, DL, MVT::i64, Read.getValue(0),
+                    Read.getValue(1)));
+  Results.push_back(Read.getOperand(0));
 }
 
 /// ExpandBITCAST - If the target supports VFP, this function is called to
@@ -4177,9 +4188,9 @@ static SDValue ExpandBITCAST(SDNode *N, SelectionDAG &DAG) {
   // Turn i64->f64 into VMOVDRR.
   if (SrcVT == MVT::i64 && TLI.isTypeLegal(DstVT)) {
     SDValue Lo = DAG.getNode(ISD::EXTRACT_ELEMENT, dl, MVT::i32, Op,
-                             DAG.getConstant(0, MVT::i32));
+                             DAG.getConstant(0, dl, MVT::i32));
     SDValue Hi = DAG.getNode(ISD::EXTRACT_ELEMENT, dl, MVT::i32, Op,
-                             DAG.getConstant(1, MVT::i32));
+                             DAG.getConstant(1, dl, MVT::i32));
     return DAG.getNode(ISD::BITCAST, dl, DstVT,
                        DAG.getNode(ARMISD::VMOVDRR, dl, MVT::f64, Lo, Hi));
   }
@@ -4187,7 +4198,7 @@ static SDValue ExpandBITCAST(SDNode *N, SelectionDAG &DAG) {
   // Turn f64->i64 into VMOVRRD.
   if (DstVT == MVT::i64 && TLI.isTypeLegal(SrcVT)) {
     SDValue Cvt;
-    if (TLI.isBigEndian() && SrcVT.isVector() &&
+    if (DAG.getDataLayout().isBigEndian() && SrcVT.isVector() &&
         SrcVT.getVectorNumElements() > 1)
       Cvt = DAG.getNode(ARMISD::VMOVRRD, dl,
                         DAG.getVTList(MVT::i32, MVT::i32),
@@ -4211,7 +4222,7 @@ static SDValue ExpandBITCAST(SDNode *N, SelectionDAG &DAG) {
 static SDValue getZeroVector(EVT VT, SelectionDAG &DAG, SDLoc dl) {
   assert(VT.isVector() && "Expected a vector type");
   // The canonical modified immediate encoding of a zero vector is....0!
-  SDValue EncodedVal = DAG.getTargetConstant(0, MVT::i32);
+  SDValue EncodedVal = DAG.getTargetConstant(0, dl, MVT::i32);
   EVT VmovVT = VT.is128BitVector() ? MVT::v4i32 : MVT::v2i32;
   SDValue Vmov = DAG.getNode(ARMISD::VMOVIMM, dl, VmovVT, EncodedVal);
   return DAG.getNode(ISD::BITCAST, dl, VT, Vmov);
@@ -4234,17 +4245,17 @@ SDValue ARMTargetLowering::LowerShiftRightParts(SDValue Op,
   assert(Op.getOpcode() == ISD::SRA_PARTS || Op.getOpcode() == ISD::SRL_PARTS);
 
   SDValue RevShAmt = DAG.getNode(ISD::SUB, dl, MVT::i32,
-                                 DAG.getConstant(VTBits, MVT::i32), ShAmt);
+                                 DAG.getConstant(VTBits, dl, MVT::i32), ShAmt);
   SDValue Tmp1 = DAG.getNode(ISD::SRL, dl, VT, ShOpLo, ShAmt);
   SDValue ExtraShAmt = DAG.getNode(ISD::SUB, dl, MVT::i32, ShAmt,
-                                   DAG.getConstant(VTBits, MVT::i32));
+                                   DAG.getConstant(VTBits, dl, MVT::i32));
   SDValue Tmp2 = DAG.getNode(ISD::SHL, dl, VT, ShOpHi, RevShAmt);
   SDValue FalseVal = DAG.getNode(ISD::OR, dl, VT, Tmp1, Tmp2);
   SDValue TrueVal = DAG.getNode(Opc, dl, VT, ShOpHi, ExtraShAmt);
 
   SDValue CCR = DAG.getRegister(ARM::CPSR, MVT::i32);
-  SDValue Cmp = getARMCmp(ExtraShAmt, DAG.getConstant(0, MVT::i32), ISD::SETGE,
-                          ARMcc, DAG, dl);
+  SDValue Cmp = getARMCmp(ExtraShAmt, DAG.getConstant(0, dl, MVT::i32),
+                          ISD::SETGE, ARMcc, DAG, dl);
   SDValue Hi = DAG.getNode(Opc, dl, VT, ShOpHi, ShAmt);
   SDValue Lo = DAG.getNode(ARMISD::CMOV, dl, VT, FalseVal, TrueVal, ARMcc,
                            CCR, Cmp);
@@ -4268,17 +4279,17 @@ SDValue ARMTargetLowering::LowerShiftLeftParts(SDValue Op,
 
   assert(Op.getOpcode() == ISD::SHL_PARTS);
   SDValue RevShAmt = DAG.getNode(ISD::SUB, dl, MVT::i32,
-                                 DAG.getConstant(VTBits, MVT::i32), ShAmt);
+                                 DAG.getConstant(VTBits, dl, MVT::i32), ShAmt);
   SDValue Tmp1 = DAG.getNode(ISD::SRL, dl, VT, ShOpLo, RevShAmt);
   SDValue ExtraShAmt = DAG.getNode(ISD::SUB, dl, MVT::i32, ShAmt,
-                                   DAG.getConstant(VTBits, MVT::i32));
+                                   DAG.getConstant(VTBits, dl, MVT::i32));
   SDValue Tmp2 = DAG.getNode(ISD::SHL, dl, VT, ShOpHi, ShAmt);
   SDValue Tmp3 = DAG.getNode(ISD::SHL, dl, VT, ShOpLo, ExtraShAmt);
 
   SDValue FalseVal = DAG.getNode(ISD::OR, dl, VT, Tmp1, Tmp2);
   SDValue CCR = DAG.getRegister(ARM::CPSR, MVT::i32);
-  SDValue Cmp = getARMCmp(ExtraShAmt, DAG.getConstant(0, MVT::i32), ISD::SETGE,
-                          ARMcc, DAG, dl);
+  SDValue Cmp = getARMCmp(ExtraShAmt, DAG.getConstant(0, dl, MVT::i32),
+                          ISD::SETGE, ARMcc, DAG, dl);
   SDValue Lo = DAG.getNode(ISD::SHL, dl, VT, ShOpLo, ShAmt);
   SDValue Hi = DAG.getNode(ARMISD::CMOV, dl, VT, FalseVal, Tmp3, ARMcc,
                            CCR, Cmp);
@@ -4295,20 +4306,94 @@ SDValue ARMTargetLowering::LowerFLT_ROUNDS_(SDValue Op,
   // so that the shift + and get folded into a bitfield extract.
   SDLoc dl(Op);
   SDValue FPSCR = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, dl, MVT::i32,
-                              DAG.getConstant(Intrinsic::arm_get_fpscr,
+                              DAG.getConstant(Intrinsic::arm_get_fpscr, dl,
                                               MVT::i32));
   SDValue FltRounds = DAG.getNode(ISD::ADD, dl, MVT::i32, FPSCR,
-                                  DAG.getConstant(1U << 22, MVT::i32));
+                                  DAG.getConstant(1U << 22, dl, MVT::i32));
   SDValue RMODE = DAG.getNode(ISD::SRL, dl, MVT::i32, FltRounds,
-                              DAG.getConstant(22, MVT::i32));
+                              DAG.getConstant(22, dl, MVT::i32));
   return DAG.getNode(ISD::AND, dl, MVT::i32, RMODE,
-                     DAG.getConstant(3, MVT::i32));
+                     DAG.getConstant(3, dl, MVT::i32));
 }
 
 static SDValue LowerCTTZ(SDNode *N, SelectionDAG &DAG,
                          const ARMSubtarget *ST) {
-  EVT VT = N->getValueType(0);
   SDLoc dl(N);
+  EVT VT = N->getValueType(0);
+  if (VT.isVector()) {
+    assert(ST->hasNEON());
+
+    // Compute the least significant set bit: LSB = X & -X
+    SDValue X = N->getOperand(0);
+    SDValue NX = DAG.getNode(ISD::SUB, dl, VT, getZeroVector(VT, DAG, dl), X);
+    SDValue LSB = DAG.getNode(ISD::AND, dl, VT, X, NX);
+
+    EVT ElemTy = VT.getVectorElementType();
+
+    if (ElemTy == MVT::i8) {
+      // Compute with: cttz(x) = ctpop(lsb - 1)
+      SDValue One = DAG.getNode(ARMISD::VMOVIMM, dl, VT,
+                                DAG.getTargetConstant(1, dl, ElemTy));
+      SDValue Bits = DAG.getNode(ISD::SUB, dl, VT, LSB, One);
+      return DAG.getNode(ISD::CTPOP, dl, VT, Bits);
+    }
+
+    if ((ElemTy == MVT::i16 || ElemTy == MVT::i32) &&
+        (N->getOpcode() == ISD::CTTZ_ZERO_UNDEF)) {
+      // Compute with: cttz(x) = (width - 1) - ctlz(lsb), if x != 0
+      unsigned NumBits = ElemTy.getSizeInBits();
+      SDValue WidthMinus1 =
+          DAG.getNode(ARMISD::VMOVIMM, dl, VT,
+                      DAG.getTargetConstant(NumBits - 1, dl, ElemTy));
+      SDValue CTLZ = DAG.getNode(ISD::CTLZ, dl, VT, LSB);
+      return DAG.getNode(ISD::SUB, dl, VT, WidthMinus1, CTLZ);
+    }
+
+    // Compute with: cttz(x) = ctpop(lsb - 1)
+
+    // Since we can only compute the number of bits in a byte with vcnt.8, we
+    // have to gather the result with pairwise addition (vpaddl) for i16, i32,
+    // and i64.
+
+    // Compute LSB - 1.
+    SDValue Bits;
+    if (ElemTy == MVT::i64) {
+      // Load constant 0xffff'ffff'ffff'ffff to register.
+      SDValue FF = DAG.getNode(ARMISD::VMOVIMM, dl, VT,
+                               DAG.getTargetConstant(0x1eff, dl, MVT::i32));
+      Bits = DAG.getNode(ISD::ADD, dl, VT, LSB, FF);
+    } else {
+      SDValue One = DAG.getNode(ARMISD::VMOVIMM, dl, VT,
+                                DAG.getTargetConstant(1, dl, ElemTy));
+      Bits = DAG.getNode(ISD::SUB, dl, VT, LSB, One);
+    }
+
+    // Count #bits with vcnt.8.
+    EVT VT8Bit = VT.is64BitVector() ? MVT::v8i8 : MVT::v16i8;
+    SDValue BitsVT8 = DAG.getNode(ISD::BITCAST, dl, VT8Bit, Bits);
+    SDValue Cnt8 = DAG.getNode(ISD::CTPOP, dl, VT8Bit, BitsVT8);
+
+    // Gather the #bits with vpaddl (pairwise add.)
+    EVT VT16Bit = VT.is64BitVector() ? MVT::v4i16 : MVT::v8i16;
+    SDValue Cnt16 = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, dl, VT16Bit,
+        DAG.getTargetConstant(Intrinsic::arm_neon_vpaddlu, dl, MVT::i32),
+        Cnt8);
+    if (ElemTy == MVT::i16)
+      return Cnt16;
+
+    EVT VT32Bit = VT.is64BitVector() ? MVT::v2i32 : MVT::v4i32;
+    SDValue Cnt32 = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, dl, VT32Bit,
+        DAG.getTargetConstant(Intrinsic::arm_neon_vpaddlu, dl, MVT::i32),
+        Cnt16);
+    if (ElemTy == MVT::i32)
+      return Cnt32;
+
+    assert(ElemTy == MVT::i64);
+    SDValue Cnt64 = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, dl, VT,
+        DAG.getTargetConstant(Intrinsic::arm_neon_vpaddlu, dl, MVT::i32),
+        Cnt32);
+    return Cnt64;
+  }
 
   if (!ST->hasV6T2Ops())
     return SDValue();
@@ -4360,10 +4445,10 @@ static SDValue lowerCTPOP16BitElements(SDNode *N, SelectionDAG &DAG) {
   if (VT.is64BitVector()) {
     SDValue Extended = DAG.getNode(ISD::ZERO_EXTEND, DL, MVT::v8i16, BitCounts);
     return DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, MVT::v4i16, Extended,
-                       DAG.getIntPtrConstant(0));
+                       DAG.getIntPtrConstant(0, DL));
   } else {
     SDValue Extracted = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, MVT::v8i8,
-                                    BitCounts, DAG.getIntPtrConstant(0));
+                                    BitCounts, DAG.getIntPtrConstant(0, DL));
     return DAG.getNode(ISD::ZERO_EXTEND, DL, MVT::v8i16, Extracted);
   }
 }
@@ -4402,10 +4487,10 @@ static SDValue lowerCTPOP32BitElements(SDNode *N, SelectionDAG &DAG) {
   if (VT.is64BitVector()) {
     SDValue Extended = DAG.getNode(ISD::ZERO_EXTEND, DL, MVT::v4i32, N2);
     return DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, MVT::v2i32, Extended,
-                       DAG.getIntPtrConstant(0));
+                       DAG.getIntPtrConstant(0, DL));
   } else {
     SDValue Extracted = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, MVT::v4i16, N2,
-                                    DAG.getIntPtrConstant(0));
+                                    DAG.getIntPtrConstant(0, DL));
     return DAG.getNode(ISD::ZERO_EXTEND, DL, MVT::v4i32, Extracted);
   }
 }
@@ -4439,7 +4524,8 @@ static SDValue LowerShift(SDNode *N, SelectionDAG &DAG,
   // Left shifts translate directly to the vshiftu intrinsic.
   if (N->getOpcode() == ISD::SHL)
     return DAG.getNode(ISD::INTRINSIC_WO_CHAIN, dl, VT,
-                       DAG.getConstant(Intrinsic::arm_neon_vshiftu, MVT::i32),
+                       DAG.getConstant(Intrinsic::arm_neon_vshiftu, dl,
+                                       MVT::i32),
                        N->getOperand(0), N->getOperand(1));
 
   assert((N->getOpcode() == ISD::SRA ||
@@ -4456,7 +4542,7 @@ static SDValue LowerShift(SDNode *N, SelectionDAG &DAG,
                              Intrinsic::arm_neon_vshifts :
                              Intrinsic::arm_neon_vshiftu);
   return DAG.getNode(ISD::INTRINSIC_WO_CHAIN, dl, VT,
-                     DAG.getConstant(vshiftInt, MVT::i32),
+                     DAG.getConstant(vshiftInt, dl, MVT::i32),
                      N->getOperand(0), NegatedCount);
 }
 
@@ -4482,9 +4568,9 @@ static SDValue Expand64BitShift(SDNode *N, SelectionDAG &DAG,
 
   // Okay, we have a 64-bit SRA or SRL of 1.  Lower this to an RRX expr.
   SDValue Lo = DAG.getNode(ISD::EXTRACT_ELEMENT, dl, MVT::i32, N->getOperand(0),
-                           DAG.getConstant(0, MVT::i32));
+                           DAG.getConstant(0, dl, MVT::i32));
   SDValue Hi = DAG.getNode(ISD::EXTRACT_ELEMENT, dl, MVT::i32, N->getOperand(0),
-                           DAG.getConstant(1, MVT::i32));
+                           DAG.getConstant(1, dl, MVT::i32));
 
   // First, build a SRA_FLAG/SRL_FLAG op, which shifts the top part by one and
   // captures the result into a carry flag.
@@ -4637,7 +4723,8 @@ static SDValue LowerVSETCC(SDValue Op, SelectionDAG &DAG) {
 /// operand (e.g., VMOV).  If so, return the encoded value.
 static SDValue isNEONModifiedImm(uint64_t SplatBits, uint64_t SplatUndef,
                                  unsigned SplatBitSize, SelectionDAG &DAG,
-                                 EVT &VT, bool is128Bits, NEONModImmType type) {
+                                 SDLoc dl, EVT &VT, bool is128Bits,
+                                 NEONModImmType type) {
   unsigned OpCmode, Imm;
 
   // SplatBitSize is set to the smallest size that splats the vector, so a
@@ -4752,7 +4839,7 @@ static SDValue isNEONModifiedImm(uint64_t SplatBits, uint64_t SplatUndef,
       ImmMask <<= 1;
     }
 
-    if (DAG.getTargetLoweringInfo().isBigEndian())
+    if (DAG.getDataLayout().isBigEndian())
       // swap higher and lower 32 bit word
       Imm = ((Imm & 0xf) << 4) | ((Imm & 0xf0) >> 4);
 
@@ -4767,7 +4854,7 @@ static SDValue isNEONModifiedImm(uint64_t SplatBits, uint64_t SplatUndef,
   }
 
   unsigned EncodedVal = ARM_AM::createNEONModImm(OpCmode, Imm);
-  return DAG.getTargetConstant(EncodedVal, MVT::i32);
+  return DAG.getTargetConstant(EncodedVal, dl, MVT::i32);
 }
 
 SDValue ARMTargetLowering::LowerConstantFP(SDValue Op, SelectionDAG &DAG,
@@ -4797,11 +4884,11 @@ SDValue ARMTargetLowering::LowerConstantFP(SDValue Op, SelectionDAG &DAG,
     // It's a float and we are trying to use NEON operations where
     // possible. Lower it to a splat followed by an extract.
     SDLoc DL(Op);
-    SDValue NewVal = DAG.getTargetConstant(ImmVal, MVT::i32);
+    SDValue NewVal = DAG.getTargetConstant(ImmVal, DL, MVT::i32);
     SDValue VecConstant = DAG.getNode(ARMISD::VMOVFPIMM, DL, MVT::v2f32,
                                       NewVal);
     return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, MVT::f32, VecConstant,
-                       DAG.getConstant(0, MVT::i32));
+                       DAG.getConstant(0, DL, MVT::i32));
   }
 
   // The rest of our options are NEON only, make sure that's allowed before
@@ -4819,8 +4906,8 @@ SDValue ARMTargetLowering::LowerConstantFP(SDValue Op, SelectionDAG &DAG,
     return SDValue();
 
   // Try a VMOV.i32 (FIXME: i8, i16, or i64 could work too).
-  SDValue NewVal = isNEONModifiedImm(iVal & 0xffffffffU, 0, 32, DAG, VMovVT,
-                                     false, VMOVModImm);
+  SDValue NewVal = isNEONModifiedImm(iVal & 0xffffffffU, 0, 32, DAG, SDLoc(Op),
+                                     VMovVT, false, VMOVModImm);
   if (NewVal != SDValue()) {
     SDLoc DL(Op);
     SDValue VecConstant = DAG.getNode(ARMISD::VMOVIMM, DL, VMovVT,
@@ -4832,11 +4919,11 @@ SDValue ARMTargetLowering::LowerConstantFP(SDValue Op, SelectionDAG &DAG,
     SDValue VecFConstant = DAG.getNode(ISD::BITCAST, DL, MVT::v2f32,
                                        VecConstant);
     return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, MVT::f32, VecFConstant,
-                       DAG.getConstant(0, MVT::i32));
+                       DAG.getConstant(0, DL, MVT::i32));
   }
 
   // Finally, try a VMVN.i32
-  NewVal = isNEONModifiedImm(~iVal & 0xffffffffU, 0, 32, DAG, VMovVT,
+  NewVal = isNEONModifiedImm(~iVal & 0xffffffffU, 0, 32, DAG, SDLoc(Op), VMovVT,
                              false, VMVNModImm);
   if (NewVal != SDValue()) {
     SDLoc DL(Op);
@@ -4849,7 +4936,7 @@ SDValue ARMTargetLowering::LowerConstantFP(SDValue Op, SelectionDAG &DAG,
     SDValue VecFConstant = DAG.getNode(ISD::BITCAST, DL, MVT::v2f32,
                                        VecConstant);
     return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, MVT::f32, VecFConstant,
-                       DAG.getConstant(0, MVT::i32));
+                       DAG.getConstant(0, DL, MVT::i32));
   }
 
   return SDValue();
@@ -4958,18 +5045,50 @@ static bool isVTBLMask(ArrayRef<int> M, EVT VT) {
   return VT == MVT::v8i8 && M.size() == 8;
 }
 
+// Checks whether the shuffle mask represents a vector transpose (VTRN) by
+// checking that pairs of elements in the shuffle mask represent the same index
+// in each vector, incrementing the expected index by 2 at each step.
+// e.g. For v1,v2 of type v4i32 a valid shuffle mask is: [0, 4, 2, 6]
+//  v1={a,b,c,d} => x=shufflevector v1, v2 shufflemask => x={a,e,c,g}
+//  v2={e,f,g,h}
+// WhichResult gives the offset for each element in the mask based on which
+// of the two results it belongs to.
+//
+// The transpose can be represented either as:
+// result1 = shufflevector v1, v2, result1_shuffle_mask
+// result2 = shufflevector v1, v2, result2_shuffle_mask
+// where v1/v2 and the shuffle masks have the same number of elements
+// (here WhichResult (see below) indicates which result is being checked)
+//
+// or as:
+// results = shufflevector v1, v2, shuffle_mask
+// where both results are returned in one vector and the shuffle mask has twice
+// as many elements as v1/v2 (here WhichResult will always be 0 if true) here we
+// want to check the low half and high half of the shuffle mask as if it were
+// the other case
 static bool isVTRNMask(ArrayRef<int> M, EVT VT, unsigned &WhichResult) {
   unsigned EltSz = VT.getVectorElementType().getSizeInBits();
   if (EltSz == 64)
     return false;
 
   unsigned NumElts = VT.getVectorNumElements();
-  WhichResult = (M[0] == 0 ? 0 : 1);
-  for (unsigned i = 0; i < NumElts; i += 2) {
-    if ((M[i] >= 0 && (unsigned) M[i] != i + WhichResult) ||
-        (M[i+1] >= 0 && (unsigned) M[i+1] != i + NumElts + WhichResult))
-      return false;
+  if (M.size() != NumElts && M.size() != NumElts*2)
+    return false;
+
+  // If the mask is twice as long as the result then we need to check the upper
+  // and lower parts of the mask
+  for (unsigned i = 0; i < M.size(); i += NumElts) {
+    WhichResult = M[i] == 0 ? 0 : 1;
+    for (unsigned j = 0; j < NumElts; j += 2) {
+      if ((M[i+j] >= 0 && (unsigned) M[i+j] != j + WhichResult) ||
+          (M[i+j+1] >= 0 && (unsigned) M[i+j+1] != j + NumElts + WhichResult))
+        return false;
+    }
   }
+
+  if (M.size() == NumElts*2)
+    WhichResult = 0;
+
   return true;
 }
 
@@ -4982,28 +5101,52 @@ static bool isVTRN_v_undef_Mask(ArrayRef<int> M, EVT VT, unsigned &WhichResult){
     return false;
 
   unsigned NumElts = VT.getVectorNumElements();
-  WhichResult = (M[0] == 0 ? 0 : 1);
-  for (unsigned i = 0; i < NumElts; i += 2) {
-    if ((M[i] >= 0 && (unsigned) M[i] != i + WhichResult) ||
-        (M[i+1] >= 0 && (unsigned) M[i+1] != i + WhichResult))
-      return false;
+  if (M.size() != NumElts && M.size() != NumElts*2)
+    return false;
+
+  for (unsigned i = 0; i < M.size(); i += NumElts) {
+    WhichResult = M[i] == 0 ? 0 : 1;
+    for (unsigned j = 0; j < NumElts; j += 2) {
+      if ((M[i+j] >= 0 && (unsigned) M[i+j] != j + WhichResult) ||
+          (M[i+j+1] >= 0 && (unsigned) M[i+j+1] != j + WhichResult))
+        return false;
+    }
   }
+
+  if (M.size() == NumElts*2)
+    WhichResult = 0;
+
   return true;
 }
 
+// Checks whether the shuffle mask represents a vector unzip (VUZP) by checking
+// that the mask elements are either all even and in steps of size 2 or all odd
+// and in steps of size 2.
+// e.g. For v1,v2 of type v4i32 a valid shuffle mask is: [0, 2, 4, 6]
+//  v1={a,b,c,d} => x=shufflevector v1, v2 shufflemask => x={a,c,e,g}
+//  v2={e,f,g,h}
+// Requires similar checks to that of isVTRNMask with
+// respect the how results are returned.
 static bool isVUZPMask(ArrayRef<int> M, EVT VT, unsigned &WhichResult) {
   unsigned EltSz = VT.getVectorElementType().getSizeInBits();
   if (EltSz == 64)
     return false;
 
   unsigned NumElts = VT.getVectorNumElements();
-  WhichResult = (M[0] == 0 ? 0 : 1);
-  for (unsigned i = 0; i != NumElts; ++i) {
-    if (M[i] < 0) continue; // ignore UNDEF indices
-    if ((unsigned) M[i] != 2 * i + WhichResult)
-      return false;
+  if (M.size() != NumElts && M.size() != NumElts*2)
+    return false;
+
+  for (unsigned i = 0; i < M.size(); i += NumElts) {
+    WhichResult = M[i] == 0 ? 0 : 1;
+    for (unsigned j = 0; j < NumElts; ++j) {
+      if (M[i+j] >= 0 && (unsigned) M[i+j] != 2 * j + WhichResult)
+        return false;
+    }
   }
 
+  if (M.size() == NumElts*2)
+    WhichResult = 0;
+
   // VUZP.32 for 64-bit vectors is a pseudo-instruction alias for VTRN.32.
   if (VT.is64BitVector() && EltSz == 32)
     return false;
@@ -5019,18 +5162,27 @@ static bool isVUZP_v_undef_Mask(ArrayRef<int> M, EVT VT, unsigned &WhichResult){
   if (EltSz == 64)
     return false;
 
-  unsigned Half = VT.getVectorNumElements() / 2;
-  WhichResult = (M[0] == 0 ? 0 : 1);
-  for (unsigned j = 0; j != 2; ++j) {
-    unsigned Idx = WhichResult;
-    for (unsigned i = 0; i != Half; ++i) {
-      int MIdx = M[i + j * Half];
-      if (MIdx >= 0 && (unsigned) MIdx != Idx)
-        return false;
-      Idx += 2;
+  unsigned NumElts = VT.getVectorNumElements();
+  if (M.size() != NumElts && M.size() != NumElts*2)
+    return false;
+
+  unsigned Half = NumElts / 2;
+  for (unsigned i = 0; i < M.size(); i += NumElts) {
+    WhichResult = M[i] == 0 ? 0 : 1;
+    for (unsigned j = 0; j < NumElts; j += Half) {
+      unsigned Idx = WhichResult;
+      for (unsigned k = 0; k < Half; ++k) {
+        int MIdx = M[i + j + k];
+        if (MIdx >= 0 && (unsigned) MIdx != Idx)
+          return false;
+        Idx += 2;
+      }
     }
   }
 
+  if (M.size() == NumElts*2)
+    WhichResult = 0;
+
   // VUZP.32 for 64-bit vectors is a pseudo-instruction alias for VTRN.32.
   if (VT.is64BitVector() && EltSz == 32)
     return false;
@@ -5038,21 +5190,37 @@ static bool isVUZP_v_undef_Mask(ArrayRef<int> M, EVT VT, unsigned &WhichResult){
   return true;
 }
 
+// Checks whether the shuffle mask represents a vector zip (VZIP) by checking
+// that pairs of elements of the shufflemask represent the same index in each
+// vector incrementing sequentially through the vectors.
+// e.g. For v1,v2 of type v4i32 a valid shuffle mask is: [0, 4, 1, 5]
+//  v1={a,b,c,d} => x=shufflevector v1, v2 shufflemask => x={a,e,b,f}
+//  v2={e,f,g,h}
+// Requires similar checks to that of isVTRNMask with respect the how results
+// are returned.
 static bool isVZIPMask(ArrayRef<int> M, EVT VT, unsigned &WhichResult) {
   unsigned EltSz = VT.getVectorElementType().getSizeInBits();
   if (EltSz == 64)
     return false;
 
   unsigned NumElts = VT.getVectorNumElements();
-  WhichResult = (M[0] == 0 ? 0 : 1);
-  unsigned Idx = WhichResult * NumElts / 2;
-  for (unsigned i = 0; i != NumElts; i += 2) {
-    if ((M[i] >= 0 && (unsigned) M[i] != Idx) ||
-        (M[i+1] >= 0 && (unsigned) M[i+1] != Idx + NumElts))
-      return false;
-    Idx += 1;
+  if (M.size() != NumElts && M.size() != NumElts*2)
+    return false;
+
+  for (unsigned i = 0; i < M.size(); i += NumElts) {
+    WhichResult = M[i] == 0 ? 0 : 1;
+    unsigned Idx = WhichResult * NumElts / 2;
+    for (unsigned j = 0; j < NumElts; j += 2) {
+      if ((M[i+j] >= 0 && (unsigned) M[i+j] != Idx) ||
+          (M[i+j+1] >= 0 && (unsigned) M[i+j+1] != Idx + NumElts))
+        return false;
+      Idx += 1;
+    }
   }
 
+  if (M.size() == NumElts*2)
+    WhichResult = 0;
+
   // VZIP.32 for 64-bit vectors is a pseudo-instruction alias for VTRN.32.
   if (VT.is64BitVector() && EltSz == 32)
     return false;
@@ -5069,15 +5237,23 @@ static bool isVZIP_v_undef_Mask(ArrayRef<int> M, EVT VT, unsigned &WhichResult){
     return false;
 
   unsigned NumElts = VT.getVectorNumElements();
-  WhichResult = (M[0] == 0 ? 0 : 1);
-  unsigned Idx = WhichResult * NumElts / 2;
-  for (unsigned i = 0; i != NumElts; i += 2) {
-    if ((M[i] >= 0 && (unsigned) M[i] != Idx) ||
-        (M[i+1] >= 0 && (unsigned) M[i+1] != Idx))
-      return false;
-    Idx += 1;
+  if (M.size() != NumElts && M.size() != NumElts*2)
+    return false;
+
+  for (unsigned i = 0; i < M.size(); i += NumElts) {
+    WhichResult = M[i] == 0 ? 0 : 1;
+    unsigned Idx = WhichResult * NumElts / 2;
+    for (unsigned j = 0; j < NumElts; j += 2) {
+      if ((M[i+j] >= 0 && (unsigned) M[i+j] != Idx) ||
+          (M[i+j+1] >= 0 && (unsigned) M[i+j+1] != Idx))
+        return false;
+      Idx += 1;
+    }
   }
 
+  if (M.size() == NumElts*2)
+    WhichResult = 0;
+
   // VZIP.32 for 64-bit vectors is a pseudo-instruction alias for VTRN.32.
   if (VT.is64BitVector() && EltSz == 32)
     return false;
@@ -5085,6 +5261,30 @@ static bool isVZIP_v_undef_Mask(ArrayRef<int> M, EVT VT, unsigned &WhichResult){
   return true;
 }
 
+/// Check if \p ShuffleMask is a NEON two-result shuffle (VZIP, VUZP, VTRN),
+/// and return the corresponding ARMISD opcode if it is, or 0 if it isn't.
+static unsigned isNEONTwoResultShuffleMask(ArrayRef<int> ShuffleMask, EVT VT,
+                                           unsigned &WhichResult,
+                                           bool &isV_UNDEF) {
+  isV_UNDEF = false;
+  if (isVTRNMask(ShuffleMask, VT, WhichResult))
+    return ARMISD::VTRN;
+  if (isVUZPMask(ShuffleMask, VT, WhichResult))
+    return ARMISD::VUZP;
+  if (isVZIPMask(ShuffleMask, VT, WhichResult))
+    return ARMISD::VZIP;
+
+  isV_UNDEF = true;
+  if (isVTRN_v_undef_Mask(ShuffleMask, VT, WhichResult))
+    return ARMISD::VTRN;
+  if (isVUZP_v_undef_Mask(ShuffleMask, VT, WhichResult))
+    return ARMISD::VUZP;
+  if (isVZIP_v_undef_Mask(ShuffleMask, VT, WhichResult))
+    return ARMISD::VZIP;
+
+  return 0;
+}
+
 /// \return true if this is a reverse operation on an vector.
 static bool isReverseMask(ArrayRef<int> M, EVT VT) {
   unsigned NumElts = VT.getVectorNumElements();
@@ -5112,10 +5312,10 @@ static SDValue IsSingleInstrConstant(SDValue N, SelectionDAG &DAG,
 
   if (ST->isThumb1Only()) {
     if (Val <= 255 || ~Val <= 255)
-      return DAG.getConstant(Val, MVT::i32);
+      return DAG.getConstant(Val, dl, MVT::i32);
   } else {
     if (ARM_AM::getSOImmVal(Val) != -1 || ARM_AM::getSOImmVal(~Val) != -1)
-      return DAG.getConstant(Val, MVT::i32);
+      return DAG.getConstant(Val, dl, MVT::i32);
   }
   return SDValue();
 }
@@ -5137,7 +5337,7 @@ SDValue ARMTargetLowering::LowerBUILD_VECTOR(SDValue Op, SelectionDAG &DAG,
       EVT VmovVT;
       SDValue Val = isNEONModifiedImm(SplatBits.getZExtValue(),
                                       SplatUndef.getZExtValue(), SplatBitSize,
-                                      DAG, VmovVT, VT.is128BitVector(),
+                                      DAG, dl, VmovVT, VT.is128BitVector(),
                                       VMOVModImm);
       if (Val.getNode()) {
         SDValue Vmov = DAG.getNode(ARMISD::VMOVIMM, dl, VmovVT, Val);
@@ -5148,7 +5348,7 @@ SDValue ARMTargetLowering::LowerBUILD_VECTOR(SDValue Op, SelectionDAG &DAG,
       uint64_t NegatedImm = (~SplatBits).getZExtValue();
       Val = isNEONModifiedImm(NegatedImm,
                                       SplatUndef.getZExtValue(), SplatBitSize,
-                                      DAG, VmovVT, VT.is128BitVector(),
+                                      DAG, dl, VmovVT, VT.is128BitVector(),
                                       VMVNModImm);
       if (Val.getNode()) {
         SDValue Vmov = DAG.getNode(ARMISD::VMVNIMM, dl, VmovVT, Val);
@@ -5159,7 +5359,7 @@ SDValue ARMTargetLowering::LowerBUILD_VECTOR(SDValue Op, SelectionDAG &DAG,
       if ((VT == MVT::v2f32 || VT == MVT::v4f32) && SplatBitSize == 32) {
         int ImmVal = ARM_AM::getFP32Imm(SplatBits);
         if (ImmVal != -1) {
-          SDValue Val = DAG.getTargetConstant(ImmVal, MVT::i32);
+          SDValue Val = DAG.getTargetConstant(ImmVal, dl, MVT::i32);
           return DAG.getNode(ARMISD::VMOVFPIMM, dl, VT, Val);
         }
       }
@@ -5241,8 +5441,8 @@ SDValue ARMTargetLowering::LowerBUILD_VECTOR(SDValue Op, SelectionDAG &DAG,
                              VT.getVectorNumElements();
           N =  DAG.getNode(ARMISD::VDUPLANE, dl, VT,
                  DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, VT, DAG.getUNDEF(VT),
-                        Value, DAG.getConstant(index, MVT::i32)),
-                           DAG.getConstant(index, MVT::i32));
+                        Value, DAG.getConstant(index, dl, MVT::i32)),
+                           DAG.getConstant(index, dl, MVT::i32));
         } else
           N = DAG.getNode(ARMISD::VDUPLANE, dl, VT,
                         Value->getOperand(0), Value->getOperand(1));
@@ -5258,7 +5458,7 @@ SDValue ARMTargetLowering::LowerBUILD_VECTOR(SDValue Op, SelectionDAG &DAG,
           SmallVector<SDValue, 3> Ops;
           Ops.push_back(N);
           Ops.push_back(Op.getOperand(I));
-          Ops.push_back(DAG.getConstant(I, MVT::i32));
+          Ops.push_back(DAG.getConstant(I, dl, MVT::i32));
           N = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, VT, Ops);
         }
       }
@@ -5322,7 +5522,7 @@ SDValue ARMTargetLowering::LowerBUILD_VECTOR(SDValue Op, SelectionDAG &DAG,
       SDValue V = Op.getOperand(i);
       if (V.getOpcode() == ISD::UNDEF)
         continue;
-      SDValue LaneIdx = DAG.getConstant(i, MVT::i32);
+      SDValue LaneIdx = DAG.getConstant(i, dl, MVT::i32);
       Vec = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, VT, Vec, V, LaneIdx);
     }
     return Vec;
@@ -5425,24 +5625,25 @@ SDValue ARMTargetLowering::ReconstructShuffle(SDValue Op,
       VEXTOffsets[i] = NumElts;
       ShuffleSrcs[i] = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, VT,
                                    SourceVecs[i],
-                                   DAG.getIntPtrConstant(NumElts));
+                                   DAG.getIntPtrConstant(NumElts, dl));
     } else if (MaxElts[i] < NumElts) {
       // The extraction can just take the first half
       VEXTOffsets[i] = 0;
       ShuffleSrcs[i] = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, VT,
                                    SourceVecs[i],
-                                   DAG.getIntPtrConstant(0));
+                                   DAG.getIntPtrConstant(0, dl));
     } else {
       // An actual VEXT is needed
       VEXTOffsets[i] = MinElts[i];
       SDValue VEXTSrc1 = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, VT,
                                      SourceVecs[i],
-                                     DAG.getIntPtrConstant(0));
+                                     DAG.getIntPtrConstant(0, dl));
       SDValue VEXTSrc2 = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, VT,
                                      SourceVecs[i],
-                                     DAG.getIntPtrConstant(NumElts));
+                                     DAG.getIntPtrConstant(NumElts, dl));
       ShuffleSrcs[i] = DAG.getNode(ARMISD::VEXT, dl, VT, VEXTSrc1, VEXTSrc2,
-                                   DAG.getConstant(VEXTOffsets[i], MVT::i32));
+                                   DAG.getConstant(VEXTOffsets[i], dl,
+                                                   MVT::i32));
     }
   }
 
@@ -5500,7 +5701,7 @@ ARMTargetLowering::isShuffleMaskLegal(const SmallVectorImpl<int> &M,
       return true;
   }
 
-  bool ReverseVEXT;
+  bool ReverseVEXT, isV_UNDEF;
   unsigned Imm, WhichResult;
 
   unsigned EltSize = VT.getVectorElementType().getSizeInBits();
@@ -5511,12 +5712,7 @@ ARMTargetLowering::isShuffleMaskLegal(const SmallVectorImpl<int> &M,
           isVREVMask(M, VT, 16) ||
           isVEXTMask(M, VT, ReverseVEXT, Imm) ||
           isVTBLMask(M, VT) ||
-          isVTRNMask(M, VT, WhichResult) ||
-          isVUZPMask(M, VT, WhichResult) ||
-          isVZIPMask(M, VT, WhichResult) ||
-          isVTRN_v_undef_Mask(M, VT, WhichResult) ||
-          isVUZP_v_undef_Mask(M, VT, WhichResult) ||
-          isVZIP_v_undef_Mask(M, VT, WhichResult) ||
+          isNEONTwoResultShuffleMask(M, VT, WhichResult, isV_UNDEF) ||
           ((VT == MVT::v8i16 || VT == MVT::v16i8) && isReverseMask(M, VT)));
 }
 
@@ -5576,13 +5772,13 @@ static SDValue GeneratePerfectShuffle(unsigned PFEntry, SDValue LHS,
   case OP_VDUP2:
   case OP_VDUP3:
     return DAG.getNode(ARMISD::VDUPLANE, dl, VT,
-                       OpLHS, DAG.getConstant(OpNum-OP_VDUP0, MVT::i32));
+                       OpLHS, DAG.getConstant(OpNum-OP_VDUP0, dl, MVT::i32));
   case OP_VEXT1:
   case OP_VEXT2:
   case OP_VEXT3:
     return DAG.getNode(ARMISD::VEXT, dl, VT,
                        OpLHS, OpRHS,
-                       DAG.getConstant(OpNum-OP_VEXT1+1, MVT::i32));
+                       DAG.getConstant(OpNum - OP_VEXT1 + 1, dl, MVT::i32));
   case OP_VUZPL:
   case OP_VUZPR:
     return DAG.getNode(ARMISD::VUZP, dl, DAG.getVTList(VT, VT),
@@ -5609,7 +5805,7 @@ static SDValue LowerVECTOR_SHUFFLEv8i8(SDValue Op,
   SmallVector<SDValue, 8> VTBLMask;
   for (ArrayRef<int>::iterator
          I = ShuffleMask.begin(), E = ShuffleMask.end(); I != E; ++I)
-    VTBLMask.push_back(DAG.getConstant(*I, MVT::i32));
+    VTBLMask.push_back(DAG.getConstant(*I, DL, MVT::i32));
 
   if (V2.getNode()->getOpcode() == ISD::UNDEF)
     return DAG.getNode(ARMISD::VTBL1, DL, MVT::v8i8, V1,
@@ -5633,7 +5829,7 @@ static SDValue LowerReverse_VECTOR_SHUFFLEv16i8_v8i16(SDValue Op,
   // into the bottom double word. The v8i16 case is similar.
   unsigned ExtractNum = (VT == MVT::v16i8) ? 8 : 4;
   return DAG.getNode(ARMISD::VEXT, DL, VT, OpLHS, OpLHS,
-                     DAG.getConstant(ExtractNum, MVT::i32));
+                     DAG.getConstant(ExtractNum, DL, MVT::i32));
 }
 
 static SDValue LowerVECTOR_SHUFFLE(SDValue Op, SelectionDAG &DAG) {
@@ -5677,7 +5873,7 @@ static SDValue LowerVECTOR_SHUFFLE(SDValue Op, SelectionDAG &DAG) {
           return DAG.getNode(ARMISD::VDUP, dl, VT, V1.getOperand(0));
       }
       return DAG.getNode(ARMISD::VDUPLANE, dl, VT, V1,
-                         DAG.getConstant(Lane, MVT::i32));
+                         DAG.getConstant(Lane, dl, MVT::i32));
     }
 
     bool ReverseVEXT;
@@ -5686,7 +5882,7 @@ static SDValue LowerVECTOR_SHUFFLE(SDValue Op, SelectionDAG &DAG) {
       if (ReverseVEXT)
         std::swap(V1, V2);
       return DAG.getNode(ARMISD::VEXT, dl, VT, V1, V2,
-                         DAG.getConstant(Imm, MVT::i32));
+                         DAG.getConstant(Imm, dl, MVT::i32));
     }
 
     if (isVREVMask(ShuffleMask, VT, 64))
@@ -5699,7 +5895,7 @@ static SDValue LowerVECTOR_SHUFFLE(SDValue Op, SelectionDAG &DAG) {
     if (V2->getOpcode() == ISD::UNDEF &&
         isSingletonVEXTMask(ShuffleMask, VT, Imm)) {
       return DAG.getNode(ARMISD::VEXT, dl, VT, V1, V1,
-                         DAG.getConstant(Imm, MVT::i32));
+                         DAG.getConstant(Imm, dl, MVT::i32));
     }
 
     // Check for Neon shuffles that modify both input vectors in place.
@@ -5708,25 +5904,53 @@ static SDValue LowerVECTOR_SHUFFLE(SDValue Op, SelectionDAG &DAG) {
     // these operations, DAG memoization will ensure that a single node is
     // used for both shuffles.
     unsigned WhichResult;
-    if (isVTRNMask(ShuffleMask, VT, WhichResult))
-      return DAG.getNode(ARMISD::VTRN, dl, DAG.getVTList(VT, VT),
-                         V1, V2).getValue(WhichResult);
-    if (isVUZPMask(ShuffleMask, VT, WhichResult))
-      return DAG.getNode(ARMISD::VUZP, dl, DAG.getVTList(VT, VT),
-                         V1, V2).getValue(WhichResult);
-    if (isVZIPMask(ShuffleMask, VT, WhichResult))
-      return DAG.getNode(ARMISD::VZIP, dl, DAG.getVTList(VT, VT),
-                         V1, V2).getValue(WhichResult);
-
-    if (isVTRN_v_undef_Mask(ShuffleMask, VT, WhichResult))
-      return DAG.getNode(ARMISD::VTRN, dl, DAG.getVTList(VT, VT),
-                         V1, V1).getValue(WhichResult);
-    if (isVUZP_v_undef_Mask(ShuffleMask, VT, WhichResult))
-      return DAG.getNode(ARMISD::VUZP, dl, DAG.getVTList(VT, VT),
-                         V1, V1).getValue(WhichResult);
-    if (isVZIP_v_undef_Mask(ShuffleMask, VT, WhichResult))
-      return DAG.getNode(ARMISD::VZIP, dl, DAG.getVTList(VT, VT),
-                         V1, V1).getValue(WhichResult);
+    bool isV_UNDEF;
+    if (unsigned ShuffleOpc = isNEONTwoResultShuffleMask(
+            ShuffleMask, VT, WhichResult, isV_UNDEF)) {
+      if (isV_UNDEF)
+        V2 = V1;
+      return DAG.getNode(ShuffleOpc, dl, DAG.getVTList(VT, VT), V1, V2)
+          .getValue(WhichResult);
+    }
+
+    // Also check for these shuffles through CONCAT_VECTORS: we canonicalize
+    // shuffles that produce a result larger than their operands with:
+    //   shuffle(concat(v1, undef), concat(v2, undef))
+    // ->
+    //   shuffle(concat(v1, v2), undef)
+    // because we can access quad vectors (see PerformVECTOR_SHUFFLECombine).
+    //
+    // This is useful in the general case, but there are special cases where
+    // native shuffles produce larger results: the two-result ops.
+    //
+    // Look through the concat when lowering them:
+    //   shuffle(concat(v1, v2), undef)
+    // ->
+    //   concat(VZIP(v1, v2):0, :1)
+    //
+    if (V1->getOpcode() == ISD::CONCAT_VECTORS &&
+        V2->getOpcode() == ISD::UNDEF) {
+      SDValue SubV1 = V1->getOperand(0);
+      SDValue SubV2 = V1->getOperand(1);
+      EVT SubVT = SubV1.getValueType();
+
+      // We expect these to have been canonicalized to -1.
+      assert(std::all_of(ShuffleMask.begin(), ShuffleMask.end(), [&](int i) {
+        return i < (int)VT.getVectorNumElements();
+      }) && "Unexpected shuffle index into UNDEF operand!");
+
+      if (unsigned ShuffleOpc = isNEONTwoResultShuffleMask(
+              ShuffleMask, SubVT, WhichResult, isV_UNDEF)) {
+        if (isV_UNDEF)
+          SubV2 = SubV1;
+        assert((WhichResult == 0) &&
+               "In-place shuffle of concat can only have one result!");
+        SDValue Res = DAG.getNode(ShuffleOpc, dl, DAG.getVTList(SubVT, SubVT),
+                                  SubV1, SubV2);
+        return DAG.getNode(ISD::CONCAT_VECTORS, dl, VT, Res.getValue(0),
+                           Res.getValue(1));
+      }
+    }
   }
 
   // If the shuffle is not directly supported and it has 4 elements, use
@@ -5767,7 +5991,7 @@ static SDValue LowerVECTOR_SHUFFLE(SDValue Op, SelectionDAG &DAG) {
         Ops.push_back(DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, EltVT,
                                   ShuffleMask[i] < (int)NumElts ? V1 : V2,
                                   DAG.getConstant(ShuffleMask[i] & (NumElts-1),
-                                                  MVT::i32)));
+                                                  dl, MVT::i32)));
     }
     SDValue Val = DAG.getNode(ARMISD::BUILD_VECTOR, dl, VecVT, Ops);
     return DAG.getNode(ISD::BITCAST, dl, VT, Val);
@@ -5822,11 +6046,11 @@ static SDValue LowerCONCAT_VECTORS(SDValue Op, SelectionDAG &DAG) {
   if (Op0.getOpcode() != ISD::UNDEF)
     Val = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, MVT::v2f64, Val,
                       DAG.getNode(ISD::BITCAST, dl, MVT::f64, Op0),
-                      DAG.getIntPtrConstant(0));
+                      DAG.getIntPtrConstant(0, dl));
   if (Op1.getOpcode() != ISD::UNDEF)
     Val = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, MVT::v2f64, Val,
                       DAG.getNode(ISD::BITCAST, dl, MVT::f64, Op1),
-                      DAG.getIntPtrConstant(1));
+                      DAG.getIntPtrConstant(1, dl));
   return DAG.getNode(ISD::BITCAST, dl, Op.getValueType(), Val);
 }
 
@@ -5842,7 +6066,7 @@ static bool isExtendedBUILD_VECTOR(SDNode *N, SelectionDAG &DAG,
     if (BVN->getValueType(0) != MVT::v4i32 ||
         BVN->getOpcode() != ISD::BUILD_VECTOR)
       return false;
-    unsigned LoElt = DAG.getTargetLoweringInfo().isBigEndian() ? 1 : 0;
+    unsigned LoElt = DAG.getDataLayout().isBigEndian() ? 1 : 0;
     unsigned HiElt = 1 - LoElt;
     ConstantSDNode *Lo0 = dyn_cast<ConstantSDNode>(BVN->getOperand(LoElt));
     ConstantSDNode *Hi0 = dyn_cast<ConstantSDNode>(BVN->getOperand(HiElt));
@@ -5987,7 +6211,7 @@ static SDValue SkipExtensionForVMULL(SDNode *N, SelectionDAG &DAG) {
     SDNode *BVN = N->getOperand(0).getNode();
     assert(BVN->getOpcode() == ISD::BUILD_VECTOR &&
            BVN->getValueType(0) == MVT::v4i32 && "expected v4i32 BUILD_VECTOR");
-    unsigned LowElt = DAG.getTargetLoweringInfo().isBigEndian() ? 1 : 0;
+    unsigned LowElt = DAG.getDataLayout().isBigEndian() ? 1 : 0;
     return DAG.getNode(ISD::BUILD_VECTOR, SDLoc(N), MVT::v2i32,
                        BVN->getOperand(LowElt), BVN->getOperand(LowElt+2));
   }
@@ -5998,14 +6222,15 @@ static SDValue SkipExtensionForVMULL(SDNode *N, SelectionDAG &DAG) {
   unsigned NumElts = VT.getVectorNumElements();
   MVT TruncVT = MVT::getIntegerVT(EltSize);
   SmallVector<SDValue, 8> Ops;
+  SDLoc dl(N);
   for (unsigned i = 0; i != NumElts; ++i) {
     ConstantSDNode *C = cast<ConstantSDNode>(N->getOperand(i));
     const APInt &CInt = C->getAPIntValue();
     // Element types smaller than 32 bits are not legal, so use i32 elements.
     // The values are implicitly truncated so sext vs. zext doesn't matter.
-    Ops.push_back(DAG.getConstant(CInt.zextOrTrunc(32), MVT::i32));
+    Ops.push_back(DAG.getConstant(CInt.zextOrTrunc(32), dl, MVT::i32));
   }
-  return DAG.getNode(ISD::BUILD_VECTOR, SDLoc(N),
+  return DAG.getNode(ISD::BUILD_VECTOR, dl,
                      MVT::getVectorVT(TruncVT, NumElts), Ops);
 }
 
@@ -6106,38 +6331,6 @@ static SDValue LowerMUL(SDValue Op, SelectionDAG &DAG) {
                                DAG.getNode(ISD::BITCAST, DL, Op1VT, N01), Op1));
 }
 
-// @LOCALMOD-START
-// An EH_RETURN is the result of lowering llvm.eh.return.i32 which in turn is
-// generated from __builtin_eh_return (offset, handler)
-// The effect of this is to adjust the stack pointer by "offset"
-// and then branch to "handler".
-SDValue ARMTargetLowering::LowerEH_RETURN(SDValue Op, SelectionDAG &DAG)
-    const {
-  SDValue Chain     = Op.getOperand(0);
-  SDValue Offset    = Op.getOperand(1);
-  SDValue Handler   = Op.getOperand(2);
-  SDLoc dl(Op);
-
-  // Store stack offset in R2, jump target in R3, dummy return value in R0
-  // The dummy return value is needed to make the use-def chains happy,
-  // because the EH_RETURN instruction uses the isReturn attribute, which
-  // means preceding code needs to define the return register (R0 on ARM).
-  // http://code.google.com/p/nativeclient/issues/detail?id=2643
-  unsigned OffsetReg = ARM::R2;
-  unsigned AddrReg = ARM::R3;
-  unsigned ReturnReg = ARM::R0;
-  Chain = DAG.getCopyToReg(Chain, dl, OffsetReg, Offset);
-  Chain = DAG.getCopyToReg(Chain, dl, AddrReg, Handler);
-  Chain = DAG.getCopyToReg(Chain, dl, ReturnReg, DAG.getIntPtrConstant(0));
-  return DAG.getNode(ARMISD::EH_RETURN, dl,
-                     MVT::Other,
-                     Chain,
-                     DAG.getRegister(OffsetReg, MVT::i32),
-                     DAG.getRegister(AddrReg, getPointerTy()));
-}
-// @LOCALMOD-END
-
-
 static SDValue
 LowerSDIV_v4i8(SDValue X, SDValue Y, SDLoc dl, SelectionDAG &DAG) {
   // Convert to float
@@ -6150,14 +6343,15 @@ LowerSDIV_v4i8(SDValue X, SDValue Y, SDLoc dl, SelectionDAG &DAG) {
   // Get reciprocal estimate.
   // float4 recip = vrecpeq_f32(yf);
   Y = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, dl, MVT::v4f32,
-                   DAG.getConstant(Intrinsic::arm_neon_vrecpe, MVT::i32), Y);
+                   DAG.getConstant(Intrinsic::arm_neon_vrecpe, dl, MVT::i32),
+                   Y);
   // Because char has a smaller range than uchar, we can actually get away
   // without any newton steps.  This requires that we use a weird bias
   // of 0xb000, however (again, this has been exhaustively tested).
   // float4 result = as_float4(as_int4(xf*recip) + 0xb000);
   X = DAG.getNode(ISD::FMUL, dl, MVT::v4f32, X, Y);
   X = DAG.getNode(ISD::BITCAST, dl, MVT::v4i32, X);
-  Y = DAG.getConstant(0xb000, MVT::i32);
+  Y = DAG.getConstant(0xb000, dl, MVT::i32);
   Y = DAG.getNode(ISD::BUILD_VECTOR, dl, MVT::v4i32, Y, Y, Y, Y);
   X = DAG.getNode(ISD::ADD, dl, MVT::v4i32, X, Y);
   X = DAG.getNode(ISD::BITCAST, dl, MVT::v4f32, X);
@@ -6182,9 +6376,10 @@ LowerSDIV_v4i16(SDValue N0, SDValue N1, SDLoc dl, SelectionDAG &DAG) {
   // float4 recip = vrecpeq_f32(yf);
   // recip *= vrecpsq_f32(yf, recip);
   N2 = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, dl, MVT::v4f32,
-                   DAG.getConstant(Intrinsic::arm_neon_vrecpe, MVT::i32), N1);
+                   DAG.getConstant(Intrinsic::arm_neon_vrecpe, dl, MVT::i32),
+                   N1);
   N1 = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, dl, MVT::v4f32,
-                   DAG.getConstant(Intrinsic::arm_neon_vrecps, MVT::i32),
+                   DAG.getConstant(Intrinsic::arm_neon_vrecps, dl, MVT::i32),
                    N1, N2);
   N2 = DAG.getNode(ISD::FMUL, dl, MVT::v4f32, N1, N2);
   // Because short has a smaller range than ushort, we can actually get away
@@ -6193,7 +6388,7 @@ LowerSDIV_v4i16(SDValue N0, SDValue N1, SDLoc dl, SelectionDAG &DAG) {
   // float4 result = as_float4(as_int4(xf*recip) + 0x89);
   N0 = DAG.getNode(ISD::FMUL, dl, MVT::v4f32, N0, N2);
   N0 = DAG.getNode(ISD::BITCAST, dl, MVT::v4i32, N0);
-  N1 = DAG.getConstant(0x89, MVT::i32);
+  N1 = DAG.getConstant(0x89, dl, MVT::i32);
   N1 = DAG.getNode(ISD::BUILD_VECTOR, dl, MVT::v4i32, N1, N1, N1, N1);
   N0 = DAG.getNode(ISD::ADD, dl, MVT::v4i32, N0, N1);
   N0 = DAG.getNode(ISD::BITCAST, dl, MVT::v4f32, N0);
@@ -6219,13 +6414,13 @@ static SDValue LowerSDIV(SDValue Op, SelectionDAG &DAG) {
     N1 = DAG.getNode(ISD::SIGN_EXTEND, dl, MVT::v8i16, N1);
 
     N2 = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, MVT::v4i16, N0,
-                     DAG.getIntPtrConstant(4));
+                     DAG.getIntPtrConstant(4, dl));
     N3 = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, MVT::v4i16, N1,
-                     DAG.getIntPtrConstant(4));
+                     DAG.getIntPtrConstant(4, dl));
     N0 = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, MVT::v4i16, N0,
-                     DAG.getIntPtrConstant(0));
+                     DAG.getIntPtrConstant(0, dl));
     N1 = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, MVT::v4i16, N1,
-                     DAG.getIntPtrConstant(0));
+                     DAG.getIntPtrConstant(0, dl));
 
     N0 = LowerSDIV_v4i8(N0, N1, dl, DAG); // v4i16
     N2 = LowerSDIV_v4i8(N2, N3, dl, DAG); // v4i16
@@ -6254,13 +6449,13 @@ static SDValue LowerUDIV(SDValue Op, SelectionDAG &DAG) {
     N1 = DAG.getNode(ISD::ZERO_EXTEND, dl, MVT::v8i16, N1);
 
     N2 = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, MVT::v4i16, N0,
-                     DAG.getIntPtrConstant(4));
+                     DAG.getIntPtrConstant(4, dl));
     N3 = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, MVT::v4i16, N1,
-                     DAG.getIntPtrConstant(4));
+                     DAG.getIntPtrConstant(4, dl));
     N0 = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, MVT::v4i16, N0,
-                     DAG.getIntPtrConstant(0));
+                     DAG.getIntPtrConstant(0, dl));
     N1 = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, MVT::v4i16, N1,
-                     DAG.getIntPtrConstant(0));
+                     DAG.getIntPtrConstant(0, dl));
 
     N0 = LowerSDIV_v4i16(N0, N1, dl, DAG); // v4i16
     N2 = LowerSDIV_v4i16(N2, N3, dl, DAG); // v4i16
@@ -6269,7 +6464,8 @@ static SDValue LowerUDIV(SDValue Op, SelectionDAG &DAG) {
     N0 = LowerCONCAT_VECTORS(N0, DAG);
 
     N0 = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, dl, MVT::v8i8,
-                     DAG.getConstant(Intrinsic::arm_neon_vqmovnsu, MVT::i32),
+                     DAG.getConstant(Intrinsic::arm_neon_vqmovnsu, dl,
+                                     MVT::i32),
                      N0);
     return N0;
   }
@@ -6287,13 +6483,14 @@ static SDValue LowerUDIV(SDValue Op, SelectionDAG &DAG) {
   // recip *= vrecpsq_f32(yf, recip);
   // recip *= vrecpsq_f32(yf, recip);
   N2 = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, dl, MVT::v4f32,
-                   DAG.getConstant(Intrinsic::arm_neon_vrecpe, MVT::i32), BN1);
+                   DAG.getConstant(Intrinsic::arm_neon_vrecpe, dl, MVT::i32),
+                   BN1);
   N1 = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, dl, MVT::v4f32,
-                   DAG.getConstant(Intrinsic::arm_neon_vrecps, MVT::i32),
+                   DAG.getConstant(Intrinsic::arm_neon_vrecps, dl, MVT::i32),
                    BN1, N2);
   N2 = DAG.getNode(ISD::FMUL, dl, MVT::v4f32, N1, N2);
   N1 = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, dl, MVT::v4f32,
-                   DAG.getConstant(Intrinsic::arm_neon_vrecps, MVT::i32),
+                   DAG.getConstant(Intrinsic::arm_neon_vrecps, dl, MVT::i32),
                    BN1, N2);
   N2 = DAG.getNode(ISD::FMUL, dl, MVT::v4f32, N1, N2);
   // Simply multiplying by the reciprocal estimate can leave us a few ulps
@@ -6302,7 +6499,7 @@ static SDValue LowerUDIV(SDValue Op, SelectionDAG &DAG) {
   // float4 result = as_float4(as_int4(xf*recip) + 2);
   N0 = DAG.getNode(ISD::FMUL, dl, MVT::v4f32, N0, N2);
   N0 = DAG.getNode(ISD::BITCAST, dl, MVT::v4i32, N0);
-  N1 = DAG.getConstant(2, MVT::i32);
+  N1 = DAG.getConstant(2, dl, MVT::i32);
   N1 = DAG.getNode(ISD::BUILD_VECTOR, dl, MVT::v4i32, N1, N1, N1, N1);
   N0 = DAG.getNode(ISD::ADD, dl, MVT::v4i32, N0, N1);
   N0 = DAG.getNode(ISD::BITCAST, dl, MVT::v4f32, N0);
@@ -6343,18 +6540,19 @@ SDValue ARMTargetLowering::LowerFSINCOS(SDValue Op, SelectionDAG &DAG) const {
   SDValue Arg = Op.getOperand(0);
   EVT ArgVT = Arg.getValueType();
   Type *ArgTy = ArgVT.getTypeForEVT(*DAG.getContext());
+  auto PtrVT = getPointerTy(DAG.getDataLayout());
 
   MachineFrameInfo *FrameInfo = DAG.getMachineFunction().getFrameInfo();
-  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
 
   // Pair of floats / doubles used to pass the result.
   StructType *RetTy = StructType::get(ArgTy, ArgTy, nullptr);
 
   // Create stack object for sret.
-  const uint64_t ByteSize = TLI.getDataLayout()->getTypeAllocSize(RetTy);
-  const unsigned StackAlign = TLI.getDataLayout()->getPrefTypeAlignment(RetTy);
+  auto &DL = DAG.getDataLayout();
+  const uint64_t ByteSize = DL.getTypeAllocSize(RetTy);
+  const unsigned StackAlign = DL.getPrefTypeAlignment(RetTy);
   int FrameIdx = FrameInfo->CreateStackObject(ByteSize, StackAlign, false);
-  SDValue SRet = DAG.getFrameIndex(FrameIdx, TLI.getPointerTy());
+  SDValue SRet = DAG.getFrameIndex(FrameIdx, getPointerTy(DL));
 
   ArgListTy Args;
   ArgListEntry Entry;
@@ -6374,7 +6572,7 @@ SDValue ARMTargetLowering::LowerFSINCOS(SDValue Op, SelectionDAG &DAG) const {
 
   const char *LibcallName  = (ArgVT == MVT::f64)
   ? "__sincos_stret" : "__sincosf_stret";
-  SDValue Callee = DAG.getExternalSymbol(LibcallName, getPointerTy());
+  SDValue Callee = DAG.getExternalSymbol(LibcallName, getPointerTy(DL));
 
   TargetLowering::CallLoweringInfo CLI(DAG);
   CLI.setDebugLoc(dl).setChain(DAG.getEntryNode())
@@ -6388,8 +6586,8 @@ SDValue ARMTargetLowering::LowerFSINCOS(SDValue Op, SelectionDAG &DAG) const {
                                 MachinePointerInfo(), false, false, false, 0);
 
   // Address of cos field.
-  SDValue Add = DAG.getNode(ISD::ADD, dl, getPointerTy(), SRet,
-                            DAG.getIntPtrConstant(ArgVT.getStoreSize()));
+  SDValue Add = DAG.getNode(ISD::ADD, dl, PtrVT, SRet,
+                            DAG.getIntPtrConstant(ArgVT.getStoreSize(), dl));
   SDValue LoadCos = DAG.getLoad(ArgVT, dl, LoadSin.getValue(1), Add,
                                 MachinePointerInfo(), false, false, false, 0);
 
@@ -6419,12 +6617,12 @@ static void ReplaceREADCYCLECOUNTER(SDNode *N,
     // Under Power Management extensions, the cycle-count is:
     //    mrc p15, #0, <Rt>, c9, c13, #0
     SDValue Ops[] = { N->getOperand(0), // Chain
-                      DAG.getConstant(Intrinsic::arm_mrc, MVT::i32),
-                      DAG.getConstant(15, MVT::i32),
-                      DAG.getConstant(0, MVT::i32),
-                      DAG.getConstant(9, MVT::i32),
-                      DAG.getConstant(13, MVT::i32),
-                      DAG.getConstant(0, MVT::i32)
+                      DAG.getConstant(Intrinsic::arm_mrc, DL, MVT::i32),
+                      DAG.getConstant(15, DL, MVT::i32),
+                      DAG.getConstant(0, DL, MVT::i32),
+                      DAG.getConstant(9, DL, MVT::i32),
+                      DAG.getConstant(13, DL, MVT::i32),
+                      DAG.getConstant(0, DL, MVT::i32)
     };
 
     Cycles32 = DAG.getNode(ISD::INTRINSIC_W_CHAIN, DL,
@@ -6434,13 +6632,13 @@ static void ReplaceREADCYCLECOUNTER(SDNode *N,
     // Intrinsic is defined to return 0 on unsupported platforms. Technically
     // there are older ARM CPUs that have implementation-specific ways of
     // obtaining this information (FIXME!).
-    Cycles32 = DAG.getConstant(0, MVT::i32);
+    Cycles32 = DAG.getConstant(0, DL, MVT::i32);
     OutChain = DAG.getEntryNode();
   }
 
 
   SDValue Cycles64 = DAG.getNode(ISD::BUILD_PAIR, DL, MVT::i64,
-                                 Cycles32, DAG.getConstant(0, MVT::i32));
+                                 Cycles32, DAG.getConstant(0, DL, MVT::i32));
   Results.push_back(Cycles64);
   Results.push_back(OutChain);
 }
@@ -6448,9 +6646,9 @@ static void ReplaceREADCYCLECOUNTER(SDNode *N,
 SDValue ARMTargetLowering::LowerOperation(SDValue Op, SelectionDAG &DAG) const {
   switch (Op.getOpcode()) {
   default: llvm_unreachable("Don't know how to custom lower this!");
+  case ISD::WRITE_REGISTER: return LowerWRITE_REGISTER(Op, DAG);
   case ISD::ConstantPool:  return LowerConstantPool(Op, DAG);
   case ISD::BlockAddress:  return LowerBlockAddress(Op, DAG);
-  case ISD::JumpTable:    return LowerJumpTable(Op, DAG); // @LOCALMOD
   case ISD::GlobalAddress:
     switch (Subtarget->getTargetTriple().getObjectFormat()) {
     default: llvm_unreachable("unknown object format");
@@ -6476,20 +6674,10 @@ SDValue ARMTargetLowering::LowerOperation(SDValue Op, SelectionDAG &DAG) const {
   case ISD::FCOPYSIGN:     return LowerFCOPYSIGN(Op, DAG);
   case ISD::RETURNADDR:    return LowerRETURNADDR(Op, DAG);
   case ISD::FRAMEADDR:     return LowerFRAMEADDR(Op, DAG);
-  // @LOCALMOD-START
-  // The exact semantics of this ISD are not completely clear.
-  // LLVM seems to always point the fp after the push ra and the old fp, i.e.
-  // two register slots after the beginning of the stack frame.
-  // It is not clear what happens when there is no frame pointer but
-  // but llvm unlike gcc seems to always force one when this node is
-  // encountered.
-  case ISD::FRAME_TO_ARGS_OFFSET: return DAG.getIntPtrConstant(2*4);
-  case ISD::EH_RETURN:            return LowerEH_RETURN(Op, DAG);
-  // @LOCALMOD-END
-   
   case ISD::GLOBAL_OFFSET_TABLE: return LowerGLOBAL_OFFSET_TABLE(Op, DAG);
   case ISD::EH_SJLJ_SETJMP: return LowerEH_SJLJ_SETJMP(Op, DAG);
   case ISD::EH_SJLJ_LONGJMP: return LowerEH_SJLJ_LONGJMP(Op, DAG);
+  case ISD::EH_SJLJ_SETUP_DISPATCH: return LowerEH_SJLJ_SETUP_DISPATCH(Op, DAG);
   case ISD::INTRINSIC_WO_CHAIN: return LowerINTRINSIC_WO_CHAIN(Op, DAG,
                                                                Subtarget);
   case ISD::BITCAST:       return ExpandBITCAST(Op.getNode(), DAG);
@@ -6499,7 +6687,8 @@ SDValue ARMTargetLowering::LowerOperation(SDValue Op, SelectionDAG &DAG) const {
   case ISD::SHL_PARTS:     return LowerShiftLeftParts(Op, DAG);
   case ISD::SRL_PARTS:
   case ISD::SRA_PARTS:     return LowerShiftRightParts(Op, DAG);
-  case ISD::CTTZ:          return LowerCTTZ(Op.getNode(), DAG, Subtarget);
+  case ISD::CTTZ:
+  case ISD::CTTZ_ZERO_UNDEF: return LowerCTTZ(Op.getNode(), DAG, Subtarget);
   case ISD::CTPOP:         return LowerCTPOP(Op.getNode(), DAG, Subtarget);
   case ISD::SETCC:         return LowerVSETCC(Op, DAG);
   case ISD::ConstantFP:    return LowerConstantFP(Op, DAG, Subtarget);
@@ -6544,6 +6733,9 @@ void ARMTargetLowering::ReplaceNodeResults(SDNode *N,
   switch (N->getOpcode()) {
   default:
     llvm_unreachable("Don't know how to custom expand this!");
+  case ISD::READ_REGISTER:
+    ExpandREAD_REGISTER(N, Results, DAG);
+    break;
   case ISD::BITCAST:
     Res = ExpandBITCAST(N, DAG);
     break;
@@ -6680,13 +6872,12 @@ SetupEntryBlockForSjLj(MachineInstr *MI, MachineBasicBlock *MBB,
   }
 }
 
-MachineBasicBlock *ARMTargetLowering::
-EmitSjLjDispatchBlock(MachineInstr *MI, MachineBasicBlock *MBB) const {
+void ARMTargetLowering::EmitSjLjDispatchBlock(MachineInstr *MI,
+                                              MachineBasicBlock *MBB) const {
   const TargetInstrInfo *TII = Subtarget->getInstrInfo();
   DebugLoc dl = MI->getDebugLoc();
   MachineFunction *MF = MBB->getParent();
   MachineRegisterInfo *MRI = &MF->getRegInfo();
-  ARMFunctionInfo *AFI = MF->getInfo<ARMFunctionInfo>();
   MachineFrameInfo *MFI = MF->getFrameInfo();
   int FI = MFI->getFunctionContextIndex();
 
@@ -6742,7 +6933,6 @@ EmitSjLjDispatchBlock(MachineInstr *MI, MachineBasicBlock *MBB) const {
   MachineJumpTableInfo *JTI =
     MF->getOrCreateJumpTableInfo(MachineJumpTableInfo::EK_Inline);
   unsigned MJTI = JTI->createJumpTableIndex(LPadList);
-  unsigned UId = AFI->createJumpTableUId();
   Reloc::Model RelocM = getTargetMachine().getRelocationModel();
 
   // Create the MBBs for the dispatch code.
@@ -6825,8 +7015,7 @@ EmitSjLjDispatchBlock(MachineInstr *MI, MachineBasicBlock *MBB) const {
 
     unsigned NewVReg3 = MRI->createVirtualRegister(TRC);
     AddDefaultPred(BuildMI(DispContBB, dl, TII->get(ARM::t2LEApcrelJT),NewVReg3)
-                   .addJumpTableIndex(MJTI)
-                   .addImm(UId));
+                   .addJumpTableIndex(MJTI));
 
     unsigned NewVReg4 = MRI->createVirtualRegister(TRC);
     AddDefaultCC(
@@ -6839,8 +7028,7 @@ EmitSjLjDispatchBlock(MachineInstr *MI, MachineBasicBlock *MBB) const {
     BuildMI(DispContBB, dl, TII->get(ARM::t2BR_JT))
       .addReg(NewVReg4, RegState::Kill)
       .addReg(NewVReg1)
-      .addJumpTableIndex(MJTI)
-      .addImm(UId);
+      .addJumpTableIndex(MJTI);
   } else if (Subtarget->isThumb()) {
     unsigned NewVReg1 = MRI->createVirtualRegister(TRC);
     AddDefaultPred(BuildMI(DispatchBB, dl, TII->get(ARM::tLDRspi), NewVReg1)
@@ -6858,9 +7046,9 @@ EmitSjLjDispatchBlock(MachineInstr *MI, MachineBasicBlock *MBB) const {
       const Constant *C = ConstantInt::get(Int32Ty, NumLPads);
 
       // MachineConstantPool wants an explicit alignment.
-      unsigned Align = getDataLayout()->getPrefTypeAlignment(Int32Ty);
+      unsigned Align = MF->getDataLayout().getPrefTypeAlignment(Int32Ty);
       if (Align == 0)
-        Align = getDataLayout()->getTypeAllocSize(C->getType());
+        Align = MF->getDataLayout().getTypeAllocSize(C->getType());
       unsigned Idx = ConstantPool->getConstantPoolIndex(C, Align);
 
       unsigned VReg1 = MRI->createVirtualRegister(TRC);
@@ -6885,8 +7073,7 @@ EmitSjLjDispatchBlock(MachineInstr *MI, MachineBasicBlock *MBB) const {
 
     unsigned NewVReg3 = MRI->createVirtualRegister(TRC);
     AddDefaultPred(BuildMI(DispContBB, dl, TII->get(ARM::tLEApcrelJT), NewVReg3)
-                   .addJumpTableIndex(MJTI)
-                   .addImm(UId));
+                   .addJumpTableIndex(MJTI));
 
     unsigned NewVReg4 = MRI->createVirtualRegister(TRC);
     AddDefaultPred(BuildMI(DispContBB, dl, TII->get(ARM::tADDrr), NewVReg4)
@@ -6915,8 +7102,7 @@ EmitSjLjDispatchBlock(MachineInstr *MI, MachineBasicBlock *MBB) const {
 
     BuildMI(DispContBB, dl, TII->get(ARM::tBR_JTr))
       .addReg(NewVReg6, RegState::Kill)
-      .addJumpTableIndex(MJTI)
-      .addImm(UId);
+      .addJumpTableIndex(MJTI);
   } else {
     unsigned NewVReg1 = MRI->createVirtualRegister(TRC);
     AddDefaultPred(BuildMI(DispatchBB, dl, TII->get(ARM::LDRi12), NewVReg1)
@@ -6950,9 +7136,9 @@ EmitSjLjDispatchBlock(MachineInstr *MI, MachineBasicBlock *MBB) const {
       const Constant *C = ConstantInt::get(Int32Ty, NumLPads);
 
       // MachineConstantPool wants an explicit alignment.
-      unsigned Align = getDataLayout()->getPrefTypeAlignment(Int32Ty);
+      unsigned Align = MF->getDataLayout().getPrefTypeAlignment(Int32Ty);
       if (Align == 0)
-        Align = getDataLayout()->getTypeAllocSize(C->getType());
+        Align = MF->getDataLayout().getTypeAllocSize(C->getType());
       unsigned Idx = ConstantPool->getConstantPoolIndex(C, Align);
 
       unsigned VReg1 = MRI->createVirtualRegister(TRC);
@@ -6977,8 +7163,7 @@ EmitSjLjDispatchBlock(MachineInstr *MI, MachineBasicBlock *MBB) const {
                      .addImm(ARM_AM::getSORegOpc(ARM_AM::lsl, 2))));
     unsigned NewVReg4 = MRI->createVirtualRegister(TRC);
     AddDefaultPred(BuildMI(DispContBB, dl, TII->get(ARM::LEApcrelJT), NewVReg4)
-                   .addJumpTableIndex(MJTI)
-                   .addImm(UId));
+                   .addJumpTableIndex(MJTI));
 
     MachineMemOperand *JTMMOLd =
       MF->getMachineMemOperand(MachinePointerInfo::getJumpTable(),
@@ -6995,13 +7180,11 @@ EmitSjLjDispatchBlock(MachineInstr *MI, MachineBasicBlock *MBB) const {
       BuildMI(DispContBB, dl, TII->get(ARM::BR_JTadd))
         .addReg(NewVReg5, RegState::Kill)
         .addReg(NewVReg4)
-        .addJumpTableIndex(MJTI)
-        .addImm(UId);
+        .addJumpTableIndex(MJTI);
     } else {
       BuildMI(DispContBB, dl, TII->get(ARM::BR_JTr))
         .addReg(NewVReg5, RegState::Kill)
-        .addJumpTableIndex(MJTI)
-        .addImm(UId);
+        .addJumpTableIndex(MJTI);
     }
   }
 
@@ -7077,8 +7260,6 @@ EmitSjLjDispatchBlock(MachineInstr *MI, MachineBasicBlock *MBB) const {
 
   // The instruction is gone now.
   MI->eraseFromParent();
-
-  return MBB;
 }
 
 static
@@ -7333,9 +7514,9 @@ ARMTargetLowering::EmitStructByval(MachineInstr *MI,
     const Constant *C = ConstantInt::get(Int32Ty, LoopSize);
 
     // MachineConstantPool wants an explicit alignment.
-    unsigned Align = getDataLayout()->getPrefTypeAlignment(Int32Ty);
+    unsigned Align = MF->getDataLayout().getPrefTypeAlignment(Int32Ty);
     if (Align == 0)
-      Align = getDataLayout()->getTypeAllocSize(C->getType());
+      Align = MF->getDataLayout().getTypeAllocSize(C->getType());
     unsigned Idx = ConstantPool->getConstantPoolIndex(C, Align);
 
     if (IsThumb1)
@@ -7659,6 +7840,9 @@ ARMTargetLowering::EmitInstrWithCustomInserter(MachineInstr *MI,
   case ARM::tInt_eh_sjlj_setjmp:
   case ARM::t2Int_eh_sjlj_setjmp:
   case ARM::t2Int_eh_sjlj_setjmp_nofp:
+    return BB;
+
+  case ARM::Int_eh_sjlj_setup_dispatch:
     EmitSjLjDispatchBlock(MI, BB);
     return BB;
 
@@ -7687,6 +7871,7 @@ ARMTargetLowering::EmitInstrWithCustomInserter(MachineInstr *MI,
 
     unsigned int ABSSrcReg = MI->getOperand(1).getReg();
     unsigned int ABSDstReg = MI->getOperand(0).getReg();
+    bool ABSSrcKIll = MI->getOperand(1).isKill();
     bool isThumb2 = Subtarget->isThumb2();
     MachineRegisterInfo &MRI = Fn->getRegInfo();
     // In Thumb mode S must not be specified if source register is the SP or
@@ -7720,7 +7905,7 @@ ARMTargetLowering::EmitInstrWithCustomInserter(MachineInstr *MI,
     // by if-conversion pass
     BuildMI(*RSBBB, RSBBB->begin(), dl,
       TII->get(isThumb2 ? ARM::t2RSBri : ARM::RSBri), NewRsbDstReg)
-      .addReg(ABSSrcReg, RegState::Kill)
+      .addReg(ABSSrcReg, ABSSrcKIll ? RegState::Kill : 0)
       .addImm(0).addImm((unsigned)ARMCC::AL).addReg(0).addReg(0);
 
     // insert PHI in SinkBB,
@@ -7861,6 +8046,7 @@ static bool isConditionalZeroOrAllOnes(SDNode *N, bool AllOnes,
       return false;
     // Fall through.
   case ISD::SIGN_EXTEND: {
+    SDLoc dl(N);
     EVT VT = N->getValueType(0);
     CC = N->getOperand(0);
     if (CC.getValueType() != MVT::i1)
@@ -7869,12 +8055,13 @@ static bool isConditionalZeroOrAllOnes(SDNode *N, bool AllOnes,
     if (AllOnes)
       // When looking for an AllOnes constant, N is an sext, and the 'other'
       // value is 0.
-      OtherOp = DAG.getConstant(0, VT);
+      OtherOp = DAG.getConstant(0, dl, VT);
     else if (N->getOpcode() == ISD::ZERO_EXTEND)
       // When looking for a 0 constant, N can be zext or sext.
-      OtherOp = DAG.getConstant(1, VT);
+      OtherOp = DAG.getConstant(1, dl, VT);
     else
-      OtherOp = DAG.getConstant(APInt::getAllOnesValue(VT.getSizeInBits()), VT);
+      OtherOp = DAG.getConstant(APInt::getAllOnesValue(VT.getSizeInBits()), dl,
+                                VT);
     return true;
   }
   }
@@ -8013,10 +8200,12 @@ static SDValue AddCombineToVPADDL(SDNode *N, SDValue N0, SDValue N1,
   SelectionDAG &DAG = DCI.DAG;
   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
 
+  SDLoc dl(N);
+
   // Build operand list.
   SmallVector<SDValue, 8> Ops;
-  Ops.push_back(DAG.getConstant(Intrinsic::arm_neon_vpaddls,
-                                TLI.getPointerTy()));
+  Ops.push_back(DAG.getConstant(Intrinsic::arm_neon_vpaddls, dl,
+                                TLI.getPointerTy(DAG.getDataLayout())));
 
   // Input is the vector.
   Ops.push_back(Vec);
@@ -8034,9 +8223,9 @@ static SDValue AddCombineToVPADDL(SDNode *N, SDValue N0, SDValue N1,
       llvm_unreachable("Invalid vector element type for padd optimization.");
   }
 
-  SDValue tmp = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, SDLoc(N), widenType, Ops);
+  SDValue tmp = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, dl, widenType, Ops);
   unsigned ExtOp = VT.bitsGT(tmp.getValueType()) ? ISD::ANY_EXTEND : ISD::TRUNCATE;
-  return DAG.getNode(ExtOp, SDLoc(N), VT, tmp);
+  return DAG.getNode(ExtOp, dl, VT, tmp);
 }
 
 static SDValue findMUL_LOHI(SDValue V) {
@@ -8061,13 +8250,13 @@ static SDValue AddCombineTo64bitMLAL(SDNode *AddcNode,
   // a glue link from the first add to the second add.
   // If we find this pattern, we can replace the U/SMUL_LOHI, ADDC, and ADDE by
   // a S/UMLAL instruction.
-  //          loAdd   UMUL_LOHI
-  //            \    / :lo    \ :hi
-  //             \  /          \          [no multiline comment]
-  //              ADDC         |  hiAdd
-  //                 \ :glue  /  /
-  //                  \      /  /
-  //                    ADDE
+  //                  UMUL_LOHI
+  //                 / :lo    \ :hi
+  //                /          \          [no multiline comment]
+  //    loAdd ->  ADDE         |
+  //                 \ :glue  /
+  //                  \      /
+  //                    ADDC   <- hiAdd
   //
   assert(AddcNode->getOpcode() == ISD::ADDC && "Expect an ADDC");
   SDValue AddcOp0 = AddcNode->getOperand(0);
@@ -8330,14 +8519,14 @@ static SDValue PerformMULCombine(SDNode *N,
                         V,
                         DAG.getNode(ISD::SHL, DL, VT,
                                     V,
-                                    DAG.getConstant(Log2_32(MulAmt - 1),
+                                    DAG.getConstant(Log2_32(MulAmt - 1), DL,
                                                     MVT::i32)));
     } else if (isPowerOf2_32(MulAmt + 1)) {
       // (mul x, 2^N - 1) => (sub (shl x, N), x)
       Res = DAG.getNode(ISD::SUB, DL, VT,
                         DAG.getNode(ISD::SHL, DL, VT,
                                     V,
-                                    DAG.getConstant(Log2_32(MulAmt + 1),
+                                    DAG.getConstant(Log2_32(MulAmt + 1), DL,
                                                     MVT::i32)),
                         V);
     } else
@@ -8350,7 +8539,7 @@ static SDValue PerformMULCombine(SDNode *N,
                         V,
                         DAG.getNode(ISD::SHL, DL, VT,
                                     V,
-                                    DAG.getConstant(Log2_32(MulAmtAbs + 1),
+                                    DAG.getConstant(Log2_32(MulAmtAbs + 1), DL,
                                                     MVT::i32)));
     } else if (isPowerOf2_32(MulAmtAbs - 1)) {
       // (mul x, -(2^N + 1)) => - (add (shl x, N), x)
@@ -8358,10 +8547,10 @@ static SDValue PerformMULCombine(SDNode *N,
                         V,
                         DAG.getNode(ISD::SHL, DL, VT,
                                     V,
-                                    DAG.getConstant(Log2_32(MulAmtAbs-1),
+                                    DAG.getConstant(Log2_32(MulAmtAbs - 1), DL,
                                                     MVT::i32)));
       Res = DAG.getNode(ISD::SUB, DL, VT,
-                        DAG.getConstant(0, MVT::i32),Res);
+                        DAG.getConstant(0, DL, MVT::i32), Res);
 
     } else
       return SDValue();
@@ -8369,7 +8558,7 @@ static SDValue PerformMULCombine(SDNode *N,
 
   if (ShiftAmt != 0)
     Res = DAG.getNode(ISD::SHL, DL, VT,
-                      Res, DAG.getConstant(ShiftAmt, MVT::i32));
+                      Res, DAG.getConstant(ShiftAmt, DL, MVT::i32));
 
   // Do not add new nodes to DAG combiner worklist.
   DCI.CombineTo(N, Res, false);
@@ -8398,7 +8587,7 @@ static SDValue PerformANDCombine(SDNode *N,
       EVT VbicVT;
       SDValue Val = isNEONModifiedImm((~SplatBits).getZExtValue(),
                                       SplatUndef.getZExtValue(), SplatBitSize,
-                                      DAG, VbicVT, VT.is128BitVector(),
+                                      DAG, dl, VbicVT, VT.is128BitVector(),
                                       OtherModImm);
       if (Val.getNode()) {
         SDValue Input =
@@ -8441,7 +8630,7 @@ static SDValue PerformORCombine(SDNode *N,
       EVT VorrVT;
       SDValue Val = isNEONModifiedImm(SplatBits.getZExtValue(),
                                       SplatUndef.getZExtValue(), SplatBitSize,
-                                      DAG, VorrVT, VT.is128BitVector(),
+                                      DAG, dl, VorrVT, VT.is128BitVector(),
                                       OtherModImm);
       if (Val.getNode()) {
         SDValue Input =
@@ -8545,8 +8734,8 @@ static SDValue PerformORCombine(SDNode *N,
       Val >>= countTrailingZeros(~Mask);
 
       Res = DAG.getNode(ARMISD::BFI, DL, VT, N00,
-                        DAG.getConstant(Val, MVT::i32),
-                        DAG.getConstant(Mask, MVT::i32));
+                        DAG.getConstant(Val, DL, MVT::i32),
+                        DAG.getConstant(Mask, DL, MVT::i32));
 
       // Do not add new nodes to DAG combiner worklist.
       DCI.CombineTo(N, Res, false);
@@ -8571,9 +8760,9 @@ static SDValue PerformORCombine(SDNode *N,
       // 2a
       unsigned amt = countTrailingZeros(Mask2);
       Res = DAG.getNode(ISD::SRL, DL, VT, N1.getOperand(0),
-                        DAG.getConstant(amt, MVT::i32));
+                        DAG.getConstant(amt, DL, MVT::i32));
       Res = DAG.getNode(ARMISD::BFI, DL, VT, N00, Res,
-                        DAG.getConstant(Mask, MVT::i32));
+                        DAG.getConstant(Mask, DL, MVT::i32));
       // Do not add new nodes to DAG combiner worklist.
       DCI.CombineTo(N, Res, false);
       return SDValue();
@@ -8587,9 +8776,9 @@ static SDValue PerformORCombine(SDNode *N,
       // 2b
       unsigned lsb = countTrailingZeros(Mask);
       Res = DAG.getNode(ISD::SRL, DL, VT, N00,
-                        DAG.getConstant(lsb, MVT::i32));
+                        DAG.getConstant(lsb, DL, MVT::i32));
       Res = DAG.getNode(ARMISD::BFI, DL, VT, N1.getOperand(0), Res,
-                        DAG.getConstant(Mask2, MVT::i32));
+                        DAG.getConstant(Mask2, DL, MVT::i32));
       // Do not add new nodes to DAG combiner worklist.
       DCI.CombineTo(N, Res, false);
       return SDValue();
@@ -8608,7 +8797,7 @@ static SDValue PerformORCombine(SDNode *N,
       return SDValue();
 
     Res = DAG.getNode(ARMISD::BFI, DL, VT, N1, N00.getOperand(0),
-                      DAG.getConstant(~Mask, MVT::i32));
+                      DAG.getConstant(~Mask, DL, MVT::i32));
 
     // Do not add new nodes to DAG combiner worklist.
     DCI.CombineTo(N, Res, false);
@@ -8689,14 +8878,14 @@ static SDValue PerformVMOVRRDCombine(SDNode *N,
                                  LD->getAlignment());
 
     SDValue OffsetPtr = DAG.getNode(ISD::ADD, DL, MVT::i32, BasePtr,
-                                    DAG.getConstant(4, MVT::i32));
+                                    DAG.getConstant(4, DL, MVT::i32));
     SDValue NewLD2 = DAG.getLoad(MVT::i32, DL, NewLD1.getValue(1), OffsetPtr,
                                  LD->getPointerInfo(), LD->isVolatile(),
                                  LD->isNonTemporal(), LD->isInvariant(),
                                  std::min(4U, LD->getAlignment() / 2));
 
     DAG.ReplaceAllUsesOfValueWith(SDValue(LD, 1), NewLD2.getValue(1));
-    if (DCI.DAG.getTargetLoweringInfo().isBigEndian())
+    if (DCI.DAG.getDataLayout().isBigEndian())
       std::swap (NewLD1, NewLD2);
     SDValue Result = DCI.CombineTo(N, NewLD1, NewLD2);
     return Result;
@@ -8855,7 +9044,7 @@ PerformARMBUILD_VECTORCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI) {
       // Make the DAGCombiner fold the bitcasts.
       DCI.AddToWorklist(V.getNode());
     }
-    SDValue LaneIdx = DAG.getConstant(Idx, MVT::i32);
+    SDValue LaneIdx = DAG.getConstant(Idx, dl, MVT::i32);
     Vec = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, VecVT, Vec, V, LaneIdx);
   }
   Vec = DAG.getNode(ISD::BITCAST, dl, VT, Vec);
@@ -8957,6 +9146,7 @@ static SDValue CombineBaseUpdate(SDNode *N,
   const unsigned AddrOpIdx = ((isIntrinsic || isStore) ? 2 : 1);
   SDValue Addr = N->getOperand(AddrOpIdx);
   MemSDNode *MemN = cast<MemSDNode>(N);
+  SDLoc dl(N);
 
   // Search for a use of the address operand that is an increment.
   for (SDNode::use_iterator UI = Addr.getNode()->use_begin(),
@@ -9118,16 +9308,16 @@ static SDValue CombineBaseUpdate(SDNode *N,
     }
 
     // For all node types, the alignment operand is always the last one.
-    Ops.push_back(DAG.getConstant(Alignment, MVT::i32));
+    Ops.push_back(DAG.getConstant(Alignment, dl, MVT::i32));
 
     // If this is a non-standard-aligned STORE, the penultimate operand is the
     // stored value.  Bitcast it to the aligned type.
     if (AlignedVecTy != VecTy && N->getOpcode() == ISD::STORE) {
       SDValue &StVal = Ops[Ops.size()-2];
-      StVal = DAG.getNode(ISD::BITCAST, SDLoc(N), AlignedVecTy, StVal);
+      StVal = DAG.getNode(ISD::BITCAST, dl, AlignedVecTy, StVal);
     }
 
-    SDValue UpdN = DAG.getMemIntrinsicNode(NewOpc, SDLoc(N), SDTys,
+    SDValue UpdN = DAG.getMemIntrinsicNode(NewOpc, dl, SDTys,
                                            Ops, AlignedVecTy,
                                            MemN->getMemOperand());
 
@@ -9140,7 +9330,7 @@ static SDValue CombineBaseUpdate(SDNode *N,
     // value.  Bitcast it to the expected result type.
     if (AlignedVecTy != VecTy && N->getOpcode() == ISD::LOAD) {
       SDValue &LdVal = NewResults[0];
-      LdVal = DAG.getNode(ISD::BITCAST, SDLoc(N), VecTy, LdVal);
+      LdVal = DAG.getNode(ISD::BITCAST, dl, VecTy, LdVal);
     }
 
     NewResults.push_back(SDValue(UpdN.getNode(), NumResultVecs+1)); // chain
@@ -9326,7 +9516,9 @@ static SDValue PerformSTORECombine(SDNode *N,
     SDValue WideVec = DAG.getNode(ISD::BITCAST, DL, WideVecVT, StVal);
     SmallVector<int, 8> ShuffleVec(NumElems * SizeRatio, -1);
     for (unsigned i = 0; i < NumElems; ++i)
-      ShuffleVec[i] = TLI.isBigEndian() ? (i+1) * SizeRatio - 1 : i * SizeRatio;
+      ShuffleVec[i] = DAG.getDataLayout().isBigEndian()
+                          ? (i + 1) * SizeRatio - 1
+                          : i * SizeRatio;
 
     // Can't shuffle using an illegal type.
     if (!TLI.isTypeLegal(WideVecVT)) return SDValue();
@@ -9353,8 +9545,8 @@ static SDValue PerformSTORECombine(SDNode *N,
     assert(StoreVecVT.getSizeInBits() == VT.getSizeInBits());
     SDValue ShuffWide = DAG.getNode(ISD::BITCAST, DL, StoreVecVT, Shuff);
     SmallVector<SDValue, 8> Chains;
-    SDValue Increment = DAG.getConstant(StoreType.getSizeInBits()/8,
-                                        TLI.getPointerTy());
+    SDValue Increment = DAG.getConstant(StoreType.getSizeInBits() / 8, DL,
+                                        TLI.getPointerTy(DAG.getDataLayout()));
     SDValue BasePtr = St->getBasePtr();
 
     // Perform one or more big stores into memory.
@@ -9362,7 +9554,7 @@ static SDValue PerformSTORECombine(SDNode *N,
     for (unsigned I = 0; I < E; I++) {
       SDValue SubVec = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL,
                                    StoreType, ShuffWide,
-                                   DAG.getIntPtrConstant(I));
+                                   DAG.getIntPtrConstant(I, DL));
       SDValue Ch = DAG.getStore(St->getChain(), DL, SubVec, BasePtr,
                                 St->getPointerInfo(), St->isVolatile(),
                                 St->isNonTemporal(), St->getAlignment());
@@ -9381,7 +9573,7 @@ static SDValue PerformSTORECombine(SDNode *N,
   if (StVal.getNode()->getOpcode() == ARMISD::VMOVDRR &&
       StVal.getNode()->hasOneUse()) {
     SelectionDAG  &DAG = DCI.DAG;
-    bool isBigEndian = DAG.getTargetLoweringInfo().isBigEndian();
+    bool isBigEndian = DAG.getDataLayout().isBigEndian();
     SDLoc DL(St);
     SDValue BasePtr = St->getBasePtr();
     SDValue NewST1 = DAG.getStore(St->getChain(), DL,
@@ -9390,7 +9582,7 @@ static SDValue PerformSTORECombine(SDNode *N,
                                   St->isNonTemporal(), St->getAlignment());
 
     SDValue OffsetPtr = DAG.getNode(ISD::ADD, DL, MVT::i32, BasePtr,
-                                    DAG.getConstant(4, MVT::i32));
+                                    DAG.getConstant(4, DL, MVT::i32));
     return DAG.getStore(NewST1.getValue(0), DL,
                         StVal.getNode()->getOperand(isBigEndian ? 0 : 1),
                         OffsetPtr, St->getPointerInfo(), St->isVolatile(),
@@ -9497,15 +9689,17 @@ static SDValue PerformVCVTCombine(SDNode *N,
     return SDValue();
   }
 
+  SDLoc dl(N);
   unsigned IntrinsicOpcode = isSigned ? Intrinsic::arm_neon_vcvtfp2fxs :
     Intrinsic::arm_neon_vcvtfp2fxu;
-  SDValue FixConv =  DAG.getNode(ISD::INTRINSIC_WO_CHAIN, SDLoc(N),
+  SDValue FixConv =  DAG.getNode(ISD::INTRINSIC_WO_CHAIN, dl,
                                  NumLanes == 2 ? MVT::v2i32 : MVT::v4i32,
-                                 DAG.getConstant(IntrinsicOpcode, MVT::i32), N0,
-                                 DAG.getConstant(Log2_64(C), MVT::i32));
+                                 DAG.getConstant(IntrinsicOpcode, dl, MVT::i32),
+                                 N0,
+                                 DAG.getConstant(Log2_64(C), dl, MVT::i32));
 
   if (IntTy.getSizeInBits() < FloatTy.getSizeInBits())
-    FixConv = DAG.getNode(ISD::TRUNCATE, SDLoc(N), N->getValueType(0), FixConv);
+    FixConv = DAG.getNode(ISD::TRUNCATE, dl, N->getValueType(0), FixConv);
 
   return FixConv;
 }
@@ -9547,19 +9741,20 @@ static SDValue PerformVDIVCombine(SDNode *N,
     return SDValue();
   }
 
+  SDLoc dl(N);
   SDValue ConvInput = Op.getOperand(0);
   unsigned NumLanes = Op.getValueType().getVectorNumElements();
   if (IntTy.getSizeInBits() < FloatTy.getSizeInBits())
     ConvInput = DAG.getNode(isSigned ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND,
-                            SDLoc(N), NumLanes == 2 ? MVT::v2i32 : MVT::v4i32,
+                            dl, NumLanes == 2 ? MVT::v2i32 : MVT::v4i32,
                             ConvInput);
 
   unsigned IntrinsicOpcode = isSigned ? Intrinsic::arm_neon_vcvtfxs2fp :
     Intrinsic::arm_neon_vcvtfxu2fp;
-  return DAG.getNode(ISD::INTRINSIC_WO_CHAIN, SDLoc(N),
+  return DAG.getNode(ISD::INTRINSIC_WO_CHAIN, dl,
                      Op.getValueType(),
-                     DAG.getConstant(IntrinsicOpcode, MVT::i32),
-                     ConvInput, DAG.getConstant(Log2_64(C), MVT::i32));
+                     DAG.getConstant(IntrinsicOpcode, dl, MVT::i32),
+                     ConvInput, DAG.getConstant(Log2_64(C), dl, MVT::i32));
 }
 
 /// Getvshiftimm - Check if this is a valid build_vector for the immediate
@@ -9587,7 +9782,7 @@ static bool getVShiftImm(SDValue Op, unsigned ElementBits, int64_t &Cnt) {
 ///   0 <= Value <= ElementBits for a long left shift.
 static bool isVShiftLImm(SDValue Op, EVT VT, bool isLong, int64_t &Cnt) {
   assert(VT.isVector() && "vector shift count is not a vector type");
-  unsigned ElementBits = VT.getVectorElementType().getSizeInBits();
+  int64_t ElementBits = VT.getVectorElementType().getSizeInBits();
   if (! getVShiftImm(Op, ElementBits, Cnt))
     return false;
   return (Cnt >= 0 && (isLong ? Cnt-1 : Cnt) < ElementBits);
@@ -9602,12 +9797,16 @@ static bool isVShiftLImm(SDValue Op, EVT VT, bool isLong, int64_t &Cnt) {
 static bool isVShiftRImm(SDValue Op, EVT VT, bool isNarrow, bool isIntrinsic,
                          int64_t &Cnt) {
   assert(VT.isVector() && "vector shift count is not a vector type");
-  unsigned ElementBits = VT.getVectorElementType().getSizeInBits();
+  int64_t ElementBits = VT.getVectorElementType().getSizeInBits();
   if (! getVShiftImm(Op, ElementBits, Cnt))
     return false;
-  if (isIntrinsic)
+  if (!isIntrinsic)
+    return (Cnt >= 1 && Cnt <= (isNarrow ? ElementBits/2 : ElementBits));
+  if (Cnt >= -(isNarrow ? ElementBits/2 : ElementBits) && Cnt <= -1) {
     Cnt = -Cnt;
-  return (Cnt >= 1 && Cnt <= (isNarrow ? ElementBits/2 : ElementBits));
+    return true;
+  }
+  return false;
 }
 
 /// PerformIntrinsicCombine - ARM-specific DAG combining for intrinsics.
@@ -9618,6 +9817,15 @@ static SDValue PerformIntrinsicCombine(SDNode *N, SelectionDAG &DAG) {
     // Don't do anything for most intrinsics.
     break;
 
+  case Intrinsic::arm_neon_vabds:
+    if (!N->getValueType(0).isInteger())
+      return SDValue();
+    return DAG.getNode(ISD::SABSDIFF, SDLoc(N), N->getValueType(0),
+                       N->getOperand(1), N->getOperand(2));
+  case Intrinsic::arm_neon_vabdu:
+    return DAG.getNode(ISD::UABSDIFF, SDLoc(N), N->getValueType(0),
+                       N->getOperand(1), N->getOperand(2));
+
   // Vector shifts: check for immediate versions and lower them.
   // Note: This is done during DAG combining instead of DAG legalizing because
   // the build_vectors for 64-bit vector element shift counts are generally
@@ -9720,8 +9928,9 @@ static SDValue PerformIntrinsicCombine(SDNode *N, SelectionDAG &DAG) {
       VShiftOpc = ARMISD::VQRSHRNsu; break;
     }
 
-    return DAG.getNode(VShiftOpc, SDLoc(N), N->getValueType(0),
-                       N->getOperand(1), DAG.getConstant(Cnt, MVT::i32));
+    SDLoc dl(N);
+    return DAG.getNode(VShiftOpc, dl, N->getValueType(0),
+                       N->getOperand(1), DAG.getConstant(Cnt, dl, MVT::i32));
   }
 
   case Intrinsic::arm_neon_vshiftins: {
@@ -9737,9 +9946,10 @@ static SDValue PerformIntrinsicCombine(SDNode *N, SelectionDAG &DAG) {
       llvm_unreachable("invalid shift count for vsli/vsri intrinsic");
     }
 
-    return DAG.getNode(VShiftOpc, SDLoc(N), N->getValueType(0),
+    SDLoc dl(N);
+    return DAG.getNode(VShiftOpc, dl, N->getValueType(0),
                        N->getOperand(1), N->getOperand(2),
-                       DAG.getConstant(Cnt, MVT::i32));
+                       DAG.getConstant(Cnt, dl, MVT::i32));
   }
 
   case Intrinsic::arm_neon_vqrshifts:
@@ -9784,9 +9994,11 @@ static SDValue PerformShiftCombine(SDNode *N, SelectionDAG &DAG,
   default: llvm_unreachable("unexpected shift opcode");
 
   case ISD::SHL:
-    if (isVShiftLImm(N->getOperand(1), VT, false, Cnt))
-      return DAG.getNode(ARMISD::VSHL, SDLoc(N), VT, N->getOperand(0),
-                         DAG.getConstant(Cnt, MVT::i32));
+    if (isVShiftLImm(N->getOperand(1), VT, false, Cnt)) {
+      SDLoc dl(N);
+      return DAG.getNode(ARMISD::VSHL, dl, VT, N->getOperand(0),
+                         DAG.getConstant(Cnt, dl, MVT::i32));
+    }
     break;
 
   case ISD::SRA:
@@ -9794,8 +10006,9 @@ static SDValue PerformShiftCombine(SDNode *N, SelectionDAG &DAG,
     if (isVShiftRImm(N->getOperand(1), VT, false, false, Cnt)) {
       unsigned VShiftOpc = (N->getOpcode() == ISD::SRA ?
                             ARMISD::VSHRs : ARMISD::VSHRu);
-      return DAG.getNode(VShiftOpc, SDLoc(N), VT, N->getOperand(0),
-                         DAG.getConstant(Cnt, MVT::i32));
+      SDLoc dl(N);
+      return DAG.getNode(VShiftOpc, dl, VT, N->getOperand(0),
+                         DAG.getConstant(Cnt, dl, MVT::i32));
     }
   }
   return SDValue();
@@ -10084,7 +10297,7 @@ bool ARMTargetLowering::allowsMisalignedMemoryAccesses(EVT VT,
     // For any little-endian targets with neon, we can support unaligned ld/st
     // of D and Q (e.g. {D0,D1}) registers by using vld1.i8/vst1.i8.
     // A big-endian target may also explicitly support unaligned accesses
-    if (Subtarget->hasNEON() && (AllowsUnaligned || isLittleEndian())) {
+    if (Subtarget->hasNEON() && (AllowsUnaligned || Subtarget->isLittle())) {
       if (Fast)
         *Fast = true;
       return true;
@@ -10323,9 +10536,10 @@ bool ARMTargetLowering::isLegalT2ScaledAddressingMode(const AddrMode &AM,
 
 /// isLegalAddressingMode - Return true if the addressing mode represented
 /// by AM is legal for this target, for a load/store of the specified type.
-bool ARMTargetLowering::isLegalAddressingMode(const AddrMode &AM,
-                                              Type *Ty) const {
-  EVT VT = getValueType(Ty, true);
+bool ARMTargetLowering::isLegalAddressingMode(const DataLayout &DL,
+                                              const AddrMode &AM, Type *Ty,
+                                              unsigned AS) const {
+  EVT VT = getValueType(DL, Ty, true);
   if (!isLegalAddressImmediate(AM.BaseOffs, VT, Subtarget))
     return false;
 
@@ -10426,7 +10640,7 @@ static bool getARMIndexedAddressParts(SDNode *Ptr, EVT VT,
       if (RHSC < 0 && RHSC > -256) {
         assert(Ptr->getOpcode() == ISD::ADD);
         isInc = false;
-        Offset = DAG.getConstant(-RHSC, RHS->getValueType(0));
+        Offset = DAG.getConstant(-RHSC, SDLoc(Ptr), RHS->getValueType(0));
         return true;
       }
     }
@@ -10440,7 +10654,7 @@ static bool getARMIndexedAddressParts(SDNode *Ptr, EVT VT,
       if (RHSC < 0 && RHSC > -0x1000) {
         assert(Ptr->getOpcode() == ISD::ADD);
         isInc = false;
-        Offset = DAG.getConstant(-RHSC, RHS->getValueType(0));
+        Offset = DAG.getConstant(-RHSC, SDLoc(Ptr), RHS->getValueType(0));
         Base = Ptr->getOperand(0);
         return true;
       }
@@ -10483,11 +10697,11 @@ static bool getT2IndexedAddressParts(SDNode *Ptr, EVT VT,
     if (RHSC < 0 && RHSC > -0x100) { // 8 bits.
       assert(Ptr->getOpcode() == ISD::ADD);
       isInc = false;
-      Offset = DAG.getConstant(-RHSC, RHS->getValueType(0));
+      Offset = DAG.getConstant(-RHSC, SDLoc(Ptr), RHS->getValueType(0));
       return true;
     } else if (RHSC > 0 && RHSC < 0x100) { // 8 bit, no zero.
       isInc = Ptr->getOpcode() == ISD::ADD;
-      Offset = DAG.getConstant(RHSC, RHS->getValueType(0));
+      Offset = DAG.getConstant(RHSC, SDLoc(Ptr), RHS->getValueType(0));
       return true;
     }
   }
@@ -10506,15 +10720,6 @@ ARMTargetLowering::getPreIndexedAddressParts(SDNode *N, SDValue &Base,
   if (Subtarget->isThumb1Only())
     return false;
 
-  // @LOCALMOD-START
-  // Avoid two reg addressing mode for loads and stores
-  const bool restrict_addressing_modes_for_nacl =
-      Subtarget->isTargetNaCl() && isa<MemSDNode>(N);
-  if (restrict_addressing_modes_for_nacl) {
-    return false;
-  }
-  // @LOCALMOD-END
-
   EVT VT;
   SDValue Ptr;
   bool isSEXTLoad = false;
@@ -10553,14 +10758,7 @@ bool ARMTargetLowering::getPostIndexedAddressParts(SDNode *N, SDNode *Op,
                                                    SelectionDAG &DAG) const {
   if (Subtarget->isThumb1Only())
     return false;
-   // @LOCALMOD-START
-  // Avoid two reg addressing mode for loads and stores
-  const bool restrict_addressing_modes_for_nacl =
-      Subtarget->isTargetNaCl() && isa<MemSDNode>(N);
-  if (restrict_addressing_modes_for_nacl) {
-    return false;
-  }
-  // @LOCALMOD-END
+
   EVT VT;
   SDValue Ptr;
   bool isSEXTLoad = false;
@@ -10685,7 +10883,7 @@ bool ARMTargetLowering::ExpandInlineAsm(CallInst *CI) const {
 /// getConstraintType - Given a constraint letter, return the type of
 /// constraint it is for this target.
 ARMTargetLowering::ConstraintType
-ARMTargetLowering::getConstraintType(const std::string &Constraint) const {
+ARMTargetLowering::getConstraintType(StringRef Constraint) const {
   if (Constraint.size() == 1) {
     switch (Constraint[0]) {
     default:  break;
@@ -10744,10 +10942,8 @@ ARMTargetLowering::getSingleConstraintMatchWeight(
 }
 
 typedef std::pair<unsigned, const TargetRegisterClass*> RCPair;
-RCPair
-ARMTargetLowering::getRegForInlineAsmConstraint(const TargetRegisterInfo *TRI,
-                                                const std::string &Constraint,
-                                                MVT VT) const {
+RCPair ARMTargetLowering::getRegForInlineAsmConstraint(
+    const TargetRegisterInfo *TRI, StringRef Constraint, MVT VT) const {
   if (Constraint.size() == 1) {
     // GCC ARM Constraint Letters
     switch (Constraint[0]) {
@@ -10951,7 +11147,7 @@ void ARMTargetLowering::LowerAsmOperandForConstraint(SDValue Op,
         }
         return;
     }
-    Result = DAG.getTargetConstant(CVal, Op.getValueType());
+    Result = DAG.getTargetConstant(CVal, SDLoc(Op), Op.getValueType());
     break;
   }
 
@@ -10963,7 +11159,8 @@ void ARMTargetLowering::LowerAsmOperandForConstraint(SDValue Op,
 }
 
 SDValue ARMTargetLowering::LowerDivRem(SDValue Op, SelectionDAG &DAG) const {
-  assert(Subtarget->isTargetAEABI() && "Register-based DivRem lowering only");
+  assert((Subtarget->isTargetAEABI() || Subtarget->isTargetAndroid()) &&
+         "Register-based DivRem lowering only");
   unsigned Opcode = Op->getOpcode();
   assert((Opcode == ISD::SDIVREM || Opcode == ISD::UDIVREM) &&
          "Invalid opcode for Div/Rem lowering");
@@ -10995,7 +11192,7 @@ SDValue ARMTargetLowering::LowerDivRem(SDValue Op, SelectionDAG &DAG) const {
   }
 
   SDValue Callee = DAG.getExternalSymbol(getLibcallName(LC),
-                                         getPointerTy());
+                                         getPointerTy(DAG.getDataLayout()));
 
   Type *RetTy = (Type*)StructType::get(Ty, Ty, nullptr);
 
@@ -11019,7 +11216,7 @@ ARMTargetLowering::LowerDYNAMIC_STACKALLOC(SDValue Op, SelectionDAG &DAG) const
   SDValue Size  = Op.getOperand(1);
 
   SDValue Words = DAG.getNode(ISD::SRL, DL, MVT::i32, Size,
-                              DAG.getConstant(2, MVT::i32));
+                              DAG.getConstant(2, DL, MVT::i32));
 
   SDValue Flag;
   Chain = DAG.getCopyToReg(Chain, DL, ARM::R4, Words, Flag);
@@ -11104,7 +11301,8 @@ bool ARMTargetLowering::getTgtMemIntrinsic(IntrinsicInfo &Info,
   case Intrinsic::arm_neon_vld4lane: {
     Info.opc = ISD::INTRINSIC_W_CHAIN;
     // Conservatively set memVT to the entire set of vectors loaded.
-    uint64_t NumElts = getDataLayout()->getTypeAllocSize(I.getType()) / 8;
+    auto &DL = I.getCalledFunction()->getParent()->getDataLayout();
+    uint64_t NumElts = DL.getTypeAllocSize(I.getType()) / 8;
     Info.memVT = EVT::getVectorVT(I.getType()->getContext(), MVT::i64, NumElts);
     Info.ptrVal = I.getArgOperand(0);
     Info.offset = 0;
@@ -11124,12 +11322,13 @@ bool ARMTargetLowering::getTgtMemIntrinsic(IntrinsicInfo &Info,
   case Intrinsic::arm_neon_vst4lane: {
     Info.opc = ISD::INTRINSIC_VOID;
     // Conservatively set memVT to the entire set of vectors stored.
+    auto &DL = I.getCalledFunction()->getParent()->getDataLayout();
     unsigned NumElts = 0;
     for (unsigned ArgI = 1, ArgE = I.getNumArgOperands(); ArgI < ArgE; ++ArgI) {
       Type *ArgTy = I.getArgOperand(ArgI)->getType();
       if (!ArgTy->isVectorTy())
         break;
-      NumElts += getDataLayout()->getTypeAllocSize(ArgTy) / 8;
+      NumElts += DL.getTypeAllocSize(ArgTy) / 8;
     }
     Info.memVT = EVT::getVectorVT(I.getType()->getContext(), MVT::i64, NumElts);
     Info.ptrVal = I.getArgOperand(0);
@@ -11143,12 +11342,13 @@ bool ARMTargetLowering::getTgtMemIntrinsic(IntrinsicInfo &Info,
   }
   case Intrinsic::arm_ldaex:
   case Intrinsic::arm_ldrex: {
+    auto &DL = I.getCalledFunction()->getParent()->getDataLayout();
     PointerType *PtrTy = cast<PointerType>(I.getArgOperand(0)->getType());
     Info.opc = ISD::INTRINSIC_W_CHAIN;
     Info.memVT = MVT::getVT(PtrTy->getElementType());
     Info.ptrVal = I.getArgOperand(0);
     Info.offset = 0;
-    Info.align = getDataLayout()->getABITypeAlignment(PtrTy->getElementType());
+    Info.align = DL.getABITypeAlignment(PtrTy->getElementType());
     Info.vol = true;
     Info.readMem = true;
     Info.writeMem = false;
@@ -11156,12 +11356,13 @@ bool ARMTargetLowering::getTgtMemIntrinsic(IntrinsicInfo &Info,
   }
   case Intrinsic::arm_stlex:
   case Intrinsic::arm_strex: {
+    auto &DL = I.getCalledFunction()->getParent()->getDataLayout();
     PointerType *PtrTy = cast<PointerType>(I.getArgOperand(1)->getType());
     Info.opc = ISD::INTRINSIC_W_CHAIN;
     Info.memVT = MVT::getVT(PtrTy->getElementType());
     Info.ptrVal = I.getArgOperand(1);
     Info.offset = 0;
-    Info.align = getDataLayout()->getABITypeAlignment(PtrTy->getElementType());
+    Info.align = DL.getABITypeAlignment(PtrTy->getElementType());
     Info.vol = true;
     Info.readMem = false;
     Info.writeMem = true;
@@ -11412,17 +11613,178 @@ Value *ARMTargetLowering::emitStoreConditional(IRBuilder<> &Builder, Value *Val,
     if (!Subtarget->isLittle())
       std::swap (Lo, Hi);
     Addr = Builder.CreateBitCast(Addr, Type::getInt8PtrTy(M->getContext()));
-    return Builder.CreateCall3(Strex, Lo, Hi, Addr);
+    return Builder.CreateCall(Strex, {Lo, Hi, Addr});
   }
 
   Intrinsic::ID Int = IsRelease ? Intrinsic::arm_stlex : Intrinsic::arm_strex;
   Type *Tys[] = { Addr->getType() };
   Function *Strex = Intrinsic::getDeclaration(M, Int, Tys);
 
-  return Builder.CreateCall2(
-      Strex, Builder.CreateZExtOrBitCast(
-                 Val, Strex->getFunctionType()->getParamType(0)),
-      Addr);
+  return Builder.CreateCall(
+      Strex, {Builder.CreateZExtOrBitCast(
+                  Val, Strex->getFunctionType()->getParamType(0)),
+              Addr});
+}
+
+/// \brief Lower an interleaved load into a vldN intrinsic.
+///
+/// E.g. Lower an interleaved load (Factor = 2):
+///        %wide.vec = load <8 x i32>, <8 x i32>* %ptr, align 4
+///        %v0 = shuffle %wide.vec, undef, <0, 2, 4, 6>  ; Extract even elements
+///        %v1 = shuffle %wide.vec, undef, <1, 3, 5, 7>  ; Extract odd elements
+///
+///      Into:
+///        %vld2 = { <4 x i32>, <4 x i32> } call llvm.arm.neon.vld2(%ptr, 4)
+///        %vec0 = extractelement { <4 x i32>, <4 x i32> } %vld2, i32 0
+///        %vec1 = extractelement { <4 x i32>, <4 x i32> } %vld2, i32 1
+bool ARMTargetLowering::lowerInterleavedLoad(
+    LoadInst *LI, ArrayRef<ShuffleVectorInst *> Shuffles,
+    ArrayRef<unsigned> Indices, unsigned Factor) const {
+  assert(Factor >= 2 && Factor <= getMaxSupportedInterleaveFactor() &&
+         "Invalid interleave factor");
+  assert(!Shuffles.empty() && "Empty shufflevector input");
+  assert(Shuffles.size() == Indices.size() &&
+         "Unmatched number of shufflevectors and indices");
+
+  VectorType *VecTy = Shuffles[0]->getType();
+  Type *EltTy = VecTy->getVectorElementType();
+
+  const DataLayout &DL = LI->getModule()->getDataLayout();
+  unsigned VecSize = DL.getTypeAllocSizeInBits(VecTy);
+  bool EltIs64Bits = DL.getTypeAllocSizeInBits(EltTy) == 64;
+
+  // Skip illegal vector types and vector types of i64/f64 element (vldN doesn't
+  // support i64/f64 element).
+  if ((VecSize != 64 && VecSize != 128) || EltIs64Bits)
+    return false;
+
+  // A pointer vector can not be the return type of the ldN intrinsics. Need to
+  // load integer vectors first and then convert to pointer vectors.
+  if (EltTy->isPointerTy())
+    VecTy =
+        VectorType::get(DL.getIntPtrType(EltTy), VecTy->getVectorNumElements());
+
+  static const Intrinsic::ID LoadInts[3] = {Intrinsic::arm_neon_vld2,
+                                            Intrinsic::arm_neon_vld3,
+                                            Intrinsic::arm_neon_vld4};
+
+  Function *VldnFunc =
+      Intrinsic::getDeclaration(LI->getModule(), LoadInts[Factor - 2], VecTy);
+
+  IRBuilder<> Builder(LI);
+  SmallVector<Value *, 2> Ops;
+
+  Type *Int8Ptr = Builder.getInt8PtrTy(LI->getPointerAddressSpace());
+  Ops.push_back(Builder.CreateBitCast(LI->getPointerOperand(), Int8Ptr));
+  Ops.push_back(Builder.getInt32(LI->getAlignment()));
+
+  CallInst *VldN = Builder.CreateCall(VldnFunc, Ops, "vldN");
+
+  // Replace uses of each shufflevector with the corresponding vector loaded
+  // by ldN.
+  for (unsigned i = 0; i < Shuffles.size(); i++) {
+    ShuffleVectorInst *SV = Shuffles[i];
+    unsigned Index = Indices[i];
+
+    Value *SubVec = Builder.CreateExtractValue(VldN, Index);
+
+    // Convert the integer vector to pointer vector if the element is pointer.
+    if (EltTy->isPointerTy())
+      SubVec = Builder.CreateIntToPtr(SubVec, SV->getType());
+
+    SV->replaceAllUsesWith(SubVec);
+  }
+
+  return true;
+}
+
+/// \brief Get a mask consisting of sequential integers starting from \p Start.
+///
+/// I.e. <Start, Start + 1, ..., Start + NumElts - 1>
+static Constant *getSequentialMask(IRBuilder<> &Builder, unsigned Start,
+                                   unsigned NumElts) {
+  SmallVector<Constant *, 16> Mask;
+  for (unsigned i = 0; i < NumElts; i++)
+    Mask.push_back(Builder.getInt32(Start + i));
+
+  return ConstantVector::get(Mask);
+}
+
+/// \brief Lower an interleaved store into a vstN intrinsic.
+///
+/// E.g. Lower an interleaved store (Factor = 3):
+///        %i.vec = shuffle <8 x i32> %v0, <8 x i32> %v1,
+///                                  <0, 4, 8, 1, 5, 9, 2, 6, 10, 3, 7, 11>
+///        store <12 x i32> %i.vec, <12 x i32>* %ptr, align 4
+///
+///      Into:
+///        %sub.v0 = shuffle <8 x i32> %v0, <8 x i32> v1, <0, 1, 2, 3>
+///        %sub.v1 = shuffle <8 x i32> %v0, <8 x i32> v1, <4, 5, 6, 7>
+///        %sub.v2 = shuffle <8 x i32> %v0, <8 x i32> v1, <8, 9, 10, 11>
+///        call void llvm.arm.neon.vst3(%ptr, %sub.v0, %sub.v1, %sub.v2, 4)
+///
+/// Note that the new shufflevectors will be removed and we'll only generate one
+/// vst3 instruction in CodeGen.
+bool ARMTargetLowering::lowerInterleavedStore(StoreInst *SI,
+                                              ShuffleVectorInst *SVI,
+                                              unsigned Factor) const {
+  assert(Factor >= 2 && Factor <= getMaxSupportedInterleaveFactor() &&
+         "Invalid interleave factor");
+
+  VectorType *VecTy = SVI->getType();
+  assert(VecTy->getVectorNumElements() % Factor == 0 &&
+         "Invalid interleaved store");
+
+  unsigned NumSubElts = VecTy->getVectorNumElements() / Factor;
+  Type *EltTy = VecTy->getVectorElementType();
+  VectorType *SubVecTy = VectorType::get(EltTy, NumSubElts);
+
+  const DataLayout &DL = SI->getModule()->getDataLayout();
+  unsigned SubVecSize = DL.getTypeAllocSizeInBits(SubVecTy);
+  bool EltIs64Bits = DL.getTypeAllocSizeInBits(EltTy) == 64;
+
+  // Skip illegal sub vector types and vector types of i64/f64 element (vstN
+  // doesn't support i64/f64 element).
+  if ((SubVecSize != 64 && SubVecSize != 128) || EltIs64Bits)
+    return false;
+
+  Value *Op0 = SVI->getOperand(0);
+  Value *Op1 = SVI->getOperand(1);
+  IRBuilder<> Builder(SI);
+
+  // StN intrinsics don't support pointer vectors as arguments. Convert pointer
+  // vectors to integer vectors.
+  if (EltTy->isPointerTy()) {
+    Type *IntTy = DL.getIntPtrType(EltTy);
+
+    // Convert to the corresponding integer vector.
+    Type *IntVecTy =
+        VectorType::get(IntTy, Op0->getType()->getVectorNumElements());
+    Op0 = Builder.CreatePtrToInt(Op0, IntVecTy);
+    Op1 = Builder.CreatePtrToInt(Op1, IntVecTy);
+
+    SubVecTy = VectorType::get(IntTy, NumSubElts);
+  }
+
+  static Intrinsic::ID StoreInts[3] = {Intrinsic::arm_neon_vst2,
+                                       Intrinsic::arm_neon_vst3,
+                                       Intrinsic::arm_neon_vst4};
+  Function *VstNFunc = Intrinsic::getDeclaration(
+      SI->getModule(), StoreInts[Factor - 2], SubVecTy);
+
+  SmallVector<Value *, 6> Ops;
+
+  Type *Int8Ptr = Builder.getInt8PtrTy(SI->getPointerAddressSpace());
+  Ops.push_back(Builder.CreateBitCast(SI->getPointerOperand(), Int8Ptr));
+
+  // Split the shufflevector operands into sub vectors for the new vstN call.
+  for (unsigned i = 0; i < Factor; i++)
+    Ops.push_back(Builder.CreateShuffleVector(
+        Op0, Op1, getSequentialMask(Builder, NumSubElts * i, NumSubElts)));
+
+  Ops.push_back(Builder.getInt32(SI->getAlignment()));
+  Builder.CreateCall(VstNFunc, Ops);
+  return true;
 }
 
 enum HABaseType {
diff --git a/lib/Target/ARM/ARMISelLowering.h b/lib/Target/ARM/ARMISelLowering.h
index 851722b5d3e..9d66be077ad 100644
--- a/lib/Target/ARM/ARMISelLowering.h
+++ b/lib/Target/ARM/ARMISelLowering.h
@@ -27,7 +27,7 @@ namespace llvm {
 
   namespace ARMISD {
     // ARM Specific DAG Nodes
-    enum NodeType {
+    enum NodeType : unsigned {
       // Start the numbering where the builtin ops and target ops leave off.
       FIRST_NUMBER = ISD::BUILTIN_OP_END,
 
@@ -36,11 +36,7 @@ namespace llvm {
       WrapperPIC,   // WrapperPIC - A wrapper node for TargetGlobalAddress in
                     // PIC mode.
       WrapperJT,    // WrapperJT - A wrapper node for TargetJumpTable
-      // @LOCALMOD-START
-      WrapperJT2,   // like WrapperJT but without the UID
-      WrapperGOT,   // A Wrapper node for GOT addresses
-      EH_RETURN,    // For LowerEH_RETURN
-      // @LOCALMOD-END
+
       // Add pseudo op to model memcpy for struct byval.
       COPY_STRUCT_BYVAL,
 
@@ -83,6 +79,7 @@ namespace llvm {
 
       EH_SJLJ_SETJMP,         // SjLj exception handling setjmp.
       EH_SJLJ_LONGJMP,        // SjLj exception handling longjmp.
+      EH_SJLJ_SETUP_DISPATCH, // SjLj exception handling setup_dispatch.
 
       TC_RETURN,    // Tail call return pseudo.
 
@@ -235,6 +232,7 @@ namespace llvm {
                                const ARMSubtarget &STI);
 
     unsigned getJumpTableEncoding() const override;
+    bool useSoftFloat() const override;
 
     SDValue LowerOperation(SDValue Op, SelectionDAG &DAG) const override;
 
@@ -252,7 +250,8 @@ namespace llvm {
     }
 
     /// getSetCCResultType - Return the value type to use for ISD::SETCC.
-    EVT getSetCCResultType(LLVMContext &Context, EVT VT) const override;
+    EVT getSetCCResultType(const DataLayout &DL, LLVMContext &Context,
+                           EVT VT) const override;
 
     MachineBasicBlock *
       EmitInstrWithCustomInserter(MachineInstr *MI,
@@ -289,7 +288,8 @@ namespace llvm {
 
     /// isLegalAddressingMode - Return true if the addressing mode represented
     /// by AM is legal for this target, for a load/store of the specified type.
-    bool isLegalAddressingMode(const AddrMode &AM, Type *Ty) const override;
+    bool isLegalAddressingMode(const DataLayout &DL, const AddrMode &AM,
+                               Type *Ty, unsigned AS) const override;
     bool isLegalT2ScaledAddressingMode(const AddrMode &AM, EVT VT) const;
 
     /// isLegalICmpImmediate - Return true if the specified immediate is legal
@@ -326,8 +326,7 @@ namespace llvm {
 
     bool ExpandInlineAsm(CallInst *CI) const override;
 
-    ConstraintType
-      getConstraintType(const std::string &Constraint) const override;
+    ConstraintType getConstraintType(StringRef Constraint) const override;
 
     /// Examine constraint string and operand type and determine a weight value.
     /// The operand object must already have been set up with the operand type.
@@ -336,8 +335,7 @@ namespace llvm {
 
     std::pair<unsigned, const TargetRegisterClass *>
     getRegForInlineAsmConstraint(const TargetRegisterInfo *TRI,
-                                 const std::string &Constraint,
-                                 MVT VT) const override;
+                                 StringRef Constraint, MVT VT) const override;
 
     /// LowerAsmOperandForConstraint - Lower the specified operand into the Ops
     /// vector.  If it is invalid, don't add anything to Ops. If hasMemory is
@@ -347,10 +345,33 @@ namespace llvm {
                                       std::vector<SDValue> &Ops,
                                       SelectionDAG &DAG) const override;
 
-    unsigned getInlineAsmMemConstraint(
-        const std::string &ConstraintCode) const override {
-      // FIXME: Map different constraints differently.
-      return InlineAsm::Constraint_m;
+    unsigned
+    getInlineAsmMemConstraint(StringRef ConstraintCode) const override {
+      if (ConstraintCode == "Q")
+        return InlineAsm::Constraint_Q;
+      else if (ConstraintCode.size() == 2) {
+        if (ConstraintCode[0] == 'U') {
+          switch(ConstraintCode[1]) {
+          default:
+            break;
+          case 'm':
+            return InlineAsm::Constraint_Um;
+          case 'n':
+            return InlineAsm::Constraint_Un;
+          case 'q':
+            return InlineAsm::Constraint_Uq;
+          case 's':
+            return InlineAsm::Constraint_Us;
+          case 't':
+            return InlineAsm::Constraint_Ut;
+          case 'v':
+            return InlineAsm::Constraint_Uv;
+          case 'y':
+            return InlineAsm::Constraint_Uy;
+          }
+        }
+      }
+      return TargetLowering::getInlineAsmMemConstraint(ConstraintCode);
     }
 
     const ARMSubtarget* getSubtarget() const {
@@ -412,6 +433,15 @@ namespace llvm {
     Instruction* emitTrailingFence(IRBuilder<> &Builder, AtomicOrdering Ord,
                            bool IsStore, bool IsLoad) const override;
 
+    unsigned getMaxSupportedInterleaveFactor() const override { return 4; }
+
+    bool lowerInterleavedLoad(LoadInst *LI,
+                              ArrayRef<ShuffleVectorInst *> Shuffles,
+                              ArrayRef<unsigned> Indices,
+                              unsigned Factor) const override;
+    bool lowerInterleavedStore(StoreInst *SI, ShuffleVectorInst *SVI,
+                               unsigned Factor) const override;
+
     bool shouldExpandAtomicLoadInIR(LoadInst *LI) const override;
     bool shouldExpandAtomicStoreInIR(StoreInst *SI) const override;
     TargetLoweringBase::AtomicRMWExpansionKind
@@ -467,6 +497,7 @@ namespace llvm {
                              ISD::ArgFlagsTy Flags) const;
     SDValue LowerEH_SJLJ_SETJMP(SDValue Op, SelectionDAG &DAG) const;
     SDValue LowerEH_SJLJ_LONGJMP(SDValue Op, SelectionDAG &DAG) const;
+    SDValue LowerEH_SJLJ_SETUP_DISPATCH(SDValue Op, SelectionDAG &DAG) const;
     SDValue LowerINTRINSIC_WO_CHAIN(SDValue Op, SelectionDAG &DAG,
                                     const ARMSubtarget *Subtarget) const;
     SDValue LowerBlockAddress(SDValue Op, SelectionDAG &DAG) const;
@@ -479,11 +510,6 @@ namespace llvm {
     SDValue LowerToTLSExecModels(GlobalAddressSDNode *GA,
                                  SelectionDAG &DAG,
                                  TLSModel::Model model) const;
-    // @LOCALMOD-START
-    SDValue LowerJumpTable(SDValue Op, SelectionDAG &DAG) const;
-    SDValue LowerEH_RETURN(SDValue Op, SelectionDAG &DAG) const;
-    // @LOCALMOD-END
-
     SDValue LowerGLOBAL_OFFSET_TABLE(SDValue Op, SelectionDAG &DAG) const;
     SDValue LowerBR_JT(SDValue Op, SelectionDAG &DAG) const;
     SDValue LowerXALUO(SDValue Op, SelectionDAG &DAG) const;
@@ -508,7 +534,8 @@ namespace llvm {
     SDValue LowerFP_TO_INT(SDValue Op, SelectionDAG &DAG) const;
     SDValue LowerINT_TO_FP(SDValue Op, SelectionDAG &DAG) const;
 
-    unsigned getRegisterByName(const char* RegName, EVT VT) const override;
+    unsigned getRegisterByName(const char* RegName, EVT VT,
+                               SelectionDAG &DAG) const override;
 
     /// isFMAFasterThanFMulAndFAdd - Return true if an FMA operation is faster
     /// than a pair of fmul and fadd instructions. fmuladd intrinsics will be
@@ -601,8 +628,7 @@ namespace llvm {
                                 MachineBasicBlock *MBB,
                                 MachineBasicBlock *DispatchBB, int FI) const;
 
-    MachineBasicBlock *EmitSjLjDispatchBlock(MachineInstr *MI,
-                                             MachineBasicBlock *MBB) const;
+    void EmitSjLjDispatchBlock(MachineInstr *MI, MachineBasicBlock *MBB) const;
 
     bool RemapAddSubWithFlags(MachineInstr *MI, MachineBasicBlock *BB) const;
 
@@ -623,7 +649,6 @@ namespace llvm {
     FastISel *createFastISel(FunctionLoweringInfo &funcInfo,
                              const TargetLibraryInfo *libInfo);
   }
-
 }
 
 #endif  // ARMISELLOWERING_H
diff --git a/lib/Target/ARM/ARMInstrInfo.cpp b/lib/Target/ARM/ARMInstrInfo.cpp
index 7c004c9696d..36f76e96d30 100644
--- a/lib/Target/ARM/ARMInstrInfo.cpp
+++ b/lib/Target/ARM/ARMInstrInfo.cpp
@@ -36,16 +36,16 @@ ARMInstrInfo::ARMInstrInfo(const ARMSubtarget &STI)
 void ARMInstrInfo::getNoopForMachoTarget(MCInst &NopInst) const {
   if (hasNOP()) {
     NopInst.setOpcode(ARM::HINT);
-    NopInst.addOperand(MCOperand::CreateImm(0));
-    NopInst.addOperand(MCOperand::CreateImm(ARMCC::AL));
-    NopInst.addOperand(MCOperand::CreateReg(0));
+    NopInst.addOperand(MCOperand::createImm(0));
+    NopInst.addOperand(MCOperand::createImm(ARMCC::AL));
+    NopInst.addOperand(MCOperand::createReg(0));
   } else {
     NopInst.setOpcode(ARM::MOVr);
-    NopInst.addOperand(MCOperand::CreateReg(ARM::R0));
-    NopInst.addOperand(MCOperand::CreateReg(ARM::R0));
-    NopInst.addOperand(MCOperand::CreateImm(ARMCC::AL));
-    NopInst.addOperand(MCOperand::CreateReg(0));
-    NopInst.addOperand(MCOperand::CreateReg(0));
+    NopInst.addOperand(MCOperand::createReg(ARM::R0));
+    NopInst.addOperand(MCOperand::createReg(ARM::R0));
+    NopInst.addOperand(MCOperand::createImm(ARMCC::AL));
+    NopInst.addOperand(MCOperand::createReg(0));
+    NopInst.addOperand(MCOperand::createReg(0));
   }
 }
 
@@ -159,7 +159,7 @@ namespace {
       ARMConstantPoolValue *CPV = ARMConstantPoolSymbol::Create(
           *Context, "_GLOBAL_OFFSET_TABLE_", ARMPCLabelIndex, PCAdj);
 
-      unsigned Align = TM.getDataLayout()->getPrefTypeAlignment(
+      unsigned Align = MF.getDataLayout().getPrefTypeAlignment(
           Type::getInt32PtrTy(*Context));
       unsigned Idx = MF.getConstantPool()->getConstantPoolIndex(CPV, Align);
 
diff --git a/lib/Target/ARM/ARMInstrInfo.td b/lib/Target/ARM/ARMInstrInfo.td
index 434b903815f..ded5533c130 100644
--- a/lib/Target/ARM/ARMInstrInfo.td
+++ b/lib/Target/ARM/ARMInstrInfo.td
@@ -33,13 +33,12 @@ def SDT_ARMCMov    : SDTypeProfile<1, 3,
 def SDT_ARMBrcond  : SDTypeProfile<0, 2,
                                    [SDTCisVT<0, OtherVT>, SDTCisVT<1, i32>]>;
 
-def SDT_ARMBrJT    : SDTypeProfile<0, 3,
-                                  [SDTCisPtrTy<0>, SDTCisVT<1, i32>,
-                                   SDTCisVT<2, i32>]>;
+def SDT_ARMBrJT    : SDTypeProfile<0, 2,
+                                  [SDTCisPtrTy<0>, SDTCisVT<1, i32>]>;
 
-def SDT_ARMBr2JT   : SDTypeProfile<0, 4,
+def SDT_ARMBr2JT   : SDTypeProfile<0, 3,
                                   [SDTCisPtrTy<0>, SDTCisVT<1, i32>,
-                                   SDTCisVT<2, i32>, SDTCisVT<3, i32>]>;
+                                   SDTCisVT<2, i32>]>;
 
 def SDT_ARMBCC_i64 : SDTypeProfile<0, 6,
                                   [SDTCisVT<0, i32>,
@@ -60,6 +59,7 @@ def SDT_ARMThreadPointer : SDTypeProfile<1, 0, [SDTCisPtrTy<0>]>;
 def SDT_ARMEH_SJLJ_Setjmp : SDTypeProfile<1, 2, [SDTCisInt<0>, SDTCisPtrTy<1>,
                                                  SDTCisInt<2>]>;
 def SDT_ARMEH_SJLJ_Longjmp: SDTypeProfile<0, 2, [SDTCisPtrTy<0>, SDTCisInt<1>]>;
+def SDT_ARMEH_SJLJ_SetupDispatch: SDTypeProfile<0, 0, []>;
 
 def SDT_ARMMEMBARRIER     : SDTypeProfile<0, 1, [SDTCisInt<0>]>;
 
@@ -96,15 +96,7 @@ def ARMSmlal         : SDNode<"ARMISD::SMLAL", SDT_ARM64bitmlal>;
 // Node definitions.
 def ARMWrapper       : SDNode<"ARMISD::Wrapper",     SDTIntUnaryOp>;
 def ARMWrapperPIC    : SDNode<"ARMISD::WrapperPIC",  SDTIntUnaryOp>;
-def ARMWrapperJT     : SDNode<"ARMISD::WrapperJT",   SDTIntBinOp>;
-// @LOCALMOD-START
-// support non-inline jumptables
-// we do not use the extre uid immediate that comes with ARMWrapperJT
-// TODO(robertm): figure out what it is used for
-def ARMWrapperJT2    : SDNode<"ARMISD::WrapperJT2",  SDTIntUnaryOp>;
-// Support for MOVW/MOVT'ing the GOT address directly into a register.
-def ARMWrapperGOT       : SDNode<"ARMISD::WrapperGOT",  SDTPtrLeaf>;
-// @LOCALMOD-END
+def ARMWrapperJT     : SDNode<"ARMISD::WrapperJT",   SDTIntUnaryOp>;
 
 def ARMcallseq_start : SDNode<"ISD::CALLSEQ_START", SDT_ARMCallSeqStart,
                               [SDNPHasChain, SDNPSideEffect, SDNPOutGlue]>;
@@ -172,6 +164,9 @@ def ARMeh_sjlj_setjmp: SDNode<"ARMISD::EH_SJLJ_SETJMP",
 def ARMeh_sjlj_longjmp: SDNode<"ARMISD::EH_SJLJ_LONGJMP",
                                SDT_ARMEH_SJLJ_Longjmp,
                                [SDNPHasChain, SDNPSideEffect]>;
+def ARMeh_sjlj_setup_dispatch: SDNode<"ARMISD::EH_SJLJ_SETUP_DISPATCH",
+                                      SDT_ARMEH_SJLJ_SetupDispatch,
+                                      [SDNPHasChain, SDNPSideEffect]>;
 
 def ARMMemBarrierMCR  : SDNode<"ARMISD::MEMBARRIER_MCR", SDT_ARMMEMBARRIER,
                                [SDNPHasChain, SDNPSideEffect]>;
@@ -279,7 +274,6 @@ def IsARM            : Predicate<"!Subtarget->isThumb()">,
 def IsMachO          : Predicate<"Subtarget->isTargetMachO()">;
 def IsNotMachO       : Predicate<"!Subtarget->isTargetMachO()">;
 def IsNaCl           : Predicate<"Subtarget->isTargetNaCl()">;
-def IsNotNaCl        : Predicate<"!Subtarget->isTargetNaCl()">; // @LOCALMOD
 def UseNaClTrap      : Predicate<"Subtarget->useNaClTrap()">,
                                  AssemblerPredicate<"FeatureNaClTrap", "NaCl">;
 def DontUseNaClTrap  : Predicate<"!Subtarget->useNaClTrap()">;
@@ -316,13 +310,8 @@ def HasSlowVDUP32 : Predicate<"Subtarget->isSwift()">;
 def UseVMOVSR : Predicate<"Subtarget->isCortexA9() || !Subtarget->useNEONForSinglePrecisionFP()">;
 def DontUseVMOVSR : Predicate<"!Subtarget->isCortexA9() && Subtarget->useNEONForSinglePrecisionFP()">;
 
-def IsLE             : Predicate<"getTargetLowering()->isLittleEndian()">;
-def IsBE             : Predicate<"getTargetLowering()->isBigEndian()">;
-
-// @LOCALMOD-BEGIN
-def UseConstIslands : Predicate<"Subtarget->useConstIslands()">;
-def DontUseConstIslands : Predicate<"!Subtarget->useConstIslands()">;
-// @LOCALMOD-END
+def IsLE             : Predicate<"MF->getDataLayout().isLittleEndian()">;
+def IsBE             : Predicate<"MF->getDataLayout().isBigEndian()">;
 
 //===----------------------------------------------------------------------===//
 // ARM Flag Definitions.
@@ -337,12 +326,12 @@ class RegConstraint<string C> {
 
 // imm_neg_XFORM - Return the negation of an i32 immediate value.
 def imm_neg_XFORM : SDNodeXForm<imm, [{
-  return CurDAG->getTargetConstant(-(int)N->getZExtValue(), MVT::i32);
+  return CurDAG->getTargetConstant(-(int)N->getZExtValue(), SDLoc(N), MVT::i32);
 }]>;
 
 // imm_not_XFORM - Return the complement of a i32 immediate value.
 def imm_not_XFORM : SDNodeXForm<imm, [{
-  return CurDAG->getTargetConstant(~(int)N->getZExtValue(), MVT::i32);
+  return CurDAG->getTargetConstant(~(int)N->getZExtValue(), SDLoc(N), MVT::i32);
 }]>;
 
 /// imm16_31 predicate - True if the 32-bit immediate is in the range [16,31].
@@ -357,7 +346,8 @@ def sext_16_node : PatLeaf<(i32 GPR:$a), [{
 
 /// Split a 32-bit immediate into two 16 bit parts.
 def hi16 : SDNodeXForm<imm, [{
-  return CurDAG->getTargetConstant((uint32_t)N->getZExtValue() >> 16, MVT::i32);
+  return CurDAG->getTargetConstant((uint32_t)N->getZExtValue() >> 16, SDLoc(N),
+                                   MVT::i32);
 }]>;
 
 def lo16AllZero : PatLeaf<(i32 imm), [{
@@ -499,10 +489,10 @@ def neon_vcvt_imm32 : Operand<i32> {
 def rot_imm_XFORM: SDNodeXForm<imm, [{
   switch (N->getZExtValue()){
   default: llvm_unreachable(nullptr);
-  case 0:  return CurDAG->getTargetConstant(0, MVT::i32);
-  case 8:  return CurDAG->getTargetConstant(1, MVT::i32);
-  case 16: return CurDAG->getTargetConstant(2, MVT::i32);
-  case 24: return CurDAG->getTargetConstant(3, MVT::i32);
+  case 0:  return CurDAG->getTargetConstant(0, SDLoc(N), MVT::i32);
+  case 8:  return CurDAG->getTargetConstant(1, SDLoc(N), MVT::i32);
+  case 16: return CurDAG->getTargetConstant(2, SDLoc(N), MVT::i32);
+  case 24: return CurDAG->getTargetConstant(3, SDLoc(N), MVT::i32);
   }
 }]>;
 def RotImmAsmOperand : AsmOperandClass {
@@ -781,7 +771,8 @@ def bf_inv_mask_imm : Operand<i32>,
 }
 
 def imm1_32_XFORM: SDNodeXForm<imm, [{
-  return CurDAG->getTargetConstant((int)N->getZExtValue() - 1, MVT::i32);
+  return CurDAG->getTargetConstant((int)N->getZExtValue() - 1, SDLoc(N),
+                                   MVT::i32);
 }]>;
 def Imm1_32AsmOperand: AsmOperandClass { let Name = "Imm1_32"; }
 def imm1_32 : Operand<i32>, PatLeaf<(imm), [{
@@ -794,7 +785,8 @@ def imm1_32 : Operand<i32>, PatLeaf<(imm), [{
 }
 
 def imm1_16_XFORM: SDNodeXForm<imm, [{
-  return CurDAG->getTargetConstant((int)N->getZExtValue() - 1, MVT::i32);
+  return CurDAG->getTargetConstant((int)N->getZExtValue() - 1, SDLoc(N),
+                                   MVT::i32);
 }]>;
 def Imm1_16AsmOperand: AsmOperandClass { let Name = "Imm1_16"; }
 def imm1_16 : Operand<i32>, PatLeaf<(imm), [{ return Imm > 0 && Imm <= 16; }],
@@ -887,8 +879,7 @@ def postidx_reg : MemOperand {
 // use explicit imm vs. reg versions above (addrmode_imm12 and ldst_so_reg).
 def AddrMode2AsmOperand : AsmOperandClass { let Name = "AddrMode2"; }
 def addrmode2 : MemOperand,
-                ComplexPattern<i32, 3, "SelectAddrMode2", [],
-                               [SDNPWantRoot]> { // @LOCALMOD
+                ComplexPattern<i32, 3, "SelectAddrMode2", []> {
   let EncoderMethod = "getAddrMode2OpValue";
   let PrintMethod = "printAddrMode2Operand";
   let ParserMatchClass = AddrMode2AsmOperand;
@@ -928,8 +919,7 @@ def am2offset_imm : MemOperand,
 // FIXME: split into imm vs. reg versions.
 def AddrMode3AsmOperand : AsmOperandClass { let Name = "AddrMode3"; }
 class AddrMode3 : MemOperand,
-                ComplexPattern<i32, 3, "SelectAddrMode3", [],
-                               [SDNPWantRoot]> { // @LOCALMOD
+                  ComplexPattern<i32, 3, "SelectAddrMode3", []> {
   let EncoderMethod = "getAddrMode3OpValue";
   let ParserMatchClass = AddrMode3AsmOperand;
   let MIOperandInfo = (ops GPR:$base, GPR:$offsreg, i32imm:$offsimm);
@@ -1827,41 +1817,6 @@ multiclass AI_str1nopc<bit isByte, string opc, InstrItinClass iii,
 // Instructions
 //===----------------------------------------------------------------------===//
 
-// @LOCALMOD-START
-
-def SFI_GUARD_LOADSTORE :
-PseudoInst<(outs GPR:$dst), (ins GPR:$a, pred:$p), NoItinerary, []>;
-
-let Defs = [CPSR] in
-def SFI_GUARD_LOADSTORE_TST :
-PseudoInst<(outs), (ins GPR:$a), NoItinerary, []>;
-
-// Like SFI_GUARD_LOADSTORE, but reserved for loads into SP.
-def SFI_GUARD_SP_LOAD :
-PseudoInst<(outs GPR:$dst), (ins GPR:$src, pred:$p), NoItinerary, []>;
-
-def SFI_GUARD_INDIRECT_CALL :
-PseudoInst<(outs GPR:$dst), (ins GPR:$a, pred:$p), NoItinerary, []>;
-
-def SFI_GUARD_INDIRECT_JMP :
-PseudoInst<(outs GPR:$dst), (ins GPR:$a, pred:$p), NoItinerary, []>;
-
-def SFI_GUARD_CALL :
-PseudoInst<(outs), (ins pred:$p), NoItinerary, []>;
-
-// NOTE: the BX_RET instruction hardcodes lr as well
-def SFI_GUARD_RETURN :
-PseudoInst<(outs), (ins pred:$p), NoItinerary, []>;
-
-def SFI_NOP_IF_AT_BUNDLE_END :
-PseudoInst<(outs), (ins), NoItinerary, []>;
-
-// Note: intention is that $src and $dst are the same register.
-def SFI_DATA_MASK :
-PseudoInst<(outs GPR:$dst), (ins GPR:$src, pred:$p), NoItinerary, []>;
-
-// @LOCALMOD-END
-
 //===----------------------------------------------------------------------===//
 //  Miscellaneous Instructions.
 //
@@ -1875,6 +1830,32 @@ def CONSTPOOL_ENTRY :
 PseudoInst<(outs), (ins cpinst_operand:$instid, cpinst_operand:$cpidx,
                     i32imm:$size), NoItinerary, []>;
 
+/// A jumptable consisting of direct 32-bit addresses of the destination basic
+/// blocks (either absolute, or relative to the start of the jump-table in PIC
+/// mode). Used mostly in ARM and Thumb-1 modes.
+def JUMPTABLE_ADDRS :
+PseudoInst<(outs), (ins cpinst_operand:$instid, cpinst_operand:$cpidx,
+                        i32imm:$size), NoItinerary, []>;
+
+/// A jumptable consisting of 32-bit jump instructions. Used for Thumb-2 tables
+/// that cannot be optimised to use TBB or TBH.
+def JUMPTABLE_INSTS :
+PseudoInst<(outs), (ins cpinst_operand:$instid, cpinst_operand:$cpidx,
+                        i32imm:$size), NoItinerary, []>;
+
+/// A jumptable consisting of 8-bit unsigned integers representing offsets from
+/// a TBB instruction.
+def JUMPTABLE_TBB :
+PseudoInst<(outs), (ins cpinst_operand:$instid, cpinst_operand:$cpidx,
+                        i32imm:$size), NoItinerary, []>;
+
+/// A jumptable consisting of 16-bit unsigned integers representing offsets from
+/// a TBH instruction.
+def JUMPTABLE_TBH :
+PseudoInst<(outs), (ins cpinst_operand:$instid, cpinst_operand:$cpidx,
+                        i32imm:$size), NoItinerary, []>;
+
+
 // FIXME: Marking these as hasSideEffects is necessary to prevent machine DCE
 // from removing one half of the matched pairs. That breaks PEI, which assumes
 // these will always be in pairs, and asserts if it finds otherwise. Better way?
@@ -2138,7 +2119,7 @@ def LEApcrel : ARMPseudoInst<(outs GPR:$Rd), (ins i32imm:$label, pred:$p),
                     4, IIC_iALUi, []>, Sched<[WriteALU, ReadALU]>;
 
 def LEApcrelJT : ARMPseudoInst<(outs GPR:$Rd),
-                      (ins i32imm:$label, nohash_imm:$id, pred:$p),
+                      (ins i32imm:$label, pred:$p),
                       4, IIC_iALUi, []>, Sched<[WriteALU, ReadALU]>;
 }
 
@@ -2190,33 +2171,6 @@ let isBranch = 1, isTerminator = 1, isBarrier = 1, isIndirectBranch = 1 in {
 
 // SP is marked as a use to prevent stack-pointer assignments that appear
 // immediately before calls from potentially appearing dead.
-
-// @LOCALMOD-START
-// Exception handling related Node and Instructions.
-// The conversion sequence is:
-// ISD::EH_RETURN -> ARMISD::EH_RETURN ->  
-// ARMeh_return -> (stack change + indirect branch)
-// 
-// ARMeh_return takes the place of regular return instruction
-// but takes two arguments.
-// R2, R3 are used for storing the offset and return address respectively.
-def SDT_ARMEHRET : SDTypeProfile<0, 2, [SDTCisInt<0>, SDTCisPtrTy<1>]>;
-
-def ARMehret : SDNode<"ARMISD::EH_RETURN", SDT_ARMEHRET,
-                      [SDNPHasChain, SDNPOptInGlue]>;
-
-
-let isTerminator = 1, isReturn = 1, isBarrier = 1,
-  Defs = [SP],
-  Uses = [SP] in {
-    def ARMeh_return : PseudoInst<(outs), 
-                               (ins GPR:$spadj, GPR:$dst),
-                               IIC_Br,
-                               [(ARMehret GPR:$spadj, GPR:$dst)]>,
-                    Requires<[IsARM]>; 
-}
-// @LOCALMOD-END
-
 let isCall = 1,
   // FIXME:  Do we really need a non-predicated version? If so, it should
   // at least be a pseudo instruction expanding to the predicated version
@@ -2300,24 +2254,24 @@ let isBranch = 1, isTerminator = 1 in {
                 [(br bb:$target)], (Bcc br_target:$target, (ops 14, zero_reg))>,
                 Sched<[WriteBr]>;
 
-    let isNotDuplicable = 1, isIndirectBranch = 1 in {
+    let Size = 4, isNotDuplicable = 1, isIndirectBranch = 1 in {
     def BR_JTr : ARMPseudoInst<(outs),
-                      (ins GPR:$target, i32imm:$jt, i32imm:$id),
+                      (ins GPR:$target, i32imm:$jt),
                       0, IIC_Br,
-                      [(ARMbrjt GPR:$target, tjumptable:$jt, imm:$id)]>,
+                      [(ARMbrjt GPR:$target, tjumptable:$jt)]>,
                       Sched<[WriteBr]>;
     // FIXME: This shouldn't use the generic "addrmode2," but rather be split
     // into i12 and rs suffixed versions.
     def BR_JTm : ARMPseudoInst<(outs),
-                     (ins addrmode2:$target, i32imm:$jt, i32imm:$id),
+                     (ins addrmode2:$target, i32imm:$jt),
                      0, IIC_Br,
-                     [(ARMbrjt (i32 (load addrmode2:$target)), tjumptable:$jt,
-                       imm:$id)]>, Sched<[WriteBrTbl]>;
+                     [(ARMbrjt (i32 (load addrmode2:$target)),
+                               tjumptable:$jt)]>, Sched<[WriteBrTbl]>;
     def BR_JTadd : ARMPseudoInst<(outs),
-                   (ins GPR:$target, GPR:$idx, i32imm:$jt, i32imm:$id),
+                   (ins GPR:$target, GPR:$idx, i32imm:$jt),
                    0, IIC_Br,
-                   [(ARMbrjt (add GPR:$target, GPR:$idx), tjumptable:$jt,
-                     imm:$id)]>, Sched<[WriteBrTbl]>;
+                   [(ARMbrjt (add GPR:$target, GPR:$idx), tjumptable:$jt)]>,
+                   Sched<[WriteBrTbl]>;
     } // isNotDuplicable = 1, isIndirectBranch = 1
   } // isBarrier = 1
 
@@ -2331,6 +2285,7 @@ def BLXi : AXI<(outs), (ins blx_target:$target), BrMiscFrm, NoItinerary,
   bits<25> target;
   let Inst{23-0} = target{24-1};
   let Inst{24} = target{0};
+  let isCall = 1;
 }
 
 // Branch and Exchange Jazelle
@@ -2341,6 +2296,7 @@ def BXJ : ABI<0b0001, (outs), (ins GPR:$func), NoItinerary, "bxj", "\t$func",
   let Inst{19-8} = 0xfff;
   let Inst{7-4} = 0b0010;
   let Inst{3-0} = func;
+  let isBranch = 1;
 }
 
 // Tail calls.
@@ -3369,69 +3325,6 @@ def MOVTi16_ga_pcrel : PseudoInst<(outs GPR:$Rd),
 
 } // Constraints
 
-// @LOCALMOD-BEGIN
-// PIC / PC-relative versions of MOVi16/MOVTi16, which have an extra 
-// operand representing the ID of the PICADD instruction that corrects
-// for relativity. This is used to materialize addresses into
-// a register in a PC-relative manner.
-// 
-// E.g. Rather than have an absolute address in $imm, and transferred to 
-// a register with:
-//    movw $Rd, :lower16:$imm
-//    movt $Rd, :upper16:$imm
-//
-// we will instead have a relative offset:
-//    movw $Rd, :lower16:$imm - ($pic_add_id + 8)
-//    ...
-//    movt $Rd, :upper16:$imm - ($pic_add_id + 8)
-//    ...
-// $pic_add_id:
-//    add  $Rd, pc, $Rd
-//
-// One way these pseudo instructions (and the corresponding PICADD) 
-// come about is during expansion of the MOVi32imm pseudo instruction
-// (see ARMExpandPseudo::ExpandMBB).
-// These pseudo instructions become real instructions when they are
-// finally lowered to MCInsts (e.g., at ARMAsmPrinter::EmitInstruction),
-// and the extra pclabel ID becomes part of the appropriate operand.
-//
-// NOTE: aside from adding the pclabel operand, all other operands should
-// be the same as the non-PIC versions to simplify conversion to the 
-// non-pseudo instructions.
-let isReMaterializable = 1, isAsCheapAsAMove = 1, isMoveImm = 1,
-    hasSideEffects = 0 in
-def MOVi16PIC : PseudoInst<(outs GPR:$Rd), (ins imm0_65535_expr:$imm,
-                                                pclabel:$pic_add_id,
-                                                pred:$p),
-                           IIC_iMOVi,
-                           []>,
-                 Requires<[IsARM, HasV6T2]>, UnaryDP;
-
-let Constraints = "$src = $Rd" in
-def MOVTi16PIC : PseudoInst<(outs GPR:$Rd), (ins GPR:$src,
-                                                 imm0_65535_expr:$imm,
-                                                 pclabel:$pic_add_id,
-                                                 pred:$p),
-                            IIC_iMOVi,
-                            []>,
-                 UnaryDP, Requires<[IsARM, HasV6T2]>;
-// @LOCALMOD-END
-
-// @LOCALMOD-BEGIN
-// Pseudo-instruction that will be expanded into MOVW / MOVT (PIC versions) w/
-// GOT as the operand. 
-// The alternative is to create a constant pool entry with the (relative)
-// GOT address and load from the constant pool. This is currently used
-// when constant islands are turned off, since MOVW / MOVT will be faster.
-let isReMaterializable = 1, isMoveImm = 1, hasSideEffects = 0 in
-def MOVGOTAddr : PseudoInst<(outs GPR:$dst), (ins),
-                           IIC_iMOVix2, // will expand to two MOVi's
-                           []>,
-                           Requires<[IsARM, UseMovt]>;
-
-def : ARMPat<(ARMWrapperGOT), (MOVGOTAddr)>;
-// @LOCALMOD-END
-
 def : ARMPat<(or GPR:$src, 0xffff0000), (MOVTi16 GPR:$src, 0xffff)>,
       Requires<[IsARM, HasV6T2]>;
 
@@ -4314,11 +4207,9 @@ def REV16 : AMiscA1I<0b01101011, 0b1011, (outs GPR:$Rd), (ins GPR:$Rm),
            Sched<[WriteALU]>;
 
 def : ARMV6Pat<(srl (bswap (extloadi16 addrmode3:$addr)), (i32 16)),
-              (REV16 (LDRH addrmode3:$addr))>,
-              Requires<[IsARM, HasV6, IsNotNaCl]>; // @LOCALMOD
+              (REV16 (LDRH addrmode3:$addr))>;
 def : ARMV6Pat<(truncstorei16 (srl (bswap GPR:$Rn), (i32 16)), addrmode3:$addr),
-               (STRH (REV16 GPR:$Rn), addrmode3:$addr)>,
-               Requires<[IsARM, HasV6, IsNotNaCl]>; // @LOCALMOD
+               (STRH (REV16 GPR:$Rn), addrmode3:$addr)>;
 
 let AddedComplexity = 5 in
 def REVSH : AMiscA1I<0b01101111, 0b1011, (outs GPR:$Rd), (ins GPR:$Rm),
@@ -5178,10 +5069,11 @@ def : ARMV5TPat<(int_arm_mrc2 imm:$cop, imm:$opc1, imm:$CRn,
                               imm:$CRm, imm:$opc2),
                 (MRC2 imm:$cop, imm:$opc1, imm:$CRn, imm:$CRm, imm:$opc2)>;
 
-class MovRRCopro<string opc, bit direction, list<dag> pattern = []>
-  : ABI<0b1100, (outs), (ins p_imm:$cop, imm0_15:$opc1,
-        GPRnopc:$Rt, GPRnopc:$Rt2, c_imm:$CRm),
-        NoItinerary, opc, "\t$cop, $opc1, $Rt, $Rt2, $CRm", pattern> {
+class MovRRCopro<string opc, bit direction, dag oops, dag iops, list<dag>
+                 pattern = []>
+  : ABI<0b1100, oops, iops, NoItinerary, opc, "\t$cop, $opc1, $Rt, $Rt2, $CRm",
+        pattern> {
+
   let Inst{23-21} = 0b010;
   let Inst{20} = direction;
 
@@ -5199,9 +5091,13 @@ class MovRRCopro<string opc, bit direction, list<dag> pattern = []>
 }
 
 def MCRR : MovRRCopro<"mcrr", 0 /* from ARM core register to coprocessor */,
+                      (outs), (ins p_imm:$cop, imm0_15:$opc1, GPRnopc:$Rt,
+                      GPRnopc:$Rt2, c_imm:$CRm),
                       [(int_arm_mcrr imm:$cop, imm:$opc1, GPRnopc:$Rt,
                                      GPRnopc:$Rt2, imm:$CRm)]>;
-def MRRC : MovRRCopro<"mrrc", 1 /* from coprocessor to ARM core register */>;
+def MRRC : MovRRCopro<"mrrc", 1 /* from coprocessor to ARM core register */,
+                      (outs GPRnopc:$Rt, GPRnopc:$Rt2),
+                      (ins p_imm:$cop, imm0_15:$opc1, c_imm:$CRm), []>;
 
 class MovRRCopro2<string opc, bit direction, list<dag> pattern = []>
   : ABXI<0b1100, (outs), (ins p_imm:$cop, imm0_15:$opc1,
@@ -5409,6 +5305,10 @@ def Int_eh_sjlj_longjmp : PseudoInst<(outs), (ins GPR:$src, GPR:$scratch),
                                 Requires<[IsARM]>;
 }
 
+let isBarrier = 1, hasSideEffects = 1, usesCustomInserter = 1 in
+def Int_eh_sjlj_setup_dispatch : PseudoInst<(outs), (ins), NoItinerary,
+            [(ARMeh_sjlj_setup_dispatch)]>;
+
 // eh.sjlj.dispatchsetup pseudo-instruction.
 // This pseudo is used for both ARM and Thumb. Any differences are handled when
 // the pseudo is expanded (which happens before any passes that need the
@@ -5439,19 +5339,6 @@ def MOVi32imm : PseudoInst<(outs GPR:$dst), (ins i32imm:$src), IIC_iMOVix2,
                            [(set GPR:$dst, (arm_i32imm:$src))]>,
                            Requires<[IsARM]>;
 
-// @LOCALMOD-START
-// MOVi32imm doesn't get a fast-isel because it's not a leaf pattern (it uses
-// arm_i32imm which differs based on whether UseMovt is true.) This means
-// ARMFastISel::ARMMaterializeInt fails when useConstIslands is disabled, and
-// ARMFastISel::ARMSimplifyAddress can't bail out of fast isel. For upstreaming
-// we could probably just split MOVi32imm directly instead of duplicating it,
-// but this is cleaner as a localmod.
-let isReMaterializable = 1, isMoveImm = 1 in
-def MOVti32imm : PseudoInst<(outs GPR:$dst), (ins i32imm:$src), IIC_iMOVix2,
-                           [(set GPR:$dst, (i32 imm:$src))]>,
-                           Requires<[IsARM, DontUseConstIslands]>;
-// @LOCALMOD-END
-
 def LDRLIT_ga_abs : PseudoInst<(outs GPR:$dst), (ins i32imm:$src), IIC_iLoad_i,
                                [(set GPR:$dst, (ARMWrapper tglobaladdr:$src))]>,
                     Requires<[IsARM, DontUseMovt]>;
@@ -5487,19 +5374,11 @@ def MOV_ga_pcrel_ldr : PseudoInst<(outs GPR:$dst), (ins i32imm:$addr),
 } // isReMaterializable
 
 // ConstantPool, GlobalAddress, and JumpTable
-def : ARMPat<(ARMWrapper  tconstpool  :$dst), (LEApcrel tconstpool  :$dst)>,
-            Requires<[IsARM, UseConstIslands]>; // @LOCALMOD see tconstpool
-                                                // with DontUseConstIslands.
+def : ARMPat<(ARMWrapper  tconstpool  :$dst), (LEApcrel tconstpool  :$dst)>;
 def : ARMPat<(ARMWrapper  tglobaladdr :$dst), (MOVi32imm tglobaladdr :$dst)>,
             Requires<[IsARM, UseMovt]>;
-// @LOCALMOD-START
-def : ARMPat<(ARMWrapper  tconstpool :$dst), (MOVi32imm tconstpool :$dst)>,
-            Requires<[IsARM, UseMovt, DontUseConstIslands]>;
-def : ARMPat<(ARMWrapperJT2  tjumptable :$dst), (MOVi32imm tjumptable :$dst)>,
-            Requires<[IsARM, UseMovt]>;
-// @LOCALMOD-END
-def : ARMPat<(ARMWrapperJT tjumptable:$dst, imm:$id),
-             (LEApcrelJT tjumptable:$dst, imm:$id)>;
+def : ARMPat<(ARMWrapperJT tjumptable:$dst),
+             (LEApcrelJT tjumptable:$dst)>;
 
 // TODO: add,sub,and, 3-instr forms?
 
diff --git a/lib/Target/ARM/ARMInstrNEON.td b/lib/Target/ARM/ARMInstrNEON.td
index a6a07a8f02e..3f47c59846d 100644
--- a/lib/Target/ARM/ARMInstrNEON.td
+++ b/lib/Target/ARM/ARMInstrNEON.td
@@ -2393,36 +2393,41 @@ def : Pat<(byte_alignedstore (v2f64 QPR:$value), addrmode6:$addr),
 // Extract D sub-registers of Q registers.
 def DSubReg_i8_reg  : SDNodeXForm<imm, [{
   assert(ARM::dsub_7 == ARM::dsub_0+7 && "Unexpected subreg numbering");
-  return CurDAG->getTargetConstant(ARM::dsub_0 + N->getZExtValue()/8, MVT::i32);
+  return CurDAG->getTargetConstant(ARM::dsub_0 + N->getZExtValue()/8, SDLoc(N),
+                                   MVT::i32);
 }]>;
 def DSubReg_i16_reg : SDNodeXForm<imm, [{
   assert(ARM::dsub_7 == ARM::dsub_0+7 && "Unexpected subreg numbering");
-  return CurDAG->getTargetConstant(ARM::dsub_0 + N->getZExtValue()/4, MVT::i32);
+  return CurDAG->getTargetConstant(ARM::dsub_0 + N->getZExtValue()/4, SDLoc(N),
+                                   MVT::i32);
 }]>;
 def DSubReg_i32_reg : SDNodeXForm<imm, [{
   assert(ARM::dsub_7 == ARM::dsub_0+7 && "Unexpected subreg numbering");
-  return CurDAG->getTargetConstant(ARM::dsub_0 + N->getZExtValue()/2, MVT::i32);
+  return CurDAG->getTargetConstant(ARM::dsub_0 + N->getZExtValue()/2, SDLoc(N),
+                                   MVT::i32);
 }]>;
 def DSubReg_f64_reg : SDNodeXForm<imm, [{
   assert(ARM::dsub_7 == ARM::dsub_0+7 && "Unexpected subreg numbering");
-  return CurDAG->getTargetConstant(ARM::dsub_0 + N->getZExtValue(), MVT::i32);
+  return CurDAG->getTargetConstant(ARM::dsub_0 + N->getZExtValue(), SDLoc(N),
+                                   MVT::i32);
 }]>;
 
 // Extract S sub-registers of Q/D registers.
 def SSubReg_f32_reg : SDNodeXForm<imm, [{
   assert(ARM::ssub_3 == ARM::ssub_0+3 && "Unexpected subreg numbering");
-  return CurDAG->getTargetConstant(ARM::ssub_0 + N->getZExtValue(), MVT::i32);
+  return CurDAG->getTargetConstant(ARM::ssub_0 + N->getZExtValue(), SDLoc(N),
+                                   MVT::i32);
 }]>;
 
 // Translate lane numbers from Q registers to D subregs.
 def SubReg_i8_lane  : SDNodeXForm<imm, [{
-  return CurDAG->getTargetConstant(N->getZExtValue() & 7, MVT::i32);
+  return CurDAG->getTargetConstant(N->getZExtValue() & 7, SDLoc(N), MVT::i32);
 }]>;
 def SubReg_i16_lane : SDNodeXForm<imm, [{
-  return CurDAG->getTargetConstant(N->getZExtValue() & 3, MVT::i32);
+  return CurDAG->getTargetConstant(N->getZExtValue() & 3, SDLoc(N), MVT::i32);
 }]>;
 def SubReg_i32_lane : SDNodeXForm<imm, [{
-  return CurDAG->getTargetConstant(N->getZExtValue() & 1, MVT::i32);
+  return CurDAG->getTargetConstant(N->getZExtValue() & 1, SDLoc(N), MVT::i32);
 }]>;
 
 //===----------------------------------------------------------------------===//
@@ -4994,10 +4999,10 @@ def  VBITq    : N3VX<1, 0, 0b10, 0b0001, 1, 1,
 //   VABD     : Vector Absolute Difference
 defm VABDs    : N3VInt_QHS<0, 0, 0b0111, 0, N3RegFrm,
                            IIC_VSUBi4D, IIC_VSUBi4D, IIC_VSUBi4Q, IIC_VSUBi4Q,
-                           "vabd", "s", int_arm_neon_vabds, 1>;
+                           "vabd", "s", sabsdiff, 1>;
 defm VABDu    : N3VInt_QHS<1, 0, 0b0111, 0, N3RegFrm,
                            IIC_VSUBi4D, IIC_VSUBi4D, IIC_VSUBi4Q, IIC_VSUBi4Q,
-                           "vabd", "u", int_arm_neon_vabdu, 1>;
+                           "vabd", "u", uabsdiff, 1>;
 def  VABDfd   : N3VDInt<1, 0, 0b10, 0b1101, 0, N3RegFrm, IIC_VBIND,
                         "vabd", "f32", v2f32, v2f32, int_arm_neon_vabds, 1>;
 def  VABDfq   : N3VQInt<1, 0, 0b10, 0b1101, 0, N3RegFrm, IIC_VBINQ,
@@ -5005,21 +5010,21 @@ def  VABDfq   : N3VQInt<1, 0, 0b10, 0b1101, 0, N3RegFrm, IIC_VBINQ,
 
 //   VABDL    : Vector Absolute Difference Long (Q = | D - D |)
 defm VABDLs   : N3VLIntExt_QHS<0,1,0b0111,0, IIC_VSUBi4Q,
-                               "vabdl", "s", int_arm_neon_vabds, zext, 1>;
+                               "vabdl", "s", sabsdiff, zext, 1>;
 defm VABDLu   : N3VLIntExt_QHS<1,1,0b0111,0, IIC_VSUBi4Q,
-                               "vabdl", "u", int_arm_neon_vabdu, zext, 1>;
+                               "vabdl", "u", uabsdiff, zext, 1>;
 
 //   VABA     : Vector Absolute Difference and Accumulate
 defm VABAs    : N3VIntOp_QHS<0,0,0b0111,1, IIC_VABAD, IIC_VABAQ,
-                             "vaba", "s", int_arm_neon_vabds, add>;
+                             "vaba", "s", sabsdiff, add>;
 defm VABAu    : N3VIntOp_QHS<1,0,0b0111,1, IIC_VABAD, IIC_VABAQ,
-                             "vaba", "u", int_arm_neon_vabdu, add>;
+                             "vaba", "u", uabsdiff, add>;
 
 //   VABAL    : Vector Absolute Difference and Accumulate Long (Q += | D - D |)
 defm VABALs   : N3VLIntExtOp_QHS<0,1,0b0101,0, IIC_VABAD,
-                                 "vabal", "s", int_arm_neon_vabds, zext, add>;
+                                 "vabal", "s", sabsdiff, zext, add>;
 defm VABALu   : N3VLIntExtOp_QHS<1,1,0b0101,0, IIC_VABAD,
-                                 "vabal", "u", int_arm_neon_vabdu, zext, add>;
+                                 "vabal", "u", uabsdiff, zext, add>;
 
 // Vector Maximum and Minimum.
 
diff --git a/lib/Target/ARM/ARMInstrThumb.td b/lib/Target/ARM/ARMInstrThumb.td
index d0ade771b5e..40414da3ca8 100644
--- a/lib/Target/ARM/ARMInstrThumb.td
+++ b/lib/Target/ARM/ARMInstrThumb.td
@@ -21,7 +21,7 @@ def ARMtcall : SDNode<"ARMISD::tCALL", SDT_ARMcall,
 
 def imm_sr_XFORM: SDNodeXForm<imm, [{
   unsigned Imm = N->getZExtValue();
-  return CurDAG->getTargetConstant((Imm == 32 ? 0 : Imm), MVT::i32);
+  return CurDAG->getTargetConstant((Imm == 32 ? 0 : Imm), SDLoc(N), MVT::i32);
 }]>;
 def ThumbSRImmAsmOperand: AsmOperandClass { let Name = "ImmThumbSR"; }
 def imm_sr : Operand<i32>, PatLeaf<(imm), [{
@@ -33,7 +33,8 @@ def imm_sr : Operand<i32>, PatLeaf<(imm), [{
 }
 
 def imm_comp_XFORM : SDNodeXForm<imm, [{
-  return CurDAG->getTargetConstant(~((uint32_t)N->getZExtValue()), MVT::i32);
+  return CurDAG->getTargetConstant(~((uint32_t)N->getZExtValue()), SDLoc(N),
+                                   MVT::i32);
 }]>;
 
 def imm0_7_neg : PatLeaf<(i32 imm), [{
@@ -61,12 +62,12 @@ def thumb_immshifted : PatLeaf<(imm), [{
 
 def thumb_immshifted_val : SDNodeXForm<imm, [{
   unsigned V = ARM_AM::getThumbImmNonShiftedVal((unsigned)N->getZExtValue());
-  return CurDAG->getTargetConstant(V, MVT::i32);
+  return CurDAG->getTargetConstant(V, SDLoc(N), MVT::i32);
 }]>;
 
 def thumb_immshifted_shamt : SDNodeXForm<imm, [{
   unsigned V = ARM_AM::getThumbImmValShift((unsigned)N->getZExtValue());
-  return CurDAG->getTargetConstant(V, MVT::i32);
+  return CurDAG->getTargetConstant(V, SDLoc(N), MVT::i32);
 }]>;
 
 // Scaled 4 immediate.
@@ -521,10 +522,11 @@ let isBranch = 1, isTerminator = 1, isBarrier = 1 in {
                           Sched<[WriteBrTbl]>;
 
   def tBR_JTr : tPseudoInst<(outs),
-                      (ins tGPR:$target, i32imm:$jt, i32imm:$id),
+                      (ins tGPR:$target, i32imm:$jt),
                       0, IIC_Br,
-                      [(ARMbrjt tGPR:$target, tjumptable:$jt, imm:$id)]>,
+                      [(ARMbrjt tGPR:$target, tjumptable:$jt)]>,
                       Sched<[WriteBrTbl]> {
+    let Size = 2;
     list<Predicate> Predicates = [IsThumb, IsThumb1Only];
   }
 }
@@ -1254,7 +1256,7 @@ def tLEApcrel   : tPseudoInst<(outs tGPR:$Rd), (ins i32imm:$label, pred:$p),
 
 let hasSideEffects = 1 in
 def tLEApcrelJT : tPseudoInst<(outs tGPR:$Rd),
-                              (ins i32imm:$label, nohash_imm:$id, pred:$p),
+                              (ins i32imm:$label, pred:$p),
                               2, IIC_iALUi, []>, Sched<[WriteALU]>;
 
 //===----------------------------------------------------------------------===//
@@ -1355,8 +1357,8 @@ def tLDRLIT_ga_abs : PseudoInst<(outs tGPR:$dst), (ins i32imm:$src),
 
 
 // JumpTable
-def : T1Pat<(ARMWrapperJT tjumptable:$dst, imm:$id),
-            (tLEApcrelJT tjumptable:$dst, imm:$id)>;
+def : T1Pat<(ARMWrapperJT tjumptable:$dst),
+            (tLEApcrelJT tjumptable:$dst)>;
 
 // Direct calls
 def : T1Pat<(ARMtcall texternalsym:$func), (tBL texternalsym:$func)>,
diff --git a/lib/Target/ARM/ARMInstrThumb2.td b/lib/Target/ARM/ARMInstrThumb2.td
index 103ee002cac..aba8a7b10fd 100644
--- a/lib/Target/ARM/ARMInstrThumb2.td
+++ b/lib/Target/ARM/ARMInstrThumb2.td
@@ -54,12 +54,14 @@ def t2_so_reg : Operand<i32>,    // reg imm
 
 // t2_so_imm_not_XFORM - Return the complement of a t2_so_imm value
 def t2_so_imm_not_XFORM : SDNodeXForm<imm, [{
-  return CurDAG->getTargetConstant(~((uint32_t)N->getZExtValue()), MVT::i32);
+  return CurDAG->getTargetConstant(~((uint32_t)N->getZExtValue()), SDLoc(N),
+                                   MVT::i32);
 }]>;
 
 // t2_so_imm_neg_XFORM - Return the negation of a t2_so_imm value
 def t2_so_imm_neg_XFORM : SDNodeXForm<imm, [{
-  return CurDAG->getTargetConstant(-((int)N->getZExtValue()), MVT::i32);
+  return CurDAG->getTargetConstant(-((int)N->getZExtValue()), SDLoc(N),
+                                   MVT::i32);
 }]>;
 
 // so_imm_notSext_XFORM - Return a so_imm value packed into the format
@@ -68,7 +70,7 @@ def t2_so_imm_neg_XFORM : SDNodeXForm<imm, [{
 def t2_so_imm_notSext16_XFORM : SDNodeXForm<imm, [{
   APInt apIntN = N->getAPIntValue();
   unsigned N16bitSignExt = apIntN.trunc(16).sext(32).getZExtValue();
-  return CurDAG->getTargetConstant(~N16bitSignExt, MVT::i32);
+  return CurDAG->getTargetConstant(~N16bitSignExt, SDLoc(N), MVT::i32);
 }]>;
 
 // t2_so_imm - Match a 32-bit immediate operand, which is an
@@ -1247,7 +1249,7 @@ def t2LEApcrel   : t2PseudoInst<(outs rGPR:$Rd), (ins i32imm:$label, pred:$p),
                                 4, IIC_iALUi, []>, Sched<[WriteALU, ReadALU]>;
 let hasSideEffects = 1 in
 def t2LEApcrelJT : t2PseudoInst<(outs rGPR:$Rd),
-                                (ins i32imm:$label, nohash_imm:$id, pred:$p),
+                                (ins i32imm:$label, pred:$p),
                                 4, IIC_iALUi,
                                 []>, Sched<[WriteALU, ReadALU]>;
 
@@ -3529,20 +3531,20 @@ def t2B   : T2I<(outs), (ins uncondbrtarget:$target), IIC_Br,
   let AsmMatchConverter = "cvtThumbBranches";
 }
 
-let isNotDuplicable = 1, isIndirectBranch = 1 in {
+let Size = 4, isNotDuplicable = 1, isIndirectBranch = 1 in {
 def t2BR_JT : t2PseudoInst<(outs),
-          (ins GPR:$target, GPR:$index, i32imm:$jt, i32imm:$id),
+          (ins GPR:$target, GPR:$index, i32imm:$jt),
            0, IIC_Br,
-          [(ARMbr2jt GPR:$target, GPR:$index, tjumptable:$jt, imm:$id)]>,
+          [(ARMbr2jt GPR:$target, GPR:$index, tjumptable:$jt)]>,
           Sched<[WriteBr]>;
 
-// FIXME: Add a non-pc based case that can be predicated.
+// FIXME: Add a case that can be predicated.
 def t2TBB_JT : t2PseudoInst<(outs),
-        (ins GPR:$index, i32imm:$jt, i32imm:$id), 0, IIC_Br, []>,
+        (ins GPR:$base, GPR:$index, i32imm:$jt, i32imm:$pclbl), 0, IIC_Br, []>,
         Sched<[WriteBr]>;
 
 def t2TBH_JT : t2PseudoInst<(outs),
-        (ins GPR:$index, i32imm:$jt, i32imm:$id), 0, IIC_Br, []>,
+        (ins GPR:$base, GPR:$index, i32imm:$jt, i32imm:$pclbl), 0, IIC_Br, []>,
         Sched<[WriteBr]>;
 
 def t2TBB : T2I<(outs), (ins addrmode_tbb:$addr), IIC_Br,
@@ -3875,8 +3877,8 @@ def : T2Pat<(ARMWrapper  tconstpool  :$dst), (t2LEApcrel tconstpool  :$dst)>;
 def : T2Pat<(ARMWrapper  tglobaladdr :$dst), (t2MOVi32imm tglobaladdr :$dst)>,
            Requires<[IsThumb2, UseMovt]>;
 
-def : T2Pat<(ARMWrapperJT tjumptable:$dst, imm:$id),
-            (t2LEApcrelJT tjumptable:$dst, imm:$id)>;
+def : T2Pat<(ARMWrapperJT tjumptable:$dst),
+            (t2LEApcrelJT tjumptable:$dst)>;
 
 // Pseudo instruction that combines ldr from constpool and add pc. This should
 // be expanded into two instructions late to allow if-conversion and
@@ -4139,11 +4141,9 @@ class t2MovRCopro<bits<4> Op, string opc, bit direction, dag oops, dag iops,
   let Inst{19-16} = CRn;
 }
 
-class t2MovRRCopro<bits<4> Op, string opc, bit direction,
+class t2MovRRCopro<bits<4> Op, string opc, bit direction, dag oops, dag iops,
                    list<dag> pattern = []>
-  : T2Cop<Op, (outs),
-          (ins p_imm:$cop, imm0_15:$opc1, GPR:$Rt, GPR:$Rt2, c_imm:$CRm),
-          opc, "\t$cop, $opc1, $Rt, $Rt2, $CRm", pattern> {
+  : T2Cop<Op, oops, iops, opc, "\t$cop, $opc1, $Rt, $Rt2, $CRm", pattern> {
   let Inst{27-24} = 0b1100;
   let Inst{23-21} = 0b010;
   let Inst{20} = direction;
@@ -4208,19 +4208,25 @@ def : T2v6Pat<(int_arm_mrc2 imm:$cop, imm:$opc1, imm:$CRn, imm:$CRm, imm:$opc2),
 
 
 /* from ARM core register to coprocessor */
-def t2MCRR : t2MovRRCopro<0b1110, "mcrr", 0,
+def t2MCRR : t2MovRRCopro<0b1110, "mcrr", 0, (outs),
+                         (ins p_imm:$cop, imm0_15:$opc1, GPR:$Rt, GPR:$Rt2,
+                         c_imm:$CRm),
                         [(int_arm_mcrr imm:$cop, imm:$opc1, GPR:$Rt, GPR:$Rt2,
                                        imm:$CRm)]>;
-def t2MCRR2 : t2MovRRCopro<0b1111, "mcrr2", 0,
-                           [(int_arm_mcrr2 imm:$cop, imm:$opc1, GPR:$Rt,
-                                           GPR:$Rt2, imm:$CRm)]> {
+def t2MCRR2 : t2MovRRCopro<0b1111, "mcrr2", 0, (outs),
+                          (ins p_imm:$cop, imm0_15:$opc1, GPR:$Rt, GPR:$Rt2,
+                           c_imm:$CRm),
+                          [(int_arm_mcrr2 imm:$cop, imm:$opc1, GPR:$Rt,
+                                          GPR:$Rt2, imm:$CRm)]> {
   let Predicates = [IsThumb2, PreV8];
 }
 
 /* from coprocessor to ARM core register */
-def t2MRRC : t2MovRRCopro<0b1110, "mrrc", 1>;
+def t2MRRC : t2MovRRCopro<0b1110, "mrrc", 1, (outs GPR:$Rt, GPR:$Rt2),
+                          (ins p_imm:$cop, imm0_15:$opc1, c_imm:$CRm)>;
 
-def t2MRRC2 : t2MovRRCopro<0b1111, "mrrc2", 1> {
+def t2MRRC2 : t2MovRRCopro<0b1111, "mrrc2", 1, (outs GPR:$Rt, GPR:$Rt2),
+                           (ins p_imm:$cop, imm0_15:$opc1, c_imm:$CRm)> {
   let Predicates = [IsThumb2, PreV8];
 }
 
diff --git a/lib/Target/ARM/ARMInstrVFP.td b/lib/Target/ARM/ARMInstrVFP.td
index afff01692a2..e83f8c85063 100644
--- a/lib/Target/ARM/ARMInstrVFP.td
+++ b/lib/Target/ARM/ARMInstrVFP.td
@@ -37,7 +37,7 @@ def vfp_f32imm : Operand<f32>,
     }], SDNodeXForm<fpimm, [{
       APFloat InVal = N->getValueAPF();
       uint32_t enc = ARM_AM::getFP32Imm(InVal);
-      return CurDAG->getTargetConstant(enc, MVT::i32);
+      return CurDAG->getTargetConstant(enc, SDLoc(N), MVT::i32);
     }]>> {
   let PrintMethod = "printFPImmOperand";
   let ParserMatchClass = FPImmOperand;
@@ -49,7 +49,7 @@ def vfp_f64imm : Operand<f64>,
     }], SDNodeXForm<fpimm, [{
       APFloat InVal = N->getValueAPF();
       uint32_t enc = ARM_AM::getFP64Imm(InVal);
-      return CurDAG->getTargetConstant(enc, MVT::i32);
+      return CurDAG->getTargetConstant(enc, SDLoc(N), MVT::i32);
     }]>> {
   let PrintMethod = "printFPImmOperand";
   let ParserMatchClass = FPImmOperand;
diff --git a/lib/Target/ARM/ARMLoadStoreOptimizer.cpp b/lib/Target/ARM/ARMLoadStoreOptimizer.cpp
index b049d92dfad..7b73c32c21e 100644
--- a/lib/Target/ARM/ARMLoadStoreOptimizer.cpp
+++ b/lib/Target/ARM/ARMLoadStoreOptimizer.cpp
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 //
-// This file contains a pass that performs load / store related peephole
-// optimizations. This pass should be run after register allocation.
+/// \file This file contains a pass that performs load / store related peephole
+/// optimizations. This pass should be run after register allocation.
 //
 //===----------------------------------------------------------------------===//
 
@@ -31,11 +31,13 @@
 #include "llvm/CodeGen/MachineInstr.h"
 #include "llvm/CodeGen/MachineInstrBuilder.h"
 #include "llvm/CodeGen/MachineRegisterInfo.h"
-#include "llvm/CodeGen/RegisterScavenging.h"
+#include "llvm/CodeGen/RegisterClassInfo.h"
 #include "llvm/CodeGen/SelectionDAGNodes.h"
+#include "llvm/CodeGen/LivePhysRegs.h"
 #include "llvm/IR/DataLayout.h"
 #include "llvm/IR/DerivedTypes.h"
 #include "llvm/IR/Function.h"
+#include "llvm/Support/Allocator.h"
 #include "llvm/Support/Debug.h"
 #include "llvm/Support/ErrorHandling.h"
 #include "llvm/Support/raw_ostream.h"
@@ -58,93 +60,104 @@ STATISTIC(NumSTRD2STM,  "Number of strd instructions turned back into stm");
 STATISTIC(NumLDRD2LDR,  "Number of ldrd instructions turned back into ldr's");
 STATISTIC(NumSTRD2STR,  "Number of strd instructions turned back into str's");
 
-/// ARMAllocLoadStoreOpt - Post- register allocation pass the combine
-/// load / store instructions to form ldm / stm instructions.
+namespace llvm {
+void initializeARMLoadStoreOptPass(PassRegistry &);
+}
+
+#define ARM_LOAD_STORE_OPT_NAME "ARM load / store optimization pass"
 
 namespace {
+  /// Post- register allocation pass the combine load / store instructions to
+  /// form ldm / stm instructions.
   struct ARMLoadStoreOpt : public MachineFunctionPass {
     static char ID;
-    ARMLoadStoreOpt() : MachineFunctionPass(ID) {}
+    ARMLoadStoreOpt() : MachineFunctionPass(ID) {
+      initializeARMLoadStoreOptPass(*PassRegistry::getPassRegistry());
+    }
 
+    const MachineFunction *MF;
     const TargetInstrInfo *TII;
     const TargetRegisterInfo *TRI;
+    const MachineRegisterInfo *MRI;
     const ARMSubtarget *STI;
     const TargetLowering *TL;
     ARMFunctionInfo *AFI;
-    RegScavenger *RS;
+    LivePhysRegs LiveRegs;
+    RegisterClassInfo RegClassInfo;
+    MachineBasicBlock::const_iterator LiveRegPos;
+    bool LiveRegsValid;
+    bool RegClassInfoValid;
     bool isThumb1, isThumb2;
 
     bool runOnMachineFunction(MachineFunction &Fn) override;
 
     const char *getPassName() const override {
-      return "ARM load / store optimization pass";
+      return ARM_LOAD_STORE_OPT_NAME;
     }
 
   private:
+    /// A set of load/store MachineInstrs with same base register sorted by
+    /// offset.
     struct MemOpQueueEntry {
-      int Offset;
-      unsigned Reg;
-      bool isKill;
-      unsigned Position;
-      MachineBasicBlock::iterator MBBI;
-      bool Merged;
-      MemOpQueueEntry(int o, unsigned r, bool k, unsigned p,
-                      MachineBasicBlock::iterator i)
-        : Offset(o), Reg(r), isKill(k), Position(p), MBBI(i), Merged(false) {}
+      MachineInstr *MI;
+      int Offset;        ///< Load/Store offset.
+      unsigned Position; ///< Position as counted from end of basic block.
+      MemOpQueueEntry(MachineInstr *MI, int Offset, unsigned Position)
+        : MI(MI), Offset(Offset), Position(Position) {}
     };
     typedef SmallVector<MemOpQueueEntry,8> MemOpQueue;
-    typedef MemOpQueue::iterator MemOpQueueIter;
 
-    void findUsesOfImpDef(SmallVectorImpl<MachineOperand *> &UsesOfImpDefs,
-                          const MemOpQueue &MemOps, unsigned DefReg,
-                          unsigned RangeBegin, unsigned RangeEnd);
+    /// A set of MachineInstrs that fulfill (nearly all) conditions to get
+    /// merged into a LDM/STM.
+    struct MergeCandidate {
+      /// List of instructions ordered by load/store offset.
+      SmallVector<MachineInstr*, 4> Instrs;
+      /// Index in Instrs of the instruction being latest in the schedule.
+      unsigned LatestMIIdx;
+      /// Index in Instrs of the instruction being earliest in the schedule.
+      unsigned EarliestMIIdx;
+      /// Index into the basic block where the merged instruction will be
+      /// inserted. (See MemOpQueueEntry.Position)
+      unsigned InsertPos;
+      /// Whether the instructions can be merged into a ldm/stm instruction.
+      bool CanMergeToLSMulti;
+      /// Whether the instructions can be merged into a ldrd/strd instruction.
+      bool CanMergeToLSDouble;
+    };
+    SpecificBumpPtrAllocator<MergeCandidate> Allocator;
+    SmallVector<const MergeCandidate*,4> Candidates;
+    SmallVector<MachineInstr*,4> MergeBaseCandidates;
+
+    void moveLiveRegsBefore(const MachineBasicBlock &MBB,
+                            MachineBasicBlock::const_iterator Before);
+    unsigned findFreeReg(const TargetRegisterClass &RegClass);
     void UpdateBaseRegUses(MachineBasicBlock &MBB,
                            MachineBasicBlock::iterator MBBI,
-                           DebugLoc dl, unsigned Base, unsigned WordOffset,
+                           DebugLoc DL, unsigned Base, unsigned WordOffset,
                            ARMCC::CondCodes Pred, unsigned PredReg);
-    bool MergeOps(MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI,
-                  int Offset, unsigned Base, bool BaseKill, int Opcode,
-                  ARMCC::CondCodes Pred, unsigned PredReg, unsigned Scratch,
-                  DebugLoc dl,
-                  ArrayRef<std::pair<unsigned, bool> > Regs,
-                  ArrayRef<unsigned> ImpDefs);
-    void MergeOpsUpdate(MachineBasicBlock &MBB,
-                        MemOpQueue &MemOps,
-                        unsigned memOpsBegin,
-                        unsigned memOpsEnd,
-                        unsigned insertAfter,
-                        int Offset,
-                        unsigned Base,
-                        bool BaseKill,
-                        int Opcode,
-                        ARMCC::CondCodes Pred,
-                        unsigned PredReg,
-                        unsigned Scratch,
-                        DebugLoc dl,
-                        SmallVectorImpl<MachineBasicBlock::iterator> &Merges);
-    void MergeLDR_STR(MachineBasicBlock &MBB, unsigned SIndex, unsigned Base,
-                      int Opcode, unsigned Size,
-                      ARMCC::CondCodes Pred, unsigned PredReg,
-                      unsigned Scratch, MemOpQueue &MemOps,
-                      SmallVectorImpl<MachineBasicBlock::iterator> &Merges);
-    void AdvanceRS(MachineBasicBlock &MBB, MemOpQueue &MemOps);
+    MachineInstr *CreateLoadStoreMulti(MachineBasicBlock &MBB,
+        MachineBasicBlock::iterator InsertBefore, int Offset, unsigned Base,
+        bool BaseKill, unsigned Opcode, ARMCC::CondCodes Pred, unsigned PredReg,
+        DebugLoc DL, ArrayRef<std::pair<unsigned, bool>> Regs);
+    MachineInstr *CreateLoadStoreDouble(MachineBasicBlock &MBB,
+        MachineBasicBlock::iterator InsertBefore, int Offset, unsigned Base,
+        bool BaseKill, unsigned Opcode, ARMCC::CondCodes Pred, unsigned PredReg,
+        DebugLoc DL, ArrayRef<std::pair<unsigned, bool>> Regs) const;
+    void FormCandidates(const MemOpQueue &MemOps);
+    MachineInstr *MergeOpsUpdate(const MergeCandidate &Cand);
     bool FixInvalidRegPairOp(MachineBasicBlock &MBB,
                              MachineBasicBlock::iterator &MBBI);
-    bool MergeBaseUpdateLoadStore(MachineBasicBlock &MBB,
-                                  MachineBasicBlock::iterator MBBI,
-                                  const TargetInstrInfo *TII,
-                                  bool &Advance,
-                                  MachineBasicBlock::iterator &I);
-    bool MergeBaseUpdateLSMultiple(MachineBasicBlock &MBB,
-                                   MachineBasicBlock::iterator MBBI,
-                                   bool &Advance,
-                                   MachineBasicBlock::iterator &I);
+    bool MergeBaseUpdateLoadStore(MachineInstr *MI);
+    bool MergeBaseUpdateLSMultiple(MachineInstr *MI);
+    bool MergeBaseUpdateLSDouble(MachineInstr &MI) const;
     bool LoadStoreMultipleOpti(MachineBasicBlock &MBB);
     bool MergeReturnIntoLDM(MachineBasicBlock &MBB);
   };
   char ARMLoadStoreOpt::ID = 0;
 }
 
+INITIALIZE_PASS(ARMLoadStoreOpt, "arm-load-store-opt", ARM_LOAD_STORE_OPT_NAME, false, false)
+
 static bool definesCPSR(const MachineInstr *MI) {
   for (const auto &MO : MI->operands()) {
     if (!MO.isReg())
@@ -159,7 +172,7 @@ static bool definesCPSR(const MachineInstr *MI) {
 }
 
 static int getMemoryOpOffset(const MachineInstr *MI) {
-  int Opcode = MI->getOpcode();
+  unsigned Opcode = MI->getOpcode();
   bool isAM3 = Opcode == ARM::LDRD || Opcode == ARM::STRD;
   unsigned NumOperands = MI->getDesc().getNumOperands();
   unsigned OffField = MI->getOperand(NumOperands-3).getImm();
@@ -186,7 +199,15 @@ static int getMemoryOpOffset(const MachineInstr *MI) {
   return Offset;
 }
 
-static int getLoadStoreMultipleOpcode(int Opcode, ARM_AM::AMSubMode Mode) {
+static const MachineOperand &getLoadStoreBaseOp(const MachineInstr &MI) {
+  return MI.getOperand(1);
+}
+
+static const MachineOperand &getLoadStoreRegOp(const MachineInstr &MI) {
+  return MI.getOperand(0);
+}
+
+static int getLoadStoreMultipleOpcode(unsigned Opcode, ARM_AM::AMSubMode Mode) {
   switch (Opcode) {
   default: llvm_unreachable("Unhandled opcode!");
   case ARM::LDRi12:
@@ -271,10 +292,7 @@ static int getLoadStoreMultipleOpcode(int Opcode, ARM_AM::AMSubMode Mode) {
   }
 }
 
-namespace llvm {
-  namespace ARM_AM {
-
-AMSubMode getLoadStoreMultipleSubMode(int Opcode) {
+static ARM_AM::AMSubMode getLoadStoreMultipleSubMode(unsigned Opcode) {
   switch (Opcode) {
   default: llvm_unreachable("Unhandled opcode!");
   case ARM::LDMIA_RET:
@@ -328,9 +346,6 @@ AMSubMode getLoadStoreMultipleSubMode(int Opcode) {
   }
 }
 
-  } // end namespace ARM_AM
-} // end namespace llvm
-
 static bool isT1i32Load(unsigned Opc) {
   return Opc == ARM::tLDRi || Opc == ARM::tLDRspi;
 }
@@ -355,6 +370,10 @@ static bool isi32Store(unsigned Opc) {
   return Opc == ARM::STRi12 || isT1i32Store(Opc) || isT2i32Store(Opc);
 }
 
+static bool isLoadSingle(unsigned Opc) {
+  return isi32Load(Opc) || Opc == ARM::VLDRS || Opc == ARM::VLDRD;
+}
+
 static unsigned getImmScale(unsigned Opc) {
   switch (Opc) {
   default: llvm_unreachable("Unhandled opcode!");
@@ -372,12 +391,55 @@ static unsigned getImmScale(unsigned Opc) {
   }
 }
 
+static unsigned getLSMultipleTransferSize(const MachineInstr *MI) {
+  switch (MI->getOpcode()) {
+  default: return 0;
+  case ARM::LDRi12:
+  case ARM::STRi12:
+  case ARM::tLDRi:
+  case ARM::tSTRi:
+  case ARM::tLDRspi:
+  case ARM::tSTRspi:
+  case ARM::t2LDRi8:
+  case ARM::t2LDRi12:
+  case ARM::t2STRi8:
+  case ARM::t2STRi12:
+  case ARM::VLDRS:
+  case ARM::VSTRS:
+    return 4;
+  case ARM::VLDRD:
+  case ARM::VSTRD:
+    return 8;
+  case ARM::LDMIA:
+  case ARM::LDMDA:
+  case ARM::LDMDB:
+  case ARM::LDMIB:
+  case ARM::STMIA:
+  case ARM::STMDA:
+  case ARM::STMDB:
+  case ARM::STMIB:
+  case ARM::tLDMIA:
+  case ARM::tLDMIA_UPD:
+  case ARM::tSTMIA_UPD:
+  case ARM::t2LDMIA:
+  case ARM::t2LDMDB:
+  case ARM::t2STMIA:
+  case ARM::t2STMDB:
+  case ARM::VLDMSIA:
+  case ARM::VSTMSIA:
+    return (MI->getNumOperands() - MI->getDesc().getNumOperands() + 1) * 4;
+  case ARM::VLDMDIA:
+  case ARM::VSTMDIA:
+    return (MI->getNumOperands() - MI->getDesc().getNumOperands() + 1) * 8;
+  }
+}
+
 /// Update future uses of the base register with the offset introduced
 /// due to writeback. This function only works on Thumb1.
 void
 ARMLoadStoreOpt::UpdateBaseRegUses(MachineBasicBlock &MBB,
                                    MachineBasicBlock::iterator MBBI,
-                                   DebugLoc dl, unsigned Base,
+                                   DebugLoc DL, unsigned Base,
                                    unsigned WordOffset,
                                    ARMCC::CondCodes Pred, unsigned PredReg) {
   assert(isThumb1 && "Can only update base register uses for Thumb1!");
@@ -405,7 +467,7 @@ ARMLoadStoreOpt::UpdateBaseRegUses(MachineBasicBlock &MBB,
         Offset = MO.getImm() - WordOffset * getImmScale(Opc);
 
         // If storing the base register, it needs to be reset first.
-        unsigned InstrSrcReg = MBBI->getOperand(0).getReg();
+        unsigned InstrSrcReg = getLoadStoreRegOp(*MBBI).getReg();
 
         if (Offset >= 0 && !(IsStore && InstrSrcReg == Base))
           MO.setImm(Offset);
@@ -446,13 +508,12 @@ ARMLoadStoreOpt::UpdateBaseRegUses(MachineBasicBlock &MBB,
 
     if (InsertSub) {
       // An instruction above couldn't be updated, so insert a sub.
-      AddDefaultT1CC(BuildMI(MBB, MBBI, dl, TII->get(ARM::tSUBi8), Base), true)
-        .addReg(Base, getKillRegState(false)).addImm(WordOffset * 4)
-        .addImm(Pred).addReg(PredReg);
+      AddDefaultT1CC(BuildMI(MBB, MBBI, DL, TII->get(ARM::tSUBi8), Base), true)
+        .addReg(Base).addImm(WordOffset * 4).addImm(Pred).addReg(PredReg);
       return;
     }
 
-    if (MBBI->killsRegister(Base))
+    if (MBBI->killsRegister(Base) || MBBI->definesRegister(Base))
       // Register got killed. Stop updating.
       return;
   }
@@ -465,32 +526,65 @@ ARMLoadStoreOpt::UpdateBaseRegUses(MachineBasicBlock &MBB,
     // See PR21029.
     if (MBBI != MBB.end()) --MBBI;
     AddDefaultT1CC(
-      BuildMI(MBB, MBBI, dl, TII->get(ARM::tSUBi8), Base), true)
-      .addReg(Base, getKillRegState(false)).addImm(WordOffset * 4)
-      .addImm(Pred).addReg(PredReg);
+      BuildMI(MBB, MBBI, DL, TII->get(ARM::tSUBi8), Base), true)
+      .addReg(Base).addImm(WordOffset * 4).addImm(Pred).addReg(PredReg);
   }
 }
 
-/// MergeOps - Create and insert a LDM or STM with Base as base register and
-/// registers in Regs as the register operands that would be loaded / stored.
-/// It returns true if the transformation is done.
-bool
-ARMLoadStoreOpt::MergeOps(MachineBasicBlock &MBB,
-                          MachineBasicBlock::iterator MBBI,
-                          int Offset, unsigned Base, bool BaseKill,
-                          int Opcode, ARMCC::CondCodes Pred,
-                          unsigned PredReg, unsigned Scratch, DebugLoc dl,
-                          ArrayRef<std::pair<unsigned, bool> > Regs,
-                          ArrayRef<unsigned> ImpDefs) {
-  // Only a single register to load / store. Don't bother.
+/// Return the first register of class \p RegClass that is not in \p Regs.
+unsigned ARMLoadStoreOpt::findFreeReg(const TargetRegisterClass &RegClass) {
+  if (!RegClassInfoValid) {
+    RegClassInfo.runOnMachineFunction(*MF);
+    RegClassInfoValid = true;
+  }
+
+  for (unsigned Reg : RegClassInfo.getOrder(&RegClass))
+    if (!LiveRegs.contains(Reg))
+      return Reg;
+  return 0;
+}
+
+/// Compute live registers just before instruction \p Before (in normal schedule
+/// direction). Computes backwards so multiple queries in the same block must
+/// come in reverse order.
+void ARMLoadStoreOpt::moveLiveRegsBefore(const MachineBasicBlock &MBB,
+    MachineBasicBlock::const_iterator Before) {
+  // Initialize if we never queried in this block.
+  if (!LiveRegsValid) {
+    LiveRegs.init(TRI);
+    LiveRegs.addLiveOuts(&MBB, true);
+    LiveRegPos = MBB.end();
+    LiveRegsValid = true;
+  }
+  // Move backward just before the "Before" position.
+  while (LiveRegPos != Before) {
+    --LiveRegPos;
+    LiveRegs.stepBackward(*LiveRegPos);
+  }
+}
+
+static bool ContainsReg(const ArrayRef<std::pair<unsigned, bool>> &Regs,
+                        unsigned Reg) {
+  for (const std::pair<unsigned, bool> &R : Regs)
+    if (R.first == Reg)
+      return true;
+  return false;
+}
+
+/// Create and insert a LDM or STM with Base as base register and registers in
+/// Regs as the register operands that would be loaded / stored.  It returns
+/// true if the transformation is done.
+MachineInstr *ARMLoadStoreOpt::CreateLoadStoreMulti(MachineBasicBlock &MBB,
+    MachineBasicBlock::iterator InsertBefore, int Offset, unsigned Base,
+    bool BaseKill, unsigned Opcode, ARMCC::CondCodes Pred, unsigned PredReg,
+    DebugLoc DL, ArrayRef<std::pair<unsigned, bool>> Regs) {
   unsigned NumRegs = Regs.size();
-  if (NumRegs <= 1)
-    return false;
+  assert(NumRegs > 1);
 
   // For Thumb1 targets, it might be necessary to clobber the CPSR to merge.
   // Compute liveness information for that register to make the decision.
   bool SafeToClobberCPSR = !isThumb1 ||
-    (MBB.computeRegisterLiveness(TRI, ARM::CPSR, std::prev(MBBI), 15) ==
+    (MBB.computeRegisterLiveness(TRI, ARM::CPSR, InsertBefore, 20) ==
      MachineBasicBlock::LQR_Dead);
 
   bool Writeback = isThumb1; // Thumb1 LDM/STM have base reg writeback.
@@ -498,17 +592,14 @@ ARMLoadStoreOpt::MergeOps(MachineBasicBlock &MBB,
   // Exception: If the base register is in the input reglist, Thumb1 LDM is
   // non-writeback.
   // It's also not possible to merge an STR of the base register in Thumb1.
-  if (isThumb1)
-    for (unsigned I = 0; I < NumRegs; ++I)
-      if (Base == Regs[I].first) {
-        assert(Base != ARM::SP && "Thumb1 does not allow SP in register list");
-        if (Opcode == ARM::tLDRi) {
-          Writeback = false;
-          break;
-        } else if (Opcode == ARM::tSTRi) {
-          return false;
-        }
-      }
+  if (isThumb1 && isi32Load(Opcode) && ContainsReg(Regs, Base)) {
+    assert(Base != ARM::SP && "Thumb1 does not allow SP in register list");
+    if (Opcode == ARM::tLDRi) {
+      Writeback = false;
+    } else if (Opcode == ARM::tSTRi) {
+      return nullptr;
+    }
+  }
 
   ARM_AM::AMSubMode Mode = ARM_AM::ia;
   // VFP and Thumb2 do not support IB or DA modes. Thumb1 only supports IA.
@@ -525,18 +616,18 @@ ARMLoadStoreOpt::MergeOps(MachineBasicBlock &MBB,
   } else if (Offset != 0 || Opcode == ARM::tLDRspi || Opcode == ARM::tSTRspi) {
     // Check if this is a supported opcode before inserting instructions to
     // calculate a new base register.
-    if (!getLoadStoreMultipleOpcode(Opcode, Mode)) return false;
+    if (!getLoadStoreMultipleOpcode(Opcode, Mode)) return nullptr;
 
     // If starting offset isn't zero, insert a MI to materialize a new base.
     // But only do so if it is cost effective, i.e. merging more than two
     // loads / stores.
     if (NumRegs <= 2)
-      return false;
+      return nullptr;
 
     // On Thumb1, it's not worth materializing a new base register without
     // clobbering the CPSR (i.e. not using ADDS/SUBS).
     if (!SafeToClobberCPSR)
-      return false;
+      return nullptr;
 
     unsigned NewBase;
     if (isi32Load(Opcode)) {
@@ -544,10 +635,17 @@ ARMLoadStoreOpt::MergeOps(MachineBasicBlock &MBB,
       // use as the new base.
       NewBase = Regs[NumRegs-1].first;
     } else {
-      // Use the scratch register to use as a new base.
-      NewBase = Scratch;
+      // Find a free register that we can use as scratch register.
+      moveLiveRegsBefore(MBB, InsertBefore);
+      // The merged instruction does not exist yet but will use several Regs if
+      // it is a Store.
+      if (!isLoadSingle(Opcode))
+        for (const std::pair<unsigned, bool> &R : Regs)
+          LiveRegs.addReg(R.first);
+
+      NewBase = findFreeReg(isThumb1 ? ARM::tGPRRegClass : ARM::GPRRegClass);
       if (NewBase == 0)
-        return false;
+        return nullptr;
     }
 
     int BaseOpc =
@@ -566,7 +664,12 @@ ARMLoadStoreOpt::MergeOps(MachineBasicBlock &MBB,
 
     if (!TL->isLegalAddImmediate(Offset))
       // FIXME: Try add with register operand?
-      return false; // Probably not worth it then.
+      return nullptr; // Probably not worth it then.
+
+    // We can only append a kill flag to the add/sub input if the value is not
+    // used in the register list of the stm as well.
+    bool KillOldBase = BaseKill &&
+      (!isi32Store(Opcode) || !ContainsReg(Regs, Base));
 
     if (isThumb1) {
       // Thumb1: depending on immediate size, use either
@@ -581,43 +684,44 @@ ARMLoadStoreOpt::MergeOps(MachineBasicBlock &MBB,
             !STI->hasV6Ops()) {
           // thumbv4t doesn't have lo->lo copies, and we can't predicate tMOVSr
           if (Pred != ARMCC::AL)
-            return false;
-          BuildMI(MBB, MBBI, dl, TII->get(ARM::tMOVSr), NewBase)
-            .addReg(Base, getKillRegState(BaseKill));
+            return nullptr;
+          BuildMI(MBB, InsertBefore, DL, TII->get(ARM::tMOVSr), NewBase)
+            .addReg(Base, getKillRegState(KillOldBase));
         } else
-          BuildMI(MBB, MBBI, dl, TII->get(ARM::tMOVr), NewBase)
-            .addReg(Base, getKillRegState(BaseKill))
+          BuildMI(MBB, InsertBefore, DL, TII->get(ARM::tMOVr), NewBase)
+            .addReg(Base, getKillRegState(KillOldBase))
             .addImm(Pred).addReg(PredReg);
 
-        // Set up BaseKill and Base correctly to insert the ADDS/SUBS below.
+        // The following ADDS/SUBS becomes an update.
         Base = NewBase;
-        BaseKill = false;
+        KillOldBase = true;
       }
       if (BaseOpc == ARM::tADDrSPi) {
         assert(Offset % 4 == 0 && "tADDrSPi offset is scaled by 4");
-        BuildMI(MBB, MBBI, dl, TII->get(BaseOpc), NewBase)
-          .addReg(Base, getKillRegState(BaseKill)).addImm(Offset/4)
+        BuildMI(MBB, InsertBefore, DL, TII->get(BaseOpc), NewBase)
+          .addReg(Base, getKillRegState(KillOldBase)).addImm(Offset/4)
           .addImm(Pred).addReg(PredReg);
       } else
-        AddDefaultT1CC(BuildMI(MBB, MBBI, dl, TII->get(BaseOpc), NewBase), true)
-          .addReg(Base, getKillRegState(BaseKill)).addImm(Offset)
+        AddDefaultT1CC(
+          BuildMI(MBB, InsertBefore, DL, TII->get(BaseOpc), NewBase), true)
+          .addReg(Base, getKillRegState(KillOldBase)).addImm(Offset)
           .addImm(Pred).addReg(PredReg);
     } else {
-      BuildMI(MBB, MBBI, dl, TII->get(BaseOpc), NewBase)
-        .addReg(Base, getKillRegState(BaseKill)).addImm(Offset)
+      BuildMI(MBB, InsertBefore, DL, TII->get(BaseOpc), NewBase)
+        .addReg(Base, getKillRegState(KillOldBase)).addImm(Offset)
         .addImm(Pred).addReg(PredReg).addReg(0);
     }
     Base = NewBase;
     BaseKill = true; // New base is always killed straight away.
   }
 
-  bool isDef = (isi32Load(Opcode) || Opcode == ARM::VLDRS ||
-                Opcode == ARM::VLDRD);
+  bool isDef = isLoadSingle(Opcode);
 
   // Get LS multiple opcode. Note that for Thumb1 this might be an opcode with
   // base register writeback.
   Opcode = getLoadStoreMultipleOpcode(Opcode, Mode);
-  if (!Opcode) return false;
+  if (!Opcode)
+    return nullptr;
 
   // Check if a Thumb1 LDM/STM merge is safe. This is the case if:
   // - There is no writeback (LDM of base register),
@@ -628,7 +732,7 @@ ARMLoadStoreOpt::MergeOps(MachineBasicBlock &MBB,
   // It's safe to return here since the code to materialize a new base register
   // above is also conditional on SafeToClobberCPSR.
   if (isThumb1 && !SafeToClobberCPSR && Writeback && !BaseKill)
-    return false;
+    return nullptr;
 
   MachineInstrBuilder MIB;
 
@@ -637,7 +741,7 @@ ARMLoadStoreOpt::MergeOps(MachineBasicBlock &MBB,
       // Update tLDMIA with writeback if necessary.
       Opcode = ARM::tLDMIA_UPD;
 
-    MIB = BuildMI(MBB, MBBI, dl, TII->get(Opcode));
+    MIB = BuildMI(MBB, InsertBefore, DL, TII->get(Opcode));
 
     // Thumb1: we might need to set base writeback when building the MI.
     MIB.addReg(Base, getDefRegState(true))
@@ -646,376 +750,263 @@ ARMLoadStoreOpt::MergeOps(MachineBasicBlock &MBB,
     // The base isn't dead after a merged instruction with writeback.
     // Insert a sub instruction after the newly formed instruction to reset.
     if (!BaseKill)
-      UpdateBaseRegUses(MBB, MBBI, dl, Base, NumRegs, Pred, PredReg);
+      UpdateBaseRegUses(MBB, InsertBefore, DL, Base, NumRegs, Pred, PredReg);
 
   } else {
     // No writeback, simply build the MachineInstr.
-    MIB = BuildMI(MBB, MBBI, dl, TII->get(Opcode));
+    MIB = BuildMI(MBB, InsertBefore, DL, TII->get(Opcode));
     MIB.addReg(Base, getKillRegState(BaseKill));
   }
 
   MIB.addImm(Pred).addReg(PredReg);
 
-  for (unsigned i = 0; i != NumRegs; ++i)
-    MIB = MIB.addReg(Regs[i].first, getDefRegState(isDef)
-                     | getKillRegState(Regs[i].second));
-
-  // Add implicit defs for super-registers.
-  for (unsigned i = 0, e = ImpDefs.size(); i != e; ++i)
-    MIB.addReg(ImpDefs[i], RegState::ImplicitDefine);
+  for (const std::pair<unsigned, bool> &R : Regs)
+    MIB.addReg(R.first, getDefRegState(isDef) | getKillRegState(R.second));
 
-  return true;
+  return MIB.getInstr();
 }
 
-/// \brief Find all instructions using a given imp-def within a range.
-///
-/// We are trying to combine a range of instructions, one of which (located at
-/// position RangeBegin) implicitly defines a register. The final LDM/STM will
-/// be placed at RangeEnd, and so any uses of this definition between RangeStart
-/// and RangeEnd must be modified to use an undefined value.
-///
-/// The live range continues until we find a second definition or one of the
-/// uses we find is a kill. Unfortunately MemOps is not sorted by Position, so
-/// we must consider all uses and decide which are relevant in a second pass.
-void ARMLoadStoreOpt::findUsesOfImpDef(
-    SmallVectorImpl<MachineOperand *> &UsesOfImpDefs, const MemOpQueue &MemOps,
-    unsigned DefReg, unsigned RangeBegin, unsigned RangeEnd) {
-  std::map<unsigned, MachineOperand *> Uses;
-  unsigned LastLivePos = RangeEnd;
-
-  // First we find all uses of this register with Position between RangeBegin
-  // and RangeEnd, any or all of these could be uses of a definition at
-  // RangeBegin. We also record the latest position a definition at RangeBegin
-  // would be considered live.
-  for (unsigned i = 0; i < MemOps.size(); ++i) {
-    MachineInstr &MI = *MemOps[i].MBBI;
-    unsigned MIPosition = MemOps[i].Position;
-    if (MIPosition <= RangeBegin || MIPosition > RangeEnd)
-      continue;
-
-    // If this instruction defines the register, then any later use will be of
-    // that definition rather than ours.
-    if (MI.definesRegister(DefReg))
-      LastLivePos = std::min(LastLivePos, MIPosition);
-
-    MachineOperand *UseOp = MI.findRegisterUseOperand(DefReg);
-    if (!UseOp)
-      continue;
-
-    // If this instruction kills the register then (assuming liveness is
-    // correct when we start) we don't need to think about anything after here.
-    if (UseOp->isKill())
-      LastLivePos = std::min(LastLivePos, MIPosition);
-
-    Uses[MIPosition] = UseOp;
-  }
-
-  // Now we traverse the list of all uses, and append the ones that actually use
-  // our definition to the requested list.
-  for (std::map<unsigned, MachineOperand *>::iterator I = Uses.begin(),
-                                                      E = Uses.end();
-       I != E; ++I) {
-    // List is sorted by position so once we've found one out of range there
-    // will be no more to consider.
-    if (I->first > LastLivePos)
-      break;
-    UsesOfImpDefs.push_back(I->second);
+MachineInstr *ARMLoadStoreOpt::CreateLoadStoreDouble(MachineBasicBlock &MBB,
+    MachineBasicBlock::iterator InsertBefore, int Offset, unsigned Base,
+    bool BaseKill, unsigned Opcode, ARMCC::CondCodes Pred, unsigned PredReg,
+    DebugLoc DL, ArrayRef<std::pair<unsigned, bool>> Regs) const {
+  bool IsLoad = isi32Load(Opcode);
+  assert((IsLoad || isi32Store(Opcode)) && "Must have integer load or store");
+  unsigned LoadStoreOpcode = IsLoad ? ARM::t2LDRDi8 : ARM::t2STRDi8;
+
+  assert(Regs.size() == 2);
+  MachineInstrBuilder MIB = BuildMI(MBB, InsertBefore, DL,
+                                    TII->get(LoadStoreOpcode));
+  if (IsLoad) {
+    MIB.addReg(Regs[0].first, RegState::Define)
+       .addReg(Regs[1].first, RegState::Define);
+  } else {
+    MIB.addReg(Regs[0].first, getKillRegState(Regs[0].second))
+       .addReg(Regs[1].first, getKillRegState(Regs[1].second));
   }
+  MIB.addReg(Base).addImm(Offset).addImm(Pred).addReg(PredReg);
+  return MIB.getInstr();
 }
 
-// MergeOpsUpdate - call MergeOps and update MemOps and merges accordingly on
-// success.
-void ARMLoadStoreOpt::MergeOpsUpdate(MachineBasicBlock &MBB,
-                                     MemOpQueue &memOps,
-                                     unsigned memOpsBegin, unsigned memOpsEnd,
-                                     unsigned insertAfter, int Offset,
-                                     unsigned Base, bool BaseKill,
-                                     int Opcode,
-                                     ARMCC::CondCodes Pred, unsigned PredReg,
-                                     unsigned Scratch,
-                                     DebugLoc dl,
-                         SmallVectorImpl<MachineBasicBlock::iterator> &Merges) {
-  // First calculate which of the registers should be killed by the merged
-  // instruction.
-  const unsigned insertPos = memOps[insertAfter].Position;
-  SmallSet<unsigned, 4> KilledRegs;
-  DenseMap<unsigned, unsigned> Killer;
-  for (unsigned i = 0, e = memOps.size(); i != e; ++i) {
-    if (i == memOpsBegin) {
-      i = memOpsEnd;
-      if (i == e)
-        break;
-    }
-    if (memOps[i].Position < insertPos && memOps[i].isKill) {
-      unsigned Reg = memOps[i].Reg;
-      KilledRegs.insert(Reg);
-      Killer[Reg] = i;
-    }
-  }
-
+/// Call MergeOps and update MemOps and merges accordingly on success.
+MachineInstr *ARMLoadStoreOpt::MergeOpsUpdate(const MergeCandidate &Cand) {
+  const MachineInstr *First = Cand.Instrs.front();
+  unsigned Opcode = First->getOpcode();
+  bool IsLoad = isLoadSingle(Opcode);
   SmallVector<std::pair<unsigned, bool>, 8> Regs;
-  SmallVector<unsigned, 8> ImpDefs;
-  SmallVector<MachineOperand *, 8> UsesOfImpDefs;
-  for (unsigned i = memOpsBegin; i < memOpsEnd; ++i) {
-    unsigned Reg = memOps[i].Reg;
-    // If we are inserting the merged operation after an operation that
-    // uses the same register, make sure to transfer any kill flag.
-    bool isKill = memOps[i].isKill || KilledRegs.count(Reg);
-    Regs.push_back(std::make_pair(Reg, isKill));
-
-    // Collect any implicit defs of super-registers. They must be preserved.
-    for (MIOperands MO(memOps[i].MBBI); MO.isValid(); ++MO) {
-      if (!MO->isReg() || !MO->isDef() || !MO->isImplicit() || MO->isDead())
-        continue;
-      unsigned DefReg = MO->getReg();
-      if (std::find(ImpDefs.begin(), ImpDefs.end(), DefReg) == ImpDefs.end())
+  SmallVector<unsigned, 4> ImpDefs;
+  DenseSet<unsigned> KilledRegs;
+  DenseSet<unsigned> UsedRegs;
+  // Determine list of registers and list of implicit super-register defs.
+  for (const MachineInstr *MI : Cand.Instrs) {
+    const MachineOperand &MO = getLoadStoreRegOp(*MI);
+    unsigned Reg = MO.getReg();
+    bool IsKill = MO.isKill();
+    if (IsKill)
+      KilledRegs.insert(Reg);
+    Regs.push_back(std::make_pair(Reg, IsKill));
+    UsedRegs.insert(Reg);
+
+    if (IsLoad) {
+      // Collect any implicit defs of super-registers, after merging we can't
+      // be sure anymore that we properly preserved these live ranges and must
+      // removed these implicit operands.
+      for (const MachineOperand &MO : MI->implicit_operands()) {
+        if (!MO.isReg() || !MO.isDef() || MO.isDead())
+          continue;
+        assert(MO.isImplicit());
+        unsigned DefReg = MO.getReg();
+
+        if (std::find(ImpDefs.begin(), ImpDefs.end(), DefReg) != ImpDefs.end())
+          continue;
+        // We can ignore cases where the super-reg is read and written.
+        if (MI->readsRegister(DefReg))
+          continue;
         ImpDefs.push_back(DefReg);
-
-      // There may be other uses of the definition between this instruction and
-      // the eventual LDM/STM position. These should be marked undef if the
-      // merge takes place.
-      findUsesOfImpDef(UsesOfImpDefs, memOps, DefReg, memOps[i].Position,
-                       insertPos);
-    }
-  }
-
-  // Try to do the merge.
-  MachineBasicBlock::iterator Loc = memOps[insertAfter].MBBI;
-  ++Loc;
-  if (!MergeOps(MBB, Loc, Offset, Base, BaseKill, Opcode,
-                Pred, PredReg, Scratch, dl, Regs, ImpDefs))
-    return;
-
-  // Merge succeeded, update records.
-  Merges.push_back(std::prev(Loc));
-
-  // In gathering loads together, we may have moved the imp-def of a register
-  // past one of its uses. This is OK, since we know better than the rest of
-  // LLVM what's OK with ARM loads and stores; but we still have to adjust the
-  // affected uses.
-  for (SmallVectorImpl<MachineOperand *>::iterator I = UsesOfImpDefs.begin(),
-                                                   E = UsesOfImpDefs.end();
-                                                   I != E; ++I)
-    (*I)->setIsUndef();
-
-  for (unsigned i = memOpsBegin; i < memOpsEnd; ++i) {
-    // Remove kill flags from any memops that come before insertPos.
-    if (Regs[i-memOpsBegin].second) {
-      unsigned Reg = Regs[i-memOpsBegin].first;
-      if (KilledRegs.count(Reg)) {
-        unsigned j = Killer[Reg];
-        int Idx = memOps[j].MBBI->findRegisterUseOperandIdx(Reg, true);
-        assert(Idx >= 0 && "Cannot find killing operand");
-        memOps[j].MBBI->getOperand(Idx).setIsKill(false);
-        memOps[j].isKill = false;
       }
-      memOps[i].isKill = true;
     }
-    MBB.erase(memOps[i].MBBI);
-    // Update this memop to refer to the merged instruction.
-    // We may need to move kill flags again.
-    memOps[i].Merged = true;
-    memOps[i].MBBI = Merges.back();
-    memOps[i].Position = insertPos;
   }
 
-  // Update memOps offsets, since they may have been modified by MergeOps.
-  for (auto &MemOp : memOps) {
-    MemOp.Offset = getMemoryOpOffset(MemOp.MBBI);
+  // Attempt the merge.
+  typedef MachineBasicBlock::iterator iterator;
+  MachineInstr *LatestMI = Cand.Instrs[Cand.LatestMIIdx];
+  iterator InsertBefore = std::next(iterator(LatestMI));
+  MachineBasicBlock &MBB = *LatestMI->getParent();
+  unsigned Offset = getMemoryOpOffset(First);
+  unsigned Base = getLoadStoreBaseOp(*First).getReg();
+  bool BaseKill = LatestMI->killsRegister(Base);
+  unsigned PredReg = 0;
+  ARMCC::CondCodes Pred = getInstrPredicate(First, PredReg);
+  DebugLoc DL = First->getDebugLoc();
+  MachineInstr *Merged = nullptr;
+  if (Cand.CanMergeToLSDouble)
+    Merged = CreateLoadStoreDouble(MBB, InsertBefore, Offset, Base, BaseKill,
+                                   Opcode, Pred, PredReg, DL, Regs);
+  if (!Merged && Cand.CanMergeToLSMulti)
+    Merged = CreateLoadStoreMulti(MBB, InsertBefore, Offset, Base, BaseKill,
+                                  Opcode, Pred, PredReg, DL, Regs);
+  if (!Merged)
+    return nullptr;
+
+  // Determine earliest instruction that will get removed. We then keep an
+  // iterator just above it so the following erases don't invalidated it.
+  iterator EarliestI(Cand.Instrs[Cand.EarliestMIIdx]);
+  bool EarliestAtBegin = false;
+  if (EarliestI == MBB.begin()) {
+    EarliestAtBegin = true;
+  } else {
+    EarliestI = std::prev(EarliestI);
   }
-}
 
-/// MergeLDR_STR - Merge a number of load / store instructions into one or more
-/// load / store multiple instructions.
-void
-ARMLoadStoreOpt::MergeLDR_STR(MachineBasicBlock &MBB, unsigned SIndex,
-                         unsigned Base, int Opcode, unsigned Size,
-                         ARMCC::CondCodes Pred, unsigned PredReg,
-                         unsigned Scratch, MemOpQueue &MemOps,
-                         SmallVectorImpl<MachineBasicBlock::iterator> &Merges) {
-  bool isNotVFP = isi32Load(Opcode) || isi32Store(Opcode);
-  int Offset = MemOps[SIndex].Offset;
-  int SOffset = Offset;
-  unsigned insertAfter = SIndex;
-  MachineBasicBlock::iterator Loc = MemOps[SIndex].MBBI;
-  DebugLoc dl = Loc->getDebugLoc();
-  const MachineOperand &PMO = Loc->getOperand(0);
-  unsigned PReg = PMO.getReg();
-  unsigned PRegNum = PMO.isUndef() ? UINT_MAX : TRI->getEncodingValue(PReg);
-  unsigned Count = 1;
-  unsigned Limit = ~0U;
-  bool BaseKill = false;
-  // vldm / vstm limit are 32 for S variants, 16 for D variants.
-
-  switch (Opcode) {
-  default: break;
-  case ARM::VSTRS:
-    Limit = 32;
-    break;
-  case ARM::VSTRD:
-    Limit = 16;
-    break;
-  case ARM::VLDRD:
-    Limit = 16;
-    break;
-  case ARM::VLDRS:
-    Limit = 32;
-    break;
-  }
+  // Remove instructions which have been merged.
+  for (MachineInstr *MI : Cand.Instrs)
+    MBB.erase(MI);
 
-  for (unsigned i = SIndex+1, e = MemOps.size(); i != e; ++i) {
-    int NewOffset = MemOps[i].Offset;
-    const MachineOperand &MO = MemOps[i].MBBI->getOperand(0);
-    unsigned Reg = MO.getReg();
-    unsigned RegNum = MO.isUndef() ? UINT_MAX : TRI->getEncodingValue(Reg);
-    // Register numbers must be in ascending order. For VFP / NEON load and
-    // store multiples, the registers must also be consecutive and within the
-    // limit on the number of registers per instruction.
-    if (Reg != ARM::SP &&
-        NewOffset == Offset + (int)Size &&
-        ((isNotVFP && RegNum > PRegNum) ||
-         ((Count < Limit) && RegNum == PRegNum+1)) &&
-        // On Swift we don't want vldm/vstm to start with a odd register num
-        // because Q register unaligned vldm/vstm need more uops.
-        (!STI->isSwift() || isNotVFP || Count != 1 || !(PRegNum & 0x1))) {
-      Offset += Size;
-      PRegNum = RegNum;
-      ++Count;
-    } else {
-      // Can't merge this in. Try merge the earlier ones first.
-      // We need to compute BaseKill here because the MemOps may have been
-      // reordered.
-      BaseKill = Loc->killsRegister(Base);
-
-      MergeOpsUpdate(MBB, MemOps, SIndex, i, insertAfter, SOffset, Base,
-                     BaseKill, Opcode, Pred, PredReg, Scratch, dl, Merges);
-      MergeLDR_STR(MBB, i, Base, Opcode, Size, Pred, PredReg, Scratch,
-                   MemOps, Merges);
-      return;
+  // Determine range between the earliest removed instruction and the new one.
+  if (EarliestAtBegin)
+    EarliestI = MBB.begin();
+  else
+    EarliestI = std::next(EarliestI);
+  auto FixupRange = make_range(EarliestI, iterator(Merged));
+
+  if (isLoadSingle(Opcode)) {
+    // If the previous loads defined a super-reg, then we have to mark earlier
+    // operands undef; Replicate the super-reg def on the merged instruction.
+    for (MachineInstr &MI : FixupRange) {
+      for (unsigned &ImpDefReg : ImpDefs) {
+        for (MachineOperand &MO : MI.implicit_operands()) {
+          if (!MO.isReg() || MO.getReg() != ImpDefReg)
+            continue;
+          if (MO.readsReg())
+            MO.setIsUndef();
+          else if (MO.isDef())
+            ImpDefReg = 0;
+        }
+      }
     }
 
-    if (MemOps[i].Position > MemOps[insertAfter].Position) {
-      insertAfter = i;
-      Loc = MemOps[i].MBBI;
+    MachineInstrBuilder MIB(*Merged->getParent()->getParent(), Merged);
+    for (unsigned ImpDef : ImpDefs)
+      MIB.addReg(ImpDef, RegState::ImplicitDefine);
+  } else {
+    // Remove kill flags: We are possibly storing the values later now.
+    assert(isi32Store(Opcode) || Opcode == ARM::VSTRS || Opcode == ARM::VSTRD);
+    for (MachineInstr &MI : FixupRange) {
+      for (MachineOperand &MO : MI.uses()) {
+        if (!MO.isReg() || !MO.isKill())
+          continue;
+        if (UsedRegs.count(MO.getReg()))
+          MO.setIsKill(false);
+      }
     }
+    assert(ImpDefs.empty());
   }
 
-  BaseKill =  Loc->killsRegister(Base);
-  MergeOpsUpdate(MBB, MemOps, SIndex, MemOps.size(), insertAfter, SOffset,
-                 Base, BaseKill, Opcode, Pred, PredReg, Scratch, dl, Merges);
+  return Merged;
 }
 
-static bool isMatchingDecrement(MachineInstr *MI, unsigned Base,
-                                unsigned Bytes, unsigned Limit,
-                                ARMCC::CondCodes Pred, unsigned PredReg) {
-  unsigned MyPredReg = 0;
-  if (!MI)
-    return false;
-
-  bool CheckCPSRDef = false;
-  switch (MI->getOpcode()) {
-  default: return false;
-  case ARM::tSUBi8:
-  case ARM::t2SUBri:
-  case ARM::SUBri:
-    CheckCPSRDef = true;
-  // fallthrough
-  case ARM::tSUBspi:
-    break;
-  }
-
-  // Make sure the offset fits in 8 bits.
-  if (Bytes == 0 || (Limit && Bytes >= Limit))
-    return false;
-
-  unsigned Scale = (MI->getOpcode() == ARM::tSUBspi ||
-                    MI->getOpcode() == ARM::tSUBi8) ? 4 : 1; // FIXME
-  if (!(MI->getOperand(0).getReg() == Base &&
-        MI->getOperand(1).getReg() == Base &&
-        (MI->getOperand(2).getImm() * Scale) == Bytes &&
-        getInstrPredicate(MI, MyPredReg) == Pred &&
-        MyPredReg == PredReg))
-    return false;
-
-  return CheckCPSRDef ? !definesCPSR(MI) : true;
+static bool isValidLSDoubleOffset(int Offset) {
+  unsigned Value = abs(Offset);
+  // t2LDRDi8/t2STRDi8 supports an 8 bit immediate which is internally
+  // multiplied by 4.
+  return (Value % 4) == 0 && Value < 1024;
 }
 
-static bool isMatchingIncrement(MachineInstr *MI, unsigned Base,
-                                unsigned Bytes, unsigned Limit,
-                                ARMCC::CondCodes Pred, unsigned PredReg) {
-  unsigned MyPredReg = 0;
-  if (!MI)
-    return false;
-
-  bool CheckCPSRDef = false;
-  switch (MI->getOpcode()) {
-  default: return false;
-  case ARM::tADDi8:
-  case ARM::t2ADDri:
-  case ARM::ADDri:
-    CheckCPSRDef = true;
-  // fallthrough
-  case ARM::tADDspi:
-    break;
-  }
-
-  if (Bytes == 0 || (Limit && Bytes >= Limit))
-    // Make sure the offset fits in 8 bits.
-    return false;
+/// Find candidates for load/store multiple merge in list of MemOpQueueEntries.
+void ARMLoadStoreOpt::FormCandidates(const MemOpQueue &MemOps) {
+  const MachineInstr *FirstMI = MemOps[0].MI;
+  unsigned Opcode = FirstMI->getOpcode();
+  bool isNotVFP = isi32Load(Opcode) || isi32Store(Opcode);
+  unsigned Size = getLSMultipleTransferSize(FirstMI);
+
+  unsigned SIndex = 0;
+  unsigned EIndex = MemOps.size();
+  do {
+    // Look at the first instruction.
+    const MachineInstr *MI = MemOps[SIndex].MI;
+    int Offset = MemOps[SIndex].Offset;
+    const MachineOperand &PMO = getLoadStoreRegOp(*MI);
+    unsigned PReg = PMO.getReg();
+    unsigned PRegNum = PMO.isUndef() ? UINT_MAX : TRI->getEncodingValue(PReg);
+    unsigned Latest = SIndex;
+    unsigned Earliest = SIndex;
+    unsigned Count = 1;
+    bool CanMergeToLSDouble =
+      STI->isThumb2() && isNotVFP && isValidLSDoubleOffset(Offset);
+    // ARM errata 602117: LDRD with base in list may result in incorrect base
+    // register when interrupted or faulted.
+    if (STI->isCortexM3() && isi32Load(Opcode) &&
+        PReg == getLoadStoreBaseOp(*MI).getReg())
+      CanMergeToLSDouble = false;
+
+    bool CanMergeToLSMulti = true;
+    // On swift vldm/vstm starting with an odd register number as that needs
+    // more uops than single vldrs.
+    if (STI->isSwift() && !isNotVFP && (PRegNum % 2) == 1)
+      CanMergeToLSMulti = false;
+
+    // LDRD/STRD do not allow SP/PC. LDM/STM do not support it or have it
+    // deprecated; LDM to PC is fine but cannot happen here.
+    if (PReg == ARM::SP || PReg == ARM::PC)
+      CanMergeToLSMulti = CanMergeToLSDouble = false;
+
+    // Merge following instructions where possible.
+    for (unsigned I = SIndex+1; I < EIndex; ++I, ++Count) {
+      int NewOffset = MemOps[I].Offset;
+      if (NewOffset != Offset + (int)Size)
+        break;
+      const MachineOperand &MO = getLoadStoreRegOp(*MemOps[I].MI);
+      unsigned Reg = MO.getReg();
+      if (Reg == ARM::SP || Reg == ARM::PC)
+        break;
 
-  unsigned Scale = (MI->getOpcode() == ARM::tADDspi ||
-                    MI->getOpcode() == ARM::tADDi8) ? 4 : 1; // FIXME
-  if (!(MI->getOperand(0).getReg() == Base &&
-        MI->getOperand(1).getReg() == Base &&
-        (MI->getOperand(2).getImm() * Scale) == Bytes &&
-        getInstrPredicate(MI, MyPredReg) == Pred &&
-        MyPredReg == PredReg))
-    return false;
+      // See if the current load/store may be part of a multi load/store.
+      unsigned RegNum = MO.isUndef() ? UINT_MAX : TRI->getEncodingValue(Reg);
+      bool PartOfLSMulti = CanMergeToLSMulti;
+      if (PartOfLSMulti) {
+        // Register numbers must be in ascending order.
+        if (RegNum <= PRegNum)
+          PartOfLSMulti = false;
+        // For VFP / NEON load/store multiples, the registers must be
+        // consecutive and within the limit on the number of registers per
+        // instruction.
+        else if (!isNotVFP && RegNum != PRegNum+1)
+          PartOfLSMulti = false;
+      }
+      // See if the current load/store may be part of a double load/store.
+      bool PartOfLSDouble = CanMergeToLSDouble && Count <= 1;
 
-  return CheckCPSRDef ? !definesCPSR(MI) : true;
-}
+      if (!PartOfLSMulti && !PartOfLSDouble)
+        break;
+      CanMergeToLSMulti &= PartOfLSMulti;
+      CanMergeToLSDouble &= PartOfLSDouble;
+      // Track MemOp with latest and earliest position (Positions are
+      // counted in reverse).
+      unsigned Position = MemOps[I].Position;
+      if (Position < MemOps[Latest].Position)
+        Latest = I;
+      else if (Position > MemOps[Earliest].Position)
+        Earliest = I;
+      // Prepare for next MemOp.
+      Offset += Size;
+      PRegNum = RegNum;
+    }
 
-static inline unsigned getLSMultipleTransferSize(MachineInstr *MI) {
-  switch (MI->getOpcode()) {
-  default: return 0;
-  case ARM::LDRi12:
-  case ARM::STRi12:
-  case ARM::tLDRi:
-  case ARM::tSTRi:
-  case ARM::tLDRspi:
-  case ARM::tSTRspi:
-  case ARM::t2LDRi8:
-  case ARM::t2LDRi12:
-  case ARM::t2STRi8:
-  case ARM::t2STRi12:
-  case ARM::VLDRS:
-  case ARM::VSTRS:
-    return 4;
-  case ARM::VLDRD:
-  case ARM::VSTRD:
-    return 8;
-  case ARM::LDMIA:
-  case ARM::LDMDA:
-  case ARM::LDMDB:
-  case ARM::LDMIB:
-  case ARM::STMIA:
-  case ARM::STMDA:
-  case ARM::STMDB:
-  case ARM::STMIB:
-  case ARM::tLDMIA:
-  case ARM::tLDMIA_UPD:
-  case ARM::tSTMIA_UPD:
-  case ARM::t2LDMIA:
-  case ARM::t2LDMDB:
-  case ARM::t2STMIA:
-  case ARM::t2STMDB:
-  case ARM::VLDMSIA:
-  case ARM::VSTMSIA:
-    return (MI->getNumOperands() - MI->getDesc().getNumOperands() + 1) * 4;
-  case ARM::VLDMDIA:
-  case ARM::VSTMDIA:
-    return (MI->getNumOperands() - MI->getDesc().getNumOperands() + 1) * 8;
-  }
+    // Form a candidate from the Ops collected so far.
+    MergeCandidate *Candidate = new(Allocator.Allocate()) MergeCandidate;
+    for (unsigned C = SIndex, CE = SIndex + Count; C < CE; ++C)
+      Candidate->Instrs.push_back(MemOps[C].MI);
+    Candidate->LatestMIIdx = Latest - SIndex;
+    Candidate->EarliestMIIdx = Earliest - SIndex;
+    Candidate->InsertPos = MemOps[Latest].Position;
+    if (Count == 1)
+      CanMergeToLSMulti = CanMergeToLSDouble = false;
+    Candidate->CanMergeToLSMulti = CanMergeToLSMulti;
+    Candidate->CanMergeToLSDouble = CanMergeToLSDouble;
+    Candidates.push_back(Candidate);
+    // Continue after the chain.
+    SIndex += Count;
+  } while (SIndex < EIndex);
 }
 
 static unsigned getUpdatingLSMultipleOpcode(unsigned Opc,
@@ -1085,8 +1076,77 @@ static unsigned getUpdatingLSMultipleOpcode(unsigned Opc,
   }
 }
 
-/// MergeBaseUpdateLSMultiple - Fold proceeding/trailing inc/dec of base
-/// register into the LDM/STM/VLDM{D|S}/VSTM{D|S} op when possible:
+/// Check if the given instruction increments or decrements a register and
+/// return the amount it is incremented/decremented. Returns 0 if the CPSR flags
+/// generated by the instruction are possibly read as well.
+static int isIncrementOrDecrement(const MachineInstr &MI, unsigned Reg,
+                                  ARMCC::CondCodes Pred, unsigned PredReg) {
+  bool CheckCPSRDef;
+  int Scale;
+  switch (MI.getOpcode()) {
+  case ARM::tADDi8:  Scale =  4; CheckCPSRDef = true; break;
+  case ARM::tSUBi8:  Scale = -4; CheckCPSRDef = true; break;
+  case ARM::t2SUBri:
+  case ARM::SUBri:   Scale = -1; CheckCPSRDef = true; break;
+  case ARM::t2ADDri:
+  case ARM::ADDri:   Scale =  1; CheckCPSRDef = true; break;
+  case ARM::tADDspi: Scale =  4; CheckCPSRDef = false; break;
+  case ARM::tSUBspi: Scale = -4; CheckCPSRDef = false; break;
+  default: return 0;
+  }
+
+  unsigned MIPredReg;
+  if (MI.getOperand(0).getReg() != Reg ||
+      MI.getOperand(1).getReg() != Reg ||
+      getInstrPredicate(&MI, MIPredReg) != Pred ||
+      MIPredReg != PredReg)
+    return 0;
+
+  if (CheckCPSRDef && definesCPSR(&MI))
+    return 0;
+  return MI.getOperand(2).getImm() * Scale;
+}
+
+/// Searches for an increment or decrement of \p Reg before \p MBBI.
+static MachineBasicBlock::iterator
+findIncDecBefore(MachineBasicBlock::iterator MBBI, unsigned Reg,
+                 ARMCC::CondCodes Pred, unsigned PredReg, int &Offset) {
+  Offset = 0;
+  MachineBasicBlock &MBB = *MBBI->getParent();
+  MachineBasicBlock::iterator BeginMBBI = MBB.begin();
+  MachineBasicBlock::iterator EndMBBI = MBB.end();
+  if (MBBI == BeginMBBI)
+    return EndMBBI;
+
+  // Skip debug values.
+  MachineBasicBlock::iterator PrevMBBI = std::prev(MBBI);
+  while (PrevMBBI->isDebugValue() && PrevMBBI != BeginMBBI)
+    --PrevMBBI;
+
+  Offset = isIncrementOrDecrement(*PrevMBBI, Reg, Pred, PredReg);
+  return Offset == 0 ? EndMBBI : PrevMBBI;
+}
+
+/// Searches for a increment or decrement of \p Reg after \p MBBI.
+static MachineBasicBlock::iterator
+findIncDecAfter(MachineBasicBlock::iterator MBBI, unsigned Reg,
+                ARMCC::CondCodes Pred, unsigned PredReg, int &Offset) {
+  Offset = 0;
+  MachineBasicBlock &MBB = *MBBI->getParent();
+  MachineBasicBlock::iterator EndMBBI = MBB.end();
+  MachineBasicBlock::iterator NextMBBI = std::next(MBBI);
+  // Skip debug values.
+  while (NextMBBI != EndMBBI && NextMBBI->isDebugValue())
+    ++NextMBBI;
+  if (NextMBBI == EndMBBI)
+    return EndMBBI;
+
+  Offset = isIncrementOrDecrement(*NextMBBI, Reg, Pred, PredReg);
+  return Offset == 0 ? EndMBBI : NextMBBI;
+}
+
+/// Fold proceeding/trailing inc/dec of base register into the
+/// LDM/STM/VLDM{D|S}/VSTM{D|S} op when possible:
 ///
 /// stmia rn, <ra, rb, rc>
 /// rn := rn + 4 * 3;
@@ -1097,22 +1157,17 @@ static unsigned getUpdatingLSMultipleOpcode(unsigned Opc,
 /// ldmia rn, <ra, rb, rc>
 /// =>
 /// ldmdb rn!, <ra, rb, rc>
-/// @LOCALMOD This is especially useful for rn == sp
-bool ARMLoadStoreOpt::MergeBaseUpdateLSMultiple(MachineBasicBlock &MBB,
-                                               MachineBasicBlock::iterator MBBI,
-                                               bool &Advance,
-                                               MachineBasicBlock::iterator &I) {
+bool ARMLoadStoreOpt::MergeBaseUpdateLSMultiple(MachineInstr *MI) {
   // Thumb1 is already using updating loads/stores.
   if (isThumb1) return false;
 
-  MachineInstr *MI = MBBI;
-  unsigned Base = MI->getOperand(0).getReg();
-  bool BaseKill = MI->getOperand(0).isKill();
-  unsigned Bytes = getLSMultipleTransferSize(MI);
+  const MachineOperand &BaseOP = MI->getOperand(0);
+  unsigned Base = BaseOP.getReg();
+  bool BaseKill = BaseOP.isKill();
   unsigned PredReg = 0;
   ARMCC::CondCodes Pred = getInstrPredicate(MI, PredReg);
-  int Opcode = MI->getOpcode();
-  DebugLoc dl = MI->getDebugLoc();
+  unsigned Opcode = MI->getOpcode();
+  DebugLoc DL = MI->getDebugLoc();
 
   // Can't use an updating ld/st if the base register is also a dest
   // register. e.g. ldmdb r0!, {r0, r1, r2}. The behavior is undefined.
@@ -1120,55 +1175,27 @@ bool ARMLoadStoreOpt::MergeBaseUpdateLSMultiple(MachineBasicBlock &MBB,
     if (MI->getOperand(i).getReg() == Base)
       return false;
 
-  bool DoMerge = false;
-  ARM_AM::AMSubMode Mode = ARM_AM::getLoadStoreMultipleSubMode(Opcode);
-
-  // Try merging with the previous instruction.
-  MachineBasicBlock::iterator BeginMBBI = MBB.begin();
-  if (MBBI != BeginMBBI) {
-    MachineBasicBlock::iterator PrevMBBI = std::prev(MBBI);
-    while (PrevMBBI != BeginMBBI && PrevMBBI->isDebugValue())
-      --PrevMBBI;
-    if (Mode == ARM_AM::ia &&
-        isMatchingDecrement(PrevMBBI, Base, Bytes, 0, Pred, PredReg)) {
-      Mode = ARM_AM::db;
-      DoMerge = true;
-    } else if (Mode == ARM_AM::ib &&
-               isMatchingDecrement(PrevMBBI, Base, Bytes, 0, Pred, PredReg)) {
-      Mode = ARM_AM::da;
-      DoMerge = true;
-    }
-    if (DoMerge)
-      MBB.erase(PrevMBBI);
-  }
-
-  // Try merging with the next instruction.
-  MachineBasicBlock::iterator EndMBBI = MBB.end();
-  if (!DoMerge && MBBI != EndMBBI) {
-    MachineBasicBlock::iterator NextMBBI = std::next(MBBI);
-    while (NextMBBI != EndMBBI && NextMBBI->isDebugValue())
-      ++NextMBBI;
-    if ((Mode == ARM_AM::ia || Mode == ARM_AM::ib) &&
-        isMatchingIncrement(NextMBBI, Base, Bytes, 0, Pred, PredReg)) {
-      DoMerge = true;
-    } else if ((Mode == ARM_AM::da || Mode == ARM_AM::db) &&
-               isMatchingDecrement(NextMBBI, Base, Bytes, 0, Pred, PredReg)) {
-      DoMerge = true;
-    }
-    if (DoMerge) {
-      if (NextMBBI == I) {
-        Advance = true;
-        ++I;
-      }
-      MBB.erase(NextMBBI);
-    }
+  int Bytes = getLSMultipleTransferSize(MI);
+  MachineBasicBlock &MBB = *MI->getParent();
+  MachineBasicBlock::iterator MBBI(MI);
+  int Offset;
+  MachineBasicBlock::iterator MergeInstr
+    = findIncDecBefore(MBBI, Base, Pred, PredReg, Offset);
+  ARM_AM::AMSubMode Mode = getLoadStoreMultipleSubMode(Opcode);
+  if (Mode == ARM_AM::ia && Offset == -Bytes) {
+    Mode = ARM_AM::db;
+  } else if (Mode == ARM_AM::ib && Offset == -Bytes) {
+    Mode = ARM_AM::da;
+  } else {
+    MergeInstr = findIncDecAfter(MBBI, Base, Pred, PredReg, Offset);
+    if (((Mode != ARM_AM::ia && Mode != ARM_AM::ib) || Offset != Bytes) &&
+        ((Mode != ARM_AM::da && Mode != ARM_AM::db) || Offset != -Bytes))
+      return false;
   }
-
-  if (!DoMerge)
-    return false;
+  MBB.erase(MergeInstr);
 
   unsigned NewOpc = getUpdatingLSMultipleOpcode(Opcode, Mode);
-  MachineInstrBuilder MIB = BuildMI(MBB, MBBI, dl, TII->get(NewOpc))
+  MachineInstrBuilder MIB = BuildMI(MBB, MBBI, DL, TII->get(NewOpc))
     .addReg(Base, getDefRegState(true)) // WB base register
     .addReg(Base, getKillRegState(BaseKill))
     .addImm(Pred).addReg(PredReg);
@@ -1234,23 +1261,17 @@ static unsigned getPostIndexedLoadStoreOpcode(unsigned Opc,
   }
 }
 
-/// MergeBaseUpdateLoadStore - Fold proceeding/trailing inc/dec of base
-/// register into the LDR/STR/FLD{D|S}/FST{D|S} op when possible:
-bool ARMLoadStoreOpt::MergeBaseUpdateLoadStore(MachineBasicBlock &MBB,
-                                               MachineBasicBlock::iterator MBBI,
-                                               const TargetInstrInfo *TII,
-                                               bool &Advance,
-                                               MachineBasicBlock::iterator &I) {
+/// Fold proceeding/trailing inc/dec of base register into the
+/// LDR/STR/FLD{D|S}/FST{D|S} op when possible:
+bool ARMLoadStoreOpt::MergeBaseUpdateLoadStore(MachineInstr *MI) {
   // Thumb1 doesn't have updating LDR/STR.
   // FIXME: Use LDM/STM with single register instead.
   if (isThumb1) return false;
 
-  MachineInstr *MI = MBBI;
-  unsigned Base = MI->getOperand(1).getReg();
-  bool BaseKill = MI->getOperand(1).isKill();
-  unsigned Bytes = getLSMultipleTransferSize(MI);
-  int Opcode = MI->getOpcode();
-  DebugLoc dl = MI->getDebugLoc();
+  unsigned Base = getLoadStoreBaseOp(*MI).getReg();
+  bool BaseKill = getLoadStoreBaseOp(*MI).isKill();
+  unsigned Opcode = MI->getOpcode();
+  DebugLoc DL = MI->getDebugLoc();
   bool isAM5 = (Opcode == ARM::VLDRD || Opcode == ARM::VLDRS ||
                 Opcode == ARM::VSTRD || Opcode == ARM::VSTRS);
   bool isAM2 = (Opcode == ARM::LDRi12 || Opcode == ARM::STRi12);
@@ -1260,7 +1281,6 @@ bool ARMLoadStoreOpt::MergeBaseUpdateLoadStore(MachineBasicBlock &MBB,
   if (isAM5 && ARM_AM::getAM5Offset(MI->getOperand(2).getImm()) != 0)
     return false;
 
-  bool isLd = isi32Load(Opcode) || Opcode == ARM::VLDRS || Opcode == ARM::VLDRD;
   // Can't do the merge if the destination register is the same as the would-be
   // writeback register.
   if (MI->getOperand(0).getReg() == Base)
@@ -1268,64 +1288,38 @@ bool ARMLoadStoreOpt::MergeBaseUpdateLoadStore(MachineBasicBlock &MBB,
 
   unsigned PredReg = 0;
   ARMCC::CondCodes Pred = getInstrPredicate(MI, PredReg);
-  bool DoMerge = false;
-  ARM_AM::AddrOpc AddSub = ARM_AM::add;
-  unsigned NewOpc = 0;
-  // AM2 - 12 bits, thumb2 - 8 bits.
-  unsigned Limit = isAM5 ? 0 : (isAM2 ? 0x1000 : 0x100);
-
-  // Try merging with the previous instruction.
-  MachineBasicBlock::iterator BeginMBBI = MBB.begin();
-  if (MBBI != BeginMBBI) {
-    MachineBasicBlock::iterator PrevMBBI = std::prev(MBBI);
-    while (PrevMBBI != BeginMBBI && PrevMBBI->isDebugValue())
-      --PrevMBBI;
-    if (isMatchingDecrement(PrevMBBI, Base, Bytes, Limit, Pred, PredReg)) {
-      DoMerge = true;
-      AddSub = ARM_AM::sub;
-    } else if (!isAM5 &&
-               isMatchingIncrement(PrevMBBI, Base, Bytes, Limit,Pred,PredReg)) {
-      DoMerge = true;
-    }
-    if (DoMerge) {
-      NewOpc = getPreIndexedLoadStoreOpcode(Opcode, AddSub);
-      MBB.erase(PrevMBBI);
-    }
-  }
-
-  // Try merging with the next instruction.
-  MachineBasicBlock::iterator EndMBBI = MBB.end();
-  if (!DoMerge && MBBI != EndMBBI) {
-    MachineBasicBlock::iterator NextMBBI = std::next(MBBI);
-    while (NextMBBI != EndMBBI && NextMBBI->isDebugValue())
-      ++NextMBBI;
-    if (!isAM5 &&
-        isMatchingDecrement(NextMBBI, Base, Bytes, Limit, Pred, PredReg)) {
-      DoMerge = true;
-      AddSub = ARM_AM::sub;
-    } else if (isMatchingIncrement(NextMBBI, Base, Bytes, Limit,Pred,PredReg)) {
-      DoMerge = true;
-    }
-    if (DoMerge) {
-      NewOpc = getPostIndexedLoadStoreOpcode(Opcode, AddSub);
-      if (NextMBBI == I) {
-        Advance = true;
-        ++I;
-      }
-      MBB.erase(NextMBBI);
-    }
+  int Bytes = getLSMultipleTransferSize(MI);
+  MachineBasicBlock &MBB = *MI->getParent();
+  MachineBasicBlock::iterator MBBI(MI);
+  int Offset;
+  MachineBasicBlock::iterator MergeInstr
+    = findIncDecBefore(MBBI, Base, Pred, PredReg, Offset);
+  unsigned NewOpc;
+  if (!isAM5 && Offset == Bytes) {
+    NewOpc = getPreIndexedLoadStoreOpcode(Opcode, ARM_AM::add);
+  } else if (Offset == -Bytes) {
+    NewOpc = getPreIndexedLoadStoreOpcode(Opcode, ARM_AM::sub);
+  } else {
+    MergeInstr = findIncDecAfter(MBBI, Base, Pred, PredReg, Offset);
+    if (Offset == Bytes) {
+      NewOpc = getPostIndexedLoadStoreOpcode(Opcode, ARM_AM::add);
+    } else if (!isAM5 && Offset == -Bytes) {
+      NewOpc = getPostIndexedLoadStoreOpcode(Opcode, ARM_AM::sub);
+    } else
+      return false;
   }
+  MBB.erase(MergeInstr);
 
-  if (!DoMerge)
-    return false;
+  ARM_AM::AddrOpc AddSub = Offset < 0 ? ARM_AM::sub : ARM_AM::add;
 
+  bool isLd = isLoadSingle(Opcode);
   if (isAM5) {
     // VLDM[SD]_UPD, VSTM[SD]_UPD
     // (There are no base-updating versions of VLDR/VSTR instructions, but the
     // updating load/store-multiple instructions can be used with only one
     // register.)
     MachineOperand &MO = MI->getOperand(0);
-    BuildMI(MBB, MBBI, dl, TII->get(NewOpc))
+    BuildMI(MBB, MBBI, DL, TII->get(NewOpc))
       .addReg(Base, getDefRegState(true)) // WB base register
       .addReg(Base, getKillRegState(isLd ? BaseKill : false))
       .addImm(Pred).addReg(PredReg)
@@ -1335,20 +1329,18 @@ bool ARMLoadStoreOpt::MergeBaseUpdateLoadStore(MachineBasicBlock &MBB,
     if (isAM2) {
       // LDR_PRE, LDR_POST
       if (NewOpc == ARM::LDR_PRE_IMM || NewOpc == ARM::LDRB_PRE_IMM) {
-        int Offset = AddSub == ARM_AM::sub ? -Bytes : Bytes;
-        BuildMI(MBB, MBBI, dl, TII->get(NewOpc), MI->getOperand(0).getReg())
+        BuildMI(MBB, MBBI, DL, TII->get(NewOpc), MI->getOperand(0).getReg())
           .addReg(Base, RegState::Define)
           .addReg(Base).addImm(Offset).addImm(Pred).addReg(PredReg);
       } else {
-        int Offset = ARM_AM::getAM2Opc(AddSub, Bytes, ARM_AM::no_shift);
-        BuildMI(MBB, MBBI, dl, TII->get(NewOpc), MI->getOperand(0).getReg())
+        int Imm = ARM_AM::getAM2Opc(AddSub, Bytes, ARM_AM::no_shift);
+        BuildMI(MBB, MBBI, DL, TII->get(NewOpc), MI->getOperand(0).getReg())
           .addReg(Base, RegState::Define)
-          .addReg(Base).addReg(0).addImm(Offset).addImm(Pred).addReg(PredReg);
+          .addReg(Base).addReg(0).addImm(Imm).addImm(Pred).addReg(PredReg);
       }
     } else {
-      int Offset = AddSub == ARM_AM::sub ? -Bytes : Bytes;
       // t2LDR_PRE, t2LDR_POST
-      BuildMI(MBB, MBBI, dl, TII->get(NewOpc), MI->getOperand(0).getReg())
+      BuildMI(MBB, MBBI, DL, TII->get(NewOpc), MI->getOperand(0).getReg())
         .addReg(Base, RegState::Define)
         .addReg(Base).addImm(Offset).addImm(Pred).addReg(PredReg);
     }
@@ -1358,15 +1350,14 @@ bool ARMLoadStoreOpt::MergeBaseUpdateLoadStore(MachineBasicBlock &MBB,
     // the vestigal zero-reg offset register. When that's fixed, this clause
     // can be removed entirely.
     if (isAM2 && NewOpc == ARM::STR_POST_IMM) {
-      int Offset = ARM_AM::getAM2Opc(AddSub, Bytes, ARM_AM::no_shift);
+      int Imm = ARM_AM::getAM2Opc(AddSub, Bytes, ARM_AM::no_shift);
       // STR_PRE, STR_POST
-      BuildMI(MBB, MBBI, dl, TII->get(NewOpc), Base)
+      BuildMI(MBB, MBBI, DL, TII->get(NewOpc), Base)
         .addReg(MO.getReg(), getKillRegState(MO.isKill()))
-        .addReg(Base).addReg(0).addImm(Offset).addImm(Pred).addReg(PredReg);
+        .addReg(Base).addReg(0).addImm(Imm).addImm(Pred).addReg(PredReg);
     } else {
-      int Offset = AddSub == ARM_AM::sub ? -Bytes : Bytes;
       // t2STR_PRE, t2STR_POST
-      BuildMI(MBB, MBBI, dl, TII->get(NewOpc), Base)
+      BuildMI(MBB, MBBI, DL, TII->get(NewOpc), Base)
         .addReg(MO.getReg(), getKillRegState(MO.isKill()))
         .addReg(Base).addImm(Offset).addImm(Pred).addReg(PredReg);
     }
@@ -1376,8 +1367,68 @@ bool ARMLoadStoreOpt::MergeBaseUpdateLoadStore(MachineBasicBlock &MBB,
   return true;
 }
 
-/// isMemoryOp - Returns true if instruction is a memory operation that this
-/// pass is capable of operating on.
+bool ARMLoadStoreOpt::MergeBaseUpdateLSDouble(MachineInstr &MI) const {
+  unsigned Opcode = MI.getOpcode();
+  assert((Opcode == ARM::t2LDRDi8 || Opcode == ARM::t2STRDi8) &&
+         "Must have t2STRDi8 or t2LDRDi8");
+  if (MI.getOperand(3).getImm() != 0)
+    return false;
+
+  // Behaviour for writeback is undefined if base register is the same as one
+  // of the others.
+  const MachineOperand &BaseOp = MI.getOperand(2);
+  unsigned Base = BaseOp.getReg();
+  const MachineOperand &Reg0Op = MI.getOperand(0);
+  const MachineOperand &Reg1Op = MI.getOperand(1);
+  if (Reg0Op.getReg() == Base || Reg1Op.getReg() == Base)
+    return false;
+
+  unsigned PredReg;
+  ARMCC::CondCodes Pred = getInstrPredicate(&MI, PredReg);
+  MachineBasicBlock::iterator MBBI(MI);
+  MachineBasicBlock &MBB = *MI.getParent();
+  int Offset;
+  MachineBasicBlock::iterator MergeInstr = findIncDecBefore(MBBI, Base, Pred,
+                                                            PredReg, Offset);
+  unsigned NewOpc;
+  if (Offset == 8 || Offset == -8) {
+    NewOpc = Opcode == ARM::t2LDRDi8 ? ARM::t2LDRD_PRE : ARM::t2STRD_PRE;
+  } else {
+    MergeInstr = findIncDecAfter(MBBI, Base, Pred, PredReg, Offset);
+    if (Offset == 8 || Offset == -8) {
+      NewOpc = Opcode == ARM::t2LDRDi8 ? ARM::t2LDRD_POST : ARM::t2STRD_POST;
+    } else
+      return false;
+  }
+  MBB.erase(MergeInstr);
+
+  DebugLoc DL = MI.getDebugLoc();
+  MachineInstrBuilder MIB = BuildMI(MBB, MBBI, DL, TII->get(NewOpc));
+  if (NewOpc == ARM::t2LDRD_PRE || NewOpc == ARM::t2LDRD_POST) {
+    MIB.addOperand(Reg0Op).addOperand(Reg1Op)
+       .addReg(BaseOp.getReg(), RegState::Define);
+  } else {
+    assert(NewOpc == ARM::t2STRD_PRE || NewOpc == ARM::t2STRD_POST);
+    MIB.addReg(BaseOp.getReg(), RegState::Define)
+       .addOperand(Reg0Op).addOperand(Reg1Op);
+  }
+  MIB.addReg(BaseOp.getReg(), RegState::Kill)
+     .addImm(Offset).addImm(Pred).addReg(PredReg);
+  assert(TII->get(Opcode).getNumOperands() == 6 &&
+         TII->get(NewOpc).getNumOperands() == 7 &&
+         "Unexpected number of operands in Opcode specification.");
+
+  // Transfer implicit operands.
+  for (const MachineOperand &MO : MI.implicit_operands())
+    MIB.addOperand(MO);
+  MIB->setMemRefs(MI.memoperands_begin(), MI.memoperands_end());
+
+  MBB.erase(MBBI);
+  return true;
+}
+
+/// Returns true if instruction is a memory operation that this pass is capable
+/// of operating on.
 static bool isMemoryOp(const MachineInstr *MI) {
   // When no memory operands are present, conservatively assume unaligned,
   // volatile, unfoldable.
@@ -1407,7 +1458,7 @@ static bool isMemoryOp(const MachineInstr *MI) {
       MI->getOperand(1).isUndef())
     return false;
 
-  int Opcode = MI->getOpcode();
+  unsigned Opcode = MI->getOpcode();
   switch (Opcode) {
   default: break;
   case ARM::VLDRS:
@@ -1431,26 +1482,10 @@ static bool isMemoryOp(const MachineInstr *MI) {
   return false;
 }
 
-/// AdvanceRS - Advance register scavenger to just before the earliest memory
-/// op that is being merged.
-void ARMLoadStoreOpt::AdvanceRS(MachineBasicBlock &MBB, MemOpQueue &MemOps) {
-  MachineBasicBlock::iterator Loc = MemOps[0].MBBI;
-  unsigned Position = MemOps[0].Position;
-  for (unsigned i = 1, e = MemOps.size(); i != e; ++i) {
-    if (MemOps[i].Position < Position) {
-      Position = MemOps[i].Position;
-      Loc = MemOps[i].MBBI;
-    }
-  }
-
-  if (Loc != MBB.begin())
-    RS->forward(std::prev(Loc));
-}
-
 static void InsertLDR_STR(MachineBasicBlock &MBB,
                           MachineBasicBlock::iterator &MBBI,
                           int Offset, bool isDef,
-                          DebugLoc dl, unsigned NewOpc,
+                          DebugLoc DL, unsigned NewOpc,
                           unsigned Reg, bool RegDeadKill, bool RegUndef,
                           unsigned BaseReg, bool BaseKill, bool BaseUndef,
                           bool OffKill, bool OffUndef,
@@ -1475,308 +1510,276 @@ bool ARMLoadStoreOpt::FixInvalidRegPairOp(MachineBasicBlock &MBB,
                                           MachineBasicBlock::iterator &MBBI) {
   MachineInstr *MI = &*MBBI;
   unsigned Opcode = MI->getOpcode();
-  if (Opcode == ARM::LDRD || Opcode == ARM::STRD ||
-      Opcode == ARM::t2LDRDi8 || Opcode == ARM::t2STRDi8) {
-    const MachineOperand &BaseOp = MI->getOperand(2);
-    unsigned BaseReg = BaseOp.getReg();
-    unsigned EvenReg = MI->getOperand(0).getReg();
-    unsigned OddReg  = MI->getOperand(1).getReg();
-    unsigned EvenRegNum = TRI->getDwarfRegNum(EvenReg, false);
-    unsigned OddRegNum  = TRI->getDwarfRegNum(OddReg, false);
-    // ARM errata 602117: LDRD with base in list may result in incorrect base
-    // register when interrupted or faulted.
-    bool Errata602117 = EvenReg == BaseReg && STI->isCortexM3();
-    if (!Errata602117 &&
-        ((EvenRegNum & 1) == 0 && (EvenRegNum + 1) == OddRegNum))
-      return false;
+  if (Opcode != ARM::LDRD && Opcode != ARM::STRD && Opcode != ARM::t2LDRDi8)
+    return false;
 
-    MachineBasicBlock::iterator NewBBI = MBBI;
-    bool isT2 = Opcode == ARM::t2LDRDi8 || Opcode == ARM::t2STRDi8;
-    bool isLd = Opcode == ARM::LDRD || Opcode == ARM::t2LDRDi8;
-    bool EvenDeadKill = isLd ?
-      MI->getOperand(0).isDead() : MI->getOperand(0).isKill();
-    bool EvenUndef = MI->getOperand(0).isUndef();
-    bool OddDeadKill  = isLd ?
-      MI->getOperand(1).isDead() : MI->getOperand(1).isKill();
-    bool OddUndef = MI->getOperand(1).isUndef();
-    bool BaseKill = BaseOp.isKill();
-    bool BaseUndef = BaseOp.isUndef();
-    bool OffKill = isT2 ? false : MI->getOperand(3).isKill();
-    bool OffUndef = isT2 ? false : MI->getOperand(3).isUndef();
-    int OffImm = getMemoryOpOffset(MI);
-    unsigned PredReg = 0;
-    ARMCC::CondCodes Pred = getInstrPredicate(MI, PredReg);
-
-    if (OddRegNum > EvenRegNum && OffImm == 0) {
-      // Ascending register numbers and no offset. It's safe to change it to a
-      // ldm or stm.
-      unsigned NewOpc = (isLd)
-        ? (isT2 ? ARM::t2LDMIA : ARM::LDMIA)
-        : (isT2 ? ARM::t2STMIA : ARM::STMIA);
-      if (isLd) {
-        BuildMI(MBB, MBBI, MBBI->getDebugLoc(), TII->get(NewOpc))
-          .addReg(BaseReg, getKillRegState(BaseKill))
-          .addImm(Pred).addReg(PredReg)
-          .addReg(EvenReg, getDefRegState(isLd) | getDeadRegState(EvenDeadKill))
-          .addReg(OddReg,  getDefRegState(isLd) | getDeadRegState(OddDeadKill));
-        ++NumLDRD2LDM;
-      } else {
-        BuildMI(MBB, MBBI, MBBI->getDebugLoc(), TII->get(NewOpc))
-          .addReg(BaseReg, getKillRegState(BaseKill))
-          .addImm(Pred).addReg(PredReg)
-          .addReg(EvenReg,
-                  getKillRegState(EvenDeadKill) | getUndefRegState(EvenUndef))
-          .addReg(OddReg,
-                  getKillRegState(OddDeadKill)  | getUndefRegState(OddUndef));
-        ++NumSTRD2STM;
-      }
-      NewBBI = std::prev(MBBI);
+  const MachineOperand &BaseOp = MI->getOperand(2);
+  unsigned BaseReg = BaseOp.getReg();
+  unsigned EvenReg = MI->getOperand(0).getReg();
+  unsigned OddReg  = MI->getOperand(1).getReg();
+  unsigned EvenRegNum = TRI->getDwarfRegNum(EvenReg, false);
+  unsigned OddRegNum  = TRI->getDwarfRegNum(OddReg, false);
+
+  // ARM errata 602117: LDRD with base in list may result in incorrect base
+  // register when interrupted or faulted.
+  bool Errata602117 = EvenReg == BaseReg &&
+    (Opcode == ARM::LDRD || Opcode == ARM::t2LDRDi8) && STI->isCortexM3();
+  // ARM LDRD/STRD needs consecutive registers.
+  bool NonConsecutiveRegs = (Opcode == ARM::LDRD || Opcode == ARM::STRD) &&
+    (EvenRegNum % 2 != 0 || EvenRegNum + 1 != OddRegNum);
+
+  if (!Errata602117 && !NonConsecutiveRegs)
+    return false;
+
+  bool isT2 = Opcode == ARM::t2LDRDi8 || Opcode == ARM::t2STRDi8;
+  bool isLd = Opcode == ARM::LDRD || Opcode == ARM::t2LDRDi8;
+  bool EvenDeadKill = isLd ?
+    MI->getOperand(0).isDead() : MI->getOperand(0).isKill();
+  bool EvenUndef = MI->getOperand(0).isUndef();
+  bool OddDeadKill  = isLd ?
+    MI->getOperand(1).isDead() : MI->getOperand(1).isKill();
+  bool OddUndef = MI->getOperand(1).isUndef();
+  bool BaseKill = BaseOp.isKill();
+  bool BaseUndef = BaseOp.isUndef();
+  bool OffKill = isT2 ? false : MI->getOperand(3).isKill();
+  bool OffUndef = isT2 ? false : MI->getOperand(3).isUndef();
+  int OffImm = getMemoryOpOffset(MI);
+  unsigned PredReg = 0;
+  ARMCC::CondCodes Pred = getInstrPredicate(MI, PredReg);
+
+  if (OddRegNum > EvenRegNum && OffImm == 0) {
+    // Ascending register numbers and no offset. It's safe to change it to a
+    // ldm or stm.
+    unsigned NewOpc = (isLd)
+      ? (isT2 ? ARM::t2LDMIA : ARM::LDMIA)
+      : (isT2 ? ARM::t2STMIA : ARM::STMIA);
+    if (isLd) {
+      BuildMI(MBB, MBBI, MBBI->getDebugLoc(), TII->get(NewOpc))
+        .addReg(BaseReg, getKillRegState(BaseKill))
+        .addImm(Pred).addReg(PredReg)
+        .addReg(EvenReg, getDefRegState(isLd) | getDeadRegState(EvenDeadKill))
+        .addReg(OddReg,  getDefRegState(isLd) | getDeadRegState(OddDeadKill));
+      ++NumLDRD2LDM;
     } else {
-      // Split into two instructions.
-      unsigned NewOpc = (isLd)
-        ? (isT2 ? (OffImm < 0 ? ARM::t2LDRi8 : ARM::t2LDRi12) : ARM::LDRi12)
-        : (isT2 ? (OffImm < 0 ? ARM::t2STRi8 : ARM::t2STRi12) : ARM::STRi12);
-      // Be extra careful for thumb2. t2LDRi8 can't reference a zero offset,
-      // so adjust and use t2LDRi12 here for that.
-      unsigned NewOpc2 = (isLd)
-        ? (isT2 ? (OffImm+4 < 0 ? ARM::t2LDRi8 : ARM::t2LDRi12) : ARM::LDRi12)
-        : (isT2 ? (OffImm+4 < 0 ? ARM::t2STRi8 : ARM::t2STRi12) : ARM::STRi12);
-      DebugLoc dl = MBBI->getDebugLoc();
-      // If this is a load and base register is killed, it may have been
-      // re-defed by the load, make sure the first load does not clobber it.
-      if (isLd &&
-          (BaseKill || OffKill) &&
-          (TRI->regsOverlap(EvenReg, BaseReg))) {
-        assert(!TRI->regsOverlap(OddReg, BaseReg));
-        InsertLDR_STR(MBB, MBBI, OffImm+4, isLd, dl, NewOpc2,
-                      OddReg, OddDeadKill, false,
-                      BaseReg, false, BaseUndef, false, OffUndef,
-                      Pred, PredReg, TII, isT2);
-        NewBBI = std::prev(MBBI);
-        InsertLDR_STR(MBB, MBBI, OffImm, isLd, dl, NewOpc,
-                      EvenReg, EvenDeadKill, false,
-                      BaseReg, BaseKill, BaseUndef, OffKill, OffUndef,
-                      Pred, PredReg, TII, isT2);
-      } else {
-        if (OddReg == EvenReg && EvenDeadKill) {
-          // If the two source operands are the same, the kill marker is
-          // probably on the first one. e.g.
-          // t2STRDi8 %R5<kill>, %R5, %R9<kill>, 0, 14, %reg0
-          EvenDeadKill = false;
-          OddDeadKill = true;
-        }
-        // Never kill the base register in the first instruction.
-        if (EvenReg == BaseReg)
-          EvenDeadKill = false;
-        InsertLDR_STR(MBB, MBBI, OffImm, isLd, dl, NewOpc,
-                      EvenReg, EvenDeadKill, EvenUndef,
-                      BaseReg, false, BaseUndef, false, OffUndef,
-                      Pred, PredReg, TII, isT2);
-        NewBBI = std::prev(MBBI);
-        InsertLDR_STR(MBB, MBBI, OffImm+4, isLd, dl, NewOpc2,
-                      OddReg, OddDeadKill, OddUndef,
-                      BaseReg, BaseKill, BaseUndef, OffKill, OffUndef,
-                      Pred, PredReg, TII, isT2);
+      BuildMI(MBB, MBBI, MBBI->getDebugLoc(), TII->get(NewOpc))
+        .addReg(BaseReg, getKillRegState(BaseKill))
+        .addImm(Pred).addReg(PredReg)
+        .addReg(EvenReg,
+                getKillRegState(EvenDeadKill) | getUndefRegState(EvenUndef))
+        .addReg(OddReg,
+                getKillRegState(OddDeadKill)  | getUndefRegState(OddUndef));
+      ++NumSTRD2STM;
+    }
+  } else {
+    // Split into two instructions.
+    unsigned NewOpc = (isLd)
+      ? (isT2 ? (OffImm < 0 ? ARM::t2LDRi8 : ARM::t2LDRi12) : ARM::LDRi12)
+      : (isT2 ? (OffImm < 0 ? ARM::t2STRi8 : ARM::t2STRi12) : ARM::STRi12);
+    // Be extra careful for thumb2. t2LDRi8 can't reference a zero offset,
+    // so adjust and use t2LDRi12 here for that.
+    unsigned NewOpc2 = (isLd)
+      ? (isT2 ? (OffImm+4 < 0 ? ARM::t2LDRi8 : ARM::t2LDRi12) : ARM::LDRi12)
+      : (isT2 ? (OffImm+4 < 0 ? ARM::t2STRi8 : ARM::t2STRi12) : ARM::STRi12);
+    DebugLoc dl = MBBI->getDebugLoc();
+    // If this is a load and base register is killed, it may have been
+    // re-defed by the load, make sure the first load does not clobber it.
+    if (isLd &&
+        (BaseKill || OffKill) &&
+        (TRI->regsOverlap(EvenReg, BaseReg))) {
+      assert(!TRI->regsOverlap(OddReg, BaseReg));
+      InsertLDR_STR(MBB, MBBI, OffImm+4, isLd, dl, NewOpc2,
+                    OddReg, OddDeadKill, false,
+                    BaseReg, false, BaseUndef, false, OffUndef,
+                    Pred, PredReg, TII, isT2);
+      InsertLDR_STR(MBB, MBBI, OffImm, isLd, dl, NewOpc,
+                    EvenReg, EvenDeadKill, false,
+                    BaseReg, BaseKill, BaseUndef, OffKill, OffUndef,
+                    Pred, PredReg, TII, isT2);
+    } else {
+      if (OddReg == EvenReg && EvenDeadKill) {
+        // If the two source operands are the same, the kill marker is
+        // probably on the first one. e.g.
+        // t2STRDi8 %R5<kill>, %R5, %R9<kill>, 0, 14, %reg0
+        EvenDeadKill = false;
+        OddDeadKill = true;
       }
-      if (isLd)
-        ++NumLDRD2LDR;
-      else
-        ++NumSTRD2STR;
+      // Never kill the base register in the first instruction.
+      if (EvenReg == BaseReg)
+        EvenDeadKill = false;
+      InsertLDR_STR(MBB, MBBI, OffImm, isLd, dl, NewOpc,
+                    EvenReg, EvenDeadKill, EvenUndef,
+                    BaseReg, false, BaseUndef, false, OffUndef,
+                    Pred, PredReg, TII, isT2);
+      InsertLDR_STR(MBB, MBBI, OffImm+4, isLd, dl, NewOpc2,
+                    OddReg, OddDeadKill, OddUndef,
+                    BaseReg, BaseKill, BaseUndef, OffKill, OffUndef,
+                    Pred, PredReg, TII, isT2);
     }
-
-    MBB.erase(MI);
-    MBBI = NewBBI;
-    return true;
+    if (isLd)
+      ++NumLDRD2LDR;
+    else
+      ++NumSTRD2STR;
   }
-  return false;
+
+  MBBI = MBB.erase(MBBI);
+  return true;
 }
 
-/// LoadStoreMultipleOpti - An optimization pass to turn multiple LDR / STR
-/// ops of the same base and incrementing offset into LDM / STM ops.
+/// An optimization pass to turn multiple LDR / STR ops of the same base and
+/// incrementing offset into LDM / STM ops.
 bool ARMLoadStoreOpt::LoadStoreMultipleOpti(MachineBasicBlock &MBB) {
-  unsigned NumMerges = 0;
-  unsigned NumMemOps = 0;
   MemOpQueue MemOps;
   unsigned CurrBase = 0;
-  int CurrOpc = -1;
-  unsigned CurrSize = 0;
+  unsigned CurrOpc = ~0u;
   ARMCC::CondCodes CurrPred = ARMCC::AL;
-  unsigned CurrPredReg = 0;
   unsigned Position = 0;
-  SmallVector<MachineBasicBlock::iterator,4> Merges;
-
-  RS->enterBasicBlock(&MBB);
-  MachineBasicBlock::iterator MBBI = MBB.begin(), E = MBB.end();
-  while (MBBI != E) {
+  assert(Candidates.size() == 0);
+  assert(MergeBaseCandidates.size() == 0);
+  LiveRegsValid = false;
+
+  for (MachineBasicBlock::iterator I = MBB.end(), MBBI; I != MBB.begin();
+       I = MBBI) {
+    // The instruction in front of the iterator is the one we look at.
+    MBBI = std::prev(I);
     if (FixInvalidRegPairOp(MBB, MBBI))
       continue;
+    ++Position;
 
-    bool Advance  = false;
-    bool TryMerge = false;
-    bool Clobber  = false;
-
-    bool isMemOp = isMemoryOp(MBBI);
-    if (isMemOp) {
-      int Opcode = MBBI->getOpcode();
-      unsigned Size = getLSMultipleTransferSize(MBBI);
+    if (isMemoryOp(MBBI)) {
+      unsigned Opcode = MBBI->getOpcode();
       const MachineOperand &MO = MBBI->getOperand(0);
       unsigned Reg = MO.getReg();
-      bool isKill = MO.isDef() ? false : MO.isKill();
-      unsigned Base = MBBI->getOperand(1).getReg();
+      unsigned Base = getLoadStoreBaseOp(*MBBI).getReg();
       unsigned PredReg = 0;
       ARMCC::CondCodes Pred = getInstrPredicate(MBBI, PredReg);
       int Offset = getMemoryOpOffset(MBBI);
-      // Watch out for:
-      // r4 := ldr [r5]
-      // r5 := ldr [r5, #4]
-      // r6 := ldr [r5, #8]
-      //
-      // The second ldr has effectively broken the chain even though it
-      // looks like the later ldr(s) use the same base register. Try to
-      // merge the ldr's so far, including this one. But don't try to
-      // combine the following ldr(s).
-      Clobber = (isi32Load(Opcode) && Base == MBBI->getOperand(0).getReg());
-
-      // Watch out for:
-      // r4 := ldr [r0, #8]
-      // r4 := ldr [r0, #4]
-      //
-      // The optimization may reorder the second ldr in front of the first
-      // ldr, which violates write after write(WAW) dependence. The same as
-      // str. Try to merge inst(s) already in MemOps.
-      bool Overlap = false;
-      for (MemOpQueueIter I = MemOps.begin(), E = MemOps.end(); I != E; ++I) {
-        if (TRI->regsOverlap(Reg, I->MBBI->getOperand(0).getReg())) {
-          Overlap = true;
-          break;
-        }
-      }
-
-      if (CurrBase == 0 && !Clobber) {
+      if (CurrBase == 0) {
         // Start of a new chain.
         CurrBase = Base;
         CurrOpc  = Opcode;
-        CurrSize = Size;
         CurrPred = Pred;
-        CurrPredReg = PredReg;
-        MemOps.push_back(MemOpQueueEntry(Offset, Reg, isKill, Position, MBBI));
-        ++NumMemOps;
-        Advance = true;
-      } else if (!Overlap) {
-        if (Clobber) {
-          TryMerge = true;
-          Advance = true;
+        MemOps.push_back(MemOpQueueEntry(MBBI, Offset, Position));
+        continue;
+      }
+      // Note: No need to match PredReg in the next if.
+      if (CurrOpc == Opcode && CurrBase == Base && CurrPred == Pred) {
+        // Watch out for:
+        //   r4 := ldr [r0, #8]
+        //   r4 := ldr [r0, #4]
+        // or
+        //   r0 := ldr [r0]
+        // If a load overrides the base register or a register loaded by
+        // another load in our chain, we cannot take this instruction.
+        bool Overlap = false;
+        if (isLoadSingle(Opcode)) {
+          Overlap = (Base == Reg);
+          if (!Overlap) {
+            for (const MemOpQueueEntry &E : MemOps) {
+              if (TRI->regsOverlap(Reg, E.MI->getOperand(0).getReg())) {
+                Overlap = true;
+                break;
+              }
+            }
+          }
         }
 
-        if (CurrOpc == Opcode && CurrBase == Base && CurrPred == Pred) {
-          // No need to match PredReg.
-          // Continue adding to the queue.
+        if (!Overlap) {
+          // Check offset and sort memory operation into the current chain.
           if (Offset > MemOps.back().Offset) {
-            MemOps.push_back(MemOpQueueEntry(Offset, Reg, isKill,
-                                             Position, MBBI));
-            ++NumMemOps;
-            Advance = true;
+            MemOps.push_back(MemOpQueueEntry(MBBI, Offset, Position));
+            continue;
           } else {
-            for (MemOpQueueIter I = MemOps.begin(), E = MemOps.end();
-                 I != E; ++I) {
-              if (Offset < I->Offset) {
-                MemOps.insert(I, MemOpQueueEntry(Offset, Reg, isKill,
-                                                 Position, MBBI));
-                ++NumMemOps;
-                Advance = true;
+            MemOpQueue::iterator MI, ME;
+            for (MI = MemOps.begin(), ME = MemOps.end(); MI != ME; ++MI) {
+              if (Offset < MI->Offset) {
+                // Found a place to insert.
                 break;
-              } else if (Offset == I->Offset) {
-                // Collision! This can't be merged!
+              }
+              if (Offset == MI->Offset) {
+                // Collision, abort.
+                MI = ME;
                 break;
               }
             }
+            if (MI != MemOps.end()) {
+              MemOps.insert(MI, MemOpQueueEntry(MBBI, Offset, Position));
+              continue;
+            }
           }
         }
       }
-    }
 
-    if (MBBI->isDebugValue()) {
-      ++MBBI;
-      if (MBBI == E)
-        // Reach the end of the block, try merging the memory instructions.
-        TryMerge = true;
-    } else if (Advance) {
-      ++Position;
-      ++MBBI;
-      if (MBBI == E)
-        // Reach the end of the block, try merging the memory instructions.
-        TryMerge = true;
-    } else {
-      TryMerge = true;
+      // Don't advance the iterator; The op will start a new chain next.
+      MBBI = I;
+      --Position;
+      // Fallthrough to look into existing chain.
+    } else if (MBBI->isDebugValue()) {
+      continue;
+    } else if (MBBI->getOpcode() == ARM::t2LDRDi8 ||
+               MBBI->getOpcode() == ARM::t2STRDi8) {
+      // ARMPreAllocLoadStoreOpt has already formed some LDRD/STRD instructions
+      // remember them because we may still be able to merge add/sub into them.
+      MergeBaseCandidates.push_back(MBBI);
     }
 
-    if (TryMerge) {
-      if (NumMemOps > 1) {
-        // Try to find a free register to use as a new base in case it's needed.
-        // First advance to the instruction just before the start of the chain.
-        AdvanceRS(MBB, MemOps);
-
-        // Find a scratch register.
-        unsigned Scratch =
-          RS->FindUnusedReg(isThumb1 ? &ARM::tGPRRegClass : &ARM::GPRRegClass);
-
-        // Process the load / store instructions.
-        RS->forward(std::prev(MBBI));
-
-        // Merge ops.
-        Merges.clear();
-        MergeLDR_STR(MBB, 0, CurrBase, CurrOpc, CurrSize,
-                     CurrPred, CurrPredReg, Scratch, MemOps, Merges);
-
-        // Try folding preceding/trailing base inc/dec into the generated
-        // LDM/STM ops.
-        for (unsigned i = 0, e = Merges.size(); i < e; ++i)
-          if (MergeBaseUpdateLSMultiple(MBB, Merges[i], Advance, MBBI))
-            ++NumMerges;
-        NumMerges += Merges.size();
-
-        // Try folding preceding/trailing base inc/dec into those load/store
-        // that were not merged to form LDM/STM ops.
-        for (unsigned i = 0; i != NumMemOps; ++i)
-          if (!MemOps[i].Merged)
-            if (MergeBaseUpdateLoadStore(MBB, MemOps[i].MBBI, TII,Advance,MBBI))
-              ++NumMerges;
-
-        // RS may be pointing to an instruction that's deleted.
-        RS->skipTo(std::prev(MBBI));
-      } else if (NumMemOps == 1) {
-        // Try folding preceding/trailing base inc/dec into the single
-        // load/store.
-        if (MergeBaseUpdateLoadStore(MBB, MemOps[0].MBBI, TII, Advance, MBBI)) {
-          ++NumMerges;
-          RS->forward(std::prev(MBBI));
-        }
-      }
 
+    // If we are here then the chain is broken; Extract candidates for a merge.
+    if (MemOps.size() > 0) {
+      FormCandidates(MemOps);
+      // Reset for the next chain.
       CurrBase = 0;
-      CurrOpc = -1;
-      CurrSize = 0;
+      CurrOpc = ~0u;
       CurrPred = ARMCC::AL;
-      CurrPredReg = 0;
-      if (NumMemOps) {
-        MemOps.clear();
-        NumMemOps = 0;
-      }
+      MemOps.clear();
+    }
+  }
+  if (MemOps.size() > 0)
+    FormCandidates(MemOps);
 
-      // If iterator hasn't been advanced and this is not a memory op, skip it.
-      // It can't start a new chain anyway.
-      if (!Advance && !isMemOp && MBBI != E) {
-        ++Position;
-        ++MBBI;
+  // Sort candidates so they get processed from end to begin of the basic
+  // block later; This is necessary for liveness calculation.
+  auto LessThan = [](const MergeCandidate* M0, const MergeCandidate *M1) {
+    return M0->InsertPos < M1->InsertPos;
+  };
+  std::sort(Candidates.begin(), Candidates.end(), LessThan);
+
+  // Go through list of candidates and merge.
+  bool Changed = false;
+  for (const MergeCandidate *Candidate : Candidates) {
+    if (Candidate->CanMergeToLSMulti || Candidate->CanMergeToLSDouble) {
+      MachineInstr *Merged = MergeOpsUpdate(*Candidate);
+      // Merge preceding/trailing base inc/dec into the merged op.
+      if (Merged) {
+        Changed = true;
+        unsigned Opcode = Merged->getOpcode();
+        if (Opcode == ARM::t2STRDi8 || Opcode == ARM::t2LDRDi8)
+          MergeBaseUpdateLSDouble(*Merged);
+        else
+          MergeBaseUpdateLSMultiple(Merged);
+      } else {
+        for (MachineInstr *MI : Candidate->Instrs) {
+          if (MergeBaseUpdateLoadStore(MI))
+            Changed = true;
+        }
       }
+    } else {
+      assert(Candidate->Instrs.size() == 1);
+      if (MergeBaseUpdateLoadStore(Candidate->Instrs.front()))
+        Changed = true;
     }
   }
-  return NumMerges > 0;
+  Candidates.clear();
+  // Try to fold add/sub into the LDRD/STRD formed by ARMPreAllocLoadStoreOpt.
+  for (MachineInstr *MI : MergeBaseCandidates)
+    MergeBaseUpdateLSDouble(*MI);
+  MergeBaseCandidates.clear();
+
+  return Changed;
 }
 
-/// MergeReturnIntoLDM - If this is a exit BB, try merging the return ops
-/// ("bx lr" and "mov pc, lr") into the preceding stack restore so it
-/// directly restore the value of LR into pc.
+/// If this is a exit BB, try merging the return ops ("bx lr" and "mov pc, lr")
+/// into the preceding stack restore so it directly restore the value of LR
+/// into pc.
 ///   ldmfd sp!, {..., lr}
 ///   bx lr
 /// or
@@ -1784,16 +1787,7 @@ bool ARMLoadStoreOpt::LoadStoreMultipleOpti(MachineBasicBlock &MBB) {
 ///   mov pc, lr
 /// =>
 ///   ldmfd sp!, {..., pc}
-// @LOCALMOD for sfi we do not want this to happen
 bool ARMLoadStoreOpt::MergeReturnIntoLDM(MachineBasicBlock &MBB) {
-  // @LOCALMOD-START
-  // For NaCl, do not load into PC directly for a return, since NaCl requires
-  // masking the address first.
-  if (STI->isTargetNaCl()) {
-    return false;
-  }
-  // @LOCALMOD-END
-
   // Thumb1 LDM doesn't allow high registers.
   if (isThumb1) return false;
   if (MBB.empty()) return false;
@@ -1825,12 +1819,14 @@ bool ARMLoadStoreOpt::MergeReturnIntoLDM(MachineBasicBlock &MBB) {
 }
 
 bool ARMLoadStoreOpt::runOnMachineFunction(MachineFunction &Fn) {
+  MF = &Fn;
   STI = &static_cast<const ARMSubtarget &>(Fn.getSubtarget());
   TL = STI->getTargetLowering();
   AFI = Fn.getInfo<ARMFunctionInfo>();
   TII = STI->getInstrInfo();
   TRI = STI->getRegisterInfo();
-  RS = new RegScavenger();
+  MRI = &Fn.getRegInfo();
+  RegClassInfoValid = false;
   isThumb2 = AFI->isThumb2Function();
   isThumb1 = AFI->isThumbFunction() && !isThumb2;
 
@@ -1843,16 +1839,13 @@ bool ARMLoadStoreOpt::runOnMachineFunction(MachineFunction &Fn) {
       Modified |= MergeReturnIntoLDM(MBB);
   }
 
-  delete RS;
+  Allocator.DestroyAll();
   return Modified;
 }
 
-
-/// ARMPreAllocLoadStoreOpt - Pre- register allocation pass that move
-/// load / stores from consecutive locations close to make it more
-/// likely they will be combined later.
-
 namespace {
+  /// Pre- register allocation pass that move load / stores from consecutive
+  /// locations close to make it more likely they will be combined later.
   struct ARMPreAllocLoadStoreOpt : public MachineFunctionPass{
     static char ID;
     ARMPreAllocLoadStoreOpt() : MachineFunctionPass(ID) {}
@@ -1887,7 +1880,7 @@ namespace {
 }
 
 bool ARMPreAllocLoadStoreOpt::runOnMachineFunction(MachineFunction &Fn) {
-  TD = Fn.getTarget().getDataLayout();
+  TD = &Fn.getDataLayout();
   STI = &static_cast<const ARMSubtarget &>(Fn.getSubtarget());
   TII = STI->getInstrInfo();
   TRI = STI->getRegisterInfo();
@@ -1949,7 +1942,7 @@ static bool IsSafeAndProfitableToMove(bool isLd, unsigned Base,
 }
 
 
-/// Copy Op0 and Op1 operands into a new array assigned to MI.
+/// Copy \p Op0 and \p Op1 operands into a new array assigned to MI.
 static void concatenateMemOperands(MachineInstr *MI, MachineInstr *Op0,
                                    MachineInstr *Op1) {
   assert(MI->memoperands_empty() && "expected a new machineinstr");
@@ -1967,10 +1960,11 @@ static void concatenateMemOperands(MachineInstr *MI, MachineInstr *Op0,
 
 bool
 ARMPreAllocLoadStoreOpt::CanFormLdStDWord(MachineInstr *Op0, MachineInstr *Op1,
-                                          DebugLoc &dl,
-                                          unsigned &NewOpc, unsigned &EvenReg,
-                                          unsigned &OddReg, unsigned &BaseReg,
-                                          int &Offset, unsigned &PredReg,
+                                          DebugLoc &dl, unsigned &NewOpc,
+                                          unsigned &FirstReg,
+                                          unsigned &SecondReg,
+                                          unsigned &BaseReg, int &Offset,
+                                          unsigned &PredReg,
                                           ARMCC::CondCodes &Pred,
                                           bool &isT2) {
   // Make sure we're allowed to generate LDRD/STRD.
@@ -2029,9 +2023,9 @@ ARMPreAllocLoadStoreOpt::CanFormLdStDWord(MachineInstr *Op0, MachineInstr *Op1,
       return false;
     Offset = ARM_AM::getAM3Opc(AddSub, OffImm);
   }
-  EvenReg = Op0->getOperand(0).getReg();
-  OddReg  = Op1->getOperand(0).getReg();
-  if (EvenReg == OddReg)
+  FirstReg = Op0->getOperand(0).getReg();
+  SecondReg = Op1->getOperand(0).getReg();
+  if (FirstReg == SecondReg)
     return false;
   BaseReg = Op0->getOperand(1).getReg();
   Pred = getInstrPredicate(Op0, PredReg);
@@ -2127,7 +2121,7 @@ bool ARMPreAllocLoadStoreOpt::RescheduleOps(MachineBasicBlock *MBB,
         // to try to allocate a pair of registers that can form register pairs.
         MachineInstr *Op0 = Ops.back();
         MachineInstr *Op1 = Ops[Ops.size()-2];
-        unsigned EvenReg = 0, OddReg = 0;
+        unsigned FirstReg = 0, SecondReg = 0;
         unsigned BaseReg = 0, PredReg = 0;
         ARMCC::CondCodes Pred = ARMCC::AL;
         bool isT2 = false;
@@ -2135,21 +2129,21 @@ bool ARMPreAllocLoadStoreOpt::RescheduleOps(MachineBasicBlock *MBB,
         int Offset = 0;
         DebugLoc dl;
         if (NumMove == 2 && CanFormLdStDWord(Op0, Op1, dl, NewOpc,
-                                             EvenReg, OddReg, BaseReg,
+                                             FirstReg, SecondReg, BaseReg,
                                              Offset, PredReg, Pred, isT2)) {
           Ops.pop_back();
           Ops.pop_back();
 
           const MCInstrDesc &MCID = TII->get(NewOpc);
           const TargetRegisterClass *TRC = TII->getRegClass(MCID, 0, TRI, *MF);
-          MRI->constrainRegClass(EvenReg, TRC);
-          MRI->constrainRegClass(OddReg, TRC);
+          MRI->constrainRegClass(FirstReg, TRC);
+          MRI->constrainRegClass(SecondReg, TRC);
 
           // Form the pair instruction.
           if (isLd) {
             MachineInstrBuilder MIB = BuildMI(*MBB, InsertPos, dl, MCID)
-              .addReg(EvenReg, RegState::Define)
-              .addReg(OddReg, RegState::Define)
+              .addReg(FirstReg, RegState::Define)
+              .addReg(SecondReg, RegState::Define)
               .addReg(BaseReg);
             // FIXME: We're converting from LDRi12 to an insn that still
             // uses addrmode2, so we need an explicit offset reg. It should
@@ -2162,8 +2156,8 @@ bool ARMPreAllocLoadStoreOpt::RescheduleOps(MachineBasicBlock *MBB,
             ++NumLDRDFormed;
           } else {
             MachineInstrBuilder MIB = BuildMI(*MBB, InsertPos, dl, MCID)
-              .addReg(EvenReg)
-              .addReg(OddReg)
+              .addReg(FirstReg)
+              .addReg(SecondReg)
               .addReg(BaseReg);
             // FIXME: We're converting from LDRi12 to an insn that still
             // uses addrmode2, so we need an explicit offset reg. It should
@@ -2178,9 +2172,11 @@ bool ARMPreAllocLoadStoreOpt::RescheduleOps(MachineBasicBlock *MBB,
           MBB->erase(Op0);
           MBB->erase(Op1);
 
-          // Add register allocation hints to form register pairs.
-          MRI->setRegAllocationHint(EvenReg, ARMRI::RegPairEven, OddReg);
-          MRI->setRegAllocationHint(OddReg,  ARMRI::RegPairOdd, EvenReg);
+          if (!isT2) {
+            // Add register allocation hints to form register pairs.
+            MRI->setRegAllocationHint(FirstReg, ARMRI::RegPairEven, SecondReg);
+            MRI->setRegAllocationHint(SecondReg,  ARMRI::RegPairOdd, FirstReg);
+          }
         } else {
           for (unsigned i = 0; i != NumMove; ++i) {
             MachineInstr *Op = Ops.back();
@@ -2230,7 +2226,7 @@ ARMPreAllocLoadStoreOpt::RescheduleLoadStoreInstrs(MachineBasicBlock *MBB) {
         continue;
 
       int Opc = MI->getOpcode();
-      bool isLd = isi32Load(Opc) || Opc == ARM::VLDRS || Opc == ARM::VLDRD;
+      bool isLd = isLoadSingle(Opc);
       unsigned Base = MI->getOperand(1).getReg();
       int Offset = getMemoryOpOffset(MI);
 
@@ -2305,10 +2301,10 @@ ARMPreAllocLoadStoreOpt::RescheduleLoadStoreInstrs(MachineBasicBlock *MBB) {
 }
 
 
-/// createARMLoadStoreOptimizationPass - returns an instance of the load / store
-/// optimization pass.
+/// Returns an instance of the load / store optimization pass.
 FunctionPass *llvm::createARMLoadStoreOptimizationPass(bool PreAlloc) {
   if (PreAlloc)
     return new ARMPreAllocLoadStoreOpt();
   return new ARMLoadStoreOpt();
 }
+
diff --git a/lib/Target/ARM/ARMMCInstLower.cpp b/lib/Target/ARM/ARMMCInstLower.cpp
index 23d61d70cf9..a2aca2d1a69 100644
--- a/lib/Target/ARM/ARMMCInstLower.cpp
+++ b/lib/Target/ARM/ARMMCInstLower.cpp
@@ -30,38 +30,38 @@ MCOperand ARMAsmPrinter::GetSymbolRef(const MachineOperand &MO,
   unsigned Option = MO.getTargetFlags() & ARMII::MO_OPTION_MASK;
   switch (Option) {
   default: {
-    Expr = MCSymbolRefExpr::Create(Symbol, MCSymbolRefExpr::VK_None,
+    Expr = MCSymbolRefExpr::create(Symbol, MCSymbolRefExpr::VK_None,
                                    OutContext);
     switch (Option) {
     default: llvm_unreachable("Unknown target flag on symbol operand");
     case ARMII::MO_NO_FLAG:
       break;
     case ARMII::MO_LO16:
-      Expr = MCSymbolRefExpr::Create(Symbol, MCSymbolRefExpr::VK_None,
+      Expr = MCSymbolRefExpr::create(Symbol, MCSymbolRefExpr::VK_None,
                                      OutContext);
-      Expr = ARMMCExpr::CreateLower16(Expr, OutContext);
+      Expr = ARMMCExpr::createLower16(Expr, OutContext);
       break;
     case ARMII::MO_HI16:
-      Expr = MCSymbolRefExpr::Create(Symbol, MCSymbolRefExpr::VK_None,
+      Expr = MCSymbolRefExpr::create(Symbol, MCSymbolRefExpr::VK_None,
                                      OutContext);
-      Expr = ARMMCExpr::CreateUpper16(Expr, OutContext);
+      Expr = ARMMCExpr::createUpper16(Expr, OutContext);
       break;
     }
     break;
   }
 
   case ARMII::MO_PLT:
-    Expr = MCSymbolRefExpr::Create(Symbol, MCSymbolRefExpr::VK_PLT,
+    Expr = MCSymbolRefExpr::create(Symbol, MCSymbolRefExpr::VK_PLT,
                                    OutContext);
     break;
   }
 
   if (!MO.isJTI() && MO.getOffset())
-    Expr = MCBinaryExpr::CreateAdd(Expr,
-                                   MCConstantExpr::Create(MO.getOffset(),
+    Expr = MCBinaryExpr::createAdd(Expr,
+                                   MCConstantExpr::create(MO.getOffset(),
                                                           OutContext),
                                    OutContext);
-  return MCOperand::CreateExpr(Expr);
+  return MCOperand::createExpr(Expr);
 
 }
 
@@ -74,13 +74,13 @@ bool ARMAsmPrinter::lowerOperand(const MachineOperand &MO,
     if (MO.isImplicit() && MO.getReg() != ARM::CPSR)
       return false;
     assert(!MO.getSubReg() && "Subregs should be eliminated!");
-    MCOp = MCOperand::CreateReg(MO.getReg());
+    MCOp = MCOperand::createReg(MO.getReg());
     break;
   case MachineOperand::MO_Immediate:
-    MCOp = MCOperand::CreateImm(MO.getImm());
+    MCOp = MCOperand::createImm(MO.getImm());
     break;
   case MachineOperand::MO_MachineBasicBlock:
-    MCOp = MCOperand::CreateExpr(MCSymbolRefExpr::Create(
+    MCOp = MCOperand::createExpr(MCSymbolRefExpr::create(
         MO.getMBB()->getSymbol(), OutContext));
     break;
   case MachineOperand::MO_GlobalAddress: {
@@ -105,7 +105,7 @@ bool ARMAsmPrinter::lowerOperand(const MachineOperand &MO,
     APFloat Val = MO.getFPImm()->getValueAPF();
     bool ignored;
     Val.convert(APFloat::IEEEdouble, APFloat::rmTowardZero, &ignored);
-    MCOp = MCOperand::CreateFPImm(Val.convertToDouble());
+    MCOp = MCOperand::createFPImm(Val.convertToDouble());
     break;
   }
   case MachineOperand::MO_RegisterMask:
@@ -161,57 +161,3 @@ void llvm::LowerARMMachineInstrToMCInst(const MachineInstr *MI, MCInst &OutMI,
     }
   }
 }
-
-// @LOCALMOD-BEGIN
-// Unlike LowerARMMachineInstrToMCInst, the opcode has already been set.
-// Otherwise, this is like LowerARMMachineInstrToMCInst, but with special
-// handling where the "immediate" is PC Relative
-// (used for MOVi16PIC / MOVTi16PIC, etc. -- see .td file)
-void llvm::LowerARMMachineInstrToMCInstPCRel(const MachineInstr *MI,
-                                             MCInst &OutMI,
-                                             ARMAsmPrinter &AP,
-                                             unsigned ImmIndex,
-                                             unsigned PCIndex,
-                                             MCSymbol *PCLabel,
-                                             unsigned PCAdjustment) {
-
-  for (unsigned i = 0, e = MI->getNumOperands(); i != e; ++i) {
-    if (i == ImmIndex) {
-      MCContext &Ctx = AP.OutContext;
-      const MCExpr *PCRelExpr = MCSymbolRefExpr::Create(PCLabel, Ctx);
-      if (PCAdjustment) {
-        const MCExpr *AdjExpr = MCConstantExpr::Create(PCAdjustment, Ctx);
-        PCRelExpr = MCBinaryExpr::CreateAdd(PCRelExpr, AdjExpr, Ctx);
-      }
-
-      // Get the usual symbol operand, then subtract the PCRelExpr.
-      const MachineOperand &MOImm = MI->getOperand(ImmIndex);
-      MCOperand SymOp;
-      bool DidLower = AP.lowerOperand(MOImm, SymOp);
-      assert (DidLower && "Immediate-like operand should have been lowered");
-
-      const MCExpr *Expr = SymOp.getExpr();
-      ARMMCExpr::VariantKind TargetKind = ARMMCExpr::VK_ARM_None;
-      /* Unwrap and rewrap the ARMMCExpr */
-      if (Expr->getKind() == MCExpr::Target) {
-        const ARMMCExpr *TargetExpr = cast<ARMMCExpr>(Expr);
-        TargetKind = TargetExpr->getKind();
-        Expr = TargetExpr->getSubExpr();
-      }
-      Expr = MCBinaryExpr::CreateSub(Expr, PCRelExpr, Ctx);
-      if (TargetKind != ARMMCExpr::VK_ARM_None) {
-        Expr = ARMMCExpr::Create(TargetKind, Expr, Ctx);
-      }
-      MCOperand MCOp = MCOperand::CreateExpr(Expr);
-      OutMI.addOperand(MCOp);
-    } else if (i == PCIndex) {  // dummy index already handled as PCLabel
-      continue;
-    } else {
-      MCOperand MCOp;
-      if (AP.lowerOperand(MI->getOperand(i), MCOp)) {
-        OutMI.addOperand(MCOp);
-      }
-    }
-  }
-}
-// @LOCALMOD-END
diff --git a/lib/Target/ARM/ARMMachineFunctionInfo.cpp b/lib/Target/ARM/ARMMachineFunctionInfo.cpp
index 229d04165ed..8ca4be725ac 100644
--- a/lib/Target/ARM/ARMMachineFunctionInfo.cpp
+++ b/lib/Target/ARM/ARMMachineFunctionInfo.cpp
@@ -1,4 +1,4 @@
-//===-- ARMMachineFuctionInfo.cpp - ARM machine function info -------------===//
+//===-- ARMMachineFunctionInfo.cpp - ARM machine function info ------------===//
 //
 //                     The LLVM Compiler Infrastructure
 //
@@ -19,6 +19,6 @@ ARMFunctionInfo::ARMFunctionInfo(MachineFunction &MF)
       StByValParamsPadding(0), ArgRegsSaveSize(0), HasStackFrame(false),
       RestoreSPFromFP(false), LRSpilledForFarJump(false),
       FramePtrSpillOffset(0), GPRCS1Offset(0), GPRCS2Offset(0), DPRCSOffset(0),
-      GPRCS1Size(0), GPRCS2Size(0), DPRCSSize(0), JumpTableUId(0),
+      GPRCS1Size(0), GPRCS2Size(0), DPRCSSize(0),
       PICLabelUId(0), VarArgsFrameIndex(0), HasITBlocks(false),
       GlobalBaseReg(0) {}
diff --git a/lib/Target/ARM/ARMMachineFunctionInfo.h b/lib/Target/ARM/ARMMachineFunctionInfo.h
index a68ab1ba912..40ace69e94c 100644
--- a/lib/Target/ARM/ARMMachineFunctionInfo.h
+++ b/lib/Target/ARM/ARMMachineFunctionInfo.h
@@ -1,4 +1,4 @@
-//===-- ARMMachineFuctionInfo.h - ARM machine function info -----*- C++ -*-===//
+//===-- ARMMachineFunctionInfo.h - ARM machine function info ----*- C++ -*-===//
 //
 //                     The LLVM Compiler Infrastructure
 //
@@ -52,7 +52,7 @@ class ARMFunctionInfo : public MachineFunctionInfo {
   unsigned ReturnRegsCount;
 
   /// HasStackFrame - True if this function has a stack frame. Set by
-  /// processFunctionBeforeCalleeSavedScan().
+  /// determineCalleeSaves().
   bool HasStackFrame;
 
   /// RestoreSPFromFP - True if epilogue should restore SP from FP. Set by
@@ -98,10 +98,6 @@ class ARMFunctionInfo : public MachineFunctionInfo {
   /// registers also aren't included in DPRCSSize above.
   unsigned NumAlignedDPRCS2Regs;
 
-  /// JumpTableUId - Unique id for jumptables.
-  ///
-  unsigned JumpTableUId;
-
   unsigned PICLabelUId;
 
   /// VarArgsFrameIndex - FrameIndex for start of varargs area.
@@ -136,8 +132,7 @@ class ARMFunctionInfo : public MachineFunctionInfo {
     LRSpilledForFarJump(false),
     FramePtrSpillOffset(0), GPRCS1Offset(0), GPRCS2Offset(0), DPRCSOffset(0),
     GPRCS1Size(0), GPRCS2Size(0), DPRCSAlignGapSize(0), DPRCSSize(0),
-    NumAlignedDPRCS2Regs(0),
-    JumpTableUId(0), PICLabelUId(0),
+    NumAlignedDPRCS2Regs(0), PICLabelUId(0),
     VarArgsFrameIndex(0), HasITBlocks(false), GlobalBaseReg(0) {}
 
   explicit ARMFunctionInfo(MachineFunction &MF);
@@ -191,14 +186,6 @@ class ARMFunctionInfo : public MachineFunctionInfo {
   unsigned getArgumentStackSize() const { return ArgumentStackSize; }
   void setArgumentStackSize(unsigned size) { ArgumentStackSize = size; }
 
-  unsigned createJumpTableUId() {
-    return JumpTableUId++;
-  }
-
-  unsigned getNumJumpTables() const {
-    return JumpTableUId;
-  }
-
   void initPICLabelUId(unsigned UId) {
     PICLabelUId = UId;
   }
diff --git a/lib/Target/ARM/ARMNaClHeaders.cpp b/lib/Target/ARM/ARMNaClHeaders.cpp
deleted file mode 100644
index d10e1ab2141..00000000000
--- a/lib/Target/ARM/ARMNaClHeaders.cpp
+++ /dev/null
@@ -1,131 +0,0 @@
-//===-- ARMNaClHeaders.cpp - Print SFI headers to an ARM .s file -----------===//
-//
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
-//
-//===----------------------------------------------------------------------===//
-//
-// This file contains the initial header string needed
-// for the Native Client target in ARM assembly.
-//
-//===----------------------------------------------------------------------===//
-
-#include "llvm/Support/raw_ostream.h"
-#include "ARMNaClRewritePass.h"
-#include <string>
-
-using namespace llvm;
-
-void EmitSFIHeaders(raw_ostream &O) {
-  O << " @ ========================================\n";
-  // NOTE: this macro does bundle alignment as follows
-  //       if current bundle pos is X emit pX data items of value "val"
-  // NOTE: that pos will be one of: 0,4,8,12
-  //
-  O <<
-    "\t.macro sfi_long_based_on_pos p0 p1 p2 p3 val\n"
-    "\t.set pos, (. - XmagicX) % 16\n"
-    "\t.fill  (((\\p3<<12)|(\\p2<<8)|(\\p1<<4)|\\p0)>>pos) & 15, 4, \\val\n"
-    "\t.endm\n"
-    "\n\n";
-
-  O <<
-    "\t.macro sfi_illegal_if_at_bundle_begining\n"
-    "\tsfi_long_based_on_pos 1 0 0 0 0xe125be70\n"
-    "\t.endm\n"
-    "\n\n";
-
-  O <<
-    "\t.macro sfi_nop_if_at_bundle_end\n"
-    "\tsfi_long_based_on_pos 0 0 0 1 0xe320f000\n"
-    "\t.endm\n"
-      "\n\n";
-
-  O <<
-    "\t.macro sfi_nops_to_force_slot3\n"
-    "\tsfi_long_based_on_pos 3 2 1 0 0xe320f000\n"
-    "\t.endm\n"
-    "\n\n";
-
-  O <<
-    "\t.macro sfi_nops_to_force_slot2\n"
-    "\tsfi_long_based_on_pos 2 1 0 3 0xe320f000\n"
-    "\t.endm\n"
-    "\n\n";
-
-  O <<
-    "\t.macro sfi_nops_to_force_slot1\n"
-    "\tsfi_long_based_on_pos 1 0 3 2 0xe320f000\n"
-    "\t.endm\n"
-    "\n\n";
-
-  O << " @ ========================================\n";
-  O <<
-    "\t.macro sfi_data_mask reg cond\n"
-    "\tbic\\cond \\reg, \\reg, #0xc0000000\n"
-    "\t.endm\n"
-    "\n\n";
-
-  O <<
-    "\t.macro sfi_data_tst reg\n"
-    "\ttst \\reg, #0xc0000000\n"
-    "\t.endm\n"
-    "\n\n";
-
-  O <<
-    "\t.macro sfi_code_mask reg cond=\n"
-    "\tbic\\cond \\reg, \\reg, #0xc000000f\n"
-    "\t.endm\n"
-    "\n\n";
-
-  O << " @ ========================================\n";
-  O <<
-    "\t.macro sfi_call_preamble cond=\n"
-    "\tsfi_nops_to_force_slot3\n"
-    "\t.endm\n"
-    "\n\n";
-
-  O <<
-    "\t.macro sfi_return_preamble reg cond=\n"
-    "\tsfi_nop_if_at_bundle_end\n"
-    "\tsfi_code_mask \\reg \\cond\n"
-    "\t.endm\n"
-    "\n\n";
-    
-  // This is used just before "bx rx"
-  O <<
-    "\t.macro sfi_indirect_jump_preamble link cond=\n"
-    "\tsfi_nop_if_at_bundle_end\n"
-    "\tsfi_code_mask \\link \\cond\n"
-    "\t.endm\n"
-    "\n\n";
-
-  // This is use just before "blx rx"
-  O <<
-    "\t.macro sfi_indirect_call_preamble link cond=\n"
-    "\tsfi_nops_to_force_slot2\n"
-    "\tsfi_code_mask \\link \\cond\n"
-    "\t.endm\n"
-    "\n\n";
-
-  O << " @ ========================================\n";
-
-  O <<
-    "\t.macro sfi_load_store_preamble reg cond\n"
-    "\tsfi_nop_if_at_bundle_end\n"
-    "\tsfi_data_mask \\reg, \\cond\n"
-    "\t.endm\n"
-    "\n\n";
-
-  O <<
-    "\t.macro sfi_cstore_preamble reg\n"
-    "\tsfi_nop_if_at_bundle_end\n"
-    "\tsfi_data_tst \\reg\n"
-    "\t.endm\n"
-    "\n\n";
-
-  O << " @ ========================================\n";
-  O << "\t.text\n";
-}
diff --git a/lib/Target/ARM/ARMNaClRewritePass.cpp b/lib/Target/ARM/ARMNaClRewritePass.cpp
deleted file mode 100644
index dbf29e606cc..00000000000
--- a/lib/Target/ARM/ARMNaClRewritePass.cpp
+++ /dev/null
@@ -1,1085 +0,0 @@
-//===-- ARMNaClRewritePass.cpp - Native Client Rewrite Pass  ------*- C++ -*-=//
-//
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
-//
-//===----------------------------------------------------------------------===//
-//
-// Native Client Rewrite Pass
-// This final pass inserts the sandboxing instructions needed to run inside
-// the Native Client sandbox. Native Client requires certain software fault
-// isolation (SFI) constructions to be put in place, to prevent escape from
-// the sandbox. Native Client refuses to execute binaries without the correct
-// SFI sequences.
-//
-// Potentially dangerous operations which are protected include:
-// * Stores
-// * Branches
-// * Changes to SP
-//
-//===----------------------------------------------------------------------===//
-
-#define DEBUG_TYPE "arm-sfi"
-#include "ARM.h"
-#include "ARMBaseInstrInfo.h"
-#include "ARMNaClRewritePass.h"
-#include "llvm/CodeGen/MachineFunctionPass.h"
-#include "llvm/CodeGen/MachineInstrBuilder.h"
-#include "llvm/IR/Function.h"
-#include "llvm/Support/Debug.h"
-#include "llvm/Support/raw_ostream.h"
-#include "llvm/Target/TargetSubtargetInfo.h"
-#include <set>
-#include <stdio.h>
-
-using namespace llvm;
-
-namespace {
-  class ARMNaClRewritePass : public MachineFunctionPass {
-  public:
-    static char ID;
-    ARMNaClRewritePass() : MachineFunctionPass(ID) {}
-
-    const ARMBaseInstrInfo *TII;
-    const TargetRegisterInfo *TRI;
-    virtual void getAnalysisUsage(AnalysisUsage &AU) const;
-    virtual bool runOnMachineFunction(MachineFunction &Fn);
-
-    virtual const char *getPassName() const {
-      return "ARM Native Client Rewrite Pass";
-    }
-
-  private:
-
-    bool SandboxMemoryReferencesInBlock(MachineBasicBlock &MBB);
-    void SandboxMemory(MachineBasicBlock &MBB,
-                       MachineBasicBlock::iterator MBBI,
-                       MachineInstr &MI,
-                       int AddrIdx,
-                       bool IsLoad);
-
-    bool SandboxBranchesInBlock(MachineBasicBlock &MBB);
-    bool SandboxStackChangesInBlock(MachineBasicBlock &MBB);
-
-    void SandboxStackChange(MachineBasicBlock &MBB,
-                              MachineBasicBlock::iterator MBBI);
-    void LightweightVerify(MachineFunction &MF);
-  };
-  char ARMNaClRewritePass::ID = 0;
-}
-
-static bool IsReturn(const MachineInstr &MI) {
-  return (MI.getOpcode() == ARM::BX_RET);
-}
-
-static bool IsIndirectJump(const MachineInstr &MI) {
-  switch (MI.getOpcode()) {
-   default: return false;
-   case ARM::BX:
-   case ARM::TAILJMPr:
-    return true;
-  }
-}
-
-static bool IsIndirectCall(const MachineInstr &MI) {
-  return MI.getOpcode() == ARM::BLX;
-}
-
-static bool IsDirectCall(const MachineInstr &MI) {
-  switch (MI.getOpcode()) {
-   default: return false;
-   case ARM::BL:
-   case ARM::BL_pred:
-   case ARM::TPsoft:
-     return true;
-  }
-}
-
-static void DumpInstructionVerbose(const MachineInstr &MI) {
-  DEBUG({
-      dbgs() << MI;
-      dbgs() << MI.getNumOperands() << " operands:" << "\n";
-      for (unsigned i = 0; i < MI.getNumOperands(); ++i) {
-        const MachineOperand& op = MI.getOperand(i);
-        dbgs() << "  " << i << "(" << (unsigned)op.getType() << "):" << op
-               << "\n";
-      }
-      dbgs() << "\n";
-    });
-}
-
-static void DumpBasicBlockVerbose(const MachineBasicBlock &MBB) {
-  DEBUG({
-      dbgs() << "\n<<<<< DUMP BASIC BLOCK START\n";
-      for (MachineBasicBlock::const_iterator
-               MBBI = MBB.begin(), MBBE = MBB.end();
-           MBBI != MBBE;
-           ++MBBI) {
-        DumpInstructionVerbose(*MBBI);
-      }
-      dbgs() << "<<<<< DUMP BASIC BLOCK END\n\n";
-    });
-}
-
-/**********************************************************************/
-/* Exported functions */
-
-namespace ARM_SFI {
-
-bool IsStackChange(const MachineInstr &MI, const TargetRegisterInfo *TRI) {
-  return MI.modifiesRegister(ARM::SP, TRI);
-}
-
-bool NextInstrMasksSP(const MachineInstr &MI) {
-  MachineBasicBlock::const_iterator It = &MI;
-  const MachineBasicBlock *MBB = MI.getParent();
-
-  MachineBasicBlock::const_iterator next = ++It;
-  if (next == MBB->end()) {
-    return false;
-  }
-
-  const MachineInstr &next_instr = *next;
-  unsigned opcode = next_instr.getOpcode();
-  return (opcode == ARM::SFI_DATA_MASK) &&
-      (next_instr.getOperand(0).getReg() == ARM::SP);
-}
-
-bool IsSandboxedStackChange(const MachineInstr &MI) {
-  // Calls do not change the stack on ARM but they have implicit-defs, so
-  // make sure they do not get sandboxed.
-  if (MI.getDesc().isCall())
-    return true;
-
-  unsigned opcode = MI.getOpcode();
-  switch (opcode) {
-    default: break;
-
-    // Our mask instructions correctly update the stack pointer.
-    case ARM::SFI_DATA_MASK:
-      return true;
-
-    // These just bump SP by a little (and access the stack),
-    // so that is okay due to guard pages.
-    case ARM::STMIA_UPD:
-    case ARM::STMDA_UPD:
-    case ARM::STMDB_UPD:
-    case ARM::STMIB_UPD:
-
-    case ARM::VSTMDIA_UPD:
-    case ARM::VSTMDDB_UPD:
-    case ARM::VSTMSIA_UPD:
-    case ARM::VSTMSDB_UPD:
-      return true;
-
-    // Similar, unless it is a load into SP...
-    case ARM::LDMIA_UPD:
-    case ARM::LDMDA_UPD:
-    case ARM::LDMDB_UPD:
-    case ARM::LDMIB_UPD:
-
-    case ARM::VLDMDIA_UPD:
-    case ARM::VLDMDDB_UPD:
-    case ARM::VLDMSIA_UPD:
-    case ARM::VLDMSDB_UPD: {
-      bool dest_SP = false;
-      // Dest regs start at operand index 4.
-      for (unsigned i = 4; i < MI.getNumOperands(); ++i) {
-        const MachineOperand &DestReg = MI.getOperand(i);
-        dest_SP = dest_SP || (DestReg.getReg() == ARM::SP);
-      }
-      if (dest_SP) {
-        break;
-      }
-      return true;
-    }
-
-    // Some localmods *should* prevent selecting a reg offset
-    // (see SelectAddrMode2 in ARMISelDAGToDAG.cpp).
-    // Otherwise, the store is already a potential violation.
-    case ARM::STR_PRE_REG:
-    case ARM::STR_PRE_IMM:
-
-    case ARM::STRH_PRE:
-
-    case ARM::STRB_PRE_REG:
-    case ARM::STRB_PRE_IMM:
-      return true;
-
-    // Similar, unless it is a load into SP...
-    case ARM::LDRi12:
-    case ARM::LDR_PRE_REG:
-    case ARM::LDR_PRE_IMM:
-    case ARM::LDRH_PRE:
-    case ARM::LDRB_PRE_REG:
-    case ARM::LDRB_PRE_IMM:
-    case ARM::LDRSH_PRE:
-    case ARM::LDRSB_PRE: {
-      const MachineOperand &DestReg = MI.getOperand(0);
-      if (DestReg.getReg() == ARM::SP) {
-        break;
-      }
-      return true;
-    }
-
-    // Here, if SP is the base / write-back reg, we need to check if
-    // a reg is used as offset (otherwise it is not a small nudge).
-    case ARM::STR_POST_REG:
-    case ARM::STR_POST_IMM:
-    case ARM::STRH_POST:
-    case ARM::STRB_POST_REG:
-    case ARM::STRB_POST_IMM: {
-      const MachineOperand &WBReg = MI.getOperand(0);
-      const MachineOperand &OffReg = MI.getOperand(3);
-      if (WBReg.getReg() == ARM::SP && OffReg.getReg() != 0) {
-        break;
-      }
-      return true;
-    }
-
-    // Similar, but also check that DestReg is not SP.
-    case ARM::LDR_POST_REG:
-    case ARM::LDR_POST_IMM:
-    case ARM::LDRB_POST_REG:
-    case ARM::LDRB_POST_IMM:
-    case ARM::LDRH_POST:
-    case ARM::LDRSH_POST:
-    case ARM::LDRSB_POST: {
-      const MachineOperand &DestReg = MI.getOperand(0);
-      if (DestReg.getReg() == ARM::SP) {
-        break;
-      }
-      const MachineOperand &WBReg = MI.getOperand(1);
-      const MachineOperand &OffReg = MI.getOperand(3);
-      if (WBReg.getReg() == ARM::SP && OffReg.getReg() != 0) {
-        break;
-      }
-      return true;
-    }
-  }
-
-  return (NextInstrMasksSP(MI));
-}
-
-bool NeedSandboxStackChange(const MachineInstr &MI,
-                               const TargetRegisterInfo *TRI) {
-  return (IsStackChange(MI, TRI) && !IsSandboxedStackChange(MI));
-}
-
-} // namespace ARM_SFI
-
-/**********************************************************************/
-
-void ARMNaClRewritePass::getAnalysisUsage(AnalysisUsage &AU) const {
-  // Slight (possibly unnecessary) efficiency tweak:
-  // Promise not to modify the CFG.
-  AU.setPreservesCFG();
-  MachineFunctionPass::getAnalysisUsage(AU);
-}
-
-/*
- * A primitive validator to catch problems at compile time.
- * E.g., it could be used along with bugpoint to reduce a bitcode file.
- */
-void ARMNaClRewritePass::LightweightVerify(MachineFunction &MF) {
-  DEBUG({
-      for (MachineFunction::iterator MFI = MF.begin(), MFE = MF.end();
-           MFI != MFE;
-           ++MFI) {
-        MachineBasicBlock &MBB = *MFI;
-        for (MachineBasicBlock::iterator MBBI = MBB.begin(), MBBE = MBB.end();
-             MBBI != MBBE;
-             ++MBBI) {
-          MachineInstr &MI = *MBBI;
-          if (ARM_SFI::NeedSandboxStackChange(MI, TRI)) {
-            dbgs() << "LightWeightVerify for function: "
-                   << MF.getFunction()->getName() << "  (BAD STACK CHANGE)\n";
-            DumpInstructionVerbose(MI);
-            DumpBasicBlockVerbose(MBB);
-          }
-        }
-      }
-    });
-}
-
-void ARMNaClRewritePass::SandboxStackChange(MachineBasicBlock &MBB,
-                                       MachineBasicBlock::iterator MBBI) {
-  // (1) Ensure there is room in the bundle for a data mask instruction
-  // (nop'ing to the next bundle if needed).
-  // (2) Do a data mask on SP after the instruction that updated SP.
-  MachineInstr &MI = *MBBI;
-
-  // Use same predicate as current instruction.
-  unsigned PredReg = 0;
-  ARMCC::CondCodes Pred = llvm::getInstrPredicate(&MI, PredReg);
-
-  BuildMI(MBB, MBBI, MI.getDebugLoc(),
-          TII->get(ARM::SFI_NOP_IF_AT_BUNDLE_END));
-
-  // Get to next instr.
-  MachineBasicBlock::iterator MBBINext = (++MBBI);
-
-  BuildMI(MBB, MBBINext, MI.getDebugLoc(),
-          TII->get(ARM::SFI_DATA_MASK))
-      .addReg(ARM::SP, RegState::Define)  // modify SP (as dst)
-      .addReg(ARM::SP, RegState::Kill)    // start with SP (as src)
-      .addImm((int64_t) Pred)             // predicate condition
-      .addReg(PredReg);                   // predicate source register (CPSR)
-}
-
-bool ARMNaClRewritePass::SandboxStackChangesInBlock(MachineBasicBlock &MBB) {
-  bool Modified = false;
-  for (MachineBasicBlock::iterator MBBI = MBB.begin(), E = MBB.end();
-       MBBI != E;
-       ++MBBI) {
-    MachineInstr &MI = *MBBI;
-    if (ARM_SFI::NeedSandboxStackChange(MI, TRI)) {
-      SandboxStackChange(MBB, MBBI);
-      Modified |= true;
-    }
-  }
-  return Modified;
-}
-
-bool ARMNaClRewritePass::SandboxBranchesInBlock(MachineBasicBlock &MBB) {
-  bool Modified = false;
-
-  for (MachineBasicBlock::iterator MBBI = MBB.begin(), E = MBB.end();
-       MBBI != E;
-       ++MBBI) {
-    MachineInstr &MI = *MBBI;
-    // Use same predicate as current instruction.
-    unsigned PredReg = 0;
-    ARMCC::CondCodes Pred = llvm::getInstrPredicate(&MI, PredReg);
-
-    if (IsReturn(MI)) {
-      BuildMI(MBB, MBBI, MI.getDebugLoc(),
-              TII->get(ARM::SFI_GUARD_RETURN))
-        .addImm((int64_t) Pred)  // predicate condition
-        .addReg(PredReg);        // predicate source register (CPSR)
-      Modified = true;
-    }
-
-    if (IsIndirectJump(MI)) {
-      unsigned Addr = MI.getOperand(0).getReg();
-      BuildMI(MBB, MBBI, MI.getDebugLoc(),
-              TII->get(ARM::SFI_GUARD_INDIRECT_JMP))
-        .addReg(Addr, RegState::Define)  // Destination definition (as dst)
-        .addReg(Addr, RegState::Kill)    // Destination read (as src)
-        .addImm((int64_t) Pred)          // predicate condition
-        .addReg(PredReg);                // predicate source register (CPSR)
-      Modified = true;
-    }
-
-    if (IsDirectCall(MI)) {
-      BuildMI(MBB, MBBI, MI.getDebugLoc(),
-              TII->get(ARM::SFI_GUARD_CALL))
-        .addImm((int64_t) Pred)  // predicate condition
-        .addReg(PredReg);        // predicate source register (CPSR)
-      Modified = true;
-    }
-
-    if (IsIndirectCall(MI)) {
-      unsigned Addr = MI.getOperand(0).getReg();
-      BuildMI(MBB, MBBI, MI.getDebugLoc(),
-              TII->get(ARM::SFI_GUARD_INDIRECT_CALL))
-        .addReg(Addr, RegState::Define)  // Destination definition (as dst)
-        .addReg(Addr, RegState::Kill)    // Destination read (as src)
-        .addImm((int64_t) Pred)          // predicate condition
-        .addReg(PredReg);                // predicate source register (CPSR)
-        Modified = true;
-    }
-  }
-
-  return Modified;
-}
-
-static bool IsDangerousLoad(const MachineInstr &MI, int *AddrIdx) {
-  unsigned Opcode = MI.getOpcode();
-  switch (Opcode) {
-  default: return false;
-
-  // Instructions with base address register in position 0...
-  case ARM::LDMIA:
-  case ARM::LDMDA:
-  case ARM::LDMDB:
-  case ARM::LDMIB:
-
-  case ARM::VLDMDIA:
-  case ARM::VLDMSIA:
-
-  case ARM::PLDi12:
-  case ARM::PLDWi12:
-  case ARM::PLIi12:
-    *AddrIdx = 0;
-    break;
-  // Instructions with base address register in position 1...
-  case ARM::LDMIA_UPD: // same reg at position 0 and position 1
-  case ARM::LDMDA_UPD:
-  case ARM::LDMDB_UPD:
-  case ARM::LDMIB_UPD:
-
-  case ARM::LDRSB:
-  case ARM::LDRH:
-  case ARM::LDRSH:
-
-  case ARM::LDRi12:
-  case ARM::LDRrs:
-  case ARM::LDRBi12:
-  case ARM::LDRBrs:
-  case ARM::VLDMDIA_UPD:
-  case ARM::VLDMDDB_UPD:
-  case ARM::VLDMSIA_UPD:
-  case ARM::VLDMSDB_UPD:
-  case ARM::VLDRS:
-  case ARM::VLDRD:
-
-  case ARM::LDREX:
-  case ARM::LDREXB:
-  case ARM::LDREXH:
-  case ARM::LDREXD:
-    *AddrIdx = 1;
-    break;
-
-  // Instructions with base address register in position 2...
-  case ARM::LDR_PRE_REG:
-  case ARM::LDR_PRE_IMM:
-  case ARM::LDR_POST_REG:
-  case ARM::LDR_POST_IMM:
-
-  case ARM::LDRB_PRE_REG:
-  case ARM::LDRB_PRE_IMM:
-  case ARM::LDRB_POST_REG:
-  case ARM::LDRB_POST_IMM:
-  case ARM::LDRSB_PRE:
-  case ARM::LDRSB_POST:
-
-  case ARM::LDRH_PRE:
-  case ARM::LDRH_POST:
-  case ARM::LDRSH_PRE:
-  case ARM::LDRSH_POST:
-
-  case ARM::LDRD:
-    *AddrIdx = 2;
-    break;
-
-  //
-  // NEON loads
-  //
-
-  // VLD1
-  case ARM::VLD1d8:
-  case ARM::VLD1d16:
-  case ARM::VLD1d32:
-  case ARM::VLD1d64:
-  case ARM::VLD1q8:
-  case ARM::VLD1q16:
-  case ARM::VLD1q32:
-  case ARM::VLD1q64:
-    *AddrIdx = 1;
-    break;
-
-  case ARM::VLD1d8wb_fixed:
-  case ARM::VLD1d16wb_fixed:
-  case ARM::VLD1d32wb_fixed:
-  case ARM::VLD1d64wb_fixed:
-  case ARM::VLD1q8wb_fixed:
-  case ARM::VLD1q16wb_fixed:
-  case ARM::VLD1q32wb_fixed:
-  case ARM::VLD1q64wb_fixed:
-  case ARM::VLD1d8wb_register:
-  case ARM::VLD1d16wb_register:
-  case ARM::VLD1d32wb_register:
-  case ARM::VLD1d64wb_register:
-  case ARM::VLD1q8wb_register:
-  case ARM::VLD1q16wb_register:
-  case ARM::VLD1q32wb_register:
-  case ARM::VLD1q64wb_register:
-    *AddrIdx = 2;
-    break;
-
-  // VLD1T
-  case ARM::VLD1d8T:
-  case ARM::VLD1d16T:
-  case ARM::VLD1d32T:
-  case ARM::VLD1d64T:
-    *AddrIdx = 1;
-    break;
-
-  case ARM::VLD1d8Twb_fixed:
-  case ARM::VLD1d16Twb_fixed:
-  case ARM::VLD1d32Twb_fixed:
-  case ARM::VLD1d64Twb_fixed:
-  case ARM::VLD1d8Twb_register:
-  case ARM::VLD1d16Twb_register:
-  case ARM::VLD1d32Twb_register:
-  case ARM::VLD1d64Twb_register:
-    *AddrIdx = 2;
-    break;
-
-  // VLD1Q
-  case ARM::VLD1d8Q:
-  case ARM::VLD1d16Q:
-  case ARM::VLD1d32Q:
-  case ARM::VLD1d64Q:
-    *AddrIdx = 1;
-    break;
-
-  case ARM::VLD1d8Qwb_fixed:
-  case ARM::VLD1d16Qwb_fixed:
-  case ARM::VLD1d32Qwb_fixed:
-  case ARM::VLD1d64Qwb_fixed:
-  case ARM::VLD1d8Qwb_register:
-  case ARM::VLD1d16Qwb_register:
-  case ARM::VLD1d32Qwb_register:
-  case ARM::VLD1d64Qwb_register:
-    *AddrIdx = 2;
-    break;
-
-  // VLD1LN
-  case ARM::VLD1LNd8:
-  case ARM::VLD1LNd16:
-  case ARM::VLD1LNd32:
-  case ARM::VLD1LNd8_UPD:
-  case ARM::VLD1LNd16_UPD:
-  case ARM::VLD1LNd32_UPD:
-
-  // VLD1DUP
-  case ARM::VLD1DUPd8:
-  case ARM::VLD1DUPd16:
-  case ARM::VLD1DUPd32:
-  case ARM::VLD1DUPq8:
-  case ARM::VLD1DUPq16:
-  case ARM::VLD1DUPq32:
-  case ARM::VLD1DUPd8wb_fixed:
-  case ARM::VLD1DUPd16wb_fixed:
-  case ARM::VLD1DUPd32wb_fixed:
-  case ARM::VLD1DUPq8wb_fixed:
-  case ARM::VLD1DUPq16wb_fixed:
-  case ARM::VLD1DUPq32wb_fixed:
-  case ARM::VLD1DUPd8wb_register:
-  case ARM::VLD1DUPd16wb_register:
-  case ARM::VLD1DUPd32wb_register:
-  case ARM::VLD1DUPq8wb_register:
-  case ARM::VLD1DUPq16wb_register:
-  case ARM::VLD1DUPq32wb_register:
-
-  // VLD2
-  case ARM::VLD2d8:
-  case ARM::VLD2d16:
-  case ARM::VLD2d32:
-  case ARM::VLD2b8:
-  case ARM::VLD2b16:
-  case ARM::VLD2b32:
-  case ARM::VLD2q8:
-  case ARM::VLD2q16:
-  case ARM::VLD2q32:
-    *AddrIdx = 1;
-    break;
-
-  case ARM::VLD2d8wb_fixed:
-  case ARM::VLD2d16wb_fixed:
-  case ARM::VLD2d32wb_fixed:
-  case ARM::VLD2b8wb_fixed:
-  case ARM::VLD2b16wb_fixed:
-  case ARM::VLD2b32wb_fixed:
-  case ARM::VLD2q8wb_fixed:
-  case ARM::VLD2q16wb_fixed:
-  case ARM::VLD2q32wb_fixed:
-  case ARM::VLD2d8wb_register:
-  case ARM::VLD2d16wb_register:
-  case ARM::VLD2d32wb_register:
-  case ARM::VLD2b8wb_register:
-  case ARM::VLD2b16wb_register:
-  case ARM::VLD2b32wb_register:
-  case ARM::VLD2q8wb_register:
-  case ARM::VLD2q16wb_register:
-  case ARM::VLD2q32wb_register:
-    *AddrIdx = 2;
-    break;
-
-  // VLD2LN
-  case ARM::VLD2LNd8:
-  case ARM::VLD2LNd16:
-  case ARM::VLD2LNd32:
-  case ARM::VLD2LNq16:
-  case ARM::VLD2LNq32:
-    *AddrIdx = 2;
-    break;
-
-  case ARM::VLD2LNd8_UPD:
-  case ARM::VLD2LNd16_UPD:
-  case ARM::VLD2LNd32_UPD:
-  case ARM::VLD2LNq16_UPD:
-  case ARM::VLD2LNq32_UPD:
-    *AddrIdx = 3;
-    break;
-
-  // VLD2DUP
-  case ARM::VLD2DUPd8:
-  case ARM::VLD2DUPd16:
-  case ARM::VLD2DUPd32:
-  case ARM::VLD2DUPd8x2:
-  case ARM::VLD2DUPd16x2:
-  case ARM::VLD2DUPd32x2:
-    *AddrIdx = 1;
-    break;
-
-  case ARM::VLD2DUPd8wb_fixed:
-  case ARM::VLD2DUPd16wb_fixed:
-  case ARM::VLD2DUPd32wb_fixed:
-  case ARM::VLD2DUPd8wb_register:
-  case ARM::VLD2DUPd16wb_register:
-  case ARM::VLD2DUPd32wb_register:
-  case ARM::VLD2DUPd8x2wb_fixed:
-  case ARM::VLD2DUPd16x2wb_fixed:
-  case ARM::VLD2DUPd32x2wb_fixed:
-  case ARM::VLD2DUPd8x2wb_register:
-  case ARM::VLD2DUPd16x2wb_register:
-  case ARM::VLD2DUPd32x2wb_register:
-    *AddrIdx = 2;
-    break;
-
-  // VLD3
-  case ARM::VLD3d8:
-  case ARM::VLD3d16:
-  case ARM::VLD3d32:
-  case ARM::VLD3q8:
-  case ARM::VLD3q16:
-  case ARM::VLD3q32:
-  case ARM::VLD3d8_UPD:
-  case ARM::VLD3d16_UPD:
-  case ARM::VLD3d32_UPD:
-  case ARM::VLD3q8_UPD:
-  case ARM::VLD3q16_UPD:
-  case ARM::VLD3q32_UPD:
-
-  // VLD3LN
-  case ARM::VLD3LNd8:
-  case ARM::VLD3LNd16:
-  case ARM::VLD3LNd32:
-  case ARM::VLD3LNq16:
-  case ARM::VLD3LNq32:
-    *AddrIdx = 3;
-    break;
-
-  case ARM::VLD3LNd8_UPD:
-  case ARM::VLD3LNd16_UPD:
-  case ARM::VLD3LNd32_UPD:
-  case ARM::VLD3LNq16_UPD:
-  case ARM::VLD3LNq32_UPD:
-    *AddrIdx = 4;
-    break;
-
-  // VLD3DUP
-  case ARM::VLD3DUPd8:
-  case ARM::VLD3DUPd16:
-  case ARM::VLD3DUPd32:
-  case ARM::VLD3DUPq8:
-  case ARM::VLD3DUPq16:
-  case ARM::VLD3DUPq32:
-    *AddrIdx = 3;
-    break;
-
-  case ARM::VLD3DUPd8_UPD:
-  case ARM::VLD3DUPd16_UPD:
-  case ARM::VLD3DUPd32_UPD:
-  case ARM::VLD3DUPq8_UPD:
-  case ARM::VLD3DUPq16_UPD:
-  case ARM::VLD3DUPq32_UPD:
-    *AddrIdx = 4;
-    break;
-
-  // VLD4
-  case ARM::VLD4d8:
-  case ARM::VLD4d16:
-  case ARM::VLD4d32:
-  case ARM::VLD4q8:
-  case ARM::VLD4q16:
-  case ARM::VLD4q32:
-    *AddrIdx = 4;
-    break;
-
-  case ARM::VLD4d8_UPD:
-  case ARM::VLD4d16_UPD:
-  case ARM::VLD4d32_UPD:
-  case ARM::VLD4q8_UPD:
-  case ARM::VLD4q16_UPD:
-  case ARM::VLD4q32_UPD:
-    *AddrIdx = 5;
-    break;
-
-  // VLD4LN
-  case ARM::VLD4LNd8:
-  case ARM::VLD4LNd16:
-  case ARM::VLD4LNd32:
-  case ARM::VLD4LNq16:
-  case ARM::VLD4LNq32:
-    *AddrIdx = 4;
-    break;
-
-  case ARM::VLD4LNd8_UPD:
-  case ARM::VLD4LNd16_UPD:
-  case ARM::VLD4LNd32_UPD:
-  case ARM::VLD4LNq16_UPD:
-  case ARM::VLD4LNq32_UPD:
-    *AddrIdx = 5;
-    break;
-
-  case ARM::VLD4DUPd8:
-  case ARM::VLD4DUPd16:
-  case ARM::VLD4DUPd32:
-  case ARM::VLD4DUPq16:
-  case ARM::VLD4DUPq32:
-    *AddrIdx = 4;
-    break;
-
-  case ARM::VLD4DUPd8_UPD:
-  case ARM::VLD4DUPd16_UPD:
-  case ARM::VLD4DUPd32_UPD:
-  case ARM::VLD4DUPq16_UPD:
-  case ARM::VLD4DUPq32_UPD:
-    *AddrIdx = 5;
-    break;
-  }
-
-  if (MI.getOperand(*AddrIdx).getReg() == ARM::SP) {
-    // The contents of SP do not require masking.
-    return false;
-  }
-
-  return true;
-}
-
-/*
- * Sandboxes a memory reference instruction by inserting an appropriate mask
- * or check operation before it.
- */
-void ARMNaClRewritePass::SandboxMemory(MachineBasicBlock &MBB,
-                                       MachineBasicBlock::iterator MBBI,
-                                       MachineInstr &MI,
-                                       int AddrIdx,
-                                       bool IsLoad) {
-  unsigned Addr = MI.getOperand(AddrIdx).getReg();
-
-  if (Addr == ARM::R9) {
-    // R9-relative loads are no longer sandboxed.
-    assert(IsLoad && "There should be no r9-relative stores");
-  } else {
-    unsigned Opcode;
-    if (IsLoad && (MI.getOperand(0).getReg() == ARM::SP)) {
-      Opcode = ARM::SFI_GUARD_SP_LOAD;
-    } else {
-      Opcode = ARM::SFI_GUARD_LOADSTORE;
-    }
-    // Use same predicate as current instruction.
-    unsigned PredReg = 0;
-    ARMCC::CondCodes Pred = llvm::getInstrPredicate(&MI, PredReg);
-    // Use the older BIC sandbox, which is universal, but incurs a stall.
-    BuildMI(MBB, MBBI, MI.getDebugLoc(), TII->get(Opcode))
-      .addReg(Addr, RegState::Define)  // Address definition (as dst).
-      .addReg(Addr, RegState::Kill)    // Address read (as src).
-      .addImm((int64_t) Pred)          // predicate condition
-      .addReg(PredReg);                // predicate source register (CPSR)
-
-    /*
-     * This pseudo-instruction is intended to generate something resembling the
-     * following, but with alignment enforced.
-     * TODO(cbiffle): move alignment into this function, use the code below.
-     *
-     *  // bic<cc> Addr, Addr, #0xC0000000
-     *  BuildMI(MBB, MBBI, MI.getDebugLoc(),
-     *          TII->get(ARM::BICri))
-     *    .addReg(Addr)            // rD
-     *    .addReg(Addr)            // rN
-     *    .addImm(0xC0000000)      // imm
-     *    .addImm((int64_t) Pred)  // predicate condition
-     *    .addReg(PredReg)         // predicate source register (CPSR)
-     *    .addReg(0);              // flag output register (0 == no flags)
-     */
-  }
-}
-
-static bool IsDangerousStore(const MachineInstr &MI, int *AddrIdx) {
-  unsigned Opcode = MI.getOpcode();
-  switch (Opcode) {
-  default: return false;
-
-  // Instructions with base address register in position 0...
-  case ARM::STMIA:
-  case ARM::STMDA:
-  case ARM::STMDB:
-  case ARM::STMIB:
-
-  case ARM::VSTMDIA:
-  case ARM::VSTMSIA:
-    *AddrIdx = 0;
-    break;
-
-  // Instructions with base address register in position 1...
-  case ARM::STMIA_UPD: // same reg at position 0 and position 1
-  case ARM::STMDA_UPD:
-  case ARM::STMDB_UPD:
-  case ARM::STMIB_UPD:
-
-  case ARM::STRH:
-  case ARM::STRi12:
-  case ARM::STRrs:
-  case ARM::STRBi12:
-  case ARM::STRBrs:
-  case ARM::VSTMDIA_UPD:
-  case ARM::VSTMDDB_UPD:
-  case ARM::VSTMSIA_UPD:
-  case ARM::VSTMSDB_UPD:
-  case ARM::VSTRS:
-  case ARM::VSTRD:
-    *AddrIdx = 1;
-    break;
-
-  //
-  // NEON stores
-  //
-
-  // VST1
-  case ARM::VST1d8:
-  case ARM::VST1d16:
-  case ARM::VST1d32:
-  case ARM::VST1d64:
-  case ARM::VST1q8:
-  case ARM::VST1q16:
-  case ARM::VST1q32:
-  case ARM::VST1q64:
-    *AddrIdx = 0;
-    break;
-
-  case ARM::VST1d8wb_fixed:
-  case ARM::VST1d16wb_fixed:
-  case ARM::VST1d32wb_fixed:
-  case ARM::VST1d64wb_fixed:
-  case ARM::VST1q8wb_fixed:
-  case ARM::VST1q16wb_fixed:
-  case ARM::VST1q32wb_fixed:
-  case ARM::VST1q64wb_fixed:
-  case ARM::VST1d8wb_register:
-  case ARM::VST1d16wb_register:
-  case ARM::VST1d32wb_register:
-  case ARM::VST1d64wb_register:
-  case ARM::VST1q8wb_register:
-  case ARM::VST1q16wb_register:
-  case ARM::VST1q32wb_register:
-  case ARM::VST1q64wb_register:
-    *AddrIdx = 1;
-    break;
-
-  // VST1LN
-  case ARM::VST1LNd8:
-  case ARM::VST1LNd16:
-  case ARM::VST1LNd32:
-    *AddrIdx = 0;
-    break;
-
-  case ARM::VST1LNd8_UPD:
-  case ARM::VST1LNd16_UPD:
-  case ARM::VST1LNd32_UPD:
-    *AddrIdx = 1;
-    break;
-
-  // VST2
-  case ARM::VST2d8:
-  case ARM::VST2d16:
-  case ARM::VST2d32:
-  case ARM::VST2q8:
-  case ARM::VST2q16:
-  case ARM::VST2q32:
-    *AddrIdx = 0;
-    break;
-
-  case ARM::VST2d8wb_fixed:
-  case ARM::VST2d16wb_fixed:
-  case ARM::VST2d32wb_fixed:
-  case ARM::VST2q8wb_fixed:
-  case ARM::VST2q16wb_fixed:
-  case ARM::VST2q32wb_fixed:
-  case ARM::VST2d8wb_register:
-  case ARM::VST2d16wb_register:
-  case ARM::VST2d32wb_register:
-  case ARM::VST2q8wb_register:
-  case ARM::VST2q16wb_register:
-  case ARM::VST2q32wb_register:
-    *AddrIdx = 1;
-    break;
-
-  // VST2LN
-  case ARM::VST2LNd8:
-  case ARM::VST2LNd16:
-  case ARM::VST2LNq16:
-  case ARM::VST2LNd32:
-  case ARM::VST2LNq32:
-    *AddrIdx = 0;
-    break;
-
-  case ARM::VST2LNd8_UPD:
-  case ARM::VST2LNd16_UPD:
-  case ARM::VST2LNq16_UPD:
-  case ARM::VST2LNd32_UPD:
-  case ARM::VST2LNq32_UPD:
-    *AddrIdx = 1;
-    break;
-
-  // VST3
-  case ARM::VST3d8:
-  case ARM::VST3d16:
-  case ARM::VST3d32:
-  case ARM::VST3q8:
-  case ARM::VST3q16:
-  case ARM::VST3q32:
-    *AddrIdx = 0;
-    break;
-
-  case ARM::VST3d8_UPD:
-  case ARM::VST3d16_UPD:
-  case ARM::VST3d32_UPD:
-  case ARM::VST3q8_UPD:
-  case ARM::VST3q16_UPD:
-  case ARM::VST3q32_UPD:
-    *AddrIdx = 1;
-    break;
-
-  // VST3LN
-  case ARM::VST3LNd8:
-  case ARM::VST3LNd16:
-  case ARM::VST3LNq16:
-  case ARM::VST3LNd32:
-  case ARM::VST3LNq32:
-    *AddrIdx = 0;
-    break;
-
-  case ARM::VST3LNd8_UPD:
-  case ARM::VST3LNd16_UPD:
-  case ARM::VST3LNq16_UPD:
-  case ARM::VST3LNd32_UPD:
-  case ARM::VST3LNq32_UPD:
-    *AddrIdx = 1;
-    break;
-
-  // VST4
-  case ARM::VST4d8:
-  case ARM::VST4d16:
-  case ARM::VST4d32:
-  case ARM::VST4q8:
-  case ARM::VST4q16:
-  case ARM::VST4q32:
-    *AddrIdx = 0;
-    break;
-
-  case ARM::VST4d8_UPD:
-  case ARM::VST4d16_UPD:
-  case ARM::VST4d32_UPD:
-  case ARM::VST4q8_UPD:
-  case ARM::VST4q16_UPD:
-  case ARM::VST4q32_UPD:
-    *AddrIdx = 1;
-    break;
-
-  // VST4LN
-  case ARM::VST4LNd8:
-  case ARM::VST4LNd16:
-  case ARM::VST4LNq16:
-  case ARM::VST4LNd32:
-  case ARM::VST4LNq32:
-    *AddrIdx = 0;
-    break;
-
-  case ARM::VST4LNd8_UPD:
-  case ARM::VST4LNd16_UPD:
-  case ARM::VST4LNq16_UPD:
-  case ARM::VST4LNd32_UPD:
-  case ARM::VST4LNq32_UPD:
-    *AddrIdx = 1;
-    break;
-
-  // Instructions with base address register in position 2...
-  case ARM::STR_PRE_REG:
-  case ARM::STR_PRE_IMM:
-  case ARM::STR_POST_REG:
-  case ARM::STR_POST_IMM:
-
-  case ARM::STRB_PRE_REG:
-  case ARM::STRB_PRE_IMM:
-  case ARM::STRB_POST_REG:
-  case ARM::STRB_POST_IMM:
-
-  case ARM::STRH_PRE:
-  case ARM::STRH_POST:
-
-
-  case ARM::STRD:
-  case ARM::STREX:
-  case ARM::STREXB:
-  case ARM::STREXH:
-  case ARM::STREXD:
-    *AddrIdx = 2;
-    break;
-  }
-
-  if (MI.getOperand(*AddrIdx).getReg() == ARM::SP) {
-    // The contents of SP do not require masking.
-    return false;
-  }
-
-  return true;
-}
-
-bool ARMNaClRewritePass::SandboxMemoryReferencesInBlock(
-    MachineBasicBlock &MBB) {
-  bool Modified = false;
-  for (MachineBasicBlock::iterator MBBI = MBB.begin(), E = MBB.end();
-       MBBI != E;
-       ++MBBI) {
-    MachineInstr &MI = *MBBI;
-    int AddrIdx;
-
-    if (IsDangerousLoad(MI, &AddrIdx)) {
-      SandboxMemory(MBB, MBBI, MI, AddrIdx, true);
-      Modified = true;
-    }
-    if (IsDangerousStore(MI, &AddrIdx)) {
-      SandboxMemory(MBB, MBBI, MI, AddrIdx, false);
-      Modified = true;
-    }
-  }
-  return Modified;
-}
-
-/**********************************************************************/
-
-bool ARMNaClRewritePass::runOnMachineFunction(MachineFunction &MF) {
-  TII = static_cast<const ARMBaseInstrInfo*>(MF.getSubtarget().getInstrInfo());
-  TRI = MF.getSubtarget().getRegisterInfo();
-
-  bool Modified = false;
-  for (MachineFunction::iterator MFI = MF.begin(), E = MF.end();
-       MFI != E;
-       ++MFI) {
-    MachineBasicBlock &MBB = *MFI;
-
-    if (MBB.hasAddressTaken()) {
-      //FIXME: use symbolic constant or get this value from some configuration
-      MBB.setAlignment(4);
-      Modified = true;
-    }
-
-    Modified |= SandboxMemoryReferencesInBlock(MBB);
-    Modified |= SandboxBranchesInBlock(MBB);
-    Modified |= SandboxStackChangesInBlock(MBB);
-  }
-  DEBUG(LightweightVerify(MF));
-  return Modified;
-}
-
-/// createARMNaClRewritePass - returns an instance of the NaClRewritePass.
-FunctionPass *llvm::createARMNaClRewritePass() {
-  return new ARMNaClRewritePass();
-}
diff --git a/lib/Target/ARM/ARMNaClRewritePass.h b/lib/Target/ARM/ARMNaClRewritePass.h
deleted file mode 100644
index 20e48308ac7..00000000000
--- a/lib/Target/ARM/ARMNaClRewritePass.h
+++ /dev/null
@@ -1,27 +0,0 @@
-//===-- ARMNaClRewritePass.h - NaCl Sandboxing Pass    ------- --*- C++ -*-===//
-//
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
-//
-//===----------------------------------------------------------------------===//
-
-#ifndef TARGET_ARMNACLREWRITEPASS_H
-#define TARGET_ARMNACLREWRITEPASS_H
-
-#include "llvm/MC/MCNaCl.h"
-#include "llvm/Target/TargetRegisterInfo.h"
-#include "llvm/CodeGen/MachineInstr.h"
-
-namespace ARM_SFI {
-
-bool IsStackChange(const llvm::MachineInstr &MI,
-                   const llvm::TargetRegisterInfo *TRI);
-bool IsSandboxedStackChange(const llvm::MachineInstr &MI);
-bool NeedSandboxStackChange(const llvm::MachineInstr &MI,
-                               const llvm::TargetRegisterInfo *TRI);
-
-} // namespace ARM_SFI
-
-#endif
diff --git a/lib/Target/ARM/ARMOptimizeBarriersPass.cpp b/lib/Target/ARM/ARMOptimizeBarriersPass.cpp
index 1c50f9e9acf..30baf4263c1 100644
--- a/lib/Target/ARM/ARMOptimizeBarriersPass.cpp
+++ b/lib/Target/ARM/ARMOptimizeBarriersPass.cpp
@@ -30,8 +30,6 @@ class ARMOptimizeBarriersPass : public MachineFunctionPass {
   const char *getPassName() const override {
     return "optimise barriers pass";
   }
-
-private:
 };
 char ARMOptimizeBarriersPass::ID = 0;
 }
diff --git a/lib/Target/ARM/ARMScheduleSwift.td b/lib/Target/ARM/ARMScheduleSwift.td
index b03d5ff44c6..6f5740fd130 100644
--- a/lib/Target/ARM/ARMScheduleSwift.td
+++ b/lib/Target/ARM/ARMScheduleSwift.td
@@ -37,1050 +37,12 @@ def SW_FDIV : FuncUnit;
 // FIXME: Add preload instruction when it is documented.
 // FIXME: Model non-pipelined nature of FP div / sqrt unit.
 
-def SwiftItineraries : ProcessorItineraries<
-  [SW_DIS0, SW_DIS1, SW_DIS2, SW_ALU0, SW_ALU1, SW_LS, SW_IDIV, SW_FDIV], [], [
-  //
-  // Move instructions, unconditional
-  InstrItinData<IIC_iMOVi   , [InstrStage<1, [SW_DIS0, SW_DIS1, SW_DIS2], 0>,
-                               InstrStage<1, [SW_ALU0, SW_ALU1]>],
-                              [1]>,
-  InstrItinData<IIC_iMOVr   , [InstrStage<1, [SW_DIS0, SW_DIS1, SW_DIS2], 0>,
-                               InstrStage<1, [SW_ALU0, SW_ALU1]>],
-                              [1]>,
-  InstrItinData<IIC_iMOVsi  , [InstrStage<1, [SW_DIS0, SW_DIS1, SW_DIS2], 0>,
-                               InstrStage<1, [SW_ALU0, SW_ALU1]>],
-                              [1]>,
-  InstrItinData<IIC_iMOVsr  , [InstrStage<1, [SW_DIS0, SW_DIS1, SW_DIS2], 0>,
-                               InstrStage<1, [SW_ALU0, SW_ALU1]>],
-                              [1]>,
-  InstrItinData<IIC_iMOVix2 , [InstrStage<1, [SW_DIS0, SW_DIS1, SW_DIS2], 0>,
-                               InstrStage<1, [SW_DIS0, SW_DIS1, SW_DIS2], 0>,
-                               InstrStage<1, [SW_ALU0, SW_ALU1]>,
-                               InstrStage<1, [SW_ALU0, SW_ALU1]>],
-                              [2]>,
-  InstrItinData<IIC_iMOVix2addpc,[InstrStage<1, [SW_DIS0, SW_DIS1, SW_DIS2], 0>,
-                                  InstrStage<1, [SW_ALU0, SW_ALU1]>,
-                                  InstrStage<1, [SW_ALU0, SW_ALU1]>,
-                                  InstrStage<1, [SW_ALU0, SW_ALU1]>],
-                                 [3]>,
-  InstrItinData<IIC_iMOVix2ld,[InstrStage<1, [SW_DIS0, SW_DIS1, SW_DIS2], 0>,
-                               InstrStage<1, [SW_ALU0, SW_ALU1]>,
-                               InstrStage<1, [SW_ALU0, SW_ALU1]>,
-                               InstrStage<1, [SW_LS]>],
-                              [5]>,
-  //
-  // MVN instructions
-  InstrItinData<IIC_iMVNi   , [InstrStage<1, [SW_DIS0, SW_DIS1, SW_DIS2], 0>,
-                               InstrStage<1, [SW_ALU0, SW_ALU1]>],
-                              [1]>,
-  InstrItinData<IIC_iMVNr   , [InstrStage<1, [SW_DIS0, SW_DIS1, SW_DIS2], 0>,
-                               InstrStage<1, [SW_ALU0, SW_ALU1]>],
-                              [1]>,
-  InstrItinData<IIC_iMVNsi  , [InstrStage<1, [SW_DIS0, SW_DIS1, SW_DIS2], 0>,
-                               InstrStage<1, [SW_ALU0, SW_ALU1]>],
-                              [1]>,
-  InstrItinData<IIC_iMVNsr  , [InstrStage<1, [SW_DIS0, SW_DIS1, SW_DIS2], 0>,
-                               InstrStage<1, [SW_ALU0, SW_ALU1]>],
-                              [1]>,
-  //
-  // No operand cycles
-  InstrItinData<IIC_iALUx   , [InstrStage<1, [SW_DIS0, SW_DIS1, SW_DIS2], 0>,
-                               InstrStage<1, [SW_ALU0, SW_ALU1]>]>,
-  //
-  // Binary Instructions that produce a result
-  InstrItinData<IIC_iALUi , [InstrStage<1, [SW_DIS0, SW_DIS1, SW_DIS2], 0>,
-                             InstrStage<1, [SW_ALU0, SW_ALU1]>],
-                            [1, 1]>,
-  InstrItinData<IIC_iALUr , [InstrStage<1, [SW_DIS0, SW_DIS1, SW_DIS2], 0>,
-                             InstrStage<1, [SW_ALU0, SW_ALU1]>],
-                            [1, 1, 1]>,
-  InstrItinData<IIC_iALUsi, [InstrStage<1, [SW_DIS0, SW_DIS1, SW_DIS2], 0>,
-                             InstrStage<1, [SW_ALU0, SW_ALU1]>],
-                            [2, 1, 1]>,
-  InstrItinData<IIC_iALUsir,[InstrStage<1, [SW_DIS0, SW_DIS1, SW_DIS2], 0>,
-                             InstrStage<1, [SW_ALU0, SW_ALU1]>],
-                            [2, 1, 1]>,
-  InstrItinData<IIC_iALUsr, [InstrStage<1, [SW_DIS0, SW_DIS1, SW_DIS2], 0>,
-                             InstrStage<1, [SW_ALU0, SW_ALU1]>],
-                            [2, 1, 1, 1]>,
-  //
-  // Bitwise Instructions that produce a result
-  InstrItinData<IIC_iBITi , [InstrStage<1, [SW_DIS0, SW_DIS1, SW_DIS2], 0>,
-                             InstrStage<1, [SW_ALU0, SW_ALU1]>],
-                            [1, 1]>,
-  InstrItinData<IIC_iBITr , [InstrStage<1, [SW_DIS0, SW_DIS1, SW_DIS2], 0>,
-                             InstrStage<1, [SW_ALU0, SW_ALU1]>],
-                            [1, 1, 1]>,
-  InstrItinData<IIC_iBITsi, [InstrStage<1, [SW_DIS0, SW_DIS1, SW_DIS2], 0>,
-                             InstrStage<1, [SW_ALU0, SW_ALU1]>],
-                            [2, 1, 1]>,
-  InstrItinData<IIC_iBITsr, [InstrStage<1, [SW_DIS0, SW_DIS1, SW_DIS2], 0>,
-                             InstrStage<1, [SW_ALU0, SW_ALU1]>],
-                            [2, 1, 1, 1]>,
-  //
-  // Unary Instructions that produce a result
-
-  // CLZ, RBIT, etc.
-  InstrItinData<IIC_iUNAr , [InstrStage<1, [SW_DIS0, SW_DIS1, SW_DIS2], 0>,
-                             InstrStage<1, [SW_ALU0, SW_ALU1]>],
-                            [1, 1]>,
-
-  // BFC, BFI, UBFX, SBFX
-  InstrItinData<IIC_iUNAsi, [InstrStage<1, [SW_DIS0, SW_DIS1, SW_DIS2], 0>,
-                             InstrStage<1, [SW_ALU0, SW_ALU1]>],
-                            [2, 1]>,
-
-  //
-  // Zero and sign extension instructions
-  InstrItinData<IIC_iEXTr , [InstrStage<1, [SW_DIS0, SW_DIS1, SW_DIS2], 0>,
-                             InstrStage<1, [SW_ALU0, SW_ALU1]>],
-                            [1, 1]>,
-  InstrItinData<IIC_iEXTAr, [InstrStage<1, [SW_DIS0, SW_DIS1, SW_DIS2], 0>,
-                             InstrStage<1, [SW_ALU0, SW_ALU1]>],
-                            [1, 1, 1]>,
-  InstrItinData<IIC_iEXTAsr,[InstrStage<1, [SW_DIS0, SW_DIS1, SW_DIS2], 0>,
-                             InstrStage<1, [SW_ALU0, SW_ALU1]>],
-                            [1, 1, 1, 1]>,
-  //
-  // Compare instructions
-  InstrItinData<IIC_iCMPi   , [InstrStage<1, [SW_DIS0, SW_DIS1, SW_DIS2], 0>,
-                               InstrStage<1, [SW_ALU0, SW_ALU1]>],
-                              [1]>,
-  InstrItinData<IIC_iCMPr   , [InstrStage<1, [SW_DIS0, SW_DIS1, SW_DIS2], 0>,
-                               InstrStage<1, [SW_ALU0, SW_ALU1]>],
-                              [1, 1]>,
-  InstrItinData<IIC_iCMPsi  , [InstrStage<1, [SW_DIS0, SW_DIS1, SW_DIS2], 0>,
-                               InstrStage<2, [SW_ALU0, SW_ALU1]>],
-                              [1, 1]>,
-  InstrItinData<IIC_iCMPsr  , [InstrStage<1, [SW_DIS0, SW_DIS1, SW_DIS2], 0>,
-                               InstrStage<2, [SW_ALU0, SW_ALU1]>],
-                              [1, 1, 1]>,
-  //
-  // Test instructions
-  InstrItinData<IIC_iTSTi   , [InstrStage<1, [SW_DIS0, SW_DIS1, SW_DIS2], 0>,
-                               InstrStage<1, [SW_ALU0, SW_ALU1]>],
-                              [1]>,
-  InstrItinData<IIC_iTSTr   , [InstrStage<1, [SW_DIS0, SW_DIS1, SW_DIS2], 0>,
-                               InstrStage<1, [SW_ALU0, SW_ALU1]>],
-                              [1, 1]>,
-  InstrItinData<IIC_iTSTsi  , [InstrStage<1, [SW_DIS0, SW_DIS1, SW_DIS2], 0>,
-                               InstrStage<2, [SW_ALU0, SW_ALU1]>],
-                              [1, 1]>,
-  InstrItinData<IIC_iTSTsr  , [InstrStage<1, [SW_DIS0, SW_DIS1, SW_DIS2], 0>,
-                               InstrStage<2, [SW_ALU0, SW_ALU1]>],
-                              [1, 1, 1]>,
-  //
-  // Move instructions, conditional
-  // FIXME: Correctly model the extra input dep on the destination.
-  InstrItinData<IIC_iCMOVi  , [InstrStage<1, [SW_DIS0, SW_DIS1, SW_DIS2], 0>,
-                               InstrStage<1, [SW_ALU0, SW_ALU1]>],
-                              [1]>,
-  InstrItinData<IIC_iCMOVr  , [InstrStage<1, [SW_DIS0, SW_DIS1, SW_DIS2], 0>,
-                               InstrStage<1, [SW_ALU0, SW_ALU1]>],
-                              [1, 1]>,
-  InstrItinData<IIC_iCMOVsi , [InstrStage<1, [SW_DIS0, SW_DIS1, SW_DIS2], 0>,
-                               InstrStage<1, [SW_ALU0, SW_ALU1]>],
-                              [1, 1]>,
-  InstrItinData<IIC_iCMOVsr , [InstrStage<1, [SW_DIS0, SW_DIS1, SW_DIS2], 0>,
-                               InstrStage<1, [SW_ALU0, SW_ALU1]>],
-                              [2, 1, 1]>,
-  InstrItinData<IIC_iCMOVix2, [InstrStage<1, [SW_DIS0, SW_DIS1, SW_DIS2], 0>,
-                               InstrStage<1, [SW_DIS0, SW_DIS1, SW_DIS2], 0>,
-                               InstrStage<1, [SW_ALU0, SW_ALU1]>,
-                               InstrStage<1, [SW_ALU0, SW_ALU1]>],
-                              [2]>,
-
-  // Integer multiply pipeline
-  //
-  InstrItinData<IIC_iMUL16  , [InstrStage<1, [SW_DIS0, SW_DIS1, SW_DIS2], 0>,
-                               InstrStage<1, [SW_ALU0]>],
-                              [3, 1, 1]>,
-  InstrItinData<IIC_iMAC16  , [InstrStage<1, [SW_DIS0, SW_DIS1, SW_DIS2], 0>,
-                               InstrStage<1, [SW_ALU0]>],
-                              [3, 1, 1, 1]>,
-  InstrItinData<IIC_iMUL32  , [InstrStage<1, [SW_DIS0, SW_DIS1, SW_DIS2], 0>,
-                               InstrStage<1, [SW_ALU0]>],
-                              [4, 1, 1]>,
-  InstrItinData<IIC_iMAC32  , [InstrStage<1, [SW_DIS0, SW_DIS1, SW_DIS2], 0>,
-                               InstrStage<1, [SW_ALU0]>],
-                              [4, 1, 1, 1]>,
-  InstrItinData<IIC_iMUL64  , [InstrStage<1, [SW_DIS0], 0>,
-                               InstrStage<1, [SW_DIS1], 0>,
-                               InstrStage<1, [SW_DIS2], 0>,
-                               InstrStage<1, [SW_ALU0], 1>,
-                               InstrStage<1, [SW_ALU0], 3>,
-                               InstrStage<1, [SW_ALU0]>],
-                              [5, 5, 1, 1]>,
-  InstrItinData<IIC_iMAC64  , [InstrStage<1, [SW_DIS0], 0>,
-                               InstrStage<1, [SW_DIS1], 0>,
-                               InstrStage<1, [SW_DIS2], 0>,
-                               InstrStage<1, [SW_ALU0], 1>,
-                               InstrStage<1, [SW_ALU0], 1>,
-                               InstrStage<1, [SW_ALU0, SW_ALU1], 3>,
-                               InstrStage<1, [SW_ALU0, SW_ALU1]>],
-                              [5, 6, 1, 1]>,
-  //
-  // Integer divide
-  InstrItinData<IIC_iDIV  , [InstrStage<1, [SW_DIS0, SW_DIS1, SW_DIS2], 0>,
-                             InstrStage<1, [SW_ALU0], 0>,
-                             InstrStage<14, [SW_IDIV]>],
-                            [14, 1, 1]>,
-
-  // Integer load pipeline
-  // FIXME: The timings are some rough approximations
-  //
-  // Immediate offset
-  InstrItinData<IIC_iLoad_i   , [InstrStage<1, [SW_DIS0, SW_DIS1, SW_DIS2], 0>,
-                                 InstrStage<1, [SW_LS]>],
-                                [3, 1]>,
-  InstrItinData<IIC_iLoad_bh_i, [InstrStage<1, [SW_DIS0, SW_DIS1, SW_DIS2], 0>,
-                                 InstrStage<1, [SW_LS]>],
-                                [3, 1]>,
-  InstrItinData<IIC_iLoad_d_i , [InstrStage<1, [SW_DIS0], 0>,
-                                 InstrStage<1, [SW_DIS1], 0>,
-                                 InstrStage<1, [SW_LS], 1>,
-                                 InstrStage<1, [SW_LS]>],
-                                [3, 4, 1]>,
-  //
-  // Register offset
-  InstrItinData<IIC_iLoad_r   , [InstrStage<1, [SW_DIS0, SW_DIS1, SW_DIS2], 0>,
-                                 InstrStage<1, [SW_LS]>],
-                                [3, 1, 1]>,
-  InstrItinData<IIC_iLoad_bh_r, [InstrStage<1, [SW_DIS0, SW_DIS1, SW_DIS2], 0>,
-                                 InstrStage<1, [SW_LS]>],
-                                [3, 1, 1]>,
-  InstrItinData<IIC_iLoad_d_r , [InstrStage<1, [SW_DIS0], 0>,
-                                 InstrStage<1, [SW_DIS1], 0>,
-                                 InstrStage<1, [SW_DIS2], 0>,
-                                 InstrStage<1, [SW_LS], 1>,
-                                 InstrStage<1, [SW_LS], 3>,
-                                 InstrStage<1, [SW_ALU0, SW_ALU1]>],
-                                [3, 4, 1, 1]>,
-  //
-  // Scaled register offset
-  InstrItinData<IIC_iLoad_si  , [InstrStage<1, [SW_DIS0], 0>,
-                                 InstrStage<1, [SW_DIS1], 0>,
-                                 InstrStage<1, [SW_ALU0, SW_ALU1], 2>,
-                                 InstrStage<1, [SW_LS]>],
-                                [5, 1, 1]>,
-  InstrItinData<IIC_iLoad_bh_si,[InstrStage<1, [SW_DIS0], 0>,
-                                 InstrStage<1, [SW_DIS1], 0>,
-                                 InstrStage<1, [SW_ALU0, SW_ALU1], 2>,
-                                 InstrStage<1, [SW_LS]>],
-                                [5, 1, 1]>,
-  //
-  // Immediate offset with update
-  InstrItinData<IIC_iLoad_iu  , [InstrStage<1, [SW_DIS0], 0>,
-                                 InstrStage<1, [SW_DIS1], 0>,
-                                 InstrStage<1, [SW_ALU0, SW_ALU1], 1>,
-                                 InstrStage<1, [SW_LS]>],
-                                [3, 1, 1]>,
-  InstrItinData<IIC_iLoad_bh_iu,[InstrStage<1, [SW_DIS0], 0>,
-                                 InstrStage<1, [SW_DIS1], 0>,
-                                 InstrStage<1, [SW_ALU0, SW_ALU1], 1>,
-                                 InstrStage<1, [SW_LS]>],
-                                [3, 1, 1]>,
-  //
-  // Register offset with update
-  InstrItinData<IIC_iLoad_ru  , [InstrStage<1, [SW_DIS0], 0>,
-                                 InstrStage<1, [SW_DIS1], 0>,
-                                 InstrStage<1, [SW_ALU0], 1>,
-                                 InstrStage<1, [SW_LS]>],
-                                [3, 1, 1, 1]>,
-  InstrItinData<IIC_iLoad_bh_ru,[InstrStage<1, [SW_DIS0], 0>,
-                                 InstrStage<1, [SW_DIS1], 0>,
-                                 InstrStage<1, [SW_ALU0], 1>,
-                                 InstrStage<1, [SW_LS]>],
-                                [3, 1, 1, 1]>,
-  InstrItinData<IIC_iLoad_d_ru, [InstrStage<1, [SW_DIS0], 0>,
-                                 InstrStage<1, [SW_DIS1], 0>,
-                                 InstrStage<1, [SW_DIS2], 0>,
-                                 InstrStage<1, [SW_ALU0, SW_ALU1], 0>,
-                                 InstrStage<1, [SW_LS], 3>,
-                                 InstrStage<1, [SW_LS], 0>,
-                                 InstrStage<1, [SW_ALU0, SW_ALU1]>],
-                                [3, 4, 1, 1]>,
-  //
-  // Scaled register offset with update
-  InstrItinData<IIC_iLoad_siu , [InstrStage<1, [SW_DIS0], 0>,
-                                 InstrStage<1, [SW_DIS1], 0>,
-                                 InstrStage<1, [SW_DIS2], 0>,
-                                 InstrStage<1, [SW_ALU0, SW_ALU1], 2>,
-                                 InstrStage<1, [SW_LS], 3>,
-                                 InstrStage<1, [SW_ALU0, SW_ALU1]>],
-                                [5, 3, 1, 1]>,
-  InstrItinData<IIC_iLoad_bh_siu,[InstrStage<1, [SW_DIS0], 0>,
-                                  InstrStage<1, [SW_DIS1], 0>,
-                                  InstrStage<1, [SW_DIS2], 0>,
-                                  InstrStage<1, [SW_ALU0, SW_ALU1], 2>,
-                                  InstrStage<1, [SW_LS], 0>,
-                                  InstrStage<1, [SW_ALU0, SW_ALU1]>],
-                                [5, 3, 1, 1]>,
-  //
-  // Load multiple, def is the 5th operand.
-  // FIXME: This assumes 3 to 4 registers.
-  InstrItinData<IIC_iLoad_m  , [InstrStage<1, [SW_DIS0], 0>,
-                                InstrStage<1, [SW_DIS1], 0>,
-                                InstrStage<1, [SW_DIS2], 0>,
-                                InstrStage<1, [SW_ALU0, SW_ALU1], 1>,
-                                InstrStage<1, [SW_LS]>],
-                               [1, 1, 1, 1, 3], [], -1>, // dynamic uops
-
-  //
-  // Load multiple + update, defs are the 1st and 5th operands.
-  InstrItinData<IIC_iLoad_mu , [InstrStage<1, [SW_DIS0], 0>,
-                                InstrStage<1, [SW_DIS1], 0>,
-                                InstrStage<1, [SW_DIS2], 0>,
-                                InstrStage<1, [SW_ALU0, SW_ALU1], 0>,
-                                InstrStage<1, [SW_LS], 3>,
-                                InstrStage<1, [SW_ALU0, SW_ALU1]>],
-                               [2, 1, 1, 1, 3], [], -1>, // dynamic uops
-  //
-  // Load multiple plus branch
-  InstrItinData<IIC_iLoad_mBr, [InstrStage<1, [SW_DIS0], 0>,
-                                InstrStage<1, [SW_DIS1], 0>,
-                                InstrStage<1, [SW_DIS2], 0>,
-                                InstrStage<1, [SW_ALU0, SW_ALU1], 1>,
-                                InstrStage<1, [SW_LS]>],
-                               [1, 1, 1, 1, 3], [], -1>, // dynamic uops
-  //
-  // Pop, def is the 3rd operand.
-  InstrItinData<IIC_iPop  ,    [InstrStage<1, [SW_DIS0], 0>,
-                                InstrStage<1, [SW_DIS1], 0>,
-                                InstrStage<1, [SW_ALU0, SW_ALU1], 1>,
-                                InstrStage<1, [SW_LS]>],
-                               [1, 1, 3], [], -1>, // dynamic uops
-  //
-  // Pop + branch, def is the 3rd operand.
-  InstrItinData<IIC_iPop_Br,   [InstrStage<1, [SW_DIS0], 0>,
-                                InstrStage<1, [SW_DIS1], 0>,
-                                InstrStage<1, [SW_ALU0, SW_ALU1], 1>,
-                                InstrStage<1, [SW_LS]>],
-                               [1, 1, 3], [], -1>, // dynamic uops
-
-  //
-  // iLoadi + iALUr for t2LDRpci_pic.
-  InstrItinData<IIC_iLoadiALU, [InstrStage<1, [SW_DIS0, SW_DIS1, SW_DIS2], 0>,
-                                InstrStage<1, [SW_LS], 3>,
-                                InstrStage<1, [SW_ALU0, SW_ALU1]>],
-                               [4, 1]>,
-
-  // Integer store pipeline
-  ///
-  // Immediate offset
-  InstrItinData<IIC_iStore_i  , [InstrStage<1, [SW_DIS0, SW_DIS1, SW_DIS2], 0>,
-                                 InstrStage<1, [SW_LS]>],
-                                [1, 1]>,
-  InstrItinData<IIC_iStore_bh_i,[InstrStage<1, [SW_DIS0, SW_DIS1, SW_DIS2], 0>,
-                                 InstrStage<1, [SW_LS]>],
-                                [1, 1]>,
-  InstrItinData<IIC_iStore_d_i, [InstrStage<1, [SW_DIS0], 0>,
-                                 InstrStage<1, [SW_DIS1], 0>,
-                                 InstrStage<1, [SW_DIS2], 0>,
-                                 InstrStage<1, [SW_LS], 0>,
-                                 InstrStage<1, [SW_ALU0, SW_ALU1], 1>,
-                                 InstrStage<1, [SW_LS]>],
-                                [1, 1]>,
-  //
-  // Register offset
-  InstrItinData<IIC_iStore_r  , [InstrStage<1, [SW_DIS0, SW_DIS1, SW_DIS2], 0>,
-                                 InstrStage<1, [SW_LS]>],
-                                [1, 1, 1]>,
-  InstrItinData<IIC_iStore_bh_r,[InstrStage<1, [SW_DIS0, SW_DIS1, SW_DIS2], 0>,
-                                 InstrStage<1, [SW_LS]>],
-                                [1, 1, 1]>,
-  InstrItinData<IIC_iStore_d_r, [InstrStage<1, [SW_DIS0], 0>,
-                                 InstrStage<1, [SW_DIS1], 0>,
-                                 InstrStage<1, [SW_DIS2], 0>,
-                                 InstrStage<1, [SW_LS], 0>,
-                                 InstrStage<1, [SW_ALU0, SW_ALU1], 1>,
-                                 InstrStage<1, [SW_LS]>],
-                                [1, 1, 1]>,
-  //
-  // Scaled register offset
-  InstrItinData<IIC_iStore_si ,  [InstrStage<1, [SW_DIS0], 0>,
-                                  InstrStage<1, [SW_DIS1], 0>,
-                                  InstrStage<1, [SW_ALU0, SW_ALU1], 2>,
-                                  InstrStage<1, [SW_LS]>],
-                                 [1, 1, 1]>,
-  InstrItinData<IIC_iStore_bh_si,[InstrStage<1, [SW_DIS0], 0>,
-                                  InstrStage<1, [SW_DIS1], 0>,
-                                  InstrStage<1, [SW_ALU0, SW_ALU1], 2>,
-                                  InstrStage<1, [SW_LS]>],
-                                 [1, 1, 1]>,
-  //
-  // Immediate offset with update
-  InstrItinData<IIC_iStore_iu ,  [InstrStage<1, [SW_DIS0], 0>,
-                                  InstrStage<1, [SW_DIS1], 0>,
-                                  InstrStage<1, [SW_ALU0, SW_ALU1], 1>,
-                                  InstrStage<1, [SW_LS]>],
-                                 [1, 1, 1]>,
-  InstrItinData<IIC_iStore_bh_iu,[InstrStage<1, [SW_DIS0], 0>,
-                                  InstrStage<1, [SW_DIS1], 0>,
-                                  InstrStage<1, [SW_ALU0, SW_ALU1], 1>,
-                                  InstrStage<1, [SW_LS]>],
-                                 [1, 1, 1]>,
-  //
-  // Register offset with update
-  InstrItinData<IIC_iStore_ru ,  [InstrStage<1, [SW_DIS0], 0>,
-                                  InstrStage<1, [SW_DIS1], 0>,
-                                  InstrStage<1, [SW_ALU0, SW_ALU1], 1>,
-                                  InstrStage<1, [SW_LS]>],
-                                 [1, 1, 1, 1]>,
-  InstrItinData<IIC_iStore_bh_ru,[InstrStage<1, [SW_DIS0], 0>,
-                                  InstrStage<1, [SW_DIS1], 0>,
-                                  InstrStage<1, [SW_ALU0, SW_ALU1], 1>,
-                                  InstrStage<1, [SW_LS]>],
-                                 [1, 1, 1, 1]>,
-  InstrItinData<IIC_iStore_d_ru, [InstrStage<1, [SW_DIS0], 0>,
-                                  InstrStage<1, [SW_DIS1], 0>,
-                                  InstrStage<1, [SW_ALU0, SW_ALU1], 1>,
-                                  InstrStage<1, [SW_LS]>],
-                                 [1, 1, 1, 1]>,
-  //
-  // Scaled register offset with update
-  InstrItinData<IIC_iStore_siu,    [InstrStage<1, [SW_DIS0], 0>,
-                                    InstrStage<1, [SW_DIS1], 0>,
-                                    InstrStage<1, [SW_ALU0, SW_ALU1], 2>,
-                                    InstrStage<1, [SW_LS], 0>,
-                                    InstrStage<1, [SW_ALU0, SW_ALU1], 1>],
-                                   [3, 1, 1, 1]>,
-  InstrItinData<IIC_iStore_bh_siu, [InstrStage<1, [SW_DIS0], 0>,
-                                    InstrStage<1, [SW_DIS1], 0>,
-                                    InstrStage<1, [SW_ALU0, SW_ALU1], 2>,
-                                    InstrStage<1, [SW_LS], 0>,
-                                    InstrStage<1, [SW_ALU0, SW_ALU1], 1>],
-                                   [3, 1, 1, 1]>,
-  //
-  // Store multiple
-  InstrItinData<IIC_iStore_m , [InstrStage<1, [SW_DIS0], 0>,
-                                InstrStage<1, [SW_DIS1], 0>,
-                                InstrStage<1, [SW_DIS2], 0>,
-                                InstrStage<1, [SW_ALU0, SW_ALU1], 1>,
-                                InstrStage<1, [SW_LS], 1>,
-                                InstrStage<1, [SW_ALU0, SW_ALU1], 1>,
-                                InstrStage<1, [SW_LS], 1>,
-                                InstrStage<1, [SW_ALU0, SW_ALU1], 1>,
-                                InstrStage<1, [SW_LS]>],
-                                [], [], -1>, // dynamic uops
-  //
-  // Store multiple + update
-  InstrItinData<IIC_iStore_mu, [InstrStage<1, [SW_DIS0], 0>,
-                                InstrStage<1, [SW_DIS1], 0>,
-                                InstrStage<1, [SW_DIS2], 0>,
-                                InstrStage<1, [SW_ALU0, SW_ALU1], 1>,
-                                InstrStage<1, [SW_LS], 1>,
-                                InstrStage<1, [SW_ALU0, SW_ALU1], 1>,
-                                InstrStage<1, [SW_LS], 1>,
-                                InstrStage<1, [SW_ALU0, SW_ALU1], 1>,
-                                InstrStage<1, [SW_LS]>],
-                               [2], [], -1>, // dynamic uops
-
-  //
-  // Preload
-  InstrItinData<IIC_Preload,   [InstrStage<1, [SW_DIS0], 0>], [1, 1]>,
-
-  // Branch
-  //
-  // no delay slots, so the latency of a branch is unimportant
-  InstrItinData<IIC_Br       , [InstrStage<1, [SW_DIS0], 0>]>,
-
-  // FP Special Register to Integer Register File Move
-  InstrItinData<IIC_fpSTAT , [InstrStage<1, [SW_DIS0, SW_DIS1, SW_DIS2], 0>,
-                              InstrStage<1, [SW_ALU0, SW_ALU1]>],
-                             [1]>,
-  //
-  // Single-precision FP Unary
-  //
-  // Most floating-point moves get issued on ALU0.
-  InstrItinData<IIC_fpUNA32 , [InstrStage<1, [SW_DIS0, SW_DIS1, SW_DIS2], 0>,
-                               InstrStage<1, [SW_ALU0]>],
-                              [2, 1]>,
-  //
-  // Double-precision FP Unary
-  InstrItinData<IIC_fpUNA64 , [InstrStage<1, [SW_DIS0, SW_DIS1, SW_DIS2], 0>,
-                               InstrStage<1, [SW_ALU0]>],
-                              [2, 1]>,
-
-  //
-  // Single-precision FP Compare
-  InstrItinData<IIC_fpCMP32 , [InstrStage<1, [SW_DIS0, SW_DIS1, SW_DIS2], 0>,
-                               InstrStage<1, [SW_ALU0]>],
-                              [1, 1]>,
-  //
-  // Double-precision FP Compare
-  InstrItinData<IIC_fpCMP64 , [InstrStage<1, [SW_DIS0, SW_DIS1, SW_DIS2], 0>,
-                               InstrStage<1, [SW_ALU0]>],
-                              [1, 1]>,
-  //
-  // Single to Double FP Convert
-  InstrItinData<IIC_fpCVTSD , [InstrStage<1, [SW_DIS0, SW_DIS1, SW_DIS2], 0>,
-                               InstrStage<1, [SW_ALU1]>],
-                              [4, 1]>,
-  //
-  // Double to Single FP Convert
-  InstrItinData<IIC_fpCVTDS , [InstrStage<1, [SW_DIS0, SW_DIS1, SW_DIS2], 0>,
-                               InstrStage<1, [SW_ALU1]>],
-                              [4, 1]>,
-
-  //
-  // Single to Half FP Convert
-  InstrItinData<IIC_fpCVTSH , [InstrStage<1, [SW_DIS0], 0>,
-                               InstrStage<1, [SW_DIS1], 0>,
-                               InstrStage<1, [SW_ALU1], 4>,
-                               InstrStage<1, [SW_ALU1]>],
-                              [6, 1]>,
-  //
-  // Half to Single FP Convert
-  InstrItinData<IIC_fpCVTHS , [InstrStage<1, [SW_DIS0, SW_DIS1, SW_DIS2], 0>,
-                               InstrStage<1, [SW_ALU1]>],
-                              [4, 1]>,
-
-  //
-  // Single-Precision FP to Integer Convert
-  InstrItinData<IIC_fpCVTSI , [InstrStage<1, [SW_DIS0, SW_DIS1, SW_DIS2], 0>,
-                               InstrStage<1, [SW_ALU1]>],
-                              [4, 1]>,
-  //
-  // Double-Precision FP to Integer Convert
-  InstrItinData<IIC_fpCVTDI , [InstrStage<1, [SW_DIS0, SW_DIS1, SW_DIS2], 0>,
-                               InstrStage<1, [SW_ALU1]>],
-                              [4, 1]>,
-  //
-  // Integer to Single-Precision FP Convert
-  InstrItinData<IIC_fpCVTIS , [InstrStage<1, [SW_DIS0, SW_DIS1, SW_DIS2], 0>,
-                               InstrStage<1, [SW_ALU1]>],
-                              [4, 1]>,
-  //
-  // Integer to Double-Precision FP Convert
-  InstrItinData<IIC_fpCVTID , [InstrStage<1, [SW_DIS0, SW_DIS1, SW_DIS2], 0>,
-                               InstrStage<1, [SW_ALU1]>],
-                              [4, 1]>,
-  //
-  // Single-precision FP ALU
-  InstrItinData<IIC_fpALU32 , [InstrStage<1, [SW_DIS0, SW_DIS1, SW_DIS2], 0>,
-                               InstrStage<1, [SW_ALU0]>],
-                              [2, 1, 1]>,
-  //
-  // Double-precision FP ALU
-  InstrItinData<IIC_fpALU64 , [InstrStage<1, [SW_DIS0, SW_DIS1, SW_DIS2], 0>,
-                               InstrStage<1, [SW_ALU0]>],
-                              [2, 1, 1]>,
-  //
-  // Single-precision FP Multiply
-  InstrItinData<IIC_fpMUL32 , [InstrStage<1, [SW_DIS0, SW_DIS1, SW_DIS2], 0>,
-                               InstrStage<1, [SW_ALU1]>],
-                              [4, 1, 1]>,
-  //
-  // Double-precision FP Multiply
-  InstrItinData<IIC_fpMUL64 , [InstrStage<1, [SW_DIS0, SW_DIS1, SW_DIS2], 0>,
-                               InstrStage<1, [SW_ALU1]>],
-                              [6, 1, 1]>,
-  //
-  // Single-precision FP MAC
-  InstrItinData<IIC_fpMAC32 , [InstrStage<1, [SW_DIS0, SW_DIS1, SW_DIS2], 0>,
-                               InstrStage<1, [SW_ALU1]>],
-                              [8, 1, 1]>,
-  //
-  // Double-precision FP MAC
-  InstrItinData<IIC_fpMAC64 , [InstrStage<1, [SW_DIS0, SW_DIS1, SW_DIS2], 0>,
-                               InstrStage<1, [SW_ALU1]>],
-                              [12, 1, 1]>,
-  //
-  // Single-precision Fused FP MAC
-  InstrItinData<IIC_fpFMAC32, [InstrStage<1, [SW_DIS0, SW_DIS1, SW_DIS2], 0>,
-                               InstrStage<1, [SW_ALU1]>],
-                              [8, 1, 1]>,
-  //
-  // Double-precision Fused FP MAC
-  InstrItinData<IIC_fpFMAC64, [InstrStage<1, [SW_DIS0, SW_DIS1, SW_DIS2], 0>,
-                               InstrStage<1, [SW_ALU1]>],
-                              [12, 1, 1]>,
-  //
-  // Single-precision FP DIV
-  InstrItinData<IIC_fpDIV32 , [InstrStage<1, [SW_DIS0, SW_DIS1, SW_DIS2], 0>,
-                               InstrStage<1, [SW_ALU1], 0>,
-                               InstrStage<15, [SW_FDIV]>],
-                              [17, 1, 1]>,
-  //
-  // Double-precision FP DIV
-  InstrItinData<IIC_fpDIV64 , [InstrStage<1, [SW_DIS0, SW_DIS1, SW_DIS2], 0>,
-                               InstrStage<1, [SW_ALU1], 0>,
-                               InstrStage<30, [SW_FDIV]>],
-                              [32, 1, 1]>,
-  //
-  // Single-precision FP SQRT
-  InstrItinData<IIC_fpSQRT32, [InstrStage<1, [SW_DIS0, SW_DIS1, SW_DIS2], 0>,
-                               InstrStage<1, [SW_ALU1], 0>,
-                               InstrStage<15, [SW_FDIV]>],
-                              [17, 1]>,
-  //
-  // Double-precision FP SQRT
-  InstrItinData<IIC_fpSQRT64, [InstrStage<1, [SW_DIS0, SW_DIS1, SW_DIS2], 0>,
-                               InstrStage<1, [SW_ALU1], 0>,
-                               InstrStage<30, [SW_FDIV]>],
-                              [32, 1, 1]>,
-
-  //
-  // Integer to Single-precision Move
-  InstrItinData<IIC_fpMOVIS,  [InstrStage<1, [SW_DIS0], 0>,
-                               InstrStage<1, [SW_DIS1], 0>,
-                               InstrStage<1, [SW_LS], 4>,
-                               InstrStage<1, [SW_ALU0]>],
-                              [6, 1]>,
-  //
-  // Integer to Double-precision Move
-  InstrItinData<IIC_fpMOVID,  [InstrStage<1, [SW_DIS0, SW_DIS1, SW_DIS2], 0>,
-                               InstrStage<1, [SW_LS]>],
-                              [4, 1]>,
-  //
-  // Single-precision to Integer Move
-  InstrItinData<IIC_fpMOVSI,  [InstrStage<1, [SW_DIS0, SW_DIS1, SW_DIS2], 0>,
-                               InstrStage<1, [SW_LS]>],
-                              [3, 1]>,
-  //
-  // Double-precision to Integer Move
-  InstrItinData<IIC_fpMOVDI,  [InstrStage<1, [SW_DIS0], 0>,
-                               InstrStage<1, [SW_DIS1], 0>,
-                               InstrStage<1, [SW_LS], 3>,
-                               InstrStage<1, [SW_LS]>],
-                              [3, 4, 1]>,
-  //
-  // Single-precision FP Load
-  InstrItinData<IIC_fpLoad32, [InstrStage<1, [SW_DIS0, SW_DIS1, SW_DIS2], 0>,
-                               InstrStage<1, [SW_LS]>],
-                              [4, 1]>,
-  //
-  // Double-precision FP Load
-  InstrItinData<IIC_fpLoad64, [InstrStage<1, [SW_DIS0, SW_DIS1, SW_DIS2], 0>,
-                               InstrStage<1, [SW_LS]>],
-                              [4, 1]>,
-  //
-  // FP Load Multiple
-  // FIXME: Assumes a single Q register.
-  InstrItinData<IIC_fpLoad_m, [InstrStage<1, [SW_DIS0, SW_DIS1, SW_DIS2], 0>,
-                               InstrStage<1, [SW_LS]>],
-                              [1, 1, 1, 4], [], -1>, // dynamic uops
-  //
-  // FP Load Multiple + update
-  // FIXME: Assumes a single Q register.
-  InstrItinData<IIC_fpLoad_mu,[InstrStage<1, [SW_DIS0], 0>,
-                               InstrStage<1, [SW_DIS1], 0>,
-                               InstrStage<1, [SW_LS], 4>,
-                               InstrStage<1, [SW_ALU0, SW_ALU1]>],
-                              [2, 1, 1, 1, 4], [], -1>, // dynamic uops
-  //
-  // Single-precision FP Store
-  InstrItinData<IIC_fpStore32,[InstrStage<1, [SW_DIS0, SW_DIS1, SW_DIS2], 0>,
-                               InstrStage<1, [SW_LS]>],
-                              [1, 1]>,
-  //
-  // Double-precision FP Store
-  InstrItinData<IIC_fpStore64,[InstrStage<1, [SW_DIS0, SW_DIS1, SW_DIS2], 0>,
-                               InstrStage<1, [SW_LS]>],
-                              [1, 1]>,
-  //
-  // FP Store Multiple
-  // FIXME: Assumes a single Q register.
-  InstrItinData<IIC_fpStore_m,[InstrStage<1, [SW_DIS0, SW_DIS1, SW_DIS2], 0>,
-                               InstrStage<1, [SW_LS]>],
-                              [1, 1, 1], [], -1>, // dynamic uops
-  //
-  // FP Store Multiple + update
-  // FIXME: Assumes a single Q register.
-  InstrItinData<IIC_fpStore_mu,[InstrStage<1, [SW_DIS0], 0>,
-                                InstrStage<1, [SW_DIS1], 0>,
-                                InstrStage<1, [SW_LS], 4>,
-                                InstrStage<1, [SW_ALU0, SW_ALU1]>],
-                               [2, 1, 1, 1], [], -1>, // dynamic uops
-  // NEON
-  //
-  // Double-register Integer Unary
-  InstrItinData<IIC_VUNAiD,   [InstrStage<1, [SW_DIS0, SW_DIS1, SW_DIS2], 0>,
-                               InstrStage<1, [SW_ALU0]>],
-                              [4, 1]>,
-  //
-  // Quad-register Integer Unary
-  InstrItinData<IIC_VUNAiQ,   [InstrStage<1, [SW_DIS0, SW_DIS1, SW_DIS2], 0>,
-                               InstrStage<1, [SW_ALU0]>],
-                              [4, 1]>,
-  //
-  // Double-register Integer Q-Unary
-  InstrItinData<IIC_VQUNAiD,  [InstrStage<1, [SW_DIS0, SW_DIS1, SW_DIS2], 0>,
-                               InstrStage<1, [SW_ALU0]>],
-                              [4, 1]>,
-  //
-  // Quad-register Integer CountQ-Unary
-  InstrItinData<IIC_VQUNAiQ,  [InstrStage<1, [SW_DIS0, SW_DIS1, SW_DIS2], 0>,
-                               InstrStage<1, [SW_ALU0]>],
-                              [4, 1]>,
-  //
-  // Double-register Integer Binary
-  InstrItinData<IIC_VBINiD,   [InstrStage<1, [SW_DIS0, SW_DIS1, SW_DIS2], 0>,
-                               InstrStage<1, [SW_ALU0]>],
-                              [2, 1, 1]>,
-  //
-  // Quad-register Integer Binary
-  InstrItinData<IIC_VBINiQ,   [InstrStage<1, [SW_DIS0, SW_DIS1, SW_DIS2], 0>,
-                               InstrStage<1, [SW_ALU0]>],
-                              [2, 1, 1]>,
-  //
-  // Double-register Integer Subtract
-  InstrItinData<IIC_VSUBiD,   [InstrStage<1, [SW_DIS0, SW_DIS1, SW_DIS2], 0>,
-                               InstrStage<1, [SW_ALU0]>],
-                              [2, 1, 1]>,
-  //
-  // Quad-register Integer Subtract
-  InstrItinData<IIC_VSUBiQ,   [InstrStage<1, [SW_DIS0, SW_DIS1, SW_DIS2], 0>,
-                               InstrStage<1, [SW_ALU0]>],
-                              [2, 1, 1]>,
-  //
-  // Double-register Integer Shift
-  InstrItinData<IIC_VSHLiD,   [InstrStage<1, [SW_DIS0, SW_DIS1, SW_DIS2], 0>,
-                               InstrStage<1, [SW_ALU0]>],
-                              [2, 1, 1]>,
-  //
-  // Quad-register Integer Shift
-  InstrItinData<IIC_VSHLiQ,   [InstrStage<1, [SW_DIS0, SW_DIS1, SW_DIS2], 0>,
-                               InstrStage<1, [SW_ALU0]>],
-                              [2, 1, 1]>,
-  //
-  // Double-register Integer Shift (4 cycle)
-  InstrItinData<IIC_VSHLi4D,  [InstrStage<1, [SW_DIS0, SW_DIS1, SW_DIS2], 0>,
-                               InstrStage<1, [SW_ALU0]>],
-                              [4, 1, 1]>,
-  //
-  // Quad-register Integer Shift (4 cycle)
-  InstrItinData<IIC_VSHLi4Q,  [InstrStage<1, [SW_DIS0, SW_DIS1, SW_DIS2], 0>,
-                               InstrStage<1, [SW_ALU0]>],
-                              [4, 1, 1]>,
-  //
-  // Double-register Integer Binary (4 cycle)
-  InstrItinData<IIC_VBINi4D,  [InstrStage<1, [SW_DIS0, SW_DIS1, SW_DIS2], 0>,
-                               InstrStage<1, [SW_ALU0]>],
-                              [4, 1, 1]>,
-  //
-  // Quad-register Integer Binary (4 cycle)
-  InstrItinData<IIC_VBINi4Q,  [InstrStage<1, [SW_DIS0, SW_DIS1, SW_DIS2], 0>,
-                               InstrStage<1, [SW_ALU0]>],
-                              [4, 1, 1]>,
-  //
-  // Double-register Integer Subtract (4 cycle)
-  InstrItinData<IIC_VSUBi4D,  [InstrStage<1, [SW_DIS0, SW_DIS1, SW_DIS2], 0>,
-                               InstrStage<1, [SW_ALU0]>],
-                              [4, 1, 1]>,
-  //
-  // Quad-register Integer Subtract (4 cycle)
-  InstrItinData<IIC_VSUBi4Q,  [InstrStage<1, [SW_DIS0, SW_DIS1, SW_DIS2], 0>,
-                               InstrStage<1, [SW_ALU0]>],
-                              [4, 1, 1]>,
-
-  //
-  // Double-register Integer Count
-  InstrItinData<IIC_VCNTiD,   [InstrStage<1, [SW_DIS0, SW_DIS1, SW_DIS2], 0>,
-                               InstrStage<1, [SW_ALU0]>],
-                              [2, 1, 1]>,
-  //
-  // Quad-register Integer Count
-  InstrItinData<IIC_VCNTiQ,   [InstrStage<1, [SW_DIS0, SW_DIS1, SW_DIS2], 0>,
-                               InstrStage<1, [SW_ALU0]>],
-                              [2, 1, 1]>,
-  //
-  // Double-register Absolute Difference and Accumulate
-  InstrItinData<IIC_VABAD,    [InstrStage<1, [SW_DIS0, SW_DIS1, SW_DIS2], 0>,
-                               InstrStage<1, [SW_ALU0]>],
-                              [4, 1, 1, 1]>,
-  //
-  // Quad-register Absolute Difference and Accumulate
-  InstrItinData<IIC_VABAQ,    [InstrStage<1, [SW_DIS0, SW_DIS1, SW_DIS2], 0>,
-                               InstrStage<1, [SW_ALU0]>],
-                              [4, 1, 1, 1]>,
-  //
-  // Double-register Integer Pair Add Long
-  InstrItinData<IIC_VPALiD,   [InstrStage<1, [SW_DIS0, SW_DIS1, SW_DIS2], 0>,
-                               InstrStage<1, [SW_ALU0]>],
-                              [4, 1, 1]>,
-  //
-  // Quad-register Integer Pair Add Long
-  InstrItinData<IIC_VPALiQ,   [InstrStage<1, [SW_DIS0, SW_DIS1, SW_DIS2], 0>,
-                               InstrStage<1, [SW_ALU0]>],
-                              [4, 1, 1]>,
-
-  //
-  // Double-register Integer Multiply (.8, .16)
-  InstrItinData<IIC_VMULi16D, [InstrStage<1, [SW_DIS0, SW_DIS1, SW_DIS2], 0>,
-                               InstrStage<1, [SW_ALU1]>],
-                              [4, 1, 1]>,
-  //
-  // Quad-register Integer Multiply (.8, .16)
-  InstrItinData<IIC_VMULi16Q, [InstrStage<1, [SW_DIS0, SW_DIS1, SW_DIS2], 0>,
-                               InstrStage<1, [SW_ALU1]>],
-                              [4, 1, 1]>,
-
-  //
-  // Double-register Integer Multiply (.32)
-  InstrItinData<IIC_VMULi32D, [InstrStage<1, [SW_DIS0, SW_DIS1, SW_DIS2], 0>,
-                               InstrStage<1, [SW_ALU1]>],
-                              [4, 1, 1]>,
-  //
-  // Quad-register Integer Multiply (.32)
-  InstrItinData<IIC_VMULi32Q, [InstrStage<1, [SW_DIS0, SW_DIS1, SW_DIS2], 0>,
-                               InstrStage<1, [SW_ALU1]>],
-                              [4, 1, 1]>,
-  //
-  // Double-register Integer Multiply-Accumulate (.8, .16)
-  InstrItinData<IIC_VMACi16D, [InstrStage<1, [SW_DIS0, SW_DIS1, SW_DIS2], 0>,
-                               InstrStage<1, [SW_ALU1]>],
-                              [4, 1, 1, 1]>,
-  //
-  // Double-register Integer Multiply-Accumulate (.32)
-  InstrItinData<IIC_VMACi32D, [InstrStage<1, [SW_DIS0, SW_DIS1, SW_DIS2], 0>,
-                               InstrStage<1, [SW_ALU1]>],
-                              [4, 1, 1, 1]>,
-  //
-  // Quad-register Integer Multiply-Accumulate (.8, .16)
-  InstrItinData<IIC_VMACi16Q, [InstrStage<1, [SW_DIS0, SW_DIS1, SW_DIS2], 0>,
-                               InstrStage<1, [SW_ALU1]>],
-                              [4, 1, 1, 1]>,
-  //
-  // Quad-register Integer Multiply-Accumulate (.32)
-  InstrItinData<IIC_VMACi32Q, [InstrStage<1, [SW_DIS0, SW_DIS1, SW_DIS2], 0>,
-                               InstrStage<1, [SW_ALU1]>],
-                              [4, 1, 1, 1]>,
-
-  //
-  // Move
-  InstrItinData<IIC_VMOV,     [InstrStage<1, [SW_DIS0, SW_DIS1, SW_DIS2], 0>,
-                               InstrStage<1, [SW_ALU0]>],
-                              [2, 1]>,
-  //
-  // Move Immediate
-  InstrItinData<IIC_VMOVImm,  [InstrStage<1, [SW_DIS0, SW_DIS1, SW_DIS2], 0>,
-                               InstrStage<1, [SW_ALU0]>],
-                              [2]>,
-  //
-  // Double-register Permute Move
-  InstrItinData<IIC_VMOVD,    [InstrStage<1, [SW_DIS0, SW_DIS1, SW_DIS2], 0>,
-                               InstrStage<1, [SW_ALU1]>],
-                              [2, 1]>,
-  //
-  // Quad-register Permute Move
-  InstrItinData<IIC_VMOVQ,    [InstrStage<1, [SW_DIS0, SW_DIS1, SW_DIS2], 0>,
-                               InstrStage<1, [SW_ALU1]>],
-                              [2, 1]>,
-  //
-  // Integer to Single-precision Move
-  InstrItinData<IIC_VMOVIS ,  [InstrStage<1, [SW_DIS0], 0>,
-                               InstrStage<1, [SW_DIS1], 0>,
-                               InstrStage<1, [SW_LS], 4>,
-                               InstrStage<1, [SW_ALU0]>],
-                              [6, 1]>,
-  //
-  // Integer to Double-precision Move
-  InstrItinData<IIC_VMOVID ,  [InstrStage<1, [SW_DIS0, SW_DIS1, SW_DIS2], 0>,
-                               InstrStage<1, [SW_LS]>],
-                              [4, 1, 1]>,
-  //
-  // Single-precision to Integer Move
-  InstrItinData<IIC_VMOVSI ,  [InstrStage<1, [SW_DIS0, SW_DIS1, SW_DIS2], 0>,
-                               InstrStage<1, [SW_LS]>],
-                              [3, 1]>,
-  //
-  // Double-precision to Integer Move
-  InstrItinData<IIC_VMOVDI ,  [InstrStage<1, [SW_DIS0], 0>,
-                               InstrStage<1, [SW_DIS1], 0>,
-                               InstrStage<1, [SW_LS], 3>,
-                               InstrStage<1, [SW_LS]>],
-                              [3, 4, 1]>,
-  //
-  // Integer to Lane Move
-  // FIXME: I think this is correct, but it is not clear from the tuning guide.
-  InstrItinData<IIC_VMOVISL , [InstrStage<1, [SW_DIS0], 0>,
-                               InstrStage<1, [SW_DIS1], 0>,
-                               InstrStage<1, [SW_LS], 4>,
-                               InstrStage<1, [SW_ALU0]>],
-                              [6, 1]>,
-
-  //
-  // Vector narrow move
-  InstrItinData<IIC_VMOVN,    [InstrStage<1, [SW_DIS0, SW_DIS1, SW_DIS2], 0>,
-                               InstrStage<1, [SW_ALU1]>],
-                              [2, 1]>,
-  //
-  // Double-register FP Unary
-  // FIXME: VRECPE / VRSQRTE has a longer latency than VABS, which is used here,
-  //        and they issue on a different pipeline.
-  InstrItinData<IIC_VUNAD,    [InstrStage<1, [SW_DIS0, SW_DIS1, SW_DIS2], 0>,
-                               InstrStage<1, [SW_ALU0]>],
-                              [2, 1]>,
-  //
-  // Quad-register FP Unary
-  // FIXME: VRECPE / VRSQRTE has a longer latency than VABS, which is used here,
-  //        and they issue on a different pipeline.
-  InstrItinData<IIC_VUNAQ,    [InstrStage<1, [SW_DIS0, SW_DIS1, SW_DIS2], 0>,
-                               InstrStage<1, [SW_ALU0]>],
-                              [2, 1]>,
-  //
-  // Double-register FP Binary
-  // FIXME: We're using this itin for many instructions.
-  InstrItinData<IIC_VBIND,    [InstrStage<1, [SW_DIS0, SW_DIS1, SW_DIS2], 0>,
-                               InstrStage<1, [SW_ALU0]>],
-                              [4, 1, 1]>,
-
-  //
-  // VPADD, etc.
-  InstrItinData<IIC_VPBIND,   [InstrStage<1, [SW_DIS0, SW_DIS1, SW_DIS2], 0>,
-                               InstrStage<1, [SW_ALU0]>],
-                              [4, 1, 1]>,
-  //
-  // Double-register FP VMUL
-  InstrItinData<IIC_VFMULD,   [InstrStage<1, [SW_DIS0, SW_DIS1, SW_DIS2], 0>,
-                               InstrStage<1, [SW_ALU1]>],
-                              [4, 1, 1]>,
-  //
-  // Quad-register FP Binary
-  InstrItinData<IIC_VBINQ,    [InstrStage<1, [SW_DIS0, SW_DIS1, SW_DIS2], 0>,
-                               InstrStage<1, [SW_ALU0]>],
-                              [4, 1, 1]>,
-  //
-  // Quad-register FP VMUL
-  InstrItinData<IIC_VFMULQ,   [InstrStage<1, [SW_DIS0, SW_DIS1, SW_DIS2], 0>,
-                               InstrStage<1, [SW_ALU1]>],
-                              [4, 1, 1]>,
-  //
-  // Double-register FP Multiple-Accumulate
-  InstrItinData<IIC_VMACD,    [InstrStage<1, [SW_DIS0, SW_DIS1, SW_DIS2], 0>,
-                               InstrStage<1, [SW_ALU1]>],
-                              [8, 1, 1]>,
-  //
-  // Quad-register FP Multiple-Accumulate
-  InstrItinData<IIC_VMACQ,    [InstrStage<1, [SW_DIS0, SW_DIS1, SW_DIS2], 0>,
-                               InstrStage<1, [SW_ALU1]>],
-                              [8, 1, 1]>,
-  //
-  // Double-register Fused FP Multiple-Accumulate
-  InstrItinData<IIC_VFMACD,   [InstrStage<1, [SW_DIS0, SW_DIS1, SW_DIS2], 0>,
-                               InstrStage<1, [SW_ALU1]>],
-                              [8, 1, 1]>,
-  //
-  // Quad-register FusedF P Multiple-Accumulate
-  InstrItinData<IIC_VFMACQ,   [InstrStage<1, [SW_DIS0, SW_DIS1, SW_DIS2], 0>,
-                               InstrStage<1, [SW_ALU1]>],
-                              [8, 1, 1]>,
-  //
-  // Double-register Reciprical Step
-  InstrItinData<IIC_VRECSD,   [InstrStage<1, [SW_DIS0, SW_DIS1, SW_DIS2], 0>,
-                               InstrStage<1, [SW_ALU1]>],
-                              [8, 1, 1]>,
-  //
-  // Quad-register Reciprical Step
-  InstrItinData<IIC_VRECSQ,   [InstrStage<1, [SW_DIS0, SW_DIS1, SW_DIS2], 0>,
-                               InstrStage<1, [SW_ALU1]>],
-                              [8, 1, 1]>,
-  //
-  // Double-register Permute
-  // FIXME: The latencies are unclear from the documentation.
-  InstrItinData<IIC_VPERMD,   [InstrStage<1, [SW_DIS0], 0>,
-                               InstrStage<1, [SW_DIS1], 0>,
-                               InstrStage<1, [SW_DIS2], 0>,
-                               InstrStage<1, [SW_ALU1], 2>,
-                               InstrStage<1, [SW_ALU1], 2>,
-                               InstrStage<1, [SW_ALU1]>],
-                              [3, 4, 3, 4]>,
-  //
-  // Quad-register Permute
-  // FIXME: The latencies are unclear from the documentation.
-  InstrItinData<IIC_VPERMQ,   [InstrStage<1, [SW_DIS0], 0>,
-                               InstrStage<1, [SW_DIS1], 0>,
-                               InstrStage<1, [SW_DIS2], 0>,
-                               InstrStage<1, [SW_ALU1], 2>,
-                               InstrStage<1, [SW_ALU1], 2>,
-                               InstrStage<1, [SW_ALU1]>],
-                              [3, 4, 3, 4]>,
-  //
-  // Quad-register Permute (3 cycle issue on A9)
-  InstrItinData<IIC_VPERMQ3,  [InstrStage<1, [SW_DIS0], 0>,
-                               InstrStage<1, [SW_DIS1], 0>,
-                               InstrStage<1, [SW_DIS2], 0>,
-                               InstrStage<1, [SW_ALU1], 2>,
-                               InstrStage<1, [SW_ALU1], 2>,
-                               InstrStage<1, [SW_ALU1]>],
-                              [3, 4, 3, 4]>,
-
-  //
-  // Double-register VEXT
-  InstrItinData<IIC_VEXTD,    [InstrStage<1, [SW_DIS0, SW_DIS1, SW_DIS2], 0>,
-                               InstrStage<1, [SW_ALU1]>],
-                              [2, 1, 1]>,
-  //
-  // Quad-register VEXT
-  InstrItinData<IIC_VEXTQ,    [InstrStage<1, [SW_DIS0, SW_DIS1, SW_DIS2], 0>,
-                               InstrStage<1, [SW_ALU1]>],
-                              [2, 1, 1]>,
-  //
-  // VTB
-  InstrItinData<IIC_VTB1,     [InstrStage<1, [SW_DIS0, SW_DIS1, SW_DIS2], 0>,
-                               InstrStage<1, [SW_ALU1]>],
-                              [2, 1, 1]>,
-  InstrItinData<IIC_VTB2,     [InstrStage<1, [SW_DIS0], 0>,
-                               InstrStage<1, [SW_DIS1], 0>,
-                               InstrStage<1, [SW_ALU1], 2>,
-                               InstrStage<1, [SW_ALU1]>],
-                              [4, 1, 3, 3]>,
-  InstrItinData<IIC_VTB3,     [InstrStage<1, [SW_DIS0], 0>,
-                               InstrStage<1, [SW_DIS1], 0>,
-                               InstrStage<1, [SW_DIS2], 0>,
-                               InstrStage<1, [SW_ALU1], 2>,
-                               InstrStage<1, [SW_ALU1], 2>,
-                               InstrStage<1, [SW_ALU1]>],
-                              [6, 1, 3, 5, 5]>,
-  InstrItinData<IIC_VTB4,     [InstrStage<1, [SW_DIS0], 0>,
-                               InstrStage<1, [SW_DIS1], 0>,
-                               InstrStage<1, [SW_DIS2], 0>,
-                               InstrStage<1, [SW_ALU1], 2>,
-                               InstrStage<1, [SW_ALU1], 2>,
-                               InstrStage<1, [SW_ALU1], 2>,
-                               InstrStage<1, [SW_ALU1]>],
-                              [8, 1, 3, 5, 7, 7]>,
-  //
-  // VTBX
-  InstrItinData<IIC_VTBX1,    [InstrStage<1, [SW_DIS0, SW_DIS1, SW_DIS2], 0>,
-                               InstrStage<1, [SW_ALU1]>],
-                              [2, 1, 1]>,
-  InstrItinData<IIC_VTBX2,    [InstrStage<1, [SW_DIS0], 0>,
-                               InstrStage<1, [SW_DIS1], 0>,
-                               InstrStage<1, [SW_ALU1], 2>,
-                               InstrStage<1, [SW_ALU1]>],
-                              [4, 1, 3, 3]>,
-  InstrItinData<IIC_VTBX3,    [InstrStage<1, [SW_DIS0], 0>,
-                               InstrStage<1, [SW_DIS1], 0>,
-                               InstrStage<1, [SW_DIS2], 0>,
-                               InstrStage<1, [SW_ALU1], 2>,
-                               InstrStage<1, [SW_ALU1], 2>,
-                               InstrStage<1, [SW_ALU1]>],
-                              [6, 1, 3, 5, 5]>,
-  InstrItinData<IIC_VTBX4,    [InstrStage<1, [SW_DIS0], 0>,
-                               InstrStage<1, [SW_DIS1], 0>,
-                               InstrStage<1, [SW_DIS2], 0>,
-                               InstrStage<1, [SW_ALU1], 2>,
-                               InstrStage<1, [SW_ALU1], 2>,
-                               InstrStage<1, [SW_ALU1], 2>,
-                               InstrStage<1, [SW_ALU1]>],
-                              [8, 1, 3, 5, 7, 7]>
-]>;
-
-// ===---------------------------------------------------------------------===//
-// This following definitions describe the simple machine model which
-// will replace itineraries.
-
 // Swift machine model for scheduling and other instruction cost heuristics.
 def SwiftModel : SchedMachineModel {
   let IssueWidth = 3; // 3 micro-ops are dispatched per cycle.
   let MicroOpBufferSize = 45; // Based on NEON renamed registers.
   let LoadLatency = 3;
   let MispredictPenalty = 14; // A branch direction mispredict.
-
-  let Itineraries = SwiftItineraries;
 }
 
 // Swift predicates.
@@ -1558,6 +520,13 @@ let SchedModel = SwiftModel in {
         (instregex "STM(IB|IA|DB|DA)_UPD", "(t2|sys|t)STM(IB|IA|DB|DA)_UPD",
         "PUSH", "tPUSH")>;
 
+  // LDRLIT pseudo instructions, they expand to LDR + PICADD
+  def : InstRW<[SwiftWriteP2ThreeCycle, WriteALU],
+        (instregex "t?LDRLIT_ga_abs", "t?LDRLIT_ga_pcrel")>;
+  // LDRLIT_ga_pcrel_ldr expands to LDR + PICLDR
+  def : InstRW<[SwiftWriteP2ThreeCycle, SwiftWriteP2ThreeCycle],
+        (instregex "LDRLIT_ga_pcrel_ldr")>;
+
   // 4.2.26 Branch
   def : WriteRes<WriteBr, [SwiftUnitP1]> { let Latency = 0; }
   def : WriteRes<WriteBrL, [SwiftUnitP1]> { let Latency = 2; }
diff --git a/lib/Target/ARM/ARMSelectionDAGInfo.cpp b/lib/Target/ARM/ARMSelectionDAGInfo.cpp
index 636205fc9e3..6cafbbb9f8e 100644
--- a/lib/Target/ARM/ARMSelectionDAGInfo.cpp
+++ b/lib/Target/ARM/ARMSelectionDAGInfo.cpp
@@ -18,10 +18,114 @@ using namespace llvm;
 
 #define DEBUG_TYPE "arm-selectiondag-info"
 
-ARMSelectionDAGInfo::ARMSelectionDAGInfo(const DataLayout &DL)
-    : TargetSelectionDAGInfo(&DL) {}
+// Emit, if possible, a specialized version of the given Libcall. Typically this
+// means selecting the appropriately aligned version, but we also convert memset
+// of 0 into memclr.
+SDValue ARMSelectionDAGInfo::
+EmitSpecializedLibcall(SelectionDAG &DAG, SDLoc dl,
+                       SDValue Chain,
+                       SDValue Dst, SDValue Src,
+                       SDValue Size, unsigned Align,
+                       RTLIB::Libcall LC) const {
+  const ARMSubtarget &Subtarget =
+      DAG.getMachineFunction().getSubtarget<ARMSubtarget>();
+  const ARMTargetLowering *TLI = Subtarget.getTargetLowering();
+
+  // Only use a specialized AEABI function if the default version of this
+  // Libcall is an AEABI function.
+  if (std::strncmp(TLI->getLibcallName(LC), "__aeabi", 7) != 0)
+    return SDValue();
+
+  // Translate RTLIB::Libcall to AEABILibcall. We only do this in order to be
+  // able to translate memset to memclr and use the value to index the function
+  // name array.
+  enum {
+    AEABI_MEMCPY = 0,
+    AEABI_MEMMOVE,
+    AEABI_MEMSET,
+    AEABI_MEMCLR
+  } AEABILibcall;
+  switch (LC) {
+  case RTLIB::MEMCPY:
+    AEABILibcall = AEABI_MEMCPY;
+    break;
+  case RTLIB::MEMMOVE:
+    AEABILibcall = AEABI_MEMMOVE;
+    break;
+  case RTLIB::MEMSET: 
+    AEABILibcall = AEABI_MEMSET;
+    if (ConstantSDNode *ConstantSrc = dyn_cast<ConstantSDNode>(Src))
+      if (ConstantSrc->getZExtValue() == 0)
+        AEABILibcall = AEABI_MEMCLR;
+    break;
+  default:
+    return SDValue();
+  }
+
+  // Choose the most-aligned libcall variant that we can
+  enum {
+    ALIGN1 = 0,
+    ALIGN4,
+    ALIGN8
+  } AlignVariant;
+  if ((Align & 7) == 0)
+    AlignVariant = ALIGN8;
+  else if ((Align & 3) == 0)
+    AlignVariant = ALIGN4;
+  else
+    AlignVariant = ALIGN1;
 
-ARMSelectionDAGInfo::~ARMSelectionDAGInfo() {
+  TargetLowering::ArgListTy Args;
+  TargetLowering::ArgListEntry Entry;
+  Entry.Ty = DAG.getDataLayout().getIntPtrType(*DAG.getContext());
+  Entry.Node = Dst;
+  Args.push_back(Entry);
+  if (AEABILibcall == AEABI_MEMCLR) {
+    Entry.Node = Size;
+    Args.push_back(Entry);
+  } else if (AEABILibcall == AEABI_MEMSET) {
+    // Adjust parameters for memset, EABI uses format (ptr, size, value),
+    // GNU library uses (ptr, value, size)
+    // See RTABI section 4.3.4
+    Entry.Node = Size;
+    Args.push_back(Entry);
+
+    // Extend or truncate the argument to be an i32 value for the call.
+    if (Src.getValueType().bitsGT(MVT::i32))
+      Src = DAG.getNode(ISD::TRUNCATE, dl, MVT::i32, Src);
+    else if (Src.getValueType().bitsLT(MVT::i32))
+      Src = DAG.getNode(ISD::ZERO_EXTEND, dl, MVT::i32, Src);
+
+    Entry.Node = Src; 
+    Entry.Ty = Type::getInt32Ty(*DAG.getContext());
+    Entry.isSExt = false;
+    Args.push_back(Entry);
+  } else {
+    Entry.Node = Src;
+    Args.push_back(Entry);
+    
+    Entry.Node = Size;
+    Args.push_back(Entry);
+  }
+
+  char const *FunctionNames[4][3] = {
+    { "__aeabi_memcpy",  "__aeabi_memcpy4",  "__aeabi_memcpy8"  },
+    { "__aeabi_memmove", "__aeabi_memmove4", "__aeabi_memmove8" },
+    { "__aeabi_memset",  "__aeabi_memset4",  "__aeabi_memset8"  },
+    { "__aeabi_memclr",  "__aeabi_memclr4",  "__aeabi_memclr8"  }
+  };
+  TargetLowering::CallLoweringInfo CLI(DAG);
+  CLI.setDebugLoc(dl)
+      .setChain(Chain)
+      .setCallee(
+           TLI->getLibcallCallingConv(LC), Type::getVoidTy(*DAG.getContext()),
+           DAG.getExternalSymbol(FunctionNames[AEABILibcall][AlignVariant],
+                                 TLI->getPointerTy(DAG.getDataLayout())),
+           std::move(Args), 0)
+      .setDiscardResult();
+  std::pair<SDValue,SDValue> CallResult = TLI->LowerCallTo(CLI);
+  
+  return CallResult.second;
 }
 
 SDValue
@@ -42,10 +146,12 @@ ARMSelectionDAGInfo::EmitTargetCodeForMemcpy(SelectionDAG &DAG, SDLoc dl,
   // within a subtarget-specific limit.
   ConstantSDNode *ConstantSize = dyn_cast<ConstantSDNode>(Size);
   if (!ConstantSize)
-    return SDValue();
+    return EmitSpecializedLibcall(DAG, dl, Chain, Dst, Src, Size, Align,
+                                  RTLIB::MEMCPY);
   uint64_t SizeVal = ConstantSize->getZExtValue();
   if (!AlwaysInline && SizeVal > Subtarget.getMaxInlineSizeThreshold())
-    return SDValue();
+    return EmitSpecializedLibcall(DAG, dl, Chain, Dst, Src, Size, Align,
+                                  RTLIB::MEMCPY);
 
   unsigned BytesLeft = SizeVal & 3;
   unsigned NumMemOps = SizeVal >> 2;
@@ -67,7 +173,7 @@ ARMSelectionDAGInfo::EmitTargetCodeForMemcpy(SelectionDAG &DAG, SDLoc dl,
          i < MAX_LOADS_IN_LDM && EmittedNumMemOps + i < NumMemOps; ++i) {
       Loads[i] = DAG.getLoad(VT, dl, Chain,
                              DAG.getNode(ISD::ADD, dl, MVT::i32, Src,
-                                         DAG.getConstant(SrcOff, MVT::i32)),
+                                         DAG.getConstant(SrcOff, dl, MVT::i32)),
                              SrcPtrInfo.getWithOffset(SrcOff), isVolatile,
                              false, false, 0);
       TFOps[i] = Loads[i].getValue(1);
@@ -80,7 +186,7 @@ ARMSelectionDAGInfo::EmitTargetCodeForMemcpy(SelectionDAG &DAG, SDLoc dl,
          i < MAX_LOADS_IN_LDM && EmittedNumMemOps + i < NumMemOps; ++i) {
       TFOps[i] = DAG.getStore(Chain, dl, Loads[i],
                               DAG.getNode(ISD::ADD, dl, MVT::i32, Dst,
-                                          DAG.getConstant(DstOff, MVT::i32)),
+                                          DAG.getConstant(DstOff, dl, MVT::i32)),
                               DstPtrInfo.getWithOffset(DstOff),
                               isVolatile, false, 0);
       DstOff += VTSize;
@@ -108,7 +214,7 @@ ARMSelectionDAGInfo::EmitTargetCodeForMemcpy(SelectionDAG &DAG, SDLoc dl,
 
     Loads[i] = DAG.getLoad(VT, dl, Chain,
                            DAG.getNode(ISD::ADD, dl, MVT::i32, Src,
-                                       DAG.getConstant(SrcOff, MVT::i32)),
+                                       DAG.getConstant(SrcOff, dl, MVT::i32)),
                            SrcPtrInfo.getWithOffset(SrcOff),
                            false, false, false, 0);
     TFOps[i] = Loads[i].getValue(1);
@@ -132,7 +238,7 @@ ARMSelectionDAGInfo::EmitTargetCodeForMemcpy(SelectionDAG &DAG, SDLoc dl,
 
     TFOps[i] = DAG.getStore(Chain, dl, Loads[i],
                             DAG.getNode(ISD::ADD, dl, MVT::i32, Dst,
-                                        DAG.getConstant(DstOff, MVT::i32)),
+                                        DAG.getConstant(DstOff, dl, MVT::i32)),
                             DstPtrInfo.getWithOffset(DstOff), false, false, 0);
     ++i;
     DstOff += VTSize;
@@ -142,59 +248,26 @@ ARMSelectionDAGInfo::EmitTargetCodeForMemcpy(SelectionDAG &DAG, SDLoc dl,
                      makeArrayRef(TFOps, i));
 }
 
-// Adjust parameters for memset, EABI uses format (ptr, size, value),
-// GNU library uses (ptr, value, size)
-// See RTABI section 4.3.4
+
+SDValue ARMSelectionDAGInfo::
+EmitTargetCodeForMemmove(SelectionDAG &DAG, SDLoc dl,
+                         SDValue Chain,
+                         SDValue Dst, SDValue Src,
+                         SDValue Size, unsigned Align,
+                         bool isVolatile,
+                         MachinePointerInfo DstPtrInfo,
+                         MachinePointerInfo SrcPtrInfo) const {
+  return EmitSpecializedLibcall(DAG, dl, Chain, Dst, Src, Size, Align,
+                                RTLIB::MEMMOVE);
+}
+
+
 SDValue ARMSelectionDAGInfo::
 EmitTargetCodeForMemset(SelectionDAG &DAG, SDLoc dl,
                         SDValue Chain, SDValue Dst,
                         SDValue Src, SDValue Size,
                         unsigned Align, bool isVolatile,
                         MachinePointerInfo DstPtrInfo) const {
-  const ARMSubtarget &Subtarget =
-      DAG.getMachineFunction().getSubtarget<ARMSubtarget>();
-  // Use default for non-AAPCS (or MachO) subtargets
-  if (!Subtarget.isAAPCS_ABI() || Subtarget.isTargetMachO() ||
-      Subtarget.isTargetWindows())
-    return SDValue();
-
-  const ARMTargetLowering &TLI = *Subtarget.getTargetLowering();
-  TargetLowering::ArgListTy Args;
-  TargetLowering::ArgListEntry Entry;
-
-  // First argument: data pointer
-  Type *IntPtrTy = TLI.getDataLayout()->getIntPtrType(*DAG.getContext());
-  Entry.Node = Dst;
-  Entry.Ty = IntPtrTy;
-  Args.push_back(Entry);
-
-  // Second argument: buffer size
-  Entry.Node = Size;
-  Entry.Ty = IntPtrTy;
-  Entry.isSExt = false;
-  Args.push_back(Entry);
-
-  // Extend or truncate the argument to be an i32 value for the call.
-  if (Src.getValueType().bitsGT(MVT::i32))
-    Src = DAG.getNode(ISD::TRUNCATE, dl, MVT::i32, Src);
-  else
-    Src = DAG.getNode(ISD::ZERO_EXTEND, dl, MVT::i32, Src);
-
-  // Third argument: value to fill
-  Entry.Node = Src;
-  Entry.Ty = Type::getInt32Ty(*DAG.getContext());
-  Entry.isSExt = true;
-  Args.push_back(Entry);
-
-  // Emit __eabi_memset call
-  TargetLowering::CallLoweringInfo CLI(DAG);
-  CLI.setDebugLoc(dl).setChain(Chain)
-    .setCallee(TLI.getLibcallCallingConv(RTLIB::MEMSET),
-               Type::getVoidTy(*DAG.getContext()),
-               DAG.getExternalSymbol(TLI.getLibcallName(RTLIB::MEMSET),
-                                     TLI.getPointerTy()), std::move(Args), 0)
-    .setDiscardResult();
-
-  std::pair<SDValue,SDValue> CallResult = TLI.LowerCallTo(CLI);
-  return CallResult.second;
+  return EmitSpecializedLibcall(DAG, dl, Chain, Dst, Src, Size, Align,
+                                RTLIB::MEMSET);
 }
diff --git a/lib/Target/ARM/ARMSelectionDAGInfo.h b/lib/Target/ARM/ARMSelectionDAGInfo.h
index 94b98e66847..289879ee1d7 100644
--- a/lib/Target/ARM/ARMSelectionDAGInfo.h
+++ b/lib/Target/ARM/ARMSelectionDAGInfo.h
@@ -37,8 +37,6 @@ namespace ARM_AM {
 
 class ARMSelectionDAGInfo : public TargetSelectionDAGInfo {
 public:
-  explicit ARMSelectionDAGInfo(const DataLayout &DL);
-  ~ARMSelectionDAGInfo();
 
   SDValue EmitTargetCodeForMemcpy(SelectionDAG &DAG, SDLoc dl,
                                   SDValue Chain,
@@ -48,6 +46,13 @@ class ARMSelectionDAGInfo : public TargetSelectionDAGInfo {
                                   MachinePointerInfo DstPtrInfo,
                                   MachinePointerInfo SrcPtrInfo) const override;
 
+  SDValue EmitTargetCodeForMemmove(SelectionDAG &DAG, SDLoc dl,
+                                   SDValue Chain,
+                                   SDValue Dst, SDValue Src,
+                                   SDValue Size, unsigned Align, bool isVolatile,
+                                   MachinePointerInfo DstPtrInfo,
+                                   MachinePointerInfo SrcPtrInfo) const override;
+
   // Adjust parameters for memset, see RTABI section 4.3.4
   SDValue EmitTargetCodeForMemset(SelectionDAG &DAG, SDLoc dl,
                                   SDValue Chain,
@@ -55,6 +60,12 @@ class ARMSelectionDAGInfo : public TargetSelectionDAGInfo {
                                   SDValue Op3, unsigned Align,
                                   bool isVolatile,
                                   MachinePointerInfo DstPtrInfo) const override;
+
+  SDValue EmitSpecializedLibcall(SelectionDAG &DAG, SDLoc dl,
+                                 SDValue Chain,
+                                 SDValue Dst, SDValue Src,
+                                 SDValue Size, unsigned Align,
+                                 RTLIB::Libcall LC) const;
 };
 
 }
diff --git a/lib/Target/ARM/ARMSubtarget.cpp b/lib/Target/ARM/ARMSubtarget.cpp
index 2e4abafa580..4c6e69654d5 100644
--- a/lib/Target/ARM/ARMSubtarget.cpp
+++ b/lib/Target/ARM/ARMSubtarget.cpp
@@ -39,47 +39,10 @@ using namespace llvm;
 #define GET_SUBTARGETINFO_CTOR
 #include "ARMGenSubtargetInfo.inc"
 
-static cl::opt<bool>
-ReserveR9("arm-reserve-r9", cl::Hidden,
-          cl::desc("Reserve R9, making it unavailable as GPR"));
-
-static cl::opt<bool>
-ArmUseMOVT("arm-use-movt", cl::init(true), cl::Hidden);
-
-// @LOCALMOD-START
-// TODO: * JITing has not been tested at all
-//       * Thumb mode operation is also not clear: it seems jump tables
-//         for thumb are broken independent of this option
-static cl::opt<bool>
-NoInlineJumpTables("no-inline-jumptables",
-                  cl::desc("Do not place jump tables inline in the code"));
-// @LOCALMOD-END
-
 static cl::opt<bool>
 UseFusedMulOps("arm-use-mulops",
                cl::init(true), cl::Hidden);
 
-namespace {
-enum AlignMode {
-  DefaultAlign,
-  StrictAlign,
-  NoStrictAlign
-};
-}
-
-static cl::opt<AlignMode>
-Align(cl::desc("Load/store alignment support"),
-      cl::Hidden, cl::init(DefaultAlign),
-      cl::values(
-          clEnumValN(DefaultAlign,  "arm-default-align",
-                     "Generate unaligned accesses only on hardware/OS "
-                     "combinations that are known to support them"),
-          clEnumValN(StrictAlign,   "arm-strict-align",
-                     "Disallow all unaligned memory accesses"),
-          clEnumValN(NoStrictAlign, "arm-no-strict-align",
-                     "Allow unaligned memory accesses"),
-          clEnumValEnd));
-
 enum ITMode {
   DefaultIT,
   RestrictedIT,
@@ -115,13 +78,12 @@ ARMFrameLowering *ARMSubtarget::initializeFrameLowering(StringRef CPU,
   return new ARMFrameLowering(STI);
 }
 
-ARMSubtarget::ARMSubtarget(const std::string &TT, const std::string &CPU,
+ARMSubtarget::ARMSubtarget(const Triple &TT, const std::string &CPU,
                            const std::string &FS,
                            const ARMBaseTargetMachine &TM, bool IsLittle)
     : ARMGenSubtargetInfo(TT, CPU, FS), ARMProcFamily(Others),
       ARMProcClass(None), stackAlignment(4), CPUString(CPU), IsLittle(IsLittle),
       TargetTriple(TT), Options(TM.Options), TM(TM),
-      TSInfo(*TM.getDataLayout()),
       FrameLowering(initializeFrameLowering(CPU, FS)),
       // At this point initializeSubtargetDependencies has been called so
       // we can query directly.
@@ -149,19 +111,16 @@ void ARMSubtarget::initializeEnvironment() {
   HasFPARMv8 = false;
   HasNEON = false;
   UseNEONForSinglePrecisionFP = false;
-  // @LOCALMOD-START
-  UseInlineJumpTables = !NoInlineJumpTables;
-  UseConstIslands = true;
-  // @LOCALMOD-END
   UseMulOps = UseFusedMulOps;
   SlowFPVMLx = false;
   HasVMLxForwarding = false;
   SlowFPBrcc = false;
   InThumbMode = false;
+  UseSoftFloat = false;
   HasThumb2 = false;
   NoARM = false;
-  IsR9Reserved = ReserveR9;
-  UseMovt = false;
+  ReserveR9 = false;
+  NoMovt = false;
   SupportsTailCall = false;
   HasFP16 = false;
   HasD16 = false;
@@ -181,9 +140,10 @@ void ARMSubtarget::initializeEnvironment() {
   HasCrypto = false;
   HasCRC = false;
   HasZeroCycleZeroing = false;
-  AllowsUnalignedMem = false;
+  StrictAlign = false;
   Thumb2DSP = false;
   UseNaClTrap = false;
+  GenLongCalls = false;
   UnsafeFPMath = false;
 }
 
@@ -199,8 +159,7 @@ void ARMSubtarget::initSubtargetFeatures(StringRef CPU, StringRef FS) {
   // Insert the architecture feature derived from the target triple into the
   // feature string. This is important for setting features that are implied
   // based on the architecture version.
-  std::string ArchFS =
-      ARM_MC::ParseARMTriple(TargetTriple.getTriple(), CPUString);
+  std::string ArchFS = ARM_MC::ParseARMTriple(TargetTriple, CPUString);
   if (!FS.empty()) {
     if (!ArchFS.empty())
       ArchFS = (Twine(ArchFS) + "," + FS).str();
@@ -228,52 +187,10 @@ void ARMSubtarget::initSubtargetFeatures(StringRef CPU, StringRef FS) {
   if (isTargetNaCl())
     stackAlignment = 16;
 
-  UseMovt = hasV6T2Ops() && ArmUseMOVT;
-
-  if (isTargetMachO()) {
-    IsR9Reserved = ReserveR9 || !HasV6Ops;
+  if (isTargetMachO())
     SupportsTailCall = !isTargetIOS() || !getTargetTriple().isOSVersionLT(5, 0);
-  } else {
-    IsR9Reserved = ReserveR9;
+  else
     SupportsTailCall = !isThumb1Only();
-  }
-
-  // @LOCALMOD-BEGIN
-  if (isTargetNaCl()) {
-    // NaCl reserves R9 for TLS.
-    IsR9Reserved = true;
-    // NaCl uses MovT to avoid generating constant islands.
-    UseMovt = true;
-    UseInlineJumpTables = false;
-    UseConstIslands = false;
-  }
-  // @LOCALMOD-END
-
-  if (Align == DefaultAlign) {
-    // Assume pre-ARMv6 doesn't support unaligned accesses.
-    //
-    // ARMv6 may or may not support unaligned accesses depending on the
-    // SCTLR.U bit, which is architecture-specific. We assume ARMv6
-    // Darwin and NetBSD targets support unaligned accesses, and others don't.
-    //
-    // ARMv7 always has SCTLR.U set to 1, but it has a new SCTLR.A bit
-    // which raises an alignment fault on unaligned accesses. Linux
-    // defaults this bit to 0 and handles it as a system-wide (not
-    // per-process) setting. It is therefore safe to assume that ARMv7+
-    // Linux targets support unaligned accesses. The same goes for NaCl.
-    //
-    // The above behavior is consistent with GCC.
-    AllowsUnalignedMem =
-      (hasV7Ops() && (isTargetLinux() || isTargetNaCl() ||
-                      isTargetNetBSD())) ||
-      (hasV6Ops() && (isTargetMachO() || isTargetNetBSD()));
-  } else {
-    AllowsUnalignedMem = !(Align == StrictAlign);
-  }
-
-  // No v6M core supports unaligned memory access (v6M ARM ARM A3.2)
-  if (isV6M())
-    AllowsUnalignedMem = false;
 
   switch (IT) {
   case DefaultIT:
@@ -288,8 +205,8 @@ void ARMSubtarget::initSubtargetFeatures(StringRef CPU, StringRef FS) {
   }
 
   // NEON f32 ops are non-IEEE 754 compliant. Darwin is ok with it by default.
-  uint64_t Bits = getFeatureBits();
-  if ((Bits & ARM::ProcA5 || Bits & ARM::ProcA8) && // Where this matters
+  const FeatureBitset &Bits = getFeatureBits();
+  if ((Bits[ARM::ProcA5] || Bits[ARM::ProcA8]) && // Where this matters
       (Options.UnsafeFPMath || isTargetDarwin()))
     UseNEONForSinglePrecisionFP = true;
 }
@@ -310,7 +227,7 @@ ARMSubtarget::GVIsIndirectSymbol(const GlobalValue *GV,
   if (RelocM == Reloc::Static)
     return false;
 
-  bool isDecl = GV->isDeclarationForLinker();
+  bool isDef = GV->isStrongDefinitionForLinker();
 
   if (!isTargetMachO()) {
     // Extra load is needed for all externally visible.
@@ -318,34 +235,22 @@ ARMSubtarget::GVIsIndirectSymbol(const GlobalValue *GV,
       return false;
     return true;
   } else {
-    if (RelocM == Reloc::PIC_) {
-      // If this is a strong reference to a definition, it is definitely not
-      // through a stub.
-      if (!isDecl && !GV->isWeakForLinker())
-        return false;
-
-      // Unless we have a symbol with hidden visibility, we have to go through a
-      // normal $non_lazy_ptr stub because this symbol might be resolved late.
-      if (!GV->hasHiddenVisibility())  // Non-hidden $non_lazy_ptr reference.
-        return true;
+    // If this is a strong reference to a definition, it is definitely not
+    // through a stub.
+    if (isDef)
+      return false;
+
+    // Unless we have a symbol with hidden visibility, we have to go through a
+    // normal $non_lazy_ptr stub because this symbol might be resolved late.
+    if (!GV->hasHiddenVisibility())  // Non-hidden $non_lazy_ptr reference.
+      return true;
 
+    if (RelocM == Reloc::PIC_) {
       // If symbol visibility is hidden, we have a stub for common symbol
       // references and external declarations.
-      if (isDecl || GV->hasCommonLinkage())
+      if (GV->isDeclarationForLinker() || GV->hasCommonLinkage())
         // Hidden $non_lazy_ptr reference.
         return true;
-
-      return false;
-    } else {
-      // If this is a strong reference to a definition, it is definitely not
-      // through a stub.
-      if (!isDecl && !GV->isWeakForLinker())
-        return false;
-
-      // Unless we have a symbol with hidden visibility, we have to go through a
-      // normal $non_lazy_ptr stub because this symbol might be resolved late.
-      if (!GV->hasHiddenVisibility())  // Non-hidden $non_lazy_ptr reference.
-        return true;
     }
   }
 
@@ -360,8 +265,19 @@ bool ARMSubtarget::hasSinCos() const {
   return getTargetTriple().isiOS() && !getTargetTriple().isOSVersionLT(7, 0);
 }
 
+bool ARMSubtarget::enableMachineScheduler() const {
+  // Enable the MachineScheduler before register allocation for out-of-order
+  // architectures where we do not use the PostRA scheduler anymore (for now
+  // restricted to swift).
+  return getSchedModel().isOutOfOrder() && isSwift();
+}
+
 // This overrides the PostRAScheduler bit in the SchedModel for any CPU.
-bool ARMSubtarget::enablePostMachineScheduler() const {
+bool ARMSubtarget::enablePostRAScheduler() const {
+  // No need for PostRA scheduling on out of order CPUs (for now restricted to
+  // swift).
+  if (getSchedModel().isOutOfOrder() && isSwift())
+    return false;
   return (!isThumb() || hasThumb2());
 }
 
@@ -373,6 +289,14 @@ bool ARMSubtarget::useMovt(const MachineFunction &MF) const {
   // NOTE Windows on ARM needs to use mov.w/mov.t pairs to materialise 32-bit
   // immediates as it is inherently position independent, and may be out of
   // range otherwise.
-  return UseMovt && (isTargetWindows() ||
-                     !MF.getFunction()->hasFnAttribute(Attribute::MinSize));
+  return !NoMovt && hasV6T2Ops() &&
+         (isTargetWindows() ||
+          !MF.getFunction()->hasFnAttribute(Attribute::MinSize));
+}
+
+bool ARMSubtarget::useFastISel() const {
+  // Thumb2 support on iOS; ARM support on iOS, Linux and NaCl.
+  return TM.Options.EnableFastISel &&
+         ((isTargetMachO() && !isThumb1Only()) ||
+          (isTargetLinux() && !isThumb()) || (isTargetNaCl() && !isThumb()));
 }
diff --git a/lib/Target/ARM/ARMSubtarget.h b/lib/Target/ARM/ARMSubtarget.h
index ea4a884bd55..d6d3d83c87e 100644
--- a/lib/Target/ARM/ARMSubtarget.h
+++ b/lib/Target/ARM/ARMSubtarget.h
@@ -39,8 +39,6 @@ class StringRef;
 class TargetOptions;
 class ARMBaseTargetMachine;
 
-extern cl::opt<bool>  EnableARMDwarfEH; // @LOCALMOD
-
 class ARMSubtarget : public ARMGenSubtargetInfo {
 protected:
   enum ARMProcFamilyEnum {
@@ -102,26 +100,21 @@ class ARMSubtarget : public ARMGenSubtargetInfo {
   /// InThumbMode - True if compiling for Thumb, false for ARM.
   bool InThumbMode;
 
+  /// UseSoftFloat - True if we're using software floating point features.
+  bool UseSoftFloat;
+
   /// HasThumb2 - True if Thumb2 instructions are supported.
   bool HasThumb2;
 
   /// NoARM - True if subtarget does not support ARM mode execution.
   bool NoARM;
 
-  /// IsR9Reserved - True if R9 is a not available as general purpose register.
-  bool IsR9Reserved;
-
-  // @LOCALMOD-START
-  /// UseInlineJumpTables - True if jump tables should be in-line in the code.
-  bool UseInlineJumpTables;
+  /// ReserveR9 - True if R9 is not available as a general purpose register.
+  bool ReserveR9;
 
-  /// UseConstIslands - True if constant islands should be used.
-  bool UseConstIslands;
-  // @LOCALMOD-END
-
-  /// UseMovt - True if MOVT / MOVW pairs are used for materialization of 32-bit
-  /// imms (including global addresses).
-  bool UseMovt;
+  /// NoMovt - True if MOVT / MOVW pairs are not used for materialization of
+  /// 32-bit imms (including global addresses).
+  bool NoMovt;
 
   /// SupportsTailCall - True if the OS supports tail call. The dynamic linker
   /// must be able to synthesize call stubs for interworking between ARM and
@@ -197,10 +190,10 @@ class ARMSubtarget : public ARMGenSubtargetInfo {
   /// particularly effective at zeroing a VFP register.
   bool HasZeroCycleZeroing;
 
-  /// AllowsUnalignedMem - If true, the subtarget allows unaligned memory
+  /// StrictAlign - If true, the subtarget disallows unaligned memory
   /// accesses for some types.  For details, see
   /// ARMTargetLowering::allowsMisalignedMemoryAccesses().
-  bool AllowsUnalignedMem;
+  bool StrictAlign;
 
   /// RestrictIT - If true, the subtarget disallows generation of deprecated IT
   ///  blocks to conform to ARMv8 rule.
@@ -213,6 +206,9 @@ class ARMSubtarget : public ARMGenSubtargetInfo {
   /// NaCl TRAP instruction is generated instead of the regular TRAP.
   bool UseNaClTrap;
 
+  /// Generate calls via indirect call instructions.
+  bool GenLongCalls;
+
   /// Target machine allowed unsafe FP math (such as use of NEON fp)
   bool UnsafeFPMath;
 
@@ -244,8 +240,8 @@ class ARMSubtarget : public ARMGenSubtargetInfo {
   /// This constructor initializes the data members to match that
   /// of the specified triple.
   ///
-  ARMSubtarget(const std::string &TT, const std::string &CPU,
-               const std::string &FS, const ARMBaseTargetMachine &TM, bool IsLittle);
+  ARMSubtarget(const Triple &TT, const std::string &CPU, const std::string &FS,
+               const ARMBaseTargetMachine &TM, bool IsLittle);
 
   /// getMaxInlineSizeThreshold - Returns the maximum memset / memcpy size
   /// that still makes it profitable to inline the call.
@@ -349,6 +345,7 @@ class ARMSubtarget : public ARMGenSubtargetInfo {
   bool hasMPExtension() const { return HasMPExtension; }
   bool hasThumb2DSP() const { return Thumb2DSP; }
   bool useNaClTrap() const { return UseNaClTrap; }
+  bool genLongCalls() const { return GenLongCalls; }
 
   bool hasFP16() const { return HasFP16; }
   bool hasD16() const { return HasD16; }
@@ -387,8 +384,7 @@ class ARMSubtarget : public ARMGenSubtargetInfo {
             TargetTriple.getEnvironment() == Triple::EABIHF ||
             TargetTriple.getEnvironment() == Triple::GNUEABIHF ||
             TargetTriple.getEnvironment() == Triple::Android) &&
-        !isTargetDarwin() && !isTargetWindows() &&
-        !isTargetNaCl() && !EnableARMDwarfEH; // @LOCALMOD
+           !isTargetDarwin() && !isTargetWindows();
   }
 
   bool isTargetHardFloat() const {
@@ -404,6 +400,7 @@ class ARMSubtarget : public ARMGenSubtargetInfo {
   bool isAPCS_ABI() const;
   bool isAAPCS_ABI() const;
 
+  bool useSoftFloat() const { return UseSoftFloat; }
   bool isThumb() const { return InThumbMode; }
   bool isThumb1Only() const { return InThumbMode && !HasThumb2; }
   bool isThumb2() const { return InThumbMode && HasThumb2; }
@@ -412,20 +409,15 @@ class ARMSubtarget : public ARMGenSubtargetInfo {
   bool isRClass() const { return ARMProcClass == RClass; }
   bool isAClass() const { return ARMProcClass == AClass; }
 
-  bool isV6M() const {
-    return isThumb1Only() && isMClass();
+  bool isR9Reserved() const {
+    return isTargetMachO() ? (ReserveR9 || !HasV6Ops) : ReserveR9;
   }
 
-  bool isR9Reserved() const { return IsR9Reserved; }
-
   bool useMovt(const MachineFunction &MF) const;
 
   bool supportsTailCall() const { return SupportsTailCall; }
 
-  // @LOCALMOD
-  bool useConstIslands() const { return UseConstIslands; }
-
-  bool allowsUnalignedMem() const { return AllowsUnalignedMem; }
+  bool allowsUnalignedMem() const { return !StrictAlign; }
 
   bool restrictIT() const { return RestrictIT; }
 
@@ -439,8 +431,11 @@ class ARMSubtarget : public ARMGenSubtargetInfo {
   /// compiler runtime or math libraries.
   bool hasSinCos() const;
 
+  /// Returns true if machine scheduler should be enabled.
+  bool enableMachineScheduler() const override;
+
   /// True for some subtargets at > -O0.
-  bool enablePostMachineScheduler() const override;
+  bool enablePostRAScheduler() const override;
 
   // enableAtomicExpand- True if we need to expand our atomics.
   bool enableAtomicExpand() const override;
@@ -460,7 +455,8 @@ class ARMSubtarget : public ARMGenSubtargetInfo {
   /// symbol.
   bool GVIsIndirectSymbol(const GlobalValue *GV, Reloc::Model RelocM) const;
 
-  bool useInlineJumpTables() const {return UseInlineJumpTables;} // @LOCALMOD
+  /// True if fast-isel is used.
+  bool useFastISel() const;
 };
 } // End llvm namespace
 
diff --git a/lib/Target/ARM/ARMTargetMachine.cpp b/lib/Target/ARM/ARMTargetMachine.cpp
index 1330a2908b5..93495d66ae7 100644
--- a/lib/Target/ARM/ARMTargetMachine.cpp
+++ b/lib/Target/ARM/ARMTargetMachine.cpp
@@ -23,9 +23,6 @@
 #include "llvm/Support/FormattedStream.h"
 #include "llvm/Support/TargetRegistry.h"
 #include "llvm/Target/TargetOptions.h"
-// @LOCALMOD-START
-#include "llvm/Transforms/NaCl.h"
-// @LOCALMOD-END
 #include "llvm/Transforms/Scalar.h"
 using namespace llvm;
 
@@ -83,8 +80,7 @@ computeTargetABI(const Triple &TT, StringRef CPU,
   // FIXME: This is duplicated code from the front end and should be unified.
   if (TT.isOSBinFormatMachO()) {
     if (TT.getEnvironment() == llvm::Triple::EABI ||
-        (TT.getOS() == llvm::Triple::UnknownOS &&
-         TT.getObjectFormat() == llvm::Triple::MachO) ||
+        (TT.getOS() == llvm::Triple::UnknownOS && TT.isOSBinFormatMachO()) ||
         CPU.startswith("cortex-m")) {
       TargetABI = ARMBaseTargetMachine::ARM_ABI_AAPCS;
     } else {
@@ -107,8 +103,8 @@ computeTargetABI(const Triple &TT, StringRef CPU,
       TargetABI = ARMBaseTargetMachine::ARM_ABI_APCS;
       break;
     default:
-      if (TT.getOS() == llvm::Triple::NetBSD)
-	TargetABI = ARMBaseTargetMachine::ARM_ABI_APCS;
+      if (TT.isOSNetBSD())
+        TargetABI = ARMBaseTargetMachine::ARM_ABI_APCS;
       else
 	TargetABI = ARMBaseTargetMachine::ARM_ABI_AAPCS;
       break;
@@ -118,11 +114,10 @@ computeTargetABI(const Triple &TT, StringRef CPU,
   return TargetABI;
 }
 
-static std::string computeDataLayout(StringRef TT, StringRef CPU,
+static std::string computeDataLayout(const Triple &TT, StringRef CPU,
                                      const TargetOptions &Options,
                                      bool isLittle) {
-  const Triple Triple(TT);
-  auto ABI = computeTargetABI(Triple, CPU, Options);
+  auto ABI = computeTargetABI(TT, CPU, Options);
   std::string Ret = "";
 
   if (isLittle)
@@ -132,7 +127,7 @@ static std::string computeDataLayout(StringRef TT, StringRef CPU,
     // Big endian.
     Ret += "E";
 
-  Ret += DataLayout::getManglingComponent(Triple);
+  Ret += DataLayout::getManglingComponent(TT);
 
   // Pointers are 32 bits and aligned to 32 bits.
   Ret += "-p:32:32";
@@ -162,7 +157,7 @@ static std::string computeDataLayout(StringRef TT, StringRef CPU,
 
   // The stack is 128 bit aligned on NaCl, 64 bit aligned on AAPCS and 32 bit
   // aligned everywhere else.
-  if (Triple.isOSNaCl())
+  if (TT.isOSNaCl())
     Ret += "-S128";
   else if (ABI == ARMBaseTargetMachine::ARM_ABI_AAPCS)
     Ret += "-S64";
@@ -174,15 +169,15 @@ static std::string computeDataLayout(StringRef TT, StringRef CPU,
 
 /// TargetMachine ctor - Create an ARM architecture model.
 ///
-ARMBaseTargetMachine::ARMBaseTargetMachine(const Target &T, StringRef TT,
+ARMBaseTargetMachine::ARMBaseTargetMachine(const Target &T, const Triple &TT,
                                            StringRef CPU, StringRef FS,
                                            const TargetOptions &Options,
                                            Reloc::Model RM, CodeModel::Model CM,
                                            CodeGenOpt::Level OL, bool isLittle)
     : LLVMTargetMachine(T, computeDataLayout(TT, CPU, Options, isLittle), TT,
                         CPU, FS, Options, RM, CM, OL),
-      TargetABI(computeTargetABI(Triple(TT), CPU, Options)),
-      TLOF(createTLOF(Triple(getTargetTriple()))),
+      TargetABI(computeTargetABI(TT, CPU, Options)),
+      TLOF(createTLOF(getTargetTriple())),
       Subtarget(TT, CPU, FS, *this, isLittle), isLittle(isLittle) {
 
   // Default to triple-appropriate float ABI
@@ -210,13 +205,15 @@ ARMBaseTargetMachine::getSubtargetImpl(const Function &F) const {
   // function before we can generate a subtarget. We also need to use
   // it as a key for the subtarget since that can be the only difference
   // between two functions.
-  Attribute SFAttr = F.getFnAttribute("use-soft-float");
-  bool SoftFloat = !SFAttr.hasAttribute(Attribute::None)
-                       ? SFAttr.getValueAsString() == "true"
-                       : Options.UseSoftFloat;
-
-  auto &I = SubtargetMap[CPU + FS + (SoftFloat ? "use-soft-float=true"
-                                               : "use-soft-float=false")];
+  bool SoftFloat =
+      F.hasFnAttribute("use-soft-float") &&
+      F.getFnAttribute("use-soft-float").getValueAsString() == "true";
+  // If the soft float attribute is set on the function turn on the soft float
+  // subtarget feature.
+  if (SoftFloat)
+    FS += FS.empty() ? "+soft-float" : ",+soft-float";
+
+  auto &I = SubtargetMap[CPU + FS];
   if (!I) {
     // This needs to be done before we create a new subtarget since any
     // creation will depend on the TM and the code generation flags on the
@@ -235,8 +232,9 @@ TargetIRAnalysis ARMBaseTargetMachine::getTargetIRAnalysis() {
 
 void ARMTargetMachine::anchor() { }
 
-ARMTargetMachine::ARMTargetMachine(const Target &T, StringRef TT, StringRef CPU,
-                                   StringRef FS, const TargetOptions &Options,
+ARMTargetMachine::ARMTargetMachine(const Target &T, const Triple &TT,
+                                   StringRef CPU, StringRef FS,
+                                   const TargetOptions &Options,
                                    Reloc::Model RM, CodeModel::Model CM,
                                    CodeGenOpt::Level OL, bool isLittle)
     : ARMBaseTargetMachine(T, TT, CPU, FS, Options, RM, CM, OL, isLittle) {
@@ -248,7 +246,7 @@ ARMTargetMachine::ARMTargetMachine(const Target &T, StringRef TT, StringRef CPU,
 
 void ARMLETargetMachine::anchor() { }
 
-ARMLETargetMachine::ARMLETargetMachine(const Target &T, StringRef TT,
+ARMLETargetMachine::ARMLETargetMachine(const Target &T, const Triple &TT,
                                        StringRef CPU, StringRef FS,
                                        const TargetOptions &Options,
                                        Reloc::Model RM, CodeModel::Model CM,
@@ -257,7 +255,7 @@ ARMLETargetMachine::ARMLETargetMachine(const Target &T, StringRef TT,
 
 void ARMBETargetMachine::anchor() { }
 
-ARMBETargetMachine::ARMBETargetMachine(const Target &T, StringRef TT,
+ARMBETargetMachine::ARMBETargetMachine(const Target &T, const Triple &TT,
                                        StringRef CPU, StringRef FS,
                                        const TargetOptions &Options,
                                        Reloc::Model RM, CodeModel::Model CM,
@@ -266,19 +264,18 @@ ARMBETargetMachine::ARMBETargetMachine(const Target &T, StringRef TT,
 
 void ThumbTargetMachine::anchor() { }
 
-ThumbTargetMachine::ThumbTargetMachine(const Target &T, StringRef TT,
+ThumbTargetMachine::ThumbTargetMachine(const Target &T, const Triple &TT,
                                        StringRef CPU, StringRef FS,
                                        const TargetOptions &Options,
                                        Reloc::Model RM, CodeModel::Model CM,
                                        CodeGenOpt::Level OL, bool isLittle)
-    : ARMBaseTargetMachine(T, TT, CPU, FS, Options, RM, CM, OL,
-                           isLittle) {
+    : ARMBaseTargetMachine(T, TT, CPU, FS, Options, RM, CM, OL, isLittle) {
   initAsmInfo();
 }
 
 void ThumbLETargetMachine::anchor() { }
 
-ThumbLETargetMachine::ThumbLETargetMachine(const Target &T, StringRef TT,
+ThumbLETargetMachine::ThumbLETargetMachine(const Target &T, const Triple &TT,
                                            StringRef CPU, StringRef FS,
                                            const TargetOptions &Options,
                                            Reloc::Model RM, CodeModel::Model CM,
@@ -287,7 +284,7 @@ ThumbLETargetMachine::ThumbLETargetMachine(const Target &T, StringRef TT,
 
 void ThumbBETargetMachine::anchor() { }
 
-ThumbBETargetMachine::ThumbBETargetMachine(const Target &T, StringRef TT,
+ThumbBETargetMachine::ThumbBETargetMachine(const Target &T, const Triple &TT,
                                            StringRef CPU, StringRef FS,
                                            const TargetOptions &Options,
                                            Reloc::Model RM, CodeModel::Model CM,
@@ -305,10 +302,6 @@ class ARMPassConfig : public TargetPassConfig {
     return getTM<ARMBaseTargetMachine>();
   }
 
-  const ARMSubtarget &getARMSubtarget() const {
-    return *getARMTargetMachine().getSubtargetImpl();
-  }
-
   void addIRPasses() override;
   bool addPreISel() override;
   bool addInstSelector() override;
@@ -323,11 +316,6 @@ TargetPassConfig *ARMBaseTargetMachine::createPassConfig(PassManagerBase &PM) {
 }
 
 void ARMPassConfig::addIRPasses() {
-  // @LOCALMOD-START
-  if (getARMSubtarget().isTargetNaCl())
-    addPass(createInsertDivideCheckPass());
-  // @LOCALMOD-END
-
   if (TM->Options.ThreadModel == ThreadModel::Single)
     addPass(createLowerAtomicPass());
   else
@@ -336,35 +324,32 @@ void ARMPassConfig::addIRPasses() {
   // Cmpxchg instructions are often used with a subsequent comparison to
   // determine whether it succeeded. We can exploit existing control-flow in
   // ldrex/strex loops to simplify this, but it needs tidying up.
-  const ARMSubtarget *Subtarget = &getARMSubtarget();
-  if (Subtarget->hasAnyDataBarrier() && !Subtarget->isThumb1Only())
-    if (TM->getOptLevel() != CodeGenOpt::None && EnableAtomicTidy)
-      addPass(createCFGSimplificationPass());
+  if (TM->getOptLevel() != CodeGenOpt::None && EnableAtomicTidy)
+    addPass(createCFGSimplificationPass(-1, [this](const Function &F) {
+      const auto &ST = this->TM->getSubtarget<ARMSubtarget>(F);
+      return ST.hasAnyDataBarrier() && !ST.isThumb1Only();
+    }));
 
   TargetPassConfig::addIRPasses();
+
+  // Match interleaved memory accesses to ldN/stN intrinsics.
+  if (TM->getOptLevel() != CodeGenOpt::None)
+    addPass(createInterleavedAccessPass(TM));
 }
 
 bool ARMPassConfig::addPreISel() {
-  // @LOCALMOD-START
-  // We disable the GlobalMerge pass for PNaCl because it causes the
-  // PNaCl ABI checker to reject the program when the PNaCl translator
-  // is run in streaming mode.  This is because GlobalMerge replaces
-  // functions' GlobalVariable references with ConstantExprs which the
-  // ABI verifier rejects.
-  // TODO(mseaborn): Make the ABI checks coexist with GlobalMerge to
-  // get back the performance benefits of GlobalMerge.
-  if (!getARMSubtarget().isTargetNaCl() &&
-      ((TM->getOptLevel() == CodeGenOpt::Aggressive &&
-        EnableGlobalMerge == cl::BOU_UNSET) ||
-       EnableGlobalMerge == cl::BOU_TRUE))
-  // @LOCALMOD-END
+  if ((TM->getOptLevel() != CodeGenOpt::None &&
+       EnableGlobalMerge == cl::BOU_UNSET) ||
+      EnableGlobalMerge == cl::BOU_TRUE) {
     // FIXME: This is using the thumb1 only constant value for
     // maximal global offset for merging globals. We may want
     // to look into using the old value for non-thumb1 code of
     // 4095 based on the TargetMachine, but this starts to become
     // tricky when doing code gen per function.
-    addPass(createGlobalMergePass(TM, 127));
-
+    bool OnlyOptimizeForSize = (TM->getOptLevel() < CodeGenOpt::Aggressive) &&
+                               (EnableGlobalMerge == cl::BOU_UNSET);
+    addPass(createGlobalMergePass(TM, 127, OnlyOptimizeForSize));
+  }
 
   return false;
 }
@@ -372,8 +357,7 @@ bool ARMPassConfig::addPreISel() {
 bool ARMPassConfig::addInstSelector() {
   addPass(createARMISelDag(getARMTargetMachine(), getOptLevel()));
 
-  if (Triple(TM->getTargetTriple()).isOSBinFormatELF() &&
-      TM->Options.EnableFastISel)
+  if (TM->getTargetTriple().isOSBinFormatELF() && TM->Options.EnableFastISel)
     addPass(createARMGlobalBaseRegPass());
   return false;
 }
@@ -404,10 +388,13 @@ void ARMPassConfig::addPreSched2() {
 
   if (getOptLevel() != CodeGenOpt::None) {
     // in v8, IfConversion depends on Thumb instruction widths
-    if (getARMSubtarget().restrictIT())
-      addPass(createThumb2SizeReductionPass());
-    if (!getARMSubtarget().isThumb1Only())
-      addPass(&IfConverterID);
+    addPass(createThumb2SizeReductionPass([this](const Function &F) {
+      return this->TM->getSubtarget<ARMSubtarget>(F).restrictIT();
+    }));
+
+    addPass(createIfConverter([this](const Function &F) {
+      return !this->TM->getSubtarget<ARMSubtarget>(F).isThumb1Only();
+    }));
   }
   addPass(createThumb2ITBlockPass());
 }
@@ -416,19 +403,13 @@ void ARMPassConfig::addPreEmitPass() {
   addPass(createThumb2SizeReductionPass());
 
   // Constant island pass work on unbundled instructions.
-  if (getARMSubtarget().isThumb2())
-    addPass(&UnpackMachineBundlesID);
-
-  addPass(createARMOptimizeBarriersPass());
-
-  // @LOCALMOD-START
-  if (getARMSubtarget().useConstIslands())
-    addPass(createARMConstantIslandPass());
-  // @LOCALMOD-END
-  // @LOCALMOD-START
-  // This pass does all the heavy sfi lifting.
-  if (getARMSubtarget().isTargetNaCl()) {
-    addPass(createARMNaClRewritePass());
-  }
-  // @LOCALMOD-END
+  addPass(createUnpackMachineBundles([this](const Function &F) {
+    return this->TM->getSubtarget<ARMSubtarget>(F).isThumb2();
+  }));
+
+  // Don't optimize barriers at -O0.
+  if (getOptLevel() != CodeGenOpt::None)
+    addPass(createARMOptimizeBarriersPass());
+
+  addPass(createARMConstantIslandPass());
 }
diff --git a/lib/Target/ARM/ARMTargetMachine.h b/lib/Target/ARM/ARMTargetMachine.h
index 20ca97b616b..8c98e082ce9 100644
--- a/lib/Target/ARM/ARMTargetMachine.h
+++ b/lib/Target/ARM/ARMTargetMachine.h
@@ -36,12 +36,10 @@ class ARMBaseTargetMachine : public LLVMTargetMachine {
   mutable StringMap<std::unique_ptr<ARMSubtarget>> SubtargetMap;
 
 public:
-  ARMBaseTargetMachine(const Target &T, StringRef TT,
-                       StringRef CPU, StringRef FS,
-                       const TargetOptions &Options,
+  ARMBaseTargetMachine(const Target &T, const Triple &TT, StringRef CPU,
+                       StringRef FS, const TargetOptions &Options,
                        Reloc::Model RM, CodeModel::Model CM,
-                       CodeGenOpt::Level OL,
-                       bool isLittle);
+                       CodeGenOpt::Level OL, bool isLittle);
   ~ARMBaseTargetMachine() override;
 
   const ARMSubtarget *getSubtargetImpl() const { return &Subtarget; }
@@ -64,8 +62,8 @@ class ARMBaseTargetMachine : public LLVMTargetMachine {
 class ARMTargetMachine : public ARMBaseTargetMachine {
   virtual void anchor();
  public:
-   ARMTargetMachine(const Target &T, StringRef TT, StringRef CPU, StringRef FS,
-                    const TargetOptions &Options, Reloc::Model RM,
+   ARMTargetMachine(const Target &T, const Triple &TT, StringRef CPU,
+                    StringRef FS, const TargetOptions &Options, Reloc::Model RM,
                     CodeModel::Model CM, CodeGenOpt::Level OL, bool isLittle);
 };
 
@@ -74,8 +72,8 @@ class ARMTargetMachine : public ARMBaseTargetMachine {
 class ARMLETargetMachine : public ARMTargetMachine {
   void anchor() override;
 public:
-  ARMLETargetMachine(const Target &T, StringRef TT,
-                     StringRef CPU, StringRef FS, const TargetOptions &Options,
+  ARMLETargetMachine(const Target &T, const Triple &TT, StringRef CPU,
+                     StringRef FS, const TargetOptions &Options,
                      Reloc::Model RM, CodeModel::Model CM,
                      CodeGenOpt::Level OL);
 };
@@ -85,9 +83,10 @@ class ARMLETargetMachine : public ARMTargetMachine {
 class ARMBETargetMachine : public ARMTargetMachine {
   void anchor() override;
 public:
-  ARMBETargetMachine(const Target &T, StringRef TT, StringRef CPU, StringRef FS,
-                     const TargetOptions &Options, Reloc::Model RM,
-                     CodeModel::Model CM, CodeGenOpt::Level OL);
+  ARMBETargetMachine(const Target &T, const Triple &TT, StringRef CPU,
+                     StringRef FS, const TargetOptions &Options,
+                     Reloc::Model RM, CodeModel::Model CM,
+                     CodeGenOpt::Level OL);
 };
 
 /// ThumbTargetMachine - Thumb target machine.
@@ -97,9 +96,10 @@ class ARMBETargetMachine : public ARMTargetMachine {
 class ThumbTargetMachine : public ARMBaseTargetMachine {
   virtual void anchor();
 public:
-  ThumbTargetMachine(const Target &T, StringRef TT, StringRef CPU, StringRef FS,
-                     const TargetOptions &Options, Reloc::Model RM,
-                     CodeModel::Model CM, CodeGenOpt::Level OL, bool isLittle);
+  ThumbTargetMachine(const Target &T, const Triple &TT, StringRef CPU,
+                     StringRef FS, const TargetOptions &Options,
+                     Reloc::Model RM, CodeModel::Model CM, CodeGenOpt::Level OL,
+                     bool isLittle);
 };
 
 /// ThumbLETargetMachine - Thumb little endian target machine.
@@ -107,7 +107,7 @@ class ThumbTargetMachine : public ARMBaseTargetMachine {
 class ThumbLETargetMachine : public ThumbTargetMachine {
   void anchor() override;
 public:
-  ThumbLETargetMachine(const Target &T, StringRef TT, StringRef CPU,
+  ThumbLETargetMachine(const Target &T, const Triple &TT, StringRef CPU,
                        StringRef FS, const TargetOptions &Options,
                        Reloc::Model RM, CodeModel::Model CM,
                        CodeGenOpt::Level OL);
@@ -118,7 +118,7 @@ class ThumbLETargetMachine : public ThumbTargetMachine {
 class ThumbBETargetMachine : public ThumbTargetMachine {
   void anchor() override;
 public:
-  ThumbBETargetMachine(const Target &T, StringRef TT, StringRef CPU,
+  ThumbBETargetMachine(const Target &T, const Triple &TT, StringRef CPU,
                        StringRef FS, const TargetOptions &Options,
                        Reloc::Model RM, CodeModel::Model CM,
                        CodeGenOpt::Level OL);
diff --git a/lib/Target/ARM/ARMTargetObjectFile.cpp b/lib/Target/ARM/ARMTargetObjectFile.cpp
index 885a797dbd3..eaed5cc6875 100644
--- a/lib/Target/ARM/ARMTargetObjectFile.cpp
+++ b/lib/Target/ARM/ARMTargetObjectFile.cpp
@@ -10,8 +10,6 @@
 #include "ARMTargetObjectFile.h"
 #include "ARMTargetMachine.h"
 #include "llvm/ADT/StringExtras.h"
-#include "llvm/ADT/Triple.h" // @LOCALMOD
-#include "llvm/CodeGen/MachineModuleInfo.h" // @LOCALMOD
 #include "llvm/IR/Mangler.h"
 #include "llvm/MC/MCAsmInfo.h"
 #include "llvm/MC/MCContext.h"
@@ -34,9 +32,7 @@ void ARMElfTargetObjectFile::Initialize(MCContext &Ctx,
   TargetLoweringObjectFileELF::Initialize(Ctx, TM);
   InitializeELF(isAAPCS_ABI);
 
-  // @LOCALMOD-BEGIN
-  if (isAAPCS_ABI && !Triple(TM.getTargetTriple()).isOSNaCl()) {
-  // @LOCALMOD-END
+  if (isAAPCS_ABI) {
     LSDASection = nullptr;
   }
 
@@ -53,20 +49,13 @@ const MCExpr *ARMElfTargetObjectFile::getTTypeGlobalReference(
         GV, Encoding, Mang, TM, MMI, Streamer);
 
   assert(Encoding == DW_EH_PE_absptr && "Can handle absptr encoding only");
-  // @LOCALMOD-BEGIN
-  // FIXME: There has got to be a better way to get this info.
-  Triple T(MMI->getModule()->getTargetTriple());
-  if (T.isOSNaCl())
-    return TargetLoweringObjectFileELF::getTTypeGlobalReference(
-        GV, Encoding, Mang, TM, MMI, Streamer);
-  // @LOCALMOD-END
 
-  return MCSymbolRefExpr::Create(TM.getSymbol(GV, Mang),
+  return MCSymbolRefExpr::create(TM.getSymbol(GV, Mang),
                                  MCSymbolRefExpr::VK_ARM_TARGET2, getContext());
 }
 
 const MCExpr *ARMElfTargetObjectFile::
 getDebugThreadLocalSymbol(const MCSymbol *Sym) const {
-  return MCSymbolRefExpr::Create(Sym, MCSymbolRefExpr::VK_ARM_TLSLDO,
+  return MCSymbolRefExpr::create(Sym, MCSymbolRefExpr::VK_ARM_TLSLDO,
                                  getContext());
 }
diff --git a/lib/Target/ARM/ARMTargetTransformInfo.cpp b/lib/Target/ARM/ARMTargetTransformInfo.cpp
index 4e1b371640b..e3c64ee8a59 100644
--- a/lib/Target/ARM/ARMTargetTransformInfo.cpp
+++ b/lib/Target/ARM/ARMTargetTransformInfo.cpp
@@ -61,14 +61,14 @@ unsigned ARMTTIImpl::getCastInstrCost(unsigned Opcode, Type *Dst, Type *Src) {
 
   if (Src->isVectorTy() && ST->hasNEON() && (ISD == ISD::FP_ROUND ||
                                           ISD == ISD::FP_EXTEND)) {
-    std::pair<unsigned, MVT> LT = TLI->getTypeLegalizationCost(Src);
+    std::pair<unsigned, MVT> LT = TLI->getTypeLegalizationCost(DL, Src);
     int Idx = CostTableLookup(NEONFltDblTbl, ISD, LT.second);
     if (Idx != -1)
       return LT.first * NEONFltDblTbl[Idx].Cost;
   }
 
-  EVT SrcTy = TLI->getValueType(Src);
-  EVT DstTy = TLI->getValueType(Dst);
+  EVT SrcTy = TLI->getValueType(DL, Src);
+  EVT DstTy = TLI->getValueType(DL, Dst);
 
   if (!SrcTy.isSimple() || !DstTy.isSimple())
     return BaseT::getCastInstrCost(Opcode, Dst, Src);
@@ -282,8 +282,8 @@ unsigned ARMTTIImpl::getCmpSelInstrCost(unsigned Opcode, Type *ValTy,
       { ISD::SELECT, MVT::v16i1, MVT::v16i64, 100 }
     };
 
-    EVT SelCondTy = TLI->getValueType(CondTy);
-    EVT SelValTy = TLI->getValueType(ValTy);
+    EVT SelCondTy = TLI->getValueType(DL, CondTy);
+    EVT SelValTy = TLI->getValueType(DL, ValTy);
     if (SelCondTy.isSimple() && SelValTy.isSimple()) {
       int Idx = ConvertCostTableLookup(NEONVectorSelectTbl, ISD,
                                        SelCondTy.getSimpleVT(),
@@ -292,7 +292,7 @@ unsigned ARMTTIImpl::getCmpSelInstrCost(unsigned Opcode, Type *ValTy,
         return NEONVectorSelectTbl[Idx].Cost;
     }
 
-    std::pair<unsigned, MVT> LT = TLI->getTypeLegalizationCost(ValTy);
+    std::pair<unsigned, MVT> LT = TLI->getTypeLegalizationCost(DL, ValTy);
     return LT.first;
   }
 
@@ -353,7 +353,7 @@ unsigned ARMTTIImpl::getShuffleCost(TTI::ShuffleKind Kind, Type *Tp, int Index,
         {ISD::VECTOR_SHUFFLE, MVT::v8i16, 2},
         {ISD::VECTOR_SHUFFLE, MVT::v16i8, 2}};
 
-    std::pair<unsigned, MVT> LT = TLI->getTypeLegalizationCost(Tp);
+    std::pair<unsigned, MVT> LT = TLI->getTypeLegalizationCost(DL, Tp);
 
     int Idx = CostTableLookup(NEONShuffleTbl, ISD::VECTOR_SHUFFLE, LT.second);
     if (Idx == -1)
@@ -379,7 +379,7 @@ unsigned ARMTTIImpl::getShuffleCost(TTI::ShuffleKind Kind, Type *Tp, int Index,
 
         {ISD::VECTOR_SHUFFLE, MVT::v16i8, 32}};
 
-    std::pair<unsigned, MVT> LT = TLI->getTypeLegalizationCost(Tp);
+    std::pair<unsigned, MVT> LT = TLI->getTypeLegalizationCost(DL, Tp);
     int Idx =
         CostTableLookup(NEONAltShuffleTbl, ISD::VECTOR_SHUFFLE, LT.second);
     if (Idx == -1)
@@ -395,7 +395,7 @@ unsigned ARMTTIImpl::getArithmeticInstrCost(
     TTI::OperandValueProperties Opd2PropInfo) {
 
   int ISDOpcode = TLI->InstructionOpcodeToISD(Opcode);
-  std::pair<unsigned, MVT> LT = TLI->getTypeLegalizationCost(Ty);
+  std::pair<unsigned, MVT> LT = TLI->getTypeLegalizationCost(DL, Ty);
 
   const unsigned FunctionCallDivCost = 20;
   const unsigned ReciprocalDivCost = 10;
@@ -468,7 +468,7 @@ unsigned ARMTTIImpl::getArithmeticInstrCost(
 unsigned ARMTTIImpl::getMemoryOpCost(unsigned Opcode, Type *Src,
                                      unsigned Alignment,
                                      unsigned AddressSpace) {
-  std::pair<unsigned, MVT> LT = TLI->getTypeLegalizationCost(Src);
+  std::pair<unsigned, MVT> LT = TLI->getTypeLegalizationCost(DL, Src);
 
   if (Src->isVectorTy() && Alignment != 16 &&
       Src->getVectorElementType()->isDoubleTy()) {
@@ -478,3 +478,28 @@ unsigned ARMTTIImpl::getMemoryOpCost(unsigned Opcode, Type *Src,
   }
   return LT.first;
 }
+
+unsigned ARMTTIImpl::getInterleavedMemoryOpCost(unsigned Opcode, Type *VecTy,
+                                                unsigned Factor,
+                                                ArrayRef<unsigned> Indices,
+                                                unsigned Alignment,
+                                                unsigned AddressSpace) {
+  assert(Factor >= 2 && "Invalid interleave factor");
+  assert(isa<VectorType>(VecTy) && "Expect a vector type");
+
+  // vldN/vstN doesn't support vector types of i64/f64 element.
+  bool EltIs64Bits = DL.getTypeAllocSizeInBits(VecTy->getScalarType()) == 64;
+
+  if (Factor <= TLI->getMaxSupportedInterleaveFactor() && !EltIs64Bits) {
+    unsigned NumElts = VecTy->getVectorNumElements();
+    Type *SubVecTy = VectorType::get(VecTy->getScalarType(), NumElts / Factor);
+    unsigned SubVecSize = DL.getTypeAllocSizeInBits(SubVecTy);
+
+    // vldN/vstN only support legal vector types of size 64 or 128 in bits.
+    if (NumElts % Factor == 0 && (SubVecSize == 64 || SubVecSize == 128))
+      return Factor;
+  }
+
+  return BaseT::getInterleavedMemoryOpCost(Opcode, VecTy, Factor, Indices,
+                                           Alignment, AddressSpace);
+}
diff --git a/lib/Target/ARM/ARMTargetTransformInfo.h b/lib/Target/ARM/ARMTargetTransformInfo.h
index 97590f60893..84f256f7372 100644
--- a/lib/Target/ARM/ARMTargetTransformInfo.h
+++ b/lib/Target/ARM/ARMTargetTransformInfo.h
@@ -42,7 +42,8 @@ class ARMTTIImpl : public BasicTTIImplBase<ARMTTIImpl> {
 
 public:
   explicit ARMTTIImpl(const ARMBaseTargetMachine *TM, Function &F)
-      : BaseT(TM), ST(TM->getSubtargetImpl(F)), TLI(ST->getTargetLowering()) {}
+      : BaseT(TM, F.getParent()->getDataLayout()), ST(TM->getSubtargetImpl(F)),
+        TLI(ST->getTargetLowering()) {}
 
   // Provide value semantics. MSVC requires that we spell all of these out.
   ARMTTIImpl(const ARMTTIImpl &Arg)
@@ -50,18 +51,6 @@ class ARMTTIImpl : public BasicTTIImplBase<ARMTTIImpl> {
   ARMTTIImpl(ARMTTIImpl &&Arg)
       : BaseT(std::move(static_cast<BaseT &>(Arg))), ST(std::move(Arg.ST)),
         TLI(std::move(Arg.TLI)) {}
-  ARMTTIImpl &operator=(const ARMTTIImpl &RHS) {
-    BaseT::operator=(static_cast<const BaseT &>(RHS));
-    ST = RHS.ST;
-    TLI = RHS.TLI;
-    return *this;
-  }
-  ARMTTIImpl &operator=(ARMTTIImpl &&RHS) {
-    BaseT::operator=(std::move(static_cast<BaseT &>(RHS)));
-    ST = std::move(RHS.ST);
-    TLI = std::move(RHS.TLI);
-    return *this;
-  }
 
   /// \name Scalar TTI Implementations
   /// @{
@@ -96,7 +85,7 @@ class ARMTTIImpl : public BasicTTIImplBase<ARMTTIImpl> {
     return 32;
   }
 
-  unsigned getMaxInterleaveFactor() {
+  unsigned getMaxInterleaveFactor(unsigned VF) {
     // These are out of order CPUs:
     if (ST->isCortexA15() || ST->isSwift())
       return 2;
@@ -126,6 +115,11 @@ class ARMTTIImpl : public BasicTTIImplBase<ARMTTIImpl> {
   unsigned getMemoryOpCost(unsigned Opcode, Type *Src, unsigned Alignment,
                            unsigned AddressSpace);
 
+  unsigned getInterleavedMemoryOpCost(unsigned Opcode, Type *VecTy,
+                                      unsigned Factor,
+                                      ArrayRef<unsigned> Indices,
+                                      unsigned Alignment,
+                                      unsigned AddressSpace);
   /// @}
 };
 
diff --git a/lib/Target/ARM/AsmParser/ARMAsmParser.cpp b/lib/Target/ARM/AsmParser/ARMAsmParser.cpp
index b9ad2c8bcd2..6d8377178e6 100644
--- a/lib/Target/ARM/AsmParser/ARMAsmParser.cpp
+++ b/lib/Target/ARM/AsmParser/ARMAsmParser.cpp
@@ -7,10 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "ARMFPUName.h"
 #include "ARMFeatures.h"
 #include "MCTargetDesc/ARMAddressingModes.h"
-#include "MCTargetDesc/ARMArchName.h"
 #include "MCTargetDesc/ARMBaseInfo.h"
 #include "MCTargetDesc/ARMMCExpr.h"
 #include "llvm/ADT/STLExtras.h"
@@ -30,6 +28,7 @@
 #include "llvm/MC/MCObjectFileInfo.h"
 #include "llvm/MC/MCParser/MCAsmLexer.h"
 #include "llvm/MC/MCParser/MCAsmParser.h"
+#include "llvm/MC/MCParser/MCAsmParserUtils.h"
 #include "llvm/MC/MCParser/MCParsedAsmOperand.h"
 #include "llvm/MC/MCRegisterInfo.h"
 #include "llvm/MC/MCSection.h"
@@ -39,6 +38,7 @@
 #include "llvm/MC/MCTargetAsmParser.h"
 #include "llvm/Support/ARMBuildAttributes.h"
 #include "llvm/Support/ARMEHABI.h"
+#include "llvm/Support/TargetParser.h"
 #include "llvm/Support/COFF.h"
 #include "llvm/Support/Debug.h"
 #include "llvm/Support/ELF.h"
@@ -189,9 +189,9 @@ class ARMAsmParser : public MCTargetAsmParser {
     return getParser().Error(L, Msg, Ranges);
   }
 
-  bool validatetLDMRegList(MCInst Inst, const OperandVector &Operands,
+  bool validatetLDMRegList(const MCInst &Inst, const OperandVector &Operands,
                            unsigned ListNo, bool IsARPop = false);
-  bool validatetSTMRegList(MCInst Inst, const OperandVector &Operands,
+  bool validatetSTMRegList(const MCInst &Inst, const OperandVector &Operands,
                            unsigned ListNo);
 
   int tryParseRegister();
@@ -242,42 +242,44 @@ class ARMAsmParser : public MCTargetAsmParser {
                              bool &CanAcceptCarrySet,
                              bool &CanAcceptPredicationCode);
 
+  void tryConvertingToTwoOperandForm(StringRef Mnemonic, bool CarrySetting,
+                                     OperandVector &Operands);
   bool isThumb() const {
     // FIXME: Can tablegen auto-generate this?
-    return (STI.getFeatureBits() & ARM::ModeThumb) != 0;
+    return STI.getFeatureBits()[ARM::ModeThumb];
   }
   bool isThumbOne() const {
-    return isThumb() && (STI.getFeatureBits() & ARM::FeatureThumb2) == 0;
+    return isThumb() && !STI.getFeatureBits()[ARM::FeatureThumb2];
   }
   bool isThumbTwo() const {
-    return isThumb() && (STI.getFeatureBits() & ARM::FeatureThumb2);
+    return isThumb() && STI.getFeatureBits()[ARM::FeatureThumb2];
   }
   bool hasThumb() const {
-    return STI.getFeatureBits() & ARM::HasV4TOps;
+    return STI.getFeatureBits()[ARM::HasV4TOps];
   }
   bool hasV6Ops() const {
-    return STI.getFeatureBits() & ARM::HasV6Ops;
+    return STI.getFeatureBits()[ARM::HasV6Ops];
   }
   bool hasV6MOps() const {
-    return STI.getFeatureBits() & ARM::HasV6MOps;
+    return STI.getFeatureBits()[ARM::HasV6MOps];
   }
   bool hasV7Ops() const {
-    return STI.getFeatureBits() & ARM::HasV7Ops;
+    return STI.getFeatureBits()[ARM::HasV7Ops];
   }
   bool hasV8Ops() const {
-    return STI.getFeatureBits() & ARM::HasV8Ops;
+    return STI.getFeatureBits()[ARM::HasV8Ops];
   }
   bool hasARM() const {
-    return !(STI.getFeatureBits() & ARM::FeatureNoARM);
+    return !STI.getFeatureBits()[ARM::FeatureNoARM];
   }
   bool hasThumb2DSP() const {
-    return STI.getFeatureBits() & ARM::FeatureDSPThumb2;
+    return STI.getFeatureBits()[ARM::FeatureDSPThumb2];
   }
   bool hasD16() const {
-    return STI.getFeatureBits() & ARM::FeatureD16;
+    return STI.getFeatureBits()[ARM::FeatureD16];
   }
   bool hasV8_1aOps() const {
-    return STI.getFeatureBits() & ARM::HasV8_1aOps;
+    return STI.getFeatureBits()[ARM::HasV8_1aOps];
   }
 
   void SwitchMode() {
@@ -285,7 +287,7 @@ class ARMAsmParser : public MCTargetAsmParser {
     setAvailableFeatures(FB);
   }
   bool isMClass() const {
-    return STI.getFeatureBits() & ARM::FeatureMClass;
+    return STI.getFeatureBits()[ARM::FeatureMClass];
   }
 
   /// @name Auto-generated Match Functions
@@ -347,7 +349,7 @@ class ARMAsmParser : public MCTargetAsmParser {
 
   ARMAsmParser(MCSubtargetInfo &STI, MCAsmParser &Parser,
                const MCInstrInfo &MII, const MCTargetOptions &Options)
-      : STI(STI), MII(MII), UC(Parser) {
+      : MCTargetAsmParser(Options), STI(STI), MII(MII), UC(Parser) {
     MCAsmParserExtension::Initialize(Parser);
 
     // Cache the MCRegisterInfo.
@@ -1052,7 +1054,7 @@ class ARMOperand : public MCParsedAsmOperand {
     if (!CE) return false;
     int64_t Value = CE->getValue();
     return (ARM_AM::getSOImmVal(Value) != -1 ||
-            ARM_AM::getSOImmVal(-Value) != -1);;
+            ARM_AM::getSOImmVal(-Value) != -1);
   }
   bool isT2SOImm() const {
     if (!isImm()) return false;
@@ -1750,62 +1752,62 @@ class ARMOperand : public MCParsedAsmOperand {
   void addExpr(MCInst &Inst, const MCExpr *Expr) const {
     // Add as immediates when possible.  Null MCExpr = 0.
     if (!Expr)
-      Inst.addOperand(MCOperand::CreateImm(0));
+      Inst.addOperand(MCOperand::createImm(0));
     else if (const MCConstantExpr *CE = dyn_cast<MCConstantExpr>(Expr))
-      Inst.addOperand(MCOperand::CreateImm(CE->getValue()));
+      Inst.addOperand(MCOperand::createImm(CE->getValue()));
     else
-      Inst.addOperand(MCOperand::CreateExpr(Expr));
+      Inst.addOperand(MCOperand::createExpr(Expr));
   }
 
   void addCondCodeOperands(MCInst &Inst, unsigned N) const {
     assert(N == 2 && "Invalid number of operands!");
-    Inst.addOperand(MCOperand::CreateImm(unsigned(getCondCode())));
+    Inst.addOperand(MCOperand::createImm(unsigned(getCondCode())));
     unsigned RegNum = getCondCode() == ARMCC::AL ? 0: ARM::CPSR;
-    Inst.addOperand(MCOperand::CreateReg(RegNum));
+    Inst.addOperand(MCOperand::createReg(RegNum));
   }
 
   void addCoprocNumOperands(MCInst &Inst, unsigned N) const {
     assert(N == 1 && "Invalid number of operands!");
-    Inst.addOperand(MCOperand::CreateImm(getCoproc()));
+    Inst.addOperand(MCOperand::createImm(getCoproc()));
   }
 
   void addCoprocRegOperands(MCInst &Inst, unsigned N) const {
     assert(N == 1 && "Invalid number of operands!");
-    Inst.addOperand(MCOperand::CreateImm(getCoproc()));
+    Inst.addOperand(MCOperand::createImm(getCoproc()));
   }
 
   void addCoprocOptionOperands(MCInst &Inst, unsigned N) const {
     assert(N == 1 && "Invalid number of operands!");
-    Inst.addOperand(MCOperand::CreateImm(CoprocOption.Val));
+    Inst.addOperand(MCOperand::createImm(CoprocOption.Val));
   }
 
   void addITMaskOperands(MCInst &Inst, unsigned N) const {
     assert(N == 1 && "Invalid number of operands!");
-    Inst.addOperand(MCOperand::CreateImm(ITMask.Mask));
+    Inst.addOperand(MCOperand::createImm(ITMask.Mask));
   }
 
   void addITCondCodeOperands(MCInst &Inst, unsigned N) const {
     assert(N == 1 && "Invalid number of operands!");
-    Inst.addOperand(MCOperand::CreateImm(unsigned(getCondCode())));
+    Inst.addOperand(MCOperand::createImm(unsigned(getCondCode())));
   }
 
   void addCCOutOperands(MCInst &Inst, unsigned N) const {
     assert(N == 1 && "Invalid number of operands!");
-    Inst.addOperand(MCOperand::CreateReg(getReg()));
+    Inst.addOperand(MCOperand::createReg(getReg()));
   }
 
   void addRegOperands(MCInst &Inst, unsigned N) const {
     assert(N == 1 && "Invalid number of operands!");
-    Inst.addOperand(MCOperand::CreateReg(getReg()));
+    Inst.addOperand(MCOperand::createReg(getReg()));
   }
 
   void addRegShiftedRegOperands(MCInst &Inst, unsigned N) const {
     assert(N == 3 && "Invalid number of operands!");
     assert(isRegShiftedReg() &&
            "addRegShiftedRegOperands() on non-RegShiftedReg!");
-    Inst.addOperand(MCOperand::CreateReg(RegShiftedReg.SrcReg));
-    Inst.addOperand(MCOperand::CreateReg(RegShiftedReg.ShiftReg));
-    Inst.addOperand(MCOperand::CreateImm(
+    Inst.addOperand(MCOperand::createReg(RegShiftedReg.SrcReg));
+    Inst.addOperand(MCOperand::createReg(RegShiftedReg.ShiftReg));
+    Inst.addOperand(MCOperand::createImm(
       ARM_AM::getSORegOpc(RegShiftedReg.ShiftTy, RegShiftedReg.ShiftImm)));
   }
 
@@ -1813,16 +1815,16 @@ class ARMOperand : public MCParsedAsmOperand {
     assert(N == 2 && "Invalid number of operands!");
     assert(isRegShiftedImm() &&
            "addRegShiftedImmOperands() on non-RegShiftedImm!");
-    Inst.addOperand(MCOperand::CreateReg(RegShiftedImm.SrcReg));
+    Inst.addOperand(MCOperand::createReg(RegShiftedImm.SrcReg));
     // Shift of #32 is encoded as 0 where permitted
     unsigned Imm = (RegShiftedImm.ShiftImm == 32 ? 0 : RegShiftedImm.ShiftImm);
-    Inst.addOperand(MCOperand::CreateImm(
+    Inst.addOperand(MCOperand::createImm(
       ARM_AM::getSORegOpc(RegShiftedImm.ShiftTy, Imm)));
   }
 
   void addShifterImmOperands(MCInst &Inst, unsigned N) const {
     assert(N == 1 && "Invalid number of operands!");
-    Inst.addOperand(MCOperand::CreateImm((ShifterImm.isASR << 5) |
+    Inst.addOperand(MCOperand::createImm((ShifterImm.isASR << 5) |
                                          ShifterImm.Imm));
   }
 
@@ -1831,7 +1833,7 @@ class ARMOperand : public MCParsedAsmOperand {
     const SmallVectorImpl<unsigned> &RegList = getRegList();
     for (SmallVectorImpl<unsigned>::const_iterator
            I = RegList.begin(), E = RegList.end(); I != E; ++I)
-      Inst.addOperand(MCOperand::CreateReg(*I));
+      Inst.addOperand(MCOperand::createReg(*I));
   }
 
   void addDPRRegListOperands(MCInst &Inst, unsigned N) const {
@@ -1845,7 +1847,7 @@ class ARMOperand : public MCParsedAsmOperand {
   void addRotImmOperands(MCInst &Inst, unsigned N) const {
     assert(N == 1 && "Invalid number of operands!");
     // Encoded as val>>3. The printer handles display as 8, 16, 24.
-    Inst.addOperand(MCOperand::CreateImm(RotImm.Imm >> 3));
+    Inst.addOperand(MCOperand::createImm(RotImm.Imm >> 3));
   }
 
   void addModImmOperands(MCInst &Inst, unsigned N) const {
@@ -1855,21 +1857,21 @@ class ARMOperand : public MCParsedAsmOperand {
     if (isImm())
       return addImmOperands(Inst, N);
 
-    Inst.addOperand(MCOperand::CreateImm(ModImm.Bits | (ModImm.Rot << 7)));
+    Inst.addOperand(MCOperand::createImm(ModImm.Bits | (ModImm.Rot << 7)));
   }
 
   void addModImmNotOperands(MCInst &Inst, unsigned N) const {
     assert(N == 1 && "Invalid number of operands!");
     const MCConstantExpr *CE = dyn_cast<MCConstantExpr>(getImm());
     uint32_t Enc = ARM_AM::getSOImmVal(~CE->getValue());
-    Inst.addOperand(MCOperand::CreateImm(Enc));
+    Inst.addOperand(MCOperand::createImm(Enc));
   }
 
   void addModImmNegOperands(MCInst &Inst, unsigned N) const {
     assert(N == 1 && "Invalid number of operands!");
     const MCConstantExpr *CE = dyn_cast<MCConstantExpr>(getImm());
     uint32_t Enc = ARM_AM::getSOImmVal(-CE->getValue());
-    Inst.addOperand(MCOperand::CreateImm(Enc));
+    Inst.addOperand(MCOperand::createImm(Enc));
   }
 
   void addBitfieldOperands(MCInst &Inst, unsigned N) const {
@@ -1880,7 +1882,7 @@ class ARMOperand : public MCParsedAsmOperand {
     // Make a 32-bit mask w/ the referenced bits clear and all other bits set.
     uint32_t Mask = ~(((uint32_t)0xffffffff >> lsb) << (32 - width) >>
                       (32 - (lsb + width)));
-    Inst.addOperand(MCOperand::CreateImm(Mask));
+    Inst.addOperand(MCOperand::createImm(Mask));
   }
 
   void addImmOperands(MCInst &Inst, unsigned N) const {
@@ -1891,20 +1893,20 @@ class ARMOperand : public MCParsedAsmOperand {
   void addFBits16Operands(MCInst &Inst, unsigned N) const {
     assert(N == 1 && "Invalid number of operands!");
     const MCConstantExpr *CE = dyn_cast<MCConstantExpr>(getImm());
-    Inst.addOperand(MCOperand::CreateImm(16 - CE->getValue()));
+    Inst.addOperand(MCOperand::createImm(16 - CE->getValue()));
   }
 
   void addFBits32Operands(MCInst &Inst, unsigned N) const {
     assert(N == 1 && "Invalid number of operands!");
     const MCConstantExpr *CE = dyn_cast<MCConstantExpr>(getImm());
-    Inst.addOperand(MCOperand::CreateImm(32 - CE->getValue()));
+    Inst.addOperand(MCOperand::createImm(32 - CE->getValue()));
   }
 
   void addFPImmOperands(MCInst &Inst, unsigned N) const {
     assert(N == 1 && "Invalid number of operands!");
     const MCConstantExpr *CE = dyn_cast<MCConstantExpr>(getImm());
     int Val = ARM_AM::getFP32Imm(APInt(32, CE->getValue()));
-    Inst.addOperand(MCOperand::CreateImm(Val));
+    Inst.addOperand(MCOperand::createImm(Val));
   }
 
   void addImm8s4Operands(MCInst &Inst, unsigned N) const {
@@ -1912,7 +1914,7 @@ class ARMOperand : public MCParsedAsmOperand {
     // FIXME: We really want to scale the value here, but the LDRD/STRD
     // instruction don't encode operands that way yet.
     const MCConstantExpr *CE = dyn_cast<MCConstantExpr>(getImm());
-    Inst.addOperand(MCOperand::CreateImm(CE->getValue()));
+    Inst.addOperand(MCOperand::createImm(CE->getValue()));
   }
 
   void addImm0_1020s4Operands(MCInst &Inst, unsigned N) const {
@@ -1920,7 +1922,7 @@ class ARMOperand : public MCParsedAsmOperand {
     // The immediate is scaled by four in the encoding and is stored
     // in the MCInst as such. Lop off the low two bits here.
     const MCConstantExpr *CE = dyn_cast<MCConstantExpr>(getImm());
-    Inst.addOperand(MCOperand::CreateImm(CE->getValue() / 4));
+    Inst.addOperand(MCOperand::createImm(CE->getValue() / 4));
   }
 
   void addImm0_508s4NegOperands(MCInst &Inst, unsigned N) const {
@@ -1928,7 +1930,7 @@ class ARMOperand : public MCParsedAsmOperand {
     // The immediate is scaled by four in the encoding and is stored
     // in the MCInst as such. Lop off the low two bits here.
     const MCConstantExpr *CE = dyn_cast<MCConstantExpr>(getImm());
-    Inst.addOperand(MCOperand::CreateImm(-(CE->getValue() / 4)));
+    Inst.addOperand(MCOperand::createImm(-(CE->getValue() / 4)));
   }
 
   void addImm0_508s4Operands(MCInst &Inst, unsigned N) const {
@@ -1936,7 +1938,7 @@ class ARMOperand : public MCParsedAsmOperand {
     // The immediate is scaled by four in the encoding and is stored
     // in the MCInst as such. Lop off the low two bits here.
     const MCConstantExpr *CE = dyn_cast<MCConstantExpr>(getImm());
-    Inst.addOperand(MCOperand::CreateImm(CE->getValue() / 4));
+    Inst.addOperand(MCOperand::createImm(CE->getValue() / 4));
   }
 
   void addImm1_16Operands(MCInst &Inst, unsigned N) const {
@@ -1944,7 +1946,7 @@ class ARMOperand : public MCParsedAsmOperand {
     // The constant encodes as the immediate-1, and we store in the instruction
     // the bits as encoded, so subtract off one here.
     const MCConstantExpr *CE = dyn_cast<MCConstantExpr>(getImm());
-    Inst.addOperand(MCOperand::CreateImm(CE->getValue() - 1));
+    Inst.addOperand(MCOperand::createImm(CE->getValue() - 1));
   }
 
   void addImm1_32Operands(MCInst &Inst, unsigned N) const {
@@ -1952,7 +1954,7 @@ class ARMOperand : public MCParsedAsmOperand {
     // The constant encodes as the immediate-1, and we store in the instruction
     // the bits as encoded, so subtract off one here.
     const MCConstantExpr *CE = dyn_cast<MCConstantExpr>(getImm());
-    Inst.addOperand(MCOperand::CreateImm(CE->getValue() - 1));
+    Inst.addOperand(MCOperand::createImm(CE->getValue() - 1));
   }
 
   void addImmThumbSROperands(MCInst &Inst, unsigned N) const {
@@ -1961,7 +1963,7 @@ class ARMOperand : public MCParsedAsmOperand {
     // zero.
     const MCConstantExpr *CE = dyn_cast<MCConstantExpr>(getImm());
     unsigned Imm = CE->getValue();
-    Inst.addOperand(MCOperand::CreateImm((Imm == 32 ? 0 : Imm)));
+    Inst.addOperand(MCOperand::createImm((Imm == 32 ? 0 : Imm)));
   }
 
   void addPKHASRImmOperands(MCInst &Inst, unsigned N) const {
@@ -1970,7 +1972,7 @@ class ARMOperand : public MCParsedAsmOperand {
     // the instruction as well.
     const MCConstantExpr *CE = dyn_cast<MCConstantExpr>(getImm());
     int Val = CE->getValue();
-    Inst.addOperand(MCOperand::CreateImm(Val == 32 ? 0 : Val));
+    Inst.addOperand(MCOperand::createImm(Val == 32 ? 0 : Val));
   }
 
   void addT2SOImmNotOperands(MCInst &Inst, unsigned N) const {
@@ -1978,7 +1980,7 @@ class ARMOperand : public MCParsedAsmOperand {
     // The operand is actually a t2_so_imm, but we have its bitwise
     // negation in the assembly source, so twiddle it here.
     const MCConstantExpr *CE = dyn_cast<MCConstantExpr>(getImm());
-    Inst.addOperand(MCOperand::CreateImm(~CE->getValue()));
+    Inst.addOperand(MCOperand::createImm(~CE->getValue()));
   }
 
   void addT2SOImmNegOperands(MCInst &Inst, unsigned N) const {
@@ -1986,7 +1988,7 @@ class ARMOperand : public MCParsedAsmOperand {
     // The operand is actually a t2_so_imm, but we have its
     // negation in the assembly source, so twiddle it here.
     const MCConstantExpr *CE = dyn_cast<MCConstantExpr>(getImm());
-    Inst.addOperand(MCOperand::CreateImm(-CE->getValue()));
+    Inst.addOperand(MCOperand::createImm(-CE->getValue()));
   }
 
   void addImm0_4095NegOperands(MCInst &Inst, unsigned N) const {
@@ -1994,18 +1996,18 @@ class ARMOperand : public MCParsedAsmOperand {
     // The operand is actually an imm0_4095, but we have its
     // negation in the assembly source, so twiddle it here.
     const MCConstantExpr *CE = dyn_cast<MCConstantExpr>(getImm());
-    Inst.addOperand(MCOperand::CreateImm(-CE->getValue()));
+    Inst.addOperand(MCOperand::createImm(-CE->getValue()));
   }
 
   void addUnsignedOffset_b8s2Operands(MCInst &Inst, unsigned N) const {
     if(const MCConstantExpr *CE = dyn_cast<MCConstantExpr>(getImm())) {
-      Inst.addOperand(MCOperand::CreateImm(CE->getValue() >> 2));
+      Inst.addOperand(MCOperand::createImm(CE->getValue() >> 2));
       return;
     }
 
     const MCSymbolRefExpr *SR = dyn_cast<MCSymbolRefExpr>(Imm.Val);
     assert(SR && "Unknown value type!");
-    Inst.addOperand(MCOperand::CreateExpr(SR));
+    Inst.addOperand(MCOperand::createExpr(SR));
   }
 
   void addThumbMemPCOperands(MCInst &Inst, unsigned N) const {
@@ -2013,40 +2015,40 @@ class ARMOperand : public MCParsedAsmOperand {
     if (isImm()) {
       const MCConstantExpr *CE = dyn_cast<MCConstantExpr>(getImm());
       if (CE) {
-        Inst.addOperand(MCOperand::CreateImm(CE->getValue()));
+        Inst.addOperand(MCOperand::createImm(CE->getValue()));
         return;
       }
 
       const MCSymbolRefExpr *SR = dyn_cast<MCSymbolRefExpr>(Imm.Val);
       assert(SR && "Unknown value type!");
-      Inst.addOperand(MCOperand::CreateExpr(SR));
+      Inst.addOperand(MCOperand::createExpr(SR));
       return;
     }
 
     assert(isMem()  && "Unknown value type!");
     assert(isa<MCConstantExpr>(Memory.OffsetImm) && "Unknown value type!");
-    Inst.addOperand(MCOperand::CreateImm(Memory.OffsetImm->getValue()));
+    Inst.addOperand(MCOperand::createImm(Memory.OffsetImm->getValue()));
   }
 
   void addMemBarrierOptOperands(MCInst &Inst, unsigned N) const {
     assert(N == 1 && "Invalid number of operands!");
-    Inst.addOperand(MCOperand::CreateImm(unsigned(getMemBarrierOpt())));
+    Inst.addOperand(MCOperand::createImm(unsigned(getMemBarrierOpt())));
   }
 
   void addInstSyncBarrierOptOperands(MCInst &Inst, unsigned N) const {
     assert(N == 1 && "Invalid number of operands!");
-    Inst.addOperand(MCOperand::CreateImm(unsigned(getInstSyncBarrierOpt())));
+    Inst.addOperand(MCOperand::createImm(unsigned(getInstSyncBarrierOpt())));
   }
 
   void addMemNoOffsetOperands(MCInst &Inst, unsigned N) const {
     assert(N == 1 && "Invalid number of operands!");
-    Inst.addOperand(MCOperand::CreateReg(Memory.BaseRegNum));
+    Inst.addOperand(MCOperand::createReg(Memory.BaseRegNum));
   }
 
   void addMemPCRelImm12Operands(MCInst &Inst, unsigned N) const {
     assert(N == 1 && "Invalid number of operands!");
     int32_t Imm = Memory.OffsetImm->getValue();
-    Inst.addOperand(MCOperand::CreateImm(Imm));
+    Inst.addOperand(MCOperand::createImm(Imm));
   }
 
   void addAdrLabelOperands(MCInst &Inst, unsigned N) const {
@@ -2056,19 +2058,19 @@ class ARMOperand : public MCParsedAsmOperand {
     // If we have an immediate that's not a constant, treat it as a label
     // reference needing a fixup. 
     if (!isa<MCConstantExpr>(getImm())) {
-      Inst.addOperand(MCOperand::CreateExpr(getImm()));
+      Inst.addOperand(MCOperand::createExpr(getImm()));
       return;
     }
 
     const MCConstantExpr *CE = dyn_cast<MCConstantExpr>(getImm());
     int Val = CE->getValue();
-    Inst.addOperand(MCOperand::CreateImm(Val));
+    Inst.addOperand(MCOperand::createImm(Val));
   }
 
   void addAlignedMemoryOperands(MCInst &Inst, unsigned N) const {
     assert(N == 2 && "Invalid number of operands!");
-    Inst.addOperand(MCOperand::CreateReg(Memory.BaseRegNum));
-    Inst.addOperand(MCOperand::CreateImm(Memory.Alignment));
+    Inst.addOperand(MCOperand::createReg(Memory.BaseRegNum));
+    Inst.addOperand(MCOperand::createImm(Memory.Alignment));
   }
 
   void addDupAlignedMemoryNoneOperands(MCInst &Inst, unsigned N) const {
@@ -2130,9 +2132,9 @@ class ARMOperand : public MCParsedAsmOperand {
       Val = ARM_AM::getAM2Opc(Memory.isNegative ? ARM_AM::sub : ARM_AM::add,
                               Memory.ShiftImm, Memory.ShiftType);
     }
-    Inst.addOperand(MCOperand::CreateReg(Memory.BaseRegNum));
-    Inst.addOperand(MCOperand::CreateReg(Memory.OffsetRegNum));
-    Inst.addOperand(MCOperand::CreateImm(Val));
+    Inst.addOperand(MCOperand::createReg(Memory.BaseRegNum));
+    Inst.addOperand(MCOperand::createReg(Memory.OffsetRegNum));
+    Inst.addOperand(MCOperand::createImm(Val));
   }
 
   void addAM2OffsetImmOperands(MCInst &Inst, unsigned N) const {
@@ -2145,8 +2147,8 @@ class ARMOperand : public MCParsedAsmOperand {
     if (Val == INT32_MIN) Val = 0;
     if (Val < 0) Val = -Val;
     Val = ARM_AM::getAM2Opc(AddSub, Val, ARM_AM::no_shift);
-    Inst.addOperand(MCOperand::CreateReg(0));
-    Inst.addOperand(MCOperand::CreateImm(Val));
+    Inst.addOperand(MCOperand::createReg(0));
+    Inst.addOperand(MCOperand::createImm(Val));
   }
 
   void addAddrMode3Operands(MCInst &Inst, unsigned N) const {
@@ -2155,9 +2157,9 @@ class ARMOperand : public MCParsedAsmOperand {
     // reference needing a fixup. If it is a constant, it's something else
     // and we reject it.
     if (isImm()) {
-      Inst.addOperand(MCOperand::CreateExpr(getImm()));
-      Inst.addOperand(MCOperand::CreateReg(0));
-      Inst.addOperand(MCOperand::CreateImm(0));
+      Inst.addOperand(MCOperand::createExpr(getImm()));
+      Inst.addOperand(MCOperand::createReg(0));
+      Inst.addOperand(MCOperand::createImm(0));
       return;
     }
 
@@ -2173,9 +2175,9 @@ class ARMOperand : public MCParsedAsmOperand {
       // here.
       Val = ARM_AM::getAM3Opc(Memory.isNegative ? ARM_AM::sub : ARM_AM::add, 0);
     }
-    Inst.addOperand(MCOperand::CreateReg(Memory.BaseRegNum));
-    Inst.addOperand(MCOperand::CreateReg(Memory.OffsetRegNum));
-    Inst.addOperand(MCOperand::CreateImm(Val));
+    Inst.addOperand(MCOperand::createReg(Memory.BaseRegNum));
+    Inst.addOperand(MCOperand::createReg(Memory.OffsetRegNum));
+    Inst.addOperand(MCOperand::createImm(Val));
   }
 
   void addAM3OffsetOperands(MCInst &Inst, unsigned N) const {
@@ -2183,8 +2185,8 @@ class ARMOperand : public MCParsedAsmOperand {
     if (Kind == k_PostIndexRegister) {
       int32_t Val =
         ARM_AM::getAM3Opc(PostIdxReg.isAdd ? ARM_AM::add : ARM_AM::sub, 0);
-      Inst.addOperand(MCOperand::CreateReg(PostIdxReg.RegNum));
-      Inst.addOperand(MCOperand::CreateImm(Val));
+      Inst.addOperand(MCOperand::createReg(PostIdxReg.RegNum));
+      Inst.addOperand(MCOperand::createImm(Val));
       return;
     }
 
@@ -2196,8 +2198,8 @@ class ARMOperand : public MCParsedAsmOperand {
     if (Val == INT32_MIN) Val = 0;
     if (Val < 0) Val = -Val;
     Val = ARM_AM::getAM3Opc(AddSub, Val);
-    Inst.addOperand(MCOperand::CreateReg(0));
-    Inst.addOperand(MCOperand::CreateImm(Val));
+    Inst.addOperand(MCOperand::createReg(0));
+    Inst.addOperand(MCOperand::createImm(Val));
   }
 
   void addAddrMode5Operands(MCInst &Inst, unsigned N) const {
@@ -2206,8 +2208,8 @@ class ARMOperand : public MCParsedAsmOperand {
     // reference needing a fixup. If it is a constant, it's something else
     // and we reject it.
     if (isImm()) {
-      Inst.addOperand(MCOperand::CreateExpr(getImm()));
-      Inst.addOperand(MCOperand::CreateImm(0));
+      Inst.addOperand(MCOperand::createExpr(getImm()));
+      Inst.addOperand(MCOperand::createImm(0));
       return;
     }
 
@@ -2218,8 +2220,8 @@ class ARMOperand : public MCParsedAsmOperand {
     if (Val == INT32_MIN) Val = 0;
     if (Val < 0) Val = -Val;
     Val = ARM_AM::getAM5Opc(AddSub, Val);
-    Inst.addOperand(MCOperand::CreateReg(Memory.BaseRegNum));
-    Inst.addOperand(MCOperand::CreateImm(Val));
+    Inst.addOperand(MCOperand::createReg(Memory.BaseRegNum));
+    Inst.addOperand(MCOperand::createImm(Val));
   }
 
   void addMemImm8s4OffsetOperands(MCInst &Inst, unsigned N) const {
@@ -2228,29 +2230,29 @@ class ARMOperand : public MCParsedAsmOperand {
     // reference needing a fixup. If it is a constant, it's something else
     // and we reject it.
     if (isImm()) {
-      Inst.addOperand(MCOperand::CreateExpr(getImm()));
-      Inst.addOperand(MCOperand::CreateImm(0));
+      Inst.addOperand(MCOperand::createExpr(getImm()));
+      Inst.addOperand(MCOperand::createImm(0));
       return;
     }
 
     int64_t Val = Memory.OffsetImm ? Memory.OffsetImm->getValue() : 0;
-    Inst.addOperand(MCOperand::CreateReg(Memory.BaseRegNum));
-    Inst.addOperand(MCOperand::CreateImm(Val));
+    Inst.addOperand(MCOperand::createReg(Memory.BaseRegNum));
+    Inst.addOperand(MCOperand::createImm(Val));
   }
 
   void addMemImm0_1020s4OffsetOperands(MCInst &Inst, unsigned N) const {
     assert(N == 2 && "Invalid number of operands!");
     // The lower two bits are always zero and as such are not encoded.
     int32_t Val = Memory.OffsetImm ? Memory.OffsetImm->getValue() / 4 : 0;
-    Inst.addOperand(MCOperand::CreateReg(Memory.BaseRegNum));
-    Inst.addOperand(MCOperand::CreateImm(Val));
+    Inst.addOperand(MCOperand::createReg(Memory.BaseRegNum));
+    Inst.addOperand(MCOperand::createImm(Val));
   }
 
   void addMemImm8OffsetOperands(MCInst &Inst, unsigned N) const {
     assert(N == 2 && "Invalid number of operands!");
     int64_t Val = Memory.OffsetImm ? Memory.OffsetImm->getValue() : 0;
-    Inst.addOperand(MCOperand::CreateReg(Memory.BaseRegNum));
-    Inst.addOperand(MCOperand::CreateImm(Val));
+    Inst.addOperand(MCOperand::createReg(Memory.BaseRegNum));
+    Inst.addOperand(MCOperand::createImm(Val));
   }
 
   void addMemPosImm8OffsetOperands(MCInst &Inst, unsigned N) const {
@@ -2266,14 +2268,14 @@ class ARMOperand : public MCParsedAsmOperand {
     // If this is an immediate, it's a label reference.
     if (isImm()) {
       addExpr(Inst, getImm());
-      Inst.addOperand(MCOperand::CreateImm(0));
+      Inst.addOperand(MCOperand::createImm(0));
       return;
     }
 
     // Otherwise, it's a normal memory reg+offset.
     int64_t Val = Memory.OffsetImm ? Memory.OffsetImm->getValue() : 0;
-    Inst.addOperand(MCOperand::CreateReg(Memory.BaseRegNum));
-    Inst.addOperand(MCOperand::CreateImm(Val));
+    Inst.addOperand(MCOperand::createReg(Memory.BaseRegNum));
+    Inst.addOperand(MCOperand::createImm(Val));
   }
 
   void addMemImm12OffsetOperands(MCInst &Inst, unsigned N) const {
@@ -2281,26 +2283,26 @@ class ARMOperand : public MCParsedAsmOperand {
     // If this is an immediate, it's a label reference.
     if (isImm()) {
       addExpr(Inst, getImm());
-      Inst.addOperand(MCOperand::CreateImm(0));
+      Inst.addOperand(MCOperand::createImm(0));
       return;
     }
 
     // Otherwise, it's a normal memory reg+offset.
     int64_t Val = Memory.OffsetImm ? Memory.OffsetImm->getValue() : 0;
-    Inst.addOperand(MCOperand::CreateReg(Memory.BaseRegNum));
-    Inst.addOperand(MCOperand::CreateImm(Val));
+    Inst.addOperand(MCOperand::createReg(Memory.BaseRegNum));
+    Inst.addOperand(MCOperand::createImm(Val));
   }
 
   void addMemTBBOperands(MCInst &Inst, unsigned N) const {
     assert(N == 2 && "Invalid number of operands!");
-    Inst.addOperand(MCOperand::CreateReg(Memory.BaseRegNum));
-    Inst.addOperand(MCOperand::CreateReg(Memory.OffsetRegNum));
+    Inst.addOperand(MCOperand::createReg(Memory.BaseRegNum));
+    Inst.addOperand(MCOperand::createReg(Memory.OffsetRegNum));
   }
 
   void addMemTBHOperands(MCInst &Inst, unsigned N) const {
     assert(N == 2 && "Invalid number of operands!");
-    Inst.addOperand(MCOperand::CreateReg(Memory.BaseRegNum));
-    Inst.addOperand(MCOperand::CreateReg(Memory.OffsetRegNum));
+    Inst.addOperand(MCOperand::createReg(Memory.BaseRegNum));
+    Inst.addOperand(MCOperand::createReg(Memory.OffsetRegNum));
   }
 
   void addMemRegOffsetOperands(MCInst &Inst, unsigned N) const {
@@ -2308,50 +2310,50 @@ class ARMOperand : public MCParsedAsmOperand {
     unsigned Val =
       ARM_AM::getAM2Opc(Memory.isNegative ? ARM_AM::sub : ARM_AM::add,
                         Memory.ShiftImm, Memory.ShiftType);
-    Inst.addOperand(MCOperand::CreateReg(Memory.BaseRegNum));
-    Inst.addOperand(MCOperand::CreateReg(Memory.OffsetRegNum));
-    Inst.addOperand(MCOperand::CreateImm(Val));
+    Inst.addOperand(MCOperand::createReg(Memory.BaseRegNum));
+    Inst.addOperand(MCOperand::createReg(Memory.OffsetRegNum));
+    Inst.addOperand(MCOperand::createImm(Val));
   }
 
   void addT2MemRegOffsetOperands(MCInst &Inst, unsigned N) const {
     assert(N == 3 && "Invalid number of operands!");
-    Inst.addOperand(MCOperand::CreateReg(Memory.BaseRegNum));
-    Inst.addOperand(MCOperand::CreateReg(Memory.OffsetRegNum));
-    Inst.addOperand(MCOperand::CreateImm(Memory.ShiftImm));
+    Inst.addOperand(MCOperand::createReg(Memory.BaseRegNum));
+    Inst.addOperand(MCOperand::createReg(Memory.OffsetRegNum));
+    Inst.addOperand(MCOperand::createImm(Memory.ShiftImm));
   }
 
   void addMemThumbRROperands(MCInst &Inst, unsigned N) const {
     assert(N == 2 && "Invalid number of operands!");
-    Inst.addOperand(MCOperand::CreateReg(Memory.BaseRegNum));
-    Inst.addOperand(MCOperand::CreateReg(Memory.OffsetRegNum));
+    Inst.addOperand(MCOperand::createReg(Memory.BaseRegNum));
+    Inst.addOperand(MCOperand::createReg(Memory.OffsetRegNum));
   }
 
   void addMemThumbRIs4Operands(MCInst &Inst, unsigned N) const {
     assert(N == 2 && "Invalid number of operands!");
     int64_t Val = Memory.OffsetImm ? (Memory.OffsetImm->getValue() / 4) : 0;
-    Inst.addOperand(MCOperand::CreateReg(Memory.BaseRegNum));
-    Inst.addOperand(MCOperand::CreateImm(Val));
+    Inst.addOperand(MCOperand::createReg(Memory.BaseRegNum));
+    Inst.addOperand(MCOperand::createImm(Val));
   }
 
   void addMemThumbRIs2Operands(MCInst &Inst, unsigned N) const {
     assert(N == 2 && "Invalid number of operands!");
     int64_t Val = Memory.OffsetImm ? (Memory.OffsetImm->getValue() / 2) : 0;
-    Inst.addOperand(MCOperand::CreateReg(Memory.BaseRegNum));
-    Inst.addOperand(MCOperand::CreateImm(Val));
+    Inst.addOperand(MCOperand::createReg(Memory.BaseRegNum));
+    Inst.addOperand(MCOperand::createImm(Val));
   }
 
   void addMemThumbRIs1Operands(MCInst &Inst, unsigned N) const {
     assert(N == 2 && "Invalid number of operands!");
     int64_t Val = Memory.OffsetImm ? (Memory.OffsetImm->getValue()) : 0;
-    Inst.addOperand(MCOperand::CreateReg(Memory.BaseRegNum));
-    Inst.addOperand(MCOperand::CreateImm(Val));
+    Inst.addOperand(MCOperand::createReg(Memory.BaseRegNum));
+    Inst.addOperand(MCOperand::createImm(Val));
   }
 
   void addMemThumbSPIOperands(MCInst &Inst, unsigned N) const {
     assert(N == 2 && "Invalid number of operands!");
     int64_t Val = Memory.OffsetImm ? (Memory.OffsetImm->getValue() / 4) : 0;
-    Inst.addOperand(MCOperand::CreateReg(Memory.BaseRegNum));
-    Inst.addOperand(MCOperand::CreateImm(Val));
+    Inst.addOperand(MCOperand::createReg(Memory.BaseRegNum));
+    Inst.addOperand(MCOperand::createImm(Val));
   }
 
   void addPostIdxImm8Operands(MCInst &Inst, unsigned N) const {
@@ -2362,7 +2364,7 @@ class ARMOperand : public MCParsedAsmOperand {
     bool isAdd = Imm >= 0;
     if (Imm == INT32_MIN) Imm = 0;
     Imm = (Imm < 0 ? -Imm : Imm) | (int)isAdd << 8;
-    Inst.addOperand(MCOperand::CreateImm(Imm));
+    Inst.addOperand(MCOperand::createImm(Imm));
   }
 
   void addPostIdxImm8s4Operands(MCInst &Inst, unsigned N) const {
@@ -2374,65 +2376,65 @@ class ARMOperand : public MCParsedAsmOperand {
     if (Imm == INT32_MIN) Imm = 0;
     // Immediate is scaled by 4.
     Imm = ((Imm < 0 ? -Imm : Imm) / 4) | (int)isAdd << 8;
-    Inst.addOperand(MCOperand::CreateImm(Imm));
+    Inst.addOperand(MCOperand::createImm(Imm));
   }
 
   void addPostIdxRegOperands(MCInst &Inst, unsigned N) const {
     assert(N == 2 && "Invalid number of operands!");
-    Inst.addOperand(MCOperand::CreateReg(PostIdxReg.RegNum));
-    Inst.addOperand(MCOperand::CreateImm(PostIdxReg.isAdd));
+    Inst.addOperand(MCOperand::createReg(PostIdxReg.RegNum));
+    Inst.addOperand(MCOperand::createImm(PostIdxReg.isAdd));
   }
 
   void addPostIdxRegShiftedOperands(MCInst &Inst, unsigned N) const {
     assert(N == 2 && "Invalid number of operands!");
-    Inst.addOperand(MCOperand::CreateReg(PostIdxReg.RegNum));
+    Inst.addOperand(MCOperand::createReg(PostIdxReg.RegNum));
     // The sign, shift type, and shift amount are encoded in a single operand
     // using the AM2 encoding helpers.
     ARM_AM::AddrOpc opc = PostIdxReg.isAdd ? ARM_AM::add : ARM_AM::sub;
     unsigned Imm = ARM_AM::getAM2Opc(opc, PostIdxReg.ShiftImm,
                                      PostIdxReg.ShiftTy);
-    Inst.addOperand(MCOperand::CreateImm(Imm));
+    Inst.addOperand(MCOperand::createImm(Imm));
   }
 
   void addMSRMaskOperands(MCInst &Inst, unsigned N) const {
     assert(N == 1 && "Invalid number of operands!");
-    Inst.addOperand(MCOperand::CreateImm(unsigned(getMSRMask())));
+    Inst.addOperand(MCOperand::createImm(unsigned(getMSRMask())));
   }
 
   void addBankedRegOperands(MCInst &Inst, unsigned N) const {
     assert(N == 1 && "Invalid number of operands!");
-    Inst.addOperand(MCOperand::CreateImm(unsigned(getBankedReg())));
+    Inst.addOperand(MCOperand::createImm(unsigned(getBankedReg())));
   }
 
   void addProcIFlagsOperands(MCInst &Inst, unsigned N) const {
     assert(N == 1 && "Invalid number of operands!");
-    Inst.addOperand(MCOperand::CreateImm(unsigned(getProcIFlags())));
+    Inst.addOperand(MCOperand::createImm(unsigned(getProcIFlags())));
   }
 
   void addVecListOperands(MCInst &Inst, unsigned N) const {
     assert(N == 1 && "Invalid number of operands!");
-    Inst.addOperand(MCOperand::CreateReg(VectorList.RegNum));
+    Inst.addOperand(MCOperand::createReg(VectorList.RegNum));
   }
 
   void addVecListIndexedOperands(MCInst &Inst, unsigned N) const {
     assert(N == 2 && "Invalid number of operands!");
-    Inst.addOperand(MCOperand::CreateReg(VectorList.RegNum));
-    Inst.addOperand(MCOperand::CreateImm(VectorList.LaneIndex));
+    Inst.addOperand(MCOperand::createReg(VectorList.RegNum));
+    Inst.addOperand(MCOperand::createImm(VectorList.LaneIndex));
   }
 
   void addVectorIndex8Operands(MCInst &Inst, unsigned N) const {
     assert(N == 1 && "Invalid number of operands!");
-    Inst.addOperand(MCOperand::CreateImm(getVectorIndex()));
+    Inst.addOperand(MCOperand::createImm(getVectorIndex()));
   }
 
   void addVectorIndex16Operands(MCInst &Inst, unsigned N) const {
     assert(N == 1 && "Invalid number of operands!");
-    Inst.addOperand(MCOperand::CreateImm(getVectorIndex()));
+    Inst.addOperand(MCOperand::createImm(getVectorIndex()));
   }
 
   void addVectorIndex32Operands(MCInst &Inst, unsigned N) const {
     assert(N == 1 && "Invalid number of operands!");
-    Inst.addOperand(MCOperand::CreateImm(getVectorIndex()));
+    Inst.addOperand(MCOperand::createImm(getVectorIndex()));
   }
 
   void addNEONi8splatOperands(MCInst &Inst, unsigned N) const {
@@ -2440,7 +2442,7 @@ class ARMOperand : public MCParsedAsmOperand {
     // The immediate encodes the type of constant as well as the value.
     // Mask in that this is an i8 splat.
     const MCConstantExpr *CE = dyn_cast<MCConstantExpr>(getImm());
-    Inst.addOperand(MCOperand::CreateImm(CE->getValue() | 0xe00));
+    Inst.addOperand(MCOperand::createImm(CE->getValue() | 0xe00));
   }
 
   void addNEONi16splatOperands(MCInst &Inst, unsigned N) const {
@@ -2449,7 +2451,7 @@ class ARMOperand : public MCParsedAsmOperand {
     const MCConstantExpr *CE = dyn_cast<MCConstantExpr>(getImm());
     unsigned Value = CE->getValue();
     Value = ARM_AM::encodeNEONi16splat(Value);
-    Inst.addOperand(MCOperand::CreateImm(Value));
+    Inst.addOperand(MCOperand::createImm(Value));
   }
 
   void addNEONi16splatNotOperands(MCInst &Inst, unsigned N) const {
@@ -2458,7 +2460,7 @@ class ARMOperand : public MCParsedAsmOperand {
     const MCConstantExpr *CE = dyn_cast<MCConstantExpr>(getImm());
     unsigned Value = CE->getValue();
     Value = ARM_AM::encodeNEONi16splat(~Value & 0xffff);
-    Inst.addOperand(MCOperand::CreateImm(Value));
+    Inst.addOperand(MCOperand::createImm(Value));
   }
 
   void addNEONi32splatOperands(MCInst &Inst, unsigned N) const {
@@ -2467,7 +2469,7 @@ class ARMOperand : public MCParsedAsmOperand {
     const MCConstantExpr *CE = dyn_cast<MCConstantExpr>(getImm());
     unsigned Value = CE->getValue();
     Value = ARM_AM::encodeNEONi32splat(Value);
-    Inst.addOperand(MCOperand::CreateImm(Value));
+    Inst.addOperand(MCOperand::createImm(Value));
   }
 
   void addNEONi32splatNotOperands(MCInst &Inst, unsigned N) const {
@@ -2476,7 +2478,7 @@ class ARMOperand : public MCParsedAsmOperand {
     const MCConstantExpr *CE = dyn_cast<MCConstantExpr>(getImm());
     unsigned Value = CE->getValue();
     Value = ARM_AM::encodeNEONi32splat(~Value);
-    Inst.addOperand(MCOperand::CreateImm(Value));
+    Inst.addOperand(MCOperand::createImm(Value));
   }
 
   void addNEONinvByteReplicateOperands(MCInst &Inst, unsigned N) const {
@@ -2490,7 +2492,7 @@ class ARMOperand : public MCParsedAsmOperand {
            "always must be replaced with VMOVv8i8 or VMOVv16i8.");
     unsigned B = ((~Value) & 0xff);
     B |= 0xe00; // cmode = 0b1110
-    Inst.addOperand(MCOperand::CreateImm(B));
+    Inst.addOperand(MCOperand::createImm(B));
   }
   void addNEONi32vmovOperands(MCInst &Inst, unsigned N) const {
     assert(N == 1 && "Invalid number of operands!");
@@ -2503,7 +2505,7 @@ class ARMOperand : public MCParsedAsmOperand {
       Value = (Value >> 16) | ((Value & 0xff) ? 0xd00 : 0x400);
     else if (Value > 0xffffff)
       Value = (Value >> 24) | 0x600;
-    Inst.addOperand(MCOperand::CreateImm(Value));
+    Inst.addOperand(MCOperand::createImm(Value));
   }
 
   void addNEONvmovByteReplicateOperands(MCInst &Inst, unsigned N) const {
@@ -2517,7 +2519,7 @@ class ARMOperand : public MCParsedAsmOperand {
            "always must be replaced with VMOVv8i8 or VMOVv16i8.");
     unsigned B = Value & 0xff;
     B |= 0xe00; // cmode = 0b1110
-    Inst.addOperand(MCOperand::CreateImm(B));
+    Inst.addOperand(MCOperand::createImm(B));
   }
   void addNEONi32vmovNegOperands(MCInst &Inst, unsigned N) const {
     assert(N == 1 && "Invalid number of operands!");
@@ -2530,7 +2532,7 @@ class ARMOperand : public MCParsedAsmOperand {
       Value = (Value >> 16) | ((Value & 0xff) ? 0xd00 : 0x400);
     else if (Value > 0xffffff)
       Value = (Value >> 24) | 0x600;
-    Inst.addOperand(MCOperand::CreateImm(Value));
+    Inst.addOperand(MCOperand::createImm(Value));
   }
 
   void addNEONi64splatOperands(MCInst &Inst, unsigned N) const {
@@ -2542,7 +2544,7 @@ class ARMOperand : public MCParsedAsmOperand {
     for (unsigned i = 0; i < 8; ++i, Value >>= 8) {
       Imm |= (Value & 1) << i;
     }
-    Inst.addOperand(MCOperand::CreateImm(Imm | 0x1e00));
+    Inst.addOperand(MCOperand::createImm(Imm | 0x1e00));
   }
 
   void print(raw_ostream &OS) const override;
@@ -2871,7 +2873,7 @@ void ARMOperand::print(raw_ostream &OS) const {
     OS << "<banked reg: " << getBankedReg() << ">";
     break;
   case k_Immediate:
-    getImm()->print(OS);
+    OS << *getImm();
     break;
   case k_MemBarrierOpt:
     OS << "<ARM_MB::" << MemBOptToString(getMemBarrierOpt(), false) << ">";
@@ -4253,7 +4255,7 @@ ARMAsmParser::parseSetEndImm(OperandVector &Operands) {
     Error(S, "'be' or 'le' operand expected");
     return MatchOperand_ParseFail;
   }
-  Operands.push_back(ARMOperand::CreateImm(MCConstantExpr::Create(Val,
+  Operands.push_back(ARMOperand::CreateImm(MCConstantExpr::create(Val,
                                                                   getContext()),
                                            S, Tok.getEndLoc()));
   return MatchOperand_Success;
@@ -4657,7 +4659,7 @@ ARMAsmParser::parseAM3Offset(OperandVector &Operands) {
       Val = INT32_MIN;
 
     Operands.push_back(
-      ARMOperand::CreateImm(MCConstantExpr::Create(Val, getContext()), S, E));
+      ARMOperand::CreateImm(MCConstantExpr::create(Val, getContext()), S, E));
 
     return MatchOperand_Success;
   }
@@ -4887,7 +4889,7 @@ bool ARMAsmParser::parseMemory(OperandVector &Operands) {
     // If the constant was #-0, represent it as INT32_MIN.
     int32_t Val = CE->getValue();
     if (isNegative && Val == 0)
-      CE = MCConstantExpr::Create(INT32_MIN, getContext());
+      CE = MCConstantExpr::create(INT32_MIN, getContext());
 
     // Now we should have the closing ']'
     if (Parser.getTok().isNot(AsmToken::RBrac))
@@ -5074,7 +5076,7 @@ ARMAsmParser::parseFPImm(OperandVector &Operands) {
     IntVal ^= (uint64_t)isNegative << 31;
     Parser.Lex(); // Eat the token.
     Operands.push_back(ARMOperand::CreateImm(
-          MCConstantExpr::Create(IntVal, getContext()),
+          MCConstantExpr::create(IntVal, getContext()),
           S, Parser.getTok().getLoc()));
     return MatchOperand_Success;
   }
@@ -5091,7 +5093,7 @@ ARMAsmParser::parseFPImm(OperandVector &Operands) {
     Val = APFloat(RealVal).bitcastToAPInt().getZExtValue();
 
     Operands.push_back(ARMOperand::CreateImm(
-        MCConstantExpr::Create(Val, getContext()), S,
+        MCConstantExpr::create(Val, getContext()), S,
         Parser.getTok().getLoc()));
     return MatchOperand_Success;
   }
@@ -5180,7 +5182,7 @@ bool ARMAsmParser::parseOperand(OperandVector &Operands, StringRef Mnemonic) {
       if (CE) {
         int32_t Val = CE->getValue();
         if (isNegative && Val == 0)
-          ImmVal = MCConstantExpr::Create(INT32_MIN, getContext());
+          ImmVal = MCConstantExpr::create(INT32_MIN, getContext());
       }
       E = SMLoc::getFromPointer(Parser.getTok().getLoc().getPointer() - 1);
       Operands.push_back(ARMOperand::CreateImm(ImmVal, S, E));
@@ -5210,7 +5212,7 @@ bool ARMAsmParser::parseOperand(OperandVector &Operands, StringRef Mnemonic) {
     if (getParser().parseExpression(SubExprVal))
       return true;
 
-    const MCExpr *ExprVal = ARMMCExpr::Create(RefKind, SubExprVal,
+    const MCExpr *ExprVal = ARMMCExpr::create(RefKind, SubExprVal,
                                               getContext());
     E = SMLoc::getFromPointer(Parser.getTok().getLoc().getPointer() - 1);
     Operands.push_back(ARMOperand::CreateImm(ExprVal, S, E));
@@ -5465,6 +5467,92 @@ void ARMAsmParser::getMnemonicAcceptInfo(StringRef Mnemonic, StringRef FullInst,
     CanAcceptPredicationCode = true;
 }
 
+// \brief Some Thumb instructions have two operand forms that are not
+// available as three operand, convert to two operand form if possible.
+//
+// FIXME: We would really like to be able to tablegen'erate this.
+void ARMAsmParser::tryConvertingToTwoOperandForm(StringRef Mnemonic,
+                                                 bool CarrySetting,
+                                                 OperandVector &Operands) {
+  if (Operands.size() != 6)
+    return;
+
+  const auto &Op3 = static_cast<ARMOperand &>(*Operands[3]);
+        auto &Op4 = static_cast<ARMOperand &>(*Operands[4]);
+  if (!Op3.isReg() || !Op4.isReg())
+    return;
+
+  auto Op3Reg = Op3.getReg();
+  auto Op4Reg = Op4.getReg();
+
+  // For most Thumb2 cases we just generate the 3 operand form and reduce
+  // it in processInstruction(), but the 3 operand form of ADD (t2ADDrr)
+  // won't accept SP or PC so we do the transformation here taking care
+  // with immediate range in the 'add sp, sp #imm' case.
+  auto &Op5 = static_cast<ARMOperand &>(*Operands[5]);
+  if (isThumbTwo()) {
+    if (Mnemonic != "add")
+      return;
+    bool TryTransform = Op3Reg == ARM::PC || Op4Reg == ARM::PC ||
+                        (Op5.isReg() && Op5.getReg() == ARM::PC);
+    if (!TryTransform) {
+      TryTransform = (Op3Reg == ARM::SP || Op4Reg == ARM::SP ||
+                      (Op5.isReg() && Op5.getReg() == ARM::SP)) &&
+                     !(Op3Reg == ARM::SP && Op4Reg == ARM::SP &&
+                       Op5.isImm() && !Op5.isImm0_508s4());
+    }
+    if (!TryTransform)
+      return;
+  } else if (!isThumbOne())
+    return;
+
+  if (!(Mnemonic == "add" || Mnemonic == "sub" || Mnemonic == "and" ||
+        Mnemonic == "eor" || Mnemonic == "lsl" || Mnemonic == "lsr" ||
+        Mnemonic == "asr" || Mnemonic == "adc" || Mnemonic == "sbc" ||
+        Mnemonic == "ror" || Mnemonic == "orr" || Mnemonic == "bic"))
+    return;
+
+  // If first 2 operands of a 3 operand instruction are the same
+  // then transform to 2 operand version of the same instruction
+  // e.g. 'adds r0, r0, #1' transforms to 'adds r0, #1'
+  bool Transform = Op3Reg == Op4Reg;
+
+  // For communtative operations, we might be able to transform if we swap
+  // Op4 and Op5.  The 'ADD Rdm, SP, Rdm' form is already handled specially
+  // as tADDrsp.
+  const ARMOperand *LastOp = &Op5;
+  bool Swap = false;
+  if (!Transform && Op5.isReg() && Op3Reg == Op5.getReg() &&
+      ((Mnemonic == "add" && Op4Reg != ARM::SP) ||
+       Mnemonic == "and" || Mnemonic == "eor" ||
+       Mnemonic == "adc" || Mnemonic == "orr")) {
+    Swap = true;
+    LastOp = &Op4;
+    Transform = true;
+  }
+
+  // If both registers are the same then remove one of them from
+  // the operand list, with certain exceptions.
+  if (Transform) {
+    // Don't transform 'adds Rd, Rd, Rm' or 'sub{s} Rd, Rd, Rm' because the
+    // 2 operand forms don't exist.
+    if (((Mnemonic == "add" && CarrySetting) || Mnemonic == "sub") &&
+        LastOp->isReg())
+      Transform = false;
+
+    // Don't transform 'add/sub{s} Rd, Rd, #imm' if the immediate fits into
+    // 3-bits because the ARMARM says not to.
+    if ((Mnemonic == "add" || Mnemonic == "sub") && LastOp->isImm0_7())
+      Transform = false;
+  }
+
+  if (Transform) {
+    if (Swap)
+      std::swap(Op4, Op5);
+    Operands.erase(Operands.begin() + 3);
+  }
+}
+
 bool ARMAsmParser::shouldOmitCCOutOperand(StringRef Mnemonic,
                                           OperandVector &Operands) {
   // FIXME: This is all horribly hacky. We really need a better way to deal
@@ -5766,7 +5854,7 @@ bool ARMAsmParser::ParseInstruction(ParseInstructionInfo &Info, StringRef Name,
   // Add the processor imod operand, if necessary.
   if (ProcessorIMod) {
     Operands.push_back(ARMOperand::CreateImm(
-          MCConstantExpr::Create(ProcessorIMod, getContext()),
+          MCConstantExpr::create(ProcessorIMod, getContext()),
                                  NameLoc, NameLoc));
   } else if (Mnemonic == "cps" && isMClass()) {
     return Error(NameLoc, "instruction 'cps' requires effect for M-class");
@@ -5838,11 +5926,13 @@ bool ARMAsmParser::ParseInstruction(ParseInstructionInfo &Info, StringRef Name,
                    "VFP/Neon double precision register expected");
   }
 
+  tryConvertingToTwoOperandForm(Mnemonic, CarrySetting, Operands);
+
   // Some instructions, mostly Thumb, have forms for the same mnemonic that
   // do and don't have a cc_out optional-def operand. With some spot-checks
   // of the operand list, we can figure out which variant we're trying to
   // parse and adjust accordingly before actually matching. We shouldn't ever
-  // try to remove a cc_out operand that was explicitly set on the the
+  // try to remove a cc_out operand that was explicitly set on the
   // mnemonic, of course (CarrySetting == true). Reason number #317 the
   // table driven matcher doesn't fit well with the ARM instruction set.
   if (!CarrySetting && shouldOmitCCOutOperand(Mnemonic, Operands))
@@ -5901,48 +5991,6 @@ bool ARMAsmParser::ParseInstruction(ParseInstructionInfo &Info, StringRef Name,
     }
   }
 
-  // If first 2 operands of a 3 operand instruction are the same
-  // then transform to 2 operand version of the same instruction
-  // e.g. 'adds r0, r0, #1' transforms to 'adds r0, #1'
-  // FIXME: We would really like to be able to tablegen'erate this.
-  if (isThumbOne() && Operands.size() == 6 &&
-       (Mnemonic == "add" || Mnemonic == "sub" || Mnemonic == "and" ||
-        Mnemonic == "eor" || Mnemonic == "lsl" || Mnemonic == "lsr" ||
-        Mnemonic == "asr" || Mnemonic == "adc" || Mnemonic == "sbc" ||
-        Mnemonic == "ror" || Mnemonic == "orr" || Mnemonic == "bic")) {
-      ARMOperand &Op3 = static_cast<ARMOperand &>(*Operands[3]);
-      ARMOperand &Op4 = static_cast<ARMOperand &>(*Operands[4]);
-      ARMOperand &Op5 = static_cast<ARMOperand &>(*Operands[5]);
-
-      // If both registers are the same then remove one of them from
-      // the operand list.
-      if (Op3.isReg() && Op4.isReg() && Op3.getReg() == Op4.getReg()) {
-          // If 3rd operand (variable Op5) is a register and the instruction is adds/sub
-          // then do not transform as the backend already handles this instruction
-          // correctly.
-          if (!Op5.isReg() || !((Mnemonic == "add" && CarrySetting) || Mnemonic == "sub")) {
-              Operands.erase(Operands.begin() + 3);
-              if (Mnemonic == "add" && !CarrySetting) {
-                  // Special case for 'add' (not 'adds') instruction must
-                  // remove the CCOut operand as well.
-                  Operands.erase(Operands.begin() + 1);
-              }
-          }
-      }
-  }
-
-  // If instruction is 'add' and first two register operands
-  // use SP register, then remove one of the SP registers from
-  // the instruction.
-  // FIXME: We would really like to be able to tablegen'erate this.
-  if (isThumbOne() && Operands.size() == 5 && Mnemonic == "add" && !CarrySetting) {
-      ARMOperand &Op2 = static_cast<ARMOperand &>(*Operands[2]);
-      ARMOperand &Op3 = static_cast<ARMOperand &>(*Operands[3]);
-      if (Op2.isReg() && Op3.isReg() && Op2.getReg() == ARM::SP && Op3.getReg() == ARM::SP) {
-          Operands.erase(Operands.begin() + 2);
-      }
-  }
-
   // GNU Assembler extension (compatibility)
   if ((Mnemonic == "ldrd" || Mnemonic == "strd")) {
     ARMOperand &Op2 = static_cast<ARMOperand &>(*Operands[2]);
@@ -5985,8 +6033,9 @@ bool ARMAsmParser::ParseInstruction(ParseInstructionInfo &Info, StringRef Name,
 // return 'true' if register list contains non-low GPR registers,
 // 'false' otherwise. If Reg is in the register list or is HiReg, set
 // 'containsReg' to true.
-static bool checkLowRegisterList(MCInst Inst, unsigned OpNo, unsigned Reg,
-                                 unsigned HiReg, bool &containsReg) {
+static bool checkLowRegisterList(const MCInst &Inst, unsigned OpNo,
+                                 unsigned Reg, unsigned HiReg,
+                                 bool &containsReg) {
   containsReg = false;
   for (unsigned i = OpNo; i < Inst.getNumOperands(); ++i) {
     unsigned OpReg = Inst.getOperand(i).getReg();
@@ -6001,8 +6050,8 @@ static bool checkLowRegisterList(MCInst Inst, unsigned OpNo, unsigned Reg,
 
 // Check if the specified regisgter is in the register list of the inst,
 // starting at the indicated operand number.
-static bool listContainsReg(MCInst &Inst, unsigned OpNo, unsigned Reg) {
-  for (unsigned i = OpNo; i < Inst.getNumOperands(); ++i) {
+static bool listContainsReg(const MCInst &Inst, unsigned OpNo, unsigned Reg) {
+  for (unsigned i = OpNo, e = Inst.getNumOperands(); i < e; ++i) {
     unsigned OpReg = Inst.getOperand(i).getReg();
     if (OpReg == Reg)
       return true;
@@ -6020,7 +6069,7 @@ static bool instIsBreakpoint(const MCInst &Inst) {
 
 }
 
-bool ARMAsmParser::validatetLDMRegList(MCInst Inst,
+bool ARMAsmParser::validatetLDMRegList(const MCInst &Inst,
                                        const OperandVector &Operands,
                                        unsigned ListNo, bool IsARPop) {
   const ARMOperand &Op = static_cast<const ARMOperand &>(*Operands[ListNo]);
@@ -6043,7 +6092,7 @@ bool ARMAsmParser::validatetLDMRegList(MCInst Inst,
   return false;
 }
 
-bool ARMAsmParser::validatetSTMRegList(MCInst Inst,
+bool ARMAsmParser::validatetSTMRegList(const MCInst &Inst,
                                        const OperandVector &Operands,
                                        unsigned ListNo) {
   const ARMOperand &Op = static_cast<const ARMOperand &>(*Operands[ListNo]);
@@ -6706,8 +6755,8 @@ bool ARMAsmParser::processInstruction(MCInst &Inst,
     TmpInst.addOperand(Inst.getOperand(0));
     TmpInst.addOperand(Inst.getOperand(1));
     TmpInst.addOperand(Inst.getOperand(1));
-    TmpInst.addOperand(MCOperand::CreateReg(0));
-    TmpInst.addOperand(MCOperand::CreateImm(0));
+    TmpInst.addOperand(MCOperand::createReg(0));
+    TmpInst.addOperand(MCOperand::createImm(0));
     TmpInst.addOperand(Inst.getOperand(2));
     TmpInst.addOperand(Inst.getOperand(3));
     Inst = TmpInst;
@@ -6724,8 +6773,8 @@ bool ARMAsmParser::processInstruction(MCInst &Inst,
     TmpInst.addOperand(Inst.getOperand(1));
     TmpInst.addOperand(Inst.getOperand(0));
     TmpInst.addOperand(Inst.getOperand(1));
-    TmpInst.addOperand(MCOperand::CreateReg(0));
-    TmpInst.addOperand(MCOperand::CreateImm(0));
+    TmpInst.addOperand(MCOperand::createReg(0));
+    TmpInst.addOperand(MCOperand::createImm(0));
     TmpInst.addOperand(Inst.getOperand(2));
     TmpInst.addOperand(Inst.getOperand(3));
     Inst = TmpInst;
@@ -6744,24 +6793,24 @@ bool ARMAsmParser::processInstruction(MCInst &Inst,
       // Immediate (mod_imm) will be in its encoded form, we must unencode it
       // before passing it to the ADR instruction.
       unsigned Enc = Inst.getOperand(2).getImm();
-      TmpInst.addOperand(MCOperand::CreateImm(
+      TmpInst.addOperand(MCOperand::createImm(
         ARM_AM::rotr32(Enc & 0xFF, (Enc & 0xF00) >> 7)));
     } else {
       // Turn PC-relative expression into absolute expression.
       // Reading PC provides the start of the current instruction + 8 and
       // the transform to adr is biased by that.
-      MCSymbol *Dot = getContext().CreateTempSymbol();
+      MCSymbol *Dot = getContext().createTempSymbol();
       Out.EmitLabel(Dot);
       const MCExpr *OpExpr = Inst.getOperand(2).getExpr();
-      const MCExpr *InstPC = MCSymbolRefExpr::Create(Dot,
+      const MCExpr *InstPC = MCSymbolRefExpr::create(Dot,
                                                      MCSymbolRefExpr::VK_None,
                                                      getContext());
-      const MCExpr *Const8 = MCConstantExpr::Create(8, getContext());
-      const MCExpr *ReadPC = MCBinaryExpr::CreateAdd(InstPC, Const8,
+      const MCExpr *Const8 = MCConstantExpr::create(8, getContext());
+      const MCExpr *ReadPC = MCBinaryExpr::createAdd(InstPC, Const8,
                                                      getContext());
-      const MCExpr *FixupAddr = MCBinaryExpr::CreateAdd(ReadPC, OpExpr,
+      const MCExpr *FixupAddr = MCBinaryExpr::createAdd(ReadPC, OpExpr,
                                                         getContext());
-      TmpInst.addOperand(MCOperand::CreateExpr(FixupAddr));
+      TmpInst.addOperand(MCOperand::createExpr(FixupAddr));
     }
     TmpInst.addOperand(Inst.getOperand(3));
     TmpInst.addOperand(Inst.getOperand(4));
@@ -6827,7 +6876,7 @@ bool ARMAsmParser::processInstruction(MCInst &Inst,
     TmpInst.addOperand(Inst.getOperand(3)); // alignment
     TmpInst.addOperand(Inst.getOperand(4)); // Rm
     TmpInst.addOperand(Inst.getOperand(0)); // Vd
-    TmpInst.addOperand(MCOperand::CreateReg(Inst.getOperand(0).getReg() +
+    TmpInst.addOperand(MCOperand::createReg(Inst.getOperand(0).getReg() +
                                             Spacing));
     TmpInst.addOperand(Inst.getOperand(1)); // lane
     TmpInst.addOperand(Inst.getOperand(5)); // CondCode
@@ -6851,9 +6900,9 @@ bool ARMAsmParser::processInstruction(MCInst &Inst,
     TmpInst.addOperand(Inst.getOperand(3)); // alignment
     TmpInst.addOperand(Inst.getOperand(4)); // Rm
     TmpInst.addOperand(Inst.getOperand(0)); // Vd
-    TmpInst.addOperand(MCOperand::CreateReg(Inst.getOperand(0).getReg() +
+    TmpInst.addOperand(MCOperand::createReg(Inst.getOperand(0).getReg() +
                                             Spacing));
-    TmpInst.addOperand(MCOperand::CreateReg(Inst.getOperand(0).getReg() +
+    TmpInst.addOperand(MCOperand::createReg(Inst.getOperand(0).getReg() +
                                             Spacing * 2));
     TmpInst.addOperand(Inst.getOperand(1)); // lane
     TmpInst.addOperand(Inst.getOperand(5)); // CondCode
@@ -6877,11 +6926,11 @@ bool ARMAsmParser::processInstruction(MCInst &Inst,
     TmpInst.addOperand(Inst.getOperand(3)); // alignment
     TmpInst.addOperand(Inst.getOperand(4)); // Rm
     TmpInst.addOperand(Inst.getOperand(0)); // Vd
-    TmpInst.addOperand(MCOperand::CreateReg(Inst.getOperand(0).getReg() +
+    TmpInst.addOperand(MCOperand::createReg(Inst.getOperand(0).getReg() +
                                             Spacing));
-    TmpInst.addOperand(MCOperand::CreateReg(Inst.getOperand(0).getReg() +
+    TmpInst.addOperand(MCOperand::createReg(Inst.getOperand(0).getReg() +
                                             Spacing * 2));
-    TmpInst.addOperand(MCOperand::CreateReg(Inst.getOperand(0).getReg() +
+    TmpInst.addOperand(MCOperand::createReg(Inst.getOperand(0).getReg() +
                                             Spacing * 3));
     TmpInst.addOperand(Inst.getOperand(1)); // lane
     TmpInst.addOperand(Inst.getOperand(5)); // CondCode
@@ -6901,7 +6950,7 @@ bool ARMAsmParser::processInstruction(MCInst &Inst,
     TmpInst.addOperand(Inst.getOperand(2)); // Rn_wb
     TmpInst.addOperand(Inst.getOperand(2)); // Rn
     TmpInst.addOperand(Inst.getOperand(3)); // alignment
-    TmpInst.addOperand(MCOperand::CreateReg(0)); // Rm
+    TmpInst.addOperand(MCOperand::createReg(0)); // Rm
     TmpInst.addOperand(Inst.getOperand(0)); // Vd
     TmpInst.addOperand(Inst.getOperand(1)); // lane
     TmpInst.addOperand(Inst.getOperand(4)); // CondCode
@@ -6923,9 +6972,9 @@ bool ARMAsmParser::processInstruction(MCInst &Inst,
     TmpInst.addOperand(Inst.getOperand(2)); // Rn_wb
     TmpInst.addOperand(Inst.getOperand(2)); // Rn
     TmpInst.addOperand(Inst.getOperand(3)); // alignment
-    TmpInst.addOperand(MCOperand::CreateReg(0)); // Rm
+    TmpInst.addOperand(MCOperand::createReg(0)); // Rm
     TmpInst.addOperand(Inst.getOperand(0)); // Vd
-    TmpInst.addOperand(MCOperand::CreateReg(Inst.getOperand(0).getReg() +
+    TmpInst.addOperand(MCOperand::createReg(Inst.getOperand(0).getReg() +
                                             Spacing));
     TmpInst.addOperand(Inst.getOperand(1)); // lane
     TmpInst.addOperand(Inst.getOperand(4)); // CondCode
@@ -6947,11 +6996,11 @@ bool ARMAsmParser::processInstruction(MCInst &Inst,
     TmpInst.addOperand(Inst.getOperand(2)); // Rn_wb
     TmpInst.addOperand(Inst.getOperand(2)); // Rn
     TmpInst.addOperand(Inst.getOperand(3)); // alignment
-    TmpInst.addOperand(MCOperand::CreateReg(0)); // Rm
+    TmpInst.addOperand(MCOperand::createReg(0)); // Rm
     TmpInst.addOperand(Inst.getOperand(0)); // Vd
-    TmpInst.addOperand(MCOperand::CreateReg(Inst.getOperand(0).getReg() +
+    TmpInst.addOperand(MCOperand::createReg(Inst.getOperand(0).getReg() +
                                             Spacing));
-    TmpInst.addOperand(MCOperand::CreateReg(Inst.getOperand(0).getReg() +
+    TmpInst.addOperand(MCOperand::createReg(Inst.getOperand(0).getReg() +
                                             Spacing * 2));
     TmpInst.addOperand(Inst.getOperand(1)); // lane
     TmpInst.addOperand(Inst.getOperand(4)); // CondCode
@@ -6973,13 +7022,13 @@ bool ARMAsmParser::processInstruction(MCInst &Inst,
     TmpInst.addOperand(Inst.getOperand(2)); // Rn_wb
     TmpInst.addOperand(Inst.getOperand(2)); // Rn
     TmpInst.addOperand(Inst.getOperand(3)); // alignment
-    TmpInst.addOperand(MCOperand::CreateReg(0)); // Rm
+    TmpInst.addOperand(MCOperand::createReg(0)); // Rm
     TmpInst.addOperand(Inst.getOperand(0)); // Vd
-    TmpInst.addOperand(MCOperand::CreateReg(Inst.getOperand(0).getReg() +
+    TmpInst.addOperand(MCOperand::createReg(Inst.getOperand(0).getReg() +
                                             Spacing));
-    TmpInst.addOperand(MCOperand::CreateReg(Inst.getOperand(0).getReg() +
+    TmpInst.addOperand(MCOperand::createReg(Inst.getOperand(0).getReg() +
                                             Spacing * 2));
-    TmpInst.addOperand(MCOperand::CreateReg(Inst.getOperand(0).getReg() +
+    TmpInst.addOperand(MCOperand::createReg(Inst.getOperand(0).getReg() +
                                             Spacing * 3));
     TmpInst.addOperand(Inst.getOperand(1)); // lane
     TmpInst.addOperand(Inst.getOperand(4)); // CondCode
@@ -7019,7 +7068,7 @@ bool ARMAsmParser::processInstruction(MCInst &Inst,
     TmpInst.addOperand(Inst.getOperand(2)); // Rn
     TmpInst.addOperand(Inst.getOperand(3)); // alignment
     TmpInst.addOperand(Inst.getOperand(0)); // Vd
-    TmpInst.addOperand(MCOperand::CreateReg(Inst.getOperand(0).getReg() +
+    TmpInst.addOperand(MCOperand::createReg(Inst.getOperand(0).getReg() +
                                             Spacing));
     TmpInst.addOperand(Inst.getOperand(1)); // lane
     TmpInst.addOperand(Inst.getOperand(4)); // CondCode
@@ -7041,9 +7090,9 @@ bool ARMAsmParser::processInstruction(MCInst &Inst,
     TmpInst.addOperand(Inst.getOperand(2)); // Rn
     TmpInst.addOperand(Inst.getOperand(3)); // alignment
     TmpInst.addOperand(Inst.getOperand(0)); // Vd
-    TmpInst.addOperand(MCOperand::CreateReg(Inst.getOperand(0).getReg() +
+    TmpInst.addOperand(MCOperand::createReg(Inst.getOperand(0).getReg() +
                                             Spacing));
-    TmpInst.addOperand(MCOperand::CreateReg(Inst.getOperand(0).getReg() +
+    TmpInst.addOperand(MCOperand::createReg(Inst.getOperand(0).getReg() +
                                             Spacing * 2));
     TmpInst.addOperand(Inst.getOperand(1)); // lane
     TmpInst.addOperand(Inst.getOperand(4)); // CondCode
@@ -7065,11 +7114,11 @@ bool ARMAsmParser::processInstruction(MCInst &Inst,
     TmpInst.addOperand(Inst.getOperand(2)); // Rn
     TmpInst.addOperand(Inst.getOperand(3)); // alignment
     TmpInst.addOperand(Inst.getOperand(0)); // Vd
-    TmpInst.addOperand(MCOperand::CreateReg(Inst.getOperand(0).getReg() +
+    TmpInst.addOperand(MCOperand::createReg(Inst.getOperand(0).getReg() +
                                             Spacing));
-    TmpInst.addOperand(MCOperand::CreateReg(Inst.getOperand(0).getReg() +
+    TmpInst.addOperand(MCOperand::createReg(Inst.getOperand(0).getReg() +
                                             Spacing * 2));
-    TmpInst.addOperand(MCOperand::CreateReg(Inst.getOperand(0).getReg() +
+    TmpInst.addOperand(MCOperand::createReg(Inst.getOperand(0).getReg() +
                                             Spacing * 3));
     TmpInst.addOperand(Inst.getOperand(1)); // lane
     TmpInst.addOperand(Inst.getOperand(4)); // CondCode
@@ -7111,14 +7160,14 @@ bool ARMAsmParser::processInstruction(MCInst &Inst,
     unsigned Spacing;
     TmpInst.setOpcode(getRealVLDOpcode(Inst.getOpcode(), Spacing));
     TmpInst.addOperand(Inst.getOperand(0)); // Vd
-    TmpInst.addOperand(MCOperand::CreateReg(Inst.getOperand(0).getReg() +
+    TmpInst.addOperand(MCOperand::createReg(Inst.getOperand(0).getReg() +
                                             Spacing));
     TmpInst.addOperand(Inst.getOperand(2)); // Rn_wb
     TmpInst.addOperand(Inst.getOperand(2)); // Rn
     TmpInst.addOperand(Inst.getOperand(3)); // alignment
     TmpInst.addOperand(Inst.getOperand(4)); // Rm
     TmpInst.addOperand(Inst.getOperand(0)); // Tied operand src (== Vd)
-    TmpInst.addOperand(MCOperand::CreateReg(Inst.getOperand(0).getReg() +
+    TmpInst.addOperand(MCOperand::createReg(Inst.getOperand(0).getReg() +
                                             Spacing));
     TmpInst.addOperand(Inst.getOperand(1)); // lane
     TmpInst.addOperand(Inst.getOperand(5)); // CondCode
@@ -7138,18 +7187,18 @@ bool ARMAsmParser::processInstruction(MCInst &Inst,
     unsigned Spacing;
     TmpInst.setOpcode(getRealVLDOpcode(Inst.getOpcode(), Spacing));
     TmpInst.addOperand(Inst.getOperand(0)); // Vd
-    TmpInst.addOperand(MCOperand::CreateReg(Inst.getOperand(0).getReg() +
+    TmpInst.addOperand(MCOperand::createReg(Inst.getOperand(0).getReg() +
                                             Spacing));
-    TmpInst.addOperand(MCOperand::CreateReg(Inst.getOperand(0).getReg() +
+    TmpInst.addOperand(MCOperand::createReg(Inst.getOperand(0).getReg() +
                                             Spacing * 2));
     TmpInst.addOperand(Inst.getOperand(2)); // Rn_wb
     TmpInst.addOperand(Inst.getOperand(2)); // Rn
     TmpInst.addOperand(Inst.getOperand(3)); // alignment
     TmpInst.addOperand(Inst.getOperand(4)); // Rm
     TmpInst.addOperand(Inst.getOperand(0)); // Tied operand src (== Vd)
-    TmpInst.addOperand(MCOperand::CreateReg(Inst.getOperand(0).getReg() +
+    TmpInst.addOperand(MCOperand::createReg(Inst.getOperand(0).getReg() +
                                             Spacing));
-    TmpInst.addOperand(MCOperand::CreateReg(Inst.getOperand(0).getReg() +
+    TmpInst.addOperand(MCOperand::createReg(Inst.getOperand(0).getReg() +
                                             Spacing * 2));
     TmpInst.addOperand(Inst.getOperand(1)); // lane
     TmpInst.addOperand(Inst.getOperand(5)); // CondCode
@@ -7169,22 +7218,22 @@ bool ARMAsmParser::processInstruction(MCInst &Inst,
     unsigned Spacing;
     TmpInst.setOpcode(getRealVLDOpcode(Inst.getOpcode(), Spacing));
     TmpInst.addOperand(Inst.getOperand(0)); // Vd
-    TmpInst.addOperand(MCOperand::CreateReg(Inst.getOperand(0).getReg() +
+    TmpInst.addOperand(MCOperand::createReg(Inst.getOperand(0).getReg() +
                                             Spacing));
-    TmpInst.addOperand(MCOperand::CreateReg(Inst.getOperand(0).getReg() +
+    TmpInst.addOperand(MCOperand::createReg(Inst.getOperand(0).getReg() +
                                             Spacing * 2));
-    TmpInst.addOperand(MCOperand::CreateReg(Inst.getOperand(0).getReg() +
+    TmpInst.addOperand(MCOperand::createReg(Inst.getOperand(0).getReg() +
                                             Spacing * 3));
     TmpInst.addOperand(Inst.getOperand(2)); // Rn_wb
     TmpInst.addOperand(Inst.getOperand(2)); // Rn
     TmpInst.addOperand(Inst.getOperand(3)); // alignment
     TmpInst.addOperand(Inst.getOperand(4)); // Rm
     TmpInst.addOperand(Inst.getOperand(0)); // Tied operand src (== Vd)
-    TmpInst.addOperand(MCOperand::CreateReg(Inst.getOperand(0).getReg() +
+    TmpInst.addOperand(MCOperand::createReg(Inst.getOperand(0).getReg() +
                                             Spacing));
-    TmpInst.addOperand(MCOperand::CreateReg(Inst.getOperand(0).getReg() +
+    TmpInst.addOperand(MCOperand::createReg(Inst.getOperand(0).getReg() +
                                             Spacing * 2));
-    TmpInst.addOperand(MCOperand::CreateReg(Inst.getOperand(0).getReg() +
+    TmpInst.addOperand(MCOperand::createReg(Inst.getOperand(0).getReg() +
                                             Spacing * 3));
     TmpInst.addOperand(Inst.getOperand(1)); // lane
     TmpInst.addOperand(Inst.getOperand(5)); // CondCode
@@ -7205,7 +7254,7 @@ bool ARMAsmParser::processInstruction(MCInst &Inst,
     TmpInst.addOperand(Inst.getOperand(2)); // Rn_wb
     TmpInst.addOperand(Inst.getOperand(2)); // Rn
     TmpInst.addOperand(Inst.getOperand(3)); // alignment
-    TmpInst.addOperand(MCOperand::CreateReg(0)); // Rm
+    TmpInst.addOperand(MCOperand::createReg(0)); // Rm
     TmpInst.addOperand(Inst.getOperand(0)); // Tied operand src (== Vd)
     TmpInst.addOperand(Inst.getOperand(1)); // lane
     TmpInst.addOperand(Inst.getOperand(4)); // CondCode
@@ -7225,14 +7274,14 @@ bool ARMAsmParser::processInstruction(MCInst &Inst,
     unsigned Spacing;
     TmpInst.setOpcode(getRealVLDOpcode(Inst.getOpcode(), Spacing));
     TmpInst.addOperand(Inst.getOperand(0)); // Vd
-    TmpInst.addOperand(MCOperand::CreateReg(Inst.getOperand(0).getReg() +
+    TmpInst.addOperand(MCOperand::createReg(Inst.getOperand(0).getReg() +
                                             Spacing));
     TmpInst.addOperand(Inst.getOperand(2)); // Rn_wb
     TmpInst.addOperand(Inst.getOperand(2)); // Rn
     TmpInst.addOperand(Inst.getOperand(3)); // alignment
-    TmpInst.addOperand(MCOperand::CreateReg(0)); // Rm
+    TmpInst.addOperand(MCOperand::createReg(0)); // Rm
     TmpInst.addOperand(Inst.getOperand(0)); // Tied operand src (== Vd)
-    TmpInst.addOperand(MCOperand::CreateReg(Inst.getOperand(0).getReg() +
+    TmpInst.addOperand(MCOperand::createReg(Inst.getOperand(0).getReg() +
                                             Spacing));
     TmpInst.addOperand(Inst.getOperand(1)); // lane
     TmpInst.addOperand(Inst.getOperand(4)); // CondCode
@@ -7252,18 +7301,18 @@ bool ARMAsmParser::processInstruction(MCInst &Inst,
     unsigned Spacing;
     TmpInst.setOpcode(getRealVLDOpcode(Inst.getOpcode(), Spacing));
     TmpInst.addOperand(Inst.getOperand(0)); // Vd
-    TmpInst.addOperand(MCOperand::CreateReg(Inst.getOperand(0).getReg() +
+    TmpInst.addOperand(MCOperand::createReg(Inst.getOperand(0).getReg() +
                                             Spacing));
-    TmpInst.addOperand(MCOperand::CreateReg(Inst.getOperand(0).getReg() +
+    TmpInst.addOperand(MCOperand::createReg(Inst.getOperand(0).getReg() +
                                             Spacing * 2));
     TmpInst.addOperand(Inst.getOperand(2)); // Rn_wb
     TmpInst.addOperand(Inst.getOperand(2)); // Rn
     TmpInst.addOperand(Inst.getOperand(3)); // alignment
-    TmpInst.addOperand(MCOperand::CreateReg(0)); // Rm
+    TmpInst.addOperand(MCOperand::createReg(0)); // Rm
     TmpInst.addOperand(Inst.getOperand(0)); // Tied operand src (== Vd)
-    TmpInst.addOperand(MCOperand::CreateReg(Inst.getOperand(0).getReg() +
+    TmpInst.addOperand(MCOperand::createReg(Inst.getOperand(0).getReg() +
                                             Spacing));
-    TmpInst.addOperand(MCOperand::CreateReg(Inst.getOperand(0).getReg() +
+    TmpInst.addOperand(MCOperand::createReg(Inst.getOperand(0).getReg() +
                                             Spacing * 2));
     TmpInst.addOperand(Inst.getOperand(1)); // lane
     TmpInst.addOperand(Inst.getOperand(4)); // CondCode
@@ -7283,22 +7332,22 @@ bool ARMAsmParser::processInstruction(MCInst &Inst,
     unsigned Spacing;
     TmpInst.setOpcode(getRealVLDOpcode(Inst.getOpcode(), Spacing));
     TmpInst.addOperand(Inst.getOperand(0)); // Vd
-    TmpInst.addOperand(MCOperand::CreateReg(Inst.getOperand(0).getReg() +
+    TmpInst.addOperand(MCOperand::createReg(Inst.getOperand(0).getReg() +
                                             Spacing));
-    TmpInst.addOperand(MCOperand::CreateReg(Inst.getOperand(0).getReg() +
+    TmpInst.addOperand(MCOperand::createReg(Inst.getOperand(0).getReg() +
                                             Spacing * 2));
-    TmpInst.addOperand(MCOperand::CreateReg(Inst.getOperand(0).getReg() +
+    TmpInst.addOperand(MCOperand::createReg(Inst.getOperand(0).getReg() +
                                             Spacing * 3));
     TmpInst.addOperand(Inst.getOperand(2)); // Rn_wb
     TmpInst.addOperand(Inst.getOperand(2)); // Rn
     TmpInst.addOperand(Inst.getOperand(3)); // alignment
-    TmpInst.addOperand(MCOperand::CreateReg(0)); // Rm
+    TmpInst.addOperand(MCOperand::createReg(0)); // Rm
     TmpInst.addOperand(Inst.getOperand(0)); // Tied operand src (== Vd)
-    TmpInst.addOperand(MCOperand::CreateReg(Inst.getOperand(0).getReg() +
+    TmpInst.addOperand(MCOperand::createReg(Inst.getOperand(0).getReg() +
                                             Spacing));
-    TmpInst.addOperand(MCOperand::CreateReg(Inst.getOperand(0).getReg() +
+    TmpInst.addOperand(MCOperand::createReg(Inst.getOperand(0).getReg() +
                                             Spacing * 2));
-    TmpInst.addOperand(MCOperand::CreateReg(Inst.getOperand(0).getReg() +
+    TmpInst.addOperand(MCOperand::createReg(Inst.getOperand(0).getReg() +
                                             Spacing * 3));
     TmpInst.addOperand(Inst.getOperand(1)); // lane
     TmpInst.addOperand(Inst.getOperand(4)); // CondCode
@@ -7337,12 +7386,12 @@ bool ARMAsmParser::processInstruction(MCInst &Inst,
     unsigned Spacing;
     TmpInst.setOpcode(getRealVLDOpcode(Inst.getOpcode(), Spacing));
     TmpInst.addOperand(Inst.getOperand(0)); // Vd
-    TmpInst.addOperand(MCOperand::CreateReg(Inst.getOperand(0).getReg() +
+    TmpInst.addOperand(MCOperand::createReg(Inst.getOperand(0).getReg() +
                                             Spacing));
     TmpInst.addOperand(Inst.getOperand(2)); // Rn
     TmpInst.addOperand(Inst.getOperand(3)); // alignment
     TmpInst.addOperand(Inst.getOperand(0)); // Tied operand src (== Vd)
-    TmpInst.addOperand(MCOperand::CreateReg(Inst.getOperand(0).getReg() +
+    TmpInst.addOperand(MCOperand::createReg(Inst.getOperand(0).getReg() +
                                             Spacing));
     TmpInst.addOperand(Inst.getOperand(1)); // lane
     TmpInst.addOperand(Inst.getOperand(4)); // CondCode
@@ -7362,16 +7411,16 @@ bool ARMAsmParser::processInstruction(MCInst &Inst,
     unsigned Spacing;
     TmpInst.setOpcode(getRealVLDOpcode(Inst.getOpcode(), Spacing));
     TmpInst.addOperand(Inst.getOperand(0)); // Vd
-    TmpInst.addOperand(MCOperand::CreateReg(Inst.getOperand(0).getReg() +
+    TmpInst.addOperand(MCOperand::createReg(Inst.getOperand(0).getReg() +
                                             Spacing));
-    TmpInst.addOperand(MCOperand::CreateReg(Inst.getOperand(0).getReg() +
+    TmpInst.addOperand(MCOperand::createReg(Inst.getOperand(0).getReg() +
                                             Spacing * 2));
     TmpInst.addOperand(Inst.getOperand(2)); // Rn
     TmpInst.addOperand(Inst.getOperand(3)); // alignment
     TmpInst.addOperand(Inst.getOperand(0)); // Tied operand src (== Vd)
-    TmpInst.addOperand(MCOperand::CreateReg(Inst.getOperand(0).getReg() +
+    TmpInst.addOperand(MCOperand::createReg(Inst.getOperand(0).getReg() +
                                             Spacing));
-    TmpInst.addOperand(MCOperand::CreateReg(Inst.getOperand(0).getReg() +
+    TmpInst.addOperand(MCOperand::createReg(Inst.getOperand(0).getReg() +
                                             Spacing * 2));
     TmpInst.addOperand(Inst.getOperand(1)); // lane
     TmpInst.addOperand(Inst.getOperand(4)); // CondCode
@@ -7391,20 +7440,20 @@ bool ARMAsmParser::processInstruction(MCInst &Inst,
     unsigned Spacing;
     TmpInst.setOpcode(getRealVLDOpcode(Inst.getOpcode(), Spacing));
     TmpInst.addOperand(Inst.getOperand(0)); // Vd
-    TmpInst.addOperand(MCOperand::CreateReg(Inst.getOperand(0).getReg() +
+    TmpInst.addOperand(MCOperand::createReg(Inst.getOperand(0).getReg() +
                                             Spacing));
-    TmpInst.addOperand(MCOperand::CreateReg(Inst.getOperand(0).getReg() +
+    TmpInst.addOperand(MCOperand::createReg(Inst.getOperand(0).getReg() +
                                             Spacing * 2));
-    TmpInst.addOperand(MCOperand::CreateReg(Inst.getOperand(0).getReg() +
+    TmpInst.addOperand(MCOperand::createReg(Inst.getOperand(0).getReg() +
                                             Spacing * 3));
     TmpInst.addOperand(Inst.getOperand(2)); // Rn
     TmpInst.addOperand(Inst.getOperand(3)); // alignment
     TmpInst.addOperand(Inst.getOperand(0)); // Tied operand src (== Vd)
-    TmpInst.addOperand(MCOperand::CreateReg(Inst.getOperand(0).getReg() +
+    TmpInst.addOperand(MCOperand::createReg(Inst.getOperand(0).getReg() +
                                             Spacing));
-    TmpInst.addOperand(MCOperand::CreateReg(Inst.getOperand(0).getReg() +
+    TmpInst.addOperand(MCOperand::createReg(Inst.getOperand(0).getReg() +
                                             Spacing * 2));
-    TmpInst.addOperand(MCOperand::CreateReg(Inst.getOperand(0).getReg() +
+    TmpInst.addOperand(MCOperand::createReg(Inst.getOperand(0).getReg() +
                                             Spacing * 3));
     TmpInst.addOperand(Inst.getOperand(1)); // lane
     TmpInst.addOperand(Inst.getOperand(4)); // CondCode
@@ -7424,9 +7473,9 @@ bool ARMAsmParser::processInstruction(MCInst &Inst,
     unsigned Spacing;
     TmpInst.setOpcode(getRealVLDOpcode(Inst.getOpcode(), Spacing));
     TmpInst.addOperand(Inst.getOperand(0)); // Vd
-    TmpInst.addOperand(MCOperand::CreateReg(Inst.getOperand(0).getReg() +
+    TmpInst.addOperand(MCOperand::createReg(Inst.getOperand(0).getReg() +
                                             Spacing));
-    TmpInst.addOperand(MCOperand::CreateReg(Inst.getOperand(0).getReg() +
+    TmpInst.addOperand(MCOperand::createReg(Inst.getOperand(0).getReg() +
                                             Spacing * 2));
     TmpInst.addOperand(Inst.getOperand(1)); // Rn
     TmpInst.addOperand(Inst.getOperand(2)); // alignment
@@ -7446,14 +7495,14 @@ bool ARMAsmParser::processInstruction(MCInst &Inst,
     unsigned Spacing;
     TmpInst.setOpcode(getRealVLDOpcode(Inst.getOpcode(), Spacing));
     TmpInst.addOperand(Inst.getOperand(0)); // Vd
-    TmpInst.addOperand(MCOperand::CreateReg(Inst.getOperand(0).getReg() +
+    TmpInst.addOperand(MCOperand::createReg(Inst.getOperand(0).getReg() +
                                             Spacing));
-    TmpInst.addOperand(MCOperand::CreateReg(Inst.getOperand(0).getReg() +
+    TmpInst.addOperand(MCOperand::createReg(Inst.getOperand(0).getReg() +
                                             Spacing * 2));
     TmpInst.addOperand(Inst.getOperand(1)); // Rn
     TmpInst.addOperand(Inst.getOperand(1)); // Rn_wb == tied Rn
     TmpInst.addOperand(Inst.getOperand(2)); // alignment
-    TmpInst.addOperand(MCOperand::CreateReg(0)); // Rm
+    TmpInst.addOperand(MCOperand::createReg(0)); // Rm
     TmpInst.addOperand(Inst.getOperand(3)); // CondCode
     TmpInst.addOperand(Inst.getOperand(4));
     Inst = TmpInst;
@@ -7470,9 +7519,9 @@ bool ARMAsmParser::processInstruction(MCInst &Inst,
     unsigned Spacing;
     TmpInst.setOpcode(getRealVLDOpcode(Inst.getOpcode(), Spacing));
     TmpInst.addOperand(Inst.getOperand(0)); // Vd
-    TmpInst.addOperand(MCOperand::CreateReg(Inst.getOperand(0).getReg() +
+    TmpInst.addOperand(MCOperand::createReg(Inst.getOperand(0).getReg() +
                                             Spacing));
-    TmpInst.addOperand(MCOperand::CreateReg(Inst.getOperand(0).getReg() +
+    TmpInst.addOperand(MCOperand::createReg(Inst.getOperand(0).getReg() +
                                             Spacing * 2));
     TmpInst.addOperand(Inst.getOperand(1)); // Rn
     TmpInst.addOperand(Inst.getOperand(1)); // Rn_wb == tied Rn
@@ -7495,9 +7544,9 @@ bool ARMAsmParser::processInstruction(MCInst &Inst,
     unsigned Spacing;
     TmpInst.setOpcode(getRealVLDOpcode(Inst.getOpcode(), Spacing));
     TmpInst.addOperand(Inst.getOperand(0)); // Vd
-    TmpInst.addOperand(MCOperand::CreateReg(Inst.getOperand(0).getReg() +
+    TmpInst.addOperand(MCOperand::createReg(Inst.getOperand(0).getReg() +
                                             Spacing));
-    TmpInst.addOperand(MCOperand::CreateReg(Inst.getOperand(0).getReg() +
+    TmpInst.addOperand(MCOperand::createReg(Inst.getOperand(0).getReg() +
                                             Spacing * 2));
     TmpInst.addOperand(Inst.getOperand(1)); // Rn
     TmpInst.addOperand(Inst.getOperand(2)); // alignment
@@ -7517,14 +7566,14 @@ bool ARMAsmParser::processInstruction(MCInst &Inst,
     unsigned Spacing;
     TmpInst.setOpcode(getRealVLDOpcode(Inst.getOpcode(), Spacing));
     TmpInst.addOperand(Inst.getOperand(0)); // Vd
-    TmpInst.addOperand(MCOperand::CreateReg(Inst.getOperand(0).getReg() +
+    TmpInst.addOperand(MCOperand::createReg(Inst.getOperand(0).getReg() +
                                             Spacing));
-    TmpInst.addOperand(MCOperand::CreateReg(Inst.getOperand(0).getReg() +
+    TmpInst.addOperand(MCOperand::createReg(Inst.getOperand(0).getReg() +
                                             Spacing * 2));
     TmpInst.addOperand(Inst.getOperand(1)); // Rn
     TmpInst.addOperand(Inst.getOperand(1)); // Rn_wb == tied Rn
     TmpInst.addOperand(Inst.getOperand(2)); // alignment
-    TmpInst.addOperand(MCOperand::CreateReg(0)); // Rm
+    TmpInst.addOperand(MCOperand::createReg(0)); // Rm
     TmpInst.addOperand(Inst.getOperand(3)); // CondCode
     TmpInst.addOperand(Inst.getOperand(4));
     Inst = TmpInst;
@@ -7541,9 +7590,9 @@ bool ARMAsmParser::processInstruction(MCInst &Inst,
     unsigned Spacing;
     TmpInst.setOpcode(getRealVLDOpcode(Inst.getOpcode(), Spacing));
     TmpInst.addOperand(Inst.getOperand(0)); // Vd
-    TmpInst.addOperand(MCOperand::CreateReg(Inst.getOperand(0).getReg() +
+    TmpInst.addOperand(MCOperand::createReg(Inst.getOperand(0).getReg() +
                                             Spacing));
-    TmpInst.addOperand(MCOperand::CreateReg(Inst.getOperand(0).getReg() +
+    TmpInst.addOperand(MCOperand::createReg(Inst.getOperand(0).getReg() +
                                             Spacing * 2));
     TmpInst.addOperand(Inst.getOperand(1)); // Rn
     TmpInst.addOperand(Inst.getOperand(1)); // Rn_wb == tied Rn
@@ -7566,11 +7615,11 @@ bool ARMAsmParser::processInstruction(MCInst &Inst,
     unsigned Spacing;
     TmpInst.setOpcode(getRealVLDOpcode(Inst.getOpcode(), Spacing));
     TmpInst.addOperand(Inst.getOperand(0)); // Vd
-    TmpInst.addOperand(MCOperand::CreateReg(Inst.getOperand(0).getReg() +
+    TmpInst.addOperand(MCOperand::createReg(Inst.getOperand(0).getReg() +
                                             Spacing));
-    TmpInst.addOperand(MCOperand::CreateReg(Inst.getOperand(0).getReg() +
+    TmpInst.addOperand(MCOperand::createReg(Inst.getOperand(0).getReg() +
                                             Spacing * 2));
-    TmpInst.addOperand(MCOperand::CreateReg(Inst.getOperand(0).getReg() +
+    TmpInst.addOperand(MCOperand::createReg(Inst.getOperand(0).getReg() +
                                             Spacing * 3));
     TmpInst.addOperand(Inst.getOperand(1)); // Rn
     TmpInst.addOperand(Inst.getOperand(2)); // alignment
@@ -7590,16 +7639,16 @@ bool ARMAsmParser::processInstruction(MCInst &Inst,
     unsigned Spacing;
     TmpInst.setOpcode(getRealVLDOpcode(Inst.getOpcode(), Spacing));
     TmpInst.addOperand(Inst.getOperand(0)); // Vd
-    TmpInst.addOperand(MCOperand::CreateReg(Inst.getOperand(0).getReg() +
+    TmpInst.addOperand(MCOperand::createReg(Inst.getOperand(0).getReg() +
                                             Spacing));
-    TmpInst.addOperand(MCOperand::CreateReg(Inst.getOperand(0).getReg() +
+    TmpInst.addOperand(MCOperand::createReg(Inst.getOperand(0).getReg() +
                                             Spacing * 2));
-    TmpInst.addOperand(MCOperand::CreateReg(Inst.getOperand(0).getReg() +
+    TmpInst.addOperand(MCOperand::createReg(Inst.getOperand(0).getReg() +
                                             Spacing * 3));
     TmpInst.addOperand(Inst.getOperand(1)); // Rn
     TmpInst.addOperand(Inst.getOperand(1)); // Rn_wb == tied Rn
     TmpInst.addOperand(Inst.getOperand(2)); // alignment
-    TmpInst.addOperand(MCOperand::CreateReg(0)); // Rm
+    TmpInst.addOperand(MCOperand::createReg(0)); // Rm
     TmpInst.addOperand(Inst.getOperand(3)); // CondCode
     TmpInst.addOperand(Inst.getOperand(4));
     Inst = TmpInst;
@@ -7616,11 +7665,11 @@ bool ARMAsmParser::processInstruction(MCInst &Inst,
     unsigned Spacing;
     TmpInst.setOpcode(getRealVLDOpcode(Inst.getOpcode(), Spacing));
     TmpInst.addOperand(Inst.getOperand(0)); // Vd
-    TmpInst.addOperand(MCOperand::CreateReg(Inst.getOperand(0).getReg() +
+    TmpInst.addOperand(MCOperand::createReg(Inst.getOperand(0).getReg() +
                                             Spacing));
-    TmpInst.addOperand(MCOperand::CreateReg(Inst.getOperand(0).getReg() +
+    TmpInst.addOperand(MCOperand::createReg(Inst.getOperand(0).getReg() +
                                             Spacing * 2));
-    TmpInst.addOperand(MCOperand::CreateReg(Inst.getOperand(0).getReg() +
+    TmpInst.addOperand(MCOperand::createReg(Inst.getOperand(0).getReg() +
                                             Spacing * 3));
     TmpInst.addOperand(Inst.getOperand(1)); // Rn
     TmpInst.addOperand(Inst.getOperand(1)); // Rn_wb == tied Rn
@@ -7643,11 +7692,11 @@ bool ARMAsmParser::processInstruction(MCInst &Inst,
     unsigned Spacing;
     TmpInst.setOpcode(getRealVLDOpcode(Inst.getOpcode(), Spacing));
     TmpInst.addOperand(Inst.getOperand(0)); // Vd
-    TmpInst.addOperand(MCOperand::CreateReg(Inst.getOperand(0).getReg() +
+    TmpInst.addOperand(MCOperand::createReg(Inst.getOperand(0).getReg() +
                                             Spacing));
-    TmpInst.addOperand(MCOperand::CreateReg(Inst.getOperand(0).getReg() +
+    TmpInst.addOperand(MCOperand::createReg(Inst.getOperand(0).getReg() +
                                             Spacing * 2));
-    TmpInst.addOperand(MCOperand::CreateReg(Inst.getOperand(0).getReg() +
+    TmpInst.addOperand(MCOperand::createReg(Inst.getOperand(0).getReg() +
                                             Spacing * 3));
     TmpInst.addOperand(Inst.getOperand(1)); // Rn
     TmpInst.addOperand(Inst.getOperand(2)); // alignment
@@ -7667,16 +7716,16 @@ bool ARMAsmParser::processInstruction(MCInst &Inst,
     unsigned Spacing;
     TmpInst.setOpcode(getRealVLDOpcode(Inst.getOpcode(), Spacing));
     TmpInst.addOperand(Inst.getOperand(0)); // Vd
-    TmpInst.addOperand(MCOperand::CreateReg(Inst.getOperand(0).getReg() +
+    TmpInst.addOperand(MCOperand::createReg(Inst.getOperand(0).getReg() +
                                             Spacing));
-    TmpInst.addOperand(MCOperand::CreateReg(Inst.getOperand(0).getReg() +
+    TmpInst.addOperand(MCOperand::createReg(Inst.getOperand(0).getReg() +
                                             Spacing * 2));
-    TmpInst.addOperand(MCOperand::CreateReg(Inst.getOperand(0).getReg() +
+    TmpInst.addOperand(MCOperand::createReg(Inst.getOperand(0).getReg() +
                                             Spacing * 3));
     TmpInst.addOperand(Inst.getOperand(1)); // Rn
     TmpInst.addOperand(Inst.getOperand(1)); // Rn_wb == tied Rn
     TmpInst.addOperand(Inst.getOperand(2)); // alignment
-    TmpInst.addOperand(MCOperand::CreateReg(0)); // Rm
+    TmpInst.addOperand(MCOperand::createReg(0)); // Rm
     TmpInst.addOperand(Inst.getOperand(3)); // CondCode
     TmpInst.addOperand(Inst.getOperand(4));
     Inst = TmpInst;
@@ -7693,11 +7742,11 @@ bool ARMAsmParser::processInstruction(MCInst &Inst,
     unsigned Spacing;
     TmpInst.setOpcode(getRealVLDOpcode(Inst.getOpcode(), Spacing));
     TmpInst.addOperand(Inst.getOperand(0)); // Vd
-    TmpInst.addOperand(MCOperand::CreateReg(Inst.getOperand(0).getReg() +
+    TmpInst.addOperand(MCOperand::createReg(Inst.getOperand(0).getReg() +
                                             Spacing));
-    TmpInst.addOperand(MCOperand::CreateReg(Inst.getOperand(0).getReg() +
+    TmpInst.addOperand(MCOperand::createReg(Inst.getOperand(0).getReg() +
                                             Spacing * 2));
-    TmpInst.addOperand(MCOperand::CreateReg(Inst.getOperand(0).getReg() +
+    TmpInst.addOperand(MCOperand::createReg(Inst.getOperand(0).getReg() +
                                             Spacing * 3));
     TmpInst.addOperand(Inst.getOperand(1)); // Rn
     TmpInst.addOperand(Inst.getOperand(1)); // Rn_wb == tied Rn
@@ -7722,9 +7771,9 @@ bool ARMAsmParser::processInstruction(MCInst &Inst,
     TmpInst.addOperand(Inst.getOperand(1)); // Rn
     TmpInst.addOperand(Inst.getOperand(2)); // alignment
     TmpInst.addOperand(Inst.getOperand(0)); // Vd
-    TmpInst.addOperand(MCOperand::CreateReg(Inst.getOperand(0).getReg() +
+    TmpInst.addOperand(MCOperand::createReg(Inst.getOperand(0).getReg() +
                                             Spacing));
-    TmpInst.addOperand(MCOperand::CreateReg(Inst.getOperand(0).getReg() +
+    TmpInst.addOperand(MCOperand::createReg(Inst.getOperand(0).getReg() +
                                             Spacing * 2));
     TmpInst.addOperand(Inst.getOperand(3)); // CondCode
     TmpInst.addOperand(Inst.getOperand(4));
@@ -7744,11 +7793,11 @@ bool ARMAsmParser::processInstruction(MCInst &Inst,
     TmpInst.addOperand(Inst.getOperand(1)); // Rn
     TmpInst.addOperand(Inst.getOperand(1)); // Rn_wb == tied Rn
     TmpInst.addOperand(Inst.getOperand(2)); // alignment
-    TmpInst.addOperand(MCOperand::CreateReg(0)); // Rm
+    TmpInst.addOperand(MCOperand::createReg(0)); // Rm
     TmpInst.addOperand(Inst.getOperand(0)); // Vd
-    TmpInst.addOperand(MCOperand::CreateReg(Inst.getOperand(0).getReg() +
+    TmpInst.addOperand(MCOperand::createReg(Inst.getOperand(0).getReg() +
                                             Spacing));
-    TmpInst.addOperand(MCOperand::CreateReg(Inst.getOperand(0).getReg() +
+    TmpInst.addOperand(MCOperand::createReg(Inst.getOperand(0).getReg() +
                                             Spacing * 2));
     TmpInst.addOperand(Inst.getOperand(3)); // CondCode
     TmpInst.addOperand(Inst.getOperand(4));
@@ -7770,9 +7819,9 @@ bool ARMAsmParser::processInstruction(MCInst &Inst,
     TmpInst.addOperand(Inst.getOperand(2)); // alignment
     TmpInst.addOperand(Inst.getOperand(3)); // Rm
     TmpInst.addOperand(Inst.getOperand(0)); // Vd
-    TmpInst.addOperand(MCOperand::CreateReg(Inst.getOperand(0).getReg() +
+    TmpInst.addOperand(MCOperand::createReg(Inst.getOperand(0).getReg() +
                                             Spacing));
-    TmpInst.addOperand(MCOperand::CreateReg(Inst.getOperand(0).getReg() +
+    TmpInst.addOperand(MCOperand::createReg(Inst.getOperand(0).getReg() +
                                             Spacing * 2));
     TmpInst.addOperand(Inst.getOperand(4)); // CondCode
     TmpInst.addOperand(Inst.getOperand(5));
@@ -7793,11 +7842,11 @@ bool ARMAsmParser::processInstruction(MCInst &Inst,
     TmpInst.addOperand(Inst.getOperand(1)); // Rn
     TmpInst.addOperand(Inst.getOperand(2)); // alignment
     TmpInst.addOperand(Inst.getOperand(0)); // Vd
-    TmpInst.addOperand(MCOperand::CreateReg(Inst.getOperand(0).getReg() +
+    TmpInst.addOperand(MCOperand::createReg(Inst.getOperand(0).getReg() +
                                             Spacing));
-    TmpInst.addOperand(MCOperand::CreateReg(Inst.getOperand(0).getReg() +
+    TmpInst.addOperand(MCOperand::createReg(Inst.getOperand(0).getReg() +
                                             Spacing * 2));
-    TmpInst.addOperand(MCOperand::CreateReg(Inst.getOperand(0).getReg() +
+    TmpInst.addOperand(MCOperand::createReg(Inst.getOperand(0).getReg() +
                                             Spacing * 3));
     TmpInst.addOperand(Inst.getOperand(3)); // CondCode
     TmpInst.addOperand(Inst.getOperand(4));
@@ -7817,13 +7866,13 @@ bool ARMAsmParser::processInstruction(MCInst &Inst,
     TmpInst.addOperand(Inst.getOperand(1)); // Rn
     TmpInst.addOperand(Inst.getOperand(1)); // Rn_wb == tied Rn
     TmpInst.addOperand(Inst.getOperand(2)); // alignment
-    TmpInst.addOperand(MCOperand::CreateReg(0)); // Rm
+    TmpInst.addOperand(MCOperand::createReg(0)); // Rm
     TmpInst.addOperand(Inst.getOperand(0)); // Vd
-    TmpInst.addOperand(MCOperand::CreateReg(Inst.getOperand(0).getReg() +
+    TmpInst.addOperand(MCOperand::createReg(Inst.getOperand(0).getReg() +
                                             Spacing));
-    TmpInst.addOperand(MCOperand::CreateReg(Inst.getOperand(0).getReg() +
+    TmpInst.addOperand(MCOperand::createReg(Inst.getOperand(0).getReg() +
                                             Spacing * 2));
-    TmpInst.addOperand(MCOperand::CreateReg(Inst.getOperand(0).getReg() +
+    TmpInst.addOperand(MCOperand::createReg(Inst.getOperand(0).getReg() +
                                             Spacing * 3));
     TmpInst.addOperand(Inst.getOperand(3)); // CondCode
     TmpInst.addOperand(Inst.getOperand(4));
@@ -7845,11 +7894,11 @@ bool ARMAsmParser::processInstruction(MCInst &Inst,
     TmpInst.addOperand(Inst.getOperand(2)); // alignment
     TmpInst.addOperand(Inst.getOperand(3)); // Rm
     TmpInst.addOperand(Inst.getOperand(0)); // Vd
-    TmpInst.addOperand(MCOperand::CreateReg(Inst.getOperand(0).getReg() +
+    TmpInst.addOperand(MCOperand::createReg(Inst.getOperand(0).getReg() +
                                             Spacing));
-    TmpInst.addOperand(MCOperand::CreateReg(Inst.getOperand(0).getReg() +
+    TmpInst.addOperand(MCOperand::createReg(Inst.getOperand(0).getReg() +
                                             Spacing * 2));
-    TmpInst.addOperand(MCOperand::CreateReg(Inst.getOperand(0).getReg() +
+    TmpInst.addOperand(MCOperand::createReg(Inst.getOperand(0).getReg() +
                                             Spacing * 3));
     TmpInst.addOperand(Inst.getOperand(4)); // CondCode
     TmpInst.addOperand(Inst.getOperand(5));
@@ -7913,14 +7962,14 @@ bool ARMAsmParser::processInstruction(MCInst &Inst,
     TmpInst.setOpcode(newOpc);
     TmpInst.addOperand(Inst.getOperand(0)); // Rd
     if (isNarrow)
-      TmpInst.addOperand(MCOperand::CreateReg(
+      TmpInst.addOperand(MCOperand::createReg(
           Inst.getOpcode() == ARM::t2MOVSsr ? ARM::CPSR : 0));
     TmpInst.addOperand(Inst.getOperand(1)); // Rn
     TmpInst.addOperand(Inst.getOperand(2)); // Rm
     TmpInst.addOperand(Inst.getOperand(4)); // CondCode
     TmpInst.addOperand(Inst.getOperand(5));
     if (!isNarrow)
-      TmpInst.addOperand(MCOperand::CreateReg(
+      TmpInst.addOperand(MCOperand::createReg(
           Inst.getOpcode() == ARM::t2MOVSsr ? ARM::CPSR : 0));
     Inst = TmpInst;
     return true;
@@ -7950,15 +7999,15 @@ bool ARMAsmParser::processInstruction(MCInst &Inst,
     TmpInst.setOpcode(newOpc);
     TmpInst.addOperand(Inst.getOperand(0)); // Rd
     if (isNarrow)
-      TmpInst.addOperand(MCOperand::CreateReg(
+      TmpInst.addOperand(MCOperand::createReg(
           Inst.getOpcode() == ARM::t2MOVSsi ? ARM::CPSR : 0));
     TmpInst.addOperand(Inst.getOperand(1)); // Rn
     if (newOpc != ARM::t2RRX)
-      TmpInst.addOperand(MCOperand::CreateImm(Amount));
+      TmpInst.addOperand(MCOperand::createImm(Amount));
     TmpInst.addOperand(Inst.getOperand(3)); // CondCode
     TmpInst.addOperand(Inst.getOperand(4));
     if (!isNarrow)
-      TmpInst.addOperand(MCOperand::CreateReg(
+      TmpInst.addOperand(MCOperand::createReg(
           Inst.getOpcode() == ARM::t2MOVSsi ? ARM::CPSR : 0));
     Inst = TmpInst;
     return true;
@@ -7982,7 +8031,7 @@ bool ARMAsmParser::processInstruction(MCInst &Inst,
     TmpInst.addOperand(Inst.getOperand(0)); // Rd
     TmpInst.addOperand(Inst.getOperand(1)); // Rn
     TmpInst.addOperand(Inst.getOperand(2)); // Rm
-    TmpInst.addOperand(MCOperand::CreateImm(Shifter)); // Shift value and ty
+    TmpInst.addOperand(MCOperand::createImm(Shifter)); // Shift value and ty
     TmpInst.addOperand(Inst.getOperand(3)); // CondCode
     TmpInst.addOperand(Inst.getOperand(4));
     TmpInst.addOperand(Inst.getOperand(5)); // cc_out
@@ -8013,7 +8062,7 @@ bool ARMAsmParser::processInstruction(MCInst &Inst,
     TmpInst.addOperand(Inst.getOperand(0)); // Rd
     TmpInst.addOperand(Inst.getOperand(1)); // Rn
     if (Opc == ARM::MOVsi)
-      TmpInst.addOperand(MCOperand::CreateImm(Shifter)); // Shift value and ty
+      TmpInst.addOperand(MCOperand::createImm(Shifter)); // Shift value and ty
     TmpInst.addOperand(Inst.getOperand(3)); // CondCode
     TmpInst.addOperand(Inst.getOperand(4));
     TmpInst.addOperand(Inst.getOperand(5)); // cc_out
@@ -8026,7 +8075,7 @@ bool ARMAsmParser::processInstruction(MCInst &Inst,
     TmpInst.setOpcode(ARM::MOVsi);
     TmpInst.addOperand(Inst.getOperand(0)); // Rd
     TmpInst.addOperand(Inst.getOperand(1)); // Rn
-    TmpInst.addOperand(MCOperand::CreateImm(Shifter)); // Shift value and ty
+    TmpInst.addOperand(MCOperand::createImm(Shifter)); // Shift value and ty
     TmpInst.addOperand(Inst.getOperand(2)); // CondCode
     TmpInst.addOperand(Inst.getOperand(3));
     TmpInst.addOperand(Inst.getOperand(4)); // cc_out
@@ -8043,7 +8092,7 @@ bool ARMAsmParser::processInstruction(MCInst &Inst,
     TmpInst.addOperand(Inst.getOperand(4)); // Rt
     TmpInst.addOperand(Inst.getOperand(0)); // Rn_wb
     TmpInst.addOperand(Inst.getOperand(1)); // Rn
-    TmpInst.addOperand(MCOperand::CreateImm(4));
+    TmpInst.addOperand(MCOperand::createImm(4));
     TmpInst.addOperand(Inst.getOperand(2)); // CondCode
     TmpInst.addOperand(Inst.getOperand(3));
     Inst = TmpInst;
@@ -8059,7 +8108,7 @@ bool ARMAsmParser::processInstruction(MCInst &Inst,
     TmpInst.addOperand(Inst.getOperand(0)); // Rn_wb
     TmpInst.addOperand(Inst.getOperand(4)); // Rt
     TmpInst.addOperand(Inst.getOperand(1)); // Rn
-    TmpInst.addOperand(MCOperand::CreateImm(-4));
+    TmpInst.addOperand(MCOperand::createImm(-4));
     TmpInst.addOperand(Inst.getOperand(2)); // CondCode
     TmpInst.addOperand(Inst.getOperand(3));
     Inst = TmpInst;
@@ -8075,8 +8124,8 @@ bool ARMAsmParser::processInstruction(MCInst &Inst,
       TmpInst.addOperand(Inst.getOperand(4)); // Rt
       TmpInst.addOperand(Inst.getOperand(0)); // Rn_wb
       TmpInst.addOperand(Inst.getOperand(1)); // Rn
-      TmpInst.addOperand(MCOperand::CreateReg(0));  // am2offset
-      TmpInst.addOperand(MCOperand::CreateImm(4));
+      TmpInst.addOperand(MCOperand::createReg(0));  // am2offset
+      TmpInst.addOperand(MCOperand::createImm(4));
       TmpInst.addOperand(Inst.getOperand(2)); // CondCode
       TmpInst.addOperand(Inst.getOperand(3));
       Inst = TmpInst;
@@ -8093,7 +8142,7 @@ bool ARMAsmParser::processInstruction(MCInst &Inst,
       TmpInst.addOperand(Inst.getOperand(0)); // Rn_wb
       TmpInst.addOperand(Inst.getOperand(4)); // Rt
       TmpInst.addOperand(Inst.getOperand(1)); // addrmode_imm12
-      TmpInst.addOperand(MCOperand::CreateImm(-4));
+      TmpInst.addOperand(MCOperand::createImm(-4));
       TmpInst.addOperand(Inst.getOperand(2)); // CondCode
       TmpInst.addOperand(Inst.getOperand(3));
       Inst = TmpInst;
@@ -8106,7 +8155,7 @@ bool ARMAsmParser::processInstruction(MCInst &Inst,
         ARM_AM::getT2SOImmVal(Inst.getOperand(2).getImm()) == -1)
       break;
     Inst.setOpcode(ARM::t2ADDri);
-    Inst.addOperand(MCOperand::CreateReg(0)); // cc_out
+    Inst.addOperand(MCOperand::createReg(0)); // cc_out
     break;
   case ARM::t2SUBri12:
     // If the immediate fits for encoding T3 (t2SUBri) and the generic "sub"
@@ -8115,7 +8164,7 @@ bool ARMAsmParser::processInstruction(MCInst &Inst,
         ARM_AM::getT2SOImmVal(Inst.getOperand(2).getImm()) == -1)
       break;
     Inst.setOpcode(ARM::t2SUBri);
-    Inst.addOperand(MCOperand::CreateReg(0)); // cc_out
+    Inst.addOperand(MCOperand::createReg(0)); // cc_out
     break;
   case ARM::tADDi8:
     // If the immediate is in the range 0-7, we want tADDi3 iff Rd was
@@ -8167,8 +8216,16 @@ bool ARMAsmParser::processInstruction(MCInst &Inst,
     // If the destination and first source operand are the same, and
     // there's no setting of the flags, use encoding T2 instead of T3.
     // Note that this is only for ADD, not SUB. This mirrors the system
-    // 'as' behaviour. Make sure the wide encoding wasn't explicit.
-    if (Inst.getOperand(0).getReg() != Inst.getOperand(1).getReg() ||
+    // 'as' behaviour.  Also take advantage of ADD being commutative.
+    // Make sure the wide encoding wasn't explicit.
+    bool Swap = false;
+    auto DestReg = Inst.getOperand(0).getReg();
+    bool Transform = DestReg == Inst.getOperand(1).getReg();
+    if (!Transform && DestReg == Inst.getOperand(2).getReg()) {
+      Transform = true;
+      Swap = true;
+    }
+    if (!Transform ||
         Inst.getOperand(5).getReg() != 0 ||
         (static_cast<ARMOperand &>(*Operands[3]).isToken() &&
          static_cast<ARMOperand &>(*Operands[3]).getToken() == ".w"))
@@ -8177,7 +8234,7 @@ bool ARMAsmParser::processInstruction(MCInst &Inst,
     TmpInst.setOpcode(ARM::tADDhirr);
     TmpInst.addOperand(Inst.getOperand(0));
     TmpInst.addOperand(Inst.getOperand(0));
-    TmpInst.addOperand(Inst.getOperand(2));
+    TmpInst.addOperand(Inst.getOperand(Swap ? 1 : 2));
     TmpInst.addOperand(Inst.getOperand(3));
     TmpInst.addOperand(Inst.getOperand(4));
     Inst = TmpInst;
@@ -8188,7 +8245,7 @@ bool ARMAsmParser::processInstruction(MCInst &Inst,
     // same, we need to use the 32-bit encoding if it's available.
     if (Inst.getOperand(0).getReg() != Inst.getOperand(2).getReg()) {
       Inst.setOpcode(ARM::t2ADDrr);
-      Inst.addOperand(MCOperand::CreateReg(0)); // cc_out
+      Inst.addOperand(MCOperand::createReg(0)); // cc_out
       return true;
     }
     break;
@@ -8241,7 +8298,7 @@ bool ARMAsmParser::processInstruction(MCInst &Inst,
       // the writeback tied operand.
       if (hasWritebackToken)
         Inst.insert(Inst.begin(),
-                    MCOperand::CreateReg(Inst.getOperand(0).getReg()));
+                    MCOperand::createReg(Inst.getOperand(0).getReg()));
       return true;
     }
     break;
@@ -8270,8 +8327,8 @@ bool ARMAsmParser::processInstruction(MCInst &Inst,
     assert (isThumbTwo());
     Inst.setOpcode(ARM::t2LDMIA_UPD);
     // Add the base register and writeback operands.
-    Inst.insert(Inst.begin(), MCOperand::CreateReg(ARM::SP));
-    Inst.insert(Inst.begin(), MCOperand::CreateReg(ARM::SP));
+    Inst.insert(Inst.begin(), MCOperand::createReg(ARM::SP));
+    Inst.insert(Inst.begin(), MCOperand::createReg(ARM::SP));
     return true;
   }
   case ARM::tPUSH: {
@@ -8281,8 +8338,8 @@ bool ARMAsmParser::processInstruction(MCInst &Inst,
     assert (isThumbTwo());
     Inst.setOpcode(ARM::t2STMDB_UPD);
     // Add the base register and writeback operands.
-    Inst.insert(Inst.begin(), MCOperand::CreateReg(ARM::SP));
-    Inst.insert(Inst.begin(), MCOperand::CreateReg(ARM::SP));
+    Inst.insert(Inst.begin(), MCOperand::createReg(ARM::SP));
+    Inst.insert(Inst.begin(), MCOperand::createReg(ARM::SP));
     return true;
   }
   case ARM::t2MOVi: {
@@ -8897,7 +8954,7 @@ bool ARMAsmParser::parseDirectiveThumbFunc(SMLoc L) {
       }
 
       MCSymbol *Func =
-          getParser().getContext().GetOrCreateSymbol(Tok.getIdentifier());
+          getParser().getContext().getOrCreateSymbol(Tok.getIdentifier());
       getParser().getStreamer().EmitThumbFunc(Func);
       Parser.Lex(); // Consume the identifier token.
       return false;
@@ -9040,15 +9097,9 @@ bool ARMAsmParser::parseDirectiveUnreq(SMLoc L) {
 bool ARMAsmParser::parseDirectiveArch(SMLoc L) {
   StringRef Arch = getParser().parseStringToEndOfStatement().trim();
 
-  unsigned ID = StringSwitch<unsigned>(Arch)
-#define ARM_ARCH_NAME(NAME, ID, DEFAULT_CPU_NAME, DEFAULT_CPU_ARCH) \
-    .Case(NAME, ARM::ID)
-#define ARM_ARCH_ALIAS(NAME, ID) \
-    .Case(NAME, ARM::ID)
-#include "MCTargetDesc/ARMArchName.def"
-    .Default(ARM::INVALID_ARCH);
+  unsigned ID = ARMTargetParser::parseArch(Arch);
 
-  if (ID == ARM::INVALID_ARCH) {
+  if (ID == ARM::AK_INVALID) {
     Error(L, "Unknown arch name");
     return false;
   }
@@ -9175,100 +9226,34 @@ bool ARMAsmParser::parseDirectiveCPU(SMLoc L) {
   StringRef CPU = getParser().parseStringToEndOfStatement().trim();
   getTargetStreamer().emitTextAttribute(ARMBuildAttrs::CPU_name, CPU);
 
+  // FIXME: This is using table-gen data, but should be moved to
+  // ARMTargetParser once that is table-gen'd.
   if (!STI.isCPUStringValid(CPU)) {
     Error(L, "Unknown CPU name");
     return false;
   }
 
-  // FIXME: This switches the CPU features globally, therefore it might
-  // happen that code you would not expect to assemble will. For details
-  // see: http://llvm.org/bugs/show_bug.cgi?id=20757
-  STI.InitMCProcessorInfo(CPU, "");
-  STI.InitCPUSchedModel(CPU);
+  STI.setDefaultFeatures(CPU);
   setAvailableFeatures(ComputeAvailableFeatures(STI.getFeatureBits()));
 
   return false;
 }
-
-// FIXME: This is duplicated in getARMFPUFeatures() in
-// tools/clang/lib/Driver/Tools.cpp
-static const struct {
-  const unsigned ID;
-  const uint64_t Enabled;
-  const uint64_t Disabled;
-} FPUs[] = {
-    {/* ID */ ARM::VFP,
-     /* Enabled */ ARM::FeatureVFP2,
-     /* Disabled */ ARM::FeatureNEON},
-    {/* ID */ ARM::VFPV2,
-     /* Enabled */ ARM::FeatureVFP2,
-     /* Disabled */ ARM::FeatureNEON},
-    {/* ID */ ARM::VFPV3,
-     /* Enabled */ ARM::FeatureVFP2 | ARM::FeatureVFP3,
-     /* Disabled */ ARM::FeatureNEON | ARM::FeatureD16},
-    {/* ID */ ARM::VFPV3_D16,
-     /* Enable */ ARM::FeatureVFP2 | ARM::FeatureVFP3 | ARM::FeatureD16,
-     /* Disabled */ ARM::FeatureNEON},
-    {/* ID */ ARM::VFPV4,
-     /* Enabled */ ARM::FeatureVFP2 | ARM::FeatureVFP3 | ARM::FeatureVFP4,
-     /* Disabled */ ARM::FeatureNEON | ARM::FeatureD16},
-    {/* ID */ ARM::VFPV4_D16,
-     /* Enabled */ ARM::FeatureVFP2 | ARM::FeatureVFP3 | ARM::FeatureVFP4 |
-         ARM::FeatureD16,
-     /* Disabled */ ARM::FeatureNEON},
-    {/* ID */ ARM::FPV5_D16,
-     /* Enabled */ ARM::FeatureVFP2 | ARM::FeatureVFP3 | ARM::FeatureVFP4 |
-         ARM::FeatureFPARMv8 | ARM::FeatureD16,
-     /* Disabled */ ARM::FeatureNEON | ARM::FeatureCrypto},
-    {/* ID */ ARM::FP_ARMV8,
-     /* Enabled */ ARM::FeatureVFP2 | ARM::FeatureVFP3 | ARM::FeatureVFP4 |
-         ARM::FeatureFPARMv8,
-     /* Disabled */ ARM::FeatureNEON | ARM::FeatureCrypto | ARM::FeatureD16},
-    {/* ID */ ARM::NEON,
-     /* Enabled */ ARM::FeatureVFP2 | ARM::FeatureVFP3 | ARM::FeatureNEON,
-     /* Disabled */ ARM::FeatureD16},
-    {/* ID */ ARM::NEON_VFPV4,
-     /* Enabled */ ARM::FeatureVFP2 | ARM::FeatureVFP3 | ARM::FeatureVFP4 |
-         ARM::FeatureNEON,
-     /* Disabled */ ARM::FeatureD16},
-    {/* ID */ ARM::NEON_FP_ARMV8,
-     /* Enabled */ ARM::FeatureVFP2 | ARM::FeatureVFP3 | ARM::FeatureVFP4 |
-         ARM::FeatureFPARMv8 | ARM::FeatureNEON,
-     /* Disabled */ ARM::FeatureCrypto | ARM::FeatureD16},
-    {/* ID */ ARM::CRYPTO_NEON_FP_ARMV8,
-     /* Enabled */ ARM::FeatureVFP2 | ARM::FeatureVFP3 | ARM::FeatureVFP4 |
-         ARM::FeatureFPARMv8 | ARM::FeatureNEON | ARM::FeatureCrypto,
-     /* Disabled */ ARM::FeatureD16},
-    {ARM::SOFTVFP, 0, 0},
-};
-
 /// parseDirectiveFPU
 ///  ::= .fpu str
 bool ARMAsmParser::parseDirectiveFPU(SMLoc L) {
   SMLoc FPUNameLoc = getTok().getLoc();
   StringRef FPU = getParser().parseStringToEndOfStatement().trim();
 
-  unsigned ID = StringSwitch<unsigned>(FPU)
-#define ARM_FPU_NAME(NAME, ID) .Case(NAME, ARM::ID)
-#include "ARMFPUName.def"
-    .Default(ARM::INVALID_FPU);
-
-  if (ID == ARM::INVALID_FPU) {
+  unsigned ID = ARMTargetParser::parseFPU(FPU);
+  std::vector<const char *> Features;
+  if (!ARMTargetParser::getFPUFeatures(ID, Features)) {
     Error(FPUNameLoc, "Unknown FPU name");
     return false;
   }
 
-  for (const auto &Entry : FPUs) {
-    if (Entry.ID != ID)
-      continue;
-
-    // Need to toggle features that should be on but are off and that
-    // should off but are on.
-    uint64_t Toggle = (Entry.Enabled & ~STI.getFeatureBits()) |
-                      (Entry.Disabled & STI.getFeatureBits());
-    setAvailableFeatures(ComputeAvailableFeatures(STI.ToggleFeature(Toggle)));
-    break;
-  }
+  for (auto Feature : Features)
+    STI.ApplyFeatureFlag(Feature);
+  setAvailableFeatures(ComputeAvailableFeatures(STI.getFeatureBits()));
 
   getTargetStreamer().emitFPU(ID);
   return false;
@@ -9372,7 +9357,7 @@ bool ARMAsmParser::parseDirectivePersonality(SMLoc L) {
   StringRef Name(Parser.getTok().getIdentifier());
   Parser.Lex();
 
-  MCSymbol *PR = getParser().getContext().GetOrCreateSymbol(Name);
+  MCSymbol *PR = getParser().getContext().getOrCreateSymbol(Name);
   getTargetStreamer().emitPersonality(PR);
   return false;
 }
@@ -9813,7 +9798,7 @@ bool ARMAsmParser::parseDirectiveTLSDescSeq(SMLoc L) {
   }
 
   const MCSymbolRefExpr *SRE =
-    MCSymbolRefExpr::Create(Parser.getTok().getIdentifier(),
+    MCSymbolRefExpr::create(Parser.getTok().getIdentifier(),
                             MCSymbolRefExpr::VK_ARM_TLSDESCSEQ, getContext());
   Lex();
 
@@ -9905,17 +9890,9 @@ bool ARMAsmParser::parseDirectiveObjectArch(SMLoc L) {
   SMLoc ArchLoc = Parser.getTok().getLoc();
   getLexer().Lex();
 
-  unsigned ID = StringSwitch<unsigned>(Arch)
-#define ARM_ARCH_NAME(NAME, ID, DEFAULT_CPU_NAME, DEFAULT_CPU_ARCH) \
-    .Case(NAME, ARM::ID)
-#define ARM_ARCH_ALIAS(NAME, ID) \
-    .Case(NAME, ARM::ID)
-#include "MCTargetDesc/ARMArchName.def"
-#undef ARM_ARCH_NAME
-#undef ARM_ARCH_ALIAS
-    .Default(ARM::INVALID_ARCH);
-
-  if (ID == ARM::INVALID_ARCH) {
+  unsigned ID = ARMTargetParser::parseArch(Arch);
+
+  if (ID == ARM::AK_INVALID) {
     Error(ArchLoc, "unknown architecture '" + Arch + "'");
     Parser.eatToEndOfStatement();
     return false;
@@ -9967,22 +9944,13 @@ bool ARMAsmParser::parseDirectiveThumbSet(SMLoc L) {
   }
   Lex();
 
+  MCSymbol *Sym;
   const MCExpr *Value;
-  if (Parser.parseExpression(Value)) {
-    TokError("missing expression");
-    Parser.eatToEndOfStatement();
-    return false;
-  }
-
-  if (getLexer().isNot(AsmToken::EndOfStatement)) {
-    TokError("unexpected token");
-    Parser.eatToEndOfStatement();
-    return false;
-  }
-  Lex();
+  if (MCParserUtils::parseAssignmentExpression(Name, /* allow_redef */ true,
+                                               Parser, Sym, Value))
+    return true;
 
-  MCSymbol *Alias = getContext().GetOrCreateSymbol(Name);
-  getTargetStreamer().emitThumbSet(Alias, Value);
+  getTargetStreamer().emitThumbSet(Sym, Value);
   return false;
 }
 
@@ -9999,33 +9967,32 @@ extern "C" void LLVMInitializeARMAsmParser() {
 #define GET_MATCHER_IMPLEMENTATION
 #include "ARMGenAsmMatcher.inc"
 
+// FIXME: This structure should be moved inside ARMTargetParser
+// when we start to table-generate them, and we can use the ARM
+// flags below, that were generated by table-gen.
 static const struct {
-  const char *Name;
+  const unsigned Kind;
   const unsigned ArchCheck;
-  const uint64_t Features;
+  const FeatureBitset Features;
 } Extensions[] = {
-  { "crc", Feature_HasV8, ARM::FeatureCRC },
-  { "crypto",  Feature_HasV8,
-    ARM::FeatureCrypto | ARM::FeatureNEON | ARM::FeatureFPARMv8 },
-  { "fp", Feature_HasV8, ARM::FeatureFPARMv8 },
-  { "idiv", Feature_HasV7 | Feature_IsNotMClass,
-    ARM::FeatureHWDiv | ARM::FeatureHWDivARM },
-  // FIXME: iWMMXT not supported
-  { "iwmmxt", Feature_None, 0 },
-  // FIXME: iWMMXT2 not supported
-  { "iwmmxt2", Feature_None, 0 },
-  // FIXME: Maverick not supported
-  { "maverick", Feature_None, 0 },
-  { "mp", Feature_HasV7 | Feature_IsNotMClass, ARM::FeatureMP },
-  // FIXME: ARMv6-m OS Extensions feature not checked
-  { "os", Feature_None, 0 },
+  { ARM::AEK_CRC, Feature_HasV8, {ARM::FeatureCRC} },
+  { ARM::AEK_CRYPTO,  Feature_HasV8,
+    {ARM::FeatureCrypto, ARM::FeatureNEON, ARM::FeatureFPARMv8} },
+  { ARM::AEK_FP, Feature_HasV8, {ARM::FeatureFPARMv8} },
+  { (ARM::AEK_HWDIV | ARM::AEK_HWDIVARM), Feature_HasV7 | Feature_IsNotMClass,
+    {ARM::FeatureHWDiv, ARM::FeatureHWDivARM} },
+  { ARM::AEK_MP, Feature_HasV7 | Feature_IsNotMClass, {ARM::FeatureMP} },
+  { ARM::AEK_SIMD, Feature_HasV8, {ARM::FeatureNEON, ARM::FeatureFPARMv8} },
   // FIXME: Also available in ARMv6-K
-  { "sec", Feature_HasV7, ARM::FeatureTrustZone },
-  { "simd", Feature_HasV8, ARM::FeatureNEON | ARM::FeatureFPARMv8 },
+  { ARM::AEK_SEC, Feature_HasV7, {ARM::FeatureTrustZone} },
   // FIXME: Only available in A-class, isel not predicated
-  { "virt", Feature_HasV7, ARM::FeatureVirtualization },
-  // FIXME: xscale not supported
-  { "xscale", Feature_None, 0 },
+  { ARM::AEK_VIRT, Feature_HasV7, {ARM::FeatureVirtualization} },
+  // FIXME: Unsupported extensions.
+  { ARM::AEK_OS, Feature_None, {} },
+  { ARM::AEK_IWMMXT, Feature_None, {} },
+  { ARM::AEK_IWMMXT2, Feature_None, {} },
+  { ARM::AEK_MAVERICK, Feature_None, {} },
+  { ARM::AEK_XSCALE, Feature_None, {} },
 };
 
 /// parseDirectiveArchExtension
@@ -10048,12 +10015,15 @@ bool ARMAsmParser::parseDirectiveArchExtension(SMLoc L) {
     EnableFeature = false;
     Name = Name.substr(2);
   }
+  unsigned FeatureKind = ARMTargetParser::parseArchExt(Name);
+  if (FeatureKind == ARM::AEK_INVALID)
+    Error(ExtLoc, "unknown architectural extension: " + Name);
 
   for (const auto &Extension : Extensions) {
-    if (Extension.Name != Name)
+    if (Extension.Kind != FeatureKind)
       continue;
 
-    if (!Extension.Features)
+    if (Extension.Features.none())
       report_fatal_error("unsupported architectural extension: " + Name);
 
     if ((getAvailableFeatures() & Extension.ArchCheck) != Extension.ArchCheck) {
@@ -10062,9 +10032,10 @@ bool ARMAsmParser::parseDirectiveArchExtension(SMLoc L) {
       return false;
     }
 
-    uint64_t ToggleFeatures = EnableFeature
-                                  ? (~STI.getFeatureBits() & Extension.Features)
-                                  : ( STI.getFeatureBits() & Extension.Features);
+    FeatureBitset ToggleFeatures = EnableFeature
+      ? (~STI.getFeatureBits() & Extension.Features)
+      : ( STI.getFeatureBits() & Extension.Features);
+
     uint64_t Features =
         ComputeAvailableFeatures(STI.ToggleFeature(ToggleFeatures));
     setAvailableFeatures(Features);
@@ -10096,7 +10067,7 @@ unsigned ARMAsmParser::validateTargetOperandClass(MCParsedAsmOperand &AsmOp,
     if (Op.isImm()) {
       const MCExpr *SOExpr = Op.getImm();
       int64_t Value;
-      if (!SOExpr->EvaluateAsAbsolute(Value))
+      if (!SOExpr->evaluateAsAbsolute(Value))
         return Match_Success;
       assert((Value >= INT32_MIN && Value <= UINT32_MAX) &&
              "expression value must be representable in 32 bits");
diff --git a/lib/Target/ARM/CMakeLists.txt b/lib/Target/ARM/CMakeLists.txt
index e1e244a590e..0b6981979cf 100644
--- a/lib/Target/ARM/CMakeLists.txt
+++ b/lib/Target/ARM/CMakeLists.txt
@@ -30,8 +30,6 @@ add_llvm_target(ARMCodeGen
   ARMLoadStoreOptimizer.cpp
   ARMMCInstLower.cpp
   ARMMachineFunctionInfo.cpp
-  ARMNaClHeaders.cpp
-  ARMNaClRewritePass.cpp
   ARMRegisterInfo.cpp
   ARMOptimizeBarriersPass.cpp
   ARMSelectionDAGInfo.cpp
diff --git a/lib/Target/ARM/Disassembler/ARMDisassembler.cpp b/lib/Target/ARM/Disassembler/ARMDisassembler.cpp
index 4c169a8ef55..097ec04e705 100644
--- a/lib/Target/ARM/Disassembler/ARMDisassembler.cpp
+++ b/lib/Target/ARM/Disassembler/ARMDisassembler.cpp
@@ -435,7 +435,7 @@ DecodeStatus ARMDisassembler::getInstruction(MCInst &MI, uint64_t &Size,
                                              raw_ostream &CS) const {
   CommentStream = &CS;
 
-  assert(!(STI.getFeatureBits() & ARM::ModeThumb) &&
+  assert(!STI.getFeatureBits()[ARM::ModeThumb] &&
          "Asked to disassemble an ARM instruction but Subtarget is in Thumb "
          "mode!");
 
@@ -582,12 +582,12 @@ static void AddThumb1SBit(MCInst &MI, bool InITBlock) {
     if (I == MI.end()) break;
     if (OpInfo[i].isOptionalDef() && OpInfo[i].RegClass == ARM::CCRRegClassID) {
       if (i > 0 && OpInfo[i-1].isPredicate()) continue;
-      MI.insert(I, MCOperand::CreateReg(InITBlock ? 0 : ARM::CPSR));
+      MI.insert(I, MCOperand::createReg(InITBlock ? 0 : ARM::CPSR));
       return;
     }
   }
 
-  MI.insert(I, MCOperand::CreateReg(InITBlock ? 0 : ARM::CPSR));
+  MI.insert(I, MCOperand::createReg(InITBlock ? 0 : ARM::CPSR));
 }
 
 // Most Thumb instructions don't have explicit predicates in the
@@ -646,22 +646,22 @@ ThumbDisassembler::AddThumbPredicate(MCInst &MI) const {
   for (unsigned i = 0; i < NumOps; ++i, ++I) {
     if (I == MI.end()) break;
     if (OpInfo[i].isPredicate()) {
-      I = MI.insert(I, MCOperand::CreateImm(CC));
+      I = MI.insert(I, MCOperand::createImm(CC));
       ++I;
       if (CC == ARMCC::AL)
-        MI.insert(I, MCOperand::CreateReg(0));
+        MI.insert(I, MCOperand::createReg(0));
       else
-        MI.insert(I, MCOperand::CreateReg(ARM::CPSR));
+        MI.insert(I, MCOperand::createReg(ARM::CPSR));
       return S;
     }
   }
 
-  I = MI.insert(I, MCOperand::CreateImm(CC));
+  I = MI.insert(I, MCOperand::createImm(CC));
   ++I;
   if (CC == ARMCC::AL)
-    MI.insert(I, MCOperand::CreateReg(0));
+    MI.insert(I, MCOperand::createReg(0));
   else
-    MI.insert(I, MCOperand::CreateReg(ARM::CPSR));
+    MI.insert(I, MCOperand::createReg(ARM::CPSR));
 
   return S;
 }
@@ -700,7 +700,7 @@ DecodeStatus ThumbDisassembler::getInstruction(MCInst &MI, uint64_t &Size,
                                                raw_ostream &CS) const {
   CommentStream = &CS;
 
-  assert((STI.getFeatureBits() & ARM::ModeThumb) &&
+  assert(STI.getFeatureBits()[ARM::ModeThumb] &&
          "Asked to disassemble in Thumb mode but Subtarget is in ARM mode!");
 
   // We want to read exactly 2 bytes of data.
@@ -894,7 +894,7 @@ static DecodeStatus DecodeGPRRegisterClass(MCInst &Inst, unsigned RegNo,
     return MCDisassembler::Fail;
 
   unsigned Register = GPRDecoderTable[RegNo];
-  Inst.addOperand(MCOperand::CreateReg(Register));
+  Inst.addOperand(MCOperand::createReg(Register));
   return MCDisassembler::Success;
 }
 
@@ -918,7 +918,7 @@ DecodeGPRwithAPSRRegisterClass(MCInst &Inst, unsigned RegNo,
 
   if (RegNo == 15)
   {
-    Inst.addOperand(MCOperand::CreateReg(ARM::APSR_NZCV));
+    Inst.addOperand(MCOperand::createReg(ARM::APSR_NZCV));
     return MCDisassembler::Success;
   }
 
@@ -949,7 +949,7 @@ static DecodeStatus DecodeGPRPairRegisterClass(MCInst &Inst, unsigned RegNo,
      S = MCDisassembler::SoftFail;
 
   unsigned RegisterPair = GPRPairDecoderTable[RegNo/2];
-  Inst.addOperand(MCOperand::CreateReg(RegisterPair));
+  Inst.addOperand(MCOperand::createReg(RegisterPair));
   return S;
 }
 
@@ -979,7 +979,7 @@ static DecodeStatus DecodetcGPRRegisterClass(MCInst &Inst, unsigned RegNo,
       return MCDisassembler::Fail;
     }
 
-  Inst.addOperand(MCOperand::CreateReg(Register));
+  Inst.addOperand(MCOperand::createReg(Register));
   return MCDisassembler::Success;
 }
 
@@ -1009,7 +1009,7 @@ static DecodeStatus DecodeSPRRegisterClass(MCInst &Inst, unsigned RegNo,
     return MCDisassembler::Fail;
 
   unsigned Register = SPRDecoderTable[RegNo];
-  Inst.addOperand(MCOperand::CreateReg(Register));
+  Inst.addOperand(MCOperand::createReg(Register));
   return MCDisassembler::Success;
 }
 
@@ -1026,15 +1026,16 @@ static const uint16_t DPRDecoderTable[] = {
 
 static DecodeStatus DecodeDPRRegisterClass(MCInst &Inst, unsigned RegNo,
                                    uint64_t Address, const void *Decoder) {
-  uint64_t featureBits = ((const MCDisassembler*)Decoder)->getSubtargetInfo()
-                                                          .getFeatureBits();
-  bool hasD16 = featureBits & ARM::FeatureD16;
+  const FeatureBitset &featureBits =
+    ((const MCDisassembler*)Decoder)->getSubtargetInfo().getFeatureBits();
+
+  bool hasD16 = featureBits[ARM::FeatureD16];
 
   if (RegNo > 31 || (hasD16 && RegNo > 15))
     return MCDisassembler::Fail;
 
   unsigned Register = DPRDecoderTable[RegNo];
-  Inst.addOperand(MCOperand::CreateReg(Register));
+  Inst.addOperand(MCOperand::createReg(Register));
   return MCDisassembler::Success;
 }
 
@@ -1068,7 +1069,7 @@ static DecodeStatus DecodeQPRRegisterClass(MCInst &Inst, unsigned RegNo,
   RegNo >>= 1;
 
   unsigned Register = QPRDecoderTable[RegNo];
-  Inst.addOperand(MCOperand::CreateReg(Register));
+  Inst.addOperand(MCOperand::createReg(Register));
   return MCDisassembler::Success;
 }
 
@@ -1087,7 +1088,7 @@ static DecodeStatus DecodeDPairRegisterClass(MCInst &Inst, unsigned RegNo,
     return MCDisassembler::Fail;
 
   unsigned Register = DPairDecoderTable[RegNo];
-  Inst.addOperand(MCOperand::CreateReg(Register));
+  Inst.addOperand(MCOperand::createReg(Register));
   return MCDisassembler::Success;
 }
 
@@ -1110,7 +1111,7 @@ static DecodeStatus DecodeDPairSpacedRegisterClass(MCInst &Inst,
     return MCDisassembler::Fail;
 
   unsigned Register = DPairSpacedDecoderTable[RegNo];
-  Inst.addOperand(MCOperand::CreateReg(Register));
+  Inst.addOperand(MCOperand::createReg(Register));
   return MCDisassembler::Success;
 }
 
@@ -1120,20 +1121,20 @@ static DecodeStatus DecodePredicateOperand(MCInst &Inst, unsigned Val,
   // AL predicate is not allowed on Thumb1 branches.
   if (Inst.getOpcode() == ARM::tBcc && Val == 0xE)
     return MCDisassembler::Fail;
-  Inst.addOperand(MCOperand::CreateImm(Val));
+  Inst.addOperand(MCOperand::createImm(Val));
   if (Val == ARMCC::AL) {
-    Inst.addOperand(MCOperand::CreateReg(0));
+    Inst.addOperand(MCOperand::createReg(0));
   } else
-    Inst.addOperand(MCOperand::CreateReg(ARM::CPSR));
+    Inst.addOperand(MCOperand::createReg(ARM::CPSR));
   return MCDisassembler::Success;
 }
 
 static DecodeStatus DecodeCCOutOperand(MCInst &Inst, unsigned Val,
                                uint64_t Address, const void *Decoder) {
   if (Val)
-    Inst.addOperand(MCOperand::CreateReg(ARM::CPSR));
+    Inst.addOperand(MCOperand::createReg(ARM::CPSR));
   else
-    Inst.addOperand(MCOperand::CreateReg(0));
+    Inst.addOperand(MCOperand::createReg(0));
   return MCDisassembler::Success;
 }
 
@@ -1169,7 +1170,7 @@ static DecodeStatus DecodeSORegImmOperand(MCInst &Inst, unsigned Val,
     Shift = ARM_AM::rrx;
 
   unsigned Op = Shift | (imm << 3);
-  Inst.addOperand(MCOperand::CreateImm(Op));
+  Inst.addOperand(MCOperand::createImm(Op));
 
   return S;
 }
@@ -1204,7 +1205,7 @@ static DecodeStatus DecodeSORegRegOperand(MCInst &Inst, unsigned Val,
       break;
   }
 
-  Inst.addOperand(MCOperand::CreateImm(Shift));
+  Inst.addOperand(MCOperand::createImm(Shift));
 
   return S;
 }
@@ -1318,7 +1319,7 @@ static DecodeStatus DecodeBitfieldMaskOperand(MCInst &Inst, unsigned Val,
   if (msb != 31) msb_mask = (1U << (msb+1)) - 1;
   uint32_t lsb_mask = (1U << lsb) - 1;
 
-  Inst.addOperand(MCOperand::CreateImm(~(msb_mask ^ lsb_mask)));
+  Inst.addOperand(MCOperand::createImm(~(msb_mask ^ lsb_mask)));
   return S;
 }
 
@@ -1373,13 +1374,13 @@ static DecodeStatus DecodeCopMemInstruction(MCInst &Inst, unsigned Insn,
       break;
   }
 
-  uint64_t featureBits = ((const MCDisassembler*)Decoder)->getSubtargetInfo()
-                                                          .getFeatureBits();
-  if ((featureBits & ARM::HasV8Ops) && (coproc != 14))
+  const FeatureBitset &featureBits =
+    ((const MCDisassembler*)Decoder)->getSubtargetInfo().getFeatureBits();
+  if (featureBits[ARM::HasV8Ops] && (coproc != 14))
     return MCDisassembler::Fail;
 
-  Inst.addOperand(MCOperand::CreateImm(coproc));
-  Inst.addOperand(MCOperand::CreateImm(CRd));
+  Inst.addOperand(MCOperand::createImm(coproc));
+  Inst.addOperand(MCOperand::createImm(CRd));
   if (!Check(S, DecodeGPRRegisterClass(Inst, Rn, Address, Decoder)))
     return MCDisassembler::Fail;
 
@@ -1417,7 +1418,7 @@ static DecodeStatus DecodeCopMemInstruction(MCInst &Inst, unsigned Insn,
     case ARM::STC_PRE:
     case ARM::STCL_PRE:
       imm = ARM_AM::getAM5Opc(U ? ARM_AM::add : ARM_AM::sub, imm);
-      Inst.addOperand(MCOperand::CreateImm(imm));
+      Inst.addOperand(MCOperand::createImm(imm));
       break;
     case ARM::t2LDC2_POST:
     case ARM::t2LDC2L_POST:
@@ -1440,7 +1441,7 @@ static DecodeStatus DecodeCopMemInstruction(MCInst &Inst, unsigned Insn,
     default:
       // The 'option' variant doesn't encode 'U' in the immediate since
       // the immediate is unsigned [0,255].
-      Inst.addOperand(MCOperand::CreateImm(imm));
+      Inst.addOperand(MCOperand::createImm(imm));
       break;
   }
 
@@ -1564,11 +1565,11 @@ DecodeAddrMode2IdxInstruction(MCInst &Inst, unsigned Insn,
       Opc = ARM_AM::rrx;
     unsigned imm = ARM_AM::getAM2Opc(Op, amt, Opc, idx_mode);
 
-    Inst.addOperand(MCOperand::CreateImm(imm));
+    Inst.addOperand(MCOperand::createImm(imm));
   } else {
-    Inst.addOperand(MCOperand::CreateReg(0));
+    Inst.addOperand(MCOperand::createReg(0));
     unsigned tmp = ARM_AM::getAM2Opc(Op, imm, ARM_AM::lsl, idx_mode);
-    Inst.addOperand(MCOperand::CreateImm(tmp));
+    Inst.addOperand(MCOperand::createImm(tmp));
   }
 
   if (!Check(S, DecodePredicateOperand(Inst, pred, Address, Decoder)))
@@ -1615,7 +1616,7 @@ static DecodeStatus DecodeSORegMemOperand(MCInst &Inst, unsigned Val,
     shift = ARM_AM::getAM2Opc(ARM_AM::add, imm, ShOp);
   else
     shift = ARM_AM::getAM2Opc(ARM_AM::sub, imm, ShOp);
-  Inst.addOperand(MCOperand::CreateImm(shift));
+  Inst.addOperand(MCOperand::createImm(shift));
 
   return S;
 }
@@ -1798,12 +1799,12 @@ DecodeAddrMode3Instruction(MCInst &Inst, unsigned Insn,
     return MCDisassembler::Fail;
 
   if (type) {
-    Inst.addOperand(MCOperand::CreateReg(0));
-    Inst.addOperand(MCOperand::CreateImm(U | (imm << 4) | Rm));
+    Inst.addOperand(MCOperand::createReg(0));
+    Inst.addOperand(MCOperand::createImm(U | (imm << 4) | Rm));
   } else {
     if (!Check(S, DecodeGPRRegisterClass(Inst, Rm, Address, Decoder)))
     return MCDisassembler::Fail;
-    Inst.addOperand(MCOperand::CreateImm(U));
+    Inst.addOperand(MCOperand::createImm(U));
   }
 
   if (!Check(S, DecodePredicateOperand(Inst, pred, Address, Decoder)))
@@ -1834,7 +1835,7 @@ static DecodeStatus DecodeRFEInstruction(MCInst &Inst, unsigned Insn,
       break;
   }
 
-  Inst.addOperand(MCOperand::CreateImm(mode));
+  Inst.addOperand(MCOperand::createImm(mode));
   if (!Check(S, DecodeGPRRegisterClass(Inst, Rn, Address, Decoder)))
     return MCDisassembler::Fail;
 
@@ -1936,7 +1937,7 @@ static DecodeStatus DecodeMemMultipleWritebackInstruction(MCInst &Inst,
         return MCDisassembler::Fail;
 
       Inst.addOperand(
-          MCOperand::CreateImm(fieldFromInstruction(Insn, 0, 4)));
+          MCOperand::createImm(fieldFromInstruction(Insn, 0, 4)));
       return S;
     }
 
@@ -1980,22 +1981,22 @@ static DecodeStatus DecodeCPSInstruction(MCInst &Inst, unsigned Insn,
 
   if (imod && M) {
     Inst.setOpcode(ARM::CPS3p);
-    Inst.addOperand(MCOperand::CreateImm(imod));
-    Inst.addOperand(MCOperand::CreateImm(iflags));
-    Inst.addOperand(MCOperand::CreateImm(mode));
+    Inst.addOperand(MCOperand::createImm(imod));
+    Inst.addOperand(MCOperand::createImm(iflags));
+    Inst.addOperand(MCOperand::createImm(mode));
   } else if (imod && !M) {
     Inst.setOpcode(ARM::CPS2p);
-    Inst.addOperand(MCOperand::CreateImm(imod));
-    Inst.addOperand(MCOperand::CreateImm(iflags));
+    Inst.addOperand(MCOperand::createImm(imod));
+    Inst.addOperand(MCOperand::createImm(iflags));
     if (mode) S = MCDisassembler::SoftFail;
   } else if (!imod && M) {
     Inst.setOpcode(ARM::CPS1p);
-    Inst.addOperand(MCOperand::CreateImm(mode));
+    Inst.addOperand(MCOperand::createImm(mode));
     if (iflags) S = MCDisassembler::SoftFail;
   } else {
     // imod == '00' && M == '0' --> UNPREDICTABLE
     Inst.setOpcode(ARM::CPS1p);
-    Inst.addOperand(MCOperand::CreateImm(mode));
+    Inst.addOperand(MCOperand::createImm(mode));
     S = MCDisassembler::SoftFail;
   }
 
@@ -2020,17 +2021,17 @@ static DecodeStatus DecodeT2CPSInstruction(MCInst &Inst, unsigned Insn,
 
   if (imod && M) {
     Inst.setOpcode(ARM::t2CPS3p);
-    Inst.addOperand(MCOperand::CreateImm(imod));
-    Inst.addOperand(MCOperand::CreateImm(iflags));
-    Inst.addOperand(MCOperand::CreateImm(mode));
+    Inst.addOperand(MCOperand::createImm(imod));
+    Inst.addOperand(MCOperand::createImm(iflags));
+    Inst.addOperand(MCOperand::createImm(mode));
   } else if (imod && !M) {
     Inst.setOpcode(ARM::t2CPS2p);
-    Inst.addOperand(MCOperand::CreateImm(imod));
-    Inst.addOperand(MCOperand::CreateImm(iflags));
+    Inst.addOperand(MCOperand::createImm(imod));
+    Inst.addOperand(MCOperand::createImm(iflags));
     if (mode) S = MCDisassembler::SoftFail;
   } else if (!imod && M) {
     Inst.setOpcode(ARM::t2CPS1p);
-    Inst.addOperand(MCOperand::CreateImm(mode));
+    Inst.addOperand(MCOperand::createImm(mode));
     if (iflags) S = MCDisassembler::SoftFail;
   } else {
     // imod == '00' && M == '0' --> this is a HINT instruction
@@ -2038,7 +2039,7 @@ static DecodeStatus DecodeT2CPSInstruction(MCInst &Inst, unsigned Insn,
     // HINT are defined only for immediate in [0..4]
     if(imm > 4) return MCDisassembler::Fail;
     Inst.setOpcode(ARM::t2HINT);
-    Inst.addOperand(MCOperand::CreateImm(imm));
+    Inst.addOperand(MCOperand::createImm(imm));
   }
 
   return S;
@@ -2063,7 +2064,7 @@ static DecodeStatus DecodeT2MOVTWInstruction(MCInst &Inst, unsigned Insn,
     return MCDisassembler::Fail;
 
   if (!tryAddingSymbolicOperand(Address, imm, false, 4, Inst, Decoder))
-    Inst.addOperand(MCOperand::CreateImm(imm));
+    Inst.addOperand(MCOperand::createImm(imm));
 
   return S;
 }
@@ -2087,7 +2088,7 @@ static DecodeStatus DecodeArmMOVTWInstruction(MCInst &Inst, unsigned Insn,
     return MCDisassembler::Fail;
 
   if (!tryAddingSymbolicOperand(Address, imm, false, 4, Inst, Decoder))
-    Inst.addOperand(MCOperand::CreateImm(imm));
+    Inst.addOperand(MCOperand::createImm(imm));
 
   if (!Check(S, DecodePredicateOperand(Inst, pred, Address, Decoder)))
     return MCDisassembler::Fail;
@@ -2151,9 +2152,10 @@ static DecodeStatus DecodeSETPANInstruction(MCInst &Inst, unsigned Insn,
   unsigned Imm = fieldFromInstruction(Insn, 9, 1);
 
   const MCDisassembler *Dis = static_cast<const MCDisassembler*>(Decoder);
-  uint64_t FeatureBits = Dis->getSubtargetInfo().getFeatureBits();
-  if ((FeatureBits & ARM::HasV8_1aOps) == 0 || 
-      (FeatureBits & ARM::HasV8Ops) == 0 )
+  const FeatureBitset &FeatureBits = Dis->getSubtargetInfo().getFeatureBits();
+
+  if (!FeatureBits[ARM::HasV8_1aOps] || 
+      !FeatureBits[ARM::HasV8Ops])
     return MCDisassembler::Fail;
 
   // Decoder can be called from DecodeTST, which does not check the full
@@ -2166,7 +2168,7 @@ static DecodeStatus DecodeSETPANInstruction(MCInst &Inst, unsigned Insn,
     S = MCDisassembler::SoftFail;
 
   Inst.setOpcode(ARM::SETPAN);
-  Inst.addOperand(MCOperand::CreateImm(Imm));
+  Inst.addOperand(MCOperand::createImm(Imm));
 
   return S;
 }
@@ -2184,7 +2186,7 @@ static DecodeStatus DecodeAddrModeImm12Operand(MCInst &Inst, unsigned Val,
 
   if (!add) imm *= -1;
   if (imm == 0 && !add) imm = INT32_MIN;
-  Inst.addOperand(MCOperand::CreateImm(imm));
+  Inst.addOperand(MCOperand::createImm(imm));
   if (Rn == 15)
     tryAddingPcLoadReferenceComment(Address, Address + imm + 8, Decoder);
 
@@ -2203,9 +2205,9 @@ static DecodeStatus DecodeAddrMode5Operand(MCInst &Inst, unsigned Val,
     return MCDisassembler::Fail;
 
   if (U)
-    Inst.addOperand(MCOperand::CreateImm(ARM_AM::getAM5Opc(ARM_AM::add, imm)));
+    Inst.addOperand(MCOperand::createImm(ARM_AM::getAM5Opc(ARM_AM::add, imm)));
   else
-    Inst.addOperand(MCOperand::CreateImm(ARM_AM::getAM5Opc(ARM_AM::sub, imm)));
+    Inst.addOperand(MCOperand::createImm(ARM_AM::getAM5Opc(ARM_AM::sub, imm)));
 
   return S;
 }
@@ -2237,7 +2239,7 @@ DecodeT2BInstruction(MCInst &Inst, unsigned Insn,
   int imm32 = SignExtend32<25>(tmp << 1);
   if (!tryAddingSymbolicOperand(Address, Address + imm32 + 4,
                                 true, 4, Inst, Decoder))
-    Inst.addOperand(MCOperand::CreateImm(imm32));
+    Inst.addOperand(MCOperand::createImm(imm32));
 
   return Status;
 }
@@ -2255,13 +2257,13 @@ DecodeBranchImmInstruction(MCInst &Inst, unsigned Insn,
     imm |= fieldFromInstruction(Insn, 24, 1) << 1;
     if (!tryAddingSymbolicOperand(Address, Address + SignExtend32<26>(imm) + 8,
                                   true, 4, Inst, Decoder))
-    Inst.addOperand(MCOperand::CreateImm(SignExtend32<26>(imm)));
+    Inst.addOperand(MCOperand::createImm(SignExtend32<26>(imm)));
     return S;
   }
 
   if (!tryAddingSymbolicOperand(Address, Address + SignExtend32<26>(imm) + 8,
                                 true, 4, Inst, Decoder))
-    Inst.addOperand(MCOperand::CreateImm(SignExtend32<26>(imm)));
+    Inst.addOperand(MCOperand::createImm(SignExtend32<26>(imm)));
   if (!Check(S, DecodePredicateOperand(Inst, pred, Address, Decoder)))
     return MCDisassembler::Fail;
 
@@ -2279,9 +2281,9 @@ static DecodeStatus DecodeAddrMode6Operand(MCInst &Inst, unsigned Val,
   if (!Check(S, DecodeGPRRegisterClass(Inst, Rm, Address, Decoder)))
     return MCDisassembler::Fail;
   if (!align)
-    Inst.addOperand(MCOperand::CreateImm(0));
+    Inst.addOperand(MCOperand::createImm(0));
   else
-    Inst.addOperand(MCOperand::CreateImm(4 << align));
+    Inst.addOperand(MCOperand::createImm(4 << align));
 
   return S;
 }
@@ -2475,7 +2477,7 @@ static DecodeStatus DecodeVLDInstruction(MCInst &Inst, unsigned Insn,
     case ARM::VLD2b8wb_register:
     case ARM::VLD2b16wb_register:
     case ARM::VLD2b32wb_register:
-      Inst.addOperand(MCOperand::CreateImm(0));
+      Inst.addOperand(MCOperand::createImm(0));
       break;
     case ARM::VLD3d8_UPD:
     case ARM::VLD3d16_UPD:
@@ -2510,7 +2512,7 @@ static DecodeStatus DecodeVLDInstruction(MCInst &Inst, unsigned Insn,
     //
     // The fixed offset encodes as Rm == 0xd, so we check for that.
     if (Rm == 0xd) {
-      Inst.addOperand(MCOperand::CreateReg(0));
+      Inst.addOperand(MCOperand::createReg(0));
       break;
     }
     // Fall through to handle the register offset variant.
@@ -2676,7 +2678,7 @@ static DecodeStatus DecodeVSTInstruction(MCInst &Inst, unsigned Insn,
     case ARM::VST2b32wb_register:
       if (Rm == 0xF)
         return MCDisassembler::Fail;
-      Inst.addOperand(MCOperand::CreateImm(0));
+      Inst.addOperand(MCOperand::createImm(0));
       break;
     case ARM::VST3d8_UPD:
     case ARM::VST3d16_UPD:
@@ -2705,7 +2707,7 @@ static DecodeStatus DecodeVSTInstruction(MCInst &Inst, unsigned Insn,
   switch (Inst.getOpcode()) {
     default:
       if (Rm == 0xD)
-        Inst.addOperand(MCOperand::CreateReg(0));
+        Inst.addOperand(MCOperand::createReg(0));
       else if (Rm != 0xF) {
         if (!Check(S, DecodeGPRRegisterClass(Inst, Rm, Address, Decoder)))
           return MCDisassembler::Fail;
@@ -2917,7 +2919,7 @@ static DecodeStatus DecodeVLD1DupInstruction(MCInst &Inst, unsigned Insn,
 
   if (!Check(S, DecodeGPRRegisterClass(Inst, Rn, Address, Decoder)))
     return MCDisassembler::Fail;
-  Inst.addOperand(MCOperand::CreateImm(align));
+  Inst.addOperand(MCOperand::createImm(align));
 
   // The fixed offset post-increment encodes Rm == 0xd. The no-writeback
   // variant encodes Rm == 0xf. Anything else is a register offset post-
@@ -2963,11 +2965,11 @@ static DecodeStatus DecodeVLD2DupInstruction(MCInst &Inst, unsigned Insn,
   }
 
   if (Rm != 0xF)
-    Inst.addOperand(MCOperand::CreateImm(0));
+    Inst.addOperand(MCOperand::createImm(0));
 
   if (!Check(S, DecodeGPRRegisterClass(Inst, Rn, Address, Decoder)))
     return MCDisassembler::Fail;
-  Inst.addOperand(MCOperand::CreateImm(align));
+  Inst.addOperand(MCOperand::createImm(align));
 
   if (Rm != 0xD && Rm != 0xF) {
     if (!Check(S, DecodeGPRRegisterClass(Inst, Rm, Address, Decoder)))
@@ -3000,10 +3002,10 @@ static DecodeStatus DecodeVLD3DupInstruction(MCInst &Inst, unsigned Insn,
 
   if (!Check(S, DecodeGPRRegisterClass(Inst, Rn, Address, Decoder)))
     return MCDisassembler::Fail;
-  Inst.addOperand(MCOperand::CreateImm(0));
+  Inst.addOperand(MCOperand::createImm(0));
 
   if (Rm == 0xD)
-    Inst.addOperand(MCOperand::CreateReg(0));
+    Inst.addOperand(MCOperand::createReg(0));
   else if (Rm != 0xF) {
     if (!Check(S, DecodeGPRRegisterClass(Inst, Rm, Address, Decoder)))
       return MCDisassembler::Fail;
@@ -3052,10 +3054,10 @@ static DecodeStatus DecodeVLD4DupInstruction(MCInst &Inst, unsigned Insn,
 
   if (!Check(S, DecodeGPRRegisterClass(Inst, Rn, Address, Decoder)))
     return MCDisassembler::Fail;
-  Inst.addOperand(MCOperand::CreateImm(align));
+  Inst.addOperand(MCOperand::createImm(align));
 
   if (Rm == 0xD)
-    Inst.addOperand(MCOperand::CreateReg(0));
+    Inst.addOperand(MCOperand::createReg(0));
   else if (Rm != 0xF) {
     if (!Check(S, DecodeGPRRegisterClass(Inst, Rm, Address, Decoder)))
       return MCDisassembler::Fail;
@@ -3086,7 +3088,7 @@ DecodeNEONModImmInstruction(MCInst &Inst, unsigned Insn,
     return MCDisassembler::Fail;
   }
 
-  Inst.addOperand(MCOperand::CreateImm(imm));
+  Inst.addOperand(MCOperand::createImm(imm));
 
   switch (Inst.getOpcode()) {
     case ARM::VORRiv4i16:
@@ -3124,32 +3126,32 @@ static DecodeStatus DecodeVSHLMaxInstruction(MCInst &Inst, unsigned Insn,
     return MCDisassembler::Fail;
   if (!Check(S, DecodeDPRRegisterClass(Inst, Rm, Address, Decoder)))
     return MCDisassembler::Fail;
-  Inst.addOperand(MCOperand::CreateImm(8 << size));
+  Inst.addOperand(MCOperand::createImm(8 << size));
 
   return S;
 }
 
 static DecodeStatus DecodeShiftRight8Imm(MCInst &Inst, unsigned Val,
                                uint64_t Address, const void *Decoder) {
-  Inst.addOperand(MCOperand::CreateImm(8 - Val));
+  Inst.addOperand(MCOperand::createImm(8 - Val));
   return MCDisassembler::Success;
 }
 
 static DecodeStatus DecodeShiftRight16Imm(MCInst &Inst, unsigned Val,
                                uint64_t Address, const void *Decoder) {
-  Inst.addOperand(MCOperand::CreateImm(16 - Val));
+  Inst.addOperand(MCOperand::createImm(16 - Val));
   return MCDisassembler::Success;
 }
 
 static DecodeStatus DecodeShiftRight32Imm(MCInst &Inst, unsigned Val,
                                uint64_t Address, const void *Decoder) {
-  Inst.addOperand(MCOperand::CreateImm(32 - Val));
+  Inst.addOperand(MCOperand::createImm(32 - Val));
   return MCDisassembler::Success;
 }
 
 static DecodeStatus DecodeShiftRight64Imm(MCInst &Inst, unsigned Val,
                                uint64_t Address, const void *Decoder) {
-  Inst.addOperand(MCOperand::CreateImm(64 - Val));
+  Inst.addOperand(MCOperand::createImm(64 - Val));
   return MCDisassembler::Success;
 }
 
@@ -3205,11 +3207,11 @@ static DecodeStatus DecodeThumbAddSpecialReg(MCInst &Inst, uint16_t Insn,
     case ARM::tADR:
       break; // tADR does not explicitly represent the PC as an operand.
     case ARM::tADDrSPi:
-      Inst.addOperand(MCOperand::CreateReg(ARM::SP));
+      Inst.addOperand(MCOperand::createReg(ARM::SP));
       break;
   }
 
-  Inst.addOperand(MCOperand::CreateImm(imm));
+  Inst.addOperand(MCOperand::createImm(imm));
   return S;
 }
 
@@ -3217,7 +3219,7 @@ static DecodeStatus DecodeThumbBROperand(MCInst &Inst, unsigned Val,
                                  uint64_t Address, const void *Decoder) {
   if (!tryAddingSymbolicOperand(Address, Address + SignExtend32<12>(Val<<1) + 4,
                                 true, 2, Inst, Decoder))
-    Inst.addOperand(MCOperand::CreateImm(SignExtend32<12>(Val << 1)));
+    Inst.addOperand(MCOperand::createImm(SignExtend32<12>(Val << 1)));
   return MCDisassembler::Success;
 }
 
@@ -3225,7 +3227,7 @@ static DecodeStatus DecodeT2BROperand(MCInst &Inst, unsigned Val,
                                  uint64_t Address, const void *Decoder) {
   if (!tryAddingSymbolicOperand(Address, Address + SignExtend32<21>(Val) + 4,
                                 true, 4, Inst, Decoder))
-    Inst.addOperand(MCOperand::CreateImm(SignExtend32<21>(Val)));
+    Inst.addOperand(MCOperand::createImm(SignExtend32<21>(Val)));
   return MCDisassembler::Success;
 }
 
@@ -3233,7 +3235,7 @@ static DecodeStatus DecodeThumbCmpBROperand(MCInst &Inst, unsigned Val,
                                  uint64_t Address, const void *Decoder) {
   if (!tryAddingSymbolicOperand(Address, Address + (Val<<1) + 4,
                                 true, 2, Inst, Decoder))
-    Inst.addOperand(MCOperand::CreateImm(Val << 1));
+    Inst.addOperand(MCOperand::createImm(Val << 1));
   return MCDisassembler::Success;
 }
 
@@ -3261,7 +3263,7 @@ static DecodeStatus DecodeThumbAddrModeIS(MCInst &Inst, unsigned Val,
 
   if (!Check(S, DecodetGPRRegisterClass(Inst, Rn, Address, Decoder)))
     return MCDisassembler::Fail;
-  Inst.addOperand(MCOperand::CreateImm(imm));
+  Inst.addOperand(MCOperand::createImm(imm));
 
   return S;
 }
@@ -3270,7 +3272,7 @@ static DecodeStatus DecodeThumbAddrModePC(MCInst &Inst, unsigned Val,
                                   uint64_t Address, const void *Decoder) {
   unsigned imm = Val << 2;
 
-  Inst.addOperand(MCOperand::CreateImm(imm));
+  Inst.addOperand(MCOperand::createImm(imm));
   tryAddingPcLoadReferenceComment(Address, (Address & ~2u) + imm + 4, Decoder);
 
   return MCDisassembler::Success;
@@ -3278,8 +3280,8 @@ static DecodeStatus DecodeThumbAddrModePC(MCInst &Inst, unsigned Val,
 
 static DecodeStatus DecodeThumbAddrModeSP(MCInst &Inst, unsigned Val,
                                   uint64_t Address, const void *Decoder) {
-  Inst.addOperand(MCOperand::CreateReg(ARM::SP));
-  Inst.addOperand(MCOperand::CreateImm(Val));
+  Inst.addOperand(MCOperand::createReg(ARM::SP));
+  Inst.addOperand(MCOperand::createImm(Val));
 
   return MCDisassembler::Success;
 }
@@ -3307,7 +3309,7 @@ static DecodeStatus DecodeT2AddrModeSOReg(MCInst &Inst, unsigned Val,
     return MCDisassembler::Fail;
   if (!Check(S, DecoderGPRRegisterClass(Inst, Rm, Address, Decoder)))
     return MCDisassembler::Fail;
-  Inst.addOperand(MCOperand::CreateImm(imm));
+  Inst.addOperand(MCOperand::createImm(imm));
 
   return S;
 }
@@ -3319,10 +3321,11 @@ static DecodeStatus DecodeT2LoadShift(MCInst &Inst, unsigned Insn,
   unsigned Rt = fieldFromInstruction(Insn, 12, 4);
   unsigned Rn = fieldFromInstruction(Insn, 16, 4);
 
-  uint64_t featureBits = ((const MCDisassembler*)Decoder)->getSubtargetInfo()
-                                                          .getFeatureBits();
-  bool hasMP = featureBits & ARM::FeatureMP;
-  bool hasV7Ops = featureBits & ARM::HasV7Ops;
+  const FeatureBitset &featureBits =
+    ((const MCDisassembler*)Decoder)->getSubtargetInfo().getFeatureBits();
+
+  bool hasMP = featureBits[ARM::FeatureMP];
+  bool hasV7Ops = featureBits[ARM::HasV7Ops];
 
   if (Rn == 15) {
     switch (Inst.getOpcode()) {
@@ -3405,10 +3408,11 @@ static DecodeStatus DecodeT2LoadImm8(MCInst &Inst, unsigned Insn,
   imm |= (Rn << 9);
   unsigned add = fieldFromInstruction(Insn, 9, 1);
 
-  uint64_t featureBits = ((const MCDisassembler*)Decoder)->getSubtargetInfo()
-                                                          .getFeatureBits();
-  bool hasMP = featureBits & ARM::FeatureMP;
-  bool hasV7Ops = featureBits & ARM::HasV7Ops;
+  const FeatureBitset &featureBits =
+    ((const MCDisassembler*)Decoder)->getSubtargetInfo().getFeatureBits();
+
+  bool hasMP = featureBits[ARM::FeatureMP];
+  bool hasV7Ops = featureBits[ARM::HasV7Ops];
 
   if (Rn == 15) {
     switch (Inst.getOpcode()) {
@@ -3485,10 +3489,11 @@ static DecodeStatus DecodeT2LoadImm12(MCInst &Inst, unsigned Insn,
   unsigned imm = fieldFromInstruction(Insn, 0, 12);
   imm |= (Rn << 13);
 
-  uint64_t featureBits = ((const MCDisassembler*)Decoder)->getSubtargetInfo()
-                                                          .getFeatureBits();
-  bool hasMP = (featureBits & ARM::FeatureMP);
-  bool hasV7Ops = (featureBits & ARM::HasV7Ops);
+  const FeatureBitset &featureBits =
+    ((const MCDisassembler*)Decoder)->getSubtargetInfo().getFeatureBits();
+
+  bool hasMP = featureBits[ARM::FeatureMP];
+  bool hasV7Ops = featureBits[ARM::HasV7Ops];
 
   if (Rn == 15) {
     switch (Inst.getOpcode()) {
@@ -3602,9 +3607,10 @@ static DecodeStatus DecodeT2LoadLabel(MCInst &Inst, unsigned Insn,
   unsigned U = fieldFromInstruction(Insn, 23, 1);
   int imm = fieldFromInstruction(Insn, 0, 12);
 
-  uint64_t featureBits = ((const MCDisassembler*)Decoder)->getSubtargetInfo()
-                                                          .getFeatureBits();
-  bool hasV7Ops = (featureBits & ARM::HasV7Ops);
+  const FeatureBitset &featureBits =
+    ((const MCDisassembler*)Decoder)->getSubtargetInfo().getFeatureBits();
+
+  bool hasV7Ops = featureBits[ARM::HasV7Ops];
 
   if (Rt == 15) {
     switch (Inst.getOpcode()) {
@@ -3641,7 +3647,7 @@ static DecodeStatus DecodeT2LoadLabel(MCInst &Inst, unsigned Insn,
     else
       imm = -imm;
   }
-  Inst.addOperand(MCOperand::CreateImm(imm));
+  Inst.addOperand(MCOperand::createImm(imm));
 
   return S;
 }
@@ -3649,12 +3655,12 @@ static DecodeStatus DecodeT2LoadLabel(MCInst &Inst, unsigned Insn,
 static DecodeStatus DecodeT2Imm8S4(MCInst &Inst, unsigned Val,
                            uint64_t Address, const void *Decoder) {
   if (Val == 0)
-    Inst.addOperand(MCOperand::CreateImm(INT32_MIN));
+    Inst.addOperand(MCOperand::createImm(INT32_MIN));
   else {
     int imm = Val & 0xFF;
 
     if (!(Val & 0x100)) imm *= -1;
-    Inst.addOperand(MCOperand::CreateImm(imm * 4));
+    Inst.addOperand(MCOperand::createImm(imm * 4));
   }
 
   return MCDisassembler::Success;
@@ -3685,7 +3691,7 @@ static DecodeStatus DecodeT2AddrModeImm0_1020s4(MCInst &Inst,unsigned Val,
   if (!Check(S, DecodeGPRnopcRegisterClass(Inst, Rn, Address, Decoder)))
     return MCDisassembler::Fail;
 
-  Inst.addOperand(MCOperand::CreateImm(imm));
+  Inst.addOperand(MCOperand::createImm(imm));
 
   return S;
 }
@@ -3697,7 +3703,7 @@ static DecodeStatus DecodeT2Imm8(MCInst &Inst, unsigned Val,
     imm = INT32_MIN;
   else if (!(Val & 0x100))
     imm *= -1;
-  Inst.addOperand(MCOperand::CreateImm(imm));
+  Inst.addOperand(MCOperand::createImm(imm));
 
   return MCDisassembler::Success;
 }
@@ -3830,7 +3836,7 @@ static DecodeStatus DecodeT2AddrModeImm12(MCInst &Inst, unsigned Val,
 
   if (!Check(S, DecodeGPRRegisterClass(Inst, Rn, Address, Decoder)))
     return MCDisassembler::Fail;
-  Inst.addOperand(MCOperand::CreateImm(imm));
+  Inst.addOperand(MCOperand::createImm(imm));
 
   return S;
 }
@@ -3840,9 +3846,9 @@ static DecodeStatus DecodeThumbAddSPImm(MCInst &Inst, uint16_t Insn,
                                 uint64_t Address, const void *Decoder) {
   unsigned imm = fieldFromInstruction(Insn, 0, 7);
 
-  Inst.addOperand(MCOperand::CreateReg(ARM::SP));
-  Inst.addOperand(MCOperand::CreateReg(ARM::SP));
-  Inst.addOperand(MCOperand::CreateImm(imm));
+  Inst.addOperand(MCOperand::createReg(ARM::SP));
+  Inst.addOperand(MCOperand::createReg(ARM::SP));
+  Inst.addOperand(MCOperand::createImm(imm));
 
   return MCDisassembler::Success;
 }
@@ -3857,14 +3863,14 @@ static DecodeStatus DecodeThumbAddSPReg(MCInst &Inst, uint16_t Insn,
 
     if (!Check(S, DecodeGPRRegisterClass(Inst, Rdm, Address, Decoder)))
     return MCDisassembler::Fail;
-    Inst.addOperand(MCOperand::CreateReg(ARM::SP));
+    Inst.addOperand(MCOperand::createReg(ARM::SP));
     if (!Check(S, DecodeGPRRegisterClass(Inst, Rdm, Address, Decoder)))
     return MCDisassembler::Fail;
   } else if (Inst.getOpcode() == ARM::tADDspr) {
     unsigned Rm = fieldFromInstruction(Insn, 3, 4);
 
-    Inst.addOperand(MCOperand::CreateReg(ARM::SP));
-    Inst.addOperand(MCOperand::CreateReg(ARM::SP));
+    Inst.addOperand(MCOperand::createReg(ARM::SP));
+    Inst.addOperand(MCOperand::createReg(ARM::SP));
     if (!Check(S, DecodeGPRRegisterClass(Inst, Rm, Address, Decoder)))
     return MCDisassembler::Fail;
   }
@@ -3877,8 +3883,8 @@ static DecodeStatus DecodeThumbCPS(MCInst &Inst, uint16_t Insn,
   unsigned imod = fieldFromInstruction(Insn, 4, 1) | 0x2;
   unsigned flags = fieldFromInstruction(Insn, 0, 3);
 
-  Inst.addOperand(MCOperand::CreateImm(imod));
-  Inst.addOperand(MCOperand::CreateImm(flags));
+  Inst.addOperand(MCOperand::createImm(imod));
+  Inst.addOperand(MCOperand::createImm(flags));
 
   return MCDisassembler::Success;
 }
@@ -3891,7 +3897,7 @@ static DecodeStatus DecodePostIdxReg(MCInst &Inst, unsigned Insn,
 
   if (!Check(S, DecodeGPRnopcRegisterClass(Inst, Rm, Address, Decoder)))
     return MCDisassembler::Fail;
-  Inst.addOperand(MCOperand::CreateImm(add));
+  Inst.addOperand(MCOperand::createImm(add));
 
   return S;
 }
@@ -3916,7 +3922,7 @@ static DecodeStatus DecodeThumbBLXOffset(MCInst &Inst, unsigned Val,
   if (!tryAddingSymbolicOperand(Address,
                                 (Address & ~2u) + imm32 + 4,
                                 true, 4, Inst, Decoder))
-    Inst.addOperand(MCOperand::CreateImm(imm32));
+    Inst.addOperand(MCOperand::createImm(imm32));
   return MCDisassembler::Success;
 }
 
@@ -3925,12 +3931,13 @@ static DecodeStatus DecodeCoprocessor(MCInst &Inst, unsigned Val,
   if (Val == 0xA || Val == 0xB)
     return MCDisassembler::Fail;
 
-  uint64_t featureBits = ((const MCDisassembler*)Decoder)->getSubtargetInfo()
-                                                          .getFeatureBits();
-  if ((featureBits & ARM::HasV8Ops) && !(Val == 14 || Val == 15))
+  const FeatureBitset &featureBits =
+    ((const MCDisassembler*)Decoder)->getSubtargetInfo().getFeatureBits();
+
+  if (featureBits[ARM::HasV8Ops] && !(Val == 14 || Val == 15))
     return MCDisassembler::Fail;
 
-  Inst.addOperand(MCOperand::CreateImm(Val));
+  Inst.addOperand(MCOperand::createImm(Val));
   return MCDisassembler::Success;
 }
 
@@ -4001,16 +4008,16 @@ static DecodeStatus DecodeT2SOImm(MCInst &Inst, unsigned Val,
     unsigned imm = fieldFromInstruction(Val, 0, 8);
     switch (byte) {
       case 0:
-        Inst.addOperand(MCOperand::CreateImm(imm));
+        Inst.addOperand(MCOperand::createImm(imm));
         break;
       case 1:
-        Inst.addOperand(MCOperand::CreateImm((imm << 16) | imm));
+        Inst.addOperand(MCOperand::createImm((imm << 16) | imm));
         break;
       case 2:
-        Inst.addOperand(MCOperand::CreateImm((imm << 24) | (imm << 8)));
+        Inst.addOperand(MCOperand::createImm((imm << 24) | (imm << 8)));
         break;
       case 3:
-        Inst.addOperand(MCOperand::CreateImm((imm << 24) | (imm << 16) |
+        Inst.addOperand(MCOperand::createImm((imm << 24) | (imm << 16) |
                                              (imm << 8)  |  imm));
         break;
     }
@@ -4018,7 +4025,7 @@ static DecodeStatus DecodeT2SOImm(MCInst &Inst, unsigned Val,
     unsigned unrot = fieldFromInstruction(Val, 0, 7) | 0x80;
     unsigned rot = fieldFromInstruction(Val, 7, 5);
     unsigned imm = (unrot >> rot) | (unrot << ((32-rot)&31));
-    Inst.addOperand(MCOperand::CreateImm(imm));
+    Inst.addOperand(MCOperand::createImm(imm));
   }
 
   return MCDisassembler::Success;
@@ -4029,7 +4036,7 @@ DecodeThumbBCCTargetOperand(MCInst &Inst, unsigned Val,
                             uint64_t Address, const void *Decoder){
   if (!tryAddingSymbolicOperand(Address, Address + SignExtend32<9>(Val<<1) + 4,
                                 true, 2, Inst, Decoder))
-    Inst.addOperand(MCOperand::CreateImm(SignExtend32<9>(Val << 1)));
+    Inst.addOperand(MCOperand::createImm(SignExtend32<9>(Val << 1)));
   return MCDisassembler::Success;
 }
 
@@ -4052,7 +4059,7 @@ static DecodeStatus DecodeThumbBLTargetOperand(MCInst &Inst, unsigned Val,
 
   if (!tryAddingSymbolicOperand(Address, Address + imm32 + 4,
                                 true, 4, Inst, Decoder))
-    Inst.addOperand(MCOperand::CreateImm(imm32));
+    Inst.addOperand(MCOperand::createImm(imm32));
   return MCDisassembler::Success;
 }
 
@@ -4061,7 +4068,7 @@ static DecodeStatus DecodeMemBarrierOption(MCInst &Inst, unsigned Val,
   if (Val & ~0xf)
     return MCDisassembler::Fail;
 
-  Inst.addOperand(MCOperand::CreateImm(Val));
+  Inst.addOperand(MCOperand::createImm(Val));
   return MCDisassembler::Success;
 }
 
@@ -4070,16 +4077,17 @@ static DecodeStatus DecodeInstSyncBarrierOption(MCInst &Inst, unsigned Val,
   if (Val & ~0xf)
     return MCDisassembler::Fail;
 
-  Inst.addOperand(MCOperand::CreateImm(Val));
+  Inst.addOperand(MCOperand::createImm(Val));
   return MCDisassembler::Success;
 }
 
 static DecodeStatus DecodeMSRMask(MCInst &Inst, unsigned Val,
                           uint64_t Address, const void *Decoder) {
   DecodeStatus S = MCDisassembler::Success;
-  uint64_t FeatureBits = ((const MCDisassembler*)Decoder)->getSubtargetInfo()
-                                                          .getFeatureBits();
-  if (FeatureBits & ARM::FeatureMClass) {
+  const FeatureBitset &FeatureBits =
+    ((const MCDisassembler*)Decoder)->getSubtargetInfo().getFeatureBits();
+
+  if (FeatureBits[ARM::FeatureMClass]) {
     unsigned ValLow = Val & 0xff;
 
     // Validate the SYSm value first.
@@ -4099,7 +4107,7 @@ static DecodeStatus DecodeMSRMask(MCInst &Inst, unsigned Val,
     case 17: // basepri
     case 18: // basepri_max
     case 19: // faultmask
-      if (!(FeatureBits & ARM::HasV7Ops))
+      if (!(FeatureBits[ARM::HasV7Ops]))
         // Values basepri, basepri_max and faultmask are only valid for v7m.
         return MCDisassembler::Fail;
       break;
@@ -4109,7 +4117,7 @@ static DecodeStatus DecodeMSRMask(MCInst &Inst, unsigned Val,
 
     if (Inst.getOpcode() == ARM::t2MSR_M) {
       unsigned Mask = fieldFromInstruction(Val, 10, 2);
-      if (!(FeatureBits & ARM::HasV7Ops)) {
+      if (!(FeatureBits[ARM::HasV7Ops])) {
         // The ARMv6-M MSR bits {11-10} can be only 0b10, other values are
         // unpredictable.
         if (Mask != 2)
@@ -4123,7 +4131,7 @@ static DecodeStatus DecodeMSRMask(MCInst &Inst, unsigned Val,
         // indicates the move for the GE{3:0} bits, the mask{0} bit can be set
         // only if the processor includes the DSP extension.
         if (Mask == 0 || (Mask != 2 && ValLow > 3) ||
-            (!(FeatureBits & ARM::FeatureDSPThumb2) && (Mask & 1)))
+            (!(FeatureBits[ARM::FeatureDSPThumb2]) && (Mask & 1)))
           S = MCDisassembler::SoftFail;
       }
     }
@@ -4132,7 +4140,7 @@ static DecodeStatus DecodeMSRMask(MCInst &Inst, unsigned Val,
     if (Val == 0)
       return MCDisassembler::Fail;
   }
-  Inst.addOperand(MCOperand::CreateImm(Val));
+  Inst.addOperand(MCOperand::createImm(Val));
   return S;
 }
 
@@ -4155,7 +4163,7 @@ static DecodeStatus DecodeBankedReg(MCInst &Inst, unsigned Val,
       return MCDisassembler::SoftFail;
   }
 
-  Inst.addOperand(MCOperand::CreateImm(Val));
+  Inst.addOperand(MCOperand::createImm(Val));
   return MCDisassembler::Success;
 }
 
@@ -4359,18 +4367,18 @@ static DecodeStatus DecodeVLD1LN(MCInst &Inst, unsigned Insn,
   }
   if (!Check(S, DecodeGPRRegisterClass(Inst, Rn, Address, Decoder)))
     return MCDisassembler::Fail;
-  Inst.addOperand(MCOperand::CreateImm(align));
+  Inst.addOperand(MCOperand::createImm(align));
   if (Rm != 0xF) {
     if (Rm != 0xD) {
       if (!Check(S, DecodeGPRRegisterClass(Inst, Rm, Address, Decoder)))
         return MCDisassembler::Fail;
     } else
-      Inst.addOperand(MCOperand::CreateReg(0));
+      Inst.addOperand(MCOperand::createReg(0));
   }
 
   if (!Check(S, DecodeDPRRegisterClass(Inst, Rd, Address, Decoder)))
     return MCDisassembler::Fail;
-  Inst.addOperand(MCOperand::CreateImm(index));
+  Inst.addOperand(MCOperand::createImm(index));
 
   return S;
 }
@@ -4424,18 +4432,18 @@ static DecodeStatus DecodeVST1LN(MCInst &Inst, unsigned Insn,
   }
   if (!Check(S, DecodeGPRRegisterClass(Inst, Rn, Address, Decoder)))
     return MCDisassembler::Fail;
-  Inst.addOperand(MCOperand::CreateImm(align));
+  Inst.addOperand(MCOperand::createImm(align));
   if (Rm != 0xF) {
     if (Rm != 0xD) {
       if (!Check(S, DecodeGPRRegisterClass(Inst, Rm, Address, Decoder)))
     return MCDisassembler::Fail;
     } else
-      Inst.addOperand(MCOperand::CreateReg(0));
+      Inst.addOperand(MCOperand::createReg(0));
   }
 
   if (!Check(S, DecodeDPRRegisterClass(Inst, Rd, Address, Decoder)))
     return MCDisassembler::Fail;
-  Inst.addOperand(MCOperand::CreateImm(index));
+  Inst.addOperand(MCOperand::createImm(index));
 
   return S;
 }
@@ -4490,20 +4498,20 @@ static DecodeStatus DecodeVLD2LN(MCInst &Inst, unsigned Insn,
   }
   if (!Check(S, DecodeGPRRegisterClass(Inst, Rn, Address, Decoder)))
     return MCDisassembler::Fail;
-  Inst.addOperand(MCOperand::CreateImm(align));
+  Inst.addOperand(MCOperand::createImm(align));
   if (Rm != 0xF) {
     if (Rm != 0xD) {
       if (!Check(S, DecodeGPRRegisterClass(Inst, Rm, Address, Decoder)))
         return MCDisassembler::Fail;
     } else
-      Inst.addOperand(MCOperand::CreateReg(0));
+      Inst.addOperand(MCOperand::createReg(0));
   }
 
   if (!Check(S, DecodeDPRRegisterClass(Inst, Rd, Address, Decoder)))
     return MCDisassembler::Fail;
   if (!Check(S, DecodeDPRRegisterClass(Inst, Rd+inc, Address, Decoder)))
     return MCDisassembler::Fail;
-  Inst.addOperand(MCOperand::CreateImm(index));
+  Inst.addOperand(MCOperand::createImm(index));
 
   return S;
 }
@@ -4553,20 +4561,20 @@ static DecodeStatus DecodeVST2LN(MCInst &Inst, unsigned Insn,
   }
   if (!Check(S, DecodeGPRRegisterClass(Inst, Rn, Address, Decoder)))
     return MCDisassembler::Fail;
-  Inst.addOperand(MCOperand::CreateImm(align));
+  Inst.addOperand(MCOperand::createImm(align));
   if (Rm != 0xF) {
     if (Rm != 0xD) {
       if (!Check(S, DecodeGPRRegisterClass(Inst, Rm, Address, Decoder)))
         return MCDisassembler::Fail;
     } else
-      Inst.addOperand(MCOperand::CreateReg(0));
+      Inst.addOperand(MCOperand::createReg(0));
   }
 
   if (!Check(S, DecodeDPRRegisterClass(Inst, Rd, Address, Decoder)))
     return MCDisassembler::Fail;
   if (!Check(S, DecodeDPRRegisterClass(Inst, Rd+inc, Address, Decoder)))
     return MCDisassembler::Fail;
-  Inst.addOperand(MCOperand::CreateImm(index));
+  Inst.addOperand(MCOperand::createImm(index));
 
   return S;
 }
@@ -4622,13 +4630,13 @@ static DecodeStatus DecodeVLD3LN(MCInst &Inst, unsigned Insn,
   }
   if (!Check(S, DecodeGPRRegisterClass(Inst, Rn, Address, Decoder)))
     return MCDisassembler::Fail;
-  Inst.addOperand(MCOperand::CreateImm(align));
+  Inst.addOperand(MCOperand::createImm(align));
   if (Rm != 0xF) {
     if (Rm != 0xD) {
       if (!Check(S, DecodeGPRRegisterClass(Inst, Rm, Address, Decoder)))
     return MCDisassembler::Fail;
     } else
-      Inst.addOperand(MCOperand::CreateReg(0));
+      Inst.addOperand(MCOperand::createReg(0));
   }
 
   if (!Check(S, DecodeDPRRegisterClass(Inst, Rd, Address, Decoder)))
@@ -4637,7 +4645,7 @@ static DecodeStatus DecodeVLD3LN(MCInst &Inst, unsigned Insn,
     return MCDisassembler::Fail;
   if (!Check(S, DecodeDPRRegisterClass(Inst, Rd+2*inc, Address, Decoder)))
     return MCDisassembler::Fail;
-  Inst.addOperand(MCOperand::CreateImm(index));
+  Inst.addOperand(MCOperand::createImm(index));
 
   return S;
 }
@@ -4685,13 +4693,13 @@ static DecodeStatus DecodeVST3LN(MCInst &Inst, unsigned Insn,
   }
   if (!Check(S, DecodeGPRRegisterClass(Inst, Rn, Address, Decoder)))
     return MCDisassembler::Fail;
-  Inst.addOperand(MCOperand::CreateImm(align));
+  Inst.addOperand(MCOperand::createImm(align));
   if (Rm != 0xF) {
     if (Rm != 0xD) {
       if (!Check(S, DecodeGPRRegisterClass(Inst, Rm, Address, Decoder)))
     return MCDisassembler::Fail;
     } else
-      Inst.addOperand(MCOperand::CreateReg(0));
+      Inst.addOperand(MCOperand::createReg(0));
   }
 
   if (!Check(S, DecodeDPRRegisterClass(Inst, Rd, Address, Decoder)))
@@ -4700,7 +4708,7 @@ static DecodeStatus DecodeVST3LN(MCInst &Inst, unsigned Insn,
     return MCDisassembler::Fail;
   if (!Check(S, DecodeDPRRegisterClass(Inst, Rd+2*inc, Address, Decoder)))
     return MCDisassembler::Fail;
-  Inst.addOperand(MCOperand::CreateImm(index));
+  Inst.addOperand(MCOperand::createImm(index));
 
   return S;
 }
@@ -4765,13 +4773,13 @@ static DecodeStatus DecodeVLD4LN(MCInst &Inst, unsigned Insn,
   }
   if (!Check(S, DecodeGPRRegisterClass(Inst, Rn, Address, Decoder)))
     return MCDisassembler::Fail;
-  Inst.addOperand(MCOperand::CreateImm(align));
+  Inst.addOperand(MCOperand::createImm(align));
   if (Rm != 0xF) {
     if (Rm != 0xD) {
       if (!Check(S, DecodeGPRRegisterClass(Inst, Rm, Address, Decoder)))
         return MCDisassembler::Fail;
     } else
-      Inst.addOperand(MCOperand::CreateReg(0));
+      Inst.addOperand(MCOperand::createReg(0));
   }
 
   if (!Check(S, DecodeDPRRegisterClass(Inst, Rd, Address, Decoder)))
@@ -4782,7 +4790,7 @@ static DecodeStatus DecodeVLD4LN(MCInst &Inst, unsigned Insn,
     return MCDisassembler::Fail;
   if (!Check(S, DecodeDPRRegisterClass(Inst, Rd+3*inc, Address, Decoder)))
     return MCDisassembler::Fail;
-  Inst.addOperand(MCOperand::CreateImm(index));
+  Inst.addOperand(MCOperand::createImm(index));
 
   return S;
 }
@@ -4837,13 +4845,13 @@ static DecodeStatus DecodeVST4LN(MCInst &Inst, unsigned Insn,
   }
   if (!Check(S, DecodeGPRRegisterClass(Inst, Rn, Address, Decoder)))
     return MCDisassembler::Fail;
-  Inst.addOperand(MCOperand::CreateImm(align));
+  Inst.addOperand(MCOperand::createImm(align));
   if (Rm != 0xF) {
     if (Rm != 0xD) {
       if (!Check(S, DecodeGPRRegisterClass(Inst, Rm, Address, Decoder)))
     return MCDisassembler::Fail;
     } else
-      Inst.addOperand(MCOperand::CreateReg(0));
+      Inst.addOperand(MCOperand::createReg(0));
   }
 
   if (!Check(S, DecodeDPRRegisterClass(Inst, Rd, Address, Decoder)))
@@ -4854,7 +4862,7 @@ static DecodeStatus DecodeVST4LN(MCInst &Inst, unsigned Insn,
     return MCDisassembler::Fail;
   if (!Check(S, DecodeDPRRegisterClass(Inst, Rd+3*inc, Address, Decoder)))
     return MCDisassembler::Fail;
-  Inst.addOperand(MCOperand::CreateImm(index));
+  Inst.addOperand(MCOperand::createImm(index));
 
   return S;
 }
@@ -4925,8 +4933,8 @@ static DecodeStatus DecodeIT(MCInst &Inst, unsigned Insn,
   if (mask == 0x0)
     return MCDisassembler::Fail;
 
-  Inst.addOperand(MCOperand::CreateImm(pred));
-  Inst.addOperand(MCOperand::CreateImm(mask));
+  Inst.addOperand(MCOperand::createImm(pred));
+  Inst.addOperand(MCOperand::createImm(mask));
   return S;
 }
 
@@ -5012,7 +5020,7 @@ static DecodeStatus DecodeT2Adr(MCInst &Inst, uint32_t Insn,
   Val |= fieldFromInstruction(Insn, 12, 3) << 8;
   Val |= fieldFromInstruction(Insn, 26, 1) << 11;
   Val |= sign1 << 12;
-  Inst.addOperand(MCOperand::CreateImm(SignExtend32<13>(Val)));
+  Inst.addOperand(MCOperand::createImm(SignExtend32<13>(Val)));
 
   return MCDisassembler::Success;
 }
@@ -5024,7 +5032,7 @@ static DecodeStatus DecodeT2ShifterImmOperand(MCInst &Inst, uint32_t Val,
 
   // Shift of "asr #32" is not allowed in Thumb2 mode.
   if (Val == 0x20) S = MCDisassembler::Fail;
-  Inst.addOperand(MCOperand::CreateImm(Val));
+  Inst.addOperand(MCOperand::createImm(Val));
   return S;
 }
 
@@ -5080,7 +5088,7 @@ static DecodeStatus DecodeVCVTD(MCInst &Inst, unsigned Insn,
     return MCDisassembler::Fail;
   if (!Check(S, DecodeDPRRegisterClass(Inst, Vm, Address, Decoder)))
     return MCDisassembler::Fail;
-  Inst.addOperand(MCOperand::CreateImm(64 - imm));
+  Inst.addOperand(MCOperand::createImm(64 - imm));
 
   return S;
 }
@@ -5110,7 +5118,7 @@ static DecodeStatus DecodeVCVTQ(MCInst &Inst, unsigned Insn,
     return MCDisassembler::Fail;
   if (!Check(S, DecodeQPRRegisterClass(Inst, Vm, Address, Decoder)))
     return MCDisassembler::Fail;
-  Inst.addOperand(MCOperand::CreateImm(64 - imm));
+  Inst.addOperand(MCOperand::createImm(64 - imm));
 
   return S;
 }
@@ -5159,13 +5167,13 @@ static DecodeStatus DecodeMRRC2(llvm::MCInst &Inst, unsigned Val,
   if (Rt == Rt2)
     S = MCDisassembler::SoftFail;
 
-  Inst.addOperand(MCOperand::CreateImm(cop));
-  Inst.addOperand(MCOperand::CreateImm(opc1));
+  Inst.addOperand(MCOperand::createImm(cop));
+  Inst.addOperand(MCOperand::createImm(opc1));
   if (!Check(S, DecodeGPRnopcRegisterClass(Inst, Rt, Address, Decoder)))
     return MCDisassembler::Fail;
   if (!Check(S, DecodeGPRnopcRegisterClass(Inst, Rt2, Address, Decoder)))
     return MCDisassembler::Fail;
-  Inst.addOperand(MCOperand::CreateImm(CRm));
+  Inst.addOperand(MCOperand::createImm(CRm));
 
   return S;
 }
diff --git a/lib/Target/ARM/InstPrinter/ARMInstPrinter.cpp b/lib/Target/ARM/InstPrinter/ARMInstPrinter.cpp
index af1b9fe4e0a..0bff52141da 100644
--- a/lib/Target/ARM/InstPrinter/ARMInstPrinter.cpp
+++ b/lib/Target/ARM/InstPrinter/ARMInstPrinter.cpp
@@ -93,7 +93,7 @@ void ARMInstPrinter::printInst(const MCInst *MI, raw_ostream &O,
       O << "\tsev";
       break;
     case 5:
-      if ((STI.getFeatureBits() & ARM::HasV8Ops)) {
+      if (STI.getFeatureBits()[ARM::HasV8Ops]) {
         O << "\tsevl";
         break;
       } // Fallthrough for non-v8
@@ -284,7 +284,7 @@ void ARMInstPrinter::printInst(const MCInst *MI, raw_ostream &O,
 
       if (isStore)
         NewMI.addOperand(MI->getOperand(0));
-      NewReg = MCOperand::CreateReg(MRI.getMatchingSuperReg(
+      NewReg = MCOperand::createReg(MRI.getMatchingSuperReg(
           Reg, ARM::gsub_0, &MRI.getRegClass(ARM::GPRPairRegClassID)));
       NewMI.addOperand(NewReg);
 
@@ -302,7 +302,7 @@ void ARMInstPrinter::printInst(const MCInst *MI, raw_ostream &O,
   case ARM::t2SUBS_PC_LR: {
     if (MI->getNumOperands() == 3 && MI->getOperand(0).isImm() &&
         MI->getOperand(0).getImm() == 0 &&
-        (STI.getFeatureBits() & ARM::FeatureVirtualization)) {
+        STI.getFeatureBits()[ARM::FeatureVirtualization]) {
       O << "\teret";
       printPredicateOperand(MI, 1, STI, O);
       printAnnotation(O, Annot);
@@ -312,71 +312,6 @@ void ARMInstPrinter::printInst(const MCInst *MI, raw_ostream &O,
   }
   }
 
-  // @LOCALMOD-BEGIN
-  // TODO(pdox): Kill this code once we switch to MC object emission
-  const char *SFIInst = NULL;
-  unsigned SFIEmitDest = ~0;
-  unsigned SFIEmitPred = ~0;
-  switch (Opcode) {
-  case ARM::SFI_NOP_IF_AT_BUNDLE_END :
-    SFIInst = "sfi_nop_if_at_bundle_end";
-    SFIEmitDest = ~0;
-    SFIEmitPred = ~0;
-    break;
-  case ARM::SFI_GUARD_LOADSTORE    :
-    SFIInst = "sfi_load_store_preamble";
-    SFIEmitDest = 0;
-    SFIEmitPred = 2;
-    break;
-  case ARM::SFI_GUARD_INDIRECT_CALL:
-    SFIInst = "sfi_indirect_call_preamble";
-    SFIEmitDest = 0;
-    SFIEmitPred = 2;
-    break;
-  case ARM::SFI_GUARD_INDIRECT_JMP :
-    SFIInst = "sfi_indirect_jump_preamble";
-    SFIEmitDest = 0;
-    SFIEmitPred = 2;
-    break;
-  case ARM::SFI_DATA_MASK          :
-    SFIInst = "sfi_data_mask";
-    SFIEmitDest = 0;
-    SFIEmitPred = 2;
-    break;
-  case ARM::SFI_GUARD_LOADSTORE_TST:
-    SFIInst = "sfi_cload_store_preamble";
-    SFIEmitDest = 0;
-    SFIEmitPred = ~0;
-    break;
-  case ARM::SFI_GUARD_CALL     :
-    SFIInst = "sfi_call_preamble";
-    SFIEmitDest = ~0;
-    SFIEmitPred = 0;
-    break;
-  case ARM::SFI_GUARD_RETURN   :
-    SFIInst = "sfi_return_preamble lr,";
-    SFIEmitDest = ~0;
-    SFIEmitPred = 0;
-    break;
-  }
-  if (SFIInst) {
-    O << '\t' << SFIInst;
-    if (SFIEmitDest != (unsigned)~0) {
-      O << ' ';
-      printOperand(MI, SFIEmitDest, STI, O);
-    }
-    if (SFIEmitDest != (unsigned)~0 && SFIEmitPred != (unsigned)~0) {
-      O << ',';
-    }
-    if (SFIEmitPred != (unsigned)~0) {
-      O << ' ';
-      printPredicateOperand(MI, SFIEmitPred, STI, O);
-    }
-    O << '\n';
-    return;
-  }
-  // @LOCALMOD-END
-
   printInstruction(MI, STI, O);
   printAnnotation(O, Annot);
 }
@@ -394,7 +329,8 @@ void ARMInstPrinter::printOperand(const MCInst *MI, unsigned OpNo,
     const MCExpr *Expr = Op.getExpr();
     switch (Expr->getKind()) {
     case MCExpr::Binary:
-      O << '#' << *Expr;
+      O << '#';
+      Expr->print(O, &MAI);
       break;
     case MCExpr::Constant: {
       // If a symbolic branch target was added as a constant expression then
@@ -402,8 +338,9 @@ void ARMInstPrinter::printOperand(const MCInst *MI, unsigned OpNo,
       // address.
       const MCConstantExpr *Constant = cast<MCConstantExpr>(Expr);
       int64_t TargetAddress;
-      if (!Constant->EvaluateAsAbsolute(TargetAddress)) {
-        O << '#' << *Expr;
+      if (!Constant->evaluateAsAbsolute(TargetAddress)) {
+        O << '#';
+        Expr->print(O, &MAI);
       } else {
         O << "0x";
         O.write_hex(static_cast<uint32_t>(TargetAddress));
@@ -413,7 +350,7 @@ void ARMInstPrinter::printOperand(const MCInst *MI, unsigned OpNo,
     default:
       // FIXME: Should we always treat this as if it is a constant literal and
       // prefix it with '#'?
-      O << *Expr;
+      Expr->print(O, &MAI);
       break;
     }
   }
@@ -424,7 +361,7 @@ void ARMInstPrinter::printThumbLdrLabelOperand(const MCInst *MI, unsigned OpNum,
                                                raw_ostream &O) {
   const MCOperand &MO1 = MI->getOperand(OpNum);
   if (MO1.isExpr()) {
-    O << *MO1.getExpr();
+    MO1.getExpr()->print(O, &MAI);
     return;
   }
 
@@ -760,7 +697,7 @@ void ARMInstPrinter::printMemBOption(const MCInst *MI, unsigned OpNum,
                                      const MCSubtargetInfo &STI,
                                      raw_ostream &O) {
   unsigned val = MI->getOperand(OpNum).getImm();
-  O << ARM_MB::MemBOptToString(val, (STI.getFeatureBits() & ARM::HasV8Ops));
+  O << ARM_MB::MemBOptToString(val, STI.getFeatureBits()[ARM::HasV8Ops]);
 }
 
 void ARMInstPrinter::printInstSyncBOption(const MCInst *MI, unsigned OpNum,
@@ -860,14 +797,14 @@ void ARMInstPrinter::printMSRMaskOperand(const MCInst *MI, unsigned OpNum,
   const MCOperand &Op = MI->getOperand(OpNum);
   unsigned SpecRegRBit = Op.getImm() >> 4;
   unsigned Mask = Op.getImm() & 0xf;
-  uint64_t FeatureBits = STI.getFeatureBits();
+  const FeatureBitset &FeatureBits = STI.getFeatureBits();
 
-  if (FeatureBits & ARM::FeatureMClass) {
+  if (FeatureBits[ARM::FeatureMClass]) {
     unsigned SYSm = Op.getImm();
     unsigned Opcode = MI->getOpcode();
 
     // For writes, handle extended mask bits if the DSP extension is present.
-    if (Opcode == ARM::t2MSR_M && (FeatureBits & ARM::FeatureDSPThumb2)) {
+    if (Opcode == ARM::t2MSR_M && FeatureBits[ARM::FeatureDSPThumb2]) {
       switch (SYSm) {
       case 0x400:
         O << "apsr_g";
@@ -899,7 +836,7 @@ void ARMInstPrinter::printMSRMaskOperand(const MCInst *MI, unsigned OpNum,
     // Handle the basic 8-bit mask.
     SYSm &= 0xff;
 
-    if (Opcode == ARM::t2MSR_M && (FeatureBits & ARM::HasV7Ops)) {
+    if (Opcode == ARM::t2MSR_M && FeatureBits [ARM::HasV7Ops]) {
       // ARMv7-M deprecates using MSR APSR without a _<bits> qualifier as an
       // alias for MSR APSR_nzcvq.
       switch (SYSm) {
@@ -1120,7 +1057,7 @@ void ARMInstPrinter::printAdrLabelOperand(const MCInst *MI, unsigned OpNum,
   const MCOperand &MO = MI->getOperand(OpNum);
 
   if (MO.isExpr()) {
-    O << *MO.getExpr();
+    MO.getExpr()->print(O, &MAI);
     return;
   }
 
@@ -1444,21 +1381,8 @@ void ARMInstPrinter::printRotImmOperand(const MCInst *MI, unsigned OpNum,
   unsigned Imm = MI->getOperand(OpNum).getImm();
   if (Imm == 0)
     return;
-  O << ", ror " << markup("<imm:") << "#";
-  switch (Imm) {
-  default:
-    assert(0 && "illegal ror immediate!");
-  case 1:
-    O << "8";
-    break;
-  case 2:
-    O << "16";
-    break;
-  case 3:
-    O << "24";
-    break;
-  }
-  O << markup(">");
+  assert(Imm <= 3 && "illegal ror immediate!");
+  O << ", ror " << markup("<imm:") << "#" << 8 * Imm << markup(">");
 }
 
 void ARMInstPrinter::printModImmOperand(const MCInst *MI, unsigned OpNum,
diff --git a/lib/Target/ARM/LLVMBuild.txt b/lib/Target/ARM/LLVMBuild.txt
index 0dc47aa0519..9ed51dfda0f 100644
--- a/lib/Target/ARM/LLVMBuild.txt
+++ b/lib/Target/ARM/LLVMBuild.txt
@@ -31,6 +31,5 @@ has_jit = 1
 type = Library
 name = ARMCodeGen
 parent = ARM
-; @LOCALMOD
-required_libraries = ARMAsmPrinter ARMDesc ARMInfo Analysis AsmPrinter CodeGen Core MC Scalar SelectionDAG Support Target NaClTransforms
+required_libraries = ARMAsmPrinter ARMDesc ARMInfo Analysis AsmPrinter CodeGen Core MC Scalar SelectionDAG Support Target
 add_to_library_groups = ARM
diff --git a/lib/Target/ARM/MCTargetDesc/ARMAddressingModes.h b/lib/Target/ARM/MCTargetDesc/ARMAddressingModes.h
index f0eed9b811d..b03cada9a64 100644
--- a/lib/Target/ARM/MCTargetDesc/ARMAddressingModes.h
+++ b/lib/Target/ARM/MCTargetDesc/ARMAddressingModes.h
@@ -622,8 +622,6 @@ namespace ARM_AM {
     return Value;
   }
 
-  AMSubMode getLoadStoreMultipleSubMode(int Opcode);
-
   //===--------------------------------------------------------------------===//
   // Floating-point Immediates
   //
diff --git a/lib/Target/ARM/MCTargetDesc/ARMArchName.def b/lib/Target/ARM/MCTargetDesc/ARMArchName.def
deleted file mode 100644
index 96a0c1a84c4..00000000000
--- a/lib/Target/ARM/MCTargetDesc/ARMArchName.def
+++ /dev/null
@@ -1,53 +0,0 @@
-//===-- ARMArchName.def - List of the ARM arch names ------------*- C++ -*-===//
-//
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
-//
-//===----------------------------------------------------------------------===//
-//
-// This file contains the list of the supported ARM architecture names,
-// i.e. the supported value for -march= option.
-//
-//===----------------------------------------------------------------------===//
-
-// NOTE: NO INCLUDE GUARD DESIRED!
-
-#ifndef ARM_ARCH_NAME
-#error "You must define ARM_ARCH_NAME before including ARMArchName.def"
-#endif
-
-// ARM_ARCH_NAME(NAME, ID, DEFAULT_CPU_NAME, DEFAULT_CPU_ARCH)
-ARM_ARCH_NAME("armv2",   ARMV2,   "2",       v4)
-ARM_ARCH_NAME("armv2a",  ARMV2A,  "2A",      v4)
-ARM_ARCH_NAME("armv3",   ARMV3,   "3",       v4)
-ARM_ARCH_NAME("armv3m",  ARMV3M,  "3M",      v4)
-ARM_ARCH_NAME("armv4",   ARMV4,   "4",       v4)
-ARM_ARCH_NAME("armv4t",  ARMV4T,  "4T",      v4T)
-ARM_ARCH_NAME("armv5",   ARMV5,   "5",       v5T)
-ARM_ARCH_NAME("armv5t",  ARMV5T,  "5T",      v5T)
-ARM_ARCH_NAME("armv5te", ARMV5TE, "5TE",     v5TE)
-ARM_ARCH_NAME("armv6",   ARMV6,   "6",       v6)
-ARM_ARCH_NAME("armv6j",  ARMV6J,  "6J",      v6)
-ARM_ARCH_NAME("armv6k",  ARMV6K,  "6K",      v6K)
-ARM_ARCH_NAME("armv6t2", ARMV6T2, "6T2",     v6T2)
-ARM_ARCH_NAME("armv6z",  ARMV6Z,  "6Z",      v6KZ)
-ARM_ARCH_NAME("armv6zk", ARMV6ZK, "6ZK",     v6KZ)
-ARM_ARCH_NAME("armv6-m", ARMV6M,  "6-M",     v6_M)
-ARM_ARCH_NAME("armv7",   ARMV7,   "7",       v7)
-ARM_ARCH_NAME("armv7-a", ARMV7A,  "7-A",     v7)
-ARM_ARCH_ALIAS("armv7a", ARMV7A)
-ARM_ARCH_NAME("armv7-r", ARMV7R,  "7-R",     v7)
-ARM_ARCH_ALIAS("armv7r", ARMV7R)
-ARM_ARCH_NAME("armv7-m", ARMV7M,  "7-M",     v7)
-ARM_ARCH_ALIAS("armv7m", ARMV7M)
-ARM_ARCH_NAME("armv8-a", ARMV8A,  "8-A",     v8)
-ARM_ARCH_ALIAS("armv8a", ARMV8A)
-ARM_ARCH_NAME("armv8.1-a", ARMV8_1A, "8.1-A", v8)
-ARM_ARCH_ALIAS("armv8.1a", ARMV8_1A)
-ARM_ARCH_NAME("iwmmxt",  IWMMXT,  "iwmmxt",  v5TE)
-ARM_ARCH_NAME("iwmmxt2", IWMMXT2, "iwmmxt2", v5TE)
-
-#undef ARM_ARCH_NAME
-#undef ARM_ARCH_ALIAS
diff --git a/lib/Target/ARM/MCTargetDesc/ARMArchName.h b/lib/Target/ARM/MCTargetDesc/ARMArchName.h
deleted file mode 100644
index bc056737a82..00000000000
--- a/lib/Target/ARM/MCTargetDesc/ARMArchName.h
+++ /dev/null
@@ -1,27 +0,0 @@
-//===-- ARMArchName.h - List of the ARM arch names --------------*- C++ -*-===//
-//
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
-//
-//===----------------------------------------------------------------------===//
-
-#ifndef LLVM_LIB_TARGET_ARM_MCTARGETDESC_ARMARCHNAME_H
-#define LLVM_LIB_TARGET_ARM_MCTARGETDESC_ARMARCHNAME_H
-
-namespace llvm {
-namespace ARM {
-
-enum ArchKind {
-  INVALID_ARCH = 0
-
-#define ARM_ARCH_NAME(NAME, ID, DEFAULT_CPU_NAME, DEFAULT_CPU_ARCH) , ID
-#define ARM_ARCH_ALIAS(NAME, ID) /* empty */
-#include "ARMArchName.def"
-};
-
-} // namespace ARM
-} // namespace llvm
-
-#endif
diff --git a/lib/Target/ARM/MCTargetDesc/ARMAsmBackend.cpp b/lib/Target/ARM/MCTargetDesc/ARMAsmBackend.cpp
index 2891a030827..11146358856 100644
--- a/lib/Target/ARM/MCTargetDesc/ARMAsmBackend.cpp
+++ b/lib/Target/ARM/MCTargetDesc/ARMAsmBackend.cpp
@@ -12,7 +12,6 @@
 #include "MCTargetDesc/ARMAsmBackend.h"
 #include "MCTargetDesc/ARMAsmBackendDarwin.h"
 #include "MCTargetDesc/ARMAsmBackendELF.h"
-#include "MCTargetDesc/ARMAsmBackendNaClELF.h" // @LOCALMOD
 #include "MCTargetDesc/ARMAsmBackendWinCOFF.h"
 #include "MCTargetDesc/ARMBaseInfo.h"
 #include "MCTargetDesc/ARMFixupKinds.h"
@@ -155,7 +154,7 @@ void ARMAsmBackend::handleAssemblerFlag(MCAssemblerFlag Flag) {
 } // end anonymous namespace
 
 unsigned ARMAsmBackend::getRelaxedOpcode(unsigned Op) const {
-  bool HasThumb2 = STI->getFeatureBits() & ARM::FeatureThumb2;
+  bool HasThumb2 = STI->getFeatureBits()[ARM::FeatureThumb2];
 
   switch (Op) {
   default:
@@ -239,9 +238,9 @@ void ARMAsmBackend::relaxInstruction(const MCInst &Inst, MCInst &Res) const {
   if ((Inst.getOpcode() == ARM::tCBZ || Inst.getOpcode() == ARM::tCBNZ) &&
       RelaxedOp == ARM::tHINT) {
     Res.setOpcode(RelaxedOp);
-    Res.addOperand(MCOperand::CreateImm(0));
-    Res.addOperand(MCOperand::CreateImm(14));
-    Res.addOperand(MCOperand::CreateReg(0));
+    Res.addOperand(MCOperand::createImm(0));
+    Res.addOperand(MCOperand::createImm(14));
+    Res.addOperand(MCOperand::createReg(0));
     return;
   }
 
@@ -261,9 +260,9 @@ bool ARMAsmBackend::writeNopData(uint64_t Count, MCObjectWriter *OW) const {
         hasNOP() ? Thumb2_16bitNopEncoding : Thumb1_16bitNopEncoding;
     uint64_t NumNops = Count / 2;
     for (uint64_t i = 0; i != NumNops; ++i)
-      OW->Write16(nopEncoding);
+      OW->write16(nopEncoding);
     if (Count & 1)
-      OW->Write8(0);
+      OW->write8(0);
     return true;
   }
   // ARM mode
@@ -271,21 +270,21 @@ bool ARMAsmBackend::writeNopData(uint64_t Count, MCObjectWriter *OW) const {
       hasNOP() ? ARMv6T2_NopEncoding : ARMv4_NopEncoding;
   uint64_t NumNops = Count / 4;
   for (uint64_t i = 0; i != NumNops; ++i)
-    OW->Write32(nopEncoding);
+    OW->write32(nopEncoding);
   // FIXME: should this function return false when unable to write exactly
   // 'Count' bytes with NOP encodings?
   switch (Count % 4) {
   default:
     break; // No leftover bytes to write
   case 1:
-    OW->Write8(0);
+    OW->write8(0);
     break;
   case 2:
-    OW->Write16(0);
+    OW->write16(0);
     break;
   case 3:
-    OW->Write16(0);
-    OW->Write8(0xa0);
+    OW->write16(0);
+    OW->write8(0xa0);
     break;
   }
 
@@ -374,7 +373,7 @@ static unsigned adjustFixupValue(const MCFixup &Fixup, uint64_t Value,
       isAdd = false;
     }
     if (Ctx && Value >= 4096)
-      Ctx->FatalError(Fixup.getLoc(), "out of range pc-relative fixup value");
+      Ctx->reportFatalError(Fixup.getLoc(), "out of range pc-relative fixup value");
     Value |= isAdd << 23;
 
     // Same addressing mode as fixup_arm_pcrel_10,
@@ -395,7 +394,7 @@ static unsigned adjustFixupValue(const MCFixup &Fixup, uint64_t Value,
       opc = 2; // 0b0010
     }
     if (Ctx && ARM_AM::getSOImmVal(Value) == -1)
-      Ctx->FatalError(Fixup.getLoc(), "out of range pc-relative fixup value");
+      Ctx->reportFatalError(Fixup.getLoc(), "out of range pc-relative fixup value");
     // Encode the immediate and shift the opcode into place.
     return ARM_AM::getSOImmVal(Value) | (opc << 21);
   }
@@ -544,7 +543,7 @@ static unsigned adjustFixupValue(const MCFixup &Fixup, uint64_t Value,
     }
     // The value has the low 4 bits encoded in [3:0] and the high 4 in [11:8].
     if (Ctx && Value >= 256)
-      Ctx->FatalError(Fixup.getLoc(), "out of range pc-relative fixup value");
+      Ctx->reportFatalError(Fixup.getLoc(), "out of range pc-relative fixup value");
     Value = (Value & 0xf) | ((Value & 0xf0) << 4);
     return Value | (isAdd << 23);
   }
@@ -563,7 +562,7 @@ static unsigned adjustFixupValue(const MCFixup &Fixup, uint64_t Value,
     // These values don't encode the low two bits since they're always zero.
     Value >>= 2;
     if (Ctx && Value >= 256)
-      Ctx->FatalError(Fixup.getLoc(), "out of range pc-relative fixup value");
+      Ctx->reportFatalError(Fixup.getLoc(), "out of range pc-relative fixup value");
     Value |= isAdd << 23;
 
     // Same addressing mode as fixup_arm_pcrel_10, but with 16-bit halfwords
@@ -602,8 +601,7 @@ void ARMAsmBackend::processFixupValue(const MCAssembler &Asm,
   // the offset when the destination has the same MCFragment.
   if (A && (unsigned)Fixup.getKind() == ARM::fixup_arm_thumb_bl) {
     const MCSymbol &Sym = A->getSymbol();
-    const MCSymbolData &SymData = Asm.getSymbolData(Sym);
-    IsResolved = (SymData.getFragment() == DF);
+    IsResolved = (Sym.getFragment() == DF);
   }
   // We must always generate a relocation for BL/BLX instructions if we have
   // a symbol to reference, as the linker relies on knowing the destination
@@ -746,10 +744,9 @@ void ARMAsmBackend::applyFixup(const MCFixup &Fixup, char *Data,
 }
 
 MCAsmBackend *llvm::createARMAsmBackend(const Target &T,
-                                        const MCRegisterInfo &MRI, StringRef TT,
-                                        StringRef CPU, bool isLittle) {
-  Triple TheTriple(TT);
-
+                                        const MCRegisterInfo &MRI,
+                                        const Triple &TheTriple, StringRef CPU,
+                                        bool isLittle) {
   switch (TheTriple.getObjectFormat()) {
   default:
     llvm_unreachable("unsupported object format");
@@ -766,42 +763,38 @@ MCAsmBackend *llvm::createARMAsmBackend(const Target &T,
             .Cases("armv7s", "thumbv7s", MachO::CPU_SUBTYPE_ARM_V7S)
             .Default(MachO::CPU_SUBTYPE_ARM_V7);
 
-    return new ARMAsmBackendDarwin(T, TT, CS);
+    return new ARMAsmBackendDarwin(T, TheTriple, CS);
   }
   case Triple::COFF:
     assert(TheTriple.isOSWindows() && "non-Windows ARM COFF is not supported");
-    return new ARMAsmBackendWinCOFF(T, TT);
+    return new ARMAsmBackendWinCOFF(T, TheTriple);
   case Triple::ELF:
     assert(TheTriple.isOSBinFormatELF() && "using ELF for non-ELF target");
-    uint8_t OSABI = MCELFObjectTargetWriter::getOSABI(Triple(TT).getOS());
-    // @LOCALMOD-END
-    if (TheTriple.isOSNaCl())
-      return new ARMAsmBackendNaClELF(T, TT, OSABI, isLittle);
-    // @LOCALMOD-END
-    return new ARMAsmBackendELF(T, TT, OSABI, isLittle);
+    uint8_t OSABI = MCELFObjectTargetWriter::getOSABI(TheTriple.getOS());
+    return new ARMAsmBackendELF(T, TheTriple, OSABI, isLittle);
   }
 }
 
 MCAsmBackend *llvm::createARMLEAsmBackend(const Target &T,
                                           const MCRegisterInfo &MRI,
-                                          StringRef TT, StringRef CPU) {
+                                          const Triple &TT, StringRef CPU) {
   return createARMAsmBackend(T, MRI, TT, CPU, true);
 }
 
 MCAsmBackend *llvm::createARMBEAsmBackend(const Target &T,
                                           const MCRegisterInfo &MRI,
-                                          StringRef TT, StringRef CPU) {
+                                          const Triple &TT, StringRef CPU) {
   return createARMAsmBackend(T, MRI, TT, CPU, false);
 }
 
 MCAsmBackend *llvm::createThumbLEAsmBackend(const Target &T,
                                             const MCRegisterInfo &MRI,
-                                            StringRef TT, StringRef CPU) {
+                                            const Triple &TT, StringRef CPU) {
   return createARMAsmBackend(T, MRI, TT, CPU, true);
 }
 
 MCAsmBackend *llvm::createThumbBEAsmBackend(const Target &T,
                                             const MCRegisterInfo &MRI,
-                                            StringRef TT, StringRef CPU) {
+                                            const Triple &TT, StringRef CPU) {
   return createARMAsmBackend(T, MRI, TT, CPU, false);
 }
diff --git a/lib/Target/ARM/MCTargetDesc/ARMAsmBackend.h b/lib/Target/ARM/MCTargetDesc/ARMAsmBackend.h
index 4fa8c79e5af..6b4abd5898e 100644
--- a/lib/Target/ARM/MCTargetDesc/ARMAsmBackend.h
+++ b/lib/Target/ARM/MCTargetDesc/ARMAsmBackend.h
@@ -23,9 +23,10 @@ class ARMAsmBackend : public MCAsmBackend {
   bool isThumbMode;    // Currently emitting Thumb code.
   bool IsLittleEndian; // Big or little endian.
 public:
-  ARMAsmBackend(const Target &T, StringRef TT, bool IsLittle)
+  ARMAsmBackend(const Target &T, const Triple &TT, bool IsLittle)
       : MCAsmBackend(), STI(ARM_MC::createARMMCSubtargetInfo(TT, "", "")),
-        isThumbMode(TT.startswith("thumb")), IsLittleEndian(IsLittle) {}
+        isThumbMode(TT.getArchName().startswith("thumb")),
+        IsLittleEndian(IsLittle) {}
 
   ~ARMAsmBackend() override { delete STI; }
 
@@ -33,7 +34,7 @@ class ARMAsmBackend : public MCAsmBackend {
     return ARM::NumTargetFixupKinds;
   }
 
-  bool hasNOP() const { return (STI->getFeatureBits() & ARM::HasV6T2Ops) != 0; }
+  bool hasNOP() const { return STI->getFeatureBits()[ARM::HasV6T2Ops]; }
 
   const MCFixupKindInfo &getFixupKindInfo(MCFixupKind Kind) const override;
 
diff --git a/lib/Target/ARM/MCTargetDesc/ARMAsmBackendDarwin.h b/lib/Target/ARM/MCTargetDesc/ARMAsmBackendDarwin.h
index ebef78937b5..a6206e3d958 100644
--- a/lib/Target/ARM/MCTargetDesc/ARMAsmBackendDarwin.h
+++ b/lib/Target/ARM/MCTargetDesc/ARMAsmBackendDarwin.h
@@ -18,7 +18,8 @@ namespace {
 class ARMAsmBackendDarwin : public ARMAsmBackend {
 public:
   const MachO::CPUSubTypeARM Subtype;
-  ARMAsmBackendDarwin(const Target &T, StringRef TT, MachO::CPUSubTypeARM st)
+  ARMAsmBackendDarwin(const Target &T, const Triple &TT,
+                      MachO::CPUSubTypeARM st)
       : ARMAsmBackend(T, TT, /* IsLittleEndian */ true), Subtype(st) {
     HasDataInCodeSupport = true;
   }
diff --git a/lib/Target/ARM/MCTargetDesc/ARMAsmBackendELF.h b/lib/Target/ARM/MCTargetDesc/ARMAsmBackendELF.h
index 263c4c488ac..68b12edd089 100644
--- a/lib/Target/ARM/MCTargetDesc/ARMAsmBackendELF.h
+++ b/lib/Target/ARM/MCTargetDesc/ARMAsmBackendELF.h
@@ -15,7 +15,8 @@ namespace {
 class ARMAsmBackendELF : public ARMAsmBackend {
 public:
   uint8_t OSABI;
-  ARMAsmBackendELF(const Target &T, StringRef TT, uint8_t OSABI, bool IsLittle)
+  ARMAsmBackendELF(const Target &T, const Triple &TT, uint8_t OSABI,
+                   bool IsLittle)
       : ARMAsmBackend(T, TT, IsLittle), OSABI(OSABI) {}
 
   MCObjectWriter *createObjectWriter(raw_pwrite_stream &OS) const override {
diff --git a/lib/Target/ARM/MCTargetDesc/ARMAsmBackendNaClELF.h b/lib/Target/ARM/MCTargetDesc/ARMAsmBackendNaClELF.h
deleted file mode 100644
index c2caf15bd16..00000000000
--- a/lib/Target/ARM/MCTargetDesc/ARMAsmBackendNaClELF.h
+++ /dev/null
@@ -1,47 +0,0 @@
-//===-- ARMAsmBackendNaClELF.h  ARM Asm Backend NaCl ELF --------*- C++ -*-===//
-//
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
-//
-//===----------------------------------------------------------------------===//
-
-#ifndef LLVM_LIB_TARGET_ARM_ARMASMBACKENDNACLELF_H
-#define LLVM_LIB_TARGET_ARM_ARMASMBACKENDNACLELF_H
-
-// This whole file is a @LOCALMOD
-#include "MCTargetDesc/ARMAsmBackendELF.h"
-#include "MCTargetDesc/ARMMCNaCl.h"
-
-using namespace llvm;
-
-namespace {
-class ARMAsmBackendNaClELF : public ARMAsmBackendELF {
-public:
-  ARMAsmBackendNaClELF(const Target &T, const StringRef TT, uint8_t _OSABI,
-                    bool isLittle)
-      : ARMAsmBackendELF(T, TT, _OSABI, isLittle),
-        STI(ARM_MC::createARMMCSubtargetInfo(TT, "", "")) {
-    assert(isLittle && "NaCl only supports little-endian");
-    State.SaveCount = 0;
-    State.I = 0;
-    State.RecursiveCall = false;
-  }
-
-  ~ARMAsmBackendNaClELF() override {}
-
-  bool CustomExpandInst(const MCInst &Inst, MCStreamer &Out) override {
-    return CustomExpandInstNaClARM(*STI, Inst, Out, State);
-  }
-
-private:
-  // TODO(jfb) When upstreaming this class we can drop STI since ARMAsmBackend
-  //           already has one. It's unfortunately private so we recreate one
-  //           here to avoid the extra localmod.
-  std::unique_ptr<MCSubtargetInfo> STI;
-  ARMMCNaClSFIState State;
-};
-} // end anonymous namespace
-
-#endif
diff --git a/lib/Target/ARM/MCTargetDesc/ARMAsmBackendWinCOFF.h b/lib/Target/ARM/MCTargetDesc/ARMAsmBackendWinCOFF.h
index f2c435820ad..170f59a4c90 100644
--- a/lib/Target/ARM/MCTargetDesc/ARMAsmBackendWinCOFF.h
+++ b/lib/Target/ARM/MCTargetDesc/ARMAsmBackendWinCOFF.h
@@ -15,8 +15,8 @@ using namespace llvm;
 namespace {
 class ARMAsmBackendWinCOFF : public ARMAsmBackend {
 public:
-  ARMAsmBackendWinCOFF(const Target &T, StringRef Triple)
-      : ARMAsmBackend(T, Triple, true) {}
+  ARMAsmBackendWinCOFF(const Target &T, const Triple &TheTriple)
+      : ARMAsmBackend(T, TheTriple, true) {}
   MCObjectWriter *createObjectWriter(raw_pwrite_stream &OS) const override {
     return createARMWinCOFFObjectWriter(OS, /*Is64Bit=*/false);
   }
diff --git a/lib/Target/ARM/MCTargetDesc/ARMELFObjectWriter.cpp b/lib/Target/ARM/MCTargetDesc/ARMELFObjectWriter.cpp
index f4fedeef650..804d3534096 100644
--- a/lib/Target/ARM/MCTargetDesc/ARMELFObjectWriter.cpp
+++ b/lib/Target/ARM/MCTargetDesc/ARMELFObjectWriter.cpp
@@ -37,7 +37,7 @@ namespace {
     unsigned GetRelocType(const MCValue &Target, const MCFixup &Fixup,
                           bool IsPCRel) const override;
 
-    bool needsRelocateWithSymbol(const MCSymbolData &SD,
+    bool needsRelocateWithSymbol(const MCSymbol &Sym,
                                  unsigned Type) const override;
   };
 }
@@ -49,7 +49,7 @@ ARMELFObjectWriter::ARMELFObjectWriter(uint8_t OSABI)
 
 ARMELFObjectWriter::~ARMELFObjectWriter() {}
 
-bool ARMELFObjectWriter::needsRelocateWithSymbol(const MCSymbolData &SD,
+bool ARMELFObjectWriter::needsRelocateWithSymbol(const MCSymbol &Sym,
                                                  unsigned Type) const {
   // FIXME: This is extremely conservative. This really needs to use a
   // whitelist with a clear explanation for why each realocation needs to
diff --git a/lib/Target/ARM/MCTargetDesc/ARMELFStreamer.cpp b/lib/Target/ARM/MCTargetDesc/ARMELFStreamer.cpp
index e7c777e26f9..d17fdb95dbd 100644
--- a/lib/Target/ARM/MCTargetDesc/ARMELFStreamer.cpp
+++ b/lib/Target/ARM/MCTargetDesc/ARMELFStreamer.cpp
@@ -13,9 +13,6 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "ARMArchName.h"
-#include "ARMFPUName.h"
-#include "ARMArchExtName.h"
 #include "ARMRegisterInfo.h"
 #include "ARMUnwindOpAsm.h"
 #include "llvm/ADT/StringExtras.h"
@@ -25,9 +22,7 @@
 #include "llvm/MC/MCAssembler.h"
 #include "llvm/MC/MCCodeEmitter.h"
 #include "llvm/MC/MCContext.h"
-#include "llvm/MC/MCELF.h"
 #include "llvm/MC/MCELFStreamer.h"
-#include "llvm/MC/MCELFSymbolFlags.h"
 #include "llvm/MC/MCExpr.h"
 #include "llvm/MC/MCInst.h"
 #include "llvm/MC/MCInstPrinter.h"
@@ -37,10 +32,11 @@
 #include "llvm/MC/MCSection.h"
 #include "llvm/MC/MCSectionELF.h"
 #include "llvm/MC/MCStreamer.h"
-#include "llvm/MC/MCSymbol.h"
+#include "llvm/MC/MCSymbolELF.h"
 #include "llvm/MC/MCValue.h"
 #include "llvm/Support/ARMBuildAttributes.h"
 #include "llvm/Support/ARMEHABI.h"
+#include "llvm/Support/TargetParser.h"
 #include "llvm/Support/Debug.h"
 #include "llvm/Support/ELF.h"
 #include "llvm/Support/FormattedStream.h"
@@ -56,69 +52,6 @@ static std::string GetAEABIUnwindPersonalityName(unsigned Index) {
   return (Twine("__aeabi_unwind_cpp_pr") + Twine(Index)).str();
 }
 
-static const char *GetFPUName(unsigned ID) {
-  switch (ID) {
-  default:
-    llvm_unreachable("Unknown FPU kind");
-    break;
-#define ARM_FPU_NAME(NAME, ID) case ARM::ID: return NAME;
-#include "ARMFPUName.def"
-  }
-  return nullptr;
-}
-
-static const char *GetArchName(unsigned ID) {
-  switch (ID) {
-  default:
-    llvm_unreachable("Unknown ARCH kind");
-    break;
-#define ARM_ARCH_NAME(NAME, ID, DEFAULT_CPU_NAME, DEFAULT_CPU_ARCH) \
-  case ARM::ID: return NAME;
-#define ARM_ARCH_ALIAS(NAME, ID) /* empty */
-#include "ARMArchName.def"
-  }
-  return nullptr;
-}
-
-static const char *GetArchDefaultCPUName(unsigned ID) {
-  switch (ID) {
-  default:
-    llvm_unreachable("Unknown ARCH kind");
-    break;
-#define ARM_ARCH_NAME(NAME, ID, DEFAULT_CPU_NAME, DEFAULT_CPU_ARCH) \
-  case ARM::ID: return DEFAULT_CPU_NAME;
-#define ARM_ARCH_ALIAS(NAME, ID) /* empty */
-#include "ARMArchName.def"
-  }
-  return nullptr;
-}
-
-static unsigned GetArchDefaultCPUArch(unsigned ID) {
-  switch (ID) {
-  default:
-    llvm_unreachable("Unknown ARCH kind");
-    break;
-#define ARM_ARCH_NAME(NAME, ID, DEFAULT_CPU_NAME, DEFAULT_CPU_ARCH) \
-  case ARM::ID: return ARMBuildAttrs::DEFAULT_CPU_ARCH;
-#define ARM_ARCH_ALIAS(NAME, ID) /* empty */
-#include "ARMArchName.def"
-  }
-  return 0;
-}
-
-static const char *GetArchExtName(unsigned ID) {
-  switch (ID) {
-  default:
-    llvm_unreachable("Unknown ARCH Extension kind");
-    break;
-#define ARM_ARCHEXT_NAME(NAME, ID)                                             \
-  case ARM::ID:                                                                \
-    return NAME;
-#include "ARMArchExtName.def"
-  }
-  return nullptr;
-}
-
 namespace {
 
 class ARMELFStreamer;
@@ -262,16 +195,16 @@ void ARMTargetAsmStreamer::emitIntTextAttribute(unsigned Attribute,
   OS << "\n";
 }
 void ARMTargetAsmStreamer::emitArch(unsigned Arch) {
-  OS << "\t.arch\t" << GetArchName(Arch) << "\n";
+  OS << "\t.arch\t" << ARMTargetParser::getArchName(Arch) << "\n";
 }
 void ARMTargetAsmStreamer::emitArchExtension(unsigned ArchExt) {
-  OS << "\t.arch_extension\t" << GetArchExtName(ArchExt) << "\n";
+  OS << "\t.arch_extension\t" << ARMTargetParser::getArchExtName(ArchExt) << "\n";
 }
 void ARMTargetAsmStreamer::emitObjectArch(unsigned Arch) {
-  OS << "\t.object_arch\t" << GetArchName(Arch) << '\n';
+  OS << "\t.object_arch\t" << ARMTargetParser::getArchName(Arch) << '\n';
 }
 void ARMTargetAsmStreamer::emitFPU(unsigned FPU) {
-  OS << "\t.fpu\t" << GetFPUName(FPU) << "\n";
+  OS << "\t.fpu\t" << ARMTargetParser::getFPUName(FPU) << "\n";
 }
 void ARMTargetAsmStreamer::finishAttributeSection() {
 }
@@ -281,14 +214,20 @@ ARMTargetAsmStreamer::AnnotateTLSDescriptorSequence(const MCSymbolRefExpr *S) {
 }
 
 void ARMTargetAsmStreamer::emitThumbSet(MCSymbol *Symbol, const MCExpr *Value) {
-  OS << "\t.thumb_set\t" << *Symbol << ", " << *Value << '\n';
+  const MCAsmInfo *MAI = Streamer.getContext().getAsmInfo();
+
+  OS << "\t.thumb_set\t";
+  Symbol->print(OS, MAI);
+  OS << ", ";
+  Value->print(OS, MAI);
+  OS << '\n';
 }
 
 void ARMTargetAsmStreamer::emitInst(uint32_t Inst, char Suffix) {
   OS << "\t.inst";
   if (Suffix)
     OS << "." << Suffix;
-  OS << "\t0x" << utohexstr(Inst) << "\n";
+  OS << "\t0x" << Twine::utohexstr(Inst) << "\n";
 }
 
 void ARMTargetAsmStreamer::emitUnwindRaw(int64_t Offset,
@@ -297,7 +236,7 @@ void ARMTargetAsmStreamer::emitUnwindRaw(int64_t Offset,
   for (SmallVectorImpl<uint8_t>::const_iterator OCI = Opcodes.begin(),
                                                 OCE = Opcodes.end();
        OCI != OCE; ++OCI)
-    OS << ", 0x" << utohexstr(*OCI);
+    OS << ", 0x" << Twine::utohexstr(*OCI);
   OS << '\n';
 }
 
@@ -340,7 +279,7 @@ class ARMTargetELFStreamer : public ARMTargetStreamer {
   unsigned EmittedArch;
   SmallVector<AttributeItem, 64> Contents;
 
-  const MCSection *AttributeSection;
+  MCSection *AttributeSection;
 
   AttributeItem *getAttributeItem(unsigned Attribute) {
     for (size_t i = 0; i < Contents.size(); ++i)
@@ -451,8 +390,8 @@ class ARMTargetELFStreamer : public ARMTargetStreamer {
 
 public:
   ARMTargetELFStreamer(MCStreamer &S)
-    : ARMTargetStreamer(S), CurrentVendor("aeabi"), FPU(ARM::INVALID_FPU),
-      Arch(ARM::INVALID_ARCH), EmittedArch(ARM::INVALID_ARCH),
+    : ARMTargetStreamer(S), CurrentVendor("aeabi"), FPU(ARM::FK_INVALID),
+      Arch(ARM::AK_INVALID), EmittedArch(ARM::AK_INVALID),
       AttributeSection(nullptr) {}
 };
 
@@ -496,8 +435,7 @@ class ARMELFStreamer : public MCELFStreamer {
   void emitRegSave(const SmallVectorImpl<unsigned> &RegList, bool isVector);
   void emitUnwindRaw(int64_t Offset, const SmallVectorImpl<uint8_t> &Opcodes);
 
-  void ChangeSection(const MCSection *Section,
-                     const MCExpr *Subsection) override {
+  void ChangeSection(MCSection *Section, const MCExpr *Subsection) override {
     // We have to keep track of the mapping symbol state of any sections we
     // use. Each one should start off as EMS_None, which is provided as the
     // default constructor by DenseMap::lookup.
@@ -573,7 +511,7 @@ class ARMELFStreamer : public MCELFStreamer {
                      const SMLoc &Loc) override {
     if (const MCSymbolRefExpr *SRE = dyn_cast_or_null<MCSymbolRefExpr>(Value))
       if (SRE->getKind() == MCSymbolRefExpr::VK_ARM_SBREL && !(Size == 4))
-        getContext().FatalError(Loc, "relocated expression must be 32-bit");
+        getContext().reportFatalError(Loc, "relocated expression must be 32-bit");
 
     EmitDataMappingSymbol();
     MCELFStreamer::EmitValueImpl(Value, Size);
@@ -625,21 +563,13 @@ class ARMELFStreamer : public MCELFStreamer {
   }
 
   void EmitMappingSymbol(StringRef Name) {
-    MCSymbol *Start = getContext().CreateTempSymbol();
-    EmitLabel(Start);
+    auto *Symbol = cast<MCSymbolELF>(getContext().getOrCreateSymbol(
+        Name + "." + Twine(MappingSymbolCounter++)));
+    EmitLabel(Symbol);
 
-    MCSymbol *Symbol =
-      getContext().GetOrCreateSymbol(Name + "." +
-                                     Twine(MappingSymbolCounter++));
-
-    MCSymbolData &SD = getAssembler().getOrCreateSymbolData(*Symbol);
-    MCELF::SetType(SD, ELF::STT_NOTYPE);
-    MCELF::SetBinding(SD, ELF::STB_LOCAL);
-    SD.setExternal(false);
-    AssignSection(Symbol, getCurrentSection().first);
-
-    const MCExpr *Value = MCSymbolRefExpr::Create(Start, getContext());
-    Symbol->setVariableValue(Value);
+    Symbol->setType(ELF::STT_NOTYPE);
+    Symbol->setBinding(ELF::STB_LOCAL);
+    Symbol->setExternal(false);
   }
 
   void EmitThumbFunc(MCSymbol *Func) override {
@@ -753,71 +683,78 @@ void ARMTargetELFStreamer::emitObjectArch(unsigned Value) {
 void ARMTargetELFStreamer::emitArchDefaultAttributes() {
   using namespace ARMBuildAttrs;
 
-  setAttributeItem(CPU_name, GetArchDefaultCPUName(Arch), false);
-  if (EmittedArch == ARM::INVALID_ARCH)
-    setAttributeItem(CPU_arch, GetArchDefaultCPUArch(Arch), false);
+  setAttributeItem(CPU_name,
+                   ARMTargetParser::getCPUAttr(Arch),
+                   false);
+
+  if (EmittedArch == ARM::AK_INVALID)
+    setAttributeItem(CPU_arch,
+                     ARMTargetParser::getArchAttr(Arch),
+                     false);
   else
-    setAttributeItem(CPU_arch, GetArchDefaultCPUArch(EmittedArch), false);
+    setAttributeItem(CPU_arch,
+                     ARMTargetParser::getArchAttr(EmittedArch),
+                     false);
 
   switch (Arch) {
-  case ARM::ARMV2:
-  case ARM::ARMV2A:
-  case ARM::ARMV3:
-  case ARM::ARMV3M:
-  case ARM::ARMV4:
-  case ARM::ARMV5:
+  case ARM::AK_ARMV2:
+  case ARM::AK_ARMV2A:
+  case ARM::AK_ARMV3:
+  case ARM::AK_ARMV3M:
+  case ARM::AK_ARMV4:
+  case ARM::AK_ARMV5:
     setAttributeItem(ARM_ISA_use, Allowed, false);
     break;
 
-  case ARM::ARMV4T:
-  case ARM::ARMV5T:
-  case ARM::ARMV5TE:
-  case ARM::ARMV6:
-  case ARM::ARMV6J:
+  case ARM::AK_ARMV4T:
+  case ARM::AK_ARMV5T:
+  case ARM::AK_ARMV5TE:
+  case ARM::AK_ARMV6:
+  case ARM::AK_ARMV6J:
     setAttributeItem(ARM_ISA_use, Allowed, false);
     setAttributeItem(THUMB_ISA_use, Allowed, false);
     break;
 
-  case ARM::ARMV6T2:
+  case ARM::AK_ARMV6T2:
     setAttributeItem(ARM_ISA_use, Allowed, false);
     setAttributeItem(THUMB_ISA_use, AllowThumb32, false);
     break;
 
-  case ARM::ARMV6K:
-  case ARM::ARMV6Z:
-  case ARM::ARMV6ZK:
+  case ARM::AK_ARMV6K:
+  case ARM::AK_ARMV6Z:
+  case ARM::AK_ARMV6ZK:
     setAttributeItem(ARM_ISA_use, Allowed, false);
     setAttributeItem(THUMB_ISA_use, Allowed, false);
     setAttributeItem(Virtualization_use, AllowTZ, false);
     break;
 
-  case ARM::ARMV6M:
+  case ARM::AK_ARMV6M:
     setAttributeItem(THUMB_ISA_use, Allowed, false);
     break;
 
-  case ARM::ARMV7:
+  case ARM::AK_ARMV7:
     setAttributeItem(THUMB_ISA_use, AllowThumb32, false);
     break;
 
-  case ARM::ARMV7A:
+  case ARM::AK_ARMV7A:
     setAttributeItem(CPU_arch_profile, ApplicationProfile, false);
     setAttributeItem(ARM_ISA_use, Allowed, false);
     setAttributeItem(THUMB_ISA_use, AllowThumb32, false);
     break;
 
-  case ARM::ARMV7R:
+  case ARM::AK_ARMV7R:
     setAttributeItem(CPU_arch_profile, RealTimeProfile, false);
     setAttributeItem(ARM_ISA_use, Allowed, false);
     setAttributeItem(THUMB_ISA_use, AllowThumb32, false);
     break;
 
-  case ARM::ARMV7M:
+  case ARM::AK_ARMV7M:
     setAttributeItem(CPU_arch_profile, MicroControllerProfile, false);
     setAttributeItem(THUMB_ISA_use, AllowThumb32, false);
     break;
 
-  case ARM::ARMV8A:
-  case ARM::ARMV8_1A:
+  case ARM::AK_ARMV8A:
+  case ARM::AK_ARMV8_1A:
     setAttributeItem(CPU_arch_profile, ApplicationProfile, false);
     setAttributeItem(ARM_ISA_use, Allowed, false);
     setAttributeItem(THUMB_ISA_use, AllowThumb32, false);
@@ -825,13 +762,13 @@ void ARMTargetELFStreamer::emitArchDefaultAttributes() {
     setAttributeItem(Virtualization_use, AllowTZVirtualization, false);
     break;
 
-  case ARM::IWMMXT:
+  case ARM::AK_IWMMXT:
     setAttributeItem(ARM_ISA_use, Allowed, false);
     setAttributeItem(THUMB_ISA_use, Allowed, false);
     setAttributeItem(WMMX_arch, AllowWMMXv1, false);
     break;
 
-  case ARM::IWMMXT2:
+  case ARM::AK_IWMMXT2:
     setAttributeItem(ARM_ISA_use, Allowed, false);
     setAttributeItem(THUMB_ISA_use, Allowed, false);
     setAttributeItem(WMMX_arch, AllowWMMXv2, false);
@@ -847,38 +784,73 @@ void ARMTargetELFStreamer::emitFPU(unsigned Value) {
 }
 void ARMTargetELFStreamer::emitFPUDefaultAttributes() {
   switch (FPU) {
-  case ARM::VFP:
-  case ARM::VFPV2:
+  case ARM::FK_VFP:
+  case ARM::FK_VFPV2:
     setAttributeItem(ARMBuildAttrs::FP_arch,
                      ARMBuildAttrs::AllowFPv2,
                      /* OverwriteExisting= */ false);
     break;
 
-  case ARM::VFPV3:
+  case ARM::FK_VFPV3:
     setAttributeItem(ARMBuildAttrs::FP_arch,
                      ARMBuildAttrs::AllowFPv3A,
                      /* OverwriteExisting= */ false);
     break;
 
-  case ARM::VFPV3_D16:
+  case ARM::FK_VFPV3_FP16:
+    setAttributeItem(ARMBuildAttrs::FP_arch,
+                     ARMBuildAttrs::AllowFPv3A,
+                     /* OverwriteExisting= */ false);
+    setAttributeItem(ARMBuildAttrs::FP_HP_extension,
+                     ARMBuildAttrs::AllowHPFP,
+                     /* OverwriteExisting= */ false);
+    break;
+
+  case ARM::FK_VFPV3_D16:
+    setAttributeItem(ARMBuildAttrs::FP_arch,
+                     ARMBuildAttrs::AllowFPv3B,
+                     /* OverwriteExisting= */ false);
+    break;
+
+  case ARM::FK_VFPV3_D16_FP16:
     setAttributeItem(ARMBuildAttrs::FP_arch,
                      ARMBuildAttrs::AllowFPv3B,
                      /* OverwriteExisting= */ false);
+    setAttributeItem(ARMBuildAttrs::FP_HP_extension,
+                     ARMBuildAttrs::AllowHPFP,
+                     /* OverwriteExisting= */ false);
     break;
 
-  case ARM::VFPV4:
+  case ARM::FK_VFPV3XD:
+    setAttributeItem(ARMBuildAttrs::FP_arch,
+                     ARMBuildAttrs::AllowFPv3B,
+                     /* OverwriteExisting= */ false);
+    break;
+  case ARM::FK_VFPV3XD_FP16:
+    setAttributeItem(ARMBuildAttrs::FP_arch,
+                     ARMBuildAttrs::AllowFPv3B,
+                     /* OverwriteExisting= */ false);
+    setAttributeItem(ARMBuildAttrs::FP_HP_extension,
+                     ARMBuildAttrs::AllowHPFP,
+                     /* OverwriteExisting= */ false);
+    break;
+
+  case ARM::FK_VFPV4:
     setAttributeItem(ARMBuildAttrs::FP_arch,
                      ARMBuildAttrs::AllowFPv4A,
                      /* OverwriteExisting= */ false);
     break;
 
-  case ARM::VFPV4_D16:
+  // ABI_HardFP_use is handled in ARMAsmPrinter, so _SP_D16 is treated the same
+  // as _D16 here.
+  case ARM::FK_FPV4_SP_D16:
+  case ARM::FK_VFPV4_D16:
     setAttributeItem(ARMBuildAttrs::FP_arch,
                      ARMBuildAttrs::AllowFPv4B,
                      /* OverwriteExisting= */ false);
     break;
 
-  case ARM::FP_ARMV8:
+  case ARM::FK_FP_ARMV8:
     setAttributeItem(ARMBuildAttrs::FP_arch,
                      ARMBuildAttrs::AllowFPARMv8A,
                      /* OverwriteExisting= */ false);
@@ -886,13 +858,14 @@ void ARMTargetELFStreamer::emitFPUDefaultAttributes() {
 
   // FPV5_D16 is identical to FP_ARMV8 except for the number of D registers, so
   // uses the FP_ARMV8_D16 build attribute.
-  case ARM::FPV5_D16:
+  case ARM::FK_FPV5_SP_D16:
+  case ARM::FK_FPV5_D16:
     setAttributeItem(ARMBuildAttrs::FP_arch,
                      ARMBuildAttrs::AllowFPARMv8B,
                      /* OverwriteExisting= */ false);
     break;
 
-  case ARM::NEON:
+  case ARM::FK_NEON:
     setAttributeItem(ARMBuildAttrs::FP_arch,
                      ARMBuildAttrs::AllowFPv3A,
                      /* OverwriteExisting= */ false);
@@ -901,7 +874,19 @@ void ARMTargetELFStreamer::emitFPUDefaultAttributes() {
                      /* OverwriteExisting= */ false);
     break;
 
-  case ARM::NEON_VFPV4:
+  case ARM::FK_NEON_FP16:
+    setAttributeItem(ARMBuildAttrs::FP_arch,
+                     ARMBuildAttrs::AllowFPv3A,
+                     /* OverwriteExisting= */ false);
+    setAttributeItem(ARMBuildAttrs::Advanced_SIMD_arch,
+                     ARMBuildAttrs::AllowNeon,
+                     /* OverwriteExisting= */ false);
+    setAttributeItem(ARMBuildAttrs::FP_HP_extension,
+                     ARMBuildAttrs::AllowHPFP,
+                     /* OverwriteExisting= */ false);
+    break;
+
+  case ARM::FK_NEON_VFPV4:
     setAttributeItem(ARMBuildAttrs::FP_arch,
                      ARMBuildAttrs::AllowFPv4A,
                      /* OverwriteExisting= */ false);
@@ -910,8 +895,8 @@ void ARMTargetELFStreamer::emitFPUDefaultAttributes() {
                      /* OverwriteExisting= */ false);
     break;
 
-  case ARM::NEON_FP_ARMV8:
-  case ARM::CRYPTO_NEON_FP_ARMV8:
+  case ARM::FK_NEON_FP_ARMV8:
+  case ARM::FK_CRYPTO_NEON_FP_ARMV8:
     setAttributeItem(ARMBuildAttrs::FP_arch,
                      ARMBuildAttrs::AllowFPARMv8A,
                      /* OverwriteExisting= */ false);
@@ -919,7 +904,8 @@ void ARMTargetELFStreamer::emitFPUDefaultAttributes() {
     // ARMAsmPrinter::emitAttributes(), depending on hasV8Ops() and hasV8_1a()
     break;
 
-  case ARM::SOFTVFP:
+  case ARM::FK_SOFTVFP:
+  case ARM::FK_NONE:
     break;
 
   default:
@@ -960,10 +946,10 @@ void ARMTargetELFStreamer::finishAttributeSection() {
   //   ]+
   // ]*
 
-  if (FPU != ARM::INVALID_FPU)
+  if (FPU != ARM::FK_INVALID)
     emitFPUDefaultAttributes();
 
-  if (Arch != ARM::INVALID_ARCH)
+  if (Arch != ARM::AK_INVALID)
     emitArchDefaultAttributes();
 
   if (Contents.empty())
@@ -1023,7 +1009,7 @@ void ARMTargetELFStreamer::finishAttributeSection() {
   }
 
   Contents.clear();
-  FPU = ARM::INVALID_FPU;
+  FPU = ARM::FK_INVALID;
 }
 
 void ARMTargetELFStreamer::emitLabel(MCSymbol *Symbol) {
@@ -1031,9 +1017,9 @@ void ARMTargetELFStreamer::emitLabel(MCSymbol *Symbol) {
   if (!Streamer.IsThumb)
     return;
 
-  const MCSymbolData &SD = Streamer.getOrCreateSymbolData(Symbol);
-  unsigned Type = MCELF::GetType(SD);
-  if (Type == ELF_STT_Func || Type == ELF_STT_GnuIFunc)
+  Streamer.getAssembler().registerSymbol(*Symbol);
+  unsigned Type = cast<MCSymbolELF>(Symbol)->getType();
+  if (Type == ELF::STT_FUNC || Type == ELF::STT_GNU_IFUNC)
     Streamer.EmitThumbFunc(Symbol);
 }
 
@@ -1083,10 +1069,10 @@ inline void ARMELFStreamer::SwitchToEHSection(const char *Prefix,
   }
 
   // Get .ARM.extab or .ARM.exidx section
-  const MCSymbol *Group = FnSection.getGroup();
+  const MCSymbolELF *Group = FnSection.getGroup();
   if (Group)
     Flags |= ELF::SHF_GROUP;
-  const MCSectionELF *EHSection =
+  MCSectionELF *EHSection =
       getContext().getELFSection(EHSecName, Type, Flags, 0, Group,
                                  FnSection.getUniqueID(), nullptr, &FnSection);
 
@@ -1114,7 +1100,7 @@ inline void ARMELFStreamer::SwitchToExIdxSection(const MCSymbol &FnStart) {
 }
 void ARMELFStreamer::EmitFixup(const MCExpr *Expr, MCFixupKind Kind) {
   MCDataFragment *Frag = getOrCreateDataFragment();
-  Frag->getFixups().push_back(MCFixup::Create(Frag->getContents().size(), Expr,
+  Frag->getFixups().push_back(MCFixup::create(Frag->getContents().size(), Expr,
                                               Kind));
 }
 
@@ -1136,7 +1122,7 @@ void ARMELFStreamer::Reset() {
 
 void ARMELFStreamer::emitFnStart() {
   assert(FnStart == nullptr);
-  FnStart = getContext().CreateTempSymbol();
+  FnStart = getContext().createTempSymbol();
   EmitLabel(FnStart);
 }
 
@@ -1154,7 +1140,7 @@ void ARMELFStreamer::emitFnEnd() {
     EmitPersonalityFixup(GetAEABIUnwindPersonalityName(PersonalityIndex));
 
   const MCSymbolRefExpr *FnStartRef =
-    MCSymbolRefExpr::Create(FnStart,
+    MCSymbolRefExpr::create(FnStart,
                             MCSymbolRefExpr::VK_ARM_PREL31,
                             getContext());
 
@@ -1165,7 +1151,7 @@ void ARMELFStreamer::emitFnEnd() {
   } else if (ExTab) {
     // Emit a reference to the unwind opcodes in the ".ARM.extab" section.
     const MCSymbolRefExpr *ExTabEntryRef =
-      MCSymbolRefExpr::Create(ExTab,
+      MCSymbolRefExpr::create(ExTab,
                               MCSymbolRefExpr::VK_ARM_PREL31,
                               getContext());
     EmitValue(ExTabEntryRef, 4);
@@ -1195,14 +1181,14 @@ void ARMELFStreamer::emitCantUnwind() { CantUnwind = true; }
 
 // Add the R_ARM_NONE fixup at the same position
 void ARMELFStreamer::EmitPersonalityFixup(StringRef Name) {
-  const MCSymbol *PersonalitySym = getContext().GetOrCreateSymbol(Name);
+  const MCSymbol *PersonalitySym = getContext().getOrCreateSymbol(Name);
 
-  const MCSymbolRefExpr *PersonalityRef = MCSymbolRefExpr::Create(
+  const MCSymbolRefExpr *PersonalityRef = MCSymbolRefExpr::create(
       PersonalitySym, MCSymbolRefExpr::VK_ARM_NONE, getContext());
 
   visitUsedExpr(*PersonalityRef);
   MCDataFragment *DF = getOrCreateDataFragment();
-  DF->getFixups().push_back(MCFixup::Create(DF->getContents().size(),
+  DF->getFixups().push_back(MCFixup::create(DF->getContents().size(),
                                             PersonalityRef,
                                             MCFixup::getKindForSize(4, false)));
 }
@@ -1239,13 +1225,13 @@ void ARMELFStreamer::FlushUnwindOpcodes(bool NoHandlerData) {
 
   // Create .ARM.extab label for offset in .ARM.exidx
   assert(!ExTab);
-  ExTab = getContext().CreateTempSymbol();
+  ExTab = getContext().createTempSymbol();
   EmitLabel(ExTab);
 
   // Emit personality
   if (Personality) {
     const MCSymbolRefExpr *PersonalityRef =
-      MCSymbolRefExpr::Create(Personality,
+      MCSymbolRefExpr::create(Personality,
                               MCSymbolRefExpr::VK_ARM_PREL31,
                               getContext());
 
@@ -1375,8 +1361,8 @@ MCTargetStreamer *createARMNullTargetStreamer(MCStreamer &S) {
 
 MCTargetStreamer *createARMObjectTargetStreamer(MCStreamer &S,
                                                 const MCSubtargetInfo &STI) {
-  Triple TT(STI.getTargetTriple());
-  if (TT.getObjectFormat() == Triple::ELF)
+  const Triple &TT = STI.getTargetTriple();
+  if (TT.isOSBinFormatELF())
     return new ARMTargetELFStreamer(S);
   return new ARMTargetStreamer(S);
 }
diff --git a/lib/Target/ARM/MCTargetDesc/ARMMCAsmInfo.cpp b/lib/Target/ARM/MCTargetDesc/ARMMCAsmInfo.cpp
index a431fdee196..1ac08159bd3 100644
--- a/lib/Target/ARM/MCTargetDesc/ARMMCAsmInfo.cpp
+++ b/lib/Target/ARM/MCTargetDesc/ARMMCAsmInfo.cpp
@@ -17,19 +17,9 @@
 
 using namespace llvm;
 
-// @LOCALMOD-BEGIN
-namespace llvm {
-cl::opt<bool>
-EnableARMDwarfEH("arm-enable-dwarf-eh",
-                 cl::desc("Use DWARF EH instead of EABI EH on ARM"),
-                 cl::init(false));
-}
-// @LOCALMOD-END
-
 void ARMMCAsmInfoDarwin::anchor() { }
 
-ARMMCAsmInfoDarwin::ARMMCAsmInfoDarwin(StringRef TT) {
-  Triple TheTriple(TT);
+ARMMCAsmInfoDarwin::ARMMCAsmInfoDarwin(const Triple &TheTriple) {
   if ((TheTriple.getArch() == Triple::armeb) ||
       (TheTriple.getArch() == Triple::thumbeb))
     IsLittleEndian = false;
@@ -50,8 +40,7 @@ ARMMCAsmInfoDarwin::ARMMCAsmInfoDarwin(StringRef TT) {
 
 void ARMELFMCAsmInfo::anchor() { }
 
-ARMELFMCAsmInfo::ARMELFMCAsmInfo(StringRef TT) {
-  Triple TheTriple(TT);
+ARMELFMCAsmInfo::ARMELFMCAsmInfo(const Triple &TheTriple) {
   if ((TheTriple.getArch() == Triple::armeb) ||
       (TheTriple.getArch() == Triple::thumbeb))
     IsLittleEndian = false;
@@ -70,16 +59,12 @@ ARMELFMCAsmInfo::ARMELFMCAsmInfo(StringRef TT) {
   switch (TheTriple.getOS()) {
   case Triple::Bitrig:
   case Triple::NetBSD:
-  case Triple::NaCl: // @LOCALMOD: NaCl uses DWARF EH
     ExceptionsType = ExceptionHandling::DwarfCFI;
     break;
   default:
     ExceptionsType = ExceptionHandling::ARM;
     break;
   }
-  // @LOCALMOD: NonSFI mode uses DWARF EH
-  if (EnableARMDwarfEH)
-    ExceptionsType = ExceptionHandling::DwarfCFI;
 
   // foo(plt) instead of foo@plt
   UseParensForSymbolVariant = true;
@@ -125,3 +110,4 @@ ARMCOFFMCAsmInfoGNU::ARMCOFFMCAsmInfoGNU() {
   UseIntegratedAssembler = false;
   DwarfRegNumForCFI = true;
 }
+
diff --git a/lib/Target/ARM/MCTargetDesc/ARMMCAsmInfo.h b/lib/Target/ARM/MCTargetDesc/ARMMCAsmInfo.h
index 6cb471537f6..99a5fff5ec2 100644
--- a/lib/Target/ARM/MCTargetDesc/ARMMCAsmInfo.h
+++ b/lib/Target/ARM/MCTargetDesc/ARMMCAsmInfo.h
@@ -19,18 +19,19 @@
 #include "llvm/MC/MCAsmInfoELF.h"
 
 namespace llvm {
+  class Triple;
 
   class ARMMCAsmInfoDarwin : public MCAsmInfoDarwin {
     virtual void anchor();
 
   public:
-    explicit ARMMCAsmInfoDarwin(StringRef TT);
+    explicit ARMMCAsmInfoDarwin(const Triple &TheTriple);
   };
 
   class ARMELFMCAsmInfo : public MCAsmInfoELF {
     void anchor() override;
   public:
-    explicit ARMELFMCAsmInfo(StringRef TT);
+    explicit ARMELFMCAsmInfo(const Triple &TT);
 
     void setUseIntegratedAssembler(bool Value) override;
   };
diff --git a/lib/Target/ARM/MCTargetDesc/ARMMCCodeEmitter.cpp b/lib/Target/ARM/MCTargetDesc/ARMMCCodeEmitter.cpp
index 6b650f09f7e..b88578309f0 100644
--- a/lib/Target/ARM/MCTargetDesc/ARMMCCodeEmitter.cpp
+++ b/lib/Target/ARM/MCTargetDesc/ARMMCCodeEmitter.cpp
@@ -51,13 +51,13 @@ class ARMMCCodeEmitter : public MCCodeEmitter {
   ~ARMMCCodeEmitter() override {}
 
   bool isThumb(const MCSubtargetInfo &STI) const {
-    return (STI.getFeatureBits() & ARM::ModeThumb) != 0;
+    return STI.getFeatureBits()[ARM::ModeThumb];
   }
   bool isThumb2(const MCSubtargetInfo &STI) const {
-    return isThumb(STI) && (STI.getFeatureBits() & ARM::FeatureThumb2) != 0;
+    return isThumb(STI) && STI.getFeatureBits()[ARM::FeatureThumb2];
   }
   bool isTargetMachO(const MCSubtargetInfo &STI) const {
-    Triple TT(STI.getTargetTriple());
+    const Triple &TT = STI.getTargetTriple();
     return TT.isOSBinFormatMachO();
   }
 
@@ -287,7 +287,7 @@ class ARMMCCodeEmitter : public MCCodeEmitter {
       // See ARMELFObjectWriter::ExplicitRelSym and
       //     ARMELFObjectWriter::GetRelocTypeInner for more details.
       MCFixupKind Kind = MCFixupKind(FK_Data_4);
-      Fixups.push_back(MCFixup::Create(0, Expr, Kind, MI.getLoc()));
+      Fixups.push_back(MCFixup::create(0, Expr, Kind, MI.getLoc()));
       return 0;
     }
 
@@ -318,7 +318,7 @@ class ARMMCCodeEmitter : public MCCodeEmitter {
       // See ARMELFObjectWriter::ExplicitRelSym and
       //     ARMELFObjectWriter::GetRelocTypeInner for more details.
       MCFixupKind Kind = MCFixupKind(FK_Data_4);
-      Fixups.push_back(MCFixup::Create(0, Expr, Kind, MI.getLoc()));
+      Fixups.push_back(MCFixup::create(0, Expr, Kind, MI.getLoc()));
       return 0;
     }
 
@@ -432,7 +432,7 @@ class ARMMCCodeEmitter : public MCCodeEmitter {
     }
   }
 
-  void EncodeInstruction(const MCInst &MI, raw_ostream &OS,
+  void encodeInstruction(const MCInst &MI, raw_ostream &OS,
                          SmallVectorImpl<MCFixup> &Fixups,
                          const MCSubtargetInfo &STI) const override;
 };
@@ -595,7 +595,7 @@ static uint32_t getBranchTargetOpValue(const MCInst &MI, unsigned OpIdx,
   assert(MO.isExpr() && "Unexpected branch target type!");
   const MCExpr *Expr = MO.getExpr();
   MCFixupKind Kind = MCFixupKind(FixupKind);
-  Fixups.push_back(MCFixup::Create(0, Expr, Kind, MI.getLoc()));
+  Fixups.push_back(MCFixup::create(0, Expr, Kind, MI.getLoc()));
 
   // All of the information is in the fixup.
   return 0;
@@ -900,7 +900,7 @@ getAddrModeImm12OpValue(const MCInst &MI, unsigned OpIdx,
         Kind = MCFixupKind(ARM::fixup_t2_ldst_pcrel_12);
       else
         Kind = MCFixupKind(ARM::fixup_arm_ldst_pcrel_12);
-      Fixups.push_back(MCFixup::Create(0, Expr, Kind, MI.getLoc()));
+      Fixups.push_back(MCFixup::create(0, Expr, Kind, MI.getLoc()));
 
       ++MCNumCPRelocations;
     } else {
@@ -979,7 +979,7 @@ getT2AddrModeImm8s4OpValue(const MCInst &MI, unsigned OpIdx,
     assert(MO.isExpr() && "Unexpected machine operand type!");
     const MCExpr *Expr = MO.getExpr();
     MCFixupKind Kind = MCFixupKind(ARM::fixup_t2_pcrel_10);
-    Fixups.push_back(MCFixup::Create(0, Expr, Kind, MI.getLoc()));
+    Fixups.push_back(MCFixup::create(0, Expr, Kind, MI.getLoc()));
 
     ++MCNumCPRelocations;
   } else
@@ -1058,14 +1058,14 @@ ARMMCCodeEmitter::getHiLo16ImmOpValue(const MCInst &MI, unsigned OpIdx,
       break;
     }
 
-    Fixups.push_back(MCFixup::Create(0, E, Kind, MI.getLoc()));
+    Fixups.push_back(MCFixup::create(0, E, Kind, MI.getLoc()));
     return 0;
   }
   // If the expression doesn't have :upper16: or :lower16: on it,
   // it's just a plain immediate expression, previously those evaluated to
   // the lower 16 bits of the expression regardless of whether
   // we have a movt or a movw, but that led to misleadingly results.
-  // This is now disallowed in the the AsmParser in validateInstruction()
+  // This is disallowed in the AsmParser in validateInstruction()
   // so this should never happen.
   llvm_unreachable("expression without :upper16: or :lower16:");
 }
@@ -1194,7 +1194,7 @@ getAddrMode3OpValue(const MCInst &MI, unsigned OpIdx,
     assert(MO.isExpr() && "Unexpected machine operand type!");
     const MCExpr *Expr = MO.getExpr();
     MCFixupKind Kind = MCFixupKind(ARM::fixup_arm_pcrel_10_unscaled);
-    Fixups.push_back(MCFixup::Create(0, Expr, Kind, MI.getLoc()));
+    Fixups.push_back(MCFixup::create(0, Expr, Kind, MI.getLoc()));
 
     ++MCNumCPRelocations;
     return (Rn << 9) | (1 << 13);
@@ -1276,7 +1276,7 @@ getAddrMode5OpValue(const MCInst &MI, unsigned OpIdx,
       Kind = MCFixupKind(ARM::fixup_t2_pcrel_10);
     else
       Kind = MCFixupKind(ARM::fixup_arm_pcrel_10);
-    Fixups.push_back(MCFixup::Create(0, Expr, Kind, MI.getLoc()));
+    Fixups.push_back(MCFixup::create(0, Expr, Kind, MI.getLoc()));
 
     ++MCNumCPRelocations;
   } else {
@@ -1666,7 +1666,7 @@ getShiftRight64Imm(const MCInst &MI, unsigned Op,
 }
 
 void ARMMCCodeEmitter::
-EncodeInstruction(const MCInst &MI, raw_ostream &OS,
+encodeInstruction(const MCInst &MI, raw_ostream &OS,
                   SmallVectorImpl<MCFixup> &Fixups,
                   const MCSubtargetInfo &STI) const {
   // Pseudo instructions don't get encoded.
diff --git a/lib/Target/ARM/MCTargetDesc/ARMMCExpr.cpp b/lib/Target/ARM/MCTargetDesc/ARMMCExpr.cpp
index 5b90de32741..2063ca6bdf3 100644
--- a/lib/Target/ARM/MCTargetDesc/ARMMCExpr.cpp
+++ b/lib/Target/ARM/MCTargetDesc/ARMMCExpr.cpp
@@ -16,12 +16,12 @@ using namespace llvm;
 #define DEBUG_TYPE "armmcexpr"
 
 const ARMMCExpr*
-ARMMCExpr::Create(VariantKind Kind, const MCExpr *Expr,
+ARMMCExpr::create(VariantKind Kind, const MCExpr *Expr,
                        MCContext &Ctx) {
   return new (Ctx) ARMMCExpr(Kind, Expr);
 }
 
-void ARMMCExpr::PrintImpl(raw_ostream &OS) const {
+void ARMMCExpr::printImpl(raw_ostream &OS, const MCAsmInfo *MAI) const {
   switch (Kind) {
   default: llvm_unreachable("Invalid kind!");
   case VK_ARM_HI16: OS << ":upper16:"; break;
@@ -31,7 +31,7 @@ void ARMMCExpr::PrintImpl(raw_ostream &OS) const {
   const MCExpr *Expr = getSubExpr();
   if (Expr->getKind() != MCExpr::SymbolRef)
     OS << '(';
-  Expr->print(OS);
+  Expr->print(OS, MAI);
   if (Expr->getKind() != MCExpr::SymbolRef)
     OS << ')';
 }
diff --git a/lib/Target/ARM/MCTargetDesc/ARMMCExpr.h b/lib/Target/ARM/MCTargetDesc/ARMMCExpr.h
index 2be98d2e9e7..9146d4def75 100644
--- a/lib/Target/ARM/MCTargetDesc/ARMMCExpr.h
+++ b/lib/Target/ARM/MCTargetDesc/ARMMCExpr.h
@@ -33,15 +33,15 @@ class ARMMCExpr : public MCTargetExpr {
   /// @name Construction
   /// @{
 
-  static const ARMMCExpr *Create(VariantKind Kind, const MCExpr *Expr,
+  static const ARMMCExpr *create(VariantKind Kind, const MCExpr *Expr,
                                       MCContext &Ctx);
 
-  static const ARMMCExpr *CreateUpper16(const MCExpr *Expr, MCContext &Ctx) {
-    return Create(VK_ARM_HI16, Expr, Ctx);
+  static const ARMMCExpr *createUpper16(const MCExpr *Expr, MCContext &Ctx) {
+    return create(VK_ARM_HI16, Expr, Ctx);
   }
 
-  static const ARMMCExpr *CreateLower16(const MCExpr *Expr, MCContext &Ctx) {
-    return Create(VK_ARM_LO16, Expr, Ctx);
+  static const ARMMCExpr *createLower16(const MCExpr *Expr, MCContext &Ctx) {
+    return create(VK_ARM_LO16, Expr, Ctx);
   }
 
   /// @}
@@ -56,15 +56,15 @@ class ARMMCExpr : public MCTargetExpr {
 
   /// @}
 
-  void PrintImpl(raw_ostream &OS) const override;
-  bool EvaluateAsRelocatableImpl(MCValue &Res,
+  void printImpl(raw_ostream &OS, const MCAsmInfo *MAI) const override;
+  bool evaluateAsRelocatableImpl(MCValue &Res,
                                  const MCAsmLayout *Layout,
                                  const MCFixup *Fixup) const override {
     return false;
   }
-  void visitUsedExpr(MCStreamer &Streamer) const override; 
-  const MCSection *FindAssociatedSection() const override {
-    return getSubExpr()->FindAssociatedSection();
+  void visitUsedExpr(MCStreamer &Streamer) const override;
+  MCSection *findAssociatedSection() const override {
+    return getSubExpr()->findAssociatedSection();
   }
 
   // There are no TLS ARMMCExprs at the moment.
diff --git a/lib/Target/ARM/MCTargetDesc/ARMMCNaCl.cpp b/lib/Target/ARM/MCTargetDesc/ARMMCNaCl.cpp
deleted file mode 100644
index 003efd39218..00000000000
--- a/lib/Target/ARM/MCTargetDesc/ARMMCNaCl.cpp
+++ /dev/null
@@ -1,323 +0,0 @@
-//=== ARMMCNaCl.cpp -  Expansion of NaCl pseudo-instructions     --*- C++ -*-=//
-//
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
-//
-//===----------------------------------------------------------------------===//
-//
-//===----------------------------------------------------------------------===//
-#define DEBUG_TYPE "arm-mc-nacl"
-
-#include "ARMAddressingModes.h"
-#include "MCTargetDesc/ARMBaseInfo.h"
-#include "MCTargetDesc/ARMMCExpr.h"
-#include "MCTargetDesc/ARMMCNaCl.h"
-#include "MCTargetDesc/ARMMCTargetDesc.h"
-#include "llvm/MC/MCInst.h"
-#include "llvm/MC/MCStreamer.h"
-#include "llvm/Support/CommandLine.h"
-#include "llvm/Support/Debug.h"
-
-using namespace llvm;
-
-/// Two helper functions for emitting the actual guard instructions
-
-static void EmitBICMask(const MCSubtargetInfo &STI, MCStreamer &Out,
-                        unsigned Addr, int64_t Pred, unsigned Mask) {
-  // bic\Pred \Addr, \Addr, #Mask
-  MCInst BICInst;
-  const int32_t EncodedMask = ARM_AM::getSOImmVal(Mask);
-  assert(EncodedMask != -1);
-  BICInst.setOpcode(ARM::BICri);
-  BICInst.addOperand(MCOperand::CreateReg(Addr)); // rD
-  BICInst.addOperand(MCOperand::CreateReg(Addr)); // rS
-  BICInst.addOperand(MCOperand::CreateImm(EncodedMask)); // imm
-  BICInst.addOperand(MCOperand::CreateImm(Pred));  // predicate
-  BICInst.addOperand(MCOperand::CreateReg(ARM::CPSR)); // CPSR
-  BICInst.addOperand(MCOperand::CreateReg(0)); // flag out
-  Out.EmitInstruction(BICInst, STI);
-}
-
-static void EmitTST(const MCSubtargetInfo &STI, MCStreamer &Out, unsigned Reg) {
-  // tst \reg, #\MASK typically 0xc0000000
-  const int32_t Mask = ARM_AM::getSOImmVal(0xC0000000U);
-  assert(Mask != -1);
-  MCInst TSTInst;
-  TSTInst.setOpcode(ARM::TSTri);
-  TSTInst.addOperand(MCOperand::CreateReg(Reg));  // rS
-  TSTInst.addOperand(MCOperand::CreateImm(Mask)); // imm
-  TSTInst.addOperand(MCOperand::CreateImm((int64_t)ARMCC::AL)); // Always
-  TSTInst.addOperand(MCOperand::CreateImm(0)); // flag out
-  Out.EmitInstruction(TSTInst, STI);
-}
-
-
-// This is ONLY used for sandboxing stack changes.
-// The reason why SFI_NOP_IF_AT_BUNDLE_END gets handled here is that
-// it must ensure that the two instructions are in the same bundle.
-// It just so happens that the SFI_NOP_IF_AT_BUNDLE_END is always
-// emitted in conjunction with a SFI_DATA_MASK
-//
-static void EmitDataMask(const MCSubtargetInfo &STI, int I, MCInst Saved[],
-                         MCStreamer &Out) {
-  assert(I == 3 && (ARM::SFI_NOP_IF_AT_BUNDLE_END == Saved[0].getOpcode()) &&
-         (ARM::SFI_DATA_MASK == Saved[2].getOpcode()) &&
-         "Unexpected SFI Pseudo while lowering");
-
-  unsigned Addr = Saved[2].getOperand(0).getReg();
-  int64_t  Pred = Saved[2].getOperand(2).getImm();
-  assert((ARM::SP == Addr) && "Unexpected register at stack guard");
-
-  Out.EmitBundleLock(false);
-  Out.EmitInstruction(Saved[1], STI);
-  EmitBICMask(STI, Out, Addr, Pred, 0xC0000000);
-  Out.EmitBundleUnlock();
-}
-
-static void EmitDirectGuardCall(const MCSubtargetInfo &STI, int I,
-                                MCInst Saved[], MCStreamer &Out) {
-  // sfi_call_preamble cond=
-  //   sfi_nops_to_force_slot3
-  assert(I == 2 && (ARM::SFI_GUARD_CALL == Saved[0].getOpcode()) &&
-         "Unexpected SFI Pseudo while lowering SFI_GUARD_CALL");
-  Out.EmitBundleLock(true);
-  Out.EmitInstruction(Saved[1], STI);
-  Out.EmitBundleUnlock();
-}
-
-static void EmitIndirectGuardCall(const MCSubtargetInfo &STI, int I,
-                                  MCInst Saved[], MCStreamer &Out) {
-  // sfi_indirect_call_preamble link cond=
-  //   sfi_nops_to_force_slot2
-  //   sfi_code_mask \link \cond
-  assert(I == 2 && (ARM::SFI_GUARD_INDIRECT_CALL == Saved[0].getOpcode()) &&
-         "Unexpected SFI Pseudo while lowering SFI_GUARD_CALL");
-  unsigned Reg = Saved[0].getOperand(0).getReg();
-  int64_t Pred = Saved[0].getOperand(2).getImm();
-  Out.EmitBundleLock(true);
-  EmitBICMask(STI, Out, Reg, Pred, 0xC000000F);
-  Out.EmitInstruction(Saved[1], STI);
-  Out.EmitBundleUnlock();
-}
-
-static void EmitIndirectGuardJmp(const MCSubtargetInfo &STI, int I,
-                                 MCInst Saved[], MCStreamer &Out) {
-  //  sfi_indirect_jump_preamble link cond=
-  //   sfi_nop_if_at_bundle_end
-  //   sfi_code_mask \link \cond
-  assert(I == 2 && (ARM::SFI_GUARD_INDIRECT_JMP == Saved[0].getOpcode()) &&
-         "Unexpected SFI Pseudo while lowering SFI_GUARD_CALL");
-  unsigned Reg = Saved[0].getOperand(0).getReg();
-  int64_t Pred = Saved[0].getOperand(2).getImm();
-
-  Out.EmitBundleLock(false);
-  EmitBICMask(STI, Out, Reg, Pred, 0xC000000F);
-  Out.EmitInstruction(Saved[1], STI);
-  Out.EmitBundleUnlock();
-}
-
-static void EmitGuardReturn(const MCSubtargetInfo &STI, int I, MCInst Saved[],
-                            MCStreamer &Out) {
-  // sfi_return_preamble reg cond=
-  //    sfi_nop_if_at_bundle_end
-  //    sfi_code_mask \reg \cond
-  assert(I == 2 && (ARM::SFI_GUARD_RETURN == Saved[0].getOpcode()) &&
-         "Unexpected SFI Pseudo while lowering SFI_GUARD_RETURN");
-  int64_t Pred = Saved[0].getOperand(0).getImm();
-
-  Out.EmitBundleLock(false);
-  EmitBICMask(STI, Out, ARM::LR, Pred, 0xC000000F);
-  Out.EmitInstruction(Saved[1], STI);
-  Out.EmitBundleUnlock();
-}
-
-static void EmitGuardLoadOrStore(const MCSubtargetInfo &STI, int I,
-                                 MCInst Saved[], MCStreamer &Out) {
-  // sfi_store_preamble reg cond ---->
-  //    sfi_nop_if_at_bundle_end
-  //    sfi_data_mask \reg, \cond
-  assert(I == 2 && (ARM::SFI_GUARD_LOADSTORE == Saved[0].getOpcode()) &&
-         "Unexpected SFI Pseudo while lowering SFI_GUARD_RETURN");
-  unsigned Reg = Saved[0].getOperand(0).getReg();
-  int64_t Pred = Saved[0].getOperand(2).getImm();
-
-  Out.EmitBundleLock(false);
-  EmitBICMask(STI, Out, Reg, Pred, 0xC0000000);
-  Out.EmitInstruction(Saved[1], STI);
-  Out.EmitBundleUnlock();
-}
-
-static void EmitGuardLoadOrStoreTst(const MCSubtargetInfo &STI, int I,
-                                    MCInst Saved[], MCStreamer &Out) {
-  // sfi_cstore_preamble reg -->
-  //   sfi_nop_if_at_bundle_end
-  //   sfi_data_tst \reg
-  assert(I == 2 && (ARM::SFI_GUARD_LOADSTORE_TST == Saved[0].getOpcode()) &&
-         "Unexpected SFI Pseudo while lowering");
-  unsigned Reg = Saved[0].getOperand(0).getReg();
-
-  Out.EmitBundleLock(false);
-  EmitTST(STI, Out, Reg);
-  Out.EmitInstruction(Saved[1], STI);
-  Out.EmitBundleUnlock();
-}
-
-// This is ONLY used for loads into the stack pointer.
-static void EmitGuardSpLoad(const MCSubtargetInfo &STI, int I, MCInst Saved[],
-                            MCStreamer &Out) {
-  assert(I == 4 &&
-         (ARM::SFI_GUARD_SP_LOAD == Saved[0].getOpcode()) &&
-         (ARM::SFI_NOP_IF_AT_BUNDLE_END == Saved[1].getOpcode()) &&
-         (ARM::SFI_DATA_MASK == Saved[3].getOpcode()) &&
-         "Unexpected SFI Pseudo while lowering");
-
-  unsigned AddrReg = Saved[0].getOperand(0).getReg();
-  unsigned SpReg = Saved[3].getOperand(0).getReg();
-  int64_t  Pred = Saved[3].getOperand(2).getImm();
-  assert((ARM::SP == SpReg) && "Unexpected register at stack guard");
-
-  Out.EmitBundleLock(false);
-  EmitBICMask(STI, Out, AddrReg, Pred, 0xC0000000);
-  Out.EmitInstruction(Saved[2], STI);
-  EmitBICMask(STI, Out, SpReg, Pred, 0xC0000000);
-  Out.EmitBundleUnlock();
-}
-
-namespace llvm {
-
-const int ARMMCNaClSFIState::MaxSaved;
-
-// CustomExpandInstNaClARM -
-//   If Inst is a NaCl pseudo instruction, emits the substitute
-//   expansion to the MCStreamer and returns true.
-//   Otherwise, returns false.
-//
-//   NOTE: Each time this function calls Out.EmitInstruction(), it will be
-//   called again recursively to rewrite the new instruction being emitted.
-//   Care must be taken to ensure that this does not result in an infinite
-//   loop. Also, global state must be managed carefully so that it is
-//   consistent during recursive calls.
-bool CustomExpandInstNaClARM(const MCSubtargetInfo &STI, const MCInst &Inst,
-                             MCStreamer &Out, ARMMCNaClSFIState &State) {
-  // Logic:
-  // This is somewhat convoluted, but in the current model, the SFI
-  // guard pseudo instructions occur PRIOR to the actual instruction.
-  // So, the bundling/alignment operation has to refer to the FOLLOWING
-  // instructions.
-  //
-  // When a SFI pseudo is detected, it is saved. Then, the saved SFI
-  // pseudo and the very next instructions (their amount depending on the kind
-  // of the SFI pseudo) are used as arguments to the Emit*() functions in
-  // this file.
-  //
-  // Some state data is used to preserve state accross calls:
-  //
-  // Saved:      the saved instructions (starting with the SFI_ pseudo).
-  // SavedCount: the amount of saved instructions required for the SFI pseudo
-  //             that's being expanded.
-  // I:          the index of the currently saved instruction - used to track
-  //             where in Saved to insert the instruction and how many more
-  //             remain.
-  //
-
-  // If we are emitting to .s, just emit all pseudo-instructions directly.
-  if (Out.hasRawTextSupport()) {
-    return false;
-  }
-
-  // Protect against recursive execution. If State.RecurseiveCall == true, it
-  // means we're already in the process of expanding a custom instruction, and
-  // we don't need to run recursively on anything generated by such an
-  // expansion.
-  if (State.RecursiveCall)
-    return false;
-
-  DEBUG(dbgs() << "CustomExpandInstNaClARM("; Inst.dump(); dbgs() << ")\n");
-
-  if ((State.I == 0) && (State.SaveCount == 0)) {
-    // Base state: no SFI guard identified yet and no saving started.
-    switch (Inst.getOpcode()) {
-      default:
-        // We don't handle non-SFI guards here
-        return false;
-      case ARM::SFI_NOP_IF_AT_BUNDLE_END:
-        // Note: SFI_NOP_IF_AT_BUNDLE_END is only emitted directly as part of
-        // a stack guard in conjunction with a SFI_DATA_MASK.
-        State.SaveCount = 3;
-        break;
-      case ARM::SFI_DATA_MASK:
-        llvm_unreachable(
-            "SFI_DATA_MASK found without preceding SFI_NOP_IF_AT_BUNDLE_END");
-        return false;
-      case ARM::SFI_GUARD_CALL:
-      case ARM::SFI_GUARD_INDIRECT_CALL:
-      case ARM::SFI_GUARD_INDIRECT_JMP:
-      case ARM::SFI_GUARD_RETURN:
-      case ARM::SFI_GUARD_LOADSTORE:
-      case ARM::SFI_GUARD_LOADSTORE_TST:
-        State.SaveCount = 2;
-        break;
-      case ARM::SFI_GUARD_SP_LOAD:
-        State.SaveCount = 4;
-        break;
-    }
-  }
-
-  // We're in "saving instructions" state
-  if (State.I < State.SaveCount) {
-    // This instruction has to be saved
-    assert(State.I < State.MaxSaved && "Trying to save too many instructions");
-    State.Saved[State.I++] = Inst;
-    if (State.I < State.SaveCount)
-      return true;
-  }
-
-  // We're in "saved enough instructions, time to emit" state
-  assert(State.I == State.SaveCount && State.SaveCount > 0 && "Bookeeping Error");
-
-  // When calling Emit* functions, do that with RecurseGuard set (the comment
-  // at the beginning of this function explains why)
-  State.RecursiveCall = true;
-  switch (State.Saved[0].getOpcode()) {
-    default:
-      break;
-    case ARM::SFI_NOP_IF_AT_BUNDLE_END:
-      EmitDataMask(STI, State.I, State.Saved, Out);
-      break;
-    case ARM::SFI_DATA_MASK:
-      llvm_unreachable("SFI_DATA_MASK can't start a SFI sequence");
-      break;
-    case ARM::SFI_GUARD_CALL:
-      EmitDirectGuardCall(STI, State.I, State.Saved, Out);
-      break;
-    case ARM::SFI_GUARD_INDIRECT_CALL:
-      EmitIndirectGuardCall(STI, State.I, State.Saved, Out);
-      break;
-    case ARM::SFI_GUARD_INDIRECT_JMP:
-      EmitIndirectGuardJmp(STI, State.I, State.Saved, Out);
-      break;
-    case ARM::SFI_GUARD_RETURN:
-      EmitGuardReturn(STI, State.I, State.Saved, Out);
-      break;
-    case ARM::SFI_GUARD_LOADSTORE:
-      EmitGuardLoadOrStore(STI, State.I, State.Saved, Out);
-      break;
-    case ARM::SFI_GUARD_LOADSTORE_TST:
-      EmitGuardLoadOrStoreTst(STI, State.I, State.Saved, Out);
-      break;
-    case ARM::SFI_GUARD_SP_LOAD:
-      EmitGuardSpLoad(STI, State.I, State.Saved, Out);
-      break;
-  }
-  assert(State.RecursiveCall && "Illegal Depth");
-  State.RecursiveCall = false;
-
-  // We're done expanding a SFI guard. Reset state vars.
-  State.SaveCount = 0;
-  State.I = 0;
-  return true;
-}
-
-} // namespace llvm
diff --git a/lib/Target/ARM/MCTargetDesc/ARMMCNaCl.h b/lib/Target/ARM/MCTargetDesc/ARMMCNaCl.h
deleted file mode 100644
index 2fdcd561301..00000000000
--- a/lib/Target/ARM/MCTargetDesc/ARMMCNaCl.h
+++ /dev/null
@@ -1,32 +0,0 @@
-//===-- ARMMCNaCl.h - Prototype for CustomExpandInstNaClARM   ---*- C++ -*-===//
-//
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
-//
-//===----------------------------------------------------------------------===//
-
-#ifndef ARMMCNACL_H
-#define ARMMCNACL_H
-#include "llvm/MC/MCInst.h"
-
-namespace llvm {
-
-class MCStreamer;
-class MCSubtargetInfo;
-
-class ARMMCNaClSFIState {
-public:
-  static const int MaxSaved = 4;
-  MCInst Saved[MaxSaved];
-  int SaveCount;
-  int I;
-  bool RecursiveCall;
-};
-
-bool CustomExpandInstNaClARM(const MCSubtargetInfo &STI, const MCInst &Inst,
-                             MCStreamer &Out, ARMMCNaClSFIState &State);
-}
-
-#endif
diff --git a/lib/Target/ARM/MCTargetDesc/ARMMCTargetDesc.cpp b/lib/Target/ARM/MCTargetDesc/ARMMCTargetDesc.cpp
index 069c1163c1f..21c9fc1e58b 100644
--- a/lib/Target/ARM/MCTargetDesc/ARMMCTargetDesc.cpp
+++ b/lib/Target/ARM/MCTargetDesc/ARMMCTargetDesc.cpp
@@ -31,9 +31,9 @@ using namespace llvm;
 #define GET_REGINFO_MC_DESC
 #include "ARMGenRegisterInfo.inc"
 
-static bool getMCRDeprecationInfo(MCInst &MI, MCSubtargetInfo &STI,
+static bool getMCRDeprecationInfo(MCInst &MI, const MCSubtargetInfo &STI,
                                   std::string &Info) {
-  if (STI.getFeatureBits() & llvm::ARM::HasV7Ops &&
+  if (STI.getFeatureBits()[llvm::ARM::HasV7Ops] &&
       (MI.getOperand(0).isImm() && MI.getOperand(0).getImm() == 15) &&
       (MI.getOperand(1).isImm() && MI.getOperand(1).getImm() == 0) &&
       // Checks for the deprecated CP15ISB encoding:
@@ -63,9 +63,9 @@ static bool getMCRDeprecationInfo(MCInst &MI, MCSubtargetInfo &STI,
   return false;
 }
 
-static bool getITDeprecationInfo(MCInst &MI, MCSubtargetInfo &STI,
+static bool getITDeprecationInfo(MCInst &MI, const MCSubtargetInfo &STI,
                                  std::string &Info) {
-  if (STI.getFeatureBits() & llvm::ARM::HasV8Ops && MI.getOperand(1).isImm() &&
+  if (STI.getFeatureBits()[llvm::ARM::HasV8Ops] && MI.getOperand(1).isImm() &&
       MI.getOperand(1).getImm() != 8) {
     Info = "applying IT instruction to more than one subsequent instruction is "
            "deprecated";
@@ -75,9 +75,9 @@ static bool getITDeprecationInfo(MCInst &MI, MCSubtargetInfo &STI,
   return false;
 }
 
-static bool getARMStoreDeprecationInfo(MCInst &MI, MCSubtargetInfo &STI,
+static bool getARMStoreDeprecationInfo(MCInst &MI, const MCSubtargetInfo &STI,
                                        std::string &Info) {
-  assert((~STI.getFeatureBits() & llvm::ARM::ModeThumb) &&
+  assert(!STI.getFeatureBits()[llvm::ARM::ModeThumb] &&
          "cannot predicate thumb instructions");
 
   assert(MI.getNumOperands() >= 4 && "expected >= 4 arguments");
@@ -92,9 +92,9 @@ static bool getARMStoreDeprecationInfo(MCInst &MI, MCSubtargetInfo &STI,
   return false;
 }
 
-static bool getARMLoadDeprecationInfo(MCInst &MI, MCSubtargetInfo &STI,
+static bool getARMLoadDeprecationInfo(MCInst &MI, const MCSubtargetInfo &STI,
                                       std::string &Info) {
-  assert((~STI.getFeatureBits() & llvm::ARM::ModeThumb) &&
+  assert(!STI.getFeatureBits()[llvm::ARM::ModeThumb] &&
          "cannot predicate thumb instructions");
 
   assert(MI.getNumOperands() >= 4 && "expected >= 4 arguments");
@@ -130,16 +130,13 @@ static bool getARMLoadDeprecationInfo(MCInst &MI, MCSubtargetInfo &STI,
 #define GET_SUBTARGETINFO_MC_DESC
 #include "ARMGenSubtargetInfo.inc"
 
-
-std::string ARM_MC::ParseARMTriple(StringRef TT, StringRef CPU) {
-  Triple triple(TT);
-
-  bool isThumb = triple.getArch() == Triple::thumb ||
-                 triple.getArch() == Triple::thumbeb;
+std::string ARM_MC::ParseARMTriple(const Triple &TT, StringRef CPU) {
+  bool isThumb =
+      TT.getArch() == Triple::thumb || TT.getArch() == Triple::thumbeb;
 
   bool NoCPU = CPU == "generic" || CPU.empty();
   std::string ARMArchFeature;
-  switch (triple.getSubArch()) {
+  switch (TT.getSubArch()) {
   default:
     llvm_unreachable("invalid sub-architecture for ARM");
   case Triple::ARMSubArch_v8:
@@ -177,7 +174,7 @@ std::string ARM_MC::ParseARMTriple(StringRef TT, StringRef CPU) {
     if (NoCPU)
       // v7em: FeatureNoARM, FeatureDB, FeatureHWDiv, FeatureDSPThumb2,
       //       FeatureT2XtPk, FeatureMClass
-      ARMArchFeature = "+v7,+noarm,+db,+hwdiv,+t2dsp,t2xtpk,+mclass";
+      ARMArchFeature = "+v7,+noarm,+db,+hwdiv,+t2dsp,+t2xtpk,+mclass";
     else
       // Use CPU to figure out the exact features.
       ARMArchFeature = "+v7";
@@ -240,7 +237,7 @@ std::string ARM_MC::ParseARMTriple(StringRef TT, StringRef CPU) {
       ARMArchFeature += ",+thumb-mode";
   }
 
-  if (triple.isOSNaCl()) {
+  if (TT.isOSNaCl()) {
     if (ARMArchFeature.empty())
       ARMArchFeature = "+nacl-trap";
     else
@@ -250,8 +247,8 @@ std::string ARM_MC::ParseARMTriple(StringRef TT, StringRef CPU) {
   return ARMArchFeature;
 }
 
-MCSubtargetInfo *ARM_MC::createARMMCSubtargetInfo(StringRef TT, StringRef CPU,
-                                                  StringRef FS) {
+MCSubtargetInfo *ARM_MC::createARMMCSubtargetInfo(const Triple &TT,
+                                                  StringRef CPU, StringRef FS) {
   std::string ArchFS = ARM_MC::ParseARMTriple(TT, CPU);
   if (!FS.empty()) {
     if (!ArchFS.empty())
@@ -260,9 +257,7 @@ MCSubtargetInfo *ARM_MC::createARMMCSubtargetInfo(StringRef TT, StringRef CPU,
       ArchFS = FS;
   }
 
-  MCSubtargetInfo *X = new MCSubtargetInfo();
-  InitARMMCSubtargetInfo(X, TT, CPU, ArchFS);
-  return X;
+  return createARMMCSubtargetInfoImpl(TT, CPU, ArchFS);
 }
 
 static MCInstrInfo *createARMMCInstrInfo() {
@@ -271,24 +266,23 @@ static MCInstrInfo *createARMMCInstrInfo() {
   return X;
 }
 
-static MCRegisterInfo *createARMMCRegisterInfo(StringRef Triple) {
+static MCRegisterInfo *createARMMCRegisterInfo(const Triple &Triple) {
   MCRegisterInfo *X = new MCRegisterInfo();
   InitARMMCRegisterInfo(X, ARM::LR, 0, 0, ARM::PC);
   return X;
 }
 
-static MCAsmInfo *createARMMCAsmInfo(const MCRegisterInfo &MRI, StringRef TT) {
-  Triple TheTriple(TT);
-
+static MCAsmInfo *createARMMCAsmInfo(const MCRegisterInfo &MRI,
+                                     const Triple &TheTriple) {
   MCAsmInfo *MAI;
   if (TheTriple.isOSDarwin() || TheTriple.isOSBinFormatMachO())
-    MAI = new ARMMCAsmInfoDarwin(TT);
-  else if (TheTriple.isWindowsItaniumEnvironment())
-    MAI = new ARMCOFFMCAsmInfoGNU();
+    MAI = new ARMMCAsmInfoDarwin(TheTriple);
   else if (TheTriple.isWindowsMSVCEnvironment())
     MAI = new ARMCOFFMCAsmInfoMicrosoft();
+  else if (TheTriple.isOSWindows())
+    MAI = new ARMCOFFMCAsmInfoGNU();
   else
-    MAI = new ARMELFMCAsmInfo(TT);
+    MAI = new ARMELFMCAsmInfo(TheTriple);
 
   unsigned Reg = MRI.getDwarfRegNum(ARM::SP, true);
   MAI->addInitialFrameState(MCCFIInstruction::createDefCfa(nullptr, Reg, 0));
@@ -296,16 +290,15 @@ static MCAsmInfo *createARMMCAsmInfo(const MCRegisterInfo &MRI, StringRef TT) {
   return MAI;
 }
 
-static MCCodeGenInfo *createARMMCCodeGenInfo(StringRef TT, Reloc::Model RM,
+static MCCodeGenInfo *createARMMCCodeGenInfo(const Triple &TT, Reloc::Model RM,
                                              CodeModel::Model CM,
                                              CodeGenOpt::Level OL) {
   MCCodeGenInfo *X = new MCCodeGenInfo();
   if (RM == Reloc::Default) {
-    Triple TheTriple(TT);
     // Default relocation model on Darwin is PIC, not DynamicNoPIC.
-    RM = TheTriple.isOSDarwin() ? Reloc::PIC_ : Reloc::DynamicNoPIC;
+    RM = TT.isOSDarwin() ? Reloc::PIC_ : Reloc::DynamicNoPIC;
   }
-  X->InitMCCodeGenInfo(RM, CM, OL);
+  X->initMCCodeGenInfo(RM, CM, OL);
   return X;
 }
 
@@ -333,10 +326,9 @@ static MCInstPrinter *createARMMCInstPrinter(const Triple &T,
   return nullptr;
 }
 
-static MCRelocationInfo *createARMMCRelocationInfo(StringRef TT,
+static MCRelocationInfo *createARMMCRelocationInfo(const Triple &TT,
                                                    MCContext &Ctx) {
-  Triple TheTriple(TT);
-  if (TheTriple.isOSBinFormatMachO())
+  if (TT.isOSBinFormatMachO())
     return createARMMachORelocationInfo(Ctx);
   // Default to the stock relocation info.
   return llvm::createMCRelocationInfo(TT, Ctx);
@@ -405,10 +397,8 @@ extern "C" void LLVMInitializeARMTargetMC() {
     TargetRegistry::RegisterMCInstrAnalysis(*T, createARMMCInstrAnalysis);
 
     TargetRegistry::RegisterELFStreamer(*T, createELFStreamer);
-#ifndef __native_client__
     TargetRegistry::RegisterCOFFStreamer(*T, createARMWinCOFFStreamer);
     TargetRegistry::RegisterMachOStreamer(*T, createARMMachOStreamer);
-#endif
 
     // Register the obj target streamer.
     TargetRegistry::RegisterObjectTargetStreamer(*T,
diff --git a/lib/Target/ARM/MCTargetDesc/ARMMCTargetDesc.h b/lib/Target/ARM/MCTargetDesc/ARMMCTargetDesc.h
index 24ca567a812..fd30623d79a 100644
--- a/lib/Target/ARM/MCTargetDesc/ARMMCTargetDesc.h
+++ b/lib/Target/ARM/MCTargetDesc/ARMMCTargetDesc.h
@@ -40,12 +40,12 @@ extern Target TheARMLETarget, TheThumbLETarget;
 extern Target TheARMBETarget, TheThumbBETarget;
 
 namespace ARM_MC {
-  std::string ParseARMTriple(StringRef TT, StringRef CPU);
+std::string ParseARMTriple(const Triple &TT, StringRef CPU);
 
-  /// Create a ARM MCSubtargetInfo instance. This is exposed so Asm parser, etc.
-  /// do not need to go through TargetRegistry.
-  MCSubtargetInfo *createARMMCSubtargetInfo(StringRef TT, StringRef CPU,
-                                            StringRef FS);
+/// Create a ARM MCSubtargetInfo instance. This is exposed so Asm parser, etc.
+/// do not need to go through TargetRegistry.
+MCSubtargetInfo *createARMMCSubtargetInfo(const Triple &TT, StringRef CPU,
+                                          StringRef FS);
 }
 
 MCTargetStreamer *createARMNullTargetStreamer(MCStreamer &S);
@@ -65,20 +65,22 @@ MCCodeEmitter *createARMBEMCCodeEmitter(const MCInstrInfo &MCII,
                                         MCContext &Ctx);
 
 MCAsmBackend *createARMAsmBackend(const Target &T, const MCRegisterInfo &MRI,
-                                  StringRef TT, StringRef CPU,
+                                  const Triple &TT, StringRef CPU,
                                   bool IsLittleEndian);
 
 MCAsmBackend *createARMLEAsmBackend(const Target &T, const MCRegisterInfo &MRI,
-                                  StringRef TT, StringRef CPU);
+                                    const Triple &TT, StringRef CPU);
 
 MCAsmBackend *createARMBEAsmBackend(const Target &T, const MCRegisterInfo &MRI,
-                                  StringRef TT, StringRef CPU);
+                                    const Triple &TT, StringRef CPU);
 
-MCAsmBackend *createThumbLEAsmBackend(const Target &T, const MCRegisterInfo &MRI,
-                                      StringRef TT, StringRef CPU);
+MCAsmBackend *createThumbLEAsmBackend(const Target &T,
+                                      const MCRegisterInfo &MRI,
+                                      const Triple &TT, StringRef CPU);
 
-MCAsmBackend *createThumbBEAsmBackend(const Target &T, const MCRegisterInfo &MRI,
-                                      StringRef TT, StringRef CPU);
+MCAsmBackend *createThumbBEAsmBackend(const Target &T,
+                                      const MCRegisterInfo &MRI,
+                                      const Triple &TT, StringRef CPU);
 
 // Construct a PE/COFF machine code streamer which will generate a PE/COFF
 // object file.
diff --git a/lib/Target/ARM/MCTargetDesc/ARMMachORelocationInfo.cpp b/lib/Target/ARM/MCTargetDesc/ARMMachORelocationInfo.cpp
index d4b00e6e4fb..4468132588c 100644
--- a/lib/Target/ARM/MCTargetDesc/ARMMachORelocationInfo.cpp
+++ b/lib/Target/ARM/MCTargetDesc/ARMMachORelocationInfo.cpp
@@ -26,9 +26,9 @@ class ARMMachORelocationInfo : public MCRelocationInfo {
                                              unsigned VariantKind) override {
     switch(VariantKind) {
     case LLVMDisassembler_VariantKind_ARM_HI16:
-      return ARMMCExpr::CreateUpper16(SubExpr, Ctx);
+      return ARMMCExpr::createUpper16(SubExpr, Ctx);
     case LLVMDisassembler_VariantKind_ARM_LO16:
-      return ARMMCExpr::CreateLower16(SubExpr, Ctx);
+      return ARMMCExpr::createLower16(SubExpr, Ctx);
     default:
       return MCRelocationInfo::createExprForCAPIVariantKind(SubExpr,
                                                             VariantKind);
diff --git a/lib/Target/ARM/MCTargetDesc/ARMMachObjectWriter.cpp b/lib/Target/ARM/MCTargetDesc/ARMMachObjectWriter.cpp
index b1f9b587918..95d7ea7c04a 100644
--- a/lib/Target/ARM/MCTargetDesc/ARMMachObjectWriter.cpp
+++ b/lib/Target/ARM/MCTargetDesc/ARMMachObjectWriter.cpp
@@ -17,8 +17,8 @@
 #include "llvm/MC/MCExpr.h"
 #include "llvm/MC/MCFixup.h"
 #include "llvm/MC/MCFixupKindInfo.h"
-#include "llvm/MC/MCMachOSymbolFlags.h"
 #include "llvm/MC/MCMachObjectWriter.h"
+#include "llvm/MC/MCSection.h"
 #include "llvm/MC/MCValue.h"
 #include "llvm/Support/ErrorHandling.h"
 #include "llvm/Support/MachO.h"
@@ -44,17 +44,14 @@ class ARMMachObjectWriter : public MCMachObjectTargetWriter {
 
   bool requiresExternRelocation(MachObjectWriter *Writer,
                                 const MCAssembler &Asm,
-                                const MCFragment &Fragment,
-                                unsigned RelocType, const MCSymbolData *SD,
-                                uint64_t FixedValue);
+                                const MCFragment &Fragment, unsigned RelocType,
+                                const MCSymbol &S, uint64_t FixedValue);
 
 public:
-  ARMMachObjectWriter(bool Is64Bit, uint32_t CPUType,
-                      uint32_t CPUSubtype)
-    : MCMachObjectTargetWriter(Is64Bit, CPUType, CPUSubtype,
-                               /*UseAggressiveSymbolFolding=*/true) {}
+  ARMMachObjectWriter(bool Is64Bit, uint32_t CPUType, uint32_t CPUSubtype)
+      : MCMachObjectTargetWriter(Is64Bit, CPUType, CPUSubtype) {}
 
-  void RecordRelocation(MachObjectWriter *Writer, MCAssembler &Asm,
+  void recordRelocation(MachObjectWriter *Writer, MCAssembler &Asm,
                         const MCAsmLayout &Layout, const MCFragment *Fragment,
                         const MCFixup &Fixup, MCValue Target,
                         uint64_t &FixedValue) override;
@@ -152,31 +149,29 @@ RecordARMScatteredHalfRelocation(MachObjectWriter *Writer,
 
   // See <reloc.h>.
   const MCSymbol *A = &Target.getSymA()->getSymbol();
-  const MCSymbolData *A_SD = &Asm.getSymbolData(*A);
 
-  if (!A_SD->getFragment())
-    Asm.getContext().FatalError(Fixup.getLoc(),
+  if (!A->getFragment())
+    Asm.getContext().reportFatalError(Fixup.getLoc(),
                        "symbol '" + A->getName() +
                        "' can not be undefined in a subtraction expression");
 
-  uint32_t Value = Writer->getSymbolAddress(A_SD, Layout);
+  uint32_t Value = Writer->getSymbolAddress(*A, Layout);
   uint32_t Value2 = 0;
-  uint64_t SecAddr =
-    Writer->getSectionAddress(A_SD->getFragment()->getParent());
+  uint64_t SecAddr = Writer->getSectionAddress(A->getFragment()->getParent());
   FixedValue += SecAddr;
 
   if (const MCSymbolRefExpr *B = Target.getSymB()) {
-    const MCSymbolData *B_SD = &Asm.getSymbolData(B->getSymbol());
+    const MCSymbol *SB = &B->getSymbol();
 
-    if (!B_SD->getFragment())
-      Asm.getContext().FatalError(Fixup.getLoc(),
+    if (!SB->getFragment())
+      Asm.getContext().reportFatalError(Fixup.getLoc(),
                          "symbol '" + B->getSymbol().getName() +
                          "' can not be undefined in a subtraction expression");
 
     // Select the appropriate difference relocation type.
     Type = MachO::ARM_RELOC_HALF_SECTDIFF;
-    Value2 = Writer->getSymbolAddress(B_SD, Layout);
-    FixedValue -= Writer->getSectionAddress(B_SD->getFragment()->getParent());
+    Value2 = Writer->getSymbolAddress(B->getSymbol(), Layout);
+    FixedValue -= Writer->getSectionAddress(SB->getFragment()->getParent());
   }
 
   // Relocations are written out in reverse order, so the PAIR comes first.
@@ -255,31 +250,30 @@ void ARMMachObjectWriter::RecordARMScatteredRelocation(MachObjectWriter *Writer,
 
   // See <reloc.h>.
   const MCSymbol *A = &Target.getSymA()->getSymbol();
-  const MCSymbolData *A_SD = &Asm.getSymbolData(*A);
 
-  if (!A_SD->getFragment())
-    Asm.getContext().FatalError(Fixup.getLoc(),
+  if (!A->getFragment())
+    Asm.getContext().reportFatalError(Fixup.getLoc(),
                        "symbol '" + A->getName() +
                        "' can not be undefined in a subtraction expression");
 
-  uint32_t Value = Writer->getSymbolAddress(A_SD, Layout);
-  uint64_t SecAddr = Writer->getSectionAddress(A_SD->getFragment()->getParent());
+  uint32_t Value = Writer->getSymbolAddress(*A, Layout);
+  uint64_t SecAddr = Writer->getSectionAddress(A->getFragment()->getParent());
   FixedValue += SecAddr;
   uint32_t Value2 = 0;
 
   if (const MCSymbolRefExpr *B = Target.getSymB()) {
     assert(Type == MachO::ARM_RELOC_VANILLA && "invalid reloc for 2 symbols");
-    const MCSymbolData *B_SD = &Asm.getSymbolData(B->getSymbol());
+    const MCSymbol *SB = &B->getSymbol();
 
-    if (!B_SD->getFragment())
-      Asm.getContext().FatalError(Fixup.getLoc(),
+    if (!SB->getFragment())
+      Asm.getContext().reportFatalError(Fixup.getLoc(),
                          "symbol '" + B->getSymbol().getName() +
                          "' can not be undefined in a subtraction expression");
 
     // Select the appropriate difference relocation type.
     Type = MachO::ARM_RELOC_SECTDIFF;
-    Value2 = Writer->getSymbolAddress(B_SD, Layout);
-    FixedValue -= Writer->getSectionAddress(B_SD->getFragment()->getParent());
+    Value2 = Writer->getSymbolAddress(B->getSymbol(), Layout);
+    FixedValue -= Writer->getSectionAddress(SB->getFragment()->getParent());
   }
 
   // Relocations are written out in reverse order, so the PAIR comes first.
@@ -309,10 +303,10 @@ bool ARMMachObjectWriter::requiresExternRelocation(MachObjectWriter *Writer,
                                                    const MCAssembler &Asm,
                                                    const MCFragment &Fragment,
                                                    unsigned RelocType,
-                                                   const MCSymbolData *SD,
+                                                   const MCSymbol &S,
                                                    uint64_t FixedValue) {
   // Most cases can be identified purely from the symbol.
-  if (Writer->doesSymbolRequireExternRelocation(SD))
+  if (Writer->doesSymbolRequireExternRelocation(S))
     return true;
   int64_t Value = (int64_t)FixedValue;  // The displacement is signed.
   int64_t Range;
@@ -334,9 +328,7 @@ bool ARMMachObjectWriter::requiresExternRelocation(MachObjectWriter *Writer,
   // BL/BLX also use external relocations when an internal relocation
   // would result in the target being out of range. This gives the linker
   // enough information to generate a branch island.
-  const MCSectionData &SymSD = Asm.getSectionData(
-    SD->getSymbol().getSection());
-  Value += Writer->getSectionAddress(&SymSD);
+  Value += Writer->getSectionAddress(&S.getSection());
   Value -= Writer->getSectionAddress(Fragment.getParent());
   // If the resultant value would be out of range for an internal relocation,
   // use an external instead.
@@ -345,7 +337,7 @@ bool ARMMachObjectWriter::requiresExternRelocation(MachObjectWriter *Writer,
   return false;
 }
 
-void ARMMachObjectWriter::RecordRelocation(MachObjectWriter *Writer,
+void ARMMachObjectWriter::recordRelocation(MachObjectWriter *Writer,
                                            MCAssembler &Asm,
                                            const MCAsmLayout &Layout,
                                            const MCFragment *Fragment,
@@ -359,7 +351,7 @@ void ARMMachObjectWriter::RecordRelocation(MachObjectWriter *Writer,
     // relocation type for the fixup kind. This happens when it's a fixup that's
     // expected to always be resolvable at assembly time and not have any
     // relocations needed.
-    Asm.getContext().FatalError(Fixup.getLoc(),
+    Asm.getContext().reportFatalError(Fixup.getLoc(),
                                 "unsupported relocation on symbol");
 
   // If this is a difference or a defined symbol plus an offset, then we need a
@@ -375,9 +367,9 @@ void ARMMachObjectWriter::RecordRelocation(MachObjectWriter *Writer,
   }
 
   // Get the symbol data, if any.
-  const MCSymbolData *SD = nullptr;
+  const MCSymbol *A = nullptr;
   if (Target.getSymA())
-    SD = &Asm.getSymbolData(Target.getSymA()->getSymbol());
+    A = &Target.getSymA()->getSymbol();
 
   // FIXME: For other platforms, we need to use scattered relocations for
   // internal relocations with offsets.  If this is an internal relocation with
@@ -387,7 +379,7 @@ void ARMMachObjectWriter::RecordRelocation(MachObjectWriter *Writer,
   uint32_t Offset = Target.getConstant();
   if (IsPCRel && RelocType == MachO::ARM_RELOC_VANILLA)
     Offset += 1 << Log2Size;
-  if (Offset && SD && !Writer->doesSymbolRequireExternRelocation(SD))
+  if (Offset && A && !Writer->doesSymbolRequireExternRelocation(*A))
     return RecordARMScatteredRelocation(Writer, Asm, Layout, Fragment, Fixup,
                                         Target, RelocType, Log2Size,
                                         FixedValue);
@@ -396,7 +388,7 @@ void ARMMachObjectWriter::RecordRelocation(MachObjectWriter *Writer,
   uint32_t FixupOffset = Layout.getFragmentOffset(Fragment)+Fixup.getOffset();
   unsigned Index = 0;
   unsigned Type = 0;
-  const MCSymbolData *RelSymbol = nullptr;
+  const MCSymbol *RelSymbol = nullptr;
 
   if (Target.isAbsolute()) { // constant
     // FIXME!
@@ -404,31 +396,30 @@ void ARMMachObjectWriter::RecordRelocation(MachObjectWriter *Writer,
                        "not yet implemented");
   } else {
     // Resolve constant variables.
-    if (SD->getSymbol().isVariable()) {
+    if (A->isVariable()) {
       int64_t Res;
-      if (SD->getSymbol().getVariableValue()->EvaluateAsAbsolute(
-            Res, Layout, Writer->getSectionAddressMap())) {
+      if (A->getVariableValue()->evaluateAsAbsolute(
+              Res, Layout, Writer->getSectionAddressMap())) {
         FixedValue = Res;
         return;
       }
     }
 
     // Check whether we need an external or internal relocation.
-    if (requiresExternRelocation(Writer, Asm, *Fragment, RelocType, SD,
+    if (requiresExternRelocation(Writer, Asm, *Fragment, RelocType, *A,
                                  FixedValue)) {
-      RelSymbol = SD;
+      RelSymbol = A;
 
       // For external relocations, make sure to offset the fixup value to
       // compensate for the addend of the symbol address, if it was
       // undefined. This occurs with weak definitions, for example.
-      if (!SD->getSymbol().isUndefined())
-        FixedValue -= Layout.getSymbolOffset(SD);
+      if (!A->isUndefined())
+        FixedValue -= Layout.getSymbolOffset(*A);
     } else {
       // The index is the section ordinal (1-based).
-      const MCSectionData &SymSD = Asm.getSectionData(
-        SD->getSymbol().getSection());
-      Index = SymSD.getOrdinal() + 1;
-      FixedValue += Writer->getSectionAddress(&SymSD);
+      const MCSection &Sec = A->getSection();
+      Index = Sec.getOrdinal() + 1;
+      FixedValue += Writer->getSectionAddress(&Sec);
     }
     if (IsPCRel)
       FixedValue -= Writer->getSectionAddress(Fragment->getParent());
diff --git a/lib/Target/ARM/MCTargetDesc/CMakeLists.txt b/lib/Target/ARM/MCTargetDesc/CMakeLists.txt
index da61b762c1d..9582e8cbef4 100644
--- a/lib/Target/ARM/MCTargetDesc/CMakeLists.txt
+++ b/lib/Target/ARM/MCTargetDesc/CMakeLists.txt
@@ -8,7 +8,6 @@ add_llvm_library(LLVMARMDesc
   ARMMCAsmInfo.cpp
   ARMMCCodeEmitter.cpp
   ARMMCExpr.cpp
-  ARMMCNaCl.cpp # LOCALMOD
   ARMMCTargetDesc.cpp
   ARMTargetStreamer.cpp
   ARMUnwindOpAsm.cpp
diff --git a/lib/Target/ARM/TargetInfo/ARMTargetInfo.cpp b/lib/Target/ARM/TargetInfo/ARMTargetInfo.cpp
index e4646713d4d..df73554372d 100644
--- a/lib/Target/ARM/TargetInfo/ARMTargetInfo.cpp
+++ b/lib/Target/ARM/TargetInfo/ARMTargetInfo.cpp
@@ -15,7 +15,7 @@ using namespace llvm;
 Target llvm::TheARMLETarget,   llvm::TheARMBETarget;
 Target llvm::TheThumbLETarget, llvm::TheThumbBETarget;
 
-extern "C" void LLVMInitializeARMTargetInfo() { 
+extern "C" void LLVMInitializeARMTargetInfo() {
   RegisterTarget<Triple::arm, /*HasJIT=*/true>
     X(TheARMLETarget, "arm", "ARM");
   RegisterTarget<Triple::armeb, /*HasJIT=*/true>
diff --git a/lib/Target/ARM/Thumb1FrameLowering.cpp b/lib/Target/ARM/Thumb1FrameLowering.cpp
index c496cd7c759..46b31c350a0 100644
--- a/lib/Target/ARM/Thumb1FrameLowering.cpp
+++ b/lib/Target/ARM/Thumb1FrameLowering.cpp
@@ -13,6 +13,7 @@
 
 #include "Thumb1FrameLowering.h"
 #include "ARMMachineFunctionInfo.h"
+#include "llvm/CodeGen/LivePhysRegs.h"
 #include "llvm/CodeGen/MachineFrameInfo.h"
 #include "llvm/CodeGen/MachineFunction.h"
 #include "llvm/CodeGen/MachineInstrBuilder.h"
@@ -82,8 +83,8 @@ eliminateCallFramePseudoInstr(MachineFunction &MF, MachineBasicBlock &MBB,
   MBB.erase(I);
 }
 
-void Thumb1FrameLowering::emitPrologue(MachineFunction &MF) const {
-  MachineBasicBlock &MBB = MF.front();
+void Thumb1FrameLowering::emitPrologue(MachineFunction &MF,
+                                       MachineBasicBlock &MBB) const {
   MachineBasicBlock::iterator MBBI = MBB.begin();
   MachineFrameInfo  *MFI = MF.getFrameInfo();
   ARMFunctionInfo *AFI = MF.getInfo<ARMFunctionInfo>();
@@ -320,11 +321,8 @@ static bool isCSRestore(MachineInstr *MI, const MCPhysReg *CSRegs) {
 
 void Thumb1FrameLowering::emitEpilogue(MachineFunction &MF,
                                    MachineBasicBlock &MBB) const {
-  MachineBasicBlock::iterator MBBI = MBB.getLastNonDebugInstr();
-  assert((MBBI->getOpcode() == ARM::tBX_RET ||
-          MBBI->getOpcode() == ARM::tPOP_RET) &&
-         "Can only insert epilog into returning blocks");
-  DebugLoc dl = MBBI->getDebugLoc();
+  MachineBasicBlock::iterator MBBI = MBB.getFirstTerminator();
+  DebugLoc dl = MBBI != MBB.end() ? MBBI->getDebugLoc() : DebugLoc();
   MachineFrameInfo *MFI = MF.getFrameInfo();
   ARMFunctionInfo *AFI = MF.getInfo<ARMFunctionInfo>();
   const ThumbRegisterInfo *RegInfo =
@@ -364,7 +362,7 @@ void Thumb1FrameLowering::emitEpilogue(MachineFunction &MF,
       // frame pointer stack slot, the target is ELF and the function has FP, or
       // the target uses var sized objects.
       if (NumBytes) {
-        assert(MF.getRegInfo().isPhysRegUsed(ARM::R4) &&
+        assert(!MFI->getPristineRegs(MF).test(ARM::R4) &&
                "No scratch register to restore SP from FP!");
         emitThumbRegPlusImmediate(MBB, MBBI, dl, ARM::R4, FramePtr, -NumBytes,
                                   TII, *RegInfo);
@@ -376,9 +374,8 @@ void Thumb1FrameLowering::emitEpilogue(MachineFunction &MF,
                                ARM::SP)
           .addReg(FramePtr));
     } else {
-      if (MBBI->getOpcode() == ARM::tBX_RET &&
-          &MBB.front() != MBBI &&
-          std::prev(MBBI)->getOpcode() == ARM::tPOP) {
+      if (MBBI != MBB.end() && MBBI->getOpcode() == ARM::tBX_RET &&
+          &MBB.front() != MBBI && std::prev(MBBI)->getOpcode() == ARM::tPOP) {
         MachineBasicBlock::iterator PMBBI = std::prev(MBBI);
         if (!tryFoldSPUpdateIntoPushPop(STI, MF, PMBBI, NumBytes))
           emitSPUpdate(MBB, PMBBI, TII, dl, *RegInfo, NumBytes);
@@ -387,66 +384,156 @@ void Thumb1FrameLowering::emitEpilogue(MachineFunction &MF,
     }
   }
 
+  if (needPopSpecialFixUp(MF)) {
+    bool Done = emitPopSpecialFixUp(MBB, /* DoIt */ true);
+    (void)Done;
+    assert(Done && "Emission of the special fixup failed!?");
+  }
+}
+
+bool Thumb1FrameLowering::canUseAsEpilogue(const MachineBasicBlock &MBB) const {
+  if (!needPopSpecialFixUp(*MBB.getParent()))
+    return true;
+
+  MachineBasicBlock *TmpMBB = const_cast<MachineBasicBlock *>(&MBB);
+  return emitPopSpecialFixUp(*TmpMBB, /* DoIt */ false);
+}
+
+bool Thumb1FrameLowering::needPopSpecialFixUp(const MachineFunction &MF) const {
+  ARMFunctionInfo *AFI =
+      const_cast<MachineFunction *>(&MF)->getInfo<ARMFunctionInfo>();
+  if (AFI->getArgRegsSaveSize())
+    return true;
+
   bool IsV4PopReturn = false;
-  for (const CalleeSavedInfo &CSI : MFI->getCalleeSavedInfo())
+  for (const CalleeSavedInfo &CSI : MF.getFrameInfo()->getCalleeSavedInfo())
     if (CSI.getReg() == ARM::LR)
       IsV4PopReturn = true;
-  IsV4PopReturn &= STI.hasV4TOps() && !STI.hasV5TOps();
-
-  // Unlike T2 and ARM mode, the T1 pop instruction cannot restore
-  // to LR, and we can't pop the value directly to the PC since
-  // we need to update the SP after popping the value. So instead
-  // we have to emit:
-  //   POP {r3}
-  //   ADD sp, #offset
-  //   BX r3
-  // If this would clobber a return value, then generate this sequence instead:
-  //   MOV ip, r3
-  //   POP {r3}
-  //   ADD sp, #offset
-  //   MOV lr, r3
-  //   MOV r3, ip
-  //   BX lr
-  if (ArgRegsSaveSize || IsV4PopReturn) {
-    // Get the last instruction, tBX_RET
-    MBBI = MBB.getLastNonDebugInstr();
-    assert (MBBI->getOpcode() == ARM::tBX_RET);
-    DebugLoc dl = MBBI->getDebugLoc();
-
-    if (AFI->getReturnRegsCount() <= 3) {
-      // Epilogue: pop saved LR to R3 and branch off it. 
-      AddDefaultPred(BuildMI(MBB, MBBI, dl, TII.get(ARM::tPOP)))
-        .addReg(ARM::R3, RegState::Define);
-
-      emitSPUpdate(MBB, MBBI, TII, dl, *RegInfo, ArgRegsSaveSize);
-
-      MachineInstrBuilder MIB =
-        BuildMI(MBB, MBBI, dl, TII.get(ARM::tBX))
-        .addReg(ARM::R3, RegState::Kill);
-      AddDefaultPred(MIB);
-      MIB.copyImplicitOps(&*MBBI);
-      // erase the old tBX_RET instruction
-      MBB.erase(MBBI);
-    } else {
-      AddDefaultPred(BuildMI(MBB, MBBI, dl, TII.get(ARM::tMOVr))
-        .addReg(ARM::R12, RegState::Define)
-        .addReg(ARM::R3, RegState::Kill));
+  return IsV4PopReturn && STI.hasV4TOps() && !STI.hasV5TOps();
+}
 
-      AddDefaultPred(BuildMI(MBB, MBBI, dl, TII.get(ARM::tPOP)))
-        .addReg(ARM::R3, RegState::Define);
+bool Thumb1FrameLowering::emitPopSpecialFixUp(MachineBasicBlock &MBB,
+                                              bool DoIt) const {
+  MachineFunction &MF = *MBB.getParent();
+  ARMFunctionInfo *AFI = MF.getInfo<ARMFunctionInfo>();
+  unsigned ArgRegsSaveSize = AFI->getArgRegsSaveSize();
+  const TargetInstrInfo &TII = *STI.getInstrInfo();
+  const ThumbRegisterInfo *RegInfo =
+      static_cast<const ThumbRegisterInfo *>(STI.getRegisterInfo());
 
-      emitSPUpdate(MBB, MBBI, TII, dl, *RegInfo, ArgRegsSaveSize);
+  // If MBBI is a return instruction, we may be able to directly restore
+  // LR in the PC.
+  // This is possible if we do not need to emit any SP update.
+  // Otherwise, we need a temporary register to pop the value
+  // and copy that value into LR.
+  auto MBBI = MBB.getFirstTerminator();
+  if (!ArgRegsSaveSize && MBBI != MBB.end() &&
+      MBBI->getOpcode() == ARM::tBX_RET) {
+    if (!DoIt)
+      return true;
+    MachineInstrBuilder MIB =
+        AddDefaultPred(
+            BuildMI(MBB, MBBI, MBBI->getDebugLoc(), TII.get(ARM::tPOP_RET)))
+            .addReg(ARM::PC, RegState::Define);
+    MIB.copyImplicitOps(&*MBBI);
+    // erase the old tBX_RET instruction
+    MBB.erase(MBBI);
+    return true;
+  }
 
-      AddDefaultPred(BuildMI(MBB, MBBI, dl, TII.get(ARM::tMOVr))
-        .addReg(ARM::LR, RegState::Define)
-        .addReg(ARM::R3, RegState::Kill));
+  // Look for a temporary register to use.
+  // First, compute the liveness information.
+  LivePhysRegs UsedRegs(STI.getRegisterInfo());
+  UsedRegs.addLiveOuts(&MBB, /*AddPristines*/ true);
+  // The semantic of pristines changed recently and now,
+  // the callee-saved registers that are touched in the function
+  // are not part of the pristines set anymore.
+  // Add those callee-saved now.
+  const TargetRegisterInfo *TRI = STI.getRegisterInfo();
+  const MCPhysReg *CSRegs = TRI->getCalleeSavedRegs(&MF);
+  for (unsigned i = 0; CSRegs[i]; ++i)
+    UsedRegs.addReg(CSRegs[i]);
+
+  DebugLoc dl = DebugLoc();
+  if (MBBI != MBB.end()) {
+    dl = MBBI->getDebugLoc();
+    auto InstUpToMBBI = MBB.end();
+    // The post-decrement is on purpose here.
+    // We want to have the liveness right before MBBI.
+    while (InstUpToMBBI-- != MBBI)
+      UsedRegs.stepBackward(*InstUpToMBBI);
+  }
 
-      AddDefaultPred(BuildMI(MBB, MBBI, dl, TII.get(ARM::tMOVr))
-        .addReg(ARM::R3, RegState::Define)
-        .addReg(ARM::R12, RegState::Kill));
-      // Keep the tBX_RET instruction
+  // Look for a register that can be directly use in the POP.
+  unsigned PopReg = 0;
+  // And some temporary register, just in case.
+  unsigned TemporaryReg = 0;
+  BitVector PopFriendly =
+      TRI->getAllocatableSet(MF, TRI->getRegClass(ARM::tGPRRegClassID));
+  assert(PopFriendly.any() && "No allocatable pop-friendly register?!");
+  // Rebuild the GPRs from the high registers because they are removed
+  // form the GPR reg class for thumb1.
+  BitVector GPRsNoLRSP =
+      TRI->getAllocatableSet(MF, TRI->getRegClass(ARM::hGPRRegClassID));
+  GPRsNoLRSP |= PopFriendly;
+  GPRsNoLRSP.reset(ARM::LR);
+  GPRsNoLRSP.reset(ARM::SP);
+  GPRsNoLRSP.reset(ARM::PC);
+  for (int Register = GPRsNoLRSP.find_first(); Register != -1;
+       Register = GPRsNoLRSP.find_next(Register)) {
+    if (!UsedRegs.contains(Register)) {
+      // Remember the first pop-friendly register and exit.
+      if (PopFriendly.test(Register)) {
+        PopReg = Register;
+        TemporaryReg = 0;
+        break;
+      }
+      // Otherwise, remember that the register will be available to
+      // save a pop-friendly register.
+      TemporaryReg = Register;
     }
   }
+
+  if (!DoIt && !PopReg && !TemporaryReg)
+    return false;
+
+  assert((PopReg || TemporaryReg) && "Cannot get LR");
+
+  if (TemporaryReg) {
+    assert(!PopReg && "Unnecessary MOV is about to be inserted");
+    PopReg = PopFriendly.find_first();
+    AddDefaultPred(BuildMI(MBB, MBBI, dl, TII.get(ARM::tMOVr))
+                       .addReg(TemporaryReg, RegState::Define)
+                       .addReg(PopReg, RegState::Kill));
+  }
+
+  assert(PopReg && "Do not know how to get LR");
+  AddDefaultPred(BuildMI(MBB, MBBI, dl, TII.get(ARM::tPOP)))
+      .addReg(PopReg, RegState::Define);
+
+  emitSPUpdate(MBB, MBBI, TII, dl, *RegInfo, ArgRegsSaveSize);
+
+  if (!TemporaryReg && MBBI != MBB.end() && MBBI->getOpcode() == ARM::tBX_RET) {
+    MachineInstrBuilder MIB = BuildMI(MBB, MBBI, dl, TII.get(ARM::tBX))
+                                  .addReg(PopReg, RegState::Kill);
+    AddDefaultPred(MIB);
+    MIB.copyImplicitOps(&*MBBI);
+    // erase the old tBX_RET instruction
+    MBB.erase(MBBI);
+    return true;
+  }
+
+  AddDefaultPred(BuildMI(MBB, MBBI, dl, TII.get(ARM::tMOVr))
+                     .addReg(ARM::LR, RegState::Define)
+                     .addReg(PopReg, RegState::Kill));
+
+  if (TemporaryReg) {
+    AddDefaultPred(BuildMI(MBB, MBBI, dl, TII.get(ARM::tMOVr))
+                       .addReg(PopReg, RegState::Define)
+                       .addReg(TemporaryReg, RegState::Kill));
+  }
+
+  return true;
 }
 
 bool Thumb1FrameLowering::
@@ -500,14 +587,14 @@ restoreCalleeSavedRegisters(MachineBasicBlock &MBB,
   const TargetInstrInfo &TII = *STI.getInstrInfo();
 
   bool isVarArg = AFI->getArgRegsSaveSize() > 0;
-  DebugLoc DL = MI->getDebugLoc();
+  DebugLoc DL = MI != MBB.end() ? MI->getDebugLoc() : DebugLoc();
   MachineInstrBuilder MIB = BuildMI(MF, DL, TII.get(ARM::tPOP));
   AddDefaultPred(MIB);
 
   bool NumRegs = false;
   for (unsigned i = CSI.size(); i != 0; --i) {
     unsigned Reg = CSI[i-1].getReg();
-    if (Reg == ARM::LR) {
+    if (Reg == ARM::LR && MBB.succ_empty()) {
       // Special epilogue for vararg functions. See emitEpilogue
       if (isVarArg)
         continue;
@@ -516,7 +603,8 @@ restoreCalleeSavedRegisters(MachineBasicBlock &MBB,
         continue;
       Reg = ARM::PC;
       (*MIB).setDesc(TII.get(ARM::tPOP_RET));
-      MIB.copyImplicitOps(&*MI);
+      if (MI != MBB.end())
+        MIB.copyImplicitOps(&*MI);
       MI = MBB.erase(MI);
     }
     MIB.addReg(Reg, getDefRegState(true));
diff --git a/lib/Target/ARM/Thumb1FrameLowering.h b/lib/Target/ARM/Thumb1FrameLowering.h
index cf932035b8d..812f9830824 100644
--- a/lib/Target/ARM/Thumb1FrameLowering.h
+++ b/lib/Target/ARM/Thumb1FrameLowering.h
@@ -27,7 +27,7 @@ class Thumb1FrameLowering : public ARMFrameLowering {
 
   /// emitProlog/emitEpilog - These methods insert prolog and epilog code into
   /// the function.
-  void emitPrologue(MachineFunction &MF) const override;
+  void emitPrologue(MachineFunction &MF, MachineBasicBlock &MBB) const override;
   void emitEpilogue(MachineFunction &MF, MachineBasicBlock &MBB) const override;
 
   bool spillCalleeSavedRegisters(MachineBasicBlock &MBB,
@@ -45,6 +45,42 @@ class Thumb1FrameLowering : public ARMFrameLowering {
   eliminateCallFramePseudoInstr(MachineFunction &MF,
                                 MachineBasicBlock &MBB,
                                 MachineBasicBlock::iterator MI) const override;
+
+  /// Check whether or not the given \p MBB can be used as a epilogue
+  /// for the target.
+  /// The epilogue will be inserted before the first terminator of that block.
+  /// This method is used by the shrink-wrapping pass to decide if
+  /// \p MBB will be correctly handled by the target.
+  bool canUseAsEpilogue(const MachineBasicBlock &MBB) const override;
+
+private:
+  /// Check if the frame lowering of \p MF needs a special fixup
+  /// code sequence for the epilogue.
+  /// Unlike T2 and ARM mode, the T1 pop instruction cannot restore
+  /// to LR, and we can't pop the value directly to the PC when
+  /// we need to update the SP after popping the value. So instead
+  /// we have to emit:
+  ///   POP {r3}
+  ///   ADD sp, #offset
+  ///   BX r3
+  /// If this would clobber a return value, then generate this sequence instead:
+  ///   MOV ip, r3
+  ///   POP {r3}
+  ///   ADD sp, #offset
+  ///   MOV lr, r3
+  ///   MOV r3, ip
+  ///   BX lr
+  bool needPopSpecialFixUp(const MachineFunction &MF) const;
+
+  /// Emit the special fixup code sequence for the epilogue.
+  /// \see needPopSpecialFixUp for more details.
+  /// \p DoIt, tells this method whether or not to actually insert
+  /// the code sequence in \p MBB. I.e., when \p DoIt is false,
+  /// \p MBB is left untouched.
+  /// \returns For \p DoIt == true: True when the emission succeeded
+  /// false otherwise. For \p DoIt == false: True when the emission
+  /// would have been possible, false otherwise.
+  bool emitPopSpecialFixUp(MachineBasicBlock &MBB, bool DoIt) const;
 };
 
 } // End llvm namespace
diff --git a/lib/Target/ARM/Thumb1InstrInfo.cpp b/lib/Target/ARM/Thumb1InstrInfo.cpp
index 29aaa1580ca..028119c264b 100644
--- a/lib/Target/ARM/Thumb1InstrInfo.cpp
+++ b/lib/Target/ARM/Thumb1InstrInfo.cpp
@@ -27,10 +27,10 @@ Thumb1InstrInfo::Thumb1InstrInfo(const ARMSubtarget &STI)
 /// getNoopForMachoTarget - Return the noop instruction to use for a noop.
 void Thumb1InstrInfo::getNoopForMachoTarget(MCInst &NopInst) const {
   NopInst.setOpcode(ARM::tMOVr);
-  NopInst.addOperand(MCOperand::CreateReg(ARM::R8));
-  NopInst.addOperand(MCOperand::CreateReg(ARM::R8));
-  NopInst.addOperand(MCOperand::CreateImm(ARMCC::AL));
-  NopInst.addOperand(MCOperand::CreateReg(0));
+  NopInst.addOperand(MCOperand::createReg(ARM::R8));
+  NopInst.addOperand(MCOperand::createReg(ARM::R8));
+  NopInst.addOperand(MCOperand::createImm(ARMCC::AL));
+  NopInst.addOperand(MCOperand::createReg(0));
 }
 
 unsigned Thumb1InstrInfo::getUnindexedOpcode(unsigned Opc) const {
diff --git a/lib/Target/ARM/Thumb2ITBlockPass.cpp b/lib/Target/ARM/Thumb2ITBlockPass.cpp
index 7bb226510a2..68736bc1dec 100644
--- a/lib/Target/ARM/Thumb2ITBlockPass.cpp
+++ b/lib/Target/ARM/Thumb2ITBlockPass.cpp
@@ -90,6 +90,19 @@ static void TrackDefUses(MachineInstr *MI,
   }
 }
 
+/// Clear kill flags for any uses in the given set.  This will likely
+/// conservatively remove more kill flags than are necessary, but removing them
+/// is safer than incorrect kill flags remaining on instructions.
+static void ClearKillFlags(MachineInstr *MI, SmallSet<unsigned, 4> &Uses) {
+  for (MachineOperand &MO : MI->operands()) {
+    if (!MO.isReg() || MO.isDef() || !MO.isKill())
+      continue;
+    if (!Uses.count(MO.getReg()))
+      continue;
+    MO.setIsKill(false);
+  }
+}
+
 static bool isCopy(MachineInstr *MI) {
   switch (MI->getOpcode()) {
   default:
@@ -222,6 +235,7 @@ bool Thumb2ITBlockPass::InsertITInstructions(MachineBasicBlock &MBB) {
             --MBBI;
             MBB.remove(NMI);
             MBB.insert(InsertPos, NMI);
+            ClearKillFlags(MI, Uses);
             ++NumMovedInsts;
             continue;
           }
diff --git a/lib/Target/ARM/Thumb2InstrInfo.cpp b/lib/Target/ARM/Thumb2InstrInfo.cpp
index 26ca7e90e4c..dc74f4e38ff 100644
--- a/lib/Target/ARM/Thumb2InstrInfo.cpp
+++ b/lib/Target/ARM/Thumb2InstrInfo.cpp
@@ -35,9 +35,9 @@ Thumb2InstrInfo::Thumb2InstrInfo(const ARMSubtarget &STI)
 /// getNoopForMachoTarget - Return the noop instruction to use for a noop.
 void Thumb2InstrInfo::getNoopForMachoTarget(MCInst &NopInst) const {
   NopInst.setOpcode(ARM::tHINT);
-  NopInst.addOperand(MCOperand::CreateImm(0));
-  NopInst.addOperand(MCOperand::CreateImm(ARMCC::AL));
-  NopInst.addOperand(MCOperand::CreateReg(0));
+  NopInst.addOperand(MCOperand::createImm(0));
+  NopInst.addOperand(MCOperand::createImm(ARMCC::AL));
+  NopInst.addOperand(MCOperand::createReg(0));
 }
 
 unsigned Thumb2InstrInfo::getUnindexedOpcode(unsigned Opc) const {
@@ -256,14 +256,19 @@ void llvm::emitT2RegPlusImmediate(MachineBasicBlock &MBB,
     if (Fits) {
       if (isSub) {
         BuildMI(MBB, MBBI, dl, TII.get(ARM::t2SUBrr), DestReg)
-          .addReg(BaseReg, RegState::Kill)
+          .addReg(BaseReg)
           .addReg(DestReg, RegState::Kill)
           .addImm((unsigned)Pred).addReg(PredReg).addReg(0)
           .setMIFlags(MIFlags);
       } else {
+        // Here we know that DestReg is not SP but we do not
+        // know anything about BaseReg. t2ADDrr is an invalid
+        // instruction is SP is used as the second argument, but
+        // is fine if SP is the first argument. To be sure we
+        // do not generate invalid encoding, put BaseReg first.
         BuildMI(MBB, MBBI, dl, TII.get(ARM::t2ADDrr), DestReg)
+          .addReg(BaseReg)
           .addReg(DestReg, RegState::Kill)
-          .addReg(BaseReg, RegState::Kill)
           .addImm((unsigned)Pred).addReg(PredReg).addReg(0)
           .setMIFlags(MIFlags);
       }
diff --git a/lib/Target/ARM/Thumb2SizeReduction.cpp b/lib/Target/ARM/Thumb2SizeReduction.cpp
index e967e537561..d9ab824995c 100644
--- a/lib/Target/ARM/Thumb2SizeReduction.cpp
+++ b/lib/Target/ARM/Thumb2SizeReduction.cpp
@@ -133,7 +133,7 @@ namespace {
   class Thumb2SizeReduce : public MachineFunctionPass {
   public:
     static char ID;
-    Thumb2SizeReduce();
+    Thumb2SizeReduce(std::function<bool(const Function &)> Ftor);
 
     const Thumb2InstrInfo *TII;
     const ARMSubtarget *STI;
@@ -198,11 +198,14 @@ namespace {
     };
 
     SmallVector<MBBInfo, 8> BlockInfo;
+
+    std::function<bool(const Function &)> PredicateFtor;
   };
   char Thumb2SizeReduce::ID = 0;
 }
 
-Thumb2SizeReduce::Thumb2SizeReduce() : MachineFunctionPass(ID) {
+Thumb2SizeReduce::Thumb2SizeReduce(std::function<bool(const Function &)> Ftor)
+    : MachineFunctionPass(ID), PredicateFtor(Ftor) {
   OptimizeSize = MinimizeSize = false;
   for (unsigned i = 0, e = array_lengthof(ReduceTable); i != e; ++i) {
     unsigned FromOpc = ReduceTable[i].WideOpc;
@@ -333,9 +336,7 @@ Thumb2SizeReduce::VerifyPredAndCC(MachineInstr *MI, const ReduceEntry &Entry,
 
 static bool VerifyLowRegs(MachineInstr *MI) {
   unsigned Opc = MI->getOpcode();
-  bool isPCOk = (Opc == ARM::t2LDMIA_RET || Opc == ARM::t2LDMIA     ||
-                 Opc == ARM::t2LDMDB     || Opc == ARM::t2LDMIA_UPD ||
-                 Opc == ARM::t2LDMDB_UPD);
+  bool isPCOk = (Opc == ARM::t2LDMIA_RET || Opc == ARM::t2LDMIA_UPD);
   bool isLROk = (Opc == ARM::t2STMDB_UPD);
   bool isSPOk = isPCOk || isLROk;
   for (unsigned i = 0, e = MI->getNumOperands(); i != e; ++i) {
@@ -413,16 +414,14 @@ Thumb2SizeReduce::ReduceLoadStore(MachineBasicBlock &MBB, MachineInstr *MI,
     HasShift = true;
     OpNum = 4;
     break;
-  case ARM::t2LDMIA:
-  case ARM::t2LDMDB: {
+  case ARM::t2LDMIA: {
     unsigned BaseReg = MI->getOperand(0).getReg();
-    if (!isARMLowRegister(BaseReg) || Entry.WideOpc != ARM::t2LDMIA)
-      return false;
+    assert(isARMLowRegister(BaseReg));
 
     // For the non-writeback version (this one), the base register must be
     // one of the registers being loaded.
     bool isOK = false;
-    for (unsigned i = 4; i < MI->getNumOperands(); ++i) {
+    for (unsigned i = 3; i < MI->getNumOperands(); ++i) {
       if (MI->getOperand(i).getReg() == BaseReg) {
         isOK = true;
         break;
@@ -446,7 +445,6 @@ Thumb2SizeReduce::ReduceLoadStore(MachineBasicBlock &MBB, MachineInstr *MI,
     break;
   }
   case ARM::t2LDMIA_UPD:
-  case ARM::t2LDMDB_UPD:
   case ARM::t2STMIA_UPD:
   case ARM::t2STMDB_UPD: {
     OpNum = 0;
@@ -470,9 +468,11 @@ Thumb2SizeReduce::ReduceLoadStore(MachineBasicBlock &MBB, MachineInstr *MI,
 
   unsigned OffsetReg = 0;
   bool OffsetKill = false;
+  bool OffsetInternal = false;
   if (HasShift) {
     OffsetReg  = MI->getOperand(2).getReg();
     OffsetKill = MI->getOperand(2).isKill();
+    OffsetInternal = MI->getOperand(2).isInternalRead();
 
     if (MI->getOperand(3).getImm())
       // Thumb1 addressing mode doesn't support shift.
@@ -502,7 +502,8 @@ Thumb2SizeReduce::ReduceLoadStore(MachineBasicBlock &MBB, MachineInstr *MI,
     assert((!HasShift || OffsetReg) && "Invalid so_reg load / store address!");
 
     if (HasOffReg)
-      MIB.addReg(OffsetReg, getKillRegState(OffsetKill));
+      MIB.addReg(OffsetReg, getKillRegState(OffsetKill) |
+                            getInternalReadRegState(OffsetInternal));
   }
 
   // Transfer the rest of operands.
@@ -571,7 +572,7 @@ Thumb2SizeReduce::ReduceSpecial(MachineBasicBlock &MBB, MachineInstr *MI,
   if (Entry.LowRegs1 && !VerifyLowRegs(MI))
     return false;
 
-  if (MI->mayLoad() || MI->mayStore())
+  if (MI->mayLoadOrStore())
     return ReduceLoadStore(MBB, MI, Entry);
 
   switch (Opc) {
@@ -1002,6 +1003,9 @@ bool Thumb2SizeReduce::ReduceMBB(MachineBasicBlock &MBB) {
 }
 
 bool Thumb2SizeReduce::runOnMachineFunction(MachineFunction &MF) {
+  if (PredicateFtor && !PredicateFtor(*MF.getFunction()))
+    return false;
+
   STI = &static_cast<const ARMSubtarget &>(MF.getSubtarget());
   if (STI->isThumb1Only() || STI->prefers32BitThumb())
     return false;
@@ -1027,6 +1031,7 @@ bool Thumb2SizeReduce::runOnMachineFunction(MachineFunction &MF) {
 
 /// createThumb2SizeReductionPass - Returns an instance of the Thumb2 size
 /// reduction pass.
-FunctionPass *llvm::createThumb2SizeReductionPass() {
-  return new Thumb2SizeReduce();
+FunctionPass *llvm::createThumb2SizeReductionPass(
+    std::function<bool(const Function &)> Ftor) {
+  return new Thumb2SizeReduce(Ftor);
 }
diff --git a/lib/Target/BPF/BPFAsmPrinter.cpp b/lib/Target/BPF/BPFAsmPrinter.cpp
index dbc7bfec256..10ec6587550 100644
--- a/lib/Target/BPF/BPFAsmPrinter.cpp
+++ b/lib/Target/BPF/BPFAsmPrinter.cpp
@@ -78,10 +78,12 @@ void BPFAsmPrinter::EmitInstruction(const MachineInstr *MI) {
 
   MCInst TmpInst;
   MCInstLowering.Lower(MI, TmpInst);
-  EmitToStreamer(OutStreamer, TmpInst);
+  EmitToStreamer(*OutStreamer, TmpInst);
 }
 
 // Force static initialization.
 extern "C" void LLVMInitializeBPFAsmPrinter() {
-  RegisterAsmPrinter<BPFAsmPrinter> X(TheBPFTarget);
+  RegisterAsmPrinter<BPFAsmPrinter> X(TheBPFleTarget);
+  RegisterAsmPrinter<BPFAsmPrinter> Y(TheBPFbeTarget);
+  RegisterAsmPrinter<BPFAsmPrinter> Z(TheBPFTarget);
 }
diff --git a/lib/Target/BPF/BPFFrameLowering.cpp b/lib/Target/BPF/BPFFrameLowering.cpp
index ae9f35520de..c2806c85f24 100644
--- a/lib/Target/BPF/BPFFrameLowering.cpp
+++ b/lib/Target/BPF/BPFFrameLowering.cpp
@@ -23,17 +23,18 @@ using namespace llvm;
 
 bool BPFFrameLowering::hasFP(const MachineFunction &MF) const { return true; }
 
-void BPFFrameLowering::emitPrologue(MachineFunction &MF) const {}
+void BPFFrameLowering::emitPrologue(MachineFunction &MF,
+                                    MachineBasicBlock &MBB) const {}
 
 void BPFFrameLowering::emitEpilogue(MachineFunction &MF,
                                     MachineBasicBlock &MBB) const {}
 
-void BPFFrameLowering::processFunctionBeforeCalleeSavedScan(
-    MachineFunction &MF, RegScavenger *RS) const {
-  MachineRegisterInfo &MRI = MF.getRegInfo();
-
-  MRI.setPhysRegUnused(BPF::R6);
-  MRI.setPhysRegUnused(BPF::R7);
-  MRI.setPhysRegUnused(BPF::R8);
-  MRI.setPhysRegUnused(BPF::R9);
+void BPFFrameLowering::determineCalleeSaves(MachineFunction &MF,
+                                            BitVector &SavedRegs,
+                                            RegScavenger *RS) const {
+  TargetFrameLowering::determineCalleeSaves(MF, SavedRegs, RS);
+  SavedRegs.reset(BPF::R6);
+  SavedRegs.reset(BPF::R7);
+  SavedRegs.reset(BPF::R8);
+  SavedRegs.reset(BPF::R9);
 }
diff --git a/lib/Target/BPF/BPFFrameLowering.h b/lib/Target/BPF/BPFFrameLowering.h
index 833046ddab6..251cda965ff 100644
--- a/lib/Target/BPF/BPFFrameLowering.h
+++ b/lib/Target/BPF/BPFFrameLowering.h
@@ -24,12 +24,12 @@ class BPFFrameLowering : public TargetFrameLowering {
   explicit BPFFrameLowering(const BPFSubtarget &sti)
       : TargetFrameLowering(TargetFrameLowering::StackGrowsDown, 8, 0) {}
 
-  void emitPrologue(MachineFunction &MF) const override;
+  void emitPrologue(MachineFunction &MF, MachineBasicBlock &MBB) const override;
   void emitEpilogue(MachineFunction &MF, MachineBasicBlock &MBB) const override;
 
   bool hasFP(const MachineFunction &MF) const override;
-  void processFunctionBeforeCalleeSavedScan(MachineFunction &MF,
-                                            RegScavenger *RS) const override;
+  void determineCalleeSaves(MachineFunction &MF, BitVector &SavedRegs,
+                            RegScavenger *RS) const override;
 
   void
   eliminateCallFramePseudoInstr(MachineFunction &MF, MachineBasicBlock &MBB,
diff --git a/lib/Target/BPF/BPFISelDAGToDAG.cpp b/lib/Target/BPF/BPFISelDAGToDAG.cpp
index b2599fe9637..d9e654c7642 100644
--- a/lib/Target/BPF/BPFISelDAGToDAG.cpp
+++ b/lib/Target/BPF/BPFISelDAGToDAG.cpp
@@ -56,9 +56,10 @@ class BPFDAGToDAGISel : public SelectionDAGISel {
 // ComplexPattern used on BPF Load/Store instructions
 bool BPFDAGToDAGISel::SelectAddr(SDValue Addr, SDValue &Base, SDValue &Offset) {
   // if Address is FI, get the TargetFrameIndex.
+  SDLoc DL(Addr);
   if (FrameIndexSDNode *FIN = dyn_cast<FrameIndexSDNode>(Addr)) {
     Base   = CurDAG->getTargetFrameIndex(FIN->getIndex(), MVT::i64);
-    Offset = CurDAG->getTargetConstant(0, MVT::i64);
+    Offset = CurDAG->getTargetConstant(0, DL, MVT::i64);
     return true;
   }
 
@@ -78,13 +79,13 @@ bool BPFDAGToDAGISel::SelectAddr(SDValue Addr, SDValue &Base, SDValue &Offset) {
       else
         Base = Addr.getOperand(0);
 
-      Offset = CurDAG->getTargetConstant(CN->getSExtValue(), MVT::i64);
+      Offset = CurDAG->getTargetConstant(CN->getSExtValue(), DL, MVT::i64);
       return true;
     }
   }
 
   Base   = Addr;
-  Offset = CurDAG->getTargetConstant(0, MVT::i64);
+  Offset = CurDAG->getTargetConstant(0, DL, MVT::i64);
   return true;
 }
 
diff --git a/lib/Target/BPF/BPFISelLowering.cpp b/lib/Target/BPF/BPFISelLowering.cpp
index 37f91648331..58498a1aec7 100644
--- a/lib/Target/BPF/BPFISelLowering.cpp
+++ b/lib/Target/BPF/BPFISelLowering.cpp
@@ -302,8 +302,9 @@ SDValue BPFTargetLowering::LowerCall(TargetLowering::CallLoweringInfo &CLI,
     DAG.getContext()->diagnose(Err);
   }
 
+  auto PtrVT = getPointerTy(MF.getDataLayout());
   Chain = DAG.getCALLSEQ_START(
-      Chain, DAG.getConstant(NumBytes, getPointerTy(), true), CLI.DL);
+      Chain, DAG.getConstant(NumBytes, CLI.DL, PtrVT, true), CLI.DL);
 
   SmallVector<std::pair<unsigned, SDValue>, 5> RegsToPass;
 
@@ -350,10 +351,10 @@ SDValue BPFTargetLowering::LowerCall(TargetLowering::CallLoweringInfo &CLI,
   // turn it into a TargetGlobalAddress node so that legalize doesn't hack it.
   // Likewise ExternalSymbol -> TargetExternalSymbol.
   if (GlobalAddressSDNode *G = dyn_cast<GlobalAddressSDNode>(Callee))
-    Callee = DAG.getTargetGlobalAddress(G->getGlobal(), CLI.DL, getPointerTy(),
+    Callee = DAG.getTargetGlobalAddress(G->getGlobal(), CLI.DL, PtrVT,
                                         G->getOffset(), 0);
   else if (ExternalSymbolSDNode *E = dyn_cast<ExternalSymbolSDNode>(Callee))
-    Callee = DAG.getTargetExternalSymbol(E->getSymbol(), getPointerTy(), 0);
+    Callee = DAG.getTargetExternalSymbol(E->getSymbol(), PtrVT, 0);
 
   // Returns a chain & a flag for retval copy to use.
   SDVTList NodeTys = DAG.getVTList(MVT::Other, MVT::Glue);
@@ -374,8 +375,8 @@ SDValue BPFTargetLowering::LowerCall(TargetLowering::CallLoweringInfo &CLI,
 
   // Create the CALLSEQ_END node.
   Chain = DAG.getCALLSEQ_END(
-      Chain, DAG.getConstant(NumBytes, getPointerTy(), true),
-      DAG.getConstant(0, getPointerTy(), true), InFlag, CLI.DL);
+      Chain, DAG.getConstant(NumBytes, CLI.DL, PtrVT, true),
+      DAG.getConstant(0, CLI.DL, PtrVT, true), InFlag, CLI.DL);
   InFlag = Chain.getValue(1);
 
   // Handle result values, copying them out of physregs into vregs that we
@@ -487,7 +488,7 @@ SDValue BPFTargetLowering::LowerBR_CC(SDValue Op, SelectionDAG &DAG) const {
   NegateCC(LHS, RHS, CC);
 
   return DAG.getNode(BPFISD::BR_CC, DL, Op.getValueType(), Chain, LHS, RHS,
-                     DAG.getConstant(CC, MVT::i64), Dest);
+                     DAG.getConstant(CC, DL, MVT::i64), Dest);
 }
 
 SDValue BPFTargetLowering::LowerSELECT_CC(SDValue Op, SelectionDAG &DAG) const {
@@ -500,7 +501,7 @@ SDValue BPFTargetLowering::LowerSELECT_CC(SDValue Op, SelectionDAG &DAG) const {
 
   NegateCC(LHS, RHS, CC);
 
-  SDValue TargetCC = DAG.getConstant(CC, MVT::i64);
+  SDValue TargetCC = DAG.getConstant(CC, DL, MVT::i64);
 
   SDVTList VTs = DAG.getVTList(Op.getValueType(), MVT::Glue);
   SDValue Ops[] = {LHS, RHS, TargetCC, TrueV, FalseV};
@@ -509,9 +510,9 @@ SDValue BPFTargetLowering::LowerSELECT_CC(SDValue Op, SelectionDAG &DAG) const {
 }
 
 const char *BPFTargetLowering::getTargetNodeName(unsigned Opcode) const {
-  switch (Opcode) {
-  default:
-    return NULL;
+  switch ((BPFISD::NodeType)Opcode) {
+  case BPFISD::FIRST_NUMBER:
+    break;
   case BPFISD::RET_FLAG:
     return "BPFISD::RET_FLAG";
   case BPFISD::CALL:
@@ -523,6 +524,7 @@ const char *BPFTargetLowering::getTargetNodeName(unsigned Opcode) const {
   case BPFISD::Wrapper:
     return "BPFISD::Wrapper";
   }
+  return nullptr;
 }
 
 SDValue BPFTargetLowering::LowerGlobalAddress(SDValue Op,
diff --git a/lib/Target/BPF/BPFISelLowering.h b/lib/Target/BPF/BPFISelLowering.h
index 5a6f0f78a58..ec71dca2fae 100644
--- a/lib/Target/BPF/BPFISelLowering.h
+++ b/lib/Target/BPF/BPFISelLowering.h
@@ -22,7 +22,7 @@
 namespace llvm {
 class BPFSubtarget;
 namespace BPFISD {
-enum {
+enum NodeType : unsigned {
   FIRST_NUMBER = ISD::BUILTIN_OP_END,
   RET_FLAG,
   CALL,
diff --git a/lib/Target/BPF/BPFInstrInfo.cpp b/lib/Target/BPF/BPFInstrInfo.cpp
index 28bd0ec6ebe..83d14efc1a6 100644
--- a/lib/Target/BPF/BPFInstrInfo.cpp
+++ b/lib/Target/BPF/BPFInstrInfo.cpp
@@ -133,7 +133,7 @@ bool BPFInstrInfo::AnalyzeBranch(MachineBasicBlock &MBB,
 unsigned BPFInstrInfo::InsertBranch(MachineBasicBlock &MBB,
                                     MachineBasicBlock *TBB,
                                     MachineBasicBlock *FBB,
-                                    const SmallVectorImpl<MachineOperand> &Cond,
+                                    ArrayRef<MachineOperand> Cond,
                                     DebugLoc DL) const {
   // Shouldn't be a fall through.
   assert(TBB && "InsertBranch must not be told to insert a fallthrough");
diff --git a/lib/Target/BPF/BPFInstrInfo.h b/lib/Target/BPF/BPFInstrInfo.h
index 4056c2efbbd..ac60188804d 100644
--- a/lib/Target/BPF/BPFInstrInfo.h
+++ b/lib/Target/BPF/BPFInstrInfo.h
@@ -51,8 +51,7 @@ class BPFInstrInfo : public BPFGenInstrInfo {
 
   unsigned RemoveBranch(MachineBasicBlock &MBB) const override;
   unsigned InsertBranch(MachineBasicBlock &MBB, MachineBasicBlock *TBB,
-                        MachineBasicBlock *FBB,
-                        const SmallVectorImpl<MachineOperand> &Cond,
+                        MachineBasicBlock *FBB, ArrayRef<MachineOperand> Cond,
                         DebugLoc DL) const override;
 };
 }
diff --git a/lib/Target/BPF/BPFMCInstLower.cpp b/lib/Target/BPF/BPFMCInstLower.cpp
index 5a695f0af36..00bd8d9c090 100644
--- a/lib/Target/BPF/BPFMCInstLower.cpp
+++ b/lib/Target/BPF/BPFMCInstLower.cpp
@@ -33,12 +33,12 @@ BPFMCInstLower::GetGlobalAddressSymbol(const MachineOperand &MO) const {
 MCOperand BPFMCInstLower::LowerSymbolOperand(const MachineOperand &MO,
                                              MCSymbol *Sym) const {
 
-  const MCExpr *Expr = MCSymbolRefExpr::Create(Sym, Ctx);
+  const MCExpr *Expr = MCSymbolRefExpr::create(Sym, Ctx);
 
   if (!MO.isJTI() && MO.getOffset())
     llvm_unreachable("unknown symbol op");
 
-  return MCOperand::CreateExpr(Expr);
+  return MCOperand::createExpr(Expr);
 }
 
 void BPFMCInstLower::Lower(const MachineInstr *MI, MCInst &OutMI) const {
@@ -56,14 +56,14 @@ void BPFMCInstLower::Lower(const MachineInstr *MI, MCInst &OutMI) const {
       // Ignore all implicit register operands.
       if (MO.isImplicit())
         continue;
-      MCOp = MCOperand::CreateReg(MO.getReg());
+      MCOp = MCOperand::createReg(MO.getReg());
       break;
     case MachineOperand::MO_Immediate:
-      MCOp = MCOperand::CreateImm(MO.getImm());
+      MCOp = MCOperand::createImm(MO.getImm());
       break;
     case MachineOperand::MO_MachineBasicBlock:
-      MCOp = MCOperand::CreateExpr(
-          MCSymbolRefExpr::Create(MO.getMBB()->getSymbol(), Ctx));
+      MCOp = MCOperand::createExpr(
+          MCSymbolRefExpr::create(MO.getMBB()->getSymbol(), Ctx));
       break;
     case MachineOperand::MO_RegisterMask:
       continue;
diff --git a/lib/Target/BPF/BPFSubtarget.cpp b/lib/Target/BPF/BPFSubtarget.cpp
index 7f7a2621315..c3a8b1caa63 100644
--- a/lib/Target/BPF/BPFSubtarget.cpp
+++ b/lib/Target/BPF/BPFSubtarget.cpp
@@ -25,7 +25,7 @@ using namespace llvm;
 
 void BPFSubtarget::anchor() {}
 
-BPFSubtarget::BPFSubtarget(const std::string &TT, const std::string &CPU,
+BPFSubtarget::BPFSubtarget(const Triple &TT, const std::string &CPU,
                            const std::string &FS, const TargetMachine &TM)
     : BPFGenSubtargetInfo(TT, CPU, FS), InstrInfo(), FrameLowering(*this),
-      TLInfo(TM, *this), TSInfo(TM.getDataLayout()) {}
+      TLInfo(TM, *this) {}
diff --git a/lib/Target/BPF/BPFSubtarget.h b/lib/Target/BPF/BPFSubtarget.h
index 347cffd82e0..5ad58db7539 100644
--- a/lib/Target/BPF/BPFSubtarget.h
+++ b/lib/Target/BPF/BPFSubtarget.h
@@ -38,8 +38,8 @@ class BPFSubtarget : public BPFGenSubtargetInfo {
 public:
   // This constructor initializes the data members to match that
   // of the specified triple.
-  BPFSubtarget(const std::string &TT, const std::string &CPU,
-               const std::string &FS, const TargetMachine &TM);
+  BPFSubtarget(const Triple &TT, const std::string &CPU, const std::string &FS,
+               const TargetMachine &TM);
 
   // ParseSubtargetFeatures - Parses features string setting specified
   // subtarget options.  Definition of function is auto generated by tblgen.
diff --git a/lib/Target/BPF/BPFTargetMachine.cpp b/lib/Target/BPF/BPFTargetMachine.cpp
index 9487427fef5..06cba2252a2 100644
--- a/lib/Target/BPF/BPFTargetMachine.cpp
+++ b/lib/Target/BPF/BPFTargetMachine.cpp
@@ -23,20 +23,26 @@ using namespace llvm;
 
 extern "C" void LLVMInitializeBPFTarget() {
   // Register the target.
-  RegisterTargetMachine<BPFTargetMachine> X(TheBPFTarget);
+  RegisterTargetMachine<BPFTargetMachine> X(TheBPFleTarget);
+  RegisterTargetMachine<BPFTargetMachine> Y(TheBPFbeTarget);
+  RegisterTargetMachine<BPFTargetMachine> Z(TheBPFTarget);
 }
 
-// DataLayout --> Little-endian, 64-bit pointer/ABI/alignment
-// The stack is always 8 byte aligned
-// On function prologue, the stack is created by decrementing
-// its pointer. Once decremented, all references are done with positive
-// offset from the stack/frame pointer.
-BPFTargetMachine::BPFTargetMachine(const Target &T, StringRef TT, StringRef CPU,
-                                   StringRef FS, const TargetOptions &Options,
+// DataLayout: little or big endian
+static std::string computeDataLayout(const Triple &TT) {
+  if (TT.getArch() == Triple::bpfeb)
+    return "E-m:e-p:64:64-i64:64-n32:64-S128";
+  else
+    return "e-m:e-p:64:64-i64:64-n32:64-S128";
+}
+
+BPFTargetMachine::BPFTargetMachine(const Target &T, const Triple &TT,
+                                   StringRef CPU, StringRef FS,
+                                   const TargetOptions &Options,
                                    Reloc::Model RM, CodeModel::Model CM,
                                    CodeGenOpt::Level OL)
-    : LLVMTargetMachine(T, "e-m:e-p:64:64-i64:64-n32:64-S128", TT, CPU, FS,
-                        Options, RM, CM, OL),
+    : LLVMTargetMachine(T, computeDataLayout(TT), TT, CPU, FS, Options, RM, CM,
+                        OL),
       TLOF(make_unique<TargetLoweringObjectFileELF>()),
       Subtarget(TT, CPU, FS, *this) {
   initAsmInfo();
diff --git a/lib/Target/BPF/BPFTargetMachine.h b/lib/Target/BPF/BPFTargetMachine.h
index 6aeafb99a2a..a0086df2d32 100644
--- a/lib/Target/BPF/BPFTargetMachine.h
+++ b/lib/Target/BPF/BPFTargetMachine.h
@@ -23,8 +23,8 @@ class BPFTargetMachine : public LLVMTargetMachine {
   BPFSubtarget Subtarget;
 
 public:
-  BPFTargetMachine(const Target &T, StringRef TT, StringRef CPU, StringRef FS,
-                   const TargetOptions &Options, Reloc::Model RM,
+  BPFTargetMachine(const Target &T, const Triple &TT, StringRef CPU,
+                   StringRef FS, const TargetOptions &Options, Reloc::Model RM,
                    CodeModel::Model CM, CodeGenOpt::Level OL);
 
   const BPFSubtarget *getSubtargetImpl() const { return &Subtarget; }
diff --git a/lib/Target/BPF/InstPrinter/BPFInstPrinter.cpp b/lib/Target/BPF/InstPrinter/BPFInstPrinter.cpp
index 05f6d825719..552288b819d 100644
--- a/lib/Target/BPF/InstPrinter/BPFInstPrinter.cpp
+++ b/lib/Target/BPF/InstPrinter/BPFInstPrinter.cpp
@@ -33,6 +33,7 @@ void BPFInstPrinter::printInst(const MCInst *MI, raw_ostream &O,
 }
 
 static void printExpr(const MCExpr *Expr, raw_ostream &O) {
+#ifndef NDEBUG
   const MCSymbolRefExpr *SRE;
 
   if (const MCBinaryExpr *BE = dyn_cast<MCBinaryExpr>(Expr))
@@ -44,6 +45,7 @@ static void printExpr(const MCExpr *Expr, raw_ostream &O) {
   MCSymbolRefExpr::VariantKind Kind = SRE->getKind();
 
   assert(Kind == MCSymbolRefExpr::VK_None);
+#endif
   O << *Expr;
 }
 
diff --git a/lib/Target/BPF/LLVMBuild.txt b/lib/Target/BPF/LLVMBuild.txt
index 11578c8ee21..66dbf86fa42 100644
--- a/lib/Target/BPF/LLVMBuild.txt
+++ b/lib/Target/BPF/LLVMBuild.txt
@@ -28,5 +28,15 @@ has_asmprinter = 1
 type = Library
 name = BPFCodeGen
 parent = BPF
-required_libraries = AsmPrinter CodeGen Core MC BPFAsmPrinter BPFDesc BPFInfo SelectionDAG Support Target
+required_libraries =
+ AsmPrinter
+ CodeGen
+ Core
+ MC
+ BPFAsmPrinter
+ BPFDesc
+ BPFInfo
+ SelectionDAG
+ Support
+ Target
 add_to_library_groups = BPF
diff --git a/lib/Target/BPF/MCTargetDesc/BPFAsmBackend.cpp b/lib/Target/BPF/MCTargetDesc/BPFAsmBackend.cpp
index 8393135a2b9..8c358cab62e 100644
--- a/lib/Target/BPF/MCTargetDesc/BPFAsmBackend.cpp
+++ b/lib/Target/BPF/MCTargetDesc/BPFAsmBackend.cpp
@@ -25,7 +25,10 @@ using namespace llvm;
 namespace {
 class BPFAsmBackend : public MCAsmBackend {
 public:
-  BPFAsmBackend() : MCAsmBackend() {}
+  bool IsLittleEndian;
+
+  BPFAsmBackend(bool IsLittleEndian)
+    : MCAsmBackend(), IsLittleEndian(IsLittleEndian) {}
   ~BPFAsmBackend() override {}
 
   void applyFixup(const MCFixup &Fixup, char *Data, unsigned DataSize,
@@ -54,7 +57,7 @@ bool BPFAsmBackend::writeNopData(uint64_t Count, MCObjectWriter *OW) const {
     return false;
 
   for (uint64_t i = 0; i < Count; i += 8)
-    OW->Write64(0x15000000);
+    OW->write64(0x15000000);
 
   return true;
 }
@@ -65,19 +68,39 @@ void BPFAsmBackend::applyFixup(const MCFixup &Fixup, char *Data,
 
   if (Fixup.getKind() == FK_SecRel_4 || Fixup.getKind() == FK_SecRel_8) {
     assert(Value == 0);
-    return;
+  } else if (Fixup.getKind() == FK_Data_4 || Fixup.getKind() == FK_Data_8) {
+    unsigned Size = Fixup.getKind() == FK_Data_4 ? 4 : 8;
+
+    for (unsigned i = 0; i != Size; ++i) {
+      unsigned Idx = IsLittleEndian ? i : Size - i;
+      Data[Fixup.getOffset() + Idx] = uint8_t(Value >> (i * 8));
+    }
+  } else {
+    assert(Fixup.getKind() == FK_PCRel_2);
+    Value = (uint16_t)((Value - 8) / 8);
+    if (IsLittleEndian) {
+      Data[Fixup.getOffset() + 2] = Value & 0xFF;
+      Data[Fixup.getOffset() + 3] = Value >> 8;
+    } else {
+      Data[Fixup.getOffset() + 2] = Value >> 8;
+      Data[Fixup.getOffset() + 3] = Value & 0xFF;
+    }
   }
-  assert(Fixup.getKind() == FK_PCRel_2);
-  *(uint16_t *)&Data[Fixup.getOffset() + 2] = (uint16_t)((Value - 8) / 8);
 }
 
 MCObjectWriter *BPFAsmBackend::createObjectWriter(raw_pwrite_stream &OS) const {
-  return createBPFELFObjectWriter(OS, 0);
+  return createBPFELFObjectWriter(OS, 0, IsLittleEndian);
 }
 }
 
 MCAsmBackend *llvm::createBPFAsmBackend(const Target &T,
-                                        const MCRegisterInfo &MRI, StringRef TT,
-                                        StringRef CPU) {
-  return new BPFAsmBackend();
+                                        const MCRegisterInfo &MRI,
+                                        const Triple &TT, StringRef CPU) {
+  return new BPFAsmBackend(/*IsLittleEndian=*/true);
+}
+
+MCAsmBackend *llvm::createBPFbeAsmBackend(const Target &T,
+                                          const MCRegisterInfo &MRI,
+                                          const Triple &TT, StringRef CPU) {
+  return new BPFAsmBackend(/*IsLittleEndian=*/false);
 }
diff --git a/lib/Target/BPF/MCTargetDesc/BPFELFObjectWriter.cpp b/lib/Target/BPF/MCTargetDesc/BPFELFObjectWriter.cpp
index a5562c1a933..87cdd5eb9da 100644
--- a/lib/Target/BPF/MCTargetDesc/BPFELFObjectWriter.cpp
+++ b/lib/Target/BPF/MCTargetDesc/BPFELFObjectWriter.cpp
@@ -44,10 +44,15 @@ unsigned BPFELFObjectWriter::GetRelocType(const MCValue &Target,
     return ELF::R_X86_64_64;
   case FK_SecRel_4:
     return ELF::R_X86_64_PC32;
+  case FK_Data_8:
+    return IsPCRel ? ELF::R_X86_64_PC64 : ELF::R_X86_64_64;
+  case FK_Data_4:
+    return IsPCRel ? ELF::R_X86_64_PC32 : ELF::R_X86_64_32;
   }
 }
 
-MCObjectWriter *llvm::createBPFELFObjectWriter(raw_pwrite_stream &OS, uint8_t OSABI) {
+MCObjectWriter *llvm::createBPFELFObjectWriter(raw_pwrite_stream &OS,
+                                               uint8_t OSABI, bool IsLittleEndian) {
   MCELFObjectTargetWriter *MOTW = new BPFELFObjectWriter(OSABI);
-  return createELFObjectWriter(MOTW, OS, /*IsLittleEndian=*/true);
+  return createELFObjectWriter(MOTW, OS, IsLittleEndian);
 }
diff --git a/lib/Target/BPF/MCTargetDesc/BPFMCAsmInfo.h b/lib/Target/BPF/MCTargetDesc/BPFMCAsmInfo.h
index ab61ae7ae66..1f440fe8787 100644
--- a/lib/Target/BPF/MCTargetDesc/BPFMCAsmInfo.h
+++ b/lib/Target/BPF/MCTargetDesc/BPFMCAsmInfo.h
@@ -16,19 +16,26 @@
 
 #include "llvm/ADT/StringRef.h"
 #include "llvm/MC/MCAsmInfo.h"
+#include "llvm/ADT/Triple.h"
 
 namespace llvm {
 class Target;
+class Triple;
 
 class BPFMCAsmInfo : public MCAsmInfo {
 public:
-  explicit BPFMCAsmInfo(StringRef TT) {
+  explicit BPFMCAsmInfo(const Triple &TT) {
+    if (TT.getArch() == Triple::bpfeb)
+      IsLittleEndian = false;
+
     PrivateGlobalPrefix = ".L";
     WeakRefDirective = "\t.weak\t";
 
     UsesELFSectionDirectiveForBSS = true;
     HasSingleParameterDotFile = false;
     HasDotTypeDotSizeDirective = false;
+
+    SupportsDebugInformation = true;
   }
 };
 }
diff --git a/lib/Target/BPF/MCTargetDesc/BPFMCCodeEmitter.cpp b/lib/Target/BPF/MCTargetDesc/BPFMCCodeEmitter.cpp
index 32d2ef56351..dc4ede30f19 100644
--- a/lib/Target/BPF/MCTargetDesc/BPFMCCodeEmitter.cpp
+++ b/lib/Target/BPF/MCTargetDesc/BPFMCCodeEmitter.cpp
@@ -30,9 +30,11 @@ class BPFMCCodeEmitter : public MCCodeEmitter {
   BPFMCCodeEmitter(const BPFMCCodeEmitter &) = delete;
   void operator=(const BPFMCCodeEmitter &) = delete;
   const MCRegisterInfo &MRI;
+  bool IsLittleEndian;
 
 public:
-  BPFMCCodeEmitter(const MCRegisterInfo &mri) : MRI(mri) {}
+  BPFMCCodeEmitter(const MCRegisterInfo &mri, bool IsLittleEndian)
+    : MRI(mri), IsLittleEndian(IsLittleEndian) {}
 
   ~BPFMCCodeEmitter() {}
 
@@ -52,7 +54,7 @@ class BPFMCCodeEmitter : public MCCodeEmitter {
                             SmallVectorImpl<MCFixup> &Fixups,
                             const MCSubtargetInfo &STI) const;
 
-  void EncodeInstruction(const MCInst &MI, raw_ostream &OS,
+  void encodeInstruction(const MCInst &MI, raw_ostream &OS,
                          SmallVectorImpl<MCFixup> &Fixups,
                          const MCSubtargetInfo &STI) const override;
 };
@@ -61,7 +63,13 @@ class BPFMCCodeEmitter : public MCCodeEmitter {
 MCCodeEmitter *llvm::createBPFMCCodeEmitter(const MCInstrInfo &MCII,
                                             const MCRegisterInfo &MRI,
                                             MCContext &Ctx) {
-  return new BPFMCCodeEmitter(MRI);
+  return new BPFMCCodeEmitter(MRI, true);
+}
+
+MCCodeEmitter *llvm::createBPFbeMCCodeEmitter(const MCInstrInfo &MCII,
+                                              const MCRegisterInfo &MRI,
+                                              MCContext &Ctx) {
+  return new BPFMCCodeEmitter(MRI, false);
 }
 
 unsigned BPFMCCodeEmitter::getMachineOpValue(const MCInst &MI,
@@ -81,69 +89,63 @@ unsigned BPFMCCodeEmitter::getMachineOpValue(const MCInst &MI,
 
   if (MI.getOpcode() == BPF::JAL)
     // func call name
-    Fixups.push_back(MCFixup::Create(0, Expr, FK_SecRel_4));
+    Fixups.push_back(MCFixup::create(0, Expr, FK_SecRel_4));
   else if (MI.getOpcode() == BPF::LD_imm64)
-    Fixups.push_back(MCFixup::Create(0, Expr, FK_SecRel_8));
+    Fixups.push_back(MCFixup::create(0, Expr, FK_SecRel_8));
   else
     // bb label
-    Fixups.push_back(MCFixup::Create(0, Expr, FK_PCRel_2));
+    Fixups.push_back(MCFixup::create(0, Expr, FK_PCRel_2));
 
   return 0;
 }
 
-// Emit one byte through output stream
-void EmitByte(unsigned char C, unsigned &CurByte, raw_ostream &OS) {
-  OS << (char)C;
-  ++CurByte;
-}
-
-// Emit a series of bytes (little endian)
-void EmitLEConstant(uint64_t Val, unsigned Size, unsigned &CurByte,
-                    raw_ostream &OS) {
-  assert(Size <= 8 && "size too big in emit constant");
-
-  for (unsigned i = 0; i != Size; ++i) {
-    EmitByte(Val & 255, CurByte, OS);
-    Val >>= 8;
-  }
-}
-
-// Emit a series of bytes (big endian)
-void EmitBEConstant(uint64_t Val, unsigned Size, unsigned &CurByte,
-                    raw_ostream &OS) {
-  assert(Size <= 8 && "size too big in emit constant");
-
-  for (int i = (Size - 1) * 8; i >= 0; i -= 8)
-    EmitByte((Val >> i) & 255, CurByte, OS);
+static uint8_t SwapBits(uint8_t Val)
+{
+  return (Val & 0x0F) << 4 | (Val & 0xF0) >> 4;
 }
 
-void BPFMCCodeEmitter::EncodeInstruction(const MCInst &MI, raw_ostream &OS,
+void BPFMCCodeEmitter::encodeInstruction(const MCInst &MI, raw_ostream &OS,
                                          SmallVectorImpl<MCFixup> &Fixups,
                                          const MCSubtargetInfo &STI) const {
   unsigned Opcode = MI.getOpcode();
-  // Keep track of the current byte being emitted
-  unsigned CurByte = 0;
+  support::endian::Writer<support::little> LE(OS);
+  support::endian::Writer<support::big> BE(OS);
 
   if (Opcode == BPF::LD_imm64 || Opcode == BPF::LD_pseudo) {
     uint64_t Value = getBinaryCodeForInstr(MI, Fixups, STI);
-    EmitByte(Value >> 56, CurByte, OS);
-    EmitByte(((Value >> 48) & 0xff), CurByte, OS);
-    EmitLEConstant(0, 2, CurByte, OS);
-    EmitLEConstant(Value & 0xffffFFFF, 4, CurByte, OS);
+    LE.write<uint8_t>(Value >> 56);
+    if (IsLittleEndian)
+      LE.write<uint8_t>((Value >> 48) & 0xff);
+    else
+      LE.write<uint8_t>(SwapBits((Value >> 48) & 0xff));
+    LE.write<uint16_t>(0);
+    if (IsLittleEndian)
+      LE.write<uint32_t>(Value & 0xffffFFFF);
+    else
+      BE.write<uint32_t>(Value & 0xffffFFFF);
 
     const MCOperand &MO = MI.getOperand(1);
     uint64_t Imm = MO.isImm() ? MO.getImm() : 0;
-    EmitByte(0, CurByte, OS);
-    EmitByte(0, CurByte, OS);
-    EmitLEConstant(0, 2, CurByte, OS);
-    EmitLEConstant(Imm >> 32, 4, CurByte, OS);
+    LE.write<uint8_t>(0);
+    LE.write<uint8_t>(0);
+    LE.write<uint16_t>(0);
+    if (IsLittleEndian)
+      LE.write<uint32_t>(Imm >> 32);
+    else
+      BE.write<uint32_t>(Imm >> 32);
   } else {
     // Get instruction encoding and emit it
     uint64_t Value = getBinaryCodeForInstr(MI, Fixups, STI);
-    EmitByte(Value >> 56, CurByte, OS);
-    EmitByte((Value >> 48) & 0xff, CurByte, OS);
-    EmitLEConstant((Value >> 32) & 0xffff, 2, CurByte, OS);
-    EmitLEConstant(Value & 0xffffFFFF, 4, CurByte, OS);
+    LE.write<uint8_t>(Value >> 56);
+    if (IsLittleEndian) {
+      LE.write<uint8_t>((Value >> 48) & 0xff);
+      LE.write<uint16_t>((Value >> 32) & 0xffff);
+      LE.write<uint32_t>(Value & 0xffffFFFF);
+    } else {
+      LE.write<uint8_t>(SwapBits((Value >> 48) & 0xff));
+      BE.write<uint16_t>((Value >> 32) & 0xffff);
+      BE.write<uint32_t>(Value & 0xffffFFFF);
+    }
   }
 }
 
diff --git a/lib/Target/BPF/MCTargetDesc/BPFMCTargetDesc.cpp b/lib/Target/BPF/MCTargetDesc/BPFMCTargetDesc.cpp
index 95f0b02c204..840570ebc39 100644
--- a/lib/Target/BPF/MCTargetDesc/BPFMCTargetDesc.cpp
+++ b/lib/Target/BPF/MCTargetDesc/BPFMCTargetDesc.cpp
@@ -40,24 +40,22 @@ static MCInstrInfo *createBPFMCInstrInfo() {
   return X;
 }
 
-static MCRegisterInfo *createBPFMCRegisterInfo(StringRef TT) {
+static MCRegisterInfo *createBPFMCRegisterInfo(const Triple &TT) {
   MCRegisterInfo *X = new MCRegisterInfo();
   InitBPFMCRegisterInfo(X, BPF::R11 /* RAReg doesn't exist */);
   return X;
 }
 
-static MCSubtargetInfo *createBPFMCSubtargetInfo(StringRef TT, StringRef CPU,
-                                                 StringRef FS) {
-  MCSubtargetInfo *X = new MCSubtargetInfo();
-  InitBPFMCSubtargetInfo(X, TT, CPU, FS);
-  return X;
+static MCSubtargetInfo *createBPFMCSubtargetInfo(const Triple &TT,
+                                                 StringRef CPU, StringRef FS) {
+  return createBPFMCSubtargetInfoImpl(TT, CPU, FS);
 }
 
-static MCCodeGenInfo *createBPFMCCodeGenInfo(StringRef TT, Reloc::Model RM,
+static MCCodeGenInfo *createBPFMCCodeGenInfo(const Triple &TT, Reloc::Model RM,
                                              CodeModel::Model CM,
                                              CodeGenOpt::Level OL) {
   MCCodeGenInfo *X = new MCCodeGenInfo();
-  X->InitMCCodeGenInfo(RM, CM, OL);
+  X->initMCCodeGenInfo(RM, CM, OL);
   return X;
 }
 
@@ -79,32 +77,43 @@ static MCInstPrinter *createBPFMCInstPrinter(const Triple &T,
 }
 
 extern "C" void LLVMInitializeBPFTargetMC() {
-  // Register the MC asm info.
-  RegisterMCAsmInfo<BPFMCAsmInfo> X(TheBPFTarget);
+  for (Target *T : {&TheBPFleTarget, &TheBPFbeTarget, &TheBPFTarget}) {
+    // Register the MC asm info.
+    RegisterMCAsmInfo<BPFMCAsmInfo> X(*T);
 
-  // Register the MC codegen info.
-  TargetRegistry::RegisterMCCodeGenInfo(TheBPFTarget, createBPFMCCodeGenInfo);
+    // Register the MC codegen info.
+    TargetRegistry::RegisterMCCodeGenInfo(*T, createBPFMCCodeGenInfo);
 
-  // Register the MC instruction info.
-  TargetRegistry::RegisterMCInstrInfo(TheBPFTarget, createBPFMCInstrInfo);
+    // Register the MC instruction info.
+    TargetRegistry::RegisterMCInstrInfo(*T, createBPFMCInstrInfo);
 
-  // Register the MC register info.
-  TargetRegistry::RegisterMCRegInfo(TheBPFTarget, createBPFMCRegisterInfo);
+    // Register the MC register info.
+    TargetRegistry::RegisterMCRegInfo(*T, createBPFMCRegisterInfo);
 
-  // Register the MC subtarget info.
-  TargetRegistry::RegisterMCSubtargetInfo(TheBPFTarget,
-                                          createBPFMCSubtargetInfo);
+    // Register the MC subtarget info.
+    TargetRegistry::RegisterMCSubtargetInfo(*T,
+                                            createBPFMCSubtargetInfo);
 
-  // Register the MC code emitter
-  TargetRegistry::RegisterMCCodeEmitter(TheBPFTarget,
-                                        llvm::createBPFMCCodeEmitter);
+    // Register the object streamer
+    TargetRegistry::RegisterELFStreamer(*T, createBPFMCStreamer);
 
-  // Register the ASM Backend
-  TargetRegistry::RegisterMCAsmBackend(TheBPFTarget, createBPFAsmBackend);
+    // Register the MCInstPrinter.
+    TargetRegistry::RegisterMCInstPrinter(*T, createBPFMCInstPrinter);
+  }
 
-  // Register the object streamer
-  TargetRegistry::RegisterELFStreamer(TheBPFTarget, createBPFMCStreamer);
+  // Register the MC code emitter
+  TargetRegistry::RegisterMCCodeEmitter(TheBPFleTarget, createBPFMCCodeEmitter);
+  TargetRegistry::RegisterMCCodeEmitter(TheBPFbeTarget, createBPFbeMCCodeEmitter);
 
-  // Register the MCInstPrinter.
-  TargetRegistry::RegisterMCInstPrinter(TheBPFTarget, createBPFMCInstPrinter);
+  // Register the ASM Backend
+  TargetRegistry::RegisterMCAsmBackend(TheBPFleTarget, createBPFAsmBackend);
+  TargetRegistry::RegisterMCAsmBackend(TheBPFbeTarget, createBPFbeAsmBackend);
+
+  if (sys::IsLittleEndianHost) {
+    TargetRegistry::RegisterMCCodeEmitter(TheBPFTarget, createBPFMCCodeEmitter);
+    TargetRegistry::RegisterMCAsmBackend(TheBPFTarget, createBPFAsmBackend);
+  } else {
+    TargetRegistry::RegisterMCCodeEmitter(TheBPFTarget, createBPFbeMCCodeEmitter);
+    TargetRegistry::RegisterMCAsmBackend(TheBPFTarget, createBPFbeAsmBackend);
+  }
 }
diff --git a/lib/Target/BPF/MCTargetDesc/BPFMCTargetDesc.h b/lib/Target/BPF/MCTargetDesc/BPFMCTargetDesc.h
index ce08b7cf76e..e2ae6526edc 100644
--- a/lib/Target/BPF/MCTargetDesc/BPFMCTargetDesc.h
+++ b/lib/Target/BPF/MCTargetDesc/BPFMCTargetDesc.h
@@ -25,21 +25,30 @@ class MCInstrInfo;
 class MCObjectWriter;
 class MCRegisterInfo;
 class MCSubtargetInfo;
-class Target;
 class StringRef;
+class Target;
+class Triple;
 class raw_ostream;
 class raw_pwrite_stream;
 
+extern Target TheBPFleTarget;
+extern Target TheBPFbeTarget;
 extern Target TheBPFTarget;
 
 MCCodeEmitter *createBPFMCCodeEmitter(const MCInstrInfo &MCII,
                                       const MCRegisterInfo &MRI,
                                       MCContext &Ctx);
+MCCodeEmitter *createBPFbeMCCodeEmitter(const MCInstrInfo &MCII,
+                                        const MCRegisterInfo &MRI,
+                                        MCContext &Ctx);
 
 MCAsmBackend *createBPFAsmBackend(const Target &T, const MCRegisterInfo &MRI,
-                                  StringRef TT, StringRef CPU);
+                                  const Triple &TT, StringRef CPU);
+MCAsmBackend *createBPFbeAsmBackend(const Target &T, const MCRegisterInfo &MRI,
+                                    const Triple &TT, StringRef CPU);
 
-MCObjectWriter *createBPFELFObjectWriter(raw_pwrite_stream &OS, uint8_t OSABI);
+MCObjectWriter *createBPFELFObjectWriter(raw_pwrite_stream &OS,
+                                         uint8_t OSABI, bool IsLittleEndian);
 }
 
 // Defines symbolic names for BPF registers.  This defines a mapping from
diff --git a/lib/Target/BPF/TargetInfo/BPFTargetInfo.cpp b/lib/Target/BPF/TargetInfo/BPFTargetInfo.cpp
index 87716e6775c..a16dbae867b 100644
--- a/lib/Target/BPF/TargetInfo/BPFTargetInfo.cpp
+++ b/lib/Target/BPF/TargetInfo/BPFTargetInfo.cpp
@@ -11,8 +11,18 @@
 #include "llvm/Support/TargetRegistry.h"
 using namespace llvm;
 
-Target llvm::TheBPFTarget;
+namespace llvm {
+Target TheBPFleTarget;
+Target TheBPFbeTarget;
+Target TheBPFTarget;
+}
 
 extern "C" void LLVMInitializeBPFTargetInfo() {
-  RegisterTarget<Triple::bpf, /*HasJIT=*/true> X(TheBPFTarget, "bpf", "BPF");
+  TargetRegistry::RegisterTarget(TheBPFTarget, "bpf",
+                                 "BPF (host endian)",
+                                 [](Triple::ArchType) { return false; }, true);
+  RegisterTarget<Triple::bpfel, /*HasJIT=*/true> X(
+      TheBPFleTarget, "bpfel", "BPF (little endian)");
+  RegisterTarget<Triple::bpfeb, /*HasJIT=*/true> Y(
+      TheBPFbeTarget, "bpfeb", "BPF (big endian)");
 }
diff --git a/lib/Target/CMakeLists.txt b/lib/Target/CMakeLists.txt
index 1805437b12f..e6d0199952f 100644
--- a/lib/Target/CMakeLists.txt
+++ b/lib/Target/CMakeLists.txt
@@ -6,6 +6,7 @@ add_llvm_library(LLVMTarget
   TargetLoweringObjectFile.cpp
   TargetMachine.cpp
   TargetMachineC.cpp
+  TargetRecip.cpp
   TargetSubtargetInfo.cpp
 
   ADDITIONAL_HEADER_DIRS
diff --git a/lib/Target/CppBackend/CPPBackend.cpp b/lib/Target/CppBackend/CPPBackend.cpp
index f1a7127e8fd..272688edb8a 100644
--- a/lib/Target/CppBackend/CPPBackend.cpp
+++ b/lib/Target/CppBackend/CPPBackend.cpp
@@ -513,6 +513,7 @@ void CppWriter::printAttributes(const AttributeSet &PAL,
       HANDLE_ATTR(StackProtect);
       HANDLE_ATTR(StackProtectReq);
       HANDLE_ATTR(StackProtectStrong);
+      HANDLE_ATTR(SafeStack);
       HANDLE_ATTR(NoCapture);
       HANDLE_ATTR(NoRedZone);
       HANDLE_ATTR(NoImplicitFloat);
@@ -645,8 +646,7 @@ void CppWriter::printType(Type* Ty) {
     if (DefinedTypes.find(Ty) == DefinedTypes.end()) {
       std::string elemName(getCppName(ET));
       Out << "ArrayType* " << typeName << " = ArrayType::get("
-          << elemName
-          << ", " << utostr(AT->getNumElements()) << ");";
+          << elemName << ", " << AT->getNumElements() << ");";
       nl(Out);
     }
     break;
@@ -658,8 +658,7 @@ void CppWriter::printType(Type* Ty) {
     if (DefinedTypes.find(Ty) == DefinedTypes.end()) {
       std::string elemName(getCppName(ET));
       Out << "PointerType* " << typeName << " = PointerType::get("
-          << elemName
-          << ", " << utostr(PT->getAddressSpace()) << ");";
+          << elemName << ", " << PT->getAddressSpace() << ");";
       nl(Out);
     }
     break;
@@ -671,8 +670,7 @@ void CppWriter::printType(Type* Ty) {
     if (DefinedTypes.find(Ty) == DefinedTypes.end()) {
       std::string elemName(getCppName(ET));
       Out << "VectorType* " << typeName << " = VectorType::get("
-          << elemName
-          << ", " << utostr(PT->getNumElements()) << ");";
+          << elemName << ", " << PT->getNumElements() << ");";
       nl(Out);
     }
     break;
@@ -1029,7 +1027,7 @@ void CppWriter::printVariableHead(const GlobalVariable *GV) {
   }
   if (GV->getAlignment()) {
     printCppName(GV);
-    Out << "->setAlignment(" << utostr(GV->getAlignment()) << ");";
+    Out << "->setAlignment(" << GV->getAlignment() << ");";
     nl(Out);
   }
   if (GV->getVisibility() != GlobalValue::DefaultVisibility) {
@@ -1680,9 +1678,8 @@ void CppWriter::printFunctionUses(const Function* F) {
                 consts.insert(GVar->getInitializer());
         } else if (Constant* C = dyn_cast<Constant>(operand)) {
           consts.insert(C);
-          for (unsigned j = 0; j < C->getNumOperands(); ++j) {
+          for (Value* operand : C->operands()) {
             // If the operand references a GVal or Constant, make a note of it
-            Value* operand = C->getOperand(j);
             printType(operand->getType());
             if (GlobalValue* GV = dyn_cast<GlobalValue>(operand)) {
               gvs.insert(GV);
@@ -2151,7 +2148,8 @@ char CppWriter::ID = 0;
 
 bool CPPTargetMachine::addPassesToEmitFile(
     PassManagerBase &PM, raw_pwrite_stream &o, CodeGenFileType FileType,
-    bool DisableVerify, AnalysisID StartAfter, AnalysisID StopAfter) {
+    bool DisableVerify, AnalysisID StartBefore, AnalysisID StartAfter,
+    AnalysisID StopAfter, MachineFunctionInitializer *MFInitializer) {
   if (FileType != TargetMachine::CGFT_AssemblyFile)
     return true;
   auto FOut = llvm::make_unique<formatted_raw_ostream>(o);
diff --git a/lib/Target/CppBackend/CPPTargetMachine.h b/lib/Target/CppBackend/CPPTargetMachine.h
index 02d705e2d8f..00e402feffb 100644
--- a/lib/Target/CppBackend/CPPTargetMachine.h
+++ b/lib/Target/CppBackend/CPPTargetMachine.h
@@ -23,16 +23,17 @@ namespace llvm {
 class formatted_raw_ostream;
 
 struct CPPTargetMachine : public TargetMachine {
-  CPPTargetMachine(const Target &T, StringRef TT, StringRef CPU, StringRef FS,
-                   const TargetOptions &Options, Reloc::Model RM,
+  CPPTargetMachine(const Target &T, const Triple &TT, StringRef CPU,
+                   StringRef FS, const TargetOptions &Options, Reloc::Model RM,
                    CodeModel::Model CM, CodeGenOpt::Level OL)
       : TargetMachine(T, "", TT, CPU, FS, Options) {}
 
 public:
   bool addPassesToEmitFile(PassManagerBase &PM, raw_pwrite_stream &Out,
                            CodeGenFileType FileType, bool DisableVerify,
-                           AnalysisID StartAfter,
-                           AnalysisID StopAfter) override;
+                           AnalysisID StartBefore, AnalysisID StartAfter,
+                           AnalysisID StopAfter,
+                           MachineFunctionInitializer *MFInitializer) override;
 };
 
 extern Target TheCppBackendTarget;
diff --git a/lib/Target/CppBackend/TargetInfo/CppBackendTargetInfo.cpp b/lib/Target/CppBackend/TargetInfo/CppBackendTargetInfo.cpp
index 096dc7350db..f88d82228ca 100644
--- a/lib/Target/CppBackend/TargetInfo/CppBackendTargetInfo.cpp
+++ b/lib/Target/CppBackend/TargetInfo/CppBackendTargetInfo.cpp
@@ -20,8 +20,8 @@ static bool CppBackend_TripleMatchQuality(Triple::ArchType Arch) {
   return false;
 }
 
-extern "C" void LLVMInitializeCppBackendTargetInfo() { 
-  TargetRegistry::RegisterTarget(TheCppBackendTarget, "cpp",    
+extern "C" void LLVMInitializeCppBackendTargetInfo() {
+  TargetRegistry::RegisterTarget(TheCppBackendTarget, "cpp",
                                   "C++ backend",
                                   &CppBackend_TripleMatchQuality);
 }
diff --git a/lib/Target/Hexagon/BitTracker.cpp b/lib/Target/Hexagon/BitTracker.cpp
new file mode 100644
index 00000000000..111d3b4d01f
--- /dev/null
+++ b/lib/Target/Hexagon/BitTracker.cpp
@@ -0,0 +1,1127 @@
+//===--- BitTracker.cpp ---------------------------------------------------===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+
+// SSA-based bit propagation.
+//
+// The purpose of this code is, for a given virtual register, to provide
+// information about the value of each bit in the register. The values
+// of bits are represented by the class BitValue, and take one of four
+// cases: 0, 1, "ref" and "bottom". The 0 and 1 are rather clear, the
+// "ref" value means that the bit is a copy of another bit (which itself
+// cannot be a copy of yet another bit---such chains are not allowed).
+// A "ref" value is associated with a BitRef structure, which indicates
+// which virtual register, and which bit in that register is the origin
+// of the value. For example, given an instruction
+//   vreg2 = ASL vreg1, 1
+// assuming that nothing is known about bits of vreg1, bit 1 of vreg2
+// will be a "ref" to (vreg1, 0). If there is a subsequent instruction
+//   vreg3 = ASL vreg2, 2
+// then bit 3 of vreg3 will be a "ref" to (vreg1, 0) as well.
+// The "bottom" case means that the bit's value cannot be determined,
+// and that this virtual register actually defines it. The "bottom" case
+// is discussed in detail in BitTracker.h. In fact, "bottom" is a "ref
+// to self", so for the vreg1 above, the bit 0 of it will be a "ref" to
+// (vreg1, 0), bit 1 will be a "ref" to (vreg1, 1), etc.
+//
+// The tracker implements the Wegman-Zadeck algorithm, originally developed
+// for SSA-based constant propagation. Each register is represented as
+// a sequence of bits, with the convention that bit 0 is the least signi-
+// ficant bit. Each bit is propagated individually. The class RegisterCell
+// implements the register's representation, and is also the subject of
+// the lattice operations in the tracker.
+//
+// The intended usage of the bit tracker is to create a target-specific
+// machine instruction evaluator, pass the evaluator to the BitTracker
+// object, and run the tracker. The tracker will then collect the bit
+// value information for a given machine function. After that, it can be
+// queried for the cells for each virtual register.
+// Sample code:
+//   const TargetSpecificEvaluator TSE(TRI, MRI);
+//   BitTracker BT(TSE, MF);
+//   BT.run();
+//   ...
+//   unsigned Reg = interestingRegister();
+//   RegisterCell RC = BT.get(Reg);
+//   if (RC[3].is(1))
+//      Reg0bit3 = 1;
+//
+// The code below is intended to be fully target-independent.
+
+#include "llvm/CodeGen/MachineBasicBlock.h"
+#include "llvm/CodeGen/MachineFunction.h"
+#include "llvm/CodeGen/MachineInstr.h"
+#include "llvm/CodeGen/MachineRegisterInfo.h"
+#include "llvm/IR/Constants.h"
+#include "llvm/Support/Debug.h"
+#include "llvm/Support/raw_ostream.h"
+#include "llvm/Target/TargetRegisterInfo.h"
+
+#include "BitTracker.h"
+
+using namespace llvm;
+
+typedef BitTracker BT;
+
+namespace {
+  // Local trickery to pretty print a register (without the whole "%vreg"
+  // business).
+  struct printv {
+    printv(unsigned r) : R(r) {}
+    unsigned R;
+  };
+  raw_ostream &operator<< (raw_ostream &OS, const printv &PV) {
+    if (PV.R)
+      OS << 'v' << TargetRegisterInfo::virtReg2Index(PV.R);
+    else
+      OS << 's';
+    return OS;
+  }
+}
+
+raw_ostream &llvm::operator<<(raw_ostream &OS, const BT::BitValue &BV) {
+  switch (BV.Type) {
+    case BT::BitValue::Top:
+      OS << 'T';
+      break;
+    case BT::BitValue::Zero:
+      OS << '0';
+      break;
+    case BT::BitValue::One:
+      OS << '1';
+      break;
+    case BT::BitValue::Ref:
+      OS << printv(BV.RefI.Reg) << '[' << BV.RefI.Pos << ']';
+      break;
+  }
+  return OS;
+}
+
+raw_ostream &llvm::operator<<(raw_ostream &OS, const BT::RegisterCell &RC) {
+  unsigned n = RC.Bits.size();
+  OS << "{ w:" << n;
+  // Instead of printing each bit value individually, try to group them
+  // into logical segments, such as sequences of 0 or 1 bits or references
+  // to consecutive bits (e.g. "bits 3-5 are same as bits 7-9 of reg xyz").
+  // "Start" will be the index of the beginning of the most recent segment.
+  unsigned Start = 0;
+  bool SeqRef = false;    // A sequence of refs to consecutive bits.
+  bool ConstRef = false;  // A sequence of refs to the same bit.
+
+  for (unsigned i = 1, n = RC.Bits.size(); i < n; ++i) {
+    const BT::BitValue &V = RC[i];
+    const BT::BitValue &SV = RC[Start];
+    bool IsRef = (V.Type == BT::BitValue::Ref);
+    // If the current value is the same as Start, skip to the next one.
+    if (!IsRef && V == SV)
+      continue;
+    if (IsRef && SV.Type == BT::BitValue::Ref && V.RefI.Reg == SV.RefI.Reg) {
+      if (Start+1 == i) {
+        SeqRef = (V.RefI.Pos == SV.RefI.Pos+1);
+        ConstRef = (V.RefI.Pos == SV.RefI.Pos);
+      }
+      if (SeqRef && V.RefI.Pos == SV.RefI.Pos+(i-Start))
+        continue;
+      if (ConstRef && V.RefI.Pos == SV.RefI.Pos)
+        continue;
+    }
+
+    // The current value is different. Print the previous one and reset
+    // the Start.
+    OS << " [" << Start;
+    unsigned Count = i - Start;
+    if (Count == 1) {
+      OS << "]:" << SV;
+    } else {
+      OS << '-' << i-1 << "]:";
+      if (SV.Type == BT::BitValue::Ref && SeqRef)
+        OS << printv(SV.RefI.Reg) << '[' << SV.RefI.Pos << '-'
+           << SV.RefI.Pos+(Count-1) << ']';
+      else
+        OS << SV;
+    }
+    Start = i;
+    SeqRef = ConstRef = false;
+  }
+
+  OS << " [" << Start;
+  unsigned Count = n - Start;
+  if (n-Start == 1) {
+    OS << "]:" << RC[Start];
+  } else {
+    OS << '-' << n-1 << "]:";
+    const BT::BitValue &SV = RC[Start];
+    if (SV.Type == BT::BitValue::Ref && SeqRef)
+      OS << printv(SV.RefI.Reg) << '[' << SV.RefI.Pos << '-'
+         << SV.RefI.Pos+(Count-1) << ']';
+    else
+      OS << SV;
+  }
+  OS << " }";
+
+  return OS;
+}
+
+BitTracker::BitTracker(const MachineEvaluator &E, MachineFunction &F)
+    : Trace(false), ME(E), MF(F), MRI(F.getRegInfo()), Map(*new CellMapType) {}
+
+BitTracker::~BitTracker() {
+  delete &Map;
+}
+
+
+// If we were allowed to update a cell for a part of a register, the meet
+// operation would need to be parametrized by the register number and the
+// exact part of the register, so that the computer BitRefs correspond to
+// the actual bits of the "self" register.
+// While this cannot happen in the current implementation, I'm not sure
+// if this should be ruled out in the future.
+bool BT::RegisterCell::meet(const RegisterCell &RC, unsigned SelfR) {
+  // An example when "meet" can be invoked with SelfR == 0 is a phi node
+  // with a physical register as an operand.
+  assert(SelfR == 0 || TargetRegisterInfo::isVirtualRegister(SelfR));
+  bool Changed = false;
+  for (uint16_t i = 0, n = Bits.size(); i < n; ++i) {
+    const BitValue &RCV = RC[i];
+    Changed |= Bits[i].meet(RCV, BitRef(SelfR, i));
+  }
+  return Changed;
+}
+
+
+// Insert the entire cell RC into the current cell at position given by M.
+BT::RegisterCell &BT::RegisterCell::insert(const BT::RegisterCell &RC,
+      const BitMask &M) {
+  uint16_t B = M.first(), E = M.last(), W = width();
+  // Sanity: M must be a valid mask for *this.
+  assert(B < W && E < W);
+  // Sanity: the masked part of *this must have the same number of bits
+  // as the source.
+  assert(B > E || E-B+1 == RC.width());      // B <= E  =>  E-B+1 = |RC|.
+  assert(B <= E || E+(W-B)+1 == RC.width()); // E < B   =>  E+(W-B)+1 = |RC|.
+  if (B <= E) {
+    for (uint16_t i = 0; i <= E-B; ++i)
+      Bits[i+B] = RC[i];
+  } else {
+    for (uint16_t i = 0; i < W-B; ++i)
+      Bits[i+B] = RC[i];
+    for (uint16_t i = 0; i <= E; ++i)
+      Bits[i] = RC[i+(W-B)];
+  }
+  return *this;
+}
+
+
+BT::RegisterCell BT::RegisterCell::extract(const BitMask &M) const {
+  uint16_t B = M.first(), E = M.last(), W = width();
+  assert(B < W && E < W);
+  if (B <= E) {
+    RegisterCell RC(E-B+1);
+    for (uint16_t i = B; i <= E; ++i)
+      RC.Bits[i-B] = Bits[i];
+    return RC;
+  }
+
+  RegisterCell RC(E+(W-B)+1);
+  for (uint16_t i = 0; i < W-B; ++i)
+    RC.Bits[i] = Bits[i+B];
+  for (uint16_t i = 0; i <= E; ++i)
+    RC.Bits[i+(W-B)] = Bits[i];
+  return RC;
+}
+
+
+BT::RegisterCell &BT::RegisterCell::rol(uint16_t Sh) {
+  // Rotate left (i.e. towards increasing bit indices).
+  // Swap the two parts:  [0..W-Sh-1] [W-Sh..W-1]
+  uint16_t W = width();
+  Sh = Sh % W;
+  if (Sh == 0)
+    return *this;
+
+  RegisterCell Tmp(W-Sh);
+  // Tmp = [0..W-Sh-1].
+  for (uint16_t i = 0; i < W-Sh; ++i)
+    Tmp[i] = Bits[i];
+  // Shift [W-Sh..W-1] to [0..Sh-1].
+  for (uint16_t i = 0; i < Sh; ++i)
+    Bits[i] = Bits[W-Sh+i];
+  // Copy Tmp to [Sh..W-1].
+  for (uint16_t i = 0; i < W-Sh; ++i)
+    Bits[i+Sh] = Tmp.Bits[i];
+  return *this;
+}
+
+
+BT::RegisterCell &BT::RegisterCell::fill(uint16_t B, uint16_t E,
+      const BitValue &V) {
+  assert(B <= E);
+  while (B < E)
+    Bits[B++] = V;
+  return *this;
+}
+
+
+BT::RegisterCell &BT::RegisterCell::cat(const RegisterCell &RC) {
+  // Append the cell given as the argument to the "this" cell.
+  // Bit 0 of RC becomes bit W of the result, where W is this->width().
+  uint16_t W = width(), WRC = RC.width();
+  Bits.resize(W+WRC);
+  for (uint16_t i = 0; i < WRC; ++i)
+    Bits[i+W] = RC.Bits[i];
+  return *this;
+}
+
+
+uint16_t BT::RegisterCell::ct(bool B) const {
+  uint16_t W = width();
+  uint16_t C = 0;
+  BitValue V = B;
+  while (C < W && Bits[C] == V)
+    C++;
+  return C;
+}
+
+
+uint16_t BT::RegisterCell::cl(bool B) const {
+  uint16_t W = width();
+  uint16_t C = 0;
+  BitValue V = B;
+  while (C < W && Bits[W-(C+1)] == V)
+    C++;
+  return C;
+}
+
+
+bool BT::RegisterCell::operator== (const RegisterCell &RC) const {
+  uint16_t W = Bits.size();
+  if (RC.Bits.size() != W)
+    return false;
+  for (uint16_t i = 0; i < W; ++i)
+    if (Bits[i] != RC[i])
+      return false;
+  return true;
+}
+
+
+uint16_t BT::MachineEvaluator::getRegBitWidth(const RegisterRef &RR) const {
+  // The general problem is with finding a register class that corresponds
+  // to a given reference reg:sub. There can be several such classes, and
+  // since we only care about the register size, it does not matter which
+  // such class we would find.
+  // The easiest way to accomplish what we want is to
+  // 1. find a physical register PhysR from the same class as RR.Reg,
+  // 2. find a physical register PhysS that corresponds to PhysR:RR.Sub,
+  // 3. find a register class that contains PhysS.
+  unsigned PhysR;
+  if (TargetRegisterInfo::isVirtualRegister(RR.Reg)) {
+    const TargetRegisterClass *VC = MRI.getRegClass(RR.Reg);
+    assert(VC->begin() != VC->end() && "Empty register class");
+    PhysR = *VC->begin();
+  } else {
+    assert(TargetRegisterInfo::isPhysicalRegister(RR.Reg));
+    PhysR = RR.Reg;
+  }
+
+  unsigned PhysS = (RR.Sub == 0) ? PhysR : TRI.getSubReg(PhysR, RR.Sub);
+  const TargetRegisterClass *RC = TRI.getMinimalPhysRegClass(PhysS);
+  uint16_t BW = RC->getSize()*8;
+  return BW;
+}
+
+
+BT::RegisterCell BT::MachineEvaluator::getCell(const RegisterRef &RR,
+      const CellMapType &M) const {
+  uint16_t BW = getRegBitWidth(RR);
+
+  // Physical registers are assumed to be present in the map with an unknown
+  // value. Don't actually insert anything in the map, just return the cell.
+  if (TargetRegisterInfo::isPhysicalRegister(RR.Reg))
+    return RegisterCell::self(0, BW);
+
+  assert(TargetRegisterInfo::isVirtualRegister(RR.Reg));
+  // For virtual registers that belong to a class that is not tracked,
+  // generate an "unknown" value as well.
+  const TargetRegisterClass *C = MRI.getRegClass(RR.Reg);
+  if (!track(C))
+    return RegisterCell::self(0, BW);
+
+  CellMapType::const_iterator F = M.find(RR.Reg);
+  if (F != M.end()) {
+    if (!RR.Sub)
+      return F->second;
+    BitMask M = mask(RR.Reg, RR.Sub);
+    return F->second.extract(M);
+  }
+  // If not found, create a "top" entry, but do not insert it in the map.
+  return RegisterCell::top(BW);
+}
+
+
+void BT::MachineEvaluator::putCell(const RegisterRef &RR, RegisterCell RC,
+      CellMapType &M) const {
+  // While updating the cell map can be done in a meaningful way for
+  // a part of a register, it makes little sense to implement it as the
+  // SSA representation would never contain such "partial definitions".
+  if (!TargetRegisterInfo::isVirtualRegister(RR.Reg))
+    return;
+  assert(RR.Sub == 0 && "Unexpected sub-register in definition");
+  // Eliminate all ref-to-reg-0 bit values: replace them with "self".
+  for (unsigned i = 0, n = RC.width(); i < n; ++i) {
+    const BitValue &V = RC[i];
+    if (V.Type == BitValue::Ref && V.RefI.Reg == 0)
+      RC[i].RefI = BitRef(RR.Reg, i);
+  }
+  M[RR.Reg] = RC;
+}
+
+
+// Check if the cell represents a compile-time integer value.
+bool BT::MachineEvaluator::isInt(const RegisterCell &A) const {
+  uint16_t W = A.width();
+  for (uint16_t i = 0; i < W; ++i)
+    if (!A[i].is(0) && !A[i].is(1))
+      return false;
+  return true;
+}
+
+
+// Convert a cell to the integer value. The result must fit in uint64_t.
+uint64_t BT::MachineEvaluator::toInt(const RegisterCell &A) const {
+  assert(isInt(A));
+  uint64_t Val = 0;
+  uint16_t W = A.width();
+  for (uint16_t i = 0; i < W; ++i) {
+    Val <<= 1;
+    Val |= A[i].is(1);
+  }
+  return Val;
+}
+
+
+// Evaluator helper functions. These implement some common operation on
+// register cells that can be used to implement target-specific instructions
+// in a target-specific evaluator.
+
+BT::RegisterCell BT::MachineEvaluator::eIMM(int64_t V, uint16_t W) const {
+  RegisterCell Res(W);
+  // For bits beyond the 63rd, this will generate the sign bit of V.
+  for (uint16_t i = 0; i < W; ++i) {
+    Res[i] = BitValue(V & 1);
+    V >>= 1;
+  }
+  return Res;
+}
+
+
+BT::RegisterCell BT::MachineEvaluator::eIMM(const ConstantInt *CI) const {
+  APInt A = CI->getValue();
+  uint16_t BW = A.getBitWidth();
+  assert((unsigned)BW == A.getBitWidth() && "BitWidth overflow");
+  RegisterCell Res(BW);
+  for (uint16_t i = 0; i < BW; ++i)
+    Res[i] = A[i];
+  return Res;
+}
+
+
+BT::RegisterCell BT::MachineEvaluator::eADD(const RegisterCell &A1,
+      const RegisterCell &A2) const {
+  uint16_t W = A1.width();
+  assert(W == A2.width());
+  RegisterCell Res(W);
+  bool Carry = false;
+  uint16_t I;
+  for (I = 0; I < W; ++I) {
+    const BitValue &V1 = A1[I];
+    const BitValue &V2 = A2[I];
+    if (!V1.num() || !V2.num())
+      break;
+    unsigned S = bool(V1) + bool(V2) + Carry;
+    Res[I] = BitValue(S & 1);
+    Carry = (S > 1);
+  }
+  for (; I < W; ++I) {
+    const BitValue &V1 = A1[I];
+    const BitValue &V2 = A2[I];
+    // If the next bit is same as Carry, the result will be 0 plus the
+    // other bit. The Carry bit will remain unchanged.
+    if (V1.is(Carry))
+      Res[I] = BitValue::ref(V2);
+    else if (V2.is(Carry))
+      Res[I] = BitValue::ref(V1);
+    else
+      break;
+  }
+  for (; I < W; ++I)
+    Res[I] = BitValue::self();
+  return Res;
+}
+
+
+BT::RegisterCell BT::MachineEvaluator::eSUB(const RegisterCell &A1,
+      const RegisterCell &A2) const {
+  uint16_t W = A1.width();
+  assert(W == A2.width());
+  RegisterCell Res(W);
+  bool Borrow = false;
+  uint16_t I;
+  for (I = 0; I < W; ++I) {
+    const BitValue &V1 = A1[I];
+    const BitValue &V2 = A2[I];
+    if (!V1.num() || !V2.num())
+      break;
+    unsigned S = bool(V1) - bool(V2) - Borrow;
+    Res[I] = BitValue(S & 1);
+    Borrow = (S > 1);
+  }
+  for (; I < W; ++I) {
+    const BitValue &V1 = A1[I];
+    const BitValue &V2 = A2[I];
+    if (V1.is(Borrow)) {
+      Res[I] = BitValue::ref(V2);
+      break;
+    }
+    if (V2.is(Borrow))
+      Res[I] = BitValue::ref(V1);
+    else
+      break;
+  }
+  for (; I < W; ++I)
+    Res[I] = BitValue::self();
+  return Res;
+}
+
+
+BT::RegisterCell BT::MachineEvaluator::eMLS(const RegisterCell &A1,
+      const RegisterCell &A2) const {
+  uint16_t W = A1.width() + A2.width();
+  uint16_t Z = A1.ct(0) + A2.ct(0);
+  RegisterCell Res(W);
+  Res.fill(0, Z, BitValue::Zero);
+  Res.fill(Z, W, BitValue::self());
+  return Res;
+}
+
+
+BT::RegisterCell BT::MachineEvaluator::eMLU(const RegisterCell &A1,
+      const RegisterCell &A2) const {
+  uint16_t W = A1.width() + A2.width();
+  uint16_t Z = A1.ct(0) + A2.ct(0);
+  RegisterCell Res(W);
+  Res.fill(0, Z, BitValue::Zero);
+  Res.fill(Z, W, BitValue::self());
+  return Res;
+}
+
+
+BT::RegisterCell BT::MachineEvaluator::eASL(const RegisterCell &A1,
+      uint16_t Sh) const {
+  assert(Sh <= A1.width());
+  RegisterCell Res = RegisterCell::ref(A1);
+  Res.rol(Sh);
+  Res.fill(0, Sh, BitValue::Zero);
+  return Res;
+}
+
+
+BT::RegisterCell BT::MachineEvaluator::eLSR(const RegisterCell &A1,
+      uint16_t Sh) const {
+  uint16_t W = A1.width();
+  assert(Sh <= W);
+  RegisterCell Res = RegisterCell::ref(A1);
+  Res.rol(W-Sh);
+  Res.fill(W-Sh, W, BitValue::Zero);
+  return Res;
+}
+
+
+BT::RegisterCell BT::MachineEvaluator::eASR(const RegisterCell &A1,
+      uint16_t Sh) const {
+  uint16_t W = A1.width();
+  assert(Sh <= W);
+  RegisterCell Res = RegisterCell::ref(A1);
+  BitValue Sign = Res[W-1];
+  Res.rol(W-Sh);
+  Res.fill(W-Sh, W, Sign);
+  return Res;
+}
+
+
+BT::RegisterCell BT::MachineEvaluator::eAND(const RegisterCell &A1,
+      const RegisterCell &A2) const {
+  uint16_t W = A1.width();
+  assert(W == A2.width());
+  RegisterCell Res(W);
+  for (uint16_t i = 0; i < W; ++i) {
+    const BitValue &V1 = A1[i];
+    const BitValue &V2 = A2[i];
+    if (V1.is(1))
+      Res[i] = BitValue::ref(V2);
+    else if (V2.is(1))
+      Res[i] = BitValue::ref(V1);
+    else if (V1.is(0) || V2.is(0))
+      Res[i] = BitValue::Zero;
+    else if (V1 == V2)
+      Res[i] = V1;
+    else
+      Res[i] = BitValue::self();
+  }
+  return Res;
+}
+
+
+BT::RegisterCell BT::MachineEvaluator::eORL(const RegisterCell &A1,
+      const RegisterCell &A2) const {
+  uint16_t W = A1.width();
+  assert(W == A2.width());
+  RegisterCell Res(W);
+  for (uint16_t i = 0; i < W; ++i) {
+    const BitValue &V1 = A1[i];
+    const BitValue &V2 = A2[i];
+    if (V1.is(1) || V2.is(1))
+      Res[i] = BitValue::One;
+    else if (V1.is(0))
+      Res[i] = BitValue::ref(V2);
+    else if (V2.is(0))
+      Res[i] = BitValue::ref(V1);
+    else if (V1 == V2)
+      Res[i] = V1;
+    else
+      Res[i] = BitValue::self();
+  }
+  return Res;
+}
+
+
+BT::RegisterCell BT::MachineEvaluator::eXOR(const RegisterCell &A1,
+      const RegisterCell &A2) const {
+  uint16_t W = A1.width();
+  assert(W == A2.width());
+  RegisterCell Res(W);
+  for (uint16_t i = 0; i < W; ++i) {
+    const BitValue &V1 = A1[i];
+    const BitValue &V2 = A2[i];
+    if (V1.is(0))
+      Res[i] = BitValue::ref(V2);
+    else if (V2.is(0))
+      Res[i] = BitValue::ref(V1);
+    else if (V1 == V2)
+      Res[i] = BitValue::Zero;
+    else
+      Res[i] = BitValue::self();
+  }
+  return Res;
+}
+
+
+BT::RegisterCell BT::MachineEvaluator::eNOT(const RegisterCell &A1) const {
+  uint16_t W = A1.width();
+  RegisterCell Res(W);
+  for (uint16_t i = 0; i < W; ++i) {
+    const BitValue &V = A1[i];
+    if (V.is(0))
+      Res[i] = BitValue::One;
+    else if (V.is(1))
+      Res[i] = BitValue::Zero;
+    else
+      Res[i] = BitValue::self();
+  }
+  return Res;
+}
+
+
+BT::RegisterCell BT::MachineEvaluator::eSET(const RegisterCell &A1,
+      uint16_t BitN) const {
+  assert(BitN < A1.width());
+  RegisterCell Res = RegisterCell::ref(A1);
+  Res[BitN] = BitValue::One;
+  return Res;
+}
+
+
+BT::RegisterCell BT::MachineEvaluator::eCLR(const RegisterCell &A1,
+      uint16_t BitN) const {
+  assert(BitN < A1.width());
+  RegisterCell Res = RegisterCell::ref(A1);
+  Res[BitN] = BitValue::Zero;
+  return Res;
+}
+
+
+BT::RegisterCell BT::MachineEvaluator::eCLB(const RegisterCell &A1, bool B,
+      uint16_t W) const {
+  uint16_t C = A1.cl(B), AW = A1.width();
+  // If the last leading non-B bit is not a constant, then we don't know
+  // the real count.
+  if ((C < AW && A1[AW-1-C].num()) || C == AW)
+    return eIMM(C, W);
+  return RegisterCell::self(0, W);
+}
+
+
+BT::RegisterCell BT::MachineEvaluator::eCTB(const RegisterCell &A1, bool B,
+      uint16_t W) const {
+  uint16_t C = A1.ct(B), AW = A1.width();
+  // If the last trailing non-B bit is not a constant, then we don't know
+  // the real count.
+  if ((C < AW && A1[C].num()) || C == AW)
+    return eIMM(C, W);
+  return RegisterCell::self(0, W);
+}
+
+
+BT::RegisterCell BT::MachineEvaluator::eSXT(const RegisterCell &A1,
+      uint16_t FromN) const {
+  uint16_t W = A1.width();
+  assert(FromN <= W);
+  RegisterCell Res = RegisterCell::ref(A1);
+  BitValue Sign = Res[FromN-1];
+  // Sign-extend "inreg".
+  Res.fill(FromN, W, Sign);
+  return Res;
+}
+
+
+BT::RegisterCell BT::MachineEvaluator::eZXT(const RegisterCell &A1,
+      uint16_t FromN) const {
+  uint16_t W = A1.width();
+  assert(FromN <= W);
+  RegisterCell Res = RegisterCell::ref(A1);
+  Res.fill(FromN, W, BitValue::Zero);
+  return Res;
+}
+
+
+BT::RegisterCell BT::MachineEvaluator::eXTR(const RegisterCell &A1,
+      uint16_t B, uint16_t E) const {
+  uint16_t W = A1.width();
+  assert(B < W && E <= W);
+  if (B == E)
+    return RegisterCell(0);
+  uint16_t Last = (E > 0) ? E-1 : W-1;
+  RegisterCell Res = RegisterCell::ref(A1).extract(BT::BitMask(B, Last));
+  // Return shorter cell.
+  return Res;
+}
+
+
+BT::RegisterCell BT::MachineEvaluator::eINS(const RegisterCell &A1,
+      const RegisterCell &A2, uint16_t AtN) const {
+  uint16_t W1 = A1.width(), W2 = A2.width();
+  (void)W1;
+  assert(AtN < W1 && AtN+W2 <= W1);
+  // Copy bits from A1, insert A2 at position AtN.
+  RegisterCell Res = RegisterCell::ref(A1);
+  if (W2 > 0)
+    Res.insert(RegisterCell::ref(A2), BT::BitMask(AtN, AtN+W2-1));
+  return Res;
+}
+
+
+BT::BitMask BT::MachineEvaluator::mask(unsigned Reg, unsigned Sub) const {
+  assert(Sub == 0 && "Generic BitTracker::mask called for Sub != 0");
+  uint16_t W = getRegBitWidth(Reg);
+  assert(W > 0 && "Cannot generate mask for empty register");
+  return BitMask(0, W-1);
+}
+
+
+bool BT::MachineEvaluator::evaluate(const MachineInstr *MI,
+      const CellMapType &Inputs, CellMapType &Outputs) const {
+  unsigned Opc = MI->getOpcode();
+  switch (Opc) {
+    case TargetOpcode::REG_SEQUENCE: {
+      RegisterRef RD = MI->getOperand(0);
+      assert(RD.Sub == 0);
+      RegisterRef RS = MI->getOperand(1);
+      unsigned SS = MI->getOperand(2).getImm();
+      RegisterRef RT = MI->getOperand(3);
+      unsigned ST = MI->getOperand(4).getImm();
+      assert(SS != ST);
+
+      uint16_t W = getRegBitWidth(RD);
+      RegisterCell Res(W);
+      Res.insert(RegisterCell::ref(getCell(RS, Inputs)), mask(RD.Reg, SS));
+      Res.insert(RegisterCell::ref(getCell(RT, Inputs)), mask(RD.Reg, ST));
+      putCell(RD, Res, Outputs);
+      break;
+    }
+
+    case TargetOpcode::COPY: {
+      // COPY can transfer a smaller register into a wider one.
+      // If that is the case, fill the remaining high bits with 0.
+      RegisterRef RD = MI->getOperand(0);
+      RegisterRef RS = MI->getOperand(1);
+      assert(RD.Sub == 0);
+      uint16_t WD = getRegBitWidth(RD);
+      uint16_t WS = getRegBitWidth(RS);
+      assert(WD >= WS);
+      RegisterCell Src = getCell(RS, Inputs);
+      RegisterCell Res(WD);
+      Res.insert(Src, BitMask(0, WS-1));
+      Res.fill(WS, WD, BitValue::Zero);
+      putCell(RD, Res, Outputs);
+      break;
+    }
+
+    default:
+      return false;
+  }
+
+  return true;
+}
+
+
+// Main W-Z implementation.
+
+void BT::visitPHI(const MachineInstr *PI) {
+  int ThisN = PI->getParent()->getNumber();
+  if (Trace)
+    dbgs() << "Visit FI(BB#" << ThisN << "): " << *PI;
+
+  const MachineOperand &MD = PI->getOperand(0);
+  assert(MD.getSubReg() == 0 && "Unexpected sub-register in definition");
+  RegisterRef DefRR(MD);
+  uint16_t DefBW = ME.getRegBitWidth(DefRR);
+
+  RegisterCell DefC = ME.getCell(DefRR, Map);
+  if (DefC == RegisterCell::self(DefRR.Reg, DefBW))    // XXX slow
+    return;
+
+  bool Changed = false;
+
+  for (unsigned i = 1, n = PI->getNumOperands(); i < n; i += 2) {
+    const MachineBasicBlock *PB = PI->getOperand(i+1).getMBB();
+    int PredN = PB->getNumber();
+    if (Trace)
+      dbgs() << "  edge BB#" << PredN << "->BB#" << ThisN;
+    if (!EdgeExec.count(CFGEdge(PredN, ThisN))) {
+      if (Trace)
+        dbgs() << " not executable\n";
+      continue;
+    }
+
+    RegisterRef RU = PI->getOperand(i);
+    RegisterCell ResC = ME.getCell(RU, Map);
+    if (Trace)
+      dbgs() << " input reg: " << PrintReg(RU.Reg, &ME.TRI, RU.Sub)
+             << " cell: " << ResC << "\n";
+    Changed |= DefC.meet(ResC, DefRR.Reg);
+  }
+
+  if (Changed) {
+    if (Trace)
+      dbgs() << "Output: " << PrintReg(DefRR.Reg, &ME.TRI, DefRR.Sub)
+             << " cell: " << DefC << "\n";
+    ME.putCell(DefRR, DefC, Map);
+    visitUsesOf(DefRR.Reg);
+  }
+}
+
+
+void BT::visitNonBranch(const MachineInstr *MI) {
+  if (Trace) {
+    int ThisN = MI->getParent()->getNumber();
+    dbgs() << "Visit MI(BB#" << ThisN << "): " << *MI;
+  }
+  if (MI->isDebugValue())
+    return;
+  assert(!MI->isBranch() && "Unexpected branch instruction");
+
+  CellMapType ResMap;
+  bool Eval = ME.evaluate(MI, Map, ResMap);
+
+  if (Trace && Eval) {
+    for (unsigned i = 0, n = MI->getNumOperands(); i < n; ++i) {
+      const MachineOperand &MO = MI->getOperand(i);
+      if (!MO.isReg() || !MO.isUse())
+        continue;
+      RegisterRef RU(MO);
+      dbgs() << "  input reg: " << PrintReg(RU.Reg, &ME.TRI, RU.Sub)
+             << " cell: " << ME.getCell(RU, Map) << "\n";
+    }
+    dbgs() << "Outputs:\n";
+    for (CellMapType::iterator I = ResMap.begin(), E = ResMap.end();
+         I != E; ++I) {
+      RegisterRef RD(I->first);
+      dbgs() << "  " << PrintReg(I->first, &ME.TRI) << " cell: "
+             << ME.getCell(RD, ResMap) << "\n";
+    }
+  }
+
+  // Iterate over all definitions of the instruction, and update the
+  // cells accordingly.
+  for (unsigned i = 0, n = MI->getNumOperands(); i < n; ++i) {
+    const MachineOperand &MO = MI->getOperand(i);
+    // Visit register defs only.
+    if (!MO.isReg() || !MO.isDef())
+      continue;
+    RegisterRef RD(MO);
+    assert(RD.Sub == 0 && "Unexpected sub-register in definition");
+    if (!TargetRegisterInfo::isVirtualRegister(RD.Reg))
+      continue;
+
+    bool Changed = false;
+    if (!Eval || ResMap.count(RD.Reg) == 0) {
+      // Set to "ref" (aka "bottom").
+      uint16_t DefBW = ME.getRegBitWidth(RD);
+      RegisterCell RefC = RegisterCell::self(RD.Reg, DefBW);
+      if (RefC != ME.getCell(RD, Map)) {
+        ME.putCell(RD, RefC, Map);
+        Changed = true;
+      }
+    } else {
+      RegisterCell DefC = ME.getCell(RD, Map);
+      RegisterCell ResC = ME.getCell(RD, ResMap);
+      // This is a non-phi instruction, so the values of the inputs come
+      // from the same registers each time this instruction is evaluated.
+      // During the propagation, the values of the inputs can become lowered
+      // in the sense of the lattice operation, which may cause different
+      // results to be calculated in subsequent evaluations. This should
+      // not cause the bottoming of the result in the map, since the new
+      // result is already reflecting the lowered inputs.
+      for (uint16_t i = 0, w = DefC.width(); i < w; ++i) {
+        BitValue &V = DefC[i];
+        // Bits that are already "bottom" should not be updated.
+        if (V.Type == BitValue::Ref && V.RefI.Reg == RD.Reg)
+          continue;
+        // Same for those that are identical in DefC and ResC.
+        if (V == ResC[i])
+          continue;
+        V = ResC[i];
+        Changed = true;
+      }
+      if (Changed)
+        ME.putCell(RD, DefC, Map);
+    }
+    if (Changed)
+      visitUsesOf(RD.Reg);
+  }
+}
+
+
+void BT::visitBranchesFrom(const MachineInstr *BI) {
+  const MachineBasicBlock &B = *BI->getParent();
+  MachineBasicBlock::const_iterator It = BI, End = B.end();
+  BranchTargetList Targets, BTs;
+  bool FallsThrough = true, DefaultToAll = false;
+  int ThisN = B.getNumber();
+
+  do {
+    BTs.clear();
+    const MachineInstr *MI = &*It;
+    if (Trace)
+      dbgs() << "Visit BR(BB#" << ThisN << "): " << *MI;
+    assert(MI->isBranch() && "Expecting branch instruction");
+    InstrExec.insert(MI);
+    bool Eval = ME.evaluate(MI, Map, BTs, FallsThrough);
+    if (!Eval) {
+      // If the evaluation failed, we will add all targets. Keep going in
+      // the loop to mark all executable branches as such.
+      DefaultToAll = true;
+      FallsThrough = true;
+      if (Trace)
+        dbgs() << "  failed to evaluate: will add all CFG successors\n";
+    } else if (!DefaultToAll) {
+      // If evaluated successfully add the targets to the cumulative list.
+      if (Trace) {
+        dbgs() << "  adding targets:";
+        for (unsigned i = 0, n = BTs.size(); i < n; ++i)
+          dbgs() << " BB#" << BTs[i]->getNumber();
+        if (FallsThrough)
+          dbgs() << "\n  falls through\n";
+        else
+          dbgs() << "\n  does not fall through\n";
+      }
+      Targets.insert(BTs.begin(), BTs.end());
+    }
+    ++It;
+  } while (FallsThrough && It != End);
+
+  typedef MachineBasicBlock::const_succ_iterator succ_iterator;
+  if (!DefaultToAll) {
+    // Need to add all CFG successors that lead to EH landing pads.
+    // There won't be explicit branches to these blocks, but they must
+    // be processed.
+    for (succ_iterator I = B.succ_begin(), E = B.succ_end(); I != E; ++I) {
+      const MachineBasicBlock *SB = *I;
+      if (SB->isLandingPad())
+        Targets.insert(SB);
+    }
+    if (FallsThrough) {
+      MachineFunction::const_iterator BIt = &B;
+      MachineFunction::const_iterator Next = std::next(BIt);
+      if (Next != MF.end())
+        Targets.insert(&*Next);
+    }
+  } else {
+    for (succ_iterator I = B.succ_begin(), E = B.succ_end(); I != E; ++I)
+      Targets.insert(*I);
+  }
+
+  for (unsigned i = 0, n = Targets.size(); i < n; ++i) {
+    int TargetN = Targets[i]->getNumber();
+    FlowQ.push(CFGEdge(ThisN, TargetN));
+  }
+}
+
+
+void BT::visitUsesOf(unsigned Reg) {
+  if (Trace)
+    dbgs() << "visiting uses of " << PrintReg(Reg, &ME.TRI) << "\n";
+
+  typedef MachineRegisterInfo::use_nodbg_iterator use_iterator;
+  use_iterator End = MRI.use_nodbg_end();
+  for (use_iterator I = MRI.use_nodbg_begin(Reg); I != End; ++I) {
+    MachineInstr *UseI = I->getParent();
+    if (!InstrExec.count(UseI))
+      continue;
+    if (UseI->isPHI())
+      visitPHI(UseI);
+    else if (!UseI->isBranch())
+      visitNonBranch(UseI);
+    else
+      visitBranchesFrom(UseI);
+  }
+}
+
+
+BT::RegisterCell BT::get(RegisterRef RR) const {
+  return ME.getCell(RR, Map);
+}
+
+
+void BT::put(RegisterRef RR, const RegisterCell &RC) {
+  ME.putCell(RR, RC, Map);
+}
+
+
+// Replace all references to bits from OldRR with the corresponding bits
+// in NewRR.
+void BT::subst(RegisterRef OldRR, RegisterRef NewRR) {
+  assert(Map.count(OldRR.Reg) > 0 && "OldRR not present in map");
+  BitMask OM = ME.mask(OldRR.Reg, OldRR.Sub);
+  BitMask NM = ME.mask(NewRR.Reg, NewRR.Sub);
+  uint16_t OMB = OM.first(), OME = OM.last();
+  uint16_t NMB = NM.first(), NME = NM.last();
+  (void)NME;
+  assert((OME-OMB == NME-NMB) &&
+         "Substituting registers of different lengths");
+  for (CellMapType::iterator I = Map.begin(), E = Map.end(); I != E; ++I) {
+    RegisterCell &RC = I->second;
+    for (uint16_t i = 0, w = RC.width(); i < w; ++i) {
+      BitValue &V = RC[i];
+      if (V.Type != BitValue::Ref || V.RefI.Reg != OldRR.Reg)
+        continue;
+      if (V.RefI.Pos < OMB || V.RefI.Pos > OME)
+        continue;
+      V.RefI.Reg = NewRR.Reg;
+      V.RefI.Pos += NMB-OMB;
+    }
+  }
+}
+
+
+// Check if the block has been "executed" during propagation. (If not, the
+// block is dead, but it may still appear to be reachable.)
+bool BT::reached(const MachineBasicBlock *B) const {
+  int BN = B->getNumber();
+  assert(BN >= 0);
+  for (EdgeSetType::iterator I = EdgeExec.begin(), E = EdgeExec.end();
+       I != E; ++I) {
+    if (I->second == BN)
+      return true;
+  }
+  return false;
+}
+
+
+void BT::reset() {
+  EdgeExec.clear();
+  InstrExec.clear();
+  Map.clear();
+}
+
+
+void BT::run() {
+  reset();
+  assert(FlowQ.empty());
+
+  typedef GraphTraits<const MachineFunction*> MachineFlowGraphTraits;
+  const MachineBasicBlock *Entry = MachineFlowGraphTraits::getEntryNode(&MF);
+
+  unsigned MaxBN = 0;
+  for (MachineFunction::const_iterator I = MF.begin(), E = MF.end();
+       I != E; ++I) {
+    assert(I->getNumber() >= 0 && "Disconnected block");
+    unsigned BN = I->getNumber();
+    if (BN > MaxBN)
+      MaxBN = BN;
+  }
+
+  // Keep track of visited blocks.
+  BitVector BlockScanned(MaxBN+1);
+
+  int EntryN = Entry->getNumber();
+  // Generate a fake edge to get something to start with.
+  FlowQ.push(CFGEdge(-1, EntryN));
+
+  while (!FlowQ.empty()) {
+    CFGEdge Edge = FlowQ.front();
+    FlowQ.pop();
+
+    if (EdgeExec.count(Edge))
+      continue;
+    EdgeExec.insert(Edge);
+
+    const MachineBasicBlock &B = *MF.getBlockNumbered(Edge.second);
+    MachineBasicBlock::const_iterator It = B.begin(), End = B.end();
+    // Visit PHI nodes first.
+    while (It != End && It->isPHI()) {
+      const MachineInstr *PI = &*It++;
+      InstrExec.insert(PI);
+      visitPHI(PI);
+    }
+
+    // If this block has already been visited through a flow graph edge,
+    // then the instructions have already been processed. Any updates to
+    // the cells would now only happen through visitUsesOf...
+    if (BlockScanned[Edge.second])
+      continue;
+    BlockScanned[Edge.second] = true;
+
+    // Visit non-branch instructions.
+    while (It != End && !It->isBranch()) {
+      const MachineInstr *MI = &*It++;
+      InstrExec.insert(MI);
+      visitNonBranch(MI);
+    }
+    // If block end has been reached, add the fall-through edge to the queue.
+    if (It == End) {
+      MachineFunction::const_iterator BIt = &B;
+      MachineFunction::const_iterator Next = std::next(BIt);
+      if (Next != MF.end()) {
+        int ThisN = B.getNumber();
+        int NextN = Next->getNumber();
+        FlowQ.push(CFGEdge(ThisN, NextN));
+      }
+    } else {
+      // Handle the remaining sequence of branches. This function will update
+      // the work queue.
+      visitBranchesFrom(It);
+    }
+  } // while (!FlowQ->empty())
+
+  if (Trace) {
+    dbgs() << "Cells after propagation:\n";
+    for (CellMapType::iterator I = Map.begin(), E = Map.end(); I != E; ++I)
+      dbgs() << PrintReg(I->first, &ME.TRI) << " -> " << I->second << "\n";
+  }
+}
+
diff --git a/lib/Target/Hexagon/BitTracker.h b/lib/Target/Hexagon/BitTracker.h
new file mode 100644
index 00000000000..fd3a68f4385
--- /dev/null
+++ b/lib/Target/Hexagon/BitTracker.h
@@ -0,0 +1,442 @@
+//===--- BitTracker.h -----------------------------------------------------===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef BITTRACKER_H
+#define BITTRACKER_H
+
+#include "llvm/ADT/SetVector.h"
+#include "llvm/ADT/SmallVector.h"
+#include "llvm/CodeGen/MachineFunction.h"
+
+#include <map>
+#include <queue>
+#include <set>
+
+namespace llvm {
+  class ConstantInt;
+  class MachineRegisterInfo;
+  class MachineBasicBlock;
+  class MachineInstr;
+  class MachineOperand;
+  class raw_ostream;
+
+struct BitTracker {
+  struct BitRef;
+  struct RegisterRef;
+  struct BitValue;
+  struct BitMask;
+  struct RegisterCell;
+  struct MachineEvaluator;
+
+  typedef SetVector<const MachineBasicBlock *> BranchTargetList;
+
+  typedef std::map<unsigned, RegisterCell> CellMapType;
+
+  BitTracker(const MachineEvaluator &E, MachineFunction &F);
+  ~BitTracker();
+
+  void run();
+  void trace(bool On = false) { Trace = On; }
+  bool has(unsigned Reg) const;
+  const RegisterCell &lookup(unsigned Reg) const;
+  RegisterCell get(RegisterRef RR) const;
+  void put(RegisterRef RR, const RegisterCell &RC);
+  void subst(RegisterRef OldRR, RegisterRef NewRR);
+  bool reached(const MachineBasicBlock *B) const;
+
+private:
+  void visitPHI(const MachineInstr *PI);
+  void visitNonBranch(const MachineInstr *MI);
+  void visitBranchesFrom(const MachineInstr *BI);
+  void visitUsesOf(unsigned Reg);
+  void reset();
+
+  typedef std::pair<int,int> CFGEdge;
+  typedef std::set<CFGEdge> EdgeSetType;
+  typedef std::set<const MachineInstr *> InstrSetType;
+  typedef std::queue<CFGEdge> EdgeQueueType;
+
+  EdgeSetType EdgeExec;       // Executable flow graph edges.
+  InstrSetType InstrExec;     // Executable instructions.
+  EdgeQueueType FlowQ;        // Work queue of CFG edges.
+  bool Trace;                 // Enable tracing for debugging.
+
+  const MachineEvaluator &ME;
+  MachineFunction &MF;
+  MachineRegisterInfo &MRI;
+  CellMapType &Map;
+};
+
+
+// Abstraction of a reference to bit at position Pos from a register Reg.
+struct BitTracker::BitRef {
+  BitRef(unsigned R = 0, uint16_t P = 0) : Reg(R), Pos(P) {}
+  BitRef(const BitRef &BR) : Reg(BR.Reg), Pos(BR.Pos) {}
+  bool operator== (const BitRef &BR) const {
+    // If Reg is 0, disregard Pos.
+    return Reg == BR.Reg && (Reg == 0 || Pos == BR.Pos);
+  }
+  unsigned Reg;
+  uint16_t Pos;
+};
+
+
+// Abstraction of a register reference in MachineOperand.  It contains the
+// register number and the subregister index.
+struct BitTracker::RegisterRef {
+  RegisterRef(unsigned R = 0, unsigned S = 0)
+    : Reg(R), Sub(S) {}
+  RegisterRef(const MachineOperand &MO)
+      : Reg(MO.getReg()), Sub(MO.getSubReg()) {}
+  unsigned Reg, Sub;
+};
+
+
+// Value that a single bit can take.  This is outside of the context of
+// any register, it is more of an abstraction of the two-element set of
+// possible bit values.  One extension here is the "Ref" type, which
+// indicates that this bit takes the same value as the bit described by
+// RefInfo.
+struct BitTracker::BitValue {
+  enum ValueType {
+    Top,    // Bit not yet defined.
+    Zero,   // Bit = 0.
+    One,    // Bit = 1.
+    Ref     // Bit value same as the one described in RefI.
+    // Conceptually, there is no explicit "bottom" value: the lattice's
+    // bottom will be expressed as a "ref to itself", which, in the context
+    // of registers, could be read as "this value of this bit is defined by
+    // this bit".
+    // The ordering is:
+    //   x <= Top,
+    //   Self <= x, where "Self" is "ref to itself".
+    // This makes the value lattice different for each virtual register
+    // (even for each bit in the same virtual register), since the "bottom"
+    // for one register will be a simple "ref" for another register.
+    // Since we do not store the "Self" bit and register number, the meet
+    // operation will need to take it as a parameter.
+    //
+    // In practice there is a special case for values that are not associa-
+    // ted with any specific virtual register. An example would be a value
+    // corresponding to a bit of a physical register, or an intermediate
+    // value obtained in some computation (such as instruction evaluation).
+    // Such cases are identical to the usual Ref type, but the register
+    // number is 0. In such case the Pos field of the reference is ignored.
+    //
+    // What is worthy of notice is that in value V (that is a "ref"), as long
+    // as the RefI.Reg is not 0, it may actually be the same register as the
+    // one in which V will be contained.  If the RefI.Pos refers to the posi-
+    // tion of V, then V is assumed to be "bottom" (as a "ref to itself"),
+    // otherwise V is taken to be identical to the referenced bit of the
+    // same register.
+    // If RefI.Reg is 0, however, such a reference to the same register is
+    // not possible.  Any value V that is a "ref", and whose RefI.Reg is 0
+    // is treated as "bottom".
+  };
+  ValueType Type;
+  BitRef RefI;
+
+  BitValue(ValueType T = Top) : Type(T) {}
+  BitValue(bool B) : Type(B ? One : Zero) {}
+  BitValue(const BitValue &V) : Type(V.Type), RefI(V.RefI) {}
+  BitValue(unsigned Reg, uint16_t Pos) : Type(Ref), RefI(Reg, Pos) {}
+
+  bool operator== (const BitValue &V) const {
+    if (Type != V.Type)
+      return false;
+    if (Type == Ref && !(RefI == V.RefI))
+      return false;
+    return true;
+  }
+  bool operator!= (const BitValue &V) const {
+    return !operator==(V);
+  }
+  bool is(unsigned T) const {
+    assert(T == 0 || T == 1);
+    return T == 0 ? Type == Zero
+                  : (T == 1 ? Type == One : false);
+  }
+
+  // The "meet" operation is the "." operation in a semilattice (L, ., T, B):
+  // (1)  x.x = x
+  // (2)  x.y = y.x
+  // (3)  x.(y.z) = (x.y).z
+  // (4)  x.T = x  (i.e. T = "top")
+  // (5)  x.B = B  (i.e. B = "bottom")
+  //
+  // This "meet" function will update the value of the "*this" object with
+  // the newly calculated one, and return "true" if the value of *this has
+  // changed, and "false" otherwise.
+  // To prove that it satisfies the conditions (1)-(5), it is sufficient
+  // to show that a relation
+  //   x <= y  <=>  x.y = x
+  // defines a partial order (i.e. that "meet" is same as "infimum").
+  bool meet(const BitValue &V, const BitRef &Self) {
+    // First, check the cases where there is nothing to be done.
+    if (Type == Ref && RefI == Self)    // Bottom.meet(V) = Bottom (i.e. This)
+      return false;
+    if (V.Type == Top)                  // This.meet(Top) = This
+      return false;
+    if (*this == V)                     // This.meet(This) = This
+      return false;
+
+    // At this point, we know that the value of "this" will change.
+    // If it is Top, it will become the same as V, otherwise it will
+    // become "bottom" (i.e. Self).
+    if (Type == Top) {
+      Type = V.Type;
+      RefI = V.RefI;  // This may be irrelevant, but copy anyway.
+      return true;
+    }
+    // Become "bottom".
+    Type = Ref;
+    RefI = Self;
+    return true;
+  }
+
+  // Create a reference to the bit value V.
+  static BitValue ref(const BitValue &V);
+  // Create a "self".
+  static BitValue self(const BitRef &Self = BitRef());
+
+  bool num() const {
+    return Type == Zero || Type == One;
+  }
+  operator bool() const {
+    assert(Type == Zero || Type == One);
+    return Type == One;
+  }
+
+  friend raw_ostream &operator<<(raw_ostream &OS, const BitValue &BV);
+};
+
+
+// This operation must be idempotent, i.e. ref(ref(V)) == ref(V).
+inline BitTracker::BitValue
+BitTracker::BitValue::ref(const BitValue &V) {
+  if (V.Type != Ref)
+    return BitValue(V.Type);
+  if (V.RefI.Reg != 0)
+    return BitValue(V.RefI.Reg, V.RefI.Pos);
+  return self();
+}
+
+
+inline BitTracker::BitValue
+BitTracker::BitValue::self(const BitRef &Self) {
+  return BitValue(Self.Reg, Self.Pos);
+}
+
+
+// A sequence of bits starting from index B up to and including index E.
+// If E < B, the mask represents two sections: [0..E] and [B..W) where
+// W is the width of the register.
+struct BitTracker::BitMask {
+  BitMask() : B(0), E(0) {}
+  BitMask(uint16_t b, uint16_t e) : B(b), E(e) {}
+  uint16_t first() const { return B; }
+  uint16_t last() const { return E; }
+private:
+  uint16_t B, E;
+};
+
+
+// Representation of a register: a list of BitValues.
+struct BitTracker::RegisterCell {
+  RegisterCell(uint16_t Width = DefaultBitN) : Bits(Width) {}
+
+  uint16_t width() const {
+    return Bits.size();
+  }
+  const BitValue &operator[](uint16_t BitN) const {
+    assert(BitN < Bits.size());
+    return Bits[BitN];
+  }
+  BitValue &operator[](uint16_t BitN) {
+    assert(BitN < Bits.size());
+    return Bits[BitN];
+  }
+
+  bool meet(const RegisterCell &RC, unsigned SelfR);
+  RegisterCell &insert(const RegisterCell &RC, const BitMask &M);
+  RegisterCell extract(const BitMask &M) const;  // Returns a new cell.
+  RegisterCell &rol(uint16_t Sh);    // Rotate left.
+  RegisterCell &fill(uint16_t B, uint16_t E, const BitValue &V);
+  RegisterCell &cat(const RegisterCell &RC);  // Concatenate.
+  uint16_t cl(bool B) const;
+  uint16_t ct(bool B) const;
+
+  bool operator== (const RegisterCell &RC) const;
+  bool operator!= (const RegisterCell &RC) const {
+    return !operator==(RC);
+  }
+
+  const RegisterCell &operator=(const RegisterCell &RC) {
+    Bits = RC.Bits;
+    return *this;
+  }
+
+  // Generate a "ref" cell for the corresponding register. In the resulting
+  // cell each bit will be described as being the same as the corresponding
+  // bit in register Reg (i.e. the cell is "defined" by register Reg).
+  static RegisterCell self(unsigned Reg, uint16_t Width);
+  // Generate a "top" cell of given size.
+  static RegisterCell top(uint16_t Width);
+  // Generate a cell that is a "ref" to another cell.
+  static RegisterCell ref(const RegisterCell &C);
+
+private:
+  // The DefaultBitN is here only to avoid frequent reallocation of the
+  // memory in the vector.
+  static const unsigned DefaultBitN = 32;
+  typedef SmallVector<BitValue, DefaultBitN> BitValueList;
+  BitValueList Bits;
+
+  friend raw_ostream &operator<<(raw_ostream &OS, const RegisterCell &RC);
+};
+
+
+inline bool BitTracker::has(unsigned Reg) const {
+  return Map.find(Reg) != Map.end();
+}
+
+
+inline const BitTracker::RegisterCell&
+BitTracker::lookup(unsigned Reg) const {
+  CellMapType::const_iterator F = Map.find(Reg);
+  assert(F != Map.end());
+  return F->second;
+}
+
+
+inline BitTracker::RegisterCell
+BitTracker::RegisterCell::self(unsigned Reg, uint16_t Width) {
+  RegisterCell RC(Width);
+  for (uint16_t i = 0; i < Width; ++i)
+    RC.Bits[i] = BitValue::self(BitRef(Reg, i));
+  return RC;
+}
+
+
+inline BitTracker::RegisterCell
+BitTracker::RegisterCell::top(uint16_t Width) {
+  RegisterCell RC(Width);
+  for (uint16_t i = 0; i < Width; ++i)
+    RC.Bits[i] = BitValue(BitValue::Top);
+  return RC;
+}
+
+
+inline BitTracker::RegisterCell
+BitTracker::RegisterCell::ref(const RegisterCell &C) {
+  uint16_t W = C.width();
+  RegisterCell RC(W);
+  for (unsigned i = 0; i < W; ++i)
+    RC[i] = BitValue::ref(C[i]);
+  return RC;
+}
+
+// A class to evaluate target's instructions and update the cell maps.
+// This is used internally by the bit tracker.  A target that wants to
+// utilize this should implement the evaluation functions (noted below)
+// in a subclass of this class.
+struct BitTracker::MachineEvaluator {
+  MachineEvaluator(const TargetRegisterInfo &T, MachineRegisterInfo &M)
+      : TRI(T), MRI(M) {}
+  virtual ~MachineEvaluator() {}
+
+  uint16_t getRegBitWidth(const RegisterRef &RR) const;
+
+  RegisterCell getCell(const RegisterRef &RR, const CellMapType &M) const;
+  void putCell(const RegisterRef &RR, RegisterCell RC, CellMapType &M) const;
+  // A result of any operation should use refs to the source cells, not
+  // the cells directly. This function is a convenience wrapper to quickly
+  // generate a ref for a cell corresponding to a register reference.
+  RegisterCell getRef(const RegisterRef &RR, const CellMapType &M) const {
+    RegisterCell RC = getCell(RR, M);
+    return RegisterCell::ref(RC);
+  }
+
+  // Helper functions.
+  // Check if a cell is an immediate value (i.e. all bits are either 0 or 1).
+  bool isInt(const RegisterCell &A) const;
+  // Convert cell to an immediate value.
+  uint64_t toInt(const RegisterCell &A) const;
+
+  // Generate cell from an immediate value.
+  RegisterCell eIMM(int64_t V, uint16_t W) const;
+  RegisterCell eIMM(const ConstantInt *CI) const;
+
+  // Arithmetic.
+  RegisterCell eADD(const RegisterCell &A1, const RegisterCell &A2) const;
+  RegisterCell eSUB(const RegisterCell &A1, const RegisterCell &A2) const;
+  RegisterCell eMLS(const RegisterCell &A1, const RegisterCell &A2) const;
+  RegisterCell eMLU(const RegisterCell &A1, const RegisterCell &A2) const;
+
+  // Shifts.
+  RegisterCell eASL(const RegisterCell &A1, uint16_t Sh) const;
+  RegisterCell eLSR(const RegisterCell &A1, uint16_t Sh) const;
+  RegisterCell eASR(const RegisterCell &A1, uint16_t Sh) const;
+
+  // Logical.
+  RegisterCell eAND(const RegisterCell &A1, const RegisterCell &A2) const;
+  RegisterCell eORL(const RegisterCell &A1, const RegisterCell &A2) const;
+  RegisterCell eXOR(const RegisterCell &A1, const RegisterCell &A2) const;
+  RegisterCell eNOT(const RegisterCell &A1) const;
+
+  // Set bit, clear bit.
+  RegisterCell eSET(const RegisterCell &A1, uint16_t BitN) const;
+  RegisterCell eCLR(const RegisterCell &A1, uint16_t BitN) const;
+
+  // Count leading/trailing bits (zeros/ones).
+  RegisterCell eCLB(const RegisterCell &A1, bool B, uint16_t W) const;
+  RegisterCell eCTB(const RegisterCell &A1, bool B, uint16_t W) const;
+
+  // Sign/zero extension.
+  RegisterCell eSXT(const RegisterCell &A1, uint16_t FromN) const;
+  RegisterCell eZXT(const RegisterCell &A1, uint16_t FromN) const;
+
+  // Extract/insert
+  // XTR R,b,e:  extract bits from A1 starting at bit b, ending at e-1.
+  // INS R,S,b:  take R and replace bits starting from b with S.
+  RegisterCell eXTR(const RegisterCell &A1, uint16_t B, uint16_t E) const;
+  RegisterCell eINS(const RegisterCell &A1, const RegisterCell &A2,
+                    uint16_t AtN) const;
+
+  // User-provided functions for individual targets:
+
+  // Return a sub-register mask that indicates which bits in Reg belong
+  // to the subregister Sub. These bits are assumed to be contiguous in
+  // the super-register, and have the same ordering in the sub-register
+  // as in the super-register. It is valid to call this function with
+  // Sub == 0, in this case, the function should return a mask that spans
+  // the entire register Reg (which is what the default implementation
+  // does).
+  virtual BitMask mask(unsigned Reg, unsigned Sub) const;
+  // Indicate whether a given register class should be tracked.
+  virtual bool track(const TargetRegisterClass *RC) const { return true; }
+  // Evaluate a non-branching machine instruction, given the cell map with
+  // the input values. Place the results in the Outputs map. Return "true"
+  // if evaluation succeeded, "false" otherwise.
+  virtual bool evaluate(const MachineInstr *MI, const CellMapType &Inputs,
+                        CellMapType &Outputs) const;
+  // Evaluate a branch, given the cell map with the input values. Fill out
+  // a list of all possible branch targets and indicate (through a flag)
+  // whether the branch could fall-through. Return "true" if this information
+  // has been successfully computed, "false" otherwise.
+  virtual bool evaluate(const MachineInstr *BI, const CellMapType &Inputs,
+                        BranchTargetList &Targets, bool &FallsThru) const = 0;
+
+  const TargetRegisterInfo &TRI;
+  MachineRegisterInfo &MRI;
+};
+
+} // end namespace llvm
+
+#endif
diff --git a/lib/Target/Hexagon/CMakeLists.txt b/lib/Target/Hexagon/CMakeLists.txt
index 758ccc74100..f8c3ca66918 100644
--- a/lib/Target/Hexagon/CMakeLists.txt
+++ b/lib/Target/Hexagon/CMakeLists.txt
@@ -12,13 +12,20 @@ tablegen(LLVM HexagonGenSubtargetInfo.inc -gen-subtarget)
 add_public_tablegen_target(HexagonCommonTableGen)
 
 add_llvm_target(HexagonCodeGen
+  BitTracker.cpp
   HexagonAsmPrinter.cpp
+  HexagonBitTracker.cpp
   HexagonCFGOptimizer.cpp
+  HexagonCommonGEP.cpp
   HexagonCopyToCombine.cpp
   HexagonExpandCondsets.cpp
   HexagonExpandPredSpillCode.cpp
   HexagonFixupHwLoops.cpp
   HexagonFrameLowering.cpp
+  HexagonGenExtract.cpp
+  HexagonGenInsert.cpp
+  HexagonGenMux.cpp
+  HexagonGenPredicate.cpp
   HexagonHardwareLoops.cpp
   HexagonInstrInfo.cpp
   HexagonISelDAGToDAG.cpp
diff --git a/lib/Target/Hexagon/Disassembler/HexagonDisassembler.cpp b/lib/Target/Hexagon/Disassembler/HexagonDisassembler.cpp
index 669af8c649e..9cc1e944d35 100644
--- a/lib/Target/Hexagon/Disassembler/HexagonDisassembler.cpp
+++ b/lib/Target/Hexagon/Disassembler/HexagonDisassembler.cpp
@@ -7,9 +7,11 @@
 //
 //===----------------------------------------------------------------------===//
 
+#include "Hexagon.h"
 #include "MCTargetDesc/HexagonBaseInfo.h"
 #include "MCTargetDesc/HexagonMCInstrInfo.h"
 #include "MCTargetDesc/HexagonMCTargetDesc.h"
+
 #include "llvm/MC/MCContext.h"
 #include "llvm/MC/MCDisassembler.h"
 #include "llvm/MC/MCExpr.h"
@@ -27,6 +29,7 @@
 #include <vector>
 
 using namespace llvm;
+using namespace Hexagon;
 
 #define DEBUG_TYPE "hexagon-disassembler"
 
@@ -37,9 +40,14 @@ namespace {
 /// \brief Hexagon disassembler for all Hexagon platforms.
 class HexagonDisassembler : public MCDisassembler {
 public:
+  std::unique_ptr<MCInst *> CurrentBundle;
   HexagonDisassembler(MCSubtargetInfo const &STI, MCContext &Ctx)
-      : MCDisassembler(STI, Ctx) {}
+      : MCDisassembler(STI, Ctx), CurrentBundle(new MCInst *) {}
 
+  DecodeStatus getSingleInstruction(MCInst &Instr, MCInst &MCB,
+                                    ArrayRef<uint8_t> Bytes, uint64_t Address,
+                                    raw_ostream &VStream, raw_ostream &CStream,
+                                    bool &Complete) const;
   DecodeStatus getInstruction(MCInst &Instr, uint64_t &Size,
                               ArrayRef<uint8_t> Bytes, uint64_t Address,
                               raw_ostream &VStream,
@@ -48,53 +56,86 @@ class HexagonDisassembler : public MCDisassembler {
 }
 
 static DecodeStatus DecodeModRegsRegisterClass(MCInst &Inst, unsigned RegNo,
-  uint64_t Address, const void *Decoder);
+                                               uint64_t Address,
+                                               const void *Decoder);
 static DecodeStatus DecodeCtrRegsRegisterClass(MCInst &Inst, unsigned RegNo,
-  uint64_t Address, const void *Decoder);
+                                               uint64_t Address,
+                                               const void *Decoder);
 static DecodeStatus DecodeCtrRegs64RegisterClass(MCInst &Inst, unsigned RegNo,
-                                   uint64_t Address, void const *Decoder);
+                                                 uint64_t Address,
+                                                 void const *Decoder);
+
+static unsigned GetSubinstOpcode(unsigned IClass, unsigned inst, unsigned &op,
+                                 raw_ostream &os);
+static void AddSubinstOperands(MCInst *MI, unsigned opcode, unsigned inst);
+
+static DecodeStatus s16ImmDecoder(MCInst &MI, unsigned tmp, uint64_t Address,
+                                  const void *Decoder);
+static DecodeStatus s12ImmDecoder(MCInst &MI, unsigned tmp, uint64_t Address,
+                                  const void *Decoder);
+static DecodeStatus s11_0ImmDecoder(MCInst &MI, unsigned tmp, uint64_t Address,
+                                    const void *Decoder);
+static DecodeStatus s11_1ImmDecoder(MCInst &MI, unsigned tmp, uint64_t Address,
+                                    const void *Decoder);
+static DecodeStatus s11_2ImmDecoder(MCInst &MI, unsigned tmp, uint64_t Address,
+                                    const void *Decoder);
+static DecodeStatus s11_3ImmDecoder(MCInst &MI, unsigned tmp, uint64_t Address,
+                                    const void *Decoder);
+static DecodeStatus s10ImmDecoder(MCInst &MI, unsigned tmp, uint64_t Address,
+                                  const void *Decoder);
+static DecodeStatus s8ImmDecoder(MCInst &MI, unsigned tmp, uint64_t Address,
+                                 const void *Decoder);
+static DecodeStatus s6_0ImmDecoder(MCInst &MI, unsigned tmp, uint64_t Address,
+                                   const void *Decoder);
+static DecodeStatus s4_0ImmDecoder(MCInst &MI, unsigned tmp, uint64_t Address,
+                                   const void *Decoder);
+static DecodeStatus s4_1ImmDecoder(MCInst &MI, unsigned tmp, uint64_t Address,
+                                   const void *Decoder);
+static DecodeStatus s4_2ImmDecoder(MCInst &MI, unsigned tmp, uint64_t Address,
+                                   const void *Decoder);
+static DecodeStatus s4_3ImmDecoder(MCInst &MI, unsigned tmp, uint64_t Address,
+                                   const void *Decoder);
 
 static const uint16_t IntRegDecoderTable[] = {
-  Hexagon::R0, Hexagon::R1, Hexagon::R2, Hexagon::R3, Hexagon::R4,
-  Hexagon::R5, Hexagon::R6, Hexagon::R7, Hexagon::R8, Hexagon::R9,
-  Hexagon::R10, Hexagon::R11, Hexagon::R12, Hexagon::R13, Hexagon::R14,
-  Hexagon::R15, Hexagon::R16, Hexagon::R17, Hexagon::R18, Hexagon::R19,
-  Hexagon::R20, Hexagon::R21, Hexagon::R22, Hexagon::R23, Hexagon::R24,
-  Hexagon::R25, Hexagon::R26, Hexagon::R27, Hexagon::R28, Hexagon::R29,
-  Hexagon::R30, Hexagon::R31 };
+    Hexagon::R0,  Hexagon::R1,  Hexagon::R2,  Hexagon::R3,  Hexagon::R4,
+    Hexagon::R5,  Hexagon::R6,  Hexagon::R7,  Hexagon::R8,  Hexagon::R9,
+    Hexagon::R10, Hexagon::R11, Hexagon::R12, Hexagon::R13, Hexagon::R14,
+    Hexagon::R15, Hexagon::R16, Hexagon::R17, Hexagon::R18, Hexagon::R19,
+    Hexagon::R20, Hexagon::R21, Hexagon::R22, Hexagon::R23, Hexagon::R24,
+    Hexagon::R25, Hexagon::R26, Hexagon::R27, Hexagon::R28, Hexagon::R29,
+    Hexagon::R30, Hexagon::R31};
 
-static const uint16_t PredRegDecoderTable[] = { Hexagon::P0, Hexagon::P1,
-Hexagon::P2, Hexagon::P3 };
+static const uint16_t PredRegDecoderTable[] = {Hexagon::P0, Hexagon::P1,
+                                               Hexagon::P2, Hexagon::P3};
 
 static DecodeStatus DecodeRegisterClass(MCInst &Inst, unsigned RegNo,
-  const uint16_t Table[], size_t Size) {
+                                        const uint16_t Table[], size_t Size) {
   if (RegNo < Size) {
-    Inst.addOperand(MCOperand::CreateReg(Table[RegNo]));
+    Inst.addOperand(MCOperand::createReg(Table[RegNo]));
     return MCDisassembler::Success;
-  }
-  else
+  } else
     return MCDisassembler::Fail;
 }
 
 static DecodeStatus DecodeIntRegsRegisterClass(MCInst &Inst, unsigned RegNo,
-  uint64_t /*Address*/,
-  void const *Decoder) {
+                                               uint64_t /*Address*/,
+                                               void const *Decoder) {
   if (RegNo > 31)
     return MCDisassembler::Fail;
 
   unsigned Register = IntRegDecoderTable[RegNo];
-  Inst.addOperand(MCOperand::CreateReg(Register));
+  Inst.addOperand(MCOperand::createReg(Register));
   return MCDisassembler::Success;
 }
 
 static DecodeStatus DecodeCtrRegsRegisterClass(MCInst &Inst, unsigned RegNo,
-  uint64_t /*Address*/, const void *Decoder) {
+                                               uint64_t /*Address*/,
+                                               const void *Decoder) {
   static const uint16_t CtrlRegDecoderTable[] = {
-    Hexagon::SA0, Hexagon::LC0, Hexagon::SA1, Hexagon::LC1,
-    Hexagon::P3_0, Hexagon::NoRegister, Hexagon::C6, Hexagon::C7,
-    Hexagon::USR, Hexagon::PC, Hexagon::UGP, Hexagon::GP,
-    Hexagon::CS0, Hexagon::CS1, Hexagon::UPCL, Hexagon::UPCH
-  };
+      Hexagon::SA0,  Hexagon::LC0,        Hexagon::SA1,  Hexagon::LC1,
+      Hexagon::P3_0, Hexagon::NoRegister, Hexagon::C6,   Hexagon::C7,
+      Hexagon::USR,  Hexagon::PC,         Hexagon::UGP,  Hexagon::GP,
+      Hexagon::CS0,  Hexagon::CS1,        Hexagon::UPCL, Hexagon::UPCH};
 
   if (RegNo >= sizeof(CtrlRegDecoderTable) / sizeof(CtrlRegDecoderTable[0]))
     return MCDisassembler::Fail;
@@ -103,22 +144,20 @@ static DecodeStatus DecodeCtrRegsRegisterClass(MCInst &Inst, unsigned RegNo,
     return MCDisassembler::Fail;
 
   unsigned Register = CtrlRegDecoderTable[RegNo];
-  Inst.addOperand(MCOperand::CreateReg(Register));
+  Inst.addOperand(MCOperand::createReg(Register));
   return MCDisassembler::Success;
 }
 
 static DecodeStatus DecodeCtrRegs64RegisterClass(MCInst &Inst, unsigned RegNo,
-                                   uint64_t /*Address*/, void const *Decoder) {
+                                                 uint64_t /*Address*/,
+                                                 void const *Decoder) {
   static const uint16_t CtrlReg64DecoderTable[] = {
-    Hexagon::C1_0, Hexagon::NoRegister,
-    Hexagon::C3_2, Hexagon::NoRegister,
-    Hexagon::NoRegister, Hexagon::NoRegister,
-    Hexagon::C7_6, Hexagon::NoRegister,
-    Hexagon::C9_8, Hexagon::NoRegister,
-    Hexagon::C11_10, Hexagon::NoRegister,
-    Hexagon::CS, Hexagon::NoRegister,
-    Hexagon::UPC, Hexagon::NoRegister
-  };
+      Hexagon::C1_0,       Hexagon::NoRegister, Hexagon::C3_2,
+      Hexagon::NoRegister, Hexagon::NoRegister, Hexagon::NoRegister,
+      Hexagon::C7_6,       Hexagon::NoRegister, Hexagon::C9_8,
+      Hexagon::NoRegister, Hexagon::C11_10,     Hexagon::NoRegister,
+      Hexagon::CS,         Hexagon::NoRegister, Hexagon::UPC,
+      Hexagon::NoRegister};
 
   if (RegNo >= sizeof(CtrlReg64DecoderTable) / sizeof(CtrlReg64DecoderTable[0]))
     return MCDisassembler::Fail;
@@ -127,12 +166,13 @@ static DecodeStatus DecodeCtrRegs64RegisterClass(MCInst &Inst, unsigned RegNo,
     return MCDisassembler::Fail;
 
   unsigned Register = CtrlReg64DecoderTable[RegNo];
-  Inst.addOperand(MCOperand::CreateReg(Register));
+  Inst.addOperand(MCOperand::createReg(Register));
   return MCDisassembler::Success;
 }
 
 static DecodeStatus DecodeModRegsRegisterClass(MCInst &Inst, unsigned RegNo,
-  uint64_t /*Address*/, const void *Decoder) {
+                                               uint64_t /*Address*/,
+                                               const void *Decoder) {
   unsigned Register = 0;
   switch (RegNo) {
   case 0:
@@ -144,32 +184,31 @@ static DecodeStatus DecodeModRegsRegisterClass(MCInst &Inst, unsigned RegNo,
   default:
     return MCDisassembler::Fail;
   }
-  Inst.addOperand(MCOperand::CreateReg(Register));
+  Inst.addOperand(MCOperand::createReg(Register));
   return MCDisassembler::Success;
 }
 
 static DecodeStatus DecodeDoubleRegsRegisterClass(MCInst &Inst, unsigned RegNo,
-  uint64_t /*Address*/, const void *Decoder) {
+                                                  uint64_t /*Address*/,
+                                                  const void *Decoder) {
   static const uint16_t DoubleRegDecoderTable[] = {
-    Hexagon::D0, Hexagon::D1, Hexagon::D2, Hexagon::D3,
-    Hexagon::D4, Hexagon::D5, Hexagon::D6, Hexagon::D7,
-    Hexagon::D8, Hexagon::D9, Hexagon::D10, Hexagon::D11,
-    Hexagon::D12, Hexagon::D13, Hexagon::D14, Hexagon::D15
-  };
+      Hexagon::D0,  Hexagon::D1,  Hexagon::D2,  Hexagon::D3,
+      Hexagon::D4,  Hexagon::D5,  Hexagon::D6,  Hexagon::D7,
+      Hexagon::D8,  Hexagon::D9,  Hexagon::D10, Hexagon::D11,
+      Hexagon::D12, Hexagon::D13, Hexagon::D14, Hexagon::D15};
 
-  return (DecodeRegisterClass(Inst, RegNo >> 1,
-    DoubleRegDecoderTable,
-    sizeof (DoubleRegDecoderTable)));
+  return (DecodeRegisterClass(Inst, RegNo >> 1, DoubleRegDecoderTable,
+                              sizeof(DoubleRegDecoderTable)));
 }
 
 static DecodeStatus DecodePredRegsRegisterClass(MCInst &Inst, unsigned RegNo,
-  uint64_t /*Address*/,
-  void const *Decoder) {
+                                                uint64_t /*Address*/,
+                                                void const *Decoder) {
   if (RegNo > 3)
     return MCDisassembler::Fail;
 
   unsigned Register = PredRegDecoderTable[RegNo];
-  Inst.addOperand(MCOperand::CreateReg(Register));
+  Inst.addOperand(MCOperand::createReg(Register));
   return MCDisassembler::Success;
 }
 
@@ -191,17 +230,778 @@ DecodeStatus HexagonDisassembler::getInstruction(MCInst &MI, uint64_t &Size,
                                                  uint64_t Address,
                                                  raw_ostream &os,
                                                  raw_ostream &cs) const {
-  Size = 4;
-  if (Bytes.size() < 4)
-    return MCDisassembler::Fail;
+  DecodeStatus Result = DecodeStatus::Success;
+  bool Complete = false;
+  Size = 0;
 
-  uint32_t insn =
+  *CurrentBundle = &MI;
+  MI.setOpcode(Hexagon::BUNDLE);
+  MI.addOperand(MCOperand::createImm(0));
+  while (Result == Success && Complete == false) {
+    if (Bytes.size() < HEXAGON_INSTR_SIZE)
+      return MCDisassembler::Fail;
+    MCInst *Inst = new (getContext()) MCInst;
+    Result = getSingleInstruction(*Inst, MI, Bytes, Address, os, cs, Complete);
+    MI.addOperand(MCOperand::createInst(Inst));
+    Size += HEXAGON_INSTR_SIZE;
+    Bytes = Bytes.slice(HEXAGON_INSTR_SIZE);
+  }
+  return Result;
+}
+
+DecodeStatus HexagonDisassembler::getSingleInstruction(
+    MCInst &MI, MCInst &MCB, ArrayRef<uint8_t> Bytes, uint64_t Address,
+    raw_ostream &os, raw_ostream &cs, bool &Complete) const {
+  assert(Bytes.size() >= HEXAGON_INSTR_SIZE);
+
+  uint32_t Instruction =
       llvm::support::endian::read<uint32_t, llvm::support::little,
                                   llvm::support::unaligned>(Bytes.data());
 
-  // Remove parse bits.
-  insn &= ~static_cast<uint32_t>(HexagonII::InstParseBits::INST_PARSE_MASK);
-  DecodeStatus Result = decodeInstruction(DecoderTable32, MI, insn, Address, this, STI);
-  HexagonMCInstrInfo::AppendImplicitOperands(MI);
+  auto BundleSize = HexagonMCInstrInfo::bundleSize(MCB);
+  if ((Instruction & HexagonII::INST_PARSE_MASK) ==
+      HexagonII::INST_PARSE_LOOP_END) {
+    if (BundleSize == 0)
+      HexagonMCInstrInfo::setInnerLoop(MCB);
+    else if (BundleSize == 1)
+      HexagonMCInstrInfo::setOuterLoop(MCB);
+    else
+      return DecodeStatus::Fail;
+  }
+
+  DecodeStatus Result = DecodeStatus::Success;
+  if ((Instruction & HexagonII::INST_PARSE_MASK) ==
+      HexagonII::INST_PARSE_DUPLEX) {
+    // Determine the instruction class of each instruction in the duplex.
+    unsigned duplexIClass, IClassLow, IClassHigh;
+
+    duplexIClass = ((Instruction >> 28) & 0xe) | ((Instruction >> 13) & 0x1);
+    switch (duplexIClass) {
+    default:
+      return MCDisassembler::Fail;
+    case 0:
+      IClassLow = HexagonII::HSIG_L1;
+      IClassHigh = HexagonII::HSIG_L1;
+      break;
+    case 1:
+      IClassLow = HexagonII::HSIG_L2;
+      IClassHigh = HexagonII::HSIG_L1;
+      break;
+    case 2:
+      IClassLow = HexagonII::HSIG_L2;
+      IClassHigh = HexagonII::HSIG_L2;
+      break;
+    case 3:
+      IClassLow = HexagonII::HSIG_A;
+      IClassHigh = HexagonII::HSIG_A;
+      break;
+    case 4:
+      IClassLow = HexagonII::HSIG_L1;
+      IClassHigh = HexagonII::HSIG_A;
+      break;
+    case 5:
+      IClassLow = HexagonII::HSIG_L2;
+      IClassHigh = HexagonII::HSIG_A;
+      break;
+    case 6:
+      IClassLow = HexagonII::HSIG_S1;
+      IClassHigh = HexagonII::HSIG_A;
+      break;
+    case 7:
+      IClassLow = HexagonII::HSIG_S2;
+      IClassHigh = HexagonII::HSIG_A;
+      break;
+    case 8:
+      IClassLow = HexagonII::HSIG_S1;
+      IClassHigh = HexagonII::HSIG_L1;
+      break;
+    case 9:
+      IClassLow = HexagonII::HSIG_S1;
+      IClassHigh = HexagonII::HSIG_L2;
+      break;
+    case 10:
+      IClassLow = HexagonII::HSIG_S1;
+      IClassHigh = HexagonII::HSIG_S1;
+      break;
+    case 11:
+      IClassLow = HexagonII::HSIG_S2;
+      IClassHigh = HexagonII::HSIG_S1;
+      break;
+    case 12:
+      IClassLow = HexagonII::HSIG_S2;
+      IClassHigh = HexagonII::HSIG_L1;
+      break;
+    case 13:
+      IClassLow = HexagonII::HSIG_S2;
+      IClassHigh = HexagonII::HSIG_L2;
+      break;
+    case 14:
+      IClassLow = HexagonII::HSIG_S2;
+      IClassHigh = HexagonII::HSIG_S2;
+      break;
+    }
+
+    // Set the MCInst to be a duplex instruction. Which one doesn't matter.
+    MI.setOpcode(Hexagon::DuplexIClass0);
+
+    // Decode each instruction in the duplex.
+    // Create an MCInst for each instruction.
+    unsigned instLow = Instruction & 0x1fff;
+    unsigned instHigh = (Instruction >> 16) & 0x1fff;
+    unsigned opLow;
+    if (GetSubinstOpcode(IClassLow, instLow, opLow, os) !=
+        MCDisassembler::Success)
+      return MCDisassembler::Fail;
+    unsigned opHigh;
+    if (GetSubinstOpcode(IClassHigh, instHigh, opHigh, os) !=
+        MCDisassembler::Success)
+      return MCDisassembler::Fail;
+    MCInst *MILow = new (getContext()) MCInst;
+    MILow->setOpcode(opLow);
+    MCInst *MIHigh = new (getContext()) MCInst;
+    MIHigh->setOpcode(opHigh);
+    AddSubinstOperands(MILow, opLow, instLow);
+    AddSubinstOperands(MIHigh, opHigh, instHigh);
+    // see ConvertToSubInst() in
+    // lib/Target/Hexagon/MCTargetDesc/HexagonMCDuplexInfo.cpp
+
+    // Add the duplex instruction MCInsts as operands to the passed in MCInst.
+    MCOperand OPLow = MCOperand::createInst(MILow);
+    MCOperand OPHigh = MCOperand::createInst(MIHigh);
+    MI.addOperand(OPLow);
+    MI.addOperand(OPHigh);
+    Complete = true;
+  } else {
+    if ((Instruction & HexagonII::INST_PARSE_MASK) ==
+        HexagonII::INST_PARSE_PACKET_END)
+      Complete = true;
+    // Calling the auto-generated decoder function.
+    Result =
+        decodeInstruction(DecoderTable32, MI, Instruction, Address, this, STI);
+  }
+
   return Result;
 }
+
+static DecodeStatus s16ImmDecoder(MCInst &MI, unsigned tmp,
+                                  uint64_t /*Address*/, const void *Decoder) {
+  uint64_t imm = SignExtend64<16>(tmp);
+  MI.addOperand(MCOperand::createImm(imm));
+  return MCDisassembler::Success;
+}
+
+static DecodeStatus s12ImmDecoder(MCInst &MI, unsigned tmp,
+                                  uint64_t /*Address*/, const void *Decoder) {
+  uint64_t imm = SignExtend64<12>(tmp);
+  MI.addOperand(MCOperand::createImm(imm));
+  return MCDisassembler::Success;
+}
+
+static DecodeStatus s11_0ImmDecoder(MCInst &MI, unsigned tmp,
+                                    uint64_t /*Address*/, const void *Decoder) {
+  uint64_t imm = SignExtend64<11>(tmp);
+  MI.addOperand(MCOperand::createImm(imm));
+  return MCDisassembler::Success;
+}
+
+static DecodeStatus s11_1ImmDecoder(MCInst &MI, unsigned tmp,
+                                    uint64_t /*Address*/, const void *Decoder) {
+  uint64_t imm = SignExtend64<12>(tmp);
+  MI.addOperand(MCOperand::createImm(imm));
+  return MCDisassembler::Success;
+}
+
+static DecodeStatus s11_2ImmDecoder(MCInst &MI, unsigned tmp,
+                                    uint64_t /*Address*/, const void *Decoder) {
+  uint64_t imm = SignExtend64<13>(tmp);
+  MI.addOperand(MCOperand::createImm(imm));
+  return MCDisassembler::Success;
+}
+
+static DecodeStatus s11_3ImmDecoder(MCInst &MI, unsigned tmp,
+                                    uint64_t /*Address*/, const void *Decoder) {
+  uint64_t imm = SignExtend64<14>(tmp);
+  MI.addOperand(MCOperand::createImm(imm));
+  return MCDisassembler::Success;
+}
+
+static DecodeStatus s10ImmDecoder(MCInst &MI, unsigned tmp,
+                                  uint64_t /*Address*/, const void *Decoder) {
+  uint64_t imm = SignExtend64<10>(tmp);
+  MI.addOperand(MCOperand::createImm(imm));
+  return MCDisassembler::Success;
+}
+
+static DecodeStatus s8ImmDecoder(MCInst &MI, unsigned tmp, uint64_t /*Address*/,
+                                 const void *Decoder) {
+  uint64_t imm = SignExtend64<8>(tmp);
+  MI.addOperand(MCOperand::createImm(imm));
+  return MCDisassembler::Success;
+}
+
+static DecodeStatus s6_0ImmDecoder(MCInst &MI, unsigned tmp,
+                                   uint64_t /*Address*/, const void *Decoder) {
+  uint64_t imm = SignExtend64<6>(tmp);
+  MI.addOperand(MCOperand::createImm(imm));
+  return MCDisassembler::Success;
+}
+
+static DecodeStatus s4_0ImmDecoder(MCInst &MI, unsigned tmp,
+                                   uint64_t /*Address*/, const void *Decoder) {
+  uint64_t imm = SignExtend64<4>(tmp);
+  MI.addOperand(MCOperand::createImm(imm));
+  return MCDisassembler::Success;
+}
+
+static DecodeStatus s4_1ImmDecoder(MCInst &MI, unsigned tmp,
+                                   uint64_t /*Address*/, const void *Decoder) {
+  uint64_t imm = SignExtend64<5>(tmp);
+  MI.addOperand(MCOperand::createImm(imm));
+  return MCDisassembler::Success;
+}
+
+static DecodeStatus s4_2ImmDecoder(MCInst &MI, unsigned tmp,
+                                   uint64_t /*Address*/, const void *Decoder) {
+  uint64_t imm = SignExtend64<6>(tmp);
+  MI.addOperand(MCOperand::createImm(imm));
+  return MCDisassembler::Success;
+}
+
+static DecodeStatus s4_3ImmDecoder(MCInst &MI, unsigned tmp,
+                                   uint64_t /*Address*/, const void *Decoder) {
+  uint64_t imm = SignExtend64<7>(tmp);
+  MI.addOperand(MCOperand::createImm(imm));
+  return MCDisassembler::Success;
+}
+
+// These values are from HexagonGenMCCodeEmitter.inc and HexagonIsetDx.td
+enum subInstBinaryValues {
+  V4_SA1_addi_BITS = 0x0000,
+  V4_SA1_addi_MASK = 0x1800,
+  V4_SA1_addrx_BITS = 0x1800,
+  V4_SA1_addrx_MASK = 0x1f00,
+  V4_SA1_addsp_BITS = 0x0c00,
+  V4_SA1_addsp_MASK = 0x1c00,
+  V4_SA1_and1_BITS = 0x1200,
+  V4_SA1_and1_MASK = 0x1f00,
+  V4_SA1_clrf_BITS = 0x1a70,
+  V4_SA1_clrf_MASK = 0x1e70,
+  V4_SA1_clrfnew_BITS = 0x1a50,
+  V4_SA1_clrfnew_MASK = 0x1e70,
+  V4_SA1_clrt_BITS = 0x1a60,
+  V4_SA1_clrt_MASK = 0x1e70,
+  V4_SA1_clrtnew_BITS = 0x1a40,
+  V4_SA1_clrtnew_MASK = 0x1e70,
+  V4_SA1_cmpeqi_BITS = 0x1900,
+  V4_SA1_cmpeqi_MASK = 0x1f00,
+  V4_SA1_combine0i_BITS = 0x1c00,
+  V4_SA1_combine0i_MASK = 0x1d18,
+  V4_SA1_combine1i_BITS = 0x1c08,
+  V4_SA1_combine1i_MASK = 0x1d18,
+  V4_SA1_combine2i_BITS = 0x1c10,
+  V4_SA1_combine2i_MASK = 0x1d18,
+  V4_SA1_combine3i_BITS = 0x1c18,
+  V4_SA1_combine3i_MASK = 0x1d18,
+  V4_SA1_combinerz_BITS = 0x1d08,
+  V4_SA1_combinerz_MASK = 0x1d08,
+  V4_SA1_combinezr_BITS = 0x1d00,
+  V4_SA1_combinezr_MASK = 0x1d08,
+  V4_SA1_dec_BITS = 0x1300,
+  V4_SA1_dec_MASK = 0x1f00,
+  V4_SA1_inc_BITS = 0x1100,
+  V4_SA1_inc_MASK = 0x1f00,
+  V4_SA1_seti_BITS = 0x0800,
+  V4_SA1_seti_MASK = 0x1c00,
+  V4_SA1_setin1_BITS = 0x1a00,
+  V4_SA1_setin1_MASK = 0x1e40,
+  V4_SA1_sxtb_BITS = 0x1500,
+  V4_SA1_sxtb_MASK = 0x1f00,
+  V4_SA1_sxth_BITS = 0x1400,
+  V4_SA1_sxth_MASK = 0x1f00,
+  V4_SA1_tfr_BITS = 0x1000,
+  V4_SA1_tfr_MASK = 0x1f00,
+  V4_SA1_zxtb_BITS = 0x1700,
+  V4_SA1_zxtb_MASK = 0x1f00,
+  V4_SA1_zxth_BITS = 0x1600,
+  V4_SA1_zxth_MASK = 0x1f00,
+  V4_SL1_loadri_io_BITS = 0x0000,
+  V4_SL1_loadri_io_MASK = 0x1000,
+  V4_SL1_loadrub_io_BITS = 0x1000,
+  V4_SL1_loadrub_io_MASK = 0x1000,
+  V4_SL2_deallocframe_BITS = 0x1f00,
+  V4_SL2_deallocframe_MASK = 0x1fc0,
+  V4_SL2_jumpr31_BITS = 0x1fc0,
+  V4_SL2_jumpr31_MASK = 0x1fc4,
+  V4_SL2_jumpr31_f_BITS = 0x1fc5,
+  V4_SL2_jumpr31_f_MASK = 0x1fc7,
+  V4_SL2_jumpr31_fnew_BITS = 0x1fc7,
+  V4_SL2_jumpr31_fnew_MASK = 0x1fc7,
+  V4_SL2_jumpr31_t_BITS = 0x1fc4,
+  V4_SL2_jumpr31_t_MASK = 0x1fc7,
+  V4_SL2_jumpr31_tnew_BITS = 0x1fc6,
+  V4_SL2_jumpr31_tnew_MASK = 0x1fc7,
+  V4_SL2_loadrb_io_BITS = 0x1000,
+  V4_SL2_loadrb_io_MASK = 0x1800,
+  V4_SL2_loadrd_sp_BITS = 0x1e00,
+  V4_SL2_loadrd_sp_MASK = 0x1f00,
+  V4_SL2_loadrh_io_BITS = 0x0000,
+  V4_SL2_loadrh_io_MASK = 0x1800,
+  V4_SL2_loadri_sp_BITS = 0x1c00,
+  V4_SL2_loadri_sp_MASK = 0x1e00,
+  V4_SL2_loadruh_io_BITS = 0x0800,
+  V4_SL2_loadruh_io_MASK = 0x1800,
+  V4_SL2_return_BITS = 0x1f40,
+  V4_SL2_return_MASK = 0x1fc4,
+  V4_SL2_return_f_BITS = 0x1f45,
+  V4_SL2_return_f_MASK = 0x1fc7,
+  V4_SL2_return_fnew_BITS = 0x1f47,
+  V4_SL2_return_fnew_MASK = 0x1fc7,
+  V4_SL2_return_t_BITS = 0x1f44,
+  V4_SL2_return_t_MASK = 0x1fc7,
+  V4_SL2_return_tnew_BITS = 0x1f46,
+  V4_SL2_return_tnew_MASK = 0x1fc7,
+  V4_SS1_storeb_io_BITS = 0x1000,
+  V4_SS1_storeb_io_MASK = 0x1000,
+  V4_SS1_storew_io_BITS = 0x0000,
+  V4_SS1_storew_io_MASK = 0x1000,
+  V4_SS2_allocframe_BITS = 0x1c00,
+  V4_SS2_allocframe_MASK = 0x1e00,
+  V4_SS2_storebi0_BITS = 0x1200,
+  V4_SS2_storebi0_MASK = 0x1f00,
+  V4_SS2_storebi1_BITS = 0x1300,
+  V4_SS2_storebi1_MASK = 0x1f00,
+  V4_SS2_stored_sp_BITS = 0x0a00,
+  V4_SS2_stored_sp_MASK = 0x1e00,
+  V4_SS2_storeh_io_BITS = 0x0000,
+  V4_SS2_storeh_io_MASK = 0x1800,
+  V4_SS2_storew_sp_BITS = 0x0800,
+  V4_SS2_storew_sp_MASK = 0x1e00,
+  V4_SS2_storewi0_BITS = 0x1000,
+  V4_SS2_storewi0_MASK = 0x1f00,
+  V4_SS2_storewi1_BITS = 0x1100,
+  V4_SS2_storewi1_MASK = 0x1f00
+};
+
+static unsigned GetSubinstOpcode(unsigned IClass, unsigned inst, unsigned &op,
+                                 raw_ostream &os) {
+  switch (IClass) {
+  case HexagonII::HSIG_L1:
+    if ((inst & V4_SL1_loadri_io_MASK) == V4_SL1_loadri_io_BITS)
+      op = Hexagon::V4_SL1_loadri_io;
+    else if ((inst & V4_SL1_loadrub_io_MASK) == V4_SL1_loadrub_io_BITS)
+      op = Hexagon::V4_SL1_loadrub_io;
+    else {
+      os << "<unknown subinstruction>";
+      return MCDisassembler::Fail;
+    }
+    break;
+  case HexagonII::HSIG_L2:
+    if ((inst & V4_SL2_deallocframe_MASK) == V4_SL2_deallocframe_BITS)
+      op = Hexagon::V4_SL2_deallocframe;
+    else if ((inst & V4_SL2_jumpr31_MASK) == V4_SL2_jumpr31_BITS)
+      op = Hexagon::V4_SL2_jumpr31;
+    else if ((inst & V4_SL2_jumpr31_f_MASK) == V4_SL2_jumpr31_f_BITS)
+      op = Hexagon::V4_SL2_jumpr31_f;
+    else if ((inst & V4_SL2_jumpr31_fnew_MASK) == V4_SL2_jumpr31_fnew_BITS)
+      op = Hexagon::V4_SL2_jumpr31_fnew;
+    else if ((inst & V4_SL2_jumpr31_t_MASK) == V4_SL2_jumpr31_t_BITS)
+      op = Hexagon::V4_SL2_jumpr31_t;
+    else if ((inst & V4_SL2_jumpr31_tnew_MASK) == V4_SL2_jumpr31_tnew_BITS)
+      op = Hexagon::V4_SL2_jumpr31_tnew;
+    else if ((inst & V4_SL2_loadrb_io_MASK) == V4_SL2_loadrb_io_BITS)
+      op = Hexagon::V4_SL2_loadrb_io;
+    else if ((inst & V4_SL2_loadrd_sp_MASK) == V4_SL2_loadrd_sp_BITS)
+      op = Hexagon::V4_SL2_loadrd_sp;
+    else if ((inst & V4_SL2_loadrh_io_MASK) == V4_SL2_loadrh_io_BITS)
+      op = Hexagon::V4_SL2_loadrh_io;
+    else if ((inst & V4_SL2_loadri_sp_MASK) == V4_SL2_loadri_sp_BITS)
+      op = Hexagon::V4_SL2_loadri_sp;
+    else if ((inst & V4_SL2_loadruh_io_MASK) == V4_SL2_loadruh_io_BITS)
+      op = Hexagon::V4_SL2_loadruh_io;
+    else if ((inst & V4_SL2_return_MASK) == V4_SL2_return_BITS)
+      op = Hexagon::V4_SL2_return;
+    else if ((inst & V4_SL2_return_f_MASK) == V4_SL2_return_f_BITS)
+      op = Hexagon::V4_SL2_return_f;
+    else if ((inst & V4_SL2_return_fnew_MASK) == V4_SL2_return_fnew_BITS)
+      op = Hexagon::V4_SL2_return_fnew;
+    else if ((inst & V4_SL2_return_t_MASK) == V4_SL2_return_t_BITS)
+      op = Hexagon::V4_SL2_return_t;
+    else if ((inst & V4_SL2_return_tnew_MASK) == V4_SL2_return_tnew_BITS)
+      op = Hexagon::V4_SL2_return_tnew;
+    else {
+      os << "<unknown subinstruction>";
+      return MCDisassembler::Fail;
+    }
+    break;
+  case HexagonII::HSIG_A:
+    if ((inst & V4_SA1_addi_MASK) == V4_SA1_addi_BITS)
+      op = Hexagon::V4_SA1_addi;
+    else if ((inst & V4_SA1_addrx_MASK) == V4_SA1_addrx_BITS)
+      op = Hexagon::V4_SA1_addrx;
+    else if ((inst & V4_SA1_addsp_MASK) == V4_SA1_addsp_BITS)
+      op = Hexagon::V4_SA1_addsp;
+    else if ((inst & V4_SA1_and1_MASK) == V4_SA1_and1_BITS)
+      op = Hexagon::V4_SA1_and1;
+    else if ((inst & V4_SA1_clrf_MASK) == V4_SA1_clrf_BITS)
+      op = Hexagon::V4_SA1_clrf;
+    else if ((inst & V4_SA1_clrfnew_MASK) == V4_SA1_clrfnew_BITS)
+      op = Hexagon::V4_SA1_clrfnew;
+    else if ((inst & V4_SA1_clrt_MASK) == V4_SA1_clrt_BITS)
+      op = Hexagon::V4_SA1_clrt;
+    else if ((inst & V4_SA1_clrtnew_MASK) == V4_SA1_clrtnew_BITS)
+      op = Hexagon::V4_SA1_clrtnew;
+    else if ((inst & V4_SA1_cmpeqi_MASK) == V4_SA1_cmpeqi_BITS)
+      op = Hexagon::V4_SA1_cmpeqi;
+    else if ((inst & V4_SA1_combine0i_MASK) == V4_SA1_combine0i_BITS)
+      op = Hexagon::V4_SA1_combine0i;
+    else if ((inst & V4_SA1_combine1i_MASK) == V4_SA1_combine1i_BITS)
+      op = Hexagon::V4_SA1_combine1i;
+    else if ((inst & V4_SA1_combine2i_MASK) == V4_SA1_combine2i_BITS)
+      op = Hexagon::V4_SA1_combine2i;
+    else if ((inst & V4_SA1_combine3i_MASK) == V4_SA1_combine3i_BITS)
+      op = Hexagon::V4_SA1_combine3i;
+    else if ((inst & V4_SA1_combinerz_MASK) == V4_SA1_combinerz_BITS)
+      op = Hexagon::V4_SA1_combinerz;
+    else if ((inst & V4_SA1_combinezr_MASK) == V4_SA1_combinezr_BITS)
+      op = Hexagon::V4_SA1_combinezr;
+    else if ((inst & V4_SA1_dec_MASK) == V4_SA1_dec_BITS)
+      op = Hexagon::V4_SA1_dec;
+    else if ((inst & V4_SA1_inc_MASK) == V4_SA1_inc_BITS)
+      op = Hexagon::V4_SA1_inc;
+    else if ((inst & V4_SA1_seti_MASK) == V4_SA1_seti_BITS)
+      op = Hexagon::V4_SA1_seti;
+    else if ((inst & V4_SA1_setin1_MASK) == V4_SA1_setin1_BITS)
+      op = Hexagon::V4_SA1_setin1;
+    else if ((inst & V4_SA1_sxtb_MASK) == V4_SA1_sxtb_BITS)
+      op = Hexagon::V4_SA1_sxtb;
+    else if ((inst & V4_SA1_sxth_MASK) == V4_SA1_sxth_BITS)
+      op = Hexagon::V4_SA1_sxth;
+    else if ((inst & V4_SA1_tfr_MASK) == V4_SA1_tfr_BITS)
+      op = Hexagon::V4_SA1_tfr;
+    else if ((inst & V4_SA1_zxtb_MASK) == V4_SA1_zxtb_BITS)
+      op = Hexagon::V4_SA1_zxtb;
+    else if ((inst & V4_SA1_zxth_MASK) == V4_SA1_zxth_BITS)
+      op = Hexagon::V4_SA1_zxth;
+    else {
+      os << "<unknown subinstruction>";
+      return MCDisassembler::Fail;
+    }
+    break;
+  case HexagonII::HSIG_S1:
+    if ((inst & V4_SS1_storeb_io_MASK) == V4_SS1_storeb_io_BITS)
+      op = Hexagon::V4_SS1_storeb_io;
+    else if ((inst & V4_SS1_storew_io_MASK) == V4_SS1_storew_io_BITS)
+      op = Hexagon::V4_SS1_storew_io;
+    else {
+      os << "<unknown subinstruction>";
+      return MCDisassembler::Fail;
+    }
+    break;
+  case HexagonII::HSIG_S2:
+    if ((inst & V4_SS2_allocframe_MASK) == V4_SS2_allocframe_BITS)
+      op = Hexagon::V4_SS2_allocframe;
+    else if ((inst & V4_SS2_storebi0_MASK) == V4_SS2_storebi0_BITS)
+      op = Hexagon::V4_SS2_storebi0;
+    else if ((inst & V4_SS2_storebi1_MASK) == V4_SS2_storebi1_BITS)
+      op = Hexagon::V4_SS2_storebi1;
+    else if ((inst & V4_SS2_stored_sp_MASK) == V4_SS2_stored_sp_BITS)
+      op = Hexagon::V4_SS2_stored_sp;
+    else if ((inst & V4_SS2_storeh_io_MASK) == V4_SS2_storeh_io_BITS)
+      op = Hexagon::V4_SS2_storeh_io;
+    else if ((inst & V4_SS2_storew_sp_MASK) == V4_SS2_storew_sp_BITS)
+      op = Hexagon::V4_SS2_storew_sp;
+    else if ((inst & V4_SS2_storewi0_MASK) == V4_SS2_storewi0_BITS)
+      op = Hexagon::V4_SS2_storewi0;
+    else if ((inst & V4_SS2_storewi1_MASK) == V4_SS2_storewi1_BITS)
+      op = Hexagon::V4_SS2_storewi1;
+    else {
+      os << "<unknown subinstruction>";
+      return MCDisassembler::Fail;
+    }
+    break;
+  default:
+    os << "<unknown>";
+    return MCDisassembler::Fail;
+  }
+  return MCDisassembler::Success;
+}
+
+static unsigned getRegFromSubinstEncoding(unsigned encoded_reg) {
+  if (encoded_reg < 8)
+    return Hexagon::R0 + encoded_reg;
+  else if (encoded_reg < 16)
+    return Hexagon::R0 + encoded_reg + 8;
+  return Hexagon::NoRegister;
+}
+
+static unsigned getDRegFromSubinstEncoding(unsigned encoded_dreg) {
+  if (encoded_dreg < 4)
+    return Hexagon::D0 + encoded_dreg;
+  else if (encoded_dreg < 8)
+    return Hexagon::D0 + encoded_dreg + 4;
+  return Hexagon::NoRegister;
+}
+
+static void AddSubinstOperands(MCInst *MI, unsigned opcode, unsigned inst) {
+  int64_t operand;
+  MCOperand Op;
+  switch (opcode) {
+  case Hexagon::V4_SL2_deallocframe:
+  case Hexagon::V4_SL2_jumpr31:
+  case Hexagon::V4_SL2_jumpr31_f:
+  case Hexagon::V4_SL2_jumpr31_fnew:
+  case Hexagon::V4_SL2_jumpr31_t:
+  case Hexagon::V4_SL2_jumpr31_tnew:
+  case Hexagon::V4_SL2_return:
+  case Hexagon::V4_SL2_return_f:
+  case Hexagon::V4_SL2_return_fnew:
+  case Hexagon::V4_SL2_return_t:
+  case Hexagon::V4_SL2_return_tnew:
+    // no operands for these instructions
+    break;
+  case Hexagon::V4_SS2_allocframe:
+    // u 8-4{5_3}
+    operand = ((inst & 0x1f0) >> 4) << 3;
+    Op = MCOperand::createImm(operand);
+    MI->addOperand(Op);
+    break;
+  case Hexagon::V4_SL1_loadri_io:
+    // Rd 3-0, Rs 7-4, u 11-8{4_2}
+    operand = getRegFromSubinstEncoding(inst & 0xf);
+    Op = MCOperand::createReg(operand);
+    MI->addOperand(Op);
+    operand = getRegFromSubinstEncoding((inst & 0xf0) >> 4);
+    Op = MCOperand::createReg(operand);
+    MI->addOperand(Op);
+    operand = (inst & 0xf00) >> 6;
+    Op = MCOperand::createImm(operand);
+    MI->addOperand(Op);
+    break;
+  case Hexagon::V4_SL1_loadrub_io:
+    // Rd 3-0, Rs 7-4, u 11-8
+    operand = getRegFromSubinstEncoding(inst & 0xf);
+    Op = MCOperand::createReg(operand);
+    MI->addOperand(Op);
+    operand = getRegFromSubinstEncoding((inst & 0xf0) >> 4);
+    Op = MCOperand::createReg(operand);
+    MI->addOperand(Op);
+    operand = (inst & 0xf00) >> 8;
+    Op = MCOperand::createImm(operand);
+    MI->addOperand(Op);
+    break;
+  case Hexagon::V4_SL2_loadrb_io:
+    // Rd 3-0, Rs 7-4, u 10-8
+    operand = getRegFromSubinstEncoding(inst & 0xf);
+    Op = MCOperand::createReg(operand);
+    MI->addOperand(Op);
+    operand = getRegFromSubinstEncoding((inst & 0xf0) >> 4);
+    Op = MCOperand::createReg(operand);
+    MI->addOperand(Op);
+    operand = (inst & 0x700) >> 8;
+    Op = MCOperand::createImm(operand);
+    MI->addOperand(Op);
+    break;
+  case Hexagon::V4_SL2_loadrh_io:
+  case Hexagon::V4_SL2_loadruh_io:
+    // Rd 3-0, Rs 7-4, u 10-8{3_1}
+    operand = getRegFromSubinstEncoding(inst & 0xf);
+    Op = MCOperand::createReg(operand);
+    MI->addOperand(Op);
+    operand = getRegFromSubinstEncoding((inst & 0xf0) >> 4);
+    Op = MCOperand::createReg(operand);
+    MI->addOperand(Op);
+    operand = ((inst & 0x700) >> 8) << 1;
+    Op = MCOperand::createImm(operand);
+    MI->addOperand(Op);
+    break;
+  case Hexagon::V4_SL2_loadrd_sp:
+    // Rdd 2-0, u 7-3{5_3}
+    operand = getDRegFromSubinstEncoding(inst & 0x7);
+    Op = MCOperand::createReg(operand);
+    MI->addOperand(Op);
+    operand = ((inst & 0x0f8) >> 3) << 3;
+    Op = MCOperand::createImm(operand);
+    MI->addOperand(Op);
+    break;
+  case Hexagon::V4_SL2_loadri_sp:
+    // Rd 3-0, u 8-4{5_2}
+    operand = getRegFromSubinstEncoding(inst & 0xf);
+    Op = MCOperand::createReg(operand);
+    MI->addOperand(Op);
+    operand = ((inst & 0x1f0) >> 4) << 2;
+    Op = MCOperand::createImm(operand);
+    MI->addOperand(Op);
+    break;
+  case Hexagon::V4_SA1_addi:
+    // Rx 3-0 (x2), s7 10-4
+    operand = getRegFromSubinstEncoding(inst & 0xf);
+    Op = MCOperand::createReg(operand);
+    MI->addOperand(Op);
+    MI->addOperand(Op);
+    operand = SignExtend64<7>((inst & 0x7f0) >> 4);
+    Op = MCOperand::createImm(operand);
+    MI->addOperand(Op);
+    break;
+  case Hexagon::V4_SA1_addrx:
+    // Rx 3-0 (x2), Rs 7-4
+    operand = getRegFromSubinstEncoding(inst & 0xf);
+    Op = MCOperand::createReg(operand);
+    MI->addOperand(Op);
+    MI->addOperand(Op);
+    operand = getRegFromSubinstEncoding((inst & 0xf0) >> 4);
+    Op = MCOperand::createReg(operand);
+    MI->addOperand(Op);
+  case Hexagon::V4_SA1_and1:
+  case Hexagon::V4_SA1_dec:
+  case Hexagon::V4_SA1_inc:
+  case Hexagon::V4_SA1_sxtb:
+  case Hexagon::V4_SA1_sxth:
+  case Hexagon::V4_SA1_tfr:
+  case Hexagon::V4_SA1_zxtb:
+  case Hexagon::V4_SA1_zxth:
+    // Rd 3-0, Rs 7-4
+    operand = getRegFromSubinstEncoding(inst & 0xf);
+    Op = MCOperand::createReg(operand);
+    MI->addOperand(Op);
+    operand = getRegFromSubinstEncoding((inst & 0xf0) >> 4);
+    Op = MCOperand::createReg(operand);
+    MI->addOperand(Op);
+    break;
+  case Hexagon::V4_SA1_addsp:
+    // Rd 3-0, u 9-4{6_2}
+    operand = getRegFromSubinstEncoding(inst & 0xf);
+    Op = MCOperand::createReg(operand);
+    MI->addOperand(Op);
+    operand = ((inst & 0x3f0) >> 4) << 2;
+    Op = MCOperand::createImm(operand);
+    MI->addOperand(Op);
+    break;
+  case Hexagon::V4_SA1_seti:
+    // Rd 3-0, u 9-4
+    operand = getRegFromSubinstEncoding(inst & 0xf);
+    Op = MCOperand::createReg(operand);
+    MI->addOperand(Op);
+    operand = (inst & 0x3f0) >> 4;
+    Op = MCOperand::createImm(operand);
+    MI->addOperand(Op);
+    break;
+  case Hexagon::V4_SA1_clrf:
+  case Hexagon::V4_SA1_clrfnew:
+  case Hexagon::V4_SA1_clrt:
+  case Hexagon::V4_SA1_clrtnew:
+  case Hexagon::V4_SA1_setin1:
+    // Rd 3-0
+    operand = getRegFromSubinstEncoding(inst & 0xf);
+    Op = MCOperand::createReg(operand);
+    MI->addOperand(Op);
+    break;
+  case Hexagon::V4_SA1_cmpeqi:
+    // Rs 7-4, u 1-0
+    operand = getRegFromSubinstEncoding((inst & 0xf0) >> 4);
+    Op = MCOperand::createReg(operand);
+    MI->addOperand(Op);
+    operand = inst & 0x3;
+    Op = MCOperand::createImm(operand);
+    MI->addOperand(Op);
+    break;
+  case Hexagon::V4_SA1_combine0i:
+  case Hexagon::V4_SA1_combine1i:
+  case Hexagon::V4_SA1_combine2i:
+  case Hexagon::V4_SA1_combine3i:
+    // Rdd 2-0, u 6-5
+    operand = getDRegFromSubinstEncoding(inst & 0x7);
+    Op = MCOperand::createReg(operand);
+    MI->addOperand(Op);
+    operand = (inst & 0x060) >> 5;
+    Op = MCOperand::createImm(operand);
+    MI->addOperand(Op);
+    break;
+  case Hexagon::V4_SA1_combinerz:
+  case Hexagon::V4_SA1_combinezr:
+    // Rdd 2-0, Rs 7-4
+    operand = getDRegFromSubinstEncoding(inst & 0x7);
+    Op = MCOperand::createReg(operand);
+    MI->addOperand(Op);
+    operand = getRegFromSubinstEncoding((inst & 0xf0) >> 4);
+    Op = MCOperand::createReg(operand);
+    MI->addOperand(Op);
+    break;
+  case Hexagon::V4_SS1_storeb_io:
+    // Rs 7-4, u 11-8, Rt 3-0
+    operand = getRegFromSubinstEncoding((inst & 0xf0) >> 4);
+    Op = MCOperand::createReg(operand);
+    MI->addOperand(Op);
+    operand = (inst & 0xf00) >> 8;
+    Op = MCOperand::createImm(operand);
+    MI->addOperand(Op);
+    operand = getRegFromSubinstEncoding(inst & 0xf);
+    Op = MCOperand::createReg(operand);
+    MI->addOperand(Op);
+    break;
+  case Hexagon::V4_SS1_storew_io:
+    // Rs 7-4, u 11-8{4_2}, Rt 3-0
+    operand = getRegFromSubinstEncoding((inst & 0xf0) >> 4);
+    Op = MCOperand::createReg(operand);
+    MI->addOperand(Op);
+    operand = ((inst & 0xf00) >> 8) << 2;
+    Op = MCOperand::createImm(operand);
+    MI->addOperand(Op);
+    operand = getRegFromSubinstEncoding(inst & 0xf);
+    Op = MCOperand::createReg(operand);
+    MI->addOperand(Op);
+    break;
+  case Hexagon::V4_SS2_storebi0:
+  case Hexagon::V4_SS2_storebi1:
+    // Rs 7-4, u 3-0
+    operand = getRegFromSubinstEncoding((inst & 0xf0) >> 4);
+    Op = MCOperand::createReg(operand);
+    MI->addOperand(Op);
+    operand = inst & 0xf;
+    Op = MCOperand::createImm(operand);
+    MI->addOperand(Op);
+    break;
+  case Hexagon::V4_SS2_storewi0:
+  case Hexagon::V4_SS2_storewi1:
+    // Rs 7-4, u 3-0{4_2}
+    operand = getRegFromSubinstEncoding((inst & 0xf0) >> 4);
+    Op = MCOperand::createReg(operand);
+    MI->addOperand(Op);
+    operand = (inst & 0xf) << 2;
+    Op = MCOperand::createImm(operand);
+    MI->addOperand(Op);
+    break;
+  case Hexagon::V4_SS2_stored_sp:
+    // s 8-3{6_3}, Rtt 2-0
+    operand = SignExtend64<9>(((inst & 0x1f8) >> 3) << 3);
+    Op = MCOperand::createImm(operand);
+    MI->addOperand(Op);
+    operand = getDRegFromSubinstEncoding(inst & 0x7);
+    Op = MCOperand::createReg(operand);
+    MI->addOperand(Op);
+  case Hexagon::V4_SS2_storeh_io:
+    // Rs 7-4, u 10-8{3_1}, Rt 3-0
+    operand = getRegFromSubinstEncoding((inst & 0xf0) >> 4);
+    Op = MCOperand::createReg(operand);
+    MI->addOperand(Op);
+    operand = ((inst & 0x700) >> 8) << 1;
+    Op = MCOperand::createImm(operand);
+    MI->addOperand(Op);
+    operand = getRegFromSubinstEncoding(inst & 0xf);
+    Op = MCOperand::createReg(operand);
+    MI->addOperand(Op);
+    break;
+  case Hexagon::V4_SS2_storew_sp:
+    // u 8-4{5_2}, Rd 3-0
+    operand = ((inst & 0x1f0) >> 4) << 2;
+    Op = MCOperand::createImm(operand);
+    MI->addOperand(Op);
+    operand = getRegFromSubinstEncoding(inst & 0xf);
+    Op = MCOperand::createReg(operand);
+    MI->addOperand(Op);
+    break;
+  default:
+    // don't crash with an invalid subinstruction
+    // llvm_unreachable("Invalid subinstruction in duplex instruction");
+    break;
+  }
+}
diff --git a/lib/Target/Hexagon/Disassembler/Makefile b/lib/Target/Hexagon/Disassembler/Makefile
index e55fd58524a..16c305fe407 100644
--- a/lib/Target/Hexagon/Disassembler/Makefile
+++ b/lib/Target/Hexagon/Disassembler/Makefile
@@ -1,16 +1,16 @@
-##===-- lib/Target/Hexagon/Disassembler/Makefile -----------*- Makefile -*-===##
-#
-#                     The LLVM Compiler Infrastructure
-#
-# This file is distributed under the University of Illinois Open Source
-# License. See LICENSE.TXT for details.
-#
-##===----------------------------------------------------------------------===##
-
-LEVEL = ../../../..
-LIBRARYNAME = LLVMHexagonDisassembler
-
-# Hack: we need to include 'main' target directory to grab private headers
-CPP.Flags += -I$(PROJ_OBJ_DIR)/.. -I$(PROJ_SRC_DIR)/..
-
-include $(LEVEL)/Makefile.common
+##===-- lib/Target/Hexagon/Disassembler/Makefile -----------*- Makefile -*-===##
+#
+#                     The LLVM Compiler Infrastructure
+#
+# This file is distributed under the University of Illinois Open Source
+# License. See LICENSE.TXT for details.
+#
+##===----------------------------------------------------------------------===##
+
+LEVEL = ../../../..
+LIBRARYNAME = LLVMHexagonDisassembler
+
+# Hack: we need to include 'main' target directory to grab private headers
+CPP.Flags += -I$(PROJ_OBJ_DIR)/.. -I$(PROJ_SRC_DIR)/..
+
+include $(LEVEL)/Makefile.common
diff --git a/lib/Target/Hexagon/Hexagon.h b/lib/Target/Hexagon/Hexagon.h
index dfe79f9ff7b..d360be2aa5b 100644
--- a/lib/Target/Hexagon/Hexagon.h
+++ b/lib/Target/Hexagon/Hexagon.h
@@ -15,50 +15,6 @@
 #ifndef LLVM_LIB_TARGET_HEXAGON_HEXAGON_H
 #define LLVM_LIB_TARGET_HEXAGON_HEXAGON_H
 
-#include "MCTargetDesc/HexagonMCTargetDesc.h"
-#include "llvm/Target/TargetLowering.h"
-#include "llvm/Target/TargetMachine.h"
-
-namespace llvm {
-  class FunctionPass;
-  class HexagonAsmPrinter;
-  class HexagonTargetMachine;
-  class MachineInstr;
-  class MCInst;
-  class ModulePass;
-  class raw_ostream;
-  class TargetMachine;
-
-  FunctionPass *createHexagonISelDag(HexagonTargetMachine &TM,
-                                     CodeGenOpt::Level OptLevel);
-  FunctionPass *createHexagonDelaySlotFillerPass(const TargetMachine &TM);
-  FunctionPass *createHexagonFPMoverPass(const TargetMachine &TM);
-  FunctionPass *createHexagonRemoveExtendArgs(const HexagonTargetMachine &TM);
-  FunctionPass *createHexagonCFGOptimizer();
-
-  FunctionPass *createHexagonSplitConst32AndConst64();
-  FunctionPass *createHexagonExpandPredSpillCode();
-  FunctionPass *createHexagonHardwareLoops();
-  FunctionPass *createHexagonPeephole();
-  FunctionPass *createHexagonFixupHwLoops();
-  FunctionPass *createHexagonNewValueJump();
-  FunctionPass *createHexagonCopyToCombine();
-  FunctionPass *createHexagonPacketizer();
-  FunctionPass *createHexagonNewValueJump();
-
-/* TODO: object output.
-  MCCodeEmitter *createHexagonMCCodeEmitter(const Target &,
-                                            const TargetMachine &TM,
-                                            MCContext &Ctx);
-*/
-/* TODO: assembler input.
-  TargetAsmBackend *createHexagonAsmBackend(const Target &,
-                                                  const std::string &);
-*/
-  void HexagonLowerToMC(MachineInstr const *MI, MCInst &MCI,
-                        HexagonAsmPrinter &AP);
-} // end namespace llvm;
-
 #define Hexagon_POINTER_SIZE 4
 
 #define Hexagon_PointerSize (Hexagon_POINTER_SIZE)
@@ -75,5 +31,33 @@ namespace llvm {
 
 // Maximum number of words and instructions in a packet.
 #define HEXAGON_PACKET_SIZE 4
+#define HEXAGON_MAX_PACKET_SIZE (HEXAGON_PACKET_SIZE * HEXAGON_INSTR_SIZE)
+// Minimum number of instructions in an end-loop packet.
+#define HEXAGON_PACKET_INNER_SIZE 2
+#define HEXAGON_PACKET_OUTER_SIZE 3
+// Maximum number of instructions in a packet before shuffling,
+// including a compound one or a duplex or an extender.
+#define HEXAGON_PRESHUFFLE_PACKET_SIZE (HEXAGON_PACKET_SIZE + 3)
+
+// Name of the global offset table as defined by the Hexagon ABI
+#define HEXAGON_GOT_SYM_NAME "_GLOBAL_OFFSET_TABLE_"
+
+#include "MCTargetDesc/HexagonMCTargetDesc.h"
+#include "llvm/Target/TargetLowering.h"
+#include "llvm/Target/TargetMachine.h"
+
+namespace llvm {
+  class MachineInstr;
+  class MCInst;
+  class MCInstrInfo;
+  class HexagonAsmPrinter;
+  class HexagonTargetMachine;
+
+  void HexagonLowerToMC(const MachineInstr *MI, MCInst &MCI,
+                        HexagonAsmPrinter &AP);
+
+  /// \brief Creates a Hexagon-specific Target Transformation Info pass.
+  ImmutablePass *createHexagonTargetTransformInfoPass(const HexagonTargetMachine *TM);
+} // end namespace llvm;
 
 #endif
diff --git a/lib/Target/Hexagon/HexagonAsmPrinter.cpp b/lib/Target/Hexagon/HexagonAsmPrinter.cpp
index f0c81e055bf..05728d2b627 100644
--- a/lib/Target/Hexagon/HexagonAsmPrinter.cpp
+++ b/lib/Target/Hexagon/HexagonAsmPrinter.cpp
@@ -20,6 +20,7 @@
 #include "HexagonTargetMachine.h"
 #include "MCTargetDesc/HexagonInstPrinter.h"
 #include "MCTargetDesc/HexagonMCInstrInfo.h"
+#include "MCTargetDesc/HexagonMCShuffler.h"
 #include "llvm/ADT/SmallString.h"
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/ADT/StringExtras.h"
@@ -78,14 +79,14 @@ void HexagonAsmPrinter::printOperand(const MachineInstr *MI, unsigned OpNo,
     O << MO.getImm();
     return;
   case MachineOperand::MO_MachineBasicBlock:
-    O << *MO.getMBB()->getSymbol();
+    MO.getMBB()->getSymbol()->print(O, MAI);
     return;
   case MachineOperand::MO_ConstantPoolIndex:
-    O << *GetCPISymbol(MO.getIndex());
+    GetCPISymbol(MO.getIndex())->print(O, MAI);
     return;
   case MachineOperand::MO_GlobalAddress:
     // Computing the address of a global symbol, not calling it.
-    O << *getSymbol(MO.getGlobal());
+    getSymbol(MO.getGlobal())->print(O, MAI);
     printOffset(MO.getOffset(), O);
     return;
   }
@@ -177,49 +178,40 @@ bool HexagonAsmPrinter::PrintAsmMemoryOperand(const MachineInstr *MI,
 /// the current output stream.
 ///
 void HexagonAsmPrinter::EmitInstruction(const MachineInstr *MI) {
-  if (MI->isBundle()) {
-    std::vector<MachineInstr const *> BundleMIs;
+  MCInst MCB;
+  MCB.setOpcode(Hexagon::BUNDLE);
+  MCB.addOperand(MCOperand::createImm(0));
 
-    const MachineBasicBlock *MBB = MI->getParent();
+  if (MI->isBundle()) {
+    const MachineBasicBlock* MBB = MI->getParent();
     MachineBasicBlock::const_instr_iterator MII = MI;
-    ++MII;
-    unsigned int IgnoreCount = 0;
-    while (MII != MBB->end() && MII->isInsideBundle()) {
-      const MachineInstr *MInst = MII;
-      if (MInst->getOpcode() == TargetOpcode::DBG_VALUE ||
-        MInst->getOpcode() == TargetOpcode::IMPLICIT_DEF) {
-        IgnoreCount++;
-        ++MII;
-        continue;
+    unsigned IgnoreCount = 0;
+
+    for (++MII; MII != MBB->end() && MII->isInsideBundle(); ++MII) {
+      if (MII->getOpcode() == TargetOpcode::DBG_VALUE ||
+          MII->getOpcode() == TargetOpcode::IMPLICIT_DEF)
+        ++IgnoreCount;
+      else {
+        HexagonLowerToMC(MII, MCB, *this);
       }
-      // BundleMIs.push_back(&*MII);
-      BundleMIs.push_back(MInst);
-      ++MII;
-    }
-    unsigned Size = BundleMIs.size();
-    assert((Size + IgnoreCount) == MI->getBundleSize() && "Corrupt Bundle!");
-    for (unsigned Index = 0; Index < Size; Index++) {
-      MCInst MCI;
-
-      HexagonLowerToMC(BundleMIs[Index], MCI, *this);
-      HexagonMCInstrInfo::AppendImplicitOperands(MCI);
-      HexagonMCInstrInfo::setPacketBegin(MCI, Index == 0);
-      HexagonMCInstrInfo::setPacketEnd(MCI, Index == (Size - 1));
-      EmitToStreamer(OutStreamer, MCI);
     }
   }
   else {
-    MCInst MCI;
-    HexagonLowerToMC(MI, MCI, *this);
-    HexagonMCInstrInfo::AppendImplicitOperands(MCI);
-    if (MI->getOpcode() == Hexagon::ENDLOOP0) {
-      HexagonMCInstrInfo::setPacketBegin(MCI, true);
-      HexagonMCInstrInfo::setPacketEnd(MCI, true);
-    }
-    EmitToStreamer(OutStreamer, MCI);
+    HexagonLowerToMC(MI, MCB, *this);
+    HexagonMCInstrInfo::padEndloop(MCB);
   }
-
-  return;
+  // Examine the packet and try to find instructions that can be converted
+  // to compounds.
+  HexagonMCInstrInfo::tryCompound(*Subtarget->getInstrInfo(),
+                                  OutStreamer->getContext(), MCB);
+  // Examine the packet and convert pairs of instructions to duplex
+  // instructions when possible.
+  SmallVector<DuplexCandidate, 8> possibleDuplexes;
+  possibleDuplexes = HexagonMCInstrInfo::getDuplexPossibilties(
+      *Subtarget->getInstrInfo(), MCB);
+  HexagonMCShuffle(*Subtarget->getInstrInfo(), *Subtarget,
+                   OutStreamer->getContext(), MCB, possibleDuplexes);
+  EmitToStreamer(*OutStreamer, MCB);
 }
 
 extern "C" void LLVMInitializeHexagonAsmPrinter() {
diff --git a/lib/Target/Hexagon/HexagonBitTracker.cpp b/lib/Target/Hexagon/HexagonBitTracker.cpp
new file mode 100644
index 00000000000..15163d43fba
--- /dev/null
+++ b/lib/Target/Hexagon/HexagonBitTracker.cpp
@@ -0,0 +1,1173 @@
+//===--- HexagonBitTracker.cpp --------------------------------------------===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/CodeGen/MachineRegisterInfo.h"
+#include "llvm/IR/Module.h"
+#include "llvm/Support/Debug.h"
+#include "llvm/Support/raw_ostream.h"
+
+#include "Hexagon.h"
+#include "HexagonInstrInfo.h"
+#include "HexagonRegisterInfo.h"
+#include "HexagonTargetMachine.h"
+#include "HexagonBitTracker.h"
+
+using namespace llvm;
+
+typedef BitTracker BT;
+
+HexagonEvaluator::HexagonEvaluator(const HexagonRegisterInfo &tri,
+                                   MachineRegisterInfo &mri,
+                                   const HexagonInstrInfo &tii,
+                                   MachineFunction &mf)
+    : MachineEvaluator(tri, mri), MF(mf), MFI(*mf.getFrameInfo()), TII(tii) {
+  // Populate the VRX map (VR to extension-type).
+  // Go over all the formal parameters of the function. If a given parameter
+  // P is sign- or zero-extended, locate the virtual register holding that
+  // parameter and create an entry in the VRX map indicating the type of ex-
+  // tension (and the source type).
+  // This is a bit complicated to do accurately, since the memory layout in-
+  // formation is necessary to precisely determine whether an aggregate para-
+  // meter will be passed in a register or in memory. What is given in MRI
+  // is the association between the physical register that is live-in (i.e.
+  // holds an argument), and the virtual register that this value will be
+  // copied into. This, by itself, is not sufficient to map back the virtual
+  // register to a formal parameter from Function (since consecutive live-ins
+  // from MRI may not correspond to consecutive formal parameters from Func-
+  // tion). To avoid the complications with in-memory arguments, only consi-
+  // der the initial sequence of formal parameters that are known to be
+  // passed via registers.
+  unsigned AttrIdx = 0;
+  unsigned InVirtReg, InPhysReg = 0;
+  const Function &F = *MF.getFunction();
+  typedef Function::const_arg_iterator arg_iterator;
+  for (arg_iterator I = F.arg_begin(), E = F.arg_end(); I != E; ++I) {
+    AttrIdx++;
+    const Argument &Arg = *I;
+    Type *ATy = Arg.getType();
+    unsigned Width = 0;
+    if (ATy->isIntegerTy())
+      Width = ATy->getIntegerBitWidth();
+    else if (ATy->isPointerTy())
+      Width = 32;
+    // If pointer size is not set through target data, it will default to
+    // Module::AnyPointerSize.
+    if (Width == 0 || Width > 64)
+      break;
+    InPhysReg = getNextPhysReg(InPhysReg, Width);
+    if (!InPhysReg)
+      break;
+    InVirtReg = getVirtRegFor(InPhysReg);
+    if (!InVirtReg)
+      continue;
+    AttributeSet Attrs = F.getAttributes();
+    if (Attrs.hasAttribute(AttrIdx, Attribute::SExt))
+      VRX.insert(std::make_pair(InVirtReg, ExtType(ExtType::SExt, Width)));
+    else if (Attrs.hasAttribute(AttrIdx, Attribute::ZExt))
+      VRX.insert(std::make_pair(InVirtReg, ExtType(ExtType::ZExt, Width)));
+  }
+}
+
+
+BT::BitMask HexagonEvaluator::mask(unsigned Reg, unsigned Sub) const {
+  if (Sub == 0)
+    return MachineEvaluator::mask(Reg, 0);
+  using namespace Hexagon;
+  const TargetRegisterClass *RC = MRI.getRegClass(Reg);
+  unsigned ID = RC->getID();
+  uint16_t RW = getRegBitWidth(RegisterRef(Reg, Sub));
+  switch (ID) {
+    case DoubleRegsRegClassID:
+      return (Sub == subreg_loreg) ? BT::BitMask(0, RW-1)
+                                   : BT::BitMask(RW, 2*RW-1);
+    default:
+      break;
+  }
+#ifndef NDEBUG
+  dbgs() << PrintReg(Reg, &TRI, Sub) << '\n';
+#endif
+  llvm_unreachable("Unexpected register/subregister");
+}
+
+namespace {
+class RegisterRefs {
+  std::vector<BT::RegisterRef> Vector;
+
+public:
+  RegisterRefs(const MachineInstr *MI) : Vector(MI->getNumOperands()) {
+    for (unsigned i = 0, n = Vector.size(); i < n; ++i) {
+      const MachineOperand &MO = MI->getOperand(i);
+      if (MO.isReg())
+        Vector[i] = BT::RegisterRef(MO);
+      // For indices that don't correspond to registers, the entry will
+      // remain constructed via the default constructor.
+    }
+  }
+
+  size_t size() const { return Vector.size(); }
+  const BT::RegisterRef &operator[](unsigned n) const {
+    // The main purpose of this operator is to assert with bad argument.
+    assert(n < Vector.size());
+    return Vector[n];
+  }
+};
+}
+
+bool HexagonEvaluator::evaluate(const MachineInstr *MI,
+      const CellMapType &Inputs, CellMapType &Outputs) const {
+  unsigned NumDefs = 0;
+
+  // Sanity verification: there should not be any defs with subregisters.
+  for (unsigned i = 0, n = MI->getNumOperands(); i < n; ++i) {
+    const MachineOperand &MO = MI->getOperand(i);
+    if (!MO.isReg() || !MO.isDef())
+      continue;
+    NumDefs++;
+    assert(MO.getSubReg() == 0);
+  }
+
+  if (NumDefs == 0)
+    return false;
+
+  if (MI->mayLoad())
+    return evaluateLoad(MI, Inputs, Outputs);
+
+  // Check COPY instructions that copy formal parameters into virtual
+  // registers. Such parameters can be sign- or zero-extended at the
+  // call site, and we should take advantage of this knowledge. The MRI
+  // keeps a list of pairs of live-in physical and virtual registers,
+  // which provides information about which virtual registers will hold
+  // the argument values. The function will still contain instructions
+  // defining those virtual registers, and in practice those are COPY
+  // instructions from a physical to a virtual register. In such cases,
+  // applying the argument extension to the virtual register can be seen
+  // as simply mirroring the extension that had already been applied to
+  // the physical register at the call site. If the defining instruction
+  // was not a COPY, it would not be clear how to mirror that extension
+  // on the callee's side. For that reason, only check COPY instructions
+  // for potential extensions.
+  if (MI->isCopy()) {
+    if (evaluateFormalCopy(MI, Inputs, Outputs))
+      return true;
+  }
+
+  // Beyond this point, if any operand is a global, skip that instruction.
+  // The reason is that certain instructions that can take an immediate
+  // operand can also have a global symbol in that operand. To avoid
+  // checking what kind of operand a given instruction has individually
+  // for each instruction, do it here. Global symbols as operands gene-
+  // rally do not provide any useful information.
+  for (unsigned i = 0, n = MI->getNumOperands(); i < n; ++i) {
+    const MachineOperand &MO = MI->getOperand(i);
+    if (MO.isGlobal() || MO.isBlockAddress() || MO.isSymbol() || MO.isJTI() ||
+        MO.isCPI())
+      return false;
+  }
+
+  RegisterRefs Reg(MI);
+  unsigned Opc = MI->getOpcode();
+  using namespace Hexagon;
+  #define op(i) MI->getOperand(i)
+  #define rc(i) RegisterCell::ref(getCell(Reg[i],Inputs))
+  #define im(i) MI->getOperand(i).getImm()
+
+  // If the instruction has no register operands, skip it.
+  if (Reg.size() == 0)
+    return false;
+
+  // Record result for register in operand 0.
+  auto rr0 = [this,Reg] (const BT::RegisterCell &Val, CellMapType &Outputs)
+        -> bool {
+    putCell(Reg[0], Val, Outputs);
+    return true;
+  };
+  // Get the cell corresponding to the N-th operand.
+  auto cop = [this,Reg,MI,Inputs] (unsigned N, uint16_t W)
+        -> BT::RegisterCell {
+    const MachineOperand &Op = MI->getOperand(N);
+    if (Op.isImm())
+      return eIMM(Op.getImm(), W);
+    if (!Op.isReg())
+      return RegisterCell::self(0, W);
+    assert(getRegBitWidth(Reg[N]) == W && "Register width mismatch");
+    return rc(N);
+  };
+  // Extract RW low bits of the cell.
+  auto lo = [this] (const BT::RegisterCell &RC, uint16_t RW)
+        -> BT::RegisterCell {
+    assert(RW <= RC.width());
+    return eXTR(RC, 0, RW);
+  };
+  // Extract RW high bits of the cell.
+  auto hi = [this] (const BT::RegisterCell &RC, uint16_t RW)
+        -> BT::RegisterCell {
+    uint16_t W = RC.width();
+    assert(RW <= W);
+    return eXTR(RC, W-RW, W);
+  };
+  // Extract N-th halfword (counting from the least significant position).
+  auto half = [this] (const BT::RegisterCell &RC, unsigned N)
+        -> BT::RegisterCell {
+    assert(N*16+16 <= RC.width());
+    return eXTR(RC, N*16, N*16+16);
+  };
+  // Shuffle bits (pick even/odd from cells and merge into result).
+  auto shuffle = [this] (const BT::RegisterCell &Rs, const BT::RegisterCell &Rt,
+                         uint16_t BW, bool Odd) -> BT::RegisterCell {
+    uint16_t I = Odd, Ws = Rs.width();
+    assert(Ws == Rt.width());
+    RegisterCell RC = eXTR(Rt, I*BW, I*BW+BW).cat(eXTR(Rs, I*BW, I*BW+BW));
+    I += 2;
+    while (I*BW < Ws) {
+      RC.cat(eXTR(Rt, I*BW, I*BW+BW)).cat(eXTR(Rs, I*BW, I*BW+BW));
+      I += 2;
+    }
+    return RC;
+  };
+
+  // The bitwidth of the 0th operand. In most (if not all) of the
+  // instructions below, the 0th operand is the defined register.
+  // Pre-compute the bitwidth here, because it is needed in many cases
+  // cases below.
+  uint16_t W0 = (Reg[0].Reg != 0) ? getRegBitWidth(Reg[0]) : 0;
+
+  switch (Opc) {
+    // Transfer immediate:
+
+    case A2_tfrsi:
+    case A2_tfrpi:
+    case CONST32:
+    case CONST32_Float_Real:
+    case CONST32_Int_Real:
+    case CONST64_Float_Real:
+    case CONST64_Int_Real:
+      return rr0(eIMM(im(1), W0), Outputs);
+    case TFR_PdFalse:
+      return rr0(RegisterCell(W0).fill(0, W0, BT::BitValue::Zero), Outputs);
+    case TFR_PdTrue:
+      return rr0(RegisterCell(W0).fill(0, W0, BT::BitValue::One), Outputs);
+    case TFR_FI: {
+      int FI = op(1).getIndex();
+      int Off = op(2).getImm();
+      unsigned A = MFI.getObjectAlignment(FI) + std::abs(Off);
+      unsigned L = Log2_32(A);
+      RegisterCell RC = RegisterCell::self(Reg[0].Reg, W0);
+      RC.fill(0, L, BT::BitValue::Zero);
+      return rr0(RC, Outputs);
+    }
+
+    // Transfer register:
+
+    case A2_tfr:
+    case A2_tfrp:
+    case C2_pxfer_map:
+      return rr0(rc(1), Outputs);
+    case C2_tfrpr: {
+      uint16_t RW = W0;
+      uint16_t PW = 8; // XXX Pred size: getRegBitWidth(Reg[1]);
+      assert(PW <= RW);
+      RegisterCell PC = eXTR(rc(1), 0, PW);
+      RegisterCell RC = RegisterCell(RW).insert(PC, BT::BitMask(0, PW-1));
+      RC.fill(PW, RW, BT::BitValue::Zero);
+      return rr0(RC, Outputs);
+    }
+    case C2_tfrrp: {
+      RegisterCell RC = RegisterCell::self(Reg[0].Reg, W0);
+      W0 = 8; // XXX Pred size
+      return rr0(eINS(RC, eXTR(rc(1), 0, W0), 0), Outputs);
+    }
+
+    // Arithmetic:
+
+    case A2_abs:
+    case A2_absp:
+      // TODO
+      break;
+
+    case A2_addsp: {
+      uint16_t W1 = getRegBitWidth(Reg[1]);
+      assert(W0 == 64 && W1 == 32);
+      RegisterCell CW = RegisterCell(W0).insert(rc(1), BT::BitMask(0, W1-1));
+      RegisterCell RC = eADD(eSXT(CW, W1), rc(2));
+      return rr0(RC, Outputs);
+    }
+    case A2_add:
+    case A2_addp:
+      return rr0(eADD(rc(1), rc(2)), Outputs);
+    case A2_addi:
+      return rr0(eADD(rc(1), eIMM(im(2), W0)), Outputs);
+    case S4_addi_asl_ri: {
+      RegisterCell RC = eADD(eIMM(im(1), W0), eASL(rc(2), im(3)));
+      return rr0(RC, Outputs);
+    }
+    case S4_addi_lsr_ri: {
+      RegisterCell RC = eADD(eIMM(im(1), W0), eLSR(rc(2), im(3)));
+      return rr0(RC, Outputs);
+    }
+    case S4_addaddi: {
+      RegisterCell RC = eADD(rc(1), eADD(rc(2), eIMM(im(3), W0)));
+      return rr0(RC, Outputs);
+    }
+    case M4_mpyri_addi: {
+      RegisterCell M = eMLS(rc(2), eIMM(im(3), W0));
+      RegisterCell RC = eADD(eIMM(im(1), W0), lo(M, W0));
+      return rr0(RC, Outputs);
+    }
+    case M4_mpyrr_addi: {
+      RegisterCell M = eMLS(rc(2), rc(3));
+      RegisterCell RC = eADD(eIMM(im(1), W0), lo(M, W0));
+      return rr0(RC, Outputs);
+    }
+    case M4_mpyri_addr_u2: {
+      RegisterCell M = eMLS(eIMM(im(2), W0), rc(3));
+      RegisterCell RC = eADD(rc(1), lo(M, W0));
+      return rr0(RC, Outputs);
+    }
+    case M4_mpyri_addr: {
+      RegisterCell M = eMLS(rc(2), eIMM(im(3), W0));
+      RegisterCell RC = eADD(rc(1), lo(M, W0));
+      return rr0(RC, Outputs);
+    }
+    case M4_mpyrr_addr: {
+      RegisterCell M = eMLS(rc(2), rc(3));
+      RegisterCell RC = eADD(rc(1), lo(M, W0));
+      return rr0(RC, Outputs);
+    }
+    case S4_subaddi: {
+      RegisterCell RC = eADD(rc(1), eSUB(eIMM(im(2), W0), rc(3)));
+      return rr0(RC, Outputs);
+    }
+    case M2_accii: {
+      RegisterCell RC = eADD(rc(1), eADD(rc(2), eIMM(im(3), W0)));
+      return rr0(RC, Outputs);
+    }
+    case M2_acci: {
+      RegisterCell RC = eADD(rc(1), eADD(rc(2), rc(3)));
+      return rr0(RC, Outputs);
+    }
+    case M2_subacc: {
+      RegisterCell RC = eADD(rc(1), eSUB(rc(2), rc(3)));
+      return rr0(RC, Outputs);
+    }
+    case S2_addasl_rrri: {
+      RegisterCell RC = eADD(rc(1), eASL(rc(2), im(3)));
+      return rr0(RC, Outputs);
+    }
+    case C4_addipc: {
+      RegisterCell RPC = RegisterCell::self(Reg[0].Reg, W0);
+      RPC.fill(0, 2, BT::BitValue::Zero);
+      return rr0(eADD(RPC, eIMM(im(2), W0)), Outputs);
+    }
+    case A2_sub:
+    case A2_subp:
+      return rr0(eSUB(rc(1), rc(2)), Outputs);
+    case A2_subri:
+      return rr0(eSUB(eIMM(im(1), W0), rc(2)), Outputs);
+    case S4_subi_asl_ri: {
+      RegisterCell RC = eSUB(eIMM(im(1), W0), eASL(rc(2), im(3)));
+      return rr0(RC, Outputs);
+    }
+    case S4_subi_lsr_ri: {
+      RegisterCell RC = eSUB(eIMM(im(1), W0), eLSR(rc(2), im(3)));
+      return rr0(RC, Outputs);
+    }
+    case M2_naccii: {
+      RegisterCell RC = eSUB(rc(1), eADD(rc(2), eIMM(im(3), W0)));
+      return rr0(RC, Outputs);
+    }
+    case M2_nacci: {
+      RegisterCell RC = eSUB(rc(1), eADD(rc(2), rc(3)));
+      return rr0(RC, Outputs);
+    }
+    // 32-bit negation is done by "Rd = A2_subri 0, Rs"
+    case A2_negp:
+      return rr0(eSUB(eIMM(0, W0), rc(1)), Outputs);
+
+    case M2_mpy_up: {
+      RegisterCell M = eMLS(rc(1), rc(2));
+      return rr0(hi(M, W0), Outputs);
+    }
+    case M2_dpmpyss_s0:
+      return rr0(eMLS(rc(1), rc(2)), Outputs);
+    case M2_dpmpyss_acc_s0:
+      return rr0(eADD(rc(1), eMLS(rc(2), rc(3))), Outputs);
+    case M2_dpmpyss_nac_s0:
+      return rr0(eSUB(rc(1), eMLS(rc(2), rc(3))), Outputs);
+    case M2_mpyi: {
+      RegisterCell M = eMLS(rc(1), rc(2));
+      return rr0(lo(M, W0), Outputs);
+    }
+    case M2_macsip: {
+      RegisterCell M = eMLS(rc(2), eIMM(im(3), W0));
+      RegisterCell RC = eADD(rc(1), lo(M, W0));
+      return rr0(RC, Outputs);
+    }
+    case M2_macsin: {
+      RegisterCell M = eMLS(rc(2), eIMM(im(3), W0));
+      RegisterCell RC = eSUB(rc(1), lo(M, W0));
+      return rr0(RC, Outputs);
+    }
+    case M2_maci: {
+      RegisterCell M = eMLS(rc(2), rc(3));
+      RegisterCell RC = eADD(rc(1), lo(M, W0));
+      return rr0(RC, Outputs);
+    }
+    case M2_mpysmi: {
+      RegisterCell M = eMLS(rc(1), eIMM(im(2), W0));
+      return rr0(lo(M, 32), Outputs);
+    }
+    case M2_mpysin: {
+      RegisterCell M = eMLS(rc(1), eIMM(-im(2), W0));
+      return rr0(lo(M, 32), Outputs);
+    }
+    case M2_mpysip: {
+      RegisterCell M = eMLS(rc(1), eIMM(im(2), W0));
+      return rr0(lo(M, 32), Outputs);
+    }
+    case M2_mpyu_up: {
+      RegisterCell M = eMLU(rc(1), rc(2));
+      return rr0(hi(M, W0), Outputs);
+    }
+    case M2_dpmpyuu_s0:
+      return rr0(eMLU(rc(1), rc(2)), Outputs);
+    case M2_dpmpyuu_acc_s0:
+      return rr0(eADD(rc(1), eMLU(rc(2), rc(3))), Outputs);
+    case M2_dpmpyuu_nac_s0:
+      return rr0(eSUB(rc(1), eMLU(rc(2), rc(3))), Outputs);
+    //case M2_mpysu_up:
+
+    // Logical/bitwise:
+
+    case A2_andir:
+      return rr0(eAND(rc(1), eIMM(im(2), W0)), Outputs);
+    case A2_and:
+    case A2_andp:
+      return rr0(eAND(rc(1), rc(2)), Outputs);
+    case A4_andn:
+    case A4_andnp:
+      return rr0(eAND(rc(1), eNOT(rc(2))), Outputs);
+    case S4_andi_asl_ri: {
+      RegisterCell RC = eAND(eIMM(im(1), W0), eASL(rc(2), im(3)));
+      return rr0(RC, Outputs);
+    }
+    case S4_andi_lsr_ri: {
+      RegisterCell RC = eAND(eIMM(im(1), W0), eLSR(rc(2), im(3)));
+      return rr0(RC, Outputs);
+    }
+    case M4_and_and:
+      return rr0(eAND(rc(1), eAND(rc(2), rc(3))), Outputs);
+    case M4_and_andn:
+      return rr0(eAND(rc(1), eAND(rc(2), eNOT(rc(3)))), Outputs);
+    case M4_and_or:
+      return rr0(eAND(rc(1), eORL(rc(2), rc(3))), Outputs);
+    case M4_and_xor:
+      return rr0(eAND(rc(1), eXOR(rc(2), rc(3))), Outputs);
+    case A2_orir:
+      return rr0(eORL(rc(1), eIMM(im(2), W0)), Outputs);
+    case A2_or:
+    case A2_orp:
+      return rr0(eORL(rc(1), rc(2)), Outputs);
+    case A4_orn:
+    case A4_ornp:
+      return rr0(eORL(rc(1), eNOT(rc(2))), Outputs);
+    case S4_ori_asl_ri: {
+      RegisterCell RC = eORL(eIMM(im(1), W0), eASL(rc(2), im(3)));
+      return rr0(RC, Outputs);
+    }
+    case S4_ori_lsr_ri: {
+      RegisterCell RC = eORL(eIMM(im(1), W0), eLSR(rc(2), im(3)));
+      return rr0(RC, Outputs);
+    }
+    case M4_or_and:
+      return rr0(eORL(rc(1), eAND(rc(2), rc(3))), Outputs);
+    case M4_or_andn:
+      return rr0(eORL(rc(1), eAND(rc(2), eNOT(rc(3)))), Outputs);
+    case S4_or_andi:
+    case S4_or_andix: {
+      RegisterCell RC = eORL(rc(1), eAND(rc(2), eIMM(im(3), W0)));
+      return rr0(RC, Outputs);
+    }
+    case S4_or_ori: {
+      RegisterCell RC = eORL(rc(1), eORL(rc(2), eIMM(im(3), W0)));
+      return rr0(RC, Outputs);
+    }
+    case M4_or_or:
+      return rr0(eORL(rc(1), eORL(rc(2), rc(3))), Outputs);
+    case M4_or_xor:
+      return rr0(eORL(rc(1), eXOR(rc(2), rc(3))), Outputs);
+    case A2_xor:
+    case A2_xorp:
+      return rr0(eXOR(rc(1), rc(2)), Outputs);
+    case M4_xor_and:
+      return rr0(eXOR(rc(1), eAND(rc(2), rc(3))), Outputs);
+    case M4_xor_andn:
+      return rr0(eXOR(rc(1), eAND(rc(2), eNOT(rc(3)))), Outputs);
+    case M4_xor_or:
+      return rr0(eXOR(rc(1), eORL(rc(2), rc(3))), Outputs);
+    case M4_xor_xacc:
+      return rr0(eXOR(rc(1), eXOR(rc(2), rc(3))), Outputs);
+    case A2_not:
+    case A2_notp:
+      return rr0(eNOT(rc(1)), Outputs);
+
+    case S2_asl_i_r:
+    case S2_asl_i_p:
+      return rr0(eASL(rc(1), im(2)), Outputs);
+    case A2_aslh:
+      return rr0(eASL(rc(1), 16), Outputs);
+    case S2_asl_i_r_acc:
+    case S2_asl_i_p_acc:
+      return rr0(eADD(rc(1), eASL(rc(2), im(3))), Outputs);
+    case S2_asl_i_r_nac:
+    case S2_asl_i_p_nac:
+      return rr0(eSUB(rc(1), eASL(rc(2), im(3))), Outputs);
+    case S2_asl_i_r_and:
+    case S2_asl_i_p_and:
+      return rr0(eAND(rc(1), eASL(rc(2), im(3))), Outputs);
+    case S2_asl_i_r_or:
+    case S2_asl_i_p_or:
+      return rr0(eORL(rc(1), eASL(rc(2), im(3))), Outputs);
+    case S2_asl_i_r_xacc:
+    case S2_asl_i_p_xacc:
+      return rr0(eXOR(rc(1), eASL(rc(2), im(3))), Outputs);
+    case S2_asl_i_vh:
+    case S2_asl_i_vw:
+      // TODO
+      break;
+
+    case S2_asr_i_r:
+    case S2_asr_i_p:
+      return rr0(eASR(rc(1), im(2)), Outputs);
+    case A2_asrh:
+      return rr0(eASR(rc(1), 16), Outputs);
+    case S2_asr_i_r_acc:
+    case S2_asr_i_p_acc:
+      return rr0(eADD(rc(1), eASR(rc(2), im(3))), Outputs);
+    case S2_asr_i_r_nac:
+    case S2_asr_i_p_nac:
+      return rr0(eSUB(rc(1), eASR(rc(2), im(3))), Outputs);
+    case S2_asr_i_r_and:
+    case S2_asr_i_p_and:
+      return rr0(eAND(rc(1), eASR(rc(2), im(3))), Outputs);
+    case S2_asr_i_r_or:
+    case S2_asr_i_p_or:
+      return rr0(eORL(rc(1), eASR(rc(2), im(3))), Outputs);
+    case S2_asr_i_r_rnd: {
+      // The input is first sign-extended to 64 bits, then the output
+      // is truncated back to 32 bits.
+      assert(W0 == 32);
+      RegisterCell XC = eSXT(rc(1).cat(eIMM(0, W0)), W0);
+      RegisterCell RC = eASR(eADD(eASR(XC, im(2)), eIMM(1, 2*W0)), 1);
+      return rr0(eXTR(RC, 0, W0), Outputs);
+    }
+    case S2_asr_i_r_rnd_goodsyntax: {
+      int64_t S = im(2);
+      if (S == 0)
+        return rr0(rc(1), Outputs);
+      // Result: S2_asr_i_r_rnd Rs, u5-1
+      RegisterCell XC = eSXT(rc(1).cat(eIMM(0, W0)), W0);
+      RegisterCell RC = eLSR(eADD(eASR(XC, S-1), eIMM(1, 2*W0)), 1);
+      return rr0(eXTR(RC, 0, W0), Outputs);
+    }
+    case S2_asr_r_vh:
+    case S2_asr_i_vw:
+    case S2_asr_i_svw_trun:
+      // TODO
+      break;
+
+    case S2_lsr_i_r:
+    case S2_lsr_i_p:
+      return rr0(eLSR(rc(1), im(2)), Outputs);
+    case S2_lsr_i_r_acc:
+    case S2_lsr_i_p_acc:
+      return rr0(eADD(rc(1), eLSR(rc(2), im(3))), Outputs);
+    case S2_lsr_i_r_nac:
+    case S2_lsr_i_p_nac:
+      return rr0(eSUB(rc(1), eLSR(rc(2), im(3))), Outputs);
+    case S2_lsr_i_r_and:
+    case S2_lsr_i_p_and:
+      return rr0(eAND(rc(1), eLSR(rc(2), im(3))), Outputs);
+    case S2_lsr_i_r_or:
+    case S2_lsr_i_p_or:
+      return rr0(eORL(rc(1), eLSR(rc(2), im(3))), Outputs);
+    case S2_lsr_i_r_xacc:
+    case S2_lsr_i_p_xacc:
+      return rr0(eXOR(rc(1), eLSR(rc(2), im(3))), Outputs);
+
+    case S2_clrbit_i: {
+      RegisterCell RC = rc(1);
+      RC[im(2)] = BT::BitValue::Zero;
+      return rr0(RC, Outputs);
+    }
+    case S2_setbit_i: {
+      RegisterCell RC = rc(1);
+      RC[im(2)] = BT::BitValue::One;
+      return rr0(RC, Outputs);
+    }
+    case S2_togglebit_i: {
+      RegisterCell RC = rc(1);
+      uint16_t BX = im(2);
+      RC[BX] = RC[BX].is(0) ? BT::BitValue::One
+                            : RC[BX].is(1) ? BT::BitValue::Zero
+                                           : BT::BitValue::self();
+      return rr0(RC, Outputs);
+    }
+
+    case A4_bitspliti: {
+      uint16_t W1 = getRegBitWidth(Reg[1]);
+      uint16_t BX = im(2);
+      // Res.uw[1] = Rs[bx+1:], Res.uw[0] = Rs[0:bx]
+      const BT::BitValue Zero = BT::BitValue::Zero;
+      RegisterCell RZ = RegisterCell(W0).fill(BX, W1, Zero)
+                                        .fill(W1+(W1-BX), W0, Zero);
+      RegisterCell BF1 = eXTR(rc(1), 0, BX), BF2 = eXTR(rc(1), BX, W1);
+      RegisterCell RC = eINS(eINS(RZ, BF1, 0), BF2, W1);
+      return rr0(RC, Outputs);
+    }
+    case S4_extract:
+    case S4_extractp:
+    case S2_extractu:
+    case S2_extractup: {
+      uint16_t Wd = im(2), Of = im(3);
+      assert(Wd <= W0);
+      if (Wd == 0)
+        return rr0(eIMM(0, W0), Outputs);
+      // If the width extends beyond the register size, pad the register
+      // with 0 bits.
+      RegisterCell Pad = (Wd+Of > W0) ? rc(1).cat(eIMM(0, Wd+Of-W0)) : rc(1);
+      RegisterCell Ext = eXTR(Pad, Of, Wd+Of);
+      // Ext is short, need to extend it with 0s or sign bit.
+      RegisterCell RC = RegisterCell(W0).insert(Ext, BT::BitMask(0, Wd-1));
+      if (Opc == S2_extractu || Opc == S2_extractup)
+        return rr0(eZXT(RC, Wd), Outputs);
+      return rr0(eSXT(RC, Wd), Outputs);
+    }
+    case S2_insert:
+    case S2_insertp: {
+      uint16_t Wd = im(3), Of = im(4);
+      assert(Wd < W0 && Of < W0);
+      // If Wd+Of exceeds W0, the inserted bits are truncated.
+      if (Wd+Of > W0)
+        Wd = W0-Of;
+      if (Wd == 0)
+        return rr0(rc(1), Outputs);
+      return rr0(eINS(rc(1), eXTR(rc(2), 0, Wd), Of), Outputs);
+    }
+
+    // Bit permutations:
+
+    case A2_combineii:
+    case A4_combineii:
+    case A4_combineir:
+    case A4_combineri:
+    case A2_combinew:
+      assert(W0 % 2 == 0);
+      return rr0(cop(2, W0/2).cat(cop(1, W0/2)), Outputs);
+    case A2_combine_ll:
+    case A2_combine_lh:
+    case A2_combine_hl:
+    case A2_combine_hh: {
+      assert(W0 == 32);
+      assert(getRegBitWidth(Reg[1]) == 32 && getRegBitWidth(Reg[2]) == 32);
+      // Low half in the output is 0 for _ll and _hl, 1 otherwise:
+      unsigned LoH = !(Opc == A2_combine_ll || Opc == A2_combine_hl);
+      // High half in the output is 0 for _ll and _lh, 1 otherwise:
+      unsigned HiH = !(Opc == A2_combine_ll || Opc == A2_combine_lh);
+      RegisterCell R1 = rc(1);
+      RegisterCell R2 = rc(2);
+      RegisterCell RC = half(R2, LoH).cat(half(R1, HiH));
+      return rr0(RC, Outputs);
+    }
+    case S2_packhl: {
+      assert(W0 == 64);
+      assert(getRegBitWidth(Reg[1]) == 32 && getRegBitWidth(Reg[2]) == 32);
+      RegisterCell R1 = rc(1);
+      RegisterCell R2 = rc(2);
+      RegisterCell RC = half(R2, 0).cat(half(R1, 0)).cat(half(R2, 1))
+                                   .cat(half(R1, 1));
+      return rr0(RC, Outputs);
+    }
+    case S2_shuffeb: {
+      RegisterCell RC = shuffle(rc(1), rc(2), 8, false);
+      return rr0(RC, Outputs);
+    }
+    case S2_shuffeh: {
+      RegisterCell RC = shuffle(rc(1), rc(2), 16, false);
+      return rr0(RC, Outputs);
+    }
+    case S2_shuffob: {
+      RegisterCell RC = shuffle(rc(1), rc(2), 8, true);
+      return rr0(RC, Outputs);
+    }
+    case S2_shuffoh: {
+      RegisterCell RC = shuffle(rc(1), rc(2), 16, true);
+      return rr0(RC, Outputs);
+    }
+    case C2_mask: {
+      uint16_t WR = W0;
+      uint16_t WP = 8; // XXX Pred size: getRegBitWidth(Reg[1]);
+      assert(WR == 64 && WP == 8);
+      RegisterCell R1 = rc(1);
+      RegisterCell RC(WR);
+      for (uint16_t i = 0; i < WP; ++i) {
+        const BT::BitValue &V = R1[i];
+        BT::BitValue F = (V.is(0) || V.is(1)) ? V : BT::BitValue::self();
+        RC.fill(i*8, i*8+8, F);
+      }
+      return rr0(RC, Outputs);
+    }
+
+    // Mux:
+
+    case C2_muxii:
+    case C2_muxir:
+    case C2_muxri:
+    case C2_mux: {
+      BT::BitValue PC0 = rc(1)[0];
+      RegisterCell R2 = cop(2, W0);
+      RegisterCell R3 = cop(3, W0);
+      if (PC0.is(0) || PC0.is(1))
+        return rr0(RegisterCell::ref(PC0 ? R2 : R3), Outputs);
+      R2.meet(R3, Reg[0].Reg);
+      return rr0(R2, Outputs);
+    }
+    case C2_vmux:
+      // TODO
+      break;
+
+    // Sign- and zero-extension:
+
+    case A2_sxtb:
+      return rr0(eSXT(rc(1), 8), Outputs);
+    case A2_sxth:
+      return rr0(eSXT(rc(1), 16), Outputs);
+    case A2_sxtw: {
+      uint16_t W1 = getRegBitWidth(Reg[1]);
+      assert(W0 == 64 && W1 == 32);
+      RegisterCell RC = eSXT(rc(1).cat(eIMM(0, W1)), W1);
+      return rr0(RC, Outputs);
+    }
+    case A2_zxtb:
+      return rr0(eZXT(rc(1), 8), Outputs);
+    case A2_zxth:
+      return rr0(eZXT(rc(1), 16), Outputs);
+
+    // Bit count:
+
+    case S2_cl0:
+    case S2_cl0p:
+      // Always produce a 32-bit result.
+      return rr0(eCLB(rc(1), 0/*bit*/, 32), Outputs);
+    case S2_cl1:
+    case S2_cl1p:
+      return rr0(eCLB(rc(1), 1/*bit*/, 32), Outputs);
+    case S2_clb:
+    case S2_clbp: {
+      uint16_t W1 = getRegBitWidth(Reg[1]);
+      RegisterCell R1 = rc(1);
+      BT::BitValue TV = R1[W1-1];
+      if (TV.is(0) || TV.is(1))
+        return rr0(eCLB(R1, TV, 32), Outputs);
+      break;
+    }
+    case S2_ct0:
+    case S2_ct0p:
+      return rr0(eCTB(rc(1), 0/*bit*/, 32), Outputs);
+    case S2_ct1:
+    case S2_ct1p:
+      return rr0(eCTB(rc(1), 1/*bit*/, 32), Outputs);
+    case S5_popcountp:
+      // TODO
+      break;
+
+    case C2_all8: {
+      RegisterCell P1 = rc(1);
+      bool Has0 = false, All1 = true;
+      for (uint16_t i = 0; i < 8/*XXX*/; ++i) {
+        if (!P1[i].is(1))
+          All1 = false;
+        if (!P1[i].is(0))
+          continue;
+        Has0 = true;
+        break;
+      }
+      if (!Has0 && !All1)
+        break;
+      RegisterCell RC(W0);
+      RC.fill(0, W0, (All1 ? BT::BitValue::One : BT::BitValue::Zero));
+      return rr0(RC, Outputs);
+    }
+    case C2_any8: {
+      RegisterCell P1 = rc(1);
+      bool Has1 = false, All0 = true;
+      for (uint16_t i = 0; i < 8/*XXX*/; ++i) {
+        if (!P1[i].is(0))
+          All0 = false;
+        if (!P1[i].is(1))
+          continue;
+        Has1 = true;
+        break;
+      }
+      if (!Has1 && !All0)
+        break;
+      RegisterCell RC(W0);
+      RC.fill(0, W0, (Has1 ? BT::BitValue::One : BT::BitValue::Zero));
+      return rr0(RC, Outputs);
+    }
+    case C2_and:
+      return rr0(eAND(rc(1), rc(2)), Outputs);
+    case C2_andn:
+      return rr0(eAND(rc(1), eNOT(rc(2))), Outputs);
+    case C2_not:
+      return rr0(eNOT(rc(1)), Outputs);
+    case C2_or:
+      return rr0(eORL(rc(1), rc(2)), Outputs);
+    case C2_orn:
+      return rr0(eORL(rc(1), eNOT(rc(2))), Outputs);
+    case C2_xor:
+      return rr0(eXOR(rc(1), rc(2)), Outputs);
+    case C4_and_and:
+      return rr0(eAND(rc(1), eAND(rc(2), rc(3))), Outputs);
+    case C4_and_andn:
+      return rr0(eAND(rc(1), eAND(rc(2), eNOT(rc(3)))), Outputs);
+    case C4_and_or:
+      return rr0(eAND(rc(1), eORL(rc(2), rc(3))), Outputs);
+    case C4_and_orn:
+      return rr0(eAND(rc(1), eORL(rc(2), eNOT(rc(3)))), Outputs);
+    case C4_or_and:
+      return rr0(eORL(rc(1), eAND(rc(2), rc(3))), Outputs);
+    case C4_or_andn:
+      return rr0(eORL(rc(1), eAND(rc(2), eNOT(rc(3)))), Outputs);
+    case C4_or_or:
+      return rr0(eORL(rc(1), eORL(rc(2), rc(3))), Outputs);
+    case C4_or_orn:
+      return rr0(eORL(rc(1), eORL(rc(2), eNOT(rc(3)))), Outputs);
+    case C2_bitsclr:
+    case C2_bitsclri:
+    case C2_bitsset:
+    case C4_nbitsclr:
+    case C4_nbitsclri:
+    case C4_nbitsset:
+      // TODO
+      break;
+    case S2_tstbit_i:
+    case S4_ntstbit_i: {
+      BT::BitValue V = rc(1)[im(2)];
+      if (V.is(0) || V.is(1)) {
+        // If instruction is S2_tstbit_i, test for 1, otherwise test for 0.
+        bool TV = (Opc == S2_tstbit_i);
+        BT::BitValue F = V.is(TV) ? BT::BitValue::One : BT::BitValue::Zero;
+        return rr0(RegisterCell(W0).fill(0, W0, F), Outputs);
+      }
+      break;
+    }
+
+    default:
+      return MachineEvaluator::evaluate(MI, Inputs, Outputs);
+  }
+  #undef im
+  #undef rc
+  #undef op
+  return false;
+}
+
+
+bool HexagonEvaluator::evaluate(const MachineInstr *BI,
+      const CellMapType &Inputs, BranchTargetList &Targets,
+      bool &FallsThru) const {
+  // We need to evaluate one branch at a time. TII::AnalyzeBranch checks
+  // all the branches in a basic block at once, so we cannot use it.
+  unsigned Opc = BI->getOpcode();
+  bool SimpleBranch = false;
+  bool Negated = false;
+  switch (Opc) {
+    case Hexagon::J2_jumpf:
+    case Hexagon::J2_jumpfnew:
+    case Hexagon::J2_jumpfnewpt:
+      Negated = true;
+    case Hexagon::J2_jumpt:
+    case Hexagon::J2_jumptnew:
+    case Hexagon::J2_jumptnewpt:
+      // Simple branch:  if([!]Pn) jump ...
+      // i.e. Op0 = predicate, Op1 = branch target.
+      SimpleBranch = true;
+      break;
+    case Hexagon::J2_jump:
+      Targets.insert(BI->getOperand(0).getMBB());
+      FallsThru = false;
+      return true;
+    default:
+      // If the branch is of unknown type, assume that all successors are
+      // executable.
+      return false;
+  }
+
+  if (!SimpleBranch)
+    return false;
+
+  // BI is a conditional branch if we got here.
+  RegisterRef PR = BI->getOperand(0);
+  RegisterCell PC = getCell(PR, Inputs);
+  const BT::BitValue &Test = PC[0];
+
+  // If the condition is neither true nor false, then it's unknown.
+  if (!Test.is(0) && !Test.is(1))
+    return false;
+
+  // "Test.is(!Negated)" means "branch condition is true".
+  if (!Test.is(!Negated)) {
+    // Condition known to be false.
+    FallsThru = true;
+    return true;
+  }
+
+  Targets.insert(BI->getOperand(1).getMBB());
+  FallsThru = false;
+  return true;
+}
+
+
+bool HexagonEvaluator::evaluateLoad(const MachineInstr *MI,
+      const CellMapType &Inputs, CellMapType &Outputs) const {
+  if (TII.isPredicated(MI))
+    return false;
+  assert(MI->mayLoad() && "A load that mayn't?");
+  unsigned Opc = MI->getOpcode();
+
+  uint16_t BitNum;
+  bool SignEx;
+  using namespace Hexagon;
+
+  switch (Opc) {
+    default:
+      return false;
+
+#if 0
+    // memb_fifo
+    case L2_loadalignb_pbr:
+    case L2_loadalignb_pcr:
+    case L2_loadalignb_pi:
+    // memh_fifo
+    case L2_loadalignh_pbr:
+    case L2_loadalignh_pcr:
+    case L2_loadalignh_pi:
+    // membh
+    case L2_loadbsw2_pbr:
+    case L2_loadbsw2_pci:
+    case L2_loadbsw2_pcr:
+    case L2_loadbsw2_pi:
+    case L2_loadbsw4_pbr:
+    case L2_loadbsw4_pci:
+    case L2_loadbsw4_pcr:
+    case L2_loadbsw4_pi:
+    // memubh
+    case L2_loadbzw2_pbr:
+    case L2_loadbzw2_pci:
+    case L2_loadbzw2_pcr:
+    case L2_loadbzw2_pi:
+    case L2_loadbzw4_pbr:
+    case L2_loadbzw4_pci:
+    case L2_loadbzw4_pcr:
+    case L2_loadbzw4_pi:
+#endif
+
+    case L2_loadrbgp:
+    case L2_loadrb_io:
+    case L2_loadrb_pbr:
+    case L2_loadrb_pci:
+    case L2_loadrb_pcr:
+    case L2_loadrb_pi:
+    case L4_loadrb_abs:
+    case L4_loadrb_ap:
+    case L4_loadrb_rr:
+    case L4_loadrb_ur:
+      BitNum = 8;
+      SignEx = true;
+      break;
+
+    case L2_loadrubgp:
+    case L2_loadrub_io:
+    case L2_loadrub_pbr:
+    case L2_loadrub_pci:
+    case L2_loadrub_pcr:
+    case L2_loadrub_pi:
+    case L4_loadrub_abs:
+    case L4_loadrub_ap:
+    case L4_loadrub_rr:
+    case L4_loadrub_ur:
+      BitNum = 8;
+      SignEx = false;
+      break;
+
+    case L2_loadrhgp:
+    case L2_loadrh_io:
+    case L2_loadrh_pbr:
+    case L2_loadrh_pci:
+    case L2_loadrh_pcr:
+    case L2_loadrh_pi:
+    case L4_loadrh_abs:
+    case L4_loadrh_ap:
+    case L4_loadrh_rr:
+    case L4_loadrh_ur:
+      BitNum = 16;
+      SignEx = true;
+      break;
+
+    case L2_loadruhgp:
+    case L2_loadruh_io:
+    case L2_loadruh_pbr:
+    case L2_loadruh_pci:
+    case L2_loadruh_pcr:
+    case L2_loadruh_pi:
+    case L4_loadruh_rr:
+    case L4_loadruh_abs:
+    case L4_loadruh_ap:
+    case L4_loadruh_ur:
+      BitNum = 16;
+      SignEx = false;
+      break;
+
+    case L2_loadrigp:
+    case L2_loadri_io:
+    case L2_loadri_pbr:
+    case L2_loadri_pci:
+    case L2_loadri_pcr:
+    case L2_loadri_pi:
+    case L2_loadw_locked:
+    case L4_loadri_abs:
+    case L4_loadri_ap:
+    case L4_loadri_rr:
+    case L4_loadri_ur:
+    case LDriw_pred:
+      BitNum = 32;
+      SignEx = true;
+      break;
+
+    case L2_loadrdgp:
+    case L2_loadrd_io:
+    case L2_loadrd_pbr:
+    case L2_loadrd_pci:
+    case L2_loadrd_pcr:
+    case L2_loadrd_pi:
+    case L4_loadd_locked:
+    case L4_loadrd_abs:
+    case L4_loadrd_ap:
+    case L4_loadrd_rr:
+    case L4_loadrd_ur:
+      BitNum = 64;
+      SignEx = true;
+      break;
+  }
+
+  const MachineOperand &MD = MI->getOperand(0);
+  assert(MD.isReg() && MD.isDef());
+  RegisterRef RD = MD;
+
+  uint16_t W = getRegBitWidth(RD);
+  assert(W >= BitNum && BitNum > 0);
+  RegisterCell Res(W);
+
+  for (uint16_t i = 0; i < BitNum; ++i)
+    Res[i] = BT::BitValue::self(BT::BitRef(RD.Reg, i));
+
+  if (SignEx) {
+    const BT::BitValue &Sign = Res[BitNum-1];
+    for (uint16_t i = BitNum; i < W; ++i)
+      Res[i] = BT::BitValue::ref(Sign);
+  } else {
+    for (uint16_t i = BitNum; i < W; ++i)
+      Res[i] = BT::BitValue::Zero;
+  }
+
+  putCell(RD, Res, Outputs);
+  return true;
+}
+
+
+bool HexagonEvaluator::evaluateFormalCopy(const MachineInstr *MI,
+      const CellMapType &Inputs, CellMapType &Outputs) const {
+  // If MI defines a formal parameter, but is not a copy (loads are handled
+  // in evaluateLoad), then it's not clear what to do.
+  assert(MI->isCopy());
+
+  RegisterRef RD = MI->getOperand(0);
+  RegisterRef RS = MI->getOperand(1);
+  assert(RD.Sub == 0);
+  if (!TargetRegisterInfo::isPhysicalRegister(RS.Reg))
+    return false;
+  RegExtMap::const_iterator F = VRX.find(RD.Reg);
+  if (F == VRX.end())
+    return false;
+
+  uint16_t EW = F->second.Width;
+  // Store RD's cell into the map. This will associate the cell with a virtual
+  // register, and make zero-/sign-extends possible (otherwise we would be ex-
+  // tending "self" bit values, which will have no effect, since "self" values
+  // cannot be references to anything).
+  putCell(RD, getCell(RS, Inputs), Outputs);
+
+  RegisterCell Res;
+  // Read RD's cell from the outputs instead of RS's cell from the inputs:
+  if (F->second.Type == ExtType::SExt)
+    Res = eSXT(getCell(RD, Outputs), EW);
+  else if (F->second.Type == ExtType::ZExt)
+    Res = eZXT(getCell(RD, Outputs), EW);
+
+  putCell(RD, Res, Outputs);
+  return true;
+}
+
+
+unsigned HexagonEvaluator::getNextPhysReg(unsigned PReg, unsigned Width) const {
+  using namespace Hexagon;
+  bool Is64 = DoubleRegsRegClass.contains(PReg);
+  assert(PReg == 0 || Is64 || IntRegsRegClass.contains(PReg));
+
+  static const unsigned Phys32[] = { R0, R1, R2, R3, R4, R5 };
+  static const unsigned Phys64[] = { D0, D1, D2 };
+  const unsigned Num32 = sizeof(Phys32)/sizeof(unsigned);
+  const unsigned Num64 = sizeof(Phys64)/sizeof(unsigned);
+
+  // Return the first parameter register of the required width.
+  if (PReg == 0)
+    return (Width <= 32) ? Phys32[0] : Phys64[0];
+
+  // Set Idx32, Idx64 in such a way that Idx+1 would give the index of the
+  // next register.
+  unsigned Idx32 = 0, Idx64 = 0;
+  if (!Is64) {
+    while (Idx32 < Num32) {
+      if (Phys32[Idx32] == PReg)
+        break;
+      Idx32++;
+    }
+    Idx64 = Idx32/2;
+  } else {
+    while (Idx64 < Num64) {
+      if (Phys64[Idx64] == PReg)
+        break;
+      Idx64++;
+    }
+    Idx32 = Idx64*2+1;
+  }
+
+  if (Width <= 32)
+    return (Idx32+1 < Num32) ? Phys32[Idx32+1] : 0;
+  return (Idx64+1 < Num64) ? Phys64[Idx64+1] : 0;
+}
+
+
+unsigned HexagonEvaluator::getVirtRegFor(unsigned PReg) const {
+  typedef MachineRegisterInfo::livein_iterator iterator;
+  for (iterator I = MRI.livein_begin(), E = MRI.livein_end(); I != E; ++I) {
+    if (I->first == PReg)
+      return I->second;
+  }
+  return 0;
+}
diff --git a/lib/Target/Hexagon/HexagonBitTracker.h b/lib/Target/Hexagon/HexagonBitTracker.h
new file mode 100644
index 00000000000..897af2d7187
--- /dev/null
+++ b/lib/Target/Hexagon/HexagonBitTracker.h
@@ -0,0 +1,64 @@
+//===--- HexagonBitTracker.h ----------------------------------------------===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef HEXAGONBITTRACKER_H
+#define HEXAGONBITTRACKER_H
+
+#include "BitTracker.h"
+#include "llvm/ADT/DenseMap.h"
+
+namespace llvm {
+  class HexagonInstrInfo;
+  class HexagonRegisterInfo;
+
+struct HexagonEvaluator : public BitTracker::MachineEvaluator {
+  typedef BitTracker::CellMapType CellMapType;
+  typedef BitTracker::RegisterRef RegisterRef;
+  typedef BitTracker::RegisterCell RegisterCell;
+  typedef BitTracker::BranchTargetList BranchTargetList;
+
+  HexagonEvaluator(const HexagonRegisterInfo &tri, MachineRegisterInfo &mri,
+                   const HexagonInstrInfo &tii, MachineFunction &mf);
+
+  bool evaluate(const MachineInstr *MI, const CellMapType &Inputs,
+                CellMapType &Outputs) const override;
+  bool evaluate(const MachineInstr *BI, const CellMapType &Inputs,
+                BranchTargetList &Targets, bool &FallsThru) const override;
+
+  BitTracker::BitMask mask(unsigned Reg, unsigned Sub) const override;
+
+  MachineFunction &MF;
+  MachineFrameInfo &MFI;
+  const HexagonInstrInfo &TII;
+
+private:
+  bool evaluateLoad(const MachineInstr *MI, const CellMapType &Inputs,
+                    CellMapType &Outputs) const;
+  bool evaluateFormalCopy(const MachineInstr *MI, const CellMapType &Inputs,
+                          CellMapType &Outputs) const;
+
+  unsigned getNextPhysReg(unsigned PReg, unsigned Width) const;
+  unsigned getVirtRegFor(unsigned PReg) const;
+
+  // Type of formal parameter extension.
+  struct ExtType {
+    enum { SExt, ZExt };
+    char Type;
+    uint16_t Width;
+    ExtType() : Type(0), Width(0) {}
+    ExtType(char t, uint16_t w) : Type(t), Width(w) {}
+  };
+  // Map VR -> extension type.
+  typedef DenseMap<unsigned, ExtType> RegExtMap;
+  RegExtMap VRX;
+};
+
+} // end namespace llvm
+
+#endif
diff --git a/lib/Target/Hexagon/HexagonCFGOptimizer.cpp b/lib/Target/Hexagon/HexagonCFGOptimizer.cpp
index 703e691e612..3753b745657 100644
--- a/lib/Target/Hexagon/HexagonCFGOptimizer.cpp
+++ b/lib/Target/Hexagon/HexagonCFGOptimizer.cpp
@@ -28,6 +28,7 @@ using namespace llvm;
 #define DEBUG_TYPE "hexagon_cfg"
 
 namespace llvm {
+  FunctionPass *createHexagonCFGOptimizer();
   void initializeHexagonCFGOptimizerPass(PassRegistry&);
 }
 
diff --git a/lib/Target/Hexagon/HexagonCommonGEP.cpp b/lib/Target/Hexagon/HexagonCommonGEP.cpp
new file mode 100644
index 00000000000..df6c17f27b2
--- /dev/null
+++ b/lib/Target/Hexagon/HexagonCommonGEP.cpp
@@ -0,0 +1,1310 @@
+//===--- HexagonCommonGEP.cpp ---------------------------------------------===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+
+#define DEBUG_TYPE "commgep"
+
+#include "llvm/Pass.h"
+#include "llvm/ADT/FoldingSet.h"
+#include "llvm/ADT/STLExtras.h"
+#include "llvm/Analysis/LoopInfo.h"
+#include "llvm/Analysis/PostDominators.h"
+#include "llvm/CodeGen/MachineFunctionAnalysis.h"
+#include "llvm/IR/Constants.h"
+#include "llvm/IR/Dominators.h"
+#include "llvm/IR/Function.h"
+#include "llvm/IR/Instructions.h"
+#include "llvm/IR/Verifier.h"
+#include "llvm/Support/Allocator.h"
+#include "llvm/Support/CommandLine.h"
+#include "llvm/Support/Debug.h"
+#include "llvm/Support/raw_ostream.h"
+#include "llvm/Transforms/Scalar.h"
+#include "llvm/Transforms/Utils/Local.h"
+
+#include <map>
+#include <set>
+#include <vector>
+
+#include "HexagonTargetMachine.h"
+
+using namespace llvm;
+
+static cl::opt<bool> OptSpeculate("commgep-speculate", cl::init(true),
+  cl::Hidden, cl::ZeroOrMore);
+
+static cl::opt<bool> OptEnableInv("commgep-inv", cl::init(true), cl::Hidden,
+  cl::ZeroOrMore);
+
+static cl::opt<bool> OptEnableConst("commgep-const", cl::init(true),
+  cl::Hidden, cl::ZeroOrMore);
+
+namespace llvm {
+  void initializeHexagonCommonGEPPass(PassRegistry&);
+}
+
+namespace {
+  struct GepNode;
+  typedef std::set<GepNode*> NodeSet;
+  typedef std::map<GepNode*,Value*> NodeToValueMap;
+  typedef std::vector<GepNode*> NodeVect;
+  typedef std::map<GepNode*,NodeVect> NodeChildrenMap;
+  typedef std::set<Use*> UseSet;
+  typedef std::map<GepNode*,UseSet> NodeToUsesMap;
+
+  // Numbering map for gep nodes. Used to keep track of ordering for
+  // gep nodes.
+  struct NodeOrdering {
+    NodeOrdering() : LastNum(0) {}
+
+    void insert(const GepNode *N) { Map.insert(std::make_pair(N, ++LastNum)); }
+    void clear() { Map.clear(); }
+
+    bool operator()(const GepNode *N1, const GepNode *N2) const {
+      auto F1 = Map.find(N1), F2 = Map.find(N2);
+      assert(F1 != Map.end() && F2 != Map.end());
+      return F1->second < F2->second;
+    }
+
+  private:
+    std::map<const GepNode *, unsigned> Map;
+    unsigned LastNum;
+  };
+
+  class HexagonCommonGEP : public FunctionPass {
+  public:
+    static char ID;
+    HexagonCommonGEP() : FunctionPass(ID) {
+      initializeHexagonCommonGEPPass(*PassRegistry::getPassRegistry());
+    }
+    virtual bool runOnFunction(Function &F);
+    virtual const char *getPassName() const {
+      return "Hexagon Common GEP";
+    }
+
+    virtual void getAnalysisUsage(AnalysisUsage &AU) const {
+      AU.addRequired<DominatorTreeWrapperPass>();
+      AU.addPreserved<DominatorTreeWrapperPass>();
+      AU.addRequired<PostDominatorTree>();
+      AU.addPreserved<PostDominatorTree>();
+      AU.addRequired<LoopInfoWrapperPass>();
+      AU.addPreserved<LoopInfoWrapperPass>();
+      FunctionPass::getAnalysisUsage(AU);
+    }
+
+  private:
+    typedef std::map<Value*,GepNode*> ValueToNodeMap;
+    typedef std::vector<Value*> ValueVect;
+    typedef std::map<GepNode*,ValueVect> NodeToValuesMap;
+
+    void getBlockTraversalOrder(BasicBlock *Root, ValueVect &Order);
+    bool isHandledGepForm(GetElementPtrInst *GepI);
+    void processGepInst(GetElementPtrInst *GepI, ValueToNodeMap &NM);
+    void collect();
+    void common();
+
+    BasicBlock *recalculatePlacement(GepNode *Node, NodeChildrenMap &NCM,
+                                     NodeToValueMap &Loc);
+    BasicBlock *recalculatePlacementRec(GepNode *Node, NodeChildrenMap &NCM,
+                                        NodeToValueMap &Loc);
+    bool isInvariantIn(Value *Val, Loop *L);
+    bool isInvariantIn(GepNode *Node, Loop *L);
+    bool isInMainPath(BasicBlock *B, Loop *L);
+    BasicBlock *adjustForInvariance(GepNode *Node, NodeChildrenMap &NCM,
+                                    NodeToValueMap &Loc);
+    void separateChainForNode(GepNode *Node, Use *U, NodeToValueMap &Loc);
+    void separateConstantChains(GepNode *Node, NodeChildrenMap &NCM,
+                                NodeToValueMap &Loc);
+    void computeNodePlacement(NodeToValueMap &Loc);
+
+    Value *fabricateGEP(NodeVect &NA, BasicBlock::iterator At,
+                        BasicBlock *LocB);
+    void getAllUsersForNode(GepNode *Node, ValueVect &Values,
+                            NodeChildrenMap &NCM);
+    void materialize(NodeToValueMap &Loc);
+
+    void removeDeadCode();
+
+    NodeVect Nodes;
+    NodeToUsesMap Uses;
+    NodeOrdering NodeOrder;   // Node ordering, for deterministic behavior.
+    SpecificBumpPtrAllocator<GepNode> *Mem;
+    LLVMContext *Ctx;
+    LoopInfo *LI;
+    DominatorTree *DT;
+    PostDominatorTree *PDT;
+    Function *Fn;
+  };
+}
+
+
+char HexagonCommonGEP::ID = 0;
+INITIALIZE_PASS_BEGIN(HexagonCommonGEP, "hcommgep", "Hexagon Common GEP",
+      false, false)
+INITIALIZE_PASS_DEPENDENCY(DominatorTreeWrapperPass)
+INITIALIZE_PASS_DEPENDENCY(PostDominatorTree)
+INITIALIZE_PASS_DEPENDENCY(LoopInfoWrapperPass)
+INITIALIZE_PASS_END(HexagonCommonGEP, "hcommgep", "Hexagon Common GEP",
+      false, false)
+
+namespace {
+  struct GepNode {
+    enum {
+      None      = 0,
+      Root      = 0x01,
+      Internal  = 0x02,
+      Used      = 0x04
+    };
+
+    uint32_t Flags;
+    union {
+      GepNode *Parent;
+      Value *BaseVal;
+    };
+    Value *Idx;
+    Type *PTy;  // Type of the pointer operand.
+
+    GepNode() : Flags(0), Parent(0), Idx(0), PTy(0) {}
+    GepNode(const GepNode *N) : Flags(N->Flags), Idx(N->Idx), PTy(N->PTy) {
+      if (Flags & Root)
+        BaseVal = N->BaseVal;
+      else
+        Parent = N->Parent;
+    }
+    friend raw_ostream &operator<< (raw_ostream &OS, const GepNode &GN);
+  };
+
+
+  Type *next_type(Type *Ty, Value *Idx) {
+    // Advance the type.
+    if (!Ty->isStructTy()) {
+      Type *NexTy = cast<SequentialType>(Ty)->getElementType();
+      return NexTy;
+    }
+    // Otherwise it is a struct type.
+    ConstantInt *CI = dyn_cast<ConstantInt>(Idx);
+    assert(CI && "Struct type with non-constant index");
+    int64_t i = CI->getValue().getSExtValue();
+    Type *NextTy = cast<StructType>(Ty)->getElementType(i);
+    return NextTy;
+  }
+
+
+  raw_ostream &operator<< (raw_ostream &OS, const GepNode &GN) {
+    OS << "{ {";
+    bool Comma = false;
+    if (GN.Flags & GepNode::Root) {
+      OS << "root";
+      Comma = true;
+    }
+    if (GN.Flags & GepNode::Internal) {
+      if (Comma)
+        OS << ',';
+      OS << "internal";
+      Comma = true;
+    }
+    if (GN.Flags & GepNode::Used) {
+      if (Comma)
+        OS << ',';
+      OS << "used";
+      Comma = true;
+    }
+    OS << "} ";
+    if (GN.Flags & GepNode::Root)
+      OS << "BaseVal:" << GN.BaseVal->getName() << '(' << GN.BaseVal << ')';
+    else
+      OS << "Parent:" << GN.Parent;
+
+    OS << " Idx:";
+    if (ConstantInt *CI = dyn_cast<ConstantInt>(GN.Idx))
+      OS << CI->getValue().getSExtValue();
+    else if (GN.Idx->hasName())
+      OS << GN.Idx->getName();
+    else
+      OS << "<anon> =" << *GN.Idx;
+
+    OS << " PTy:";
+    if (GN.PTy->isStructTy()) {
+      StructType *STy = cast<StructType>(GN.PTy);
+      if (!STy->isLiteral())
+        OS << GN.PTy->getStructName();
+      else
+        OS << "<anon-struct>:" << *STy;
+    }
+    else
+      OS << *GN.PTy;
+    OS << " }";
+    return OS;
+  }
+
+
+  template <typename NodeContainer>
+  void dump_node_container(raw_ostream &OS, const NodeContainer &S) {
+    typedef typename NodeContainer::const_iterator const_iterator;
+    for (const_iterator I = S.begin(), E = S.end(); I != E; ++I)
+      OS << *I << ' ' << **I << '\n';
+  }
+
+  raw_ostream &operator<< (raw_ostream &OS,
+                           const NodeVect &S) LLVM_ATTRIBUTE_UNUSED;
+  raw_ostream &operator<< (raw_ostream &OS, const NodeVect &S) {
+    dump_node_container(OS, S);
+    return OS;
+  }
+
+
+  raw_ostream &operator<< (raw_ostream &OS,
+                           const NodeToUsesMap &M) LLVM_ATTRIBUTE_UNUSED;
+  raw_ostream &operator<< (raw_ostream &OS, const NodeToUsesMap &M){
+    typedef NodeToUsesMap::const_iterator const_iterator;
+    for (const_iterator I = M.begin(), E = M.end(); I != E; ++I) {
+      const UseSet &Us = I->second;
+      OS << I->first << " -> #" << Us.size() << '{';
+      for (UseSet::const_iterator J = Us.begin(), F = Us.end(); J != F; ++J) {
+        User *R = (*J)->getUser();
+        if (R->hasName())
+          OS << ' ' << R->getName();
+        else
+          OS << " <?>(" << *R << ')';
+      }
+      OS << " }\n";
+    }
+    return OS;
+  }
+
+
+  struct in_set {
+    in_set(const NodeSet &S) : NS(S) {}
+    bool operator() (GepNode *N) const {
+      return NS.find(N) != NS.end();
+    }
+  private:
+    const NodeSet &NS;
+  };
+}
+
+
+inline void *operator new(size_t, SpecificBumpPtrAllocator<GepNode> &A) {
+  return A.Allocate();
+}
+
+
+void HexagonCommonGEP::getBlockTraversalOrder(BasicBlock *Root,
+      ValueVect &Order) {
+  // Compute block ordering for a typical DT-based traversal of the flow
+  // graph: "before visiting a block, all of its dominators must have been
+  // visited".
+
+  Order.push_back(Root);
+  DomTreeNode *DTN = DT->getNode(Root);
+  typedef GraphTraits<DomTreeNode*> GTN;
+  typedef GTN::ChildIteratorType Iter;
+  for (Iter I = GTN::child_begin(DTN), E = GTN::child_end(DTN); I != E; ++I)
+    getBlockTraversalOrder((*I)->getBlock(), Order);
+}
+
+
+bool HexagonCommonGEP::isHandledGepForm(GetElementPtrInst *GepI) {
+  // No vector GEPs.
+  if (!GepI->getType()->isPointerTy())
+    return false;
+  // No GEPs without any indices.  (Is this possible?)
+  if (GepI->idx_begin() == GepI->idx_end())
+    return false;
+  return true;
+}
+
+
+void HexagonCommonGEP::processGepInst(GetElementPtrInst *GepI,
+      ValueToNodeMap &NM) {
+  DEBUG(dbgs() << "Visiting GEP: " << *GepI << '\n');
+  GepNode *N = new (*Mem) GepNode;
+  Value *PtrOp = GepI->getPointerOperand();
+  ValueToNodeMap::iterator F = NM.find(PtrOp);
+  if (F == NM.end()) {
+    N->BaseVal = PtrOp;
+    N->Flags |= GepNode::Root;
+  } else {
+    // If PtrOp was a GEP instruction, it must have already been processed.
+    // The ValueToNodeMap entry for it is the last gep node in the generated
+    // chain. Link to it here.
+    N->Parent = F->second;
+  }
+  N->PTy = PtrOp->getType();
+  N->Idx = *GepI->idx_begin();
+
+  // Collect the list of users of this GEP instruction. Will add it to the
+  // last node created for it.
+  UseSet Us;
+  for (Value::user_iterator UI = GepI->user_begin(), UE = GepI->user_end();
+       UI != UE; ++UI) {
+    // Check if this gep is used by anything other than other geps that
+    // we will process.
+    if (isa<GetElementPtrInst>(*UI)) {
+      GetElementPtrInst *UserG = cast<GetElementPtrInst>(*UI);
+      if (isHandledGepForm(UserG))
+        continue;
+    }
+    Us.insert(&UI.getUse());
+  }
+  Nodes.push_back(N);
+  NodeOrder.insert(N);
+
+  // Skip the first index operand, since we only handle 0. This dereferences
+  // the pointer operand.
+  GepNode *PN = N;
+  Type *PtrTy = cast<PointerType>(PtrOp->getType())->getElementType();
+  for (User::op_iterator OI = GepI->idx_begin()+1, OE = GepI->idx_end();
+       OI != OE; ++OI) {
+    Value *Op = *OI;
+    GepNode *Nx = new (*Mem) GepNode;
+    Nx->Parent = PN;  // Link Nx to the previous node.
+    Nx->Flags |= GepNode::Internal;
+    Nx->PTy = PtrTy;
+    Nx->Idx = Op;
+    Nodes.push_back(Nx);
+    NodeOrder.insert(Nx);
+    PN = Nx;
+
+    PtrTy = next_type(PtrTy, Op);
+  }
+
+  // After last node has been created, update the use information.
+  if (!Us.empty()) {
+    PN->Flags |= GepNode::Used;
+    Uses[PN].insert(Us.begin(), Us.end());
+  }
+
+  // Link the last node with the originating GEP instruction. This is to
+  // help with linking chained GEP instructions.
+  NM.insert(std::make_pair(GepI, PN));
+}
+
+
+void HexagonCommonGEP::collect() {
+  // Establish depth-first traversal order of the dominator tree.
+  ValueVect BO;
+  getBlockTraversalOrder(Fn->begin(), BO);
+
+  // The creation of gep nodes requires DT-traversal. When processing a GEP
+  // instruction that uses another GEP instruction as the base pointer, the
+  // gep node for the base pointer should already exist.
+  ValueToNodeMap NM;
+  for (ValueVect::iterator I = BO.begin(), E = BO.end(); I != E; ++I) {
+    BasicBlock *B = cast<BasicBlock>(*I);
+    for (BasicBlock::iterator J = B->begin(), F = B->end(); J != F; ++J) {
+      if (!isa<GetElementPtrInst>(J))
+        continue;
+      GetElementPtrInst *GepI = cast<GetElementPtrInst>(J);
+      if (isHandledGepForm(GepI))
+        processGepInst(GepI, NM);
+    }
+  }
+
+  DEBUG(dbgs() << "Gep nodes after initial collection:\n" << Nodes);
+}
+
+
+namespace {
+  void invert_find_roots(const NodeVect &Nodes, NodeChildrenMap &NCM,
+        NodeVect &Roots) {
+    typedef NodeVect::const_iterator const_iterator;
+    for (const_iterator I = Nodes.begin(), E = Nodes.end(); I != E; ++I) {
+      GepNode *N = *I;
+      if (N->Flags & GepNode::Root) {
+        Roots.push_back(N);
+        continue;
+      }
+      GepNode *PN = N->Parent;
+      NCM[PN].push_back(N);
+    }
+  }
+
+  void nodes_for_root(GepNode *Root, NodeChildrenMap &NCM, NodeSet &Nodes) {
+    NodeVect Work;
+    Work.push_back(Root);
+    Nodes.insert(Root);
+
+    while (!Work.empty()) {
+      NodeVect::iterator First = Work.begin();
+      GepNode *N = *First;
+      Work.erase(First);
+      NodeChildrenMap::iterator CF = NCM.find(N);
+      if (CF != NCM.end()) {
+        Work.insert(Work.end(), CF->second.begin(), CF->second.end());
+        Nodes.insert(CF->second.begin(), CF->second.end());
+      }
+    }
+  }
+}
+
+
+namespace {
+  typedef std::set<NodeSet> NodeSymRel;
+  typedef std::pair<GepNode*,GepNode*> NodePair;
+  typedef std::set<NodePair> NodePairSet;
+
+  const NodeSet *node_class(GepNode *N, NodeSymRel &Rel) {
+    for (NodeSymRel::iterator I = Rel.begin(), E = Rel.end(); I != E; ++I)
+      if (I->count(N))
+        return &*I;
+    return 0;
+  }
+
+  // Create an ordered pair of GepNode pointers. The pair will be used in
+  // determining equality. The only purpose of the ordering is to eliminate
+  // duplication due to the commutativity of equality/non-equality.
+  NodePair node_pair(GepNode *N1, GepNode *N2) {
+    uintptr_t P1 = uintptr_t(N1), P2 = uintptr_t(N2);
+    if (P1 <= P2)
+      return std::make_pair(N1, N2);
+    return std::make_pair(N2, N1);
+  }
+
+  unsigned node_hash(GepNode *N) {
+    // Include everything except flags and parent.
+    FoldingSetNodeID ID;
+    ID.AddPointer(N->Idx);
+    ID.AddPointer(N->PTy);
+    return ID.ComputeHash();
+  }
+
+  bool node_eq(GepNode *N1, GepNode *N2, NodePairSet &Eq, NodePairSet &Ne) {
+    // Don't cache the result for nodes with different hashes. The hash
+    // comparison is fast enough.
+    if (node_hash(N1) != node_hash(N2))
+      return false;
+
+    NodePair NP = node_pair(N1, N2);
+    NodePairSet::iterator FEq = Eq.find(NP);
+    if (FEq != Eq.end())
+      return true;
+    NodePairSet::iterator FNe = Ne.find(NP);
+    if (FNe != Ne.end())
+      return false;
+    // Not previously compared.
+    bool Root1 = N1->Flags & GepNode::Root;
+    bool Root2 = N2->Flags & GepNode::Root;
+    NodePair P = node_pair(N1, N2);
+    // If the Root flag has different values, the nodes are different.
+    // If both nodes are root nodes, but their base pointers differ,
+    // they are different.
+    if (Root1 != Root2 || (Root1 && N1->BaseVal != N2->BaseVal)) {
+      Ne.insert(P);
+      return false;
+    }
+    // Here the root flags are identical, and for root nodes the
+    // base pointers are equal, so the root nodes are equal.
+    // For non-root nodes, compare their parent nodes.
+    if (Root1 || node_eq(N1->Parent, N2->Parent, Eq, Ne)) {
+      Eq.insert(P);
+      return true;
+    }
+    return false;
+  }
+}
+
+
+void HexagonCommonGEP::common() {
+  // The essence of this commoning is finding gep nodes that are equal.
+  // To do this we need to compare all pairs of nodes. To save time,
+  // first, partition the set of all nodes into sets of potentially equal
+  // nodes, and then compare pairs from within each partition.
+  typedef std::map<unsigned,NodeSet> NodeSetMap;
+  NodeSetMap MaybeEq;
+
+  for (NodeVect::iterator I = Nodes.begin(), E = Nodes.end(); I != E; ++I) {
+    GepNode *N = *I;
+    unsigned H = node_hash(N);
+    MaybeEq[H].insert(N);
+  }
+
+  // Compute the equivalence relation for the gep nodes.  Use two caches,
+  // one for equality and the other for non-equality.
+  NodeSymRel EqRel;  // Equality relation (as set of equivalence classes).
+  NodePairSet Eq, Ne;  // Caches.
+  for (NodeSetMap::iterator I = MaybeEq.begin(), E = MaybeEq.end();
+       I != E; ++I) {
+    NodeSet &S = I->second;
+    for (NodeSet::iterator NI = S.begin(), NE = S.end(); NI != NE; ++NI) {
+      GepNode *N = *NI;
+      // If node already has a class, then the class must have been created
+      // in a prior iteration of this loop. Since equality is transitive,
+      // nothing more will be added to that class, so skip it.
+      if (node_class(N, EqRel))
+        continue;
+
+      // Create a new class candidate now.
+      NodeSet C;
+      for (NodeSet::iterator NJ = std::next(NI); NJ != NE; ++NJ)
+        if (node_eq(N, *NJ, Eq, Ne))
+          C.insert(*NJ);
+      // If Tmp is empty, N would be the only element in it. Don't bother
+      // creating a class for it then.
+      if (!C.empty()) {
+        C.insert(N);  // Finalize the set before adding it to the relation.
+        std::pair<NodeSymRel::iterator, bool> Ins = EqRel.insert(C);
+        (void)Ins;
+        assert(Ins.second && "Cannot add a class");
+      }
+    }
+  }
+
+  DEBUG({
+    dbgs() << "Gep node equality:\n";
+    for (NodePairSet::iterator I = Eq.begin(), E = Eq.end(); I != E; ++I)
+      dbgs() << "{ " << I->first << ", " << I->second << " }\n";
+
+    dbgs() << "Gep equivalence classes:\n";
+    for (NodeSymRel::iterator I = EqRel.begin(), E = EqRel.end(); I != E; ++I) {
+      dbgs() << '{';
+      const NodeSet &S = *I;
+      for (NodeSet::const_iterator J = S.begin(), F = S.end(); J != F; ++J) {
+        if (J != S.begin())
+          dbgs() << ',';
+        dbgs() << ' ' << *J;
+      }
+      dbgs() << " }\n";
+    }
+  });
+
+
+  // Create a projection from a NodeSet to the minimal element in it.
+  typedef std::map<const NodeSet*,GepNode*> ProjMap;
+  ProjMap PM;
+  for (NodeSymRel::iterator I = EqRel.begin(), E = EqRel.end(); I != E; ++I) {
+    const NodeSet &S = *I;
+    GepNode *Min = *std::min_element(S.begin(), S.end(), NodeOrder);
+    std::pair<ProjMap::iterator,bool> Ins = PM.insert(std::make_pair(&S, Min));
+    (void)Ins;
+    assert(Ins.second && "Cannot add minimal element");
+
+    // Update the min element's flags, and user list.
+    uint32_t Flags = 0;
+    UseSet &MinUs = Uses[Min];
+    for (NodeSet::iterator J = S.begin(), F = S.end(); J != F; ++J) {
+      GepNode *N = *J;
+      uint32_t NF = N->Flags;
+      // If N is used, append all original values of N to the list of
+      // original values of Min.
+      if (NF & GepNode::Used)
+        MinUs.insert(Uses[N].begin(), Uses[N].end());
+      Flags |= NF;
+    }
+    if (MinUs.empty())
+      Uses.erase(Min);
+
+    // The collected flags should include all the flags from the min element.
+    assert((Min->Flags & Flags) == Min->Flags);
+    Min->Flags = Flags;
+  }
+
+  // Commoning: for each non-root gep node, replace "Parent" with the
+  // selected (minimum) node from the corresponding equivalence class.
+  // If a given parent does not have an equivalence class, leave it
+  // unchanged (it means that it's the only element in its class).
+  for (NodeVect::iterator I = Nodes.begin(), E = Nodes.end(); I != E; ++I) {
+    GepNode *N = *I;
+    if (N->Flags & GepNode::Root)
+      continue;
+    const NodeSet *PC = node_class(N->Parent, EqRel);
+    if (!PC)
+      continue;
+    ProjMap::iterator F = PM.find(PC);
+    if (F == PM.end())
+      continue;
+    // Found a replacement, use it.
+    GepNode *Rep = F->second;
+    N->Parent = Rep;
+  }
+
+  DEBUG(dbgs() << "Gep nodes after commoning:\n" << Nodes);
+
+  // Finally, erase the nodes that are no longer used.
+  NodeSet Erase;
+  for (NodeVect::iterator I = Nodes.begin(), E = Nodes.end(); I != E; ++I) {
+    GepNode *N = *I;
+    const NodeSet *PC = node_class(N, EqRel);
+    if (!PC)
+      continue;
+    ProjMap::iterator F = PM.find(PC);
+    if (F == PM.end())
+      continue;
+    if (N == F->second)
+      continue;
+    // Node for removal.
+    Erase.insert(*I);
+  }
+  NodeVect::iterator NewE = std::remove_if(Nodes.begin(), Nodes.end(),
+                                           in_set(Erase));
+  Nodes.resize(std::distance(Nodes.begin(), NewE));
+
+  DEBUG(dbgs() << "Gep nodes after post-commoning cleanup:\n" << Nodes);
+}
+
+
+namespace {
+  template <typename T>
+  BasicBlock *nearest_common_dominator(DominatorTree *DT, T &Blocks) {
+    DEBUG({
+      dbgs() << "NCD of {";
+      for (typename T::iterator I = Blocks.begin(), E = Blocks.end();
+           I != E; ++I) {
+        if (!*I)
+          continue;
+        BasicBlock *B = cast<BasicBlock>(*I);
+        dbgs() << ' ' << B->getName();
+      }
+      dbgs() << " }\n";
+    });
+
+    // Allow null basic blocks in Blocks.  In such cases, return 0.
+    typename T::iterator I = Blocks.begin(), E = Blocks.end();
+    if (I == E || !*I)
+      return 0;
+    BasicBlock *Dom = cast<BasicBlock>(*I);
+    while (++I != E) {
+      BasicBlock *B = cast_or_null<BasicBlock>(*I);
+      Dom = B ? DT->findNearestCommonDominator(Dom, B) : 0;
+      if (!Dom)
+        return 0;
+    }
+    DEBUG(dbgs() << "computed:" << Dom->getName() << '\n');
+    return Dom;
+  }
+
+  template <typename T>
+  BasicBlock *nearest_common_dominatee(DominatorTree *DT, T &Blocks) {
+    // If two blocks, A and B, dominate a block C, then A dominates B,
+    // or B dominates A.
+    typename T::iterator I = Blocks.begin(), E = Blocks.end();
+    // Find the first non-null block.
+    while (I != E && !*I)
+      ++I;
+    if (I == E)
+      return DT->getRoot();
+    BasicBlock *DomB = cast<BasicBlock>(*I);
+    while (++I != E) {
+      if (!*I)
+        continue;
+      BasicBlock *B = cast<BasicBlock>(*I);
+      if (DT->dominates(B, DomB))
+        continue;
+      if (!DT->dominates(DomB, B))
+        return 0;
+      DomB = B;
+    }
+    return DomB;
+  }
+
+  // Find the first use in B of any value from Values. If no such use,
+  // return B->end().
+  template <typename T>
+  BasicBlock::iterator first_use_of_in_block(T &Values, BasicBlock *B) {
+    BasicBlock::iterator FirstUse = B->end(), BEnd = B->end();
+    typedef typename T::iterator iterator;
+    for (iterator I = Values.begin(), E = Values.end(); I != E; ++I) {
+      Value *V = *I;
+      // If V is used in a PHI node, the use belongs to the incoming block,
+      // not the block with the PHI node. In the incoming block, the use
+      // would be considered as being at the end of it, so it cannot
+      // influence the position of the first use (which is assumed to be
+      // at the end to start with).
+      if (isa<PHINode>(V))
+        continue;
+      if (!isa<Instruction>(V))
+        continue;
+      Instruction *In = cast<Instruction>(V);
+      if (In->getParent() != B)
+        continue;
+      BasicBlock::iterator It = In;
+      if (std::distance(FirstUse, BEnd) < std::distance(It, BEnd))
+        FirstUse = It;
+    }
+    return FirstUse;
+  }
+
+  bool is_empty(const BasicBlock *B) {
+    return B->empty() || (&*B->begin() == B->getTerminator());
+  }
+}
+
+
+BasicBlock *HexagonCommonGEP::recalculatePlacement(GepNode *Node,
+      NodeChildrenMap &NCM, NodeToValueMap &Loc) {
+  DEBUG(dbgs() << "Loc for node:" << Node << '\n');
+  // Recalculate the placement for Node, assuming that the locations of
+  // its children in Loc are valid.
+  // Return 0 if there is no valid placement for Node (for example, it
+  // uses an index value that is not available at the location required
+  // to dominate all children, etc.).
+
+  // Find the nearest common dominator for:
+  // - all users, if the node is used, and
+  // - all children.
+  ValueVect Bs;
+  if (Node->Flags & GepNode::Used) {
+    // Append all blocks with uses of the original values to the
+    // block vector Bs.
+    NodeToUsesMap::iterator UF = Uses.find(Node);
+    assert(UF != Uses.end() && "Used node with no use information");
+    UseSet &Us = UF->second;
+    for (UseSet::iterator I = Us.begin(), E = Us.end(); I != E; ++I) {
+      Use *U = *I;
+      User *R = U->getUser();
+      if (!isa<Instruction>(R))
+        continue;
+      BasicBlock *PB = isa<PHINode>(R)
+          ? cast<PHINode>(R)->getIncomingBlock(*U)
+          : cast<Instruction>(R)->getParent();
+      Bs.push_back(PB);
+    }
+  }
+  // Append the location of each child.
+  NodeChildrenMap::iterator CF = NCM.find(Node);
+  if (CF != NCM.end()) {
+    NodeVect &Cs = CF->second;
+    for (NodeVect::iterator I = Cs.begin(), E = Cs.end(); I != E; ++I) {
+      GepNode *CN = *I;
+      NodeToValueMap::iterator LF = Loc.find(CN);
+      // If the child is only used in GEP instructions (i.e. is not used in
+      // non-GEP instructions), the nearest dominator computed for it may
+      // have been null. In such case it won't have a location available.
+      if (LF == Loc.end())
+        continue;
+      Bs.push_back(LF->second);
+    }
+  }
+
+  BasicBlock *DomB = nearest_common_dominator(DT, Bs);
+  if (!DomB)
+    return 0;
+  // Check if the index used by Node dominates the computed dominator.
+  Instruction *IdxI = dyn_cast<Instruction>(Node->Idx);
+  if (IdxI && !DT->dominates(IdxI->getParent(), DomB))
+    return 0;
+
+  // Avoid putting nodes into empty blocks.
+  while (is_empty(DomB)) {
+    DomTreeNode *N = (*DT)[DomB]->getIDom();
+    if (!N)
+      break;
+    DomB = N->getBlock();
+  }
+
+  // Otherwise, DomB is fine. Update the location map.
+  Loc[Node] = DomB;
+  return DomB;
+}
+
+
+BasicBlock *HexagonCommonGEP::recalculatePlacementRec(GepNode *Node,
+      NodeChildrenMap &NCM, NodeToValueMap &Loc) {
+  DEBUG(dbgs() << "LocRec begin for node:" << Node << '\n');
+  // Recalculate the placement of Node, after recursively recalculating the
+  // placements of all its children.
+  NodeChildrenMap::iterator CF = NCM.find(Node);
+  if (CF != NCM.end()) {
+    NodeVect &Cs = CF->second;
+    for (NodeVect::iterator I = Cs.begin(), E = Cs.end(); I != E; ++I)
+      recalculatePlacementRec(*I, NCM, Loc);
+  }
+  BasicBlock *LB = recalculatePlacement(Node, NCM, Loc);
+  DEBUG(dbgs() << "LocRec end for node:" << Node << '\n');
+  return LB;
+}
+
+
+bool HexagonCommonGEP::isInvariantIn(Value *Val, Loop *L) {
+  if (isa<Constant>(Val) || isa<Argument>(Val))
+    return true;
+  Instruction *In = dyn_cast<Instruction>(Val);
+  if (!In)
+    return false;
+  BasicBlock *HdrB = L->getHeader(), *DefB = In->getParent();
+  return DT->properlyDominates(DefB, HdrB);
+}
+
+
+bool HexagonCommonGEP::isInvariantIn(GepNode *Node, Loop *L) {
+  if (Node->Flags & GepNode::Root)
+    if (!isInvariantIn(Node->BaseVal, L))
+      return false;
+  return isInvariantIn(Node->Idx, L);
+}
+
+
+bool HexagonCommonGEP::isInMainPath(BasicBlock *B, Loop *L) {
+  BasicBlock *HB = L->getHeader();
+  BasicBlock *LB = L->getLoopLatch();
+  // B must post-dominate the loop header or dominate the loop latch.
+  if (PDT->dominates(B, HB))
+    return true;
+  if (LB && DT->dominates(B, LB))
+    return true;
+  return false;
+}
+
+
+namespace {
+  BasicBlock *preheader(DominatorTree *DT, Loop *L) {
+    if (BasicBlock *PH = L->getLoopPreheader())
+      return PH;
+    if (!OptSpeculate)
+      return 0;
+    DomTreeNode *DN = DT->getNode(L->getHeader());
+    if (!DN)
+      return 0;
+    return DN->getIDom()->getBlock();
+  }
+}
+
+
+BasicBlock *HexagonCommonGEP::adjustForInvariance(GepNode *Node,
+      NodeChildrenMap &NCM, NodeToValueMap &Loc) {
+  // Find the "topmost" location for Node: it must be dominated by both,
+  // its parent (or the BaseVal, if it's a root node), and by the index
+  // value.
+  ValueVect Bs;
+  if (Node->Flags & GepNode::Root) {
+    if (Instruction *PIn = dyn_cast<Instruction>(Node->BaseVal))
+      Bs.push_back(PIn->getParent());
+  } else {
+    Bs.push_back(Loc[Node->Parent]);
+  }
+  if (Instruction *IIn = dyn_cast<Instruction>(Node->Idx))
+    Bs.push_back(IIn->getParent());
+  BasicBlock *TopB = nearest_common_dominatee(DT, Bs);
+
+  // Traverse the loop nest upwards until we find a loop in which Node
+  // is no longer invariant, or until we get to the upper limit of Node's
+  // placement. The traversal will also stop when a suitable "preheader"
+  // cannot be found for a given loop. The "preheader" may actually be
+  // a regular block outside of the loop (i.e. not guarded), in which case
+  // the Node will be speculated.
+  // For nodes that are not in the main path of the containing loop (i.e.
+  // are not executed in each iteration), do not move them out of the loop.
+  BasicBlock *LocB = cast_or_null<BasicBlock>(Loc[Node]);
+  if (LocB) {
+    Loop *Lp = LI->getLoopFor(LocB);
+    while (Lp) {
+      if (!isInvariantIn(Node, Lp) || !isInMainPath(LocB, Lp))
+        break;
+      BasicBlock *NewLoc = preheader(DT, Lp);
+      if (!NewLoc || !DT->dominates(TopB, NewLoc))
+        break;
+      Lp = Lp->getParentLoop();
+      LocB = NewLoc;
+    }
+  }
+  Loc[Node] = LocB;
+
+  // Recursively compute the locations of all children nodes.
+  NodeChildrenMap::iterator CF = NCM.find(Node);
+  if (CF != NCM.end()) {
+    NodeVect &Cs = CF->second;
+    for (NodeVect::iterator I = Cs.begin(), E = Cs.end(); I != E; ++I)
+      adjustForInvariance(*I, NCM, Loc);
+  }
+  return LocB;
+}
+
+
+namespace {
+  struct LocationAsBlock {
+    LocationAsBlock(const NodeToValueMap &L) : Map(L) {}
+    const NodeToValueMap &Map;
+  };
+
+  raw_ostream &operator<< (raw_ostream &OS,
+                           const LocationAsBlock &Loc) LLVM_ATTRIBUTE_UNUSED ;
+  raw_ostream &operator<< (raw_ostream &OS, const LocationAsBlock &Loc) {
+    for (NodeToValueMap::const_iterator I = Loc.Map.begin(), E = Loc.Map.end();
+         I != E; ++I) {
+      OS << I->first << " -> ";
+      BasicBlock *B = cast<BasicBlock>(I->second);
+      OS << B->getName() << '(' << B << ')';
+      OS << '\n';
+    }
+    return OS;
+  }
+
+  inline bool is_constant(GepNode *N) {
+    return isa<ConstantInt>(N->Idx);
+  }
+}
+
+
+void HexagonCommonGEP::separateChainForNode(GepNode *Node, Use *U,
+      NodeToValueMap &Loc) {
+  User *R = U->getUser();
+  DEBUG(dbgs() << "Separating chain for node (" << Node << ") user: "
+               << *R << '\n');
+  BasicBlock *PB = cast<Instruction>(R)->getParent();
+
+  GepNode *N = Node;
+  GepNode *C = 0, *NewNode = 0;
+  while (is_constant(N) && !(N->Flags & GepNode::Root)) {
+    // XXX if (single-use) dont-replicate;
+    GepNode *NewN = new (*Mem) GepNode(N);
+    Nodes.push_back(NewN);
+    Loc[NewN] = PB;
+
+    if (N == Node)
+      NewNode = NewN;
+    NewN->Flags &= ~GepNode::Used;
+    if (C)
+      C->Parent = NewN;
+    C = NewN;
+    N = N->Parent;
+  }
+  if (!NewNode)
+    return;
+
+  // Move over all uses that share the same user as U from Node to NewNode.
+  NodeToUsesMap::iterator UF = Uses.find(Node);
+  assert(UF != Uses.end());
+  UseSet &Us = UF->second;
+  UseSet NewUs;
+  for (UseSet::iterator I = Us.begin(); I != Us.end(); ) {
+    User *S = (*I)->getUser();
+    UseSet::iterator Nx = std::next(I);
+    if (S == R) {
+      NewUs.insert(*I);
+      Us.erase(I);
+    }
+    I = Nx;
+  }
+  if (Us.empty()) {
+    Node->Flags &= ~GepNode::Used;
+    Uses.erase(UF);
+  }
+
+  // Should at least have U in NewUs.
+  NewNode->Flags |= GepNode::Used;
+  DEBUG(dbgs() << "new node: " << NewNode << "  " << *NewNode << '\n');
+  assert(!NewUs.empty());
+  Uses[NewNode] = NewUs;
+}
+
+
+void HexagonCommonGEP::separateConstantChains(GepNode *Node,
+      NodeChildrenMap &NCM, NodeToValueMap &Loc) {
+  // First approximation: extract all chains.
+  NodeSet Ns;
+  nodes_for_root(Node, NCM, Ns);
+
+  DEBUG(dbgs() << "Separating constant chains for node: " << Node << '\n');
+  // Collect all used nodes together with the uses from loads and stores,
+  // where the GEP node could be folded into the load/store instruction.
+  NodeToUsesMap FNs; // Foldable nodes.
+  for (NodeSet::iterator I = Ns.begin(), E = Ns.end(); I != E; ++I) {
+    GepNode *N = *I;
+    if (!(N->Flags & GepNode::Used))
+      continue;
+    NodeToUsesMap::iterator UF = Uses.find(N);
+    assert(UF != Uses.end());
+    UseSet &Us = UF->second;
+    // Loads/stores that use the node N.
+    UseSet LSs;
+    for (UseSet::iterator J = Us.begin(), F = Us.end(); J != F; ++J) {
+      Use *U = *J;
+      User *R = U->getUser();
+      // We're interested in uses that provide the address. It can happen
+      // that the value may also be provided via GEP, but we won't handle
+      // those cases here for now.
+      if (LoadInst *Ld = dyn_cast<LoadInst>(R)) {
+        unsigned PtrX = LoadInst::getPointerOperandIndex();
+        if (&Ld->getOperandUse(PtrX) == U)
+          LSs.insert(U);
+      } else if (StoreInst *St = dyn_cast<StoreInst>(R)) {
+        unsigned PtrX = StoreInst::getPointerOperandIndex();
+        if (&St->getOperandUse(PtrX) == U)
+          LSs.insert(U);
+      }
+    }
+    // Even if the total use count is 1, separating the chain may still be
+    // beneficial, since the constant chain may be longer than the GEP alone
+    // would be (e.g. if the parent node has a constant index and also has
+    // other children).
+    if (!LSs.empty())
+      FNs.insert(std::make_pair(N, LSs));
+  }
+
+  DEBUG(dbgs() << "Nodes with foldable users:\n" << FNs);
+
+  for (NodeToUsesMap::iterator I = FNs.begin(), E = FNs.end(); I != E; ++I) {
+    GepNode *N = I->first;
+    UseSet &Us = I->second;
+    for (UseSet::iterator J = Us.begin(), F = Us.end(); J != F; ++J)
+      separateChainForNode(N, *J, Loc);
+  }
+}
+
+
+void HexagonCommonGEP::computeNodePlacement(NodeToValueMap &Loc) {
+  // Compute the inverse of the Node.Parent links. Also, collect the set
+  // of root nodes.
+  NodeChildrenMap NCM;
+  NodeVect Roots;
+  invert_find_roots(Nodes, NCM, Roots);
+
+  // Compute the initial placement determined by the users' locations, and
+  // the locations of the child nodes.
+  for (NodeVect::iterator I = Roots.begin(), E = Roots.end(); I != E; ++I)
+    recalculatePlacementRec(*I, NCM, Loc);
+
+  DEBUG(dbgs() << "Initial node placement:\n" << LocationAsBlock(Loc));
+
+  if (OptEnableInv) {
+    for (NodeVect::iterator I = Roots.begin(), E = Roots.end(); I != E; ++I)
+      adjustForInvariance(*I, NCM, Loc);
+
+    DEBUG(dbgs() << "Node placement after adjustment for invariance:\n"
+                 << LocationAsBlock(Loc));
+  }
+  if (OptEnableConst) {
+    for (NodeVect::iterator I = Roots.begin(), E = Roots.end(); I != E; ++I)
+      separateConstantChains(*I, NCM, Loc);
+  }
+  DEBUG(dbgs() << "Node use information:\n" << Uses);
+
+  // At the moment, there is no further refinement of the initial placement.
+  // Such a refinement could include splitting the nodes if they are placed
+  // too far from some of its users.
+
+  DEBUG(dbgs() << "Final node placement:\n" << LocationAsBlock(Loc));
+}
+
+
+Value *HexagonCommonGEP::fabricateGEP(NodeVect &NA, BasicBlock::iterator At,
+      BasicBlock *LocB) {
+  DEBUG(dbgs() << "Fabricating GEP in " << LocB->getName()
+               << " for nodes:\n" << NA);
+  unsigned Num = NA.size();
+  GepNode *RN = NA[0];
+  assert((RN->Flags & GepNode::Root) && "Creating GEP for non-root");
+
+  Value *NewInst = 0;
+  Value *Input = RN->BaseVal;
+  Value **IdxList = new Value*[Num+1];
+  unsigned nax = 0;
+  do {
+    unsigned IdxC = 0;
+    // If the type of the input of the first node is not a pointer,
+    // we need to add an artificial i32 0 to the indices (because the
+    // actual input in the IR will be a pointer).
+    if (!NA[nax]->PTy->isPointerTy()) {
+      Type *Int32Ty = Type::getInt32Ty(*Ctx);
+      IdxList[IdxC++] = ConstantInt::get(Int32Ty, 0);
+    }
+
+    // Keep adding indices from NA until we have to stop and generate
+    // an "intermediate" GEP.
+    while (++nax <= Num) {
+      GepNode *N = NA[nax-1];
+      IdxList[IdxC++] = N->Idx;
+      if (nax < Num) {
+        // We have to stop, if the expected type of the output of this node
+        // is not the same as the input type of the next node.
+        Type *NextTy = next_type(N->PTy, N->Idx);
+        if (NextTy != NA[nax]->PTy)
+          break;
+      }
+    }
+    ArrayRef<Value*> A(IdxList, IdxC);
+    Type *InpTy = Input->getType();
+    Type *ElTy = cast<PointerType>(InpTy->getScalarType())->getElementType();
+    NewInst = GetElementPtrInst::Create(ElTy, Input, A, "cgep", At);
+    DEBUG(dbgs() << "new GEP: " << *NewInst << '\n');
+    Input = NewInst;
+  } while (nax <= Num);
+
+  delete[] IdxList;
+  return NewInst;
+}
+
+
+void HexagonCommonGEP::getAllUsersForNode(GepNode *Node, ValueVect &Values,
+      NodeChildrenMap &NCM) {
+  NodeVect Work;
+  Work.push_back(Node);
+
+  while (!Work.empty()) {
+    NodeVect::iterator First = Work.begin();
+    GepNode *N = *First;
+    Work.erase(First);
+    if (N->Flags & GepNode::Used) {
+      NodeToUsesMap::iterator UF = Uses.find(N);
+      assert(UF != Uses.end() && "No use information for used node");
+      UseSet &Us = UF->second;
+      for (UseSet::iterator I = Us.begin(), E = Us.end(); I != E; ++I)
+        Values.push_back((*I)->getUser());
+    }
+    NodeChildrenMap::iterator CF = NCM.find(N);
+    if (CF != NCM.end()) {
+      NodeVect &Cs = CF->second;
+      Work.insert(Work.end(), Cs.begin(), Cs.end());
+    }
+  }
+}
+
+
+void HexagonCommonGEP::materialize(NodeToValueMap &Loc) {
+  DEBUG(dbgs() << "Nodes before materialization:\n" << Nodes << '\n');
+  NodeChildrenMap NCM;
+  NodeVect Roots;
+  // Compute the inversion again, since computing placement could alter
+  // "parent" relation between nodes.
+  invert_find_roots(Nodes, NCM, Roots);
+
+  while (!Roots.empty()) {
+    NodeVect::iterator First = Roots.begin();
+    GepNode *Root = *First, *Last = *First;
+    Roots.erase(First);
+
+    NodeVect NA;  // Nodes to assemble.
+    // Append to NA all child nodes up to (and including) the first child
+    // that:
+    // (1) has more than 1 child, or
+    // (2) is used, or
+    // (3) has a child located in a different block.
+    bool LastUsed = false;
+    unsigned LastCN = 0;
+    // The location may be null if the computation failed (it can legitimately
+    // happen for nodes created from dead GEPs).
+    Value *LocV = Loc[Last];
+    if (!LocV)
+      continue;
+    BasicBlock *LastB = cast<BasicBlock>(LocV);
+    do {
+      NA.push_back(Last);
+      LastUsed = (Last->Flags & GepNode::Used);
+      if (LastUsed)
+        break;
+      NodeChildrenMap::iterator CF = NCM.find(Last);
+      LastCN = (CF != NCM.end()) ? CF->second.size() : 0;
+      if (LastCN != 1)
+        break;
+      GepNode *Child = CF->second.front();
+      BasicBlock *ChildB = cast_or_null<BasicBlock>(Loc[Child]);
+      if (ChildB != 0 && LastB != ChildB)
+        break;
+      Last = Child;
+    } while (true);
+
+    BasicBlock::iterator InsertAt = LastB->getTerminator();
+    if (LastUsed || LastCN > 0) {
+      ValueVect Urs;
+      getAllUsersForNode(Root, Urs, NCM);
+      BasicBlock::iterator FirstUse = first_use_of_in_block(Urs, LastB);
+      if (FirstUse != LastB->end())
+        InsertAt = FirstUse;
+    }
+
+    // Generate a new instruction for NA.
+    Value *NewInst = fabricateGEP(NA, InsertAt, LastB);
+
+    // Convert all the children of Last node into roots, and append them
+    // to the Roots list.
+    if (LastCN > 0) {
+      NodeVect &Cs = NCM[Last];
+      for (NodeVect::iterator I = Cs.begin(), E = Cs.end(); I != E; ++I) {
+        GepNode *CN = *I;
+        CN->Flags &= ~GepNode::Internal;
+        CN->Flags |= GepNode::Root;
+        CN->BaseVal = NewInst;
+        Roots.push_back(CN);
+      }
+    }
+
+    // Lastly, if the Last node was used, replace all uses with the new GEP.
+    // The uses reference the original GEP values.
+    if (LastUsed) {
+      NodeToUsesMap::iterator UF = Uses.find(Last);
+      assert(UF != Uses.end() && "No use information found");
+      UseSet &Us = UF->second;
+      for (UseSet::iterator I = Us.begin(), E = Us.end(); I != E; ++I) {
+        Use *U = *I;
+        U->set(NewInst);
+      }
+    }
+  }
+}
+
+
+void HexagonCommonGEP::removeDeadCode() {
+  ValueVect BO;
+  BO.push_back(&Fn->front());
+
+  for (unsigned i = 0; i < BO.size(); ++i) {
+    BasicBlock *B = cast<BasicBlock>(BO[i]);
+    DomTreeNode *N = DT->getNode(B);
+    typedef GraphTraits<DomTreeNode*> GTN;
+    typedef GTN::ChildIteratorType Iter;
+    for (Iter I = GTN::child_begin(N), E = GTN::child_end(N); I != E; ++I)
+      BO.push_back((*I)->getBlock());
+  }
+
+  for (unsigned i = BO.size(); i > 0; --i) {
+    BasicBlock *B = cast<BasicBlock>(BO[i-1]);
+    BasicBlock::InstListType &IL = B->getInstList();
+    typedef BasicBlock::InstListType::reverse_iterator reverse_iterator;
+    ValueVect Ins;
+    for (reverse_iterator I = IL.rbegin(), E = IL.rend(); I != E; ++I)
+      Ins.push_back(&*I);
+    for (ValueVect::iterator I = Ins.begin(), E = Ins.end(); I != E; ++I) {
+      Instruction *In = cast<Instruction>(*I);
+      if (isInstructionTriviallyDead(In))
+        In->eraseFromParent();
+    }
+  }
+}
+
+
+bool HexagonCommonGEP::runOnFunction(Function &F) {
+  // For now bail out on C++ exception handling.
+  for (Function::iterator A = F.begin(), Z = F.end(); A != Z; ++A)
+    for (BasicBlock::iterator I = A->begin(), E = A->end(); I != E; ++I)
+      if (isa<InvokeInst>(I) || isa<LandingPadInst>(I))
+        return false;
+
+  Fn = &F;
+  DT = &getAnalysis<DominatorTreeWrapperPass>().getDomTree();
+  PDT = &getAnalysis<PostDominatorTree>();
+  LI = &getAnalysis<LoopInfoWrapperPass>().getLoopInfo();
+  Ctx = &F.getContext();
+
+  Nodes.clear();
+  Uses.clear();
+  NodeOrder.clear();
+
+  SpecificBumpPtrAllocator<GepNode> Allocator;
+  Mem = &Allocator;
+
+  collect();
+  common();
+
+  NodeToValueMap Loc;
+  computeNodePlacement(Loc);
+  materialize(Loc);
+  removeDeadCode();
+
+#ifdef XDEBUG
+  // Run this only when expensive checks are enabled.
+  verifyFunction(F);
+#endif
+  return true;
+}
+
+
+namespace llvm {
+  FunctionPass *createHexagonCommonGEP() {
+    return new HexagonCommonGEP();
+  }
+}
diff --git a/lib/Target/Hexagon/HexagonCopyToCombine.cpp b/lib/Target/Hexagon/HexagonCopyToCombine.cpp
index 5a26045fe4c..9fd863f6e15 100644
--- a/lib/Target/Hexagon/HexagonCopyToCombine.cpp
+++ b/lib/Target/Hexagon/HexagonCopyToCombine.cpp
@@ -49,6 +49,7 @@ MaxNumOfInstsBetweenNewValueStoreAndTFR("max-num-inst-between-tfr-and-nv-store",
                             "consider the store still to be newifiable"));
 
 namespace llvm {
+  FunctionPass *createHexagonCopyToCombine();
   void initializeHexagonCopyToCombinePass(PassRegistry&);
 }
 
@@ -116,12 +117,14 @@ static bool isCombinableInstType(MachineInstr *MI,
   switch(MI->getOpcode()) {
   case Hexagon::A2_tfr: {
     // A COPY instruction can be combined if its arguments are IntRegs (32bit).
-    assert(MI->getOperand(0).isReg() && MI->getOperand(1).isReg());
+    const MachineOperand &Op0 = MI->getOperand(0);
+    const MachineOperand &Op1 = MI->getOperand(1);
+    assert(Op0.isReg() && Op1.isReg());
 
-    unsigned DestReg = MI->getOperand(0).getReg();
-    unsigned SrcReg = MI->getOperand(1).getReg();
+    unsigned DestReg = Op0.getReg();
+    unsigned SrcReg = Op1.getReg();
     return Hexagon::IntRegsRegClass.contains(DestReg) &&
-      Hexagon::IntRegsRegClass.contains(SrcReg);
+           Hexagon::IntRegsRegClass.contains(SrcReg);
   }
 
   case Hexagon::A2_tfrsi: {
@@ -144,21 +147,6 @@ static bool isCombinableInstType(MachineInstr *MI,
            (ShouldCombineAggressively || NotExt);
   }
 
-  case Hexagon::TFRI_V4: {
-    if (!ShouldCombineAggressively)
-      return false;
-    assert(MI->getOperand(0).isReg() && MI->getOperand(1).isGlobal());
-
-    // Ensure that TargetFlags are MO_NO_FLAG for a global. This is a
-    // workaround for an ABI bug that prevents GOT relocations on combine
-    // instructions
-    if (MI->getOperand(1).getTargetFlags() != HexagonII::MO_NO_FLAG)
-      return false;
-
-    unsigned DestReg = MI->getOperand(0).getReg();
-    return Hexagon::IntRegsRegClass.contains(DestReg);
-  }
-
   default:
     break;
   }
@@ -166,13 +154,14 @@ static bool isCombinableInstType(MachineInstr *MI,
   return false;
 }
 
-static bool isGreaterThan8BitTFRI(MachineInstr *I) {
-  return I->getOpcode() == Hexagon::A2_tfrsi &&
-    !isInt<8>(I->getOperand(1).getImm());
-}
-static bool isGreaterThan6BitTFRI(MachineInstr *I) {
-  return I->getOpcode() == Hexagon::A2_tfrsi &&
-    !isUInt<6>(I->getOperand(1).getImm());
+template <unsigned N>
+static bool isGreaterThanNBitTFRI(const MachineInstr *I) {
+  if (I->getOpcode() == Hexagon::TFRI64_V4 ||
+      I->getOpcode() == Hexagon::A2_tfrsi) {
+    const MachineOperand &Op = I->getOperand(1);
+    return !Op.isImm() || !isInt<N>(Op.getImm());
+  }
+  return false;
 }
 
 /// areCombinableOperations - Returns true if the two instruction can be merge
@@ -180,19 +169,17 @@ static bool isGreaterThan6BitTFRI(MachineInstr *I) {
 static bool areCombinableOperations(const TargetRegisterInfo *TRI,
                                     MachineInstr *HighRegInst,
                                     MachineInstr *LowRegInst) {
-  assert((HighRegInst->getOpcode() == Hexagon::A2_tfr ||
-          HighRegInst->getOpcode() == Hexagon::A2_tfrsi ||
-          HighRegInst->getOpcode() == Hexagon::TFRI_V4) &&
-         (LowRegInst->getOpcode() == Hexagon::A2_tfr ||
-          LowRegInst->getOpcode() == Hexagon::A2_tfrsi ||
-          LowRegInst->getOpcode() == Hexagon::TFRI_V4) &&
+  unsigned HiOpc = HighRegInst->getOpcode();
+  unsigned LoOpc = LowRegInst->getOpcode();
+  (void)HiOpc; // Fix compiler warning
+  (void)LoOpc; // Fix compiler warning
+  assert((HiOpc == Hexagon::A2_tfr || HiOpc == Hexagon::A2_tfrsi) &&
+         (LoOpc == Hexagon::A2_tfr || LoOpc == Hexagon::A2_tfrsi) &&
          "Assume individual instructions are of a combinable type");
 
   // There is no combine of two constant extended values.
-  if ((HighRegInst->getOpcode() == Hexagon::TFRI_V4 ||
-       isGreaterThan8BitTFRI(HighRegInst)) &&
-      (LowRegInst->getOpcode() == Hexagon::TFRI_V4 ||
-       isGreaterThan6BitTFRI(LowRegInst)))
+  if (isGreaterThanNBitTFRI<8>(HighRegInst) &&
+      isGreaterThanNBitTFRI<6>(LowRegInst))
     return false;
 
   return true;
@@ -219,10 +206,14 @@ static bool isUnsafeToMoveAcross(MachineInstr *I, unsigned UseReg,
                                   unsigned DestReg,
                                   const TargetRegisterInfo *TRI) {
   return (UseReg && (I->modifiesRegister(UseReg, TRI))) ||
-          I->modifiesRegister(DestReg, TRI) ||
-          I->readsRegister(DestReg, TRI) ||
-          I->hasUnmodeledSideEffects() ||
-          I->isInlineAsm() || I->isDebugValue();
+         I->modifiesRegister(DestReg, TRI) ||
+         I->readsRegister(DestReg, TRI) ||
+         I->hasUnmodeledSideEffects() ||
+         I->isInlineAsm() || I->isDebugValue();
+}
+
+static unsigned UseReg(const MachineOperand& MO) {
+  return MO.isReg() ? MO.getReg() : 0;
 }
 
 /// isSafeToMoveTogether - Returns true if it is safe to move I1 next to I2 such
@@ -232,9 +223,7 @@ bool HexagonCopyToCombine::isSafeToMoveTogether(MachineInstr *I1,
                                                 unsigned I1DestReg,
                                                 unsigned I2DestReg,
                                                 bool &DoInsertAtI1) {
-
-  bool IsImmUseReg = I2->getOperand(1).isImm() || I2->getOperand(1).isGlobal();
-  unsigned I2UseReg = IsImmUseReg ? 0 : I2->getOperand(1).getReg();
+  unsigned I2UseReg = UseReg(I2->getOperand(1));
 
   // It is not safe to move I1 and I2 into one combine if I2 has a true
   // dependence on I1.
@@ -298,8 +287,7 @@ bool HexagonCopyToCombine::isSafeToMoveTogether(MachineInstr *I1,
     // At O3 we got better results (dhrystone) by being more conservative here.
     if (!ShouldCombineAggressively)
       End = std::next(MachineBasicBlock::iterator(I2));
-    IsImmUseReg = I1->getOperand(1).isImm() || I1->getOperand(1).isGlobal();
-    unsigned I1UseReg = IsImmUseReg ? 0 : I1->getOperand(1).getReg();
+    unsigned I1UseReg = UseReg(I1->getOperand(1));
     // Track killed operands. If we move across an instruction that kills our
     // operand, we need to update the kill information on the moved I1. It kills
     // the operand now.
@@ -558,7 +546,7 @@ void HexagonCopyToCombine::emitCombineII(MachineBasicBlock::iterator &InsertPt,
   DebugLoc DL = InsertPt->getDebugLoc();
   MachineBasicBlock *BB = InsertPt->getParent();
 
-  // Handle  globals.
+  // Handle globals.
   if (HiOperand.isGlobal()) {
     BuildMI(*BB, InsertPt, DL, TII->get(Hexagon::A2_combineii), DoubleDestReg)
       .addGlobalAddress(HiOperand.getGlobal(), HiOperand.getOffset(),
@@ -574,17 +562,64 @@ void HexagonCopyToCombine::emitCombineII(MachineBasicBlock::iterator &InsertPt,
     return;
   }
 
-  // Handle constant extended immediates.
-  if (!isInt<8>(HiOperand.getImm())) {
-    assert(isInt<8>(LoOperand.getImm()));
+  // Handle block addresses.
+  if (HiOperand.isBlockAddress()) {
     BuildMI(*BB, InsertPt, DL, TII->get(Hexagon::A2_combineii), DoubleDestReg)
+      .addBlockAddress(HiOperand.getBlockAddress(), HiOperand.getOffset(),
+                       HiOperand.getTargetFlags())
+      .addImm(LoOperand.getImm());
+    return;
+  }
+  if (LoOperand.isBlockAddress()) {
+    BuildMI(*BB, InsertPt, DL, TII->get(Hexagon::A4_combineii), DoubleDestReg)
       .addImm(HiOperand.getImm())
+      .addBlockAddress(LoOperand.getBlockAddress(), LoOperand.getOffset(),
+                       LoOperand.getTargetFlags());
+    return;
+  }
+
+  // Handle jump tables.
+  if (HiOperand.isJTI()) {
+    BuildMI(*BB, InsertPt, DL, TII->get(Hexagon::A2_combineii), DoubleDestReg)
+      .addJumpTableIndex(HiOperand.getIndex(), HiOperand.getTargetFlags())
+      .addImm(LoOperand.getImm());
+    return;
+  }
+  if (LoOperand.isJTI()) {
+    BuildMI(*BB, InsertPt, DL, TII->get(Hexagon::A4_combineii), DoubleDestReg)
+      .addImm(HiOperand.getImm())
+      .addJumpTableIndex(LoOperand.getIndex(), LoOperand.getTargetFlags());
+    return;
+  }
+
+  // Handle constant pools.
+  if (HiOperand.isCPI()) {
+    BuildMI(*BB, InsertPt, DL, TII->get(Hexagon::A2_combineii), DoubleDestReg)
+      .addConstantPoolIndex(HiOperand.getIndex(), HiOperand.getOffset(),
+                            HiOperand.getTargetFlags())
       .addImm(LoOperand.getImm());
     return;
   }
+  if (LoOperand.isCPI()) {
+    BuildMI(*BB, InsertPt, DL, TII->get(Hexagon::A4_combineii), DoubleDestReg)
+      .addImm(HiOperand.getImm())
+      .addConstantPoolIndex(LoOperand.getIndex(), LoOperand.getOffset(),
+                            LoOperand.getTargetFlags());
+    return;
+  }
 
-  if (!isUInt<6>(LoOperand.getImm())) {
-    assert(isInt<8>(HiOperand.getImm()));
+  // First preference should be given to Hexagon::A2_combineii instruction
+  // as it can include U6 (in Hexagon::A4_combineii) as well.
+  // In this instruction, HiOperand is const extended, if required.
+  if (isInt<8>(LoOperand.getImm())) {
+    BuildMI(*BB, InsertPt, DL, TII->get(Hexagon::A2_combineii), DoubleDestReg)
+      .addImm(HiOperand.getImm())
+      .addImm(LoOperand.getImm());
+      return;
+  }
+
+  // In this instruction, LoOperand is const extended, if required.
+  if (isInt<8>(HiOperand.getImm())) {
     BuildMI(*BB, InsertPt, DL, TII->get(Hexagon::A4_combineii), DoubleDestReg)
       .addImm(HiOperand.getImm())
       .addImm(LoOperand.getImm());
@@ -608,7 +643,7 @@ void HexagonCopyToCombine::emitCombineIR(MachineBasicBlock::iterator &InsertPt,
   DebugLoc DL = InsertPt->getDebugLoc();
   MachineBasicBlock *BB = InsertPt->getParent();
 
-  // Handle global.
+  // Handle globals.
   if (HiOperand.isGlobal()) {
     BuildMI(*BB, InsertPt, DL, TII->get(Hexagon::A4_combineir), DoubleDestReg)
       .addGlobalAddress(HiOperand.getGlobal(), HiOperand.getOffset(),
@@ -616,6 +651,29 @@ void HexagonCopyToCombine::emitCombineIR(MachineBasicBlock::iterator &InsertPt,
       .addReg(LoReg, LoRegKillFlag);
     return;
   }
+  // Handle block addresses.
+  if (HiOperand.isBlockAddress()) {
+    BuildMI(*BB, InsertPt, DL, TII->get(Hexagon::A4_combineir), DoubleDestReg)
+      .addBlockAddress(HiOperand.getBlockAddress(), HiOperand.getOffset(),
+                       HiOperand.getTargetFlags())
+      .addReg(LoReg, LoRegKillFlag);
+    return;
+  }
+  // Handle jump tables.
+  if (HiOperand.isJTI()) {
+    BuildMI(*BB, InsertPt, DL, TII->get(Hexagon::A4_combineir), DoubleDestReg)
+      .addJumpTableIndex(HiOperand.getIndex(), HiOperand.getTargetFlags())
+      .addReg(LoReg, LoRegKillFlag);
+    return;
+  }
+  // Handle constant pools.
+  if (HiOperand.isCPI()) {
+    BuildMI(*BB, InsertPt, DL, TII->get(Hexagon::A4_combineir), DoubleDestReg)
+      .addConstantPoolIndex(HiOperand.getIndex(), HiOperand.getOffset(),
+                            HiOperand.getTargetFlags())
+      .addReg(LoReg, LoRegKillFlag);
+    return;
+  }
   // Insert new combine instruction.
   //  DoubleRegDest = combine #HiImm, LoReg
   BuildMI(*BB, InsertPt, DL, TII->get(Hexagon::A4_combineir), DoubleDestReg)
@@ -641,6 +699,29 @@ void HexagonCopyToCombine::emitCombineRI(MachineBasicBlock::iterator &InsertPt,
                         LoOperand.getTargetFlags());
     return;
   }
+  // Handle block addresses.
+  if (LoOperand.isBlockAddress()) {
+    BuildMI(*BB, InsertPt, DL, TII->get(Hexagon::A4_combineri), DoubleDestReg)
+      .addReg(HiReg, HiRegKillFlag)
+      .addBlockAddress(LoOperand.getBlockAddress(), LoOperand.getOffset(),
+                       LoOperand.getTargetFlags());
+    return;
+  }
+  // Handle jump tables.
+  if (LoOperand.isJTI()) {
+    BuildMI(*BB, InsertPt, DL, TII->get(Hexagon::A4_combineri), DoubleDestReg)
+      .addReg(HiOperand.getReg(), HiRegKillFlag)
+      .addJumpTableIndex(LoOperand.getIndex(), LoOperand.getTargetFlags());
+    return;
+  }
+  // Handle constant pools.
+  if (LoOperand.isCPI()) {
+    BuildMI(*BB, InsertPt, DL, TII->get(Hexagon::A4_combineri), DoubleDestReg)
+      .addReg(HiOperand.getReg(), HiRegKillFlag)
+      .addConstantPoolIndex(LoOperand.getIndex(), LoOperand.getOffset(),
+                            LoOperand.getTargetFlags());
+    return;
+  }
 
   // Insert new combine instruction.
   //  DoubleRegDest = combine HiReg, #LoImm
diff --git a/lib/Target/Hexagon/HexagonExpandCondsets.cpp b/lib/Target/Hexagon/HexagonExpandCondsets.cpp
index 37ed173a79c..ce10aeadef9 100644
--- a/lib/Target/Hexagon/HexagonExpandCondsets.cpp
+++ b/lib/Target/Hexagon/HexagonExpandCondsets.cpp
@@ -1,3 +1,12 @@
+//===--- HexagonExpandCondsets.cpp ----------------------------------------===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+
 // Replace mux instructions with the corresponding legal instructions.
 // It is meant to work post-SSA, but still on virtual registers. It was
 // originally placed between register coalescing and machine instruction
diff --git a/lib/Target/Hexagon/HexagonExpandPredSpillCode.cpp b/lib/Target/Hexagon/HexagonExpandPredSpillCode.cpp
index 40059fb2737..e4c8d8f7b28 100644
--- a/lib/Target/Hexagon/HexagonExpandPredSpillCode.cpp
+++ b/lib/Target/Hexagon/HexagonExpandPredSpillCode.cpp
@@ -41,6 +41,7 @@ using namespace llvm;
 
 
 namespace llvm {
+  FunctionPass *createHexagonExpandPredSpillCode();
   void initializeHexagonExpandPredSpillCodePass(PassRegistry&);
 }
 
diff --git a/lib/Target/Hexagon/HexagonFixupHwLoops.cpp b/lib/Target/Hexagon/HexagonFixupHwLoops.cpp
index e8d8f1497bd..d0c7f9c8960 100644
--- a/lib/Target/Hexagon/HexagonFixupHwLoops.cpp
+++ b/lib/Target/Hexagon/HexagonFixupHwLoops.cpp
@@ -6,9 +6,8 @@
 // License. See LICENSE.TXT for details.
 //
 // The loop start address in the LOOPn instruction is encoded as a distance
-// from the LOOPn instruction itself.  If the start address is too far from
-// the LOOPn instruction, the loop needs to be set up manually, i.e. via
-// direct transfers to SAn and LCn.
+// from the LOOPn instruction itself. If the start address is too far from
+// the LOOPn instruction, the instruction needs to use a constant extender.
 // This pass will identify and convert such LOOPn instructions to a proper
 // form.
 //===----------------------------------------------------------------------===//
@@ -21,13 +20,17 @@
 #include "llvm/CodeGen/MachineFunctionPass.h"
 #include "llvm/CodeGen/MachineInstrBuilder.h"
 #include "llvm/CodeGen/Passes.h"
-#include "llvm/CodeGen/RegisterScavenging.h"
 #include "llvm/PassSupport.h"
 #include "llvm/Target/TargetInstrInfo.h"
 
 using namespace llvm;
 
+static cl::opt<unsigned> MaxLoopRange(
+    "hexagon-loop-range", cl::Hidden, cl::init(200),
+    cl::desc("Restrict range of loopN instructions (testing only)"));
+
 namespace llvm {
+  FunctionPass *createHexagonFixupHwLoops();
   void initializeHexagonFixupHwLoopsPass(PassRegistry&);
 }
 
@@ -52,20 +55,15 @@ namespace {
     }
 
   private:
-    /// \brief Maximum distance between the loop instr and the basic block.
-    /// Just an estimate.
-    static const unsigned MAX_LOOP_DISTANCE = 200;
-
     /// \brief Check the offset between each loop instruction and
     /// the loop basic block to determine if we can use the LOOP instruction
     /// or if we need to set the LC/SA registers explicitly.
     bool fixupLoopInstrs(MachineFunction &MF);
 
-    /// \brief Add the instruction to set the LC and SA registers explicitly.
-    void convertLoopInstr(MachineFunction &MF,
-                          MachineBasicBlock::iterator &MII,
-                          RegScavenger &RS);
-
+    /// \brief Replace loop instruction with the constant extended
+    /// version if the loop label is too far from the loop instruction.
+    void useExtLoopInstr(MachineFunction &MF,
+                         MachineBasicBlock::iterator &MII);
   };
 
   char HexagonFixupHwLoops::ID = 0;
@@ -78,20 +76,18 @@ FunctionPass *llvm::createHexagonFixupHwLoops() {
   return new HexagonFixupHwLoops();
 }
 
-
 /// \brief Returns true if the instruction is a hardware loop instruction.
 static bool isHardwareLoop(const MachineInstr *MI) {
   return MI->getOpcode() == Hexagon::J2_loop0r ||
-         MI->getOpcode() == Hexagon::J2_loop0i;
+         MI->getOpcode() == Hexagon::J2_loop0i ||
+         MI->getOpcode() == Hexagon::J2_loop1r ||
+         MI->getOpcode() == Hexagon::J2_loop1i;
 }
 
-
 bool HexagonFixupHwLoops::runOnMachineFunction(MachineFunction &MF) {
-  bool Changed = fixupLoopInstrs(MF);
-  return Changed;
+  return fixupLoopInstrs(MF);
 }
 
-
 /// \brief For Hexagon, if the loop label is to far from the
 /// loop instruction then we need to set the LC0 and SA0 registers
 /// explicitly instead of using LOOP(start,count).  This function
@@ -105,41 +101,49 @@ bool HexagonFixupHwLoops::fixupLoopInstrs(MachineFunction &MF) {
   // Offset of the current instruction from the start.
   unsigned InstOffset = 0;
   // Map for each basic block to it's first instruction.
-  DenseMap<MachineBasicBlock*, unsigned> BlockToInstOffset;
+  DenseMap<const MachineBasicBlock *, unsigned> BlockToInstOffset;
+
+  const HexagonInstrInfo *HII =
+      static_cast<const HexagonInstrInfo *>(MF.getSubtarget().getInstrInfo());
 
   // First pass - compute the offset of each basic block.
-  for (MachineFunction::iterator MBB = MF.begin(), MBBe = MF.end();
-       MBB != MBBe; ++MBB) {
-    BlockToInstOffset[MBB] = InstOffset;
-    InstOffset += (MBB->size() * 4);
+  for (const MachineBasicBlock &MBB : MF) {
+    if (MBB.getAlignment()) {
+      // Although we don't know the exact layout of the final code, we need
+      // to account for alignment padding somehow. This heuristic pads each
+      // aligned basic block according to the alignment value.
+      int ByteAlign = (1u << MBB.getAlignment()) - 1;
+      InstOffset = (InstOffset + ByteAlign) & ~(ByteAlign);
+    }
+
+    BlockToInstOffset[&MBB] = InstOffset;
+    for (const MachineInstr &MI : MBB)
+      InstOffset += HII->getSize(&MI);
   }
 
-  // Second pass - check each loop instruction to see if it needs to
-  // be converted.
+  // Second pass - check each loop instruction to see if it needs to be
+  // converted.
   InstOffset = 0;
   bool Changed = false;
-  RegScavenger RS;
-
-  // Loop over all the basic blocks.
-  for (MachineFunction::iterator MBB = MF.begin(), MBBe = MF.end();
-       MBB != MBBe; ++MBB) {
-    InstOffset = BlockToInstOffset[MBB];
-    RS.enterBasicBlock(MBB);
+  for (MachineBasicBlock &MBB : MF) {
+    InstOffset = BlockToInstOffset[&MBB];
 
     // Loop over all the instructions.
-    MachineBasicBlock::iterator MIE = MBB->end();
-    MachineBasicBlock::iterator MII = MBB->begin();
+    MachineBasicBlock::iterator MII = MBB.begin();
+    MachineBasicBlock::iterator MIE = MBB.end();
     while (MII != MIE) {
+      InstOffset += HII->getSize(&*MII);
+      if (MII->isDebugValue()) {
+        ++MII;
+        continue;
+      }
       if (isHardwareLoop(MII)) {
-        RS.forward(MII);
         assert(MII->getOperand(0).isMBB() &&
                "Expect a basic block as loop operand");
-        int Sub = InstOffset - BlockToInstOffset[MII->getOperand(0).getMBB()];
-        unsigned Dist = Sub > 0 ? Sub : -Sub;
-        if (Dist > MAX_LOOP_DISTANCE) {
-          // Convert to explicity setting LC0 and SA0.
-          convertLoopInstr(MF, MII, RS);
-          MII = MBB->erase(MII);
+        int diff = InstOffset - BlockToInstOffset[MII->getOperand(0).getMBB()];
+        if ((unsigned)abs(diff) > MaxLoopRange) {
+          useExtLoopInstr(MF, MII);
+          MII = MBB.erase(MII);
           Changed = true;
         } else {
           ++MII;
@@ -147,39 +151,38 @@ bool HexagonFixupHwLoops::fixupLoopInstrs(MachineFunction &MF) {
       } else {
         ++MII;
       }
-      InstOffset += 4;
     }
   }
 
   return Changed;
 }
 
-
-/// \brief convert a loop instruction to a sequence of instructions that
-/// set the LC0 and SA0 register explicitly.
-void HexagonFixupHwLoops::convertLoopInstr(MachineFunction &MF,
-                                           MachineBasicBlock::iterator &MII,
-                                           RegScavenger &RS) {
+/// \brief Replace loop instructions with the constant extended version.
+void HexagonFixupHwLoops::useExtLoopInstr(MachineFunction &MF,
+                                          MachineBasicBlock::iterator &MII) {
   const TargetInstrInfo *TII = MF.getSubtarget().getInstrInfo();
   MachineBasicBlock *MBB = MII->getParent();
   DebugLoc DL = MII->getDebugLoc();
-  unsigned Scratch = RS.scavengeRegister(&Hexagon::IntRegsRegClass, MII, 0);
-
-  // First, set the LC0 with the trip count.
-  if (MII->getOperand(1).isReg()) {
-    // Trip count is a register
-    BuildMI(*MBB, MII, DL, TII->get(Hexagon::A2_tfrrcr), Hexagon::LC0)
-      .addReg(MII->getOperand(1).getReg());
-  } else {
-    // Trip count is an immediate.
-    BuildMI(*MBB, MII, DL, TII->get(Hexagon::A2_tfrsi), Scratch)
-      .addImm(MII->getOperand(1).getImm());
-    BuildMI(*MBB, MII, DL, TII->get(Hexagon::A2_tfrrcr), Hexagon::LC0)
-      .addReg(Scratch);
+  MachineInstrBuilder MIB;
+  unsigned newOp;
+  switch (MII->getOpcode()) {
+  case Hexagon::J2_loop0r:
+    newOp = Hexagon::J2_loop0rext;
+    break;
+  case Hexagon::J2_loop0i:
+    newOp = Hexagon::J2_loop0iext;
+    break;
+  case Hexagon::J2_loop1r:
+    newOp = Hexagon::J2_loop1rext;
+    break;
+  case Hexagon::J2_loop1i:
+    newOp = Hexagon::J2_loop1iext;
+    break;
+  default:
+    llvm_unreachable("Invalid Hardware Loop Instruction.");
   }
-  // Then, set the SA0 with the loop start address.
-  BuildMI(*MBB, MII, DL, TII->get(Hexagon::CONST32_Label), Scratch)
-    .addMBB(MII->getOperand(0).getMBB());
-  BuildMI(*MBB, MII, DL, TII->get(Hexagon::A2_tfrrcr), Hexagon::SA0)
-    .addReg(Scratch);
+  MIB = BuildMI(*MBB, MII, DL, TII->get(newOp));
+
+  for (unsigned i = 0; i < MII->getNumOperands(); ++i)
+    MIB.addOperand(MII->getOperand(i));
 }
diff --git a/lib/Target/Hexagon/HexagonFrameLowering.cpp b/lib/Target/Hexagon/HexagonFrameLowering.cpp
index 65d689b4d80..29283c81877 100644
--- a/lib/Target/Hexagon/HexagonFrameLowering.cpp
+++ b/lib/Target/Hexagon/HexagonFrameLowering.cpp
@@ -8,6 +8,8 @@
 //
 //===----------------------------------------------------------------------===//
 
+#define DEBUG_TYPE "hexagon-pei"
+
 #include "HexagonFrameLowering.h"
 #include "Hexagon.h"
 #include "HexagonInstrInfo.h"
@@ -16,331 +18,1275 @@
 #include "HexagonSubtarget.h"
 #include "HexagonTargetMachine.h"
 #include "llvm/ADT/BitVector.h"
+#include "llvm/ADT/PostOrderIterator.h"
 #include "llvm/ADT/STLExtras.h"
-#include "llvm/CodeGen/AsmPrinter.h"
-#include "llvm/CodeGen/MachineFrameInfo.h"
+#include "llvm/CodeGen/MachineDominators.h"
+#include "llvm/CodeGen/MachineInstrBuilder.h"
 #include "llvm/CodeGen/MachineFunction.h"
 #include "llvm/CodeGen/MachineFunctionPass.h"
 #include "llvm/CodeGen/MachineInstrBuilder.h"
 #include "llvm/CodeGen/MachineModuleInfo.h"
+#include "llvm/CodeGen/MachinePostDominators.h"
 #include "llvm/CodeGen/MachineRegisterInfo.h"
 #include "llvm/CodeGen/RegisterScavenging.h"
 #include "llvm/IR/Function.h"
 #include "llvm/IR/Type.h"
-#include "llvm/MC/MCAsmInfo.h"
-#include "llvm/MC/MachineLocation.h"
 #include "llvm/Support/CommandLine.h"
+#include "llvm/Support/Debug.h"
+#include "llvm/Support/raw_ostream.h"
 #include "llvm/Target/TargetInstrInfo.h"
 #include "llvm/Target/TargetMachine.h"
 #include "llvm/Target/TargetOptions.h"
 
+// Hexagon stack frame layout as defined by the ABI:
+//
+//                                                       Incoming arguments
+//                                                       passed via stack
+//                                                                      |
+//                                                                      |
+//        SP during function's                 FP during function's     |
+//    +-- runtime (top of stack)               runtime (bottom) --+     |
+//    |                                                           |     |
+// --++---------------------+------------------+-----------------++-+-------
+//   |  parameter area for  |  variable-size   |   fixed-size    |LR|  arg
+//   |   called functions   |  local objects   |  local objects  |FP|
+// --+----------------------+------------------+-----------------+--+-------
+//    <-    size known    -> <- size unknown -> <- size known  ->
+//
+// Low address                                                 High address
+//
+// <--- stack growth
+//
+//
+// - In any circumstances, the outgoing function arguments are always accessi-
+//   ble using the SP, and the incoming arguments are accessible using the FP.
+// - If the local objects are not aligned, they can always be accessed using
+//   the FP.
+// - If there are no variable-sized objects, the local objects can always be
+//   accessed using the SP, regardless whether they are aligned or not. (The
+//   alignment padding will be at the bottom of the stack (highest address),
+//   and so the offset with respect to the SP will be known at the compile-
+//   -time.)
+//
+// The only complication occurs if there are both, local aligned objects, and
+// dynamically allocated (variable-sized) objects. The alignment pad will be
+// placed between the FP and the local objects, thus preventing the use of the
+// FP to access the local objects. At the same time, the variable-sized objects
+// will be between the SP and the local objects, thus introducing an unknown
+// distance from the SP to the locals.
+//
+// To avoid this problem, a new register is created that holds the aligned
+// address of the bottom of the stack, referred in the sources as AP (aligned
+// pointer). The AP will be equal to "FP-p", where "p" is the smallest pad
+// that aligns AP to the required boundary (a maximum of the alignments of
+// all stack objects, fixed- and variable-sized). All local objects[1] will
+// then use AP as the base pointer.
+// [1] The exception is with "fixed" stack objects. "Fixed" stack objects get
+// their name from being allocated at fixed locations on the stack, relative
+// to the FP. In the presence of dynamic allocation and local alignment, such
+// objects can only be accessed through the FP.
+//
+// Illustration of the AP:
+//                                                                FP --+
+//                                                                     |
+// ---------------+---------------------+-----+-----------------------++-+--
+//   Rest of the  | Local stack objects | Pad |  Fixed stack objects  |LR|
+//   stack frame  | (aligned)           |     |  (CSR, spills, etc.)  |FP|
+// ---------------+---------------------+-----+-----------------+-----+--+--
+//                                      |<-- Multiple of the -->|
+//                                           stack alignment    +-- AP
+//
+// The AP is set up at the beginning of the function. Since it is not a dedi-
+// cated (reserved) register, it needs to be kept live throughout the function
+// to be available as the base register for local object accesses.
+// Normally, an address of a stack objects is obtained by a pseudo-instruction
+// TFR_FI. To access local objects with the AP register present, a different
+// pseudo-instruction needs to be used: TFR_FIA. The TFR_FIA takes one extra
+// argument compared to TFR_FI: the first input register is the AP register.
+// This keeps the register live between its definition and its uses.
+
+// The AP register is originally set up using pseudo-instruction ALIGNA:
+//   AP = ALIGNA A
+// where
+//   A  - required stack alignment
+// The alignment value must be the maximum of all alignments required by
+// any stack object.
+
+// The dynamic allocation uses a pseudo-instruction ALLOCA:
+//   Rd = ALLOCA Rs, A
+// where
+//   Rd - address of the allocated space
+//   Rs - minimum size (the actual allocated can be larger to accommodate
+//        alignment)
+//   A  - required alignment
+
+
 using namespace llvm;
 
-static cl::opt<bool> DisableDeallocRet(
-                       "disable-hexagon-dealloc-ret",
-                       cl::Hidden,
-                       cl::desc("Disable Dealloc Return for Hexagon target"));
+static cl::opt<bool> DisableDeallocRet("disable-hexagon-dealloc-ret",
+    cl::Hidden, cl::desc("Disable Dealloc Return for Hexagon target"));
 
-/// determineFrameLayout - Determine the size of the frame and maximum call
-/// frame size.
-void HexagonFrameLowering::determineFrameLayout(MachineFunction &MF) const {
-  MachineFrameInfo *MFI = MF.getFrameInfo();
 
-  // Get the number of bytes to allocate from the FrameInfo.
-  unsigned FrameSize = MFI->getStackSize();
+static cl::opt<int> NumberScavengerSlots("number-scavenger-slots",
+    cl::Hidden, cl::desc("Set the number of scavenger slots"), cl::init(2),
+    cl::ZeroOrMore);
 
-  // Get the alignments provided by the target.
-  unsigned TargetAlign =
-      MF.getSubtarget().getFrameLowering()->getStackAlignment();
-  // Get the maximum call frame size of all the calls.
-  unsigned maxCallFrameSize = MFI->getMaxCallFrameSize();
+static cl::opt<int> SpillFuncThreshold("spill-func-threshold",
+    cl::Hidden, cl::desc("Specify O2(not Os) spill func threshold"),
+    cl::init(6), cl::ZeroOrMore);
 
-  // If we have dynamic alloca then maxCallFrameSize needs to be aligned so
-  // that allocations will be aligned.
-  if (MFI->hasVarSizedObjects())
-    maxCallFrameSize = RoundUpToAlignment(maxCallFrameSize, TargetAlign);
+static cl::opt<int> SpillFuncThresholdOs("spill-func-threshold-Os",
+    cl::Hidden, cl::desc("Specify Os spill func threshold"),
+    cl::init(1), cl::ZeroOrMore);
 
-  // Update maximum call frame size.
-  MFI->setMaxCallFrameSize(maxCallFrameSize);
+static cl::opt<bool> EnableShrinkWrapping("hexagon-shrink-frame",
+    cl::init(true), cl::Hidden, cl::ZeroOrMore,
+    cl::desc("Enable stack frame shrink wrapping"));
 
-  // Include call frame size in total.
-  FrameSize += maxCallFrameSize;
+static cl::opt<unsigned> ShrinkLimit("shrink-frame-limit", cl::init(UINT_MAX),
+    cl::Hidden, cl::ZeroOrMore, cl::desc("Max count of stack frame "
+    "shrink-wraps"));
 
-  // Make sure the frame is aligned.
-  FrameSize = RoundUpToAlignment(FrameSize, TargetAlign);
+namespace {
+  /// Map a register pair Reg to the subregister that has the greater "number",
+  /// i.e. D3 (aka R7:6) will be mapped to R7, etc.
+  unsigned getMax32BitSubRegister(unsigned Reg, const TargetRegisterInfo &TRI,
+                                  bool hireg = true) {
+    if (Reg < Hexagon::D0 || Reg > Hexagon::D15)
+      return Reg;
 
-  // Update frame info.
-  MFI->setStackSize(FrameSize);
+    unsigned RegNo = 0;
+    for (MCSubRegIterator SubRegs(Reg, &TRI); SubRegs.isValid(); ++SubRegs) {
+      if (hireg) {
+        if (*SubRegs > RegNo)
+          RegNo = *SubRegs;
+      } else {
+        if (!RegNo || *SubRegs < RegNo)
+          RegNo = *SubRegs;
+      }
+    }
+    return RegNo;
+  }
+
+  /// Returns the callee saved register with the largest id in the vector.
+  unsigned getMaxCalleeSavedReg(const std::vector<CalleeSavedInfo> &CSI,
+                                const TargetRegisterInfo &TRI) {
+    assert(Hexagon::R1 > 0 &&
+           "Assume physical registers are encoded as positive integers");
+    if (CSI.empty())
+      return 0;
+
+    unsigned Max = getMax32BitSubRegister(CSI[0].getReg(), TRI);
+    for (unsigned I = 1, E = CSI.size(); I < E; ++I) {
+      unsigned Reg = getMax32BitSubRegister(CSI[I].getReg(), TRI);
+      if (Reg > Max)
+        Max = Reg;
+    }
+    return Max;
+  }
+
+  /// Checks if the basic block contains any instruction that needs a stack
+  /// frame to be already in place.
+  bool needsStackFrame(const MachineBasicBlock &MBB, const BitVector &CSR) {
+    for (auto &I : MBB) {
+      const MachineInstr *MI = &I;
+      if (MI->isCall())
+        return true;
+      unsigned Opc = MI->getOpcode();
+      switch (Opc) {
+        case Hexagon::ALLOCA:
+        case Hexagon::ALIGNA:
+          return true;
+        default:
+          break;
+      }
+      // Check individual operands.
+      for (const MachineOperand &MO : MI->operands()) {
+        // While the presence of a frame index does not prove that a stack
+        // frame will be required, all frame indexes should be within alloc-
+        // frame/deallocframe. Otherwise, the code that translates a frame
+        // index into an offset would have to be aware of the placement of
+        // the frame creation/destruction instructions.
+        if (MO.isFI())
+          return true;
+        if (!MO.isReg())
+          continue;
+        unsigned R = MO.getReg();
+        // Virtual registers will need scavenging, which then may require
+        // a stack slot.
+        if (TargetRegisterInfo::isVirtualRegister(R))
+          return true;
+        if (CSR[R])
+          return true;
+      }
+    }
+    return false;
+  }
+
+  /// Returns true if MBB has a machine instructions that indicates a tail call
+  /// in the block.
+  bool hasTailCall(const MachineBasicBlock &MBB) {
+    MachineBasicBlock::const_iterator I = MBB.getLastNonDebugInstr();
+    unsigned RetOpc = I->getOpcode();
+    return RetOpc == Hexagon::TCRETURNi || RetOpc == Hexagon::TCRETURNr;
+  }
+
+  /// Returns true if MBB contains an instruction that returns.
+  bool hasReturn(const MachineBasicBlock &MBB) {
+    for (auto I = MBB.getFirstTerminator(), E = MBB.end(); I != E; ++I)
+      if (I->isReturn())
+        return true;
+    return false;
+  }
+}
+
+
+/// Implements shrink-wrapping of the stack frame. By default, stack frame
+/// is created in the function entry block, and is cleaned up in every block
+/// that returns. This function finds alternate blocks: one for the frame
+/// setup (prolog) and one for the cleanup (epilog).
+void HexagonFrameLowering::findShrunkPrologEpilog(MachineFunction &MF,
+      MachineBasicBlock *&PrologB, MachineBasicBlock *&EpilogB) const {
+  static unsigned ShrinkCounter = 0;
+
+  if (ShrinkLimit.getPosition()) {
+    if (ShrinkCounter >= ShrinkLimit)
+      return;
+    ShrinkCounter++;
+  }
+
+  auto &HST = static_cast<const HexagonSubtarget&>(MF.getSubtarget());
+  auto &HRI = *HST.getRegisterInfo();
+
+  MachineDominatorTree MDT;
+  MDT.runOnMachineFunction(MF);
+  MachinePostDominatorTree MPT;
+  MPT.runOnMachineFunction(MF);
+
+  typedef DenseMap<unsigned,unsigned> UnsignedMap;
+  UnsignedMap RPO;
+  typedef ReversePostOrderTraversal<const MachineFunction*> RPOTType;
+  RPOTType RPOT(&MF);
+  unsigned RPON = 0;
+  for (RPOTType::rpo_iterator I = RPOT.begin(), E = RPOT.end(); I != E; ++I)
+    RPO[(*I)->getNumber()] = RPON++;
+
+  // Don't process functions that have loops, at least for now. Placement
+  // of prolog and epilog must take loop structure into account. For simpli-
+  // city don't do it right now.
+  for (auto &I : MF) {
+    unsigned BN = RPO[I.getNumber()];
+    for (auto SI = I.succ_begin(), SE = I.succ_end(); SI != SE; ++SI) {
+      // If found a back-edge, return.
+      if (RPO[(*SI)->getNumber()] <= BN)
+        return;
+    }
+  }
+
+  // Collect the set of blocks that need a stack frame to execute. Scan
+  // each block for uses/defs of callee-saved registers, calls, etc.
+  SmallVector<MachineBasicBlock*,16> SFBlocks;
+  BitVector CSR(Hexagon::NUM_TARGET_REGS);
+  for (const MCPhysReg *P = HRI.getCalleeSavedRegs(&MF); *P; ++P)
+    CSR[*P] = true;
+
+  for (auto &I : MF)
+    if (needsStackFrame(I, CSR))
+      SFBlocks.push_back(&I);
+
+  DEBUG({
+    dbgs() << "Blocks needing SF: {";
+    for (auto &B : SFBlocks)
+      dbgs() << " BB#" << B->getNumber();
+    dbgs() << " }\n";
+  });
+  // No frame needed?
+  if (SFBlocks.empty())
+    return;
+
+  // Pick a common dominator and a common post-dominator.
+  MachineBasicBlock *DomB = SFBlocks[0];
+  for (unsigned i = 1, n = SFBlocks.size(); i < n; ++i) {
+    DomB = MDT.findNearestCommonDominator(DomB, SFBlocks[i]);
+    if (!DomB)
+      break;
+  }
+  MachineBasicBlock *PDomB = SFBlocks[0];
+  for (unsigned i = 1, n = SFBlocks.size(); i < n; ++i) {
+    PDomB = MPT.findNearestCommonDominator(PDomB, SFBlocks[i]);
+    if (!PDomB)
+      break;
+  }
+  DEBUG({
+    dbgs() << "Computed dom block: BB#";
+    if (DomB) dbgs() << DomB->getNumber();
+    else      dbgs() << "<null>";
+    dbgs() << ", computed pdom block: BB#";
+    if (PDomB) dbgs() << PDomB->getNumber();
+    else       dbgs() << "<null>";
+    dbgs() << "\n";
+  });
+  if (!DomB || !PDomB)
+    return;
+
+  // Make sure that DomB dominates PDomB and PDomB post-dominates DomB.
+  if (!MDT.dominates(DomB, PDomB)) {
+    DEBUG(dbgs() << "Dom block does not dominate pdom block\n");
+    return;
+  }
+  if (!MPT.dominates(PDomB, DomB)) {
+    DEBUG(dbgs() << "PDom block does not post-dominate dom block\n");
+    return;
+  }
+
+  // Finally, everything seems right.
+  PrologB = DomB;
+  EpilogB = PDomB;
+}
+
+/// Perform most of the PEI work here:
+/// - saving/restoring of the callee-saved registers,
+/// - stack frame creation and destruction.
+/// Normally, this work is distributed among various functions, but doing it
+/// in one place allows shrink-wrapping of the stack frame.
+void HexagonFrameLowering::emitPrologue(MachineFunction &MF,
+                                        MachineBasicBlock &MBB) const {
+  auto &HST = static_cast<const HexagonSubtarget&>(MF.getSubtarget());
+  auto &HRI = *HST.getRegisterInfo();
+
+  assert(&MF.front() == &MBB && "Shrink-wrapping not yet supported");
+  MachineFrameInfo *MFI = MF.getFrameInfo();
+  const std::vector<CalleeSavedInfo> &CSI = MFI->getCalleeSavedInfo();
+
+  MachineBasicBlock *PrologB = &MF.front(), *EpilogB = nullptr;
+  if (EnableShrinkWrapping)
+    findShrunkPrologEpilog(MF, PrologB, EpilogB);
+
+  insertCSRSpillsInBlock(*PrologB, CSI, HRI);
+  insertPrologueInBlock(*PrologB);
+
+  if (EpilogB) {
+    insertCSRRestoresInBlock(*EpilogB, CSI, HRI);
+    insertEpilogueInBlock(*EpilogB);
+  } else {
+    for (auto &B : MF)
+      if (!B.empty() && B.back().isReturn())
+        insertCSRRestoresInBlock(B, CSI, HRI);
+
+    for (auto &B : MF)
+      if (!B.empty() && B.back().isReturn())
+        insertEpilogueInBlock(B);
+  }
 }
 
 
-void HexagonFrameLowering::emitPrologue(MachineFunction &MF) const {
-  MachineBasicBlock &MBB = MF.front();
+void HexagonFrameLowering::insertPrologueInBlock(MachineBasicBlock &MBB) const {
+  MachineFunction &MF = *MBB.getParent();
   MachineFrameInfo *MFI = MF.getFrameInfo();
+  MachineModuleInfo &MMI = MF.getMMI();
   MachineBasicBlock::iterator MBBI = MBB.begin();
-  const HexagonRegisterInfo *QRI =
-      MF.getSubtarget<HexagonSubtarget>().getRegisterInfo();
-  DebugLoc dl = MBBI != MBB.end() ? MBBI->getDebugLoc() : DebugLoc();
-  determineFrameLayout(MF);
+  auto &HTM = static_cast<const HexagonTargetMachine&>(MF.getTarget());
+  auto &HST = static_cast<const HexagonSubtarget&>(MF.getSubtarget());
+  auto &HII = *HST.getInstrInfo();
+  auto &HRI = *HST.getRegisterInfo();
+  DebugLoc dl;
 
+  unsigned MaxAlign = std::max(MFI->getMaxAlignment(), getStackAlignment());
+
+  // Calculate the total stack frame size.
   // Get the number of bytes to allocate from the FrameInfo.
-  int NumBytes = (int) MFI->getStackSize();
+  unsigned FrameSize = MFI->getStackSize();
+  // Round up the max call frame size to the max alignment on the stack.
+  unsigned MaxCFA = RoundUpToAlignment(MFI->getMaxCallFrameSize(), MaxAlign);
+  MFI->setMaxCallFrameSize(MaxCFA);
+
+  FrameSize = MaxCFA + RoundUpToAlignment(FrameSize, MaxAlign);
+  MFI->setStackSize(FrameSize);
 
-  // LLVM expects allocframe not to be the first instruction in the
-  // basic block.
+  bool AlignStack = (MaxAlign > getStackAlignment());
+
+  // Check if frame moves are needed for EH.
+  bool needsFrameMoves = MMI.hasDebugInfo() ||
+    MF.getFunction()->needsUnwindTableEntry();
+
+  // Get the number of bytes to allocate from the FrameInfo.
+  unsigned NumBytes = MFI->getStackSize();
+  unsigned SP = HRI.getStackRegister();
+  unsigned MaxCF = MFI->getMaxCallFrameSize();
   MachineBasicBlock::iterator InsertPt = MBB.begin();
 
-  //
-  // ALLOCA adjust regs.  Iterate over ADJDYNALLOC nodes and change the offset.
-  //
-  HexagonMachineFunctionInfo *FuncInfo =
-    MF.getInfo<HexagonMachineFunctionInfo>();
-  const std::vector<MachineInstr*>& AdjustRegs =
-    FuncInfo->getAllocaAdjustInsts();
-  for (std::vector<MachineInstr*>::const_iterator i = AdjustRegs.begin(),
-         e = AdjustRegs.end();
-       i != e; ++i) {
-    MachineInstr* MI = *i;
-    assert((MI->getOpcode() == Hexagon::ADJDYNALLOC) &&
-           "Expected adjust alloca node");
+  auto *FuncInfo = MF.getInfo<HexagonMachineFunctionInfo>();
+  auto &AdjustRegs = FuncInfo->getAllocaAdjustInsts();
 
-    MachineOperand& MO = MI->getOperand(2);
-    assert(MO.isImm() && "Expected immediate");
-    MO.setImm(MFI->getMaxCallFrameSize());
+  for (auto MI : AdjustRegs) {
+    assert((MI->getOpcode() == Hexagon::ALLOCA) && "Expected alloca");
+    expandAlloca(MI, HII, SP, MaxCF);
+    MI->eraseFromParent();
   }
 
   //
-  // Only insert ALLOCFRAME if we need to.
+  // Only insert ALLOCFRAME if we need to or at -O0 for the debugger.  Think
+  // that this shouldn't be required, but doing so now because gcc does and
+  // gdb can't break at the start of the function without it.  Will remove if
+  // this turns out to be a gdb bug.
   //
-  if (hasFP(MF)) {
-    // Check for overflow.
-    // Hexagon_TODO: Ugh! hardcoding. Is there an API that can be used?
-    const int ALLOCFRAME_MAX = 16384;
-    const TargetInstrInfo &TII = *MF.getSubtarget().getInstrInfo();
-
-    if (NumBytes >= ALLOCFRAME_MAX) {
-      // Emit allocframe(#0).
-      BuildMI(MBB, InsertPt, dl, TII.get(Hexagon::S2_allocframe)).addImm(0);
-
-      // Subtract offset from frame pointer.
-      BuildMI(MBB, InsertPt, dl, TII.get(Hexagon::CONST32_Int_Real),
-                                      HEXAGON_RESERVED_REG_1).addImm(NumBytes);
-      BuildMI(MBB, InsertPt, dl, TII.get(Hexagon::A2_sub),
-                                      QRI->getStackRegister()).
-                                      addReg(QRI->getStackRegister()).
-                                      addReg(HEXAGON_RESERVED_REG_1);
-    } else {
-      BuildMI(MBB, InsertPt, dl, TII.get(Hexagon::S2_allocframe)).addImm(NumBytes);
-    }
+  bool NoOpt = (HTM.getOptLevel() == CodeGenOpt::None);
+  if (!NoOpt && !FuncInfo->hasClobberLR() && !hasFP(MF))
+    return;
+
+  // Check for overflow.
+  // Hexagon_TODO: Ugh! hardcoding. Is there an API that can be used?
+  const unsigned int ALLOCFRAME_MAX = 16384;
+
+  // Create a dummy memory operand to avoid allocframe from being treated as
+  // a volatile memory reference.
+  MachineMemOperand *MMO =
+    MF.getMachineMemOperand(MachinePointerInfo(), MachineMemOperand::MOStore,
+                            4, 4);
+
+  if (NumBytes >= ALLOCFRAME_MAX) {
+    // Emit allocframe(#0).
+    BuildMI(MBB, InsertPt, dl, HII.get(Hexagon::S2_allocframe))
+      .addImm(0)
+      .addMemOperand(MMO);
+
+    // Subtract offset from frame pointer.
+    // We use a caller-saved non-parameter register for that.
+    unsigned CallerSavedReg = HRI.getFirstCallerSavedNonParamReg();
+    BuildMI(MBB, InsertPt, dl, HII.get(Hexagon::CONST32_Int_Real),
+            CallerSavedReg).addImm(NumBytes);
+    BuildMI(MBB, InsertPt, dl, HII.get(Hexagon::A2_sub), SP)
+      .addReg(SP)
+      .addReg(CallerSavedReg);
+  } else {
+    BuildMI(MBB, InsertPt, dl, HII.get(Hexagon::S2_allocframe))
+      .addImm(NumBytes)
+      .addMemOperand(MMO);
   }
-}
-// Returns true if MBB has a machine instructions that indicates a tail call
-// in the block.
-bool HexagonFrameLowering::hasTailCall(MachineBasicBlock &MBB) const {
-  MachineBasicBlock::iterator MBBI = MBB.getLastNonDebugInstr();
-  unsigned RetOpcode = MBBI->getOpcode();
 
-  return RetOpcode == Hexagon::TCRETURNi || RetOpcode == Hexagon::TCRETURNr;
+  if (AlignStack) {
+    BuildMI(MBB, InsertPt, dl, HII.get(Hexagon::A2_andir), SP)
+        .addReg(SP)
+        .addImm(-int64_t(MaxAlign));
+  }
+
+  if (needsFrameMoves) {
+    std::vector<MCCFIInstruction> Instructions = MMI.getFrameInstructions();
+    MCSymbol *FrameLabel = MMI.getContext().createTempSymbol();
+
+    // Advance CFA. DW_CFA_def_cfa
+    unsigned DwFPReg = HRI.getDwarfRegNum(HRI.getFrameRegister(), true);
+    unsigned DwRAReg = HRI.getDwarfRegNum(HRI.getRARegister(), true);
+
+    // CFA = FP + 8
+    unsigned CFIIndex = MMI.addFrameInst(MCCFIInstruction::createDefCfa(
+                                               FrameLabel, DwFPReg, -8));
+    BuildMI(MBB, MBBI, dl, HII.get(TargetOpcode::CFI_INSTRUCTION))
+           .addCFIIndex(CFIIndex);
+
+    // R31 (return addr) = CFA - #4
+    CFIIndex = MMI.addFrameInst(MCCFIInstruction::createOffset(
+                                               FrameLabel, DwRAReg, -4));
+    BuildMI(MBB, MBBI, dl, HII.get(TargetOpcode::CFI_INSTRUCTION))
+           .addCFIIndex(CFIIndex);
+
+    // R30 (frame ptr) = CFA - #8)
+    CFIIndex = MMI.addFrameInst(MCCFIInstruction::createOffset(
+                                               FrameLabel, DwFPReg, -8));
+    BuildMI(MBB, MBBI, dl, HII.get(TargetOpcode::CFI_INSTRUCTION))
+           .addCFIIndex(CFIIndex);
+
+    unsigned int regsToMove[] = {
+      Hexagon::R1,  Hexagon::R0,  Hexagon::R3,  Hexagon::R2,
+      Hexagon::R17, Hexagon::R16, Hexagon::R19, Hexagon::R18,
+      Hexagon::R21, Hexagon::R20, Hexagon::R23, Hexagon::R22,
+      Hexagon::R25, Hexagon::R24, Hexagon::R27, Hexagon::R26,
+      Hexagon::D0,  Hexagon::D1,  Hexagon::D8,  Hexagon::D9,  Hexagon::D10,
+      Hexagon::D11, Hexagon::D12, Hexagon::D13, Hexagon::NoRegister
+    };
+
+    const std::vector<CalleeSavedInfo> &CSI = MFI->getCalleeSavedInfo();
+
+    for (unsigned i = 0; regsToMove[i] != Hexagon::NoRegister; ++i) {
+      for (unsigned I = 0, E = CSI.size(); I < E; ++I) {
+        if (CSI[I].getReg() == regsToMove[i]) {
+          // Subtract 8 to make room for R30 and R31, which are added above.
+          int64_t Offset = getFrameIndexOffset(MF, CSI[I].getFrameIdx()) - 8;
+
+          if (regsToMove[i] < Hexagon::D0 || regsToMove[i] > Hexagon::D15) {
+            unsigned DwarfReg = HRI.getDwarfRegNum(regsToMove[i], true);
+            unsigned CFIIndex = MMI.addFrameInst(
+                                    MCCFIInstruction::createOffset(FrameLabel,
+                                                        DwarfReg, Offset));
+            BuildMI(MBB, MBBI, dl, HII.get(TargetOpcode::CFI_INSTRUCTION))
+                   .addCFIIndex(CFIIndex);
+          } else {
+            // Split the double regs into subregs, and generate appropriate
+            // cfi_offsets.
+            // The only reason, we are split double regs is, llvm-mc does not
+            // understand paired registers for cfi_offset.
+            // Eg .cfi_offset r1:0, -64
+            unsigned HiReg = getMax32BitSubRegister(regsToMove[i], HRI);
+            unsigned LoReg = getMax32BitSubRegister(regsToMove[i], HRI, false);
+            unsigned HiDwarfReg = HRI.getDwarfRegNum(HiReg, true);
+            unsigned LoDwarfReg = HRI.getDwarfRegNum(LoReg, true);
+            unsigned HiCFIIndex = MMI.addFrameInst(
+                                    MCCFIInstruction::createOffset(FrameLabel,
+                                                        HiDwarfReg, Offset+4));
+            BuildMI(MBB, MBBI, dl, HII.get(TargetOpcode::CFI_INSTRUCTION))
+                   .addCFIIndex(HiCFIIndex);
+            unsigned LoCFIIndex = MMI.addFrameInst(
+                                    MCCFIInstruction::createOffset(FrameLabel,
+                                                        LoDwarfReg, Offset));
+            BuildMI(MBB, MBBI, dl, HII.get(TargetOpcode::CFI_INSTRUCTION))
+                   .addCFIIndex(LoCFIIndex);
+          }
+          break;
+        }
+      } // for CSI.size()
+    } // for regsToMove
+  } // needsFrameMoves
 }
 
-void HexagonFrameLowering::emitEpilogue(MachineFunction &MF,
-                                     MachineBasicBlock &MBB) const {
-  MachineBasicBlock::iterator MBBI = std::prev(MBB.end());
-  DebugLoc dl = MBBI->getDebugLoc();
+void HexagonFrameLowering::insertEpilogueInBlock(MachineBasicBlock &MBB) const {
+  MachineFunction &MF = *MBB.getParent();
   //
   // Only insert deallocframe if we need to.  Also at -O0.  See comment
-  // in emitPrologue above.
+  // in insertPrologueInBlock above.
   //
-  if (hasFP(MF) || MF.getTarget().getOptLevel() == CodeGenOpt::None) {
-    MachineBasicBlock::iterator MBBI = std::prev(MBB.end());
-    MachineBasicBlock::iterator MBBI_end = MBB.end();
-
-    const TargetInstrInfo &TII = *MF.getSubtarget().getInstrInfo();
-    // Handle EH_RETURN.
-    if (MBBI->getOpcode() == Hexagon::EH_RETURN_JMPR) {
-      assert(MBBI->getOperand(0).isReg() && "Offset should be in register!");
-      BuildMI(MBB, MBBI, dl, TII.get(Hexagon::L2_deallocframe));
-      BuildMI(MBB, MBBI, dl, TII.get(Hexagon::A2_add),
-              Hexagon::R29).addReg(Hexagon::R29).addReg(Hexagon::R28);
-      return;
-    }
-    // Replace 'jumpr r31' instruction with dealloc_return for V4 and higher
-    // versions.
-    if (MBBI->getOpcode() == Hexagon::JMPret && !DisableDeallocRet) {
-      // Check for RESTORE_DEALLOC_RET_JMP_V4 call. Don't emit an extra DEALLOC
-      // instruction if we encounter it.
-      MachineBasicBlock::iterator BeforeJMPR =
-        MBB.begin() == MBBI ? MBBI : std::prev(MBBI);
-      if (BeforeJMPR != MBBI &&
-          BeforeJMPR->getOpcode() == Hexagon::RESTORE_DEALLOC_RET_JMP_V4) {
-        // Remove the JMPR node.
-        MBB.erase(MBBI);
-        return;
-      }
+  if (!hasFP(MF) && MF.getTarget().getOptLevel() != CodeGenOpt::None)
+    return;
 
-      // Add dealloc_return.
-      MachineInstrBuilder MIB =
-        BuildMI(MBB, MBBI_end, dl, TII.get(Hexagon::L4_return));
-      // Transfer the function live-out registers.
-      MIB->copyImplicitOps(*MBB.getParent(), &*MBBI);
-      // Remove the JUMPR node.
-      MBB.erase(MBBI);
-    } else { // Add deallocframe for V2 and V3, and V4 tail calls.
-      // Check for RESTORE_DEALLOC_BEFORE_TAILCALL_V4. We don't need an extra
-      // DEALLOCFRAME instruction after it.
-      MachineBasicBlock::iterator Term = MBB.getFirstTerminator();
-      MachineBasicBlock::iterator I =
-        Term == MBB.begin() ?  MBB.end() : std::prev(Term);
-      if (I != MBB.end() &&
-          I->getOpcode() == Hexagon::RESTORE_DEALLOC_BEFORE_TAILCALL_V4)
-        return;
+  auto &HST = static_cast<const HexagonSubtarget&>(MF.getSubtarget());
+  auto &HII = *HST.getInstrInfo();
+  auto &HRI = *HST.getRegisterInfo();
+  unsigned SP = HRI.getStackRegister();
+
+  MachineInstr *RetI = nullptr;
+  for (auto &I : MBB) {
+    if (!I.isReturn())
+      continue;
+    RetI = &I;
+    break;
+  }
+  unsigned RetOpc = RetI ? RetI->getOpcode() : 0;
 
-      BuildMI(MBB, MBBI, dl, TII.get(Hexagon::L2_deallocframe));
+  MachineBasicBlock::iterator InsertPt = MBB.getFirstTerminator();
+  DebugLoc DL;
+  if (InsertPt != MBB.end())
+    DL = InsertPt->getDebugLoc();
+  else if (!MBB.empty())
+    DL = std::prev(MBB.end())->getDebugLoc();
+
+  // Handle EH_RETURN.
+  if (RetOpc == Hexagon::EH_RETURN_JMPR) {
+    BuildMI(MBB, InsertPt, DL, HII.get(Hexagon::L2_deallocframe));
+    BuildMI(MBB, InsertPt, DL, HII.get(Hexagon::A2_add), SP)
+        .addReg(SP)
+        .addReg(Hexagon::R28);
+    return;
+  }
+
+  // Check for RESTORE_DEALLOC_RET* tail call. Don't emit an extra dealloc-
+  // frame instruction if we encounter it.
+  if (RetOpc == Hexagon::RESTORE_DEALLOC_RET_JMP_V4) {
+    MachineBasicBlock::iterator It = RetI;
+    ++It;
+    // Delete all instructions after the RESTORE (except labels).
+    while (It != MBB.end()) {
+      if (!It->isLabel())
+        It = MBB.erase(It);
+      else
+        ++It;
     }
+    return;
+  }
+
+  // It is possible that the restoring code is a call to a library function.
+  // All of the restore* functions include "deallocframe", so we need to make
+  // sure that we don't add an extra one.
+  bool NeedsDeallocframe = true;
+  if (!MBB.empty() && InsertPt != MBB.begin()) {
+    MachineBasicBlock::iterator PrevIt = std::prev(InsertPt);
+    unsigned COpc = PrevIt->getOpcode();
+    if (COpc == Hexagon::RESTORE_DEALLOC_BEFORE_TAILCALL_V4)
+      NeedsDeallocframe = false;
+  }
+
+  if (!NeedsDeallocframe)
+    return;
+  // If the returning instruction is JMPret, replace it with dealloc_return,
+  // otherwise just add deallocframe. The function could be returning via a
+  // tail call.
+  if (RetOpc != Hexagon::JMPret || DisableDeallocRet) {
+    BuildMI(MBB, InsertPt, DL, HII.get(Hexagon::L2_deallocframe));
+    return;
   }
+  unsigned NewOpc = Hexagon::L4_return;
+  MachineInstr *NewI = BuildMI(MBB, RetI, DL, HII.get(NewOpc));
+  // Transfer the function live-out registers.
+  NewI->copyImplicitOps(MF, RetI);
+  MBB.erase(RetI);
 }
 
+
 bool HexagonFrameLowering::hasFP(const MachineFunction &MF) const {
   const MachineFrameInfo *MFI = MF.getFrameInfo();
   const HexagonMachineFunctionInfo *FuncInfo =
     MF.getInfo<HexagonMachineFunctionInfo>();
-  return (MFI->hasCalls() || (MFI->getStackSize() > 0) ||
-          FuncInfo->hasClobberLR() );
+  return MFI->hasCalls() || MFI->getStackSize() > 0 ||
+         FuncInfo->hasClobberLR();
 }
 
-static inline
-unsigned uniqueSuperReg(unsigned Reg, const TargetRegisterInfo *TRI) {
-  MCSuperRegIterator SRI(Reg, TRI);
-  assert(SRI.isValid() && "Expected a superreg");
-  unsigned SuperReg = *SRI;
-  ++SRI;
-  assert(!SRI.isValid() && "Expected exactly one superreg");
-  return SuperReg;
-}
 
-bool
-HexagonFrameLowering::spillCalleeSavedRegisters(
-                                        MachineBasicBlock &MBB,
-                                        MachineBasicBlock::iterator MI,
-                                        const std::vector<CalleeSavedInfo> &CSI,
-                                        const TargetRegisterInfo *TRI) const {
-  MachineFunction *MF = MBB.getParent();
-  const TargetInstrInfo &TII = *MF->getSubtarget().getInstrInfo();
+enum SpillKind {
+  SK_ToMem,
+  SK_FromMem,
+  SK_FromMemTailcall
+};
 
-  if (CSI.empty()) {
-    return false;
+static const char *
+getSpillFunctionFor(unsigned MaxReg, SpillKind SpillType) {
+  const char * V4SpillToMemoryFunctions[] = {
+    "__save_r16_through_r17",
+    "__save_r16_through_r19",
+    "__save_r16_through_r21",
+    "__save_r16_through_r23",
+    "__save_r16_through_r25",
+    "__save_r16_through_r27" };
+
+  const char * V4SpillFromMemoryFunctions[] = {
+    "__restore_r16_through_r17_and_deallocframe",
+    "__restore_r16_through_r19_and_deallocframe",
+    "__restore_r16_through_r21_and_deallocframe",
+    "__restore_r16_through_r23_and_deallocframe",
+    "__restore_r16_through_r25_and_deallocframe",
+    "__restore_r16_through_r27_and_deallocframe" };
+
+  const char * V4SpillFromMemoryTailcallFunctions[] = {
+    "__restore_r16_through_r17_and_deallocframe_before_tailcall",
+    "__restore_r16_through_r19_and_deallocframe_before_tailcall",
+    "__restore_r16_through_r21_and_deallocframe_before_tailcall",
+    "__restore_r16_through_r23_and_deallocframe_before_tailcall",
+    "__restore_r16_through_r25_and_deallocframe_before_tailcall",
+    "__restore_r16_through_r27_and_deallocframe_before_tailcall"
+  };
+
+  const char **SpillFunc = nullptr;
+
+  switch(SpillType) {
+  case SK_ToMem:
+    SpillFunc = V4SpillToMemoryFunctions;
+    break;
+  case SK_FromMem:
+    SpillFunc = V4SpillFromMemoryFunctions;
+    break;
+  case SK_FromMemTailcall:
+    SpillFunc = V4SpillFromMemoryTailcallFunctions;
+    break;
+  }
+  assert(SpillFunc && "Unknown spill kind");
+
+  // Spill all callee-saved registers up to the highest register used.
+  switch (MaxReg) {
+  case Hexagon::R17:
+    return SpillFunc[0];
+  case Hexagon::R19:
+    return SpillFunc[1];
+  case Hexagon::R21:
+    return SpillFunc[2];
+  case Hexagon::R23:
+    return SpillFunc[3];
+  case Hexagon::R25:
+    return SpillFunc[4];
+  case Hexagon::R27:
+    return SpillFunc[5];
+  default:
+    llvm_unreachable("Unhandled maximum callee save register");
   }
+  return 0;
+}
 
-  // We can only schedule double loads if we spill contiguous callee-saved regs
-  // For instance, we cannot scheduled double-word loads if we spill r24,
-  // r26, and r27.
-  // Hexagon_TODO: We can try to double-word align odd registers for -O2 and
-  // above.
-  bool ContiguousRegs = true;
+/// Adds all callee-saved registers up to MaxReg to the instruction.
+static void addCalleeSaveRegistersAsImpOperand(MachineInstr *Inst,
+                                           unsigned MaxReg, bool IsDef) {
+  // Add the callee-saved registers as implicit uses.
+  for (unsigned R = Hexagon::R16; R <= MaxReg; ++R) {
+    MachineOperand ImpUse = MachineOperand::CreateReg(R, IsDef, true);
+    Inst->addOperand(ImpUse);
+  }
+}
 
-  for (unsigned i = 0; i < CSI.size(); ++i) {
+
+int HexagonFrameLowering::getFrameIndexOffset(const MachineFunction &MF,
+      int FI) const {
+  return MF.getFrameInfo()->getObjectOffset(FI);
+}
+
+
+bool HexagonFrameLowering::insertCSRSpillsInBlock(MachineBasicBlock &MBB,
+      const CSIVect &CSI, const HexagonRegisterInfo &HRI) const {
+  if (CSI.empty())
+    return true;
+
+  MachineBasicBlock::iterator MI = MBB.begin();
+  MachineFunction &MF = *MBB.getParent();
+  const TargetInstrInfo &TII = *MF.getSubtarget().getInstrInfo();
+
+  if (useSpillFunction(MF, CSI)) {
+    unsigned MaxReg = getMaxCalleeSavedReg(CSI, HRI);
+    const char *SpillFun = getSpillFunctionFor(MaxReg, SK_ToMem);
+    // Call spill function.
+    DebugLoc DL = MI != MBB.end() ? MI->getDebugLoc() : DebugLoc();
+    MachineInstr *SaveRegsCall =
+        BuildMI(MBB, MI, DL, TII.get(Hexagon::SAVE_REGISTERS_CALL_V4))
+          .addExternalSymbol(SpillFun);
+    // Add callee-saved registers as use.
+    addCalleeSaveRegistersAsImpOperand(SaveRegsCall, MaxReg, false);
+    // Add live in registers.
+    for (unsigned I = 0; I < CSI.size(); ++I)
+      MBB.addLiveIn(CSI[I].getReg());
+    return true;
+  }
+
+  for (unsigned i = 0, n = CSI.size(); i < n; ++i) {
     unsigned Reg = CSI[i].getReg();
+    // Add live in registers. We treat eh_return callee saved register r0 - r3
+    // specially. They are not really callee saved registers as they are not
+    // supposed to be killed.
+    bool IsKill = !HRI.isEHReturnCalleeSaveReg(Reg);
+    int FI = CSI[i].getFrameIdx();
+    const TargetRegisterClass *RC = HRI.getMinimalPhysRegClass(Reg);
+    TII.storeRegToStackSlot(MBB, MI, Reg, IsKill, FI, RC, &HRI);
+    if (IsKill)
+      MBB.addLiveIn(Reg);
+  }
+  return true;
+}
 
-    //
-    // Check if we can use a double-word store.
-    //
-    unsigned SuperReg = uniqueSuperReg(Reg, TRI);
-    bool CanUseDblStore = false;
-    const TargetRegisterClass* SuperRegClass = nullptr;
-
-    if (ContiguousRegs && (i < CSI.size()-1)) {
-      unsigned SuperRegNext = uniqueSuperReg(CSI[i+1].getReg(), TRI);
-      SuperRegClass = TRI->getMinimalPhysRegClass(SuperReg);
-      CanUseDblStore = (SuperRegNext == SuperReg);
-    }
 
+bool HexagonFrameLowering::insertCSRRestoresInBlock(MachineBasicBlock &MBB,
+      const CSIVect &CSI, const HexagonRegisterInfo &HRI) const {
+  if (CSI.empty())
+    return false;
+
+  MachineBasicBlock::iterator MI = MBB.getFirstTerminator();
+  MachineFunction &MF = *MBB.getParent();
+  const TargetInstrInfo &TII = *MF.getSubtarget().getInstrInfo();
 
-    if (CanUseDblStore) {
-      TII.storeRegToStackSlot(MBB, MI, SuperReg, true,
-                              CSI[i+1].getFrameIdx(), SuperRegClass, TRI);
-      MBB.addLiveIn(SuperReg);
-      ++i;
+  if (useRestoreFunction(MF, CSI)) {
+    bool HasTC = hasTailCall(MBB) || !hasReturn(MBB);
+    unsigned MaxR = getMaxCalleeSavedReg(CSI, HRI);
+    SpillKind Kind = HasTC ? SK_FromMemTailcall : SK_FromMem;
+    const char *RestoreFn = getSpillFunctionFor(MaxR, Kind);
+
+    // Call spill function.
+    DebugLoc DL = MI != MBB.end() ? MI->getDebugLoc()
+                                  : MBB.getLastNonDebugInstr()->getDebugLoc();
+    MachineInstr *DeallocCall = nullptr;
+
+    if (HasTC) {
+      unsigned ROpc = Hexagon::RESTORE_DEALLOC_BEFORE_TAILCALL_V4;
+      DeallocCall = BuildMI(MBB, MI, DL, TII.get(ROpc))
+          .addExternalSymbol(RestoreFn);
     } else {
-      // Cannot use a double-word store.
-      ContiguousRegs = false;
-      const TargetRegisterClass *RC = TRI->getMinimalPhysRegClass(Reg);
-      TII.storeRegToStackSlot(MBB, MI, Reg, true, CSI[i].getFrameIdx(), RC,
-                              TRI);
-      MBB.addLiveIn(Reg);
+      // The block has a return.
+      MachineBasicBlock::iterator It = MBB.getFirstTerminator();
+      assert(It->isReturn() && std::next(It) == MBB.end());
+      unsigned ROpc = Hexagon::RESTORE_DEALLOC_RET_JMP_V4;
+      DeallocCall = BuildMI(MBB, It, DL, TII.get(ROpc))
+          .addExternalSymbol(RestoreFn);
+      // Transfer the function live-out registers.
+      DeallocCall->copyImplicitOps(MF, It);
     }
+    addCalleeSaveRegistersAsImpOperand(DeallocCall, MaxR, true);
+    return true;
+  }
+
+  for (unsigned i = 0; i < CSI.size(); ++i) {
+    unsigned Reg = CSI[i].getReg();
+    const TargetRegisterClass *RC = HRI.getMinimalPhysRegClass(Reg);
+    int FI = CSI[i].getFrameIdx();
+    TII.loadRegFromStackSlot(MBB, MI, Reg, FI, RC, &HRI);
   }
   return true;
 }
 
 
-bool HexagonFrameLowering::restoreCalleeSavedRegisters(
-                                        MachineBasicBlock &MBB,
-                                        MachineBasicBlock::iterator MI,
-                                        const std::vector<CalleeSavedInfo> &CSI,
-                                        const TargetRegisterInfo *TRI) const {
+void HexagonFrameLowering::eliminateCallFramePseudoInstr(MachineFunction &MF,
+      MachineBasicBlock &MBB, MachineBasicBlock::iterator I) const {
+  MachineInstr &MI = *I;
+  unsigned Opc = MI.getOpcode();
+  (void)Opc; // Silence compiler warning.
+  assert((Opc == Hexagon::ADJCALLSTACKDOWN || Opc == Hexagon::ADJCALLSTACKUP) &&
+         "Cannot handle this call frame pseudo instruction");
+  MBB.erase(I);
+}
+
+
+void HexagonFrameLowering::processFunctionBeforeFrameFinalized(
+    MachineFunction &MF, RegScavenger *RS) const {
+  // If this function has uses aligned stack and also has variable sized stack
+  // objects, then we need to map all spill slots to fixed positions, so that
+  // they can be accessed through FP. Otherwise they would have to be accessed
+  // via AP, which may not be available at the particular place in the program.
+  MachineFrameInfo *MFI = MF.getFrameInfo();
+  bool HasAlloca = MFI->hasVarSizedObjects();
+  bool HasAligna = (MFI->getMaxAlignment() > getStackAlignment());
+
+  if (!HasAlloca || !HasAligna)
+    return;
+
+  unsigned LFS = MFI->getLocalFrameSize();
+  int Offset = -LFS;
+  for (int i = 0, e = MFI->getObjectIndexEnd(); i != e; ++i) {
+    if (!MFI->isSpillSlotObjectIndex(i) || MFI->isDeadObjectIndex(i))
+      continue;
+    int S = MFI->getObjectSize(i);
+    LFS += S;
+    Offset -= S;
+    MFI->mapLocalFrameObject(i, Offset);
+  }
+
+  MFI->setLocalFrameSize(LFS);
+  unsigned A = MFI->getLocalFrameMaxAlign();
+  assert(A <= 8 && "Unexpected local frame alignment");
+  if (A == 0)
+    MFI->setLocalFrameMaxAlign(8);
+  MFI->setUseLocalStackAllocationBlock(true);
+}
 
-  MachineFunction *MF = MBB.getParent();
-  const TargetInstrInfo &TII = *MF->getSubtarget().getInstrInfo();
+/// Returns true if there is no caller saved registers available.
+static bool needToReserveScavengingSpillSlots(MachineFunction &MF,
+                                              const HexagonRegisterInfo &HRI) {
+  MachineRegisterInfo &MRI = MF.getRegInfo();
+  const MCPhysReg *CallerSavedRegs = HRI.getCallerSavedRegs(&MF);
+  // Check for an unused caller-saved register.
+  for ( ; *CallerSavedRegs; ++CallerSavedRegs) {
+    MCPhysReg FreeReg = *CallerSavedRegs;
+    if (!MRI.reg_nodbg_empty(FreeReg))
+      continue;
 
-  if (CSI.empty()) {
+    // Check aliased register usage.
+    bool IsCurrentRegUsed = false;
+    for (MCRegAliasIterator AI(FreeReg, &HRI, false); AI.isValid(); ++AI)
+      if (!MRI.reg_nodbg_empty(*AI)) {
+        IsCurrentRegUsed = true;
+        break;
+      }
+    if (IsCurrentRegUsed)
+      continue;
+
+    // Neither directly used nor used through an aliased register.
     return false;
   }
+  // All caller-saved registers are used.
+  return true;
+}
 
-  // We can only schedule double loads if we spill contiguous callee-saved regs
-  // For instance, we cannot scheduled double-word loads if we spill r24,
-  // r26, and r27.
-  // Hexagon_TODO: We can try to double-word align odd registers for -O2 and
-  // above.
-  bool ContiguousRegs = true;
 
-  for (unsigned i = 0; i < CSI.size(); ++i) {
-    unsigned Reg = CSI[i].getReg();
+/// Replaces the predicate spill code pseudo instructions by valid instructions.
+bool HexagonFrameLowering::replacePredRegPseudoSpillCode(MachineFunction &MF)
+      const {
+  auto &HST = static_cast<const HexagonSubtarget&>(MF.getSubtarget());
+  auto &HII = *HST.getInstrInfo();
+  MachineRegisterInfo &MRI = MF.getRegInfo();
+  bool HasReplacedPseudoInst = false;
+  // Replace predicate spill pseudo instructions by real code.
+  // Loop over all of the basic blocks.
+  for (MachineFunction::iterator MBBb = MF.begin(), MBBe = MF.end();
+       MBBb != MBBe; ++MBBb) {
+    MachineBasicBlock* MBB = MBBb;
+    // Traverse the basic block.
+    MachineBasicBlock::iterator NextII;
+    for (MachineBasicBlock::iterator MII = MBB->begin(); MII != MBB->end();
+         MII = NextII) {
+      MachineInstr *MI = MII;
+      NextII = std::next(MII);
+      int Opc = MI->getOpcode();
+      if (Opc == Hexagon::STriw_pred) {
+        HasReplacedPseudoInst = true;
+        // STriw_pred FI, 0, SrcReg;
+        unsigned VirtReg = MRI.createVirtualRegister(&Hexagon::IntRegsRegClass);
+        unsigned SrcReg = MI->getOperand(2).getReg();
+        bool IsOrigSrcRegKilled = MI->getOperand(2).isKill();
+
+        assert(MI->getOperand(0).isFI() && "Expect a frame index");
+        assert(Hexagon::PredRegsRegClass.contains(SrcReg) &&
+               "Not a predicate register");
+
+        // Insert transfer to general purpose register.
+        //   VirtReg = C2_tfrpr SrcPredReg
+        BuildMI(*MBB, MII, MI->getDebugLoc(), HII.get(Hexagon::C2_tfrpr),
+                VirtReg).addReg(SrcReg, getKillRegState(IsOrigSrcRegKilled));
+
+        // Change instruction to S2_storeri_io.
+        //   S2_storeri_io FI, 0, VirtReg
+        MI->setDesc(HII.get(Hexagon::S2_storeri_io));
+        MI->getOperand(2).setReg(VirtReg);
+        MI->getOperand(2).setIsKill();
 
-    //
-    // Check if we can use a double-word load.
-    //
-    unsigned SuperReg = uniqueSuperReg(Reg, TRI);
-    const TargetRegisterClass* SuperRegClass = nullptr;
-    bool CanUseDblLoad = false;
-    if (ContiguousRegs && (i < CSI.size()-1)) {
-      unsigned SuperRegNext = uniqueSuperReg(CSI[i+1].getReg(), TRI);
-      SuperRegClass = TRI->getMinimalPhysRegClass(SuperReg);
-      CanUseDblLoad = (SuperRegNext == SuperReg);
+      } else if (Opc == Hexagon::LDriw_pred) {
+        // DstReg = LDriw_pred FI, 0
+        MachineOperand &M0 = MI->getOperand(0);
+        if (M0.isDead()) {
+          MBB->erase(MII);
+          continue;
+        }
+
+        unsigned VirtReg = MRI.createVirtualRegister(&Hexagon::IntRegsRegClass);
+        unsigned DestReg = MI->getOperand(0).getReg();
+
+        assert(MI->getOperand(1).isFI() && "Expect a frame index");
+        assert(Hexagon::PredRegsRegClass.contains(DestReg) &&
+               "Not a predicate register");
+
+        // Change instruction to L2_loadri_io.
+        //   VirtReg = L2_loadri_io FI, 0
+        MI->setDesc(HII.get(Hexagon::L2_loadri_io));
+        MI->getOperand(0).setReg(VirtReg);
+
+        // Insert transfer to general purpose register.
+        //   DestReg = C2_tfrrp VirtReg
+        const MCInstrDesc &D = HII.get(Hexagon::C2_tfrrp);
+        BuildMI(*MBB, std::next(MII), MI->getDebugLoc(), D, DestReg)
+          .addReg(VirtReg, getKillRegState(true));
+        HasReplacedPseudoInst = true;
+      }
     }
+  }
+  return HasReplacedPseudoInst;
+}
 
 
-    if (CanUseDblLoad) {
-      TII.loadRegFromStackSlot(MBB, MI, SuperReg, CSI[i+1].getFrameIdx(),
-                               SuperRegClass, TRI);
-      MBB.addLiveIn(SuperReg);
-      ++i;
-    } else {
-      // Cannot use a double-word load.
-      ContiguousRegs = false;
-      const TargetRegisterClass *RC = TRI->getMinimalPhysRegClass(Reg);
-      TII.loadRegFromStackSlot(MBB, MI, Reg, CSI[i].getFrameIdx(), RC, TRI);
-      MBB.addLiveIn(Reg);
+void HexagonFrameLowering::determineCalleeSaves(MachineFunction &MF,
+                                                BitVector &SavedRegs,
+                                                RegScavenger *RS) const {
+  TargetFrameLowering::determineCalleeSaves(MF, SavedRegs, RS);
+
+  auto &HST = static_cast<const HexagonSubtarget&>(MF.getSubtarget());
+  auto &HRI = *HST.getRegisterInfo();
+
+  bool HasEHReturn = MF.getInfo<HexagonMachineFunctionInfo>()->hasEHReturn();
+
+  // If we have a function containing __builtin_eh_return we want to spill and
+  // restore all callee saved registers. Pretend that they are used.
+  if (HasEHReturn) {
+    for (const MCPhysReg *CSRegs = HRI.getCalleeSavedRegs(&MF); *CSRegs;
+         ++CSRegs)
+      SavedRegs.set(*CSRegs);
+  }
+
+  const TargetRegisterClass &RC = Hexagon::IntRegsRegClass;
+
+  // Replace predicate register pseudo spill code.
+  bool HasReplacedPseudoInst = replacePredRegPseudoSpillCode(MF);
+
+  // We need to reserve a a spill slot if scavenging could potentially require
+  // spilling a scavenged register.
+  if (HasReplacedPseudoInst && needToReserveScavengingSpillSlots(MF, HRI)) {
+    MachineFrameInfo *MFI = MF.getFrameInfo();
+    for (int i=0; i < NumberScavengerSlots; i++)
+      RS->addScavengingFrameIndex(
+        MFI->CreateSpillStackObject(RC.getSize(), RC.getAlignment()));
+  }
+}
+
+
+#ifndef NDEBUG
+static void dump_registers(BitVector &Regs, const TargetRegisterInfo &TRI) {
+  dbgs() << '{';
+  for (int x = Regs.find_first(); x >= 0; x = Regs.find_next(x)) {
+    unsigned R = x;
+    dbgs() << ' ' << PrintReg(R, &TRI);
+  }
+  dbgs() << " }";
+}
+#endif
+
+
+bool HexagonFrameLowering::assignCalleeSavedSpillSlots(MachineFunction &MF,
+      const TargetRegisterInfo *TRI, std::vector<CalleeSavedInfo> &CSI) const {
+  DEBUG(dbgs() << LLVM_FUNCTION_NAME << " on "
+               << MF.getFunction()->getName() << '\n');
+  MachineFrameInfo *MFI = MF.getFrameInfo();
+  BitVector SRegs(Hexagon::NUM_TARGET_REGS);
+
+  // Generate a set of unique, callee-saved registers (SRegs), where each
+  // register in the set is maximal in terms of sub-/super-register relation,
+  // i.e. for each R in SRegs, no proper super-register of R is also in SRegs.
+
+  // (1) For each callee-saved register, add that register and all of its
+  // sub-registers to SRegs.
+  DEBUG(dbgs() << "Initial CS registers: {");
+  for (unsigned i = 0, n = CSI.size(); i < n; ++i) {
+    unsigned R = CSI[i].getReg();
+    DEBUG(dbgs() << ' ' << PrintReg(R, TRI));
+    for (MCSubRegIterator SR(R, TRI, true); SR.isValid(); ++SR)
+      SRegs[*SR] = true;
+  }
+  DEBUG(dbgs() << " }\n");
+  DEBUG(dbgs() << "SRegs.1: "; dump_registers(SRegs, *TRI); dbgs() << "\n");
+
+  // (2) For each reserved register, remove that register and all of its
+  // sub- and super-registers from SRegs.
+  BitVector Reserved = TRI->getReservedRegs(MF);
+  for (int x = Reserved.find_first(); x >= 0; x = Reserved.find_next(x)) {
+    unsigned R = x;
+    for (MCSuperRegIterator SR(R, TRI, true); SR.isValid(); ++SR)
+      SRegs[*SR] = false;
+  }
+  DEBUG(dbgs() << "Res:     "; dump_registers(Reserved, *TRI); dbgs() << "\n");
+  DEBUG(dbgs() << "SRegs.2: "; dump_registers(SRegs, *TRI); dbgs() << "\n");
+
+  // (3) Collect all registers that have at least one sub-register in SRegs,
+  // and also have no sub-registers that are reserved. These will be the can-
+  // didates for saving as a whole instead of their individual sub-registers.
+  // (Saving R17:16 instead of R16 is fine, but only if R17 was not reserved.)
+  BitVector TmpSup(Hexagon::NUM_TARGET_REGS);
+  for (int x = SRegs.find_first(); x >= 0; x = SRegs.find_next(x)) {
+    unsigned R = x;
+    for (MCSuperRegIterator SR(R, TRI); SR.isValid(); ++SR)
+      TmpSup[*SR] = true;
+  }
+  for (int x = TmpSup.find_first(); x >= 0; x = TmpSup.find_next(x)) {
+    unsigned R = x;
+    for (MCSubRegIterator SR(R, TRI, true); SR.isValid(); ++SR) {
+      if (!Reserved[*SR])
+        continue;
+      TmpSup[R] = false;
+      break;
     }
   }
+  DEBUG(dbgs() << "TmpSup:  "; dump_registers(TmpSup, *TRI); dbgs() << "\n");
+
+  // (4) Include all super-registers found in (3) into SRegs.
+  SRegs |= TmpSup;
+  DEBUG(dbgs() << "SRegs.4: "; dump_registers(SRegs, *TRI); dbgs() << "\n");
+
+  // (5) For each register R in SRegs, if any super-register of R is in SRegs,
+  // remove R from SRegs.
+  for (int x = SRegs.find_first(); x >= 0; x = SRegs.find_next(x)) {
+    unsigned R = x;
+    for (MCSuperRegIterator SR(R, TRI); SR.isValid(); ++SR) {
+      if (!SRegs[*SR])
+        continue;
+      SRegs[R] = false;
+      break;
+    }
+  }
+  DEBUG(dbgs() << "SRegs.5: "; dump_registers(SRegs, *TRI); dbgs() << "\n");
+
+  // Now, for each register that has a fixed stack slot, create the stack
+  // object for it.
+  CSI.clear();
+
+  typedef TargetFrameLowering::SpillSlot SpillSlot;
+  unsigned NumFixed;
+  int MinOffset = 0;  // CS offsets are negative.
+  const SpillSlot *FixedSlots = getCalleeSavedSpillSlots(NumFixed);
+  for (const SpillSlot *S = FixedSlots; S != FixedSlots+NumFixed; ++S) {
+    if (!SRegs[S->Reg])
+      continue;
+    const TargetRegisterClass *RC = TRI->getMinimalPhysRegClass(S->Reg);
+    int FI = MFI->CreateFixedSpillStackObject(RC->getSize(), S->Offset);
+    MinOffset = std::min(MinOffset, S->Offset);
+    CSI.push_back(CalleeSavedInfo(S->Reg, FI));
+    SRegs[S->Reg] = false;
+  }
+
+  // There can be some registers that don't have fixed slots. For example,
+  // we need to store R0-R3 in functions with exception handling. For each
+  // such register, create a non-fixed stack object.
+  for (int x = SRegs.find_first(); x >= 0; x = SRegs.find_next(x)) {
+    unsigned R = x;
+    const TargetRegisterClass *RC = TRI->getMinimalPhysRegClass(R);
+    int Off = MinOffset - RC->getSize();
+    unsigned Align = std::min(RC->getAlignment(), getStackAlignment());
+    assert(isPowerOf2_32(Align));
+    Off &= -Align;
+    int FI = MFI->CreateFixedSpillStackObject(RC->getSize(), Off);
+    MinOffset = std::min(MinOffset, Off);
+    CSI.push_back(CalleeSavedInfo(R, FI));
+    SRegs[R] = false;
+  }
+
+  DEBUG({
+    dbgs() << "CS information: {";
+    for (unsigned i = 0, n = CSI.size(); i < n; ++i) {
+      int FI = CSI[i].getFrameIdx();
+      int Off = MFI->getObjectOffset(FI);
+      dbgs() << ' ' << PrintReg(CSI[i].getReg(), TRI) << ":fi#" << FI << ":sp";
+      if (Off >= 0)
+        dbgs() << '+';
+      dbgs() << Off;
+    }
+    dbgs() << " }\n";
+  });
+
+#ifndef NDEBUG
+  // Verify that all registers were handled.
+  bool MissedReg = false;
+  for (int x = SRegs.find_first(); x >= 0; x = SRegs.find_next(x)) {
+    unsigned R = x;
+    dbgs() << PrintReg(R, TRI) << ' ';
+    MissedReg = true;
+  }
+  if (MissedReg)
+    llvm_unreachable("...there are unhandled callee-saved registers!");
+#endif
+
   return true;
 }
 
-void HexagonFrameLowering::
-eliminateCallFramePseudoInstr(MachineFunction &MF, MachineBasicBlock &MBB,
-                              MachineBasicBlock::iterator I) const {
-  MachineInstr &MI = *I;
 
-  if (MI.getOpcode() == Hexagon::ADJCALLSTACKDOWN) {
-    // Hexagon_TODO: add code
-  } else if (MI.getOpcode() == Hexagon::ADJCALLSTACKUP) {
-    // Hexagon_TODO: add code
-  } else {
-    llvm_unreachable("Cannot handle this call frame pseudo instruction");
+void HexagonFrameLowering::expandAlloca(MachineInstr *AI,
+      const HexagonInstrInfo &HII, unsigned SP, unsigned CF) const {
+  MachineBasicBlock &MB = *AI->getParent();
+  DebugLoc DL = AI->getDebugLoc();
+  unsigned A = AI->getOperand(2).getImm();
+
+  // Have
+  //    Rd  = alloca Rs, #A
+  //
+  // If Rs and Rd are different registers, use this sequence:
+  //    Rd  = sub(r29, Rs)
+  //    r29 = sub(r29, Rs)
+  //    Rd  = and(Rd, #-A)    ; if necessary
+  //    r29 = and(r29, #-A)   ; if necessary
+  //    Rd  = add(Rd, #CF)    ; CF size aligned to at most A
+  // otherwise, do
+  //    Rd  = sub(r29, Rs)
+  //    Rd  = and(Rd, #-A)    ; if necessary
+  //    r29 = Rd
+  //    Rd  = add(Rd, #CF)    ; CF size aligned to at most A
+
+  MachineOperand &RdOp = AI->getOperand(0);
+  MachineOperand &RsOp = AI->getOperand(1);
+  unsigned Rd = RdOp.getReg(), Rs = RsOp.getReg();
+
+  // Rd = sub(r29, Rs)
+  BuildMI(MB, AI, DL, HII.get(Hexagon::A2_sub), Rd)
+      .addReg(SP)
+      .addReg(Rs);
+  if (Rs != Rd) {
+    // r29 = sub(r29, Rs)
+    BuildMI(MB, AI, DL, HII.get(Hexagon::A2_sub), SP)
+        .addReg(SP)
+        .addReg(Rs);
+  }
+  if (A > 8) {
+    // Rd  = and(Rd, #-A)
+    BuildMI(MB, AI, DL, HII.get(Hexagon::A2_andir), Rd)
+        .addReg(Rd)
+        .addImm(-int64_t(A));
+    if (Rs != Rd)
+      BuildMI(MB, AI, DL, HII.get(Hexagon::A2_andir), SP)
+          .addReg(SP)
+          .addImm(-int64_t(A));
+  }
+  if (Rs == Rd) {
+    // r29 = Rd
+    BuildMI(MB, AI, DL, HII.get(TargetOpcode::COPY), SP)
+        .addReg(Rd);
+  }
+  if (CF > 0) {
+    // Rd = add(Rd, #CF)
+    BuildMI(MB, AI, DL, HII.get(Hexagon::A2_addi), Rd)
+        .addReg(Rd)
+        .addImm(CF);
   }
-  MBB.erase(I);
 }
 
-int HexagonFrameLowering::getFrameIndexOffset(const MachineFunction &MF,
-                                              int FI) const {
-  return MF.getFrameInfo()->getObjectOffset(FI);
+
+bool HexagonFrameLowering::needsAligna(const MachineFunction &MF) const {
+  const MachineFrameInfo *MFI = MF.getFrameInfo();
+  if (!MFI->hasVarSizedObjects())
+    return false;
+  unsigned MaxA = MFI->getMaxAlignment();
+  if (MaxA <= getStackAlignment())
+    return false;
+  return true;
+}
+
+
+MachineInstr *HexagonFrameLowering::getAlignaInstr(MachineFunction &MF) const {
+  for (auto &B : MF)
+    for (auto &I : B)
+      if (I.getOpcode() == Hexagon::ALIGNA)
+        return &I;
+  return nullptr;
+}
+
+
+inline static bool isOptSize(const MachineFunction &MF) {
+  AttributeSet AF = MF.getFunction()->getAttributes();
+  return AF.hasAttribute(AttributeSet::FunctionIndex,
+                         Attribute::OptimizeForSize);
+}
+
+inline static bool isMinSize(const MachineFunction &MF) {
+  AttributeSet AF = MF.getFunction()->getAttributes();
+  return AF.hasAttribute(AttributeSet::FunctionIndex, Attribute::MinSize);
+}
+
+
+/// Determine whether the callee-saved register saves and restores should
+/// be generated via inline code. If this function returns "true", inline
+/// code will be generated. If this function returns "false", additional
+/// checks are performed, which may still lead to the inline code.
+bool HexagonFrameLowering::shouldInlineCSR(MachineFunction &MF,
+      const CSIVect &CSI) const {
+  if (MF.getInfo<HexagonMachineFunctionInfo>()->hasEHReturn())
+    return true;
+  if (!isOptSize(MF) && !isMinSize(MF))
+    if (MF.getTarget().getOptLevel() > CodeGenOpt::Default)
+      return true;
+
+  // Check if CSI only has double registers, and if the registers form
+  // a contiguous block starting from D8.
+  BitVector Regs(Hexagon::NUM_TARGET_REGS);
+  for (unsigned i = 0, n = CSI.size(); i < n; ++i) {
+    unsigned R = CSI[i].getReg();
+    if (!Hexagon::DoubleRegsRegClass.contains(R))
+      return true;
+    Regs[R] = true;
+  }
+  int F = Regs.find_first();
+  if (F != Hexagon::D8)
+    return true;
+  while (F >= 0) {
+    int N = Regs.find_next(F);
+    if (N >= 0 && N != F+1)
+      return true;
+    F = N;
+  }
+
+  return false;
+}
+
+
+bool HexagonFrameLowering::useSpillFunction(MachineFunction &MF,
+      const CSIVect &CSI) const {
+  if (shouldInlineCSR(MF, CSI))
+    return false;
+  unsigned NumCSI = CSI.size();
+  if (NumCSI <= 1)
+    return false;
+
+  unsigned Threshold = isOptSize(MF) ? SpillFuncThresholdOs
+                                     : SpillFuncThreshold;
+  return Threshold < NumCSI;
+}
+
+
+bool HexagonFrameLowering::useRestoreFunction(MachineFunction &MF,
+      const CSIVect &CSI) const {
+  if (shouldInlineCSR(MF, CSI))
+    return false;
+  unsigned NumCSI = CSI.size();
+  unsigned Threshold = isOptSize(MF) ? SpillFuncThresholdOs-1
+                                     : SpillFuncThreshold;
+  return Threshold < NumCSI;
 }
+
diff --git a/lib/Target/Hexagon/HexagonFrameLowering.h b/lib/Target/Hexagon/HexagonFrameLowering.h
index 2d6b4579380..d39ee2c7719 100644
--- a/lib/Target/Hexagon/HexagonFrameLowering.h
+++ b/lib/Target/Hexagon/HexagonFrameLowering.h
@@ -15,35 +15,88 @@
 
 namespace llvm {
 
-class HexagonFrameLowering : public TargetFrameLowering {
-private:
-  void determineFrameLayout(MachineFunction &MF) const;
+class HexagonInstrInfo;
+class HexagonRegisterInfo;
 
+class HexagonFrameLowering : public TargetFrameLowering {
 public:
-  explicit HexagonFrameLowering() : TargetFrameLowering(StackGrowsDown, 8, 0) {}
+  explicit HexagonFrameLowering()
+      : TargetFrameLowering(StackGrowsDown, 8, 0, 1, true) {}
 
-  /// emitProlog/emitEpilog - These methods insert prolog and epilog code into
-  /// the function.
-  void emitPrologue(MachineFunction &MF) const override;
-  void emitEpilogue(MachineFunction &MF, MachineBasicBlock &MBB) const override;
+  // All of the prolog/epilog functionality, including saving and restoring
+  // callee-saved registers is handled in emitPrologue. This is to have the
+  // logic for shrink-wrapping in one place.
+  void emitPrologue(MachineFunction &MF, MachineBasicBlock &MBB) const
+      override;
+  void emitEpilogue(MachineFunction &MF, MachineBasicBlock &MBB) const
+      override {}
   bool spillCalleeSavedRegisters(MachineBasicBlock &MBB,
-                                 MachineBasicBlock::iterator MI,
-                                 const std::vector<CalleeSavedInfo> &CSI,
-                                 const TargetRegisterInfo *TRI) const override;
-
-  void
-  eliminateCallFramePseudoInstr(MachineFunction &MF,
-                                MachineBasicBlock &MBB,
-                                MachineBasicBlock::iterator I) const override;
-
-  bool
-  restoreCalleeSavedRegisters(MachineBasicBlock &MBB,
-                              MachineBasicBlock::iterator MI,
-                              const std::vector<CalleeSavedInfo> &CSI,
-                              const TargetRegisterInfo *TRI) const override;
+      MachineBasicBlock::iterator MI, const std::vector<CalleeSavedInfo> &CSI,
+      const TargetRegisterInfo *TRI) const override {
+    return true;
+  }
+  bool restoreCalleeSavedRegisters(MachineBasicBlock &MBB,
+      MachineBasicBlock::iterator MI, const std::vector<CalleeSavedInfo> &CSI,
+      const TargetRegisterInfo *TRI) const override {
+    return true;
+  }
+
+  void eliminateCallFramePseudoInstr(MachineFunction &MF,
+      MachineBasicBlock &MBB, MachineBasicBlock::iterator I) const override;
+  void processFunctionBeforeFrameFinalized(MachineFunction &MF,
+        RegScavenger *RS = nullptr) const override;
+  void determineCalleeSaves(MachineFunction &MF, BitVector &SavedRegs,
+        RegScavenger *RS) const override;
+
+  bool targetHandlesStackFrameRounding() const override {
+    return true;
+  }
   int getFrameIndexOffset(const MachineFunction &MF, int FI) const override;
   bool hasFP(const MachineFunction &MF) const override;
-  bool hasTailCall(MachineBasicBlock &MBB) const;
+
+  const SpillSlot *getCalleeSavedSpillSlots(unsigned &NumEntries)
+        const override {
+    static const SpillSlot Offsets[] = {
+      { Hexagon::R17, -4 }, { Hexagon::R16, -8 }, { Hexagon::D8, -8 },
+      { Hexagon::R19, -12 }, { Hexagon::R18, -16 }, { Hexagon::D9, -16 },
+      { Hexagon::R21, -20 }, { Hexagon::R20, -24 }, { Hexagon::D10, -24 },
+      { Hexagon::R23, -28 }, { Hexagon::R22, -32 }, { Hexagon::D11, -32 },
+      { Hexagon::R25, -36 }, { Hexagon::R24, -40 }, { Hexagon::D12, -40 },
+      { Hexagon::R27, -44 }, { Hexagon::R26, -48 }, { Hexagon::D13, -48 }
+    };
+    NumEntries = array_lengthof(Offsets);
+    return Offsets;
+  }
+
+  bool assignCalleeSavedSpillSlots(MachineFunction &MF,
+      const TargetRegisterInfo *TRI, std::vector<CalleeSavedInfo> &CSI)
+      const override;
+
+  bool needsAligna(const MachineFunction &MF) const;
+  MachineInstr *getAlignaInstr(MachineFunction &MF) const;
+
+private:
+  typedef std::vector<CalleeSavedInfo> CSIVect;
+
+  void expandAlloca(MachineInstr *AI, const HexagonInstrInfo &TII,
+      unsigned SP, unsigned CF) const;
+  void insertPrologueInBlock(MachineBasicBlock &MBB) const;
+  void insertEpilogueInBlock(MachineBasicBlock &MBB) const;
+  bool insertCSRSpillsInBlock(MachineBasicBlock &MBB, const CSIVect &CSI,
+      const HexagonRegisterInfo &HRI) const;
+  bool insertCSRRestoresInBlock(MachineBasicBlock &MBB, const CSIVect &CSI,
+      const HexagonRegisterInfo &HRI) const;
+
+  void adjustForCalleeSavedRegsSpillCall(MachineFunction &MF) const;
+  bool replacePredRegPseudoSpillCode(MachineFunction &MF) const;
+  bool replaceVecPredRegPseudoSpillCode(MachineFunction &MF) const;
+
+  void findShrunkPrologEpilog(MachineFunction &MF, MachineBasicBlock *&PrologB,
+      MachineBasicBlock *&EpilogB) const;
+
+  bool shouldInlineCSR(llvm::MachineFunction&, const CSIVect&) const;
+  bool useSpillFunction(MachineFunction &MF, const CSIVect &CSI) const;
+  bool useRestoreFunction(MachineFunction &MF, const CSIVect &CSI) const;
 };
 
 } // End llvm namespace
diff --git a/lib/Target/Hexagon/HexagonGenExtract.cpp b/lib/Target/Hexagon/HexagonGenExtract.cpp
new file mode 100644
index 00000000000..4d32208bd5a
--- /dev/null
+++ b/lib/Target/Hexagon/HexagonGenExtract.cpp
@@ -0,0 +1,259 @@
+//===--- HexagonGenExtract.cpp --------------------------------------------===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/ADT/STLExtras.h"
+#include "llvm/CodeGen/MachineFunctionAnalysis.h"
+#include "llvm/IR/Constants.h"
+#include "llvm/IR/Dominators.h"
+#include "llvm/IR/Function.h"
+#include "llvm/IR/Instructions.h"
+#include "llvm/IR/IntrinsicInst.h"
+#include "llvm/IR/IRBuilder.h"
+#include "llvm/IR/PatternMatch.h"
+#include "llvm/Pass.h"
+#include "llvm/Support/CommandLine.h"
+#include "llvm/Support/Debug.h"
+#include "llvm/Support/MathExtras.h"
+#include "llvm/Support/raw_ostream.h"
+
+using namespace llvm;
+
+static cl::opt<unsigned> ExtractCutoff("extract-cutoff", cl::init(~0U),
+  cl::Hidden, cl::desc("Cutoff for generating \"extract\""
+  " instructions"));
+
+// This prevents generating extract instructions that have the offset of 0.
+// One of the reasons for "extract" is to put a sequence of bits in a regis-
+// ter, starting at offset 0 (so that these bits can then be used by an
+// "insert"). If the bits are already at offset 0, it is better not to gene-
+// rate "extract", since logical bit operations can be merged into compound
+// instructions (as opposed to "extract").
+static cl::opt<bool> NoSR0("extract-nosr0", cl::init(true), cl::Hidden,
+  cl::desc("No extract instruction with offset 0"));
+
+static cl::opt<bool> NeedAnd("extract-needand", cl::init(true), cl::Hidden,
+  cl::desc("Require & in extract patterns"));
+
+namespace llvm {
+  void initializeHexagonGenExtractPass(PassRegistry&);
+  FunctionPass *createHexagonGenExtract();
+}
+
+
+namespace {
+  class HexagonGenExtract : public FunctionPass {
+  public:
+    static char ID;
+    HexagonGenExtract() : FunctionPass(ID), ExtractCount(0) {
+      initializeHexagonGenExtractPass(*PassRegistry::getPassRegistry());
+    }
+    virtual const char *getPassName() const override {
+      return "Hexagon generate \"extract\" instructions";
+    }
+    virtual bool runOnFunction(Function &F) override;
+    virtual void getAnalysisUsage(AnalysisUsage &AU) const override {
+      AU.addRequired<DominatorTreeWrapperPass>();
+      AU.addPreserved<DominatorTreeWrapperPass>();
+      AU.addPreserved<MachineFunctionAnalysis>();
+      FunctionPass::getAnalysisUsage(AU);
+    }
+  private:
+    bool visitBlock(BasicBlock *B);
+    bool convert(Instruction *In);
+
+    unsigned ExtractCount;
+    DominatorTree *DT;
+  };
+
+  char HexagonGenExtract::ID = 0;
+}
+
+INITIALIZE_PASS_BEGIN(HexagonGenExtract, "hextract", "Hexagon generate "
+  "\"extract\" instructions", false, false)
+INITIALIZE_PASS_DEPENDENCY(DominatorTreeWrapperPass)
+INITIALIZE_PASS_END(HexagonGenExtract, "hextract", "Hexagon generate "
+  "\"extract\" instructions", false, false)
+
+
+bool HexagonGenExtract::convert(Instruction *In) {
+  using namespace PatternMatch;
+  Value *BF = 0;
+  ConstantInt *CSL = 0, *CSR = 0, *CM = 0;
+  BasicBlock *BB = In->getParent();
+  LLVMContext &Ctx = BB->getContext();
+  bool LogicalSR;
+
+  // (and (shl (lshr x, #sr), #sl), #m)
+  LogicalSR = true;
+  bool Match = match(In, m_And(m_Shl(m_LShr(m_Value(BF), m_ConstantInt(CSR)),
+                               m_ConstantInt(CSL)),
+                         m_ConstantInt(CM)));
+
+  if (!Match) {
+    // (and (shl (ashr x, #sr), #sl), #m)
+    LogicalSR = false;
+    Match = match(In, m_And(m_Shl(m_AShr(m_Value(BF), m_ConstantInt(CSR)),
+                            m_ConstantInt(CSL)),
+                      m_ConstantInt(CM)));
+  }
+  if (!Match) {
+    // (and (shl x, #sl), #m)
+    LogicalSR = true;
+    CSR = ConstantInt::get(Type::getInt32Ty(Ctx), 0);
+    Match = match(In, m_And(m_Shl(m_Value(BF), m_ConstantInt(CSL)),
+                      m_ConstantInt(CM)));
+    if (Match && NoSR0)
+      return false;
+  }
+  if (!Match) {
+    // (and (lshr x, #sr), #m)
+    LogicalSR = true;
+    CSL = ConstantInt::get(Type::getInt32Ty(Ctx), 0);
+    Match = match(In, m_And(m_LShr(m_Value(BF), m_ConstantInt(CSR)),
+                            m_ConstantInt(CM)));
+  }
+  if (!Match) {
+    // (and (ashr x, #sr), #m)
+    LogicalSR = false;
+    CSL = ConstantInt::get(Type::getInt32Ty(Ctx), 0);
+    Match = match(In, m_And(m_AShr(m_Value(BF), m_ConstantInt(CSR)),
+                            m_ConstantInt(CM)));
+  }
+  if (!Match) {
+    CM = 0;
+    // (shl (lshr x, #sr), #sl)
+    LogicalSR = true;
+    Match = match(In, m_Shl(m_LShr(m_Value(BF), m_ConstantInt(CSR)),
+                            m_ConstantInt(CSL)));
+  }
+  if (!Match) {
+    CM = 0;
+    // (shl (ashr x, #sr), #sl)
+    LogicalSR = false;
+    Match = match(In, m_Shl(m_AShr(m_Value(BF), m_ConstantInt(CSR)),
+                            m_ConstantInt(CSL)));
+  }
+  if (!Match)
+    return false;
+
+  Type *Ty = BF->getType();
+  if (!Ty->isIntegerTy())
+    return false;
+  unsigned BW = Ty->getPrimitiveSizeInBits();
+  if (BW != 32 && BW != 64)
+    return false;
+
+  uint32_t SR = CSR->getZExtValue();
+  uint32_t SL = CSL->getZExtValue();
+
+  if (!CM) {
+    // If there was no and, and the shift left did not remove all potential
+    // sign bits created by the shift right, then extractu cannot reproduce
+    // this value.
+    if (!LogicalSR && (SR > SL))
+      return false;
+    APInt A = APInt(BW, ~0ULL).lshr(SR).shl(SL);
+    CM = ConstantInt::get(Ctx, A);
+  }
+
+  // CM is the shifted-left mask. Shift it back right to remove the zero
+  // bits on least-significant positions.
+  APInt M = CM->getValue().lshr(SL);
+  uint32_t T = M.countTrailingOnes();
+
+  // During the shifts some of the bits will be lost. Calculate how many
+  // of the original value will remain after shift right and then left.
+  uint32_t U = BW - std::max(SL, SR);
+  // The width of the extracted field is the minimum of the original bits
+  // that remain after the shifts and the number of contiguous 1s in the mask.
+  uint32_t W = std::min(U, T);
+  if (W == 0)
+    return false;
+
+  // Check if the extracted bits are contained within the mask that it is
+  // and-ed with. The extract operation will copy these bits, and so the
+  // mask cannot any holes in it that would clear any of the bits of the
+  // extracted field.
+  if (!LogicalSR) {
+    // If the shift right was arithmetic, it could have included some 1 bits.
+    // It is still ok to generate extract, but only if the mask eliminates
+    // those bits (i.e. M does not have any bits set beyond U).
+    APInt C = APInt::getHighBitsSet(BW, BW-U);
+    if (M.intersects(C) || !APIntOps::isMask(W, M))
+      return false;
+  } else {
+    // Check if M starts with a contiguous sequence of W times 1 bits. Get
+    // the low U bits of M (which eliminates the 0 bits shifted in on the
+    // left), and check if the result is APInt's "mask":
+    if (!APIntOps::isMask(W, M.getLoBits(U)))
+      return false;
+  }
+
+  IRBuilder<> IRB(BB, In);
+  Intrinsic::ID IntId = (BW == 32) ? Intrinsic::hexagon_S2_extractu
+                                   : Intrinsic::hexagon_S2_extractup;
+  Module *Mod = BB->getParent()->getParent();
+  Value *ExtF = Intrinsic::getDeclaration(Mod, IntId);
+  Value *NewIn = IRB.CreateCall(ExtF, {BF, IRB.getInt32(W), IRB.getInt32(SR)});
+  if (SL != 0)
+    NewIn = IRB.CreateShl(NewIn, SL, CSL->getName());
+  In->replaceAllUsesWith(NewIn);
+  return true;
+}
+
+
+bool HexagonGenExtract::visitBlock(BasicBlock *B) {
+  // Depth-first, bottom-up traversal.
+  DomTreeNode *DTN = DT->getNode(B);
+  typedef GraphTraits<DomTreeNode*> GTN;
+  typedef GTN::ChildIteratorType Iter;
+  for (Iter I = GTN::child_begin(DTN), E = GTN::child_end(DTN); I != E; ++I)
+    visitBlock((*I)->getBlock());
+
+  // Allow limiting the number of generated extracts for debugging purposes.
+  bool HasCutoff = ExtractCutoff.getPosition();
+  unsigned Cutoff = ExtractCutoff;
+
+  bool Changed = false;
+  BasicBlock::iterator I = std::prev(B->end()), NextI, Begin = B->begin();
+  while (true) {
+    if (HasCutoff && (ExtractCount >= Cutoff))
+      return Changed;
+    bool Last = (I == Begin);
+    if (!Last)
+      NextI = std::prev(I);
+    Instruction *In = &*I;
+    bool Done = convert(In);
+    if (HasCutoff && Done)
+      ExtractCount++;
+    Changed |= Done;
+    if (Last)
+      break;
+    I = NextI;
+  }
+  return Changed;
+}
+
+
+bool HexagonGenExtract::runOnFunction(Function &F) {
+  DT = &getAnalysis<DominatorTreeWrapperPass>().getDomTree();
+  bool Changed;
+
+  // Traverse the function bottom-up, to see super-expressions before their
+  // sub-expressions.
+  BasicBlock *Entry = GraphTraits<Function*>::getEntryNode(&F);
+  Changed = visitBlock(Entry);
+
+  return Changed;
+}
+
+
+FunctionPass *llvm::createHexagonGenExtract() {
+  return new HexagonGenExtract();
+}
diff --git a/lib/Target/Hexagon/HexagonGenInsert.cpp b/lib/Target/Hexagon/HexagonGenInsert.cpp
new file mode 100644
index 00000000000..096da949e77
--- /dev/null
+++ b/lib/Target/Hexagon/HexagonGenInsert.cpp
@@ -0,0 +1,1598 @@
+//===--- HexagonGenInsert.cpp ---------------------------------------------===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+
+#define DEBUG_TYPE "hexinsert"
+
+#include "llvm/Pass.h"
+#include "llvm/PassRegistry.h"
+#include "llvm/ADT/BitVector.h"
+#include "llvm/ADT/DenseMap.h"
+#include "llvm/ADT/DenseSet.h"
+#include "llvm/ADT/PostOrderIterator.h"
+#include "llvm/CodeGen/MachineDominators.h"
+#include "llvm/CodeGen/MachineFunction.h"
+#include "llvm/CodeGen/MachineFunctionPass.h"
+#include "llvm/CodeGen/MachineInstrBuilder.h"
+#include "llvm/CodeGen/MachineRegisterInfo.h"
+#include "llvm/IR/Constants.h"
+#include "llvm/Support/CommandLine.h"
+#include "llvm/Support/Debug.h"
+#include "llvm/Support/raw_ostream.h"
+#include "llvm/Support/Timer.h"
+#include "llvm/Target/TargetMachine.h"
+#include "llvm/Target/TargetRegisterInfo.h"
+
+#include "Hexagon.h"
+#include "HexagonRegisterInfo.h"
+#include "HexagonTargetMachine.h"
+#include "HexagonBitTracker.h"
+
+#include <map>
+#include <vector>
+
+using namespace llvm;
+
+static cl::opt<unsigned> VRegIndexCutoff("insert-vreg-cutoff", cl::init(~0U),
+  cl::Hidden, cl::ZeroOrMore, cl::desc("Vreg# cutoff for insert generation."));
+// The distance cutoff is selected based on the precheckin-perf results:
+// cutoffs 20, 25, 35, and 40 are worse than 30.
+static cl::opt<unsigned> VRegDistCutoff("insert-dist-cutoff", cl::init(30U),
+  cl::Hidden, cl::ZeroOrMore, cl::desc("Vreg distance cutoff for insert "
+  "generation."));
+
+static cl::opt<bool> OptTiming("insert-timing", cl::init(false), cl::Hidden,
+  cl::ZeroOrMore, cl::desc("Enable timing of insert generation"));
+static cl::opt<bool> OptTimingDetail("insert-timing-detail", cl::init(false),
+  cl::Hidden, cl::ZeroOrMore, cl::desc("Enable detailed timing of insert "
+  "generation"));
+
+static cl::opt<bool> OptSelectAll0("insert-all0", cl::init(false), cl::Hidden,
+  cl::ZeroOrMore);
+static cl::opt<bool> OptSelectHas0("insert-has0", cl::init(false), cl::Hidden,
+  cl::ZeroOrMore);
+// Whether to construct constant values via "insert". Could eliminate constant
+// extenders, but often not practical.
+static cl::opt<bool> OptConst("insert-const", cl::init(false), cl::Hidden,
+  cl::ZeroOrMore);
+
+namespace {
+  // The preprocessor gets confused when the DEBUG macro is passed larger
+  // chunks of code. Use this function to detect debugging.
+  inline bool isDebug() {
+#ifndef NDEBUG
+    return ::llvm::DebugFlag && ::llvm::isCurrentDebugType(DEBUG_TYPE);
+#else
+    return false;
+#endif
+  }
+}
+
+
+namespace {
+  // Set of virtual registers, based on BitVector.
+  struct RegisterSet : private BitVector {
+    RegisterSet() : BitVector() {}
+    explicit RegisterSet(unsigned s, bool t = false) : BitVector(s, t) {}
+    RegisterSet(const RegisterSet &RS) : BitVector(RS) {}
+
+    using BitVector::clear;
+
+    unsigned find_first() const {
+      int First = BitVector::find_first();
+      if (First < 0)
+        return 0;
+      return x2v(First);
+    }
+
+    unsigned find_next(unsigned Prev) const {
+      int Next = BitVector::find_next(v2x(Prev));
+      if (Next < 0)
+        return 0;
+      return x2v(Next);
+    }
+
+    RegisterSet &insert(unsigned R) {
+      unsigned Idx = v2x(R);
+      ensure(Idx);
+      return static_cast<RegisterSet&>(BitVector::set(Idx));
+    }
+    RegisterSet &remove(unsigned R) {
+      unsigned Idx = v2x(R);
+      if (Idx >= size())
+        return *this;
+      return static_cast<RegisterSet&>(BitVector::reset(Idx));
+    }
+
+    RegisterSet &insert(const RegisterSet &Rs) {
+      return static_cast<RegisterSet&>(BitVector::operator|=(Rs));
+    }
+    RegisterSet &remove(const RegisterSet &Rs) {
+      return static_cast<RegisterSet&>(BitVector::reset(Rs));
+    }
+
+    reference operator[](unsigned R) {
+      unsigned Idx = v2x(R);
+      ensure(Idx);
+      return BitVector::operator[](Idx);
+    }
+    bool operator[](unsigned R) const {
+      unsigned Idx = v2x(R);
+      assert(Idx < size());
+      return BitVector::operator[](Idx);
+    }
+    bool has(unsigned R) const {
+      unsigned Idx = v2x(R);
+      if (Idx >= size())
+        return false;
+      return BitVector::test(Idx);
+    }
+
+    bool empty() const {
+      return !BitVector::any();
+    }
+    bool includes(const RegisterSet &Rs) const {
+      // A.BitVector::test(B)  <=>  A-B != {}
+      return !Rs.BitVector::test(*this);
+    }
+    bool intersects(const RegisterSet &Rs) const {
+      return BitVector::anyCommon(Rs);
+    }
+
+  private:
+    void ensure(unsigned Idx) {
+      if (size() <= Idx)
+        resize(std::max(Idx+1, 32U));
+    }
+    static inline unsigned v2x(unsigned v) {
+      return TargetRegisterInfo::virtReg2Index(v);
+    }
+    static inline unsigned x2v(unsigned x) {
+      return TargetRegisterInfo::index2VirtReg(x);
+    }
+  };
+
+
+  struct PrintRegSet {
+    PrintRegSet(const RegisterSet &S, const TargetRegisterInfo *RI)
+      : RS(S), TRI(RI) {}
+    friend raw_ostream &operator<< (raw_ostream &OS,
+          const PrintRegSet &P);
+  private:
+    const RegisterSet &RS;
+    const TargetRegisterInfo *TRI;
+  };
+
+  raw_ostream &operator<< (raw_ostream &OS, const PrintRegSet &P) {
+    OS << '{';
+    for (unsigned R = P.RS.find_first(); R; R = P.RS.find_next(R))
+      OS << ' ' << PrintReg(R, P.TRI);
+    OS << " }";
+    return OS;
+  }
+}
+
+
+namespace {
+  // A convenience class to associate unsigned numbers (such as virtual
+  // registers) with unsigned numbers.
+  struct UnsignedMap : public DenseMap<unsigned,unsigned> {
+    UnsignedMap() : BaseType() {}
+  private:
+    typedef DenseMap<unsigned,unsigned> BaseType;
+  };
+
+  // A utility to establish an ordering between virtual registers:
+  // VRegA < VRegB  <=>  RegisterOrdering[VRegA] < RegisterOrdering[VRegB]
+  // This is meant as a cache for the ordering of virtual registers defined
+  // by a potentially expensive comparison function, or obtained by a proce-
+  // dure that should not be repeated each time two registers are compared.
+  struct RegisterOrdering : public UnsignedMap {
+    RegisterOrdering() : UnsignedMap() {}
+    unsigned operator[](unsigned VR) const {
+      const_iterator F = find(VR);
+      assert(F != end());
+      return F->second;
+    }
+    // Add operator(), so that objects of this class can be used as
+    // comparators in std::sort et al.
+    bool operator() (unsigned VR1, unsigned VR2) const {
+      return operator[](VR1) < operator[](VR2);
+    }
+  };
+}
+
+
+namespace {
+  // Ordering of bit values. This class does not have operator[], but
+  // is supplies a comparison operator() for use in std:: algorithms.
+  // The order is as follows:
+  // - 0 < 1 < ref
+  // - ref1 < ref2, if ord(ref1.Reg) < ord(ref2.Reg),
+  //   or ord(ref1.Reg) == ord(ref2.Reg), and ref1.Pos < ref2.Pos.
+  struct BitValueOrdering {
+    BitValueOrdering(const RegisterOrdering &RB) : BaseOrd(RB) {}
+    bool operator() (const BitTracker::BitValue &V1,
+          const BitTracker::BitValue &V2) const;
+    const RegisterOrdering &BaseOrd;
+  };
+}
+
+
+bool BitValueOrdering::operator() (const BitTracker::BitValue &V1,
+      const BitTracker::BitValue &V2) const {
+  if (V1 == V2)
+    return false;
+  // V1==0 => true, V2==0 => false
+  if (V1.is(0) || V2.is(0))
+    return V1.is(0);
+  // Neither of V1,V2 is 0, and V1!=V2.
+  // V2==1 => false, V1==1 => true
+  if (V2.is(1) || V1.is(1))
+    return !V2.is(1);
+  // Both V1,V2 are refs.
+  unsigned Ind1 = BaseOrd[V1.RefI.Reg], Ind2 = BaseOrd[V2.RefI.Reg];
+  if (Ind1 != Ind2)
+    return Ind1 < Ind2;
+  // If V1.Pos==V2.Pos
+  assert(V1.RefI.Pos != V2.RefI.Pos && "Bit values should be different");
+  return V1.RefI.Pos < V2.RefI.Pos;
+}
+
+
+namespace {
+  // Cache for the BitTracker's cell map. Map lookup has a logarithmic
+  // complexity, this class will memoize the lookup results to reduce
+  // the access time for repeated lookups of the same cell.
+  struct CellMapShadow {
+    CellMapShadow(const BitTracker &T) : BT(T) {}
+    const BitTracker::RegisterCell &lookup(unsigned VR) {
+      unsigned RInd = TargetRegisterInfo::virtReg2Index(VR);
+      // Grow the vector to at least 32 elements.
+      if (RInd >= CVect.size())
+        CVect.resize(std::max(RInd+16, 32U), 0);
+      const BitTracker::RegisterCell *CP = CVect[RInd];
+      if (CP == 0)
+        CP = CVect[RInd] = &BT.lookup(VR);
+      return *CP;
+    }
+
+    const BitTracker &BT;
+
+  private:
+    typedef std::vector<const BitTracker::RegisterCell*> CellVectType;
+    CellVectType CVect;
+  };
+}
+
+
+namespace {
+  // Comparator class for lexicographic ordering of virtual registers
+  // according to the corresponding BitTracker::RegisterCell objects.
+  struct RegisterCellLexCompare {
+    RegisterCellLexCompare(const BitValueOrdering &BO, CellMapShadow &M)
+      : BitOrd(BO), CM(M) {}
+    bool operator() (unsigned VR1, unsigned VR2) const;
+  private:
+    const BitValueOrdering &BitOrd;
+    CellMapShadow &CM;
+  };
+
+  // Comparator class for lexicographic ordering of virtual registers
+  // according to the specified bits of the corresponding BitTracker::
+  // RegisterCell objects.
+  // Specifically, this class will be used to compare bit B of a register
+  // cell for a selected virtual register R with bit N of any register
+  // other than R.
+  struct RegisterCellBitCompareSel {
+    RegisterCellBitCompareSel(unsigned R, unsigned B, unsigned N,
+          const BitValueOrdering &BO, CellMapShadow &M)
+      : SelR(R), SelB(B), BitN(N), BitOrd(BO), CM(M) {}
+    bool operator() (unsigned VR1, unsigned VR2) const;
+  private:
+    const unsigned SelR, SelB;
+    const unsigned BitN;
+    const BitValueOrdering &BitOrd;
+    CellMapShadow &CM;
+  };
+}
+
+
+bool RegisterCellLexCompare::operator() (unsigned VR1, unsigned VR2) const {
+  // Ordering of registers, made up from two given orderings:
+  // - the ordering of the register numbers, and
+  // - the ordering of register cells.
+  // Def. R1 < R2 if:
+  // - cell(R1) < cell(R2), or
+  // - cell(R1) == cell(R2), and index(R1) < index(R2).
+  //
+  // For register cells, the ordering is lexicographic, with index 0 being
+  // the most significant.
+  if (VR1 == VR2)
+    return false;
+
+  const BitTracker::RegisterCell &RC1 = CM.lookup(VR1), &RC2 = CM.lookup(VR2);
+  uint16_t W1 = RC1.width(), W2 = RC2.width();
+  for (uint16_t i = 0, w = std::min(W1, W2); i < w; ++i) {
+    const BitTracker::BitValue &V1 = RC1[i], &V2 = RC2[i];
+    if (V1 != V2)
+      return BitOrd(V1, V2);
+  }
+  // Cells are equal up until the common length.
+  if (W1 != W2)
+    return W1 < W2;
+
+  return BitOrd.BaseOrd[VR1] < BitOrd.BaseOrd[VR2];
+}
+
+
+bool RegisterCellBitCompareSel::operator() (unsigned VR1, unsigned VR2) const {
+  if (VR1 == VR2)
+    return false;
+  const BitTracker::RegisterCell &RC1 = CM.lookup(VR1);
+  const BitTracker::RegisterCell &RC2 = CM.lookup(VR2);
+  uint16_t W1 = RC1.width(), W2 = RC2.width();
+  uint16_t Bit1 = (VR1 == SelR) ? SelB : BitN;
+  uint16_t Bit2 = (VR2 == SelR) ? SelB : BitN;
+  // If Bit1 exceeds the width of VR1, then:
+  // - return false, if at the same time Bit2 exceeds VR2, or
+  // - return true, otherwise.
+  // (I.e. "a bit value that does not exist is less than any bit value
+  // that does exist".)
+  if (W1 <= Bit1)
+    return Bit2 < W2;
+  // If Bit1 is within VR1, but Bit2 is not within VR2, return false.
+  if (W2 <= Bit2)
+    return false;
+
+  const BitTracker::BitValue &V1 = RC1[Bit1], V2 = RC2[Bit2];
+  if (V1 != V2)
+    return BitOrd(V1, V2);
+  return false;
+}
+
+
+namespace {
+  class OrderedRegisterList {
+    typedef std::vector<unsigned> ListType;
+  public:
+    OrderedRegisterList(const RegisterOrdering &RO) : Ord(RO) {}
+    void insert(unsigned VR);
+    void remove(unsigned VR);
+    unsigned operator[](unsigned Idx) const {
+      assert(Idx < Seq.size());
+      return Seq[Idx];
+    }
+    unsigned size() const {
+      return Seq.size();
+    }
+
+    typedef ListType::iterator iterator;
+    typedef ListType::const_iterator const_iterator;
+    iterator begin() { return Seq.begin(); }
+    iterator end() { return Seq.end(); }
+    const_iterator begin() const { return Seq.begin(); }
+    const_iterator end() const { return Seq.end(); }
+
+    // Convenience function to convert an iterator to the corresponding index.
+    unsigned idx(iterator It) const { return It-begin(); }
+  private:
+    ListType Seq;
+    const RegisterOrdering &Ord;
+  };
+
+
+  struct PrintORL {
+    PrintORL(const OrderedRegisterList &L, const TargetRegisterInfo *RI)
+      : RL(L), TRI(RI) {}
+    friend raw_ostream &operator<< (raw_ostream &OS, const PrintORL &P);
+  private:
+    const OrderedRegisterList &RL;
+    const TargetRegisterInfo *TRI;
+  };
+
+  raw_ostream &operator<< (raw_ostream &OS, const PrintORL &P) {
+    OS << '(';
+    OrderedRegisterList::const_iterator B = P.RL.begin(), E = P.RL.end();
+    for (OrderedRegisterList::const_iterator I = B; I != E; ++I) {
+      if (I != B)
+        OS << ", ";
+      OS << PrintReg(*I, P.TRI);
+    }
+    OS << ')';
+    return OS;
+  }
+}
+
+
+void OrderedRegisterList::insert(unsigned VR) {
+  iterator L = std::lower_bound(Seq.begin(), Seq.end(), VR, Ord);
+  if (L == Seq.end())
+    Seq.push_back(VR);
+  else
+    Seq.insert(L, VR);
+}
+
+
+void OrderedRegisterList::remove(unsigned VR) {
+  iterator L = std::lower_bound(Seq.begin(), Seq.end(), VR, Ord);
+  assert(L != Seq.end());
+  Seq.erase(L);
+}
+
+
+namespace {
+  // A record of the insert form. The fields correspond to the operands
+  // of the "insert" instruction:
+  // ... = insert(SrcR, InsR, #Wdh, #Off)
+  struct IFRecord {
+    IFRecord(unsigned SR = 0, unsigned IR = 0, uint16_t W = 0, uint16_t O = 0)
+      : SrcR(SR), InsR(IR), Wdh(W), Off(O) {}
+    unsigned SrcR, InsR;
+    uint16_t Wdh, Off;
+  };
+
+  struct PrintIFR {
+    PrintIFR(const IFRecord &R, const TargetRegisterInfo *RI)
+      : IFR(R), TRI(RI) {}
+  private:
+    const IFRecord &IFR;
+    const TargetRegisterInfo *TRI;
+    friend raw_ostream &operator<< (raw_ostream &OS, const PrintIFR &P);
+  };
+
+  raw_ostream &operator<< (raw_ostream &OS, const PrintIFR &P) {
+    unsigned SrcR = P.IFR.SrcR, InsR = P.IFR.InsR;
+    OS << '(' << PrintReg(SrcR, P.TRI) << ',' << PrintReg(InsR, P.TRI)
+       << ",#" << P.IFR.Wdh << ",#" << P.IFR.Off << ')';
+    return OS;
+  }
+
+  typedef std::pair<IFRecord,RegisterSet> IFRecordWithRegSet;
+}
+
+
+namespace llvm {
+  void initializeHexagonGenInsertPass(PassRegistry&);
+  FunctionPass *createHexagonGenInsert();
+}
+
+
+namespace {
+  class HexagonGenInsert : public MachineFunctionPass {
+  public:
+    static char ID;
+    HexagonGenInsert() : MachineFunctionPass(ID), HII(0), HRI(0) {
+      initializeHexagonGenInsertPass(*PassRegistry::getPassRegistry());
+    }
+    virtual const char *getPassName() const {
+      return "Hexagon generate \"insert\" instructions";
+    }
+    virtual void getAnalysisUsage(AnalysisUsage &AU) const {
+      AU.addRequired<MachineDominatorTree>();
+      AU.addPreserved<MachineDominatorTree>();
+      MachineFunctionPass::getAnalysisUsage(AU);
+    }
+    virtual bool runOnMachineFunction(MachineFunction &MF);
+
+  private:
+    typedef DenseMap<std::pair<unsigned,unsigned>,unsigned> PairMapType;
+
+    void buildOrderingMF(RegisterOrdering &RO) const;
+    void buildOrderingBT(RegisterOrdering &RB, RegisterOrdering &RO) const;
+    bool isIntClass(const TargetRegisterClass *RC) const;
+    bool isConstant(unsigned VR) const;
+    bool isSmallConstant(unsigned VR) const;
+    bool isValidInsertForm(unsigned DstR, unsigned SrcR, unsigned InsR,
+          uint16_t L, uint16_t S) const;
+    bool findSelfReference(unsigned VR) const;
+    bool findNonSelfReference(unsigned VR) const;
+    void getInstrDefs(const MachineInstr *MI, RegisterSet &Defs) const;
+    void getInstrUses(const MachineInstr *MI, RegisterSet &Uses) const;
+    unsigned distance(const MachineBasicBlock *FromB,
+          const MachineBasicBlock *ToB, const UnsignedMap &RPO,
+          PairMapType &M) const;
+    unsigned distance(MachineBasicBlock::const_iterator FromI,
+          MachineBasicBlock::const_iterator ToI, const UnsignedMap &RPO,
+          PairMapType &M) const;
+    bool findRecordInsertForms(unsigned VR, OrderedRegisterList &AVs);
+    void collectInBlock(MachineBasicBlock *B, OrderedRegisterList &AVs);
+    void findRemovableRegisters(unsigned VR, IFRecord IF,
+          RegisterSet &RMs) const;
+    void computeRemovableRegisters();
+
+    void pruneEmptyLists();
+    void pruneCoveredSets(unsigned VR);
+    void pruneUsesTooFar(unsigned VR, const UnsignedMap &RPO, PairMapType &M);
+    void pruneRegCopies(unsigned VR);
+    void pruneCandidates();
+    void selectCandidates();
+    bool generateInserts();
+
+    bool removeDeadCode(MachineDomTreeNode *N);
+
+    // IFRecord coupled with a set of potentially removable registers:
+    typedef std::vector<IFRecordWithRegSet> IFListType;
+    typedef DenseMap<unsigned,IFListType> IFMapType;  // vreg -> IFListType
+
+    void dump_map() const;
+
+    const HexagonInstrInfo *HII;
+    const HexagonRegisterInfo *HRI;
+
+    MachineFunction *MFN;
+    MachineRegisterInfo *MRI;
+    MachineDominatorTree *MDT;
+    CellMapShadow *CMS;
+
+    RegisterOrdering BaseOrd;
+    RegisterOrdering CellOrd;
+    IFMapType IFMap;
+  };
+
+  char HexagonGenInsert::ID = 0;
+}
+
+
+void HexagonGenInsert::dump_map() const {
+  typedef IFMapType::const_iterator iterator;
+  for (iterator I = IFMap.begin(), E = IFMap.end(); I != E; ++I) {
+    dbgs() << "  " << PrintReg(I->first, HRI) << ":\n";
+    const IFListType &LL = I->second;
+    for (unsigned i = 0, n = LL.size(); i < n; ++i)
+      dbgs() << "    " << PrintIFR(LL[i].first, HRI) << ", "
+             << PrintRegSet(LL[i].second, HRI) << '\n';
+  }
+}
+
+
+void HexagonGenInsert::buildOrderingMF(RegisterOrdering &RO) const {
+  unsigned Index = 0;
+  typedef MachineFunction::const_iterator mf_iterator;
+  for (mf_iterator A = MFN->begin(), Z = MFN->end(); A != Z; ++A) {
+    const MachineBasicBlock &B = *A;
+    if (!CMS->BT.reached(&B))
+      continue;
+    typedef MachineBasicBlock::const_iterator mb_iterator;
+    for (mb_iterator I = B.begin(), E = B.end(); I != E; ++I) {
+      const MachineInstr *MI = &*I;
+      for (unsigned i = 0, n = MI->getNumOperands(); i < n; ++i) {
+        const MachineOperand &MO = MI->getOperand(i);
+        if (MO.isReg() && MO.isDef()) {
+          unsigned R = MO.getReg();
+          assert(MO.getSubReg() == 0 && "Unexpected subregister in definition");
+          if (TargetRegisterInfo::isVirtualRegister(R))
+            RO.insert(std::make_pair(R, Index++));
+        }
+      }
+    }
+  }
+  // Since some virtual registers may have had their def and uses eliminated,
+  // they are no longer referenced in the code, and so they will not appear
+  // in the map.
+}
+
+
+void HexagonGenInsert::buildOrderingBT(RegisterOrdering &RB,
+      RegisterOrdering &RO) const {
+  // Create a vector of all virtual registers (collect them from the base
+  // ordering RB), and then sort it using the RegisterCell comparator.
+  BitValueOrdering BVO(RB);
+  RegisterCellLexCompare LexCmp(BVO, *CMS);
+  typedef std::vector<unsigned> SortableVectorType;
+  SortableVectorType VRs;
+  for (RegisterOrdering::iterator I = RB.begin(), E = RB.end(); I != E; ++I)
+    VRs.push_back(I->first);
+  std::sort(VRs.begin(), VRs.end(), LexCmp);
+  // Transfer the results to the outgoing register ordering.
+  for (unsigned i = 0, n = VRs.size(); i < n; ++i)
+    RO.insert(std::make_pair(VRs[i], i));
+}
+
+
+inline bool HexagonGenInsert::isIntClass(const TargetRegisterClass *RC) const {
+  return RC == &Hexagon::IntRegsRegClass || RC == &Hexagon::DoubleRegsRegClass;
+}
+
+
+bool HexagonGenInsert::isConstant(unsigned VR) const {
+  const BitTracker::RegisterCell &RC = CMS->lookup(VR);
+  uint16_t W = RC.width();
+  for (uint16_t i = 0; i < W; ++i) {
+    const BitTracker::BitValue &BV = RC[i];
+    if (BV.is(0) || BV.is(1))
+      continue;
+    return false;
+  }
+  return true;
+}
+
+
+bool HexagonGenInsert::isSmallConstant(unsigned VR) const {
+  const BitTracker::RegisterCell &RC = CMS->lookup(VR);
+  uint16_t W = RC.width();
+  if (W > 64)
+    return false;
+  uint64_t V = 0, B = 1;
+  for (uint16_t i = 0; i < W; ++i) {
+    const BitTracker::BitValue &BV = RC[i];
+    if (BV.is(1))
+      V |= B;
+    else if (!BV.is(0))
+      return false;
+    B <<= 1;
+  }
+
+  // For 32-bit registers, consider: Rd = #s16.
+  if (W == 32)
+    return isInt<16>(V);
+
+  // For 64-bit registers, it's Rdd = #s8 or Rdd = combine(#s8,#s8)
+  return isInt<8>(Lo_32(V)) && isInt<8>(Hi_32(V));
+}
+
+
+bool HexagonGenInsert::isValidInsertForm(unsigned DstR, unsigned SrcR,
+      unsigned InsR, uint16_t L, uint16_t S) const {
+  const TargetRegisterClass *DstRC = MRI->getRegClass(DstR);
+  const TargetRegisterClass *SrcRC = MRI->getRegClass(SrcR);
+  const TargetRegisterClass *InsRC = MRI->getRegClass(InsR);
+  // Only integet (32-/64-bit) register classes.
+  if (!isIntClass(DstRC) || !isIntClass(SrcRC) || !isIntClass(InsRC))
+    return false;
+  // The "source" register must be of the same class as DstR.
+  if (DstRC != SrcRC)
+    return false;
+  if (DstRC == InsRC)
+    return true;
+  // A 64-bit register can only be generated from other 64-bit registers.
+  if (DstRC == &Hexagon::DoubleRegsRegClass)
+    return false;
+  // Otherwise, the L and S cannot span 32-bit word boundary.
+  if (S < 32 && S+L > 32)
+    return false;
+  return true;
+}
+
+
+bool HexagonGenInsert::findSelfReference(unsigned VR) const {
+  const BitTracker::RegisterCell &RC = CMS->lookup(VR);
+  for (uint16_t i = 0, w = RC.width(); i < w; ++i) {
+    const BitTracker::BitValue &V = RC[i];
+    if (V.Type == BitTracker::BitValue::Ref && V.RefI.Reg == VR)
+      return true;
+  }
+  return false;
+}
+
+
+bool HexagonGenInsert::findNonSelfReference(unsigned VR) const {
+  BitTracker::RegisterCell RC = CMS->lookup(VR);
+  for (uint16_t i = 0, w = RC.width(); i < w; ++i) {
+    const BitTracker::BitValue &V = RC[i];
+    if (V.Type == BitTracker::BitValue::Ref && V.RefI.Reg != VR)
+      return true;
+  }
+  return false;
+}
+
+
+void HexagonGenInsert::getInstrDefs(const MachineInstr *MI,
+      RegisterSet &Defs) const {
+  for (unsigned i = 0, n = MI->getNumOperands(); i < n; ++i) {
+    const MachineOperand &MO = MI->getOperand(i);
+    if (!MO.isReg() || !MO.isDef())
+      continue;
+    unsigned R = MO.getReg();
+    if (!TargetRegisterInfo::isVirtualRegister(R))
+      continue;
+    Defs.insert(R);
+  }
+}
+
+
+void HexagonGenInsert::getInstrUses(const MachineInstr *MI,
+      RegisterSet &Uses) const {
+  for (unsigned i = 0, n = MI->getNumOperands(); i < n; ++i) {
+    const MachineOperand &MO = MI->getOperand(i);
+    if (!MO.isReg() || !MO.isUse())
+      continue;
+    unsigned R = MO.getReg();
+    if (!TargetRegisterInfo::isVirtualRegister(R))
+      continue;
+    Uses.insert(R);
+  }
+}
+
+
+unsigned HexagonGenInsert::distance(const MachineBasicBlock *FromB,
+      const MachineBasicBlock *ToB, const UnsignedMap &RPO,
+      PairMapType &M) const {
+  // Forward distance from the end of a block to the beginning of it does
+  // not make sense. This function should not be called with FromB == ToB.
+  assert(FromB != ToB);
+
+  unsigned FromN = FromB->getNumber(), ToN = ToB->getNumber();
+  // If we have already computed it, return the cached result.
+  PairMapType::iterator F = M.find(std::make_pair(FromN, ToN));
+  if (F != M.end())
+    return F->second;
+  unsigned ToRPO = RPO.lookup(ToN);
+
+  unsigned MaxD = 0;
+  typedef MachineBasicBlock::const_pred_iterator pred_iterator;
+  for (pred_iterator I = ToB->pred_begin(), E = ToB->pred_end(); I != E; ++I) {
+    const MachineBasicBlock *PB = *I;
+    // Skip back edges. Also, if FromB is a predecessor of ToB, the distance
+    // along that path will be 0, and we don't need to do any calculations
+    // on it.
+    if (PB == FromB || RPO.lookup(PB->getNumber()) >= ToRPO)
+      continue;
+    unsigned D = PB->size() + distance(FromB, PB, RPO, M);
+    if (D > MaxD)
+      MaxD = D;
+  }
+
+  // Memoize the result for later lookup.
+  M.insert(std::make_pair(std::make_pair(FromN, ToN), MaxD));
+  return MaxD;
+}
+
+
+unsigned HexagonGenInsert::distance(MachineBasicBlock::const_iterator FromI,
+      MachineBasicBlock::const_iterator ToI, const UnsignedMap &RPO,
+      PairMapType &M) const {
+  const MachineBasicBlock *FB = FromI->getParent(), *TB = ToI->getParent();
+  if (FB == TB)
+    return std::distance(FromI, ToI);
+  unsigned D1 = std::distance(TB->begin(), ToI);
+  unsigned D2 = distance(FB, TB, RPO, M);
+  unsigned D3 = std::distance(FromI, FB->end());
+  return D1+D2+D3;
+}
+
+
+bool HexagonGenInsert::findRecordInsertForms(unsigned VR,
+      OrderedRegisterList &AVs) {
+  if (isDebug()) {
+    dbgs() << LLVM_FUNCTION_NAME << ": " << PrintReg(VR, HRI)
+           << "  AVs: " << PrintORL(AVs, HRI) << "\n";
+  }
+  if (AVs.size() == 0)
+    return false;
+
+  typedef OrderedRegisterList::iterator iterator;
+  BitValueOrdering BVO(BaseOrd);
+  const BitTracker::RegisterCell &RC = CMS->lookup(VR);
+  uint16_t W = RC.width();
+
+  typedef std::pair<unsigned,uint16_t> RSRecord;  // (reg,shift)
+  typedef std::vector<RSRecord> RSListType;
+  // Have a map, with key being the matching prefix length, and the value
+  // being the list of pairs (R,S), where R's prefix matches VR at S.
+  // (DenseMap<uint16_t,RSListType> fails to instantiate.)
+  typedef DenseMap<unsigned,RSListType> LRSMapType;
+  LRSMapType LM;
+
+  // Conceptually, rotate the cell RC right (i.e. towards the LSB) by S,
+  // and find matching prefixes from AVs with the rotated RC. Such a prefix
+  // would match a string of bits (of length L) in RC starting at S.
+  for (uint16_t S = 0; S < W; ++S) {
+    iterator B = AVs.begin(), E = AVs.end();
+    // The registers in AVs are ordered according to the lexical order of
+    // the corresponding register cells. This means that the range of regis-
+    // ters in AVs that match a prefix of length L+1 will be contained in
+    // the range that matches a prefix of length L. This means that we can
+    // keep narrowing the search space as the prefix length goes up. This
+    // helps reduce the overall complexity of the search.
+    uint16_t L;
+    for (L = 0; L < W-S; ++L) {
+      // Compare against VR's bits starting at S, which emulates rotation
+      // of VR by S.
+      RegisterCellBitCompareSel RCB(VR, S+L, L, BVO, *CMS);
+      iterator NewB = std::lower_bound(B, E, VR, RCB);
+      iterator NewE = std::upper_bound(NewB, E, VR, RCB);
+      // For the registers that are eliminated from the next range, L is
+      // the longest prefix matching VR at position S (their prefixes
+      // differ from VR at S+L). If L>0, record this information for later
+      // use.
+      if (L > 0) {
+        for (iterator I = B; I != NewB; ++I)
+          LM[L].push_back(std::make_pair(*I, S));
+        for (iterator I = NewE; I != E; ++I)
+          LM[L].push_back(std::make_pair(*I, S));
+      }
+      B = NewB, E = NewE;
+      if (B == E)
+        break;
+    }
+    // Record the final register range. If this range is non-empty, then
+    // L=W-S.
+    assert(B == E || L == W-S);
+    if (B != E) {
+      for (iterator I = B; I != E; ++I)
+        LM[L].push_back(std::make_pair(*I, S));
+      // If B!=E, then we found a range of registers whose prefixes cover the
+      // rest of VR from position S. There is no need to further advance S.
+      break;
+    }
+  }
+
+  if (isDebug()) {
+    dbgs() << "Prefixes matching register " << PrintReg(VR, HRI) << "\n";
+    for (LRSMapType::iterator I = LM.begin(), E = LM.end(); I != E; ++I) {
+      dbgs() << "  L=" << I->first << ':';
+      const RSListType &LL = I->second;
+      for (unsigned i = 0, n = LL.size(); i < n; ++i)
+        dbgs() << " (" << PrintReg(LL[i].first, HRI) << ",@"
+               << LL[i].second << ')';
+      dbgs() << '\n';
+    }
+  }
+
+
+  bool Recorded = false;
+
+  for (iterator I = AVs.begin(), E = AVs.end(); I != E; ++I) {
+    unsigned SrcR = *I;
+    int FDi = -1, LDi = -1;   // First/last different bit.
+    const BitTracker::RegisterCell &AC = CMS->lookup(SrcR);
+    uint16_t AW = AC.width();
+    for (uint16_t i = 0, w = std::min(W, AW); i < w; ++i) {
+      if (RC[i] == AC[i])
+        continue;
+      if (FDi == -1)
+        FDi = i;
+      LDi = i;
+    }
+    if (FDi == -1)
+      continue;  // TODO (future): Record identical registers.
+    // Look for a register whose prefix could patch the range [FD..LD]
+    // where VR and SrcR differ.
+    uint16_t FD = FDi, LD = LDi;  // Switch to unsigned type.
+    uint16_t MinL = LD-FD+1;
+    for (uint16_t L = MinL; L < W; ++L) {
+      LRSMapType::iterator F = LM.find(L);
+      if (F == LM.end())
+        continue;
+      RSListType &LL = F->second;
+      for (unsigned i = 0, n = LL.size(); i < n; ++i) {
+        uint16_t S = LL[i].second;
+        // MinL is the minimum length of the prefix. Any length above MinL
+        // allows some flexibility as to where the prefix can start:
+        // given the extra length EL=L-MinL, the prefix must start between
+        // max(0,FD-EL) and FD.
+        if (S > FD)   // Starts too late.
+          continue;
+        uint16_t EL = L-MinL;
+        uint16_t LowS = (EL < FD) ? FD-EL : 0;
+        if (S < LowS) // Starts too early.
+          continue;
+        unsigned InsR = LL[i].first;
+        if (!isValidInsertForm(VR, SrcR, InsR, L, S))
+          continue;
+        if (isDebug()) {
+          dbgs() << PrintReg(VR, HRI) << " = insert(" << PrintReg(SrcR, HRI)
+                 << ',' << PrintReg(InsR, HRI) << ",#" << L << ",#"
+                 << S << ")\n";
+        }
+        IFRecordWithRegSet RR(IFRecord(SrcR, InsR, L, S), RegisterSet());
+        IFMap[VR].push_back(RR);
+        Recorded = true;
+      }
+    }
+  }
+
+  return Recorded;
+}
+
+
+void HexagonGenInsert::collectInBlock(MachineBasicBlock *B,
+      OrderedRegisterList &AVs) {
+  if (isDebug())
+    dbgs() << "visiting block BB#" << B->getNumber() << "\n";
+
+  // First, check if this block is reachable at all. If not, the bit tracker
+  // will not have any information about registers in it.
+  if (!CMS->BT.reached(B))
+    return;
+
+  bool DoConst = OptConst;
+  // Keep a separate set of registers defined in this block, so that we
+  // can remove them from the list of available registers once all DT
+  // successors have been processed.
+  RegisterSet BlockDefs, InsDefs;
+  for (MachineBasicBlock::iterator I = B->begin(), E = B->end(); I != E; ++I) {
+    MachineInstr *MI = &*I;
+    InsDefs.clear();
+    getInstrDefs(MI, InsDefs);
+    // Leave those alone. They are more transparent than "insert".
+    bool Skip = MI->isCopy() || MI->isRegSequence();
+
+    if (!Skip) {
+      // Visit all defined registers, and attempt to find the corresponding
+      // "insert" representations.
+      for (unsigned VR = InsDefs.find_first(); VR; VR = InsDefs.find_next(VR)) {
+        // Do not collect registers that are known to be compile-time cons-
+        // tants, unless requested.
+        if (!DoConst && isConstant(VR))
+          continue;
+        // If VR's cell contains a reference to VR, then VR cannot be defined
+        // via "insert". If VR is a constant that can be generated in a single
+        // instruction (without constant extenders), generating it via insert
+        // makes no sense.
+        if (findSelfReference(VR) || isSmallConstant(VR))
+          continue;
+
+        findRecordInsertForms(VR, AVs);
+      }
+    }
+
+    // Insert the defined registers into the list of available registers
+    // after they have been processed.
+    for (unsigned VR = InsDefs.find_first(); VR; VR = InsDefs.find_next(VR))
+      AVs.insert(VR);
+    BlockDefs.insert(InsDefs);
+  }
+
+  MachineDomTreeNode *N = MDT->getNode(B);
+  typedef GraphTraits<MachineDomTreeNode*> GTN;
+  typedef GTN::ChildIteratorType ChildIter;
+  for (ChildIter I = GTN::child_begin(N), E = GTN::child_end(N); I != E; ++I) {
+    MachineBasicBlock *SB = (*I)->getBlock();
+    collectInBlock(SB, AVs);
+  }
+
+  for (unsigned VR = BlockDefs.find_first(); VR; VR = BlockDefs.find_next(VR))
+    AVs.remove(VR);
+}
+
+
+void HexagonGenInsert::findRemovableRegisters(unsigned VR, IFRecord IF,
+      RegisterSet &RMs) const {
+  // For a given register VR and a insert form, find the registers that are
+  // used by the current definition of VR, and which would no longer be
+  // needed for it after the definition of VR is replaced with the insert
+  // form. These are the registers that could potentially become dead.
+  RegisterSet Regs[2];
+
+  unsigned S = 0;  // Register set selector.
+  Regs[S].insert(VR);
+
+  while (!Regs[S].empty()) {
+    // Breadth-first search.
+    unsigned OtherS = 1-S;
+    Regs[OtherS].clear();
+    for (unsigned R = Regs[S].find_first(); R; R = Regs[S].find_next(R)) {
+      Regs[S].remove(R);
+      if (R == IF.SrcR || R == IF.InsR)
+        continue;
+      // Check if a given register has bits that are references to any other
+      // registers. This is to detect situations where the instruction that
+      // defines register R takes register Q as an operand, but R itself does
+      // not contain any bits from Q. Loads are examples of how this could
+      // happen:
+      //   R = load Q
+      // In this case (assuming we do not have any knowledge about the loaded
+      // value), we must not treat R as a "conveyance" of the bits from Q.
+      // (The information in BT about R's bits would have them as constants,
+      // in case of zero-extending loads, or refs to R.)
+      if (!findNonSelfReference(R))
+        continue;
+      RMs.insert(R);
+      const MachineInstr *DefI = MRI->getVRegDef(R);
+      assert(DefI);
+      // Do not iterate past PHI nodes to avoid infinite loops. This can
+      // make the final set a bit less accurate, but the removable register
+      // sets are an approximation anyway.
+      if (DefI->isPHI())
+        continue;
+      getInstrUses(DefI, Regs[OtherS]);
+    }
+    S = OtherS;
+  }
+  // The register VR is added to the list as a side-effect of the algorithm,
+  // but it is not "potentially removable". A potentially removable register
+  // is one that may become unused (dead) after conversion to the insert form
+  // IF, and obviously VR (or its replacement) will not become dead by apply-
+  // ing IF.
+  RMs.remove(VR);
+}
+
+
+void HexagonGenInsert::computeRemovableRegisters() {
+  for (IFMapType::iterator I = IFMap.begin(), E = IFMap.end(); I != E; ++I) {
+    IFListType &LL = I->second;
+    for (unsigned i = 0, n = LL.size(); i < n; ++i)
+      findRemovableRegisters(I->first, LL[i].first, LL[i].second);
+  }
+}
+
+
+void HexagonGenInsert::pruneEmptyLists() {
+  // Remove all entries from the map, where the register has no insert forms
+  // associated with it.
+  typedef SmallVector<IFMapType::iterator,16> IterListType;
+  IterListType Prune;
+  for (IFMapType::iterator I = IFMap.begin(), E = IFMap.end(); I != E; ++I) {
+    if (I->second.size() == 0)
+      Prune.push_back(I);
+  }
+  for (unsigned i = 0, n = Prune.size(); i < n; ++i)
+    IFMap.erase(Prune[i]);
+}
+
+
+void HexagonGenInsert::pruneCoveredSets(unsigned VR) {
+  IFMapType::iterator F = IFMap.find(VR);
+  assert(F != IFMap.end());
+  IFListType &LL = F->second;
+
+  // First, examine the IF candidates for register VR whose removable-regis-
+  // ter sets are empty. This means that a given candidate will not help eli-
+  // minate any registers, but since "insert" is not a constant-extendable
+  // instruction, using such a candidate may reduce code size if the defini-
+  // tion of VR is constant-extended.
+  // If there exists a candidate with a non-empty set, the ones with empty
+  // sets will not be used and can be removed.
+  MachineInstr *DefVR = MRI->getVRegDef(VR);
+  bool DefEx = HII->isConstExtended(DefVR);
+  bool HasNE = false;
+  for (unsigned i = 0, n = LL.size(); i < n; ++i) {
+    if (LL[i].second.empty())
+      continue;
+    HasNE = true;
+    break;
+  }
+  if (!DefEx || HasNE) {
+    // The definition of VR is not constant-extended, or there is a candidate
+    // with a non-empty set. Remove all candidates with empty sets.
+    auto IsEmpty = [] (const IFRecordWithRegSet &IR) -> bool {
+      return IR.second.empty();
+    };
+    auto End = std::remove_if(LL.begin(), LL.end(), IsEmpty);
+    if (End != LL.end())
+      LL.erase(End, LL.end());
+  } else {
+    // The definition of VR is constant-extended, and all candidates have
+    // empty removable-register sets. Pick the maximum candidate, and remove
+    // all others. The "maximum" does not have any special meaning here, it
+    // is only so that the candidate that will remain on the list is selec-
+    // ted deterministically.
+    IFRecord MaxIF = LL[0].first;
+    for (unsigned i = 1, n = LL.size(); i < n; ++i) {
+      // If LL[MaxI] < LL[i], then MaxI = i.
+      const IFRecord &IF = LL[i].first;
+      unsigned M0 = BaseOrd[MaxIF.SrcR], M1 = BaseOrd[MaxIF.InsR];
+      unsigned R0 = BaseOrd[IF.SrcR], R1 = BaseOrd[IF.InsR];
+      if (M0 > R0)
+        continue;
+      if (M0 == R0) {
+        if (M1 > R1)
+          continue;
+        if (M1 == R1) {
+          if (MaxIF.Wdh > IF.Wdh)
+            continue;
+          if (MaxIF.Wdh == IF.Wdh && MaxIF.Off >= IF.Off)
+            continue;
+        }
+      }
+      // MaxIF < IF.
+      MaxIF = IF;
+    }
+    // Remove everything except the maximum candidate. All register sets
+    // are empty, so no need to preserve anything.
+    LL.clear();
+    LL.push_back(std::make_pair(MaxIF, RegisterSet()));
+  }
+
+  // Now, remove those whose sets of potentially removable registers are
+  // contained in another IF candidate for VR. For example, given these
+  // candidates for vreg45,
+  //   %vreg45:
+  //     (%vreg44,%vreg41,#9,#8), { %vreg42 }
+  //     (%vreg43,%vreg41,#9,#8), { %vreg42 %vreg44 }
+  // remove the first one, since it is contained in the second one.
+  for (unsigned i = 0, n = LL.size(); i < n; ) {
+    const RegisterSet &RMi = LL[i].second;
+    unsigned j = 0;
+    while (j < n) {
+      if (j != i && LL[j].second.includes(RMi))
+        break;
+      j++;
+    }
+    if (j == n) {   // RMi not contained in anything else.
+      i++;
+      continue;
+    }
+    LL.erase(LL.begin()+i);
+    n = LL.size();
+  }
+}
+
+
+void HexagonGenInsert::pruneUsesTooFar(unsigned VR, const UnsignedMap &RPO,
+      PairMapType &M) {
+  IFMapType::iterator F = IFMap.find(VR);
+  assert(F != IFMap.end());
+  IFListType &LL = F->second;
+  unsigned Cutoff = VRegDistCutoff;
+  const MachineInstr *DefV = MRI->getVRegDef(VR);
+
+  for (unsigned i = LL.size(); i > 0; --i) {
+    unsigned SR = LL[i-1].first.SrcR, IR = LL[i-1].first.InsR;
+    const MachineInstr *DefS = MRI->getVRegDef(SR);
+    const MachineInstr *DefI = MRI->getVRegDef(IR);
+    unsigned DSV = distance(DefS, DefV, RPO, M);
+    if (DSV < Cutoff) {
+      unsigned DIV = distance(DefI, DefV, RPO, M);
+      if (DIV < Cutoff)
+        continue;
+    }
+    LL.erase(LL.begin()+(i-1));
+  }
+}
+
+
+void HexagonGenInsert::pruneRegCopies(unsigned VR) {
+  IFMapType::iterator F = IFMap.find(VR);
+  assert(F != IFMap.end());
+  IFListType &LL = F->second;
+
+  auto IsCopy = [] (const IFRecordWithRegSet &IR) -> bool {
+    return IR.first.Wdh == 32 && (IR.first.Off == 0 || IR.first.Off == 32);
+  };
+  auto End = std::remove_if(LL.begin(), LL.end(), IsCopy);
+  if (End != LL.end())
+    LL.erase(End, LL.end());
+}
+
+
+void HexagonGenInsert::pruneCandidates() {
+  // Remove candidates that are not beneficial, regardless of the final
+  // selection method.
+  // First, remove candidates whose potentially removable set is a subset
+  // of another candidate's set.
+  for (IFMapType::iterator I = IFMap.begin(), E = IFMap.end(); I != E; ++I)
+    pruneCoveredSets(I->first);
+
+  UnsignedMap RPO;
+  typedef ReversePostOrderTraversal<const MachineFunction*> RPOTType;
+  RPOTType RPOT(MFN);
+  unsigned RPON = 0;
+  for (RPOTType::rpo_iterator I = RPOT.begin(), E = RPOT.end(); I != E; ++I)
+    RPO[(*I)->getNumber()] = RPON++;
+
+  PairMapType Memo; // Memoization map for distance calculation.
+  // Remove candidates that would use registers defined too far away.
+  for (IFMapType::iterator I = IFMap.begin(), E = IFMap.end(); I != E; ++I)
+    pruneUsesTooFar(I->first, RPO, Memo);
+
+  pruneEmptyLists();
+
+  for (IFMapType::iterator I = IFMap.begin(), E = IFMap.end(); I != E; ++I)
+    pruneRegCopies(I->first);
+}
+
+
+namespace {
+  // Class for comparing IF candidates for registers that have multiple of
+  // them. The smaller the candidate, according to this ordering, the better.
+  // First, compare the number of zeros in the associated potentially remova-
+  // ble register sets. "Zero" indicates that the register is very likely to
+  // become dead after this transformation.
+  // Second, compare "averages", i.e. use-count per size. The lower wins.
+  // After that, it does not really matter which one is smaller. Resolve
+  // the tie in some deterministic way.
+  struct IFOrdering {
+    IFOrdering(const UnsignedMap &UC, const RegisterOrdering &BO)
+      : UseC(UC), BaseOrd(BO) {}
+    bool operator() (const IFRecordWithRegSet &A,
+          const IFRecordWithRegSet &B) const;
+  private:
+    void stats(const RegisterSet &Rs, unsigned &Size, unsigned &Zero,
+          unsigned &Sum) const;
+    const UnsignedMap &UseC;
+    const RegisterOrdering &BaseOrd;
+  };
+}
+
+
+bool IFOrdering::operator() (const IFRecordWithRegSet &A,
+      const IFRecordWithRegSet &B) const {
+  unsigned SizeA = 0, ZeroA = 0, SumA = 0;
+  unsigned SizeB = 0, ZeroB = 0, SumB = 0;
+  stats(A.second, SizeA, ZeroA, SumA);
+  stats(B.second, SizeB, ZeroB, SumB);
+
+  // We will pick the minimum element. The more zeros, the better.
+  if (ZeroA != ZeroB)
+    return ZeroA > ZeroB;
+  // Compare SumA/SizeA with SumB/SizeB, lower is better.
+  uint64_t AvgA = SumA*SizeB, AvgB = SumB*SizeA;
+  if (AvgA != AvgB)
+    return AvgA < AvgB;
+
+  // The sets compare identical so far. Resort to comparing the IF records.
+  // The actual values don't matter, this is only for determinism.
+  unsigned OSA = BaseOrd[A.first.SrcR], OSB = BaseOrd[B.first.SrcR];
+  if (OSA != OSB)
+    return OSA < OSB;
+  unsigned OIA = BaseOrd[A.first.InsR], OIB = BaseOrd[B.first.InsR];
+  if (OIA != OIB)
+    return OIA < OIB;
+  if (A.first.Wdh != B.first.Wdh)
+    return A.first.Wdh < B.first.Wdh;
+  return A.first.Off < B.first.Off;
+}
+
+
+void IFOrdering::stats(const RegisterSet &Rs, unsigned &Size, unsigned &Zero,
+      unsigned &Sum) const {
+  for (unsigned R = Rs.find_first(); R; R = Rs.find_next(R)) {
+    UnsignedMap::const_iterator F = UseC.find(R);
+    assert(F != UseC.end());
+    unsigned UC = F->second;
+    if (UC == 0)
+      Zero++;
+    Sum += UC;
+    Size++;
+  }
+}
+
+
+void HexagonGenInsert::selectCandidates() {
+  // Some registers may have multiple valid candidates. Pick the best one
+  // (or decide not to use any).
+
+  // Compute the "removability" measure of R:
+  // For each potentially removable register R, record the number of regis-
+  // ters with IF candidates, where R appears in at least one set.
+  RegisterSet AllRMs;
+  UnsignedMap UseC, RemC;
+  IFMapType::iterator End = IFMap.end();
+
+  for (IFMapType::iterator I = IFMap.begin(); I != End; ++I) {
+    const IFListType &LL = I->second;
+    RegisterSet TT;
+    for (unsigned i = 0, n = LL.size(); i < n; ++i)
+      TT.insert(LL[i].second);
+    for (unsigned R = TT.find_first(); R; R = TT.find_next(R))
+      RemC[R]++;
+    AllRMs.insert(TT);
+  }
+
+  for (unsigned R = AllRMs.find_first(); R; R = AllRMs.find_next(R)) {
+    typedef MachineRegisterInfo::use_nodbg_iterator use_iterator;
+    typedef SmallSet<const MachineInstr*,16> InstrSet;
+    InstrSet UIs;
+    // Count as the number of instructions in which R is used, not the
+    // number of operands.
+    use_iterator E = MRI->use_nodbg_end();
+    for (use_iterator I = MRI->use_nodbg_begin(R); I != E; ++I)
+      UIs.insert(I->getParent());
+    unsigned C = UIs.size();
+    // Calculate a measure, which is the number of instructions using R,
+    // minus the "removability" count computed earlier.
+    unsigned D = RemC[R];
+    UseC[R] = (C > D) ? C-D : 0;  // doz
+  }
+
+
+  bool SelectAll0 = OptSelectAll0, SelectHas0 = OptSelectHas0;
+  if (!SelectAll0 && !SelectHas0)
+    SelectAll0 = true;
+
+  // The smaller the number UseC for a given register R, the "less used"
+  // R is aside from the opportunities for removal offered by generating
+  // "insert" instructions.
+  // Iterate over the IF map, and for those registers that have multiple
+  // candidates, pick the minimum one according to IFOrdering.
+  IFOrdering IFO(UseC, BaseOrd);
+  for (IFMapType::iterator I = IFMap.begin(); I != End; ++I) {
+    IFListType &LL = I->second;
+    if (LL.empty())
+      continue;
+    // Get the minimum element, remember it and clear the list. If the
+    // element found is adequate, we will put it back on the list, other-
+    // wise the list will remain empty, and the entry for this register
+    // will be removed (i.e. this register will not be replaced by insert).
+    IFListType::iterator MinI = std::min_element(LL.begin(), LL.end(), IFO);
+    assert(MinI != LL.end());
+    IFRecordWithRegSet M = *MinI;
+    LL.clear();
+
+    // We want to make sure that this replacement will have a chance to be
+    // beneficial, and that means that we want to have indication that some
+    // register will be removed. The most likely registers to be eliminated
+    // are the use operands in the definition of I->first. Accept/reject a
+    // candidate based on how many of its uses it can potentially eliminate.
+
+    RegisterSet Us;
+    const MachineInstr *DefI = MRI->getVRegDef(I->first);
+    getInstrUses(DefI, Us);
+    bool Accept = false;
+
+    if (SelectAll0) {
+      bool All0 = true;
+      for (unsigned R = Us.find_first(); R; R = Us.find_next(R)) {
+        if (UseC[R] == 0)
+          continue;
+        All0 = false;
+        break;
+      }
+      Accept = All0;
+    } else if (SelectHas0) {
+      bool Has0 = false;
+      for (unsigned R = Us.find_first(); R; R = Us.find_next(R)) {
+        if (UseC[R] != 0)
+          continue;
+        Has0 = true;
+        break;
+      }
+      Accept = Has0;
+    }
+    if (Accept)
+      LL.push_back(M);
+  }
+
+  // Remove candidates that add uses of removable registers, unless the
+  // removable registers are among replacement candidates.
+  // Recompute the removable registers, since some candidates may have
+  // been eliminated.
+  AllRMs.clear();
+  for (IFMapType::iterator I = IFMap.begin(); I != End; ++I) {
+    const IFListType &LL = I->second;
+    if (LL.size() > 0)
+      AllRMs.insert(LL[0].second);
+  }
+  for (IFMapType::iterator I = IFMap.begin(); I != End; ++I) {
+    IFListType &LL = I->second;
+    if (LL.size() == 0)
+      continue;
+    unsigned SR = LL[0].first.SrcR, IR = LL[0].first.InsR;
+    if (AllRMs[SR] || AllRMs[IR])
+      LL.clear();
+  }
+
+  pruneEmptyLists();
+}
+
+
+bool HexagonGenInsert::generateInserts() {
+  // Create a new register for each one from IFMap, and store them in the
+  // map.
+  UnsignedMap RegMap;
+  for (IFMapType::iterator I = IFMap.begin(), E = IFMap.end(); I != E; ++I) {
+    unsigned VR = I->first;
+    const TargetRegisterClass *RC = MRI->getRegClass(VR);
+    unsigned NewVR = MRI->createVirtualRegister(RC);
+    RegMap[VR] = NewVR;
+  }
+
+  // We can generate the "insert" instructions using potentially stale re-
+  // gisters: SrcR and InsR for a given VR may be among other registers that
+  // are also replaced. This is fine, we will do the mass "rauw" a bit later.
+  for (IFMapType::iterator I = IFMap.begin(), E = IFMap.end(); I != E; ++I) {
+    MachineInstr *MI = MRI->getVRegDef(I->first);
+    MachineBasicBlock &B = *MI->getParent();
+    DebugLoc DL = MI->getDebugLoc();
+    unsigned NewR = RegMap[I->first];
+    bool R32 = MRI->getRegClass(NewR) == &Hexagon::IntRegsRegClass;
+    const MCInstrDesc &D = R32 ? HII->get(Hexagon::S2_insert)
+                               : HII->get(Hexagon::S2_insertp);
+    IFRecord IF = I->second[0].first;
+    unsigned Wdh = IF.Wdh, Off = IF.Off;
+    unsigned InsS = 0;
+    if (R32 && MRI->getRegClass(IF.InsR) == &Hexagon::DoubleRegsRegClass) {
+      InsS = Hexagon::subreg_loreg;
+      if (Off >= 32) {
+        InsS = Hexagon::subreg_hireg;
+        Off -= 32;
+      }
+    }
+    // Advance to the proper location for inserting instructions. This could
+    // be B.end().
+    MachineBasicBlock::iterator At = MI;
+    if (MI->isPHI())
+      At = B.getFirstNonPHI();
+
+    BuildMI(B, At, DL, D, NewR)
+      .addReg(IF.SrcR)
+      .addReg(IF.InsR, 0, InsS)
+      .addImm(Wdh)
+      .addImm(Off);
+
+    MRI->clearKillFlags(IF.SrcR);
+    MRI->clearKillFlags(IF.InsR);
+  }
+
+  for (IFMapType::iterator I = IFMap.begin(), E = IFMap.end(); I != E; ++I) {
+    MachineInstr *DefI = MRI->getVRegDef(I->first);
+    MRI->replaceRegWith(I->first, RegMap[I->first]);
+    DefI->eraseFromParent();
+  }
+
+  return true;
+}
+
+
+bool HexagonGenInsert::removeDeadCode(MachineDomTreeNode *N) {
+  bool Changed = false;
+  typedef GraphTraits<MachineDomTreeNode*> GTN;
+  for (auto I = GTN::child_begin(N), E = GTN::child_end(N); I != E; ++I)
+    Changed |= removeDeadCode(*I);
+
+  MachineBasicBlock *B = N->getBlock();
+  std::vector<MachineInstr*> Instrs;
+  for (auto I = B->rbegin(), E = B->rend(); I != E; ++I)
+    Instrs.push_back(&*I);
+
+  for (auto I = Instrs.begin(), E = Instrs.end(); I != E; ++I) {
+    MachineInstr *MI = *I;
+    unsigned Opc = MI->getOpcode();
+    // Do not touch lifetime markers. This is why the target-independent DCE
+    // cannot be used.
+    if (Opc == TargetOpcode::LIFETIME_START ||
+        Opc == TargetOpcode::LIFETIME_END)
+      continue;
+    bool Store = false;
+    if (MI->isInlineAsm() || !MI->isSafeToMove(nullptr, Store))
+      continue;
+
+    bool AllDead = true;
+    SmallVector<unsigned,2> Regs;
+    for (ConstMIOperands Op(MI); Op.isValid(); ++Op) {
+      if (!Op->isReg() || !Op->isDef())
+        continue;
+      unsigned R = Op->getReg();
+      if (!TargetRegisterInfo::isVirtualRegister(R) ||
+          !MRI->use_nodbg_empty(R)) {
+        AllDead = false;
+        break;
+      }
+      Regs.push_back(R);
+    }
+    if (!AllDead)
+      continue;
+
+    B->erase(MI);
+    for (unsigned I = 0, N = Regs.size(); I != N; ++I)
+      MRI->markUsesInDebugValueAsUndef(Regs[I]);
+    Changed = true;
+  }
+
+  return Changed;
+}
+
+
+bool HexagonGenInsert::runOnMachineFunction(MachineFunction &MF) {
+  bool Timing = OptTiming, TimingDetail = Timing && OptTimingDetail;
+  bool Changed = false;
+  TimerGroup __G("hexinsert");
+  NamedRegionTimer __T("hexinsert", Timing && !TimingDetail);
+
+  // Sanity check: one, but not both.
+  assert(!OptSelectAll0 || !OptSelectHas0);
+
+  IFMap.clear();
+  BaseOrd.clear();
+  CellOrd.clear();
+
+  const auto &ST = MF.getSubtarget<HexagonSubtarget>();
+  HII = ST.getInstrInfo();
+  HRI = ST.getRegisterInfo();
+  MFN = &MF;
+  MRI = &MF.getRegInfo();
+  MDT = &getAnalysis<MachineDominatorTree>();
+
+  // Clean up before any further processing, so that dead code does not
+  // get used in a newly generated "insert" instruction. Have a custom
+  // version of DCE that preserves lifetime markers. Without it, merging
+  // of stack objects can fail to recognize and merge disjoint objects
+  // leading to unnecessary stack growth.
+  Changed |= removeDeadCode(MDT->getRootNode());
+
+  const HexagonEvaluator HE(*HRI, *MRI, *HII, MF);
+  BitTracker BTLoc(HE, MF);
+  BTLoc.trace(isDebug());
+  BTLoc.run();
+  CellMapShadow MS(BTLoc);
+  CMS = &MS;
+
+  buildOrderingMF(BaseOrd);
+  buildOrderingBT(BaseOrd, CellOrd);
+
+  if (isDebug()) {
+    dbgs() << "Cell ordering:\n";
+    for (RegisterOrdering::iterator I = CellOrd.begin(), E = CellOrd.end();
+        I != E; ++I) {
+      unsigned VR = I->first, Pos = I->second;
+      dbgs() << PrintReg(VR, HRI) << " -> " << Pos << "\n";
+    }
+  }
+
+  // Collect candidates for conversion into the insert forms.
+  MachineBasicBlock *RootB = MDT->getRoot();
+  OrderedRegisterList AvailR(CellOrd);
+
+  {
+    NamedRegionTimer _T("collection", "hexinsert", TimingDetail);
+    collectInBlock(RootB, AvailR);
+    // Complete the information gathered in IFMap.
+    computeRemovableRegisters();
+  }
+
+  if (isDebug()) {
+    dbgs() << "Candidates after collection:\n";
+    dump_map();
+  }
+
+  if (IFMap.empty())
+    return false;
+
+  {
+    NamedRegionTimer _T("pruning", "hexinsert", TimingDetail);
+    pruneCandidates();
+  }
+
+  if (isDebug()) {
+    dbgs() << "Candidates after pruning:\n";
+    dump_map();
+  }
+
+  if (IFMap.empty())
+    return false;
+
+  {
+    NamedRegionTimer _T("selection", "hexinsert", TimingDetail);
+    selectCandidates();
+  }
+
+  if (isDebug()) {
+    dbgs() << "Candidates after selection:\n";
+    dump_map();
+  }
+
+  // Filter out vregs beyond the cutoff.
+  if (VRegIndexCutoff.getPosition()) {
+    unsigned Cutoff = VRegIndexCutoff;
+    typedef SmallVector<IFMapType::iterator,16> IterListType;
+    IterListType Out;
+    for (IFMapType::iterator I = IFMap.begin(), E = IFMap.end(); I != E; ++I) {
+      unsigned Idx = TargetRegisterInfo::virtReg2Index(I->first);
+      if (Idx >= Cutoff)
+        Out.push_back(I);
+    }
+    for (unsigned i = 0, n = Out.size(); i < n; ++i)
+      IFMap.erase(Out[i]);
+  }
+
+  {
+    NamedRegionTimer _T("generation", "hexinsert", TimingDetail);
+    Changed = generateInserts();
+  }
+
+  return Changed;
+}
+
+
+FunctionPass *llvm::createHexagonGenInsert() {
+  return new HexagonGenInsert();
+}
+
+
+//===----------------------------------------------------------------------===//
+//                         Public Constructor Functions
+//===----------------------------------------------------------------------===//
+
+INITIALIZE_PASS_BEGIN(HexagonGenInsert, "hexinsert",
+  "Hexagon generate \"insert\" instructions", false, false)
+INITIALIZE_PASS_DEPENDENCY(MachineDominatorTree)
+INITIALIZE_PASS_END(HexagonGenInsert, "hexinsert",
+  "Hexagon generate \"insert\" instructions", false, false)
diff --git a/lib/Target/Hexagon/HexagonGenMux.cpp b/lib/Target/Hexagon/HexagonGenMux.cpp
new file mode 100644
index 00000000000..b4ebd9140e7
--- /dev/null
+++ b/lib/Target/Hexagon/HexagonGenMux.cpp
@@ -0,0 +1,319 @@
+//===--- HexagonGenMux.cpp ------------------------------------------------===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+
+// During instruction selection, MUX instructions are generated for
+// conditional assignments. Since such assignments often present an
+// opportunity to predicate instructions, HexagonExpandCondsets
+// expands MUXes into pairs of conditional transfers, and then proceeds
+// with predication of the producers/consumers of the registers involved.
+// This happens after exiting from the SSA form, but before the machine
+// instruction scheduler. After the scheduler and after the register
+// allocation there can be cases of pairs of conditional transfers
+// resulting from a MUX where neither of them was further predicated. If
+// these transfers are now placed far enough from the instruction defining
+// the predicate register, they cannot use the .new form. In such cases it
+// is better to collapse them back to a single MUX instruction.
+
+#define DEBUG_TYPE "hexmux"
+
+#include "llvm/CodeGen/Passes.h"
+#include "llvm/CodeGen/MachineFunctionPass.h"
+#include "llvm/CodeGen/MachineInstrBuilder.h"
+#include "llvm/CodeGen/MachineRegisterInfo.h"
+#include "HexagonTargetMachine.h"
+
+using namespace llvm;
+
+namespace llvm {
+  FunctionPass *createHexagonGenMux();
+  void initializeHexagonGenMuxPass(PassRegistry& Registry);
+}
+
+namespace {
+  class HexagonGenMux : public MachineFunctionPass {
+  public:
+    static char ID;
+    HexagonGenMux() : MachineFunctionPass(ID), HII(0), HRI(0) {
+      initializeHexagonGenMuxPass(*PassRegistry::getPassRegistry());
+    }
+    const char *getPassName() const override {
+      return "Hexagon generate mux instructions";
+    }
+    void getAnalysisUsage(AnalysisUsage &AU) const override {
+      MachineFunctionPass::getAnalysisUsage(AU);
+    }
+    bool runOnMachineFunction(MachineFunction &MF) override;
+
+  private:
+    const HexagonInstrInfo *HII;
+    const HexagonRegisterInfo *HRI;
+
+    struct CondsetInfo {
+      unsigned PredR;
+      unsigned TrueX, FalseX;
+      CondsetInfo() : PredR(0), TrueX(UINT_MAX), FalseX(UINT_MAX) {}
+    };
+    struct DefUseInfo {
+      BitVector Defs, Uses;
+      DefUseInfo() : Defs(), Uses() {}
+      DefUseInfo(const BitVector &D, const BitVector &U) : Defs(D), Uses(U) {}
+    };
+    struct MuxInfo {
+      MachineBasicBlock::iterator At;
+      unsigned DefR, PredR;
+      MachineOperand *SrcT, *SrcF;
+      MachineInstr *Def1, *Def2;
+      MuxInfo(MachineBasicBlock::iterator It, unsigned DR, unsigned PR,
+            MachineOperand *TOp, MachineOperand *FOp,
+            MachineInstr *D1, MachineInstr *D2)
+        : At(It), DefR(DR), PredR(PR), SrcT(TOp), SrcF(FOp), Def1(D1),
+          Def2(D2) {}
+    };
+    typedef DenseMap<MachineInstr*,unsigned> InstrIndexMap;
+    typedef DenseMap<unsigned,DefUseInfo> DefUseInfoMap;
+    typedef SmallVector<MuxInfo,4> MuxInfoList;
+
+    bool isRegPair(unsigned Reg) const {
+      return Hexagon::DoubleRegsRegClass.contains(Reg);
+    }
+    void getSubRegs(unsigned Reg, BitVector &SRs) const;
+    void expandReg(unsigned Reg, BitVector &Set) const;
+    void getDefsUses(const MachineInstr *MI, BitVector &Defs,
+          BitVector &Uses) const;
+    void buildMaps(MachineBasicBlock &B, InstrIndexMap &I2X,
+          DefUseInfoMap &DUM);
+    bool isCondTransfer(unsigned Opc) const;
+    unsigned getMuxOpcode(const MachineOperand &Src1,
+          const MachineOperand &Src2) const;
+    bool genMuxInBlock(MachineBasicBlock &B);
+  };
+
+  char HexagonGenMux::ID = 0;
+}
+
+INITIALIZE_PASS(HexagonGenMux, "hexagon-mux",
+  "Hexagon generate mux instructions", false, false)
+
+
+void HexagonGenMux::getSubRegs(unsigned Reg, BitVector &SRs) const {
+  for (MCSubRegIterator I(Reg, HRI); I.isValid(); ++I)
+    SRs[*I] = true;
+}
+
+
+void HexagonGenMux::expandReg(unsigned Reg, BitVector &Set) const {
+  if (isRegPair(Reg))
+    getSubRegs(Reg, Set);
+  else
+    Set[Reg] = true;
+}
+
+
+void HexagonGenMux::getDefsUses(const MachineInstr *MI, BitVector &Defs,
+      BitVector &Uses) const {
+  // First, get the implicit defs and uses for this instruction.
+  unsigned Opc = MI->getOpcode();
+  const MCInstrDesc &D = HII->get(Opc);
+  if (const uint16_t *R = D.ImplicitDefs)
+    while (*R)
+      expandReg(*R++, Defs);
+  if (const uint16_t *R = D.ImplicitUses)
+    while (*R)
+      expandReg(*R++, Uses);
+
+  // Look over all operands, and collect explicit defs and uses.
+  for (ConstMIOperands Mo(MI); Mo.isValid(); ++Mo) {
+    if (!Mo->isReg() || Mo->isImplicit())
+      continue;
+    unsigned R = Mo->getReg();
+    BitVector &Set = Mo->isDef() ? Defs : Uses;
+    expandReg(R, Set);
+  }
+}
+
+
+void HexagonGenMux::buildMaps(MachineBasicBlock &B, InstrIndexMap &I2X,
+      DefUseInfoMap &DUM) {
+  unsigned Index = 0;
+  unsigned NR = HRI->getNumRegs();
+  BitVector Defs(NR), Uses(NR);
+
+  for (MachineBasicBlock::iterator I = B.begin(), E = B.end(); I != E; ++I) {
+    MachineInstr *MI = &*I;
+    I2X.insert(std::make_pair(MI, Index));
+    Defs.reset();
+    Uses.reset();
+    getDefsUses(MI, Defs, Uses);
+    DUM.insert(std::make_pair(Index, DefUseInfo(Defs, Uses)));
+    Index++;
+  }
+}
+
+
+bool HexagonGenMux::isCondTransfer(unsigned Opc) const {
+  switch (Opc) {
+    case Hexagon::A2_tfrt:
+    case Hexagon::A2_tfrf:
+    case Hexagon::C2_cmoveit:
+    case Hexagon::C2_cmoveif:
+      return true;
+  }
+  return false;
+}
+
+
+unsigned HexagonGenMux::getMuxOpcode(const MachineOperand &Src1,
+      const MachineOperand &Src2) const {
+  bool IsReg1 = Src1.isReg(), IsReg2 = Src2.isReg();
+  if (IsReg1)
+    return IsReg2 ? Hexagon::C2_mux : Hexagon::C2_muxir;
+  if (IsReg2)
+    return Hexagon::C2_muxri;
+
+  // Neither is a register. The first source is extendable, but the second
+  // is not (s8).
+  if (Src2.isImm() && isInt<8>(Src2.getImm()))
+    return Hexagon::C2_muxii;
+
+  return 0;
+}
+
+
+bool HexagonGenMux::genMuxInBlock(MachineBasicBlock &B) {
+  bool Changed = false;
+  InstrIndexMap I2X;
+  DefUseInfoMap DUM;
+  buildMaps(B, I2X, DUM);
+
+  typedef DenseMap<unsigned,CondsetInfo> CondsetMap;
+  CondsetMap CM;
+  MuxInfoList ML;
+
+  MachineBasicBlock::iterator NextI, End = B.end();
+  for (MachineBasicBlock::iterator I = B.begin(); I != End; I = NextI) {
+    MachineInstr *MI = &*I;
+    NextI = std::next(I);
+    unsigned Opc = MI->getOpcode();
+    if (!isCondTransfer(Opc))
+      continue;
+    unsigned DR = MI->getOperand(0).getReg();
+    if (isRegPair(DR))
+      continue;
+
+    unsigned PR = MI->getOperand(1).getReg();
+    unsigned Idx = I2X.lookup(MI);
+    CondsetMap::iterator F = CM.find(DR);
+    bool IfTrue = HII->isPredicatedTrue(Opc);
+
+    // If there is no record of a conditional transfer for this register,
+    // or the predicate register differs, create a new record for it.
+    if (F != CM.end() && F->second.PredR != PR) {
+      CM.erase(F);
+      F = CM.end();
+    }
+    if (F == CM.end()) {
+      auto It = CM.insert(std::make_pair(DR, CondsetInfo()));
+      F = It.first;
+      F->second.PredR = PR;
+    }
+    CondsetInfo &CI = F->second;
+    if (IfTrue)
+      CI.TrueX = Idx;
+    else
+      CI.FalseX = Idx;
+    if (CI.TrueX == UINT_MAX || CI.FalseX == UINT_MAX)
+      continue;
+
+    // There is now a complete definition of DR, i.e. we have the predicate
+    // register, the definition if-true, and definition if-false.
+
+    // First, check if both definitions are far enough from the definition
+    // of the predicate register.
+    unsigned MinX = std::min(CI.TrueX, CI.FalseX);
+    unsigned MaxX = std::max(CI.TrueX, CI.FalseX);
+    unsigned SearchX = (MaxX > 4) ? MaxX-4 : 0;
+    bool NearDef = false;
+    for (unsigned X = SearchX; X < MaxX; ++X) {
+      const DefUseInfo &DU = DUM.lookup(X);
+      if (!DU.Defs[PR])
+        continue;
+      NearDef = true;
+      break;
+    }
+    if (NearDef)
+      continue;
+
+    // The predicate register is not defined in the last few instructions.
+    // Check if the conversion to MUX is possible (either "up", i.e. at the
+    // place of the earlier partial definition, or "down", where the later
+    // definition is located). Examine all defs and uses between these two
+    // definitions.
+    // SR1, SR2 - source registers from the first and the second definition.
+    MachineBasicBlock::iterator It1 = B.begin(), It2 = B.begin();
+    std::advance(It1, MinX);
+    std::advance(It2, MaxX);
+    MachineInstr *Def1 = It1, *Def2 = It2;
+    MachineOperand *Src1 = &Def1->getOperand(2), *Src2 = &Def2->getOperand(2);
+    unsigned SR1 = Src1->isReg() ? Src1->getReg() : 0;
+    unsigned SR2 = Src2->isReg() ? Src2->getReg() : 0;
+    bool Failure = false, CanUp = true, CanDown = true;
+    for (unsigned X = MinX+1; X < MaxX; X++) {
+      const DefUseInfo &DU = DUM.lookup(X);
+      if (DU.Defs[PR] || DU.Defs[DR] || DU.Uses[DR]) {
+        Failure = true;
+        break;
+      }
+      if (CanDown && DU.Defs[SR1])
+        CanDown = false;
+      if (CanUp && DU.Defs[SR2])
+        CanUp = false;
+    }
+    if (Failure || (!CanUp && !CanDown))
+      continue;
+
+    MachineOperand *SrcT = (MinX == CI.TrueX) ? Src1 : Src2;
+    MachineOperand *SrcF = (MinX == CI.FalseX) ? Src1 : Src2;
+    // Prefer "down", since this will move the MUX farther away from the
+    // predicate definition.
+    MachineBasicBlock::iterator At = CanDown ? Def2 : Def1;
+    ML.push_back(MuxInfo(At, DR, PR, SrcT, SrcF, Def1, Def2));
+  }
+
+  for (unsigned I = 0, N = ML.size(); I < N; ++I) {
+    MuxInfo &MX = ML[I];
+    MachineBasicBlock &B = *MX.At->getParent();
+    DebugLoc DL = MX.At->getDebugLoc();
+    unsigned MxOpc = getMuxOpcode(*MX.SrcT, *MX.SrcF);
+    if (!MxOpc)
+      continue;
+    BuildMI(B, MX.At, DL, HII->get(MxOpc), MX.DefR)
+      .addReg(MX.PredR)
+      .addOperand(*MX.SrcT)
+      .addOperand(*MX.SrcF);
+    B.erase(MX.Def1);
+    B.erase(MX.Def2);
+    Changed = true;
+  }
+
+  return Changed;
+}
+
+bool HexagonGenMux::runOnMachineFunction(MachineFunction &MF) {
+  HII = MF.getSubtarget<HexagonSubtarget>().getInstrInfo();
+  HRI = MF.getSubtarget<HexagonSubtarget>().getRegisterInfo();
+  bool Changed = false;
+  for (auto &I : MF)
+    Changed |= genMuxInBlock(I);
+  return Changed;
+}
+
+FunctionPass *llvm::createHexagonGenMux() {
+  return new HexagonGenMux();
+}
+
diff --git a/lib/Target/Hexagon/HexagonGenPredicate.cpp b/lib/Target/Hexagon/HexagonGenPredicate.cpp
new file mode 100644
index 00000000000..6905c4f6d12
--- /dev/null
+++ b/lib/Target/Hexagon/HexagonGenPredicate.cpp
@@ -0,0 +1,525 @@
+//===--- HexagonGenPredicate.cpp ------------------------------------------===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+
+#define DEBUG_TYPE "gen-pred"
+
+#include "llvm/ADT/SetVector.h"
+#include "llvm/CodeGen/Passes.h"
+#include "llvm/CodeGen/MachineDominators.h"
+#include "llvm/CodeGen/MachineFunctionPass.h"
+#include "llvm/CodeGen/MachineInstrBuilder.h"
+#include "llvm/CodeGen/MachineLoopInfo.h"
+#include "llvm/CodeGen/MachineRegisterInfo.h"
+#include "llvm/Support/CommandLine.h"
+#include "llvm/Support/Debug.h"
+#include "llvm/Support/raw_ostream.h"
+#include "llvm/Target/TargetMachine.h"
+#include "llvm/Target/TargetInstrInfo.h"
+#include "HexagonTargetMachine.h"
+
+#include <functional>
+#include <queue>
+#include <set>
+#include <vector>
+
+using namespace llvm;
+
+namespace llvm {
+  void initializeHexagonGenPredicatePass(PassRegistry& Registry);
+  FunctionPass *createHexagonGenPredicate();
+}
+
+namespace {
+  struct Register {
+    unsigned R, S;
+    Register(unsigned r = 0, unsigned s = 0) : R(r), S(s) {}
+    Register(const MachineOperand &MO) : R(MO.getReg()), S(MO.getSubReg()) {}
+    bool operator== (const Register &Reg) const {
+      return R == Reg.R && S == Reg.S;
+    }
+    bool operator< (const Register &Reg) const {
+      return R < Reg.R || (R == Reg.R && S < Reg.S);
+    }
+  };
+  struct PrintRegister {
+    PrintRegister(Register R, const TargetRegisterInfo &I) : Reg(R), TRI(I) {}
+    friend raw_ostream &operator<< (raw_ostream &OS, const PrintRegister &PR);
+  private:
+    Register Reg;
+    const TargetRegisterInfo &TRI;
+  };
+  raw_ostream &operator<< (raw_ostream &OS, const PrintRegister &PR)
+    LLVM_ATTRIBUTE_UNUSED;
+  raw_ostream &operator<< (raw_ostream &OS, const PrintRegister &PR) {
+    return OS << PrintReg(PR.Reg.R, &PR.TRI, PR.Reg.S);
+  }
+
+  class HexagonGenPredicate : public MachineFunctionPass {
+  public:
+    static char ID;
+    HexagonGenPredicate() : MachineFunctionPass(ID), TII(0), TRI(0), MRI(0) {
+      initializeHexagonGenPredicatePass(*PassRegistry::getPassRegistry());
+    }
+    virtual const char *getPassName() const {
+      return "Hexagon generate predicate operations";
+    }
+    virtual void getAnalysisUsage(AnalysisUsage &AU) const {
+      AU.addRequired<MachineDominatorTree>();
+      AU.addPreserved<MachineDominatorTree>();
+      MachineFunctionPass::getAnalysisUsage(AU);
+    }
+    virtual bool runOnMachineFunction(MachineFunction &MF);
+
+  private:
+    typedef SetVector<MachineInstr*> VectOfInst;
+    typedef std::set<Register> SetOfReg;
+    typedef std::map<Register,Register> RegToRegMap;
+
+    const HexagonInstrInfo *TII;
+    const HexagonRegisterInfo *TRI;
+    MachineRegisterInfo *MRI;
+    SetOfReg PredGPRs;
+    VectOfInst PUsers;
+    RegToRegMap G2P;
+
+    bool isPredReg(unsigned R);
+    void collectPredicateGPR(MachineFunction &MF);
+    void processPredicateGPR(const Register &Reg);
+    unsigned getPredForm(unsigned Opc);
+    bool isConvertibleToPredForm(const MachineInstr *MI);
+    bool isScalarCmp(unsigned Opc);
+    bool isScalarPred(Register PredReg);
+    Register getPredRegFor(const Register &Reg);
+    bool convertToPredForm(MachineInstr *MI);
+    bool eliminatePredCopies(MachineFunction &MF);
+  };
+
+  char HexagonGenPredicate::ID = 0;
+}
+
+INITIALIZE_PASS_BEGIN(HexagonGenPredicate, "hexagon-gen-pred",
+  "Hexagon generate predicate operations", false, false)
+INITIALIZE_PASS_DEPENDENCY(MachineDominatorTree)
+INITIALIZE_PASS_END(HexagonGenPredicate, "hexagon-gen-pred",
+  "Hexagon generate predicate operations", false, false)
+
+bool HexagonGenPredicate::isPredReg(unsigned R) {
+  if (!TargetRegisterInfo::isVirtualRegister(R))
+    return false;
+  const TargetRegisterClass *RC = MRI->getRegClass(R);
+  return RC == &Hexagon::PredRegsRegClass;
+}
+
+
+unsigned HexagonGenPredicate::getPredForm(unsigned Opc) {
+  using namespace Hexagon;
+
+  switch (Opc) {
+    case A2_and:
+    case A2_andp:
+      return C2_and;
+    case A4_andn:
+    case A4_andnp:
+      return C2_andn;
+    case M4_and_and:
+      return C4_and_and;
+    case M4_and_andn:
+      return C4_and_andn;
+    case M4_and_or:
+      return C4_and_or;
+
+    case A2_or:
+    case A2_orp:
+      return C2_or;
+    case A4_orn:
+    case A4_ornp:
+      return C2_orn;
+    case M4_or_and:
+      return C4_or_and;
+    case M4_or_andn:
+      return C4_or_andn;
+    case M4_or_or:
+      return C4_or_or;
+
+    case A2_xor:
+    case A2_xorp:
+      return C2_xor;
+
+    case C2_tfrrp:
+      return COPY;
+  }
+  // The opcode corresponding to 0 is TargetOpcode::PHI. We can use 0 here
+  // to denote "none", but we need to make sure that none of the valid opcodes
+  // that we return will ever be 0.
+  assert(PHI == 0 && "Use different value for <none>");
+  return 0;
+}
+
+
+bool HexagonGenPredicate::isConvertibleToPredForm(const MachineInstr *MI) {
+  unsigned Opc = MI->getOpcode();
+  if (getPredForm(Opc) != 0)
+    return true;
+
+  // Comparisons against 0 are also convertible. This does not apply to
+  // A4_rcmpeqi or A4_rcmpneqi, since they produce values 0 or 1, which
+  // may not match the value that the predicate register would have if
+  // it was converted to a predicate form.
+  switch (Opc) {
+    case Hexagon::C2_cmpeqi:
+    case Hexagon::C4_cmpneqi:
+      if (MI->getOperand(2).isImm() && MI->getOperand(2).getImm() == 0)
+        return true;
+      break;
+  }
+  return false;
+}
+
+
+void HexagonGenPredicate::collectPredicateGPR(MachineFunction &MF) {
+  for (MachineFunction::iterator A = MF.begin(), Z = MF.end(); A != Z; ++A) {
+    MachineBasicBlock &B = *A;
+    for (MachineBasicBlock::iterator I = B.begin(), E = B.end(); I != E; ++I) {
+      MachineInstr *MI = &*I;
+      unsigned Opc = MI->getOpcode();
+      switch (Opc) {
+        case Hexagon::C2_tfrpr:
+        case TargetOpcode::COPY:
+          if (isPredReg(MI->getOperand(1).getReg())) {
+            Register RD = MI->getOperand(0);
+            if (TargetRegisterInfo::isVirtualRegister(RD.R))
+              PredGPRs.insert(RD);
+          }
+          break;
+      }
+    }
+  }
+}
+
+
+void HexagonGenPredicate::processPredicateGPR(const Register &Reg) {
+  DEBUG(dbgs() << LLVM_FUNCTION_NAME << ": "
+               << PrintReg(Reg.R, TRI, Reg.S) << "\n");
+  typedef MachineRegisterInfo::use_iterator use_iterator;
+  use_iterator I = MRI->use_begin(Reg.R), E = MRI->use_end();
+  if (I == E) {
+    DEBUG(dbgs() << "Dead reg: " << PrintReg(Reg.R, TRI, Reg.S) << '\n');
+    MachineInstr *DefI = MRI->getVRegDef(Reg.R);
+    DefI->eraseFromParent();
+    return;
+  }
+
+  for (; I != E; ++I) {
+    MachineInstr *UseI = I->getParent();
+    if (isConvertibleToPredForm(UseI))
+      PUsers.insert(UseI);
+  }
+}
+
+
+Register HexagonGenPredicate::getPredRegFor(const Register &Reg) {
+  // Create a predicate register for a given Reg. The newly created register
+  // will have its value copied from Reg, so that it can be later used as
+  // an operand in other instructions.
+  assert(TargetRegisterInfo::isVirtualRegister(Reg.R));
+  RegToRegMap::iterator F = G2P.find(Reg);
+  if (F != G2P.end())
+    return F->second;
+
+  DEBUG(dbgs() << LLVM_FUNCTION_NAME << ": " << PrintRegister(Reg, *TRI));
+  MachineInstr *DefI = MRI->getVRegDef(Reg.R);
+  assert(DefI);
+  unsigned Opc = DefI->getOpcode();
+  if (Opc == Hexagon::C2_tfrpr || Opc == TargetOpcode::COPY) {
+    assert(DefI->getOperand(0).isDef() && DefI->getOperand(1).isUse());
+    Register PR = DefI->getOperand(1);
+    G2P.insert(std::make_pair(Reg, PR));
+    DEBUG(dbgs() << " -> " << PrintRegister(PR, *TRI) << '\n');
+    return PR;
+  }
+
+  MachineBasicBlock &B = *DefI->getParent();
+  DebugLoc DL = DefI->getDebugLoc();
+  const TargetRegisterClass *PredRC = &Hexagon::PredRegsRegClass;
+  unsigned NewPR = MRI->createVirtualRegister(PredRC);
+
+  // For convertible instructions, do not modify them, so that they can
+  // be coverted later.  Generate a copy from Reg to NewPR.
+  if (isConvertibleToPredForm(DefI)) {
+    MachineBasicBlock::iterator DefIt = DefI;
+    BuildMI(B, std::next(DefIt), DL, TII->get(TargetOpcode::COPY), NewPR)
+      .addReg(Reg.R, 0, Reg.S);
+    G2P.insert(std::make_pair(Reg, Register(NewPR)));
+    DEBUG(dbgs() << " -> !" << PrintRegister(Register(NewPR), *TRI) << '\n');
+    return Register(NewPR);
+  }
+
+  llvm_unreachable("Invalid argument");
+}
+
+
+bool HexagonGenPredicate::isScalarCmp(unsigned Opc) {
+  switch (Opc) {
+    case Hexagon::C2_cmpeq:
+    case Hexagon::C2_cmpgt:
+    case Hexagon::C2_cmpgtu:
+    case Hexagon::C2_cmpeqp:
+    case Hexagon::C2_cmpgtp:
+    case Hexagon::C2_cmpgtup:
+    case Hexagon::C2_cmpeqi:
+    case Hexagon::C2_cmpgti:
+    case Hexagon::C2_cmpgtui:
+    case Hexagon::C2_cmpgei:
+    case Hexagon::C2_cmpgeui:
+    case Hexagon::C4_cmpneqi:
+    case Hexagon::C4_cmpltei:
+    case Hexagon::C4_cmplteui:
+    case Hexagon::C4_cmpneq:
+    case Hexagon::C4_cmplte:
+    case Hexagon::C4_cmplteu:
+    case Hexagon::A4_cmpbeq:
+    case Hexagon::A4_cmpbeqi:
+    case Hexagon::A4_cmpbgtu:
+    case Hexagon::A4_cmpbgtui:
+    case Hexagon::A4_cmpbgt:
+    case Hexagon::A4_cmpbgti:
+    case Hexagon::A4_cmpheq:
+    case Hexagon::A4_cmphgt:
+    case Hexagon::A4_cmphgtu:
+    case Hexagon::A4_cmpheqi:
+    case Hexagon::A4_cmphgti:
+    case Hexagon::A4_cmphgtui:
+      return true;
+  }
+  return false;
+}
+
+
+bool HexagonGenPredicate::isScalarPred(Register PredReg) {
+  std::queue<Register> WorkQ;
+  WorkQ.push(PredReg);
+
+  while (!WorkQ.empty()) {
+    Register PR = WorkQ.front();
+    WorkQ.pop();
+    const MachineInstr *DefI = MRI->getVRegDef(PR.R);
+    if (!DefI)
+      return false;
+    unsigned DefOpc = DefI->getOpcode();
+    switch (DefOpc) {
+      case TargetOpcode::COPY: {
+        const TargetRegisterClass *PredRC = &Hexagon::PredRegsRegClass;
+        if (MRI->getRegClass(PR.R) != PredRC)
+          return false;
+        // If it is a copy between two predicate registers, fall through.
+      }
+      case Hexagon::C2_and:
+      case Hexagon::C2_andn:
+      case Hexagon::C4_and_and:
+      case Hexagon::C4_and_andn:
+      case Hexagon::C4_and_or:
+      case Hexagon::C2_or:
+      case Hexagon::C2_orn:
+      case Hexagon::C4_or_and:
+      case Hexagon::C4_or_andn:
+      case Hexagon::C4_or_or:
+      case Hexagon::C4_or_orn:
+      case Hexagon::C2_xor:
+        // Add operands to the queue.
+        for (ConstMIOperands Mo(DefI); Mo.isValid(); ++Mo)
+          if (Mo->isReg() && Mo->isUse())
+            WorkQ.push(Register(Mo->getReg()));
+        break;
+
+      // All non-vector compares are ok, everything else is bad.
+      default:
+        return isScalarCmp(DefOpc);
+    }
+  }
+
+  return true;
+}
+
+
+bool HexagonGenPredicate::convertToPredForm(MachineInstr *MI) {
+  DEBUG(dbgs() << LLVM_FUNCTION_NAME << ": " << MI << " " << *MI);
+
+  unsigned Opc = MI->getOpcode();
+  assert(isConvertibleToPredForm(MI));
+  unsigned NumOps = MI->getNumOperands();
+  for (unsigned i = 0; i < NumOps; ++i) {
+    MachineOperand &MO = MI->getOperand(i);
+    if (!MO.isReg() || !MO.isUse())
+      continue;
+    Register Reg(MO);
+    if (Reg.S && Reg.S != Hexagon::subreg_loreg)
+      return false;
+    if (!PredGPRs.count(Reg))
+      return false;
+  }
+
+  MachineBasicBlock &B = *MI->getParent();
+  DebugLoc DL = MI->getDebugLoc();
+
+  unsigned NewOpc = getPredForm(Opc);
+  // Special case for comparisons against 0.
+  if (NewOpc == 0) {
+    switch (Opc) {
+      case Hexagon::C2_cmpeqi:
+        NewOpc = Hexagon::C2_not;
+        break;
+      case Hexagon::C4_cmpneqi:
+        NewOpc = TargetOpcode::COPY;
+        break;
+      default:
+        return false;
+    }
+
+    // If it's a scalar predicate register, then all bits in it are
+    // the same. Otherwise, to determine whether all bits are 0 or not
+    // we would need to use any8.
+    Register PR = getPredRegFor(MI->getOperand(1));
+    if (!isScalarPred(PR))
+      return false;
+    // This will skip the immediate argument when creating the predicate
+    // version instruction.
+    NumOps = 2;
+  }
+
+  // Some sanity: check that def is in operand #0.
+  MachineOperand &Op0 = MI->getOperand(0);
+  assert(Op0.isDef());
+  Register OutR(Op0);
+
+  // Don't use getPredRegFor, since it will create an association between
+  // the argument and a created predicate register (i.e. it will insert a
+  // copy if a new predicate register is created).
+  const TargetRegisterClass *PredRC = &Hexagon::PredRegsRegClass;
+  Register NewPR = MRI->createVirtualRegister(PredRC);
+  MachineInstrBuilder MIB = BuildMI(B, MI, DL, TII->get(NewOpc), NewPR.R);
+
+  // Add predicate counterparts of the GPRs.
+  for (unsigned i = 1; i < NumOps; ++i) {
+    Register GPR = MI->getOperand(i);
+    Register Pred = getPredRegFor(GPR);
+    MIB.addReg(Pred.R, 0, Pred.S);
+  }
+  DEBUG(dbgs() << "generated: " << *MIB);
+
+  // Generate a copy-out: NewGPR = NewPR, and replace all uses of OutR
+  // with NewGPR.
+  const TargetRegisterClass *RC = MRI->getRegClass(OutR.R);
+  unsigned NewOutR = MRI->createVirtualRegister(RC);
+  BuildMI(B, MI, DL, TII->get(TargetOpcode::COPY), NewOutR)
+    .addReg(NewPR.R, 0, NewPR.S);
+  MRI->replaceRegWith(OutR.R, NewOutR);
+  MI->eraseFromParent();
+
+  // If the processed instruction was C2_tfrrp (i.e. Rn = Pm; Pk = Rn),
+  // then the output will be a predicate register.  Do not visit the
+  // users of it.
+  if (!isPredReg(NewOutR)) {
+    Register R(NewOutR);
+    PredGPRs.insert(R);
+    processPredicateGPR(R);
+  }
+  return true;
+}
+
+
+bool HexagonGenPredicate::eliminatePredCopies(MachineFunction &MF) {
+  DEBUG(dbgs() << LLVM_FUNCTION_NAME << "\n");
+  const TargetRegisterClass *PredRC = &Hexagon::PredRegsRegClass;
+  bool Changed = false;
+  VectOfInst Erase;
+
+  // First, replace copies
+  //   IntR = PredR1
+  //   PredR2 = IntR
+  // with
+  //   PredR2 = PredR1
+  // Such sequences can be generated when a copy-into-pred is generated from
+  // a gpr register holding a result of a convertible instruction. After
+  // the convertible instruction is converted, its predicate result will be
+  // copied back into the original gpr.
+
+  for (MachineFunction::iterator A = MF.begin(), Z = MF.end(); A != Z; ++A) {
+    MachineBasicBlock &B = *A;
+    for (MachineBasicBlock::iterator I = B.begin(), E = B.end(); I != E; ++I) {
+      if (I->getOpcode() != TargetOpcode::COPY)
+        continue;
+      Register DR = I->getOperand(0);
+      Register SR = I->getOperand(1);
+      if (!TargetRegisterInfo::isVirtualRegister(DR.R))
+        continue;
+      if (!TargetRegisterInfo::isVirtualRegister(SR.R))
+        continue;
+      if (MRI->getRegClass(DR.R) != PredRC)
+        continue;
+      if (MRI->getRegClass(SR.R) != PredRC)
+        continue;
+      assert(!DR.S && !SR.S && "Unexpected subregister");
+      MRI->replaceRegWith(DR.R, SR.R);
+      Erase.insert(I);
+      Changed = true;
+    }
+  }
+
+  for (VectOfInst::iterator I = Erase.begin(), E = Erase.end(); I != E; ++I)
+    (*I)->eraseFromParent();
+
+  return Changed;
+}
+
+
+bool HexagonGenPredicate::runOnMachineFunction(MachineFunction &MF) {
+  TII = MF.getSubtarget<HexagonSubtarget>().getInstrInfo();
+  TRI = MF.getSubtarget<HexagonSubtarget>().getRegisterInfo();
+  MRI = &MF.getRegInfo();
+  PredGPRs.clear();
+  PUsers.clear();
+  G2P.clear();
+
+  bool Changed = false;
+  collectPredicateGPR(MF);
+  for (SetOfReg::iterator I = PredGPRs.begin(), E = PredGPRs.end(); I != E; ++I)
+    processPredicateGPR(*I);
+
+  bool Again;
+  do {
+    Again = false;
+    VectOfInst Processed, Copy;
+
+    typedef VectOfInst::iterator iterator;
+    Copy = PUsers;
+    for (iterator I = Copy.begin(), E = Copy.end(); I != E; ++I) {
+      MachineInstr *MI = *I;
+      bool Done = convertToPredForm(MI);
+      if (Done) {
+        Processed.insert(MI);
+        Again = true;
+      }
+    }
+    Changed |= Again;
+
+    auto Done = [Processed] (MachineInstr *MI) -> bool {
+      return Processed.count(MI);
+    };
+    PUsers.remove_if(Done);
+  } while (Again);
+
+  Changed |= eliminatePredCopies(MF);
+  return Changed;
+}
+
+
+FunctionPass *llvm::createHexagonGenPredicate() {
+  return new HexagonGenPredicate();
+}
+
diff --git a/lib/Target/Hexagon/HexagonHardwareLoops.cpp b/lib/Target/Hexagon/HexagonHardwareLoops.cpp
index c47ee9c8e2e..53b6bf617e8 100644
--- a/lib/Target/Hexagon/HexagonHardwareLoops.cpp
+++ b/lib/Target/Hexagon/HexagonHardwareLoops.cpp
@@ -21,7 +21,6 @@
 //  - Countable loops (w/ ind. var for a trip count)
 //  - Assumes loops are normalized by IndVarSimplify
 //  - Try inner-most loops first
-//  - No nested hardware loops.
 //  - No function calls in loops.
 //
 //===----------------------------------------------------------------------===//
@@ -49,12 +48,22 @@ using namespace llvm;
 #define DEBUG_TYPE "hwloops"
 
 #ifndef NDEBUG
-static cl::opt<int> HWLoopLimit("max-hwloop", cl::Hidden, cl::init(-1));
+static cl::opt<int> HWLoopLimit("hexagon-max-hwloop", cl::Hidden, cl::init(-1));
+
+// Option to create preheader only for a specific function.
+static cl::opt<std::string> PHFn("hexagon-hwloop-phfn", cl::Hidden,
+                                 cl::init(""));
 #endif
 
+// Option to create a preheader if one doesn't exist.
+static cl::opt<bool> HWCreatePreheader("hexagon-hwloop-preheader",
+    cl::Hidden, cl::init(true),
+    cl::desc("Add a preheader to a hardware loop if one doesn't exist"));
+
 STATISTIC(NumHWLoops, "Number of loops converted to hardware loops");
 
 namespace llvm {
+  FunctionPass *createHexagonHardwareLoops();
   void initializeHexagonHardwareLoopsPass(PassRegistry&);
 }
 
@@ -87,14 +96,16 @@ namespace {
     }
 
   private:
+    typedef std::map<unsigned, MachineInstr *> LoopFeederMap;
+
     /// Kinds of comparisons in the compare instructions.
     struct Comparison {
       enum Kind {
         EQ  = 0x01,
         NE  = 0x02,
-        L   = 0x04, // Less-than property.
-        G   = 0x08, // Greater-than property.
-        U   = 0x40, // Unsigned property.
+        L   = 0x04,
+        G   = 0x08,
+        U   = 0x40,
         LTs = L,
         LEs = L | EQ,
         GTs = G,
@@ -111,6 +122,23 @@ namespace {
           return (Kind)(Cmp ^ (L|G));
         return Cmp;
       }
+
+      static Kind getNegatedComparison(Kind Cmp) {
+        if ((Cmp & L) || (Cmp & G))
+          return (Kind)((Cmp ^ (L | G)) ^ EQ);
+        if ((Cmp & NE) || (Cmp & EQ))
+          return (Kind)(Cmp ^ (EQ | NE));
+        return (Kind)0;
+      }
+
+      static bool isSigned(Kind Cmp) {
+        return (Cmp & (L | G) && !(Cmp & U));
+      }
+
+      static bool isUnsigned(Kind Cmp) {
+        return (Cmp & U);
+      }
+
     };
 
     /// \brief Find the register that contains the loop controlling
@@ -128,6 +156,12 @@ namespace {
     bool findInductionRegister(MachineLoop *L, unsigned &Reg,
                                int64_t &IVBump, MachineInstr *&IVOp) const;
 
+    /// \brief Return the comparison kind for the specified opcode.
+    Comparison::Kind getComparisonKind(unsigned CondOpc,
+                                       MachineOperand *InitialValue,
+                                       const MachineOperand *Endvalue,
+                                       int64_t IVBump) const;
+
     /// \brief Analyze the statements in a loop to determine if the loop
     /// has a computable trip count and, if so, return a value that represents
     /// the trip count expression.
@@ -141,24 +175,22 @@ namespace {
     /// If the trip count is not directly available (as an immediate value,
     /// or a register), the function will attempt to insert computation of it
     /// to the loop's preheader.
-    CountValue *computeCount(MachineLoop *Loop,
-                             const MachineOperand *Start,
-                             const MachineOperand *End,
-                             unsigned IVReg,
-                             int64_t IVBump,
-                             Comparison::Kind Cmp) const;
+    CountValue *computeCount(MachineLoop *Loop, const MachineOperand *Start,
+                             const MachineOperand *End, unsigned IVReg,
+                             int64_t IVBump, Comparison::Kind Cmp) const;
 
     /// \brief Return true if the instruction is not valid within a hardware
     /// loop.
-    bool isInvalidLoopOperation(const MachineInstr *MI) const;
+    bool isInvalidLoopOperation(const MachineInstr *MI,
+                                bool IsInnerHWLoop) const;
 
     /// \brief Return true if the loop contains an instruction that inhibits
     /// using the hardware loop.
-    bool containsInvalidInstruction(MachineLoop *L) const;
+    bool containsInvalidInstruction(MachineLoop *L, bool IsInnerHWLoop) const;
 
     /// \brief Given a loop, check if we can convert it to a hardware loop.
     /// If so, then perform the conversion and return true.
-    bool convertToHardwareLoop(MachineLoop *L);
+    bool convertToHardwareLoop(MachineLoop *L, bool &L0used, bool &L1used);
 
     /// \brief Return true if the instruction is now dead.
     bool isDead(const MachineInstr *MI,
@@ -173,14 +205,44 @@ namespace {
     /// defined.  If the instructions are out of order, try to reorder them.
     bool orderBumpCompare(MachineInstr *BumpI, MachineInstr *CmpI);
 
-    /// \brief Get the instruction that loads an immediate value into \p R,
-    /// or 0 if such an instruction does not exist.
-    MachineInstr *defWithImmediate(unsigned R);
+    /// \brief Return true if MO and MI pair is visited only once. If visited
+    /// more than once, this indicates there is recursion. In such a case,
+    /// return false.
+    bool isLoopFeeder(MachineLoop *L, MachineBasicBlock *A, MachineInstr *MI,
+                      const MachineOperand *MO,
+                      LoopFeederMap &LoopFeederPhi) const;
+
+    /// \brief Return true if the Phi may generate a value that may underflow,
+    /// or may wrap.
+    bool phiMayWrapOrUnderflow(MachineInstr *Phi, const MachineOperand *EndVal,
+                               MachineBasicBlock *MBB, MachineLoop *L,
+                               LoopFeederMap &LoopFeederPhi) const;
+
+    /// \brief Return true if the induction variable may underflow an unsigned
+    /// value in the first iteration.
+    bool loopCountMayWrapOrUnderFlow(const MachineOperand *InitVal,
+                                     const MachineOperand *EndVal,
+                                     MachineBasicBlock *MBB, MachineLoop *L,
+                                     LoopFeederMap &LoopFeederPhi) const;
+
+    /// \brief Check if the given operand has a compile-time known constant
+    /// value. Return true if yes, and false otherwise. When returning true, set
+    /// Val to the corresponding constant value.
+    bool checkForImmediate(const MachineOperand &MO, int64_t &Val) const;
+
+    /// \brief Check if the operand has a compile-time known constant value.
+    bool isImmediate(const MachineOperand &MO) const {
+      int64_t V;
+      return checkForImmediate(MO, V);
+    }
 
-    /// \brief Get the immediate value referenced to by \p MO, either for
-    /// immediate operands, or for register operands, where the register
-    /// was defined with an immediate value.
-    int64_t getImmediate(MachineOperand &MO);
+    /// \brief Return the immediate for the specified operand.
+    int64_t getImmediate(const MachineOperand &MO) const {
+      int64_t V;
+      if (!checkForImmediate(MO, V))
+        llvm_unreachable("Invalid operand");
+      return V;
+    }
 
     /// \brief Reset the given machine operand to now refer to a new immediate
     /// value.  Assumes that the operand was already referencing an immediate
@@ -278,18 +340,10 @@ INITIALIZE_PASS_DEPENDENCY(MachineLoopInfo)
 INITIALIZE_PASS_END(HexagonHardwareLoops, "hwloops",
                     "Hexagon Hardware Loops", false, false)
 
-
-/// \brief Returns true if the instruction is a hardware loop instruction.
-static bool isHardwareLoop(const MachineInstr *MI) {
-  return MI->getOpcode() == Hexagon::J2_loop0r ||
-    MI->getOpcode() == Hexagon::J2_loop0i;
-}
-
 FunctionPass *llvm::createHexagonHardwareLoops() {
   return new HexagonHardwareLoops();
 }
 
-
 bool HexagonHardwareLoops::runOnMachineFunction(MachineFunction &MF) {
   DEBUG(dbgs() << "********* Hexagon Hardware Loops *********\n");
 
@@ -300,16 +354,28 @@ bool HexagonHardwareLoops::runOnMachineFunction(MachineFunction &MF) {
   MDT = &getAnalysis<MachineDominatorTree>();
   TII = MF.getSubtarget<HexagonSubtarget>().getInstrInfo();
 
-  for (MachineLoopInfo::iterator I = MLI->begin(), E = MLI->end();
-       I != E; ++I) {
-    MachineLoop *L = *I;
-    if (!L->getParentLoop())
-      Changed |= convertToHardwareLoop(L);
-  }
+  for (auto &L : *MLI)
+    if (!L->getParentLoop()) {
+      bool L0Used = false;
+      bool L1Used = false;
+      Changed |= convertToHardwareLoop(L, L0Used, L1Used);
+    }
 
   return Changed;
 }
 
+/// \brief Return the latch block if it's one of the exiting blocks. Otherwise,
+/// return the exiting block. Return 'null' when multiple exiting blocks are
+/// present.
+static MachineBasicBlock* getExitingBlock(MachineLoop *L) {
+  if (MachineBasicBlock *Latch = L->getLoopLatch()) {
+    if (L->isLoopExiting(Latch))
+      return Latch;
+    else
+      return L->getExitingBlock();
+  }
+  return nullptr;
+}
 
 bool HexagonHardwareLoops::findInductionRegister(MachineLoop *L,
                                                  unsigned &Reg,
@@ -319,7 +385,8 @@ bool HexagonHardwareLoops::findInductionRegister(MachineLoop *L,
   MachineBasicBlock *Header = L->getHeader();
   MachineBasicBlock *Preheader = L->getLoopPreheader();
   MachineBasicBlock *Latch = L->getLoopLatch();
-  if (!Header || !Preheader || !Latch)
+  MachineBasicBlock *ExitingBlock = getExitingBlock(L);
+  if (!Header || !Preheader || !Latch || !ExitingBlock)
     return false;
 
   // This pair represents an induction register together with an immediate
@@ -349,15 +416,16 @@ bool HexagonHardwareLoops::findInductionRegister(MachineLoop *L,
       unsigned PhiOpReg = Phi->getOperand(i).getReg();
       MachineInstr *DI = MRI->getVRegDef(PhiOpReg);
       unsigned UpdOpc = DI->getOpcode();
-      bool isAdd = (UpdOpc == Hexagon::A2_addi);
+      bool isAdd = (UpdOpc == Hexagon::A2_addi || UpdOpc == Hexagon::A2_addp);
 
       if (isAdd) {
-        // If the register operand to the add is the PHI we're
-        // looking at, this meets the induction pattern.
+        // If the register operand to the add is the PHI we're looking at, this
+        // meets the induction pattern.
         unsigned IndReg = DI->getOperand(1).getReg();
-        if (MRI->getVRegDef(IndReg) == Phi) {
+        MachineOperand &Opnd2 = DI->getOperand(2);
+        int64_t V;
+        if (MRI->getVRegDef(IndReg) == Phi && checkForImmediate(Opnd2, V)) {
           unsigned UpdReg = DI->getOperand(0).getReg();
-          int64_t V = DI->getOperand(2).getImm();
           IndMap.insert(std::make_pair(UpdReg, std::make_pair(IndReg, V)));
         }
       }
@@ -366,13 +434,13 @@ bool HexagonHardwareLoops::findInductionRegister(MachineLoop *L,
 
   SmallVector<MachineOperand,2> Cond;
   MachineBasicBlock *TB = nullptr, *FB = nullptr;
-  bool NotAnalyzed = TII->AnalyzeBranch(*Latch, TB, FB, Cond, false);
+  bool NotAnalyzed = TII->AnalyzeBranch(*ExitingBlock, TB, FB, Cond, false);
   if (NotAnalyzed)
     return false;
 
-  unsigned CSz = Cond.size();
-  assert (CSz == 1 || CSz == 2);
-  unsigned PredR = Cond[CSz-1].getReg();
+  unsigned PredR, PredPos, PredRegFlags;
+  if (!TII->getPredReg(Cond, PredR, PredPos, PredRegFlags))
+    return false;
 
   MachineInstr *PredI = MRI->getVRegDef(PredR);
   if (!PredI->isCompare())
@@ -384,7 +452,7 @@ bool HexagonHardwareLoops::findInductionRegister(MachineLoop *L,
                                          CmpMask, CmpImm);
   // Fail if the compare was not analyzed, or it's not comparing a register
   // with an immediate value.  Not checking the mask here, since we handle
-  // the individual compare opcodes (including CMPb) later on.
+  // the individual compare opcodes (including A4_cmpb*) later on.
   if (!CmpAnalyzed)
     return false;
 
@@ -414,6 +482,44 @@ bool HexagonHardwareLoops::findInductionRegister(MachineLoop *L,
   return true;
 }
 
+// Return the comparison kind for the specified opcode.
+HexagonHardwareLoops::Comparison::Kind
+HexagonHardwareLoops::getComparisonKind(unsigned CondOpc,
+                                        MachineOperand *InitialValue,
+                                        const MachineOperand *EndValue,
+                                        int64_t IVBump) const {
+  Comparison::Kind Cmp = (Comparison::Kind)0;
+  switch (CondOpc) {
+  case Hexagon::C2_cmpeqi:
+  case Hexagon::C2_cmpeq:
+  case Hexagon::C2_cmpeqp:
+    Cmp = Comparison::EQ;
+    break;
+  case Hexagon::C4_cmpneq:
+  case Hexagon::C4_cmpneqi:
+    Cmp = Comparison::NE;
+    break;
+  case Hexagon::C4_cmplte:
+    Cmp = Comparison::LEs;
+    break;
+  case Hexagon::C4_cmplteu:
+    Cmp = Comparison::LEu;
+    break;
+  case Hexagon::C2_cmpgtui:
+  case Hexagon::C2_cmpgtu:
+  case Hexagon::C2_cmpgtup:
+    Cmp = Comparison::GTu;
+    break;
+  case Hexagon::C2_cmpgti:
+  case Hexagon::C2_cmpgt:
+  case Hexagon::C2_cmpgtp:
+    Cmp = Comparison::GTs;
+    break;
+  default:
+    return (Comparison::Kind)0;
+  }
+  return Cmp;
+}
 
 /// \brief Analyze the statements in a loop to determine if the loop has
 /// a computable trip count and, if so, return a value that represents
@@ -423,7 +529,7 @@ bool HexagonHardwareLoops::findInductionRegister(MachineLoop *L,
 /// induction variable patterns that are used in the calculation for
 /// the number of time the loop is executed.
 CountValue *HexagonHardwareLoops::getLoopTripCount(MachineLoop *L,
-                                    SmallVectorImpl<MachineInstr *> &OldInsts) {
+    SmallVectorImpl<MachineInstr *> &OldInsts) {
   MachineBasicBlock *TopMBB = L->getTopBlock();
   MachineBasicBlock::pred_iterator PI = TopMBB->pred_begin();
   assert(PI != TopMBB->pred_end() &&
@@ -447,8 +553,8 @@ CountValue *HexagonHardwareLoops::getLoopTripCount(MachineLoop *L,
   // Look for the cmp instruction to determine if we can get a useful trip
   // count.  The trip count can be either a register or an immediate.  The
   // location of the value depends upon the type (reg or imm).
-  MachineBasicBlock *Latch = L->getLoopLatch();
-  if (!Latch)
+  MachineBasicBlock *ExitingBlock = getExitingBlock(L);
+  if (!ExitingBlock)
     return nullptr;
 
   unsigned IVReg = 0;
@@ -462,6 +568,7 @@ CountValue *HexagonHardwareLoops::getLoopTripCount(MachineLoop *L,
 
   MachineOperand *InitialValue = nullptr;
   MachineInstr *IV_Phi = MRI->getVRegDef(IVReg);
+  MachineBasicBlock *Latch = L->getLoopLatch();
   for (unsigned i = 1, n = IV_Phi->getNumOperands(); i < n; i += 2) {
     MachineBasicBlock *MBB = IV_Phi->getOperand(i+1).getMBB();
     if (MBB == Preheader)
@@ -474,7 +581,7 @@ CountValue *HexagonHardwareLoops::getLoopTripCount(MachineLoop *L,
 
   SmallVector<MachineOperand,2> Cond;
   MachineBasicBlock *TB = nullptr, *FB = nullptr;
-  bool NotAnalyzed = TII->AnalyzeBranch(*Latch, TB, FB, Cond, false);
+  bool NotAnalyzed = TII->AnalyzeBranch(*ExitingBlock, TB, FB, Cond, false);
   if (NotAnalyzed)
     return nullptr;
 
@@ -482,7 +589,18 @@ CountValue *HexagonHardwareLoops::getLoopTripCount(MachineLoop *L,
   // TB must be non-null.  If FB is also non-null, one of them must be
   // the header.  Otherwise, branch to TB could be exiting the loop, and
   // the fall through can go to the header.
-  assert (TB && "Latch block without a branch?");
+  assert (TB && "Exit block without a branch?");
+  if (ExitingBlock != Latch && (TB == Latch || FB == Latch)) {
+    MachineBasicBlock *LTB = 0, *LFB = 0;
+    SmallVector<MachineOperand,2> LCond;
+    bool NotAnalyzed = TII->AnalyzeBranch(*Latch, LTB, LFB, LCond, false);
+    if (NotAnalyzed)
+      return nullptr;
+    if (TB == Latch)
+      TB = (LTB == Header) ? LTB : LFB;
+    else
+      FB = (LTB == Header) ? LTB: LFB;
+  }
   assert ((!FB || TB == Header || FB == Header) && "Branches not to header?");
   if (!TB || (FB && TB != Header && FB != Header))
     return nullptr;
@@ -491,8 +609,10 @@ CountValue *HexagonHardwareLoops::getLoopTripCount(MachineLoop *L,
   // to put imm(0), followed by P in the vector Cond.
   // If TB is not the header, it means that the "not-taken" path must lead
   // to the header.
-  bool Negated = (Cond.size() > 1) ^ (TB != Header);
-  unsigned PredReg = Cond[Cond.size()-1].getReg();
+  bool Negated = TII->predOpcodeHasNot(Cond) ^ (TB != Header);
+  unsigned PredReg, PredPos, PredRegFlags;
+  if (!TII->getPredReg(Cond, PredReg, PredPos, PredRegFlags))
+    return nullptr;
   MachineInstr *CondI = MRI->getVRegDef(PredReg);
   unsigned CondOpc = CondI->getOpcode();
 
@@ -531,57 +651,13 @@ CountValue *HexagonHardwareLoops::getLoopTripCount(MachineLoop *L,
   if (!EndValue)
     return nullptr;
 
-  switch (CondOpc) {
-    case Hexagon::C2_cmpeqi:
-    case Hexagon::C2_cmpeq:
-      Cmp = !Negated ? Comparison::EQ : Comparison::NE;
-      break;
-    case Hexagon::C2_cmpgtui:
-    case Hexagon::C2_cmpgtu:
-      Cmp = !Negated ? Comparison::GTu : Comparison::LEu;
-      break;
-    case Hexagon::C2_cmpgti:
-    case Hexagon::C2_cmpgt:
-      Cmp = !Negated ? Comparison::GTs : Comparison::LEs;
-      break;
-    // Very limited support for byte/halfword compares.
-    case Hexagon::A4_cmpbeqi:
-    case Hexagon::A4_cmpheqi: {
-      if (IVBump != 1)
-        return nullptr;
-
-      int64_t InitV, EndV;
-      // Since the comparisons are "ri", the EndValue should be an
-      // immediate.  Check it just in case.
-      assert(EndValue->isImm() && "Unrecognized latch comparison");
-      EndV = EndValue->getImm();
-      // Allow InitialValue to be a register defined with an immediate.
-      if (InitialValue->isReg()) {
-        if (!defWithImmediate(InitialValue->getReg()))
-          return nullptr;
-        InitV = getImmediate(*InitialValue);
-      } else {
-        assert(InitialValue->isImm());
-        InitV = InitialValue->getImm();
-      }
-      if (InitV >= EndV)
-        return nullptr;
-      if (CondOpc == Hexagon::A4_cmpbeqi) {
-        if (!isInt<8>(InitV) || !isInt<8>(EndV))
-          return nullptr;
-      } else {  // Hexagon::CMPhEQri_V4
-        if (!isInt<16>(InitV) || !isInt<16>(EndV))
-          return nullptr;
-      }
-      Cmp = !Negated ? Comparison::EQ : Comparison::NE;
-      break;
-    }
-    default:
-      return nullptr;
-  }
-
+  Cmp = getComparisonKind(CondOpc, InitialValue, EndValue, IVBump);
+  if (!Cmp)
+    return nullptr;
+  if (Negated)
+    Cmp = Comparison::getNegatedComparison(Cmp);
   if (isSwapped)
-   Cmp = Comparison::getSwappedComparison(Cmp);
+    Cmp = Comparison::getSwappedComparison(Cmp);
 
   if (InitialValue->isReg()) {
     unsigned R = InitialValue->getReg();
@@ -595,6 +671,7 @@ CountValue *HexagonHardwareLoops::getLoopTripCount(MachineLoop *L,
     MachineBasicBlock *DefBB = MRI->getVRegDef(R)->getParent();
     if (!MDT->properlyDominates(DefBB, Header))
       return nullptr;
+    OldInsts.push_back(MRI->getVRegDef(R));
   }
 
   return computeCount(L, InitialValue, EndValue, IVReg, IVBump, Cmp);
@@ -618,32 +695,45 @@ CountValue *HexagonHardwareLoops::computeCount(MachineLoop *Loop,
   // If so, use the immediate value rather than the register.
   if (Start->isReg()) {
     const MachineInstr *StartValInstr = MRI->getVRegDef(Start->getReg());
-    if (StartValInstr && StartValInstr->getOpcode() == Hexagon::A2_tfrsi)
+    if (StartValInstr && (StartValInstr->getOpcode() == Hexagon::A2_tfrsi ||
+                          StartValInstr->getOpcode() == Hexagon::A2_tfrpi))
       Start = &StartValInstr->getOperand(1);
   }
   if (End->isReg()) {
     const MachineInstr *EndValInstr = MRI->getVRegDef(End->getReg());
-    if (EndValInstr && EndValInstr->getOpcode() == Hexagon::A2_tfrsi)
+    if (EndValInstr && (EndValInstr->getOpcode() == Hexagon::A2_tfrsi ||
+                        EndValInstr->getOpcode() == Hexagon::A2_tfrpi))
       End = &EndValInstr->getOperand(1);
   }
 
-  assert (Start->isReg() || Start->isImm());
-  assert (End->isReg() || End->isImm());
+  if (!Start->isReg() && !Start->isImm())
+    return nullptr;
+  if (!End->isReg() && !End->isImm())
+    return nullptr;
 
   bool CmpLess =     Cmp & Comparison::L;
   bool CmpGreater =  Cmp & Comparison::G;
   bool CmpHasEqual = Cmp & Comparison::EQ;
 
   // Avoid certain wrap-arounds.  This doesn't detect all wrap-arounds.
-  // If loop executes while iv is "less" with the iv value going down, then
-  // the iv must wrap.
   if (CmpLess && IVBump < 0)
+    // Loop going while iv is "less" with the iv value going down.  Must wrap.
     return nullptr;
-  // If loop executes while iv is "greater" with the iv value going up, then
-  // the iv must wrap.
+
   if (CmpGreater && IVBump > 0)
+    // Loop going while iv is "greater" with the iv value going up.  Must wrap.
     return nullptr;
 
+  // Phis that may feed into the loop.
+  LoopFeederMap LoopFeederPhi;
+
+  // Check if the inital value may be zero and can be decremented in the first
+  // iteration. If the value is zero, the endloop instruction will not decrement
+  // the loop counter, so we shoudn't generate a hardware loop in this case.
+  if (loopCountMayWrapOrUnderFlow(Start, End, Loop->getLoopPreheader(), Loop,
+                                  LoopFeederPhi))
+      return nullptr;
+
   if (Start->isImm() && End->isImm()) {
     // Both, start and end are immediates.
     int64_t StartV = Start->getImm();
@@ -666,14 +756,16 @@ CountValue *HexagonHardwareLoops::computeCount(MachineLoop *Loop,
     if (CmpHasEqual)
       Dist = Dist > 0 ? Dist+1 : Dist-1;
 
-    // assert (CmpLess => Dist > 0);
-    assert ((!CmpLess || Dist > 0) && "Loop should never iterate!");
-    // assert (CmpGreater => Dist < 0);
-    assert ((!CmpGreater || Dist < 0) && "Loop should never iterate!");
+    // For the loop to iterate, CmpLess should imply Dist > 0.  Similarly,
+    // CmpGreater should imply Dist < 0.  These conditions could actually
+    // fail, for example, in unreachable code (which may still appear to be
+    // reachable in the CFG).
+    if ((CmpLess && Dist < 0) || (CmpGreater && Dist > 0))
+      return nullptr;
 
     // "Normalized" distance, i.e. with the bump set to +-1.
-    int64_t Dist1 = (IVBump > 0) ? (Dist +  (IVBump-1)) /   IVBump
-                               :  (-Dist + (-IVBump-1)) / (-IVBump);
+    int64_t Dist1 = (IVBump > 0) ? (Dist +  (IVBump - 1)) / IVBump
+                                 : (-Dist + (-IVBump - 1)) / (-IVBump);
     assert (Dist1 > 0 && "Fishy thing.  Both operands have the same sign.");
 
     uint64_t Count = Dist1;
@@ -696,8 +788,9 @@ CountValue *HexagonHardwareLoops::computeCount(MachineLoop *Loop,
   MachineBasicBlock *PH = Loop->getLoopPreheader();
   assert (PH && "Should have a preheader by now");
   MachineBasicBlock::iterator InsertPos = PH->getFirstTerminator();
-  DebugLoc DL = (InsertPos != PH->end()) ? InsertPos->getDebugLoc()
-                                         : DebugLoc();
+  DebugLoc DL;
+  if (InsertPos != PH->end())
+    DL = InsertPos->getDebugLoc();
 
   // If Start is an immediate and End is a register, the trip count
   // will be "reg - imm".  Hexagon's "subtract immediate" instruction
@@ -776,21 +869,35 @@ CountValue *HexagonHardwareLoops::computeCount(MachineLoop *Loop,
     const MCInstrDesc &SubD = RegToReg ? TII->get(Hexagon::A2_sub) :
                               (RegToImm ? TII->get(Hexagon::A2_subri) :
                                           TII->get(Hexagon::A2_addi));
-    unsigned SubR = MRI->createVirtualRegister(IntRC);
-    MachineInstrBuilder SubIB =
-      BuildMI(*PH, InsertPos, DL, SubD, SubR);
-
-    if (RegToReg) {
-      SubIB.addReg(End->getReg(), 0, End->getSubReg())
-           .addReg(Start->getReg(), 0, Start->getSubReg());
-    } else if (RegToImm) {
-      SubIB.addImm(EndV)
-           .addReg(Start->getReg(), 0, Start->getSubReg());
-    } else { // ImmToReg
-      SubIB.addReg(End->getReg(), 0, End->getSubReg())
-           .addImm(-StartV);
+    if (RegToReg || RegToImm) {
+      unsigned SubR = MRI->createVirtualRegister(IntRC);
+      MachineInstrBuilder SubIB =
+        BuildMI(*PH, InsertPos, DL, SubD, SubR);
+
+      if (RegToReg)
+        SubIB.addReg(End->getReg(), 0, End->getSubReg())
+          .addReg(Start->getReg(), 0, Start->getSubReg());
+      else
+        SubIB.addImm(EndV)
+          .addReg(Start->getReg(), 0, Start->getSubReg());
+      DistR = SubR;
+    } else {
+      // If the loop has been unrolled, we should use the original loop count
+      // instead of recalculating the value. This will avoid additional
+      // 'Add' instruction.
+      const MachineInstr *EndValInstr = MRI->getVRegDef(End->getReg());
+      if (EndValInstr->getOpcode() == Hexagon::A2_addi &&
+          EndValInstr->getOperand(2).getImm() == StartV) {
+        DistR = EndValInstr->getOperand(1).getReg();
+      } else {
+        unsigned SubR = MRI->createVirtualRegister(IntRC);
+        MachineInstrBuilder SubIB =
+          BuildMI(*PH, InsertPos, DL, SubD, SubR);
+        SubIB.addReg(End->getReg(), 0, End->getSubReg())
+             .addImm(-StartV);
+        DistR = SubR;
+      }
     }
-    DistR = SubR;
     DistSR = 0;
   }
 
@@ -836,50 +943,50 @@ CountValue *HexagonHardwareLoops::computeCount(MachineLoop *Loop,
   return new CountValue(CountValue::CV_Register, CountR, CountSR);
 }
 
-
 /// \brief Return true if the operation is invalid within hardware loop.
-bool HexagonHardwareLoops::isInvalidLoopOperation(
-      const MachineInstr *MI) const {
+bool HexagonHardwareLoops::isInvalidLoopOperation(const MachineInstr *MI,
+                                                  bool IsInnerHWLoop) const {
 
-  // call is not allowed because the callee may use a hardware loop
-  if (MI->getDesc().isCall())
+  // Call is not allowed because the callee may use a hardware loop except for
+  // the case when the call never returns.
+  if (MI->getDesc().isCall() && MI->getOpcode() != Hexagon::CALLv3nr)
     return true;
 
-  // do not allow nested hardware loops
-  if (isHardwareLoop(MI))
-    return true;
-
-  // check if the instruction defines a hardware loop register
+  // Check if the instruction defines a hardware loop register.
   for (unsigned i = 0, e = MI->getNumOperands(); i != e; ++i) {
     const MachineOperand &MO = MI->getOperand(i);
     if (!MO.isReg() || !MO.isDef())
       continue;
     unsigned R = MO.getReg();
-    if (R == Hexagon::LC0 || R == Hexagon::LC1 ||
-        R == Hexagon::SA0 || R == Hexagon::SA1)
+    if (IsInnerHWLoop && (R == Hexagon::LC0 || R == Hexagon::SA0 ||
+                          R == Hexagon::LC1 || R == Hexagon::SA1))
+      return true;
+    if (!IsInnerHWLoop && (R == Hexagon::LC1 || R == Hexagon::SA1))
       return true;
   }
   return false;
 }
 
-
-/// \brief - Return true if the loop contains an instruction that inhibits
-/// the use of the hardware loop function.
-bool HexagonHardwareLoops::containsInvalidInstruction(MachineLoop *L) const {
+/// \brief Return true if the loop contains an instruction that inhibits
+/// the use of the hardware loop instruction.
+bool HexagonHardwareLoops::containsInvalidInstruction(MachineLoop *L,
+    bool IsInnerHWLoop) const {
   const std::vector<MachineBasicBlock *> &Blocks = L->getBlocks();
+  DEBUG(dbgs() << "\nhw_loop head, BB#" << Blocks[0]->getNumber(););
   for (unsigned i = 0, e = Blocks.size(); i != e; ++i) {
     MachineBasicBlock *MBB = Blocks[i];
     for (MachineBasicBlock::iterator
            MII = MBB->begin(), E = MBB->end(); MII != E; ++MII) {
       const MachineInstr *MI = &*MII;
-      if (isInvalidLoopOperation(MI))
+      if (isInvalidLoopOperation(MI, IsInnerHWLoop)) {
+        DEBUG(dbgs()<< "\nCannot convert to hw_loop due to:"; MI->dump(););
         return true;
+      }
     }
   }
   return false;
 }
 
-
 /// \brief Returns true if the instruction is dead.  This was essentially
 /// copied from DeadMachineInstructionElim::isDead, but with special cases
 /// for inline asm, physical registers and instructions with side effects
@@ -920,7 +1027,7 @@ bool HexagonHardwareLoops::isDead(const MachineInstr *MI,
         MachineOperand &Use = *J;
         MachineInstr *UseMI = Use.getParent();
 
-        // If the phi node has a user that is not MI, bail...
+        // If the phi node has a user that is not MI, bail.
         if (MI != UseMI)
           return false;
       }
@@ -957,8 +1064,6 @@ void HexagonHardwareLoops::removeIfDead(MachineInstr *MI) {
           continue;
         if (Use.isDebug())
           UseMI->getOperand(0).setReg(0U);
-        // This may also be a "instr -> phi -> instr" case which can
-        // be removed too.
       }
     }
 
@@ -976,19 +1081,47 @@ void HexagonHardwareLoops::removeIfDead(MachineInstr *MI) {
 ///
 /// The code makes several assumptions about the representation of the loop
 /// in llvm.
-bool HexagonHardwareLoops::convertToHardwareLoop(MachineLoop *L) {
+bool HexagonHardwareLoops::convertToHardwareLoop(MachineLoop *L,
+                                                 bool &RecL0used,
+                                                 bool &RecL1used) {
   // This is just for sanity.
   assert(L->getHeader() && "Loop without a header?");
 
   bool Changed = false;
+  bool L0Used = false;
+  bool L1Used = false;
+
   // Process nested loops first.
-  for (MachineLoop::iterator I = L->begin(), E = L->end(); I != E; ++I)
-    Changed |= convertToHardwareLoop(*I);
+  for (MachineLoop::iterator I = L->begin(), E = L->end(); I != E; ++I) {
+    Changed |= convertToHardwareLoop(*I, RecL0used, RecL1used);
+    L0Used |= RecL0used;
+    L1Used |= RecL1used;
+  }
 
   // If a nested loop has been converted, then we can't convert this loop.
-  if (Changed)
+  if (Changed && L0Used && L1Used)
     return Changed;
 
+  unsigned LOOP_i;
+  unsigned LOOP_r;
+  unsigned ENDLOOP;
+
+  // Flag used to track loopN instruction:
+  // 1 - Hardware loop is being generated for the inner most loop.
+  // 0 - Hardware loop is being generated for the outer loop.
+  unsigned IsInnerHWLoop = 1;
+
+  if (L0Used) {
+    LOOP_i = Hexagon::J2_loop1i;
+    LOOP_r = Hexagon::J2_loop1r;
+    ENDLOOP = Hexagon::ENDLOOP1;
+    IsInnerHWLoop = 0;
+  } else {
+    LOOP_i = Hexagon::J2_loop0i;
+    LOOP_r = Hexagon::J2_loop0r;
+    ENDLOOP = Hexagon::ENDLOOP0;
+  }
+
 #ifndef NDEBUG
   // Stop trying after reaching the limit (if any).
   int Limit = HWLoopLimit;
@@ -1000,14 +1133,10 @@ bool HexagonHardwareLoops::convertToHardwareLoop(MachineLoop *L) {
 #endif
 
   // Does the loop contain any invalid instructions?
-  if (containsInvalidInstruction(L))
+  if (containsInvalidInstruction(L, IsInnerHWLoop))
     return false;
 
-  // Is the induction variable bump feeding the latch condition?
-  if (!fixupInductionVariable(L))
-    return false;
-
-  MachineBasicBlock *LastMBB = L->getExitingBlock();
+  MachineBasicBlock *LastMBB = getExitingBlock(L);
   // Don't generate hw loop if the loop has more than one exit.
   if (!LastMBB)
     return false;
@@ -1016,16 +1145,19 @@ bool HexagonHardwareLoops::convertToHardwareLoop(MachineLoop *L) {
   if (LastI == LastMBB->end())
     return false;
 
+  // Is the induction variable bump feeding the latch condition?
+  if (!fixupInductionVariable(L))
+    return false;
+
   // Ensure the loop has a preheader: the loop instruction will be
   // placed there.
-  bool NewPreheader = false;
   MachineBasicBlock *Preheader = L->getLoopPreheader();
   if (!Preheader) {
     Preheader = createPreheaderForLoop(L);
     if (!Preheader)
       return false;
-    NewPreheader = true;
   }
+
   MachineBasicBlock::iterator InsertPos = Preheader->getFirstTerminator();
 
   SmallVector<MachineInstr*, 2> OldInsts;
@@ -1040,31 +1172,30 @@ bool HexagonHardwareLoops::convertToHardwareLoop(MachineLoop *L) {
     // so make sure that the register is actually defined at that point.
     MachineInstr *TCDef = MRI->getVRegDef(TripCount->getReg());
     MachineBasicBlock *BBDef = TCDef->getParent();
-    if (!NewPreheader) {
-      if (!MDT->dominates(BBDef, Preheader))
-        return false;
-    } else {
-      // If we have just created a preheader, the dominator tree won't be
-      // aware of it.  Check if the definition of the register dominates
-      // the header, but is not the header itself.
-      if (!MDT->properlyDominates(BBDef, L->getHeader()))
-        return false;
-    }
+    if (!MDT->dominates(BBDef, Preheader))
+      return false;
   }
 
   // Determine the loop start.
-  MachineBasicBlock *LoopStart = L->getTopBlock();
-  if (L->getLoopLatch() != LastMBB) {
-    // When the exit and latch are not the same, use the latch block as the
-    // start.
-    // The loop start address is used only after the 1st iteration, and the
-    // loop latch may contains instrs. that need to be executed after the
-    // first iteration.
-    LoopStart = L->getLoopLatch();
-    // Make sure the latch is a successor of the exit, otherwise it won't work.
-    if (!LastMBB->isSuccessor(LoopStart))
+  MachineBasicBlock *TopBlock = L->getTopBlock();
+  MachineBasicBlock *ExitingBlock = getExitingBlock(L);
+  MachineBasicBlock *LoopStart = 0;
+  if (ExitingBlock !=  L->getLoopLatch()) {
+    MachineBasicBlock *TB = 0, *FB = 0;
+    SmallVector<MachineOperand, 2> Cond;
+
+    if (TII->AnalyzeBranch(*ExitingBlock, TB, FB, Cond, false))
+      return false;
+
+    if (L->contains(TB))
+      LoopStart = TB;
+    else if (L->contains(FB))
+      LoopStart = FB;
+    else
       return false;
   }
+  else
+    LoopStart = TopBlock;
 
   // Convert the loop to a hardware loop.
   DEBUG(dbgs() << "Change to hardware loop at "; L->dump());
@@ -1078,8 +1209,7 @@ bool HexagonHardwareLoops::convertToHardwareLoop(MachineLoop *L) {
     BuildMI(*Preheader, InsertPos, DL, TII->get(TargetOpcode::COPY), CountReg)
       .addReg(TripCount->getReg(), 0, TripCount->getSubReg());
     // Add the Loop instruction to the beginning of the loop.
-    BuildMI(*Preheader, InsertPos, DL, TII->get(Hexagon::J2_loop0r))
-      .addMBB(LoopStart)
+    BuildMI(*Preheader, InsertPos, DL, TII->get(LOOP_r)).addMBB(LoopStart)
       .addReg(CountReg);
   } else {
     assert(TripCount->isImm() && "Expecting immediate value for trip count");
@@ -1087,14 +1217,14 @@ bool HexagonHardwareLoops::convertToHardwareLoop(MachineLoop *L) {
     // if the immediate fits in the instructions.  Otherwise, we need to
     // create a new virtual register.
     int64_t CountImm = TripCount->getImm();
-    if (!TII->isValidOffset(Hexagon::J2_loop0i, CountImm)) {
+    if (!TII->isValidOffset(LOOP_i, CountImm)) {
       unsigned CountReg = MRI->createVirtualRegister(&Hexagon::IntRegsRegClass);
       BuildMI(*Preheader, InsertPos, DL, TII->get(Hexagon::A2_tfrsi), CountReg)
         .addImm(CountImm);
-      BuildMI(*Preheader, InsertPos, DL, TII->get(Hexagon::J2_loop0r))
+      BuildMI(*Preheader, InsertPos, DL, TII->get(LOOP_r))
         .addMBB(LoopStart).addReg(CountReg);
     } else
-      BuildMI(*Preheader, InsertPos, DL, TII->get(Hexagon::J2_loop0i))
+      BuildMI(*Preheader, InsertPos, DL, TII->get(LOOP_i))
         .addMBB(LoopStart).addImm(CountImm);
   }
 
@@ -1108,8 +1238,7 @@ bool HexagonHardwareLoops::convertToHardwareLoop(MachineLoop *L) {
 
   // Replace the loop branch with an endloop instruction.
   DebugLoc LastIDL = LastI->getDebugLoc();
-  BuildMI(*LastMBB, LastI, LastIDL,
-          TII->get(Hexagon::ENDLOOP0)).addMBB(LoopStart);
+  BuildMI(*LastMBB, LastI, LastIDL, TII->get(ENDLOOP)).addMBB(LoopStart);
 
   // The loop ends with either:
   //  - a conditional branch followed by an unconditional branch, or
@@ -1137,10 +1266,18 @@ bool HexagonHardwareLoops::convertToHardwareLoop(MachineLoop *L) {
     removeIfDead(OldInsts[i]);
 
   ++NumHWLoops;
+
+  // Set RecL1used and RecL0used only after hardware loop has been
+  // successfully generated. Doing it earlier can cause wrong loop instruction
+  // to be used.
+  if (L0Used) // Loop0 was already used. So, the correct loop must be loop1.
+    RecL1used = true;
+  else
+    RecL0used = true;
+
   return true;
 }
 
-
 bool HexagonHardwareLoops::orderBumpCompare(MachineInstr *BumpI,
                                             MachineInstr *CmpI) {
   assert (BumpI != CmpI && "Bump and compare in the same instruction?");
@@ -1181,35 +1318,226 @@ bool HexagonHardwareLoops::orderBumpCompare(MachineInstr *BumpI,
   return FoundBump;
 }
 
+/// This function is required to break recursion. Visiting phis in a loop may
+/// result in recursion during compilation. We break the recursion by making
+/// sure that we visit a MachineOperand and its definition in a
+/// MachineInstruction only once. If we attempt to visit more than once, then
+/// there is recursion, and will return false.
+bool HexagonHardwareLoops::isLoopFeeder(MachineLoop *L, MachineBasicBlock *A,
+                                        MachineInstr *MI,
+                                        const MachineOperand *MO,
+                                        LoopFeederMap &LoopFeederPhi) const {
+  if (LoopFeederPhi.find(MO->getReg()) == LoopFeederPhi.end()) {
+    const std::vector<MachineBasicBlock *> &Blocks = L->getBlocks();
+    DEBUG(dbgs() << "\nhw_loop head, BB#" << Blocks[0]->getNumber(););
+    // Ignore all BBs that form Loop.
+    for (unsigned i = 0, e = Blocks.size(); i != e; ++i) {
+      MachineBasicBlock *MBB = Blocks[i];
+      if (A == MBB)
+        return false;
+    }
+    MachineInstr *Def = MRI->getVRegDef(MO->getReg());
+    LoopFeederPhi.insert(std::make_pair(MO->getReg(), Def));
+    return true;
+  } else
+    // Already visited node.
+    return false;
+}
+
+/// Return true if a Phi may generate a value that can underflow.
+/// This function calls loopCountMayWrapOrUnderFlow for each Phi operand.
+bool HexagonHardwareLoops::phiMayWrapOrUnderflow(
+    MachineInstr *Phi, const MachineOperand *EndVal, MachineBasicBlock *MBB,
+    MachineLoop *L, LoopFeederMap &LoopFeederPhi) const {
+  assert(Phi->isPHI() && "Expecting a Phi.");
+  // Walk through each Phi, and its used operands. Make sure that
+  // if there is recursion in Phi, we won't generate hardware loops.
+  for (int i = 1, n = Phi->getNumOperands(); i < n; i += 2)
+    if (isLoopFeeder(L, MBB, Phi, &(Phi->getOperand(i)), LoopFeederPhi))
+      if (loopCountMayWrapOrUnderFlow(&(Phi->getOperand(i)), EndVal,
+                                      Phi->getParent(), L, LoopFeederPhi))
+        return true;
+  return false;
+}
+
+/// Return true if the induction variable can underflow in the first iteration.
+/// An example, is an initial unsigned value that is 0 and is decrement in the
+/// first itertion of a do-while loop.  In this case, we cannot generate a
+/// hardware loop because the endloop instruction does not decrement the loop
+/// counter if it is <= 1. We only need to perform this analysis if the
+/// initial value is a register.
+///
+/// This function assumes the initial value may underfow unless proven
+/// otherwise. If the type is signed, then we don't care because signed
+/// underflow is undefined. We attempt to prove the initial value is not
+/// zero by perfoming a crude analysis of the loop counter. This function
+/// checks if the initial value is used in any comparison prior to the loop
+/// and, if so, assumes the comparison is a range check. This is inexact,
+/// but will catch the simple cases.
+bool HexagonHardwareLoops::loopCountMayWrapOrUnderFlow(
+    const MachineOperand *InitVal, const MachineOperand *EndVal,
+    MachineBasicBlock *MBB, MachineLoop *L,
+    LoopFeederMap &LoopFeederPhi) const {
+  // Only check register values since they are unknown.
+  if (!InitVal->isReg())
+    return false;
+
+  if (!EndVal->isImm())
+    return false;
+
+  // A register value that is assigned an immediate is a known value, and it
+  // won't underflow in the first iteration.
+  int64_t Imm;
+  if (checkForImmediate(*InitVal, Imm))
+    return (EndVal->getImm() == Imm);
+
+  unsigned Reg = InitVal->getReg();
+
+  // We don't know the value of a physical register.
+  if (!TargetRegisterInfo::isVirtualRegister(Reg))
+    return true;
+
+  MachineInstr *Def = MRI->getVRegDef(Reg);
+  if (!Def)
+    return true;
 
-MachineInstr *HexagonHardwareLoops::defWithImmediate(unsigned R) {
+  // If the initial value is a Phi or copy and the operands may not underflow,
+  // then the definition cannot be underflow either.
+  if (Def->isPHI() && !phiMayWrapOrUnderflow(Def, EndVal, Def->getParent(),
+                                             L, LoopFeederPhi))
+    return false;
+  if (Def->isCopy() && !loopCountMayWrapOrUnderFlow(&(Def->getOperand(1)),
+                                                    EndVal, Def->getParent(),
+                                                    L, LoopFeederPhi))
+    return false;
+
+  // Iterate over the uses of the initial value. If the initial value is used
+  // in a compare, then we assume this is a range check that ensures the loop
+  // doesn't underflow. This is not an exact test and should be improved.
+  for (MachineRegisterInfo::use_instr_nodbg_iterator I = MRI->use_instr_nodbg_begin(Reg),
+         E = MRI->use_instr_nodbg_end(); I != E; ++I) {
+    MachineInstr *MI = &*I;
+    unsigned CmpReg1 = 0, CmpReg2 = 0;
+    int CmpMask = 0, CmpValue = 0;
+
+    if (!TII->analyzeCompare(MI, CmpReg1, CmpReg2, CmpMask, CmpValue))
+      continue;
+
+    MachineBasicBlock *TBB = 0, *FBB = 0;
+    SmallVector<MachineOperand, 2> Cond;
+    if (TII->AnalyzeBranch(*MI->getParent(), TBB, FBB, Cond, false))
+      continue;
+
+    Comparison::Kind Cmp = getComparisonKind(MI->getOpcode(), 0, 0, 0);
+    if (Cmp == 0)
+      continue;
+    if (TII->predOpcodeHasNot(Cond) ^ (TBB != MBB))
+      Cmp = Comparison::getNegatedComparison(Cmp);
+    if (CmpReg2 != 0 && CmpReg2 == Reg)
+      Cmp = Comparison::getSwappedComparison(Cmp);
+
+    // Signed underflow is undefined.
+    if (Comparison::isSigned(Cmp))
+      return false;
+
+    // Check if there is a comparison of the inital value. If the initial value
+    // is greater than or not equal to another value, then assume this is a
+    // range check.
+    if ((Cmp & Comparison::G) || Cmp == Comparison::NE)
+      return false;
+  }
+
+  // OK - this is a hack that needs to be improved. We really need to analyze
+  // the instructions performed on the initial value. This works on the simplest
+  // cases only.
+  if (!Def->isCopy() && !Def->isPHI())
+    return false;
+
+  return true;
+}
+
+bool HexagonHardwareLoops::checkForImmediate(const MachineOperand &MO,
+                                             int64_t &Val) const {
+  if (MO.isImm()) {
+    Val = MO.getImm();
+    return true;
+  }
+  if (!MO.isReg())
+    return false;
+
+  // MO is a register. Check whether it is defined as an immediate value,
+  // and if so, get the value of it in TV. That value will then need to be
+  // processed to handle potential subregisters in MO.
+  int64_t TV;
+
+  unsigned R = MO.getReg();
+  if (!TargetRegisterInfo::isVirtualRegister(R))
+    return false;
   MachineInstr *DI = MRI->getVRegDef(R);
   unsigned DOpc = DI->getOpcode();
   switch (DOpc) {
+    case TargetOpcode::COPY:
     case Hexagon::A2_tfrsi:
     case Hexagon::A2_tfrpi:
     case Hexagon::CONST32_Int_Real:
-    case Hexagon::CONST64_Int_Real:
-      return DI;
-  }
-  return nullptr;
-}
+    case Hexagon::CONST64_Int_Real: {
+      // Call recursively to avoid an extra check whether operand(1) is
+      // indeed an immediate (it could be a global address, for example),
+      // plus we can handle COPY at the same time.
+      if (!checkForImmediate(DI->getOperand(1), TV))
+        return false;
+      break;
+    }
+    case Hexagon::A2_combineii:
+    case Hexagon::A4_combineir:
+    case Hexagon::A4_combineii:
+    case Hexagon::A4_combineri:
+    case Hexagon::A2_combinew: {
+      const MachineOperand &S1 = DI->getOperand(1);
+      const MachineOperand &S2 = DI->getOperand(2);
+      int64_t V1, V2;
+      if (!checkForImmediate(S1, V1) || !checkForImmediate(S2, V2))
+        return false;
+      TV = V2 | (V1 << 32);
+      break;
+    }
+    case TargetOpcode::REG_SEQUENCE: {
+      const MachineOperand &S1 = DI->getOperand(1);
+      const MachineOperand &S3 = DI->getOperand(3);
+      int64_t V1, V3;
+      if (!checkForImmediate(S1, V1) || !checkForImmediate(S3, V3))
+        return false;
+      unsigned Sub2 = DI->getOperand(2).getImm();
+      unsigned Sub4 = DI->getOperand(4).getImm();
+      if (Sub2 == Hexagon::subreg_loreg && Sub4 == Hexagon::subreg_hireg)
+        TV = V1 | (V3 << 32);
+      else if (Sub2 == Hexagon::subreg_hireg && Sub4 == Hexagon::subreg_loreg)
+        TV = V3 | (V1 << 32);
+      else
+        llvm_unreachable("Unexpected form of REG_SEQUENCE");
+      break;
+    }
 
+    default:
+      return false;
+  }
 
-int64_t HexagonHardwareLoops::getImmediate(MachineOperand &MO) {
-  if (MO.isImm())
-    return MO.getImm();
-  assert(MO.isReg());
-  unsigned R = MO.getReg();
-  MachineInstr *DI = defWithImmediate(R);
-  assert(DI && "Need an immediate operand");
-  // All currently supported "define-with-immediate" instructions have the
-  // actual immediate value in the operand(1).
-  int64_t v = DI->getOperand(1).getImm();
-  return v;
+  // By now, we should have successfuly obtained the immediate value defining
+  // the register referenced in MO. Handle a potential use of a subregister.
+  switch (MO.getSubReg()) {
+    case Hexagon::subreg_loreg:
+      Val = TV & 0xFFFFFFFFULL;
+      break;
+    case Hexagon::subreg_hireg:
+      Val = (TV >> 32) & 0xFFFFFFFFULL;
+      break;
+    default:
+      Val = TV;
+      break;
+  }
+  return true;
 }
 
-
 void HexagonHardwareLoops::setImmediate(MachineOperand &MO, int64_t Val) {
   if (MO.isImm()) {
     MO.setImm(Val);
@@ -1218,30 +1546,32 @@ void HexagonHardwareLoops::setImmediate(MachineOperand &MO, int64_t Val) {
 
   assert(MO.isReg());
   unsigned R = MO.getReg();
-  MachineInstr *DI = defWithImmediate(R);
-  if (MRI->hasOneNonDBGUse(R)) {
-    // If R has only one use, then just change its defining instruction to
-    // the new immediate value.
-    DI->getOperand(1).setImm(Val);
-    return;
-  }
+  MachineInstr *DI = MRI->getVRegDef(R);
 
   const TargetRegisterClass *RC = MRI->getRegClass(R);
   unsigned NewR = MRI->createVirtualRegister(RC);
   MachineBasicBlock &B = *DI->getParent();
   DebugLoc DL = DI->getDebugLoc();
-  BuildMI(B, DI, DL, TII->get(DI->getOpcode()), NewR)
-    .addImm(Val);
+  BuildMI(B, DI, DL, TII->get(DI->getOpcode()), NewR).addImm(Val);
   MO.setReg(NewR);
 }
 
+static bool isImmValidForOpcode(unsigned CmpOpc, int64_t Imm) {
+  // These two instructions are not extendable.
+  if (CmpOpc == Hexagon::A4_cmpbeqi)
+    return isUInt<8>(Imm);
+  if (CmpOpc == Hexagon::A4_cmpbgti)
+    return isInt<8>(Imm);
+  // The rest of the comparison-with-immediate instructions are extendable.
+  return true;
+}
 
 bool HexagonHardwareLoops::fixupInductionVariable(MachineLoop *L) {
   MachineBasicBlock *Header = L->getHeader();
-  MachineBasicBlock *Preheader = L->getLoopPreheader();
   MachineBasicBlock *Latch = L->getLoopLatch();
+  MachineBasicBlock *ExitingBlock = getExitingBlock(L);
 
-  if (!Header || !Preheader || !Latch)
+  if (!(Header && Latch && ExitingBlock))
     return false;
 
   // These data structures follow the same concept as the corresponding
@@ -1269,15 +1599,16 @@ bool HexagonHardwareLoops::fixupInductionVariable(MachineLoop *L) {
       unsigned PhiReg = Phi->getOperand(i).getReg();
       MachineInstr *DI = MRI->getVRegDef(PhiReg);
       unsigned UpdOpc = DI->getOpcode();
-      bool isAdd = (UpdOpc == Hexagon::A2_addi);
+      bool isAdd = (UpdOpc == Hexagon::A2_addi || UpdOpc == Hexagon::A2_addp);
 
       if (isAdd) {
         // If the register operand to the add/sub is the PHI we are looking
         // at, this meets the induction pattern.
         unsigned IndReg = DI->getOperand(1).getReg();
-        if (MRI->getVRegDef(IndReg) == Phi) {
+        MachineOperand &Opnd2 = DI->getOperand(2);
+        int64_t V;
+        if (MRI->getVRegDef(IndReg) == Phi && checkForImmediate(Opnd2, V)) {
           unsigned UpdReg = DI->getOperand(0).getReg();
-          int64_t V = DI->getOperand(2).getImm();
           IndRegs.insert(std::make_pair(UpdReg, std::make_pair(IndReg, V)));
         }
       }
@@ -1290,17 +1621,38 @@ bool HexagonHardwareLoops::fixupInductionVariable(MachineLoop *L) {
   MachineBasicBlock *TB = nullptr, *FB = nullptr;
   SmallVector<MachineOperand,2> Cond;
   // AnalyzeBranch returns true if it fails to analyze branch.
-  bool NotAnalyzed = TII->AnalyzeBranch(*Latch, TB, FB, Cond, false);
-  if (NotAnalyzed)
+  bool NotAnalyzed = TII->AnalyzeBranch(*ExitingBlock, TB, FB, Cond, false);
+  if (NotAnalyzed || Cond.empty())
     return false;
 
-  // Check if the latch branch is unconditional.
-  if (Cond.empty())
-    return false;
+  if (ExitingBlock != Latch && (TB == Latch || FB == Latch)) {
+    MachineBasicBlock *LTB = 0, *LFB = 0;
+    SmallVector<MachineOperand,2> LCond;
+    bool NotAnalyzed = TII->AnalyzeBranch(*Latch, LTB, LFB, LCond, false);
+    if (NotAnalyzed)
+      return false;
 
-  if (TB != Header && FB != Header)
-    // The latch does not go back to the header.  Not a latch we know and love.
-    return false;
+    // Since latch is not the exiting block, the latch branch should be an
+    // unconditional branch to the loop header.
+    if (TB == Latch)
+      TB = (LTB == Header) ? LTB : LFB;
+    else
+      FB = (LTB == Header) ? LTB : LFB;
+  }
+  if (TB != Header) {
+    if (FB != Header) {
+      // The latch/exit block does not go back to the header.
+      return false;
+    }
+    // FB is the header (i.e., uncond. jump to branch header)
+    // In this case, the LoopBody -> TB should not be a back edge otherwise
+    // it could result in an infinite loop after conversion to hw_loop.
+    // This case can happen when the Latch has two jumps like this:
+    // Jmp_c OuterLoopHeader <-- TB
+    // Jmp   InnerLoopHeader <-- FB
+    if (MDT->dominates(TB, FB))
+      return false;
+  }
 
   // Expecting a predicate register as a condition.  It won't be a hardware
   // predicate register at this point yet, just a vreg.
@@ -1311,6 +1663,9 @@ bool HexagonHardwareLoops::fixupInductionVariable(MachineLoop *L) {
   if (CSz != 1 && CSz != 2)
     return false;
 
+  if (!Cond[CSz-1].isReg())
+    return false;
+
   unsigned P = Cond[CSz-1].getReg();
   MachineInstr *PredDef = MRI->getVRegDef(P);
 
@@ -1332,8 +1687,7 @@ bool HexagonHardwareLoops::fixupInductionVariable(MachineLoop *L) {
       if (MO.isImplicit())
         continue;
       if (MO.isUse()) {
-        unsigned R = MO.getReg();
-        if (!defWithImmediate(R)) {
+        if (!isImmediate(MO)) {
           CmpRegs.insert(MO.getReg());
           continue;
         }
@@ -1366,20 +1720,70 @@ bool HexagonHardwareLoops::fixupInductionVariable(MachineLoop *L) {
     // compared against an immediate, we can fix it.
     const RegisterBump &RB = I->second;
     if (CmpRegs.count(RB.first)) {
-      if (!CmpImmOp)
+      if (!CmpImmOp) {
+        // If both operands to the compare instruction are registers, see if
+        // it can be changed to use induction register as one of the operands.
+        MachineInstr *IndI = nullptr;
+        MachineInstr *nonIndI = nullptr;
+        MachineOperand *IndMO = nullptr;
+        MachineOperand *nonIndMO = nullptr;
+
+        for (unsigned i = 1, n = PredDef->getNumOperands(); i < n; ++i) {
+          MachineOperand &MO = PredDef->getOperand(i);
+          if (MO.isReg() && MO.getReg() == RB.first) {
+            DEBUG(dbgs() << "\n DefMI(" << i << ") = "
+                         << *(MRI->getVRegDef(I->first)));
+            if (IndI)
+              return false;
+
+            IndI = MRI->getVRegDef(I->first);
+            IndMO = &MO;
+          } else if (MO.isReg()) {
+            DEBUG(dbgs() << "\n DefMI(" << i << ") = "
+                         << *(MRI->getVRegDef(MO.getReg())));
+            if (nonIndI)
+              return false;
+
+            nonIndI = MRI->getVRegDef(MO.getReg());
+            nonIndMO = &MO;
+          }
+        }
+        if (IndI && nonIndI &&
+            nonIndI->getOpcode() == Hexagon::A2_addi &&
+            nonIndI->getOperand(2).isImm() &&
+            nonIndI->getOperand(2).getImm() == - RB.second) {
+          bool Order = orderBumpCompare(IndI, PredDef);
+          if (Order) {
+            IndMO->setReg(I->first);
+            nonIndMO->setReg(nonIndI->getOperand(1).getReg());
+            return true;
+          }
+        }
         return false;
+      }
 
+      // It is not valid to do this transformation on an unsigned comparison
+      // because it may underflow.
+      Comparison::Kind Cmp = getComparisonKind(PredDef->getOpcode(), 0, 0, 0);
+      if (!Cmp || Comparison::isUnsigned(Cmp))
+        return false;
+
+      // If the register is being compared against an immediate, try changing
+      // the compare instruction to use induction register and adjust the
+      // immediate operand.
       int64_t CmpImm = getImmediate(*CmpImmOp);
       int64_t V = RB.second;
-      if (V > 0 && CmpImm+V < CmpImm)  // Overflow (64-bit).
-        return false;
-      if (V < 0 && CmpImm+V > CmpImm)  // Overflow (64-bit).
+      // Handle Overflow (64-bit).
+      if (((V > 0) && (CmpImm > INT64_MAX - V)) ||
+          ((V < 0) && (CmpImm < INT64_MIN - V)))
         return false;
       CmpImm += V;
-      // Some forms of cmp-immediate allow u9 and s10.  Assume the worst case
-      // scenario, i.e. an 8-bit value.
-      if (CmpImmOp->isImm() && !isInt<8>(CmpImm))
-        return false;
+      // Most comparisons of register against an immediate value allow
+      // the immediate to be constant-extended. There are some exceptions
+      // though. Make sure the new combination will work.
+      if (CmpImmOp->isImm())
+        if (!isImmValidForOpcode(PredDef->getOpcode(), CmpImm))
+          return false;
 
       // Make sure that the compare happens after the bump.  Otherwise,
       // after the fixup, the compare would use a yet-undefined register.
@@ -1403,19 +1807,27 @@ bool HexagonHardwareLoops::fixupInductionVariable(MachineLoop *L) {
   return false;
 }
 
-
 /// \brief Create a preheader for a given loop.
 MachineBasicBlock *HexagonHardwareLoops::createPreheaderForLoop(
       MachineLoop *L) {
   if (MachineBasicBlock *TmpPH = L->getLoopPreheader())
     return TmpPH;
 
+  if (!HWCreatePreheader)
+    return nullptr;
+
   MachineBasicBlock *Header = L->getHeader();
   MachineBasicBlock *Latch = L->getLoopLatch();
+  MachineBasicBlock *ExitingBlock = getExitingBlock(L);
   MachineFunction *MF = Header->getParent();
   DebugLoc DL;
 
-  if (!Latch || Header->hasAddressTaken())
+#ifndef NDEBUG
+  if ((PHFn != "") && (PHFn != MF->getName()))
+    return nullptr;
+#endif
+
+  if (!Latch || !ExitingBlock || Header->hasAddressTaken())
     return nullptr;
 
   typedef MachineBasicBlock::instr_iterator instr_iterator;
@@ -1427,16 +1839,14 @@ MachineBasicBlock *HexagonHardwareLoops::createPreheaderForLoop(
   SmallVector<MachineOperand,2> Tmp1;
   MachineBasicBlock *TB = nullptr, *FB = nullptr;
 
-  if (TII->AnalyzeBranch(*Latch, TB, FB, Tmp1, false))
+  if (TII->AnalyzeBranch(*ExitingBlock, TB, FB, Tmp1, false))
     return nullptr;
 
   for (MBBVector::iterator I = Preds.begin(), E = Preds.end(); I != E; ++I) {
     MachineBasicBlock *PB = *I;
-    if (PB != Latch) {
-      bool NotAnalyzed = TII->AnalyzeBranch(*PB, TB, FB, Tmp1, false);
-      if (NotAnalyzed)
-        return nullptr;
-    }
+    bool NotAnalyzed = TII->AnalyzeBranch(*PB, TB, FB, Tmp1, false);
+    if (NotAnalyzed)
+      return nullptr;
   }
 
   MachineBasicBlock *NewPH = MF->CreateMachineBasicBlock();
@@ -1445,7 +1855,7 @@ MachineBasicBlock *HexagonHardwareLoops::createPreheaderForLoop(
   if (Header->pred_size() > 2) {
     // Ensure that the header has only two predecessors: the preheader and
     // the loop latch.  Any additional predecessors of the header should
-    // join at the newly created preheader.  Inspect all PHI nodes from the
+    // join at the newly created preheader. Inspect all PHI nodes from the
     // header and create appropriate corresponding PHI nodes in the preheader.
 
     for (instr_iterator I = Header->instr_begin(), E = Header->instr_end();
@@ -1465,11 +1875,14 @@ MachineBasicBlock *HexagonHardwareLoops::createPreheaderForLoop(
       // created PHI node in the preheader.
       for (unsigned i = 1, n = PN->getNumOperands(); i < n; i += 2) {
         unsigned PredR = PN->getOperand(i).getReg();
+        unsigned PredRSub = PN->getOperand(i).getSubReg();
         MachineBasicBlock *PredB = PN->getOperand(i+1).getMBB();
         if (PredB == Latch)
           continue;
 
-        NewPN->addOperand(MachineOperand::CreateReg(PredR, false));
+        MachineOperand MO = MachineOperand::CreateReg(PredR, false);
+        MO.setSubReg(PredRSub);
+        NewPN->addOperand(MO);
         NewPN->addOperand(MachineOperand::CreateMBB(PredB));
       }
 
@@ -1539,5 +1952,16 @@ MachineBasicBlock *HexagonHardwareLoops::createPreheaderForLoop(
   TII->InsertBranch(*NewPH, Header, nullptr, EmptyCond, DL);
   NewPH->addSuccessor(Header);
 
+  MachineLoop *ParentLoop = L->getParentLoop();
+  if (ParentLoop)
+    ParentLoop->addBasicBlockToLoop(NewPH, MLI->getBase());
+
+  // Update the dominator information with the new preheader.
+  if (MDT) {
+    MachineDomTreeNode *HDom = MDT->getNode(Header);
+    MDT->addNewBlock(NewPH, HDom->getIDom()->getBlock());
+    MDT->changeImmediateDominator(Header, NewPH);
+  }
+
   return NewPH;
 }
diff --git a/lib/Target/Hexagon/HexagonISelDAGToDAG.cpp b/lib/Target/Hexagon/HexagonISelDAGToDAG.cpp
index aaccac8b57a..523d6eb0c2c 100644
--- a/lib/Target/Hexagon/HexagonISelDAGToDAG.cpp
+++ b/lib/Target/Hexagon/HexagonISelDAGToDAG.cpp
@@ -13,8 +13,11 @@
 
 #include "Hexagon.h"
 #include "HexagonISelLowering.h"
+#include "HexagonMachineFunctionInfo.h"
 #include "HexagonTargetMachine.h"
 #include "llvm/ADT/DenseMap.h"
+#include "llvm/CodeGen/FunctionLoweringInfo.h"
+#include "llvm/CodeGen/MachineInstrBuilder.h"
 #include "llvm/CodeGen/SelectionDAGISel.h"
 #include "llvm/IR/Intrinsics.h"
 #include "llvm/Support/CommandLine.h"
@@ -62,6 +65,7 @@ class HexagonDAGToDAGISel : public SelectionDAGISel {
   }
 
   virtual void PreprocessISelDAG() override;
+  virtual void EmitFunctionEntryCode() override;
 
   SDNode *Select(SDNode *N) override;
 
@@ -100,28 +104,29 @@ class HexagonDAGToDAGISel : public SelectionDAGISel {
   SDNode *SelectConstantFP(SDNode *N);
   SDNode *SelectAdd(SDNode *N);
   SDNode *SelectBitOp(SDNode *N);
+  bool isConstExtProfitable(SDNode *N) const;
 
   // XformMskToBitPosU5Imm - Returns the bit position which
   // the single bit 32 bit mask represents.
   // Used in Clr and Set bit immediate memops.
-  SDValue XformMskToBitPosU5Imm(uint32_t Imm) {
+  SDValue XformMskToBitPosU5Imm(uint32_t Imm, SDLoc DL) {
     int32_t bitPos;
     bitPos = Log2_32(Imm);
     assert(bitPos >= 0 && bitPos < 32 &&
            "Constant out of range for 32 BitPos Memops");
-    return CurDAG->getTargetConstant(bitPos, MVT::i32);
+    return CurDAG->getTargetConstant(bitPos, DL, MVT::i32);
   }
 
   // XformMskToBitPosU4Imm - Returns the bit position which the single-bit
   // 16 bit mask represents. Used in Clr and Set bit immediate memops.
-  SDValue XformMskToBitPosU4Imm(uint16_t Imm) {
-    return XformMskToBitPosU5Imm(Imm);
+  SDValue XformMskToBitPosU4Imm(uint16_t Imm, SDLoc DL) {
+    return XformMskToBitPosU5Imm(Imm, DL);
   }
 
   // XformMskToBitPosU3Imm - Returns the bit position which the single-bit
   // 8 bit mask represents. Used in Clr and Set bit immediate memops.
-  SDValue XformMskToBitPosU3Imm(uint8_t Imm) {
-    return XformMskToBitPosU5Imm(Imm);
+  SDValue XformMskToBitPosU3Imm(uint8_t Imm, SDLoc DL) {
+    return XformMskToBitPosU5Imm(Imm, DL);
   }
 
   // Return true if there is exactly one bit set in V, i.e., if V is one of the
@@ -133,37 +138,37 @@ class HexagonDAGToDAGISel : public SelectionDAGISel {
   // XformM5ToU5Imm - Return a target constant with the specified value, of
   // type i32 where the negative literal is transformed into a positive literal
   // for use in -= memops.
-  inline SDValue XformM5ToU5Imm(signed Imm) {
+  inline SDValue XformM5ToU5Imm(signed Imm, SDLoc DL) {
      assert( (Imm >= -31 && Imm <= -1)  && "Constant out of range for Memops");
-     return CurDAG->getTargetConstant( - Imm, MVT::i32);
+     return CurDAG->getTargetConstant( - Imm, DL, MVT::i32);
   }
 
   // XformU7ToU7M1Imm - Return a target constant decremented by 1, in range
   // [1..128], used in cmpb.gtu instructions.
-  inline SDValue XformU7ToU7M1Imm(signed Imm) {
+  inline SDValue XformU7ToU7M1Imm(signed Imm, SDLoc DL) {
     assert((Imm >= 1 && Imm <= 128) && "Constant out of range for cmpb op");
-    return CurDAG->getTargetConstant(Imm - 1, MVT::i8);
+    return CurDAG->getTargetConstant(Imm - 1, DL, MVT::i8);
   }
 
   // XformS8ToS8M1Imm - Return a target constant decremented by 1.
-  inline SDValue XformSToSM1Imm(signed Imm) {
-    return CurDAG->getTargetConstant(Imm - 1, MVT::i32);
+  inline SDValue XformSToSM1Imm(signed Imm, SDLoc DL) {
+    return CurDAG->getTargetConstant(Imm - 1, DL, MVT::i32);
   }
 
   // XformU8ToU8M1Imm - Return a target constant decremented by 1.
-  inline SDValue XformUToUM1Imm(unsigned Imm) {
+  inline SDValue XformUToUM1Imm(unsigned Imm, SDLoc DL) {
     assert((Imm >= 1) && "Cannot decrement unsigned int less than 1");
-    return CurDAG->getTargetConstant(Imm - 1, MVT::i32);
+    return CurDAG->getTargetConstant(Imm - 1, DL, MVT::i32);
   }
 
   // XformSToSM2Imm - Return a target constant decremented by 2.
-  inline SDValue XformSToSM2Imm(unsigned Imm) {
-    return CurDAG->getTargetConstant(Imm - 2, MVT::i32);
+  inline SDValue XformSToSM2Imm(unsigned Imm, SDLoc DL) {
+    return CurDAG->getTargetConstant(Imm - 2, DL, MVT::i32);
   }
 
   // XformSToSM3Imm - Return a target constant decremented by 3.
-  inline SDValue XformSToSM3Imm(unsigned Imm) {
-    return CurDAG->getTargetConstant(Imm - 3, MVT::i32);
+  inline SDValue XformSToSM3Imm(unsigned Imm, SDLoc DL) {
+    return CurDAG->getTargetConstant(Imm - 3, DL, MVT::i32);
   }
 
   // Include the pieces autogenerated from the target description.
@@ -255,7 +260,7 @@ SDNode *HexagonDAGToDAGISel::SelectIndexedLoadSignExtend64(LoadSDNode *LD,
 
   const HexagonInstrInfo &TII = *HST->getInstrInfo();
   if (TII.isValidAutoIncImm(LoadedVT, Val)) {
-    SDValue TargetConst = CurDAG->getTargetConstant(Val, MVT::i32);
+    SDValue TargetConst = CurDAG->getTargetConstant(Val, dl, MVT::i32);
     SDNode *Result_1 = CurDAG->getMachineNode(Opcode, dl, MVT::i32, MVT::i32,
                                               MVT::Other, Base, TargetConst,
                                               Chain);
@@ -274,8 +279,8 @@ SDNode *HexagonDAGToDAGISel::SelectIndexedLoadSignExtend64(LoadSDNode *LD,
     return Result_2;
   }
 
-  SDValue TargetConst0 = CurDAG->getTargetConstant(0, MVT::i32);
-  SDValue TargetConstVal = CurDAG->getTargetConstant(Val, MVT::i32);
+  SDValue TargetConst0 = CurDAG->getTargetConstant(0, dl, MVT::i32);
+  SDValue TargetConstVal = CurDAG->getTargetConstant(Val, dl, MVT::i32);
   SDNode *Result_1 = CurDAG->getMachineNode(Opcode, dl, MVT::i32, MVT::Other,
                                             Base, TargetConst0, Chain);
   SDNode *Result_2 = CurDAG->getMachineNode(Hexagon::A2_sxtw, dl, MVT::i64,
@@ -309,8 +314,8 @@ SDNode *HexagonDAGToDAGISel::SelectIndexedLoadZeroExtend64(LoadSDNode *LD,
 
   const HexagonInstrInfo &TII = *HST->getInstrInfo();
   if (TII.isValidAutoIncImm(LoadedVT, Val)) {
-    SDValue TargetConstVal = CurDAG->getTargetConstant(Val, MVT::i32);
-    SDValue TargetConst0 = CurDAG->getTargetConstant(0, MVT::i32);
+    SDValue TargetConstVal = CurDAG->getTargetConstant(Val, dl, MVT::i32);
+    SDValue TargetConst0 = CurDAG->getTargetConstant(0, dl, MVT::i32);
     SDNode *Result_1 = CurDAG->getMachineNode(Opcode, dl, MVT::i32,
                                               MVT::i32, MVT::Other, Base,
                                               TargetConstVal, Chain);
@@ -332,8 +337,8 @@ SDNode *HexagonDAGToDAGISel::SelectIndexedLoadZeroExtend64(LoadSDNode *LD,
   }
 
   // Generate an indirect load.
-  SDValue TargetConst0 = CurDAG->getTargetConstant(0, MVT::i32);
-  SDValue TargetConstVal = CurDAG->getTargetConstant(Val, MVT::i32);
+  SDValue TargetConst0 = CurDAG->getTargetConstant(0, dl, MVT::i32);
+  SDValue TargetConstVal = CurDAG->getTargetConstant(Val, dl, MVT::i32);
   SDNode *Result_1 = CurDAG->getMachineNode(Opcode, dl, MVT::i32,
                                             MVT::Other, Base, TargetConst0,
                                             Chain);
@@ -407,7 +412,7 @@ SDNode *HexagonDAGToDAGISel::SelectIndexedLoad(LoadSDNode *LD, SDLoc dl) {
     return SelectIndexedLoadSignExtend64(LD, Opcode, dl);
 
   if (TII.isValidAutoIncImm(LoadedVT, Val)) {
-    SDValue TargetConstVal = CurDAG->getTargetConstant(Val, MVT::i32);
+    SDValue TargetConstVal = CurDAG->getTargetConstant(Val, dl, MVT::i32);
     SDNode* Result = CurDAG->getMachineNode(Opcode, dl,
                                             LD->getValueType(0),
                                             MVT::i32, MVT::Other, Base,
@@ -426,8 +431,8 @@ SDNode *HexagonDAGToDAGISel::SelectIndexedLoad(LoadSDNode *LD, SDLoc dl) {
     ReplaceUses(Froms, Tos, 3);
     return Result;
   } else {
-    SDValue TargetConst0 = CurDAG->getTargetConstant(0, MVT::i32);
-    SDValue TargetConstVal = CurDAG->getTargetConstant(Val, MVT::i32);
+    SDValue TargetConst0 = CurDAG->getTargetConstant(0, dl, MVT::i32);
+    SDValue TargetConstVal = CurDAG->getTargetConstant(Val, dl, MVT::i32);
     SDNode* Result_1 = CurDAG->getMachineNode(Opcode, dl,
                                               LD->getValueType(0),
                                               MVT::Other, Base, TargetConst0,
@@ -498,7 +503,7 @@ SDNode *HexagonDAGToDAGISel::SelectIndexedStore(StoreSDNode *ST, SDLoc dl) {
       Value = CurDAG->getTargetExtractSubreg(Hexagon::subreg_loreg,
                                              dl, MVT::i32, Value);
     }
-    SDValue Ops[] = {Base, CurDAG->getTargetConstant(Val, MVT::i32), Value,
+    SDValue Ops[] = {Base, CurDAG->getTargetConstant(Val, dl, MVT::i32), Value,
                      Chain};
     // Build post increment store.
     SDNode* Result = CurDAG->getMachineNode(Opcode, dl, MVT::i32,
@@ -516,7 +521,7 @@ SDNode *HexagonDAGToDAGISel::SelectIndexedStore(StoreSDNode *ST, SDLoc dl) {
   // def S2_storerd_io
   //   : STInst<(outs), (ins IntRegs:$base, imm:$offset, DoubleRegs:$src1), ...
   // and it differs for POST_ST* for instance.
-  SDValue Ops[] = { Base, CurDAG->getTargetConstant(0, MVT::i32), Value,
+  SDValue Ops[] = { Base, CurDAG->getTargetConstant(0, dl, MVT::i32), Value,
                     Chain};
   unsigned Opcode = 0;
 
@@ -528,7 +533,7 @@ SDNode *HexagonDAGToDAGISel::SelectIndexedStore(StoreSDNode *ST, SDLoc dl) {
   else llvm_unreachable("unknown memory type");
 
   // Build regular store.
-  SDValue TargetConstVal = CurDAG->getTargetConstant(Val, MVT::i32);
+  SDValue TargetConstVal = CurDAG->getTargetConstant(Val, dl, MVT::i32);
   SDNode* Result_1 = CurDAG->getMachineNode(Opcode, dl, MVT::Other, Ops);
   // Build splitted incriment instruction.
   SDNode* Result_2 = CurDAG->getMachineNode(Hexagon::A2_addi, dl, MVT::i32,
@@ -595,7 +600,7 @@ SDNode *HexagonDAGToDAGISel::SelectMul(SDNode *N) {
       }
 
       SDValue Chain = LD->getChain();
-      SDValue TargetConst0 = CurDAG->getTargetConstant(0, MVT::i32);
+      SDValue TargetConst0 = CurDAG->getTargetConstant(0, dl, MVT::i32);
       OP0 = SDValue(CurDAG->getMachineNode(Hexagon::L2_loadri_io, dl, MVT::i32,
                                             MVT::Other,
                                             LD->getBasePtr(), TargetConst0,
@@ -621,7 +626,7 @@ SDNode *HexagonDAGToDAGISel::SelectMul(SDNode *N) {
       }
 
       SDValue Chain = LD->getChain();
-      SDValue TargetConst0 = CurDAG->getTargetConstant(0, MVT::i32);
+      SDValue TargetConst0 = CurDAG->getTargetConstant(0, dl, MVT::i32);
       OP1 = SDValue(CurDAG->getMachineNode(Hexagon::L2_loadri_io, dl, MVT::i32,
                                             MVT::Other,
                                             LD->getBasePtr(), TargetConst0,
@@ -657,7 +662,7 @@ SDNode *HexagonDAGToDAGISel::SelectSHL(SDNode *N) {
           int32_t MulConst =
             cast<ConstantSDNode>(Mul_1.getNode())->getSExtValue();
           int32_t ValConst = MulConst << ShlConst;
-          SDValue Val = CurDAG->getTargetConstant(ValConst,
+          SDValue Val = CurDAG->getTargetConstant(ValConst, dl,
                                                   MVT::i32);
           if (ConstantSDNode *CN = dyn_cast<ConstantSDNode>(Val.getNode()))
             if (isInt<9>(CN->getSExtValue())) {
@@ -685,7 +690,8 @@ SDNode *HexagonDAGToDAGISel::SelectSHL(SDNode *N) {
                 int32_t Shl2Const =
                   cast<ConstantSDNode>(Shl2_1.getNode())->getSExtValue();
                 int32_t ValConst = 1 << (ShlConst+Shl2Const);
-                SDValue Val = CurDAG->getTargetConstant(-ValConst, MVT::i32);
+                SDValue Val = CurDAG->getTargetConstant(-ValConst, dl,
+                                                        MVT::i32);
                 if (ConstantSDNode *CN =
                     dyn_cast<ConstantSDNode>(Val.getNode()))
                   if (isInt<9>(CN->getSExtValue())) {
@@ -734,13 +740,14 @@ SDNode *HexagonDAGToDAGISel::SelectZeroExtend(SDNode *N) {
       MV |= Bit;
       Bit <<= ES;
     }
-    SDValue Ones = CurDAG->getTargetConstant(MV, MVT::i64);
+    SDValue Ones = CurDAG->getTargetConstant(MV, dl, MVT::i64);
     SDNode *OnesReg = CurDAG->getMachineNode(Hexagon::CONST64_Int_Real, dl,
                                              MVT::i64, Ones);
     if (ExVT.getSizeInBits() == 32) {
       SDNode *And = CurDAG->getMachineNode(Hexagon::A2_andp, dl, MVT::i64,
                                            SDValue(Mask,0), SDValue(OnesReg,0));
-      SDValue SubR = CurDAG->getTargetConstant(Hexagon::subreg_loreg, MVT::i32);
+      SDValue SubR = CurDAG->getTargetConstant(Hexagon::subreg_loreg, dl,
+                                               MVT::i32);
       return CurDAG->getMachineNode(Hexagon::EXTRACT_SUBREG, dl, ExVT,
                                     SDValue(And,0), SubR);
     }
@@ -756,7 +763,7 @@ SDNode *HexagonDAGToDAGISel::SelectZeroExtend(SDNode *N) {
       // Now we need to differentiate target data types.
       if (N->getValueType(0) == MVT::i64) {
         // Convert the zero_extend to Rs = Pd followed by A2_combinew(0,Rs).
-        SDValue TargetConst0 = CurDAG->getTargetConstant(0, MVT::i32);
+        SDValue TargetConst0 = CurDAG->getTargetConstant(0, dl, MVT::i32);
         SDNode *Result_1 = CurDAG->getMachineNode(Hexagon::C2_tfrpr, dl,
                                                   MVT::i32,
                                                   SDValue(IsIntrinsic, 0));
@@ -863,7 +870,7 @@ SDNode *HexagonDAGToDAGISel::SelectIntrinsicWChain(SDNode *N) {
     Ops.push_back(Load);
     Ops.push_back(ModifierExpr);
     int32_t Val = cast<ConstantSDNode>(Offset.getNode())->getSExtValue();
-    Ops.push_back(CurDAG->getTargetConstant(Val, MVT::i32));
+    Ops.push_back(CurDAG->getTargetConstant(Val, dl, MVT::i32));
     Ops.push_back(Chain);
     SDNode* Result = CurDAG->getMachineNode(opc, dl, ResTys, Ops);
 
@@ -1018,11 +1025,11 @@ SDNode *HexagonDAGToDAGISel::SelectConstantFP(SDNode *N) {
   APFloat APF = CN->getValueAPF();
   if (N->getValueType(0) == MVT::f32) {
     return CurDAG->getMachineNode(Hexagon::TFRI_f, dl, MVT::f32,
-              CurDAG->getTargetConstantFP(APF.convertToFloat(), MVT::f32));
+              CurDAG->getTargetConstantFP(APF.convertToFloat(), dl, MVT::f32));
   }
   else if (N->getValueType(0) == MVT::f64) {
     return CurDAG->getMachineNode(Hexagon::CONST64_Float_Real, dl, MVT::f64,
-              CurDAG->getTargetConstantFP(APF.convertToDouble(), MVT::f64));
+              CurDAG->getTargetConstantFP(APF.convertToDouble(), dl, MVT::f64));
   }
 
   return SelectCode(N);
@@ -1162,7 +1169,7 @@ SDNode *HexagonDAGToDAGISel::SelectBitOp(SDNode *N) {
 
   SDNode *Result;
   // Get the right SDVal for the opcode.
-  SDValue SDVal = CurDAG->getTargetConstant(bitpos, MVT::i32);
+  SDValue SDVal = CurDAG->getTargetConstant(bitpos, dl, MVT::i32);
 
   if (ValueVT == MVT::i32 || ValueVT == MVT::f32) {
     Result = CurDAG->getMachineNode(BitOpc, dl, ValueVT,
@@ -1177,11 +1184,11 @@ SDNode *HexagonDAGToDAGISel::SelectBitOp(SDNode *N) {
 
     SDNode *Reg = N->getOperand(0).getNode();
     SDValue RegClass = CurDAG->getTargetConstant(Hexagon::DoubleRegsRegClassID,
-                                                 MVT::i64);
+                                                 dl, MVT::i64);
 
-    SDValue SubregHiIdx = CurDAG->getTargetConstant(Hexagon::subreg_hireg,
+    SDValue SubregHiIdx = CurDAG->getTargetConstant(Hexagon::subreg_hireg, dl,
                                                     MVT::i32);
-    SDValue SubregLoIdx = CurDAG->getTargetConstant(Hexagon::subreg_loreg,
+    SDValue SubregLoIdx = CurDAG->getTargetConstant(Hexagon::subreg_loreg, dl,
                                                     MVT::i32);
 
     SDValue SubregHI = CurDAG->getTargetExtractSubreg(Hexagon::subreg_hireg, dl,
@@ -1200,7 +1207,7 @@ SDNode *HexagonDAGToDAGISel::SelectBitOp(SDNode *N) {
                                       dl, ValueVT, Ops);
     } else {
       if (Opc != ISD::FABS && Opc != ISD::FNEG)
-        SDVal = CurDAG->getTargetConstant(bitpos-32, MVT::i32);
+        SDVal = CurDAG->getTargetConstant(bitpos - 32, dl, MVT::i32);
       SDNode *Result0 = CurDAG->getMachineNode(BitOpc, dl, SubValueVT,
                                                SubregHI, SDVal);
       const SDValue Ops[] = { RegClass, SDValue(Result0, 0), SubregHiIdx,
@@ -1216,12 +1223,30 @@ SDNode *HexagonDAGToDAGISel::SelectBitOp(SDNode *N) {
 
 
 SDNode *HexagonDAGToDAGISel::SelectFrameIndex(SDNode *N) {
+  MachineFrameInfo *MFI = MF->getFrameInfo();
+  const HexagonFrameLowering *HFI = HST->getFrameLowering();
   int FX = cast<FrameIndexSDNode>(N)->getIndex();
+  unsigned StkA = HFI->getStackAlignment();
+  unsigned MaxA = MFI->getMaxAlignment();
   SDValue FI = CurDAG->getTargetFrameIndex(FX, MVT::i32);
-  SDValue Zero = CurDAG->getTargetConstant(0, MVT::i32);
   SDLoc DL(N);
-
-  SDNode *R = CurDAG->getMachineNode(Hexagon::TFR_FI, DL, MVT::i32, FI, Zero);
+  SDValue Zero = CurDAG->getTargetConstant(0, DL, MVT::i32);
+  SDNode *R = 0;
+
+  // Use TFR_FI when:
+  // - the object is fixed, or
+  // - there are no objects with higher-than-default alignment, or
+  // - there are no dynamically allocated objects.
+  // Otherwise, use TFR_FIA.
+  if (FX < 0 || MaxA <= StkA || !MFI->hasVarSizedObjects()) {
+    R = CurDAG->getMachineNode(Hexagon::TFR_FI, DL, MVT::i32, FI, Zero);
+  } else {
+    auto &HMFI = *MF->getInfo<HexagonMachineFunctionInfo>();
+    unsigned AR = HMFI.getStackAlignBaseVReg();
+    SDValue CH = CurDAG->getEntryNode();
+    SDValue Ops[] = { CurDAG->getCopyFromReg(CH, DL, AR, MVT::i32), FI, Zero };
+    R = CurDAG->getMachineNode(Hexagon::TFR_FIA, DL, MVT::i32, Ops);
+  }
 
   if (N->getHasDebugValue())
     CurDAG->TransferDbgValues(SDValue(N, 0), SDValue(R, 0));
@@ -1280,7 +1305,6 @@ SDNode *HexagonDAGToDAGISel::Select(SDNode *N) {
   return SelectCode(N);
 }
 
-
 bool HexagonDAGToDAGISel::
 SelectInlineAsmMemoryOperand(const SDValue &Op, unsigned ConstraintID,
                              std::vector<SDValue> &OutOps) {
@@ -1300,15 +1324,29 @@ SelectInlineAsmMemoryOperand(const SDValue &Op, unsigned ConstraintID,
     break;
   }
 
-  OutOps.push_back(CurDAG->getTargetConstant(0, MVT::i32));
+  OutOps.push_back(CurDAG->getTargetConstant(0, SDLoc(Op), MVT::i32));
   return false;
 }
 
+bool HexagonDAGToDAGISel::isConstExtProfitable(SDNode *N) const {
+  unsigned UseCount = 0;
+  unsigned CallCount = 0;
+  for (SDNode::use_iterator I = N->use_begin(), E = N->use_end(); I != E; ++I) {
+    // Ignore call instructions.
+    if (I->getOpcode() == ISD::CopyToReg)
+      ++CallCount;
+    UseCount++;
+  }
+
+  return (UseCount <= 1) || (CallCount > 1);
+
+}
+
 void HexagonDAGToDAGISel::PreprocessISelDAG() {
   SelectionDAG &DAG = *CurDAG;
   std::vector<SDNode*> Nodes;
-  for (auto I = DAG.allnodes_begin(), E = DAG.allnodes_end(); I != E; ++I)
-    Nodes.push_back(I);
+  for (SDNode &Node : DAG.allnodes())
+    Nodes.push_back(&Node);
 
   // Simplify: (or (select c x 0) z)  ->  (select c (or x z) z)
   //           (or (select c 0 y) z)  ->  (select c z (or y z))
@@ -1352,12 +1390,32 @@ void HexagonDAGToDAGISel::PreprocessISelDAG() {
   }
 }
 
+void HexagonDAGToDAGISel::EmitFunctionEntryCode() {
+  auto &HST = static_cast<const HexagonSubtarget&>(MF->getSubtarget());
+  auto &HFI = *HST.getFrameLowering();
+  if (!HFI.needsAligna(*MF))
+    return;
+
+  MachineFrameInfo *MFI = MF->getFrameInfo();
+  MachineBasicBlock *EntryBB = MF->begin();
+  unsigned AR = FuncInfo->CreateReg(MVT::i32);
+  unsigned MaxA = MFI->getMaxAlignment();
+  auto &HII = *HST.getInstrInfo();
+  BuildMI(EntryBB, DebugLoc(), HII.get(Hexagon::ALIGNA), AR)
+      .addImm(MaxA);
+  MF->getInfo<HexagonMachineFunctionInfo>()->setStackAlignBaseVReg(AR);
+}
 
+// Match a frame index that can be used in an addressing mode.
 bool HexagonDAGToDAGISel::SelectAddrFI(SDValue& N, SDValue &R) {
   if (N.getOpcode() != ISD::FrameIndex)
     return false;
-  FrameIndexSDNode *FX = cast<FrameIndexSDNode>(N);
-  R = CurDAG->getTargetFrameIndex(FX->getIndex(), MVT::i32);
+  auto &HFI = *HST->getFrameLowering();
+  MachineFrameInfo *MFI = MF->getFrameInfo();
+  int FX = cast<FrameIndexSDNode>(N)->getIndex();
+  if (!MFI->isFixedObjectIndex(FX) && HFI.needsAligna(*MF))
+    return false;
+  R = CurDAG->getTargetFrameIndex(FX, MVT::i32);
   return true;
 }
 
diff --git a/lib/Target/Hexagon/HexagonISelLowering.cpp b/lib/Target/Hexagon/HexagonISelLowering.cpp
index 63900e02042..c739afb70c1 100644
--- a/lib/Target/Hexagon/HexagonISelLowering.cpp
+++ b/lib/Target/Hexagon/HexagonISelLowering.cpp
@@ -43,11 +43,48 @@ using namespace llvm;
 
 static cl::opt<bool>
 EmitJumpTables("hexagon-emit-jump-tables", cl::init(true), cl::Hidden,
-               cl::desc("Control jump table emission on Hexagon target"));
+  cl::desc("Control jump table emission on Hexagon target"));
+
+static cl::opt<bool> EnableHexSDNodeSched("enable-hexagon-sdnode-sched",
+  cl::Hidden, cl::ZeroOrMore, cl::init(false),
+  cl::desc("Enable Hexagon SDNode scheduling"));
+
+static cl::opt<bool> EnableFastMath("ffast-math",
+  cl::Hidden, cl::ZeroOrMore, cl::init(false),
+  cl::desc("Enable Fast Math processing"));
+
+static cl::opt<int> MinimumJumpTables("minimum-jump-tables",
+  cl::Hidden, cl::ZeroOrMore, cl::init(5),
+  cl::desc("Set minimum jump tables"));
+
+static cl::opt<int> MaxStoresPerMemcpyCL("max-store-memcpy",
+  cl::Hidden, cl::ZeroOrMore, cl::init(6),
+  cl::desc("Max #stores to inline memcpy"));
+
+static cl::opt<int> MaxStoresPerMemcpyOptSizeCL("max-store-memcpy-Os",
+  cl::Hidden, cl::ZeroOrMore, cl::init(4),
+  cl::desc("Max #stores to inline memcpy"));
+
+static cl::opt<int> MaxStoresPerMemmoveCL("max-store-memmove",
+  cl::Hidden, cl::ZeroOrMore, cl::init(6),
+  cl::desc("Max #stores to inline memmove"));
+
+static cl::opt<int> MaxStoresPerMemmoveOptSizeCL("max-store-memmove-Os",
+  cl::Hidden, cl::ZeroOrMore, cl::init(4),
+  cl::desc("Max #stores to inline memmove"));
+
+static cl::opt<int> MaxStoresPerMemsetCL("max-store-memset",
+  cl::Hidden, cl::ZeroOrMore, cl::init(8),
+  cl::desc("Max #stores to inline memset"));
+
+static cl::opt<int> MaxStoresPerMemsetOptSizeCL("max-store-memset-Os",
+  cl::Hidden, cl::ZeroOrMore, cl::init(4),
+  cl::desc("Max #stores to inline memset"));
+
 
 namespace {
 class HexagonCCState : public CCState {
-  int NumNamedVarArgParams;
+  unsigned NumNamedVarArgParams;
 
 public:
   HexagonCCState(CallingConv::ID CC, bool isVarArg, MachineFunction &MF,
@@ -56,7 +93,7 @@ class HexagonCCState : public CCState {
       : CCState(CC, isVarArg, MF, locs, C),
         NumNamedVarArgParams(NumNamedVarArgParams) {}
 
-  int getNumNamedVarArgParams() const { return NumNamedVarArgParams; }
+  unsigned getNumNamedVarArgParams() const { return NumNamedVarArgParams; }
 };
 }
 
@@ -97,11 +134,7 @@ CC_Hexagon_VarArg (unsigned ValNo, MVT ValVT,
             ISD::ArgFlagsTy ArgFlags, CCState &State) {
   HexagonCCState &HState = static_cast<HexagonCCState &>(State);
 
-  // NumNamedVarArgParams can not be zero for a VarArg function.
-  assert((HState.getNumNamedVarArgParams() > 0) &&
-         "NumNamedVarArgParams is not bigger than zero.");
-
-  if ((int)ValNo < HState.getNumNamedVarArgParams()) {
+  if (ValNo < HState.getNumNamedVarArgParams()) {
     // Deal with named arguments.
     return CC_Hexagon(ValNo, ValVT, LocVT, LocInfo, ArgFlags, State);
   }
@@ -111,9 +144,8 @@ CC_Hexagon_VarArg (unsigned ValNo, MVT ValVT,
   if (ArgFlags.isByVal()) {
     // If pass-by-value, the size allocated on stack is decided
     // by ArgFlags.getByValSize(), not by the size of LocVT.
-    assert ((ArgFlags.getByValSize() > 8) &&
-            "ByValSize must be bigger than 8 bytes");
-    ofst = State.AllocateStack(ArgFlags.getByValSize(), 4);
+    ofst = State.AllocateStack(ArgFlags.getByValSize(),
+                               ArgFlags.getByValAlign());
     State.addLoc(CCValAssign::getMem(ValNo, ValVT, ofst, LocVT, LocInfo));
     return false;
   }
@@ -148,9 +180,8 @@ CC_Hexagon (unsigned ValNo, MVT ValVT,
 
   if (ArgFlags.isByVal()) {
     // Passed on stack.
-    assert ((ArgFlags.getByValSize() > 8) &&
-            "ByValSize must be bigger than 8 bytes");
-    unsigned Offset = State.AllocateStack(ArgFlags.getByValSize(), 4);
+    unsigned Offset = State.AllocateStack(ArgFlags.getByValSize(),
+                                          ArgFlags.getByValAlign());
     State.addLoc(CCValAssign::getMem(ValNo, ValVT, Offset, LocVT, LocInfo));
     return false;
   }
@@ -313,7 +344,7 @@ CreateCopyOfByValArgument(SDValue Src, SDValue Dst, SDValue Chain,
                           ISD::ArgFlagsTy Flags, SelectionDAG &DAG,
                           SDLoc dl) {
 
-  SDValue SizeNode = DAG.getConstant(Flags.getByValSize(), MVT::i32);
+  SDValue SizeNode = DAG.getConstant(Flags.getByValSize(), dl, MVT::i32);
   return DAG.getMemcpy(Chain, dl, Dst, Src, SizeNode, Flags.getByValAlign(),
                        /*isVolatile=*/false, /*AlwaysInline=*/false,
                        /*isTailCall=*/false,
@@ -364,8 +395,15 @@ HexagonTargetLowering::LowerReturn(SDValue Chain,
   return DAG.getNode(HexagonISD::RET_FLAG, dl, MVT::Other, RetOps);
 }
 
+bool HexagonTargetLowering::mayBeEmittedAsTailCall(CallInst *CI) const {
+  // If either no tail call or told not to tail call at all, don't.
+  auto Attr =
+      CI->getParent()->getParent()->getFnAttribute("disable-tail-calls");
+  if (!CI->isTailCall() || Attr.getValueAsString() == "true")
+    return false;
 
-
+  return true;
+}
 
 /// LowerCallResult - Lower the result values of an ISD::CALL into the
 /// appropriate copies out of appropriate physical registers.  This assumes that
@@ -420,6 +458,8 @@ HexagonTargetLowering::LowerCall(TargetLowering::CallLoweringInfo &CLI,
   bool doesNotReturn                    = CLI.DoesNotReturn;
 
   bool IsStructRet    = (Outs.empty()) ? false : Outs[0].Flags.isSRet();
+  MachineFunction &MF = DAG.getMachineFunction();
+  auto PtrVT = getPointerTy(MF.getDataLayout());
 
   // Check for varargs.
   int NumNamedVarArgParams = -1;
@@ -444,41 +484,40 @@ HexagonTargetLowering::LowerCall(TargetLowering::CallLoweringInfo &CLI,
   HexagonCCState CCInfo(CallConv, isVarArg, DAG.getMachineFunction(), ArgLocs,
                         *DAG.getContext(), NumNamedVarArgParams);
 
-  if (NumNamedVarArgParams > 0)
+  if (isVarArg)
     CCInfo.AnalyzeCallOperands(Outs, CC_Hexagon_VarArg);
   else
     CCInfo.AnalyzeCallOperands(Outs, CC_Hexagon);
 
+  auto Attr = MF.getFunction()->getFnAttribute("disable-tail-calls");
+  if (Attr.getValueAsString() == "true")
+    isTailCall = false;
 
-  if(isTailCall) {
-    bool StructAttrFlag =
-      DAG.getMachineFunction().getFunction()->hasStructRetAttr();
+  if (isTailCall) {
+    bool StructAttrFlag = MF.getFunction()->hasStructRetAttr();
     isTailCall = IsEligibleForTailCallOptimization(Callee, CallConv,
                                                    isVarArg, IsStructRet,
                                                    StructAttrFlag,
                                                    Outs, OutVals, Ins, DAG);
-    for (unsigned i = 0, e = ArgLocs.size(); i != e; ++i){
+    for (unsigned i = 0, e = ArgLocs.size(); i != e; ++i) {
       CCValAssign &VA = ArgLocs[i];
       if (VA.isMemLoc()) {
         isTailCall = false;
         break;
       }
     }
-    if (isTailCall) {
-      DEBUG(dbgs () << "Eligible for Tail Call\n");
-    } else {
-      DEBUG(dbgs () <<
-            "Argument must be passed on stack. Not eligible for Tail Call\n");
-    }
+    DEBUG(dbgs() << (isTailCall ? "Eligible for Tail Call\n"
+                                : "Argument must be passed on stack. "
+                                  "Not eligible for Tail Call\n"));
   }
   // Get a count of how many bytes are to be pushed on the stack.
   unsigned NumBytes = CCInfo.getNextStackOffset();
   SmallVector<std::pair<unsigned, SDValue>, 16> RegsToPass;
   SmallVector<SDValue, 8> MemOpChains;
 
-  const HexagonRegisterInfo *QRI = Subtarget->getRegisterInfo();
+  auto &HRI = *Subtarget.getRegisterInfo();
   SDValue StackPtr =
-      DAG.getCopyFromReg(Chain, dl, QRI->getStackRegister(), getPointerTy());
+      DAG.getCopyFromReg(Chain, dl, HRI.getStackRegister(), PtrVT);
 
   // Walk the register/memloc assignments, inserting copies/loads.
   for (unsigned i = 0, e = ArgLocs.size(); i != e; ++i) {
@@ -491,6 +530,7 @@ HexagonTargetLowering::LowerCall(TargetLowering::CallLoweringInfo &CLI,
       default:
         // Loc info must be one of Full, SExt, ZExt, or AExt.
         llvm_unreachable("Unknown loc info!");
+      case CCValAssign::BCvt:
       case CCValAssign::Full:
         break;
       case CCValAssign::SExt:
@@ -506,41 +546,38 @@ HexagonTargetLowering::LowerCall(TargetLowering::CallLoweringInfo &CLI,
 
     if (VA.isMemLoc()) {
       unsigned LocMemOffset = VA.getLocMemOffset();
-      SDValue PtrOff = DAG.getConstant(LocMemOffset, StackPtr.getValueType());
-      PtrOff = DAG.getNode(ISD::ADD, dl, MVT::i32, StackPtr, PtrOff);
-
+      SDValue MemAddr = DAG.getConstant(LocMemOffset, dl,
+                                        StackPtr.getValueType());
+      MemAddr = DAG.getNode(ISD::ADD, dl, MVT::i32, StackPtr, MemAddr);
       if (Flags.isByVal()) {
         // The argument is a struct passed by value. According to LLVM, "Arg"
         // is is pointer.
-        MemOpChains.push_back(CreateCopyOfByValArgument(Arg, PtrOff, Chain,
+        MemOpChains.push_back(CreateCopyOfByValArgument(Arg, MemAddr, Chain,
                                                         Flags, DAG, dl));
       } else {
-        // The argument is not passed by value. "Arg" is a buildin type. It is
-        // not a pointer.
-        MemOpChains.push_back(DAG.getStore(Chain, dl, Arg, PtrOff,
-                                           MachinePointerInfo(),false, false,
-                                           0));
+        MachinePointerInfo LocPI = MachinePointerInfo::getStack(LocMemOffset);
+        SDValue S = DAG.getStore(Chain, dl, Arg, MemAddr, LocPI, false,
+                                 false, 0);
+        MemOpChains.push_back(S);
       }
       continue;
     }
 
     // Arguments that can be passed on register must be kept at RegsToPass
     // vector.
-    if (VA.isRegLoc()) {
+    if (VA.isRegLoc())
       RegsToPass.push_back(std::make_pair(VA.getLocReg(), Arg));
-    }
   }
 
   // Transform all store nodes into one single node because all store
   // nodes are independent of each other.
-  if (!MemOpChains.empty()) {
+  if (!MemOpChains.empty())
     Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, MemOpChains);
-  }
 
-  if (!isTailCall)
-    Chain = DAG.getCALLSEQ_START(Chain, DAG.getConstant(NumBytes,
-                                                        getPointerTy(), true),
-                                 dl);
+  if (!isTailCall) {
+    SDValue C = DAG.getConstant(NumBytes, dl, PtrVT, true);
+    Chain = DAG.getCALLSEQ_START(Chain, C, dl);
+  }
 
   // Build a sequence of copy-to-reg nodes chained together with token
   // chain and flag operands which copy the outgoing args into registers.
@@ -553,10 +590,9 @@ HexagonTargetLowering::LowerCall(TargetLowering::CallLoweringInfo &CLI,
                                RegsToPass[i].second, InFlag);
       InFlag = Chain.getValue(1);
     }
-  }
-
-  // For tail calls lower the arguments to the 'real' stack slot.
-  if (isTailCall) {
+  } else {
+    // For tail calls lower the arguments to the 'real' stack slot.
+    //
     // Force all the incoming stack arguments to be loaded from the stack
     // before any new outgoing arguments are stored to the stack, because the
     // outgoing stack slots may alias the incoming argument stack slots, and
@@ -571,7 +607,7 @@ HexagonTargetLowering::LowerCall(TargetLowering::CallLoweringInfo &CLI,
                                RegsToPass[i].second, InFlag);
       InFlag = Chain.getValue(1);
     }
-    InFlag =SDValue();
+    InFlag = SDValue();
   }
 
   // If the callee is a GlobalAddress/ExternalSymbol node (quite common, every
@@ -580,14 +616,13 @@ HexagonTargetLowering::LowerCall(TargetLowering::CallLoweringInfo &CLI,
   if (flag_aligned_memcpy) {
     const char *MemcpyName =
       "__hexagon_memcpy_likely_aligned_min32bytes_mult8bytes";
-    Callee =
-      DAG.getTargetExternalSymbol(MemcpyName, getPointerTy());
+    Callee = DAG.getTargetExternalSymbol(MemcpyName, PtrVT);
     flag_aligned_memcpy = false;
   } else if (GlobalAddressSDNode *G = dyn_cast<GlobalAddressSDNode>(Callee)) {
-    Callee = DAG.getTargetGlobalAddress(G->getGlobal(), dl, getPointerTy());
+    Callee = DAG.getTargetGlobalAddress(G->getGlobal(), dl, PtrVT);
   } else if (ExternalSymbolSDNode *S =
              dyn_cast<ExternalSymbolSDNode>(Callee)) {
-    Callee = DAG.getTargetExternalSymbol(S->getSymbol(), getPointerTy());
+    Callee = DAG.getTargetExternalSymbol(S->getSymbol(), PtrVT);
   }
 
   // Returns a chain & a flag for retval copy to use.
@@ -603,20 +638,21 @@ HexagonTargetLowering::LowerCall(TargetLowering::CallLoweringInfo &CLI,
                                   RegsToPass[i].second.getValueType()));
   }
 
-  if (InFlag.getNode()) {
+  if (InFlag.getNode())
     Ops.push_back(InFlag);
-  }
 
-  if (isTailCall)
+  if (isTailCall) {
+    MF.getFrameInfo()->setHasTailCall();
     return DAG.getNode(HexagonISD::TC_RETURN, dl, NodeTys, Ops);
+  }
 
   int OpCode = doesNotReturn ? HexagonISD::CALLv3nr : HexagonISD::CALLv3;
   Chain = DAG.getNode(OpCode, dl, NodeTys, Ops);
   InFlag = Chain.getValue(1);
 
   // Create the CALLSEQ_END node.
-  Chain = DAG.getCALLSEQ_END(Chain, DAG.getIntPtrConstant(NumBytes, true),
-                             DAG.getIntPtrConstant(0, true), InFlag, dl);
+  Chain = DAG.getCALLSEQ_END(Chain, DAG.getIntPtrConstant(NumBytes, dl, true),
+                             DAG.getIntPtrConstant(0, dl, true), InFlag, dl);
   InFlag = Chain.getValue(1);
 
   // Handle result values, copying them out of physregs into vregs that we
@@ -630,7 +666,7 @@ static bool getIndexedAddressParts(SDNode *Ptr, EVT VT,
                                    SDValue &Offset, bool &isInc,
                                    SelectionDAG &DAG) {
   if (Ptr->getOpcode() != ISD::ADD)
-  return false;
+    return false;
 
   if (VT == MVT::i64 || VT == MVT::i32 || VT == MVT::i16 || VT == MVT::i8) {
     isInc = (Ptr->getOpcode() == ISD::ADD);
@@ -702,8 +738,7 @@ SDValue HexagonTargetLowering::LowerINLINEASM(SDValue Op,
                                               SelectionDAG &DAG) const {
   SDNode *Node = Op.getNode();
   MachineFunction &MF = DAG.getMachineFunction();
-  HexagonMachineFunctionInfo *FuncInfo =
-    MF.getInfo<HexagonMachineFunctionInfo>();
+  auto &FuncInfo = *MF.getInfo<HexagonMachineFunctionInfo>();
   switch (Node->getOpcode()) {
     case ISD::INLINEASM: {
       unsigned NumOps = Node->getNumOperands();
@@ -711,7 +746,7 @@ SDValue HexagonTargetLowering::LowerINLINEASM(SDValue Op,
         --NumOps;  // Ignore the flag operand.
 
       for (unsigned i = InlineAsm::Op_FirstOperand; i != NumOps;) {
-        if (FuncInfo->hasClobberLR())
+        if (FuncInfo.hasClobberLR())
           break;
         unsigned Flags =
           cast<ConstantSDNode>(Node->getOperand(i))->getZExtValue();
@@ -734,9 +769,9 @@ SDValue HexagonTargetLowering::LowerINLINEASM(SDValue Op,
                 cast<RegisterSDNode>(Node->getOperand(i))->getReg();
 
               // Check it to be lr
-              const HexagonRegisterInfo *QRI = Subtarget->getRegisterInfo();
+              const HexagonRegisterInfo *QRI = Subtarget.getRegisterInfo();
               if (Reg == QRI->getRARegister()) {
-                FuncInfo->setHasClobberLR(true);
+                FuncInfo.setHasClobberLR(true);
                 break;
               }
             }
@@ -777,10 +812,10 @@ LowerBR_JT(SDValue Op, SelectionDAG &DAG) const
     BlockAddress::get(const_cast<BasicBlock *>(MBB->getBasicBlock()));
   }
 
-  SDValue JumpTableBase = DAG.getNode(HexagonISD::JT, dl,
-                                      getPointerTy(), TargetJT);
+  SDValue JumpTableBase = DAG.getNode(
+      HexagonISD::JT, dl, getPointerTy(DAG.getDataLayout()), TargetJT);
   SDValue ShiftIndex = DAG.getNode(ISD::SHL, dl, MVT::i32, Index,
-                                   DAG.getConstant(2, MVT::i32));
+                                   DAG.getConstant(2, dl, MVT::i32));
   SDValue JTAddress = DAG.getNode(ISD::ADD, dl, MVT::i32, JumpTableBase,
                                   ShiftIndex);
   SDValue LoadTarget = DAG.getLoad(MVT::i32, dl, Chain, JTAddress,
@@ -795,43 +830,27 @@ HexagonTargetLowering::LowerDYNAMIC_STACKALLOC(SDValue Op,
                                                SelectionDAG &DAG) const {
   SDValue Chain = Op.getOperand(0);
   SDValue Size = Op.getOperand(1);
+  SDValue Align = Op.getOperand(2);
   SDLoc dl(Op);
 
-  unsigned SPReg = getStackPointerRegisterToSaveRestore();
+  ConstantSDNode *AlignConst = dyn_cast<ConstantSDNode>(Align);
+  assert(AlignConst && "Non-constant Align in LowerDYNAMIC_STACKALLOC");
 
-  // Get a reference to the stack pointer.
-  SDValue StackPointer = DAG.getCopyFromReg(Chain, dl, SPReg, MVT::i32);
+  unsigned A = AlignConst->getSExtValue();
+  auto &HFI = *Subtarget.getFrameLowering();
+  // "Zero" means natural stack alignment.
+  if (A == 0)
+    A = HFI.getStackAlignment();
 
-  // Subtract the dynamic size from the actual stack size to
-  // obtain the new stack size.
-  SDValue Sub = DAG.getNode(ISD::SUB, dl, MVT::i32, StackPointer, Size);
+  DEBUG({
+    dbgs () << LLVM_FUNCTION_NAME << " Align: " << A << " Size: ";
+    Size.getNode()->dump(&DAG);
+    dbgs() << "\n";
+  });
 
-  //
-  // For Hexagon, the outgoing memory arguments area should be on top of the
-  // alloca area on the stack i.e., the outgoing memory arguments should be
-  // at a lower address than the alloca area. Move the alloca area down the
-  // stack by adding back the space reserved for outgoing arguments to SP
-  // here.
-  //
-  // We do not know what the size of the outgoing args is at this point.
-  // So, we add a pseudo instruction ADJDYNALLOC that will adjust the
-  // stack pointer. We patch this instruction with the correct, known
-  // offset in emitPrologue().
-  //
-  // Use a placeholder immediate (zero) for now. This will be patched up
-  // by emitPrologue().
-  SDValue ArgAdjust = DAG.getNode(HexagonISD::ADJDYNALLOC, dl,
-                                  MVT::i32,
-                                  Sub,
-                                  DAG.getConstant(0, MVT::i32));
-
-  // The Sub result contains the new stack start address, so it
-  // must be placed in the stack pointer register.
-  const HexagonRegisterInfo *QRI = Subtarget->getRegisterInfo();
-  SDValue CopyChain = DAG.getCopyToReg(Chain, dl, QRI->getStackRegister(), Sub);
-
-  SDValue Ops[2] = { ArgAdjust, CopyChain };
-  return DAG.getMergeValues(Ops, dl);
+  SDValue AC = DAG.getConstant(A, dl, MVT::i32);
+  SDVTList VTs = DAG.getVTList(MVT::i32, MVT::Other);
+  return DAG.getNode(HexagonISD::ALLOCA, dl, VTs, Chain, Size, AC);
 }
 
 SDValue
@@ -847,9 +866,7 @@ const {
   MachineFunction &MF = DAG.getMachineFunction();
   MachineFrameInfo *MFI = MF.getFrameInfo();
   MachineRegisterInfo &RegInfo = MF.getRegInfo();
-  HexagonMachineFunctionInfo *FuncInfo =
-    MF.getInfo<HexagonMachineFunctionInfo>();
-
+  auto &FuncInfo = *MF.getInfo<HexagonMachineFunctionInfo>();
 
   // Assign locations to all of the incoming arguments.
   SmallVector<CCValAssign, 16> ArgLocs;
@@ -938,7 +955,7 @@ const {
                                             HEXAGON_LRFP_SIZE +
                                             CCInfo.getNextStackOffset(),
                                             true);
-    FuncInfo->setVarArgsFrameIndex(FrameIndex);
+    FuncInfo.setVarArgsFrameIndex(FrameIndex);
   }
 
   return Chain;
@@ -984,7 +1001,7 @@ SDValue HexagonTargetLowering::LowerCTPOP(SDValue Op, SelectionDAG &DAG) const {
   SDValue InpVal = Op.getOperand(0);
   if (isa<ConstantSDNode>(InpVal)) {
     uint64_t V = cast<ConstantSDNode>(InpVal)->getZExtValue();
-    return DAG.getTargetConstant(countPopulation(V), MVT::i64);
+    return DAG.getTargetConstant(countPopulation(V), dl, MVT::i64);
   }
   SDValue PopOut = DAG.getNode(HexagonISD::POPCOUNT, dl, MVT::i32, InpVal);
   return DAG.getNode(ISD::ZERO_EXTEND, dl, MVT::i64, PopOut);
@@ -1085,7 +1102,7 @@ SDValue HexagonTargetLowering::LowerLOAD(SDValue Op, SelectionDAG &DAG) const {
                                 LoadNode->isInvariant(),
                                 Alignment);
       // Base+2 load.
-      SDValue Increment = DAG.getConstant(2, MVT::i32);
+      SDValue Increment = DAG.getConstant(2, DL, MVT::i32);
       Ptr = DAG.getNode(ISD::ADD, DL, Base.getValueType(), Base, Increment);
       Loads[1] = DAG.getExtLoad(Ext, DL, MVT::i32, Chain, Ptr,
                                 LoadNode->getPointerInfo(), MVT::i16,
@@ -1094,11 +1111,11 @@ SDValue HexagonTargetLowering::LowerLOAD(SDValue Op, SelectionDAG &DAG) const {
                                 LoadNode->isInvariant(),
                                 Alignment);
       // SHL 16, then OR base and base+2.
-      SDValue ShiftAmount = DAG.getConstant(16, MVT::i32);
+      SDValue ShiftAmount = DAG.getConstant(16, DL, MVT::i32);
       SDValue Tmp1 = DAG.getNode(ISD::SHL, DL, MVT::i32, Loads[1], ShiftAmount);
       SDValue Tmp2 = DAG.getNode(ISD::OR, DL, MVT::i32, Tmp1, Loads[0]);
       // Base + 4.
-      Increment = DAG.getConstant(4, MVT::i32);
+      Increment = DAG.getConstant(4, DL, MVT::i32);
       Ptr = DAG.getNode(ISD::ADD, DL, Base.getValueType(), Base, Increment);
       Loads[2] = DAG.getExtLoad(Ext, DL, MVT::i32, Chain, Ptr,
                                 LoadNode->getPointerInfo(), MVT::i16,
@@ -1107,7 +1124,7 @@ SDValue HexagonTargetLowering::LowerLOAD(SDValue Op, SelectionDAG &DAG) const {
                                 LoadNode->isInvariant(),
                                 Alignment);
       // Base + 6.
-      Increment = DAG.getConstant(6, MVT::i32);
+      Increment = DAG.getConstant(6, DL, MVT::i32);
       Ptr = DAG.getNode(ISD::ADD, DL, Base.getValueType(), Base, Increment);
       Loads[3] = DAG.getExtLoad(Ext, DL, MVT::i32, Chain, Ptr,
                                 LoadNode->getPointerInfo(), MVT::i16,
@@ -1155,15 +1172,15 @@ HexagonTargetLowering::LowerConstantPool(SDValue Op, SelectionDAG &DAG) const {
   else
     Res = DAG.getTargetConstantPool(CP->getConstVal(), ValTy,
                                     CP->getAlignment());
-  return DAG.getNode(HexagonISD::CONST32, dl, ValTy, Res);
+  return DAG.getNode(HexagonISD::CP, dl, ValTy, Res);
 }
 
 SDValue
 HexagonTargetLowering::LowerRETURNADDR(SDValue Op, SelectionDAG &DAG) const {
-  const TargetRegisterInfo *TRI = Subtarget->getRegisterInfo();
+  const HexagonRegisterInfo &HRI = *Subtarget.getRegisterInfo();
   MachineFunction &MF = DAG.getMachineFunction();
-  MachineFrameInfo *MFI = MF.getFrameInfo();
-  MFI->setReturnAddressIsTaken(true);
+  MachineFrameInfo &MFI = *MF.getFrameInfo();
+  MFI.setReturnAddressIsTaken(true);
 
   if (verifyReturnAddressArgumentIsConstant(Op, DAG))
     return SDValue();
@@ -1173,28 +1190,28 @@ HexagonTargetLowering::LowerRETURNADDR(SDValue Op, SelectionDAG &DAG) const {
   unsigned Depth = cast<ConstantSDNode>(Op.getOperand(0))->getZExtValue();
   if (Depth) {
     SDValue FrameAddr = LowerFRAMEADDR(Op, DAG);
-    SDValue Offset = DAG.getConstant(4, MVT::i32);
+    SDValue Offset = DAG.getConstant(4, dl, MVT::i32);
     return DAG.getLoad(VT, dl, DAG.getEntryNode(),
                        DAG.getNode(ISD::ADD, dl, VT, FrameAddr, Offset),
                        MachinePointerInfo(), false, false, false, 0);
   }
 
   // Return LR, which contains the return address. Mark it an implicit live-in.
-  unsigned Reg = MF.addLiveIn(TRI->getRARegister(), getRegClassFor(MVT::i32));
+  unsigned Reg = MF.addLiveIn(HRI.getRARegister(), getRegClassFor(MVT::i32));
   return DAG.getCopyFromReg(DAG.getEntryNode(), dl, Reg, VT);
 }
 
 SDValue
 HexagonTargetLowering::LowerFRAMEADDR(SDValue Op, SelectionDAG &DAG) const {
-  const HexagonRegisterInfo *TRI = Subtarget->getRegisterInfo();
-  MachineFrameInfo *MFI = DAG.getMachineFunction().getFrameInfo();
-  MFI->setFrameAddressIsTaken(true);
+  const HexagonRegisterInfo &HRI = *Subtarget.getRegisterInfo();
+  MachineFrameInfo &MFI = *DAG.getMachineFunction().getFrameInfo();
+  MFI.setFrameAddressIsTaken(true);
 
   EVT VT = Op.getValueType();
   SDLoc dl(Op);
   unsigned Depth = cast<ConstantSDNode>(Op.getOperand(0))->getZExtValue();
   SDValue FrameAddr = DAG.getCopyFromReg(DAG.getEntryNode(), dl,
-                                         TRI->getFrameRegister(), VT);
+                                         HRI.getFrameRegister(), VT);
   while (Depth--)
     FrameAddr = DAG.getLoad(VT, dl, DAG.getEntryNode(), FrameAddr,
                             MachinePointerInfo(),
@@ -1215,16 +1232,17 @@ SDValue HexagonTargetLowering::LowerGLOBALADDRESS(SDValue Op,
   const GlobalValue *GV = cast<GlobalAddressSDNode>(Op)->getGlobal();
   int64_t Offset = cast<GlobalAddressSDNode>(Op)->getOffset();
   SDLoc dl(Op);
-  Result = DAG.getTargetGlobalAddress(GV, dl, getPointerTy(), Offset);
+  auto PtrVT = getPointerTy(DAG.getDataLayout());
+  Result = DAG.getTargetGlobalAddress(GV, dl, PtrVT, Offset);
 
   const HexagonTargetObjectFile *TLOF =
       static_cast<const HexagonTargetObjectFile *>(
           getTargetMachine().getObjFileLowering());
   if (TLOF->IsGlobalInSmallSection(GV, getTargetMachine())) {
-    return DAG.getNode(HexagonISD::CONST32_GP, dl, getPointerTy(), Result);
+    return DAG.getNode(HexagonISD::CONST32_GP, dl, PtrVT, Result);
   }
 
-  return DAG.getNode(HexagonISD::CONST32, dl, getPointerTy(), Result);
+  return DAG.getNode(HexagonISD::CONST32, dl, PtrVT, Result);
 }
 
 // Specifies that for loads and stores VT can be promoted to PromotedLdStVT.
@@ -1245,7 +1263,8 @@ HexagonTargetLowering::LowerBlockAddress(SDValue Op, SelectionDAG &DAG) const {
   const BlockAddress *BA = cast<BlockAddressSDNode>(Op)->getBlockAddress();
   SDValue BA_SD =  DAG.getTargetBlockAddress(BA, MVT::i32);
   SDLoc dl(Op);
-  return DAG.getNode(HexagonISD::CONST32_GP, dl, getPointerTy(), BA_SD);
+  return DAG.getNode(HexagonISD::CONST32_GP, dl,
+                     getPointerTy(DAG.getDataLayout()), BA_SD);
 }
 
 //===----------------------------------------------------------------------===//
@@ -1254,619 +1273,514 @@ HexagonTargetLowering::LowerBlockAddress(SDValue Op, SelectionDAG &DAG) const {
 
 HexagonTargetLowering::HexagonTargetLowering(const TargetMachine &TM,
                                              const HexagonSubtarget &STI)
-    : TargetLowering(TM), Subtarget(&STI) {
-
-  // Set up the register classes.
-  addRegisterClass(MVT::v2i1, &Hexagon::PredRegsRegClass);  // bbbbaaaa
-  addRegisterClass(MVT::v4i1, &Hexagon::PredRegsRegClass);  // ddccbbaa
-  addRegisterClass(MVT::v8i1, &Hexagon::PredRegsRegClass);  // hgfedcba
-  addRegisterClass(MVT::i32, &Hexagon::IntRegsRegClass);
-  addRegisterClass(MVT::v4i8, &Hexagon::IntRegsRegClass);
+    : TargetLowering(TM), HTM(static_cast<const HexagonTargetMachine&>(TM)),
+      Subtarget(STI) {
+  bool IsV4 = !Subtarget.hasV5TOps();
+  auto &HRI = *Subtarget.getRegisterInfo();
+
+  setPrefLoopAlignment(4);
+  setPrefFunctionAlignment(4);
+  setMinFunctionAlignment(2);
+  setInsertFencesForAtomic(false);
+  setExceptionPointerRegister(Hexagon::R0);
+  setExceptionSelectorRegister(Hexagon::R1);
+  setStackPointerRegisterToSaveRestore(HRI.getStackRegister());
+
+  if (EnableHexSDNodeSched)
+    setSchedulingPreference(Sched::VLIW);
+  else
+    setSchedulingPreference(Sched::Source);
+
+  // Limits for inline expansion of memcpy/memmove
+  MaxStoresPerMemcpy = MaxStoresPerMemcpyCL;
+  MaxStoresPerMemcpyOptSize = MaxStoresPerMemcpyOptSizeCL;
+  MaxStoresPerMemmove = MaxStoresPerMemmoveCL;
+  MaxStoresPerMemmoveOptSize = MaxStoresPerMemmoveOptSizeCL;
+  MaxStoresPerMemset = MaxStoresPerMemsetCL;
+  MaxStoresPerMemsetOptSize = MaxStoresPerMemsetOptSizeCL;
+
+  //
+  // Set up register classes.
+  //
+
+  addRegisterClass(MVT::i1,    &Hexagon::PredRegsRegClass);
+  addRegisterClass(MVT::v2i1,  &Hexagon::PredRegsRegClass);  // bbbbaaaa
+  addRegisterClass(MVT::v4i1,  &Hexagon::PredRegsRegClass);  // ddccbbaa
+  addRegisterClass(MVT::v8i1,  &Hexagon::PredRegsRegClass);  // hgfedcba
+  addRegisterClass(MVT::i32,   &Hexagon::IntRegsRegClass);
+  addRegisterClass(MVT::v4i8,  &Hexagon::IntRegsRegClass);
   addRegisterClass(MVT::v2i16, &Hexagon::IntRegsRegClass);
-  promoteLdStType(MVT::v4i8, MVT::i32);
-  promoteLdStType(MVT::v2i16, MVT::i32);
+  addRegisterClass(MVT::i64,   &Hexagon::DoubleRegsRegClass);
+  addRegisterClass(MVT::v8i8,  &Hexagon::DoubleRegsRegClass);
+  addRegisterClass(MVT::v4i16, &Hexagon::DoubleRegsRegClass);
+  addRegisterClass(MVT::v2i32, &Hexagon::DoubleRegsRegClass);
 
-  if (Subtarget->hasV5TOps()) {
+  if (Subtarget.hasV5TOps()) {
     addRegisterClass(MVT::f32, &Hexagon::IntRegsRegClass);
     addRegisterClass(MVT::f64, &Hexagon::DoubleRegsRegClass);
   }
 
-  addRegisterClass(MVT::i64, &Hexagon::DoubleRegsRegClass);
-  addRegisterClass(MVT::v8i8, &Hexagon::DoubleRegsRegClass);
-  addRegisterClass(MVT::v4i16, &Hexagon::DoubleRegsRegClass);
-  addRegisterClass(MVT::v2i32, &Hexagon::DoubleRegsRegClass);
-  promoteLdStType(MVT::v8i8, MVT::i64);
+  //
+  // Handling of scalar operations.
+  //
+  // All operations default to "legal", except:
+  // - indexed loads and stores (pre-/post-incremented),
+  // - ANY_EXTEND_VECTOR_INREG, ATOMIC_CMP_SWAP_WITH_SUCCESS, CONCAT_VECTORS,
+  //   ConstantFP, DEBUGTRAP, FCEIL, FCOPYSIGN, FEXP, FEXP2, FFLOOR, FGETSIGN,
+  //   FLOG, FLOG2, FLOG10, FMAXNUM, FMINNUM, FNEARBYINT, FRINT, FROUND, TRAP,
+  //   FTRUNC, PREFETCH, SIGN_EXTEND_VECTOR_INREG, ZERO_EXTEND_VECTOR_INREG,
+  // which default to "expand" for at least one type.
+
+  // Misc operations.
+  setOperationAction(ISD::ConstantFP, MVT::f32, Legal); // Default: expand
+  setOperationAction(ISD::ConstantFP, MVT::f64, Legal); // Default: expand
 
-  // Custom lower v4i16 load only. Let v4i16 store to be
-  // promoted for now.
-  setOperationAction(ISD::LOAD, MVT::v4i16, Custom);
-  AddPromotedToType(ISD::LOAD, MVT::v4i16, MVT::i64);
-  setOperationAction(ISD::STORE, MVT::v4i16, Promote);
-  AddPromotedToType(ISD::STORE, MVT::v4i16, MVT::i64);
-  promoteLdStType(MVT::v2i32, MVT::i64);
+  setOperationAction(ISD::ConstantPool, MVT::i32, Custom);
+  setOperationAction(ISD::BUILD_PAIR, MVT::i64, Expand);
+  setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::i1, Expand);
+  setOperationAction(ISD::INLINEASM, MVT::Other, Custom);
+  setOperationAction(ISD::EH_RETURN, MVT::Other, Custom);
+  setOperationAction(ISD::ATOMIC_FENCE, MVT::Other, Custom);
 
-  for (unsigned i = (unsigned) MVT::FIRST_VECTOR_VALUETYPE;
-       i <= (unsigned) MVT::LAST_VECTOR_VALUETYPE; ++i) {
-    MVT::SimpleValueType VT = (MVT::SimpleValueType) i;
-
-    // Hexagon does not have support for the following operations,
-    // so they need to be expanded.
-    setOperationAction(ISD::SELECT, VT, Expand);
-    setOperationAction(ISD::SDIV, VT, Expand);
-    setOperationAction(ISD::SREM, VT, Expand);
-    setOperationAction(ISD::UDIV, VT, Expand);
-    setOperationAction(ISD::UREM, VT, Expand);
-    setOperationAction(ISD::ROTL, VT, Expand);
-    setOperationAction(ISD::ROTR, VT, Expand);
-    setOperationAction(ISD::FDIV, VT, Expand);
-    setOperationAction(ISD::FNEG, VT, Expand);
-    setOperationAction(ISD::UMUL_LOHI, VT, Expand);
-    setOperationAction(ISD::SMUL_LOHI, VT, Expand);
-    setOperationAction(ISD::UDIVREM, VT, Expand);
-    setOperationAction(ISD::SDIVREM, VT, Expand);
-    setOperationAction(ISD::FPOW, VT, Expand);
-    setOperationAction(ISD::CTPOP, VT, Expand);
-    setOperationAction(ISD::CTLZ, VT, Expand);
-    setOperationAction(ISD::CTLZ_ZERO_UNDEF, VT, Expand);
-    setOperationAction(ISD::CTTZ, VT, Expand);
-    setOperationAction(ISD::CTTZ_ZERO_UNDEF, VT, Expand);
-
-    // Expand all any extend loads.
-    for (unsigned j = (unsigned) MVT::FIRST_VECTOR_VALUETYPE;
-                  j <= (unsigned) MVT::LAST_VECTOR_VALUETYPE; ++j)
-      setLoadExtAction(ISD::EXTLOAD, (MVT::SimpleValueType) j, VT, Expand);
-
-    // Expand all trunc stores.
-    for (unsigned TargetVT = (unsigned) MVT::FIRST_VECTOR_VALUETYPE;
-         TargetVT <= (unsigned) MVT::LAST_VECTOR_VALUETYPE; ++TargetVT)
-      setTruncStoreAction(VT, (MVT::SimpleValueType) TargetVT, Expand);
-
-    setOperationAction(ISD::VECTOR_SHUFFLE, VT, Expand);
-    setOperationAction(ISD::ConstantPool, VT, Expand);
-    setOperationAction(ISD::SCALAR_TO_VECTOR, VT, Expand);
-    setOperationAction(ISD::EXTRACT_VECTOR_ELT, VT, Expand);
-    setOperationAction(ISD::INSERT_VECTOR_ELT, VT, Expand);
-    setOperationAction(ISD::BUILD_VECTOR, VT, Expand);
-    setOperationAction(ISD::EXTRACT_SUBVECTOR, VT, Expand);
-    setOperationAction(ISD::INSERT_SUBVECTOR, VT, Expand);
-    setOperationAction(ISD::CONCAT_VECTORS, VT, Expand);
-    setOperationAction(ISD::SRA, VT, Custom);
-    setOperationAction(ISD::SHL, VT, Custom);
-    setOperationAction(ISD::SRL, VT, Custom);
+  // Custom legalize GlobalAddress nodes into CONST32.
+  setOperationAction(ISD::GlobalAddress, MVT::i32, Custom);
+  setOperationAction(ISD::GlobalAddress, MVT::i8,  Custom);
+  setOperationAction(ISD::BlockAddress,  MVT::i32, Custom);
 
-    if (!isTypeLegal(VT))
-      continue;
+  // Hexagon needs to optimize cases with negative constants.
+  setOperationAction(ISD::SETCC, MVT::i8,  Custom);
+  setOperationAction(ISD::SETCC, MVT::i16, Custom);
 
-    setOperationAction(ISD::ADD, VT, Legal);
-    setOperationAction(ISD::SUB, VT, Legal);
-    setOperationAction(ISD::MUL, VT, Legal);
+  // VASTART needs to be custom lowered to use the VarArgsFrameIndex.
+  setOperationAction(ISD::VASTART, MVT::Other, Custom);
+  setOperationAction(ISD::VAEND,   MVT::Other, Expand);
+  setOperationAction(ISD::VAARG,   MVT::Other, Expand);
 
-    setOperationAction(ISD::BUILD_VECTOR, VT, Custom);
-    setOperationAction(ISD::EXTRACT_VECTOR_ELT, VT, Custom);
-    setOperationAction(ISD::INSERT_VECTOR_ELT, VT, Custom);
-    setOperationAction(ISD::EXTRACT_SUBVECTOR, VT, Custom);
-    setOperationAction(ISD::INSERT_SUBVECTOR, VT, Custom);
-    setOperationAction(ISD::CONCAT_VECTORS, VT, Custom);
-  }
+  setOperationAction(ISD::STACKSAVE, MVT::Other, Expand);
+  setOperationAction(ISD::STACKRESTORE, MVT::Other, Expand);
+  setOperationAction(ISD::DYNAMIC_STACKALLOC, MVT::i32, Custom);
 
-  setOperationAction(ISD::SETCC, MVT::v2i16, Custom);
-  setOperationAction(ISD::VSELECT, MVT::v2i16, Custom);
-  setOperationAction(ISD::VECTOR_SHUFFLE, MVT::v8i8, Custom);
-  setOperationAction(ISD::VECTOR_SHUFFLE, MVT::v4i16, Custom);
+  if (EmitJumpTables)
+    setOperationAction(ISD::BR_JT, MVT::Other, Custom);
+  else
+    setOperationAction(ISD::BR_JT, MVT::Other, Expand);
+  // Increase jump tables cutover to 5, was 4.
+  setMinimumJumpTableEntries(MinimumJumpTables);
+
+  // Hexagon has instructions for add/sub with carry. The problem with
+  // modeling these instructions is that they produce 2 results: Rdd and Px.
+  // To model the update of Px, we will have to use Defs[p0..p3] which will
+  // cause any predicate live range to spill. So, we pretend we dont't have
+  // these instructions.
+  setOperationAction(ISD::ADDE, MVT::i8,  Expand);
+  setOperationAction(ISD::ADDE, MVT::i16, Expand);
+  setOperationAction(ISD::ADDE, MVT::i32, Expand);
+  setOperationAction(ISD::ADDE, MVT::i64, Expand);
+  setOperationAction(ISD::SUBE, MVT::i8,  Expand);
+  setOperationAction(ISD::SUBE, MVT::i16, Expand);
+  setOperationAction(ISD::SUBE, MVT::i32, Expand);
+  setOperationAction(ISD::SUBE, MVT::i64, Expand);
+  setOperationAction(ISD::ADDC, MVT::i8,  Expand);
+  setOperationAction(ISD::ADDC, MVT::i16, Expand);
+  setOperationAction(ISD::ADDC, MVT::i32, Expand);
+  setOperationAction(ISD::ADDC, MVT::i64, Expand);
+  setOperationAction(ISD::SUBC, MVT::i8,  Expand);
+  setOperationAction(ISD::SUBC, MVT::i16, Expand);
+  setOperationAction(ISD::SUBC, MVT::i32, Expand);
+  setOperationAction(ISD::SUBC, MVT::i64, Expand);
 
-  setOperationAction(ISD::ConstantPool, MVT::i32, Custom);
+  // Only add and sub that detect overflow are the saturating ones.
+  for (MVT VT : MVT::integer_valuetypes()) {
+    setOperationAction(ISD::UADDO, VT, Expand);
+    setOperationAction(ISD::SADDO, VT, Expand);
+    setOperationAction(ISD::USUBO, VT, Expand);
+    setOperationAction(ISD::SSUBO, VT, Expand);
+  }
 
-  addRegisterClass(MVT::i1, &Hexagon::PredRegsRegClass);
+  setOperationAction(ISD::CTLZ, MVT::i8,  Promote);
+  setOperationAction(ISD::CTLZ, MVT::i16, Promote);
+  setOperationAction(ISD::CTTZ, MVT::i8,  Promote);
+  setOperationAction(ISD::CTTZ, MVT::i16, Promote);
+  setOperationAction(ISD::CTLZ_ZERO_UNDEF, MVT::i8,  Promote);
+  setOperationAction(ISD::CTLZ_ZERO_UNDEF, MVT::i16, Promote);
+  setOperationAction(ISD::CTTZ_ZERO_UNDEF, MVT::i8,  Promote);
+  setOperationAction(ISD::CTTZ_ZERO_UNDEF, MVT::i16, Promote);
+
+  // In V5, popcount can count # of 1s in i64 but returns i32.
+  // On V4 it will be expanded (set later).
+  setOperationAction(ISD::CTPOP, MVT::i8,  Promote);
+  setOperationAction(ISD::CTPOP, MVT::i16, Promote);
+  setOperationAction(ISD::CTPOP, MVT::i32, Promote);
+  setOperationAction(ISD::CTPOP, MVT::i64, Custom);
+
+  // We custom lower i64 to i64 mul, so that it is not considered as a legal
+  // operation. There is a pattern that will match i64 mul and transform it
+  // to a series of instructions.
+  setOperationAction(ISD::MUL,   MVT::i64, Expand);
+  setOperationAction(ISD::MULHS, MVT::i64, Expand);
 
-  computeRegisterProperties(Subtarget->getRegisterInfo());
+  for (unsigned IntExpOp :
+       {ISD::SDIV, ISD::UDIV, ISD::SREM, ISD::UREM, ISD::SDIVREM, ISD::UDIVREM,
+        ISD::ROTL, ISD::ROTR, ISD::BSWAP, ISD::SHL_PARTS, ISD::SRA_PARTS,
+        ISD::SRL_PARTS, ISD::SMUL_LOHI, ISD::UMUL_LOHI}) {
+    setOperationAction(IntExpOp, MVT::i32, Expand);
+    setOperationAction(IntExpOp, MVT::i64, Expand);
+  }
 
-  // Align loop entry
-  setPrefLoopAlignment(4);
+  for (unsigned FPExpOp :
+       {ISD::FDIV, ISD::FREM, ISD::FSQRT, ISD::FSIN, ISD::FCOS, ISD::FSINCOS,
+        ISD::FPOW, ISD::FCOPYSIGN}) {
+    setOperationAction(FPExpOp, MVT::f32, Expand);
+    setOperationAction(FPExpOp, MVT::f64, Expand);
+  }
 
-  // Limits for inline expansion of memcpy/memmove
-  MaxStoresPerMemcpy = 6;
-  MaxStoresPerMemmove = 6;
+  // No extending loads from i32.
+  for (MVT VT : MVT::integer_valuetypes()) {
+    setLoadExtAction(ISD::ZEXTLOAD, VT, MVT::i32, Expand);
+    setLoadExtAction(ISD::SEXTLOAD, VT, MVT::i32, Expand);
+    setLoadExtAction(ISD::EXTLOAD,  VT, MVT::i32, Expand);
+  }
+  // Turn FP truncstore into trunc + store.
+  setTruncStoreAction(MVT::f64, MVT::f32, Expand);
+  // Turn FP extload into load/fextend.
+  for (MVT VT : MVT::fp_valuetypes())
+    setLoadExtAction(ISD::EXTLOAD, VT, MVT::f32, Expand);
+
+  // Expand BR_CC and SELECT_CC for all integer and fp types.
+  for (MVT VT : MVT::integer_valuetypes()) {
+    setOperationAction(ISD::BR_CC,     VT, Expand);
+    setOperationAction(ISD::SELECT_CC, VT, Expand);
+  }
+  for (MVT VT : MVT::fp_valuetypes()) {
+    setOperationAction(ISD::BR_CC,     VT, Expand);
+    setOperationAction(ISD::SELECT_CC, VT, Expand);
+  }
+  setOperationAction(ISD::BR_CC, MVT::Other, Expand);
 
   //
-  // Library calls for unsupported operations
+  // Handling of vector operations.
   //
 
-  setLibcallName(RTLIB::SINTTOFP_I128_F64, "__hexagon_floattidf");
-  setLibcallName(RTLIB::SINTTOFP_I128_F32, "__hexagon_floattisf");
-
-  setLibcallName(RTLIB::FPTOUINT_F32_I128, "__hexagon_fixunssfti");
-  setLibcallName(RTLIB::FPTOUINT_F64_I128, "__hexagon_fixunsdfti");
-
-  setLibcallName(RTLIB::FPTOSINT_F32_I128, "__hexagon_fixsfti");
-  setLibcallName(RTLIB::FPTOSINT_F64_I128, "__hexagon_fixdfti");
-
-  setLibcallName(RTLIB::SDIV_I32, "__hexagon_divsi3");
-  setOperationAction(ISD::SDIV, MVT::i32, Expand);
-  setLibcallName(RTLIB::SREM_I32, "__hexagon_umodsi3");
-  setOperationAction(ISD::SREM, MVT::i32, Expand);
-
-  setLibcallName(RTLIB::SDIV_I64, "__hexagon_divdi3");
-  setOperationAction(ISD::SDIV, MVT::i64, Expand);
-  setLibcallName(RTLIB::SREM_I64, "__hexagon_moddi3");
-  setOperationAction(ISD::SREM, MVT::i64, Expand);
+  // Custom lower v4i16 load only. Let v4i16 store to be
+  // promoted for now.
+  promoteLdStType(MVT::v4i8,  MVT::i32);
+  promoteLdStType(MVT::v2i16, MVT::i32);
+  promoteLdStType(MVT::v8i8,  MVT::i64);
+  promoteLdStType(MVT::v2i32, MVT::i64);
 
-  setLibcallName(RTLIB::UDIV_I32, "__hexagon_udivsi3");
-  setOperationAction(ISD::UDIV, MVT::i32, Expand);
+  setOperationAction(ISD::LOAD,  MVT::v4i16, Custom);
+  setOperationAction(ISD::STORE, MVT::v4i16, Promote);
+  AddPromotedToType(ISD::LOAD,  MVT::v4i16, MVT::i64);
+  AddPromotedToType(ISD::STORE, MVT::v4i16, MVT::i64);
 
-  setLibcallName(RTLIB::UDIV_I64, "__hexagon_udivdi3");
-  setOperationAction(ISD::UDIV, MVT::i64, Expand);
+  // Set the action for vector operations to "expand", then override it with
+  // either "custom" or "legal" for specific cases.
+  static unsigned VectExpOps[] = {
+    // Integer arithmetic:
+    ISD::ADD,     ISD::SUB,     ISD::MUL,     ISD::SDIV,    ISD::UDIV,
+    ISD::SREM,    ISD::UREM,    ISD::SDIVREM, ISD::UDIVREM, ISD::ADDC,
+    ISD::SUBC,    ISD::SADDO,   ISD::UADDO,   ISD::SSUBO,   ISD::USUBO,
+    ISD::SMUL_LOHI,             ISD::UMUL_LOHI,
+    // Logical/bit:
+    ISD::AND,     ISD::OR,      ISD::XOR,     ISD::ROTL,    ISD::ROTR,
+    ISD::CTPOP,   ISD::CTLZ,    ISD::CTTZ,    ISD::CTLZ_ZERO_UNDEF,
+    ISD::CTTZ_ZERO_UNDEF,
+    // Floating point arithmetic/math functions:
+    ISD::FADD,    ISD::FSUB,    ISD::FMUL,    ISD::FMA,     ISD::FDIV,
+    ISD::FREM,    ISD::FNEG,    ISD::FABS,    ISD::FSQRT,   ISD::FSIN,
+    ISD::FCOS,    ISD::FPOWI,   ISD::FPOW,    ISD::FLOG,    ISD::FLOG2,
+    ISD::FLOG10,  ISD::FEXP,    ISD::FEXP2,   ISD::FCEIL,   ISD::FTRUNC,
+    ISD::FRINT,   ISD::FNEARBYINT,            ISD::FROUND,  ISD::FFLOOR,
+    ISD::FMINNUM, ISD::FMAXNUM, ISD::FSINCOS,
+    // Misc:
+    ISD::SELECT,  ISD::ConstantPool,
+    // Vector:
+    ISD::BUILD_VECTOR,          ISD::SCALAR_TO_VECTOR,
+    ISD::EXTRACT_VECTOR_ELT,    ISD::INSERT_VECTOR_ELT,
+    ISD::EXTRACT_SUBVECTOR,     ISD::INSERT_SUBVECTOR,
+    ISD::CONCAT_VECTORS,        ISD::VECTOR_SHUFFLE
+  };
 
-  setLibcallName(RTLIB::UREM_I32, "__hexagon_umodsi3");
-  setOperationAction(ISD::UREM, MVT::i32, Expand);
+  for (MVT VT : MVT::vector_valuetypes()) {
+    for (unsigned VectExpOp : VectExpOps)
+      setOperationAction(VectExpOp, VT, Expand);
 
-  setLibcallName(RTLIB::UREM_I64, "__hexagon_umoddi3");
-  setOperationAction(ISD::UREM, MVT::i64, Expand);
+    // Expand all extended loads and truncating stores:
+    for (MVT TargetVT : MVT::vector_valuetypes()) {
+      setLoadExtAction(ISD::EXTLOAD, TargetVT, VT, Expand);
+      setTruncStoreAction(VT, TargetVT, Expand);
+    }
 
-  setLibcallName(RTLIB::DIV_F32, "__hexagon_divsf3");
-  setOperationAction(ISD::FDIV, MVT::f32, Expand);
+    setOperationAction(ISD::SRA, VT, Custom);
+    setOperationAction(ISD::SHL, VT, Custom);
+    setOperationAction(ISD::SRL, VT, Custom);
+  }
 
-  setLibcallName(RTLIB::DIV_F64, "__hexagon_divdf3");
-  setOperationAction(ISD::FDIV, MVT::f64, Expand);
+  // Types natively supported:
+  for (MVT NativeVT : {MVT::v2i1, MVT::v4i1, MVT::v8i1, MVT::v32i1, MVT::v64i1,
+                       MVT::v4i8, MVT::v8i8, MVT::v2i16, MVT::v4i16, MVT::v1i32,
+                       MVT::v2i32, MVT::v1i64}) {
+    setOperationAction(ISD::BUILD_VECTOR,       NativeVT, Custom);
+    setOperationAction(ISD::EXTRACT_VECTOR_ELT, NativeVT, Custom);
+    setOperationAction(ISD::INSERT_VECTOR_ELT,  NativeVT, Custom);
+    setOperationAction(ISD::EXTRACT_SUBVECTOR,  NativeVT, Custom);
+    setOperationAction(ISD::INSERT_SUBVECTOR,   NativeVT, Custom);
+    setOperationAction(ISD::CONCAT_VECTORS,     NativeVT, Custom);
 
-  setLibcallName(RTLIB::ADD_F64, "__hexagon_adddf3");
-  setLibcallName(RTLIB::SUB_F64, "__hexagon_subdf3");
-  setLibcallName(RTLIB::MUL_F64, "__hexagon_muldf3");
+    setOperationAction(ISD::ADD, NativeVT, Legal);
+    setOperationAction(ISD::SUB, NativeVT, Legal);
+    setOperationAction(ISD::MUL, NativeVT, Legal);
+    setOperationAction(ISD::AND, NativeVT, Legal);
+    setOperationAction(ISD::OR,  NativeVT, Legal);
+    setOperationAction(ISD::XOR, NativeVT, Legal);
+  }
 
-  setOperationAction(ISD::FSQRT, MVT::f32, Expand);
-  setOperationAction(ISD::FSQRT, MVT::f64, Expand);
-  setOperationAction(ISD::FSIN, MVT::f32, Expand);
-  setOperationAction(ISD::FSIN, MVT::f64, Expand);
+  setOperationAction(ISD::SETCC,          MVT::v2i16, Custom);
+  setOperationAction(ISD::VSELECT,        MVT::v2i16, Custom);
+  setOperationAction(ISD::VECTOR_SHUFFLE, MVT::v4i16, Custom);
+  setOperationAction(ISD::VECTOR_SHUFFLE, MVT::v8i8,  Custom);
 
-  if (Subtarget->hasV5TOps()) {
-    // Hexagon V5 Support.
-    setOperationAction(ISD::FADD, MVT::f32, Legal);
+  // Subtarget-specific operation actions.
+  //
+  if (Subtarget.hasV5TOps()) {
+    setOperationAction(ISD::FMA,  MVT::f64, Expand);
     setOperationAction(ISD::FADD, MVT::f64, Expand);
-    setOperationAction(ISD::FSUB, MVT::f32, Legal);
     setOperationAction(ISD::FSUB, MVT::f64, Expand);
     setOperationAction(ISD::FMUL, MVT::f64, Expand);
-    setOperationAction(ISD::FP_EXTEND, MVT::f32, Legal);
-    setCondCodeAction(ISD::SETOEQ, MVT::f32, Legal);
-    setCondCodeAction(ISD::SETOEQ, MVT::f64, Legal);
-    setCondCodeAction(ISD::SETUEQ, MVT::f32, Legal);
-    setCondCodeAction(ISD::SETUEQ, MVT::f64, Legal);
-
-    setCondCodeAction(ISD::SETOGE, MVT::f32, Legal);
-    setCondCodeAction(ISD::SETOGE, MVT::f64, Legal);
-    setCondCodeAction(ISD::SETUGE, MVT::f32, Legal);
-    setCondCodeAction(ISD::SETUGE, MVT::f64, Legal);
-
-    setCondCodeAction(ISD::SETOGT, MVT::f32, Legal);
-    setCondCodeAction(ISD::SETOGT, MVT::f64, Legal);
-    setCondCodeAction(ISD::SETUGT, MVT::f32, Legal);
-    setCondCodeAction(ISD::SETUGT, MVT::f64, Legal);
-
-    setCondCodeAction(ISD::SETOLE, MVT::f32, Legal);
-    setCondCodeAction(ISD::SETOLE, MVT::f64, Legal);
-    setCondCodeAction(ISD::SETOLT, MVT::f32, Legal);
-    setCondCodeAction(ISD::SETOLT, MVT::f64, Legal);
-
-    setOperationAction(ISD::ConstantFP, MVT::f32, Legal);
-    setOperationAction(ISD::ConstantFP, MVT::f64, Legal);
-
-    setOperationAction(ISD::FP_TO_UINT, MVT::i1, Promote);
-    setOperationAction(ISD::FP_TO_SINT, MVT::i1, Promote);
-    setOperationAction(ISD::UINT_TO_FP, MVT::i1, Promote);
-    setOperationAction(ISD::SINT_TO_FP, MVT::i1, Promote);
-
-    setOperationAction(ISD::FP_TO_UINT, MVT::i8, Promote);
-    setOperationAction(ISD::FP_TO_SINT, MVT::i8, Promote);
-    setOperationAction(ISD::UINT_TO_FP, MVT::i8, Promote);
-    setOperationAction(ISD::SINT_TO_FP, MVT::i8, Promote);
 
+    setOperationAction(ISD::FP_TO_UINT, MVT::i1,  Promote);
+    setOperationAction(ISD::FP_TO_UINT, MVT::i8,  Promote);
     setOperationAction(ISD::FP_TO_UINT, MVT::i16, Promote);
+    setOperationAction(ISD::FP_TO_SINT, MVT::i1,  Promote);
+    setOperationAction(ISD::FP_TO_SINT, MVT::i8,  Promote);
     setOperationAction(ISD::FP_TO_SINT, MVT::i16, Promote);
+    setOperationAction(ISD::UINT_TO_FP, MVT::i1,  Promote);
+    setOperationAction(ISD::UINT_TO_FP, MVT::i8,  Promote);
     setOperationAction(ISD::UINT_TO_FP, MVT::i16, Promote);
+    setOperationAction(ISD::SINT_TO_FP, MVT::i1,  Promote);
+    setOperationAction(ISD::SINT_TO_FP, MVT::i8,  Promote);
     setOperationAction(ISD::SINT_TO_FP, MVT::i16, Promote);
 
-    setOperationAction(ISD::FP_TO_UINT, MVT::i32, Legal);
-    setOperationAction(ISD::FP_TO_SINT, MVT::i32, Legal);
-    setOperationAction(ISD::UINT_TO_FP, MVT::i32, Legal);
-    setOperationAction(ISD::SINT_TO_FP, MVT::i32, Legal);
-
-    setOperationAction(ISD::FP_TO_UINT, MVT::i64, Legal);
-    setOperationAction(ISD::FP_TO_SINT, MVT::i64, Legal);
-    setOperationAction(ISD::UINT_TO_FP, MVT::i64, Legal);
-    setOperationAction(ISD::SINT_TO_FP, MVT::i64, Legal);
-
-    setOperationAction(ISD::FABS, MVT::f32, Legal);
-    setOperationAction(ISD::FABS, MVT::f64, Expand);
-
-    setOperationAction(ISD::FNEG, MVT::f32, Legal);
-    setOperationAction(ISD::FNEG, MVT::f64, Expand);
-  } else {
-
-    // Expand fp<->uint.
-    setOperationAction(ISD::FP_TO_SINT, MVT::i32, Expand);
-    setOperationAction(ISD::FP_TO_UINT, MVT::i32, Expand);
-
+  } else { // V4
     setOperationAction(ISD::SINT_TO_FP, MVT::i32, Expand);
+    setOperationAction(ISD::SINT_TO_FP, MVT::i64, Expand);
     setOperationAction(ISD::UINT_TO_FP, MVT::i32, Expand);
-
-    setLibcallName(RTLIB::SINTTOFP_I64_F32, "__hexagon_floatdisf");
-    setLibcallName(RTLIB::UINTTOFP_I64_F32, "__hexagon_floatundisf");
-
-    setLibcallName(RTLIB::UINTTOFP_I32_F32, "__hexagon_floatunsisf");
-    setLibcallName(RTLIB::SINTTOFP_I32_F32, "__hexagon_floatsisf");
-
-    setLibcallName(RTLIB::SINTTOFP_I64_F64, "__hexagon_floatdidf");
-    setLibcallName(RTLIB::UINTTOFP_I64_F64, "__hexagon_floatundidf");
-
-    setLibcallName(RTLIB::UINTTOFP_I32_F64, "__hexagon_floatunsidf");
-    setLibcallName(RTLIB::SINTTOFP_I32_F64, "__hexagon_floatsidf");
-
-    setLibcallName(RTLIB::FPTOUINT_F32_I32, "__hexagon_fixunssfsi");
-    setLibcallName(RTLIB::FPTOUINT_F32_I64, "__hexagon_fixunssfdi");
-
-    setLibcallName(RTLIB::FPTOSINT_F64_I64, "__hexagon_fixdfdi");
-    setLibcallName(RTLIB::FPTOSINT_F32_I64, "__hexagon_fixsfdi");
-
-    setLibcallName(RTLIB::FPTOUINT_F64_I32, "__hexagon_fixunsdfsi");
-    setLibcallName(RTLIB::FPTOUINT_F64_I64, "__hexagon_fixunsdfdi");
-
-
-    setLibcallName(RTLIB::ADD_F32, "__hexagon_addsf3");
-    setOperationAction(ISD::FADD, MVT::f32, Expand);
-    setOperationAction(ISD::FADD, MVT::f64, Expand);
-
-    setLibcallName(RTLIB::SUB_F32, "__hexagon_subsf3");
-    setOperationAction(ISD::FSUB, MVT::f32, Expand);
-    setOperationAction(ISD::FSUB, MVT::f64, Expand);
-
-    setLibcallName(RTLIB::FPEXT_F32_F64, "__hexagon_extendsfdf2");
-    setOperationAction(ISD::FP_EXTEND, MVT::f32, Expand);
-
-    setLibcallName(RTLIB::OEQ_F32, "__hexagon_eqsf2");
-    setCondCodeAction(ISD::SETOEQ, MVT::f32, Expand);
-
-    setLibcallName(RTLIB::OEQ_F64, "__hexagon_eqdf2");
-    setCondCodeAction(ISD::SETOEQ, MVT::f64, Expand);
-
-    setLibcallName(RTLIB::OGE_F32, "__hexagon_gesf2");
-    setCondCodeAction(ISD::SETOGE, MVT::f32, Expand);
-
-    setLibcallName(RTLIB::OGE_F64, "__hexagon_gedf2");
-    setCondCodeAction(ISD::SETOGE, MVT::f64, Expand);
-
-    setLibcallName(RTLIB::OGT_F32, "__hexagon_gtsf2");
-    setCondCodeAction(ISD::SETOGT, MVT::f32, Expand);
-
-    setLibcallName(RTLIB::OGT_F64, "__hexagon_gtdf2");
-    setCondCodeAction(ISD::SETOGT, MVT::f64, Expand);
-
-    setLibcallName(RTLIB::FPTOSINT_F64_I32, "__hexagon_fixdfsi");
+    setOperationAction(ISD::UINT_TO_FP, MVT::i64, Expand);
     setOperationAction(ISD::FP_TO_SINT, MVT::f64, Expand);
-
-    setLibcallName(RTLIB::FPTOSINT_F32_I32, "__hexagon_fixsfsi");
     setOperationAction(ISD::FP_TO_SINT, MVT::f32, Expand);
-
-    setLibcallName(RTLIB::OLE_F64, "__hexagon_ledf2");
-    setCondCodeAction(ISD::SETOLE, MVT::f64, Expand);
-
-    setLibcallName(RTLIB::OLE_F32, "__hexagon_lesf2");
-    setCondCodeAction(ISD::SETOLE, MVT::f32, Expand);
-
-    setLibcallName(RTLIB::OLT_F64, "__hexagon_ltdf2");
-    setCondCodeAction(ISD::SETOLT, MVT::f64, Expand);
-
-    setLibcallName(RTLIB::OLT_F32, "__hexagon_ltsf2");
-    setCondCodeAction(ISD::SETOLT, MVT::f32, Expand);
-
-    setOperationAction(ISD::FMUL, MVT::f64, Expand);
-
-    setLibcallName(RTLIB::MUL_F32, "__hexagon_mulsf3");
-    setOperationAction(ISD::MUL, MVT::f32, Expand);
-
-    setLibcallName(RTLIB::UNE_F64, "__hexagon_nedf2");
+    setOperationAction(ISD::FP_EXTEND,  MVT::f32, Expand);
+    setOperationAction(ISD::FP_ROUND,   MVT::f64, Expand);
     setCondCodeAction(ISD::SETUNE, MVT::f64, Expand);
 
-    setLibcallName(RTLIB::UNE_F32, "__hexagon_nesf2");
-
-    setLibcallName(RTLIB::SUB_F64, "__hexagon_subdf3");
-    setOperationAction(ISD::SUB, MVT::f64, Expand);
-
-    setLibcallName(RTLIB::SUB_F32, "__hexagon_subsf3");
-    setOperationAction(ISD::SUB, MVT::f32, Expand);
-
-    setLibcallName(RTLIB::FPROUND_F64_F32, "__hexagon_truncdfsf2");
-    setOperationAction(ISD::FP_ROUND, MVT::f64, Expand);
+    setOperationAction(ISD::CTPOP, MVT::i8,  Expand);
+    setOperationAction(ISD::CTPOP, MVT::i16, Expand);
+    setOperationAction(ISD::CTPOP, MVT::i32, Expand);
+    setOperationAction(ISD::CTPOP, MVT::i64, Expand);
 
-    setLibcallName(RTLIB::UO_F64, "__hexagon_unorddf2");
-    setCondCodeAction(ISD::SETUO, MVT::f64, Expand);
-
-    setLibcallName(RTLIB::O_F64, "__hexagon_unorddf2");
-    setCondCodeAction(ISD::SETO, MVT::f64, Expand);
-
-    setLibcallName(RTLIB::O_F32, "__hexagon_unordsf2");
-    setCondCodeAction(ISD::SETO, MVT::f32, Expand);
-
-    setLibcallName(RTLIB::UO_F32, "__hexagon_unordsf2");
-    setCondCodeAction(ISD::SETUO, MVT::f32, Expand);
+    // Expand these operations for both f32 and f64:
+    for (unsigned FPExpOpV4 :
+         {ISD::FADD, ISD::FSUB, ISD::FMUL, ISD::FABS, ISD::FNEG, ISD::FMA}) {
+      setOperationAction(FPExpOpV4, MVT::f32, Expand);
+      setOperationAction(FPExpOpV4, MVT::f64, Expand);
+    }
 
-    setOperationAction(ISD::FABS, MVT::f32, Expand);
-    setOperationAction(ISD::FABS, MVT::f64, Expand);
-    setOperationAction(ISD::FNEG, MVT::f32, Expand);
-    setOperationAction(ISD::FNEG, MVT::f64, Expand);
+    for (ISD::CondCode FPExpCCV4 :
+         {ISD::SETOEQ, ISD::SETOGT, ISD::SETOLT, ISD::SETOGE, ISD::SETOLE,
+          ISD::SETUO, ISD::SETO}) {
+      setCondCodeAction(FPExpCCV4, MVT::f32, Expand);
+      setCondCodeAction(FPExpCCV4, MVT::f64, Expand);
+    }
   }
 
-  setLibcallName(RTLIB::SREM_I32, "__hexagon_modsi3");
-  setOperationAction(ISD::SREM, MVT::i32, Expand);
-
-  setIndexedLoadAction(ISD::POST_INC, MVT::i8, Legal);
-  setIndexedLoadAction(ISD::POST_INC, MVT::i16, Legal);
-  setIndexedLoadAction(ISD::POST_INC, MVT::i32, Legal);
-  setIndexedLoadAction(ISD::POST_INC, MVT::i64, Legal);
-
-  setIndexedStoreAction(ISD::POST_INC, MVT::i8, Legal);
-  setIndexedStoreAction(ISD::POST_INC, MVT::i16, Legal);
-  setIndexedStoreAction(ISD::POST_INC, MVT::i32, Legal);
-  setIndexedStoreAction(ISD::POST_INC, MVT::i64, Legal);
-
-  setOperationAction(ISD::BUILD_PAIR, MVT::i64, Expand);
-
-  // Turn FP extload into load/fextend.
-  for (MVT VT : MVT::fp_valuetypes())
-    setLoadExtAction(ISD::EXTLOAD, VT, MVT::f32, Expand);
-
-  // No extending loads from i32.
-  for (MVT VT : MVT::integer_valuetypes()) {
-    setLoadExtAction(ISD::ZEXTLOAD, VT, MVT::i32, Expand);
-    setLoadExtAction(ISD::SEXTLOAD, VT, MVT::i32, Expand);
-    setLoadExtAction(ISD::EXTLOAD, VT, MVT::i32, Expand);
+  // Handling of indexed loads/stores: default is "expand".
+  //
+  for (MVT LSXTy : {MVT::i8, MVT::i16, MVT::i32, MVT::i64}) {
+    setIndexedLoadAction(ISD::POST_INC, LSXTy, Legal);
+    setIndexedStoreAction(ISD::POST_INC, LSXTy, Legal);
   }
 
-  // Turn FP truncstore into trunc + store.
-  setTruncStoreAction(MVT::f64, MVT::f32, Expand);
-
-  // Custom legalize GlobalAddress nodes into CONST32.
-  setOperationAction(ISD::GlobalAddress, MVT::i32, Custom);
-  setOperationAction(ISD::GlobalAddress, MVT::i8, Custom);
-  setOperationAction(ISD::BlockAddress, MVT::i32, Custom);
-  // Truncate action?
-  setOperationAction(ISD::TRUNCATE, MVT::i64, Expand);
-
-  // Hexagon doesn't have sext_inreg, replace them with shl/sra.
-  setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::i1, Expand);
-
-  // Hexagon has no REM or DIVREM operations.
-  setOperationAction(ISD::UREM, MVT::i32, Expand);
-  setOperationAction(ISD::SREM, MVT::i32, Expand);
-  setOperationAction(ISD::SDIVREM, MVT::i32, Expand);
-  setOperationAction(ISD::UDIVREM, MVT::i32, Expand);
-  setOperationAction(ISD::SREM, MVT::i64, Expand);
-  setOperationAction(ISD::SDIVREM, MVT::i64, Expand);
-  setOperationAction(ISD::UDIVREM, MVT::i64, Expand);
-
-  setOperationAction(ISD::BSWAP, MVT::i64, Expand);
-
-  // Lower SELECT_CC to SETCC and SELECT.
-  setOperationAction(ISD::SELECT_CC, MVT::i1, Expand);
-  setOperationAction(ISD::SELECT_CC, MVT::i32, Expand);
-  setOperationAction(ISD::SELECT_CC, MVT::i64, Expand);
+  computeRegisterProperties(&HRI);
 
-  if (Subtarget->hasV5TOps()) {
-
-    // We need to make the operation type of SELECT node to be Custom,
-    // such that we don't go into the infinite loop of
-    // select ->  setcc -> select_cc -> select loop.
-    setOperationAction(ISD::SELECT, MVT::f32, Custom);
-    setOperationAction(ISD::SELECT, MVT::f64, Custom);
+  //
+  // Library calls for unsupported operations
+  //
+  bool FastMath  = EnableFastMath;
 
-    setOperationAction(ISD::SELECT_CC, MVT::f32, Expand);
-    setOperationAction(ISD::SELECT_CC, MVT::f64, Expand);
+  setLibcallName(RTLIB::SDIV_I32, "__hexagon_divsi3");
+  setLibcallName(RTLIB::SDIV_I64, "__hexagon_divdi3");
+  setLibcallName(RTLIB::UDIV_I32, "__hexagon_udivsi3");
+  setLibcallName(RTLIB::UDIV_I64, "__hexagon_udivdi3");
+  setLibcallName(RTLIB::SREM_I32, "__hexagon_modsi3");
+  setLibcallName(RTLIB::SREM_I64, "__hexagon_moddi3");
+  setLibcallName(RTLIB::UREM_I32, "__hexagon_umodsi3");
+  setLibcallName(RTLIB::UREM_I64, "__hexagon_umoddi3");
 
-  } else {
+  setLibcallName(RTLIB::SINTTOFP_I128_F64, "__hexagon_floattidf");
+  setLibcallName(RTLIB::SINTTOFP_I128_F32, "__hexagon_floattisf");
+  setLibcallName(RTLIB::FPTOUINT_F32_I128, "__hexagon_fixunssfti");
+  setLibcallName(RTLIB::FPTOUINT_F64_I128, "__hexagon_fixunsdfti");
+  setLibcallName(RTLIB::FPTOSINT_F32_I128, "__hexagon_fixsfti");
+  setLibcallName(RTLIB::FPTOSINT_F64_I128, "__hexagon_fixdfti");
 
-    // Hexagon has no select or setcc: expand to SELECT_CC.
-    setOperationAction(ISD::SELECT, MVT::f32, Expand);
-    setOperationAction(ISD::SELECT, MVT::f64, Expand);
+  if (IsV4) {
+    // Handle single-precision floating point operations on V4.
+    if (FastMath) {
+      setLibcallName(RTLIB::ADD_F32, "__hexagon_fast_addsf3");
+      setLibcallName(RTLIB::SUB_F32, "__hexagon_fast_subsf3");
+      setLibcallName(RTLIB::MUL_F32, "__hexagon_fast_mulsf3");
+      setLibcallName(RTLIB::OGT_F32, "__hexagon_fast_gtsf2");
+      setLibcallName(RTLIB::OLT_F32, "__hexagon_fast_ltsf2");
+      // Double-precision compares.
+      setLibcallName(RTLIB::OGT_F64, "__hexagon_fast_gtdf2");
+      setLibcallName(RTLIB::OLT_F64, "__hexagon_fast_ltdf2");
+    } else {
+      setLibcallName(RTLIB::ADD_F32, "__hexagon_addsf3");
+      setLibcallName(RTLIB::SUB_F32, "__hexagon_subsf3");
+      setLibcallName(RTLIB::MUL_F32, "__hexagon_mulsf3");
+      setLibcallName(RTLIB::OGT_F32, "__hexagon_gtsf2");
+      setLibcallName(RTLIB::OLT_F32, "__hexagon_ltsf2");
+      // Double-precision compares.
+      setLibcallName(RTLIB::OGT_F64, "__hexagon_gtdf2");
+      setLibcallName(RTLIB::OLT_F64, "__hexagon_ltdf2");
+    }
   }
 
-  // Hexagon needs to optimize cases with negative constants.
-  setOperationAction(ISD::SETCC, MVT::i16, Custom);
-  setOperationAction(ISD::SETCC, MVT::i8, Custom);
-
-  if (EmitJumpTables) {
-    setOperationAction(ISD::BR_JT, MVT::Other, Custom);
+  // This is the only fast library function for sqrtd.
+  if (FastMath)
+    setLibcallName(RTLIB::SQRT_F64, "__hexagon_fast2_sqrtdf2");
+
+  // Prefix is: nothing  for "slow-math",
+  //            "fast2_" for V4 fast-math and V5+ fast-math double-precision
+  // (actually, keep fast-math and fast-math2 separate for now)
+  if (FastMath) {
+    setLibcallName(RTLIB::ADD_F64, "__hexagon_fast_adddf3");
+    setLibcallName(RTLIB::SUB_F64, "__hexagon_fast_subdf3");
+    setLibcallName(RTLIB::MUL_F64, "__hexagon_fast_muldf3");
+    setLibcallName(RTLIB::DIV_F64, "__hexagon_fast_divdf3");
+    // Calling __hexagon_fast2_divsf3 with fast-math on V5 (ok).
+    setLibcallName(RTLIB::DIV_F32, "__hexagon_fast_divsf3");
   } else {
-    setOperationAction(ISD::BR_JT, MVT::Other, Expand);
+    setLibcallName(RTLIB::ADD_F64, "__hexagon_adddf3");
+    setLibcallName(RTLIB::SUB_F64, "__hexagon_subdf3");
+    setLibcallName(RTLIB::MUL_F64, "__hexagon_muldf3");
+    setLibcallName(RTLIB::DIV_F64, "__hexagon_divdf3");
+    setLibcallName(RTLIB::DIV_F32, "__hexagon_divsf3");
   }
-  // Increase jump tables cutover to 5, was 4.
-  setMinimumJumpTableEntries(5);
-
-  setOperationAction(ISD::BR_CC, MVT::f32, Expand);
-  setOperationAction(ISD::BR_CC, MVT::f64, Expand);
-  setOperationAction(ISD::BR_CC, MVT::i1, Expand);
-  setOperationAction(ISD::BR_CC, MVT::i32, Expand);
-  setOperationAction(ISD::BR_CC, MVT::i64, Expand);
-
-  setOperationAction(ISD::ATOMIC_FENCE, MVT::Other, Custom);
-
-  setOperationAction(ISD::FSIN, MVT::f64, Expand);
-  setOperationAction(ISD::FCOS, MVT::f64, Expand);
-  setOperationAction(ISD::FREM, MVT::f64, Expand);
-  setOperationAction(ISD::FSIN, MVT::f32, Expand);
-  setOperationAction(ISD::FCOS, MVT::f32, Expand);
-  setOperationAction(ISD::FREM, MVT::f32, Expand);
-  setOperationAction(ISD::FSINCOS, MVT::f64, Expand);
-  setOperationAction(ISD::FSINCOS, MVT::f32, Expand);
-
-  // In V4, we have double word add/sub with carry. The problem with
-  // modelling this instruction is that it produces 2 results - Rdd and Px.
-  // To model update of Px, we will have to use Defs[p0..p3] which will
-  // cause any predicate live range to spill. So, we pretend we dont't
-  // have these instructions.
-  setOperationAction(ISD::ADDE, MVT::i8, Expand);
-  setOperationAction(ISD::ADDE, MVT::i16, Expand);
-  setOperationAction(ISD::ADDE, MVT::i32, Expand);
-  setOperationAction(ISD::ADDE, MVT::i64, Expand);
-  setOperationAction(ISD::SUBE, MVT::i8, Expand);
-  setOperationAction(ISD::SUBE, MVT::i16, Expand);
-  setOperationAction(ISD::SUBE, MVT::i32, Expand);
-  setOperationAction(ISD::SUBE, MVT::i64, Expand);
-  setOperationAction(ISD::ADDC, MVT::i8, Expand);
-  setOperationAction(ISD::ADDC, MVT::i16, Expand);
-  setOperationAction(ISD::ADDC, MVT::i32, Expand);
-  setOperationAction(ISD::ADDC, MVT::i64, Expand);
-  setOperationAction(ISD::SUBC, MVT::i8, Expand);
-  setOperationAction(ISD::SUBC, MVT::i16, Expand);
-  setOperationAction(ISD::SUBC, MVT::i32, Expand);
-  setOperationAction(ISD::SUBC, MVT::i64, Expand);
 
-  // Only add and sub that detect overflow are the saturating ones.
-  for (MVT VT : MVT::integer_valuetypes()) {
-    setOperationAction(ISD::UADDO, VT, Expand);
-    setOperationAction(ISD::SADDO, VT, Expand);
-    setOperationAction(ISD::USUBO, VT, Expand);
-    setOperationAction(ISD::SSUBO, VT, Expand);
+  if (Subtarget.hasV5TOps()) {
+    if (FastMath)
+      setLibcallName(RTLIB::SQRT_F32, "__hexagon_fast2_sqrtf");
+    else
+      setLibcallName(RTLIB::SQRT_F32, "__hexagon_sqrtf");
+  } else {
+    // V4
+    setLibcallName(RTLIB::SINTTOFP_I32_F32, "__hexagon_floatsisf");
+    setLibcallName(RTLIB::SINTTOFP_I32_F64, "__hexagon_floatsidf");
+    setLibcallName(RTLIB::SINTTOFP_I64_F32, "__hexagon_floatdisf");
+    setLibcallName(RTLIB::SINTTOFP_I64_F64, "__hexagon_floatdidf");
+    setLibcallName(RTLIB::UINTTOFP_I32_F32, "__hexagon_floatunsisf");
+    setLibcallName(RTLIB::UINTTOFP_I32_F64, "__hexagon_floatunsidf");
+    setLibcallName(RTLIB::UINTTOFP_I64_F32, "__hexagon_floatundisf");
+    setLibcallName(RTLIB::UINTTOFP_I64_F64, "__hexagon_floatundidf");
+    setLibcallName(RTLIB::FPTOUINT_F32_I32, "__hexagon_fixunssfsi");
+    setLibcallName(RTLIB::FPTOUINT_F32_I64, "__hexagon_fixunssfdi");
+    setLibcallName(RTLIB::FPTOUINT_F64_I32, "__hexagon_fixunsdfsi");
+    setLibcallName(RTLIB::FPTOUINT_F64_I64, "__hexagon_fixunsdfdi");
+    setLibcallName(RTLIB::FPTOSINT_F32_I32, "__hexagon_fixsfsi");
+    setLibcallName(RTLIB::FPTOSINT_F32_I64, "__hexagon_fixsfdi");
+    setLibcallName(RTLIB::FPTOSINT_F64_I32, "__hexagon_fixdfsi");
+    setLibcallName(RTLIB::FPTOSINT_F64_I64, "__hexagon_fixdfdi");
+    setLibcallName(RTLIB::FPEXT_F32_F64,    "__hexagon_extendsfdf2");
+    setLibcallName(RTLIB::FPROUND_F64_F32,  "__hexagon_truncdfsf2");
+    setLibcallName(RTLIB::OEQ_F32, "__hexagon_eqsf2");
+    setLibcallName(RTLIB::OEQ_F64, "__hexagon_eqdf2");
+    setLibcallName(RTLIB::OGE_F32, "__hexagon_gesf2");
+    setLibcallName(RTLIB::OGE_F64, "__hexagon_gedf2");
+    setLibcallName(RTLIB::OLE_F32, "__hexagon_lesf2");
+    setLibcallName(RTLIB::OLE_F64, "__hexagon_ledf2");
+    setLibcallName(RTLIB::UNE_F32, "__hexagon_nesf2");
+    setLibcallName(RTLIB::UNE_F64, "__hexagon_nedf2");
+    setLibcallName(RTLIB::UO_F32,  "__hexagon_unordsf2");
+    setLibcallName(RTLIB::UO_F64,  "__hexagon_unorddf2");
+    setLibcallName(RTLIB::O_F32,   "__hexagon_unordsf2");
+    setLibcallName(RTLIB::O_F64,   "__hexagon_unorddf2");
   }
 
-  setOperationAction(ISD::CTPOP, MVT::i32, Expand);
-  setOperationAction(ISD::CTPOP, MVT::i64, Expand);
-  setOperationAction(ISD::CTTZ, MVT::i32, Expand);
-  setOperationAction(ISD::CTTZ, MVT::i64, Expand);
-  setOperationAction(ISD::CTTZ_ZERO_UNDEF, MVT::i32, Expand);
-  setOperationAction(ISD::CTTZ_ZERO_UNDEF, MVT::i64, Expand);
-  setOperationAction(ISD::CTLZ, MVT::i32, Expand);
-  setOperationAction(ISD::CTLZ, MVT::i64, Expand);
-  setOperationAction(ISD::CTLZ_ZERO_UNDEF, MVT::i32, Expand);
-  setOperationAction(ISD::CTLZ_ZERO_UNDEF, MVT::i64, Expand);
-
-  setOperationAction(ISD::ROTL, MVT::i32, Expand);
-  setOperationAction(ISD::ROTR, MVT::i32, Expand);
-  setOperationAction(ISD::BSWAP, MVT::i32, Expand);
-  setOperationAction(ISD::ROTL, MVT::i64, Expand);
-  setOperationAction(ISD::ROTR, MVT::i64, Expand);
-  setOperationAction(ISD::SHL_PARTS, MVT::i64, Expand);
-  setOperationAction(ISD::SRA_PARTS, MVT::i64, Expand);
-  setOperationAction(ISD::SRL_PARTS, MVT::i64, Expand);
-  setOperationAction(ISD::BR_CC, MVT::i64, Expand);
-
-  setOperationAction(ISD::FCOPYSIGN, MVT::f64, Expand);
-  setOperationAction(ISD::FCOPYSIGN, MVT::f32, Expand);
-  setOperationAction(ISD::FPOW, MVT::f64, Expand);
-  setOperationAction(ISD::FPOW, MVT::f32, Expand);
-
-  setOperationAction(ISD::SHL_PARTS, MVT::i32, Expand);
-  setOperationAction(ISD::SRA_PARTS, MVT::i32, Expand);
-  setOperationAction(ISD::SRL_PARTS, MVT::i32, Expand);
-
-  setOperationAction(ISD::UMUL_LOHI, MVT::i32, Expand);
-  setOperationAction(ISD::SMUL_LOHI, MVT::i32, Expand);
-
-  setOperationAction(ISD::MULHS, MVT::i64, Expand);
-  setOperationAction(ISD::SMUL_LOHI, MVT::i64, Expand);
-  setOperationAction(ISD::UMUL_LOHI, MVT::i64, Expand);
-
-  setOperationAction(ISD::EH_RETURN, MVT::Other, Custom);
-
-  setExceptionPointerRegister(Hexagon::R0);
-  setExceptionSelectorRegister(Hexagon::R1);
-
-  // VASTART needs to be custom lowered to use the VarArgsFrameIndex.
-  setOperationAction(ISD::VASTART, MVT::Other, Custom);
-
-  // Use the default implementation.
-  setOperationAction(ISD::VAARG, MVT::Other, Expand);
-  setOperationAction(ISD::VACOPY, MVT::Other, Expand);
-  setOperationAction(ISD::VAEND, MVT::Other, Expand);
-  setOperationAction(ISD::STACKSAVE, MVT::Other, Expand);
-  setOperationAction(ISD::STACKRESTORE, MVT::Other, Expand);
-
-  setOperationAction(ISD::DYNAMIC_STACKALLOC, MVT::i32, Custom);
-  setOperationAction(ISD::INLINEASM, MVT::Other, Custom);
-
-  setMinFunctionAlignment(2);
-
-  // Needed for DYNAMIC_STACKALLOC expansion.
-  const HexagonRegisterInfo *QRI = Subtarget->getRegisterInfo();
-  setStackPointerRegisterToSaveRestore(QRI->getStackRegister());
-  setSchedulingPreference(Sched::VLIW);
+  // These cause problems when the shift amount is non-constant.
+  setLibcallName(RTLIB::SHL_I128, nullptr);
+  setLibcallName(RTLIB::SRL_I128, nullptr);
+  setLibcallName(RTLIB::SRA_I128, nullptr);
 }
 
-const char*
-HexagonTargetLowering::getTargetNodeName(unsigned Opcode) const {
-  switch (Opcode) {
-  default: return nullptr;
-  case HexagonISD::CONST32:     return "HexagonISD::CONST32";
-  case HexagonISD::CONST32_GP: return "HexagonISD::CONST32_GP";
-  case HexagonISD::CONST32_Int_Real: return "HexagonISD::CONST32_Int_Real";
-  case HexagonISD::ADJDYNALLOC: return "HexagonISD::ADJDYNALLOC";
-  case HexagonISD::CMPICC:      return "HexagonISD::CMPICC";
-  case HexagonISD::CMPFCC:      return "HexagonISD::CMPFCC";
-  case HexagonISD::BRICC:       return "HexagonISD::BRICC";
-  case HexagonISD::BRFCC:       return "HexagonISD::BRFCC";
-  case HexagonISD::SELECT_ICC:  return "HexagonISD::SELECT_ICC";
-  case HexagonISD::SELECT_FCC:  return "HexagonISD::SELECT_FCC";
-  case HexagonISD::Hi:          return "HexagonISD::Hi";
-  case HexagonISD::Lo:          return "HexagonISD::Lo";
-  case HexagonISD::JT: return "HexagonISD::JT";
-  case HexagonISD::CP: return "HexagonISD::CP";
-  case HexagonISD::POPCOUNT: return "HexagonISD::POPCOUNT";
-  case HexagonISD::COMBINE: return "HexagonISD::COMBINE";
-  case HexagonISD::PACKHL: return "HexagonISD::PACKHL";
-  case HexagonISD::VSPLATB: return "HexagonISD::VSPLTB";
-  case HexagonISD::VSPLATH: return "HexagonISD::VSPLATH";
-  case HexagonISD::SHUFFEB: return "HexagonISD::SHUFFEB";
-  case HexagonISD::SHUFFEH: return "HexagonISD::SHUFFEH";
-  case HexagonISD::SHUFFOB: return "HexagonISD::SHUFFOB";
-  case HexagonISD::SHUFFOH: return "HexagonISD::SHUFFOH";
-  case HexagonISD::VSXTBH: return "HexagonISD::VSXTBH";
-  case HexagonISD::VSXTBW: return "HexagonISD::VSXTBW";
-  case HexagonISD::VSRAW: return "HexagonISD::VSRAW";
-  case HexagonISD::VSRAH: return "HexagonISD::VSRAH";
-  case HexagonISD::VSRLW: return "HexagonISD::VSRLW";
-  case HexagonISD::VSRLH: return "HexagonISD::VSRLH";
-  case HexagonISD::VSHLW: return "HexagonISD::VSHLW";
-  case HexagonISD::VSHLH: return "HexagonISD::VSHLH";
-  case HexagonISD::VCMPBEQ: return "HexagonISD::VCMPBEQ";
-  case HexagonISD::VCMPBGT: return "HexagonISD::VCMPBGT";
-  case HexagonISD::VCMPBGTU: return "HexagonISD::VCMPBGTU";
-  case HexagonISD::VCMPHEQ: return "HexagonISD::VCMPHEQ";
-  case HexagonISD::VCMPHGT: return "HexagonISD::VCMPHGT";
-  case HexagonISD::VCMPHGTU: return "HexagonISD::VCMPHGTU";
-  case HexagonISD::VCMPWEQ: return "HexagonISD::VCMPWEQ";
-  case HexagonISD::VCMPWGT: return "HexagonISD::VCMPWGT";
-  case HexagonISD::VCMPWGTU: return "HexagonISD::VCMPWGTU";
-  case HexagonISD::INSERT_ri: return "HexagonISD::INSERT_ri";
-  case HexagonISD::INSERT_rd: return "HexagonISD::INSERT_rd";
-  case HexagonISD::INSERT_riv: return "HexagonISD::INSERT_riv";
-  case HexagonISD::INSERT_rdv: return "HexagonISD::INSERT_rdv";
-  case HexagonISD::EXTRACTU_ri: return "HexagonISD::EXTRACTU_ri";
-  case HexagonISD::EXTRACTU_rd: return "HexagonISD::EXTRACTU_rd";
-  case HexagonISD::EXTRACTU_riv: return "HexagonISD::EXTRACTU_riv";
-  case HexagonISD::EXTRACTU_rdv: return "HexagonISD::EXTRACTU_rdv";
-  case HexagonISD::FTOI:        return "HexagonISD::FTOI";
-  case HexagonISD::ITOF:        return "HexagonISD::ITOF";
-  case HexagonISD::CALLv3:      return "HexagonISD::CALLv3";
-  case HexagonISD::CALLv3nr:    return "HexagonISD::CALLv3nr";
-  case HexagonISD::CALLR:       return "HexagonISD::CALLR";
-  case HexagonISD::RET_FLAG:    return "HexagonISD::RET_FLAG";
-  case HexagonISD::BR_JT:       return "HexagonISD::BR_JT";
-  case HexagonISD::TC_RETURN:   return "HexagonISD::TC_RETURN";
-  case HexagonISD::EH_RETURN: return "HexagonISD::EH_RETURN";
-  }
+
+const char* HexagonTargetLowering::getTargetNodeName(unsigned Opcode) const {
+  switch ((HexagonISD::NodeType)Opcode) {
+  case HexagonISD::ALLOCA:        return "HexagonISD::ALLOCA";
+  case HexagonISD::ARGEXTEND:     return "HexagonISD::ARGEXTEND";
+  case HexagonISD::AT_GOT:        return "HexagonISD::AT_GOT";
+  case HexagonISD::AT_PCREL:      return "HexagonISD::AT_PCREL";
+  case HexagonISD::BARRIER:       return "HexagonISD::BARRIER";
+  case HexagonISD::BR_JT:         return "HexagonISD::BR_JT";
+  case HexagonISD::CALLR:         return "HexagonISD::CALLR";
+  case HexagonISD::CALLv3nr:      return "HexagonISD::CALLv3nr";
+  case HexagonISD::CALLv3:        return "HexagonISD::CALLv3";
+  case HexagonISD::COMBINE:       return "HexagonISD::COMBINE";
+  case HexagonISD::CONST32_GP:    return "HexagonISD::CONST32_GP";
+  case HexagonISD::CONST32:       return "HexagonISD::CONST32";
+  case HexagonISD::CP:            return "HexagonISD::CP";
+  case HexagonISD::DCFETCH:       return "HexagonISD::DCFETCH";
+  case HexagonISD::EH_RETURN:     return "HexagonISD::EH_RETURN";
+  case HexagonISD::EXTRACTU:      return "HexagonISD::EXTRACTU";
+  case HexagonISD::EXTRACTURP:    return "HexagonISD::EXTRACTURP";
+  case HexagonISD::FCONST32:      return "HexagonISD::FCONST32";
+  case HexagonISD::INSERT:        return "HexagonISD::INSERT";
+  case HexagonISD::INSERTRP:      return "HexagonISD::INSERTRP";
+  case HexagonISD::JT:            return "HexagonISD::JT";
+  case HexagonISD::PACKHL:        return "HexagonISD::PACKHL";
+  case HexagonISD::PIC_ADD:       return "HexagonISD::PIC_ADD";
+  case HexagonISD::POPCOUNT:      return "HexagonISD::POPCOUNT";
+  case HexagonISD::RET_FLAG:      return "HexagonISD::RET_FLAG";
+  case HexagonISD::SHUFFEB:       return "HexagonISD::SHUFFEB";
+  case HexagonISD::SHUFFEH:       return "HexagonISD::SHUFFEH";
+  case HexagonISD::SHUFFOB:       return "HexagonISD::SHUFFOB";
+  case HexagonISD::SHUFFOH:       return "HexagonISD::SHUFFOH";
+  case HexagonISD::TC_RETURN:     return "HexagonISD::TC_RETURN";
+  case HexagonISD::VCMPBEQ:       return "HexagonISD::VCMPBEQ";
+  case HexagonISD::VCMPBGT:       return "HexagonISD::VCMPBGT";
+  case HexagonISD::VCMPBGTU:      return "HexagonISD::VCMPBGTU";
+  case HexagonISD::VCMPHEQ:       return "HexagonISD::VCMPHEQ";
+  case HexagonISD::VCMPHGT:       return "HexagonISD::VCMPHGT";
+  case HexagonISD::VCMPHGTU:      return "HexagonISD::VCMPHGTU";
+  case HexagonISD::VCMPWEQ:       return "HexagonISD::VCMPWEQ";
+  case HexagonISD::VCMPWGT:       return "HexagonISD::VCMPWGT";
+  case HexagonISD::VCMPWGTU:      return "HexagonISD::VCMPWGTU";
+  case HexagonISD::VSHLH:         return "HexagonISD::VSHLH";
+  case HexagonISD::VSHLW:         return "HexagonISD::VSHLW";
+  case HexagonISD::VSPLATB:       return "HexagonISD::VSPLTB";
+  case HexagonISD::VSPLATH:       return "HexagonISD::VSPLATH";
+  case HexagonISD::VSRAH:         return "HexagonISD::VSRAH";
+  case HexagonISD::VSRAW:         return "HexagonISD::VSRAW";
+  case HexagonISD::VSRLH:         return "HexagonISD::VSRLH";
+  case HexagonISD::VSRLW:         return "HexagonISD::VSRLW";
+  case HexagonISD::VSXTBH:        return "HexagonISD::VSXTBH";
+  case HexagonISD::VSXTBW:        return "HexagonISD::VSXTBW";
+  case HexagonISD::OP_END:        break;
+  }
+  return nullptr;
 }
 
-bool
-HexagonTargetLowering::isTruncateFree(Type *Ty1, Type *Ty2) const {
+bool HexagonTargetLowering::isTruncateFree(Type *Ty1, Type *Ty2) const {
   EVT MTy1 = EVT::getEVT(Ty1);
   EVT MTy2 = EVT::getEVT(Ty2);
-  if (!MTy1.isSimple() || !MTy2.isSimple()) {
+  if (!MTy1.isSimple() || !MTy2.isSimple())
     return false;
-  }
-  return ((MTy1.getSimpleVT() == MVT::i64) && (MTy2.getSimpleVT() == MVT::i32));
+  return (MTy1.getSimpleVT() == MVT::i64) && (MTy2.getSimpleVT() == MVT::i32);
 }
 
 bool HexagonTargetLowering::isTruncateFree(EVT VT1, EVT VT2) const {
-  if (!VT1.isSimple() || !VT2.isSimple()) {
+  if (!VT1.isSimple() || !VT2.isSimple())
     return false;
-  }
-  return ((VT1.getSimpleVT() == MVT::i64) && (VT2.getSimpleVT() == MVT::i32));
+  return (VT1.getSimpleVT() == MVT::i64) && (VT2.getSimpleVT() == MVT::i32);
 }
 
 // shouldExpandBuildVectorWithShuffles
@@ -1918,7 +1832,7 @@ static SDValue LowerVECTOR_SHUFFLE(SDValue Op, SelectionDAG &DAG) {
       if (IsScalarToVector)
         return createSplat(DAG, dl, VT, V1.getOperand(0));
     }
-    return createSplat(DAG, dl, VT, DAG.getConstant(Lane, MVT::i32));
+    return createSplat(DAG, dl, VT, DAG.getConstant(Lane, dl, MVT::i32));
   }
 
   // FIXME: We need to support more general vector shuffles.  See
@@ -2026,7 +1940,7 @@ HexagonTargetLowering::LowerBUILD_VECTOR(SDValue Op, SelectionDAG &DAG) const {
     unsigned SplatBits = APSplatBits.getZExtValue();
     int32_t SextVal = ((int32_t) (SplatBits << (32 - SplatBitSize)) >>
                        (32 - SplatBitSize));
-    return createSplat(DAG, dl, VT, DAG.getConstant(SextVal, MVT::i32));
+    return createSplat(DAG, dl, VT, DAG.getConstant(SextVal, dl, MVT::i32));
   }
 
   // Try to generate COMBINE to build v2i32 vectors.
@@ -2035,9 +1949,9 @@ HexagonTargetLowering::LowerBUILD_VECTOR(SDValue Op, SelectionDAG &DAG) const {
     SDValue V1 = BVN->getOperand(1);
 
     if (V0.getOpcode() == ISD::UNDEF)
-      V0 = DAG.getConstant(0, MVT::i32);
+      V0 = DAG.getConstant(0, dl, MVT::i32);
     if (V1.getOpcode() == ISD::UNDEF)
-      V1 = DAG.getConstant(0, MVT::i32);
+      V1 = DAG.getConstant(0, dl, MVT::i32);
 
     ConstantSDNode *C0 = dyn_cast<ConstantSDNode>(V0);
     ConstantSDNode *C1 = dyn_cast<ConstantSDNode>(V1);
@@ -2098,17 +2012,17 @@ HexagonTargetLowering::LowerBUILD_VECTOR(SDValue Op, SelectionDAG &DAG) const {
   }
 
   if (Size == 64)
-    ConstVal = DAG.getConstant(Res, MVT::i64);
+    ConstVal = DAG.getConstant(Res, dl, MVT::i64);
   else
-    ConstVal = DAG.getConstant(Res, MVT::i32);
+    ConstVal = DAG.getConstant(Res, dl, MVT::i32);
 
   // When there are non constant operands, add them with INSERT_VECTOR_ELT to
   // ConstVal, the constant part of the vector.
   if (HasNonConstantElements) {
     EVT EltVT = VT.getVectorElementType();
-    SDValue Width = DAG.getConstant(EltVT.getSizeInBits(), MVT::i64);
+    SDValue Width = DAG.getConstant(EltVT.getSizeInBits(), dl, MVT::i64);
     SDValue Shifted = DAG.getNode(ISD::SHL, dl, MVT::i64, Width,
-                                  DAG.getConstant(32, MVT::i64));
+                                  DAG.getConstant(32, dl, MVT::i64));
 
     for (unsigned i = 0, e = NElts; i != e; ++i) {
       // LLVM's BUILD_VECTOR operands are in Little Endian mode, whereas Hexagon
@@ -2121,19 +2035,19 @@ HexagonTargetLowering::LowerBUILD_VECTOR(SDValue Op, SelectionDAG &DAG) const {
 
       if (VT.getSizeInBits() == 64 &&
           Operand.getValueType().getSizeInBits() == 32) {
-        SDValue C = DAG.getConstant(0, MVT::i32);
+        SDValue C = DAG.getConstant(0, dl, MVT::i32);
         Operand = DAG.getNode(HexagonISD::COMBINE, dl, VT, C, Operand);
       }
 
-      SDValue Idx = DAG.getConstant(OpIdx, MVT::i64);
+      SDValue Idx = DAG.getConstant(OpIdx, dl, MVT::i64);
       SDValue Offset = DAG.getNode(ISD::MUL, dl, MVT::i64, Idx, Width);
       SDValue Combined = DAG.getNode(ISD::OR, dl, MVT::i64, Shifted, Offset);
       const SDValue Ops[] = {ConstVal, Operand, Combined};
 
       if (VT.getSizeInBits() == 32)
-        ConstVal = DAG.getNode(HexagonISD::INSERT_riv, dl, MVT::i32, Ops);
+        ConstVal = DAG.getNode(HexagonISD::INSERTRP, dl, MVT::i32, Ops);
       else
-        ConstVal = DAG.getNode(HexagonISD::INSERT_rdv, dl, MVT::i64, Ops);
+        ConstVal = DAG.getNode(HexagonISD::INSERTRP, dl, MVT::i64, Ops);
     }
   }
 
@@ -2148,10 +2062,10 @@ HexagonTargetLowering::LowerCONCAT_VECTORS(SDValue Op,
   unsigned NElts = Op.getNumOperands();
   SDValue Vec = Op.getOperand(0);
   EVT VecVT = Vec.getValueType();
-  SDValue Width = DAG.getConstant(VecVT.getSizeInBits(), MVT::i64);
+  SDValue Width = DAG.getConstant(VecVT.getSizeInBits(), dl, MVT::i64);
   SDValue Shifted = DAG.getNode(ISD::SHL, dl, MVT::i64, Width,
-                                DAG.getConstant(32, MVT::i64));
-  SDValue ConstVal = DAG.getConstant(0, MVT::i64);
+                                DAG.getConstant(32, dl, MVT::i64));
+  SDValue ConstVal = DAG.getConstant(0, dl, MVT::i64);
 
   ConstantSDNode *W = dyn_cast<ConstantSDNode>(Width);
   ConstantSDNode *S = dyn_cast<ConstantSDNode>(Shifted);
@@ -2180,19 +2094,19 @@ HexagonTargetLowering::LowerCONCAT_VECTORS(SDValue Op,
 
     if (VT.getSizeInBits() == 64 &&
         Operand.getValueType().getSizeInBits() == 32) {
-      SDValue C = DAG.getConstant(0, MVT::i32);
+      SDValue C = DAG.getConstant(0, dl, MVT::i32);
       Operand = DAG.getNode(HexagonISD::COMBINE, dl, VT, C, Operand);
     }
 
-    SDValue Idx = DAG.getConstant(OpIdx, MVT::i64);
+    SDValue Idx = DAG.getConstant(OpIdx, dl, MVT::i64);
     SDValue Offset = DAG.getNode(ISD::MUL, dl, MVT::i64, Idx, Width);
     SDValue Combined = DAG.getNode(ISD::OR, dl, MVT::i64, Shifted, Offset);
     const SDValue Ops[] = {ConstVal, Operand, Combined};
 
     if (VT.getSizeInBits() == 32)
-      ConstVal = DAG.getNode(HexagonISD::INSERT_riv, dl, MVT::i32, Ops);
+      ConstVal = DAG.getNode(HexagonISD::INSERTRP, dl, MVT::i32, Ops);
     else
-      ConstVal = DAG.getNode(HexagonISD::INSERT_rdv, dl, MVT::i64, Ops);
+      ConstVal = DAG.getNode(HexagonISD::INSERTRP, dl, MVT::i64, Ops);
   }
 
   return DAG.getNode(ISD::BITCAST, dl, VT, ConstVal);
@@ -2210,54 +2124,41 @@ HexagonTargetLowering::LowerEXTRACT_VECTOR(SDValue Op,
   EVT EltVT = VecVT.getVectorElementType();
   int EltSize = EltVT.getSizeInBits();
   SDValue Width = DAG.getConstant(Op.getOpcode() == ISD::EXTRACT_VECTOR_ELT ?
-                                  EltSize : VTN * EltSize, MVT::i64);
+                                  EltSize : VTN * EltSize, dl, MVT::i64);
 
   // Constant element number.
-  if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Idx)) {
-    SDValue Offset = DAG.getConstant(C->getZExtValue() * EltSize, MVT::i32);
+  if (ConstantSDNode *CI = dyn_cast<ConstantSDNode>(Idx)) {
+    uint64_t X = CI->getZExtValue();
+    SDValue Offset = DAG.getConstant(X * EltSize, dl, MVT::i32);
     const SDValue Ops[] = {Vec, Width, Offset};
 
-    ConstantSDNode *W = dyn_cast<ConstantSDNode>(Width);
-    assert(W && "Non constant width in LowerEXTRACT_VECTOR");
+    ConstantSDNode *CW = dyn_cast<ConstantSDNode>(Width);
+    assert(CW && "Non constant width in LowerEXTRACT_VECTOR");
 
     SDValue N;
-    // For certain extracts, it is a simple _hi/_lo subreg.
-    if (VecVT.getSimpleVT() == MVT::v2i32) {
-      // v2i32 -> i32 vselect.
-      if (C->getZExtValue() == 0)
-        N = DAG.getTargetExtractSubreg(Hexagon::subreg_loreg, dl,
-                                       MVT::i32, Vec);
-      else if (C->getZExtValue() == 1)
-        N = DAG.getTargetExtractSubreg(Hexagon::subreg_hireg, dl,
-                                       MVT::i32, Vec);
-      else
-        llvm_unreachable("Bad offset");
-    } else if ((VecVT.getSimpleVT() == MVT::v4i16) &&
-               (W->getZExtValue() == 32)) {
-      // v4i16 -> v2i16/i32 vselect.
-      if (C->getZExtValue() == 0)
-        N = DAG.getTargetExtractSubreg(Hexagon::subreg_loreg, dl,
-                                       MVT::i32, Vec);
-      else if (C->getZExtValue() == 2)
-        N = DAG.getTargetExtractSubreg(Hexagon::subreg_hireg, dl,
-                                       MVT::i32, Vec);
-      else
-        llvm_unreachable("Bad offset");
-    }  else if ((VecVT.getSimpleVT() == MVT::v8i8) &&
-               (W->getZExtValue() == 32)) {
-      // v8i8 -> v4i8/i32 vselect.
-      if (C->getZExtValue() == 0)
-        N = DAG.getTargetExtractSubreg(Hexagon::subreg_loreg, dl,
-                                       MVT::i32, Vec);
-      else if (C->getZExtValue() == 4)
-        N = DAG.getTargetExtractSubreg(Hexagon::subreg_hireg, dl,
-                                       MVT::i32, Vec);
+    MVT SVT = VecVT.getSimpleVT();
+    uint64_t W = CW->getZExtValue();
+
+    if (W == 32) {
+      // Translate this node into EXTRACT_SUBREG.
+      unsigned Subreg = (X == 0) ? Hexagon::subreg_loreg : 0;
+
+      if (X == 0)
+        Subreg = Hexagon::subreg_loreg;
+      else if (SVT == MVT::v2i32 && X == 1)
+        Subreg = Hexagon::subreg_hireg;
+      else if (SVT == MVT::v4i16 && X == 2)
+        Subreg = Hexagon::subreg_hireg;
+      else if (SVT == MVT::v8i8 && X == 4)
+        Subreg = Hexagon::subreg_hireg;
       else
         llvm_unreachable("Bad offset");
+      N = DAG.getTargetExtractSubreg(Subreg, dl, MVT::i32, Vec);
+
     } else if (VecVT.getSizeInBits() == 32) {
-        N = DAG.getNode(HexagonISD::EXTRACTU_ri, dl, MVT::i32, Ops);
+      N = DAG.getNode(HexagonISD::EXTRACTU, dl, MVT::i32, Ops);
     } else {
-      N = DAG.getNode(HexagonISD::EXTRACTU_rd, dl, MVT::i64, Ops);
+      N = DAG.getNode(HexagonISD::EXTRACTU, dl, MVT::i64, Ops);
       if (VT.getSizeInBits() == 32)
         N = DAG.getTargetExtractSubreg(Hexagon::subreg_loreg, dl, MVT::i32, N);
     }
@@ -2267,18 +2168,18 @@ HexagonTargetLowering::LowerEXTRACT_VECTOR(SDValue Op,
 
   // Variable element number.
   SDValue Offset = DAG.getNode(ISD::MUL, dl, MVT::i32, Idx,
-                               DAG.getConstant(EltSize, MVT::i32));
+                               DAG.getConstant(EltSize, dl, MVT::i32));
   SDValue Shifted = DAG.getNode(ISD::SHL, dl, MVT::i64, Width,
-                                DAG.getConstant(32, MVT::i64));
+                                DAG.getConstant(32, dl, MVT::i64));
   SDValue Combined = DAG.getNode(ISD::OR, dl, MVT::i64, Shifted, Offset);
 
   const SDValue Ops[] = {Vec, Combined};
 
   SDValue N;
   if (VecVT.getSizeInBits() == 32) {
-    N = DAG.getNode(HexagonISD::EXTRACTU_riv, dl, MVT::i32, Ops);
+    N = DAG.getNode(HexagonISD::EXTRACTURP, dl, MVT::i32, Ops);
   } else {
-    N = DAG.getNode(HexagonISD::EXTRACTU_rdv, dl, MVT::i64, Ops);
+    N = DAG.getNode(HexagonISD::EXTRACTURP, dl, MVT::i64, Ops);
     if (VT.getSizeInBits() == 32)
       N = DAG.getTargetExtractSubreg(Hexagon::subreg_loreg, dl, MVT::i32, N);
   }
@@ -2298,31 +2199,31 @@ HexagonTargetLowering::LowerINSERT_VECTOR(SDValue Op,
   EVT EltVT = VecVT.getVectorElementType();
   int EltSize = EltVT.getSizeInBits();
   SDValue Width = DAG.getConstant(Op.getOpcode() == ISD::INSERT_VECTOR_ELT ?
-                                  EltSize : VTN * EltSize, MVT::i64);
+                                  EltSize : VTN * EltSize, dl, MVT::i64);
 
   if (ConstantSDNode *C = cast<ConstantSDNode>(Idx)) {
-    SDValue Offset = DAG.getConstant(C->getSExtValue() * EltSize, MVT::i32);
+    SDValue Offset = DAG.getConstant(C->getSExtValue() * EltSize, dl, MVT::i32);
     const SDValue Ops[] = {Vec, Val, Width, Offset};
 
     SDValue N;
     if (VT.getSizeInBits() == 32)
-      N = DAG.getNode(HexagonISD::INSERT_ri, dl, MVT::i32, Ops);
+      N = DAG.getNode(HexagonISD::INSERT, dl, MVT::i32, Ops);
     else
-      N = DAG.getNode(HexagonISD::INSERT_rd, dl, MVT::i64, Ops);
+      N = DAG.getNode(HexagonISD::INSERT, dl, MVT::i64, Ops);
 
     return DAG.getNode(ISD::BITCAST, dl, VT, N);
   }
 
   // Variable element number.
   SDValue Offset = DAG.getNode(ISD::MUL, dl, MVT::i32, Idx,
-                               DAG.getConstant(EltSize, MVT::i32));
+                               DAG.getConstant(EltSize, dl, MVT::i32));
   SDValue Shifted = DAG.getNode(ISD::SHL, dl, MVT::i64, Width,
-                                DAG.getConstant(32, MVT::i64));
+                                DAG.getConstant(32, dl, MVT::i64));
   SDValue Combined = DAG.getNode(ISD::OR, dl, MVT::i64, Shifted, Offset);
 
   if (VT.getSizeInBits() == 64 &&
       Val.getValueType().getSizeInBits() == 32) {
-    SDValue C = DAG.getConstant(0, MVT::i32);
+    SDValue C = DAG.getConstant(0, dl, MVT::i32);
     Val = DAG.getNode(HexagonISD::COMBINE, dl, VT, C, Val);
   }
 
@@ -2330,9 +2231,9 @@ HexagonTargetLowering::LowerINSERT_VECTOR(SDValue Op,
 
   SDValue N;
   if (VT.getSizeInBits() == 32)
-    N = DAG.getNode(HexagonISD::INSERT_riv, dl, MVT::i32, Ops);
+    N = DAG.getNode(HexagonISD::INSERTRP, dl, MVT::i32, Ops);
   else
-    N = DAG.getNode(HexagonISD::INSERT_rdv, dl, MVT::i64, Ops);
+    N = DAG.getNode(HexagonISD::INSERTRP, dl, MVT::i64, Ops);
 
   return DAG.getNode(ISD::BITCAST, dl, VT, N);
 }
@@ -2356,6 +2257,7 @@ HexagonTargetLowering::LowerEH_RETURN(SDValue Op, SelectionDAG &DAG) const {
   SDValue Offset    = Op.getOperand(1);
   SDValue Handler   = Op.getOperand(2);
   SDLoc dl(Op);
+  auto PtrVT = getPointerTy(DAG.getDataLayout());
 
   // Mark function as containing a call to EH_RETURN.
   HexagonMachineFunctionInfo *FuncInfo =
@@ -2364,9 +2266,9 @@ HexagonTargetLowering::LowerEH_RETURN(SDValue Op, SelectionDAG &DAG) const {
 
   unsigned OffsetReg = Hexagon::R28;
 
-  SDValue StoreAddr = DAG.getNode(ISD::ADD, dl, getPointerTy(),
-                                  DAG.getRegister(Hexagon::R30, getPointerTy()),
-                                  DAG.getIntPtrConstant(4));
+  SDValue StoreAddr =
+      DAG.getNode(ISD::ADD, dl, PtrVT, DAG.getRegister(Hexagon::R30, PtrVT),
+                  DAG.getIntPtrConstant(4, dl));
   Chain = DAG.getStore(Chain, dl, Handler, StoreAddr, MachinePointerInfo(),
                        false, false, 0);
   Chain = DAG.getCopyToReg(Chain, dl, OffsetReg, Offset);
@@ -2379,60 +2281,54 @@ HexagonTargetLowering::LowerEH_RETURN(SDValue Op, SelectionDAG &DAG) const {
 
 SDValue
 HexagonTargetLowering::LowerOperation(SDValue Op, SelectionDAG &DAG) const {
-  switch (Op.getOpcode()) {
-    default: llvm_unreachable("Should not custom lower this!");
-    case ISD::CONCAT_VECTORS:     return LowerCONCAT_VECTORS(Op, DAG);
-    case ISD::INSERT_SUBVECTOR:   return LowerINSERT_VECTOR(Op, DAG);
-    case ISD::INSERT_VECTOR_ELT:  return LowerINSERT_VECTOR(Op, DAG);
-    case ISD::EXTRACT_SUBVECTOR:  return LowerEXTRACT_VECTOR(Op, DAG);
-    case ISD::EXTRACT_VECTOR_ELT: return LowerEXTRACT_VECTOR(Op, DAG);
-    case ISD::BUILD_VECTOR:       return LowerBUILD_VECTOR(Op, DAG);
-    case ISD::VECTOR_SHUFFLE:     return LowerVECTOR_SHUFFLE(Op, DAG);
+  unsigned Opc = Op.getOpcode();
+  switch (Opc) {
+    default:
+#ifndef NDEBUG
+      Op.getNode()->dumpr(&DAG);
+      if (Opc > HexagonISD::OP_BEGIN && Opc < HexagonISD::OP_END)
+        errs() << "Check for a non-legal type in this operation\n";
+#endif
+      llvm_unreachable("Should not custom lower this!");
+    case ISD::CONCAT_VECTORS:       return LowerCONCAT_VECTORS(Op, DAG);
+    case ISD::INSERT_SUBVECTOR:     return LowerINSERT_VECTOR(Op, DAG);
+    case ISD::INSERT_VECTOR_ELT:    return LowerINSERT_VECTOR(Op, DAG);
+    case ISD::EXTRACT_SUBVECTOR:    return LowerEXTRACT_VECTOR(Op, DAG);
+    case ISD::EXTRACT_VECTOR_ELT:   return LowerEXTRACT_VECTOR(Op, DAG);
+    case ISD::BUILD_VECTOR:         return LowerBUILD_VECTOR(Op, DAG);
+    case ISD::VECTOR_SHUFFLE:       return LowerVECTOR_SHUFFLE(Op, DAG);
     case ISD::SRA:
     case ISD::SHL:
-    case ISD::SRL:
-      return LowerVECTOR_SHIFT(Op, DAG);
-    case ISD::ConstantPool:
-      return LowerConstantPool(Op, DAG);
-    case ISD::EH_RETURN:          return LowerEH_RETURN(Op, DAG);
-      // Frame & Return address.  Currently unimplemented.
-    case ISD::RETURNADDR:         return LowerRETURNADDR(Op, DAG);
-    case ISD::FRAMEADDR:          return LowerFRAMEADDR(Op, DAG);
-    case ISD::GlobalTLSAddress:
-                          llvm_unreachable("TLS not implemented for Hexagon.");
-    case ISD::ATOMIC_FENCE:       return LowerATOMIC_FENCE(Op, DAG);
-    case ISD::GlobalAddress:      return LowerGLOBALADDRESS(Op, DAG);
-    case ISD::BlockAddress:       return LowerBlockAddress(Op, DAG);
-    case ISD::VASTART:            return LowerVASTART(Op, DAG);
-    case ISD::BR_JT:              return LowerBR_JT(Op, DAG);
+    case ISD::SRL:                  return LowerVECTOR_SHIFT(Op, DAG);
+    case ISD::ConstantPool:         return LowerConstantPool(Op, DAG);
+    case ISD::EH_RETURN:            return LowerEH_RETURN(Op, DAG);
+      // Frame & Return address. Currently unimplemented.
+    case ISD::RETURNADDR:           return LowerRETURNADDR(Op, DAG);
+    case ISD::FRAMEADDR:            return LowerFRAMEADDR(Op, DAG);
+    case ISD::ATOMIC_FENCE:         return LowerATOMIC_FENCE(Op, DAG);
+    case ISD::GlobalAddress:        return LowerGLOBALADDRESS(Op, DAG);
+    case ISD::BlockAddress:         return LowerBlockAddress(Op, DAG);
+    case ISD::VASTART:              return LowerVASTART(Op, DAG);
+    case ISD::BR_JT:                return LowerBR_JT(Op, DAG);
     // Custom lower some vector loads.
-    case ISD::LOAD:               return LowerLOAD(Op, DAG);
-
-    case ISD::DYNAMIC_STACKALLOC: return LowerDYNAMIC_STACKALLOC(Op, DAG);
-    case ISD::SELECT:             return Op;
-    case ISD::SETCC:              return LowerSETCC(Op, DAG);
-    case ISD::VSELECT:            return LowerVSELECT(Op, DAG);
-    case ISD::CTPOP:              return LowerCTPOP(Op, DAG);
-    case ISD::INTRINSIC_WO_CHAIN: return LowerINTRINSIC_WO_CHAIN(Op, DAG);
-    case ISD::INLINEASM:          return LowerINLINEASM(Op, DAG);
-
+    case ISD::LOAD:                 return LowerLOAD(Op, DAG);
+    case ISD::DYNAMIC_STACKALLOC:   return LowerDYNAMIC_STACKALLOC(Op, DAG);
+    case ISD::SETCC:                return LowerSETCC(Op, DAG);
+    case ISD::VSELECT:              return LowerVSELECT(Op, DAG);
+    case ISD::CTPOP:                return LowerCTPOP(Op, DAG);
+    case ISD::INTRINSIC_WO_CHAIN:   return LowerINTRINSIC_WO_CHAIN(Op, DAG);
+    case ISD::INLINEASM:            return LowerINLINEASM(Op, DAG);
   }
 }
 
-
-
-//===----------------------------------------------------------------------===//
-//                           Hexagon Scheduler Hooks
-//===----------------------------------------------------------------------===//
 MachineBasicBlock *
 HexagonTargetLowering::EmitInstrWithCustomInserter(MachineInstr *MI,
                                                    MachineBasicBlock *BB)
-const {
+      const {
   switch (MI->getOpcode()) {
-    case Hexagon::ADJDYNALLOC: {
+    case Hexagon::ALLOCA: {
       MachineFunction *MF = BB->getParent();
-      HexagonMachineFunctionInfo *FuncInfo =
-        MF->getInfo<HexagonMachineFunctionInfo>();
+      auto *FuncInfo = MF->getInfo<HexagonMachineFunctionInfo>();
       FuncInfo->addAllocaAdjustInst(MI);
       return BB;
     }
@@ -2446,8 +2342,7 @@ const {
 
 std::pair<unsigned, const TargetRegisterClass *>
 HexagonTargetLowering::getRegForInlineAsmConstraint(
-    const TargetRegisterInfo *TRI, const std::string &Constraint,
-    MVT VT) const {
+    const TargetRegisterInfo *TRI, StringRef Constraint, MVT VT) const {
   if (Constraint.size() == 1) {
     switch (Constraint[0]) {
     case 'r':   // R0-R31
@@ -2475,22 +2370,21 @@ HexagonTargetLowering::getRegForInlineAsmConstraint(
 /// specified FP immediate natively. If false, the legalizer will
 /// materialize the FP immediate as a load from a constant pool.
 bool HexagonTargetLowering::isFPImmLegal(const APFloat &Imm, EVT VT) const {
-  return Subtarget->hasV5TOps();
+  return Subtarget.hasV5TOps();
 }
 
 /// isLegalAddressingMode - Return true if the addressing mode represented by
 /// AM is legal for this target, for a load/store of the specified type.
-bool HexagonTargetLowering::isLegalAddressingMode(const AddrMode &AM,
-                                                  Type *Ty) const {
+bool HexagonTargetLowering::isLegalAddressingMode(const DataLayout &DL,
+                                                  const AddrMode &AM, Type *Ty,
+                                                  unsigned AS) const {
   // Allows a signed-extended 11-bit immediate field.
-  if (AM.BaseOffs <= -(1LL << 13) || AM.BaseOffs >= (1LL << 13)-1) {
+  if (AM.BaseOffs <= -(1LL << 13) || AM.BaseOffs >= (1LL << 13)-1)
     return false;
-  }
 
   // No global is ever allowed as a base.
-  if (AM.BaseGV) {
+  if (AM.BaseGV)
     return false;
-  }
 
   int Scale = AM.Scale;
   if (Scale < 0) Scale = -Scale;
@@ -2572,3 +2466,45 @@ bool llvm::isPositiveHalfWord(SDNode *N) {
     return true;
   }
 }
+
+Value *HexagonTargetLowering::emitLoadLinked(IRBuilder<> &Builder, Value *Addr,
+      AtomicOrdering Ord) const {
+  BasicBlock *BB = Builder.GetInsertBlock();
+  Module *M = BB->getParent()->getParent();
+  Type *Ty = cast<PointerType>(Addr->getType())->getElementType();
+  unsigned SZ = Ty->getPrimitiveSizeInBits();
+  assert((SZ == 32 || SZ == 64) && "Only 32/64-bit atomic loads supported");
+  Intrinsic::ID IntID = (SZ == 32) ? Intrinsic::hexagon_L2_loadw_locked
+                                   : Intrinsic::hexagon_L4_loadd_locked;
+  Value *Fn = Intrinsic::getDeclaration(M, IntID);
+  return Builder.CreateCall(Fn, Addr, "larx");
+}
+
+/// Perform a store-conditional operation to Addr. Return the status of the
+/// store. This should be 0 if the store succeeded, non-zero otherwise.
+Value *HexagonTargetLowering::emitStoreConditional(IRBuilder<> &Builder,
+      Value *Val, Value *Addr, AtomicOrdering Ord) const {
+  BasicBlock *BB = Builder.GetInsertBlock();
+  Module *M = BB->getParent()->getParent();
+  Type *Ty = Val->getType();
+  unsigned SZ = Ty->getPrimitiveSizeInBits();
+  assert((SZ == 32 || SZ == 64) && "Only 32/64-bit atomic stores supported");
+  Intrinsic::ID IntID = (SZ == 32) ? Intrinsic::hexagon_S2_storew_locked
+                                   : Intrinsic::hexagon_S4_stored_locked;
+  Value *Fn = Intrinsic::getDeclaration(M, IntID);
+  Value *Call = Builder.CreateCall(Fn, {Addr, Val}, "stcx");
+  Value *Cmp = Builder.CreateICmpEQ(Call, Builder.getInt32(0), "");
+  Value *Ext = Builder.CreateZExt(Cmp, Type::getInt32Ty(M->getContext()));
+  return Ext;
+}
+
+bool HexagonTargetLowering::shouldExpandAtomicLoadInIR(LoadInst *LI) const {
+  // Do not expand loads and stores that don't exceed 64 bits.
+  return LI->getType()->getPrimitiveSizeInBits() > 64;
+}
+
+bool HexagonTargetLowering::shouldExpandAtomicStoreInIR(StoreInst *SI) const {
+  // Do not expand loads and stores that don't exceed 64 bits.
+  return SI->getValueOperand()->getType()->getPrimitiveSizeInBits() > 64;
+}
+
diff --git a/lib/Target/Hexagon/HexagonISelLowering.h b/lib/Target/Hexagon/HexagonISelLowering.h
index 34b1ebb4d3e..2642abffadd 100644
--- a/lib/Target/Hexagon/HexagonISelLowering.h
+++ b/lib/Target/Hexagon/HexagonISelLowering.h
@@ -26,33 +26,19 @@ namespace llvm {
 bool isPositiveHalfWord(SDNode *N);
 
   namespace HexagonISD {
-    enum {
-      FIRST_NUMBER = ISD::BUILTIN_OP_END,
+    enum NodeType : unsigned {
+      OP_BEGIN = ISD::BUILTIN_OP_END,
 
-      CONST32,
+      CONST32 = OP_BEGIN,
       CONST32_GP,  // For marking data present in GP.
-      CONST32_Int_Real,
       FCONST32,
-      SETCC,
-      ADJDYNALLOC,
+      ALLOCA,
       ARGEXTEND,
 
       PIC_ADD,
       AT_GOT,
       AT_PCREL,
 
-      CMPICC,      // Compare two GPR operands, set icc.
-      CMPFCC,      // Compare two FP operands, set fcc.
-      BRICC,       // Branch to dest on icc condition
-      BRFCC,       // Branch to dest on fcc condition
-      SELECT_ICC,  // Select between two values using the current ICC flags.
-      SELECT_FCC,  // Select between two values using the current FCC flags.
-
-      Hi, Lo,      // Hi/Lo operations, typically on a global address.
-
-      FTOI,        // FP to Int within a FP register.
-      ITOF,        // Int to FP within a FP register.
-
       CALLv3,      // A V3+ call instruction.
       CALLv3nr,    // A V3+ call instruction that doesn't return.
       CALLR,
@@ -62,6 +48,7 @@ bool isPositiveHalfWord(SDNode *N);
       BARRIER,     // Memory barrier.
       JT,          // Jump table.
       CP,          // Constant pool.
+
       POPCOUNT,
       COMBINE,
       PACKHL,
@@ -88,17 +75,16 @@ bool isPositiveHalfWord(SDNode *N);
       VCMPWEQ,
       VCMPWGT,
       VCMPWGTU,
-      INSERT_ri,
-      INSERT_rd,
-      INSERT_riv,
-      INSERT_rdv,
-      EXTRACTU_ri,
-      EXTRACTU_rd,
-      EXTRACTU_riv,
-      EXTRACTU_rdv,
+
+      INSERT,
+      INSERTRP,
+      EXTRACTU,
+      EXTRACTURP,
       TC_RETURN,
       EH_RETURN,
-      DCFETCH
+      DCFETCH,
+
+      OP_END
     };
   }
 
@@ -107,30 +93,24 @@ bool isPositiveHalfWord(SDNode *N);
   class HexagonTargetLowering : public TargetLowering {
     int VarArgsFrameOffset;   // Frame offset to start of varargs area.
 
-    bool CanReturnSmallStruct(const Function* CalleeFn,
-                              unsigned& RetSize) const;
-
+    bool CanReturnSmallStruct(const Function* CalleeFn, unsigned& RetSize)
+        const;
     void promoteLdStType(EVT VT, EVT PromotedLdStVT);
+    const HexagonTargetMachine &HTM;
+    const HexagonSubtarget &Subtarget;
 
   public:
-    const HexagonSubtarget *Subtarget;
     explicit HexagonTargetLowering(const TargetMachine &TM,
-                                   const HexagonSubtarget &Subtarget);
+                                   const HexagonSubtarget &ST);
 
     /// IsEligibleForTailCallOptimization - Check whether the call is eligible
     /// for tail call optimization. Targets which want to do tail call
     /// optimization should implement this function.
-    bool
-    IsEligibleForTailCallOptimization(SDValue Callee,
-                                      CallingConv::ID CalleeCC,
-                                      bool isVarArg,
-                                      bool isCalleeStructRet,
-                                      bool isCallerStructRet,
-                                      const
-                                      SmallVectorImpl<ISD::OutputArg> &Outs,
-                                      const SmallVectorImpl<SDValue> &OutVals,
-                                      const SmallVectorImpl<ISD::InputArg> &Ins,
-                                      SelectionDAG& DAG) const;
+    bool IsEligibleForTailCallOptimization(SDValue Callee,
+        CallingConv::ID CalleeCC, bool isVarArg, bool isCalleeStructRet,
+        bool isCallerStructRet, const SmallVectorImpl<ISD::OutputArg> &Outs,
+        const SmallVectorImpl<SDValue> &OutVals,
+        const SmallVectorImpl<ISD::InputArg> &Ins, SelectionDAG& DAG) const;
 
     bool isTruncateFree(Type *Ty1, Type *Ty2) const override;
     bool isTruncateFree(EVT VT1, EVT VT2) const override;
@@ -139,7 +119,7 @@ bool isPositiveHalfWord(SDNode *N);
 
     // Should we expand the build vector with shuffles?
     bool shouldExpandBuildVectorWithShuffles(EVT VT,
-                                        unsigned DefinedValues) const override;
+        unsigned DefinedValues) const override;
 
     SDValue LowerOperation(SDValue Op, SelectionDAG &DAG) const override;
     const char *getTargetNodeName(unsigned Opcode) const override;
@@ -152,24 +132,19 @@ bool isPositiveHalfWord(SDNode *N);
     SDValue LowerINLINEASM(SDValue Op, SelectionDAG &DAG) const;
     SDValue LowerEH_LABEL(SDValue Op, SelectionDAG &DAG) const;
     SDValue LowerEH_RETURN(SDValue Op, SelectionDAG &DAG) const;
-    SDValue LowerFormalArguments(SDValue Chain,
-                                 CallingConv::ID CallConv, bool isVarArg,
-                                 const SmallVectorImpl<ISD::InputArg> &Ins,
-                                 SDLoc dl, SelectionDAG &DAG,
-                                 SmallVectorImpl<SDValue> &InVals) const override;
+    SDValue LowerFormalArguments(SDValue Chain, CallingConv::ID CallConv,
+        bool isVarArg, const SmallVectorImpl<ISD::InputArg> &Ins, SDLoc dl,
+        SelectionDAG &DAG, SmallVectorImpl<SDValue> &InVals) const override;
     SDValue LowerGLOBALADDRESS(SDValue Op, SelectionDAG &DAG) const;
     SDValue LowerBlockAddress(SDValue Op, SelectionDAG &DAG) const;
 
     SDValue LowerCall(TargetLowering::CallLoweringInfo &CLI,
-                      SmallVectorImpl<SDValue> &InVals) const override;
-
+        SmallVectorImpl<SDValue> &InVals) const override;
     SDValue LowerCallResult(SDValue Chain, SDValue InFlag,
-                            CallingConv::ID CallConv, bool isVarArg,
-                            const SmallVectorImpl<ISD::InputArg> &Ins,
-                            SDLoc dl, SelectionDAG &DAG,
-                            SmallVectorImpl<SDValue> &InVals,
-                            const SmallVectorImpl<SDValue> &OutVals,
-                            SDValue Callee) const;
+        CallingConv::ID CallConv, bool isVarArg,
+        const SmallVectorImpl<ISD::InputArg> &Ins, SDLoc dl,
+        SelectionDAG &DAG, SmallVectorImpl<SDValue> &InVals,
+        const SmallVectorImpl<SDValue> &OutVals, SDValue Callee) const;
 
     SDValue LowerSETCC(SDValue Op, SelectionDAG &DAG) const;
     SDValue LowerVSELECT(SDValue Op, SelectionDAG &DAG) const;
@@ -179,19 +154,19 @@ bool isPositiveHalfWord(SDNode *N);
     SDValue LowerRETURNADDR(SDValue Op, SelectionDAG &DAG) const;
     SDValue LowerLOAD(SDValue Op, SelectionDAG &DAG) const;
 
-    SDValue LowerReturn(SDValue Chain,
-                        CallingConv::ID CallConv, bool isVarArg,
-                        const SmallVectorImpl<ISD::OutputArg> &Outs,
-                        const SmallVectorImpl<SDValue> &OutVals,
-                        SDLoc dl, SelectionDAG &DAG) const override;
+    SDValue LowerReturn(SDValue Chain, CallingConv::ID CallConv,
+        bool isVarArg, const SmallVectorImpl<ISD::OutputArg> &Outs,
+        const SmallVectorImpl<SDValue> &OutVals, SDLoc dl,
+        SelectionDAG &DAG) const override;
 
-    MachineBasicBlock *
-    EmitInstrWithCustomInserter(MachineInstr *MI,
-                                MachineBasicBlock *BB) const override;
+    bool mayBeEmittedAsTailCall(CallInst *CI) const override;
+    MachineBasicBlock * EmitInstrWithCustomInserter(MachineInstr *MI,
+        MachineBasicBlock *BB) const override;
 
-    SDValue  LowerVASTART(SDValue Op, SelectionDAG &DAG) const;
-    SDValue  LowerConstantPool(SDValue Op, SelectionDAG &DAG) const;
-    EVT getSetCCResultType(LLVMContext &C, EVT VT) const override {
+    SDValue LowerVASTART(SDValue Op, SelectionDAG &DAG) const;
+    SDValue LowerConstantPool(SDValue Op, SelectionDAG &DAG) const;
+    EVT getSetCCResultType(const DataLayout &, LLVMContext &C,
+                           EVT VT) const override {
       if (!VT.isVector())
         return MVT::i1;
       else
@@ -205,11 +180,10 @@ bool isPositiveHalfWord(SDNode *N);
 
     std::pair<unsigned, const TargetRegisterClass *>
     getRegForInlineAsmConstraint(const TargetRegisterInfo *TRI,
-                                 const std::string &Constraint,
-                                 MVT VT) const override;
+                                 StringRef Constraint, MVT VT) const override;
 
-    unsigned getInlineAsmMemConstraint(
-        const std::string &ConstraintCode) const override {
+    unsigned
+    getInlineAsmMemConstraint(StringRef ConstraintCode) const override {
       if (ConstraintCode == "o")
         return InlineAsm::Constraint_o;
       else if (ConstraintCode == "v")
@@ -224,7 +198,8 @@ bool isPositiveHalfWord(SDNode *N);
     /// The type may be VoidTy, in which case only return true if the addressing
     /// mode is legal for a load/store of any legal type.
     /// TODO: Handle pre/postinc as well.
-    bool isLegalAddressingMode(const AddrMode &AM, Type *Ty) const override;
+    bool isLegalAddressingMode(const DataLayout &DL, const AddrMode &AM,
+                               Type *Ty, unsigned AS) const override;
     bool isFPImmLegal(const APFloat &Imm, EVT VT) const override;
 
     /// isLegalICmpImmediate - Return true if the specified immediate is legal
@@ -232,6 +207,21 @@ bool isPositiveHalfWord(SDNode *N);
     /// compare a register against the immediate without having to materialize
     /// the immediate into a register.
     bool isLegalICmpImmediate(int64_t Imm) const override;
+
+    // Handling of atomic RMW instructions.
+    bool hasLoadLinkedStoreConditional() const override {
+      return true;
+    }
+    Value *emitLoadLinked(IRBuilder<> &Builder, Value *Addr,
+        AtomicOrdering Ord) const override;
+    Value *emitStoreConditional(IRBuilder<> &Builder, Value *Val,
+        Value *Addr, AtomicOrdering Ord) const override;
+    bool shouldExpandAtomicLoadInIR(LoadInst *LI) const override;
+    bool shouldExpandAtomicStoreInIR(StoreInst *SI) const override;
+    AtomicRMWExpansionKind shouldExpandAtomicRMWInIR(AtomicRMWInst *AI)
+        const override {
+      return AtomicRMWExpansionKind::LLSC;
+    }
   };
 } // end namespace llvm
 
diff --git a/lib/Target/Hexagon/HexagonInstrFormats.td b/lib/Target/Hexagon/HexagonInstrFormats.td
index 36a7e9f642c..44bab292f32 100644
--- a/lib/Target/Hexagon/HexagonInstrFormats.td
+++ b/lib/Target/Hexagon/HexagonInstrFormats.td
@@ -66,10 +66,8 @@ def DoubleWordAccess : MemAccessSize<4>;// Double word access instruction (memd)
 class OpcodeHexagon {
   field bits<32> Inst = ?; // Default to an invalid insn.
   bits<4> IClass = 0; // ICLASS
-  bits<2> IParse = 0; // Parse bits.
 
   let Inst{31-28} = IClass;
-  let Inst{15-14} = IParse;
 
   bits<1> zero = 0;
 }
diff --git a/lib/Target/Hexagon/HexagonInstrFormatsV4.td b/lib/Target/Hexagon/HexagonInstrFormatsV4.td
index 7f7b2c96dba..db83ef6bc47 100644
--- a/lib/Target/Hexagon/HexagonInstrFormatsV4.td
+++ b/lib/Target/Hexagon/HexagonInstrFormatsV4.td
@@ -146,6 +146,11 @@ class EXTENDERInst<dag outs, dag ins, string asmstr, list<dag> pattern = []>
   : InstHexagon<outs, ins, asmstr, pattern, "", EXTENDER_tc_1_SLOT0123,
                 TypePREFIX>, OpcodeHexagon;
 
+class SUBInst<dag outs, dag ins, string asmstr, list<dag> pattern = [],
+              string cstr = "">
+  : InstHexagon<outs, ins, asmstr, pattern, "", PREFIX, TypeDUPLEX>,
+    OpcodeHexagon;
+
 class CJInst<dag outs, dag ins, string asmstr, list<dag> pattern = [],
               string cstr = "">
   : InstHexagon<outs, ins, asmstr, pattern, cstr, COMPOUND, TypeCOMPOUND>,
diff --git a/lib/Target/Hexagon/HexagonInstrInfo.cpp b/lib/Target/Hexagon/HexagonInstrInfo.cpp
index ff4bcadaabb..3cb082349b4 100644
--- a/lib/Target/Hexagon/HexagonInstrInfo.cpp
+++ b/lib/Target/Hexagon/HexagonInstrInfo.cpp
@@ -115,72 +115,172 @@ unsigned HexagonInstrInfo::isStoreToStackSlot(const MachineInstr *MI,
   return 0;
 }
 
+// Find the hardware loop instruction used to set-up the specified loop.
+// On Hexagon, we have two instructions used to set-up the hardware loop
+// (LOOP0, LOOP1) with corresponding endloop (ENDLOOP0, ENDLOOP1) instructions
+// to indicate the end of a loop.
+static MachineInstr *
+findLoopInstr(MachineBasicBlock *BB, int EndLoopOp,
+              SmallPtrSet<MachineBasicBlock *, 8> &Visited) {
+  int LOOPi;
+  int LOOPr;
+  if (EndLoopOp == Hexagon::ENDLOOP0) {
+    LOOPi = Hexagon::J2_loop0i;
+    LOOPr = Hexagon::J2_loop0r;
+  } else { // EndLoopOp == Hexagon::EndLOOP1
+    LOOPi = Hexagon::J2_loop1i;
+    LOOPr = Hexagon::J2_loop1r;
+  }
 
-unsigned
-HexagonInstrInfo::InsertBranch(MachineBasicBlock &MBB,MachineBasicBlock *TBB,
-                             MachineBasicBlock *FBB,
-                             const SmallVectorImpl<MachineOperand> &Cond,
-                             DebugLoc DL) const{
-
-    int BOpc   = Hexagon::J2_jump;
-    int BccOpc = Hexagon::J2_jumpt;
-
-    assert(TBB && "InsertBranch must not be told to insert a fallthrough");
-
-    int regPos = 0;
-    // Check if ReverseBranchCondition has asked to reverse this branch
-    // If we want to reverse the branch an odd number of times, we want
-    // JMP_f.
-    if (!Cond.empty() && Cond[0].isImm() && Cond[0].getImm() == 0) {
-      BccOpc = Hexagon::J2_jumpf;
-      regPos = 1;
+  // The loop set-up instruction will be in a predecessor block
+  for (MachineBasicBlock::pred_iterator PB = BB->pred_begin(),
+         PE = BB->pred_end(); PB != PE; ++PB) {
+    // If this has been visited, already skip it.
+    if (!Visited.insert(*PB).second)
+      continue;
+    if (*PB == BB)
+      continue;
+    for (MachineBasicBlock::reverse_instr_iterator I = (*PB)->instr_rbegin(),
+           E = (*PB)->instr_rend(); I != E; ++I) {
+      int Opc = I->getOpcode();
+      if (Opc == LOOPi || Opc == LOOPr)
+        return &*I;
+      // We've reached a different loop, which means the loop0 has been removed.
+      if (Opc == EndLoopOp)
+        return 0;
     }
+    // Check the predecessors for the LOOP instruction.
+    MachineInstr *loop = findLoopInstr(*PB, EndLoopOp, Visited);
+    if (loop)
+      return loop;
+  }
+  return 0;
+}
 
-    if (!FBB) {
-      if (Cond.empty()) {
-        // Due to a bug in TailMerging/CFG Optimization, we need to add a
-        // special case handling of a predicated jump followed by an
-        // unconditional jump. If not, Tail Merging and CFG Optimization go
-        // into an infinite loop.
-        MachineBasicBlock *NewTBB, *NewFBB;
-        SmallVector<MachineOperand, 4> Cond;
-        MachineInstr *Term = MBB.getFirstTerminator();
-        if (isPredicated(Term) && !AnalyzeBranch(MBB, NewTBB, NewFBB, Cond,
-                                                 false)) {
-          MachineBasicBlock *NextBB =
-            std::next(MachineFunction::iterator(&MBB));
-          if (NewTBB == NextBB) {
-            ReverseBranchCondition(Cond);
-            RemoveBranch(MBB);
-            return InsertBranch(MBB, TBB, nullptr, Cond, DL);
-          }
+unsigned HexagonInstrInfo::InsertBranch(
+    MachineBasicBlock &MBB,MachineBasicBlock *TBB, MachineBasicBlock *FBB,
+    ArrayRef<MachineOperand> Cond, DebugLoc DL) const {
+
+  Opcode_t BOpc   = Hexagon::J2_jump;
+  Opcode_t BccOpc = Hexagon::J2_jumpt;
+
+  assert(TBB && "InsertBranch must not be told to insert a fallthrough");
+
+  // Check if ReverseBranchCondition has asked to reverse this branch
+  // If we want to reverse the branch an odd number of times, we want
+  // J2_jumpf.
+  if (!Cond.empty() && Cond[0].isImm())
+    BccOpc = Cond[0].getImm();
+
+  if (!FBB) {
+    if (Cond.empty()) {
+      // Due to a bug in TailMerging/CFG Optimization, we need to add a
+      // special case handling of a predicated jump followed by an
+      // unconditional jump. If not, Tail Merging and CFG Optimization go
+      // into an infinite loop.
+      MachineBasicBlock *NewTBB, *NewFBB;
+      SmallVector<MachineOperand, 4> Cond;
+      MachineInstr *Term = MBB.getFirstTerminator();
+      if (Term != MBB.end() && isPredicated(Term) &&
+          !AnalyzeBranch(MBB, NewTBB, NewFBB, Cond, false)) {
+        MachineBasicBlock *NextBB =
+          std::next(MachineFunction::iterator(&MBB));
+        if (NewTBB == NextBB) {
+          ReverseBranchCondition(Cond);
+          RemoveBranch(MBB);
+          return InsertBranch(MBB, TBB, nullptr, Cond, DL);
         }
-        BuildMI(&MBB, DL, get(BOpc)).addMBB(TBB);
-      } else {
-        // If Cond[0] is a basic block, insert ENDLOOP0.
-        if (Cond[0].isMBB())
-          BuildMI(&MBB, DL, get(Hexagon::ENDLOOP0)).addMBB(Cond[0].getMBB());
-        else
-          BuildMI(&MBB, DL,
-                  get(BccOpc)).addReg(Cond[regPos].getReg()).addMBB(TBB);
       }
-      return 1;
+      BuildMI(&MBB, DL, get(BOpc)).addMBB(TBB);
+    } else if (isEndLoopN(Cond[0].getImm())) {
+      int EndLoopOp = Cond[0].getImm();
+      assert(Cond[1].isMBB());
+      // Since we're adding an ENDLOOP, there better be a LOOP instruction.
+      // Check for it, and change the BB target if needed.
+      SmallPtrSet<MachineBasicBlock *, 8> VisitedBBs;
+      MachineInstr *Loop = findLoopInstr(TBB, EndLoopOp, VisitedBBs);
+      assert(Loop != 0 && "Inserting an ENDLOOP without a LOOP");
+      Loop->getOperand(0).setMBB(TBB);
+      // Add the ENDLOOP after the finding the LOOP0.
+      BuildMI(&MBB, DL, get(EndLoopOp)).addMBB(TBB);
+    } else if (isNewValueJump(Cond[0].getImm())) {
+      assert((Cond.size() == 3) && "Only supporting rr/ri version of nvjump");
+      // New value jump
+      // (ins IntRegs:$src1, IntRegs:$src2, brtarget:$offset)
+      // (ins IntRegs:$src1, u5Imm:$src2, brtarget:$offset)
+      unsigned Flags1 = getUndefRegState(Cond[1].isUndef());
+      DEBUG(dbgs() << "\nInserting NVJump for BB#" << MBB.getNumber(););
+      if (Cond[2].isReg()) {
+        unsigned Flags2 = getUndefRegState(Cond[2].isUndef());
+        BuildMI(&MBB, DL, get(BccOpc)).addReg(Cond[1].getReg(), Flags1).
+          addReg(Cond[2].getReg(), Flags2).addMBB(TBB);
+      } else if(Cond[2].isImm()) {
+        BuildMI(&MBB, DL, get(BccOpc)).addReg(Cond[1].getReg(), Flags1).
+          addImm(Cond[2].getImm()).addMBB(TBB);
+      } else
+        llvm_unreachable("Invalid condition for branching");
+    } else {
+      assert((Cond.size() == 2) && "Malformed cond vector");
+      const MachineOperand &RO = Cond[1];
+      unsigned Flags = getUndefRegState(RO.isUndef());
+      BuildMI(&MBB, DL, get(BccOpc)).addReg(RO.getReg(), Flags).addMBB(TBB);
     }
-
-    // We don't handle ENDLOOP0 with a conditional branch in AnalyzeBranch.
-    BuildMI(&MBB, DL, get(BccOpc)).addReg(Cond[regPos].getReg()).addMBB(TBB);
-    BuildMI(&MBB, DL, get(BOpc)).addMBB(FBB);
-    return 2;
-}
-
-
+    return 1;
+  }
+  assert((!Cond.empty()) &&
+         "Cond. cannot be empty when multiple branchings are required");
+  assert((!isNewValueJump(Cond[0].getImm())) &&
+         "NV-jump cannot be inserted with another branch");
+  // Special case for hardware loops.  The condition is a basic block.
+  if (isEndLoopN(Cond[0].getImm())) {
+    int EndLoopOp = Cond[0].getImm();
+    assert(Cond[1].isMBB());
+    // Since we're adding an ENDLOOP, there better be a LOOP instruction.
+    // Check for it, and change the BB target if needed.
+    SmallPtrSet<MachineBasicBlock *, 8> VisitedBBs;
+    MachineInstr *Loop = findLoopInstr(TBB, EndLoopOp, VisitedBBs);
+    assert(Loop != 0 && "Inserting an ENDLOOP without a LOOP");
+    Loop->getOperand(0).setMBB(TBB);
+    // Add the ENDLOOP after the finding the LOOP0.
+    BuildMI(&MBB, DL, get(EndLoopOp)).addMBB(TBB);
+  } else {
+    const MachineOperand &RO = Cond[1];
+    unsigned Flags = getUndefRegState(RO.isUndef());
+    BuildMI(&MBB, DL, get(BccOpc)).addReg(RO.getReg(), Flags).addMBB(TBB);
+  }
+  BuildMI(&MBB, DL, get(BOpc)).addMBB(FBB);
+
+  return 2;
+}
+
+
+/// This function can analyze one/two way branching only and should (mostly) be
+/// called by target independent side.
+/// First entry is always the opcode of the branching instruction, except when
+/// the Cond vector is supposed to be empty, e.g., when AnalyzeBranch fails, a
+/// BB with only unconditional jump. Subsequent entries depend upon the opcode,
+/// e.g. Jump_c p will have
+/// Cond[0] = Jump_c
+/// Cond[1] = p
+/// HW-loop ENDLOOP:
+/// Cond[0] = ENDLOOP
+/// Cond[1] = MBB
+/// New value jump:
+/// Cond[0] = Hexagon::CMPEQri_f_Jumpnv_t_V4 -- specific opcode
+/// Cond[1] = R
+/// Cond[2] = Imm
+/// @note Related function is \fn findInstrPredicate which fills in
+/// Cond. vector when a predicated instruction is passed to it.
+/// We follow same protocol in that case too.
+///
 bool HexagonInstrInfo::AnalyzeBranch(MachineBasicBlock &MBB,
                                      MachineBasicBlock *&TBB,
-                                 MachineBasicBlock *&FBB,
-                                 SmallVectorImpl<MachineOperand> &Cond,
-                                 bool AllowModify) const {
+                                     MachineBasicBlock *&FBB,
+                                     SmallVectorImpl<MachineOperand> &Cond,
+                                     bool AllowModify) const {
   TBB = nullptr;
   FBB = nullptr;
+  Cond.clear();
 
   // If the block has no terminators, it just falls into the block after it.
   MachineBasicBlock::instr_iterator I = MBB.instr_end();
@@ -202,6 +302,7 @@ bool HexagonInstrInfo::AnalyzeBranch(MachineBasicBlock &MBB,
   do {
     --I;
     if (I->isEHLabel())
+      // Don't analyze EH branches.
       return true;
   } while (I != MBB.instr_begin());
 
@@ -216,7 +317,7 @@ bool HexagonInstrInfo::AnalyzeBranch(MachineBasicBlock &MBB,
   
   bool JumpToBlock = I->getOpcode() == Hexagon::J2_jump &&
                      I->getOperand(0).isMBB();
-  // Delete the JMP if it's equivalent to a fall-through.
+  // Delete the J2_jump if it's equivalent to a fall-through.
   if (AllowModify && JumpToBlock &&
       MBB.isLayoutSuccessor(I->getOperand(0).getMBB())) {
     DEBUG(dbgs()<< "\nErasing the jump to successor block\n";);
@@ -257,7 +358,7 @@ bool HexagonInstrInfo::AnalyzeBranch(MachineBasicBlock &MBB,
     return true;
 
   bool LastOpcodeHasJMP_c = PredOpcodeHasJMP_c(LastOpcode);
-  bool LastOpcodeHasNot = PredOpcodeHasNot(LastOpcode);
+  bool LastOpcodeHasNVJump = isNewValueJump(LastInst);
 
   // If there is only one terminator instruction, process it.
   if (LastInst && !SecondLastInst) {
@@ -265,34 +366,54 @@ bool HexagonInstrInfo::AnalyzeBranch(MachineBasicBlock &MBB,
       TBB = LastInst->getOperand(0).getMBB();
       return false;
     }
-    if (LastOpcode == Hexagon::ENDLOOP0) {
+    if (isEndLoopN(LastOpcode)) {
       TBB = LastInst->getOperand(0).getMBB();
+      Cond.push_back(MachineOperand::CreateImm(LastInst->getOpcode()));
       Cond.push_back(LastInst->getOperand(0));
       return false;
     }
     if (LastOpcodeHasJMP_c) {
       TBB = LastInst->getOperand(1).getMBB();
-      if (LastOpcodeHasNot) {
-        Cond.push_back(MachineOperand::CreateImm(0));
-      }
+      Cond.push_back(MachineOperand::CreateImm(LastInst->getOpcode()));
       Cond.push_back(LastInst->getOperand(0));
       return false;
     }
+    // Only supporting rr/ri versions of new-value jumps.
+    if (LastOpcodeHasNVJump && (LastInst->getNumExplicitOperands() == 3)) {
+      TBB = LastInst->getOperand(2).getMBB();
+      Cond.push_back(MachineOperand::CreateImm(LastInst->getOpcode()));
+      Cond.push_back(LastInst->getOperand(0));
+      Cond.push_back(LastInst->getOperand(1));
+      return false;
+    }
+    DEBUG(dbgs() << "\nCant analyze BB#" << MBB.getNumber()
+                 << " with one jump\n";);
     // Otherwise, don't know what this is.
     return true;
   }
 
   bool SecLastOpcodeHasJMP_c = PredOpcodeHasJMP_c(SecLastOpcode);
-  bool SecLastOpcodeHasNot = PredOpcodeHasNot(SecLastOpcode);
+  bool SecLastOpcodeHasNVJump = isNewValueJump(SecondLastInst);
   if (SecLastOpcodeHasJMP_c && (LastOpcode == Hexagon::J2_jump)) {
     TBB =  SecondLastInst->getOperand(1).getMBB();
-    if (SecLastOpcodeHasNot)
-      Cond.push_back(MachineOperand::CreateImm(0));
+    Cond.push_back(MachineOperand::CreateImm(SecondLastInst->getOpcode()));
     Cond.push_back(SecondLastInst->getOperand(0));
     FBB = LastInst->getOperand(0).getMBB();
     return false;
   }
 
+  // Only supporting rr/ri versions of new-value jumps.
+  if (SecLastOpcodeHasNVJump &&
+      (SecondLastInst->getNumExplicitOperands() == 3) &&
+      (LastOpcode == Hexagon::J2_jump)) {
+    TBB = SecondLastInst->getOperand(2).getMBB();
+    Cond.push_back(MachineOperand::CreateImm(SecondLastInst->getOpcode()));
+    Cond.push_back(SecondLastInst->getOperand(0));
+    Cond.push_back(SecondLastInst->getOperand(1));
+    FBB = LastInst->getOperand(0).getMBB();
+    return false;
+  }
+
   // If the block ends with two Hexagon:JMPs, handle it.  The second one is not
   // executed, so remove it.
   if (SecLastOpcode == Hexagon::J2_jump && LastOpcode == Hexagon::J2_jump) {
@@ -303,53 +424,40 @@ bool HexagonInstrInfo::AnalyzeBranch(MachineBasicBlock &MBB,
     return false;
   }
 
-  // If the block ends with an ENDLOOP, and JMP, handle it.
-  if (SecLastOpcode == Hexagon::ENDLOOP0 &&
-      LastOpcode == Hexagon::J2_jump) {
+  // If the block ends with an ENDLOOP, and J2_jump, handle it.
+  if (isEndLoopN(SecLastOpcode) && LastOpcode == Hexagon::J2_jump) {
     TBB = SecondLastInst->getOperand(0).getMBB();
+    Cond.push_back(MachineOperand::CreateImm(SecondLastInst->getOpcode()));
     Cond.push_back(SecondLastInst->getOperand(0));
     FBB = LastInst->getOperand(0).getMBB();
     return false;
   }
-
+  DEBUG(dbgs() << "\nCant analyze BB#" << MBB.getNumber()
+               << " with two jumps";);
   // Otherwise, can't handle this.
   return true;
 }
 
-
 unsigned HexagonInstrInfo::RemoveBranch(MachineBasicBlock &MBB) const {
+  DEBUG(dbgs() << "\nRemoving branches out of BB#" << MBB.getNumber());
   MachineBasicBlock::iterator I = MBB.end();
-  if (I == MBB.begin()) return 0;
-  --I;
-  unsigned Opc1 = I->getOpcode();
-  switch (Opc1) {
-    case Hexagon::J2_jump:
-    case Hexagon::J2_jumpt:
-    case Hexagon::J2_jumpf:
-    case Hexagon::ENDLOOP0:
-      I->eraseFromParent();
-      break;
-    default:
-      return 0;
-  }
-
-  I = MBB.end();
-
-  if (I == MBB.begin()) return 1;
-  --I;
-  unsigned Opc2 = I->getOpcode();
-  switch (Opc2) {
-    case Hexagon::J2_jumpt:
-    case Hexagon::J2_jumpf:
-    case Hexagon::ENDLOOP0:
-      I->eraseFromParent();
-      return 2;
-    default:
-      return 1;
+  unsigned Count = 0;
+  while (I != MBB.begin()) {
+    --I;
+    if (I->isDebugValue())
+      continue;
+    // Only removing branches from end of MBB.
+    if (!I->isBranch())
+      return Count;
+    if (Count && (I->getOpcode() == Hexagon::J2_jump))
+      llvm_unreachable("Malformed basic block: unconditional branch not last");
+    MBB.erase(&MBB.back());
+    I = MBB.end();
+    ++Count;
   }
+  return Count;
 }
 
-
 /// \brief For a comparison instruction, return the source registers in
 /// \p SrcReg and \p SrcReg2 if having two register operands, and the value it
 /// compares against in CmpValue. Return true if the comparison instruction
@@ -361,31 +469,39 @@ bool HexagonInstrInfo::analyzeCompare(const MachineInstr *MI,
 
   // Set mask and the first source register.
   switch (Opc) {
-    case Hexagon::C2_cmpeqp:
-    case Hexagon::C2_cmpeqi:
     case Hexagon::C2_cmpeq:
+    case Hexagon::C2_cmpeqp:
+    case Hexagon::C2_cmpgt:
     case Hexagon::C2_cmpgtp:
-    case Hexagon::C2_cmpgtup:
-    case Hexagon::C2_cmpgtui:
     case Hexagon::C2_cmpgtu:
+    case Hexagon::C2_cmpgtup:
+    case Hexagon::C4_cmpneq:
+    case Hexagon::C4_cmplte:
+    case Hexagon::C4_cmplteu:
+    case Hexagon::C2_cmpeqi:
     case Hexagon::C2_cmpgti:
-    case Hexagon::C2_cmpgt:
+    case Hexagon::C2_cmpgtui:
+    case Hexagon::C4_cmpneqi:
+    case Hexagon::C4_cmplteui:
+    case Hexagon::C4_cmpltei:
       SrcReg = MI->getOperand(1).getReg();
       Mask = ~0;
       break;
-    case Hexagon::A4_cmpbeqi:
     case Hexagon::A4_cmpbeq:
-    case Hexagon::A4_cmpbgtui:
-    case Hexagon::A4_cmpbgtu:
     case Hexagon::A4_cmpbgt:
+    case Hexagon::A4_cmpbgtu:
+    case Hexagon::A4_cmpbeqi:
+    case Hexagon::A4_cmpbgti:
+    case Hexagon::A4_cmpbgtui:
       SrcReg = MI->getOperand(1).getReg();
       Mask = 0xFF;
       break;
-    case Hexagon::A4_cmpheqi:
     case Hexagon::A4_cmpheq:
-    case Hexagon::A4_cmphgtui:
-    case Hexagon::A4_cmphgtu:
     case Hexagon::A4_cmphgt:
+    case Hexagon::A4_cmphgtu:
+    case Hexagon::A4_cmpheqi:
+    case Hexagon::A4_cmphgti:
+    case Hexagon::A4_cmphgtui:
       SrcReg = MI->getOperand(1).getReg();
       Mask = 0xFFFF;
       break;
@@ -393,27 +509,35 @@ bool HexagonInstrInfo::analyzeCompare(const MachineInstr *MI,
 
   // Set the value/second source register.
   switch (Opc) {
-    case Hexagon::C2_cmpeqp:
     case Hexagon::C2_cmpeq:
+    case Hexagon::C2_cmpeqp:
+    case Hexagon::C2_cmpgt:
     case Hexagon::C2_cmpgtp:
-    case Hexagon::C2_cmpgtup:
     case Hexagon::C2_cmpgtu:
-    case Hexagon::C2_cmpgt:
+    case Hexagon::C2_cmpgtup:
     case Hexagon::A4_cmpbeq:
-    case Hexagon::A4_cmpbgtu:
     case Hexagon::A4_cmpbgt:
+    case Hexagon::A4_cmpbgtu:
     case Hexagon::A4_cmpheq:
-    case Hexagon::A4_cmphgtu:
     case Hexagon::A4_cmphgt:
+    case Hexagon::A4_cmphgtu:
+    case Hexagon::C4_cmpneq:
+    case Hexagon::C4_cmplte:
+    case Hexagon::C4_cmplteu:
       SrcReg2 = MI->getOperand(2).getReg();
       return true;
 
     case Hexagon::C2_cmpeqi:
     case Hexagon::C2_cmpgtui:
     case Hexagon::C2_cmpgti:
+    case Hexagon::C4_cmpneqi:
+    case Hexagon::C4_cmplteui:
+    case Hexagon::C4_cmpltei:
     case Hexagon::A4_cmpbeqi:
+    case Hexagon::A4_cmpbgti:
     case Hexagon::A4_cmpbgtui:
     case Hexagon::A4_cmpheqi:
+    case Hexagon::A4_cmphgti:
     case Hexagon::A4_cmphgtui:
       SrcReg2 = 0;
       Value = MI->getOperand(2).getImm();
@@ -573,6 +697,12 @@ HexagonInstrInfo::expandPostRAPseudo(MachineBasicBlock::iterator MI) const {
   unsigned Opc = MI->getOpcode();
 
   switch (Opc) {
+    case Hexagon::ALIGNA:
+      BuildMI(MBB, MI, DL, get(Hexagon::A2_andir), MI->getOperand(0).getReg())
+          .addReg(TRI.getFrameRegister())
+          .addImm(-MI->getOperand(1).getImm());
+      MBB.erase(MI);
+      return true;
     case Hexagon::TFR_PdTrue: {
       unsigned Reg = MI->getOperand(0).getReg();
       BuildMI(MBB, MI, DL, get(Hexagon::C2_orn), Reg)
@@ -649,10 +779,9 @@ HexagonInstrInfo::expandPostRAPseudo(MachineBasicBlock::iterator MI) const {
   return false;
 }
 
-MachineInstr *HexagonInstrInfo::foldMemoryOperandImpl(MachineFunction &MF,
-                                                      MachineInstr *MI,
-                                                      ArrayRef<unsigned> Ops,
-                                                      int FI) const {
+MachineInstr *HexagonInstrInfo::foldMemoryOperandImpl(
+    MachineFunction &MF, MachineInstr *MI, ArrayRef<unsigned> Ops,
+    MachineBasicBlock::iterator InsertPt, int FI) const {
   // Hexagon_TODO: Implement.
   return nullptr;
 }
@@ -725,6 +854,16 @@ bool HexagonInstrInfo::isNewValueInst(const MachineInstr *MI) const {
   return false;
 }
 
+bool HexagonInstrInfo::isNewValue(const MachineInstr* MI) const {
+  const uint64_t F = MI->getDesc().TSFlags;
+  return ((F >> HexagonII::NewValuePos) & HexagonII::NewValueMask);
+}
+
+bool HexagonInstrInfo::isNewValue(Opcode_t Opcode) const {
+  const uint64_t F = get(Opcode).TSFlags;
+  return ((F >> HexagonII::NewValuePos) & HexagonII::NewValueMask);
+}
+
 bool HexagonInstrInfo::isSaveCalleeSavedRegsCall(const MachineInstr *MI) const {
   return MI->getOpcode() == Hexagon::SAVE_REGISTERS_CALL_V4;
 }
@@ -874,149 +1013,52 @@ int HexagonInstrInfo::getCondOpcode(int Opc, bool invertPredicate) const {
 
 bool HexagonInstrInfo::
 PredicateInstruction(MachineInstr *MI,
-                     const SmallVectorImpl<MachineOperand> &Cond) const {
+                     ArrayRef<MachineOperand> Cond) const {
+  if (Cond.empty() || isEndLoopN(Cond[0].getImm())) {
+    DEBUG(dbgs() << "\nCannot predicate:"; MI->dump(););
+    return false;
+  }
   int Opc = MI->getOpcode();
   assert (isPredicable(MI) && "Expected predicable instruction");
-  bool invertJump = (!Cond.empty() && Cond[0].isImm() &&
-                     (Cond[0].getImm() == 0));
-
-  // This will change MI's opcode to its predicate version.
-  // However, its operand list is still the old one, i.e. the
-  // non-predicate one.
-  MI->setDesc(get(getCondOpcode(Opc, invertJump)));
-
-  int oper = -1;
-  unsigned int GAIdx = 0;
-
-  // Indicates whether the current MI has a GlobalAddress operand
-  bool hasGAOpnd = false;
-  std::vector<MachineOperand> tmpOpnds;
-
-  // Indicates whether we need to shift operands to right.
-  bool needShift = true;
-
-  // The predicate is ALWAYS the FIRST input operand !!!
-  if (MI->getNumOperands() == 0) {
-    // The non-predicate version of MI does not take any operands,
-    // i.e. no outs and no ins. In this condition, the predicate
-    // operand will be directly placed at Operands[0]. No operand
-    // shift is needed.
-    // Example: BARRIER
-    needShift = false;
-    oper = -1;
-  }
-  else if (   MI->getOperand(MI->getNumOperands()-1).isReg()
-           && MI->getOperand(MI->getNumOperands()-1).isDef()
-           && !MI->getOperand(MI->getNumOperands()-1).isImplicit()) {
-    // The non-predicate version of MI does not have any input operands.
-    // In this condition, we extend the length of Operands[] by one and
-    // copy the original last operand to the newly allocated slot.
-    // At this moment, it is just a place holder. Later, we will put
-    // predicate operand directly into it. No operand shift is needed.
-    // Example: r0=BARRIER (this is a faked insn used here for illustration)
-    MI->addOperand(MI->getOperand(MI->getNumOperands()-1));
-    needShift = false;
-    oper = MI->getNumOperands() - 2;
-  }
-  else {
-    // We need to right shift all input operands by one. Duplicate the
-    // last operand into the newly allocated slot.
-    MI->addOperand(MI->getOperand(MI->getNumOperands()-1));
-  }
+  bool invertJump = predOpcodeHasNot(Cond);
 
-  if (needShift)
-  {
-    // Operands[ MI->getNumOperands() - 2 ] has been copied into
-    // Operands[ MI->getNumOperands() - 1 ], so we start from
-    // Operands[ MI->getNumOperands() - 3 ].
-    // oper is a signed int.
-    // It is ok if "MI->getNumOperands()-3" is -3, -2, or -1.
-    for (oper = MI->getNumOperands() - 3; oper >= 0; --oper)
-    {
-      MachineOperand &MO = MI->getOperand(oper);
-
-      // Opnd[0] Opnd[1] Opnd[2] Opnd[3] Opnd[4]   Opnd[5]   Opnd[6]   Opnd[7]
-      // <Def0>  <Def1>  <Use0>  <Use1>  <ImpDef0> <ImpDef1> <ImpUse0> <ImpUse1>
-      //               /\~
-      //              /||\~
-      //               ||
-      //        Predicate Operand here
-      if (MO.isReg() && !MO.isUse() && !MO.isImplicit()) {
-        break;
-      }
-      if (MO.isReg()) {
-        MI->getOperand(oper+1).ChangeToRegister(MO.getReg(), MO.isDef(),
-                                                MO.isImplicit(), MO.isKill(),
-                                                MO.isDead(), MO.isUndef(),
-                                                MO.isDebug());
-      }
-      else if (MO.isImm()) {
-        MI->getOperand(oper+1).ChangeToImmediate(MO.getImm());
-      }
-      else if (MO.isGlobal()) {
-        // MI can not have more than one GlobalAddress operand.
-        assert(hasGAOpnd == false && "MI can only have one GlobalAddress opnd");
-
-        // There is no member function called "ChangeToGlobalAddress" in the
-        // MachineOperand class (not like "ChangeToRegister" and
-        // "ChangeToImmediate"). So we have to remove them from Operands[] list
-        // first, and then add them back after we have inserted the predicate
-        // operand. tmpOpnds[] is to remember these operands before we remove
-        // them.
-        tmpOpnds.push_back(MO);
-
-        // Operands[oper] is a GlobalAddress operand;
-        // Operands[oper+1] has been copied into Operands[oper+2];
-        hasGAOpnd = true;
-        GAIdx = oper;
-        continue;
-      }
-      else {
-        llvm_unreachable("Unexpected operand type");
-      }
-    }
-  }
+  // We have to predicate MI "in place", i.e. after this function returns,
+  // MI will need to be transformed into a predicated form. To avoid com-
+  // plicated manipulations with the operands (handling tied operands,
+  // etc.), build a new temporary instruction, then overwrite MI with it.
 
-  int regPos = invertJump ? 1 : 0;
-  MachineOperand PredMO = Cond[regPos];
+  MachineBasicBlock &B = *MI->getParent();
+  DebugLoc DL = MI->getDebugLoc();
+  unsigned PredOpc = getCondOpcode(Opc, invertJump);
+  MachineInstrBuilder T = BuildMI(B, MI, DL, get(PredOpc));
+  unsigned NOp = 0, NumOps = MI->getNumOperands();
+  while (NOp < NumOps) {
+    MachineOperand &Op = MI->getOperand(NOp);
+    if (!Op.isReg() || !Op.isDef() || Op.isImplicit())
+      break;
+    T.addOperand(Op);
+    NOp++;
+  }
 
-  // [oper] now points to the last explicit Def. Predicate operand must be
-  // located at [oper+1]. See diagram above.
-  // This assumes that the predicate is always the first operand,
-  // i.e. Operands[0+numResults], in the set of inputs
-  // It is better to have an assert here to check this. But I don't know how
-  // to write this assert because findFirstPredOperandIdx() would return -1
-  if (oper < -1) oper = -1;
+  unsigned PredReg, PredRegPos, PredRegFlags;
+  bool GotPredReg = getPredReg(Cond, PredReg, PredRegPos, PredRegFlags);
+  (void)GotPredReg;
+  assert(GotPredReg);
+  T.addReg(PredReg, PredRegFlags);
+  while (NOp < NumOps)
+    T.addOperand(MI->getOperand(NOp++));
 
-  MI->getOperand(oper+1).ChangeToRegister(PredMO.getReg(), PredMO.isDef(),
-                                          PredMO.isImplicit(), false,
-                                          PredMO.isDead(), PredMO.isUndef(),
-                                          PredMO.isDebug());
+  MI->setDesc(get(PredOpc));
+  while (unsigned n = MI->getNumOperands())
+    MI->RemoveOperand(n-1);
+  for (unsigned i = 0, n = T->getNumOperands(); i < n; ++i)
+    MI->addOperand(T->getOperand(i));
 
-  MachineRegisterInfo &RegInfo = MI->getParent()->getParent()->getRegInfo();
-  RegInfo.clearKillFlags(PredMO.getReg());
+  MachineBasicBlock::instr_iterator TI = &*T;
+  B.erase(TI);
 
-  if (hasGAOpnd)
-  {
-    unsigned int i;
-
-    // Operands[GAIdx] is the original GlobalAddress operand, which is
-    // already copied into tmpOpnds[0].
-    // Operands[GAIdx] now stores a copy of Operands[GAIdx-1]
-    // Operands[GAIdx+1] has already been copied into Operands[GAIdx+2],
-    // so we start from [GAIdx+2]
-    for (i = GAIdx + 2; i < MI->getNumOperands(); ++i)
-      tmpOpnds.push_back(MI->getOperand(i));
-
-    // Remove all operands in range [ (GAIdx+1) ... (MI->getNumOperands()-1) ]
-    // It is very important that we always remove from the end of Operands[]
-    // MI->getNumOperands() is at least 2 if program goes to here.
-    for (i = MI->getNumOperands() - 1; i > GAIdx; --i)
-      MI->RemoveOperand(i);
-
-    for (i = 0; i < tmpOpnds.size(); ++i)
-      MI->addOperand(tmpOpnds[i]);
-  }
+  MachineRegisterInfo &MRI = B.getParent()->getRegInfo();
+  MRI.clearKillFlags(PredReg);
 
   return true;
 }
@@ -1120,8 +1162,8 @@ HexagonInstrInfo::DefinesPredicate(MachineInstr *MI,
 
 bool
 HexagonInstrInfo::
-SubsumesPredicate(const SmallVectorImpl<MachineOperand> &Pred1,
-                  const SmallVectorImpl<MachineOperand> &Pred2) const {
+SubsumesPredicate(ArrayRef<MachineOperand> Pred1,
+                  ArrayRef<MachineOperand> Pred2) const {
   // TODO: Fix this
   return false;
 }
@@ -1129,17 +1171,20 @@ SubsumesPredicate(const SmallVectorImpl<MachineOperand> &Pred1,
 
 //
 // We indicate that we want to reverse the branch by
-// inserting a 0 at the beginning of the Cond vector.
+// inserting the reversed branching opcode.
 //
-bool HexagonInstrInfo::
-ReverseBranchCondition(SmallVectorImpl<MachineOperand> &Cond) const {
-  if (!Cond.empty() && Cond[0].isMBB())
+bool HexagonInstrInfo::ReverseBranchCondition(
+    SmallVectorImpl<MachineOperand> &Cond) const {
+  if (Cond.empty())
     return true;
-  if (!Cond.empty() && Cond[0].isImm() && Cond[0].getImm() == 0) {
-    Cond.erase(Cond.begin());
-  } else {
-    Cond.insert(Cond.begin(), MachineOperand::CreateImm(0));
-  }
+  assert(Cond[0].isImm() && "First entry in the cond vector not imm-val");
+  Opcode_t opcode = Cond[0].getImm();
+  //unsigned temp;
+  assert(get(opcode).isBranch() && "Should be a branching condition.");
+  if (isEndLoopN(opcode))
+    return true;
+  Opcode_t NewOpcode = getInvertedPredicatedOpcode(opcode);
+  Cond[0].setImm(NewOpcode);
   return false;
 }
 
@@ -1165,10 +1210,10 @@ bool HexagonInstrInfo::isDeallocRet(const MachineInstr *MI) const {
 }
 
 
-bool HexagonInstrInfo::
-isValidOffset(const int Opcode, const int Offset) const {
+bool HexagonInstrInfo::isValidOffset(unsigned Opcode, int Offset,
+      bool Extend) const {
   // This function is to check whether the "Offset" is in the correct range of
-  // the given "Opcode". If "Offset" is not in the correct range, "ADD_ri" is
+  // the given "Opcode". If "Offset" is not in the correct range, "A2_addi" is
   // inserted to calculate the final address. Due to this reason, the function
   // assumes that the "Offset" has correct alignment.
   // We used to assert if the offset was not properly aligned, however,
@@ -1176,8 +1221,16 @@ isValidOffset(const int Opcode, const int Offset) const {
   // problem, and we need to allow for it. The front end warns of such
   // misaligns with respect to load size.
 
-  switch(Opcode) {
+  switch (Opcode) {
+  case Hexagon::J2_loop0i:
+  case Hexagon::J2_loop1i:
+    return isUInt<10>(Offset);
+  }
+
+  if (Extend)
+    return true;
 
+  switch (Opcode) {
   case Hexagon::L2_loadri_io:
   case Hexagon::S2_storeri_io:
     return (Offset >= Hexagon_MEMW_OFFSET_MIN) &&
@@ -1201,7 +1254,6 @@ isValidOffset(const int Opcode, const int Offset) const {
       (Offset <= Hexagon_MEMB_OFFSET_MAX);
 
   case Hexagon::A2_addi:
-  case Hexagon::TFR_FI:
     return (Offset >= Hexagon_ADDI_OFFSET_MIN) &&
       (Offset <= Hexagon_ADDI_OFFSET_MAX);
 
@@ -1235,10 +1287,8 @@ isValidOffset(const int Opcode, const int Offset) const {
   case Hexagon::LDriw_pred:
     return true;
 
-  case Hexagon::J2_loop0i:
-    return isUInt<10>(Offset);
-
-  // INLINEASM is very special.
+  case Hexagon::TFR_FI:
+  case Hexagon::TFR_FIA:
   case Hexagon::INLINEASM:
     return true;
   }
@@ -1572,13 +1622,12 @@ bool HexagonInstrInfo::isNewValueJump(const MachineInstr *MI) const {
   return false;
 }
 
-bool HexagonInstrInfo::isPostIncrement (const MachineInstr* MI) const {
-  return (getAddrMode(MI) == HexagonII::PostInc);
+bool HexagonInstrInfo::isNewValueJump(Opcode_t Opcode) const {
+  return isNewValue(Opcode) && get(Opcode).isBranch() && isPredicated(Opcode);
 }
 
-bool HexagonInstrInfo::isNewValue(const MachineInstr* MI) const {
-  const uint64_t F = MI->getDesc().TSFlags;
-  return ((F >> HexagonII::NewValuePos) & HexagonII::NewValueMask);
+bool HexagonInstrInfo::isPostIncrement (const MachineInstr* MI) const {
+  return (getAddrMode(MI) == HexagonII::PostInc);
 }
 
 // Returns true, if any one of the operands is a dot new
@@ -1731,14 +1780,14 @@ bool HexagonInstrInfo::isSchedulingBoundary(const MachineInstr *MI,
   return false;
 }
 
-bool HexagonInstrInfo::isConstExtended(MachineInstr *MI) const {
+bool HexagonInstrInfo::isConstExtended(const MachineInstr *MI) const {
   const uint64_t F = MI->getDesc().TSFlags;
   unsigned isExtended = (F >> HexagonII::ExtendedPos) & HexagonII::ExtendedMask;
   if (isExtended) // Instruction must be extended.
     return true;
 
-  unsigned isExtendable = (F >> HexagonII::ExtendablePos)
-                          & HexagonII::ExtendableMask;
+  unsigned isExtendable =
+    (F >> HexagonII::ExtendablePos) & HexagonII::ExtendableMask;
   if (!isExtendable)
     return false;
 
@@ -1759,7 +1808,8 @@ bool HexagonInstrInfo::isConstExtended(MachineInstr *MI) const {
   // We currently only handle isGlobal() because it is the only kind of
   // object we are going to end up with here for now.
   // In the future we probably should add isSymbol(), etc.
-  if (MO.isGlobal() || MO.isSymbol() || MO.isBlockAddress())
+  if (MO.isGlobal() || MO.isSymbol() || MO.isBlockAddress() ||
+      MO.isJTI() || MO.isCPI())
     return true;
 
   // If the extendable operand is not 'Immediate' type, the instruction should
@@ -1773,6 +1823,27 @@ bool HexagonInstrInfo::isConstExtended(MachineInstr *MI) const {
   return (ImmValue < MinValue || ImmValue > MaxValue);
 }
 
+// Return the number of bytes required to encode the instruction.
+// Hexagon instructions are fixed length, 4 bytes, unless they
+// use a constant extender, which requires another 4 bytes.
+// For debug instructions and prolog labels, return 0.
+unsigned HexagonInstrInfo::getSize(const MachineInstr *MI) const {
+
+  if (MI->isDebugValue() || MI->isPosition())
+    return 0;
+
+  unsigned Size = MI->getDesc().getSize();
+  if (!Size)
+    // Assume the default insn size in case it cannot be determined
+    // for whatever reason.
+    Size = HEXAGON_INSTR_SIZE;
+
+  if (isConstExtended(MI) || isExtended(MI))
+    Size += HEXAGON_INSTR_SIZE;
+
+  return Size;
+}
+
 // Returns the opcode to use when converting MI, which is a conditional jump,
 // into a conditional instruction which uses the .new value of the predicate.
 // We also use branch probabilities to add a hint to the jump.
@@ -1911,8 +1982,35 @@ bool HexagonInstrInfo::PredOpcodeHasJMP_c(Opcode_t Opcode) const {
          (Opcode == Hexagon::J2_jumpf);
 }
 
-bool HexagonInstrInfo::PredOpcodeHasNot(Opcode_t Opcode) const {
-  return (Opcode == Hexagon::J2_jumpf) ||
-         (Opcode == Hexagon::J2_jumpfnewpt) ||
-         (Opcode == Hexagon::J2_jumpfnew);
+bool HexagonInstrInfo::predOpcodeHasNot(ArrayRef<MachineOperand> Cond) const {
+  if (Cond.empty() || !isPredicated(Cond[0].getImm()))
+    return false;
+  return !isPredicatedTrue(Cond[0].getImm());
 }
+
+bool HexagonInstrInfo::isEndLoopN(Opcode_t Opcode) const {
+  return (Opcode == Hexagon::ENDLOOP0 ||
+          Opcode == Hexagon::ENDLOOP1);
+}
+
+bool HexagonInstrInfo::getPredReg(ArrayRef<MachineOperand> Cond,
+                                  unsigned &PredReg, unsigned &PredRegPos,
+                                  unsigned &PredRegFlags) const {
+  if (Cond.empty())
+    return false;
+  assert(Cond.size() == 2);
+  if (isNewValueJump(Cond[0].getImm()) || Cond[1].isMBB()) {
+     DEBUG(dbgs() << "No predregs for new-value jumps/endloop");
+     return false;
+  }
+  PredReg = Cond[1].getReg();
+  PredRegPos = 1;
+  // See IfConversion.cpp why we add RegState::Implicit | RegState::Undef
+  PredRegFlags = 0;
+  if (Cond[1].isImplicit())
+    PredRegFlags = RegState::Implicit;
+  if (Cond[1].isUndef())
+    PredRegFlags |= RegState::Undef;
+  return true;
+}
+
diff --git a/lib/Target/Hexagon/HexagonInstrInfo.h b/lib/Target/Hexagon/HexagonInstrInfo.h
index 284dde1f91e..d0b8a4631c1 100644
--- a/lib/Target/Hexagon/HexagonInstrInfo.h
+++ b/lib/Target/Hexagon/HexagonInstrInfo.h
@@ -1,3 +1,4 @@
+
 //===- HexagonInstrInfo.h - Hexagon Instruction Information -----*- C++ -*-===//
 //
 //                     The LLVM Compiler Infrastructure
@@ -31,9 +32,10 @@ class HexagonInstrInfo : public HexagonGenInstrInfo {
   virtual void anchor();
   const HexagonRegisterInfo RI;
   const HexagonSubtarget &Subtarget;
-  typedef unsigned Opcode_t;
 
 public:
+  typedef unsigned Opcode_t;
+
   explicit HexagonInstrInfo(HexagonSubtarget &ST);
 
   /// getRegisterInfo - TargetInstrInfo is a superset of MRegister info.  As
@@ -67,8 +69,7 @@ class HexagonInstrInfo : public HexagonGenInstrInfo {
   unsigned RemoveBranch(MachineBasicBlock &MBB) const override;
 
   unsigned InsertBranch(MachineBasicBlock &MBB, MachineBasicBlock *TBB,
-                        MachineBasicBlock *FBB,
-                        const SmallVectorImpl<MachineOperand> &Cond,
+                        MachineBasicBlock *FBB, ArrayRef<MachineOperand> Cond,
                         DebugLoc DL) const override;
 
   bool analyzeCompare(const MachineInstr *MI,
@@ -112,10 +113,12 @@ class HexagonInstrInfo : public HexagonGenInstrInfo {
 
   MachineInstr *foldMemoryOperandImpl(MachineFunction &MF, MachineInstr *MI,
                                       ArrayRef<unsigned> Ops,
+                                      MachineBasicBlock::iterator InsertPt,
                                       int FrameIndex) const override;
 
   MachineInstr *foldMemoryOperandImpl(MachineFunction &MF, MachineInstr *MI,
                                       ArrayRef<unsigned> Ops,
+                                      MachineBasicBlock::iterator InsertPt,
                                       MachineInstr *LoadMI) const override {
     return nullptr;
   }
@@ -125,7 +128,7 @@ class HexagonInstrInfo : public HexagonGenInstrInfo {
   bool isBranch(const MachineInstr *MI) const;
   bool isPredicable(MachineInstr *MI) const override;
   bool PredicateInstruction(MachineInstr *MI,
-                    const SmallVectorImpl<MachineOperand> &Cond) const override;
+                            ArrayRef<MachineOperand> Cond) const override;
 
   bool isProfitableToIfCvt(MachineBasicBlock &MBB, unsigned NumCycles,
                            unsigned ExtraPredCycles,
@@ -145,8 +148,8 @@ class HexagonInstrInfo : public HexagonGenInstrInfo {
   bool isPredicatedNew(unsigned Opcode) const;
   bool DefinesPredicate(MachineInstr *MI,
                         std::vector<MachineOperand> &Pred) const override;
-  bool SubsumesPredicate(const SmallVectorImpl<MachineOperand> &Pred1,
-                   const SmallVectorImpl<MachineOperand> &Pred2) const override;
+  bool SubsumesPredicate(ArrayRef<MachineOperand> Pred1,
+                         ArrayRef<MachineOperand> Pred2) const override;
 
   bool
   ReverseBranchCondition(SmallVectorImpl<MachineOperand> &Cond) const override;
@@ -160,7 +163,7 @@ class HexagonInstrInfo : public HexagonGenInstrInfo {
   bool isSchedulingBoundary(const MachineInstr *MI,
                             const MachineBasicBlock *MBB,
                             const MachineFunction &MF) const override;
-  bool isValidOffset(const int Opcode, const int Offset) const;
+  bool isValidOffset(unsigned Opcode, int Offset, bool Extend = true) const;
   bool isValidAutoIncImm(const EVT VT, const int Offset) const;
   bool isMemOp(const MachineInstr *MI) const;
   bool isSpillPredRegOp(const MachineInstr *MI) const;
@@ -184,6 +187,7 @@ class HexagonInstrInfo : public HexagonGenInstrInfo {
   bool isConditionalStore(const MachineInstr* MI) const;
   bool isNewValueInst(const MachineInstr* MI) const;
   bool isNewValue(const MachineInstr* MI) const;
+  bool isNewValue(Opcode_t Opcode) const;
   bool isDotNewInst(const MachineInstr* MI) const;
   int GetDotOldOp(const int opc) const;
   int GetDotNewOp(const MachineInstr* MI) const;
@@ -199,11 +203,13 @@ class HexagonInstrInfo : public HexagonGenInstrInfo {
   bool isNewValueStore(const MachineInstr* MI) const;
   bool isNewValueStore(unsigned Opcode) const;
   bool isNewValueJump(const MachineInstr* MI) const;
+  bool isNewValueJump(Opcode_t Opcode) const;
   bool isNewValueJumpCandidate(const MachineInstr *MI) const;
 
 
   void immediateExtend(MachineInstr *MI) const;
-  bool isConstExtended(MachineInstr *MI) const;
+  bool isConstExtended(const MachineInstr *MI) const;
+  unsigned getSize(const MachineInstr *MI) const;  
   int getDotNewPredJumpOp(MachineInstr *MI,
                       const MachineBranchProbabilityInfo *MBPI) const;
   unsigned getAddrMode(const MachineInstr* MI) const;
@@ -215,7 +221,10 @@ class HexagonInstrInfo : public HexagonGenInstrInfo {
   bool NonExtEquivalentExists (const MachineInstr *MI) const;
   short getNonExtOpcode(const MachineInstr *MI) const;
   bool PredOpcodeHasJMP_c(Opcode_t Opcode) const;
-  bool PredOpcodeHasNot(Opcode_t Opcode) const;
+  bool predOpcodeHasNot(ArrayRef<MachineOperand> Cond) const;
+  bool isEndLoopN(Opcode_t Opcode) const;
+  bool getPredReg(ArrayRef<MachineOperand> Cond, unsigned &PredReg,
+                  unsigned &PredRegPos, unsigned &PredRegFlags) const;
   int getCondOpcode(int Opc, bool sense) const;
 
 };
diff --git a/lib/Target/Hexagon/HexagonInstrInfo.td b/lib/Target/Hexagon/HexagonInstrInfo.td
index 19cf9932452..3b32c10ed5b 100644
--- a/lib/Target/Hexagon/HexagonInstrInfo.td
+++ b/lib/Target/Hexagon/HexagonInstrInfo.td
@@ -36,28 +36,28 @@ def HiReg: OutPatFrag<(ops node:$Rs),
 def DEC_CONST_SIGNED : SDNodeXForm<imm, [{
    // Return the byte immediate const-1 as an SDNode.
    int32_t imm = N->getSExtValue();
-   return XformSToSM1Imm(imm);
+   return XformSToSM1Imm(imm, SDLoc(N));
 }]>;
 
 // SDNode for converting immediate C to C-2.
 def DEC2_CONST_SIGNED : SDNodeXForm<imm, [{
    // Return the byte immediate const-2 as an SDNode.
    int32_t imm = N->getSExtValue();
-   return XformSToSM2Imm(imm);
+   return XformSToSM2Imm(imm, SDLoc(N));
 }]>;
 
 // SDNode for converting immediate C to C-3.
 def DEC3_CONST_SIGNED : SDNodeXForm<imm, [{
    // Return the byte immediate const-3 as an SDNode.
    int32_t imm = N->getSExtValue();
-   return XformSToSM3Imm(imm);
+   return XformSToSM3Imm(imm, SDLoc(N));
 }]>;
 
 // SDNode for converting immediate C to C-1.
 def DEC_CONST_UNSIGNED : SDNodeXForm<imm, [{
    // Return the byte immediate const-1 as an SDNode.
    uint32_t imm = N->getZExtValue();
-   return XformUToUM1Imm(imm);
+   return XformUToUM1Imm(imm, SDLoc(N));
 }]>;
 
 //===----------------------------------------------------------------------===//
@@ -4191,12 +4191,27 @@ def S2_clb     : T_COUNT_LEADING_32<"clb",     0b000, 0b100>;
 def S2_clbp    : T_COUNT_LEADING_64<"clb",     0b010, 0b000>;
 def S2_clbnorm : T_COUNT_LEADING_32<"normamt", 0b000, 0b111>;
 
-def: Pat<(i32 (ctlz I32:$Rs)),                (S2_cl0 I32:$Rs)>;
-def: Pat<(i32 (ctlz (not I32:$Rs))),          (S2_cl1 I32:$Rs)>;
-def: Pat<(i32 (cttz I32:$Rs)),                (S2_ct0 I32:$Rs)>;
-def: Pat<(i32 (cttz (not I32:$Rs))),          (S2_ct1 I32:$Rs)>;
-def: Pat<(i32 (trunc (ctlz I64:$Rss))),       (S2_cl0p I64:$Rss)>;
+// Count leading zeros.
+def: Pat<(i32 (ctlz I32:$Rs)), (S2_cl0 I32:$Rs)>;
+def: Pat<(i32 (trunc (ctlz I64:$Rss))), (S2_cl0p I64:$Rss)>;
+def: Pat<(i32 (ctlz_zero_undef I32:$Rs)), (S2_cl0 I32:$Rs)>;
+def: Pat<(i32 (trunc (ctlz_zero_undef I64:$Rss))), (S2_cl0p I64:$Rss)>;
+
+// Count trailing zeros: 32-bit.
+def: Pat<(i32 (cttz I32:$Rs)), (S2_ct0 I32:$Rs)>;
+def: Pat<(i32 (cttz_zero_undef I32:$Rs)), (S2_ct0 I32:$Rs)>;
+
+// Count leading ones.
+def: Pat<(i32 (ctlz (not I32:$Rs))), (S2_cl1 I32:$Rs)>;
 def: Pat<(i32 (trunc (ctlz (not I64:$Rss)))), (S2_cl1p I64:$Rss)>;
+def: Pat<(i32 (ctlz_zero_undef (not I32:$Rs))), (S2_cl1 I32:$Rs)>;
+def: Pat<(i32 (trunc (ctlz_zero_undef (not I64:$Rss)))), (S2_cl1p I64:$Rss)>;
+
+// Count trailing ones: 32-bit.
+def: Pat<(i32 (cttz (not I32:$Rs))), (S2_ct1 I32:$Rs)>;
+def: Pat<(i32 (cttz_zero_undef (not I32:$Rs))), (S2_ct1 I32:$Rs)>;
+
+// The 64-bit counts leading/trailing are defined in HexagonInstrInfoV4.td.
 
 // Bit set/clear/toggle
 
@@ -4526,10 +4541,18 @@ def Y2_barrier : SYSInst<(outs), (ins),
 // SYSTEM/SUPER -
 //===----------------------------------------------------------------------===//
 
-// Generate frameindex addresses.
+// Generate frameindex addresses. The main reason for the offset operand is
+// that every instruction that is allowed to have frame index as an operand
+// will then have that operand followed by an immediate operand (the offset).
+// This simplifies the frame-index elimination code.
+//
 let isMoveImm = 1, isAsCheapAsAMove = 1, isReMaterializable = 1,
-    isPseudo = 1, isCodeGenOnly = 1, hasSideEffects = 0 in
-def TFR_FI: ALU32_ri<(outs IntRegs:$Rd), (ins IntRegs:$fi, s32Imm:$Off), "">;
+    isPseudo = 1, isCodeGenOnly = 1, hasSideEffects = 0 in {
+  def TFR_FI  : ALU32_ri<(outs IntRegs:$Rd),
+                         (ins IntRegs:$fi, s32Imm:$off), "">;
+  def TFR_FIA : ALU32_ri<(outs IntRegs:$Rd),
+                         (ins IntRegs:$Rs, IntRegs:$fi, s32Imm:$off), "">;
+}
 
 //===----------------------------------------------------------------------===//
 // CRUSER - Type.
@@ -4815,12 +4838,6 @@ def CONST32 : CONSTLDInst<(outs IntRegs:$dst), (ins globaladdress:$global),
               [(set (i32 IntRegs:$dst),
                     (load (HexagonCONST32 tglobaltlsaddr:$global)))]>;
 
-let isReMaterializable = 1, isMoveImm = 1 in
-def CONST32_set_jt : CONSTLDInst<(outs IntRegs:$dst), (ins jumptablebase:$jt),
-                     "$dst = CONST32(#$jt)",
-                     [(set (i32 IntRegs:$dst),
-                           (HexagonCONST32 tjumptable:$jt))]>;
-
 let isReMaterializable = 1, isMoveImm = 1, isAsmParserOnly = 1 in
 def CONST32_Int_Real : CONSTLDInst<(outs IntRegs:$dst), (ins i32imm:$global),
                        "$dst = CONST32(#$global)",
@@ -4828,12 +4845,7 @@ def CONST32_Int_Real : CONSTLDInst<(outs IntRegs:$dst), (ins i32imm:$global),
 
 // Map TLS addressses to a CONST32 instruction
 def: Pat<(HexagonCONST32 tglobaltlsaddr:$addr), (A2_tfrsi s16Ext:$addr)>;
-def: Pat<(HexagonCONST32 bbl:$label), (A2_tfrsi s16Ext:$label)>;
-
-let isReMaterializable = 1, isMoveImm = 1, isAsmParserOnly = 1 in
-def CONST32_Label : LDInst2<(outs IntRegs:$dst), (ins bblabel:$label),
-                    "$dst = CONST32($label)",
-                    [(set (i32 IntRegs:$dst), (HexagonCONST32 bbl:$label))]>;
+def: Pat<(HexagonCONST32 bbl:$label),           (A2_tfrsi s16Ext:$label)>;
 
 let isReMaterializable = 1, isMoveImm = 1, isAsmParserOnly = 1 in
 def CONST64_Int_Real : CONSTLDInst<(outs DoubleRegs:$dst), (ins i64imm:$global),
@@ -5105,21 +5117,25 @@ def : Pat <(mulhu (i64 DoubleRegs:$src1), (i64 DoubleRegs:$src2)),
 )>;
 
 // Hexagon specific ISD nodes.
-def SDTHexagonADJDYNALLOC : SDTypeProfile<1, 2, [SDTCisVT<0, i32>,
-                                                 SDTCisVT<1, i32>]>;
-def SDTHexagonARGEXTEND   : SDTypeProfile<1, 1, [SDTCisVT<0, i32>]>;
-
-def Hexagon_ADJDYNALLOC : SDNode<"HexagonISD::ADJDYNALLOC",
-                                  SDTHexagonADJDYNALLOC>;
-def Hexagon_ARGEXTEND   : SDNode<"HexagonISD::ARGEXTEND", SDTHexagonARGEXTEND>;
-
-// Needed to tag these instructions for stack layout.
-let isCodeGenOnly = 1, usesCustomInserter = 1 in
-def ADJDYNALLOC : T_Addri<s6Imm>;
-
-def: Pat<(Hexagon_ADJDYNALLOC I32:$Rs, s16ImmPred:$s16),
-         (ADJDYNALLOC I32:$Rs, imm:$s16)>;
-
+def SDTHexagonALLOCA : SDTypeProfile<1, 2,
+      [SDTCisVT<0, i32>, SDTCisVT<1, i32>]>;
+def HexagonALLOCA : SDNode<"HexagonISD::ALLOCA", SDTHexagonALLOCA,
+      [SDNPHasChain]>;
+
+// The reason for the custom inserter is to record all ALLOCA instructions
+// in MachineFunctionInfo.
+let Defs = [R29], isCodeGenOnly = 1, isPseudo = 1, hasSideEffects = 1,
+    usesCustomInserter = 1 in
+def ALLOCA: ALU32Inst<(outs IntRegs:$Rd),
+      (ins IntRegs:$Rs, u32Imm:$A), "",
+      [(set (i32 IntRegs:$Rd),
+            (HexagonALLOCA (i32 IntRegs:$Rs), (i32 imm:$A)))]>;
+
+let isCodeGenOnly = 1, isPseudo = 1, Uses = [R30], hasSideEffects = 0 in
+def ALIGNA : ALU32Inst<(outs IntRegs:$Rd), (ins u32Imm:$A), "", []>;
+
+def SDTHexagonARGEXTEND : SDTypeProfile<1, 1, [SDTCisVT<0, i32>]>;
+def Hexagon_ARGEXTEND : SDNode<"HexagonISD::ARGEXTEND", SDTHexagonARGEXTEND>;
 let isCodeGenOnly = 1 in
 def ARGEXTEND : ALU32_rr <(outs IntRegs:$dst), (ins IntRegs:$src1),
                 "$dst = $src1",
@@ -5133,10 +5149,8 @@ def: Pat<(i32 (sext_inreg (Hexagon_ARGEXTEND (i32 IntRegs:$src1)), i16)),
 def HexagonJT:     SDNode<"HexagonISD::JT", SDTIntUnaryOp>;
 def HexagonCP:     SDNode<"HexagonISD::CP", SDTIntUnaryOp>;
 
-def: Pat<(HexagonJT tjumptable:$dst),
-         (CONST32_set_jt tjumptable:$dst)>;
-def: Pat<(HexagonCP tconstpool :$dst),
-         (CONST32_set_jt tconstpool:$dst)>;
+def: Pat<(HexagonJT tjumptable:$dst), (A2_tfrsi s16Ext:$dst)>;
+def: Pat<(HexagonCP tconstpool:$dst), (A2_tfrsi s16Ext:$dst)>;
 
 // XTYPE/SHIFT
 //
@@ -5574,41 +5588,34 @@ def S2_insertp_rp : T_S3op_insert<"insert", DoubleRegs>;
 def S2_insertp    : T_S2op_insert <0b0011, DoubleRegs, u6Imm>;
 
 
-def SDTHexagonINSERT_ri : SDTypeProfile<1, 4, [SDTCisVT<0, i32>,
-                                               SDTCisVT<1, i32>,
-                                               SDTCisVT<2, i32>,
-                                               SDTCisVT<3, i32>,
-                                               SDTCisVT<4, i32>]>;
-def SDTHexagonINSERT_rd : SDTypeProfile<1, 4, [SDTCisVT<0, i64>,
-                                               SDTCisVT<1, i64>,
-                                               SDTCisVT<2, i64>,
-                                               SDTCisVT<3, i32>,
-                                               SDTCisVT<4, i32>]>;
-def SDTHexagonINSERT_riv : SDTypeProfile<1, 3, [SDTCisVT<0, i32>,
-                                                SDTCisVT<1, i32>,
-                                                SDTCisVT<2, i32>,
-                                                SDTCisVT<3, i64>]>;
-def SDTHexagonINSERT_rdv : SDTypeProfile<1, 3, [SDTCisVT<0, i64>,
-                                                SDTCisVT<1, i64>,
-                                                SDTCisVT<2, i64>,
-                                                SDTCisVT<3, i64>]>;
-def HexagonINSERT_ri : SDNode<"HexagonISD::INSERT_ri",  SDTHexagonINSERT_ri>;
-def HexagonINSERT_rd : SDNode<"HexagonISD::INSERT_rd",  SDTHexagonINSERT_rd>;
-def HexagonINSERT_riv: SDNode<"HexagonISD::INSERT_riv", SDTHexagonINSERT_riv>;
-def HexagonINSERT_rdv: SDNode<"HexagonISD::INSERT_rdv", SDTHexagonINSERT_rdv>;
-
-def: Pat<(HexagonINSERT_ri I32:$Rs, I32:$Rt, u5ImmPred:$u1, u5ImmPred:$u2),
-         (S2_insert I32:$Rs, I32:$Rt, u5ImmPred:$u1, u5ImmPred:$u2)>;
+def SDTHexagonINSERT:
+  SDTypeProfile<1, 4, [SDTCisSameAs<0, 1>, SDTCisSameAs<0, 2>,
+                       SDTCisInt<0>, SDTCisVT<3, i32>, SDTCisVT<4, i32>]>;
+def SDTHexagonINSERTRP:
+  SDTypeProfile<1, 3, [SDTCisSameAs<0, 1>, SDTCisSameAs<0, 2>,
+                       SDTCisInt<0>, SDTCisVT<3, i64>]>;
 
-def: Pat<(HexagonINSERT_rd I64:$Rs, I64:$Rt, u6ImmPred:$u1, u6ImmPred:$u2),
-         (S2_insertp I64:$Rs, I64:$Rt, u6ImmPred:$u1, u6ImmPred:$u2)>;
+def HexagonINSERT   : SDNode<"HexagonISD::INSERT",   SDTHexagonINSERT>;
+def HexagonINSERTRP : SDNode<"HexagonISD::INSERTRP", SDTHexagonINSERTRP>;
 
-def: Pat<(HexagonINSERT_riv I32:$Rs, I32:$Rt, I64:$Ru),
+def: Pat<(HexagonINSERT I32:$Rs, I32:$Rt, u5ImmPred:$u1, u5ImmPred:$u2),
+         (S2_insert I32:$Rs, I32:$Rt, u5ImmPred:$u1, u5ImmPred:$u2)>;
+def: Pat<(HexagonINSERT I64:$Rs, I64:$Rt, u6ImmPred:$u1, u6ImmPred:$u2),
+         (S2_insertp I64:$Rs, I64:$Rt, u6ImmPred:$u1, u6ImmPred:$u2)>;
+def: Pat<(HexagonINSERTRP I32:$Rs, I32:$Rt, I64:$Ru),
          (S2_insert_rp I32:$Rs, I32:$Rt, I64:$Ru)>;
-
-def: Pat<(HexagonINSERT_rdv I64:$Rs, I64:$Rt, I64:$Ru),
+def: Pat<(HexagonINSERTRP I64:$Rs, I64:$Rt, I64:$Ru),
          (S2_insertp_rp I64:$Rs, I64:$Rt, I64:$Ru)>;
 
+let AddedComplexity = 100 in
+def: Pat<(or (or (shl (HexagonINSERT (i32 (zextloadi8 (add I32:$b, 2))),
+                                     (i32 (extloadi8  (add I32:$b, 3))),
+                                     24, 8),
+                      (i32 16)),
+                 (shl (i32 (zextloadi8 (add I32:$b, 1))), (i32 8))),
+             (zextloadi8 I32:$b)),
+         (A2_swiz (L2_loadri_io I32:$b, 0))>;
+
 
 //===----------------------------------------------------------------------===//
 // Template class for 'extract bitfield' instructions
@@ -5676,35 +5683,23 @@ let hasNewValue = 1 in {
   def S2_extractu    : T_S2op_extract <"extractu", 0b1101, IntRegs, u5Imm>;
 }
 
-def SDTHexagonEXTRACTU_ri : SDTypeProfile<1, 3, [SDTCisVT<0, i32>,
-                                                 SDTCisVT<1, i32>,
-                                                 SDTCisVT<2, i32>,
-                                                 SDTCisVT<3, i32>]>;
-def SDTHexagonEXTRACTU_rd : SDTypeProfile<1, 3, [SDTCisVT<0, i64>,
-                                                 SDTCisVT<1, i64>,
-                                                 SDTCisVT<2, i32>,
-                                                 SDTCisVT<3, i32>]>;
-def SDTHexagonEXTRACTU_riv : SDTypeProfile<1, 2, [SDTCisVT<0, i32>,
-                                                  SDTCisVT<1, i32>,
-                                                  SDTCisVT<2, i64>]>;
-def SDTHexagonEXTRACTU_rdv : SDTypeProfile<1, 2, [SDTCisVT<0, i64>,
-                                                  SDTCisVT<1, i64>,
-                                                  SDTCisVT<2, i64>]>;
-def HexagonEXTRACTU_ri : SDNode<"HexagonISD::EXTRACTU_ri",  SDTHexagonEXTRACTU_ri>;
-def HexagonEXTRACTU_rd : SDNode<"HexagonISD::EXTRACTU_rd",  SDTHexagonEXTRACTU_rd>;
-def HexagonEXTRACTU_riv: SDNode<"HexagonISD::EXTRACTU_riv", SDTHexagonEXTRACTU_riv>;
-def HexagonEXTRACTU_rdv: SDNode<"HexagonISD::EXTRACTU_rdv", SDTHexagonEXTRACTU_rdv>;
-
-def: Pat<(HexagonEXTRACTU_ri I32:$src1, u5ImmPred:$src2, u5ImmPred:$src3),
-         (S2_extractu I32:$src1, u5ImmPred:$src2, u5ImmPred:$src3)>;
+def SDTHexagonEXTRACTU:
+  SDTypeProfile<1, 3, [SDTCisSameAs<0, 1>, SDTCisInt<0>, SDTCisInt<1>,
+                       SDTCisVT<2, i32>, SDTCisVT<3, i32>]>;
+def SDTHexagonEXTRACTURP:
+  SDTypeProfile<1, 2, [SDTCisSameAs<0, 1>, SDTCisInt<0>, SDTCisInt<1>,
+                       SDTCisVT<2, i64>]>;
 
-def: Pat<(HexagonEXTRACTU_rd I64:$src1, u6ImmPred:$src2, u6ImmPred:$src3),
-         (S2_extractup I64:$src1, u6ImmPred:$src2, u6ImmPred:$src3)>;
+def HexagonEXTRACTU   : SDNode<"HexagonISD::EXTRACTU",   SDTHexagonEXTRACTU>;
+def HexagonEXTRACTURP : SDNode<"HexagonISD::EXTRACTURP", SDTHexagonEXTRACTURP>;
 
-def: Pat<(HexagonEXTRACTU_riv I32:$src1, I64:$src2),
+def: Pat<(HexagonEXTRACTU I32:$src1, u5ImmPred:$src2, u5ImmPred:$src3),
+         (S2_extractu I32:$src1, u5ImmPred:$src2, u5ImmPred:$src3)>;
+def: Pat<(HexagonEXTRACTU I64:$src1, u6ImmPred:$src2, u6ImmPred:$src3),
+         (S2_extractup I64:$src1, u6ImmPred:$src2, u6ImmPred:$src3)>;
+def: Pat<(HexagonEXTRACTURP I32:$src1, I64:$src2),
          (S2_extractu_rp I32:$src1, I64:$src2)>;
-
-def: Pat<(HexagonEXTRACTU_rdv I64:$src1, I64:$src2),
+def: Pat<(HexagonEXTRACTURP I64:$src1, I64:$src2),
          (S2_extractup_rp I64:$src1, I64:$src2)>;
 
 // Change the sign of the immediate for Rd=-mpyi(Rs,#u8)
diff --git a/lib/Target/Hexagon/HexagonInstrInfoV4.td b/lib/Target/Hexagon/HexagonInstrInfoV4.td
index 918b482ec70..65b0f497436 100644
--- a/lib/Target/Hexagon/HexagonInstrInfoV4.td
+++ b/lib/Target/Hexagon/HexagonInstrInfoV4.td
@@ -57,9 +57,10 @@ def BITPOS32 : SDNodeXForm<imm, [{
    // Return the bit position we will set [0-31].
    // As an SDNode.
    int32_t imm = N->getSExtValue();
-   return XformMskToBitPosU5Imm(imm);
+   return XformMskToBitPosU5Imm(imm, SDLoc(N));
 }]>;
 
+
 // Hexagon V4 Architecture spec defines 8 instruction classes:
 // LD ST ALU32 XTYPE J JR MEMOP NV CR SYSTEM(system is not implemented in the
 // compiler)
@@ -498,10 +499,23 @@ multiclass T_LoadAbsReg_Pat <PatFrag ldOp, InstHexagon MI, ValueType VT = i32> {
   def  : Pat <(VT (ldOp (add (shl IntRegs:$src1, u2ImmPred:$src2),
                              (HexagonCONST32 tglobaladdr:$src3)))),
               (MI IntRegs:$src1, u2ImmPred:$src2, tglobaladdr:$src3)>;
-
   def  : Pat <(VT (ldOp (add IntRegs:$src1,
                              (HexagonCONST32 tglobaladdr:$src2)))),
               (MI IntRegs:$src1, 0, tglobaladdr:$src2)>;
+
+  def  : Pat <(VT (ldOp (add (shl IntRegs:$src1, u2ImmPred:$src2),
+                             (HexagonCONST32 tconstpool:$src3)))),
+              (MI IntRegs:$src1, u2ImmPred:$src2, tconstpool:$src3)>;
+  def  : Pat <(VT (ldOp (add IntRegs:$src1,
+                             (HexagonCONST32 tconstpool:$src2)))),
+              (MI IntRegs:$src1, 0, tconstpool:$src2)>;
+
+  def  : Pat <(VT (ldOp (add (shl IntRegs:$src1, u2ImmPred:$src2),
+                             (HexagonCONST32 tjumptable:$src3)))),
+              (MI IntRegs:$src1, u2ImmPred:$src2, tjumptable:$src3)>;
+  def  : Pat <(VT (ldOp (add IntRegs:$src1,
+                             (HexagonCONST32 tjumptable:$src2)))),
+              (MI IntRegs:$src1, 0, tjumptable:$src2)>;
 }
 
 let AddedComplexity  = 60 in {
@@ -1139,14 +1153,14 @@ def IMM_BYTE : SDNodeXForm<imm, [{
   // -1 etc is  represented as 255 etc
   // assigning to a byte restores our desired signed value.
   int8_t imm = N->getSExtValue();
-  return CurDAG->getTargetConstant(imm, MVT::i32);
+  return CurDAG->getTargetConstant(imm, SDLoc(N), MVT::i32);
 }]>;
 
 def IMM_HALF : SDNodeXForm<imm, [{
   // -1 etc is  represented as 65535 etc
   // assigning to a short restores our desired signed value.
   int16_t imm = N->getSExtValue();
-  return CurDAG->getTargetConstant(imm, MVT::i32);
+  return CurDAG->getTargetConstant(imm, SDLoc(N), MVT::i32);
 }]>;
 
 def IMM_WORD : SDNodeXForm<imm, [{
@@ -1155,7 +1169,7 @@ def IMM_WORD : SDNodeXForm<imm, [{
   // might convert -1 to a large +ve number.
   // assigning to a word restores our desired signed value.
   int32_t imm = N->getSExtValue();
-  return CurDAG->getTargetConstant(imm, MVT::i32);
+  return CurDAG->getTargetConstant(imm, SDLoc(N), MVT::i32);
 }]>;
 
 def ToImmByte : OutPatFrag<(ops node:$R), (IMM_BYTE $R)>;
@@ -2311,10 +2325,24 @@ def S2_ct0p : T_COUNT_LEADING_64<"ct0", 0b111, 0b010>;
 def S2_ct1p : T_COUNT_LEADING_64<"ct1", 0b111, 0b100>;
 def S4_clbpnorm : T_COUNT_LEADING_64<"normamt", 0b011, 0b000>;
 
-def: Pat<(i32 (trunc (cttz (i64 DoubleRegs:$Rss)))),
-         (S2_ct0p (i64 DoubleRegs:$Rss))>;
-def: Pat<(i32 (trunc (cttz (not (i64 DoubleRegs:$Rss))))),
-         (S2_ct1p (i64 DoubleRegs:$Rss))>;
+// Count trailing zeros: 64-bit.
+def: Pat<(i32 (trunc (cttz I64:$Rss))), (S2_ct0p I64:$Rss)>;
+def: Pat<(i32 (trunc (cttz_zero_undef I64:$Rss))), (S2_ct0p I64:$Rss)>;
+
+// Count trailing ones: 64-bit.
+def: Pat<(i32 (trunc (cttz (not I64:$Rss)))), (S2_ct1p I64:$Rss)>;
+def: Pat<(i32 (trunc (cttz_zero_undef (not I64:$Rss)))), (S2_ct1p I64:$Rss)>;
+
+// Define leading/trailing patterns that require zero-extensions to 64 bits.
+def: Pat<(i64 (ctlz I64:$Rss)), (Zext64 (S2_cl0p I64:$Rss))>;
+def: Pat<(i64 (ctlz_zero_undef I64:$Rss)), (Zext64 (S2_cl0p I64:$Rss))>;
+def: Pat<(i64 (cttz I64:$Rss)), (Zext64 (S2_ct0p I64:$Rss))>;
+def: Pat<(i64 (cttz_zero_undef I64:$Rss)), (Zext64 (S2_ct0p I64:$Rss))>;
+def: Pat<(i64 (ctlz (not I64:$Rss))), (Zext64 (S2_cl1p I64:$Rss))>;
+def: Pat<(i64 (ctlz_zero_undef (not I64:$Rss))), (Zext64 (S2_cl1p I64:$Rss))>;
+def: Pat<(i64 (cttz (not I64:$Rss))), (Zext64 (S2_ct1p I64:$Rss))>;
+def: Pat<(i64 (cttz_zero_undef (not I64:$Rss))), (Zext64 (S2_ct1p I64:$Rss))>;
+
 
 let hasSideEffects = 0, hasNewValue = 1 in
 def S4_clbaddi : SInst<(outs IntRegs:$Rd), (ins IntRegs:$Rs, s6Imm:$s6),
@@ -2777,7 +2805,7 @@ def MEMOPIMM : SDNodeXForm<imm, [{
   // Call the transformation function XformM5ToU5Imm to get the negative
   // immediate's positive counterpart.
   int32_t imm = N->getSExtValue();
-  return XformM5ToU5Imm(imm);
+  return XformM5ToU5Imm(imm, SDLoc(N));
 }]>;
 
 def MEMOPIMM_HALF : SDNodeXForm<imm, [{
@@ -2786,7 +2814,7 @@ def MEMOPIMM_HALF : SDNodeXForm<imm, [{
   // Call the transformation function XformM5ToU5Imm to get the negative
   // immediate's positive counterpart.
   int16_t imm = N->getSExtValue();
-  return XformM5ToU5Imm(imm);
+  return XformM5ToU5Imm(imm, SDLoc(N));
 }]>;
 
 def MEMOPIMM_BYTE : SDNodeXForm<imm, [{
@@ -2795,14 +2823,14 @@ def MEMOPIMM_BYTE : SDNodeXForm<imm, [{
   // Call the transformation function XformM5ToU5Imm to get the negative
   // immediate's positive counterpart.
   int8_t imm = N->getSExtValue();
-  return XformM5ToU5Imm(imm);
+  return XformM5ToU5Imm(imm, SDLoc(N));
 }]>;
 
 def SETMEMIMM : SDNodeXForm<imm, [{
    // Return the bit position we will set [0-31].
    // As an SDNode.
    int32_t imm = N->getSExtValue();
-   return XformMskToBitPosU5Imm(imm);
+   return XformMskToBitPosU5Imm(imm, SDLoc(N));
 }]>;
 
 def CLRMEMIMM : SDNodeXForm<imm, [{
@@ -2810,14 +2838,14 @@ def CLRMEMIMM : SDNodeXForm<imm, [{
    // As an SDNode.
    // we bit negate the value first
    int32_t imm = ~(N->getSExtValue());
-   return XformMskToBitPosU5Imm(imm);
+   return XformMskToBitPosU5Imm(imm, SDLoc(N));
 }]>;
 
 def SETMEMIMM_SHORT : SDNodeXForm<imm, [{
    // Return the bit position we will set [0-15].
    // As an SDNode.
    int16_t imm = N->getSExtValue();
-   return XformMskToBitPosU4Imm(imm);
+   return XformMskToBitPosU4Imm(imm, SDLoc(N));
 }]>;
 
 def CLRMEMIMM_SHORT : SDNodeXForm<imm, [{
@@ -2825,14 +2853,14 @@ def CLRMEMIMM_SHORT : SDNodeXForm<imm, [{
    // As an SDNode.
    // we bit negate the value first
    int16_t imm = ~(N->getSExtValue());
-   return XformMskToBitPosU4Imm(imm);
+   return XformMskToBitPosU4Imm(imm, SDLoc(N));
 }]>;
 
 def SETMEMIMM_BYTE : SDNodeXForm<imm, [{
    // Return the bit position we will set [0-7].
    // As an SDNode.
    int8_t imm =  N->getSExtValue();
-   return XformMskToBitPosU3Imm(imm);
+   return XformMskToBitPosU3Imm(imm, SDLoc(N));
 }]>;
 
 def CLRMEMIMM_BYTE : SDNodeXForm<imm, [{
@@ -2840,7 +2868,7 @@ def CLRMEMIMM_BYTE : SDNodeXForm<imm, [{
    // As an SDNode.
    // we bit negate the value first
    int8_t imm = ~(N->getSExtValue());
-   return XformMskToBitPosU3Imm(imm);
+   return XformMskToBitPosU3Imm(imm, SDLoc(N));
 }]>;
 
 //===----------------------------------------------------------------------===//
@@ -3174,7 +3202,7 @@ def: Pat<(i1 (setne (i32 IntRegs:$src1), s32ImmPred:$src2)),
 def DEC_CONST_BYTE : SDNodeXForm<imm, [{
    // Return the byte immediate const-1 as an SDNode.
    int32_t imm = N->getSExtValue();
-   return XformU7ToU7M1Imm(imm);
+   return XformU7ToU7M1Imm(imm, SDLoc(N));
 }]>;
 
 // For the sequence
@@ -3770,9 +3798,6 @@ def: Pat<(HexagonCONST32 tglobaladdr:$Rs),      (A2_tfrsi s16Ext:$Rs)>;
 def: Pat<(HexagonCONST32_GP tblockaddress:$Rs), (A2_tfrsi s16Ext:$Rs)>;
 def: Pat<(HexagonCONST32_GP tglobaladdr:$Rs),   (A2_tfrsi s16Ext:$Rs)>;
 
-def: Pat<(i64 (ctlz I64:$src1)), (Zext64 (S2_cl0p I64:$src1))>;
-def: Pat<(i64 (cttz I64:$src1)), (Zext64 (S2_ct0p I64:$src1))>;
-
 let AddedComplexity  = 30 in {
   def: Storea_pat<truncstorei8,  I32, u32ImmPred, S2_storerbabs>;
   def: Storea_pat<truncstorei16, I32, u32ImmPred, S2_storerhabs>;
@@ -4238,3 +4263,7 @@ def J4_jumpsetr: CJInst <
     let Inst{19-16} = Rs;
     let Inst{7-1} = r9_2{8-2};
   }
+
+// Duplex instructions
+//===----------------------------------------------------------------------===//
+include "HexagonIsetDx.td"
diff --git a/lib/Target/Hexagon/HexagonIntrinsics.td b/lib/Target/Hexagon/HexagonIntrinsics.td
index 4275230ba71..1d0d015f798 100644
--- a/lib/Target/Hexagon/HexagonIntrinsics.td
+++ b/lib/Target/Hexagon/HexagonIntrinsics.td
@@ -676,6 +676,7 @@ def  : Pat <(int_hexagon_A2_tfrih (I32:$Rs), u16_0ImmPred:$Is),
 //  Transfer Register/immediate.
 def : T_R_pat <A2_tfr, int_hexagon_A2_tfr>;
 def : T_I_pat <A2_tfrsi, int_hexagon_A2_tfrsi>;
+def : T_I_pat <A2_tfrpi, int_hexagon_A2_tfrpi>;
 
 // Assembler mapped from Rdd32=Rss32 to Rdd32=combine(Rss.H32,Rss.L32)
 def : Pat<(int_hexagon_A2_tfrp DoubleRegs:$src),
@@ -690,15 +691,15 @@ def: T_RR_pat<A2_combine_hl, int_hexagon_A2_combine_hl>;
 def: T_RR_pat<A2_combine_lh, int_hexagon_A2_combine_lh>;
 def: T_RR_pat<A2_combine_ll, int_hexagon_A2_combine_ll>;
 
-def: T_II_pat<A2_combineii, int_hexagon_A2_combineii, s32ImmPred, s8ImmPred>;
+def: T_II_pat<A2_combineii, int_hexagon_A2_combineii, s8ExtPred, s8ImmPred>;
 
 def: Pat<(i32 (int_hexagon_C2_mux (I32:$Rp), (I32:$Rs), (I32:$Rt))),
          (i32 (C2_mux (C2_tfrrp IntRegs:$Rp), IntRegs:$Rs, IntRegs:$Rt))>;
 
 // Mux
-def : T_QRI_pat<C2_muxir, int_hexagon_C2_muxir, s32ImmPred>;
-def : T_QIR_pat<C2_muxri, int_hexagon_C2_muxri, s32ImmPred>;
-def : T_QII_pat<C2_muxii, int_hexagon_C2_muxii, s32ImmPred, s8ImmPred>;
+def : T_QRI_pat<C2_muxir, int_hexagon_C2_muxir, s8ExtPred>;
+def : T_QIR_pat<C2_muxri, int_hexagon_C2_muxri, s8ExtPred>;
+def : T_QII_pat<C2_muxii, int_hexagon_C2_muxii, s8ExtPred, s8ImmPred>;
 
 // Shift halfword
 def : T_R_pat<A2_aslh, int_hexagon_A2_aslh>;
@@ -719,17 +720,17 @@ def : T_RR_pat<C2_cmpeq,  int_hexagon_C2_cmpeq>;
 def : T_RR_pat<C2_cmpgt,  int_hexagon_C2_cmpgt>;
 def : T_RR_pat<C2_cmpgtu, int_hexagon_C2_cmpgtu>;
 
-def : T_RI_pat<C2_cmpeqi, int_hexagon_C2_cmpeqi, s32ImmPred>;
-def : T_RI_pat<C2_cmpgti, int_hexagon_C2_cmpgti, s32ImmPred>;
-def : T_RI_pat<C2_cmpgtui, int_hexagon_C2_cmpgtui, u32ImmPred>;
+def : T_RI_pat<C2_cmpeqi, int_hexagon_C2_cmpeqi, s10ExtPred>;
+def : T_RI_pat<C2_cmpgti, int_hexagon_C2_cmpgti, s10ExtPred>;
+def : T_RI_pat<C2_cmpgtui, int_hexagon_C2_cmpgtui, u9ExtPred>;
 
-def : Pat <(i32 (int_hexagon_C2_cmpgei (I32:$src1), s32ImmPred:$src2)),
+def : Pat <(i32 (int_hexagon_C2_cmpgei (I32:$src1), s8ExtPred:$src2)),
       (i32 (C2_cmpgti (I32:$src1),
-                      (DEC_CONST_SIGNED s32ImmPred:$src2)))>;
+                      (DEC_CONST_SIGNED s8ExtPred:$src2)))>;
 
-def : Pat <(i32 (int_hexagon_C2_cmpgeui (I32:$src1), u32ImmPred:$src2)),
+def : Pat <(i32 (int_hexagon_C2_cmpgeui (I32:$src1), u8ExtPred:$src2)),
       (i32 (C2_cmpgtui (I32:$src1),
-                       (DEC_CONST_UNSIGNED u32ImmPred:$src2)))>;
+                       (DEC_CONST_UNSIGNED u8ExtPred:$src2)))>;
 
 // The instruction, Pd=cmp.geu(Rs, #u8) -> Pd=cmp.eq(Rs,Rs) when #u8 == 0.
 def : Pat <(i32 (int_hexagon_C2_cmpgeui (I32:$src1), 0)),
@@ -923,6 +924,10 @@ def: qi_CRInst_qiqi_pat<C2_or,    int_hexagon_C2_or>;
 def: qi_CRInst_qiqi_pat<C2_orn,   int_hexagon_C2_orn>;
 def: qi_CRInst_qiqi_pat<C2_xor,   int_hexagon_C2_xor>;
 
+// Assembler mapped from  Pd4=Ps4 to Pd4=or(Ps4,Ps4)
+def : Pat<(int_hexagon_C2_pxfer_map PredRegs:$src),
+          (C2_pxfer_map PredRegs:$src)>;
+
 // Multiply 32x32 and use lower result
 def : T_RRI_pat <M2_macsip, int_hexagon_M2_macsip>;
 def : T_RRI_pat <M2_macsin, int_hexagon_M2_macsin>;
diff --git a/lib/Target/Hexagon/HexagonIsetDx.td b/lib/Target/Hexagon/HexagonIsetDx.td
new file mode 100644
index 00000000000..0ca95e99985
--- /dev/null
+++ b/lib/Target/Hexagon/HexagonIsetDx.td
@@ -0,0 +1,728 @@
+//=- HexagonIsetDx.td - Target Desc. for Hexagon Target -*- tablegen -*-=//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This file describes the Hexagon duplex instructions.
+//
+//===----------------------------------------------------------------------===//
+
+// SA1_combine1i: Combines.
+let isCodeGenOnly = 1, hasSideEffects = 0 in
+def V4_SA1_combine1i: SUBInst <
+  (outs DoubleRegs:$Rdd),
+  (ins u2Imm:$u2),
+  "$Rdd = combine(#1, #$u2)"> {
+    bits<3> Rdd;
+    bits<2> u2;
+
+    let Inst{12-10} = 0b111;
+    let Inst{8} = 0b0;
+    let Inst{4-3} = 0b01;
+    let Inst{2-0} = Rdd;
+    let Inst{6-5} = u2;
+  }
+
+// SL2_jumpr31_f: Indirect conditional jump if false.
+// SL2_jumpr31_f -> SL2_jumpr31_fnew
+let Defs = [PC], Uses = [P0, R31], isCodeGenOnly = 1, isPredicated = 1, isPredicatedFalse = 1, isBranch = 1, isIndirectBranch = 1, hasSideEffects = 0 in
+def V4_SL2_jumpr31_f: SUBInst <
+  (outs ),
+  (ins ),
+  "if (!p0) jumpr r31"> {
+    let Inst{12-6} = 0b1111111;
+    let Inst{2-0} = 0b101;
+  }
+
+// SL2_deallocframe: Deallocate stack frame.
+let Defs = [R31, R29, R30], Uses = [R30], isCodeGenOnly = 1, mayLoad = 1, accessSize = DoubleWordAccess in
+def V4_SL2_deallocframe: SUBInst <
+  (outs ),
+  (ins ),
+  "deallocframe"> {
+    let Inst{12-6} = 0b1111100;
+    let Inst{2} = 0b0;
+  }
+
+// SL2_return_f: Deallocate stack frame and return.
+// SL2_return_f -> SL2_return_fnew
+let Defs = [PC, R31, R29, R30], Uses = [R30, P0], isCodeGenOnly = 1, isPredicated = 1, isPredicatedFalse = 1, mayLoad = 1, accessSize = DoubleWordAccess, isBranch = 1, isIndirectBranch = 1 in
+def V4_SL2_return_f: SUBInst <
+  (outs ),
+  (ins ),
+  "if (!p0) dealloc_return"> {
+    let Inst{12-6} = 0b1111101;
+    let Inst{2-0} = 0b101;
+  }
+
+// SA1_combine3i: Combines.
+let isCodeGenOnly = 1, hasSideEffects = 0 in
+def V4_SA1_combine3i: SUBInst <
+  (outs DoubleRegs:$Rdd),
+  (ins u2Imm:$u2),
+  "$Rdd = combine(#3, #$u2)"> {
+    bits<3> Rdd;
+    bits<2> u2;
+
+    let Inst{12-10} = 0b111;
+    let Inst{8} = 0b0;
+    let Inst{4-3} = 0b11;
+    let Inst{2-0} = Rdd;
+    let Inst{6-5} = u2;
+  }
+
+// SS2_storebi0: Store byte.
+let isCodeGenOnly = 1, mayStore = 1, accessSize = ByteAccess in
+def V4_SS2_storebi0: SUBInst <
+  (outs ),
+  (ins IntRegs:$Rs, u4_0Imm:$u4_0),
+  "memb($Rs + #$u4_0)=#0"> {
+    bits<4> Rs;
+    bits<4> u4_0;
+
+    let Inst{12-8} = 0b10010;
+    let Inst{7-4} = Rs;
+    let Inst{3-0} = u4_0;
+  }
+
+// SA1_clrtnew: Clear if true.
+let Uses = [P0], isCodeGenOnly = 1, isPredicated = 1, isPredicatedNew = 1, hasSideEffects = 0, hasNewValue = 1, opNewValue = 0 in
+def V4_SA1_clrtnew: SUBInst <
+  (outs IntRegs:$Rd),
+  (ins ),
+  "if (p0.new) $Rd = #0"> {
+    bits<4> Rd;
+
+    let Inst{12-9} = 0b1101;
+    let Inst{6-4} = 0b100;
+    let Inst{3-0} = Rd;
+  }
+
+// SL2_loadruh_io: Load half.
+let isCodeGenOnly = 1, mayLoad = 1, accessSize = HalfWordAccess, hasNewValue = 1, opNewValue = 0 in
+def V4_SL2_loadruh_io: SUBInst <
+  (outs IntRegs:$Rd),
+  (ins IntRegs:$Rs, u3_1Imm:$u3_1),
+  "$Rd = memuh($Rs + #$u3_1)"> {
+    bits<4> Rd;
+    bits<4> Rs;
+    bits<4> u3_1;
+
+    let Inst{12-11} = 0b01;
+    let Inst{3-0} = Rd;
+    let Inst{7-4} = Rs;
+    let Inst{10-8} = u3_1{3-1};
+  }
+
+// SL2_jumpr31_tnew: Indirect conditional jump if true.
+let Defs = [PC], Uses = [P0, R31], isCodeGenOnly = 1, isPredicated = 1, isPredicatedNew = 1, isBranch = 1, isIndirectBranch = 1, hasSideEffects = 0 in
+def V4_SL2_jumpr31_tnew: SUBInst <
+  (outs ),
+  (ins ),
+  "if (p0.new) jumpr:nt r31"> {
+    let Inst{12-6} = 0b1111111;
+    let Inst{2-0} = 0b110;
+  }
+
+// SA1_addi: Add.
+let isCodeGenOnly = 1, hasSideEffects = 0, hasNewValue = 1, opNewValue = 0, isExtendable = 1, isExtentSigned = 1, opExtentBits = 7, opExtendable = 2 in
+def V4_SA1_addi: SUBInst <
+  (outs IntRegs:$Rx),
+  (ins IntRegs:$_src_, s7Ext:$s7),
+  "$Rx = add($_src_, #$s7)" ,
+  [] ,
+  "$_src_ = $Rx"> {
+    bits<4> Rx;
+    bits<7> s7;
+
+    let Inst{12-11} = 0b00;
+    let Inst{3-0} = Rx;
+    let Inst{10-4} = s7;
+  }
+
+// SL1_loadrub_io: Load byte.
+let isCodeGenOnly = 1, mayLoad = 1, accessSize = ByteAccess, hasNewValue = 1, opNewValue = 0 in
+def V4_SL1_loadrub_io: SUBInst <
+  (outs IntRegs:$Rd),
+  (ins IntRegs:$Rs, u4_0Imm:$u4_0),
+  "$Rd = memub($Rs + #$u4_0)"> {
+    bits<4> Rd;
+    bits<4> Rs;
+    bits<4> u4_0;
+
+    let Inst{12} = 0b1;
+    let Inst{3-0} = Rd;
+    let Inst{7-4} = Rs;
+    let Inst{11-8} = u4_0;
+  }
+
+// SL1_loadri_io: Load word.
+let isCodeGenOnly = 1, mayLoad = 1, accessSize = WordAccess, hasNewValue = 1, opNewValue = 0 in
+def V4_SL1_loadri_io: SUBInst <
+  (outs IntRegs:$Rd),
+  (ins IntRegs:$Rs, u4_2Imm:$u4_2),
+  "$Rd = memw($Rs + #$u4_2)"> {
+    bits<4> Rd;
+    bits<4> Rs;
+    bits<6> u4_2;
+
+    let Inst{12} = 0b0;
+    let Inst{3-0} = Rd;
+    let Inst{7-4} = Rs;
+    let Inst{11-8} = u4_2{5-2};
+  }
+
+// SA1_cmpeqi: Compareimmed.
+let Defs = [P0], isCodeGenOnly = 1, hasSideEffects = 0 in
+def V4_SA1_cmpeqi: SUBInst <
+  (outs ),
+  (ins IntRegs:$Rs, u2Imm:$u2),
+  "p0 = cmp.eq($Rs, #$u2)"> {
+    bits<4> Rs;
+    bits<2> u2;
+
+    let Inst{12-8} = 0b11001;
+    let Inst{7-4} = Rs;
+    let Inst{1-0} = u2;
+  }
+
+// SA1_combinerz: Combines.
+let isCodeGenOnly = 1, hasSideEffects = 0 in
+def V4_SA1_combinerz: SUBInst <
+  (outs DoubleRegs:$Rdd),
+  (ins IntRegs:$Rs),
+  "$Rdd = combine($Rs, #0)"> {
+    bits<3> Rdd;
+    bits<4> Rs;
+
+    let Inst{12-10} = 0b111;
+    let Inst{8} = 0b1;
+    let Inst{3} = 0b1;
+    let Inst{2-0} = Rdd;
+    let Inst{7-4} = Rs;
+  }
+
+// SL2_return_t: Deallocate stack frame and return.
+// SL2_return_t -> SL2_return_tnew
+let Defs = [PC, R31, R29, R30], Uses = [R30, P0], isCodeGenOnly = 1, isPredicated = 1, mayLoad = 1, accessSize = DoubleWordAccess, isBranch = 1, isIndirectBranch = 1 in
+def V4_SL2_return_t: SUBInst <
+  (outs ),
+  (ins ),
+  "if (p0) dealloc_return"> {
+    let Inst{12-6} = 0b1111101;
+    let Inst{2-0} = 0b100;
+  }
+
+// SS2_allocframe: Allocate stack frame.
+let Defs = [R29, R30], Uses = [R30, R31, R29], isCodeGenOnly = 1, mayStore = 1, accessSize = DoubleWordAccess in
+def V4_SS2_allocframe: SUBInst <
+  (outs ),
+  (ins u5_3Imm:$u5_3),
+  "allocframe(#$u5_3)"> {
+    bits<8> u5_3;
+
+    let Inst{12-9} = 0b1110;
+    let Inst{8-4} = u5_3{7-3};
+  }
+
+// SS2_storeh_io: Store half.
+let isCodeGenOnly = 1, mayStore = 1, accessSize = HalfWordAccess in
+def V4_SS2_storeh_io: SUBInst <
+  (outs ),
+  (ins IntRegs:$Rs, u3_1Imm:$u3_1, IntRegs:$Rt),
+  "memh($Rs + #$u3_1) = $Rt"> {
+    bits<4> Rs;
+    bits<4> u3_1;
+    bits<4> Rt;
+
+    let Inst{12-11} = 0b00;
+    let Inst{7-4} = Rs;
+    let Inst{10-8} = u3_1{3-1};
+    let Inst{3-0} = Rt;
+  }
+
+// SS2_storewi0: Store word.
+let isCodeGenOnly = 1, mayStore = 1, accessSize = WordAccess in
+def V4_SS2_storewi0: SUBInst <
+  (outs ),
+  (ins IntRegs:$Rs, u4_2Imm:$u4_2),
+  "memw($Rs + #$u4_2)=#0"> {
+    bits<4> Rs;
+    bits<6> u4_2;
+
+    let Inst{12-8} = 0b10000;
+    let Inst{7-4} = Rs;
+    let Inst{3-0} = u4_2{5-2};
+  }
+
+// SS2_storewi1: Store word.
+let isCodeGenOnly = 1, mayStore = 1, accessSize = WordAccess in
+def V4_SS2_storewi1: SUBInst <
+  (outs ),
+  (ins IntRegs:$Rs, u4_2Imm:$u4_2),
+  "memw($Rs + #$u4_2)=#1"> {
+    bits<4> Rs;
+    bits<6> u4_2;
+
+    let Inst{12-8} = 0b10001;
+    let Inst{7-4} = Rs;
+    let Inst{3-0} = u4_2{5-2};
+  }
+
+// SL2_jumpr31: Indirect conditional jump if true.
+let Defs = [PC], Uses = [R31], isCodeGenOnly = 1, isBranch = 1, isIndirectBranch = 1, hasSideEffects = 0 in
+def V4_SL2_jumpr31: SUBInst <
+  (outs ),
+  (ins ),
+  "jumpr r31"> {
+    let Inst{12-6} = 0b1111111;
+    let Inst{2} = 0b0;
+  }
+
+// SA1_combinezr: Combines.
+let isCodeGenOnly = 1, hasSideEffects = 0 in
+def V4_SA1_combinezr: SUBInst <
+  (outs DoubleRegs:$Rdd),
+  (ins IntRegs:$Rs),
+  "$Rdd = combine(#0, $Rs)"> {
+    bits<3> Rdd;
+    bits<4> Rs;
+
+    let Inst{12-10} = 0b111;
+    let Inst{8} = 0b1;
+    let Inst{3} = 0b0;
+    let Inst{2-0} = Rdd;
+    let Inst{7-4} = Rs;
+  }
+
+// SL2_loadrh_io: Load half.
+let isCodeGenOnly = 1, mayLoad = 1, accessSize = HalfWordAccess, hasNewValue = 1, opNewValue = 0 in
+def V4_SL2_loadrh_io: SUBInst <
+  (outs IntRegs:$Rd),
+  (ins IntRegs:$Rs, u3_1Imm:$u3_1),
+  "$Rd = memh($Rs + #$u3_1)"> {
+    bits<4> Rd;
+    bits<4> Rs;
+    bits<4> u3_1;
+
+    let Inst{12-11} = 0b00;
+    let Inst{3-0} = Rd;
+    let Inst{7-4} = Rs;
+    let Inst{10-8} = u3_1{3-1};
+  }
+
+// SA1_addrx: Add.
+let isCodeGenOnly = 1, hasSideEffects = 0, hasNewValue = 1, opNewValue = 0 in
+def V4_SA1_addrx: SUBInst <
+  (outs IntRegs:$Rx),
+  (ins IntRegs:$_src_, IntRegs:$Rs),
+  "$Rx = add($_src_, $Rs)" ,
+  [] ,
+  "$_src_ = $Rx"> {
+    bits<4> Rx;
+    bits<4> Rs;
+
+    let Inst{12-8} = 0b11000;
+    let Inst{3-0} = Rx;
+    let Inst{7-4} = Rs;
+  }
+
+// SA1_setin1: Set to -1.
+let isCodeGenOnly = 1, hasSideEffects = 0, hasNewValue = 1, opNewValue = 0 in
+def V4_SA1_setin1: SUBInst <
+  (outs IntRegs:$Rd),
+  (ins ),
+  "$Rd = #-1"> {
+    bits<4> Rd;
+
+    let Inst{12-9} = 0b1101;
+    let Inst{6} = 0b0;
+    let Inst{3-0} = Rd;
+  }
+
+// SA1_sxth: Sxth.
+let isCodeGenOnly = 1, hasSideEffects = 0, hasNewValue = 1, opNewValue = 0 in
+def V4_SA1_sxth: SUBInst <
+  (outs IntRegs:$Rd),
+  (ins IntRegs:$Rs),
+  "$Rd = sxth($Rs)"> {
+    bits<4> Rd;
+    bits<4> Rs;
+
+    let Inst{12-8} = 0b10100;
+    let Inst{3-0} = Rd;
+    let Inst{7-4} = Rs;
+  }
+
+// SA1_combine0i: Combines.
+let isCodeGenOnly = 1, hasSideEffects = 0 in
+def V4_SA1_combine0i: SUBInst <
+  (outs DoubleRegs:$Rdd),
+  (ins u2Imm:$u2),
+  "$Rdd = combine(#0, #$u2)"> {
+    bits<3> Rdd;
+    bits<2> u2;
+
+    let Inst{12-10} = 0b111;
+    let Inst{8} = 0b0;
+    let Inst{4-3} = 0b00;
+    let Inst{2-0} = Rdd;
+    let Inst{6-5} = u2;
+  }
+
+// SA1_combine2i: Combines.
+let isCodeGenOnly = 1, hasSideEffects = 0 in
+def V4_SA1_combine2i: SUBInst <
+  (outs DoubleRegs:$Rdd),
+  (ins u2Imm:$u2),
+  "$Rdd = combine(#2, #$u2)"> {
+    bits<3> Rdd;
+    bits<2> u2;
+
+    let Inst{12-10} = 0b111;
+    let Inst{8} = 0b0;
+    let Inst{4-3} = 0b10;
+    let Inst{2-0} = Rdd;
+    let Inst{6-5} = u2;
+  }
+
+// SA1_sxtb: Sxtb.
+let isCodeGenOnly = 1, hasSideEffects = 0, hasNewValue = 1, opNewValue = 0 in
+def V4_SA1_sxtb: SUBInst <
+  (outs IntRegs:$Rd),
+  (ins IntRegs:$Rs),
+  "$Rd = sxtb($Rs)"> {
+    bits<4> Rd;
+    bits<4> Rs;
+
+    let Inst{12-8} = 0b10101;
+    let Inst{3-0} = Rd;
+    let Inst{7-4} = Rs;
+  }
+
+// SA1_clrf: Clear if false.
+// SA1_clrf -> SA1_clrfnew
+let Uses = [P0], isCodeGenOnly = 1, isPredicated = 1, isPredicatedFalse = 1, hasSideEffects = 0, hasNewValue = 1, opNewValue = 0 in
+def V4_SA1_clrf: SUBInst <
+  (outs IntRegs:$Rd),
+  (ins ),
+  "if (!p0) $Rd = #0"> {
+    bits<4> Rd;
+
+    let Inst{12-9} = 0b1101;
+    let Inst{6-4} = 0b111;
+    let Inst{3-0} = Rd;
+  }
+
+// SL2_loadrb_io: Load byte.
+let isCodeGenOnly = 1, mayLoad = 1, accessSize = ByteAccess, hasNewValue = 1, opNewValue = 0 in
+def V4_SL2_loadrb_io: SUBInst <
+  (outs IntRegs:$Rd),
+  (ins IntRegs:$Rs, u3_0Imm:$u3_0),
+  "$Rd = memb($Rs + #$u3_0)"> {
+    bits<4> Rd;
+    bits<4> Rs;
+    bits<3> u3_0;
+
+    let Inst{12-11} = 0b10;
+    let Inst{3-0} = Rd;
+    let Inst{7-4} = Rs;
+    let Inst{10-8} = u3_0;
+  }
+
+// SA1_tfr: Tfr.
+let isCodeGenOnly = 1, hasSideEffects = 0, hasNewValue = 1, opNewValue = 0 in
+def V4_SA1_tfr: SUBInst <
+  (outs IntRegs:$Rd),
+  (ins IntRegs:$Rs),
+  "$Rd = $Rs"> {
+    bits<4> Rd;
+    bits<4> Rs;
+
+    let Inst{12-8} = 0b10000;
+    let Inst{3-0} = Rd;
+    let Inst{7-4} = Rs;
+  }
+
+// SL2_loadrd_sp: Load dword.
+let Uses = [R29], isCodeGenOnly = 1, mayLoad = 1, accessSize = DoubleWordAccess in
+def V4_SL2_loadrd_sp: SUBInst <
+  (outs DoubleRegs:$Rdd),
+  (ins u5_3Imm:$u5_3),
+  "$Rdd = memd(r29 + #$u5_3)"> {
+    bits<3> Rdd;
+    bits<8> u5_3;
+
+    let Inst{12-8} = 0b11110;
+    let Inst{2-0} = Rdd;
+    let Inst{7-3} = u5_3{7-3};
+  }
+
+// SA1_and1: And #1.
+let isCodeGenOnly = 1, hasSideEffects = 0, hasNewValue = 1, opNewValue = 0 in
+def V4_SA1_and1: SUBInst <
+  (outs IntRegs:$Rd),
+  (ins IntRegs:$Rs),
+  "$Rd = and($Rs, #1)"> {
+    bits<4> Rd;
+    bits<4> Rs;
+
+    let Inst{12-8} = 0b10010;
+    let Inst{3-0} = Rd;
+    let Inst{7-4} = Rs;
+  }
+
+// SS2_storebi1: Store byte.
+let isCodeGenOnly = 1, mayStore = 1, accessSize = ByteAccess in
+def V4_SS2_storebi1: SUBInst <
+  (outs ),
+  (ins IntRegs:$Rs, u4_0Imm:$u4_0),
+  "memb($Rs + #$u4_0)=#1"> {
+    bits<4> Rs;
+    bits<4> u4_0;
+
+    let Inst{12-8} = 0b10011;
+    let Inst{7-4} = Rs;
+    let Inst{3-0} = u4_0;
+  }
+
+// SA1_inc: Inc.
+let isCodeGenOnly = 1, hasSideEffects = 0, hasNewValue = 1, opNewValue = 0 in
+def V4_SA1_inc: SUBInst <
+  (outs IntRegs:$Rd),
+  (ins IntRegs:$Rs),
+  "$Rd = add($Rs, #1)"> {
+    bits<4> Rd;
+    bits<4> Rs;
+
+    let Inst{12-8} = 0b10001;
+    let Inst{3-0} = Rd;
+    let Inst{7-4} = Rs;
+  }
+
+// SS2_stored_sp: Store dword.
+let Uses = [R29], isCodeGenOnly = 1, mayStore = 1, accessSize = DoubleWordAccess in
+def V4_SS2_stored_sp: SUBInst <
+  (outs ),
+  (ins s6_3Imm:$s6_3, DoubleRegs:$Rtt),
+  "memd(r29 + #$s6_3) = $Rtt"> {
+    bits<9> s6_3;
+    bits<3> Rtt;
+
+    let Inst{12-9} = 0b0101;
+    let Inst{8-3} = s6_3{8-3};
+    let Inst{2-0} = Rtt;
+  }
+
+// SS2_storew_sp: Store word.
+let Uses = [R29], isCodeGenOnly = 1, mayStore = 1, accessSize = WordAccess in
+def V4_SS2_storew_sp: SUBInst <
+  (outs ),
+  (ins u5_2Imm:$u5_2, IntRegs:$Rt),
+  "memw(r29 + #$u5_2) = $Rt"> {
+    bits<7> u5_2;
+    bits<4> Rt;
+
+    let Inst{12-9} = 0b0100;
+    let Inst{8-4} = u5_2{6-2};
+    let Inst{3-0} = Rt;
+  }
+
+// SL2_jumpr31_fnew: Indirect conditional jump if false.
+let Defs = [PC], Uses = [P0, R31], isCodeGenOnly = 1, isPredicated = 1, isPredicatedFalse = 1, isPredicatedNew = 1, isBranch = 1, isIndirectBranch = 1, hasSideEffects = 0 in
+def V4_SL2_jumpr31_fnew: SUBInst <
+  (outs ),
+  (ins ),
+  "if (!p0.new) jumpr:nt r31"> {
+    let Inst{12-6} = 0b1111111;
+    let Inst{2-0} = 0b111;
+  }
+
+// SA1_clrt: Clear if true.
+// SA1_clrt -> SA1_clrtnew
+let Uses = [P0], isCodeGenOnly = 1, isPredicated = 1, hasSideEffects = 0, hasNewValue = 1, opNewValue = 0 in
+def V4_SA1_clrt: SUBInst <
+  (outs IntRegs:$Rd),
+  (ins ),
+  "if (p0) $Rd = #0"> {
+    bits<4> Rd;
+
+    let Inst{12-9} = 0b1101;
+    let Inst{6-4} = 0b110;
+    let Inst{3-0} = Rd;
+  }
+
+// SL2_return: Deallocate stack frame and return.
+let Defs = [PC, R31, R29, R30], Uses = [R30], isCodeGenOnly = 1, mayLoad = 1, accessSize = DoubleWordAccess, isBranch = 1, isIndirectBranch = 1 in
+def V4_SL2_return: SUBInst <
+  (outs ),
+  (ins ),
+  "dealloc_return"> {
+    let Inst{12-6} = 0b1111101;
+    let Inst{2} = 0b0;
+  }
+
+// SA1_dec: Dec.
+let isCodeGenOnly = 1, hasSideEffects = 0, hasNewValue = 1, opNewValue = 0 in
+def V4_SA1_dec: SUBInst <
+  (outs IntRegs:$Rd),
+  (ins IntRegs:$Rs),
+  "$Rd = add($Rs,#-1)"> {
+    bits<4> Rd;
+    bits<4> Rs;
+
+    let Inst{12-8} = 0b10011;
+    let Inst{3-0} = Rd;
+    let Inst{7-4} = Rs;
+  }
+
+// SA1_seti: Set immed.
+let isCodeGenOnly = 1, hasSideEffects = 0, hasNewValue = 1, opNewValue = 0, isExtendable = 1, isExtentSigned = 0, opExtentBits = 6, opExtendable = 1 in
+def V4_SA1_seti: SUBInst <
+  (outs IntRegs:$Rd),
+  (ins u6Ext:$u6),
+  "$Rd = #$u6"> {
+    bits<4> Rd;
+    bits<6> u6;
+
+    let Inst{12-10} = 0b010;
+    let Inst{3-0} = Rd;
+    let Inst{9-4} = u6;
+  }
+
+// SL2_jumpr31_t: Indirect conditional jump if true.
+// SL2_jumpr31_t -> SL2_jumpr31_tnew
+let Defs = [PC], Uses = [P0, R31], isCodeGenOnly = 1, isPredicated = 1, isBranch = 1, isIndirectBranch = 1, hasSideEffects = 0 in
+def V4_SL2_jumpr31_t: SUBInst <
+  (outs ),
+  (ins ),
+  "if (p0) jumpr r31"> {
+    let Inst{12-6} = 0b1111111;
+    let Inst{2-0} = 0b100;
+  }
+
+// SA1_clrfnew: Clear if false.
+let Uses = [P0], isCodeGenOnly = 1, isPredicated = 1, isPredicatedFalse = 1, isPredicatedNew = 1, hasSideEffects = 0, hasNewValue = 1, opNewValue = 0 in
+def V4_SA1_clrfnew: SUBInst <
+  (outs IntRegs:$Rd),
+  (ins ),
+  "if (!p0.new) $Rd = #0"> {
+    bits<4> Rd;
+
+    let Inst{12-9} = 0b1101;
+    let Inst{6-4} = 0b101;
+    let Inst{3-0} = Rd;
+  }
+
+// SS1_storew_io: Store word.
+let isCodeGenOnly = 1, mayStore = 1, accessSize = WordAccess in
+def V4_SS1_storew_io: SUBInst <
+  (outs ),
+  (ins IntRegs:$Rs, u4_2Imm:$u4_2, IntRegs:$Rt),
+  "memw($Rs + #$u4_2) = $Rt"> {
+    bits<4> Rs;
+    bits<6> u4_2;
+    bits<4> Rt;
+
+    let Inst{12} = 0b0;
+    let Inst{7-4} = Rs;
+    let Inst{11-8} = u4_2{5-2};
+    let Inst{3-0} = Rt;
+  }
+
+// SA1_zxtb: Zxtb.
+let isCodeGenOnly = 1, hasSideEffects = 0, hasNewValue = 1, opNewValue = 0 in
+def V4_SA1_zxtb: SUBInst <
+  (outs IntRegs:$Rd),
+  (ins IntRegs:$Rs),
+  "$Rd = and($Rs, #255)"> {
+    bits<4> Rd;
+    bits<4> Rs;
+
+    let Inst{12-8} = 0b10111;
+    let Inst{3-0} = Rd;
+    let Inst{7-4} = Rs;
+  }
+
+// SA1_addsp: Add.
+let Uses = [R29], isCodeGenOnly = 1, hasSideEffects = 0, hasNewValue = 1, opNewValue = 0 in
+def V4_SA1_addsp: SUBInst <
+  (outs IntRegs:$Rd),
+  (ins u6_2Imm:$u6_2),
+  "$Rd = add(r29, #$u6_2)"> {
+    bits<4> Rd;
+    bits<8> u6_2;
+
+    let Inst{12-10} = 0b011;
+    let Inst{3-0} = Rd;
+    let Inst{9-4} = u6_2{7-2};
+  }
+
+// SL2_loadri_sp: Load word.
+let Uses = [R29], isCodeGenOnly = 1, mayLoad = 1, accessSize = WordAccess, hasNewValue = 1, opNewValue = 0 in
+def V4_SL2_loadri_sp: SUBInst <
+  (outs IntRegs:$Rd),
+  (ins u5_2Imm:$u5_2),
+  "$Rd = memw(r29 + #$u5_2)"> {
+    bits<4> Rd;
+    bits<7> u5_2;
+
+    let Inst{12-9} = 0b1110;
+    let Inst{3-0} = Rd;
+    let Inst{8-4} = u5_2{6-2};
+  }
+
+// SS1_storeb_io: Store byte.
+let isCodeGenOnly = 1, mayStore = 1, accessSize = ByteAccess in
+def V4_SS1_storeb_io: SUBInst <
+  (outs ),
+  (ins IntRegs:$Rs, u4_0Imm:$u4_0, IntRegs:$Rt),
+  "memb($Rs + #$u4_0) = $Rt"> {
+    bits<4> Rs;
+    bits<4> u4_0;
+    bits<4> Rt;
+
+    let Inst{12} = 0b1;
+    let Inst{7-4} = Rs;
+    let Inst{11-8} = u4_0;
+    let Inst{3-0} = Rt;
+  }
+
+// SL2_return_tnew: Deallocate stack frame and return.
+let Defs = [PC, R31, R29, R30], Uses = [R30, P0], isCodeGenOnly = 1, isPredicated = 1, isPredicatedNew = 1, mayLoad = 1, accessSize = DoubleWordAccess, isBranch = 1, isIndirectBranch = 1 in
+def V4_SL2_return_tnew: SUBInst <
+  (outs ),
+  (ins ),
+  "if (p0.new) dealloc_return:nt"> {
+    let Inst{12-6} = 0b1111101;
+    let Inst{2-0} = 0b110;
+  }
+
+// SL2_return_fnew: Deallocate stack frame and return.
+let Defs = [PC, R31, R29, R30], Uses = [R30, P0], isCodeGenOnly = 1, isPredicated = 1, isPredicatedFalse = 1, isPredicatedNew = 1, mayLoad = 1, accessSize = DoubleWordAccess, isBranch = 1, isIndirectBranch = 1 in
+def V4_SL2_return_fnew: SUBInst <
+  (outs ),
+  (ins ),
+  "if (!p0.new) dealloc_return:nt"> {
+    let Inst{12-6} = 0b1111101;
+    let Inst{2-0} = 0b111;
+  }
+
+// SA1_zxth: Zxth.
+let isCodeGenOnly = 1, hasSideEffects = 0, hasNewValue = 1, opNewValue = 0 in
+def V4_SA1_zxth: SUBInst <
+  (outs IntRegs:$Rd),
+  (ins IntRegs:$Rs),
+  "$Rd = zxth($Rs)"> {
+    bits<4> Rd;
+    bits<4> Rs;
+
+    let Inst{12-8} = 0b10110;
+    let Inst{3-0} = Rd;
+    let Inst{7-4} = Rs;
+  }
+
diff --git a/lib/Target/Hexagon/HexagonMCInstLower.cpp b/lib/Target/Hexagon/HexagonMCInstLower.cpp
index 9c9f3afdd81..75189b696ea 100644
--- a/lib/Target/Hexagon/HexagonMCInstLower.cpp
+++ b/lib/Target/Hexagon/HexagonMCInstLower.cpp
@@ -15,9 +15,12 @@
 #include "Hexagon.h"
 #include "HexagonAsmPrinter.h"
 #include "HexagonMachineFunctionInfo.h"
+#include "MCTargetDesc/HexagonMCInstrInfo.h"
+
 #include "llvm/CodeGen/MachineBasicBlock.h"
 #include "llvm/IR/Constants.h"
 #include "llvm/IR/Mangler.h"
+#include "llvm/MC/MCContext.h"
 #include "llvm/MC/MCExpr.h"
 #include "llvm/MC/MCInst.h"
 
@@ -28,19 +31,30 @@ static MCOperand GetSymbolRef(const MachineOperand& MO, const MCSymbol* Symbol,
   MCContext &MC = Printer.OutContext;
   const MCExpr *ME;
 
-  ME = MCSymbolRefExpr::Create(Symbol, MCSymbolRefExpr::VK_None, MC);
+  ME = MCSymbolRefExpr::create(Symbol, MCSymbolRefExpr::VK_None, MC);
 
   if (!MO.isJTI() && MO.getOffset())
-    ME = MCBinaryExpr::CreateAdd(ME, MCConstantExpr::Create(MO.getOffset(), MC),
+    ME = MCBinaryExpr::createAdd(ME, MCConstantExpr::create(MO.getOffset(), MC),
                                  MC);
 
-  return (MCOperand::CreateExpr(ME));
+  return (MCOperand::createExpr(ME));
 }
 
 // Create an MCInst from a MachineInstr
-void llvm::HexagonLowerToMC(MachineInstr const* MI, MCInst& MCI,
+void llvm::HexagonLowerToMC(MachineInstr const* MI, MCInst& MCB,
                             HexagonAsmPrinter& AP) {
-  MCI.setOpcode(MI->getOpcode());
+  if(MI->getOpcode() == Hexagon::ENDLOOP0){
+    HexagonMCInstrInfo::setInnerLoop(MCB);
+    return;
+  }
+  if(MI->getOpcode() == Hexagon::ENDLOOP1){
+    HexagonMCInstrInfo::setOuterLoop(MCB);
+    return;
+  }
+  MCInst* MCI = new (AP.OutContext) MCInst;
+  MCI->setOpcode(MI->getOpcode());
+  assert(MCI->getOpcode() == static_cast<unsigned>(MI->getOpcode()) &&
+         "MCI opcode should have been set on construction");
 
   for (unsigned i = 0, e = MI->getNumOperands(); i < e; i++) {
     const MachineOperand &MO = MI->getOperand(i);
@@ -53,21 +67,21 @@ void llvm::HexagonLowerToMC(MachineInstr const* MI, MCInst& MCI,
     case MachineOperand::MO_Register:
       // Ignore all implicit register operands.
       if (MO.isImplicit()) continue;
-      MCO = MCOperand::CreateReg(MO.getReg());
+      MCO = MCOperand::createReg(MO.getReg());
       break;
     case MachineOperand::MO_FPImmediate: {
       APFloat Val = MO.getFPImm()->getValueAPF();
       // FP immediates are used only when setting GPRs, so they may be dealt
       // with like regular immediates from this point on.
-      MCO = MCOperand::CreateImm(*Val.bitcastToAPInt().getRawData());
+      MCO = MCOperand::createImm(*Val.bitcastToAPInt().getRawData());
       break;
     }
     case MachineOperand::MO_Immediate:
-      MCO = MCOperand::CreateImm(MO.getImm());
+      MCO = MCOperand::createImm(MO.getImm());
       break;
     case MachineOperand::MO_MachineBasicBlock:
-      MCO = MCOperand::CreateExpr
-              (MCSymbolRefExpr::Create(MO.getMBB()->getSymbol(),
+      MCO = MCOperand::createExpr
+              (MCSymbolRefExpr::create(MO.getMBB()->getSymbol(),
                AP.OutContext));
       break;
     case MachineOperand::MO_GlobalAddress:
@@ -88,6 +102,7 @@ void llvm::HexagonLowerToMC(MachineInstr const* MI, MCInst& MCI,
       break;
     }
 
-    MCI.addOperand(MCO);
+    MCI->addOperand(MCO);
   }
+  MCB.addOperand(MCOperand::createInst(MCI));
 }
diff --git a/lib/Target/Hexagon/HexagonMachineFunctionInfo.h b/lib/Target/Hexagon/HexagonMachineFunctionInfo.h
index cb18df6ed19..76723586c66 100644
--- a/lib/Target/Hexagon/HexagonMachineFunctionInfo.h
+++ b/lib/Target/Hexagon/HexagonMachineFunctionInfo.h
@@ -27,6 +27,7 @@ class HexagonMachineFunctionInfo : public MachineFunctionInfo {
   // returning the value of the returned struct in a register. This field
   // holds the virtual register into which the sret argument is passed.
   unsigned SRetReturnReg;
+  unsigned StackAlignBaseReg;
   std::vector<MachineInstr*> AllocaAdjustInsts;
   int VarArgsFrameIndex;
   bool HasClobberLR;
@@ -35,10 +36,11 @@ class HexagonMachineFunctionInfo : public MachineFunctionInfo {
   virtual void anchor();
 
 public:
-  HexagonMachineFunctionInfo() : SRetReturnReg(0), HasClobberLR(0),
-    HasEHReturn(false) {}
+  HexagonMachineFunctionInfo() : SRetReturnReg(0), StackAlignBaseReg(0),
+    HasClobberLR(0), HasEHReturn(false) {}
 
   HexagonMachineFunctionInfo(MachineFunction &MF) : SRetReturnReg(0),
+                                                    StackAlignBaseReg(0),
                                                     HasClobberLR(0),
                                                     HasEHReturn(false) {}
 
@@ -74,6 +76,9 @@ class HexagonMachineFunctionInfo : public MachineFunctionInfo {
 
   bool hasEHReturn() const { return HasEHReturn; };
   void setHasEHReturn(bool H = true) { HasEHReturn = H; };
+
+  void setStackAlignBaseVReg(unsigned R) { StackAlignBaseReg = R; }
+  unsigned getStackAlignBaseVReg() const { return StackAlignBaseReg; }
 };
 } // End llvm namespace
 
diff --git a/lib/Target/Hexagon/HexagonNewValueJump.cpp b/lib/Target/Hexagon/HexagonNewValueJump.cpp
index 81af4db912c..707bfdbb6ab 100644
--- a/lib/Target/Hexagon/HexagonNewValueJump.cpp
+++ b/lib/Target/Hexagon/HexagonNewValueJump.cpp
@@ -60,6 +60,7 @@ static cl::opt<bool> DisableNewValueJumps("disable-nvjump", cl::Hidden,
     cl::desc("Disable New Value Jumps"));
 
 namespace llvm {
+  FunctionPass *createHexagonNewValueJump();
   void initializeHexagonNewValueJumpPass(PassRegistry&);
 }
 
diff --git a/lib/Target/Hexagon/HexagonOperands.td b/lib/Target/Hexagon/HexagonOperands.td
index 450f59432e3..2bece8f42f5 100644
--- a/lib/Target/Hexagon/HexagonOperands.td
+++ b/lib/Target/Hexagon/HexagonOperands.td
@@ -7,31 +7,24 @@
 //
 //===----------------------------------------------------------------------===//
 
+def s4_0ImmOperand : AsmOperandClass { let Name = "s4_0Imm"; }
+def s4_1ImmOperand : AsmOperandClass { let Name = "s4_1Imm"; }
+def s4_2ImmOperand : AsmOperandClass { let Name = "s4_2Imm"; }
+def s4_3ImmOperand : AsmOperandClass { let Name = "s4_3Imm"; }
+
 // Immediate operands.
 
 let PrintMethod = "printImmOperand" in {
-  // f32Ext type is used to identify constant extended floating point immediates.
-  def f32Ext : Operand<f32>;
   def s32Imm : Operand<i32>;
-  def s26_6Imm : Operand<i32>;
-  def s16Imm : Operand<i32>;
-  def s12Imm : Operand<i32>;
-  def s11Imm : Operand<i32>;
-  def s11_0Imm : Operand<i32>;
-  def s11_1Imm : Operand<i32>;
-  def s11_2Imm : Operand<i32>;
-  def s11_3Imm : Operand<i32>;
-  def s10Imm : Operand<i32>;
-  def s9Imm : Operand<i32>;
-  def m9Imm : Operand<i32>;
   def s8Imm : Operand<i32>;
   def s8Imm64 : Operand<i64>;
   def s6Imm : Operand<i32>;
+  def s6_3Imm : Operand<i32>;
   def s4Imm : Operand<i32>;
-  def s4_0Imm : Operand<i32>;
-  def s4_1Imm : Operand<i32>;
-  def s4_2Imm : Operand<i32>;
-  def s4_3Imm : Operand<i32>;
+  def s4_0Imm : Operand<i32> { let DecoderMethod = "s4_0ImmDecoder"; }
+  def s4_1Imm : Operand<i32> { let DecoderMethod = "s4_1ImmDecoder"; }
+  def s4_2Imm : Operand<i32> { let DecoderMethod = "s4_2ImmDecoder"; }
+  def s4_3Imm : Operand<i32> { let DecoderMethod = "s4_3ImmDecoder"; }
   def u64Imm : Operand<i64>;
   def u32Imm : Operand<i32>;
   def u26_6Imm : Operand<i32>;
@@ -51,8 +44,14 @@ let PrintMethod = "printImmOperand" in {
   def u6_2Imm : Operand<i32>;
   def u6_3Imm : Operand<i32>;
   def u5Imm : Operand<i32>;
+  def u5_2Imm : Operand<i32>;
+  def u5_3Imm : Operand<i32>;
   def u4Imm : Operand<i32>;
+  def u4_0Imm : Operand<i32>;
+  def u4_2Imm : Operand<i32>;
   def u3Imm : Operand<i32>;
+  def u3_0Imm : Operand<i32>;
+  def u3_1Imm : Operand<i32>;
   def u2Imm : Operand<i32>;
   def u1Imm : Operand<i32>;
   def n8Imm : Operand<i32>;
@@ -439,16 +438,18 @@ def SetClr3ImmPred : PatLeaf<(i32 imm), [{
 // Extendable immediate operands.
 
 let PrintMethod = "printExtOperand" in {
-  def s16Ext : Operand<i32>;
-  def s12Ext : Operand<i32>;
-  def s10Ext : Operand<i32>;
-  def s9Ext : Operand<i32>;
-  def s8Ext : Operand<i32>;
-  def s6Ext : Operand<i32>;
-  def s11_0Ext : Operand<i32>;
-  def s11_1Ext : Operand<i32>;
-  def s11_2Ext : Operand<i32>;
-  def s11_3Ext : Operand<i32>;
+  def f32Ext : Operand<f32>;
+  def s16Ext : Operand<i32> { let DecoderMethod = "s16ImmDecoder"; }
+  def s12Ext : Operand<i32> { let DecoderMethod = "s12ImmDecoder"; }
+  def s11_0Ext : Operand<i32> { let DecoderMethod = "s11_0ImmDecoder"; }
+  def s11_1Ext : Operand<i32> { let DecoderMethod = "s11_1ImmDecoder"; }
+  def s11_2Ext : Operand<i32> { let DecoderMethod = "s11_2ImmDecoder"; }
+  def s11_3Ext : Operand<i32> { let DecoderMethod = "s11_3ImmDecoder"; }
+  def s10Ext : Operand<i32> { let DecoderMethod = "s10ImmDecoder"; }
+  def s9Ext : Operand<i32> { let DecoderMethod = "s90ImmDecoder"; }
+  def s8Ext : Operand<i32> { let DecoderMethod = "s8ImmDecoder"; }
+  def s7Ext : Operand<i32>;
+  def s6Ext : Operand<i32> { let DecoderMethod = "s6_0ImmDecoder"; }
   def u6Ext : Operand<i32>;
   def u7Ext : Operand<i32>;
   def u8Ext : Operand<i32>;
@@ -460,6 +461,46 @@ let PrintMethod = "printExtOperand" in {
   def u6_3Ext : Operand<i32>;
 }
 
+def s10ExtPred  : PatLeaf<(i32 imm), [{
+  int64_t v = (int64_t)N->getSExtValue();
+  if (isInt<10>(v))
+    return true;
+
+  // Return true if extending this immediate is profitable and the value
+  // can fit in a 32-bit signed field.
+  return isConstExtProfitable(Node) && isInt<32>(v);
+}]>;
+
+def s8ExtPred  : PatLeaf<(i32 imm), [{
+  int64_t v = (int64_t)N->getSExtValue();
+  if (isInt<8>(v))
+    return true;
+
+  // Return true if extending this immediate is profitable and the value
+  // can fit in a 32-bit signed field.
+  return isConstExtProfitable(Node) && isInt<32>(v);
+}]>;
+
+def u8ExtPred  : PatLeaf<(i32 imm), [{
+  int64_t v = (int64_t)N->getSExtValue();
+  if (isUInt<8>(v))
+    return true;
+
+  // Return true if extending this immediate is profitable and the value
+  // can fit in a 32-bit unsigned field.
+  return isConstExtProfitable(Node) && isUInt<32>(v);
+}]>;
+
+def u9ExtPred  : PatLeaf<(i32 imm), [{
+  int64_t v = (int64_t)N->getSExtValue();
+  if (isUInt<9>(v))
+    return true;
+
+  // Return true if extending this immediate is profitable and the value
+  // can fit in a 32-bit unsigned field.
+  return isConstExtProfitable(Node) && isUInt<32>(v);
+}]>;
+
 
 // This complex pattern exists only to create a machine instruction operand
 // of type "frame index". There doesn't seem to be a way to do that directly
@@ -483,7 +524,9 @@ let PrintMethod = "printJumpTable" in
 def jumptablebase : Operand<i32>;
 
 def brtarget : Operand<OtherVT>;
-def brtargetExt : Operand<OtherVT>;
+def brtargetExt : Operand<OtherVT> {
+  let PrintMethod = "printExtBrtarget";
+}
 def calltarget : Operand<i32>;
 
 def bblabel : Operand<i32>;
diff --git a/lib/Target/Hexagon/HexagonPeephole.cpp b/lib/Target/Hexagon/HexagonPeephole.cpp
index 503bfdb6b3e..93dcbe233b2 100644
--- a/lib/Target/Hexagon/HexagonPeephole.cpp
+++ b/lib/Target/Hexagon/HexagonPeephole.cpp
@@ -75,6 +75,7 @@ static cl::opt<bool> DisableOptExtTo64("disable-hexagon-opt-ext-to-64",
     cl::desc("Disable Optimization of extensions to i64."));
 
 namespace llvm {
+  FunctionPass *createHexagonPeephole();
   void initializeHexagonPeepholePass(PassRegistry&);
 }
 
diff --git a/lib/Target/Hexagon/HexagonRegisterInfo.cpp b/lib/Target/Hexagon/HexagonRegisterInfo.cpp
index 86eaee8ddbc..a3548630ca3 100644
--- a/lib/Target/Hexagon/HexagonRegisterInfo.cpp
+++ b/lib/Target/Hexagon/HexagonRegisterInfo.cpp
@@ -30,7 +30,9 @@
 #include "llvm/IR/Type.h"
 #include "llvm/MC/MachineLocation.h"
 #include "llvm/Support/CommandLine.h"
+#include "llvm/Support/Debug.h"
 #include "llvm/Support/ErrorHandling.h"
+#include "llvm/Support/raw_ostream.h"
 #include "llvm/Target/TargetInstrInfo.h"
 #include "llvm/Target/TargetMachine.h"
 #include "llvm/Target/TargetOptions.h"
@@ -40,6 +42,37 @@ using namespace llvm;
 HexagonRegisterInfo::HexagonRegisterInfo()
     : HexagonGenRegisterInfo(Hexagon::R31) {}
 
+
+bool HexagonRegisterInfo::isEHReturnCalleeSaveReg(unsigned R) const {
+  return R == Hexagon::R0 || R == Hexagon::R1 || R == Hexagon::R2 ||
+         R == Hexagon::R3 || R == Hexagon::D0 || R == Hexagon::D1;
+}
+
+bool HexagonRegisterInfo::isCalleeSaveReg(unsigned Reg) const {
+  return Hexagon::R16 <= Reg && Reg <= Hexagon::R27;
+}
+
+
+const MCPhysReg *
+HexagonRegisterInfo::getCallerSavedRegs(const MachineFunction *MF) const {
+  static const MCPhysReg CallerSavedRegsV4[] = {
+    Hexagon::R0, Hexagon::R1, Hexagon::R2, Hexagon::R3, Hexagon::R4,
+    Hexagon::R5, Hexagon::R6, Hexagon::R7, Hexagon::R8, Hexagon::R9,
+    Hexagon::R10, Hexagon::R11, Hexagon::R12, Hexagon::R13, Hexagon::R14,
+    Hexagon::R15, 0
+  };
+
+  auto &HST = static_cast<const HexagonSubtarget&>(MF->getSubtarget());
+  switch (HST.getHexagonArchVersion()) {
+  case HexagonSubtarget::V4:
+  case HexagonSubtarget::V5:
+    return CallerSavedRegsV4;
+  }
+  llvm_unreachable(
+    "Callee saved registers requested for unknown archtecture version");
+}
+
+
 const MCPhysReg *
 HexagonRegisterInfo::getCalleeSavedRegs(const MachineFunction *MF) const {
   static const MCPhysReg CalleeSavedRegsV3[] = {
@@ -75,173 +108,147 @@ BitVector HexagonRegisterInfo::getReservedRegs(const MachineFunction &MF)
 }
 
 
-const TargetRegisterClass* const*
-HexagonRegisterInfo::getCalleeSavedRegClasses(const MachineFunction *MF) const {
-  static const TargetRegisterClass * const CalleeSavedRegClassesV3[] = {
-    &Hexagon::IntRegsRegClass,     &Hexagon::IntRegsRegClass,
-    &Hexagon::IntRegsRegClass,     &Hexagon::IntRegsRegClass,
-    &Hexagon::IntRegsRegClass,     &Hexagon::IntRegsRegClass,
-    &Hexagon::IntRegsRegClass,     &Hexagon::IntRegsRegClass,
-    &Hexagon::IntRegsRegClass,     &Hexagon::IntRegsRegClass,
-    &Hexagon::IntRegsRegClass,     &Hexagon::IntRegsRegClass,
-  };
-
-  switch (MF->getSubtarget<HexagonSubtarget>().getHexagonArchVersion()) {
-  case HexagonSubtarget::V4:
-  case HexagonSubtarget::V5:
-    return CalleeSavedRegClassesV3;
-  }
-  llvm_unreachable("Callee saved register classes requested for unknown "
-                   "architecture version");
-}
-
 void HexagonRegisterInfo::eliminateFrameIndex(MachineBasicBlock::iterator II,
-                                              int SPAdj, unsigned FIOperandNum,
+                                              int SPAdj, unsigned FIOp,
                                               RegScavenger *RS) const {
   //
   // Hexagon_TODO: Do we need to enforce this for Hexagon?
   assert(SPAdj == 0 && "Unexpected");
 
   MachineInstr &MI = *II;
-  int FrameIndex = MI.getOperand(FIOperandNum).getIndex();
 
-  // Addressable stack objects are accessed using neg. offsets from %fp.
-  MachineFunction &MF = *MI.getParent()->getParent();
-  const HexagonInstrInfo &TII =
-      *static_cast<const HexagonInstrInfo *>(MF.getSubtarget().getInstrInfo());
-  int Offset = MF.getFrameInfo()->getObjectOffset(FrameIndex);
+  MachineBasicBlock &MB = *MI.getParent();
+  MachineFunction &MF = *MB.getParent();
   MachineFrameInfo &MFI = *MF.getFrameInfo();
-
-  unsigned FrameReg = getFrameRegister(MF);
-  const TargetFrameLowering *TFI = MF.getSubtarget().getFrameLowering();
-  if (!TFI->hasFP(MF)) {
+  auto &HST = static_cast<const HexagonSubtarget&>(MF.getSubtarget());
+  auto &HII = *HST.getInstrInfo();
+  auto &HFI = *HST.getFrameLowering();
+
+  int FI = MI.getOperand(FIOp).getIndex();
+  int Offset = MFI.getObjectOffset(FI) + MI.getOperand(FIOp+1).getImm();
+  bool HasAlloca = MFI.hasVarSizedObjects();
+  bool HasAlign = needsStackRealignment(MF);
+
+  // XXX: Fixed objects cannot be accessed through SP if there are aligned
+  // objects in the local frame, or if there are dynamically allocated objects.
+  // In such cases, there has to be FP available.
+  if (!HFI.hasFP(MF)) {
+    assert(!HasAlloca && !HasAlign && "This function must have frame pointer");
     // We will not reserve space on the stack for the lr and fp registers.
-    Offset -= 2 * Hexagon_WordSize;
+    Offset -= 8;
   }
 
+  unsigned SP = getStackRegister(), FP = getFrameRegister();
+  unsigned AP = 0;
+  if (MachineInstr *AI = HFI.getAlignaInstr(MF))
+    AP = AI->getOperand(0).getReg();
   unsigned FrameSize = MFI.getStackSize();
-  if (MI.getOpcode() == Hexagon::TFR_FI)
-    MI.setDesc(TII.get(Hexagon::A2_addi));
-
-  if (!MFI.hasVarSizedObjects() &&
-      TII.isValidOffset(MI.getOpcode(), (FrameSize+Offset)) &&
-      !TII.isSpillPredRegOp(&MI)) {
-    // Replace frame index with a stack pointer reference.
-    MI.getOperand(FIOperandNum).ChangeToRegister(getStackRegister(), false,
-                                                 false, true);
-    MI.getOperand(FIOperandNum + 1).ChangeToImmediate(FrameSize+Offset);
+
+  // Special handling of dbg_value instructions and INLINEASM.
+  if (MI.isDebugValue() || MI.isInlineAsm()) {
+    MI.getOperand(FIOp).ChangeToRegister(SP, false /*isDef*/);
+    MI.getOperand(FIOp+1).ChangeToImmediate(Offset+FrameSize);
+    return;
+  }
+
+  bool UseFP = false, UseAP = false;  // Default: use SP.
+  if (MFI.isFixedObjectIndex(FI) || MFI.isObjectPreAllocated(FI)) {
+    UseFP = HasAlloca || HasAlign;
   } else {
-    // Replace frame index with a frame pointer reference.
-    if (!TII.isValidOffset(MI.getOpcode(), Offset)) {
-
-      // If the offset overflows, then correct it.
-      //
-      // For loads, we do not need a reserved register
-      // r0 = memw(r30 + #10000) to:
-      //
-      // r0 = add(r30, #10000)
-      // r0 = memw(r0)
-      if ( (MI.getOpcode() == Hexagon::L2_loadri_io)  ||
-           (MI.getOpcode() == Hexagon::L2_loadrd_io)   ||
-           (MI.getOpcode() == Hexagon::L2_loadrh_io) ||
-           (MI.getOpcode() == Hexagon::L2_loadruh_io) ||
-           (MI.getOpcode() == Hexagon::L2_loadrb_io) ||
-           (MI.getOpcode() == Hexagon::L2_loadrub_io)) {
-        unsigned dstReg = (MI.getOpcode() == Hexagon::L2_loadrd_io) ?
-          getSubReg(MI.getOperand(0).getReg(), Hexagon::subreg_loreg) :
-          MI.getOperand(0).getReg();
-
-        // Check if offset can fit in addi.
-        if (!TII.isValidOffset(Hexagon::A2_addi, Offset)) {
-          BuildMI(*MI.getParent(), II, MI.getDebugLoc(),
-                  TII.get(Hexagon::CONST32_Int_Real), dstReg).addImm(Offset);
-          BuildMI(*MI.getParent(), II, MI.getDebugLoc(),
-                  TII.get(Hexagon::A2_add),
-                  dstReg).addReg(FrameReg).addReg(dstReg);
-        } else {
-          BuildMI(*MI.getParent(), II, MI.getDebugLoc(),
-                  TII.get(Hexagon::A2_addi),
-                  dstReg).addReg(FrameReg).addImm(Offset);
-        }
-
-        MI.getOperand(FIOperandNum).ChangeToRegister(dstReg, false, false,true);
-        MI.getOperand(FIOperandNum+1).ChangeToImmediate(0);
-      } else if ((MI.getOpcode() == Hexagon::S2_storeri_io) ||
-                 (MI.getOpcode() == Hexagon::S2_storerd_io) ||
-                 (MI.getOpcode() == Hexagon::S2_storerh_io) ||
-                 (MI.getOpcode() == Hexagon::S2_storerb_io)) {
-        // For stores, we need a reserved register. Change
-        // memw(r30 + #10000) = r0 to:
-        //
-        // rs = add(r30, #10000);
-        // memw(rs) = r0
-        unsigned resReg = HEXAGON_RESERVED_REG_1;
-
-        // Check if offset can fit in addi.
-        if (!TII.isValidOffset(Hexagon::A2_addi, Offset)) {
-          BuildMI(*MI.getParent(), II, MI.getDebugLoc(),
-                  TII.get(Hexagon::CONST32_Int_Real), resReg).addImm(Offset);
-          BuildMI(*MI.getParent(), II, MI.getDebugLoc(),
-                  TII.get(Hexagon::A2_add),
-                  resReg).addReg(FrameReg).addReg(resReg);
-        } else {
-          BuildMI(*MI.getParent(), II, MI.getDebugLoc(),
-                  TII.get(Hexagon::A2_addi),
-                  resReg).addReg(FrameReg).addImm(Offset);
-        }
-        MI.getOperand(FIOperandNum).ChangeToRegister(resReg, false, false,true);
-        MI.getOperand(FIOperandNum+1).ChangeToImmediate(0);
-      } else if (TII.isMemOp(&MI)) {
-        // use the constant extender if the instruction provides it
-        if (TII.isConstExtended(&MI)) {
-          MI.getOperand(FIOperandNum).ChangeToRegister(FrameReg, false);
-          MI.getOperand(FIOperandNum+1).ChangeToImmediate(Offset);
-          TII.immediateExtend(&MI);
-        } else {
-          llvm_unreachable("Need to implement for memops");
-        }
-      } else {
-        unsigned dstReg = MI.getOperand(0).getReg();
-        BuildMI(*MI.getParent(), II, MI.getDebugLoc(),
-                TII.get(Hexagon::CONST32_Int_Real), dstReg).addImm(Offset);
-        BuildMI(*MI.getParent(), II, MI.getDebugLoc(),
-                TII.get(Hexagon::A2_add),
-                dstReg).addReg(FrameReg).addReg(dstReg);
-        // Can we delete MI??? r2 = add (r2, #0).
-        MI.getOperand(FIOperandNum).ChangeToRegister(dstReg, false, false,true);
-        MI.getOperand(FIOperandNum+1).ChangeToImmediate(0);
-      }
-    } else {
-      // If the offset is small enough to fit in the immediate field, directly
-      // encode it.
-      MI.getOperand(FIOperandNum).ChangeToRegister(FrameReg, false);
-      MI.getOperand(FIOperandNum+1).ChangeToImmediate(Offset);
+    if (HasAlloca) {
+      if (HasAlign)
+        UseAP = true;
+      else
+        UseFP = true;
     }
   }
 
+  unsigned Opc = MI.getOpcode();
+  bool ValidSP = HII.isValidOffset(Opc, FrameSize+Offset);
+  bool ValidFP = HII.isValidOffset(Opc, Offset);
+
+  // Calculate the actual offset in the instruction.
+  int64_t RealOffset = Offset;
+  if (!UseFP && !UseAP)
+    RealOffset = FrameSize+Offset;
+
+  switch (Opc) {
+    case Hexagon::TFR_FIA:
+      MI.setDesc(HII.get(Hexagon::A2_addi));
+      MI.getOperand(FIOp).ChangeToImmediate(RealOffset);
+      MI.RemoveOperand(FIOp+1);
+      return;
+    case Hexagon::TFR_FI:
+      // Set up the instruction for updating below.
+      MI.setDesc(HII.get(Hexagon::A2_addi));
+      break;
+  }
+
+  unsigned BP = 0;
+  bool Valid = false;
+  if (UseFP) {
+    BP = FP;
+    Valid = ValidFP;
+  } else if (UseAP) {
+    BP = AP;
+    Valid = ValidFP;
+  } else {
+    BP = SP;
+    Valid = ValidSP;
+  }
+
+  if (Valid) {
+    MI.getOperand(FIOp).ChangeToRegister(BP, false);
+    MI.getOperand(FIOp+1).ChangeToImmediate(RealOffset);
+    return;
+  }
+
+#ifndef NDEBUG
+  const Function *F = MF.getFunction();
+  dbgs() << "In function ";
+  if (F) dbgs() << F->getName();
+  else   dbgs() << "<?>";
+  dbgs() << ", BB#" << MB.getNumber() << "\n" << MI;
+#endif
+  llvm_unreachable("Unhandled instruction");
 }
 
+
 unsigned HexagonRegisterInfo::getRARegister() const {
   return Hexagon::R31;
 }
 
+
 unsigned HexagonRegisterInfo::getFrameRegister(const MachineFunction
                                                &MF) const {
-  const TargetFrameLowering *TFI = MF.getSubtarget().getFrameLowering();
-  if (TFI->hasFP(MF)) {
+  const HexagonFrameLowering *TFI = getFrameLowering(MF);
+  if (TFI->hasFP(MF))
     return Hexagon::R30;
-  }
-
   return Hexagon::R29;
 }
 
+
 unsigned HexagonRegisterInfo::getFrameRegister() const {
   return Hexagon::R30;
 }
 
+
 unsigned HexagonRegisterInfo::getStackRegister() const {
   return Hexagon::R29;
 }
 
+
+bool
+HexagonRegisterInfo::useFPForScavengingIndex(const MachineFunction &MF) const {
+  const HexagonFrameLowering *TFI = getFrameLowering(MF);
+  return TFI->hasFP(MF);
+}
+
+
+unsigned HexagonRegisterInfo::getFirstCallerSavedNonParamReg() const {
+  return Hexagon::R6;
+}
+
+
 #define GET_REGINFO_TARGET_DESC
 #include "HexagonGenRegisterInfo.inc"
diff --git a/lib/Target/Hexagon/HexagonRegisterInfo.h b/lib/Target/Hexagon/HexagonRegisterInfo.h
index dc6dd2a156f..db7e0f27815 100644
--- a/lib/Target/Hexagon/HexagonRegisterInfo.h
+++ b/lib/Target/Hexagon/HexagonRegisterInfo.h
@@ -37,31 +37,35 @@
 #define HEXAGON_RESERVED_REG_2 Hexagon::R11
 
 namespace llvm {
-struct HexagonRegisterInfo : public HexagonGenRegisterInfo {
+class HexagonRegisterInfo : public HexagonGenRegisterInfo {
+public:
   HexagonRegisterInfo();
 
   /// Code Generation virtual methods...
-  const MCPhysReg *getCalleeSavedRegs(const MachineFunction *MF) const override;
+  const MCPhysReg *getCalleeSavedRegs(const MachineFunction *MF)
+        const override;
 
-  const TargetRegisterClass* const*
-  getCalleeSavedRegClasses(const MachineFunction *MF = nullptr) const;
 
   BitVector getReservedRegs(const MachineFunction &MF) const override;
 
-  void eliminateFrameIndex(MachineBasicBlock::iterator II,
-                           int SPAdj, unsigned FIOperandNum,
-                           RegScavenger *RS = nullptr) const override;
+  void eliminateFrameIndex(MachineBasicBlock::iterator II, int SPAdj,
+        unsigned FIOperandNum, RegScavenger *RS = nullptr) const override;
 
-  /// determineFrameLayout - Determine the size of the frame and maximum call
-  /// frame size.
-  void determineFrameLayout(MachineFunction &MF) const;
-
-  /// requiresRegisterScavenging - returns true since we may need scavenging for
-  /// a temporary register when generating hardware loop instructions.
+  /// Returns true since we may need scavenging for a temporary register
+  /// when generating hardware loop instructions.
   bool requiresRegisterScavenging(const MachineFunction &MF) const override {
     return true;
   }
 
+  /// Returns true. Spill code for predicate registers might need an extra
+  /// register.
+  bool requiresFrameIndexScavenging(const MachineFunction &MF) const override {
+    return true;
+  }
+
+  /// Returns true if the frame pointer is valid.
+  bool useFPForScavengingIndex(const MachineFunction &MF) const override;
+
   bool trackLivenessAfterRegAlloc(const MachineFunction &MF) const override {
     return true;
   }
@@ -71,6 +75,13 @@ struct HexagonRegisterInfo : public HexagonGenRegisterInfo {
   unsigned getFrameRegister(const MachineFunction &MF) const override;
   unsigned getFrameRegister() const;
   unsigned getStackRegister() const;
+
+  const MCPhysReg *getCallerSavedRegs(const MachineFunction *MF) const;
+
+  unsigned getFirstCallerSavedNonParamReg() const;
+
+  bool isEHReturnCalleeSaveReg(unsigned Reg) const;
+  bool isCalleeSaveReg(unsigned Reg) const;
 };
 
 } // end namespace llvm
diff --git a/lib/Target/Hexagon/HexagonRemoveSZExtArgs.cpp b/lib/Target/Hexagon/HexagonRemoveSZExtArgs.cpp
index 0c240750886..7069ad36e21 100644
--- a/lib/Target/Hexagon/HexagonRemoveSZExtArgs.cpp
+++ b/lib/Target/Hexagon/HexagonRemoveSZExtArgs.cpp
@@ -24,6 +24,7 @@
 using namespace llvm;
 
 namespace llvm {
+  FunctionPass *createHexagonRemoveExtendArgs(const HexagonTargetMachine &TM);
   void initializeHexagonRemoveExtendArgsPass(PassRegistry&);
 }
 
diff --git a/lib/Target/Hexagon/HexagonSelectionDAGInfo.cpp b/lib/Target/Hexagon/HexagonSelectionDAGInfo.cpp
index b5db997eb1b..276cc69eed0 100644
--- a/lib/Target/Hexagon/HexagonSelectionDAGInfo.cpp
+++ b/lib/Target/Hexagon/HexagonSelectionDAGInfo.cpp
@@ -18,12 +18,6 @@ using namespace llvm;
 
 bool llvm::flag_aligned_memcpy;
 
-HexagonSelectionDAGInfo::HexagonSelectionDAGInfo(const DataLayout &DL)
-    : TargetSelectionDAGInfo(&DL) {}
-
-HexagonSelectionDAGInfo::~HexagonSelectionDAGInfo() {
-}
-
 SDValue
 HexagonSelectionDAGInfo::
 EmitTargetCodeForMemcpy(SelectionDAG &DAG, SDLoc dl, SDValue Chain,
diff --git a/lib/Target/Hexagon/HexagonSelectionDAGInfo.h b/lib/Target/Hexagon/HexagonSelectionDAGInfo.h
index 8ac2e43f929..80ac5d7bd9e 100644
--- a/lib/Target/Hexagon/HexagonSelectionDAGInfo.h
+++ b/lib/Target/Hexagon/HexagonSelectionDAGInfo.h
@@ -20,8 +20,6 @@ namespace llvm {
 
 class HexagonSelectionDAGInfo : public TargetSelectionDAGInfo {
 public:
-  explicit HexagonSelectionDAGInfo(const DataLayout &DL);
-  ~HexagonSelectionDAGInfo();
 
   SDValue EmitTargetCodeForMemcpy(SelectionDAG &DAG, SDLoc dl,
                                   SDValue Chain,
diff --git a/lib/Target/Hexagon/HexagonSplitConst32AndConst64.cpp b/lib/Target/Hexagon/HexagonSplitConst32AndConst64.cpp
index 1a4c7ae6f2b..d3eb56f4ba0 100644
--- a/lib/Target/Hexagon/HexagonSplitConst32AndConst64.cpp
+++ b/lib/Target/Hexagon/HexagonSplitConst32AndConst64.cpp
@@ -45,6 +45,11 @@ using namespace llvm;
 
 #define DEBUG_TYPE "xfer"
 
+namespace llvm {
+  FunctionPass *createHexagonSplitConst32AndConst64();
+  void initializeHexagonSplitConst32AndConst64Pass(PassRegistry&);
+}
+
 namespace {
 
 class HexagonSplitConst32AndConst64 : public MachineFunctionPass {
@@ -83,19 +88,8 @@ bool HexagonSplitConst32AndConst64::runOnMachineFunction(MachineFunction &Fn) {
     while (MII != MIE) {
       MachineInstr *MI = MII;
       int Opc = MI->getOpcode();
-      if (Opc == Hexagon::CONST32_set_jt) {
-        int DestReg = MI->getOperand(0).getReg();
-        MachineOperand &Symbol = MI->getOperand (1);
-        BuildMI (*MBB, MII, MI->getDebugLoc(),
-                 TII->get(Hexagon::A2_tfrsi), DestReg).addOperand(Symbol);
-
-        // MBB->erase returns the iterator to the next instruction, which is the
-        // one we want to process next
-        MII = MBB->erase (MI);
-        continue;
-      }
-      else if (Opc == Hexagon::CONST32_Int_Real &&
-               MI->getOperand(1).isBlockAddress()) {
+      if (Opc == Hexagon::CONST32_Int_Real &&
+          MI->getOperand(1).isBlockAddress()) {
         int DestReg = MI->getOperand(0).getReg();
         MachineOperand &Symbol = MI->getOperand (1);
 
diff --git a/lib/Target/Hexagon/HexagonSubtarget.cpp b/lib/Target/Hexagon/HexagonSubtarget.cpp
index d61cc5418a4..cd482b3e3af 100644
--- a/lib/Target/Hexagon/HexagonSubtarget.cpp
+++ b/lib/Target/Hexagon/HexagonSubtarget.cpp
@@ -70,11 +70,11 @@ HexagonSubtarget::initializeSubtargetDependencies(StringRef CPU, StringRef FS) {
   return *this;
 }
 
-HexagonSubtarget::HexagonSubtarget(StringRef TT, StringRef CPU, StringRef FS,
-                                   const TargetMachine &TM)
+HexagonSubtarget::HexagonSubtarget(const Triple &TT, StringRef CPU,
+                                   StringRef FS, const TargetMachine &TM)
     : HexagonGenSubtargetInfo(TT, CPU, FS), CPUString(CPU),
       InstrInfo(initializeSubtargetDependencies(CPU, FS)), TLInfo(TM, *this),
-      TSInfo(*TM.getDataLayout()), FrameLowering() {
+      FrameLowering() {
 
   // Initialize scheduling itinerary for the specified CPU.
   InstrItins = getInstrItineraryForCPU(CPUString);
diff --git a/lib/Target/Hexagon/HexagonSubtarget.h b/lib/Target/Hexagon/HexagonSubtarget.h
index 780567bcd36..34cdad786f8 100644
--- a/lib/Target/Hexagon/HexagonSubtarget.h
+++ b/lib/Target/Hexagon/HexagonSubtarget.h
@@ -52,7 +52,7 @@ class HexagonSubtarget : public HexagonGenSubtargetInfo {
   InstrItineraryData InstrItins;
 
 public:
-  HexagonSubtarget(StringRef TT, StringRef CPU, StringRef FS,
+  HexagonSubtarget(const Triple &TT, StringRef CPU, StringRef FS,
                    const TargetMachine &TM);
 
   /// getInstrItins - Return the instruction itineraries based on subtarget
diff --git a/lib/Target/Hexagon/HexagonTargetMachine.cpp b/lib/Target/Hexagon/HexagonTargetMachine.cpp
index 06798665cb0..ba589a2ac27 100644
--- a/lib/Target/Hexagon/HexagonTargetMachine.cpp
+++ b/lib/Target/Hexagon/HexagonTargetMachine.cpp
@@ -37,6 +37,21 @@ static cl::opt<bool> EnableExpandCondsets("hexagon-expand-condsets",
   cl::init(true), cl::Hidden, cl::ZeroOrMore,
   cl::desc("Early expansion of MUX"));
 
+static cl::opt<bool> EnableGenInsert("hexagon-insert", cl::init(true),
+  cl::Hidden, cl::desc("Generate \"insert\" instructions"));
+
+static cl::opt<bool> EnableCommGEP("hexagon-commgep", cl::init(true),
+  cl::Hidden, cl::ZeroOrMore, cl::desc("Enable commoning of GEP instructions"));
+
+static cl::opt<bool> EnableGenExtract("hexagon-extract", cl::init(true),
+  cl::Hidden, cl::desc("Generate \"extract\" instructions"));
+
+static cl::opt<bool> EnableGenMux("hexagon-mux", cl::init(true), cl::Hidden,
+  cl::desc("Enable converting conditional transfers into MUX instructions"));
+
+static cl::opt<bool> EnableGenPred("hexagon-gen-pred", cl::init(true),
+  cl::Hidden, cl::desc("Enable conversion of arithmetic operations to "
+  "predicate instructions"));
 
 /// HexagonTargetMachineModule - Note that this is used on hosts that
 /// cannot link in a library unless there are references into the
@@ -60,15 +75,32 @@ SchedCustomRegistry("hexagon", "Run Hexagon's custom scheduler",
                     createVLIWMachineSched);
 
 namespace llvm {
+  FunctionPass *createHexagonCFGOptimizer();
+  FunctionPass *createHexagonCommonGEP();
+  FunctionPass *createHexagonCopyToCombine();
   FunctionPass *createHexagonExpandCondsets();
-}
+  FunctionPass *createHexagonExpandPredSpillCode();
+  FunctionPass *createHexagonFixupHwLoops();
+  FunctionPass *createHexagonGenExtract();
+  FunctionPass *createHexagonGenInsert();
+  FunctionPass *createHexagonGenMux();
+  FunctionPass *createHexagonGenPredicate();
+  FunctionPass *createHexagonHardwareLoops();
+  FunctionPass *createHexagonISelDag(HexagonTargetMachine &TM,
+                                     CodeGenOpt::Level OptLevel);
+  FunctionPass *createHexagonNewValueJump();
+  FunctionPass *createHexagonPacketizer();
+  FunctionPass *createHexagonPeephole();
+  FunctionPass *createHexagonRemoveExtendArgs(const HexagonTargetMachine &TM);
+  FunctionPass *createHexagonSplitConst32AndConst64();
+} // end namespace llvm;
 
 /// HexagonTargetMachine ctor - Create an ILP32 architecture model.
 ///
 
 /// Hexagon_TODO: Do I need an aggregate alignment?
 ///
-HexagonTargetMachine::HexagonTargetMachine(const Target &T, StringRef TT,
+HexagonTargetMachine::HexagonTargetMachine(const Target &T, const Triple &TT,
                                            StringRef CPU, StringRef FS,
                                            const TargetOptions &Options,
                                            Reloc::Model RM, CodeModel::Model CM,
@@ -106,6 +138,7 @@ class HexagonPassConfig : public TargetPassConfig {
     return createVLIWMachineSched(C);
   }
 
+  void addIRPasses() override;
   bool addInstSelector() override;
   void addPreRegAlloc() override;
   void addPostRegAlloc() override;
@@ -118,6 +151,20 @@ TargetPassConfig *HexagonTargetMachine::createPassConfig(PassManagerBase &PM) {
   return new HexagonPassConfig(this, PM);
 }
 
+void HexagonPassConfig::addIRPasses() {
+  TargetPassConfig::addIRPasses();
+  bool NoOpt = (getOptLevel() == CodeGenOpt::None);
+
+  addPass(createAtomicExpandPass(TM));
+  if (!NoOpt) {
+    if (EnableCommGEP)
+      addPass(createHexagonCommonGEP());
+    // Replace certain combinations of shifts and ands with extracts.
+    if (EnableGenExtract)
+      addPass(createHexagonGenExtract());
+  }
+}
+
 bool HexagonPassConfig::addInstSelector() {
   HexagonTargetMachine &TM = getHexagonTargetMachine();
   bool NoOpt = (getOptLevel() == CodeGenOpt::None);
@@ -128,8 +175,13 @@ bool HexagonPassConfig::addInstSelector() {
   addPass(createHexagonISelDag(TM, getOptLevel()));
 
   if (!NoOpt) {
+    // Create logical operations on predicate registers.
+    if (EnableGenPred)
+      addPass(createHexagonGenPredicate(), false);
     addPass(createHexagonPeephole());
     printAndVerify("After hexagon peephole pass");
+    if (EnableGenInsert)
+      addPass(createHexagonGenInsert(), false);
   }
 
   return false;
@@ -167,6 +219,10 @@ void HexagonPassConfig::addPreEmitPass() {
   if (!NoOpt) {
     if (!DisableHardwareLoops)
       addPass(createHexagonFixupHwLoops(), false);
+    // Generate MUX from pairs of conditional transfers.
+    if (EnableGenMux)
+      addPass(createHexagonGenMux(), false);
+
     addPass(createHexagonPacketizer(), false);
   }
 }
diff --git a/lib/Target/Hexagon/HexagonTargetMachine.h b/lib/Target/Hexagon/HexagonTargetMachine.h
index 5774f7e195b..115eadb98c3 100644
--- a/lib/Target/Hexagon/HexagonTargetMachine.h
+++ b/lib/Target/Hexagon/HexagonTargetMachine.h
@@ -27,7 +27,7 @@ class HexagonTargetMachine : public LLVMTargetMachine {
   HexagonSubtarget Subtarget;
 
 public:
-  HexagonTargetMachine(const Target &T, StringRef TT,StringRef CPU,
+  HexagonTargetMachine(const Target &T, const Triple &TT, StringRef CPU,
                        StringRef FS, const TargetOptions &Options,
                        Reloc::Model RM, CodeModel::Model CM,
                        CodeGenOpt::Level OL);
diff --git a/lib/Target/Hexagon/HexagonTargetObjectFile.cpp b/lib/Target/Hexagon/HexagonTargetObjectFile.cpp
index d8660d3138e..f6983aa1251 100644
--- a/lib/Target/Hexagon/HexagonTargetObjectFile.cpp
+++ b/lib/Target/Hexagon/HexagonTargetObjectFile.cpp
@@ -75,13 +75,14 @@ IsGlobalInSmallSection(const GlobalValue *GV, const TargetMachine &TM,
 
   if (Kind.isBSS() || Kind.isDataNoRel() || Kind.isCommon()) {
     Type *Ty = GV->getType()->getElementType();
-    return IsInSmallSection(TM.getDataLayout()->getTypeAllocSize(Ty));
+    return IsInSmallSection(
+        GV->getParent()->getDataLayout().getTypeAllocSize(Ty));
   }
 
   return false;
 }
 
-const MCSection *
+MCSection *
 HexagonTargetObjectFile::SelectSectionForGlobal(const GlobalValue *GV,
                                                 SectionKind Kind, Mangler &Mang,
                                                 const TargetMachine &TM) const {
diff --git a/lib/Target/Hexagon/HexagonTargetObjectFile.h b/lib/Target/Hexagon/HexagonTargetObjectFile.h
index c9742042724..da0eeeb3fd2 100644
--- a/lib/Target/Hexagon/HexagonTargetObjectFile.h
+++ b/lib/Target/Hexagon/HexagonTargetObjectFile.h
@@ -16,8 +16,9 @@
 namespace llvm {
 
   class HexagonTargetObjectFile : public TargetLoweringObjectFileELF {
-    const MCSectionELF *SmallDataSection;
-    const MCSectionELF *SmallBSSSection;
+    MCSectionELF *SmallDataSection;
+    MCSectionELF *SmallBSSSection;
+
   public:
     void Initialize(MCContext &Ctx, const TargetMachine &TM) override;
 
@@ -30,9 +31,9 @@ namespace llvm {
                                 const TargetMachine &TM) const;
 
     bool IsSmallDataEnabled () const;
-    const MCSection *SelectSectionForGlobal(const GlobalValue *GV,
-                                        SectionKind Kind, Mangler &Mang,
-                                        const TargetMachine &TM) const override;
+    MCSection *SelectSectionForGlobal(const GlobalValue *GV, SectionKind Kind,
+                                      Mangler &Mang,
+                                      const TargetMachine &TM) const override;
   };
 
 } // namespace llvm
diff --git a/lib/Target/Hexagon/HexagonTargetStreamer.h b/lib/Target/Hexagon/HexagonTargetStreamer.h
new file mode 100644
index 00000000000..e19c404450e
--- /dev/null
+++ b/lib/Target/Hexagon/HexagonTargetStreamer.h
@@ -0,0 +1,31 @@
+//===-- HexagonTargetStreamer.h - Hexagon Target Streamer ------*- C++ -*--===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef HEXAGONTARGETSTREAMER_H
+#define HEXAGONTARGETSTREAMER_H
+
+#include "llvm/MC/MCStreamer.h"
+
+namespace llvm {
+class HexagonTargetStreamer : public MCTargetStreamer {
+public:
+  HexagonTargetStreamer(MCStreamer &S) : MCTargetStreamer(S) {}
+  virtual void EmitCodeAlignment(unsigned ByteAlignment,
+                                 unsigned MaxBytesToEmit = 0){};
+  virtual void emitFAlign(unsigned Size, unsigned MaxBytesToEmit){};
+  virtual void EmitCommonSymbolSorted(MCSymbol *Symbol, uint64_t Size,
+                                      unsigned ByteAlignment,
+                                      unsigned AccessGranularity){};
+  virtual void EmitLocalCommonSymbolSorted(MCSymbol *Symbol, uint64_t Size,
+                                           unsigned ByteAlign,
+                                           unsigned AccessGranularity){};
+};
+}
+
+#endif
diff --git a/lib/Target/Hexagon/HexagonVLIWPacketizer.cpp b/lib/Target/Hexagon/HexagonVLIWPacketizer.cpp
index 4ca628ef812..b91a3f6f8c6 100644
--- a/lib/Target/Hexagon/HexagonVLIWPacketizer.cpp
+++ b/lib/Target/Hexagon/HexagonVLIWPacketizer.cpp
@@ -57,6 +57,7 @@ static cl::opt<bool> PacketizeVolatiles("hexagon-packetize-volatiles",
       cl::desc("Allow non-solo packetization of volatile memory references"));
 
 namespace llvm {
+  FunctionPass *createHexagonPacketizer();
   void initializeHexagonPacketizerPass(PassRegistry&);
 }
 
@@ -271,9 +272,8 @@ static bool IsIndirectCall(MachineInstr* MI) {
 // reservation fail.
 void HexagonPacketizerList::reserveResourcesForConstExt(MachineInstr* MI) {
   const HexagonInstrInfo *QII = (const HexagonInstrInfo *) TII;
-  MachineFunction *MF = MI->getParent()->getParent();
-  MachineInstr *PseudoMI = MF->CreateMachineInstr(QII->get(Hexagon::A4_ext),
-                                                  MI->getDebugLoc());
+  MachineInstr *PseudoMI = MF.CreateMachineInstr(QII->get(Hexagon::A4_ext),
+                                                 MI->getDebugLoc());
 
   if (ResourceTracker->canReserveResources(PseudoMI)) {
     ResourceTracker->reserveResources(PseudoMI);
@@ -289,11 +289,10 @@ bool HexagonPacketizerList::canReserveResourcesForConstExt(MachineInstr *MI) {
   const HexagonInstrInfo *QII = (const HexagonInstrInfo *) TII;
   assert((QII->isExtended(MI) || QII->isConstExtended(MI)) &&
          "Should only be called for constant extended instructions");
-  MachineFunction *MF = MI->getParent()->getParent();
-  MachineInstr *PseudoMI = MF->CreateMachineInstr(QII->get(Hexagon::A4_ext),
-                                                  MI->getDebugLoc());
+  MachineInstr *PseudoMI = MF.CreateMachineInstr(QII->get(Hexagon::A4_ext),
+                                                 MI->getDebugLoc());
   bool CanReserve = ResourceTracker->canReserveResources(PseudoMI);
-  MF->DeleteMachineInstr(PseudoMI);
+  MF.DeleteMachineInstr(PseudoMI);
   return CanReserve;
 }
 
@@ -301,9 +300,8 @@ bool HexagonPacketizerList::canReserveResourcesForConstExt(MachineInstr *MI) {
 // true, otherwise, return false.
 bool HexagonPacketizerList::tryAllocateResourcesForConstExt(MachineInstr* MI) {
   const HexagonInstrInfo *QII = (const HexagonInstrInfo *) TII;
-  MachineFunction *MF = MI->getParent()->getParent();
-  MachineInstr *PseudoMI = MF->CreateMachineInstr(QII->get(Hexagon::A4_ext),
-                                                  MI->getDebugLoc());
+  MachineInstr *PseudoMI = MF.CreateMachineInstr(QII->get(Hexagon::A4_ext),
+                                                 MI->getDebugLoc());
 
   if (ResourceTracker->canReserveResources(PseudoMI)) {
     ResourceTracker->reserveResources(PseudoMI);
@@ -950,6 +948,9 @@ bool HexagonPacketizerList::ignorePseudoInstruction(MachineInstr *MI,
   if (MI->isDebugValue())
     return true;
 
+  if (MI->isCFIInstruction())
+    return false;
+
   // We must print out inline assembly
   if (MI->isInlineAsm())
     return false;
@@ -967,11 +968,10 @@ bool HexagonPacketizerList::ignorePseudoInstruction(MachineInstr *MI,
 // isSoloInstruction: - Returns true for instructions that must be
 // scheduled in their own packet.
 bool HexagonPacketizerList::isSoloInstruction(MachineInstr *MI) {
-
-  if (MI->isInlineAsm())
+  if (MI->isEHLabel() || MI->isCFIInstruction())
     return true;
 
-  if (MI->isEHLabel())
+  if (MI->isInlineAsm())
     return true;
 
   // From Hexagon V4 Programmer's Reference Manual 3.4.4 Grouping constraints:
diff --git a/lib/Target/Hexagon/LLVMBuild.txt b/lib/Target/Hexagon/LLVMBuild.txt
index 6ffd26a2022..9d288af0214 100644
--- a/lib/Target/Hexagon/LLVMBuild.txt
+++ b/lib/Target/Hexagon/LLVMBuild.txt
@@ -28,5 +28,16 @@ has_asmprinter = 1
 type = Library
 name = HexagonCodeGen
 parent = Hexagon
-required_libraries = Analysis AsmPrinter CodeGen Core HexagonDesc HexagonInfo MC SelectionDAG Support Target
+required_libraries =
+ Analysis
+ AsmPrinter
+ CodeGen
+ Core
+ HexagonDesc
+ HexagonInfo
+ MC
+ SelectionDAG
+ Support
+ Target
+ TransformUtils
 add_to_library_groups = Hexagon
diff --git a/lib/Target/Hexagon/MCTargetDesc/CMakeLists.txt b/lib/Target/Hexagon/MCTargetDesc/CMakeLists.txt
index 4c987ed32a6..5403b106cbb 100644
--- a/lib/Target/Hexagon/MCTargetDesc/CMakeLists.txt
+++ b/lib/Target/Hexagon/MCTargetDesc/CMakeLists.txt
@@ -4,8 +4,13 @@ add_llvm_library(LLVMHexagonDesc
   HexagonInstPrinter.cpp
   HexagonMCAsmInfo.cpp
   HexagonMCCodeEmitter.cpp
+  HexagonMCCompound.cpp
+  HexagonMCDuplexInfo.cpp
+  HexagonMCELFStreamer.cpp
   HexagonMCInstrInfo.cpp
+  HexagonMCShuffler.cpp
   HexagonMCTargetDesc.cpp
+  HexagonShuffler.cpp
   )
 
 add_dependencies(LLVMHexagonDesc HexagonCommonTableGen)
diff --git a/lib/Target/Hexagon/MCTargetDesc/HexagonAsmBackend.cpp b/lib/Target/Hexagon/MCTargetDesc/HexagonAsmBackend.cpp
index 155aa9ef955..99ea2fabf86 100644
--- a/lib/Target/Hexagon/MCTargetDesc/HexagonAsmBackend.cpp
+++ b/lib/Target/Hexagon/MCTargetDesc/HexagonAsmBackend.cpp
@@ -7,19 +7,153 @@
 //
 //===----------------------------------------------------------------------===//
 
+#include "Hexagon.h"
+#include "HexagonFixupKinds.h"
 #include "HexagonMCTargetDesc.h"
+#include "MCTargetDesc/HexagonBaseInfo.h"
+#include "MCTargetDesc/HexagonMCInstrInfo.h"
 #include "llvm/MC/MCAsmBackend.h"
+#include "llvm/MC/MCAssembler.h"
 #include "llvm/MC/MCELFObjectWriter.h"
+#include "llvm/MC/MCFixupKindInfo.h"
+#include "llvm/MC/MCInstrInfo.h"
+#include "llvm/Support/Debug.h"
+#include "llvm/Support/TargetRegistry.h"
 
 using namespace llvm;
+using namespace Hexagon;
+
+#define DEBUG_TYPE "hexagon-asm-backend"
 
 namespace {
 
 class HexagonAsmBackend : public MCAsmBackend {
+  uint8_t OSABI;
+  StringRef CPU;
+  mutable uint64_t relaxedCnt;
+  std::unique_ptr <MCInstrInfo> MCII;
+  std::unique_ptr <MCInst *> RelaxTarget;
 public:
-  HexagonAsmBackend(Target const & /*T*/) {}
+  HexagonAsmBackend(Target const &T,  uint8_t OSABI, StringRef CPU) :
+    OSABI(OSABI), MCII (T.createMCInstrInfo()), RelaxTarget(new MCInst *){}
+
+  MCObjectWriter *createObjectWriter(raw_pwrite_stream &OS) const override {
+    return createHexagonELFObjectWriter(OS, OSABI, CPU);
+  }
+
+  unsigned getNumFixupKinds() const override {
+    return Hexagon::NumTargetFixupKinds;
+  }
+
+  const MCFixupKindInfo &getFixupKindInfo(MCFixupKind Kind) const override {
+    const static MCFixupKindInfo Infos[Hexagon::NumTargetFixupKinds] = {
+        // This table *must* be in same the order of fixup_* kinds in
+        // HexagonFixupKinds.h.
+        //
+        // namei                          offset  bits    flags
+        {"fixup_Hexagon_B22_PCREL", 0, 32, MCFixupKindInfo::FKF_IsPCRel},
+        {"fixup_Hexagon_B15_PCREL", 0, 32, MCFixupKindInfo::FKF_IsPCRel},
+        {"fixup_Hexagon_B7_PCREL", 0, 32, MCFixupKindInfo::FKF_IsPCRel},
+        {"fixup_Hexagon_LO16", 0, 32, 0},
+        {"fixup_Hexagon_HI16", 0, 32, 0},
+        {"fixup_Hexagon_32", 0, 32, 0},
+        {"fixup_Hexagon_16", 0, 32, 0},
+        {"fixup_Hexagon_8", 0, 32, 0},
+        {"fixup_Hexagon_GPREL16_0", 0, 32, 0},
+        {"fixup_Hexagon_GPREL16_1", 0, 32, 0},
+        {"fixup_Hexagon_GPREL16_2", 0, 32, 0},
+        {"fixup_Hexagon_GPREL16_3", 0, 32, 0},
+        {"fixup_Hexagon_HL16", 0, 32, 0},
+        {"fixup_Hexagon_B13_PCREL", 0, 32, MCFixupKindInfo::FKF_IsPCRel},
+        {"fixup_Hexagon_B9_PCREL", 0, 32, MCFixupKindInfo::FKF_IsPCRel},
+        {"fixup_Hexagon_B32_PCREL_X", 0, 32, MCFixupKindInfo::FKF_IsPCRel},
+        {"fixup_Hexagon_32_6_X", 0, 32, 0},
+        {"fixup_Hexagon_B22_PCREL_X", 0, 32, MCFixupKindInfo::FKF_IsPCRel},
+        {"fixup_Hexagon_B15_PCREL_X", 0, 32, MCFixupKindInfo::FKF_IsPCRel},
+        {"fixup_Hexagon_B13_PCREL_X", 0, 32, MCFixupKindInfo::FKF_IsPCRel},
+        {"fixup_Hexagon_B9_PCREL_X", 0, 32, MCFixupKindInfo::FKF_IsPCRel},
+        {"fixup_Hexagon_B7_PCREL_X", 0, 32, MCFixupKindInfo::FKF_IsPCRel},
+        {"fixup_Hexagon_16_X", 0, 32, 0},
+        {"fixup_Hexagon_12_X", 0, 32, 0},
+        {"fixup_Hexagon_11_X", 0, 32, 0},
+        {"fixup_Hexagon_10_X", 0, 32, 0},
+        {"fixup_Hexagon_9_X", 0, 32, 0},
+        {"fixup_Hexagon_8_X", 0, 32, 0},
+        {"fixup_Hexagon_7_X", 0, 32, 0},
+        {"fixup_Hexagon_6_X", 0, 32, 0},
+        {"fixup_Hexagon_32_PCREL", 0, 32, MCFixupKindInfo::FKF_IsPCRel},
+        {"fixup_Hexagon_COPY", 0, 32, 0},
+        {"fixup_Hexagon_GLOB_DAT", 0, 32, 0},
+        {"fixup_Hexagon_JMP_SLOT", 0, 32, 0},
+        {"fixup_Hexagon_RELATIVE", 0, 32, 0},
+        {"fixup_Hexagon_PLT_B22_PCREL", 0, 32, MCFixupKindInfo::FKF_IsPCRel},
+        {"fixup_Hexagon_GOTREL_LO16", 0, 32, 0},
+        {"fixup_Hexagon_GOTREL_HI16", 0, 32, 0},
+        {"fixup_Hexagon_GOTREL_32", 0, 32, 0},
+        {"fixup_Hexagon_GOT_LO16", 0, 32, 0},
+        {"fixup_Hexagon_GOT_HI16", 0, 32, 0},
+        {"fixup_Hexagon_GOT_32", 0, 32, 0},
+        {"fixup_Hexagon_GOT_16", 0, 32, 0},
+        {"fixup_Hexagon_DTPMOD_32", 0, 32, 0},
+        {"fixup_Hexagon_DTPREL_LO16", 0, 32, 0},
+        {"fixup_Hexagon_DTPREL_HI16", 0, 32, 0},
+        {"fixup_Hexagon_DTPREL_32", 0, 32, 0},
+        {"fixup_Hexagon_DTPREL_16", 0, 32, 0},
+        {"fixup_Hexagon_GD_PLT_B22_PCREL", 0, 32, MCFixupKindInfo::FKF_IsPCRel},
+        {"fixup_Hexagon_LD_PLT_B22_PCREL", 0, 32, MCFixupKindInfo::FKF_IsPCRel},
+        {"fixup_Hexagon_GD_GOT_LO16", 0, 32, 0},
+        {"fixup_Hexagon_GD_GOT_HI16", 0, 32, 0},
+        {"fixup_Hexagon_GD_GOT_32", 0, 32, 0},
+        {"fixup_Hexagon_GD_GOT_16", 0, 32, 0},
+        {"fixup_Hexagon_LD_GOT_LO16", 0, 32, 0},
+        {"fixup_Hexagon_LD_GOT_HI16", 0, 32, 0},
+        {"fixup_Hexagon_LD_GOT_32", 0, 32, 0},
+        {"fixup_Hexagon_LD_GOT_16", 0, 32, 0},
+        {"fixup_Hexagon_IE_LO16", 0, 32, 0},
+        {"fixup_Hexagon_IE_HI16", 0, 32, 0},
+        {"fixup_Hexagon_IE_32", 0, 32, 0},
+        {"fixup_Hexagon_IE_16", 0, 32, 0},
+        {"fixup_Hexagon_IE_GOT_LO16", 0, 32, 0},
+        {"fixup_Hexagon_IE_GOT_HI16", 0, 32, 0},
+        {"fixup_Hexagon_IE_GOT_32", 0, 32, 0},
+        {"fixup_Hexagon_IE_GOT_16", 0, 32, 0},
+        {"fixup_Hexagon_TPREL_LO16", 0, 32, 0},
+        {"fixup_Hexagon_TPREL_HI16", 0, 32, 0},
+        {"fixup_Hexagon_TPREL_32", 0, 32, 0},
+        {"fixup_Hexagon_TPREL_16", 0, 32, 0},
+        {"fixup_Hexagon_6_PCREL_X", 0, 32, MCFixupKindInfo::FKF_IsPCRel},
+        {"fixup_Hexagon_GOTREL_32_6_X", 0, 32, 0},
+        {"fixup_Hexagon_GOTREL_16_X", 0, 32, 0},
+        {"fixup_Hexagon_GOTREL_11_X", 0, 32, 0},
+        {"fixup_Hexagon_GOT_32_6_X", 0, 32, 0},
+        {"fixup_Hexagon_GOT_16_X", 0, 32, 0},
+        {"fixup_Hexagon_GOT_11_X", 0, 32, 0},
+        {"fixup_Hexagon_DTPREL_32_6_X", 0, 32, 0},
+        {"fixup_Hexagon_DTPREL_16_X", 0, 32, 0},
+        {"fixup_Hexagon_DTPREL_11_X", 0, 32, 0},
+        {"fixup_Hexagon_GD_GOT_32_6_X", 0, 32, 0},
+        {"fixup_Hexagon_GD_GOT_16_X", 0, 32, 0},
+        {"fixup_Hexagon_GD_GOT_11_X", 0, 32, 0},
+        {"fixup_Hexagon_LD_GOT_32_6_X", 0, 32, 0},
+        {"fixup_Hexagon_LD_GOT_16_X", 0, 32, 0},
+        {"fixup_Hexagon_LD_GOT_11_X", 0, 32, 0},
+        {"fixup_Hexagon_IE_32_6_X", 0, 32, 0},
+        {"fixup_Hexagon_IE_16_X", 0, 32, 0},
+        {"fixup_Hexagon_IE_GOT_32_6_X", 0, 32, 0},
+        {"fixup_Hexagon_IE_GOT_16_X", 0, 32, 0},
+        {"fixup_Hexagon_IE_GOT_11_X", 0, 32, 0},
+        {"fixup_Hexagon_TPREL_32_6_X", 0, 32, 0},
+        {"fixup_Hexagon_TPREL_16_X", 0, 32, 0},
+        {"fixup_Hexagon_TPREL_11_X", 0, 32, 0}};
 
-  unsigned getNumFixupKinds() const override { return 0; }
+    if (Kind < FirstTargetFixupKind) {
+      return MCAsmBackend::getFixupKindInfo(Kind);
+    }
+
+    assert(unsigned(Kind - FirstTargetFixupKind) < getNumFixupKinds() &&
+           "Invalid kind!");
+    return Infos[Kind - FirstTargetFixupKind];
+  }
 
   void applyFixup(MCFixup const & /*Fixup*/, char * /*Data*/,
                   unsigned /*DataSize*/, uint64_t /*Value*/,
@@ -27,14 +161,119 @@ class HexagonAsmBackend : public MCAsmBackend {
     return;
   }
 
-  bool mayNeedRelaxation(MCInst const & /*Inst*/) const override {
+  bool isInstRelaxable(MCInst const &HMI) const {
+    const MCInstrDesc &MCID = HexagonMCInstrInfo::getDesc(*MCII, HMI);
+    bool Relaxable = false;
+    // Branches and loop-setup insns are handled as necessary by relaxation.
+    if (llvm::HexagonMCInstrInfo::getType(*MCII, HMI) == HexagonII::TypeJ ||
+        (llvm::HexagonMCInstrInfo::getType(*MCII, HMI) == HexagonII::TypeNV &&
+         MCID.isBranch()) ||
+        (llvm::HexagonMCInstrInfo::getType(*MCII, HMI) == HexagonII::TypeCR &&
+         HMI.getOpcode() != Hexagon::C4_addipc))
+      if (HexagonMCInstrInfo::isExtendable(*MCII, HMI))
+        Relaxable = true;
+
+    return Relaxable;
+  }
+
+  /// MayNeedRelaxation - Check whether the given instruction may need
+  /// relaxation.
+  ///
+  /// \param Inst - The instruction to test.
+  bool mayNeedRelaxation(MCInst const &Inst) const override {
+    assert(HexagonMCInstrInfo::isBundle(Inst));
+    bool PreviousIsExtender = false;
+    for (auto const &I : HexagonMCInstrInfo::bundleInstructions(Inst)) {
+      auto const &Inst = *I.getInst();
+      if (!PreviousIsExtender) {
+        if (isInstRelaxable(Inst))
+          return true;
+      }
+      PreviousIsExtender = HexagonMCInstrInfo::isImmext(Inst);
+    }
+    return false;
+  }
+
+  /// fixupNeedsRelaxation - Target specific predicate for whether a given
+  /// fixup requires the associated instruction to be relaxed.
+  bool fixupNeedsRelaxationAdvanced(const MCFixup &Fixup, bool Resolved,
+                                    uint64_t Value,
+                                    const MCRelaxableFragment *DF,
+                                    const MCAsmLayout &Layout) const override {
+    MCInst const &MCB = DF->getInst();
+    assert(HexagonMCInstrInfo::isBundle(MCB));
+
+    *RelaxTarget = nullptr;
+    MCInst &MCI = const_cast<MCInst &>(HexagonMCInstrInfo::instruction(
+        MCB, Fixup.getOffset() / HEXAGON_INSTR_SIZE));
+    // If we cannot resolve the fixup value, it requires relaxation.
+    if (!Resolved) {
+      switch ((unsigned)Fixup.getKind()) {
+      case fixup_Hexagon_B22_PCREL:
+      // GetFixupCount assumes B22 won't relax
+      // Fallthrough
+      default:
+        return false;
+        break;
+      case fixup_Hexagon_B13_PCREL:
+      case fixup_Hexagon_B15_PCREL:
+      case fixup_Hexagon_B9_PCREL:
+      case fixup_Hexagon_B7_PCREL: {
+        if (HexagonMCInstrInfo::bundleSize(MCB) < HEXAGON_PACKET_SIZE) {
+          ++relaxedCnt;
+          *RelaxTarget = &MCI;
+          return true;
+        } else {
+          return false;
+        }
+        break;
+      }
+      }
+    }
+    bool Relaxable = isInstRelaxable(MCI);
+    if (Relaxable == false)
+      return false;
+
+    MCFixupKind Kind = Fixup.getKind();
+    int64_t sValue = Value;
+    int64_t maxValue;
+
+    switch ((unsigned)Kind) {
+    case fixup_Hexagon_B7_PCREL:
+      maxValue = 1 << 8;
+      break;
+    case fixup_Hexagon_B9_PCREL:
+      maxValue = 1 << 10;
+      break;
+    case fixup_Hexagon_B15_PCREL:
+      maxValue = 1 << 16;
+      break;
+    case fixup_Hexagon_B22_PCREL:
+      maxValue = 1 << 23;
+      break;
+    default:
+      maxValue = INT64_MAX;
+      break;
+    }
+
+    bool isFarAway = -maxValue > sValue || sValue > maxValue - 1;
+
+    if (isFarAway) {
+      if (HexagonMCInstrInfo::bundleSize(MCB) < HEXAGON_PACKET_SIZE) {
+        ++relaxedCnt;
+        *RelaxTarget = &MCI;
+        return true;
+      }
+    }
+
     return false;
   }
 
-  bool fixupNeedsRelaxation(MCFixup const & /*Fixup*/, uint64_t /*Value*/,
-                            MCRelaxableFragment const * /*DF*/,
-                            MCAsmLayout const & /*Layout*/) const override {
-    llvm_unreachable("fixupNeedsRelaxation() unimplemented");
+  /// Simple predicate for targets where !Resolved implies requiring relaxation
+  bool fixupNeedsRelaxation(const MCFixup &Fixup, uint64_t Value,
+                            const MCRelaxableFragment *DF,
+                            const MCAsmLayout &Layout) const override {
+    llvm_unreachable("Handled by fixupNeedsRelaxationAdvanced");
   }
 
   void relaxInstruction(MCInst const & /*Inst*/,
@@ -42,24 +281,27 @@ class HexagonAsmBackend : public MCAsmBackend {
     llvm_unreachable("relaxInstruction() unimplemented");
   }
 
-  bool writeNopData(uint64_t /*Count*/,
-                    MCObjectWriter * /*OW*/) const override {
-    return true;
-  }
-};
-} // end anonymous namespace
-
-namespace {
-class ELFHexagonAsmBackend : public HexagonAsmBackend {
-  uint8_t OSABI;
+  bool writeNopData(uint64_t Count,
+                    MCObjectWriter * OW) const override {
+    static const uint32_t Nopcode  = 0x7f000000, // Hard-coded NOP.
+                          ParseIn  = 0x00004000, // In packet parse-bits.
+                          ParseEnd = 0x0000c000; // End of packet parse-bits.
 
-public:
-  ELFHexagonAsmBackend(Target const &T, uint8_t OSABI)
-      : HexagonAsmBackend(T), OSABI(OSABI) {}
+    while(Count % HEXAGON_INSTR_SIZE) {
+      DEBUG(dbgs() << "Alignment not a multiple of the instruction size:" <<
+          Count % HEXAGON_INSTR_SIZE << "/" << HEXAGON_INSTR_SIZE << "\n");
+      --Count;
+      OW->write8(0);
+    }
 
-  MCObjectWriter *createObjectWriter(raw_pwrite_stream &OS) const override {
-    StringRef CPU("HexagonV4");
-    return createHexagonELFObjectWriter(OS, OSABI, CPU);
+    while(Count) {
+      Count -= HEXAGON_INSTR_SIZE;
+      // Close the packet whenever a multiple of the maximum packet size remains
+      uint32_t ParseBits = (Count % (HEXAGON_PACKET_SIZE * HEXAGON_INSTR_SIZE))?
+                           ParseIn: ParseEnd;
+      OW->write32(Nopcode | ParseBits);
+    }
+    return true;
   }
 };
 } // end anonymous namespace
@@ -67,8 +309,8 @@ class ELFHexagonAsmBackend : public HexagonAsmBackend {
 namespace llvm {
 MCAsmBackend *createHexagonAsmBackend(Target const &T,
                                       MCRegisterInfo const & /*MRI*/,
-                                      StringRef TT, StringRef /*CPU*/) {
-  uint8_t OSABI = MCELFObjectTargetWriter::getOSABI(Triple(TT).getOS());
-  return new ELFHexagonAsmBackend(T, OSABI);
+                                      const Triple &TT, StringRef CPU) {
+  uint8_t OSABI = MCELFObjectTargetWriter::getOSABI(TT.getOS());
+  return new HexagonAsmBackend(T, OSABI, CPU);
 }
 }
diff --git a/lib/Target/Hexagon/MCTargetDesc/HexagonBaseInfo.h b/lib/Target/Hexagon/MCTargetDesc/HexagonBaseInfo.h
index 8e02f799d7e..f4d162ccf6a 100644
--- a/lib/Target/Hexagon/MCTargetDesc/HexagonBaseInfo.h
+++ b/lib/Target/Hexagon/MCTargetDesc/HexagonBaseInfo.h
@@ -43,6 +43,7 @@ namespace HexagonII {
     TypeXTYPE   = 8,
     TypeMEMOP   = 9,
     TypeNV      = 10,
+    TypeDUPLEX  = 11,
     TypePREFIX  = 30, // Such as extenders.
     TypeENDLOOP = 31  // Such as end of a HW loop.
   };
@@ -70,7 +71,7 @@ namespace HexagonII {
     PostInc        = 6   // Post increment addressing mode
   };
 
-  enum MemAccessSize {
+  enum class MemAccessSize {
     NoMemAccess = 0,            // Not a memory acces instruction.
     ByteAccess = 1,             // Byte access instruction (memb).
     HalfWordAccess = 2,         // Half word access instruction (memh).
@@ -190,7 +191,26 @@ namespace HexagonII {
     MO_GPREL
   };
 
-  enum class InstParseBits : uint32_t {
+  // Hexagon Sub-instruction classes.
+  enum SubInstructionGroup {
+    HSIG_None = 0,
+    HSIG_L1,
+    HSIG_L2,
+    HSIG_S1,
+    HSIG_S2,
+    HSIG_A,
+    HSIG_Compound
+  };
+
+  // Hexagon Compound classes.
+  enum CompoundGroup {
+    HCG_None = 0,
+    HCG_A,
+    HCG_B,
+    HCG_C
+  };
+
+  enum InstParseBits {
     INST_PARSE_MASK       = 0x0000c000,
     INST_PARSE_PACKET_END = 0x0000c000,
     INST_PARSE_LOOP_END   = 0x00008000,
diff --git a/lib/Target/Hexagon/MCTargetDesc/HexagonELFObjectWriter.cpp b/lib/Target/Hexagon/MCTargetDesc/HexagonELFObjectWriter.cpp
index fde935b2758..da5d4d1da69 100644
--- a/lib/Target/Hexagon/MCTargetDesc/HexagonELFObjectWriter.cpp
+++ b/lib/Target/Hexagon/MCTargetDesc/HexagonELFObjectWriter.cpp
@@ -8,6 +8,7 @@
 //===----------------------------------------------------------------------===//
 
 #include "Hexagon.h"
+#include "MCTargetDesc/HexagonFixupKinds.h"
 #include "llvm/MC/MCAssembler.h"
 #include "llvm/MC/MCELFObjectWriter.h"
 #include "llvm/Support/Debug.h"
@@ -37,22 +38,209 @@ HexagonELFObjectWriter::HexagonELFObjectWriter(uint8_t OSABI, StringRef C)
                               /*HasRelocationAddend*/ true),
       CPU(C) {}
 
-unsigned HexagonELFObjectWriter::GetRelocType(MCValue const &/*Target*/,
+unsigned HexagonELFObjectWriter::GetRelocType(MCValue const & /*Target*/,
                                               MCFixup const &Fixup,
                                               bool IsPCRel) const {
-  unsigned Type = (unsigned)ELF::R_HEX_NONE;
-  llvm::MCFixupKind Kind = Fixup.getKind();
-
-  switch (Kind) {
+  switch ((unsigned)Fixup.getKind()) {
   default:
     DEBUG(dbgs() << "unrecognized relocation " << Fixup.getKind() << "\n");
     llvm_unreachable("Unimplemented Fixup kind!");
-    break;
+    return ELF::R_HEX_NONE;
   case FK_Data_4:
-    Type = (IsPCRel) ? ELF::R_HEX_32_PCREL : ELF::R_HEX_32;
-    break;
+    return (IsPCRel) ? ELF::R_HEX_32_PCREL : ELF::R_HEX_32;
+  case FK_PCRel_4:
+    return ELF::R_HEX_32_PCREL;
+  case FK_Data_2:
+    return ELF::R_HEX_16;
+  case FK_Data_1:
+    return ELF::R_HEX_8;
+  case fixup_Hexagon_B22_PCREL:
+    return ELF::R_HEX_B22_PCREL;
+  case fixup_Hexagon_B15_PCREL:
+    return ELF::R_HEX_B15_PCREL;
+  case fixup_Hexagon_B7_PCREL:
+    return ELF::R_HEX_B7_PCREL;
+  case fixup_Hexagon_LO16:
+    return ELF::R_HEX_LO16;
+  case fixup_Hexagon_HI16:
+    return ELF::R_HEX_HI16;
+  case fixup_Hexagon_32:
+    return ELF::R_HEX_32;
+  case fixup_Hexagon_16:
+    return ELF::R_HEX_16;
+  case fixup_Hexagon_8:
+    return ELF::R_HEX_8;
+  case fixup_Hexagon_GPREL16_0:
+    return ELF::R_HEX_GPREL16_0;
+  case fixup_Hexagon_GPREL16_1:
+    return ELF::R_HEX_GPREL16_1;
+  case fixup_Hexagon_GPREL16_2:
+    return ELF::R_HEX_GPREL16_2;
+  case fixup_Hexagon_GPREL16_3:
+    return ELF::R_HEX_GPREL16_3;
+  case fixup_Hexagon_HL16:
+    return ELF::R_HEX_HL16;
+  case fixup_Hexagon_B13_PCREL:
+    return ELF::R_HEX_B13_PCREL;
+  case fixup_Hexagon_B9_PCREL:
+    return ELF::R_HEX_B9_PCREL;
+  case fixup_Hexagon_B32_PCREL_X:
+    return ELF::R_HEX_B32_PCREL_X;
+  case fixup_Hexagon_32_6_X:
+    return ELF::R_HEX_32_6_X;
+  case fixup_Hexagon_B22_PCREL_X:
+    return ELF::R_HEX_B22_PCREL_X;
+  case fixup_Hexagon_B15_PCREL_X:
+    return ELF::R_HEX_B15_PCREL_X;
+  case fixup_Hexagon_B13_PCREL_X:
+    return ELF::R_HEX_B13_PCREL_X;
+  case fixup_Hexagon_B9_PCREL_X:
+    return ELF::R_HEX_B9_PCREL_X;
+  case fixup_Hexagon_B7_PCREL_X:
+    return ELF::R_HEX_B7_PCREL_X;
+  case fixup_Hexagon_16_X:
+    return ELF::R_HEX_16_X;
+  case fixup_Hexagon_12_X:
+    return ELF::R_HEX_12_X;
+  case fixup_Hexagon_11_X:
+    return ELF::R_HEX_11_X;
+  case fixup_Hexagon_10_X:
+    return ELF::R_HEX_10_X;
+  case fixup_Hexagon_9_X:
+    return ELF::R_HEX_9_X;
+  case fixup_Hexagon_8_X:
+    return ELF::R_HEX_8_X;
+  case fixup_Hexagon_7_X:
+    return ELF::R_HEX_7_X;
+  case fixup_Hexagon_6_X:
+    return ELF::R_HEX_6_X;
+  case fixup_Hexagon_32_PCREL:
+    return ELF::R_HEX_32_PCREL;
+  case fixup_Hexagon_COPY:
+    return ELF::R_HEX_COPY;
+  case fixup_Hexagon_GLOB_DAT:
+    return ELF::R_HEX_GLOB_DAT;
+  case fixup_Hexagon_JMP_SLOT:
+    return ELF::R_HEX_JMP_SLOT;
+  case fixup_Hexagon_RELATIVE:
+    return ELF::R_HEX_RELATIVE;
+  case fixup_Hexagon_PLT_B22_PCREL:
+    return ELF::R_HEX_PLT_B22_PCREL;
+  case fixup_Hexagon_GOTREL_LO16:
+    return ELF::R_HEX_GOTREL_LO16;
+  case fixup_Hexagon_GOTREL_HI16:
+    return ELF::R_HEX_GOTREL_HI16;
+  case fixup_Hexagon_GOTREL_32:
+    return ELF::R_HEX_GOTREL_32;
+  case fixup_Hexagon_GOT_LO16:
+    return ELF::R_HEX_GOT_LO16;
+  case fixup_Hexagon_GOT_HI16:
+    return ELF::R_HEX_GOT_HI16;
+  case fixup_Hexagon_GOT_32:
+    return ELF::R_HEX_GOT_32;
+  case fixup_Hexagon_GOT_16:
+    return ELF::R_HEX_GOT_16;
+  case fixup_Hexagon_DTPMOD_32:
+    return ELF::R_HEX_DTPMOD_32;
+  case fixup_Hexagon_DTPREL_LO16:
+    return ELF::R_HEX_DTPREL_LO16;
+  case fixup_Hexagon_DTPREL_HI16:
+    return ELF::R_HEX_DTPREL_HI16;
+  case fixup_Hexagon_DTPREL_32:
+    return ELF::R_HEX_DTPREL_32;
+  case fixup_Hexagon_DTPREL_16:
+    return ELF::R_HEX_DTPREL_16;
+  case fixup_Hexagon_GD_PLT_B22_PCREL:
+    return ELF::R_HEX_GD_PLT_B22_PCREL;
+  case fixup_Hexagon_LD_PLT_B22_PCREL:
+    return ELF::R_HEX_LD_PLT_B22_PCREL;
+  case fixup_Hexagon_GD_GOT_LO16:
+    return ELF::R_HEX_GD_GOT_LO16;
+  case fixup_Hexagon_GD_GOT_HI16:
+    return ELF::R_HEX_GD_GOT_HI16;
+  case fixup_Hexagon_GD_GOT_32:
+    return ELF::R_HEX_GD_GOT_32;
+  case fixup_Hexagon_GD_GOT_16:
+    return ELF::R_HEX_GD_GOT_16;
+  case fixup_Hexagon_LD_GOT_LO16:
+    return ELF::R_HEX_LD_GOT_LO16;
+  case fixup_Hexagon_LD_GOT_HI16:
+    return ELF::R_HEX_LD_GOT_HI16;
+  case fixup_Hexagon_LD_GOT_32:
+    return ELF::R_HEX_LD_GOT_32;
+  case fixup_Hexagon_LD_GOT_16:
+    return ELF::R_HEX_LD_GOT_16;
+  case fixup_Hexagon_IE_LO16:
+    return ELF::R_HEX_IE_LO16;
+  case fixup_Hexagon_IE_HI16:
+    return ELF::R_HEX_IE_HI16;
+  case fixup_Hexagon_IE_32:
+    return ELF::R_HEX_IE_32;
+  case fixup_Hexagon_IE_GOT_LO16:
+    return ELF::R_HEX_IE_GOT_LO16;
+  case fixup_Hexagon_IE_GOT_HI16:
+    return ELF::R_HEX_IE_GOT_HI16;
+  case fixup_Hexagon_IE_GOT_32:
+    return ELF::R_HEX_IE_GOT_32;
+  case fixup_Hexagon_IE_GOT_16:
+    return ELF::R_HEX_IE_GOT_16;
+  case fixup_Hexagon_TPREL_LO16:
+    return ELF::R_HEX_TPREL_LO16;
+  case fixup_Hexagon_TPREL_HI16:
+    return ELF::R_HEX_TPREL_HI16;
+  case fixup_Hexagon_TPREL_32:
+    return ELF::R_HEX_TPREL_32;
+  case fixup_Hexagon_TPREL_16:
+    return ELF::R_HEX_TPREL_16;
+  case fixup_Hexagon_6_PCREL_X:
+    return ELF::R_HEX_6_PCREL_X;
+  case fixup_Hexagon_GOTREL_32_6_X:
+    return ELF::R_HEX_GOTREL_32_6_X;
+  case fixup_Hexagon_GOTREL_16_X:
+    return ELF::R_HEX_GOTREL_16_X;
+  case fixup_Hexagon_GOTREL_11_X:
+    return ELF::R_HEX_GOTREL_11_X;
+  case fixup_Hexagon_GOT_32_6_X:
+    return ELF::R_HEX_GOT_32_6_X;
+  case fixup_Hexagon_GOT_16_X:
+    return ELF::R_HEX_GOT_16_X;
+  case fixup_Hexagon_GOT_11_X:
+    return ELF::R_HEX_GOT_11_X;
+  case fixup_Hexagon_DTPREL_32_6_X:
+    return ELF::R_HEX_DTPREL_32_6_X;
+  case fixup_Hexagon_DTPREL_16_X:
+    return ELF::R_HEX_DTPREL_16_X;
+  case fixup_Hexagon_DTPREL_11_X:
+    return ELF::R_HEX_DTPREL_11_X;
+  case fixup_Hexagon_GD_GOT_32_6_X:
+    return ELF::R_HEX_GD_GOT_32_6_X;
+  case fixup_Hexagon_GD_GOT_16_X:
+    return ELF::R_HEX_GD_GOT_16_X;
+  case fixup_Hexagon_GD_GOT_11_X:
+    return ELF::R_HEX_GD_GOT_11_X;
+  case fixup_Hexagon_LD_GOT_32_6_X:
+    return ELF::R_HEX_LD_GOT_32_6_X;
+  case fixup_Hexagon_LD_GOT_16_X:
+    return ELF::R_HEX_LD_GOT_16_X;
+  case fixup_Hexagon_LD_GOT_11_X:
+    return ELF::R_HEX_LD_GOT_11_X;
+  case fixup_Hexagon_IE_32_6_X:
+    return ELF::R_HEX_IE_32_6_X;
+  case fixup_Hexagon_IE_16_X:
+    return ELF::R_HEX_IE_16_X;
+  case fixup_Hexagon_IE_GOT_32_6_X:
+    return ELF::R_HEX_IE_GOT_32_6_X;
+  case fixup_Hexagon_IE_GOT_16_X:
+    return ELF::R_HEX_IE_GOT_16_X;
+  case fixup_Hexagon_IE_GOT_11_X:
+    return ELF::R_HEX_IE_GOT_11_X;
+  case fixup_Hexagon_TPREL_32_6_X:
+    return ELF::R_HEX_TPREL_32_6_X;
+  case fixup_Hexagon_TPREL_16_X:
+    return ELF::R_HEX_TPREL_16_X;
+  case fixup_Hexagon_TPREL_11_X:
+    return ELF::R_HEX_TPREL_11_X;
   }
-  return Type;
 }
 
 MCObjectWriter *llvm::createHexagonELFObjectWriter(raw_pwrite_stream &OS,
diff --git a/lib/Target/Hexagon/MCTargetDesc/HexagonFixupKinds.h b/lib/Target/Hexagon/MCTargetDesc/HexagonFixupKinds.h
new file mode 100644
index 00000000000..4bbfbec883c
--- /dev/null
+++ b/lib/Target/Hexagon/MCTargetDesc/HexagonFixupKinds.h
@@ -0,0 +1,137 @@
+//===-- HexagonFixupKinds.h - Hexagon Specific Fixup Entries --------------===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_HEXAGON_HEXAGONFIXUPKINDS_H
+#define LLVM_HEXAGON_HEXAGONFIXUPKINDS_H
+
+#include "llvm/MC/MCFixup.h"
+
+namespace llvm {
+namespace Hexagon {
+enum Fixups {
+  // Branch fixups for R_HEX_B{22,15,7}_PCREL.
+  fixup_Hexagon_B22_PCREL = FirstTargetFixupKind,
+  fixup_Hexagon_B15_PCREL,
+  fixup_Hexagon_B7_PCREL,
+  fixup_Hexagon_LO16,
+  fixup_Hexagon_HI16,
+  fixup_Hexagon_32,
+  fixup_Hexagon_16,
+  fixup_Hexagon_8,
+  fixup_Hexagon_GPREL16_0,
+  fixup_Hexagon_GPREL16_1,
+  fixup_Hexagon_GPREL16_2,
+  fixup_Hexagon_GPREL16_3,
+  fixup_Hexagon_HL16,
+  fixup_Hexagon_B13_PCREL,
+  fixup_Hexagon_B9_PCREL,
+  fixup_Hexagon_B32_PCREL_X,
+  fixup_Hexagon_32_6_X,
+  fixup_Hexagon_B22_PCREL_X,
+  fixup_Hexagon_B15_PCREL_X,
+  fixup_Hexagon_B13_PCREL_X,
+  fixup_Hexagon_B9_PCREL_X,
+  fixup_Hexagon_B7_PCREL_X,
+  fixup_Hexagon_16_X,
+  fixup_Hexagon_12_X,
+  fixup_Hexagon_11_X,
+  fixup_Hexagon_10_X,
+  fixup_Hexagon_9_X,
+  fixup_Hexagon_8_X,
+  fixup_Hexagon_7_X,
+  fixup_Hexagon_6_X,
+  fixup_Hexagon_32_PCREL,
+  fixup_Hexagon_COPY,
+  fixup_Hexagon_GLOB_DAT,
+  fixup_Hexagon_JMP_SLOT,
+  fixup_Hexagon_RELATIVE,
+  fixup_Hexagon_PLT_B22_PCREL,
+  fixup_Hexagon_GOTREL_LO16,
+  fixup_Hexagon_GOTREL_HI16,
+  fixup_Hexagon_GOTREL_32,
+  fixup_Hexagon_GOT_LO16,
+  fixup_Hexagon_GOT_HI16,
+  fixup_Hexagon_GOT_32,
+  fixup_Hexagon_GOT_16,
+  fixup_Hexagon_DTPMOD_32,
+  fixup_Hexagon_DTPREL_LO16,
+  fixup_Hexagon_DTPREL_HI16,
+  fixup_Hexagon_DTPREL_32,
+  fixup_Hexagon_DTPREL_16,
+  fixup_Hexagon_GD_PLT_B22_PCREL,
+  fixup_Hexagon_LD_PLT_B22_PCREL,
+  fixup_Hexagon_GD_GOT_LO16,
+  fixup_Hexagon_GD_GOT_HI16,
+  fixup_Hexagon_GD_GOT_32,
+  fixup_Hexagon_GD_GOT_16,
+  fixup_Hexagon_LD_GOT_LO16,
+  fixup_Hexagon_LD_GOT_HI16,
+  fixup_Hexagon_LD_GOT_32,
+  fixup_Hexagon_LD_GOT_16,
+  fixup_Hexagon_IE_LO16,
+  fixup_Hexagon_IE_HI16,
+  fixup_Hexagon_IE_32,
+  fixup_Hexagon_IE_16,
+  fixup_Hexagon_IE_GOT_LO16,
+  fixup_Hexagon_IE_GOT_HI16,
+  fixup_Hexagon_IE_GOT_32,
+  fixup_Hexagon_IE_GOT_16,
+  fixup_Hexagon_TPREL_LO16,
+  fixup_Hexagon_TPREL_HI16,
+  fixup_Hexagon_TPREL_32,
+  fixup_Hexagon_TPREL_16,
+  fixup_Hexagon_6_PCREL_X,
+  fixup_Hexagon_GOTREL_32_6_X,
+  fixup_Hexagon_GOTREL_16_X,
+  fixup_Hexagon_GOTREL_11_X,
+  fixup_Hexagon_GOT_32_6_X,
+  fixup_Hexagon_GOT_16_X,
+  fixup_Hexagon_GOT_11_X,
+  fixup_Hexagon_DTPREL_32_6_X,
+  fixup_Hexagon_DTPREL_16_X,
+  fixup_Hexagon_DTPREL_11_X,
+  fixup_Hexagon_GD_GOT_32_6_X,
+  fixup_Hexagon_GD_GOT_16_X,
+  fixup_Hexagon_GD_GOT_11_X,
+  fixup_Hexagon_LD_GOT_32_6_X,
+  fixup_Hexagon_LD_GOT_16_X,
+  fixup_Hexagon_LD_GOT_11_X,
+  fixup_Hexagon_IE_32_6_X,
+  fixup_Hexagon_IE_16_X,
+  fixup_Hexagon_IE_GOT_32_6_X,
+  fixup_Hexagon_IE_GOT_16_X,
+  fixup_Hexagon_IE_GOT_11_X,
+  fixup_Hexagon_TPREL_32_6_X,
+  fixup_Hexagon_TPREL_16_X,
+  fixup_Hexagon_TPREL_11_X,
+
+  LastTargetFixupKind,
+  NumTargetFixupKinds = LastTargetFixupKind - FirstTargetFixupKind
+};
+enum FixupBitmaps : unsigned {
+  Word8 = 0xff,
+  Word16 = 0xffff,
+  Word32 = 0xffffffff,
+  Word32_LO = 0x00c03fff,
+  Word32_HL = 0x0, // Not Implemented
+  Word32_GP = 0x0, // Not Implemented
+  Word32_B7 = 0x00001f18,
+  Word32_B9 = 0x003000fe,
+  Word32_B13 = 0x00202ffe,
+  Word32_B15 = 0x00df20fe,
+  Word32_B22 = 0x01ff3ffe,
+  Word32_R6 = 0x000007e0,
+  Word32_U6 = 0x0,  // Not Implemented
+  Word32_U16 = 0x0, // Not Implemented
+  Word32_X26 = 0x0fff3fff
+};
+} // namespace Hexagon
+} // namespace llvm
+
+#endif // LLVM_HEXAGON_HEXAGONFIXUPKINDS_H
diff --git a/lib/Target/Hexagon/MCTargetDesc/HexagonInstPrinter.cpp b/lib/Target/Hexagon/MCTargetDesc/HexagonInstPrinter.cpp
index ec552342d8e..36f81465eef 100644
--- a/lib/Target/Hexagon/MCTargetDesc/HexagonInstPrinter.cpp
+++ b/lib/Target/Hexagon/MCTargetDesc/HexagonInstPrinter.cpp
@@ -28,7 +28,47 @@ using namespace llvm;
 #define GET_INSTRUCTION_NAME
 #include "HexagonGenAsmWriter.inc"
 
-const char HexagonInstPrinter::PacketPadding = '\t';
+HexagonAsmInstPrinter::HexagonAsmInstPrinter(MCInstPrinter *RawPrinter)
+    : MCInstPrinter(*RawPrinter), RawPrinter(RawPrinter) {}
+
+void HexagonAsmInstPrinter::printInst(MCInst const *MI, raw_ostream &O,
+                                      StringRef Annot,
+                                      MCSubtargetInfo const &STI) {
+  assert(HexagonMCInstrInfo::isBundle(*MI));
+  assert(HexagonMCInstrInfo::bundleSize(*MI) <= HEXAGON_PACKET_SIZE);
+  std::string Buffer;
+  {
+    raw_string_ostream TempStream(Buffer);
+    RawPrinter->printInst(MI, TempStream, "", STI);
+  }
+  StringRef Contents(Buffer);
+  auto PacketBundle = Contents.rsplit('\n');
+  auto HeadTail = PacketBundle.first.split('\n');
+  auto Preamble = "\t{\n\t\t";
+  auto Separator = "";
+  while(!HeadTail.first.empty()) {
+    O << Separator;
+    StringRef Inst;
+    auto Duplex = HeadTail.first.split('\v');
+    if(!Duplex.second.empty()){
+      O << Duplex.first << "\n";
+      Inst = Duplex.second;
+    }
+    else
+      Inst = Duplex.first;
+    O << Preamble;
+    O << Inst;
+    HeadTail = HeadTail.second.split('\n');
+    Preamble = "";
+    Separator = "\n\t\t";
+  }
+  O << "\n\t}" << PacketBundle.second;
+}
+
+void HexagonAsmInstPrinter::printRegName(raw_ostream &O, unsigned RegNo) const {
+  RawPrinter->printRegName(O, RegNo);
+}
+
 // Return the minimum value that a constant extendable operand can have
 // without being extended.
 static int getMinValue(uint64_t TSFlags) {
@@ -73,52 +113,48 @@ StringRef HexagonInstPrinter::getOpcodeName(unsigned Opcode) const {
   return MII.getName(Opcode);
 }
 
-StringRef HexagonInstPrinter::getRegName(unsigned RegNo) const {
-  return getRegisterName(RegNo);
+void HexagonInstPrinter::printRegName(raw_ostream &OS, unsigned RegNo) const {
+  OS << getRegisterName(RegNo);
 }
 
-void HexagonInstPrinter::printInst(MCInst const *MI, raw_ostream &O,
-                                   StringRef Annot,
-                                   const MCSubtargetInfo &STI) {
-  const char startPacket = '{',
-             endPacket = '}';
-  // TODO: add outer HW loop when it's supported too.
-  if (MI->getOpcode() == Hexagon::ENDLOOP0) {
-    // Ending a harware loop is different from ending an regular packet.
-    assert(HexagonMCInstrInfo::isPacketEnd(*MI) && "Loop-end must also end the packet");
-
-    if (HexagonMCInstrInfo::isPacketBegin(*MI)) {
-      // There must be a packet to end a loop.
-      // FIXME: when shuffling is always run, this shouldn't be needed.
-      MCInst Nop;
-      StringRef NoAnnot;
-
-      Nop.setOpcode (Hexagon::A2_nop);
-      HexagonMCInstrInfo::setPacketBegin (Nop, HexagonMCInstrInfo::isPacketBegin(*MI));
-      printInst (&Nop, O, NoAnnot, STI);
-    }
+void HexagonInstPrinter::setExtender(MCInst const &MCI) {
+  HasExtender = HexagonMCInstrInfo::isImmext(MCI);
+}
 
-    // Close the packet.
-    if (HexagonMCInstrInfo::isPacketEnd(*MI))
-      O << PacketPadding << endPacket;
+void HexagonInstPrinter::printInst(MCInst const *MI, raw_ostream &OS,
+                                   StringRef Annot,
+                                   MCSubtargetInfo const &STI) {
+  assert(HexagonMCInstrInfo::isBundle(*MI));
+  assert(HexagonMCInstrInfo::bundleSize(*MI) <= HEXAGON_PACKET_SIZE);
+  HasExtender = false;
+  for (auto const &I : HexagonMCInstrInfo::bundleInstructions(*MI)) {
+    MCInst const &MCI = *I.getInst();
+    if (HexagonMCInstrInfo::isDuplex(MII, MCI)) {
+      printInstruction(MCI.getOperand(1).getInst(), OS);
+      OS << '\v';
+      HasExtender = false;
+      printInstruction(MCI.getOperand(0).getInst(), OS);
+    } else
+      printInstruction(&MCI, OS);
+    setExtender(MCI);
+    OS << "\n";
+  }
 
-    printInstruction(MI, O);
+  auto Separator = "";
+  if (HexagonMCInstrInfo::isInnerLoop(*MI)) {
+    OS << Separator;
+    Separator = " ";
+    MCInst ME;
+    ME.setOpcode(Hexagon::ENDLOOP0);
+    printInstruction(&ME, OS);
   }
-  else {
-    // Prefix the insn opening the packet.
-    if (HexagonMCInstrInfo::isPacketBegin(*MI))
-      O << PacketPadding << startPacket << '\n';
-
-    printInstruction(MI, O);
-
-    // Suffix the insn closing the packet.
-    if (HexagonMCInstrInfo::isPacketEnd(*MI))
-      // Suffix the packet in a new line always, since the GNU assembler has
-      // issues with a closing brace on the same line as CONST{32,64}.
-      O << '\n' << PacketPadding << endPacket;
+  if (HexagonMCInstrInfo::isOuterLoop(*MI)) {
+    OS << Separator;
+    Separator = " ";
+    MCInst ME;
+    ME.setOpcode(Hexagon::ENDLOOP1);
+    printInstruction(&ME, OS);
   }
-
-  printAnnotation(O, Annot);
 }
 
 void HexagonInstPrinter::printOperand(const MCInst *MI, unsigned OpNo,
@@ -126,9 +162,9 @@ void HexagonInstPrinter::printOperand(const MCInst *MI, unsigned OpNo,
   const MCOperand& MO = MI->getOperand(OpNo);
 
   if (MO.isReg()) {
-    O << getRegisterName(MO.getReg());
+    printRegName(O, MO.getReg());
   } else if(MO.isExpr()) {
-    O << *MO.getExpr();
+    MO.getExpr()->print(O, &MAI);
   } else if(MO.isImm()) {
     printImmOperand(MI, OpNo, O);
   } else {
@@ -141,7 +177,7 @@ void HexagonInstPrinter::printImmOperand(const MCInst *MI, unsigned OpNo,
   const MCOperand& MO = MI->getOperand(OpNo);
 
   if(MO.isExpr()) {
-    O << *MO.getExpr();
+    MO.getExpr()->print(O, &MAI);
   } else if(MO.isImm()) {
     O << MI->getOperand(OpNo).getImm();
   } else {
@@ -188,7 +224,7 @@ void HexagonInstPrinter::printMEMriOperand(const MCInst *MI, unsigned OpNo,
   const MCOperand& MO0 = MI->getOperand(OpNo);
   const MCOperand& MO1 = MI->getOperand(OpNo + 1);
 
-  O << getRegisterName(MO0.getReg());
+  printRegName(O, MO0.getReg());
   O << " + #" << MO1.getImm();
 }
 
@@ -197,7 +233,8 @@ void HexagonInstPrinter::printFrameIndexOperand(const MCInst *MI, unsigned OpNo,
   const MCOperand& MO0 = MI->getOperand(OpNo);
   const MCOperand& MO1 = MI->getOperand(OpNo + 1);
 
-  O << getRegisterName(MO0.getReg()) << ", #" << MO1.getImm();
+  printRegName(O, MO0.getReg());
+  O << ", #" << MO1.getImm();
 }
 
 void HexagonInstPrinter::printGlobalOperand(const MCInst *MI, unsigned OpNo,
@@ -248,3 +285,17 @@ void HexagonInstPrinter::printSymbol(const MCInst *MI, unsigned OpNo,
   printOperand(MI, OpNo, O);
   O << ')';
 }
+
+void HexagonInstPrinter::printExtBrtarget(const MCInst *MI, unsigned OpNo,
+                                          raw_ostream &O) const {
+  const MCOperand &MO = MI->getOperand(OpNo);
+  const MCInstrDesc &MII = getMII().get(MI->getOpcode());
+
+  assert((isExtendable(MII.TSFlags) || isExtended(MII.TSFlags)) &&
+         "Expecting an extendable operand");
+
+  if (MO.isExpr() || isExtended(MII.TSFlags)) {
+    O << "##";
+  }
+  printOperand(MI, OpNo, O);
+}
diff --git a/lib/Target/Hexagon/MCTargetDesc/HexagonInstPrinter.h b/lib/Target/Hexagon/MCTargetDesc/HexagonInstPrinter.h
index 98fb99bb6f0..534ac237d63 100644
--- a/lib/Target/Hexagon/MCTargetDesc/HexagonInstPrinter.h
+++ b/lib/Target/Hexagon/MCTargetDesc/HexagonInstPrinter.h
@@ -18,6 +18,21 @@
 #include "llvm/MC/MCInstrInfo.h"
 
 namespace llvm {
+class HexagonAsmInstPrinter : public MCInstPrinter {
+public:
+  HexagonAsmInstPrinter(MCInstPrinter *RawPrinter);
+  void printInst(MCInst const *MI, raw_ostream &O, StringRef Annot,
+                 MCSubtargetInfo const &STI) override;
+  void printRegName(raw_ostream &O, unsigned RegNo) const override;
+  std::unique_ptr<MCInstPrinter> RawPrinter;
+};
+/// Prints bundles as a newline separated list of individual instructions
+/// Duplexes are separated by a vertical tab \v character
+/// A trailing line includes bundle properties such as endloop0/1
+///
+/// r0 = add(r1, r2)
+/// r0 = #0 \v jump 0x0
+/// :endloop0 :endloop1
   class HexagonInstPrinter : public MCInstPrinter {
   public:
     explicit HexagonInstPrinter(MCAsmInfo const &MAI,
@@ -29,7 +44,7 @@ namespace llvm {
                    const MCSubtargetInfo &STI) override;
     virtual StringRef getOpcodeName(unsigned Opcode) const;
     void printInstruction(const MCInst *MI, raw_ostream &O);
-    StringRef getRegName(unsigned RegNo) const;
+    void printRegName(raw_ostream &OS, unsigned RegNo) const override;
     static const char *getRegisterName(unsigned RegNo);
 
     void printOperand(const MCInst *MI, unsigned OpNo, raw_ostream &O) const;
@@ -56,6 +71,7 @@ namespace llvm {
     void printGlobalOperand(const MCInst *MI, unsigned OpNo, raw_ostream &O)
            const;
     void printJumpTable(const MCInst *MI, unsigned OpNo, raw_ostream &O) const;
+    void printExtBrtarget(const MCInst *MI, unsigned OpNo, raw_ostream &O) const;
 
     void printConstantPool(const MCInst *MI, unsigned OpNo,
                            raw_ostream &O) const;
@@ -73,11 +89,11 @@ namespace llvm {
     void printSymbol(const MCInst *MI, unsigned OpNo, raw_ostream &O, bool hi)
            const;
 
-    static const char PacketPadding;
-
   private:
     const MCInstrInfo &MII;
 
+    bool HasExtender;
+    void setExtender(MCInst const &MCI);
   };
 
 } // end namespace llvm
diff --git a/lib/Target/Hexagon/MCTargetDesc/HexagonMCAsmInfo.cpp b/lib/Target/Hexagon/MCTargetDesc/HexagonMCAsmInfo.cpp
index ad5e0fb15e7..51d2f1c878d 100644
--- a/lib/Target/Hexagon/MCTargetDesc/HexagonMCAsmInfo.cpp
+++ b/lib/Target/Hexagon/MCTargetDesc/HexagonMCAsmInfo.cpp
@@ -18,7 +18,7 @@ using namespace llvm;
 // Pin the vtable to this file.
 void HexagonMCAsmInfo::anchor() {}
 
-HexagonMCAsmInfo::HexagonMCAsmInfo(StringRef TT) {
+HexagonMCAsmInfo::HexagonMCAsmInfo(const Triple &TT) {
   Data16bitsDirective = "\t.half\t";
   Data32bitsDirective = "\t.word\t";
   Data64bitsDirective = nullptr;  // .xword is only supported by V9.
diff --git a/lib/Target/Hexagon/MCTargetDesc/HexagonMCAsmInfo.h b/lib/Target/Hexagon/MCTargetDesc/HexagonMCAsmInfo.h
index ab18f0b37ba..dc070699478 100644
--- a/lib/Target/Hexagon/MCTargetDesc/HexagonMCAsmInfo.h
+++ b/lib/Target/Hexagon/MCTargetDesc/HexagonMCAsmInfo.h
@@ -18,10 +18,12 @@
 #include "llvm/MC/MCAsmInfoELF.h"
 
 namespace llvm {
+  class Triple;
+
   class HexagonMCAsmInfo : public MCAsmInfoELF {
     void anchor() override;
   public:
-    explicit HexagonMCAsmInfo(StringRef TT);
+    explicit HexagonMCAsmInfo(const Triple &TT);
   };
 
 } // namespace llvm
diff --git a/lib/Target/Hexagon/MCTargetDesc/HexagonMCCodeEmitter.cpp b/lib/Target/Hexagon/MCTargetDesc/HexagonMCCodeEmitter.cpp
index eac7d6d2c4e..9fc4e2aeaba 100644
--- a/lib/Target/Hexagon/MCTargetDesc/HexagonMCCodeEmitter.cpp
+++ b/lib/Target/Hexagon/MCTargetDesc/HexagonMCCodeEmitter.cpp
@@ -9,6 +9,7 @@
 
 #include "Hexagon.h"
 #include "MCTargetDesc/HexagonBaseInfo.h"
+#include "MCTargetDesc/HexagonFixupKinds.h"
 #include "MCTargetDesc/HexagonMCCodeEmitter.h"
 #include "MCTargetDesc/HexagonMCInstrInfo.h"
 #include "MCTargetDesc/HexagonMCTargetDesc.h"
@@ -21,6 +22,7 @@
 #include "llvm/MC/MCRegisterInfo.h"
 #include "llvm/MC/MCSubtargetInfo.h"
 #include "llvm/Support/Debug.h"
+#include "llvm/Support/EndianStream.h"
 #include "llvm/Support/raw_ostream.h"
 
 #define DEBUG_TYPE "mccodeemitter"
@@ -30,39 +32,678 @@ using namespace Hexagon;
 
 STATISTIC(MCNumEmitted, "Number of MC instructions emitted");
 
-namespace {
-/// \brief 10.6 Instruction Packets
-/// Possible values for instruction packet parse field.
-enum class ParseField { duplex = 0x0, last0 = 0x1, last1 = 0x2, end = 0x3 };
-/// \brief Returns the packet bits based on instruction position.
-uint32_t getPacketBits(MCInst const &HMI) {
-  unsigned const ParseFieldOffset = 14;
-  ParseField Field = HexagonMCInstrInfo::isPacketEnd(HMI) ? ParseField::end : ParseField::last0;
-  return static_cast <uint32_t> (Field) << ParseFieldOffset;
-}
-void emitLittleEndian(uint64_t Binary, raw_ostream &OS) {
-  OS << static_cast<uint8_t>((Binary >> 0x00) & 0xff);
-  OS << static_cast<uint8_t>((Binary >> 0x08) & 0xff);
-  OS << static_cast<uint8_t>((Binary >> 0x10) & 0xff);
-  OS << static_cast<uint8_t>((Binary >> 0x18) & 0xff);
-}
-}
-
 HexagonMCCodeEmitter::HexagonMCCodeEmitter(MCInstrInfo const &aMII,
                                            MCContext &aMCT)
-    : MCT(aMCT), MCII(aMII) {}
+    : MCT(aMCT), MCII(aMII), Addend(new unsigned(0)),
+      Extended(new bool(false)), CurrentBundle(new MCInst const *) {}
+
+uint32_t HexagonMCCodeEmitter::parseBits(size_t Instruction, size_t Last,
+                                         MCInst const &MCB,
+                                         MCInst const &MCI) const {
+  bool Duplex = HexagonMCInstrInfo::isDuplex(MCII, MCI);
+  if (Instruction == 0) {
+    if (HexagonMCInstrInfo::isInnerLoop(MCB)) {
+      assert(!Duplex);
+      assert(Instruction != Last);
+      return HexagonII::INST_PARSE_LOOP_END;
+    }
+  }
+  if (Instruction == 1) {
+    if (HexagonMCInstrInfo::isOuterLoop(MCB)) {
+      assert(!Duplex);
+      assert(Instruction != Last);
+      return HexagonII::INST_PARSE_LOOP_END;
+    }
+  }
+  if (Duplex) {
+    assert(Instruction == Last);
+    return HexagonII::INST_PARSE_DUPLEX;
+  }
+  if(Instruction == Last)
+    return HexagonII::INST_PARSE_PACKET_END;
+  return HexagonII::INST_PARSE_NOT_END;
+}
 
-void HexagonMCCodeEmitter::EncodeInstruction(MCInst const &MI, raw_ostream &OS,
+void HexagonMCCodeEmitter::encodeInstruction(MCInst const &MI, raw_ostream &OS,
                                              SmallVectorImpl<MCFixup> &Fixups,
                                              MCSubtargetInfo const &STI) const {
-  uint64_t Binary = getBinaryCodeForInstr(MI, Fixups, STI) | getPacketBits(MI);
-  assert(HexagonMCInstrInfo::getDesc(MCII, MI).getSize() == 4 &&
-         "All instructions should be 32bit");
-  (void)&MCII;
-  emitLittleEndian(Binary, OS);
+  MCInst &HMB = const_cast<MCInst &>(MI);
+
+  assert(HexagonMCInstrInfo::isBundle(HMB));
+  DEBUG(dbgs() << "Encoding bundle\n";);
+  *Addend = 0;
+  *Extended = false;
+  *CurrentBundle = &MI;
+  size_t Instruction = 0;
+  size_t Last = HexagonMCInstrInfo::bundleSize(HMB) - 1;
+  for (auto &I : HexagonMCInstrInfo::bundleInstructions(HMB)) {
+    MCInst &HMI = const_cast<MCInst &>(*I.getInst());
+    EncodeSingleInstruction(HMI, OS, Fixups, STI,
+                            parseBits(Instruction, Last, HMB, HMI),
+                            Instruction);
+    *Extended = HexagonMCInstrInfo::isImmext(HMI);
+    *Addend += HEXAGON_INSTR_SIZE;
+    ++Instruction;
+  }
+  return;
+}
+
+/// EncodeSingleInstruction - Emit a single
+void HexagonMCCodeEmitter::EncodeSingleInstruction(
+    const MCInst &MI, raw_ostream &OS, SmallVectorImpl<MCFixup> &Fixups,
+    const MCSubtargetInfo &STI, uint32_t Parse, size_t Index) const {
+  MCInst HMB = MI;
+  assert(!HexagonMCInstrInfo::isBundle(HMB));
+  uint64_t Binary;
+
+  // Pseudo instructions don't get encoded and shouldn't be here
+  // in the first place!
+  assert(!HexagonMCInstrInfo::getDesc(MCII, HMB).isPseudo() &&
+         "pseudo-instruction found");
+  DEBUG(dbgs() << "Encoding insn"
+                  " `" << HexagonMCInstrInfo::getName(MCII, HMB) << "'"
+                                                                    "\n");
+
+  if (HexagonMCInstrInfo::isNewValue(MCII, HMB)) {
+    // Calculate the new value distance to the associated producer
+    MCOperand &MCO =
+        HMB.getOperand(HexagonMCInstrInfo::getNewValueOp(MCII, HMB));
+    unsigned SOffset = 0;
+    unsigned Register = MCO.getReg();
+    unsigned Register1;
+    auto Instructions = HexagonMCInstrInfo::bundleInstructions(**CurrentBundle);
+    auto i = Instructions.begin() + Index - 1;
+    for (;; --i) {
+      assert(i != Instructions.begin() - 1 && "Couldn't find producer");
+      MCInst const &Inst = *i->getInst();
+      if (HexagonMCInstrInfo::isImmext(Inst))
+        continue;
+      ++SOffset;
+      Register1 =
+          HexagonMCInstrInfo::hasNewValue(MCII, Inst)
+              ? HexagonMCInstrInfo::getNewValueOperand(MCII, Inst).getReg()
+              : static_cast<unsigned>(Hexagon::NoRegister);
+      if (Register != Register1)
+        // This isn't the register we're looking for
+        continue;
+      if (!HexagonMCInstrInfo::isPredicated(MCII, Inst))
+        // Producer is unpredicated
+        break;
+      assert(HexagonMCInstrInfo::isPredicated(MCII, HMB) &&
+             "Unpredicated consumer depending on predicated producer");
+      if (HexagonMCInstrInfo::isPredicatedTrue(MCII, Inst) ==
+          HexagonMCInstrInfo::isPredicatedTrue(MCII, HMB))
+        // Producer predicate sense matched ours
+        break;
+    }
+    // Hexagon PRM 10.11 Construct Nt from distance
+    unsigned Offset = SOffset;
+    Offset <<= 1;
+    MCO.setReg(Offset + Hexagon::R0);
+  }
+
+  Binary = getBinaryCodeForInstr(HMB, Fixups, STI);
+  // Check for unimplemented instructions. Immediate extenders
+  // are encoded as zero, so they need to be accounted for.
+  if ((!Binary) &&
+      ((HMB.getOpcode() != DuplexIClass0) && (HMB.getOpcode() != A4_ext) &&
+       (HMB.getOpcode() != A4_ext_b) && (HMB.getOpcode() != A4_ext_c) &&
+       (HMB.getOpcode() != A4_ext_g))) {
+    // Use a A2_nop for unimplemented instructions.
+    DEBUG(dbgs() << "Unimplemented inst: "
+                    " `" << HexagonMCInstrInfo::getName(MCII, HMB) << "'"
+                                                                      "\n");
+    llvm_unreachable("Unimplemented Instruction");
+  }
+  Binary |= Parse;
+
+  // if we need to emit a duplexed instruction
+  if (HMB.getOpcode() >= Hexagon::DuplexIClass0 &&
+      HMB.getOpcode() <= Hexagon::DuplexIClassF) {
+    assert(Parse == HexagonII::INST_PARSE_DUPLEX &&
+           "Emitting duplex without duplex parse bits");
+    unsigned dupIClass;
+    switch (HMB.getOpcode()) {
+    case Hexagon::DuplexIClass0:
+      dupIClass = 0;
+      break;
+    case Hexagon::DuplexIClass1:
+      dupIClass = 1;
+      break;
+    case Hexagon::DuplexIClass2:
+      dupIClass = 2;
+      break;
+    case Hexagon::DuplexIClass3:
+      dupIClass = 3;
+      break;
+    case Hexagon::DuplexIClass4:
+      dupIClass = 4;
+      break;
+    case Hexagon::DuplexIClass5:
+      dupIClass = 5;
+      break;
+    case Hexagon::DuplexIClass6:
+      dupIClass = 6;
+      break;
+    case Hexagon::DuplexIClass7:
+      dupIClass = 7;
+      break;
+    case Hexagon::DuplexIClass8:
+      dupIClass = 8;
+      break;
+    case Hexagon::DuplexIClass9:
+      dupIClass = 9;
+      break;
+    case Hexagon::DuplexIClassA:
+      dupIClass = 10;
+      break;
+    case Hexagon::DuplexIClassB:
+      dupIClass = 11;
+      break;
+    case Hexagon::DuplexIClassC:
+      dupIClass = 12;
+      break;
+    case Hexagon::DuplexIClassD:
+      dupIClass = 13;
+      break;
+    case Hexagon::DuplexIClassE:
+      dupIClass = 14;
+      break;
+    case Hexagon::DuplexIClassF:
+      dupIClass = 15;
+      break;
+    default:
+      llvm_unreachable("Unimplemented DuplexIClass");
+      break;
+    }
+    // 29 is the bit position.
+    // 0b1110 =0xE bits are masked off and down shifted by 1 bit.
+    // Last bit is moved to bit position 13
+    Binary = ((dupIClass & 0xE) << (29 - 1)) | ((dupIClass & 0x1) << 13);
+
+    const MCInst *subInst0 = HMB.getOperand(0).getInst();
+    const MCInst *subInst1 = HMB.getOperand(1).getInst();
+
+    // get subinstruction slot 0
+    unsigned subInstSlot0Bits = getBinaryCodeForInstr(*subInst0, Fixups, STI);
+    // get subinstruction slot 1
+    unsigned subInstSlot1Bits = getBinaryCodeForInstr(*subInst1, Fixups, STI);
+
+    Binary |= subInstSlot0Bits | (subInstSlot1Bits << 16);
+  }
+  support::endian::Writer<support::little>(OS).write<uint32_t>(Binary);
   ++MCNumEmitted;
 }
 
+static Hexagon::Fixups getFixupNoBits(MCInstrInfo const &MCII, const MCInst &MI,
+                                      const MCOperand &MO,
+                                      const MCSymbolRefExpr::VariantKind kind) {
+  const MCInstrDesc &MCID = HexagonMCInstrInfo::getDesc(MCII, MI);
+  unsigned insnType = llvm::HexagonMCInstrInfo::getType(MCII, MI);
+
+  if (insnType == HexagonII::TypePREFIX) {
+    switch (kind) {
+    case llvm::MCSymbolRefExpr::VK_GOTOFF:
+      return Hexagon::fixup_Hexagon_GOTREL_32_6_X;
+    case llvm::MCSymbolRefExpr::VK_GOT:
+      return Hexagon::fixup_Hexagon_GOT_32_6_X;
+    case llvm::MCSymbolRefExpr::VK_TPREL:
+      return Hexagon::fixup_Hexagon_TPREL_32_6_X;
+    case llvm::MCSymbolRefExpr::VK_DTPREL:
+      return Hexagon::fixup_Hexagon_DTPREL_32_6_X;
+    case llvm::MCSymbolRefExpr::VK_Hexagon_GD_GOT:
+      return Hexagon::fixup_Hexagon_GD_GOT_32_6_X;
+    case llvm::MCSymbolRefExpr::VK_Hexagon_LD_GOT:
+      return Hexagon::fixup_Hexagon_LD_GOT_32_6_X;
+    case llvm::MCSymbolRefExpr::VK_Hexagon_IE:
+      return Hexagon::fixup_Hexagon_IE_32_6_X;
+    case llvm::MCSymbolRefExpr::VK_Hexagon_IE_GOT:
+      return Hexagon::fixup_Hexagon_IE_GOT_32_6_X;
+    default:
+      if (MCID.isBranch())
+        return Hexagon::fixup_Hexagon_B32_PCREL_X;
+      else
+        return Hexagon::fixup_Hexagon_32_6_X;
+    }
+  } else if (MCID.isBranch())
+    return (Hexagon::fixup_Hexagon_B13_PCREL);
+
+  switch (MCID.getOpcode()) {
+  case Hexagon::HI:
+  case Hexagon::A2_tfrih:
+    switch (kind) {
+    case llvm::MCSymbolRefExpr::VK_GOT:
+      return Hexagon::fixup_Hexagon_GOT_HI16;
+    case llvm::MCSymbolRefExpr::VK_GOTOFF:
+      return Hexagon::fixup_Hexagon_GOTREL_HI16;
+    case llvm::MCSymbolRefExpr::VK_Hexagon_GD_GOT:
+      return Hexagon::fixup_Hexagon_GD_GOT_HI16;
+    case llvm::MCSymbolRefExpr::VK_Hexagon_LD_GOT:
+      return Hexagon::fixup_Hexagon_LD_GOT_HI16;
+    case llvm::MCSymbolRefExpr::VK_Hexagon_IE:
+      return Hexagon::fixup_Hexagon_IE_HI16;
+    case llvm::MCSymbolRefExpr::VK_Hexagon_IE_GOT:
+      return Hexagon::fixup_Hexagon_IE_GOT_HI16;
+    case llvm::MCSymbolRefExpr::VK_TPREL:
+      return Hexagon::fixup_Hexagon_TPREL_HI16;
+    case llvm::MCSymbolRefExpr::VK_DTPREL:
+      return Hexagon::fixup_Hexagon_DTPREL_HI16;
+    default:
+      return Hexagon::fixup_Hexagon_HI16;
+    }
+
+  case Hexagon::LO:
+  case Hexagon::A2_tfril:
+    switch (kind) {
+    case llvm::MCSymbolRefExpr::VK_GOT:
+      return Hexagon::fixup_Hexagon_GOT_LO16;
+    case llvm::MCSymbolRefExpr::VK_GOTOFF:
+      return Hexagon::fixup_Hexagon_GOTREL_LO16;
+    case llvm::MCSymbolRefExpr::VK_Hexagon_GD_GOT:
+      return Hexagon::fixup_Hexagon_GD_GOT_LO16;
+    case llvm::MCSymbolRefExpr::VK_Hexagon_LD_GOT:
+      return Hexagon::fixup_Hexagon_LD_GOT_LO16;
+    case llvm::MCSymbolRefExpr::VK_Hexagon_IE:
+      return Hexagon::fixup_Hexagon_IE_LO16;
+    case llvm::MCSymbolRefExpr::VK_Hexagon_IE_GOT:
+      return Hexagon::fixup_Hexagon_IE_GOT_LO16;
+    case llvm::MCSymbolRefExpr::VK_TPREL:
+      return Hexagon::fixup_Hexagon_TPREL_LO16;
+    case llvm::MCSymbolRefExpr::VK_DTPREL:
+      return Hexagon::fixup_Hexagon_DTPREL_LO16;
+    default:
+      return Hexagon::fixup_Hexagon_LO16;
+    }
+
+  // The only relocs left should be GP relative:
+  default:
+    if (MCID.mayStore() || MCID.mayLoad()) {
+      for (const uint16_t *ImpUses = MCID.getImplicitUses(); *ImpUses;
+           ++ImpUses) {
+        if (*ImpUses == Hexagon::GP) {
+          switch (HexagonMCInstrInfo::getAccessSize(MCII, MI)) {
+          case HexagonII::MemAccessSize::ByteAccess:
+            return fixup_Hexagon_GPREL16_0;
+          case HexagonII::MemAccessSize::HalfWordAccess:
+            return fixup_Hexagon_GPREL16_1;
+          case HexagonII::MemAccessSize::WordAccess:
+            return fixup_Hexagon_GPREL16_2;
+          case HexagonII::MemAccessSize::DoubleWordAccess:
+            return fixup_Hexagon_GPREL16_3;
+          default:
+            llvm_unreachable("unhandled fixup");
+          }
+        }
+      }
+    } else
+      llvm_unreachable("unhandled fixup");
+  }
+
+  return LastTargetFixupKind;
+}
+
+namespace llvm {
+extern const MCInstrDesc HexagonInsts[];
+}
+
+namespace {
+  bool isPCRel (unsigned Kind) {
+    switch(Kind){
+    case fixup_Hexagon_B22_PCREL:
+    case fixup_Hexagon_B15_PCREL:
+    case fixup_Hexagon_B7_PCREL:
+    case fixup_Hexagon_B13_PCREL:
+    case fixup_Hexagon_B9_PCREL:
+    case fixup_Hexagon_B32_PCREL_X:
+    case fixup_Hexagon_B22_PCREL_X:
+    case fixup_Hexagon_B15_PCREL_X:
+    case fixup_Hexagon_B13_PCREL_X:
+    case fixup_Hexagon_B9_PCREL_X:
+    case fixup_Hexagon_B7_PCREL_X:
+    case fixup_Hexagon_32_PCREL:
+    case fixup_Hexagon_PLT_B22_PCREL:
+    case fixup_Hexagon_GD_PLT_B22_PCREL:
+    case fixup_Hexagon_LD_PLT_B22_PCREL:
+    case fixup_Hexagon_6_PCREL_X:
+      return true;
+    default:
+      return false;
+    }
+  }
+}
+
+unsigned HexagonMCCodeEmitter::getExprOpValue(const MCInst &MI,
+                                              const MCOperand &MO,
+                                              const MCExpr *ME,
+                                              SmallVectorImpl<MCFixup> &Fixups,
+                                              const MCSubtargetInfo &STI) const
+
+{
+  int64_t Res;
+
+  if (ME->evaluateAsAbsolute(Res))
+    return Res;
+
+  MCExpr::ExprKind MK = ME->getKind();
+  if (MK == MCExpr::Constant) {
+    return cast<MCConstantExpr>(ME)->getValue();
+  }
+  if (MK == MCExpr::Binary) {
+    unsigned Res;
+    Res = getExprOpValue(MI, MO, cast<MCBinaryExpr>(ME)->getLHS(), Fixups, STI);
+    Res +=
+        getExprOpValue(MI, MO, cast<MCBinaryExpr>(ME)->getRHS(), Fixups, STI);
+    return 0;
+  }
+
+  assert(MK == MCExpr::SymbolRef);
+
+  Hexagon::Fixups FixupKind =
+      Hexagon::Fixups(Hexagon::fixup_Hexagon_TPREL_LO16);
+  const MCSymbolRefExpr *MCSRE = static_cast<const MCSymbolRefExpr *>(ME);
+  const MCInstrDesc &MCID = HexagonMCInstrInfo::getDesc(MCII, MI);
+  unsigned bits = HexagonMCInstrInfo::getExtentBits(MCII, MI) -
+                  HexagonMCInstrInfo::getExtentAlignment(MCII, MI);
+  const MCSymbolRefExpr::VariantKind kind = MCSRE->getKind();
+
+  DEBUG(dbgs() << "----------------------------------------\n");
+  DEBUG(dbgs() << "Opcode Name: " << HexagonMCInstrInfo::getName(MCII, MI)
+               << "\n");
+  DEBUG(dbgs() << "Opcode: " << MCID.getOpcode() << "\n");
+  DEBUG(dbgs() << "Relocation bits: " << bits << "\n");
+  DEBUG(dbgs() << "Addend: " << *Addend << "\n");
+  DEBUG(dbgs() << "----------------------------------------\n");
+
+  switch (bits) {
+  default:
+    DEBUG(dbgs() << "unrecognized bit count of " << bits << '\n');
+    break;
+
+  case 32:
+    switch (kind) {
+    case llvm::MCSymbolRefExpr::VK_Hexagon_PCREL:
+      FixupKind = Hexagon::fixup_Hexagon_32_PCREL;
+      break;
+    case llvm::MCSymbolRefExpr::VK_GOT:
+      FixupKind = *Extended ? Hexagon::fixup_Hexagon_GOT_32_6_X
+                            : Hexagon::fixup_Hexagon_GOT_32;
+      break;
+    case llvm::MCSymbolRefExpr::VK_GOTOFF:
+      FixupKind = *Extended ? Hexagon::fixup_Hexagon_GOTREL_32_6_X
+                            : Hexagon::fixup_Hexagon_GOTREL_32;
+      break;
+    case llvm::MCSymbolRefExpr::VK_Hexagon_GD_GOT:
+      FixupKind = *Extended ? Hexagon::fixup_Hexagon_GD_GOT_32_6_X
+                            : Hexagon::fixup_Hexagon_GD_GOT_32;
+      break;
+    case llvm::MCSymbolRefExpr::VK_Hexagon_LD_GOT:
+      FixupKind = *Extended ? Hexagon::fixup_Hexagon_LD_GOT_32_6_X
+                            : Hexagon::fixup_Hexagon_LD_GOT_32;
+      break;
+    case llvm::MCSymbolRefExpr::VK_Hexagon_IE:
+      FixupKind = *Extended ? Hexagon::fixup_Hexagon_IE_32_6_X
+                            : Hexagon::fixup_Hexagon_IE_32;
+      break;
+    case llvm::MCSymbolRefExpr::VK_Hexagon_IE_GOT:
+      FixupKind = *Extended ? Hexagon::fixup_Hexagon_IE_GOT_32_6_X
+                            : Hexagon::fixup_Hexagon_IE_GOT_32;
+      break;
+    case llvm::MCSymbolRefExpr::VK_TPREL:
+      FixupKind = *Extended ? Hexagon::fixup_Hexagon_TPREL_32_6_X
+                            : Hexagon::fixup_Hexagon_TPREL_32;
+      break;
+    case llvm::MCSymbolRefExpr::VK_DTPREL:
+      FixupKind = *Extended ? Hexagon::fixup_Hexagon_DTPREL_32_6_X
+                            : Hexagon::fixup_Hexagon_DTPREL_32;
+      break;
+    default:
+      FixupKind =
+          *Extended ? Hexagon::fixup_Hexagon_32_6_X : Hexagon::fixup_Hexagon_32;
+      break;
+    }
+    break;
+
+  case 22:
+    switch (kind) {
+    case llvm::MCSymbolRefExpr::VK_Hexagon_GD_PLT:
+      FixupKind = Hexagon::fixup_Hexagon_GD_PLT_B22_PCREL;
+      break;
+    case llvm::MCSymbolRefExpr::VK_Hexagon_LD_PLT:
+      FixupKind = Hexagon::fixup_Hexagon_LD_PLT_B22_PCREL;
+      break;
+    default:
+      if (MCID.isBranch() || MCID.isCall()) {
+        FixupKind = *Extended ? Hexagon::fixup_Hexagon_B22_PCREL_X
+                              : Hexagon::fixup_Hexagon_B22_PCREL;
+      } else {
+        errs() << "unrecognized relocation, bits: " << bits << "\n";
+        errs() << "name = " << HexagonMCInstrInfo::getName(MCII, MI) << "\n";
+      }
+      break;
+    }
+    break;
+
+  case 16:
+    if (*Extended) {
+      switch (kind) {
+      default:
+        FixupKind = Hexagon::fixup_Hexagon_16_X;
+        break;
+      case llvm::MCSymbolRefExpr::VK_GOT:
+        FixupKind = Hexagon::fixup_Hexagon_GOT_16_X;
+        break;
+      case llvm::MCSymbolRefExpr::VK_GOTOFF:
+        FixupKind = Hexagon::fixup_Hexagon_GOTREL_16_X;
+        break;
+      case llvm::MCSymbolRefExpr::VK_Hexagon_GD_GOT:
+        FixupKind = Hexagon::fixup_Hexagon_GD_GOT_16_X;
+        break;
+      case llvm::MCSymbolRefExpr::VK_Hexagon_LD_GOT:
+        FixupKind = Hexagon::fixup_Hexagon_LD_GOT_16_X;
+        break;
+      case llvm::MCSymbolRefExpr::VK_Hexagon_IE:
+        FixupKind = Hexagon::fixup_Hexagon_IE_16_X;
+        break;
+      case llvm::MCSymbolRefExpr::VK_Hexagon_IE_GOT:
+        FixupKind = Hexagon::fixup_Hexagon_IE_GOT_16_X;
+        break;
+      case llvm::MCSymbolRefExpr::VK_TPREL:
+        FixupKind = Hexagon::fixup_Hexagon_TPREL_16_X;
+        break;
+      case llvm::MCSymbolRefExpr::VK_DTPREL:
+        FixupKind = Hexagon::fixup_Hexagon_DTPREL_16_X;
+        break;
+      }
+    } else
+      switch (kind) {
+      default:
+        errs() << "unrecognized relocation, bits " << bits << "\n";
+        errs() << "name = " << HexagonMCInstrInfo::getName(MCII, MI) << "\n";
+        break;
+      case llvm::MCSymbolRefExpr::VK_GOTOFF:
+        if ((MCID.getOpcode() == Hexagon::HI) ||
+            (MCID.getOpcode() == Hexagon::LO_H))
+          FixupKind = Hexagon::fixup_Hexagon_GOTREL_HI16;
+        else
+          FixupKind = Hexagon::fixup_Hexagon_GOTREL_LO16;
+        break;
+      case llvm::MCSymbolRefExpr::VK_Hexagon_GPREL:
+        FixupKind = Hexagon::fixup_Hexagon_GPREL16_0;
+        break;
+      case llvm::MCSymbolRefExpr::VK_Hexagon_LO16:
+        FixupKind = Hexagon::fixup_Hexagon_LO16;
+        break;
+      case llvm::MCSymbolRefExpr::VK_Hexagon_HI16:
+        FixupKind = Hexagon::fixup_Hexagon_HI16;
+        break;
+      case llvm::MCSymbolRefExpr::VK_Hexagon_GD_GOT:
+        FixupKind = Hexagon::fixup_Hexagon_GD_GOT_16;
+        break;
+      case llvm::MCSymbolRefExpr::VK_Hexagon_LD_GOT:
+        FixupKind = Hexagon::fixup_Hexagon_LD_GOT_16;
+        break;
+      case llvm::MCSymbolRefExpr::VK_Hexagon_IE_GOT:
+        FixupKind = Hexagon::fixup_Hexagon_IE_GOT_16;
+        break;
+      case llvm::MCSymbolRefExpr::VK_TPREL:
+        FixupKind = Hexagon::fixup_Hexagon_TPREL_16;
+        break;
+      case llvm::MCSymbolRefExpr::VK_DTPREL:
+        FixupKind = Hexagon::fixup_Hexagon_DTPREL_16;
+        break;
+      }
+    break;
+
+  case 15:
+    if (MCID.isBranch() || MCID.isCall())
+      FixupKind = *Extended ? Hexagon::fixup_Hexagon_B15_PCREL_X
+                            : Hexagon::fixup_Hexagon_B15_PCREL;
+    break;
+
+  case 13:
+    if (MCID.isBranch())
+      FixupKind = Hexagon::fixup_Hexagon_B13_PCREL;
+    else {
+      errs() << "unrecognized relocation, bits " << bits << "\n";
+      errs() << "name = " << HexagonMCInstrInfo::getName(MCII, MI) << "\n";
+    }
+    break;
+
+  case 12:
+    if (*Extended)
+      switch (kind) {
+      default:
+        FixupKind = Hexagon::fixup_Hexagon_12_X;
+        break;
+      // There isn't a GOT_12_X, both 11_X and 16_X resolve to 6/26
+      case llvm::MCSymbolRefExpr::VK_GOT:
+        FixupKind = Hexagon::fixup_Hexagon_GOT_16_X;
+        break;
+      case llvm::MCSymbolRefExpr::VK_GOTOFF:
+        FixupKind = Hexagon::fixup_Hexagon_GOTREL_16_X;
+        break;
+      }
+    else {
+      errs() << "unrecognized relocation, bits " << bits << "\n";
+      errs() << "name = " << HexagonMCInstrInfo::getName(MCII, MI) << "\n";
+    }
+    break;
+
+  case 11:
+    if (*Extended)
+      switch (kind) {
+      default:
+        FixupKind = Hexagon::fixup_Hexagon_11_X;
+        break;
+      case llvm::MCSymbolRefExpr::VK_GOT:
+        FixupKind = Hexagon::fixup_Hexagon_GOT_11_X;
+        break;
+      case llvm::MCSymbolRefExpr::VK_GOTOFF:
+        FixupKind = Hexagon::fixup_Hexagon_GOTREL_11_X;
+        break;
+      case llvm::MCSymbolRefExpr::VK_Hexagon_GD_GOT:
+        FixupKind = Hexagon::fixup_Hexagon_GD_GOT_11_X;
+        break;
+      case llvm::MCSymbolRefExpr::VK_Hexagon_LD_GOT:
+        FixupKind = Hexagon::fixup_Hexagon_LD_GOT_11_X;
+        break;
+      case llvm::MCSymbolRefExpr::VK_Hexagon_IE_GOT:
+        FixupKind = Hexagon::fixup_Hexagon_IE_GOT_11_X;
+        break;
+      case llvm::MCSymbolRefExpr::VK_TPREL:
+        FixupKind = Hexagon::fixup_Hexagon_TPREL_11_X;
+        break;
+      case llvm::MCSymbolRefExpr::VK_DTPREL:
+        FixupKind = Hexagon::fixup_Hexagon_DTPREL_11_X;
+        break;
+      }
+    else {
+      errs() << "unrecognized relocation, bits " << bits << "\n";
+      errs() << "name = " << HexagonMCInstrInfo::getName(MCII, MI) << "\n";
+    }
+    break;
+
+  case 10:
+    if (*Extended)
+      FixupKind = Hexagon::fixup_Hexagon_10_X;
+    break;
+
+  case 9:
+    if (MCID.isBranch() ||
+        (llvm::HexagonMCInstrInfo::getType(MCII, MI) == HexagonII::TypeCR))
+      FixupKind = *Extended ? Hexagon::fixup_Hexagon_B9_PCREL_X
+                            : Hexagon::fixup_Hexagon_B9_PCREL;
+    else if (*Extended)
+      FixupKind = Hexagon::fixup_Hexagon_9_X;
+    else {
+      errs() << "unrecognized relocation, bits " << bits << "\n";
+      errs() << "name = " << HexagonMCInstrInfo::getName(MCII, MI) << "\n";
+    }
+    break;
+
+  case 8:
+    if (*Extended)
+      FixupKind = Hexagon::fixup_Hexagon_8_X;
+    else {
+      errs() << "unrecognized relocation, bits " << bits << "\n";
+      errs() << "name = " << HexagonMCInstrInfo::getName(MCII, MI) << "\n";
+    }
+    break;
+
+  case 7:
+    if (MCID.isBranch() ||
+        (llvm::HexagonMCInstrInfo::getType(MCII, MI) == HexagonII::TypeCR))
+      FixupKind = *Extended ? Hexagon::fixup_Hexagon_B7_PCREL_X
+                            : Hexagon::fixup_Hexagon_B7_PCREL;
+    else if (*Extended)
+      FixupKind = Hexagon::fixup_Hexagon_7_X;
+    else {
+      errs() << "unrecognized relocation, bits " << bits << "\n";
+      errs() << "name = " << HexagonMCInstrInfo::getName(MCII, MI) << "\n";
+    }
+    break;
+
+  case 6:
+    if (*Extended) {
+      switch (kind) {
+      default:
+        FixupKind = Hexagon::fixup_Hexagon_6_X;
+        break;
+      case llvm::MCSymbolRefExpr::VK_Hexagon_PCREL:
+        FixupKind = Hexagon::fixup_Hexagon_6_PCREL_X;
+        break;
+      // This is part of an extender, GOT_11 is a
+      // Word32_U6 unsigned/truncated reloc.
+      case llvm::MCSymbolRefExpr::VK_GOT:
+        FixupKind = Hexagon::fixup_Hexagon_GOT_11_X;
+        break;
+      case llvm::MCSymbolRefExpr::VK_GOTOFF:
+        FixupKind = Hexagon::fixup_Hexagon_GOTREL_11_X;
+        break;
+      }
+    } else {
+      errs() << "unrecognized relocation, bits " << bits << "\n";
+      errs() << "name = " << HexagonMCInstrInfo::getName(MCII, MI) << "\n";
+    }
+    break;
+
+  case 0:
+    FixupKind = getFixupNoBits(MCII, MI, MO, kind);
+    break;
+  }
+
+  MCExpr const *FixupExpression = (*Addend > 0 && isPCRel(FixupKind)) ?
+    MCBinaryExpr::createAdd(MO.getExpr(),
+                            MCConstantExpr::create(*Addend, MCT), MCT) :
+    MO.getExpr();
+
+  MCFixup fixup = MCFixup::create(*Addend, FixupExpression, 
+                                  MCFixupKind(FixupKind), MI.getLoc());
+  Fixups.push_back(fixup);
+  // All of the information is in the fixup.
+  return (0);
+}
+
 unsigned
 HexagonMCCodeEmitter::getMachineOpValue(MCInst const &MI, MCOperand const &MO,
                                         SmallVectorImpl<MCFixup> &Fixups,
@@ -71,7 +712,10 @@ HexagonMCCodeEmitter::getMachineOpValue(MCInst const &MI, MCOperand const &MO,
     return MCT.getRegisterInfo()->getEncodingValue(MO.getReg());
   if (MO.isImm())
     return static_cast<unsigned>(MO.getImm());
-  llvm_unreachable("Only Immediates and Registers implemented right now");
+
+  // MO must be an ME.
+  assert(MO.isExpr());
+  return getExprOpValue(MI, MO, MO.getExpr(), Fixups, STI);
 }
 
 MCCodeEmitter *llvm::createHexagonMCCodeEmitter(MCInstrInfo const &MII,
diff --git a/lib/Target/Hexagon/MCTargetDesc/HexagonMCCodeEmitter.h b/lib/Target/Hexagon/MCTargetDesc/HexagonMCCodeEmitter.h
index 768c10ed92d..2a154da26c5 100644
--- a/lib/Target/Hexagon/MCTargetDesc/HexagonMCCodeEmitter.h
+++ b/lib/Target/Hexagon/MCTargetDesc/HexagonMCCodeEmitter.h
@@ -28,16 +28,31 @@ namespace llvm {
 class HexagonMCCodeEmitter : public MCCodeEmitter {
   MCContext &MCT;
   MCInstrInfo const &MCII;
+  std::unique_ptr<unsigned> Addend;
+  std::unique_ptr<bool> Extended;
+  std::unique_ptr<MCInst const *> CurrentBundle;
+
+  // helper routine for getMachineOpValue()
+  unsigned getExprOpValue(const MCInst &MI, const MCOperand &MO,
+                          const MCExpr *ME, SmallVectorImpl<MCFixup> &Fixups,
+                          const MCSubtargetInfo &STI) const;
 
 public:
   HexagonMCCodeEmitter(MCInstrInfo const &aMII, MCContext &aMCT);
 
-  MCSubtargetInfo const &getSubtargetInfo() const;
+  // Return parse bits for instruction `MCI' inside bundle `MCB'
+  uint32_t parseBits(size_t Instruction, size_t Last, MCInst const &MCB,
+                    MCInst const &MCI) const;
 
-  void EncodeInstruction(MCInst const &MI, raw_ostream &OS,
+  void encodeInstruction(MCInst const &MI, raw_ostream &OS,
                          SmallVectorImpl<MCFixup> &Fixups,
                          MCSubtargetInfo const &STI) const override;
 
+  void EncodeSingleInstruction(const MCInst &MI, raw_ostream &OS,
+                               SmallVectorImpl<MCFixup> &Fixups,
+                               const MCSubtargetInfo &STI,
+                               uint32_t Parse, size_t Index) const;
+
   // \brief TableGen'erated function for getting the
   // binary encoding for an instruction.
   uint64_t getBinaryCodeForInstr(MCInst const &MI,
@@ -48,10 +63,6 @@ class HexagonMCCodeEmitter : public MCCodeEmitter {
   unsigned getMachineOpValue(MCInst const &MI, MCOperand const &MO,
                              SmallVectorImpl<MCFixup> &Fixups,
                              MCSubtargetInfo const &STI) const;
-
-private:
-  HexagonMCCodeEmitter(HexagonMCCodeEmitter const &) = delete;
-  void operator=(HexagonMCCodeEmitter const &) = delete;
 }; // class HexagonMCCodeEmitter
 
 } // namespace llvm
diff --git a/lib/Target/Hexagon/MCTargetDesc/HexagonMCCompound.cpp b/lib/Target/Hexagon/MCTargetDesc/HexagonMCCompound.cpp
new file mode 100644
index 00000000000..886f8db3bc6
--- /dev/null
+++ b/lib/Target/Hexagon/MCTargetDesc/HexagonMCCompound.cpp
@@ -0,0 +1,420 @@
+
+//=== HexagonMCCompound.cpp - Hexagon Compound checker  -------===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is looks at a packet and tries to form compound insns
+//
+//===----------------------------------------------------------------------===//
+#include "Hexagon.h"
+#include "MCTargetDesc/HexagonBaseInfo.h"
+#include "MCTargetDesc/HexagonMCShuffler.h"
+#include "llvm/ADT/StringExtras.h"
+#include "llvm/MC/MCAssembler.h"
+#include "llvm/MC/MCContext.h"
+#include "llvm/MC/MCInst.h"
+#include "llvm/MC/MCSectionELF.h"
+#include "llvm/MC/MCStreamer.h"
+#include "llvm/MC/MCSymbol.h"
+#include "llvm/Support/Debug.h"
+#include "llvm/Support/raw_ostream.h"
+
+using namespace llvm;
+using namespace Hexagon;
+
+#define DEBUG_TYPE "hexagon-mccompound"
+
+enum OpcodeIndex {
+  fp0_jump_nt = 0,
+  fp0_jump_t,
+  fp1_jump_nt,
+  fp1_jump_t,
+  tp0_jump_nt,
+  tp0_jump_t,
+  tp1_jump_nt,
+  tp1_jump_t
+};
+
+static const unsigned tstBitOpcode[8] = {
+    J4_tstbit0_fp0_jump_nt, J4_tstbit0_fp0_jump_t,  J4_tstbit0_fp1_jump_nt,
+    J4_tstbit0_fp1_jump_t,  J4_tstbit0_tp0_jump_nt, J4_tstbit0_tp0_jump_t,
+    J4_tstbit0_tp1_jump_nt, J4_tstbit0_tp1_jump_t};
+static const unsigned cmpeqBitOpcode[8] = {
+    J4_cmpeq_fp0_jump_nt, J4_cmpeq_fp0_jump_t,  J4_cmpeq_fp1_jump_nt,
+    J4_cmpeq_fp1_jump_t,  J4_cmpeq_tp0_jump_nt, J4_cmpeq_tp0_jump_t,
+    J4_cmpeq_tp1_jump_nt, J4_cmpeq_tp1_jump_t};
+static const unsigned cmpgtBitOpcode[8] = {
+    J4_cmpgt_fp0_jump_nt, J4_cmpgt_fp0_jump_t,  J4_cmpgt_fp1_jump_nt,
+    J4_cmpgt_fp1_jump_t,  J4_cmpgt_tp0_jump_nt, J4_cmpgt_tp0_jump_t,
+    J4_cmpgt_tp1_jump_nt, J4_cmpgt_tp1_jump_t};
+static const unsigned cmpgtuBitOpcode[8] = {
+    J4_cmpgtu_fp0_jump_nt, J4_cmpgtu_fp0_jump_t,  J4_cmpgtu_fp1_jump_nt,
+    J4_cmpgtu_fp1_jump_t,  J4_cmpgtu_tp0_jump_nt, J4_cmpgtu_tp0_jump_t,
+    J4_cmpgtu_tp1_jump_nt, J4_cmpgtu_tp1_jump_t};
+static const unsigned cmpeqiBitOpcode[8] = {
+    J4_cmpeqi_fp0_jump_nt, J4_cmpeqi_fp0_jump_t,  J4_cmpeqi_fp1_jump_nt,
+    J4_cmpeqi_fp1_jump_t,  J4_cmpeqi_tp0_jump_nt, J4_cmpeqi_tp0_jump_t,
+    J4_cmpeqi_tp1_jump_nt, J4_cmpeqi_tp1_jump_t};
+static const unsigned cmpgtiBitOpcode[8] = {
+    J4_cmpgti_fp0_jump_nt, J4_cmpgti_fp0_jump_t,  J4_cmpgti_fp1_jump_nt,
+    J4_cmpgti_fp1_jump_t,  J4_cmpgti_tp0_jump_nt, J4_cmpgti_tp0_jump_t,
+    J4_cmpgti_tp1_jump_nt, J4_cmpgti_tp1_jump_t};
+static const unsigned cmpgtuiBitOpcode[8] = {
+    J4_cmpgtui_fp0_jump_nt, J4_cmpgtui_fp0_jump_t,  J4_cmpgtui_fp1_jump_nt,
+    J4_cmpgtui_fp1_jump_t,  J4_cmpgtui_tp0_jump_nt, J4_cmpgtui_tp0_jump_t,
+    J4_cmpgtui_tp1_jump_nt, J4_cmpgtui_tp1_jump_t};
+static const unsigned cmpeqn1BitOpcode[8] = {
+    J4_cmpeqn1_fp0_jump_nt, J4_cmpeqn1_fp0_jump_t,  J4_cmpeqn1_fp1_jump_nt,
+    J4_cmpeqn1_fp1_jump_t,  J4_cmpeqn1_tp0_jump_nt, J4_cmpeqn1_tp0_jump_t,
+    J4_cmpeqn1_tp1_jump_nt, J4_cmpeqn1_tp1_jump_t};
+static const unsigned cmpgtn1BitOpcode[8] = {
+    J4_cmpgtn1_fp0_jump_nt, J4_cmpgtn1_fp0_jump_t,  J4_cmpgtn1_fp1_jump_nt,
+    J4_cmpgtn1_fp1_jump_t,  J4_cmpgtn1_tp0_jump_nt, J4_cmpgtn1_tp0_jump_t,
+    J4_cmpgtn1_tp1_jump_nt, J4_cmpgtn1_tp1_jump_t,
+};
+
+// enum HexagonII::CompoundGroup
+namespace {
+unsigned getCompoundCandidateGroup(MCInst const &MI, bool IsExtended) {
+  unsigned DstReg, SrcReg, Src1Reg, Src2Reg;
+
+  switch (MI.getOpcode()) {
+  default:
+    return HexagonII::HCG_None;
+  //
+  // Compound pairs.
+  // "p0=cmp.eq(Rs16,Rt16); if (p0.new) jump:nt #r9:2"
+  // "Rd16=#U6 ; jump #r9:2"
+  // "Rd16=Rs16 ; jump #r9:2"
+  //
+  case Hexagon::C2_cmpeq:
+  case Hexagon::C2_cmpgt:
+  case Hexagon::C2_cmpgtu:
+    if (IsExtended)
+      return false;
+    DstReg = MI.getOperand(0).getReg();
+    Src1Reg = MI.getOperand(1).getReg();
+    Src2Reg = MI.getOperand(2).getReg();
+    if ((Hexagon::P0 == DstReg || Hexagon::P1 == DstReg) &&
+        HexagonMCInstrInfo::isIntRegForSubInst(Src1Reg) &&
+        HexagonMCInstrInfo::isIntRegForSubInst(Src2Reg))
+      return HexagonII::HCG_A;
+    break;
+  case Hexagon::C2_cmpeqi:
+  case Hexagon::C2_cmpgti:
+  case Hexagon::C2_cmpgtui:
+    if (IsExtended)
+      return false;
+    // P0 = cmp.eq(Rs,#u2)
+    DstReg = MI.getOperand(0).getReg();
+    SrcReg = MI.getOperand(1).getReg();
+    if ((Hexagon::P0 == DstReg || Hexagon::P1 == DstReg) &&
+        HexagonMCInstrInfo::isIntRegForSubInst(SrcReg) &&
+        MI.getOperand(2).isImm() && ((isUInt<5>(MI.getOperand(2).getImm())) ||
+                                     (MI.getOperand(2).getImm() == -1)))
+      return HexagonII::HCG_A;
+    break;
+  case Hexagon::A2_tfr:
+    if (IsExtended)
+      return false;
+    // Rd = Rs
+    DstReg = MI.getOperand(0).getReg();
+    SrcReg = MI.getOperand(1).getReg();
+    if (HexagonMCInstrInfo::isIntRegForSubInst(DstReg) &&
+        HexagonMCInstrInfo::isIntRegForSubInst(SrcReg))
+      return HexagonII::HCG_A;
+    break;
+  case Hexagon::A2_tfrsi:
+    if (IsExtended)
+      return false;
+    // Rd = #u6
+    DstReg = MI.getOperand(0).getReg();
+    if (MI.getOperand(1).isImm() && MI.getOperand(1).getImm() <= 63 &&
+        MI.getOperand(1).getImm() >= 0 &&
+        HexagonMCInstrInfo::isIntRegForSubInst(DstReg))
+      return HexagonII::HCG_A;
+    break;
+  case Hexagon::S2_tstbit_i:
+    if (IsExtended)
+      return false;
+    DstReg = MI.getOperand(0).getReg();
+    Src1Reg = MI.getOperand(1).getReg();
+    if ((Hexagon::P0 == DstReg || Hexagon::P1 == DstReg) &&
+        MI.getOperand(2).isImm() &&
+        HexagonMCInstrInfo::isIntRegForSubInst(Src1Reg) &&
+        (MI.getOperand(2).getImm() == 0))
+      return HexagonII::HCG_A;
+    break;
+  // The fact that .new form is used pretty much guarantees
+  // that predicate register will match. Nevertheless,
+  // there could be some false positives without additional
+  // checking.
+  case Hexagon::J2_jumptnew:
+  case Hexagon::J2_jumpfnew:
+  case Hexagon::J2_jumptnewpt:
+  case Hexagon::J2_jumpfnewpt:
+    Src1Reg = MI.getOperand(0).getReg();
+    if (Hexagon::P0 == Src1Reg || Hexagon::P1 == Src1Reg)
+      return HexagonII::HCG_B;
+    break;
+  // Transfer and jump:
+  // Rd=#U6 ; jump #r9:2
+  // Rd=Rs ; jump #r9:2
+  // Do not test for jump range here.
+  case Hexagon::J2_jump:
+  case Hexagon::RESTORE_DEALLOC_RET_JMP_V4:
+    return HexagonII::HCG_C;
+    break;
+  }
+
+  return HexagonII::HCG_None;
+}
+}
+
+/// getCompoundOp - Return the index from 0-7 into the above opcode lists.
+namespace {
+unsigned getCompoundOp(MCInst const &HMCI) {
+  const MCOperand &Predicate = HMCI.getOperand(0);
+  unsigned PredReg = Predicate.getReg();
+
+  assert((PredReg == Hexagon::P0) || (PredReg == Hexagon::P1) ||
+         (PredReg == Hexagon::P2) || (PredReg == Hexagon::P3));
+
+  switch (HMCI.getOpcode()) {
+  default:
+    llvm_unreachable("Expected match not found.\n");
+    break;
+  case Hexagon::J2_jumpfnew:
+    return (PredReg == Hexagon::P0) ? fp0_jump_nt : fp1_jump_nt;
+  case Hexagon::J2_jumpfnewpt:
+    return (PredReg == Hexagon::P0) ? fp0_jump_t : fp1_jump_t;
+  case Hexagon::J2_jumptnew:
+    return (PredReg == Hexagon::P0) ? tp0_jump_nt : tp1_jump_nt;
+  case Hexagon::J2_jumptnewpt:
+    return (PredReg == Hexagon::P0) ? tp0_jump_t : tp1_jump_t;
+  }
+}
+}
+
+namespace {
+MCInst *getCompoundInsn(MCContext &Context, MCInst const &L, MCInst const &R) {
+  MCInst *CompoundInsn = 0;
+  unsigned compoundOpcode;
+  MCOperand Rs, Rt;
+
+  switch (L.getOpcode()) {
+  default:
+    DEBUG(dbgs() << "Possible compound ignored\n");
+    return CompoundInsn;
+
+  case Hexagon::A2_tfrsi:
+    Rt = L.getOperand(0);
+    compoundOpcode = J4_jumpseti;
+    CompoundInsn = new (Context) MCInst;
+    CompoundInsn->setOpcode(compoundOpcode);
+
+    CompoundInsn->addOperand(Rt);
+    CompoundInsn->addOperand(L.getOperand(1)); // Immediate
+    CompoundInsn->addOperand(R.getOperand(0)); // Jump target
+    break;
+
+  case Hexagon::A2_tfr:
+    Rt = L.getOperand(0);
+    Rs = L.getOperand(1);
+
+    compoundOpcode = J4_jumpsetr;
+    CompoundInsn = new (Context) MCInst;
+    CompoundInsn->setOpcode(compoundOpcode);
+    CompoundInsn->addOperand(Rt);
+    CompoundInsn->addOperand(Rs);
+    CompoundInsn->addOperand(R.getOperand(0)); // Jump target.
+
+    break;
+
+  case Hexagon::C2_cmpeq:
+    DEBUG(dbgs() << "CX: C2_cmpeq\n");
+    Rs = L.getOperand(1);
+    Rt = L.getOperand(2);
+
+    compoundOpcode = cmpeqBitOpcode[getCompoundOp(R)];
+    CompoundInsn = new (Context) MCInst;
+    CompoundInsn->setOpcode(compoundOpcode);
+    CompoundInsn->addOperand(Rs);
+    CompoundInsn->addOperand(Rt);
+    CompoundInsn->addOperand(R.getOperand(1));
+    break;
+
+  case Hexagon::C2_cmpgt:
+    DEBUG(dbgs() << "CX: C2_cmpgt\n");
+    Rs = L.getOperand(1);
+    Rt = L.getOperand(2);
+
+    compoundOpcode = cmpgtBitOpcode[getCompoundOp(R)];
+    CompoundInsn = new (Context) MCInst;
+    CompoundInsn->setOpcode(compoundOpcode);
+    CompoundInsn->addOperand(Rs);
+    CompoundInsn->addOperand(Rt);
+    CompoundInsn->addOperand(R.getOperand(1));
+    break;
+
+  case Hexagon::C2_cmpgtu:
+    DEBUG(dbgs() << "CX: C2_cmpgtu\n");
+    Rs = L.getOperand(1);
+    Rt = L.getOperand(2);
+
+    compoundOpcode = cmpgtuBitOpcode[getCompoundOp(R)];
+    CompoundInsn = new (Context) MCInst;
+    CompoundInsn->setOpcode(compoundOpcode);
+    CompoundInsn->addOperand(Rs);
+    CompoundInsn->addOperand(Rt);
+    CompoundInsn->addOperand(R.getOperand(1));
+    break;
+
+  case Hexagon::C2_cmpeqi:
+    DEBUG(dbgs() << "CX: C2_cmpeqi\n");
+    if (L.getOperand(2).getImm() == -1)
+      compoundOpcode = cmpeqn1BitOpcode[getCompoundOp(R)];
+    else
+      compoundOpcode = cmpeqiBitOpcode[getCompoundOp(R)];
+
+    Rs = L.getOperand(1);
+    CompoundInsn = new (Context) MCInst;
+    CompoundInsn->setOpcode(compoundOpcode);
+    CompoundInsn->addOperand(Rs);
+    if (L.getOperand(2).getImm() != -1)
+      CompoundInsn->addOperand(L.getOperand(2));
+    CompoundInsn->addOperand(R.getOperand(1));
+    break;
+
+  case Hexagon::C2_cmpgti:
+    DEBUG(dbgs() << "CX: C2_cmpgti\n");
+    if (L.getOperand(2).getImm() == -1)
+      compoundOpcode = cmpgtn1BitOpcode[getCompoundOp(R)];
+    else
+      compoundOpcode = cmpgtiBitOpcode[getCompoundOp(R)];
+
+    Rs = L.getOperand(1);
+    CompoundInsn = new (Context) MCInst;
+    CompoundInsn->setOpcode(compoundOpcode);
+    CompoundInsn->addOperand(Rs);
+    if (L.getOperand(2).getImm() != -1)
+      CompoundInsn->addOperand(L.getOperand(2));
+    CompoundInsn->addOperand(R.getOperand(1));
+    break;
+
+  case Hexagon::C2_cmpgtui:
+    DEBUG(dbgs() << "CX: C2_cmpgtui\n");
+    Rs = L.getOperand(1);
+    compoundOpcode = cmpgtuiBitOpcode[getCompoundOp(R)];
+    CompoundInsn = new (Context) MCInst;
+    CompoundInsn->setOpcode(compoundOpcode);
+    CompoundInsn->addOperand(Rs);
+    CompoundInsn->addOperand(L.getOperand(2));
+    CompoundInsn->addOperand(R.getOperand(1));
+    break;
+
+  case Hexagon::S2_tstbit_i:
+    DEBUG(dbgs() << "CX: S2_tstbit_i\n");
+    Rs = L.getOperand(1);
+    compoundOpcode = tstBitOpcode[getCompoundOp(R)];
+    CompoundInsn = new (Context) MCInst;
+    CompoundInsn->setOpcode(compoundOpcode);
+    CompoundInsn->addOperand(Rs);
+    CompoundInsn->addOperand(R.getOperand(1));
+    break;
+  }
+
+  return CompoundInsn;
+}
+}
+
+/// Non-Symmetrical. See if these two instructions are fit for compound pair.
+namespace {
+bool isOrderedCompoundPair(MCInst const &MIa, bool IsExtendedA,
+                           MCInst const &MIb, bool IsExtendedB) {
+  unsigned MIaG = getCompoundCandidateGroup(MIa, IsExtendedA);
+  unsigned MIbG = getCompoundCandidateGroup(MIb, IsExtendedB);
+  // We have two candidates - check that this is the same register
+  // we are talking about.
+  unsigned Opca = MIa.getOpcode();
+  if (MIaG == HexagonII::HCG_A && MIbG == HexagonII::HCG_C &&
+      (Opca == Hexagon::A2_tfr || Opca == Hexagon::A2_tfrsi))
+    return true;
+  return ((MIaG == HexagonII::HCG_A && MIbG == HexagonII::HCG_B) &&
+          (MIa.getOperand(0).getReg() == MIb.getOperand(0).getReg()));
+}
+}
+
+namespace {
+bool lookForCompound(MCInstrInfo const &MCII, MCContext &Context, MCInst &MCI) {
+  assert(HexagonMCInstrInfo::isBundle(MCI));
+  bool JExtended = false;
+  for (MCInst::iterator J =
+           MCI.begin() + HexagonMCInstrInfo::bundleInstructionsOffset;
+       J != MCI.end(); ++J) {
+    MCInst const *JumpInst = J->getInst();
+    if (HexagonMCInstrInfo::isImmext(*JumpInst)) {
+      JExtended = true;
+      continue;
+    }
+    if (llvm::HexagonMCInstrInfo::getType(MCII, *JumpInst) ==
+        HexagonII::TypeJ) {
+      // Try to pair with another insn (B)undled with jump.
+      bool BExtended = false;
+      for (MCInst::iterator B =
+               MCI.begin() + HexagonMCInstrInfo::bundleInstructionsOffset;
+           B != MCI.end(); ++B) {
+        MCInst const *Inst = B->getInst();
+        if (JumpInst == Inst)
+          continue;
+        if (HexagonMCInstrInfo::isImmext(*Inst)) {
+          BExtended = true;
+          continue;
+        }
+        DEBUG(dbgs() << "J,B: " << JumpInst->getOpcode() << ","
+                     << Inst->getOpcode() << "\n");
+        if (isOrderedCompoundPair(*Inst, BExtended, *JumpInst, JExtended)) {
+          MCInst *CompoundInsn = getCompoundInsn(Context, *Inst, *JumpInst);
+          if (CompoundInsn) {
+            DEBUG(dbgs() << "B: " << Inst->getOpcode() << ","
+                         << JumpInst->getOpcode() << " Compounds to "
+                         << CompoundInsn->getOpcode() << "\n");
+            J->setInst(CompoundInsn);
+            MCI.erase(B);
+            return true;
+          }
+        }
+        BExtended = false;
+      }
+    }
+    JExtended = false;
+  }
+  return false;
+}
+}
+
+/// tryCompound - Given a bundle check for compound insns when one
+/// is found update the contents fo the bundle with the compound insn.
+/// If a compound instruction is found then the bundle will have one
+/// additional slot.
+void HexagonMCInstrInfo::tryCompound(MCInstrInfo const &MCII,
+                                     MCContext &Context, MCInst &MCI) {
+  assert(MCI.getOpcode() == Hexagon::BUNDLE &&
+         "Non-Bundle where Bundle expected");
+
+  // By definition a compound must have 2 insn.
+  if (MCI.size() < 2)
+    return;
+
+  // Look for compounds until none are found, only update the bundle when
+  // a compound is found.
+  while (lookForCompound(MCII, Context, MCI))
+    ;
+
+  return;
+}
diff --git a/lib/Target/Hexagon/MCTargetDesc/HexagonMCDuplexInfo.cpp b/lib/Target/Hexagon/MCTargetDesc/HexagonMCDuplexInfo.cpp
new file mode 100644
index 00000000000..7e9247cef6a
--- /dev/null
+++ b/lib/Target/Hexagon/MCTargetDesc/HexagonMCDuplexInfo.cpp
@@ -0,0 +1,1099 @@
+//===----- HexagonMCDuplexInfo.cpp - Instruction bundle checking ----------===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This implements duplexing of instructions to reduce code size
+//
+//===----------------------------------------------------------------------===//
+
+#include "HexagonBaseInfo.h"
+#include "MCTargetDesc/HexagonMCInstrInfo.h"
+
+#include "llvm/ADT/SmallVector.h"
+#include "llvm/Support/Debug.h"
+#include "llvm/Support/raw_ostream.h"
+
+#include <map>
+
+using namespace llvm;
+using namespace Hexagon;
+
+#define DEBUG_TYPE "hexagon-mcduplex-info"
+
+// pair table of subInstructions with opcodes
+static std::pair<unsigned, unsigned> opcodeData[] = {
+    std::make_pair((unsigned)V4_SA1_addi, 0),
+    std::make_pair((unsigned)V4_SA1_addrx, 6144),
+    std::make_pair((unsigned)V4_SA1_addsp, 3072),
+    std::make_pair((unsigned)V4_SA1_and1, 4608),
+    std::make_pair((unsigned)V4_SA1_clrf, 6768),
+    std::make_pair((unsigned)V4_SA1_clrfnew, 6736),
+    std::make_pair((unsigned)V4_SA1_clrt, 6752),
+    std::make_pair((unsigned)V4_SA1_clrtnew, 6720),
+    std::make_pair((unsigned)V4_SA1_cmpeqi, 6400),
+    std::make_pair((unsigned)V4_SA1_combine0i, 7168),
+    std::make_pair((unsigned)V4_SA1_combine1i, 7176),
+    std::make_pair((unsigned)V4_SA1_combine2i, 7184),
+    std::make_pair((unsigned)V4_SA1_combine3i, 7192),
+    std::make_pair((unsigned)V4_SA1_combinerz, 7432),
+    std::make_pair((unsigned)V4_SA1_combinezr, 7424),
+    std::make_pair((unsigned)V4_SA1_dec, 4864),
+    std::make_pair((unsigned)V4_SA1_inc, 4352),
+    std::make_pair((unsigned)V4_SA1_seti, 2048),
+    std::make_pair((unsigned)V4_SA1_setin1, 6656),
+    std::make_pair((unsigned)V4_SA1_sxtb, 5376),
+    std::make_pair((unsigned)V4_SA1_sxth, 5120),
+    std::make_pair((unsigned)V4_SA1_tfr, 4096),
+    std::make_pair((unsigned)V4_SA1_zxtb, 5888),
+    std::make_pair((unsigned)V4_SA1_zxth, 5632),
+    std::make_pair((unsigned)V4_SL1_loadri_io, 0),
+    std::make_pair((unsigned)V4_SL1_loadrub_io, 4096),
+    std::make_pair((unsigned)V4_SL2_deallocframe, 7936),
+    std::make_pair((unsigned)V4_SL2_jumpr31, 8128),
+    std::make_pair((unsigned)V4_SL2_jumpr31_f, 8133),
+    std::make_pair((unsigned)V4_SL2_jumpr31_fnew, 8135),
+    std::make_pair((unsigned)V4_SL2_jumpr31_t, 8132),
+    std::make_pair((unsigned)V4_SL2_jumpr31_tnew, 8134),
+    std::make_pair((unsigned)V4_SL2_loadrb_io, 4096),
+    std::make_pair((unsigned)V4_SL2_loadrd_sp, 7680),
+    std::make_pair((unsigned)V4_SL2_loadrh_io, 0),
+    std::make_pair((unsigned)V4_SL2_loadri_sp, 7168),
+    std::make_pair((unsigned)V4_SL2_loadruh_io, 2048),
+    std::make_pair((unsigned)V4_SL2_return, 8000),
+    std::make_pair((unsigned)V4_SL2_return_f, 8005),
+    std::make_pair((unsigned)V4_SL2_return_fnew, 8007),
+    std::make_pair((unsigned)V4_SL2_return_t, 8004),
+    std::make_pair((unsigned)V4_SL2_return_tnew, 8006),
+    std::make_pair((unsigned)V4_SS1_storeb_io, 4096),
+    std::make_pair((unsigned)V4_SS1_storew_io, 0),
+    std::make_pair((unsigned)V4_SS2_allocframe, 7168),
+    std::make_pair((unsigned)V4_SS2_storebi0, 4608),
+    std::make_pair((unsigned)V4_SS2_storebi1, 4864),
+    std::make_pair((unsigned)V4_SS2_stored_sp, 2560),
+    std::make_pair((unsigned)V4_SS2_storeh_io, 0),
+    std::make_pair((unsigned)V4_SS2_storew_sp, 2048),
+    std::make_pair((unsigned)V4_SS2_storewi0, 4096),
+    std::make_pair((unsigned)V4_SS2_storewi1, 4352)};
+
+static std::map<unsigned, unsigned>
+    subinstOpcodeMap(opcodeData,
+                     opcodeData + sizeof(opcodeData) / sizeof(opcodeData[0]));
+
+bool HexagonMCInstrInfo::isDuplexPairMatch(unsigned Ga, unsigned Gb) {
+  switch (Ga) {
+  case HexagonII::HSIG_None:
+  default:
+    return false;
+  case HexagonII::HSIG_L1:
+    return (Gb == HexagonII::HSIG_L1 || Gb == HexagonII::HSIG_A);
+  case HexagonII::HSIG_L2:
+    return (Gb == HexagonII::HSIG_L1 || Gb == HexagonII::HSIG_L2 ||
+            Gb == HexagonII::HSIG_A);
+  case HexagonII::HSIG_S1:
+    return (Gb == HexagonII::HSIG_L1 || Gb == HexagonII::HSIG_L2 ||
+            Gb == HexagonII::HSIG_S1 || Gb == HexagonII::HSIG_A);
+  case HexagonII::HSIG_S2:
+    return (Gb == HexagonII::HSIG_L1 || Gb == HexagonII::HSIG_L2 ||
+            Gb == HexagonII::HSIG_S1 || Gb == HexagonII::HSIG_S2 ||
+            Gb == HexagonII::HSIG_A);
+  case HexagonII::HSIG_A:
+    return (Gb == HexagonII::HSIG_A);
+  case HexagonII::HSIG_Compound:
+    return (Gb == HexagonII::HSIG_Compound);
+  }
+  return false;
+}
+
+unsigned HexagonMCInstrInfo::iClassOfDuplexPair(unsigned Ga, unsigned Gb) {
+  switch (Ga) {
+  case HexagonII::HSIG_None:
+  default:
+    break;
+  case HexagonII::HSIG_L1:
+    switch (Gb) {
+    default:
+      break;
+    case HexagonII::HSIG_L1:
+      return 0;
+    case HexagonII::HSIG_A:
+      return 0x4;
+    }
+  case HexagonII::HSIG_L2:
+    switch (Gb) {
+    default:
+      break;
+    case HexagonII::HSIG_L1:
+      return 0x1;
+    case HexagonII::HSIG_L2:
+      return 0x2;
+    case HexagonII::HSIG_A:
+      return 0x5;
+    }
+  case HexagonII::HSIG_S1:
+    switch (Gb) {
+    default:
+      break;
+    case HexagonII::HSIG_L1:
+      return 0x8;
+    case HexagonII::HSIG_L2:
+      return 0x9;
+    case HexagonII::HSIG_S1:
+      return 0xA;
+    case HexagonII::HSIG_A:
+      return 0x6;
+    }
+  case HexagonII::HSIG_S2:
+    switch (Gb) {
+    default:
+      break;
+    case HexagonII::HSIG_L1:
+      return 0xC;
+    case HexagonII::HSIG_L2:
+      return 0xD;
+    case HexagonII::HSIG_S1:
+      return 0xB;
+    case HexagonII::HSIG_S2:
+      return 0xE;
+    case HexagonII::HSIG_A:
+      return 0x7;
+    }
+  case HexagonII::HSIG_A:
+    switch (Gb) {
+    default:
+      break;
+    case HexagonII::HSIG_A:
+      return 0x3;
+    }
+  case HexagonII::HSIG_Compound:
+    switch (Gb) {
+    case HexagonII::HSIG_Compound:
+      return 0xFFFFFFFF;
+    }
+  }
+  return 0xFFFFFFFF;
+}
+
+unsigned HexagonMCInstrInfo::getDuplexCandidateGroup(MCInst const &MCI) {
+  unsigned DstReg, PredReg, SrcReg, Src1Reg, Src2Reg;
+
+  switch (MCI.getOpcode()) {
+  default:
+    return HexagonII::HSIG_None;
+  //
+  // Group L1:
+  //
+  // Rd = memw(Rs+#u4:2)
+  // Rd = memub(Rs+#u4:0)
+  case Hexagon::L2_loadri_io:
+    DstReg = MCI.getOperand(0).getReg();
+    SrcReg = MCI.getOperand(1).getReg();
+    // Special case this one from Group L2.
+    // Rd = memw(r29+#u5:2)
+    if (HexagonMCInstrInfo::isIntRegForSubInst(DstReg)) {
+      if (HexagonMCInstrInfo::isIntReg(SrcReg) && Hexagon::R29 == SrcReg &&
+          MCI.getOperand(2).isImm() &&
+          isShiftedUInt<5, 2>(MCI.getOperand(2).getImm())) {
+        return HexagonII::HSIG_L2;
+      }
+      // Rd = memw(Rs+#u4:2)
+      if (HexagonMCInstrInfo::isIntRegForSubInst(SrcReg) &&
+          (MCI.getOperand(2).isImm() &&
+           isShiftedUInt<4, 2>(MCI.getOperand(2).getImm()))) {
+        return HexagonII::HSIG_L1;
+      }
+    }
+    break;
+  case Hexagon::L2_loadrub_io:
+    // Rd = memub(Rs+#u4:0)
+    DstReg = MCI.getOperand(0).getReg();
+    SrcReg = MCI.getOperand(1).getReg();
+    if (HexagonMCInstrInfo::isIntRegForSubInst(DstReg) &&
+        HexagonMCInstrInfo::isIntRegForSubInst(SrcReg) &&
+        MCI.getOperand(2).isImm() && isUInt<4>(MCI.getOperand(2).getImm())) {
+      return HexagonII::HSIG_L1;
+    }
+    break;
+  //
+  // Group L2:
+  //
+  // Rd = memh/memuh(Rs+#u3:1)
+  // Rd = memb(Rs+#u3:0)
+  // Rd = memw(r29+#u5:2) - Handled above.
+  // Rdd = memd(r29+#u5:3)
+  // deallocframe
+  // [if ([!]p0[.new])] dealloc_return
+  // [if ([!]p0[.new])] jumpr r31
+  case Hexagon::L2_loadrh_io:
+  case Hexagon::L2_loadruh_io:
+    // Rd = memh/memuh(Rs+#u3:1)
+    DstReg = MCI.getOperand(0).getReg();
+    SrcReg = MCI.getOperand(1).getReg();
+    if (HexagonMCInstrInfo::isIntRegForSubInst(DstReg) &&
+        HexagonMCInstrInfo::isIntRegForSubInst(SrcReg) &&
+        MCI.getOperand(2).isImm() &&
+        isShiftedUInt<3, 1>(MCI.getOperand(2).getImm())) {
+      return HexagonII::HSIG_L2;
+    }
+    break;
+  case Hexagon::L2_loadrb_io:
+    // Rd = memb(Rs+#u3:0)
+    DstReg = MCI.getOperand(0).getReg();
+    SrcReg = MCI.getOperand(1).getReg();
+    if (HexagonMCInstrInfo::isIntRegForSubInst(DstReg) &&
+        HexagonMCInstrInfo::isIntRegForSubInst(SrcReg) &&
+        MCI.getOperand(2).isImm() && isUInt<3>(MCI.getOperand(2).getImm())) {
+      return HexagonII::HSIG_L2;
+    }
+    break;
+  case Hexagon::L2_loadrd_io:
+    // Rdd = memd(r29+#u5:3)
+    DstReg = MCI.getOperand(0).getReg();
+    SrcReg = MCI.getOperand(1).getReg();
+    if (HexagonMCInstrInfo::isDblRegForSubInst(DstReg) &&
+        HexagonMCInstrInfo::isIntReg(SrcReg) && Hexagon::R29 == SrcReg &&
+        MCI.getOperand(2).isImm() &&
+        isShiftedUInt<5, 3>(MCI.getOperand(2).getImm())) {
+      return HexagonII::HSIG_L2;
+    }
+    break;
+
+  case Hexagon::L4_return:
+
+  case Hexagon::L2_deallocframe:
+
+    return HexagonII::HSIG_L2;
+  case Hexagon::EH_RETURN_JMPR:
+
+  case Hexagon::J2_jumpr:
+  case Hexagon::JMPret:
+    // jumpr r31
+    // Actual form JMPR %PC<imp-def>, %R31<imp-use>, %R0<imp-use,internal>.
+    DstReg = MCI.getOperand(0).getReg();
+    if (Hexagon::R31 == DstReg) {
+      return HexagonII::HSIG_L2;
+    }
+    break;
+
+  case Hexagon::J2_jumprt:
+  case Hexagon::J2_jumprf:
+  case Hexagon::J2_jumprtnew:
+  case Hexagon::J2_jumprfnew:
+  case Hexagon::JMPrett:
+  case Hexagon::JMPretf:
+  case Hexagon::JMPrettnew:
+  case Hexagon::JMPretfnew:
+  case Hexagon::JMPrettnewpt:
+  case Hexagon::JMPretfnewpt:
+    DstReg = MCI.getOperand(1).getReg();
+    SrcReg = MCI.getOperand(0).getReg();
+    // [if ([!]p0[.new])] jumpr r31
+    if ((HexagonMCInstrInfo::isPredReg(SrcReg) && (Hexagon::P0 == SrcReg)) &&
+        (Hexagon::R31 == DstReg)) {
+      return HexagonII::HSIG_L2;
+    }
+    break;
+  case Hexagon::L4_return_t:
+
+  case Hexagon::L4_return_f:
+
+  case Hexagon::L4_return_tnew_pnt:
+
+  case Hexagon::L4_return_fnew_pnt:
+
+  case Hexagon::L4_return_tnew_pt:
+
+  case Hexagon::L4_return_fnew_pt:
+    // [if ([!]p0[.new])] dealloc_return
+    SrcReg = MCI.getOperand(0).getReg();
+    if (Hexagon::P0 == SrcReg) {
+      return HexagonII::HSIG_L2;
+    }
+    break;
+  //
+  // Group S1:
+  //
+  // memw(Rs+#u4:2) = Rt
+  // memb(Rs+#u4:0) = Rt
+  case Hexagon::S2_storeri_io:
+    // Special case this one from Group S2.
+    // memw(r29+#u5:2) = Rt
+    Src1Reg = MCI.getOperand(0).getReg();
+    Src2Reg = MCI.getOperand(2).getReg();
+    if (HexagonMCInstrInfo::isIntReg(Src1Reg) &&
+        HexagonMCInstrInfo::isIntRegForSubInst(Src2Reg) &&
+        Hexagon::R29 == Src1Reg && MCI.getOperand(1).isImm() &&
+        isShiftedUInt<5, 2>(MCI.getOperand(1).getImm())) {
+      return HexagonII::HSIG_S2;
+    }
+    // memw(Rs+#u4:2) = Rt
+    if (HexagonMCInstrInfo::isIntRegForSubInst(Src1Reg) &&
+        HexagonMCInstrInfo::isIntRegForSubInst(Src2Reg) &&
+        MCI.getOperand(1).isImm() &&
+        isShiftedUInt<4, 2>(MCI.getOperand(1).getImm())) {
+      return HexagonII::HSIG_S1;
+    }
+    break;
+  case Hexagon::S2_storerb_io:
+    // memb(Rs+#u4:0) = Rt
+    Src1Reg = MCI.getOperand(0).getReg();
+    Src2Reg = MCI.getOperand(2).getReg();
+    if (HexagonMCInstrInfo::isIntRegForSubInst(Src1Reg) &&
+        HexagonMCInstrInfo::isIntRegForSubInst(Src2Reg) &&
+        MCI.getOperand(1).isImm() && isUInt<4>(MCI.getOperand(1).getImm())) {
+      return HexagonII::HSIG_S1;
+    }
+    break;
+  //
+  // Group S2:
+  //
+  // memh(Rs+#u3:1) = Rt
+  // memw(r29+#u5:2) = Rt
+  // memd(r29+#s6:3) = Rtt
+  // memw(Rs+#u4:2) = #U1
+  // memb(Rs+#u4) = #U1
+  // allocframe(#u5:3)
+  case Hexagon::S2_storerh_io:
+    // memh(Rs+#u3:1) = Rt
+    Src1Reg = MCI.getOperand(0).getReg();
+    Src2Reg = MCI.getOperand(2).getReg();
+    if (HexagonMCInstrInfo::isIntRegForSubInst(Src1Reg) &&
+        HexagonMCInstrInfo::isIntRegForSubInst(Src2Reg) &&
+        MCI.getOperand(1).isImm() &&
+        isShiftedUInt<3, 1>(MCI.getOperand(1).getImm())) {
+      return HexagonII::HSIG_S2;
+    }
+    break;
+  case Hexagon::S2_storerd_io:
+    // memd(r29+#s6:3) = Rtt
+    Src1Reg = MCI.getOperand(0).getReg();
+    Src2Reg = MCI.getOperand(2).getReg();
+    if (HexagonMCInstrInfo::isDblRegForSubInst(Src2Reg) &&
+        HexagonMCInstrInfo::isIntReg(Src1Reg) && Hexagon::R29 == Src1Reg &&
+        MCI.getOperand(1).isImm() &&
+        isShiftedInt<6, 3>(MCI.getOperand(1).getImm())) {
+      return HexagonII::HSIG_S2;
+    }
+    break;
+  case Hexagon::S4_storeiri_io:
+    // memw(Rs+#u4:2) = #U1
+    Src1Reg = MCI.getOperand(0).getReg();
+    if (HexagonMCInstrInfo::isIntRegForSubInst(Src1Reg) &&
+        MCI.getOperand(1).isImm() &&
+        isShiftedUInt<4, 2>(MCI.getOperand(1).getImm()) &&
+        MCI.getOperand(2).isImm() && isUInt<1>(MCI.getOperand(2).getImm())) {
+      return HexagonII::HSIG_S2;
+    }
+    break;
+  case Hexagon::S4_storeirb_io:
+    // memb(Rs+#u4) = #U1
+    Src1Reg = MCI.getOperand(0).getReg();
+    if (HexagonMCInstrInfo::isIntRegForSubInst(Src1Reg) &&
+        MCI.getOperand(1).isImm() && isUInt<4>(MCI.getOperand(1).getImm()) &&
+        MCI.getOperand(2).isImm() && isUInt<1>(MCI.getOperand(2).getImm())) {
+      return HexagonII::HSIG_S2;
+    }
+    break;
+  case Hexagon::S2_allocframe:
+    if (MCI.getOperand(0).isImm() &&
+        isShiftedUInt<5, 3>(MCI.getOperand(0).getImm())) {
+      return HexagonII::HSIG_S2;
+    }
+    break;
+  //
+  // Group A:
+  //
+  // Rx = add(Rx,#s7)
+  // Rd = Rs
+  // Rd = #u6
+  // Rd = #-1
+  // if ([!]P0[.new]) Rd = #0
+  // Rd = add(r29,#u6:2)
+  // Rx = add(Rx,Rs)
+  // P0 = cmp.eq(Rs,#u2)
+  // Rdd = combine(#0,Rs)
+  // Rdd = combine(Rs,#0)
+  // Rdd = combine(#u2,#U2)
+  // Rd = add(Rs,#1)
+  // Rd = add(Rs,#-1)
+  // Rd = sxth/sxtb/zxtb/zxth(Rs)
+  // Rd = and(Rs,#1)
+  case Hexagon::A2_addi:
+    DstReg = MCI.getOperand(0).getReg();
+    SrcReg = MCI.getOperand(1).getReg();
+    if (HexagonMCInstrInfo::isIntRegForSubInst(DstReg)) {
+      // Rd = add(r29,#u6:2)
+      if (HexagonMCInstrInfo::isIntReg(SrcReg) && Hexagon::R29 == SrcReg &&
+          MCI.getOperand(2).isImm() &&
+          isShiftedUInt<6, 2>(MCI.getOperand(2).getImm())) {
+        return HexagonII::HSIG_A;
+      }
+      // Rx = add(Rx,#s7)
+      if (DstReg == SrcReg) {
+        return HexagonII::HSIG_A;
+      }
+      // Rd = add(Rs,#1)
+      // Rd = add(Rs,#-1)
+      if (HexagonMCInstrInfo::isIntRegForSubInst(SrcReg) &&
+          MCI.getOperand(2).isImm() && ((MCI.getOperand(2).getImm() == 1) ||
+                                        (MCI.getOperand(2).getImm() == -1))) {
+        return HexagonII::HSIG_A;
+      }
+    }
+    break;
+  case Hexagon::A2_add:
+    // Rx = add(Rx,Rs)
+    DstReg = MCI.getOperand(0).getReg();
+    Src1Reg = MCI.getOperand(1).getReg();
+    Src2Reg = MCI.getOperand(2).getReg();
+    if (HexagonMCInstrInfo::isIntRegForSubInst(DstReg) && (DstReg == Src1Reg) &&
+        HexagonMCInstrInfo::isIntRegForSubInst(Src2Reg)) {
+      return HexagonII::HSIG_A;
+    }
+    break;
+  case Hexagon::A2_andir:
+    DstReg = MCI.getOperand(0).getReg();
+    SrcReg = MCI.getOperand(1).getReg();
+    if (HexagonMCInstrInfo::isIntRegForSubInst(DstReg) &&
+        HexagonMCInstrInfo::isIntRegForSubInst(SrcReg) &&
+        MCI.getOperand(2).isImm() && ((MCI.getOperand(2).getImm() == 1) ||
+                                      (MCI.getOperand(2).getImm() == 255))) {
+      return HexagonII::HSIG_A;
+    }
+    break;
+  case Hexagon::A2_tfr:
+    // Rd = Rs
+    DstReg = MCI.getOperand(0).getReg();
+    SrcReg = MCI.getOperand(1).getReg();
+    if (HexagonMCInstrInfo::isIntRegForSubInst(DstReg) &&
+        HexagonMCInstrInfo::isIntRegForSubInst(SrcReg)) {
+      return HexagonII::HSIG_A;
+    }
+    break;
+  case Hexagon::A2_tfrsi:
+    DstReg = MCI.getOperand(0).getReg();
+
+    if (HexagonMCInstrInfo::isIntRegForSubInst(DstReg)) {
+      return HexagonII::HSIG_A;
+    }
+    break;
+  case Hexagon::C2_cmoveit:
+  case Hexagon::C2_cmovenewit:
+  case Hexagon::C2_cmoveif:
+  case Hexagon::C2_cmovenewif:
+    // if ([!]P0[.new]) Rd = #0
+    // Actual form:
+    // %R16<def> = C2_cmovenewit %P0<internal>, 0, %R16<imp-use,undef>;
+    DstReg = MCI.getOperand(0).getReg();  // Rd
+    PredReg = MCI.getOperand(1).getReg(); // P0
+    if (HexagonMCInstrInfo::isIntRegForSubInst(DstReg) &&
+        Hexagon::P0 == PredReg && MCI.getOperand(2).isImm() &&
+        MCI.getOperand(2).getImm() == 0) {
+      return HexagonII::HSIG_A;
+    }
+    break;
+  case Hexagon::C2_cmpeqi:
+    // P0 = cmp.eq(Rs,#u2)
+    DstReg = MCI.getOperand(0).getReg();
+    SrcReg = MCI.getOperand(1).getReg();
+    if (Hexagon::P0 == DstReg &&
+        HexagonMCInstrInfo::isIntRegForSubInst(SrcReg) &&
+        MCI.getOperand(2).isImm() && isUInt<2>(MCI.getOperand(2).getImm())) {
+      return HexagonII::HSIG_A;
+    }
+    break;
+  case Hexagon::A2_combineii:
+  case Hexagon::A4_combineii:
+    // Rdd = combine(#u2,#U2)
+    DstReg = MCI.getOperand(0).getReg();
+    if (HexagonMCInstrInfo::isDblRegForSubInst(DstReg) &&
+        // TODO: Handle Globals/Symbols
+        (MCI.getOperand(1).isImm() && isUInt<2>(MCI.getOperand(1).getImm())) &&
+        ((MCI.getOperand(2).isImm() &&
+          isUInt<2>(MCI.getOperand(2).getImm())))) {
+      return HexagonII::HSIG_A;
+    }
+    break;
+  case Hexagon::A4_combineri:
+    // Rdd = combine(Rs,#0)
+    DstReg = MCI.getOperand(0).getReg();
+    SrcReg = MCI.getOperand(1).getReg();
+    if (HexagonMCInstrInfo::isDblRegForSubInst(DstReg) &&
+        HexagonMCInstrInfo::isIntRegForSubInst(SrcReg) &&
+        (MCI.getOperand(2).isImm() && MCI.getOperand(2).getImm() == 0)) {
+      return HexagonII::HSIG_A;
+    }
+    break;
+  case Hexagon::A4_combineir:
+    // Rdd = combine(#0,Rs)
+    DstReg = MCI.getOperand(0).getReg();
+    SrcReg = MCI.getOperand(2).getReg();
+    if (HexagonMCInstrInfo::isDblRegForSubInst(DstReg) &&
+        HexagonMCInstrInfo::isIntRegForSubInst(SrcReg) &&
+        (MCI.getOperand(1).isImm() && MCI.getOperand(1).getImm() == 0)) {
+      return HexagonII::HSIG_A;
+    }
+    break;
+  case Hexagon::A2_sxtb:
+  case Hexagon::A2_sxth:
+  case Hexagon::A2_zxtb:
+  case Hexagon::A2_zxth:
+    // Rd = sxth/sxtb/zxtb/zxth(Rs)
+    DstReg = MCI.getOperand(0).getReg();
+    SrcReg = MCI.getOperand(1).getReg();
+    if (HexagonMCInstrInfo::isIntRegForSubInst(DstReg) &&
+        HexagonMCInstrInfo::isIntRegForSubInst(SrcReg)) {
+      return HexagonII::HSIG_A;
+    }
+    break;
+  }
+
+  return HexagonII::HSIG_None;
+}
+
+bool HexagonMCInstrInfo::subInstWouldBeExtended(MCInst const &potentialDuplex) {
+
+  unsigned DstReg, SrcReg;
+
+  switch (potentialDuplex.getOpcode()) {
+  case Hexagon::A2_addi:
+    // testing for case of: Rx = add(Rx,#s7)
+    DstReg = potentialDuplex.getOperand(0).getReg();
+    SrcReg = potentialDuplex.getOperand(1).getReg();
+    if (DstReg == SrcReg && HexagonMCInstrInfo::isIntRegForSubInst(DstReg)) {
+      if (potentialDuplex.getOperand(2).isExpr())
+        return true;
+      if (potentialDuplex.getOperand(2).isImm() &&
+          !(isShiftedInt<7, 0>(potentialDuplex.getOperand(2).getImm())))
+        return true;
+    }
+    break;
+  case Hexagon::A2_tfrsi:
+    DstReg = potentialDuplex.getOperand(0).getReg();
+
+    if (HexagonMCInstrInfo::isIntRegForSubInst(DstReg)) {
+      if (potentialDuplex.getOperand(1).isExpr())
+        return true;
+      // Check for case of Rd = #-1.
+      if (potentialDuplex.getOperand(1).isImm() &&
+          (potentialDuplex.getOperand(1).getImm() == -1))
+        return false;
+      // Check for case of Rd = #u6.
+      if (potentialDuplex.getOperand(1).isImm() &&
+          !isShiftedUInt<6, 0>(potentialDuplex.getOperand(1).getImm()))
+        return true;
+    }
+    break;
+  default:
+    break;
+  }
+  return false;
+}
+
+/// non-Symmetrical. See if these two instructions are fit for duplex pair.
+bool HexagonMCInstrInfo::isOrderedDuplexPair(MCInstrInfo const &MCII,
+                                             MCInst const &MIa, bool ExtendedA,
+                                             MCInst const &MIb, bool ExtendedB,
+                                             bool bisReversable) {
+  // Slot 1 cannot be extended in duplexes PRM 10.5
+  if (ExtendedA)
+    return false;
+  // Only A2_addi and A2_tfrsi can be extended in duplex form PRM 10.5
+  if (ExtendedB) {
+    unsigned Opcode = MIb.getOpcode();
+    if ((Opcode != Hexagon::A2_addi) && (Opcode != Hexagon::A2_tfrsi))
+      return false;
+  }
+  unsigned MIaG = HexagonMCInstrInfo::getDuplexCandidateGroup(MIa),
+           MIbG = HexagonMCInstrInfo::getDuplexCandidateGroup(MIb);
+
+  // If a duplex contains 2 insns in the same group, the insns must be
+  // ordered such that the numerically smaller opcode is in slot 1.
+  if ((MIaG != HexagonII::HSIG_None) && (MIaG == MIbG) && bisReversable) {
+    MCInst SubInst0 = HexagonMCInstrInfo::deriveSubInst(MIa);
+    MCInst SubInst1 = HexagonMCInstrInfo::deriveSubInst(MIb);
+
+    unsigned zeroedSubInstS0 =
+        subinstOpcodeMap.find(SubInst0.getOpcode())->second;
+    unsigned zeroedSubInstS1 =
+        subinstOpcodeMap.find(SubInst1.getOpcode())->second;
+
+    if (zeroedSubInstS0 < zeroedSubInstS1)
+      // subinstS0 (maps to slot 0) must be greater than
+      // subinstS1 (maps to slot 1)
+      return false;
+  }
+
+  // allocframe must always be in slot 0
+  if (MIb.getOpcode() == Hexagon::S2_allocframe)
+    return false;
+
+  if ((MIaG != HexagonII::HSIG_None) && (MIbG != HexagonII::HSIG_None)) {
+    // Prevent 2 instructions with extenders from duplexing
+    // Note that MIb (slot1) can be extended and MIa (slot0)
+    //   can never be extended
+    if (subInstWouldBeExtended(MIa))
+      return false;
+
+    // If duplexing produces an extender, but the original did not
+    //   have an extender, do not duplex.
+    if (subInstWouldBeExtended(MIb) && !ExtendedB)
+      return false;
+  }
+
+  // If jumpr r31 appears, it must be in slot 0, and never slot 1 (MIb).
+  if (MIbG == HexagonII::HSIG_L2) {
+    if ((MIb.getNumOperands() > 1) && MIb.getOperand(1).isReg() &&
+        (MIb.getOperand(1).getReg() == Hexagon::R31))
+      return false;
+    if ((MIb.getNumOperands() > 0) && MIb.getOperand(0).isReg() &&
+        (MIb.getOperand(0).getReg() == Hexagon::R31))
+      return false;
+  }
+
+  // If a store appears, it must be in slot 0 (MIa) 1st, and then slot 1 (MIb);
+  //   therefore, not duplexable if slot 1 is a store, and slot 0 is not.
+  if ((MIbG == HexagonII::HSIG_S1) || (MIbG == HexagonII::HSIG_S2)) {
+    if ((MIaG != HexagonII::HSIG_S1) && (MIaG != HexagonII::HSIG_S2))
+      return false;
+  }
+
+  return (isDuplexPairMatch(MIaG, MIbG));
+}
+
+/// Symmetrical. See if these two instructions are fit for duplex pair.
+bool HexagonMCInstrInfo::isDuplexPair(MCInst const &MIa, MCInst const &MIb) {
+  unsigned MIaG = getDuplexCandidateGroup(MIa),
+           MIbG = getDuplexCandidateGroup(MIb);
+  return (isDuplexPairMatch(MIaG, MIbG) || isDuplexPairMatch(MIbG, MIaG));
+}
+
+inline static void addOps(MCInst &subInstPtr, MCInst const &Inst,
+                          unsigned opNum) {
+  if (Inst.getOperand(opNum).isReg()) {
+    switch (Inst.getOperand(opNum).getReg()) {
+    default:
+      llvm_unreachable("Not Duplexable Register");
+      break;
+    case Hexagon::R0:
+    case Hexagon::R1:
+    case Hexagon::R2:
+    case Hexagon::R3:
+    case Hexagon::R4:
+    case Hexagon::R5:
+    case Hexagon::R6:
+    case Hexagon::R7:
+    case Hexagon::D0:
+    case Hexagon::D1:
+    case Hexagon::D2:
+    case Hexagon::D3:
+    case Hexagon::R16:
+    case Hexagon::R17:
+    case Hexagon::R18:
+    case Hexagon::R19:
+    case Hexagon::R20:
+    case Hexagon::R21:
+    case Hexagon::R22:
+    case Hexagon::R23:
+    case Hexagon::D8:
+    case Hexagon::D9:
+    case Hexagon::D10:
+    case Hexagon::D11:
+      subInstPtr.addOperand(Inst.getOperand(opNum));
+      break;
+    }
+  } else
+    subInstPtr.addOperand(Inst.getOperand(opNum));
+}
+
+MCInst HexagonMCInstrInfo::deriveSubInst(MCInst const &Inst) {
+  MCInst Result;
+  switch (Inst.getOpcode()) {
+  default:
+    // dbgs() << "opcode: "<< Inst->getOpcode() << "\n";
+    llvm_unreachable("Unimplemented subinstruction \n");
+    break;
+  case Hexagon::A2_addi:
+    if (Inst.getOperand(2).isImm() && Inst.getOperand(2).getImm() == 1) {
+      Result.setOpcode(Hexagon::V4_SA1_inc);
+      addOps(Result, Inst, 0);
+      addOps(Result, Inst, 1);
+      break;
+    } //  1,2 SUBInst $Rd = add($Rs, #1)
+    else if (Inst.getOperand(2).isImm() && Inst.getOperand(2).getImm() == -1) {
+      Result.setOpcode(Hexagon::V4_SA1_dec);
+      addOps(Result, Inst, 0);
+      addOps(Result, Inst, 1);
+      break;
+    } //  1,2 SUBInst $Rd = add($Rs,#-1)
+    else if (Inst.getOperand(1).getReg() == Hexagon::R29) {
+      Result.setOpcode(Hexagon::V4_SA1_addsp);
+      addOps(Result, Inst, 0);
+      addOps(Result, Inst, 2);
+      break;
+    } //  1,3 SUBInst $Rd = add(r29, #$u6_2)
+    else {
+      Result.setOpcode(Hexagon::V4_SA1_addi);
+      addOps(Result, Inst, 0);
+      addOps(Result, Inst, 1);
+      addOps(Result, Inst, 2);
+      break;
+    } //    1,2,3 SUBInst $Rx = add($Rx, #$s7)
+  case Hexagon::A2_add:
+    Result.setOpcode(Hexagon::V4_SA1_addrx);
+    addOps(Result, Inst, 0);
+    addOps(Result, Inst, 1);
+    addOps(Result, Inst, 2);
+    break; //    1,2,3 SUBInst $Rx = add($_src_, $Rs)
+  case Hexagon::S2_allocframe:
+    Result.setOpcode(Hexagon::V4_SS2_allocframe);
+    addOps(Result, Inst, 0);
+    break; //    1 SUBInst allocframe(#$u5_3)
+  case Hexagon::A2_andir:
+    if (Inst.getOperand(2).getImm() == 255) {
+      Result.setOpcode(Hexagon::V4_SA1_zxtb);
+      addOps(Result, Inst, 0);
+      addOps(Result, Inst, 1);
+      break; //    1,2    $Rd = and($Rs, #255)
+    } else {
+      Result.setOpcode(Hexagon::V4_SA1_and1);
+      addOps(Result, Inst, 0);
+      addOps(Result, Inst, 1);
+      break; //    1,2 SUBInst $Rd = and($Rs, #1)
+    }
+  case Hexagon::C2_cmpeqi:
+    Result.setOpcode(Hexagon::V4_SA1_cmpeqi);
+    addOps(Result, Inst, 1);
+    addOps(Result, Inst, 2);
+    break; //    2,3 SUBInst p0 = cmp.eq($Rs, #$u2)
+  case Hexagon::A4_combineii:
+  case Hexagon::A2_combineii:
+    if (Inst.getOperand(1).getImm() == 1) {
+      Result.setOpcode(Hexagon::V4_SA1_combine1i);
+      addOps(Result, Inst, 0);
+      addOps(Result, Inst, 2);
+      break; //  1,3 SUBInst $Rdd = combine(#1, #$u2)
+    }
+
+    if (Inst.getOperand(1).getImm() == 3) {
+      Result.setOpcode(Hexagon::V4_SA1_combine3i);
+      addOps(Result, Inst, 0);
+      addOps(Result, Inst, 2);
+      break; //  1,3 SUBInst $Rdd = combine(#3, #$u2)
+    }
+    if (Inst.getOperand(1).getImm() == 0) {
+      Result.setOpcode(Hexagon::V4_SA1_combine0i);
+      addOps(Result, Inst, 0);
+      addOps(Result, Inst, 2);
+      break; //  1,3 SUBInst $Rdd = combine(#0, #$u2)
+    }
+    if (Inst.getOperand(1).getImm() == 2) {
+      Result.setOpcode(Hexagon::V4_SA1_combine2i);
+      addOps(Result, Inst, 0);
+      addOps(Result, Inst, 2);
+      break; //  1,3 SUBInst $Rdd = combine(#2, #$u2)
+    }
+  case Hexagon::A4_combineir:
+    Result.setOpcode(Hexagon::V4_SA1_combinezr);
+    addOps(Result, Inst, 0);
+    addOps(Result, Inst, 2);
+    break; //    1,3 SUBInst $Rdd = combine(#0, $Rs)
+
+  case Hexagon::A4_combineri:
+    Result.setOpcode(Hexagon::V4_SA1_combinerz);
+    addOps(Result, Inst, 0);
+    addOps(Result, Inst, 1);
+    break; //    1,2 SUBInst $Rdd = combine($Rs, #0)
+  case Hexagon::L4_return_tnew_pnt:
+  case Hexagon::L4_return_tnew_pt:
+    Result.setOpcode(Hexagon::V4_SL2_return_tnew);
+    break; //    none  SUBInst if (p0.new) dealloc_return:nt
+  case Hexagon::L4_return_fnew_pnt:
+  case Hexagon::L4_return_fnew_pt:
+    Result.setOpcode(Hexagon::V4_SL2_return_fnew);
+    break; //    none  SUBInst if (!p0.new) dealloc_return:nt
+  case Hexagon::L4_return_f:
+    Result.setOpcode(Hexagon::V4_SL2_return_f);
+    break; //    none  SUBInst if (!p0) dealloc_return
+  case Hexagon::L4_return_t:
+    Result.setOpcode(Hexagon::V4_SL2_return_t);
+    break; //    none  SUBInst if (p0) dealloc_return
+  case Hexagon::L4_return:
+    Result.setOpcode(Hexagon::V4_SL2_return);
+    break; //    none  SUBInst dealloc_return
+  case Hexagon::L2_deallocframe:
+    Result.setOpcode(Hexagon::V4_SL2_deallocframe);
+    break; //    none  SUBInst deallocframe
+  case Hexagon::EH_RETURN_JMPR:
+  case Hexagon::J2_jumpr:
+  case Hexagon::JMPret:
+    Result.setOpcode(Hexagon::V4_SL2_jumpr31);
+    break; //    none  SUBInst jumpr r31
+  case Hexagon::J2_jumprf:
+  case Hexagon::JMPretf:
+    Result.setOpcode(Hexagon::V4_SL2_jumpr31_f);
+    break; //    none  SUBInst if (!p0) jumpr r31
+  case Hexagon::J2_jumprfnew:
+  case Hexagon::JMPretfnewpt:
+  case Hexagon::JMPretfnew:
+    Result.setOpcode(Hexagon::V4_SL2_jumpr31_fnew);
+    break; //    none  SUBInst if (!p0.new) jumpr:nt r31
+  case Hexagon::J2_jumprt:
+  case Hexagon::JMPrett:
+    Result.setOpcode(Hexagon::V4_SL2_jumpr31_t);
+    break; //    none  SUBInst if (p0) jumpr r31
+  case Hexagon::J2_jumprtnew:
+  case Hexagon::JMPrettnewpt:
+  case Hexagon::JMPrettnew:
+    Result.setOpcode(Hexagon::V4_SL2_jumpr31_tnew);
+    break; //    none  SUBInst if (p0.new) jumpr:nt r31
+  case Hexagon::L2_loadrb_io:
+    Result.setOpcode(Hexagon::V4_SL2_loadrb_io);
+    addOps(Result, Inst, 0);
+    addOps(Result, Inst, 1);
+    addOps(Result, Inst, 2);
+    break; //    1,2,3 SUBInst $Rd = memb($Rs + #$u3_0)
+  case Hexagon::L2_loadrd_io:
+    Result.setOpcode(Hexagon::V4_SL2_loadrd_sp);
+    addOps(Result, Inst, 0);
+    addOps(Result, Inst, 2);
+    break; //    1,3 SUBInst $Rdd = memd(r29 + #$u5_3)
+  case Hexagon::L2_loadrh_io:
+    Result.setOpcode(Hexagon::V4_SL2_loadrh_io);
+    addOps(Result, Inst, 0);
+    addOps(Result, Inst, 1);
+    addOps(Result, Inst, 2);
+    break; //    1,2,3 SUBInst $Rd = memh($Rs + #$u3_1)
+  case Hexagon::L2_loadrub_io:
+    Result.setOpcode(Hexagon::V4_SL1_loadrub_io);
+    addOps(Result, Inst, 0);
+    addOps(Result, Inst, 1);
+    addOps(Result, Inst, 2);
+    break; //    1,2,3 SUBInst $Rd = memub($Rs + #$u4_0)
+  case Hexagon::L2_loadruh_io:
+    Result.setOpcode(Hexagon::V4_SL2_loadruh_io);
+    addOps(Result, Inst, 0);
+    addOps(Result, Inst, 1);
+    addOps(Result, Inst, 2);
+    break; //    1,2,3 SUBInst $Rd = memuh($Rs + #$u3_1)
+  case Hexagon::L2_loadri_io:
+    if (Inst.getOperand(1).getReg() == Hexagon::R29) {
+      Result.setOpcode(Hexagon::V4_SL2_loadri_sp);
+      addOps(Result, Inst, 0);
+      addOps(Result, Inst, 2);
+      break; //  2 1,3 SUBInst $Rd = memw(r29 + #$u5_2)
+    } else {
+      Result.setOpcode(Hexagon::V4_SL1_loadri_io);
+      addOps(Result, Inst, 0);
+      addOps(Result, Inst, 1);
+      addOps(Result, Inst, 2);
+      break; //    1,2,3 SUBInst $Rd = memw($Rs + #$u4_2)
+    }
+  case Hexagon::S4_storeirb_io:
+    if (Inst.getOperand(2).getImm() == 0) {
+      Result.setOpcode(Hexagon::V4_SS2_storebi0);
+      addOps(Result, Inst, 0);
+      addOps(Result, Inst, 1);
+      break; //    1,2 SUBInst memb($Rs + #$u4_0)=#0
+    } else if (Inst.getOperand(2).getImm() == 1) {
+      Result.setOpcode(Hexagon::V4_SS2_storebi1);
+      addOps(Result, Inst, 0);
+      addOps(Result, Inst, 1);
+      break; //  2 1,2 SUBInst memb($Rs + #$u4_0)=#1
+    }
+  case Hexagon::S2_storerb_io:
+    Result.setOpcode(Hexagon::V4_SS1_storeb_io);
+    addOps(Result, Inst, 0);
+    addOps(Result, Inst, 1);
+    addOps(Result, Inst, 2);
+    break; //    1,2,3 SUBInst memb($Rs + #$u4_0) = $Rt
+  case Hexagon::S2_storerd_io:
+    Result.setOpcode(Hexagon::V4_SS2_stored_sp);
+    addOps(Result, Inst, 1);
+    addOps(Result, Inst, 2);
+    break; //    2,3 SUBInst memd(r29 + #$s6_3) = $Rtt
+  case Hexagon::S2_storerh_io:
+    Result.setOpcode(Hexagon::V4_SS2_storeh_io);
+    addOps(Result, Inst, 0);
+    addOps(Result, Inst, 1);
+    addOps(Result, Inst, 2);
+    break; //    1,2,3 SUBInst memb($Rs + #$u4_0) = $Rt
+  case Hexagon::S4_storeiri_io:
+    if (Inst.getOperand(2).getImm() == 0) {
+      Result.setOpcode(Hexagon::V4_SS2_storewi0);
+      addOps(Result, Inst, 0);
+      addOps(Result, Inst, 1);
+      break; //  3 1,2 SUBInst memw($Rs + #$u4_2)=#0
+    } else if (Inst.getOperand(2).getImm() == 1) {
+      Result.setOpcode(Hexagon::V4_SS2_storewi1);
+      addOps(Result, Inst, 0);
+      addOps(Result, Inst, 1);
+      break; //  3 1,2 SUBInst memw($Rs + #$u4_2)=#1
+    } else if (Inst.getOperand(0).getReg() == Hexagon::R29) {
+      Result.setOpcode(Hexagon::V4_SS2_storew_sp);
+      addOps(Result, Inst, 1);
+      addOps(Result, Inst, 2);
+      break; //  1 2,3 SUBInst memw(r29 + #$u5_2) = $Rt
+    }
+  case Hexagon::S2_storeri_io:
+    if (Inst.getOperand(0).getReg() == Hexagon::R29) {
+      Result.setOpcode(Hexagon::V4_SS2_storew_sp);
+      addOps(Result, Inst, 1);
+      addOps(Result, Inst, 2); //  1,2,3 SUBInst memw(sp + #$u5_2) = $Rt
+    } else {
+      Result.setOpcode(Hexagon::V4_SS1_storew_io);
+      addOps(Result, Inst, 0);
+      addOps(Result, Inst, 1);
+      addOps(Result, Inst, 2); //  1,2,3 SUBInst memw($Rs + #$u4_2) = $Rt
+    }
+    break;
+  case Hexagon::A2_sxtb:
+    Result.setOpcode(Hexagon::V4_SA1_sxtb);
+    addOps(Result, Inst, 0);
+    addOps(Result, Inst, 1);
+    break; //  1,2 SUBInst $Rd = sxtb($Rs)
+  case Hexagon::A2_sxth:
+    Result.setOpcode(Hexagon::V4_SA1_sxth);
+    addOps(Result, Inst, 0);
+    addOps(Result, Inst, 1);
+    break; //  1,2 SUBInst $Rd = sxth($Rs)
+  case Hexagon::A2_tfr:
+    Result.setOpcode(Hexagon::V4_SA1_tfr);
+    addOps(Result, Inst, 0);
+    addOps(Result, Inst, 1);
+    break; //  1,2 SUBInst $Rd = $Rs
+  case Hexagon::C2_cmovenewif:
+    Result.setOpcode(Hexagon::V4_SA1_clrfnew);
+    addOps(Result, Inst, 0);
+    break; //  2 SUBInst if (!p0.new) $Rd = #0
+  case Hexagon::C2_cmovenewit:
+    Result.setOpcode(Hexagon::V4_SA1_clrtnew);
+    addOps(Result, Inst, 0);
+    break; //  2 SUBInst if (p0.new) $Rd = #0
+  case Hexagon::C2_cmoveif:
+    Result.setOpcode(Hexagon::V4_SA1_clrf);
+    addOps(Result, Inst, 0);
+    break; //  2 SUBInst if (!p0) $Rd = #0
+  case Hexagon::C2_cmoveit:
+    Result.setOpcode(Hexagon::V4_SA1_clrt);
+    addOps(Result, Inst, 0);
+    break; //  2 SUBInst if (p0) $Rd = #0
+  case Hexagon::A2_tfrsi:
+    if (Inst.getOperand(1).isImm() && Inst.getOperand(1).getImm() == -1) {
+      Result.setOpcode(Hexagon::V4_SA1_setin1);
+      addOps(Result, Inst, 0);
+      break; //  2 1 SUBInst $Rd = #-1
+    } else {
+      Result.setOpcode(Hexagon::V4_SA1_seti);
+      addOps(Result, Inst, 0);
+      addOps(Result, Inst, 1);
+      break; //    1,2 SUBInst $Rd = #$u6
+    }
+  case Hexagon::A2_zxtb:
+    Result.setOpcode(Hexagon::V4_SA1_zxtb);
+    addOps(Result, Inst, 0);
+    addOps(Result, Inst, 1);
+    break; //    1,2    $Rd = and($Rs, #255)
+
+  case Hexagon::A2_zxth:
+    Result.setOpcode(Hexagon::V4_SA1_zxth);
+    addOps(Result, Inst, 0);
+    addOps(Result, Inst, 1);
+    break; //    1,2 SUBInst $Rd = zxth($Rs)
+  }
+  return Result;
+}
+
+static bool isStoreInst(unsigned opCode) {
+  switch (opCode) {
+  case Hexagon::S2_storeri_io:
+  case Hexagon::S2_storerb_io:
+  case Hexagon::S2_storerh_io:
+  case Hexagon::S2_storerd_io:
+  case Hexagon::S4_storeiri_io:
+  case Hexagon::S4_storeirb_io:
+  case Hexagon::S2_allocframe:
+    return true;
+  default:
+    return false;
+  }
+}
+
+SmallVector<DuplexCandidate, 8>
+HexagonMCInstrInfo::getDuplexPossibilties(MCInstrInfo const &MCII,
+                                          MCInst const &MCB) {
+  assert(isBundle(MCB));
+  SmallVector<DuplexCandidate, 8> duplexToTry;
+  // Use an "order matters" version of isDuplexPair.
+  unsigned numInstrInPacket = MCB.getNumOperands();
+
+  for (unsigned distance = 1; distance < numInstrInPacket; ++distance) {
+    for (unsigned j = HexagonMCInstrInfo::bundleInstructionsOffset,
+                  k = j + distance;
+         (j < numInstrInPacket) && (k < numInstrInPacket); ++j, ++k) {
+
+      // Check if reversable.
+      bool bisReversable = true;
+      if (isStoreInst(MCB.getOperand(j).getInst()->getOpcode()) &&
+          isStoreInst(MCB.getOperand(k).getInst()->getOpcode())) {
+        DEBUG(dbgs() << "skip out of order write pair: " << k << "," << j
+                     << "\n");
+        bisReversable = false;
+      }
+
+      // Try in order.
+      if (isOrderedDuplexPair(
+              MCII, *MCB.getOperand(k).getInst(),
+              HexagonMCInstrInfo::hasExtenderForIndex(MCB, k - 1),
+              *MCB.getOperand(j).getInst(),
+              HexagonMCInstrInfo::hasExtenderForIndex(MCB, j - 1),
+              bisReversable)) {
+        // Get iClass.
+        unsigned iClass = iClassOfDuplexPair(
+            getDuplexCandidateGroup(*MCB.getOperand(k).getInst()),
+            getDuplexCandidateGroup(*MCB.getOperand(j).getInst()));
+
+        // Save off pairs for duplex checking.
+        duplexToTry.push_back(DuplexCandidate(j, k, iClass));
+        DEBUG(dbgs() << "adding pair: " << j << "," << k << ":"
+                     << MCB.getOperand(j).getInst()->getOpcode() << ","
+                     << MCB.getOperand(k).getInst()->getOpcode() << "\n");
+        continue;
+      } else {
+        DEBUG(dbgs() << "skipping pair: " << j << "," << k << ":"
+                     << MCB.getOperand(j).getInst()->getOpcode() << ","
+                     << MCB.getOperand(k).getInst()->getOpcode() << "\n");
+      }
+
+      // Try reverse.
+      if (bisReversable) {
+        if (isOrderedDuplexPair(
+                MCII, *MCB.getOperand(j).getInst(),
+                HexagonMCInstrInfo::hasExtenderForIndex(MCB, j - 1),
+                *MCB.getOperand(k).getInst(),
+                HexagonMCInstrInfo::hasExtenderForIndex(MCB, k - 1),
+                bisReversable)) {
+          // Get iClass.
+          unsigned iClass = iClassOfDuplexPair(
+              getDuplexCandidateGroup(*MCB.getOperand(j).getInst()),
+              getDuplexCandidateGroup(*MCB.getOperand(k).getInst()));
+
+          // Save off pairs for duplex checking.
+          duplexToTry.push_back(DuplexCandidate(k, j, iClass));
+          DEBUG(dbgs() << "adding pair:" << k << "," << j << ":"
+                       << MCB.getOperand(j).getInst()->getOpcode() << ","
+                       << MCB.getOperand(k).getInst()->getOpcode() << "\n");
+        } else {
+          DEBUG(dbgs() << "skipping pair: " << k << "," << j << ":"
+                       << MCB.getOperand(j).getInst()->getOpcode() << ","
+                       << MCB.getOperand(k).getInst()->getOpcode() << "\n");
+        }
+      }
+    }
+  }
+  return duplexToTry;
+}
diff --git a/lib/Target/Hexagon/MCTargetDesc/HexagonMCELFStreamer.cpp b/lib/Target/Hexagon/MCTargetDesc/HexagonMCELFStreamer.cpp
new file mode 100644
index 00000000000..bf51c3515e9
--- /dev/null
+++ b/lib/Target/Hexagon/MCTargetDesc/HexagonMCELFStreamer.cpp
@@ -0,0 +1,152 @@
+//=== HexagonMCELFStreamer.cpp - Hexagon subclass of MCELFStreamer -------===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is a stub that parses a MCInst bundle and passes the
+// instructions on to the real streamer.
+//
+//===----------------------------------------------------------------------===//
+#define DEBUG_TYPE "hexagonmcelfstreamer"
+
+#include "Hexagon.h"
+#include "HexagonMCELFStreamer.h"
+#include "MCTargetDesc/HexagonBaseInfo.h"
+#include "MCTargetDesc/HexagonMCShuffler.h"
+#include "llvm/ADT/StringExtras.h"
+#include "llvm/MC/MCAssembler.h"
+#include "llvm/MC/MCContext.h"
+#include "llvm/MC/MCSectionELF.h"
+#include "llvm/MC/MCStreamer.h"
+#include "llvm/MC/MCSymbol.h"
+#include "llvm/MC/MCSymbolELF.h"
+#include "llvm/Support/CommandLine.h"
+#include "llvm/Support/Debug.h"
+#include "llvm/Support/raw_ostream.h"
+
+using namespace llvm;
+
+static cl::opt<unsigned>
+    GPSize("gpsize", cl::NotHidden,
+           cl::desc("Global Pointer Addressing Size.  The default size is 8."),
+           cl::Prefix, cl::init(8));
+
+void HexagonMCELFStreamer::EmitInstruction(const MCInst &MCK,
+                                           const MCSubtargetInfo &STI) {
+  MCInst HMI;
+  HMI.setOpcode(Hexagon::BUNDLE);
+  HMI.addOperand(MCOperand::createImm(0));
+  MCInst *MCB;
+
+  if (MCK.getOpcode() != Hexagon::BUNDLE) {
+    HMI.addOperand(MCOperand::createInst(&MCK));
+    MCB = &HMI;
+  } else
+    MCB = const_cast<MCInst *>(&MCK);
+
+  // Examines packet and pad the packet, if needed, when an
+  // end-loop is in the bundle.
+  HexagonMCInstrInfo::padEndloop(*MCB);
+  HexagonMCShuffle(*MCII, STI, *MCB);
+
+  assert(HexagonMCInstrInfo::bundleSize(*MCB) <= HEXAGON_PACKET_SIZE);
+  bool Extended = false;
+  for (auto &I : HexagonMCInstrInfo::bundleInstructions(*MCB)) {
+    MCInst *MCI = const_cast<MCInst *>(I.getInst());
+    if (Extended) {
+      if (HexagonMCInstrInfo::isDuplex(*MCII, *MCI)) {
+        MCInst *SubInst = const_cast<MCInst *>(MCI->getOperand(1).getInst());
+        HexagonMCInstrInfo::clampExtended(*MCII, *SubInst);
+      } else {
+        HexagonMCInstrInfo::clampExtended(*MCII, *MCI);
+      }
+      Extended = false;
+    } else {
+      Extended = HexagonMCInstrInfo::isImmext(*MCI);
+    }
+  }
+
+  // At this point, MCB is a bundle
+  // Iterate through the bundle and assign addends for the instructions
+  for (auto const &I : HexagonMCInstrInfo::bundleInstructions(*MCB)) {
+    MCInst *MCI = const_cast<MCInst *>(I.getInst());
+    EmitSymbol(*MCI);
+  }
+  MCObjectStreamer::EmitInstruction(*MCB, STI);
+}
+
+void HexagonMCELFStreamer::EmitSymbol(const MCInst &Inst) {
+  // Scan for values.
+  for (unsigned i = Inst.getNumOperands(); i--;)
+    if (Inst.getOperand(i).isExpr())
+      visitUsedExpr(*Inst.getOperand(i).getExpr());
+}
+
+// EmitCommonSymbol and EmitLocalCommonSymbol are extended versions of the
+// functions found in MCELFStreamer.cpp taking AccessSize as an additional
+// parameter.
+void HexagonMCELFStreamer::HexagonMCEmitCommonSymbol(MCSymbol *Symbol,
+                                                     uint64_t Size,
+                                                     unsigned ByteAlignment,
+                                                     unsigned AccessSize) {
+  getAssembler().registerSymbol(*Symbol);
+  StringRef sbss[4] = {".sbss.1", ".sbss.2", ".sbss.4", ".sbss.8"};
+
+  auto ELFSymbol = cast<MCSymbolELF>(Symbol);
+  if (!ELFSymbol->isBindingSet()) {
+    ELFSymbol->setBinding(ELF::STB_GLOBAL);
+    ELFSymbol->setExternal(true);
+  }
+
+  ELFSymbol->setType(ELF::STT_OBJECT);
+
+  if (ELFSymbol->getBinding() == ELF::STB_LOCAL) {
+    StringRef SectionName =
+        ((AccessSize == 0) || (Size == 0) || (Size > GPSize))
+            ? ".bss"
+            : sbss[(Log2_64(AccessSize))];
+
+    MCSection *CrntSection = getCurrentSection().first;
+    MCSection *Section = getAssembler().getContext().getELFSection(
+        SectionName, ELF::SHT_NOBITS, ELF::SHF_WRITE | ELF::SHF_ALLOC);
+    SwitchSection(Section);
+    AssignSection(Symbol, Section);
+
+    MCELFStreamer::EmitCommonSymbol(Symbol, Size, ByteAlignment);
+    SwitchSection(CrntSection);
+  } else {
+    if (ELFSymbol->declareCommon(Size, ByteAlignment))
+      report_fatal_error("Symbol: " + Symbol->getName() +
+                         " redeclared as different type");
+    if ((AccessSize) && (Size <= GPSize)) {
+      uint64_t SectionIndex =
+          (AccessSize <= GPSize)
+              ? ELF::SHN_HEXAGON_SCOMMON + (Log2_64(AccessSize) + 1)
+              : (unsigned)ELF::SHN_HEXAGON_SCOMMON;
+      ELFSymbol->setIndex(SectionIndex);
+    }
+  }
+
+  ELFSymbol->setSize(MCConstantExpr::create(Size, getContext()));
+}
+
+void HexagonMCELFStreamer::HexagonMCEmitLocalCommonSymbol(
+    MCSymbol *Symbol, uint64_t Size, unsigned ByteAlignment,
+    unsigned AccessSize) {
+  getAssembler().registerSymbol(*Symbol);
+  auto ELFSymbol = cast<MCSymbolELF>(Symbol);
+  ELFSymbol->setBinding(ELF::STB_LOCAL);
+  ELFSymbol->setExternal(false);
+  HexagonMCEmitCommonSymbol(Symbol, Size, ByteAlignment, AccessSize);
+}
+
+namespace llvm {
+MCStreamer *createHexagonELFStreamer(MCContext &Context, MCAsmBackend &MAB,
+                                     raw_pwrite_stream &OS, MCCodeEmitter *CE) {
+  return new HexagonMCELFStreamer(Context, MAB, OS, CE);
+}
+}
diff --git a/lib/Target/Hexagon/MCTargetDesc/HexagonMCELFStreamer.h b/lib/Target/Hexagon/MCTargetDesc/HexagonMCELFStreamer.h
new file mode 100644
index 00000000000..d77c0cd16b3
--- /dev/null
+++ b/lib/Target/Hexagon/MCTargetDesc/HexagonMCELFStreamer.h
@@ -0,0 +1,45 @@
+//===- HexagonMCELFStreamer.h - Hexagon subclass of MCElfStreamer ---------===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef HEXAGONMCELFSTREAMER_H
+#define HEXAGONMCELFSTREAMER_H
+
+#include "MCTargetDesc/HexagonMCCodeEmitter.h"
+#include "MCTargetDesc/HexagonMCInstrInfo.h"
+#include "MCTargetDesc/HexagonMCTargetDesc.h"
+#include "llvm/MC/MCELFStreamer.h"
+#include "HexagonTargetStreamer.h"
+
+namespace llvm {
+
+class HexagonMCELFStreamer : public MCELFStreamer {
+  std::unique_ptr<MCInstrInfo> MCII;
+
+public:
+  HexagonMCELFStreamer(MCContext &Context, MCAsmBackend &TAB,
+                       raw_pwrite_stream &OS, MCCodeEmitter *Emitter)
+      : MCELFStreamer(Context, TAB, OS, Emitter),
+        MCII(createHexagonMCInstrInfo()) {}
+
+  virtual void EmitInstruction(const MCInst &Inst,
+                               const MCSubtargetInfo &STI) override;
+  void EmitSymbol(const MCInst &Inst);
+  void HexagonMCEmitLocalCommonSymbol(MCSymbol *Symbol, uint64_t Size,
+                                      unsigned ByteAlignment,
+                                      unsigned AccessSize);
+  void HexagonMCEmitCommonSymbol(MCSymbol *Symbol, uint64_t Size,
+                                 unsigned ByteAlignment, unsigned AccessSize);
+};
+
+MCStreamer *createHexagonELFStreamer(MCContext &Context, MCAsmBackend &MAB,
+                                     raw_pwrite_stream &OS, MCCodeEmitter *CE);
+
+} // namespace llvm
+
+#endif
diff --git a/lib/Target/Hexagon/MCTargetDesc/HexagonMCInstrInfo.cpp b/lib/Target/Hexagon/MCTargetDesc/HexagonMCInstrInfo.cpp
index 33e7c81904f..48b15f85a78 100644
--- a/lib/Target/Hexagon/MCTargetDesc/HexagonMCInstrInfo.cpp
+++ b/lib/Target/Hexagon/MCTargetDesc/HexagonMCInstrInfo.cpp
@@ -12,12 +12,76 @@
 //===----------------------------------------------------------------------===//
 
 #include "HexagonMCInstrInfo.h"
+
+#include "Hexagon.h"
 #include "HexagonBaseInfo.h"
 
+#include "llvm/MC/MCContext.h"
+#include "llvm/MC/MCInstrInfo.h"
+#include "llvm/MC/MCSubtargetInfo.h"
+
 namespace llvm {
-void HexagonMCInstrInfo::AppendImplicitOperands(MCInst &MCI) {
-  MCI.addOperand(MCOperand::CreateImm(0));
-  MCI.addOperand(MCOperand::CreateInst(nullptr));
+iterator_range<MCInst::const_iterator>
+HexagonMCInstrInfo::bundleInstructions(MCInst const &MCI) {
+  assert(isBundle(MCI));
+  return iterator_range<MCInst::const_iterator>(
+      MCI.begin() + bundleInstructionsOffset, MCI.end());
+}
+
+size_t HexagonMCInstrInfo::bundleSize(MCInst const &MCI) {
+  if (HexagonMCInstrInfo::isBundle(MCI))
+    return (MCI.size() - bundleInstructionsOffset);
+  else
+    return (1);
+}
+
+void HexagonMCInstrInfo::clampExtended(MCInstrInfo const &MCII, MCInst &MCI) {
+  assert(HexagonMCInstrInfo::isExtendable(MCII, MCI) ||
+         HexagonMCInstrInfo::isExtended(MCII, MCI));
+  MCOperand &exOp =
+      MCI.getOperand(HexagonMCInstrInfo::getExtendableOp(MCII, MCI));
+  // If the extended value is a constant, then use it for the extended and
+  // for the extender instructions, masking off the lower 6 bits and
+  // including the assumed bits.
+  if (exOp.isImm()) {
+    unsigned Shift = HexagonMCInstrInfo::getExtentAlignment(MCII, MCI);
+    int64_t Bits = exOp.getImm();
+    exOp.setImm((Bits & 0x3f) << Shift);
+  }
+}
+
+MCInst *HexagonMCInstrInfo::deriveDuplex(MCContext &Context, unsigned iClass,
+                                         MCInst const &inst0,
+                                         MCInst const &inst1) {
+  assert((iClass <= 0xf) && "iClass must have range of 0 to 0xf");
+  MCInst *duplexInst = new (Context) MCInst;
+  duplexInst->setOpcode(Hexagon::DuplexIClass0 + iClass);
+
+  MCInst *SubInst0 = new (Context) MCInst(deriveSubInst(inst0));
+  MCInst *SubInst1 = new (Context) MCInst(deriveSubInst(inst1));
+  duplexInst->addOperand(MCOperand::createInst(SubInst0));
+  duplexInst->addOperand(MCOperand::createInst(SubInst1));
+  return duplexInst;
+}
+
+MCInst const *HexagonMCInstrInfo::extenderForIndex(MCInst const &MCB,
+                                                   size_t Index) {
+  assert(Index <= bundleSize(MCB));
+  if (Index == 0)
+    return nullptr;
+  MCInst const *Inst =
+      MCB.getOperand(Index + bundleInstructionsOffset - 1).getInst();
+  if (isImmext(*Inst))
+    return Inst;
+  return nullptr;
+}
+
+HexagonII::MemAccessSize
+HexagonMCInstrInfo::getAccessSize(MCInstrInfo const &MCII, MCInst const &MCI) {
+  const uint64_t F = HexagonMCInstrInfo::getDesc(MCII, MCI).TSFlags;
+
+  return (HexagonII::MemAccessSize((F >> HexagonII::MemAccessSizePos) &
+                                   HexagonII::MemAccesSizeMask));
 }
 
 unsigned HexagonMCInstrInfo::getBitCount(MCInstrInfo const &MCII,
@@ -38,10 +102,34 @@ MCInstrDesc const &HexagonMCInstrInfo::getDesc(MCInstrInfo const &MCII,
   return (MCII.get(MCI.getOpcode()));
 }
 
-std::bitset<16> HexagonMCInstrInfo::GetImplicitBits(MCInst const &MCI) {
-  SanityCheckImplicitOperands(MCI);
-  std::bitset<16> Bits(MCI.getOperand(MCI.getNumOperands() - 2).getImm());
-  return Bits;
+unsigned short HexagonMCInstrInfo::getExtendableOp(MCInstrInfo const &MCII,
+                                                   MCInst const &MCI) {
+  const uint64_t F = HexagonMCInstrInfo::getDesc(MCII, MCI).TSFlags;
+  return ((F >> HexagonII::ExtendableOpPos) & HexagonII::ExtendableOpMask);
+}
+
+MCOperand const &
+HexagonMCInstrInfo::getExtendableOperand(MCInstrInfo const &MCII,
+                                         MCInst const &MCI) {
+  unsigned O = HexagonMCInstrInfo::getExtendableOp(MCII, MCI);
+  MCOperand const &MO = MCI.getOperand(O);
+
+  assert((HexagonMCInstrInfo::isExtendable(MCII, MCI) ||
+          HexagonMCInstrInfo::isExtended(MCII, MCI)) &&
+         (MO.isImm() || MO.isExpr()));
+  return (MO);
+}
+
+unsigned HexagonMCInstrInfo::getExtentAlignment(MCInstrInfo const &MCII,
+                                                MCInst const &MCI) {
+  const uint64_t F = HexagonMCInstrInfo::getDesc(MCII, MCI).TSFlags;
+  return ((F >> HexagonII::ExtentAlignPos) & HexagonII::ExtentAlignMask);
+}
+
+unsigned HexagonMCInstrInfo::getExtentBits(MCInstrInfo const &MCII,
+                                           MCInst const &MCI) {
+  const uint64_t F = HexagonMCInstrInfo::getDesc(MCII, MCI).TSFlags;
+  return ((F >> HexagonII::ExtentBitsPos) & HexagonII::ExtentBitsMask);
 }
 
 // Return the max value that a constant extendable operand can have
@@ -74,9 +162,19 @@ int HexagonMCInstrInfo::getMinValue(MCInstrInfo const &MCII,
     return 0;
 }
 
-// Return the operand that consumes or produces a new value.
-MCOperand const &HexagonMCInstrInfo::getNewValue(MCInstrInfo const &MCII,
+char const *HexagonMCInstrInfo::getName(MCInstrInfo const &MCII,
+                                        MCInst const &MCI) {
+  return MCII.getName(MCI.getOpcode());
+}
+
+unsigned short HexagonMCInstrInfo::getNewValueOp(MCInstrInfo const &MCII,
                                                  MCInst const &MCI) {
+  const uint64_t F = HexagonMCInstrInfo::getDesc(MCII, MCI).TSFlags;
+  return ((F >> HexagonII::NewValueOpPos) & HexagonII::NewValueOpMask);
+}
+
+MCOperand const &HexagonMCInstrInfo::getNewValueOperand(MCInstrInfo const &MCII,
+                                                        MCInst const &MCI) {
   uint64_t const F = HexagonMCInstrInfo::getDesc(MCII, MCI).TSFlags;
   unsigned const O =
       (F >> HexagonII::NewValueOpPos) & HexagonII::NewValueOpMask;
@@ -88,6 +186,21 @@ MCOperand const &HexagonMCInstrInfo::getNewValue(MCInstrInfo const &MCII,
   return (MCO);
 }
 
+int HexagonMCInstrInfo::getSubTarget(MCInstrInfo const &MCII,
+                                     MCInst const &MCI) {
+  const uint64_t F = HexagonMCInstrInfo::getDesc(MCII, MCI).TSFlags;
+
+  HexagonII::SubTarget Target = static_cast<HexagonII::SubTarget>(
+      (F >> HexagonII::validSubTargetPos) & HexagonII::validSubTargetMask);
+
+  switch (Target) {
+  default:
+    return Hexagon::ArchV4;
+  case HexagonII::HasV5SubT:
+    return Hexagon::ArchV5;
+  }
+}
+
 // Return the Hexagon ISA class for the insn.
 unsigned HexagonMCInstrInfo::getType(MCInstrInfo const &MCII,
                                      MCInst const &MCI) {
@@ -96,6 +209,32 @@ unsigned HexagonMCInstrInfo::getType(MCInstrInfo const &MCII,
   return ((F >> HexagonII::TypePos) & HexagonII::TypeMask);
 }
 
+unsigned HexagonMCInstrInfo::getUnits(MCInstrInfo const &MCII,
+                                      MCSubtargetInfo const &STI,
+                                      MCInst const &MCI) {
+
+  const InstrItinerary *II = STI.getSchedModel().InstrItineraries;
+  int SchedClass = HexagonMCInstrInfo::getDesc(MCII, MCI).getSchedClass();
+  return ((II[SchedClass].FirstStage + HexagonStages)->getUnits());
+}
+
+bool HexagonMCInstrInfo::hasImmExt(MCInst const &MCI) {
+  if (!HexagonMCInstrInfo::isBundle(MCI))
+    return false;
+
+  for (const auto &I : HexagonMCInstrInfo::bundleInstructions(MCI)) {
+    auto MI = I.getInst();
+    if (isImmext(*MI))
+      return true;
+  }
+
+  return false;
+}
+
+bool HexagonMCInstrInfo::hasExtenderForIndex(MCInst const &MCB, size_t Index) {
+  return extenderForIndex(MCB, Index) != nullptr;
+}
+
 // Return whether the instruction is a legal new-value producer.
 bool HexagonMCInstrInfo::hasNewValue(MCInstrInfo const &MCII,
                                      MCInst const &MCI) {
@@ -103,6 +242,18 @@ bool HexagonMCInstrInfo::hasNewValue(MCInstrInfo const &MCII,
   return ((F >> HexagonII::hasNewValuePos) & HexagonII::hasNewValueMask);
 }
 
+MCInst const &HexagonMCInstrInfo::instruction(MCInst const &MCB, size_t Index) {
+  assert(isBundle(MCB));
+  assert(Index < HEXAGON_PACKET_SIZE);
+  return *MCB.getOperand(bundleInstructionsOffset + Index).getInst();
+}
+
+bool HexagonMCInstrInfo::isBundle(MCInst const &MCI) {
+  auto Result = Hexagon::BUNDLE == MCI.getOpcode();
+  assert(!Result || (MCI.size() > 0 && MCI.getOperand(0).isImm()));
+  return Result;
+}
+
 // Return whether the insn is an actual insn.
 bool HexagonMCInstrInfo::isCanon(MCInstrInfo const &MCII, MCInst const &MCI) {
   return (!HexagonMCInstrInfo::getDesc(MCII, MCI).isPseudo() &&
@@ -110,6 +261,15 @@ bool HexagonMCInstrInfo::isCanon(MCInstrInfo const &MCII, MCInst const &MCI) {
           HexagonMCInstrInfo::getType(MCII, MCI) != HexagonII::TypeENDLOOP);
 }
 
+bool HexagonMCInstrInfo::isDblRegForSubInst(unsigned Reg) {
+  return ((Reg >= Hexagon::D0 && Reg <= Hexagon::D3) ||
+          (Reg >= Hexagon::D8 && Reg <= Hexagon::D11));
+}
+
+bool HexagonMCInstrInfo::isDuplex(MCInstrInfo const &MCII, MCInst const &MCI) {
+  return HexagonII::TypeDUPLEX == HexagonMCInstrInfo::getType(MCII, MCI);
+}
+
 // Return whether the instruction needs to be constant extended.
 // 1) Always return true if the instruction has 'isExtended' flag set.
 //
@@ -148,20 +308,44 @@ bool HexagonMCInstrInfo::isConstExtended(MCInstrInfo const &MCII,
   return (ImmValue < MinValue || ImmValue > MaxValue);
 }
 
-// Return true if the instruction may be extended based on the operand value.
 bool HexagonMCInstrInfo::isExtendable(MCInstrInfo const &MCII,
                                       MCInst const &MCI) {
   uint64_t const F = HexagonMCInstrInfo::getDesc(MCII, MCI).TSFlags;
   return (F >> HexagonII::ExtendablePos) & HexagonII::ExtendableMask;
 }
 
-// Return whether the instruction must be always extended.
 bool HexagonMCInstrInfo::isExtended(MCInstrInfo const &MCII,
                                     MCInst const &MCI) {
   uint64_t const F = HexagonMCInstrInfo::getDesc(MCII, MCI).TSFlags;
   return (F >> HexagonII::ExtendedPos) & HexagonII::ExtendedMask;
 }
 
+bool HexagonMCInstrInfo::isFloat(MCInstrInfo const &MCII, MCInst const &MCI) {
+  const uint64_t F = HexagonMCInstrInfo::getDesc(MCII, MCI).TSFlags;
+  return ((F >> HexagonII::FPPos) & HexagonII::FPMask);
+}
+
+bool HexagonMCInstrInfo::isImmext(MCInst const &MCI) {
+  auto Op = MCI.getOpcode();
+  return (Op == Hexagon::A4_ext_b || Op == Hexagon::A4_ext_c ||
+          Op == Hexagon::A4_ext_g || Op == Hexagon::A4_ext);
+}
+
+bool HexagonMCInstrInfo::isInnerLoop(MCInst const &MCI) {
+  assert(isBundle(MCI));
+  int64_t Flags = MCI.getOperand(0).getImm();
+  return (Flags & innerLoopMask) != 0;
+}
+
+bool HexagonMCInstrInfo::isIntReg(unsigned Reg) {
+  return (Reg >= Hexagon::R0 && Reg <= Hexagon::R31);
+}
+
+bool HexagonMCInstrInfo::isIntRegForSubInst(unsigned Reg) {
+  return ((Reg >= Hexagon::R0 && Reg <= Hexagon::R7) ||
+          (Reg >= Hexagon::R16 && Reg <= Hexagon::R23));
+}
+
 // Return whether the insn is a new-value consumer.
 bool HexagonMCInstrInfo::isNewValue(MCInstrInfo const &MCII,
                                     MCInst const &MCI) {
@@ -178,46 +362,103 @@ bool HexagonMCInstrInfo::isOperandExtended(MCInstrInfo const &MCII,
          OperandNum;
 }
 
-bool HexagonMCInstrInfo::isPacketBegin(MCInst const &MCI) {
-  std::bitset<16> Bits(GetImplicitBits(MCI));
-  return Bits.test(packetBeginIndex);
+bool HexagonMCInstrInfo::isOuterLoop(MCInst const &MCI) {
+  assert(isBundle(MCI));
+  int64_t Flags = MCI.getOperand(0).getImm();
+  return (Flags & outerLoopMask) != 0;
 }
 
-bool HexagonMCInstrInfo::isPacketEnd(MCInst const &MCI) {
-  std::bitset<16> Bits(GetImplicitBits(MCI));
-  return Bits.test(packetEndIndex);
+bool HexagonMCInstrInfo::isPredicated(MCInstrInfo const &MCII,
+                                      MCInst const &MCI) {
+  const uint64_t F = HexagonMCInstrInfo::getDesc(MCII, MCI).TSFlags;
+  return ((F >> HexagonII::PredicatedPos) & HexagonII::PredicatedMask);
+}
+
+bool HexagonMCInstrInfo::isPredicatedTrue(MCInstrInfo const &MCII,
+                                          MCInst const &MCI) {
+  const uint64_t F = HexagonMCInstrInfo::getDesc(MCII, MCI).TSFlags;
+  return (
+      !((F >> HexagonII::PredicatedFalsePos) & HexagonII::PredicatedFalseMask));
+}
+
+bool HexagonMCInstrInfo::isPredReg(unsigned Reg) {
+  return (Reg >= Hexagon::P0 && Reg <= Hexagon::P3_0);
 }
 
-// Return whether the insn is a prefix.
 bool HexagonMCInstrInfo::isPrefix(MCInstrInfo const &MCII, MCInst const &MCI) {
   return (HexagonMCInstrInfo::getType(MCII, MCI) == HexagonII::TypePREFIX);
 }
 
-// Return whether the insn is solo, i.e., cannot be in a packet.
 bool HexagonMCInstrInfo::isSolo(MCInstrInfo const &MCII, MCInst const &MCI) {
   const uint64_t F = HexagonMCInstrInfo::getDesc(MCII, MCI).TSFlags;
   return ((F >> HexagonII::SoloPos) & HexagonII::SoloMask);
 }
 
-void HexagonMCInstrInfo::resetPacket(MCInst &MCI) {
-  setPacketBegin(MCI, false);
-  setPacketEnd(MCI, false);
+bool HexagonMCInstrInfo::isSoloAX(MCInstrInfo const &MCII, MCInst const &MCI) {
+  const uint64_t F = HexagonMCInstrInfo::getDesc(MCII, MCI).TSFlags;
+  return ((F >> HexagonII::SoloAXPos) & HexagonII::SoloAXMask);
+}
+
+bool HexagonMCInstrInfo::isSoloAin1(MCInstrInfo const &MCII,
+                                    MCInst const &MCI) {
+  const uint64_t F = HexagonMCInstrInfo::getDesc(MCII, MCI).TSFlags;
+  return ((F >> HexagonII::SoloAin1Pos) & HexagonII::SoloAin1Mask);
+}
+
+void HexagonMCInstrInfo::padEndloop(MCInst &MCB) {
+  MCInst Nop;
+  Nop.setOpcode(Hexagon::A2_nop);
+  assert(isBundle(MCB));
+  while ((HexagonMCInstrInfo::isInnerLoop(MCB) &&
+          (HexagonMCInstrInfo::bundleSize(MCB) < HEXAGON_PACKET_INNER_SIZE)) ||
+         ((HexagonMCInstrInfo::isOuterLoop(MCB) &&
+           (HexagonMCInstrInfo::bundleSize(MCB) < HEXAGON_PACKET_OUTER_SIZE))))
+    MCB.addOperand(MCOperand::createInst(new MCInst(Nop)));
+}
+
+bool HexagonMCInstrInfo::prefersSlot3(MCInstrInfo const &MCII,
+                                      MCInst const &MCI) {
+  if (HexagonMCInstrInfo::getType(MCII, MCI) == HexagonII::TypeCR)
+    return false;
+
+  unsigned SchedClass = HexagonMCInstrInfo::getDesc(MCII, MCI).getSchedClass();
+  switch (SchedClass) {
+  case Hexagon::Sched::ALU32_3op_tc_2_SLOT0123:
+  case Hexagon::Sched::ALU64_tc_2_SLOT23:
+  case Hexagon::Sched::ALU64_tc_3x_SLOT23:
+  case Hexagon::Sched::M_tc_2_SLOT23:
+  case Hexagon::Sched::M_tc_3x_SLOT23:
+  case Hexagon::Sched::S_2op_tc_2_SLOT23:
+  case Hexagon::Sched::S_3op_tc_2_SLOT23:
+  case Hexagon::Sched::S_3op_tc_3x_SLOT23:
+    return true;
+  }
+  return false;
 }
 
-void HexagonMCInstrInfo::SetImplicitBits(MCInst &MCI, std::bitset<16> Bits) {
-  SanityCheckImplicitOperands(MCI);
-  MCI.getOperand(MCI.getNumOperands() - 2).setImm(Bits.to_ulong());
+void HexagonMCInstrInfo::replaceDuplex(MCContext &Context, MCInst &MCB,
+                                       DuplexCandidate Candidate) {
+  assert(Candidate.packetIndexI < MCB.size());
+  assert(Candidate.packetIndexJ < MCB.size());
+  assert(isBundle(MCB));
+  MCInst *Duplex =
+      deriveDuplex(Context, Candidate.iClass,
+                   *MCB.getOperand(Candidate.packetIndexJ).getInst(),
+                   *MCB.getOperand(Candidate.packetIndexI).getInst());
+  assert(Duplex != nullptr);
+  MCB.getOperand(Candidate.packetIndexI).setInst(Duplex);
+  MCB.erase(MCB.begin() + Candidate.packetIndexJ);
 }
 
-void HexagonMCInstrInfo::setPacketBegin(MCInst &MCI, bool f) {
-  std::bitset<16> Bits(GetImplicitBits(MCI));
-  Bits.set(packetBeginIndex, f);
-  SetImplicitBits(MCI, Bits);
+void HexagonMCInstrInfo::setInnerLoop(MCInst &MCI) {
+  assert(isBundle(MCI));
+  MCOperand &Operand = MCI.getOperand(0);
+  Operand.setImm(Operand.getImm() | innerLoopMask);
 }
 
-void HexagonMCInstrInfo::setPacketEnd(MCInst &MCI, bool f) {
-  std::bitset<16> Bits(GetImplicitBits(MCI));
-  Bits.set(packetEndIndex, f);
-  SetImplicitBits(MCI, Bits);
+void HexagonMCInstrInfo::setOuterLoop(MCInst &MCI) {
+  assert(isBundle(MCI));
+  MCOperand &Operand = MCI.getOperand(0);
+  Operand.setImm(Operand.getImm() | outerLoopMask);
 }
 }
diff --git a/lib/Target/Hexagon/MCTargetDesc/HexagonMCInstrInfo.h b/lib/Target/Hexagon/MCTargetDesc/HexagonMCInstrInfo.h
index 10fc0f3bdeb..32d61a4a7be 100644
--- a/lib/Target/Hexagon/MCTargetDesc/HexagonMCInstrInfo.h
+++ b/lib/Target/Hexagon/MCTargetDesc/HexagonMCInstrInfo.h
@@ -1,4 +1,4 @@
-//===- HexagonMCInstrInfo.cpp - Hexagon sub-class of MCInst ---------------===//
+//===- HexagonMCInstrInfo.cpp - Utility functions on Hexagon MCInsts ------===//
 //
 //                     The LLVM Compiler Infrastructure
 //
@@ -14,17 +14,55 @@
 #ifndef LLVM_LIB_TARGET_HEXAGON_MCTARGETDESC_HEXAGONMCINSTRINFO_H
 #define LLVM_LIB_TARGET_HEXAGON_MCTARGETDESC_HEXAGONMCINSTRINFO_H
 
-#include "llvm/MC/MCInstrInfo.h"
-
-#include <bitset>
+#include "llvm/MC/MCInst.h"
 
 namespace llvm {
+class MCContext;
 class MCInstrDesc;
 class MCInstrInfo;
 class MCInst;
 class MCOperand;
+class MCSubtargetInfo;
+namespace HexagonII {
+enum class MemAccessSize;
+}
+class DuplexCandidate {
+public:
+  unsigned packetIndexI, packetIndexJ, iClass;
+  DuplexCandidate(unsigned i, unsigned j, unsigned iClass)
+      : packetIndexI(i), packetIndexJ(j), iClass(iClass) {}
+};
 namespace HexagonMCInstrInfo {
-void AppendImplicitOperands(MCInst &MCI);
+size_t const innerLoopOffset = 0;
+int64_t const innerLoopMask = 1 << innerLoopOffset;
+
+size_t const outerLoopOffset = 1;
+int64_t const outerLoopMask = 1 << outerLoopOffset;
+
+size_t const bundleInstructionsOffset = 1;
+
+// Returns a iterator range of instructions in this bundle
+iterator_range<MCInst::const_iterator> bundleInstructions(MCInst const &MCI);
+
+// Returns the number of instructions in the bundle
+size_t bundleSize(MCInst const &MCI);
+
+// Clamp off upper 26 bits of extendable operand for emission
+void clampExtended(MCInstrInfo const &MCII, MCInst &MCI);
+
+// Create a duplex instruction given the two subinsts
+MCInst *deriveDuplex(MCContext &Context, unsigned iClass, MCInst const &inst0,
+                     MCInst const &inst1);
+
+// Convert this instruction in to a duplex subinst
+MCInst deriveSubInst(MCInst const &Inst);
+
+// Return the extender for instruction at Index or nullptr if none
+MCInst const *extenderForIndex(MCInst const &MCB, size_t Index);
+
+// Return memory access size
+HexagonII::MemAccessSize getAccessSize(MCInstrInfo const &MCII,
+                                       MCInst const &MCI);
 
 // Return number of bits in the constant extended operand.
 unsigned getBitCount(MCInstrInfo const &MCII, MCInst const &MCI);
@@ -34,7 +72,25 @@ unsigned short getCExtOpNum(MCInstrInfo const &MCII, MCInst const &MCI);
 
 MCInstrDesc const &getDesc(MCInstrInfo const &MCII, MCInst const &MCI);
 
-std::bitset<16> GetImplicitBits(MCInst const &MCI);
+// Return which duplex group this instruction belongs to
+unsigned getDuplexCandidateGroup(MCInst const &MI);
+
+// Return a list of all possible instruction duplex combinations
+SmallVector<DuplexCandidate, 8> getDuplexPossibilties(MCInstrInfo const &MCII,
+                                                      MCInst const &MCB);
+
+// Return the index of the extendable operand
+unsigned short getExtendableOp(MCInstrInfo const &MCII, MCInst const &MCI);
+
+// Return a reference to the extendable operand
+MCOperand const &getExtendableOperand(MCInstrInfo const &MCII,
+                                      MCInst const &MCI);
+
+// Return the implicit alignment of the extendable operand
+unsigned getExtentAlignment(MCInstrInfo const &MCII, MCInst const &MCI);
+
+// Return the number of logical bits of the extendable operand
+unsigned getExtentBits(MCInstrInfo const &MCII, MCInst const &MCI);
 
 // Return the max value that a constant extendable operand can have
 // without being extended.
@@ -44,27 +100,80 @@ int getMaxValue(MCInstrInfo const &MCII, MCInst const &MCI);
 // without being extended.
 int getMinValue(MCInstrInfo const &MCII, MCInst const &MCI);
 
+// Return instruction name
+char const *getName(MCInstrInfo const &MCII, MCInst const &MCI);
+
+// Return the operand index for the new value.
+unsigned short getNewValueOp(MCInstrInfo const &MCII, MCInst const &MCI);
+
 // Return the operand that consumes or produces a new value.
-MCOperand const &getNewValue(MCInstrInfo const &MCII, MCInst const &MCI);
+MCOperand const &getNewValueOperand(MCInstrInfo const &MCII, MCInst const &MCI);
+
+int getSubTarget(MCInstrInfo const &MCII, MCInst const &MCI);
 
 // Return the Hexagon ISA class for the insn.
 unsigned getType(MCInstrInfo const &MCII, MCInst const &MCI);
 
+/// Return the slots used by the insn.
+unsigned getUnits(MCInstrInfo const &MCII, MCSubtargetInfo const &STI,
+                  MCInst const &MCI);
+
+// Does the packet have an extender for the instruction at Index
+bool hasExtenderForIndex(MCInst const &MCB, size_t Index);
+
+bool hasImmExt(MCInst const &MCI);
+
 // Return whether the instruction is a legal new-value producer.
 bool hasNewValue(MCInstrInfo const &MCII, MCInst const &MCI);
-  
+
+// Return the instruction at Index
+MCInst const &instruction(MCInst const &MCB, size_t Index);
+
+// Returns whether this MCInst is a wellformed bundle
+bool isBundle(MCInst const &MCI);
+
 // Return whether the insn is an actual insn.
 bool isCanon(MCInstrInfo const &MCII, MCInst const &MCI);
 
+// Return the duplex iclass given the two duplex classes
+unsigned iClassOfDuplexPair(unsigned Ga, unsigned Gb);
+
 // Return whether the instruction needs to be constant extended.
 bool isConstExtended(MCInstrInfo const &MCII, MCInst const &MCI);
 
+// Is this double register suitable for use in a duplex subinst
+bool isDblRegForSubInst(unsigned Reg);
+
+// Is this a duplex instruction
+bool isDuplex(MCInstrInfo const &MCII, MCInst const &MCI);
+
+// Can these instructions be duplexed
+bool isDuplexPair(MCInst const &MIa, MCInst const &MIb);
+
+// Can these duplex classes be combine in to a duplex instruction
+bool isDuplexPairMatch(unsigned Ga, unsigned Gb);
+
 // Return true if the insn may be extended based on the operand value.
 bool isExtendable(MCInstrInfo const &MCII, MCInst const &MCI);
 
 // Return whether the instruction must be always extended.
 bool isExtended(MCInstrInfo const &MCII, MCInst const &MCI);
 
+/// Return whether it is a floating-point insn.
+bool isFloat(MCInstrInfo const &MCII, MCInst const &MCI);
+
+// Returns whether this instruction is an immediate extender
+bool isImmext(MCInst const &MCI);
+
+// Returns whether this bundle is an endloop0
+bool isInnerLoop(MCInst const &MCI);
+
+// Is this an integer register
+bool isIntReg(unsigned Reg);
+
+// Is this register suitable for use in a duplex subinst
+bool isIntRegForSubInst(unsigned Reg);
+
 // Return whether the insn is a new-value consumer.
 bool isNewValue(MCInstrInfo const &MCII, MCInst const &MCI);
 
@@ -72,9 +181,22 @@ bool isNewValue(MCInstrInfo const &MCII, MCInst const &MCI);
 bool isOperandExtended(MCInstrInfo const &MCII, MCInst const &MCI,
                        unsigned short OperandNum);
 
-bool isPacketBegin(MCInst const &MCI);
+// Can these two instructions be duplexed
+bool isOrderedDuplexPair(MCInstrInfo const &MCII, MCInst const &MIa,
+                         bool ExtendedA, MCInst const &MIb, bool ExtendedB,
+                         bool bisReversable);
+
+// Returns whether this bundle is an endloop1
+bool isOuterLoop(MCInst const &MCI);
+
+// Return whether this instruction is predicated
+bool isPredicated(MCInstrInfo const &MCII, MCInst const &MCI);
 
-bool isPacketEnd(MCInst const &MCI);
+// Return whether the predicate sense is true
+bool isPredicatedTrue(MCInstrInfo const &MCII, MCInst const &MCI);
+
+// Is this a predicate register
+bool isPredReg(unsigned Reg);
 
 // Return whether the insn is a prefix.
 bool isPrefix(MCInstrInfo const &MCII, MCInst const &MCI);
@@ -82,24 +204,31 @@ bool isPrefix(MCInstrInfo const &MCII, MCInst const &MCI);
 // Return whether the insn is solo, i.e., cannot be in a packet.
 bool isSolo(MCInstrInfo const &MCII, MCInst const &MCI);
 
-static const size_t packetBeginIndex = 0;
-static const size_t packetEndIndex = 1;
+/// Return whether the insn can be packaged only with A and X-type insns.
+bool isSoloAX(MCInstrInfo const &MCII, MCInst const &MCI);
 
-void resetPacket(MCInst &MCI);
+/// Return whether the insn can be packaged only with an A-type insn in slot #1.
+bool isSoloAin1(MCInstrInfo const &MCII, MCInst const &MCI);
 
-inline void SanityCheckImplicitOperands(MCInst const &MCI) {
-  assert(MCI.getNumOperands() >= 2 && "At least the two implicit operands");
-  assert(MCI.getOperand(MCI.getNumOperands() - 1).isInst() &&
-          "Implicit bits and flags");
-  assert(MCI.getOperand(MCI.getNumOperands() - 2).isImm() &&
-          "Parent pointer");
-}
+// Pad the bundle with nops to satisfy endloop requirements
+void padEndloop(MCInst &MCI);
+
+bool prefersSlot3(MCInstrInfo const &MCII, MCInst const &MCI);
+
+// Replace the instructions inside MCB, represented by Candidate
+void replaceDuplex(MCContext &Context, MCInst &MCB, DuplexCandidate Candidate);
+
+// Marks a bundle as endloop0
+void setInnerLoop(MCInst &MCI);
 
-void SetImplicitBits(MCInst &MCI, std::bitset<16> Bits);
+// Marks a bundle as endloop1
+void setOuterLoop(MCInst &MCI);
 
-void setPacketBegin(MCInst &MCI, bool Y);
+// Would duplexing this instruction create a requirement to extend
+bool subInstWouldBeExtended(MCInst const &potentialDuplex);
 
-void setPacketEnd(MCInst &MCI, bool Y);
+// Attempt to find and replace compound pairs
+void tryCompound(MCInstrInfo const &MCII, MCContext &Context, MCInst &MCI);
 }
 }
 
diff --git a/lib/Target/Hexagon/MCTargetDesc/HexagonMCShuffler.cpp b/lib/Target/Hexagon/MCTargetDesc/HexagonMCShuffler.cpp
new file mode 100644
index 00000000000..8e70280c1a0
--- /dev/null
+++ b/lib/Target/Hexagon/MCTargetDesc/HexagonMCShuffler.cpp
@@ -0,0 +1,237 @@
+//===----- HexagonMCShuffler.cpp - MC bundle shuffling --------------------===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This implements the shuffling of insns inside a bundle according to the
+// packet formation rules of the Hexagon ISA.
+//
+//===----------------------------------------------------------------------===//
+
+#define DEBUG_TYPE "hexagon-shuffle"
+
+#include "Hexagon.h"
+#include "MCTargetDesc/HexagonMCInstrInfo.h"
+#include "MCTargetDesc/HexagonMCShuffler.h"
+#include "MCTargetDesc/HexagonMCTargetDesc.h"
+#include "llvm/Support/CommandLine.h"
+#include "llvm/Support/Debug.h"
+#include "llvm/Support/raw_ostream.h"
+
+using namespace llvm;
+
+static cl::opt<bool>
+    DisableShuffle("disable-hexagon-shuffle", cl::Hidden, cl::init(false),
+                   cl::desc("Disable Hexagon instruction shuffling"));
+
+void HexagonMCShuffler::init(MCInst &MCB) {
+  if (HexagonMCInstrInfo::isBundle(MCB)) {
+    MCInst const *Extender = nullptr;
+    // Copy the bundle for the shuffling.
+    for (const auto &I : HexagonMCInstrInfo::bundleInstructions(MCB)) {
+      assert(!HexagonMCInstrInfo::getDesc(MCII, *I.getInst()).isPseudo());
+      MCInst *MI = const_cast<MCInst *>(I.getInst());
+
+      if (!HexagonMCInstrInfo::isImmext(*MI)) {
+        append(MI, Extender, HexagonMCInstrInfo::getUnits(MCII, STI, *MI),
+               false);
+        Extender = nullptr;
+      } else
+        Extender = MI;
+    }
+  }
+
+  BundleFlags = MCB.getOperand(0).getImm();
+}
+
+void HexagonMCShuffler::init(MCInst &MCB, MCInst const *AddMI,
+                             bool bInsertAtFront) {
+  if (HexagonMCInstrInfo::isBundle(MCB)) {
+    if (bInsertAtFront && AddMI)
+      append(AddMI, nullptr, HexagonMCInstrInfo::getUnits(MCII, STI, *AddMI),
+             false);
+    MCInst const *Extender = nullptr;
+    // Copy the bundle for the shuffling.
+    for (auto const &I : HexagonMCInstrInfo::bundleInstructions(MCB)) {
+      assert(!HexagonMCInstrInfo::getDesc(MCII, *I.getInst()).isPseudo());
+      MCInst *MI = const_cast<MCInst *>(I.getInst());
+      if (!HexagonMCInstrInfo::isImmext(*MI)) {
+        append(MI, Extender, HexagonMCInstrInfo::getUnits(MCII, STI, *MI),
+               false);
+        Extender = nullptr;
+      } else
+        Extender = MI;
+    }
+    if (!bInsertAtFront && AddMI)
+      append(AddMI, nullptr, HexagonMCInstrInfo::getUnits(MCII, STI, *AddMI),
+             false);
+  }
+
+  BundleFlags = MCB.getOperand(0).getImm();
+}
+
+void HexagonMCShuffler::copyTo(MCInst &MCB) {
+  MCB.clear();
+  MCB.addOperand(MCOperand::createImm(BundleFlags));
+  // Copy the results into the bundle.
+  for (HexagonShuffler::iterator I = begin(); I != end(); ++I) {
+
+    MCInst const *MI = I->getDesc();
+    MCInst const *Extender = I->getExtender();
+    if (Extender)
+      MCB.addOperand(MCOperand::createInst(Extender));
+    MCB.addOperand(MCOperand::createInst(MI));
+  }
+}
+
+bool HexagonMCShuffler::reshuffleTo(MCInst &MCB) {
+  if (shuffle()) {
+    // Copy the results into the bundle.
+    copyTo(MCB);
+  } else
+    DEBUG(MCB.dump());
+
+  return (!getError());
+}
+
+bool llvm::HexagonMCShuffle(MCInstrInfo const &MCII, MCSubtargetInfo const &STI,
+                            MCInst &MCB) {
+  HexagonMCShuffler MCS(MCII, STI, MCB);
+
+  if (DisableShuffle)
+    // Ignore if user chose so.
+    return false;
+
+  if (!HexagonMCInstrInfo::bundleSize(MCB)) {
+    // There once was a bundle:
+    //    BUNDLE %D2<imp-def>, %R4<imp-def>, %R5<imp-def>, %D7<imp-def>, ...
+    //      * %D2<def> = IMPLICIT_DEF; flags:
+    //      * %D7<def> = IMPLICIT_DEF; flags:
+    // After the IMPLICIT_DEFs were removed by the asm printer, the bundle
+    // became empty.
+    DEBUG(dbgs() << "Skipping empty bundle");
+    return false;
+  } else if (!HexagonMCInstrInfo::isBundle(MCB)) {
+    DEBUG(dbgs() << "Skipping stand-alone insn");
+    return false;
+  }
+
+  // Reorder the bundle and copy the result.
+  if (!MCS.reshuffleTo(MCB)) {
+    // Unless there is any error, which should not happen at this point.
+    unsigned shuffleError = MCS.getError();
+    switch (shuffleError) {
+    default:
+      llvm_unreachable("unknown error");
+    case HexagonShuffler::SHUFFLE_ERROR_INVALID:
+      llvm_unreachable("invalid packet");
+    case HexagonShuffler::SHUFFLE_ERROR_STORES:
+      llvm_unreachable("too many stores");
+    case HexagonShuffler::SHUFFLE_ERROR_LOADS:
+      llvm_unreachable("too many loads");
+    case HexagonShuffler::SHUFFLE_ERROR_BRANCHES:
+      llvm_unreachable("too many branches");
+    case HexagonShuffler::SHUFFLE_ERROR_NOSLOTS:
+      llvm_unreachable("no suitable slot");
+    case HexagonShuffler::SHUFFLE_ERROR_SLOTS:
+      llvm_unreachable("over-subscribed slots");
+    case HexagonShuffler::SHUFFLE_SUCCESS: // Single instruction case.
+      return true;
+    }
+  }
+
+  return true;
+}
+
+unsigned
+llvm::HexagonMCShuffle(MCInstrInfo const &MCII, MCSubtargetInfo const &STI,
+                       MCContext &Context, MCInst &MCB,
+                       SmallVector<DuplexCandidate, 8> possibleDuplexes) {
+
+  if (DisableShuffle)
+    return HexagonShuffler::SHUFFLE_SUCCESS;
+
+  if (!HexagonMCInstrInfo::bundleSize(MCB)) {
+    // There once was a bundle:
+    //    BUNDLE %D2<imp-def>, %R4<imp-def>, %R5<imp-def>, %D7<imp-def>, ...
+    //      * %D2<def> = IMPLICIT_DEF; flags:
+    //      * %D7<def> = IMPLICIT_DEF; flags:
+    // After the IMPLICIT_DEFs were removed by the asm printer, the bundle
+    // became empty.
+    DEBUG(dbgs() << "Skipping empty bundle");
+    return HexagonShuffler::SHUFFLE_SUCCESS;
+  } else if (!HexagonMCInstrInfo::isBundle(MCB)) {
+    DEBUG(dbgs() << "Skipping stand-alone insn");
+    return HexagonShuffler::SHUFFLE_SUCCESS;
+  }
+
+  bool doneShuffling = false;
+  unsigned shuffleError;
+  while (possibleDuplexes.size() > 0 && (!doneShuffling)) {
+    // case of Duplex Found
+    DuplexCandidate duplexToTry = possibleDuplexes.pop_back_val();
+    MCInst Attempt(MCB);
+    HexagonMCInstrInfo::replaceDuplex(Context, Attempt, duplexToTry);
+    HexagonMCShuffler MCS(MCII, STI, Attempt); // copy packet to the shuffler
+    if (MCS.size() == 1) {                     // case of one duplex
+      // copy the created duplex in the shuffler to the bundle
+      MCS.copyTo(MCB);
+      doneShuffling = true;
+      return HexagonShuffler::SHUFFLE_SUCCESS;
+    }
+    // try shuffle with this duplex
+    doneShuffling = MCS.reshuffleTo(MCB);
+    shuffleError = MCS.getError();
+
+    if (doneShuffling)
+      break;
+  }
+
+  if (doneShuffling == false) {
+    HexagonMCShuffler MCS(MCII, STI, MCB);
+    doneShuffling = MCS.reshuffleTo(MCB); // shuffle
+    shuffleError = MCS.getError();
+  }
+  if (!doneShuffling)
+    return shuffleError;
+
+  return HexagonShuffler::SHUFFLE_SUCCESS;
+}
+
+bool llvm::HexagonMCShuffle(MCInstrInfo const &MCII, MCSubtargetInfo const &STI,
+                            MCInst &MCB, MCInst const *AddMI, int fixupCount) {
+  if (!HexagonMCInstrInfo::isBundle(MCB) || !AddMI)
+    return false;
+
+  // if fixups present, make sure we don't insert too many nops that would
+  // later prevent an extender from being inserted.
+  unsigned int bundleSize = HexagonMCInstrInfo::bundleSize(MCB);
+  if (bundleSize >= HEXAGON_PACKET_SIZE)
+    return false;
+  if (fixupCount >= 2) {
+    return false;
+  } else {
+    if (bundleSize == HEXAGON_PACKET_SIZE - 1 && fixupCount)
+      return false;
+  }
+
+  if (DisableShuffle)
+    return false;
+
+  HexagonMCShuffler MCS(MCII, STI, MCB, AddMI);
+  if (!MCS.reshuffleTo(MCB)) {
+    unsigned shuffleError = MCS.getError();
+    switch (shuffleError) {
+    default:
+      return false;
+    case HexagonShuffler::SHUFFLE_SUCCESS: // single instruction case
+      return true;
+    }
+  }
+
+  return true;
+}
diff --git a/lib/Target/Hexagon/MCTargetDesc/HexagonMCShuffler.h b/lib/Target/Hexagon/MCTargetDesc/HexagonMCShuffler.h
new file mode 100644
index 00000000000..a21cce1fc24
--- /dev/null
+++ b/lib/Target/Hexagon/MCTargetDesc/HexagonMCShuffler.h
@@ -0,0 +1,65 @@
+//=-- HexagonMCShuffler.h ---------------------------------------------------=//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This declares the shuffling of insns inside a bundle according to the
+// packet formation rules of the Hexagon ISA.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef HEXAGONMCSHUFFLER_H
+#define HEXAGONMCSHUFFLER_H
+
+#include "MCTargetDesc/HexagonShuffler.h"
+
+namespace llvm {
+
+class MCInst;
+
+// Insn bundle shuffler.
+class HexagonMCShuffler : public HexagonShuffler {
+  bool immext_present;
+  bool duplex_present;
+
+public:
+  HexagonMCShuffler(MCInstrInfo const &MCII, MCSubtargetInfo const &STI,
+                    MCInst &MCB)
+      : HexagonShuffler(MCII, STI) {
+    init(MCB);
+  };
+  HexagonMCShuffler(MCInstrInfo const &MCII, MCSubtargetInfo const &STI,
+                    MCInst &MCB, const MCInst *AddMI,
+                    bool bInsertAtFront = false)
+      : HexagonShuffler(MCII, STI) {
+    init(MCB, AddMI, bInsertAtFront);
+  };
+
+  // Copy reordered bundle to another.
+  void copyTo(MCInst &MCB);
+  // Reorder and copy result to another.
+  bool reshuffleTo(MCInst &MCB);
+
+  bool immextPresent() const { return immext_present; };
+  bool duplexPresent() const { return duplex_present; };
+
+private:
+  void init(MCInst &MCB);
+  void init(MCInst &MCB, const MCInst *AddMI, bool bInsertAtFront = false);
+};
+
+// Invocation of the shuffler.
+bool HexagonMCShuffle(MCInstrInfo const &MCII, MCSubtargetInfo const &STI,
+                      MCInst &);
+bool HexagonMCShuffle(MCInstrInfo const &MCII, MCSubtargetInfo const &STI,
+                      MCInst &, const MCInst *, int);
+unsigned HexagonMCShuffle(MCInstrInfo const &MCII, MCSubtargetInfo const &STI,
+                          MCContext &Context, MCInst &,
+                          SmallVector<DuplexCandidate, 8>);
+}
+
+#endif // HEXAGONMCSHUFFLER_H
diff --git a/lib/Target/Hexagon/MCTargetDesc/HexagonMCTargetDesc.cpp b/lib/Target/Hexagon/MCTargetDesc/HexagonMCTargetDesc.cpp
index 2e10d810b8f..53305d85fd8 100644
--- a/lib/Target/Hexagon/MCTargetDesc/HexagonMCTargetDesc.cpp
+++ b/lib/Target/Hexagon/MCTargetDesc/HexagonMCTargetDesc.cpp
@@ -12,15 +12,20 @@
 //===----------------------------------------------------------------------===//
 
 #include "HexagonMCTargetDesc.h"
+#include "Hexagon.h"
 #include "HexagonMCAsmInfo.h"
+#include "HexagonMCELFStreamer.h"
 #include "MCTargetDesc/HexagonInstPrinter.h"
 #include "llvm/MC/MCCodeGenInfo.h"
+#include "llvm/MC/MCContext.h"
 #include "llvm/MC/MCELFStreamer.h"
 #include "llvm/MC/MCInstrInfo.h"
+#include "llvm/MC/MCObjectStreamer.h"
 #include "llvm/MC/MCRegisterInfo.h"
 #include "llvm/MC/MCStreamer.h"
 #include "llvm/MC/MCSubtargetInfo.h"
 #include "llvm/MC/MachineLocation.h"
+#include "llvm/Support/ELF.h"
 #include "llvm/Support/ErrorHandling.h"
 #include "llvm/Support/TargetRegistry.h"
 
@@ -41,21 +46,99 @@ MCInstrInfo *llvm::createHexagonMCInstrInfo() {
   return X;
 }
 
-static MCRegisterInfo *createHexagonMCRegisterInfo(StringRef TT) {
+static MCRegisterInfo *createHexagonMCRegisterInfo(const Triple &TT) {
   MCRegisterInfo *X = new MCRegisterInfo();
   InitHexagonMCRegisterInfo(X, Hexagon::R0);
   return X;
 }
 
 static MCSubtargetInfo *
-createHexagonMCSubtargetInfo(StringRef TT, StringRef CPU, StringRef FS) {
-  MCSubtargetInfo *X = new MCSubtargetInfo();
-  InitHexagonMCSubtargetInfo(X, TT, CPU, FS);
-  return X;
+createHexagonMCSubtargetInfo(const Triple &TT, StringRef CPU, StringRef FS) {
+  return createHexagonMCSubtargetInfoImpl(TT, CPU, FS);
+}
+
+namespace {
+class HexagonTargetAsmStreamer : public HexagonTargetStreamer {
+public:
+  HexagonTargetAsmStreamer(MCStreamer &S,
+                           formatted_raw_ostream &, bool,
+                           MCInstPrinter &)
+      : HexagonTargetStreamer(S) {}
+  void prettyPrintAsm(MCInstPrinter &InstPrinter, raw_ostream &OS,
+                      const MCInst &Inst, const MCSubtargetInfo &STI) override {
+    assert(HexagonMCInstrInfo::isBundle(Inst));
+    assert(HexagonMCInstrInfo::bundleSize(Inst) <= HEXAGON_PACKET_SIZE);
+    std::string Buffer;
+    {
+      raw_string_ostream TempStream(Buffer);
+      InstPrinter.printInst(&Inst, TempStream, "", STI);
+    }
+    StringRef Contents(Buffer);
+    auto PacketBundle = Contents.rsplit('\n');
+    auto HeadTail = PacketBundle.first.split('\n');
+    auto Preamble = "\t{\n\t\t";
+    auto Separator = "";
+    while(!HeadTail.first.empty()) {
+      OS << Separator;
+      StringRef Inst;
+      auto Duplex = HeadTail.first.split('\v');
+      if(!Duplex.second.empty()){
+        OS << Duplex.first << "\n";
+        Inst = Duplex.second;
+      }
+      else {
+        if(!HeadTail.first.startswith("immext"))
+          Inst = Duplex.first;
+      }
+      OS << Preamble;
+      OS << Inst;
+      HeadTail = HeadTail.second.split('\n');
+      Preamble = "";
+      Separator = "\n\t\t";
+    }
+    if(HexagonMCInstrInfo::bundleSize(Inst) != 0)
+      OS << "\n\t}" << PacketBundle.second;
+  }
+};
+}
+
+namespace {
+class HexagonTargetELFStreamer : public HexagonTargetStreamer {
+public:
+  MCELFStreamer &getStreamer() {
+    return static_cast<MCELFStreamer &>(Streamer);
+  }
+  HexagonTargetELFStreamer(MCStreamer &S, MCSubtargetInfo const &STI)
+      : HexagonTargetStreamer(S) {
+    auto Bits = STI.getFeatureBits();
+    unsigned Flags;
+    if (Bits.to_ullong() & llvm::Hexagon::ArchV5)
+      Flags = ELF::EF_HEXAGON_MACH_V5;
+    else
+      Flags = ELF::EF_HEXAGON_MACH_V4;
+    getStreamer().getAssembler().setELFHeaderEFlags(Flags);
+  }
+  void EmitCommonSymbolSorted(MCSymbol *Symbol, uint64_t Size,
+                              unsigned ByteAlignment,
+                              unsigned AccessSize) override {
+    HexagonMCELFStreamer &HexagonELFStreamer =
+        static_cast<HexagonMCELFStreamer &>(getStreamer());
+    HexagonELFStreamer.HexagonMCEmitCommonSymbol(Symbol, Size, ByteAlignment,
+                                                 AccessSize);
+  }
+  void EmitLocalCommonSymbolSorted(MCSymbol *Symbol, uint64_t Size,
+                                   unsigned ByteAlignment,
+                                   unsigned AccessSize) override {
+    HexagonMCELFStreamer &HexagonELFStreamer =
+        static_cast<HexagonMCELFStreamer &>(getStreamer());
+    HexagonELFStreamer.HexagonMCEmitLocalCommonSymbol(
+        Symbol, Size, ByteAlignment, AccessSize);
+  }
+};
 }
 
 static MCAsmInfo *createHexagonMCAsmInfo(const MCRegisterInfo &MRI,
-                                         StringRef TT) {
+                                         const Triple &TT) {
   MCAsmInfo *MAI = new HexagonMCAsmInfo(TT);
 
   // VirtualFP = (R30 + #0).
@@ -66,13 +149,14 @@ static MCAsmInfo *createHexagonMCAsmInfo(const MCRegisterInfo &MRI,
   return MAI;
 }
 
-static MCCodeGenInfo *createHexagonMCCodeGenInfo(StringRef TT, Reloc::Model RM,
+static MCCodeGenInfo *createHexagonMCCodeGenInfo(const Triple &TT,
+                                                 Reloc::Model RM,
                                                  CodeModel::Model CM,
                                                  CodeGenOpt::Level OL) {
   MCCodeGenInfo *X = new MCCodeGenInfo();
   // For the time being, use static relocations, since there's really no
   // support for PIC yet.
-  X->InitMCCodeGenInfo(Reloc::Static, CM, OL);
+  X->initMCCodeGenInfo(Reloc::Static, CM, OL);
   return X;
 }
 
@@ -82,9 +166,27 @@ static MCInstPrinter *createHexagonMCInstPrinter(const Triple &T,
                                                  const MCInstrInfo &MII,
                                                  const MCRegisterInfo &MRI) {
   if (SyntaxVariant == 0)
-    return(new HexagonInstPrinter(MAI, MII, MRI));
+    return (new HexagonInstPrinter(MAI, MII, MRI));
   else
-   return nullptr;
+    return nullptr;
+}
+
+static MCTargetStreamer *createMCAsmTargetStreamer(MCStreamer &S,
+                                                   formatted_raw_ostream &OS,
+                                                   MCInstPrinter *InstPrint,
+                                                   bool IsVerboseAsm) {
+  return new HexagonTargetAsmStreamer(S,  OS, IsVerboseAsm, *InstPrint);
+}
+
+static MCStreamer *createMCStreamer(Triple const &T, MCContext &Context,
+                                    MCAsmBackend &MAB, raw_pwrite_stream &OS,
+                                    MCCodeEmitter *Emitter, bool RelaxAll) {
+  return createHexagonELFStreamer(Context, MAB, OS, Emitter);
+}
+
+static MCTargetStreamer *
+createHexagonObjectTargetStreamer(MCStreamer &S, MCSubtargetInfo const &STI) {
+  return new HexagonTargetELFStreamer(S, STI);
 }
 
 // Force static initialization.
@@ -112,11 +214,21 @@ extern "C" void LLVMInitializeHexagonTargetMC() {
   TargetRegistry::RegisterMCCodeEmitter(TheHexagonTarget,
                                         createHexagonMCCodeEmitter);
 
+  // Register the asm backend
+  TargetRegistry::RegisterMCAsmBackend(TheHexagonTarget,
+                                       createHexagonAsmBackend);
+
+  // Register the obj streamer
+  TargetRegistry::RegisterELFStreamer(TheHexagonTarget, createMCStreamer);
+
+  // Register the asm streamer
+  TargetRegistry::RegisterAsmTargetStreamer(TheHexagonTarget,
+                                            createMCAsmTargetStreamer);
+
   // Register the MC Inst Printer
   TargetRegistry::RegisterMCInstPrinter(TheHexagonTarget,
                                         createHexagonMCInstPrinter);
 
-  // Register the asm backend
-  TargetRegistry::RegisterMCAsmBackend(TheHexagonTarget,
-                                       createHexagonAsmBackend);
+  TargetRegistry::RegisterObjectTargetStreamer(
+      TheHexagonTarget, createHexagonObjectTargetStreamer);
 }
diff --git a/lib/Target/Hexagon/MCTargetDesc/HexagonMCTargetDesc.h b/lib/Target/Hexagon/MCTargetDesc/HexagonMCTargetDesc.h
index de63fd271ae..cb626503313 100644
--- a/lib/Target/Hexagon/MCTargetDesc/HexagonMCTargetDesc.h
+++ b/lib/Target/Hexagon/MCTargetDesc/HexagonMCTargetDesc.h
@@ -17,6 +17,8 @@
 #include <cstdint>
 
 namespace llvm {
+struct InstrItinerary;
+struct InstrStage;
 class MCAsmBackend;
 class MCCodeEmitter;
 class MCContext;
@@ -25,12 +27,15 @@ class MCObjectWriter;
 class MCRegisterInfo;
 class MCSubtargetInfo;
 class Target;
+class Triple;
 class StringRef;
 class raw_ostream;
 class raw_pwrite_stream;
 
 extern Target TheHexagonTarget;
 
+extern const InstrStage HexagonStages[];
+
 MCInstrInfo *createHexagonMCInstrInfo();
 
 MCCodeEmitter *createHexagonMCCodeEmitter(MCInstrInfo const &MCII,
@@ -38,8 +43,8 @@ MCCodeEmitter *createHexagonMCCodeEmitter(MCInstrInfo const &MCII,
                                           MCContext &MCT);
 
 MCAsmBackend *createHexagonAsmBackend(Target const &T,
-                                      MCRegisterInfo const &MRI, StringRef TT,
-                                      StringRef CPU);
+                                      MCRegisterInfo const &MRI,
+                                      const Triple &TT, StringRef CPU);
 
 MCObjectWriter *createHexagonELFObjectWriter(raw_pwrite_stream &OS,
                                              uint8_t OSABI, StringRef CPU);
diff --git a/lib/Target/Hexagon/MCTargetDesc/HexagonShuffler.cpp b/lib/Target/Hexagon/MCTargetDesc/HexagonShuffler.cpp
new file mode 100644
index 00000000000..41112ac0b46
--- /dev/null
+++ b/lib/Target/Hexagon/MCTargetDesc/HexagonShuffler.cpp
@@ -0,0 +1,388 @@
+//===----- HexagonShuffler.cpp - Instruction bundle shuffling -------------===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This implements the shuffling of insns inside a bundle according to the
+// packet formation rules of the Hexagon ISA.
+//
+//===----------------------------------------------------------------------===//
+
+#define DEBUG_TYPE "hexagon-shuffle"
+
+#include <algorithm>
+#include <utility>
+#include "Hexagon.h"
+#include "MCTargetDesc/HexagonBaseInfo.h"
+#include "MCTargetDesc/HexagonMCTargetDesc.h"
+#include "MCTargetDesc/HexagonMCInstrInfo.h"
+#include "HexagonShuffler.h"
+#include "llvm/Support/Debug.h"
+#include "llvm/Support/MathExtras.h"
+#include "llvm/Support/raw_ostream.h"
+
+using namespace llvm;
+
+// Insn shuffling priority.
+class HexagonBid {
+  // The priority is directly proportional to how restricted the insn is based
+  // on its flexibility to run on the available slots.  So, the fewer slots it
+  // may run on, the higher its priority.
+  enum { MAX = 360360 }; // LCD of 1/2, 1/3, 1/4,... 1/15.
+  unsigned Bid;
+
+public:
+  HexagonBid() : Bid(0){};
+  HexagonBid(unsigned B) { Bid = B ? MAX / countPopulation(B) : 0; };
+
+  // Check if the insn priority is overflowed.
+  bool isSold() const { return (Bid >= MAX); };
+
+  HexagonBid &operator+=(const HexagonBid &B) {
+    Bid += B.Bid;
+    return *this;
+  };
+};
+
+// Slot shuffling allocation.
+class HexagonUnitAuction {
+  HexagonBid Scores[HEXAGON_PACKET_SIZE];
+  // Mask indicating which slot is unavailable.
+  unsigned isSold : HEXAGON_PACKET_SIZE;
+
+public:
+  HexagonUnitAuction() : isSold(0){};
+
+  // Allocate slots.
+  bool bid(unsigned B) {
+    // Exclude already auctioned slots from the bid.
+    unsigned b = B & ~isSold;
+    if (b) {
+      for (unsigned i = 0; i < HEXAGON_PACKET_SIZE; ++i)
+        if (b & (1 << i)) {
+          // Request candidate slots.
+          Scores[i] += HexagonBid(b);
+          isSold |= Scores[i].isSold() << i;
+        }
+      return true;
+      ;
+    } else
+      // Error if the desired slots are already full.
+      return false;
+  };
+};
+
+unsigned HexagonResource::setWeight(unsigned s) {
+  const unsigned SlotWeight = 8;
+  const unsigned MaskWeight = SlotWeight - 1;
+  bool Key = (1 << s) & getUnits();
+
+  // TODO: Improve this API so that we can prevent misuse statically.
+  assert(SlotWeight * s < 32 && "Argument to setWeight too large.");
+
+  // Calculate relative weight of the insn for the given slot, weighing it the
+  // heavier the more restrictive the insn is and the lowest the slots that the
+  // insn may be executed in.
+  Weight =
+      (Key << (SlotWeight * s)) * ((MaskWeight - countPopulation(getUnits()))
+                                   << countTrailingZeros(getUnits()));
+  return (Weight);
+}
+
+HexagonShuffler::HexagonShuffler(MCInstrInfo const &MCII,
+                                 MCSubtargetInfo const &STI)
+    : MCII(MCII), STI(STI) {
+  reset();
+}
+
+void HexagonShuffler::reset() {
+  Packet.clear();
+  BundleFlags = 0;
+  Error = SHUFFLE_SUCCESS;
+}
+
+void HexagonShuffler::append(MCInst const *ID, MCInst const *Extender,
+                             unsigned S, bool X) {
+  HexagonInstr PI(ID, Extender, S, X);
+
+  Packet.push_back(PI);
+}
+
+/// Check that the packet is legal and enforce relative insn order.
+bool HexagonShuffler::check() {
+  // Descriptive slot masks.
+  const unsigned slotSingleLoad = 0x1, slotSingleStore = 0x1, slotOne = 0x2,
+                 slotThree = 0x8, slotFirstJump = 0x8, slotLastJump = 0x4,
+                 slotFirstLoadStore = 0x2, slotLastLoadStore = 0x1;
+  // Highest slots for branches and stores used to keep their original order.
+  unsigned slotJump = slotFirstJump;
+  unsigned slotLoadStore = slotFirstLoadStore;
+  // Number of branches, solo branches, indirect branches.
+  unsigned jumps = 0, jump1 = 0, jumpr = 0;
+  // Number of memory operations, loads, solo loads, stores, solo stores, single
+  // stores.
+  unsigned memory = 0, loads = 0, load0 = 0, stores = 0, store0 = 0, store1 = 0;
+  // Number of duplex insns, solo insns.
+  unsigned duplex = 0, solo = 0;
+  // Number of insns restricting other insns in the packet to A and X types,
+  // which is neither A or X types.
+  unsigned onlyAX = 0, neitherAnorX = 0;
+  // Number of insns restricting other insns in slot #1 to A type.
+  unsigned onlyAin1 = 0;
+  // Number of insns restricting any insn in slot #1, except A2_nop.
+  unsigned onlyNo1 = 0;
+  unsigned xtypeFloat = 0;
+  unsigned pSlot3Cnt = 0;
+  iterator slot3ISJ = end();
+
+  // Collect information from the insns in the packet.
+  for (iterator ISJ = begin(); ISJ != end(); ++ISJ) {
+    MCInst const *ID = ISJ->getDesc();
+
+    if (HexagonMCInstrInfo::isSolo(MCII, *ID))
+      solo += !ISJ->isSoloException();
+    else if (HexagonMCInstrInfo::isSoloAX(MCII, *ID))
+      onlyAX += !ISJ->isSoloException();
+    else if (HexagonMCInstrInfo::isSoloAin1(MCII, *ID))
+      onlyAin1 += !ISJ->isSoloException();
+    if (HexagonMCInstrInfo::getType(MCII, *ID) != HexagonII::TypeALU32 &&
+        HexagonMCInstrInfo::getType(MCII, *ID) != HexagonII::TypeXTYPE)
+      ++neitherAnorX;
+    if (HexagonMCInstrInfo::prefersSlot3(MCII, *ID)) {
+      ++pSlot3Cnt;
+      slot3ISJ = ISJ;
+    }
+
+    switch (HexagonMCInstrInfo::getType(MCII, *ID)) {
+    case HexagonII::TypeXTYPE:
+      if (HexagonMCInstrInfo::isFloat(MCII, *ID))
+        ++xtypeFloat;
+      break;
+    case HexagonII::TypeJR:
+      ++jumpr;
+    // Fall-through.
+    case HexagonII::TypeJ:
+      ++jumps;
+      break;
+    case HexagonII::TypeLD:
+      ++loads;
+      ++memory;
+      if (ISJ->Core.getUnits() == slotSingleLoad)
+        ++load0;
+      if (HexagonMCInstrInfo::getDesc(MCII, *ID).isReturn())
+        ++jumps, ++jump1; // DEALLOC_RETURN is of type LD.
+      break;
+    case HexagonII::TypeST:
+      ++stores;
+      ++memory;
+      if (ISJ->Core.getUnits() == slotSingleStore)
+        ++store0;
+      break;
+    case HexagonII::TypeMEMOP:
+      ++loads;
+      ++stores;
+      ++store1;
+      ++memory;
+      break;
+    case HexagonII::TypeNV:
+      ++memory; // NV insns are memory-like.
+      if (HexagonMCInstrInfo::getDesc(MCII, *ID).isBranch())
+        ++jumps, ++jump1;
+      break;
+    case HexagonII::TypeCR:
+    // Legacy conditional branch predicated on a register.
+    case HexagonII::TypeSYSTEM:
+      if (HexagonMCInstrInfo::getDesc(MCII, *ID).mayLoad())
+        ++loads;
+      break;
+    }
+  }
+
+  // Check if the packet is legal.
+  if ((load0 > 1 || store0 > 1) || (duplex > 1 || (duplex && memory)) ||
+      (solo && size() > 1) || (onlyAX && neitherAnorX > 1) ||
+      (onlyAX && xtypeFloat)) {
+    Error = SHUFFLE_ERROR_INVALID;
+    return false;
+  }
+
+  if (jump1 && jumps > 1) {
+    // Error if single branch with another branch.
+    Error = SHUFFLE_ERROR_BRANCHES;
+    return false;
+  }
+
+  // Modify packet accordingly.
+  // TODO: need to reserve slots #0 and #1 for duplex insns.
+  bool bOnlySlot3 = false;
+  for (iterator ISJ = begin(); ISJ != end(); ++ISJ) {
+    MCInst const *ID = ISJ->getDesc();
+
+    if (!ISJ->Core.getUnits()) {
+      // Error if insn may not be executed in any slot.
+      Error = SHUFFLE_ERROR_UNKNOWN;
+      return false;
+    }
+
+    // Exclude from slot #1 any insn but A2_nop.
+    if (HexagonMCInstrInfo::getDesc(MCII, *ID).getOpcode() != Hexagon::A2_nop)
+      if (onlyNo1)
+        ISJ->Core.setUnits(ISJ->Core.getUnits() & ~slotOne);
+
+    // Exclude from slot #1 any insn but A-type.
+    if (HexagonMCInstrInfo::getType(MCII, *ID) != HexagonII::TypeALU32)
+      if (onlyAin1)
+        ISJ->Core.setUnits(ISJ->Core.getUnits() & ~slotOne);
+
+    // Branches must keep the original order.
+    if (HexagonMCInstrInfo::getDesc(MCII, *ID).isBranch() ||
+        HexagonMCInstrInfo::getDesc(MCII, *ID).isCall())
+      if (jumps > 1) {
+        if (jumpr || slotJump < slotLastJump) {
+          // Error if indirect branch with another branch or
+          // no more slots available for branches.
+          Error = SHUFFLE_ERROR_BRANCHES;
+          return false;
+        }
+        // Pin the branch to the highest slot available to it.
+        ISJ->Core.setUnits(ISJ->Core.getUnits() & slotJump);
+        // Update next highest slot available to branches.
+        slotJump >>= 1;
+      }
+
+    // A single load must use slot #0.
+    if (HexagonMCInstrInfo::getDesc(MCII, *ID).mayLoad()) {
+      if (loads == 1 && loads == memory)
+        // Pin the load to slot #0.
+        ISJ->Core.setUnits(ISJ->Core.getUnits() & slotSingleLoad);
+    }
+
+    // A single store must use slot #0.
+    if (HexagonMCInstrInfo::getDesc(MCII, *ID).mayStore()) {
+      if (!store0) {
+        if (stores == 1)
+          ISJ->Core.setUnits(ISJ->Core.getUnits() & slotSingleStore);
+        else if (stores > 1) {
+          if (slotLoadStore < slotLastLoadStore) {
+            // Error if no more slots available for stores.
+            Error = SHUFFLE_ERROR_STORES;
+            return false;
+          }
+          // Pin the store to the highest slot available to it.
+          ISJ->Core.setUnits(ISJ->Core.getUnits() & slotLoadStore);
+          // Update the next highest slot available to stores.
+          slotLoadStore >>= 1;
+        }
+      }
+      if (store1 && stores > 1) {
+        // Error if a single store with another store.
+        Error = SHUFFLE_ERROR_STORES;
+        return false;
+      }
+    }
+
+    // flag if an instruction can only be executed in slot 3
+    if (ISJ->Core.getUnits() == slotThree)
+      bOnlySlot3 = true;
+
+    if (!ISJ->Core.getUnits()) {
+      // Error if insn may not be executed in any slot.
+      Error = SHUFFLE_ERROR_NOSLOTS;
+      return false;
+    }
+  }
+
+  bool validateSlots = true;
+  if (bOnlySlot3 == false && pSlot3Cnt == 1 && slot3ISJ != end()) {
+    // save off slot mask of instruction marked with A_PREFER_SLOT3
+    // and then pin it to slot #3
+    unsigned saveUnits = slot3ISJ->Core.getUnits();
+    slot3ISJ->Core.setUnits(saveUnits & slotThree);
+
+    HexagonUnitAuction AuctionCore;
+    std::sort(begin(), end(), HexagonInstr::lessCore);
+
+    // see if things ok with that instruction being pinned to slot #3
+    bool bFail = false;
+    for (iterator I = begin(); I != end() && bFail != true; ++I)
+      if (!AuctionCore.bid(I->Core.getUnits()))
+        bFail = true;
+
+    // if yes, great, if not then restore original slot mask
+    if (!bFail)
+      validateSlots = false; // all good, no need to re-do auction
+    else
+      for (iterator ISJ = begin(); ISJ != end(); ++ISJ) {
+        MCInst const *ID = ISJ->getDesc();
+        if (HexagonMCInstrInfo::prefersSlot3(MCII, *ID))
+          ISJ->Core.setUnits(saveUnits);
+      }
+  }
+
+  // Check if any slot, core, is over-subscribed.
+  // Verify the core slot subscriptions.
+  if (validateSlots) {
+    HexagonUnitAuction AuctionCore;
+
+    std::sort(begin(), end(), HexagonInstr::lessCore);
+
+    for (iterator I = begin(); I != end(); ++I)
+      if (!AuctionCore.bid(I->Core.getUnits())) {
+        Error = SHUFFLE_ERROR_SLOTS;
+        return false;
+      }
+  }
+
+  Error = SHUFFLE_SUCCESS;
+  return true;
+}
+
+bool HexagonShuffler::shuffle() {
+  if (size() > HEXAGON_PACKET_SIZE) {
+    // Ignore a packet with with more than what a packet can hold
+    // or with compound or duplex insns for now.
+    Error = SHUFFLE_ERROR_INVALID;
+    return false;
+  }
+
+  // Check and prepare packet.
+  if (size() > 1 && check())
+    // Reorder the handles for each slot.
+    for (unsigned nSlot = 0, emptySlots = 0; nSlot < HEXAGON_PACKET_SIZE;
+         ++nSlot) {
+      iterator ISJ, ISK;
+      unsigned slotSkip, slotWeight;
+
+      // Prioritize the handles considering their restrictions.
+      for (ISJ = ISK = Packet.begin(), slotSkip = slotWeight = 0;
+           ISK != Packet.end(); ++ISK, ++slotSkip)
+        if (slotSkip < nSlot - emptySlots)
+          // Note which handle to begin at.
+          ++ISJ;
+        else
+          // Calculate the weight of the slot.
+          slotWeight += ISK->Core.setWeight(HEXAGON_PACKET_SIZE - nSlot - 1);
+
+      if (slotWeight)
+        // Sort the packet, favoring source order,
+        // beginning after the previous slot.
+        std::sort(ISJ, Packet.end());
+      else
+        // Skip unused slot.
+        ++emptySlots;
+    }
+
+  for (iterator ISJ = begin(); ISJ != end(); ++ISJ)
+    DEBUG(dbgs().write_hex(ISJ->Core.getUnits());
+          dbgs() << ':'
+                 << HexagonMCInstrInfo::getDesc(MCII, *ISJ->getDesc())
+                        .getOpcode();
+          dbgs() << '\n');
+  DEBUG(dbgs() << '\n');
+
+  return (!getError());
+}
diff --git a/lib/Target/Hexagon/MCTargetDesc/HexagonShuffler.h b/lib/Target/Hexagon/MCTargetDesc/HexagonShuffler.h
new file mode 100644
index 00000000000..8b6c72ee25e
--- /dev/null
+++ b/lib/Target/Hexagon/MCTargetDesc/HexagonShuffler.h
@@ -0,0 +1,138 @@
+//===----- HexagonShuffler.h - Instruction bundle shuffling ---------------===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This implements the shuffling of insns inside a bundle according to the
+// packet formation rules of the Hexagon ISA.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef HEXAGONSHUFFLER_H
+#define HEXAGONSHUFFLER_H
+
+#include "Hexagon.h"
+#include "MCTargetDesc/HexagonMCInstrInfo.h"
+
+#include "llvm/ADT/SmallVector.h"
+#include "llvm/MC/MCInstrInfo.h"
+
+using namespace llvm;
+
+namespace llvm {
+// Insn resources.
+class HexagonResource {
+  // Mask of the slots or units that may execute the insn and
+  // the weight or priority that the insn requires to be assigned a slot.
+  unsigned Slots, Weight;
+
+public:
+  HexagonResource(unsigned s) { setUnits(s); };
+
+  void setUnits(unsigned s) {
+    Slots = s & ~(~0U << HEXAGON_PACKET_SIZE);
+  };
+  unsigned setWeight(unsigned s);
+
+  unsigned getUnits() const { return (Slots); };
+  unsigned getWeight() const { return (Weight); };
+
+  // Check if the resources are in ascending slot order.
+  static bool lessUnits(const HexagonResource &A, const HexagonResource &B) {
+    return (countPopulation(A.getUnits()) < countPopulation(B.getUnits()));
+  };
+  // Check if the resources are in ascending weight order.
+  static bool lessWeight(const HexagonResource &A, const HexagonResource &B) {
+    return (A.getWeight() < B.getWeight());
+  };
+};
+
+// Handle to an insn used by the shuffling algorithm.
+class HexagonInstr {
+  friend class HexagonShuffler;
+
+  MCInst const *ID;
+  MCInst const *Extender;
+  HexagonResource Core;
+  bool SoloException;
+
+public:
+  HexagonInstr(MCInst const *id, MCInst const *Extender, unsigned s,
+               bool x = false)
+      : ID(id), Extender(Extender), Core(s), SoloException(x){};
+
+  MCInst const *getDesc() const { return (ID); };
+
+  MCInst const *getExtender() const { return Extender; }
+
+  unsigned isSoloException() const { return (SoloException); };
+
+  // Check if the handles are in ascending order for shuffling purposes.
+  bool operator<(const HexagonInstr &B) const {
+    return (HexagonResource::lessWeight(B.Core, Core));
+  };
+  // Check if the handles are in ascending order by core slots.
+  static bool lessCore(const HexagonInstr &A, const HexagonInstr &B) {
+    return (HexagonResource::lessUnits(A.Core, B.Core));
+  };
+};
+
+// Bundle shuffler.
+class HexagonShuffler {
+  typedef SmallVector<HexagonInstr, HEXAGON_PRESHUFFLE_PACKET_SIZE>
+      HexagonPacket;
+
+  // Insn handles in a bundle.
+  HexagonPacket Packet;
+
+  // Shuffling error code.
+  unsigned Error;
+
+protected:
+  int64_t BundleFlags;
+  MCInstrInfo const &MCII;
+  MCSubtargetInfo const &STI;
+
+public:
+  typedef HexagonPacket::iterator iterator;
+
+  enum {
+    SHUFFLE_SUCCESS = 0,    ///< Successful operation.
+    SHUFFLE_ERROR_INVALID,  ///< Invalid bundle.
+    SHUFFLE_ERROR_STORES,   ///< No free slots for store insns.
+    SHUFFLE_ERROR_LOADS,    ///< No free slots for load insns.
+    SHUFFLE_ERROR_BRANCHES, ///< No free slots for branch insns.
+    SHUFFLE_ERROR_NOSLOTS,  ///< No free slots for other insns.
+    SHUFFLE_ERROR_SLOTS,    ///< Over-subscribed slots.
+    SHUFFLE_ERROR_UNKNOWN   ///< Unknown error.
+  };
+
+  explicit HexagonShuffler(MCInstrInfo const &MCII, MCSubtargetInfo const &STI);
+
+  // Reset to initial state.
+  void reset();
+  // Check if the bundle may be validly shuffled.
+  bool check();
+  // Reorder the insn handles in the bundle.
+  bool shuffle();
+
+  unsigned size() const { return (Packet.size()); };
+
+  iterator begin() { return (Packet.begin()); };
+  iterator end() { return (Packet.end()); };
+
+  // Add insn handle to the bundle .
+  void append(MCInst const *ID, MCInst const *Extender, unsigned S,
+              bool X = false);
+
+  // Return the error code for the last check or shuffling of the bundle.
+  void setError(unsigned Err) { Error = Err; };
+  unsigned getError() const { return (Error); };
+};
+}
+
+#endif // HEXAGONSHUFFLER_H
diff --git a/lib/Target/Hexagon/Makefile b/lib/Target/Hexagon/Makefile
index 329c9d3018f..04b41e5986a 100644
--- a/lib/Target/Hexagon/Makefile
+++ b/lib/Target/Hexagon/Makefile
@@ -14,12 +14,12 @@ TARGET = Hexagon
 BUILT_SOURCES = HexagonGenRegisterInfo.inc \
                 HexagonGenInstrInfo.inc  \
                 HexagonGenAsmWriter.inc \
-                HexagonGenDAGISel.inc HexagonGenSubtargetInfo.inc \
-                HexagonGenCallingConv.inc \
-                HexagonGenDFAPacketizer.inc \
-                HexagonGenMCCodeEmitter.inc \
-                HexagonGenDisassemblerTables.inc
-
-DIRS = TargetInfo MCTargetDesc Disassembler
-
-include $(LEVEL)/Makefile.common
+                HexagonGenDAGISel.inc HexagonGenSubtargetInfo.inc \
+                HexagonGenCallingConv.inc \
+                HexagonGenDFAPacketizer.inc \
+                HexagonGenMCCodeEmitter.inc \
+                HexagonGenDisassemblerTables.inc
+
+DIRS = TargetInfo MCTargetDesc Disassembler
+
+include $(LEVEL)/Makefile.common
diff --git a/lib/Target/JSBackend/JSBackend.cpp b/lib/Target/JSBackend/JSBackend.cpp
index 836f34ce9e0..329d81ffb41 100644
--- a/lib/Target/JSBackend/JSBackend.cpp
+++ b/lib/Target/JSBackend/JSBackend.cpp
@@ -711,7 +711,7 @@ static void emitDebugInfo(raw_ostream& Code, const Instruction *I) {
   auto &Loc = I->getDebugLoc();
   if (Loc) {
     unsigned Line = Loc.getLine();
-    auto *Scope = cast_or_null<MDScope>(Loc.getScope());
+    auto *Scope = cast_or_null<DIScope>(Loc.getScope());
     if (Scope) {
       StringRef File = Scope->getFilename();
       if (Line > 0)
@@ -3080,6 +3080,7 @@ void JSWriter::printModuleBody() {
       // which doesn't require the intrinsic function itself to be declared.
       if (I->isIntrinsic()) {
         switch (I->getIntrinsicID()) {
+        default: break;
         case Intrinsic::dbg_declare:
         case Intrinsic::dbg_value:
         case Intrinsic::lifetime_start:
@@ -3600,12 +3601,11 @@ Pass *createCheckTriplePass() {
 //                       External Interface declaration
 //===----------------------------------------------------------------------===//
 
-bool JSTargetMachine::addPassesToEmitFile(PassManagerBase &PM,
-                                          raw_pwrite_stream &o,
-                                          CodeGenFileType FileType,
-                                          bool DisableVerify,
-                                          AnalysisID StartAfter,
-                                          AnalysisID StopAfter) {
+bool JSTargetMachine::addPassesToEmitFile(
+      PassManagerBase &PM, raw_pwrite_stream &Out, CodeGenFileType FileType,
+      bool DisableVerify, AnalysisID StartBefore,
+      AnalysisID StartAfter, AnalysisID StopAfter,
+      MachineFunctionInitializer *MFInitializer) {
   assert(FileType == TargetMachine::CGFT_AssemblyFile);
 
   PM.add(createCheckTriplePass());
@@ -3623,7 +3623,7 @@ bool JSTargetMachine::addPassesToEmitFile(PassManagerBase &PM,
   PM.add(createEmscriptenRemoveLLVMAssumePass());
   PM.add(createEmscriptenExpandBigSwitchesPass());
 
-  PM.add(new JSWriter(o, OptLevel));
+  PM.add(new JSWriter(Out, OptLevel));
 
   return false;
 }
diff --git a/lib/Target/JSBackend/JSTargetMachine.cpp b/lib/Target/JSBackend/JSTargetMachine.cpp
index 9676a905cc8..c14717ae25b 100644
--- a/lib/Target/JSBackend/JSTargetMachine.cpp
+++ b/lib/Target/JSBackend/JSTargetMachine.cpp
@@ -17,20 +17,33 @@
 #include "llvm/Support/TargetRegistry.h"
 using namespace llvm;
 
-JSTargetMachine::JSTargetMachine(const Target &T, StringRef Triple,
+extern const llvm::SubtargetFeatureKV JSSubTypeKV[] = {
+  { "asmjs", "Select the asmjs processor", { }, { } }
+};
+
+static const llvm::SubtargetInfoKV JSProcSchedModels[] = {
+  { "asmjs", &MCSchedModel::GetDefaultSchedModel() }
+};
+
+JSSubtarget::JSSubtarget(const TargetMachine& TM, const Triple &TT) :
+  TargetSubtargetInfo(TT, "asmjs", "asmjs", None, makeArrayRef(JSSubTypeKV, 1), JSProcSchedModels, nullptr, nullptr, nullptr, nullptr, nullptr, nullptr),
+  TL(TM)
+ {}
+
+
+JSTargetMachine::JSTargetMachine(const Target &T, const Triple &TT,
                                  StringRef CPU, StringRef FS, const TargetOptions &Options,
                                  Reloc::Model RM, CodeModel::Model CM,
                                  CodeGenOpt::Level OL)
-    : TargetMachine(T, "e-p:32:32-i64:64-v128:32:128-n32-S128", Triple, CPU,
+    : TargetMachine(T, "e-p:32:32-i64:64-v128:32:128-n32-S128", TT, CPU,
                     FS, Options),
-      ST(*this) {
-  CodeGenInfo = T.createMCCodeGenInfo(Triple, RM, CM, OL);
+      ST(*this, TT) {
+  CodeGenInfo = T.createMCCodeGenInfo("asmjs", RM, CM, OL);
 }
 
-
 TargetIRAnalysis JSTargetMachine::getTargetIRAnalysis() {
   return TargetIRAnalysis([this](Function &F) {
-    return TargetTransformInfo(JSTTIImpl(this));
+    return TargetTransformInfo(JSTTIImpl(this, F));
   });
 }
 
diff --git a/lib/Target/JSBackend/JSTargetMachine.h b/lib/Target/JSBackend/JSTargetMachine.h
index 622e8306252..055eb0c8601 100644
--- a/lib/Target/JSBackend/JSTargetMachine.h
+++ b/lib/Target/JSBackend/JSTargetMachine.h
@@ -33,7 +33,7 @@ class JSSubtarget : public TargetSubtargetInfo {
   JSTargetLowering TL;
 
 public:
-  JSSubtarget(const TargetMachine& TM) : TL(TM) {}
+  JSSubtarget(const TargetMachine& TM, const Triple &TT);
 
   const TargetLowering *getTargetLowering() const override {
     return &TL;
@@ -44,21 +44,26 @@ class JSTargetMachine : public TargetMachine {
   const JSSubtarget ST;
 
 public:
-  JSTargetMachine(const Target &T, StringRef Triple,
+  JSTargetMachine(const Target &T, const Triple &TT,
                   StringRef CPU, StringRef FS, const TargetOptions &Options,
                   Reloc::Model RM, CodeModel::Model CM,
                   CodeGenOpt::Level OL);
 
-  bool addPassesToEmitFile(PassManagerBase &PM, raw_pwrite_stream &Out,
-                           CodeGenFileType FileType, bool DisableVerify,
-                           AnalysisID StartAfter,
-                           AnalysisID StopAfter) override;
+  bool addPassesToEmitFile(
+      PassManagerBase &PM, raw_pwrite_stream &Out, CodeGenFileType FileType,
+      bool DisableVerify = true, AnalysisID StartBefore = nullptr,
+      AnalysisID StartAfter = nullptr, AnalysisID StopAfter = nullptr,
+      MachineFunctionInitializer *MFInitializer = nullptr) override;
 
   TargetIRAnalysis getTargetIRAnalysis() override;
 
   const TargetSubtargetInfo *getJSSubtargetImpl() const {
     return &ST;
   }
+
+  const JSSubtarget *getSubtargetImpl(const Function &F) const override {
+    return &ST;
+  }
 };
 
 } // End llvm namespace
diff --git a/lib/Target/JSBackend/JSTargetTransformInfo.cpp b/lib/Target/JSBackend/JSTargetTransformInfo.cpp
index 204c3c53137..c1e29fc3d8b 100644
--- a/lib/Target/JSBackend/JSTargetTransformInfo.cpp
+++ b/lib/Target/JSBackend/JSTargetTransformInfo.cpp
@@ -35,6 +35,12 @@ void JSTTIImpl::getUnrollingPreferences(Loop *L,
   UP.Runtime = false;
 }
 
+unsigned JSTTIImpl::getNumberOfRegisters(bool Vector) {
+  if (Vector) return 16; // like NEON, x86_64, etc.
+
+  return 8; // like x86, thumb, etc.
+}
+
 unsigned JSTTIImpl::getRegisterBitWidth(bool Vector) {
   if (Vector) {
     return 128;
diff --git a/lib/Target/JSBackend/JSTargetTransformInfo.h b/lib/Target/JSBackend/JSTargetTransformInfo.h
index 8c8bfde19aa..c8e707c3c84 100644
--- a/lib/Target/JSBackend/JSTargetTransformInfo.h
+++ b/lib/Target/JSBackend/JSTargetTransformInfo.h
@@ -36,11 +36,9 @@ class JSTTIImpl : public BasicTTIImplBase<JSTTIImpl> {
   const TargetLoweringBase *getTLI() const { return TLI; }
 
 public:
-  explicit JSTTIImpl(const JSTargetMachine *TM)
-      : BaseT(TM), ST(TM->getJSSubtargetImpl()), TLI(ST->getTargetLowering()) {
-    assert(ST);
-    assert(TLI);
-  }
+  explicit JSTTIImpl(const JSTargetMachine *TM, Function &F)
+      : BaseT(TM, F.getParent()->getDataLayout()), ST(TM->getSubtargetImpl(F)),
+        TLI(ST->getTargetLowering()) {}
 
   // Provide value semantics. MSVC requires that we spell all of these out.
   JSTTIImpl(const JSTTIImpl &Arg)
@@ -48,6 +46,7 @@ class JSTTIImpl : public BasicTTIImplBase<JSTTIImpl> {
   JSTTIImpl(JSTTIImpl &&Arg)
       : BaseT(std::move(static_cast<BaseT &>(Arg))), ST(std::move(Arg.ST)),
         TLI(std::move(Arg.TLI)) {}
+/*
   JSTTIImpl &operator=(const JSTTIImpl &RHS) {
     BaseT::operator=(static_cast<const BaseT &>(RHS));
     ST = RHS.ST;
@@ -60,6 +59,7 @@ class JSTTIImpl : public BasicTTIImplBase<JSTTIImpl> {
     TLI = std::move(RHS.TLI);
     return *this;
   }
+*/
 
   bool hasBranchDivergence() { return true; }
 
@@ -72,6 +72,8 @@ class JSTTIImpl : public BasicTTIImplBase<JSTTIImpl> {
     return TargetTransformInfo::PSK_Software;
   }
 
+  unsigned getNumberOfRegisters(bool Vector);
+
   unsigned getRegisterBitWidth(bool Vector);
 
   unsigned getArithmeticInstrCost(
diff --git a/lib/Target/JSBackend/MCTargetDesc/JSBackendMCTargetDesc.cpp b/lib/Target/JSBackend/MCTargetDesc/JSBackendMCTargetDesc.cpp
index f7ba0686839..687357df811 100644
--- a/lib/Target/JSBackend/MCTargetDesc/JSBackendMCTargetDesc.cpp
+++ b/lib/Target/JSBackend/MCTargetDesc/JSBackendMCTargetDesc.cpp
@@ -16,11 +16,12 @@
 #include "llvm/Support/TargetRegistry.h"
 using namespace llvm;
 
-static MCCodeGenInfo *createJSBackendMCCodeGenInfo(StringRef Triple, Reloc::Model RM,
+
+static MCCodeGenInfo *createJSBackendMCCodeGenInfo(const Triple &TT, Reloc::Model RM,
                                                    CodeModel::Model CM,
                                                    CodeGenOpt::Level OL) {
   MCCodeGenInfo *X = new MCCodeGenInfo();
-  X->InitMCCodeGenInfo(RM, CM, OL);
+  X->initMCCodeGenInfo(RM, CM, OL);
   return X;
 }
 
diff --git a/lib/Target/LLVMBuild.txt b/lib/Target/LLVMBuild.txt
index e5496698be5..276f8292ad5 100644
--- a/lib/Target/LLVMBuild.txt
+++ b/lib/Target/LLVMBuild.txt
@@ -15,8 +15,26 @@
 ;
 ;===------------------------------------------------------------------------===;
 
+; Please keep these as one per line so that out-of-tree merges
+; will typically require only insertion of a line.
 [common]
-subdirectories = JSBackend ARM AArch64 BPF CppBackend Hexagon MSP430 NVPTX Mips PowerPC R600 Sparc SystemZ X86 XCore
+subdirectories =
+ AMDGPU
+ ARM
+ AArch64
+ BPF
+ CppBackend
+ JSBackend
+ Hexagon
+ MSP430
+ NVPTX
+ Mips
+ PowerPC
+ Sparc
+ SystemZ
+ WebAssembly
+ X86
+ XCore
 
 ; This is a special group whose required libraries are extended (by llvm-build)
 ; with the best execution engine (the native JIT, if available, or the
diff --git a/lib/Target/MSP430/CMakeLists.txt b/lib/Target/MSP430/CMakeLists.txt
index a8f9b52746a..3f377631c01 100644
--- a/lib/Target/MSP430/CMakeLists.txt
+++ b/lib/Target/MSP430/CMakeLists.txt
@@ -18,7 +18,6 @@ add_llvm_target(MSP430CodeGen
   MSP430RegisterInfo.cpp
   MSP430Subtarget.cpp
   MSP430TargetMachine.cpp
-  MSP430SelectionDAGInfo.cpp
   MSP430AsmPrinter.cpp
   MSP430MCInstLower.cpp
   )
diff --git a/lib/Target/MSP430/InstPrinter/MSP430InstPrinter.cpp b/lib/Target/MSP430/InstPrinter/MSP430InstPrinter.cpp
index 6c43d97837c..be6d1a84a37 100644
--- a/lib/Target/MSP430/InstPrinter/MSP430InstPrinter.cpp
+++ b/lib/Target/MSP430/InstPrinter/MSP430InstPrinter.cpp
@@ -39,7 +39,7 @@ void MSP430InstPrinter::printPCRelImmOperand(const MCInst *MI, unsigned OpNo,
     O << Op.getImm();
   else {
     assert(Op.isExpr() && "unknown pcrel immediate operand");
-    O << *Op.getExpr();
+    Op.getExpr()->print(O, &MAI);
   }
 }
 
@@ -53,7 +53,8 @@ void MSP430InstPrinter::printOperand(const MCInst *MI, unsigned OpNo,
     O << '#' << Op.getImm();
   } else {
     assert(Op.isExpr() && "unknown operand kind in printOperand");
-    O << '#' << *Op.getExpr();
+    O << '#';
+    Op.getExpr()->print(O, &MAI);
   }
 }
 
@@ -75,7 +76,7 @@ void MSP430InstPrinter::printSrcMemOperand(const MCInst *MI, unsigned OpNo,
     O << '&';
 
   if (Disp.isExpr())
-    O << *Disp.getExpr();
+    Disp.getExpr()->print(O, &MAI);
   else {
     assert(Disp.isImm() && "Expected immediate in displacement field");
     O << Disp.getImm();
diff --git a/lib/Target/MSP430/MCTargetDesc/LLVMBuild.txt b/lib/Target/MSP430/MCTargetDesc/LLVMBuild.txt
index b8f3d02ab4d..a305b2db868 100644
--- a/lib/Target/MSP430/MCTargetDesc/LLVMBuild.txt
+++ b/lib/Target/MSP430/MCTargetDesc/LLVMBuild.txt
@@ -19,5 +19,5 @@
 type = Library
 name = MSP430Desc
 parent = MSP430
-required_libraries = MC MSP430AsmPrinter MSP430Info
+required_libraries = MC MSP430AsmPrinter MSP430Info Support
 add_to_library_groups = MSP430
diff --git a/lib/Target/MSP430/MCTargetDesc/MSP430MCAsmInfo.cpp b/lib/Target/MSP430/MCTargetDesc/MSP430MCAsmInfo.cpp
index df1aa1a41f1..c26b3081dbc 100644
--- a/lib/Target/MSP430/MCTargetDesc/MSP430MCAsmInfo.cpp
+++ b/lib/Target/MSP430/MCTargetDesc/MSP430MCAsmInfo.cpp
@@ -12,12 +12,11 @@
 //===----------------------------------------------------------------------===//
 
 #include "MSP430MCAsmInfo.h"
-#include "llvm/ADT/StringRef.h"
 using namespace llvm;
 
 void MSP430MCAsmInfo::anchor() { }
 
-MSP430MCAsmInfo::MSP430MCAsmInfo(StringRef TT) {
+MSP430MCAsmInfo::MSP430MCAsmInfo(const Triple &TT) {
   PointerSize = CalleeSaveStackSlotSize = 2;
 
   CommentString = ";";
diff --git a/lib/Target/MSP430/MCTargetDesc/MSP430MCAsmInfo.h b/lib/Target/MSP430/MCTargetDesc/MSP430MCAsmInfo.h
index 2c9532d321e..ff5b0b6d858 100644
--- a/lib/Target/MSP430/MCTargetDesc/MSP430MCAsmInfo.h
+++ b/lib/Target/MSP430/MCTargetDesc/MSP430MCAsmInfo.h
@@ -17,12 +17,12 @@
 #include "llvm/MC/MCAsmInfoELF.h"
 
 namespace llvm {
-  class StringRef;
+  class Triple;
 
   class MSP430MCAsmInfo : public MCAsmInfoELF {
     void anchor() override;
   public:
-    explicit MSP430MCAsmInfo(StringRef TT);
+    explicit MSP430MCAsmInfo(const Triple &TT);
   };
 
 } // namespace llvm
diff --git a/lib/Target/MSP430/MCTargetDesc/MSP430MCTargetDesc.cpp b/lib/Target/MSP430/MCTargetDesc/MSP430MCTargetDesc.cpp
index 775c0b2806a..807d1129b5f 100644
--- a/lib/Target/MSP430/MCTargetDesc/MSP430MCTargetDesc.cpp
+++ b/lib/Target/MSP430/MCTargetDesc/MSP430MCTargetDesc.cpp
@@ -37,24 +37,23 @@ static MCInstrInfo *createMSP430MCInstrInfo() {
   return X;
 }
 
-static MCRegisterInfo *createMSP430MCRegisterInfo(StringRef TT) {
+static MCRegisterInfo *createMSP430MCRegisterInfo(const Triple &TT) {
   MCRegisterInfo *X = new MCRegisterInfo();
   InitMSP430MCRegisterInfo(X, MSP430::PC);
   return X;
 }
 
-static MCSubtargetInfo *createMSP430MCSubtargetInfo(StringRef TT, StringRef CPU,
-                                                    StringRef FS) {
-  MCSubtargetInfo *X = new MCSubtargetInfo();
-  InitMSP430MCSubtargetInfo(X, TT, CPU, FS);
-  return X;
+static MCSubtargetInfo *
+createMSP430MCSubtargetInfo(const Triple &TT, StringRef CPU, StringRef FS) {
+  return createMSP430MCSubtargetInfoImpl(TT, CPU, FS);
 }
 
-static MCCodeGenInfo *createMSP430MCCodeGenInfo(StringRef TT, Reloc::Model RM,
+static MCCodeGenInfo *createMSP430MCCodeGenInfo(const Triple &TT,
+                                                Reloc::Model RM,
                                                 CodeModel::Model CM,
                                                 CodeGenOpt::Level OL) {
   MCCodeGenInfo *X = new MCCodeGenInfo();
-  X->InitMCCodeGenInfo(RM, CM, OL);
+  X->initMCCodeGenInfo(RM, CM, OL);
   return X;
 }
 
diff --git a/lib/Target/MSP430/MSP430AsmPrinter.cpp b/lib/Target/MSP430/MSP430AsmPrinter.cpp
index fb7823e306b..4342c10a1bf 100644
--- a/lib/Target/MSP430/MSP430AsmPrinter.cpp
+++ b/lib/Target/MSP430/MSP430AsmPrinter.cpp
@@ -75,7 +75,7 @@ void MSP430AsmPrinter::printOperand(const MachineInstr *MI, int OpNum,
     O << MO.getImm();
     return;
   case MachineOperand::MO_MachineBasicBlock:
-    O << *MO.getMBB()->getSymbol();
+    MO.getMBB()->getSymbol()->print(O, MAI);
     return;
   case MachineOperand::MO_GlobalAddress: {
     bool isMemOp  = Modifier && !strcmp(Modifier, "mem");
@@ -92,7 +92,7 @@ void MSP430AsmPrinter::printOperand(const MachineInstr *MI, int OpNum,
     if (Offset)
       O << '(' << Offset << '+';
 
-    O << *getSymbol(MO.getGlobal());
+    getSymbol(MO.getGlobal())->print(O, MAI);
 
     if (Offset)
       O << ')';
@@ -152,7 +152,7 @@ void MSP430AsmPrinter::EmitInstruction(const MachineInstr *MI) {
 
   MCInst TmpInst;
   MCInstLowering.Lower(MI, TmpInst);
-  EmitToStreamer(OutStreamer, TmpInst);
+  EmitToStreamer(*OutStreamer, TmpInst);
 }
 
 // Force static initialization.
diff --git a/lib/Target/MSP430/MSP430FrameLowering.cpp b/lib/Target/MSP430/MSP430FrameLowering.cpp
index d6cb9f6459a..eb720809e47 100644
--- a/lib/Target/MSP430/MSP430FrameLowering.cpp
+++ b/lib/Target/MSP430/MSP430FrameLowering.cpp
@@ -39,8 +39,9 @@ bool MSP430FrameLowering::hasReservedCallFrame(const MachineFunction &MF) const
   return !MF.getFrameInfo()->hasVarSizedObjects();
 }
 
-void MSP430FrameLowering::emitPrologue(MachineFunction &MF) const {
-  MachineBasicBlock &MBB = MF.front();   // Prolog goes in entry BB
+void MSP430FrameLowering::emitPrologue(MachineFunction &MF,
+                                       MachineBasicBlock &MBB) const {
+  assert(&MF.front() == &MBB && "Shrink-wrapping not yet supported");
   MachineFrameInfo *MFI = MF.getFrameInfo();
   MSP430MachineFunctionInfo *MSP430FI = MF.getInfo<MSP430MachineFunctionInfo>();
   const MSP430InstrInfo &TII =
diff --git a/lib/Target/MSP430/MSP430FrameLowering.h b/lib/Target/MSP430/MSP430FrameLowering.h
index 1941af26e85..48c4dc866a6 100644
--- a/lib/Target/MSP430/MSP430FrameLowering.h
+++ b/lib/Target/MSP430/MSP430FrameLowering.h
@@ -27,7 +27,7 @@ class MSP430FrameLowering : public TargetFrameLowering {
 
   /// emitProlog/emitEpilog - These methods insert prolog and epilog code into
   /// the function.
-  void emitPrologue(MachineFunction &MF) const override;
+  void emitPrologue(MachineFunction &MF, MachineBasicBlock &MBB) const override;
   void emitEpilogue(MachineFunction &MF, MachineBasicBlock &MBB) const override;
 
   void eliminateCallFramePseudoInstr(MachineFunction &MF,
diff --git a/lib/Target/MSP430/MSP430ISelDAGToDAG.cpp b/lib/Target/MSP430/MSP430ISelDAGToDAG.cpp
index 591ceb55d57..8a01334ee2d 100644
--- a/lib/Target/MSP430/MSP430ISelDAGToDAG.cpp
+++ b/lib/Target/MSP430/MSP430ISelDAGToDAG.cpp
@@ -254,10 +254,11 @@ bool MSP430DAGToDAGISel::SelectAddr(SDValue N,
       AM.Base.Reg = CurDAG->getRegister(0, VT);
   }
 
-  Base  = (AM.BaseType == MSP430ISelAddressMode::FrameIndexBase) ?
-    CurDAG->getTargetFrameIndex(AM.Base.FrameIndex,
-                                getTargetLowering()->getPointerTy()) :
-    AM.Base.Reg;
+  Base = (AM.BaseType == MSP430ISelAddressMode::FrameIndexBase)
+             ? CurDAG->getTargetFrameIndex(
+                   AM.Base.FrameIndex,
+                   getTargetLowering()->getPointerTy(CurDAG->getDataLayout()))
+             : AM.Base.Reg;
 
   if (AM.GV)
     Disp = CurDAG->getTargetGlobalAddress(AM.GV, SDLoc(N),
@@ -274,7 +275,7 @@ bool MSP430DAGToDAGISel::SelectAddr(SDValue N,
     Disp = CurDAG->getTargetBlockAddress(AM.BlockAddr, MVT::i32, 0,
                                          0/*AM.SymbolFlags*/);
   else
-    Disp = CurDAG->getTargetConstant(AM.Disp, MVT::i16);
+    Disp = CurDAG->getTargetConstant(AM.Disp, SDLoc(N), MVT::i16);
 
   return true;
 }
@@ -401,10 +402,10 @@ SDNode *MSP430DAGToDAGISel::Select(SDNode *Node) {
     int FI = cast<FrameIndexSDNode>(Node)->getIndex();
     SDValue TFI = CurDAG->getTargetFrameIndex(FI, MVT::i16);
     if (Node->hasOneUse())
-      return CurDAG->SelectNodeTo(Node, MSP430::ADD16ri, MVT::i16,
-                                  TFI, CurDAG->getTargetConstant(0, MVT::i16));
-    return CurDAG->getMachineNode(MSP430::ADD16ri, dl, MVT::i16,
-                                  TFI, CurDAG->getTargetConstant(0, MVT::i16));
+      return CurDAG->SelectNodeTo(Node, MSP430::ADD16ri, MVT::i16, TFI,
+                                  CurDAG->getTargetConstant(0, dl, MVT::i16));
+    return CurDAG->getMachineNode(MSP430::ADD16ri, dl, MVT::i16, TFI,
+                                  CurDAG->getTargetConstant(0, dl, MVT::i16));
   }
   case ISD::LOAD:
     if (SDNode *ResNode = SelectIndexedLoad(Node))
diff --git a/lib/Target/MSP430/MSP430ISelLowering.cpp b/lib/Target/MSP430/MSP430ISelLowering.cpp
index 08f41a80971..29bc8b33988 100644
--- a/lib/Target/MSP430/MSP430ISelLowering.cpp
+++ b/lib/Target/MSP430/MSP430ISelLowering.cpp
@@ -213,7 +213,7 @@ SDValue MSP430TargetLowering::LowerOperation(SDValue Op,
 /// getConstraintType - Given a constraint letter, return the type of
 /// constraint it is for this target.
 TargetLowering::ConstraintType
-MSP430TargetLowering::getConstraintType(const std::string &Constraint) const {
+MSP430TargetLowering::getConstraintType(StringRef Constraint) const {
   if (Constraint.size() == 1) {
     switch (Constraint[0]) {
     case 'r':
@@ -227,8 +227,7 @@ MSP430TargetLowering::getConstraintType(const std::string &Constraint) const {
 
 std::pair<unsigned, const TargetRegisterClass *>
 MSP430TargetLowering::getRegForInlineAsmConstraint(
-    const TargetRegisterInfo *TRI, const std::string &Constraint,
-    MVT VT) const {
+    const TargetRegisterInfo *TRI, StringRef Constraint, MVT VT) const {
   if (Constraint.size() == 1) {
     // GCC Constraint Letters
     switch (Constraint[0]) {
@@ -494,7 +493,7 @@ MSP430TargetLowering::LowerCCCArguments(SDValue Chain,
       if (Flags.isByVal()) {
         int FI = MFI->CreateFixedObject(Flags.getByValSize(),
                                         VA.getLocMemOffset(), true);
-        InVal = DAG.getFrameIndex(FI, getPointerTy());
+        InVal = DAG.getFrameIndex(FI, getPointerTy(DAG.getDataLayout()));
       } else {
         // Load the argument to a virtual register
         unsigned ObjSize = VA.getLocVT().getSizeInBits()/8;
@@ -592,10 +591,10 @@ MSP430TargetLowering::LowerCCCCallTo(SDValue Chain, SDValue Callee,
 
   // Get a count of how many bytes are to be pushed on the stack.
   unsigned NumBytes = CCInfo.getNextStackOffset();
+  auto PtrVT = getPointerTy(DAG.getDataLayout());
 
-  Chain = DAG.getCALLSEQ_START(Chain ,DAG.getConstant(NumBytes,
-                                                      getPointerTy(), true),
-                               dl);
+  Chain = DAG.getCALLSEQ_START(Chain,
+                               DAG.getConstant(NumBytes, dl, PtrVT, true), dl);
 
   SmallVector<std::pair<unsigned, SDValue>, 4> RegsToPass;
   SmallVector<SDValue, 12> MemOpChains;
@@ -630,17 +629,17 @@ MSP430TargetLowering::LowerCCCCallTo(SDValue Chain, SDValue Callee,
       assert(VA.isMemLoc());
 
       if (!StackPtr.getNode())
-        StackPtr = DAG.getCopyFromReg(Chain, dl, MSP430::SP, getPointerTy());
+        StackPtr = DAG.getCopyFromReg(Chain, dl, MSP430::SP, PtrVT);
 
-      SDValue PtrOff = DAG.getNode(ISD::ADD, dl, getPointerTy(),
-                                   StackPtr,
-                                   DAG.getIntPtrConstant(VA.getLocMemOffset()));
+      SDValue PtrOff =
+          DAG.getNode(ISD::ADD, dl, PtrVT, StackPtr,
+                      DAG.getIntPtrConstant(VA.getLocMemOffset(), dl));
 
       SDValue MemOp;
       ISD::ArgFlagsTy Flags = Outs[i].Flags;
 
       if (Flags.isByVal()) {
-        SDValue SizeNode = DAG.getConstant(Flags.getByValSize(), MVT::i16);
+        SDValue SizeNode = DAG.getConstant(Flags.getByValSize(), dl, MVT::i16);
         MemOp = DAG.getMemcpy(Chain, dl, PtrOff, Arg, SizeNode,
                               Flags.getByValAlign(),
                               /*isVolatile*/false,
@@ -699,10 +698,8 @@ MSP430TargetLowering::LowerCCCCallTo(SDValue Chain, SDValue Callee,
   InFlag = Chain.getValue(1);
 
   // Create the CALLSEQ_END node.
-  Chain = DAG.getCALLSEQ_END(Chain,
-                             DAG.getConstant(NumBytes, getPointerTy(), true),
-                             DAG.getConstant(0, getPointerTy(), true),
-                             InFlag, dl);
+  Chain = DAG.getCALLSEQ_END(Chain, DAG.getConstant(NumBytes, dl, PtrVT, true),
+                             DAG.getConstant(0, dl, PtrVT, true), InFlag, dl);
   InFlag = Chain.getValue(1);
 
   // Handle result values, copying them out of physregs into vregs that we
@@ -786,30 +783,31 @@ SDValue MSP430TargetLowering::LowerGlobalAddress(SDValue Op,
                                                  SelectionDAG &DAG) const {
   const GlobalValue *GV = cast<GlobalAddressSDNode>(Op)->getGlobal();
   int64_t Offset = cast<GlobalAddressSDNode>(Op)->getOffset();
+  auto PtrVT = getPointerTy(DAG.getDataLayout());
 
   // Create the TargetGlobalAddress node, folding in the constant offset.
-  SDValue Result = DAG.getTargetGlobalAddress(GV, SDLoc(Op),
-                                              getPointerTy(), Offset);
-  return DAG.getNode(MSP430ISD::Wrapper, SDLoc(Op),
-                     getPointerTy(), Result);
+  SDValue Result = DAG.getTargetGlobalAddress(GV, SDLoc(Op), PtrVT, Offset);
+  return DAG.getNode(MSP430ISD::Wrapper, SDLoc(Op), PtrVT, Result);
 }
 
 SDValue MSP430TargetLowering::LowerExternalSymbol(SDValue Op,
                                                   SelectionDAG &DAG) const {
   SDLoc dl(Op);
   const char *Sym = cast<ExternalSymbolSDNode>(Op)->getSymbol();
-  SDValue Result = DAG.getTargetExternalSymbol(Sym, getPointerTy());
+  auto PtrVT = getPointerTy(DAG.getDataLayout());
+  SDValue Result = DAG.getTargetExternalSymbol(Sym, PtrVT);
 
-  return DAG.getNode(MSP430ISD::Wrapper, dl, getPointerTy(), Result);
+  return DAG.getNode(MSP430ISD::Wrapper, dl, PtrVT, Result);
 }
 
 SDValue MSP430TargetLowering::LowerBlockAddress(SDValue Op,
                                                 SelectionDAG &DAG) const {
   SDLoc dl(Op);
+  auto PtrVT = getPointerTy(DAG.getDataLayout());
   const BlockAddress *BA = cast<BlockAddressSDNode>(Op)->getBlockAddress();
-  SDValue Result = DAG.getTargetBlockAddress(BA, getPointerTy());
+  SDValue Result = DAG.getTargetBlockAddress(BA, PtrVT);
 
-  return DAG.getNode(MSP430ISD::Wrapper, dl, getPointerTy(), Result);
+  return DAG.getNode(MSP430ISD::Wrapper, dl, PtrVT, Result);
 }
 
 static SDValue EmitCMP(SDValue &LHS, SDValue &RHS, SDValue &TargetCC,
@@ -843,7 +841,7 @@ static SDValue EmitCMP(SDValue &LHS, SDValue &RHS, SDValue &TargetCC,
     // fold constant into instruction.
     if (const ConstantSDNode * C = dyn_cast<ConstantSDNode>(LHS)) {
       LHS = RHS;
-      RHS = DAG.getConstant(C->getSExtValue() + 1, C->getValueType(0));
+      RHS = DAG.getConstant(C->getSExtValue() + 1, dl, C->getValueType(0));
       TCC = MSP430CC::COND_LO;
       break;
     }
@@ -856,7 +854,7 @@ static SDValue EmitCMP(SDValue &LHS, SDValue &RHS, SDValue &TargetCC,
     // fold constant into instruction.
     if (const ConstantSDNode * C = dyn_cast<ConstantSDNode>(LHS)) {
       LHS = RHS;
-      RHS = DAG.getConstant(C->getSExtValue() + 1, C->getValueType(0));
+      RHS = DAG.getConstant(C->getSExtValue() + 1, dl, C->getValueType(0));
       TCC = MSP430CC::COND_HS;
       break;
     }
@@ -869,7 +867,7 @@ static SDValue EmitCMP(SDValue &LHS, SDValue &RHS, SDValue &TargetCC,
     // fold constant into instruction.
     if (const ConstantSDNode * C = dyn_cast<ConstantSDNode>(LHS)) {
       LHS = RHS;
-      RHS = DAG.getConstant(C->getSExtValue() + 1, C->getValueType(0));
+      RHS = DAG.getConstant(C->getSExtValue() + 1, dl, C->getValueType(0));
       TCC = MSP430CC::COND_L;
       break;
     }
@@ -882,7 +880,7 @@ static SDValue EmitCMP(SDValue &LHS, SDValue &RHS, SDValue &TargetCC,
     // fold constant into instruction.
     if (const ConstantSDNode * C = dyn_cast<ConstantSDNode>(LHS)) {
       LHS = RHS;
-      RHS = DAG.getConstant(C->getSExtValue() + 1, C->getValueType(0));
+      RHS = DAG.getConstant(C->getSExtValue() + 1, dl, C->getValueType(0));
       TCC = MSP430CC::COND_GE;
       break;
     }
@@ -890,7 +888,7 @@ static SDValue EmitCMP(SDValue &LHS, SDValue &RHS, SDValue &TargetCC,
     break;
   }
 
-  TargetCC = DAG.getConstant(TCC, MVT::i8);
+  TargetCC = DAG.getConstant(TCC, dl, MVT::i8);
   return DAG.getNode(MSP430ISD::CMP, dl, MVT::Glue, LHS, RHS);
 }
 
@@ -967,7 +965,7 @@ SDValue MSP430TargetLowering::LowerSETCC(SDValue Op, SelectionDAG &DAG) const {
      break;
   }
   EVT VT = Op.getValueType();
-  SDValue One  = DAG.getConstant(1, VT);
+  SDValue One  = DAG.getConstant(1, dl, VT);
   if (Convert) {
     SDValue SR = DAG.getCopyFromReg(DAG.getEntryNode(), dl, MSP430::SR,
                                     MVT::i16, Flag);
@@ -979,7 +977,7 @@ SDValue MSP430TargetLowering::LowerSETCC(SDValue Op, SelectionDAG &DAG) const {
       SR = DAG.getNode(ISD::XOR, dl, MVT::i16, SR, One);
     return SR;
   } else {
-    SDValue Zero = DAG.getConstant(0, VT);
+    SDValue Zero = DAG.getConstant(0, dl, VT);
     SDVTList VTs = DAG.getVTList(Op.getValueType(), MVT::Glue);
     SDValue Ops[] = {One, Zero, TargetCC, Flag};
     return DAG.getNode(MSP430ISD::SELECT_CC, dl, VTs, Ops);
@@ -1022,16 +1020,17 @@ MSP430TargetLowering::getReturnAddressFrameIndex(SelectionDAG &DAG) const {
   MachineFunction &MF = DAG.getMachineFunction();
   MSP430MachineFunctionInfo *FuncInfo = MF.getInfo<MSP430MachineFunctionInfo>();
   int ReturnAddrIndex = FuncInfo->getRAIndex();
+  auto PtrVT = getPointerTy(MF.getDataLayout());
 
   if (ReturnAddrIndex == 0) {
     // Set up a frame object for the return address.
-    uint64_t SlotSize = getDataLayout()->getPointerSize();
+    uint64_t SlotSize = MF.getDataLayout().getPointerSize();
     ReturnAddrIndex = MF.getFrameInfo()->CreateFixedObject(SlotSize, -SlotSize,
                                                            true);
     FuncInfo->setRAIndex(ReturnAddrIndex);
   }
 
-  return DAG.getFrameIndex(ReturnAddrIndex, getPointerTy());
+  return DAG.getFrameIndex(ReturnAddrIndex, PtrVT);
 }
 
 SDValue MSP430TargetLowering::LowerRETURNADDR(SDValue Op,
@@ -1044,21 +1043,21 @@ SDValue MSP430TargetLowering::LowerRETURNADDR(SDValue Op,
 
   unsigned Depth = cast<ConstantSDNode>(Op.getOperand(0))->getZExtValue();
   SDLoc dl(Op);
+  auto PtrVT = getPointerTy(DAG.getDataLayout());
 
   if (Depth > 0) {
     SDValue FrameAddr = LowerFRAMEADDR(Op, DAG);
     SDValue Offset =
-        DAG.getConstant(getDataLayout()->getPointerSize(), MVT::i16);
-    return DAG.getLoad(getPointerTy(), dl, DAG.getEntryNode(),
-                       DAG.getNode(ISD::ADD, dl, getPointerTy(),
-                                   FrameAddr, Offset),
+        DAG.getConstant(DAG.getDataLayout().getPointerSize(), dl, MVT::i16);
+    return DAG.getLoad(PtrVT, dl, DAG.getEntryNode(),
+                       DAG.getNode(ISD::ADD, dl, PtrVT, FrameAddr, Offset),
                        MachinePointerInfo(), false, false, false, 0);
   }
 
   // Just load the return address.
   SDValue RetAddrFI = getReturnAddressFrameIndex(DAG);
-  return DAG.getLoad(getPointerTy(), dl, DAG.getEntryNode(),
-                     RetAddrFI, MachinePointerInfo(), false, false, false, 0);
+  return DAG.getLoad(PtrVT, dl, DAG.getEntryNode(), RetAddrFI,
+                     MachinePointerInfo(), false, false, false, 0);
 }
 
 SDValue MSP430TargetLowering::LowerFRAMEADDR(SDValue Op,
@@ -1082,10 +1081,11 @@ SDValue MSP430TargetLowering::LowerVASTART(SDValue Op,
                                            SelectionDAG &DAG) const {
   MachineFunction &MF = DAG.getMachineFunction();
   MSP430MachineFunctionInfo *FuncInfo = MF.getInfo<MSP430MachineFunctionInfo>();
+  auto PtrVT = getPointerTy(DAG.getDataLayout());
 
   // Frame index of first vararg argument
-  SDValue FrameIndex = DAG.getFrameIndex(FuncInfo->getVarArgsFrameIndex(),
-                                         getPointerTy());
+  SDValue FrameIndex =
+      DAG.getFrameIndex(FuncInfo->getVarArgsFrameIndex(), PtrVT);
   const Value *SV = cast<SrcValueSDNode>(Op.getOperand(2))->getValue();
 
   // Create a store of the frame index to the location operand
@@ -1097,9 +1097,9 @@ SDValue MSP430TargetLowering::LowerVASTART(SDValue Op,
 SDValue MSP430TargetLowering::LowerJumpTable(SDValue Op,
                                              SelectionDAG &DAG) const {
     JumpTableSDNode *JT = cast<JumpTableSDNode>(Op);
-    SDValue Result = DAG.getTargetJumpTable(JT->getIndex(), getPointerTy());
-    return DAG.getNode(MSP430ISD::Wrapper, SDLoc(JT),
-                       getPointerTy(), Result);
+    auto PtrVT = getPointerTy(DAG.getDataLayout());
+    SDValue Result = DAG.getTargetJumpTable(JT->getIndex(), PtrVT);
+    return DAG.getNode(MSP430ISD::Wrapper, SDLoc(JT), PtrVT, Result);
 }
 
 /// getPostIndexedAddressParts - returns true by value, base pointer and
@@ -1129,7 +1129,7 @@ bool MSP430TargetLowering::getPostIndexedAddressParts(SDNode *N, SDNode *Op,
       return false;
 
     Base = Op->getOperand(0);
-    Offset = DAG.getConstant(RHSC, VT);
+    Offset = DAG.getConstant(RHSC, SDLoc(N), VT);
     AM = ISD::POST_INC;
     return true;
   }
@@ -1139,8 +1139,8 @@ bool MSP430TargetLowering::getPostIndexedAddressParts(SDNode *N, SDNode *Op,
 
 
 const char *MSP430TargetLowering::getTargetNodeName(unsigned Opcode) const {
-  switch (Opcode) {
-  default: return nullptr;
+  switch ((MSP430ISD::NodeType)Opcode) {
+  case MSP430ISD::FIRST_NUMBER:       break;
   case MSP430ISD::RET_FLAG:           return "MSP430ISD::RET_FLAG";
   case MSP430ISD::RETI_FLAG:          return "MSP430ISD::RETI_FLAG";
   case MSP430ISD::RRA:                return "MSP430ISD::RRA";
@@ -1150,10 +1150,13 @@ const char *MSP430TargetLowering::getTargetNodeName(unsigned Opcode) const {
   case MSP430ISD::Wrapper:            return "MSP430ISD::Wrapper";
   case MSP430ISD::BR_CC:              return "MSP430ISD::BR_CC";
   case MSP430ISD::CMP:                return "MSP430ISD::CMP";
+  case MSP430ISD::SETCC:              return "MSP430ISD::SETCC";
   case MSP430ISD::SELECT_CC:          return "MSP430ISD::SELECT_CC";
   case MSP430ISD::SHL:                return "MSP430ISD::SHL";
   case MSP430ISD::SRA:                return "MSP430ISD::SRA";
+  case MSP430ISD::SRL:                return "MSP430ISD::SRL";
   }
+  return nullptr;
 }
 
 bool MSP430TargetLowering::isTruncateFree(Type *Ty1,
diff --git a/lib/Target/MSP430/MSP430ISelLowering.h b/lib/Target/MSP430/MSP430ISelLowering.h
index 9266c3b19df..2d63852c185 100644
--- a/lib/Target/MSP430/MSP430ISelLowering.h
+++ b/lib/Target/MSP430/MSP430ISelLowering.h
@@ -21,7 +21,7 @@
 
 namespace llvm {
   namespace MSP430ISD {
-    enum {
+    enum NodeType : unsigned {
       FIRST_NUMBER = ISD::BUILTIN_OP_END,
 
       /// Return with a flag operand. Operand 0 is the chain operand.
@@ -72,7 +72,9 @@ namespace llvm {
     explicit MSP430TargetLowering(const TargetMachine &TM,
                                   const MSP430Subtarget &STI);
 
-    MVT getScalarShiftAmountTy(EVT LHSTy) const override { return MVT::i8; }
+    MVT getScalarShiftAmountTy(const DataLayout &, EVT) const override {
+      return MVT::i8;
+    }
 
     /// LowerOperation - Provide custom lowering hooks for some operations.
     SDValue LowerOperation(SDValue Op, SelectionDAG &DAG) const override;
@@ -96,11 +98,10 @@ namespace llvm {
     SDValue getReturnAddressFrameIndex(SelectionDAG &DAG) const;
 
     TargetLowering::ConstraintType
-    getConstraintType(const std::string &Constraint) const override;
+    getConstraintType(StringRef Constraint) const override;
     std::pair<unsigned, const TargetRegisterClass *>
     getRegForInlineAsmConstraint(const TargetRegisterInfo *TRI,
-                                 const std::string &Constraint,
-                                 MVT VT) const override;
+                                 StringRef Constraint, MVT VT) const override;
 
     /// isTruncateFree - Return true if it's free to truncate a value of type
     /// Ty1 to type Ty2. e.g. On msp430 it's free to truncate a i16 value in
diff --git a/lib/Target/MSP430/MSP430InstrInfo.cpp b/lib/Target/MSP430/MSP430InstrInfo.cpp
index 27681aae606..72b1780fd1c 100644
--- a/lib/Target/MSP430/MSP430InstrInfo.cpp
+++ b/lib/Target/MSP430/MSP430InstrInfo.cpp
@@ -262,7 +262,7 @@ bool MSP430InstrInfo::AnalyzeBranch(MachineBasicBlock &MBB,
 unsigned
 MSP430InstrInfo::InsertBranch(MachineBasicBlock &MBB, MachineBasicBlock *TBB,
                               MachineBasicBlock *FBB,
-                              const SmallVectorImpl<MachineOperand> &Cond,
+                              ArrayRef<MachineOperand> Cond,
                               DebugLoc DL) const {
   // Shouldn't be a fall through.
   assert(TBB && "InsertBranch must not be told to insert a fallthrough");
diff --git a/lib/Target/MSP430/MSP430InstrInfo.h b/lib/Target/MSP430/MSP430InstrInfo.h
index f9b25b63962..3cf3b1bb8ab 100644
--- a/lib/Target/MSP430/MSP430InstrInfo.h
+++ b/lib/Target/MSP430/MSP430InstrInfo.h
@@ -82,8 +82,7 @@ class MSP430InstrInfo : public MSP430GenInstrInfo {
 
   unsigned RemoveBranch(MachineBasicBlock &MBB) const override;
   unsigned InsertBranch(MachineBasicBlock &MBB, MachineBasicBlock *TBB,
-                        MachineBasicBlock *FBB,
-                        const SmallVectorImpl<MachineOperand> &Cond,
+                        MachineBasicBlock *FBB, ArrayRef<MachineOperand> Cond,
                         DebugLoc DL) const override;
 
 };
diff --git a/lib/Target/MSP430/MSP430MCInstLower.cpp b/lib/Target/MSP430/MSP430MCInstLower.cpp
index c63a57c60dd..47b0e270c5b 100644
--- a/lib/Target/MSP430/MSP430MCInstLower.cpp
+++ b/lib/Target/MSP430/MSP430MCInstLower.cpp
@@ -50,9 +50,9 @@ GetExternalSymbolSymbol(const MachineOperand &MO) const {
 
 MCSymbol *MSP430MCInstLower::
 GetJumpTableSymbol(const MachineOperand &MO) const {
-  const DataLayout *DL = Printer.TM.getDataLayout();
+  const DataLayout &DL = Printer.getDataLayout();
   SmallString<256> Name;
-  raw_svector_ostream(Name) << DL->getPrivateGlobalPrefix() << "JTI"
+  raw_svector_ostream(Name) << DL.getPrivateGlobalPrefix() << "JTI"
                             << Printer.getFunctionNumber() << '_'
                             << MO.getIndex();
 
@@ -62,14 +62,14 @@ GetJumpTableSymbol(const MachineOperand &MO) const {
   }
 
   // Create a symbol for the name.
-  return Ctx.GetOrCreateSymbol(Name);
+  return Ctx.getOrCreateSymbol(Name);
 }
 
 MCSymbol *MSP430MCInstLower::
 GetConstantPoolIndexSymbol(const MachineOperand &MO) const {
-  const DataLayout *DL = Printer.TM.getDataLayout();
+  const DataLayout &DL = Printer.getDataLayout();
   SmallString<256> Name;
-  raw_svector_ostream(Name) << DL->getPrivateGlobalPrefix() << "CPI"
+  raw_svector_ostream(Name) << DL.getPrivateGlobalPrefix() << "CPI"
                             << Printer.getFunctionNumber() << '_'
                             << MO.getIndex();
 
@@ -79,7 +79,7 @@ GetConstantPoolIndexSymbol(const MachineOperand &MO) const {
   }
 
   // Create a symbol for the name.
-  return Ctx.GetOrCreateSymbol(Name);
+  return Ctx.getOrCreateSymbol(Name);
 }
 
 MCSymbol *MSP430MCInstLower::
@@ -96,7 +96,7 @@ MCOperand MSP430MCInstLower::
 LowerSymbolOperand(const MachineOperand &MO, MCSymbol *Sym) const {
   // FIXME: We would like an efficient form for this, so we don't have to do a
   // lot of extra uniquing.
-  const MCExpr *Expr = MCSymbolRefExpr::Create(Sym, Ctx);
+  const MCExpr *Expr = MCSymbolRefExpr::create(Sym, Ctx);
 
   switch (MO.getTargetFlags()) {
   default: llvm_unreachable("Unknown target flag on GV operand");
@@ -104,10 +104,10 @@ LowerSymbolOperand(const MachineOperand &MO, MCSymbol *Sym) const {
   }
 
   if (!MO.isJTI() && MO.getOffset())
-    Expr = MCBinaryExpr::CreateAdd(Expr,
-                                   MCConstantExpr::Create(MO.getOffset(), Ctx),
+    Expr = MCBinaryExpr::createAdd(Expr,
+                                   MCConstantExpr::create(MO.getOffset(), Ctx),
                                    Ctx);
-  return MCOperand::CreateExpr(Expr);
+  return MCOperand::createExpr(Expr);
 }
 
 void MSP430MCInstLower::Lower(const MachineInstr *MI, MCInst &OutMI) const {
@@ -124,13 +124,13 @@ void MSP430MCInstLower::Lower(const MachineInstr *MI, MCInst &OutMI) const {
     case MachineOperand::MO_Register:
       // Ignore all implicit register operands.
       if (MO.isImplicit()) continue;
-      MCOp = MCOperand::CreateReg(MO.getReg());
+      MCOp = MCOperand::createReg(MO.getReg());
       break;
     case MachineOperand::MO_Immediate:
-      MCOp = MCOperand::CreateImm(MO.getImm());
+      MCOp = MCOperand::createImm(MO.getImm());
       break;
     case MachineOperand::MO_MachineBasicBlock:
-      MCOp = MCOperand::CreateExpr(MCSymbolRefExpr::Create(
+      MCOp = MCOperand::createExpr(MCSymbolRefExpr::create(
                          MO.getMBB()->getSymbol(), Ctx));
       break;
     case MachineOperand::MO_GlobalAddress:
diff --git a/lib/Target/MSP430/MSP430MachineFunctionInfo.cpp b/lib/Target/MSP430/MSP430MachineFunctionInfo.cpp
index 0f753990845..08981b9d49a 100644
--- a/lib/Target/MSP430/MSP430MachineFunctionInfo.cpp
+++ b/lib/Target/MSP430/MSP430MachineFunctionInfo.cpp
@@ -1,4 +1,4 @@
-//===-- MSP430MachineFuctionInfo.cpp - MSP430 machine function info -------===//
+//===-- MSP430MachineFucntionInfo.cpp - MSP430 machine function info ------===//
 //
 //                     The LLVM Compiler Infrastructure
 //
diff --git a/lib/Target/MSP430/MSP430MachineFunctionInfo.h b/lib/Target/MSP430/MSP430MachineFunctionInfo.h
index fcc5f5b8860..2d937318c7e 100644
--- a/lib/Target/MSP430/MSP430MachineFunctionInfo.h
+++ b/lib/Target/MSP430/MSP430MachineFunctionInfo.h
@@ -1,4 +1,4 @@
-//===- MSP430MachineFuctionInfo.h - MSP430 machine function info -*- C++ -*-==//
+//=== MSP430MachineFunctionInfo.h - MSP430 machine function info -*- C++ -*-==//
 //
 //                     The LLVM Compiler Infrastructure
 //
diff --git a/lib/Target/MSP430/MSP430RegisterInfo.cpp b/lib/Target/MSP430/MSP430RegisterInfo.cpp
index 614467bcd24..2fb82e535e8 100644
--- a/lib/Target/MSP430/MSP430RegisterInfo.cpp
+++ b/lib/Target/MSP430/MSP430RegisterInfo.cpp
@@ -37,7 +37,7 @@ MSP430RegisterInfo::MSP430RegisterInfo()
 
 const MCPhysReg*
 MSP430RegisterInfo::getCalleeSavedRegs(const MachineFunction *MF) const {
-  const TargetFrameLowering *TFI = MF->getSubtarget().getFrameLowering();
+  const MSP430FrameLowering *TFI = getFrameLowering(*MF);
   const Function* F = MF->getFunction();
   static const MCPhysReg CalleeSavedRegs[] = {
     MSP430::FP, MSP430::R5, MSP430::R6, MSP430::R7,
@@ -73,7 +73,7 @@ MSP430RegisterInfo::getCalleeSavedRegs(const MachineFunction *MF) const {
 
 BitVector MSP430RegisterInfo::getReservedRegs(const MachineFunction &MF) const {
   BitVector Reserved(getNumRegs());
-  const TargetFrameLowering *TFI = MF.getSubtarget().getFrameLowering();
+  const MSP430FrameLowering *TFI = getFrameLowering(MF);
 
   // Mark 4 special registers with subregisters as reserved.
   Reserved.set(MSP430::PCB);
@@ -109,7 +109,7 @@ MSP430RegisterInfo::eliminateFrameIndex(MachineBasicBlock::iterator II,
   MachineInstr &MI = *II;
   MachineBasicBlock &MBB = *MI.getParent();
   MachineFunction &MF = *MBB.getParent();
-  const TargetFrameLowering *TFI = MF.getSubtarget().getFrameLowering();
+  const MSP430FrameLowering *TFI = getFrameLowering(MF);
   DebugLoc dl = MI.getDebugLoc();
   int FrameIndex = MI.getOperand(FIOperandNum).getIndex();
 
@@ -156,7 +156,6 @@ MSP430RegisterInfo::eliminateFrameIndex(MachineBasicBlock::iterator II,
 }
 
 unsigned MSP430RegisterInfo::getFrameRegister(const MachineFunction &MF) const {
-  const TargetFrameLowering *TFI = MF.getSubtarget().getFrameLowering();
-
+  const MSP430FrameLowering *TFI = getFrameLowering(MF);
   return TFI->hasFP(MF) ? MSP430::FP : MSP430::SP;
 }
diff --git a/lib/Target/MSP430/MSP430SelectionDAGInfo.cpp b/lib/Target/MSP430/MSP430SelectionDAGInfo.cpp
deleted file mode 100644
index 3897ef684d4..00000000000
--- a/lib/Target/MSP430/MSP430SelectionDAGInfo.cpp
+++ /dev/null
@@ -1,23 +0,0 @@
-//===-- MSP430SelectionDAGInfo.cpp - MSP430 SelectionDAG Info -------------===//
-//
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
-//
-//===----------------------------------------------------------------------===//
-//
-// This file implements the MSP430SelectionDAGInfo class.
-//
-//===----------------------------------------------------------------------===//
-
-#include "MSP430TargetMachine.h"
-using namespace llvm;
-
-#define DEBUG_TYPE "msp430-selectiondag-info"
-
-MSP430SelectionDAGInfo::MSP430SelectionDAGInfo(const DataLayout &DL)
-    : TargetSelectionDAGInfo(&DL) {}
-
-MSP430SelectionDAGInfo::~MSP430SelectionDAGInfo() {
-}
diff --git a/lib/Target/MSP430/MSP430SelectionDAGInfo.h b/lib/Target/MSP430/MSP430SelectionDAGInfo.h
deleted file mode 100644
index 61a6b19111d..00000000000
--- a/lib/Target/MSP430/MSP430SelectionDAGInfo.h
+++ /dev/null
@@ -1,31 +0,0 @@
-//===-- MSP430SelectionDAGInfo.h - MSP430 SelectionDAG Info -----*- C++ -*-===//
-//
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
-//
-//===----------------------------------------------------------------------===//
-//
-// This file defines the MSP430 subclass for TargetSelectionDAGInfo.
-//
-//===----------------------------------------------------------------------===//
-
-#ifndef LLVM_LIB_TARGET_MSP430_MSP430SELECTIONDAGINFO_H
-#define LLVM_LIB_TARGET_MSP430_MSP430SELECTIONDAGINFO_H
-
-#include "llvm/Target/TargetSelectionDAGInfo.h"
-
-namespace llvm {
-
-class MSP430TargetMachine;
-
-class MSP430SelectionDAGInfo : public TargetSelectionDAGInfo {
-public:
-  explicit MSP430SelectionDAGInfo(const DataLayout &DL);
-  ~MSP430SelectionDAGInfo();
-};
-
-}
-
-#endif
diff --git a/lib/Target/MSP430/MSP430Subtarget.cpp b/lib/Target/MSP430/MSP430Subtarget.cpp
index 3dda3bf95e5..6216348e4d7 100644
--- a/lib/Target/MSP430/MSP430Subtarget.cpp
+++ b/lib/Target/MSP430/MSP430Subtarget.cpp
@@ -31,8 +31,7 @@ MSP430Subtarget::initializeSubtargetDependencies(StringRef CPU, StringRef FS) {
   return *this;
 }
 
-MSP430Subtarget::MSP430Subtarget(const std::string &TT, const std::string &CPU,
+MSP430Subtarget::MSP430Subtarget(const Triple &TT, const std::string &CPU,
                                  const std::string &FS, const TargetMachine &TM)
     : MSP430GenSubtargetInfo(TT, CPU, FS), FrameLowering(),
-      InstrInfo(initializeSubtargetDependencies(CPU, FS)), TLInfo(TM, *this),
-      TSInfo(*TM.getDataLayout()) {}
+      InstrInfo(initializeSubtargetDependencies(CPU, FS)), TLInfo(TM, *this) {}
diff --git a/lib/Target/MSP430/MSP430Subtarget.h b/lib/Target/MSP430/MSP430Subtarget.h
index 30d46d389ee..ff2656d26dd 100644
--- a/lib/Target/MSP430/MSP430Subtarget.h
+++ b/lib/Target/MSP430/MSP430Subtarget.h
@@ -18,8 +18,8 @@
 #include "MSP430ISelLowering.h"
 #include "MSP430InstrInfo.h"
 #include "MSP430RegisterInfo.h"
-#include "MSP430SelectionDAGInfo.h"
 #include "llvm/IR/DataLayout.h"
+#include "llvm/Target/TargetSelectionDAGInfo.h"
 #include "llvm/Target/TargetSubtargetInfo.h"
 #include <string>
 
@@ -35,13 +35,13 @@ class MSP430Subtarget : public MSP430GenSubtargetInfo {
   MSP430FrameLowering FrameLowering;
   MSP430InstrInfo InstrInfo;
   MSP430TargetLowering TLInfo;
-  MSP430SelectionDAGInfo TSInfo;
+  TargetSelectionDAGInfo TSInfo;
 
 public:
   /// This constructor initializes the data members to match that
   /// of the specified triple.
   ///
-  MSP430Subtarget(const std::string &TT, const std::string &CPU,
+  MSP430Subtarget(const Triple &TT, const std::string &CPU,
                   const std::string &FS, const TargetMachine &TM);
 
   MSP430Subtarget &initializeSubtargetDependencies(StringRef CPU, StringRef FS);
@@ -60,7 +60,7 @@ class MSP430Subtarget : public MSP430GenSubtargetInfo {
   const MSP430TargetLowering *getTargetLowering() const override {
     return &TLInfo;
   }
-  const MSP430SelectionDAGInfo *getSelectionDAGInfo() const override {
+  const TargetSelectionDAGInfo *getSelectionDAGInfo() const override {
     return &TSInfo;
   }
 };
diff --git a/lib/Target/MSP430/MSP430TargetMachine.cpp b/lib/Target/MSP430/MSP430TargetMachine.cpp
index d6cc4ae5ecd..97a4047d1d6 100644
--- a/lib/Target/MSP430/MSP430TargetMachine.cpp
+++ b/lib/Target/MSP430/MSP430TargetMachine.cpp
@@ -25,7 +25,7 @@ extern "C" void LLVMInitializeMSP430Target() {
   RegisterTargetMachine<MSP430TargetMachine> X(TheMSP430Target);
 }
 
-MSP430TargetMachine::MSP430TargetMachine(const Target &T, StringRef TT,
+MSP430TargetMachine::MSP430TargetMachine(const Target &T, const Triple &TT,
                                          StringRef CPU, StringRef FS,
                                          const TargetOptions &Options,
                                          Reloc::Model RM, CodeModel::Model CM,
diff --git a/lib/Target/MSP430/MSP430TargetMachine.h b/lib/Target/MSP430/MSP430TargetMachine.h
index 6ccd30d393f..4f955a8049c 100644
--- a/lib/Target/MSP430/MSP430TargetMachine.h
+++ b/lib/Target/MSP430/MSP430TargetMachine.h
@@ -28,8 +28,8 @@ class MSP430TargetMachine : public LLVMTargetMachine {
   MSP430Subtarget        Subtarget;
 
 public:
-  MSP430TargetMachine(const Target &T, StringRef TT,
-                      StringRef CPU, StringRef FS, const TargetOptions &Options,
+  MSP430TargetMachine(const Target &T, const Triple &TT, StringRef CPU,
+                      StringRef FS, const TargetOptions &Options,
                       Reloc::Model RM, CodeModel::Model CM,
                       CodeGenOpt::Level OL);
   ~MSP430TargetMachine() override;
diff --git a/lib/Target/Mips/AsmParser/MipsAsmParser.cpp b/lib/Target/Mips/AsmParser/MipsAsmParser.cpp
index 6f7e3c16a9b..9855bf87eed 100644
--- a/lib/Target/Mips/AsmParser/MipsAsmParser.cpp
+++ b/lib/Target/Mips/AsmParser/MipsAsmParser.cpp
@@ -43,18 +43,18 @@ class MCInstrInfo;
 namespace {
 class MipsAssemblerOptions {
 public:
-  MipsAssemblerOptions(uint64_t Features_) : 
+  MipsAssemblerOptions(const FeatureBitset &Features_) :
     ATReg(1), Reorder(true), Macro(true), Features(Features_) {}
 
   MipsAssemblerOptions(const MipsAssemblerOptions *Opts) {
-    ATReg = Opts->getATRegNum();
+    ATReg = Opts->getATRegIndex();
     Reorder = Opts->isReorder();
     Macro = Opts->isMacro();
     Features = Opts->getFeatures();
   }
 
-  unsigned getATRegNum() const { return ATReg; }
-  bool setATReg(unsigned Reg) {
+  unsigned getATRegIndex() const { return ATReg; }
+  bool setATRegIndex(unsigned Reg) {
     if (Reg > 31)
       return false;
 
@@ -70,32 +70,35 @@ class MipsAssemblerOptions {
   void setMacro() { Macro = true; }
   void setNoMacro() { Macro = false; }
 
-  uint64_t getFeatures() const { return Features; }
-  void setFeatures(uint64_t Features_) { Features = Features_; }
+  const FeatureBitset &getFeatures() const { return Features; }
+  void setFeatures(const FeatureBitset &Features_) { Features = Features_; }
 
   // Set of features that are either architecture features or referenced
   // by them (e.g.: FeatureNaN2008 implied by FeatureMips32r6).
   // The full table can be found in MipsGenSubtargetInfo.inc (MipsFeatureKV[]).
   // The reason we need this mask is explained in the selectArch function.
   // FIXME: Ideally we would like TableGen to generate this information.
-  static const uint64_t AllArchRelatedMask =
-      Mips::FeatureMips1 | Mips::FeatureMips2 | Mips::FeatureMips3 |
-      Mips::FeatureMips3_32 | Mips::FeatureMips3_32r2 | Mips::FeatureMips4 |
-      Mips::FeatureMips4_32 | Mips::FeatureMips4_32r2 | Mips::FeatureMips5 |
-      Mips::FeatureMips5_32r2 | Mips::FeatureMips32 | Mips::FeatureMips32r2 |
-      Mips::FeatureMips32r3 | Mips::FeatureMips32r5 | Mips::FeatureMips32r6 |
-      Mips::FeatureMips64 | Mips::FeatureMips64r2 | Mips::FeatureMips64r3 |
-      Mips::FeatureMips64r5 | Mips::FeatureMips64r6 | Mips::FeatureCnMips |
-      Mips::FeatureFP64Bit | Mips::FeatureGP64Bit | Mips::FeatureNaN2008;
+  static const FeatureBitset AllArchRelatedMask;
 
 private:
   unsigned ATReg;
   bool Reorder;
   bool Macro;
-  uint64_t Features;
+  FeatureBitset Features;
 };
 }
 
+const FeatureBitset MipsAssemblerOptions::AllArchRelatedMask = {
+    Mips::FeatureMips1, Mips::FeatureMips2, Mips::FeatureMips3,
+    Mips::FeatureMips3_32, Mips::FeatureMips3_32r2, Mips::FeatureMips4,
+    Mips::FeatureMips4_32, Mips::FeatureMips4_32r2, Mips::FeatureMips5,
+    Mips::FeatureMips5_32r2, Mips::FeatureMips32, Mips::FeatureMips32r2,
+    Mips::FeatureMips32r3, Mips::FeatureMips32r5, Mips::FeatureMips32r6,
+    Mips::FeatureMips64, Mips::FeatureMips64r2, Mips::FeatureMips64r3,
+    Mips::FeatureMips64r5, Mips::FeatureMips64r6, Mips::FeatureCnMips,
+    Mips::FeatureFP64Bit, Mips::FeatureGP64Bit, Mips::FeatureNaN2008
+};
+
 namespace {
 class MipsAsmParser : public MCTargetAsmParser {
   MipsTargetStreamer &getTargetStreamer() {
@@ -110,6 +113,7 @@ class MipsAsmParser : public MCTargetAsmParser {
                        // nullptr, which indicates that no function is currently
                        // selected. This usually happens after an '.end func'
                        // directive.
+  bool IsLittleEndian;
 
   // Print a warning along with its fix-it message at the given range.
   void printWarningWithFixIt(const Twine &Msg, const Twine &FixMsg,
@@ -179,20 +183,25 @@ class MipsAsmParser : public MCTargetAsmParser {
   bool expandJalWithRegs(MCInst &Inst, SMLoc IDLoc,
                          SmallVectorImpl<MCInst> &Instructions);
 
-  bool expandLoadImm(MCInst &Inst, SMLoc IDLoc,
+  bool loadImmediate(int64_t ImmValue, unsigned DstReg, unsigned SrcReg,
+                     bool Is32BitImm, SMLoc IDLoc,
+                     SmallVectorImpl<MCInst> &Instructions);
+
+  bool loadAndAddSymbolAddress(const MCExpr *SymExpr, unsigned DstReg,
+                               unsigned SrcReg, bool Is32BitSym, SMLoc IDLoc,
+                               SmallVectorImpl<MCInst> &Instructions);
+
+  bool expandLoadImm(MCInst &Inst, bool Is32BitImm, SMLoc IDLoc,
                      SmallVectorImpl<MCInst> &Instructions);
 
-  bool expandLoadAddressImm(MCInst &Inst, SMLoc IDLoc,
+  bool expandLoadAddressImm(MCInst &Inst, bool Is32BitImm, SMLoc IDLoc,
                             SmallVectorImpl<MCInst> &Instructions);
 
-  bool expandLoadAddressReg(MCInst &Inst, SMLoc IDLoc,
+  bool expandLoadAddressReg(MCInst &Inst, bool Is32BitImm, SMLoc IDLoc,
                             SmallVectorImpl<MCInst> &Instructions);
   bool expandUncondBranchMMPseudo(MCInst &Inst, SMLoc IDLoc,
                                   SmallVectorImpl<MCInst> &Instructions);
 
-  void expandLoadAddressSym(MCInst &Inst, SMLoc IDLoc,
-                            SmallVectorImpl<MCInst> &Instructions);
-
   void expandMemInst(MCInst &Inst, SMLoc IDLoc,
                      SmallVectorImpl<MCInst> &Instructions, bool isLoad,
                      bool isImmOpnd);
@@ -200,9 +209,24 @@ class MipsAsmParser : public MCTargetAsmParser {
   bool expandLoadStoreMultiple(MCInst &Inst, SMLoc IDLoc,
                                SmallVectorImpl<MCInst> &Instructions);
 
+  bool expandBranchImm(MCInst &Inst, SMLoc IDLoc,
+                       SmallVectorImpl<MCInst> &Instructions);
+
+  bool expandCondBranches(MCInst &Inst, SMLoc IDLoc,
+                          SmallVectorImpl<MCInst> &Instructions);
+
+  bool expandUlhu(MCInst &Inst, SMLoc IDLoc,
+                  SmallVectorImpl<MCInst> &Instructions);
+
+  bool expandUlw(MCInst &Inst, SMLoc IDLoc,
+                 SmallVectorImpl<MCInst> &Instructions);
+
   void createNop(bool hasShortDelaySlot, SMLoc IDLoc,
                  SmallVectorImpl<MCInst> &Instructions);
 
+  void createAddu(unsigned DstReg, unsigned SrcReg, unsigned TrgReg,
+                  bool Is64Bit, SmallVectorImpl<MCInst> &Instructions);
+
   bool reportParseError(Twine ErrorMsg);
   bool reportParseError(SMLoc Loc, Twine ErrorMsg);
 
@@ -234,8 +258,12 @@ class MipsAsmParser : public MCTargetAsmParser {
   bool parseSetMips16Directive();
   bool parseSetNoMips16Directive();
   bool parseSetFpDirective();
+  bool parseSetOddSPRegDirective();
+  bool parseSetNoOddSPRegDirective();
   bool parseSetPopDirective();
   bool parseSetPushDirective();
+  bool parseSetSoftFloatDirective();
+  bool parseSetHardFloatDirective();
 
   bool parseSetAssignment();
 
@@ -309,28 +337,38 @@ class MipsAsmParser : public MCTargetAsmParser {
   // FeatureMipsGP64 | FeatureMips1)
   // Clearing Mips3 is equivalent to clear (FeatureMips3 | FeatureMips4).
   void selectArch(StringRef ArchFeature) {
-    uint64_t FeatureBits = STI.getFeatureBits();
+    FeatureBitset FeatureBits = STI.getFeatureBits();
     FeatureBits &= ~MipsAssemblerOptions::AllArchRelatedMask;
     STI.setFeatureBits(FeatureBits);
     setAvailableFeatures(
         ComputeAvailableFeatures(STI.ToggleFeature(ArchFeature)));
-    AssemblerOptions.back()->setFeatures(getAvailableFeatures());
+    AssemblerOptions.back()->setFeatures(STI.getFeatureBits());
   }
 
   void setFeatureBits(uint64_t Feature, StringRef FeatureString) {
-    if (!(STI.getFeatureBits() & Feature)) {
+    if (!(STI.getFeatureBits()[Feature])) {
       setAvailableFeatures(
           ComputeAvailableFeatures(STI.ToggleFeature(FeatureString)));
+      AssemblerOptions.back()->setFeatures(STI.getFeatureBits());
     }
-    AssemblerOptions.back()->setFeatures(getAvailableFeatures());
   }
 
   void clearFeatureBits(uint64_t Feature, StringRef FeatureString) {
-    if (STI.getFeatureBits() & Feature) {
+    if (STI.getFeatureBits()[Feature]) {
       setAvailableFeatures(
           ComputeAvailableFeatures(STI.ToggleFeature(FeatureString)));
+      AssemblerOptions.back()->setFeatures(STI.getFeatureBits());
     }
-    AssemblerOptions.back()->setFeatures(getAvailableFeatures());
+  }
+
+  void setModuleFeatureBits(uint64_t Feature, StringRef FeatureString) {
+    setFeatureBits(Feature, FeatureString);
+    AssemblerOptions.front()->setFeatures(STI.getFeatureBits());
+  }
+
+  void clearModuleFeatureBits(uint64_t Feature, StringRef FeatureString) {
+    clearFeatureBits(Feature, FeatureString);
+    AssemblerOptions.front()->setFeatures(STI.getFeatureBits());
   }
 
 public:
@@ -344,21 +382,23 @@ class MipsAsmParser : public MCTargetAsmParser {
 
   MipsAsmParser(MCSubtargetInfo &sti, MCAsmParser &parser,
                 const MCInstrInfo &MII, const MCTargetOptions &Options)
-      : MCTargetAsmParser(), STI(sti),
+      : MCTargetAsmParser(Options), STI(sti),
         ABI(MipsABIInfo::computeTargetABI(Triple(sti.getTargetTriple()),
                                           sti.getCPU(), Options)) {
     MCAsmParserExtension::Initialize(parser);
 
+    parser.addAliasForDirective(".asciiz", ".asciz");
+
     // Initialize the set of available features.
     setAvailableFeatures(ComputeAvailableFeatures(STI.getFeatureBits()));
     
     // Remember the initial assembler options. The user can not modify these.
     AssemblerOptions.push_back(
-                     make_unique<MipsAssemblerOptions>(getAvailableFeatures()));
+        llvm::make_unique<MipsAssemblerOptions>(STI.getFeatureBits()));
     
     // Create an assembler options environment for the user to modify.
     AssemblerOptions.push_back(
-                     make_unique<MipsAssemblerOptions>(getAvailableFeatures()));
+        llvm::make_unique<MipsAssemblerOptions>(STI.getFeatureBits()));
 
     getTargetStreamer().updateABIInfo(*this);
 
@@ -366,76 +406,90 @@ class MipsAsmParser : public MCTargetAsmParser {
       report_fatal_error("-mno-odd-spreg requires the O32 ABI");
 
     CurrentFn = nullptr;
+
+    Triple TheTriple(sti.getTargetTriple());
+    if ((TheTriple.getArch() == Triple::mips) ||
+        (TheTriple.getArch() == Triple::mips64))
+      IsLittleEndian = false;
+    else
+      IsLittleEndian = true;
   }
 
   /// True if all of $fcc0 - $fcc7 exist for the current ISA.
   bool hasEightFccRegisters() const { return hasMips4() || hasMips32(); }
 
-  bool isGP64bit() const { return STI.getFeatureBits() & Mips::FeatureGP64Bit; }
-  bool isFP64bit() const { return STI.getFeatureBits() & Mips::FeatureFP64Bit; }
+  bool isGP64bit() const { return STI.getFeatureBits()[Mips::FeatureGP64Bit]; }
+  bool isFP64bit() const { return STI.getFeatureBits()[Mips::FeatureFP64Bit]; }
   const MipsABIInfo &getABI() const { return ABI; }
   bool isABI_N32() const { return ABI.IsN32(); }
   bool isABI_N64() const { return ABI.IsN64(); }
   bool isABI_O32() const { return ABI.IsO32(); }
-  bool isABI_FPXX() const { return STI.getFeatureBits() & Mips::FeatureFPXX; }
+  bool isABI_FPXX() const { return STI.getFeatureBits()[Mips::FeatureFPXX]; }
 
   bool useOddSPReg() const {
-    return !(STI.getFeatureBits() & Mips::FeatureNoOddSPReg);
+    return !(STI.getFeatureBits()[Mips::FeatureNoOddSPReg]);
   }
 
   bool inMicroMipsMode() const {
-    return STI.getFeatureBits() & Mips::FeatureMicroMips;
+    return STI.getFeatureBits()[Mips::FeatureMicroMips];
   }
-  bool hasMips1() const { return STI.getFeatureBits() & Mips::FeatureMips1; }
-  bool hasMips2() const { return STI.getFeatureBits() & Mips::FeatureMips2; }
-  bool hasMips3() const { return STI.getFeatureBits() & Mips::FeatureMips3; }
-  bool hasMips4() const { return STI.getFeatureBits() & Mips::FeatureMips4; }
-  bool hasMips5() const { return STI.getFeatureBits() & Mips::FeatureMips5; }
+  bool hasMips1() const { return STI.getFeatureBits()[Mips::FeatureMips1]; }
+  bool hasMips2() const { return STI.getFeatureBits()[Mips::FeatureMips2]; }
+  bool hasMips3() const { return STI.getFeatureBits()[Mips::FeatureMips3]; }
+  bool hasMips4() const { return STI.getFeatureBits()[Mips::FeatureMips4]; }
+  bool hasMips5() const { return STI.getFeatureBits()[Mips::FeatureMips5]; }
   bool hasMips32() const {
-    return (STI.getFeatureBits() & Mips::FeatureMips32);
+    return STI.getFeatureBits()[Mips::FeatureMips32];
   }
   bool hasMips64() const {
-    return (STI.getFeatureBits() & Mips::FeatureMips64);
+    return STI.getFeatureBits()[Mips::FeatureMips64];
   }
   bool hasMips32r2() const {
-    return (STI.getFeatureBits() & Mips::FeatureMips32r2);
+    return STI.getFeatureBits()[Mips::FeatureMips32r2];
   }
   bool hasMips64r2() const {
-    return (STI.getFeatureBits() & Mips::FeatureMips64r2);
+    return STI.getFeatureBits()[Mips::FeatureMips64r2];
   }
   bool hasMips32r3() const {
-    return (STI.getFeatureBits() & Mips::FeatureMips32r3);
+    return (STI.getFeatureBits()[Mips::FeatureMips32r3]);
   }
   bool hasMips64r3() const {
-    return (STI.getFeatureBits() & Mips::FeatureMips64r3);
+    return (STI.getFeatureBits()[Mips::FeatureMips64r3]);
   }
   bool hasMips32r5() const {
-    return (STI.getFeatureBits() & Mips::FeatureMips32r5);
+    return (STI.getFeatureBits()[Mips::FeatureMips32r5]);
   }
   bool hasMips64r5() const {
-    return (STI.getFeatureBits() & Mips::FeatureMips64r5);
+    return (STI.getFeatureBits()[Mips::FeatureMips64r5]);
   }
   bool hasMips32r6() const {
-    return (STI.getFeatureBits() & Mips::FeatureMips32r6);
+    return STI.getFeatureBits()[Mips::FeatureMips32r6];
   }
   bool hasMips64r6() const {
-    return (STI.getFeatureBits() & Mips::FeatureMips64r6);
+    return STI.getFeatureBits()[Mips::FeatureMips64r6];
   }
+
+  bool hasDSP() const { return STI.getFeatureBits()[Mips::FeatureDSP]; }
+  bool hasDSPR2() const { return STI.getFeatureBits()[Mips::FeatureDSPR2]; }
+  bool hasMSA() const { return STI.getFeatureBits()[Mips::FeatureMSA]; }
   bool hasCnMips() const {
-    return (STI.getFeatureBits() & Mips::FeatureCnMips);
+    return (STI.getFeatureBits()[Mips::FeatureCnMips]);
   }
-  bool hasDSP() const { return (STI.getFeatureBits() & Mips::FeatureDSP); }
-  bool hasDSPR2() const { return (STI.getFeatureBits() & Mips::FeatureDSPR2); }
-  bool hasMSA() const { return (STI.getFeatureBits() & Mips::FeatureMSA); }
 
   bool inMips16Mode() const {
-    return STI.getFeatureBits() & Mips::FeatureMips16;
+    return STI.getFeatureBits()[Mips::FeatureMips16];
+  }
+
+  bool useSoftFloat() const {
+    return STI.getFeatureBits()[Mips::FeatureSoftFloat];
   }
-  // TODO: see how can we get this info.
-  bool abiUsesSoftFloat() const { return false; }
 
-  /// Warn if RegNo is the current assembler temporary.
-  void warnIfAssemblerTemporary(int RegNo, SMLoc Loc);
+  /// Warn if RegIndex is the same as the current AT.
+  void warnIfRegIndexIsAT(unsigned RegIndex, SMLoc Loc);
+
+  void warnIfNoMacro(SMLoc Loc);
+
+  bool isLittle() const { return IsLittleEndian; }
 };
 }
 
@@ -460,11 +514,11 @@ class MipsOperand : public MCParsedAsmOperand {
     RegKind_CCR = 128,    /// CCR
     RegKind_HWRegs = 256, /// HWRegs
     RegKind_COP3 = 512,   /// COP3
-
+    RegKind_COP0 = 1024,  /// COP0
     /// Potentially any (e.g. $1)
     RegKind_Numeric = RegKind_GPR | RegKind_FGR | RegKind_FCC | RegKind_MSA128 |
                       RegKind_MSACtrl | RegKind_COP2 | RegKind_ACC |
-                      RegKind_CCR | RegKind_HWRegs | RegKind_COP3
+                      RegKind_CCR | RegKind_HWRegs | RegKind_COP3 | RegKind_COP0
   };
 
 private:
@@ -544,7 +598,7 @@ class MipsOperand : public MCParsedAsmOperand {
   /// target.
   unsigned getGPR32Reg() const {
     assert(isRegIdx() && (RegIdx.Kind & RegKind_GPR) && "Invalid access!");
-    AsmParser.warnIfAssemblerTemporary(RegIdx.Index, StartLoc);
+    AsmParser.warnIfRegIndexIsAT(RegIdx.Index, StartLoc);
     unsigned ClassID = Mips::GPR32RegClassID;
     return RegIdx.RegInfo->getRegClass(ClassID).getRegister(RegIdx.Index);
   }
@@ -626,6 +680,14 @@ class MipsOperand : public MCParsedAsmOperand {
     return RegIdx.RegInfo->getRegClass(ClassID).getRegister(RegIdx.Index);
   }
 
+  /// Coerce the register to COP0 and return the real register for the
+  /// current target.
+  unsigned getCOP0Reg() const {
+    assert(isRegIdx() && (RegIdx.Kind & RegKind_COP0) && "Invalid access!");
+    unsigned ClassID = Mips::COP0RegClassID;
+    return RegIdx.RegInfo->getRegClass(ClassID).getRegister(RegIdx.Index);
+  }
+
   /// Coerce the register to COP2 and return the real register for the
   /// current target.
   unsigned getCOP2Reg() const {
@@ -686,11 +748,11 @@ class MipsOperand : public MCParsedAsmOperand {
   void addExpr(MCInst &Inst, const MCExpr *Expr) const {
     // Add as immediate when possible.  Null MCExpr = 0.
     if (!Expr)
-      Inst.addOperand(MCOperand::CreateImm(0));
+      Inst.addOperand(MCOperand::createImm(0));
     else if (const MCConstantExpr *CE = dyn_cast<MCConstantExpr>(Expr))
-      Inst.addOperand(MCOperand::CreateImm(CE->getValue()));
+      Inst.addOperand(MCOperand::createImm(CE->getValue()));
     else
-      Inst.addOperand(MCOperand::CreateExpr(Expr));
+      Inst.addOperand(MCOperand::createExpr(Expr));
   }
 
   void addRegOperands(MCInst &Inst, unsigned N) const {
@@ -702,22 +764,22 @@ class MipsOperand : public MCParsedAsmOperand {
   /// is not a k_RegisterIndex compatible with RegKind_GPR
   void addGPR32AsmRegOperands(MCInst &Inst, unsigned N) const {
     assert(N == 1 && "Invalid number of operands!");
-    Inst.addOperand(MCOperand::CreateReg(getGPR32Reg()));
+    Inst.addOperand(MCOperand::createReg(getGPR32Reg()));
   }
 
   void addGPRMM16AsmRegOperands(MCInst &Inst, unsigned N) const {
     assert(N == 1 && "Invalid number of operands!");
-    Inst.addOperand(MCOperand::CreateReg(getGPRMM16Reg()));
+    Inst.addOperand(MCOperand::createReg(getGPRMM16Reg()));
   }
 
   void addGPRMM16AsmRegZeroOperands(MCInst &Inst, unsigned N) const {
     assert(N == 1 && "Invalid number of operands!");
-    Inst.addOperand(MCOperand::CreateReg(getGPRMM16Reg()));
+    Inst.addOperand(MCOperand::createReg(getGPRMM16Reg()));
   }
 
   void addGPRMM16AsmRegMovePOperands(MCInst &Inst, unsigned N) const {
     assert(N == 1 && "Invalid number of operands!");
-    Inst.addOperand(MCOperand::CreateReg(getGPRMM16Reg()));
+    Inst.addOperand(MCOperand::createReg(getGPRMM16Reg()));
   }
 
   /// Render the operand to an MCInst as a GPR64
@@ -725,22 +787,22 @@ class MipsOperand : public MCParsedAsmOperand {
   /// is not a k_RegisterIndex compatible with RegKind_GPR
   void addGPR64AsmRegOperands(MCInst &Inst, unsigned N) const {
     assert(N == 1 && "Invalid number of operands!");
-    Inst.addOperand(MCOperand::CreateReg(getGPR64Reg()));
+    Inst.addOperand(MCOperand::createReg(getGPR64Reg()));
   }
 
   void addAFGR64AsmRegOperands(MCInst &Inst, unsigned N) const {
     assert(N == 1 && "Invalid number of operands!");
-    Inst.addOperand(MCOperand::CreateReg(getAFGR64Reg()));
+    Inst.addOperand(MCOperand::createReg(getAFGR64Reg()));
   }
 
   void addFGR64AsmRegOperands(MCInst &Inst, unsigned N) const {
     assert(N == 1 && "Invalid number of operands!");
-    Inst.addOperand(MCOperand::CreateReg(getFGR64Reg()));
+    Inst.addOperand(MCOperand::createReg(getFGR64Reg()));
   }
 
   void addFGR32AsmRegOperands(MCInst &Inst, unsigned N) const {
     assert(N == 1 && "Invalid number of operands!");
-    Inst.addOperand(MCOperand::CreateReg(getFGR32Reg()));
+    Inst.addOperand(MCOperand::createReg(getFGR32Reg()));
     // FIXME: We ought to do this for -integrated-as without -via-file-asm too.
     if (!AsmParser.useOddSPReg() && RegIdx.Index & 1)
       AsmParser.Error(StartLoc, "-mno-odd-spreg prohibits the use of odd FPU "
@@ -749,57 +811,62 @@ class MipsOperand : public MCParsedAsmOperand {
 
   void addFGRH32AsmRegOperands(MCInst &Inst, unsigned N) const {
     assert(N == 1 && "Invalid number of operands!");
-    Inst.addOperand(MCOperand::CreateReg(getFGRH32Reg()));
+    Inst.addOperand(MCOperand::createReg(getFGRH32Reg()));
   }
 
   void addFCCAsmRegOperands(MCInst &Inst, unsigned N) const {
     assert(N == 1 && "Invalid number of operands!");
-    Inst.addOperand(MCOperand::CreateReg(getFCCReg()));
+    Inst.addOperand(MCOperand::createReg(getFCCReg()));
   }
 
   void addMSA128AsmRegOperands(MCInst &Inst, unsigned N) const {
     assert(N == 1 && "Invalid number of operands!");
-    Inst.addOperand(MCOperand::CreateReg(getMSA128Reg()));
+    Inst.addOperand(MCOperand::createReg(getMSA128Reg()));
   }
 
   void addMSACtrlAsmRegOperands(MCInst &Inst, unsigned N) const {
     assert(N == 1 && "Invalid number of operands!");
-    Inst.addOperand(MCOperand::CreateReg(getMSACtrlReg()));
+    Inst.addOperand(MCOperand::createReg(getMSACtrlReg()));
+  }
+
+  void addCOP0AsmRegOperands(MCInst &Inst, unsigned N) const {
+    assert(N == 1 && "Invalid number of operands!");
+    Inst.addOperand(MCOperand::createReg(getCOP0Reg()));
   }
 
   void addCOP2AsmRegOperands(MCInst &Inst, unsigned N) const {
     assert(N == 1 && "Invalid number of operands!");
-    Inst.addOperand(MCOperand::CreateReg(getCOP2Reg()));
+    Inst.addOperand(MCOperand::createReg(getCOP2Reg()));
   }
 
   void addCOP3AsmRegOperands(MCInst &Inst, unsigned N) const {
     assert(N == 1 && "Invalid number of operands!");
-    Inst.addOperand(MCOperand::CreateReg(getCOP3Reg()));
+    Inst.addOperand(MCOperand::createReg(getCOP3Reg()));
   }
 
   void addACC64DSPAsmRegOperands(MCInst &Inst, unsigned N) const {
     assert(N == 1 && "Invalid number of operands!");
-    Inst.addOperand(MCOperand::CreateReg(getACC64DSPReg()));
+    Inst.addOperand(MCOperand::createReg(getACC64DSPReg()));
   }
 
   void addHI32DSPAsmRegOperands(MCInst &Inst, unsigned N) const {
     assert(N == 1 && "Invalid number of operands!");
-    Inst.addOperand(MCOperand::CreateReg(getHI32DSPReg()));
+    Inst.addOperand(MCOperand::createReg(getHI32DSPReg()));
   }
 
   void addLO32DSPAsmRegOperands(MCInst &Inst, unsigned N) const {
     assert(N == 1 && "Invalid number of operands!");
-    Inst.addOperand(MCOperand::CreateReg(getLO32DSPReg()));
+    Inst.addOperand(MCOperand::createReg(getLO32DSPReg()));
   }
 
   void addCCRAsmRegOperands(MCInst &Inst, unsigned N) const {
     assert(N == 1 && "Invalid number of operands!");
-    Inst.addOperand(MCOperand::CreateReg(getCCRReg()));
+    Inst.addOperand(MCOperand::createReg(getCCRReg()));
   }
 
   void addHWRegsAsmRegOperands(MCInst &Inst, unsigned N) const {
     assert(N == 1 && "Invalid number of operands!");
-    Inst.addOperand(MCOperand::CreateReg(getHWRegsReg()));
+    Inst.addOperand(MCOperand::createReg(getHWRegsReg()));
   }
 
   void addImmOperands(MCInst &Inst, unsigned N) const {
@@ -811,7 +878,7 @@ class MipsOperand : public MCParsedAsmOperand {
   void addMemOperands(MCInst &Inst, unsigned N) const {
     assert(N == 2 && "Invalid number of operands!");
 
-    Inst.addOperand(MCOperand::CreateReg(getMemBase()->getGPR32Reg()));
+    Inst.addOperand(MCOperand::createReg(getMemBase()->getGPR32Reg()));
 
     const MCExpr *Expr = getMemOff();
     addExpr(Inst, Expr);
@@ -820,7 +887,7 @@ class MipsOperand : public MCParsedAsmOperand {
   void addMicroMipsMemOperands(MCInst &Inst, unsigned N) const {
     assert(N == 2 && "Invalid number of operands!");
 
-    Inst.addOperand(MCOperand::CreateReg(getMemBase()->getGPRMM16Reg()));
+    Inst.addOperand(MCOperand::createReg(getMemBase()->getGPRMM16Reg()));
 
     const MCExpr *Expr = getMemOff();
     addExpr(Inst, Expr);
@@ -830,20 +897,20 @@ class MipsOperand : public MCParsedAsmOperand {
     assert(N == 1 && "Invalid number of operands!");
 
     for (auto RegNo : getRegList())
-      Inst.addOperand(MCOperand::CreateReg(RegNo));
+      Inst.addOperand(MCOperand::createReg(RegNo));
   }
 
   void addRegPairOperands(MCInst &Inst, unsigned N) const {
     assert(N == 2 && "Invalid number of operands!");
     unsigned RegNo = getRegPair();
-    Inst.addOperand(MCOperand::CreateReg(RegNo++));
-    Inst.addOperand(MCOperand::CreateReg(RegNo));
+    Inst.addOperand(MCOperand::createReg(RegNo++));
+    Inst.addOperand(MCOperand::createReg(RegNo));
   }
 
   void addMovePRegPairOperands(MCInst &Inst, unsigned N) const {
     assert(N == 2 && "Invalid number of operands!");
     for (auto RegNo : getRegList())
-      Inst.addOperand(MCOperand::CreateReg(RegNo));
+      Inst.addOperand(MCOperand::createReg(RegNo));
   }
 
   bool isReg() const override {
@@ -859,6 +926,9 @@ class MipsOperand : public MCParsedAsmOperand {
   bool isConstantImm() const {
     return isImm() && dyn_cast<MCConstantExpr>(getImm());
   }
+  template <unsigned Bits> bool isUImm() const {
+    return isImm() && isConstantImm() && isUInt<Bits>(getConstantImm());
+  }
   bool isToken() const override {
     // Note: It's not possible to pretend that other operand kinds are tokens.
     // The matcher emitter checks tokens first.
@@ -1139,6 +1209,9 @@ class MipsOperand : public MCParsedAsmOperand {
   bool isACCAsmReg() const {
     return isRegIdx() && RegIdx.Kind & RegKind_ACC && RegIdx.Index <= 3;
   }
+  bool isCOP0AsmReg() const {
+    return isRegIdx() && RegIdx.Kind & RegKind_COP0 && RegIdx.Index <= 31;
+  }
   bool isCOP2AsmReg() const {
     return isRegIdx() && RegIdx.Kind & RegKind_COP2 && RegIdx.Index <= 31;
   }
@@ -1178,14 +1251,14 @@ class MipsOperand : public MCParsedAsmOperand {
     switch (Kind) {
     case k_Immediate:
       OS << "Imm<";
-      Imm.Val->print(OS);
+      OS << *Imm.Val;
       OS << ">";
       break;
     case k_Memory:
       OS << "Mem<";
       Mem.Base->print(OS);
       OS << ", ";
-      Mem.Off->print(OS);
+      OS << *Mem.Off;
       OS << ">";
       break;
     case k_PhysRegister:
@@ -1381,14 +1454,6 @@ bool MipsAsmParser::processInstruction(MCInst &Inst, SMLoc IDLoc,
     }
   }
 
-  // If this instruction has a delay slot and .set reorder is active,
-  // emit a NOP after it.
-  if (MCID.hasDelaySlot() && AssemblerOptions.back()->isReorder()) {
-    Instructions.push_back(Inst);
-    createNop(hasShortDelaySlot(Inst.getOpcode()), IDLoc, Instructions);
-    return false;
-  }
-
   if (MCID.mayLoad() || MCID.mayStore()) {
     // Check the offset of memory operand, if it is a symbol
     // reference or immediate we may have to expand instructions.
@@ -1442,9 +1507,9 @@ bool MipsAsmParser::processInstruction(MCInst &Inst, SMLoc IDLoc,
               MCInst TmpInst;
               TmpInst.setLoc(IDLoc);
               TmpInst.setOpcode(Mips::LWGP_MM);
-              TmpInst.addOperand(MCOperand::CreateReg(DstReg.getReg()));
-              TmpInst.addOperand(MCOperand::CreateReg(Mips::GP));
-              TmpInst.addOperand(MCOperand::CreateImm(MemOffset));
+              TmpInst.addOperand(MCOperand::createReg(DstReg.getReg()));
+              TmpInst.addOperand(MCOperand::createReg(Mips::GP));
+              TmpInst.addOperand(MCOperand::createImm(MemOffset));
               Instructions.push_back(TmpInst);
               return false;
             }
@@ -1578,11 +1643,17 @@ bool MipsAsmParser::processInstruction(MCInst &Inst, SMLoc IDLoc,
     }
   }
 
-  if (needsExpansion(Inst))
-    return expandInstruction(Inst, IDLoc, Instructions);
-  else
+  if (needsExpansion(Inst)) {
+    if (expandInstruction(Inst, IDLoc, Instructions))
+      return true;
+  } else
     Instructions.push_back(Inst);
 
+  // If this instruction has a delay slot and .set reorder is active,
+  // emit a NOP after it.
+  if (MCID.hasDelaySlot() && AssemblerOptions.back()->isReorder())
+    createNop(hasShortDelaySlot(Inst.getOpcode()), IDLoc, Instructions);
+
   return false;
 }
 
@@ -1598,6 +1669,18 @@ bool MipsAsmParser::needsExpansion(MCInst &Inst) {
   case Mips::SWM_MM:
   case Mips::JalOneReg:
   case Mips::JalTwoReg:
+  case Mips::BneImm:
+  case Mips::BeqImm:
+  case Mips::BLT:
+  case Mips::BLE:
+  case Mips::BGE:
+  case Mips::BGT:
+  case Mips::BLTU:
+  case Mips::BLEU:
+  case Mips::BGEU:
+  case Mips::BGTU:
+  case Mips::Ulhu:
+  case Mips::Ulw:
     return true;
   default:
     return false;
@@ -1609,17 +1692,13 @@ bool MipsAsmParser::expandInstruction(MCInst &Inst, SMLoc IDLoc,
   switch (Inst.getOpcode()) {
   default: llvm_unreachable("unimplemented expansion");
   case Mips::LoadImm32:
-    return expandLoadImm(Inst, IDLoc, Instructions);
+    return expandLoadImm(Inst, true, IDLoc, Instructions);
   case Mips::LoadImm64:
-    if (!isGP64bit()) {
-      Error(IDLoc, "instruction requires a 64-bit architecture");
-      return true;
-    }
-    return expandLoadImm(Inst, IDLoc, Instructions);
+    return expandLoadImm(Inst, false, IDLoc, Instructions);
   case Mips::LoadAddrImm32:
-    return expandLoadAddressImm(Inst, IDLoc, Instructions);
+    return expandLoadAddressImm(Inst, true, IDLoc, Instructions);
   case Mips::LoadAddrReg32:
-    return expandLoadAddressReg(Inst, IDLoc, Instructions);
+    return expandLoadAddressReg(Inst, true, IDLoc, Instructions);
   case Mips::B_MM_Pseudo:
     return expandUncondBranchMMPseudo(Inst, IDLoc, Instructions);
   case Mips::SWM_MM:
@@ -1628,37 +1707,86 @@ bool MipsAsmParser::expandInstruction(MCInst &Inst, SMLoc IDLoc,
   case Mips::JalOneReg:
   case Mips::JalTwoReg:
     return expandJalWithRegs(Inst, IDLoc, Instructions);
+  case Mips::BneImm:
+  case Mips::BeqImm:
+    return expandBranchImm(Inst, IDLoc, Instructions);
+  case Mips::BLT:
+  case Mips::BLE:
+  case Mips::BGE:
+  case Mips::BGT:
+  case Mips::BLTU:
+  case Mips::BLEU:
+  case Mips::BGEU:
+  case Mips::BGTU:
+    return expandCondBranches(Inst, IDLoc, Instructions);
+  case Mips::Ulhu:
+    return expandUlhu(Inst, IDLoc, Instructions);
+  case Mips::Ulw:
+    return expandUlw(Inst, IDLoc, Instructions);
   }
 }
 
 namespace {
-template <bool PerformShift>
-void createShiftOr(MCOperand Operand, unsigned RegNo, SMLoc IDLoc,
-                   SmallVectorImpl<MCInst> &Instructions) {
+void emitRX(unsigned Opcode, unsigned DstReg, MCOperand Imm, SMLoc IDLoc,
+            SmallVectorImpl<MCInst> &Instructions) {
   MCInst tmpInst;
-  if (PerformShift) {
-    tmpInst.setOpcode(Mips::DSLL);
-    tmpInst.addOperand(MCOperand::CreateReg(RegNo));
-    tmpInst.addOperand(MCOperand::CreateReg(RegNo));
-    tmpInst.addOperand(MCOperand::CreateImm(16));
-    tmpInst.setLoc(IDLoc);
-    Instructions.push_back(tmpInst);
-    tmpInst.clear();
-  }
-  tmpInst.setOpcode(Mips::ORi);
-  tmpInst.addOperand(MCOperand::CreateReg(RegNo));
-  tmpInst.addOperand(MCOperand::CreateReg(RegNo));
-  tmpInst.addOperand(Operand);
+  tmpInst.setOpcode(Opcode);
+  tmpInst.addOperand(MCOperand::createReg(DstReg));
+  tmpInst.addOperand(Imm);
+  tmpInst.setLoc(IDLoc);
+  Instructions.push_back(tmpInst);
+}
+
+void emitRI(unsigned Opcode, unsigned DstReg, int16_t Imm, SMLoc IDLoc,
+            SmallVectorImpl<MCInst> &Instructions) {
+  emitRX(Opcode, DstReg, MCOperand::createImm(Imm), IDLoc, Instructions);
+}
+
+
+void emitRRX(unsigned Opcode, unsigned DstReg, unsigned SrcReg, MCOperand Imm,
+             SMLoc IDLoc, SmallVectorImpl<MCInst> &Instructions) {
+  MCInst tmpInst;
+  tmpInst.setOpcode(Opcode);
+  tmpInst.addOperand(MCOperand::createReg(DstReg));
+  tmpInst.addOperand(MCOperand::createReg(SrcReg));
+  tmpInst.addOperand(Imm);
   tmpInst.setLoc(IDLoc);
   Instructions.push_back(tmpInst);
 }
 
-template <int Shift, bool PerformShift>
-void createShiftOr(int64_t Value, unsigned RegNo, SMLoc IDLoc,
-                   SmallVectorImpl<MCInst> &Instructions) {
-  createShiftOr<PerformShift>(
-      MCOperand::CreateImm(((Value & (0xffffLL << Shift)) >> Shift)), RegNo,
-      IDLoc, Instructions);
+void emitRRR(unsigned Opcode, unsigned DstReg, unsigned SrcReg,
+             unsigned SrcReg2, SMLoc IDLoc,
+             SmallVectorImpl<MCInst> &Instructions) {
+  emitRRX(Opcode, DstReg, SrcReg, MCOperand::createReg(SrcReg2), IDLoc,
+          Instructions);
+}
+
+void emitRRI(unsigned Opcode, unsigned DstReg, unsigned SrcReg, int16_t Imm,
+             SMLoc IDLoc, SmallVectorImpl<MCInst> &Instructions) {
+  emitRRX(Opcode, DstReg, SrcReg, MCOperand::createImm(Imm), IDLoc,
+          Instructions);
+}
+
+template <int16_t ShiftAmount>
+void createLShiftOri(MCOperand Operand, unsigned RegNo, SMLoc IDLoc,
+                     SmallVectorImpl<MCInst> &Instructions) {
+  if (ShiftAmount >= 32)
+    emitRRI(Mips::DSLL32, RegNo, RegNo, ShiftAmount - 32, IDLoc, Instructions);
+  else if (ShiftAmount > 0)
+    emitRRI(Mips::DSLL, RegNo, RegNo, ShiftAmount, IDLoc, Instructions);
+
+  // There's no need for an ORi if the immediate is 0.
+  if (Operand.isImm() && Operand.getImm() == 0)
+    return;
+
+  emitRRX(Mips::ORi, RegNo, RegNo, Operand, IDLoc, Instructions);
+}
+
+template <unsigned ShiftAmount>
+void createLShiftOri(int64_t Value, unsigned RegNo, SMLoc IDLoc,
+                     SmallVectorImpl<MCInst> &Instructions) {
+  createLShiftOri<ShiftAmount>(MCOperand::createImm(Value), RegNo, IDLoc,
+                               Instructions);
 }
 }
 
@@ -1677,7 +1805,7 @@ bool MipsAsmParser::expandJalWithRegs(MCInst &Inst, SMLoc IDLoc,
       JalrInst.addOperand(FirstRegOp);
     } else {
       JalrInst.setOpcode(Mips::JALR);
-      JalrInst.addOperand(MCOperand::CreateReg(Mips::RA));
+      JalrInst.addOperand(MCOperand::createReg(Mips::RA));
       JalrInst.addOperand(FirstRegOp);
     }
   } else if (Opcode == Mips::JalTwoReg) {
@@ -1695,64 +1823,99 @@ bool MipsAsmParser::expandJalWithRegs(MCInst &Inst, SMLoc IDLoc,
     // do not have a short delay slot.
     MCInst NopInst;
     NopInst.setOpcode(Mips::SLL);
-    NopInst.addOperand(MCOperand::CreateReg(Mips::ZERO));
-    NopInst.addOperand(MCOperand::CreateReg(Mips::ZERO));
-    NopInst.addOperand(MCOperand::CreateImm(0));
+    NopInst.addOperand(MCOperand::createReg(Mips::ZERO));
+    NopInst.addOperand(MCOperand::createReg(Mips::ZERO));
+    NopInst.addOperand(MCOperand::createImm(0));
     Instructions.push_back(NopInst);
   }
 
   return false;
 }
 
-bool MipsAsmParser::expandLoadImm(MCInst &Inst, SMLoc IDLoc,
+bool MipsAsmParser::loadImmediate(int64_t ImmValue, unsigned DstReg,
+                                  unsigned SrcReg, bool Is32BitImm, SMLoc IDLoc,
                                   SmallVectorImpl<MCInst> &Instructions) {
-  MCInst tmpInst;
-  const MCOperand &ImmOp = Inst.getOperand(1);
-  assert(ImmOp.isImm() && "expected immediate operand kind");
-  const MCOperand &RegOp = Inst.getOperand(0);
-  assert(RegOp.isReg() && "expected register operand kind");
+  if (!Is32BitImm && !isGP64bit()) {
+    Error(IDLoc, "instruction requires a 64-bit architecture");
+    return true;
+  }
+
+  if (Is32BitImm) {
+    if (isInt<32>(ImmValue) || isUInt<32>(ImmValue)) {
+      // Sign extend up to 64-bit so that the predicates match the hardware
+      // behaviour. In particular, isInt<16>(0xffff8000) and similar should be
+      // true.
+      ImmValue = SignExtend64<32>(ImmValue);
+    } else {
+      Error(IDLoc, "instruction requires a 32-bit immediate");
+      return true;
+    }
+  }
+
+  bool UseSrcReg = false;
+  if (SrcReg != Mips::NoRegister)
+    UseSrcReg = true;
+
+  unsigned TmpReg = DstReg;
+  if (UseSrcReg && (DstReg == SrcReg)) {
+    // At this point we need AT to perform the expansions and we exit if it is
+    // not available.
+    unsigned ATReg = getATReg(IDLoc);
+    if (!ATReg)
+      return true;
+    TmpReg = ATReg;
+  }
 
-  int64_t ImmValue = ImmOp.getImm();
-  tmpInst.setLoc(IDLoc);
   // FIXME: gas has a special case for values that are 000...1111, which
   // becomes a li -1 and then a dsrl
-  if (0 <= ImmValue && ImmValue <= 65535) {
-    // For unsigned and positive signed 16-bit values (0 <= j <= 65535):
-    // li d,j => ori d,$zero,j
-    tmpInst.setOpcode(Mips::ORi);
-    tmpInst.addOperand(MCOperand::CreateReg(RegOp.getReg()));
-    tmpInst.addOperand(MCOperand::CreateReg(Mips::ZERO));
-    tmpInst.addOperand(MCOperand::CreateImm(ImmValue));
-    Instructions.push_back(tmpInst);
-  } else if (ImmValue < 0 && ImmValue >= -32768) {
-    // For negative signed 16-bit values (-32768 <= j < 0):
+  if (isInt<16>(ImmValue)) {
     // li d,j => addiu d,$zero,j
-    tmpInst.setOpcode(Mips::ADDiu);
-    tmpInst.addOperand(MCOperand::CreateReg(RegOp.getReg()));
-    tmpInst.addOperand(MCOperand::CreateReg(Mips::ZERO));
-    tmpInst.addOperand(MCOperand::CreateImm(ImmValue));
-    Instructions.push_back(tmpInst);
-  } else if ((ImmValue & 0xffffffff) == ImmValue) {
+    if (!UseSrcReg)
+      SrcReg = Mips::ZERO;
+    emitRRI(Mips::ADDiu, DstReg, SrcReg, ImmValue, IDLoc, Instructions);
+  } else if (isUInt<16>(ImmValue)) {
+    // li d,j => ori d,$zero,j
+    unsigned TmpReg = DstReg;
+    if (SrcReg == DstReg) {
+      unsigned ATReg = getATReg(IDLoc);
+      if (!ATReg)
+        return true;
+      TmpReg = ATReg;
+    }
+
+    emitRRI(Mips::ORi, TmpReg, Mips::ZERO, ImmValue, IDLoc, Instructions);
+    if (UseSrcReg)
+      emitRRR(Mips::ADDu, DstReg, TmpReg, SrcReg, IDLoc, Instructions);
+  } else if (isInt<32>(ImmValue) || isUInt<32>(ImmValue)) {
+    warnIfNoMacro(IDLoc);
+
     // For all other values which are representable as a 32-bit integer:
     // li d,j => lui d,hi16(j)
     //           ori d,d,lo16(j)
-    tmpInst.setOpcode(Mips::LUi);
-    tmpInst.addOperand(MCOperand::CreateReg(RegOp.getReg()));
-    tmpInst.addOperand(MCOperand::CreateImm((ImmValue & 0xffff0000) >> 16));
-    Instructions.push_back(tmpInst);
-    createShiftOr<0, false>(ImmValue, RegOp.getReg(), IDLoc, Instructions);
+    uint16_t Bits31To16 = (ImmValue >> 16) & 0xffff;
+    uint16_t Bits15To0 = ImmValue & 0xffff;
+
+    if (!Is32BitImm && !isInt<32>(ImmValue)) {
+      // For DLI, expand to an ORi instead of a LUi to avoid sign-extending the
+      // upper 32 bits.
+      emitRRI(Mips::ORi, TmpReg, Mips::ZERO, Bits31To16, IDLoc, Instructions);
+      emitRRI(Mips::DSLL, TmpReg, TmpReg, 16, IDLoc, Instructions);
+    } else
+      emitRI(Mips::LUi, TmpReg, Bits31To16, IDLoc, Instructions);
+    createLShiftOri<0>(Bits15To0, TmpReg, IDLoc, Instructions);
+
+    if (UseSrcReg)
+      createAddu(DstReg, TmpReg, SrcReg, !Is32BitImm, Instructions);
+
   } else if ((ImmValue & (0xffffLL << 48)) == 0) {
-    if (!isGP64bit()) {
-      Error(IDLoc, "instruction requires a 64-bit architecture");
-      return true;
-    }
+    warnIfNoMacro(IDLoc);
 
     //            <-------  lo32 ------>
     // <-------  hi32 ------>
     // <- hi16 ->             <- lo16 ->
     //  _________________________________
     // |          |          |          |
-    // | 16-bytes | 16-bytes | 16-bytes |
+    // | 16-bits  | 16-bits  | 16-bits  |
     // |__________|__________|__________|
     //
     // For any 64-bit value that is representable as a 48-bit integer:
@@ -1760,24 +1923,25 @@ bool MipsAsmParser::expandLoadImm(MCInst &Inst, SMLoc IDLoc,
     //           ori d,d,hi16(lo32(j))
     //           dsll d,d,16
     //           ori d,d,lo16(lo32(j))
-    tmpInst.setOpcode(Mips::LUi);
-    tmpInst.addOperand(MCOperand::CreateReg(RegOp.getReg()));
-    tmpInst.addOperand(
-        MCOperand::CreateImm((ImmValue & (0xffffLL << 32)) >> 32));
-    Instructions.push_back(tmpInst);
-    createShiftOr<16, false>(ImmValue, RegOp.getReg(), IDLoc, Instructions);
-    createShiftOr<0, true>(ImmValue, RegOp.getReg(), IDLoc, Instructions);
+    uint16_t Bits47To32 = (ImmValue >> 32) & 0xffff;
+    uint16_t Bits31To16 = (ImmValue >> 16) & 0xffff;
+    uint16_t Bits15To0 = ImmValue & 0xffff;
+
+    emitRI(Mips::LUi, TmpReg, Bits47To32, IDLoc, Instructions);
+    createLShiftOri<0>(Bits31To16, TmpReg, IDLoc, Instructions);
+    createLShiftOri<16>(Bits15To0, TmpReg, IDLoc, Instructions);
+
+    if (UseSrcReg)
+      createAddu(DstReg, TmpReg, SrcReg, !Is32BitImm, Instructions);
+
   } else {
-    if (!isGP64bit()) {
-      Error(IDLoc, "instruction requires a 64-bit architecture");
-      return true;
-    }
+    warnIfNoMacro(IDLoc);
 
     // <-------  hi32 ------> <-------  lo32 ------>
     // <- hi16 ->                        <- lo16 ->
     //  ___________________________________________
     // |          |          |          |          |
-    // | 16-bytes | 16-bytes | 16-bytes | 16-bytes |
+    // | 16-bits  | 16-bits  | 16-bits  | 16-bits  |
     // |__________|__________|__________|__________|
     //
     // For all other values which are representable as a 64-bit integer:
@@ -1787,133 +1951,124 @@ bool MipsAsmParser::expandLoadImm(MCInst &Inst, SMLoc IDLoc,
     //           ori d,d,hi16(lo32(j))
     //           dsll d,d,16
     //           ori d,d,lo16(lo32(j))
-    tmpInst.setOpcode(Mips::LUi);
-    tmpInst.addOperand(MCOperand::CreateReg(RegOp.getReg()));
-    tmpInst.addOperand(
-        MCOperand::CreateImm((ImmValue & (0xffffLL << 48)) >> 48));
-    Instructions.push_back(tmpInst);
-    createShiftOr<32, false>(ImmValue, RegOp.getReg(), IDLoc, Instructions);
-    createShiftOr<16, true>(ImmValue, RegOp.getReg(), IDLoc, Instructions);
-    createShiftOr<0, true>(ImmValue, RegOp.getReg(), IDLoc, Instructions);
+    uint16_t Bits63To48 = (ImmValue >> 48) & 0xffff;
+    uint16_t Bits47To32 = (ImmValue >> 32) & 0xffff;
+    uint16_t Bits31To16 = (ImmValue >> 16) & 0xffff;
+    uint16_t Bits15To0 = ImmValue & 0xffff;
+
+    emitRI(Mips::LUi, TmpReg, Bits63To48, IDLoc, Instructions);
+    createLShiftOri<0>(Bits47To32, TmpReg, IDLoc, Instructions);
+
+    // When Bits31To16 is 0, do a left shift of 32 bits instead of doing
+    // two left shifts of 16 bits.
+    if (Bits31To16 == 0) {
+      createLShiftOri<32>(Bits15To0, TmpReg, IDLoc, Instructions);
+    } else {
+      createLShiftOri<16>(Bits31To16, TmpReg, IDLoc, Instructions);
+      createLShiftOri<16>(Bits15To0, TmpReg, IDLoc, Instructions);
+    }
+
+    if (UseSrcReg)
+      createAddu(DstReg, TmpReg, SrcReg, !Is32BitImm, Instructions);
   }
   return false;
 }
 
+bool MipsAsmParser::expandLoadImm(MCInst &Inst, bool Is32BitImm, SMLoc IDLoc,
+                                  SmallVectorImpl<MCInst> &Instructions) {
+  const MCOperand &ImmOp = Inst.getOperand(1);
+  assert(ImmOp.isImm() && "expected immediate operand kind");
+  const MCOperand &DstRegOp = Inst.getOperand(0);
+  assert(DstRegOp.isReg() && "expected register operand kind");
+
+  if (loadImmediate(ImmOp.getImm(), DstRegOp.getReg(), Mips::NoRegister,
+                    Is32BitImm, IDLoc, Instructions))
+    return true;
+
+  return false;
+}
+
 bool
-MipsAsmParser::expandLoadAddressReg(MCInst &Inst, SMLoc IDLoc,
+MipsAsmParser::expandLoadAddressReg(MCInst &Inst, bool Is32BitImm, SMLoc IDLoc,
                                     SmallVectorImpl<MCInst> &Instructions) {
-  MCInst tmpInst;
+  const MCOperand &DstRegOp = Inst.getOperand(0);
+  assert(DstRegOp.isReg() && "expected register operand kind");
+
+  const MCOperand &SrcRegOp = Inst.getOperand(1);
+  assert(SrcRegOp.isReg() && "expected register operand kind");
+
   const MCOperand &ImmOp = Inst.getOperand(2);
   assert((ImmOp.isImm() || ImmOp.isExpr()) &&
          "expected immediate operand kind");
   if (!ImmOp.isImm()) {
-    expandLoadAddressSym(Inst, IDLoc, Instructions);
+    if (loadAndAddSymbolAddress(ImmOp.getExpr(), DstRegOp.getReg(),
+                                SrcRegOp.getReg(), Is32BitImm, IDLoc,
+                                Instructions))
+      return true;
+
     return false;
   }
-  const MCOperand &SrcRegOp = Inst.getOperand(1);
-  assert(SrcRegOp.isReg() && "expected register operand kind");
-  const MCOperand &DstRegOp = Inst.getOperand(0);
-  assert(DstRegOp.isReg() && "expected register operand kind");
-  int ImmValue = ImmOp.getImm();
-  if (-32768 <= ImmValue && ImmValue <= 65535) {
-    // For -32768 <= j <= 65535.
-    // la d,j(s) => addiu d,s,j
-    tmpInst.setOpcode(Mips::ADDiu);
-    tmpInst.addOperand(MCOperand::CreateReg(DstRegOp.getReg()));
-    tmpInst.addOperand(MCOperand::CreateReg(SrcRegOp.getReg()));
-    tmpInst.addOperand(MCOperand::CreateImm(ImmValue));
-    Instructions.push_back(tmpInst);
-  } else {
-    // For any other value of j that is representable as a 32-bit integer.
-    // la d,j(s) => lui d,hi16(j)
-    //              ori d,d,lo16(j)
-    //              addu d,d,s
-    tmpInst.setOpcode(Mips::LUi);
-    tmpInst.addOperand(MCOperand::CreateReg(DstRegOp.getReg()));
-    tmpInst.addOperand(MCOperand::CreateImm((ImmValue & 0xffff0000) >> 16));
-    Instructions.push_back(tmpInst);
-    tmpInst.clear();
-    tmpInst.setOpcode(Mips::ORi);
-    tmpInst.addOperand(MCOperand::CreateReg(DstRegOp.getReg()));
-    tmpInst.addOperand(MCOperand::CreateReg(DstRegOp.getReg()));
-    tmpInst.addOperand(MCOperand::CreateImm(ImmValue & 0xffff));
-    Instructions.push_back(tmpInst);
-    tmpInst.clear();
-    tmpInst.setOpcode(Mips::ADDu);
-    tmpInst.addOperand(MCOperand::CreateReg(DstRegOp.getReg()));
-    tmpInst.addOperand(MCOperand::CreateReg(DstRegOp.getReg()));
-    tmpInst.addOperand(MCOperand::CreateReg(SrcRegOp.getReg()));
-    Instructions.push_back(tmpInst);
-  }
+
+  if (loadImmediate(ImmOp.getImm(), DstRegOp.getReg(), SrcRegOp.getReg(),
+                    Is32BitImm, IDLoc, Instructions))
+    return true;
+
   return false;
 }
 
 bool
-MipsAsmParser::expandLoadAddressImm(MCInst &Inst, SMLoc IDLoc,
+MipsAsmParser::expandLoadAddressImm(MCInst &Inst, bool Is32BitImm, SMLoc IDLoc,
                                     SmallVectorImpl<MCInst> &Instructions) {
-  MCInst tmpInst;
+  const MCOperand &DstRegOp = Inst.getOperand(0);
+  assert(DstRegOp.isReg() && "expected register operand kind");
+
   const MCOperand &ImmOp = Inst.getOperand(1);
   assert((ImmOp.isImm() || ImmOp.isExpr()) &&
          "expected immediate operand kind");
   if (!ImmOp.isImm()) {
-    expandLoadAddressSym(Inst, IDLoc, Instructions);
+    if (loadAndAddSymbolAddress(ImmOp.getExpr(), DstRegOp.getReg(),
+                                Mips::NoRegister, Is32BitImm, IDLoc,
+                                Instructions))
+      return true;
+
     return false;
   }
-  const MCOperand &RegOp = Inst.getOperand(0);
-  assert(RegOp.isReg() && "expected register operand kind");
-  int ImmValue = ImmOp.getImm();
-  if (-32768 <= ImmValue && ImmValue <= 65535) {
-    // For -32768 <= j <= 65535.
-    // la d,j => addiu d,$zero,j
-    tmpInst.setOpcode(Mips::ADDiu);
-    tmpInst.addOperand(MCOperand::CreateReg(RegOp.getReg()));
-    tmpInst.addOperand(MCOperand::CreateReg(Mips::ZERO));
-    tmpInst.addOperand(MCOperand::CreateImm(ImmValue));
-    Instructions.push_back(tmpInst);
-  } else {
-    // For any other value of j that is representable as a 32-bit integer.
-    // la d,j => lui d,hi16(j)
-    //           ori d,d,lo16(j)
-    tmpInst.setOpcode(Mips::LUi);
-    tmpInst.addOperand(MCOperand::CreateReg(RegOp.getReg()));
-    tmpInst.addOperand(MCOperand::CreateImm((ImmValue & 0xffff0000) >> 16));
-    Instructions.push_back(tmpInst);
-    tmpInst.clear();
-    tmpInst.setOpcode(Mips::ORi);
-    tmpInst.addOperand(MCOperand::CreateReg(RegOp.getReg()));
-    tmpInst.addOperand(MCOperand::CreateReg(RegOp.getReg()));
-    tmpInst.addOperand(MCOperand::CreateImm(ImmValue & 0xffff));
-    Instructions.push_back(tmpInst);
-  }
+
+  if (loadImmediate(ImmOp.getImm(), DstRegOp.getReg(), Mips::NoRegister,
+                    Is32BitImm, IDLoc, Instructions))
+    return true;
+
   return false;
 }
 
-void
-MipsAsmParser::expandLoadAddressSym(MCInst &Inst, SMLoc IDLoc,
-                                    SmallVectorImpl<MCInst> &Instructions) {
-  // FIXME: If we do have a valid at register to use, we should generate a
-  // slightly shorter sequence here.
+bool MipsAsmParser::loadAndAddSymbolAddress(
+    const MCExpr *SymExpr, unsigned DstReg, unsigned SrcReg, bool Is32BitSym,
+    SMLoc IDLoc, SmallVectorImpl<MCInst> &Instructions) {
+  warnIfNoMacro(IDLoc);
+
+  if (Is32BitSym && isABI_N64())
+    Warning(IDLoc, "instruction loads the 32-bit address of a 64-bit symbol");
+
   MCInst tmpInst;
-  int ExprOperandNo = 1;
-  // Sometimes the assembly parser will get the immediate expression as
-  // a $zero + an immediate.
-  if (Inst.getNumOperands() == 3) {
-    assert(Inst.getOperand(1).getReg() ==
-           (isGP64bit() ? Mips::ZERO_64 : Mips::ZERO));
-    ExprOperandNo = 2;
-  }
-  const MCOperand &SymOp = Inst.getOperand(ExprOperandNo);
-  assert(SymOp.isExpr() && "expected symbol operand kind");
-  const MCOperand &RegOp = Inst.getOperand(0);
-  unsigned RegNo = RegOp.getReg();
-  const MCSymbolRefExpr *Symbol = cast<MCSymbolRefExpr>(SymOp.getExpr());
-  const MCSymbolRefExpr *HiExpr =
-      MCSymbolRefExpr::Create(Symbol->getSymbol().getName(),
-                              MCSymbolRefExpr::VK_Mips_ABS_HI, getContext());
-  const MCSymbolRefExpr *LoExpr =
-      MCSymbolRefExpr::Create(Symbol->getSymbol().getName(),
-                              MCSymbolRefExpr::VK_Mips_ABS_LO, getContext());
-  if (isGP64bit()) {
+  const MCSymbolRefExpr *Symbol = cast<MCSymbolRefExpr>(SymExpr);
+  const MCSymbolRefExpr *HiExpr = MCSymbolRefExpr::create(
+      &Symbol->getSymbol(), MCSymbolRefExpr::VK_Mips_ABS_HI, getContext());
+  const MCSymbolRefExpr *LoExpr = MCSymbolRefExpr::create(
+      &Symbol->getSymbol(), MCSymbolRefExpr::VK_Mips_ABS_LO, getContext());
+
+  bool UseSrcReg = SrcReg != Mips::NoRegister;
+
+  unsigned TmpReg = DstReg;
+  if (UseSrcReg && (DstReg == SrcReg)) {
+    // At this point we need AT to perform the expansions and we exit if it is
+    // not available.
+    unsigned ATReg = getATReg(IDLoc);
+    if (!ATReg)
+      return true;
+    TmpReg = ATReg;
+  }
+
+  if (!Is32BitSym) {
     // If it's a 64-bit architecture, expand to:
     // la d,sym => lui  d,highest(sym)
     //             ori  d,d,higher(sym)
@@ -1921,36 +2076,39 @@ MipsAsmParser::expandLoadAddressSym(MCInst &Inst, SMLoc IDLoc,
     //             ori  d,d,hi16(sym)
     //             dsll d,d,16
     //             ori  d,d,lo16(sym)
-    const MCSymbolRefExpr *HighestExpr =
-        MCSymbolRefExpr::Create(Symbol->getSymbol().getName(),
-                                MCSymbolRefExpr::VK_Mips_HIGHEST, getContext());
-    const MCSymbolRefExpr *HigherExpr =
-        MCSymbolRefExpr::Create(Symbol->getSymbol().getName(),
-                                MCSymbolRefExpr::VK_Mips_HIGHER, getContext());
+    const MCSymbolRefExpr *HighestExpr = MCSymbolRefExpr::create(
+        &Symbol->getSymbol(), MCSymbolRefExpr::VK_Mips_HIGHEST, getContext());
+    const MCSymbolRefExpr *HigherExpr = MCSymbolRefExpr::create(
+        &Symbol->getSymbol(), MCSymbolRefExpr::VK_Mips_HIGHER, getContext());
 
     tmpInst.setOpcode(Mips::LUi);
-    tmpInst.addOperand(MCOperand::CreateReg(RegNo));
-    tmpInst.addOperand(MCOperand::CreateExpr(HighestExpr));
+    tmpInst.addOperand(MCOperand::createReg(TmpReg));
+    tmpInst.addOperand(MCOperand::createExpr(HighestExpr));
     Instructions.push_back(tmpInst);
 
-    createShiftOr<false>(MCOperand::CreateExpr(HigherExpr), RegNo, SMLoc(),
-                         Instructions);
-    createShiftOr<true>(MCOperand::CreateExpr(HiExpr), RegNo, SMLoc(),
+    createLShiftOri<0>(MCOperand::createExpr(HigherExpr), TmpReg, SMLoc(),
+                       Instructions);
+    createLShiftOri<16>(MCOperand::createExpr(HiExpr), TmpReg, SMLoc(),
                         Instructions);
-    createShiftOr<true>(MCOperand::CreateExpr(LoExpr), RegNo, SMLoc(),
+    createLShiftOri<16>(MCOperand::createExpr(LoExpr), TmpReg, SMLoc(),
                         Instructions);
   } else {
     // Otherwise, expand to:
     // la d,sym => lui  d,hi16(sym)
     //             ori  d,d,lo16(sym)
     tmpInst.setOpcode(Mips::LUi);
-    tmpInst.addOperand(MCOperand::CreateReg(RegNo));
-    tmpInst.addOperand(MCOperand::CreateExpr(HiExpr));
+    tmpInst.addOperand(MCOperand::createReg(TmpReg));
+    tmpInst.addOperand(MCOperand::createExpr(HiExpr));
     Instructions.push_back(tmpInst);
 
-    createShiftOr<false>(MCOperand::CreateExpr(LoExpr), RegNo, SMLoc(),
-                         Instructions);
+    emitRRX(Mips::ADDiu, TmpReg, TmpReg, MCOperand::createExpr(LoExpr), SMLoc(),
+            Instructions);
   }
+
+  if (UseSrcReg)
+    createAddu(DstReg, TmpReg, SrcReg, !Is32BitSym, Instructions);
+
+  return false;
 }
 
 bool MipsAsmParser::expandUncondBranchMMPseudo(
@@ -1962,9 +2120,9 @@ bool MipsAsmParser::expandUncondBranchMMPseudo(
   if (Offset.isExpr()) {
     Inst.clear();
     Inst.setOpcode(Mips::BEQ_MM);
-    Inst.addOperand(MCOperand::CreateReg(Mips::ZERO));
-    Inst.addOperand(MCOperand::CreateReg(Mips::ZERO));
-    Inst.addOperand(MCOperand::CreateExpr(Offset.getExpr()));
+    Inst.addOperand(MCOperand::createReg(Mips::ZERO));
+    Inst.addOperand(MCOperand::createReg(Mips::ZERO));
+    Inst.addOperand(MCOperand::createExpr(Offset.getExpr()));
   } else {
     assert(Offset.isImm() && "expected immediate operand kind");
     if (isIntN(11, Offset.getImm())) {
@@ -1978,9 +2136,9 @@ bool MipsAsmParser::expandUncondBranchMMPseudo(
         Error(IDLoc, "branch to misaligned address");
       Inst.clear();
       Inst.setOpcode(Mips::BEQ_MM);
-      Inst.addOperand(MCOperand::CreateReg(Mips::ZERO));
-      Inst.addOperand(MCOperand::CreateReg(Mips::ZERO));
-      Inst.addOperand(MCOperand::CreateImm(Offset.getImm()));
+      Inst.addOperand(MCOperand::createReg(Mips::ZERO));
+      Inst.addOperand(MCOperand::createReg(Mips::ZERO));
+      Inst.addOperand(MCOperand::createImm(Offset.getImm()));
     }
   }
   Instructions.push_back(Inst);
@@ -1992,10 +2150,62 @@ bool MipsAsmParser::expandUncondBranchMMPseudo(
   return false;
 }
 
+bool MipsAsmParser::expandBranchImm(MCInst &Inst, SMLoc IDLoc,
+                                    SmallVectorImpl<MCInst> &Instructions) {
+  const MCOperand &DstRegOp = Inst.getOperand(0);
+  assert(DstRegOp.isReg() && "expected register operand kind");
+
+  const MCOperand &ImmOp = Inst.getOperand(1);
+  assert(ImmOp.isImm() && "expected immediate operand kind");
+
+  const MCOperand &MemOffsetOp = Inst.getOperand(2);
+  assert(MemOffsetOp.isImm() && "expected immediate operand kind");
+
+  unsigned OpCode = 0;
+  switch(Inst.getOpcode()) {
+    case Mips::BneImm:
+      OpCode = Mips::BNE;
+      break;
+    case Mips::BeqImm:
+      OpCode = Mips::BEQ;
+      break;
+    default:
+      llvm_unreachable("Unknown immediate branch pseudo-instruction.");
+      break;
+  }
+
+  int64_t ImmValue = ImmOp.getImm();
+  if (ImmValue == 0) {
+    MCInst BranchInst;
+    BranchInst.setOpcode(OpCode);
+    BranchInst.addOperand(DstRegOp);
+    BranchInst.addOperand(MCOperand::createReg(Mips::ZERO));
+    BranchInst.addOperand(MemOffsetOp);
+    Instructions.push_back(BranchInst);
+  } else {
+    warnIfNoMacro(IDLoc);
+
+    unsigned ATReg = getATReg(IDLoc);
+    if (!ATReg)
+      return true;
+
+    if (loadImmediate(ImmValue, ATReg, Mips::NoRegister, !isGP64bit(), IDLoc,
+                      Instructions))
+      return true;
+
+    MCInst BranchInst;
+    BranchInst.setOpcode(OpCode);
+    BranchInst.addOperand(DstRegOp);
+    BranchInst.addOperand(MCOperand::createReg(ATReg));
+    BranchInst.addOperand(MemOffsetOp);
+    Instructions.push_back(BranchInst);
+  }
+  return false;
+}
+
 void MipsAsmParser::expandMemInst(MCInst &Inst, SMLoc IDLoc,
                                   SmallVectorImpl<MCInst> &Instructions,
                                   bool isLoad, bool isImmOpnd) {
-  const MCSymbolRefExpr *SR;
   MCInst TempInst;
   unsigned ImmOffset, HiOffset, LoOffset;
   const MCExpr *ExprOffset;
@@ -2058,20 +2268,12 @@ void MipsAsmParser::expandMemInst(MCInst &Inst, SMLoc IDLoc,
   }
 
   TempInst.setOpcode(Mips::LUi);
-  TempInst.addOperand(MCOperand::CreateReg(TmpRegNum));
+  TempInst.addOperand(MCOperand::createReg(TmpRegNum));
   if (isImmOpnd)
-    TempInst.addOperand(MCOperand::CreateImm(HiOffset));
+    TempInst.addOperand(MCOperand::createImm(HiOffset));
   else {
-    if (ExprOffset->getKind() == MCExpr::SymbolRef) {
-      SR = static_cast<const MCSymbolRefExpr *>(ExprOffset);
-      const MCSymbolRefExpr *HiExpr = MCSymbolRefExpr::Create(
-          SR->getSymbol().getName(), MCSymbolRefExpr::VK_Mips_ABS_HI,
-          getContext());
-      TempInst.addOperand(MCOperand::CreateExpr(HiExpr));
-    } else {
-      const MCExpr *HiExpr = evaluateRelocExpr(ExprOffset, "hi");
-      TempInst.addOperand(MCOperand::CreateExpr(HiExpr));
-    }
+    const MCExpr *HiExpr = evaluateRelocExpr(ExprOffset, "hi");
+    TempInst.addOperand(MCOperand::createExpr(HiExpr));
   }
   // Add the instruction to the list.
   Instructions.push_back(TempInst);
@@ -2080,29 +2282,22 @@ void MipsAsmParser::expandMemInst(MCInst &Inst, SMLoc IDLoc,
   // Add temp register to base.
   if (BaseRegNum != Mips::ZERO) {
     TempInst.setOpcode(Mips::ADDu);
-    TempInst.addOperand(MCOperand::CreateReg(TmpRegNum));
-    TempInst.addOperand(MCOperand::CreateReg(TmpRegNum));
-    TempInst.addOperand(MCOperand::CreateReg(BaseRegNum));
+    TempInst.addOperand(MCOperand::createReg(TmpRegNum));
+    TempInst.addOperand(MCOperand::createReg(TmpRegNum));
+    TempInst.addOperand(MCOperand::createReg(BaseRegNum));
     Instructions.push_back(TempInst);
     TempInst.clear();
   }
   // And finally, create original instruction with low part
   // of offset and new base.
   TempInst.setOpcode(Inst.getOpcode());
-  TempInst.addOperand(MCOperand::CreateReg(RegOpNum));
-  TempInst.addOperand(MCOperand::CreateReg(TmpRegNum));
+  TempInst.addOperand(MCOperand::createReg(RegOpNum));
+  TempInst.addOperand(MCOperand::createReg(TmpRegNum));
   if (isImmOpnd)
-    TempInst.addOperand(MCOperand::CreateImm(LoOffset));
+    TempInst.addOperand(MCOperand::createImm(LoOffset));
   else {
-    if (ExprOffset->getKind() == MCExpr::SymbolRef) {
-      const MCSymbolRefExpr *LoExpr = MCSymbolRefExpr::Create(
-          SR->getSymbol().getName(), MCSymbolRefExpr::VK_Mips_ABS_LO,
-          getContext());
-      TempInst.addOperand(MCOperand::CreateExpr(LoExpr));
-    } else {
-      const MCExpr *LoExpr = evaluateRelocExpr(ExprOffset, "lo");
-      TempInst.addOperand(MCOperand::CreateExpr(LoExpr));
-    }
+    const MCExpr *LoExpr = evaluateRelocExpr(ExprOffset, "lo");
+    TempInst.addOperand(MCOperand::createExpr(LoExpr));
   }
   Instructions.push_back(TempInst);
   TempInst.clear();
@@ -2131,22 +2326,397 @@ MipsAsmParser::expandLoadStoreMultiple(MCInst &Inst, SMLoc IDLoc,
   return false;
 }
 
+bool MipsAsmParser::expandCondBranches(MCInst &Inst, SMLoc IDLoc,
+                                       SmallVectorImpl<MCInst> &Instructions) {
+  unsigned PseudoOpcode = Inst.getOpcode();
+  unsigned SrcReg = Inst.getOperand(0).getReg();
+  unsigned TrgReg = Inst.getOperand(1).getReg();
+  const MCExpr *OffsetExpr = Inst.getOperand(2).getExpr();
+
+  unsigned ZeroSrcOpcode, ZeroTrgOpcode;
+  bool ReverseOrderSLT, IsUnsigned, AcceptsEquality;
+
+  switch (PseudoOpcode) {
+  case Mips::BLT:
+  case Mips::BLTU:
+    AcceptsEquality = false;
+    ReverseOrderSLT = false;
+    IsUnsigned = (PseudoOpcode == Mips::BLTU);
+    ZeroSrcOpcode = Mips::BGTZ;
+    ZeroTrgOpcode = Mips::BLTZ;
+    break;
+  case Mips::BLE:
+  case Mips::BLEU:
+    AcceptsEquality = true;
+    ReverseOrderSLT = true;
+    IsUnsigned = (PseudoOpcode == Mips::BLEU);
+    ZeroSrcOpcode = Mips::BGEZ;
+    ZeroTrgOpcode = Mips::BLEZ;
+    break;
+  case Mips::BGE:
+  case Mips::BGEU:
+    AcceptsEquality = true;
+    ReverseOrderSLT = false;
+    IsUnsigned = (PseudoOpcode == Mips::BGEU);
+    ZeroSrcOpcode = Mips::BLEZ;
+    ZeroTrgOpcode = Mips::BGEZ;
+    break;
+  case Mips::BGT:
+  case Mips::BGTU:
+    AcceptsEquality = false;
+    ReverseOrderSLT = true;
+    IsUnsigned = (PseudoOpcode == Mips::BGTU);
+    ZeroSrcOpcode = Mips::BLTZ;
+    ZeroTrgOpcode = Mips::BGTZ;
+    break;
+  default:
+    llvm_unreachable("unknown opcode for branch pseudo-instruction");
+  }
+
+  MCInst BranchInst;
+  bool IsTrgRegZero = (TrgReg == Mips::ZERO);
+  bool IsSrcRegZero = (SrcReg == Mips::ZERO);
+  if (IsSrcRegZero && IsTrgRegZero) {
+    // FIXME: All of these Opcode-specific if's are needed for compatibility
+    // with GAS' behaviour. However, they may not generate the most efficient
+    // code in some circumstances.
+    if (PseudoOpcode == Mips::BLT) {
+      BranchInst.setOpcode(Mips::BLTZ);
+      BranchInst.addOperand(MCOperand::createReg(Mips::ZERO));
+      BranchInst.addOperand(MCOperand::createExpr(OffsetExpr));
+      Instructions.push_back(BranchInst);
+      return false;
+    }
+    if (PseudoOpcode == Mips::BLE) {
+      BranchInst.setOpcode(Mips::BLEZ);
+      BranchInst.addOperand(MCOperand::createReg(Mips::ZERO));
+      BranchInst.addOperand(MCOperand::createExpr(OffsetExpr));
+      Instructions.push_back(BranchInst);
+      Warning(IDLoc, "branch is always taken");
+      return false;
+    }
+    if (PseudoOpcode == Mips::BGE) {
+      BranchInst.setOpcode(Mips::BGEZ);
+      BranchInst.addOperand(MCOperand::createReg(Mips::ZERO));
+      BranchInst.addOperand(MCOperand::createExpr(OffsetExpr));
+      Instructions.push_back(BranchInst);
+      Warning(IDLoc, "branch is always taken");
+      return false;
+    }
+    if (PseudoOpcode == Mips::BGT) {
+      BranchInst.setOpcode(Mips::BGTZ);
+      BranchInst.addOperand(MCOperand::createReg(Mips::ZERO));
+      BranchInst.addOperand(MCOperand::createExpr(OffsetExpr));
+      Instructions.push_back(BranchInst);
+      return false;
+    }
+    if (PseudoOpcode == Mips::BGTU) {
+      BranchInst.setOpcode(Mips::BNE);
+      BranchInst.addOperand(MCOperand::createReg(Mips::ZERO));
+      BranchInst.addOperand(MCOperand::createReg(Mips::ZERO));
+      BranchInst.addOperand(MCOperand::createExpr(OffsetExpr));
+      Instructions.push_back(BranchInst);
+      return false;
+    }
+    if (AcceptsEquality) {
+      // If both registers are $0 and the pseudo-branch accepts equality, it
+      // will always be taken, so we emit an unconditional branch.
+      BranchInst.setOpcode(Mips::BEQ);
+      BranchInst.addOperand(MCOperand::createReg(Mips::ZERO));
+      BranchInst.addOperand(MCOperand::createReg(Mips::ZERO));
+      BranchInst.addOperand(MCOperand::createExpr(OffsetExpr));
+      Instructions.push_back(BranchInst);
+      Warning(IDLoc, "branch is always taken");
+      return false;
+    }
+    // If both registers are $0 and the pseudo-branch does not accept
+    // equality, it will never be taken, so we don't have to emit anything.
+    return false;
+  }
+  if (IsSrcRegZero || IsTrgRegZero) {
+    if ((IsSrcRegZero && PseudoOpcode == Mips::BGTU) ||
+        (IsTrgRegZero && PseudoOpcode == Mips::BLTU)) {
+      // If the $rs is $0 and the pseudo-branch is BGTU (0 > x) or
+      // if the $rt is $0 and the pseudo-branch is BLTU (x < 0),
+      // the pseudo-branch will never be taken, so we don't emit anything.
+      // This only applies to unsigned pseudo-branches.
+      return false;
+    }
+    if ((IsSrcRegZero && PseudoOpcode == Mips::BLEU) ||
+        (IsTrgRegZero && PseudoOpcode == Mips::BGEU)) {
+      // If the $rs is $0 and the pseudo-branch is BLEU (0 <= x) or
+      // if the $rt is $0 and the pseudo-branch is BGEU (x >= 0),
+      // the pseudo-branch will always be taken, so we emit an unconditional
+      // branch.
+      // This only applies to unsigned pseudo-branches.
+      BranchInst.setOpcode(Mips::BEQ);
+      BranchInst.addOperand(MCOperand::createReg(Mips::ZERO));
+      BranchInst.addOperand(MCOperand::createReg(Mips::ZERO));
+      BranchInst.addOperand(MCOperand::createExpr(OffsetExpr));
+      Instructions.push_back(BranchInst);
+      Warning(IDLoc, "branch is always taken");
+      return false;
+    }
+    if (IsUnsigned) {
+      // If the $rs is $0 and the pseudo-branch is BLTU (0 < x) or
+      // if the $rt is $0 and the pseudo-branch is BGTU (x > 0),
+      // the pseudo-branch will be taken only when the non-zero register is
+      // different from 0, so we emit a BNEZ.
+      //
+      // If the $rs is $0 and the pseudo-branch is BGEU (0 >= x) or
+      // if the $rt is $0 and the pseudo-branch is BLEU (x <= 0),
+      // the pseudo-branch will be taken only when the non-zero register is
+      // equal to 0, so we emit a BEQZ.
+      //
+      // Because only BLEU and BGEU branch on equality, we can use the
+      // AcceptsEquality variable to decide when to emit the BEQZ.
+      BranchInst.setOpcode(AcceptsEquality ? Mips::BEQ : Mips::BNE);
+      BranchInst.addOperand(
+          MCOperand::createReg(IsSrcRegZero ? TrgReg : SrcReg));
+      BranchInst.addOperand(MCOperand::createReg(Mips::ZERO));
+      BranchInst.addOperand(MCOperand::createExpr(OffsetExpr));
+      Instructions.push_back(BranchInst);
+      return false;
+    }
+    // If we have a signed pseudo-branch and one of the registers is $0,
+    // we can use an appropriate compare-to-zero branch. We select which one
+    // to use in the switch statement above.
+    BranchInst.setOpcode(IsSrcRegZero ? ZeroSrcOpcode : ZeroTrgOpcode);
+    BranchInst.addOperand(MCOperand::createReg(IsSrcRegZero ? TrgReg : SrcReg));
+    BranchInst.addOperand(MCOperand::createExpr(OffsetExpr));
+    Instructions.push_back(BranchInst);
+    return false;
+  }
+
+  // If neither the SrcReg nor the TrgReg are $0, we need AT to perform the
+  // expansions. If it is not available, we return.
+  unsigned ATRegNum = getATReg(IDLoc);
+  if (!ATRegNum)
+    return true;
+
+  warnIfNoMacro(IDLoc);
+
+  // SLT fits well with 2 of our 4 pseudo-branches:
+  //   BLT, where $rs < $rt, translates into "slt $at, $rs, $rt" and
+  //   BGT, where $rs > $rt, translates into "slt $at, $rt, $rs".
+  // If the result of the SLT is 1, we branch, and if it's 0, we don't.
+  // This is accomplished by using a BNEZ with the result of the SLT.
+  //
+  // The other 2 pseudo-branches are opposites of the above 2 (BGE with BLT
+  // and BLE with BGT), so we change the BNEZ into a a BEQZ.
+  // Because only BGE and BLE branch on equality, we can use the
+  // AcceptsEquality variable to decide when to emit the BEQZ.
+  // Note that the order of the SLT arguments doesn't change between
+  // opposites.
+  //
+  // The same applies to the unsigned variants, except that SLTu is used
+  // instead of SLT.
+  MCInst SetInst;
+  SetInst.setOpcode(IsUnsigned ? Mips::SLTu : Mips::SLT);
+  SetInst.addOperand(MCOperand::createReg(ATRegNum));
+  SetInst.addOperand(MCOperand::createReg(ReverseOrderSLT ? TrgReg : SrcReg));
+  SetInst.addOperand(MCOperand::createReg(ReverseOrderSLT ? SrcReg : TrgReg));
+  Instructions.push_back(SetInst);
+
+  BranchInst.setOpcode(AcceptsEquality ? Mips::BEQ : Mips::BNE);
+  BranchInst.addOperand(MCOperand::createReg(ATRegNum));
+  BranchInst.addOperand(MCOperand::createReg(Mips::ZERO));
+  BranchInst.addOperand(MCOperand::createExpr(OffsetExpr));
+  Instructions.push_back(BranchInst);
+  return false;
+}
+
+bool MipsAsmParser::expandUlhu(MCInst &Inst, SMLoc IDLoc,
+                               SmallVectorImpl<MCInst> &Instructions) {
+  if (hasMips32r6() || hasMips64r6()) {
+    Error(IDLoc, "instruction not supported on mips32r6 or mips64r6");
+    return false;
+  }
+
+  warnIfNoMacro(IDLoc);
+
+  const MCOperand &DstRegOp = Inst.getOperand(0);
+  assert(DstRegOp.isReg() && "expected register operand kind");
+
+  const MCOperand &SrcRegOp = Inst.getOperand(1);
+  assert(SrcRegOp.isReg() && "expected register operand kind");
+
+  const MCOperand &OffsetImmOp = Inst.getOperand(2);
+  assert(OffsetImmOp.isImm() && "expected immediate operand kind");
+
+  unsigned DstReg = DstRegOp.getReg();
+  unsigned SrcReg = SrcRegOp.getReg();
+  int64_t OffsetValue = OffsetImmOp.getImm();
+
+  // NOTE: We always need AT for ULHU, as it is always used as the source
+  // register for one of the LBu's.
+  unsigned ATReg = getATReg(IDLoc);
+  if (!ATReg)
+    return true;
+
+  // When the value of offset+1 does not fit in 16 bits, we have to load the
+  // offset in AT, (D)ADDu the original source register (if there was one), and
+  // then use AT as the source register for the 2 generated LBu's.
+  bool LoadedOffsetInAT = false;
+  if (!isInt<16>(OffsetValue + 1) || !isInt<16>(OffsetValue)) {
+    LoadedOffsetInAT = true;
+
+    if (loadImmediate(OffsetValue, ATReg, Mips::NoRegister, !ABI.ArePtrs64bit(),
+                      IDLoc, Instructions))
+      return true;
+
+    // NOTE: We do this (D)ADDu here instead of doing it in loadImmediate()
+    // because it will make our output more similar to GAS'. For example,
+    // generating an "ori $1, $zero, 32768" followed by an "addu $1, $1, $9",
+    // instead of just an "ori $1, $9, 32768".
+    // NOTE: If there is no source register specified in the ULHU, the parser
+    // will interpret it as $0.
+    if (SrcReg != Mips::ZERO && SrcReg != Mips::ZERO_64)
+      createAddu(ATReg, ATReg, SrcReg, ABI.ArePtrs64bit(), Instructions);
+  }
+
+  unsigned FirstLbuDstReg = LoadedOffsetInAT ? DstReg : ATReg;
+  unsigned SecondLbuDstReg = LoadedOffsetInAT ? ATReg : DstReg;
+  unsigned LbuSrcReg = LoadedOffsetInAT ? ATReg : SrcReg;
+
+  int64_t FirstLbuOffset = 0, SecondLbuOffset = 0;
+  if (isLittle()) {
+    FirstLbuOffset = LoadedOffsetInAT ? 1 : (OffsetValue + 1);
+    SecondLbuOffset = LoadedOffsetInAT ? 0 : OffsetValue;
+  } else {
+    FirstLbuOffset = LoadedOffsetInAT ? 0 : OffsetValue;
+    SecondLbuOffset = LoadedOffsetInAT ? 1 : (OffsetValue + 1);
+  }
+
+  unsigned SllReg = LoadedOffsetInAT ? DstReg : ATReg;
+
+  MCInst TmpInst;
+  TmpInst.setOpcode(Mips::LBu);
+  TmpInst.addOperand(MCOperand::createReg(FirstLbuDstReg));
+  TmpInst.addOperand(MCOperand::createReg(LbuSrcReg));
+  TmpInst.addOperand(MCOperand::createImm(FirstLbuOffset));
+  Instructions.push_back(TmpInst);
+
+  TmpInst.clear();
+  TmpInst.setOpcode(Mips::LBu);
+  TmpInst.addOperand(MCOperand::createReg(SecondLbuDstReg));
+  TmpInst.addOperand(MCOperand::createReg(LbuSrcReg));
+  TmpInst.addOperand(MCOperand::createImm(SecondLbuOffset));
+  Instructions.push_back(TmpInst);
+
+  TmpInst.clear();
+  TmpInst.setOpcode(Mips::SLL);
+  TmpInst.addOperand(MCOperand::createReg(SllReg));
+  TmpInst.addOperand(MCOperand::createReg(SllReg));
+  TmpInst.addOperand(MCOperand::createImm(8));
+  Instructions.push_back(TmpInst);
+
+  TmpInst.clear();
+  TmpInst.setOpcode(Mips::OR);
+  TmpInst.addOperand(MCOperand::createReg(DstReg));
+  TmpInst.addOperand(MCOperand::createReg(DstReg));
+  TmpInst.addOperand(MCOperand::createReg(ATReg));
+  Instructions.push_back(TmpInst);
+
+  return false;
+}
+
+bool MipsAsmParser::expandUlw(MCInst &Inst, SMLoc IDLoc,
+                              SmallVectorImpl<MCInst> &Instructions) {
+  if (hasMips32r6() || hasMips64r6()) {
+    Error(IDLoc, "instruction not supported on mips32r6 or mips64r6");
+    return false;
+  }
+
+  const MCOperand &DstRegOp = Inst.getOperand(0);
+  assert(DstRegOp.isReg() && "expected register operand kind");
+
+  const MCOperand &SrcRegOp = Inst.getOperand(1);
+  assert(SrcRegOp.isReg() && "expected register operand kind");
+
+  const MCOperand &OffsetImmOp = Inst.getOperand(2);
+  assert(OffsetImmOp.isImm() && "expected immediate operand kind");
+
+  unsigned SrcReg = SrcRegOp.getReg();
+  int64_t OffsetValue = OffsetImmOp.getImm();
+  unsigned ATReg = 0;
+
+  // When the value of offset+3 does not fit in 16 bits, we have to load the
+  // offset in AT, (D)ADDu the original source register (if there was one), and
+  // then use AT as the source register for the generated LWL and LWR.
+  bool LoadedOffsetInAT = false;
+  if (!isInt<16>(OffsetValue + 3) || !isInt<16>(OffsetValue)) {
+    ATReg = getATReg(IDLoc);
+    if (!ATReg)
+      return true;
+    LoadedOffsetInAT = true;
+
+    warnIfNoMacro(IDLoc);
+
+    if (loadImmediate(OffsetValue, ATReg, Mips::NoRegister, !ABI.ArePtrs64bit(),
+                      IDLoc, Instructions))
+      return true;
+
+    // NOTE: We do this (D)ADDu here instead of doing it in loadImmediate()
+    // because it will make our output more similar to GAS'. For example,
+    // generating an "ori $1, $zero, 32768" followed by an "addu $1, $1, $9",
+    // instead of just an "ori $1, $9, 32768".
+    // NOTE: If there is no source register specified in the ULW, the parser
+    // will interpret it as $0.
+    if (SrcReg != Mips::ZERO && SrcReg != Mips::ZERO_64)
+      createAddu(ATReg, ATReg, SrcReg, ABI.ArePtrs64bit(), Instructions);
+  }
+
+  unsigned FinalSrcReg = LoadedOffsetInAT ? ATReg : SrcReg;
+  int64_t LeftLoadOffset = 0, RightLoadOffset  = 0;
+  if (isLittle()) {
+    LeftLoadOffset = LoadedOffsetInAT ? 3 : (OffsetValue + 3);
+    RightLoadOffset  = LoadedOffsetInAT ? 0 : OffsetValue;
+  } else {
+    LeftLoadOffset = LoadedOffsetInAT ? 0 : OffsetValue;
+    RightLoadOffset  = LoadedOffsetInAT ? 3 : (OffsetValue + 3);
+  }
+
+  MCInst LeftLoadInst;
+  LeftLoadInst.setOpcode(Mips::LWL);
+  LeftLoadInst.addOperand(DstRegOp);
+  LeftLoadInst.addOperand(MCOperand::createReg(FinalSrcReg));
+  LeftLoadInst.addOperand(MCOperand::createImm(LeftLoadOffset));
+  Instructions.push_back(LeftLoadInst);
+
+  MCInst RightLoadInst;
+  RightLoadInst.setOpcode(Mips::LWR);
+  RightLoadInst.addOperand(DstRegOp);
+  RightLoadInst.addOperand(MCOperand::createReg(FinalSrcReg));
+  RightLoadInst.addOperand(MCOperand::createImm(RightLoadOffset ));
+  Instructions.push_back(RightLoadInst);
+
+  return false;
+}
+
 void MipsAsmParser::createNop(bool hasShortDelaySlot, SMLoc IDLoc,
                               SmallVectorImpl<MCInst> &Instructions) {
   MCInst NopInst;
   if (hasShortDelaySlot) {
     NopInst.setOpcode(Mips::MOVE16_MM);
-    NopInst.addOperand(MCOperand::CreateReg(Mips::ZERO));
-    NopInst.addOperand(MCOperand::CreateReg(Mips::ZERO));
+    NopInst.addOperand(MCOperand::createReg(Mips::ZERO));
+    NopInst.addOperand(MCOperand::createReg(Mips::ZERO));
   } else {
     NopInst.setOpcode(Mips::SLL);
-    NopInst.addOperand(MCOperand::CreateReg(Mips::ZERO));
-    NopInst.addOperand(MCOperand::CreateReg(Mips::ZERO));
-    NopInst.addOperand(MCOperand::CreateImm(0));
+    NopInst.addOperand(MCOperand::createReg(Mips::ZERO));
+    NopInst.addOperand(MCOperand::createReg(Mips::ZERO));
+    NopInst.addOperand(MCOperand::createImm(0));
   }
   Instructions.push_back(NopInst);
 }
 
+void MipsAsmParser::createAddu(unsigned DstReg, unsigned SrcReg,
+                               unsigned TrgReg, bool Is64Bit,
+                               SmallVectorImpl<MCInst> &Instructions) {
+  emitRRR(Is64Bit ? Mips::DADDu : Mips::ADDu, DstReg, SrcReg, TrgReg, SMLoc(),
+          Instructions);
+}
+
 unsigned MipsAsmParser::checkTargetMatchPredicate(MCInst &Inst) {
   // As described by the Mips32r2 spec, the registers Rd and Rs for
   // jalr.hb must be different.
@@ -2203,15 +2773,15 @@ bool MipsAsmParser::MatchAndEmitInstruction(SMLoc IDLoc, unsigned &Opcode,
   llvm_unreachable("Implement any new match types added!");
 }
 
-void MipsAsmParser::warnIfAssemblerTemporary(int RegIndex, SMLoc Loc) {
-  if ((RegIndex != 0) && 
-      ((int)AssemblerOptions.back()->getATRegNum() == RegIndex)) {
-    if (RegIndex == 1)
-      Warning(Loc, "used $at without \".set noat\"");
-    else
-      Warning(Loc, Twine("used $") + Twine(RegIndex) + " with \".set at=$" +
-                       Twine(RegIndex) + "\"");
-  }
+void MipsAsmParser::warnIfRegIndexIsAT(unsigned RegIndex, SMLoc Loc) {
+  if (RegIndex != 0 && AssemblerOptions.back()->getATRegIndex() == RegIndex)
+    Warning(Loc, "used $at (currently $" + Twine(RegIndex) +
+                     ") without \".set noat\"");
+}
+
+void MipsAsmParser::warnIfNoMacro(SMLoc Loc) {
+  if (!AssemblerOptions.back()->isMacro())
+    Warning(Loc, "macro instruction expanded into multiple instructions");
 }
 
 void
@@ -2386,7 +2956,7 @@ int MipsAsmParser::matchMSA128CtrlRegisterName(StringRef Name) {
 }
 
 unsigned MipsAsmParser::getATReg(SMLoc Loc) {
-  unsigned ATIndex = AssemblerOptions.back()->getATRegNum();
+  unsigned ATIndex = AssemblerOptions.back()->getATRegIndex();
   if (ATIndex == 0) {
     reportParseError(Loc,
                      "pseudo-instruction requires $at, which is not available");
@@ -2453,10 +3023,10 @@ bool MipsAsmParser::parseOperand(OperandVector &Operands, StringRef Mnemonic) {
       return true;
 
     SMLoc E = SMLoc::getFromPointer(Parser.getTok().getLoc().getPointer() - 1);
-    MCSymbol *Sym = getContext().GetOrCreateSymbol("$" + Identifier);
+    MCSymbol *Sym = getContext().getOrCreateSymbol("$" + Identifier);
     // Otherwise create a symbol reference.
     const MCExpr *Res =
-        MCSymbolRefExpr::Create(Sym, MCSymbolRefExpr::VK_None, getContext());
+        MCSymbolRefExpr::create(Sym, MCSymbolRefExpr::VK_None, getContext());
 
     Operands.push_back(MipsOperand::CreateImm(Res, S, E, *this));
     return false;
@@ -2516,14 +3086,14 @@ const MCExpr *MipsAsmParser::evaluateRelocExpr(const MCExpr *Expr,
     default:
       report_fatal_error("unsupported reloc value");
     }
-    return MCConstantExpr::Create(Val, getContext());
+    return MCConstantExpr::create(Val, getContext());
   }
 
   if (const MCSymbolRefExpr *MSRE = dyn_cast<MCSymbolRefExpr>(Expr)) {
     // It's a symbol, create a symbolic expression from the symbol.
-    StringRef Symbol = MSRE->getSymbol().getName();
+    const MCSymbol *Symbol = &MSRE->getSymbol();
     MCSymbolRefExpr::VariantKind VK = getVariantKind(RelocStr);
-    Res = MCSymbolRefExpr::Create(Symbol, VK, getContext());
+    Res = MCSymbolRefExpr::create(Symbol, VK, getContext());
     return Res;
   }
 
@@ -2532,17 +3102,17 @@ const MCExpr *MipsAsmParser::evaluateRelocExpr(const MCExpr *Expr,
 
     // Try to create target expression.
     if (MipsMCExpr::isSupportedBinaryExpr(VK, BE))
-      return MipsMCExpr::Create(VK, Expr, getContext());
+      return MipsMCExpr::create(VK, Expr, getContext());
 
     const MCExpr *LExp = evaluateRelocExpr(BE->getLHS(), RelocStr);
     const MCExpr *RExp = evaluateRelocExpr(BE->getRHS(), RelocStr);
-    Res = MCBinaryExpr::Create(BE->getOpcode(), LExp, RExp, getContext());
+    Res = MCBinaryExpr::create(BE->getOpcode(), LExp, RExp, getContext());
     return Res;
   }
 
   if (const MCUnaryExpr *UN = dyn_cast<MCUnaryExpr>(Expr)) {
     const MCExpr *UnExp = evaluateRelocExpr(UN->getSubExpr(), RelocStr);
-    Res = MCUnaryExpr::Create(UN->getOpcode(), UnExp, getContext());
+    Res = MCUnaryExpr::create(UN->getOpcode(), UnExp, getContext());
     return Res;
   }
   // Just return the original expression.
@@ -2643,9 +3213,12 @@ bool MipsAsmParser::parseMemOffset(const MCExpr *&Res, bool isParenExpr) {
   MCAsmParser &Parser = getParser();
   SMLoc S;
   bool Result = true;
+  unsigned NumOfLParen = 0;
 
-  while (getLexer().getKind() == AsmToken::LParen)
+  while (getLexer().getKind() == AsmToken::LParen) {
     Parser.Lex();
+    ++NumOfLParen;
+  }
 
   switch (getLexer().getKind()) {
   default:
@@ -2656,7 +3229,7 @@ bool MipsAsmParser::parseMemOffset(const MCExpr *&Res, bool isParenExpr) {
   case AsmToken::Minus:
   case AsmToken::Plus:
     if (isParenExpr)
-      Result = getParser().parseParenExpression(Res, S);
+      Result = getParser().parseParenExprOfDepth(NumOfLParen, Res, S);
     else
       Result = (getParser().parseExpression(Res));
     while (getLexer().getKind() == AsmToken::RParen)
@@ -2730,7 +3303,7 @@ MipsAsmParser::parseMemOperand(OperandVector &Operands) {
   Parser.Lex(); // Eat the ')' token.
 
   if (!IdVal)
-    IdVal = MCConstantExpr::Create(0, getContext());
+    IdVal = MCConstantExpr::create(0, getContext());
 
   // Replace the register operand with the memory operand.
   std::unique_ptr<MipsOperand> op(
@@ -2741,10 +3314,10 @@ MipsAsmParser::parseMemOperand(OperandVector &Operands) {
   // Add the memory operand.
   if (const MCBinaryExpr *BE = dyn_cast<MCBinaryExpr>(IdVal)) {
     int64_t Imm;
-    if (IdVal->EvaluateAsAbsolute(Imm))
-      IdVal = MCConstantExpr::Create(Imm, getContext());
+    if (IdVal->evaluateAsAbsolute(Imm))
+      IdVal = MCConstantExpr::create(Imm, getContext());
     else if (BE->getLHS()->getKind() != MCExpr::SymbolRef)
-      IdVal = MCBinaryExpr::Create(BE->getOpcode(), BE->getRHS(), BE->getLHS(),
+      IdVal = MCBinaryExpr::create(BE->getOpcode(), BE->getRHS(), BE->getLHS(),
                                    getContext());
   }
 
@@ -2754,7 +3327,7 @@ MipsAsmParser::parseMemOperand(OperandVector &Operands) {
 
 bool MipsAsmParser::searchSymbolAlias(OperandVector &Operands) {
   MCAsmParser &Parser = getParser();
-  MCSymbol *Sym = getContext().LookupSymbol(Parser.getTok().getIdentifier());
+  MCSymbol *Sym = getContext().lookupSymbol(Parser.getTok().getIdentifier());
   if (Sym) {
     SMLoc S = Parser.getTok().getLoc();
     const MCExpr *Expr;
@@ -2961,7 +3534,7 @@ MipsAsmParser::parseInvNum(OperandVector &Operands) {
   int64_t Val = MCE->getValue();
   SMLoc E = SMLoc::getFromPointer(Parser.getTok().getLoc().getPointer() - 1);
   Operands.push_back(MipsOperand::CreateImm(
-      MCConstantExpr::Create(0 - Val, getContext()), S, E, *this));
+      MCConstantExpr::create(0 - Val, getContext()), S, E, *this));
   return MatchOperand_Success;
 }
 
@@ -2985,7 +3558,7 @@ MipsAsmParser::parseLSAImm(OperandVector &Operands) {
     return MatchOperand_ParseFail;
 
   int64_t Val;
-  if (!Expr->EvaluateAsAbsolute(Val)) {
+  if (!Expr->evaluateAsAbsolute(Val)) {
     Error(S, "expected immediate value");
     return MatchOperand_ParseFail;
   }
@@ -3300,7 +3873,7 @@ bool MipsAsmParser::parseSetNoAtDirective() {
   // Line should look like: ".set noat".
 
   // Set the $at register to $0.
-  AssemblerOptions.back()->setATReg(0);
+  AssemblerOptions.back()->setATRegIndex(0);
 
   Parser.Lex(); // Eat "noat".
 
@@ -3323,7 +3896,7 @@ bool MipsAsmParser::parseSetAtDirective() {
 
   if (getLexer().is(AsmToken::EndOfStatement)) {
     // No register was specified, so we set $at to $1.
-    AssemblerOptions.back()->setATReg(1);
+    AssemblerOptions.back()->setATRegIndex(1);
 
     getTargetStreamer().emitDirectiveSetAt();
     Parser.Lex(); // Consume the EndOfStatement.
@@ -3360,7 +3933,7 @@ bool MipsAsmParser::parseSetAtDirective() {
   }
 
   // Check if $reg is a valid register. If it is, set $at to $reg.
-  if (!AssemblerOptions.back()->setATReg(AtRegNo)) {
+  if (!AssemblerOptions.back()->setATRegIndex(AtRegNo)) {
     reportParseError("invalid register");
     return false;
   }
@@ -3415,6 +3988,7 @@ bool MipsAsmParser::parseSetMacroDirective() {
     return false;
   }
   AssemblerOptions.back()->setMacro();
+  getTargetStreamer().emitDirectiveSetMacro();
   Parser.Lex(); // Consume the EndOfStatement.
   return false;
 }
@@ -3432,6 +4006,7 @@ bool MipsAsmParser::parseSetNoMacroDirective() {
     return false;
   }
   AssemblerOptions.back()->setNoMacro();
+  getTargetStreamer().emitDirectiveSetNoMacro();
   Parser.Lex(); // Consume the EndOfStatement.
   return false;
 }
@@ -3536,6 +4111,34 @@ bool MipsAsmParser::parseSetFpDirective() {
   return false;
 }
 
+bool MipsAsmParser::parseSetOddSPRegDirective() {
+  MCAsmParser &Parser = getParser();
+
+  Parser.Lex(); // Eat "oddspreg".
+  if (getLexer().isNot(AsmToken::EndOfStatement)) {
+    reportParseError("unexpected token, expected end of statement");
+    return false;
+  }
+
+  clearFeatureBits(Mips::FeatureNoOddSPReg, "nooddspreg");
+  getTargetStreamer().emitDirectiveSetOddSPReg();
+  return false;
+}
+
+bool MipsAsmParser::parseSetNoOddSPRegDirective() {
+  MCAsmParser &Parser = getParser();
+
+  Parser.Lex(); // Eat "nooddspreg".
+  if (getLexer().isNot(AsmToken::EndOfStatement)) {
+    reportParseError("unexpected token, expected end of statement");
+    return false;
+  }
+
+  setFeatureBits(Mips::FeatureNoOddSPReg, "nooddspreg");
+  getTargetStreamer().emitDirectiveSetNoOddSPReg();
+  return false;
+}
+
 bool MipsAsmParser::parseSetPopDirective() {
   MCAsmParser &Parser = getParser();
   SMLoc Loc = getLexer().getLoc();
@@ -3550,7 +4153,9 @@ bool MipsAsmParser::parseSetPopDirective() {
     return reportParseError(Loc, ".set pop with no .set push");
 
   AssemblerOptions.pop_back();
-  setAvailableFeatures(AssemblerOptions.back()->getFeatures());
+  setAvailableFeatures(
+      ComputeAvailableFeatures(AssemblerOptions.back()->getFeatures()));
+  STI.setFeatureBits(AssemblerOptions.back()->getFeatures());
 
   getTargetStreamer().emitDirectiveSetPop();
   return false;
@@ -3570,6 +4175,28 @@ bool MipsAsmParser::parseSetPushDirective() {
   return false;
 }
 
+bool MipsAsmParser::parseSetSoftFloatDirective() {
+  MCAsmParser &Parser = getParser();
+  Parser.Lex();
+  if (getLexer().isNot(AsmToken::EndOfStatement))
+    return reportParseError("unexpected token, expected end of statement");
+
+  setFeatureBits(Mips::FeatureSoftFloat, "soft-float");
+  getTargetStreamer().emitDirectiveSetSoftFloat();
+  return false;
+}
+
+bool MipsAsmParser::parseSetHardFloatDirective() {
+  MCAsmParser &Parser = getParser();
+  Parser.Lex();
+  if (getLexer().isNot(AsmToken::EndOfStatement))
+    return reportParseError("unexpected token, expected end of statement");
+
+  clearFeatureBits(Mips::FeatureSoftFloat, "soft-float");
+  getTargetStreamer().emitDirectiveSetHardFloat();
+  return false;
+}
+
 bool MipsAsmParser::parseSetAssignment() {
   StringRef Name;
   const MCExpr *Value;
@@ -3585,7 +4212,7 @@ bool MipsAsmParser::parseSetAssignment() {
   if (Parser.parseExpression(Value))
     return reportParseError("expected valid expression after comma");
 
-  MCSymbol *Sym = getContext().GetOrCreateSymbol(Name);
+  MCSymbol *Sym = getContext().getOrCreateSymbol(Name);
   Sym->setVariableValue(Value);
 
   return false;
@@ -3598,7 +4225,9 @@ bool MipsAsmParser::parseSetMips0Directive() {
     return reportParseError("unexpected token, expected end of statement");
 
   // Reset assembler options to their initial values.
-  setAvailableFeatures(AssemblerOptions.front()->getFeatures());
+  setAvailableFeatures(
+      ComputeAvailableFeatures(AssemblerOptions.front()->getFeatures()));
+  STI.setFeatureBits(AssemblerOptions.front()->getFeatures());
   AssemblerOptions.back()->setFeatures(AssemblerOptions.front()->getFeatures());
 
   getTargetStreamer().emitDirectiveSetMips0();
@@ -3872,6 +4501,10 @@ bool MipsAsmParser::parseDirectiveSet() {
     return parseSetArchDirective();
   } else if (Tok.getString() == "fp") {
     return parseSetFpDirective();
+  } else if (Tok.getString() == "oddspreg") {
+    return parseSetOddSPRegDirective();
+  } else if (Tok.getString() == "nooddspreg") {
+    return parseSetNoOddSPRegDirective();
   } else if (Tok.getString() == "pop") {
     return parseSetPopDirective();
   } else if (Tok.getString() == "push") {
@@ -3934,6 +4567,10 @@ bool MipsAsmParser::parseDirectiveSet() {
     return parseSetMsaDirective();
   } else if (Tok.getString() == "nomsa") {
     return parseSetNoMsaDirective();
+  } else if (Tok.getString() == "softfloat") {
+    return parseSetSoftFloatDirective();
+  } else if (Tok.getString() == "hardfloat") {
+    return parseSetHardFloatDirective();
   } else {
     // It is just an identifier, look for an assignment.
     parseSetAssignment();
@@ -4067,6 +4704,8 @@ bool MipsAsmParser::parseInsnDirective() {
 ///  ::= .module oddspreg
 ///  ::= .module nooddspreg
 ///  ::= .module fp=value
+///  ::= .module softfloat
+///  ::= .module hardfloat
 bool MipsAsmParser::parseDirectiveModule() {
   MCAsmParser &Parser = getParser();
   MCAsmLexer &Lexer = getLexer();
@@ -4085,8 +4724,16 @@ bool MipsAsmParser::parseDirectiveModule() {
   }
 
   if (Option == "oddspreg") {
-    getTargetStreamer().emitDirectiveModuleOddSPReg(true, isABI_O32());
-    clearFeatureBits(Mips::FeatureNoOddSPReg, "nooddspreg");
+    clearModuleFeatureBits(Mips::FeatureNoOddSPReg, "nooddspreg");
+
+    // Synchronize the abiflags information with the FeatureBits information we
+    // changed above.
+    getTargetStreamer().updateABIInfo(*this);
+
+    // If printing assembly, use the recently updated abiflags information.
+    // If generating ELF, don't do anything (the .MIPS.abiflags section gets
+    // emitted at the end).
+    getTargetStreamer().emitDirectiveModuleOddSPReg();
 
     // If this is not the end of the statement, report an error.
     if (getLexer().isNot(AsmToken::EndOfStatement)) {
@@ -4101,8 +4748,16 @@ bool MipsAsmParser::parseDirectiveModule() {
       return false;
     }
 
-    getTargetStreamer().emitDirectiveModuleOddSPReg(false, isABI_O32());
-    setFeatureBits(Mips::FeatureNoOddSPReg, "nooddspreg");
+    setModuleFeatureBits(Mips::FeatureNoOddSPReg, "nooddspreg");
+
+    // Synchronize the abiflags information with the FeatureBits information we
+    // changed above.
+    getTargetStreamer().updateABIInfo(*this);
+
+    // If printing assembly, use the recently updated abiflags information.
+    // If generating ELF, don't do anything (the .MIPS.abiflags section gets
+    // emitted at the end).
+    getTargetStreamer().emitDirectiveModuleOddSPReg();
 
     // If this is not the end of the statement, report an error.
     if (getLexer().isNot(AsmToken::EndOfStatement)) {
@@ -4113,6 +4768,44 @@ bool MipsAsmParser::parseDirectiveModule() {
     return false; // parseDirectiveModule has finished successfully.
   } else if (Option == "fp") {
     return parseDirectiveModuleFP();
+  } else if (Option == "softfloat") {
+    setModuleFeatureBits(Mips::FeatureSoftFloat, "soft-float");
+
+    // Synchronize the ABI Flags information with the FeatureBits information we
+    // updated above.
+    getTargetStreamer().updateABIInfo(*this);
+
+    // If printing assembly, use the recently updated ABI Flags information.
+    // If generating ELF, don't do anything (the .MIPS.abiflags section gets
+    // emitted later).
+    getTargetStreamer().emitDirectiveModuleSoftFloat();
+
+    // If this is not the end of the statement, report an error.
+    if (getLexer().isNot(AsmToken::EndOfStatement)) {
+      reportParseError("unexpected token, expected end of statement");
+      return false;
+    }
+
+    return false; // parseDirectiveModule has finished successfully.
+  } else if (Option == "hardfloat") {
+    clearModuleFeatureBits(Mips::FeatureSoftFloat, "soft-float");
+
+    // Synchronize the ABI Flags information with the FeatureBits information we
+    // updated above.
+    getTargetStreamer().updateABIInfo(*this);
+
+    // If printing assembly, use the recently updated ABI Flags information.
+    // If generating ELF, don't do anything (the .MIPS.abiflags section gets
+    // emitted later).
+    getTargetStreamer().emitDirectiveModuleHardFloat();
+
+    // If this is not the end of the statement, report an error.
+    if (getLexer().isNot(AsmToken::EndOfStatement)) {
+      reportParseError("unexpected token, expected end of statement");
+      return false;
+    }
+
+    return false; // parseDirectiveModule has finished successfully.
   } else {
     return Error(L, "'" + Twine(Option) + "' is not a valid .module option.");
   }
@@ -4141,8 +4834,15 @@ bool MipsAsmParser::parseDirectiveModuleFP() {
     return false;
   }
 
-  // Emit appropriate flags.
-  getTargetStreamer().emitDirectiveModuleFP(FpABI, isABI_O32());
+  // Synchronize the abiflags information with the FeatureBits information we
+  // changed above.
+  getTargetStreamer().updateABIInfo(*this);
+
+  // If printing assembly, use the recently updated abiflags information.
+  // If generating ELF, don't do anything (the .MIPS.abiflags section gets
+  // emitted at the end).
+  getTargetStreamer().emitDirectiveModuleFP();
+
   Parser.Lex(); // Consume the EndOfStatement.
   return false;
 }
@@ -4151,6 +4851,7 @@ bool MipsAsmParser::parseFpABIValue(MipsABIFlagsSection::FpABIKind &FpABI,
                                     StringRef Directive) {
   MCAsmParser &Parser = getParser();
   MCAsmLexer &Lexer = getLexer();
+  bool ModuleLevelOptions = Directive == ".module";
 
   if (Lexer.is(AsmToken::Identifier)) {
     StringRef Value = Parser.getTok().getString();
@@ -4167,6 +4868,13 @@ bool MipsAsmParser::parseFpABIValue(MipsABIFlagsSection::FpABIKind &FpABI,
     }
 
     FpABI = MipsABIFlagsSection::FpABIKind::XX;
+    if (ModuleLevelOptions) {
+      setModuleFeatureBits(Mips::FeatureFPXX, "fpxx");
+      clearModuleFeatureBits(Mips::FeatureFP64Bit, "fp64");
+    } else {
+      setFeatureBits(Mips::FeatureFPXX, "fpxx");
+      clearFeatureBits(Mips::FeatureFP64Bit, "fp64");
+    }
     return true;
   }
 
@@ -4186,8 +4894,23 @@ bool MipsAsmParser::parseFpABIValue(MipsABIFlagsSection::FpABIKind &FpABI,
       }
 
       FpABI = MipsABIFlagsSection::FpABIKind::S32;
-    } else
+      if (ModuleLevelOptions) {
+        clearModuleFeatureBits(Mips::FeatureFPXX, "fpxx");
+        clearModuleFeatureBits(Mips::FeatureFP64Bit, "fp64");
+      } else {
+        clearFeatureBits(Mips::FeatureFPXX, "fpxx");
+        clearFeatureBits(Mips::FeatureFP64Bit, "fp64");
+      }
+    } else {
       FpABI = MipsABIFlagsSection::FpABIKind::S64;
+      if (ModuleLevelOptions) {
+        clearModuleFeatureBits(Mips::FeatureFPXX, "fpxx");
+        setModuleFeatureBits(Mips::FeatureFP64Bit, "fp64");
+      } else {
+        clearFeatureBits(Mips::FeatureFPXX, "fpxx");
+        setFeatureBits(Mips::FeatureFP64Bit, "fp64");
+      }
+    }
 
     return true;
   }
@@ -4235,7 +4958,7 @@ bool MipsAsmParser::ParseDirective(AsmToken DirectiveID) {
         reportParseError("expected number after comma");
         return false;
       }
-      if (!DummyNumber->EvaluateAsAbsolute(DummyNumberVal)) {
+      if (!DummyNumber->evaluateAsAbsolute(DummyNumberVal)) {
         reportParseError("expected an absolute expression after comma");
         return false;
       }
@@ -4247,7 +4970,7 @@ bool MipsAsmParser::ParseDirective(AsmToken DirectiveID) {
       return false;
     }
 
-    MCSymbol *Sym = getContext().GetOrCreateSymbol(SymbolName);
+    MCSymbol *Sym = getContext().getOrCreateSymbol(SymbolName);
 
     getTargetStreamer().emitDirectiveEnt(*Sym);
     CurrentFn = Sym;
@@ -4315,7 +5038,7 @@ bool MipsAsmParser::ParseDirective(AsmToken DirectiveID) {
       return false;
     }
 
-    if (!FrameSize->EvaluateAsAbsolute(FrameSizeVal)) {
+    if (!FrameSize->evaluateAsAbsolute(FrameSizeVal)) {
       reportParseError("frame size not an absolute expression");
       return false;
     }
@@ -4376,7 +5099,7 @@ bool MipsAsmParser::ParseDirective(AsmToken DirectiveID) {
       return false;
     }
 
-    if (!BitMask->EvaluateAsAbsolute(BitMaskVal)) {
+    if (!BitMask->evaluateAsAbsolute(BitMaskVal)) {
       reportParseError("bitmask not an absolute expression");
       return false;
     }
@@ -4397,7 +5120,7 @@ bool MipsAsmParser::ParseDirective(AsmToken DirectiveID) {
       return false;
     }
 
-    if (!FrameOffset->EvaluateAsAbsolute(FrameOffsetVal)) {
+    if (!FrameOffset->evaluateAsAbsolute(FrameOffsetVal)) {
       reportParseError("frame offset not an absolute expression");
       return false;
     }
diff --git a/lib/Target/Mips/CMakeLists.txt b/lib/Target/Mips/CMakeLists.txt
index 36ba8e559e0..bde843afd3d 100644
--- a/lib/Target/Mips/CMakeLists.txt
+++ b/lib/Target/Mips/CMakeLists.txt
@@ -46,7 +46,6 @@ add_llvm_target(MipsCodeGen
   MipsSubtarget.cpp
   MipsTargetMachine.cpp
   MipsTargetObjectFile.cpp
-  MipsSelectionDAGInfo.cpp
   )
 
 add_subdirectory(InstPrinter)
diff --git a/lib/Target/Mips/Disassembler/MipsDisassembler.cpp b/lib/Target/Mips/Disassembler/MipsDisassembler.cpp
index 8849366e837..a34ba3bd0ee 100644
--- a/lib/Target/Mips/Disassembler/MipsDisassembler.cpp
+++ b/lib/Target/Mips/Disassembler/MipsDisassembler.cpp
@@ -36,16 +36,18 @@ class MipsDisassembler : public MCDisassembler {
 public:
   MipsDisassembler(const MCSubtargetInfo &STI, MCContext &Ctx, bool IsBigEndian)
       : MCDisassembler(STI, Ctx),
-        IsMicroMips(STI.getFeatureBits() & Mips::FeatureMicroMips),
+        IsMicroMips(STI.getFeatureBits()[Mips::FeatureMicroMips]),
         IsBigEndian(IsBigEndian) {}
 
-  bool hasMips3() const { return STI.getFeatureBits() & Mips::FeatureMips3; }
-  bool hasMips32() const { return STI.getFeatureBits() & Mips::FeatureMips32; }
+  bool hasMips3() const { return STI.getFeatureBits()[Mips::FeatureMips3]; }
+  bool hasMips32() const { return STI.getFeatureBits()[Mips::FeatureMips32]; }
   bool hasMips32r6() const {
-    return STI.getFeatureBits() & Mips::FeatureMips32r6;
+    return STI.getFeatureBits()[Mips::FeatureMips32r6];
   }
 
-  bool isGP64() const { return STI.getFeatureBits() & Mips::FeatureGP64Bit; }
+  bool isGP64() const { return STI.getFeatureBits()[Mips::FeatureGP64Bit]; }
+
+  bool hasCnMips() const { return STI.getFeatureBits()[Mips::FeatureCnMips]; }
 
   bool hasCOP3() const {
     // Only present in MIPS-I and MIPS-II
@@ -176,6 +178,11 @@ static DecodeStatus DecodeMSACtrlRegisterClass(MCInst &Inst,
                                                uint64_t Address,
                                                const void *Decoder);
 
+static DecodeStatus DecodeCOP0RegisterClass(MCInst &Inst,
+                                            unsigned RegNo,
+                                            uint64_t Address,
+                                            const void *Decoder);
+
 static DecodeStatus DecodeCOP2RegisterClass(MCInst &Inst,
                                             unsigned RegNo,
                                             uint64_t Address,
@@ -490,13 +497,13 @@ static DecodeStatus DecodeINSVE_DF(MCInst &MI, InsnType insn, uint64_t Address,
     return MCDisassembler::Fail;
   // $n
   tmp = fieldFromInstruction(insn, 16, NSize);
-  MI.addOperand(MCOperand::CreateImm(tmp));
+  MI.addOperand(MCOperand::createImm(tmp));
   // $ws
   tmp = fieldFromInstruction(insn, 11, 5);
   if (RegDecoder(MI, tmp, Address, Decoder) == MCDisassembler::Fail)
     return MCDisassembler::Fail;
   // $n2
-  MI.addOperand(MCOperand::CreateImm(0));
+  MI.addOperand(MCOperand::createImm(0));
 
   return MCDisassembler::Success;
 }
@@ -530,12 +537,12 @@ static DecodeStatus DecodeAddiGroupBranch(MCInst &MI, InsnType insn,
     MI.setOpcode(Mips::BEQZALC);
 
   if (HasRs)
-    MI.addOperand(MCOperand::CreateReg(getReg(Decoder, Mips::GPR32RegClassID,
+    MI.addOperand(MCOperand::createReg(getReg(Decoder, Mips::GPR32RegClassID,
                                        Rs)));
 
-  MI.addOperand(MCOperand::CreateReg(getReg(Decoder, Mips::GPR32RegClassID,
+  MI.addOperand(MCOperand::createReg(getReg(Decoder, Mips::GPR32RegClassID,
                                      Rt)));
-  MI.addOperand(MCOperand::CreateImm(Imm));
+  MI.addOperand(MCOperand::createImm(Imm));
 
   return MCDisassembler::Success;
 }
@@ -569,12 +576,12 @@ static DecodeStatus DecodeDaddiGroupBranch(MCInst &MI, InsnType insn,
     MI.setOpcode(Mips::BNEZALC);
 
   if (HasRs)
-    MI.addOperand(MCOperand::CreateReg(getReg(Decoder, Mips::GPR32RegClassID,
+    MI.addOperand(MCOperand::createReg(getReg(Decoder, Mips::GPR32RegClassID,
                                        Rs)));
 
-  MI.addOperand(MCOperand::CreateReg(getReg(Decoder, Mips::GPR32RegClassID,
+  MI.addOperand(MCOperand::createReg(getReg(Decoder, Mips::GPR32RegClassID,
                                      Rt)));
-  MI.addOperand(MCOperand::CreateImm(Imm));
+  MI.addOperand(MCOperand::createImm(Imm));
 
   return MCDisassembler::Success;
 }
@@ -611,13 +618,13 @@ static DecodeStatus DecodeBlezlGroupBranch(MCInst &MI, InsnType insn,
   }
 
   if (HasRs)
-    MI.addOperand(MCOperand::CreateReg(getReg(Decoder, Mips::GPR32RegClassID,
+    MI.addOperand(MCOperand::createReg(getReg(Decoder, Mips::GPR32RegClassID,
                                        Rs)));
 
-  MI.addOperand(MCOperand::CreateReg(getReg(Decoder, Mips::GPR32RegClassID,
+  MI.addOperand(MCOperand::createReg(getReg(Decoder, Mips::GPR32RegClassID,
                                      Rt)));
 
-  MI.addOperand(MCOperand::CreateImm(Imm));
+  MI.addOperand(MCOperand::createImm(Imm));
 
   return MCDisassembler::Success;
 }
@@ -655,13 +662,13 @@ static DecodeStatus DecodeBgtzlGroupBranch(MCInst &MI, InsnType insn,
   }
 
   if (HasRs)
-    MI.addOperand(MCOperand::CreateReg(getReg(Decoder, Mips::GPR32RegClassID,
+    MI.addOperand(MCOperand::createReg(getReg(Decoder, Mips::GPR32RegClassID,
                                               Rs)));
 
-  MI.addOperand(MCOperand::CreateReg(getReg(Decoder, Mips::GPR32RegClassID,
+  MI.addOperand(MCOperand::createReg(getReg(Decoder, Mips::GPR32RegClassID,
                                      Rt)));
 
-  MI.addOperand(MCOperand::CreateImm(Imm));
+  MI.addOperand(MCOperand::createImm(Imm));
 
   return MCDisassembler::Success;
 }
@@ -703,14 +710,14 @@ static DecodeStatus DecodeBgtzGroupBranch(MCInst &MI, InsnType insn,
   }
 
   if (HasRs)
-    MI.addOperand(MCOperand::CreateReg(getReg(Decoder, Mips::GPR32RegClassID,
+    MI.addOperand(MCOperand::createReg(getReg(Decoder, Mips::GPR32RegClassID,
                                        Rs)));
 
   if (HasRt)
-    MI.addOperand(MCOperand::CreateReg(getReg(Decoder, Mips::GPR32RegClassID,
+    MI.addOperand(MCOperand::createReg(getReg(Decoder, Mips::GPR32RegClassID,
                                        Rt)));
 
-  MI.addOperand(MCOperand::CreateImm(Imm));
+  MI.addOperand(MCOperand::createImm(Imm));
 
   return MCDisassembler::Success;
 }
@@ -747,12 +754,12 @@ static DecodeStatus DecodeBlezGroupBranch(MCInst &MI, InsnType insn,
   }
 
   if (HasRs)
-    MI.addOperand(MCOperand::CreateReg(getReg(Decoder, Mips::GPR32RegClassID,
+    MI.addOperand(MCOperand::createReg(getReg(Decoder, Mips::GPR32RegClassID,
                                        Rs)));
-  MI.addOperand(MCOperand::CreateReg(getReg(Decoder, Mips::GPR32RegClassID,
+  MI.addOperand(MCOperand::createReg(getReg(Decoder, Mips::GPR32RegClassID,
                                      Rt)));
 
-  MI.addOperand(MCOperand::CreateImm(Imm));
+  MI.addOperand(MCOperand::createImm(Imm));
 
   return MCDisassembler::Success;
 }
@@ -837,10 +844,17 @@ DecodeStatus MipsDisassembler::getInstruction(MCInst &Instr, uint64_t &Size,
     if (Result == MCDisassembler::Fail)
       return MCDisassembler::Fail;
 
-    DEBUG(dbgs() << "Trying MicroMips32 table (32-bit instructions):\n");
-    // Calling the auto-generated decoder function.
-    Result = decodeInstruction(DecoderTableMicroMips32, Instr, Insn, Address,
-                               this, STI);
+    if (hasMips32r6()) {
+      DEBUG(dbgs() << "Trying MicroMips32r632 table (32-bit instructions):\n");
+      // Calling the auto-generated decoder function.
+      Result = decodeInstruction(DecoderTableMicroMips32r632, Instr, Insn, Address,
+                                 this, STI);
+    } else {
+      DEBUG(dbgs() << "Trying MicroMips32 table (32-bit instructions):\n");
+      // Calling the auto-generated decoder function.
+      Result = decodeInstruction(DecoderTableMicroMips32, Instr, Insn, Address,
+                                 this, STI);
+    }
     if (Result != MCDisassembler::Fail) {
       Size = 4;
       return Result;
@@ -882,6 +896,16 @@ DecodeStatus MipsDisassembler::getInstruction(MCInst &Instr, uint64_t &Size,
     }
   }
 
+  if (hasCnMips()) {
+    DEBUG(dbgs() << "Trying CnMips table (32-bit opcodes):\n");
+    Result = decodeInstruction(DecoderTableCnMips32, Instr, Insn,
+                               Address, this, STI);
+    if (Result != MCDisassembler::Fail) {
+      Size = 4;
+      return Result;
+    }
+  }
+
   if (isGP64()) {
     DEBUG(dbgs() << "Trying Mips64 (GPR64) table (32-bit opcodes):\n");
     Result = decodeInstruction(DecoderTableMips6432, Instr, Insn,
@@ -922,7 +946,7 @@ static DecodeStatus DecodeGPR64RegisterClass(MCInst &Inst,
     return MCDisassembler::Fail;
 
   unsigned Reg = getReg(Decoder, Mips::GPR64RegClassID, RegNo);
-  Inst.addOperand(MCOperand::CreateReg(Reg));
+  Inst.addOperand(MCOperand::createReg(Reg));
   return MCDisassembler::Success;
 }
 
@@ -933,7 +957,7 @@ static DecodeStatus DecodeGPRMM16RegisterClass(MCInst &Inst,
   if (RegNo > 7)
     return MCDisassembler::Fail;
   unsigned Reg = getReg(Decoder, Mips::GPRMM16RegClassID, RegNo);
-  Inst.addOperand(MCOperand::CreateReg(Reg));
+  Inst.addOperand(MCOperand::createReg(Reg));
   return MCDisassembler::Success;
 }
 
@@ -944,7 +968,7 @@ static DecodeStatus DecodeGPRMM16ZeroRegisterClass(MCInst &Inst,
   if (RegNo > 7)
     return MCDisassembler::Fail;
   unsigned Reg = getReg(Decoder, Mips::GPRMM16ZeroRegClassID, RegNo);
-  Inst.addOperand(MCOperand::CreateReg(Reg));
+  Inst.addOperand(MCOperand::createReg(Reg));
   return MCDisassembler::Success;
 }
 
@@ -955,7 +979,7 @@ static DecodeStatus DecodeGPRMM16MovePRegisterClass(MCInst &Inst,
   if (RegNo > 7)
     return MCDisassembler::Fail;
   unsigned Reg = getReg(Decoder, Mips::GPRMM16MovePRegClassID, RegNo);
-  Inst.addOperand(MCOperand::CreateReg(Reg));
+  Inst.addOperand(MCOperand::createReg(Reg));
   return MCDisassembler::Success;
 }
 
@@ -966,7 +990,7 @@ static DecodeStatus DecodeGPR32RegisterClass(MCInst &Inst,
   if (RegNo > 31)
     return MCDisassembler::Fail;
   unsigned Reg = getReg(Decoder, Mips::GPR32RegClassID, RegNo);
-  Inst.addOperand(MCOperand::CreateReg(Reg));
+  Inst.addOperand(MCOperand::createReg(Reg));
   return MCDisassembler::Success;
 }
 
@@ -995,7 +1019,7 @@ static DecodeStatus DecodeFGR64RegisterClass(MCInst &Inst,
     return MCDisassembler::Fail;
 
   unsigned Reg = getReg(Decoder, Mips::FGR64RegClassID, RegNo);
-  Inst.addOperand(MCOperand::CreateReg(Reg));
+  Inst.addOperand(MCOperand::createReg(Reg));
   return MCDisassembler::Success;
 }
 
@@ -1007,7 +1031,7 @@ static DecodeStatus DecodeFGR32RegisterClass(MCInst &Inst,
     return MCDisassembler::Fail;
 
   unsigned Reg = getReg(Decoder, Mips::FGR32RegClassID, RegNo);
-  Inst.addOperand(MCOperand::CreateReg(Reg));
+  Inst.addOperand(MCOperand::createReg(Reg));
   return MCDisassembler::Success;
 }
 
@@ -1018,7 +1042,7 @@ static DecodeStatus DecodeCCRRegisterClass(MCInst &Inst,
   if (RegNo > 31)
     return MCDisassembler::Fail;
   unsigned Reg = getReg(Decoder, Mips::CCRRegClassID, RegNo);
-  Inst.addOperand(MCOperand::CreateReg(Reg));
+  Inst.addOperand(MCOperand::createReg(Reg));
   return MCDisassembler::Success;
 }
 
@@ -1029,7 +1053,7 @@ static DecodeStatus DecodeFCCRegisterClass(MCInst &Inst,
   if (RegNo > 7)
     return MCDisassembler::Fail;
   unsigned Reg = getReg(Decoder, Mips::FCCRegClassID, RegNo);
-  Inst.addOperand(MCOperand::CreateReg(Reg));
+  Inst.addOperand(MCOperand::createReg(Reg));
   return MCDisassembler::Success;
 }
 
@@ -1040,7 +1064,7 @@ static DecodeStatus DecodeFGRCCRegisterClass(MCInst &Inst, unsigned RegNo,
     return MCDisassembler::Fail;
 
   unsigned Reg = getReg(Decoder, Mips::FGRCCRegClassID, RegNo);
-  Inst.addOperand(MCOperand::CreateReg(Reg));
+  Inst.addOperand(MCOperand::createReg(Reg));
   return MCDisassembler::Success;
 }
 
@@ -1057,12 +1081,12 @@ static DecodeStatus DecodeMem(MCInst &Inst,
 
   if(Inst.getOpcode() == Mips::SC ||
      Inst.getOpcode() == Mips::SCD){
-    Inst.addOperand(MCOperand::CreateReg(Reg));
+    Inst.addOperand(MCOperand::createReg(Reg));
   }
 
-  Inst.addOperand(MCOperand::CreateReg(Reg));
-  Inst.addOperand(MCOperand::CreateReg(Base));
-  Inst.addOperand(MCOperand::CreateImm(Offset));
+  Inst.addOperand(MCOperand::createReg(Reg));
+  Inst.addOperand(MCOperand::createReg(Base));
+  Inst.addOperand(MCOperand::createImm(Offset));
 
   return MCDisassembler::Success;
 }
@@ -1077,9 +1101,9 @@ static DecodeStatus DecodeCacheOp(MCInst &Inst,
 
   Base = getReg(Decoder, Mips::GPR32RegClassID, Base);
 
-  Inst.addOperand(MCOperand::CreateReg(Base));
-  Inst.addOperand(MCOperand::CreateImm(Offset));
-  Inst.addOperand(MCOperand::CreateImm(Hint));
+  Inst.addOperand(MCOperand::createReg(Base));
+  Inst.addOperand(MCOperand::createImm(Offset));
+  Inst.addOperand(MCOperand::createImm(Hint));
 
   return MCDisassembler::Success;
 }
@@ -1094,9 +1118,9 @@ static DecodeStatus DecodeCacheOpMM(MCInst &Inst,
 
   Base = getReg(Decoder, Mips::GPR32RegClassID, Base);
 
-  Inst.addOperand(MCOperand::CreateReg(Base));
-  Inst.addOperand(MCOperand::CreateImm(Offset));
-  Inst.addOperand(MCOperand::CreateImm(Hint));
+  Inst.addOperand(MCOperand::createReg(Base));
+  Inst.addOperand(MCOperand::createImm(Offset));
+  Inst.addOperand(MCOperand::createImm(Hint));
 
   return MCDisassembler::Success;
 }
@@ -1111,9 +1135,9 @@ static DecodeStatus DecodeCacheOpR6(MCInst &Inst,
 
   Base = getReg(Decoder, Mips::GPR32RegClassID, Base);
 
-  Inst.addOperand(MCOperand::CreateReg(Base));
-  Inst.addOperand(MCOperand::CreateImm(Offset));
-  Inst.addOperand(MCOperand::CreateImm(Hint));
+  Inst.addOperand(MCOperand::createReg(Base));
+  Inst.addOperand(MCOperand::createImm(Offset));
+  Inst.addOperand(MCOperand::createImm(Hint));
 
   return MCDisassembler::Success;
 }
@@ -1127,8 +1151,8 @@ static DecodeStatus DecodeSyncI(MCInst &Inst,
 
   Base = getReg(Decoder, Mips::GPR32RegClassID, Base);
 
-  Inst.addOperand(MCOperand::CreateReg(Base));
-  Inst.addOperand(MCOperand::CreateImm(Offset));
+  Inst.addOperand(MCOperand::createReg(Base));
+  Inst.addOperand(MCOperand::createImm(Offset));
 
   return MCDisassembler::Success;
 }
@@ -1142,8 +1166,8 @@ static DecodeStatus DecodeMSA128Mem(MCInst &Inst, unsigned Insn,
   Reg = getReg(Decoder, Mips::MSA128BRegClassID, Reg);
   Base = getReg(Decoder, Mips::GPR32RegClassID, Base);
 
-  Inst.addOperand(MCOperand::CreateReg(Reg));
-  Inst.addOperand(MCOperand::CreateReg(Base));
+  Inst.addOperand(MCOperand::createReg(Reg));
+  Inst.addOperand(MCOperand::createReg(Base));
 
   // The immediate field of an LD/ST instruction is scaled which means it must
   // be multiplied (when decoding) by the size (in bytes) of the instructions'
@@ -1160,19 +1184,19 @@ static DecodeStatus DecodeMSA128Mem(MCInst &Inst, unsigned Insn,
     break;
   case Mips::LD_B:
   case Mips::ST_B:
-    Inst.addOperand(MCOperand::CreateImm(Offset));
+    Inst.addOperand(MCOperand::createImm(Offset));
     break;
   case Mips::LD_H:
   case Mips::ST_H:
-    Inst.addOperand(MCOperand::CreateImm(Offset * 2));
+    Inst.addOperand(MCOperand::createImm(Offset * 2));
     break;
   case Mips::LD_W:
   case Mips::ST_W:
-    Inst.addOperand(MCOperand::CreateImm(Offset * 4));
+    Inst.addOperand(MCOperand::createImm(Offset * 4));
     break;
   case Mips::LD_D:
   case Mips::ST_D:
-    Inst.addOperand(MCOperand::CreateImm(Offset * 8));
+    Inst.addOperand(MCOperand::createImm(Offset * 8));
     break;
   }
 
@@ -1211,20 +1235,20 @@ static DecodeStatus DecodeMemMMImm4(MCInst &Inst,
   switch (Inst.getOpcode()) {
     case Mips::LBU16_MM:
       if (Offset == 0xf)
-        Inst.addOperand(MCOperand::CreateImm(-1));
+        Inst.addOperand(MCOperand::createImm(-1));
       else
-        Inst.addOperand(MCOperand::CreateImm(Offset));
+        Inst.addOperand(MCOperand::createImm(Offset));
       break;
     case Mips::SB16_MM:
-      Inst.addOperand(MCOperand::CreateImm(Offset));
+      Inst.addOperand(MCOperand::createImm(Offset));
       break;
     case Mips::LHU16_MM:
     case Mips::SH16_MM:
-      Inst.addOperand(MCOperand::CreateImm(Offset << 1));
+      Inst.addOperand(MCOperand::createImm(Offset << 1));
       break;
     case Mips::LW16_MM:
     case Mips::SW16_MM:
-      Inst.addOperand(MCOperand::CreateImm(Offset << 2));
+      Inst.addOperand(MCOperand::createImm(Offset << 2));
       break;
   }
 
@@ -1240,9 +1264,9 @@ static DecodeStatus DecodeMemMMSPImm5Lsl2(MCInst &Inst,
 
   Reg = getReg(Decoder, Mips::GPR32RegClassID, Reg);
 
-  Inst.addOperand(MCOperand::CreateReg(Reg));
-  Inst.addOperand(MCOperand::CreateReg(Mips::SP));
-  Inst.addOperand(MCOperand::CreateImm(Offset << 2));
+  Inst.addOperand(MCOperand::createReg(Reg));
+  Inst.addOperand(MCOperand::createReg(Mips::SP));
+  Inst.addOperand(MCOperand::createImm(Offset << 2));
 
   return MCDisassembler::Success;
 }
@@ -1256,9 +1280,9 @@ static DecodeStatus DecodeMemMMGPImm7Lsl2(MCInst &Inst,
 
   Reg = getReg(Decoder, Mips::GPR32RegClassID, Reg);
 
-  Inst.addOperand(MCOperand::CreateReg(Reg));
-  Inst.addOperand(MCOperand::CreateReg(Mips::GP));
-  Inst.addOperand(MCOperand::CreateImm(Offset << 2));
+  Inst.addOperand(MCOperand::createReg(Reg));
+  Inst.addOperand(MCOperand::createReg(Mips::GP));
+  Inst.addOperand(MCOperand::createImm(Offset << 2));
 
   return MCDisassembler::Success;
 }
@@ -1273,8 +1297,8 @@ static DecodeStatus DecodeMemMMReglistImm4Lsl2(MCInst &Inst,
       == MCDisassembler::Fail)
     return MCDisassembler::Fail;
 
-  Inst.addOperand(MCOperand::CreateReg(Mips::SP));
-  Inst.addOperand(MCOperand::CreateImm(Offset << 2));
+  Inst.addOperand(MCOperand::createReg(Mips::SP));
+  Inst.addOperand(MCOperand::createImm(Offset << 2));
 
   return MCDisassembler::Success;
 }
@@ -1296,19 +1320,19 @@ static DecodeStatus DecodeMemMMImm12(MCInst &Inst,
     if (DecodeRegListOperand(Inst, Insn, Address, Decoder)
         == MCDisassembler::Fail)
       return MCDisassembler::Fail;
-    Inst.addOperand(MCOperand::CreateReg(Base));
-    Inst.addOperand(MCOperand::CreateImm(Offset));
+    Inst.addOperand(MCOperand::createReg(Base));
+    Inst.addOperand(MCOperand::createImm(Offset));
     break;
   case Mips::SC_MM:
-    Inst.addOperand(MCOperand::CreateReg(Reg));
+    Inst.addOperand(MCOperand::createReg(Reg));
     // fallthrough
   default:
-    Inst.addOperand(MCOperand::CreateReg(Reg));
+    Inst.addOperand(MCOperand::createReg(Reg));
     if (Inst.getOpcode() == Mips::LWP_MM || Inst.getOpcode() == Mips::SWP_MM)
-      Inst.addOperand(MCOperand::CreateReg(Reg+1));
+      Inst.addOperand(MCOperand::createReg(Reg+1));
 
-    Inst.addOperand(MCOperand::CreateReg(Base));
-    Inst.addOperand(MCOperand::CreateImm(Offset));
+    Inst.addOperand(MCOperand::createReg(Base));
+    Inst.addOperand(MCOperand::createImm(Offset));
   }
 
   return MCDisassembler::Success;
@@ -1325,9 +1349,9 @@ static DecodeStatus DecodeMemMMImm16(MCInst &Inst,
   Reg = getReg(Decoder, Mips::GPR32RegClassID, Reg);
   Base = getReg(Decoder, Mips::GPR32RegClassID, Base);
 
-  Inst.addOperand(MCOperand::CreateReg(Reg));
-  Inst.addOperand(MCOperand::CreateReg(Base));
-  Inst.addOperand(MCOperand::CreateImm(Offset));
+  Inst.addOperand(MCOperand::createReg(Reg));
+  Inst.addOperand(MCOperand::createReg(Base));
+  Inst.addOperand(MCOperand::createImm(Offset));
 
   return MCDisassembler::Success;
 }
@@ -1343,9 +1367,9 @@ static DecodeStatus DecodeFMem(MCInst &Inst,
   Reg = getReg(Decoder, Mips::FGR64RegClassID, Reg);
   Base = getReg(Decoder, Mips::GPR32RegClassID, Base);
 
-  Inst.addOperand(MCOperand::CreateReg(Reg));
-  Inst.addOperand(MCOperand::CreateReg(Base));
-  Inst.addOperand(MCOperand::CreateImm(Offset));
+  Inst.addOperand(MCOperand::createReg(Reg));
+  Inst.addOperand(MCOperand::createReg(Base));
+  Inst.addOperand(MCOperand::createImm(Offset));
 
   return MCDisassembler::Success;
 }
@@ -1361,9 +1385,9 @@ static DecodeStatus DecodeFMem2(MCInst &Inst,
   Reg = getReg(Decoder, Mips::COP2RegClassID, Reg);
   Base = getReg(Decoder, Mips::GPR32RegClassID, Base);
 
-  Inst.addOperand(MCOperand::CreateReg(Reg));
-  Inst.addOperand(MCOperand::CreateReg(Base));
-  Inst.addOperand(MCOperand::CreateImm(Offset));
+  Inst.addOperand(MCOperand::createReg(Reg));
+  Inst.addOperand(MCOperand::createReg(Base));
+  Inst.addOperand(MCOperand::createImm(Offset));
 
   return MCDisassembler::Success;
 }
@@ -1379,9 +1403,9 @@ static DecodeStatus DecodeFMem3(MCInst &Inst,
   Reg = getReg(Decoder, Mips::COP3RegClassID, Reg);
   Base = getReg(Decoder, Mips::GPR32RegClassID, Base);
 
-  Inst.addOperand(MCOperand::CreateReg(Reg));
-  Inst.addOperand(MCOperand::CreateReg(Base));
-  Inst.addOperand(MCOperand::CreateImm(Offset));
+  Inst.addOperand(MCOperand::createReg(Reg));
+  Inst.addOperand(MCOperand::createReg(Base));
+  Inst.addOperand(MCOperand::createImm(Offset));
 
   return MCDisassembler::Success;
 }
@@ -1397,9 +1421,9 @@ static DecodeStatus DecodeFMemCop2R6(MCInst &Inst,
   Reg = getReg(Decoder, Mips::COP2RegClassID, Reg);
   Base = getReg(Decoder, Mips::GPR32RegClassID, Base);
 
-  Inst.addOperand(MCOperand::CreateReg(Reg));
-  Inst.addOperand(MCOperand::CreateReg(Base));
-  Inst.addOperand(MCOperand::CreateImm(Offset));
+  Inst.addOperand(MCOperand::createReg(Reg));
+  Inst.addOperand(MCOperand::createReg(Base));
+  Inst.addOperand(MCOperand::createImm(Offset));
 
   return MCDisassembler::Success;
 }
@@ -1415,12 +1439,12 @@ static DecodeStatus DecodeSpecial3LlSc(MCInst &Inst,
   Base = getReg(Decoder, Mips::GPR32RegClassID, Base);
 
   if(Inst.getOpcode() == Mips::SC_R6 || Inst.getOpcode() == Mips::SCD_R6){
-    Inst.addOperand(MCOperand::CreateReg(Rt));
+    Inst.addOperand(MCOperand::createReg(Rt));
   }
 
-  Inst.addOperand(MCOperand::CreateReg(Rt));
-  Inst.addOperand(MCOperand::CreateReg(Base));
-  Inst.addOperand(MCOperand::CreateImm(Offset));
+  Inst.addOperand(MCOperand::createReg(Rt));
+  Inst.addOperand(MCOperand::createReg(Base));
+  Inst.addOperand(MCOperand::createImm(Offset));
 
   return MCDisassembler::Success;
 }
@@ -1432,7 +1456,7 @@ static DecodeStatus DecodeHWRegsRegisterClass(MCInst &Inst,
   // Currently only hardware register 29 is supported.
   if (RegNo != 29)
     return  MCDisassembler::Fail;
-  Inst.addOperand(MCOperand::CreateReg(Mips::HWR29));
+  Inst.addOperand(MCOperand::createReg(Mips::HWR29));
   return MCDisassembler::Success;
 }
 
@@ -1445,7 +1469,7 @@ static DecodeStatus DecodeAFGR64RegisterClass(MCInst &Inst,
 
   ;
   unsigned Reg = getReg(Decoder, Mips::AFGR64RegClassID, RegNo /2);
-  Inst.addOperand(MCOperand::CreateReg(Reg));
+  Inst.addOperand(MCOperand::createReg(Reg));
   return MCDisassembler::Success;
 }
 
@@ -1457,7 +1481,7 @@ static DecodeStatus DecodeACC64DSPRegisterClass(MCInst &Inst,
     return MCDisassembler::Fail;
 
   unsigned Reg = getReg(Decoder, Mips::ACC64DSPRegClassID, RegNo);
-  Inst.addOperand(MCOperand::CreateReg(Reg));
+  Inst.addOperand(MCOperand::createReg(Reg));
   return MCDisassembler::Success;
 }
 
@@ -1469,7 +1493,7 @@ static DecodeStatus DecodeHI32DSPRegisterClass(MCInst &Inst,
     return MCDisassembler::Fail;
 
   unsigned Reg = getReg(Decoder, Mips::HI32DSPRegClassID, RegNo);
-  Inst.addOperand(MCOperand::CreateReg(Reg));
+  Inst.addOperand(MCOperand::createReg(Reg));
   return MCDisassembler::Success;
 }
 
@@ -1481,7 +1505,7 @@ static DecodeStatus DecodeLO32DSPRegisterClass(MCInst &Inst,
     return MCDisassembler::Fail;
 
   unsigned Reg = getReg(Decoder, Mips::LO32DSPRegClassID, RegNo);
-  Inst.addOperand(MCOperand::CreateReg(Reg));
+  Inst.addOperand(MCOperand::createReg(Reg));
   return MCDisassembler::Success;
 }
 
@@ -1493,7 +1517,7 @@ static DecodeStatus DecodeMSA128BRegisterClass(MCInst &Inst,
     return MCDisassembler::Fail;
 
   unsigned Reg = getReg(Decoder, Mips::MSA128BRegClassID, RegNo);
-  Inst.addOperand(MCOperand::CreateReg(Reg));
+  Inst.addOperand(MCOperand::createReg(Reg));
   return MCDisassembler::Success;
 }
 
@@ -1505,7 +1529,7 @@ static DecodeStatus DecodeMSA128HRegisterClass(MCInst &Inst,
     return MCDisassembler::Fail;
 
   unsigned Reg = getReg(Decoder, Mips::MSA128HRegClassID, RegNo);
-  Inst.addOperand(MCOperand::CreateReg(Reg));
+  Inst.addOperand(MCOperand::createReg(Reg));
   return MCDisassembler::Success;
 }
 
@@ -1517,7 +1541,7 @@ static DecodeStatus DecodeMSA128WRegisterClass(MCInst &Inst,
     return MCDisassembler::Fail;
 
   unsigned Reg = getReg(Decoder, Mips::MSA128WRegClassID, RegNo);
-  Inst.addOperand(MCOperand::CreateReg(Reg));
+  Inst.addOperand(MCOperand::createReg(Reg));
   return MCDisassembler::Success;
 }
 
@@ -1529,7 +1553,7 @@ static DecodeStatus DecodeMSA128DRegisterClass(MCInst &Inst,
     return MCDisassembler::Fail;
 
   unsigned Reg = getReg(Decoder, Mips::MSA128DRegClassID, RegNo);
-  Inst.addOperand(MCOperand::CreateReg(Reg));
+  Inst.addOperand(MCOperand::createReg(Reg));
   return MCDisassembler::Success;
 }
 
@@ -1541,7 +1565,19 @@ static DecodeStatus DecodeMSACtrlRegisterClass(MCInst &Inst,
     return MCDisassembler::Fail;
 
   unsigned Reg = getReg(Decoder, Mips::MSACtrlRegClassID, RegNo);
-  Inst.addOperand(MCOperand::CreateReg(Reg));
+  Inst.addOperand(MCOperand::createReg(Reg));
+  return MCDisassembler::Success;
+}
+
+static DecodeStatus DecodeCOP0RegisterClass(MCInst &Inst,
+                                            unsigned RegNo,
+                                            uint64_t Address,
+                                            const void *Decoder) {
+  if (RegNo > 31)
+    return MCDisassembler::Fail;
+
+  unsigned Reg = getReg(Decoder, Mips::COP0RegClassID, RegNo);
+  Inst.addOperand(MCOperand::createReg(Reg));
   return MCDisassembler::Success;
 }
 
@@ -1553,7 +1589,7 @@ static DecodeStatus DecodeCOP2RegisterClass(MCInst &Inst,
     return MCDisassembler::Fail;
 
   unsigned Reg = getReg(Decoder, Mips::COP2RegClassID, RegNo);
-  Inst.addOperand(MCOperand::CreateReg(Reg));
+  Inst.addOperand(MCOperand::createReg(Reg));
   return MCDisassembler::Success;
 }
 
@@ -1562,7 +1598,7 @@ static DecodeStatus DecodeBranchTarget(MCInst &Inst,
                                        uint64_t Address,
                                        const void *Decoder) {
   int32_t BranchOffset = (SignExtend32<16>(Offset) * 4) + 4;
-  Inst.addOperand(MCOperand::CreateImm(BranchOffset));
+  Inst.addOperand(MCOperand::createImm(BranchOffset));
   return MCDisassembler::Success;
 }
 
@@ -1572,7 +1608,7 @@ static DecodeStatus DecodeJumpTarget(MCInst &Inst,
                                      const void *Decoder) {
 
   unsigned JumpOffset = fieldFromInstruction(Insn, 0, 26) << 2;
-  Inst.addOperand(MCOperand::CreateImm(JumpOffset));
+  Inst.addOperand(MCOperand::createImm(JumpOffset));
   return MCDisassembler::Success;
 }
 
@@ -1582,7 +1618,7 @@ static DecodeStatus DecodeBranchTarget21(MCInst &Inst,
                                          const void *Decoder) {
   int32_t BranchOffset = SignExtend32<21>(Offset) * 4;
 
-  Inst.addOperand(MCOperand::CreateImm(BranchOffset));
+  Inst.addOperand(MCOperand::createImm(BranchOffset));
   return MCDisassembler::Success;
 }
 
@@ -1592,7 +1628,7 @@ static DecodeStatus DecodeBranchTarget26(MCInst &Inst,
                                          const void *Decoder) {
   int32_t BranchOffset = SignExtend32<26>(Offset) * 4;
 
-  Inst.addOperand(MCOperand::CreateImm(BranchOffset));
+  Inst.addOperand(MCOperand::createImm(BranchOffset));
   return MCDisassembler::Success;
 }
 
@@ -1601,7 +1637,7 @@ static DecodeStatus DecodeBranchTarget7MM(MCInst &Inst,
                                           uint64_t Address,
                                           const void *Decoder) {
   int32_t BranchOffset = SignExtend32<7>(Offset) << 1;
-  Inst.addOperand(MCOperand::CreateImm(BranchOffset));
+  Inst.addOperand(MCOperand::createImm(BranchOffset));
   return MCDisassembler::Success;
 }
 
@@ -1610,7 +1646,7 @@ static DecodeStatus DecodeBranchTarget10MM(MCInst &Inst,
                                            uint64_t Address,
                                            const void *Decoder) {
   int32_t BranchOffset = SignExtend32<10>(Offset) << 1;
-  Inst.addOperand(MCOperand::CreateImm(BranchOffset));
+  Inst.addOperand(MCOperand::createImm(BranchOffset));
   return MCDisassembler::Success;
 }
 
@@ -1619,7 +1655,7 @@ static DecodeStatus DecodeBranchTargetMM(MCInst &Inst,
                                          uint64_t Address,
                                          const void *Decoder) {
   int32_t BranchOffset = SignExtend32<16>(Offset) * 2;
-  Inst.addOperand(MCOperand::CreateImm(BranchOffset));
+  Inst.addOperand(MCOperand::createImm(BranchOffset));
   return MCDisassembler::Success;
 }
 
@@ -1628,7 +1664,7 @@ static DecodeStatus DecodeJumpTargetMM(MCInst &Inst,
                                        uint64_t Address,
                                        const void *Decoder) {
   unsigned JumpOffset = fieldFromInstruction(Insn, 0, 26) << 1;
-  Inst.addOperand(MCOperand::CreateImm(JumpOffset));
+  Inst.addOperand(MCOperand::createImm(JumpOffset));
   return MCDisassembler::Success;
 }
 
@@ -1637,11 +1673,11 @@ static DecodeStatus DecodeAddiur2Simm7(MCInst &Inst,
                                        uint64_t Address,
                                        const void *Decoder) {
   if (Value == 0)
-    Inst.addOperand(MCOperand::CreateImm(1));
+    Inst.addOperand(MCOperand::createImm(1));
   else if (Value == 0x7)
-    Inst.addOperand(MCOperand::CreateImm(-1));
+    Inst.addOperand(MCOperand::createImm(-1));
   else
-    Inst.addOperand(MCOperand::CreateImm(Value << 2));
+    Inst.addOperand(MCOperand::createImm(Value << 2));
   return MCDisassembler::Success;
 }
 
@@ -1649,7 +1685,7 @@ static DecodeStatus DecodeUImm6Lsl2(MCInst &Inst,
                                     unsigned Value,
                                     uint64_t Address,
                                     const void *Decoder) {
-  Inst.addOperand(MCOperand::CreateImm(Value << 2));
+  Inst.addOperand(MCOperand::createImm(Value << 2));
   return MCDisassembler::Success;
 }
 
@@ -1658,9 +1694,9 @@ static DecodeStatus DecodeLiSimm7(MCInst &Inst,
                                   uint64_t Address,
                                   const void *Decoder) {
   if (Value == 0x7F)
-    Inst.addOperand(MCOperand::CreateImm(-1));
+    Inst.addOperand(MCOperand::createImm(-1));
   else
-    Inst.addOperand(MCOperand::CreateImm(Value));
+    Inst.addOperand(MCOperand::createImm(Value));
   return MCDisassembler::Success;
 }
 
@@ -1668,7 +1704,7 @@ static DecodeStatus DecodeSimm4(MCInst &Inst,
                                 unsigned Value,
                                 uint64_t Address,
                                 const void *Decoder) {
-  Inst.addOperand(MCOperand::CreateImm(SignExtend32<4>(Value)));
+  Inst.addOperand(MCOperand::createImm(SignExtend32<4>(Value)));
   return MCDisassembler::Success;
 }
 
@@ -1676,7 +1712,7 @@ static DecodeStatus DecodeSimm16(MCInst &Inst,
                                  unsigned Insn,
                                  uint64_t Address,
                                  const void *Decoder) {
-  Inst.addOperand(MCOperand::CreateImm(SignExtend32<16>(Insn)));
+  Inst.addOperand(MCOperand::createImm(SignExtend32<16>(Insn)));
   return MCDisassembler::Success;
 }
 
@@ -1685,7 +1721,7 @@ static DecodeStatus DecodeLSAImm(MCInst &Inst,
                                  uint64_t Address,
                                  const void *Decoder) {
   // We add one to the immediate field as it was encoded as 'imm - 1'.
-  Inst.addOperand(MCOperand::CreateImm(Insn + 1));
+  Inst.addOperand(MCOperand::createImm(Insn + 1));
   return MCDisassembler::Success;
 }
 
@@ -1696,7 +1732,7 @@ static DecodeStatus DecodeInsSize(MCInst &Inst,
   // First we need to grab the pos(lsb) from MCInst.
   int Pos = Inst.getOperand(2).getImm();
   int Size = (int) Insn - Pos + 1;
-  Inst.addOperand(MCOperand::CreateImm(SignExtend32<16>(Size)));
+  Inst.addOperand(MCOperand::createImm(SignExtend32<16>(Size)));
   return MCDisassembler::Success;
 }
 
@@ -1705,19 +1741,19 @@ static DecodeStatus DecodeExtSize(MCInst &Inst,
                                   uint64_t Address,
                                   const void *Decoder) {
   int Size = (int) Insn  + 1;
-  Inst.addOperand(MCOperand::CreateImm(SignExtend32<16>(Size)));
+  Inst.addOperand(MCOperand::createImm(SignExtend32<16>(Size)));
   return MCDisassembler::Success;
 }
 
 static DecodeStatus DecodeSimm19Lsl2(MCInst &Inst, unsigned Insn,
                                      uint64_t Address, const void *Decoder) {
-  Inst.addOperand(MCOperand::CreateImm(SignExtend32<19>(Insn) * 4));
+  Inst.addOperand(MCOperand::createImm(SignExtend32<19>(Insn) * 4));
   return MCDisassembler::Success;
 }
 
 static DecodeStatus DecodeSimm18Lsl3(MCInst &Inst, unsigned Insn,
                                      uint64_t Address, const void *Decoder) {
-  Inst.addOperand(MCOperand::CreateImm(SignExtend32<18>(Insn) * 8));
+  Inst.addOperand(MCOperand::createImm(SignExtend32<18>(Insn) * 8));
   return MCDisassembler::Success;
 }
 
@@ -1731,7 +1767,7 @@ static DecodeStatus DecodeSimm9SP(MCInst &Inst, unsigned Insn,
   case 511: DecodedValue = -257; break;
   default: DecodedValue = SignExtend32<9>(Insn); break;
   }
-  Inst.addOperand(MCOperand::CreateImm(DecodedValue * 4));
+  Inst.addOperand(MCOperand::createImm(DecodedValue * 4));
   return MCDisassembler::Success;
 }
 
@@ -1741,13 +1777,13 @@ static DecodeStatus DecodeANDI16Imm(MCInst &Inst, unsigned Insn,
   assert(Insn < 16);
   int32_t DecodedValues[] = {128, 1, 2, 3, 4, 7, 8, 15, 16, 31, 32, 63, 64,
                              255, 32768, 65535};
-  Inst.addOperand(MCOperand::CreateImm(DecodedValues[Insn]));
+  Inst.addOperand(MCOperand::createImm(DecodedValues[Insn]));
   return MCDisassembler::Success;
 }
 
 static DecodeStatus DecodeUImm5lsl2(MCInst &Inst, unsigned Insn,
                                     uint64_t Address, const void *Decoder) {
-  Inst.addOperand(MCOperand::CreateImm(Insn << 2));
+  Inst.addOperand(MCOperand::createImm(Insn << 2));
   return MCDisassembler::Success;
 }
 
@@ -1766,10 +1802,10 @@ static DecodeStatus DecodeRegListOperand(MCInst &Inst,
 
   RegNum = RegLst & 0xf;
   for (unsigned i = 0; i < RegNum; i++)
-    Inst.addOperand(MCOperand::CreateReg(Regs[i]));
+    Inst.addOperand(MCOperand::createReg(Regs[i]));
 
   if (RegLst & 0x10)
-    Inst.addOperand(MCOperand::CreateReg(Mips::RA));
+    Inst.addOperand(MCOperand::createReg(Mips::RA));
 
   return MCDisassembler::Success;
 }
@@ -1782,9 +1818,9 @@ static DecodeStatus DecodeRegListOperand16(MCInst &Inst, unsigned Insn,
   unsigned RegNum = RegLst & 0x3;
 
   for (unsigned i = 0; i <= RegNum; i++)
-    Inst.addOperand(MCOperand::CreateReg(Regs[i]));
+    Inst.addOperand(MCOperand::createReg(Regs[i]));
 
-  Inst.addOperand(MCOperand::CreateReg(Mips::RA));
+  Inst.addOperand(MCOperand::createReg(Mips::RA));
 
   return MCDisassembler::Success;
 }
@@ -1798,36 +1834,36 @@ static DecodeStatus DecodeMovePRegPair(MCInst &Inst, unsigned Insn,
   default:
     return MCDisassembler::Fail;
   case 0:
-    Inst.addOperand(MCOperand::CreateReg(Mips::A1));
-    Inst.addOperand(MCOperand::CreateReg(Mips::A2));
+    Inst.addOperand(MCOperand::createReg(Mips::A1));
+    Inst.addOperand(MCOperand::createReg(Mips::A2));
     break;
   case 1:
-    Inst.addOperand(MCOperand::CreateReg(Mips::A1));
-    Inst.addOperand(MCOperand::CreateReg(Mips::A3));
+    Inst.addOperand(MCOperand::createReg(Mips::A1));
+    Inst.addOperand(MCOperand::createReg(Mips::A3));
     break;
   case 2:
-    Inst.addOperand(MCOperand::CreateReg(Mips::A2));
-    Inst.addOperand(MCOperand::CreateReg(Mips::A3));
+    Inst.addOperand(MCOperand::createReg(Mips::A2));
+    Inst.addOperand(MCOperand::createReg(Mips::A3));
     break;
   case 3:
-    Inst.addOperand(MCOperand::CreateReg(Mips::A0));
-    Inst.addOperand(MCOperand::CreateReg(Mips::S5));
+    Inst.addOperand(MCOperand::createReg(Mips::A0));
+    Inst.addOperand(MCOperand::createReg(Mips::S5));
     break;
   case 4:
-    Inst.addOperand(MCOperand::CreateReg(Mips::A0));
-    Inst.addOperand(MCOperand::CreateReg(Mips::S6));
+    Inst.addOperand(MCOperand::createReg(Mips::A0));
+    Inst.addOperand(MCOperand::createReg(Mips::S6));
     break;
   case 5:
-    Inst.addOperand(MCOperand::CreateReg(Mips::A0));
-    Inst.addOperand(MCOperand::CreateReg(Mips::A1));
+    Inst.addOperand(MCOperand::createReg(Mips::A0));
+    Inst.addOperand(MCOperand::createReg(Mips::A1));
     break;
   case 6:
-    Inst.addOperand(MCOperand::CreateReg(Mips::A0));
-    Inst.addOperand(MCOperand::CreateReg(Mips::A2));
+    Inst.addOperand(MCOperand::createReg(Mips::A0));
+    Inst.addOperand(MCOperand::createReg(Mips::A2));
     break;
   case 7:
-    Inst.addOperand(MCOperand::CreateReg(Mips::A0));
-    Inst.addOperand(MCOperand::CreateReg(Mips::A3));
+    Inst.addOperand(MCOperand::createReg(Mips::A0));
+    Inst.addOperand(MCOperand::createReg(Mips::A3));
     break;
   }
 
@@ -1836,6 +1872,6 @@ static DecodeStatus DecodeMovePRegPair(MCInst &Inst, unsigned Insn,
 
 static DecodeStatus DecodeSimm23Lsl2(MCInst &Inst, unsigned Insn,
                                      uint64_t Address, const void *Decoder) {
-  Inst.addOperand(MCOperand::CreateImm(SignExtend32<23>(Insn) << 2));
+  Inst.addOperand(MCOperand::createImm(SignExtend32<25>(Insn << 2)));
   return MCDisassembler::Success;
 }
diff --git a/lib/Target/Mips/InstPrinter/MipsInstPrinter.cpp b/lib/Target/Mips/InstPrinter/MipsInstPrinter.cpp
index e80a47b9014..a5637b16b63 100644
--- a/lib/Target/Mips/InstPrinter/MipsInstPrinter.cpp
+++ b/lib/Target/Mips/InstPrinter/MipsInstPrinter.cpp
@@ -122,7 +122,8 @@ void MipsInstPrinter::printInst(const MCInst *MI, raw_ostream &O,
   }
 }
 
-static void printExpr(const MCExpr *Expr, raw_ostream &OS) {
+static void printExpr(const MCExpr *Expr, const MCAsmInfo *MAI,
+                      raw_ostream &OS) {
   int Offset = 0;
   const MCSymbolRefExpr *SRE;
 
@@ -132,7 +133,7 @@ static void printExpr(const MCExpr *Expr, raw_ostream &OS) {
     assert(SRE && CE && "Binary expression must be sym+const.");
     Offset = CE->getValue();
   } else if (const MipsMCExpr *ME = dyn_cast<MipsMCExpr>(Expr)) {
-    ME->print(OS);
+    ME->print(OS, MAI);
     return;
   } else
     SRE = cast<MCSymbolRefExpr>(Expr);
@@ -170,7 +171,7 @@ static void printExpr(const MCExpr *Expr, raw_ostream &OS) {
   case MCSymbolRefExpr::VK_Mips_PCREL_LO16: OS << "%pcrel_lo("; break;
   }
 
-  OS << SRE->getSymbol();
+  SRE->getSymbol().print(OS, MAI);
 
   if (Offset) {
     if (Offset > 0)
@@ -199,7 +200,7 @@ void MipsInstPrinter::printOperand(const MCInst *MI, unsigned OpNo,
   }
 
   assert(Op.isExpr() && "unknown operand kind in printOperand");
-  printExpr(Op.getExpr(), O);
+  printExpr(Op.getExpr(), &MAI, O);
 }
 
 void MipsInstPrinter::printUnsignedImm(const MCInst *MI, int opNum,
diff --git a/lib/Target/Mips/LLVMBuild.txt b/lib/Target/Mips/LLVMBuild.txt
index 2f311e0e9b0..06af8a10a4d 100644
--- a/lib/Target/Mips/LLVMBuild.txt
+++ b/lib/Target/Mips/LLVMBuild.txt
@@ -31,5 +31,16 @@ has_jit = 1
 type = Library
 name = MipsCodeGen
 parent = Mips
-required_libraries = Analysis AsmPrinter CodeGen Core MC MipsAsmPrinter MipsDesc MipsInfo SelectionDAG Support Target NaClTransforms ;LOCALMOD
+required_libraries =
+ Analysis
+ AsmPrinter
+ CodeGen
+ Core
+ MC
+ MipsAsmPrinter
+ MipsDesc
+ MipsInfo
+ SelectionDAG
+ Support
+ Target
 add_to_library_groups = Mips
diff --git a/lib/Target/Mips/MCTargetDesc/MipsABIFlagsSection.cpp b/lib/Target/Mips/MCTargetDesc/MipsABIFlagsSection.cpp
index 5b0f950b076..70b9cca8cf6 100644
--- a/lib/Target/Mips/MCTargetDesc/MipsABIFlagsSection.cpp
+++ b/lib/Target/Mips/MCTargetDesc/MipsABIFlagsSection.cpp
@@ -14,15 +14,18 @@ using namespace llvm;
 uint8_t MipsABIFlagsSection::getFpABIValue() {
   switch (FpABI) {
   case FpABIKind::ANY:
-    return Val_GNU_MIPS_ABI_FP_ANY;
+    return Mips::Val_GNU_MIPS_ABI_FP_ANY;
+  case FpABIKind::SOFT:
+    return Mips::Val_GNU_MIPS_ABI_FP_SOFT;
   case FpABIKind::XX:
-    return Val_GNU_MIPS_ABI_FP_XX;
+    return Mips::Val_GNU_MIPS_ABI_FP_XX;
   case FpABIKind::S32:
-    return Val_GNU_MIPS_ABI_FP_DOUBLE;
+    return Mips::Val_GNU_MIPS_ABI_FP_DOUBLE;
   case FpABIKind::S64:
     if (Is32BitABI)
-      return OddSPReg ? Val_GNU_MIPS_ABI_FP_64 : Val_GNU_MIPS_ABI_FP_64A;
-    return Val_GNU_MIPS_ABI_FP_DOUBLE;
+      return OddSPReg ? Mips::Val_GNU_MIPS_ABI_FP_64
+                      : Mips::Val_GNU_MIPS_ABI_FP_64A;
+    return Mips::Val_GNU_MIPS_ABI_FP_DOUBLE;
   }
 
   llvm_unreachable("unexpected fp abi value");
@@ -43,7 +46,7 @@ StringRef MipsABIFlagsSection::getFpABIString(FpABIKind Value) {
 
 uint8_t MipsABIFlagsSection::getCPR1SizeValue() {
   if (FpABI == FpABIKind::XX)
-    return (uint8_t)AFL_REG_32;
+    return (uint8_t)Mips::AFL_REG_32;
   return (uint8_t)CPR1Size;
 }
 
diff --git a/lib/Target/Mips/MCTargetDesc/MipsABIFlagsSection.h b/lib/Target/Mips/MCTargetDesc/MipsABIFlagsSection.h
index 473f4f244d0..b078cd30a87 100644
--- a/lib/Target/Mips/MCTargetDesc/MipsABIFlagsSection.h
+++ b/lib/Target/Mips/MCTargetDesc/MipsABIFlagsSection.h
@@ -11,74 +11,16 @@
 #define LLVM_LIB_TARGET_MIPS_MCTARGETDESC_MIPSABIFLAGSSECTION_H
 
 #include "llvm/MC/MCStreamer.h"
+#include "llvm/Support/ErrorHandling.h"
+#include "llvm/Support/MipsABIFlags.h"
 
 namespace llvm {
 
 class MCStreamer;
 
 struct MipsABIFlagsSection {
-  // Values for the xxx_size bytes of an ABI flags structure.
-  enum AFL_REG {
-    AFL_REG_NONE = 0x00, // No registers.
-    AFL_REG_32 = 0x01,   // 32-bit registers.
-    AFL_REG_64 = 0x02,   // 64-bit registers.
-    AFL_REG_128 = 0x03   // 128-bit registers.
-  };
-
-  // Masks for the ases word of an ABI flags structure.
-  enum AFL_ASE {
-    AFL_ASE_DSP = 0x00000001,       // DSP ASE.
-    AFL_ASE_DSPR2 = 0x00000002,     // DSP R2 ASE.
-    AFL_ASE_EVA = 0x00000004,       // Enhanced VA Scheme.
-    AFL_ASE_MCU = 0x00000008,       // MCU (MicroController) ASE.
-    AFL_ASE_MDMX = 0x00000010,      // MDMX ASE.
-    AFL_ASE_MIPS3D = 0x00000020,    // MIPS-3D ASE.
-    AFL_ASE_MT = 0x00000040,        // MT ASE.
-    AFL_ASE_SMARTMIPS = 0x00000080, // SmartMIPS ASE.
-    AFL_ASE_VIRT = 0x00000100,      // VZ ASE.
-    AFL_ASE_MSA = 0x00000200,       // MSA ASE.
-    AFL_ASE_MIPS16 = 0x00000400,    // MIPS16 ASE.
-    AFL_ASE_MICROMIPS = 0x00000800, // MICROMIPS ASE.
-    AFL_ASE_XPA = 0x00001000        // XPA ASE.
-  };
-
-  // Values for the isa_ext word of an ABI flags structure.
-  enum AFL_EXT {
-    AFL_EXT_XLR = 1,          // RMI Xlr instruction.
-    AFL_EXT_OCTEON2 = 2,      // Cavium Networks Octeon2.
-    AFL_EXT_OCTEONP = 3,      // Cavium Networks OcteonP.
-    AFL_EXT_LOONGSON_3A = 4,  // Loongson 3A.
-    AFL_EXT_OCTEON = 5,       // Cavium Networks Octeon.
-    AFL_EXT_5900 = 6,         // MIPS R5900 instruction.
-    AFL_EXT_4650 = 7,         // MIPS R4650 instruction.
-    AFL_EXT_4010 = 8,         // LSI R4010 instruction.
-    AFL_EXT_4100 = 9,         // NEC VR4100 instruction.
-    AFL_EXT_3900 = 10,        // Toshiba R3900 instruction.
-    AFL_EXT_10000 = 11,       // MIPS R10000 instruction.
-    AFL_EXT_SB1 = 12,         // Broadcom SB-1 instruction.
-    AFL_EXT_4111 = 13,        // NEC VR4111/VR4181 instruction.
-    AFL_EXT_4120 = 14,        // NEC VR4120 instruction.
-    AFL_EXT_5400 = 15,        // NEC VR5400 instruction.
-    AFL_EXT_5500 = 16,        // NEC VR5500 instruction.
-    AFL_EXT_LOONGSON_2E = 17, // ST Microelectronics Loongson 2E.
-    AFL_EXT_LOONGSON_2F = 18  // ST Microelectronics Loongson 2F.
-  };
-
-  // Values for the fp_abi word of an ABI flags structure.
-  enum Val_GNU_MIPS_ABI {
-    Val_GNU_MIPS_ABI_FP_ANY = 0,
-    Val_GNU_MIPS_ABI_FP_DOUBLE = 1,
-    Val_GNU_MIPS_ABI_FP_XX = 5,
-    Val_GNU_MIPS_ABI_FP_64 = 6,
-    Val_GNU_MIPS_ABI_FP_64A = 7
-  };
-
-  enum AFL_FLAGS1 {
-    AFL_FLAGS1_ODDSPREG = 1
-  };
-
-  // Internal representation of the values used in .module fp=value
-  enum class FpABIKind { ANY, XX, S32, S64 };
+  // Internal representation of the fp_abi related values used in .module.
+  enum class FpABIKind { ANY, XX, S32, S64, SOFT };
 
   // Version of flags structure.
   uint16_t Version;
@@ -87,11 +29,11 @@ struct MipsABIFlagsSection {
   // The revision of ISA: 0 for MIPS V and below, 1-n otherwise.
   uint8_t ISARevision;
   // The size of general purpose registers.
-  AFL_REG GPRSize;
+  Mips::AFL_REG GPRSize;
   // The size of co-processor 1 registers.
-  AFL_REG CPR1Size;
+  Mips::AFL_REG CPR1Size;
   // The size of co-processor 2 registers.
-  AFL_REG CPR2Size;
+  Mips::AFL_REG CPR2Size;
   // Processor-specific extension.
   uint32_t ISAExtensionSet;
   // Mask of ASEs used.
@@ -107,9 +49,10 @@ struct MipsABIFlagsSection {
 
 public:
   MipsABIFlagsSection()
-      : Version(0), ISALevel(0), ISARevision(0), GPRSize(AFL_REG_NONE),
-        CPR1Size(AFL_REG_NONE), CPR2Size(AFL_REG_NONE), ISAExtensionSet(0),
-        ASESet(0), OddSPReg(false), Is32BitABI(false), FpABI(FpABIKind::ANY) {}
+      : Version(0), ISALevel(0), ISARevision(0), GPRSize(Mips::AFL_REG_NONE),
+        CPR1Size(Mips::AFL_REG_NONE), CPR2Size(Mips::AFL_REG_NONE),
+        ISAExtensionSet(0), ASESet(0), OddSPReg(false), Is32BitABI(false),
+        FpABI(FpABIKind::ANY) {}
 
   uint16_t getVersionValue() { return (uint16_t)Version; }
   uint8_t getISALevelValue() { return (uint8_t)ISALevel; }
@@ -125,7 +68,7 @@ struct MipsABIFlagsSection {
     uint32_t Value = 0;
 
     if (OddSPReg)
-      Value |= (uint32_t)AFL_FLAGS1_ODDSPREG;
+      Value |= (uint32_t)Mips::AFL_FLAGS1_ODDSPREG;
 
     return Value;
   }
@@ -184,32 +127,32 @@ struct MipsABIFlagsSection {
 
   template <class PredicateLibrary>
   void setGPRSizeFromPredicates(const PredicateLibrary &P) {
-    GPRSize = P.isGP64bit() ? AFL_REG_64 : AFL_REG_32;
+    GPRSize = P.isGP64bit() ? Mips::AFL_REG_64 : Mips::AFL_REG_32;
   }
 
   template <class PredicateLibrary>
   void setCPR1SizeFromPredicates(const PredicateLibrary &P) {
-    if (P.abiUsesSoftFloat())
-      CPR1Size = AFL_REG_NONE;
+    if (P.useSoftFloat())
+      CPR1Size = Mips::AFL_REG_NONE;
     else if (P.hasMSA())
-      CPR1Size = AFL_REG_128;
+      CPR1Size = Mips::AFL_REG_128;
     else
-      CPR1Size = P.isFP64bit() ? AFL_REG_64 : AFL_REG_32;
+      CPR1Size = P.isFP64bit() ? Mips::AFL_REG_64 : Mips::AFL_REG_32;
   }
 
   template <class PredicateLibrary>
   void setASESetFromPredicates(const PredicateLibrary &P) {
     ASESet = 0;
     if (P.hasDSP())
-      ASESet |= AFL_ASE_DSP;
+      ASESet |= Mips::AFL_ASE_DSP;
     if (P.hasDSPR2())
-      ASESet |= AFL_ASE_DSPR2;
+      ASESet |= Mips::AFL_ASE_DSPR2;
     if (P.hasMSA())
-      ASESet |= AFL_ASE_MSA;
+      ASESet |= Mips::AFL_ASE_MSA;
     if (P.inMicroMipsMode())
-      ASESet |= AFL_ASE_MICROMIPS;
+      ASESet |= Mips::AFL_ASE_MICROMIPS;
     if (P.inMips16Mode())
-      ASESet |= AFL_ASE_MIPS16;
+      ASESet |= Mips::AFL_ASE_MIPS16;
   }
 
   template <class PredicateLibrary>
@@ -217,7 +160,9 @@ struct MipsABIFlagsSection {
     Is32BitABI = P.isABI_O32();
 
     FpABI = FpABIKind::ANY;
-    if (P.isABI_N32() || P.isABI_N64())
+    if (P.useSoftFloat())
+      FpABI = FpABIKind::SOFT;
+    else if (P.isABI_N32() || P.isABI_N64())
       FpABI = FpABIKind::S64;
     else if (P.isABI_O32()) {
       if (P.isABI_FPXX())
diff --git a/lib/Target/Mips/MCTargetDesc/MipsABIInfo.cpp b/lib/Target/Mips/MCTargetDesc/MipsABIInfo.cpp
index faf974198ca..8e6c9e69b22 100644
--- a/lib/Target/Mips/MCTargetDesc/MipsABIInfo.cpp
+++ b/lib/Target/Mips/MCTargetDesc/MipsABIInfo.cpp
@@ -47,7 +47,7 @@ unsigned MipsABIInfo::GetCalleeAllocdArgSizeInBytes(CallingConv::ID CC) const {
   llvm_unreachable("Unhandled ABI");
 }
 
-MipsABIInfo MipsABIInfo::computeTargetABI(Triple TT, StringRef CPU,
+MipsABIInfo MipsABIInfo::computeTargetABI(const Triple &TT, StringRef CPU,
                                           const MCTargetOptions &Options) {
   if (Options.getABIName().startswith("o32"))
     return MipsABIInfo::O32();
@@ -90,3 +90,39 @@ MipsABIInfo MipsABIInfo::computeTargetABI(Triple TT, StringRef CPU,
       .Case("octeon", MipsABIInfo::N64())
       .Default(MipsABIInfo::Unknown());
 }
+
+unsigned MipsABIInfo::GetStackPtr() const {
+  return ArePtrs64bit() ? Mips::SP_64 : Mips::SP;
+}
+
+unsigned MipsABIInfo::GetFramePtr() const {
+  return ArePtrs64bit() ? Mips::FP_64 : Mips::FP;
+}
+
+unsigned MipsABIInfo::GetBasePtr() const {
+  return ArePtrs64bit() ? Mips::S7_64 : Mips::S7;
+}
+
+unsigned MipsABIInfo::GetNullPtr() const {
+  return ArePtrs64bit() ? Mips::ZERO_64 : Mips::ZERO;
+}
+
+unsigned MipsABIInfo::GetPtrAdduOp() const {
+  return ArePtrs64bit() ? Mips::DADDu : Mips::ADDu;
+}
+
+unsigned MipsABIInfo::GetPtrAddiuOp() const {
+  return ArePtrs64bit() ? Mips::DADDiu : Mips::ADDiu;
+}
+
+unsigned MipsABIInfo::GetEhDataReg(unsigned I) const {
+  static const unsigned EhDataReg[] = {
+    Mips::A0, Mips::A1, Mips::A2, Mips::A3
+  };
+  static const unsigned EhDataReg64[] = {
+    Mips::A0_64, Mips::A1_64, Mips::A2_64, Mips::A3_64
+  };
+
+  return IsN64() ? EhDataReg64[I] : EhDataReg[I];
+}
+
diff --git a/lib/Target/Mips/MCTargetDesc/MipsABIInfo.h b/lib/Target/Mips/MCTargetDesc/MipsABIInfo.h
index 008e08ecfde..40c5681acc1 100644
--- a/lib/Target/Mips/MCTargetDesc/MipsABIInfo.h
+++ b/lib/Target/Mips/MCTargetDesc/MipsABIInfo.h
@@ -19,6 +19,7 @@ namespace llvm {
 
 class MCTargetOptions;
 class StringRef;
+class TargetRegisterClass;
 
 class MipsABIInfo {
 public:
@@ -35,7 +36,7 @@ class MipsABIInfo {
   static MipsABIInfo N32() { return MipsABIInfo(ABI::N32); }
   static MipsABIInfo N64() { return MipsABIInfo(ABI::N64); }
   static MipsABIInfo EABI() { return MipsABIInfo(ABI::EABI); }
-  static MipsABIInfo computeTargetABI(Triple TT, StringRef CPU,
+  static MipsABIInfo computeTargetABI(const Triple &TT, StringRef CPU,
                                       const MCTargetOptions &Options);
 
   bool IsKnown() const { return ThisABI != ABI::Unknown; }
@@ -61,6 +62,16 @@ class MipsABIInfo {
   bool operator<(const MipsABIInfo Other) const {
     return ThisABI < Other.GetEnumValue();
   }
+
+  unsigned GetStackPtr() const;
+  unsigned GetFramePtr() const;
+  unsigned GetBasePtr() const;
+  unsigned GetNullPtr() const;
+  unsigned GetPtrAdduOp() const;
+  unsigned GetPtrAddiuOp() const;
+  inline bool ArePtrs64bit() const { return IsN64(); }
+
+  unsigned GetEhDataReg(unsigned I) const;
 };
 }
 
diff --git a/lib/Target/Mips/MCTargetDesc/MipsAsmBackend.cpp b/lib/Target/Mips/MCTargetDesc/MipsAsmBackend.cpp
index dbcd867b402..328e71720ca 100644
--- a/lib/Target/Mips/MCTargetDesc/MipsAsmBackend.cpp
+++ b/lib/Target/Mips/MCTargetDesc/MipsAsmBackend.cpp
@@ -59,23 +59,19 @@ static unsigned adjustFixupValue(const MCFixup &Fixup, uint64_t Value,
   case Mips::fixup_MIPS_PCLO16:
     break;
   case Mips::fixup_Mips_PC16:
-    // So far we are only using this type for branches.
-    // For branches we start 1 instruction after the branch
-    // so the displacement will be one instruction size less.
-    Value -= 4;
     // The displacement is then divided by 4 to give us an 18 bit
     // address range. Forcing a signed division because Value can be negative.
     Value = (int64_t)Value / 4;
     // We now check if Value can be encoded as a 16-bit signed immediate.
     if (!isIntN(16, Value) && Ctx)
-      Ctx->FatalError(Fixup.getLoc(), "out of range PC16 fixup");
+      Ctx->reportFatalError(Fixup.getLoc(), "out of range PC16 fixup");
     break;
   case Mips::fixup_MIPS_PC19_S2:
     // Forcing a signed division because Value can be negative.
     Value = (int64_t)Value / 4;
     // We now check if Value can be encoded as a 19-bit signed immediate.
     if (!isIntN(19, Value) && Ctx)
-      Ctx->FatalError(Fixup.getLoc(), "out of range PC19 fixup");
+      Ctx->reportFatalError(Fixup.getLoc(), "out of range PC19 fixup");
     break;
   case Mips::fixup_Mips_26:
     // So far we are only using this type for jumps.
@@ -109,7 +105,7 @@ static unsigned adjustFixupValue(const MCFixup &Fixup, uint64_t Value,
     Value = (int64_t) Value / 2;
     // We now check if Value can be encoded as a 7-bit signed immediate.
     if (!isIntN(7, Value) && Ctx)
-      Ctx->FatalError(Fixup.getLoc(), "out of range PC7 fixup");
+      Ctx->reportFatalError(Fixup.getLoc(), "out of range PC7 fixup");
     break;
   case Mips::fixup_MICROMIPS_PC10_S1:
     Value -= 2;
@@ -117,7 +113,7 @@ static unsigned adjustFixupValue(const MCFixup &Fixup, uint64_t Value,
     Value = (int64_t) Value / 2;
     // We now check if Value can be encoded as a 10-bit signed immediate.
     if (!isIntN(10, Value) && Ctx)
-      Ctx->FatalError(Fixup.getLoc(), "out of range PC10 fixup");
+      Ctx->reportFatalError(Fixup.getLoc(), "out of range PC10 fixup");
     break;
   case Mips::fixup_MICROMIPS_PC16_S1:
     Value -= 4;
@@ -125,30 +121,28 @@ static unsigned adjustFixupValue(const MCFixup &Fixup, uint64_t Value,
     Value = (int64_t)Value / 2;
     // We now check if Value can be encoded as a 16-bit signed immediate.
     if (!isIntN(16, Value) && Ctx)
-      Ctx->FatalError(Fixup.getLoc(), "out of range PC16 fixup");
+      Ctx->reportFatalError(Fixup.getLoc(), "out of range PC16 fixup");
     break;
   case Mips::fixup_MIPS_PC18_S3:
     // Forcing a signed division because Value can be negative.
     Value = (int64_t)Value / 8;
     // We now check if Value can be encoded as a 18-bit signed immediate.
     if (!isIntN(18, Value) && Ctx)
-      Ctx->FatalError(Fixup.getLoc(), "out of range PC18 fixup");
+      Ctx->reportFatalError(Fixup.getLoc(), "out of range PC18 fixup");
     break;
   case Mips::fixup_MIPS_PC21_S2:
-    Value -= 4;
     // Forcing a signed division because Value can be negative.
     Value = (int64_t) Value / 4;
     // We now check if Value can be encoded as a 21-bit signed immediate.
     if (!isIntN(21, Value) && Ctx)
-      Ctx->FatalError(Fixup.getLoc(), "out of range PC21 fixup");
+      Ctx->reportFatalError(Fixup.getLoc(), "out of range PC21 fixup");
     break;
   case Mips::fixup_MIPS_PC26_S2:
-    Value -= 4;
     // Forcing a signed division because Value can be negative.
     Value = (int64_t) Value / 4;
     // We now check if Value can be encoded as a 26-bit signed immediate.
     if (!isIntN(26, Value) && Ctx)
-      Ctx->FatalError(Fixup.getLoc(), "out of range PC26 fixup");
+      Ctx->reportFatalError(Fixup.getLoc(), "out of range PC26 fixup");
     break;
   }
 
@@ -394,12 +388,7 @@ bool MipsAsmBackend::writeNopData(uint64_t Count, MCObjectWriter *OW) const {
   // If the count is not 4-byte aligned, we must be writing data into the text
   // section (otherwise we have unaligned instructions, and thus have far
   // bigger problems), so just write zeros instead.
-  for (uint64_t i = 0, e = Count % 4; i != e; ++i)
-    OW->Write8(0);
-
-  uint64_t NumNops = Count / 4;
-  for (uint64_t i = 0; i != NumNops; ++i)
-    OW->Write32(0);
+  OW->WriteZeros(Count);
   return true;
 }
 
@@ -422,32 +411,27 @@ void MipsAsmBackend::processFixupValue(const MCAssembler &Asm,
 // MCAsmBackend
 MCAsmBackend *llvm::createMipsAsmBackendEL32(const Target &T,
                                              const MCRegisterInfo &MRI,
-                                             StringRef TT,
-                                             StringRef CPU) {
-  return new MipsAsmBackend(T, Triple(TT).getOS(),
-                            /*IsLittle*/true, /*Is64Bit*/false);
+                                             const Triple &TT, StringRef CPU) {
+  return new MipsAsmBackend(T, TT.getOS(), /*IsLittle*/ true,
+                            /*Is64Bit*/ false);
 }
 
 MCAsmBackend *llvm::createMipsAsmBackendEB32(const Target &T,
                                              const MCRegisterInfo &MRI,
-                                             StringRef TT,
-                                             StringRef CPU) {
-  return new MipsAsmBackend(T, Triple(TT).getOS(),
-                            /*IsLittle*/false, /*Is64Bit*/false);
+                                             const Triple &TT, StringRef CPU) {
+  return new MipsAsmBackend(T, TT.getOS(), /*IsLittle*/ false,
+                            /*Is64Bit*/ false);
 }
 
 MCAsmBackend *llvm::createMipsAsmBackendEL64(const Target &T,
                                              const MCRegisterInfo &MRI,
-                                             StringRef TT,
-                                             StringRef CPU) {
-  return new MipsAsmBackend(T, Triple(TT).getOS(),
-                            /*IsLittle*/true, /*Is64Bit*/true);
+                                             const Triple &TT, StringRef CPU) {
+  return new MipsAsmBackend(T, TT.getOS(), /*IsLittle*/ true, /*Is64Bit*/ true);
 }
 
 MCAsmBackend *llvm::createMipsAsmBackendEB64(const Target &T,
                                              const MCRegisterInfo &MRI,
-                                             StringRef TT,
-                                             StringRef CPU) {
-  return new MipsAsmBackend(T, Triple(TT).getOS(),
-                            /*IsLittle*/false, /*Is64Bit*/true);
+                                             const Triple &TT, StringRef CPU) {
+  return new MipsAsmBackend(T, TT.getOS(), /*IsLittle*/ false,
+                            /*Is64Bit*/ true);
 }
diff --git a/lib/Target/Mips/MCTargetDesc/MipsELFObjectWriter.cpp b/lib/Target/Mips/MCTargetDesc/MipsELFObjectWriter.cpp
index 8d9e3e31105..9b2952720ed 100644
--- a/lib/Target/Mips/MCTargetDesc/MipsELFObjectWriter.cpp
+++ b/lib/Target/Mips/MCTargetDesc/MipsELFObjectWriter.cpp
@@ -12,10 +12,10 @@
 #include "MCTargetDesc/MipsMCTargetDesc.h"
 #include "llvm/ADT/STLExtras.h"
 #include "llvm/MC/MCAssembler.h"
-#include "llvm/MC/MCELF.h"
 #include "llvm/MC/MCELFObjectWriter.h"
 #include "llvm/MC/MCExpr.h"
 #include "llvm/MC/MCSection.h"
+#include "llvm/MC/MCSymbolELF.h"
 #include "llvm/MC/MCValue.h"
 #include "llvm/Support/ErrorHandling.h"
 #include <list>
@@ -46,7 +46,7 @@ struct MipsRelocationEntry {
 
     unsigned GetRelocType(const MCValue &Target, const MCFixup &Fixup,
                           bool IsPCRel) const override;
-    bool needsRelocateWithSymbol(const MCSymbolData &SD,
+    bool needsRelocateWithSymbol(const MCSymbol &Sym,
                                  unsigned Type) const override;
     virtual void sortRelocs(const MCAssembler &Asm,
                             std::vector<ELFRelocationEntry> &Relocs) override;
@@ -64,182 +64,149 @@ MipsELFObjectWriter::~MipsELFObjectWriter() {}
 unsigned MipsELFObjectWriter::GetRelocType(const MCValue &Target,
                                            const MCFixup &Fixup,
                                            bool IsPCRel) const {
-  // determine the type of the relocation
-  unsigned Type = (unsigned)ELF::R_MIPS_NONE;
+  // Determine the type of the relocation.
   unsigned Kind = (unsigned)Fixup.getKind();
 
   switch (Kind) {
-  default:
-    llvm_unreachable("invalid fixup kind!");
+  case Mips::fixup_Mips_16:
+  case FK_Data_2:
+    return IsPCRel ? ELF::R_MIPS_PC16 : ELF::R_MIPS_16;
   case Mips::fixup_Mips_32:
   case FK_Data_4:
-    Type = ELF::R_MIPS_32;
-    break;
+    return IsPCRel ? ELF::R_MIPS_PC32 : ELF::R_MIPS_32;
+  }
+
+  if (IsPCRel) {
+    switch (Kind) {
+    case Mips::fixup_Mips_Branch_PCRel:
+    case Mips::fixup_Mips_PC16:
+      return ELF::R_MIPS_PC16;
+    case Mips::fixup_MICROMIPS_PC7_S1:
+      return ELF::R_MICROMIPS_PC7_S1;
+    case Mips::fixup_MICROMIPS_PC10_S1:
+      return ELF::R_MICROMIPS_PC10_S1;
+    case Mips::fixup_MICROMIPS_PC16_S1:
+      return ELF::R_MICROMIPS_PC16_S1;
+    case Mips::fixup_MIPS_PC19_S2:
+      return ELF::R_MIPS_PC19_S2;
+    case Mips::fixup_MIPS_PC18_S3:
+      return ELF::R_MIPS_PC18_S3;
+    case Mips::fixup_MIPS_PC21_S2:
+      return ELF::R_MIPS_PC21_S2;
+    case Mips::fixup_MIPS_PC26_S2:
+      return ELF::R_MIPS_PC26_S2;
+    case Mips::fixup_MIPS_PCHI16:
+      return ELF::R_MIPS_PCHI16;
+    case Mips::fixup_MIPS_PCLO16:
+      return ELF::R_MIPS_PCLO16;
+    }
+
+    llvm_unreachable("invalid PC-relative fixup kind!");
+  }
+
+  switch (Kind) {
   case Mips::fixup_Mips_64:
   case FK_Data_8:
-    Type = ELF::R_MIPS_64;
-    break;
+    return ELF::R_MIPS_64;
   case FK_GPRel_4:
     if (isN64()) {
+      unsigned Type = (unsigned)ELF::R_MIPS_NONE;
       Type = setRType((unsigned)ELF::R_MIPS_GPREL32, Type);
       Type = setRType2((unsigned)ELF::R_MIPS_64, Type);
       Type = setRType3((unsigned)ELF::R_MIPS_NONE, Type);
+      return Type;
     }
-    else
-      Type = ELF::R_MIPS_GPREL32;
-    break;
+    return ELF::R_MIPS_GPREL32;
   case Mips::fixup_Mips_GPREL16:
-    Type = ELF::R_MIPS_GPREL16;
-    break;
+    return ELF::R_MIPS_GPREL16;
   case Mips::fixup_Mips_26:
-    Type = ELF::R_MIPS_26;
-    break;
+    return ELF::R_MIPS_26;
   case Mips::fixup_Mips_CALL16:
-    Type = ELF::R_MIPS_CALL16;
-    break;
+    return ELF::R_MIPS_CALL16;
   case Mips::fixup_Mips_GOT_Global:
   case Mips::fixup_Mips_GOT_Local:
-    Type = ELF::R_MIPS_GOT16;
-    break;
+    return ELF::R_MIPS_GOT16;
   case Mips::fixup_Mips_HI16:
-    Type = ELF::R_MIPS_HI16;
-    break;
+    return ELF::R_MIPS_HI16;
   case Mips::fixup_Mips_LO16:
-    Type = ELF::R_MIPS_LO16;
-    break;
+    return ELF::R_MIPS_LO16;
   case Mips::fixup_Mips_TLSGD:
-    Type = ELF::R_MIPS_TLS_GD;
-    break;
+    return ELF::R_MIPS_TLS_GD;
   case Mips::fixup_Mips_GOTTPREL:
-    Type = ELF::R_MIPS_TLS_GOTTPREL;
-    break;
+    return ELF::R_MIPS_TLS_GOTTPREL;
   case Mips::fixup_Mips_TPREL_HI:
-    Type = ELF::R_MIPS_TLS_TPREL_HI16;
-    break;
+    return ELF::R_MIPS_TLS_TPREL_HI16;
   case Mips::fixup_Mips_TPREL_LO:
-    Type = ELF::R_MIPS_TLS_TPREL_LO16;
-    break;
+    return ELF::R_MIPS_TLS_TPREL_LO16;
   case Mips::fixup_Mips_TLSLDM:
-    Type = ELF::R_MIPS_TLS_LDM;
-    break;
+    return ELF::R_MIPS_TLS_LDM;
   case Mips::fixup_Mips_DTPREL_HI:
-    Type = ELF::R_MIPS_TLS_DTPREL_HI16;
-    break;
+    return ELF::R_MIPS_TLS_DTPREL_HI16;
   case Mips::fixup_Mips_DTPREL_LO:
-    Type = ELF::R_MIPS_TLS_DTPREL_LO16;
-    break;
-  case Mips::fixup_Mips_Branch_PCRel:
-  case Mips::fixup_Mips_PC16:
-    Type = ELF::R_MIPS_PC16;
-    break;
+    return ELF::R_MIPS_TLS_DTPREL_LO16;
   case Mips::fixup_Mips_GOT_PAGE:
-    Type = ELF::R_MIPS_GOT_PAGE;
-    break;
+    return ELF::R_MIPS_GOT_PAGE;
   case Mips::fixup_Mips_GOT_OFST:
-    Type = ELF::R_MIPS_GOT_OFST;
-    break;
+    return ELF::R_MIPS_GOT_OFST;
   case Mips::fixup_Mips_GOT_DISP:
-    Type = ELF::R_MIPS_GOT_DISP;
-    break;
-  case Mips::fixup_Mips_GPOFF_HI:
+    return ELF::R_MIPS_GOT_DISP;
+  case Mips::fixup_Mips_GPOFF_HI: {
+    unsigned Type = (unsigned)ELF::R_MIPS_NONE;
     Type = setRType((unsigned)ELF::R_MIPS_GPREL16, Type);
     Type = setRType2((unsigned)ELF::R_MIPS_SUB, Type);
     Type = setRType3((unsigned)ELF::R_MIPS_HI16, Type);
-    break;
-  case Mips::fixup_Mips_GPOFF_LO:
+    return Type;
+  }
+  case Mips::fixup_Mips_GPOFF_LO: {
+    unsigned Type = (unsigned)ELF::R_MIPS_NONE;
     Type = setRType((unsigned)ELF::R_MIPS_GPREL16, Type);
     Type = setRType2((unsigned)ELF::R_MIPS_SUB, Type);
     Type = setRType3((unsigned)ELF::R_MIPS_LO16, Type);
-    break;
+    return Type;
+  }
   case Mips::fixup_Mips_HIGHER:
-    Type = ELF::R_MIPS_HIGHER;
-    break;
+    return ELF::R_MIPS_HIGHER;
   case Mips::fixup_Mips_HIGHEST:
-    Type = ELF::R_MIPS_HIGHEST;
-    break;
+    return ELF::R_MIPS_HIGHEST;
   case Mips::fixup_Mips_GOT_HI16:
-    Type = ELF::R_MIPS_GOT_HI16;
-    break;
+    return ELF::R_MIPS_GOT_HI16;
   case Mips::fixup_Mips_GOT_LO16:
-    Type = ELF::R_MIPS_GOT_LO16;
-    break;
+    return ELF::R_MIPS_GOT_LO16;
   case Mips::fixup_Mips_CALL_HI16:
-    Type = ELF::R_MIPS_CALL_HI16;
-    break;
+    return ELF::R_MIPS_CALL_HI16;
   case Mips::fixup_Mips_CALL_LO16:
-    Type = ELF::R_MIPS_CALL_LO16;
-    break;
+    return ELF::R_MIPS_CALL_LO16;
   case Mips::fixup_MICROMIPS_26_S1:
-    Type = ELF::R_MICROMIPS_26_S1;
-    break;
+    return ELF::R_MICROMIPS_26_S1;
   case Mips::fixup_MICROMIPS_HI16:
-    Type = ELF::R_MICROMIPS_HI16;
-    break;
+    return ELF::R_MICROMIPS_HI16;
   case Mips::fixup_MICROMIPS_LO16:
-    Type = ELF::R_MICROMIPS_LO16;
-    break;
+    return ELF::R_MICROMIPS_LO16;
   case Mips::fixup_MICROMIPS_GOT16:
-    Type = ELF::R_MICROMIPS_GOT16;
-    break;
-  case Mips::fixup_MICROMIPS_PC7_S1:
-    Type = ELF::R_MICROMIPS_PC7_S1;
-    break;
-  case Mips::fixup_MICROMIPS_PC10_S1:
-    Type = ELF::R_MICROMIPS_PC10_S1;
-    break;
-  case Mips::fixup_MICROMIPS_PC16_S1:
-    Type = ELF::R_MICROMIPS_PC16_S1;
-    break;
+    return ELF::R_MICROMIPS_GOT16;
   case Mips::fixup_MICROMIPS_CALL16:
-    Type = ELF::R_MICROMIPS_CALL16;
-    break;
+    return ELF::R_MICROMIPS_CALL16;
   case Mips::fixup_MICROMIPS_GOT_DISP:
-    Type = ELF::R_MICROMIPS_GOT_DISP;
-    break;
+    return ELF::R_MICROMIPS_GOT_DISP;
   case Mips::fixup_MICROMIPS_GOT_PAGE:
-    Type = ELF::R_MICROMIPS_GOT_PAGE;
-    break;
+    return ELF::R_MICROMIPS_GOT_PAGE;
   case Mips::fixup_MICROMIPS_GOT_OFST:
-    Type = ELF::R_MICROMIPS_GOT_OFST;
-    break;
+    return ELF::R_MICROMIPS_GOT_OFST;
   case Mips::fixup_MICROMIPS_TLS_GD:
-    Type = ELF::R_MICROMIPS_TLS_GD;
-    break;
+    return ELF::R_MICROMIPS_TLS_GD;
   case Mips::fixup_MICROMIPS_TLS_LDM:
-    Type = ELF::R_MICROMIPS_TLS_LDM;
-    break;
+    return ELF::R_MICROMIPS_TLS_LDM;
   case Mips::fixup_MICROMIPS_TLS_DTPREL_HI16:
-    Type = ELF::R_MICROMIPS_TLS_DTPREL_HI16;
-    break;
+    return ELF::R_MICROMIPS_TLS_DTPREL_HI16;
   case Mips::fixup_MICROMIPS_TLS_DTPREL_LO16:
-    Type = ELF::R_MICROMIPS_TLS_DTPREL_LO16;
-    break;
+    return ELF::R_MICROMIPS_TLS_DTPREL_LO16;
   case Mips::fixup_MICROMIPS_TLS_TPREL_HI16:
-    Type = ELF::R_MICROMIPS_TLS_TPREL_HI16;
-    break;
+    return ELF::R_MICROMIPS_TLS_TPREL_HI16;
   case Mips::fixup_MICROMIPS_TLS_TPREL_LO16:
-    Type = ELF::R_MICROMIPS_TLS_TPREL_LO16;
-    break;
-  case Mips::fixup_MIPS_PC19_S2:
-    Type = ELF::R_MIPS_PC19_S2;
-    break;
-  case Mips::fixup_MIPS_PC18_S3:
-    Type = ELF::R_MIPS_PC18_S3;
-    break;
-  case Mips::fixup_MIPS_PC21_S2:
-    Type = ELF::R_MIPS_PC21_S2;
-    break;
-  case Mips::fixup_MIPS_PC26_S2:
-    Type = ELF::R_MIPS_PC26_S2;
-    break;
-  case Mips::fixup_MIPS_PCHI16:
-    Type = ELF::R_MIPS_PCHI16;
-    break;
-  case Mips::fixup_MIPS_PCLO16:
-    Type = ELF::R_MIPS_PCLO16;
-    break;
+    return ELF::R_MICROMIPS_TLS_TPREL_LO16;
   }
-  return Type;
+
+  llvm_unreachable("invalid fixup kind!");
 }
 
 // Sort entries by SortOffset in descending order.
@@ -271,9 +238,7 @@ static unsigned getMatchingLoType(const MCAssembler &Asm,
   if (Type == ELF::R_MIPS16_HI16)
     return ELF::R_MIPS16_LO16;
 
-  const MCSymbolData &SD = Asm.getSymbolData(*Reloc.Symbol);
-
-  if (MCELF::GetBinding(SD) != ELF::STB_LOCAL)
+  if (Reloc.Symbol->getBinding() != ELF::STB_LOCAL)
     return ELF::R_MIPS_NONE;
 
   if (Type == ELF::R_MIPS_GOT16)
@@ -405,9 +370,8 @@ void MipsELFObjectWriter::sortRelocs(const MCAssembler &Asm,
     Relocs[I] = MipsRelocs[I].R;
 }
 
-bool
-MipsELFObjectWriter::needsRelocateWithSymbol(const MCSymbolData &SD,
-                                             unsigned Type) const {
+bool MipsELFObjectWriter::needsRelocateWithSymbol(const MCSymbol &Sym,
+                                                  unsigned Type) const {
   // FIXME: This is extremely conservative. This really needs to use a
   // whitelist with a clear explanation for why each realocation needs to
   // point to the symbol, not to the section.
@@ -434,7 +398,7 @@ MipsELFObjectWriter::needsRelocateWithSymbol(const MCSymbolData &SD,
     return true;
 
   case ELF::R_MIPS_32:
-    if (MCELF::getOther(SD) & (ELF::STO_MIPS_MICROMIPS >> 2))
+    if (cast<MCSymbolELF>(Sym).getOther() & ELF::STO_MIPS_MICROMIPS)
       return true;
     // falltrough
   case ELF::R_MIPS_26:
diff --git a/lib/Target/Mips/MCTargetDesc/MipsELFStreamer.cpp b/lib/Target/Mips/MCTargetDesc/MipsELFStreamer.cpp
index 6d1d9f4f223..b45d9cf621d 100644
--- a/lib/Target/Mips/MCTargetDesc/MipsELFStreamer.cpp
+++ b/lib/Target/Mips/MCTargetDesc/MipsELFStreamer.cpp
@@ -9,8 +9,8 @@
 
 #include "MipsELFStreamer.h"
 #include "MipsTargetStreamer.h"
-#include "llvm/MC/MCELF.h"
 #include "llvm/MC/MCInst.h"
+#include "llvm/MC/MCSymbolELF.h"
 #include "llvm/Support/ELF.h"
 
 using namespace llvm;
@@ -41,12 +41,10 @@ void MipsELFStreamer::createPendingLabelRelocs() {
 
   // FIXME: Also mark labels when in MIPS16 mode.
   if (ELFTargetStreamer->isMicroMipsEnabled()) {
-    for (auto Label : Labels) {
-      MCSymbolData &Data = getOrCreateSymbolData(Label);
-      // The "other" values are stored in the last 6 bits of the second byte.
-      // The traditional defines for STO values assume the full byte and thus
-      // the shift to pack it.
-      MCELF::setOther(Data, ELF::STO_MIPS_MICROMIPS >> 2);
+    for (auto *L : Labels) {
+      auto *Label = cast<MCSymbolELF>(L);
+      getAssembler().registerSymbol(*Label);
+      Label->setOther(ELF::STO_MIPS_MICROMIPS);
     }
   }
 
@@ -58,7 +56,7 @@ void MipsELFStreamer::EmitLabel(MCSymbol *Symbol) {
   Labels.push_back(Symbol);
 }
 
-void MipsELFStreamer::SwitchSection(const MCSection * Section,
+void MipsELFStreamer::SwitchSection(MCSection *Section,
                                     const MCExpr *Subsection) {
   MCELFStreamer::SwitchSection(Section, Subsection);
   Labels.clear();
diff --git a/lib/Target/Mips/MCTargetDesc/MipsELFStreamer.h b/lib/Target/Mips/MCTargetDesc/MipsELFStreamer.h
index 4e3090171e3..af9311fa428 100644
--- a/lib/Target/Mips/MCTargetDesc/MipsELFStreamer.h
+++ b/lib/Target/Mips/MCTargetDesc/MipsELFStreamer.h
@@ -55,7 +55,7 @@ class MipsELFStreamer : public MCELFStreamer {
 
   /// Overriding this function allows us to dismiss all labels that are
   /// candidates for marking as microMIPS when .section directive is processed.
-  void SwitchSection(const MCSection *Section,
+  void SwitchSection(MCSection *Section,
                      const MCExpr *Subsection = nullptr) override;
 
   /// Overriding this function allows us to dismiss all labels that are
diff --git a/lib/Target/Mips/MCTargetDesc/MipsMCAsmInfo.cpp b/lib/Target/Mips/MCTargetDesc/MipsMCAsmInfo.cpp
index e2bd5a815ab..4d554583dc7 100644
--- a/lib/Target/Mips/MCTargetDesc/MipsMCAsmInfo.cpp
+++ b/lib/Target/Mips/MCTargetDesc/MipsMCAsmInfo.cpp
@@ -18,8 +18,7 @@ using namespace llvm;
 
 void MipsMCAsmInfo::anchor() { }
 
-MipsMCAsmInfo::MipsMCAsmInfo(StringRef TT) {
-  Triple TheTriple(TT);
+MipsMCAsmInfo::MipsMCAsmInfo(const Triple &TheTriple) {
   if ((TheTriple.getArch() == Triple::mips) ||
       (TheTriple.getArch() == Triple::mips64))
     IsLittleEndian = false;
diff --git a/lib/Target/Mips/MCTargetDesc/MipsMCAsmInfo.h b/lib/Target/Mips/MCTargetDesc/MipsMCAsmInfo.h
index 59ff1c41ed6..5d23fcbd7a4 100644
--- a/lib/Target/Mips/MCTargetDesc/MipsMCAsmInfo.h
+++ b/lib/Target/Mips/MCTargetDesc/MipsMCAsmInfo.h
@@ -17,12 +17,12 @@
 #include "llvm/MC/MCAsmInfoELF.h"
 
 namespace llvm {
-  class StringRef;
+  class Triple;
 
   class MipsMCAsmInfo : public MCAsmInfoELF {
     void anchor() override;
   public:
-    explicit MipsMCAsmInfo(StringRef TT);
+    explicit MipsMCAsmInfo(const Triple &TheTriple);
   };
 
 } // namespace llvm
diff --git a/lib/Target/Mips/MCTargetDesc/MipsMCCodeEmitter.cpp b/lib/Target/Mips/MCTargetDesc/MipsMCCodeEmitter.cpp
index 1c2f2da6c96..e36263d54ca 100644
--- a/lib/Target/Mips/MCTargetDesc/MipsMCCodeEmitter.cpp
+++ b/lib/Target/Mips/MCTargetDesc/MipsMCCodeEmitter.cpp
@@ -23,6 +23,7 @@
 #include "llvm/MC/MCFixup.h"
 #include "llvm/MC/MCInst.h"
 #include "llvm/MC/MCInstrInfo.h"
+#include "llvm/MC/MCRegisterInfo.h"
 #include "llvm/MC/MCSubtargetInfo.h"
 #include "llvm/Support/raw_ostream.h"
 
@@ -112,7 +113,11 @@ static void LowerDextDins(MCInst& InstIn) {
 }
 
 bool MipsMCCodeEmitter::isMicroMips(const MCSubtargetInfo &STI) const {
-  return STI.getFeatureBits() & Mips::FeatureMicroMips;
+  return STI.getFeatureBits()[Mips::FeatureMicroMips];
+}
+
+bool MipsMCCodeEmitter::isMips32r6(const MCSubtargetInfo &STI) const {
+  return STI.getFeatureBits()[Mips::FeatureMips32r6];
 }
 
 void MipsMCCodeEmitter::EmitByte(unsigned char C, raw_ostream &OS) const {
@@ -137,10 +142,10 @@ void MipsMCCodeEmitter::EmitInstruction(uint64_t Val, unsigned Size,
   }
 }
 
-/// EncodeInstruction - Emit the instruction.
+/// encodeInstruction - Emit the instruction.
 /// Size the instruction with Desc.getSize().
 void MipsMCCodeEmitter::
-EncodeInstruction(const MCInst &MI, raw_ostream &OS,
+encodeInstruction(const MCInst &MI, raw_ostream &OS,
                   SmallVectorImpl<MCFixup> &Fixups,
                   const MCSubtargetInfo &STI) const
 {
@@ -173,13 +178,22 @@ EncodeInstruction(const MCInst &MI, raw_ostream &OS,
   unsigned Opcode = TmpInst.getOpcode();
   if ((Opcode != Mips::NOP) && (Opcode != Mips::SLL) &&
       (Opcode != Mips::SLL_MM) && !Binary)
-    llvm_unreachable("unimplemented opcode in EncodeInstruction()");
+    llvm_unreachable("unimplemented opcode in encodeInstruction()");
+
+  int NewOpcode = -1;
+  if (isMicroMips(STI)) {
+    if (isMips32r6(STI)) {
+      NewOpcode = Mips::MipsR62MicroMipsR6(Opcode, Mips::Arch_micromipsr6);
+      if (NewOpcode == -1)
+        NewOpcode = Mips::Std2MicroMipsR6(Opcode, Mips::Arch_micromipsr6);
+    }
+    else
+      NewOpcode = Mips::Std2MicroMips(Opcode, Mips::Arch_micromips);
 
-  if (STI.getFeatureBits() & Mips::FeatureMicroMips) {
-    int NewOpcode = Mips::Std2MicroMips (Opcode, Mips::Arch_micromips);
     if (NewOpcode != -1) {
       if (Fixups.size() > N)
         Fixups.pop_back();
+
       Opcode = NewOpcode;
       TmpInst.setOpcode (NewOpcode);
       Binary = getBinaryCodeForInstr(TmpInst, Fixups, STI);
@@ -212,8 +226,9 @@ getBranchTargetOpValue(const MCInst &MI, unsigned OpNo,
   assert(MO.isExpr() &&
          "getBranchTargetOpValue expects only expressions or immediates");
 
-  const MCExpr *Expr = MO.getExpr();
-  Fixups.push_back(MCFixup::Create(0, Expr,
+  const MCExpr *FixupExpression = MCBinaryExpr::createAdd(
+      MO.getExpr(), MCConstantExpr::create(-4, Ctx), Ctx);
+  Fixups.push_back(MCFixup::create(0, FixupExpression,
                                    MCFixupKind(Mips::fixup_Mips_PC16)));
   return 0;
 }
@@ -235,7 +250,7 @@ getBranchTarget7OpValueMM(const MCInst &MI, unsigned OpNo,
          "getBranchTargetOpValueMM expects only expressions or immediates");
 
   const MCExpr *Expr = MO.getExpr();
-  Fixups.push_back(MCFixup::Create(0, Expr,
+  Fixups.push_back(MCFixup::create(0, Expr,
                                    MCFixupKind(Mips::fixup_MICROMIPS_PC7_S1)));
   return 0;
 }
@@ -257,7 +272,7 @@ getBranchTargetOpValueMMPC10(const MCInst &MI, unsigned OpNo,
          "getBranchTargetOpValuePC10 expects only expressions or immediates");
 
   const MCExpr *Expr = MO.getExpr();
-  Fixups.push_back(MCFixup::Create(0, Expr,
+  Fixups.push_back(MCFixup::create(0, Expr,
                    MCFixupKind(Mips::fixup_MICROMIPS_PC10_S1)));
   return 0;
 }
@@ -279,7 +294,7 @@ getBranchTargetOpValueMM(const MCInst &MI, unsigned OpNo,
          "getBranchTargetOpValueMM expects only expressions or immediates");
 
   const MCExpr *Expr = MO.getExpr();
-  Fixups.push_back(MCFixup::Create(0, Expr,
+  Fixups.push_back(MCFixup::create(0, Expr,
                    MCFixupKind(Mips::
                                fixup_MICROMIPS_PC16_S1)));
   return 0;
@@ -301,8 +316,9 @@ getBranchTarget21OpValue(const MCInst &MI, unsigned OpNo,
   assert(MO.isExpr() &&
          "getBranchTarget21OpValue expects only expressions or immediates");
 
-  const MCExpr *Expr = MO.getExpr();
-  Fixups.push_back(MCFixup::Create(0, Expr,
+  const MCExpr *FixupExpression = MCBinaryExpr::createAdd(
+      MO.getExpr(), MCConstantExpr::create(-4, Ctx), Ctx);
+  Fixups.push_back(MCFixup::create(0, FixupExpression,
                                    MCFixupKind(Mips::fixup_MIPS_PC21_S2)));
   return 0;
 }
@@ -323,8 +339,9 @@ getBranchTarget26OpValue(const MCInst &MI, unsigned OpNo,
   assert(MO.isExpr() &&
          "getBranchTarget26OpValue expects only expressions or immediates");
 
-  const MCExpr *Expr = MO.getExpr();
-  Fixups.push_back(MCFixup::Create(0, Expr,
+  const MCExpr *FixupExpression = MCBinaryExpr::createAdd(
+      MO.getExpr(), MCConstantExpr::create(-4, Ctx), Ctx);
+  Fixups.push_back(MCFixup::create(0, FixupExpression,
                                    MCFixupKind(Mips::fixup_MIPS_PC26_S2)));
   return 0;
 }
@@ -364,7 +381,7 @@ getJumpTargetOpValue(const MCInst &MI, unsigned OpNo,
          "getJumpTargetOpValue expects only expressions or an immediate");
 
   const MCExpr *Expr = MO.getExpr();
-  Fixups.push_back(MCFixup::Create(0, Expr,
+  Fixups.push_back(MCFixup::create(0, Expr,
                                    MCFixupKind(Mips::fixup_Mips_26)));
   return 0;
 }
@@ -382,7 +399,7 @@ getJumpTargetOpValueMM(const MCInst &MI, unsigned OpNo,
          "getJumpTargetOpValueMM expects only expressions or an immediate");
 
   const MCExpr *Expr = MO.getExpr();
-  Fixups.push_back(MCFixup::Create(0, Expr,
+  Fixups.push_back(MCFixup::create(0, Expr,
                                    MCFixupKind(Mips::fixup_MICROMIPS_26_S1)));
   return 0;
 }
@@ -453,7 +470,7 @@ getExprOpValue(const MCExpr *Expr, SmallVectorImpl<MCFixup> &Fixups,
                const MCSubtargetInfo &STI) const {
   int64_t Res;
 
-  if (Expr->EvaluateAsAbsolute(Res))
+  if (Expr->evaluateAsAbsolute(Res))
     return Res;
 
   MCExpr::ExprKind Kind = Expr->getKind();
@@ -488,7 +505,7 @@ getExprOpValue(const MCExpr *Expr, SmallVectorImpl<MCFixup> &Fixups,
                                    : Mips::fixup_Mips_LO16;
       break;
     }
-    Fixups.push_back(MCFixup::Create(0, MipsExpr, MCFixupKind(FixupKind)));
+    Fixups.push_back(MCFixup::create(0, MipsExpr, MCFixupKind(FixupKind)));
     return 0;
   }
 
@@ -595,7 +612,7 @@ getExprOpValue(const MCExpr *Expr, SmallVectorImpl<MCFixup> &Fixups,
       break;
     } // switch
 
-    Fixups.push_back(MCFixup::Create(0, Expr, MCFixupKind(FixupKind)));
+    Fixups.push_back(MCFixup::create(0, Expr, MCFixupKind(FixupKind)));
     return 0;
   }
   return 0;
@@ -846,7 +863,7 @@ MipsMCCodeEmitter::getSimm19Lsl2Encoding(const MCInst &MI, unsigned OpNo,
          "getSimm19Lsl2Encoding expects only expressions or an immediate");
 
   const MCExpr *Expr = MO.getExpr();
-  Fixups.push_back(MCFixup::Create(0, Expr,
+  Fixups.push_back(MCFixup::create(0, Expr,
                                    MCFixupKind(Mips::fixup_MIPS_PC19_S2)));
   return 0;
 }
@@ -867,7 +884,7 @@ MipsMCCodeEmitter::getSimm18Lsl3Encoding(const MCInst &MI, unsigned OpNo,
          "getSimm18Lsl2Encoding expects only expressions or an immediate");
 
   const MCExpr *Expr = MO.getExpr();
-  Fixups.push_back(MCFixup::Create(0, Expr,
+  Fixups.push_back(MCFixup::create(0, Expr,
                                    MCFixupKind(Mips::fixup_MIPS_PC18_S3)));
   return 0;
 }
diff --git a/lib/Target/Mips/MCTargetDesc/MipsMCCodeEmitter.h b/lib/Target/Mips/MCTargetDesc/MipsMCCodeEmitter.h
index cc40e2e789a..911cc2f77a4 100644
--- a/lib/Target/Mips/MCTargetDesc/MipsMCCodeEmitter.h
+++ b/lib/Target/Mips/MCTargetDesc/MipsMCCodeEmitter.h
@@ -38,6 +38,7 @@ class MipsMCCodeEmitter : public MCCodeEmitter {
   bool IsLittleEndian;
 
   bool isMicroMips(const MCSubtargetInfo &STI) const;
+  bool isMips32r6(const MCSubtargetInfo &STI) const;
 
 public:
   MipsMCCodeEmitter(const MCInstrInfo &mcii, MCContext &Ctx_, bool IsLittle)
@@ -50,7 +51,7 @@ class MipsMCCodeEmitter : public MCCodeEmitter {
   void EmitInstruction(uint64_t Val, unsigned Size, const MCSubtargetInfo &STI,
                        raw_ostream &OS) const;
 
-  void EncodeInstruction(const MCInst &MI, raw_ostream &OS,
+  void encodeInstruction(const MCInst &MI, raw_ostream &OS,
                          SmallVectorImpl<MCFixup> &Fixups,
                          const MCSubtargetInfo &STI) const override;
 
diff --git a/lib/Target/Mips/MCTargetDesc/MipsMCExpr.cpp b/lib/Target/Mips/MCTargetDesc/MipsMCExpr.cpp
index 74490f334b3..c85fc4816b0 100644
--- a/lib/Target/Mips/MCTargetDesc/MipsMCExpr.cpp
+++ b/lib/Target/Mips/MCTargetDesc/MipsMCExpr.cpp
@@ -41,7 +41,7 @@ bool MipsMCExpr::isSupportedBinaryExpr(MCSymbolRefExpr::VariantKind VK,
 }
 
 const MipsMCExpr*
-MipsMCExpr::Create(MCSymbolRefExpr::VariantKind VK, const MCExpr *Expr,
+MipsMCExpr::create(MCSymbolRefExpr::VariantKind VK, const MCExpr *Expr,
                    MCContext &Ctx) {
   VariantKind Kind;
   switch (VK) {
@@ -64,7 +64,7 @@ MipsMCExpr::Create(MCSymbolRefExpr::VariantKind VK, const MCExpr *Expr,
   return new (Ctx) MipsMCExpr(Kind, Expr);
 }
 
-void MipsMCExpr::PrintImpl(raw_ostream &OS) const {
+void MipsMCExpr::printImpl(raw_ostream &OS, const MCAsmInfo *MAI) const {
   switch (Kind) {
   default: llvm_unreachable("Invalid kind!");
   case VK_Mips_LO: OS << "%lo"; break;
@@ -74,15 +74,15 @@ void MipsMCExpr::PrintImpl(raw_ostream &OS) const {
   }
 
   OS << '(';
-  Expr->print(OS);
+  Expr->print(OS, MAI);
   OS << ')';
 }
 
 bool
-MipsMCExpr::EvaluateAsRelocatableImpl(MCValue &Res,
+MipsMCExpr::evaluateAsRelocatableImpl(MCValue &Res,
                                       const MCAsmLayout *Layout,
                                       const MCFixup *Fixup) const {
-  return getSubExpr()->EvaluateAsRelocatable(Res, Layout, Fixup);
+  return getSubExpr()->evaluateAsRelocatable(Res, Layout, Fixup);
 }
 
 void MipsMCExpr::visitUsedExpr(MCStreamer &Streamer) const {
diff --git a/lib/Target/Mips/MCTargetDesc/MipsMCExpr.h b/lib/Target/Mips/MCTargetDesc/MipsMCExpr.h
index 2b8f0c89a2e..fd2ed17ee78 100644
--- a/lib/Target/Mips/MCTargetDesc/MipsMCExpr.h
+++ b/lib/Target/Mips/MCTargetDesc/MipsMCExpr.h
@@ -37,7 +37,7 @@ class MipsMCExpr : public MCTargetExpr {
   static bool isSupportedBinaryExpr(MCSymbolRefExpr::VariantKind VK,
                                     const MCBinaryExpr *BE);
 
-  static const MipsMCExpr *Create(MCSymbolRefExpr::VariantKind VK,
+  static const MipsMCExpr *create(MCSymbolRefExpr::VariantKind VK,
                                   const MCExpr *Expr, MCContext &Ctx);
 
   /// getOpcode - Get the kind of this expression.
@@ -46,13 +46,13 @@ class MipsMCExpr : public MCTargetExpr {
   /// getSubExpr - Get the child of this expression.
   const MCExpr *getSubExpr() const { return Expr; }
 
-  void PrintImpl(raw_ostream &OS) const override;
-  bool EvaluateAsRelocatableImpl(MCValue &Res,
+  void printImpl(raw_ostream &OS, const MCAsmInfo *MAI) const override;
+  bool evaluateAsRelocatableImpl(MCValue &Res,
                                  const MCAsmLayout *Layout,
                                  const MCFixup *Fixup) const override;
   void visitUsedExpr(MCStreamer &Streamer) const override;
-  const MCSection *FindAssociatedSection() const override {
-    return getSubExpr()->FindAssociatedSection();
+  MCSection *findAssociatedSection() const override {
+    return getSubExpr()->findAssociatedSection();
   }
 
   // There are no TLS MipsMCExprs at the moment.
diff --git a/lib/Target/Mips/MCTargetDesc/MipsMCTargetDesc.cpp b/lib/Target/Mips/MCTargetDesc/MipsMCTargetDesc.cpp
index a75d27d4b21..949ee1474f9 100644
--- a/lib/Target/Mips/MCTargetDesc/MipsMCTargetDesc.cpp
+++ b/lib/Target/Mips/MCTargetDesc/MipsMCTargetDesc.cpp
@@ -43,11 +43,9 @@ using namespace llvm;
 
 /// Select the Mips CPU for the given triple and cpu name.
 /// FIXME: Merge with the copy in MipsSubtarget.cpp
-StringRef MIPS_MC::selectMipsCPU(StringRef TT, StringRef CPU) {
+StringRef MIPS_MC::selectMipsCPU(const Triple &TT, StringRef CPU) {
   if (CPU.empty() || CPU == "generic") {
-    Triple TheTriple(TT);
-    if (TheTriple.getArch() == Triple::mips ||
-        TheTriple.getArch() == Triple::mipsel)
+    if (TT.getArch() == Triple::mips || TT.getArch() == Triple::mipsel)
       CPU = "mips32";
     else
       CPU = "mips64";
@@ -61,21 +59,20 @@ static MCInstrInfo *createMipsMCInstrInfo() {
   return X;
 }
 
-static MCRegisterInfo *createMipsMCRegisterInfo(StringRef TT) {
+static MCRegisterInfo *createMipsMCRegisterInfo(const Triple &TT) {
   MCRegisterInfo *X = new MCRegisterInfo();
   InitMipsMCRegisterInfo(X, Mips::RA);
   return X;
 }
 
-static MCSubtargetInfo *createMipsMCSubtargetInfo(StringRef TT, StringRef CPU,
-                                                  StringRef FS) {
+static MCSubtargetInfo *createMipsMCSubtargetInfo(const Triple &TT,
+                                                  StringRef CPU, StringRef FS) {
   CPU = MIPS_MC::selectMipsCPU(TT, CPU);
-  MCSubtargetInfo *X = new MCSubtargetInfo();
-  InitMipsMCSubtargetInfo(X, TT, CPU, FS);
-  return X;
+  return createMipsMCSubtargetInfoImpl(TT, CPU, FS);
 }
 
-static MCAsmInfo *createMipsMCAsmInfo(const MCRegisterInfo &MRI, StringRef TT) {
+static MCAsmInfo *createMipsMCAsmInfo(const MCRegisterInfo &MRI,
+                                      const Triple &TT) {
   MCAsmInfo *MAI = new MipsMCAsmInfo(TT);
 
   unsigned SP = MRI.getDwarfRegNum(Mips::SP, true);
@@ -85,7 +82,7 @@ static MCAsmInfo *createMipsMCAsmInfo(const MCRegisterInfo &MRI, StringRef TT) {
   return MAI;
 }
 
-static MCCodeGenInfo *createMipsMCCodeGenInfo(StringRef TT, Reloc::Model RM,
+static MCCodeGenInfo *createMipsMCCodeGenInfo(const Triple &TT, Reloc::Model RM,
                                               CodeModel::Model CM,
                                               CodeGenOpt::Level OL) {
   MCCodeGenInfo *X = new MCCodeGenInfo();
@@ -93,7 +90,7 @@ static MCCodeGenInfo *createMipsMCCodeGenInfo(StringRef TT, Reloc::Model RM,
     RM = Reloc::Static;
   else if (RM == Reloc::Default)
     RM = Reloc::PIC_;
-  X->InitMCCodeGenInfo(RM, CM, OL);
+  X->initMCCodeGenInfo(RM, CM, OL);
   return X;
 }
 
diff --git a/lib/Target/Mips/MCTargetDesc/MipsMCTargetDesc.h b/lib/Target/Mips/MCTargetDesc/MipsMCTargetDesc.h
index 577a8b3ea3b..4069d7d184e 100644
--- a/lib/Target/Mips/MCTargetDesc/MipsMCTargetDesc.h
+++ b/lib/Target/Mips/MCTargetDesc/MipsMCTargetDesc.h
@@ -26,6 +26,7 @@ class MCRegisterInfo;
 class MCSubtargetInfo;
 class StringRef;
 class Target;
+class Triple;
 class raw_ostream;
 class raw_pwrite_stream;
 
@@ -42,23 +43,23 @@ MCCodeEmitter *createMipsMCCodeEmitterEL(const MCInstrInfo &MCII,
                                          MCContext &Ctx);
 
 MCAsmBackend *createMipsAsmBackendEB32(const Target &T,
-                                       const MCRegisterInfo &MRI, StringRef TT,
-                                       StringRef CPU);
+                                       const MCRegisterInfo &MRI,
+                                       const Triple &TT, StringRef CPU);
 MCAsmBackend *createMipsAsmBackendEL32(const Target &T,
-                                       const MCRegisterInfo &MRI, StringRef TT,
-                                       StringRef CPU);
+                                       const MCRegisterInfo &MRI,
+                                       const Triple &TT, StringRef CPU);
 MCAsmBackend *createMipsAsmBackendEB64(const Target &T,
-                                       const MCRegisterInfo &MRI, StringRef TT,
-                                       StringRef CPU);
+                                       const MCRegisterInfo &MRI,
+                                       const Triple &TT, StringRef CPU);
 MCAsmBackend *createMipsAsmBackendEL64(const Target &T,
-                                       const MCRegisterInfo &MRI, StringRef TT,
-                                       StringRef CPU);
+                                       const MCRegisterInfo &MRI,
+                                       const Triple &TT, StringRef CPU);
 
 MCObjectWriter *createMipsELFObjectWriter(raw_pwrite_stream &OS, uint8_t OSABI,
                                           bool IsLittleEndian, bool Is64Bit);
 
 namespace MIPS_MC {
-StringRef selectMipsCPU(StringRef TT, StringRef CPU);
+StringRef selectMipsCPU(const Triple &TT, StringRef CPU);
 }
 
 } // End llvm namespace
diff --git a/lib/Target/Mips/MCTargetDesc/MipsNaClELFStreamer.cpp b/lib/Target/Mips/MCTargetDesc/MipsNaClELFStreamer.cpp
index 35348aa9e35..aef9bd3a8e2 100644
--- a/lib/Target/Mips/MCTargetDesc/MipsNaClELFStreamer.cpp
+++ b/lib/Target/Mips/MCTargetDesc/MipsNaClELFStreamer.cpp
@@ -94,9 +94,9 @@ class MipsNaClELFStreamer : public MipsELFStreamer {
                 const MCSubtargetInfo &STI) {
     MCInst MaskInst;
     MaskInst.setOpcode(Mips::AND);
-    MaskInst.addOperand(MCOperand::CreateReg(AddrReg));
-    MaskInst.addOperand(MCOperand::CreateReg(AddrReg));
-    MaskInst.addOperand(MCOperand::CreateReg(MaskReg));
+    MaskInst.addOperand(MCOperand::createReg(AddrReg));
+    MaskInst.addOperand(MCOperand::createReg(AddrReg));
+    MaskInst.addOperand(MCOperand::createReg(MaskReg));
     MipsELFStreamer::EmitInstruction(MaskInst, STI);
   }
 
diff --git a/lib/Target/Mips/MCTargetDesc/MipsOptionRecord.cpp b/lib/Target/Mips/MCTargetDesc/MipsOptionRecord.cpp
index 188e3e8e1db..24b602810d6 100644
--- a/lib/Target/Mips/MCTargetDesc/MipsOptionRecord.cpp
+++ b/lib/Target/Mips/MCTargetDesc/MipsOptionRecord.cpp
@@ -28,10 +28,11 @@ void MipsRegInfoRecord::EmitMipsOptionRecord() {
   if (MTS->getABI().IsN64()) {
     // The EntrySize value of 1 seems strange since the records are neither
     // 1-byte long nor fixed length but it matches the value GAS emits.
-    const MCSectionELF *Sec =
+    MCSectionELF *Sec =
         Context.getELFSection(".MIPS.options", ELF::SHT_MIPS_OPTIONS,
                               ELF::SHF_ALLOC | ELF::SHF_MIPS_NOSTRIP, 1, "");
-    MCA.getOrCreateSectionData(*Sec).setAlignment(8);
+    MCA.registerSection(*Sec);
+    Sec->setAlignment(8);
     Streamer->SwitchSection(Sec);
 
     Streamer->EmitIntValue(ELF::ODK_REGINFO, 1);  // kind
@@ -46,10 +47,10 @@ void MipsRegInfoRecord::EmitMipsOptionRecord() {
     Streamer->EmitIntValue(ri_cprmask[3], 4);
     Streamer->EmitIntValue(ri_gp_value, 8);
   } else {
-    const MCSectionELF *Sec = Context.getELFSection(
-        ".reginfo", ELF::SHT_MIPS_REGINFO, ELF::SHF_ALLOC, 24, "");
-    MCA.getOrCreateSectionData(*Sec)
-        .setAlignment(MTS->getABI().IsN32() ? 8 : 4);
+    MCSectionELF *Sec = Context.getELFSection(".reginfo", ELF::SHT_MIPS_REGINFO,
+                                              ELF::SHF_ALLOC, 24, "");
+    MCA.registerSection(*Sec);
+    Sec->setAlignment(MTS->getABI().IsN32() ? 8 : 4);
     Streamer->SwitchSection(Sec);
 
     Streamer->EmitIntValue(ri_gprmask, 4);
@@ -78,6 +79,9 @@ void MipsRegInfoRecord::SetPhysRegUsed(unsigned Reg,
     if (GPR32RegClass->contains(CurrentSubReg) ||
         GPR64RegClass->contains(CurrentSubReg))
       ri_gprmask |= Value;
+    else if (COP0RegClass->contains(CurrentSubReg))
+      ri_cprmask[0] |= Value;
+    // MIPS COP1 is the FPU.
     else if (FGR32RegClass->contains(CurrentSubReg) ||
              FGR64RegClass->contains(CurrentSubReg) ||
              AFGR64RegClass->contains(CurrentSubReg) ||
diff --git a/lib/Target/Mips/MCTargetDesc/MipsTargetStreamer.cpp b/lib/Target/Mips/MCTargetDesc/MipsTargetStreamer.cpp
index cfd56c6a5cd..e4da2df75d4 100644
--- a/lib/Target/Mips/MCTargetDesc/MipsTargetStreamer.cpp
+++ b/lib/Target/Mips/MCTargetDesc/MipsTargetStreamer.cpp
@@ -17,10 +17,9 @@
 #include "MipsTargetObjectFile.h"
 #include "MipsTargetStreamer.h"
 #include "llvm/MC/MCContext.h"
-#include "llvm/MC/MCELF.h"
 #include "llvm/MC/MCSectionELF.h"
 #include "llvm/MC/MCSubtargetInfo.h"
-#include "llvm/MC/MCSymbol.h"
+#include "llvm/MC/MCSymbolELF.h"
 #include "llvm/Support/CommandLine.h"
 #include "llvm/Support/ELF.h"
 #include "llvm/Support/ErrorHandling.h"
@@ -81,21 +80,35 @@ void MipsTargetStreamer::emitDirectiveSetMips64R5() { forbidModuleDirective(); }
 void MipsTargetStreamer::emitDirectiveSetMips64R6() { forbidModuleDirective(); }
 void MipsTargetStreamer::emitDirectiveSetPop() { forbidModuleDirective(); }
 void MipsTargetStreamer::emitDirectiveSetPush() { forbidModuleDirective(); }
+void MipsTargetStreamer::emitDirectiveSetSoftFloat() {
+  forbidModuleDirective();
+}
+void MipsTargetStreamer::emitDirectiveSetHardFloat() {
+  forbidModuleDirective();
+}
 void MipsTargetStreamer::emitDirectiveSetDsp() { forbidModuleDirective(); }
 void MipsTargetStreamer::emitDirectiveSetNoDsp() { forbidModuleDirective(); }
 void MipsTargetStreamer::emitDirectiveCpLoad(unsigned RegNo) {}
 void MipsTargetStreamer::emitDirectiveCpsetup(unsigned RegNo, int RegOrOffset,
                                               const MCSymbol &Sym, bool IsReg) {
 }
-void MipsTargetStreamer::emitDirectiveModuleOddSPReg(bool Enabled,
-                                                     bool IsO32ABI) {
-  if (!Enabled && !IsO32ABI)
+
+void MipsTargetStreamer::emitDirectiveModuleFP() {}
+
+void MipsTargetStreamer::emitDirectiveModuleOddSPReg() {
+  if (!ABIFlagsSection.OddSPReg && !ABIFlagsSection.Is32BitABI)
     report_fatal_error("+nooddspreg is only valid for O32");
 }
+void MipsTargetStreamer::emitDirectiveModuleSoftFloat() {}
+void MipsTargetStreamer::emitDirectiveModuleHardFloat() {}
 void MipsTargetStreamer::emitDirectiveSetFp(
     MipsABIFlagsSection::FpABIKind Value) {
   forbidModuleDirective();
 }
+void MipsTargetStreamer::emitDirectiveSetOddSPReg() { forbidModuleDirective(); }
+void MipsTargetStreamer::emitDirectiveSetNoOddSPReg() {
+  forbidModuleDirective();
+}
 
 MipsTargetAsmStreamer::MipsTargetAsmStreamer(MCStreamer &S,
                                              formatted_raw_ostream &OS)
@@ -308,6 +321,16 @@ void MipsTargetAsmStreamer::emitDirectiveSetPush() {
  MipsTargetStreamer::emitDirectiveSetPush();
 }
 
+void MipsTargetAsmStreamer::emitDirectiveSetSoftFloat() {
+  OS << "\t.set\tsoftfloat\n";
+  MipsTargetStreamer::emitDirectiveSetSoftFloat();
+}
+
+void MipsTargetAsmStreamer::emitDirectiveSetHardFloat() {
+  OS << "\t.set\thardfloat\n";
+  MipsTargetStreamer::emitDirectiveSetHardFloat();
+}
+
 // Print a 32 bit hex number with all numbers.
 static void printHex32(unsigned Value, raw_ostream &OS) {
   OS << "0x";
@@ -354,29 +377,41 @@ void MipsTargetAsmStreamer::emitDirectiveCpsetup(unsigned RegNo,
   forbidModuleDirective();
 }
 
-void MipsTargetAsmStreamer::emitDirectiveModuleFP(
-    MipsABIFlagsSection::FpABIKind Value, bool Is32BitABI) {
-  MipsTargetStreamer::emitDirectiveModuleFP(Value, Is32BitABI);
-
-  StringRef ModuleValue;
+void MipsTargetAsmStreamer::emitDirectiveModuleFP() {
   OS << "\t.module\tfp=";
-  OS << ABIFlagsSection.getFpABIString(Value) << "\n";
+  OS << ABIFlagsSection.getFpABIString(ABIFlagsSection.getFpABI()) << "\n";
 }
 
 void MipsTargetAsmStreamer::emitDirectiveSetFp(
     MipsABIFlagsSection::FpABIKind Value) {
   MipsTargetStreamer::emitDirectiveSetFp(Value);
 
-  StringRef ModuleValue;
   OS << "\t.set\tfp=";
   OS << ABIFlagsSection.getFpABIString(Value) << "\n";
 }
 
-void MipsTargetAsmStreamer::emitDirectiveModuleOddSPReg(bool Enabled,
-                                                        bool IsO32ABI) {
-  MipsTargetStreamer::emitDirectiveModuleOddSPReg(Enabled, IsO32ABI);
+void MipsTargetAsmStreamer::emitDirectiveModuleOddSPReg() {
+  MipsTargetStreamer::emitDirectiveModuleOddSPReg();
+
+  OS << "\t.module\t" << (ABIFlagsSection.OddSPReg ? "" : "no") << "oddspreg\n";
+}
+
+void MipsTargetAsmStreamer::emitDirectiveSetOddSPReg() {
+  MipsTargetStreamer::emitDirectiveSetOddSPReg();
+  OS << "\t.set\toddspreg\n";
+}
+
+void MipsTargetAsmStreamer::emitDirectiveSetNoOddSPReg() {
+  MipsTargetStreamer::emitDirectiveSetNoOddSPReg();
+  OS << "\t.set\tnooddspreg\n";
+}
+
+void MipsTargetAsmStreamer::emitDirectiveModuleSoftFloat() {
+  OS << "\t.module\tsoftfloat\n";
+}
 
-  OS << "\t.module\t" << (Enabled ? "" : "no") << "oddspreg\n";
+void MipsTargetAsmStreamer::emitDirectiveModuleHardFloat() {
+  OS << "\t.module\thardfloat\n";
 }
 
 // This part is for ELF object output.
@@ -386,7 +421,7 @@ MipsTargetELFStreamer::MipsTargetELFStreamer(MCStreamer &S,
   MCAssembler &MCA = getStreamer().getAssembler();
   Pic = MCA.getContext().getObjectFileInfo()->getRelocM() == Reloc::PIC_;
 
-  uint64_t Features = STI.getFeatureBits();
+  const FeatureBitset &Features = STI.getFeatureBits();
 
   // Set the header flags that we can in the constructor.
   // FIXME: This is a fairly terrible hack. We set the rest
@@ -402,35 +437,35 @@ MipsTargetELFStreamer::MipsTargetELFStreamer(MCStreamer &S,
   unsigned EFlags = MCA.getELFHeaderEFlags();
 
   // Architecture
-  if (Features & Mips::FeatureMips64r6)
+  if (Features[Mips::FeatureMips64r6])
     EFlags |= ELF::EF_MIPS_ARCH_64R6;
-  else if (Features & Mips::FeatureMips64r2 ||
-           Features & Mips::FeatureMips64r3 ||
-           Features & Mips::FeatureMips64r5)
+  else if (Features[Mips::FeatureMips64r2] ||
+           Features[Mips::FeatureMips64r3] ||
+           Features[Mips::FeatureMips64r5])
     EFlags |= ELF::EF_MIPS_ARCH_64R2;
-  else if (Features & Mips::FeatureMips64)
+  else if (Features[Mips::FeatureMips64])
     EFlags |= ELF::EF_MIPS_ARCH_64;
-  else if (Features & Mips::FeatureMips5)
+  else if (Features[Mips::FeatureMips5])
     EFlags |= ELF::EF_MIPS_ARCH_5;
-  else if (Features & Mips::FeatureMips4)
+  else if (Features[Mips::FeatureMips4])
     EFlags |= ELF::EF_MIPS_ARCH_4;
-  else if (Features & Mips::FeatureMips3)
+  else if (Features[Mips::FeatureMips3])
     EFlags |= ELF::EF_MIPS_ARCH_3;
-  else if (Features & Mips::FeatureMips32r6)
+  else if (Features[Mips::FeatureMips32r6])
     EFlags |= ELF::EF_MIPS_ARCH_32R6;
-  else if (Features & Mips::FeatureMips32r2 ||
-           Features & Mips::FeatureMips32r3 ||
-           Features & Mips::FeatureMips32r5)
+  else if (Features[Mips::FeatureMips32r2] ||
+           Features[Mips::FeatureMips32r3] ||
+           Features[Mips::FeatureMips32r5])
     EFlags |= ELF::EF_MIPS_ARCH_32R2;
-  else if (Features & Mips::FeatureMips32)
+  else if (Features[Mips::FeatureMips32])
     EFlags |= ELF::EF_MIPS_ARCH_32;
-  else if (Features & Mips::FeatureMips2)
+  else if (Features[Mips::FeatureMips2])
     EFlags |= ELF::EF_MIPS_ARCH_2;
   else
     EFlags |= ELF::EF_MIPS_ARCH_1;
 
   // Other options.
-  if (Features & Mips::FeatureNaN2008)
+  if (Features[Mips::FeatureNaN2008])
     EFlags |= ELF::EF_MIPS_NAN2008;
 
   // -mabicalls and -mplt are not implemented but we should act as if they were
@@ -440,18 +475,16 @@ MipsTargetELFStreamer::MipsTargetELFStreamer(MCStreamer &S,
   MCA.setELFHeaderEFlags(EFlags);
 }
 
-void MipsTargetELFStreamer::emitLabel(MCSymbol *Symbol) {
+void MipsTargetELFStreamer::emitLabel(MCSymbol *S) {
+  auto *Symbol = cast<MCSymbolELF>(S);
   if (!isMicroMipsEnabled())
     return;
-  MCSymbolData &Data = getStreamer().getOrCreateSymbolData(Symbol);
-  uint8_t Type = MCELF::GetType(Data);
+  getStreamer().getAssembler().registerSymbol(*Symbol);
+  uint8_t Type = Symbol->getType();
   if (Type != ELF::STT_FUNC)
     return;
 
-  // The "other" values are stored in the last 6 bits of the second byte
-  // The traditional defines for STO values assume the full byte and thus
-  // the shift to pack it.
-  MCELF::setOther(Data, ELF::STO_MIPS_MICROMIPS >> 2);
+  Symbol->setOther(ELF::STO_MIPS_MICROMIPS);
 }
 
 void MipsTargetELFStreamer::finish() {
@@ -459,18 +492,18 @@ void MipsTargetELFStreamer::finish() {
   const MCObjectFileInfo &OFI = *MCA.getContext().getObjectFileInfo();
 
   // .bss, .text and .data are always at least 16-byte aligned.
-  MCSectionData &TextSectionData =
-      MCA.getOrCreateSectionData(*OFI.getTextSection());
-  MCSectionData &DataSectionData =
-      MCA.getOrCreateSectionData(*OFI.getDataSection());
-  MCSectionData &BSSSectionData =
-      MCA.getOrCreateSectionData(*OFI.getBSSSection());
+  MCSection &TextSection = *OFI.getTextSection();
+  MCA.registerSection(TextSection);
+  MCSection &DataSection = *OFI.getDataSection();
+  MCA.registerSection(DataSection);
+  MCSection &BSSSection = *OFI.getBSSSection();
+  MCA.registerSection(BSSSection);
 
-  TextSectionData.setAlignment(std::max(16u, TextSectionData.getAlignment()));
-  DataSectionData.setAlignment(std::max(16u, DataSectionData.getAlignment()));
-  BSSSectionData.setAlignment(std::max(16u, BSSSectionData.getAlignment()));
+  TextSection.setAlignment(std::max(16u, TextSection.getAlignment()));
+  DataSection.setAlignment(std::max(16u, DataSection.getAlignment()));
+  BSSSection.setAlignment(std::max(16u, BSSSection.getAlignment()));
 
-  uint64_t Features = STI.getFeatureBits();
+  const FeatureBitset &Features = STI.getFeatureBits();
 
   // Update e_header flags. See the FIXME and comment above in
   // the constructor for a full rundown on this.
@@ -483,10 +516,10 @@ void MipsTargetELFStreamer::finish() {
   else if (getABI().IsN32())
     EFlags |= ELF::EF_MIPS_ABI2;
 
-  if (Features & Mips::FeatureGP64Bit) {
+  if (Features[Mips::FeatureGP64Bit]) {
     if (getABI().IsO32())
       EFlags |= ELF::EF_MIPS_32BITMODE; /* Compatibility Mode */
-  } else if (Features & Mips::FeatureMips64r2 || Features & Mips::FeatureMips64)
+  } else if (Features[Mips::FeatureMips64r2] || Features[Mips::FeatureMips64])
     EFlags |= ELF::EF_MIPS_32BITMODE;
 
   // If we've set the cpic eflag and we're n64, go ahead and set the pic
@@ -505,23 +538,18 @@ void MipsTargetELFStreamer::finish() {
   emitMipsAbiFlags();
 }
 
-void MipsTargetELFStreamer::emitAssignment(MCSymbol *Symbol,
-                                           const MCExpr *Value) {
+void MipsTargetELFStreamer::emitAssignment(MCSymbol *S, const MCExpr *Value) {
+  auto *Symbol = cast<MCSymbolELF>(S);
   // If on rhs is micromips symbol then mark Symbol as microMips.
   if (Value->getKind() != MCExpr::SymbolRef)
     return;
-  const MCSymbol &RhsSym =
-      static_cast<const MCSymbolRefExpr *>(Value)->getSymbol();
-  MCSymbolData &Data = getStreamer().getOrCreateSymbolData(&RhsSym);
+  const auto &RhsSym = cast<MCSymbolELF>(
+      static_cast<const MCSymbolRefExpr *>(Value)->getSymbol());
 
-  if (!(MCELF::getOther(Data) & (ELF::STO_MIPS_MICROMIPS >> 2)))
+  if (!(RhsSym.getOther() & ELF::STO_MIPS_MICROMIPS))
     return;
 
-  MCSymbolData &SymbolData = getStreamer().getOrCreateSymbolData(Symbol);
-  // The "other" values are stored in the last 6 bits of the second byte.
-  // The traditional defines for STO values assume the full byte and thus
-  // the shift to pack it.
-  MCELF::setOther(SymbolData, ELF::STO_MIPS_MICROMIPS >> 2);
+  Symbol->setOther(ELF::STO_MIPS_MICROMIPS);
 }
 
 MCELFStreamer &MipsTargetELFStreamer::getStreamer() {
@@ -564,14 +592,14 @@ void MipsTargetELFStreamer::emitDirectiveEnd(StringRef Name) {
   MCContext &Context = MCA.getContext();
   MCStreamer &OS = getStreamer();
 
-  const MCSectionELF *Sec = Context.getELFSection(
-      ".pdr", ELF::SHT_PROGBITS, ELF::SHF_ALLOC | ELF::SHT_REL);
+  MCSectionELF *Sec = Context.getELFSection(".pdr", ELF::SHT_PROGBITS,
+                                            ELF::SHF_ALLOC | ELF::SHT_REL);
 
   const MCSymbolRefExpr *ExprRef =
-      MCSymbolRefExpr::Create(Name, MCSymbolRefExpr::VK_None, Context);
+      MCSymbolRefExpr::create(Name, MCSymbolRefExpr::VK_None, Context);
 
-  MCSectionData &SecData = MCA.getOrCreateSectionData(*Sec);
-  SecData.setAlignment(4);
+  MCA.registerSection(*Sec);
+  Sec->setAlignment(4);
 
   OS.PushSection();
 
@@ -692,33 +720,33 @@ void MipsTargetELFStreamer::emitDirectiveCpLoad(unsigned RegNo) {
 
   StringRef SymName("_gp_disp");
   MCAssembler &MCA = getStreamer().getAssembler();
-  MCSymbol *GP_Disp = MCA.getContext().GetOrCreateSymbol(SymName);
-  MCA.getOrCreateSymbolData(*GP_Disp);
+  MCSymbol *GP_Disp = MCA.getContext().getOrCreateSymbol(SymName);
+  MCA.registerSymbol(*GP_Disp);
 
   MCInst TmpInst;
   TmpInst.setOpcode(Mips::LUi);
-  TmpInst.addOperand(MCOperand::CreateReg(Mips::GP));
-  const MCSymbolRefExpr *HiSym = MCSymbolRefExpr::Create(
+  TmpInst.addOperand(MCOperand::createReg(Mips::GP));
+  const MCSymbolRefExpr *HiSym = MCSymbolRefExpr::create(
       "_gp_disp", MCSymbolRefExpr::VK_Mips_ABS_HI, MCA.getContext());
-  TmpInst.addOperand(MCOperand::CreateExpr(HiSym));
+  TmpInst.addOperand(MCOperand::createExpr(HiSym));
   getStreamer().EmitInstruction(TmpInst, STI);
 
   TmpInst.clear();
 
   TmpInst.setOpcode(Mips::ADDiu);
-  TmpInst.addOperand(MCOperand::CreateReg(Mips::GP));
-  TmpInst.addOperand(MCOperand::CreateReg(Mips::GP));
-  const MCSymbolRefExpr *LoSym = MCSymbolRefExpr::Create(
+  TmpInst.addOperand(MCOperand::createReg(Mips::GP));
+  TmpInst.addOperand(MCOperand::createReg(Mips::GP));
+  const MCSymbolRefExpr *LoSym = MCSymbolRefExpr::create(
       "_gp_disp", MCSymbolRefExpr::VK_Mips_ABS_LO, MCA.getContext());
-  TmpInst.addOperand(MCOperand::CreateExpr(LoSym));
+  TmpInst.addOperand(MCOperand::createExpr(LoSym));
   getStreamer().EmitInstruction(TmpInst, STI);
 
   TmpInst.clear();
 
   TmpInst.setOpcode(Mips::ADDu);
-  TmpInst.addOperand(MCOperand::CreateReg(Mips::GP));
-  TmpInst.addOperand(MCOperand::CreateReg(Mips::GP));
-  TmpInst.addOperand(MCOperand::CreateReg(RegNo));
+  TmpInst.addOperand(MCOperand::createReg(Mips::GP));
+  TmpInst.addOperand(MCOperand::createReg(Mips::GP));
+  TmpInst.addOperand(MCOperand::createReg(RegNo));
   getStreamer().EmitInstruction(TmpInst, STI);
 
   forbidModuleDirective();
@@ -739,44 +767,44 @@ void MipsTargetELFStreamer::emitDirectiveCpsetup(unsigned RegNo,
   if (IsReg) {
     // move $save, $gpreg
     Inst.setOpcode(Mips::DADDu);
-    Inst.addOperand(MCOperand::CreateReg(RegOrOffset));
-    Inst.addOperand(MCOperand::CreateReg(Mips::GP));
-    Inst.addOperand(MCOperand::CreateReg(Mips::ZERO));
+    Inst.addOperand(MCOperand::createReg(RegOrOffset));
+    Inst.addOperand(MCOperand::createReg(Mips::GP));
+    Inst.addOperand(MCOperand::createReg(Mips::ZERO));
   } else {
     // sd $gpreg, offset($sp)
     Inst.setOpcode(Mips::SD);
-    Inst.addOperand(MCOperand::CreateReg(Mips::GP));
-    Inst.addOperand(MCOperand::CreateReg(Mips::SP));
-    Inst.addOperand(MCOperand::CreateImm(RegOrOffset));
+    Inst.addOperand(MCOperand::createReg(Mips::GP));
+    Inst.addOperand(MCOperand::createReg(Mips::SP));
+    Inst.addOperand(MCOperand::createImm(RegOrOffset));
   }
   getStreamer().EmitInstruction(Inst, STI);
   Inst.clear();
 
-  const MCSymbolRefExpr *HiExpr = MCSymbolRefExpr::Create(
+  const MCSymbolRefExpr *HiExpr = MCSymbolRefExpr::create(
       &Sym, MCSymbolRefExpr::VK_Mips_GPOFF_HI, MCA.getContext());
-  const MCSymbolRefExpr *LoExpr = MCSymbolRefExpr::Create(
+  const MCSymbolRefExpr *LoExpr = MCSymbolRefExpr::create(
       &Sym, MCSymbolRefExpr::VK_Mips_GPOFF_LO, MCA.getContext());
 
   // lui $gp, %hi(%neg(%gp_rel(funcSym)))
   Inst.setOpcode(Mips::LUi);
-  Inst.addOperand(MCOperand::CreateReg(Mips::GP));
-  Inst.addOperand(MCOperand::CreateExpr(HiExpr));
+  Inst.addOperand(MCOperand::createReg(Mips::GP));
+  Inst.addOperand(MCOperand::createExpr(HiExpr));
   getStreamer().EmitInstruction(Inst, STI);
   Inst.clear();
 
   // addiu  $gp, $gp, %lo(%neg(%gp_rel(funcSym)))
   Inst.setOpcode(Mips::ADDiu);
-  Inst.addOperand(MCOperand::CreateReg(Mips::GP));
-  Inst.addOperand(MCOperand::CreateReg(Mips::GP));
-  Inst.addOperand(MCOperand::CreateExpr(LoExpr));
+  Inst.addOperand(MCOperand::createReg(Mips::GP));
+  Inst.addOperand(MCOperand::createReg(Mips::GP));
+  Inst.addOperand(MCOperand::createExpr(LoExpr));
   getStreamer().EmitInstruction(Inst, STI);
   Inst.clear();
 
   // daddu  $gp, $gp, $funcreg
   Inst.setOpcode(Mips::DADDu);
-  Inst.addOperand(MCOperand::CreateReg(Mips::GP));
-  Inst.addOperand(MCOperand::CreateReg(Mips::GP));
-  Inst.addOperand(MCOperand::CreateReg(RegNo));
+  Inst.addOperand(MCOperand::createReg(Mips::GP));
+  Inst.addOperand(MCOperand::createReg(Mips::GP));
+  Inst.addOperand(MCOperand::createReg(RegNo));
   getStreamer().EmitInstruction(Inst, STI);
 
   forbidModuleDirective();
@@ -786,18 +814,11 @@ void MipsTargetELFStreamer::emitMipsAbiFlags() {
   MCAssembler &MCA = getStreamer().getAssembler();
   MCContext &Context = MCA.getContext();
   MCStreamer &OS = getStreamer();
-  const MCSectionELF *Sec = Context.getELFSection(
+  MCSectionELF *Sec = Context.getELFSection(
       ".MIPS.abiflags", ELF::SHT_MIPS_ABIFLAGS, ELF::SHF_ALLOC, 24, "");
-  MCSectionData &ABIShndxSD = MCA.getOrCreateSectionData(*Sec);
-  ABIShndxSD.setAlignment(8);
+  MCA.registerSection(*Sec);
+  Sec->setAlignment(8);
   OS.SwitchSection(Sec);
 
   OS << ABIFlagsSection;
 }
-
-void MipsTargetELFStreamer::emitDirectiveModuleOddSPReg(bool Enabled,
-                                                        bool IsO32ABI) {
-  MipsTargetStreamer::emitDirectiveModuleOddSPReg(Enabled, IsO32ABI);
-
-  ABIFlagsSection.OddSPReg = Enabled;
-}
diff --git a/lib/Target/Mips/MicroMips32r6InstrFormats.td b/lib/Target/Mips/MicroMips32r6InstrFormats.td
new file mode 100644
index 00000000000..187a022b256
--- /dev/null
+++ b/lib/Target/Mips/MicroMips32r6InstrFormats.td
@@ -0,0 +1,289 @@
+//=- MicroMips32r6InstrFormats.td - Mips32r6 Instruction Formats -*- tablegen -*-==//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This file describes microMIPS32r6 instruction formats.
+//
+//===----------------------------------------------------------------------===//
+
+class MMR6Arch<string opstr> {
+  string Arch = "micromipsr6";
+  string BaseOpcode = opstr;
+}
+
+class POOL32A_BITSWAP_FM_MMR6<bits<6> funct> : MipsR6Inst {
+  bits<5> rd;
+  bits<5> rt;
+
+  bits<32> Inst;
+
+  let Inst{31-26} = 0b000000;
+  let Inst{25-21} = rt;
+  let Inst{20-16} = rd;
+  let Inst{15-12} = 0b0000;
+  let Inst{11-6} = funct;
+  let Inst{5-0} = 0b111100;
+}
+
+class CACHE_PREF_FM_MMR6<bits<6> opgroup, bits<4> funct> : MipsR6Inst {
+  bits<21> addr;
+  bits<5> hint;
+
+  bits<32> Inst;
+
+  let Inst{31-26} = opgroup;
+  let Inst{25-21} = hint;
+  let Inst{20-16} = addr{20-16};
+  let Inst{15-12} = funct;
+  let Inst{11-0}  = addr{11-0};
+}
+
+class ARITH_FM_MMR6<string instr_asm, bits<10> funct> : MMR6Arch<instr_asm> {
+  bits<5> rd;
+  bits<5> rt;
+  bits<5> rs;
+
+  bits<32> Inst;
+
+  let Inst{31-26} = 0;
+  let Inst{25-21} = rt;
+  let Inst{20-16} = rs;
+  let Inst{15-11} = rd;
+  let Inst{10}    = 0;
+  let Inst{9-0}   = funct;
+}
+
+class ADDI_FM_MMR6<string instr_asm, bits<6> op> : MMR6Arch<instr_asm> {
+  bits<5>  rt;
+  bits<5>  rs;
+  bits<16> imm16;
+
+  bits<32> Inst;
+
+  let Inst{31-26} = op;
+  let Inst{25-21} = rt;
+  let Inst{20-16} = rs;
+  let Inst{15-0}  = imm16;
+}
+
+class SIGN_EXTEND_FM_MMR6<string instr_asm, bits<10> funct>
+    : MMR6Arch<instr_asm> {
+  bits<5> rd;
+  bits<5> rt;
+
+  bits<32> Inst;
+
+  let Inst{31-26} = 0b000000;
+  let Inst{25-21} = rd;
+  let Inst{20-16} = rt;
+  let Inst{15-6}  = funct;
+  let Inst{5-0}   = 0b111100;
+}
+
+class PCREL19_FM_MMR6<bits<2> funct> : MipsR6Inst {
+  bits<5> rt;
+  bits<19> imm;
+
+  bits<32> Inst;
+
+  let Inst{31-26} = 0b011110;
+  let Inst{25-21} = rt;
+  let Inst{20-19} = funct;
+  let Inst{18-0}  = imm;
+}
+
+class PCREL16_FM_MMR6<bits<5> funct> : MipsR6Inst {
+  bits<5> rt;
+  bits<16> imm;
+
+  bits<32> Inst;
+
+  let Inst{31-26} = 0b011110;
+  let Inst{25-21} = rt;
+  let Inst{20-16} = funct;
+  let Inst{15-0}  = imm;
+}
+
+class POOL32A_FM_MMR6<bits<10> funct> : MipsR6Inst {
+  bits<5> rd;
+  bits<5> rs;
+  bits<5> rt;
+
+  bits<32> Inst;
+
+  let Inst{31-26} = 0b000000;
+  let Inst{25-21} = rt;
+  let Inst{20-16} = rs;
+  let Inst{15-11} = rd;
+  let Inst{10}    = 0;
+  let Inst{9-0}   = funct;
+}
+
+class POOL32A_2R_FM_MMR6<bits<10> funct> : MipsR6Inst {
+  bits<5> rs;
+  bits<5> rt;
+
+  bits<32> Inst;
+
+  let Inst{31-26} = 0b000000;
+  let Inst{25-21} = rt;
+  let Inst{20-16} = rs;
+  let Inst{15-6}  = funct;
+  let Inst{5-0}   = 0b111100;
+}
+
+class SPECIAL_2R_FM_MMR6<bits<6> funct> : MipsR6Inst {
+  bits<5> rs;
+  bits<5> rt;
+
+  bits<32> Inst;
+
+  let Inst{31-26} = 0b000000;
+  let Inst{25-21} = rs;
+  let Inst{20-16} = 0b00000;
+  let Inst{15-11} = rt;
+  let Inst{10-6}  = 0b00001;
+  let Inst{5-0}   = funct;
+}
+
+class POOL32A_ALIGN_FM_MMR6<bits<6> funct> : MipsR6Inst {
+  bits<5> rd;
+  bits<5> rs;
+  bits<5> rt;
+  bits<2> bp;
+
+  bits<32> Inst;
+
+  let Inst{31-26} = 0b000000;
+  let Inst{25-21} = rs;
+  let Inst{20-16} = rt;
+  let Inst{15-11} = rd;
+  let Inst{10-9}  = bp;
+  let Inst{8-6}   = 0b000;
+  let Inst{5-0}   = funct;
+}
+
+class AUI_FM_MMR6 : MipsR6Inst {
+  bits<5> rs;
+  bits<5> rt;
+  bits<16> imm;
+
+  bits<32> Inst;
+
+  let Inst{31-26} = 0b000100;
+  let Inst{25-21} = rt;
+  let Inst{20-16} = rs;
+  let Inst{15-0} = imm;
+}
+
+class POOL32A_LSA_FM<bits<6> funct> : MipsR6Inst {
+  bits<5> rd;
+  bits<5> rs;
+  bits<5> rt;
+  bits<2> imm2;
+
+  bits<32> Inst;
+
+  let Inst{31-26} = 0b000000;
+  let Inst{25-21} = rt;
+  let Inst{20-16} = rs;
+  let Inst{15-11} = rd;
+  let Inst{10-9}  = imm2;
+  let Inst{8-6}   = 0b000;
+  let Inst{5-0}   = funct;
+}
+
+class CMP_BRANCH_1R_RT_OFF16_FM_MMR6<bits<6> funct> : MipsR6Inst {
+  bits<5> rt;
+  bits<16> offset;
+
+  bits<32> Inst;
+
+  let Inst{31-26} = funct;
+  let Inst{25-21} = rt;
+  let Inst{20-16} = 0b00000;
+  let Inst{15-0}  = offset;
+}
+
+class CMP_BRANCH_1R_BOTH_OFF16_FM_MMR6<bits<6> funct> : MipsR6Inst {
+  bits<5> rt;
+  bits<16> offset;
+
+  bits<32> Inst;
+
+  let Inst{31-26} = funct;
+  let Inst{25-21} = rt;
+  let Inst{20-16} = rt;
+  let Inst{15-0}  = offset;
+}
+
+class ERET_FM_MMR6<string instr_asm> : MMR6Arch<instr_asm> {
+  bits<32> Inst;
+
+  let Inst{31-26} = 0x00;
+  let Inst{25-16} = 0x00;
+  let Inst{15-6}  = 0x3cd;
+  let Inst{5-0}   = 0x3c;
+}
+
+class ERETNC_FM_MMR6<string instr_asm> : MMR6Arch<instr_asm> {
+  bits<32> Inst;
+
+  let Inst{31-26} = 0x00;
+  let Inst{25-17} = 0x00;
+  let Inst{16-16} = 0x01;
+  let Inst{15-6}  = 0x3cd;
+  let Inst{5-0}   = 0x3c;
+}
+
+class BREAK_MMR6_ENC<string instr_asm> : MMR6Arch<instr_asm> {
+  bits<10> code_1;
+  bits<10> code_2;
+  bits<32> Inst;
+  let Inst{31-26} = 0x0;
+  let Inst{25-16} = code_1;
+  let Inst{15-6}  = code_2;
+  let Inst{5-0}   = 0x07;
+}
+
+class BARRIER_MMR6_ENC<string instr_asm, bits<5> op> : MMR6Arch<instr_asm> {
+  bits<32> Inst;
+
+  let Inst{31-26} = 0x0;
+  let Inst{25-21} = 0x0;
+  let Inst{20-16} = 0x0;
+  let Inst{15-11} = op;
+  let Inst{10-6}  = 0x0;
+  let Inst{5-0}   = 0x0;
+}
+
+class EIDI_MMR6_ENC<string instr_asm, bits<10> funct> : MMR6Arch<instr_asm> {
+  bits<32> Inst;
+  bits<5> rt; // Actually rs but we're sharing code with the standard encodings which call it rt
+
+  let Inst{31-26} = 0x00;
+  let Inst{25-21} = 0x00;
+  let Inst{20-16} = rt;
+  let Inst{15-6}  = funct;
+  let Inst{5-0}   = 0x3c;
+}
+
+class SHIFT_MMR6_ENC<string instr_asm, bits<10> funct, bit rotate> : MMR6Arch<instr_asm> {
+  bits<5> rd;
+  bits<5> rt;
+  bits<5> shamt;
+
+  bits<32> Inst;
+
+  let Inst{31-26} = 0;
+  let Inst{25-21} = rd;
+  let Inst{20-16} = rt;
+  let Inst{15-11} = shamt;
+  let Inst{10}    = rotate;
+  let Inst{9-0}   = funct;
+}
diff --git a/lib/Target/Mips/MicroMips32r6InstrInfo.td b/lib/Target/Mips/MicroMips32r6InstrInfo.td
new file mode 100644
index 00000000000..53bde1379e2
--- /dev/null
+++ b/lib/Target/Mips/MicroMips32r6InstrInfo.td
@@ -0,0 +1,360 @@
+//=- MicroMips32r6InstrInfo.td - MicroMips r6 Instruction Information -*- tablegen -*-=//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This file describes microMIPSr6 instructions.
+//
+//===----------------------------------------------------------------------===//
+
+//===----------------------------------------------------------------------===//
+//
+// Instruction Encodings
+//
+//===----------------------------------------------------------------------===//
+class ADD_MMR6_ENC : ARITH_FM_MMR6<"add", 0x110>;
+class ADDIU_MMR6_ENC : ADDI_FM_MMR6<"addiu", 0xc>;
+class ADDU_MMR6_ENC : ARITH_FM_MMR6<"addu", 0x150>;
+class ADDIUPC_MMR6_ENC : PCREL19_FM_MMR6<0b00>;
+class ALUIPC_MMR6_ENC : PCREL16_FM_MMR6<0b11111>;
+class AND_MMR6_ENC : ARITH_FM_MMR6<"and", 0x250>;
+class ANDI_MMR6_ENC : ADDI_FM_MMR6<"andi", 0x34>;
+class AUIPC_MMR6_ENC  : PCREL16_FM_MMR6<0b11110>;
+class ALIGN_MMR6_ENC : POOL32A_ALIGN_FM_MMR6<0b011111>;
+class AUI_MMR6_ENC : AUI_FM_MMR6;
+class BALC_MMR6_ENC  : BRANCH_OFF26_FM<0b101101>;
+class BC_MMR6_ENC : BRANCH_OFF26_FM<0b100101>;
+class BITSWAP_MMR6_ENC : POOL32A_BITSWAP_FM_MMR6<0b101100>;
+class BRK_MMR6_ENC : BREAK_MMR6_ENC<"break">;
+class BEQZALC_MMR6_ENC : CMP_BRANCH_1R_RT_OFF16_FM_MMR6<0b011101>;
+class BNEZALC_MMR6_ENC : CMP_BRANCH_1R_RT_OFF16_FM_MMR6<0b011111>;
+class BGTZALC_MMR6_ENC : CMP_BRANCH_1R_RT_OFF16_FM_MMR6<0b111000>;
+class BLTZALC_MMR6_ENC : CMP_BRANCH_1R_BOTH_OFF16_FM_MMR6<0b111000>;
+class BGEZALC_MMR6_ENC : CMP_BRANCH_1R_BOTH_OFF16_FM_MMR6<0b110000>;
+class BLEZALC_MMR6_ENC : CMP_BRANCH_1R_RT_OFF16_FM_MMR6<0b110000>;
+class CACHE_MMR6_ENC : CACHE_PREF_FM_MMR6<0b001000, 0b0110>;
+class CLO_MMR6_ENC : POOL32A_2R_FM_MMR6<0b0100101100>;
+class CLZ_MMR6_ENC : SPECIAL_2R_FM_MMR6<0b010000>;
+class DIV_MMR6_ENC : ARITH_FM_MMR6<"div", 0x118>;
+class DIVU_MMR6_ENC : ARITH_FM_MMR6<"divu", 0x198>;
+class EHB_MMR6_ENC : BARRIER_MMR6_ENC<"ehb", 0x3>;
+class EI_MMR6_ENC : EIDI_MMR6_ENC<"ei", 0x15d>;
+class ERET_MMR6_ENC : ERET_FM_MMR6<"eret">;
+class ERETNC_MMR6_ENC : ERETNC_FM_MMR6<"eretnc">;
+class JIALC_MMR6_ENC : JMP_IDX_COMPACT_FM<0b100000>;
+class JIC_MMR6_ENC   : JMP_IDX_COMPACT_FM<0b101000>;
+class LSA_MMR6_ENC : POOL32A_LSA_FM<0b001111>;
+class LWPC_MMR6_ENC  : PCREL19_FM_MMR6<0b01>;
+class MOD_MMR6_ENC : ARITH_FM_MMR6<"mod", 0x158>;
+class MODU_MMR6_ENC : ARITH_FM_MMR6<"modu", 0x1d8>;
+class MUL_MMR6_ENC : ARITH_FM_MMR6<"mul", 0x18>;
+class MUH_MMR6_ENC : ARITH_FM_MMR6<"muh", 0x58>;
+class MULU_MMR6_ENC : ARITH_FM_MMR6<"mulu", 0x98>;
+class MUHU_MMR6_ENC : ARITH_FM_MMR6<"muhu", 0xd8>;
+class NOR_MMR6_ENC : ARITH_FM_MMR6<"nor", 0x2d0>;
+class OR_MMR6_ENC : ARITH_FM_MMR6<"or", 0x290>;
+class ORI_MMR6_ENC : ADDI_FM_MMR6<"ori", 0x14>;
+class PREF_MMR6_ENC : CACHE_PREF_FM_MMR6<0b011000, 0b0010>;
+class SEB_MMR6_ENC : SIGN_EXTEND_FM_MMR6<"seb", 0b0010101100>;
+class SEH_MMR6_ENC : SIGN_EXTEND_FM_MMR6<"seh", 0b0011101100>;
+class SELEQZ_MMR6_ENC : POOL32A_FM_MMR6<0b0101000000>;
+class SELNEZ_MMR6_ENC : POOL32A_FM_MMR6<0b0110000000>;
+class SLL_MMR6_ENC : SHIFT_MMR6_ENC<"sll", 0x00, 0b0>;
+class SUB_MMR6_ENC : ARITH_FM_MMR6<"sub", 0x190>;
+class SUBU_MMR6_ENC : ARITH_FM_MMR6<"subu", 0x1d0>;
+class XOR_MMR6_ENC : ARITH_FM_MMR6<"xor", 0x310>;
+class XORI_MMR6_ENC : ADDI_FM_MMR6<"xori", 0x1c>;
+
+class CMP_CBR_RT_Z_MMR6_DESC_BASE<string instr_asm, DAGOperand opnd,
+                                  RegisterOperand GPROpnd>
+    : BRANCH_DESC_BASE, MMR6Arch<instr_asm> {
+  dag InOperandList = (ins GPROpnd:$rt, opnd:$offset);
+  dag OutOperandList = (outs);
+  string AsmString = !strconcat(instr_asm, "\t$rt, $offset");
+  list<Register> Defs = [AT];
+}
+
+class BEQZALC_MMR6_DESC : CMP_CBR_RT_Z_MMR6_DESC_BASE<"beqzalc", brtarget_mm,
+                                                      GPR32Opnd> {
+  list<Register> Defs = [RA];
+}
+
+class BGEZALC_MMR6_DESC : CMP_CBR_RT_Z_MMR6_DESC_BASE<"bgezalc", brtarget_mm,
+                                                      GPR32Opnd> {
+  list<Register> Defs = [RA];
+}
+
+class BGTZALC_MMR6_DESC : CMP_CBR_RT_Z_MMR6_DESC_BASE<"bgtzalc", brtarget_mm,
+                                                      GPR32Opnd> {
+  list<Register> Defs = [RA];
+}
+
+class BLEZALC_MMR6_DESC : CMP_CBR_RT_Z_MMR6_DESC_BASE<"blezalc", brtarget_mm,
+                                                      GPR32Opnd> {
+  list<Register> Defs = [RA];
+}
+
+class BLTZALC_MMR6_DESC : CMP_CBR_RT_Z_MMR6_DESC_BASE<"bltzalc", brtarget_mm,
+                                                      GPR32Opnd> {
+  list<Register> Defs = [RA];
+}
+
+class BNEZALC_MMR6_DESC : CMP_CBR_RT_Z_MMR6_DESC_BASE<"bnezalc", brtarget_mm,
+                                                      GPR32Opnd> {
+  list<Register> Defs = [RA];
+}
+
+//===----------------------------------------------------------------------===//
+//
+// Instruction Descriptions
+//
+//===----------------------------------------------------------------------===//
+
+class ADD_MMR6_DESC : ArithLogicR<"add", GPR32Opnd>;
+class ADDIU_MMR6_DESC : ArithLogicI<"addiu", simm16, GPR32Opnd>;
+class ADDU_MMR6_DESC : ArithLogicR<"addu", GPR32Opnd>;
+class MUL_MMR6_DESC : ArithLogicR<"mul", GPR32Opnd>;
+class MUH_MMR6_DESC : ArithLogicR<"muh", GPR32Opnd>;
+class MULU_MMR6_DESC : ArithLogicR<"mulu", GPR32Opnd>;
+class MUHU_MMR6_DESC : ArithLogicR<"muhu", GPR32Opnd>;
+
+class BC_MMR6_DESC_BASE<string instr_asm, DAGOperand opnd>
+    : BRANCH_DESC_BASE, MMR6Arch<instr_asm> {
+  dag InOperandList = (ins opnd:$offset);
+  dag OutOperandList = (outs);
+  string AsmString = !strconcat(instr_asm, "\t$offset");
+  bit isBarrier = 1;
+}
+
+class BALC_MMR6_DESC : BC_MMR6_DESC_BASE<"balc", brtarget26> {
+  bit isCall = 1;
+  list<Register> Defs = [RA];
+}
+class BC_MMR6_DESC : BC_MMR6_DESC_BASE<"bc", brtarget26>;
+class SUB_MMR6_DESC : ArithLogicR<"sub", GPR32Opnd>;
+class SUBU_MMR6_DESC : ArithLogicR<"subu", GPR32Opnd>;
+
+class BITSWAP_MMR6_DESC_BASE<string instr_asm, RegisterOperand GPROpnd>
+    : MMR6Arch<instr_asm> {
+  dag OutOperandList = (outs GPROpnd:$rd);
+  dag InOperandList = (ins GPROpnd:$rt);
+  string AsmString = !strconcat(instr_asm, "\t$rd, $rt");
+  list<dag> Pattern = [];
+}
+
+class BITSWAP_MMR6_DESC : BITSWAP_MMR6_DESC_BASE<"bitswap", GPR32Opnd>;
+
+class BRK_MMR6_DESC : BRK_FT<"break">;
+
+class CACHE_HINT_MMR6_DESC<string instr_asm, Operand MemOpnd,
+                           RegisterOperand GPROpnd> : MMR6Arch<instr_asm> {
+  dag OutOperandList = (outs);
+  dag InOperandList = (ins MemOpnd:$addr, uimm5:$hint);
+  string AsmString = !strconcat(instr_asm, "\t$hint, $addr");
+  list<dag> Pattern = [];
+  string DecoderMethod = "DecodeCacheOpMM";
+}
+
+class CACHE_MMR6_DESC : CACHE_HINT_MMR6_DESC<"cache", mem_mm_12, GPR32Opnd>;
+class PREF_MMR6_DESC : CACHE_HINT_MMR6_DESC<"pref", mem_mm_12, GPR32Opnd>;
+
+class CLO_CLZ_MMR6_DESC_BASE<string instr_asm, RegisterOperand GPROpnd>
+    : MMR6Arch<instr_asm> {
+  dag OutOperandList = (outs GPROpnd:$rt);
+  dag InOperandList = (ins GPROpnd:$rs);
+  string AsmString = !strconcat(instr_asm, "\t$rt, $rs");
+}
+
+class CLO_MMR6_DESC : CLO_CLZ_MMR6_DESC_BASE<"clo", GPR32Opnd>;
+class CLZ_MMR6_DESC : CLO_CLZ_MMR6_DESC_BASE<"clz", GPR32Opnd>;
+
+class EHB_MMR6_DESC : Barrier<"ehb">;
+class EI_MMR6_DESC : DEI_FT<"ei", GPR32Opnd>;
+
+class ERET_MMR6_DESC : ER_FT<"eret">;
+class ERETNC_MMR6_DESC : ER_FT<"eretnc">;
+
+class JMP_MMR6_IDX_COMPACT_DESC_BASE<string opstr, DAGOperand opnd,
+                                     RegisterOperand GPROpnd>
+    : MMR6Arch<opstr> {
+  dag InOperandList = (ins GPROpnd:$rt, opnd:$offset);
+  string AsmString = !strconcat(opstr, "\t$rt, $offset");
+  list<dag> Pattern = [];
+  bit isTerminator = 1;
+  bit hasDelaySlot = 0;
+}
+
+class JIALC_MMR6_DESC : JMP_MMR6_IDX_COMPACT_DESC_BASE<"jialc", calloffset16,
+                                                       GPR32Opnd> {
+  bit isCall = 1;
+  list<Register> Defs = [RA];
+}
+
+class JIC_MMR6_DESC : JMP_MMR6_IDX_COMPACT_DESC_BASE<"jic", jmpoffset16,
+                                                     GPR32Opnd> {
+  bit isBarrier = 1;
+  list<Register> Defs = [AT];
+}
+
+class ALIGN_MMR6_DESC_BASE<string instr_asm, RegisterOperand GPROpnd,
+                      Operand ImmOpnd>  : MMR6Arch<instr_asm> {
+  dag OutOperandList = (outs GPROpnd:$rd);
+  dag InOperandList = (ins GPROpnd:$rs, GPROpnd:$rt, ImmOpnd:$bp);
+  string AsmString = !strconcat(instr_asm, "\t$rd, $rs, $rt, $bp");
+  list<dag> Pattern = [];
+}
+
+class ALIGN_MMR6_DESC : ALIGN_MMR6_DESC_BASE<"align", GPR32Opnd, uimm2>;
+
+class AUI_MMR6_DESC_BASE<string instr_asm, RegisterOperand GPROpnd>
+    : MMR6Arch<instr_asm> {
+  dag OutOperandList = (outs GPROpnd:$rt);
+  dag InOperandList = (ins GPROpnd:$rs, simm16:$imm);
+  string AsmString = !strconcat(instr_asm, "\t$rt, $rs, $imm");
+  list<dag> Pattern = [];
+}
+
+class AUI_MMR6_DESC : AUI_MMR6_DESC_BASE<"aui", GPR32Opnd>;
+
+class SEB_MMR6_DESC : SignExtInReg<"seb", i8, GPR32Opnd, II_SEB>;
+class SEH_MMR6_DESC : SignExtInReg<"seh", i16, GPR32Opnd, II_SEH>;
+class ALUIPC_MMR6_DESC_BASE<string instr_asm, RegisterOperand GPROpnd>
+    : MMR6Arch<instr_asm> {
+  dag OutOperandList = (outs GPROpnd:$rt);
+  dag InOperandList = (ins simm16:$imm);
+  string AsmString = !strconcat(instr_asm, "\t$rt, $imm");
+  list<dag> Pattern = [];
+}
+
+class ALUIPC_MMR6_DESC : ALUIPC_MMR6_DESC_BASE<"aluipc", GPR32Opnd>;
+class AUIPC_MMR6_DESC : ALUIPC_MMR6_DESC_BASE<"auipc", GPR32Opnd>;
+
+class LSA_MMR6_DESC_BASE<string instr_asm, RegisterOperand GPROpnd,
+                         Operand ImmOpnd> : MMR6Arch<instr_asm> {
+  dag OutOperandList = (outs GPROpnd:$rd);
+  dag InOperandList = (ins GPROpnd:$rs, GPROpnd:$rt, ImmOpnd:$imm2);
+  string AsmString = !strconcat(instr_asm, "\t$rt, $rs, $rd, $imm2");
+  list<dag> Pattern = [];
+}
+
+class LSA_MMR6_DESC : LSA_MMR6_DESC_BASE<"lsa", GPR32Opnd, uimm2>;
+
+class PCREL_MMR6_DESC_BASE<string instr_asm, RegisterOperand GPROpnd,
+                           Operand ImmOpnd> : MMR6Arch<instr_asm> {
+  dag OutOperandList = (outs GPROpnd:$rt);
+  dag InOperandList = (ins ImmOpnd:$imm);
+  string AsmString = !strconcat(instr_asm, "\t$rt, $imm");
+  list<dag> Pattern = [];
+}
+
+class ADDIUPC_MMR6_DESC : PCREL_MMR6_DESC_BASE<"addiupc", GPR32Opnd, simm19_lsl2>;
+class LWPC_MMR6_DESC: PCREL_MMR6_DESC_BASE<"lwpc", GPR32Opnd, simm19_lsl2>;
+
+class SELEQNE_Z_MMR6_DESC_BASE<string instr_asm, RegisterOperand GPROpnd>
+    : MMR6Arch<instr_asm> {
+  dag OutOperandList = (outs GPROpnd:$rd);
+  dag InOperandList = (ins GPROpnd:$rs, GPROpnd:$rt);
+  string AsmString = !strconcat(instr_asm, "\t$rd, $rs, $rt");
+  list<dag> Pattern = [];
+}
+
+class SELEQZ_MMR6_DESC : SELEQNE_Z_MMR6_DESC_BASE<"seleqz", GPR32Opnd>;
+class SELNEZ_MMR6_DESC : SELEQNE_Z_MMR6_DESC_BASE<"selnez", GPR32Opnd>;
+class SLL_MMR6_DESC : shift_rotate_imm<"sll", uimm5, GPR32Opnd, II_SLL>;
+class DIV_MMR6_DESC : ArithLogicR<"div", GPR32Opnd>;
+class DIVU_MMR6_DESC : ArithLogicR<"divu", GPR32Opnd>;
+class MOD_MMR6_DESC : ArithLogicR<"mod", GPR32Opnd>;
+class MODU_MMR6_DESC : ArithLogicR<"modu", GPR32Opnd>;
+class AND_MMR6_DESC : ArithLogicR<"and", GPR32Opnd>;
+class ANDI_MMR6_DESC : ArithLogicI<"andi", simm16, GPR32Opnd>;
+class NOR_MMR6_DESC : ArithLogicR<"nor", GPR32Opnd>;
+class OR_MMR6_DESC : ArithLogicR<"or", GPR32Opnd>;
+class ORI_MMR6_DESC : ArithLogicI<"ori", simm16, GPR32Opnd>;
+class XOR_MMR6_DESC : ArithLogicR<"xor", GPR32Opnd>;
+class XORI_MMR6_DESC : ArithLogicI<"xori", simm16, GPR32Opnd>;
+
+//===----------------------------------------------------------------------===//
+//
+// Instruction Definitions
+//
+//===----------------------------------------------------------------------===//
+
+let DecoderNamespace = "MicroMips32r6" in {
+def ADD_MMR6 : StdMMR6Rel, ADD_MMR6_DESC, ADD_MMR6_ENC, ISA_MICROMIPS32R6;
+def ADDIU_MMR6 : StdMMR6Rel, ADDIU_MMR6_DESC, ADDIU_MMR6_ENC, ISA_MICROMIPS32R6;
+def ADDU_MMR6 : StdMMR6Rel, ADDU_MMR6_DESC, ADDU_MMR6_ENC, ISA_MICROMIPS32R6;
+def ADDIUPC_MMR6 : R6MMR6Rel, ADDIUPC_MMR6_ENC, ADDIUPC_MMR6_DESC,
+                   ISA_MICROMIPS32R6;
+def ALUIPC_MMR6 : R6MMR6Rel, ALUIPC_MMR6_ENC, ALUIPC_MMR6_DESC,
+                  ISA_MICROMIPS32R6;
+def AND_MMR6 : StdMMR6Rel, AND_MMR6_DESC, AND_MMR6_ENC, ISA_MICROMIPS32R6;
+def ANDI_MMR6 : StdMMR6Rel, ANDI_MMR6_DESC, ANDI_MMR6_ENC, ISA_MICROMIPS32R6;
+def AUIPC_MMR6 : R6MMR6Rel, AUIPC_MMR6_ENC, AUIPC_MMR6_DESC, ISA_MICROMIPS32R6;
+def ALIGN_MMR6 : R6MMR6Rel, ALIGN_MMR6_ENC, ALIGN_MMR6_DESC, ISA_MICROMIPS32R6;
+def AUI_MMR6 : R6MMR6Rel, AUI_MMR6_ENC, AUI_MMR6_DESC, ISA_MICROMIPS32R6;
+def BALC_MMR6 : R6MMR6Rel, BALC_MMR6_ENC, BALC_MMR6_DESC, ISA_MICROMIPS32R6;
+def BC_MMR6 : R6MMR6Rel, BC_MMR6_ENC, BC_MMR6_DESC, ISA_MICROMIPS32R6;
+def BITSWAP_MMR6 : R6MMR6Rel, BITSWAP_MMR6_ENC, BITSWAP_MMR6_DESC,
+                   ISA_MICROMIPS32R6;
+def BEQZALC_MMR6 : R6MMR6Rel, BEQZALC_MMR6_ENC, BEQZALC_MMR6_DESC,
+                   ISA_MICROMIPS32R6;
+def BGEZALC_MMR6 : R6MMR6Rel, BGEZALC_MMR6_ENC, BGEZALC_MMR6_DESC,
+                   ISA_MICROMIPS32R6;
+def BGTZALC_MMR6 : R6MMR6Rel, BGTZALC_MMR6_ENC, BGTZALC_MMR6_DESC,
+                   ISA_MICROMIPS32R6;
+def BLEZALC_MMR6 : R6MMR6Rel, BLEZALC_MMR6_ENC, BLEZALC_MMR6_DESC,
+                   ISA_MICROMIPS32R6;
+def BLTZALC_MMR6 : R6MMR6Rel, BLTZALC_MMR6_ENC, BLTZALC_MMR6_DESC,
+                   ISA_MICROMIPS32R6;
+def BNEZALC_MMR6 : R6MMR6Rel, BNEZALC_MMR6_ENC, BNEZALC_MMR6_DESC,
+                   ISA_MICROMIPS32R6;
+def BREAK_MMR6 : StdMMR6Rel, BRK_MMR6_DESC, BRK_MMR6_ENC, ISA_MICROMIPS32R6;
+def CACHE_MMR6 : R6MMR6Rel, CACHE_MMR6_ENC, CACHE_MMR6_DESC, ISA_MICROMIPS32R6;
+def CLO_MMR6 : R6MMR6Rel, CLO_MMR6_ENC, CLO_MMR6_DESC, ISA_MICROMIPS32R6;
+def CLZ_MMR6 : R6MMR6Rel, CLZ_MMR6_ENC, CLZ_MMR6_DESC, ISA_MICROMIPS32R6;
+def DIV_MMR6 : R6MMR6Rel, DIV_MMR6_DESC, DIV_MMR6_ENC, ISA_MICROMIPS32R6;
+def DIVU_MMR6 : R6MMR6Rel, DIVU_MMR6_DESC, DIVU_MMR6_ENC, ISA_MICROMIPS32R6;
+def EHB_MMR6 : StdMMR6Rel, EHB_MMR6_DESC, EHB_MMR6_ENC, ISA_MICROMIPS32R6;
+def EI_MMR6 : StdMMR6Rel, EI_MMR6_DESC, EI_MMR6_ENC, ISA_MICROMIPS32R6;
+def ERET_MMR6 : R6MMR6Rel, ERET_MMR6_DESC, ERET_MMR6_ENC, ISA_MICROMIPS32R6;
+def ERETNC_MMR6 : R6MMR6Rel, ERETNC_MMR6_DESC, ERETNC_MMR6_ENC,
+                  ISA_MICROMIPS32R6;
+def JIALC_MMR6 : R6MMR6Rel, JIALC_MMR6_ENC, JIALC_MMR6_DESC, ISA_MICROMIPS32R6;
+def JIC_MMR6 : R6MMR6Rel, JIC_MMR6_ENC, JIC_MMR6_DESC, ISA_MICROMIPS32R6;
+def LSA_MMR6 : R6MMR6Rel, LSA_MMR6_ENC, LSA_MMR6_DESC, ISA_MICROMIPS32R6;
+def LWPC_MMR6 : R6MMR6Rel, LWPC_MMR6_ENC, LWPC_MMR6_DESC, ISA_MICROMIPS32R6;
+def MOD_MMR6 : R6MMR6Rel, MOD_MMR6_DESC, MOD_MMR6_ENC, ISA_MICROMIPS32R6;
+def MODU_MMR6 : R6MMR6Rel, MODU_MMR6_DESC, MODU_MMR6_ENC, ISA_MICROMIPS32R6;
+def MUL_MMR6 : R6MMR6Rel, MUL_MMR6_DESC, MUL_MMR6_ENC, ISA_MICROMIPS32R6;
+def MUH_MMR6 : R6MMR6Rel, MUH_MMR6_DESC, MUH_MMR6_ENC, ISA_MICROMIPS32R6;
+def MULU_MMR6 : R6MMR6Rel, MULU_MMR6_DESC, MULU_MMR6_ENC, ISA_MICROMIPS32R6;
+def MUHU_MMR6 : R6MMR6Rel, MUHU_MMR6_DESC, MUHU_MMR6_ENC, ISA_MICROMIPS32R6;
+def NOR_MMR6 : StdMMR6Rel, NOR_MMR6_DESC, NOR_MMR6_ENC, ISA_MICROMIPS32R6;
+def OR_MMR6 : StdMMR6Rel, OR_MMR6_DESC, OR_MMR6_ENC, ISA_MICROMIPS32R6;
+def ORI_MMR6 : StdMMR6Rel, ORI_MMR6_DESC, ORI_MMR6_ENC, ISA_MICROMIPS32R6;
+def PREF_MMR6 : R6MMR6Rel, PREF_MMR6_ENC, PREF_MMR6_DESC, ISA_MICROMIPS32R6;
+def SEB_MMR6 : StdMMR6Rel, SEB_MMR6_DESC, SEB_MMR6_ENC, ISA_MICROMIPS32R6;
+def SEH_MMR6 : StdMMR6Rel, SEH_MMR6_DESC, SEH_MMR6_ENC, ISA_MICROMIPS32R6;
+def SELEQZ_MMR6 : R6MMR6Rel, SELEQZ_MMR6_ENC, SELEQZ_MMR6_DESC,
+                  ISA_MICROMIPS32R6;
+def SELNEZ_MMR6 : R6MMR6Rel, SELNEZ_MMR6_ENC, SELNEZ_MMR6_DESC,
+                  ISA_MICROMIPS32R6;
+def SLL_MMR6 : StdMMR6Rel, SLL_MMR6_DESC, SLL_MMR6_ENC, ISA_MICROMIPS32R6;
+def SUB_MMR6 : StdMMR6Rel, SUB_MMR6_DESC, SUB_MMR6_ENC, ISA_MICROMIPS32R6;
+def SUBU_MMR6 : StdMMR6Rel, SUBU_MMR6_DESC, SUBU_MMR6_ENC, ISA_MICROMIPS32R6;
+def XOR_MMR6 : StdMMR6Rel, XOR_MMR6_DESC, XOR_MMR6_ENC, ISA_MICROMIPS32R6;
+def XORI_MMR6 : StdMMR6Rel, XORI_MMR6_DESC, XORI_MMR6_ENC, ISA_MICROMIPS32R6;
+}
+
+//===----------------------------------------------------------------------===//
+//
+// MicroMips instruction aliases
+//
+//===----------------------------------------------------------------------===//
+
+def : MipsInstAlias<"ei", (EI_MMR6 ZERO), 1>, ISA_MICROMIPS32R6;
+def : MipsInstAlias<"nop", (SLL_MMR6 ZERO, ZERO, 0), 1>, ISA_MICROMIPS32R6;
diff --git a/lib/Target/Mips/MicroMipsInstrFPU.td b/lib/Target/Mips/MicroMipsInstrFPU.td
index fae70598fa3..004b0d51f4b 100644
--- a/lib/Target/Mips/MicroMipsInstrFPU.td
+++ b/lib/Target/Mips/MicroMipsInstrFPU.td
@@ -95,7 +95,7 @@ def FNEG_MM : MMRel, ABSS_FT<"neg.d", AFGR64Opnd, AFGR64Opnd, II_NEG, fneg>,
               ABS_FM_MM<1, 0x2d>;
 
 def FMOV_D32_MM : MMRel, ABSS_FT<"mov.d", AFGR64Opnd, AFGR64Opnd, II_MOV_D>,
-                  ABS_FM_MM<1, 0x1>, AdditionalRequires<[NotFP64bit]>;
+                  ABS_FM_MM<1, 0x1>, FGR_32;
 
 def MOVZ_I_S_MM : MMRel, CMov_I_F_FT<"movz.s", GPR32Opnd, FGR32Opnd,
                                      II_MOVZ_S>, CMov_I_F_FM_MM<0x78, 0>;
@@ -124,9 +124,9 @@ def MFC1_MM : MMRel, MFC1_FT<"mfc1", GPR32Opnd, FGR32Opnd,
 def MTC1_MM : MMRel, MTC1_FT<"mtc1", FGR32Opnd, GPR32Opnd,
                              II_MTC1, bitconvert>, MFC1_FM_MM<0xa0>;
 def MFHC1_MM : MMRel, MFC1_FT<"mfhc1", GPR32Opnd, AFGR64Opnd, II_MFHC1>,
-               MFC1_FM_MM<0xc0>, ISA_MIPS32R2, AdditionalRequires<[NotFP64bit]>;
+               MFC1_FM_MM<0xc0>, ISA_MIPS32R2, FGR_32;
 def MTHC1_MM : MMRel, MTC1_64_FT<"mthc1", AFGR64Opnd, GPR32Opnd, II_MTHC1>,
-               MFC1_FM_MM<0xe0>, ISA_MIPS32R2, AdditionalRequires<[NotFP64bit]>;
+               MFC1_FM_MM<0xe0>, ISA_MIPS32R2, FGR_32;
 
 def MADD_S_MM : MMRel, MADDS_FT<"madd.s", FGR32Opnd, II_MADD_S, fadd>,
                 MADDS_FM_MM<0x1>;
diff --git a/lib/Target/Mips/MicroMipsInstrInfo.td b/lib/Target/Mips/MicroMipsInstrInfo.td
index 2aab739dbd2..39393840c6f 100644
--- a/lib/Target/Mips/MicroMipsInstrInfo.td
+++ b/lib/Target/Mips/MicroMipsInstrInfo.td
@@ -934,3 +934,7 @@ class UncondBranchMMPseudo<string opstr> :
   def : MipsInstAlias<"nop", (SLL_MM ZERO, ZERO, 0), 1>;
   def : MipsInstAlias<"nop", (MOVE16_MM ZERO, ZERO), 1>;
 }
+
+let Predicates = [InMicroMips] in {
+def : MipsInstAlias<"ei", (EI_MM ZERO), 1>, ISA_MIPS32R2;
+}
diff --git a/lib/Target/Mips/Mips.td b/lib/Target/Mips/Mips.td
index ca2474101ec..dbb5f7b71d8 100644
--- a/lib/Target/Mips/Mips.td
+++ b/lib/Target/Mips/Mips.td
@@ -28,12 +28,15 @@ class PredicateControl {
   list<Predicate> FGRPredicates = [];
   // Predicates for the instruction group membership such as ISA's and ASE's
   list<Predicate> InsnPredicates = [];
+  // Predicate for marking the instruction as usable in hard-float mode only.
+  list<Predicate> HardFloatPredicate = [];
   // Predicates for anything else
   list<Predicate> AdditionalPredicates = [];
   list<Predicate> Predicates = !listconcat(EncodingPredicates,
                                            GPRPredicates,
                                            FGRPredicates,
                                            InsnPredicates,
+                                           HardFloatPredicate,
                                            AdditionalPredicates);
 }
 
@@ -69,6 +72,8 @@ def FeatureNaN2008     : SubtargetFeature<"nan2008", "IsNaN2008bit", "true",
                                 "IEEE 754-2008 NaN encoding">;
 def FeatureSingleFloat : SubtargetFeature<"single-float", "IsSingleFloat",
                                 "true", "Only supports single precision float">;
+def FeatureSoftFloat   : SubtargetFeature<"soft-float", "IsSoftFloat", "true",
+                                "Does not support floating point instructions">;
 def FeatureNoOddSPReg  : SubtargetFeature<"nooddspreg", "UseOddSPReg", "false",
                               "Disable odd numbered single-precision "
                               "registers">;
diff --git a/lib/Target/Mips/Mips16FrameLowering.cpp b/lib/Target/Mips/Mips16FrameLowering.cpp
index 5828fbdee40..46cc99c6239 100644
--- a/lib/Target/Mips/Mips16FrameLowering.cpp
+++ b/lib/Target/Mips/Mips16FrameLowering.cpp
@@ -32,8 +32,9 @@ using namespace llvm;
 Mips16FrameLowering::Mips16FrameLowering(const MipsSubtarget &STI)
     : MipsFrameLowering(STI, STI.stackAlignment()) {}
 
-void Mips16FrameLowering::emitPrologue(MachineFunction &MF) const {
-  MachineBasicBlock &MBB = MF.front();
+void Mips16FrameLowering::emitPrologue(MachineFunction &MF,
+                                       MachineBasicBlock &MBB) const {
+  assert(&MF.front() == &MBB && "Shrink-wrapping not yet supported");
   MachineFrameInfo *MFI = MF.getFrameInfo();
   const Mips16InstrInfo &TII =
       *static_cast<const Mips16InstrInfo *>(STI.getInstrInfo());
@@ -151,18 +152,19 @@ Mips16FrameLowering::hasReservedCallFrame(const MachineFunction &MF) const {
   return isInt<15>(MFI->getMaxCallFrameSize()) && !MFI->hasVarSizedObjects();
 }
 
-void Mips16FrameLowering::
-processFunctionBeforeCalleeSavedScan(MachineFunction &MF,
-                                     RegScavenger *RS) const {
+void Mips16FrameLowering::determineCalleeSaves(MachineFunction &MF,
+                                               BitVector &SavedRegs,
+                                               RegScavenger *RS) const {
+  TargetFrameLowering::determineCalleeSaves(MF, SavedRegs, RS);
   const Mips16InstrInfo &TII =
       *static_cast<const Mips16InstrInfo *>(STI.getInstrInfo());
   const MipsRegisterInfo &RI = TII.getRegisterInfo();
   const BitVector Reserved = RI.getReservedRegs(MF);
   bool SaveS2 = Reserved[Mips::S2];
   if (SaveS2)
-    MF.getRegInfo().setPhysRegUsed(Mips::S2);
+    SavedRegs.set(Mips::S2);
   if (hasFP(MF))
-    MF.getRegInfo().setPhysRegUsed(Mips::S0);
+    SavedRegs.set(Mips::S0);
 }
 
 const MipsFrameLowering *
diff --git a/lib/Target/Mips/Mips16FrameLowering.h b/lib/Target/Mips/Mips16FrameLowering.h
index 0287e59243a..b48ed4641ea 100644
--- a/lib/Target/Mips/Mips16FrameLowering.h
+++ b/lib/Target/Mips/Mips16FrameLowering.h
@@ -23,7 +23,7 @@ class Mips16FrameLowering : public MipsFrameLowering {
 
   /// emitProlog/emitEpilog - These methods insert prolog and epilog code into
   /// the function.
-  void emitPrologue(MachineFunction &MF) const override;
+  void emitPrologue(MachineFunction &MF, MachineBasicBlock &MBB) const override;
   void emitEpilogue(MachineFunction &MF, MachineBasicBlock &MBB) const override;
 
   bool spillCalleeSavedRegisters(MachineBasicBlock &MBB,
@@ -38,8 +38,8 @@ class Mips16FrameLowering : public MipsFrameLowering {
 
   bool hasReservedCallFrame(const MachineFunction &MF) const override;
 
-  void processFunctionBeforeCalleeSavedScan(MachineFunction &MF,
-                                            RegScavenger *RS) const override;
+  void determineCalleeSaves(MachineFunction &MF, BitVector &SavedRegs,
+                            RegScavenger *RS) const override;
 };
 
 } // End llvm namespace
diff --git a/lib/Target/Mips/Mips16ISelDAGToDAG.cpp b/lib/Target/Mips/Mips16ISelDAGToDAG.cpp
index 3221ccb7749..bce2c1eb448 100644
--- a/lib/Target/Mips/Mips16ISelDAGToDAG.cpp
+++ b/lib/Target/Mips/Mips16ISelDAGToDAG.cpp
@@ -120,13 +120,13 @@ void Mips16DAGToDAGISel::processFunctionAfterISel(MachineFunction &MF) {
 SDValue Mips16DAGToDAGISel::getMips16SPAliasReg() {
   unsigned Mips16SPAliasReg =
     MF->getInfo<MipsFunctionInfo>()->getMips16SPAliasReg();
-  return CurDAG->getRegister(Mips16SPAliasReg,
-                             getTargetLowering()->getPointerTy());
+  auto PtrVT = getTargetLowering()->getPointerTy(CurDAG->getDataLayout());
+  return CurDAG->getRegister(Mips16SPAliasReg, PtrVT);
 }
 
 void Mips16DAGToDAGISel::getMips16SPRefReg(SDNode *Parent, SDValue &AliasReg) {
-  SDValue AliasFPReg = CurDAG->getRegister(Mips::S0,
-                                           getTargetLowering()->getPointerTy());
+  auto PtrVT = getTargetLowering()->getPointerTy(CurDAG->getDataLayout());
+  SDValue AliasFPReg = CurDAG->getRegister(Mips::S0, PtrVT);
   if (Parent) {
     switch (Parent->getOpcode()) {
       case ISD::LOAD: {
@@ -155,7 +155,7 @@ void Mips16DAGToDAGISel::getMips16SPRefReg(SDNode *Parent, SDValue &AliasReg) {
       }
     }
   }
-  AliasReg = CurDAG->getRegister(Mips::SP, getTargetLowering()->getPointerTy());
+  AliasReg = CurDAG->getRegister(Mips::SP, PtrVT);
   return;
 
 }
@@ -163,14 +163,15 @@ void Mips16DAGToDAGISel::getMips16SPRefReg(SDNode *Parent, SDValue &AliasReg) {
 bool Mips16DAGToDAGISel::selectAddr16(
   SDNode *Parent, SDValue Addr, SDValue &Base, SDValue &Offset,
   SDValue &Alias) {
+  SDLoc DL(Addr);
   EVT ValTy = Addr.getValueType();
 
-  Alias = CurDAG->getTargetConstant(0, ValTy);
+  Alias = CurDAG->getTargetConstant(0, DL, ValTy);
 
   // if Address is FI, get the TargetFrameIndex.
   if (FrameIndexSDNode *FIN = dyn_cast<FrameIndexSDNode>(Addr)) {
     Base   = CurDAG->getTargetFrameIndex(FIN->getIndex(), ValTy);
-    Offset = CurDAG->getTargetConstant(0, ValTy);
+    Offset = CurDAG->getTargetConstant(0, DL, ValTy);
     getMips16SPRefReg(Parent, Alias);
     return true;
   }
@@ -199,7 +200,7 @@ bool Mips16DAGToDAGISel::selectAddr16(
       else
         Base = Addr.getOperand(0);
 
-      Offset = CurDAG->getTargetConstant(CN->getZExtValue(), ValTy);
+      Offset = CurDAG->getTargetConstant(CN->getZExtValue(), DL, ValTy);
       return true;
     }
   }
@@ -235,7 +236,7 @@ bool Mips16DAGToDAGISel::selectAddr16(
     }
   }
   Base   = Addr;
-  Offset = CurDAG->getTargetConstant(0, ValTy);
+  Offset = CurDAG->getTargetConstant(0, DL, ValTy);
   return true;
 }
 
diff --git a/lib/Target/Mips/Mips16ISelLowering.cpp b/lib/Target/Mips/Mips16ISelLowering.cpp
index ede4f3748f1..3522cbb1f36 100644
--- a/lib/Target/Mips/Mips16ISelLowering.cpp
+++ b/lib/Target/Mips/Mips16ISelLowering.cpp
@@ -127,7 +127,7 @@ Mips16TargetLowering::Mips16TargetLowering(const MipsTargetMachine &TM,
   // Set up the register classes
   addRegisterClass(MVT::i32, &Mips::CPU16RegsRegClass);
 
-  if (!TM.Options.UseSoftFloat)
+  if (!Subtarget.useSoftFloat())
     setMips16HardFloatLibCalls();
 
   setOperationAction(ISD::ATOMIC_FENCE,       MVT::Other, Expand);
@@ -502,7 +502,8 @@ getOpndList(SmallVectorImpl<SDValue> &Ops,
     unsigned V0Reg = Mips::V0;
     if (NeedMips16Helper) {
       RegsToPass.push_front(std::make_pair(V0Reg, Callee));
-      JumpTarget = DAG.getExternalSymbol(Mips16HelperFunction, getPointerTy());
+      JumpTarget = DAG.getExternalSymbol(Mips16HelperFunction,
+                                         getPointerTy(DAG.getDataLayout()));
       ExternalSymbolSDNode *S = cast<ExternalSymbolSDNode>(JumpTarget);
       JumpTarget = getAddrGlobal(S, CLI.DL, JumpTarget.getValueType(), DAG,
                                  MipsII::MO_GOT, Chain,
diff --git a/lib/Target/Mips/Mips32r6InstrFormats.td b/lib/Target/Mips/Mips32r6InstrFormats.td
index e9a4289ec24..13216bebe5b 100644
--- a/lib/Target/Mips/Mips32r6InstrFormats.td
+++ b/lib/Target/Mips/Mips32r6InstrFormats.td
@@ -11,6 +11,25 @@
 //
 //===----------------------------------------------------------------------===//
 
+class R6MMR6Rel;
+
+def MipsR62MicroMipsR6 : InstrMapping {
+  let FilterClass = "R6MMR6Rel";
+  // Instructions with the same BaseOpcode and isNVStore values form a row.
+  let RowFields = ["BaseOpcode"];
+  // Instructions with the same predicate sense form a column.
+  let ColFields = ["Arch"];
+  // The key column is the unpredicated instructions.
+  let KeyCol = ["mipsr6"];
+  // Value columns are PredSense=true and PredSense=false
+  let ValueCols = [["mipsr6"], ["micromipsr6"]];
+}
+
+class MipsR6Arch<string opstr> {
+  string Arch = "mipsr6";
+  string BaseOpcode = opstr;
+}
+
 class MipsR6Inst : MipsInst<(outs), (ins), "", [], NoItinerary, FrmOther>,
                    PredicateControl {
   let DecoderNamespace = "Mips32r6_64r6";
diff --git a/lib/Target/Mips/Mips32r6InstrInfo.td b/lib/Target/Mips/Mips32r6InstrInfo.td
index 49c63226dc0..d6ab8a6e541 100644
--- a/lib/Target/Mips/Mips32r6InstrInfo.td
+++ b/lib/Target/Mips/Mips32r6InstrInfo.td
@@ -188,52 +188,52 @@ multiclass CMP_CC_M <FIELD_CMP_FORMAT Format, string Typestr,
                      RegisterOperand FGROpnd>{
   def CMP_F_#NAME : COP1_CMP_CONDN_FM<Format, FIELD_CMP_COND_AF>,
                     CMP_CONDN_DESC_BASE<"af", Typestr, FGROpnd>,
-                    ISA_MIPS32R6;
+                    ISA_MIPS32R6, HARDFLOAT;
   def CMP_UN_#NAME : COP1_CMP_CONDN_FM<Format, FIELD_CMP_COND_UN>,
                      CMP_CONDN_DESC_BASE<"un", Typestr, FGROpnd, setuo>,
-                     ISA_MIPS32R6;
+                     ISA_MIPS32R6, HARDFLOAT;
   def CMP_EQ_#NAME : COP1_CMP_CONDN_FM<Format, FIELD_CMP_COND_EQ>,
                      CMP_CONDN_DESC_BASE<"eq", Typestr, FGROpnd, setoeq>,
-                     ISA_MIPS32R6;
+                     ISA_MIPS32R6, HARDFLOAT;
   def CMP_UEQ_#NAME : COP1_CMP_CONDN_FM<Format, FIELD_CMP_COND_UEQ>,
                       CMP_CONDN_DESC_BASE<"ueq", Typestr, FGROpnd, setueq>,
-                      ISA_MIPS32R6;
+                      ISA_MIPS32R6, HARDFLOAT;
   def CMP_LT_#NAME : COP1_CMP_CONDN_FM<Format, FIELD_CMP_COND_LT>,
                      CMP_CONDN_DESC_BASE<"lt", Typestr, FGROpnd, setolt>,
-                     ISA_MIPS32R6;
+                     ISA_MIPS32R6, HARDFLOAT;
   def CMP_ULT_#NAME : COP1_CMP_CONDN_FM<Format, FIELD_CMP_COND_ULT>,
                       CMP_CONDN_DESC_BASE<"ult", Typestr, FGROpnd, setult>,
-                      ISA_MIPS32R6;
+                      ISA_MIPS32R6, HARDFLOAT;
   def CMP_LE_#NAME : COP1_CMP_CONDN_FM<Format, FIELD_CMP_COND_LE>,
                      CMP_CONDN_DESC_BASE<"le", Typestr, FGROpnd, setole>,
-                     ISA_MIPS32R6;
+                     ISA_MIPS32R6, HARDFLOAT;
   def CMP_ULE_#NAME : COP1_CMP_CONDN_FM<Format, FIELD_CMP_COND_ULE>,
                       CMP_CONDN_DESC_BASE<"ule", Typestr, FGROpnd, setule>,
-                      ISA_MIPS32R6;
+                      ISA_MIPS32R6, HARDFLOAT;
   def CMP_SAF_#NAME : COP1_CMP_CONDN_FM<Format, FIELD_CMP_COND_SAF>,
                       CMP_CONDN_DESC_BASE<"saf", Typestr, FGROpnd>,
-                      ISA_MIPS32R6;
+                      ISA_MIPS32R6, HARDFLOAT;
   def CMP_SUN_#NAME : COP1_CMP_CONDN_FM<Format, FIELD_CMP_COND_SUN>,
                       CMP_CONDN_DESC_BASE<"sun", Typestr, FGROpnd>,
-                      ISA_MIPS32R6;
+                      ISA_MIPS32R6, HARDFLOAT;
   def CMP_SEQ_#NAME : COP1_CMP_CONDN_FM<Format, FIELD_CMP_COND_SEQ>,
                       CMP_CONDN_DESC_BASE<"seq", Typestr, FGROpnd>,
-                      ISA_MIPS32R6;
+                      ISA_MIPS32R6, HARDFLOAT;
   def CMP_SUEQ_#NAME : COP1_CMP_CONDN_FM<Format, FIELD_CMP_COND_SUEQ>,
                        CMP_CONDN_DESC_BASE<"sueq", Typestr, FGROpnd>,
-                       ISA_MIPS32R6;
+                       ISA_MIPS32R6, HARDFLOAT;
   def CMP_SLT_#NAME : COP1_CMP_CONDN_FM<Format, FIELD_CMP_COND_SLT>,
                       CMP_CONDN_DESC_BASE<"slt", Typestr, FGROpnd>,
-                      ISA_MIPS32R6;
+                      ISA_MIPS32R6, HARDFLOAT;
   def CMP_SULT_#NAME : COP1_CMP_CONDN_FM<Format, FIELD_CMP_COND_SULT>,
                        CMP_CONDN_DESC_BASE<"sult", Typestr, FGROpnd>,
-                       ISA_MIPS32R6;
+                       ISA_MIPS32R6, HARDFLOAT;
   def CMP_SLE_#NAME : COP1_CMP_CONDN_FM<Format, FIELD_CMP_COND_SLE>,
                       CMP_CONDN_DESC_BASE<"sle", Typestr, FGROpnd>,
-                      ISA_MIPS32R6;
+                      ISA_MIPS32R6, HARDFLOAT;
   def CMP_SULE_#NAME : COP1_CMP_CONDN_FM<Format, FIELD_CMP_COND_SULE>,
                        CMP_CONDN_DESC_BASE<"sule", Typestr, FGROpnd>,
-                       ISA_MIPS32R6;
+                       ISA_MIPS32R6, HARDFLOAT;
 }
 
 //===----------------------------------------------------------------------===//
@@ -243,7 +243,7 @@ multiclass CMP_CC_M <FIELD_CMP_FORMAT Format, string Typestr,
 //===----------------------------------------------------------------------===//
 
 class PCREL_DESC_BASE<string instr_asm, RegisterOperand GPROpnd,
-                      Operand ImmOpnd> {
+                      Operand ImmOpnd> : MipsR6Arch<instr_asm> {
   dag OutOperandList = (outs GPROpnd:$rs);
   dag InOperandList = (ins ImmOpnd:$imm);
   string AsmString = !strconcat(instr_asm, "\t$rs, $imm");
@@ -255,7 +255,7 @@ class LWPC_DESC: PCREL_DESC_BASE<"lwpc", GPR32Opnd, simm19_lsl2>;
 class LWUPC_DESC: PCREL_DESC_BASE<"lwupc", GPR32Opnd, simm19_lsl2>;
 
 class ALIGN_DESC_BASE<string instr_asm, RegisterOperand GPROpnd,
-                      Operand ImmOpnd> {
+                      Operand ImmOpnd>  : MipsR6Arch<instr_asm> {
   dag OutOperandList = (outs GPROpnd:$rd);
   dag InOperandList = (ins GPROpnd:$rs, GPROpnd:$rt, ImmOpnd:$bp);
   string AsmString = !strconcat(instr_asm, "\t$rd, $rs, $rt, $bp");
@@ -264,7 +264,8 @@ class ALIGN_DESC_BASE<string instr_asm, RegisterOperand GPROpnd,
 
 class ALIGN_DESC : ALIGN_DESC_BASE<"align", GPR32Opnd, uimm2>;
 
-class ALUIPC_DESC_BASE<string instr_asm, RegisterOperand GPROpnd> {
+class ALUIPC_DESC_BASE<string instr_asm, RegisterOperand GPROpnd>
+    : MipsR6Arch<instr_asm> {
   dag OutOperandList = (outs GPROpnd:$rs);
   dag InOperandList = (ins simm16:$imm);
   string AsmString = !strconcat(instr_asm, "\t$rs, $imm");
@@ -274,7 +275,8 @@ class ALUIPC_DESC_BASE<string instr_asm, RegisterOperand GPROpnd> {
 class ALUIPC_DESC : ALUIPC_DESC_BASE<"aluipc", GPR32Opnd>;
 class AUIPC_DESC : ALUIPC_DESC_BASE<"auipc", GPR32Opnd>;
 
-class AUI_DESC_BASE<string instr_asm, RegisterOperand GPROpnd> {
+class AUI_DESC_BASE<string instr_asm, RegisterOperand GPROpnd>
+    : MipsR6Arch<instr_asm> {
   dag OutOperandList = (outs GPROpnd:$rs);
   dag InOperandList = (ins GPROpnd:$rt, simm16:$imm);
   string AsmString = !strconcat(instr_asm, "\t$rs, $rt, $imm");
@@ -289,7 +291,8 @@ class BRANCH_DESC_BASE {
   bit hasDelaySlot = 0;
 }
 
-class BC_DESC_BASE<string instr_asm, DAGOperand opnd> : BRANCH_DESC_BASE {
+class BC_DESC_BASE<string instr_asm, DAGOperand opnd> : BRANCH_DESC_BASE,
+    MipsR6Arch<instr_asm> {
   dag InOperandList = (ins opnd:$offset);
   dag OutOperandList = (outs);
   string AsmString = !strconcat(instr_asm, "\t$offset");
@@ -313,7 +316,8 @@ class CMP_CBR_EQNE_Z_DESC_BASE<string instr_asm, DAGOperand opnd,
 }
 
 class CMP_CBR_RT_Z_DESC_BASE<string instr_asm, DAGOperand opnd,
-                             RegisterOperand GPROpnd> : BRANCH_DESC_BASE {
+                             RegisterOperand GPROpnd>
+    : BRANCH_DESC_BASE, MipsR6Arch<instr_asm> {
   dag InOperandList = (ins GPROpnd:$rt, opnd:$offset);
   dag OutOperandList = (outs);
   string AsmString = !strconcat(instr_asm, "\t$rt, $offset");
@@ -373,7 +377,8 @@ class BOVC_DESC   : CMP_BC_DESC_BASE<"bovc", brtarget, GPR32Opnd>;
 class BNVC_DESC   : CMP_BC_DESC_BASE<"bnvc", brtarget, GPR32Opnd>;
 
 class JMP_IDX_COMPACT_DESC_BASE<string opstr, DAGOperand opnd,
-                                RegisterOperand GPROpnd> {
+                                RegisterOperand GPROpnd>
+    : MipsR6Arch<opstr> {
   dag InOperandList = (ins GPROpnd:$rt, opnd:$offset);
   string AsmString = !strconcat(opstr, "\t$rt, $offset");
   list<dag> Pattern = [];
@@ -400,7 +405,8 @@ class JR_HB_R6_DESC : JR_HB_DESC_BASE<"jr.hb", GPR32Opnd> {
   bit isBarrier=1;
 }
 
-class BITSWAP_DESC_BASE<string instr_asm, RegisterOperand GPROpnd> {
+class BITSWAP_DESC_BASE<string instr_asm, RegisterOperand GPROpnd>
+    : MipsR6Arch<instr_asm> {
   dag OutOperandList = (outs GPROpnd:$rd);
   dag InOperandList = (ins GPROpnd:$rt);
   string AsmString = !strconcat(instr_asm, "\t$rd, $rt");
@@ -410,7 +416,8 @@ class BITSWAP_DESC_BASE<string instr_asm, RegisterOperand GPROpnd> {
 class BITSWAP_DESC : BITSWAP_DESC_BASE<"bitswap", GPR32Opnd>;
 
 class DIVMOD_DESC_BASE<string instr_asm, RegisterOperand GPROpnd,
-                       SDPatternOperator Op=null_frag> {
+                       SDPatternOperator Op=null_frag>
+    : MipsR6Arch<instr_asm> {
   dag OutOperandList = (outs GPROpnd:$rd);
   dag InOperandList = (ins GPROpnd:$rs, GPROpnd:$rt);
   string AsmString = !strconcat(instr_asm, "\t$rd, $rs, $rt");
@@ -451,7 +458,7 @@ class BNEZALC_DESC : CMP_CBR_RT_Z_DESC_BASE<"bnezalc", brtarget, GPR32Opnd> {
 }
 
 class MUL_R6_DESC_BASE<string instr_asm, RegisterOperand GPROpnd,
-                       SDPatternOperator Op=null_frag> {
+                       SDPatternOperator Op=null_frag> : MipsR6Arch<instr_asm> {
   dag OutOperandList = (outs GPROpnd:$rd);
   dag InOperandList = (ins GPROpnd:$rs, GPROpnd:$rt);
   string AsmString = !strconcat(instr_asm, "\t$rd, $rs, $rt");
@@ -479,7 +486,8 @@ class SEL_D_DESC : COP1_SEL_DESC_BASE<"sel.d", FGR64Opnd> {
 }
 class SEL_S_DESC : COP1_SEL_DESC_BASE<"sel.s", FGR32Opnd>;
 
-class SELEQNE_Z_DESC_BASE<string instr_asm, RegisterOperand GPROpnd> {
+class SELEQNE_Z_DESC_BASE<string instr_asm, RegisterOperand GPROpnd>
+    : MipsR6Arch<instr_asm> {
   dag OutOperandList = (outs GPROpnd:$rd);
   dag InOperandList = (ins GPROpnd:$rs, GPROpnd:$rt);
   string AsmString = !strconcat(instr_asm, "\t$rd, $rs, $rt");
@@ -544,7 +552,7 @@ class CLASS_S_DESC : CLASS_RINT_DESC_BASE<"class.s", FGR32Opnd>;
 class CLASS_D_DESC : CLASS_RINT_DESC_BASE<"class.d", FGR64Opnd>;
 
 class CACHE_HINT_DESC<string instr_asm, Operand MemOpnd,
-                      RegisterOperand GPROpnd> {
+                      RegisterOperand GPROpnd> : MipsR6Arch<instr_asm> {
   dag OutOperandList = (outs);
   dag InOperandList = (ins MemOpnd:$addr, uimm5:$hint);
   string AsmString = !strconcat(instr_asm, "\t$hint, $addr");
@@ -580,7 +588,7 @@ class SDC2_R6_DESC : COP2ST_DESC_BASE<"sdc2", COP2Opnd>;
 class SWC2_R6_DESC : COP2ST_DESC_BASE<"swc2", COP2Opnd>;
 
 class LSA_R6_DESC_BASE<string instr_asm, RegisterOperand GPROpnd,
-                       Operand ImmOpnd> {
+                       Operand ImmOpnd> : MipsR6Arch<instr_asm> {
   dag OutOperandList = (outs GPROpnd:$rd);
   dag InOperandList = (ins GPROpnd:$rs, GPROpnd:$rt, ImmOpnd:$imm2);
   string AsmString = !strconcat(instr_asm, "\t$rd, $rs, $rt, $imm2");
@@ -610,7 +618,8 @@ class SC_R6_DESC_BASE<string instr_asm, RegisterOperand GPROpnd> {
 
 class SC_R6_DESC : SC_R6_DESC_BASE<"sc", GPR32Opnd>;
 
-class CLO_CLZ_R6_DESC_BASE<string instr_asm, RegisterOperand GPROpnd> {
+class CLO_CLZ_R6_DESC_BASE<string instr_asm, RegisterOperand GPROpnd>
+    : MipsR6Arch<instr_asm> {
   dag OutOperandList = (outs GPROpnd:$rd);
   dag InOperandList = (ins GPROpnd:$rs);
   string AsmString = !strconcat(instr_asm, "\t$rd, $rs");
@@ -642,90 +651,90 @@ class SDBBP_R6_DESC {
 //
 //===----------------------------------------------------------------------===//
 
-def ADDIUPC : ADDIUPC_ENC, ADDIUPC_DESC, ISA_MIPS32R6;
-def ALIGN : ALIGN_ENC, ALIGN_DESC, ISA_MIPS32R6;
-def ALUIPC : ALUIPC_ENC, ALUIPC_DESC, ISA_MIPS32R6;
-def AUI : AUI_ENC, AUI_DESC, ISA_MIPS32R6;
-def AUIPC : AUIPC_ENC, AUIPC_DESC, ISA_MIPS32R6;
+def ADDIUPC : R6MMR6Rel, ADDIUPC_ENC, ADDIUPC_DESC, ISA_MIPS32R6;
+def ALIGN : R6MMR6Rel, ALIGN_ENC, ALIGN_DESC, ISA_MIPS32R6;
+def ALUIPC : R6MMR6Rel, ALUIPC_ENC, ALUIPC_DESC, ISA_MIPS32R6;
+def AUI : R6MMR6Rel, AUI_ENC, AUI_DESC, ISA_MIPS32R6;
+def AUIPC : R6MMR6Rel, AUIPC_ENC, AUIPC_DESC, ISA_MIPS32R6;
 def BAL : BAL_ENC, BAL_DESC, ISA_MIPS32R6;
-def BALC : BALC_ENC, BALC_DESC, ISA_MIPS32R6;
-def BC1EQZ : BC1EQZ_ENC, BC1EQZ_DESC, ISA_MIPS32R6;
-def BC1NEZ : BC1NEZ_ENC, BC1NEZ_DESC, ISA_MIPS32R6;
+def BALC : R6MMR6Rel, BALC_ENC, BALC_DESC, ISA_MIPS32R6;
+def BC1EQZ : BC1EQZ_ENC, BC1EQZ_DESC, ISA_MIPS32R6, HARDFLOAT;
+def BC1NEZ : BC1NEZ_ENC, BC1NEZ_DESC, ISA_MIPS32R6, HARDFLOAT;
 def BC2EQZ : BC2EQZ_ENC, BC2EQZ_DESC, ISA_MIPS32R6;
 def BC2NEZ : BC2NEZ_ENC, BC2NEZ_DESC, ISA_MIPS32R6;
-def BC : BC_ENC, BC_DESC, ISA_MIPS32R6;
+def BC : R6MMR6Rel, BC_ENC, BC_DESC, ISA_MIPS32R6;
 def BEQC : BEQC_ENC, BEQC_DESC, ISA_MIPS32R6;
-def BEQZALC : BEQZALC_ENC, BEQZALC_DESC, ISA_MIPS32R6;
+def BEQZALC : R6MMR6Rel, BEQZALC_ENC, BEQZALC_DESC, ISA_MIPS32R6;
 def BEQZC : BEQZC_ENC, BEQZC_DESC, ISA_MIPS32R6;
 def BGEC : BGEC_ENC, BGEC_DESC, ISA_MIPS32R6;
 def BGEUC : BGEUC_ENC, BGEUC_DESC, ISA_MIPS32R6;
-def BGEZALC : BGEZALC_ENC, BGEZALC_DESC, ISA_MIPS32R6;
+def BGEZALC : R6MMR6Rel, BGEZALC_ENC, BGEZALC_DESC, ISA_MIPS32R6;
 def BGEZC : BGEZC_ENC, BGEZC_DESC, ISA_MIPS32R6;
-def BGTZALC : BGTZALC_ENC, BGTZALC_DESC, ISA_MIPS32R6;
+def BGTZALC : R6MMR6Rel, BGTZALC_ENC, BGTZALC_DESC, ISA_MIPS32R6;
 def BGTZC : BGTZC_ENC, BGTZC_DESC, ISA_MIPS32R6;
-def BITSWAP : BITSWAP_ENC, BITSWAP_DESC, ISA_MIPS32R6;
-def BLEZALC : BLEZALC_ENC, BLEZALC_DESC, ISA_MIPS32R6;
+def BITSWAP : R6MMR6Rel, BITSWAP_ENC, BITSWAP_DESC, ISA_MIPS32R6;
+def BLEZALC : R6MMR6Rel, BLEZALC_ENC, BLEZALC_DESC, ISA_MIPS32R6;
 def BLEZC : BLEZC_ENC, BLEZC_DESC, ISA_MIPS32R6;
 def BLTC : BLTC_ENC, BLTC_DESC, ISA_MIPS32R6;
 def BLTUC : BLTUC_ENC, BLTUC_DESC, ISA_MIPS32R6;
-def BLTZALC : BLTZALC_ENC, BLTZALC_DESC, ISA_MIPS32R6;
+def BLTZALC : R6MMR6Rel, BLTZALC_ENC, BLTZALC_DESC, ISA_MIPS32R6;
 def BLTZC : BLTZC_ENC, BLTZC_DESC, ISA_MIPS32R6;
 def BNEC : BNEC_ENC, BNEC_DESC, ISA_MIPS32R6;
-def BNEZALC : BNEZALC_ENC, BNEZALC_DESC, ISA_MIPS32R6;
+def BNEZALC : R6MMR6Rel, BNEZALC_ENC, BNEZALC_DESC, ISA_MIPS32R6;
 def BNEZC : BNEZC_ENC, BNEZC_DESC, ISA_MIPS32R6;
 def BNVC : BNVC_ENC, BNVC_DESC, ISA_MIPS32R6;
 def BOVC : BOVC_ENC, BOVC_DESC, ISA_MIPS32R6;
-def CACHE_R6 : CACHE_ENC, CACHE_DESC, ISA_MIPS32R6;
-def CLASS_D : CLASS_D_ENC, CLASS_D_DESC, ISA_MIPS32R6;
-def CLASS_S : CLASS_S_ENC, CLASS_S_DESC, ISA_MIPS32R6;
-def CLO_R6 : CLO_R6_ENC, CLO_R6_DESC, ISA_MIPS32R6;
-def CLZ_R6 : CLZ_R6_ENC, CLZ_R6_DESC, ISA_MIPS32R6;
+def CACHE_R6 : R6MMR6Rel, CACHE_ENC, CACHE_DESC, ISA_MIPS32R6;
+def CLASS_D : CLASS_D_ENC, CLASS_D_DESC, ISA_MIPS32R6, HARDFLOAT;
+def CLASS_S : CLASS_S_ENC, CLASS_S_DESC, ISA_MIPS32R6, HARDFLOAT;
+def CLO_R6 : R6MMR6Rel, CLO_R6_ENC, CLO_R6_DESC, ISA_MIPS32R6;
+def CLZ_R6 : R6MMR6Rel, CLZ_R6_ENC, CLZ_R6_DESC, ISA_MIPS32R6;
 defm S : CMP_CC_M<FIELD_CMP_FORMAT_S, "s", FGR32Opnd>;
 defm D : CMP_CC_M<FIELD_CMP_FORMAT_D, "d", FGR64Opnd>;
-def DIV : DIV_ENC, DIV_DESC, ISA_MIPS32R6;
-def DIVU : DIVU_ENC, DIVU_DESC, ISA_MIPS32R6;
-def JIALC : JIALC_ENC, JIALC_DESC, ISA_MIPS32R6;
-def JIC : JIC_ENC, JIC_DESC, ISA_MIPS32R6;
+def DIV : R6MMR6Rel, DIV_ENC, DIV_DESC, ISA_MIPS32R6;
+def DIVU : R6MMR6Rel, DIVU_ENC, DIVU_DESC, ISA_MIPS32R6;
+def JIALC : R6MMR6Rel, JIALC_ENC, JIALC_DESC, ISA_MIPS32R6;
+def JIC : R6MMR6Rel, JIC_ENC, JIC_DESC, ISA_MIPS32R6;
 def JR_HB_R6 : JR_HB_R6_ENC, JR_HB_R6_DESC, ISA_MIPS32R6;
 def LDC2_R6 : LDC2_R6_ENC, LDC2_R6_DESC, ISA_MIPS32R6;
 def LL_R6 : LL_R6_ENC, LL_R6_DESC, ISA_MIPS32R6;
-def LSA_R6 : LSA_R6_ENC, LSA_R6_DESC, ISA_MIPS32R6;
+def LSA_R6 : R6MMR6Rel, LSA_R6_ENC, LSA_R6_DESC, ISA_MIPS32R6;
 def LWC2_R6 : LWC2_R6_ENC, LWC2_R6_DESC, ISA_MIPS32R6;
-def LWPC : LWPC_ENC, LWPC_DESC, ISA_MIPS32R6;
+def LWPC : R6MMR6Rel, LWPC_ENC, LWPC_DESC, ISA_MIPS32R6;
 def LWUPC : LWUPC_ENC, LWUPC_DESC, ISA_MIPS32R6;
-def MADDF_S : MADDF_S_ENC, MADDF_S_DESC, ISA_MIPS32R6;
-def MADDF_D : MADDF_D_ENC, MADDF_D_DESC, ISA_MIPS32R6;
-def MAXA_D : MAXA_D_ENC, MAXA_D_DESC, ISA_MIPS32R6;
-def MAXA_S : MAXA_S_ENC, MAXA_S_DESC, ISA_MIPS32R6;
-def MAX_D : MAX_D_ENC, MAX_D_DESC, ISA_MIPS32R6;
-def MAX_S : MAX_S_ENC, MAX_S_DESC, ISA_MIPS32R6;
-def MINA_D : MINA_D_ENC, MINA_D_DESC, ISA_MIPS32R6;
-def MINA_S : MINA_S_ENC, MINA_S_DESC, ISA_MIPS32R6;
-def MIN_D : MIN_D_ENC, MIN_D_DESC, ISA_MIPS32R6;
-def MIN_S : MIN_S_ENC, MIN_S_DESC, ISA_MIPS32R6;
-def MOD : MOD_ENC, MOD_DESC, ISA_MIPS32R6;
-def MODU : MODU_ENC, MODU_DESC, ISA_MIPS32R6;
-def MSUBF_S : MSUBF_S_ENC, MSUBF_S_DESC, ISA_MIPS32R6;
-def MSUBF_D : MSUBF_D_ENC, MSUBF_D_DESC, ISA_MIPS32R6;
-def MUH    : MUH_ENC, MUH_DESC, ISA_MIPS32R6;
-def MUHU   : MUHU_ENC, MUHU_DESC, ISA_MIPS32R6;
-def MUL_R6 : MUL_R6_ENC, MUL_R6_DESC, ISA_MIPS32R6;
-def MULU   : MULU_ENC, MULU_DESC, ISA_MIPS32R6;
+def MADDF_S : MADDF_S_ENC, MADDF_S_DESC, ISA_MIPS32R6, HARDFLOAT;
+def MADDF_D : MADDF_D_ENC, MADDF_D_DESC, ISA_MIPS32R6, HARDFLOAT;
+def MAXA_D  : MAXA_D_ENC, MAXA_D_DESC, ISA_MIPS32R6, HARDFLOAT;
+def MAXA_S  : MAXA_S_ENC, MAXA_S_DESC, ISA_MIPS32R6, HARDFLOAT;
+def MAX_D   : MAX_D_ENC, MAX_D_DESC, ISA_MIPS32R6, HARDFLOAT;
+def MAX_S   : MAX_S_ENC, MAX_S_DESC, ISA_MIPS32R6, HARDFLOAT;
+def MINA_D  : MINA_D_ENC, MINA_D_DESC, ISA_MIPS32R6, HARDFLOAT;
+def MINA_S  : MINA_S_ENC, MINA_S_DESC, ISA_MIPS32R6, HARDFLOAT;
+def MIN_D   : MIN_D_ENC, MIN_D_DESC, ISA_MIPS32R6, HARDFLOAT;
+def MIN_S   : MIN_S_ENC, MIN_S_DESC, ISA_MIPS32R6, HARDFLOAT;
+def MOD : R6MMR6Rel, MOD_ENC, MOD_DESC, ISA_MIPS32R6;
+def MODU : R6MMR6Rel, MODU_ENC, MODU_DESC, ISA_MIPS32R6;
+def MSUBF_S : MSUBF_S_ENC, MSUBF_S_DESC, ISA_MIPS32R6, HARDFLOAT;
+def MSUBF_D : MSUBF_D_ENC, MSUBF_D_DESC, ISA_MIPS32R6, HARDFLOAT;
+def MUH    : R6MMR6Rel, MUH_ENC, MUH_DESC, ISA_MIPS32R6;
+def MUHU   : R6MMR6Rel, MUHU_ENC, MUHU_DESC, ISA_MIPS32R6;
+def MUL_R6 : R6MMR6Rel, MUL_R6_ENC, MUL_R6_DESC, ISA_MIPS32R6;
+def MULU   : R6MMR6Rel, MULU_ENC, MULU_DESC, ISA_MIPS32R6;
 def NAL; // BAL with rd=0
-def PREF_R6 : PREF_ENC, PREF_DESC, ISA_MIPS32R6;
-def RINT_D : RINT_D_ENC, RINT_D_DESC, ISA_MIPS32R6;
-def RINT_S : RINT_S_ENC, RINT_S_DESC, ISA_MIPS32R6;
+def PREF_R6 : R6MMR6Rel, PREF_ENC, PREF_DESC, ISA_MIPS32R6;
+def RINT_D : RINT_D_ENC, RINT_D_DESC, ISA_MIPS32R6, HARDFLOAT;
+def RINT_S : RINT_S_ENC, RINT_S_DESC, ISA_MIPS32R6, HARDFLOAT;
 def SC_R6 : SC_R6_ENC, SC_R6_DESC, ISA_MIPS32R6;
 def SDBBP_R6 : SDBBP_R6_ENC, SDBBP_R6_DESC, ISA_MIPS32R6;
 def SDC2_R6 : SDC2_R6_ENC, SDC2_R6_DESC, ISA_MIPS32R6;
-def SELEQZ : SELEQZ_ENC, SELEQZ_DESC, ISA_MIPS32R6, GPR_32;
-def SELEQZ_D : SELEQZ_D_ENC, SELEQZ_D_DESC, ISA_MIPS32R6;
-def SELEQZ_S : SELEQZ_S_ENC, SELEQZ_S_DESC, ISA_MIPS32R6;
-def SELNEZ : SELNEZ_ENC, SELNEZ_DESC, ISA_MIPS32R6, GPR_32;
-def SELNEZ_D : SELNEZ_D_ENC, SELNEZ_D_DESC, ISA_MIPS32R6;
-def SELNEZ_S : SELNEZ_S_ENC, SELNEZ_S_DESC, ISA_MIPS32R6;
-def SEL_D : SEL_D_ENC, SEL_D_DESC, ISA_MIPS32R6;
-def SEL_S : SEL_S_ENC, SEL_S_DESC, ISA_MIPS32R6;
+def SELEQZ : R6MMR6Rel, SELEQZ_ENC, SELEQZ_DESC, ISA_MIPS32R6, GPR_32;
+def SELEQZ_D : SELEQZ_D_ENC, SELEQZ_D_DESC, ISA_MIPS32R6, HARDFLOAT;
+def SELEQZ_S : SELEQZ_S_ENC, SELEQZ_S_DESC, ISA_MIPS32R6, HARDFLOAT;
+def SELNEZ : R6MMR6Rel, SELNEZ_ENC, SELNEZ_DESC, ISA_MIPS32R6, GPR_32;
+def SELNEZ_D : SELNEZ_D_ENC, SELNEZ_D_DESC, ISA_MIPS32R6, HARDFLOAT;
+def SELNEZ_S : SELNEZ_S_ENC, SELNEZ_S_DESC, ISA_MIPS32R6, HARDFLOAT;
+def SEL_D    : SEL_D_ENC, SEL_D_DESC, ISA_MIPS32R6, HARDFLOAT;
+def SEL_S    : SEL_S_ENC, SEL_S_DESC, ISA_MIPS32R6, HARDFLOAT;
 def SWC2_R6 : SWC2_R6_ENC, SWC2_R6_DESC, ISA_MIPS32R6;
 
 //===----------------------------------------------------------------------===//
diff --git a/lib/Target/Mips/Mips64InstrInfo.td b/lib/Target/Mips/Mips64InstrInfo.td
index b1cb7f7352c..c37cf95cadc 100644
--- a/lib/Target/Mips/Mips64InstrInfo.td
+++ b/lib/Target/Mips/Mips64InstrInfo.td
@@ -27,8 +27,6 @@ def uimm16_64      : Operand<i64> {
 // Signed Operand
 def simm10_64 : Operand<i64>;
 
-def imm64: Operand<i64>;
-
 // Transformation Function - get Imm - 32.
 def Subtract32 : SDNodeXForm<imm, [{
   return getImm(N, (unsigned)N->getZExtValue() - 32);
@@ -305,8 +303,9 @@ let isCodeGenOnly = 1, rs = 0, shamt = 0 in {
 def LONG_BRANCH_DADDiu : PseudoSE<(outs GPR64Opnd:$dst),
   (ins GPR64Opnd:$src, brtarget:$tgt, brtarget:$baltgt), []>;
 
-// Cavium Octeon cmMIPS instructions
-let EncodingPredicates = []<Predicate>, // FIXME: The lack of HasStdEnc is probably a bug
+// Cavium Octeon cnMIPS instructions
+let DecoderNamespace = "CnMips",
+    EncodingPredicates = []<Predicate>, // FIXME: The lack of HasStdEnc is probably a bug
     AdditionalPredicates = [HasCnMips] in {
 
 class Count1s<string opstr, RegisterOperand RO>:
@@ -353,6 +352,10 @@ class CBranchBitNum<string opstr, DAGOperand opnd, PatFrag cond_op,
   let Defs = [AT];
 }
 
+class MFC2OP<string asmstr, RegisterOperand RO> :
+  InstSE<(outs RO:$rt, uimm16:$imm16), (ins),
+         !strconcat(asmstr, "\t$rt, $imm16"), [], NoItinerary, FrmFR>;
+
 // Unsigned Byte Add
 let Pattern = [(set GPR64Opnd:$rd,
                     (and (add GPR64Opnd:$rs, GPR64Opnd:$rt), 255))] in
@@ -415,16 +418,19 @@ let Defs = [MPL1, MPL2, P0, P1, P2] in
 def VMULU : ArithLogicR<"vmulu", GPR64Opnd, 0, II_DMUL>,
                                  ADD_FM<0x1c, 0x0f>;
 
+// Move between CPU and coprocessor registers
+def DMFC2_OCTEON : MFC2OP<"dmfc2", GPR64Opnd>, MFC2OP_FM<0x12, 1>;
+def DMTC2_OCTEON : MFC2OP<"dmtc2", GPR64Opnd>, MFC2OP_FM<0x12, 5>;
 }
 
 }
 
 /// Move between CPU and coprocessor registers
 let DecoderNamespace = "Mips64", Predicates = [HasMips64] in {
-def DMFC0 : MFC3OP<"dmfc0", GPR64Opnd>, MFC3OP_FM<0x10, 1>;
-def DMTC0 : MFC3OP<"dmtc0", GPR64Opnd>, MFC3OP_FM<0x10, 5>, ISA_MIPS3;
-def DMFC2 : MFC3OP<"dmfc2", GPR64Opnd>, MFC3OP_FM<0x12, 1>, ISA_MIPS3;
-def DMTC2 : MFC3OP<"dmtc2", GPR64Opnd>, MFC3OP_FM<0x12, 5>, ISA_MIPS3;
+def DMFC0 : MFC3OP<"dmfc0", GPR64Opnd, COP0Opnd>, MFC3OP_FM<0x10, 1>, ISA_MIPS3;
+def DMTC0 : MTC3OP<"dmtc0", COP0Opnd, GPR64Opnd>, MFC3OP_FM<0x10, 5>, ISA_MIPS3;
+def DMFC2 : MFC3OP<"dmfc2", GPR64Opnd, COP2Opnd>, MFC3OP_FM<0x12, 1>, ISA_MIPS3;
+def DMTC2 : MTC3OP<"dmtc2", COP2Opnd, GPR64Opnd>, MFC3OP_FM<0x12, 5>, ISA_MIPS3;
 }
 
 //===----------------------------------------------------------------------===//
@@ -502,6 +508,16 @@ def : MipsPat<(i32 (trunc (or GPR64:$lhs, GPR64:$rhs))),
 def : MipsPat<(i32 (trunc (xor GPR64:$lhs, GPR64:$rhs))),
               (EXTRACT_SUBREG (XOR64 GPR64:$lhs, GPR64:$rhs), sub_32)>;
 
+// variable shift instructions patterns
+def : MipsPat<(shl GPR64:$rt, (i32 (trunc GPR64:$rs))),
+              (DSLLV GPR64:$rt, (EXTRACT_SUBREG GPR64:$rs, sub_32))>;
+def : MipsPat<(srl GPR64:$rt, (i32 (trunc GPR64:$rs))),
+              (DSRLV GPR64:$rt, (EXTRACT_SUBREG GPR64:$rs, sub_32))>;
+def : MipsPat<(sra GPR64:$rt, (i32 (trunc GPR64:$rs))),
+              (DSRAV GPR64:$rt, (EXTRACT_SUBREG GPR64:$rs, sub_32))>;
+def : MipsPat<(rotr GPR64:$rt, (i32 (trunc GPR64:$rs))),
+              (DROTRV GPR64:$rt, (EXTRACT_SUBREG GPR64:$rs, sub_32))>;
+
 // 32-to-64-bit extension
 def : MipsPat<(i64 (anyext GPR32:$src)), (SLL64_32 GPR32:$src)>;
 def : MipsPat<(i64 (zext GPR32:$src)), (DSRL (DSLL64_32 GPR32:$src), 32)>;
@@ -557,6 +573,15 @@ def : MipsInstAlias<"dadd $rs, $imm",
 def : MipsInstAlias<"dsll $rd, $rt, $rs",
                     (DSLLV GPR64Opnd:$rd, GPR64Opnd:$rt, GPR32Opnd:$rs), 0>,
                     ISA_MIPS3;
+def : MipsInstAlias<"dneg $rt, $rs",
+                    (DSUB GPR64Opnd:$rt, ZERO_64, GPR64Opnd:$rs), 1>,
+                    ISA_MIPS3;
+def : MipsInstAlias<"dneg $rt",
+                    (DSUB GPR64Opnd:$rt, ZERO_64, GPR64Opnd:$rt), 0>,
+                    ISA_MIPS3;
+def : MipsInstAlias<"dnegu $rt, $rs",
+                    (DSUBu GPR64Opnd:$rt, ZERO_64, GPR64Opnd:$rs), 1>,
+                    ISA_MIPS3;
 def : MipsInstAlias<"dsubu $rt, $rs, $imm",
                     (DADDiu GPR64Opnd:$rt, GPR64Opnd:$rs,
                             InvertedImOperand64:$imm), 0>, ISA_MIPS3;
@@ -588,10 +613,10 @@ def : MipsInstAlias<"dsrl $rd, $rt, $rs",
                     ISA_MIPS3;
 
 // Two operand (implicit 0 selector) versions:
-def : MipsInstAlias<"dmfc0 $rt, $rd", (DMFC0 GPR64Opnd:$rt, GPR64Opnd:$rd, 0), 0>;
-def : MipsInstAlias<"dmtc0 $rt, $rd", (DMTC0 GPR64Opnd:$rt, GPR64Opnd:$rd, 0), 0>;
-def : MipsInstAlias<"dmfc2 $rt, $rd", (DMFC2 GPR64Opnd:$rt, GPR64Opnd:$rd, 0), 0>;
-def : MipsInstAlias<"dmtc2 $rt, $rd", (DMTC2 GPR64Opnd:$rt, GPR64Opnd:$rd, 0), 0>;
+def : MipsInstAlias<"dmfc0 $rt, $rd", (DMFC0 GPR64Opnd:$rt, COP0Opnd:$rd, 0), 0>;
+def : MipsInstAlias<"dmtc0 $rt, $rd", (DMTC0 COP0Opnd:$rd, GPR64Opnd:$rt, 0), 0>;
+def : MipsInstAlias<"dmfc2 $rt, $rd", (DMFC2 GPR64Opnd:$rt, COP2Opnd:$rd, 0), 0>;
+def : MipsInstAlias<"dmtc2 $rt, $rd", (DMTC2 COP2Opnd:$rd, GPR64Opnd:$rt, 0), 0>;
 
 let Predicates = [HasMips64, HasCnMips] in {
 def : MipsInstAlias<"synciobdma", (SYNC 0x2), 0>;
diff --git a/lib/Target/Mips/MipsAsmPrinter.cpp b/lib/Target/Mips/MipsAsmPrinter.cpp
index cbf5b96c850..027c4e2a5ed 100644
--- a/lib/Target/Mips/MipsAsmPrinter.cpp
+++ b/lib/Target/Mips/MipsAsmPrinter.cpp
@@ -22,7 +22,6 @@
 #include "MipsTargetMachine.h"
 #include "MipsTargetStreamer.h"
 #include "llvm/ADT/SmallString.h"
-#include "llvm/ADT/StringExtras.h"
 #include "llvm/ADT/Twine.h"
 #include "llvm/CodeGen/MachineConstantPool.h"
 #include "llvm/CodeGen/MachineFrameInfo.h"
@@ -40,10 +39,9 @@
 #include "llvm/MC/MCELFStreamer.h"
 #include "llvm/MC/MCExpr.h"
 #include "llvm/MC/MCInst.h"
-#include "llvm/MC/MCNaCl.h"
 #include "llvm/MC/MCSection.h"
 #include "llvm/MC/MCSectionELF.h"
-#include "llvm/MC/MCSymbol.h"
+#include "llvm/MC/MCSymbolELF.h"
 #include "llvm/Support/ELF.h"
 #include "llvm/Support/TargetRegistry.h"
 #include "llvm/Support/raw_ostream.h"
@@ -56,7 +54,7 @@ using namespace llvm;
 #define DEBUG_TYPE "mips-asm-printer"
 
 MipsTargetStreamer &MipsAsmPrinter::getTargetStreamer() const {
-  return static_cast<MipsTargetStreamer &>(*OutStreamer.getTargetStreamer());
+  return static_cast<MipsTargetStreamer &>(*OutStreamer->getTargetStreamer());
 }
 
 bool MipsAsmPrinter::runOnMachineFunction(MachineFunction &MF) {
@@ -122,7 +120,7 @@ void MipsAsmPrinter::emitPseudoIndirectBranch(MCStreamer &OutStreamer,
 
   if (HasLinkReg) {
     unsigned ZeroReg = Subtarget->isGP64bit() ? Mips::ZERO_64 : Mips::ZERO;
-    TmpInst0.addOperand(MCOperand::CreateReg(ZeroReg));
+    TmpInst0.addOperand(MCOperand::createReg(ZeroReg));
   }
 
   lowerOperand(MI->getOperand(0), MCOp);
@@ -145,7 +143,7 @@ void MipsAsmPrinter::EmitInstruction(const MachineInstr *MI) {
 
   // If we just ended a constant pool, mark it as such.
   if (InConstantPool && MI->getOpcode() != Mips::CONSTPOOL_ENTRY) {
-    OutStreamer.EmitDataRegion(MCDR_DataRegionEnd);
+    OutStreamer->EmitDataRegion(MCDR_DataRegionEnd);
     InConstantPool = false;
   }
   if (MI->getOpcode() == Mips::CONSTPOOL_ENTRY) {
@@ -161,17 +159,17 @@ void MipsAsmPrinter::EmitInstruction(const MachineInstr *MI) {
 
     // If this is the first entry of the pool, mark it.
     if (!InConstantPool) {
-      OutStreamer.EmitDataRegion(MCDR_DataRegion);
+      OutStreamer->EmitDataRegion(MCDR_DataRegion);
       InConstantPool = true;
     }
 
-    OutStreamer.EmitLabel(GetCPISymbol(LabelId));
+    OutStreamer->EmitLabel(GetCPISymbol(LabelId));
 
     const MachineConstantPoolEntry &MCPE = MCP->getConstants()[CPIdx];
     if (MCPE.isMachineConstantPoolEntry())
       EmitMachineConstantPoolValue(MCPE.Val.MachineCPVal);
     else
-      EmitGlobalConstant(MCPE.Val.ConstVal);
+      EmitGlobalConstant(MF->getDataLayout(), MCPE.Val.ConstVal);
     return;
   }
 
@@ -181,14 +179,14 @@ void MipsAsmPrinter::EmitInstruction(const MachineInstr *MI) {
 
   do {
     // Do any auto-generated pseudo lowerings.
-    if (emitPseudoExpansionLowering(OutStreamer, &*I))
+    if (emitPseudoExpansionLowering(*OutStreamer, &*I))
       continue;
 
     if (I->getOpcode() == Mips::PseudoReturn ||
         I->getOpcode() == Mips::PseudoReturn64 ||
         I->getOpcode() == Mips::PseudoIndirectBranch ||
         I->getOpcode() == Mips::PseudoIndirectBranch64) {
-      emitPseudoIndirectBranch(OutStreamer, &*I);
+      emitPseudoIndirectBranch(*OutStreamer, &*I);
       continue;
     }
 
@@ -205,7 +203,7 @@ void MipsAsmPrinter::EmitInstruction(const MachineInstr *MI) {
 
     MCInst TmpInst0;
     MCInstLowering.Lower(I, TmpInst0);
-    EmitToStreamer(OutStreamer, TmpInst0);
+    EmitToStreamer(*OutStreamer, TmpInst0);
   } while ((++I != E) && I->isInsideBundle()); // Delay slot check
 }
 
@@ -339,15 +337,9 @@ void MipsAsmPrinter::EmitFunctionEntryLabel() {
     TS.emitDirectiveSetNoMips16();
 
   TS.emitDirectiveEnt(*CurrentFnSym);
-  OutStreamer.EmitLabel(CurrentFnSym);
+  OutStreamer->EmitLabel(CurrentFnSym);
 }
 
-// @LOCALMOD-START
-bool MipsAsmPrinter::UseReadOnlyJumpTables() const {
-  return Subtarget->isTargetNaCl();
-}
-// @LOCALMOD-END
-
 /// EmitFunctionBodyStart - Targets can override this to emit stuff before
 /// the first basic block in the function.
 void MipsAsmPrinter::EmitFunctionBodyStart() {
@@ -388,7 +380,13 @@ void MipsAsmPrinter::EmitFunctionBodyEnd() {
   if (!InConstantPool)
     return;
   InConstantPool = false;
-  OutStreamer.EmitDataRegion(MCDR_DataRegionEnd);
+  OutStreamer->EmitDataRegion(MCDR_DataRegionEnd);
+}
+
+void MipsAsmPrinter::EmitBasicBlockEnd(const MachineBasicBlock &MBB) {
+  MipsTargetStreamer &TS = getTargetStreamer();
+  if (MBB.size() == 0)
+    TS.emitDirectiveInsn();
 }
 
 /// isBlockOnlyReachableByFallthough - Return true if the basic block has
@@ -449,12 +447,12 @@ bool MipsAsmPrinter::PrintAsmOperand(const MachineInstr *MI, unsigned OpNum,
     case 'X': // hex const int
       if ((MO.getType()) != MachineOperand::MO_Immediate)
         return true;
-      O << "0x" << StringRef(utohexstr(MO.getImm())).lower();
+      O << "0x" << Twine::utohexstr(MO.getImm());
       return false;
     case 'x': // hex const int (low 16 bits)
       if ((MO.getType()) != MachineOperand::MO_Immediate)
         return true;
-      O << "0x" << StringRef(utohexstr(MO.getImm() & 0xffff)).lower();
+      O << "0x" << Twine::utohexstr(MO.getImm() & 0xffff);
       return false;
     case 'd': // decimal const int
       if ((MO.getType()) != MachineOperand::MO_Immediate)
@@ -561,7 +559,6 @@ bool MipsAsmPrinter::PrintAsmMemoryOperand(const MachineInstr *MI,
 
 void MipsAsmPrinter::printOperand(const MachineInstr *MI, int opNum,
                                   raw_ostream &O) {
-  const DataLayout *DL = TM.getDataLayout();
   const MachineOperand &MO = MI->getOperand(opNum);
   bool closeP = false;
 
@@ -596,11 +593,11 @@ void MipsAsmPrinter::printOperand(const MachineInstr *MI, int opNum,
       break;
 
     case MachineOperand::MO_MachineBasicBlock:
-      O << *MO.getMBB()->getSymbol();
+      MO.getMBB()->getSymbol()->print(O, MAI);
       return;
 
     case MachineOperand::MO_GlobalAddress:
-      O << *getSymbol(MO.getGlobal());
+      getSymbol(MO.getGlobal())->print(O, MAI);
       break;
 
     case MachineOperand::MO_BlockAddress: {
@@ -610,7 +607,7 @@ void MipsAsmPrinter::printOperand(const MachineInstr *MI, int opNum,
     }
 
     case MachineOperand::MO_ConstantPoolIndex:
-      O << DL->getPrivateGlobalPrefix() << "CPI"
+      O << getDataLayout().getPrivateGlobalPrefix() << "CPI"
         << getFunctionNumber() << "_" << MO.getIndex();
       if (MO.getOffset())
         O << "+" << MO.getOffset();
@@ -696,9 +693,8 @@ void MipsAsmPrinter::EmitStartOfAsmFile(Module &M) {
   // clean anyhow.
   // FIXME: For ifunc related functions we could iterate over and look
   // for a feature string that doesn't match the default one.
-  StringRef TT = TM.getTargetTriple();
-  StringRef CPU =
-      MIPS_MC::selectMipsCPU(TM.getTargetTriple(), TM.getTargetCPU());
+  const Triple &TT = TM.getTargetTriple();
+  StringRef CPU = MIPS_MC::selectMipsCPU(TT, TM.getTargetCPU());
   StringRef FS = TM.getTargetFeatureString();
   const MipsTargetMachine &MTM = static_cast<const MipsTargetMachine &>(TM);
   const MipsSubtarget STI(TT, CPU, FS, MTM.isLittleEndian(), MTM);
@@ -718,7 +714,7 @@ void MipsAsmPrinter::EmitStartOfAsmFile(Module &M) {
 
   // Tell the assembler which ABI we are using
   std::string SectionName = std::string(".mdebug.") + getCurrentABIString();
-  OutStreamer.SwitchSection(
+  OutStreamer->SwitchSection(
       OutContext.getELFSection(SectionName, ELF::SHT_PROGBITS, 0));
 
   // NaN: At the moment we only support:
@@ -731,11 +727,11 @@ void MipsAsmPrinter::EmitStartOfAsmFile(Module &M) {
 
   if (ABI.IsEABI()) {
     if (STI.isGP32bit())
-      OutStreamer.SwitchSection(OutContext.getELFSection(".gcc_compiled_long32",
-                                                         ELF::SHT_PROGBITS, 0));
+      OutStreamer->SwitchSection(OutContext.getELFSection(".gcc_compiled_long32",
+                                                          ELF::SHT_PROGBITS, 0));
     else
-      OutStreamer.SwitchSection(OutContext.getELFSection(".gcc_compiled_long64",
-                                                         ELF::SHT_PROGBITS, 0));
+      OutStreamer->SwitchSection(OutContext.getELFSection(".gcc_compiled_long64",
+                                                          ELF::SHT_PROGBITS, 0));
   }
 
   getTargetStreamer().updateABIInfo(STI);
@@ -750,34 +746,8 @@ void MipsAsmPrinter::EmitStartOfAsmFile(Module &M) {
   // accept it. We therefore emit it when it contradicts the default or an
   // option has changed the default (i.e. FPXX) and omit it otherwise.
   if (ABI.IsO32() && (!STI.useOddSPReg() || STI.isABI_FPXX()))
-    getTargetStreamer().emitDirectiveModuleOddSPReg(STI.useOddSPReg(),
-                                                    ABI.IsO32());
-
-  // @LOCALMOD-START
-  if (STI.isTargetNaCl()) {
-    initializeNaClMCStreamer(OutStreamer, OutContext, Triple(TT));
-  }
-  // @LOCALMOD-END
-}
-
-// @LOCALMOD-START
-unsigned MipsAsmPrinter::GetTargetLabelAlign(const MachineInstr *MI) const {
-  if (Subtarget->isTargetNaCl()) {
-    switch (MI->getOpcode()) {
-      default: return 0;
-      // These labels may indicate an indirect entry point that is
-      // externally reachable and hence must be bundle aligned.
-      // Note: these labels appear to be always at basic block beginnings
-      // so it may be possible to simply set the MBB alignment.
-      // However, it is unclear whether this always holds.
-      case TargetOpcode::EH_LABEL:
-      case TargetOpcode::GC_LABEL:
-        return 4;
-    }
-  }
-  return 0;
+    getTargetStreamer().emitDirectiveModuleOddSPReg();
 }
-// @LOCALMOD-END
 
 void MipsAsmPrinter::emitInlineAsmStart() const {
   MipsTargetStreamer &TS = getTargetStreamer();
@@ -792,12 +762,12 @@ void MipsAsmPrinter::emitInlineAsmStart() const {
   TS.emitDirectiveSetAt();
   TS.emitDirectiveSetMacro();
   TS.emitDirectiveSetReorder();
-  OutStreamer.AddBlankLine();
+  OutStreamer->AddBlankLine();
 }
 
 void MipsAsmPrinter::emitInlineAsmEnd(const MCSubtargetInfo &StartInfo,
                                       const MCSubtargetInfo *EndInfo) const {
-  OutStreamer.AddBlankLine();
+  OutStreamer->AddBlankLine();
   getTargetStreamer().emitDirectiveSetPop();
 }
 
@@ -805,16 +775,16 @@ void MipsAsmPrinter::EmitJal(const MCSubtargetInfo &STI, MCSymbol *Symbol) {
   MCInst I;
   I.setOpcode(Mips::JAL);
   I.addOperand(
-      MCOperand::CreateExpr(MCSymbolRefExpr::Create(Symbol, OutContext)));
-  OutStreamer.EmitInstruction(I, STI);
+      MCOperand::createExpr(MCSymbolRefExpr::create(Symbol, OutContext)));
+  OutStreamer->EmitInstruction(I, STI);
 }
 
 void MipsAsmPrinter::EmitInstrReg(const MCSubtargetInfo &STI, unsigned Opcode,
                                   unsigned Reg) {
   MCInst I;
   I.setOpcode(Opcode);
-  I.addOperand(MCOperand::CreateReg(Reg));
-  OutStreamer.EmitInstruction(I, STI);
+  I.addOperand(MCOperand::createReg(Reg));
+  OutStreamer->EmitInstruction(I, STI);
 }
 
 void MipsAsmPrinter::EmitInstrRegReg(const MCSubtargetInfo &STI,
@@ -832,9 +802,9 @@ void MipsAsmPrinter::EmitInstrRegReg(const MCSubtargetInfo &STI,
     Reg2 = Temp;
   }
   I.setOpcode(Opcode);
-  I.addOperand(MCOperand::CreateReg(Reg1));
-  I.addOperand(MCOperand::CreateReg(Reg2));
-  OutStreamer.EmitInstruction(I, STI);
+  I.addOperand(MCOperand::createReg(Reg1));
+  I.addOperand(MCOperand::createReg(Reg2));
+  OutStreamer->EmitInstruction(I, STI);
 }
 
 void MipsAsmPrinter::EmitInstrRegRegReg(const MCSubtargetInfo &STI,
@@ -842,10 +812,10 @@ void MipsAsmPrinter::EmitInstrRegRegReg(const MCSubtargetInfo &STI,
                                         unsigned Reg2, unsigned Reg3) {
   MCInst I;
   I.setOpcode(Opcode);
-  I.addOperand(MCOperand::CreateReg(Reg1));
-  I.addOperand(MCOperand::CreateReg(Reg2));
-  I.addOperand(MCOperand::CreateReg(Reg3));
-  OutStreamer.EmitInstruction(I, STI);
+  I.addOperand(MCOperand::createReg(Reg1));
+  I.addOperand(MCOperand::createReg(Reg2));
+  I.addOperand(MCOperand::createReg(Reg3));
+  OutStreamer->EmitInstruction(I, STI);
 }
 
 void MipsAsmPrinter::EmitMovFPIntPair(const MCSubtargetInfo &STI,
@@ -919,7 +889,7 @@ void MipsAsmPrinter::EmitSwapFPIntRetval(
 
 void MipsAsmPrinter::EmitFPCallStub(
     const char *Symbol, const Mips16HardFloatInfo::FuncSignature *Signature) {
-  MCSymbol *MSymbol = OutContext.GetOrCreateSymbol(StringRef(Symbol));
+  MCSymbol *MSymbol = OutContext.getOrCreateSymbol(StringRef(Symbol));
   using namespace Mips16HardFloatInfo;
   bool LE = getDataLayout().isLittleEndian();
   // Construct a local MCSubtargetInfo here.
@@ -927,12 +897,13 @@ void MipsAsmPrinter::EmitFPCallStub(
   // freed) and since we're at the global level we can use the default
   // constructed subtarget.
   std::unique_ptr<MCSubtargetInfo> STI(TM.getTarget().createMCSubtargetInfo(
-      TM.getTargetTriple(), TM.getTargetCPU(), TM.getTargetFeatureString()));
+      TM.getTargetTriple().str(), TM.getTargetCPU(),
+      TM.getTargetFeatureString()));
 
   //
   // .global xxxx
   //
-  OutStreamer.EmitSymbolAttribute(MSymbol, MCSA_Global);
+  OutStreamer->EmitSymbolAttribute(MSymbol, MCSA_Global);
   const char *RetType;
   //
   // make the comment field identifying the return and parameter
@@ -980,23 +951,23 @@ void MipsAsmPrinter::EmitFPCallStub(
     Parms = "";
     break;
   }
-  OutStreamer.AddComment("\t# Stub function to call " + Twine(RetType) + " " +
-                         Twine(Symbol) + " (" + Twine(Parms) + ")");
+  OutStreamer->AddComment("\t# Stub function to call " + Twine(RetType) + " " +
+                          Twine(Symbol) + " (" + Twine(Parms) + ")");
   //
   // probably not necessary but we save and restore the current section state
   //
-  OutStreamer.PushSection();
+  OutStreamer->PushSection();
   //
   // .section mips16.call.fpxxxx,"ax",@progbits
   //
-  const MCSectionELF *M = OutContext.getELFSection(
+  MCSectionELF *M = OutContext.getELFSection(
       ".mips16.call.fp." + std::string(Symbol), ELF::SHT_PROGBITS,
       ELF::SHF_ALLOC | ELF::SHF_EXECINSTR);
-  OutStreamer.SwitchSection(M, nullptr);
+  OutStreamer->SwitchSection(M, nullptr);
   //
   // .align 2
   //
-  OutStreamer.EmitValueToAlignment(4);
+  OutStreamer->EmitValueToAlignment(4);
   MipsTargetStreamer &TS = getTargetStreamer();
   //
   // .set nomips16
@@ -1010,12 +981,13 @@ void MipsAsmPrinter::EmitFPCallStub(
   //  __call_stub_fp_xxxx:
   //
   std::string x = "__call_stub_fp_" + std::string(Symbol);
-  MCSymbol *Stub = OutContext.GetOrCreateSymbol(StringRef(x));
+  MCSymbolELF *Stub =
+      cast<MCSymbolELF>(OutContext.getOrCreateSymbol(StringRef(x)));
   TS.emitDirectiveEnt(*Stub);
   MCSymbol *MType =
-      OutContext.GetOrCreateSymbol("__call_stub_fp_" + Twine(Symbol));
-  OutStreamer.EmitSymbolAttribute(MType, MCSA_ELF_TypeFunction);
-  OutStreamer.EmitLabel(Stub);
+      OutContext.getOrCreateSymbol("__call_stub_fp_" + Twine(Symbol));
+  OutStreamer->EmitSymbolAttribute(MType, MCSA_ELF_TypeFunction);
+  OutStreamer->EmitLabel(Stub);
 
   // Only handle non-pic for now.
   assert(TM.getRelocationModel() != Reloc::PIC_ &&
@@ -1053,14 +1025,14 @@ void MipsAsmPrinter::EmitFPCallStub(
   // else
   EmitInstrReg(*STI, Mips::JR, Mips::S2);
 
-  MCSymbol *Tmp = OutContext.CreateTempSymbol();
-  OutStreamer.EmitLabel(Tmp);
-  const MCSymbolRefExpr *E = MCSymbolRefExpr::Create(Stub, OutContext);
-  const MCSymbolRefExpr *T = MCSymbolRefExpr::Create(Tmp, OutContext);
-  const MCExpr *T_min_E = MCBinaryExpr::CreateSub(T, E, OutContext);
-  OutStreamer.EmitELFSize(Stub, T_min_E);
+  MCSymbol *Tmp = OutContext.createTempSymbol();
+  OutStreamer->EmitLabel(Tmp);
+  const MCSymbolRefExpr *E = MCSymbolRefExpr::create(Stub, OutContext);
+  const MCSymbolRefExpr *T = MCSymbolRefExpr::create(Tmp, OutContext);
+  const MCExpr *T_min_E = MCBinaryExpr::createSub(T, E, OutContext);
+  OutStreamer->emitELFSize(Stub, T_min_E);
   TS.emitDirectiveEnd(x);
-  OutStreamer.PopSection();
+  OutStreamer->PopSection();
 }
 
 void MipsAsmPrinter::EmitEndOfAsmFile(Module &M) {
@@ -1076,7 +1048,7 @@ void MipsAsmPrinter::EmitEndOfAsmFile(Module &M) {
     EmitFPCallStub(Symbol, Signature);
   }
   // return to the text section
-  OutStreamer.SwitchSection(OutContext.getObjectFileInfo()->getTextSection());
+  OutStreamer->SwitchSection(OutContext.getObjectFileInfo()->getTextSection());
 }
 
 void MipsAsmPrinter::PrintDebugValueComment(const MachineInstr *MI,
diff --git a/lib/Target/Mips/MipsAsmPrinter.h b/lib/Target/Mips/MipsAsmPrinter.h
index f239dd0a559..a7f3304a3da 100644
--- a/lib/Target/Mips/MipsAsmPrinter.h
+++ b/lib/Target/Mips/MipsAsmPrinter.h
@@ -124,6 +124,7 @@ class LLVM_LIBRARY_VISIBILITY MipsAsmPrinter : public AsmPrinter {
   void EmitFunctionEntryLabel() override;
   void EmitFunctionBodyStart() override;
   void EmitFunctionBodyEnd() override;
+  void EmitBasicBlockEnd(const MachineBasicBlock &MBB) override;
   bool isBlockOnlyReachableByFallthrough(
                                    const MachineBasicBlock* MBB) const override;
   bool PrintAsmOperand(const MachineInstr *MI, unsigned OpNo,
@@ -143,12 +144,6 @@ class LLVM_LIBRARY_VISIBILITY MipsAsmPrinter : public AsmPrinter {
   void EmitStartOfAsmFile(Module &M) override;
   void EmitEndOfAsmFile(Module &M) override;
   void PrintDebugValueComment(const MachineInstr *MI, raw_ostream &OS);
-
-  // @LOCALMOD-START
-  unsigned GetTargetLabelAlign(const MachineInstr *MI) const override;
-  /// UseReadOnlyJumpTables - true if JumpTableInfo must be in rodata.
-  bool UseReadOnlyJumpTables() const override;
-  // @LOCALMOD-END
 };
 }
 
diff --git a/lib/Target/Mips/MipsCallingConv.td b/lib/Target/Mips/MipsCallingConv.td
index dcd88f25d25..93e1908083c 100644
--- a/lib/Target/Mips/MipsCallingConv.td
+++ b/lib/Target/Mips/MipsCallingConv.td
@@ -64,9 +64,9 @@ def RetCC_F128HardFloat : CallingConv<[
 // Handle F128 specially since we can't identify the original type during the
 // tablegen-erated code.
 def RetCC_F128 : CallingConv<[
-  CCIfSubtarget<"abiUsesSoftFloat()",
+  CCIfSubtarget<"useSoftFloat()",
       CCIfType<[i64], CCDelegateTo<RetCC_F128SoftFloat>>>,
-  CCIfSubtargetNot<"abiUsesSoftFloat()",
+  CCIfSubtargetNot<"useSoftFloat()",
       CCIfType<[i64], CCDelegateTo<RetCC_F128HardFloat>>>
 ]>;
 
@@ -90,6 +90,9 @@ def CC_MipsO32 : CallingConv<[
 // Only the return rules are defined here for O32. The rules for argument
 // passing are defined in MipsISelLowering.cpp.
 def RetCC_MipsO32 : CallingConv<[
+  // Promote i1/i8/i16 return values to i32.
+  CCIfType<[i1, i8, i16], CCPromoteToType<i32>>,
+
   // i32 are returned in registers V0, V1, A0, A1
   CCIfType<[i32], CCAssignToReg<[V0, V1, A0, A1]>>,
 
@@ -131,7 +134,7 @@ def CC_MipsN : CallingConv<[
   CCIfType<[i8, i16, i32], CCIfOrigArgWasNotFloat<CCPromoteToType<i64>>>,
 
   // The only i32's we have left are soft-float arguments.
-  CCIfSubtarget<"abiUsesSoftFloat()", CCIfType<[i32], CCDelegateTo<CC_MipsN_SoftFloat>>>,
+  CCIfSubtarget<"useSoftFloat()", CCIfType<[i32], CCDelegateTo<CC_MipsN_SoftFloat>>>,
 
   // Integer arguments are passed in integer registers.
   CCIfType<[i64], CCAssignToRegWithShadow<[A0_64, A1_64, A2_64, A3_64,
@@ -369,7 +372,7 @@ def CC_Mips_FixedArg : CallingConv<[
   // f128 should only occur for the N64 ABI where long double is 128-bit. On
   // N32, long double is equivalent to double.
   CCIfType<[i64],
-      CCIfSubtargetNot<"abiUsesSoftFloat()",
+      CCIfSubtargetNot<"useSoftFloat()",
           CCIfOrigArgWasF128<CCBitConvertToType<f64>>>>,
 
   CCIfCC<"CallingConv::Fast", CCDelegateTo<CC_Mips_FastCC>>,
diff --git a/lib/Target/Mips/MipsCondMov.td b/lib/Target/Mips/MipsCondMov.td
index af10cd4e576..2d96d9b48c0 100644
--- a/lib/Target/Mips/MipsCondMov.td
+++ b/lib/Target/Mips/MipsCondMov.td
@@ -27,7 +27,8 @@ class CMov_I_I_FT<string opstr, RegisterOperand CRC, RegisterOperand DRC,
 class CMov_I_F_FT<string opstr, RegisterOperand CRC, RegisterOperand DRC,
                   InstrItinClass Itin> :
   InstSE<(outs DRC:$fd), (ins DRC:$fs, CRC:$rt, DRC:$F),
-         !strconcat(opstr, "\t$fd, $fs, $rt"), [], Itin, FrmFR, opstr> {
+         !strconcat(opstr, "\t$fd, $fs, $rt"), [], Itin, FrmFR, opstr>,
+  HARDFLOAT {
   let Constraints = "$F = $fd";
 }
 
@@ -37,7 +38,7 @@ class CMov_F_I_FT<string opstr, RegisterOperand RC, InstrItinClass Itin,
   InstSE<(outs RC:$rd), (ins RC:$rs, FCCRegsOpnd:$fcc, RC:$F),
          !strconcat(opstr, "\t$rd, $rs, $fcc"),
          [(set RC:$rd, (OpNode RC:$rs, FCCRegsOpnd:$fcc, RC:$F))],
-         Itin, FrmFR, opstr> {
+         Itin, FrmFR, opstr>, HARDFLOAT {
   let Constraints = "$F = $rd";
 }
 
@@ -47,7 +48,7 @@ class CMov_F_F_FT<string opstr, RegisterOperand RC, InstrItinClass Itin,
   InstSE<(outs RC:$fd), (ins RC:$fs, FCCRegsOpnd:$fcc, RC:$F),
          !strconcat(opstr, "\t$fd, $fs, $fcc"),
          [(set RC:$fd, (OpNode RC:$fs, FCCRegsOpnd:$fcc, RC:$F))],
-         Itin, FrmFR, opstr> {
+         Itin, FrmFR, opstr>, HARDFLOAT {
   let Constraints = "$F = $fd";
 }
 
diff --git a/lib/Target/Mips/MipsConstantIslandPass.cpp b/lib/Target/Mips/MipsConstantIslandPass.cpp
index 96553d28fc5..f246439f3e6 100644
--- a/lib/Target/Mips/MipsConstantIslandPass.cpp
+++ b/lib/Target/Mips/MipsConstantIslandPass.cpp
@@ -560,7 +560,7 @@ MipsConstantIslands::doInitialPlacement(std::vector<MachineInstr*> &CPEMIs) {
   // identity mapping of CPI's to CPE's.
   const std::vector<MachineConstantPoolEntry> &CPs = MCP->getConstants();
 
-  const DataLayout &TD = *MF->getTarget().getDataLayout();
+  const DataLayout &TD = MF->getDataLayout();
   for (unsigned i = 0, e = CPs.size(); i != e; ++i) {
     unsigned Size = TD.getTypeAllocSize(CPs[i].getType());
     assert(Size >= 4 && "Too small constant pool entry");
diff --git a/lib/Target/Mips/MipsDelaySlotFiller.cpp b/lib/Target/Mips/MipsDelaySlotFiller.cpp
index 606964dc129..7c9cd2b45c0 100644
--- a/lib/Target/Mips/MipsDelaySlotFiller.cpp
+++ b/lib/Target/Mips/MipsDelaySlotFiller.cpp
@@ -315,6 +315,14 @@ void RegDefsUses::init(const MachineInstr &MI) {
 void RegDefsUses::setCallerSaved(const MachineInstr &MI) {
   assert(MI.isCall());
 
+  // Add RA/RA_64 to Defs to prevent users of RA/RA_64 from going into
+  // the delay slot. The reason is that RA/RA_64 must not be changed
+  // in the delay slot so that the callee can return to the caller.
+  if (MI.definesRegister(Mips::RA) || MI.definesRegister(Mips::RA_64)) {
+    Defs.set(Mips::RA);
+    Defs.set(Mips::RA_64);
+  }
+
   // If MI is a call, add all caller-saved registers to Defs.
   BitVector CallerSavedRegs(TRI.getNumRegs(), true);
 
@@ -643,18 +651,34 @@ template<typename IterTy>
 bool Filler::searchRange(MachineBasicBlock &MBB, IterTy Begin, IterTy End,
                          RegDefsUses &RegDU, InspectMemInstr& IM, Iter Slot,
                          IterTy &Filler) const {
-  for (IterTy I = Begin; I != End; ++I) {
+  bool IsReverseIter = std::is_convertible<IterTy, ReverseIter>::value;
+
+  for (IterTy I = Begin; I != End;) {
+    IterTy CurrI = I;
+    ++I;
+
     // skip debug value
-    if (I->isDebugValue())
+    if (CurrI->isDebugValue())
       continue;
 
-    if (terminateSearch(*I))
+    if (terminateSearch(*CurrI))
       break;
 
-    assert((!I->isCall() && !I->isReturn() && !I->isBranch()) &&
+    assert((!CurrI->isCall() && !CurrI->isReturn() && !CurrI->isBranch()) &&
            "Cannot put calls, returns or branches in delay slot.");
 
-    if (delayHasHazard(*I, RegDU, IM))
+    if (CurrI->isKill()) {
+      CurrI->eraseFromParent();
+
+      // This special case is needed for reverse iterators, because when we
+      // erase an instruction, the iterators are updated to point to the next
+      // instruction.
+      if (IsReverseIter && I != End)
+        I = CurrI;
+      continue;
+    }
+
+    if (delayHasHazard(*CurrI, RegDU, IM))
       continue;
 
     const MipsSubtarget &STI = MBB.getParent()->getSubtarget<MipsSubtarget>();
@@ -664,21 +688,21 @@ bool Filler::searchRange(MachineBasicBlock &MBB, IterTy Begin, IterTy End,
       // branches are not checked because non-NaCl targets never put them in
       // delay slots.
       unsigned AddrIdx;
-      if ((isBasePlusOffsetMemoryAccess(I->getOpcode(), &AddrIdx) &&
-           baseRegNeedsLoadStoreMask(I->getOperand(AddrIdx).getReg())) ||
-          I->modifiesRegister(Mips::SP, STI.getRegisterInfo()))
+      if ((isBasePlusOffsetMemoryAccess(CurrI->getOpcode(), &AddrIdx) &&
+           baseRegNeedsLoadStoreMask(CurrI->getOperand(AddrIdx).getReg())) ||
+          CurrI->modifiesRegister(Mips::SP, STI.getRegisterInfo()))
         continue;
     }
 
     bool InMicroMipsMode = STI.inMicroMipsMode();
     const MipsInstrInfo *TII = STI.getInstrInfo();
     unsigned Opcode = (*Slot).getOpcode();
-    if (InMicroMipsMode && TII->GetInstSizeInBytes(&(*I)) == 2 &&
+    if (InMicroMipsMode && TII->GetInstSizeInBytes(&(*CurrI)) == 2 &&
         (Opcode == Mips::JR || Opcode == Mips::PseudoIndirectBranch ||
          Opcode == Mips::PseudoReturn))
       continue;
 
-    Filler = I;
+    Filler = CurrI;
     return true;
   }
 
@@ -689,8 +713,9 @@ bool Filler::searchBackward(MachineBasicBlock &MBB, Iter Slot) const {
   if (DisableBackwardSearch)
     return false;
 
-  RegDefsUses RegDU(*MBB.getParent()->getSubtarget().getRegisterInfo());
-  MemDefsUses MemDU(*TM.getDataLayout(), MBB.getParent()->getFrameInfo());
+  auto *Fn = MBB.getParent();
+  RegDefsUses RegDU(*Fn->getSubtarget().getRegisterInfo());
+  MemDefsUses MemDU(Fn->getDataLayout(), Fn->getFrameInfo());
   ReverseIter Filler;
 
   RegDU.init(*Slot);
@@ -739,6 +764,7 @@ bool Filler::searchSuccBBs(MachineBasicBlock &MBB, Iter Slot) const {
   BB2BrMap BrMap;
   std::unique_ptr<InspectMemInstr> IM;
   Iter Filler;
+  auto *Fn = MBB.getParent();
 
   // Iterate over SuccBB's predecessor list.
   for (MachineBasicBlock::pred_iterator PI = SuccBB->pred_begin(),
@@ -748,15 +774,15 @@ bool Filler::searchSuccBBs(MachineBasicBlock &MBB, Iter Slot) const {
 
   // Do not allow moving instructions which have unallocatable register operands
   // across basic block boundaries.
-  RegDU.setUnallocatableRegs(*MBB.getParent());
+  RegDU.setUnallocatableRegs(*Fn);
 
   // Only allow moving loads from stack or constants if any of the SuccBB's
   // predecessors have multiple successors.
   if (HasMultipleSuccs) {
     IM.reset(new LoadFromStackOrConst());
   } else {
-    const MachineFrameInfo *MFI = MBB.getParent()->getFrameInfo();
-    IM.reset(new MemDefsUses(*TM.getDataLayout(), MFI));
+    const MachineFrameInfo *MFI = Fn->getFrameInfo();
+    IM.reset(new MemDefsUses(Fn->getDataLayout(), MFI));
   }
 
   if (!searchRange(MBB, SuccBB->begin(), SuccBB->end(), RegDU, *IM, Slot,
@@ -843,7 +869,10 @@ bool Filler::examinePred(MachineBasicBlock &Pred, const MachineBasicBlock &Succ,
 
 bool Filler::delayHasHazard(const MachineInstr &Candidate, RegDefsUses &RegDU,
                             InspectMemInstr &IM) const {
-  bool HasHazard = (Candidate.isImplicitDef() || Candidate.isKill());
+  assert(!Candidate.isKill() &&
+         "KILL instructions should have been eliminated at this point.");
+
+  bool HasHazard = Candidate.isImplicitDef();
 
   HasHazard |= IM.hasHazard(Candidate);
   HasHazard |= RegDU.update(Candidate, 0, Candidate.getNumOperands());
diff --git a/lib/Target/Mips/MipsFastISel.cpp b/lib/Target/Mips/MipsFastISel.cpp
index e8e3d3d4b92..bdeb94942ff 100644
--- a/lib/Target/Mips/MipsFastISel.cpp
+++ b/lib/Target/Mips/MipsFastISel.cpp
@@ -13,8 +13,10 @@
 #include "llvm/CodeGen/FunctionLoweringInfo.h"
 #include "llvm/CodeGen/MachineInstrBuilder.h"
 #include "llvm/CodeGen/MachineRegisterInfo.h"
+#include "llvm/IR/GetElementPtrTypeIterator.h"
 #include "llvm/IR/GlobalAlias.h"
 #include "llvm/IR/GlobalVariable.h"
+#include "llvm/MC/MCSymbol.h"
 #include "llvm/Target/TargetInstrInfo.h"
 
 using namespace llvm;
@@ -81,6 +83,7 @@ class MipsFastISel final : public FastISel {
   LLVMContext *Context;
 
   bool fastLowerCall(CallLoweringInfo &CLI) override;
+  bool fastLowerIntrinsicCall(const IntrinsicInst *II) override;
 
   bool TargetSupported;
   bool UnsupportedFPMode; // To allow fast-isel to proceed and just not handle
@@ -93,6 +96,7 @@ class MipsFastISel final : public FastISel {
   bool selectLoad(const Instruction *I);
   bool selectStore(const Instruction *I);
   bool selectBranch(const Instruction *I);
+  bool selectSelect(const Instruction *I);
   bool selectCmp(const Instruction *I);
   bool selectFPExt(const Instruction *I);
   bool selectFPTrunc(const Instruction *I);
@@ -100,6 +104,8 @@ class MipsFastISel final : public FastISel {
   bool selectRet(const Instruction *I);
   bool selectTrunc(const Instruction *I);
   bool selectIntExt(const Instruction *I);
+  bool selectShift(const Instruction *I);
+  bool selectDivRem(const Instruction *I, unsigned ISDOpcode);
 
   // Utility helper routines.
   bool isTypeLegal(Type *Ty, MVT &VT);
@@ -138,6 +144,7 @@ class MipsFastISel final : public FastISel {
   unsigned materializeGV(const GlobalValue *GV, MVT VT);
   unsigned materializeInt(const Constant *C, MVT VT);
   unsigned materialize32BitInt(int64_t Imm, const TargetRegisterClass *RC);
+  unsigned materializeExternalCallSym(MCSymbol *Syn);
 
   MachineInstrBuilder emitInst(unsigned Opc) {
     return BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, TII.get(Opc));
@@ -154,6 +161,12 @@ class MipsFastISel final : public FastISel {
                                    unsigned MemReg, int64_t MemOffset) {
     return emitInst(Opc, DstReg).addReg(MemReg).addImm(MemOffset);
   }
+
+  unsigned fastEmitInst_rr(unsigned MachineInstOpcode,
+                           const TargetRegisterClass *RC,
+                           unsigned Op0, bool Op0IsKill,
+                           unsigned Op1, bool Op1IsKill);
+
   // for some reason, this default is not generated by tablegen
   // so we explicitly generate it here.
   //
@@ -186,6 +199,7 @@ class MipsFastISel final : public FastISel {
     UnsupportedFPMode = Subtarget->isFP64bit();
   }
 
+  unsigned fastMaterializeAlloca(const AllocaInst *AI) override;
   unsigned fastMaterializeConstant(const Constant *C) override;
   bool fastSelectInstruction(const Instruction *I) override;
 
@@ -252,17 +266,31 @@ unsigned MipsFastISel::emitLogicalOp(unsigned ISDOpc, MVT RetVT,
   return ResultReg;
 }
 
+unsigned MipsFastISel::fastMaterializeAlloca(const AllocaInst *AI) {
+  assert(TLI.getValueType(DL, AI->getType(), true) == MVT::i32 &&
+         "Alloca should always return a pointer.");
+
+  DenseMap<const AllocaInst *, int>::iterator SI =
+      FuncInfo.StaticAllocaMap.find(AI);
+
+  if (SI != FuncInfo.StaticAllocaMap.end()) {
+    unsigned ResultReg = createResultReg(&Mips::GPR32RegClass);
+    BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, TII.get(Mips::LEA_ADDiu),
+            ResultReg)
+        .addFrameIndex(SI->second)
+        .addImm(0);
+    return ResultReg;
+  }
+
+  return 0;
+}
+
 unsigned MipsFastISel::materializeInt(const Constant *C, MVT VT) {
   if (VT != MVT::i32 && VT != MVT::i16 && VT != MVT::i8 && VT != MVT::i1)
     return 0;
   const TargetRegisterClass *RC = &Mips::GPR32RegClass;
   const ConstantInt *CI = cast<ConstantInt>(C);
-  int64_t Imm;
-  if ((VT != MVT::i1) && CI->isNegative())
-    Imm = CI->getSExtValue();
-  else
-    Imm = CI->getZExtValue();
-  return materialize32BitInt(Imm, RC);
+  return materialize32BitInt(CI->getZExtValue(), RC);
 }
 
 unsigned MipsFastISel::materialize32BitInt(int64_t Imm,
@@ -337,10 +365,19 @@ unsigned MipsFastISel::materializeGV(const GlobalValue *GV, MVT VT) {
   return DestReg;
 }
 
+unsigned MipsFastISel::materializeExternalCallSym(MCSymbol *Sym) {
+  const TargetRegisterClass *RC = &Mips::GPR32RegClass;
+  unsigned DestReg = createResultReg(RC);
+  emitInst(Mips::LW, DestReg)
+      .addReg(MFI->getGlobalBaseReg())
+      .addSym(Sym, MipsII::MO_GOT);
+  return DestReg;
+}
+
 // Materialize a constant into a register, and return the register
 // number (or zero if we failed to handle it).
 unsigned MipsFastISel::fastMaterializeConstant(const Constant *C) {
-  EVT CEVT = TLI.getValueType(C->getType(), true);
+  EVT CEVT = TLI.getValueType(DL, C->getType(), true);
 
   // Only handle simple types.
   if (!CEVT.isSimple())
@@ -369,8 +406,10 @@ bool MipsFastISel::computeAddress(const Value *Obj, Address &Addr) {
       Opcode = I->getOpcode();
       U = I;
     }
-  } else if (isa<ConstantExpr>(Obj))
-    return false;
+  } else if (const ConstantExpr *C = dyn_cast<ConstantExpr>(Obj)) {
+    Opcode = C->getOpcode();
+    U = C;
+  }
   switch (Opcode) {
   default:
     break;
@@ -439,20 +478,57 @@ bool MipsFastISel::computeAddress(const Value *Obj, Address &Addr) {
 }
 
 bool MipsFastISel::computeCallAddress(const Value *V, Address &Addr) {
-  const GlobalValue *GV = dyn_cast<GlobalValue>(V);
-  if (GV && isa<Function>(GV) && cast<Function>(GV)->isIntrinsic())
-    return false;
-  if (!GV)
-    return false;
+  const User *U = nullptr;
+  unsigned Opcode = Instruction::UserOp1;
+
+  if (const auto *I = dyn_cast<Instruction>(V)) {
+    // Check if the value is defined in the same basic block. This information
+    // is crucial to know whether or not folding an operand is valid.
+    if (I->getParent() == FuncInfo.MBB->getBasicBlock()) {
+      Opcode = I->getOpcode();
+      U = I;
+    }
+  } else if (const auto *C = dyn_cast<ConstantExpr>(V)) {
+    Opcode = C->getOpcode();
+    U = C;
+  }
+
+  switch (Opcode) {
+  default:
+    break;
+  case Instruction::BitCast:
+    // Look past bitcasts if its operand is in the same BB.
+      return computeCallAddress(U->getOperand(0), Addr);
+    break;
+  case Instruction::IntToPtr:
+    // Look past no-op inttoptrs if its operand is in the same BB.
+    if (TLI.getValueType(DL, U->getOperand(0)->getType()) ==
+        TLI.getPointerTy(DL))
+      return computeCallAddress(U->getOperand(0), Addr);
+    break;
+  case Instruction::PtrToInt:
+    // Look past no-op ptrtoints if its operand is in the same BB.
+    if (TLI.getValueType(DL, U->getType()) == TLI.getPointerTy(DL))
+      return computeCallAddress(U->getOperand(0), Addr);
+    break;
+  }
+
   if (const GlobalValue *GV = dyn_cast<GlobalValue>(V)) {
     Addr.setGlobalValue(GV);
     return true;
   }
+
+  // If all else fails, try to materialize the value in a register.
+  if (!Addr.getGlobalValue()) {
+    Addr.setReg(getRegForValue(V));
+    return Addr.getReg() != 0;
+  }
+
   return false;
 }
 
 bool MipsFastISel::isTypeLegal(Type *Ty, MVT &VT) {
-  EVT evt = TLI.getValueType(Ty, true);
+  EVT evt = TLI.getValueType(DL, Ty, true);
   // Only handle simple types.
   if (evt == MVT::Other || !evt.isSimple())
     return false;
@@ -851,8 +927,8 @@ bool MipsFastISel::selectFPExt(const Instruction *I) {
   if (UnsupportedFPMode)
     return false;
   Value *Src = I->getOperand(0);
-  EVT SrcVT = TLI.getValueType(Src->getType(), true);
-  EVT DestVT = TLI.getValueType(I->getType(), true);
+  EVT SrcVT = TLI.getValueType(DL, Src->getType(), true);
+  EVT DestVT = TLI.getValueType(DL, I->getType(), true);
 
   if (SrcVT != MVT::f32 || DestVT != MVT::f64)
     return false;
@@ -869,13 +945,64 @@ bool MipsFastISel::selectFPExt(const Instruction *I) {
   return true;
 }
 
+bool MipsFastISel::selectSelect(const Instruction *I) {
+  assert(isa<SelectInst>(I) && "Expected a select instruction.");
+
+  MVT VT;
+  if (!isTypeSupported(I->getType(), VT))
+    return false;
+
+  unsigned CondMovOpc;
+  const TargetRegisterClass *RC;
+
+  if (VT.isInteger() && !VT.isVector() && VT.getSizeInBits() <= 32) {
+    CondMovOpc = Mips::MOVN_I_I;
+    RC = &Mips::GPR32RegClass;
+  } else if (VT == MVT::f32) {
+    CondMovOpc = Mips::MOVN_I_S;
+    RC = &Mips::FGR32RegClass;
+  } else if (VT == MVT::f64) {
+    CondMovOpc = Mips::MOVN_I_D32;
+    RC = &Mips::AFGR64RegClass;
+  } else
+    return false;
+
+  const SelectInst *SI = cast<SelectInst>(I);
+  const Value *Cond = SI->getCondition();
+  unsigned Src1Reg = getRegForValue(SI->getTrueValue());
+  unsigned Src2Reg = getRegForValue(SI->getFalseValue());
+  unsigned CondReg = getRegForValue(Cond);
+
+  if (!Src1Reg || !Src2Reg || !CondReg)
+    return false;
+
+  unsigned ZExtCondReg = createResultReg(&Mips::GPR32RegClass);
+  if (!ZExtCondReg)
+    return false;
+
+  if (!emitIntExt(MVT::i1, CondReg, MVT::i32, ZExtCondReg, true))
+    return false;
+
+  unsigned ResultReg = createResultReg(RC);
+  unsigned TempReg = createResultReg(RC);
+
+  if (!ResultReg || !TempReg)
+    return false;
+
+  emitInst(TargetOpcode::COPY, TempReg).addReg(Src2Reg);
+  emitInst(CondMovOpc, ResultReg)
+    .addReg(Src1Reg).addReg(ZExtCondReg).addReg(TempReg);
+  updateValueMap(I, ResultReg);
+  return true;
+}
+
 // Attempt to fast-select a floating-point truncate instruction.
 bool MipsFastISel::selectFPTrunc(const Instruction *I) {
   if (UnsupportedFPMode)
     return false;
   Value *Src = I->getOperand(0);
-  EVT SrcVT = TLI.getValueType(Src->getType(), true);
-  EVT DestVT = TLI.getValueType(I->getType(), true);
+  EVT SrcVT = TLI.getValueType(DL, Src->getType(), true);
+  EVT DestVT = TLI.getValueType(DL, I->getType(), true);
 
   if (SrcVT != MVT::f64 || DestVT != MVT::f32)
     return false;
@@ -977,7 +1104,9 @@ bool MipsFastISel::processCallArgs(CallLoweringInfo &CLI,
         }
       }
     }
-    if (((ArgVT == MVT::i32) || (ArgVT == MVT::f32)) && VA.isMemLoc()) {
+    if (((ArgVT == MVT::i32) || (ArgVT == MVT::f32) || (ArgVT == MVT::i16) ||
+         (ArgVT == MVT::i8)) &&
+        VA.isMemLoc()) {
       switch (VA.getLocMemOffset()) {
       case 0:
         VA.convertToReg(Mips::A0);
@@ -1091,6 +1220,8 @@ bool MipsFastISel::finishCall(CallLoweringInfo &CLI, MVT RetVT,
       CopyVT = MVT::i32;
 
     unsigned ResultReg = createResultReg(TLI.getRegClassFor(CopyVT));
+    if (!ResultReg)
+      return false;
     BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc,
             TII.get(TargetOpcode::COPY),
             ResultReg).addReg(RVLocs[0].getLocReg());
@@ -1107,7 +1238,11 @@ bool MipsFastISel::fastLowerCall(CallLoweringInfo &CLI) {
   bool IsTailCall = CLI.IsTailCall;
   bool IsVarArg = CLI.IsVarArg;
   const Value *Callee = CLI.Callee;
-  // const char *SymName = CLI.SymName;
+  MCSymbol *Symbol = CLI.Symbol;
+
+  // Do not handle FastCC.
+  if (CC == CallingConv::Fast)
+    return false;
 
   // Allow SelectionDAG isel to handle tail calls.
   if (IsTailCall)
@@ -1121,7 +1256,7 @@ bool MipsFastISel::fastLowerCall(CallLoweringInfo &CLI) {
   MVT RetVT;
   if (CLI.RetTy->isVoidTy())
     RetVT = MVT::isVoid;
-  else if (!isTypeLegal(CLI.RetTy, RetVT))
+  else if (!isTypeSupported(CLI.RetTy, RetVT))
     return false;
 
   for (auto Flag : CLI.OutFlags)
@@ -1154,8 +1289,15 @@ bool MipsFastISel::fastLowerCall(CallLoweringInfo &CLI) {
   if (!processCallArgs(CLI, OutVTs, NumBytes))
     return false;
 
+  if (!Addr.getGlobalValue())
+    return false;
+
   // Issue the call.
-  unsigned DestAddress = materializeGV(Addr.getGlobalValue(), MVT::i32);
+  unsigned DestAddress;
+  if (Symbol)
+    DestAddress = materializeExternalCallSym(Symbol);
+  else
+    DestAddress = materializeGV(Addr.getGlobalValue(), MVT::i32);
   emitInst(TargetOpcode::COPY, Mips::T9).addReg(DestAddress);
   MachineInstrBuilder MIB =
       BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, TII.get(Mips::JALR),
@@ -1175,6 +1317,98 @@ bool MipsFastISel::fastLowerCall(CallLoweringInfo &CLI) {
   return finishCall(CLI, RetVT, NumBytes);
 }
 
+bool MipsFastISel::fastLowerIntrinsicCall(const IntrinsicInst *II) {
+  switch (II->getIntrinsicID()) {
+  default:
+    return false;
+  case Intrinsic::bswap: {
+    Type *RetTy = II->getCalledFunction()->getReturnType();
+
+    MVT VT;
+    if (!isTypeSupported(RetTy, VT))
+      return false;
+
+    unsigned SrcReg = getRegForValue(II->getOperand(0));
+    if (SrcReg == 0)
+      return false;
+    unsigned DestReg = createResultReg(&Mips::GPR32RegClass);
+    if (DestReg == 0)
+      return false;
+    if (VT == MVT::i16) {
+      if (Subtarget->hasMips32r2()) {
+        emitInst(Mips::WSBH, DestReg).addReg(SrcReg);
+        updateValueMap(II, DestReg);
+        return true;
+      } else {
+        unsigned TempReg[3];
+        for (int i = 0; i < 3; i++) {
+          TempReg[i] = createResultReg(&Mips::GPR32RegClass);
+          if (TempReg[i] == 0)
+            return false;
+        }
+        emitInst(Mips::SLL, TempReg[0]).addReg(SrcReg).addImm(8);
+        emitInst(Mips::SRL, TempReg[1]).addReg(SrcReg).addImm(8);
+        emitInst(Mips::OR, TempReg[2]).addReg(TempReg[0]).addReg(TempReg[1]);
+        emitInst(Mips::ANDi, DestReg).addReg(TempReg[2]).addImm(0xFFFF);
+        updateValueMap(II, DestReg);
+        return true;
+      }
+    } else if (VT == MVT::i32) {
+      if (Subtarget->hasMips32r2()) {
+        unsigned TempReg = createResultReg(&Mips::GPR32RegClass);
+        emitInst(Mips::WSBH, TempReg).addReg(SrcReg);
+        emitInst(Mips::ROTR, DestReg).addReg(TempReg).addImm(16);
+        updateValueMap(II, DestReg);
+        return true;
+      } else {
+        unsigned TempReg[8];
+        for (int i = 0; i < 8; i++) {
+          TempReg[i] = createResultReg(&Mips::GPR32RegClass);
+          if (TempReg[i] == 0)
+            return false;
+        }
+
+        emitInst(Mips::SRL, TempReg[0]).addReg(SrcReg).addImm(8);
+        emitInst(Mips::SRL, TempReg[1]).addReg(SrcReg).addImm(24);
+        emitInst(Mips::ANDi, TempReg[2]).addReg(TempReg[0]).addImm(0xFF00);
+        emitInst(Mips::OR, TempReg[3]).addReg(TempReg[1]).addReg(TempReg[2]);
+
+        emitInst(Mips::ANDi, TempReg[4]).addReg(SrcReg).addImm(0xFF00);
+        emitInst(Mips::SLL, TempReg[5]).addReg(TempReg[4]).addImm(8);
+
+        emitInst(Mips::SLL, TempReg[6]).addReg(SrcReg).addImm(24);
+        emitInst(Mips::OR, TempReg[7]).addReg(TempReg[3]).addReg(TempReg[5]);
+        emitInst(Mips::OR, DestReg).addReg(TempReg[6]).addReg(TempReg[7]);
+        updateValueMap(II, DestReg);
+        return true;
+      }
+    }
+    return false;
+  }
+  case Intrinsic::memcpy:
+  case Intrinsic::memmove: {
+    const auto *MTI = cast<MemTransferInst>(II);
+    // Don't handle volatile.
+    if (MTI->isVolatile())
+      return false;
+    if (!MTI->getLength()->getType()->isIntegerTy(32))
+      return false;
+    const char *IntrMemName = isa<MemCpyInst>(II) ? "memcpy" : "memmove";
+    return lowerCallTo(II, IntrMemName, II->getNumArgOperands() - 2);
+  }
+  case Intrinsic::memset: {
+    const MemSetInst *MSI = cast<MemSetInst>(II);
+    // Don't handle volatile.
+    if (MSI->isVolatile())
+      return false;
+    if (!MSI->getLength()->getType()->isIntegerTy(32))
+      return false;
+    return lowerCallTo(II, "memset", II->getNumArgOperands() - 2);
+  }
+  }
+  return false;
+}
+
 bool MipsFastISel::selectRet(const Instruction *I) {
   const Function &F = *I->getParent()->getParent();
   const ReturnInst *Ret = cast<ReturnInst>(I);
@@ -1187,8 +1421,14 @@ bool MipsFastISel::selectRet(const Instruction *I) {
 
   if (Ret->getNumOperands() > 0) {
     CallingConv::ID CC = F.getCallingConv();
+
+    // Do not handle FastCC.
+    if (CC == CallingConv::Fast)
+      return false;
+
     SmallVector<ISD::OutputArg, 4> Outs;
-    GetReturnInfo(F.getReturnType(), F.getAttributes(), Outs, TLI);
+    GetReturnInfo(F.getReturnType(), F.getAttributes(), Outs, TLI, DL);
+
     // Analyze operands of the call, assigning locations to each operand.
     SmallVector<CCValAssign, 16> ValLocs;
     MipsCCState CCInfo(CC, F.isVarArg(), *FuncInfo.MF, ValLocs,
@@ -1222,7 +1462,7 @@ bool MipsFastISel::selectRet(const Instruction *I) {
     if (!MRI.getRegClass(SrcReg)->contains(DestReg))
       return false;
 
-    EVT RVEVT = TLI.getValueType(RV->getType());
+    EVT RVEVT = TLI.getValueType(DL, RV->getType());
     if (!RVEVT.isSimple())
       return false;
 
@@ -1239,13 +1479,12 @@ bool MipsFastISel::selectRet(const Instruction *I) {
       if (RVVT != MVT::i1 && RVVT != MVT::i8 && RVVT != MVT::i16)
         return false;
 
-      if (!Outs[0].Flags.isZExt() && !Outs[0].Flags.isSExt())
-        return false;
-
-      bool IsZExt = Outs[0].Flags.isZExt();
-      SrcReg = emitIntExt(RVVT, SrcReg, DestVT, IsZExt);
-      if (SrcReg == 0)
-        return false;
+      if (Outs[0].Flags.isZExt() || Outs[0].Flags.isSExt()) {
+        bool IsZExt = Outs[0].Flags.isZExt();
+        SrcReg = emitIntExt(RVVT, SrcReg, DestVT, IsZExt);
+        if (SrcReg == 0)
+          return false;
+      }
     }
 
     // Make the copy.
@@ -1267,8 +1506,8 @@ bool MipsFastISel::selectTrunc(const Instruction *I) {
   Value *Op = I->getOperand(0);
 
   EVT SrcVT, DestVT;
-  SrcVT = TLI.getValueType(Op->getType(), true);
-  DestVT = TLI.getValueType(I->getType(), true);
+  SrcVT = TLI.getValueType(DL, Op->getType(), true);
+  DestVT = TLI.getValueType(DL, I->getType(), true);
 
   if (SrcVT != MVT::i32 && SrcVT != MVT::i16 && SrcVT != MVT::i8)
     return false;
@@ -1295,8 +1534,8 @@ bool MipsFastISel::selectIntExt(const Instruction *I) {
     return false;
 
   EVT SrcEVT, DestEVT;
-  SrcEVT = TLI.getValueType(SrcTy, true);
-  DestEVT = TLI.getValueType(DestTy, true);
+  SrcEVT = TLI.getValueType(DL, SrcTy, true);
+  DestEVT = TLI.getValueType(DL, DestTy, true);
   if (!SrcEVT.isSimple())
     return false;
   if (!DestEVT.isSimple())
@@ -1374,6 +1613,13 @@ bool MipsFastISel::emitIntZExt(MVT SrcVT, unsigned SrcReg, MVT DestVT,
 
 bool MipsFastISel::emitIntExt(MVT SrcVT, unsigned SrcReg, MVT DestVT,
                               unsigned DestReg, bool IsZExt) {
+  // FastISel does not have plumbing to deal with extensions where the SrcVT or
+  // DestVT are odd things, so test to make sure that they are both types we can
+  // handle (i1/i8/i16/i32 for SrcVT and i8/i16/i32/i64 for DestVT), otherwise
+  // bail out to SelectionDAG.
+  if (((DestVT != MVT::i8) && (DestVT != MVT::i16) && (DestVT != MVT::i32)) ||
+      ((SrcVT != MVT::i1) && (SrcVT != MVT::i8) && (SrcVT != MVT::i16)))
+    return false;
   if (IsZExt)
     return emitIntZExt(SrcVT, SrcReg, DestVT, DestReg);
   return emitIntSExt(SrcVT, SrcReg, DestVT, DestReg);
@@ -1386,6 +1632,125 @@ unsigned MipsFastISel::emitIntExt(MVT SrcVT, unsigned SrcReg, MVT DestVT,
   return Success ? DestReg : 0;
 }
 
+bool MipsFastISel::selectDivRem(const Instruction *I, unsigned ISDOpcode) {
+  EVT DestEVT = TLI.getValueType(DL, I->getType(), true);
+  if (!DestEVT.isSimple())
+    return false;
+
+  MVT DestVT = DestEVT.getSimpleVT();
+  if (DestVT != MVT::i32)
+    return false;
+
+  unsigned DivOpc;
+  switch (ISDOpcode) {
+  default:
+    return false;
+  case ISD::SDIV:
+  case ISD::SREM:
+    DivOpc = Mips::SDIV;
+    break;
+  case ISD::UDIV:
+  case ISD::UREM:
+    DivOpc = Mips::UDIV;
+    break;
+  }
+
+  unsigned Src0Reg = getRegForValue(I->getOperand(0));
+  unsigned Src1Reg = getRegForValue(I->getOperand(1));
+  if (!Src0Reg || !Src1Reg)
+    return false;
+
+  emitInst(DivOpc).addReg(Src0Reg).addReg(Src1Reg);
+  emitInst(Mips::TEQ).addReg(Src1Reg).addReg(Mips::ZERO).addImm(7);
+
+  unsigned ResultReg = createResultReg(&Mips::GPR32RegClass);
+  if (!ResultReg)
+    return false;
+
+  unsigned MFOpc = (ISDOpcode == ISD::SREM || ISDOpcode == ISD::UREM)
+                       ? Mips::MFHI
+                       : Mips::MFLO;
+  emitInst(MFOpc, ResultReg);
+
+  updateValueMap(I, ResultReg);
+  return true;
+}
+
+bool MipsFastISel::selectShift(const Instruction *I) {
+  MVT RetVT;
+
+  if (!isTypeSupported(I->getType(), RetVT))
+    return false;
+
+  unsigned ResultReg = createResultReg(&Mips::GPR32RegClass);
+  if (!ResultReg)
+    return false;
+
+  unsigned Opcode = I->getOpcode();
+  const Value *Op0 = I->getOperand(0);
+  unsigned Op0Reg = getRegForValue(Op0);
+  if (!Op0Reg)
+    return false;
+
+  // If AShr or LShr, then we need to make sure the operand0 is sign extended.
+  if (Opcode == Instruction::AShr || Opcode == Instruction::LShr) {
+    unsigned TempReg = createResultReg(&Mips::GPR32RegClass);
+    if (!TempReg)
+      return false;
+
+    MVT Op0MVT = TLI.getValueType(DL, Op0->getType(), true).getSimpleVT();
+    bool IsZExt = Opcode == Instruction::LShr;
+    if (!emitIntExt(Op0MVT, Op0Reg, MVT::i32, TempReg, IsZExt))
+      return false;
+
+    Op0Reg = TempReg;
+  }
+
+  if (const auto *C = dyn_cast<ConstantInt>(I->getOperand(1))) {
+    uint64_t ShiftVal = C->getZExtValue();
+
+    switch (Opcode) {
+    default:
+      llvm_unreachable("Unexpected instruction.");
+    case Instruction::Shl:
+      Opcode = Mips::SLL;
+      break;
+    case Instruction::AShr:
+      Opcode = Mips::SRA;
+      break;
+    case Instruction::LShr:
+      Opcode = Mips::SRL;
+      break;
+    }
+
+    emitInst(Opcode, ResultReg).addReg(Op0Reg).addImm(ShiftVal);
+    updateValueMap(I, ResultReg);
+    return true;
+  }
+
+  unsigned Op1Reg = getRegForValue(I->getOperand(1));
+  if (!Op1Reg)
+    return false;
+
+  switch (Opcode) {
+  default:
+    llvm_unreachable("Unexpected instruction.");
+  case Instruction::Shl:
+    Opcode = Mips::SLLV;
+    break;
+  case Instruction::AShr:
+    Opcode = Mips::SRAV;
+    break;
+  case Instruction::LShr:
+    Opcode = Mips::SRLV;
+    break;
+  }
+
+  emitInst(Opcode, ResultReg).addReg(Op0Reg).addReg(Op1Reg);
+  updateValueMap(I, ResultReg);
+  return true;
+}
+
 bool MipsFastISel::fastSelectInstruction(const Instruction *I) {
   if (!TargetSupported)
     return false;
@@ -1396,6 +1761,26 @@ bool MipsFastISel::fastSelectInstruction(const Instruction *I) {
     return selectLoad(I);
   case Instruction::Store:
     return selectStore(I);
+  case Instruction::SDiv:
+    if (!selectBinaryOp(I, ISD::SDIV))
+      return selectDivRem(I, ISD::SDIV);
+    return true;
+  case Instruction::UDiv:
+    if (!selectBinaryOp(I, ISD::UDIV))
+      return selectDivRem(I, ISD::UDIV);
+    return true;
+  case Instruction::SRem:
+    if (!selectBinaryOp(I, ISD::SREM))
+      return selectDivRem(I, ISD::SREM);
+    return true;
+  case Instruction::URem:
+    if (!selectBinaryOp(I, ISD::UREM))
+      return selectDivRem(I, ISD::UREM);
+    return true;
+  case Instruction::Shl:
+  case Instruction::LShr:
+  case Instruction::AShr:
+    return selectShift(I);
   case Instruction::And:
   case Instruction::Or:
   case Instruction::Xor:
@@ -1420,6 +1805,8 @@ bool MipsFastISel::fastSelectInstruction(const Instruction *I) {
   case Instruction::ICmp:
   case Instruction::FCmp:
     return selectCmp(I);
+  case Instruction::Select:
+    return selectSelect(I);
   }
   return false;
 }
@@ -1429,7 +1816,7 @@ unsigned MipsFastISel::getRegEnsuringSimpleIntegerWidening(const Value *V,
   unsigned VReg = getRegForValue(V);
   if (VReg == 0)
     return 0;
-  MVT VMVT = TLI.getValueType(V->getType(), true).getSimpleVT();
+  MVT VMVT = TLI.getValueType(DL, V->getType(), true).getSimpleVT();
   if ((VMVT == MVT::i8) || (VMVT == MVT::i16)) {
     unsigned TempReg = createResultReg(&Mips::GPR32RegClass);
     if (!emitIntExt(VMVT, VReg, MVT::i32, TempReg, IsUnsigned))
@@ -1450,6 +1837,33 @@ void MipsFastISel::simplifyAddress(Address &Addr) {
   }
 }
 
+unsigned MipsFastISel::fastEmitInst_rr(unsigned MachineInstOpcode,
+                                       const TargetRegisterClass *RC,
+                                       unsigned Op0, bool Op0IsKill,
+                                       unsigned Op1, bool Op1IsKill) {
+  // We treat the MUL instruction in a special way because it clobbers
+  // the HI0 & LO0 registers. The TableGen definition of this instruction can
+  // mark these registers only as implicitly defined. As a result, the
+  // register allocator runs out of registers when this instruction is
+  // followed by another instruction that defines the same registers too.
+  // We can fix this by explicitly marking those registers as dead.
+  if (MachineInstOpcode == Mips::MUL) {
+    unsigned ResultReg = createResultReg(RC);
+    const MCInstrDesc &II = TII.get(MachineInstOpcode);
+    Op0 = constrainOperandRegClass(II, Op0, II.getNumDefs());
+    Op1 = constrainOperandRegClass(II, Op1, II.getNumDefs() + 1);
+    BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, II, ResultReg)
+      .addReg(Op0, getKillRegState(Op0IsKill))
+      .addReg(Op1, getKillRegState(Op1IsKill))
+      .addReg(Mips::HI0, RegState::ImplicitDefine | RegState::Dead)
+      .addReg(Mips::LO0, RegState::ImplicitDefine | RegState::Dead);
+    return ResultReg;
+  }
+
+  return FastISel::fastEmitInst_rr(MachineInstOpcode, RC, Op0, Op0IsKill, Op1,
+                                   Op1IsKill);
+}
+
 namespace llvm {
 FastISel *Mips::createFastISel(FunctionLoweringInfo &funcInfo,
                                const TargetLibraryInfo *libInfo) {
diff --git a/lib/Target/Mips/MipsFrameLowering.cpp b/lib/Target/Mips/MipsFrameLowering.cpp
index 826fbaf4d00..a74c8abd2e2 100644
--- a/lib/Target/Mips/MipsFrameLowering.cpp
+++ b/lib/Target/Mips/MipsFrameLowering.cpp
@@ -90,12 +90,23 @@ const MipsFrameLowering *MipsFrameLowering::create(const MipsSubtarget &ST) {
 }
 
 // hasFP - Return true if the specified function should have a dedicated frame
-// pointer register.  This is true if the function has variable sized allocas or
-// if frame pointer elimination is disabled.
+// pointer register.  This is true if the function has variable sized allocas,
+// if it needs dynamic stack realignment, if frame pointer elimination is
+// disabled, or if the frame address is taken.
 bool MipsFrameLowering::hasFP(const MachineFunction &MF) const {
   const MachineFrameInfo *MFI = MF.getFrameInfo();
+  const TargetRegisterInfo *TRI = STI.getRegisterInfo();
+
   return MF.getTarget().Options.DisableFramePointerElim(MF) ||
-      MFI->hasVarSizedObjects() || MFI->isFrameAddressTaken();
+      MFI->hasVarSizedObjects() || MFI->isFrameAddressTaken() ||
+      TRI->needsStackRealignment(MF);
+}
+
+bool MipsFrameLowering::hasBP(const MachineFunction &MF) const {
+  const MachineFrameInfo *MFI = MF.getFrameInfo();
+  const TargetRegisterInfo *TRI = STI.getRegisterInfo();
+
+  return MFI->hasVarSizedObjects() && TRI->needsStackRealignment(MF);
 }
 
 uint64_t MipsFrameLowering::estimateStackSize(const MachineFunction &MF) const {
diff --git a/lib/Target/Mips/MipsFrameLowering.h b/lib/Target/Mips/MipsFrameLowering.h
index 96d1e29d6c5..5eabd58e868 100644
--- a/lib/Target/Mips/MipsFrameLowering.h
+++ b/lib/Target/Mips/MipsFrameLowering.h
@@ -32,6 +32,10 @@ class MipsFrameLowering : public TargetFrameLowering {
 
   bool hasFP(const MachineFunction &MF) const override;
 
+  bool hasBP(const MachineFunction &MF) const;
+
+  bool isFPCloseToIncomingSP() const override { return false; }
+
   void
   eliminateCallFramePseudoInstr(MachineFunction &MF,
                                 MachineBasicBlock &MBB,
diff --git a/lib/Target/Mips/MipsISelDAGToDAG.cpp b/lib/Target/Mips/MipsISelDAGToDAG.cpp
index c78c3296762..06502397b6b 100644
--- a/lib/Target/Mips/MipsISelDAGToDAG.cpp
+++ b/lib/Target/Mips/MipsISelDAGToDAG.cpp
@@ -59,8 +59,9 @@ bool MipsDAGToDAGISel::runOnMachineFunction(MachineFunction &MF) {
 /// GOT address into a register.
 SDNode *MipsDAGToDAGISel::getGlobalBaseReg() {
   unsigned GlobalBaseReg = MF->getInfo<MipsFunctionInfo>()->getGlobalBaseReg();
-  return CurDAG->getRegister(GlobalBaseReg,
-                             getTargetLowering()->getPointerTy()).getNode();
+  return CurDAG->getRegister(GlobalBaseReg, getTargetLowering()->getPointerTy(
+                                                CurDAG->getDataLayout()))
+      .getNode();
 }
 
 /// ComplexPattern used on MipsInstrInfo
@@ -113,7 +114,8 @@ bool MipsDAGToDAGISel::selectAddr16(SDNode *Parent, SDValue N, SDValue &Base,
   return false;
 }
 
-bool MipsDAGToDAGISel::selectVSplat(SDNode *N, APInt &Imm) const {
+bool MipsDAGToDAGISel::selectVSplat(SDNode *N, APInt &Imm,
+                                    unsigned MinSizeInBits) const {
   llvm_unreachable("Unimplemented function.");
   return false;
 }
diff --git a/lib/Target/Mips/MipsISelDAGToDAG.h b/lib/Target/Mips/MipsISelDAGToDAG.h
index aec731e0dff..1426d0fbf51 100644
--- a/lib/Target/Mips/MipsISelDAGToDAG.h
+++ b/lib/Target/Mips/MipsISelDAGToDAG.h
@@ -84,7 +84,8 @@ class MipsDAGToDAGISel : public SelectionDAGISel {
                             SDValue &Offset, SDValue &Alias);
 
   /// \brief Select constant vector splats.
-  virtual bool selectVSplat(SDNode *N, APInt &Imm) const;
+  virtual bool selectVSplat(SDNode *N, APInt &Imm,
+                            unsigned MinSizeInBits) const;
   /// \brief Select constant vector splats whose value fits in a uimm1.
   virtual bool selectVSplatUimm1(SDValue N, SDValue &Imm) const;
   /// \brief Select constant vector splats whose value fits in a uimm2.
@@ -119,7 +120,7 @@ class MipsDAGToDAGISel : public SelectionDAGISel {
 
   // getImm - Return a target constant with the specified value.
   inline SDValue getImm(const SDNode *Node, uint64_t Imm) {
-    return CurDAG->getTargetConstant(Imm, Node->getValueType(0));
+    return CurDAG->getTargetConstant(Imm, SDLoc(Node), Node->getValueType(0));
   }
 
   virtual void processFunctionAfterISel(MachineFunction &MF) = 0;
diff --git a/lib/Target/Mips/MipsISelLowering.cpp b/lib/Target/Mips/MipsISelLowering.cpp
index a61fac8249c..b3f85fd4e61 100644
--- a/lib/Target/Mips/MipsISelLowering.cpp
+++ b/lib/Target/Mips/MipsISelLowering.cpp
@@ -27,6 +27,7 @@
 #include "llvm/CodeGen/MachineInstrBuilder.h"
 #include "llvm/CodeGen/MachineJumpTableInfo.h"
 #include "llvm/CodeGen/MachineRegisterInfo.h"
+#include "llvm/CodeGen/FunctionLoweringInfo.h"
 #include "llvm/CodeGen/SelectionDAGISel.h"
 #include "llvm/CodeGen/ValueTypes.h"
 #include "llvm/IR/CallingConv.h"
@@ -53,11 +54,6 @@ NoZeroDivCheck("mno-check-zero-division", cl::Hidden,
                cl::desc("MIPS: Don't trap on integer division by zero."),
                cl::init(false));
 
-cl::opt<bool>
-EnableMipsFastISel("mips-fast-isel", cl::Hidden,
-  cl::desc("Allow mips-fast-isel to be used"),
-  cl::init(false));
-
 static const MCPhysReg Mips64DPRegs[8] = {
   Mips::D12_64, Mips::D13_64, Mips::D14_64, Mips::D15_64,
   Mips::D16_64, Mips::D17_64, Mips::D18_64, Mips::D19_64
@@ -112,7 +108,8 @@ SDValue MipsTargetLowering::getTargetNode(ConstantPoolSDNode *N, EVT Ty,
 }
 
 const char *MipsTargetLowering::getTargetNodeName(unsigned Opcode) const {
-  switch (Opcode) {
+  switch ((MipsISD::NodeType)Opcode) {
+  case MipsISD::FIRST_NUMBER:      break;
   case MipsISD::JmpLink:           return "MipsISD::JmpLink";
   case MipsISD::TailCall:          return "MipsISD::TailCall";
   case MipsISD::Hi:                return "MipsISD::Hi";
@@ -142,6 +139,7 @@ const char *MipsTargetLowering::getTargetNodeName(unsigned Opcode) const {
   case MipsISD::BuildPairF64:      return "MipsISD::BuildPairF64";
   case MipsISD::ExtractElementF64: return "MipsISD::ExtractElementF64";
   case MipsISD::Wrapper:           return "MipsISD::Wrapper";
+  case MipsISD::DynAlloc:          return "MipsISD::DynAlloc";
   case MipsISD::Sync:              return "MipsISD::Sync";
   case MipsISD::Ext:               return "MipsISD::Ext";
   case MipsISD::Ins:               return "MipsISD::Ins";
@@ -161,6 +159,28 @@ const char *MipsTargetLowering::getTargetNodeName(unsigned Opcode) const {
   case MipsISD::EXTR_RS_W:         return "MipsISD::EXTR_RS_W";
   case MipsISD::SHILO:             return "MipsISD::SHILO";
   case MipsISD::MTHLIP:            return "MipsISD::MTHLIP";
+  case MipsISD::MULSAQ_S_W_PH:     return "MipsISD::MULSAQ_S_W_PH";
+  case MipsISD::MAQ_S_W_PHL:       return "MipsISD::MAQ_S_W_PHL";
+  case MipsISD::MAQ_S_W_PHR:       return "MipsISD::MAQ_S_W_PHR";
+  case MipsISD::MAQ_SA_W_PHL:      return "MipsISD::MAQ_SA_W_PHL";
+  case MipsISD::MAQ_SA_W_PHR:      return "MipsISD::MAQ_SA_W_PHR";
+  case MipsISD::DPAU_H_QBL:        return "MipsISD::DPAU_H_QBL";
+  case MipsISD::DPAU_H_QBR:        return "MipsISD::DPAU_H_QBR";
+  case MipsISD::DPSU_H_QBL:        return "MipsISD::DPSU_H_QBL";
+  case MipsISD::DPSU_H_QBR:        return "MipsISD::DPSU_H_QBR";
+  case MipsISD::DPAQ_S_W_PH:       return "MipsISD::DPAQ_S_W_PH";
+  case MipsISD::DPSQ_S_W_PH:       return "MipsISD::DPSQ_S_W_PH";
+  case MipsISD::DPAQ_SA_L_W:       return "MipsISD::DPAQ_SA_L_W";
+  case MipsISD::DPSQ_SA_L_W:       return "MipsISD::DPSQ_SA_L_W";
+  case MipsISD::DPA_W_PH:          return "MipsISD::DPA_W_PH";
+  case MipsISD::DPS_W_PH:          return "MipsISD::DPS_W_PH";
+  case MipsISD::DPAQX_S_W_PH:      return "MipsISD::DPAQX_S_W_PH";
+  case MipsISD::DPAQX_SA_W_PH:     return "MipsISD::DPAQX_SA_W_PH";
+  case MipsISD::DPAX_W_PH:         return "MipsISD::DPAX_W_PH";
+  case MipsISD::DPSX_W_PH:         return "MipsISD::DPSX_W_PH";
+  case MipsISD::DPSQX_S_W_PH:      return "MipsISD::DPSQX_S_W_PH";
+  case MipsISD::DPSQX_SA_W_PH:     return "MipsISD::DPSQX_SA_W_PH";
+  case MipsISD::MULSA_W_PH:        return "MipsISD::MULSA_W_PH";
   case MipsISD::MULT:              return "MipsISD::MULT";
   case MipsISD::MULTU:             return "MipsISD::MULTU";
   case MipsISD::MADD_DSP:          return "MipsISD::MADD_DSP";
@@ -197,8 +217,8 @@ const char *MipsTargetLowering::getTargetNodeName(unsigned Opcode) const {
   case MipsISD::PCKEV:             return "MipsISD::PCKEV";
   case MipsISD::PCKOD:             return "MipsISD::PCKOD";
   case MipsISD::INSVE:             return "MipsISD::INSVE";
-  default:                         return nullptr;
   }
+  return nullptr;
 }
 
 MipsTargetLowering::MipsTargetLowering(const MipsTargetMachine &TM,
@@ -221,9 +241,23 @@ MipsTargetLowering::MipsTargetLowering(const MipsTargetMachine &TM,
     setLoadExtAction(ISD::SEXTLOAD, VT, MVT::i1,  Promote);
   }
 
-  // MIPS doesn't have extending float->double load/store
-  for (MVT VT : MVT::fp_valuetypes())
+  // MIPS doesn't have extending float->double load/store.  Set LoadExtAction
+  // for f32, f16
+  for (MVT VT : MVT::fp_valuetypes()) {
     setLoadExtAction(ISD::EXTLOAD, VT, MVT::f32, Expand);
+    setLoadExtAction(ISD::EXTLOAD, VT, MVT::f16, Expand);
+  }
+
+  // Set LoadExtAction for f16 vectors to Expand
+  for (MVT VT : MVT::fp_vector_valuetypes()) {
+    MVT F16VT = MVT::getVectorVT(MVT::f16, VT.getVectorNumElements());
+    if (F16VT.isValid())
+      setLoadExtAction(ISD::EXTLOAD, VT, F16VT, Expand);
+  }
+
+  setTruncStoreAction(MVT::f32, MVT::f16, Expand);
+  setTruncStoreAction(MVT::f64, MVT::f16, Expand);
+
   setTruncStoreAction(MVT::f64, MVT::f32, Expand);
 
   // Used by legalize types to correctly generate the setcc result.
@@ -339,6 +373,12 @@ MipsTargetLowering::MipsTargetLowering(const MipsTargetMachine &TM,
   setOperationAction(ISD::FREM,              MVT::f32,   Expand);
   setOperationAction(ISD::FREM,              MVT::f64,   Expand);
 
+  // Lower f16 conversion operations into library calls
+  setOperationAction(ISD::FP16_TO_FP,        MVT::f32,   Expand);
+  setOperationAction(ISD::FP_TO_FP16,        MVT::f32,   Expand);
+  setOperationAction(ISD::FP16_TO_FP,        MVT::f64,   Expand);
+  setOperationAction(ISD::FP_TO_FP16,        MVT::f64,   Expand);
+
   setOperationAction(ISD::EH_RETURN, MVT::Other, Custom);
 
   setOperationAction(ISD::VASTART,           MVT::Other, Custom);
@@ -346,13 +386,6 @@ MipsTargetLowering::MipsTargetLowering(const MipsTargetMachine &TM,
   setOperationAction(ISD::VACOPY,            MVT::Other, Expand);
   setOperationAction(ISD::VAEND,             MVT::Other, Expand);
 
-  // @LOCALMOD-BEGIN
-  if (Subtarget.isTargetNaCl())
-    setOperationAction(ISD::INTRINSIC_WO_CHAIN, MVT::Other, Custom);
-  else
-    setOperationAction(ISD::INTRINSIC_WO_CHAIN, MVT::i64, Custom);
-  // @LOCALMOD-END
-
   // Use the default for now
   setOperationAction(ISD::STACKSAVE,         MVT::Other, Expand);
   setOperationAction(ISD::STACKRESTORE,      MVT::Other, Expand);
@@ -424,12 +457,13 @@ const MipsTargetLowering *MipsTargetLowering::create(const MipsTargetMachine &TM
 FastISel *
 MipsTargetLowering::createFastISel(FunctionLoweringInfo &funcInfo,
                                   const TargetLibraryInfo *libInfo) const {
-  if (!EnableMipsFastISel)
+  if (!funcInfo.MF->getTarget().Options.EnableFastISel)
     return TargetLowering::createFastISel(funcInfo, libInfo);
   return Mips::createFastISel(funcInfo, libInfo);
 }
 
-EVT MipsTargetLowering::getSetCCResultType(LLVMContext &, EVT VT) const {
+EVT MipsTargetLowering::getSetCCResultType(const DataLayout &, LLVMContext &,
+                                           EVT VT) const {
   if (!VT.isVector())
     return MVT::i32;
   return VT.changeVectorElementTypeToInteger();
@@ -531,7 +565,7 @@ static SDValue createFPCmp(SelectionDAG &DAG, const SDValue &Op) {
   ISD::CondCode CC = cast<CondCodeSDNode>(Op.getOperand(2))->get();
 
   return DAG.getNode(MipsISD::FPCmp, DL, MVT::Glue, LHS, RHS,
-                     DAG.getConstant(condCodeToFCC(CC), MVT::i32));
+                     DAG.getConstant(condCodeToFCC(CC), DL, MVT::i32));
 }
 
 // Creates and returns a CMovFPT/F node.
@@ -686,9 +720,11 @@ static SDValue performANDCombine(SDNode *N, SelectionDAG &DAG,
   if (SMPos != 0 || Pos + SMSize > ValTy.getSizeInBits())
     return SDValue();
 
-  return DAG.getNode(MipsISD::Ext, SDLoc(N), ValTy,
-                     ShiftRight.getOperand(0), DAG.getConstant(Pos, MVT::i32),
-                     DAG.getConstant(SMSize, MVT::i32));
+  SDLoc DL(N);
+  return DAG.getNode(MipsISD::Ext, DL, ValTy,
+                     ShiftRight.getOperand(0),
+                     DAG.getConstant(Pos, DL, MVT::i32),
+                     DAG.getConstant(SMSize, DL, MVT::i32));
 }
 
 static SDValue performORCombine(SDNode *N, SelectionDAG &DAG,
@@ -740,9 +776,11 @@ static SDValue performORCombine(SDNode *N, SelectionDAG &DAG,
   if ((Shamt != SMPos0) || (SMPos0 + SMSize0 > ValTy.getSizeInBits()))
     return SDValue();
 
-  return DAG.getNode(MipsISD::Ins, SDLoc(N), ValTy, Shl.getOperand(0),
-                     DAG.getConstant(SMPos0, MVT::i32),
-                     DAG.getConstant(SMSize0, MVT::i32), And0.getOperand(0));
+  SDLoc DL(N);
+  return DAG.getNode(MipsISD::Ins, DL, ValTy, Shl.getOperand(0),
+                     DAG.getConstant(SMPos0, DL, MVT::i32),
+                     DAG.getConstant(SMSize0, DL, MVT::i32),
+                     And0.getOperand(0));
 }
 
 static SDValue performADDCombine(SDNode *N, SelectionDAG &DAG,
@@ -1538,12 +1576,13 @@ SDValue MipsTargetLowering::lowerBR_JT(SDValue Op, SelectionDAG &DAG) const {
   SDValue Table = Op.getOperand(1);
   SDValue Index = Op.getOperand(2);
   SDLoc DL(Op);
-  EVT PTy = getPointerTy();
+  auto &TD = DAG.getDataLayout();
+  EVT PTy = getPointerTy(TD);
   unsigned EntrySize =
-    DAG.getMachineFunction().getJumpTableInfo()->getEntrySize(*getDataLayout());
+      DAG.getMachineFunction().getJumpTableInfo()->getEntrySize(TD);
 
   Index = DAG.getNode(ISD::MUL, DL, PTy, Index,
-                      DAG.getConstant(EntrySize, PTy));
+                      DAG.getConstant(EntrySize, DL, PTy));
   SDValue Addr = DAG.getNode(ISD::ADD, DL, PTy, Index, Table);
 
   EVT MemVT = EVT::getIntegerVT(*DAG.getContext(), EntrySize * 8);
@@ -1581,7 +1620,7 @@ SDValue MipsTargetLowering::lowerBRCOND(SDValue Op, SelectionDAG &DAG) const {
   Mips::CondCode CC =
     (Mips::CondCode)cast<ConstantSDNode>(CCNode)->getZExtValue();
   unsigned Opc = invertFPCondCodeUser(CC) ? Mips::BRANCH_F : Mips::BRANCH_T;
-  SDValue BrCode = DAG.getConstant(Opc, MVT::i32);
+  SDValue BrCode = DAG.getConstant(Opc, DL, MVT::i32);
   SDValue FCC0 = DAG.getRegister(Mips::FCC0, MVT::i32);
   return DAG.getNode(MipsISD::FPBrcond, DL, Op.getValueType(), Chain, BrCode,
                      FCC0, Dest, CondRes);
@@ -1606,10 +1645,10 @@ lowerSELECT_CC(SDValue Op, SelectionDAG &DAG) const
 {
   SDLoc DL(Op);
   EVT Ty = Op.getOperand(0).getValueType();
-  SDValue Cond = DAG.getNode(ISD::SETCC, DL,
-                             getSetCCResultType(*DAG.getContext(), Ty),
-                             Op.getOperand(0), Op.getOperand(1),
-                             Op.getOperand(4));
+  SDValue Cond =
+      DAG.getNode(ISD::SETCC, DL, getSetCCResultType(DAG.getDataLayout(),
+                                                     *DAG.getContext(), Ty),
+                  Op.getOperand(0), Op.getOperand(1), Op.getOperand(4));
 
   return DAG.getNode(ISD::SELECT, DL, Op.getValueType(), Cond, Op.getOperand(2),
                      Op.getOperand(3));
@@ -1622,10 +1661,11 @@ SDValue MipsTargetLowering::lowerSETCC(SDValue Op, SelectionDAG &DAG) const {
   assert(Cond.getOpcode() == MipsISD::FPCmp &&
          "Floating point operand expected.");
 
-  SDValue True  = DAG.getConstant(1, MVT::i32);
-  SDValue False = DAG.getConstant(0, MVT::i32);
+  SDLoc DL(Op);
+  SDValue True  = DAG.getConstant(1, DL, MVT::i32);
+  SDValue False = DAG.getConstant(0, DL, MVT::i32);
 
-  return createCMovFP(DAG, Cond, True, False, SDLoc(Op));
+  return createCMovFP(DAG, Cond, True, False, DL);
 }
 
 SDValue MipsTargetLowering::lowerGlobalAddress(SDValue Op,
@@ -1671,50 +1711,6 @@ SDValue MipsTargetLowering::lowerBlockAddress(SDValue Op,
   return getAddrLocal(N, SDLoc(N), Ty, DAG, ABI.IsN32() || ABI.IsN64());
 }
 
-// @LOCALMOD-BEGIN
-SDValue MipsTargetLowering::
-GetNaClThreadPointer(SelectionDAG &DAG, SDLoc DL) const {
-  EVT PtrVT = getPointerTy();
-  SDValue ThreadPointer;
-  if (llvm::TLSUseCall) {
-    unsigned PtrSize = PtrVT.getSizeInBits();
-    IntegerType *PtrTy = Type::getIntNTy(*DAG.getContext(), PtrSize);
-
-    // We must check whether the __nacl_read_tp is defined in the module because
-    // local and global pic functions are called differently. If the function
-    // is local the address is calculated with %got and %lo relocations.
-    // Otherwise, the address is calculated with %call16 relocation.
-    const Function *NaClReadTp = NULL;
-    const Module *M = DAG.getMachineFunction().getFunction()->getParent();
-    for (Module::const_iterator I = M->getFunctionList().begin(),
-           E = M->getFunctionList().end(); I != E; ++I) {
-      if (I->getName() == "__nacl_read_tp") {
-        NaClReadTp = I;
-        break;
-      }
-    }
-
-    SDValue TlsReadTp;
-    if (NaClReadTp == NULL)
-      TlsReadTp = DAG.getExternalSymbol("__nacl_read_tp", PtrVT);
-    else
-      TlsReadTp = DAG.getGlobalAddress(NaClReadTp, DL, PtrVT);
-
-    ArgListTy Args;
-    TargetLowering::CallLoweringInfo CLI(DAG);
-    CLI.setDebugLoc(DL).setChain(DAG.getEntryNode())
-      .setCallee(CallingConv::C, PtrTy, TlsReadTp, std::move(Args), 0);
-    std::pair<SDValue, SDValue> CallResult = LowerCallTo(CLI);
-
-    ThreadPointer = CallResult.first;
-  } else {
-    ThreadPointer = DAG.getCopyFromReg(DAG.getEntryNode(), DL,
-                                       Mips::T8, PtrVT);
-  }
-  return ThreadPointer;
-}
-// @LOCALMOD-END
-
 SDValue MipsTargetLowering::
 lowerGlobalTLSAddress(SDValue Op, SelectionDAG &DAG) const
 {
@@ -1723,33 +1719,15 @@ lowerGlobalTLSAddress(SDValue Op, SelectionDAG &DAG) const
   // Local Exec TLS Model.
 
   GlobalAddressSDNode *GA = cast<GlobalAddressSDNode>(Op);
+  if (DAG.getTarget().Options.EmulatedTLS)
+    return LowerToTLSEmulatedModel(GA, DAG);
+
   SDLoc DL(GA);
   const GlobalValue *GV = GA->getGlobal();
-  EVT PtrVT = getPointerTy();
+  EVT PtrVT = getPointerTy(DAG.getDataLayout());
 
   TLSModel::Model model = getTargetMachine().getTLSModel(GV);
 
-  // @LOCALMOD-BEGIN
-  if (Subtarget.isTargetNaCl()) {
-    SDVTList VTs = DAG.getVTList(MVT::i32);
-    SDValue TGAHi = DAG.getTargetGlobalAddress(GV, DL, MVT::i32, 0,
-                                                 MipsII::MO_TPREL_HI);
-    SDValue TGALo = DAG.getTargetGlobalAddress(GV, DL, MVT::i32, 0,
-                                                 MipsII::MO_TPREL_LO);
-    SDValue Hi = DAG.getNode(MipsISD::Hi, DL, VTs, TGAHi);
-    SDValue Lo = DAG.getNode(MipsISD::Lo, DL, MVT::i32, TGALo);
-    SDValue Offset = DAG.getNode(ISD::ADD, DL, MVT::i32, Hi, Lo);
-
-    SDValue ThreadPointer = GetNaClThreadPointer(DAG, DL);
-    // tprel_hi and tprel_lo relocations expect that thread pointer is offset
-    // by 0x7000 from the start of the TLS data area.
-    SDValue TPOffset = DAG.getConstant(0x7000, MVT::i32);
-    SDValue ThreadPointer2 = DAG.getNode(ISD::ADD, DL, PtrVT, ThreadPointer,
-                                         TPOffset);
-    return DAG.getNode(ISD::ADD, DL, PtrVT, ThreadPointer2, Offset);
-  }
-  // @LOCALMOD-END
-
   if (model == TLSModel::GeneralDynamic || model == TLSModel::LocalDynamic) {
     // General Dynamic and Local Dynamic TLS Model.
     unsigned Flag = (model == TLSModel::LocalDynamic) ? MipsII::MO_TLSLDM
@@ -1838,7 +1816,8 @@ lowerConstantPool(SDValue Op, SelectionDAG &DAG) const
         static_cast<const MipsTargetObjectFile *>(
             getTargetMachine().getObjFileLowering());
 
-    if (TLOF->IsConstantInSmallSection(N->getConstVal(), getTargetMachine()))
+    if (TLOF->IsConstantInSmallSection(DAG.getDataLayout(), N->getConstVal(),
+                                       getTargetMachine()))
       // %gp_rel relocation
       return getAddrGPRel(N, SDLoc(N), Ty, DAG);
 
@@ -1854,7 +1833,7 @@ SDValue MipsTargetLowering::lowerVASTART(SDValue Op, SelectionDAG &DAG) const {
 
   SDLoc DL(Op);
   SDValue FI = DAG.getFrameIndex(FuncInfo->getVarArgsFrameIndex(),
-                                 getPointerTy());
+                                 getPointerTy(MF.getDataLayout()));
 
   // vastart just stores the address of the VarArgsFrameIndex slot into the
   // memory location argument.
@@ -1873,9 +1852,9 @@ SDValue MipsTargetLowering::lowerVAARG(SDValue Op, SelectionDAG &DAG) const {
   SDLoc DL(Node);
   unsigned ArgSlotSizeInBytes = (ABI.IsN32() || ABI.IsN64()) ? 8 : 4;
 
-  SDValue VAListLoad = DAG.getLoad(getPointerTy(), DL, Chain, VAListPtr,
-                                   MachinePointerInfo(SV), false, false, false,
-                                   0);
+  SDValue VAListLoad =
+      DAG.getLoad(getPointerTy(DAG.getDataLayout()), DL, Chain, VAListPtr,
+                  MachinePointerInfo(SV), false, false, false, 0);
   SDValue VAList = VAListLoad;
 
   // Re-align the pointer if necessary.
@@ -1889,19 +1868,21 @@ SDValue MipsTargetLowering::lowerVAARG(SDValue Op, SelectionDAG &DAG) const {
     assert(((Align & (Align-1)) == 0) && "Expected Align to be a power of 2");
 
     VAList = DAG.getNode(ISD::ADD, DL, VAList.getValueType(), VAList,
-                         DAG.getConstant(Align - 1,
-                                         VAList.getValueType()));
+                         DAG.getConstant(Align - 1, DL, VAList.getValueType()));
 
     VAList = DAG.getNode(ISD::AND, DL, VAList.getValueType(), VAList,
-                         DAG.getConstant(-(int64_t)Align,
+                         DAG.getConstant(-(int64_t)Align, DL,
                                          VAList.getValueType()));
   }
 
   // Increment the pointer, VAList, to the next vaarg.
-  unsigned ArgSizeInBytes = getDataLayout()->getTypeAllocSize(VT.getTypeForEVT(*DAG.getContext()));
+  auto &TD = DAG.getDataLayout();
+  unsigned ArgSizeInBytes =
+      TD.getTypeAllocSize(VT.getTypeForEVT(*DAG.getContext()));
   SDValue Tmp3 = DAG.getNode(ISD::ADD, DL, VAList.getValueType(), VAList,
-                             DAG.getConstant(RoundUpToAlignment(ArgSizeInBytes, ArgSlotSizeInBytes),
-                                             VAList.getValueType()));
+                             DAG.getConstant(RoundUpToAlignment(ArgSizeInBytes,
+                                                            ArgSlotSizeInBytes),
+                                             DL, VAList.getValueType()));
   // Store the incremented VAList to the legalized pointer
   Chain = DAG.getStore(VAListLoad.getValue(1), DL, Tmp3, VAListPtr,
                       MachinePointerInfo(SV), false, false, 0);
@@ -1914,7 +1895,7 @@ SDValue MipsTargetLowering::lowerVAARG(SDValue Op, SelectionDAG &DAG) const {
   if (!Subtarget.isLittle() && ArgSizeInBytes < ArgSlotSizeInBytes) {
     unsigned Adjustment = ArgSlotSizeInBytes - ArgSizeInBytes;
     VAList = DAG.getNode(ISD::ADD, DL, VAListPtr.getValueType(), VAList,
-                         DAG.getIntPtrConstant(Adjustment));
+                         DAG.getIntPtrConstant(Adjustment, DL));
   }
   // Load the actual argument out of the pointer VAList
   return DAG.getLoad(VT, DL, Chain, VAList, MachinePointerInfo(), false, false,
@@ -1925,9 +1906,9 @@ static SDValue lowerFCOPYSIGN32(SDValue Op, SelectionDAG &DAG,
                                 bool HasExtractInsert) {
   EVT TyX = Op.getOperand(0).getValueType();
   EVT TyY = Op.getOperand(1).getValueType();
-  SDValue Const1 = DAG.getConstant(1, MVT::i32);
-  SDValue Const31 = DAG.getConstant(31, MVT::i32);
   SDLoc DL(Op);
+  SDValue Const1 = DAG.getConstant(1, DL, MVT::i32);
+  SDValue Const31 = DAG.getConstant(31, DL, MVT::i32);
   SDValue Res;
 
   // If operand is of type f64, extract the upper 32-bit. Otherwise, bitcast it
@@ -1963,7 +1944,8 @@ static SDValue lowerFCOPYSIGN32(SDValue Op, SelectionDAG &DAG,
     return DAG.getNode(ISD::BITCAST, DL, Op.getOperand(0).getValueType(), Res);
 
   SDValue LowX = DAG.getNode(MipsISD::ExtractElementF64, DL, MVT::i32,
-                             Op.getOperand(0), DAG.getConstant(0, MVT::i32));
+                             Op.getOperand(0),
+                             DAG.getConstant(0, DL, MVT::i32));
   return DAG.getNode(MipsISD::BuildPairF64, DL, MVT::f64, LowX, Res);
 }
 
@@ -1972,8 +1954,8 @@ static SDValue lowerFCOPYSIGN64(SDValue Op, SelectionDAG &DAG,
   unsigned WidthX = Op.getOperand(0).getValueSizeInBits();
   unsigned WidthY = Op.getOperand(1).getValueSizeInBits();
   EVT TyX = MVT::getIntegerVT(WidthX), TyY = MVT::getIntegerVT(WidthY);
-  SDValue Const1 = DAG.getConstant(1, MVT::i32);
   SDLoc DL(Op);
+  SDValue Const1 = DAG.getConstant(1, DL, MVT::i32);
 
   // Bitcast to integer nodes.
   SDValue X = DAG.getNode(ISD::BITCAST, DL, TyX, Op.getOperand(0));
@@ -1983,7 +1965,7 @@ static SDValue lowerFCOPYSIGN64(SDValue Op, SelectionDAG &DAG,
     // ext  E, Y, width(Y) - 1, 1  ; extract bit width(Y)-1 of Y
     // ins  X, E, width(X) - 1, 1  ; insert extracted bit at bit width(X)-1 of X
     SDValue E = DAG.getNode(MipsISD::Ext, DL, TyY, Y,
-                            DAG.getConstant(WidthY - 1, MVT::i32), Const1);
+                            DAG.getConstant(WidthY - 1, DL, MVT::i32), Const1);
 
     if (WidthX > WidthY)
       E = DAG.getNode(ISD::ZERO_EXTEND, DL, TyX, E);
@@ -1991,7 +1973,8 @@ static SDValue lowerFCOPYSIGN64(SDValue Op, SelectionDAG &DAG,
       E = DAG.getNode(ISD::TRUNCATE, DL, TyX, E);
 
     SDValue I = DAG.getNode(MipsISD::Ins, DL, TyX, E,
-                            DAG.getConstant(WidthX - 1, MVT::i32), Const1, X);
+                            DAG.getConstant(WidthX - 1, DL, MVT::i32), Const1,
+                            X);
     return DAG.getNode(ISD::BITCAST, DL, Op.getOperand(0).getValueType(), I);
   }
 
@@ -2003,7 +1986,7 @@ static SDValue lowerFCOPYSIGN64(SDValue Op, SelectionDAG &DAG,
   SDValue SllX = DAG.getNode(ISD::SHL, DL, TyX, X, Const1);
   SDValue SrlX = DAG.getNode(ISD::SRL, DL, TyX, SllX, Const1);
   SDValue SrlY = DAG.getNode(ISD::SRL, DL, TyY, Y,
-                             DAG.getConstant(WidthY - 1, MVT::i32));
+                             DAG.getConstant(WidthY - 1, DL, MVT::i32));
 
   if (WidthX > WidthY)
     SrlY = DAG.getNode(ISD::ZERO_EXTEND, DL, TyX, SrlY);
@@ -2011,7 +1994,7 @@ static SDValue lowerFCOPYSIGN64(SDValue Op, SelectionDAG &DAG,
     SrlY = DAG.getNode(ISD::TRUNCATE, DL, TyX, SrlY);
 
   SDValue SllY = DAG.getNode(ISD::SHL, DL, TyX, SrlY,
-                             DAG.getConstant(WidthX - 1, MVT::i32));
+                             DAG.getConstant(WidthX - 1, DL, MVT::i32));
   SDValue Or = DAG.getNode(ISD::OR, DL, TyX, SrlX, SllY);
   return DAG.getNode(ISD::BITCAST, DL, Op.getOperand(0).getValueType(), Or);
 }
@@ -2083,7 +2066,7 @@ SDValue MipsTargetLowering::lowerEH_RETURN(SDValue Op, SelectionDAG &DAG)
   Chain = DAG.getCopyToReg(Chain, DL, AddrReg, Handler, Chain.getValue(1));
   return DAG.getNode(MipsISD::EH_RETURN, DL, MVT::Other, Chain,
                      DAG.getRegister(OffsetReg, Ty),
-                     DAG.getRegister(AddrReg, getPointerTy()),
+                     DAG.getRegister(AddrReg, getPointerTy(MF.getDataLayout())),
                      Chain.getValue(1));
 }
 
@@ -2094,7 +2077,7 @@ SDValue MipsTargetLowering::lowerATOMIC_FENCE(SDValue Op,
   unsigned SType = 0;
   SDLoc DL(Op);
   return DAG.getNode(MipsISD::Sync, DL, MVT::Other, Op.getOperand(0),
-                     DAG.getConstant(SType, MVT::i32));
+                     DAG.getConstant(SType, DL, MVT::i32));
 }
 
 SDValue MipsTargetLowering::lowerShiftLeftParts(SDValue Op,
@@ -2111,17 +2094,17 @@ SDValue MipsTargetLowering::lowerShiftLeftParts(SDValue Op,
   //  lo = 0
   //  hi = (shl lo, shamt[4:0])
   SDValue Not = DAG.getNode(ISD::XOR, DL, MVT::i32, Shamt,
-                            DAG.getConstant(-1, MVT::i32));
+                            DAG.getConstant(-1, DL, MVT::i32));
   SDValue ShiftRight1Lo = DAG.getNode(ISD::SRL, DL, VT, Lo,
-                                      DAG.getConstant(1, VT));
+                                      DAG.getConstant(1, DL, VT));
   SDValue ShiftRightLo = DAG.getNode(ISD::SRL, DL, VT, ShiftRight1Lo, Not);
   SDValue ShiftLeftHi = DAG.getNode(ISD::SHL, DL, VT, Hi, Shamt);
   SDValue Or = DAG.getNode(ISD::OR, DL, VT, ShiftLeftHi, ShiftRightLo);
   SDValue ShiftLeftLo = DAG.getNode(ISD::SHL, DL, VT, Lo, Shamt);
   SDValue Cond = DAG.getNode(ISD::AND, DL, MVT::i32, Shamt,
-                             DAG.getConstant(0x20, MVT::i32));
+                             DAG.getConstant(VT.getSizeInBits(), DL, MVT::i32));
   Lo = DAG.getNode(ISD::SELECT, DL, VT, Cond,
-                   DAG.getConstant(0, VT), ShiftLeftLo);
+                   DAG.getConstant(0, DL, VT), ShiftLeftLo);
   Hi = DAG.getNode(ISD::SELECT, DL, VT, Cond, ShiftLeftLo, Or);
 
   SDValue Ops[2] = {Lo, Hi};
@@ -2149,20 +2132,21 @@ SDValue MipsTargetLowering::lowerShiftRightParts(SDValue Op, SelectionDAG &DAG,
   //   lo = (srl hi, shamt[4:0])
   //   hi = 0
   SDValue Not = DAG.getNode(ISD::XOR, DL, MVT::i32, Shamt,
-                            DAG.getConstant(-1, MVT::i32));
+                            DAG.getConstant(-1, DL, MVT::i32));
   SDValue ShiftLeft1Hi = DAG.getNode(ISD::SHL, DL, VT, Hi,
-                                     DAG.getConstant(1, VT));
+                                     DAG.getConstant(1, DL, VT));
   SDValue ShiftLeftHi = DAG.getNode(ISD::SHL, DL, VT, ShiftLeft1Hi, Not);
   SDValue ShiftRightLo = DAG.getNode(ISD::SRL, DL, VT, Lo, Shamt);
   SDValue Or = DAG.getNode(ISD::OR, DL, VT, ShiftLeftHi, ShiftRightLo);
   SDValue ShiftRightHi = DAG.getNode(IsSRA ? ISD::SRA : ISD::SRL,
                                      DL, VT, Hi, Shamt);
   SDValue Cond = DAG.getNode(ISD::AND, DL, MVT::i32, Shamt,
-                             DAG.getConstant(0x20, MVT::i32));
-  SDValue Shift31 = DAG.getNode(ISD::SRA, DL, VT, Hi, DAG.getConstant(31, VT));
+                             DAG.getConstant(VT.getSizeInBits(), DL, MVT::i32));
+  SDValue Ext = DAG.getNode(ISD::SRA, DL, VT, Hi,
+                            DAG.getConstant(VT.getSizeInBits() - 1, DL, VT));
   Lo = DAG.getNode(ISD::SELECT, DL, VT, Cond, ShiftRightHi, Or);
   Hi = DAG.getNode(ISD::SELECT, DL, VT, Cond,
-                   IsSRA ? Shift31 : DAG.getConstant(0, VT), ShiftRightHi);
+                   IsSRA ? Ext : DAG.getConstant(0, DL, VT), ShiftRightHi);
 
   SDValue Ops[2] = {Lo, Hi};
   return DAG.getMergeValues(Ops, DL);
@@ -2178,7 +2162,7 @@ static SDValue createLoadLR(unsigned Opc, SelectionDAG &DAG, LoadSDNode *LD,
 
   if (Offset)
     Ptr = DAG.getNode(ISD::ADD, DL, BasePtrVT, Ptr,
-                      DAG.getConstant(Offset, BasePtrVT));
+                      DAG.getConstant(Offset, DL, BasePtrVT));
 
   SDValue Ops[] = { Chain, Ptr, Src };
   return DAG.getMemIntrinsicNode(Opc, DL, VTList, Ops, MemVT,
@@ -2243,7 +2227,7 @@ SDValue MipsTargetLowering::lowerLOAD(SDValue Op, SelectionDAG &DAG) const {
   //  (set tmp2, (shl tmp1, 32))
   //  (set dst, (srl tmp2, 32))
   SDLoc DL(LD);
-  SDValue Const32 = DAG.getConstant(32, MVT::i32);
+  SDValue Const32 = DAG.getConstant(32, DL, MVT::i32);
   SDValue SLL = DAG.getNode(ISD::SHL, DL, MVT::i64, LWR, Const32);
   SDValue SRL = DAG.getNode(ISD::SRL, DL, MVT::i64, SLL, Const32);
   SDValue Ops[] = { SRL, LWR.getValue(1) };
@@ -2259,7 +2243,7 @@ static SDValue createStoreLR(unsigned Opc, SelectionDAG &DAG, StoreSDNode *SD,
 
   if (Offset)
     Ptr = DAG.getNode(ISD::ADD, DL, BasePtrVT, Ptr,
-                      DAG.getConstant(Offset, BasePtrVT));
+                      DAG.getConstant(Offset, DL, BasePtrVT));
 
   SDValue Ops[] = { Chain, Value, Ptr };
   return DAG.getMemIntrinsicNode(Opc, DL, VTList, Ops, MemVT,
@@ -2341,8 +2325,9 @@ SDValue MipsTargetLowering::lowerADD(SDValue Op, SelectionDAG &DAG) const {
   EVT ValTy = Op->getValueType(0);
   int FI = MFI->CreateFixedObject(Op.getValueSizeInBits() / 8, 0, false);
   SDValue InArgsAddr = DAG.getFrameIndex(FI, ValTy);
-  return DAG.getNode(ISD::ADD, SDLoc(Op), ValTy, InArgsAddr,
-                     DAG.getConstant(0, ValTy));
+  SDLoc DL(Op);
+  return DAG.getNode(ISD::ADD, DL, ValTy, InArgsAddr,
+                     DAG.getConstant(0, DL, ValTy));
 }
 
 SDValue MipsTargetLowering::lowerFP_TO_SINT(SDValue Op,
@@ -2498,15 +2483,16 @@ MipsTargetLowering::passArgOnStack(SDValue StackPtr, unsigned Offset,
                                    SDValue Chain, SDValue Arg, SDLoc DL,
                                    bool IsTailCall, SelectionDAG &DAG) const {
   if (!IsTailCall) {
-    SDValue PtrOff = DAG.getNode(ISD::ADD, DL, getPointerTy(), StackPtr,
-                                 DAG.getIntPtrConstant(Offset));
+    SDValue PtrOff =
+        DAG.getNode(ISD::ADD, DL, getPointerTy(DAG.getDataLayout()), StackPtr,
+                    DAG.getIntPtrConstant(Offset, DL));
     return DAG.getStore(Chain, DL, Arg, PtrOff, MachinePointerInfo(), false,
                         false, 0);
   }
 
   MachineFrameInfo *MFI = DAG.getMachineFunction().getFrameInfo();
   int FI = MFI->CreateFixedObject(Arg.getValueSizeInBits() / 8, Offset, false);
-  SDValue FIN = DAG.getFrameIndex(FI, getPointerTy());
+  SDValue FIN = DAG.getFrameIndex(FI, getPointerTy(DAG.getDataLayout()));
   return DAG.getStore(Chain, DL, Arg, FIN, MachinePointerInfo(),
                       /*isVolatile=*/ true, false, 0);
 }
@@ -2625,13 +2611,14 @@ MipsTargetLowering::LowerCall(TargetLowering::CallLoweringInfo &CLI,
   // byval arguments to the stack.
   unsigned StackAlignment = TFL->getStackAlignment();
   NextStackOffset = RoundUpToAlignment(NextStackOffset, StackAlignment);
-  SDValue NextStackOffsetVal = DAG.getIntPtrConstant(NextStackOffset, true);
+  SDValue NextStackOffsetVal = DAG.getIntPtrConstant(NextStackOffset, DL, true);
 
   if (!IsTailCall)
     Chain = DAG.getCALLSEQ_START(Chain, NextStackOffsetVal, DL);
 
-  SDValue StackPtr = DAG.getCopyFromReg(
-      Chain, DL, ABI.IsN64() ? Mips::SP_64 : Mips::SP, getPointerTy());
+  SDValue StackPtr =
+      DAG.getCopyFromReg(Chain, DL, ABI.IsN64() ? Mips::SP_64 : Mips::SP,
+                         getPointerTy(DAG.getDataLayout()));
 
   // With EABI is it possible to have 16 args on registers.
   std::deque< std::pair<unsigned, SDValue> > RegsToPass;
@@ -2677,9 +2664,9 @@ MipsTargetLowering::LowerCall(TargetLowering::CallLoweringInfo &CLI,
           Arg = DAG.getNode(ISD::BITCAST, DL, LocVT, Arg);
         else if (ValVT == MVT::f64 && LocVT == MVT::i32) {
           SDValue Lo = DAG.getNode(MipsISD::ExtractElementF64, DL, MVT::i32,
-                                   Arg, DAG.getConstant(0, MVT::i32));
+                                   Arg, DAG.getConstant(0, DL, MVT::i32));
           SDValue Hi = DAG.getNode(MipsISD::ExtractElementF64, DL, MVT::i32,
-                                   Arg, DAG.getConstant(1, MVT::i32));
+                                   Arg, DAG.getConstant(1, DL, MVT::i32));
           if (!Subtarget.isLittle())
             std::swap(Lo, Hi);
           unsigned LocRegLo = VA.getLocReg();
@@ -2718,7 +2705,7 @@ MipsTargetLowering::LowerCall(TargetLowering::CallLoweringInfo &CLI,
       unsigned LocSizeInBits = VA.getLocVT().getSizeInBits();
       Arg = DAG.getNode(
           ISD::SHL, DL, VA.getLocVT(), Arg,
-          DAG.getConstant(LocSizeInBits - ValSizeInBits, VA.getLocVT()));
+          DAG.getConstant(LocSizeInBits - ValSizeInBits, DL, VA.getLocVT()));
     }
 
     // Arguments that can be passed on register must be kept at
@@ -2769,7 +2756,8 @@ MipsTargetLowering::LowerCall(TargetLowering::CallLoweringInfo &CLI,
         IsCallReloc = true;
       }
     } else
-      Callee = DAG.getTargetGlobalAddress(G->getGlobal(), DL, getPointerTy(), 0,
+      Callee = DAG.getTargetGlobalAddress(G->getGlobal(), DL,
+                                          getPointerTy(DAG.getDataLayout()), 0,
                                           MipsII::MO_NO_FLAG);
     GlobalOrExternal = true;
   }
@@ -2777,8 +2765,8 @@ MipsTargetLowering::LowerCall(TargetLowering::CallLoweringInfo &CLI,
     const char *Sym = S->getSymbol();
 
     if (!ABI.IsN64() && !IsPIC) // !N64 && static
-      Callee =
-          DAG.getTargetExternalSymbol(Sym, getPointerTy(), MipsII::MO_NO_FLAG);
+      Callee = DAG.getTargetExternalSymbol(
+          Sym, getPointerTy(DAG.getDataLayout()), MipsII::MO_NO_FLAG);
     else if (LargeGOT) {
       Callee = getAddrGlobalLargeGOT(S, DL, Ty, DAG, MipsII::MO_CALL_HI16,
                                      MipsII::MO_CALL_LO16, Chain,
@@ -2807,7 +2795,7 @@ MipsTargetLowering::LowerCall(TargetLowering::CallLoweringInfo &CLI,
 
   // Create the CALLSEQ_END node.
   Chain = DAG.getCALLSEQ_END(Chain, NextStackOffsetVal,
-                             DAG.getIntPtrConstant(0, true), InFlag, DL);
+                             DAG.getIntPtrConstant(0, DL, true), InFlag, DL);
   InFlag = Chain.getValue(1);
 
   // Handle result values, copying them out of physregs into vregs that we
@@ -2846,7 +2834,7 @@ SDValue MipsTargetLowering::LowerCallResult(
           VA.getLocInfo() == CCValAssign::ZExtUpper ? ISD::SRL : ISD::SRA;
       Val = DAG.getNode(
           Shift, DL, VA.getLocVT(), Val,
-          DAG.getConstant(LocSizeInBits - ValSizeInBits, VA.getLocVT()));
+          DAG.getConstant(LocSizeInBits - ValSizeInBits, DL, VA.getLocVT()));
     }
 
     switch (VA.getLocInfo()) {
@@ -2899,7 +2887,7 @@ static SDValue UnpackFromArgumentSlot(SDValue Val, const CCValAssign &VA,
         VA.getLocInfo() == CCValAssign::ZExtUpper ? ISD::SRL : ISD::SRA;
     Val = DAG.getNode(
         Opcode, DL, VA.getLocVT(), Val,
-        DAG.getConstant(LocSizeInBits - ValSizeInBits, VA.getLocVT()));
+        DAG.getConstant(LocSizeInBits - ValSizeInBits, DL, VA.getLocVT()));
     break;
   }
   }
@@ -3036,7 +3024,7 @@ MipsTargetLowering::LowerFormalArguments(SDValue Chain,
         // when allocating floating point values to integer registers.
         // This shouldn't influence how we load the value into registers unless
         // we are targetting softfloat.
-        if (VA.getValVT().isFloatingPoint() && !Subtarget.abiUsesSoftFloat())
+        if (VA.getValVT().isFloatingPoint() && !Subtarget.useSoftFloat())
           LocVT = VA.getValVT();
       }
 
@@ -3048,7 +3036,7 @@ MipsTargetLowering::LowerFormalArguments(SDValue Chain,
                                       VA.getLocMemOffset(), true);
 
       // Create load nodes to retrieve arguments from the stack
-      SDValue FIN = DAG.getFrameIndex(FI, getPointerTy());
+      SDValue FIN = DAG.getFrameIndex(FI, getPointerTy(DAG.getDataLayout()));
       SDValue ArgValue = DAG.getLoad(LocVT, DL, Chain, FIN,
                                      MachinePointerInfo::getFixedStack(FI),
                                      false, false, false, 0);
@@ -3106,7 +3094,7 @@ MipsTargetLowering::CanLowerReturn(CallingConv::ID CallConv,
 
 bool
 MipsTargetLowering::shouldSignExtendTypeInLibCall(EVT Type, bool IsSigned) const {
-  if (Subtarget.hasMips3() && Subtarget.abiUsesSoftFloat()) {
+  if (Subtarget.hasMips3() && Subtarget.useSoftFloat()) {
     if (Type == MVT::i32)
       return true;
   }
@@ -3173,7 +3161,7 @@ MipsTargetLowering::LowerReturn(SDValue Chain,
       unsigned LocSizeInBits = VA.getLocVT().getSizeInBits();
       Val = DAG.getNode(
           ISD::SHL, DL, VA.getLocVT(), Val,
-          DAG.getConstant(LocSizeInBits - ValSizeInBits, VA.getLocVT()));
+          DAG.getConstant(LocSizeInBits - ValSizeInBits, DL, VA.getLocVT()));
     }
 
     Chain = DAG.getCopyToReg(Chain, DL, VA.getLocReg(), Val, Flag);
@@ -3193,12 +3181,13 @@ MipsTargetLowering::LowerReturn(SDValue Chain,
 
     if (!Reg)
       llvm_unreachable("sret virtual register not created in the entry block");
-    SDValue Val = DAG.getCopyFromReg(Chain, DL, Reg, getPointerTy());
+    SDValue Val =
+        DAG.getCopyFromReg(Chain, DL, Reg, getPointerTy(DAG.getDataLayout()));
     unsigned V0 = ABI.IsN64() ? Mips::V0_64 : Mips::V0;
 
     Chain = DAG.getCopyToReg(Chain, DL, V0, Val, Flag);
     Flag = Chain.getValue(1);
-    RetOps.push_back(DAG.getRegister(V0, getPointerTy()));
+    RetOps.push_back(DAG.getRegister(V0, getPointerTy(DAG.getDataLayout())));
   }
 
   RetOps[0] = Chain;  // Update chain.
@@ -3217,9 +3206,8 @@ MipsTargetLowering::LowerReturn(SDValue Chain,
 
 /// getConstraintType - Given a constraint letter, return the type of
 /// constraint it is for this target.
-MipsTargetLowering::ConstraintType MipsTargetLowering::
-getConstraintType(const std::string &Constraint) const
-{
+MipsTargetLowering::ConstraintType
+MipsTargetLowering::getConstraintType(StringRef Constraint) const {
   // Mips specific constraints
   // GCC config/mips/constraints.md
   //
@@ -3309,9 +3297,8 @@ MipsTargetLowering::getSingleConstraintMatchWeight(
 /// into non-numeric and numeric parts (Prefix and Reg). The first boolean flag
 /// that is returned indicates whether parsing was successful. The second flag
 /// is true if the numeric part exists.
-static std::pair<bool, bool>
-parsePhysicalReg(StringRef C, std::string &Prefix,
-                 unsigned long long &Reg) {
+static std::pair<bool, bool> parsePhysicalReg(StringRef C, StringRef &Prefix,
+                                              unsigned long long &Reg) {
   if (C.front() != '{' || C.back() != '}')
     return std::make_pair(false, false);
 
@@ -3319,7 +3306,7 @@ parsePhysicalReg(StringRef C, std::string &Prefix,
   StringRef::const_iterator I, B = C.begin() + 1, E = C.end() - 1;
   I = std::find_if(B, E, std::ptr_fun(isdigit));
 
-  Prefix.assign(B, I - B);
+  Prefix = StringRef(B, I - B);
 
   // The second flag is set to false if no numeric characters were found.
   if (I == E)
@@ -3335,7 +3322,7 @@ parseRegForInlineAsmConstraint(StringRef C, MVT VT) const {
   const TargetRegisterInfo *TRI =
       Subtarget.getRegisterInfo();
   const TargetRegisterClass *RC;
-  std::string Prefix;
+  StringRef Prefix;
   unsigned long long Reg;
 
   std::pair<bool, bool> R = parsePhysicalReg(C, Prefix, Reg);
@@ -3351,7 +3338,7 @@ parseRegForInlineAsmConstraint(StringRef C, MVT VT) const {
     RC = TRI->getRegClass(Prefix == "hi" ?
                           Mips::HI32RegClassID : Mips::LO32RegClassID);
     return std::make_pair(*(RC->begin()), RC);
-  } else if (Prefix.compare(0, 4, "$msa") == 0) {
+  } else if (Prefix.startswith("$msa")) {
     // Parse $msa(ir|csr|access|save|modify|request|map|unmap)
 
     // No numeric characters follow the name.
@@ -3409,7 +3396,7 @@ parseRegForInlineAsmConstraint(StringRef C, MVT VT) const {
 /// pointer.
 std::pair<unsigned, const TargetRegisterClass *>
 MipsTargetLowering::getRegForInlineAsmConstraint(const TargetRegisterInfo *TRI,
-                                                 const std::string &Constraint,
+                                                 StringRef Constraint,
                                                  MVT VT) const {
   if (Constraint.size() == 1) {
     switch (Constraint[0]) {
@@ -3475,6 +3462,7 @@ void MipsTargetLowering::LowerAsmOperandForConstraint(SDValue Op,
                                                      std::string &Constraint,
                                                      std::vector<SDValue>&Ops,
                                                      SelectionDAG &DAG) const {
+  SDLoc DL(Op);
   SDValue Result;
 
   // Only support length 1 constraints for now.
@@ -3489,7 +3477,7 @@ void MipsTargetLowering::LowerAsmOperandForConstraint(SDValue Op,
       EVT Type = Op.getValueType();
       int64_t Val = C->getSExtValue();
       if (isInt<16>(Val)) {
-        Result = DAG.getTargetConstant(Val, Type);
+        Result = DAG.getTargetConstant(Val, DL, Type);
         break;
       }
     }
@@ -3499,7 +3487,7 @@ void MipsTargetLowering::LowerAsmOperandForConstraint(SDValue Op,
       EVT Type = Op.getValueType();
       int64_t Val = C->getZExtValue();
       if (Val == 0) {
-        Result = DAG.getTargetConstant(0, Type);
+        Result = DAG.getTargetConstant(0, DL, Type);
         break;
       }
     }
@@ -3509,7 +3497,7 @@ void MipsTargetLowering::LowerAsmOperandForConstraint(SDValue Op,
       EVT Type = Op.getValueType();
       uint64_t Val = (uint64_t)C->getZExtValue();
       if (isUInt<16>(Val)) {
-        Result = DAG.getTargetConstant(Val, Type);
+        Result = DAG.getTargetConstant(Val, DL, Type);
         break;
       }
     }
@@ -3519,7 +3507,7 @@ void MipsTargetLowering::LowerAsmOperandForConstraint(SDValue Op,
       EVT Type = Op.getValueType();
       int64_t Val = C->getSExtValue();
       if ((isInt<32>(Val)) && ((Val & 0xffff) == 0)){
-        Result = DAG.getTargetConstant(Val, Type);
+        Result = DAG.getTargetConstant(Val, DL, Type);
         break;
       }
     }
@@ -3529,7 +3517,7 @@ void MipsTargetLowering::LowerAsmOperandForConstraint(SDValue Op,
       EVT Type = Op.getValueType();
       int64_t Val = C->getSExtValue();
       if ((Val >= -65535) && (Val <= -1)) {
-        Result = DAG.getTargetConstant(Val, Type);
+        Result = DAG.getTargetConstant(Val, DL, Type);
         break;
       }
     }
@@ -3539,7 +3527,7 @@ void MipsTargetLowering::LowerAsmOperandForConstraint(SDValue Op,
       EVT Type = Op.getValueType();
       int64_t Val = C->getSExtValue();
       if ((isInt<15>(Val))) {
-        Result = DAG.getTargetConstant(Val, Type);
+        Result = DAG.getTargetConstant(Val, DL, Type);
         break;
       }
     }
@@ -3549,7 +3537,7 @@ void MipsTargetLowering::LowerAsmOperandForConstraint(SDValue Op,
       EVT Type = Op.getValueType();
       int64_t Val = C->getSExtValue();
       if ((Val <= 65535) && (Val >= 1)) {
-        Result = DAG.getTargetConstant(Val, Type);
+        Result = DAG.getTargetConstant(Val, DL, Type);
         break;
       }
     }
@@ -3564,8 +3552,9 @@ void MipsTargetLowering::LowerAsmOperandForConstraint(SDValue Op,
   TargetLowering::LowerAsmOperandForConstraint(Op, Constraint, Ops, DAG);
 }
 
-bool MipsTargetLowering::isLegalAddressingMode(const AddrMode &AM,
-                                               Type *Ty) const {
+bool MipsTargetLowering::isLegalAddressingMode(const DataLayout &DL,
+                                               const AddrMode &AM, Type *Ty,
+                                               unsigned AS) const {
   // No global is ever allowed as a base.
   if (AM.BaseGV)
     return false;
@@ -3616,6 +3605,10 @@ unsigned MipsTargetLowering::getJumpTableEncoding() const {
   return TargetLowering::getJumpTableEncoding();
 }
 
+bool MipsTargetLowering::useSoftFloat() const {
+  return Subtarget.useSoftFloat();
+}
+
 void MipsTargetLowering::copyByValRegs(
     SDValue Chain, SDLoc DL, std::vector<SDValue> &OutChains, SelectionDAG &DAG,
     const ISD::ArgFlagsTy &Flags, SmallVectorImpl<SDValue> &InVals,
@@ -3638,7 +3631,7 @@ void MipsTargetLowering::copyByValRegs(
     FrameObjOffset = VA.getLocMemOffset();
 
   // Create frame object.
-  EVT PtrTy = getPointerTy();
+  EVT PtrTy = getPointerTy(DAG.getDataLayout());
   int FI = MFI->CreateFixedObject(FrameObjSize, FrameObjOffset, true);
   SDValue FIN = DAG.getFrameIndex(FI, PtrTy);
   InVals.push_back(FIN);
@@ -3655,7 +3648,7 @@ void MipsTargetLowering::copyByValRegs(
     unsigned VReg = addLiveIn(MF, ArgReg, RC);
     unsigned Offset = I * GPRSizeInBytes;
     SDValue StorePtr = DAG.getNode(ISD::ADD, DL, PtrTy, FIN,
-                                   DAG.getConstant(Offset, PtrTy));
+                                   DAG.getConstant(Offset, DL, PtrTy));
     SDValue Store = DAG.getStore(Chain, DL, DAG.getRegister(VReg, RegTy),
                                  StorePtr, MachinePointerInfo(FuncArg, Offset),
                                  false, false, 0);
@@ -3675,7 +3668,8 @@ void MipsTargetLowering::passByValArg(
   unsigned OffsetInBytes = 0; // From beginning of struct
   unsigned RegSizeInBytes = Subtarget.getGPRSizeInBytes();
   unsigned Alignment = std::min(Flags.getByValAlign(), RegSizeInBytes);
-  EVT PtrTy = getPointerTy(), RegTy = MVT::getIntegerVT(RegSizeInBytes * 8);
+  EVT PtrTy = getPointerTy(DAG.getDataLayout()),
+      RegTy = MVT::getIntegerVT(RegSizeInBytes * 8);
   unsigned NumRegs = LastReg - FirstReg;
 
   if (NumRegs) {
@@ -3686,7 +3680,7 @@ void MipsTargetLowering::passByValArg(
     // Copy words to registers.
     for (; I < NumRegs - LeftoverBytes; ++I, OffsetInBytes += RegSizeInBytes) {
       SDValue LoadPtr = DAG.getNode(ISD::ADD, DL, PtrTy, Arg,
-                                    DAG.getConstant(OffsetInBytes, PtrTy));
+                                    DAG.getConstant(OffsetInBytes, DL, PtrTy));
       SDValue LoadVal = DAG.getLoad(RegTy, DL, Chain, LoadPtr,
                                     MachinePointerInfo(), false, false, false,
                                     Alignment);
@@ -3712,7 +3706,8 @@ void MipsTargetLowering::passByValArg(
 
         // Load subword.
         SDValue LoadPtr = DAG.getNode(ISD::ADD, DL, PtrTy, Arg,
-                                      DAG.getConstant(OffsetInBytes, PtrTy));
+                                      DAG.getConstant(OffsetInBytes, DL,
+                                                      PtrTy));
         SDValue LoadVal = DAG.getExtLoad(
             ISD::ZEXTLOAD, DL, RegTy, Chain, LoadPtr, MachinePointerInfo(),
             MVT::getIntegerVT(LoadSizeInBytes * 8), false, false, false,
@@ -3728,7 +3723,7 @@ void MipsTargetLowering::passByValArg(
           Shamt = (RegSizeInBytes - (TotalBytesLoaded + LoadSizeInBytes)) * 8;
 
         SDValue Shift = DAG.getNode(ISD::SHL, DL, RegTy, LoadVal,
-                                    DAG.getConstant(Shamt, MVT::i32));
+                                    DAG.getConstant(Shamt, DL, MVT::i32));
 
         if (Val.getNode())
           Val = DAG.getNode(ISD::OR, DL, RegTy, Val, Shift);
@@ -3749,10 +3744,11 @@ void MipsTargetLowering::passByValArg(
   // Copy remainder of byval arg to it with memcpy.
   unsigned MemCpySize = ByValSizeInBytes - OffsetInBytes;
   SDValue Src = DAG.getNode(ISD::ADD, DL, PtrTy, Arg,
-                            DAG.getConstant(OffsetInBytes, PtrTy));
+                            DAG.getConstant(OffsetInBytes, DL, PtrTy));
   SDValue Dst = DAG.getNode(ISD::ADD, DL, PtrTy, StackPtr,
-                            DAG.getIntPtrConstant(VA.getLocMemOffset()));
-  Chain = DAG.getMemcpy(Chain, DL, Dst, Src, DAG.getConstant(MemCpySize, PtrTy),
+                            DAG.getIntPtrConstant(VA.getLocMemOffset(), DL));
+  Chain = DAG.getMemcpy(Chain, DL, Dst, Src,
+                        DAG.getConstant(MemCpySize, DL, PtrTy),
                         Alignment, /*isVolatile=*/false, /*AlwaysInline=*/false,
                         /*isTailCall=*/false,
                         MachinePointerInfo(), MachinePointerInfo());
@@ -3798,7 +3794,7 @@ void MipsTargetLowering::writeVarArgRegs(std::vector<SDValue> &OutChains,
     unsigned Reg = addLiveIn(MF, ArgRegs[I], RC);
     SDValue ArgValue = DAG.getCopyFromReg(Chain, DL, Reg, RegTy);
     FI = MFI->CreateFixedObject(RegSizeInBytes, VaArgOffset, true);
-    SDValue PtrOff = DAG.getFrameIndex(FI, getPointerTy());
+    SDValue PtrOff = DAG.getFrameIndex(FI, getPointerTy(DAG.getDataLayout()));
     SDValue Store = DAG.getStore(Chain, DL, ArgValue, PtrOff,
                                  MachinePointerInfo(), false, false, 0);
     cast<StoreSDNode>(Store.getNode())->getMemOperand()->setValue(
@@ -3931,8 +3927,8 @@ MipsTargetLowering::emitPseudoSELECT(MachineInstr *MI, MachineBasicBlock *BB,
 
 // FIXME? Maybe this could be a TableGen attribute on some registers and
 // this table could be generated automatically from RegInfo.
-unsigned MipsTargetLowering::getRegisterByName(const char* RegName,
-                                               EVT VT) const {
+unsigned MipsTargetLowering::getRegisterByName(const char* RegName, EVT VT,
+                                               SelectionDAG &DAG) const {
   // Named registers is expected to be fairly rare. For now, just support $28
   // since the linux kernel uses it.
   if (Subtarget.isGP64bit()) {
diff --git a/lib/Target/Mips/MipsISelLowering.h b/lib/Target/Mips/MipsISelLowering.h
index c8c77aafe1f..6fe8f830d35 100644
--- a/lib/Target/Mips/MipsISelLowering.h
+++ b/lib/Target/Mips/MipsISelLowering.h
@@ -27,7 +27,7 @@
 
 namespace llvm {
   namespace MipsISD {
-    enum NodeType {
+    enum NodeType : unsigned {
       // Start the numbering from where ISD NodeType finishes.
       FIRST_NUMBER = ISD::BUILTIN_OP_END,
 
@@ -227,7 +227,9 @@ namespace llvm {
     FastISel *createFastISel(FunctionLoweringInfo &funcInfo,
                              const TargetLibraryInfo *libInfo) const override;
 
-    MVT getScalarShiftAmountTy(EVT LHSTy) const override { return MVT::i32; }
+    MVT getScalarShiftAmountTy(const DataLayout &, EVT) const override {
+      return MVT::i32;
+    }
 
     void LowerOperationWrapper(SDNode *N,
                                SmallVectorImpl<SDValue> &Results,
@@ -247,7 +249,8 @@ namespace llvm {
     const char *getTargetNodeName(unsigned Opcode) const override;
 
     /// getSetCCResultType - get the ISD::SETCC result ValueType
-    EVT getSetCCResultType(LLVMContext &Context, EVT VT) const override;
+    EVT getSetCCResultType(const DataLayout &DL, LLVMContext &Context,
+                           EVT VT) const override;
 
     SDValue PerformDAGCombine(SDNode *N, DAGCombinerInfo &DCI) const override;
 
@@ -263,7 +266,8 @@ namespace llvm {
 
     void HandleByVal(CCState *, unsigned &, unsigned) const override;
 
-    unsigned getRegisterByName(const char* RegName, EVT VT) const override;
+    unsigned getRegisterByName(const char* RegName, EVT VT,
+                               SelectionDAG &DAG) const override;
 
   protected:
     SDValue getGlobalReg(SelectionDAG &DAG, EVT Ty) const;
@@ -365,10 +369,6 @@ namespace llvm {
     // Cache the ABI from the TargetMachine, we use it everywhere.
     const MipsABIInfo &ABI;
 
-    // @LOCALMOD-BEGIN
-    SDValue GetNaClThreadPointer(SelectionDAG &DAG, SDLoc DL) const;
-    // @LOCALMOD-END
-
   private:
     // Create a TargetGlobalAddress node.
     SDValue getTargetNode(GlobalAddressSDNode *N, EVT Ty, SelectionDAG &DAG,
@@ -482,8 +482,7 @@ namespace llvm {
     bool shouldSignExtendTypeInLibCall(EVT Type, bool IsSigned) const override;
 
     // Inline asm support
-    ConstraintType
-      getConstraintType(const std::string &Constraint) const override;
+    ConstraintType getConstraintType(StringRef Constraint) const override;
 
     /// Examine constraint string and operand type and determine a weight value.
     /// The operand object must already have been set up with the operand type.
@@ -497,8 +496,7 @@ namespace llvm {
 
     std::pair<unsigned, const TargetRegisterClass *>
     getRegForInlineAsmConstraint(const TargetRegisterInfo *TRI,
-                                 const std::string &Constraint,
-                                 MVT VT) const override;
+                                 StringRef Constraint, MVT VT) const override;
 
     /// LowerAsmOperandForConstraint - Lower the specified operand into the Ops
     /// vector.  If it is invalid, don't add anything to Ops. If hasMemory is
@@ -509,8 +507,8 @@ namespace llvm {
                                       std::vector<SDValue> &Ops,
                                       SelectionDAG &DAG) const override;
 
-    unsigned getInlineAsmMemConstraint(
-        const std::string &ConstraintCode) const override {
+    unsigned
+    getInlineAsmMemConstraint(StringRef ConstraintCode) const override {
       if (ConstraintCode == "R")
         return InlineAsm::Constraint_R;
       else if (ConstraintCode == "ZC")
@@ -518,7 +516,8 @@ namespace llvm {
       return TargetLowering::getInlineAsmMemConstraint(ConstraintCode);
     }
 
-    bool isLegalAddressingMode(const AddrMode &AM, Type *Ty) const override;
+    bool isLegalAddressingMode(const DataLayout &DL, const AddrMode &AM,
+                               Type *Ty, unsigned AS) const override;
 
     bool isOffsetFoldingLegal(const GlobalAddressSDNode *GA) const override;
 
@@ -534,6 +533,7 @@ namespace llvm {
     bool isFPImmLegal(const APFloat &Imm, EVT VT) const override;
 
     unsigned getJumpTableEncoding() const override;
+    bool useSoftFloat() const override;
 
     /// Emit a sign-extension using sll/sra, seb, or seh appropriately.
     MachineBasicBlock *emitSignExtendToI32InReg(MachineInstr *MI,
diff --git a/lib/Target/Mips/MipsInstrFPU.td b/lib/Target/Mips/MipsInstrFPU.td
index ed97cb46192..cb912253b28 100644
--- a/lib/Target/Mips/MipsInstrFPU.td
+++ b/lib/Target/Mips/MipsInstrFPU.td
@@ -65,6 +65,8 @@ def IsSingleFloat    : Predicate<"Subtarget->isSingleFloat()">,
                        AssemblerPredicate<"FeatureSingleFloat">;
 def IsNotSingleFloat : Predicate<"!Subtarget->isSingleFloat()">,
                        AssemblerPredicate<"!FeatureSingleFloat">;
+def IsNotSoftFloat   : Predicate<"!Subtarget->useSoftFloat()">,
+                       AssemblerPredicate<"!FeatureSoftFloat">;
 
 //===----------------------------------------------------------------------===//
 // Mips FGR size adjectives.
@@ -73,6 +75,7 @@ def IsNotSingleFloat : Predicate<"!Subtarget->isSingleFloat()">,
 
 class FGR_32 { list<Predicate> FGRPredicates = [NotFP64bit]; }
 class FGR_64 { list<Predicate> FGRPredicates = [IsFP64bit]; }
+class HARDFLOAT { list<Predicate> HardFloatPredicate = [IsNotSoftFloat]; }
 
 //===----------------------------------------------------------------------===//
 
@@ -98,22 +101,19 @@ def fpimm0neg : PatLeaf<(fpimm), [{
 //
 // Only S32 and D32 are supported right now.
 //===----------------------------------------------------------------------===//
-
 class ADDS_FT<string opstr, RegisterOperand RC, InstrItinClass Itin, bit IsComm,
               SDPatternOperator OpNode= null_frag> :
   InstSE<(outs RC:$fd), (ins RC:$fs, RC:$ft),
          !strconcat(opstr, "\t$fd, $fs, $ft"),
-         [(set RC:$fd, (OpNode RC:$fs, RC:$ft))], Itin, FrmFR, opstr> {
+         [(set RC:$fd, (OpNode RC:$fs, RC:$ft))], Itin, FrmFR, opstr>,
+  HARDFLOAT {
   let isCommutable = IsComm;
 }
 
 multiclass ADDS_M<string opstr, InstrItinClass Itin, bit IsComm,
                   SDPatternOperator OpNode = null_frag> {
-  def _D32 : MMRel, ADDS_FT<opstr, AFGR64Opnd, Itin, IsComm, OpNode>,
-             AdditionalRequires<[NotFP64bit]>;
-  def _D64 : ADDS_FT<opstr, FGR64Opnd, Itin,
-                     IsComm, OpNode>,
-             AdditionalRequires<[IsFP64bit]> {
+  def _D32 : MMRel, ADDS_FT<opstr, AFGR64Opnd, Itin, IsComm, OpNode>, FGR_32;
+  def _D64 : ADDS_FT<opstr, FGR64Opnd, Itin, IsComm, OpNode>, FGR_64 {
     string DecoderNamespace = "Mips64";
   }
 }
@@ -122,23 +122,21 @@ class ABSS_FT<string opstr, RegisterOperand DstRC, RegisterOperand SrcRC,
               InstrItinClass Itin, SDPatternOperator OpNode= null_frag> :
   InstSE<(outs DstRC:$fd), (ins SrcRC:$fs), !strconcat(opstr, "\t$fd, $fs"),
          [(set DstRC:$fd, (OpNode SrcRC:$fs))], Itin, FrmFR, opstr>,
+  HARDFLOAT,
   NeverHasSideEffects;
 
 multiclass ABSS_M<string opstr, InstrItinClass Itin,
                   SDPatternOperator OpNode= null_frag> {
   def _D32 : MMRel, ABSS_FT<opstr, AFGR64Opnd, AFGR64Opnd, Itin, OpNode>,
-             AdditionalRequires<[NotFP64bit]>;
-  def _D64 : ABSS_FT<opstr, FGR64Opnd, FGR64Opnd, Itin, OpNode>,
-             AdditionalRequires<[IsFP64bit]> {
+             FGR_32;
+  def _D64 : ABSS_FT<opstr, FGR64Opnd, FGR64Opnd, Itin, OpNode>, FGR_64 {
     string DecoderNamespace = "Mips64";
   }
 }
 
 multiclass ROUND_M<string opstr, InstrItinClass Itin> {
-  def _D32 : MMRel, ABSS_FT<opstr, FGR32Opnd, AFGR64Opnd, Itin>,
-             AdditionalRequires<[NotFP64bit]>;
-  def _D64 : ABSS_FT<opstr, FGR32Opnd, FGR64Opnd, Itin>,
-             AdditionalRequires<[IsFP64bit]> {
+  def _D32 : MMRel, ABSS_FT<opstr, FGR32Opnd, AFGR64Opnd, Itin>, FGR_32;
+  def _D64 : ABSS_FT<opstr, FGR32Opnd, FGR64Opnd, Itin>, FGR_64 {
     let DecoderNamespace = "Mips64";
   }
 }
@@ -146,17 +144,17 @@ multiclass ROUND_M<string opstr, InstrItinClass Itin> {
 class MFC1_FT<string opstr, RegisterOperand DstRC, RegisterOperand SrcRC,
               InstrItinClass Itin, SDPatternOperator OpNode= null_frag> :
   InstSE<(outs DstRC:$rt), (ins SrcRC:$fs), !strconcat(opstr, "\t$rt, $fs"),
-         [(set DstRC:$rt, (OpNode SrcRC:$fs))], Itin, FrmFR, opstr>;
+         [(set DstRC:$rt, (OpNode SrcRC:$fs))], Itin, FrmFR, opstr>, HARDFLOAT;
 
 class MTC1_FT<string opstr, RegisterOperand DstRC, RegisterOperand SrcRC,
               InstrItinClass Itin, SDPatternOperator OpNode= null_frag> :
   InstSE<(outs DstRC:$fs), (ins SrcRC:$rt), !strconcat(opstr, "\t$rt, $fs"),
-         [(set DstRC:$fs, (OpNode SrcRC:$rt))], Itin, FrmFR, opstr>;
+         [(set DstRC:$fs, (OpNode SrcRC:$rt))], Itin, FrmFR, opstr>, HARDFLOAT;
 
 class MTC1_64_FT<string opstr, RegisterOperand DstRC, RegisterOperand SrcRC,
                  InstrItinClass Itin> :
   InstSE<(outs DstRC:$fs), (ins DstRC:$fs_in, SrcRC:$rt),
-         !strconcat(opstr, "\t$rt, $fs"), [], Itin, FrmFR, opstr> {
+         !strconcat(opstr, "\t$rt, $fs"), [], Itin, FrmFR, opstr>, HARDFLOAT {
   // $fs_in is part of a white lie to work around a widespread bug in the FPU
   // implementation. See expandBuildPairF64 for details.
   let Constraints = "$fs = $fs_in";
@@ -165,7 +163,8 @@ class MTC1_64_FT<string opstr, RegisterOperand DstRC, RegisterOperand SrcRC,
 class LW_FT<string opstr, RegisterOperand RC, InstrItinClass Itin,
             SDPatternOperator OpNode= null_frag> :
   InstSE<(outs RC:$rt), (ins mem:$addr), !strconcat(opstr, "\t$rt, $addr"),
-         [(set RC:$rt, (OpNode addrDefault:$addr))], Itin, FrmFI, opstr> {
+         [(set RC:$rt, (OpNode addrDefault:$addr))], Itin, FrmFI, opstr>,
+  HARDFLOAT {
   let DecoderMethod = "DecodeFMem";
   let mayLoad = 1;
 }
@@ -173,7 +172,7 @@ class LW_FT<string opstr, RegisterOperand RC, InstrItinClass Itin,
 class SW_FT<string opstr, RegisterOperand RC, InstrItinClass Itin,
             SDPatternOperator OpNode= null_frag> :
   InstSE<(outs), (ins RC:$rt, mem:$addr), !strconcat(opstr, "\t$rt, $addr"),
-         [(OpNode RC:$rt, addrDefault:$addr)], Itin, FrmFI, opstr> {
+         [(OpNode RC:$rt, addrDefault:$addr)], Itin, FrmFI, opstr>, HARDFLOAT {
   let DecoderMethod = "DecodeFMem";
   let mayStore = 1;
 }
@@ -183,21 +182,21 @@ class MADDS_FT<string opstr, RegisterOperand RC, InstrItinClass Itin,
   InstSE<(outs RC:$fd), (ins RC:$fr, RC:$fs, RC:$ft),
          !strconcat(opstr, "\t$fd, $fr, $fs, $ft"),
          [(set RC:$fd, (OpNode (fmul RC:$fs, RC:$ft), RC:$fr))], Itin,
-         FrmFR, opstr>;
+         FrmFR, opstr>, HARDFLOAT;
 
 class NMADDS_FT<string opstr, RegisterOperand RC, InstrItinClass Itin,
                 SDPatternOperator OpNode = null_frag> :
   InstSE<(outs RC:$fd), (ins RC:$fr, RC:$fs, RC:$ft),
          !strconcat(opstr, "\t$fd, $fr, $fs, $ft"),
          [(set RC:$fd, (fsub fpimm0, (OpNode (fmul RC:$fs, RC:$ft), RC:$fr)))],
-         Itin, FrmFR, opstr>;
+         Itin, FrmFR, opstr>, HARDFLOAT;
 
 class LWXC1_FT<string opstr, RegisterOperand DRC,
                InstrItinClass Itin, SDPatternOperator OpNode = null_frag> :
   InstSE<(outs DRC:$fd), (ins PtrRC:$base, PtrRC:$index),
          !strconcat(opstr, "\t$fd, ${index}(${base})"),
          [(set DRC:$fd, (OpNode (add iPTR:$base, iPTR:$index)))], Itin,
-         FrmFI, opstr> {
+         FrmFI, opstr>, HARDFLOAT {
   let AddedComplexity = 20;
 }
 
@@ -206,7 +205,7 @@ class SWXC1_FT<string opstr, RegisterOperand DRC,
   InstSE<(outs), (ins DRC:$fs, PtrRC:$base, PtrRC:$index),
          !strconcat(opstr, "\t$fs, ${index}(${base})"),
          [(OpNode DRC:$fs, (add iPTR:$base, iPTR:$index))], Itin,
-         FrmFI, opstr> {
+         FrmFI, opstr>, HARDFLOAT {
   let AddedComplexity = 20;
 }
 
@@ -215,7 +214,7 @@ class BC1F_FT<string opstr, DAGOperand opnd, InstrItinClass Itin,
   InstSE<(outs), (ins FCCRegsOpnd:$fcc, opnd:$offset),
          !strconcat(opstr, "\t$fcc, $offset"),
          [(MipsFPBrcond Op, FCCRegsOpnd:$fcc, bb:$offset)], Itin,
-         FrmFI, opstr> {
+         FrmFI, opstr>, HARDFLOAT {
   let isBranch = 1;
   let isTerminator = 1;
   let hasDelaySlot = DelaySlot;
@@ -227,7 +226,7 @@ class CEQS_FT<string typestr, RegisterClass RC, InstrItinClass Itin,
   InstSE<(outs), (ins RC:$fs, RC:$ft, condcode:$cond),
          !strconcat("c.$cond.", typestr, "\t$fs, $ft"),
          [(OpNode RC:$fs, RC:$ft, imm:$cond)], Itin, FrmFR,
-         !strconcat("c.$cond.", typestr)> {
+         !strconcat("c.$cond.", typestr)>, HARDFLOAT {
   let Defs = [FCC0];
   let isCodeGenOnly = 1;
 }
@@ -236,7 +235,7 @@ class C_COND_FT<string CondStr, string Typestr, RegisterOperand RC,
                 InstrItinClass itin>  :
    InstSE<(outs), (ins RC:$fs, RC:$ft),
           !strconcat("c.", CondStr, ".", Typestr, "\t$fs, $ft"), [], itin,
-          FrmFR>;
+          FrmFR>, HARDFLOAT;
 
 multiclass C_COND_M<string TypeStr, RegisterOperand RC, bits<5> fmt,
                     InstrItinClass itin> {
@@ -260,10 +259,10 @@ multiclass C_COND_M<string TypeStr, RegisterOperand RC, bits<5> fmt,
 
 defm S : C_COND_M<"s", FGR32Opnd, 16, II_C_CC_S>, ISA_MIPS1_NOT_32R6_64R6;
 defm D32 : C_COND_M<"d", AFGR64Opnd, 17, II_C_CC_D>, ISA_MIPS1_NOT_32R6_64R6,
-           AdditionalRequires<[NotFP64bit]>;
+           FGR_32;
 let DecoderNamespace = "Mips64" in
 defm D64 : C_COND_M<"d", FGR64Opnd, 17, II_C_CC_D>, ISA_MIPS1_NOT_32R6_64R6,
-           AdditionalRequires<[IsFP64bit]>;
+           FGR_64;
 
 //===----------------------------------------------------------------------===//
 // Floating Point Instructions
@@ -363,15 +362,15 @@ def MFC1 : MMRel, MFC1_FT<"mfc1", GPR32Opnd, FGR32Opnd, II_MFC1,
 def MTC1 : MMRel, MTC1_FT<"mtc1", FGR32Opnd, GPR32Opnd, II_MTC1,
                           bitconvert>, MFC1_FM<4>;
 def MFHC1_D32 : MMRel, MFC1_FT<"mfhc1", GPR32Opnd, AFGR64Opnd, II_MFHC1>,
-                MFC1_FM<3>, ISA_MIPS32R2, AdditionalRequires<[NotFP64bit]>;
+                MFC1_FM<3>, ISA_MIPS32R2, FGR_32;
 def MFHC1_D64 : MFC1_FT<"mfhc1", GPR32Opnd, FGR64Opnd, II_MFHC1>,
-                MFC1_FM<3>, ISA_MIPS32R2, AdditionalRequires<[IsFP64bit]> {
+                MFC1_FM<3>, ISA_MIPS32R2, FGR_64 {
   let DecoderNamespace = "Mips64";
 }
 def MTHC1_D32 : MMRel, MTC1_64_FT<"mthc1", AFGR64Opnd, GPR32Opnd, II_MTHC1>,
-                MFC1_FM<7>, ISA_MIPS32R2, AdditionalRequires<[NotFP64bit]>;
+                MFC1_FM<7>, ISA_MIPS32R2, FGR_32;
 def MTHC1_D64 : MTC1_64_FT<"mthc1", FGR64Opnd, GPR32Opnd, II_MTHC1>,
-                MFC1_FM<7>, ISA_MIPS32R2, AdditionalRequires<[IsFP64bit]> {
+                MFC1_FM<7>, ISA_MIPS32R2, FGR_64 {
   let DecoderNamespace = "Mips64";
 }
 def DMFC1 : MFC1_FT<"dmfc1", GPR64Opnd, FGR64Opnd, II_DMFC1,
@@ -382,9 +381,9 @@ def DMTC1 : MTC1_FT<"dmtc1", FGR64Opnd, GPR64Opnd, II_DMTC1,
 def FMOV_S   : MMRel, ABSS_FT<"mov.s", FGR32Opnd, FGR32Opnd, II_MOV_S>,
                ABSS_FM<0x6, 16>;
 def FMOV_D32 : MMRel, ABSS_FT<"mov.d", AFGR64Opnd, AFGR64Opnd, II_MOV_D>,
-               ABSS_FM<0x6, 17>, AdditionalRequires<[NotFP64bit]>;
+               ABSS_FM<0x6, 17>, FGR_32;
 def FMOV_D64 : ABSS_FT<"mov.d", FGR64Opnd, FGR64Opnd, II_MOV_D>,
-               ABSS_FM<0x6, 17>, AdditionalRequires<[IsFP64bit]> {
+               ABSS_FM<0x6, 17>, FGR_64 {
                  let DecoderNamespace = "Mips64";
 }
 
@@ -513,36 +512,14 @@ def BC1T : MMRel, BC1F_FT<"bc1t", brtarget, IIBranch, MIPS_BRANCH_T>,
 def BC1TL : MMRel, BC1F_FT<"bc1tl", brtarget, IIBranch, MIPS_BRANCH_T, 0>,
             BC1F_FM<1, 1>, ISA_MIPS2_NOT_32R6_64R6;
 
-//===----------------------------------------------------------------------===//
-// Floating Point Flag Conditions
-//===----------------------------------------------------------------------===//
-// Mips condition codes. They must correspond to condcode in MipsInstrInfo.h.
-// They must be kept in synch.
-def MIPS_FCOND_F    : PatLeaf<(i32 0)>;
-def MIPS_FCOND_UN   : PatLeaf<(i32 1)>;
-def MIPS_FCOND_OEQ  : PatLeaf<(i32 2)>;
-def MIPS_FCOND_UEQ  : PatLeaf<(i32 3)>;
-def MIPS_FCOND_OLT  : PatLeaf<(i32 4)>;
-def MIPS_FCOND_ULT  : PatLeaf<(i32 5)>;
-def MIPS_FCOND_OLE  : PatLeaf<(i32 6)>;
-def MIPS_FCOND_ULE  : PatLeaf<(i32 7)>;
-def MIPS_FCOND_SF   : PatLeaf<(i32 8)>;
-def MIPS_FCOND_NGLE : PatLeaf<(i32 9)>;
-def MIPS_FCOND_SEQ  : PatLeaf<(i32 10)>;
-def MIPS_FCOND_NGL  : PatLeaf<(i32 11)>;
-def MIPS_FCOND_LT   : PatLeaf<(i32 12)>;
-def MIPS_FCOND_NGE  : PatLeaf<(i32 13)>;
-def MIPS_FCOND_LE   : PatLeaf<(i32 14)>;
-def MIPS_FCOND_NGT  : PatLeaf<(i32 15)>;
-
 /// Floating Point Compare
 def FCMP_S32 : MMRel, CEQS_FT<"s", FGR32, II_C_CC_S, MipsFPCmp>, CEQS_FM<16>,
                ISA_MIPS1_NOT_32R6_64R6;
 def FCMP_D32 : MMRel, CEQS_FT<"d", AFGR64, II_C_CC_D, MipsFPCmp>, CEQS_FM<17>,
-               ISA_MIPS1_NOT_32R6_64R6, AdditionalRequires<[NotFP64bit]>;
+               ISA_MIPS1_NOT_32R6_64R6, FGR_32;
 let DecoderNamespace = "Mips64" in
 def FCMP_D64 : CEQS_FT<"d", FGR64, II_C_CC_D, MipsFPCmp>, CEQS_FM<17>,
-               ISA_MIPS1_NOT_32R6_64R6, AdditionalRequires<[IsFP64bit]>;
+               ISA_MIPS1_NOT_32R6_64R6, FGR_64;
 
 //===----------------------------------------------------------------------===//
 // Floating Point Pseudo-Instructions
@@ -554,10 +531,8 @@ class BuildPairF64Base<RegisterOperand RO> :
   PseudoSE<(outs RO:$dst), (ins GPR32Opnd:$lo, GPR32Opnd:$hi),
            [(set RO:$dst, (MipsBuildPairF64 GPR32Opnd:$lo, GPR32Opnd:$hi))]>;
 
-def BuildPairF64 : BuildPairF64Base<AFGR64Opnd>,
-                   AdditionalRequires<[NotFP64bit]>;
-def BuildPairF64_64 : BuildPairF64Base<FGR64Opnd>,
-                      AdditionalRequires<[IsFP64bit]>;
+def BuildPairF64 : BuildPairF64Base<AFGR64Opnd>, FGR_32, HARDFLOAT;
+def BuildPairF64_64 : BuildPairF64Base<FGR64Opnd>, FGR_64, HARDFLOAT;
 
 // This pseudo instr gets expanded into 2 mfc1 instrs after register
 // allocation.
@@ -567,22 +542,20 @@ class ExtractElementF64Base<RegisterOperand RO> :
   PseudoSE<(outs GPR32Opnd:$dst), (ins RO:$src, i32imm:$n),
            [(set GPR32Opnd:$dst, (MipsExtractElementF64 RO:$src, imm:$n))]>;
 
-def ExtractElementF64 : ExtractElementF64Base<AFGR64Opnd>,
-                        AdditionalRequires<[NotFP64bit]>;
-def ExtractElementF64_64 : ExtractElementF64Base<FGR64Opnd>,
-                           AdditionalRequires<[IsFP64bit]>;
+def ExtractElementF64 : ExtractElementF64Base<AFGR64Opnd>, FGR_32, HARDFLOAT;
+def ExtractElementF64_64 : ExtractElementF64Base<FGR64Opnd>, FGR_64, HARDFLOAT;
 
 //===----------------------------------------------------------------------===//
 // InstAliases.
 //===----------------------------------------------------------------------===//
 def : MipsInstAlias<"bc1t $offset", (BC1T FCC0, brtarget:$offset)>,
-      ISA_MIPS1_NOT_32R6_64R6;
+      ISA_MIPS1_NOT_32R6_64R6, HARDFLOAT;
 def : MipsInstAlias<"bc1tl $offset", (BC1TL FCC0, brtarget:$offset)>,
-      ISA_MIPS2_NOT_32R6_64R6;
+      ISA_MIPS2_NOT_32R6_64R6, HARDFLOAT;
 def : MipsInstAlias<"bc1f $offset", (BC1F FCC0, brtarget:$offset)>,
-      ISA_MIPS1_NOT_32R6_64R6;
+      ISA_MIPS1_NOT_32R6_64R6, HARDFLOAT;
 def : MipsInstAlias<"bc1fl $offset", (BC1FL FCC0, brtarget:$offset)>,
-      ISA_MIPS2_NOT_32R6_64R6;
+      ISA_MIPS2_NOT_32R6_64R6, HARDFLOAT;
 
 //===----------------------------------------------------------------------===//
 // Floating Point Patterns
diff --git a/lib/Target/Mips/MipsInstrFormats.td b/lib/Target/Mips/MipsInstrFormats.td
index 8cc16037351..73886689b3e 100644
--- a/lib/Target/Mips/MipsInstrFormats.td
+++ b/lib/Target/Mips/MipsInstrFormats.td
@@ -50,6 +50,20 @@ def Std2MicroMips : InstrMapping {
   let ValueCols = [["se"], ["micromips"]];
 }
 
+class StdMMR6Rel;
+
+def Std2MicroMipsR6 : InstrMapping {
+  let FilterClass = "StdMMR6Rel";
+  // Instructions with the same BaseOpcode and isNVStore values form a row.
+  let RowFields = ["BaseOpcode"];
+  // Instructions with the same predicate sense form a column.
+  let ColFields = ["Arch"];
+  // The key column is the unpredicated instructions.
+  let KeyCol = ["se"];
+  // Value columns are PredSense=true and PredSense=false
+  let ValueCols = [["se"], ["micromipsr6"]];
+}
+
 class StdArch {
   string Arch = "se";
 }
@@ -212,6 +226,18 @@ class MFC3OP_FM<bits<6> op, bits<5> mfmt>
   let Inst{2-0}   = sel;
 }
 
+class MFC2OP_FM<bits<6> op, bits<5> mfmt> : StdArch {
+  bits<5>  rt;
+  bits<16> imm16;
+
+  bits<32> Inst;
+
+  let Inst{31-26} = op;
+  let Inst{25-21} = mfmt;
+  let Inst{20-16} = rt;
+  let Inst{15-0}  = imm16;
+}
+
 class ADD_FM<bits<6> op, bits<6> funct> : StdArch {
   bits<5> rd;
   bits<5> rs;
@@ -618,16 +644,16 @@ class BRK_FM<bits<6> funct> : StdArch
 //  Exception return format <Cop0|1|0|funct>
 //===----------------------------------------------------------------------===//
 
-class ER_FM<bits<6> funct> : StdArch
+class ER_FM<bits<6> funct, bit LLBit> : StdArch
 {
   bits<32> Inst;
   let Inst{31-26} = 0x10;
   let Inst{25}    = 1;
-  let Inst{24-6}  = 0;
+  let Inst{24-7}  = 0;
+  let Inst{6} = LLBit;
   let Inst{5-0}   = funct;
 }
 
-
 //===----------------------------------------------------------------------===//
 //  Enable/disable interrupt instruction format <Cop0|MFMC0|rt|12|0|sc|0|0>
 //===----------------------------------------------------------------------===//
diff --git a/lib/Target/Mips/MipsInstrInfo.cpp b/lib/Target/Mips/MipsInstrInfo.cpp
index 0839147984b..bb23cc04e69 100644
--- a/lib/Target/Mips/MipsInstrInfo.cpp
+++ b/lib/Target/Mips/MipsInstrInfo.cpp
@@ -96,8 +96,7 @@ bool MipsInstrInfo::AnalyzeBranch(MachineBasicBlock &MBB,
 
 void
 MipsInstrInfo::BuildCondBr(MachineBasicBlock &MBB, MachineBasicBlock *TBB,
-                           DebugLoc DL,
-                           const SmallVectorImpl<MachineOperand> &Cond) const {
+                           DebugLoc DL, ArrayRef<MachineOperand> Cond) const {
   unsigned Opc = Cond[0].getImm();
   const MCInstrDesc &MCID = get(Opc);
   MachineInstrBuilder MIB = BuildMI(&MBB, DL, MCID);
@@ -115,7 +114,7 @@ MipsInstrInfo::BuildCondBr(MachineBasicBlock &MBB, MachineBasicBlock *TBB,
 
 unsigned MipsInstrInfo::InsertBranch(
     MachineBasicBlock &MBB, MachineBasicBlock *TBB, MachineBasicBlock *FBB,
-    const SmallVectorImpl<MachineOperand> &Cond, DebugLoc DL) const {
+    ArrayRef<MachineOperand> Cond, DebugLoc DL) const {
   // Shouldn't be a fall through.
   assert(TBB && "InsertBranch must not be told to insert a fallthrough");
 
diff --git a/lib/Target/Mips/MipsInstrInfo.h b/lib/Target/Mips/MipsInstrInfo.h
index 45895355e1a..08efc350904 100644
--- a/lib/Target/Mips/MipsInstrInfo.h
+++ b/lib/Target/Mips/MipsInstrInfo.h
@@ -59,8 +59,7 @@ class MipsInstrInfo : public MipsGenInstrInfo {
   unsigned RemoveBranch(MachineBasicBlock &MBB) const override;
 
   unsigned InsertBranch(MachineBasicBlock &MBB, MachineBasicBlock *TBB,
-                        MachineBasicBlock *FBB,
-                        const SmallVectorImpl<MachineOperand> &Cond,
+                        MachineBasicBlock *FBB, ArrayRef<MachineOperand> Cond,
                         DebugLoc DL) const override;
 
   bool
@@ -140,7 +139,7 @@ class MipsInstrInfo : public MipsGenInstrInfo {
                      SmallVectorImpl<MachineOperand> &Cond) const;
 
   void BuildCondBr(MachineBasicBlock &MBB, MachineBasicBlock *TBB, DebugLoc DL,
-                   const SmallVectorImpl<MachineOperand>& Cond) const;
+                   ArrayRef<MachineOperand> Cond) const;
 };
 
 /// Create MipsInstrInfo objects.
diff --git a/lib/Target/Mips/MipsInstrInfo.td b/lib/Target/Mips/MipsInstrInfo.td
index a1fad663b1c..0862285941f 100644
--- a/lib/Target/Mips/MipsInstrInfo.td
+++ b/lib/Target/Mips/MipsInstrInfo.td
@@ -166,6 +166,8 @@ def HasMips32    :    Predicate<"Subtarget->hasMips32()">,
                       AssemblerPredicate<"FeatureMips32">;
 def HasMips32r2  :    Predicate<"Subtarget->hasMips32r2()">,
                       AssemblerPredicate<"FeatureMips32r2">;
+def HasMips32r5  :    Predicate<"Subtarget->hasMips32r5()">,
+                      AssemblerPredicate<"FeatureMips32r5">;
 def HasMips32r6  :    Predicate<"Subtarget->hasMips32r6()">,
                       AssemblerPredicate<"FeatureMips32r6">;
 def NotMips32r6  :    Predicate<"!Subtarget->hasMips32r6()">,
@@ -182,6 +184,8 @@ def HasMips64r6  :    Predicate<"Subtarget->hasMips64r6()">,
                       AssemblerPredicate<"FeatureMips64r6">;
 def NotMips64r6  :    Predicate<"!Subtarget->hasMips64r6()">,
                       AssemblerPredicate<"!FeatureMips64r6">;
+def HasMicroMips32r6 : Predicate<"Subtarget->inMicroMips32r6Mode()">,
+                       AssemblerPredicate<"FeatureMicroMips,FeatureMips32r6">;
 def InMips16Mode :    Predicate<"Subtarget->inMips16Mode()">,
                       AssemblerPredicate<"FeatureMips16">;
 def HasCnMips    :    Predicate<"Subtarget->hasCnMips()">,
@@ -240,6 +244,7 @@ class ISA_MIPS32R2 { list<Predicate> InsnPredicates = [HasMips32r2]; }
 class ISA_MIPS32R2_NOT_32R6_64R6 {
   list<Predicate> InsnPredicates = [HasMips32r2, NotMips32r6, NotMips64r6];
 }
+class ISA_MIPS32R5 { list<Predicate> InsnPredicates = [HasMips32r5]; }
 class ISA_MIPS64   { list<Predicate> InsnPredicates = [HasMips64]; }
 class ISA_MIPS64_NOT_64R6 {
   list<Predicate> InsnPredicates = [HasMips64, NotMips64r6];
@@ -247,6 +252,9 @@ class ISA_MIPS64_NOT_64R6 {
 class ISA_MIPS64R2 { list<Predicate> InsnPredicates = [HasMips64r2]; }
 class ISA_MIPS32R6 { list<Predicate> InsnPredicates = [HasMips32r6]; }
 class ISA_MIPS64R6 { list<Predicate> InsnPredicates = [HasMips64r6]; }
+class ISA_MICROMIPS32R6 {
+  list<Predicate> InsnPredicates = [HasMicroMips32r6];
+}
 
 // The portions of MIPS-III that were also added to MIPS32
 class INSN_MIPS3_32 { list<Predicate> InsnPredicates = [HasMips3_32]; }
@@ -353,6 +361,8 @@ def calltarget  : Operand<iPTR> {
   let ParserMatchClass = MipsJumpTargetAsmOperand;
 }
 
+def imm64: Operand<i64>;
+
 def simm9 : Operand<i32>;
 def simm10 : Operand<i32>;
 def simm11 : Operand<i32>;
@@ -379,7 +389,15 @@ def simm20      : Operand<i32> {
 def uimm20      : Operand<i32> {
 }
 
+def MipsUImm10AsmOperand : AsmOperandClass {
+  let Name = "UImm10";
+  let RenderMethod = "addImmOperands";
+  let ParserMethod = "parseImm";
+  let PredicateMethod = "isUImm<10>";
+}
+
 def uimm10      : Operand<i32> {
+  let ParserMatchClass = MipsUImm10AsmOperand;
 }
 
 def simm16_64   : Operand<i64> {
@@ -1035,8 +1053,12 @@ class SCBase<string opstr, RegisterOperand RO> :
   let Constraints = "$rt = $dst";
 }
 
-class MFC3OP<string asmstr, RegisterOperand RO> :
-  InstSE<(outs RO:$rt, RO:$rd, uimm16:$sel), (ins),
+class MFC3OP<string asmstr, RegisterOperand RO, RegisterOperand RD> :
+  InstSE<(outs RO:$rt), (ins RD:$rd, uimm16:$sel),
+         !strconcat(asmstr, "\t$rt, $rd, $sel"), [], NoItinerary, FrmFR>;
+
+class MTC3OP<string asmstr, RegisterOperand RO, RegisterOperand RD> :
+  InstSE<(outs RO:$rd), (ins RD:$rt, uimm16:$sel),
          !strconcat(asmstr, "\t$rt, $rd, $sel"), [], NoItinerary, FrmFR>;
 
 class TrapBase<Instruction RealInst>
@@ -1118,8 +1140,9 @@ def LONG_BRANCH_ADDiu : PseudoSE<(outs GPR32Opnd:$dst),
 
 /// Arithmetic Instructions (ALU Immediate)
 let AdditionalPredicates = [NotInMicroMips] in {
-def ADDiu : MMRel, ArithLogicI<"addiu", simm16, GPR32Opnd, II_ADDIU, immSExt16,
-                               add>, ADDI_FM<0x9>, IsAsCheapAsAMove;
+def ADDiu : MMRel, StdMMR6Rel, ArithLogicI<"addiu", simm16, GPR32Opnd,
+                                           II_ADDIU, immSExt16, add>,
+            ADDI_FM<0x9>, IsAsCheapAsAMove;
 }
 def ADDi  : MMRel, ArithLogicI<"addi", simm16, GPR32Opnd>, ADDI_FM<0x8>,
             ISA_MIPS1_NOT_32R6_64R6;
@@ -1128,19 +1151,20 @@ def SLTi  : MMRel, SetCC_I<"slti", setlt, simm16, immSExt16, GPR32Opnd>,
 def SLTiu : MMRel, SetCC_I<"sltiu", setult, simm16, immSExt16, GPR32Opnd>,
             SLTI_FM<0xb>;
 let AdditionalPredicates = [NotInMicroMips] in {
-def ANDi  : MMRel, ArithLogicI<"andi", uimm16, GPR32Opnd, II_ANDI, immZExt16,
-                               and>, ADDI_FM<0xc>;
+def ANDi  : MMRel, StdMMR6Rel,
+            ArithLogicI<"andi", uimm16, GPR32Opnd, II_ANDI, immZExt16, and>,
+            ADDI_FM<0xc>;
 }
-def ORi   : MMRel, ArithLogicI<"ori", uimm16, GPR32Opnd, II_ORI, immZExt16,
-                               or>,
+def ORi   : MMRel, StdMMR6Rel,
+            ArithLogicI<"ori", uimm16, GPR32Opnd, II_ORI, immZExt16, or>,
             ADDI_FM<0xd>;
-def XORi  : MMRel, ArithLogicI<"xori", uimm16, GPR32Opnd, II_XORI, immZExt16,
-                               xor>,
+def XORi  : MMRel, StdMMR6Rel,
+            ArithLogicI<"xori", uimm16, GPR32Opnd, II_XORI, immZExt16, xor>,
             ADDI_FM<0xe>;
 def LUi   : MMRel, LoadUpper<"lui", GPR32Opnd, uimm16>, LUI_FM;
 let AdditionalPredicates = [NotInMicroMips] in {
 /// Arithmetic Instructions (3-Operand, R-Type)
-def ADDu  : MMRel, ArithLogicR<"addu", GPR32Opnd, 1, II_ADDU, add>,
+def ADDu  : MMRel, StdMMR6Rel, ArithLogicR<"addu", GPR32Opnd, 1, II_ADDU, add>,
             ADD_FM<0, 0x21>;
 def SUBu  : MMRel, ArithLogicR<"subu", GPR32Opnd, 0, II_SUBU, sub>,
             ADD_FM<0, 0x23>;
@@ -1148,19 +1172,19 @@ def SUBu  : MMRel, ArithLogicR<"subu", GPR32Opnd, 0, II_SUBU, sub>,
 let Defs = [HI0, LO0] in
 def MUL   : MMRel, ArithLogicR<"mul", GPR32Opnd, 1, II_MUL, mul>,
             ADD_FM<0x1c, 2>, ISA_MIPS32_NOT_32R6_64R6;
-def ADD   : MMRel, ArithLogicR<"add", GPR32Opnd>, ADD_FM<0, 0x20>;
+def ADD   : MMRel, StdMMR6Rel, ArithLogicR<"add", GPR32Opnd>, ADD_FM<0, 0x20>;
 def SUB   : MMRel, ArithLogicR<"sub", GPR32Opnd>, ADD_FM<0, 0x22>;
 def SLT   : MMRel, SetCC_R<"slt", setlt, GPR32Opnd>, ADD_FM<0, 0x2a>;
 def SLTu  : MMRel, SetCC_R<"sltu", setult, GPR32Opnd>, ADD_FM<0, 0x2b>;
 let AdditionalPredicates = [NotInMicroMips] in {
-def AND   : MMRel, ArithLogicR<"and", GPR32Opnd, 1, II_AND, and>,
+def AND   : MMRel, StdMMR6Rel, ArithLogicR<"and", GPR32Opnd, 1, II_AND, and>,
             ADD_FM<0, 0x24>;
-def OR    : MMRel, ArithLogicR<"or", GPR32Opnd, 1, II_OR, or>,
+def OR    : MMRel, StdMMR6Rel, ArithLogicR<"or", GPR32Opnd, 1, II_OR, or>,
             ADD_FM<0, 0x25>;
-def XOR   : MMRel, ArithLogicR<"xor", GPR32Opnd, 1, II_XOR, xor>,
+def XOR   : MMRel, StdMMR6Rel, ArithLogicR<"xor", GPR32Opnd, 1, II_XOR, xor>,
             ADD_FM<0, 0x26>;
 }
-def NOR   : MMRel, LogicNOR<"nor", GPR32Opnd>, ADD_FM<0, 0x27>;
+def NOR   : MMRel, StdMMR6Rel, LogicNOR<"nor", GPR32Opnd>, ADD_FM<0, 0x27>;
 
 /// Shift Instructions
 let AdditionalPredicates = [NotInMicroMips] in {
@@ -1261,15 +1285,22 @@ def TTLTIU : MMRel, TEQI_FT<"tltiu", GPR32Opnd>, TEQI_FM<0xb>,
 def TNEI : MMRel, TEQI_FT<"tnei", GPR32Opnd>, TEQI_FM<0xe>,
            ISA_MIPS2_NOT_32R6_64R6;
 
-def BREAK : MMRel, BRK_FT<"break">, BRK_FM<0xd>;
+let AdditionalPredicates = [NotInMicroMips] in {
+def BREAK : MMRel, StdMMR6Rel, BRK_FT<"break">, BRK_FM<0xd>;
+}
 def SYSCALL : MMRel, SYS_FT<"syscall">, SYS_FM<0xc>;
 def TRAP : TrapBase<BREAK>;
 def SDBBP : MMRel, SYS_FT<"sdbbp">, SDBBP_FM, ISA_MIPS32_NOT_32R6_64R6;
 
-def ERET : MMRel, ER_FT<"eret">, ER_FM<0x18>, INSN_MIPS3_32;
-def DERET : MMRel, ER_FT<"deret">, ER_FM<0x1f>, ISA_MIPS32;
+let AdditionalPredicates = [NotInMicroMips] in {
+def ERET : MMRel, ER_FT<"eret">, ER_FM<0x18, 0x0>, INSN_MIPS3_32;
+def ERETNC : MMRel, ER_FT<"eretnc">, ER_FM<0x18, 0x1>, ISA_MIPS32R5;
+}
+def DERET : MMRel, ER_FT<"deret">, ER_FM<0x1f, 0x0>, ISA_MIPS32;
 
-def EI : MMRel, DEI_FT<"ei", GPR32Opnd>, EI_FM<1>, ISA_MIPS32R2;
+let AdditionalPredicates = [NotInMicroMips] in {
+def EI : MMRel, StdMMR6Rel, DEI_FT<"ei", GPR32Opnd>, EI_FM<1>, ISA_MIPS32R2;
+}
 def DI : MMRel, DEI_FT<"di", GPR32Opnd>, EI_FM<0>, ISA_MIPS32R2;
 
 let EncodingPredicates = []<Predicate>, // FIXME: Lack of HasStdEnc is probably a bug
@@ -1403,9 +1434,9 @@ def MFLO : MMRel, MoveFromLOHI<"mflo", GPR32Opnd, AC0>, MFLO_FM<0x12>,
 }
 
 /// Sign Ext In Register Instructions.
-def SEB : MMRel, SignExtInReg<"seb", i8, GPR32Opnd, II_SEB>,
+def SEB : MMRel, StdMMR6Rel, SignExtInReg<"seb", i8, GPR32Opnd, II_SEB>,
           SEB_FM<0x10, 0x20>, ISA_MIPS32R2;
-def SEH : MMRel, SignExtInReg<"seh", i16, GPR32Opnd, II_SEH>,
+def SEH : MMRel, StdMMR6Rel, SignExtInReg<"seh", i16, GPR32Opnd, II_SEH>,
           SEB_FM<0x18, 0x20>, ISA_MIPS32R2;
 
 /// Count Leading
@@ -1465,10 +1496,10 @@ def EXT : MMRel, ExtBase<"ext", GPR32Opnd, uimm5, MipsExt>, EXT_FM<0>;
 def INS : MMRel, InsBase<"ins", GPR32Opnd, uimm5, MipsIns>, EXT_FM<4>;
 
 /// Move Control Registers From/To CPU Registers
-def MFC0 : MFC3OP<"mfc0", GPR32Opnd>, MFC3OP_FM<0x10, 0>, ISA_MIPS32;
-def MTC0 : MFC3OP<"mtc0", GPR32Opnd>, MFC3OP_FM<0x10, 4>, ISA_MIPS32;
-def MFC2 : MFC3OP<"mfc2", GPR32Opnd>, MFC3OP_FM<0x12, 0>;
-def MTC2 : MFC3OP<"mtc2", GPR32Opnd>, MFC3OP_FM<0x12, 4>;
+def MFC0 : MFC3OP<"mfc0", GPR32Opnd, COP0Opnd>, MFC3OP_FM<0x10, 0>, ISA_MIPS32;
+def MTC0 : MTC3OP<"mtc0", COP0Opnd, GPR32Opnd>, MFC3OP_FM<0x10, 4>, ISA_MIPS32;
+def MFC2 : MFC3OP<"mfc2", GPR32Opnd, COP2Opnd>, MFC3OP_FM<0x12, 0>;
+def MTC2 : MTC3OP<"mtc2", COP2Opnd, GPR32Opnd>, MFC3OP_FM<0x12, 4>;
 
 class Barrier<string asmstr> : InstSE<(outs), (ins), asmstr, [], NoItinerary,
                                       FrmOther, asmstr>;
@@ -1584,11 +1615,13 @@ def : MipsInstAlias<"or $rs, $rt, $imm",
                     (ORi GPR32Opnd:$rs, GPR32Opnd:$rt, uimm16:$imm), 0>;
 def : MipsInstAlias<"or $rs, $imm",
                     (ORi GPR32Opnd:$rs, GPR32Opnd:$rs, uimm16:$imm), 0>;
+let AdditionalPredicates = [NotInMicroMips] in {
 def : MipsInstAlias<"nop", (SLL ZERO, ZERO, 0), 1>;
-def : MipsInstAlias<"mfc0 $rt, $rd", (MFC0 GPR32Opnd:$rt, GPR32Opnd:$rd, 0), 0>;
-def : MipsInstAlias<"mtc0 $rt, $rd", (MTC0 GPR32Opnd:$rt, GPR32Opnd:$rd, 0), 0>;
-def : MipsInstAlias<"mfc2 $rt, $rd", (MFC2 GPR32Opnd:$rt, GPR32Opnd:$rd, 0), 0>;
-def : MipsInstAlias<"mtc2 $rt, $rd", (MTC2 GPR32Opnd:$rt, GPR32Opnd:$rd, 0), 0>;
+}
+def : MipsInstAlias<"mfc0 $rt, $rd", (MFC0 GPR32Opnd:$rt, COP0Opnd:$rd, 0), 0>;
+def : MipsInstAlias<"mtc0 $rt, $rd", (MTC0 COP0Opnd:$rd, GPR32Opnd:$rt, 0), 0>;
+def : MipsInstAlias<"mfc2 $rt, $rd", (MFC2 GPR32Opnd:$rt, COP2Opnd:$rd, 0), 0>;
+def : MipsInstAlias<"mtc2 $rt, $rd", (MTC2 COP2Opnd:$rd, GPR32Opnd:$rt, 0), 0>;
 let AdditionalPredicates = [NotInMicroMips] in {
 def : MipsInstAlias<"b $offset", (BEQ ZERO, ZERO, brtarget:$offset), 0>;
 }
@@ -1604,7 +1637,9 @@ def : MipsInstAlias<"syscall", (SYSCALL 0), 1>;
     
 def : MipsInstAlias<"break", (BREAK 0, 0), 1>;
 def : MipsInstAlias<"break $imm", (BREAK uimm10:$imm, 0), 1>;
+let AdditionalPredicates = [NotInMicroMips] in {
 def : MipsInstAlias<"ei", (EI ZERO), 1>, ISA_MIPS32R2;
+}
 def : MipsInstAlias<"di", (DI ZERO), 1>, ISA_MIPS32R2;
 
 def : MipsInstAlias<"teq $rs, $rt",
@@ -1665,6 +1700,35 @@ def JalTwoReg : MipsAsmPseudoInst<(outs GPR32Opnd:$rd), (ins GPR32Opnd:$rs),
 def JalOneReg : MipsAsmPseudoInst<(outs), (ins GPR32Opnd:$rs),
                       "jal\t$rs"> ;
 
+let hasDelaySlot = 1 in {
+def BneImm : MipsAsmPseudoInst<(outs GPR32Opnd:$rt),
+                               (ins imm64:$imm64, brtarget:$offset),
+                               "bne\t$rt, $imm64, $offset">;
+def BeqImm : MipsAsmPseudoInst<(outs GPR32Opnd:$rt),
+                               (ins imm64:$imm64, brtarget:$offset),
+                               "beq\t$rt, $imm64, $offset">;
+
+class CondBranchPseudo<string instr_asm> :
+  MipsAsmPseudoInst<(outs), (ins GPR32Opnd:$rs, GPR32Opnd:$rt,
+                                 brtarget:$offset),
+                    !strconcat(instr_asm, "\t$rs, $rt, $offset")>;
+}
+
+def BLT : CondBranchPseudo<"blt">;
+def BLE : CondBranchPseudo<"ble">;
+def BGE : CondBranchPseudo<"bge">;
+def BGT : CondBranchPseudo<"bgt">;
+def BLTU : CondBranchPseudo<"bltu">;
+def BLEU : CondBranchPseudo<"bleu">;
+def BGEU : CondBranchPseudo<"bgeu">;
+def BGTU : CondBranchPseudo<"bgtu">;
+
+def Ulhu : MipsAsmPseudoInst<(outs GPR32Opnd:$rt), (ins mem:$addr),
+                             "ulhu\t$rt, $addr">, ISA_MIPS1_NOT_32R6_64R6;
+
+def Ulw : MipsAsmPseudoInst<(outs GPR32Opnd:$rt), (ins mem:$addr),
+                            "ulw\t$rt, $addr">, ISA_MIPS1_NOT_32R6_64R6;
+
 //===----------------------------------------------------------------------===//
 //  Arbitrary patterns that map to one or more instructions
 //===----------------------------------------------------------------------===//
@@ -1908,3 +1972,7 @@ include "MipsMSAInstrInfo.td"
 include "MicroMipsInstrFormats.td"
 include "MicroMipsInstrInfo.td"
 include "MicroMipsInstrFPU.td"
+
+// Micromips r6
+include "MicroMips32r6InstrFormats.td"
+include "MicroMips32r6InstrInfo.td"
diff --git a/lib/Target/Mips/MipsMCInstLower.cpp b/lib/Target/Mips/MipsMCInstLower.cpp
index 5258181f910..80d9b75b85b 100644
--- a/lib/Target/Mips/MipsMCInstLower.cpp
+++ b/lib/Target/Mips/MipsMCInstLower.cpp
@@ -88,6 +88,11 @@ MCOperand MipsMCInstLower::LowerSymbolOperand(const MachineOperand &MO,
     Offset += MO.getOffset();
     break;
 
+  case MachineOperand::MO_MCSymbol:
+    Symbol = MO.getMCSymbol();
+    Offset += MO.getOffset();
+    break;
+
   case MachineOperand::MO_JumpTableIndex:
     Symbol = AsmPrinter.GetJTISymbol(MO.getIndex());
     break;
@@ -101,17 +106,17 @@ MCOperand MipsMCInstLower::LowerSymbolOperand(const MachineOperand &MO,
     llvm_unreachable("<unknown operand type>");
   }
 
-  const MCSymbolRefExpr *MCSym = MCSymbolRefExpr::Create(Symbol, Kind, *Ctx);
+  const MCSymbolRefExpr *MCSym = MCSymbolRefExpr::create(Symbol, Kind, *Ctx);
 
   if (!Offset)
-    return MCOperand::CreateExpr(MCSym);
+    return MCOperand::createExpr(MCSym);
 
   // Assume offset is never negative.
   assert(Offset > 0);
 
-  const MCConstantExpr *OffsetExpr =  MCConstantExpr::Create(Offset, *Ctx);
-  const MCBinaryExpr *Add = MCBinaryExpr::CreateAdd(MCSym, OffsetExpr, *Ctx);
-  return MCOperand::CreateExpr(Add);
+  const MCConstantExpr *OffsetExpr =  MCConstantExpr::create(Offset, *Ctx);
+  const MCBinaryExpr *Add = MCBinaryExpr::createAdd(MCSym, OffsetExpr, *Ctx);
+  return MCOperand::createExpr(Add);
 }
 
 /*
@@ -135,12 +140,13 @@ MCOperand MipsMCInstLower::LowerOperand(const MachineOperand &MO,
   case MachineOperand::MO_Register:
     // Ignore all implicit register operands.
     if (MO.isImplicit()) break;
-    return MCOperand::CreateReg(MO.getReg());
+    return MCOperand::createReg(MO.getReg());
   case MachineOperand::MO_Immediate:
-    return MCOperand::CreateImm(MO.getImm() + offset);
+    return MCOperand::createImm(MO.getImm() + offset);
   case MachineOperand::MO_MachineBasicBlock:
   case MachineOperand::MO_GlobalAddress:
   case MachineOperand::MO_ExternalSymbol:
+  case MachineOperand::MO_MCSymbol:
   case MachineOperand::MO_JumpTableIndex:
   case MachineOperand::MO_ConstantPoolIndex:
   case MachineOperand::MO_BlockAddress:
@@ -155,11 +161,11 @@ MCOperand MipsMCInstLower::LowerOperand(const MachineOperand &MO,
 MCOperand MipsMCInstLower::createSub(MachineBasicBlock *BB1,
                                      MachineBasicBlock *BB2,
                                      MCSymbolRefExpr::VariantKind Kind) const {
-  const MCSymbolRefExpr *Sym1 = MCSymbolRefExpr::Create(BB1->getSymbol(), *Ctx);
-  const MCSymbolRefExpr *Sym2 = MCSymbolRefExpr::Create(BB2->getSymbol(), *Ctx);
-  const MCBinaryExpr *Sub = MCBinaryExpr::CreateSub(Sym1, Sym2, *Ctx);
+  const MCSymbolRefExpr *Sym1 = MCSymbolRefExpr::create(BB1->getSymbol(), *Ctx);
+  const MCSymbolRefExpr *Sym2 = MCSymbolRefExpr::create(BB2->getSymbol(), *Ctx);
+  const MCBinaryExpr *Sub = MCBinaryExpr::createSub(Sym1, Sym2, *Ctx);
 
-  return MCOperand::CreateExpr(MipsMCExpr::Create(Kind, Sub, *Ctx));
+  return MCOperand::createExpr(MipsMCExpr::create(Kind, Sub, *Ctx));
 }
 
 void MipsMCInstLower::
diff --git a/lib/Target/Mips/MipsMSAInstrInfo.td b/lib/Target/Mips/MipsMSAInstrInfo.td
index 68230e6957a..970e98ea9e1 100644
--- a/lib/Target/Mips/MipsMSAInstrInfo.td
+++ b/lib/Target/Mips/MipsMSAInstrInfo.td
@@ -63,6 +63,9 @@ def MipsVExtractSExt : SDNode<"MipsISD::VEXTRACT_SEXT_ELT",
 def MipsVExtractZExt : SDNode<"MipsISD::VEXTRACT_ZEXT_ELT",
     SDTypeProfile<1, 3, [SDTCisPtrTy<2>]>, []>;
 
+def immZExt4Ptr : ImmLeaf<iPTR, [{return isUInt<4>(Imm);}]>;
+def immZExt6Ptr : ImmLeaf<iPTR, [{return isUInt<6>(Imm);}]>;
+
 // Operands
 
 // The immediate of an LSA instruction needs special handling
@@ -84,6 +87,14 @@ def uimm4 : Operand<i32> {
   let PrintMethod = "printUnsignedImm8";
 }
 
+def uimm4_ptr : Operand<iPTR> {
+  let PrintMethod = "printUnsignedImm8";
+}
+
+def uimm6_ptr : Operand<iPTR> {
+  let PrintMethod = "printUnsignedImm8";
+}
+
 def uimm8 : Operand<i32> {
   let PrintMethod = "printUnsignedImm8";
 }
@@ -364,7 +375,7 @@ def vsplat_imm_eq_1 : PatLeaf<(build_vector), [{
   APInt Imm;
   EVT EltTy = N->getValueType(0).getVectorElementType();
 
-  return selectVSplat (N, Imm) &&
+  return selectVSplat(N, Imm, EltTy.getSizeInBits()) &&
          Imm.getBitWidth() == EltTy.getSizeInBits() && Imm == 1;
 }]>;
 
@@ -373,7 +384,7 @@ def vsplati64_imm_eq_1 : PatLeaf<(bitconvert (v4i32 (build_vector))), [{
   SDNode *BV = N->getOperand(0).getNode();
   EVT EltTy = N->getValueType(0).getVectorElementType();
 
-  return selectVSplat (BV, Imm) &&
+  return selectVSplat(BV, Imm, EltTy.getSizeInBits()) &&
          Imm.getBitWidth() == EltTy.getSizeInBits() && Imm == 1;
 }]>;
 
@@ -1273,9 +1284,9 @@ class MSA_COPY_DESC_BASE<string instr_asm, SDPatternOperator OpNode,
                          RegisterOperand ROWS,
                          InstrItinClass itin = NoItinerary> {
   dag OutOperandList = (outs ROD:$rd);
-  dag InOperandList = (ins ROWS:$ws, uimm4:$n);
+  dag InOperandList = (ins ROWS:$ws, uimm4_ptr:$n);
   string AsmString = !strconcat(instr_asm, "\t$rd, $ws[$n]");
-  list<dag> Pattern = [(set ROD:$rd, (OpNode (VecTy ROWS:$ws), immZExt4:$n))];
+  list<dag> Pattern = [(set ROD:$rd, (OpNode (VecTy ROWS:$ws), immZExt4Ptr:$n))];
   InstrItinClass Itinerary = itin;
 }
 
@@ -1293,8 +1304,8 @@ class MSA_ELM_SLD_DESC_BASE<string instr_asm, SDPatternOperator OpNode,
 
 class MSA_COPY_PSEUDO_BASE<SDPatternOperator OpNode, ValueType VecTy,
                            RegisterClass RCD, RegisterClass RCWS> :
-      MSAPseudo<(outs RCD:$wd), (ins RCWS:$ws, uimm4:$n),
-                [(set RCD:$wd, (OpNode (VecTy RCWS:$ws), immZExt4:$n))]> {
+      MSAPseudo<(outs RCD:$wd), (ins RCWS:$ws, uimm4_ptr:$n),
+                [(set RCD:$wd, (OpNode (VecTy RCWS:$ws), immZExt4Ptr:$n))]> {
   bit usesCustomInserter = 1;
 }
 
@@ -1479,29 +1490,30 @@ class MSA_INSERT_DESC_BASE<string instr_asm, SDPatternOperator OpNode,
                            RegisterOperand ROWD, RegisterOperand ROS,
                            InstrItinClass itin = NoItinerary> {
   dag OutOperandList = (outs ROWD:$wd);
-  dag InOperandList = (ins ROWD:$wd_in, ROS:$rs, uimm6:$n);
+  dag InOperandList = (ins ROWD:$wd_in, ROS:$rs, uimm6_ptr:$n);
   string AsmString = !strconcat(instr_asm, "\t$wd[$n], $rs");
   list<dag> Pattern = [(set ROWD:$wd, (OpNode ROWD:$wd_in,
                                               ROS:$rs,
-                                              immZExt6:$n))];
+                                              immZExt6Ptr:$n))];
   InstrItinClass Itinerary = itin;
   string Constraints = "$wd = $wd_in";
 }
 
 class MSA_INSERT_PSEUDO_BASE<SDPatternOperator OpNode, ValueType Ty,
                              RegisterOperand ROWD, RegisterOperand ROFS> :
-      MSAPseudo<(outs ROWD:$wd), (ins ROWD:$wd_in, uimm6:$n, ROFS:$fs),
+      MSAPseudo<(outs ROWD:$wd), (ins ROWD:$wd_in, uimm6_ptr:$n, ROFS:$fs),
                 [(set ROWD:$wd, (OpNode (Ty ROWD:$wd_in), ROFS:$fs,
-                                        immZExt6:$n))]> {
+                                        immZExt6Ptr:$n))]> {
   bit usesCustomInserter = 1;
   string Constraints = "$wd = $wd_in";
 }
 
 class MSA_INSERT_VIDX_PSEUDO_BASE<SDPatternOperator OpNode, ValueType Ty,
-                                  RegisterOperand ROWD, RegisterOperand ROFS> :
-      MSAPseudo<(outs ROWD:$wd), (ins ROWD:$wd_in, GPR32Opnd:$n, ROFS:$fs),
+                                  RegisterOperand ROWD, RegisterOperand ROFS,
+                                  RegisterOperand ROIdx> :
+      MSAPseudo<(outs ROWD:$wd), (ins ROWD:$wd_in, ROIdx:$n, ROFS:$fs),
                 [(set ROWD:$wd, (OpNode (Ty ROWD:$wd_in), ROFS:$fs,
-                                        GPR32Opnd:$n))]> {
+                                        ROIdx:$n))]> {
   bit usesCustomInserter = 1;
   string Constraints = "$wd = $wd_in";
 }
@@ -2302,13 +2314,13 @@ class INSERT_D_DESC : MSA_INSERT_DESC_BASE<"insert.d", vinsert_v2i64,
                                            MSA128DOpnd, GPR64Opnd>;
 
 class INSERT_B_VIDX_PSEUDO_DESC :
-    MSA_INSERT_VIDX_PSEUDO_BASE<vector_insert, v16i8, MSA128BOpnd, GPR32Opnd>;
+    MSA_INSERT_VIDX_PSEUDO_BASE<vector_insert, v16i8, MSA128BOpnd, GPR32Opnd, GPR32Opnd>;
 class INSERT_H_VIDX_PSEUDO_DESC :
-    MSA_INSERT_VIDX_PSEUDO_BASE<vector_insert, v8i16, MSA128HOpnd, GPR32Opnd>;
+    MSA_INSERT_VIDX_PSEUDO_BASE<vector_insert, v8i16, MSA128HOpnd, GPR32Opnd, GPR32Opnd>;
 class INSERT_W_VIDX_PSEUDO_DESC :
-    MSA_INSERT_VIDX_PSEUDO_BASE<vector_insert, v4i32, MSA128WOpnd, GPR32Opnd>;
+    MSA_INSERT_VIDX_PSEUDO_BASE<vector_insert, v4i32, MSA128WOpnd, GPR32Opnd, GPR32Opnd>;
 class INSERT_D_VIDX_PSEUDO_DESC :
-    MSA_INSERT_VIDX_PSEUDO_BASE<vector_insert, v2i64, MSA128DOpnd, GPR64Opnd>;
+    MSA_INSERT_VIDX_PSEUDO_BASE<vector_insert, v2i64, MSA128DOpnd, GPR64Opnd, GPR32Opnd>;
 
 class INSERT_FW_PSEUDO_DESC : MSA_INSERT_PSEUDO_BASE<vector_insert, v4f32,
                                                      MSA128WOpnd, FGR32Opnd>;
@@ -2316,9 +2328,23 @@ class INSERT_FD_PSEUDO_DESC : MSA_INSERT_PSEUDO_BASE<vector_insert, v2f64,
                                                      MSA128DOpnd, FGR64Opnd>;
 
 class INSERT_FW_VIDX_PSEUDO_DESC :
-    MSA_INSERT_VIDX_PSEUDO_BASE<vector_insert, v4f32, MSA128WOpnd, FGR32Opnd>;
+    MSA_INSERT_VIDX_PSEUDO_BASE<vector_insert, v4f32, MSA128WOpnd, FGR32Opnd, GPR32Opnd>;
 class INSERT_FD_VIDX_PSEUDO_DESC :
-    MSA_INSERT_VIDX_PSEUDO_BASE<vector_insert, v2f64, MSA128DOpnd, FGR64Opnd>;
+    MSA_INSERT_VIDX_PSEUDO_BASE<vector_insert, v2f64, MSA128DOpnd, FGR64Opnd, GPR32Opnd>;
+
+class INSERT_B_VIDX64_PSEUDO_DESC :
+    MSA_INSERT_VIDX_PSEUDO_BASE<vector_insert, v16i8, MSA128BOpnd, GPR32Opnd, GPR64Opnd>;
+class INSERT_H_VIDX64_PSEUDO_DESC :
+    MSA_INSERT_VIDX_PSEUDO_BASE<vector_insert, v8i16, MSA128HOpnd, GPR32Opnd, GPR64Opnd>;
+class INSERT_W_VIDX64_PSEUDO_DESC :
+    MSA_INSERT_VIDX_PSEUDO_BASE<vector_insert, v4i32, MSA128WOpnd, GPR32Opnd, GPR64Opnd>;
+class INSERT_D_VIDX64_PSEUDO_DESC :
+    MSA_INSERT_VIDX_PSEUDO_BASE<vector_insert, v2i64, MSA128DOpnd, GPR64Opnd, GPR64Opnd>;
+
+class INSERT_FW_VIDX64_PSEUDO_DESC :
+    MSA_INSERT_VIDX_PSEUDO_BASE<vector_insert, v4f32, MSA128WOpnd, FGR32Opnd, GPR64Opnd>;
+class INSERT_FD_VIDX64_PSEUDO_DESC :
+    MSA_INSERT_VIDX_PSEUDO_BASE<vector_insert, v2f64, MSA128DOpnd, FGR64Opnd, GPR64Opnd>;
 
 class INSVE_B_DESC : MSA_INSVE_DESC_BASE<"insve.b", insve_v16i8,
                                          MSA128BOpnd>;
@@ -3236,6 +3262,13 @@ def INSERT_D_VIDX_PSEUDO : INSERT_D_VIDX_PSEUDO_DESC;
 def INSERT_FW_VIDX_PSEUDO : INSERT_FW_VIDX_PSEUDO_DESC;
 def INSERT_FD_VIDX_PSEUDO : INSERT_FD_VIDX_PSEUDO_DESC;
 
+def INSERT_B_VIDX64_PSEUDO : INSERT_B_VIDX64_PSEUDO_DESC;
+def INSERT_H_VIDX64_PSEUDO : INSERT_H_VIDX64_PSEUDO_DESC;
+def INSERT_W_VIDX64_PSEUDO : INSERT_W_VIDX64_PSEUDO_DESC;
+def INSERT_D_VIDX64_PSEUDO : INSERT_D_VIDX64_PSEUDO_DESC;
+def INSERT_FW_VIDX64_PSEUDO : INSERT_FW_VIDX64_PSEUDO_DESC;
+def INSERT_FD_VIDX64_PSEUDO : INSERT_FD_VIDX64_PSEUDO_DESC;
+
 def LD_B: LD_B_ENC, LD_B_DESC;
 def LD_H: LD_H_ENC, LD_H_DESC;
 def LD_W: LD_W_ENC, LD_W_DESC;
@@ -3805,3 +3838,93 @@ def : MSAPat<(f64 (vector_extract v2f64:$ws, i32:$idx)),
              (f64 (EXTRACT_SUBREG (SPLAT_D v2f64:$ws,
                                            i32:$idx),
                                   sub_64))>;
+
+// Vector extraction with variable index (N64 ABI)
+def : MSAPat<
+  (i32 (vextract_sext_i8 v16i8:$ws, i64:$idx)),
+  (SRA (COPY_TO_REGCLASS
+         (i32 (EXTRACT_SUBREG
+                (SPLAT_B v16i8:$ws,
+                  (COPY_TO_REGCLASS
+                    (i32 (EXTRACT_SUBREG i64:$idx, sub_32)), GPR32)),
+                sub_lo)),
+         GPR32),
+       (i32 24))>;
+def : MSAPat<
+  (i32 (vextract_sext_i16 v8i16:$ws, i64:$idx)),
+  (SRA (COPY_TO_REGCLASS
+         (i32 (EXTRACT_SUBREG
+                (SPLAT_H v8i16:$ws,
+                  (COPY_TO_REGCLASS
+                    (i32 (EXTRACT_SUBREG i64:$idx, sub_32)), GPR32)),
+                sub_lo)),
+         GPR32),
+       (i32 16))>;
+def : MSAPat<
+  (i32 (vextract_sext_i32 v4i32:$ws, i64:$idx)),
+  (COPY_TO_REGCLASS
+    (i32 (EXTRACT_SUBREG
+           (SPLAT_W v4i32:$ws,
+             (COPY_TO_REGCLASS
+               (i32 (EXTRACT_SUBREG i64:$idx, sub_32)), GPR32)),
+           sub_lo)),
+    GPR32)>;
+def : MSAPat<
+  (i64 (vextract_sext_i64 v2i64:$ws, i64:$idx)),
+  (COPY_TO_REGCLASS
+    (i64 (EXTRACT_SUBREG
+           (SPLAT_D v2i64:$ws,
+             (COPY_TO_REGCLASS (i32 (EXTRACT_SUBREG i64:$idx, sub_32)), GPR32)),
+           sub_64)),
+    GPR64), [HasMSA, IsGP64bit]>;
+
+def : MSAPat<
+  (i32 (vextract_zext_i8 v16i8:$ws, i64:$idx)),
+  (SRL (COPY_TO_REGCLASS
+         (i32 (EXTRACT_SUBREG
+                 (SPLAT_B v16i8:$ws,
+                   (COPY_TO_REGCLASS
+                     (i32 (EXTRACT_SUBREG i64:$idx, sub_32)), GPR32)),
+                 sub_lo)),
+         GPR32),
+       (i32 24))>;
+def : MSAPat<
+  (i32 (vextract_zext_i16 v8i16:$ws, i64:$idx)),
+  (SRL (COPY_TO_REGCLASS
+         (i32 (EXTRACT_SUBREG
+                (SPLAT_H v8i16:$ws,
+                  (COPY_TO_REGCLASS
+                    (i32 (EXTRACT_SUBREG i64:$idx, sub_32)), GPR32)),
+                sub_lo)),
+         GPR32),
+       (i32 16))>;
+def : MSAPat<
+  (i32 (vextract_zext_i32 v4i32:$ws, i64:$idx)),
+  (COPY_TO_REGCLASS
+    (i32 (EXTRACT_SUBREG
+           (SPLAT_W v4i32:$ws,
+             (COPY_TO_REGCLASS (i32 (EXTRACT_SUBREG i64:$idx, sub_32)), GPR32)),
+           sub_lo)),
+    GPR32)>;
+def : MSAPat<
+  (i64 (vextract_zext_i64 v2i64:$ws, i64:$idx)),
+  (COPY_TO_REGCLASS
+    (i64 (EXTRACT_SUBREG
+           (SPLAT_D v2i64:$ws,
+             (COPY_TO_REGCLASS (i32 (EXTRACT_SUBREG i64:$idx, sub_32)), GPR32)),
+           sub_64)),
+    GPR64),
+  [HasMSA, IsGP64bit]>;
+
+def : MSAPat<
+  (f32 (vector_extract v4f32:$ws, i64:$idx)),
+  (f32 (EXTRACT_SUBREG
+         (SPLAT_W v4f32:$ws,
+           (COPY_TO_REGCLASS (i32 (EXTRACT_SUBREG i64:$idx, sub_32)), GPR32)),
+         sub_lo))>;
+def : MSAPat<
+  (f64 (vector_extract v2f64:$ws, i64:$idx)),
+  (f64 (EXTRACT_SUBREG
+         (SPLAT_D v2f64:$ws,
+           (COPY_TO_REGCLASS (i32 (EXTRACT_SUBREG i64:$idx, sub_32)), GPR32)),
+         sub_64))>;
diff --git a/lib/Target/Mips/MipsOptionRecord.h b/lib/Target/Mips/MipsOptionRecord.h
index 746feab1d9a..23f0b7070d6 100644
--- a/lib/Target/Mips/MipsOptionRecord.h
+++ b/lib/Target/Mips/MipsOptionRecord.h
@@ -49,6 +49,7 @@ class MipsRegInfoRecord : public MipsOptionRecord {
     FGR64RegClass = &(TRI->getRegClass(Mips::FGR64RegClassID));
     AFGR64RegClass = &(TRI->getRegClass(Mips::AFGR64RegClassID));
     MSA128BRegClass = &(TRI->getRegClass(Mips::MSA128BRegClassID));
+    COP0RegClass = &(TRI->getRegClass(Mips::COP0RegClassID));
     COP2RegClass = &(TRI->getRegClass(Mips::COP2RegClassID));
     COP3RegClass = &(TRI->getRegClass(Mips::COP3RegClassID));
   }
@@ -66,6 +67,7 @@ class MipsRegInfoRecord : public MipsOptionRecord {
   const MCRegisterClass *FGR64RegClass;
   const MCRegisterClass *AFGR64RegClass;
   const MCRegisterClass *MSA128BRegClass;
+  const MCRegisterClass *COP0RegClass;
   const MCRegisterClass *COP2RegClass;
   const MCRegisterClass *COP3RegClass;
   uint32_t ri_gprmask;
diff --git a/lib/Target/Mips/MipsRegisterInfo.cpp b/lib/Target/Mips/MipsRegisterInfo.cpp
index 0ea48b19891..775d8006fdf 100644
--- a/lib/Target/Mips/MipsRegisterInfo.cpp
+++ b/lib/Target/Mips/MipsRegisterInfo.cpp
@@ -21,6 +21,7 @@
 #include "llvm/ADT/BitVector.h"
 #include "llvm/ADT/STLExtras.h"
 #include "llvm/CodeGen/MachineFrameInfo.h"
+#include "llvm/CodeGen/MachineRegisterInfo.h"
 #include "llvm/CodeGen/MachineFunction.h"
 #include "llvm/CodeGen/MachineInstrBuilder.h"
 #include "llvm/IR/Constants.h"
@@ -50,8 +51,8 @@ unsigned MipsRegisterInfo::getPICCallReg() { return Mips::T9; }
 const TargetRegisterClass *
 MipsRegisterInfo::getPointerRegClass(const MachineFunction &MF,
                                      unsigned Kind) const {
-  const MipsSubtarget &Subtarget = MF.getSubtarget<MipsSubtarget>();
-  return Subtarget.isABI_N64() ? &Mips::GPR64RegClass : &Mips::GPR32RegClass;
+  MipsABIInfo ABI = MF.getSubtarget<MipsSubtarget>().getABI();
+  return ABI.ArePtrs64bit() ? &Mips::GPR64RegClass : &Mips::GPR32RegClass;
 }
 
 unsigned
@@ -178,6 +179,15 @@ getReservedRegs(const MachineFunction &MF) const {
     else {
       Reserved.set(Mips::FP);
       Reserved.set(Mips::FP_64);
+
+      // Reserve the base register if we need to both realign the stack and
+      // allocate variable-sized objects at runtime. This should test the
+      // same conditions as MipsFrameLowering::hasBP().
+      if (needsStackRealignment(MF) &&
+          MF.getFrameInfo()->hasVarSizedObjects()) {
+        Reserved.set(Mips::S7);
+        Reserved.set(Mips::S7_64);
+      }
     }
   }
 
@@ -271,6 +281,38 @@ getFrameRegister(const MachineFunction &MF) const {
   else
     return TFI->hasFP(MF) ? (IsN64 ? Mips::FP_64 : Mips::FP) :
                             (IsN64 ? Mips::SP_64 : Mips::SP);
-
 }
 
+bool MipsRegisterInfo::canRealignStack(const MachineFunction &MF) const {
+  // Avoid realigning functions that explicitly do not want to be realigned.
+  // Normally, we should report an error when a function should be dynamically
+  // realigned but also has the attribute no-realign-stack. Unfortunately,
+  // with this attribute, MachineFrameInfo clamps each new object's alignment
+  // to that of the stack's alignment as specified by the ABI. As a result,
+  // the information of whether we have objects with larger alignment
+  // requirement than the stack's alignment is already lost at this point.
+  if (!TargetRegisterInfo::canRealignStack(MF))
+    return false;
+
+  const MipsSubtarget &Subtarget = MF.getSubtarget<MipsSubtarget>();
+  unsigned FP = Subtarget.isGP32bit() ? Mips::FP : Mips::FP_64;
+  unsigned BP = Subtarget.isGP32bit() ? Mips::S7 : Mips::S7_64;
+
+  // Support dynamic stack realignment only for targets with standard encoding.
+  if (!Subtarget.hasStandardEncoding())
+    return false;
+
+  // We can't perform dynamic stack realignment if we can't reserve the
+  // frame pointer register.
+  if (!MF.getRegInfo().canReserveReg(FP))
+    return false;
+
+  // We can realign the stack if we know the maximum call frame size and we
+  // don't have variable sized objects.
+  if (Subtarget.getFrameLowering()->hasReservedCallFrame(MF))
+    return true;
+
+  // We have to reserve the base pointer register in the presence of variable
+  // sized objects.
+  return MF.getRegInfo().canReserveReg(BP);
+}
diff --git a/lib/Target/Mips/MipsRegisterInfo.h b/lib/Target/Mips/MipsRegisterInfo.h
index 031b93ef9e4..5de68a21b73 100644
--- a/lib/Target/Mips/MipsRegisterInfo.h
+++ b/lib/Target/Mips/MipsRegisterInfo.h
@@ -60,6 +60,9 @@ class MipsRegisterInfo : public MipsGenRegisterInfo {
   void processFunctionBeforeFrameFinalized(MachineFunction &MF,
                                        RegScavenger *RS = nullptr) const;
 
+  // Stack realignment queries.
+  bool canRealignStack(const MachineFunction &MF) const override;
+
   /// Debug information queries.
   unsigned getFrameRegister(const MachineFunction &MF) const override;
 
diff --git a/lib/Target/Mips/MipsRegisterInfo.td b/lib/Target/Mips/MipsRegisterInfo.td
index 7497a255673..02bcac5a3dd 100644
--- a/lib/Target/Mips/MipsRegisterInfo.td
+++ b/lib/Target/Mips/MipsRegisterInfo.td
@@ -201,6 +201,10 @@ let Namespace = "Mips" in {
   foreach I = 0-7 in
   def FCC#I : MipsReg<#I, "fcc"#I>;
 
+  // COP0 registers.
+  foreach I = 0-31 in
+  def COP0#I : MipsReg<#I, ""#I>;
+
   // COP2 registers.
   foreach I = 0-31 in
   def COP2#I : MipsReg<#I, ""#I>;
@@ -431,6 +435,10 @@ def ACC64DSP : RegisterClass<"Mips", [untyped], 64, (sequence "AC%u", 0, 3)> {
 
 def DSPCC : RegisterClass<"Mips", [v4i8, v2i16], 32, (add DSPCCond)>;
 
+// Coprocessor 0 registers.
+def COP0 : RegisterClass<"Mips", [i32], 32, (sequence "COP0%u", 0, 31)>,
+           Unallocatable;
+
 // Coprocessor 2 registers.
 def COP2 : RegisterClass<"Mips", [i32], 32, (sequence "COP2%u", 0, 31)>,
            Unallocatable;
@@ -559,6 +567,10 @@ def HWRegsAsmOperand : MipsAsmRegOperand {
   let Name = "HWRegsAsmReg";
 }
 
+def COP0AsmOperand : MipsAsmRegOperand {
+  let Name = "COP0AsmReg";
+}
+
 def COP2AsmOperand : MipsAsmRegOperand {
   let Name = "COP2AsmReg";
 }
@@ -609,6 +621,10 @@ def ACC64DSPOpnd : RegisterOperand<ACC64DSP> {
   let ParserMatchClass = ACC64DSPAsmOperand;
 }
 
+def COP0Opnd : RegisterOperand<COP0> {
+  let ParserMatchClass = COP0AsmOperand;
+}
+
 def COP2Opnd : RegisterOperand<COP2> {
   let ParserMatchClass = COP2AsmOperand;
 }
diff --git a/lib/Target/Mips/MipsSEFrameLowering.cpp b/lib/Target/Mips/MipsSEFrameLowering.cpp
index 23feb5c3861..a19d1504e1e 100644
--- a/lib/Target/Mips/MipsSEFrameLowering.cpp
+++ b/lib/Target/Mips/MipsSEFrameLowering.cpp
@@ -364,19 +364,9 @@ bool ExpandPseudo::expandExtractElementF64(MachineBasicBlock &MBB,
 MipsSEFrameLowering::MipsSEFrameLowering(const MipsSubtarget &STI)
     : MipsFrameLowering(STI, STI.stackAlignment()) {}
 
-unsigned MipsSEFrameLowering::ehDataReg(unsigned I) const {
-  static const unsigned EhDataReg[] = {
-    Mips::A0, Mips::A1, Mips::A2, Mips::A3
-  };
-  static const unsigned EhDataReg64[] = {
-    Mips::A0_64, Mips::A1_64, Mips::A2_64, Mips::A3_64
-  };
-
-  return STI.isABI_N64() ? EhDataReg64[I] : EhDataReg[I];
-}
-
-void MipsSEFrameLowering::emitPrologue(MachineFunction &MF) const {
-  MachineBasicBlock &MBB   = MF.front();
+void MipsSEFrameLowering::emitPrologue(MachineFunction &MF,
+                                       MachineBasicBlock &MBB) const {
+  assert(&MF.front() == &MBB && "Shrink-wrapping not yet supported");
   MachineFrameInfo *MFI    = MF.getFrameInfo();
   MipsFunctionInfo *MipsFI = MF.getInfo<MipsFunctionInfo>();
 
@@ -387,10 +377,16 @@ void MipsSEFrameLowering::emitPrologue(MachineFunction &MF) const {
 
   MachineBasicBlock::iterator MBBI = MBB.begin();
   DebugLoc dl = MBBI != MBB.end() ? MBBI->getDebugLoc() : DebugLoc();
-  unsigned SP = STI.isABI_N64() ? Mips::SP_64 : Mips::SP;
-  unsigned FP = STI.isABI_N64() ? Mips::FP_64 : Mips::FP;
-  unsigned ZERO = STI.isABI_N64() ? Mips::ZERO_64 : Mips::ZERO;
-  unsigned ADDu = STI.isABI_N64() ? Mips::DADDu : Mips::ADDu;
+  MipsABIInfo ABI = STI.getABI();
+  unsigned SP = ABI.GetStackPtr();
+  unsigned FP = ABI.GetFramePtr();
+  unsigned ZERO = ABI.GetNullPtr();
+  unsigned ADDu = ABI.GetPtrAdduOp();
+  unsigned ADDiu = ABI.GetPtrAddiuOp();
+  unsigned AND = ABI.IsN64() ? Mips::AND64 : Mips::AND;
+
+  const TargetRegisterClass *RC = ABI.ArePtrs64bit() ?
+        &Mips::GPR64RegClass : &Mips::GPR32RegClass;
 
   // First, compute final stack size.
   uint64_t StackSize = MFI->getStackSize();
@@ -473,21 +469,18 @@ void MipsSEFrameLowering::emitPrologue(MachineFunction &MF) const {
   }
 
   if (MipsFI->callsEhReturn()) {
-    const TargetRegisterClass *RC = STI.isABI_N64() ?
-        &Mips::GPR64RegClass : &Mips::GPR32RegClass;
-
     // Insert instructions that spill eh data registers.
     for (int I = 0; I < 4; ++I) {
-      if (!MBB.isLiveIn(ehDataReg(I)))
-        MBB.addLiveIn(ehDataReg(I));
-      TII.storeRegToStackSlot(MBB, MBBI, ehDataReg(I), false,
+      if (!MBB.isLiveIn(ABI.GetEhDataReg(I)))
+        MBB.addLiveIn(ABI.GetEhDataReg(I));
+      TII.storeRegToStackSlot(MBB, MBBI, ABI.GetEhDataReg(I), false,
                               MipsFI->getEhDataRegFI(I), RC, &RegInfo);
     }
 
     // Emit .cfi_offset directives for eh data registers.
     for (int I = 0; I < 4; ++I) {
       int64_t Offset = MFI->getObjectOffset(MipsFI->getEhDataRegFI(I));
-      unsigned Reg = MRI->getDwarfRegNum(ehDataReg(I), true);
+      unsigned Reg = MRI->getDwarfRegNum(ABI.GetEhDataReg(I), true);
       unsigned CFIIndex = MMI.addFrameInst(
           MCCFIInstruction::createOffset(nullptr, Reg, Offset));
       BuildMI(MBB, MBBI, dl, TII.get(TargetOpcode::CFI_INSTRUCTION))
@@ -506,6 +499,26 @@ void MipsSEFrameLowering::emitPrologue(MachineFunction &MF) const {
         nullptr, MRI->getDwarfRegNum(FP, true)));
     BuildMI(MBB, MBBI, dl, TII.get(TargetOpcode::CFI_INSTRUCTION))
         .addCFIIndex(CFIIndex);
+
+    if (RegInfo.needsStackRealignment(MF)) {
+      // addiu $Reg, $zero, -MaxAlignment
+      // andi $sp, $sp, $Reg
+      unsigned VR = MF.getRegInfo().createVirtualRegister(RC);
+      assert(isInt<16>(MFI->getMaxAlignment()) &&
+             "Function's alignment size requirement is not supported.");
+      int MaxAlign = - (signed) MFI->getMaxAlignment();
+
+      BuildMI(MBB, MBBI, dl, TII.get(ADDiu), VR).addReg(ZERO) .addImm(MaxAlign);
+      BuildMI(MBB, MBBI, dl, TII.get(AND), SP).addReg(SP).addReg(VR);
+
+      if (hasBP(MF)) {
+        // move $s7, $sp
+        unsigned BP = STI.isABI_N64() ? Mips::S7_64 : Mips::S7;
+        BuildMI(MBB, MBBI, dl, TII.get(ADDu), BP)
+          .addReg(SP)
+          .addReg(ZERO);
+      }
+    }
   }
 }
 
@@ -521,10 +534,11 @@ void MipsSEFrameLowering::emitEpilogue(MachineFunction &MF,
       *static_cast<const MipsRegisterInfo *>(STI.getRegisterInfo());
 
   DebugLoc dl = MBBI->getDebugLoc();
-  unsigned SP = STI.isABI_N64() ? Mips::SP_64 : Mips::SP;
-  unsigned FP = STI.isABI_N64() ? Mips::FP_64 : Mips::FP;
-  unsigned ZERO = STI.isABI_N64() ? Mips::ZERO_64 : Mips::ZERO;
-  unsigned ADDu = STI.isABI_N64() ? Mips::DADDu : Mips::ADDu;
+  MipsABIInfo ABI = STI.getABI();
+  unsigned SP = ABI.GetStackPtr();
+  unsigned FP = ABI.GetFramePtr();
+  unsigned ZERO = ABI.GetNullPtr();
+  unsigned ADDu = ABI.GetPtrAdduOp();
 
   // if framepointer enabled, restore the stack pointer.
   if (hasFP(MF)) {
@@ -539,8 +553,8 @@ void MipsSEFrameLowering::emitEpilogue(MachineFunction &MF,
   }
 
   if (MipsFI->callsEhReturn()) {
-    const TargetRegisterClass *RC = STI.isABI_N64() ?
-        &Mips::GPR64RegClass : &Mips::GPR32RegClass;
+    const TargetRegisterClass *RC =
+        ABI.ArePtrs64bit() ? &Mips::GPR64RegClass : &Mips::GPR32RegClass;
 
     // Find first instruction that restores a callee-saved register.
     MachineBasicBlock::iterator I = MBBI;
@@ -549,8 +563,8 @@ void MipsSEFrameLowering::emitEpilogue(MachineFunction &MF,
 
     // Insert instructions that restore eh data registers.
     for (int J = 0; J < 4; ++J) {
-      TII.loadRegFromStackSlot(MBB, I, ehDataReg(J), MipsFI->getEhDataRegFI(J),
-                               RC, &RegInfo);
+      TII.loadRegFromStackSlot(MBB, I, ABI.GetEhDataReg(J),
+                               MipsFI->getEhDataRegFI(J), RC, &RegInfo);
     }
   }
 
@@ -607,16 +621,29 @@ MipsSEFrameLowering::hasReservedCallFrame(const MachineFunction &MF) const {
     !MFI->hasVarSizedObjects();
 }
 
-void MipsSEFrameLowering::
-processFunctionBeforeCalleeSavedScan(MachineFunction &MF,
-                                     RegScavenger *RS) const {
-  MachineRegisterInfo &MRI = MF.getRegInfo();
+/// Mark \p Reg and all registers aliasing it in the bitset.
+static void setAliasRegs(MachineFunction &MF, BitVector &SavedRegs,
+                         unsigned Reg) {
+  const TargetRegisterInfo *TRI = MF.getSubtarget().getRegisterInfo();
+  for (MCRegAliasIterator AI(Reg, TRI, true); AI.isValid(); ++AI)
+    SavedRegs.set(*AI);
+}
+
+void MipsSEFrameLowering::determineCalleeSaves(MachineFunction &MF,
+                                               BitVector &SavedRegs,
+                                               RegScavenger *RS) const {
+  TargetFrameLowering::determineCalleeSaves(MF, SavedRegs, RS);
   MipsFunctionInfo *MipsFI = MF.getInfo<MipsFunctionInfo>();
-  unsigned FP = STI.isABI_N64() ? Mips::FP_64 : Mips::FP;
+  MipsABIInfo ABI = STI.getABI();
+  unsigned FP = ABI.GetFramePtr();
+  unsigned BP = ABI.IsN64() ? Mips::S7_64 : Mips::S7;
 
   // Mark $fp as used if function has dedicated frame pointer.
   if (hasFP(MF))
-    MRI.setPhysRegUsed(FP);
+    setAliasRegs(MF, SavedRegs, FP);
+  // Mark $s7 as used if function has dedicated base pointer.
+  if (hasBP(MF))
+    setAliasRegs(MF, SavedRegs, BP);
 
   // Create spill slots for eh data registers if function calls eh_return.
   if (MipsFI->callsEhReturn())
@@ -641,8 +668,8 @@ processFunctionBeforeCalleeSavedScan(MachineFunction &MF,
   if (isInt<16>(MaxSPOffset))
     return;
 
-  const TargetRegisterClass *RC = STI.isABI_N64() ?
-    &Mips::GPR64RegClass : &Mips::GPR32RegClass;
+  const TargetRegisterClass *RC =
+      ABI.ArePtrs64bit() ? &Mips::GPR64RegClass : &Mips::GPR32RegClass;
   int FI = MF.getFrameInfo()->CreateStackObject(RC->getSize(),
                                                 RC->getAlignment(), false);
   RS->addScavengingFrameIndex(FI);
diff --git a/lib/Target/Mips/MipsSEFrameLowering.h b/lib/Target/Mips/MipsSEFrameLowering.h
index 22448a433b7..9cb32e6c782 100644
--- a/lib/Target/Mips/MipsSEFrameLowering.h
+++ b/lib/Target/Mips/MipsSEFrameLowering.h
@@ -24,7 +24,7 @@ class MipsSEFrameLowering : public MipsFrameLowering {
 
   /// emitProlog/emitEpilog - These methods insert prolog and epilog code into
   /// the function.
-  void emitPrologue(MachineFunction &MF) const override;
+  void emitPrologue(MachineFunction &MF, MachineBasicBlock &MBB) const override;
   void emitEpilogue(MachineFunction &MF, MachineBasicBlock &MBB) const override;
 
   bool spillCalleeSavedRegisters(MachineBasicBlock &MBB,
@@ -34,8 +34,8 @@ class MipsSEFrameLowering : public MipsFrameLowering {
 
   bool hasReservedCallFrame(const MachineFunction &MF) const override;
 
-  void processFunctionBeforeCalleeSavedScan(MachineFunction &MF,
-                                            RegScavenger *RS) const override;
+  void determineCalleeSaves(MachineFunction &MF, BitVector &SavedRegs,
+                            RegScavenger *RS) const override;
   unsigned ehDataReg(unsigned I) const;
 };
 
diff --git a/lib/Target/Mips/MipsSEISelDAGToDAG.cpp b/lib/Target/Mips/MipsSEISelDAGToDAG.cpp
index 6daa632b7a3..cb46d731da2 100644
--- a/lib/Target/Mips/MipsSEISelDAGToDAG.cpp
+++ b/lib/Target/Mips/MipsSEISelDAGToDAG.cpp
@@ -253,9 +253,10 @@ SDNode *MipsSEDAGToDAGISel::selectAddESubE(unsigned MOp, SDValue InFlag,
     // that SLTu64 produces an i32. We need to fix this in the long run but for
     // now, just make the DAG type-correct by asserting the upper bits are zero.
     Carry = CurDAG->getMachineNode(Mips::SUBREG_TO_REG, DL, VT,
-                                   CurDAG->getTargetConstant(0, VT),
+                                   CurDAG->getTargetConstant(0, DL, VT),
                                    SDValue(Carry, 0),
-                                   CurDAG->getTargetConstant(Mips::sub_32, VT));
+                                   CurDAG->getTargetConstant(Mips::sub_32, DL,
+                                                             VT));
   }
 
   // Generate a second addition only if we know that RHS is not a
@@ -276,7 +277,7 @@ bool MipsSEDAGToDAGISel::selectAddrFrameIndex(SDValue Addr, SDValue &Base,
     EVT ValTy = Addr.getValueType();
 
     Base   = CurDAG->getTargetFrameIndex(FIN->getIndex(), ValTy);
-    Offset = CurDAG->getTargetConstant(0, ValTy);
+    Offset = CurDAG->getTargetConstant(0, SDLoc(Addr), ValTy);
     return true;
   }
   return false;
@@ -298,7 +299,8 @@ bool MipsSEDAGToDAGISel::selectAddrFrameIndexOffset(SDValue Addr, SDValue &Base,
       else
         Base = Addr.getOperand(0);
 
-      Offset = CurDAG->getTargetConstant(CN->getZExtValue(), ValTy);
+      Offset = CurDAG->getTargetConstant(CN->getZExtValue(), SDLoc(Addr),
+                                         ValTy);
       return true;
     }
   }
@@ -372,7 +374,7 @@ bool MipsSEDAGToDAGISel::selectAddrRegReg(SDValue Addr, SDValue &Base,
 bool MipsSEDAGToDAGISel::selectAddrDefault(SDValue Addr, SDValue &Base,
                                            SDValue &Offset) const {
   Base = Addr;
-  Offset = CurDAG->getTargetConstant(0, Addr.getValueType());
+  Offset = CurDAG->getTargetConstant(0, SDLoc(Addr), Addr.getValueType());
   return true;
 }
 
@@ -471,7 +473,8 @@ bool MipsSEDAGToDAGISel::selectIntAddrMSA(SDValue Addr, SDValue &Base,
 // Returns true and sets Imm if:
 // * MSA is enabled
 // * N is a ISD::BUILD_VECTOR representing a constant splat
-bool MipsSEDAGToDAGISel::selectVSplat(SDNode *N, APInt &Imm) const {
+bool MipsSEDAGToDAGISel::selectVSplat(SDNode *N, APInt &Imm,
+                                      unsigned MinSizeInBits) const {
   if (!Subtarget->hasMSA())
     return false;
 
@@ -484,9 +487,8 @@ bool MipsSEDAGToDAGISel::selectVSplat(SDNode *N, APInt &Imm) const {
   unsigned SplatBitSize;
   bool HasAnyUndefs;
 
-  if (!Node->isConstantSplat(SplatValue, SplatUndef, SplatBitSize,
-                             HasAnyUndefs, 8,
-                             !Subtarget->isLittle()))
+  if (!Node->isConstantSplat(SplatValue, SplatUndef, SplatBitSize, HasAnyUndefs,
+                             MinSizeInBits, !Subtarget->isLittle()))
     return false;
 
   Imm = SplatValue;
@@ -519,11 +521,12 @@ selectVSplatCommon(SDValue N, SDValue &Imm, bool Signed,
   if (N->getOpcode() == ISD::BITCAST)
     N = N->getOperand(0);
 
-  if (selectVSplat (N.getNode(), ImmValue) &&
+  if (selectVSplat(N.getNode(), ImmValue, EltTy.getSizeInBits()) &&
       ImmValue.getBitWidth() == EltTy.getSizeInBits()) {
+
     if (( Signed && ImmValue.isSignedIntN(ImmBitSize)) ||
         (!Signed && ImmValue.isIntN(ImmBitSize))) {
-      Imm = CurDAG->getTargetConstant(ImmValue, EltTy);
+      Imm = CurDAG->getTargetConstant(ImmValue, SDLoc(N), EltTy);
       return true;
     }
   }
@@ -594,12 +597,12 @@ bool MipsSEDAGToDAGISel::selectVSplatUimmPow2(SDValue N, SDValue &Imm) const {
   if (N->getOpcode() == ISD::BITCAST)
     N = N->getOperand(0);
 
-  if (selectVSplat (N.getNode(), ImmValue) &&
+  if (selectVSplat(N.getNode(), ImmValue, EltTy.getSizeInBits()) &&
       ImmValue.getBitWidth() == EltTy.getSizeInBits()) {
     int32_t Log2 = ImmValue.exactLogBase2();
 
     if (Log2 != -1) {
-      Imm = CurDAG->getTargetConstant(Log2, EltTy);
+      Imm = CurDAG->getTargetConstant(Log2, SDLoc(N), EltTy);
       return true;
     }
   }
@@ -625,14 +628,15 @@ bool MipsSEDAGToDAGISel::selectVSplatMaskL(SDValue N, SDValue &Imm) const {
   if (N->getOpcode() == ISD::BITCAST)
     N = N->getOperand(0);
 
-  if (selectVSplat(N.getNode(), ImmValue) &&
+  if (selectVSplat(N.getNode(), ImmValue, EltTy.getSizeInBits()) &&
       ImmValue.getBitWidth() == EltTy.getSizeInBits()) {
     // Extract the run of set bits starting with bit zero from the bitwise
     // inverse of ImmValue, and test that the inverse of this is the same
     // as the original value.
     if (ImmValue == ~(~ImmValue & ~(~ImmValue + 1))) {
 
-      Imm = CurDAG->getTargetConstant(ImmValue.countPopulation(), EltTy);
+      Imm = CurDAG->getTargetConstant(ImmValue.countPopulation(), SDLoc(N),
+                                      EltTy);
       return true;
     }
   }
@@ -658,12 +662,13 @@ bool MipsSEDAGToDAGISel::selectVSplatMaskR(SDValue N, SDValue &Imm) const {
   if (N->getOpcode() == ISD::BITCAST)
     N = N->getOperand(0);
 
-  if (selectVSplat(N.getNode(), ImmValue) &&
+  if (selectVSplat(N.getNode(), ImmValue, EltTy.getSizeInBits()) &&
       ImmValue.getBitWidth() == EltTy.getSizeInBits()) {
     // Extract the run of set bits starting with bit zero, and test that the
     // result is the same as the original value
     if (ImmValue == (ImmValue & ~(ImmValue + 1))) {
-      Imm = CurDAG->getTargetConstant(ImmValue.countPopulation(), EltTy);
+      Imm = CurDAG->getTargetConstant(ImmValue.countPopulation(), SDLoc(N),
+                                      EltTy);
       return true;
     }
   }
@@ -679,12 +684,12 @@ bool MipsSEDAGToDAGISel::selectVSplatUimmInvPow2(SDValue N,
   if (N->getOpcode() == ISD::BITCAST)
     N = N->getOperand(0);
 
-  if (selectVSplat(N.getNode(), ImmValue) &&
+  if (selectVSplat(N.getNode(), ImmValue, EltTy.getSizeInBits()) &&
       ImmValue.getBitWidth() == EltTy.getSizeInBits()) {
     int32_t Log2 = (~ImmValue).exactLogBase2();
 
     if (Log2 != -1) {
-      Imm = CurDAG->getTargetConstant(Log2, EltTy);
+      Imm = CurDAG->getTargetConstant(Log2, SDLoc(N), EltTy);
       return true;
     }
   }
@@ -762,7 +767,7 @@ std::pair<bool, SDNode*> MipsSEDAGToDAGISel::selectNode(SDNode *Node) {
     SDLoc DL(CN);
     SDNode *RegOpnd;
     SDValue ImmOpnd = CurDAG->getTargetConstant(SignExtend64<16>(Inst->ImmOpnd),
-                                                MVT::i64);
+                                                DL, MVT::i64);
 
     // The first instruction can be a LUi which is different from other
     // instructions (ADDiu, ORI and SLL) in that it does not have a register
@@ -777,7 +782,7 @@ std::pair<bool, SDNode*> MipsSEDAGToDAGISel::selectNode(SDNode *Node) {
 
     // The remaining instructions in the sequence are handled here.
     for (++Inst; Inst != Seq.end(); ++Inst) {
-      ImmOpnd = CurDAG->getTargetConstant(SignExtend64<16>(Inst->ImmOpnd),
+      ImmOpnd = CurDAG->getTargetConstant(SignExtend64<16>(Inst->ImmOpnd), DL,
                                           MVT::i64);
       RegOpnd = CurDAG->getMachineNode(Inst->Opc, DL, MVT::i64,
                                        SDValue(RegOpnd, 0), ImmOpnd);
@@ -836,7 +841,7 @@ std::pair<bool, SDNode*> MipsSEDAGToDAGISel::selectNode(SDNode *Node) {
   }
 
   case MipsISD::ThreadPointer: {
-    EVT PtrVT = getTargetLowering()->getPointerTy();
+    EVT PtrVT = getTargetLowering()->getPointerTy(CurDAG->getDataLayout());
     unsigned RdhwrOpc, DestReg;
 
     if (PtrVT == MVT::i32) {
@@ -848,7 +853,7 @@ std::pair<bool, SDNode*> MipsSEDAGToDAGISel::selectNode(SDNode *Node) {
     }
 
     SDNode *Rdhwr =
-      CurDAG->getMachineNode(RdhwrOpc, SDLoc(Node),
+      CurDAG->getMachineNode(RdhwrOpc, DL,
                              Node->getValueType(0),
                              CurDAG->getRegister(Mips::HWR29, MVT::i32));
     SDValue Chain = CurDAG->getCopyToReg(CurDAG->getEntryNode(), DL, DestReg,
@@ -911,10 +916,10 @@ std::pair<bool, SDNode*> MipsSEDAGToDAGISel::selectNode(SDNode *Node) {
     if (!SplatValue.isSignedIntN(10))
       return std::make_pair(false, nullptr);
 
-    SDValue Imm = CurDAG->getTargetConstant(SplatValue,
+    SDValue Imm = CurDAG->getTargetConstant(SplatValue, DL,
                                             ViaVecTy.getVectorElementType());
 
-    SDNode *Res = CurDAG->getMachineNode(LdiOp, SDLoc(Node), ViaVecTy, Imm);
+    SDNode *Res = CurDAG->getMachineNode(LdiOp, DL, ViaVecTy, Imm);
 
     if (ResVecTy != ViaVecTy) {
       // If LdiOp is writing to a different register class to ResVecTy, then
@@ -924,9 +929,9 @@ std::pair<bool, SDNode*> MipsSEDAGToDAGISel::selectNode(SDNode *Node) {
       const TargetLowering *TLI = getTargetLowering();
       MVT ResVecTySimple = ResVecTy.getSimpleVT();
       const TargetRegisterClass *RC = TLI->getRegClassFor(ResVecTySimple);
-      Res = CurDAG->getMachineNode(Mips::COPY_TO_REGCLASS, SDLoc(Node),
+      Res = CurDAG->getMachineNode(Mips::COPY_TO_REGCLASS, DL,
                                    ResVecTy, SDValue(Res, 0),
-                                   CurDAG->getTargetConstant(RC->getID(),
+                                   CurDAG->getTargetConstant(RC->getID(), DL,
                                                              MVT::i32));
     }
 
@@ -949,7 +954,7 @@ SelectInlineAsmMemoryOperand(const SDValue &Op, unsigned ConstraintID,
   // All memory constraints can at least accept raw pointers.
   case InlineAsm::Constraint_i:
     OutOps.push_back(Op);
-    OutOps.push_back(CurDAG->getTargetConstant(0, MVT::i32));
+    OutOps.push_back(CurDAG->getTargetConstant(0, SDLoc(Op), MVT::i32));
     return false;
   case InlineAsm::Constraint_m:
     if (selectAddrRegImm16(Op, Base, Offset)) {
@@ -958,7 +963,7 @@ SelectInlineAsmMemoryOperand(const SDValue &Op, unsigned ConstraintID,
       return false;
     }
     OutOps.push_back(Op);
-    OutOps.push_back(CurDAG->getTargetConstant(0, MVT::i32));
+    OutOps.push_back(CurDAG->getTargetConstant(0, SDLoc(Op), MVT::i32));
     return false;
   case InlineAsm::Constraint_R:
     // The 'R' constraint is supposed to be much more complicated than this.
@@ -972,7 +977,7 @@ SelectInlineAsmMemoryOperand(const SDValue &Op, unsigned ConstraintID,
       return false;
     }
     OutOps.push_back(Op);
-    OutOps.push_back(CurDAG->getTargetConstant(0, MVT::i32));
+    OutOps.push_back(CurDAG->getTargetConstant(0, SDLoc(Op), MVT::i32));
     return false;
   case InlineAsm::Constraint_ZC:
     // ZC matches whatever the pref, ll, and sc instructions can handle for the
@@ -999,7 +1004,7 @@ SelectInlineAsmMemoryOperand(const SDValue &Op, unsigned ConstraintID,
     }
     // In all cases, 0-bit offsets are acceptable.
     OutOps.push_back(Op);
-    OutOps.push_back(CurDAG->getTargetConstant(0, MVT::i32));
+    OutOps.push_back(CurDAG->getTargetConstant(0, SDLoc(Op), MVT::i32));
     return false;
   }
   return true;
diff --git a/lib/Target/Mips/MipsSEISelDAGToDAG.h b/lib/Target/Mips/MipsSEISelDAGToDAG.h
index a11fcf4e2e0..a894034020e 100644
--- a/lib/Target/Mips/MipsSEISelDAGToDAG.h
+++ b/lib/Target/Mips/MipsSEISelDAGToDAG.h
@@ -78,7 +78,8 @@ class MipsSEDAGToDAGISel : public MipsDAGToDAGISel {
                         SDValue &Offset) const override;
 
   /// \brief Select constant vector splats.
-  bool selectVSplat(SDNode *N, APInt &Imm) const override;
+  bool selectVSplat(SDNode *N, APInt &Imm,
+                    unsigned MinSizeInBits) const override;
   /// \brief Select constant vector splats whose value fits in a given integer.
   bool selectVSplatCommon(SDValue N, SDValue &Imm, bool Signed,
                                   unsigned ImmBitSize) const;
diff --git a/lib/Target/Mips/MipsSEISelLowering.cpp b/lib/Target/Mips/MipsSEISelLowering.cpp
index a3a1726cfc6..b319fd07884 100644
--- a/lib/Target/Mips/MipsSEISelLowering.cpp
+++ b/lib/Target/Mips/MipsSEISelLowering.cpp
@@ -99,7 +99,7 @@ MipsSETargetLowering::MipsSETargetLowering(const MipsTargetMachine &TM,
     setTargetDAGCombine(ISD::XOR);
   }
 
-  if (!Subtarget.abiUsesSoftFloat()) {
+  if (!Subtarget.useSoftFloat()) {
     addRegisterClass(MVT::f32, &Mips::FGR32RegClass);
 
     // When dealing with single precision only, use libcalls
@@ -800,7 +800,7 @@ static SDValue genConstMult(SDValue X, uint64_t C, SDLoc DL, EVT VT,
 
   // Return 0.
   if (C == 0)
-    return DAG.getConstant(0, VT);
+    return DAG.getConstant(0, DL, VT);
 
   // Return x.
   if (C == 1)
@@ -809,7 +809,7 @@ static SDValue genConstMult(SDValue X, uint64_t C, SDLoc DL, EVT VT,
   // If c is power of 2, return (shl x, log2(c)).
   if (isPowerOf2_64(C))
     return DAG.getNode(ISD::SHL, DL, VT, X,
-                       DAG.getConstant(Log2_64(C), ShiftTy));
+                       DAG.getConstant(Log2_64(C), DL, ShiftTy));
 
   unsigned Log2Ceil = Log2_64_Ceil(C);
   uint64_t Floor = 1LL << Log2_64(C);
@@ -838,8 +838,9 @@ static SDValue performMULCombine(SDNode *N, SelectionDAG &DAG,
 
   if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(N->getOperand(1)))
     if (!VT.isVector())
-      return genConstMult(N->getOperand(0), C->getZExtValue(), SDLoc(N),
-                          VT, TL->getScalarShiftAmountTy(VT), DAG);
+      return genConstMult(N->getOperand(0), C->getZExtValue(), SDLoc(N), VT,
+                          TL->getScalarShiftAmountTy(DAG.getDataLayout(), VT),
+                          DAG);
 
   return SDValue(N, 0);
 }
@@ -864,8 +865,9 @@ static SDValue performDSPShiftCombine(unsigned Opc, SDNode *N, EVT Ty,
       (SplatValue.getZExtValue() >= EltSize))
     return SDValue();
 
-  return DAG.getNode(Opc, SDLoc(N), Ty, N->getOperand(0),
-                     DAG.getConstant(SplatValue.getZExtValue(), MVT::i32));
+  SDLoc DL(N);
+  return DAG.getNode(Opc, DL, Ty, N->getOperand(0),
+                     DAG.getConstant(SplatValue.getZExtValue(), DL, MVT::i32));
 }
 
 static SDValue performSHLCombine(SDNode *N, SelectionDAG &DAG,
@@ -1145,16 +1147,22 @@ MipsSETargetLowering::EmitInstrWithCustomInserter(MachineInstr *MI,
   case Mips::INSERT_FD_PSEUDO:
     return emitINSERT_FD(MI, BB);
   case Mips::INSERT_B_VIDX_PSEUDO:
+  case Mips::INSERT_B_VIDX64_PSEUDO:
     return emitINSERT_DF_VIDX(MI, BB, 1, false);
   case Mips::INSERT_H_VIDX_PSEUDO:
+  case Mips::INSERT_H_VIDX64_PSEUDO:
     return emitINSERT_DF_VIDX(MI, BB, 2, false);
   case Mips::INSERT_W_VIDX_PSEUDO:
+  case Mips::INSERT_W_VIDX64_PSEUDO:
     return emitINSERT_DF_VIDX(MI, BB, 4, false);
   case Mips::INSERT_D_VIDX_PSEUDO:
+  case Mips::INSERT_D_VIDX64_PSEUDO:
     return emitINSERT_DF_VIDX(MI, BB, 8, false);
   case Mips::INSERT_FW_VIDX_PSEUDO:
+  case Mips::INSERT_FW_VIDX64_PSEUDO:
     return emitINSERT_DF_VIDX(MI, BB, 4, true);
   case Mips::INSERT_FD_VIDX_PSEUDO:
+  case Mips::INSERT_FD_VIDX64_PSEUDO:
     return emitINSERT_DF_VIDX(MI, BB, 8, true);
   case Mips::FILL_FW_PSEUDO:
     return emitFILL_FW(MI, BB);
@@ -1212,7 +1220,7 @@ SDValue MipsSETargetLowering::lowerLOAD(SDValue Op, SelectionDAG &DAG) const {
                            Nd.getAlignment());
 
   // i32 load from higher address.
-  Ptr = DAG.getNode(ISD::ADD, DL, PtrVT, Ptr, DAG.getConstant(4, PtrVT));
+  Ptr = DAG.getNode(ISD::ADD, DL, PtrVT, Ptr, DAG.getConstant(4, DL, PtrVT));
   SDValue Hi = DAG.getLoad(MVT::i32, DL, Lo.getValue(1), Ptr,
                            MachinePointerInfo(), Nd.isVolatile(),
                            Nd.isNonTemporal(), Nd.isInvariant(),
@@ -1237,9 +1245,9 @@ SDValue MipsSETargetLowering::lowerSTORE(SDValue Op, SelectionDAG &DAG) const {
   SDValue Val = Nd.getValue(), Ptr = Nd.getBasePtr(), Chain = Nd.getChain();
   EVT PtrVT = Ptr.getValueType();
   SDValue Lo = DAG.getNode(MipsISD::ExtractElementF64, DL, MVT::i32,
-                           Val, DAG.getConstant(0, MVT::i32));
+                           Val, DAG.getConstant(0, DL, MVT::i32));
   SDValue Hi = DAG.getNode(MipsISD::ExtractElementF64, DL, MVT::i32,
-                           Val, DAG.getConstant(1, MVT::i32));
+                           Val, DAG.getConstant(1, DL, MVT::i32));
 
   if (!Subtarget.isLittle())
     std::swap(Lo, Hi);
@@ -1250,7 +1258,7 @@ SDValue MipsSETargetLowering::lowerSTORE(SDValue Op, SelectionDAG &DAG) const {
                        Nd.getAAInfo());
 
   // i32 store to higher address.
-  Ptr = DAG.getNode(ISD::ADD, DL, PtrVT, Ptr, DAG.getConstant(4, PtrVT));
+  Ptr = DAG.getNode(ISD::ADD, DL, PtrVT, Ptr, DAG.getConstant(4, DL, PtrVT));
   return DAG.getStore(Chain, DL, Hi, Ptr, MachinePointerInfo(),
                       Nd.isVolatile(), Nd.isNonTemporal(),
                       std::min(Nd.getAlignment(), 4U), Nd.getAAInfo());
@@ -1283,9 +1291,9 @@ SDValue MipsSETargetLowering::lowerMulDiv(SDValue Op, unsigned NewOpc,
 
 static SDValue initAccumulator(SDValue In, SDLoc DL, SelectionDAG &DAG) {
   SDValue InLo = DAG.getNode(ISD::EXTRACT_ELEMENT, DL, MVT::i32, In,
-                             DAG.getConstant(0, MVT::i32));
+                             DAG.getConstant(0, DL, MVT::i32));
   SDValue InHi = DAG.getNode(ISD::EXTRACT_ELEMENT, DL, MVT::i32, In,
-                             DAG.getConstant(1, MVT::i32));
+                             DAG.getConstant(1, DL, MVT::i32));
   return DAG.getNode(MipsISD::MTLOHI, DL, MVT::Untyped, InLo, InHi);
 }
 
@@ -1381,7 +1389,7 @@ static SDValue lowerMSASplatZExt(SDValue Op, unsigned OpNr, SelectionDAG &DAG) {
   SDValue LaneB = Op->getOperand(2);
 
   if (ResVecTy == MVT::v2i64) {
-    LaneA = DAG.getConstant(0, MVT::i32);
+    LaneA = DAG.getConstant(0, DL, MVT::i32);
     ViaVecTy = MVT::v4i32;
   } else
     LaneA = LaneB;
@@ -1399,7 +1407,8 @@ static SDValue lowerMSASplatZExt(SDValue Op, unsigned OpNr, SelectionDAG &DAG) {
 }
 
 static SDValue lowerMSASplatImm(SDValue Op, unsigned ImmOp, SelectionDAG &DAG) {
-  return DAG.getConstant(Op->getConstantOperandVal(ImmOp), Op->getValueType(0));
+  return DAG.getConstant(Op->getConstantOperandVal(ImmOp), SDLoc(Op),
+                         Op->getValueType(0));
 }
 
 static SDValue getBuildVectorSplat(EVT VecTy, SDValue SplatValue,
@@ -1415,7 +1424,7 @@ static SDValue getBuildVectorSplat(EVT VecTy, SDValue SplatValue,
 
     SplatValueA = DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, SplatValue);
     SplatValueB = DAG.getNode(ISD::SRL, DL, MVT::i64, SplatValue,
-                              DAG.getConstant(32, MVT::i32));
+                              DAG.getConstant(32, DL, MVT::i32));
     SplatValueB = DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, SplatValueB);
   }
 
@@ -1451,8 +1460,9 @@ static SDValue lowerMSABinaryBitImmIntr(SDValue Op, SelectionDAG &DAG,
     if (ConstantSDNode *CImm = dyn_cast<ConstantSDNode>(Imm)) {
       APInt BitImm = APInt(64, 1) << CImm->getAPIntValue();
 
-      SDValue BitImmHiOp = DAG.getConstant(BitImm.lshr(32).trunc(32), MVT::i32);
-      SDValue BitImmLoOp = DAG.getConstant(BitImm.trunc(32), MVT::i32);
+      SDValue BitImmHiOp = DAG.getConstant(BitImm.lshr(32).trunc(32), DL,
+                                           MVT::i32);
+      SDValue BitImmLoOp = DAG.getConstant(BitImm.trunc(32), DL, MVT::i32);
 
       if (BigEndian)
         std::swap(BitImmLoOp, BitImmHiOp);
@@ -1474,8 +1484,8 @@ static SDValue lowerMSABinaryBitImmIntr(SDValue Op, SelectionDAG &DAG,
 
     Exp2Imm = getBuildVectorSplat(VecTy, Imm, BigEndian, DAG);
 
-    Exp2Imm =
-        DAG.getNode(ISD::SHL, DL, VecTy, DAG.getConstant(1, VecTy), Exp2Imm);
+    Exp2Imm = DAG.getNode(ISD::SHL, DL, VecTy, DAG.getConstant(1, DL, VecTy),
+                          Exp2Imm);
   }
 
   return DAG.getNode(Opc, DL, VecTy, Op->getOperand(1), Exp2Imm);
@@ -1484,7 +1494,7 @@ static SDValue lowerMSABinaryBitImmIntr(SDValue Op, SelectionDAG &DAG,
 static SDValue lowerMSABitClear(SDValue Op, SelectionDAG &DAG) {
   EVT ResTy = Op->getValueType(0);
   SDLoc DL(Op);
-  SDValue One = DAG.getConstant(1, ResTy);
+  SDValue One = DAG.getConstant(1, DL, ResTy);
   SDValue Bit = DAG.getNode(ISD::SHL, DL, ResTy, One, Op->getOperand(2));
 
   return DAG.getNode(ISD::AND, DL, ResTy, Op->getOperand(1),
@@ -1496,7 +1506,7 @@ static SDValue lowerMSABitClearImm(SDValue Op, SelectionDAG &DAG) {
   EVT ResTy = Op->getValueType(0);
   APInt BitImm = APInt(ResTy.getVectorElementType().getSizeInBits(), 1)
                  << cast<ConstantSDNode>(Op->getOperand(2))->getAPIntValue();
-  SDValue BitMask = DAG.getConstant(~BitImm, ResTy);
+  SDValue BitMask = DAG.getConstant(~BitImm, DL, ResTy);
 
   return DAG.getNode(ISD::AND, DL, ResTy, Op->getOperand(1), BitMask);
 }
@@ -1508,10 +1518,6 @@ SDValue MipsSETargetLowering::lowerINTRINSIC_WO_CHAIN(SDValue Op,
   switch (cast<ConstantSDNode>(Op->getOperand(0))->getZExtValue()) {
   default:
     return SDValue();
-  // @LOCALMOD-BEGIN
-  case Intrinsic::nacl_read_tp:
-    return GetNaClThreadPointer(DAG, SDLoc(Op));
-  // @LOCALMOD-END
   case Intrinsic::mips_shilo:
     return lowerDSPIntr(Op, DAG, MipsISD::SHILO);
   case Intrinsic::mips_dpau_h_qbl:
@@ -1582,8 +1588,8 @@ SDValue MipsSETargetLowering::lowerINTRINSIC_WO_CHAIN(SDValue Op,
     APInt Mask = APInt::getHighBitsSet(EltTy.getSizeInBits(),
                                        Op->getConstantOperandVal(3));
     return DAG.getNode(ISD::VSELECT, DL, VecTy,
-                       DAG.getConstant(Mask, VecTy, true), Op->getOperand(2),
-                       Op->getOperand(1));
+                       DAG.getConstant(Mask, DL, VecTy, true),
+                       Op->getOperand(2), Op->getOperand(1));
   }
   case Intrinsic::mips_binsri_b:
   case Intrinsic::mips_binsri_h:
@@ -1595,8 +1601,8 @@ SDValue MipsSETargetLowering::lowerINTRINSIC_WO_CHAIN(SDValue Op,
     APInt Mask = APInt::getLowBitsSet(EltTy.getSizeInBits(),
                                       Op->getConstantOperandVal(3));
     return DAG.getNode(ISD::VSELECT, DL, VecTy,
-                       DAG.getConstant(Mask, VecTy, true), Op->getOperand(2),
-                       Op->getOperand(1));
+                       DAG.getConstant(Mask, DL, VecTy, true),
+                       Op->getOperand(2), Op->getOperand(1));
   }
   case Intrinsic::mips_bmnz_v:
     return DAG.getNode(ISD::VSELECT, DL, Op->getValueType(0), Op->getOperand(3),
@@ -1617,7 +1623,7 @@ SDValue MipsSETargetLowering::lowerINTRINSIC_WO_CHAIN(SDValue Op,
   case Intrinsic::mips_bneg_w:
   case Intrinsic::mips_bneg_d: {
     EVT VecTy = Op->getValueType(0);
-    SDValue One = DAG.getConstant(1, VecTy);
+    SDValue One = DAG.getConstant(1, DL, VecTy);
 
     return DAG.getNode(ISD::XOR, DL, VecTy, Op->getOperand(1),
                        DAG.getNode(ISD::SHL, DL, VecTy, One,
@@ -1653,7 +1659,7 @@ SDValue MipsSETargetLowering::lowerINTRINSIC_WO_CHAIN(SDValue Op,
   case Intrinsic::mips_bset_w:
   case Intrinsic::mips_bset_d: {
     EVT VecTy = Op->getValueType(0);
-    SDValue One = DAG.getConstant(1, VecTy);
+    SDValue One = DAG.getConstant(1, DL, VecTy);
 
     return DAG.getNode(ISD::OR, DL, VecTy, Op->getOperand(1),
                        DAG.getNode(ISD::SHL, DL, VecTy, One,
@@ -1927,7 +1933,7 @@ SDValue MipsSETargetLowering::lowerINTRINSIC_WO_CHAIN(SDValue Op,
   case Intrinsic::mips_insve_d:
     return DAG.getNode(MipsISD::INSVE, DL, Op->getValueType(0),
                        Op->getOperand(1), Op->getOperand(2), Op->getOperand(3),
-                       DAG.getConstant(0, MVT::i32));
+                       DAG.getConstant(0, DL, MVT::i32));
   case Intrinsic::mips_ldi_b:
   case Intrinsic::mips_ldi_h:
   case Intrinsic::mips_ldi_w:
@@ -2367,7 +2373,7 @@ SDValue MipsSETargetLowering::lowerBUILD_VECTOR(SDValue Op,
     }
 
     // SelectionDAG::getConstant will promote SplatValue appropriately.
-    SDValue Result = DAG.getConstant(SplatValue, ViaVecTy);
+    SDValue Result = DAG.getConstant(SplatValue, DL, ViaVecTy);
 
     // Bitcast to the type we originally wanted
     if (ViaVecTy != ResTy)
@@ -2389,7 +2395,7 @@ SDValue MipsSETargetLowering::lowerBUILD_VECTOR(SDValue Op,
     for (unsigned i = 0; i < NumElts; ++i) {
       Vector = DAG.getNode(ISD::INSERT_VECTOR_ELT, DL, ResTy, Vector,
                            Node->getOperand(i),
-                           DAG.getConstant(i, MVT::i32));
+                           DAG.getConstant(i, DL, MVT::i32));
     }
     return Vector;
   }
@@ -2405,7 +2411,7 @@ SDValue MipsSETargetLowering::lowerBUILD_VECTOR(SDValue Op,
 // It is therefore possible to lower into SHF when the mask takes the form:
 //   <a, b, c, d, a+4, b+4, c+4, d+4, a+8, b+8, c+8, d+8, ...>
 // When undef's appear they are treated as if they were whatever value is
-// necessary in order to fit the above form.
+// necessary in order to fit the above forms.
 //
 // For example:
 //   %2 = shufflevector <8 x i16> %0, <8 x i16> undef,
@@ -2459,181 +2465,331 @@ static SDValue lowerVECTOR_SHUFFLE_SHF(SDValue Op, EVT ResTy,
     Imm |= Idx & 0x3;
   }
 
-  return DAG.getNode(MipsISD::SHF, SDLoc(Op), ResTy,
-                     DAG.getConstant(Imm, MVT::i32), Op->getOperand(0));
+  SDLoc DL(Op);
+  return DAG.getNode(MipsISD::SHF, DL, ResTy,
+                     DAG.getConstant(Imm, DL, MVT::i32), Op->getOperand(0));
+}
+
+/// Determine whether a range fits a regular pattern of values.
+/// This function accounts for the possibility of jumping over the End iterator.
+template <typename ValType>
+static bool
+fitsRegularPattern(typename SmallVectorImpl<ValType>::const_iterator Begin,
+                   unsigned CheckStride,
+                   typename SmallVectorImpl<ValType>::const_iterator End,
+                   ValType ExpectedIndex, unsigned ExpectedIndexStride) {
+  auto &I = Begin;
+
+  while (I != End) {
+    if (*I != -1 && *I != ExpectedIndex)
+      return false;
+    ExpectedIndex += ExpectedIndexStride;
+
+    // Incrementing past End is undefined behaviour so we must increment one
+    // step at a time and check for End at each step.
+    for (unsigned n = 0; n < CheckStride && I != End; ++n, ++I)
+      ; // Empty loop body.
+  }
+  return true;
+}
+
+// Determine whether VECTOR_SHUFFLE is a SPLATI.
+//
+// It is a SPLATI when the mask is:
+//   <x, x, x, ...>
+// where x is any valid index.
+//
+// When undef's appear in the mask they are treated as if they were whatever
+// value is necessary in order to fit the above form.
+static bool isVECTOR_SHUFFLE_SPLATI(SDValue Op, EVT ResTy,
+                                    SmallVector<int, 16> Indices,
+                                    SelectionDAG &DAG) {
+  assert((Indices.size() % 2) == 0);
+
+  int SplatIndex = -1;
+  for (const auto &V : Indices) {
+    if (V != -1) {
+      SplatIndex = V;
+      break;
+    }
+  }
+
+  return fitsRegularPattern<int>(Indices.begin(), 1, Indices.end(), SplatIndex,
+                                 0);
 }
 
 // Lower VECTOR_SHUFFLE into ILVEV (if possible).
 //
 // ILVEV interleaves the even elements from each vector.
 //
-// It is possible to lower into ILVEV when the mask takes the form:
-//   <0, n, 2, n+2, 4, n+4, ...>
+// It is possible to lower into ILVEV when the mask consists of two of the
+// following forms interleaved:
+//   <0, 2, 4, ...>
+//   <n, n+2, n+4, ...>
 // where n is the number of elements in the vector.
+// For example:
+//   <0, 0, 2, 2, 4, 4, ...>
+//   <0, n, 2, n+2, 4, n+4, ...>
 //
 // When undef's appear in the mask they are treated as if they were whatever
-// value is necessary in order to fit the above form.
+// value is necessary in order to fit the above forms.
 static SDValue lowerVECTOR_SHUFFLE_ILVEV(SDValue Op, EVT ResTy,
                                          SmallVector<int, 16> Indices,
                                          SelectionDAG &DAG) {
-  assert ((Indices.size() % 2) == 0);
-  int WsIdx = 0;
-  int WtIdx = ResTy.getVectorNumElements();
+  assert((Indices.size() % 2) == 0);
+
+  SDValue Wt;
+  SDValue Ws;
+  const auto &Begin = Indices.begin();
+  const auto &End = Indices.end();
+
+  // Check even elements are taken from the even elements of one half or the
+  // other and pick an operand accordingly.
+  if (fitsRegularPattern<int>(Begin, 2, End, 0, 2))
+    Wt = Op->getOperand(0);
+  else if (fitsRegularPattern<int>(Begin, 2, End, Indices.size(), 2))
+    Wt = Op->getOperand(1);
+  else
+    return SDValue();
 
-  for (unsigned i = 0; i < Indices.size(); i += 2) {
-    if (Indices[i] != -1 && Indices[i] != WsIdx)
-      return SDValue();
-    if (Indices[i+1] != -1 && Indices[i+1] != WtIdx)
-      return SDValue();
-    WsIdx += 2;
-    WtIdx += 2;
-  }
+  // Check odd elements are taken from the even elements of one half or the
+  // other and pick an operand accordingly.
+  if (fitsRegularPattern<int>(Begin + 1, 2, End, 0, 2))
+    Ws = Op->getOperand(0);
+  else if (fitsRegularPattern<int>(Begin + 1, 2, End, Indices.size(), 2))
+    Ws = Op->getOperand(1);
+  else
+    return SDValue();
 
-  return DAG.getNode(MipsISD::ILVEV, SDLoc(Op), ResTy, Op->getOperand(0),
-                     Op->getOperand(1));
+  return DAG.getNode(MipsISD::ILVEV, SDLoc(Op), ResTy, Ws, Wt);
 }
 
 // Lower VECTOR_SHUFFLE into ILVOD (if possible).
 //
 // ILVOD interleaves the odd elements from each vector.
 //
-// It is possible to lower into ILVOD when the mask takes the form:
-//   <1, n+1, 3, n+3, 5, n+5, ...>
+// It is possible to lower into ILVOD when the mask consists of two of the
+// following forms interleaved:
+//   <1, 3, 5, ...>
+//   <n+1, n+3, n+5, ...>
 // where n is the number of elements in the vector.
+// For example:
+//   <1, 1, 3, 3, 5, 5, ...>
+//   <1, n+1, 3, n+3, 5, n+5, ...>
 //
 // When undef's appear in the mask they are treated as if they were whatever
-// value is necessary in order to fit the above form.
+// value is necessary in order to fit the above forms.
 static SDValue lowerVECTOR_SHUFFLE_ILVOD(SDValue Op, EVT ResTy,
                                          SmallVector<int, 16> Indices,
                                          SelectionDAG &DAG) {
-  assert ((Indices.size() % 2) == 0);
-  int WsIdx = 1;
-  int WtIdx = ResTy.getVectorNumElements() + 1;
+  assert((Indices.size() % 2) == 0);
+
+  SDValue Wt;
+  SDValue Ws;
+  const auto &Begin = Indices.begin();
+  const auto &End = Indices.end();
+
+  // Check even elements are taken from the odd elements of one half or the
+  // other and pick an operand accordingly.
+  if (fitsRegularPattern<int>(Begin, 2, End, 1, 2))
+    Wt = Op->getOperand(0);
+  else if (fitsRegularPattern<int>(Begin, 2, End, Indices.size() + 1, 2))
+    Wt = Op->getOperand(1);
+  else
+    return SDValue();
 
-  for (unsigned i = 0; i < Indices.size(); i += 2) {
-    if (Indices[i] != -1 && Indices[i] != WsIdx)
-      return SDValue();
-    if (Indices[i+1] != -1 && Indices[i+1] != WtIdx)
-      return SDValue();
-    WsIdx += 2;
-    WtIdx += 2;
-  }
+  // Check odd elements are taken from the odd elements of one half or the
+  // other and pick an operand accordingly.
+  if (fitsRegularPattern<int>(Begin + 1, 2, End, 1, 2))
+    Ws = Op->getOperand(0);
+  else if (fitsRegularPattern<int>(Begin + 1, 2, End, Indices.size() + 1, 2))
+    Ws = Op->getOperand(1);
+  else
+    return SDValue();
 
-  return DAG.getNode(MipsISD::ILVOD, SDLoc(Op), ResTy, Op->getOperand(0),
-                     Op->getOperand(1));
+  return DAG.getNode(MipsISD::ILVOD, SDLoc(Op), ResTy, Wt, Ws);
 }
 
-// Lower VECTOR_SHUFFLE into ILVL (if possible).
+// Lower VECTOR_SHUFFLE into ILVR (if possible).
 //
-// ILVL interleaves consecutive elements from the left half of each vector.
+// ILVR interleaves consecutive elements from the right (lowest-indexed) half of
+// each vector.
 //
-// It is possible to lower into ILVL when the mask takes the form:
-//   <0, n, 1, n+1, 2, n+2, ...>
+// It is possible to lower into ILVR when the mask consists of two of the
+// following forms interleaved:
+//   <0, 1, 2, ...>
+//   <n, n+1, n+2, ...>
 // where n is the number of elements in the vector.
+// For example:
+//   <0, 0, 1, 1, 2, 2, ...>
+//   <0, n, 1, n+1, 2, n+2, ...>
 //
 // When undef's appear in the mask they are treated as if they were whatever
-// value is necessary in order to fit the above form.
-static SDValue lowerVECTOR_SHUFFLE_ILVL(SDValue Op, EVT ResTy,
+// value is necessary in order to fit the above forms.
+static SDValue lowerVECTOR_SHUFFLE_ILVR(SDValue Op, EVT ResTy,
                                         SmallVector<int, 16> Indices,
                                         SelectionDAG &DAG) {
-  assert ((Indices.size() % 2) == 0);
-  int WsIdx = 0;
-  int WtIdx = ResTy.getVectorNumElements();
+  assert((Indices.size() % 2) == 0);
+
+  SDValue Wt;
+  SDValue Ws;
+  const auto &Begin = Indices.begin();
+  const auto &End = Indices.end();
+
+  // Check even elements are taken from the right (lowest-indexed) elements of
+  // one half or the other and pick an operand accordingly.
+  if (fitsRegularPattern<int>(Begin, 2, End, 0, 1))
+    Wt = Op->getOperand(0);
+  else if (fitsRegularPattern<int>(Begin, 2, End, Indices.size(), 1))
+    Wt = Op->getOperand(1);
+  else
+    return SDValue();
 
-  for (unsigned i = 0; i < Indices.size(); i += 2) {
-    if (Indices[i] != -1 && Indices[i] != WsIdx)
-      return SDValue();
-    if (Indices[i+1] != -1 && Indices[i+1] != WtIdx)
-      return SDValue();
-    WsIdx ++;
-    WtIdx ++;
-  }
+  // Check odd elements are taken from the right (lowest-indexed) elements of
+  // one half or the other and pick an operand accordingly.
+  if (fitsRegularPattern<int>(Begin + 1, 2, End, 0, 1))
+    Ws = Op->getOperand(0);
+  else if (fitsRegularPattern<int>(Begin + 1, 2, End, Indices.size(), 1))
+    Ws = Op->getOperand(1);
+  else
+    return SDValue();
 
-  return DAG.getNode(MipsISD::ILVL, SDLoc(Op), ResTy, Op->getOperand(0),
-                     Op->getOperand(1));
+  return DAG.getNode(MipsISD::ILVR, SDLoc(Op), ResTy, Ws, Wt);
 }
 
-// Lower VECTOR_SHUFFLE into ILVR (if possible).
+// Lower VECTOR_SHUFFLE into ILVL (if possible).
 //
-// ILVR interleaves consecutive elements from the right half of each vector.
+// ILVL interleaves consecutive elements from the left (highest-indexed) half
+// of each vector.
 //
-// It is possible to lower into ILVR when the mask takes the form:
-//   <x, n+x, x+1, n+x+1, x+2, n+x+2, ...>
+// It is possible to lower into ILVL when the mask consists of two of the
+// following forms interleaved:
+//   <x, x+1, x+2, ...>
+//   <n+x, n+x+1, n+x+2, ...>
 // where n is the number of elements in the vector and x is half n.
+// For example:
+//   <x, x, x+1, x+1, x+2, x+2, ...>
+//   <x, n+x, x+1, n+x+1, x+2, n+x+2, ...>
 //
 // When undef's appear in the mask they are treated as if they were whatever
-// value is necessary in order to fit the above form.
-static SDValue lowerVECTOR_SHUFFLE_ILVR(SDValue Op, EVT ResTy,
+// value is necessary in order to fit the above forms.
+static SDValue lowerVECTOR_SHUFFLE_ILVL(SDValue Op, EVT ResTy,
                                         SmallVector<int, 16> Indices,
                                         SelectionDAG &DAG) {
-  assert ((Indices.size() % 2) == 0);
-  unsigned NumElts = ResTy.getVectorNumElements();
-  int WsIdx = NumElts / 2;
-  int WtIdx = NumElts + NumElts / 2;
+  assert((Indices.size() % 2) == 0);
+
+  unsigned HalfSize = Indices.size() / 2;
+  SDValue Wt;
+  SDValue Ws;
+  const auto &Begin = Indices.begin();
+  const auto &End = Indices.end();
+
+  // Check even elements are taken from the left (highest-indexed) elements of
+  // one half or the other and pick an operand accordingly.
+  if (fitsRegularPattern<int>(Begin, 2, End, HalfSize, 1))
+    Wt = Op->getOperand(0);
+  else if (fitsRegularPattern<int>(Begin, 2, End, Indices.size() + HalfSize, 1))
+    Wt = Op->getOperand(1);
+  else
+    return SDValue();
 
-  for (unsigned i = 0; i < Indices.size(); i += 2) {
-    if (Indices[i] != -1 && Indices[i] != WsIdx)
-      return SDValue();
-    if (Indices[i+1] != -1 && Indices[i+1] != WtIdx)
-      return SDValue();
-    WsIdx ++;
-    WtIdx ++;
-  }
+  // Check odd elements are taken from the left (highest-indexed) elements of
+  // one half or the other and pick an operand accordingly.
+  if (fitsRegularPattern<int>(Begin + 1, 2, End, HalfSize, 1))
+    Ws = Op->getOperand(0);
+  else if (fitsRegularPattern<int>(Begin + 1, 2, End, Indices.size() + HalfSize,
+                                   1))
+    Ws = Op->getOperand(1);
+  else
+    return SDValue();
 
-  return DAG.getNode(MipsISD::ILVR, SDLoc(Op), ResTy, Op->getOperand(0),
-                     Op->getOperand(1));
+  return DAG.getNode(MipsISD::ILVL, SDLoc(Op), ResTy, Ws, Wt);
 }
 
 // Lower VECTOR_SHUFFLE into PCKEV (if possible).
 //
 // PCKEV copies the even elements of each vector into the result vector.
 //
-// It is possible to lower into PCKEV when the mask takes the form:
-//   <0, 2, 4, ..., n, n+2, n+4, ...>
+// It is possible to lower into PCKEV when the mask consists of two of the
+// following forms concatenated:
+//   <0, 2, 4, ...>
+//   <n, n+2, n+4, ...>
 // where n is the number of elements in the vector.
+// For example:
+//   <0, 2, 4, ..., 0, 2, 4, ...>
+//   <0, 2, 4, ..., n, n+2, n+4, ...>
 //
 // When undef's appear in the mask they are treated as if they were whatever
-// value is necessary in order to fit the above form.
+// value is necessary in order to fit the above forms.
 static SDValue lowerVECTOR_SHUFFLE_PCKEV(SDValue Op, EVT ResTy,
                                          SmallVector<int, 16> Indices,
                                          SelectionDAG &DAG) {
-  assert ((Indices.size() % 2) == 0);
-  int Idx = 0;
+  assert((Indices.size() % 2) == 0);
+
+  SDValue Wt;
+  SDValue Ws;
+  const auto &Begin = Indices.begin();
+  const auto &Mid = Indices.begin() + Indices.size() / 2;
+  const auto &End = Indices.end();
+
+  if (fitsRegularPattern<int>(Begin, 1, Mid, 0, 2))
+    Wt = Op->getOperand(0);
+  else if (fitsRegularPattern<int>(Begin, 1, Mid, Indices.size(), 2))
+    Wt = Op->getOperand(1);
+  else
+    return SDValue();
 
-  for (unsigned i = 0; i < Indices.size(); ++i) {
-    if (Indices[i] != -1 && Indices[i] != Idx)
-      return SDValue();
-    Idx += 2;
-  }
+  if (fitsRegularPattern<int>(Mid, 1, End, 0, 2))
+    Ws = Op->getOperand(0);
+  else if (fitsRegularPattern<int>(Mid, 1, End, Indices.size(), 2))
+    Ws = Op->getOperand(1);
+  else
+    return SDValue();
 
-  return DAG.getNode(MipsISD::PCKEV, SDLoc(Op), ResTy, Op->getOperand(0),
-                     Op->getOperand(1));
+  return DAG.getNode(MipsISD::PCKEV, SDLoc(Op), ResTy, Ws, Wt);
 }
 
 // Lower VECTOR_SHUFFLE into PCKOD (if possible).
 //
 // PCKOD copies the odd elements of each vector into the result vector.
 //
-// It is possible to lower into PCKOD when the mask takes the form:
-//   <1, 3, 5, ..., n+1, n+3, n+5, ...>
+// It is possible to lower into PCKOD when the mask consists of two of the
+// following forms concatenated:
+//   <1, 3, 5, ...>
+//   <n+1, n+3, n+5, ...>
 // where n is the number of elements in the vector.
+// For example:
+//   <1, 3, 5, ..., 1, 3, 5, ...>
+//   <1, 3, 5, ..., n+1, n+3, n+5, ...>
 //
 // When undef's appear in the mask they are treated as if they were whatever
-// value is necessary in order to fit the above form.
+// value is necessary in order to fit the above forms.
 static SDValue lowerVECTOR_SHUFFLE_PCKOD(SDValue Op, EVT ResTy,
                                          SmallVector<int, 16> Indices,
                                          SelectionDAG &DAG) {
-  assert ((Indices.size() % 2) == 0);
-  int Idx = 1;
+  assert((Indices.size() % 2) == 0);
+
+  SDValue Wt;
+  SDValue Ws;
+  const auto &Begin = Indices.begin();
+  const auto &Mid = Indices.begin() + Indices.size() / 2;
+  const auto &End = Indices.end();
+
+  if (fitsRegularPattern<int>(Begin, 1, Mid, 1, 2))
+    Wt = Op->getOperand(0);
+  else if (fitsRegularPattern<int>(Begin, 1, Mid, Indices.size() + 1, 2))
+    Wt = Op->getOperand(1);
+  else
+    return SDValue();
 
-  for (unsigned i = 0; i < Indices.size(); ++i) {
-    if (Indices[i] != -1 && Indices[i] != Idx)
-      return SDValue();
-    Idx += 2;
-  }
+  if (fitsRegularPattern<int>(Mid, 1, End, 1, 2))
+    Ws = Op->getOperand(0);
+  else if (fitsRegularPattern<int>(Mid, 1, End, Indices.size() + 1, 2))
+    Ws = Op->getOperand(1);
+  else
+    return SDValue();
 
-  return DAG.getNode(MipsISD::PCKOD, SDLoc(Op), ResTy, Op->getOperand(0),
-                     Op->getOperand(1));
+  return DAG.getNode(MipsISD::PCKOD, SDLoc(Op), ResTy, Ws, Wt);
 }
 
 // Lower VECTOR_SHUFFLE into VSHF.
@@ -2669,7 +2825,7 @@ static SDValue lowerVECTOR_SHUFFLE_VSHF(SDValue Op, EVT ResTy,
 
   for (SmallVector<int, 16>::iterator I = Indices.begin(); I != Indices.end();
        ++I)
-    Ops.push_back(DAG.getTargetConstant(*I, MaskEltTy));
+    Ops.push_back(DAG.getTargetConstant(*I, DL, MaskEltTy));
 
   SDValue MaskVec = DAG.getNode(ISD::BUILD_VECTOR, DL, MaskVecTy, Ops);
 
@@ -2709,10 +2865,11 @@ SDValue MipsSETargetLowering::lowerVECTOR_SHUFFLE(SDValue Op,
   for (int i = 0; i < ResTyNumElts; ++i)
     Indices.push_back(Node->getMaskElt(i));
 
-  SDValue Result = lowerVECTOR_SHUFFLE_SHF(Op, ResTy, Indices, DAG);
-  if (Result.getNode())
-    return Result;
-  Result = lowerVECTOR_SHUFFLE_ILVEV(Op, ResTy, Indices, DAG);
+  // splati.[bhwd] is preferable to the others but is matched from
+  // MipsISD::VSHF.
+  if (isVECTOR_SHUFFLE_SPLATI(Op, ResTy, Indices, DAG))
+    return lowerVECTOR_SHUFFLE_VSHF(Op, ResTy, Indices, DAG);
+  SDValue Result = lowerVECTOR_SHUFFLE_ILVEV(Op, ResTy, Indices, DAG);
   if (Result.getNode())
     return Result;
   Result = lowerVECTOR_SHUFFLE_ILVOD(Op, ResTy, Indices, DAG);
@@ -2728,6 +2885,9 @@ SDValue MipsSETargetLowering::lowerVECTOR_SHUFFLE(SDValue Op,
   if (Result.getNode())
     return Result;
   Result = lowerVECTOR_SHUFFLE_PCKOD(Op, ResTy, Indices, DAG);
+  if (Result.getNode())
+    return Result;
+  Result = lowerVECTOR_SHUFFLE_SHF(Op, ResTy, Indices, DAG);
   if (Result.getNode())
     return Result;
   return lowerVECTOR_SHUFFLE_VSHF(Op, ResTy, Indices, DAG);
@@ -3043,7 +3203,7 @@ MipsSETargetLowering::emitINSERT_DF_VIDX(MachineInstr *MI,
 
   const TargetRegisterClass *VecRC = nullptr;
   const TargetRegisterClass *GPRRC =
-      Subtarget.isGP64bit() ? &Mips::GPR64RegClass : &Mips::GPR32RegClass;
+      Subtarget.isABI_N64() ? &Mips::GPR64RegClass : &Mips::GPR32RegClass;
   unsigned EltLog2Size;
   unsigned InsertOp = 0;
   unsigned InsveOp = 0;
@@ -3121,8 +3281,9 @@ MipsSETargetLowering::emitINSERT_DF_VIDX(MachineInstr *MI,
   // sld.df inteprets $rt modulo the number of columns so we only need to negate
   // the lane index to do this.
   unsigned LaneTmp2 = RegInfo.createVirtualRegister(GPRRC);
-  BuildMI(*BB, MI, DL, TII->get(Mips::SUB), LaneTmp2)
-      .addReg(Mips::ZERO)
+  BuildMI(*BB, MI, DL, TII->get(Subtarget.isABI_N64() ? Mips::DSUB : Mips::SUB),
+          LaneTmp2)
+      .addReg(Subtarget.isABI_N64() ? Mips::ZERO_64 : Mips::ZERO)
       .addReg(LaneReg);
   BuildMI(*BB, MI, DL, TII->get(Mips::SLD_B), Wd)
       .addReg(WdTmp2)
diff --git a/lib/Target/Mips/MipsSEInstrInfo.cpp b/lib/Target/Mips/MipsSEInstrInfo.cpp
index cb38393f30d..786307b95f8 100644
--- a/lib/Target/Mips/MipsSEInstrInfo.cpp
+++ b/lib/Target/Mips/MipsSEInstrInfo.cpp
@@ -359,10 +359,10 @@ unsigned MipsSEInstrInfo::getOppositeBranchOpc(unsigned Opc) const {
 void MipsSEInstrInfo::adjustStackPtr(unsigned SP, int64_t Amount,
                                      MachineBasicBlock &MBB,
                                      MachineBasicBlock::iterator I) const {
-  const MipsSubtarget &STI = Subtarget;
+  MipsABIInfo ABI = Subtarget.getABI();
   DebugLoc DL = I != MBB.end() ? I->getDebugLoc() : DebugLoc();
-  unsigned ADDu = STI.isABI_N64() ? Mips::DADDu : Mips::ADDu;
-  unsigned ADDiu = STI.isABI_N64() ? Mips::DADDiu : Mips::ADDiu;
+  unsigned ADDu = ABI.GetPtrAdduOp();
+  unsigned ADDiu = ABI.GetPtrAddiuOp();
 
   if (Amount == 0)
     return;
@@ -610,7 +610,8 @@ void MipsSEInstrInfo::expandEhReturn(MachineBasicBlock &MBB,
   // This pseudo instruction is generated as part of the lowering of
   // ISD::EH_RETURN. We convert it to a stack increment by OffsetReg, and
   // indirect jump to TargetReg
-  unsigned ADDU = Subtarget.isABI_N64() ? Mips::DADDu : Mips::ADDu;
+  MipsABIInfo ABI = Subtarget.getABI();
+  unsigned ADDU = ABI.GetPtrAdduOp();
   unsigned SP = Subtarget.isGP64bit() ? Mips::SP_64 : Mips::SP;
   unsigned RA = Subtarget.isGP64bit() ? Mips::RA_64 : Mips::RA;
   unsigned T9 = Subtarget.isGP64bit() ? Mips::T9_64 : Mips::T9;
diff --git a/lib/Target/Mips/MipsSERegisterInfo.cpp b/lib/Target/Mips/MipsSERegisterInfo.cpp
index b89207ea1d2..132c3a1001a 100644
--- a/lib/Target/Mips/MipsSERegisterInfo.cpp
+++ b/lib/Target/Mips/MipsSERegisterInfo.cpp
@@ -110,8 +110,11 @@ void MipsSERegisterInfo::eliminateFI(MachineBasicBlock::iterator II,
   MachineFunction &MF = *MI.getParent()->getParent();
   MachineFrameInfo *MFI = MF.getFrameInfo();
   MipsFunctionInfo *MipsFI = MF.getInfo<MipsFunctionInfo>();
-  bool isN64 =
-      static_cast<const MipsTargetMachine &>(MF.getTarget()).getABI().IsN64();
+
+  MipsABIInfo ABI =
+      static_cast<const MipsTargetMachine &>(MF.getTarget()).getABI();
+  const MipsRegisterInfo *RegInfo =
+    static_cast<const MipsRegisterInfo *>(MF.getSubtarget().getRegisterInfo());
 
   const std::vector<CalleeSavedInfo> &CSI = MFI->getCalleeSavedInfo();
   int MinCSFI = 0;
@@ -134,8 +137,15 @@ void MipsSERegisterInfo::eliminateFI(MachineBasicBlock::iterator II,
   unsigned FrameReg;
 
   if ((FrameIndex >= MinCSFI && FrameIndex <= MaxCSFI) || EhDataRegFI)
-    FrameReg = isN64 ? Mips::SP_64 : Mips::SP;
-  else
+    FrameReg = ABI.GetStackPtr();
+  else if (RegInfo->needsStackRealignment(MF)) {
+    if (MFI->hasVarSizedObjects() && !MFI->isFixedObjectIndex(FrameIndex))
+      FrameReg = ABI.GetBasePtr();
+    else if (MFI->isFixedObjectIndex(FrameIndex))
+      FrameReg = getFrameRegister(MF);
+    else
+      FrameReg = ABI.GetStackPtr();
+  } else
     FrameReg = getFrameRegister(MF);
 
   // Calculate final offset.
@@ -167,15 +177,16 @@ void MipsSERegisterInfo::eliminateFI(MachineBasicBlock::iterator II,
       // (where n < 16) and doesn't, but does fit into 16-bits then use an ADDiu
       MachineBasicBlock &MBB = *MI.getParent();
       DebugLoc DL = II->getDebugLoc();
-      unsigned ADDiu = isN64 ? Mips::DADDiu : Mips::ADDiu;
-      const TargetRegisterClass *RC =
-          isN64 ? &Mips::GPR64RegClass : &Mips::GPR32RegClass;
+      const TargetRegisterClass *PtrRC =
+          ABI.ArePtrs64bit() ? &Mips::GPR64RegClass : &Mips::GPR32RegClass;
       MachineRegisterInfo &RegInfo = MBB.getParent()->getRegInfo();
-      unsigned Reg = RegInfo.createVirtualRegister(RC);
+      unsigned Reg = RegInfo.createVirtualRegister(PtrRC);
       const MipsSEInstrInfo &TII =
           *static_cast<const MipsSEInstrInfo *>(
               MBB.getParent()->getSubtarget().getInstrInfo());
-      BuildMI(MBB, II, DL, TII.get(ADDiu), Reg).addReg(FrameReg).addImm(Offset);
+      BuildMI(MBB, II, DL, TII.get(ABI.GetPtrAddiuOp()), Reg)
+          .addReg(FrameReg)
+          .addImm(Offset);
 
       FrameReg = Reg;
       Offset = 0;
@@ -185,14 +196,13 @@ void MipsSERegisterInfo::eliminateFI(MachineBasicBlock::iterator II,
       // instructions.
       MachineBasicBlock &MBB = *MI.getParent();
       DebugLoc DL = II->getDebugLoc();
-      unsigned ADDu = isN64 ? Mips::DADDu : Mips::ADDu;
       unsigned NewImm = 0;
       const MipsSEInstrInfo &TII =
           *static_cast<const MipsSEInstrInfo *>(
               MBB.getParent()->getSubtarget().getInstrInfo());
       unsigned Reg = TII.loadImmediate(Offset, MBB, II, DL,
                                        OffsetBitSize == 16 ? &NewImm : nullptr);
-      BuildMI(MBB, II, DL, TII.get(ADDu), Reg).addReg(FrameReg)
+      BuildMI(MBB, II, DL, TII.get(ABI.GetPtrAdduOp()), Reg).addReg(FrameReg)
         .addReg(Reg, RegState::Kill);
 
       FrameReg = Reg;
diff --git a/lib/Target/Mips/MipsSelectionDAGInfo.cpp b/lib/Target/Mips/MipsSelectionDAGInfo.cpp
deleted file mode 100644
index edd8f670707..00000000000
--- a/lib/Target/Mips/MipsSelectionDAGInfo.cpp
+++ /dev/null
@@ -1,23 +0,0 @@
-//===-- MipsSelectionDAGInfo.cpp - Mips SelectionDAG Info -----------------===//
-//
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
-//
-//===----------------------------------------------------------------------===//
-//
-// This file implements the MipsSelectionDAGInfo class.
-//
-//===----------------------------------------------------------------------===//
-
-#include "MipsTargetMachine.h"
-using namespace llvm;
-
-#define DEBUG_TYPE "mips-selectiondag-info"
-
-MipsSelectionDAGInfo::MipsSelectionDAGInfo(const DataLayout &DL)
-    : TargetSelectionDAGInfo(&DL) {}
-
-MipsSelectionDAGInfo::~MipsSelectionDAGInfo() {
-}
diff --git a/lib/Target/Mips/MipsSelectionDAGInfo.h b/lib/Target/Mips/MipsSelectionDAGInfo.h
deleted file mode 100644
index 061423fbeb8..00000000000
--- a/lib/Target/Mips/MipsSelectionDAGInfo.h
+++ /dev/null
@@ -1,31 +0,0 @@
-//===-- MipsSelectionDAGInfo.h - Mips SelectionDAG Info ---------*- C++ -*-===//
-//
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
-//
-//===----------------------------------------------------------------------===//
-//
-// This file defines the Mips subclass for TargetSelectionDAGInfo.
-//
-//===----------------------------------------------------------------------===//
-
-#ifndef LLVM_LIB_TARGET_MIPS_MIPSSELECTIONDAGINFO_H
-#define LLVM_LIB_TARGET_MIPS_MIPSSELECTIONDAGINFO_H
-
-#include "llvm/Target/TargetSelectionDAGInfo.h"
-
-namespace llvm {
-
-class MipsTargetMachine;
-
-class MipsSelectionDAGInfo : public TargetSelectionDAGInfo {
-public:
-  explicit MipsSelectionDAGInfo(const DataLayout &DL);
-  ~MipsSelectionDAGInfo();
-};
-
-}
-
-#endif
diff --git a/lib/Target/Mips/MipsSubtarget.cpp b/lib/Target/Mips/MipsSubtarget.cpp
index 26f39a2aa99..471b6e19a8b 100644
--- a/lib/Target/Mips/MipsSubtarget.cpp
+++ b/lib/Target/Mips/MipsSubtarget.cpp
@@ -59,18 +59,18 @@ static cl::opt<bool>
 
 void MipsSubtarget::anchor() { }
 
-MipsSubtarget::MipsSubtarget(const std::string &TT, const std::string &CPU,
+MipsSubtarget::MipsSubtarget(const Triple &TT, const std::string &CPU,
                              const std::string &FS, bool little,
                              const MipsTargetMachine &TM)
     : MipsGenSubtargetInfo(TT, CPU, FS), MipsArchVersion(MipsDefault),
-      IsLittle(little), IsSingleFloat(false), IsFPXX(false), NoABICalls(false),
-      IsFP64bit(false), UseOddSPReg(true), IsNaN2008bit(false),
-      IsGP64bit(false), HasVFPU(false), HasCnMips(false), HasMips3_32(false),
-      HasMips3_32r2(false), HasMips4_32(false), HasMips4_32r2(false),
-      HasMips5_32r2(false), InMips16Mode(false),
+      IsLittle(little), IsSoftFloat(false), IsSingleFloat(false), IsFPXX(false),
+      NoABICalls(false), IsFP64bit(false), UseOddSPReg(true),
+      IsNaN2008bit(false), IsGP64bit(false), HasVFPU(false), HasCnMips(false),
+      HasMips3_32(false), HasMips3_32r2(false), HasMips4_32(false),
+      HasMips4_32r2(false), HasMips5_32r2(false), InMips16Mode(false),
       InMips16HardFloat(Mips16HardFloat), InMicroMipsMode(false), HasDSP(false),
       HasDSPR2(false), AllowMixed16_32(Mixed16_32 | Mips_Os16), Os16(Mips_Os16),
-      HasMSA(false), TM(TM), TargetTriple(TT), TSInfo(*TM.getDataLayout()),
+      HasMSA(false), TM(TM), TargetTriple(TT), TSInfo(),
       InstrInfo(
           MipsInstrInfo::create(initializeSubtargetDependencies(CPU, FS, TM))),
       FrameLowering(MipsFrameLowering::create(*this)),
@@ -126,7 +126,7 @@ MipsSubtarget::MipsSubtarget(const std::string &TT, const std::string &CPU,
 }
 
 /// This overrides the PostRAScheduler bit in the SchedModel for any CPU.
-bool MipsSubtarget::enablePostMachineScheduler() const { return true; }
+bool MipsSubtarget::enablePostRAScheduler() const { return true; }
 
 void MipsSubtarget::getCriticalPathRCs(RegClassVector &CriticalPathRCs) const {
   CriticalPathRCs.clear();
@@ -148,16 +148,12 @@ MipsSubtarget::initializeSubtargetDependencies(StringRef CPU, StringRef FS,
   // Initialize scheduling itinerary for the specified CPU.
   InstrItins = getInstrItineraryForCPU(CPUName);
 
-  if (InMips16Mode && !TM.Options.UseSoftFloat)
+  if (InMips16Mode && !IsSoftFloat)
     InMips16HardFloat = true;
 
   return *this;
 }
 
-bool MipsSubtarget::abiUsesSoftFloat() const {
-  return TM.Options.UseSoftFloat && !InMips16HardFloat;
-}
-
 bool MipsSubtarget::useConstantIslands() {
   DEBUG(dbgs() << "use constant islands " << Mips16ConstantIslands << "\n");
   return Mips16ConstantIslands;
diff --git a/lib/Target/Mips/MipsSubtarget.h b/lib/Target/Mips/MipsSubtarget.h
index faded8a3d6c..1e3a808d963 100644
--- a/lib/Target/Mips/MipsSubtarget.h
+++ b/lib/Target/Mips/MipsSubtarget.h
@@ -18,10 +18,10 @@
 #include "MipsFrameLowering.h"
 #include "MipsISelLowering.h"
 #include "MipsInstrInfo.h"
-#include "MipsSelectionDAGInfo.h"
 #include "llvm/IR/DataLayout.h"
 #include "llvm/MC/MCInstrItineraries.h"
 #include "llvm/Support/ErrorHandling.h"
+#include "llvm/Target/TargetSelectionDAGInfo.h"
 #include "llvm/Target/TargetSubtargetInfo.h"
 #include <string>
 
@@ -48,6 +48,9 @@ class MipsSubtarget : public MipsGenSubtargetInfo {
   // IsLittle - The target is Little Endian
   bool IsLittle;
 
+  // IsSoftFloat - The target does not support any floating point instructions.
+  bool IsSoftFloat;
+
   // IsSingleFloat - The target only supports single precision float
   // point operations. This enable the target to use all 32 32-bit
   // floating point registers instead of only using even ones.
@@ -137,14 +140,14 @@ class MipsSubtarget : public MipsGenSubtargetInfo {
 
   Triple TargetTriple;
 
-  const MipsSelectionDAGInfo TSInfo;
+  const TargetSelectionDAGInfo TSInfo;
   std::unique_ptr<const MipsInstrInfo> InstrInfo;
   std::unique_ptr<const MipsFrameLowering> FrameLowering;
   std::unique_ptr<const MipsTargetLowering> TLInfo;
 
 public:
   /// This overrides the PostRAScheduler bit in the SchedModel for each CPU.
-  bool enablePostMachineScheduler() const override;
+  bool enablePostRAScheduler() const override;
   void getCriticalPathRCs(RegClassVector &CriticalPathRCs) const override;
   CodeGenOpt::Level getOptLevelToEnablePostRAScheduler() const override;
 
@@ -158,9 +161,8 @@ class MipsSubtarget : public MipsGenSubtargetInfo {
 
   /// This constructor initializes the data members to match that
   /// of the specified triple.
-  MipsSubtarget(const std::string &TT, const std::string &CPU,
-                const std::string &FS, bool little,
-                const MipsTargetMachine &TM);
+  MipsSubtarget(const Triple &TT, const std::string &CPU, const std::string &FS,
+                bool little, const MipsTargetMachine &TM);
 
   /// ParseSubtargetFeatures - Parses features string setting specified
   /// subtarget options.  Definition of function is auto generated by tblgen.
@@ -187,7 +189,7 @@ class MipsSubtarget : public MipsGenSubtargetInfo {
   }
   bool hasMips32r5() const {
     return (MipsArchVersion >= Mips32r5 && MipsArchVersion < Mips32Max) ||
-           hasMips64r2();
+           hasMips64r5();
   }
   bool hasMips32r6() const {
     return (MipsArchVersion >= Mips32r6 && MipsArchVersion < Mips32Max) ||
@@ -225,6 +227,7 @@ class MipsSubtarget : public MipsGenSubtargetInfo {
     return inMips16Mode() && InMips16HardFloat;
   }
   bool inMicroMipsMode() const { return InMicroMipsMode; }
+  bool inMicroMips32r6Mode() const { return InMicroMipsMode && hasMips32r6(); }
   bool hasDSP() const { return HasDSP; }
   bool hasDSPR2() const { return HasDSPR2; }
   bool hasMSA() const { return HasMSA; }
@@ -232,7 +235,7 @@ class MipsSubtarget : public MipsGenSubtargetInfo {
 
   bool hasStandardEncoding() const { return !inMips16Mode(); }
 
-  bool abiUsesSoftFloat() const;
+  bool useSoftFloat() const { return IsSoftFloat; }
 
   bool enableLongBranchPass() const {
     return hasStandardEncoding() || allowMixed16_32();
@@ -272,7 +275,7 @@ class MipsSubtarget : public MipsGenSubtargetInfo {
   void setHelperClassesMips16();
   void setHelperClassesMipsSE();
 
-  const MipsSelectionDAGInfo *getSelectionDAGInfo() const override {
+  const TargetSelectionDAGInfo *getSelectionDAGInfo() const override {
     return &TSInfo;
   }
   const MipsInstrInfo *getInstrInfo() const override { return InstrInfo.get(); }
diff --git a/lib/Target/Mips/MipsTargetMachine.cpp b/lib/Target/Mips/MipsTargetMachine.cpp
index e600f595368..1c77745d130 100644
--- a/lib/Target/Mips/MipsTargetMachine.cpp
+++ b/lib/Target/Mips/MipsTargetMachine.cpp
@@ -32,28 +32,10 @@
 #include "llvm/Support/raw_ostream.h"
 #include "llvm/Transforms/Scalar.h"
 
-// @LOCALMOD-START
-#if !defined(PNACL_BROWSER_TRANSLATOR)
-#include "llvm/Support/CommandLine.h"
-#include "llvm/Transforms/IPO.h"
-#include "llvm/Transforms/NaCl.h"
-#endif
-// @LOCALMOD-END
-
 using namespace llvm;
 
 #define DEBUG_TYPE "mips"
 
-// @LOCALMOD-START
-#if !defined(PNACL_BROWSER_TRANSLATOR)
-static cl::opt<bool>
-DirectToNaCl("direct-to-nacl",
-    cl::init(false),
-    cl::desc("MIPS-D2N: add passes required for direct-to-nacl."),
-    cl::Hidden);
-#endif
-// @LOCALMOD-END
-
 extern "C" void LLVMInitializeMipsTarget() {
   // Register the target.
   RegisterTargetMachine<MipsebTargetMachine> X(TheMipsTarget);
@@ -62,12 +44,11 @@ extern "C" void LLVMInitializeMipsTarget() {
   RegisterTargetMachine<MipselTargetMachine> B(TheMips64elTarget);
 }
 
-static std::string computeDataLayout(StringRef TT, StringRef CPU,
+static std::string computeDataLayout(const Triple &TT, StringRef CPU,
                                      const TargetOptions &Options,
                                      bool isLittle) {
   std::string Ret = "";
-  MipsABIInfo ABI =
-      MipsABIInfo::computeTargetABI(Triple(TT), CPU, Options.MCOptions);
+  MipsABIInfo ABI = MipsABIInfo::computeTargetABI(TT, CPU, Options.MCOptions);
 
   // There are both little and big endian mips.
   if (isLittle)
@@ -81,7 +62,7 @@ static std::string computeDataLayout(StringRef TT, StringRef CPU,
   if (!ABI.IsN64())
     Ret += "-p:32:32";
 
-  // 8 and 16 bit integers only need no have natural alignment, but try to
+  // 8 and 16 bit integers only need to have natural alignment, but try to
   // align them to 32 bits. 64 bit integers have natural alignment.
   Ret += "-i8:8:32-i16:16:32-i64:64";
 
@@ -101,7 +82,7 @@ static std::string computeDataLayout(StringRef TT, StringRef CPU,
 // offset from the stack/frame pointer, using StackGrowsUp enables
 // an easier handling.
 // Using CodeModel::Large enables different CALL behavior.
-MipsTargetMachine::MipsTargetMachine(const Target &T, StringRef TT,
+MipsTargetMachine::MipsTargetMachine(const Target &T, const Triple &TT,
                                      StringRef CPU, StringRef FS,
                                      const TargetOptions &Options,
                                      Reloc::Model RM, CodeModel::Model CM,
@@ -109,7 +90,7 @@ MipsTargetMachine::MipsTargetMachine(const Target &T, StringRef TT,
     : LLVMTargetMachine(T, computeDataLayout(TT, CPU, Options, isLittle), TT,
                         CPU, FS, Options, RM, CM, OL),
       isLittle(isLittle), TLOF(make_unique<MipsTargetObjectFile>()),
-      ABI(MipsABIInfo::computeTargetABI(Triple(TT), CPU, Options.MCOptions)),
+      ABI(MipsABIInfo::computeTargetABI(TT, CPU, Options.MCOptions)),
       Subtarget(nullptr), DefaultSubtarget(TT, CPU, FS, isLittle, *this),
       NoMips16Subtarget(TT, CPU, FS.empty() ? "-mips16" : FS.str() + ",-mips16",
                         isLittle, *this),
@@ -123,21 +104,21 @@ MipsTargetMachine::~MipsTargetMachine() {}
 
 void MipsebTargetMachine::anchor() { }
 
-MipsebTargetMachine::
-MipsebTargetMachine(const Target &T, StringRef TT,
-                    StringRef CPU, StringRef FS, const TargetOptions &Options,
-                    Reloc::Model RM, CodeModel::Model CM,
-                    CodeGenOpt::Level OL)
-  : MipsTargetMachine(T, TT, CPU, FS, Options, RM, CM, OL, false) {}
+MipsebTargetMachine::MipsebTargetMachine(const Target &T, const Triple &TT,
+                                         StringRef CPU, StringRef FS,
+                                         const TargetOptions &Options,
+                                         Reloc::Model RM, CodeModel::Model CM,
+                                         CodeGenOpt::Level OL)
+    : MipsTargetMachine(T, TT, CPU, FS, Options, RM, CM, OL, false) {}
 
 void MipselTargetMachine::anchor() { }
 
-MipselTargetMachine::
-MipselTargetMachine(const Target &T, StringRef TT,
-                    StringRef CPU, StringRef FS, const TargetOptions &Options,
-                    Reloc::Model RM, CodeModel::Model CM,
-                    CodeGenOpt::Level OL)
-  : MipsTargetMachine(T, TT, CPU, FS, Options, RM, CM, OL, true) {}
+MipselTargetMachine::MipselTargetMachine(const Target &T, const Triple &TT,
+                                         StringRef CPU, StringRef FS,
+                                         const TargetOptions &Options,
+                                         Reloc::Model RM, CodeModel::Model CM,
+                                         CodeGenOpt::Level OL)
+    : MipsTargetMachine(T, TT, CPU, FS, Options, RM, CM, OL, true) {}
 
 const MipsSubtarget *
 MipsTargetMachine::getSubtargetImpl(const Function &F) const {
@@ -157,27 +138,26 @@ MipsTargetMachine::getSubtargetImpl(const Function &F) const {
 
   // FIXME: This is related to the code below to reset the target options,
   // we need to know whether or not the soft float flag is set on the
-  // function before we can generate a subtarget. We also need to use
-  // it as a key for the subtarget since that can be the only difference
-  // between two functions.
-  Attribute SFAttr = F.getFnAttribute("use-soft-float");
-  bool softFloat = !SFAttr.hasAttribute(Attribute::None)
-                       ? SFAttr.getValueAsString() == "true"
-                       : Options.UseSoftFloat;
+  // function, so we can enable it as a subtarget feature.
+  bool softFloat =
+      F.hasFnAttribute("use-soft-float") &&
+      F.getFnAttribute("use-soft-float").getValueAsString() == "true";
 
   if (hasMips16Attr)
     FS += FS.empty() ? "+mips16" : ",+mips16";
   else if (hasNoMips16Attr)
     FS += FS.empty() ? "-mips16" : ",-mips16";
+  if (softFloat)
+    FS += FS.empty() ? "+soft-float" : ",+soft-float";
 
-  auto &I = SubtargetMap[CPU + FS + (softFloat ? "use-soft-float=true"
-                                               : "use-soft-float=false")];
+  auto &I = SubtargetMap[CPU + FS];
   if (!I) {
     // This needs to be done before we create a new subtarget since any
     // creation will depend on the TM and the code generation flags on the
     // function that reside in TargetOptions.
     resetTargetOptions(F);
-    I = llvm::make_unique<MipsSubtarget>(TargetTriple, CPU, FS, isLittle, *this);
+    I = llvm::make_unique<MipsSubtarget>(TargetTriple, CPU, FS, isLittle,
+                                         *this);
   }
   return I.get();
 }
@@ -232,17 +212,6 @@ void MipsPassConfig::addIRPasses() {
     addPass(createMipsOs16Pass(getMipsTargetMachine()));
   if (getMipsSubtarget().inMips16HardFloat())
     addPass(createMips16HardFloatPass(getMipsTargetMachine()));
-
-  // @LOCALMOD-START
-#if !defined(PNACL_BROWSER_TRANSLATOR)
-  if (DirectToNaCl) {
-    addPass(createExpandByValPass());
-
-    addPass(createAddPNaClExternalDeclsPass());
-    addPass(createResolvePNaClIntrinsicsPass());
-  }
-#endif
-  // @LOCALMOD-END
 }
 // Install an instruction selector pass using
 // the ISelDag to gen Mips code.
@@ -268,7 +237,7 @@ TargetIRAnalysis MipsTargetMachine::getTargetIRAnalysis() {
     if (Subtarget->allowMixed16_32()) {
       DEBUG(errs() << "No Target Transform Info Pass Added\n");
       // FIXME: This is no longer necessary as the TTI returned is per-function.
-      return TargetTransformInfo(getDataLayout());
+      return TargetTransformInfo(F.getParent()->getDataLayout());
     }
 
     DEBUG(errs() << "Target Transform Info Pass Added\n");
diff --git a/lib/Target/Mips/MipsTargetMachine.h b/lib/Target/Mips/MipsTargetMachine.h
index 5427d6a8304..38b2ecff7d7 100644
--- a/lib/Target/Mips/MipsTargetMachine.h
+++ b/lib/Target/Mips/MipsTargetMachine.h
@@ -39,8 +39,8 @@ class MipsTargetMachine : public LLVMTargetMachine {
   mutable StringMap<std::unique_ptr<MipsSubtarget>> SubtargetMap;
 
 public:
-  MipsTargetMachine(const Target &T, StringRef TT, StringRef CPU, StringRef FS,
-                    const TargetOptions &Options, Reloc::Model RM,
+  MipsTargetMachine(const Target &T, const Triple &TT, StringRef CPU,
+                    StringRef FS, const TargetOptions &Options, Reloc::Model RM,
                     CodeModel::Model CM, CodeGenOpt::Level OL, bool isLittle);
   ~MipsTargetMachine() override;
 
@@ -73,8 +73,8 @@ class MipsTargetMachine : public LLVMTargetMachine {
 class MipsebTargetMachine : public MipsTargetMachine {
   virtual void anchor();
 public:
-  MipsebTargetMachine(const Target &T, StringRef TT,
-                      StringRef CPU, StringRef FS, const TargetOptions &Options,
+  MipsebTargetMachine(const Target &T, const Triple &TT, StringRef CPU,
+                      StringRef FS, const TargetOptions &Options,
                       Reloc::Model RM, CodeModel::Model CM,
                       CodeGenOpt::Level OL);
 };
@@ -84,8 +84,8 @@ class MipsebTargetMachine : public MipsTargetMachine {
 class MipselTargetMachine : public MipsTargetMachine {
   virtual void anchor();
 public:
-  MipselTargetMachine(const Target &T, StringRef TT,
-                      StringRef CPU, StringRef FS, const TargetOptions &Options,
+  MipselTargetMachine(const Target &T, const Triple &TT, StringRef CPU,
+                      StringRef FS, const TargetOptions &Options,
                       Reloc::Model RM, CodeModel::Model CM,
                       CodeGenOpt::Level OL);
 };
diff --git a/lib/Target/Mips/MipsTargetObjectFile.cpp b/lib/Target/Mips/MipsTargetObjectFile.cpp
index 723b63bb41c..205926622fc 100644
--- a/lib/Target/Mips/MipsTargetObjectFile.cpp
+++ b/lib/Target/Mips/MipsTargetObjectFile.cpp
@@ -107,12 +107,14 @@ IsGlobalInSmallSectionImpl(const GlobalValue *GV,
     return false;
 
   Type *Ty = GV->getType()->getElementType();
-  return IsInSmallSection(TM.getDataLayout()->getTypeAllocSize(Ty));
+  return IsInSmallSection(
+      GV->getParent()->getDataLayout().getTypeAllocSize(Ty));
 }
 
-const MCSection *MipsTargetObjectFile::
-SelectSectionForGlobal(const GlobalValue *GV, SectionKind Kind,
-                       Mangler &Mang, const TargetMachine &TM) const {
+MCSection *
+MipsTargetObjectFile::SelectSectionForGlobal(const GlobalValue *GV,
+                                             SectionKind Kind, Mangler &Mang,
+                                             const TargetMachine &TM) const {
   // TODO: Could also support "weak" symbols as well with ".gnu.linkonce.s.*"
   // sections?
 
@@ -127,20 +129,20 @@ SelectSectionForGlobal(const GlobalValue *GV, SectionKind Kind,
 }
 
 /// Return true if this constant should be placed into small data section.
-bool MipsTargetObjectFile::
-IsConstantInSmallSection(const Constant *CN, const TargetMachine &TM) const {
+bool MipsTargetObjectFile::IsConstantInSmallSection(
+    const DataLayout &DL, const Constant *CN, const TargetMachine &TM) const {
   return (static_cast<const MipsTargetMachine &>(TM)
               .getSubtargetImpl()
               ->useSmallSection() &&
-          LocalSData && IsInSmallSection(TM.getDataLayout()->getTypeAllocSize(
-                            CN->getType())));
+          LocalSData && IsInSmallSection(DL.getTypeAllocSize(CN->getType())));
 }
 
-const MCSection *MipsTargetObjectFile::
-getSectionForConstant(SectionKind Kind, const Constant *C) const {
-  if (IsConstantInSmallSection(C, *TM))
+/// Return true if this constant should be placed into small data section.
+MCSection *MipsTargetObjectFile::getSectionForConstant(
+    const DataLayout &DL, SectionKind Kind, const Constant *C) const {
+  if (IsConstantInSmallSection(DL, C, *TM))
     return SmallDataSection;
 
   // Otherwise, we work the same as ELF.
-  return TargetLoweringObjectFileELF::getSectionForConstant(Kind, C);
+  return TargetLoweringObjectFileELF::getSectionForConstant(DL, Kind, C);
 }
diff --git a/lib/Target/Mips/MipsTargetObjectFile.h b/lib/Target/Mips/MipsTargetObjectFile.h
index 45ed9d05f9c..ba04343bad8 100644
--- a/lib/Target/Mips/MipsTargetObjectFile.h
+++ b/lib/Target/Mips/MipsTargetObjectFile.h
@@ -15,8 +15,8 @@
 namespace llvm {
 class MipsTargetMachine;
   class MipsTargetObjectFile : public TargetLoweringObjectFileELF {
-    const MCSection *SmallDataSection;
-    const MCSection *SmallBSSSection;
+    MCSection *SmallDataSection;
+    MCSection *SmallBSSSection;
     const MipsTargetMachine *TM;
   public:
 
@@ -31,16 +31,16 @@ class MipsTargetMachine;
     bool IsGlobalInSmallSectionImpl(const GlobalValue *GV,
                                     const TargetMachine &TM) const;
 
-    const MCSection *SelectSectionForGlobal(const GlobalValue *GV,
-                                        SectionKind Kind, Mangler &Mang,
-                                        const TargetMachine &TM) const override;
+    MCSection *SelectSectionForGlobal(const GlobalValue *GV, SectionKind Kind,
+                                      Mangler &Mang,
+                                      const TargetMachine &TM) const override;
 
     /// Return true if this constant should be placed into small data section.
-    bool IsConstantInSmallSection(const Constant *CN,
+    bool IsConstantInSmallSection(const DataLayout &DL, const Constant *CN,
                                   const TargetMachine &TM) const;
 
-    const MCSection *getSectionForConstant(SectionKind Kind,
-                                           const Constant *C) const override;
+    MCSection *getSectionForConstant(const DataLayout &DL, SectionKind Kind,
+                                     const Constant *C) const override;
   };
 } // end namespace llvm
 
diff --git a/lib/Target/Mips/MipsTargetStreamer.h b/lib/Target/Mips/MipsTargetStreamer.h
index 22b0c6c6685..6ce1be707d0 100644
--- a/lib/Target/Mips/MipsTargetStreamer.h
+++ b/lib/Target/Mips/MipsTargetStreamer.h
@@ -72,28 +72,23 @@ class MipsTargetStreamer : public MCTargetStreamer {
   virtual void emitDirectiveSetNoDsp();
   virtual void emitDirectiveSetPop();
   virtual void emitDirectiveSetPush();
+  virtual void emitDirectiveSetSoftFloat();
+  virtual void emitDirectiveSetHardFloat();
 
   // PIC support
   virtual void emitDirectiveCpLoad(unsigned RegNo);
   virtual void emitDirectiveCpsetup(unsigned RegNo, int RegOrOffset,
                                     const MCSymbol &Sym, bool IsReg);
 
-  /// Emit a '.module fp=value' directive using the given values.
-  /// Updates the .MIPS.abiflags section
-  virtual void emitDirectiveModuleFP(MipsABIFlagsSection::FpABIKind Value,
-                                     bool Is32BitABI) {
-    ABIFlagsSection.setFpABI(Value, Is32BitABI);
-  }
-
-  /// Emit a '.module fp=value' directive using the current values of the
-  /// .MIPS.abiflags section.
-  void emitDirectiveModuleFP() {
-    emitDirectiveModuleFP(ABIFlagsSection.getFpABI(),
-                          ABIFlagsSection.Is32BitABI);
-  }
-
-  virtual void emitDirectiveModuleOddSPReg(bool Enabled, bool IsO32ABI);
+  // FP abiflags directives
+  virtual void emitDirectiveModuleFP();
+  virtual void emitDirectiveModuleOddSPReg();
+  virtual void emitDirectiveModuleSoftFloat();
+  virtual void emitDirectiveModuleHardFloat();
   virtual void emitDirectiveSetFp(MipsABIFlagsSection::FpABIKind Value);
+  virtual void emitDirectiveSetOddSPReg();
+  virtual void emitDirectiveSetNoOddSPReg();
+
   void forbidModuleDirective() { ModuleDirectiveAllowed = false; }
   void reallowModuleDirective() { ModuleDirectiveAllowed = true; }
   bool isModuleDirectiveAllowed() { return ModuleDirectiveAllowed; }
@@ -188,17 +183,22 @@ class MipsTargetAsmStreamer : public MipsTargetStreamer {
   void emitDirectiveSetNoDsp() override;
   void emitDirectiveSetPop() override;
   void emitDirectiveSetPush() override;
+  void emitDirectiveSetSoftFloat() override;
+  void emitDirectiveSetHardFloat() override;
 
   // PIC support
   void emitDirectiveCpLoad(unsigned RegNo) override;
   void emitDirectiveCpsetup(unsigned RegNo, int RegOrOffset,
                             const MCSymbol &Sym, bool IsReg) override;
 
-  // ABI Flags
-  void emitDirectiveModuleFP(MipsABIFlagsSection::FpABIKind Value,
-                             bool Is32BitABI) override;
-  void emitDirectiveModuleOddSPReg(bool Enabled, bool IsO32ABI) override;
+  // FP abiflags directives
+  void emitDirectiveModuleFP() override;
+  void emitDirectiveModuleOddSPReg() override;
+  void emitDirectiveModuleSoftFloat() override;
+  void emitDirectiveModuleHardFloat() override;
   void emitDirectiveSetFp(MipsABIFlagsSection::FpABIKind Value) override;
+  void emitDirectiveSetOddSPReg() override;
+  void emitDirectiveSetNoOddSPReg() override;
 };
 
 // This part is for ELF object output
@@ -240,8 +240,6 @@ class MipsTargetELFStreamer : public MipsTargetStreamer {
   void emitDirectiveCpsetup(unsigned RegNo, int RegOrOffset,
                             const MCSymbol &Sym, bool IsReg) override;
 
-  // ABI Flags
-  void emitDirectiveModuleOddSPReg(bool Enabled, bool IsO32ABI) override;
   void emitMipsAbiFlags();
 };
 }
diff --git a/lib/Target/Mips/TargetInfo/MipsTargetInfo.cpp b/lib/Target/Mips/TargetInfo/MipsTargetInfo.cpp
index 3615c146a52..6a65943515b 100644
--- a/lib/Target/Mips/TargetInfo/MipsTargetInfo.cpp
+++ b/lib/Target/Mips/TargetInfo/MipsTargetInfo.cpp
@@ -23,9 +23,9 @@ extern "C" void LLVMInitializeMipsTargetInfo() {
         /*HasJIT=*/true> Y(TheMipselTarget, "mipsel", "Mipsel");
 
   RegisterTarget<Triple::mips64,
-        /*HasJIT=*/false> A(TheMips64Target, "mips64", "Mips64 [experimental]");
+        /*HasJIT=*/true> A(TheMips64Target, "mips64", "Mips64 [experimental]");
 
   RegisterTarget<Triple::mips64el,
-        /*HasJIT=*/false> B(TheMips64elTarget,
+        /*HasJIT=*/true> B(TheMips64elTarget,
                             "mips64el", "Mips64el [experimental]");
 }
diff --git a/lib/Target/NVPTX/CMakeLists.txt b/lib/Target/NVPTX/CMakeLists.txt
index cdd2f1f5944..05fe06dbc07 100644
--- a/lib/Target/NVPTX/CMakeLists.txt
+++ b/lib/Target/NVPTX/CMakeLists.txt
@@ -20,7 +20,9 @@ set(NVPTXCodeGen_sources
   NVPTXImageOptimizer.cpp
   NVPTXInstrInfo.cpp
   NVPTXLowerAggrCopies.cpp
-  NVPTXLowerStructArgs.cpp
+  NVPTXLowerKernelArgs.cpp
+  NVPTXLowerAlloca.cpp
+  NVPTXPeephole.cpp
   NVPTXMCExpr.cpp
   NVPTXPrologEpilogPass.cpp
   NVPTXRegisterInfo.cpp
diff --git a/lib/Target/NVPTX/InstPrinter/NVPTXInstPrinter.cpp b/lib/Target/NVPTX/InstPrinter/NVPTXInstPrinter.cpp
index ac92df90124..4594c22b870 100644
--- a/lib/Target/NVPTX/InstPrinter/NVPTXInstPrinter.cpp
+++ b/lib/Target/NVPTX/InstPrinter/NVPTXInstPrinter.cpp
@@ -85,7 +85,7 @@ void NVPTXInstPrinter::printOperand(const MCInst *MI, unsigned OpNo,
     O << markup("<imm:") << formatImm(Op.getImm()) << markup(">");
   } else {
     assert(Op.isExpr() && "Unknown operand kind in printOperand");
-    O << *Op.getExpr();
+    Op.getExpr()->print(O, &MAI);
   }
 }
 
diff --git a/lib/Target/NVPTX/MCTargetDesc/NVPTXMCAsmInfo.cpp b/lib/Target/NVPTX/MCTargetDesc/NVPTXMCAsmInfo.cpp
index b9df3d18f94..ef36c13b49f 100644
--- a/lib/Target/NVPTX/MCTargetDesc/NVPTXMCAsmInfo.cpp
+++ b/lib/Target/NVPTX/MCTargetDesc/NVPTXMCAsmInfo.cpp
@@ -25,8 +25,7 @@ static cl::opt<bool> CompileForDebugging("debug-compile",
 
 void NVPTXMCAsmInfo::anchor() {}
 
-NVPTXMCAsmInfo::NVPTXMCAsmInfo(StringRef TT) {
-  Triple TheTriple(TT);
+NVPTXMCAsmInfo::NVPTXMCAsmInfo(const Triple &TheTriple) {
   if (TheTriple.getArch() == Triple::nvptx64) {
     PointerSize = CalleeSaveStackSlotSize = 8;
   }
diff --git a/lib/Target/NVPTX/MCTargetDesc/NVPTXMCAsmInfo.h b/lib/Target/NVPTX/MCTargetDesc/NVPTXMCAsmInfo.h
index c3242866b17..b432e065c2f 100644
--- a/lib/Target/NVPTX/MCTargetDesc/NVPTXMCAsmInfo.h
+++ b/lib/Target/NVPTX/MCTargetDesc/NVPTXMCAsmInfo.h
@@ -18,12 +18,12 @@
 
 namespace llvm {
 class Target;
-class StringRef;
+class Triple;
 
 class NVPTXMCAsmInfo : public MCAsmInfo {
   virtual void anchor();
 public:
-  explicit NVPTXMCAsmInfo(StringRef TT);
+  explicit NVPTXMCAsmInfo(const Triple &TheTriple);
 };
 } // namespace llvm
 
diff --git a/lib/Target/NVPTX/MCTargetDesc/NVPTXMCTargetDesc.cpp b/lib/Target/NVPTX/MCTargetDesc/NVPTXMCTargetDesc.cpp
index f9e43247200..ad7302037ca 100644
--- a/lib/Target/NVPTX/MCTargetDesc/NVPTXMCTargetDesc.cpp
+++ b/lib/Target/NVPTX/MCTargetDesc/NVPTXMCTargetDesc.cpp
@@ -37,7 +37,7 @@ static MCInstrInfo *createNVPTXMCInstrInfo() {
   return X;
 }
 
-static MCRegisterInfo *createNVPTXMCRegisterInfo(StringRef TT) {
+static MCRegisterInfo *createNVPTXMCRegisterInfo(const Triple &TT) {
   MCRegisterInfo *X = new MCRegisterInfo();
   // PTX does not have a return address register.
   InitNVPTXMCRegisterInfo(X, 0);
@@ -45,16 +45,19 @@ static MCRegisterInfo *createNVPTXMCRegisterInfo(StringRef TT) {
 }
 
 static MCSubtargetInfo *
-createNVPTXMCSubtargetInfo(StringRef TT, StringRef CPU, StringRef FS) {
-  MCSubtargetInfo *X = new MCSubtargetInfo();
-  InitNVPTXMCSubtargetInfo(X, TT, CPU, FS);
-  return X;
+createNVPTXMCSubtargetInfo(const Triple &TT, StringRef CPU, StringRef FS) {
+  return createNVPTXMCSubtargetInfoImpl(TT, CPU, FS);
 }
 
-static MCCodeGenInfo *createNVPTXMCCodeGenInfo(
-    StringRef TT, Reloc::Model RM, CodeModel::Model CM, CodeGenOpt::Level OL) {
+static MCCodeGenInfo *createNVPTXMCCodeGenInfo(const Triple &TT,
+                                               Reloc::Model RM,
+                                               CodeModel::Model CM,
+                                               CodeGenOpt::Level OL) {
   MCCodeGenInfo *X = new MCCodeGenInfo();
-  X->InitMCCodeGenInfo(RM, CM, OL);
+
+  // The default relocation model is used regardless of what the client has
+  // specified, as it is the only relocation model currently supported.
+  X->initMCCodeGenInfo(Reloc::Default, CM, OL);
   return X;
 }
 
diff --git a/lib/Target/NVPTX/NVPTX.h b/lib/Target/NVPTX/NVPTX.h
index 382525d27a2..fe28214e958 100644
--- a/lib/Target/NVPTX/NVPTX.h
+++ b/lib/Target/NVPTX/NVPTX.h
@@ -69,7 +69,9 @@ ModulePass *createNVVMReflectPass(const StringMap<int>& Mapping);
 MachineFunctionPass *createNVPTXPrologEpilogPass();
 MachineFunctionPass *createNVPTXReplaceImageHandlesPass();
 FunctionPass *createNVPTXImageOptimizerPass();
-FunctionPass *createNVPTXLowerStructArgsPass();
+FunctionPass *createNVPTXLowerKernelArgsPass(const NVPTXTargetMachine *TM);
+BasicBlockPass *createNVPTXLowerAllocaPass();
+MachineFunctionPass *createNVPTXPeephole();
 
 bool isImageOrSamplerVal(const Value *, const Module *);
 
diff --git a/lib/Target/NVPTX/NVPTXAsmPrinter.cpp b/lib/Target/NVPTX/NVPTXAsmPrinter.cpp
index 9a719644c56..0577881c49e 100644
--- a/lib/Target/NVPTX/NVPTXAsmPrinter.cpp
+++ b/lib/Target/NVPTX/NVPTXAsmPrinter.cpp
@@ -37,6 +37,7 @@
 #include "llvm/IR/Mangler.h"
 #include "llvm/IR/Module.h"
 #include "llvm/IR/Operator.h"
+#include "llvm/MC/MCInst.h"
 #include "llvm/MC/MCStreamer.h"
 #include "llvm/MC/MCSymbol.h"
 #include "llvm/Support/CommandLine.h"
@@ -129,7 +130,7 @@ void NVPTXAsmPrinter::emitLineNumberAsDotLoc(const MachineInstr &MI) {
   if (!curLoc)
     return;
 
-  auto *Scope = cast_or_null<MDScope>(curLoc.getScope());
+  auto *Scope = cast_or_null<DIScope>(curLoc.getScope());
   if (!Scope)
      return;
 
@@ -151,7 +152,7 @@ void NVPTXAsmPrinter::emitLineNumberAsDotLoc(const MachineInstr &MI) {
   std::stringstream temp;
   temp << "\t.loc " << filenameMap[fileName] << " " << curLoc.getLine()
        << " " << curLoc.getCol();
-  OutStreamer.EmitRawText(temp.str());
+  OutStreamer->EmitRawText(temp.str());
 }
 
 void NVPTXAsmPrinter::EmitInstruction(const MachineInstr *MI) {
@@ -162,7 +163,7 @@ void NVPTXAsmPrinter::EmitInstruction(const MachineInstr *MI) {
 
   MCInst Inst;
   lowerToMCInst(MI, Inst);
-  EmitToStreamer(OutStreamer, Inst);
+  EmitToStreamer(*OutStreamer, Inst);
 }
 
 // Handle symbol backtracking for targets that do not support image handles
@@ -224,7 +225,7 @@ void NVPTXAsmPrinter::lowerImageHandleSymbol(unsigned Index, MCOperand &MCOp) {
   const char *Sym = MFI->getImageHandleSymbol(Index);
   std::string *SymNamePtr =
     nvTM.getManagedStrPool()->getManagedString(Sym);
-  MCOp = GetSymbolRef(OutContext.GetOrCreateSymbol(
+  MCOp = GetSymbolRef(OutContext.getOrCreateSymbol(
     StringRef(SymNamePtr->c_str())));
 }
 
@@ -234,7 +235,7 @@ void NVPTXAsmPrinter::lowerToMCInst(const MachineInstr *MI, MCInst &OutMI) {
   if (MI->getOpcode() == NVPTX::CALL_PROTOTYPE) {
     const MachineOperand &MO = MI->getOperand(0);
     OutMI.addOperand(GetSymbolRef(
-      OutContext.GetOrCreateSymbol(Twine(MO.getSymbolName()))));
+      OutContext.getOrCreateSymbol(Twine(MO.getSymbolName()))));
     return;
   }
 
@@ -259,13 +260,13 @@ bool NVPTXAsmPrinter::lowerOperand(const MachineOperand &MO,
   switch (MO.getType()) {
   default: llvm_unreachable("unknown operand type");
   case MachineOperand::MO_Register:
-    MCOp = MCOperand::CreateReg(encodeVirtualRegister(MO.getReg()));
+    MCOp = MCOperand::createReg(encodeVirtualRegister(MO.getReg()));
     break;
   case MachineOperand::MO_Immediate:
-    MCOp = MCOperand::CreateImm(MO.getImm());
+    MCOp = MCOperand::createImm(MO.getImm());
     break;
   case MachineOperand::MO_MachineBasicBlock:
-    MCOp = MCOperand::CreateExpr(MCSymbolRefExpr::Create(
+    MCOp = MCOperand::createExpr(MCSymbolRefExpr::create(
         MO.getMBB()->getSymbol(), OutContext));
     break;
   case MachineOperand::MO_ExternalSymbol:
@@ -281,12 +282,12 @@ bool NVPTXAsmPrinter::lowerOperand(const MachineOperand &MO,
     switch (Cnt->getType()->getTypeID()) {
     default: report_fatal_error("Unsupported FP type"); break;
     case Type::FloatTyID:
-      MCOp = MCOperand::CreateExpr(
-        NVPTXFloatMCExpr::CreateConstantFPSingle(Val, OutContext));
+      MCOp = MCOperand::createExpr(
+        NVPTXFloatMCExpr::createConstantFPSingle(Val, OutContext));
       break;
     case Type::DoubleTyID:
-      MCOp = MCOperand::CreateExpr(
-        NVPTXFloatMCExpr::CreateConstantFPDouble(Val, OutContext));
+      MCOp = MCOperand::createExpr(
+        NVPTXFloatMCExpr::createConstantFPDouble(Val, OutContext));
       break;
     }
     break;
@@ -333,13 +334,13 @@ unsigned NVPTXAsmPrinter::encodeVirtualRegister(unsigned Reg) {
 
 MCOperand NVPTXAsmPrinter::GetSymbolRef(const MCSymbol *Symbol) {
   const MCExpr *Expr;
-  Expr = MCSymbolRefExpr::Create(Symbol, MCSymbolRefExpr::VK_None,
+  Expr = MCSymbolRefExpr::create(Symbol, MCSymbolRefExpr::VK_None,
                                  OutContext);
-  return MCOperand::CreateExpr(Expr);
+  return MCOperand::createExpr(Expr);
 }
 
 void NVPTXAsmPrinter::printReturnValStr(const Function *F, raw_ostream &O) {
-  const DataLayout *TD = TM.getDataLayout();
+  const DataLayout &DL = getDataLayout();
   const TargetLowering *TLI = nvptxSubtarget->getTargetLowering();
 
   Type *Ty = F->getReturnType();
@@ -365,20 +366,20 @@ void NVPTXAsmPrinter::printReturnValStr(const Function *F, raw_ostream &O) {
 
       O << ".param .b" << size << " func_retval0";
     } else if (isa<PointerType>(Ty)) {
-      O << ".param .b" << TLI->getPointerTy().getSizeInBits()
+      O << ".param .b" << TLI->getPointerTy(DL).getSizeInBits()
         << " func_retval0";
     } else if ((Ty->getTypeID() == Type::StructTyID) || isa<VectorType>(Ty)) {
-       unsigned totalsz = TD->getTypeAllocSize(Ty);
+      unsigned totalsz = DL.getTypeAllocSize(Ty);
        unsigned retAlignment = 0;
        if (!llvm::getAlign(*F, 0, retAlignment))
-         retAlignment = TD->getABITypeAlignment(Ty);
+         retAlignment = DL.getABITypeAlignment(Ty);
        O << ".param .align " << retAlignment << " .b8 func_retval0[" << totalsz
          << "]";
     } else
       llvm_unreachable("Unknown return type");
   } else {
     SmallVector<EVT, 16> vtparts;
-    ComputeValueVTs(*TLI, Ty, vtparts);
+    ComputeValueVTs(*TLI, DL, Ty, vtparts);
     unsigned idx = 0;
     for (unsigned i = 0, e = vtparts.size(); i != e; ++i) {
       unsigned elems = 1;
@@ -417,9 +418,8 @@ void NVPTXAsmPrinter::printReturnValStr(const MachineFunction &MF,
 bool NVPTXAsmPrinter::isLoopHeaderOfNoUnroll(
     const MachineBasicBlock &MBB) const {
   MachineLoopInfo &LI = getAnalysis<MachineLoopInfo>();
-  // TODO: isLoopHeader() should take "const MachineBasicBlock *".
   // We insert .pragma "nounroll" only to the loop header.
-  if (!LI.isLoopHeader(const_cast<MachineBasicBlock *>(&MBB)))
+  if (!LI.isLoopHeader(&MBB))
     return false;
 
   // llvm.loop.unroll.disable is marked on the back edges of a loop. Therefore,
@@ -444,7 +444,7 @@ bool NVPTXAsmPrinter::isLoopHeaderOfNoUnroll(
 void NVPTXAsmPrinter::EmitBasicBlockStart(const MachineBasicBlock &MBB) const {
   AsmPrinter::EmitBasicBlockStart(MBB);
   if (isLoopHeaderOfNoUnroll(MBB))
-    OutStreamer.EmitRawText(StringRef("\t.pragma \"nounroll\";\n"));
+    OutStreamer->EmitRawText(StringRef("\t.pragma \"nounroll\";\n"));
 }
 
 void NVPTXAsmPrinter::EmitFunctionEntryLabel() {
@@ -467,44 +467,44 @@ void NVPTXAsmPrinter::EmitFunctionEntryLabel() {
     printReturnValStr(*MF, O);
   }
 
-  O << *CurrentFnSym;
+  CurrentFnSym->print(O, MAI);
 
   emitFunctionParamList(*MF, O);
 
   if (llvm::isKernelFunction(*F))
     emitKernelFunctionDirectives(*F, O);
 
-  OutStreamer.EmitRawText(O.str());
+  OutStreamer->EmitRawText(O.str());
 
   prevDebugLoc = DebugLoc();
 }
 
 void NVPTXAsmPrinter::EmitFunctionBodyStart() {
   VRegMapping.clear();
-  OutStreamer.EmitRawText(StringRef("{\n"));
+  OutStreamer->EmitRawText(StringRef("{\n"));
   setAndEmitFunctionVirtualRegisters(*MF);
 
   SmallString<128> Str;
   raw_svector_ostream O(Str);
   emitDemotedVars(MF->getFunction(), O);
-  OutStreamer.EmitRawText(O.str());
+  OutStreamer->EmitRawText(O.str());
 }
 
 void NVPTXAsmPrinter::EmitFunctionBodyEnd() {
-  OutStreamer.EmitRawText(StringRef("}\n"));
+  OutStreamer->EmitRawText(StringRef("}\n"));
   VRegMapping.clear();
 }
 
 void NVPTXAsmPrinter::emitImplicitDef(const MachineInstr *MI) const {
   unsigned RegNo = MI->getOperand(0).getReg();
   if (TargetRegisterInfo::isVirtualRegister(RegNo)) {
-    OutStreamer.AddComment(Twine("implicit-def: ") +
-                           getVirtualRegisterName(RegNo));
+    OutStreamer->AddComment(Twine("implicit-def: ") +
+                            getVirtualRegisterName(RegNo));
   } else {
-    OutStreamer.AddComment(Twine("implicit-def: ") +
-                           nvptxSubtarget->getRegisterInfo()->getName(RegNo));
+    OutStreamer->AddComment(Twine("implicit-def: ") +
+                            nvptxSubtarget->getRegisterInfo()->getName(RegNo));
   }
-  OutStreamer.AddBlankLine();
+  OutStreamer->AddBlankLine();
 }
 
 void NVPTXAsmPrinter::emitKernelFunctionDirectives(const Function &F,
@@ -624,7 +624,8 @@ void NVPTXAsmPrinter::emitDeclaration(const Function *F, raw_ostream &O) {
   else
     O << ".func ";
   printReturnValStr(F, O);
-  O << *getSymbol(F) << "\n";
+  getSymbol(F)->print(O, MAI);
+  O << "\n";
   emitFunctionParamList(F, O);
   O << ";\n";
 }
@@ -634,9 +635,7 @@ static bool usedInGlobalVarDef(const Constant *C) {
     return false;
 
   if (const GlobalVariable *GV = dyn_cast<GlobalVariable>(C)) {
-    if (GV->getName() == "llvm.used")
-      return false;
-    return true;
+    return GV->getName() != "llvm.used";
   }
 
   for (const User *U : C->users())
@@ -773,7 +772,7 @@ void NVPTXAsmPrinter::recordAndEmitFilenames(Module &M) {
   DbgFinder.processModule(M);
 
   unsigned i = 1;
-  for (const MDCompileUnit *DIUnit : DbgFinder.compile_units()) {
+  for (const DICompileUnit *DIUnit : DbgFinder.compile_units()) {
     StringRef Filename = DIUnit->getFilename();
     StringRef Dirname = DIUnit->getDirectory();
     SmallString<128> FullPathName = Dirname;
@@ -784,11 +783,11 @@ void NVPTXAsmPrinter::recordAndEmitFilenames(Module &M) {
     if (filenameMap.find(Filename) != filenameMap.end())
       continue;
     filenameMap[Filename] = i;
-    OutStreamer.EmitDwarfFileDirective(i, "", Filename);
+    OutStreamer->EmitDwarfFileDirective(i, "", Filename);
     ++i;
   }
 
-  for (MDSubprogram *SP : DbgFinder.subprograms()) {
+  for (DISubprogram *SP : DbgFinder.subprograms()) {
     StringRef Filename = SP->getFilename();
     StringRef Dirname = SP->getDirectory();
     SmallString<128> FullPathName = Dirname;
@@ -807,7 +806,7 @@ bool NVPTXAsmPrinter::doInitialization(Module &M) {
   // Construct a default subtarget off of the TargetMachine defaults. The
   // rest of NVPTX isn't friendly to change subtargets per function and
   // so the default TargetMachine will have all of the options.
-  StringRef TT = TM.getTargetTriple();
+  const Triple &TT = TM.getTargetTriple();
   StringRef CPU = TM.getTargetCPU();
   StringRef FS = TM.getTargetFeatureString();
   const NVPTXTargetMachine &NTM = static_cast<const NVPTXTargetMachine &>(TM);
@@ -817,7 +816,6 @@ bool NVPTXAsmPrinter::doInitialization(Module &M) {
   raw_svector_ostream OS1(Str1);
 
   MMI = getAnalysisIfAvailable<MachineModuleInfo>();
-  MMI->AnalyzeModule(M);
 
   // We need to call the parent's one explicitly.
   //bool Result = AsmPrinter::doInitialization(M);
@@ -826,27 +824,27 @@ bool NVPTXAsmPrinter::doInitialization(Module &M) {
   const_cast<TargetLoweringObjectFile &>(getObjFileLowering())
       .Initialize(OutContext, TM);
 
-  Mang = new Mangler(TM.getDataLayout());
+  Mang = new Mangler();
 
   // Emit header before any dwarf directives are emitted below.
   emitHeader(M, OS1, STI);
-  OutStreamer.EmitRawText(OS1.str());
+  OutStreamer->EmitRawText(OS1.str());
 
   // Already commented out
   //bool Result = AsmPrinter::doInitialization(M);
 
   // Emit module-level inline asm if it exists.
   if (!M.getModuleInlineAsm().empty()) {
-    OutStreamer.AddComment("Start of file scope inline assembly");
-    OutStreamer.AddBlankLine();
-    OutStreamer.EmitRawText(StringRef(M.getModuleInlineAsm()));
-    OutStreamer.AddBlankLine();
-    OutStreamer.AddComment("End of file scope inline assembly");
-    OutStreamer.AddBlankLine();
+    OutStreamer->AddComment("Start of file scope inline assembly");
+    OutStreamer->AddBlankLine();
+    OutStreamer->EmitRawText(StringRef(M.getModuleInlineAsm()));
+    OutStreamer->AddBlankLine();
+    OutStreamer->AddComment("End of file scope inline assembly");
+    OutStreamer->AddBlankLine();
   }
 
   // If we're not NVCL we're CUDA, go ahead and emit filenames.
-  if (Triple(TM.getTargetTriple()).getOS() != Triple::NVCL)
+  if (TM.getTargetTriple().getOS() != Triple::NVCL)
     recordAndEmitFilenames(M);
 
   GlobalsEmitted = false;
@@ -884,7 +882,7 @@ void NVPTXAsmPrinter::emitGlobals(const Module &M) {
 
   OS2 << '\n';
 
-  OutStreamer.EmitRawText(OS2.str());
+  OutStreamer->EmitRawText(OS2.str());
 }
 
 void NVPTXAsmPrinter::emitHeader(Module &M, raw_ostream &O,
@@ -1029,7 +1027,7 @@ void NVPTXAsmPrinter::printModuleLevelGV(const GlobalVariable *GVar,
       GVar->getName().startswith("nvvm."))
     return;
 
-  const DataLayout *TD = TM.getDataLayout();
+  const DataLayout &DL = getDataLayout();
 
   // GlobalVariables are always constant pointers themselves.
   const PointerType *PTy = GVar->getType();
@@ -1159,7 +1157,7 @@ void NVPTXAsmPrinter::printModuleLevelGV(const GlobalVariable *GVar,
   }
 
   if (GVar->getAlignment() == 0)
-    O << " .align " << (int) TD->getPrefTypeAlignment(ETy);
+    O << " .align " << (int)DL.getPrefTypeAlignment(ETy);
   else
     O << " .align " << GVar->getAlignment();
 
@@ -1171,7 +1169,7 @@ void NVPTXAsmPrinter::printModuleLevelGV(const GlobalVariable *GVar,
     else
       O << getPTXFundamentalTypeStr(ETy, false);
     O << " ";
-    O << *getSymbol(GVar);
+    getSymbol(GVar)->print(O, MAI);
 
     // Ptx allows variable initilization only for constant and global state
     // spaces.
@@ -1179,7 +1177,7 @@ void NVPTXAsmPrinter::printModuleLevelGV(const GlobalVariable *GVar,
       if ((PTy->getAddressSpace() == llvm::ADDRESS_SPACE_GLOBAL) ||
           (PTy->getAddressSpace() == llvm::ADDRESS_SPACE_CONST)) {
         const Constant *Initializer = GVar->getInitializer();
-        // 'undef' is treated as there is no value spefied.
+        // 'undef' is treated as there is no value specified.
         if (!Initializer->isNullValue() && !isa<UndefValue>(Initializer)) {
           O << " = ";
           printScalarConstant(Initializer, O);
@@ -1188,11 +1186,9 @@ void NVPTXAsmPrinter::printModuleLevelGV(const GlobalVariable *GVar,
         // The frontend adds zero-initializer to variables that don't have an
         // initial value, so skip warning for this case.
         if (!GVar->getInitializer()->isNullValue()) {
-          std::string warnMsg =
-              ("initial value of '" + GVar->getName() +
-               "' is not allowed in addrspace(" +
-               Twine(llvm::utostr_32(PTy->getAddressSpace())) + ")").str();
-          report_fatal_error(warnMsg.c_str());
+          report_fatal_error("initial value of '" + GVar->getName() +
+                             "' is not allowed in addrspace(" +
+                             Twine(PTy->getAddressSpace()) + ")");
         }
       }
     }
@@ -1207,7 +1203,7 @@ void NVPTXAsmPrinter::printModuleLevelGV(const GlobalVariable *GVar,
     case Type::StructTyID:
     case Type::ArrayTyID:
     case Type::VectorTyID:
-      ElementSize = TD->getTypeStoreSize(ETy);
+      ElementSize = DL.getTypeStoreSize(ETy);
       // Ptx allows variable initilization only for constant and
       // global state spaces.
       if (((PTy->getAddressSpace() == llvm::ADDRESS_SPACE_GLOBAL) ||
@@ -1219,15 +1215,21 @@ void NVPTXAsmPrinter::printModuleLevelGV(const GlobalVariable *GVar,
           bufferAggregateConstant(Initializer, &aggBuffer);
           if (aggBuffer.numSymbols) {
             if (static_cast<const NVPTXTargetMachine &>(TM).is64Bit()) {
-              O << " .u64 " << *getSymbol(GVar) << "[";
+              O << " .u64 ";
+              getSymbol(GVar)->print(O, MAI);
+              O << "[";
               O << ElementSize / 8;
             } else {
-              O << " .u32 " << *getSymbol(GVar) << "[";
+              O << " .u32 ";
+              getSymbol(GVar)->print(O, MAI);
+              O << "[";
               O << ElementSize / 4;
             }
             O << "]";
           } else {
-            O << " .b8 " << *getSymbol(GVar) << "[";
+            O << " .b8 ";
+            getSymbol(GVar)->print(O, MAI);
+            O << "[";
             O << ElementSize;
             O << "]";
           }
@@ -1235,7 +1237,8 @@ void NVPTXAsmPrinter::printModuleLevelGV(const GlobalVariable *GVar,
           aggBuffer.print();
           O << "}";
         } else {
-          O << " .b8 " << *getSymbol(GVar);
+          O << " .b8 ";
+          getSymbol(GVar)->print(O, MAI);
           if (ElementSize) {
             O << "[";
             O << ElementSize;
@@ -1243,7 +1246,8 @@ void NVPTXAsmPrinter::printModuleLevelGV(const GlobalVariable *GVar,
           }
         }
       } else {
-        O << " .b8 " << *getSymbol(GVar);
+        O << " .b8 ";
+        getSymbol(GVar)->print(O, MAI);
         if (ElementSize) {
           O << "[";
           O << ElementSize;
@@ -1333,7 +1337,7 @@ NVPTXAsmPrinter::getPTXFundamentalTypeStr(const Type *Ty, bool useB4PTR) const {
 void NVPTXAsmPrinter::emitPTXGlobalVariable(const GlobalVariable *GVar,
                                             raw_ostream &O) {
 
-  const DataLayout *TD = TM.getDataLayout();
+  const DataLayout &DL = getDataLayout();
 
   // GlobalVariables are always constant pointers themselves.
   const PointerType *PTy = GVar->getType();
@@ -1342,7 +1346,7 @@ void NVPTXAsmPrinter::emitPTXGlobalVariable(const GlobalVariable *GVar,
   O << ".";
   emitPTXAddressSpace(PTy->getAddressSpace(), O);
   if (GVar->getAlignment() == 0)
-    O << " .align " << (int) TD->getPrefTypeAlignment(ETy);
+    O << " .align " << (int)DL.getPrefTypeAlignment(ETy);
   else
     O << " .align " << GVar->getAlignment();
 
@@ -1350,7 +1354,7 @@ void NVPTXAsmPrinter::emitPTXGlobalVariable(const GlobalVariable *GVar,
     O << " .";
     O << getPTXFundamentalTypeStr(ETy);
     O << " ";
-    O << *getSymbol(GVar);
+    getSymbol(GVar)->print(O, MAI);
     return;
   }
 
@@ -1364,10 +1368,12 @@ void NVPTXAsmPrinter::emitPTXGlobalVariable(const GlobalVariable *GVar,
   case Type::StructTyID:
   case Type::ArrayTyID:
   case Type::VectorTyID:
-    ElementSize = TD->getTypeStoreSize(ETy);
-    O << " .b8 " << *getSymbol(GVar) << "[";
+    ElementSize = DL.getTypeStoreSize(ETy);
+    O << " .b8 ";
+    getSymbol(GVar)->print(O, MAI);
+    O << "[";
     if (ElementSize) {
-      O << itostr(ElementSize);
+      O << ElementSize;
     }
     O << "]";
     break;
@@ -1377,13 +1383,13 @@ void NVPTXAsmPrinter::emitPTXGlobalVariable(const GlobalVariable *GVar,
   return;
 }
 
-static unsigned int getOpenCLAlignment(const DataLayout *TD, Type *Ty) {
+static unsigned int getOpenCLAlignment(const DataLayout &DL, Type *Ty) {
   if (Ty->isSingleValueType())
-    return TD->getPrefTypeAlignment(Ty);
+    return DL.getPrefTypeAlignment(Ty);
 
   const ArrayType *ATy = dyn_cast<ArrayType>(Ty);
   if (ATy)
-    return getOpenCLAlignment(TD, ATy->getElementType());
+    return getOpenCLAlignment(DL, ATy->getElementType());
 
   const StructType *STy = dyn_cast<StructType>(Ty);
   if (STy) {
@@ -1392,7 +1398,7 @@ static unsigned int getOpenCLAlignment(const DataLayout *TD, Type *Ty) {
     // largest alignment.
     for (unsigned i = 0, e = STy->getNumElements(); i != e; i++) {
       Type *ETy = STy->getElementType(i);
-      unsigned int align = getOpenCLAlignment(TD, ETy);
+      unsigned int align = getOpenCLAlignment(DL, ETy);
       if (align > alignStruct)
         alignStruct = align;
     }
@@ -1401,21 +1407,23 @@ static unsigned int getOpenCLAlignment(const DataLayout *TD, Type *Ty) {
 
   const FunctionType *FTy = dyn_cast<FunctionType>(Ty);
   if (FTy)
-    return TD->getPointerPrefAlignment();
-  return TD->getPrefTypeAlignment(Ty);
+    return DL.getPointerPrefAlignment();
+  return DL.getPrefTypeAlignment(Ty);
 }
 
 void NVPTXAsmPrinter::printParamName(Function::const_arg_iterator I,
                                      int paramIndex, raw_ostream &O) {
-  O << *getSymbol(I->getParent()) << "_param_" << paramIndex;
+  getSymbol(I->getParent())->print(O, MAI);
+  O << "_param_" << paramIndex;
 }
 
 void NVPTXAsmPrinter::printParamName(int paramIndex, raw_ostream &O) {
-  O << *CurrentFnSym << "_param_" << paramIndex;
+  CurrentFnSym->print(O, MAI);
+  O << "_param_" << paramIndex;
 }
 
 void NVPTXAsmPrinter::emitFunctionParamList(const Function *F, raw_ostream &O) {
-  const DataLayout *TD = TM.getDataLayout();
+  const DataLayout &DL = getDataLayout();
   const AttributeSet &PAL = F->getAttributes();
   const TargetLowering *TLI = nvptxSubtarget->getTargetLowering();
   Function::const_arg_iterator I, E;
@@ -1423,7 +1431,7 @@ void NVPTXAsmPrinter::emitFunctionParamList(const Function *F, raw_ostream &O) {
   bool first = true;
   bool isKernelFunc = llvm::isKernelFunction(*F);
   bool isABI = (nvptxSubtarget->getSmVersion() >= 20);
-  MVT thePointerTy = TLI->getPointerTy();
+  MVT thePointerTy = TLI->getPointerTy(DL);
 
   O << "(\n";
 
@@ -1445,21 +1453,24 @@ void NVPTXAsmPrinter::emitFunctionParamList(const Function *F, raw_ostream &O) {
               O << "\t.param .u64 .ptr .surfref ";
             else
               O << "\t.param .surfref ";
-            O << *CurrentFnSym << "_param_" << paramIndex;
+            CurrentFnSym->print(O, MAI);
+            O << "_param_" << paramIndex;
           }
           else { // Default image is read_only
             if (nvptxSubtarget->hasImageHandles())
               O << "\t.param .u64 .ptr .texref ";
             else
               O << "\t.param .texref ";
-            O << *CurrentFnSym << "_param_" << paramIndex;
+            CurrentFnSym->print(O, MAI);
+            O << "_param_" << paramIndex;
           }
         } else {
           if (nvptxSubtarget->hasImageHandles())
             O << "\t.param .u64 .ptr .samplerref ";
           else
             O << "\t.param .samplerref ";
-          O << *CurrentFnSym << "_param_" << paramIndex;
+          CurrentFnSym->print(O, MAI);
+          O << "_param_" << paramIndex;
         }
         continue;
       }
@@ -1472,9 +1483,9 @@ void NVPTXAsmPrinter::emitFunctionParamList(const Function *F, raw_ostream &O) {
         // size = typeallocsize of element type
         unsigned align = PAL.getParamAlignment(paramIndex + 1);
         if (align == 0)
-          align = TD->getABITypeAlignment(Ty);
+          align = DL.getABITypeAlignment(Ty);
 
-        unsigned sz = TD->getTypeAllocSize(Ty);
+        unsigned sz = DL.getTypeAllocSize(Ty);
         O << "\t.param .align " << align << " .b8 ";
         printParamName(I, paramIndex, O);
         O << "[" << sz << "]";
@@ -1506,7 +1517,7 @@ void NVPTXAsmPrinter::emitFunctionParamList(const Function *F, raw_ostream &O) {
               O << ".ptr .global ";
               break;
             }
-            O << ".align " << (int) getOpenCLAlignment(TD, ETy) << " ";
+            O << ".align " << (int)getOpenCLAlignment(DL, ETy) << " ";
           }
           printParamName(I, paramIndex, O);
           continue;
@@ -1553,9 +1564,9 @@ void NVPTXAsmPrinter::emitFunctionParamList(const Function *F, raw_ostream &O) {
       // size = typeallocsize of element type
       unsigned align = PAL.getParamAlignment(paramIndex + 1);
       if (align == 0)
-        align = TD->getABITypeAlignment(ETy);
+        align = DL.getABITypeAlignment(ETy);
 
-      unsigned sz = TD->getTypeAllocSize(ETy);
+      unsigned sz = DL.getTypeAllocSize(ETy);
       O << "\t.param .align " << align << " .b8 ";
       printParamName(I, paramIndex, O);
       O << "[" << sz << "]";
@@ -1566,7 +1577,7 @@ void NVPTXAsmPrinter::emitFunctionParamList(const Function *F, raw_ostream &O) {
       // Further, if a part is vector, print the above for
       // each vector element.
       SmallVector<EVT, 16> vtparts;
-      ComputeValueVTs(*TLI, ETy, vtparts);
+      ComputeValueVTs(*TLI, DL, ETy, vtparts);
       for (unsigned i = 0, e = vtparts.size(); i != e; ++i) {
         unsigned elems = 1;
         EVT elemtype = vtparts[i];
@@ -1666,7 +1677,7 @@ void NVPTXAsmPrinter::setAndEmitFunctionVirtualRegisters(
     }
   }
 
-  OutStreamer.EmitRawText(O.str());
+  OutStreamer->EmitRawText(O.str());
 }
 
 void NVPTXAsmPrinter::printFPConstant(const ConstantFP *Fp, raw_ostream &O) {
@@ -1715,10 +1726,10 @@ void NVPTXAsmPrinter::printScalarConstant(const Constant *CPV, raw_ostream &O) {
     }
     if (EmitGeneric && !isa<Function>(CPV) && !IsNonGenericPointer) {
       O << "generic(";
-      O << *getSymbol(GVar);
+      getSymbol(GVar)->print(O, MAI);
       O << ")";
     } else {
-      O << *getSymbol(GVar);
+      getSymbol(GVar)->print(O, MAI);
     }
     return;
   }
@@ -1732,63 +1743,87 @@ void NVPTXAsmPrinter::printScalarConstant(const Constant *CPV, raw_ostream &O) {
     if (const GlobalValue *GVar = dyn_cast<GlobalValue>(v)) {
       if (EmitGeneric && !isa<Function>(v) && !IsNonGenericPointer) {
         O << "generic(";
-        O << *getSymbol(GVar);
+        getSymbol(GVar)->print(O, MAI);
         O << ")";
       } else {
-        O << *getSymbol(GVar);
+        getSymbol(GVar)->print(O, MAI);
       }
       return;
     } else {
-      O << *lowerConstant(CPV);
+      lowerConstant(CPV)->print(O, MAI);
       return;
     }
   }
   llvm_unreachable("Not scalar type found in printScalarConstant()");
 }
 
+// These utility functions assure we get the right sequence of bytes for a given
+// type even for big-endian machines
+template <typename T> static void ConvertIntToBytes(unsigned char *p, T val) {
+  int64_t vp = (int64_t)val;
+  for (unsigned i = 0; i < sizeof(T); ++i) {
+    p[i] = (unsigned char)vp;
+    vp >>= 8;
+  }
+}
+static void ConvertFloatToBytes(unsigned char *p, float val) {
+  int32_t *vp = (int32_t *)&val;
+  for (unsigned i = 0; i < sizeof(int32_t); ++i) {
+    p[i] = (unsigned char)*vp;
+    *vp >>= 8;
+  }
+}
+static void ConvertDoubleToBytes(unsigned char *p, double val) {
+  int64_t *vp = (int64_t *)&val;
+  for (unsigned i = 0; i < sizeof(int64_t); ++i) {
+    p[i] = (unsigned char)*vp;
+    *vp >>= 8;
+  }
+}
+
 void NVPTXAsmPrinter::bufferLEByte(const Constant *CPV, int Bytes,
                                    AggBuffer *aggBuffer) {
 
-  const DataLayout *TD = TM.getDataLayout();
+  const DataLayout &DL = getDataLayout();
 
   if (isa<UndefValue>(CPV) || CPV->isNullValue()) {
-    int s = TD->getTypeAllocSize(CPV->getType());
+    int s = DL.getTypeAllocSize(CPV->getType());
     if (s < Bytes)
       s = Bytes;
     aggBuffer->addZeros(s);
     return;
   }
 
-  unsigned char *ptr;
+  unsigned char ptr[8];
   switch (CPV->getType()->getTypeID()) {
 
   case Type::IntegerTyID: {
     const Type *ETy = CPV->getType();
     if (ETy == Type::getInt8Ty(CPV->getContext())) {
       unsigned char c = (unsigned char)cast<ConstantInt>(CPV)->getZExtValue();
-      ptr = &c;
+      ConvertIntToBytes<>(ptr, c);
       aggBuffer->addBytes(ptr, 1, Bytes);
     } else if (ETy == Type::getInt16Ty(CPV->getContext())) {
       short int16 = (short)cast<ConstantInt>(CPV)->getZExtValue();
-      ptr = (unsigned char *)&int16;
+      ConvertIntToBytes<>(ptr, int16);
       aggBuffer->addBytes(ptr, 2, Bytes);
     } else if (ETy == Type::getInt32Ty(CPV->getContext())) {
       if (const ConstantInt *constInt = dyn_cast<ConstantInt>(CPV)) {
         int int32 = (int)(constInt->getZExtValue());
-        ptr = (unsigned char *)&int32;
+        ConvertIntToBytes<>(ptr, int32);
         aggBuffer->addBytes(ptr, 4, Bytes);
         break;
       } else if (const ConstantExpr *Cexpr = dyn_cast<ConstantExpr>(CPV)) {
         if (const ConstantInt *constInt = dyn_cast<ConstantInt>(
-                ConstantFoldConstantExpression(Cexpr, *TD))) {
+                ConstantFoldConstantExpression(Cexpr, DL))) {
           int int32 = (int)(constInt->getZExtValue());
-          ptr = (unsigned char *)&int32;
+          ConvertIntToBytes<>(ptr, int32);
           aggBuffer->addBytes(ptr, 4, Bytes);
           break;
         }
         if (Cexpr->getOpcode() == Instruction::PtrToInt) {
           Value *v = Cexpr->getOperand(0)->stripPointerCasts();
-          aggBuffer->addSymbol(v);
+          aggBuffer->addSymbol(v, Cexpr->getOperand(0));
           aggBuffer->addZeros(4);
           break;
         }
@@ -1797,20 +1832,20 @@ void NVPTXAsmPrinter::bufferLEByte(const Constant *CPV, int Bytes,
     } else if (ETy == Type::getInt64Ty(CPV->getContext())) {
       if (const ConstantInt *constInt = dyn_cast<ConstantInt>(CPV)) {
         long long int64 = (long long)(constInt->getZExtValue());
-        ptr = (unsigned char *)&int64;
+        ConvertIntToBytes<>(ptr, int64);
         aggBuffer->addBytes(ptr, 8, Bytes);
         break;
       } else if (const ConstantExpr *Cexpr = dyn_cast<ConstantExpr>(CPV)) {
         if (const ConstantInt *constInt = dyn_cast<ConstantInt>(
-                ConstantFoldConstantExpression(Cexpr, *TD))) {
+                ConstantFoldConstantExpression(Cexpr, DL))) {
           long long int64 = (long long)(constInt->getZExtValue());
-          ptr = (unsigned char *)&int64;
+          ConvertIntToBytes<>(ptr, int64);
           aggBuffer->addBytes(ptr, 8, Bytes);
           break;
         }
         if (Cexpr->getOpcode() == Instruction::PtrToInt) {
           Value *v = Cexpr->getOperand(0)->stripPointerCasts();
-          aggBuffer->addSymbol(v);
+          aggBuffer->addSymbol(v, Cexpr->getOperand(0));
           aggBuffer->addZeros(8);
           break;
         }
@@ -1826,11 +1861,11 @@ void NVPTXAsmPrinter::bufferLEByte(const Constant *CPV, int Bytes,
     const Type *Ty = CFP->getType();
     if (Ty == Type::getFloatTy(CPV->getContext())) {
       float float32 = (float) CFP->getValueAPF().convertToFloat();
-      ptr = (unsigned char *)&float32;
+      ConvertFloatToBytes(ptr, float32);
       aggBuffer->addBytes(ptr, 4, Bytes);
     } else if (Ty == Type::getDoubleTy(CPV->getContext())) {
       double float64 = CFP->getValueAPF().convertToDouble();
-      ptr = (unsigned char *)&float64;
+      ConvertDoubleToBytes(ptr, float64);
       aggBuffer->addBytes(ptr, 8, Bytes);
     } else {
       llvm_unreachable("unsupported fp const type");
@@ -1839,12 +1874,12 @@ void NVPTXAsmPrinter::bufferLEByte(const Constant *CPV, int Bytes,
   }
   case Type::PointerTyID: {
     if (const GlobalValue *GVar = dyn_cast<GlobalValue>(CPV)) {
-      aggBuffer->addSymbol(GVar);
+      aggBuffer->addSymbol(GVar, GVar);
     } else if (const ConstantExpr *Cexpr = dyn_cast<ConstantExpr>(CPV)) {
       const Value *v = Cexpr->stripPointerCasts();
-      aggBuffer->addSymbol(v);
+      aggBuffer->addSymbol(v, Cexpr);
     }
-    unsigned int s = TD->getTypeAllocSize(CPV->getType());
+    unsigned int s = DL.getTypeAllocSize(CPV->getType());
     aggBuffer->addZeros(s);
     break;
   }
@@ -1854,7 +1889,7 @@ void NVPTXAsmPrinter::bufferLEByte(const Constant *CPV, int Bytes,
   case Type::StructTyID: {
     if (isa<ConstantArray>(CPV) || isa<ConstantVector>(CPV) ||
         isa<ConstantStruct>(CPV) || isa<ConstantDataSequential>(CPV)) {
-      int ElementSize = TD->getTypeAllocSize(CPV->getType());
+      int ElementSize = DL.getTypeAllocSize(CPV->getType());
       bufferAggregateConstant(CPV, aggBuffer);
       if (Bytes > ElementSize)
         aggBuffer->addZeros(Bytes - ElementSize);
@@ -1872,7 +1907,7 @@ void NVPTXAsmPrinter::bufferLEByte(const Constant *CPV, int Bytes,
 
 void NVPTXAsmPrinter::bufferAggregateConstant(const Constant *CPV,
                                               AggBuffer *aggBuffer) {
-  const DataLayout *TD = TM.getDataLayout();
+  const DataLayout &DL = getDataLayout();
   int Bytes;
 
   // Old constants
@@ -1897,12 +1932,12 @@ void NVPTXAsmPrinter::bufferAggregateConstant(const Constant *CPV,
       StructType *ST = cast<StructType>(CPV->getType());
       for (unsigned i = 0, e = CPV->getNumOperands(); i != e; ++i) {
         if (i == (e - 1))
-          Bytes = TD->getStructLayout(ST)->getElementOffset(0) +
-                  TD->getTypeAllocSize(ST) -
-                  TD->getStructLayout(ST)->getElementOffset(i);
+          Bytes = DL.getStructLayout(ST)->getElementOffset(0) +
+                  DL.getTypeAllocSize(ST) -
+                  DL.getStructLayout(ST)->getElementOffset(i);
         else
-          Bytes = TD->getStructLayout(ST)->getElementOffset(i + 1) -
-                  TD->getStructLayout(ST)->getElementOffset(i);
+          Bytes = DL.getStructLayout(ST)->getElementOffset(i + 1) -
+                  DL.getStructLayout(ST)->getElementOffset(i);
         bufferLEByte(cast<Constant>(CPV->getOperand(i)), Bytes, aggBuffer);
       }
     }
@@ -1918,12 +1953,9 @@ bool NVPTXAsmPrinter::isImageType(const Type *Ty) {
 
   std::map<const Type *, std::string>::iterator PI = TypeNameMap.find(Ty);
 
-  if (PI != TypeNameMap.end() && (!PI->second.compare("struct._image1d_t") ||
-                                  !PI->second.compare("struct._image2d_t") ||
-                                  !PI->second.compare("struct._image3d_t")))
-    return true;
-
-  return false;
+  return PI != TypeNameMap.end() && (!PI->second.compare("struct._image1d_t") ||
+                                     !PI->second.compare("struct._image2d_t") ||
+                                     !PI->second.compare("struct._image3d_t"));
 }
 
 
@@ -1983,6 +2015,212 @@ bool NVPTXAsmPrinter::ignoreLoc(const MachineInstr &MI) {
   return false;
 }
 
+/// lowerConstantForGV - Return an MCExpr for the given Constant.  This is mostly
+/// a copy from AsmPrinter::lowerConstant, except customized to only handle
+/// expressions that are representable in PTX and create
+/// NVPTXGenericMCSymbolRefExpr nodes for addrspacecast instructions.
+const MCExpr *
+NVPTXAsmPrinter::lowerConstantForGV(const Constant *CV, bool ProcessingGeneric) {
+  MCContext &Ctx = OutContext;
+
+  if (CV->isNullValue() || isa<UndefValue>(CV))
+    return MCConstantExpr::create(0, Ctx);
+
+  if (const ConstantInt *CI = dyn_cast<ConstantInt>(CV))
+    return MCConstantExpr::create(CI->getZExtValue(), Ctx);
+
+  if (const GlobalValue *GV = dyn_cast<GlobalValue>(CV)) {
+    const MCSymbolRefExpr *Expr =
+      MCSymbolRefExpr::create(getSymbol(GV), Ctx);
+    if (ProcessingGeneric) {
+      return NVPTXGenericMCSymbolRefExpr::create(Expr, Ctx);
+    } else {
+      return Expr;
+    }
+  }
+
+  const ConstantExpr *CE = dyn_cast<ConstantExpr>(CV);
+  if (!CE) {
+    llvm_unreachable("Unknown constant value to lower!");
+  }
+
+  switch (CE->getOpcode()) {
+  default:
+    // If the code isn't optimized, there may be outstanding folding
+    // opportunities. Attempt to fold the expression using DataLayout as a
+    // last resort before giving up.
+    if (Constant *C = ConstantFoldConstantExpression(CE, getDataLayout()))
+      if (C != CE)
+        return lowerConstantForGV(C, ProcessingGeneric);
+
+    // Otherwise report the problem to the user.
+    {
+      std::string S;
+      raw_string_ostream OS(S);
+      OS << "Unsupported expression in static initializer: ";
+      CE->printAsOperand(OS, /*PrintType=*/false,
+                     !MF ? nullptr : MF->getFunction()->getParent());
+      report_fatal_error(OS.str());
+    }
+
+  case Instruction::AddrSpaceCast: {
+    // Strip the addrspacecast and pass along the operand
+    PointerType *DstTy = cast<PointerType>(CE->getType());
+    if (DstTy->getAddressSpace() == 0) {
+      return lowerConstantForGV(cast<const Constant>(CE->getOperand(0)), true);
+    }
+    std::string S;
+    raw_string_ostream OS(S);
+    OS << "Unsupported expression in static initializer: ";
+    CE->printAsOperand(OS, /*PrintType=*/ false,
+                       !MF ? 0 : MF->getFunction()->getParent());
+    report_fatal_error(OS.str());
+  }
+
+  case Instruction::GetElementPtr: {
+    const DataLayout &DL = getDataLayout();
+
+    // Generate a symbolic expression for the byte address
+    APInt OffsetAI(DL.getPointerTypeSizeInBits(CE->getType()), 0);
+    cast<GEPOperator>(CE)->accumulateConstantOffset(DL, OffsetAI);
+
+    const MCExpr *Base = lowerConstantForGV(CE->getOperand(0),
+                                            ProcessingGeneric);
+    if (!OffsetAI)
+      return Base;
+
+    int64_t Offset = OffsetAI.getSExtValue();
+    return MCBinaryExpr::createAdd(Base, MCConstantExpr::create(Offset, Ctx),
+                                   Ctx);
+  }
+
+  case Instruction::Trunc:
+    // We emit the value and depend on the assembler to truncate the generated
+    // expression properly.  This is important for differences between
+    // blockaddress labels.  Since the two labels are in the same function, it
+    // is reasonable to treat their delta as a 32-bit value.
+    // FALL THROUGH.
+  case Instruction::BitCast:
+    return lowerConstantForGV(CE->getOperand(0), ProcessingGeneric);
+
+  case Instruction::IntToPtr: {
+    const DataLayout &DL = getDataLayout();
+
+    // Handle casts to pointers by changing them into casts to the appropriate
+    // integer type.  This promotes constant folding and simplifies this code.
+    Constant *Op = CE->getOperand(0);
+    Op = ConstantExpr::getIntegerCast(Op, DL.getIntPtrType(CV->getType()),
+                                      false/*ZExt*/);
+    return lowerConstantForGV(Op, ProcessingGeneric);
+  }
+
+  case Instruction::PtrToInt: {
+    const DataLayout &DL = getDataLayout();
+
+    // Support only foldable casts to/from pointers that can be eliminated by
+    // changing the pointer to the appropriately sized integer type.
+    Constant *Op = CE->getOperand(0);
+    Type *Ty = CE->getType();
+
+    const MCExpr *OpExpr = lowerConstantForGV(Op, ProcessingGeneric);
+
+    // We can emit the pointer value into this slot if the slot is an
+    // integer slot equal to the size of the pointer.
+    if (DL.getTypeAllocSize(Ty) == DL.getTypeAllocSize(Op->getType()))
+      return OpExpr;
+
+    // Otherwise the pointer is smaller than the resultant integer, mask off
+    // the high bits so we are sure to get a proper truncation if the input is
+    // a constant expr.
+    unsigned InBits = DL.getTypeAllocSizeInBits(Op->getType());
+    const MCExpr *MaskExpr = MCConstantExpr::create(~0ULL >> (64-InBits), Ctx);
+    return MCBinaryExpr::createAnd(OpExpr, MaskExpr, Ctx);
+  }
+
+  // The MC library also has a right-shift operator, but it isn't consistently
+  // signed or unsigned between different targets.
+  case Instruction::Add: {
+    const MCExpr *LHS = lowerConstantForGV(CE->getOperand(0), ProcessingGeneric);
+    const MCExpr *RHS = lowerConstantForGV(CE->getOperand(1), ProcessingGeneric);
+    switch (CE->getOpcode()) {
+    default: llvm_unreachable("Unknown binary operator constant cast expr");
+    case Instruction::Add: return MCBinaryExpr::createAdd(LHS, RHS, Ctx);
+    }
+  }
+  }
+}
+
+// Copy of MCExpr::print customized for NVPTX
+void NVPTXAsmPrinter::printMCExpr(const MCExpr &Expr, raw_ostream &OS) {
+  switch (Expr.getKind()) {
+  case MCExpr::Target:
+    return cast<MCTargetExpr>(&Expr)->printImpl(OS, MAI);
+  case MCExpr::Constant:
+    OS << cast<MCConstantExpr>(Expr).getValue();
+    return;
+
+  case MCExpr::SymbolRef: {
+    const MCSymbolRefExpr &SRE = cast<MCSymbolRefExpr>(Expr);
+    const MCSymbol &Sym = SRE.getSymbol();
+    Sym.print(OS, MAI);
+    return;
+  }
+
+  case MCExpr::Unary: {
+    const MCUnaryExpr &UE = cast<MCUnaryExpr>(Expr);
+    switch (UE.getOpcode()) {
+    case MCUnaryExpr::LNot:  OS << '!'; break;
+    case MCUnaryExpr::Minus: OS << '-'; break;
+    case MCUnaryExpr::Not:   OS << '~'; break;
+    case MCUnaryExpr::Plus:  OS << '+'; break;
+    }
+    printMCExpr(*UE.getSubExpr(), OS);
+    return;
+  }
+
+  case MCExpr::Binary: {
+    const MCBinaryExpr &BE = cast<MCBinaryExpr>(Expr);
+
+    // Only print parens around the LHS if it is non-trivial.
+    if (isa<MCConstantExpr>(BE.getLHS()) || isa<MCSymbolRefExpr>(BE.getLHS()) ||
+        isa<NVPTXGenericMCSymbolRefExpr>(BE.getLHS())) {
+      printMCExpr(*BE.getLHS(), OS);
+    } else {
+      OS << '(';
+      printMCExpr(*BE.getLHS(), OS);
+      OS<< ')';
+    }
+
+    switch (BE.getOpcode()) {
+    case MCBinaryExpr::Add:
+      // Print "X-42" instead of "X+-42".
+      if (const MCConstantExpr *RHSC = dyn_cast<MCConstantExpr>(BE.getRHS())) {
+        if (RHSC->getValue() < 0) {
+          OS << RHSC->getValue();
+          return;
+        }
+      }
+
+      OS <<  '+';
+      break;
+    default: llvm_unreachable("Unhandled binary operator");
+    }
+
+    // Only print parens around the LHS if it is non-trivial.
+    if (isa<MCConstantExpr>(BE.getRHS()) || isa<MCSymbolRefExpr>(BE.getRHS())) {
+      printMCExpr(*BE.getRHS(), OS);
+    } else {
+      OS << '(';
+      printMCExpr(*BE.getRHS(), OS);
+      OS << ')';
+    }
+    return;
+  }
+  }
+
+  llvm_unreachable("Invalid expression kind!");
+}
+
 /// PrintAsmOperand - Print out an operand for an inline asm expression.
 ///
 bool NVPTXAsmPrinter::PrintAsmOperand(const MachineInstr *MI, unsigned OpNo,
@@ -2049,11 +2287,11 @@ void NVPTXAsmPrinter::printOperand(const MachineInstr *MI, int opNum,
     break;
 
   case MachineOperand::MO_GlobalAddress:
-    O << *getSymbol(MO.getGlobal());
+    getSymbol(MO.getGlobal())->print(O, MAI);
     break;
 
   case MachineOperand::MO_MachineBasicBlock:
-    O << *MO.getMBB()->getSymbol();
+    MO.getMBB()->getSymbol()->print(O, MAI);
     return;
 
   default:
@@ -2087,7 +2325,7 @@ void NVPTXAsmPrinter::emitSrcInText(StringRef filename, unsigned line) {
   temp << " ";
   temp << reader->readLine(line);
   temp << "\n";
-  this->OutStreamer.EmitRawText(temp.str());
+  this->OutStreamer->EmitRawText(temp.str());
 }
 
 LineReader *NVPTXAsmPrinter::getReader(std::string filename) {
diff --git a/lib/Target/NVPTX/NVPTXAsmPrinter.h b/lib/Target/NVPTX/NVPTXAsmPrinter.h
index 9b11e70b2ae..f6f7685e76f 100644
--- a/lib/Target/NVPTX/NVPTXAsmPrinter.h
+++ b/lib/Target/NVPTX/NVPTXAsmPrinter.h
@@ -40,6 +40,7 @@
 // (subclass of MCStreamer).
 
 namespace llvm {
+  class MCOperand;
 
 class LineReader {
 private:
@@ -86,6 +87,14 @@ class LLVM_LIBRARY_VISIBILITY NVPTXAsmPrinter : public AsmPrinter {
     std::vector<unsigned char> buffer; // the buffer
     SmallVector<unsigned, 4> symbolPosInBuffer;
     SmallVector<const Value *, 4> Symbols;
+    // SymbolsBeforeStripping[i] is the original form of Symbols[i] before
+    // stripping pointer casts, i.e.,
+    // Symbols[i] == SymbolsBeforeStripping[i]->stripPointerCasts().
+    //
+    // We need to keep these values because AggBuffer::print decides whether to
+    // emit a "generic()" cast for Symbols[i] depending on the address space of
+    // SymbolsBeforeStripping[i].
+    SmallVector<const Value *, 4> SymbolsBeforeStripping;
     unsigned curpos;
     raw_ostream &O;
     NVPTXAsmPrinter &AP;
@@ -119,9 +128,10 @@ class LLVM_LIBRARY_VISIBILITY NVPTXAsmPrinter : public AsmPrinter {
       }
       return curpos;
     }
-    void addSymbol(const Value *GVar) {
+    void addSymbol(const Value *GVar, const Value *GVarBeforeStripping) {
       symbolPosInBuffer.push_back(curpos);
       Symbols.push_back(GVar);
+      SymbolsBeforeStripping.push_back(GVarBeforeStripping);
       numSymbols++;
     }
     void print() {
@@ -145,22 +155,25 @@ class LLVM_LIBRARY_VISIBILITY NVPTXAsmPrinter : public AsmPrinter {
             O << ", ";
           if (pos == nextSymbolPos) {
             const Value *v = Symbols[nSym];
+            const Value *v0 = SymbolsBeforeStripping[nSym];
             if (const GlobalValue *GVar = dyn_cast<GlobalValue>(v)) {
               MCSymbol *Name = AP.getSymbol(GVar);
-              PointerType *PTy = dyn_cast<PointerType>(GVar->getType());
-              bool IsNonGenericPointer = false;
+              PointerType *PTy = dyn_cast<PointerType>(v0->getType());
+              bool IsNonGenericPointer = false; // Is v0 a non-generic pointer?
               if (PTy && PTy->getAddressSpace() != 0) {
                 IsNonGenericPointer = true;
               }
               if (EmitGeneric && !isa<Function>(v) && !IsNonGenericPointer) {
                 O << "generic(";
-                O << *Name;
+                Name->print(O, AP.MAI);
                 O << ")";
               } else {
-                O << *Name;
+                Name->print(O, AP.MAI);
               }
-            } else if (const ConstantExpr *Cexpr = dyn_cast<ConstantExpr>(v)) {
-              O << *AP.lowerConstant(Cexpr);
+            } else if (const ConstantExpr *CExpr = dyn_cast<ConstantExpr>(v0)) {
+              const MCExpr *Expr =
+                AP.lowerConstantForGV(cast<Constant>(CExpr), false);
+              AP.printMCExpr(*Expr, O);
             } else
               llvm_unreachable("symbol type unknown");
             nSym++;
@@ -231,6 +244,10 @@ class LLVM_LIBRARY_VISIBILITY NVPTXAsmPrinter : public AsmPrinter {
   bool PrintAsmMemoryOperand(const MachineInstr *MI, unsigned OpNo,
                              unsigned AsmVariant, const char *ExtraCode,
                              raw_ostream &) override;
+
+  const MCExpr *lowerConstantForGV(const Constant *CV, bool ProcessingGeneric);
+  void printMCExpr(const MCExpr &Expr, raw_ostream &OS);
+
 protected:
   bool doInitialization(Module &M) override;
   bool doFinalization(Module &M) override;
diff --git a/lib/Target/NVPTX/NVPTXFavorNonGenericAddrSpaces.cpp b/lib/Target/NVPTX/NVPTXFavorNonGenericAddrSpaces.cpp
index ae63caec132..69a229e32f4 100644
--- a/lib/Target/NVPTX/NVPTXFavorNonGenericAddrSpaces.cpp
+++ b/lib/Target/NVPTX/NVPTXFavorNonGenericAddrSpaces.cpp
@@ -10,34 +10,54 @@
 // When a load/store accesses the generic address space, checks whether the
 // address is casted from a non-generic address space. If so, remove this
 // addrspacecast because accessing non-generic address spaces is typically
-// faster. Besides seeking addrspacecasts, this optimization also traces into
-// the base pointer of a GEP.
+// faster. Besides removing addrspacecasts directly used by loads/stores, this
+// optimization also recursively traces into a GEP's pointer operand and a
+// bitcast's source to find more eliminable addrspacecasts.
 //
 // For instance, the code below loads a float from an array allocated in
 // addrspace(3).
 //
-// %0 = addrspacecast [10 x float] addrspace(3)* @a to [10 x float]*
-// %1 = gep [10 x float]* %0, i64 0, i64 %i
-// %2 = load float* %1 ; emits ld.f32
+//   %0 = addrspacecast [10 x float] addrspace(3)* @a to [10 x float]*
+//   %1 = gep [10 x float]* %0, i64 0, i64 %i
+//   %2 = bitcast float* %1 to i32*
+//   %3 = load i32* %2 ; emits ld.u32
 //
-// First, function hoistAddrSpaceCastFromGEP reorders the addrspacecast
-// and the GEP to expose more optimization opportunities to function
+// First, function hoistAddrSpaceCastFrom reorders the addrspacecast, the GEP,
+// and the bitcast to expose more optimization opportunities to function
 // optimizeMemoryInst. The intermediate code looks like:
 //
-// %0 = gep [10 x float] addrspace(3)* @a, i64 0, i64 %i
-// %1 = addrspacecast float addrspace(3)* %0 to float*
-// %2 = load float* %1 ; still emits ld.f32, but will be optimized shortly
+//   %0 = gep [10 x float] addrspace(3)* @a, i64 0, i64 %i
+//   %1 = bitcast float addrspace(3)* %0 to i32 addrspace(3)*
+//   %2 = addrspacecast i32 addrspace(3)* %1 to i32*
+//   %3 = load i32* %2 ; still emits ld.u32, but will be optimized shortly
 //
 // Then, function optimizeMemoryInstruction detects a load from addrspacecast'ed
 // generic pointers, and folds the load and the addrspacecast into a load from
 // the original address space. The final code looks like:
 //
-// %0 = gep [10 x float] addrspace(3)* @a, i64 0, i64 %i
-// %2 = load float addrspace(3)* %0 ; emits ld.shared.f32
+//   %0 = gep [10 x float] addrspace(3)* @a, i64 0, i64 %i
+//   %1 = bitcast float addrspace(3)* %0 to i32 addrspace(3)*
+//   %3 = load i32 addrspace(3)* %1 ; emits ld.shared.f32
 //
 // This pass may remove an addrspacecast in a different BB. Therefore, we
 // implement it as a FunctionPass.
 //
+// TODO:
+// The current implementation doesn't handle PHINodes. Eliminating
+// addrspacecasts used by PHINodes is trickier because PHINodes can introduce
+// loops in data flow. For example,
+//
+//     %generic.input = addrspacecast float addrspace(3)* %input to float*
+//   loop:
+//     %y = phi [ %generic.input, %y2 ]
+//     %y2 = getelementptr %y, 1
+//     %v = load %y2
+//     br ..., label %loop, ...
+//
+// Marking %y2 shared depends on marking %y shared, but %y also data-flow
+// depends on %y2. We probably need an iterative fix-point algorithm on handle
+// this case.
+//
 //===----------------------------------------------------------------------===//
 
 #include "NVPTX.h"
@@ -62,17 +82,30 @@ class NVPTXFavorNonGenericAddrSpaces : public FunctionPass {
 public:
   static char ID;
   NVPTXFavorNonGenericAddrSpaces() : FunctionPass(ID) {}
-
   bool runOnFunction(Function &F) override;
 
+private:
   /// Optimizes load/store instructions. Idx is the index of the pointer operand
   /// (0 for load, and 1 for store). Returns true if it changes anything.
   bool optimizeMemoryInstruction(Instruction *I, unsigned Idx);
-  /// Transforms "gep (addrspacecast X), indices" into "addrspacecast (gep X,
-  /// indices)".  This reordering exposes to optimizeMemoryInstruction more
-  /// optimization opportunities on loads and stores. Returns true if it changes
-  /// the program.
-  bool hoistAddrSpaceCastFromGEP(GEPOperator *GEP);
+  /// Recursively traces into a GEP's pointer operand or a bitcast's source to
+  /// find an eliminable addrspacecast, and hoists that addrspacecast to the
+  /// outermost level. For example, this function transforms
+  ///   bitcast(gep(gep(addrspacecast(X))))
+  /// to
+  ///   addrspacecast(bitcast(gep(gep(X)))).
+  ///
+  /// This reordering exposes to optimizeMemoryInstruction more
+  /// optimization opportunities on loads and stores.
+  ///
+  /// If this function succesfully hoists an eliminable addrspacecast or V is
+  /// already such an addrspacecast, it returns the transformed value (which is
+  /// guaranteed to be an addrspacecast); otherwise, it returns nullptr.
+  Value *hoistAddrSpaceCastFrom(Value *V, int Depth = 0);
+  /// Helper function for GEPs.
+  Value *hoistAddrSpaceCastFromGEP(GEPOperator *GEP, int Depth);
+  /// Helper function for bitcasts.
+  Value *hoistAddrSpaceCastFromBitCast(BitCastOperator *BC, int Depth);
 };
 }
 
@@ -85,11 +118,12 @@ INITIALIZE_PASS(NVPTXFavorNonGenericAddrSpaces, "nvptx-favor-non-generic",
                 "Remove unnecessary non-generic-to-generic addrspacecasts",
                 false, false)
 
-// Decides whether removing Cast is valid and beneficial. Cast can be an
-// instruction or a constant expression.
-static bool IsEliminableAddrSpaceCast(Operator *Cast) {
-  // Returns false if not even an addrspacecast.
-  if (Cast->getOpcode() != Instruction::AddrSpaceCast)
+// Decides whether V is an addrspacecast and shortcutting V in load/store is
+// valid and beneficial.
+static bool isEliminableAddrSpaceCast(Value *V) {
+  // Returns false if V is not even an addrspacecast.
+  Operator *Cast = dyn_cast<Operator>(V);
+  if (Cast == nullptr || Cast->getOpcode() != Instruction::AddrSpaceCast)
     return false;
 
   Value *Src = Cast->getOperand(0);
@@ -108,46 +142,109 @@ static bool IsEliminableAddrSpaceCast(Operator *Cast) {
           DestTy->getAddressSpace() == AddressSpace::ADDRESS_SPACE_GENERIC);
 }
 
-bool NVPTXFavorNonGenericAddrSpaces::hoistAddrSpaceCastFromGEP(
-    GEPOperator *GEP) {
-  Operator *Cast = dyn_cast<Operator>(GEP->getPointerOperand());
-  if (!Cast)
-    return false;
+Value *NVPTXFavorNonGenericAddrSpaces::hoistAddrSpaceCastFromGEP(
+    GEPOperator *GEP, int Depth) {
+  Value *NewOperand =
+      hoistAddrSpaceCastFrom(GEP->getPointerOperand(), Depth + 1);
+  if (NewOperand == nullptr)
+    return nullptr;
 
-  if (!IsEliminableAddrSpaceCast(Cast))
-    return false;
+  // hoistAddrSpaceCastFrom returns an eliminable addrspacecast or nullptr.
+  assert(isEliminableAddrSpaceCast(NewOperand));
+  Operator *Cast = cast<Operator>(NewOperand);
 
   SmallVector<Value *, 8> Indices(GEP->idx_begin(), GEP->idx_end());
+  Value *NewASC;
   if (Instruction *GEPI = dyn_cast<Instruction>(GEP)) {
-    // %1 = gep (addrspacecast X), indices
+    // GEP = gep (addrspacecast X), indices
     // =>
-    // %0 = gep X, indices
-    // %1 = addrspacecast %0
-    GetElementPtrInst *NewGEPI = GetElementPtrInst::Create(
+    // NewGEP = gep X, indices
+    // NewASC = addrspacecast NewGEP
+    GetElementPtrInst *NewGEP = GetElementPtrInst::Create(
         GEP->getSourceElementType(), Cast->getOperand(0), Indices,
-        GEP->getName(), GEPI);
-    NewGEPI->setIsInBounds(GEP->isInBounds());
-    GEP->replaceAllUsesWith(
-        new AddrSpaceCastInst(NewGEPI, GEP->getType(), "", GEPI));
+        "", GEPI);
+    NewGEP->setIsInBounds(GEP->isInBounds());
+    NewASC = new AddrSpaceCastInst(NewGEP, GEP->getType(), "", GEPI);
+    NewASC->takeName(GEP);
+    // Without RAUWing GEP, the compiler would visit GEP again and emit
+    // redundant instructions. This is exercised in test @rauw in
+    // access-non-generic.ll.
+    GEP->replaceAllUsesWith(NewASC);
   } else {
     // GEP is a constant expression.
-    Constant *NewGEPCE = ConstantExpr::getGetElementPtr(
+    Constant *NewGEP = ConstantExpr::getGetElementPtr(
         GEP->getSourceElementType(), cast<Constant>(Cast->getOperand(0)),
         Indices, GEP->isInBounds());
-    GEP->replaceAllUsesWith(
-        ConstantExpr::getAddrSpaceCast(NewGEPCE, GEP->getType()));
+    NewASC = ConstantExpr::getAddrSpaceCast(NewGEP, GEP->getType());
   }
+  return NewASC;
+}
 
-  return true;
+Value *NVPTXFavorNonGenericAddrSpaces::hoistAddrSpaceCastFromBitCast(
+    BitCastOperator *BC, int Depth) {
+  Value *NewOperand = hoistAddrSpaceCastFrom(BC->getOperand(0), Depth + 1);
+  if (NewOperand == nullptr)
+    return nullptr;
+
+  // hoistAddrSpaceCastFrom returns an eliminable addrspacecast or nullptr.
+  assert(isEliminableAddrSpaceCast(NewOperand));
+  Operator *Cast = cast<Operator>(NewOperand);
+
+  // Cast  = addrspacecast Src
+  // BC    = bitcast Cast
+  //   =>
+  // Cast' = bitcast Src
+  // BC'   = addrspacecast Cast'
+  Value *Src = Cast->getOperand(0);
+  Type *TypeOfNewCast =
+      PointerType::get(BC->getType()->getPointerElementType(),
+                       Src->getType()->getPointerAddressSpace());
+  Value *NewBC;
+  if (BitCastInst *BCI = dyn_cast<BitCastInst>(BC)) {
+    Value *NewCast = new BitCastInst(Src, TypeOfNewCast, "", BCI);
+    NewBC = new AddrSpaceCastInst(NewCast, BC->getType(), "", BCI);
+    NewBC->takeName(BC);
+    // Without RAUWing BC, the compiler would visit BC again and emit
+    // redundant instructions. This is exercised in test @rauw in
+    // access-non-generic.ll.
+    BC->replaceAllUsesWith(NewBC);
+  } else {
+    // BC is a constant expression.
+    Constant *NewCast =
+        ConstantExpr::getBitCast(cast<Constant>(Src), TypeOfNewCast);
+    NewBC = ConstantExpr::getAddrSpaceCast(NewCast, BC->getType());
+  }
+  return NewBC;
+}
+
+Value *NVPTXFavorNonGenericAddrSpaces::hoistAddrSpaceCastFrom(Value *V,
+                                                              int Depth) {
+  // Returns V if V is already an eliminable addrspacecast.
+  if (isEliminableAddrSpaceCast(V))
+    return V;
+
+  // Limit the depth to prevent this recursive function from running too long.
+  const int MaxDepth = 20;
+  if (Depth >= MaxDepth)
+    return nullptr;
+
+  // If V is a GEP or bitcast, hoist the addrspacecast if any from its pointer
+  // operand. This enables optimizeMemoryInstruction to shortcut addrspacecasts
+  // that are not directly used by the load/store.
+  if (GEPOperator *GEP = dyn_cast<GEPOperator>(V))
+    return hoistAddrSpaceCastFromGEP(GEP, Depth);
+
+  if (BitCastOperator *BC = dyn_cast<BitCastOperator>(V))
+    return hoistAddrSpaceCastFromBitCast(BC, Depth);
+
+  return nullptr;
 }
 
 bool NVPTXFavorNonGenericAddrSpaces::optimizeMemoryInstruction(Instruction *MI,
                                                                unsigned Idx) {
-  // If the pointer operand is a GEP, hoist the addrspacecast if any from the
-  // GEP to expose more optimization opportunites.
-  if (GEPOperator *GEP = dyn_cast<GEPOperator>(MI->getOperand(Idx))) {
-    hoistAddrSpaceCastFromGEP(GEP);
-  }
+  Value *NewOperand = hoistAddrSpaceCastFrom(MI->getOperand(Idx));
+  if (NewOperand == nullptr)
+    return false;
 
   // load/store (addrspacecast X) => load/store X if shortcutting the
   // addrspacecast is valid and can improve performance.
@@ -159,14 +256,10 @@ bool NVPTXFavorNonGenericAddrSpaces::optimizeMemoryInstruction(Instruction *MI,
   // %2 = load float addrspace(3)* %0
   //
   // Note: the addrspacecast can also be a constant expression.
-  if (Operator *Cast = dyn_cast<Operator>(MI->getOperand(Idx))) {
-    if (IsEliminableAddrSpaceCast(Cast)) {
-      MI->setOperand(Idx, Cast->getOperand(0));
-      return true;
-    }
-  }
-
-  return false;
+  assert(isEliminableAddrSpaceCast(NewOperand));
+  Operator *ASC = dyn_cast<Operator>(NewOperand);
+  MI->setOperand(Idx, ASC->getOperand(0));
+  return true;
 }
 
 bool NVPTXFavorNonGenericAddrSpaces::runOnFunction(Function &F) {
diff --git a/lib/Target/NVPTX/NVPTXFrameLowering.cpp b/lib/Target/NVPTX/NVPTXFrameLowering.cpp
index 34d3a66adce..9b34aef3fde 100644
--- a/lib/Target/NVPTX/NVPTXFrameLowering.cpp
+++ b/lib/Target/NVPTX/NVPTXFrameLowering.cpp
@@ -31,38 +31,37 @@ NVPTXFrameLowering::NVPTXFrameLowering()
 
 bool NVPTXFrameLowering::hasFP(const MachineFunction &MF) const { return true; }
 
-void NVPTXFrameLowering::emitPrologue(MachineFunction &MF) const {
+void NVPTXFrameLowering::emitPrologue(MachineFunction &MF,
+                                      MachineBasicBlock &MBB) const {
   if (MF.getFrameInfo()->hasStackObjects()) {
-    MachineBasicBlock &MBB = MF.front();
-    // Insert "mov.u32 %SP, %Depot"
-    MachineBasicBlock::iterator MBBI = MBB.begin();
+    assert(&MF.front() == &MBB && "Shrink-wrapping not yet supported");
+    MachineInstr *MI = MBB.begin();
+    MachineRegisterInfo &MR = MF.getRegInfo();
+
     // This instruction really occurs before first instruction
     // in the BB, so giving it no debug location.
     DebugLoc dl = DebugLoc();
 
-    MachineRegisterInfo &MRI = MF.getRegInfo();
-
-    // mov %SPL, %depot;
-    // cvta.local %SP, %SPL;
-    if (static_cast<const NVPTXTargetMachine &>(MF.getTarget()).is64Bit()) {
-      unsigned LocalReg = MRI.createVirtualRegister(&NVPTX::Int64RegsRegClass);
-      MachineInstr *MI =
-          BuildMI(MBB, MBBI, dl, MF.getSubtarget().getInstrInfo()->get(
-                                     NVPTX::cvta_local_yes_64),
-                  NVPTX::VRFrame).addReg(LocalReg);
-      BuildMI(MBB, MI, dl,
-              MF.getSubtarget().getInstrInfo()->get(NVPTX::MOV_DEPOT_ADDR_64),
-              LocalReg).addImm(MF.getFunctionNumber());
-    } else {
-      unsigned LocalReg = MRI.createVirtualRegister(&NVPTX::Int32RegsRegClass);
-      MachineInstr *MI =
-          BuildMI(MBB, MBBI, dl,
-                  MF.getSubtarget().getInstrInfo()->get(NVPTX::cvta_local_yes),
-                  NVPTX::VRFrame).addReg(LocalReg);
-      BuildMI(MBB, MI, dl,
-              MF.getSubtarget().getInstrInfo()->get(NVPTX::MOV_DEPOT_ADDR),
-              LocalReg).addImm(MF.getFunctionNumber());
+    // Emits
+    //   mov %SPL, %depot;
+    //   cvta.local %SP, %SPL;
+    // for local address accesses in MF.
+    bool Is64Bit =
+        static_cast<const NVPTXTargetMachine &>(MF.getTarget()).is64Bit();
+    unsigned CvtaLocalOpcode =
+        (Is64Bit ? NVPTX::cvta_local_yes_64 : NVPTX::cvta_local_yes);
+    unsigned MovDepotOpcode =
+        (Is64Bit ? NVPTX::MOV_DEPOT_ADDR_64 : NVPTX::MOV_DEPOT_ADDR);
+    if (!MR.use_empty(NVPTX::VRFrame)) {
+      // If %SP is not used, do not bother emitting "cvta.local %SP, %SPL".
+      MI = BuildMI(MBB, MI, dl,
+                   MF.getSubtarget().getInstrInfo()->get(CvtaLocalOpcode),
+                   NVPTX::VRFrame)
+               .addReg(NVPTX::VRFrameLocal);
     }
+    BuildMI(MBB, MI, dl, MF.getSubtarget().getInstrInfo()->get(MovDepotOpcode),
+            NVPTX::VRFrameLocal)
+        .addImm(MF.getFunctionNumber());
   }
 }
 
diff --git a/lib/Target/NVPTX/NVPTXFrameLowering.h b/lib/Target/NVPTX/NVPTXFrameLowering.h
index d1e0a5ceb30..14f8bb7b98f 100644
--- a/lib/Target/NVPTX/NVPTXFrameLowering.h
+++ b/lib/Target/NVPTX/NVPTXFrameLowering.h
@@ -23,7 +23,7 @@ class NVPTXFrameLowering : public TargetFrameLowering {
   explicit NVPTXFrameLowering();
 
   bool hasFP(const MachineFunction &MF) const override;
-  void emitPrologue(MachineFunction &MF) const override;
+  void emitPrologue(MachineFunction &MF, MachineBasicBlock &MBB) const override;
   void emitEpilogue(MachineFunction &MF, MachineBasicBlock &MBB) const override;
 
   void
diff --git a/lib/Target/NVPTX/NVPTXISelDAGToDAG.cpp b/lib/Target/NVPTX/NVPTXISelDAGToDAG.cpp
index 52c5e1b8788..ddf77b91e80 100644
--- a/lib/Target/NVPTX/NVPTXISelDAGToDAG.cpp
+++ b/lib/Target/NVPTX/NVPTXISelDAGToDAG.cpp
@@ -12,6 +12,7 @@
 //===----------------------------------------------------------------------===//
 
 #include "NVPTXISelDAGToDAG.h"
+#include "llvm/Analysis/ValueTracking.h"
 #include "llvm/IR/GlobalValue.h"
 #include "llvm/IR/Instructions.h"
 #include "llvm/Support/CommandLine.h"
@@ -544,6 +545,21 @@ static unsigned int getCodeAddrSpace(MemSDNode *N) {
   return NVPTX::PTXLdStInstCode::GENERIC;
 }
 
+static bool canLowerToLDG(MemSDNode *N, const NVPTXSubtarget &Subtarget,
+                          unsigned codeAddrSpace, const DataLayout &DL) {
+  if (!Subtarget.hasLDG() || codeAddrSpace != NVPTX::PTXLdStInstCode::GLOBAL) {
+    return false;
+  }
+
+  // Check whether load operates on a readonly argument.
+  bool canUseLDG = false;
+  if (const Argument *A = dyn_cast<const Argument>(
+          GetUnderlyingObject(N->getMemOperand()->getValue(), DL)))
+    canUseLDG = A->onlyReadsMemory() && A->hasNoAliasAttr();
+
+  return canUseLDG;
+}
+
 SDNode *NVPTXDAGToDAGISel::SelectIntrinsicNoChain(SDNode *N) {
   unsigned IID = cast<ConstantSDNode>(N->getOperand(0))->getZExtValue();
   switch (IID) {
@@ -613,6 +629,10 @@ SDNode *NVPTXDAGToDAGISel::SelectAddrSpaceCast(SDNode *N) {
       Opc =
           TM.is64Bit() ? NVPTX::cvta_to_local_yes_64 : NVPTX::cvta_to_local_yes;
       break;
+    case ADDRESS_SPACE_PARAM:
+      Opc = TM.is64Bit() ? NVPTX::nvvm_ptr_gen_to_param_64
+                         : NVPTX::nvvm_ptr_gen_to_param;
+      break;
     }
     return CurDAG->getMachineNode(Opc, SDLoc(N), N->getValueType(0), Src);
   }
@@ -634,6 +654,10 @@ SDNode *NVPTXDAGToDAGISel::SelectLoad(SDNode *N) {
   // Address Space Setting
   unsigned int codeAddrSpace = getCodeAddrSpace(LD);
 
+  if (canLowerToLDG(LD, *Subtarget, codeAddrSpace, CurDAG->getDataLayout())) {
+    return SelectLDGLDU(N);
+  }
+
   // Volatile Setting
   // - .volatile is only availalble for .global and .shared
   bool isVolatile = LD->isVolatile();
@@ -703,9 +727,9 @@ SDNode *NVPTXDAGToDAGISel::SelectLoad(SDNode *N) {
     default:
       return nullptr;
     }
-    SDValue Ops[] = { getI32Imm(isVolatile), getI32Imm(codeAddrSpace),
-                      getI32Imm(vecType), getI32Imm(fromType),
-                      getI32Imm(fromTypeWidth), Addr, Chain };
+    SDValue Ops[] = { getI32Imm(isVolatile, dl), getI32Imm(codeAddrSpace, dl),
+                      getI32Imm(vecType, dl), getI32Imm(fromType, dl),
+                      getI32Imm(fromTypeWidth, dl), Addr, Chain };
     NVPTXLD = CurDAG->getMachineNode(Opcode, dl, TargetVT, MVT::Other, Ops);
   } else if (TM.is64Bit() ? SelectADDRsi64(N1.getNode(), N1, Base, Offset)
                           : SelectADDRsi(N1.getNode(), N1, Base, Offset)) {
@@ -731,9 +755,9 @@ SDNode *NVPTXDAGToDAGISel::SelectLoad(SDNode *N) {
     default:
       return nullptr;
     }
-    SDValue Ops[] = { getI32Imm(isVolatile), getI32Imm(codeAddrSpace),
-                      getI32Imm(vecType), getI32Imm(fromType),
-                      getI32Imm(fromTypeWidth), Base, Offset, Chain };
+    SDValue Ops[] = { getI32Imm(isVolatile, dl), getI32Imm(codeAddrSpace, dl),
+                      getI32Imm(vecType, dl), getI32Imm(fromType, dl),
+                      getI32Imm(fromTypeWidth, dl), Base, Offset, Chain };
     NVPTXLD = CurDAG->getMachineNode(Opcode, dl, TargetVT, MVT::Other, Ops);
   } else if (TM.is64Bit() ? SelectADDRri64(N1.getNode(), N1, Base, Offset)
                           : SelectADDRri(N1.getNode(), N1, Base, Offset)) {
@@ -784,9 +808,9 @@ SDNode *NVPTXDAGToDAGISel::SelectLoad(SDNode *N) {
         return nullptr;
       }
     }
-    SDValue Ops[] = { getI32Imm(isVolatile), getI32Imm(codeAddrSpace),
-                      getI32Imm(vecType), getI32Imm(fromType),
-                      getI32Imm(fromTypeWidth), Base, Offset, Chain };
+    SDValue Ops[] = { getI32Imm(isVolatile, dl), getI32Imm(codeAddrSpace, dl),
+                      getI32Imm(vecType, dl), getI32Imm(fromType, dl),
+                      getI32Imm(fromTypeWidth, dl), Base, Offset, Chain };
     NVPTXLD = CurDAG->getMachineNode(Opcode, dl, TargetVT, MVT::Other, Ops);
   } else {
     if (TM.is64Bit()) {
@@ -836,9 +860,9 @@ SDNode *NVPTXDAGToDAGISel::SelectLoad(SDNode *N) {
         return nullptr;
       }
     }
-    SDValue Ops[] = { getI32Imm(isVolatile), getI32Imm(codeAddrSpace),
-                      getI32Imm(vecType), getI32Imm(fromType),
-                      getI32Imm(fromTypeWidth), N1, Chain };
+    SDValue Ops[] = { getI32Imm(isVolatile, dl), getI32Imm(codeAddrSpace, dl),
+                      getI32Imm(vecType, dl), getI32Imm(fromType, dl),
+                      getI32Imm(fromTypeWidth, dl), N1, Chain };
     NVPTXLD = CurDAG->getMachineNode(Opcode, dl, TargetVT, MVT::Other, Ops);
   }
 
@@ -868,6 +892,10 @@ SDNode *NVPTXDAGToDAGISel::SelectLoadVector(SDNode *N) {
   // Address Space Setting
   unsigned int CodeAddrSpace = getCodeAddrSpace(MemSD);
 
+  if (canLowerToLDG(MemSD, *Subtarget, CodeAddrSpace, CurDAG->getDataLayout())) {
+    return SelectLDGLDU(N);
+  }
+
   // Volatile Setting
   // - .volatile is only availalble for .global and .shared
   bool IsVolatile = MemSD->isVolatile();
@@ -962,9 +990,9 @@ SDNode *NVPTXDAGToDAGISel::SelectLoadVector(SDNode *N) {
       break;
     }
 
-    SDValue Ops[] = { getI32Imm(IsVolatile), getI32Imm(CodeAddrSpace),
-                      getI32Imm(VecType), getI32Imm(FromType),
-                      getI32Imm(FromTypeWidth), Addr, Chain };
+    SDValue Ops[] = { getI32Imm(IsVolatile, DL), getI32Imm(CodeAddrSpace, DL),
+                      getI32Imm(VecType, DL), getI32Imm(FromType, DL),
+                      getI32Imm(FromTypeWidth, DL), Addr, Chain };
     LD = CurDAG->getMachineNode(Opcode, DL, N->getVTList(), Ops);
   } else if (TM.is64Bit() ? SelectADDRsi64(Op1.getNode(), Op1, Base, Offset)
                           : SelectADDRsi(Op1.getNode(), Op1, Base, Offset)) {
@@ -1015,9 +1043,9 @@ SDNode *NVPTXDAGToDAGISel::SelectLoadVector(SDNode *N) {
       break;
     }
 
-    SDValue Ops[] = { getI32Imm(IsVolatile), getI32Imm(CodeAddrSpace),
-                      getI32Imm(VecType), getI32Imm(FromType),
-                      getI32Imm(FromTypeWidth), Base, Offset, Chain };
+    SDValue Ops[] = { getI32Imm(IsVolatile, DL), getI32Imm(CodeAddrSpace, DL),
+                      getI32Imm(VecType, DL), getI32Imm(FromType, DL),
+                      getI32Imm(FromTypeWidth, DL), Base, Offset, Chain };
     LD = CurDAG->getMachineNode(Opcode, DL, N->getVTList(), Ops);
   } else if (TM.is64Bit() ? SelectADDRri64(Op1.getNode(), Op1, Base, Offset)
                           : SelectADDRri(Op1.getNode(), Op1, Base, Offset)) {
@@ -1117,9 +1145,9 @@ SDNode *NVPTXDAGToDAGISel::SelectLoadVector(SDNode *N) {
       }
     }
 
-    SDValue Ops[] = { getI32Imm(IsVolatile), getI32Imm(CodeAddrSpace),
-                      getI32Imm(VecType), getI32Imm(FromType),
-                      getI32Imm(FromTypeWidth), Base, Offset, Chain };
+    SDValue Ops[] = { getI32Imm(IsVolatile, DL), getI32Imm(CodeAddrSpace, DL),
+                      getI32Imm(VecType, DL), getI32Imm(FromType, DL),
+                      getI32Imm(FromTypeWidth, DL), Base, Offset, Chain };
 
     LD = CurDAG->getMachineNode(Opcode, DL, N->getVTList(), Ops);
   } else {
@@ -1219,9 +1247,9 @@ SDNode *NVPTXDAGToDAGISel::SelectLoadVector(SDNode *N) {
       }
     }
 
-    SDValue Ops[] = { getI32Imm(IsVolatile), getI32Imm(CodeAddrSpace),
-                      getI32Imm(VecType), getI32Imm(FromType),
-                      getI32Imm(FromTypeWidth), Op1, Chain };
+    SDValue Ops[] = { getI32Imm(IsVolatile, DL), getI32Imm(CodeAddrSpace, DL),
+                      getI32Imm(VecType, DL), getI32Imm(FromType, DL),
+                      getI32Imm(FromTypeWidth, DL), Op1, Chain };
     LD = CurDAG->getMachineNode(Opcode, DL, N->getVTList(), Ops);
   }
 
@@ -1421,6 +1449,7 @@ SDNode *NVPTXDAGToDAGISel::SelectLDGLDU(SDNode *N) {
       switch (N->getOpcode()) {
       default:
         return nullptr;
+      case ISD::LOAD:
       case ISD::INTRINSIC_W_CHAIN:
         if (IsLDG) {
           switch (EltVT.getSimpleVT().SimpleTy) {
@@ -1470,6 +1499,7 @@ SDNode *NVPTXDAGToDAGISel::SelectLDGLDU(SDNode *N) {
           }
         }
         break;
+      case NVPTXISD::LoadV2:
       case NVPTXISD::LDGV2:
         switch (EltVT.getSimpleVT().SimpleTy) {
         default:
@@ -1518,6 +1548,7 @@ SDNode *NVPTXDAGToDAGISel::SelectLDGLDU(SDNode *N) {
           break;
         }
         break;
+      case NVPTXISD::LoadV4:
       case NVPTXISD::LDGV4:
         switch (EltVT.getSimpleVT().SimpleTy) {
         default:
@@ -1559,6 +1590,7 @@ SDNode *NVPTXDAGToDAGISel::SelectLDGLDU(SDNode *N) {
       switch (N->getOpcode()) {
       default:
         return nullptr;
+      case ISD::LOAD:
       case ISD::INTRINSIC_W_CHAIN:
         if (IsLDG) {
           switch (EltVT.getSimpleVT().SimpleTy) {
@@ -1608,6 +1640,7 @@ SDNode *NVPTXDAGToDAGISel::SelectLDGLDU(SDNode *N) {
           }
         }
         break;
+      case NVPTXISD::LoadV2:
       case NVPTXISD::LDGV2:
         switch (EltVT.getSimpleVT().SimpleTy) {
         default:
@@ -1656,6 +1689,7 @@ SDNode *NVPTXDAGToDAGISel::SelectLDGLDU(SDNode *N) {
           break;
         }
         break;
+      case NVPTXISD::LoadV4:
       case NVPTXISD::LDGV4:
         switch (EltVT.getSimpleVT().SimpleTy) {
         default:
@@ -1703,6 +1737,7 @@ SDNode *NVPTXDAGToDAGISel::SelectLDGLDU(SDNode *N) {
       switch (N->getOpcode()) {
       default:
         return nullptr;
+      case ISD::LOAD:
       case ISD::INTRINSIC_W_CHAIN:
         if (IsLDG) {
           switch (EltVT.getSimpleVT().SimpleTy) {
@@ -1752,6 +1787,7 @@ SDNode *NVPTXDAGToDAGISel::SelectLDGLDU(SDNode *N) {
           }
         }
         break;
+      case NVPTXISD::LoadV2:
       case NVPTXISD::LDGV2:
         switch (EltVT.getSimpleVT().SimpleTy) {
         default:
@@ -1800,6 +1836,7 @@ SDNode *NVPTXDAGToDAGISel::SelectLDGLDU(SDNode *N) {
           break;
         }
         break;
+      case NVPTXISD::LoadV4:
       case NVPTXISD::LDGV4:
         switch (EltVT.getSimpleVT().SimpleTy) {
         default:
@@ -1841,6 +1878,7 @@ SDNode *NVPTXDAGToDAGISel::SelectLDGLDU(SDNode *N) {
       switch (N->getOpcode()) {
       default:
         return nullptr;
+      case ISD::LOAD:
       case ISD::INTRINSIC_W_CHAIN:
         if (IsLDG) {
           switch (EltVT.getSimpleVT().SimpleTy) {
@@ -1890,6 +1928,7 @@ SDNode *NVPTXDAGToDAGISel::SelectLDGLDU(SDNode *N) {
           }
         }
         break;
+      case NVPTXISD::LoadV2:
       case NVPTXISD::LDGV2:
         switch (EltVT.getSimpleVT().SimpleTy) {
         default:
@@ -1938,6 +1977,7 @@ SDNode *NVPTXDAGToDAGISel::SelectLDGLDU(SDNode *N) {
           break;
         }
         break;
+      case NVPTXISD::LoadV4:
       case NVPTXISD::LDGV4:
         switch (EltVT.getSimpleVT().SimpleTy) {
         default:
@@ -2068,9 +2108,10 @@ SDNode *NVPTXDAGToDAGISel::SelectStore(SDNode *N) {
     default:
       return nullptr;
     }
-    SDValue Ops[] = { N1, getI32Imm(isVolatile), getI32Imm(codeAddrSpace),
-                      getI32Imm(vecType), getI32Imm(toType),
-                      getI32Imm(toTypeWidth), Addr, Chain };
+    SDValue Ops[] = { N1, getI32Imm(isVolatile, dl),
+                      getI32Imm(codeAddrSpace, dl), getI32Imm(vecType, dl),
+                      getI32Imm(toType, dl), getI32Imm(toTypeWidth, dl), Addr,
+                      Chain };
     NVPTXST = CurDAG->getMachineNode(Opcode, dl, MVT::Other, Ops);
   } else if (TM.is64Bit() ? SelectADDRsi64(N2.getNode(), N2, Base, Offset)
                           : SelectADDRsi(N2.getNode(), N2, Base, Offset)) {
@@ -2096,9 +2137,10 @@ SDNode *NVPTXDAGToDAGISel::SelectStore(SDNode *N) {
     default:
       return nullptr;
     }
-    SDValue Ops[] = { N1, getI32Imm(isVolatile), getI32Imm(codeAddrSpace),
-                      getI32Imm(vecType), getI32Imm(toType),
-                      getI32Imm(toTypeWidth), Base, Offset, Chain };
+    SDValue Ops[] = { N1, getI32Imm(isVolatile, dl),
+                      getI32Imm(codeAddrSpace, dl), getI32Imm(vecType, dl),
+                      getI32Imm(toType, dl), getI32Imm(toTypeWidth, dl), Base,
+                      Offset, Chain };
     NVPTXST = CurDAG->getMachineNode(Opcode, dl, MVT::Other, Ops);
   } else if (TM.is64Bit() ? SelectADDRri64(N2.getNode(), N2, Base, Offset)
                           : SelectADDRri(N2.getNode(), N2, Base, Offset)) {
@@ -2149,9 +2191,10 @@ SDNode *NVPTXDAGToDAGISel::SelectStore(SDNode *N) {
         return nullptr;
       }
     }
-    SDValue Ops[] = { N1, getI32Imm(isVolatile), getI32Imm(codeAddrSpace),
-                      getI32Imm(vecType), getI32Imm(toType),
-                      getI32Imm(toTypeWidth), Base, Offset, Chain };
+    SDValue Ops[] = { N1, getI32Imm(isVolatile, dl),
+                      getI32Imm(codeAddrSpace, dl), getI32Imm(vecType, dl),
+                      getI32Imm(toType, dl), getI32Imm(toTypeWidth, dl), Base,
+                      Offset, Chain };
     NVPTXST = CurDAG->getMachineNode(Opcode, dl, MVT::Other, Ops);
   } else {
     if (TM.is64Bit()) {
@@ -2201,9 +2244,10 @@ SDNode *NVPTXDAGToDAGISel::SelectStore(SDNode *N) {
         return nullptr;
       }
     }
-    SDValue Ops[] = { N1, getI32Imm(isVolatile), getI32Imm(codeAddrSpace),
-                      getI32Imm(vecType), getI32Imm(toType),
-                      getI32Imm(toTypeWidth), N2, Chain };
+    SDValue Ops[] = { N1, getI32Imm(isVolatile, dl),
+                      getI32Imm(codeAddrSpace, dl), getI32Imm(vecType, dl),
+                      getI32Imm(toType, dl), getI32Imm(toTypeWidth, dl), N2,
+                      Chain };
     NVPTXST = CurDAG->getMachineNode(Opcode, dl, MVT::Other, Ops);
   }
 
@@ -2277,11 +2321,11 @@ SDNode *NVPTXDAGToDAGISel::SelectStoreVector(SDNode *N) {
     return nullptr;
   }
 
-  StOps.push_back(getI32Imm(IsVolatile));
-  StOps.push_back(getI32Imm(CodeAddrSpace));
-  StOps.push_back(getI32Imm(VecType));
-  StOps.push_back(getI32Imm(ToType));
-  StOps.push_back(getI32Imm(ToTypeWidth));
+  StOps.push_back(getI32Imm(IsVolatile, DL));
+  StOps.push_back(getI32Imm(CodeAddrSpace, DL));
+  StOps.push_back(getI32Imm(VecType, DL));
+  StOps.push_back(getI32Imm(ToType, DL));
+  StOps.push_back(getI32Imm(ToTypeWidth, DL));
 
   if (SelectDirectAddr(N2, Addr)) {
     switch (N->getOpcode()) {
@@ -2710,13 +2754,11 @@ SDNode *NVPTXDAGToDAGISel::SelectLoadParam(SDNode *Node) {
   unsigned OffsetVal = cast<ConstantSDNode>(Offset)->getZExtValue();
 
   SmallVector<SDValue, 2> Ops;
-  Ops.push_back(CurDAG->getTargetConstant(OffsetVal, MVT::i32));
+  Ops.push_back(CurDAG->getTargetConstant(OffsetVal, DL, MVT::i32));
   Ops.push_back(Chain);
   Ops.push_back(Flag);
 
-  SDNode *Ret =
-      CurDAG->getMachineNode(Opc, DL, VTs, Ops);
-  return Ret;
+  return CurDAG->getMachineNode(Opc, DL, VTs, Ops);
 }
 
 SDNode *NVPTXDAGToDAGISel::SelectStoreRetval(SDNode *N) {
@@ -2746,7 +2788,7 @@ SDNode *NVPTXDAGToDAGISel::SelectStoreRetval(SDNode *N) {
   SmallVector<SDValue, 6> Ops;
   for (unsigned i = 0; i < NumElts; ++i)
     Ops.push_back(N->getOperand(i + 2));
-  Ops.push_back(CurDAG->getTargetConstant(OffsetVal, MVT::i32));
+  Ops.push_back(CurDAG->getTargetConstant(OffsetVal, DL, MVT::i32));
   Ops.push_back(Chain);
 
   // Determine target opcode
@@ -2874,8 +2916,8 @@ SDNode *NVPTXDAGToDAGISel::SelectStoreParam(SDNode *N) {
   SmallVector<SDValue, 8> Ops;
   for (unsigned i = 0; i < NumElts; ++i)
     Ops.push_back(N->getOperand(i + 3));
-  Ops.push_back(CurDAG->getTargetConstant(ParamVal, MVT::i32));
-  Ops.push_back(CurDAG->getTargetConstant(OffsetVal, MVT::i32));
+  Ops.push_back(CurDAG->getTargetConstant(ParamVal, DL, MVT::i32));
+  Ops.push_back(CurDAG->getTargetConstant(OffsetVal, DL, MVT::i32));
   Ops.push_back(Chain);
   Ops.push_back(Flag);
 
@@ -2970,7 +3012,7 @@ SDNode *NVPTXDAGToDAGISel::SelectStoreParam(SDNode *N) {
   // the selected StoreParam node.
   case NVPTXISD::StoreParamU32: {
     Opcode = NVPTX::StoreParamI32;
-    SDValue CvtNone = CurDAG->getTargetConstant(NVPTX::PTXCvtMode::NONE,
+    SDValue CvtNone = CurDAG->getTargetConstant(NVPTX::PTXCvtMode::NONE, DL,
                                                 MVT::i32);
     SDNode *Cvt = CurDAG->getMachineNode(NVPTX::CVT_u32_u16, DL,
                                          MVT::i32, Ops[0], CvtNone);
@@ -2979,7 +3021,7 @@ SDNode *NVPTXDAGToDAGISel::SelectStoreParam(SDNode *N) {
   }
   case NVPTXISD::StoreParamS32: {
     Opcode = NVPTX::StoreParamI32;
-    SDValue CvtNone = CurDAG->getTargetConstant(NVPTX::PTXCvtMode::NONE,
+    SDValue CvtNone = CurDAG->getTargetConstant(NVPTX::PTXCvtMode::NONE, DL,
                                                 MVT::i32);
     SDNode *Cvt = CurDAG->getMachineNode(NVPTX::CVT_s32_s16, DL,
                                          MVT::i32, Ops[0], CvtNone);
@@ -4727,6 +4769,7 @@ SDNode *NVPTXDAGToDAGISel::SelectSurfaceIntrinsic(SDNode *N) {
 /// SelectBFE - Look for instruction sequences that can be made more efficient
 /// by using the 'bfe' (bit-field extract) PTX instruction
 SDNode *NVPTXDAGToDAGISel::SelectBFE(SDNode *N) {
+  SDLoc DL(N);
   SDValue LHS = N->getOperand(0);
   SDValue RHS = N->getOperand(1);
   SDValue Len;
@@ -4758,7 +4801,7 @@ SDNode *NVPTXDAGToDAGISel::SelectBFE(SDNode *N) {
 
     // How many bits are in our mask?
     uint64_t NumBits = countTrailingOnes(MaskVal);
-    Len = CurDAG->getTargetConstant(NumBits, MVT::i32);
+    Len = CurDAG->getTargetConstant(NumBits, DL, MVT::i32);
 
     if (LHS.getOpcode() == ISD::SRL || LHS.getOpcode() == ISD::SRA) {
       // We have a 'srl/and' pair, extract the effective start bit and length
@@ -4776,7 +4819,7 @@ SDNode *NVPTXDAGToDAGISel::SelectBFE(SDNode *N) {
           // emitting the srl/and pair.
           return NULL;
         }
-        Start = CurDAG->getTargetConstant(StartVal, MVT::i32);
+        Start = CurDAG->getTargetConstant(StartVal, DL, MVT::i32);
       } else {
         // Do not handle the case where the shift amount (can be zero if no srl
         // was found) is not constant. We could handle this case, but it would
@@ -4841,8 +4884,8 @@ SDNode *NVPTXDAGToDAGISel::SelectBFE(SDNode *N) {
       }
 
       Val = AndLHS;
-      Start = CurDAG->getTargetConstant(ShiftAmt, MVT::i32);
-      Len = CurDAG->getTargetConstant(NumBits, MVT::i32);
+      Start = CurDAG->getTargetConstant(ShiftAmt, DL, MVT::i32);
+      Len = CurDAG->getTargetConstant(NumBits, DL, MVT::i32);
     } else if (LHS->getOpcode() == ISD::SHL) {
       // Here, we have a pattern like:
       //
@@ -4882,10 +4925,10 @@ SDNode *NVPTXDAGToDAGISel::SelectBFE(SDNode *N) {
       }
 
       Start =
-        CurDAG->getTargetConstant(OuterShiftAmt - InnerShiftAmt, MVT::i32);
+        CurDAG->getTargetConstant(OuterShiftAmt - InnerShiftAmt, DL, MVT::i32);
       Len =
         CurDAG->getTargetConstant(Val.getValueType().getSizeInBits() -
-                                  OuterShiftAmt, MVT::i32);
+                                  OuterShiftAmt, DL, MVT::i32);
 
       if (N->getOpcode() == ISD::SRA) {
         // If we have a arithmetic right shift, we need to use the signed bfe
@@ -4926,10 +4969,7 @@ SDNode *NVPTXDAGToDAGISel::SelectBFE(SDNode *N) {
     Val, Start, Len
   };
 
-  SDNode *Ret =
-    CurDAG->getMachineNode(Opc, SDLoc(N), N->getVTList(), Ops);
-
-  return Ret;
+  return CurDAG->getMachineNode(Opc, DL, N->getVTList(), Ops);
 }
 
 // SelectDirectAddr - Match a direct address for DAG.
@@ -4961,7 +5001,8 @@ bool NVPTXDAGToDAGISel::SelectADDRsi_imp(
     if (ConstantSDNode *CN = dyn_cast<ConstantSDNode>(Addr.getOperand(1))) {
       SDValue base = Addr.getOperand(0);
       if (SelectDirectAddr(base, Base)) {
-        Offset = CurDAG->getTargetConstant(CN->getZExtValue(), mvt);
+        Offset = CurDAG->getTargetConstant(CN->getZExtValue(), SDLoc(OpNode),
+                                           mvt);
         return true;
       }
     }
@@ -4986,7 +5027,7 @@ bool NVPTXDAGToDAGISel::SelectADDRri_imp(
     SDNode *OpNode, SDValue Addr, SDValue &Base, SDValue &Offset, MVT mvt) {
   if (FrameIndexSDNode *FIN = dyn_cast<FrameIndexSDNode>(Addr)) {
     Base = CurDAG->getTargetFrameIndex(FIN->getIndex(), mvt);
-    Offset = CurDAG->getTargetConstant(0, mvt);
+    Offset = CurDAG->getTargetConstant(0, SDLoc(OpNode), mvt);
     return true;
   }
   if (Addr.getOpcode() == ISD::TargetExternalSymbol ||
@@ -5004,7 +5045,8 @@ bool NVPTXDAGToDAGISel::SelectADDRri_imp(
         Base = CurDAG->getTargetFrameIndex(FIN->getIndex(), mvt);
       else
         Base = Addr.getOperand(0);
-      Offset = CurDAG->getTargetConstant(CN->getZExtValue(), mvt);
+      Offset = CurDAG->getTargetConstant(CN->getZExtValue(), SDLoc(OpNode),
+                                         mvt);
       return true;
     }
   }
@@ -5049,7 +5091,7 @@ bool NVPTXDAGToDAGISel::SelectInlineAsmMemoryOperand(
   case InlineAsm::Constraint_m: // memory
     if (SelectDirectAddr(Op, Op0)) {
       OutOps.push_back(Op0);
-      OutOps.push_back(CurDAG->getTargetConstant(0, MVT::i32));
+      OutOps.push_back(CurDAG->getTargetConstant(0, SDLoc(Op), MVT::i32));
       return false;
     }
     if (SelectADDRri(Op.getNode(), Op, Op0, Op1)) {
diff --git a/lib/Target/NVPTX/NVPTXISelDAGToDAG.h b/lib/Target/NVPTX/NVPTXISelDAGToDAG.h
index 6d845c9d5d1..fe20580c83a 100644
--- a/lib/Target/NVPTX/NVPTXISelDAGToDAG.h
+++ b/lib/Target/NVPTX/NVPTXISelDAGToDAG.h
@@ -71,8 +71,8 @@ class LLVM_LIBRARY_VISIBILITY NVPTXDAGToDAGISel : public SelectionDAGISel {
   SDNode *SelectSurfaceIntrinsic(SDNode *N);
   SDNode *SelectBFE(SDNode *N);
         
-  inline SDValue getI32Imm(unsigned Imm) {
-    return CurDAG->getTargetConstant(Imm, MVT::i32);
+  inline SDValue getI32Imm(unsigned Imm, SDLoc DL) {
+    return CurDAG->getTargetConstant(Imm, DL, MVT::i32);
   }
 
   // Match direct address complex pattern.
diff --git a/lib/Target/NVPTX/NVPTXISelLowering.cpp b/lib/Target/NVPTX/NVPTXISelLowering.cpp
index 8b0665708b9..a0c2bf31be7 100644
--- a/lib/Target/NVPTX/NVPTXISelLowering.cpp
+++ b/lib/Target/NVPTX/NVPTXISelLowering.cpp
@@ -80,14 +80,14 @@ static bool IsPTXVectorType(MVT VT) {
 /// NOTE: This is a band-aid for code that expects ComputeValueVTs to return the
 /// same number of types as the Ins/Outs arrays in LowerFormalArguments,
 /// LowerCall, and LowerReturn.
-static void ComputePTXValueVTs(const TargetLowering &TLI, Type *Ty,
-                               SmallVectorImpl<EVT> &ValueVTs,
+static void ComputePTXValueVTs(const TargetLowering &TLI, const DataLayout &DL,
+                               Type *Ty, SmallVectorImpl<EVT> &ValueVTs,
                                SmallVectorImpl<uint64_t> *Offsets = nullptr,
                                uint64_t StartingOffset = 0) {
   SmallVector<EVT, 16> TempVTs;
   SmallVector<uint64_t, 16> TempOffsets;
 
-  ComputeValueVTs(TLI, Ty, TempVTs, &TempOffsets, StartingOffset);
+  ComputeValueVTs(TLI, DL, Ty, TempVTs, &TempOffsets, StartingOffset);
   for (unsigned i = 0, e = TempVTs.size(); i != e; ++i) {
     EVT VT = TempVTs[i];
     uint64_t Off = TempOffsets[i];
@@ -206,7 +206,14 @@ NVPTXTargetLowering::NVPTXTargetLowering(const NVPTXTargetMachine &TM,
   setLoadExtAction(ISD::EXTLOAD, MVT::f32, MVT::f16, Expand);
   setLoadExtAction(ISD::EXTLOAD, MVT::f64, MVT::f16, Expand);
   setLoadExtAction(ISD::EXTLOAD, MVT::f64, MVT::f32, Expand);
+  setLoadExtAction(ISD::EXTLOAD, MVT::v2f32, MVT::v2f16, Expand);
+  setLoadExtAction(ISD::EXTLOAD, MVT::v2f64, MVT::v2f16, Expand);
+  setLoadExtAction(ISD::EXTLOAD, MVT::v2f64, MVT::v2f32, Expand);
+  setLoadExtAction(ISD::EXTLOAD, MVT::v4f32, MVT::v4f16, Expand);
+  setLoadExtAction(ISD::EXTLOAD, MVT::v4f64, MVT::v4f16, Expand);
+  setLoadExtAction(ISD::EXTLOAD, MVT::v4f64, MVT::v4f32, Expand);
   // Turn FP truncstore into trunc + store.
+  // FIXME: vector types should also be expanded
   setTruncStoreAction(MVT::f32, MVT::f16, Expand);
   setTruncStoreAction(MVT::f64, MVT::f16, Expand);
   setTruncStoreAction(MVT::f64, MVT::f32, Expand);
@@ -275,13 +282,15 @@ NVPTXTargetLowering::NVPTXTargetLowering(const NVPTXTargetMachine &TM,
 }
 
 const char *NVPTXTargetLowering::getTargetNodeName(unsigned Opcode) const {
-  switch (Opcode) {
-  default:
-    return nullptr;
+  switch ((NVPTXISD::NodeType)Opcode) {
+  case NVPTXISD::FIRST_NUMBER:
+    break;
   case NVPTXISD::CALL:
     return "NVPTXISD::CALL";
   case NVPTXISD::RET_FLAG:
     return "NVPTXISD::RET_FLAG";
+  case NVPTXISD::LOAD_PARAM:
+    return "NVPTXISD::LOAD_PARAM";
   case NVPTXISD::Wrapper:
     return "NVPTXISD::Wrapper";
   case NVPTXISD::DeclareParam:
@@ -290,10 +299,14 @@ const char *NVPTXTargetLowering::getTargetNodeName(unsigned Opcode) const {
     return "NVPTXISD::DeclareScalarParam";
   case NVPTXISD::DeclareRet:
     return "NVPTXISD::DeclareRet";
+  case NVPTXISD::DeclareScalarRet:
+    return "NVPTXISD::DeclareScalarRet";
   case NVPTXISD::DeclareRetParam:
     return "NVPTXISD::DeclareRetParam";
   case NVPTXISD::PrintCall:
     return "NVPTXISD::PrintCall";
+  case NVPTXISD::PrintCallUni:
+    return "NVPTXISD::PrintCallUni";
   case NVPTXISD::LoadParam:
     return "NVPTXISD::LoadParam";
   case NVPTXISD::LoadParamV2:
@@ -366,6 +379,8 @@ const char *NVPTXTargetLowering::getTargetNodeName(unsigned Opcode) const {
     return "NVPTXISD::FUN_SHFR_CLAMP";
   case NVPTXISD::IMAD:
     return "NVPTXISD::IMAD";
+  case NVPTXISD::Dummy:
+    return "NVPTXISD::Dummy";
   case NVPTXISD::MUL_WIDE_SIGNED:
     return "NVPTXISD::MUL_WIDE_SIGNED";
   case NVPTXISD::MUL_WIDE_UNSIGNED:
@@ -855,6 +870,7 @@ const char *NVPTXTargetLowering::getTargetNodeName(unsigned Opcode) const {
   case NVPTXISD::Suld3DV4I16Zero:       return "NVPTXISD::Suld3DV4I16Zero";
   case NVPTXISD::Suld3DV4I32Zero:       return "NVPTXISD::Suld3DV4I32Zero";
   }
+  return nullptr;
 }
 
 TargetLoweringBase::LegalizeTypeAction
@@ -869,15 +885,16 @@ SDValue
 NVPTXTargetLowering::LowerGlobalAddress(SDValue Op, SelectionDAG &DAG) const {
   SDLoc dl(Op);
   const GlobalValue *GV = cast<GlobalAddressSDNode>(Op)->getGlobal();
-  Op = DAG.getTargetGlobalAddress(GV, dl, getPointerTy());
-  return DAG.getNode(NVPTXISD::Wrapper, dl, getPointerTy(), Op);
+  auto PtrVT = getPointerTy(DAG.getDataLayout());
+  Op = DAG.getTargetGlobalAddress(GV, dl, PtrVT);
+  return DAG.getNode(NVPTXISD::Wrapper, dl, PtrVT, Op);
 }
 
-std::string
-NVPTXTargetLowering::getPrototype(Type *retTy, const ArgListTy &Args,
-                                  const SmallVectorImpl<ISD::OutputArg> &Outs,
-                                  unsigned retAlignment,
-                                  const ImmutableCallSite *CS) const {
+std::string NVPTXTargetLowering::getPrototype(
+    const DataLayout &DL, Type *retTy, const ArgListTy &Args,
+    const SmallVectorImpl<ISD::OutputArg> &Outs, unsigned retAlignment,
+    const ImmutableCallSite *CS) const {
+  auto PtrVT = getPointerTy(DL);
 
   bool isABI = (STI.getSmVersion() >= 20);
   assert(isABI && "Non-ABI compilation is not supported");
@@ -905,13 +922,12 @@ NVPTXTargetLowering::getPrototype(Type *retTy, const ArgListTy &Args,
 
       O << ".param .b" << size << " _";
     } else if (isa<PointerType>(retTy)) {
-      O << ".param .b" << getPointerTy().getSizeInBits() << " _";
+      O << ".param .b" << PtrVT.getSizeInBits() << " _";
     } else if ((retTy->getTypeID() == Type::StructTyID) ||
                isa<VectorType>(retTy)) {
-      O << ".param .align "
-        << retAlignment
-        << " .b8 _["
-        << getDataLayout()->getTypeAllocSize(retTy) << "]";
+      auto &DL = CS->getCalledFunction()->getParent()->getDataLayout();
+      O << ".param .align " << retAlignment << " .b8 _["
+        << DL.getTypeAllocSize(retTy) << "]";
     } else {
       llvm_unreachable("Unknown return type");
     }
@@ -920,7 +936,6 @@ NVPTXTargetLowering::getPrototype(Type *retTy, const ArgListTy &Args,
   O << "_ (";
 
   bool first = true;
-  MVT thePointerTy = getPointerTy();
 
   unsigned OIdx = 0;
   for (unsigned i = 0, e = Args.size(); i != e; ++i, ++OIdx) {
@@ -934,24 +949,23 @@ NVPTXTargetLowering::getPrototype(Type *retTy, const ArgListTy &Args,
       if (Ty->isAggregateType() || Ty->isVectorTy()) {
         unsigned align = 0;
         const CallInst *CallI = cast<CallInst>(CS->getInstruction());
-        const DataLayout *TD = getDataLayout();
         // +1 because index 0 is reserved for return type alignment
         if (!llvm::getAlign(*CallI, i + 1, align))
-          align = TD->getABITypeAlignment(Ty);
-        unsigned sz = TD->getTypeAllocSize(Ty);
+          align = DL.getABITypeAlignment(Ty);
+        unsigned sz = DL.getTypeAllocSize(Ty);
         O << ".param .align " << align << " .b8 ";
         O << "_";
         O << "[" << sz << "]";
         // update the index for Outs
         SmallVector<EVT, 16> vtparts;
-        ComputeValueVTs(*this, Ty, vtparts);
+        ComputeValueVTs(*this, DL, Ty, vtparts);
         if (unsigned len = vtparts.size())
           OIdx += len - 1;
         continue;
       }
        // i8 types in IR will be i16 types in SDAG
-      assert((getValueType(Ty) == Outs[OIdx].VT ||
-             (getValueType(Ty) == MVT::i8 && Outs[OIdx].VT == MVT::i16)) &&
+      assert((getValueType(DL, Ty) == Outs[OIdx].VT ||
+              (getValueType(DL, Ty) == MVT::i8 && Outs[OIdx].VT == MVT::i16)) &&
              "type mismatch between callee prototype and arguments");
       // scalar type
       unsigned sz = 0;
@@ -960,7 +974,7 @@ NVPTXTargetLowering::getPrototype(Type *retTy, const ArgListTy &Args,
         if (sz < 32)
           sz = 32;
       } else if (isa<PointerType>(Ty))
-        sz = thePointerTy.getSizeInBits();
+        sz = PtrVT.getSizeInBits();
       else
         sz = Ty->getPrimitiveSizeInBits();
       O << ".param .b" << sz << " ";
@@ -972,7 +986,7 @@ NVPTXTargetLowering::getPrototype(Type *retTy, const ArgListTy &Args,
     Type *ETy = PTy->getElementType();
 
     unsigned align = Outs[OIdx].Flags.getByValAlign();
-    unsigned sz = getDataLayout()->getTypeAllocSize(ETy);
+    unsigned sz = DL.getTypeAllocSize(ETy);
     O << ".param .align " << align << " .b8 ";
     O << "_";
     O << "[" << sz << "]";
@@ -986,7 +1000,6 @@ NVPTXTargetLowering::getArgumentAlignment(SDValue Callee,
                                           const ImmutableCallSite *CS,
                                           Type *Ty,
                                           unsigned Idx) const {
-  const DataLayout *TD = getDataLayout();
   unsigned Align = 0;
   const Value *DirectCallee = CS->getCalledFunction();
 
@@ -1027,7 +1040,8 @@ NVPTXTargetLowering::getArgumentAlignment(SDValue Callee,
 
   // Call is indirect or alignment information is not available, fall back to
   // the ABI type alignment
-  return TD->getABITypeAlignment(Ty);
+  auto &DL = CS->getCaller()->getParent()->getDataLayout();
+  return DL.getABITypeAlignment(Ty);
 }
 
 SDValue NVPTXTargetLowering::LowerCall(TargetLowering::CallLoweringInfo &CLI,
@@ -1048,14 +1062,14 @@ SDValue NVPTXTargetLowering::LowerCall(TargetLowering::CallLoweringInfo &CLI,
   assert(isABI && "Non-ABI compilation is not supported");
   if (!isABI)
     return Chain;
-  const DataLayout *TD = getDataLayout();
   MachineFunction &MF = DAG.getMachineFunction();
   const Function *F = MF.getFunction();
+  auto &DL = MF.getDataLayout();
 
   SDValue tempChain = Chain;
-  Chain =
-      DAG.getCALLSEQ_START(Chain, DAG.getIntPtrConstant(uniqueCallSite, true),
-                           dl);
+  Chain = DAG.getCALLSEQ_START(Chain,
+                               DAG.getIntPtrConstant(uniqueCallSite, dl, true),
+                               dl);
   SDValue InFlag = Chain.getValue(1);
 
   unsigned paramCount = 0;
@@ -1080,15 +1094,18 @@ SDValue NVPTXTargetLowering::LowerCall(TargetLowering::CallLoweringInfo &CLI,
         // aggregate
         SmallVector<EVT, 16> vtparts;
         SmallVector<uint64_t, 16> Offsets;
-        ComputePTXValueVTs(*this, Ty, vtparts, &Offsets, 0);
+        ComputePTXValueVTs(*this, DAG.getDataLayout(), Ty, vtparts, &Offsets,
+                           0);
 
         unsigned align = getArgumentAlignment(Callee, CS, Ty, paramCount + 1);
         // declare .param .align <align> .b8 .param<n>[<size>];
-        unsigned sz = TD->getTypeAllocSize(Ty);
+        unsigned sz = DL.getTypeAllocSize(Ty);
         SDVTList DeclareParamVTs = DAG.getVTList(MVT::Other, MVT::Glue);
-        SDValue DeclareParamOps[] = { Chain, DAG.getConstant(align, MVT::i32),
-                                      DAG.getConstant(paramCount, MVT::i32),
-                                      DAG.getConstant(sz, MVT::i32), InFlag };
+        SDValue DeclareParamOps[] = { Chain, DAG.getConstant(align, dl,
+                                                             MVT::i32),
+                                      DAG.getConstant(paramCount, dl, MVT::i32),
+                                      DAG.getConstant(sz, dl, MVT::i32),
+                                      InFlag };
         Chain = DAG.getNode(NVPTXISD::DeclareParam, dl, DeclareParamVTs,
                             DeclareParamOps);
         InFlag = Chain.getValue(1);
@@ -1103,8 +1120,8 @@ SDValue NVPTXTargetLowering::LowerCall(TargetLowering::CallLoweringInfo &CLI,
           }
           SDVTList CopyParamVTs = DAG.getVTList(MVT::Other, MVT::Glue);
           SDValue CopyParamOps[] = { Chain,
-                                     DAG.getConstant(paramCount, MVT::i32),
-                                     DAG.getConstant(Offsets[j], MVT::i32),
+                                     DAG.getConstant(paramCount, dl, MVT::i32),
+                                     DAG.getConstant(Offsets[j], dl, MVT::i32),
                                      StVal, InFlag };
           Chain = DAG.getMemIntrinsicNode(NVPTXISD::StoreParam, dl,
                                           CopyParamVTs, CopyParamOps,
@@ -1119,14 +1136,16 @@ SDValue NVPTXTargetLowering::LowerCall(TargetLowering::CallLoweringInfo &CLI,
         continue;
       }
       if (Ty->isVectorTy()) {
-        EVT ObjectVT = getValueType(Ty);
+        EVT ObjectVT = getValueType(DL, Ty);
         unsigned align = getArgumentAlignment(Callee, CS, Ty, paramCount + 1);
         // declare .param .align <align> .b8 .param<n>[<size>];
-        unsigned sz = TD->getTypeAllocSize(Ty);
+        unsigned sz = DL.getTypeAllocSize(Ty);
         SDVTList DeclareParamVTs = DAG.getVTList(MVT::Other, MVT::Glue);
-        SDValue DeclareParamOps[] = { Chain, DAG.getConstant(align, MVT::i32),
-                                      DAG.getConstant(paramCount, MVT::i32),
-                                      DAG.getConstant(sz, MVT::i32), InFlag };
+        SDValue DeclareParamOps[] = { Chain,
+                                      DAG.getConstant(align, dl, MVT::i32),
+                                      DAG.getConstant(paramCount, dl, MVT::i32),
+                                      DAG.getConstant(sz, dl, MVT::i32),
+                                      InFlag };
         Chain = DAG.getNode(NVPTXISD::DeclareParam, dl, DeclareParamVTs,
                             DeclareParamOps);
         InFlag = Chain.getValue(1);
@@ -1147,8 +1166,8 @@ SDValue NVPTXTargetLowering::LowerCall(TargetLowering::CallLoweringInfo &CLI,
 
           SDVTList CopyParamVTs = DAG.getVTList(MVT::Other, MVT::Glue);
           SDValue CopyParamOps[] = { Chain,
-                                     DAG.getConstant(paramCount, MVT::i32),
-                                     DAG.getConstant(0, MVT::i32), Elt,
+                                     DAG.getConstant(paramCount, dl, MVT::i32),
+                                     DAG.getConstant(0, dl, MVT::i32), Elt,
                                      InFlag };
           Chain = DAG.getMemIntrinsicNode(NVPTXISD::StoreParam, dl,
                                           CopyParamVTs, CopyParamOps,
@@ -1164,9 +1183,9 @@ SDValue NVPTXTargetLowering::LowerCall(TargetLowering::CallLoweringInfo &CLI,
 
           SDVTList CopyParamVTs = DAG.getVTList(MVT::Other, MVT::Glue);
           SDValue CopyParamOps[] = { Chain,
-                                     DAG.getConstant(paramCount, MVT::i32),
-                                     DAG.getConstant(0, MVT::i32), Elt0, Elt1,
-                                     InFlag };
+                                     DAG.getConstant(paramCount, dl, MVT::i32),
+                                     DAG.getConstant(0, dl, MVT::i32), Elt0,
+                                     Elt1, InFlag };
           Chain = DAG.getMemIntrinsicNode(NVPTXISD::StoreParamV2, dl,
                                           CopyParamVTs, CopyParamOps,
                                           MemVT, MachinePointerInfo());
@@ -1196,8 +1215,8 @@ SDValue NVPTXTargetLowering::LowerCall(TargetLowering::CallLoweringInfo &CLI,
             SDValue StoreVal;
             SmallVector<SDValue, 8> Ops;
             Ops.push_back(Chain);
-            Ops.push_back(DAG.getConstant(paramCount, MVT::i32));
-            Ops.push_back(DAG.getConstant(curOffset, MVT::i32));
+            Ops.push_back(DAG.getConstant(paramCount, dl, MVT::i32));
+            Ops.push_back(DAG.getConstant(curOffset, dl, MVT::i32));
 
             unsigned Opc = NVPTXISD::StoreParamV2;
 
@@ -1264,9 +1283,9 @@ SDValue NVPTXTargetLowering::LowerCall(TargetLowering::CallLoweringInfo &CLI,
       }
       SDVTList DeclareParamVTs = DAG.getVTList(MVT::Other, MVT::Glue);
       SDValue DeclareParamOps[] = { Chain,
-                                    DAG.getConstant(paramCount, MVT::i32),
-                                    DAG.getConstant(sz, MVT::i32),
-                                    DAG.getConstant(0, MVT::i32), InFlag };
+                                    DAG.getConstant(paramCount, dl, MVT::i32),
+                                    DAG.getConstant(sz, dl, MVT::i32),
+                                    DAG.getConstant(0, dl, MVT::i32), InFlag };
       Chain = DAG.getNode(NVPTXISD::DeclareScalarParam, dl, DeclareParamVTs,
                           DeclareParamOps);
       InFlag = Chain.getValue(1);
@@ -1279,8 +1298,10 @@ SDValue NVPTXTargetLowering::LowerCall(TargetLowering::CallLoweringInfo &CLI,
         OutV = DAG.getNode(opc, dl, MVT::i16, OutV);
       }
       SDVTList CopyParamVTs = DAG.getVTList(MVT::Other, MVT::Glue);
-      SDValue CopyParamOps[] = { Chain, DAG.getConstant(paramCount, MVT::i32),
-                                 DAG.getConstant(0, MVT::i32), OutV, InFlag };
+      SDValue CopyParamOps[] = { Chain,
+                                 DAG.getConstant(paramCount, dl, MVT::i32),
+                                 DAG.getConstant(0, dl, MVT::i32), OutV,
+                                 InFlag };
 
       unsigned opcode = NVPTXISD::StoreParam;
       if (Outs[OIdx].Flags.isZExt())
@@ -1299,7 +1320,8 @@ SDValue NVPTXTargetLowering::LowerCall(TargetLowering::CallLoweringInfo &CLI,
     SmallVector<uint64_t, 16> Offsets;
     const PointerType *PTy = dyn_cast<PointerType>(Args[i].Ty);
     assert(PTy && "Type of a byval parameter should be pointer");
-    ComputePTXValueVTs(*this, PTy->getElementType(), vtparts, &Offsets, 0);
+    ComputePTXValueVTs(*this, DAG.getDataLayout(), PTy->getElementType(),
+                       vtparts, &Offsets, 0);
 
     // declare .param .align <align> .b8 .param<n>[<size>];
     unsigned sz = Outs[OIdx].Flags.getByValSize();
@@ -1309,9 +1331,9 @@ SDValue NVPTXTargetLowering::LowerCall(TargetLowering::CallLoweringInfo &CLI,
     // so we don't need to worry about natural alignment or not.
     // See TargetLowering::LowerCallTo().
     SDValue DeclareParamOps[] = {
-      Chain, DAG.getConstant(Outs[OIdx].Flags.getByValAlign(), MVT::i32),
-      DAG.getConstant(paramCount, MVT::i32), DAG.getConstant(sz, MVT::i32),
-      InFlag
+      Chain, DAG.getConstant(Outs[OIdx].Flags.getByValAlign(), dl, MVT::i32),
+      DAG.getConstant(paramCount, dl, MVT::i32),
+      DAG.getConstant(sz, dl, MVT::i32), InFlag
     };
     Chain = DAG.getNode(NVPTXISD::DeclareParam, dl, DeclareParamVTs,
                         DeclareParamOps);
@@ -1320,9 +1342,9 @@ SDValue NVPTXTargetLowering::LowerCall(TargetLowering::CallLoweringInfo &CLI,
       EVT elemtype = vtparts[j];
       int curOffset = Offsets[j];
       unsigned PartAlign = GreatestCommonDivisor64(ArgAlign, curOffset);
-      SDValue srcAddr =
-          DAG.getNode(ISD::ADD, dl, getPointerTy(), OutVals[OIdx],
-                      DAG.getConstant(curOffset, getPointerTy()));
+      auto PtrVT = getPointerTy(DAG.getDataLayout());
+      SDValue srcAddr = DAG.getNode(ISD::ADD, dl, PtrVT, OutVals[OIdx],
+                                    DAG.getConstant(curOffset, dl, PtrVT));
       SDValue theVal = DAG.getLoad(elemtype, dl, tempChain, srcAddr,
                                    MachinePointerInfo(), false, false, false,
                                    PartAlign);
@@ -1330,9 +1352,10 @@ SDValue NVPTXTargetLowering::LowerCall(TargetLowering::CallLoweringInfo &CLI,
         theVal = DAG.getNode(ISD::ANY_EXTEND, dl, MVT::i16, theVal);
       }
       SDVTList CopyParamVTs = DAG.getVTList(MVT::Other, MVT::Glue);
-      SDValue CopyParamOps[] = { Chain, DAG.getConstant(paramCount, MVT::i32),
-                                 DAG.getConstant(curOffset, MVT::i32), theVal,
-                                 InFlag };
+      SDValue CopyParamOps[] = { Chain,
+                                 DAG.getConstant(paramCount, dl, MVT::i32),
+                                 DAG.getConstant(curOffset, dl, MVT::i32),
+                                 theVal, InFlag };
       Chain = DAG.getMemIntrinsicNode(NVPTXISD::StoreParam, dl, CopyParamVTs,
                                       CopyParamOps, elemtype,
                                       MachinePointerInfo());
@@ -1348,12 +1371,12 @@ SDValue NVPTXTargetLowering::LowerCall(TargetLowering::CallLoweringInfo &CLI,
   // Handle Result
   if (Ins.size() > 0) {
     SmallVector<EVT, 16> resvtparts;
-    ComputeValueVTs(*this, retTy, resvtparts);
+    ComputeValueVTs(*this, DL, retTy, resvtparts);
 
     // Declare
     //  .param .align 16 .b8 retval0[<size-in-bytes>], or
     //  .param .b<size-in-bits> retval0
-    unsigned resultsz = TD->getTypeAllocSizeInBits(retTy);
+    unsigned resultsz = DL.getTypeAllocSizeInBits(retTy);
     // Emit ".param .b<size-in-bits> retval0" instead of byte arrays only for
     // these three types to match the logic in
     // NVPTXAsmPrinter::printReturnValStr and NVPTXTargetLowering::getPrototype.
@@ -1364,9 +1387,9 @@ SDValue NVPTXTargetLowering::LowerCall(TargetLowering::CallLoweringInfo &CLI,
       if (resultsz < 32)
         resultsz = 32;
       SDVTList DeclareRetVTs = DAG.getVTList(MVT::Other, MVT::Glue);
-      SDValue DeclareRetOps[] = { Chain, DAG.getConstant(1, MVT::i32),
-                                  DAG.getConstant(resultsz, MVT::i32),
-                                  DAG.getConstant(0, MVT::i32), InFlag };
+      SDValue DeclareRetOps[] = { Chain, DAG.getConstant(1, dl, MVT::i32),
+                                  DAG.getConstant(resultsz, dl, MVT::i32),
+                                  DAG.getConstant(0, dl, MVT::i32), InFlag };
       Chain = DAG.getNode(NVPTXISD::DeclareRet, dl, DeclareRetVTs,
                           DeclareRetOps);
       InFlag = Chain.getValue(1);
@@ -1374,9 +1397,9 @@ SDValue NVPTXTargetLowering::LowerCall(TargetLowering::CallLoweringInfo &CLI,
       retAlignment = getArgumentAlignment(Callee, CS, retTy, 0);
       SDVTList DeclareRetVTs = DAG.getVTList(MVT::Other, MVT::Glue);
       SDValue DeclareRetOps[] = { Chain,
-                                  DAG.getConstant(retAlignment, MVT::i32),
-                                  DAG.getConstant(resultsz / 8, MVT::i32),
-                                  DAG.getConstant(0, MVT::i32), InFlag };
+                                  DAG.getConstant(retAlignment, dl, MVT::i32),
+                                  DAG.getConstant(resultsz / 8, dl, MVT::i32),
+                                  DAG.getConstant(0, dl, MVT::i32), InFlag };
       Chain = DAG.getNode(NVPTXISD::DeclareRetParam, dl, DeclareRetVTs,
                           DeclareRetOps);
       InFlag = Chain.getValue(1);
@@ -1392,7 +1415,8 @@ SDValue NVPTXTargetLowering::LowerCall(TargetLowering::CallLoweringInfo &CLI,
     // The prototype is embedded in a string and put as the operand for a
     // CallPrototype SDNode which will print out to the value of the string.
     SDVTList ProtoVTs = DAG.getVTList(MVT::Other, MVT::Glue);
-    std::string Proto = getPrototype(retTy, Args, Outs, retAlignment, CS);
+    std::string Proto =
+        getPrototype(DAG.getDataLayout(), retTy, Args, Outs, retAlignment, CS);
     const char *ProtoStr =
       nvTM->getManagedStrPool()->getManagedString(Proto.c_str())->c_str();
     SDValue ProtoOps[] = {
@@ -1404,7 +1428,7 @@ SDValue NVPTXTargetLowering::LowerCall(TargetLowering::CallLoweringInfo &CLI,
   // Op to just print "call"
   SDVTList PrintCallVTs = DAG.getVTList(MVT::Other, MVT::Glue);
   SDValue PrintCallOps[] = {
-    Chain, DAG.getConstant((Ins.size() == 0) ? 0 : 1, MVT::i32), InFlag
+    Chain, DAG.getConstant((Ins.size() == 0) ? 0 : 1, dl, MVT::i32), InFlag
   };
   Chain = DAG.getNode(Func ? (NVPTXISD::PrintCallUni) : (NVPTXISD::PrintCall),
                       dl, PrintCallVTs, PrintCallOps);
@@ -1430,20 +1454,22 @@ SDValue NVPTXTargetLowering::LowerCall(TargetLowering::CallLoweringInfo &CLI,
     else
       opcode = NVPTXISD::CallArg;
     SDVTList CallArgVTs = DAG.getVTList(MVT::Other, MVT::Glue);
-    SDValue CallArgOps[] = { Chain, DAG.getConstant(1, MVT::i32),
-                             DAG.getConstant(i, MVT::i32), InFlag };
+    SDValue CallArgOps[] = { Chain, DAG.getConstant(1, dl, MVT::i32),
+                             DAG.getConstant(i, dl, MVT::i32), InFlag };
     Chain = DAG.getNode(opcode, dl, CallArgVTs, CallArgOps);
     InFlag = Chain.getValue(1);
   }
   SDVTList CallArgEndVTs = DAG.getVTList(MVT::Other, MVT::Glue);
-  SDValue CallArgEndOps[] = { Chain, DAG.getConstant(Func ? 1 : 0, MVT::i32),
+  SDValue CallArgEndOps[] = { Chain,
+                              DAG.getConstant(Func ? 1 : 0, dl, MVT::i32),
                               InFlag };
   Chain = DAG.getNode(NVPTXISD::CallArgEnd, dl, CallArgEndVTs, CallArgEndOps);
   InFlag = Chain.getValue(1);
 
   if (!Func) {
     SDVTList PrototypeVTs = DAG.getVTList(MVT::Other, MVT::Glue);
-    SDValue PrototypeOps[] = { Chain, DAG.getConstant(uniqueCallSite, MVT::i32),
+    SDValue PrototypeOps[] = { Chain,
+                               DAG.getConstant(uniqueCallSite, dl, MVT::i32),
                                InFlag };
     Chain = DAG.getNode(NVPTXISD::Prototype, dl, PrototypeVTs, PrototypeOps);
     InFlag = Chain.getValue(1);
@@ -1452,7 +1478,7 @@ SDValue NVPTXTargetLowering::LowerCall(TargetLowering::CallLoweringInfo &CLI,
   // Generate loads from param memory/moves from registers for result
   if (Ins.size() > 0) {
     if (retTy && retTy->isVectorTy()) {
-      EVT ObjectVT = getValueType(retTy);
+      EVT ObjectVT = getValueType(DL, retTy);
       unsigned NumElts = ObjectVT.getVectorNumElements();
       EVT EltVT = ObjectVT.getVectorElementType();
       assert(STI.getTargetLowering()->getNumRegisters(F->getContext(),
@@ -1474,8 +1500,8 @@ SDValue NVPTXTargetLowering::LowerCall(TargetLowering::CallLoweringInfo &CLI,
           LoadRetVTs.push_back(EltVT);
         LoadRetVTs.push_back(MVT::Other);
         LoadRetVTs.push_back(MVT::Glue);
-        SDValue LoadRetOps[] = {Chain, DAG.getConstant(1, MVT::i32),
-                                DAG.getConstant(0, MVT::i32), InFlag};
+        SDValue LoadRetOps[] = {Chain, DAG.getConstant(1, dl, MVT::i32),
+                                DAG.getConstant(0, dl, MVT::i32), InFlag};
         SDValue retval = DAG.getMemIntrinsicNode(
             NVPTXISD::LoadParam, dl,
             DAG.getVTList(LoadRetVTs), LoadRetOps, EltVT, MachinePointerInfo());
@@ -1501,8 +1527,8 @@ SDValue NVPTXTargetLowering::LowerCall(TargetLowering::CallLoweringInfo &CLI,
         }
         LoadRetVTs.push_back(MVT::Other);
         LoadRetVTs.push_back(MVT::Glue);
-        SDValue LoadRetOps[] = {Chain, DAG.getConstant(1, MVT::i32),
-                                DAG.getConstant(0, MVT::i32), InFlag};
+        SDValue LoadRetOps[] = {Chain, DAG.getConstant(1, dl, MVT::i32),
+                                DAG.getConstant(0, dl, MVT::i32), InFlag};
         SDValue retval = DAG.getMemIntrinsicNode(
             NVPTXISD::LoadParamV2, dl,
             DAG.getVTList(LoadRetVTs), LoadRetOps, EltVT, MachinePointerInfo());
@@ -1544,8 +1570,8 @@ SDValue NVPTXTargetLowering::LowerCall(TargetLowering::CallLoweringInfo &CLI,
           }
           LoadRetVTs.push_back(MVT::Other);
           LoadRetVTs.push_back(MVT::Glue);
-          SDValue LoadRetOps[] = {Chain, DAG.getConstant(1, MVT::i32),
-                                  DAG.getConstant(Ofst, MVT::i32), InFlag};
+          SDValue LoadRetOps[] = {Chain, DAG.getConstant(1, dl, MVT::i32),
+                                  DAG.getConstant(Ofst, dl, MVT::i32), InFlag};
           SDValue retval = DAG.getMemIntrinsicNode(
               Opc, dl, DAG.getVTList(LoadRetVTs),
               LoadRetOps, EltVT, MachinePointerInfo());
@@ -1565,13 +1591,13 @@ SDValue NVPTXTargetLowering::LowerCall(TargetLowering::CallLoweringInfo &CLI,
               Elt = DAG.getNode(ISD::TRUNCATE, dl, EltVT, Elt);
             InVals.push_back(Elt);
           }
-          Ofst += TD->getTypeAllocSize(VecVT.getTypeForEVT(F->getContext()));
+          Ofst += DL.getTypeAllocSize(VecVT.getTypeForEVT(F->getContext()));
         }
       }
     } else {
       SmallVector<EVT, 16> VTs;
       SmallVector<uint64_t, 16> Offsets;
-      ComputePTXValueVTs(*this, retTy, VTs, &Offsets, 0);
+      ComputePTXValueVTs(*this, DAG.getDataLayout(), retTy, VTs, &Offsets, 0);
       assert(VTs.size() == Ins.size() && "Bad value decomposition");
       unsigned RetAlign = getArgumentAlignment(Callee, CS, retTy, 0);
       for (unsigned i = 0, e = Ins.size(); i != e; ++i) {
@@ -1583,8 +1609,7 @@ SDValue NVPTXTargetLowering::LowerCall(TargetLowering::CallLoweringInfo &CLI,
 
         SmallVector<EVT, 4> LoadRetVTs;
         EVT TheLoadType = VTs[i];
-        if (retTy->isIntegerTy() &&
-            TD->getTypeAllocSizeInBits(retTy) < 32) {
+        if (retTy->isIntegerTy() && DL.getTypeAllocSizeInBits(retTy) < 32) {
           // This is for integer types only, and specifically not for
           // aggregates.
           LoadRetVTs.push_back(MVT::i32);
@@ -1599,8 +1624,9 @@ SDValue NVPTXTargetLowering::LowerCall(TargetLowering::CallLoweringInfo &CLI,
         LoadRetVTs.push_back(MVT::Other);
         LoadRetVTs.push_back(MVT::Glue);
 
-        SDValue LoadRetOps[] = {Chain, DAG.getConstant(1, MVT::i32),
-                                DAG.getConstant(Offsets[i], MVT::i32), InFlag};
+        SDValue LoadRetOps[] = {Chain, DAG.getConstant(1, dl, MVT::i32),
+                                DAG.getConstant(Offsets[i], dl, MVT::i32),
+                                InFlag};
         SDValue retval = DAG.getMemIntrinsicNode(
             NVPTXISD::LoadParam, dl,
             DAG.getVTList(LoadRetVTs), LoadRetOps,
@@ -1615,8 +1641,10 @@ SDValue NVPTXTargetLowering::LowerCall(TargetLowering::CallLoweringInfo &CLI,
     }
   }
 
-  Chain = DAG.getCALLSEQ_END(Chain, DAG.getIntPtrConstant(uniqueCallSite, true),
-                             DAG.getIntPtrConstant(uniqueCallSite + 1, true),
+  Chain = DAG.getCALLSEQ_END(Chain,
+                             DAG.getIntPtrConstant(uniqueCallSite, dl, true),
+                             DAG.getIntPtrConstant(uniqueCallSite + 1, dl,
+                                                   true),
                              InFlag, dl);
   uniqueCallSite++;
 
@@ -1642,7 +1670,7 @@ NVPTXTargetLowering::LowerCONCAT_VECTORS(SDValue Op, SelectionDAG &DAG) const {
     unsigned NumSubElem = VVT.getVectorNumElements();
     for (unsigned j = 0; j < NumSubElem; ++j) {
       Ops.push_back(DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, EltVT, SubOp,
-                                DAG.getIntPtrConstant(j)));
+                                DAG.getIntPtrConstant(j, dl)));
     }
   }
   return DAG.getNode(ISD::BUILD_VECTOR, dl, Node->getValueType(0), Ops);
@@ -1691,16 +1719,18 @@ SDValue NVPTXTargetLowering::LowerShiftRightParts(SDValue Op,
     //      dHi = aHi >> Amt
 
     SDValue RevShAmt = DAG.getNode(ISD::SUB, dl, MVT::i32,
-                                   DAG.getConstant(VTBits, MVT::i32), ShAmt);
+                                   DAG.getConstant(VTBits, dl, MVT::i32),
+                                   ShAmt);
     SDValue Tmp1 = DAG.getNode(ISD::SRL, dl, VT, ShOpLo, ShAmt);
     SDValue ExtraShAmt = DAG.getNode(ISD::SUB, dl, MVT::i32, ShAmt,
-                                     DAG.getConstant(VTBits, MVT::i32));
+                                     DAG.getConstant(VTBits, dl, MVT::i32));
     SDValue Tmp2 = DAG.getNode(ISD::SHL, dl, VT, ShOpHi, RevShAmt);
     SDValue FalseVal = DAG.getNode(ISD::OR, dl, VT, Tmp1, Tmp2);
     SDValue TrueVal = DAG.getNode(Opc, dl, VT, ShOpHi, ExtraShAmt);
 
     SDValue Cmp = DAG.getSetCC(dl, MVT::i1, ShAmt,
-                               DAG.getConstant(VTBits, MVT::i32), ISD::SETGE);
+                               DAG.getConstant(VTBits, dl, MVT::i32),
+                               ISD::SETGE);
     SDValue Hi = DAG.getNode(Opc, dl, VT, ShOpHi, ShAmt);
     SDValue Lo = DAG.getNode(ISD::SELECT, dl, VT, Cmp, TrueVal, FalseVal);
 
@@ -1751,16 +1781,18 @@ SDValue NVPTXTargetLowering::LowerShiftLeftParts(SDValue Op,
     //      dHi = (aHi << Amt) | (aLo >> (size-Amt))
 
     SDValue RevShAmt = DAG.getNode(ISD::SUB, dl, MVT::i32,
-                                   DAG.getConstant(VTBits, MVT::i32), ShAmt);
+                                   DAG.getConstant(VTBits, dl, MVT::i32),
+                                   ShAmt);
     SDValue Tmp1 = DAG.getNode(ISD::SHL, dl, VT, ShOpHi, ShAmt);
     SDValue ExtraShAmt = DAG.getNode(ISD::SUB, dl, MVT::i32, ShAmt,
-                                     DAG.getConstant(VTBits, MVT::i32));
+                                     DAG.getConstant(VTBits, dl, MVT::i32));
     SDValue Tmp2 = DAG.getNode(ISD::SRL, dl, VT, ShOpLo, RevShAmt);
     SDValue FalseVal = DAG.getNode(ISD::OR, dl, VT, Tmp1, Tmp2);
     SDValue TrueVal = DAG.getNode(ISD::SHL, dl, VT, ShOpLo, ExtraShAmt);
 
     SDValue Cmp = DAG.getSetCC(dl, MVT::i1, ShAmt,
-                               DAG.getConstant(VTBits, MVT::i32), ISD::SETGE);
+                               DAG.getConstant(VTBits, dl, MVT::i32),
+                               ISD::SETGE);
     SDValue Lo = DAG.getNode(ISD::SHL, dl, VT, ShOpLo, ShAmt);
     SDValue Hi = DAG.getNode(ISD::SELECT, dl, VT, Cmp, TrueVal, FalseVal);
 
@@ -1888,11 +1920,11 @@ NVPTXTargetLowering::LowerSTOREVector(SDValue Op, SelectionDAG &DAG) const {
     }
 
     MemSDNode *MemSD = cast<MemSDNode>(N);
-    const DataLayout *TD = getDataLayout();
+    const DataLayout &TD = DAG.getDataLayout();
 
     unsigned Align = MemSD->getAlignment();
     unsigned PrefAlign =
-      TD->getPrefTypeAlignment(ValVT.getTypeForEVT(*DAG.getContext()));
+        TD.getPrefTypeAlignment(ValVT.getTypeForEVT(*DAG.getContext()));
     if (Align < PrefAlign) {
       // This store is not sufficiently aligned, so bail out and let this vector
       // store be scalarized.  Note that we may still be able to emit smaller
@@ -1933,7 +1965,7 @@ NVPTXTargetLowering::LowerSTOREVector(SDValue Op, SelectionDAG &DAG) const {
     // Then the split values
     for (unsigned i = 0; i < NumElts; ++i) {
       SDValue ExtVal = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, EltVT, Val,
-                                   DAG.getIntPtrConstant(i));
+                                   DAG.getIntPtrConstant(i, DL));
       if (NeedExt)
         ExtVal = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i16, ExtVal);
       Ops.push_back(ExtVal);
@@ -2032,7 +2064,8 @@ SDValue NVPTXTargetLowering::LowerFormalArguments(
     const SmallVectorImpl<ISD::InputArg> &Ins, SDLoc dl, SelectionDAG &DAG,
     SmallVectorImpl<SDValue> &InVals) const {
   MachineFunction &MF = DAG.getMachineFunction();
-  const DataLayout *TD = getDataLayout();
+  const DataLayout &DL = DAG.getDataLayout();
+  auto PtrVT = getPointerTy(DAG.getDataLayout());
 
   const Function *F = MF.getFunction();
   const AttributeSet &PAL = F->getAttributes();
@@ -2077,7 +2110,7 @@ SDValue NVPTXTargetLowering::LowerFormalArguments(
             (theArgs[i]->getParent() ? theArgs[i]->getParent()->getParent()
                                      : nullptr))) {
       assert(isKernel && "Only kernels can have image/sampler params");
-      InVals.push_back(DAG.getConstant(i + 1, MVT::i32));
+      InVals.push_back(DAG.getConstant(i + 1, dl, MVT::i32));
       continue;
     }
 
@@ -2086,7 +2119,7 @@ SDValue NVPTXTargetLowering::LowerFormalArguments(
       if (Ty->isAggregateType()) {
         SmallVector<EVT, 16> vtparts;
 
-        ComputePTXValueVTs(*this, Ty, vtparts);
+        ComputePTXValueVTs(*this, DAG.getDataLayout(), Ty, vtparts);
         assert(vtparts.size() > 0 && "empty aggregate type not expected");
         for (unsigned parti = 0, parte = vtparts.size(); parti != parte;
              ++parti) {
@@ -2098,7 +2131,7 @@ SDValue NVPTXTargetLowering::LowerFormalArguments(
         continue;
       }
       if (Ty->isVectorTy()) {
-        EVT ObjectVT = getValueType(Ty);
+        EVT ObjectVT = getValueType(DL, Ty);
         unsigned NumRegs = TLI->getNumRegisters(F->getContext(), ObjectVT);
         for (unsigned parti = 0; parti < NumRegs; ++parti) {
           InVals.push_back(DAG.getNode(ISD::UNDEF, dl, Ins[InsIdx].VT));
@@ -2124,13 +2157,14 @@ SDValue NVPTXTargetLowering::LowerFormalArguments(
         // NOTE: Here, we lose the ability to issue vector loads for vectors
         // that are a part of a struct.  This should be investigated in the
         // future.
-        ComputePTXValueVTs(*this, Ty, vtparts, &offsets, 0);
+        ComputePTXValueVTs(*this, DAG.getDataLayout(), Ty, vtparts, &offsets,
+                           0);
         assert(vtparts.size() > 0 && "empty aggregate type not expected");
         bool aggregateIsPacked = false;
         if (StructType *STy = llvm::dyn_cast<StructType>(Ty))
           aggregateIsPacked = STy->isPacked();
 
-        SDValue Arg = getParamSymbol(DAG, idx, getPointerTy());
+        SDValue Arg = getParamSymbol(DAG, idx, PtrVT);
         for (unsigned parti = 0, parte = vtparts.size(); parti != parte;
              ++parti) {
           EVT partVT = vtparts[parti];
@@ -2138,12 +2172,12 @@ SDValue NVPTXTargetLowering::LowerFormalArguments(
               PointerType::get(partVT.getTypeForEVT(F->getContext()),
                                llvm::ADDRESS_SPACE_PARAM));
           SDValue srcAddr =
-              DAG.getNode(ISD::ADD, dl, getPointerTy(), Arg,
-                          DAG.getConstant(offsets[parti], getPointerTy()));
-          unsigned partAlign =
-              aggregateIsPacked ? 1
-                                : TD->getABITypeAlignment(
-                                      partVT.getTypeForEVT(F->getContext()));
+              DAG.getNode(ISD::ADD, dl, PtrVT, Arg,
+                          DAG.getConstant(offsets[parti], dl, PtrVT));
+          unsigned partAlign = aggregateIsPacked
+                                   ? 1
+                                   : DL.getABITypeAlignment(
+                                         partVT.getTypeForEVT(F->getContext()));
           SDValue p;
           if (Ins[InsIdx].VT.getSizeInBits() > partVT.getSizeInBits()) {
             ISD::LoadExtType ExtOp = Ins[InsIdx].Flags.isSExt() ? 
@@ -2166,8 +2200,8 @@ SDValue NVPTXTargetLowering::LowerFormalArguments(
         continue;
       }
       if (Ty->isVectorTy()) {
-        EVT ObjectVT = getValueType(Ty);
-        SDValue Arg = getParamSymbol(DAG, idx, getPointerTy());
+        EVT ObjectVT = getValueType(DL, Ty);
+        SDValue Arg = getParamSymbol(DAG, idx, PtrVT);
         unsigned NumElts = ObjectVT.getVectorNumElements();
         assert(TLI->getNumRegisters(F->getContext(), ObjectVT) == NumElts &&
                "Vector was not scalarized");
@@ -2180,9 +2214,9 @@ SDValue NVPTXTargetLowering::LowerFormalArguments(
           Value *SrcValue = Constant::getNullValue(PointerType::get(
               EltVT.getTypeForEVT(F->getContext()), llvm::ADDRESS_SPACE_PARAM));
           SDValue P = DAG.getLoad(
-              EltVT, dl, Root, Arg, MachinePointerInfo(SrcValue), false,
-              false, true,
-              TD->getABITypeAlignment(EltVT.getTypeForEVT(F->getContext())));
+              EltVT, dl, Root, Arg, MachinePointerInfo(SrcValue), false, false,
+              true,
+              DL.getABITypeAlignment(EltVT.getTypeForEVT(F->getContext())));
           if (P.getNode())
             P.getNode()->setIROrder(idx + 1);
 
@@ -2197,16 +2231,16 @@ SDValue NVPTXTargetLowering::LowerFormalArguments(
           Value *SrcValue = Constant::getNullValue(PointerType::get(
               VecVT.getTypeForEVT(F->getContext()), llvm::ADDRESS_SPACE_PARAM));
           SDValue P = DAG.getLoad(
-              VecVT, dl, Root, Arg, MachinePointerInfo(SrcValue), false,
-              false, true,
-              TD->getABITypeAlignment(VecVT.getTypeForEVT(F->getContext())));
+              VecVT, dl, Root, Arg, MachinePointerInfo(SrcValue), false, false,
+              true,
+              DL.getABITypeAlignment(VecVT.getTypeForEVT(F->getContext())));
           if (P.getNode())
             P.getNode()->setIROrder(idx + 1);
 
           SDValue Elt0 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, EltVT, P,
-                                     DAG.getIntPtrConstant(0));
+                                     DAG.getIntPtrConstant(0, dl));
           SDValue Elt1 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, EltVT, P,
-                                     DAG.getIntPtrConstant(1));
+                                     DAG.getIntPtrConstant(1, dl));
 
           if (Ins[InsIdx].VT.getSizeInBits() > EltVT.getSizeInBits()) {
             Elt0 = DAG.getNode(ISD::ANY_EXTEND, dl, Ins[InsIdx].VT, Elt0);
@@ -2237,13 +2271,12 @@ SDValue NVPTXTargetLowering::LowerFormalArguments(
             Value *SrcValue = Constant::getNullValue(
                 PointerType::get(VecVT.getTypeForEVT(F->getContext()),
                                  llvm::ADDRESS_SPACE_PARAM));
-            SDValue SrcAddr =
-                DAG.getNode(ISD::ADD, dl, getPointerTy(), Arg,
-                            DAG.getConstant(Ofst, getPointerTy()));
+            SDValue SrcAddr = DAG.getNode(ISD::ADD, dl, PtrVT, Arg,
+                                          DAG.getConstant(Ofst, dl, PtrVT));
             SDValue P = DAG.getLoad(
                 VecVT, dl, Root, SrcAddr, MachinePointerInfo(SrcValue), false,
                 false, true,
-                TD->getABITypeAlignment(VecVT.getTypeForEVT(F->getContext())));
+                DL.getABITypeAlignment(VecVT.getTypeForEVT(F->getContext())));
             if (P.getNode())
               P.getNode()->setIROrder(idx + 1);
 
@@ -2251,12 +2284,12 @@ SDValue NVPTXTargetLowering::LowerFormalArguments(
               if (i + j >= NumElts)
                 break;
               SDValue Elt = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, EltVT, P,
-                                        DAG.getIntPtrConstant(j));
+                                        DAG.getIntPtrConstant(j, dl));
               if (Ins[InsIdx].VT.getSizeInBits() > EltVT.getSizeInBits())
                 Elt = DAG.getNode(ISD::ANY_EXTEND, dl, Ins[InsIdx].VT, Elt);
               InVals.push_back(Elt);
             }
-            Ofst += TD->getTypeAllocSize(VecVT.getTypeForEVT(F->getContext()));
+            Ofst += DL.getTypeAllocSize(VecVT.getTypeForEVT(F->getContext()));
           }
           InsIdx += NumElts;
         }
@@ -2266,23 +2299,24 @@ SDValue NVPTXTargetLowering::LowerFormalArguments(
         continue;
       }
       // A plain scalar.
-      EVT ObjectVT = getValueType(Ty);
+      EVT ObjectVT = getValueType(DL, Ty);
       // If ABI, load from the param symbol
-      SDValue Arg = getParamSymbol(DAG, idx, getPointerTy());
+      SDValue Arg = getParamSymbol(DAG, idx, PtrVT);
       Value *srcValue = Constant::getNullValue(PointerType::get(
           ObjectVT.getTypeForEVT(F->getContext()), llvm::ADDRESS_SPACE_PARAM));
       SDValue p;
        if (ObjectVT.getSizeInBits() < Ins[InsIdx].VT.getSizeInBits()) {
         ISD::LoadExtType ExtOp = Ins[InsIdx].Flags.isSExt() ? 
                                        ISD::SEXTLOAD : ISD::ZEXTLOAD;
-        p = DAG.getExtLoad(ExtOp, dl, Ins[InsIdx].VT, Root, Arg,
-                           MachinePointerInfo(srcValue), ObjectVT, false, false,
-                           false,
-        TD->getABITypeAlignment(ObjectVT.getTypeForEVT(F->getContext())));
+        p = DAG.getExtLoad(
+            ExtOp, dl, Ins[InsIdx].VT, Root, Arg, MachinePointerInfo(srcValue),
+            ObjectVT, false, false, false,
+            DL.getABITypeAlignment(ObjectVT.getTypeForEVT(F->getContext())));
       } else {
-        p = DAG.getLoad(Ins[InsIdx].VT, dl, Root, Arg,
-                        MachinePointerInfo(srcValue), false, false, false,
-        TD->getABITypeAlignment(ObjectVT.getTypeForEVT(F->getContext())));
+        p = DAG.getLoad(
+            Ins[InsIdx].VT, dl, Root, Arg, MachinePointerInfo(srcValue), false,
+            false, false,
+            DL.getABITypeAlignment(ObjectVT.getTypeForEVT(F->getContext())));
       }
       if (p.getNode())
         p.getNode()->setIROrder(idx + 1);
@@ -2297,10 +2331,10 @@ SDValue NVPTXTargetLowering::LowerFormalArguments(
     // machine instruction fails because TargetExternalSymbol
     // (not lowered) is target dependent, and CopyToReg assumes
     // the source is lowered.
-    EVT ObjectVT = getValueType(Ty);
+    EVT ObjectVT = getValueType(DL, Ty);
     assert(ObjectVT == Ins[InsIdx].VT &&
            "Ins type did not match function type");
-    SDValue Arg = getParamSymbol(DAG, idx, getPointerTy());
+    SDValue Arg = getParamSymbol(DAG, idx, PtrVT);
     SDValue p = DAG.getNode(NVPTXISD::MoveParam, dl, ObjectVT, Arg);
     if (p.getNode())
       p.getNode()->setIROrder(idx + 1);
@@ -2309,7 +2343,7 @@ SDValue NVPTXTargetLowering::LowerFormalArguments(
     else {
       SDValue p2 = DAG.getNode(
           ISD::INTRINSIC_WO_CHAIN, dl, ObjectVT,
-          DAG.getConstant(Intrinsic::nvvm_ptr_local_to_gen, MVT::i32), p);
+          DAG.getConstant(Intrinsic::nvvm_ptr_local_to_gen, dl, MVT::i32), p);
       InVals.push_back(p2);
     }
   }
@@ -2338,7 +2372,7 @@ NVPTXTargetLowering::LowerReturn(SDValue Chain, CallingConv::ID CallConv,
   MachineFunction &MF = DAG.getMachineFunction();
   const Function *F = MF.getFunction();
   Type *RetTy = F->getReturnType();
-  const DataLayout *TD = getDataLayout();
+  const DataLayout &TD = DAG.getDataLayout();
 
   bool isABI = (STI.getSmVersion() >= 20);
   assert(isABI && "Non-ABI compilation is not supported");
@@ -2352,7 +2386,7 @@ NVPTXTargetLowering::LowerReturn(SDValue Chain, CallingConv::ID CallConv,
     assert(NumElts == Outs.size() && "Bad scalarization of return value");
 
     // const_cast can be removed in later LLVM versions
-    EVT EltVT = getValueType(RetTy).getVectorElementType();
+    EVT EltVT = getValueType(TD, RetTy).getVectorElementType();
     bool NeedExtend = false;
     if (EltVT.getSizeInBits() < 16)
       NeedExtend = true;
@@ -2363,7 +2397,7 @@ NVPTXTargetLowering::LowerReturn(SDValue Chain, CallingConv::ID CallConv,
       // We only have one element, so just directly store it
       if (NeedExtend)
         StoreVal = DAG.getNode(ISD::ZERO_EXTEND, dl, MVT::i16, StoreVal);
-      SDValue Ops[] = { Chain, DAG.getConstant(0, MVT::i32), StoreVal };
+      SDValue Ops[] = { Chain, DAG.getConstant(0, dl, MVT::i32), StoreVal };
       Chain = DAG.getMemIntrinsicNode(NVPTXISD::StoreRetval, dl,
                                       DAG.getVTList(MVT::Other), Ops,
                                       EltVT, MachinePointerInfo());
@@ -2378,7 +2412,7 @@ NVPTXTargetLowering::LowerReturn(SDValue Chain, CallingConv::ID CallConv,
         StoreVal1 = DAG.getNode(ISD::ZERO_EXTEND, dl, MVT::i16, StoreVal1);
       }
 
-      SDValue Ops[] = { Chain, DAG.getConstant(0, MVT::i32), StoreVal0,
+      SDValue Ops[] = { Chain, DAG.getConstant(0, dl, MVT::i32), StoreVal0,
                         StoreVal1 };
       Chain = DAG.getMemIntrinsicNode(NVPTXISD::StoreRetvalV2, dl,
                                       DAG.getVTList(MVT::Other), Ops,
@@ -2403,14 +2437,14 @@ NVPTXTargetLowering::LowerReturn(SDValue Chain, CallingConv::ID CallConv,
       EVT VecVT =
           EVT::getVectorVT(F->getContext(), EltVT, VecSize);
       unsigned PerStoreOffset =
-          TD->getTypeAllocSize(VecVT.getTypeForEVT(F->getContext()));
+          TD.getTypeAllocSize(VecVT.getTypeForEVT(F->getContext()));
 
       for (unsigned i = 0; i < NumElts; i += VecSize) {
         // Get values
         SDValue StoreVal;
         SmallVector<SDValue, 8> Ops;
         Ops.push_back(Chain);
-        Ops.push_back(DAG.getConstant(Offset, MVT::i32));
+        Ops.push_back(DAG.getConstant(Offset, dl, MVT::i32));
         unsigned Opc = NVPTXISD::StoreRetvalV2;
         EVT ExtendedVT = (NeedExtend) ? MVT::i16 : OutVals[0].getValueType();
 
@@ -2461,7 +2495,7 @@ NVPTXTargetLowering::LowerReturn(SDValue Chain, CallingConv::ID CallConv,
   } else {
     SmallVector<EVT, 16> ValVTs;
     SmallVector<uint64_t, 16> Offsets;
-    ComputePTXValueVTs(*this, RetTy, ValVTs, &Offsets, 0);
+    ComputePTXValueVTs(*this, DAG.getDataLayout(), RetTy, ValVTs, &Offsets, 0);
     assert(ValVTs.size() == OutVals.size() && "Bad return value decomposition");
 
     for (unsigned i = 0, e = Outs.size(); i != e; ++i) {
@@ -2475,10 +2509,9 @@ NVPTXTargetLowering::LowerReturn(SDValue Chain, CallingConv::ID CallConv,
         if (TheValType.isVector())
           TmpVal = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl,
                                TheValType.getVectorElementType(), TmpVal,
-                               DAG.getIntPtrConstant(j));
+                               DAG.getIntPtrConstant(j, dl));
         EVT TheStoreType = ValVTs[i];
-        if (RetTy->isIntegerTy() &&
-            TD->getTypeAllocSizeInBits(RetTy) < 32) {
+        if (RetTy->isIntegerTy() && TD.getTypeAllocSizeInBits(RetTy) < 32) {
           // The following zero-extension is for integer types only, and
           // specifically not for aggregates.
           TmpVal = DAG.getNode(ISD::ZERO_EXTEND, dl, MVT::i32, TmpVal);
@@ -2489,7 +2522,7 @@ NVPTXTargetLowering::LowerReturn(SDValue Chain, CallingConv::ID CallConv,
 
         SDValue Ops[] = {
           Chain,
-          DAG.getConstant(Offsets[i], MVT::i32),
+          DAG.getConstant(Offsets[i], dl, MVT::i32),
           TmpVal };
         Chain = DAG.getMemIntrinsicNode(NVPTXISD::StoreRetval, dl,
                                         DAG.getVTList(MVT::Other), Ops,
@@ -3259,14 +3292,14 @@ bool NVPTXTargetLowering::getTgtMemIntrinsic(
   case Intrinsic::nvvm_ldu_global_i:
   case Intrinsic::nvvm_ldu_global_f:
   case Intrinsic::nvvm_ldu_global_p: {
-
+    auto &DL = I.getModule()->getDataLayout();
     Info.opc = ISD::INTRINSIC_W_CHAIN;
     if (Intrinsic == Intrinsic::nvvm_ldu_global_i)
-      Info.memVT = getValueType(I.getType());
+      Info.memVT = getValueType(DL, I.getType());
     else if(Intrinsic == Intrinsic::nvvm_ldu_global_p)
-      Info.memVT = getPointerTy();
+      Info.memVT = getPointerTy(DL);
     else
-      Info.memVT = getValueType(I.getType());
+      Info.memVT = getValueType(DL, I.getType());
     Info.ptrVal = I.getArgOperand(0);
     Info.offset = 0;
     Info.vol = 0;
@@ -3279,14 +3312,15 @@ bool NVPTXTargetLowering::getTgtMemIntrinsic(
   case Intrinsic::nvvm_ldg_global_i:
   case Intrinsic::nvvm_ldg_global_f:
   case Intrinsic::nvvm_ldg_global_p: {
+    auto &DL = I.getModule()->getDataLayout();
 
     Info.opc = ISD::INTRINSIC_W_CHAIN;
     if (Intrinsic == Intrinsic::nvvm_ldg_global_i)
-      Info.memVT = getValueType(I.getType());
+      Info.memVT = getValueType(DL, I.getType());
     else if(Intrinsic == Intrinsic::nvvm_ldg_global_p)
-      Info.memVT = getPointerTy();
+      Info.memVT = getPointerTy(DL);
     else
-      Info.memVT = getValueType(I.getType());
+      Info.memVT = getValueType(DL, I.getType());
     Info.ptrVal = I.getArgOperand(0);
     Info.offset = 0;
     Info.vol = 0;
@@ -3699,8 +3733,9 @@ bool NVPTXTargetLowering::getTgtMemIntrinsic(
 /// Used to guide target specific optimizations, like loop strength reduction
 /// (LoopStrengthReduce.cpp) and memory optimization for address mode
 /// (CodeGenPrepare.cpp)
-bool NVPTXTargetLowering::isLegalAddressingMode(const AddrMode &AM,
-                                                Type *Ty) const {
+bool NVPTXTargetLowering::isLegalAddressingMode(const DataLayout &DL,
+                                                const AddrMode &AM, Type *Ty,
+                                                unsigned AS) const {
 
   // AddrMode - This represents an addressing mode of:
   //    BaseGV + BaseOffs + BaseReg + Scale*ScaleReg
@@ -3712,9 +3747,7 @@ bool NVPTXTargetLowering::isLegalAddressingMode(const AddrMode &AM,
   // - [immAddr]
 
   if (AM.BaseGV) {
-    if (AM.BaseOffs || AM.HasBaseReg || AM.Scale)
-      return false;
-    return true;
+    return !AM.BaseOffs && !AM.HasBaseReg && !AM.Scale;
   }
 
   switch (AM.Scale) {
@@ -3739,7 +3772,7 @@ bool NVPTXTargetLowering::isLegalAddressingMode(const AddrMode &AM,
 /// getConstraintType - Given a constraint letter, return the type of
 /// constraint it is for this target.
 NVPTXTargetLowering::ConstraintType
-NVPTXTargetLowering::getConstraintType(const std::string &Constraint) const {
+NVPTXTargetLowering::getConstraintType(StringRef Constraint) const {
   if (Constraint.size() == 1) {
     switch (Constraint[0]) {
     default:
@@ -3761,7 +3794,7 @@ NVPTXTargetLowering::getConstraintType(const std::string &Constraint) const {
 
 std::pair<unsigned, const TargetRegisterClass *>
 NVPTXTargetLowering::getRegForInlineAsmConstraint(const TargetRegisterInfo *TRI,
-                                                  const std::string &Constraint,
+                                                  StringRef Constraint,
                                                   MVT VT) const {
   if (Constraint.size() == 1) {
     switch (Constraint[0]) {
@@ -4078,25 +4111,16 @@ static bool AreMulWideOperandsDemotable(SDValue LHS, SDValue RHS,
   if (ConstantSDNode *CI = dyn_cast<ConstantSDNode>(RHS)) {
     APInt Val = CI->getAPIntValue();
     if (LHSSign == Unsigned) {
-      if (Val.isIntN(OptSize)) {
-        return true;
-      }
-      return false;
+      return Val.isIntN(OptSize);
     } else {
-      if (Val.isSignedIntN(OptSize)) {
-        return true;
-      }
-      return false;
+      return Val.isSignedIntN(OptSize);
     }
   } else {
     OperandSignedness RHSSign;
     if (!IsMulWideOperandDemotable(RHS, OptSize, RHSSign))
       return false;
 
-    if (LHSSign != RHSSign)
-      return false;
-
-    return true;
+    return LHSSign == RHSSign;
   }
 }
 
@@ -4111,6 +4135,7 @@ static SDValue TryMULWIDECombine(SDNode *N,
     return SDValue();
   }
 
+  SDLoc DL(N);
   unsigned OptSize = MulType.getSizeInBits() >> 1;
   SDValue LHS = N->getOperand(0);
   SDValue RHS = N->getOperand(1);
@@ -4133,7 +4158,7 @@ static SDValue TryMULWIDECombine(SDNode *N,
     unsigned BitWidth = MulType.getSizeInBits();
     if (ShiftAmt.sge(0) && ShiftAmt.slt(BitWidth)) {
       APInt MulVal = APInt(BitWidth, 1) << ShiftAmt;
-      RHS = DCI.DAG.getConstant(MulVal, MulType);
+      RHS = DCI.DAG.getConstant(MulVal, DL, MulType);
     } else {
       return SDValue();
     }
@@ -4155,9 +4180,9 @@ static SDValue TryMULWIDECombine(SDNode *N,
   // Truncate the operands to the correct size. Note that these are just for
   // type consistency and will (likely) be eliminated in later phases.
   SDValue TruncLHS =
-    DCI.DAG.getNode(ISD::TRUNCATE, SDLoc(N), DemotedVT, LHS);
+    DCI.DAG.getNode(ISD::TRUNCATE, DL, DemotedVT, LHS);
   SDValue TruncRHS =
-    DCI.DAG.getNode(ISD::TRUNCATE, SDLoc(N), DemotedVT, RHS);
+    DCI.DAG.getNode(ISD::TRUNCATE, DL, DemotedVT, RHS);
 
   unsigned Opc;
   if (Signed) {
@@ -4166,7 +4191,7 @@ static SDValue TryMULWIDECombine(SDNode *N,
     Opc = NVPTXISD::MUL_WIDE_UNSIGNED;
   }
 
-  return DCI.DAG.getNode(Opc, SDLoc(N), MulType, TruncLHS, TruncRHS);
+  return DCI.DAG.getNode(Opc, DL, MulType, TruncLHS, TruncRHS);
 }
 
 /// PerformMULCombine - Runs PTX-specific DAG combine patterns on MUL nodes.
@@ -4217,7 +4242,6 @@ SDValue NVPTXTargetLowering::PerformDAGCombine(SDNode *N,
 
 /// ReplaceVectorLoad - Convert vector loads into multi-output scalar loads.
 static void ReplaceLoadVector(SDNode *N, SelectionDAG &DAG,
-                              const DataLayout *TD,
                               SmallVectorImpl<SDValue> &Results) {
   EVT ResVT = N->getValueType(0);
   SDLoc DL(N);
@@ -4248,8 +4272,9 @@ static void ReplaceLoadVector(SDNode *N, SelectionDAG &DAG,
   LoadSDNode *LD = cast<LoadSDNode>(N);
 
   unsigned Align = LD->getAlignment();
+  auto &TD = DAG.getDataLayout();
   unsigned PrefAlign =
-    TD->getPrefTypeAlignment(ResVT.getTypeForEVT(*DAG.getContext()));
+      TD.getPrefTypeAlignment(ResVT.getTypeForEVT(*DAG.getContext()));
   if (Align < PrefAlign) {
     // This load is not sufficiently aligned, so bail out and let this vector
     // load be scalarized.  Note that we may still be able to emit smaller
@@ -4294,7 +4319,7 @@ static void ReplaceLoadVector(SDNode *N, SelectionDAG &DAG,
 
   // The select routine does not have access to the LoadSDNode instance, so
   // pass along the extension information
-  OtherOps.push_back(DAG.getIntPtrConstant(LD->getExtensionType()));
+  OtherOps.push_back(DAG.getIntPtrConstant(LD->getExtensionType(), DL));
 
   SDValue NewLD = DAG.getMemIntrinsicNode(Opcode, DL, LdResVTs, OtherOps,
                                           LD->getMemoryVT(),
@@ -4461,7 +4486,7 @@ void NVPTXTargetLowering::ReplaceNodeResults(
   default:
     report_fatal_error("Unhandled custom legalization");
   case ISD::LOAD:
-    ReplaceLoadVector(N, DAG, getDataLayout(), Results);
+    ReplaceLoadVector(N, DAG, Results);
     return;
   case ISD::INTRINSIC_W_CHAIN:
     ReplaceINTRINSIC_W_CHAIN(N, DAG, Results);
@@ -4494,7 +4519,7 @@ NVPTXTargetObjectFile::~NVPTXTargetObjectFile() {
   delete DwarfRangesSection;
 }
 
-const MCSection *
+MCSection *
 NVPTXTargetObjectFile::SelectSectionForGlobal(const GlobalValue *GV,
                                               SectionKind Kind, Mangler &Mang,
                                               const TargetMachine &TM) const {
diff --git a/lib/Target/NVPTX/NVPTXISelLowering.h b/lib/Target/NVPTX/NVPTXISelLowering.h
index 8594364d290..e5c37321a33 100644
--- a/lib/Target/NVPTX/NVPTXISelLowering.h
+++ b/lib/Target/NVPTX/NVPTXISelLowering.h
@@ -21,7 +21,7 @@
 
 namespace llvm {
 namespace NVPTXISD {
-enum NodeType {
+enum NodeType : unsigned {
   // Start the numbering from where ISD NodeType finishes.
   FIRST_NUMBER = ISD::BUILTIN_OP_END,
   Wrapper,
@@ -456,23 +456,23 @@ class NVPTXTargetLowering : public TargetLowering {
   /// Used to guide target specific optimizations, like loop strength
   /// reduction (LoopStrengthReduce.cpp) and memory optimization for
   /// address mode (CodeGenPrepare.cpp)
-  bool isLegalAddressingMode(const AddrMode &AM, Type *Ty) const override;
+  bool isLegalAddressingMode(const DataLayout &DL, const AddrMode &AM, Type *Ty,
+                             unsigned AS) const override;
 
   /// getFunctionAlignment - Return the Log2 alignment of this function.
   unsigned getFunctionAlignment(const Function *F) const;
 
-  EVT getSetCCResultType(LLVMContext &Ctx, EVT VT) const override {
+  EVT getSetCCResultType(const DataLayout &DL, LLVMContext &Ctx,
+                         EVT VT) const override {
     if (VT.isVector())
       return EVT::getVectorVT(Ctx, MVT::i1, VT.getVectorNumElements());
     return MVT::i1;
   }
 
-  ConstraintType
-  getConstraintType(const std::string &Constraint) const override;
+  ConstraintType getConstraintType(StringRef Constraint) const override;
   std::pair<unsigned, const TargetRegisterClass *>
   getRegForInlineAsmConstraint(const TargetRegisterInfo *TRI,
-                               const std::string &Constraint,
-                               MVT VT) const override;
+                               StringRef Constraint, MVT VT) const override;
 
   SDValue LowerFormalArguments(
       SDValue Chain, CallingConv::ID CallConv, bool isVarArg,
@@ -482,7 +482,7 @@ class NVPTXTargetLowering : public TargetLowering {
   SDValue LowerCall(CallLoweringInfo &CLI,
                     SmallVectorImpl<SDValue> &InVals) const override;
 
-  std::string getPrototype(Type *, const ArgListTy &,
+  std::string getPrototype(const DataLayout &DL, Type *, const ArgListTy &,
                            const SmallVectorImpl<ISD::OutputArg> &,
                            unsigned retAlignment,
                            const ImmutableCallSite *CS) const;
@@ -497,16 +497,12 @@ class NVPTXTargetLowering : public TargetLowering {
                                     std::vector<SDValue> &Ops,
                                     SelectionDAG &DAG) const override;
 
-  unsigned getInlineAsmMemConstraint(
-      const std::string &ConstraintCode) const override {
-    // FIXME: Map different constraints differently.
-    return InlineAsm::Constraint_m;
-  }
-
   const NVPTXTargetMachine *nvTM;
 
   // PTX always uses 32-bit shift amounts
-  MVT getScalarShiftAmountTy(EVT LHSTy) const override { return MVT::i32; }
+  MVT getScalarShiftAmountTy(const DataLayout &, EVT) const override {
+    return MVT::i32;
+  }
 
   TargetLoweringBase::LegalizeTypeAction
   getPreferredVectorAction(EVT VT) const override;
diff --git a/lib/Target/NVPTX/NVPTXInstrInfo.cpp b/lib/Target/NVPTX/NVPTXInstrInfo.cpp
index dabc3be43a3..76d6597c6e2 100644
--- a/lib/Target/NVPTX/NVPTXInstrInfo.cpp
+++ b/lib/Target/NVPTX/NVPTXInstrInfo.cpp
@@ -248,7 +248,7 @@ unsigned NVPTXInstrInfo::RemoveBranch(MachineBasicBlock &MBB) const {
 
 unsigned NVPTXInstrInfo::InsertBranch(
     MachineBasicBlock &MBB, MachineBasicBlock *TBB, MachineBasicBlock *FBB,
-    const SmallVectorImpl<MachineOperand> &Cond, DebugLoc DL) const {
+    ArrayRef<MachineOperand> Cond, DebugLoc DL) const {
   // Shouldn't be a fall through.
   assert(TBB && "InsertBranch must not be told to insert a fallthrough");
   assert((Cond.size() == 1 || Cond.size() == 0) &&
diff --git a/lib/Target/NVPTX/NVPTXInstrInfo.h b/lib/Target/NVPTX/NVPTXInstrInfo.h
index 9b5d491dfeb..179c0688719 100644
--- a/lib/Target/NVPTX/NVPTXInstrInfo.h
+++ b/lib/Target/NVPTX/NVPTXInstrInfo.h
@@ -66,7 +66,7 @@ class NVPTXInstrInfo : public NVPTXGenInstrInfo {
   unsigned RemoveBranch(MachineBasicBlock &MBB) const override;
   unsigned InsertBranch(
       MachineBasicBlock &MBB, MachineBasicBlock *TBB, MachineBasicBlock *FBB,
-      const SmallVectorImpl<MachineOperand> &Cond, DebugLoc DL) const override;
+      ArrayRef<MachineOperand> Cond, DebugLoc DL) const override;
   unsigned getLdStCodeAddrSpace(const MachineInstr &MI) const {
     return MI.getOperand(2).getImm();
   }
diff --git a/lib/Target/NVPTX/NVPTXInstrInfo.td b/lib/Target/NVPTX/NVPTXInstrInfo.td
index 68f0d9fa41e..6fdd60f3ed2 100644
--- a/lib/Target/NVPTX/NVPTXInstrInfo.td
+++ b/lib/Target/NVPTX/NVPTXInstrInfo.td
@@ -452,13 +452,13 @@ def Int4Const : PatLeaf<(imm), [{
 def SHL2MUL32 : SDNodeXForm<imm, [{
   const APInt &v = N->getAPIntValue();
   APInt temp(32, 1);
-  return CurDAG->getTargetConstant(temp.shl(v), MVT::i32);
+  return CurDAG->getTargetConstant(temp.shl(v), SDLoc(N), MVT::i32);
 }]>;
 
 def SHL2MUL16 : SDNodeXForm<imm, [{
   const APInt &v = N->getAPIntValue();
   APInt temp(16, 1);
-  return CurDAG->getTargetConstant(temp.shl(v), MVT::i16);
+  return CurDAG->getTargetConstant(temp.shl(v), SDLoc(N), MVT::i16);
 }]>;
 
 def MULWIDES64
@@ -1138,7 +1138,7 @@ def ROT32imm_sw : NVPTXInst<(outs Int32Regs:$dst),
     []>;
 
 def SUB_FRM_32 : SDNodeXForm<imm, [{
-    return CurDAG->getTargetConstant(32-N->getZExtValue(), MVT::i32);
+    return CurDAG->getTargetConstant(32-N->getZExtValue(), SDLoc(N), MVT::i32);
 }]>;
 
 def : Pat<(rotl Int32Regs:$src, (i32 imm:$amt)),
@@ -1189,7 +1189,7 @@ def ROT64imm_sw : NVPTXInst<(outs Int64Regs:$dst), (ins Int64Regs:$src,
     []>;
 
 def SUB_FRM_64 : SDNodeXForm<imm, [{
-    return CurDAG->getTargetConstant(64-N->getZExtValue(), MVT::i32);
+    return CurDAG->getTargetConstant(64-N->getZExtValue(), SDLoc(N), MVT::i32);
 }]>;
 
 def : Pat<(rotl Int64Regs:$src, (i32 imm:$amt)),
diff --git a/lib/Target/NVPTX/NVPTXLowerAggrCopies.cpp b/lib/Target/NVPTX/NVPTXLowerAggrCopies.cpp
index 6ab0fadf9a3..35298565e58 100644
--- a/lib/Target/NVPTX/NVPTXLowerAggrCopies.cpp
+++ b/lib/Target/NVPTX/NVPTXLowerAggrCopies.cpp
@@ -6,6 +6,8 @@
 // License. See LICENSE.TXT for details.
 //
 //===----------------------------------------------------------------------===//
+//
+// \file
 // Lower aggregate copies, memset, memcpy, memmov intrinsics into loops when
 // the size is large or is not a compile-time constant.
 //
@@ -18,19 +20,20 @@
 #include "llvm/IR/DataLayout.h"
 #include "llvm/IR/Function.h"
 #include "llvm/IR/IRBuilder.h"
-#include "llvm/IR/InstIterator.h"
 #include "llvm/IR/Instructions.h"
 #include "llvm/IR/IntrinsicInst.h"
 #include "llvm/IR/Intrinsics.h"
 #include "llvm/IR/LLVMContext.h"
 #include "llvm/IR/Module.h"
 #include "llvm/Support/Debug.h"
+#include "llvm/Transforms/Utils/BasicBlockUtils.h"
 
 #define DEBUG_TYPE "nvptx"
 
 using namespace llvm;
 
 namespace {
+
 // actual analysis class, which is a functionpass
 struct NVPTXLowerAggrCopies : public FunctionPass {
   static char ID;
@@ -50,183 +53,299 @@ struct NVPTXLowerAggrCopies : public FunctionPass {
     return "Lower aggregate copies/intrinsics into loops";
   }
 };
-} // namespace
 
 char NVPTXLowerAggrCopies::ID = 0;
 
-// Lower MemTransferInst or load-store pair to loop
-static void convertTransferToLoop(
-    Instruction *splitAt, Value *srcAddr, Value *dstAddr, Value *len,
-    //unsigned numLoads,
-    bool srcVolatile, bool dstVolatile, LLVMContext &Context, Function &F) {
-  Type *indType = len->getType();
+// Lower memcpy to loop.
+void convertMemCpyToLoop(Instruction *ConvertedInst, Value *SrcAddr,
+                         Value *DstAddr, Value *CopyLen, bool SrcIsVolatile,
+                         bool DstIsVolatile, LLVMContext &Context,
+                         Function &F) {
+  Type *TypeOfCopyLen = CopyLen->getType();
 
-  BasicBlock *origBB = splitAt->getParent();
-  BasicBlock *newBB = splitAt->getParent()->splitBasicBlock(splitAt, "split");
-  BasicBlock *loopBB = BasicBlock::Create(Context, "loadstoreloop", &F, newBB);
+  BasicBlock *OrigBB = ConvertedInst->getParent();
+  BasicBlock *NewBB =
+      ConvertedInst->getParent()->splitBasicBlock(ConvertedInst, "split");
+  BasicBlock *LoopBB = BasicBlock::Create(Context, "loadstoreloop", &F, NewBB);
 
-  origBB->getTerminator()->setSuccessor(0, loopBB);
-  IRBuilder<> builder(origBB, origBB->getTerminator());
+  OrigBB->getTerminator()->setSuccessor(0, LoopBB);
+  IRBuilder<> Builder(OrigBB, OrigBB->getTerminator());
 
-  // srcAddr and dstAddr are expected to be pointer types,
+  // SrcAddr and DstAddr are expected to be pointer types,
   // so no check is made here.
-  unsigned srcAS = cast<PointerType>(srcAddr->getType())->getAddressSpace();
-  unsigned dstAS = cast<PointerType>(dstAddr->getType())->getAddressSpace();
+  unsigned SrcAS = cast<PointerType>(SrcAddr->getType())->getAddressSpace();
+  unsigned DstAS = cast<PointerType>(DstAddr->getType())->getAddressSpace();
 
   // Cast pointers to (char *)
-  srcAddr = builder.CreateBitCast(srcAddr, Type::getInt8PtrTy(Context, srcAS));
-  dstAddr = builder.CreateBitCast(dstAddr, Type::getInt8PtrTy(Context, dstAS));
-
-  IRBuilder<> loop(loopBB);
-  // The loop index (ind) is a phi node.
-  PHINode *ind = loop.CreatePHI(indType, 0);
-  // Incoming value for ind is 0
-  ind->addIncoming(ConstantInt::get(indType, 0), origBB);
-
-  // load from srcAddr+ind
-  Value *val = loop.CreateLoad(loop.CreateGEP(loop.getInt8Ty(), srcAddr, ind),
-                               srcVolatile);
-  // store at dstAddr+ind
-  loop.CreateStore(val, loop.CreateGEP(loop.getInt8Ty(), dstAddr, ind),
-                   dstVolatile);
-
-  // The value for ind coming from backedge is (ind + 1)
-  Value *newind = loop.CreateAdd(ind, ConstantInt::get(indType, 1));
-  ind->addIncoming(newind, loopBB);
-
-  loop.CreateCondBr(loop.CreateICmpULT(newind, len), loopBB, newBB);
+  SrcAddr = Builder.CreateBitCast(SrcAddr, Builder.getInt8PtrTy(SrcAS));
+  DstAddr = Builder.CreateBitCast(DstAddr, Builder.getInt8PtrTy(DstAS));
+
+  IRBuilder<> LoopBuilder(LoopBB);
+  PHINode *LoopIndex = LoopBuilder.CreatePHI(TypeOfCopyLen, 0);
+  LoopIndex->addIncoming(ConstantInt::get(TypeOfCopyLen, 0), OrigBB);
+
+  // load from SrcAddr+LoopIndex
+  // TODO: we can leverage the align parameter of llvm.memcpy for more efficient
+  // word-sized loads and stores.
+  Value *Element =
+      LoopBuilder.CreateLoad(LoopBuilder.CreateInBoundsGEP(
+                                 LoopBuilder.getInt8Ty(), SrcAddr, LoopIndex),
+                             SrcIsVolatile);
+  // store at DstAddr+LoopIndex
+  LoopBuilder.CreateStore(Element,
+                          LoopBuilder.CreateInBoundsGEP(LoopBuilder.getInt8Ty(),
+                                                        DstAddr, LoopIndex),
+                          DstIsVolatile);
+
+  // The value for LoopIndex coming from backedge is (LoopIndex + 1)
+  Value *NewIndex =
+      LoopBuilder.CreateAdd(LoopIndex, ConstantInt::get(TypeOfCopyLen, 1));
+  LoopIndex->addIncoming(NewIndex, LoopBB);
+
+  LoopBuilder.CreateCondBr(LoopBuilder.CreateICmpULT(NewIndex, CopyLen), LoopBB,
+                           NewBB);
 }
 
-// Lower MemSetInst to loop
-static void convertMemSetToLoop(Instruction *splitAt, Value *dstAddr,
-                                Value *len, Value *val, LLVMContext &Context,
-                                Function &F) {
-  BasicBlock *origBB = splitAt->getParent();
-  BasicBlock *newBB = splitAt->getParent()->splitBasicBlock(splitAt, "split");
-  BasicBlock *loopBB = BasicBlock::Create(Context, "loadstoreloop", &F, newBB);
+// Lower memmove to IR. memmove is required to correctly copy overlapping memory
+// regions; therefore, it has to check the relative positions of the source and
+// destination pointers and choose the copy direction accordingly.
+//
+// The code below is an IR rendition of this C function:
+//
+// void* memmove(void* dst, const void* src, size_t n) {
+//   unsigned char* d = dst;
+//   const unsigned char* s = src;
+//   if (s < d) {
+//     // copy backwards
+//     while (n--) {
+//       d[n] = s[n];
+//     }
+//   } else {
+//     // copy forward
+//     for (size_t i = 0; i < n; ++i) {
+//       d[i] = s[i];
+//     }
+//   }
+//   return dst;
+// }
+void convertMemMoveToLoop(Instruction *ConvertedInst, Value *SrcAddr,
+                          Value *DstAddr, Value *CopyLen, bool SrcIsVolatile,
+                          bool DstIsVolatile, LLVMContext &Context,
+                          Function &F) {
+  Type *TypeOfCopyLen = CopyLen->getType();
+  BasicBlock *OrigBB = ConvertedInst->getParent();
+
+  // Create the a comparison of src and dst, based on which we jump to either
+  // the forward-copy part of the function (if src >= dst) or the backwards-copy
+  // part (if src < dst).
+  // SplitBlockAndInsertIfThenElse conveniently creates the basic if-then-else
+  // structure. Its block terminators (unconditional branches) are replaced by
+  // the appropriate conditional branches when the loop is built.
+  ICmpInst *PtrCompare = new ICmpInst(ConvertedInst, ICmpInst::ICMP_ULT,
+                                      SrcAddr, DstAddr, "compare_src_dst");
+  TerminatorInst *ThenTerm, *ElseTerm;
+  SplitBlockAndInsertIfThenElse(PtrCompare, ConvertedInst, &ThenTerm,
+                                &ElseTerm);
+
+  // Each part of the function consists of two blocks:
+  //   copy_backwards:        used to skip the loop when n == 0
+  //   copy_backwards_loop:   the actual backwards loop BB
+  //   copy_forward:          used to skip the loop when n == 0
+  //   copy_forward_loop:     the actual forward loop BB
+  BasicBlock *CopyBackwardsBB = ThenTerm->getParent();
+  CopyBackwardsBB->setName("copy_backwards");
+  BasicBlock *CopyForwardBB = ElseTerm->getParent();
+  CopyForwardBB->setName("copy_forward");
+  BasicBlock *ExitBB = ConvertedInst->getParent();
+  ExitBB->setName("memmove_done");
+
+  // Initial comparison of n == 0 that lets us skip the loops altogether. Shared
+  // between both backwards and forward copy clauses.
+  ICmpInst *CompareN =
+      new ICmpInst(OrigBB->getTerminator(), ICmpInst::ICMP_EQ, CopyLen,
+                   ConstantInt::get(TypeOfCopyLen, 0), "compare_n_to_0");
+
+  // Copying backwards.
+  BasicBlock *LoopBB =
+      BasicBlock::Create(Context, "copy_backwards_loop", &F, CopyForwardBB);
+  IRBuilder<> LoopBuilder(LoopBB);
+  PHINode *LoopPhi = LoopBuilder.CreatePHI(TypeOfCopyLen, 0);
+  Value *IndexPtr = LoopBuilder.CreateSub(
+      LoopPhi, ConstantInt::get(TypeOfCopyLen, 1), "index_ptr");
+  Value *Element = LoopBuilder.CreateLoad(
+      LoopBuilder.CreateInBoundsGEP(SrcAddr, IndexPtr), "element");
+  LoopBuilder.CreateStore(Element,
+                          LoopBuilder.CreateInBoundsGEP(DstAddr, IndexPtr));
+  LoopBuilder.CreateCondBr(
+      LoopBuilder.CreateICmpEQ(IndexPtr, ConstantInt::get(TypeOfCopyLen, 0)),
+      ExitBB, LoopBB);
+  LoopPhi->addIncoming(IndexPtr, LoopBB);
+  LoopPhi->addIncoming(CopyLen, CopyBackwardsBB);
+  BranchInst::Create(ExitBB, LoopBB, CompareN, ThenTerm);
+  ThenTerm->eraseFromParent();
+
+  // Copying forward.
+  BasicBlock *FwdLoopBB =
+      BasicBlock::Create(Context, "copy_forward_loop", &F, ExitBB);
+  IRBuilder<> FwdLoopBuilder(FwdLoopBB);
+  PHINode *FwdCopyPhi = FwdLoopBuilder.CreatePHI(TypeOfCopyLen, 0, "index_ptr");
+  Value *FwdElement = FwdLoopBuilder.CreateLoad(
+      FwdLoopBuilder.CreateInBoundsGEP(SrcAddr, FwdCopyPhi), "element");
+  FwdLoopBuilder.CreateStore(
+      FwdElement, FwdLoopBuilder.CreateInBoundsGEP(DstAddr, FwdCopyPhi));
+  Value *FwdIndexPtr = FwdLoopBuilder.CreateAdd(
+      FwdCopyPhi, ConstantInt::get(TypeOfCopyLen, 1), "index_increment");
+  FwdLoopBuilder.CreateCondBr(FwdLoopBuilder.CreateICmpEQ(FwdIndexPtr, CopyLen),
+                              ExitBB, FwdLoopBB);
+  FwdCopyPhi->addIncoming(FwdIndexPtr, FwdLoopBB);
+  FwdCopyPhi->addIncoming(ConstantInt::get(TypeOfCopyLen, 0), CopyForwardBB);
+
+  BranchInst::Create(ExitBB, FwdLoopBB, CompareN, ElseTerm);
+  ElseTerm->eraseFromParent();
+}
 
-  origBB->getTerminator()->setSuccessor(0, loopBB);
-  IRBuilder<> builder(origBB, origBB->getTerminator());
+// Lower memset to loop.
+void convertMemSetToLoop(Instruction *ConvertedInst, Value *DstAddr,
+                         Value *CopyLen, Value *SetValue, LLVMContext &Context,
+                         Function &F) {
+  BasicBlock *OrigBB = ConvertedInst->getParent();
+  BasicBlock *NewBB =
+      ConvertedInst->getParent()->splitBasicBlock(ConvertedInst, "split");
+  BasicBlock *LoopBB = BasicBlock::Create(Context, "loadstoreloop", &F, NewBB);
 
-  unsigned dstAS = cast<PointerType>(dstAddr->getType())->getAddressSpace();
+  OrigBB->getTerminator()->setSuccessor(0, LoopBB);
+  IRBuilder<> Builder(OrigBB, OrigBB->getTerminator());
 
   // Cast pointer to the type of value getting stored
-  dstAddr =
-      builder.CreateBitCast(dstAddr, PointerType::get(val->getType(), dstAS));
+  unsigned dstAS = cast<PointerType>(DstAddr->getType())->getAddressSpace();
+  DstAddr = Builder.CreateBitCast(DstAddr,
+                                  PointerType::get(SetValue->getType(), dstAS));
 
-  IRBuilder<> loop(loopBB);
-  PHINode *ind = loop.CreatePHI(len->getType(), 0);
-  ind->addIncoming(ConstantInt::get(len->getType(), 0), origBB);
+  IRBuilder<> LoopBuilder(LoopBB);
+  PHINode *LoopIndex = LoopBuilder.CreatePHI(CopyLen->getType(), 0);
+  LoopIndex->addIncoming(ConstantInt::get(CopyLen->getType(), 0), OrigBB);
 
-  loop.CreateStore(val, loop.CreateGEP(val->getType(), dstAddr, ind), false);
+  LoopBuilder.CreateStore(
+      SetValue,
+      LoopBuilder.CreateInBoundsGEP(SetValue->getType(), DstAddr, LoopIndex),
+      false);
 
-  Value *newind = loop.CreateAdd(ind, ConstantInt::get(len->getType(), 1));
-  ind->addIncoming(newind, loopBB);
+  Value *NewIndex =
+      LoopBuilder.CreateAdd(LoopIndex, ConstantInt::get(CopyLen->getType(), 1));
+  LoopIndex->addIncoming(NewIndex, LoopBB);
 
-  loop.CreateCondBr(loop.CreateICmpULT(newind, len), loopBB, newBB);
+  LoopBuilder.CreateCondBr(LoopBuilder.CreateICmpULT(NewIndex, CopyLen), LoopBB,
+                           NewBB);
 }
 
 bool NVPTXLowerAggrCopies::runOnFunction(Function &F) {
-  SmallVector<LoadInst *, 4> aggrLoads;
-  SmallVector<MemTransferInst *, 4> aggrMemcpys;
-  SmallVector<MemSetInst *, 4> aggrMemsets;
+  SmallVector<LoadInst *, 4> AggrLoads;
+  SmallVector<MemIntrinsic *, 4> MemCalls;
 
   const DataLayout &DL = F.getParent()->getDataLayout();
   LLVMContext &Context = F.getParent()->getContext();
 
-  //
-  // Collect all the aggrLoads, aggrMemcpys and addrMemsets.
-  //
-  //const BasicBlock *firstBB = &F.front();  // first BB in F
+  // Collect all aggregate loads and mem* calls.
   for (Function::iterator BI = F.begin(), BE = F.end(); BI != BE; ++BI) {
-    //BasicBlock *bb = BI;
     for (BasicBlock::iterator II = BI->begin(), IE = BI->end(); II != IE;
          ++II) {
-      if (LoadInst *load = dyn_cast<LoadInst>(II)) {
-
-        if (!load->hasOneUse())
+      if (LoadInst *LI = dyn_cast<LoadInst>(II)) {
+        if (!LI->hasOneUse())
           continue;
 
-        if (DL.getTypeStoreSize(load->getType()) < MaxAggrCopySize)
+        if (DL.getTypeStoreSize(LI->getType()) < MaxAggrCopySize)
           continue;
 
-        User *use = load->user_back();
-        if (StoreInst *store = dyn_cast<StoreInst>(use)) {
-          if (store->getOperand(0) != load) //getValueOperand
+        if (StoreInst *SI = dyn_cast<StoreInst>(LI->user_back())) {
+          if (SI->getOperand(0) != LI)
             continue;
-          aggrLoads.push_back(load);
-        }
-      } else if (MemTransferInst *intr = dyn_cast<MemTransferInst>(II)) {
-        Value *len = intr->getLength();
-        // If the number of elements being copied is greater
-        // than MaxAggrCopySize, lower it to a loop
-        if (ConstantInt *len_int = dyn_cast<ConstantInt>(len)) {
-          if (len_int->getZExtValue() >= MaxAggrCopySize) {
-            aggrMemcpys.push_back(intr);
-          }
-        } else {
-          // turn variable length memcpy/memmov into loop
-          aggrMemcpys.push_back(intr);
+          AggrLoads.push_back(LI);
         }
-      } else if (MemSetInst *memsetintr = dyn_cast<MemSetInst>(II)) {
-        Value *len = memsetintr->getLength();
-        if (ConstantInt *len_int = dyn_cast<ConstantInt>(len)) {
-          if (len_int->getZExtValue() >= MaxAggrCopySize) {
-            aggrMemsets.push_back(memsetintr);
+      } else if (MemIntrinsic *IntrCall = dyn_cast<MemIntrinsic>(II)) {
+        // Convert intrinsic calls with variable size or with constant size
+        // larger than the MaxAggrCopySize threshold.
+        if (ConstantInt *LenCI = dyn_cast<ConstantInt>(IntrCall->getLength())) {
+          if (LenCI->getZExtValue() >= MaxAggrCopySize) {
+            MemCalls.push_back(IntrCall);
           }
         } else {
-          // turn variable length memset into loop
-          aggrMemsets.push_back(memsetintr);
+          MemCalls.push_back(IntrCall);
         }
       }
     }
   }
-  if ((aggrLoads.size() == 0) && (aggrMemcpys.size() == 0) &&
-      (aggrMemsets.size() == 0))
+
+  if (AggrLoads.size() == 0 && MemCalls.size() == 0) {
     return false;
+  }
 
   //
   // Do the transformation of an aggr load/copy/set to a loop
   //
-  for (unsigned i = 0, e = aggrLoads.size(); i != e; ++i) {
-    LoadInst *load = aggrLoads[i];
-    StoreInst *store = dyn_cast<StoreInst>(*load->user_begin());
-    Value *srcAddr = load->getOperand(0);
-    Value *dstAddr = store->getOperand(1);
-    unsigned numLoads = DL.getTypeStoreSize(load->getType());
-    Value *len = ConstantInt::get(Type::getInt32Ty(Context), numLoads);
-
-    convertTransferToLoop(store, srcAddr, dstAddr, len, load->isVolatile(),
-                          store->isVolatile(), Context, F);
-
-    store->eraseFromParent();
-    load->eraseFromParent();
+  for (LoadInst *LI : AggrLoads) {
+    StoreInst *SI = dyn_cast<StoreInst>(*LI->user_begin());
+    Value *SrcAddr = LI->getOperand(0);
+    Value *DstAddr = SI->getOperand(1);
+    unsigned NumLoads = DL.getTypeStoreSize(LI->getType());
+    Value *CopyLen = ConstantInt::get(Type::getInt32Ty(Context), NumLoads);
+
+    convertMemCpyToLoop(/* ConvertedInst */ SI,
+                        /* SrcAddr */ SrcAddr, /* DstAddr */ DstAddr,
+                        /* CopyLen */ CopyLen,
+                        /* SrcIsVolatile */ LI->isVolatile(),
+                        /* DstIsVolatile */ SI->isVolatile(),
+                        /* Context */ Context,
+                        /* Function F */ F);
+
+    SI->eraseFromParent();
+    LI->eraseFromParent();
   }
 
-  for (unsigned i = 0, e = aggrMemcpys.size(); i != e; ++i) {
-    MemTransferInst *cpy = aggrMemcpys[i];
-    Value *len = cpy->getLength();
-    // llvm 2.7 version of memcpy does not have volatile
-    // operand yet. So always making it non-volatile
-    // optimistically, so that we don't see unnecessary
-    // st.volatile in ptx
-    convertTransferToLoop(cpy, cpy->getSource(), cpy->getDest(), len, false,
-                          false, Context, F);
-    cpy->eraseFromParent();
-  }
-
-  for (unsigned i = 0, e = aggrMemsets.size(); i != e; ++i) {
-    MemSetInst *memsetinst = aggrMemsets[i];
-    Value *len = memsetinst->getLength();
-    Value *val = memsetinst->getValue();
-    convertMemSetToLoop(memsetinst, memsetinst->getDest(), len, val, Context,
-                        F);
-    memsetinst->eraseFromParent();
+  // Transform mem* intrinsic calls.
+  for (MemIntrinsic *MemCall : MemCalls) {
+    if (MemCpyInst *Memcpy = dyn_cast<MemCpyInst>(MemCall)) {
+      convertMemCpyToLoop(/* ConvertedInst */ Memcpy,
+                          /* SrcAddr */ Memcpy->getRawSource(),
+                          /* DstAddr */ Memcpy->getRawDest(),
+                          /* CopyLen */ Memcpy->getLength(),
+                          /* SrcIsVolatile */ Memcpy->isVolatile(),
+                          /* DstIsVolatile */ Memcpy->isVolatile(),
+                          /* Context */ Context,
+                          /* Function F */ F);
+    } else if (MemMoveInst *Memmove = dyn_cast<MemMoveInst>(MemCall)) {
+      convertMemMoveToLoop(/* ConvertedInst */ Memmove,
+                           /* SrcAddr */ Memmove->getRawSource(),
+                           /* DstAddr */ Memmove->getRawDest(),
+                           /* CopyLen */ Memmove->getLength(),
+                           /* SrcIsVolatile */ Memmove->isVolatile(),
+                           /* DstIsVolatile */ Memmove->isVolatile(),
+                           /* Context */ Context,
+                           /* Function F */ F);
+
+    } else if (MemSetInst *Memset = dyn_cast<MemSetInst>(MemCall)) {
+      convertMemSetToLoop(/* ConvertedInst */ Memset,
+                          /* DstAddr */ Memset->getRawDest(),
+                          /* CopyLen */ Memset->getLength(),
+                          /* SetValue */ Memset->getValue(),
+                          /* Context */ Context,
+                          /* Function F */ F);
+    }
+    MemCall->eraseFromParent();
   }
 
   return true;
 }
 
+} // namespace
+
+namespace llvm {
+void initializeNVPTXLowerAggrCopiesPass(PassRegistry &);
+}
+
+INITIALIZE_PASS(NVPTXLowerAggrCopies, "nvptx-lower-aggr-copies",
+                "Lower aggregate copies, and llvm.mem* intrinsics into loops",
+                false, false)
+
 FunctionPass *llvm::createLowerAggrCopies() {
   return new NVPTXLowerAggrCopies();
 }
diff --git a/lib/Target/NVPTX/NVPTXLowerAlloca.cpp b/lib/Target/NVPTX/NVPTXLowerAlloca.cpp
new file mode 100644
index 00000000000..93d0025d8f5
--- /dev/null
+++ b/lib/Target/NVPTX/NVPTXLowerAlloca.cpp
@@ -0,0 +1,115 @@
+//===-- NVPTXLowerAlloca.cpp - Make alloca to use local memory =====--===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// For all alloca instructions, and add a pair of cast to local address for
+// each of them. For example,
+//
+//   %A = alloca i32
+//   store i32 0, i32* %A ; emits st.u32
+//
+// will be transformed to
+//
+//   %A = alloca i32
+//   %Local = addrspacecast i32* %A to i32 addrspace(5)*
+//   %Generic = addrspacecast i32 addrspace(5)* %A to i32*
+//   store i32 0, i32 addrspace(5)* %Generic ; emits st.local.u32
+//
+// And we will rely on NVPTXFavorNonGenericAddrSpace to combine the last
+// two instructions.
+//
+//===----------------------------------------------------------------------===//
+
+#include "NVPTX.h"
+#include "NVPTXUtilities.h"
+#include "llvm/IR/Function.h"
+#include "llvm/IR/Instructions.h"
+#include "llvm/IR/IntrinsicInst.h"
+#include "llvm/IR/Module.h"
+#include "llvm/IR/Type.h"
+#include "llvm/Pass.h"
+
+using namespace llvm;
+
+namespace llvm {
+void initializeNVPTXLowerAllocaPass(PassRegistry &);
+}
+
+namespace {
+class NVPTXLowerAlloca : public BasicBlockPass {
+  bool runOnBasicBlock(BasicBlock &BB) override;
+
+public:
+  static char ID; // Pass identification, replacement for typeid
+  NVPTXLowerAlloca() : BasicBlockPass(ID) {}
+  const char *getPassName() const override {
+    return "convert address space of alloca'ed memory to local";
+  }
+};
+} // namespace
+
+char NVPTXLowerAlloca::ID = 1;
+
+INITIALIZE_PASS(NVPTXLowerAlloca, "nvptx-lower-alloca",
+                "Lower Alloca", false, false)
+
+// =============================================================================
+// Main function for this pass.
+// =============================================================================
+bool NVPTXLowerAlloca::runOnBasicBlock(BasicBlock &BB) {
+  bool Changed = false;
+  for (auto &I : BB) {
+    if (auto allocaInst = dyn_cast<AllocaInst>(&I)) {
+      Changed = true;
+      auto PTy = dyn_cast<PointerType>(allocaInst->getType());
+      auto ETy = PTy->getElementType();
+      auto LocalAddrTy = PointerType::get(ETy, ADDRESS_SPACE_LOCAL);
+      auto NewASCToLocal = new AddrSpaceCastInst(allocaInst, LocalAddrTy, "");
+      auto GenericAddrTy = PointerType::get(ETy, ADDRESS_SPACE_GENERIC);
+      auto NewASCToGeneric = new AddrSpaceCastInst(NewASCToLocal,
+                                                    GenericAddrTy, "");
+      NewASCToLocal->insertAfter(allocaInst);
+      NewASCToGeneric->insertAfter(NewASCToLocal);
+      for (Value::use_iterator UI = allocaInst->use_begin(),
+                                UE = allocaInst->use_end();
+            UI != UE; ) {
+        // Check Load, Store, GEP, and BitCast Uses on alloca and make them
+        // use the converted generic address, in order to expose non-generic
+        // addrspacecast to NVPTXFavorNonGenericAddrSpace. For other types
+        // of instructions this is unecessary and may introduce redudant
+        // address cast.
+        const auto &AllocaUse = *UI++;
+        auto LI = dyn_cast<LoadInst>(AllocaUse.getUser());
+        if (LI && LI->getPointerOperand() == allocaInst && !LI->isVolatile()) {
+          LI->setOperand(LI->getPointerOperandIndex(), NewASCToGeneric);
+          continue;
+        }
+        auto SI = dyn_cast<StoreInst>(AllocaUse.getUser());
+        if (SI && SI->getPointerOperand() == allocaInst && !SI->isVolatile()) {
+          SI->setOperand(SI->getPointerOperandIndex(), NewASCToGeneric);
+          continue;
+        }
+        auto GI = dyn_cast<GetElementPtrInst>(AllocaUse.getUser());
+        if (GI && GI->getPointerOperand() == allocaInst) {
+          GI->setOperand(GI->getPointerOperandIndex(), NewASCToGeneric);
+          continue;
+        }
+        auto BI = dyn_cast<BitCastInst>(AllocaUse.getUser());
+        if (BI && BI->getOperand(0) == allocaInst) {
+          BI->setOperand(0, NewASCToGeneric);
+          continue;
+        }
+      }
+    }
+  }
+  return Changed;
+}
+
+BasicBlockPass *llvm::createNVPTXLowerAllocaPass() {
+  return new NVPTXLowerAlloca();
+}
diff --git a/lib/Target/NVPTX/NVPTXLowerKernelArgs.cpp b/lib/Target/NVPTX/NVPTXLowerKernelArgs.cpp
new file mode 100644
index 00000000000..b533f316d8a
--- /dev/null
+++ b/lib/Target/NVPTX/NVPTXLowerKernelArgs.cpp
@@ -0,0 +1,174 @@
+//===-- NVPTXLowerKernelArgs.cpp - Lower kernel arguments -----------------===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// Pointer arguments to kernel functions need to be lowered specially.
+//
+// 1. Copy byval struct args to local memory. This is a preparation for handling
+//    cases like
+//
+//    kernel void foo(struct A arg, ...)
+//    {
+//      struct A *p = &arg;
+//      ...
+//      ... = p->filed1 ...  (this is no generic address for .param)
+//      p->filed2 = ...      (this is no write access to .param)
+//    }
+//
+// 2. Convert non-byval pointer arguments of CUDA kernels to pointers in the
+//    global address space. This allows later optimizations to emit
+//    ld.global.*/st.global.* for accessing these pointer arguments. For
+//    example,
+//
+//    define void @foo(float* %input) {
+//      %v = load float, float* %input, align 4
+//      ...
+//    }
+//
+//    becomes
+//
+//    define void @foo(float* %input) {
+//      %input2 = addrspacecast float* %input to float addrspace(1)*
+//      %input3 = addrspacecast float addrspace(1)* %input2 to float*
+//      %v = load float, float* %input3, align 4
+//      ...
+//    }
+//
+//    Later, NVPTXFavorNonGenericAddrSpaces will optimize it to
+//
+//    define void @foo(float* %input) {
+//      %input2 = addrspacecast float* %input to float addrspace(1)*
+//      %v = load float, float addrspace(1)* %input2, align 4
+//      ...
+//    }
+//
+// TODO: merge this pass with NVPTXFavorNonGenericAddrSpace so that other passes
+// don't cancel the addrspacecast pair this pass emits.
+//===----------------------------------------------------------------------===//
+
+#include "NVPTX.h"
+#include "NVPTXUtilities.h"
+#include "NVPTXTargetMachine.h"
+#include "llvm/IR/Function.h"
+#include "llvm/IR/Instructions.h"
+#include "llvm/IR/Module.h"
+#include "llvm/IR/Type.h"
+#include "llvm/Pass.h"
+
+using namespace llvm;
+
+namespace llvm {
+void initializeNVPTXLowerKernelArgsPass(PassRegistry &);
+}
+
+namespace {
+class NVPTXLowerKernelArgs : public FunctionPass {
+  bool runOnFunction(Function &F) override;
+
+  // handle byval parameters
+  void handleByValParam(Argument *);
+  // handle non-byval pointer parameters
+  void handlePointerParam(Argument *);
+
+public:
+  static char ID; // Pass identification, replacement for typeid
+  NVPTXLowerKernelArgs(const NVPTXTargetMachine *TM = nullptr)
+      : FunctionPass(ID), TM(TM) {}
+  const char *getPassName() const override {
+    return "Lower pointer arguments of CUDA kernels";
+  }
+
+private:
+  const NVPTXTargetMachine *TM;
+};
+} // namespace
+
+char NVPTXLowerKernelArgs::ID = 1;
+
+INITIALIZE_PASS(NVPTXLowerKernelArgs, "nvptx-lower-kernel-args",
+                "Lower kernel arguments (NVPTX)", false, false)
+
+// =============================================================================
+// If the function had a byval struct ptr arg, say foo(%struct.x *byval %d),
+// then add the following instructions to the first basic block:
+//
+// %temp = alloca %struct.x, align 8
+// %tempd = addrspacecast %struct.x* %d to %struct.x addrspace(101)*
+// %tv = load %struct.x addrspace(101)* %tempd
+// store %struct.x %tv, %struct.x* %temp, align 8
+//
+// The above code allocates some space in the stack and copies the incoming
+// struct from param space to local space.
+// Then replace all occurences of %d by %temp.
+// =============================================================================
+void NVPTXLowerKernelArgs::handleByValParam(Argument *Arg) {
+  Function *Func = Arg->getParent();
+  Instruction *FirstInst = &(Func->getEntryBlock().front());
+  PointerType *PType = dyn_cast<PointerType>(Arg->getType());
+
+  assert(PType && "Expecting pointer type in handleByValParam");
+
+  Type *StructType = PType->getElementType();
+  AllocaInst *AllocA = new AllocaInst(StructType, Arg->getName(), FirstInst);
+  // Set the alignment to alignment of the byval parameter. This is because,
+  // later load/stores assume that alignment, and we are going to replace
+  // the use of the byval parameter with this alloca instruction.
+  AllocA->setAlignment(Func->getParamAlignment(Arg->getArgNo() + 1));
+  Arg->replaceAllUsesWith(AllocA);
+
+  Value *ArgInParam = new AddrSpaceCastInst(
+      Arg, PointerType::get(StructType, ADDRESS_SPACE_PARAM), Arg->getName(),
+      FirstInst);
+  LoadInst *LI = new LoadInst(ArgInParam, Arg->getName(), FirstInst);
+  new StoreInst(LI, AllocA, FirstInst);
+}
+
+void NVPTXLowerKernelArgs::handlePointerParam(Argument *Arg) {
+  assert(!Arg->hasByValAttr() &&
+         "byval params should be handled by handleByValParam");
+
+  // Do nothing if the argument already points to the global address space.
+  if (Arg->getType()->getPointerAddressSpace() == ADDRESS_SPACE_GLOBAL)
+    return;
+
+  Instruction *FirstInst = Arg->getParent()->getEntryBlock().begin();
+  Instruction *ArgInGlobal = new AddrSpaceCastInst(
+      Arg, PointerType::get(Arg->getType()->getPointerElementType(),
+                            ADDRESS_SPACE_GLOBAL),
+      Arg->getName(), FirstInst);
+  Value *ArgInGeneric = new AddrSpaceCastInst(ArgInGlobal, Arg->getType(),
+                                              Arg->getName(), FirstInst);
+  // Replace with ArgInGeneric all uses of Args except ArgInGlobal.
+  Arg->replaceAllUsesWith(ArgInGeneric);
+  ArgInGlobal->setOperand(0, Arg);
+}
+
+
+// =============================================================================
+// Main function for this pass.
+// =============================================================================
+bool NVPTXLowerKernelArgs::runOnFunction(Function &F) {
+  // Skip non-kernels. See the comments at the top of this file.
+  if (!isKernelFunction(F))
+    return false;
+
+  for (Argument &Arg : F.args()) {
+    if (Arg.getType()->isPointerTy()) {
+      if (Arg.hasByValAttr())
+        handleByValParam(&Arg);
+      else if (TM && TM->getDrvInterface() == NVPTX::CUDA)
+        handlePointerParam(&Arg);
+    }
+  }
+  return true;
+}
+
+FunctionPass *
+llvm::createNVPTXLowerKernelArgsPass(const NVPTXTargetMachine *TM) {
+  return new NVPTXLowerKernelArgs(TM);
+}
diff --git a/lib/Target/NVPTX/NVPTXLowerStructArgs.cpp b/lib/Target/NVPTX/NVPTXLowerStructArgs.cpp
deleted file mode 100644
index 68dfbb71613..00000000000
--- a/lib/Target/NVPTX/NVPTXLowerStructArgs.cpp
+++ /dev/null
@@ -1,136 +0,0 @@
-//===-- NVPTXLowerStructArgs.cpp - Copy struct args to local memory =====--===//
-//
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
-//
-//===----------------------------------------------------------------------===//
-//
-// Copy struct args to local memory. This is needed for kernel functions only.
-// This is a preparation for handling cases like
-//
-// kernel void foo(struct A arg, ...)
-// {
-//     struct A *p = &arg;
-//     ...
-//     ... = p->filed1 ...  (this is no generic address for .param)
-//     p->filed2 = ...      (this is no write access to .param)
-// }
-//
-//===----------------------------------------------------------------------===//
-
-#include "NVPTX.h"
-#include "NVPTXUtilities.h"
-#include "llvm/IR/Function.h"
-#include "llvm/IR/Instructions.h"
-#include "llvm/IR/IntrinsicInst.h"
-#include "llvm/IR/Module.h"
-#include "llvm/IR/Type.h"
-#include "llvm/Pass.h"
-
-using namespace llvm;
-
-namespace llvm {
-void initializeNVPTXLowerStructArgsPass(PassRegistry &);
-}
-
-namespace {
-class NVPTXLowerStructArgs : public FunctionPass {
-  bool runOnFunction(Function &F) override;
-
-  void handleStructPtrArgs(Function &);
-  void handleParam(Argument *);
-
-public:
-  static char ID; // Pass identification, replacement for typeid
-  NVPTXLowerStructArgs() : FunctionPass(ID) {}
-  const char *getPassName() const override {
-    return "Copy structure (byval *) arguments to stack";
-  }
-};
-} // namespace
-
-char NVPTXLowerStructArgs::ID = 1;
-
-INITIALIZE_PASS(NVPTXLowerStructArgs, "nvptx-lower-struct-args",
-                "Lower structure arguments (NVPTX)", false, false)
-
-void NVPTXLowerStructArgs::handleParam(Argument *Arg) {
-  Function *Func = Arg->getParent();
-  Instruction *FirstInst = &(Func->getEntryBlock().front());
-  PointerType *PType = dyn_cast<PointerType>(Arg->getType());
-
-  assert(PType && "Expecting pointer type in handleParam");
-
-  Type *StructType = PType->getElementType();
-  AllocaInst *AllocA = new AllocaInst(StructType, Arg->getName(), FirstInst);
-
-  /* Set the alignment to alignment of the byval parameter. This is because,
-   * later load/stores assume that alignment, and we are going to replace
-   * the use of the byval parameter with this alloca instruction.
-   */
-  AllocA->setAlignment(Func->getParamAlignment(Arg->getArgNo() + 1));
-
-  Arg->replaceAllUsesWith(AllocA);
-
-  // Get the cvt.gen.to.param intrinsic
-  Type *CvtTypes[] = {
-      Type::getInt8PtrTy(Func->getParent()->getContext(), ADDRESS_SPACE_PARAM),
-      Type::getInt8PtrTy(Func->getParent()->getContext(),
-                         ADDRESS_SPACE_GENERIC)};
-  Function *CvtFunc = Intrinsic::getDeclaration(
-      Func->getParent(), Intrinsic::nvvm_ptr_gen_to_param, CvtTypes);
-
-  Value *BitcastArgs[] = {
-      new BitCastInst(Arg, Type::getInt8PtrTy(Func->getParent()->getContext(),
-                                              ADDRESS_SPACE_GENERIC),
-                      Arg->getName(), FirstInst)};
-  CallInst *CallCVT =
-      CallInst::Create(CvtFunc, BitcastArgs, "cvt_to_param", FirstInst);
-
-  BitCastInst *BitCast = new BitCastInst(
-      CallCVT, PointerType::get(StructType, ADDRESS_SPACE_PARAM),
-      Arg->getName(), FirstInst);
-  LoadInst *LI = new LoadInst(BitCast, Arg->getName(), FirstInst);
-  new StoreInst(LI, AllocA, FirstInst);
-}
-
-// =============================================================================
-// If the function had a struct ptr arg, say foo(%struct.x *byval %d), then
-// add the following instructions to the first basic block :
-//
-// %temp = alloca %struct.x, align 8
-// %tt1 = bitcast %struct.x * %d to i8 *
-// %tt2 = llvm.nvvm.cvt.gen.to.param %tt2
-// %tempd = bitcast i8 addrspace(101) * to %struct.x addrspace(101) *
-// %tv = load %struct.x addrspace(101) * %tempd
-// store %struct.x %tv, %struct.x * %temp, align 8
-//
-// The above code allocates some space in the stack and copies the incoming
-// struct from param space to local space.
-// Then replace all occurences of %d by %temp.
-// =============================================================================
-void NVPTXLowerStructArgs::handleStructPtrArgs(Function &F) {
-  for (Argument &Arg : F.args()) {
-    if (Arg.getType()->isPointerTy() && Arg.hasByValAttr()) {
-      handleParam(&Arg);
-    }
-  }
-}
-
-// =============================================================================
-// Main function for this pass.
-// =============================================================================
-bool NVPTXLowerStructArgs::runOnFunction(Function &F) {
-  // Skip non-kernels. See the comments at the top of this file.
-  if (!isKernelFunction(F))
-    return false;
-
-  handleStructPtrArgs(F);
-  return true;
-}
-
-FunctionPass *llvm::createNVPTXLowerStructArgsPass() {
-  return new NVPTXLowerStructArgs();
-}
diff --git a/lib/Target/NVPTX/NVPTXMCExpr.cpp b/lib/Target/NVPTX/NVPTXMCExpr.cpp
index 137248b19a6..3c98b9febf8 100644
--- a/lib/Target/NVPTX/NVPTXMCExpr.cpp
+++ b/lib/Target/NVPTX/NVPTXMCExpr.cpp
@@ -16,11 +16,11 @@ using namespace llvm;
 #define DEBUG_TYPE "nvptx-mcexpr"
 
 const NVPTXFloatMCExpr*
-NVPTXFloatMCExpr::Create(VariantKind Kind, APFloat Flt, MCContext &Ctx) {
+NVPTXFloatMCExpr::create(VariantKind Kind, APFloat Flt, MCContext &Ctx) {
   return new (Ctx) NVPTXFloatMCExpr(Kind, Flt);
 }
 
-void NVPTXFloatMCExpr::PrintImpl(raw_ostream &OS) const {
+void NVPTXFloatMCExpr::printImpl(raw_ostream &OS, const MCAsmInfo *MAI) const {
   bool Ignored;
   unsigned NumHex;
   APFloat APF = getAPFloat();
@@ -45,3 +45,16 @@ void NVPTXFloatMCExpr::PrintImpl(raw_ostream &OS) const {
     OS << std::string(NumHex - HexStr.length(), '0');
   OS << utohexstr(API.getZExtValue());
 }
+
+const NVPTXGenericMCSymbolRefExpr*
+NVPTXGenericMCSymbolRefExpr::create(const MCSymbolRefExpr *SymExpr,
+                                    MCContext &Ctx) {
+  return new (Ctx) NVPTXGenericMCSymbolRefExpr(SymExpr);
+}
+
+void NVPTXGenericMCSymbolRefExpr::printImpl(raw_ostream &OS,
+                                            const MCAsmInfo *MAI) const {
+  OS << "generic(";
+  SymExpr->print(OS, MAI);
+  OS << ")";
+}
diff --git a/lib/Target/NVPTX/NVPTXMCExpr.h b/lib/Target/NVPTX/NVPTXMCExpr.h
index f075b8b1733..46b4b33e7e4 100644
--- a/lib/Target/NVPTX/NVPTXMCExpr.h
+++ b/lib/Target/NVPTX/NVPTXMCExpr.h
@@ -36,17 +36,17 @@ class NVPTXFloatMCExpr : public MCTargetExpr {
   /// @name Construction
   /// @{
 
-  static const NVPTXFloatMCExpr *Create(VariantKind Kind, APFloat Flt,
+  static const NVPTXFloatMCExpr *create(VariantKind Kind, APFloat Flt,
                                         MCContext &Ctx);
 
-  static const NVPTXFloatMCExpr *CreateConstantFPSingle(APFloat Flt,
+  static const NVPTXFloatMCExpr *createConstantFPSingle(APFloat Flt,
                                                         MCContext &Ctx) {
-    return Create(VK_NVPTX_SINGLE_PREC_FLOAT, Flt, Ctx);
+    return create(VK_NVPTX_SINGLE_PREC_FLOAT, Flt, Ctx);
   }
 
-  static const NVPTXFloatMCExpr *CreateConstantFPDouble(APFloat Flt,
+  static const NVPTXFloatMCExpr *createConstantFPDouble(APFloat Flt,
                                                         MCContext &Ctx) {
-    return Create(VK_NVPTX_DOUBLE_PREC_FLOAT, Flt, Ctx);
+    return create(VK_NVPTX_DOUBLE_PREC_FLOAT, Flt, Ctx);
   }
 
   /// @}
@@ -61,16 +61,14 @@ class NVPTXFloatMCExpr : public MCTargetExpr {
 
 /// @}
 
-  void PrintImpl(raw_ostream &OS) const override;
-  bool EvaluateAsRelocatableImpl(MCValue &Res,
+  void printImpl(raw_ostream &OS, const MCAsmInfo *MAI) const override;
+  bool evaluateAsRelocatableImpl(MCValue &Res,
                                  const MCAsmLayout *Layout,
                                  const MCFixup *Fixup) const override {
     return false;
   }
   void visitUsedExpr(MCStreamer &Streamer) const override {};
-  const MCSection *FindAssociatedSection() const override {
-    return nullptr;
-  }
+  MCSection *findAssociatedSection() const override { return nullptr; }
 
   // There are no TLS NVPTXMCExprs at the moment.
   void fixELFSymbolsInTLSFixups(MCAssembler &Asm) const override {}
@@ -79,6 +77,48 @@ class NVPTXFloatMCExpr : public MCTargetExpr {
     return E->getKind() == MCExpr::Target;
   }
 };
+
+/// A wrapper for MCSymbolRefExpr that tells the assembly printer that the
+/// symbol should be enclosed by generic().
+class NVPTXGenericMCSymbolRefExpr : public MCTargetExpr {
+private:
+  const MCSymbolRefExpr *SymExpr;
+
+  explicit NVPTXGenericMCSymbolRefExpr(const MCSymbolRefExpr *_SymExpr)
+      : SymExpr(_SymExpr) {}
+
+public:
+  /// @name Construction
+  /// @{
+
+  static const NVPTXGenericMCSymbolRefExpr
+  *create(const MCSymbolRefExpr *SymExpr, MCContext &Ctx);
+
+  /// @}
+  /// @name Accessors
+  /// @{
+
+  /// getOpcode - Get the kind of this expression.
+  const MCSymbolRefExpr *getSymbolExpr() const { return SymExpr; }
+
+  /// @}
+
+  void printImpl(raw_ostream &OS, const MCAsmInfo *MAI) const override;
+  bool evaluateAsRelocatableImpl(MCValue &Res,
+                                 const MCAsmLayout *Layout,
+                                 const MCFixup *Fixup) const override {
+    return false;
+  }
+  void visitUsedExpr(MCStreamer &Streamer) const override {};
+  MCSection *findAssociatedSection() const override { return nullptr; }
+
+  // There are no TLS NVPTXMCExprs at the moment.
+  void fixELFSymbolsInTLSFixups(MCAssembler &Asm) const override {}
+
+  static bool classof(const MCExpr *E) {
+    return E->getKind() == MCExpr::Target;
+  }
+  };
 } // end namespace llvm
 
 #endif
diff --git a/lib/Target/NVPTX/NVPTXPeephole.cpp b/lib/Target/NVPTX/NVPTXPeephole.cpp
new file mode 100644
index 00000000000..a61c291d233
--- /dev/null
+++ b/lib/Target/NVPTX/NVPTXPeephole.cpp
@@ -0,0 +1,154 @@
+//===-- NVPTXPeephole.cpp - NVPTX Peephole Optimiztions -------------------===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// In NVPTX, NVPTXFrameLowering will emit following instruction at the beginning
+// of a MachineFunction.
+//
+//   mov %SPL, %depot
+//   cvta.local %SP, %SPL
+//
+// Because Frame Index is a generic address and alloca can only return generic
+// pointer, without this pass the instructions producing alloca'ed address will
+// be based on %SP. NVPTXLowerAlloca tends to help replace store and load on
+// this address with their .local versions, but this may introduce a lot of
+// cvta.to.local instructions. Performance can be improved if we avoid casting
+// address back and forth and directly calculate local address based on %SPL.
+// This peephole pass optimizes these cases, for example
+//
+// It will transform the following pattern
+//    %vreg0<def> = LEA_ADDRi64 %VRFrame, 4
+//    %vreg1<def> = cvta_to_local_yes_64 %vreg0
+//
+// into
+//    %vreg1<def> = LEA_ADDRi64 %VRFrameLocal, 4
+//
+// %VRFrameLocal is the virtual register name of %SPL
+//
+//===----------------------------------------------------------------------===//
+
+#include "NVPTX.h"
+#include "llvm/CodeGen/MachineFunctionPass.h"
+#include "llvm/CodeGen/MachineInstrBuilder.h"
+#include "llvm/CodeGen/MachineRegisterInfo.h"
+#include "llvm/Target/TargetRegisterInfo.h"
+#include "llvm/Target/TargetInstrInfo.h"
+
+using namespace llvm;
+
+#define DEBUG_TYPE "nvptx-peephole"
+
+namespace llvm {
+void initializeNVPTXPeepholePass(PassRegistry &);
+}
+
+namespace {
+struct NVPTXPeephole : public MachineFunctionPass {
+ public:
+  static char ID;
+  NVPTXPeephole() : MachineFunctionPass(ID) {
+    initializeNVPTXPeepholePass(*PassRegistry::getPassRegistry());
+  }
+
+  bool runOnMachineFunction(MachineFunction &MF) override;
+
+  const char *getPassName() const override {
+    return "NVPTX optimize redundant cvta.to.local instruction";
+  }
+
+  void getAnalysisUsage(AnalysisUsage &AU) const override {
+    MachineFunctionPass::getAnalysisUsage(AU);
+  }
+};
+}
+
+char NVPTXPeephole::ID = 0;
+
+INITIALIZE_PASS(NVPTXPeephole, "nvptx-peephole", "NVPTX Peephole", false, false)
+
+static bool isCVTAToLocalCombinationCandidate(MachineInstr &Root) {
+  auto &MBB = *Root.getParent();
+  auto &MF = *MBB.getParent();
+  // Check current instruction is cvta.to.local
+  if (Root.getOpcode() != NVPTX::cvta_to_local_yes_64 &&
+      Root.getOpcode() != NVPTX::cvta_to_local_yes)
+    return false;
+
+  auto &Op = Root.getOperand(1);
+  const auto &MRI = MF.getRegInfo();
+  MachineInstr *GenericAddrDef = nullptr;
+  if (Op.isReg() && TargetRegisterInfo::isVirtualRegister(Op.getReg())) {
+    GenericAddrDef = MRI.getUniqueVRegDef(Op.getReg());
+  }
+
+  // Check the register operand is uniquely defined by LEA_ADDRi instruction
+  if (!GenericAddrDef || GenericAddrDef->getParent() != &MBB ||
+      (GenericAddrDef->getOpcode() != NVPTX::LEA_ADDRi64 &&
+       GenericAddrDef->getOpcode() != NVPTX::LEA_ADDRi)) {
+    return false;
+  }
+
+  // Check the LEA_ADDRi operand is Frame index
+  auto &BaseAddrOp = GenericAddrDef->getOperand(1);
+  if (BaseAddrOp.isReg() && BaseAddrOp.getReg() == NVPTX::VRFrame) {
+    return true;
+  }
+
+  return false;
+}
+
+static void CombineCVTAToLocal(MachineInstr &Root) {
+  auto &MBB = *Root.getParent();
+  auto &MF = *MBB.getParent();
+  const auto &MRI = MF.getRegInfo();
+  const TargetInstrInfo *TII = MF.getSubtarget().getInstrInfo();
+  auto &Prev = *MRI.getUniqueVRegDef(Root.getOperand(1).getReg());
+
+  MachineInstrBuilder MIB =
+      BuildMI(MF, Root.getDebugLoc(), TII->get(Prev.getOpcode()),
+              Root.getOperand(0).getReg())
+          .addReg(NVPTX::VRFrameLocal)
+          .addOperand(Prev.getOperand(2));
+
+  MBB.insert((MachineBasicBlock::iterator)&Root, MIB);
+
+  // Check if MRI has only one non dbg use, which is Root
+  if (MRI.hasOneNonDBGUse(Prev.getOperand(0).getReg())) {
+    Prev.eraseFromParentAndMarkDBGValuesForRemoval();
+  }
+  Root.eraseFromParentAndMarkDBGValuesForRemoval();
+}
+
+bool NVPTXPeephole::runOnMachineFunction(MachineFunction &MF) {
+  bool Changed = false;
+  // Loop over all of the basic blocks.
+  for (auto &MBB : MF) {
+    // Traverse the basic block.
+    auto BlockIter = MBB.begin();
+
+    while (BlockIter != MBB.end()) {
+      auto &MI = *BlockIter++;
+      if (isCVTAToLocalCombinationCandidate(MI)) {
+        CombineCVTAToLocal(MI);
+        Changed = true;
+      }
+    }  // Instruction
+  }    // Basic Block
+
+  // Remove unnecessary %VRFrame = cvta.local %VRFrameLocal
+  const auto &MRI = MF.getRegInfo();
+  if (MRI.use_empty(NVPTX::VRFrame)) {
+    if (auto MI = MRI.getUniqueVRegDef(NVPTX::VRFrame)) {
+      MI->eraseFromParentAndMarkDBGValuesForRemoval();
+    }
+  }
+
+  return Changed;
+}
+
+MachineFunctionPass *llvm::createNVPTXPeephole() { return new NVPTXPeephole(); }
diff --git a/lib/Target/NVPTX/NVPTXPrologEpilogPass.cpp b/lib/Target/NVPTX/NVPTXPrologEpilogPass.cpp
index c1c67e3872e..5fd69a6815a 100644
--- a/lib/Target/NVPTX/NVPTXPrologEpilogPass.cpp
+++ b/lib/Target/NVPTX/NVPTXPrologEpilogPass.cpp
@@ -68,7 +68,7 @@ bool NVPTXPrologEpilogPass::runOnMachineFunction(MachineFunction &MF) {
   }
 
   // Add function prolog/epilog
-  TFI.emitPrologue(MF);
+  TFI.emitPrologue(MF, MF.front());
 
   for (MachineFunction::iterator I = MF.begin(), E = MF.end(); I != E; ++I) {
     // If last instruction is a return instruction, add an epilogue
diff --git a/lib/Target/NVPTX/NVPTXRegisterInfo.td b/lib/Target/NVPTX/NVPTXRegisterInfo.td
index efcee6b6f2b..ff6ccc457db 100644
--- a/lib/Target/NVPTX/NVPTXRegisterInfo.td
+++ b/lib/Target/NVPTX/NVPTXRegisterInfo.td
@@ -65,5 +65,5 @@ def Float32ArgRegs : NVPTXRegClass<[f32], 32, (add (sequence "fa%u", 0, 4))>;
 def Float64ArgRegs : NVPTXRegClass<[f64], 64, (add (sequence "da%u", 0, 4))>;
 
 // Read NVPTXRegisterInfo.cpp to see how VRFrame and VRDepot are used.
-def SpecialRegs : NVPTXRegClass<[i32], 32, (add VRFrame, VRDepot,
+def SpecialRegs : NVPTXRegClass<[i32], 32, (add VRFrame, VRFrameLocal, VRDepot,
                                             (sequence "ENVREG%u", 0, 31))>;
diff --git a/lib/Target/NVPTX/NVPTXReplaceImageHandles.cpp b/lib/Target/NVPTX/NVPTXReplaceImageHandles.cpp
index e83f735a551..5a83371b07f 100644
--- a/lib/Target/NVPTX/NVPTXReplaceImageHandles.cpp
+++ b/lib/Target/NVPTX/NVPTXReplaceImageHandles.cpp
@@ -2,7 +2,7 @@
 //
 //                     The LLVM Compiler Infrastructure
 //
-// This file is distributed under the University of Illinois Open Source 
+// This file is distributed under the University of Illinois Open Source
 // License. See LICENSE.TXT for details.
 //
 //===----------------------------------------------------------------------===//
@@ -115,7 +115,7 @@ bool NVPTXReplaceImageHandles::processInstr(MachineInstr &MI) {
 
     replaceImageHandle(Handle, MF);
 
-    return true; 
+    return true;
   }
 
   return false;
diff --git a/lib/Target/NVPTX/NVPTXSubtarget.cpp b/lib/Target/NVPTX/NVPTXSubtarget.cpp
index 069d6e179dd..bd2509a3c8c 100644
--- a/lib/Target/NVPTX/NVPTXSubtarget.cpp
+++ b/lib/Target/NVPTX/NVPTXSubtarget.cpp
@@ -43,12 +43,12 @@ NVPTXSubtarget &NVPTXSubtarget::initializeSubtargetDependencies(StringRef CPU,
   return *this;
 }
 
-NVPTXSubtarget::NVPTXSubtarget(const std::string &TT, const std::string &CPU,
+NVPTXSubtarget::NVPTXSubtarget(const Triple &TT, const std::string &CPU,
                                const std::string &FS,
                                const NVPTXTargetMachine &TM)
     : NVPTXGenSubtargetInfo(TT, CPU, FS), PTXVersion(0), SmVersion(20), TM(TM),
       InstrInfo(), TLInfo(TM, initializeSubtargetDependencies(CPU, FS)),
-      TSInfo(TM.getDataLayout()), FrameLowering() {}
+      FrameLowering() {}
 
 bool NVPTXSubtarget::hasImageHandles() const {
   // Enable handles for Kepler+, where CUDA supports indirect surfaces and
diff --git a/lib/Target/NVPTX/NVPTXSubtarget.h b/lib/Target/NVPTX/NVPTXSubtarget.h
index e9833e5823c..c7287719be5 100644
--- a/lib/Target/NVPTX/NVPTXSubtarget.h
+++ b/lib/Target/NVPTX/NVPTXSubtarget.h
@@ -52,7 +52,7 @@ class NVPTXSubtarget : public NVPTXGenSubtargetInfo {
   /// This constructor initializes the data members to match that
   /// of the specified module.
   ///
-  NVPTXSubtarget(const std::string &TT, const std::string &CPU,
+  NVPTXSubtarget(const Triple &TT, const std::string &CPU,
                  const std::string &FS, const NVPTXTargetMachine &TM);
 
   const TargetFrameLowering *getFrameLowering() const override {
diff --git a/lib/Target/NVPTX/NVPTXTargetMachine.cpp b/lib/Target/NVPTX/NVPTXTargetMachine.cpp
index 1b6bc714505..706314c9ed3 100644
--- a/lib/Target/NVPTX/NVPTXTargetMachine.cpp
+++ b/lib/Target/NVPTX/NVPTXTargetMachine.cpp
@@ -53,7 +53,9 @@ void initializeGenericToNVVMPass(PassRegistry&);
 void initializeNVPTXAllocaHoistingPass(PassRegistry &);
 void initializeNVPTXAssignValidGlobalNamesPass(PassRegistry&);
 void initializeNVPTXFavorNonGenericAddrSpacesPass(PassRegistry &);
-void initializeNVPTXLowerStructArgsPass(PassRegistry &);
+void initializeNVPTXLowerAggrCopiesPass(PassRegistry &);
+void initializeNVPTXLowerKernelArgsPass(PassRegistry &);
+void initializeNVPTXLowerAllocaPass(PassRegistry &);
 }
 
 extern "C" void LLVMInitializeNVPTXTarget() {
@@ -63,13 +65,15 @@ extern "C" void LLVMInitializeNVPTXTarget() {
 
   // FIXME: This pass is really intended to be invoked during IR optimization,
   // but it's very NVPTX-specific.
-  initializeNVVMReflectPass(*PassRegistry::getPassRegistry());
-  initializeGenericToNVVMPass(*PassRegistry::getPassRegistry());
-  initializeNVPTXAllocaHoistingPass(*PassRegistry::getPassRegistry());
-  initializeNVPTXAssignValidGlobalNamesPass(*PassRegistry::getPassRegistry());
-  initializeNVPTXFavorNonGenericAddrSpacesPass(
-    *PassRegistry::getPassRegistry());
-  initializeNVPTXLowerStructArgsPass(*PassRegistry::getPassRegistry());
+  PassRegistry &PR = *PassRegistry::getPassRegistry();
+  initializeNVVMReflectPass(PR);
+  initializeGenericToNVVMPass(PR);
+  initializeNVPTXAllocaHoistingPass(PR);
+  initializeNVPTXAssignValidGlobalNamesPass(PR);
+  initializeNVPTXFavorNonGenericAddrSpacesPass(PR);
+  initializeNVPTXLowerKernelArgsPass(PR);
+  initializeNVPTXLowerAllocaPass(PR);
+  initializeNVPTXLowerAggrCopiesPass(PR);
 }
 
 static std::string computeDataLayout(bool is64Bit) {
@@ -83,7 +87,7 @@ static std::string computeDataLayout(bool is64Bit) {
   return Ret;
 }
 
-NVPTXTargetMachine::NVPTXTargetMachine(const Target &T, StringRef TT,
+NVPTXTargetMachine::NVPTXTargetMachine(const Target &T, const Triple &TT,
                                        StringRef CPU, StringRef FS,
                                        const TargetOptions &Options,
                                        Reloc::Model RM, CodeModel::Model CM,
@@ -92,7 +96,7 @@ NVPTXTargetMachine::NVPTXTargetMachine(const Target &T, StringRef TT,
                         CM, OL),
       is64bit(is64bit), TLOF(make_unique<NVPTXTargetObjectFile>()),
       Subtarget(TT, CPU, FS, *this) {
-  if (Triple(TT).getOS() == Triple::NVCL)
+  if (TT.getOS() == Triple::NVCL)
     drvInterface = NVPTX::NVCL;
   else
     drvInterface = NVPTX::CUDA;
@@ -103,18 +107,20 @@ NVPTXTargetMachine::~NVPTXTargetMachine() {}
 
 void NVPTXTargetMachine32::anchor() {}
 
-NVPTXTargetMachine32::NVPTXTargetMachine32(
-    const Target &T, StringRef TT, StringRef CPU, StringRef FS,
-    const TargetOptions &Options, Reloc::Model RM, CodeModel::Model CM,
-    CodeGenOpt::Level OL)
+NVPTXTargetMachine32::NVPTXTargetMachine32(const Target &T, const Triple &TT,
+                                           StringRef CPU, StringRef FS,
+                                           const TargetOptions &Options,
+                                           Reloc::Model RM, CodeModel::Model CM,
+                                           CodeGenOpt::Level OL)
     : NVPTXTargetMachine(T, TT, CPU, FS, Options, RM, CM, OL, false) {}
 
 void NVPTXTargetMachine64::anchor() {}
 
-NVPTXTargetMachine64::NVPTXTargetMachine64(
-    const Target &T, StringRef TT, StringRef CPU, StringRef FS,
-    const TargetOptions &Options, Reloc::Model RM, CodeModel::Model CM,
-    CodeGenOpt::Level OL)
+NVPTXTargetMachine64::NVPTXTargetMachine64(const Target &T, const Triple &TT,
+                                           StringRef CPU, StringRef FS,
+                                           const TargetOptions &Options,
+                                           Reloc::Model RM, CodeModel::Model CM,
+                                           CodeGenOpt::Level OL)
     : NVPTXTargetMachine(T, TT, CPU, FS, Options, RM, CM, OL, true) {}
 
 namespace {
@@ -135,6 +141,10 @@ class NVPTXPassConfig : public TargetPassConfig {
   FunctionPass *createTargetRegisterAllocator(bool) override;
   void addFastRegAlloc(FunctionPass *RegAllocPass) override;
   void addOptimizedRegAlloc(FunctionPass *RegAllocPass) override;
+
+private:
+  // if the opt level is aggressive, add GVN; otherwise, add EarlyCSE.
+  void addEarlyCSEOrGVNPass();
 };
 } // end anonymous namespace
 
@@ -144,8 +154,16 @@ TargetPassConfig *NVPTXTargetMachine::createPassConfig(PassManagerBase &PM) {
 }
 
 TargetIRAnalysis NVPTXTargetMachine::getTargetIRAnalysis() {
-  return TargetIRAnalysis(
-      [this](Function &) { return TargetTransformInfo(NVPTXTTIImpl(this)); });
+  return TargetIRAnalysis([this](Function &F) {
+    return TargetTransformInfo(NVPTXTTIImpl(this, F));
+  });
+}
+
+void NVPTXPassConfig::addEarlyCSEOrGVNPass() {
+  if (getOptLevel() == CodeGenOpt::Aggressive)
+    addPass(createGVNPass());
+  else
+    addPass(createEarlyCSEPass());
 }
 
 void NVPTXPassConfig::addIRPasses() {
@@ -156,33 +174,55 @@ void NVPTXPassConfig::addIRPasses() {
   // NVPTXPrologEpilog pass (see NVPTXPrologEpilogPass.cpp).
   disablePass(&PrologEpilogCodeInserterID);
   disablePass(&MachineCopyPropagationID);
-  disablePass(&BranchFolderPassID);
   disablePass(&TailDuplicateID);
 
   addPass(createNVPTXImageOptimizerPass());
-  TargetPassConfig::addIRPasses();
   addPass(createNVPTXAssignValidGlobalNamesPass());
   addPass(createGenericToNVVMPass());
+
+  // === Propagate special address spaces ===
+  addPass(createNVPTXLowerKernelArgsPass(&getNVPTXTargetMachine()));
+  // NVPTXLowerKernelArgs emits alloca for byval parameters which can often
+  // be eliminated by SROA.
+  addPass(createSROAPass());
+  addPass(createNVPTXLowerAllocaPass());
   addPass(createNVPTXFavorNonGenericAddrSpacesPass());
-  addPass(createStraightLineStrengthReducePass());
-  addPass(createSeparateConstOffsetFromGEPPass());
-  // The SeparateConstOffsetFromGEP pass creates variadic bases that can be used
-  // by multiple GEPs. Run GVN or EarlyCSE to really reuse them. GVN generates
-  // significantly better code than EarlyCSE for some of our benchmarks.
-  if (getOptLevel() == CodeGenOpt::Aggressive)
-    addPass(createGVNPass());
-  else
-    addPass(createEarlyCSEPass());
-  // Both FavorNonGenericAddrSpaces and SeparateConstOffsetFromGEP may leave
-  // some dead code.  We could remove dead code in an ad-hoc manner, but that
+  // FavorNonGenericAddrSpaces shortcuts unnecessary addrspacecasts, and leave
+  // them unused. We could remove dead code in an ad-hoc manner, but that
   // requires manual work and might be error-prone.
+  addPass(createDeadCodeEliminationPass());
+
+  // === Straight-line scalar optimizations ===
+  addPass(createSeparateConstOffsetFromGEPPass());
+  addPass(createSpeculativeExecutionPass());
+  // ReassociateGEPs exposes more opportunites for SLSR. See
+  // the example in reassociate-geps-and-slsr.ll.
+  addPass(createStraightLineStrengthReducePass());
+  // SeparateConstOffsetFromGEP and SLSR creates common expressions which GVN or
+  // EarlyCSE can reuse. GVN generates significantly better code than EarlyCSE
+  // for some of our benchmarks.
+  addEarlyCSEOrGVNPass();
+  // Run NaryReassociate after EarlyCSE/GVN to be more effective.
+  addPass(createNaryReassociatePass());
+  // NaryReassociate on GEPs creates redundant common expressions, so run
+  // EarlyCSE after it.
+  addPass(createEarlyCSEPass());
+
+  // === LSR and other generic IR passes ===
+  TargetPassConfig::addIRPasses();
+  // EarlyCSE is not always strong enough to clean up what LSR produces. For
+  // example, GVN can combine
   //
-  // The FavorNonGenericAddrSpaces pass shortcuts unnecessary addrspacecasts,
-  // and leave them unused.
+  //   %0 = add %a, %b
+  //   %1 = add %b, %a
   //
-  // SeparateConstOffsetFromGEP rebuilds a new index from the old index, and the
-  // old index and some of its intermediate results may become unused.
-  addPass(createDeadCodeEliminationPass());
+  // and
+  //
+  //   %0 = shl nsw %a, 2
+  //   %1 = shl %a, 2
+  //
+  // but EarlyCSE can do neither of them.
+  addEarlyCSEOrGVNPass();
 }
 
 bool NVPTXPassConfig::addInstSelector() {
@@ -200,6 +240,10 @@ bool NVPTXPassConfig::addInstSelector() {
 
 void NVPTXPassConfig::addPostRegAlloc() {
   addPass(createNVPTXPrologEpilogPass(), false);
+  // NVPTXPrologEpilogPass calculates frame object offset and replace frame
+  // index with VRFrame register. NVPTXPeephole need to be run after that and
+  // will replace VRFrame with VRFrameLocal when possible.
+  addPass(createNVPTXPeephole());
 }
 
 FunctionPass *NVPTXPassConfig::createTargetRegisterAllocator(bool) {
diff --git a/lib/Target/NVPTX/NVPTXTargetMachine.h b/lib/Target/NVPTX/NVPTXTargetMachine.h
index 2cd10e87f62..da7f62bf9d9 100644
--- a/lib/Target/NVPTX/NVPTXTargetMachine.h
+++ b/lib/Target/NVPTX/NVPTXTargetMachine.h
@@ -34,9 +34,10 @@ class NVPTXTargetMachine : public LLVMTargetMachine {
   ManagedStringPool ManagedStrPool;
 
 public:
-  NVPTXTargetMachine(const Target &T, StringRef TT, StringRef CPU, StringRef FS,
-                     const TargetOptions &Options, Reloc::Model RM,
-                     CodeModel::Model CM, CodeGenOpt::Level OP, bool is64bit);
+  NVPTXTargetMachine(const Target &T, const Triple &TT, StringRef CPU,
+                     StringRef FS, const TargetOptions &Options,
+                     Reloc::Model RM, CodeModel::Model CM, CodeGenOpt::Level OP,
+                     bool is64bit);
 
   ~NVPTXTargetMachine() override;
   const NVPTXSubtarget *getSubtargetImpl(const Function &) const override {
@@ -67,7 +68,7 @@ class NVPTXTargetMachine : public LLVMTargetMachine {
 class NVPTXTargetMachine32 : public NVPTXTargetMachine {
   virtual void anchor();
 public:
-  NVPTXTargetMachine32(const Target &T, StringRef TT, StringRef CPU,
+  NVPTXTargetMachine32(const Target &T, const Triple &TT, StringRef CPU,
                        StringRef FS, const TargetOptions &Options,
                        Reloc::Model RM, CodeModel::Model CM,
                        CodeGenOpt::Level OL);
@@ -76,7 +77,7 @@ class NVPTXTargetMachine32 : public NVPTXTargetMachine {
 class NVPTXTargetMachine64 : public NVPTXTargetMachine {
   virtual void anchor();
 public:
-  NVPTXTargetMachine64(const Target &T, StringRef TT, StringRef CPU,
+  NVPTXTargetMachine64(const Target &T, const Triple &TT, StringRef CPU,
                        StringRef FS, const TargetOptions &Options,
                        Reloc::Model RM, CodeModel::Model CM,
                        CodeGenOpt::Level OL);
diff --git a/lib/Target/NVPTX/NVPTXTargetObjectFile.h b/lib/Target/NVPTX/NVPTXTargetObjectFile.h
index 5d9ab0d511f..a17d1b9fdae 100644
--- a/lib/Target/NVPTX/NVPTXTargetObjectFile.h
+++ b/lib/Target/NVPTX/NVPTXTargetObjectFile.h
@@ -84,20 +84,20 @@ class NVPTXTargetObjectFile : public TargetLoweringObjectFile {
         new NVPTXSection(MCSection::SV_ELF, SectionKind::getMetadata());
   }
 
-  const MCSection *getSectionForConstant(SectionKind Kind,
-                                         const Constant *C) const override {
+  MCSection *getSectionForConstant(const DataLayout &DL, SectionKind Kind,
+                                   const Constant *C) const override {
     return ReadOnlySection;
   }
 
-  const MCSection *getExplicitSectionGlobal(const GlobalValue *GV,
-                                       SectionKind Kind, Mangler &Mang,
-                                       const TargetMachine &TM) const override {
+  MCSection *getExplicitSectionGlobal(const GlobalValue *GV, SectionKind Kind,
+                                      Mangler &Mang,
+                                      const TargetMachine &TM) const override {
     return DataSection;
   }
 
-  const MCSection *
-  SelectSectionForGlobal(const GlobalValue *GV, SectionKind Kind, Mangler &Mang,
-                         const TargetMachine &TM) const override;
+  MCSection *SelectSectionForGlobal(const GlobalValue *GV, SectionKind Kind,
+                                    Mangler &Mang,
+                                    const TargetMachine &TM) const override;
 };
 
 } // end namespace llvm
diff --git a/lib/Target/NVPTX/NVPTXTargetTransformInfo.cpp b/lib/Target/NVPTX/NVPTXTargetTransformInfo.cpp
index dc81802f4b5..e7250cdba5a 100644
--- a/lib/Target/NVPTX/NVPTXTargetTransformInfo.cpp
+++ b/lib/Target/NVPTX/NVPTXTargetTransformInfo.cpp
@@ -94,7 +94,7 @@ unsigned NVPTXTTIImpl::getArithmeticInstrCost(
     TTI::OperandValueKind Opd2Info, TTI::OperandValueProperties Opd1PropInfo,
     TTI::OperandValueProperties Opd2PropInfo) {
   // Legalize the type.
-  std::pair<unsigned, MVT> LT = TLI->getTypeLegalizationCost(Ty);
+  std::pair<unsigned, MVT> LT = TLI->getTypeLegalizationCost(DL, Ty);
 
   int ISD = TLI->InstructionOpcodeToISD(Opcode);
 
@@ -117,3 +117,15 @@ unsigned NVPTXTTIImpl::getArithmeticInstrCost(
                                          Opd1PropInfo, Opd2PropInfo);
   }
 }
+
+void NVPTXTTIImpl::getUnrollingPreferences(Loop *L,
+                                           TTI::UnrollingPreferences &UP) {
+  BaseT::getUnrollingPreferences(L, UP);
+
+  // Enable partial unrolling and runtime unrolling, but reduce the
+  // threshold.  This partially unrolls small loops which are often
+  // unrolled by the PTX to SASS compiler and unrolling earlier can be
+  // beneficial.
+  UP.Partial = UP.Runtime = true;
+  UP.PartialThreshold = UP.Threshold / 4;
+}
diff --git a/lib/Target/NVPTX/NVPTXTargetTransformInfo.h b/lib/Target/NVPTX/NVPTXTargetTransformInfo.h
index 4280888988f..5bcd1e27a55 100644
--- a/lib/Target/NVPTX/NVPTXTargetTransformInfo.h
+++ b/lib/Target/NVPTX/NVPTXTargetTransformInfo.h
@@ -37,8 +37,9 @@ class NVPTXTTIImpl : public BasicTTIImplBase<NVPTXTTIImpl> {
   const NVPTXTargetLowering *getTLI() const { return TLI; };
 
 public:
-  explicit NVPTXTTIImpl(const NVPTXTargetMachine *TM)
-      : BaseT(TM), ST(TM->getSubtargetImpl()), TLI(ST->getTargetLowering()) {}
+  explicit NVPTXTTIImpl(const NVPTXTargetMachine *TM, const Function &F)
+      : BaseT(TM, F.getParent()->getDataLayout()), ST(TM->getSubtargetImpl()),
+        TLI(ST->getTargetLowering()) {}
 
   // Provide value semantics. MSVC requires that we spell all of these out.
   NVPTXTTIImpl(const NVPTXTTIImpl &Arg)
@@ -46,18 +47,6 @@ class NVPTXTTIImpl : public BasicTTIImplBase<NVPTXTTIImpl> {
   NVPTXTTIImpl(NVPTXTTIImpl &&Arg)
       : BaseT(std::move(static_cast<BaseT &>(Arg))), ST(std::move(Arg.ST)),
         TLI(std::move(Arg.TLI)) {}
-  NVPTXTTIImpl &operator=(const NVPTXTTIImpl &RHS) {
-    BaseT::operator=(static_cast<const BaseT &>(RHS));
-    ST = RHS.ST;
-    TLI = RHS.TLI;
-    return *this;
-  }
-  NVPTXTTIImpl &operator=(NVPTXTTIImpl &&RHS) {
-    BaseT::operator=(std::move(static_cast<BaseT &>(RHS)));
-    ST = std::move(RHS.ST);
-    TLI = std::move(RHS.TLI);
-    return *this;
-  }
 
   bool hasBranchDivergence() { return true; }
 
@@ -69,6 +58,8 @@ class NVPTXTTIImpl : public BasicTTIImplBase<NVPTXTTIImpl> {
       TTI::OperandValueKind Opd2Info = TTI::OK_AnyValue,
       TTI::OperandValueProperties Opd1PropInfo = TTI::OP_None,
       TTI::OperandValueProperties Opd2PropInfo = TTI::OP_None);
+
+  void getUnrollingPreferences(Loop *L, TTI::UnrollingPreferences &UP);
 };
 
 } // end namespace llvm
diff --git a/lib/Target/NVPTX/NVPTXUtilities.cpp b/lib/Target/NVPTX/NVPTXUtilities.cpp
index 1f178af4167..51782c66155 100644
--- a/lib/Target/NVPTX/NVPTXUtilities.cpp
+++ b/lib/Target/NVPTX/NVPTXUtilities.cpp
@@ -336,18 +336,16 @@ bool llvm::getAlign(const CallInst &I, unsigned index, unsigned &align) {
 }
 
 bool llvm::isBarrierIntrinsic(Intrinsic::ID id) {
-  if ((id == Intrinsic::nvvm_barrier0) ||
-      (id == Intrinsic::nvvm_barrier0_popc) ||
-      (id == Intrinsic::nvvm_barrier0_and) ||
-      (id == Intrinsic::nvvm_barrier0_or) ||
-      (id == Intrinsic::cuda_syncthreads))
-    return true;
-  return false;
+  return (id == Intrinsic::nvvm_barrier0) ||
+         (id == Intrinsic::nvvm_barrier0_popc) ||
+         (id == Intrinsic::nvvm_barrier0_and) ||
+         (id == Intrinsic::nvvm_barrier0_or) ||
+         (id == Intrinsic::cuda_syncthreads);
 }
 
 // Interface for checking all memory space transfer related intrinsics
 bool llvm::isMemorySpaceTransferIntrinsic(Intrinsic::ID id) {
-  if (id == Intrinsic::nvvm_ptr_local_to_gen ||
+  return id == Intrinsic::nvvm_ptr_local_to_gen ||
       id == Intrinsic::nvvm_ptr_shared_to_gen ||
       id == Intrinsic::nvvm_ptr_global_to_gen ||
       id == Intrinsic::nvvm_ptr_constant_to_gen ||
@@ -355,11 +353,7 @@ bool llvm::isMemorySpaceTransferIntrinsic(Intrinsic::ID id) {
       id == Intrinsic::nvvm_ptr_gen_to_shared ||
       id == Intrinsic::nvvm_ptr_gen_to_local ||
       id == Intrinsic::nvvm_ptr_gen_to_constant ||
-      id == Intrinsic::nvvm_ptr_gen_to_param) {
-    return true;
-  }
-
-  return false;
+      id == Intrinsic::nvvm_ptr_gen_to_param;
 }
 
 // consider several special intrinsics in striping pointer casts, and
diff --git a/lib/Target/NVPTX/NVPTXVector.td b/lib/Target/NVPTX/NVPTXVector.td
index 85aa34e9aea..a237247e483 100644
--- a/lib/Target/NVPTX/NVPTXVector.td
+++ b/lib/Target/NVPTX/NVPTXVector.td
@@ -735,19 +735,19 @@ def VecShuffle_v2i64 : NVPTXVecInst<(outs V2I64Regs:$dst),
 
 def ShuffleMask0 : SDNodeXForm<vector_shuffle, [{
   ShuffleVectorSDNode *SVOp = cast<ShuffleVectorSDNode>(N);
-  return CurDAG->getTargetConstant(SVOp->getMaskElt(0), MVT::i32);
+  return CurDAG->getTargetConstant(SVOp->getMaskElt(0), SDLoc(N), MVT::i32);
 }]>;
 def ShuffleMask1 : SDNodeXForm<vector_shuffle, [{
   ShuffleVectorSDNode *SVOp = cast<ShuffleVectorSDNode>(N);
-  return CurDAG->getTargetConstant(SVOp->getMaskElt(1), MVT::i32);
+  return CurDAG->getTargetConstant(SVOp->getMaskElt(1), SDLoc(N), MVT::i32);
 }]>;
 def ShuffleMask2 : SDNodeXForm<vector_shuffle, [{
   ShuffleVectorSDNode *SVOp = cast<ShuffleVectorSDNode>(N);
-  return CurDAG->getTargetConstant(SVOp->getMaskElt(2), MVT::i32);
+  return CurDAG->getTargetConstant(SVOp->getMaskElt(2), SDLoc(N), MVT::i32);
 }]>;
 def ShuffleMask3 : SDNodeXForm<vector_shuffle, [{
   ShuffleVectorSDNode *SVOp = cast<ShuffleVectorSDNode>(N);
-  return CurDAG->getTargetConstant(SVOp->getMaskElt(3), MVT::i32);
+  return CurDAG->getTargetConstant(SVOp->getMaskElt(3), SDLoc(N), MVT::i32);
 }]>;
 
 // The spurious call is here to silence a compiler warning about N being
diff --git a/lib/Target/PowerPC/AsmParser/PPCAsmParser.cpp b/lib/Target/PowerPC/AsmParser/PPCAsmParser.cpp
index 90ab7a558f8..1ad3d07831a 100644
--- a/lib/Target/PowerPC/AsmParser/PPCAsmParser.cpp
+++ b/lib/Target/PowerPC/AsmParser/PPCAsmParser.cpp
@@ -24,6 +24,7 @@
 #include "llvm/MC/MCParser/MCParsedAsmOperand.h"
 #include "llvm/MC/MCRegisterInfo.h"
 #include "llvm/MC/MCStreamer.h"
+#include "llvm/MC/MCSymbolELF.h"
 #include "llvm/MC/MCSubtargetInfo.h"
 #include "llvm/MC/MCTargetAsmParser.h"
 #include "llvm/Support/SourceMgr.h"
@@ -132,6 +133,25 @@ static const MCPhysReg VSFRegs[64] = {
   PPC::VF24, PPC::VF25, PPC::VF26, PPC::VF27,
   PPC::VF28, PPC::VF29, PPC::VF30, PPC::VF31
 };
+static const MCPhysReg VSSRegs[64] = {
+  PPC::F0,  PPC::F1,  PPC::F2,  PPC::F3,
+  PPC::F4,  PPC::F5,  PPC::F6,  PPC::F7,
+  PPC::F8,  PPC::F9,  PPC::F10, PPC::F11,
+  PPC::F12, PPC::F13, PPC::F14, PPC::F15,
+  PPC::F16, PPC::F17, PPC::F18, PPC::F19,
+  PPC::F20, PPC::F21, PPC::F22, PPC::F23,
+  PPC::F24, PPC::F25, PPC::F26, PPC::F27,
+  PPC::F28, PPC::F29, PPC::F30, PPC::F31,
+
+  PPC::VF0,  PPC::VF1,  PPC::VF2,  PPC::VF3,
+  PPC::VF4,  PPC::VF5,  PPC::VF6,  PPC::VF7,
+  PPC::VF8,  PPC::VF9,  PPC::VF10, PPC::VF11,
+  PPC::VF12, PPC::VF13, PPC::VF14, PPC::VF15,
+  PPC::VF16, PPC::VF17, PPC::VF18, PPC::VF19,
+  PPC::VF20, PPC::VF21, PPC::VF22, PPC::VF23,
+  PPC::VF24, PPC::VF25, PPC::VF26, PPC::VF27,
+  PPC::VF28, PPC::VF29, PPC::VF30, PPC::VF31
+};
 static unsigned QFRegs[32] = {
   PPC::QF0,  PPC::QF1,  PPC::QF2,  PPC::QF3,
   PPC::QF4,  PPC::QF5,  PPC::QF6,  PPC::QF7,
@@ -273,7 +293,7 @@ class PPCAsmParser : public MCTargetAsmParser {
 public:
   PPCAsmParser(MCSubtargetInfo &STI, MCAsmParser &, const MCInstrInfo &MII,
                const MCTargetOptions &Options)
-      : MCTargetAsmParser(), STI(STI), MII(MII) {
+      : MCTargetAsmParser(Options), STI(STI), MII(MII) {
     // Check for 64-bit vs. 32-bit pointer mode.
     Triple TheTriple(STI.getTargetTriple());
     IsPPC64 = (TheTriple.getArch() == Triple::ppc64 ||
@@ -441,6 +461,8 @@ struct PPCOperand : public MCParsedAsmOperand {
   bool isU8ImmX8() const { return Kind == Immediate &&
                                   isUInt<8>(getImm()) &&
                                   (getImm() & 7) == 0; }
+  
+  bool isU10Imm() const { return Kind == Immediate && isUInt<10>(getImm()); }
   bool isU12Imm() const { return Kind == Immediate && isUInt<12>(getImm()); }
   bool isU16Imm() const {
     switch (Kind) {
@@ -520,22 +542,22 @@ struct PPCOperand : public MCParsedAsmOperand {
 
   void addRegGPRCOperands(MCInst &Inst, unsigned N) const {
     assert(N == 1 && "Invalid number of operands!");
-    Inst.addOperand(MCOperand::CreateReg(RRegs[getReg()]));
+    Inst.addOperand(MCOperand::createReg(RRegs[getReg()]));
   }
 
   void addRegGPRCNoR0Operands(MCInst &Inst, unsigned N) const {
     assert(N == 1 && "Invalid number of operands!");
-    Inst.addOperand(MCOperand::CreateReg(RRegsNoR0[getReg()]));
+    Inst.addOperand(MCOperand::createReg(RRegsNoR0[getReg()]));
   }
 
   void addRegG8RCOperands(MCInst &Inst, unsigned N) const {
     assert(N == 1 && "Invalid number of operands!");
-    Inst.addOperand(MCOperand::CreateReg(XRegs[getReg()]));
+    Inst.addOperand(MCOperand::createReg(XRegs[getReg()]));
   }
 
   void addRegG8RCNoX0Operands(MCInst &Inst, unsigned N) const {
     assert(N == 1 && "Invalid number of operands!");
-    Inst.addOperand(MCOperand::CreateReg(XRegsNoX0[getReg()]));
+    Inst.addOperand(MCOperand::createReg(XRegsNoX0[getReg()]));
   }
 
   void addRegGxRCOperands(MCInst &Inst, unsigned N) const {
@@ -554,78 +576,83 @@ struct PPCOperand : public MCParsedAsmOperand {
 
   void addRegF4RCOperands(MCInst &Inst, unsigned N) const {
     assert(N == 1 && "Invalid number of operands!");
-    Inst.addOperand(MCOperand::CreateReg(FRegs[getReg()]));
+    Inst.addOperand(MCOperand::createReg(FRegs[getReg()]));
   }
 
   void addRegF8RCOperands(MCInst &Inst, unsigned N) const {
     assert(N == 1 && "Invalid number of operands!");
-    Inst.addOperand(MCOperand::CreateReg(FRegs[getReg()]));
+    Inst.addOperand(MCOperand::createReg(FRegs[getReg()]));
   }
 
   void addRegVRRCOperands(MCInst &Inst, unsigned N) const {
     assert(N == 1 && "Invalid number of operands!");
-    Inst.addOperand(MCOperand::CreateReg(VRegs[getReg()]));
+    Inst.addOperand(MCOperand::createReg(VRegs[getReg()]));
   }
 
   void addRegVSRCOperands(MCInst &Inst, unsigned N) const {
     assert(N == 1 && "Invalid number of operands!");
-    Inst.addOperand(MCOperand::CreateReg(VSRegs[getVSReg()]));
+    Inst.addOperand(MCOperand::createReg(VSRegs[getVSReg()]));
   }
 
   void addRegVSFRCOperands(MCInst &Inst, unsigned N) const {
     assert(N == 1 && "Invalid number of operands!");
-    Inst.addOperand(MCOperand::CreateReg(VSFRegs[getVSReg()]));
+    Inst.addOperand(MCOperand::createReg(VSFRegs[getVSReg()]));
+  }
+
+  void addRegVSSRCOperands(MCInst &Inst, unsigned N) const {
+    assert(N == 1 && "Invalid number of operands!");
+    Inst.addOperand(MCOperand::createReg(VSSRegs[getVSReg()]));
   }
 
   void addRegQFRCOperands(MCInst &Inst, unsigned N) const {
     assert(N == 1 && "Invalid number of operands!");
-    Inst.addOperand(MCOperand::CreateReg(QFRegs[getReg()]));
+    Inst.addOperand(MCOperand::createReg(QFRegs[getReg()]));
   }
 
   void addRegQSRCOperands(MCInst &Inst, unsigned N) const {
     assert(N == 1 && "Invalid number of operands!");
-    Inst.addOperand(MCOperand::CreateReg(QFRegs[getReg()]));
+    Inst.addOperand(MCOperand::createReg(QFRegs[getReg()]));
   }
 
   void addRegQBRCOperands(MCInst &Inst, unsigned N) const {
     assert(N == 1 && "Invalid number of operands!");
-    Inst.addOperand(MCOperand::CreateReg(QFRegs[getReg()]));
+    Inst.addOperand(MCOperand::createReg(QFRegs[getReg()]));
   }
 
   void addRegCRBITRCOperands(MCInst &Inst, unsigned N) const {
     assert(N == 1 && "Invalid number of operands!");
-    Inst.addOperand(MCOperand::CreateReg(CRBITRegs[getCRBit()]));
+    Inst.addOperand(MCOperand::createReg(CRBITRegs[getCRBit()]));
   }
 
   void addRegCRRCOperands(MCInst &Inst, unsigned N) const {
     assert(N == 1 && "Invalid number of operands!");
-    Inst.addOperand(MCOperand::CreateReg(CRRegs[getCCReg()]));
+    Inst.addOperand(MCOperand::createReg(CRRegs[getCCReg()]));
   }
 
   void addCRBitMaskOperands(MCInst &Inst, unsigned N) const {
     assert(N == 1 && "Invalid number of operands!");
-    Inst.addOperand(MCOperand::CreateReg(CRRegs[getCRBitMask()]));
+    Inst.addOperand(MCOperand::createReg(CRRegs[getCRBitMask()]));
   }
 
   void addImmOperands(MCInst &Inst, unsigned N) const {
     assert(N == 1 && "Invalid number of operands!");
     if (Kind == Immediate)
-      Inst.addOperand(MCOperand::CreateImm(getImm()));
+      Inst.addOperand(MCOperand::createImm(getImm()));
     else
-      Inst.addOperand(MCOperand::CreateExpr(getExpr()));
+      Inst.addOperand(MCOperand::createExpr(getExpr()));
   }
 
   void addS16ImmOperands(MCInst &Inst, unsigned N) const {
     assert(N == 1 && "Invalid number of operands!");
     switch (Kind) {
       case Immediate:
-        Inst.addOperand(MCOperand::CreateImm(getImm()));
+        Inst.addOperand(MCOperand::createImm(getImm()));
         break;
       case ContextImmediate:
-        Inst.addOperand(MCOperand::CreateImm(getImmS16Context()));
+        Inst.addOperand(MCOperand::createImm(getImmS16Context()));
         break;
       default:
-        Inst.addOperand(MCOperand::CreateExpr(getExpr()));
+        Inst.addOperand(MCOperand::createExpr(getExpr()));
         break;
     }
   }
@@ -634,13 +661,13 @@ struct PPCOperand : public MCParsedAsmOperand {
     assert(N == 1 && "Invalid number of operands!");
     switch (Kind) {
       case Immediate:
-        Inst.addOperand(MCOperand::CreateImm(getImm()));
+        Inst.addOperand(MCOperand::createImm(getImm()));
         break;
       case ContextImmediate:
-        Inst.addOperand(MCOperand::CreateImm(getImmU16Context()));
+        Inst.addOperand(MCOperand::createImm(getImmU16Context()));
         break;
       default:
-        Inst.addOperand(MCOperand::CreateExpr(getExpr()));
+        Inst.addOperand(MCOperand::createExpr(getExpr()));
         break;
     }
   }
@@ -648,14 +675,14 @@ struct PPCOperand : public MCParsedAsmOperand {
   void addBranchTargetOperands(MCInst &Inst, unsigned N) const {
     assert(N == 1 && "Invalid number of operands!");
     if (Kind == Immediate)
-      Inst.addOperand(MCOperand::CreateImm(getImm() / 4));
+      Inst.addOperand(MCOperand::createImm(getImm() / 4));
     else
-      Inst.addOperand(MCOperand::CreateExpr(getExpr()));
+      Inst.addOperand(MCOperand::createExpr(getExpr()));
   }
 
   void addTLSRegOperands(MCInst &Inst, unsigned N) const {
     assert(N == 1 && "Invalid number of operands!");
-    Inst.addOperand(MCOperand::CreateExpr(getTLSReg()));
+    Inst.addOperand(MCOperand::createExpr(getTLSReg()));
   }
 
   StringRef getToken() const {
@@ -746,7 +773,7 @@ struct PPCOperand : public MCParsedAsmOperand {
 
     if (const PPCMCExpr *TE = dyn_cast<PPCMCExpr>(Val)) {
       int64_t Res;
-      if (TE->EvaluateAsConstant(Res))
+      if (TE->evaluateAsConstant(Res))
         return CreateContextImm(Res, S, E, IsPPC64);
     }
 
@@ -766,10 +793,10 @@ void PPCOperand::print(raw_ostream &OS) const {
     OS << getImm();
     break;
   case Expression:
-    getExpr()->print(OS);
+    OS << *getExpr();
     break;
   case TLSRegister:
-    getTLSReg()->print(OS);
+    OS << *getTLSReg();
     break;
   }
 }
@@ -777,30 +804,64 @@ void PPCOperand::print(raw_ostream &OS) const {
 static void
 addNegOperand(MCInst &Inst, MCOperand &Op, MCContext &Ctx) {
   if (Op.isImm()) {
-    Inst.addOperand(MCOperand::CreateImm(-Op.getImm()));
+    Inst.addOperand(MCOperand::createImm(-Op.getImm()));
     return;
   }
   const MCExpr *Expr = Op.getExpr();
   if (const MCUnaryExpr *UnExpr = dyn_cast<MCUnaryExpr>(Expr)) {
     if (UnExpr->getOpcode() == MCUnaryExpr::Minus) {
-      Inst.addOperand(MCOperand::CreateExpr(UnExpr->getSubExpr()));
+      Inst.addOperand(MCOperand::createExpr(UnExpr->getSubExpr()));
       return;
     }
   } else if (const MCBinaryExpr *BinExpr = dyn_cast<MCBinaryExpr>(Expr)) {
     if (BinExpr->getOpcode() == MCBinaryExpr::Sub) {
-      const MCExpr *NE = MCBinaryExpr::CreateSub(BinExpr->getRHS(),
+      const MCExpr *NE = MCBinaryExpr::createSub(BinExpr->getRHS(),
                                                  BinExpr->getLHS(), Ctx);
-      Inst.addOperand(MCOperand::CreateExpr(NE));
+      Inst.addOperand(MCOperand::createExpr(NE));
       return;
     }
   }
-  Inst.addOperand(MCOperand::CreateExpr(MCUnaryExpr::CreateMinus(Expr, Ctx)));
+  Inst.addOperand(MCOperand::createExpr(MCUnaryExpr::createMinus(Expr, Ctx)));
 }
 
 void PPCAsmParser::ProcessInstruction(MCInst &Inst,
                                       const OperandVector &Operands) {
   int Opcode = Inst.getOpcode();
   switch (Opcode) {
+  case PPC::DCBTx:
+  case PPC::DCBTT:
+  case PPC::DCBTSTx:
+  case PPC::DCBTSTT: {
+    MCInst TmpInst;
+    TmpInst.setOpcode((Opcode == PPC::DCBTx || Opcode == PPC::DCBTT) ?
+                      PPC::DCBT : PPC::DCBTST);
+    TmpInst.addOperand(MCOperand::createImm(
+      (Opcode == PPC::DCBTx || Opcode == PPC::DCBTSTx) ? 0 : 16));
+    TmpInst.addOperand(Inst.getOperand(0));
+    TmpInst.addOperand(Inst.getOperand(1));
+    Inst = TmpInst;
+    break;
+  }
+  case PPC::DCBTCT:
+  case PPC::DCBTDS: {
+    MCInst TmpInst;
+    TmpInst.setOpcode(PPC::DCBT);
+    TmpInst.addOperand(Inst.getOperand(2));
+    TmpInst.addOperand(Inst.getOperand(0));
+    TmpInst.addOperand(Inst.getOperand(1));
+    Inst = TmpInst;
+    break;
+  }
+  case PPC::DCBTSTCT:
+  case PPC::DCBTSTDS: {
+    MCInst TmpInst;
+    TmpInst.setOpcode(PPC::DCBTST);
+    TmpInst.addOperand(Inst.getOperand(2));
+    TmpInst.addOperand(Inst.getOperand(0));
+    TmpInst.addOperand(Inst.getOperand(1));
+    Inst = TmpInst;
+    break;
+  }
   case PPC::LAx: {
     MCInst TmpInst;
     TmpInst.setOpcode(PPC::LA);
@@ -854,9 +915,9 @@ void PPCAsmParser::ProcessInstruction(MCInst &Inst,
     TmpInst.setOpcode(Opcode == PPC::EXTLWI? PPC::RLWINM : PPC::RLWINMo);
     TmpInst.addOperand(Inst.getOperand(0));
     TmpInst.addOperand(Inst.getOperand(1));
-    TmpInst.addOperand(MCOperand::CreateImm(B));
-    TmpInst.addOperand(MCOperand::CreateImm(0));
-    TmpInst.addOperand(MCOperand::CreateImm(N - 1));
+    TmpInst.addOperand(MCOperand::createImm(B));
+    TmpInst.addOperand(MCOperand::createImm(0));
+    TmpInst.addOperand(MCOperand::createImm(N - 1));
     Inst = TmpInst;
     break;
   }
@@ -868,9 +929,9 @@ void PPCAsmParser::ProcessInstruction(MCInst &Inst,
     TmpInst.setOpcode(Opcode == PPC::EXTRWI? PPC::RLWINM : PPC::RLWINMo);
     TmpInst.addOperand(Inst.getOperand(0));
     TmpInst.addOperand(Inst.getOperand(1));
-    TmpInst.addOperand(MCOperand::CreateImm(B + N));
-    TmpInst.addOperand(MCOperand::CreateImm(32 - N));
-    TmpInst.addOperand(MCOperand::CreateImm(31));
+    TmpInst.addOperand(MCOperand::createImm(B + N));
+    TmpInst.addOperand(MCOperand::createImm(32 - N));
+    TmpInst.addOperand(MCOperand::createImm(31));
     Inst = TmpInst;
     break;
   }
@@ -883,9 +944,9 @@ void PPCAsmParser::ProcessInstruction(MCInst &Inst,
     TmpInst.addOperand(Inst.getOperand(0));
     TmpInst.addOperand(Inst.getOperand(0));
     TmpInst.addOperand(Inst.getOperand(1));
-    TmpInst.addOperand(MCOperand::CreateImm(32 - B));
-    TmpInst.addOperand(MCOperand::CreateImm(B));
-    TmpInst.addOperand(MCOperand::CreateImm((B + N) - 1));
+    TmpInst.addOperand(MCOperand::createImm(32 - B));
+    TmpInst.addOperand(MCOperand::createImm(B));
+    TmpInst.addOperand(MCOperand::createImm((B + N) - 1));
     Inst = TmpInst;
     break;
   }
@@ -898,9 +959,9 @@ void PPCAsmParser::ProcessInstruction(MCInst &Inst,
     TmpInst.addOperand(Inst.getOperand(0));
     TmpInst.addOperand(Inst.getOperand(0));
     TmpInst.addOperand(Inst.getOperand(1));
-    TmpInst.addOperand(MCOperand::CreateImm(32 - (B + N)));
-    TmpInst.addOperand(MCOperand::CreateImm(B));
-    TmpInst.addOperand(MCOperand::CreateImm((B + N) - 1));
+    TmpInst.addOperand(MCOperand::createImm(32 - (B + N)));
+    TmpInst.addOperand(MCOperand::createImm(B));
+    TmpInst.addOperand(MCOperand::createImm((B + N) - 1));
     Inst = TmpInst;
     break;
   }
@@ -911,9 +972,9 @@ void PPCAsmParser::ProcessInstruction(MCInst &Inst,
     TmpInst.setOpcode(Opcode == PPC::ROTRWI? PPC::RLWINM : PPC::RLWINMo);
     TmpInst.addOperand(Inst.getOperand(0));
     TmpInst.addOperand(Inst.getOperand(1));
-    TmpInst.addOperand(MCOperand::CreateImm(32 - N));
-    TmpInst.addOperand(MCOperand::CreateImm(0));
-    TmpInst.addOperand(MCOperand::CreateImm(31));
+    TmpInst.addOperand(MCOperand::createImm(32 - N));
+    TmpInst.addOperand(MCOperand::createImm(0));
+    TmpInst.addOperand(MCOperand::createImm(31));
     Inst = TmpInst;
     break;
   }
@@ -924,9 +985,9 @@ void PPCAsmParser::ProcessInstruction(MCInst &Inst,
     TmpInst.setOpcode(Opcode == PPC::SLWI? PPC::RLWINM : PPC::RLWINMo);
     TmpInst.addOperand(Inst.getOperand(0));
     TmpInst.addOperand(Inst.getOperand(1));
-    TmpInst.addOperand(MCOperand::CreateImm(N));
-    TmpInst.addOperand(MCOperand::CreateImm(0));
-    TmpInst.addOperand(MCOperand::CreateImm(31 - N));
+    TmpInst.addOperand(MCOperand::createImm(N));
+    TmpInst.addOperand(MCOperand::createImm(0));
+    TmpInst.addOperand(MCOperand::createImm(31 - N));
     Inst = TmpInst;
     break;
   }
@@ -937,9 +998,9 @@ void PPCAsmParser::ProcessInstruction(MCInst &Inst,
     TmpInst.setOpcode(Opcode == PPC::SRWI? PPC::RLWINM : PPC::RLWINMo);
     TmpInst.addOperand(Inst.getOperand(0));
     TmpInst.addOperand(Inst.getOperand(1));
-    TmpInst.addOperand(MCOperand::CreateImm(32 - N));
-    TmpInst.addOperand(MCOperand::CreateImm(N));
-    TmpInst.addOperand(MCOperand::CreateImm(31));
+    TmpInst.addOperand(MCOperand::createImm(32 - N));
+    TmpInst.addOperand(MCOperand::createImm(N));
+    TmpInst.addOperand(MCOperand::createImm(31));
     Inst = TmpInst;
     break;
   }
@@ -950,9 +1011,9 @@ void PPCAsmParser::ProcessInstruction(MCInst &Inst,
     TmpInst.setOpcode(Opcode == PPC::CLRRWI? PPC::RLWINM : PPC::RLWINMo);
     TmpInst.addOperand(Inst.getOperand(0));
     TmpInst.addOperand(Inst.getOperand(1));
-    TmpInst.addOperand(MCOperand::CreateImm(0));
-    TmpInst.addOperand(MCOperand::CreateImm(0));
-    TmpInst.addOperand(MCOperand::CreateImm(31 - N));
+    TmpInst.addOperand(MCOperand::createImm(0));
+    TmpInst.addOperand(MCOperand::createImm(0));
+    TmpInst.addOperand(MCOperand::createImm(31 - N));
     Inst = TmpInst;
     break;
   }
@@ -964,9 +1025,9 @@ void PPCAsmParser::ProcessInstruction(MCInst &Inst,
     TmpInst.setOpcode(Opcode == PPC::CLRLSLWI? PPC::RLWINM : PPC::RLWINMo);
     TmpInst.addOperand(Inst.getOperand(0));
     TmpInst.addOperand(Inst.getOperand(1));
-    TmpInst.addOperand(MCOperand::CreateImm(N));
-    TmpInst.addOperand(MCOperand::CreateImm(B - N));
-    TmpInst.addOperand(MCOperand::CreateImm(31 - N));
+    TmpInst.addOperand(MCOperand::createImm(N));
+    TmpInst.addOperand(MCOperand::createImm(B - N));
+    TmpInst.addOperand(MCOperand::createImm(31 - N));
     Inst = TmpInst;
     break;
   }
@@ -978,8 +1039,8 @@ void PPCAsmParser::ProcessInstruction(MCInst &Inst,
     TmpInst.setOpcode(Opcode == PPC::EXTLDI? PPC::RLDICR : PPC::RLDICRo);
     TmpInst.addOperand(Inst.getOperand(0));
     TmpInst.addOperand(Inst.getOperand(1));
-    TmpInst.addOperand(MCOperand::CreateImm(B));
-    TmpInst.addOperand(MCOperand::CreateImm(N - 1));
+    TmpInst.addOperand(MCOperand::createImm(B));
+    TmpInst.addOperand(MCOperand::createImm(N - 1));
     Inst = TmpInst;
     break;
   }
@@ -991,8 +1052,8 @@ void PPCAsmParser::ProcessInstruction(MCInst &Inst,
     TmpInst.setOpcode(Opcode == PPC::EXTRDI? PPC::RLDICL : PPC::RLDICLo);
     TmpInst.addOperand(Inst.getOperand(0));
     TmpInst.addOperand(Inst.getOperand(1));
-    TmpInst.addOperand(MCOperand::CreateImm(B + N));
-    TmpInst.addOperand(MCOperand::CreateImm(64 - N));
+    TmpInst.addOperand(MCOperand::createImm(B + N));
+    TmpInst.addOperand(MCOperand::createImm(64 - N));
     Inst = TmpInst;
     break;
   }
@@ -1005,8 +1066,8 @@ void PPCAsmParser::ProcessInstruction(MCInst &Inst,
     TmpInst.addOperand(Inst.getOperand(0));
     TmpInst.addOperand(Inst.getOperand(0));
     TmpInst.addOperand(Inst.getOperand(1));
-    TmpInst.addOperand(MCOperand::CreateImm(64 - (B + N)));
-    TmpInst.addOperand(MCOperand::CreateImm(B));
+    TmpInst.addOperand(MCOperand::createImm(64 - (B + N)));
+    TmpInst.addOperand(MCOperand::createImm(B));
     Inst = TmpInst;
     break;
   }
@@ -1017,8 +1078,8 @@ void PPCAsmParser::ProcessInstruction(MCInst &Inst,
     TmpInst.setOpcode(Opcode == PPC::ROTRDI? PPC::RLDICL : PPC::RLDICLo);
     TmpInst.addOperand(Inst.getOperand(0));
     TmpInst.addOperand(Inst.getOperand(1));
-    TmpInst.addOperand(MCOperand::CreateImm(64 - N));
-    TmpInst.addOperand(MCOperand::CreateImm(0));
+    TmpInst.addOperand(MCOperand::createImm(64 - N));
+    TmpInst.addOperand(MCOperand::createImm(0));
     Inst = TmpInst;
     break;
   }
@@ -1029,8 +1090,8 @@ void PPCAsmParser::ProcessInstruction(MCInst &Inst,
     TmpInst.setOpcode(Opcode == PPC::SLDI? PPC::RLDICR : PPC::RLDICRo);
     TmpInst.addOperand(Inst.getOperand(0));
     TmpInst.addOperand(Inst.getOperand(1));
-    TmpInst.addOperand(MCOperand::CreateImm(N));
-    TmpInst.addOperand(MCOperand::CreateImm(63 - N));
+    TmpInst.addOperand(MCOperand::createImm(N));
+    TmpInst.addOperand(MCOperand::createImm(63 - N));
     Inst = TmpInst;
     break;
   }
@@ -1041,8 +1102,8 @@ void PPCAsmParser::ProcessInstruction(MCInst &Inst,
     TmpInst.setOpcode(Opcode == PPC::SRDI? PPC::RLDICL : PPC::RLDICLo);
     TmpInst.addOperand(Inst.getOperand(0));
     TmpInst.addOperand(Inst.getOperand(1));
-    TmpInst.addOperand(MCOperand::CreateImm(64 - N));
-    TmpInst.addOperand(MCOperand::CreateImm(N));
+    TmpInst.addOperand(MCOperand::createImm(64 - N));
+    TmpInst.addOperand(MCOperand::createImm(N));
     Inst = TmpInst;
     break;
   }
@@ -1053,8 +1114,8 @@ void PPCAsmParser::ProcessInstruction(MCInst &Inst,
     TmpInst.setOpcode(Opcode == PPC::CLRRDI? PPC::RLDICR : PPC::RLDICRo);
     TmpInst.addOperand(Inst.getOperand(0));
     TmpInst.addOperand(Inst.getOperand(1));
-    TmpInst.addOperand(MCOperand::CreateImm(0));
-    TmpInst.addOperand(MCOperand::CreateImm(63 - N));
+    TmpInst.addOperand(MCOperand::createImm(0));
+    TmpInst.addOperand(MCOperand::createImm(63 - N));
     Inst = TmpInst;
     break;
   }
@@ -1066,8 +1127,8 @@ void PPCAsmParser::ProcessInstruction(MCInst &Inst,
     TmpInst.setOpcode(Opcode == PPC::CLRLSLDI? PPC::RLDIC : PPC::RLDICo);
     TmpInst.addOperand(Inst.getOperand(0));
     TmpInst.addOperand(Inst.getOperand(1));
-    TmpInst.addOperand(MCOperand::CreateImm(N));
-    TmpInst.addOperand(MCOperand::CreateImm(B - N));
+    TmpInst.addOperand(MCOperand::createImm(N));
+    TmpInst.addOperand(MCOperand::createImm(B - N));
     Inst = TmpInst;
     break;
   }
@@ -1083,8 +1144,8 @@ void PPCAsmParser::ProcessInstruction(MCInst &Inst,
     TmpInst.addOperand(Inst.getOperand(0));
     TmpInst.addOperand(Inst.getOperand(1));
     TmpInst.addOperand(Inst.getOperand(2));
-    TmpInst.addOperand(MCOperand::CreateImm(MB));
-    TmpInst.addOperand(MCOperand::CreateImm(ME));
+    TmpInst.addOperand(MCOperand::createImm(MB));
+    TmpInst.addOperand(MCOperand::createImm(ME));
     Inst = TmpInst;
     break;
   }
@@ -1101,8 +1162,8 @@ void PPCAsmParser::ProcessInstruction(MCInst &Inst,
     TmpInst.addOperand(Inst.getOperand(0)); // The tied operand.
     TmpInst.addOperand(Inst.getOperand(1));
     TmpInst.addOperand(Inst.getOperand(2));
-    TmpInst.addOperand(MCOperand::CreateImm(MB));
-    TmpInst.addOperand(MCOperand::CreateImm(ME));
+    TmpInst.addOperand(MCOperand::createImm(MB));
+    TmpInst.addOperand(MCOperand::createImm(ME));
     Inst = TmpInst;
     break;
   }
@@ -1118,11 +1179,18 @@ void PPCAsmParser::ProcessInstruction(MCInst &Inst,
     TmpInst.addOperand(Inst.getOperand(0));
     TmpInst.addOperand(Inst.getOperand(1));
     TmpInst.addOperand(Inst.getOperand(2));
-    TmpInst.addOperand(MCOperand::CreateImm(MB));
-    TmpInst.addOperand(MCOperand::CreateImm(ME));
+    TmpInst.addOperand(MCOperand::createImm(MB));
+    TmpInst.addOperand(MCOperand::createImm(ME));
     Inst = TmpInst;
     break;
   }
+  case PPC::MFTB: {
+    if (STI.getFeatureBits()[PPC::FeatureMFTB]) {
+      assert(Inst.getNumOperands() == 2 && "Expecting two operands");
+      Inst.setOpcode(PPC::MFSPR);
+    }
+    break;
+  }
   }
 }
 
@@ -1185,10 +1253,18 @@ MatchRegisterName(const AsmToken &Tok, unsigned &RegNo, int64_t &IntVal) {
                !Name.substr(1).getAsInteger(10, IntVal) && IntVal < 32) {
       RegNo = FRegs[IntVal];
       return false;
+    } else if (Name.startswith_lower("vs") &&
+               !Name.substr(2).getAsInteger(10, IntVal) && IntVal < 64) {
+      RegNo = VSRegs[IntVal];
+      return false;
     } else if (Name.startswith_lower("v") &&
                !Name.substr(1).getAsInteger(10, IntVal) && IntVal < 32) {
       RegNo = VRegs[IntVal];
       return false;
+    } else if (Name.startswith_lower("q") &&
+               !Name.substr(1).getAsInteger(10, IntVal) && IntVal < 32) {
+      RegNo = QFRegs[IntVal];
+      return false;
     } else if (Name.startswith_lower("cr") &&
                !Name.substr(2).getAsInteger(10, IntVal) && IntVal < 8) {
       RegNo = CRRegs[IntVal];
@@ -1262,7 +1338,7 @@ ExtractModifierFromExpr(const MCExpr *E,
       return nullptr;
     }
 
-    return MCSymbolRefExpr::Create(&SRE->getSymbol(), Context);
+    return MCSymbolRefExpr::create(&SRE->getSymbol(), Context);
   }
 
   case MCExpr::Unary: {
@@ -1270,7 +1346,7 @@ ExtractModifierFromExpr(const MCExpr *E,
     const MCExpr *Sub = ExtractModifierFromExpr(UE->getSubExpr(), Variant);
     if (!Sub)
       return nullptr;
-    return MCUnaryExpr::Create(UE->getOpcode(), Sub, Context);
+    return MCUnaryExpr::create(UE->getOpcode(), Sub, Context);
   }
 
   case MCExpr::Binary: {
@@ -1294,7 +1370,7 @@ ExtractModifierFromExpr(const MCExpr *E,
     else
       return nullptr;
 
-    return MCBinaryExpr::Create(BE->getOpcode(), LHS, RHS, Context);
+    return MCBinaryExpr::create(BE->getOpcode(), LHS, RHS, Context);
   }
   }
 
@@ -1328,7 +1404,7 @@ FixupVariantKind(const MCExpr *E) {
     default:
       return E;
     }
-    return MCSymbolRefExpr::Create(&SRE->getSymbol(), Variant, Context);
+    return MCSymbolRefExpr::create(&SRE->getSymbol(), Variant, Context);
   }
 
   case MCExpr::Unary: {
@@ -1336,7 +1412,7 @@ FixupVariantKind(const MCExpr *E) {
     const MCExpr *Sub = FixupVariantKind(UE->getSubExpr());
     if (Sub == UE->getSubExpr())
       return E;
-    return MCUnaryExpr::Create(UE->getOpcode(), Sub, Context);
+    return MCUnaryExpr::create(UE->getOpcode(), Sub, Context);
   }
 
   case MCExpr::Binary: {
@@ -1345,7 +1421,7 @@ FixupVariantKind(const MCExpr *E) {
     const MCExpr *RHS = FixupVariantKind(BE->getRHS());
     if (LHS == BE->getLHS() && RHS == BE->getRHS())
       return E;
-    return MCBinaryExpr::Create(BE->getOpcode(), LHS, RHS, Context);
+    return MCBinaryExpr::create(BE->getOpcode(), LHS, RHS, Context);
   }
   }
 
@@ -1370,7 +1446,7 @@ ParseExpression(const MCExpr *&EVal) {
   PPCMCExpr::VariantKind Variant;
   const MCExpr *E = ExtractModifierFromExpr(EVal, Variant);
   if (E)
-    EVal = PPCMCExpr::Create(Variant, E, false, getParser().getContext());
+    EVal = PPCMCExpr::create(Variant, E, false, getParser().getContext());
 
   return false;
 }
@@ -1417,7 +1493,7 @@ ParseDarwinExpression(const MCExpr *&EVal) {
     if (getLexer().isNot(AsmToken::RParen))
       return Error(Parser.getTok().getLoc(), "expected ')'");
     Parser.Lex(); // Eat the ')'
-    EVal = PPCMCExpr::Create(Variant, EVal, false, getParser().getContext());
+    EVal = PPCMCExpr::create(Variant, EVal, false, getParser().getContext());
   }
   return false;
 }
@@ -1606,6 +1682,21 @@ bool PPCAsmParser::ParseInstruction(ParseInstructionInfo &Info, StringRef Name,
       return true;
   }
 
+  // We'll now deal with an unfortunate special case: the syntax for the dcbt
+  // and dcbtst instructions differs for server vs. embedded cores.
+  //  The syntax for dcbt is:
+  //    dcbt ra, rb, th [server]
+  //    dcbt th, ra, rb [embedded]
+  //  where th can be omitted when it is 0. dcbtst is the same. We take the
+  //  server form to be the default, so swap the operands if we're parsing for
+  //  an embedded core (they'll be swapped again upon printing).
+  if (STI.getFeatureBits()[PPC::FeatureBookE] &&
+      Operands.size() == 4 &&
+      (Name == "dcbt" || Name == "dcbtst")) {
+    std::swap(Operands[1], Operands[3]);
+    std::swap(Operands[2], Operands[1]);
+  }
+
   return false;
 }
 
@@ -1780,7 +1871,7 @@ bool PPCAsmParser::ParseDirectiveLocalEntry(SMLoc L) {
     Error(L, "expected identifier in directive");
     return false;
   }
-  MCSymbol *Sym = getContext().GetOrCreateSymbol(Name);
+  MCSymbolELF *Sym = cast<MCSymbolELF>(getContext().getOrCreateSymbol(Name));
 
   if (getLexer().isNot(AsmToken::Comma)) {
     Error(L, "unexpected token in directive");
@@ -1853,19 +1944,19 @@ PPCAsmParser::applyModifierToExpr(const MCExpr *E,
                                   MCContext &Ctx) {
   switch (Variant) {
   case MCSymbolRefExpr::VK_PPC_LO:
-    return PPCMCExpr::Create(PPCMCExpr::VK_PPC_LO, E, false, Ctx);
+    return PPCMCExpr::create(PPCMCExpr::VK_PPC_LO, E, false, Ctx);
   case MCSymbolRefExpr::VK_PPC_HI:
-    return PPCMCExpr::Create(PPCMCExpr::VK_PPC_HI, E, false, Ctx);
+    return PPCMCExpr::create(PPCMCExpr::VK_PPC_HI, E, false, Ctx);
   case MCSymbolRefExpr::VK_PPC_HA:
-    return PPCMCExpr::Create(PPCMCExpr::VK_PPC_HA, E, false, Ctx);
+    return PPCMCExpr::create(PPCMCExpr::VK_PPC_HA, E, false, Ctx);
   case MCSymbolRefExpr::VK_PPC_HIGHER:
-    return PPCMCExpr::Create(PPCMCExpr::VK_PPC_HIGHER, E, false, Ctx);
+    return PPCMCExpr::create(PPCMCExpr::VK_PPC_HIGHER, E, false, Ctx);
   case MCSymbolRefExpr::VK_PPC_HIGHERA:
-    return PPCMCExpr::Create(PPCMCExpr::VK_PPC_HIGHERA, E, false, Ctx);
+    return PPCMCExpr::create(PPCMCExpr::VK_PPC_HIGHERA, E, false, Ctx);
   case MCSymbolRefExpr::VK_PPC_HIGHEST:
-    return PPCMCExpr::Create(PPCMCExpr::VK_PPC_HIGHEST, E, false, Ctx);
+    return PPCMCExpr::create(PPCMCExpr::VK_PPC_HIGHEST, E, false, Ctx);
   case MCSymbolRefExpr::VK_PPC_HIGHESTA:
-    return PPCMCExpr::Create(PPCMCExpr::VK_PPC_HIGHESTA, E, false, Ctx);
+    return PPCMCExpr::create(PPCMCExpr::VK_PPC_HIGHESTA, E, false, Ctx);
   default:
     return nullptr;
   }
diff --git a/lib/Target/PowerPC/CMakeLists.txt b/lib/Target/PowerPC/CMakeLists.txt
index 936ed7f2b1b..c0c83cc258b 100644
--- a/lib/Target/PowerPC/CMakeLists.txt
+++ b/lib/Target/PowerPC/CMakeLists.txt
@@ -32,10 +32,11 @@ add_llvm_target(PowerPCCodeGen
   PPCTargetMachine.cpp
   PPCTargetObjectFile.cpp
   PPCTargetTransformInfo.cpp
-  PPCSelectionDAGInfo.cpp
+  PPCTOCRegDeps.cpp
   PPCTLSDynamicCall.cpp
   PPCVSXCopy.cpp
   PPCVSXFMAMutate.cpp
+  PPCVSXSwapRemoval.cpp
   )
 
 add_subdirectory(AsmParser)
diff --git a/lib/Target/PowerPC/Disassembler/PPCDisassembler.cpp b/lib/Target/PowerPC/Disassembler/PPCDisassembler.cpp
index 5cbf3d9a189..93a503c3758 100644
--- a/lib/Target/PowerPC/Disassembler/PPCDisassembler.cpp
+++ b/lib/Target/PowerPC/Disassembler/PPCDisassembler.cpp
@@ -12,6 +12,7 @@
 #include "llvm/MC/MCFixedLenDisassembler.h"
 #include "llvm/MC/MCInst.h"
 #include "llvm/MC/MCSubtargetInfo.h"
+#include "llvm/Support/Endian.h"
 #include "llvm/Support/TargetRegistry.h"
 
 using namespace llvm;
@@ -22,10 +23,12 @@ typedef MCDisassembler::DecodeStatus DecodeStatus;
 
 namespace {
 class PPCDisassembler : public MCDisassembler {
+  bool IsLittleEndian;
+
 public:
-  PPCDisassembler(const MCSubtargetInfo &STI, MCContext &Ctx)
-    : MCDisassembler(STI, Ctx) {}
-  ~PPCDisassembler() override {}
+  PPCDisassembler(const MCSubtargetInfo &STI, MCContext &Ctx,
+                  bool IsLittleEndian)
+      : MCDisassembler(STI, Ctx), IsLittleEndian(IsLittleEndian) {}
 
   DecodeStatus getInstruction(MCInst &Instr, uint64_t &Size,
                               ArrayRef<uint8_t> Bytes, uint64_t Address,
@@ -37,7 +40,13 @@ class PPCDisassembler : public MCDisassembler {
 static MCDisassembler *createPPCDisassembler(const Target &T,
                                              const MCSubtargetInfo &STI,
                                              MCContext &Ctx) {
-  return new PPCDisassembler(STI, Ctx);
+  return new PPCDisassembler(STI, Ctx, /*IsLittleEndian=*/false);
+}
+
+static MCDisassembler *createPPCLEDisassembler(const Target &T,
+                                               const MCSubtargetInfo &STI,
+                                               MCContext &Ctx) {
+  return new PPCDisassembler(STI, Ctx, /*IsLittleEndian=*/true);
 }
 
 extern "C" void LLVMInitializePowerPCDisassembler() {
@@ -47,7 +56,7 @@ extern "C" void LLVMInitializePowerPCDisassembler() {
   TargetRegistry::RegisterMCDisassembler(ThePPC64Target,
                                          createPPCDisassembler);
   TargetRegistry::RegisterMCDisassembler(ThePPC64LETarget,
-                                         createPPCDisassembler);
+                                         createPPCLEDisassembler);
 }
 
 // FIXME: These can be generated by TableGen from the existing register
@@ -131,6 +140,26 @@ static const unsigned VSFRegs[] = {
   PPC::VF28, PPC::VF29, PPC::VF30, PPC::VF31
 };
 
+static const unsigned VSSRegs[] = {
+  PPC::F0, PPC::F1, PPC::F2, PPC::F3,
+  PPC::F4, PPC::F5, PPC::F6, PPC::F7,
+  PPC::F8, PPC::F9, PPC::F10, PPC::F11,
+  PPC::F12, PPC::F13, PPC::F14, PPC::F15,
+  PPC::F16, PPC::F17, PPC::F18, PPC::F19,
+  PPC::F20, PPC::F21, PPC::F22, PPC::F23,
+  PPC::F24, PPC::F25, PPC::F26, PPC::F27,
+  PPC::F28, PPC::F29, PPC::F30, PPC::F31,
+
+  PPC::VF0, PPC::VF1, PPC::VF2, PPC::VF3,
+  PPC::VF4, PPC::VF5, PPC::VF6, PPC::VF7,
+  PPC::VF8, PPC::VF9, PPC::VF10, PPC::VF11,
+  PPC::VF12, PPC::VF13, PPC::VF14, PPC::VF15,
+  PPC::VF16, PPC::VF17, PPC::VF18, PPC::VF19,
+  PPC::VF20, PPC::VF21, PPC::VF22, PPC::VF23,
+  PPC::VF24, PPC::VF25, PPC::VF26, PPC::VF27,
+  PPC::VF28, PPC::VF29, PPC::VF30, PPC::VF31
+};
+
 static const unsigned GPRegs[] = {
   PPC::R0, PPC::R1, PPC::R2, PPC::R3,
   PPC::R4, PPC::R5, PPC::R6, PPC::R7,
@@ -179,7 +208,7 @@ template <std::size_t N>
 static DecodeStatus decodeRegisterClass(MCInst &Inst, uint64_t RegNo,
                                         const unsigned (&Regs)[N]) {
   assert(RegNo < N && "Invalid register number");
-  Inst.addOperand(MCOperand::CreateReg(Regs[RegNo]));
+  Inst.addOperand(MCOperand::createReg(Regs[RegNo]));
   return MCDisassembler::Success;
 }
 
@@ -231,6 +260,12 @@ static DecodeStatus DecodeVSFRCRegisterClass(MCInst &Inst, uint64_t RegNo,
   return decodeRegisterClass(Inst, RegNo, VSFRegs);
 }
 
+static DecodeStatus DecodeVSSRCRegisterClass(MCInst &Inst, uint64_t RegNo,
+                                            uint64_t Address,
+                                            const void *Decoder) {
+  return decodeRegisterClass(Inst, RegNo, VSSRegs);
+}
+
 static DecodeStatus DecodeGPRCRegisterClass(MCInst &Inst, uint64_t RegNo,
                                             uint64_t Address,
                                             const void *Decoder) {
@@ -265,7 +300,7 @@ template<unsigned N>
 static DecodeStatus decodeUImmOperand(MCInst &Inst, uint64_t Imm,
                                       int64_t Address, const void *Decoder) {
   assert(isUInt<N>(Imm) && "Invalid immediate");
-  Inst.addOperand(MCOperand::CreateImm(Imm));
+  Inst.addOperand(MCOperand::createImm(Imm));
   return MCDisassembler::Success;
 }
 
@@ -273,7 +308,7 @@ template<unsigned N>
 static DecodeStatus decodeSImmOperand(MCInst &Inst, uint64_t Imm,
                                       int64_t Address, const void *Decoder) {
   assert(isUInt<N>(Imm) && "Invalid immediate");
-  Inst.addOperand(MCOperand::CreateImm(SignExtend64<N>(Imm)));
+  Inst.addOperand(MCOperand::createImm(SignExtend64<N>(Imm)));
   return MCDisassembler::Success;
 }
 
@@ -296,19 +331,19 @@ static DecodeStatus decodeMemRIOperands(MCInst &Inst, uint64_t Imm,
   case PPC::LFSU:
   case PPC::LFDU:
     // Add the tied output operand.
-    Inst.addOperand(MCOperand::CreateReg(GP0Regs[Base]));
+    Inst.addOperand(MCOperand::createReg(GP0Regs[Base]));
     break;
   case PPC::STBU:
   case PPC::STHU:
   case PPC::STWU:
   case PPC::STFSU:
   case PPC::STFDU:
-    Inst.insert(Inst.begin(), MCOperand::CreateReg(GP0Regs[Base]));
+    Inst.insert(Inst.begin(), MCOperand::createReg(GP0Regs[Base]));
     break;
   }
 
-  Inst.addOperand(MCOperand::CreateImm(SignExtend64<16>(Disp)));
-  Inst.addOperand(MCOperand::CreateReg(GP0Regs[Base]));
+  Inst.addOperand(MCOperand::createImm(SignExtend64<16>(Disp)));
+  Inst.addOperand(MCOperand::createReg(GP0Regs[Base]));
   return MCDisassembler::Success;
 }
 
@@ -324,12 +359,12 @@ static DecodeStatus decodeMemRIXOperands(MCInst &Inst, uint64_t Imm,
 
   if (Inst.getOpcode() == PPC::LDU)
     // Add the tied output operand.
-    Inst.addOperand(MCOperand::CreateReg(GP0Regs[Base]));
+    Inst.addOperand(MCOperand::createReg(GP0Regs[Base]));
   else if (Inst.getOpcode() == PPC::STDU)
-    Inst.insert(Inst.begin(), MCOperand::CreateReg(GP0Regs[Base]));
+    Inst.insert(Inst.begin(), MCOperand::createReg(GP0Regs[Base]));
 
-  Inst.addOperand(MCOperand::CreateImm(SignExtend64<16>(Disp << 2)));
-  Inst.addOperand(MCOperand::CreateReg(GP0Regs[Base]));
+  Inst.addOperand(MCOperand::createImm(SignExtend64<16>(Disp << 2)));
+  Inst.addOperand(MCOperand::createReg(GP0Regs[Base]));
   return MCDisassembler::Success;
 }
 
@@ -340,7 +375,7 @@ static DecodeStatus decodeCRBitMOperand(MCInst &Inst, uint64_t Imm,
   unsigned Zeros = countTrailingZeros(Imm);
   assert(Zeros < 8 && "Invalid CR bit value");
 
-  Inst.addOperand(MCOperand::CreateReg(CRRegs[7 - Zeros]));
+  Inst.addOperand(MCOperand::createReg(CRRegs[7 - Zeros]));
   return MCDisassembler::Success;
 }
 
@@ -357,11 +392,11 @@ DecodeStatus PPCDisassembler::getInstruction(MCInst &MI, uint64_t &Size,
     return MCDisassembler::Fail;
   }
 
-  // The instruction is big-endian encoded.
-  uint32_t Inst =
-      (Bytes[0] << 24) | (Bytes[1] << 16) | (Bytes[2] << 8) | (Bytes[3] << 0);
+  // Read the instruction in the proper endianness.
+  uint32_t Inst = IsLittleEndian ? support::endian::read32le(Bytes.data())
+                                 : support::endian::read32be(Bytes.data());
 
-  if ((STI.getFeatureBits() & PPC::FeatureQPX) != 0) {
+  if (STI.getFeatureBits()[PPC::FeatureQPX]) {
     DecodeStatus result =
       decodeInstruction(DecoderTableQPX32, MI, Inst, Address, this, STI);
     if (result != MCDisassembler::Fail)
diff --git a/lib/Target/PowerPC/InstPrinter/PPCInstPrinter.cpp b/lib/Target/PowerPC/InstPrinter/PPCInstPrinter.cpp
index 15765440e87..5e1d2278905 100644
--- a/lib/Target/PowerPC/InstPrinter/PPCInstPrinter.cpp
+++ b/lib/Target/PowerPC/InstPrinter/PPCInstPrinter.cpp
@@ -17,6 +17,8 @@
 #include "llvm/MC/MCExpr.h"
 #include "llvm/MC/MCInst.h"
 #include "llvm/MC/MCInstrInfo.h"
+#include "llvm/MC/MCRegisterInfo.h"
+#include "llvm/MC/MCSubtargetInfo.h"
 #include "llvm/MC/MCSymbol.h"
 #include "llvm/Support/CommandLine.h"
 #include "llvm/Support/raw_ostream.h"
@@ -31,6 +33,7 @@ static cl::opt<bool>
 FullRegNames("ppc-asm-full-reg-names", cl::Hidden, cl::init(false),
              cl::desc("Use full register names when printing assembly"));
 
+#define PRINT_ALIAS_INSTR
 #include "PPCGenAsmWriter.inc"
 
 void PPCInstPrinter::printRegName(raw_ostream &OS, unsigned RegNo) const {
@@ -100,6 +103,38 @@ void PPCInstPrinter::printInst(const MCInst *MI, raw_ostream &O,
       return;
     }
   }
+
+  // dcbt[st] is printed manually here because:
+  //  1. The assembly syntax is different between embedded and server targets
+  //  2. We must print the short mnemonics for TH == 0 because the
+  //     embedded/server syntax default will not be stable across assemblers
+  //  The syntax for dcbt is:
+  //    dcbt ra, rb, th [server]
+  //    dcbt th, ra, rb [embedded]
+  //  where th can be omitted when it is 0. dcbtst is the same.
+  if (MI->getOpcode() == PPC::DCBT || MI->getOpcode() == PPC::DCBTST) {
+    unsigned char TH = MI->getOperand(0).getImm();
+    O << "\tdcbt";
+    if (MI->getOpcode() == PPC::DCBTST)
+      O << "st";
+    if (TH == 16)
+      O << "t";
+    O << " ";
+
+    bool IsBookE = STI.getFeatureBits()[PPC::FeatureBookE];
+    if (IsBookE && TH != 0 && TH != 16)
+      O << (unsigned int) TH << ", ";
+
+    printOperand(MI, 1, O);
+    O << ", ";
+    printOperand(MI, 2, O);
+
+    if (!IsBookE && TH != 0 && TH != 16)
+      O << ", " << (unsigned int) TH;
+
+    printAnnotation(O, Annot);
+    return;
+  }
   
   // For fast-isel, a COPY_TO_REGCLASS may survive this long.  This is
   // used when converting a 32-bit float to a 64-bit float as part of
@@ -111,8 +146,9 @@ void PPCInstPrinter::printInst(const MCInst *MI, raw_ostream &O,
   // precision).  FIXME: Is there a better solution?
   if (MI->getOpcode() == TargetOpcode::COPY_TO_REGCLASS)
     return;
-  
-  printInstruction(MI, O);
+
+  if (!printAliasInstr(MI, O))
+    printInstruction(MI, O);
   printAnnotation(O, Annot);
 }
 
@@ -263,6 +299,13 @@ void PPCInstPrinter::printU6ImmOperand(const MCInst *MI, unsigned OpNo,
   O << (unsigned int)Value;
 }
 
+void PPCInstPrinter::printU10ImmOperand(const MCInst *MI, unsigned OpNo,
+                                        raw_ostream &O) {
+  unsigned short Value = MI->getOperand(OpNo).getImm();
+  assert(Value <= 1023 && "Invalid u10imm argument!");
+  O << (unsigned short)Value;
+}
+
 void PPCInstPrinter::printU12ImmOperand(const MCInst *MI, unsigned OpNo,
                                         raw_ostream &O) {
   unsigned short Value = MI->getOperand(OpNo).getImm();
@@ -402,6 +445,6 @@ void PPCInstPrinter::printOperand(const MCInst *MI, unsigned OpNo,
   }
   
   assert(Op.isExpr() && "unknown operand kind in printOperand");
-  O << *Op.getExpr();
+  Op.getExpr()->print(O, &MAI);
 }
 
diff --git a/lib/Target/PowerPC/InstPrinter/PPCInstPrinter.h b/lib/Target/PowerPC/InstPrinter/PPCInstPrinter.h
index eca37eb3679..8e187834430 100644
--- a/lib/Target/PowerPC/InstPrinter/PPCInstPrinter.h
+++ b/lib/Target/PowerPC/InstPrinter/PPCInstPrinter.h
@@ -39,6 +39,10 @@ class PPCInstPrinter : public MCInstPrinter {
   void printInstruction(const MCInst *MI, raw_ostream &O);
   static const char *getRegisterName(unsigned RegNo);
   
+  bool printAliasInstr(const MCInst *MI, raw_ostream &OS);
+  void printCustomAliasOperand(const MCInst *MI, unsigned OpIdx,
+                               unsigned PrintMethodIdx,
+                               raw_ostream &OS);
 
   void printOperand(const MCInst *MI, unsigned OpNo, raw_ostream &O);
   void printPredicateOperand(const MCInst *MI, unsigned OpNo,
@@ -51,6 +55,7 @@ class PPCInstPrinter : public MCInstPrinter {
   void printS5ImmOperand(const MCInst *MI, unsigned OpNo, raw_ostream &O);
   void printU5ImmOperand(const MCInst *MI, unsigned OpNo, raw_ostream &O);
   void printU6ImmOperand(const MCInst *MI, unsigned OpNo, raw_ostream &O);
+  void printU10ImmOperand(const MCInst *MI, unsigned OpNo, raw_ostream &O);
   void printU12ImmOperand(const MCInst *MI, unsigned OpNo, raw_ostream &O);
   void printS16ImmOperand(const MCInst *MI, unsigned OpNo, raw_ostream &O);
   void printU16ImmOperand(const MCInst *MI, unsigned OpNo, raw_ostream &O);
diff --git a/lib/Target/PowerPC/MCTargetDesc/PPCAsmBackend.cpp b/lib/Target/PowerPC/MCTargetDesc/PPCAsmBackend.cpp
index 420c5c809a5..b6dd595ffb0 100644
--- a/lib/Target/PowerPC/MCTargetDesc/PPCAsmBackend.cpp
+++ b/lib/Target/PowerPC/MCTargetDesc/PPCAsmBackend.cpp
@@ -11,12 +11,12 @@
 #include "MCTargetDesc/PPCFixupKinds.h"
 #include "llvm/MC/MCAsmBackend.h"
 #include "llvm/MC/MCAssembler.h"
-#include "llvm/MC/MCELF.h"
 #include "llvm/MC/MCELFObjectWriter.h"
 #include "llvm/MC/MCFixupKindInfo.h"
 #include "llvm/MC/MCMachObjectWriter.h"
 #include "llvm/MC/MCObjectWriter.h"
 #include "llvm/MC/MCSectionMachO.h"
+#include "llvm/MC/MCSymbolELF.h"
 #include "llvm/MC/MCValue.h"
 #include "llvm/Support/ELF.h"
 #include "llvm/Support/ErrorHandling.h"
@@ -142,13 +142,14 @@ class PPCAsmBackend : public MCAsmBackend {
       // to resolve the fixup directly.  Emit a relocation and leave
       // resolution of the final target address to the linker.
       if (const MCSymbolRefExpr *A = Target.getSymA()) {
-        const MCSymbolData &Data = Asm.getSymbolData(A->getSymbol());
-        // The "other" values are stored in the last 6 bits of the second byte.
-        // The traditional defines for STO values assume the full byte and thus
-        // the shift to pack it.
-        unsigned Other = MCELF::getOther(Data) << 2;
-        if ((Other & ELF::STO_PPC64_LOCAL_MASK) != 0)
-          IsResolved = false;
+        if (const auto *S = dyn_cast<MCSymbolELF>(&A->getSymbol())) {
+          // The "other" values are stored in the last 6 bits of the second
+          // byte. The traditional defines for STO values assume the full byte
+          // and thus the shift to pack it.
+          unsigned Other = S->getOther() << 2;
+          if ((Other & ELF::STO_PPC64_LOCAL_MASK) != 0)
+            IsResolved = false;
+        }
       }
       break;
     }
@@ -176,14 +177,9 @@ class PPCAsmBackend : public MCAsmBackend {
   bool writeNopData(uint64_t Count, MCObjectWriter *OW) const override {
     uint64_t NumNops = Count / 4;
     for (uint64_t i = 0; i != NumNops; ++i)
-      OW->Write32(0x60000000);
+      OW->write32(0x60000000);
 
-    switch (Count % 4) {
-    default: break; // No leftover bytes to write
-    case 1: OW->Write8(0); break;
-    case 2: OW->Write16(0); break;
-    case 3: OW->Write16(0); OW->Write8(0); break;
-    }
+    OW->WriteZeros(Count % 4);
 
     return true;
   }
@@ -234,11 +230,11 @@ namespace {
 
 MCAsmBackend *llvm::createPPCAsmBackend(const Target &T,
                                         const MCRegisterInfo &MRI,
-                                        StringRef TT, StringRef CPU) {
-  if (Triple(TT).isOSDarwin())
+                                        const Triple &TT, StringRef CPU) {
+  if (TT.isOSDarwin())
     return new DarwinPPCAsmBackend(T);
 
-  uint8_t OSABI = MCELFObjectTargetWriter::getOSABI(Triple(TT).getOS());
-  bool IsLittleEndian = Triple(TT).getArch() == Triple::ppc64le;
+  uint8_t OSABI = MCELFObjectTargetWriter::getOSABI(TT.getOS());
+  bool IsLittleEndian = TT.getArch() == Triple::ppc64le;
   return new ELFPPCAsmBackend(T, IsLittleEndian, OSABI);
 }
diff --git a/lib/Target/PowerPC/MCTargetDesc/PPCELFObjectWriter.cpp b/lib/Target/PowerPC/MCTargetDesc/PPCELFObjectWriter.cpp
index 3e3489fc46a..992be5b966c 100644
--- a/lib/Target/PowerPC/MCTargetDesc/PPCELFObjectWriter.cpp
+++ b/lib/Target/PowerPC/MCTargetDesc/PPCELFObjectWriter.cpp
@@ -11,9 +11,9 @@
 #include "MCTargetDesc/PPCFixupKinds.h"
 #include "MCTargetDesc/PPCMCExpr.h"
 #include "llvm/ADT/STLExtras.h"
-#include "llvm/MC/MCELF.h"
 #include "llvm/MC/MCELFObjectWriter.h"
 #include "llvm/MC/MCExpr.h"
+#include "llvm/MC/MCSymbolELF.h"
 #include "llvm/MC/MCValue.h"
 #include "llvm/Support/ErrorHandling.h"
 
@@ -28,7 +28,7 @@ namespace {
     unsigned GetRelocType(const MCValue &Target, const MCFixup &Fixup,
                           bool IsPCRel) const override;
 
-    bool needsRelocateWithSymbol(const MCSymbolData &SD,
+    bool needsRelocateWithSymbol(const MCSymbol &Sym,
                                  unsigned Type) const override;
   };
 }
@@ -395,7 +395,7 @@ unsigned PPCELFObjectWriter::GetRelocType(const MCValue &Target,
   return Type;
 }
 
-bool PPCELFObjectWriter::needsRelocateWithSymbol(const MCSymbolData &SD,
+bool PPCELFObjectWriter::needsRelocateWithSymbol(const MCSymbol &Sym,
                                                  unsigned Type) const {
   switch (Type) {
     default:
@@ -407,7 +407,7 @@ bool PPCELFObjectWriter::needsRelocateWithSymbol(const MCSymbolData &SD,
       // The "other" values are stored in the last 6 bits of the second byte.
       // The traditional defines for STO values assume the full byte and thus
       // the shift to pack it.
-      unsigned Other = MCELF::getOther(SD) << 2;
+      unsigned Other = cast<MCSymbolELF>(Sym).getOther() << 2;
       return (Other & ELF::STO_PPC64_LOCAL_MASK) != 0;
   }
 }
diff --git a/lib/Target/PowerPC/MCTargetDesc/PPCMCCodeEmitter.cpp b/lib/Target/PowerPC/MCTargetDesc/PPCMCCodeEmitter.cpp
index 725b47ba791..b7291561c75 100644
--- a/lib/Target/PowerPC/MCTargetDesc/PPCMCCodeEmitter.cpp
+++ b/lib/Target/PowerPC/MCTargetDesc/PPCMCCodeEmitter.cpp
@@ -20,7 +20,9 @@
 #include "llvm/MC/MCExpr.h"
 #include "llvm/MC/MCInst.h"
 #include "llvm/MC/MCInstrInfo.h"
+#include "llvm/MC/MCRegisterInfo.h"
 #include "llvm/MC/MCSubtargetInfo.h"
+#include "llvm/Support/EndianStream.h"
 #include "llvm/Support/ErrorHandling.h"
 #include "llvm/Support/raw_ostream.h"
 #include "llvm/Target/TargetOpcodes.h"
@@ -97,7 +99,7 @@ class PPCMCCodeEmitter : public MCCodeEmitter {
   uint64_t getBinaryCodeForInstr(const MCInst &MI,
                                  SmallVectorImpl<MCFixup> &Fixups,
                                  const MCSubtargetInfo &STI) const;
-  void EncodeInstruction(const MCInst &MI, raw_ostream &OS,
+  void encodeInstruction(const MCInst &MI, raw_ostream &OS,
                          SmallVectorImpl<MCFixup> &Fixups,
                          const MCSubtargetInfo &STI) const override {
     // For fast-isel, a float COPY_TO_REGCLASS can survive this long.
@@ -115,38 +117,19 @@ class PPCMCCodeEmitter : public MCCodeEmitter {
     switch (Size) {
     case 4:
       if (IsLittleEndian) {
-        OS << (char)(Bits);
-        OS << (char)(Bits >> 8);
-        OS << (char)(Bits >> 16);
-        OS << (char)(Bits >> 24);
+        support::endian::Writer<support::little>(OS).write<uint32_t>(Bits);
       } else {
-        OS << (char)(Bits >> 24);
-        OS << (char)(Bits >> 16);
-        OS << (char)(Bits >> 8);
-        OS << (char)(Bits);
+        support::endian::Writer<support::big>(OS).write<uint32_t>(Bits);
       }
       break;
     case 8:
       // If we emit a pair of instructions, the first one is
       // always in the top 32 bits, even on little-endian.
       if (IsLittleEndian) {
-        OS << (char)(Bits >> 32);
-        OS << (char)(Bits >> 40);
-        OS << (char)(Bits >> 48);
-        OS << (char)(Bits >> 56);
-        OS << (char)(Bits);
-        OS << (char)(Bits >> 8);
-        OS << (char)(Bits >> 16);
-        OS << (char)(Bits >> 24);
+        uint64_t Swapped = (Bits << 32) | (Bits >> 32);
+        support::endian::Writer<support::little>(OS).write<uint64_t>(Swapped);
       } else {
-        OS << (char)(Bits >> 56);
-        OS << (char)(Bits >> 48);
-        OS << (char)(Bits >> 40);
-        OS << (char)(Bits >> 32);
-        OS << (char)(Bits >> 24);
-        OS << (char)(Bits >> 16);
-        OS << (char)(Bits >> 8);
-        OS << (char)(Bits);
+        support::endian::Writer<support::big>(OS).write<uint64_t>(Bits);
       }
       break;
     default:
@@ -174,7 +157,7 @@ getDirectBrEncoding(const MCInst &MI, unsigned OpNo,
   if (MO.isReg() || MO.isImm()) return getMachineOpValue(MI, MO, Fixups, STI);
   
   // Add a fixup for the branch target.
-  Fixups.push_back(MCFixup::Create(0, MO.getExpr(),
+  Fixups.push_back(MCFixup::create(0, MO.getExpr(),
                                    (MCFixupKind)PPC::fixup_ppc_br24));
   return 0;
 }
@@ -186,7 +169,7 @@ unsigned PPCMCCodeEmitter::getCondBrEncoding(const MCInst &MI, unsigned OpNo,
   if (MO.isReg() || MO.isImm()) return getMachineOpValue(MI, MO, Fixups, STI);
 
   // Add a fixup for the branch target.
-  Fixups.push_back(MCFixup::Create(0, MO.getExpr(),
+  Fixups.push_back(MCFixup::create(0, MO.getExpr(),
                                    (MCFixupKind)PPC::fixup_ppc_brcond14));
   return 0;
 }
@@ -199,7 +182,7 @@ getAbsDirectBrEncoding(const MCInst &MI, unsigned OpNo,
   if (MO.isReg() || MO.isImm()) return getMachineOpValue(MI, MO, Fixups, STI);
 
   // Add a fixup for the branch target.
-  Fixups.push_back(MCFixup::Create(0, MO.getExpr(),
+  Fixups.push_back(MCFixup::create(0, MO.getExpr(),
                                    (MCFixupKind)PPC::fixup_ppc_br24abs));
   return 0;
 }
@@ -212,7 +195,7 @@ getAbsCondBrEncoding(const MCInst &MI, unsigned OpNo,
   if (MO.isReg() || MO.isImm()) return getMachineOpValue(MI, MO, Fixups, STI);
 
   // Add a fixup for the branch target.
-  Fixups.push_back(MCFixup::Create(0, MO.getExpr(),
+  Fixups.push_back(MCFixup::create(0, MO.getExpr(),
                                    (MCFixupKind)PPC::fixup_ppc_brcond14abs));
   return 0;
 }
@@ -224,7 +207,7 @@ unsigned PPCMCCodeEmitter::getImm16Encoding(const MCInst &MI, unsigned OpNo,
   if (MO.isReg() || MO.isImm()) return getMachineOpValue(MI, MO, Fixups, STI);
   
   // Add a fixup for the immediate field.
-  Fixups.push_back(MCFixup::Create(IsLittleEndian? 0 : 2, MO.getExpr(),
+  Fixups.push_back(MCFixup::create(IsLittleEndian? 0 : 2, MO.getExpr(),
                                    (MCFixupKind)PPC::fixup_ppc_half16));
   return 0;
 }
@@ -242,7 +225,7 @@ unsigned PPCMCCodeEmitter::getMemRIEncoding(const MCInst &MI, unsigned OpNo,
     return (getMachineOpValue(MI, MO, Fixups, STI) & 0xFFFF) | RegBits;
   
   // Add a fixup for the displacement field.
-  Fixups.push_back(MCFixup::Create(IsLittleEndian? 0 : 2, MO.getExpr(),
+  Fixups.push_back(MCFixup::create(IsLittleEndian? 0 : 2, MO.getExpr(),
                                    (MCFixupKind)PPC::fixup_ppc_half16));
   return RegBits;
 }
@@ -261,7 +244,7 @@ unsigned PPCMCCodeEmitter::getMemRIXEncoding(const MCInst &MI, unsigned OpNo,
     return ((getMachineOpValue(MI, MO, Fixups, STI) >> 2) & 0x3FFF) | RegBits;
   
   // Add a fixup for the displacement field.
-  Fixups.push_back(MCFixup::Create(IsLittleEndian? 0 : 2, MO.getExpr(),
+  Fixups.push_back(MCFixup::create(IsLittleEndian? 0 : 2, MO.getExpr(),
                                    (MCFixupKind)PPC::fixup_ppc_half16ds));
   return RegBits;
 }
@@ -324,9 +307,9 @@ unsigned PPCMCCodeEmitter::getTLSRegEncoding(const MCInst &MI, unsigned OpNo,
   // Add a fixup for the TLS register, which simply provides a relocation
   // hint to the linker that this statement is part of a relocation sequence.
   // Return the thread-pointer register's encoding.
-  Fixups.push_back(MCFixup::Create(0, MO.getExpr(),
+  Fixups.push_back(MCFixup::create(0, MO.getExpr(),
                                    (MCFixupKind)PPC::fixup_ppc_nofixup));
-  Triple TT(STI.getTargetTriple());
+  const Triple &TT = STI.getTargetTriple();
   bool isPPC64 = TT.getArch() == Triple::ppc64 || TT.getArch() == Triple::ppc64le;
   return CTX.getRegisterInfo()->getEncodingValue(isPPC64 ? PPC::X13 : PPC::R2);
 }
@@ -338,7 +321,7 @@ unsigned PPCMCCodeEmitter::getTLSCallEncoding(const MCInst &MI, unsigned OpNo,
   // (__tls_get_addr), which we create via getDirectBrEncoding as usual,
   // and one for the TLSGD or TLSLD symbol, which is emitted here.
   const MCOperand &MO = MI.getOperand(OpNo+1);
-  Fixups.push_back(MCFixup::Create(0, MO.getExpr(),
+  Fixups.push_back(MCFixup::create(0, MO.getExpr(),
                                    (MCFixupKind)PPC::fixup_ppc_nofixup));
   return getDirectBrEncoding(MI, OpNo, Fixups, STI);
 }
diff --git a/lib/Target/PowerPC/MCTargetDesc/PPCMCExpr.cpp b/lib/Target/PowerPC/MCTargetDesc/PPCMCExpr.cpp
index 7204befe15e..6b97d4c1456 100644
--- a/lib/Target/PowerPC/MCTargetDesc/PPCMCExpr.cpp
+++ b/lib/Target/PowerPC/MCTargetDesc/PPCMCExpr.cpp
@@ -19,12 +19,12 @@ using namespace llvm;
 #define DEBUG_TYPE "ppcmcexpr"
 
 const PPCMCExpr*
-PPCMCExpr::Create(VariantKind Kind, const MCExpr *Expr,
+PPCMCExpr::create(VariantKind Kind, const MCExpr *Expr,
                   bool isDarwin, MCContext &Ctx) {
   return new (Ctx) PPCMCExpr(Kind, Expr, isDarwin);
 }
 
-void PPCMCExpr::PrintImpl(raw_ostream &OS) const {
+void PPCMCExpr::printImpl(raw_ostream &OS, const MCAsmInfo *MAI) const {
   if (isDarwinSyntax()) {
     switch (Kind) {
     default: llvm_unreachable("Invalid kind!");
@@ -34,10 +34,10 @@ void PPCMCExpr::PrintImpl(raw_ostream &OS) const {
     }
 
     OS << '(';
-    getSubExpr()->print(OS);
+    getSubExpr()->print(OS, MAI);
     OS << ')';
   } else {
-    getSubExpr()->print(OS);
+    getSubExpr()->print(OS, MAI);
 
     switch (Kind) {
     default: llvm_unreachable("Invalid kind!");
@@ -53,21 +53,21 @@ void PPCMCExpr::PrintImpl(raw_ostream &OS) const {
 }
 
 bool
-PPCMCExpr::EvaluateAsConstant(int64_t &Res) const {
+PPCMCExpr::evaluateAsConstant(int64_t &Res) const {
   MCValue Value;
 
-  if (!getSubExpr()->EvaluateAsRelocatable(Value, nullptr, nullptr))
+  if (!getSubExpr()->evaluateAsRelocatable(Value, nullptr, nullptr))
     return false;
 
   if (!Value.isAbsolute())
     return false;
 
-  Res = EvaluateAsInt64(Value.getConstant());
+  Res = evaluateAsInt64(Value.getConstant());
   return true;
 }
 
 int64_t
-PPCMCExpr::EvaluateAsInt64(int64_t Value) const {
+PPCMCExpr::evaluateAsInt64(int64_t Value) const {
   switch (Kind) {
     case VK_PPC_LO:
       return Value & 0xffff;
@@ -90,16 +90,16 @@ PPCMCExpr::EvaluateAsInt64(int64_t Value) const {
 }
 
 bool
-PPCMCExpr::EvaluateAsRelocatableImpl(MCValue &Res,
+PPCMCExpr::evaluateAsRelocatableImpl(MCValue &Res,
                                      const MCAsmLayout *Layout,
                                      const MCFixup *Fixup) const {
   MCValue Value;
 
-  if (!getSubExpr()->EvaluateAsRelocatable(Value, Layout, Fixup))
+  if (!getSubExpr()->evaluateAsRelocatable(Value, Layout, Fixup))
     return false;
 
   if (Value.isAbsolute()) {
-    int64_t Result = EvaluateAsInt64(Value.getConstant());
+    int64_t Result = evaluateAsInt64(Value.getConstant());
     if ((Fixup == nullptr || (unsigned)Fixup->getKind() != PPC::fixup_ppc_half16) &&
         (Result >= 0x8000))
       return false;
@@ -138,7 +138,7 @@ PPCMCExpr::EvaluateAsRelocatableImpl(MCValue &Res,
         Modifier = MCSymbolRefExpr::VK_PPC_HIGHESTA;
         break;
     }
-    Sym = MCSymbolRefExpr::Create(&Sym->getSymbol(), Modifier, Context);
+    Sym = MCSymbolRefExpr::create(&Sym->getSymbol(), Modifier, Context);
     Res = MCValue::get(Sym, Value.getSymB(), Value.getConstant());
   }
 
diff --git a/lib/Target/PowerPC/MCTargetDesc/PPCMCExpr.h b/lib/Target/PowerPC/MCTargetDesc/PPCMCExpr.h
index 1c840d998e0..a641780516b 100644
--- a/lib/Target/PowerPC/MCTargetDesc/PPCMCExpr.h
+++ b/lib/Target/PowerPC/MCTargetDesc/PPCMCExpr.h
@@ -34,7 +34,7 @@ class PPCMCExpr : public MCTargetExpr {
   const MCExpr *Expr;
   bool IsDarwin;
 
-  int64_t EvaluateAsInt64(int64_t Value) const;
+  int64_t evaluateAsInt64(int64_t Value) const;
 
   explicit PPCMCExpr(VariantKind Kind, const MCExpr *Expr, bool IsDarwin)
       : Kind(Kind), Expr(Expr), IsDarwin(IsDarwin) {}
@@ -43,22 +43,22 @@ class PPCMCExpr : public MCTargetExpr {
   /// @name Construction
   /// @{
 
-  static const PPCMCExpr *Create(VariantKind Kind, const MCExpr *Expr,
+  static const PPCMCExpr *create(VariantKind Kind, const MCExpr *Expr,
                                  bool isDarwin, MCContext &Ctx);
 
-  static const PPCMCExpr *CreateLo(const MCExpr *Expr,
+  static const PPCMCExpr *createLo(const MCExpr *Expr,
                                    bool isDarwin, MCContext &Ctx) {
-    return Create(VK_PPC_LO, Expr, isDarwin, Ctx);
+    return create(VK_PPC_LO, Expr, isDarwin, Ctx);
   }
 
-  static const PPCMCExpr *CreateHi(const MCExpr *Expr,
+  static const PPCMCExpr *createHi(const MCExpr *Expr,
                                    bool isDarwin, MCContext &Ctx) {
-    return Create(VK_PPC_HI, Expr, isDarwin, Ctx);
+    return create(VK_PPC_HI, Expr, isDarwin, Ctx);
   }
 
-  static const PPCMCExpr *CreateHa(const MCExpr *Expr,
+  static const PPCMCExpr *createHa(const MCExpr *Expr,
                                    bool isDarwin, MCContext &Ctx) {
-    return Create(VK_PPC_HA, Expr, isDarwin, Ctx);
+    return create(VK_PPC_HA, Expr, isDarwin, Ctx);
   }
 
   /// @}
@@ -77,19 +77,19 @@ class PPCMCExpr : public MCTargetExpr {
 
   /// @}
 
-  void PrintImpl(raw_ostream &OS) const override;
-  bool EvaluateAsRelocatableImpl(MCValue &Res,
+  void printImpl(raw_ostream &OS, const MCAsmInfo *MAI) const override;
+  bool evaluateAsRelocatableImpl(MCValue &Res,
                                  const MCAsmLayout *Layout,
                                  const MCFixup *Fixup) const override;
   void visitUsedExpr(MCStreamer &Streamer) const override;
-  const MCSection *FindAssociatedSection() const override {
-    return getSubExpr()->FindAssociatedSection();
+  MCSection *findAssociatedSection() const override {
+    return getSubExpr()->findAssociatedSection();
   }
 
   // There are no TLS PPCMCExprs at the moment.
   void fixELFSymbolsInTLSFixups(MCAssembler &Asm) const override {}
 
-  bool EvaluateAsConstant(int64_t &Res) const;
+  bool evaluateAsConstant(int64_t &Res) const;
 
   static bool classof(const MCExpr *E) {
     return E->getKind() == MCExpr::Target;
diff --git a/lib/Target/PowerPC/MCTargetDesc/PPCMCTargetDesc.cpp b/lib/Target/PowerPC/MCTargetDesc/PPCMCTargetDesc.cpp
index 423e427a7a8..30f232a9a91 100644
--- a/lib/Target/PowerPC/MCTargetDesc/PPCMCTargetDesc.cpp
+++ b/lib/Target/PowerPC/MCTargetDesc/PPCMCTargetDesc.cpp
@@ -16,14 +16,14 @@
 #include "PPCMCAsmInfo.h"
 #include "PPCTargetStreamer.h"
 #include "llvm/MC/MCCodeGenInfo.h"
-#include "llvm/MC/MCELF.h"
+#include "llvm/MC/MCContext.h"
 #include "llvm/MC/MCELFStreamer.h"
 #include "llvm/MC/MCExpr.h"
 #include "llvm/MC/MCInstrInfo.h"
 #include "llvm/MC/MCRegisterInfo.h"
 #include "llvm/MC/MCStreamer.h"
 #include "llvm/MC/MCSubtargetInfo.h"
-#include "llvm/MC/MCSymbol.h"
+#include "llvm/MC/MCSymbolELF.h"
 #include "llvm/MC/MachineLocation.h"
 #include "llvm/Support/ELF.h"
 #include "llvm/Support/ErrorHandling.h"
@@ -51,10 +51,9 @@ static MCInstrInfo *createPPCMCInstrInfo() {
   return X;
 }
 
-static MCRegisterInfo *createPPCMCRegisterInfo(StringRef TT) {
-  Triple TheTriple(TT);
-  bool isPPC64 = (TheTriple.getArch() == Triple::ppc64 ||
-                  TheTriple.getArch() == Triple::ppc64le);
+static MCRegisterInfo *createPPCMCRegisterInfo(const Triple &TT) {
+  bool isPPC64 =
+      (TT.getArch() == Triple::ppc64 || TT.getArch() == Triple::ppc64le);
   unsigned Flavour = isPPC64 ? 0 : 1;
   unsigned RA = isPPC64 ? PPC::LR8 : PPC::LR;
 
@@ -63,15 +62,13 @@ static MCRegisterInfo *createPPCMCRegisterInfo(StringRef TT) {
   return X;
 }
 
-static MCSubtargetInfo *createPPCMCSubtargetInfo(StringRef TT, StringRef CPU,
-                                                 StringRef FS) {
-  MCSubtargetInfo *X = new MCSubtargetInfo();
-  InitPPCMCSubtargetInfo(X, TT, CPU, FS);
-  return X;
+static MCSubtargetInfo *createPPCMCSubtargetInfo(const Triple &TT,
+                                                 StringRef CPU, StringRef FS) {
+  return createPPCMCSubtargetInfoImpl(TT, CPU, FS);
 }
 
-static MCAsmInfo *createPPCMCAsmInfo(const MCRegisterInfo &MRI, StringRef TT) {
-  Triple TheTriple(TT);
+static MCAsmInfo *createPPCMCAsmInfo(const MCRegisterInfo &MRI,
+                                     const Triple &TheTriple) {
   bool isPPC64 = (TheTriple.getArch() == Triple::ppc64 ||
                   TheTriple.getArch() == Triple::ppc64le);
 
@@ -90,25 +87,23 @@ static MCAsmInfo *createPPCMCAsmInfo(const MCRegisterInfo &MRI, StringRef TT) {
   return MAI;
 }
 
-static MCCodeGenInfo *createPPCMCCodeGenInfo(StringRef TT, Reloc::Model RM,
+static MCCodeGenInfo *createPPCMCCodeGenInfo(const Triple &TT, Reloc::Model RM,
                                              CodeModel::Model CM,
                                              CodeGenOpt::Level OL) {
   MCCodeGenInfo *X = new MCCodeGenInfo();
 
   if (RM == Reloc::Default) {
-    Triple T(TT);
-    if (T.isOSDarwin())
+    if (TT.isOSDarwin())
       RM = Reloc::DynamicNoPIC;
     else
       RM = Reloc::Static;
   }
   if (CM == CodeModel::Default) {
-    Triple T(TT);
-    if (!T.isOSDarwin() &&
-        (T.getArch() == Triple::ppc64 || T.getArch() == Triple::ppc64le))
+    if (!TT.isOSDarwin() &&
+        (TT.getArch() == Triple::ppc64 || TT.getArch() == Triple::ppc64le))
       CM = CodeModel::Medium;
   }
-  X->InitMCCodeGenInfo(RM, CM, OL);
+  X->initMCCodeGenInfo(RM, CM, OL);
   return X;
 }
 
@@ -132,8 +127,14 @@ class PPCTargetAsmStreamer : public PPCTargetStreamer {
   void emitAbiVersion(int AbiVersion) override {
     OS << "\t.abiversion " << AbiVersion << '\n';
   }
-  void emitLocalEntry(MCSymbol *S, const MCExpr *LocalOffset) override {
-    OS << "\t.localentry\t" << *S << ", " << *LocalOffset << '\n';
+  void emitLocalEntry(MCSymbolELF *S, const MCExpr *LocalOffset) override {
+    const MCAsmInfo *MAI = Streamer.getContext().getAsmInfo();
+
+    OS << "\t.localentry\t";
+    S->print(OS, MAI);
+    OS << ", ";
+    LocalOffset->print(OS, MAI);
+    OS << '\n';
   }
 };
 
@@ -159,25 +160,21 @@ class PPCTargetELFStreamer : public PPCTargetStreamer {
     Flags |= (AbiVersion & ELF::EF_PPC64_ABI);
     MCA.setELFHeaderEFlags(Flags);
   }
-  void emitLocalEntry(MCSymbol *S, const MCExpr *LocalOffset) override {
+  void emitLocalEntry(MCSymbolELF *S, const MCExpr *LocalOffset) override {
     MCAssembler &MCA = getStreamer().getAssembler();
-    MCSymbolData &Data = getStreamer().getOrCreateSymbolData(S);
 
     int64_t Res;
-    if (!LocalOffset->EvaluateAsAbsolute(Res, MCA))
+    if (!LocalOffset->evaluateAsAbsolute(Res, MCA))
       report_fatal_error(".localentry expression must be absolute.");
 
     unsigned Encoded = ELF::encodePPC64LocalEntryOffset(Res);
     if (Res != ELF::decodePPC64LocalEntryOffset(Encoded))
       report_fatal_error(".localentry expression cannot be encoded.");
 
-    // The "other" values are stored in the last 6 bits of the second byte.
-    // The traditional defines for STO values assume the full byte and thus
-    // the shift to pack it.
-    unsigned Other = MCELF::getOther(Data) << 2;
+    unsigned Other = S->getOther();
     Other &= ~ELF::STO_PPC64_LOCAL_MASK;
     Other |= Encoded;
-    MCELF::setOther(Data, Other >> 2);
+    S->setOther(Other);
 
     // For GAS compatibility, unless we already saw a .abiversion directive,
     // set e_flags to indicate ELFv2 ABI.
@@ -185,22 +182,18 @@ class PPCTargetELFStreamer : public PPCTargetStreamer {
     if ((Flags & ELF::EF_PPC64_ABI) == 0)
       MCA.setELFHeaderEFlags(Flags | 2);
   }
-  void emitAssignment(MCSymbol *Symbol, const MCExpr *Value) override {
+  void emitAssignment(MCSymbol *S, const MCExpr *Value) override {
+    auto *Symbol = cast<MCSymbolELF>(S);
     // When encoding an assignment to set symbol A to symbol B, also copy
     // the st_other bits encoding the local entry point offset.
     if (Value->getKind() != MCExpr::SymbolRef)
       return;
-    const MCSymbol &RhsSym =
-        static_cast<const MCSymbolRefExpr *>(Value)->getSymbol();
-    MCSymbolData &Data = getStreamer().getOrCreateSymbolData(&RhsSym);
-    MCSymbolData &SymbolData = getStreamer().getOrCreateSymbolData(Symbol);
-    // The "other" values are stored in the last 6 bits of the second byte.
-    // The traditional defines for STO values assume the full byte and thus
-    // the shift to pack it.
-    unsigned Other = MCELF::getOther(SymbolData) << 2;
+    const auto &RhsSym = cast<MCSymbolELF>(
+        static_cast<const MCSymbolRefExpr *>(Value)->getSymbol());
+    unsigned Other = Symbol->getOther();
     Other &= ~ELF::STO_PPC64_LOCAL_MASK;
-    Other |= (MCELF::getOther(Data) << 2) & ELF::STO_PPC64_LOCAL_MASK;
-    MCELF::setOther(SymbolData, Other >> 2);
+    Other |= RhsSym.getOther() & ELF::STO_PPC64_LOCAL_MASK;
+    Symbol->setOther(Other);
   }
 };
 
@@ -217,7 +210,7 @@ class PPCTargetMachOStreamer : public PPCTargetStreamer {
   void emitAbiVersion(int AbiVersion) override {
     llvm_unreachable("Unknown pseudo-op: .abiversion");
   }
-  void emitLocalEntry(MCSymbol *S, const MCExpr *LocalOffset) override {
+  void emitLocalEntry(MCSymbolELF *S, const MCExpr *LocalOffset) override {
     llvm_unreachable("Unknown pseudo-op: .localentry");
   }
 };
@@ -232,8 +225,8 @@ static MCTargetStreamer *createAsmTargetStreamer(MCStreamer &S,
 
 static MCTargetStreamer *
 createObjectTargetStreamer(MCStreamer &S, const MCSubtargetInfo &STI) {
-  Triple TT(STI.getTargetTriple());
-  if (TT.getObjectFormat() == Triple::ELF)
+  const Triple &TT = STI.getTargetTriple();
+  if (TT.isOSBinFormatELF())
     return new PPCTargetELFStreamer(S);
   return new PPCTargetMachOStreamer(S);
 }
diff --git a/lib/Target/PowerPC/MCTargetDesc/PPCMCTargetDesc.h b/lib/Target/PowerPC/MCTargetDesc/PPCMCTargetDesc.h
index 5f2117c88e4..77fe4588228 100644
--- a/lib/Target/PowerPC/MCTargetDesc/PPCMCTargetDesc.h
+++ b/lib/Target/PowerPC/MCTargetDesc/PPCMCTargetDesc.h
@@ -29,6 +29,7 @@ class MCObjectWriter;
 class MCRegisterInfo;
 class MCSubtargetInfo;
 class Target;
+class Triple;
 class StringRef;
 class raw_pwrite_stream;
 class raw_ostream;
@@ -42,7 +43,7 @@ MCCodeEmitter *createPPCMCCodeEmitter(const MCInstrInfo &MCII,
                                       MCContext &Ctx);
 
 MCAsmBackend *createPPCAsmBackend(const Target &T, const MCRegisterInfo &MRI,
-                                  StringRef TT, StringRef CPU);
+                                  const Triple &TT, StringRef CPU);
 
 /// Construct an PPC ELF object writer.
 MCObjectWriter *createPPCELFObjectWriter(raw_pwrite_stream &OS, bool Is64Bit,
diff --git a/lib/Target/PowerPC/MCTargetDesc/PPCMachObjectWriter.cpp b/lib/Target/PowerPC/MCTargetDesc/PPCMachObjectWriter.cpp
index 44e69b7565d..9d7289658f0 100644
--- a/lib/Target/PowerPC/MCTargetDesc/PPCMachObjectWriter.cpp
+++ b/lib/Target/PowerPC/MCTargetDesc/PPCMachObjectWriter.cpp
@@ -24,7 +24,7 @@ using namespace llvm;
 
 namespace {
 class PPCMachObjectWriter : public MCMachObjectTargetWriter {
-  bool RecordScatteredRelocation(MachObjectWriter *Writer,
+  bool recordScatteredRelocation(MachObjectWriter *Writer,
                                  const MCAssembler &Asm,
                                  const MCAsmLayout &Layout,
                                  const MCFragment *Fragment,
@@ -38,10 +38,9 @@ class PPCMachObjectWriter : public MCMachObjectTargetWriter {
 
 public:
   PPCMachObjectWriter(bool Is64Bit, uint32_t CPUType, uint32_t CPUSubtype)
-      : MCMachObjectTargetWriter(Is64Bit, CPUType, CPUSubtype,
-                                 /*UseAggressiveSymbolFolding=*/Is64Bit) {}
+      : MCMachObjectTargetWriter(Is64Bit, CPUType, CPUSubtype) {}
 
-  void RecordRelocation(MachObjectWriter *Writer, MCAssembler &Asm,
+  void recordRelocation(MachObjectWriter *Writer, MCAssembler &Asm,
                         const MCAsmLayout &Layout, const MCFragment *Fragment,
                         const MCFixup &Fixup, MCValue Target,
                         uint64_t &FixedValue) override {
@@ -187,9 +186,9 @@ static uint32_t getFixupOffset(const MCAsmLayout &Layout,
 
 /// \return false if falling back to using non-scattered relocation,
 /// otherwise true for normal scattered relocation.
-/// based on X86MachObjectWriter::RecordScatteredRelocation
-/// and ARMMachObjectWriter::RecordScatteredRelocation
-bool PPCMachObjectWriter::RecordScatteredRelocation(
+/// based on X86MachObjectWriter::recordScatteredRelocation
+/// and ARMMachObjectWriter::recordScatteredRelocation
+bool PPCMachObjectWriter::recordScatteredRelocation(
     MachObjectWriter *Writer, const MCAssembler &Asm, const MCAsmLayout &Layout,
     const MCFragment *Fragment, const MCFixup &Fixup, MCValue Target,
     unsigned Log2Size, uint64_t &FixedValue) {
@@ -206,28 +205,26 @@ bool PPCMachObjectWriter::RecordScatteredRelocation(
 
   // See <reloc.h>.
   const MCSymbol *A = &Target.getSymA()->getSymbol();
-  const MCSymbolData *A_SD = &Asm.getSymbolData(*A);
 
-  if (!A_SD->getFragment())
+  if (!A->getFragment())
     report_fatal_error("symbol '" + A->getName() +
                        "' can not be undefined in a subtraction expression");
 
-  uint32_t Value = Writer->getSymbolAddress(A_SD, Layout);
-  uint64_t SecAddr =
-      Writer->getSectionAddress(A_SD->getFragment()->getParent());
+  uint32_t Value = Writer->getSymbolAddress(*A, Layout);
+  uint64_t SecAddr = Writer->getSectionAddress(A->getFragment()->getParent());
   FixedValue += SecAddr;
   uint32_t Value2 = 0;
 
   if (const MCSymbolRefExpr *B = Target.getSymB()) {
-    const MCSymbolData *B_SD = &Asm.getSymbolData(B->getSymbol());
+    const MCSymbol *SB = &B->getSymbol();
 
-    if (!B_SD->getFragment())
+    if (!SB->getFragment())
       report_fatal_error("symbol '" + B->getSymbol().getName() +
                          "' can not be undefined in a subtraction expression");
 
     // FIXME: is Type correct? see include/llvm/Support/MachO.h
-    Value2 = Writer->getSymbolAddress(B_SD, Layout);
-    FixedValue -= Writer->getSectionAddress(B_SD->getFragment()->getParent());
+    Value2 = Writer->getSymbolAddress(B->getSymbol(), Layout);
+    FixedValue -= Writer->getSectionAddress(SB->getFragment()->getParent());
   }
   // FIXME: does FixedValue get used??
 
@@ -244,7 +241,7 @@ bool PPCMachObjectWriter::RecordScatteredRelocation(
     if (FixupOffset > 0xffffff) {
       char Buffer[32];
       format("0x%x", FixupOffset).print(Buffer, sizeof(Buffer));
-      Asm.getContext().FatalError(Fixup.getLoc(),
+      Asm.getContext().reportFatalError(Fixup.getLoc(),
                                   Twine("Section too large, can't encode "
                                         "r_address (") +
                                       Buffer + ") into 24 bits of scattered "
@@ -253,7 +250,7 @@ bool PPCMachObjectWriter::RecordScatteredRelocation(
     }
 
     // Is this supposed to follow MCTarget/PPCAsmBackend.cpp:adjustFixupValue()?
-    // see PPCMCExpr::EvaluateAsRelocatableImpl()
+    // see PPCMCExpr::evaluateAsRelocatableImpl()
     uint32_t other_half = 0;
     switch (Type) {
     case MachO::PPC_RELOC_LO16_SECTDIFF:
@@ -317,23 +314,23 @@ void PPCMachObjectWriter::RecordPPCRelocation(
       // Q: are branch targets ever scattered?
       RelocType != MachO::PPC_RELOC_BR24 &&
       RelocType != MachO::PPC_RELOC_BR14) {
-    RecordScatteredRelocation(Writer, Asm, Layout, Fragment, Fixup, Target,
+    recordScatteredRelocation(Writer, Asm, Layout, Fragment, Fixup, Target,
                               Log2Size, FixedValue);
     return;
   }
 
   // this doesn't seem right for RIT_PPC_BR24
   // Get the symbol data, if any.
-  const MCSymbolData *SD = nullptr;
+  const MCSymbol *A = nullptr;
   if (Target.getSymA())
-    SD = &Asm.getSymbolData(Target.getSymA()->getSymbol());
+    A = &Target.getSymA()->getSymbol();
 
   // See <reloc.h>.
   const uint32_t FixupOffset = getFixupOffset(Layout, Fragment, Fixup);
   unsigned Index = 0;
   unsigned Type = RelocType;
 
-  const MCSymbolData *RelSymbol = nullptr;
+  const MCSymbol *RelSymbol = nullptr;
   if (Target.isAbsolute()) { // constant
                              // SymbolNum of 0 indicates the absolute section.
                              //
@@ -344,9 +341,9 @@ void PPCMachObjectWriter::RecordPPCRelocation(
     // the above line stolen from ARM, not sure
   } else {
     // Resolve constant variables.
-    if (SD->getSymbol().isVariable()) {
+    if (A->isVariable()) {
       int64_t Res;
-      if (SD->getSymbol().getVariableValue()->EvaluateAsAbsolute(
+      if (A->getVariableValue()->evaluateAsAbsolute(
               Res, Layout, Writer->getSectionAddressMap())) {
         FixedValue = Res;
         return;
@@ -354,19 +351,18 @@ void PPCMachObjectWriter::RecordPPCRelocation(
     }
 
     // Check whether we need an external or internal relocation.
-    if (Writer->doesSymbolRequireExternRelocation(SD)) {
-      RelSymbol = SD;
+    if (Writer->doesSymbolRequireExternRelocation(*A)) {
+      RelSymbol = A;
       // For external relocations, make sure to offset the fixup value to
       // compensate for the addend of the symbol address, if it was
       // undefined. This occurs with weak definitions, for example.
-      if (!SD->getSymbol().isUndefined())
-        FixedValue -= Layout.getSymbolOffset(SD);
+      if (!A->isUndefined())
+        FixedValue -= Layout.getSymbolOffset(*A);
     } else {
       // The index is the section ordinal (1-based).
-      const MCSectionData &SymSD =
-          Asm.getSectionData(SD->getSymbol().getSection());
-      Index = SymSD.getOrdinal() + 1;
-      FixedValue += Writer->getSectionAddress(&SymSD);
+      const MCSection &Sec = A->getSection();
+      Index = Sec.getOrdinal() + 1;
+      FixedValue += Writer->getSectionAddress(&Sec);
     }
     if (IsPCRel)
       FixedValue -= Writer->getSectionAddress(Fragment->getParent());
diff --git a/lib/Target/PowerPC/PPC.h b/lib/Target/PowerPC/PPC.h
index 5e5a9b1999d..ae8d8b4f5df 100644
--- a/lib/Target/PowerPC/PPC.h
+++ b/lib/Target/PowerPC/PPC.h
@@ -36,9 +36,11 @@ namespace llvm {
 #endif
   FunctionPass *createPPCLoopDataPrefetchPass();
   FunctionPass *createPPCLoopPreIncPrepPass(PPCTargetMachine &TM);
+  FunctionPass *createPPCTOCRegDepsPass();
   FunctionPass *createPPCEarlyReturnPass();
   FunctionPass *createPPCVSXCopyPass();
   FunctionPass *createPPCVSXFMAMutatePass();
+  FunctionPass *createPPCVSXSwapRemovalPass();
   FunctionPass *createPPCBranchSelectionPass();
   FunctionPass *createPPCISelDag(PPCTargetMachine &TM);
   FunctionPass *createPPCTLSDynamicCallPass();
diff --git a/lib/Target/PowerPC/PPC.td b/lib/Target/PowerPC/PPC.td
index 1a02bcca936..641b2377de4 100644
--- a/lib/Target/PowerPC/PPC.td
+++ b/lib/Target/PowerPC/PPC.td
@@ -135,9 +135,9 @@ def FeatureInvariantFunctionDescriptors :
                    "Assume function descriptors are invariant">;
 def FeatureHTM : SubtargetFeature<"htm", "HasHTM", "true",
                                   "Enable Hardware Transactional Memory instructions">;
+def FeatureMFTB   : SubtargetFeature<"", "FeatureMFTB", "true",
+                                        "Implement mftb using the mfspr instruction">;
 
-def DeprecatedMFTB   : SubtargetFeature<"", "DeprecatedMFTB", "true",
-                                        "Treat mftb as deprecated">;
 def DeprecatedDST    : SubtargetFeature<"", "DeprecatedDST", "true",
   "Treat vector data stream cache control instructions as deprecated">;
 
@@ -165,7 +165,7 @@ def ProcessorFeatures {
        FeaturePOPCNTD, FeatureCMPB, FeatureLDBRX,
        Feature64Bit /*, Feature64BitRegs */,
        FeatureBPERMD, FeatureExtDiv,
-       DeprecatedMFTB, DeprecatedDST];
+       FeatureMFTB, DeprecatedDST];
   list<SubtargetFeature> Power8SpecificFeatures =
       [DirectivePwr8, FeatureP8Altivec, FeatureP8Vector, FeatureP8Crypto,
        FeatureHTM, FeatureDirectMove, FeatureICBT, FeaturePartwordAtomic];
@@ -247,61 +247,75 @@ include "PPCInstrInfo.td"
 // PowerPC processors supported.
 //
 
-def : Processor<"generic", G3Itineraries, [Directive32]>;
+def : Processor<"generic", G3Itineraries, [Directive32, FeatureMFTB]>;
 def : ProcessorModel<"440", PPC440Model, [Directive440, FeatureISEL,
                                           FeatureFRES, FeatureFRSQRTE,
                                           FeatureICBT, FeatureBookE, 
-                                          FeatureMSYNC, DeprecatedMFTB]>;
+                                          FeatureMSYNC, FeatureMFTB]>;
 def : ProcessorModel<"450", PPC440Model, [Directive440, FeatureISEL,
                                           FeatureFRES, FeatureFRSQRTE,
                                           FeatureICBT, FeatureBookE, 
-                                          FeatureMSYNC, DeprecatedMFTB]>;
+                                          FeatureMSYNC, FeatureMFTB]>;
 def : Processor<"601", G3Itineraries, [Directive601]>;
-def : Processor<"602", G3Itineraries, [Directive602]>;
+def : Processor<"602", G3Itineraries, [Directive602,
+                                       FeatureMFTB]>;
 def : Processor<"603", G3Itineraries, [Directive603,
-                                       FeatureFRES, FeatureFRSQRTE]>;
+                                       FeatureFRES, FeatureFRSQRTE,
+                                       FeatureMFTB]>;
 def : Processor<"603e", G3Itineraries, [Directive603,
-                                        FeatureFRES, FeatureFRSQRTE]>;
+                                        FeatureFRES, FeatureFRSQRTE,
+                                        FeatureMFTB]>;
 def : Processor<"603ev", G3Itineraries, [Directive603,
-                                         FeatureFRES, FeatureFRSQRTE]>;
+                                         FeatureFRES, FeatureFRSQRTE,
+                                         FeatureMFTB]>;
 def : Processor<"604", G3Itineraries, [Directive604,
-                                       FeatureFRES, FeatureFRSQRTE]>;
+                                       FeatureFRES, FeatureFRSQRTE,
+                                       FeatureMFTB]>;
 def : Processor<"604e", G3Itineraries, [Directive604,
-                                        FeatureFRES, FeatureFRSQRTE]>;
+                                        FeatureFRES, FeatureFRSQRTE,
+                                        FeatureMFTB]>;
 def : Processor<"620", G3Itineraries, [Directive620,
-                                       FeatureFRES, FeatureFRSQRTE]>;
+                                       FeatureFRES, FeatureFRSQRTE,
+                                       FeatureMFTB]>;
 def : Processor<"750", G4Itineraries, [Directive750,
-                                       FeatureFRES, FeatureFRSQRTE]>;
+                                       FeatureFRES, FeatureFRSQRTE,
+                                       FeatureMFTB]>;
 def : Processor<"g3", G3Itineraries, [Directive750,
-                                      FeatureFRES, FeatureFRSQRTE]>;
+                                      FeatureFRES, FeatureFRSQRTE,
+                                      FeatureMFTB]>;
 def : Processor<"7400", G4Itineraries, [Directive7400, FeatureAltivec,
-                                        FeatureFRES, FeatureFRSQRTE]>;
+                                        FeatureFRES, FeatureFRSQRTE,
+                                        FeatureMFTB]>;
 def : Processor<"g4", G4Itineraries, [Directive7400, FeatureAltivec,
-                                      FeatureFRES, FeatureFRSQRTE]>;
+                                      FeatureFRES, FeatureFRSQRTE,
+                                      FeatureMFTB]>;
 def : Processor<"7450", G4PlusItineraries, [Directive7400, FeatureAltivec,
-                                            FeatureFRES, FeatureFRSQRTE]>;
+                                            FeatureFRES, FeatureFRSQRTE,
+                                            FeatureMFTB]>;
 def : Processor<"g4+", G4PlusItineraries, [Directive7400, FeatureAltivec,
-                                           FeatureFRES, FeatureFRSQRTE]>;
+                                           FeatureFRES, FeatureFRSQRTE, 
+                                           FeatureMFTB]>;
 
 def : ProcessorModel<"970", G5Model,
                   [Directive970, FeatureAltivec,
                    FeatureMFOCRF, FeatureFSqrt,
                    FeatureFRES, FeatureFRSQRTE, FeatureSTFIWX,
-                   Feature64Bit /*, Feature64BitRegs */]>;
+                   Feature64Bit /*, Feature64BitRegs */,
+                   FeatureMFTB]>;
 def : ProcessorModel<"g5", G5Model,
                   [Directive970, FeatureAltivec,
                    FeatureMFOCRF, FeatureFSqrt, FeatureSTFIWX,
                    FeatureFRES, FeatureFRSQRTE,
                    Feature64Bit /*, Feature64BitRegs */,
-                   DeprecatedMFTB, DeprecatedDST]>;
+                   FeatureMFTB, DeprecatedDST]>;
 def : ProcessorModel<"e500mc", PPCE500mcModel,
                   [DirectiveE500mc, FeatureMFOCRF,
                    FeatureSTFIWX, FeatureICBT, FeatureBookE, 
-                   FeatureISEL, DeprecatedMFTB]>;
+                   FeatureISEL, FeatureMFTB]>;
 def : ProcessorModel<"e5500", PPCE5500Model,
                   [DirectiveE5500, FeatureMFOCRF, Feature64Bit,
                    FeatureSTFIWX, FeatureICBT, FeatureBookE, 
-                   FeatureISEL, DeprecatedMFTB]>;
+                   FeatureISEL, FeatureMFTB]>;
 def : ProcessorModel<"a2", PPCA2Model,
                   [DirectiveA2, FeatureICBT, FeatureBookE, FeatureMFOCRF,
                    FeatureFCPSGN, FeatureFSqrt, FeatureFRE, FeatureFRES,
@@ -309,7 +323,7 @@ def : ProcessorModel<"a2", PPCA2Model,
                    FeatureSTFIWX, FeatureLFIWAX,
                    FeatureFPRND, FeatureFPCVT, FeatureISEL,
                    FeaturePOPCNTD, FeatureCMPB, FeatureLDBRX, Feature64Bit
-               /*, Feature64BitRegs */, DeprecatedMFTB]>;
+               /*, Feature64BitRegs */, FeatureMFTB]>;
 def : ProcessorModel<"a2q", PPCA2Model,
                   [DirectiveA2, FeatureICBT, FeatureBookE, FeatureMFOCRF,
                    FeatureFCPSGN, FeatureFSqrt, FeatureFRE, FeatureFRES,
@@ -317,7 +331,7 @@ def : ProcessorModel<"a2q", PPCA2Model,
                    FeatureSTFIWX, FeatureLFIWAX,
                    FeatureFPRND, FeatureFPCVT, FeatureISEL,
                    FeaturePOPCNTD, FeatureCMPB, FeatureLDBRX, Feature64Bit
-               /*, Feature64BitRegs */, FeatureQPX, DeprecatedMFTB]>;
+               /*, Feature64BitRegs */, FeatureQPX, FeatureMFTB]>;
 def : ProcessorModel<"pwr3", G5Model,
                   [DirectivePwr3, FeatureAltivec,
                    FeatureFRES, FeatureFRSQRTE, FeatureMFOCRF,
@@ -325,41 +339,42 @@ def : ProcessorModel<"pwr3", G5Model,
 def : ProcessorModel<"pwr4", G5Model,
                   [DirectivePwr4, FeatureAltivec, FeatureMFOCRF,
                    FeatureFSqrt, FeatureFRES, FeatureFRSQRTE,
-                   FeatureSTFIWX, Feature64Bit]>;
+                   FeatureSTFIWX, Feature64Bit, FeatureMFTB]>;
 def : ProcessorModel<"pwr5", G5Model,
                   [DirectivePwr5, FeatureAltivec, FeatureMFOCRF,
                    FeatureFSqrt, FeatureFRE, FeatureFRES,
                    FeatureFRSQRTE, FeatureFRSQRTES,
                    FeatureSTFIWX, Feature64Bit,
-                   DeprecatedMFTB, DeprecatedDST]>;
+                   FeatureMFTB, DeprecatedDST]>;
 def : ProcessorModel<"pwr5x", G5Model,
                   [DirectivePwr5x, FeatureAltivec, FeatureMFOCRF,
                    FeatureFSqrt, FeatureFRE, FeatureFRES,
                    FeatureFRSQRTE, FeatureFRSQRTES,
                    FeatureSTFIWX, FeatureFPRND, Feature64Bit,
-                   DeprecatedMFTB, DeprecatedDST]>;
+                   FeatureMFTB, DeprecatedDST]>;
 def : ProcessorModel<"pwr6", G5Model,
                   [DirectivePwr6, FeatureAltivec,
                    FeatureMFOCRF, FeatureFCPSGN, FeatureFSqrt, FeatureFRE,
                    FeatureFRES, FeatureFRSQRTE, FeatureFRSQRTES,
                    FeatureRecipPrec, FeatureSTFIWX, FeatureLFIWAX, FeatureCMPB,
                    FeatureFPRND, Feature64Bit /*, Feature64BitRegs */,
-                   DeprecatedMFTB, DeprecatedDST]>;
+                   FeatureMFTB, DeprecatedDST]>;
 def : ProcessorModel<"pwr6x", G5Model,
                   [DirectivePwr5x, FeatureAltivec, FeatureMFOCRF,
                    FeatureFCPSGN, FeatureFSqrt, FeatureFRE, FeatureFRES,
                    FeatureFRSQRTE, FeatureFRSQRTES, FeatureRecipPrec,
                    FeatureSTFIWX, FeatureLFIWAX, FeatureCMPB,
                    FeatureFPRND, Feature64Bit,
-                   DeprecatedMFTB, DeprecatedDST]>;
+                   FeatureMFTB, DeprecatedDST]>;
 def : ProcessorModel<"pwr7", P7Model, ProcessorFeatures.Power7FeatureList>;
 def : ProcessorModel<"pwr8", P8Model, ProcessorFeatures.Power8FeatureList>;
-def : Processor<"ppc", G3Itineraries, [Directive32]>;
+def : Processor<"ppc", G3Itineraries, [Directive32, FeatureMFTB]>;
 def : ProcessorModel<"ppc64", G5Model,
                   [Directive64, FeatureAltivec,
                    FeatureMFOCRF, FeatureFSqrt, FeatureFRES,
                    FeatureFRSQRTE, FeatureSTFIWX,
-                   Feature64Bit /*, Feature64BitRegs */]>;
+                   Feature64Bit /*, Feature64BitRegs */,
+                   FeatureMFTB]>;
 def : ProcessorModel<"ppc64le", P8Model, ProcessorFeatures.Power8FeatureList>;
 
 //===----------------------------------------------------------------------===//
diff --git a/lib/Target/PowerPC/PPCAsmPrinter.cpp b/lib/Target/PowerPC/PPCAsmPrinter.cpp
index 383a1e2fe88..2ce7501d020 100644
--- a/lib/Target/PowerPC/PPCAsmPrinter.cpp
+++ b/lib/Target/PowerPC/PPCAsmPrinter.cpp
@@ -49,7 +49,7 @@
 #include "llvm/MC/MCSectionELF.h"
 #include "llvm/MC/MCSectionMachO.h"
 #include "llvm/MC/MCStreamer.h"
-#include "llvm/MC/MCSymbol.h"
+#include "llvm/MC/MCSymbolELF.h"
 #include "llvm/Support/CommandLine.h"
 #include "llvm/Support/Debug.h"
 #include "llvm/Support/ELF.h"
@@ -163,7 +163,7 @@ static const char *stripRegisterPrefix(const char *RegName) {
 
 void PPCAsmPrinter::printOperand(const MachineInstr *MI, unsigned OpNo,
                                  raw_ostream &O) {
-  const DataLayout *DL = TM.getDataLayout();
+  const DataLayout &DL = getDataLayout();
   const MachineOperand &MO = MI->getOperand(OpNo);
   
   switch (MO.getType()) {
@@ -181,14 +181,14 @@ void PPCAsmPrinter::printOperand(const MachineInstr *MI, unsigned OpNo,
     return;
 
   case MachineOperand::MO_MachineBasicBlock:
-    O << *MO.getMBB()->getSymbol();
+    MO.getMBB()->getSymbol()->print(O, MAI);
     return;
   case MachineOperand::MO_ConstantPoolIndex:
-    O << DL->getPrivateGlobalPrefix() << "CPI" << getFunctionNumber()
-      << '_' << MO.getIndex();
+    O << DL.getPrivateGlobalPrefix() << "CPI" << getFunctionNumber() << '_'
+      << MO.getIndex();
     return;
   case MachineOperand::MO_BlockAddress:
-    O << *GetBlockAddressSymbol(MO.getBlockAddress());
+    GetBlockAddressSymbol(MO.getBlockAddress())->print(O, MAI);
     return;
   case MachineOperand::MO_GlobalAddress: {
     // Computing the address of a global symbol, not calling it.
@@ -197,7 +197,7 @@ void PPCAsmPrinter::printOperand(const MachineInstr *MI, unsigned OpNo,
 
     // External or weakly linked global variables need non-lazily-resolved stubs
     if (TM.getRelocationModel() != Reloc::Static &&
-        (GV->isDeclaration() || GV->isWeakForLinker())) {
+        !GV->isStrongDefinitionForLinker()) {
       if (!GV->hasHiddenVisibility()) {
         SymToPrint = getSymbolWithGlobalValueBase(GV, "$non_lazy_ptr");
         MachineModuleInfoImpl::StubValueTy &StubSym = 
@@ -222,8 +222,8 @@ void PPCAsmPrinter::printOperand(const MachineInstr *MI, unsigned OpNo,
     } else {
       SymToPrint = getSymbol(GV);
     }
-    
-    O << *SymToPrint;
+
+    SymToPrint->print(O, MAI);
 
     printOffset(MO.getOffset(), O);
     return;
@@ -363,34 +363,90 @@ void PPCAsmPrinter::LowerPATCHPOINT(MCStreamer &OutStreamer, StackMaps &SM,
   SM.recordPatchPoint(MI);
   PatchPointOpers Opers(&MI);
 
-  int64_t CallTarget = Opers.getMetaOper(PatchPointOpers::TargetPos).getImm();
   unsigned EncodedBytes = 0;
-  if (CallTarget) {
-    assert((CallTarget & 0xFFFFFFFFFFFF) == CallTarget &&
-           "High 16 bits of call target should be zero.");
-    unsigned ScratchReg = MI.getOperand(Opers.getNextScratchIdx()).getReg();
-    EncodedBytes = 6*4;
-    // Materialize the jump address:
-    EmitToStreamer(OutStreamer, MCInstBuilder(PPC::LI8)
-                                    .addReg(ScratchReg)
-                                    .addImm((CallTarget >> 32) & 0xFFFF));
-    EmitToStreamer(OutStreamer, MCInstBuilder(PPC::RLDIC)
-                                    .addReg(ScratchReg)
-                                    .addReg(ScratchReg)
-                                    .addImm(32).addImm(16));
-    EmitToStreamer(OutStreamer, MCInstBuilder(PPC::ORIS8)
-                                    .addReg(ScratchReg)
-                                    .addReg(ScratchReg)
-                                    .addImm((CallTarget >> 16) & 0xFFFF));
-    EmitToStreamer(OutStreamer, MCInstBuilder(PPC::ORI8)
-                                    .addReg(ScratchReg)
-                                    .addReg(ScratchReg)
-                                    .addImm(CallTarget & 0xFFFF));
-
-    EmitToStreamer(OutStreamer, MCInstBuilder(PPC::MTCTR8).addReg(ScratchReg));
-    EmitToStreamer(OutStreamer, MCInstBuilder(PPC::BCTRL8));
+  const MachineOperand &CalleeMO =
+    Opers.getMetaOper(PatchPointOpers::TargetPos);
+
+  if (CalleeMO.isImm()) {
+    int64_t CallTarget = Opers.getMetaOper(PatchPointOpers::TargetPos).getImm();
+    if (CallTarget) {
+      assert((CallTarget & 0xFFFFFFFFFFFF) == CallTarget &&
+             "High 16 bits of call target should be zero.");
+      unsigned ScratchReg = MI.getOperand(Opers.getNextScratchIdx()).getReg();
+      EncodedBytes = 0;
+      // Materialize the jump address:
+      EmitToStreamer(OutStreamer, MCInstBuilder(PPC::LI8)
+                                      .addReg(ScratchReg)
+                                      .addImm((CallTarget >> 32) & 0xFFFF));
+      ++EncodedBytes;
+      EmitToStreamer(OutStreamer, MCInstBuilder(PPC::RLDIC)
+                                      .addReg(ScratchReg)
+                                      .addReg(ScratchReg)
+                                      .addImm(32).addImm(16));
+      ++EncodedBytes;
+      EmitToStreamer(OutStreamer, MCInstBuilder(PPC::ORIS8)
+                                      .addReg(ScratchReg)
+                                      .addReg(ScratchReg)
+                                      .addImm((CallTarget >> 16) & 0xFFFF));
+      ++EncodedBytes;
+      EmitToStreamer(OutStreamer, MCInstBuilder(PPC::ORI8)
+                                      .addReg(ScratchReg)
+                                      .addReg(ScratchReg)
+                                      .addImm(CallTarget & 0xFFFF));
+
+      // Save the current TOC pointer before the remote call.
+      int TOCSaveOffset = Subtarget->isELFv2ABI() ? 24 : 40;
+      EmitToStreamer(OutStreamer, MCInstBuilder(PPC::STD)
+                                      .addReg(PPC::X2)
+                                      .addImm(TOCSaveOffset)
+                                      .addReg(PPC::X1));
+      ++EncodedBytes;
+
+
+      // If we're on ELFv1, then we need to load the actual function pointer
+      // from the function descriptor.
+      if (!Subtarget->isELFv2ABI()) {
+	// Load the new TOC pointer and the function address, but not r11
+	// (needing this is rare, and loading it here would prevent passing it
+	// via a 'nest' parameter.
+        EmitToStreamer(OutStreamer, MCInstBuilder(PPC::LD)
+                                        .addReg(PPC::X2)
+                                        .addImm(8)
+                                        .addReg(ScratchReg));
+        ++EncodedBytes;
+        EmitToStreamer(OutStreamer, MCInstBuilder(PPC::LD)
+                                        .addReg(ScratchReg)
+                                        .addImm(0)
+                                        .addReg(ScratchReg));
+        ++EncodedBytes;
+      }
+
+      EmitToStreamer(OutStreamer, MCInstBuilder(PPC::MTCTR8)
+                                      .addReg(ScratchReg));
+      ++EncodedBytes;
+      EmitToStreamer(OutStreamer, MCInstBuilder(PPC::BCTRL8));
+      ++EncodedBytes;
+
+      // Restore the TOC pointer after the call.
+      EmitToStreamer(OutStreamer, MCInstBuilder(PPC::LD)
+                                      .addReg(PPC::X2)
+                                      .addImm(TOCSaveOffset)
+                                      .addReg(PPC::X1));
+      ++EncodedBytes;
+    }
+  } else if (CalleeMO.isGlobal()) {
+    const GlobalValue *GValue = CalleeMO.getGlobal();
+    MCSymbol *MOSymbol = getSymbol(GValue);
+    const MCExpr *SymVar = MCSymbolRefExpr::create(MOSymbol, OutContext);
+
+    EmitToStreamer(OutStreamer, MCInstBuilder(PPC::BL8_NOP)
+                                    .addExpr(SymVar));
+    EncodedBytes += 2;
   }
 
+  // Each instruction is 4 bytes.
+  EncodedBytes *= 4;
+
   // Emit padding.
   unsigned NumBytes = Opers.getMetaOper(PatchPointOpers::NBytesPos).getImm();
   assert(NumBytes >= EncodedBytes &&
@@ -406,7 +462,7 @@ void PPCAsmPrinter::LowerPATCHPOINT(MCStreamer &OutStreamer, StackMaps &SM,
 void PPCAsmPrinter::EmitTlsCall(const MachineInstr *MI,
                                 MCSymbolRefExpr::VariantKind VK) {
   StringRef Name = "__tls_get_addr";
-  MCSymbol *TlsGetAddr = OutContext.GetOrCreateSymbol(Name);
+  MCSymbol *TlsGetAddr = OutContext.getOrCreateSymbol(Name);
   MCSymbolRefExpr::VariantKind Kind = MCSymbolRefExpr::VK_None;
 
   assert(MI->getOperand(0).isReg() &&
@@ -422,12 +478,12 @@ void PPCAsmPrinter::EmitTlsCall(const MachineInstr *MI,
       TM.getRelocationModel() == Reloc::PIC_)
     Kind = MCSymbolRefExpr::VK_PLT;
   const MCSymbolRefExpr *TlsRef =
-    MCSymbolRefExpr::Create(TlsGetAddr, Kind, OutContext);
+    MCSymbolRefExpr::create(TlsGetAddr, Kind, OutContext);
   const MachineOperand &MO = MI->getOperand(2);
   const GlobalValue *GValue = MO.getGlobal();
   MCSymbol *MOSymbol = getSymbol(GValue);
-  const MCExpr *SymVar = MCSymbolRefExpr::Create(MOSymbol, VK, OutContext);
-  EmitToStreamer(OutStreamer,
+  const MCExpr *SymVar = MCSymbolRefExpr::create(MOSymbol, VK, OutContext);
+  EmitToStreamer(*OutStreamer,
                  MCInstBuilder(Subtarget->isPPC64() ?
                                PPC::BL8_NOP_TLS : PPC::BL_TLS)
                  .addExpr(TlsRef)
@@ -440,7 +496,7 @@ void PPCAsmPrinter::EmitTlsCall(const MachineInstr *MI,
 void PPCAsmPrinter::EmitInstruction(const MachineInstr *MI) {
   MCInst TmpInst;
   bool isPPC64 = Subtarget->isPPC64();
-  bool isDarwin = Triple(TM.getTargetTriple()).isOSDarwin();
+  bool isDarwin = TM.getTargetTriple().isOSDarwin();
   const Module *M = MF->getFunction()->getParent();
   PICLevel::Level PL = M->getPICLevel();
   
@@ -450,9 +506,9 @@ void PPCAsmPrinter::EmitInstruction(const MachineInstr *MI) {
   case TargetOpcode::DBG_VALUE:
     llvm_unreachable("Should be handled target independently");
   case TargetOpcode::STACKMAP:
-    return LowerSTACKMAP(OutStreamer, SM, *MI);
+    return LowerSTACKMAP(*OutStreamer, SM, *MI);
   case TargetOpcode::PATCHPOINT:
-    return LowerPATCHPOINT(OutStreamer, SM, *MI);
+    return LowerPATCHPOINT(*OutStreamer, SM, *MI);
 
   case PPC::MoveGOTtoLR: {
     // Transform %LR = MoveGOTtoLR
@@ -462,16 +518,16 @@ void PPCAsmPrinter::EmitInstruction(const MachineInstr *MI) {
     //      blrl
     // This will return the pointer to _GLOBAL_OFFSET_TABLE_@local
     MCSymbol *GOTSymbol =
-      OutContext.GetOrCreateSymbol(StringRef("_GLOBAL_OFFSET_TABLE_"));
+      OutContext.getOrCreateSymbol(StringRef("_GLOBAL_OFFSET_TABLE_"));
     const MCExpr *OffsExpr =
-      MCBinaryExpr::CreateSub(MCSymbolRefExpr::Create(GOTSymbol,
+      MCBinaryExpr::createSub(MCSymbolRefExpr::create(GOTSymbol,
                                                       MCSymbolRefExpr::VK_PPC_LOCAL,
                                                       OutContext),
-                              MCConstantExpr::Create(4, OutContext),
+                              MCConstantExpr::create(4, OutContext),
                               OutContext);
 
     // Emit the 'bl'.
-    EmitToStreamer(OutStreamer, MCInstBuilder(PPC::BL).addExpr(OffsExpr));
+    EmitToStreamer(*OutStreamer, MCInstBuilder(PPC::BL).addExpr(OffsExpr));
     return;
   }
   case PPC::MovePCtoLR:
@@ -483,13 +539,13 @@ void PPCAsmPrinter::EmitInstruction(const MachineInstr *MI) {
     MCSymbol *PICBase = MF->getPICBaseSymbol();
     
     // Emit the 'bl'.
-    EmitToStreamer(OutStreamer, MCInstBuilder(PPC::BL)
+    EmitToStreamer(*OutStreamer, MCInstBuilder(PPC::BL)
       // FIXME: We would like an efficient form for this, so we don't have to do
       // a lot of extra uniquing.
-      .addExpr(MCSymbolRefExpr::Create(PICBase, OutContext)));
+      .addExpr(MCSymbolRefExpr::create(PICBase, OutContext)));
     
     // Emit the label.
-    OutStreamer.EmitLabel(PICBase);
+    OutStreamer->EmitLabel(PICBase);
     return;
   }
   case PPC::UpdateGBR: {
@@ -502,9 +558,9 @@ void PPCAsmPrinter::EmitInstruction(const MachineInstr *MI) {
       MF->getInfo<PPCFunctionInfo>()->getPICOffsetSymbol();
     TmpInst.setOpcode(PPC::LWZ);
     const MCExpr *Exp =
-      MCSymbolRefExpr::Create(PICOffset, MCSymbolRefExpr::VK_None, OutContext);
+      MCSymbolRefExpr::create(PICOffset, MCSymbolRefExpr::VK_None, OutContext);
     const MCExpr *PB =
-      MCSymbolRefExpr::Create(MF->getPICBaseSymbol(),
+      MCSymbolRefExpr::create(MF->getPICBaseSymbol(),
                               MCSymbolRefExpr::VK_None,
                               OutContext);
     const MCOperand TR = TmpInst.getOperand(1);
@@ -512,16 +568,16 @@ void PPCAsmPrinter::EmitInstruction(const MachineInstr *MI) {
 
     // Step 1: lwz %Rt, .L$poff - .L$pb(%Ri)
     TmpInst.getOperand(1) =
-        MCOperand::CreateExpr(MCBinaryExpr::CreateSub(Exp, PB, OutContext));
+        MCOperand::createExpr(MCBinaryExpr::createSub(Exp, PB, OutContext));
     TmpInst.getOperand(0) = TR;
     TmpInst.getOperand(2) = PICR;
-    EmitToStreamer(OutStreamer, TmpInst);
+    EmitToStreamer(*OutStreamer, TmpInst);
 
     TmpInst.setOpcode(PPC::ADD4);
     TmpInst.getOperand(0) = PICR;
     TmpInst.getOperand(1) = TR;
     TmpInst.getOperand(2) = PICR;
-    EmitToStreamer(OutStreamer, TmpInst);
+    EmitToStreamer(*OutStreamer, TmpInst);
     return;
   }
   case PPC::LWZtoc: {
@@ -547,22 +603,22 @@ void PPCAsmPrinter::EmitInstruction(const MachineInstr *MI) {
 
     if (PL == PICLevel::Small) {
       const MCExpr *Exp =
-        MCSymbolRefExpr::Create(MOSymbol, MCSymbolRefExpr::VK_GOT,
+        MCSymbolRefExpr::create(MOSymbol, MCSymbolRefExpr::VK_GOT,
                                 OutContext);
-      TmpInst.getOperand(1) = MCOperand::CreateExpr(Exp);
+      TmpInst.getOperand(1) = MCOperand::createExpr(Exp);
     } else {
       MCSymbol *TOCEntry = lookUpOrCreateTOCEntry(MOSymbol);
 
       const MCExpr *Exp =
-        MCSymbolRefExpr::Create(TOCEntry, MCSymbolRefExpr::VK_None,
+        MCSymbolRefExpr::create(TOCEntry, MCSymbolRefExpr::VK_None,
                                 OutContext);
       const MCExpr *PB =
-        MCSymbolRefExpr::Create(OutContext.GetOrCreateSymbol(Twine(".LTOC")),
+        MCSymbolRefExpr::create(OutContext.getOrCreateSymbol(Twine(".LTOC")),
                                                              OutContext);
-      Exp = MCBinaryExpr::CreateSub(Exp, PB, OutContext);
-      TmpInst.getOperand(1) = MCOperand::CreateExpr(Exp);
+      Exp = MCBinaryExpr::createSub(Exp, PB, OutContext);
+      TmpInst.getOperand(1) = MCOperand::createExpr(Exp);
     }
-    EmitToStreamer(OutStreamer, TmpInst);
+    EmitToStreamer(*OutStreamer, TmpInst);
     return;
   }
   case PPC::LDtocJTI:
@@ -592,10 +648,10 @@ void PPCAsmPrinter::EmitInstruction(const MachineInstr *MI) {
     MCSymbol *TOCEntry = lookUpOrCreateTOCEntry(MOSymbol);
 
     const MCExpr *Exp =
-      MCSymbolRefExpr::Create(TOCEntry, MCSymbolRefExpr::VK_PPC_TOC,
+      MCSymbolRefExpr::create(TOCEntry, MCSymbolRefExpr::VK_PPC_TOC,
                               OutContext);
-    TmpInst.getOperand(1) = MCOperand::CreateExpr(Exp);
-    EmitToStreamer(OutStreamer, TmpInst);
+    TmpInst.getOperand(1) = MCOperand::createExpr(Exp);
+    EmitToStreamer(*OutStreamer, TmpInst);
     return;
   }
       
@@ -624,7 +680,7 @@ void PPCAsmPrinter::EmitInstruction(const MachineInstr *MI) {
       IsExternal = GV->isDeclaration();
       IsCommon = GV->hasCommonLinkage();
       IsNonLocalFunction = GV->getType()->getElementType()->isFunctionTy() &&
-        (GV->isDeclaration() || GV->isWeakForLinker());
+        !GV->isStrongDefinitionForLinker();
       IsAvailExt = GV->hasAvailableExternallyLinkage();
     } else if (MO.isCPI())
       MOSymbol = GetCPISymbol(MO.getIndex());
@@ -639,10 +695,10 @@ void PPCAsmPrinter::EmitInstruction(const MachineInstr *MI) {
       MOSymbol = lookUpOrCreateTOCEntry(MOSymbol);
 
     const MCExpr *Exp =
-      MCSymbolRefExpr::Create(MOSymbol, MCSymbolRefExpr::VK_PPC_TOC_HA,
+      MCSymbolRefExpr::create(MOSymbol, MCSymbolRefExpr::VK_PPC_TOC_HA,
                               OutContext);
-    TmpInst.getOperand(2) = MCOperand::CreateExpr(Exp);
-    EmitToStreamer(OutStreamer, TmpInst);
+    TmpInst.getOperand(2) = MCOperand::createExpr(Exp);
+    EmitToStreamer(*OutStreamer, TmpInst);
     return;
   }
   case PPC::LDtocL: {
@@ -681,10 +737,10 @@ void PPCAsmPrinter::EmitInstruction(const MachineInstr *MI) {
     }
 
     const MCExpr *Exp =
-      MCSymbolRefExpr::Create(MOSymbol, MCSymbolRefExpr::VK_PPC_TOC_LO,
+      MCSymbolRefExpr::create(MOSymbol, MCSymbolRefExpr::VK_PPC_TOC_LO,
                               OutContext);
-    TmpInst.getOperand(1) = MCOperand::CreateExpr(Exp);
-    EmitToStreamer(OutStreamer, TmpInst);
+    TmpInst.getOperand(1) = MCOperand::createExpr(Exp);
+    EmitToStreamer(*OutStreamer, TmpInst);
     return;
   }
   case PPC::ADDItocL: {
@@ -706,7 +762,7 @@ void PPCAsmPrinter::EmitInstruction(const MachineInstr *MI) {
       MOSymbol = getSymbol(GV);
       IsExternal = GV->isDeclaration();
       IsNonLocalFunction = GV->getType()->getElementType()->isFunctionTy() &&
-        (GV->isDeclaration() || GV->isWeakForLinker());
+        !GV->isStrongDefinitionForLinker();
     } else if (MO.isCPI())
       MOSymbol = GetCPISymbol(MO.getIndex());
 
@@ -715,10 +771,10 @@ void PPCAsmPrinter::EmitInstruction(const MachineInstr *MI) {
       MOSymbol = lookUpOrCreateTOCEntry(MOSymbol);
 
     const MCExpr *Exp =
-      MCSymbolRefExpr::Create(MOSymbol, MCSymbolRefExpr::VK_PPC_TOC_LO,
+      MCSymbolRefExpr::create(MOSymbol, MCSymbolRefExpr::VK_PPC_TOC_LO,
                               OutContext);
-    TmpInst.getOperand(2) = MCOperand::CreateExpr(Exp);
-    EmitToStreamer(OutStreamer, TmpInst);
+    TmpInst.getOperand(2) = MCOperand::createExpr(Exp);
+    EmitToStreamer(*OutStreamer, TmpInst);
     return;
   }
   case PPC::ADDISgotTprelHA: {
@@ -729,12 +785,12 @@ void PPCAsmPrinter::EmitInstruction(const MachineInstr *MI) {
     const GlobalValue *GValue = MO.getGlobal();
     MCSymbol *MOSymbol = getSymbol(GValue);
     const MCExpr *SymGotTprel =
-      MCSymbolRefExpr::Create(MOSymbol, MCSymbolRefExpr::VK_PPC_GOT_TPREL_HA,
+      MCSymbolRefExpr::create(MOSymbol, MCSymbolRefExpr::VK_PPC_GOT_TPREL_HA,
                               OutContext);
-    EmitToStreamer(OutStreamer, MCInstBuilder(PPC::ADDIS8)
-                                .addReg(MI->getOperand(0).getReg())
-                                .addReg(MI->getOperand(1).getReg())
-                                .addExpr(SymGotTprel));
+    EmitToStreamer(*OutStreamer, MCInstBuilder(PPC::ADDIS8)
+                                 .addReg(MI->getOperand(0).getReg())
+                                 .addReg(MI->getOperand(1).getReg())
+                                 .addExpr(SymGotTprel));
     return;
   }
   case PPC::LDgotTprelL:
@@ -748,56 +804,56 @@ void PPCAsmPrinter::EmitInstruction(const MachineInstr *MI) {
     const GlobalValue *GValue = MO.getGlobal();
     MCSymbol *MOSymbol = getSymbol(GValue);
     const MCExpr *Exp =
-      MCSymbolRefExpr::Create(MOSymbol, MCSymbolRefExpr::VK_PPC_GOT_TPREL_LO,
+      MCSymbolRefExpr::create(MOSymbol, MCSymbolRefExpr::VK_PPC_GOT_TPREL_LO,
                               OutContext);
-    TmpInst.getOperand(1) = MCOperand::CreateExpr(Exp);
-    EmitToStreamer(OutStreamer, TmpInst);
+    TmpInst.getOperand(1) = MCOperand::createExpr(Exp);
+    EmitToStreamer(*OutStreamer, TmpInst);
     return;
   }
 
   case PPC::PPC32PICGOT: {
-    MCSymbol *GOTSymbol = OutContext.GetOrCreateSymbol(StringRef("_GLOBAL_OFFSET_TABLE_"));
-    MCSymbol *GOTRef = OutContext.CreateTempSymbol();
-    MCSymbol *NextInstr = OutContext.CreateTempSymbol();
+    MCSymbol *GOTSymbol = OutContext.getOrCreateSymbol(StringRef("_GLOBAL_OFFSET_TABLE_"));
+    MCSymbol *GOTRef = OutContext.createTempSymbol();
+    MCSymbol *NextInstr = OutContext.createTempSymbol();
 
-    EmitToStreamer(OutStreamer, MCInstBuilder(PPC::BL)
+    EmitToStreamer(*OutStreamer, MCInstBuilder(PPC::BL)
       // FIXME: We would like an efficient form for this, so we don't have to do
       // a lot of extra uniquing.
-      .addExpr(MCSymbolRefExpr::Create(NextInstr, OutContext)));
+      .addExpr(MCSymbolRefExpr::create(NextInstr, OutContext)));
     const MCExpr *OffsExpr =
-      MCBinaryExpr::CreateSub(MCSymbolRefExpr::Create(GOTSymbol, OutContext),
-                                MCSymbolRefExpr::Create(GOTRef, OutContext),
+      MCBinaryExpr::createSub(MCSymbolRefExpr::create(GOTSymbol, OutContext),
+                                MCSymbolRefExpr::create(GOTRef, OutContext),
         OutContext);
-    OutStreamer.EmitLabel(GOTRef);
-    OutStreamer.EmitValue(OffsExpr, 4);
-    OutStreamer.EmitLabel(NextInstr);
-    EmitToStreamer(OutStreamer, MCInstBuilder(PPC::MFLR)
-                                .addReg(MI->getOperand(0).getReg()));
-    EmitToStreamer(OutStreamer, MCInstBuilder(PPC::LWZ)
-                                .addReg(MI->getOperand(1).getReg())
-                                .addImm(0)
-                                .addReg(MI->getOperand(0).getReg()));
-    EmitToStreamer(OutStreamer, MCInstBuilder(PPC::ADD4)
-                                .addReg(MI->getOperand(0).getReg())
-                                .addReg(MI->getOperand(1).getReg())
-                                .addReg(MI->getOperand(0).getReg()));
+    OutStreamer->EmitLabel(GOTRef);
+    OutStreamer->EmitValue(OffsExpr, 4);
+    OutStreamer->EmitLabel(NextInstr);
+    EmitToStreamer(*OutStreamer, MCInstBuilder(PPC::MFLR)
+                                 .addReg(MI->getOperand(0).getReg()));
+    EmitToStreamer(*OutStreamer, MCInstBuilder(PPC::LWZ)
+                                 .addReg(MI->getOperand(1).getReg())
+                                 .addImm(0)
+                                 .addReg(MI->getOperand(0).getReg()));
+    EmitToStreamer(*OutStreamer, MCInstBuilder(PPC::ADD4)
+                                 .addReg(MI->getOperand(0).getReg())
+                                 .addReg(MI->getOperand(1).getReg())
+                                 .addReg(MI->getOperand(0).getReg()));
     return;
   }
   case PPC::PPC32GOT: {
-    MCSymbol *GOTSymbol = OutContext.GetOrCreateSymbol(StringRef("_GLOBAL_OFFSET_TABLE_"));
+    MCSymbol *GOTSymbol = OutContext.getOrCreateSymbol(StringRef("_GLOBAL_OFFSET_TABLE_"));
     const MCExpr *SymGotTlsL =
-      MCSymbolRefExpr::Create(GOTSymbol, MCSymbolRefExpr::VK_PPC_LO,
+      MCSymbolRefExpr::create(GOTSymbol, MCSymbolRefExpr::VK_PPC_LO,
                               OutContext);
     const MCExpr *SymGotTlsHA =                               
-      MCSymbolRefExpr::Create(GOTSymbol, MCSymbolRefExpr::VK_PPC_HA,
+      MCSymbolRefExpr::create(GOTSymbol, MCSymbolRefExpr::VK_PPC_HA,
                               OutContext);
-    EmitToStreamer(OutStreamer, MCInstBuilder(PPC::LI)
-                                .addReg(MI->getOperand(0).getReg())
-                                .addExpr(SymGotTlsL));
-    EmitToStreamer(OutStreamer, MCInstBuilder(PPC::ADDIS)
-                                .addReg(MI->getOperand(0).getReg())
-                                .addReg(MI->getOperand(0).getReg())
-                                .addExpr(SymGotTlsHA));
+    EmitToStreamer(*OutStreamer, MCInstBuilder(PPC::LI)
+                                 .addReg(MI->getOperand(0).getReg())
+                                 .addExpr(SymGotTlsL));
+    EmitToStreamer(*OutStreamer, MCInstBuilder(PPC::ADDIS)
+                                 .addReg(MI->getOperand(0).getReg())
+                                 .addReg(MI->getOperand(0).getReg())
+                                 .addExpr(SymGotTlsHA));
     return;
   }
   case PPC::ADDIStlsgdHA: {
@@ -808,12 +864,12 @@ void PPCAsmPrinter::EmitInstruction(const MachineInstr *MI) {
     const GlobalValue *GValue = MO.getGlobal();
     MCSymbol *MOSymbol = getSymbol(GValue);
     const MCExpr *SymGotTlsGD =
-      MCSymbolRefExpr::Create(MOSymbol, MCSymbolRefExpr::VK_PPC_GOT_TLSGD_HA,
+      MCSymbolRefExpr::create(MOSymbol, MCSymbolRefExpr::VK_PPC_GOT_TLSGD_HA,
                               OutContext);
-    EmitToStreamer(OutStreamer, MCInstBuilder(PPC::ADDIS8)
-                                .addReg(MI->getOperand(0).getReg())
-                                .addReg(MI->getOperand(1).getReg())
-                                .addExpr(SymGotTlsGD));
+    EmitToStreamer(*OutStreamer, MCInstBuilder(PPC::ADDIS8)
+                                 .addReg(MI->getOperand(0).getReg())
+                                 .addReg(MI->getOperand(1).getReg())
+                                 .addExpr(SymGotTlsGD));
     return;
   }
   case PPC::ADDItlsgdL:
@@ -825,11 +881,11 @@ void PPCAsmPrinter::EmitInstruction(const MachineInstr *MI) {
     const MachineOperand &MO = MI->getOperand(2);
     const GlobalValue *GValue = MO.getGlobal();
     MCSymbol *MOSymbol = getSymbol(GValue);
-    const MCExpr *SymGotTlsGD = MCSymbolRefExpr::Create(
+    const MCExpr *SymGotTlsGD = MCSymbolRefExpr::create(
         MOSymbol, Subtarget->isPPC64() ? MCSymbolRefExpr::VK_PPC_GOT_TLSGD_LO
                                        : MCSymbolRefExpr::VK_PPC_GOT_TLSGD,
         OutContext);
-    EmitToStreamer(OutStreamer,
+    EmitToStreamer(*OutStreamer,
                    MCInstBuilder(Subtarget->isPPC64() ? PPC::ADDI8 : PPC::ADDI)
                    .addReg(MI->getOperand(0).getReg())
                    .addReg(MI->getOperand(1).getReg())
@@ -853,12 +909,12 @@ void PPCAsmPrinter::EmitInstruction(const MachineInstr *MI) {
     const GlobalValue *GValue = MO.getGlobal();
     MCSymbol *MOSymbol = getSymbol(GValue);
     const MCExpr *SymGotTlsLD =
-      MCSymbolRefExpr::Create(MOSymbol, MCSymbolRefExpr::VK_PPC_GOT_TLSLD_HA,
+      MCSymbolRefExpr::create(MOSymbol, MCSymbolRefExpr::VK_PPC_GOT_TLSLD_HA,
                               OutContext);
-    EmitToStreamer(OutStreamer, MCInstBuilder(PPC::ADDIS8)
-                                .addReg(MI->getOperand(0).getReg())
-                                .addReg(MI->getOperand(1).getReg())
-                                .addExpr(SymGotTlsLD));
+    EmitToStreamer(*OutStreamer, MCInstBuilder(PPC::ADDIS8)
+                                 .addReg(MI->getOperand(0).getReg())
+                                 .addReg(MI->getOperand(1).getReg())
+                                 .addExpr(SymGotTlsLD));
     return;
   }
   case PPC::ADDItlsldL:
@@ -870,11 +926,11 @@ void PPCAsmPrinter::EmitInstruction(const MachineInstr *MI) {
     const MachineOperand &MO = MI->getOperand(2);
     const GlobalValue *GValue = MO.getGlobal();
     MCSymbol *MOSymbol = getSymbol(GValue);
-    const MCExpr *SymGotTlsLD = MCSymbolRefExpr::Create(
+    const MCExpr *SymGotTlsLD = MCSymbolRefExpr::create(
         MOSymbol, Subtarget->isPPC64() ? MCSymbolRefExpr::VK_PPC_GOT_TLSLD_LO
                                        : MCSymbolRefExpr::VK_PPC_GOT_TLSLD,
         OutContext);
-    EmitToStreamer(OutStreamer,
+    EmitToStreamer(*OutStreamer,
                    MCInstBuilder(Subtarget->isPPC64() ? PPC::ADDI8 : PPC::ADDI)
                        .addReg(MI->getOperand(0).getReg())
                        .addReg(MI->getOperand(1).getReg())
@@ -900,10 +956,10 @@ void PPCAsmPrinter::EmitInstruction(const MachineInstr *MI) {
     const GlobalValue *GValue = MO.getGlobal();
     MCSymbol *MOSymbol = getSymbol(GValue);
     const MCExpr *SymDtprel =
-      MCSymbolRefExpr::Create(MOSymbol, MCSymbolRefExpr::VK_PPC_DTPREL_HA,
+      MCSymbolRefExpr::create(MOSymbol, MCSymbolRefExpr::VK_PPC_DTPREL_HA,
                               OutContext);
     EmitToStreamer(
-        OutStreamer,
+        *OutStreamer,
         MCInstBuilder(Subtarget->isPPC64() ? PPC::ADDIS8 : PPC::ADDIS)
             .addReg(MI->getOperand(0).getReg())
             .addReg(Subtarget->isPPC64() ? PPC::X3 : PPC::R3)
@@ -920,9 +976,9 @@ void PPCAsmPrinter::EmitInstruction(const MachineInstr *MI) {
     const GlobalValue *GValue = MO.getGlobal();
     MCSymbol *MOSymbol = getSymbol(GValue);
     const MCExpr *SymDtprel =
-      MCSymbolRefExpr::Create(MOSymbol, MCSymbolRefExpr::VK_PPC_DTPREL_LO,
+      MCSymbolRefExpr::create(MOSymbol, MCSymbolRefExpr::VK_PPC_DTPREL_LO,
                               OutContext);
-    EmitToStreamer(OutStreamer,
+    EmitToStreamer(*OutStreamer,
                    MCInstBuilder(Subtarget->isPPC64() ? PPC::ADDI8 : PPC::ADDI)
                        .addReg(MI->getOperand(0).getReg())
                        .addReg(MI->getOperand(1).getReg())
@@ -936,9 +992,9 @@ void PPCAsmPrinter::EmitInstruction(const MachineInstr *MI) {
       // Into:      %R3 = MFCR   ;; cr7
       unsigned NewOpcode =
         MI->getOpcode() == PPC::MFOCRF ? PPC::MFCR : PPC::MFCR8;
-      OutStreamer.AddComment(PPCInstPrinter::
-                             getRegisterName(MI->getOperand(1).getReg()));
-      EmitToStreamer(OutStreamer, MCInstBuilder(NewOpcode)
+      OutStreamer->AddComment(PPCInstPrinter::
+                              getRegisterName(MI->getOperand(1).getReg()));
+      EmitToStreamer(*OutStreamer, MCInstBuilder(NewOpcode)
                                   .addReg(MI->getOperand(0).getReg()));
       return;
     }
@@ -952,11 +1008,11 @@ void PPCAsmPrinter::EmitInstruction(const MachineInstr *MI) {
         MI->getOpcode() == PPC::MTOCRF ? PPC::MTCRF : PPC::MTCRF8;
       unsigned Mask = 0x80 >> OutContext.getRegisterInfo()
                               ->getEncodingValue(MI->getOperand(0).getReg());
-      OutStreamer.AddComment(PPCInstPrinter::
-                             getRegisterName(MI->getOperand(0).getReg()));
-      EmitToStreamer(OutStreamer, MCInstBuilder(NewOpcode)
-                                  .addImm(Mask)
-                                  .addReg(MI->getOperand(1).getReg()));
+      OutStreamer->AddComment(PPCInstPrinter::
+                              getRegisterName(MI->getOperand(0).getReg()));
+      EmitToStreamer(*OutStreamer, MCInstBuilder(NewOpcode)
+                                     .addImm(Mask)
+                                     .addReg(MI->getOperand(1).getReg()));
       return;
     }
     break;
@@ -982,13 +1038,13 @@ void PPCAsmPrinter::EmitInstruction(const MachineInstr *MI) {
   }
 
   LowerPPCMachineInstrToMCInst(MI, TmpInst, *this, isDarwin);
-  EmitToStreamer(OutStreamer, TmpInst);
+  EmitToStreamer(*OutStreamer, TmpInst);
 }
 
 void PPCLinuxAsmPrinter::EmitStartOfAsmFile(Module &M) {
   if (static_cast<const PPCTargetMachine &>(TM).isELFv2ABI()) {
     PPCTargetStreamer *TS =
-      static_cast<PPCTargetStreamer *>(OutStreamer.getTargetStreamer());
+      static_cast<PPCTargetStreamer *>(OutStreamer->getTargetStreamer());
 
     if (TS)
       TS->emitAbiVersion(2);
@@ -1001,24 +1057,24 @@ void PPCLinuxAsmPrinter::EmitStartOfAsmFile(Module &M) {
   if (M.getPICLevel() == PICLevel::Small)
     return AsmPrinter::EmitStartOfAsmFile(M);
 
-  OutStreamer.SwitchSection(OutContext.getELFSection(
+  OutStreamer->SwitchSection(OutContext.getELFSection(
       ".got2", ELF::SHT_PROGBITS, ELF::SHF_WRITE | ELF::SHF_ALLOC));
 
-  MCSymbol *TOCSym = OutContext.GetOrCreateSymbol(Twine(".LTOC"));
-  MCSymbol *CurrentPos = OutContext.CreateTempSymbol();
+  MCSymbol *TOCSym = OutContext.getOrCreateSymbol(Twine(".LTOC"));
+  MCSymbol *CurrentPos = OutContext.createTempSymbol();
 
-  OutStreamer.EmitLabel(CurrentPos);
+  OutStreamer->EmitLabel(CurrentPos);
 
   // The GOT pointer points to the middle of the GOT, in order to reference the
   // entire 64kB range.  0x8000 is the midpoint.
   const MCExpr *tocExpr =
-    MCBinaryExpr::CreateAdd(MCSymbolRefExpr::Create(CurrentPos, OutContext),
-                            MCConstantExpr::Create(0x8000, OutContext),
+    MCBinaryExpr::createAdd(MCSymbolRefExpr::create(CurrentPos, OutContext),
+                            MCConstantExpr::create(0x8000, OutContext),
                             OutContext);
 
-  OutStreamer.EmitAssignment(TOCSym, tocExpr);
+  OutStreamer->EmitAssignment(TOCSym, tocExpr);
 
-  OutStreamer.SwitchSection(getObjFileLowering().getTextSection());
+  OutStreamer->SwitchSection(getObjFileLowering().getTextSection());
 }
 
 void PPCLinuxAsmPrinter::EmitFunctionEntryLabel() {
@@ -1033,16 +1089,16 @@ void PPCLinuxAsmPrinter::EmitFunctionEntryLabel() {
   	if (PPCFI->usesPICBase()) {
       MCSymbol *RelocSymbol = PPCFI->getPICOffsetSymbol();
       MCSymbol *PICBase = MF->getPICBaseSymbol();
-      OutStreamer.EmitLabel(RelocSymbol);
+      OutStreamer->EmitLabel(RelocSymbol);
 
       const MCExpr *OffsExpr =
-        MCBinaryExpr::CreateSub(
-          MCSymbolRefExpr::Create(OutContext.GetOrCreateSymbol(Twine(".LTOC")),
+        MCBinaryExpr::createSub(
+          MCSymbolRefExpr::create(OutContext.getOrCreateSymbol(Twine(".LTOC")),
                                                                OutContext),
-                                  MCSymbolRefExpr::Create(PICBase, OutContext),
+                                  MCSymbolRefExpr::create(PICBase, OutContext),
           OutContext);
-      OutStreamer.EmitValue(OffsExpr, 4);
-      OutStreamer.EmitLabel(CurrentFnSym);
+      OutStreamer->EmitValue(OffsExpr, 4);
+      OutStreamer->EmitLabel(CurrentFnSym);
       return;
     } else
       return AsmPrinter::EmitFunctionEntryLabel();
@@ -1053,55 +1109,55 @@ void PPCLinuxAsmPrinter::EmitFunctionEntryLabel() {
     return AsmPrinter::EmitFunctionEntryLabel();
 
   // Emit an official procedure descriptor.
-  MCSectionSubPair Current = OutStreamer.getCurrentSection();
-  const MCSectionELF *Section = OutStreamer.getContext().getELFSection(
+  MCSectionSubPair Current = OutStreamer->getCurrentSection();
+  MCSectionELF *Section = OutStreamer->getContext().getELFSection(
       ".opd", ELF::SHT_PROGBITS, ELF::SHF_WRITE | ELF::SHF_ALLOC);
-  OutStreamer.SwitchSection(Section);
-  OutStreamer.EmitLabel(CurrentFnSym);
-  OutStreamer.EmitValueToAlignment(8);
+  OutStreamer->SwitchSection(Section);
+  OutStreamer->EmitLabel(CurrentFnSym);
+  OutStreamer->EmitValueToAlignment(8);
   MCSymbol *Symbol1 = CurrentFnSymForSize;
   // Generates a R_PPC64_ADDR64 (from FK_DATA_8) relocation for the function
   // entry point.
-  OutStreamer.EmitValue(MCSymbolRefExpr::Create(Symbol1, OutContext),
-			8 /*size*/);
-  MCSymbol *Symbol2 = OutContext.GetOrCreateSymbol(StringRef(".TOC."));
+  OutStreamer->EmitValue(MCSymbolRefExpr::create(Symbol1, OutContext),
+                         8 /*size*/);
+  MCSymbol *Symbol2 = OutContext.getOrCreateSymbol(StringRef(".TOC."));
   // Generates a R_PPC64_TOC relocation for TOC base insertion.
-  OutStreamer.EmitValue(MCSymbolRefExpr::Create(Symbol2,
-                        MCSymbolRefExpr::VK_PPC_TOCBASE, OutContext),
-                        8/*size*/);
+  OutStreamer->EmitValue(
+    MCSymbolRefExpr::create(Symbol2, MCSymbolRefExpr::VK_PPC_TOCBASE, OutContext),
+    8/*size*/);
   // Emit a null environment pointer.
-  OutStreamer.EmitIntValue(0, 8 /* size */);
-  OutStreamer.SwitchSection(Current.first, Current.second);
+  OutStreamer->EmitIntValue(0, 8 /* size */);
+  OutStreamer->SwitchSection(Current.first, Current.second);
 }
 
 
 bool PPCLinuxAsmPrinter::doFinalization(Module &M) {
-  const DataLayout *TD = TM.getDataLayout();
+  const DataLayout &DL = getDataLayout();
 
-  bool isPPC64 = TD->getPointerSizeInBits() == 64;
+  bool isPPC64 = DL.getPointerSizeInBits() == 64;
 
   PPCTargetStreamer &TS =
-      static_cast<PPCTargetStreamer &>(*OutStreamer.getTargetStreamer());
+      static_cast<PPCTargetStreamer &>(*OutStreamer->getTargetStreamer());
 
   if (!TOC.empty()) {
-    const MCSectionELF *Section;
-    
+    MCSectionELF *Section;
+
     if (isPPC64)
-      Section = OutStreamer.getContext().getELFSection(
+      Section = OutStreamer->getContext().getELFSection(
           ".toc", ELF::SHT_PROGBITS, ELF::SHF_WRITE | ELF::SHF_ALLOC);
         else
-          Section = OutStreamer.getContext().getELFSection(
+          Section = OutStreamer->getContext().getELFSection(
               ".got2", ELF::SHT_PROGBITS, ELF::SHF_WRITE | ELF::SHF_ALLOC);
-    OutStreamer.SwitchSection(Section);
+    OutStreamer->SwitchSection(Section);
 
     for (MapVector<MCSymbol*, MCSymbol*>::iterator I = TOC.begin(),
          E = TOC.end(); I != E; ++I) {
-      OutStreamer.EmitLabel(I->second);
+      OutStreamer->EmitLabel(I->second);
       MCSymbol *S = I->first;
       if (isPPC64)
         TS.emitTCEntry(*S);
       else
-        OutStreamer.EmitSymbolValue(S, 4);
+        OutStreamer->EmitSymbolValue(S, 4);
     }
   }
 
@@ -1130,43 +1186,43 @@ void PPCLinuxAsmPrinter::EmitFunctionBodyStart() {
       // Only do all that if the function uses r2 in the first place.
       && !MF->getRegInfo().use_empty(PPC::X2)) {
 
-    MCSymbol *GlobalEntryLabel = OutContext.CreateTempSymbol();
-    OutStreamer.EmitLabel(GlobalEntryLabel);
+    MCSymbol *GlobalEntryLabel = OutContext.createTempSymbol();
+    OutStreamer->EmitLabel(GlobalEntryLabel);
     const MCSymbolRefExpr *GlobalEntryLabelExp =
-      MCSymbolRefExpr::Create(GlobalEntryLabel, OutContext);
+      MCSymbolRefExpr::create(GlobalEntryLabel, OutContext);
 
-    MCSymbol *TOCSymbol = OutContext.GetOrCreateSymbol(StringRef(".TOC."));
+    MCSymbol *TOCSymbol = OutContext.getOrCreateSymbol(StringRef(".TOC."));
     const MCExpr *TOCDeltaExpr =
-      MCBinaryExpr::CreateSub(MCSymbolRefExpr::Create(TOCSymbol, OutContext),
+      MCBinaryExpr::createSub(MCSymbolRefExpr::create(TOCSymbol, OutContext),
                               GlobalEntryLabelExp, OutContext);
 
     const MCExpr *TOCDeltaHi =
-      PPCMCExpr::CreateHa(TOCDeltaExpr, false, OutContext);
-    EmitToStreamer(OutStreamer, MCInstBuilder(PPC::ADDIS)
-                                .addReg(PPC::X2)
-                                .addReg(PPC::X12)
-                                .addExpr(TOCDeltaHi));
+      PPCMCExpr::createHa(TOCDeltaExpr, false, OutContext);
+    EmitToStreamer(*OutStreamer, MCInstBuilder(PPC::ADDIS)
+                                 .addReg(PPC::X2)
+                                 .addReg(PPC::X12)
+                                 .addExpr(TOCDeltaHi));
 
     const MCExpr *TOCDeltaLo =
-      PPCMCExpr::CreateLo(TOCDeltaExpr, false, OutContext);
-    EmitToStreamer(OutStreamer, MCInstBuilder(PPC::ADDI)
-                                .addReg(PPC::X2)
-                                .addReg(PPC::X2)
-                                .addExpr(TOCDeltaLo));
-
-    MCSymbol *LocalEntryLabel = OutContext.CreateTempSymbol();
-    OutStreamer.EmitLabel(LocalEntryLabel);
+      PPCMCExpr::createLo(TOCDeltaExpr, false, OutContext);
+    EmitToStreamer(*OutStreamer, MCInstBuilder(PPC::ADDI)
+                                 .addReg(PPC::X2)
+                                 .addReg(PPC::X2)
+                                 .addExpr(TOCDeltaLo));
+
+    MCSymbol *LocalEntryLabel = OutContext.createTempSymbol();
+    OutStreamer->EmitLabel(LocalEntryLabel);
     const MCSymbolRefExpr *LocalEntryLabelExp =
-       MCSymbolRefExpr::Create(LocalEntryLabel, OutContext);
+       MCSymbolRefExpr::create(LocalEntryLabel, OutContext);
     const MCExpr *LocalOffsetExp =
-      MCBinaryExpr::CreateSub(LocalEntryLabelExp,
+      MCBinaryExpr::createSub(LocalEntryLabelExp,
                               GlobalEntryLabelExp, OutContext);
 
     PPCTargetStreamer *TS =
-      static_cast<PPCTargetStreamer *>(OutStreamer.getTargetStreamer());
+      static_cast<PPCTargetStreamer *>(OutStreamer->getTargetStreamer());
 
     if (TS)
-      TS->emitLocalEntry(CurrentFnSym, LocalOffsetExp);
+      TS->emitLocalEntry(cast<MCSymbolELF>(CurrentFnSym), LocalOffsetExp);
   }
 }
 
@@ -1182,8 +1238,8 @@ void PPCLinuxAsmPrinter::EmitFunctionBodyEnd() {
   // the PPC64 ELF ABI (this is a low-priority item because GDB does not
   // currently make use of these fields).
   if (Subtarget->isPPC64()) {
-    OutStreamer.EmitIntValue(0, 4/*size*/);
-    OutStreamer.EmitIntValue(0, 8/*size*/);
+    OutStreamer->EmitIntValue(0, 4/*size*/);
+    OutStreamer->EmitIntValue(0, 8/*size*/);
   }
 }
 
@@ -1232,51 +1288,52 @@ void PPCDarwinAsmPrinter::EmitStartOfAsmFile(Module &M) {
   assert(Directive < array_lengthof(CPUDirectives) &&
          "CPUDirectives[] might not be up-to-date!");
   PPCTargetStreamer &TStreamer =
-      *static_cast<PPCTargetStreamer *>(OutStreamer.getTargetStreamer());
+      *static_cast<PPCTargetStreamer *>(OutStreamer->getTargetStreamer());
   TStreamer.emitMachine(CPUDirectives[Directive]);
 
   // Prime text sections so they are adjacent.  This reduces the likelihood a
   // large data or debug section causes a branch to exceed 16M limit.
   const TargetLoweringObjectFileMachO &TLOFMacho = 
     static_cast<const TargetLoweringObjectFileMachO &>(getObjFileLowering());
-  OutStreamer.SwitchSection(TLOFMacho.getTextCoalSection());
+  OutStreamer->SwitchSection(TLOFMacho.getTextCoalSection());
   if (TM.getRelocationModel() == Reloc::PIC_) {
-    OutStreamer.SwitchSection(
+    OutStreamer->SwitchSection(
            OutContext.getMachOSection("__TEXT", "__picsymbolstub1",
                                       MachO::S_SYMBOL_STUBS |
                                       MachO::S_ATTR_PURE_INSTRUCTIONS,
                                       32, SectionKind::getText()));
   } else if (TM.getRelocationModel() == Reloc::DynamicNoPIC) {
-    OutStreamer.SwitchSection(
+    OutStreamer->SwitchSection(
            OutContext.getMachOSection("__TEXT","__symbol_stub1",
                                       MachO::S_SYMBOL_STUBS |
                                       MachO::S_ATTR_PURE_INSTRUCTIONS,
                                       16, SectionKind::getText()));
   }
-  OutStreamer.SwitchSection(getObjFileLowering().getTextSection());
+  OutStreamer->SwitchSection(getObjFileLowering().getTextSection());
 }
 
 static MCSymbol *GetLazyPtr(MCSymbol *Sym, MCContext &Ctx) {
   // Remove $stub suffix, add $lazy_ptr.
   StringRef NoStub = Sym->getName().substr(0, Sym->getName().size()-5);
-  return Ctx.GetOrCreateSymbol(NoStub + "$lazy_ptr");
+  return Ctx.getOrCreateSymbol(NoStub + "$lazy_ptr");
 }
 
 static MCSymbol *GetAnonSym(MCSymbol *Sym, MCContext &Ctx) {
   // Add $tmp suffix to $stub, yielding $stub$tmp.
-  return Ctx.GetOrCreateSymbol(Sym->getName() + "$tmp");
+  return Ctx.getOrCreateSymbol(Sym->getName() + "$tmp");
 }
 
 void PPCDarwinAsmPrinter::
 EmitFunctionStubs(const MachineModuleInfoMachO::SymbolListTy &Stubs) {
-  bool isPPC64 = TM.getDataLayout()->getPointerSizeInBits() == 64;
+  bool isPPC64 = getDataLayout().getPointerSizeInBits() == 64;
 
   // Construct a local MCSubtargetInfo and shadow EmitToStreamer here.
   // This is because the MachineFunction won't exist (but have not yet been
   // freed) and since we're at the global level we can use the default
   // constructed subtarget.
   std::unique_ptr<MCSubtargetInfo> STI(TM.getTarget().createMCSubtargetInfo(
-      TM.getTargetTriple(), TM.getTargetCPU(), TM.getTargetFeatureString()));
+      TM.getTargetTriple().str(), TM.getTargetCPU(),
+      TM.getTargetFeatureString()));
   auto EmitToStreamer = [&STI] (MCStreamer &S, const MCInst &Inst) {
     S.EmitInstruction(Inst, *STI);
   };
@@ -1285,17 +1342,16 @@ EmitFunctionStubs(const MachineModuleInfoMachO::SymbolListTy &Stubs) {
     static_cast<const TargetLoweringObjectFileMachO &>(getObjFileLowering());
 
   // .lazy_symbol_pointer
-  const MCSection *LSPSection = TLOFMacho.getLazySymbolPointerSection();
-  
+  MCSection *LSPSection = TLOFMacho.getLazySymbolPointerSection();
+
   // Output stubs for dynamically-linked functions
   if (TM.getRelocationModel() == Reloc::PIC_) {
-    const MCSection *StubSection = 
-    OutContext.getMachOSection("__TEXT", "__picsymbolstub1",
-                               MachO::S_SYMBOL_STUBS |
-                               MachO::S_ATTR_PURE_INSTRUCTIONS,
-                               32, SectionKind::getText());
+    MCSection *StubSection = OutContext.getMachOSection(
+        "__TEXT", "__picsymbolstub1",
+        MachO::S_SYMBOL_STUBS | MachO::S_ATTR_PURE_INSTRUCTIONS, 32,
+        SectionKind::getText());
     for (unsigned i = 0, e = Stubs.size(); i != e; ++i) {
-      OutStreamer.SwitchSection(StubSection);
+      OutStreamer->SwitchSection(StubSection);
       EmitAlignment(4);
       
       MCSymbol *Stub = Stubs[i].first;
@@ -1303,118 +1359,117 @@ EmitFunctionStubs(const MachineModuleInfoMachO::SymbolListTy &Stubs) {
       MCSymbol *LazyPtr = GetLazyPtr(Stub, OutContext);
       MCSymbol *AnonSymbol = GetAnonSym(Stub, OutContext);
                                            
-      OutStreamer.EmitLabel(Stub);
-      OutStreamer.EmitSymbolAttribute(RawSym, MCSA_IndirectSymbol);
+      OutStreamer->EmitLabel(Stub);
+      OutStreamer->EmitSymbolAttribute(RawSym, MCSA_IndirectSymbol);
 
-      const MCExpr *Anon = MCSymbolRefExpr::Create(AnonSymbol, OutContext);
-      const MCExpr *LazyPtrExpr = MCSymbolRefExpr::Create(LazyPtr, OutContext);
+      const MCExpr *Anon = MCSymbolRefExpr::create(AnonSymbol, OutContext);
+      const MCExpr *LazyPtrExpr = MCSymbolRefExpr::create(LazyPtr, OutContext);
       const MCExpr *Sub =
-        MCBinaryExpr::CreateSub(LazyPtrExpr, Anon, OutContext);
+        MCBinaryExpr::createSub(LazyPtrExpr, Anon, OutContext);
 
       // mflr r0
-      EmitToStreamer(OutStreamer, MCInstBuilder(PPC::MFLR).addReg(PPC::R0));
+      EmitToStreamer(*OutStreamer, MCInstBuilder(PPC::MFLR).addReg(PPC::R0));
       // bcl 20, 31, AnonSymbol
-      EmitToStreamer(OutStreamer, MCInstBuilder(PPC::BCLalways).addExpr(Anon));
-      OutStreamer.EmitLabel(AnonSymbol);
+      EmitToStreamer(*OutStreamer, MCInstBuilder(PPC::BCLalways).addExpr(Anon));
+      OutStreamer->EmitLabel(AnonSymbol);
       // mflr r11
-      EmitToStreamer(OutStreamer, MCInstBuilder(PPC::MFLR).addReg(PPC::R11));
+      EmitToStreamer(*OutStreamer, MCInstBuilder(PPC::MFLR).addReg(PPC::R11));
       // addis r11, r11, ha16(LazyPtr - AnonSymbol)
-      const MCExpr *SubHa16 = PPCMCExpr::CreateHa(Sub, true, OutContext);
-      EmitToStreamer(OutStreamer, MCInstBuilder(PPC::ADDIS)
+      const MCExpr *SubHa16 = PPCMCExpr::createHa(Sub, true, OutContext);
+      EmitToStreamer(*OutStreamer, MCInstBuilder(PPC::ADDIS)
         .addReg(PPC::R11)
         .addReg(PPC::R11)
         .addExpr(SubHa16));
       // mtlr r0
-      EmitToStreamer(OutStreamer, MCInstBuilder(PPC::MTLR).addReg(PPC::R0));
+      EmitToStreamer(*OutStreamer, MCInstBuilder(PPC::MTLR).addReg(PPC::R0));
 
       // ldu r12, lo16(LazyPtr - AnonSymbol)(r11)
       // lwzu r12, lo16(LazyPtr - AnonSymbol)(r11)
-      const MCExpr *SubLo16 = PPCMCExpr::CreateLo(Sub, true, OutContext);
-      EmitToStreamer(OutStreamer, MCInstBuilder(isPPC64 ? PPC::LDU : PPC::LWZU)
+      const MCExpr *SubLo16 = PPCMCExpr::createLo(Sub, true, OutContext);
+      EmitToStreamer(*OutStreamer, MCInstBuilder(isPPC64 ? PPC::LDU : PPC::LWZU)
         .addReg(PPC::R12)
         .addExpr(SubLo16).addExpr(SubLo16)
         .addReg(PPC::R11));
       // mtctr r12
-      EmitToStreamer(OutStreamer, MCInstBuilder(PPC::MTCTR).addReg(PPC::R12));
+      EmitToStreamer(*OutStreamer, MCInstBuilder(PPC::MTCTR).addReg(PPC::R12));
       // bctr
-      EmitToStreamer(OutStreamer, MCInstBuilder(PPC::BCTR));
+      EmitToStreamer(*OutStreamer, MCInstBuilder(PPC::BCTR));
 
-      OutStreamer.SwitchSection(LSPSection);
-      OutStreamer.EmitLabel(LazyPtr);
-      OutStreamer.EmitSymbolAttribute(RawSym, MCSA_IndirectSymbol);
+      OutStreamer->SwitchSection(LSPSection);
+      OutStreamer->EmitLabel(LazyPtr);
+      OutStreamer->EmitSymbolAttribute(RawSym, MCSA_IndirectSymbol);
 
       MCSymbol *DyldStubBindingHelper =
-        OutContext.GetOrCreateSymbol(StringRef("dyld_stub_binding_helper"));
+        OutContext.getOrCreateSymbol(StringRef("dyld_stub_binding_helper"));
       if (isPPC64) {
         // .quad dyld_stub_binding_helper
-        OutStreamer.EmitSymbolValue(DyldStubBindingHelper, 8);
+        OutStreamer->EmitSymbolValue(DyldStubBindingHelper, 8);
       } else {
         // .long dyld_stub_binding_helper
-        OutStreamer.EmitSymbolValue(DyldStubBindingHelper, 4);
+        OutStreamer->EmitSymbolValue(DyldStubBindingHelper, 4);
       }
     }
-    OutStreamer.AddBlankLine();
+    OutStreamer->AddBlankLine();
     return;
   }
-  
-  const MCSection *StubSection =
-    OutContext.getMachOSection("__TEXT","__symbol_stub1",
-                               MachO::S_SYMBOL_STUBS |
-                               MachO::S_ATTR_PURE_INSTRUCTIONS,
-                               16, SectionKind::getText());
+
+  MCSection *StubSection = OutContext.getMachOSection(
+      "__TEXT", "__symbol_stub1",
+      MachO::S_SYMBOL_STUBS | MachO::S_ATTR_PURE_INSTRUCTIONS, 16,
+      SectionKind::getText());
   for (unsigned i = 0, e = Stubs.size(); i != e; ++i) {
     MCSymbol *Stub = Stubs[i].first;
     MCSymbol *RawSym = Stubs[i].second.getPointer();
     MCSymbol *LazyPtr = GetLazyPtr(Stub, OutContext);
-    const MCExpr *LazyPtrExpr = MCSymbolRefExpr::Create(LazyPtr, OutContext);
+    const MCExpr *LazyPtrExpr = MCSymbolRefExpr::create(LazyPtr, OutContext);
 
-    OutStreamer.SwitchSection(StubSection);
+    OutStreamer->SwitchSection(StubSection);
     EmitAlignment(4);
-    OutStreamer.EmitLabel(Stub);
-    OutStreamer.EmitSymbolAttribute(RawSym, MCSA_IndirectSymbol);
+    OutStreamer->EmitLabel(Stub);
+    OutStreamer->EmitSymbolAttribute(RawSym, MCSA_IndirectSymbol);
 
     // lis r11, ha16(LazyPtr)
     const MCExpr *LazyPtrHa16 =
-      PPCMCExpr::CreateHa(LazyPtrExpr, true, OutContext);
-    EmitToStreamer(OutStreamer, MCInstBuilder(PPC::LIS)
+      PPCMCExpr::createHa(LazyPtrExpr, true, OutContext);
+    EmitToStreamer(*OutStreamer, MCInstBuilder(PPC::LIS)
       .addReg(PPC::R11)
       .addExpr(LazyPtrHa16));
 
     // ldu r12, lo16(LazyPtr)(r11)
     // lwzu r12, lo16(LazyPtr)(r11)
     const MCExpr *LazyPtrLo16 =
-      PPCMCExpr::CreateLo(LazyPtrExpr, true, OutContext);
-    EmitToStreamer(OutStreamer, MCInstBuilder(isPPC64 ? PPC::LDU : PPC::LWZU)
+      PPCMCExpr::createLo(LazyPtrExpr, true, OutContext);
+    EmitToStreamer(*OutStreamer, MCInstBuilder(isPPC64 ? PPC::LDU : PPC::LWZU)
       .addReg(PPC::R12)
       .addExpr(LazyPtrLo16).addExpr(LazyPtrLo16)
       .addReg(PPC::R11));
 
     // mtctr r12
-    EmitToStreamer(OutStreamer, MCInstBuilder(PPC::MTCTR).addReg(PPC::R12));
+    EmitToStreamer(*OutStreamer, MCInstBuilder(PPC::MTCTR).addReg(PPC::R12));
     // bctr
-    EmitToStreamer(OutStreamer, MCInstBuilder(PPC::BCTR));
+    EmitToStreamer(*OutStreamer, MCInstBuilder(PPC::BCTR));
 
-    OutStreamer.SwitchSection(LSPSection);
-    OutStreamer.EmitLabel(LazyPtr);
-    OutStreamer.EmitSymbolAttribute(RawSym, MCSA_IndirectSymbol);
+    OutStreamer->SwitchSection(LSPSection);
+    OutStreamer->EmitLabel(LazyPtr);
+    OutStreamer->EmitSymbolAttribute(RawSym, MCSA_IndirectSymbol);
 
     MCSymbol *DyldStubBindingHelper =
-      OutContext.GetOrCreateSymbol(StringRef("dyld_stub_binding_helper"));
+      OutContext.getOrCreateSymbol(StringRef("dyld_stub_binding_helper"));
     if (isPPC64) {
       // .quad dyld_stub_binding_helper
-      OutStreamer.EmitSymbolValue(DyldStubBindingHelper, 8);
+      OutStreamer->EmitSymbolValue(DyldStubBindingHelper, 8);
     } else {
       // .long dyld_stub_binding_helper
-      OutStreamer.EmitSymbolValue(DyldStubBindingHelper, 4);
+      OutStreamer->EmitSymbolValue(DyldStubBindingHelper, 4);
     }
   }
   
-  OutStreamer.AddBlankLine();
+  OutStreamer->AddBlankLine();
 }
 
 
 bool PPCDarwinAsmPrinter::doFinalization(Module &M) {
-  bool isPPC64 = TM.getDataLayout()->getPointerSizeInBits() == 64;
+  bool isPPC64 = getDataLayout().getPointerSizeInBits() == 64;
 
   // Darwin/PPC always uses mach-o.
   const TargetLoweringObjectFileMachO &TLOFMacho = 
@@ -1447,19 +1502,19 @@ bool PPCDarwinAsmPrinter::doFinalization(Module &M) {
   // Output macho stubs for external and common global variables.
   if (!Stubs.empty()) {
     // Switch with ".non_lazy_symbol_pointer" directive.
-    OutStreamer.SwitchSection(TLOFMacho.getNonLazySymbolPointerSection());
+    OutStreamer->SwitchSection(TLOFMacho.getNonLazySymbolPointerSection());
     EmitAlignment(isPPC64 ? 3 : 2);
     
     for (unsigned i = 0, e = Stubs.size(); i != e; ++i) {
       // L_foo$stub:
-      OutStreamer.EmitLabel(Stubs[i].first);
+      OutStreamer->EmitLabel(Stubs[i].first);
       //   .indirect_symbol _foo
       MachineModuleInfoImpl::StubValueTy &MCSym = Stubs[i].second;
-      OutStreamer.EmitSymbolAttribute(MCSym.getPointer(), MCSA_IndirectSymbol);
+      OutStreamer->EmitSymbolAttribute(MCSym.getPointer(), MCSA_IndirectSymbol);
 
       if (MCSym.getInt())
         // External to current translation unit.
-        OutStreamer.EmitIntValue(0, isPPC64 ? 8 : 4/*size*/);
+        OutStreamer->EmitIntValue(0, isPPC64 ? 8 : 4/*size*/);
       else
         // Internal to current translation unit.
         //
@@ -1467,32 +1522,32 @@ bool PPCDarwinAsmPrinter::doFinalization(Module &M) {
         // need to be indirect and pc-rel. We accomplish this by using NLPs.
         // However, sometimes the types are local to the file. So we need to
         // fill in the value for the NLP in those cases.
-        OutStreamer.EmitValue(MCSymbolRefExpr::Create(MCSym.getPointer(),
-                                                      OutContext),
+        OutStreamer->EmitValue(MCSymbolRefExpr::create(MCSym.getPointer(),
+                                                       OutContext),
                               isPPC64 ? 8 : 4/*size*/);
     }
 
     Stubs.clear();
-    OutStreamer.AddBlankLine();
+    OutStreamer->AddBlankLine();
   }
 
   Stubs = MMIMacho.GetHiddenGVStubList();
   if (!Stubs.empty()) {
-    OutStreamer.SwitchSection(getObjFileLowering().getDataSection());
+    OutStreamer->SwitchSection(getObjFileLowering().getDataSection());
     EmitAlignment(isPPC64 ? 3 : 2);
     
     for (unsigned i = 0, e = Stubs.size(); i != e; ++i) {
       // L_foo$stub:
-      OutStreamer.EmitLabel(Stubs[i].first);
+      OutStreamer->EmitLabel(Stubs[i].first);
       //   .long _foo
-      OutStreamer.EmitValue(MCSymbolRefExpr::
-                            Create(Stubs[i].second.getPointer(),
-                                   OutContext),
-                            isPPC64 ? 8 : 4/*size*/);
+      OutStreamer->EmitValue(MCSymbolRefExpr::
+                             create(Stubs[i].second.getPointer(),
+                                    OutContext),
+                             isPPC64 ? 8 : 4/*size*/);
     }
 
     Stubs.clear();
-    OutStreamer.AddBlankLine();
+    OutStreamer->AddBlankLine();
   }
 
   // Funny Darwin hack: This flag tells the linker that no global symbols
@@ -1500,7 +1555,7 @@ bool PPCDarwinAsmPrinter::doFinalization(Module &M) {
   // implementation of multiple entry points).  If this doesn't occur, the
   // linker can safely perform dead code stripping.  Since LLVM never generates
   // code that does this, it is always safe to set.
-  OutStreamer.EmitAssemblerFlag(MCAF_SubsectionsViaSymbols);
+  OutStreamer->EmitAssemblerFlag(MCAF_SubsectionsViaSymbols);
 
   return AsmPrinter::doFinalization(M);
 }
@@ -1512,7 +1567,7 @@ bool PPCDarwinAsmPrinter::doFinalization(Module &M) {
 static AsmPrinter *
 createPPCAsmPrinterPass(TargetMachine &tm,
                         std::unique_ptr<MCStreamer> &&Streamer) {
-  if (Triple(tm.getTargetTriple()).isMacOSX())
+  if (tm.getTargetTriple().isMacOSX())
     return new PPCDarwinAsmPrinter(tm, std::move(Streamer));
   return new PPCLinuxAsmPrinter(tm, std::move(Streamer));
 }
diff --git a/lib/Target/PowerPC/PPCCTRLoops.cpp b/lib/Target/PowerPC/PPCCTRLoops.cpp
index c595f44388e..baadf081a64 100644
--- a/lib/Target/PowerPC/PPCCTRLoops.cpp
+++ b/lib/Target/PowerPC/PPCCTRLoops.cpp
@@ -351,8 +351,9 @@ bool PPCCTRLoops::mightUseCTR(const Triple &TT, BasicBlock *BB) {
             Opcode = ISD::FTRUNC; break;
           }
 
-          MVT VTy =
-            TLI->getSimpleValueType(CI->getArgOperand(0)->getType(), true);
+          auto &DL = CI->getModule()->getDataLayout();
+          MVT VTy = TLI->getSimpleValueType(DL, CI->getArgOperand(0)->getType(),
+                                            true);
           if (VTy == MVT::Other)
             return true;
           
@@ -417,8 +418,8 @@ bool PPCCTRLoops::mightUseCTR(const Triple &TT, BasicBlock *BB) {
 bool PPCCTRLoops::convertToCTRLoop(Loop *L) {
   bool MadeChange = false;
 
-  Triple TT = Triple(L->getHeader()->getParent()->getParent()->
-                     getTargetTriple());
+  const Triple TT =
+      Triple(L->getHeader()->getParent()->getParent()->getTargetTriple());
   if (!TT.isArch32Bit() && !TT.isArch64Bit())
     return MadeChange; // Unknown arch. type.
 
@@ -553,7 +554,7 @@ bool PPCCTRLoops::convertToCTRLoop(Loop *L) {
   IRBuilder<> CondBuilder(CountedExitBranch);
   Value *DecFunc =
     Intrinsic::getDeclaration(M, Intrinsic::ppc_is_decremented_ctr_nonzero);
-  Value *NewCond = CondBuilder.CreateCall(DecFunc);
+  Value *NewCond = CondBuilder.CreateCall(DecFunc, {});
   Value *OldCond = CountedExitBranch->getCondition();
   CountedExitBranch->setCondition(NewCond);
 
diff --git a/lib/Target/PowerPC/PPCCallingConv.td b/lib/Target/PowerPC/PPCCallingConv.td
index 045fca3c747..5bc9124f808 100644
--- a/lib/Target/PowerPC/PPCCallingConv.td
+++ b/lib/Target/PowerPC/PPCCallingConv.td
@@ -62,7 +62,8 @@ def RetCC_PPC : CallingConv<[
  
   // Vector types returned as "direct" go into V2 .. V9; note that only the
   // ELFv2 ABI fully utilizes all these registers.
-  CCIfType<[v16i8, v8i16, v4i32, v4f32], CCIfSubtarget<"hasAltivec()",
+  CCIfType<[v16i8, v8i16, v4i32, v2i64, v1i128, v4f32], 
+           CCIfSubtarget<"hasAltivec()",
            CCAssignToReg<[V2, V3, V4, V5, V6, V7, V8, V9]>>>,
   CCIfType<[v2f64, v2i64], CCIfSubtarget<"hasVSX()",
            CCAssignToReg<[VSH2, VSH3, VSH4, VSH5, VSH6, VSH7, VSH8, VSH9]>>>
@@ -114,7 +115,8 @@ def RetCC_PPC64_ELF_FIS : CallingConv<[
   CCIfType<[f64],  CCAssignToReg<[F1, F2, F3, F4, F5, F6, F7, F8]>>,
   CCIfType<[v4f64, v4f32, v4i1],
            CCIfSubtarget<"hasQPX()", CCAssignToReg<[QF1, QF2]>>>,
-  CCIfType<[v16i8, v8i16, v4i32, v4f32], CCIfSubtarget<"hasAltivec()",
+  CCIfType<[v16i8, v8i16, v4i32, v2i64, v1i128, v4f32], 
+           CCIfSubtarget<"hasAltivec()",
            CCAssignToReg<[V2, V3, V4, V5, V6, V7, V8, V9]>>>,
   CCIfType<[v2f64, v2i64], CCIfSubtarget<"hasVSX()",
            CCAssignToReg<[VSH2, VSH3, VSH4, VSH5, VSH6, VSH7, VSH8, VSH9]>>>
@@ -131,6 +133,9 @@ def CC_PPC32_SVR4_Common : CallingConv<[
   // register having an odd register number.
   CCIfType<[i32], CCIfSplit<CCCustom<"CC_PPC32_SVR4_Custom_AlignArgRegs">>>,
 
+  // The 'nest' parameter, if any, is passed in R11.
+  CCIfNest<CCAssignToReg<[R11]>>,
+
   // The first 8 integer arguments are passed in integer registers.
   CCIfType<[i32], CCAssignToReg<[R3, R4, R5, R6, R7, R8, R9, R10]>>,
 
@@ -172,9 +177,9 @@ def CC_PPC32_SVR4 : CallingConv<[
     CCAssignToReg<[QF1, QF2, QF3, QF4, QF5, QF6, QF7, QF8]>>>,
 
   // The first 12 Vector arguments are passed in AltiVec registers.
-  CCIfType<[v16i8, v8i16, v4i32, v4f32], CCIfSubtarget<"hasAltivec()",
-           CCAssignToReg<[V2, V3, V4, V5, V6, V7, V8, V9,
-                          V10, V11, V12, V13]>>>,
+  CCIfType<[v16i8, v8i16, v4i32, v2i64, v1i128, v4f32], 
+           CCIfSubtarget<"hasAltivec()", CCAssignToReg<[V2, V3, V4, V5, V6, V7,
+                          V8, V9, V10, V11, V12, V13]>>>,
   CCIfType<[v2f64, v2i64], CCIfSubtarget<"hasVSX()",
            CCAssignToReg<[VSH2, VSH3, VSH4, VSH5, VSH6, VSH7, VSH8, VSH9,
                           VSH10, VSH11, VSH12, VSH13]>>>,
diff --git a/lib/Target/PowerPC/PPCEarlyReturn.cpp b/lib/Target/PowerPC/PPCEarlyReturn.cpp
index 08673cc093d..fc89753ed94 100644
--- a/lib/Target/PowerPC/PPCEarlyReturn.cpp
+++ b/lib/Target/PowerPC/PPCEarlyReturn.cpp
@@ -151,12 +151,13 @@ namespace {
       if (Changed && !ReturnMBB.hasAddressTaken()) {
         // We now might be able to merge this blr-only block into its
         // by-layout predecessor.
-        if (ReturnMBB.pred_size() == 1 &&
-            (*ReturnMBB.pred_begin())->isLayoutSuccessor(&ReturnMBB)) {
-          // Move the blr into the preceding block.
+        if (ReturnMBB.pred_size() == 1) {
           MachineBasicBlock &PrevMBB = **ReturnMBB.pred_begin();
-          PrevMBB.splice(PrevMBB.end(), &ReturnMBB, I);
-          PrevMBB.removeSuccessor(&ReturnMBB);
+          if (PrevMBB.isLayoutSuccessor(&ReturnMBB) && PrevMBB.canFallThrough()) {
+            // Move the blr into the preceding block.
+            PrevMBB.splice(PrevMBB.end(), &ReturnMBB, I);
+            PrevMBB.removeSuccessor(&ReturnMBB);
+          }
         }
 
         if (ReturnMBB.pred_empty())
diff --git a/lib/Target/PowerPC/PPCFastISel.cpp b/lib/Target/PowerPC/PPCFastISel.cpp
index 002616babd4..700df20f8aa 100644
--- a/lib/Target/PowerPC/PPCFastISel.cpp
+++ b/lib/Target/PowerPC/PPCFastISel.cpp
@@ -144,9 +144,13 @@ class PPCFastISel final : public FastISel {
   private:
     bool isTypeLegal(Type *Ty, MVT &VT);
     bool isLoadTypeLegal(Type *Ty, MVT &VT);
+    bool isValueAvailable(const Value *V) const;
     bool isVSFRCRegister(unsigned Register) const {
       return MRI.getRegClass(Register)->getID() == PPC::VSFRCRegClassID;
     }
+    bool isVSSRCRegister(unsigned Register) const {
+      return MRI.getRegClass(Register)->getID() == PPC::VSSRCRegClassID;
+    }
     bool PPCEmitCmp(const Value *Src1Value, const Value *Src2Value,
                     bool isZExt, unsigned DestReg);
     bool PPCEmitLoad(MVT VT, unsigned &ResultReg, Address &Addr,
@@ -160,7 +164,8 @@ class PPCFastISel final : public FastISel {
                            unsigned DestReg, bool IsZExt);
     unsigned PPCMaterializeFP(const ConstantFP *CFP, MVT VT);
     unsigned PPCMaterializeGV(const GlobalValue *GV, MVT VT);
-    unsigned PPCMaterializeInt(const Constant *C, MVT VT, bool UseSExt = true);
+    unsigned PPCMaterializeInt(const ConstantInt *CI, MVT VT,
+                               bool UseSExt = true);
     unsigned PPCMaterialize32BitInt(int64_t Imm,
                                     const TargetRegisterClass *RC);
     unsigned PPCMaterialize64BitInt(int64_t Imm,
@@ -258,7 +263,7 @@ static Optional<PPC::Predicate> getComparePred(CmpInst::Predicate Pred) {
 // fast-isel, and return its equivalent machine type in VT.
 // FIXME: Copied directly from ARM -- factor into base class?
 bool PPCFastISel::isTypeLegal(Type *Ty, MVT &VT) {
-  EVT Evt = TLI.getValueType(Ty, true);
+  EVT Evt = TLI.getValueType(DL, Ty, true);
 
   // Only handle simple types.
   if (Evt == MVT::Other || !Evt.isSimple()) return false;
@@ -283,6 +288,17 @@ bool PPCFastISel::isLoadTypeLegal(Type *Ty, MVT &VT) {
   return false;
 }
 
+bool PPCFastISel::isValueAvailable(const Value *V) const {
+  if (!isa<Instruction>(V))
+    return true;
+
+  const auto *I = cast<Instruction>(V);
+  if (FuncInfo.MBBMap[I->getParent()] == FuncInfo.MBB)
+    return true;
+
+  return false;
+}
+
 // Given a value Obj, create an Address object Addr that represents its
 // address.  Return false if we can't handle it.
 bool PPCFastISel::PPCComputeAddress(const Value *Obj, Address &Addr) {
@@ -309,12 +325,13 @@ bool PPCFastISel::PPCComputeAddress(const Value *Obj, Address &Addr) {
       return PPCComputeAddress(U->getOperand(0), Addr);
     case Instruction::IntToPtr:
       // Look past no-op inttoptrs.
-      if (TLI.getValueType(U->getOperand(0)->getType()) == TLI.getPointerTy())
+      if (TLI.getValueType(DL, U->getOperand(0)->getType()) ==
+          TLI.getPointerTy(DL))
         return PPCComputeAddress(U->getOperand(0), Addr);
       break;
     case Instruction::PtrToInt:
       // Look past no-op ptrtoints.
-      if (TLI.getValueType(U->getType()) == TLI.getPointerTy())
+      if (TLI.getValueType(DL, U->getType()) == TLI.getPointerTy(DL))
         return PPCComputeAddress(U->getOperand(0), Addr);
       break;
     case Instruction::GetElementPtr: {
@@ -491,8 +508,11 @@ bool PPCFastISel::PPCEmitLoad(MVT VT, unsigned &ResultReg, Address &Addr,
 
   // If this is a potential VSX load with an offset of 0, a VSX indexed load can
   // be used.
+  bool IsVSSRC = (ResultReg != 0) && isVSSRCRegister(ResultReg);
   bool IsVSFRC = (ResultReg != 0) && isVSFRCRegister(ResultReg);
-  if (IsVSFRC && (Opc == PPC::LFD) && 
+  bool Is32VSXLoad = IsVSSRC && Opc == PPC::LFS;
+  bool Is64VSXLoad = IsVSSRC && Opc == PPC::LFD;
+  if ((Is32VSXLoad || Is64VSXLoad) &&
       (Addr.BaseType != Address::FrameIndexBase) && UseOffset &&
       (Addr.Offset == 0)) {
     UseOffset = false;
@@ -506,7 +526,7 @@ bool PPCFastISel::PPCEmitLoad(MVT VT, unsigned &ResultReg, Address &Addr,
   // into a RegBase.
   if (Addr.BaseType == Address::FrameIndexBase) {
     // VSX only provides an indexed load.
-    if (IsVSFRC && Opc == PPC::LFD) return false;
+    if (Is32VSXLoad || Is64VSXLoad) return false;
 
     MachineMemOperand *MMO =
       FuncInfo.MF->getMachineMemOperand(
@@ -520,7 +540,7 @@ bool PPCFastISel::PPCEmitLoad(MVT VT, unsigned &ResultReg, Address &Addr,
   // Base reg with offset in range.
   } else if (UseOffset) {
     // VSX only provides an indexed load.
-    if (IsVSFRC && Opc == PPC::LFD) return false;
+    if (Is32VSXLoad || Is64VSXLoad) return false;
 
     BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, TII.get(Opc), ResultReg)
       .addImm(Addr.Offset).addReg(Addr.Base.Reg);
@@ -543,7 +563,7 @@ bool PPCFastISel::PPCEmitLoad(MVT VT, unsigned &ResultReg, Address &Addr,
       case PPC::LWA:    Opc = PPC::LWAX;    break;
       case PPC::LWA_32: Opc = PPC::LWAX_32; break;
       case PPC::LD:     Opc = PPC::LDX;     break;
-      case PPC::LFS:    Opc = PPC::LFSX;    break;
+      case PPC::LFS:    Opc = IsVSSRC ? PPC::LXSSPX : PPC::LFSX; break;
       case PPC::LFD:    Opc = IsVSFRC ? PPC::LXSDX : PPC::LFDX; break;
     }
     BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, TII.get(Opc), ResultReg)
@@ -624,9 +644,12 @@ bool PPCFastISel::PPCEmitStore(MVT VT, unsigned SrcReg, Address &Addr) {
 
   // If this is a potential VSX store with an offset of 0, a VSX indexed store
   // can be used.
+  bool IsVSSRC = isVSSRCRegister(SrcReg);
   bool IsVSFRC = isVSFRCRegister(SrcReg);
-  if (IsVSFRC && (Opc == PPC::STFD) && 
-      (Addr.BaseType != Address::FrameIndexBase) && UseOffset && 
+  bool Is32VSXStore = IsVSSRC && Opc == PPC::STFS;
+  bool Is64VSXStore = IsVSFRC && Opc == PPC::STFD;
+  if ((Is32VSXStore || Is64VSXStore) &&
+      (Addr.BaseType != Address::FrameIndexBase) && UseOffset &&
       (Addr.Offset == 0)) {
     UseOffset = false;
   }
@@ -636,7 +659,7 @@ bool PPCFastISel::PPCEmitStore(MVT VT, unsigned SrcReg, Address &Addr) {
   // into a RegBase.
   if (Addr.BaseType == Address::FrameIndexBase) {
     // VSX only provides an indexed store.
-    if (IsVSFRC && Opc == PPC::STFD) return false;
+    if (Is32VSXStore || Is64VSXStore) return false;
 
     MachineMemOperand *MMO =
       FuncInfo.MF->getMachineMemOperand(
@@ -653,7 +676,7 @@ bool PPCFastISel::PPCEmitStore(MVT VT, unsigned SrcReg, Address &Addr) {
   // Base reg with offset in range.
   } else if (UseOffset) {
     // VSX only provides an indexed store.
-    if (IsVSFRC && Opc == PPC::STFD) return false;
+    if (Is32VSXStore || Is64VSXStore) return false;
     
     BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, TII.get(Opc))
       .addReg(SrcReg).addImm(Addr.Offset).addReg(Addr.Base.Reg);
@@ -672,7 +695,7 @@ bool PPCFastISel::PPCEmitStore(MVT VT, unsigned SrcReg, Address &Addr) {
       case PPC::STH8: Opc = PPC::STHX8; break;
       case PPC::STW8: Opc = PPC::STWX8; break;
       case PPC::STD:  Opc = PPC::STDX;  break;
-      case PPC::STFS: Opc = PPC::STFSX; break;
+      case PPC::STFS: Opc = IsVSSRC ? PPC::STXSSPX : PPC::STFSX; break;
       case PPC::STFD: Opc = IsVSFRC ? PPC::STXSDX : PPC::STFDX; break;
     }
 
@@ -731,30 +754,31 @@ bool PPCFastISel::SelectBranch(const Instruction *I) {
 
   // For now, just try the simplest case where it's fed by a compare.
   if (const CmpInst *CI = dyn_cast<CmpInst>(BI->getCondition())) {
-    Optional<PPC::Predicate> OptPPCPred = getComparePred(CI->getPredicate());
-    if (!OptPPCPred)
-      return false;
+    if (isValueAvailable(CI)) {
+      Optional<PPC::Predicate> OptPPCPred = getComparePred(CI->getPredicate());
+      if (!OptPPCPred)
+        return false;
 
-    PPC::Predicate PPCPred = OptPPCPred.getValue();
+      PPC::Predicate PPCPred = OptPPCPred.getValue();
 
-    // Take advantage of fall-through opportunities.
-    if (FuncInfo.MBB->isLayoutSuccessor(TBB)) {
-      std::swap(TBB, FBB);
-      PPCPred = PPC::InvertPredicate(PPCPred);
-    }
-
-    unsigned CondReg = createResultReg(&PPC::CRRCRegClass);
+      // Take advantage of fall-through opportunities.
+      if (FuncInfo.MBB->isLayoutSuccessor(TBB)) {
+        std::swap(TBB, FBB);
+        PPCPred = PPC::InvertPredicate(PPCPred);
+      }
 
-    if (!PPCEmitCmp(CI->getOperand(0), CI->getOperand(1), CI->isUnsigned(),
-                    CondReg))
-      return false;
+      unsigned CondReg = createResultReg(&PPC::CRRCRegClass);
 
-    BuildMI(*BrBB, FuncInfo.InsertPt, DbgLoc, TII.get(PPC::BCC))
-      .addImm(PPCPred).addReg(CondReg).addMBB(TBB);
-    fastEmitBranch(FBB, DbgLoc);
-    FuncInfo.MBB->addSuccessor(TBB);
-    return true;
+      if (!PPCEmitCmp(CI->getOperand(0), CI->getOperand(1), CI->isUnsigned(),
+                      CondReg))
+        return false;
 
+      BuildMI(*BrBB, FuncInfo.InsertPt, DbgLoc, TII.get(PPC::BCC))
+        .addImm(PPCPred).addReg(CondReg).addMBB(TBB);
+      fastEmitBranch(FBB, DbgLoc);
+      FuncInfo.MBB->addSuccessor(TBB);
+      return true;
+    }
   } else if (const ConstantInt *CI =
              dyn_cast<ConstantInt>(BI->getCondition())) {
     uint64_t Imm = CI->getZExtValue();
@@ -777,7 +801,7 @@ bool PPCFastISel::SelectBranch(const Instruction *I) {
 bool PPCFastISel::PPCEmitCmp(const Value *SrcValue1, const Value *SrcValue2,
                              bool IsZExt, unsigned DestReg) {
   Type *Ty = SrcValue1->getType();
-  EVT SrcEVT = TLI.getValueType(Ty, true);
+  EVT SrcEVT = TLI.getValueType(DL, Ty, true);
   if (!SrcEVT.isSimple())
     return false;
   MVT SrcVT = SrcEVT.getSimpleVT();
@@ -871,8 +895,8 @@ bool PPCFastISel::PPCEmitCmp(const Value *SrcValue1, const Value *SrcValue2,
 // Attempt to fast-select a floating-point extend instruction.
 bool PPCFastISel::SelectFPExt(const Instruction *I) {
   Value *Src  = I->getOperand(0);
-  EVT SrcVT  = TLI.getValueType(Src->getType(), true);
-  EVT DestVT = TLI.getValueType(I->getType(), true);
+  EVT SrcVT = TLI.getValueType(DL, Src->getType(), true);
+  EVT DestVT = TLI.getValueType(DL, I->getType(), true);
 
   if (SrcVT != MVT::f32 || DestVT != MVT::f64)
     return false;
@@ -889,8 +913,8 @@ bool PPCFastISel::SelectFPExt(const Instruction *I) {
 // Attempt to fast-select a floating-point truncate instruction.
 bool PPCFastISel::SelectFPTrunc(const Instruction *I) {
   Value *Src  = I->getOperand(0);
-  EVT SrcVT  = TLI.getValueType(Src->getType(), true);
-  EVT DestVT = TLI.getValueType(I->getType(), true);
+  EVT SrcVT = TLI.getValueType(DL, Src->getType(), true);
+  EVT DestVT = TLI.getValueType(DL, I->getType(), true);
 
   if (SrcVT != MVT::f64 || DestVT != MVT::f32)
     return false;
@@ -970,7 +994,7 @@ bool PPCFastISel::SelectIToFP(const Instruction *I, bool IsSigned) {
     return false;
 
   Value *Src = I->getOperand(0);
-  EVT SrcEVT = TLI.getValueType(Src->getType(), true);
+  EVT SrcEVT = TLI.getValueType(DL, Src->getType(), true);
   if (!SrcEVT.isSimple())
     return false;
 
@@ -1135,7 +1159,7 @@ bool PPCFastISel::SelectFPToI(const Instruction *I, bool IsSigned) {
 // Attempt to fast-select a binary integer operation that isn't already
 // handled automatically.
 bool PPCFastISel::SelectBinaryIntOp(const Instruction *I, unsigned ISDOpcode) {
-  EVT DestVT  = TLI.getValueType(I->getType(), true);
+  EVT DestVT = TLI.getValueType(DL, I->getType(), true);
 
   // We can get here in the case when we have a binary operation on a non-legal
   // type and the target independent selector doesn't know how to handle it.
@@ -1426,9 +1450,9 @@ bool PPCFastISel::fastLowerCall(CallLoweringInfo &CLI) {
   bool IsTailCall     = CLI.IsTailCall;
   bool IsVarArg       = CLI.IsVarArg;
   const Value *Callee = CLI.Callee;
-  const char *SymName = CLI.SymName;
+  const MCSymbol *Symbol = CLI.Symbol;
 
-  if (!Callee && !SymName)
+  if (!Callee && !Symbol)
     return false;
 
   // Allow SelectionDAG isel to handle tail calls.
@@ -1572,7 +1596,7 @@ bool PPCFastISel::SelectRet(const Instruction *I) {
 
   if (Ret->getNumOperands() > 0) {
     SmallVector<ISD::OutputArg, 4> Outs;
-    GetReturnInfo(F.getReturnType(), F.getAttributes(), Outs, TLI);
+    GetReturnInfo(F.getReturnType(), F.getAttributes(), Outs, TLI, DL);
 
     // Analyze operands of the call, assigning locations to each operand.
     SmallVector<CCValAssign, 16> ValLocs;
@@ -1584,21 +1608,18 @@ bool PPCFastISel::SelectRet(const Instruction *I) {
     if (ValLocs.size() > 1)
       return false;
 
-    // Special case for returning a constant integer of any size.
-    // Materialize the constant as an i64 and copy it to the return
-    // register. We still need to worry about properly extending the sign. E.g:
-    // If the constant has only one bit, it means it is a boolean. Therefore
-    // we can't use PPCMaterializeInt because it extends the sign which will
-    // cause negations of the returned value to be incorrect as they are
-    // implemented as the flip of the least significant bit.
-    if (isa<ConstantInt>(*RV)) {
-      const Constant *C = cast<Constant>(RV);
-
+    // Special case for returning a constant integer of any size - materialize
+    // the constant as an i64 and copy it to the return register.
+    if (const ConstantInt *CI = dyn_cast<ConstantInt>(RV)) {
       CCValAssign &VA = ValLocs[0];
 
       unsigned RetReg = VA.getLocReg();
-      unsigned SrcReg = PPCMaterializeInt(C, MVT::i64,
-                                          VA.getLocInfo() == CCValAssign::SExt);
+      // We still need to worry about properly extending the sign. For example,
+      // we could have only a single bit or a constant that needs zero
+      // extension rather than sign extension. Make sure we pass the return
+      // value extension property to integer materialization.
+      unsigned SrcReg =
+          PPCMaterializeInt(CI, MVT::i64, VA.getLocInfo() == CCValAssign::SExt);
 
       BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc,
             TII.get(TargetOpcode::COPY), RetReg).addReg(SrcReg);
@@ -1619,7 +1640,7 @@ bool PPCFastISel::SelectRet(const Instruction *I) {
         RetRegs.push_back(VA.getLocReg());
         unsigned SrcReg = Reg + VA.getValNo();
 
-        EVT RVEVT = TLI.getValueType(RV->getType());
+        EVT RVEVT = TLI.getValueType(DL, RV->getType());
         if (!RVEVT.isSimple())
           return false;
         MVT RVVT = RVEVT.getSimpleVT();
@@ -1747,8 +1768,8 @@ bool PPCFastISel::SelectIndirectBr(const Instruction *I) {
 // Attempt to fast-select an integer truncate instruction.
 bool PPCFastISel::SelectTrunc(const Instruction *I) {
   Value *Src  = I->getOperand(0);
-  EVT SrcVT  = TLI.getValueType(Src->getType(), true);
-  EVT DestVT = TLI.getValueType(I->getType(), true);
+  EVT SrcVT = TLI.getValueType(DL, Src->getType(), true);
+  EVT DestVT = TLI.getValueType(DL, I->getType(), true);
 
   if (SrcVT != MVT::i64 && SrcVT != MVT::i32 && SrcVT != MVT::i16)
     return false;
@@ -1784,8 +1805,8 @@ bool PPCFastISel::SelectIntExt(const Instruction *I) {
   if (!SrcReg) return false;
 
   EVT SrcEVT, DestEVT;
-  SrcEVT = TLI.getValueType(SrcTy, true);
-  DestEVT = TLI.getValueType(DestTy, true);
+  SrcEVT = TLI.getValueType(DL, SrcTy, true);
+  DestEVT = TLI.getValueType(DL, DestTy, true);
   if (!SrcEVT.isSimple())
     return false;
   if (!DestEVT.isSimple())
@@ -1957,7 +1978,7 @@ unsigned PPCFastISel::PPCMaterializeGV(const GlobalValue *GV, MVT VT) {
     // on the "if" path here.
     if (CModel == CodeModel::Large ||
         (GV->getType()->getElementType()->isFunctionTy() &&
-         (GV->isDeclaration() || GV->isWeakForLinker())) ||
+         !GV->isStrongDefinitionForLinker()) ||
         GV->isDeclaration() || GV->hasCommonLinkage() ||
         GV->hasAvailableExternallyLinkage())
       BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, TII.get(PPC::LDtocL),
@@ -2062,12 +2083,11 @@ unsigned PPCFastISel::PPCMaterialize64BitInt(int64_t Imm,
 
 // Materialize an integer constant into a register, and return
 // the register number (or zero if we failed to handle it).
-unsigned PPCFastISel::PPCMaterializeInt(const Constant *C, MVT VT,
-                                                           bool UseSExt) {
+unsigned PPCFastISel::PPCMaterializeInt(const ConstantInt *CI, MVT VT,
+                                        bool UseSExt) {
   // If we're using CR bit registers for i1 values, handle that as a special
   // case first.
   if (VT == MVT::i1 && PPCSubTarget->useCRBits()) {
-    const ConstantInt *CI = cast<ConstantInt>(C);
     unsigned ImmReg = createResultReg(&PPC::CRBITRCRegClass);
     BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc,
             TII.get(CI->isZero() ? PPC::CRUNSET : PPC::CRSET), ImmReg);
@@ -2082,12 +2102,17 @@ unsigned PPCFastISel::PPCMaterializeInt(const Constant *C, MVT VT,
                                    &PPC::GPRCRegClass);
 
   // If the constant is in range, use a load-immediate.
-  const ConstantInt *CI = cast<ConstantInt>(C);
-  if (isInt<16>(CI->getSExtValue())) {
+  if (UseSExt && isInt<16>(CI->getSExtValue())) {
+    unsigned Opc = (VT == MVT::i64) ? PPC::LI8 : PPC::LI;
+    unsigned ImmReg = createResultReg(RC);
+    BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, TII.get(Opc), ImmReg)
+        .addImm(CI->getSExtValue());
+    return ImmReg;
+  } else if (!UseSExt && isUInt<16>(CI->getZExtValue())) {
     unsigned Opc = (VT == MVT::i64) ? PPC::LI8 : PPC::LI;
     unsigned ImmReg = createResultReg(RC);
     BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, TII.get(Opc), ImmReg)
-      .addImm( (UseSExt) ? CI->getSExtValue() : CI->getZExtValue() );
+        .addImm(CI->getZExtValue());
     return ImmReg;
   }
 
@@ -2105,7 +2130,7 @@ unsigned PPCFastISel::PPCMaterializeInt(const Constant *C, MVT VT,
 // Materialize a constant into a register, and return the register
 // number (or zero if we failed to handle it).
 unsigned PPCFastISel::fastMaterializeConstant(const Constant *C) {
-  EVT CEVT = TLI.getValueType(C->getType(), true);
+  EVT CEVT = TLI.getValueType(DL, C->getType(), true);
 
   // Only handle simple types.
   if (!CEVT.isSimple()) return 0;
@@ -2115,8 +2140,8 @@ unsigned PPCFastISel::fastMaterializeConstant(const Constant *C) {
     return PPCMaterializeFP(CFP, VT);
   else if (const GlobalValue *GV = dyn_cast<GlobalValue>(C))
     return PPCMaterializeGV(GV, VT);
-  else if (isa<ConstantInt>(C))
-    return PPCMaterializeInt(C, VT, VT != MVT::i1);
+  else if (const ConstantInt *CI = dyn_cast<ConstantInt>(C))
+    return PPCMaterializeInt(CI, VT, VT != MVT::i1);
 
   return 0;
 }
diff --git a/lib/Target/PowerPC/PPCFrameLowering.cpp b/lib/Target/PowerPC/PPCFrameLowering.cpp
index f997fea4d93..87229d80d9c 100644
--- a/lib/Target/PowerPC/PPCFrameLowering.cpp
+++ b/lib/Target/PowerPC/PPCFrameLowering.cpp
@@ -306,9 +306,10 @@ static void HandleVRSaveUpdate(MachineInstr *MI, const TargetInstrInfo &TII) {
   const TargetRegisterInfo *TRI = MF->getSubtarget().getRegisterInfo();
   DebugLoc dl = MI->getDebugLoc();
 
+  const MachineRegisterInfo &MRI = MF->getRegInfo();
   unsigned UsedRegMask = 0;
   for (unsigned i = 0; i != 32; ++i)
-    if (MF->getRegInfo().isPhysRegUsed(VRRegNo[i]))
+    if (MRI.isPhysRegModified(VRRegNo[i]))
       UsedRegMask |= 1 << (31-i);
 
   // Live in and live out values already must be in the mask, so don't bother
@@ -555,8 +556,9 @@ void PPCFrameLowering::replaceFPWithRealFP(MachineFunction &MF) const {
     }
 }
 
-void PPCFrameLowering::emitPrologue(MachineFunction &MF) const {
-  MachineBasicBlock &MBB = MF.front();   // Prolog goes in entry BB
+void PPCFrameLowering::emitPrologue(MachineFunction &MF,
+                                    MachineBasicBlock &MBB) const {
+  assert(&MF.front() == &MBB && "Shrink-wrapping not yet supported");
   MachineBasicBlock::iterator MBBI = MBB.begin();
   MachineFrameInfo *MFI = MF.getFrameInfo();
   const PPCInstrInfo &TII =
@@ -1157,9 +1159,11 @@ void PPCFrameLowering::emitEpilogue(MachineFunction &MF,
   }
 }
 
-void
-PPCFrameLowering::processFunctionBeforeCalleeSavedScan(MachineFunction &MF,
-                                                   RegScavenger *) const {
+void PPCFrameLowering::determineCalleeSaves(MachineFunction &MF,
+                                            BitVector &SavedRegs,
+                                            RegScavenger *RS) const {
+  TargetFrameLowering::determineCalleeSaves(MF, SavedRegs, RS);
+
   const PPCRegisterInfo *RegInfo =
       static_cast<const PPCRegisterInfo *>(Subtarget.getRegisterInfo());
 
@@ -1167,8 +1171,7 @@ PPCFrameLowering::processFunctionBeforeCalleeSavedScan(MachineFunction &MF,
   PPCFunctionInfo *FI = MF.getInfo<PPCFunctionInfo>();
   unsigned LR = RegInfo->getRARegister();
   FI->setMustSaveLR(MustSaveLR(MF, LR));
-  MachineRegisterInfo &MRI = MF.getRegInfo();
-  MRI.setPhysRegUnused(LR);
+  SavedRegs.reset(LR);
 
   //  Save R31 if necessary
   int FPSI = FI->getFramePointerSaveIndex();
@@ -1213,9 +1216,9 @@ PPCFrameLowering::processFunctionBeforeCalleeSavedScan(MachineFunction &MF,
   // For 32-bit SVR4, allocate the nonvolatile CR spill slot iff the
   // function uses CR 2, 3, or 4.
   if (!isPPC64 && !isDarwinABI &&
-      (MRI.isPhysRegUsed(PPC::CR2) ||
-       MRI.isPhysRegUsed(PPC::CR3) ||
-       MRI.isPhysRegUsed(PPC::CR4))) {
+      (SavedRegs.test(PPC::CR2) ||
+       SavedRegs.test(PPC::CR3) ||
+       SavedRegs.test(PPC::CR4))) {
     int FrameIdx = MFI->CreateFixedObject((uint64_t)4, (int64_t)-4, true);
     FI->setCRSpillFrameIndex(FrameIdx);
   }
diff --git a/lib/Target/PowerPC/PPCFrameLowering.h b/lib/Target/PowerPC/PPCFrameLowering.h
index dddabb80892..d6a389bfbf0 100644
--- a/lib/Target/PowerPC/PPCFrameLowering.h
+++ b/lib/Target/PowerPC/PPCFrameLowering.h
@@ -38,15 +38,15 @@ class PPCFrameLowering: public TargetFrameLowering {
 
   /// emitProlog/emitEpilog - These methods insert prolog and epilog code into
   /// the function.
-  void emitPrologue(MachineFunction &MF) const override;
+  void emitPrologue(MachineFunction &MF, MachineBasicBlock &MBB) const override;
   void emitEpilogue(MachineFunction &MF, MachineBasicBlock &MBB) const override;
 
   bool hasFP(const MachineFunction &MF) const override;
   bool needsFP(const MachineFunction &MF) const;
   void replaceFPWithRealFP(MachineFunction &MF) const;
 
-  void processFunctionBeforeCalleeSavedScan(MachineFunction &MF,
-                                     RegScavenger *RS = nullptr) const override;
+  void determineCalleeSaves(MachineFunction &MF, BitVector &SavedRegs,
+                            RegScavenger *RS = nullptr) const override;
   void processFunctionBeforeFrameFinalized(MachineFunction &MF,
                                      RegScavenger *RS = nullptr) const override;
   void addScavengingSpillSlot(MachineFunction &MF, RegScavenger *RS) const;
diff --git a/lib/Target/PowerPC/PPCISelDAGToDAG.cpp b/lib/Target/PowerPC/PPCISelDAGToDAG.cpp
index 4f8d01b059c..01a3acb742e 100644
--- a/lib/Target/PowerPC/PPCISelDAGToDAG.cpp
+++ b/lib/Target/PowerPC/PPCISelDAGToDAG.cpp
@@ -90,19 +90,20 @@ namespace {
 
     /// getI32Imm - Return a target constant with the specified value, of type
     /// i32.
-    inline SDValue getI32Imm(unsigned Imm) {
-      return CurDAG->getTargetConstant(Imm, MVT::i32);
+    inline SDValue getI32Imm(unsigned Imm, SDLoc dl) {
+      return CurDAG->getTargetConstant(Imm, dl, MVT::i32);
     }
 
     /// getI64Imm - Return a target constant with the specified value, of type
     /// i64.
-    inline SDValue getI64Imm(uint64_t Imm) {
-      return CurDAG->getTargetConstant(Imm, MVT::i64);
+    inline SDValue getI64Imm(uint64_t Imm, SDLoc dl) {
+      return CurDAG->getTargetConstant(Imm, dl, MVT::i64);
     }
 
     /// getSmallIPtrImm - Return a target constant of pointer type.
-    inline SDValue getSmallIPtrImm(unsigned Imm) {
-      return CurDAG->getTargetConstant(Imm, PPCLowering->getPointerTy());
+    inline SDValue getSmallIPtrImm(unsigned Imm, SDLoc dl) {
+      return CurDAG->getTargetConstant(
+          Imm, dl, PPCLowering->getPointerTy(CurDAG->getDataLayout()));
     }
 
     /// isRotateAndMask - Returns true if Mask and Shift can be folded into a
@@ -197,10 +198,11 @@ namespace {
         // (because we might end up lowering this as 0(%op)).
         const TargetRegisterInfo *TRI = PPCSubTarget->getRegisterInfo();
         const TargetRegisterClass *TRC = TRI->getPointerRegClass(*MF, /*Kind=*/1);
-        SDValue RC = CurDAG->getTargetConstant(TRC->getID(), MVT::i32);
+        SDLoc dl(Op);
+        SDValue RC = CurDAG->getTargetConstant(TRC->getID(), dl, MVT::i32);
         SDValue NewOp =
           SDValue(CurDAG->getMachineNode(TargetOpcode::COPY_TO_REGCLASS,
-                                         SDLoc(Op), Op.getValueType(),
+                                         dl, Op.getValueType(),
                                          Op, RC), 0);
 
         OutOps.push_back(NewOp);
@@ -312,7 +314,7 @@ SDNode *PPCDAGToDAGISel::getGlobalBaseReg() {
     const Module *M = MF->getFunction()->getParent();
     DebugLoc dl;
 
-    if (PPCLowering->getPointerTy() == MVT::i32) {
+    if (PPCLowering->getPointerTy(CurDAG->getDataLayout()) == MVT::i32) {
       if (PPCSubTarget->isTargetELF()) {
         GlobalBaseReg = PPC::R30;
         if (M->getPICLevel() == PICLevel::Small) {
@@ -341,7 +343,8 @@ SDNode *PPCDAGToDAGISel::getGlobalBaseReg() {
     }
   }
   return CurDAG->getRegister(GlobalBaseReg,
-                             PPCLowering->getPointerTy()).getNode();
+                             PPCLowering->getPointerTy(CurDAG->getDataLayout()))
+      .getNode();
 }
 
 /// isIntS16Immediate - This method tests to see if the node is either a 32-bit
@@ -406,9 +409,9 @@ SDNode *PPCDAGToDAGISel::getFrameIndex(SDNode *SN, SDNode *N, unsigned Offset) {
   unsigned Opc = N->getValueType(0) == MVT::i32 ? PPC::ADDI : PPC::ADDI8;
   if (SN->hasOneUse())
     return CurDAG->SelectNodeTo(SN, Opc, N->getValueType(0), TFI,
-                                getSmallIPtrImm(Offset));
+                                getSmallIPtrImm(Offset, dl));
   return CurDAG->getMachineNode(Opc, dl, N->getValueType(0), TFI,
-                                getSmallIPtrImm(Offset));
+                                getSmallIPtrImm(Offset, dl));
 }
 
 bool PPCDAGToDAGISel::isRotateAndMask(SDNode *N, unsigned Mask,
@@ -523,8 +526,8 @@ SDNode *PPCDAGToDAGISel::SelectBitfieldInsert(SDNode *N) {
       }
 
       SH &= 31;
-      SDValue Ops[] = { Op0, Op1, getI32Imm(SH), getI32Imm(MB),
-                          getI32Imm(ME) };
+      SDValue Ops[] = { Op0, Op1, getI32Imm(SH, dl), getI32Imm(MB, dl),
+                          getI32Imm(ME, dl) };
       return CurDAG->getMachineNode(PPC::RLWIMI, dl, MVT::i32, Ops);
     }
   }
@@ -652,8 +655,8 @@ static SDNode *SelectInt64Direct(SelectionDAG *CurDAG, SDLoc dl, int64_t Imm) {
   unsigned Lo = Imm & 0xFFFF;
   unsigned Hi = (Imm >> 16) & 0xFFFF;
 
-  auto getI32Imm = [CurDAG](unsigned Imm) {
-      return CurDAG->getTargetConstant(Imm, MVT::i32);
+  auto getI32Imm = [CurDAG, dl](unsigned Imm) {
+      return CurDAG->getTargetConstant(Imm, dl, MVT::i32);
   };
 
   // Simple value.
@@ -743,8 +746,8 @@ static SDNode *SelectInt64(SelectionDAG *CurDAG, SDLoc dl, int64_t Imm) {
   if (!RMin)
     return SelectInt64Direct(CurDAG, dl, Imm);
 
-  auto getI32Imm = [CurDAG](unsigned Imm) {
-      return CurDAG->getTargetConstant(Imm, MVT::i32);
+  auto getI32Imm = [CurDAG, dl](unsigned Imm) {
+      return CurDAG->getTargetConstant(Imm, dl, MVT::i32);
   };
 
   SDValue Val = SDValue(SelectInt64Direct(CurDAG, dl, MatImm), 0);
@@ -1194,8 +1197,8 @@ class BitPermutationSelector {
     }
   }
 
-  SDValue getI32Imm(unsigned Imm) {
-    return CurDAG->getTargetConstant(Imm, MVT::i32);
+  SDValue getI32Imm(unsigned Imm, SDLoc dl) {
+    return CurDAG->getTargetConstant(Imm, dl, MVT::i32);
   }
 
   uint64_t getZerosMask() {
@@ -1267,7 +1270,8 @@ class BitPermutationSelector {
       SDValue VRot;
       if (VRI.RLAmt) {
         SDValue Ops[] =
-          { VRI.V, getI32Imm(VRI.RLAmt), getI32Imm(0), getI32Imm(31) };
+          { VRI.V, getI32Imm(VRI.RLAmt, dl), getI32Imm(0, dl),
+            getI32Imm(31, dl) };
         VRot = SDValue(CurDAG->getMachineNode(PPC::RLWINM, dl, MVT::i32,
                                               Ops), 0);
       } else {
@@ -1277,10 +1281,10 @@ class BitPermutationSelector {
       SDValue ANDIVal, ANDISVal;
       if (ANDIMask != 0)
         ANDIVal = SDValue(CurDAG->getMachineNode(PPC::ANDIo, dl, MVT::i32,
-                            VRot, getI32Imm(ANDIMask)), 0);
+                            VRot, getI32Imm(ANDIMask, dl)), 0);
       if (ANDISMask != 0)
         ANDISVal = SDValue(CurDAG->getMachineNode(PPC::ANDISo, dl, MVT::i32,
-                             VRot, getI32Imm(ANDISMask)), 0);
+                             VRot, getI32Imm(ANDISMask, dl)), 0);
 
       SDValue TotalVal;
       if (!ANDIVal)
@@ -1299,12 +1303,9 @@ class BitPermutationSelector {
 
       // Now, remove all groups with this underlying value and rotation
       // factor.
-      for (auto I = BitGroups.begin(); I != BitGroups.end();) {
-        if (I->V == VRI.V && I->RLAmt == VRI.RLAmt)
-          I = BitGroups.erase(I);
-        else
-          ++I;
-      }
+      eraseMatchingBitGroups([VRI](const BitGroup &BG) {
+        return BG.V == VRI.V && BG.RLAmt == VRI.RLAmt;
+      });
     }
   }
 
@@ -1326,19 +1327,18 @@ class BitPermutationSelector {
       if (VRI.RLAmt) {
         if (InstCnt) *InstCnt += 1;
         SDValue Ops[] =
-          { VRI.V, getI32Imm(VRI.RLAmt), getI32Imm(0), getI32Imm(31) };
-        Res = SDValue(CurDAG->getMachineNode(PPC::RLWINM, dl, MVT::i32, Ops), 0);
+          { VRI.V, getI32Imm(VRI.RLAmt, dl), getI32Imm(0, dl),
+            getI32Imm(31, dl) };
+        Res = SDValue(CurDAG->getMachineNode(PPC::RLWINM, dl, MVT::i32, Ops),
+                      0);
       } else {
         Res = VRI.V;
       }
 
       // Now, remove all groups with this underlying value and rotation factor.
-      for (auto I = BitGroups.begin(); I != BitGroups.end();) {
-        if (I->V == VRI.V && I->RLAmt == VRI.RLAmt)
-          I = BitGroups.erase(I);
-        else
-          ++I;
-      }
+      eraseMatchingBitGroups([VRI](const BitGroup &BG) {
+        return BG.V == VRI.V && BG.RLAmt == VRI.RLAmt;
+      });
     }
 
     if (InstCnt) *InstCnt += BitGroups.size();
@@ -1347,13 +1347,15 @@ class BitPermutationSelector {
     for (auto &BG : BitGroups) {
       if (!Res) {
         SDValue Ops[] =
-          { BG.V, getI32Imm(BG.RLAmt), getI32Imm(Bits.size() - BG.EndIdx - 1),
-            getI32Imm(Bits.size() - BG.StartIdx - 1) };
+          { BG.V, getI32Imm(BG.RLAmt, dl),
+            getI32Imm(Bits.size() - BG.EndIdx - 1, dl),
+            getI32Imm(Bits.size() - BG.StartIdx - 1, dl) };
         Res = SDValue(CurDAG->getMachineNode(PPC::RLWINM, dl, MVT::i32, Ops), 0);
       } else {
         SDValue Ops[] =
-          { Res, BG.V, getI32Imm(BG.RLAmt), getI32Imm(Bits.size() - BG.EndIdx - 1),
-            getI32Imm(Bits.size() - BG.StartIdx - 1) };
+          { Res, BG.V, getI32Imm(BG.RLAmt, dl),
+              getI32Imm(Bits.size() - BG.EndIdx - 1, dl),
+            getI32Imm(Bits.size() - BG.StartIdx - 1, dl) };
         Res = SDValue(CurDAG->getMachineNode(PPC::RLWIMI, dl, MVT::i32, Ops), 0);
       }
     }
@@ -1372,10 +1374,10 @@ class BitPermutationSelector {
       SDValue ANDIVal, ANDISVal;
       if (ANDIMask != 0)
         ANDIVal = SDValue(CurDAG->getMachineNode(PPC::ANDIo, dl, MVT::i32,
-                            Res, getI32Imm(ANDIMask)), 0);
+                            Res, getI32Imm(ANDIMask, dl)), 0);
       if (ANDISMask != 0)
         ANDISVal = SDValue(CurDAG->getMachineNode(PPC::ANDISo, dl, MVT::i32,
-                             Res, getI32Imm(ANDISMask)), 0);
+                             Res, getI32Imm(ANDISMask, dl)), 0);
 
       if (!ANDIVal)
         Res = ANDISVal;
@@ -1426,27 +1428,27 @@ class BitPermutationSelector {
       assert(InstMaskStart >= 32 && "Mask cannot start out of range");
       assert(InstMaskEnd   >= 32 && "Mask cannot end out of range");
       SDValue Ops[] =
-        { V, getI32Imm(RLAmt), getI32Imm(InstMaskStart - 32),
-          getI32Imm(InstMaskEnd - 32) };
+        { V, getI32Imm(RLAmt, dl), getI32Imm(InstMaskStart - 32, dl),
+          getI32Imm(InstMaskEnd - 32, dl) };
       return SDValue(CurDAG->getMachineNode(PPC::RLWINM8, dl, MVT::i64,
                                             Ops), 0);
     }
 
     if (InstMaskEnd == 63) {
       SDValue Ops[] =
-        { V, getI32Imm(RLAmt), getI32Imm(InstMaskStart) };
+        { V, getI32Imm(RLAmt, dl), getI32Imm(InstMaskStart, dl) };
       return SDValue(CurDAG->getMachineNode(PPC::RLDICL, dl, MVT::i64, Ops), 0);
     }
 
     if (InstMaskStart == 0) {
       SDValue Ops[] =
-        { V, getI32Imm(RLAmt), getI32Imm(InstMaskEnd) };
+        { V, getI32Imm(RLAmt, dl), getI32Imm(InstMaskEnd, dl) };
       return SDValue(CurDAG->getMachineNode(PPC::RLDICR, dl, MVT::i64, Ops), 0);
     }
 
     if (InstMaskEnd == 63 - RLAmt) {
       SDValue Ops[] =
-        { V, getI32Imm(RLAmt), getI32Imm(InstMaskStart) };
+        { V, getI32Imm(RLAmt, dl), getI32Imm(InstMaskStart, dl) };
       return SDValue(CurDAG->getMachineNode(PPC::RLDIC, dl, MVT::i64, Ops), 0);
     }
 
@@ -1487,15 +1489,15 @@ class BitPermutationSelector {
       assert(InstMaskStart >= 32 && "Mask cannot start out of range");
       assert(InstMaskEnd   >= 32 && "Mask cannot end out of range");
       SDValue Ops[] =
-        { Base, V, getI32Imm(RLAmt), getI32Imm(InstMaskStart - 32),
-          getI32Imm(InstMaskEnd - 32) };
+        { Base, V, getI32Imm(RLAmt, dl), getI32Imm(InstMaskStart - 32, dl),
+          getI32Imm(InstMaskEnd - 32, dl) };
       return SDValue(CurDAG->getMachineNode(PPC::RLWIMI8, dl, MVT::i64,
                                             Ops), 0);
     }
 
     if (InstMaskEnd == 63 - RLAmt) {
       SDValue Ops[] =
-        { Base, V, getI32Imm(RLAmt), getI32Imm(InstMaskStart) };
+        { Base, V, getI32Imm(RLAmt, dl), getI32Imm(InstMaskStart, dl) };
       return SDValue(CurDAG->getMachineNode(PPC::RLDIMI, dl, MVT::i64, Ops), 0);
     }
 
@@ -1538,7 +1540,7 @@ class BitPermutationSelector {
       // Repl32 true, but are trivially convertable to Repl32 false. Such a
       // group is trivially convertable if it overlaps only with the lower 32
       // bits, and the group has not been coalesced.
-      auto MatchingBG = [VRI](BitGroup &BG) {
+      auto MatchingBG = [VRI](const BitGroup &BG) {
         if (VRI.V != BG.V)
           return false;
 
@@ -1642,10 +1644,10 @@ class BitPermutationSelector {
         SDValue ANDIVal, ANDISVal;
         if (ANDIMask != 0)
           ANDIVal = SDValue(CurDAG->getMachineNode(PPC::ANDIo8, dl, MVT::i64,
-                              VRot, getI32Imm(ANDIMask)), 0);
+                              VRot, getI32Imm(ANDIMask, dl)), 0);
         if (ANDISMask != 0)
           ANDISVal = SDValue(CurDAG->getMachineNode(PPC::ANDISo8, dl, MVT::i64,
-                               VRot, getI32Imm(ANDISMask)), 0);
+                               VRot, getI32Imm(ANDISMask, dl)), 0);
 
         if (!ANDIVal)
           TotalVal = ANDISVal;
@@ -1669,12 +1671,7 @@ class BitPermutationSelector {
 
       // Now, remove all groups with this underlying value and rotation
       // factor.
-      for (auto I = BitGroups.begin(); I != BitGroups.end();) {
-        if (MatchingBG(*I))
-          I = BitGroups.erase(I);
-        else
-          ++I;
-      }
+      eraseMatchingBitGroups(MatchingBG);
     }
   }
 
@@ -1734,12 +1731,10 @@ class BitPermutationSelector {
 
       // Now, remove all groups with this underlying value and rotation factor.
       if (Res)
-        for (auto I = BitGroups.begin(); I != BitGroups.end();) {
-          if (I->V == VRI.V && I->RLAmt == VRI.RLAmt && I->Repl32 == VRI.Repl32)
-            I = BitGroups.erase(I);
-          else
-            ++I;
-        }
+        eraseMatchingBitGroups([VRI](const BitGroup &BG) {
+          return BG.V == VRI.V && BG.RLAmt == VRI.RLAmt &&
+                 BG.Repl32 == VRI.Repl32;
+        });
     }
 
     // Because 64-bit rotates are more flexible than inserts, we might have a
@@ -1792,10 +1787,10 @@ class BitPermutationSelector {
         SDValue ANDIVal, ANDISVal;
         if (ANDIMask != 0)
           ANDIVal = SDValue(CurDAG->getMachineNode(PPC::ANDIo8, dl, MVT::i64,
-                              Res, getI32Imm(ANDIMask)), 0);
+                              Res, getI32Imm(ANDIMask, dl)), 0);
         if (ANDISMask != 0)
           ANDISVal = SDValue(CurDAG->getMachineNode(PPC::ANDISo8, dl, MVT::i64,
-                               Res, getI32Imm(ANDISMask)), 0);
+                               Res, getI32Imm(ANDISMask, dl)), 0);
 
         if (!ANDIVal)
           Res = ANDISVal;
@@ -1840,6 +1835,11 @@ class BitPermutationSelector {
     return nullptr;
   }
 
+  void eraseMatchingBitGroups(function_ref<bool(const BitGroup &)> F) {
+    BitGroups.erase(std::remove_if(BitGroups.begin(), BitGroups.end(), F),
+                    BitGroups.end());
+  }
+
   SmallVector<ValueBit, 64> Bits;
 
   bool HasZeros;
@@ -1940,11 +1940,13 @@ SDValue PPCDAGToDAGISel::SelectCC(SDValue LHS, SDValue RHS,
         // SETEQ/SETNE comparison with 16-bit immediate, fold it.
         if (isUInt<16>(Imm))
           return SDValue(CurDAG->getMachineNode(PPC::CMPLWI, dl, MVT::i32, LHS,
-                                                getI32Imm(Imm & 0xFFFF)), 0);
+                                                getI32Imm(Imm & 0xFFFF, dl)),
+                         0);
         // If this is a 16-bit signed immediate, fold it.
         if (isInt<16>((int)Imm))
           return SDValue(CurDAG->getMachineNode(PPC::CMPWI, dl, MVT::i32, LHS,
-                                                getI32Imm(Imm & 0xFFFF)), 0);
+                                                getI32Imm(Imm & 0xFFFF, dl)),
+                         0);
 
         // For non-equality comparisons, the default code would materialize the
         // constant, then compare against it, like this:
@@ -1956,21 +1958,22 @@ SDValue PPCDAGToDAGISel::SelectCC(SDValue LHS, SDValue RHS,
         //   cmplwi cr0,r0,0x5678
         //   beq cr0,L6
         SDValue Xor(CurDAG->getMachineNode(PPC::XORIS, dl, MVT::i32, LHS,
-                                           getI32Imm(Imm >> 16)), 0);
+                                           getI32Imm(Imm >> 16, dl)), 0);
         return SDValue(CurDAG->getMachineNode(PPC::CMPLWI, dl, MVT::i32, Xor,
-                                              getI32Imm(Imm & 0xFFFF)), 0);
+                                              getI32Imm(Imm & 0xFFFF, dl)), 0);
       }
       Opc = PPC::CMPLW;
     } else if (ISD::isUnsignedIntSetCC(CC)) {
       if (isInt32Immediate(RHS, Imm) && isUInt<16>(Imm))
         return SDValue(CurDAG->getMachineNode(PPC::CMPLWI, dl, MVT::i32, LHS,
-                                              getI32Imm(Imm & 0xFFFF)), 0);
+                                              getI32Imm(Imm & 0xFFFF, dl)), 0);
       Opc = PPC::CMPLW;
     } else {
       short SImm;
       if (isIntS16Immediate(RHS, SImm))
         return SDValue(CurDAG->getMachineNode(PPC::CMPWI, dl, MVT::i32, LHS,
-                                              getI32Imm((int)SImm & 0xFFFF)),
+                                              getI32Imm((int)SImm & 0xFFFF,
+                                                        dl)),
                          0);
       Opc = PPC::CMPW;
     }
@@ -1981,11 +1984,13 @@ SDValue PPCDAGToDAGISel::SelectCC(SDValue LHS, SDValue RHS,
         // SETEQ/SETNE comparison with 16-bit immediate, fold it.
         if (isUInt<16>(Imm))
           return SDValue(CurDAG->getMachineNode(PPC::CMPLDI, dl, MVT::i64, LHS,
-                                                getI32Imm(Imm & 0xFFFF)), 0);
+                                                getI32Imm(Imm & 0xFFFF, dl)),
+                         0);
         // If this is a 16-bit signed immediate, fold it.
         if (isInt<16>(Imm))
           return SDValue(CurDAG->getMachineNode(PPC::CMPDI, dl, MVT::i64, LHS,
-                                                getI32Imm(Imm & 0xFFFF)), 0);
+                                                getI32Imm(Imm & 0xFFFF, dl)),
+                         0);
 
         // For non-equality comparisons, the default code would materialize the
         // constant, then compare against it, like this:
@@ -1998,22 +2003,23 @@ SDValue PPCDAGToDAGISel::SelectCC(SDValue LHS, SDValue RHS,
         //   beq cr0,L6
         if (isUInt<32>(Imm)) {
           SDValue Xor(CurDAG->getMachineNode(PPC::XORIS8, dl, MVT::i64, LHS,
-                                             getI64Imm(Imm >> 16)), 0);
+                                             getI64Imm(Imm >> 16, dl)), 0);
           return SDValue(CurDAG->getMachineNode(PPC::CMPLDI, dl, MVT::i64, Xor,
-                                                getI64Imm(Imm & 0xFFFF)), 0);
+                                                getI64Imm(Imm & 0xFFFF, dl)),
+                         0);
         }
       }
       Opc = PPC::CMPLD;
     } else if (ISD::isUnsignedIntSetCC(CC)) {
       if (isInt64Immediate(RHS.getNode(), Imm) && isUInt<16>(Imm))
         return SDValue(CurDAG->getMachineNode(PPC::CMPLDI, dl, MVT::i64, LHS,
-                                              getI64Imm(Imm & 0xFFFF)), 0);
+                                              getI64Imm(Imm & 0xFFFF, dl)), 0);
       Opc = PPC::CMPLD;
     } else {
       short SImm;
       if (isIntS16Immediate(RHS, SImm))
         return SDValue(CurDAG->getMachineNode(PPC::CMPDI, dl, MVT::i64, LHS,
-                                              getI64Imm(SImm & 0xFFFF)),
+                                              getI64Imm(SImm & 0xFFFF, dl)),
                          0);
       Opc = PPC::CMPD;
     }
@@ -2201,7 +2207,8 @@ SDNode *PPCDAGToDAGISel::SelectSETCC(SDNode *N) {
   SDLoc dl(N);
   unsigned Imm;
   ISD::CondCode CC = cast<CondCodeSDNode>(N->getOperand(2))->get();
-  EVT PtrVT = CurDAG->getTargetLoweringInfo().getPointerTy();
+  EVT PtrVT =
+      CurDAG->getTargetLoweringInfo().getPointerTy(CurDAG->getDataLayout());
   bool isPPC64 = (PtrVT == MVT::i64);
 
   if (!PPCSubTarget->useCRBits() &&
@@ -2215,26 +2222,29 @@ SDNode *PPCDAGToDAGISel::SelectSETCC(SDNode *N) {
       default: break;
       case ISD::SETEQ: {
         Op = SDValue(CurDAG->getMachineNode(PPC::CNTLZW, dl, MVT::i32, Op), 0);
-        SDValue Ops[] = { Op, getI32Imm(27), getI32Imm(5), getI32Imm(31) };
+        SDValue Ops[] = { Op, getI32Imm(27, dl), getI32Imm(5, dl),
+                          getI32Imm(31, dl) };
         return CurDAG->SelectNodeTo(N, PPC::RLWINM, MVT::i32, Ops);
       }
       case ISD::SETNE: {
         if (isPPC64) break;
         SDValue AD =
           SDValue(CurDAG->getMachineNode(PPC::ADDIC, dl, MVT::i32, MVT::Glue,
-                                         Op, getI32Imm(~0U)), 0);
+                                         Op, getI32Imm(~0U, dl)), 0);
         return CurDAG->SelectNodeTo(N, PPC::SUBFE, MVT::i32, AD, Op,
                                     AD.getValue(1));
       }
       case ISD::SETLT: {
-        SDValue Ops[] = { Op, getI32Imm(1), getI32Imm(31), getI32Imm(31) };
+        SDValue Ops[] = { Op, getI32Imm(1, dl), getI32Imm(31, dl),
+                          getI32Imm(31, dl) };
         return CurDAG->SelectNodeTo(N, PPC::RLWINM, MVT::i32, Ops);
       }
       case ISD::SETGT: {
         SDValue T =
           SDValue(CurDAG->getMachineNode(PPC::NEG, dl, MVT::i32, Op), 0);
         T = SDValue(CurDAG->getMachineNode(PPC::ANDC, dl, MVT::i32, T, Op), 0);
-        SDValue Ops[] = { T, getI32Imm(1), getI32Imm(31), getI32Imm(31) };
+        SDValue Ops[] = { T, getI32Imm(1, dl), getI32Imm(31, dl),
+                          getI32Imm(31, dl) };
         return CurDAG->SelectNodeTo(N, PPC::RLWINM, MVT::i32, Ops);
       }
       }
@@ -2245,34 +2255,35 @@ SDNode *PPCDAGToDAGISel::SelectSETCC(SDNode *N) {
       case ISD::SETEQ:
         if (isPPC64) break;
         Op = SDValue(CurDAG->getMachineNode(PPC::ADDIC, dl, MVT::i32, MVT::Glue,
-                                            Op, getI32Imm(1)), 0);
+                                            Op, getI32Imm(1, dl)), 0);
         return CurDAG->SelectNodeTo(N, PPC::ADDZE, MVT::i32,
                               SDValue(CurDAG->getMachineNode(PPC::LI, dl,
                                                              MVT::i32,
-                                                             getI32Imm(0)), 0),
-                                      Op.getValue(1));
+                                                             getI32Imm(0, dl)),
+                                      0), Op.getValue(1));
       case ISD::SETNE: {
         if (isPPC64) break;
         Op = SDValue(CurDAG->getMachineNode(PPC::NOR, dl, MVT::i32, Op, Op), 0);
         SDNode *AD = CurDAG->getMachineNode(PPC::ADDIC, dl, MVT::i32, MVT::Glue,
-                                            Op, getI32Imm(~0U));
+                                            Op, getI32Imm(~0U, dl));
         return CurDAG->SelectNodeTo(N, PPC::SUBFE, MVT::i32, SDValue(AD, 0),
                                     Op, SDValue(AD, 1));
       }
       case ISD::SETLT: {
         SDValue AD = SDValue(CurDAG->getMachineNode(PPC::ADDI, dl, MVT::i32, Op,
-                                                    getI32Imm(1)), 0);
+                                                    getI32Imm(1, dl)), 0);
         SDValue AN = SDValue(CurDAG->getMachineNode(PPC::AND, dl, MVT::i32, AD,
                                                     Op), 0);
-        SDValue Ops[] = { AN, getI32Imm(1), getI32Imm(31), getI32Imm(31) };
+        SDValue Ops[] = { AN, getI32Imm(1, dl), getI32Imm(31, dl),
+                          getI32Imm(31, dl) };
         return CurDAG->SelectNodeTo(N, PPC::RLWINM, MVT::i32, Ops);
       }
       case ISD::SETGT: {
-        SDValue Ops[] = { Op, getI32Imm(1), getI32Imm(31), getI32Imm(31) };
-        Op = SDValue(CurDAG->getMachineNode(PPC::RLWINM, dl, MVT::i32, Ops),
-                     0);
+        SDValue Ops[] = { Op, getI32Imm(1, dl), getI32Imm(31, dl),
+                          getI32Imm(31, dl) };
+        Op = SDValue(CurDAG->getMachineNode(PPC::RLWINM, dl, MVT::i32, Ops), 0);
         return CurDAG->SelectNodeTo(N, PPC::XORI, MVT::i32, Op,
-                                    getI32Imm(1));
+                                    getI32Imm(1, dl));
       }
       }
     }
@@ -2322,15 +2333,15 @@ SDNode *PPCDAGToDAGISel::SelectSETCC(SDNode *N) {
   IntCR = SDValue(CurDAG->getMachineNode(PPC::MFOCRF, dl, MVT::i32, CR7Reg,
                                          CCReg), 0);
 
-  SDValue Ops[] = { IntCR, getI32Imm((32-(3-Idx)) & 31),
-                      getI32Imm(31), getI32Imm(31) };
+  SDValue Ops[] = { IntCR, getI32Imm((32 - (3 - Idx)) & 31, dl),
+                      getI32Imm(31, dl), getI32Imm(31, dl) };
   if (!Inv)
     return CurDAG->SelectNodeTo(N, PPC::RLWINM, MVT::i32, Ops);
 
   // Get the specified bit.
   SDValue Tmp =
     SDValue(CurDAG->getMachineNode(PPC::RLWINM, dl, MVT::i32, Ops), 0);
-  return CurDAG->SelectNodeTo(N, PPC::XORI, MVT::i32, Tmp, getI32Imm(1));
+  return CurDAG->SelectNodeTo(N, PPC::XORI, MVT::i32, Tmp, getI32Imm(1, dl));
 }
 
 SDNode *PPCDAGToDAGISel::transferMemOperands(SDNode *N, SDNode *Result) {
@@ -2398,7 +2409,8 @@ SDNode *PPCDAGToDAGISel::Select(SDNode *N) {
     SDValue N0 = N->getOperand(0);
     SDValue ShiftAmt =
       CurDAG->getTargetConstant(*cast<ConstantSDNode>(N->getOperand(1))->
-                                  getConstantIntValue(), N->getValueType(0));
+                                  getConstantIntValue(), dl,
+                                  N->getValueType(0));
     if (N->getValueType(0) == MVT::i64) {
       SDNode *Op =
         CurDAG->getMachineNode(PPC::SRADI, dl, MVT::i64, MVT::Glue,
@@ -2459,10 +2471,11 @@ SDNode *PPCDAGToDAGISel::Select(SDNode *N) {
       SDValue Chain = LD->getChain();
       SDValue Base = LD->getBasePtr();
       SDValue Ops[] = { Offset, Base, Chain };
-      return transferMemOperands(N, CurDAG->getMachineNode(Opcode, dl,
-                                      LD->getValueType(0),
-                                      PPCLowering->getPointerTy(),
-                                      MVT::Other, Ops));
+      return transferMemOperands(
+          N, CurDAG->getMachineNode(
+                 Opcode, dl, LD->getValueType(0),
+                 PPCLowering->getPointerTy(CurDAG->getDataLayout()), MVT::Other,
+                 Ops));
     } else {
       unsigned Opcode;
       bool isSExt = LD->getExtensionType() == ISD::SEXTLOAD;
@@ -2497,10 +2510,11 @@ SDNode *PPCDAGToDAGISel::Select(SDNode *N) {
       SDValue Chain = LD->getChain();
       SDValue Base = LD->getBasePtr();
       SDValue Ops[] = { Base, Offset, Chain };
-      return transferMemOperands(N, CurDAG->getMachineNode(Opcode, dl,
-                                      LD->getValueType(0),
-                                      PPCLowering->getPointerTy(),
-                                      MVT::Other, Ops));
+      return transferMemOperands(
+          N, CurDAG->getMachineNode(
+                 Opcode, dl, LD->getValueType(0),
+                 PPCLowering->getPointerTy(CurDAG->getDataLayout()), MVT::Other,
+                 Ops));
     }
   }
 
@@ -2513,7 +2527,8 @@ SDNode *PPCDAGToDAGISel::Select(SDNode *N) {
     if (isInt32Immediate(N->getOperand(1), Imm) &&
         isRotateAndMask(N->getOperand(0).getNode(), Imm, false, SH, MB, ME)) {
       SDValue Val = N->getOperand(0).getOperand(0);
-      SDValue Ops[] = { Val, getI32Imm(SH), getI32Imm(MB), getI32Imm(ME) };
+      SDValue Ops[] = { Val, getI32Imm(SH, dl), getI32Imm(MB, dl),
+                        getI32Imm(ME, dl) };
       return CurDAG->SelectNodeTo(N, PPC::RLWINM, MVT::i32, Ops);
     }
     // If this is just a masked value where the input is not handled above, and
@@ -2522,7 +2537,8 @@ SDNode *PPCDAGToDAGISel::Select(SDNode *N) {
         isRunOfOnes(Imm, MB, ME) &&
         N->getOperand(0).getOpcode() != ISD::ROTL) {
       SDValue Val = N->getOperand(0);
-      SDValue Ops[] = { Val, getI32Imm(0), getI32Imm(MB), getI32Imm(ME) };
+      SDValue Ops[] = { Val, getI32Imm(0, dl), getI32Imm(MB, dl),
+                        getI32Imm(ME, dl) };
       return CurDAG->SelectNodeTo(N, PPC::RLWINM, MVT::i32, Ops);
     }
     // If this is a 64-bit zero-extension mask, emit rldicl.
@@ -2544,7 +2560,7 @@ SDNode *PPCDAGToDAGISel::Select(SDNode *N) {
         SH = 64 - Imm;
       }
 
-      SDValue Ops[] = { Val, getI32Imm(SH), getI32Imm(MB) };
+      SDValue Ops[] = { Val, getI32Imm(SH, dl), getI32Imm(MB, dl) };
       return CurDAG->SelectNodeTo(N, PPC::RLDICL, MVT::i64, Ops);
     }
     // AND X, 0 -> 0, not "rlwinm 32".
@@ -2562,7 +2578,8 @@ SDNode *PPCDAGToDAGISel::Select(SDNode *N) {
       if (isRunOfOnes(Imm, MB, ME)) {
         SDValue Ops[] = { N->getOperand(0).getOperand(0),
                             N->getOperand(0).getOperand(1),
-                            getI32Imm(0), getI32Imm(MB),getI32Imm(ME) };
+                            getI32Imm(0, dl), getI32Imm(MB, dl),
+                            getI32Imm(ME, dl) };
         return CurDAG->getMachineNode(PPC::RLWIMI, dl, MVT::i32, Ops);
       }
     }
@@ -2603,7 +2620,8 @@ SDNode *PPCDAGToDAGISel::Select(SDNode *N) {
     if (isOpcWithIntImmediate(N->getOperand(0).getNode(), ISD::AND, Imm) &&
         isRotateAndMask(N, Imm, true, SH, MB, ME)) {
       SDValue Ops[] = { N->getOperand(0).getOperand(0),
-                          getI32Imm(SH), getI32Imm(MB), getI32Imm(ME) };
+                          getI32Imm(SH, dl), getI32Imm(MB, dl),
+                          getI32Imm(ME, dl) };
       return CurDAG->SelectNodeTo(N, PPC::RLWINM, MVT::i32, Ops);
     }
 
@@ -2615,7 +2633,8 @@ SDNode *PPCDAGToDAGISel::Select(SDNode *N) {
     if (isOpcWithIntImmediate(N->getOperand(0).getNode(), ISD::AND, Imm) &&
         isRotateAndMask(N, Imm, true, SH, MB, ME)) {
       SDValue Ops[] = { N->getOperand(0).getOperand(0),
-                          getI32Imm(SH), getI32Imm(MB), getI32Imm(ME) };
+                          getI32Imm(SH, dl), getI32Imm(MB, dl),
+                          getI32Imm(ME, dl) };
       return CurDAG->SelectNodeTo(N, PPC::RLWINM, MVT::i32, Ops);
     }
 
@@ -2635,11 +2654,12 @@ SDNode *PPCDAGToDAGISel::Select(SDNode *N) {
     unsigned Opcode = (InVT == MVT::i64) ? PPC::ANDIo8 : PPC::ANDIo;
     SDValue AndI(CurDAG->getMachineNode(Opcode, dl, InVT, MVT::Glue,
                                         N->getOperand(0),
-                                        CurDAG->getTargetConstant(1, InVT)), 0);
+                                        CurDAG->getTargetConstant(1, dl, InVT)),
+                 0);
     SDValue CR0Reg = CurDAG->getRegister(PPC::CR0, MVT::i32);
     SDValue SRIdxVal =
       CurDAG->getTargetConstant(N->getOpcode() == PPCISD::ANDIo_1_EQ_BIT ?
-                                PPC::sub_eq : PPC::sub_gt, MVT::i32);
+                                PPC::sub_eq : PPC::sub_gt, dl, MVT::i32);
 
     return CurDAG->SelectNodeTo(N, TargetOpcode::EXTRACT_SUBREG, MVT::i1,
                                 CR0Reg, SRIdxVal,
@@ -2647,7 +2667,8 @@ SDNode *PPCDAGToDAGISel::Select(SDNode *N) {
   }
   case ISD::SELECT_CC: {
     ISD::CondCode CC = cast<CondCodeSDNode>(N->getOperand(4))->get();
-    EVT PtrVT = CurDAG->getTargetLoweringInfo().getPointerTy();
+    EVT PtrVT =
+        CurDAG->getTargetLoweringInfo().getPointerTy(CurDAG->getDataLayout());
     bool isPPC64 = (PtrVT == MVT::i64);
 
     // If this is a select of i1 operands, we'll pattern match it.
@@ -2666,7 +2687,7 @@ SDNode *PPCDAGToDAGISel::Select(SDNode *N) {
                 N->getValueType(0) == MVT::i32) {
               SDNode *Tmp =
                 CurDAG->getMachineNode(PPC::ADDIC, dl, MVT::i32, MVT::Glue,
-                                       N->getOperand(0), getI32Imm(~0U));
+                                       N->getOperand(0), getI32Imm(~0U, dl));
               return CurDAG->SelectNodeTo(N, PPC::SUBFE, MVT::i32,
                                           SDValue(Tmp, 0), N->getOperand(0),
                                           SDValue(Tmp, 1));
@@ -2711,7 +2732,10 @@ SDNode *PPCDAGToDAGISel::Select(SDNode *N) {
     else if (N->getValueType(0) == MVT::i64)
       SelectCCOp = PPC::SELECT_CC_I8;
     else if (N->getValueType(0) == MVT::f32)
-      SelectCCOp = PPC::SELECT_CC_F4;
+      if (PPCSubTarget->hasP8Vector())
+        SelectCCOp = PPC::SELECT_CC_VSSRC;
+      else
+        SelectCCOp = PPC::SELECT_CC_F4;
     else if (N->getValueType(0) == MVT::f64)
       if (PPCSubTarget->hasVSX())
         SelectCCOp = PPC::SELECT_CC_VSFRC;
@@ -2730,7 +2754,7 @@ SDNode *PPCDAGToDAGISel::Select(SDNode *N) {
       SelectCCOp = PPC::SELECT_CC_VRRC;
 
     SDValue Ops[] = { CCReg, N->getOperand(2), N->getOperand(3),
-                        getI32Imm(BROpc) };
+                        getI32Imm(BROpc, dl) };
     return CurDAG->SelectNodeTo(N, SelectCCOp, N->getValueType(0), Ops);
   }
   case ISD::VSELECT:
@@ -2755,17 +2779,6 @@ SDNode *PPCDAGToDAGISel::Select(SDNode *N) {
         else
           DM[i] = 1;
 
-      // For little endian, we must swap the input operands and adjust
-      // the mask elements (reverse and invert them).
-      if (PPCSubTarget->isLittleEndian()) {
-        std::swap(Op1, Op2);
-        unsigned tmp = DM[0];
-        DM[0] = 1 - DM[1];
-        DM[1] = 1 - tmp;
-      }
-
-      SDValue DMV = CurDAG->getTargetConstant(DM[1] | (DM[0] << 1), MVT::i32);
-
       if (Op1 == Op2 && DM[0] == 0 && DM[1] == 0 &&
           Op1.getOpcode() == ISD::SCALAR_TO_VECTOR &&
           isa<LoadSDNode>(Op1.getOperand(0))) {
@@ -2781,6 +2794,17 @@ SDNode *PPCDAGToDAGISel::Select(SDNode *N) {
         }
       }
 
+      // For little endian, we must swap the input operands and adjust
+      // the mask elements (reverse and invert them).
+      if (PPCSubTarget->isLittleEndian()) {
+        std::swap(Op1, Op2);
+        unsigned tmp = DM[0];
+        DM[0] = 1 - DM[1];
+        DM[1] = 1 - tmp;
+      }
+
+      SDValue DMV = CurDAG->getTargetConstant(DM[1] | (DM[0] << 1), dl,
+                                              MVT::i32);
       SDValue Ops[] = { Op1, Op2, DMV };
       return CurDAG->SelectNodeTo(N, PPC::XXPERMDI, N->getValueType(0), Ops);
     }
@@ -2803,7 +2827,7 @@ SDNode *PPCDAGToDAGISel::Select(SDNode *N) {
     // Op #4 is the Flag.
     // Prevent PPC::PRED_* from being selected into LI.
     SDValue Pred =
-      getI32Imm(cast<ConstantSDNode>(N->getOperand(1))->getZExtValue());
+      getI32Imm(cast<ConstantSDNode>(N->getOperand(1))->getZExtValue(), dl);
     SDValue Ops[] = { Pred, N->getOperand(2), N->getOperand(3),
       N->getOperand(0), N->getOperand(4) };
     return CurDAG->SelectNodeTo(N, PPC::BCC, MVT::Other, Ops);
@@ -2833,7 +2857,7 @@ SDNode *PPCDAGToDAGISel::Select(SDNode *N) {
     }
 
     SDValue CondCode = SelectCC(N->getOperand(2), N->getOperand(3), CC, dl);
-    SDValue Ops[] = { getI32Imm(PCC), CondCode,
+    SDValue Ops[] = { getI32Imm(PCC, dl), CondCode,
                         N->getOperand(4), N->getOperand(0) };
     return CurDAG->SelectNodeTo(N, PPC::BCC, MVT::Other, Ops);
   }
@@ -2883,7 +2907,7 @@ SDNode *PPCDAGToDAGISel::Select(SDNode *N) {
     if (GlobalAddressSDNode *G = dyn_cast<GlobalAddressSDNode>(GA)) {
       const GlobalValue *GValue = G->getGlobal();
       if ((GValue->getType()->getElementType()->isFunctionTy() &&
-           (GValue->isDeclaration() || GValue->isWeakForLinker())) ||
+           !GValue->isStrongDefinitionForLinker()) ||
           GValue->isDeclaration() || GValue->hasCommonLinkage() ||
           GValue->hasAvailableExternallyLinkage())
         return transferMemOperands(N, CurDAG->getMachineNode(PPC::LDtocL, dl,
@@ -2897,7 +2921,9 @@ SDNode *PPCDAGToDAGISel::Select(SDNode *N) {
     // Generate a PIC-safe GOT reference.
     assert(!PPCSubTarget->isPPC64() && PPCSubTarget->isSVR4ABI() &&
       "PPCISD::PPC32_PICGOT is only supported for 32-bit SVR4");
-    return CurDAG->SelectNodeTo(N, PPC::PPC32PICGOT, PPCLowering->getPointerTy(),  MVT::i32);
+    return CurDAG->SelectNodeTo(
+        N, PPC::PPC32PICGOT, PPCLowering->getPointerTy(CurDAG->getDataLayout()),
+        MVT::i32);
   }
   case PPCISD::VADD_SPLAT: {
     // This expands into one of three sequences, depending on whether
@@ -2936,7 +2962,7 @@ SDNode *PPCDAGToDAGISel::Select(SDNode *N) {
       // Into:    tmp = VSPLTIS[BHW] elt
       //          VADDU[BHW]M tmp, tmp
       // Where:   [BHW] = B for size = 1, H for size = 2, W for size = 4
-      SDValue EltVal = getI32Imm(Elt >> 1);
+      SDValue EltVal = getI32Imm(Elt >> 1, dl);
       SDNode *Tmp = CurDAG->getMachineNode(Opc1, dl, VT, EltVal);
       SDValue TmpVal = SDValue(Tmp, 0);
       return CurDAG->getMachineNode(Opc2, dl, VT, TmpVal, TmpVal);
@@ -2948,9 +2974,9 @@ SDNode *PPCDAGToDAGISel::Select(SDNode *N) {
       // Into:    tmp1 = VSPLTIS[BHW] elt-16
       //          tmp2 = VSPLTIS[BHW] -16
       //          VSUBU[BHW]M tmp1, tmp2
-      SDValue EltVal = getI32Imm(Elt - 16);
+      SDValue EltVal = getI32Imm(Elt - 16, dl);
       SDNode *Tmp1 = CurDAG->getMachineNode(Opc1, dl, VT, EltVal);
-      EltVal = getI32Imm(-16);
+      EltVal = getI32Imm(-16, dl);
       SDNode *Tmp2 = CurDAG->getMachineNode(Opc1, dl, VT, EltVal);
       return CurDAG->getMachineNode(Opc3, dl, VT, SDValue(Tmp1, 0),
                                     SDValue(Tmp2, 0));
@@ -2962,9 +2988,9 @@ SDNode *PPCDAGToDAGISel::Select(SDNode *N) {
       // Into:    tmp1 = VSPLTIS[BHW] elt+16
       //          tmp2 = VSPLTIS[BHW] -16
       //          VADDU[BHW]M tmp1, tmp2
-      SDValue EltVal = getI32Imm(Elt + 16);
+      SDValue EltVal = getI32Imm(Elt + 16, dl);
       SDNode *Tmp1 = CurDAG->getMachineNode(Opc1, dl, VT, EltVal);
-      EltVal = getI32Imm(-16);
+      EltVal = getI32Imm(-16, dl);
       SDNode *Tmp2 = CurDAG->getMachineNode(Opc1, dl, VT, EltVal);
       return CurDAG->getMachineNode(Opc2, dl, VT, SDValue(Tmp1, 0),
                                     SDValue(Tmp2, 0));
@@ -3173,7 +3199,8 @@ SDValue PPCDAGToDAGISel::combineToCMPB(SDNode *N) {
   bool NonTrivialMask = ((int64_t) Mask) != INT64_C(-1);
   if (NonTrivialMask && !Alt) {
     // Res = Mask & CMPB
-    Res = CurDAG->getNode(ISD::AND, dl, VT, Res, CurDAG->getConstant(Mask, VT));
+    Res = CurDAG->getNode(ISD::AND, dl, VT, Res,
+                          CurDAG->getConstant(Mask, dl, VT));
   } else if (Alt) {
     // Res = (CMPB & Mask) | (~CMPB & Alt)
     // Which, as suggested here:
@@ -3182,8 +3209,9 @@ SDValue PPCDAGToDAGISel::combineToCMPB(SDNode *N) {
     // Res = Alt ^ ((Alt ^ Mask) & CMPB)
     // useful because the (Alt ^ Mask) can be pre-computed.
     Res = CurDAG->getNode(ISD::AND, dl, VT, Res,
-                          CurDAG->getConstant(Mask ^ Alt, VT));
-    Res = CurDAG->getNode(ISD::XOR, dl, VT, Res, CurDAG->getConstant(Alt, VT));
+                          CurDAG->getConstant(Mask ^ Alt, dl, VT));
+    Res = CurDAG->getNode(ISD::XOR, dl, VT, Res,
+                          CurDAG->getConstant(Alt, dl, VT));
   }
 
   return Res;
@@ -3215,20 +3243,20 @@ void PPCDAGToDAGISel::foldBoolExts(SDValue &Res, SDNode *&N) {
   EVT VT = N->getValueType(0);
   SDValue Cond = N->getOperand(0);
   SDValue ConstTrue =
-    CurDAG->getConstant(N->getOpcode() == ISD::SIGN_EXTEND ? -1 : 1, VT);
-  SDValue ConstFalse = CurDAG->getConstant(0, VT);
+    CurDAG->getConstant(N->getOpcode() == ISD::SIGN_EXTEND ? -1 : 1, dl, VT);
+  SDValue ConstFalse = CurDAG->getConstant(0, dl, VT);
 
   do {
     SDNode *User = *N->use_begin();
     if (User->getNumOperands() != 2)
       break;
 
-    auto TryFold = [this, N, User](SDValue Val) {
+    auto TryFold = [this, N, User, dl](SDValue Val) {
       SDValue UserO0 = User->getOperand(0), UserO1 = User->getOperand(1);
       SDValue O0 = UserO0.getNode() == N ? Val : UserO0;
       SDValue O1 = UserO1.getNode() == N ? Val : UserO1;
 
-      return CurDAG->FoldConstantArithmetic(User->getOpcode(),
+      return CurDAG->FoldConstantArithmetic(User->getOpcode(), dl,
                                             User->getValueType(0),
                                             O0.getNode(), O1.getNode());
     };
@@ -3378,9 +3406,8 @@ void PPCDAGToDAGISel::PeepholeCROps() {
   bool IsModified;
   do {
     IsModified = false;
-    for (SelectionDAG::allnodes_iterator I = CurDAG->allnodes_begin(),
-         E = CurDAG->allnodes_end(); I != E; ++I) {
-      MachineSDNode *MachineNode = dyn_cast<MachineSDNode>(I);
+    for (SDNode &Node : CurDAG->allnodes()) {
+      MachineSDNode *MachineNode = dyn_cast<MachineSDNode>(&Node);
       if (!MachineNode || MachineNode->use_empty())
         continue;
       SDNode *ResNode = MachineNode;
@@ -3423,6 +3450,7 @@ void PPCDAGToDAGISel::PeepholeCROps() {
       case PPC::SELECT_QBRC:
       case PPC::SELECT_VRRC:
       case PPC::SELECT_VSFRC:
+      case PPC::SELECT_VSSRC:
       case PPC::SELECT_VSRC: {
         SDValue Op = MachineNode->getOperand(0);
         if (Op.isMachineOpcode()) {
@@ -3733,6 +3761,7 @@ void PPCDAGToDAGISel::PeepholeCROps() {
       case PPC::SELECT_QBRC:
       case PPC::SELECT_VRRC:
       case PPC::SELECT_VSFRC:
+      case PPC::SELECT_VSSRC:
       case PPC::SELECT_VSRC:
         if (Op1Set)
           ResNode = MachineNode->getOperand(1).getNode();
diff --git a/lib/Target/PowerPC/PPCISelLowering.cpp b/lib/Target/PowerPC/PPCISelLowering.cpp
index 4c0b6a6e871..94171a6cc47 100644
--- a/lib/Target/PowerPC/PPCISelLowering.cpp
+++ b/lib/Target/PowerPC/PPCISelLowering.cpp
@@ -39,6 +39,7 @@
 #include "llvm/Support/MathExtras.h"
 #include "llvm/Support/raw_ostream.h"
 #include "llvm/Target/TargetOptions.h"
+
 using namespace llvm;
 
 // FIXME: Remove this once soft-float is supported.
@@ -404,9 +405,9 @@ PPCTargetLowering::PPCTargetLowering(const PPCTargetMachine &TM,
       // add/sub are legal for all supported vector VT's.
       setOperationAction(ISD::ADD , VT, Legal);
       setOperationAction(ISD::SUB , VT, Legal);
-
+      
       // Vector instructions introduced in P8
-      if (Subtarget.hasP8Altivec()) {
+      if (Subtarget.hasP8Altivec() && (VT.SimpleTy != MVT::v1i128)) {
         setOperationAction(ISD::CTPOP, VT, Legal);
         setOperationAction(ISD::CTLZ, VT, Legal);
       }
@@ -574,8 +575,12 @@ PPCTargetLowering::PPCTargetLowering(const PPCTargetMachine &TM,
 
       setOperationAction(ISD::VECTOR_SHUFFLE, MVT::v2f64, Legal);
 
+      if (Subtarget.hasP8Vector())
+        addRegisterClass(MVT::f32, &PPC::VSSRCRegClass);
+
       addRegisterClass(MVT::f64, &PPC::VSFRCRegClass);
 
+      addRegisterClass(MVT::v4i32, &PPC::VSRCRegClass);
       addRegisterClass(MVT::v4f32, &PPC::VSRCRegClass);
       addRegisterClass(MVT::v2f64, &PPC::VSRCRegClass);
 
@@ -620,8 +625,10 @@ PPCTargetLowering::PPCTargetLowering(const PPCTargetMachine &TM,
       addRegisterClass(MVT::v2i64, &PPC::VSRCRegClass);
     }
 
-    if (Subtarget.hasP8Altivec()) 
+    if (Subtarget.hasP8Altivec()) {
       addRegisterClass(MVT::v2i64, &PPC::VRRCRegClass);
+      addRegisterClass(MVT::v1i128, &PPC::VRRCRegClass);
+    }
   }
 
   if (Subtarget.hasQPX()) {
@@ -933,9 +940,9 @@ static void getMaxByValAlign(Type *Ty, unsigned &MaxAlign,
     if (EltAlign > MaxAlign)
       MaxAlign = EltAlign;
   } else if (StructType *STy = dyn_cast<StructType>(Ty)) {
-    for (unsigned i = 0, e = STy->getNumElements(); i != e; ++i) {
+    for (auto *EltTy : STy->elements()) {
       unsigned EltAlign = 0;
-      getMaxByValAlign(STy->getElementType(i), EltAlign, MaxMaxAlign);
+      getMaxByValAlign(EltTy, EltAlign, MaxMaxAlign);
       if (EltAlign > MaxAlign)
         MaxAlign = EltAlign;
       if (MaxAlign == MaxMaxAlign)
@@ -946,7 +953,8 @@ static void getMaxByValAlign(Type *Ty, unsigned &MaxAlign,
 
 /// getByValTypeAlignment - Return the desired alignment for ByVal aggregate
 /// function arguments in the caller parameter area.
-unsigned PPCTargetLowering::getByValTypeAlignment(Type *Ty) const {
+unsigned PPCTargetLowering::getByValTypeAlignment(Type *Ty,
+                                                  const DataLayout &DL) const {
   // Darwin passes everything on 4 byte boundary.
   if (Subtarget.isDarwin())
     return 4;
@@ -960,8 +968,8 @@ unsigned PPCTargetLowering::getByValTypeAlignment(Type *Ty) const {
 }
 
 const char *PPCTargetLowering::getTargetNodeName(unsigned Opcode) const {
-  switch (Opcode) {
-  default: return nullptr;
+  switch ((PPCISD::NodeType)Opcode) {
+  case PPCISD::FIRST_NUMBER:    break;
   case PPCISD::FSEL:            return "PPCISD::FSEL";
   case PPCISD::FCFID:           return "PPCISD::FCFID";
   case PPCISD::FCFIDU:          return "PPCISD::FCFIDU";
@@ -986,6 +994,7 @@ const char *PPCTargetLowering::getTargetNodeName(unsigned Opcode) const {
   case PPCISD::SRL:             return "PPCISD::SRL";
   case PPCISD::SRA:             return "PPCISD::SRA";
   case PPCISD::SHL:             return "PPCISD::SHL";
+  case PPCISD::SRA_ADDZE:       return "PPCISD::SRA_ADDZE";
   case PPCISD::CALL:            return "PPCISD::CALL";
   case PPCISD::CALL_NOP:        return "PPCISD::CALL_NOP";
   case PPCISD::MTCTR:           return "PPCISD::MTCTR";
@@ -999,12 +1008,16 @@ const char *PPCTargetLowering::getTargetNodeName(unsigned Opcode) const {
   case PPCISD::MFVSR:           return "PPCISD::MFVSR";
   case PPCISD::MTVSRA:          return "PPCISD::MTVSRA";
   case PPCISD::MTVSRZ:          return "PPCISD::MTVSRZ";
+  case PPCISD::ANDIo_1_EQ_BIT:  return "PPCISD::ANDIo_1_EQ_BIT";
+  case PPCISD::ANDIo_1_GT_BIT:  return "PPCISD::ANDIo_1_GT_BIT";
   case PPCISD::VCMP:            return "PPCISD::VCMP";
   case PPCISD::VCMPo:           return "PPCISD::VCMPo";
   case PPCISD::LBRX:            return "PPCISD::LBRX";
   case PPCISD::STBRX:           return "PPCISD::STBRX";
   case PPCISD::LFIWAX:          return "PPCISD::LFIWAX";
   case PPCISD::LFIWZX:          return "PPCISD::LFIWZX";
+  case PPCISD::LXVD2X:          return "PPCISD::LXVD2X";
+  case PPCISD::STXVD2X:         return "PPCISD::STXVD2X";
   case PPCISD::COND_BRANCH:     return "PPCISD::COND_BRANCH";
   case PPCISD::BDNZ:            return "PPCISD::BDNZ";
   case PPCISD::BDZ:             return "PPCISD::BDZ";
@@ -1014,6 +1027,7 @@ const char *PPCTargetLowering::getTargetNodeName(unsigned Opcode) const {
   case PPCISD::CR6SET:          return "PPCISD::CR6SET";
   case PPCISD::CR6UNSET:        return "PPCISD::CR6UNSET";
   case PPCISD::PPC32_GOT:       return "PPCISD::PPC32_GOT";
+  case PPCISD::PPC32_PICGOT:    return "PPCISD::PPC32_PICGOT";
   case PPCISD::ADDIS_GOT_TPREL_HA: return "PPCISD::ADDIS_GOT_TPREL_HA";
   case PPCISD::LD_GOT_TPREL_L:  return "PPCISD::LD_GOT_TPREL_L";
   case PPCISD::ADD_TLS:         return "PPCISD::ADD_TLS";
@@ -1029,6 +1043,10 @@ const char *PPCTargetLowering::getTargetNodeName(unsigned Opcode) const {
   case PPCISD::ADDI_DTPREL_L:   return "PPCISD::ADDI_DTPREL_L";
   case PPCISD::VADD_SPLAT:      return "PPCISD::VADD_SPLAT";
   case PPCISD::SC:              return "PPCISD::SC";
+  case PPCISD::CLRBHRB:         return "PPCISD::CLRBHRB";
+  case PPCISD::MFBHRBE:         return "PPCISD::MFBHRBE";
+  case PPCISD::RFEBB:           return "PPCISD::RFEBB";
+  case PPCISD::XXSWAPD:         return "PPCISD::XXSWAPD";
   case PPCISD::QVFPERM:         return "PPCISD::QVFPERM";
   case PPCISD::QVGPCI:          return "PPCISD::QVGPCI";
   case PPCISD::QVALIGNI:        return "PPCISD::QVALIGNI";
@@ -1036,9 +1054,11 @@ const char *PPCTargetLowering::getTargetNodeName(unsigned Opcode) const {
   case PPCISD::QBFLT:           return "PPCISD::QBFLT";
   case PPCISD::QVLFSb:          return "PPCISD::QVLFSb";
   }
+  return nullptr;
 }
 
-EVT PPCTargetLowering::getSetCCResultType(LLVMContext &C, EVT VT) const {
+EVT PPCTargetLowering::getSetCCResultType(const DataLayout &DL, LLVMContext &C,
+                                          EVT VT) const {
   if (!VT.isVector())
     return Subtarget.useCRBits() ? MVT::i1 : MVT::i32;
 
@@ -1080,11 +1100,11 @@ static bool isConstantOrUndef(int Op, int Val) {
 /// VPKUHUM instruction.
 /// The ShuffleKind distinguishes between big-endian operations with
 /// two different inputs (0), either-endian operations with two identical
-/// inputs (1), and little-endian operantion with two different inputs (2).
+/// inputs (1), and little-endian operations with two different inputs (2).
 /// For the latter, the input operands are swapped (see PPCInstrAltivec.td).
 bool PPC::isVPKUHUMShuffleMask(ShuffleVectorSDNode *N, unsigned ShuffleKind,
                                SelectionDAG &DAG) {
-  bool IsLE = DAG.getTarget().getDataLayout()->isLittleEndian();
+  bool IsLE = DAG.getDataLayout().isLittleEndian();
   if (ShuffleKind == 0) {
     if (IsLE)
       return false;
@@ -1111,11 +1131,11 @@ bool PPC::isVPKUHUMShuffleMask(ShuffleVectorSDNode *N, unsigned ShuffleKind,
 /// VPKUWUM instruction.
 /// The ShuffleKind distinguishes between big-endian operations with
 /// two different inputs (0), either-endian operations with two identical
-/// inputs (1), and little-endian operantion with two different inputs (2).
+/// inputs (1), and little-endian operations with two different inputs (2).
 /// For the latter, the input operands are swapped (see PPCInstrAltivec.td).
 bool PPC::isVPKUWUMShuffleMask(ShuffleVectorSDNode *N, unsigned ShuffleKind,
                                SelectionDAG &DAG) {
-  bool IsLE = DAG.getTarget().getDataLayout()->isLittleEndian();
+  bool IsLE = DAG.getDataLayout().isLittleEndian();
   if (ShuffleKind == 0) {
     if (IsLE)
       return false;
@@ -1142,6 +1162,56 @@ bool PPC::isVPKUWUMShuffleMask(ShuffleVectorSDNode *N, unsigned ShuffleKind,
   return true;
 }
 
+/// isVPKUDUMShuffleMask - Return true if this is the shuffle mask for a
+/// VPKUDUM instruction, AND the VPKUDUM instruction exists for the
+/// current subtarget.
+///
+/// The ShuffleKind distinguishes between big-endian operations with
+/// two different inputs (0), either-endian operations with two identical
+/// inputs (1), and little-endian operations with two different inputs (2).
+/// For the latter, the input operands are swapped (see PPCInstrAltivec.td).
+bool PPC::isVPKUDUMShuffleMask(ShuffleVectorSDNode *N, unsigned ShuffleKind,
+                               SelectionDAG &DAG) {
+  const PPCSubtarget& Subtarget =
+    static_cast<const PPCSubtarget&>(DAG.getSubtarget());
+  if (!Subtarget.hasP8Vector())
+    return false;
+
+  bool IsLE = DAG.getDataLayout().isLittleEndian();
+  if (ShuffleKind == 0) {
+    if (IsLE)
+      return false;
+    for (unsigned i = 0; i != 16; i += 4)
+      if (!isConstantOrUndef(N->getMaskElt(i  ),  i*2+4) ||
+          !isConstantOrUndef(N->getMaskElt(i+1),  i*2+5) ||
+          !isConstantOrUndef(N->getMaskElt(i+2),  i*2+6) ||
+          !isConstantOrUndef(N->getMaskElt(i+3),  i*2+7))
+        return false;
+  } else if (ShuffleKind == 2) {
+    if (!IsLE)
+      return false;
+    for (unsigned i = 0; i != 16; i += 4)
+      if (!isConstantOrUndef(N->getMaskElt(i  ),  i*2) ||
+          !isConstantOrUndef(N->getMaskElt(i+1),  i*2+1) ||
+          !isConstantOrUndef(N->getMaskElt(i+2),  i*2+2) ||
+          !isConstantOrUndef(N->getMaskElt(i+3),  i*2+3))
+        return false;
+  } else if (ShuffleKind == 1) {
+    unsigned j = IsLE ? 0 : 4;
+    for (unsigned i = 0; i != 8; i += 4)
+      if (!isConstantOrUndef(N->getMaskElt(i  ),  i*2+j)   ||
+          !isConstantOrUndef(N->getMaskElt(i+1),  i*2+j+1) ||
+          !isConstantOrUndef(N->getMaskElt(i+2),  i*2+j+2) ||
+          !isConstantOrUndef(N->getMaskElt(i+3),  i*2+j+3) ||
+          !isConstantOrUndef(N->getMaskElt(i+8),  i*2+j)   ||
+          !isConstantOrUndef(N->getMaskElt(i+9),  i*2+j+1) ||
+          !isConstantOrUndef(N->getMaskElt(i+10), i*2+j+2) ||
+          !isConstantOrUndef(N->getMaskElt(i+11), i*2+j+3))
+        return false;
+  }
+  return true;
+}
+
 /// isVMerge - Common function, used to match vmrg* shuffles.
 ///
 static bool isVMerge(ShuffleVectorSDNode *N, unsigned UnitSize,
@@ -1170,7 +1240,7 @@ static bool isVMerge(ShuffleVectorSDNode *N, unsigned UnitSize,
 /// the input operands are swapped (see PPCInstrAltivec.td).
 bool PPC::isVMRGLShuffleMask(ShuffleVectorSDNode *N, unsigned UnitSize,
                              unsigned ShuffleKind, SelectionDAG &DAG) {
-  if (DAG.getTarget().getDataLayout()->isLittleEndian()) {
+  if (DAG.getDataLayout().isLittleEndian()) {
     if (ShuffleKind == 1) // unary
       return isVMerge(N, UnitSize, 0, 0);
     else if (ShuffleKind == 2) // swapped
@@ -1195,7 +1265,7 @@ bool PPC::isVMRGLShuffleMask(ShuffleVectorSDNode *N, unsigned UnitSize,
 /// the input operands are swapped (see PPCInstrAltivec.td).
 bool PPC::isVMRGHShuffleMask(ShuffleVectorSDNode *N, unsigned UnitSize,
                              unsigned ShuffleKind, SelectionDAG &DAG) {
-  if (DAG.getTarget().getDataLayout()->isLittleEndian()) {
+  if (DAG.getDataLayout().isLittleEndian()) {
     if (ShuffleKind == 1) // unary
       return isVMerge(N, UnitSize, 8, 8);
     else if (ShuffleKind == 2) // swapped
@@ -1212,6 +1282,99 @@ bool PPC::isVMRGHShuffleMask(ShuffleVectorSDNode *N, unsigned UnitSize,
   }
 }
 
+/**
+ * \brief Common function used to match vmrgew and vmrgow shuffles
+ *
+ * The indexOffset determines whether to look for even or odd words in
+ * the shuffle mask. This is based on the of the endianness of the target
+ * machine.
+ *   - Little Endian:
+ *     - Use offset of 0 to check for odd elements
+ *     - Use offset of 4 to check for even elements
+ *   - Big Endian:
+ *     - Use offset of 0 to check for even elements
+ *     - Use offset of 4 to check for odd elements
+ * A detailed description of the vector element ordering for little endian and
+ * big endian can be found at
+ * http://www.ibm.com/developerworks/library/l-ibm-xl-c-cpp-compiler/index.html
+ * Targeting your applications - what little endian and big endian IBM XL C/C++
+ * compiler differences mean to you
+ *
+ * The mask to the shuffle vector instruction specifies the indices of the
+ * elements from the two input vectors to place in the result. The elements are
+ * numbered in array-access order, starting with the first vector. These vectors
+ * are always of type v16i8, thus each vector will contain 16 elements of size
+ * 8. More info on the shuffle vector can be found in the
+ * http://llvm.org/docs/LangRef.html#shufflevector-instruction
+ * Language Reference.
+ *
+ * The RHSStartValue indicates whether the same input vectors are used (unary)
+ * or two different input vectors are used, based on the following:
+ *   - If the instruction uses the same vector for both inputs, the range of the
+ *     indices will be 0 to 15. In this case, the RHSStart value passed should
+ *     be 0.
+ *   - If the instruction has two different vectors then the range of the
+ *     indices will be 0 to 31. In this case, the RHSStart value passed should
+ *     be 16 (indices 0-15 specify elements in the first vector while indices 16
+ *     to 31 specify elements in the second vector).
+ *
+ * \param[in] N The shuffle vector SD Node to analyze
+ * \param[in] IndexOffset Specifies whether to look for even or odd elements
+ * \param[in] RHSStartValue Specifies the starting index for the righthand input
+ * vector to the shuffle_vector instruction
+ * \return true iff this shuffle vector represents an even or odd word merge
+ */
+static bool isVMerge(ShuffleVectorSDNode *N, unsigned IndexOffset,
+                     unsigned RHSStartValue) {
+  if (N->getValueType(0) != MVT::v16i8)
+    return false;
+
+  for (unsigned i = 0; i < 2; ++i)
+    for (unsigned j = 0; j < 4; ++j)
+      if (!isConstantOrUndef(N->getMaskElt(i*4+j),
+                             i*RHSStartValue+j+IndexOffset) ||
+          !isConstantOrUndef(N->getMaskElt(i*4+j+8),
+                             i*RHSStartValue+j+IndexOffset+8))
+        return false;
+  return true;
+}
+
+/**
+ * \brief Determine if the specified shuffle mask is suitable for the vmrgew or
+ * vmrgow instructions.
+ *
+ * \param[in] N The shuffle vector SD Node to analyze
+ * \param[in] CheckEven Check for an even merge (true) or an odd merge (false)
+ * \param[in] ShuffleKind Identify the type of merge:
+ *   - 0 = big-endian merge with two different inputs;
+ *   - 1 = either-endian merge with two identical inputs;
+ *   - 2 = little-endian merge with two different inputs (inputs are swapped for
+ *     little-endian merges).
+ * \param[in] DAG The current SelectionDAG
+ * \return true iff this shuffle mask 
+ */
+bool PPC::isVMRGEOShuffleMask(ShuffleVectorSDNode *N, bool CheckEven,
+                              unsigned ShuffleKind, SelectionDAG &DAG) {
+  if (DAG.getDataLayout().isLittleEndian()) {
+    unsigned indexOffset = CheckEven ? 4 : 0;
+    if (ShuffleKind == 1) // Unary
+      return isVMerge(N, indexOffset, 0);
+    else if (ShuffleKind == 2) // swapped
+      return isVMerge(N, indexOffset, 16);
+    else
+      return false;
+  }
+  else {
+    unsigned indexOffset = CheckEven ? 0 : 4;
+    if (ShuffleKind == 1) // Unary
+      return isVMerge(N, indexOffset, 0);
+    else if (ShuffleKind == 0) // Normal
+      return isVMerge(N, indexOffset, 16);
+    else
+      return false;
+  }
+  return false;
+}
 
 /// isVSLDOIShuffleMask - If this is a vsldoi shuffle mask, return the shift
 /// amount, otherwise return -1.
@@ -1239,7 +1402,7 @@ int PPC::isVSLDOIShuffleMask(SDNode *N, unsigned ShuffleKind,
   if (ShiftAmt < i) return -1;
 
   ShiftAmt -= i;
-  bool isLE = DAG.getTarget().getDataLayout()->isLittleEndian();
+  bool isLE = DAG.getDataLayout().isLittleEndian();
 
   if ((ShuffleKind == 0 && !isLE) || (ShuffleKind == 2 && isLE)) {
     // Check the rest of the elements to see if they are consecutive.
@@ -1254,7 +1417,7 @@ int PPC::isVSLDOIShuffleMask(SDNode *N, unsigned ShuffleKind,
   } else
     return -1;
 
-  if (ShuffleKind == 2 && isLE)
+  if (isLE)
     ShiftAmt = 16 - ShiftAmt;
 
   return ShiftAmt;
@@ -1267,6 +1430,11 @@ bool PPC::isSplatShuffleMask(ShuffleVectorSDNode *N, unsigned EltSize) {
   assert(N->getValueType(0) == MVT::v16i8 &&
          (EltSize == 1 || EltSize == 2 || EltSize == 4));
 
+  // The consecutive indices need to specify an element, not part of two
+  // different elements.  So abandon ship early if this isn't the case.
+  if (N->getMaskElt(0) % EltSize != 0)
+    return false;
+
   // This is a splat operation if each element of the permute is the same, and
   // if the value doesn't reference the second vector.
   unsigned ElementBase = N->getMaskElt(0);
@@ -1296,7 +1464,7 @@ unsigned PPC::getVSPLTImmediate(SDNode *N, unsigned EltSize,
                                 SelectionDAG &DAG) {
   ShuffleVectorSDNode *SVOp = cast<ShuffleVectorSDNode>(N);
   assert(isSplatShuffleMask(SVOp, EltSize));
-  if (DAG.getTarget().getDataLayout()->isLittleEndian())
+  if (DAG.getDataLayout().isLittleEndian())
     return (16 / EltSize) - 1 - (SVOp->getMaskElt(0) / EltSize);
   else
     return SVOp->getMaskElt(0) / EltSize;
@@ -1349,17 +1517,17 @@ SDValue PPC::get_VSPLTI_elt(SDNode *N, unsigned ByteSize, SelectionDAG &DAG) {
     // Finally, check the least significant entry.
     if (LeadingZero) {
       if (!UniquedVals[Multiple-1].getNode())
-        return DAG.getTargetConstant(0, MVT::i32);  // 0,0,0,undef
+        return DAG.getTargetConstant(0, SDLoc(N), MVT::i32);  // 0,0,0,undef
       int Val = cast<ConstantSDNode>(UniquedVals[Multiple-1])->getZExtValue();
-      if (Val < 16)
-        return DAG.getTargetConstant(Val, MVT::i32);  // 0,0,0,4 -> vspltisw(4)
+      if (Val < 16)                                   // 0,0,0,4 -> vspltisw(4)
+        return DAG.getTargetConstant(Val, SDLoc(N), MVT::i32);
     }
     if (LeadingOnes) {
       if (!UniquedVals[Multiple-1].getNode())
-        return DAG.getTargetConstant(~0U, MVT::i32);  // -1,-1,-1,undef
+        return DAG.getTargetConstant(~0U, SDLoc(N), MVT::i32); // -1,-1,-1,undef
       int Val =cast<ConstantSDNode>(UniquedVals[Multiple-1])->getSExtValue();
       if (Val >= -16)                            // -1,-1,-1,-2 -> vspltisw(-2)
-        return DAG.getTargetConstant(Val, MVT::i32);
+        return DAG.getTargetConstant(Val, SDLoc(N), MVT::i32);
     }
 
     return SDValue();
@@ -1403,7 +1571,7 @@ SDValue PPC::get_VSPLTI_elt(SDNode *N, unsigned ByteSize, SelectionDAG &DAG) {
 
   // Finally, if this value fits in a 5 bit sext field, return it
   if (SignExtend32<5>(MaskVal) == MaskVal)
-    return DAG.getTargetConstant(MaskVal, MVT::i32);
+    return DAG.getTargetConstant(MaskVal, SDLoc(N), MVT::i32);
   return SDValue();
 }
 
@@ -1562,7 +1730,7 @@ bool PPCTargetLowering::SelectAddressRegImm(SDValue N, SDValue &Disp,
     short imm = 0;
     if (isIntS16Immediate(N.getOperand(1), imm) &&
         (!Aligned || (imm & 3) == 0)) {
-      Disp = DAG.getTargetConstant(imm, N.getValueType());
+      Disp = DAG.getTargetConstant(imm, dl, N.getValueType());
       if (FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(N.getOperand(0))) {
         Base = DAG.getTargetFrameIndex(FI->getIndex(), N.getValueType());
         fixupFuncForFI(DAG, FI->getIndex(), N.getValueType());
@@ -1602,7 +1770,7 @@ bool PPCTargetLowering::SelectAddressRegImm(SDValue N, SDValue &Disp,
         } else {
           Base = N.getOperand(0);
         }
-        Disp = DAG.getTargetConstant(imm, N.getValueType());
+        Disp = DAG.getTargetConstant(imm, dl, N.getValueType());
         return true;
       }
     }
@@ -1613,7 +1781,7 @@ bool PPCTargetLowering::SelectAddressRegImm(SDValue N, SDValue &Disp,
     // this as "d, 0"
     short Imm;
     if (isIntS16Immediate(CN, Imm) && (!Aligned || (Imm & 3) == 0)) {
-      Disp = DAG.getTargetConstant(Imm, CN->getValueType(0));
+      Disp = DAG.getTargetConstant(Imm, dl, CN->getValueType(0));
       Base = DAG.getRegister(Subtarget.isPPC64() ? PPC::ZERO8 : PPC::ZERO,
                              CN->getValueType(0));
       return true;
@@ -1626,16 +1794,17 @@ bool PPCTargetLowering::SelectAddressRegImm(SDValue N, SDValue &Disp,
       int Addr = (int)CN->getZExtValue();
 
       // Otherwise, break this down into an LIS + disp.
-      Disp = DAG.getTargetConstant((short)Addr, MVT::i32);
+      Disp = DAG.getTargetConstant((short)Addr, dl, MVT::i32);
 
-      Base = DAG.getTargetConstant((Addr - (signed short)Addr) >> 16, MVT::i32);
+      Base = DAG.getTargetConstant((Addr - (signed short)Addr) >> 16, dl,
+                                   MVT::i32);
       unsigned Opc = CN->getValueType(0) == MVT::i32 ? PPC::LIS : PPC::LIS8;
       Base = SDValue(DAG.getMachineNode(Opc, dl, CN->getValueType(0), Base), 0);
       return true;
     }
   }
 
-  Disp = DAG.getTargetConstant(0, getPointerTy());
+  Disp = DAG.getTargetConstant(0, dl, getPointerTy(DAG.getDataLayout()));
   if (FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(N)) {
     Base = DAG.getTargetFrameIndex(FI->getIndex(), N.getValueType());
     fixupFuncForFI(DAG, FI->getIndex(), N.getValueType());
@@ -1794,9 +1963,9 @@ static bool GetLabelAccessInfo(const TargetMachine &TM,
 
 static SDValue LowerLabelRef(SDValue HiPart, SDValue LoPart, bool isPIC,
                              SelectionDAG &DAG) {
-  EVT PtrVT = HiPart.getValueType();
-  SDValue Zero = DAG.getConstant(0, PtrVT);
   SDLoc DL(HiPart);
+  EVT PtrVT = HiPart.getValueType();
+  SDValue Zero = DAG.getConstant(0, DL, PtrVT);
 
   SDValue Hi = DAG.getNode(PPCISD::Hi, DL, PtrVT, HiPart, Zero);
   SDValue Lo = DAG.getNode(PPCISD::Lo, DL, PtrVT, LoPart, Zero);
@@ -1921,9 +2090,12 @@ SDValue PPCTargetLowering::LowerGlobalTLSAddress(SDValue Op,
   // large models could be added if users need it, at the cost of
   // additional complexity.
   GlobalAddressSDNode *GA = cast<GlobalAddressSDNode>(Op);
+  if (DAG.getTarget().Options.EmulatedTLS)
+    return LowerToTLSEmulatedModel(GA, DAG);
+
   SDLoc dl(GA);
   const GlobalValue *GV = GA->getGlobal();
-  EVT PtrVT = getPointerTy();
+  EVT PtrVT = getPointerTy(DAG.getDataLayout());
   bool is64bit = Subtarget.isPPC64();
   const Module *M = DAG.getMachineFunction().getFunction()->getParent();
   PICLevel::Level picLevel = M->getPICLevel();
@@ -2080,7 +2252,7 @@ SDValue PPCTargetLowering::LowerSETCC(SDValue Op, SelectionDAG &DAG) const {
       unsigned Log2b = Log2_32(VT.getSizeInBits());
       SDValue Clz = DAG.getNode(ISD::CTLZ, dl, VT, Zext);
       SDValue Scc = DAG.getNode(ISD::SRL, dl, VT, Clz,
-                                DAG.getConstant(Log2b, MVT::i32));
+                                DAG.getConstant(Log2b, dl, MVT::i32));
       return DAG.getNode(ISD::TRUNCATE, dl, MVT::i32, Scc);
     }
     // Leave comparisons against 0 and -1 alone for now, since they're usually
@@ -2100,7 +2272,7 @@ SDValue PPCTargetLowering::LowerSETCC(SDValue Op, SelectionDAG &DAG) const {
     EVT VT = Op.getValueType();
     SDValue Sub = DAG.getNode(ISD::XOR, dl, LHSVT, Op.getOperand(0),
                                 Op.getOperand(1));
-    return DAG.getSetCC(dl, VT, Sub, DAG.getConstant(0, LHSVT), CC);
+    return DAG.getSetCC(dl, VT, Sub, DAG.getConstant(0, dl, LHSVT), CC);
   }
   return SDValue();
 }
@@ -2109,7 +2281,7 @@ SDValue PPCTargetLowering::LowerVAARG(SDValue Op, SelectionDAG &DAG,
                                       const PPCSubtarget &Subtarget) const {
   SDNode *Node = Op.getNode();
   EVT VT = Node->getValueType(0);
-  EVT PtrVT = DAG.getTargetLoweringInfo().getPointerTy();
+  EVT PtrVT = DAG.getTargetLoweringInfo().getPointerTy(DAG.getDataLayout());
   SDValue InChain = Node->getOperand(0);
   SDValue VAListPtr = Node->getOperand(1);
   const Value *SV = cast<SrcValueSDNode>(Node->getOperand(2))->getValue();
@@ -2126,11 +2298,11 @@ SDValue PPCTargetLowering::LowerVAARG(SDValue Op, SelectionDAG &DAG,
   if (VT == MVT::i64) {
     // Check if GprIndex is even
     SDValue GprAnd = DAG.getNode(ISD::AND, dl, MVT::i32, GprIndex,
-                                 DAG.getConstant(1, MVT::i32));
+                                 DAG.getConstant(1, dl, MVT::i32));
     SDValue CC64 = DAG.getSetCC(dl, MVT::i32, GprAnd,
-                                DAG.getConstant(0, MVT::i32), ISD::SETNE);
+                                DAG.getConstant(0, dl, MVT::i32), ISD::SETNE);
     SDValue GprIndexPlusOne = DAG.getNode(ISD::ADD, dl, MVT::i32, GprIndex,
-                                          DAG.getConstant(1, MVT::i32));
+                                          DAG.getConstant(1, dl, MVT::i32));
     // Align GprIndex to be even if it isn't
     GprIndex = DAG.getNode(ISD::SELECT, dl, MVT::i32, CC64, GprIndexPlusOne,
                            GprIndex);
@@ -2138,7 +2310,7 @@ SDValue PPCTargetLowering::LowerVAARG(SDValue Op, SelectionDAG &DAG,
 
   // fpr index is 1 byte after gpr
   SDValue FprPtr = DAG.getNode(ISD::ADD, dl, PtrVT, VAListPtr,
-                               DAG.getConstant(1, MVT::i32));
+                               DAG.getConstant(1, dl, MVT::i32));
 
   // fpr
   SDValue FprIndex = DAG.getExtLoad(ISD::ZEXTLOAD, dl, MVT::i32, InChain,
@@ -2147,10 +2319,10 @@ SDValue PPCTargetLowering::LowerVAARG(SDValue Op, SelectionDAG &DAG,
   InChain = FprIndex.getValue(1);
 
   SDValue RegSaveAreaPtr = DAG.getNode(ISD::ADD, dl, PtrVT, VAListPtr,
-                                       DAG.getConstant(8, MVT::i32));
+                                       DAG.getConstant(8, dl, MVT::i32));
 
   SDValue OverflowAreaPtr = DAG.getNode(ISD::ADD, dl, PtrVT, VAListPtr,
-                                        DAG.getConstant(4, MVT::i32));
+                                        DAG.getConstant(4, dl, MVT::i32));
 
   // areas
   SDValue OverflowArea = DAG.getLoad(MVT::i32, dl, InChain, OverflowAreaPtr,
@@ -2165,12 +2337,12 @@ SDValue PPCTargetLowering::LowerVAARG(SDValue Op, SelectionDAG &DAG,
 
   // select overflow_area if index > 8
   SDValue CC = DAG.getSetCC(dl, MVT::i32, VT.isInteger() ? GprIndex : FprIndex,
-                            DAG.getConstant(8, MVT::i32), ISD::SETLT);
+                            DAG.getConstant(8, dl, MVT::i32), ISD::SETLT);
 
   // adjustment constant gpr_index * 4/8
   SDValue RegConstant = DAG.getNode(ISD::MUL, dl, MVT::i32,
                                     VT.isInteger() ? GprIndex : FprIndex,
-                                    DAG.getConstant(VT.isInteger() ? 4 : 8,
+                                    DAG.getConstant(VT.isInteger() ? 4 : 8, dl,
                                                     MVT::i32));
 
   // OurReg = RegSaveArea + RegConstant
@@ -2180,12 +2352,12 @@ SDValue PPCTargetLowering::LowerVAARG(SDValue Op, SelectionDAG &DAG,
   // Floating types are 32 bytes into RegSaveArea
   if (VT.isFloatingPoint())
     OurReg = DAG.getNode(ISD::ADD, dl, PtrVT, OurReg,
-                         DAG.getConstant(32, MVT::i32));
+                         DAG.getConstant(32, dl, MVT::i32));
 
   // increase {f,g}pr_index by 1 (or 2 if VT is i64)
   SDValue IndexPlus1 = DAG.getNode(ISD::ADD, dl, MVT::i32,
                                    VT.isInteger() ? GprIndex : FprIndex,
-                                   DAG.getConstant(VT == MVT::i64 ? 2 : 1,
+                                   DAG.getConstant(VT == MVT::i64 ? 2 : 1, dl,
                                                    MVT::i32));
 
   InChain = DAG.getTruncStore(InChain, dl, IndexPlus1,
@@ -2199,7 +2371,7 @@ SDValue PPCTargetLowering::LowerVAARG(SDValue Op, SelectionDAG &DAG,
   // increase overflow_area by 4/8 if gpr/fpr > 8
   SDValue OverflowAreaPlusN = DAG.getNode(ISD::ADD, dl, PtrVT, OverflowArea,
                                           DAG.getConstant(VT.isInteger() ? 4 : 8,
-                                          MVT::i32));
+                                          dl, MVT::i32));
 
   OverflowArea = DAG.getNode(ISD::SELECT, dl, MVT::i32, CC, OverflowArea,
                              OverflowAreaPlusN);
@@ -2221,8 +2393,8 @@ SDValue PPCTargetLowering::LowerVACOPY(SDValue Op, SelectionDAG &DAG,
   // 2*sizeof(char) + 2 Byte alignment + 2*sizeof(char*) = 12 Byte
   return DAG.getMemcpy(Op.getOperand(0), Op,
                        Op.getOperand(1), Op.getOperand(2),
-                       DAG.getConstant(12, MVT::i32), 8, false, true, false,
-                       MachinePointerInfo(), MachinePointerInfo());
+                       DAG.getConstant(12, SDLoc(Op), MVT::i32), 8, false, true,
+                       false, MachinePointerInfo(), MachinePointerInfo());
 }
 
 SDValue PPCTargetLowering::LowerADJUST_TRAMPOLINE(SDValue Op,
@@ -2238,11 +2410,9 @@ SDValue PPCTargetLowering::LowerINIT_TRAMPOLINE(SDValue Op,
   SDValue Nest = Op.getOperand(3); // 'nest' parameter value
   SDLoc dl(Op);
 
-  EVT PtrVT = DAG.getTargetLoweringInfo().getPointerTy();
+  EVT PtrVT = DAG.getTargetLoweringInfo().getPointerTy(DAG.getDataLayout());
   bool isPPC64 = (PtrVT == MVT::i64);
-  Type *IntPtrTy =
-    DAG.getTargetLoweringInfo().getDataLayout()->getIntPtrType(
-                                                             *DAG.getContext());
+  Type *IntPtrTy = DAG.getDataLayout().getIntPtrType(*DAG.getContext());
 
   TargetLowering::ArgListTy Args;
   TargetLowering::ArgListEntry Entry;
@@ -2251,7 +2421,7 @@ SDValue PPCTargetLowering::LowerINIT_TRAMPOLINE(SDValue Op,
   Entry.Node = Trmp; Args.push_back(Entry);
 
   // TrampSize == (isPPC64 ? 48 : 40);
-  Entry.Node = DAG.getConstant(isPPC64 ? 48 : 40,
+  Entry.Node = DAG.getConstant(isPPC64 ? 48 : 40, dl,
                                isPPC64 ? MVT::i64 : MVT::i32);
   Args.push_back(Entry);
 
@@ -2279,7 +2449,7 @@ SDValue PPCTargetLowering::LowerVASTART(SDValue Op, SelectionDAG &DAG,
   if (Subtarget.isDarwinABI() || Subtarget.isPPC64()) {
     // vastart just stores the address of the VarArgsFrameIndex slot into the
     // memory location argument.
-    EVT PtrVT = DAG.getTargetLoweringInfo().getPointerTy();
+    EVT PtrVT = DAG.getTargetLoweringInfo().getPointerTy(MF.getDataLayout());
     SDValue FR = DAG.getFrameIndex(FuncInfo->getVarArgsFrameIndex(), PtrVT);
     const Value *SV = cast<SrcValueSDNode>(Op.getOperand(2))->getValue();
     return DAG.getStore(Op.getOperand(0), dl, FR, Op.getOperand(1),
@@ -2312,11 +2482,10 @@ SDValue PPCTargetLowering::LowerVASTART(SDValue Op, SelectionDAG &DAG,
   // } va_list[1];
 
 
-  SDValue ArgGPR = DAG.getConstant(FuncInfo->getVarArgsNumGPR(), MVT::i32);
-  SDValue ArgFPR = DAG.getConstant(FuncInfo->getVarArgsNumFPR(), MVT::i32);
-
+  SDValue ArgGPR = DAG.getConstant(FuncInfo->getVarArgsNumGPR(), dl, MVT::i32);
+  SDValue ArgFPR = DAG.getConstant(FuncInfo->getVarArgsNumFPR(), dl, MVT::i32);
 
-  EVT PtrVT = DAG.getTargetLoweringInfo().getPointerTy();
+  EVT PtrVT = DAG.getTargetLoweringInfo().getPointerTy(MF.getDataLayout());
 
   SDValue StackOffsetFI = DAG.getFrameIndex(FuncInfo->getVarArgsStackOffset(),
                                             PtrVT);
@@ -2324,13 +2493,13 @@ SDValue PPCTargetLowering::LowerVASTART(SDValue Op, SelectionDAG &DAG,
                                  PtrVT);
 
   uint64_t FrameOffset = PtrVT.getSizeInBits()/8;
-  SDValue ConstFrameOffset = DAG.getConstant(FrameOffset, PtrVT);
+  SDValue ConstFrameOffset = DAG.getConstant(FrameOffset, dl, PtrVT);
 
   uint64_t StackOffset = PtrVT.getSizeInBits()/8 - 1;
-  SDValue ConstStackOffset = DAG.getConstant(StackOffset, PtrVT);
+  SDValue ConstStackOffset = DAG.getConstant(StackOffset, dl, PtrVT);
 
   uint64_t FPROffset = 1;
-  SDValue ConstFPROffset = DAG.getConstant(FPROffset, PtrVT);
+  SDValue ConstFPROffset = DAG.getConstant(FPROffset, dl, PtrVT);
 
   const Value *SV = cast<SrcValueSDNode>(Op.getOperand(2))->getValue();
 
@@ -2472,7 +2641,8 @@ static unsigned CalculateStackSlotAlignment(EVT ArgVT, EVT OrigVT,
   // Altivec parameters are padded to a 16 byte boundary.
   if (ArgVT == MVT::v4f32 || ArgVT == MVT::v4i32 ||
       ArgVT == MVT::v8i16 || ArgVT == MVT::v16i8 ||
-      ArgVT == MVT::v2f64 || ArgVT == MVT::v2i64)
+      ArgVT == MVT::v2f64 || ArgVT == MVT::v2i64 ||
+      ArgVT == MVT::v1i128)
     Align = 16;
   // QPX vector types stored in double-precision are padded to a 32 byte
   // boundary.
@@ -2551,7 +2721,8 @@ static bool CalculateStackSlotUsed(EVT ArgVT, EVT OrigVT,
       }
     if (ArgVT == MVT::v4f32 || ArgVT == MVT::v4i32 ||
         ArgVT == MVT::v8i16 || ArgVT == MVT::v16i8 ||
-        ArgVT == MVT::v2f64 || ArgVT == MVT::v2i64)
+        ArgVT == MVT::v2f64 || ArgVT == MVT::v2i64 ||
+        ArgVT == MVT::v1i128)
       if (AvailableVRs > 0) {
         --AvailableVRs;
         return false;
@@ -2634,7 +2805,7 @@ PPCTargetLowering::LowerFormalArguments_32SVR4(
   MachineFrameInfo *MFI = MF.getFrameInfo();
   PPCFunctionInfo *FuncInfo = MF.getInfo<PPCFunctionInfo>();
 
-  EVT PtrVT = DAG.getTargetLoweringInfo().getPointerTy();
+  EVT PtrVT = DAG.getTargetLoweringInfo().getPointerTy(MF.getDataLayout());
   // Potential tail calls could cause overwriting of argument stack slots.
   bool isImmutable = !(getTargetMachine().Options.GuaranteedTailCallOpt &&
                        (CallConv == CallingConv::Fast));
@@ -2667,7 +2838,10 @@ PPCTargetLowering::LowerFormalArguments_32SVR4(
           RC = &PPC::GPRCRegClass;
           break;
         case MVT::f32:
-          RC = &PPC::F4RCRegClass;
+          if (Subtarget.hasP8Vector())
+            RC = &PPC::VSSRCRegClass;
+          else
+            RC = &PPC::F4RCRegClass;
           break;
         case MVT::f64:
           if (Subtarget.hasVSX())
@@ -2791,7 +2965,7 @@ PPCTargetLowering::LowerFormalArguments_32SVR4(
                                    MachinePointerInfo(), false, false, 0);
       MemOps.push_back(Store);
       // Increment the address by four for the next argument to store
-      SDValue PtrOff = DAG.getConstant(PtrVT.getSizeInBits()/8, PtrVT);
+      SDValue PtrOff = DAG.getConstant(PtrVT.getSizeInBits()/8, dl, PtrVT);
       FIN = DAG.getNode(ISD::ADD, dl, PtrOff.getValueType(), FIN, PtrOff);
     }
 
@@ -2810,7 +2984,7 @@ PPCTargetLowering::LowerFormalArguments_32SVR4(
                                    MachinePointerInfo(), false, false, 0);
       MemOps.push_back(Store);
       // Increment the address by eight for the next argument to store
-      SDValue PtrOff = DAG.getConstant(MVT(MVT::f64).getSizeInBits()/8,
+      SDValue PtrOff = DAG.getConstant(MVT(MVT::f64).getSizeInBits()/8, dl,
                                          PtrVT);
       FIN = DAG.getNode(ISD::ADD, dl, PtrOff.getValueType(), FIN, PtrOff);
     }
@@ -2857,7 +3031,7 @@ PPCTargetLowering::LowerFormalArguments_64SVR4(
   assert(!(CallConv == CallingConv::Fast && isVarArg) &&
          "fastcc not supported on varargs functions");
 
-  EVT PtrVT = DAG.getTargetLoweringInfo().getPointerTy();
+  EVT PtrVT = DAG.getTargetLoweringInfo().getPointerTy(MF.getDataLayout());
   // Potential tail calls could cause overwriting of argument stack slots.
   bool isImmutable = !(getTargetMachine().Options.GuaranteedTailCallOpt &&
                        (CallConv == CallingConv::Fast));
@@ -2893,12 +3067,16 @@ PPCTargetLowering::LowerFormalArguments_64SVR4(
   unsigned NumBytes = LinkageSize;
   unsigned AvailableFPRs = Num_FPR_Regs;
   unsigned AvailableVRs = Num_VR_Regs;
-  for (unsigned i = 0, e = Ins.size(); i != e; ++i)
+  for (unsigned i = 0, e = Ins.size(); i != e; ++i) {
+    if (Ins[i].Flags.isNest())
+      continue;
+
     if (CalculateStackSlotUsed(Ins[i].VT, Ins[i].ArgVT, Ins[i].Flags,
                                PtrByteSize, LinkageSize, ParamAreaSize,
                                NumBytes, AvailableFPRs, AvailableVRs,
                                Subtarget.hasQPX()))
       HasParameterArea = true;
+  }
 
   // Add DAG nodes to load the arguments or copy them out of registers.  On
   // entry to a function on PPC, the arguments start after the linkage area,
@@ -2986,7 +3164,7 @@ PPCTargetLowering::LowerFormalArguments_64SVR4(
         // address of the enclosing doubleword on big-endian systems.
         SDValue Arg = FIN;
         if (!isLittleEndian) {
-          SDValue ArgOff = DAG.getConstant(PtrByteSize - ObjSize, PtrVT);
+          SDValue ArgOff = DAG.getConstant(PtrByteSize - ObjSize, dl, PtrVT);
           Arg = DAG.getNode(ISD::ADD, dl, ArgOff.getValueType(), Arg, ArgOff);
         }
         InVals.push_back(Arg);
@@ -3032,7 +3210,7 @@ PPCTargetLowering::LowerFormalArguments_64SVR4(
         SDValue Val = DAG.getCopyFromReg(Chain, dl, VReg, PtrVT);
         SDValue Addr = FIN;
         if (j) {
-          SDValue Off = DAG.getConstant(j, PtrVT);
+          SDValue Off = DAG.getConstant(j, dl, PtrVT);
           Addr = DAG.getNode(ISD::ADD, dl, Off.getValueType(), Addr, Off);
         }
         SDValue Store = DAG.getStore(Val.getValue(1), dl, Val, Addr,
@@ -3050,6 +3228,17 @@ PPCTargetLowering::LowerFormalArguments_64SVR4(
     case MVT::i1:
     case MVT::i32:
     case MVT::i64:
+      if (Flags.isNest()) {
+        // The 'nest' parameter, if any, is passed in R11.
+        unsigned VReg = MF.addLiveIn(PPC::X11, &PPC::G8RCRegClass);
+        ArgVal = DAG.getCopyFromReg(Chain, dl, VReg, MVT::i64);
+
+        if (ObjectVT == MVT::i32 || ObjectVT == MVT::i1)
+          ArgVal = extendArgForPPC64(Flags, ObjectVT, DAG, ArgVal, dl);
+
+        break;
+      }
+
       // These can be scalar arguments or elements of an integer array type
       // passed directly.  Clang may use those instead of "byval" aggregate
       // types to avoid forcing arguments to memory unnecessarily.
@@ -3081,7 +3270,10 @@ PPCTargetLowering::LowerFormalArguments_64SVR4(
         unsigned VReg;
 
         if (ObjectVT == MVT::f32)
-          VReg = MF.addLiveIn(FPR[FPR_idx], &PPC::F4RCRegClass);
+          VReg = MF.addLiveIn(FPR[FPR_idx],
+                              Subtarget.hasP8Vector()
+                                  ? &PPC::VSSRCRegClass
+                                  : &PPC::F4RCRegClass);
         else
           VReg = MF.addLiveIn(FPR[FPR_idx], Subtarget.hasVSX()
                                                 ? &PPC::VSFRCRegClass
@@ -3102,7 +3294,7 @@ PPCTargetLowering::LowerFormalArguments_64SVR4(
         if (ObjectVT == MVT::f32) {
           if ((ArgOffset % PtrByteSize) == (isLittleEndian ? 4 : 0))
             ArgVal = DAG.getNode(ISD::SRL, dl, MVT::i64, ArgVal,
-                                 DAG.getConstant(32, MVT::i32));
+                                 DAG.getConstant(32, dl, MVT::i32));
           ArgVal = DAG.getNode(ISD::TRUNCATE, dl, MVT::i32, ArgVal);
         }
 
@@ -3130,6 +3322,7 @@ PPCTargetLowering::LowerFormalArguments_64SVR4(
     case MVT::v16i8:
     case MVT::v2f64:
     case MVT::v2i64:
+    case MVT::v1i128:
       if (!Subtarget.hasQPX()) {
       // These can be scalar arguments or elements of a vector array type
       // passed directly.  The latter are used to implement ELFv2 homogenous
@@ -3230,7 +3423,7 @@ PPCTargetLowering::LowerFormalArguments_64SVR4(
                                    MachinePointerInfo(), false, false, 0);
       MemOps.push_back(Store);
       // Increment the address by four for the next argument to store
-      SDValue PtrOff = DAG.getConstant(PtrByteSize, PtrVT);
+      SDValue PtrOff = DAG.getConstant(PtrByteSize, dl, PtrVT);
       FIN = DAG.getNode(ISD::ADD, dl, PtrOff.getValueType(), FIN, PtrOff);
     }
   }
@@ -3255,7 +3448,7 @@ PPCTargetLowering::LowerFormalArguments_Darwin(
   MachineFrameInfo *MFI = MF.getFrameInfo();
   PPCFunctionInfo *FuncInfo = MF.getInfo<PPCFunctionInfo>();
 
-  EVT PtrVT = DAG.getTargetLoweringInfo().getPointerTy();
+  EVT PtrVT = DAG.getTargetLoweringInfo().getPointerTy(MF.getDataLayout());
   bool isPPC64 = PtrVT == MVT::i64;
   // Potential tail calls could cause overwriting of argument stack slots.
   bool isImmutable = !(getTargetMachine().Options.GuaranteedTailCallOpt &&
@@ -3596,7 +3789,7 @@ PPCTargetLowering::LowerFormalArguments_Darwin(
                                    MachinePointerInfo(), false, false, 0);
       MemOps.push_back(Store);
       // Increment the address by four for the next argument to store
-      SDValue PtrOff = DAG.getConstant(PtrVT.getSizeInBits()/8, PtrVT);
+      SDValue PtrOff = DAG.getConstant(PtrVT.getSizeInBits()/8, dl, PtrVT);
       FIN = DAG.getNode(ISD::ADD, dl, PtrOff.getValueType(), FIN, PtrOff);
     }
   }
@@ -3674,8 +3867,9 @@ static SDNode *isBLACompatibleAddress(SDValue Op, SelectionDAG &DAG) {
       SignExtend32<26>(Addr) != Addr)
     return nullptr;  // Top 6 bits have to be sext of immediate.
 
-  return DAG.getConstant((int)C->getZExtValue() >> 2,
-                         DAG.getTargetLoweringInfo().getPointerTy()).getNode();
+  return DAG.getConstant((int)C->getZExtValue() >> 2, SDLoc(Op),
+                         DAG.getTargetLoweringInfo().getPointerTy(
+                             DAG.getDataLayout())).getNode();
 }
 
 namespace {
@@ -3806,7 +4000,7 @@ static SDValue
 CreateCopyOfByValArgument(SDValue Src, SDValue Dst, SDValue Chain,
                           ISD::ArgFlagsTy Flags, SelectionDAG &DAG,
                           SDLoc dl) {
-  SDValue SizeNode = DAG.getConstant(Flags.getByValSize(), MVT::i32);
+  SDValue SizeNode = DAG.getConstant(Flags.getByValSize(), dl, MVT::i32);
   return DAG.getMemcpy(Chain, dl, Dst, Src, SizeNode, Flags.getByValAlign(),
                        false, false, false, MachinePointerInfo(),
                        MachinePointerInfo());
@@ -3821,7 +4015,7 @@ LowerMemOpCallTo(SelectionDAG &DAG, MachineFunction &MF, SDValue Chain,
                  bool isVector, SmallVectorImpl<SDValue> &MemOpChains,
                  SmallVectorImpl<TailCallArgumentInfo> &TailCallArguments,
                  SDLoc dl) {
-  EVT PtrVT = DAG.getTargetLoweringInfo().getPointerTy();
+  EVT PtrVT = DAG.getTargetLoweringInfo().getPointerTy(DAG.getDataLayout());
   if (!isTailCall) {
     if (isVector) {
       SDValue StackPtr;
@@ -3830,7 +4024,7 @@ LowerMemOpCallTo(SelectionDAG &DAG, MachineFunction &MF, SDValue Chain,
       else
         StackPtr = DAG.getRegister(PPC::R1, MVT::i32);
       PtrOff = DAG.getNode(ISD::ADD, dl, PtrVT, StackPtr,
-                           DAG.getConstant(ArgOffset, PtrVT));
+                           DAG.getConstant(ArgOffset, dl, PtrVT));
     }
     MemOpChains.push_back(DAG.getStore(Chain, dl, Arg, PtrOff,
                                        MachinePointerInfo(), false, false, 0));
@@ -3861,8 +4055,8 @@ void PrepareTailCall(SelectionDAG &DAG, SDValue &InFlag, SDValue &Chain,
                                         isPPC64, isDarwinABI, dl);
 
   // Emit callseq_end just before tailcall node.
-  Chain = DAG.getCALLSEQ_END(Chain, DAG.getIntPtrConstant(NumBytes, true),
-                             DAG.getIntPtrConstant(0, true), InFlag, dl);
+  Chain = DAG.getCALLSEQ_END(Chain, DAG.getIntPtrConstant(NumBytes, dl, true),
+                             DAG.getIntPtrConstant(0, dl, true), InFlag, dl);
   InFlag = Chain.getValue(1);
 }
 
@@ -3883,7 +4077,7 @@ static bool isFunctionGlobalAddress(SDValue Callee) {
 static
 unsigned PrepareCall(SelectionDAG &DAG, SDValue &Callee, SDValue &InFlag,
                      SDValue &Chain, SDValue CallSeqStart, SDLoc dl, int SPDiff,
-                     bool isTailCall, bool IsPatchPoint,
+                     bool isTailCall, bool IsPatchPoint, bool hasNest,
                      SmallVectorImpl<std::pair<unsigned, SDValue> > &RegsToPass,
                      SmallVectorImpl<SDValue> &Ops, std::vector<EVT> &NodeTys,
                      ImmutableCallSite *CS, const PPCSubtarget &Subtarget) {
@@ -3892,7 +4086,7 @@ unsigned PrepareCall(SelectionDAG &DAG, SDValue &Callee, SDValue &InFlag,
   bool isSVR4ABI = Subtarget.isSVR4ABI();
   bool isELFv2ABI = Subtarget.isELFv2ABI();
 
-  EVT PtrVT = DAG.getTargetLoweringInfo().getPointerTy();
+  EVT PtrVT = DAG.getTargetLoweringInfo().getPointerTy(DAG.getDataLayout());
   NodeTys.push_back(MVT::Other);   // Returns a chain
   NodeTys.push_back(MVT::Glue);    // Returns a flag for retval copy to use.
 
@@ -3914,8 +4108,7 @@ unsigned PrepareCall(SelectionDAG &DAG, SDValue &Callee, SDValue &InFlag,
     if ((DAG.getTarget().getRelocationModel() != Reloc::Static &&
          (Subtarget.getTargetTriple().isMacOSX() &&
           Subtarget.getTargetTriple().isMacOSXVersionLT(10, 5)) &&
-         (G->getGlobal()->isDeclaration() ||
-          G->getGlobal()->isWeakForLinker())) ||
+         !G->getGlobal()->isStrongDefinitionForLinker()) ||
         (Subtarget.isTargetELF() && !isPPC64 &&
          !G->getGlobal()->hasLocalLinkage() &&
          DAG.getTarget().getRelocationModel() == Reloc::PIC_)) {
@@ -4008,13 +4201,13 @@ unsigned PrepareCall(SelectionDAG &DAG, SDValue &Callee, SDValue &InFlag,
                                         false, false, LoadsInv, 8);
 
       // Load environment pointer into r11.
-      SDValue PtrOff = DAG.getIntPtrConstant(16);
+      SDValue PtrOff = DAG.getIntPtrConstant(16, dl);
       SDValue AddPtr = DAG.getNode(ISD::ADD, dl, MVT::i64, Callee, PtrOff);
       SDValue LoadEnvPtr = DAG.getLoad(MVT::i64, dl, LDChain, AddPtr,
                                        MPI.getWithOffset(16), false, false,
                                        LoadsInv, 8);
 
-      SDValue TOCOff = DAG.getIntPtrConstant(8);
+      SDValue TOCOff = DAG.getIntPtrConstant(8, dl);
       SDValue AddTOC = DAG.getNode(ISD::ADD, dl, MVT::i64, Callee, TOCOff);
       SDValue TOCPtr = DAG.getLoad(MVT::i64, dl, LDChain, AddTOC,
                                    MPI.getWithOffset(8), false, false,
@@ -4026,11 +4219,15 @@ unsigned PrepareCall(SelectionDAG &DAG, SDValue &Callee, SDValue &InFlag,
       Chain = TOCVal.getValue(0);
       InFlag = TOCVal.getValue(1);
 
-      SDValue EnvVal = DAG.getCopyToReg(Chain, dl, PPC::X11, LoadEnvPtr,
-                                        InFlag);
+      // If the function call has an explicit 'nest' parameter, it takes the
+      // place of the environment pointer.
+      if (!hasNest) {
+        SDValue EnvVal = DAG.getCopyToReg(Chain, dl, PPC::X11, LoadEnvPtr,
+                                          InFlag);
 
-      Chain = EnvVal.getValue(0);
-      InFlag = EnvVal.getValue(1);
+        Chain = EnvVal.getValue(0);
+        InFlag = EnvVal.getValue(1);
+      }
 
       MTCTROps[0] = Chain;
       MTCTROps[1] = LoadFuncPtr;
@@ -4048,7 +4245,7 @@ unsigned PrepareCall(SelectionDAG &DAG, SDValue &Callee, SDValue &InFlag,
     CallOpc = PPCISD::BCTRL;
     Callee.setNode(nullptr);
     // Add use of X11 (holding environment pointer)
-    if (isSVR4ABI && isPPC64 && !isELFv2ABI)
+    if (isSVR4ABI && isPPC64 && !isELFv2ABI && !hasNest)
       Ops.push_back(DAG.getRegister(PPC::X11, PtrVT));
     // Add CTR register as callee so a bctr can be emitted later.
     if (isTailCall)
@@ -4062,7 +4259,7 @@ unsigned PrepareCall(SelectionDAG &DAG, SDValue &Callee, SDValue &InFlag,
   }
   // If this is a tail call add stack pointer delta.
   if (isTailCall)
-    Ops.push_back(DAG.getConstant(SPDiff, MVT::i32));
+    Ops.push_back(DAG.getConstant(SPDiff, dl, MVT::i32));
 
   // Add argument registers to the end of the list so that they are known live
   // into the call.
@@ -4084,8 +4281,7 @@ static
 bool isLocalCall(const SDValue &Callee)
 {
   if (GlobalAddressSDNode *G = dyn_cast<GlobalAddressSDNode>(Callee))
-    return !G->getGlobal()->isDeclaration() &&
-           !G->getGlobal()->isWeakForLinker();
+    return G->getGlobal()->isStrongDefinitionForLinker();
   return false;
 }
 
@@ -4138,7 +4334,7 @@ PPCTargetLowering::LowerCallResult(SDValue Chain, SDValue InFlag,
 SDValue
 PPCTargetLowering::FinishCall(CallingConv::ID CallConv, SDLoc dl,
                               bool isTailCall, bool isVarArg, bool IsPatchPoint,
-                              SelectionDAG &DAG,
+                              bool hasNest, SelectionDAG &DAG,
                               SmallVector<std::pair<unsigned, SDValue>, 8>
                                 &RegsToPass,
                               SDValue InFlag, SDValue Chain,
@@ -4151,8 +4347,8 @@ PPCTargetLowering::FinishCall(CallingConv::ID CallConv, SDLoc dl,
   std::vector<EVT> NodeTys;
   SmallVector<SDValue, 8> Ops;
   unsigned CallOpc = PrepareCall(DAG, Callee, InFlag, Chain, CallSeqStart, dl,
-                                 SPDiff, isTailCall, IsPatchPoint, RegsToPass,
-                                 Ops, NodeTys, CS, Subtarget);
+                                 SPDiff, isTailCall, IsPatchPoint, hasNest,
+                                 RegsToPass, Ops, NodeTys, CS, Subtarget);
 
   // Add implicit use of CR bit 6 for 32-bit SVR4 vararg calls
   if (isVarArg && Subtarget.isSVR4ABI() && !Subtarget.isPPC64())
@@ -4184,6 +4380,7 @@ PPCTargetLowering::FinishCall(CallingConv::ID CallConv, SDLoc dl,
             isa<ConstantSDNode>(Callee)) &&
     "Expecting an global address, external symbol, absolute value or register");
 
+    DAG.getMachineFunction().getFrameInfo()->setHasTailCall();
     return DAG.getNode(PPCISD::TC_RETURN, dl, MVT::Other, Ops);
   }
 
@@ -4210,10 +4407,10 @@ PPCTargetLowering::FinishCall(CallingConv::ID CallConv, SDLoc dl,
       // allocated and an unnecessary move instruction being generated.
       CallOpc = PPCISD::BCTRL_LOAD_TOC;
 
-      EVT PtrVT = DAG.getTargetLoweringInfo().getPointerTy();
+      EVT PtrVT = DAG.getTargetLoweringInfo().getPointerTy(DAG.getDataLayout());
       SDValue StackPtr = DAG.getRegister(PPC::X1, PtrVT);
       unsigned TOCSaveOffset = Subtarget.getFrameLowering()->getTOCSaveOffset();
-      SDValue TOCOff = DAG.getIntPtrConstant(TOCSaveOffset);
+      SDValue TOCOff = DAG.getIntPtrConstant(TOCSaveOffset, dl);
       SDValue AddTOC = DAG.getNode(ISD::ADD, dl, MVT::i64, StackPtr, TOCOff);
 
       // The address needs to go after the chain input but before the flag (or
@@ -4229,8 +4426,8 @@ PPCTargetLowering::FinishCall(CallingConv::ID CallConv, SDLoc dl,
   Chain = DAG.getNode(CallOpc, dl, NodeTys, Ops);
   InFlag = Chain.getValue(1);
 
-  Chain = DAG.getCALLSEQ_END(Chain, DAG.getIntPtrConstant(NumBytes, true),
-                             DAG.getIntPtrConstant(BytesCalleePops, true),
+  Chain = DAG.getCALLSEQ_END(Chain, DAG.getIntPtrConstant(NumBytes, dl, true),
+                             DAG.getIntPtrConstant(BytesCalleePops, dl, true),
                              InFlag, dl);
   if (!Ins.empty())
     InFlag = Chain.getValue(1);
@@ -4374,7 +4571,7 @@ PPCTargetLowering::LowerCall_32SVR4(SDValue Chain, SDValue Callee,
 
   // Adjust the stack pointer for the new arguments...
   // These operations are automatically eliminated by the prolog/epilog pass
-  Chain = DAG.getCALLSEQ_START(Chain, DAG.getIntPtrConstant(NumBytes, true),
+  Chain = DAG.getCALLSEQ_START(Chain, DAG.getIntPtrConstant(NumBytes, dl, true),
                                dl);
   SDValue CallSeqStart = Chain;
 
@@ -4414,8 +4611,9 @@ PPCTargetLowering::LowerCall_32SVR4(SDValue Chain, SDValue Callee,
       // Memory reserved in the local variable space of the callers stack frame.
       unsigned LocMemOffset = ByValVA.getLocMemOffset();
 
-      SDValue PtrOff = DAG.getIntPtrConstant(LocMemOffset);
-      PtrOff = DAG.getNode(ISD::ADD, dl, getPointerTy(), StackPtr, PtrOff);
+      SDValue PtrOff = DAG.getIntPtrConstant(LocMemOffset, dl);
+      PtrOff = DAG.getNode(ISD::ADD, dl, getPointerTy(MF.getDataLayout()),
+                           StackPtr, PtrOff);
 
       // Create a copy of the argument in the local area of the current
       // stack frame.
@@ -4451,8 +4649,9 @@ PPCTargetLowering::LowerCall_32SVR4(SDValue Chain, SDValue Callee,
       unsigned LocMemOffset = VA.getLocMemOffset();
 
       if (!isTailCall) {
-        SDValue PtrOff = DAG.getIntPtrConstant(LocMemOffset);
-        PtrOff = DAG.getNode(ISD::ADD, dl, getPointerTy(), StackPtr, PtrOff);
+        SDValue PtrOff = DAG.getIntPtrConstant(LocMemOffset, dl);
+        PtrOff = DAG.getNode(ISD::ADD, dl, getPointerTy(MF.getDataLayout()),
+                             StackPtr, PtrOff);
 
         MemOpChains.push_back(DAG.getStore(Chain, dl, Arg, PtrOff,
                                            MachinePointerInfo(),
@@ -4493,7 +4692,8 @@ PPCTargetLowering::LowerCall_32SVR4(SDValue Chain, SDValue Callee,
     PrepareTailCall(DAG, InFlag, Chain, dl, false, SPDiff, NumBytes, LROp, FPOp,
                     false, TailCallArguments);
 
-  return FinishCall(CallConv, dl, isTailCall, isVarArg, IsPatchPoint, DAG,
+  return FinishCall(CallConv, dl, isTailCall, isVarArg, IsPatchPoint,
+                    /* unused except on PPC64 ELFv1 */ false, DAG,
                     RegsToPass, InFlag, Chain, CallSeqStart, Callee, SPDiff,
                     NumBytes, Ins, InVals, CS);
 }
@@ -4532,8 +4732,9 @@ PPCTargetLowering::LowerCall_64SVR4(SDValue Chain, SDValue Callee,
   bool isELFv2ABI = Subtarget.isELFv2ABI();
   bool isLittleEndian = Subtarget.isLittleEndian();
   unsigned NumOps = Outs.size();
+  bool hasNest = false;
 
-  EVT PtrVT = DAG.getTargetLoweringInfo().getPointerTy();
+  EVT PtrVT = DAG.getTargetLoweringInfo().getPointerTy(DAG.getDataLayout());
   unsigned PtrByteSize = 8;
 
   MachineFunction &MF = DAG.getMachineFunction();
@@ -4587,6 +4788,9 @@ PPCTargetLowering::LowerCall_64SVR4(SDValue Chain, SDValue Callee,
     EVT ArgVT = Outs[i].VT;
     EVT OrigVT = Outs[i].ArgVT;
 
+    if (Flags.isNest())
+      continue;
+
     if (CallConv == CallingConv::Fast) {
       if (Flags.isByVal())
         NumGPRsUsed += (Flags.getByValSize()+7)/8;
@@ -4604,6 +4808,7 @@ PPCTargetLowering::LowerCall_64SVR4(SDValue Chain, SDValue Callee,
         case MVT::v16i8:
         case MVT::v2f64:
         case MVT::v2i64:
+        case MVT::v1i128:
           if (++NumVRsUsed <= NumVRs)
             continue;
           break;
@@ -4664,7 +4869,7 @@ PPCTargetLowering::LowerCall_64SVR4(SDValue Chain, SDValue Callee,
 
   // Adjust the stack pointer for the new arguments...
   // These operations are automatically eliminated by the prolog/epilog pass
-  Chain = DAG.getCALLSEQ_START(Chain, DAG.getIntPtrConstant(NumBytes, true),
+  Chain = DAG.getCALLSEQ_START(Chain, DAG.getIntPtrConstant(NumBytes, dl, true),
                                dl);
   SDValue CallSeqStart = Chain;
 
@@ -4708,7 +4913,7 @@ PPCTargetLowering::LowerCall_64SVR4(SDValue Chain, SDValue Callee,
         CalculateStackSlotAlignment(ArgVT, OrigVT, Flags, PtrByteSize);
       ArgOffset = ((ArgOffset + Align - 1) / Align) * Align;
 
-      PtrOff = DAG.getConstant(ArgOffset, StackPtr.getValueType());
+      PtrOff = DAG.getConstant(ArgOffset, dl, StackPtr.getValueType());
 
       PtrOff = DAG.getNode(ISD::ADD, dl, PtrVT, StackPtr, PtrOff);
     };
@@ -4765,7 +4970,7 @@ PPCTargetLowering::LowerCall_64SVR4(SDValue Chain, SDValue Callee,
       if (GPR_idx == NumGPRs && Size < 8) {
         SDValue AddPtr = PtrOff;
         if (!isLittleEndian) {
-          SDValue Const = DAG.getConstant(PtrByteSize - Size,
+          SDValue Const = DAG.getConstant(PtrByteSize - Size, dl,
                                           PtrOff.getValueType());
           AddPtr = DAG.getNode(ISD::ADD, dl, PtrVT, PtrOff, Const);
         }
@@ -4805,7 +5010,7 @@ PPCTargetLowering::LowerCall_64SVR4(SDValue Chain, SDValue Callee,
         // parameter save area instead of a new local variable.
         SDValue AddPtr = PtrOff;
         if (!isLittleEndian) {
-          SDValue Const = DAG.getConstant(8 - Size, PtrOff.getValueType());
+          SDValue Const = DAG.getConstant(8 - Size, dl, PtrOff.getValueType());
           AddPtr = DAG.getNode(ISD::ADD, dl, PtrVT, PtrOff, Const);
         }
         Chain = CallSeqStart = createMemcpyOutsideCallSeq(Arg, AddPtr,
@@ -4827,7 +5032,7 @@ PPCTargetLowering::LowerCall_64SVR4(SDValue Chain, SDValue Callee,
       // For aggregates larger than PtrByteSize, copy the pieces of the
       // object that fit into registers from the parameter save area.
       for (unsigned j=0; j<Size; j+=PtrByteSize) {
-        SDValue Const = DAG.getConstant(j, PtrOff.getValueType());
+        SDValue Const = DAG.getConstant(j, dl, PtrOff.getValueType());
         SDValue AddArg = DAG.getNode(ISD::ADD, dl, PtrVT, Arg, Const);
         if (GPR_idx != NumGPRs) {
           SDValue Load = DAG.getLoad(PtrVT, dl, Chain, AddArg,
@@ -4849,6 +5054,13 @@ PPCTargetLowering::LowerCall_64SVR4(SDValue Chain, SDValue Callee,
     case MVT::i1:
     case MVT::i32:
     case MVT::i64:
+      if (Flags.isNest()) {
+        // The 'nest' parameter, if any, is passed in R11.
+        RegsToPass.push_back(std::make_pair(PPC::X11, Arg));
+        hasNest = true;
+        break;
+      }
+
       // These can be scalar arguments or elements of an integer array type
       // passed directly.  Clang may use those instead of "byval" aggregate
       // types to avoid forcing arguments to memory unnecessarily.
@@ -4922,7 +5134,7 @@ PPCTargetLowering::LowerCall_64SVR4(SDValue Chain, SDValue Callee,
           ArgVal = DAG.getNode(ISD::ANY_EXTEND, dl, MVT::i64, ArgVal);
           if (!isLittleEndian)
             ArgVal = DAG.getNode(ISD::SHL, dl, MVT::i64, ArgVal,
-                                 DAG.getConstant(32, MVT::i32));
+                                 DAG.getConstant(32, dl, MVT::i32));
 
         // Non-final even elements are skipped; they will be handled
         // together the with subsequent argument on the next go-around.
@@ -4939,7 +5151,7 @@ PPCTargetLowering::LowerCall_64SVR4(SDValue Chain, SDValue Callee,
         // second (rightmost) word of the stack doubleword.
         if (Arg.getValueType() == MVT::f32 &&
             !isLittleEndian && !Flags.isInConsecutiveRegs()) {
-          SDValue ConstFour = DAG.getConstant(4, PtrOff.getValueType());
+          SDValue ConstFour = DAG.getConstant(4, dl, PtrOff.getValueType());
           PtrOff = DAG.getNode(ISD::ADD, dl, PtrVT, PtrOff, ConstFour);
         }
 
@@ -4966,6 +5178,7 @@ PPCTargetLowering::LowerCall_64SVR4(SDValue Chain, SDValue Callee,
     case MVT::v16i8:
     case MVT::v2f64:
     case MVT::v2i64:
+    case MVT::v1i128:
       if (!Subtarget.hasQPX()) {
       // These can be scalar arguments or elements of a vector array type
       // passed directly.  The latter are used to implement ELFv2 homogenous
@@ -4999,7 +5212,7 @@ PPCTargetLowering::LowerCall_64SVR4(SDValue Chain, SDValue Callee,
           if (GPR_idx == NumGPRs)
             break;
           SDValue Ix = DAG.getNode(ISD::ADD, dl, PtrVT, PtrOff,
-                                  DAG.getConstant(i, PtrVT));
+                                   DAG.getConstant(i, dl, PtrVT));
           SDValue Load = DAG.getLoad(PtrVT, dl, Store, Ix, MachinePointerInfo(),
                                      false, false, false, 0);
           MemOpChains.push_back(Load.getValue(1));
@@ -5057,7 +5270,7 @@ PPCTargetLowering::LowerCall_64SVR4(SDValue Chain, SDValue Callee,
           if (GPR_idx == NumGPRs)
             break;
           SDValue Ix = DAG.getNode(ISD::ADD, dl, PtrVT, PtrOff,
-                                  DAG.getConstant(i, PtrVT));
+                                   DAG.getConstant(i, dl, PtrVT));
           SDValue Load = DAG.getLoad(PtrVT, dl, Store, Ix, MachinePointerInfo(),
                                      false, false, false, 0);
           MemOpChains.push_back(Load.getValue(1));
@@ -5104,7 +5317,7 @@ PPCTargetLowering::LowerCall_64SVR4(SDValue Chain, SDValue Callee,
     SDValue Val = DAG.getCopyFromReg(Chain, dl, PPC::X2, MVT::i64);
     // TOC save area offset.
     unsigned TOCSaveOffset = Subtarget.getFrameLowering()->getTOCSaveOffset();
-    SDValue PtrOff = DAG.getIntPtrConstant(TOCSaveOffset);
+    SDValue PtrOff = DAG.getIntPtrConstant(TOCSaveOffset, dl);
     SDValue AddPtr = DAG.getNode(ISD::ADD, dl, PtrVT, StackPtr, PtrOff);
     Chain = DAG.getStore(Val.getValue(1), dl, Val, AddPtr,
                          MachinePointerInfo::getStack(TOCSaveOffset),
@@ -5129,9 +5342,9 @@ PPCTargetLowering::LowerCall_64SVR4(SDValue Chain, SDValue Callee,
     PrepareTailCall(DAG, InFlag, Chain, dl, true, SPDiff, NumBytes, LROp,
                     FPOp, true, TailCallArguments);
 
-  return FinishCall(CallConv, dl, isTailCall, isVarArg, IsPatchPoint, DAG,
-                    RegsToPass, InFlag, Chain, CallSeqStart, Callee, SPDiff,
-                    NumBytes, Ins, InVals, CS);
+  return FinishCall(CallConv, dl, isTailCall, isVarArg, IsPatchPoint,
+		    hasNest, DAG, RegsToPass, InFlag, Chain, CallSeqStart,
+                    Callee, SPDiff, NumBytes, Ins, InVals, CS);
 }
 
 SDValue
@@ -5147,7 +5360,7 @@ PPCTargetLowering::LowerCall_Darwin(SDValue Chain, SDValue Callee,
 
   unsigned NumOps = Outs.size();
 
-  EVT PtrVT = DAG.getTargetLoweringInfo().getPointerTy();
+  EVT PtrVT = DAG.getTargetLoweringInfo().getPointerTy(DAG.getDataLayout());
   bool isPPC64 = PtrVT == MVT::i64;
   unsigned PtrByteSize = isPPC64 ? 8 : 4;
 
@@ -5223,7 +5436,7 @@ PPCTargetLowering::LowerCall_Darwin(SDValue Chain, SDValue Callee,
 
   // Adjust the stack pointer for the new arguments...
   // These operations are automatically eliminated by the prolog/epilog pass
-  Chain = DAG.getCALLSEQ_START(Chain, DAG.getIntPtrConstant(NumBytes, true),
+  Chain = DAG.getCALLSEQ_START(Chain, DAG.getIntPtrConstant(NumBytes, dl, true),
                                dl);
   SDValue CallSeqStart = Chain;
 
@@ -5279,7 +5492,7 @@ PPCTargetLowering::LowerCall_Darwin(SDValue Chain, SDValue Callee,
     // register cannot be found for it.
     SDValue PtrOff;
 
-    PtrOff = DAG.getConstant(ArgOffset, StackPtr.getValueType());
+    PtrOff = DAG.getConstant(ArgOffset, dl, StackPtr.getValueType());
 
     PtrOff = DAG.getNode(ISD::ADD, dl, PtrVT, StackPtr, PtrOff);
 
@@ -5308,7 +5521,7 @@ PPCTargetLowering::LowerCall_Darwin(SDValue Chain, SDValue Callee,
 
           ArgOffset += PtrByteSize;
         } else {
-          SDValue Const = DAG.getConstant(PtrByteSize - Size,
+          SDValue Const = DAG.getConstant(PtrByteSize - Size, dl,
                                           PtrOff.getValueType());
           SDValue AddPtr = DAG.getNode(ISD::ADD, dl, PtrVT, PtrOff, Const);
           Chain = CallSeqStart = createMemcpyOutsideCallSeq(Arg, AddPtr,
@@ -5329,7 +5542,7 @@ PPCTargetLowering::LowerCall_Darwin(SDValue Chain, SDValue Callee,
       // copy the pieces of the object that fit into registers from the
       // parameter save area.
       for (unsigned j=0; j<Size; j+=PtrByteSize) {
-        SDValue Const = DAG.getConstant(j, PtrOff.getValueType());
+        SDValue Const = DAG.getConstant(j, dl, PtrOff.getValueType());
         SDValue AddArg = DAG.getNode(ISD::ADD, dl, PtrVT, Arg, Const);
         if (GPR_idx != NumGPRs) {
           SDValue Load = DAG.getLoad(PtrVT, dl, Chain, AddArg,
@@ -5382,7 +5595,7 @@ PPCTargetLowering::LowerCall_Darwin(SDValue Chain, SDValue Callee,
             RegsToPass.push_back(std::make_pair(GPR[GPR_idx++], Load));
           }
           if (GPR_idx != NumGPRs && Arg.getValueType() == MVT::f64 && !isPPC64){
-            SDValue ConstFour = DAG.getConstant(4, PtrOff.getValueType());
+            SDValue ConstFour = DAG.getConstant(4, dl, PtrOff.getValueType());
             PtrOff = DAG.getNode(ISD::ADD, dl, PtrVT, PtrOff, ConstFour);
             SDValue Load = DAG.getLoad(PtrVT, dl, Store, PtrOff,
                                        MachinePointerInfo(),
@@ -5427,7 +5640,7 @@ PPCTargetLowering::LowerCall_Darwin(SDValue Chain, SDValue Callee,
         // We could elide this store in the case where the object fits
         // entirely in R registers.  Maybe later.
         PtrOff = DAG.getNode(ISD::ADD, dl, PtrVT, StackPtr,
-                            DAG.getConstant(ArgOffset, PtrVT));
+                             DAG.getConstant(ArgOffset, dl, PtrVT));
         SDValue Store = DAG.getStore(Chain, dl, Arg, PtrOff,
                                      MachinePointerInfo(), false, false, 0);
         MemOpChains.push_back(Store);
@@ -5443,7 +5656,7 @@ PPCTargetLowering::LowerCall_Darwin(SDValue Chain, SDValue Callee,
           if (GPR_idx == NumGPRs)
             break;
           SDValue Ix = DAG.getNode(ISD::ADD, dl, PtrVT, PtrOff,
-                                  DAG.getConstant(i, PtrVT));
+                                   DAG.getConstant(i, dl, PtrVT));
           SDValue Load = DAG.getLoad(PtrVT, dl, Store, Ix, MachinePointerInfo(),
                                      false, false, false, 0);
           MemOpChains.push_back(Load.getValue(1));
@@ -5520,7 +5733,8 @@ PPCTargetLowering::LowerCall_Darwin(SDValue Chain, SDValue Callee,
     PrepareTailCall(DAG, InFlag, Chain, dl, isPPC64, SPDiff, NumBytes, LROp,
                     FPOp, true, TailCallArguments);
 
-  return FinishCall(CallConv, dl, isTailCall, isVarArg, IsPatchPoint, DAG,
+  return FinishCall(CallConv, dl, isTailCall, isVarArg, IsPatchPoint,
+                    /* unused except on PPC64 ELFv1 */ false, DAG,
                     RegsToPass, InFlag, Chain, CallSeqStart, Callee, SPDiff,
                     NumBytes, Ins, InVals, CS);
 }
@@ -5591,7 +5805,7 @@ SDValue PPCTargetLowering::LowerSTACKRESTORE(SDValue Op, SelectionDAG &DAG,
   SDLoc dl(Op);
 
   // Get the corect type for pointers.
-  EVT PtrVT = DAG.getTargetLoweringInfo().getPointerTy();
+  EVT PtrVT = DAG.getTargetLoweringInfo().getPointerTy(DAG.getDataLayout());
 
   // Construct the stack pointer operand.
   bool isPPC64 = Subtarget.isPPC64();
@@ -5621,7 +5835,7 @@ SDValue
 PPCTargetLowering::getReturnAddrFrameIndex(SelectionDAG & DAG) const {
   MachineFunction &MF = DAG.getMachineFunction();
   bool isPPC64 = Subtarget.isPPC64();
-  EVT PtrVT = DAG.getTargetLoweringInfo().getPointerTy();
+  EVT PtrVT = DAG.getTargetLoweringInfo().getPointerTy(MF.getDataLayout());
 
   // Get current frame pointer save index.  The users of this index will be
   // primarily DYNALLOC instructions.
@@ -5644,7 +5858,7 @@ SDValue
 PPCTargetLowering::getFramePointerFrameIndex(SelectionDAG & DAG) const {
   MachineFunction &MF = DAG.getMachineFunction();
   bool isPPC64 = Subtarget.isPPC64();
-  EVT PtrVT = DAG.getTargetLoweringInfo().getPointerTy();
+  EVT PtrVT = DAG.getTargetLoweringInfo().getPointerTy(MF.getDataLayout());
 
   // Get current frame pointer save index.  The users of this index will be
   // primarily DYNALLOC instructions.
@@ -5672,10 +5886,10 @@ SDValue PPCTargetLowering::LowerDYNAMIC_STACKALLOC(SDValue Op,
   SDLoc dl(Op);
 
   // Get the corect type for pointers.
-  EVT PtrVT = DAG.getTargetLoweringInfo().getPointerTy();
+  EVT PtrVT = DAG.getTargetLoweringInfo().getPointerTy(DAG.getDataLayout());
   // Negate the size.
   SDValue NegSize = DAG.getNode(ISD::SUB, dl, PtrVT,
-                                  DAG.getConstant(0, PtrVT), Size);
+                                DAG.getConstant(0, dl, PtrVT), Size);
   // Construct a node for the frame pointer save index.
   SDValue FPSIdx = getFramePointerFrameIndex(DAG);
   // Build a DYNALLOC node.
@@ -5715,8 +5929,9 @@ SDValue PPCTargetLowering::LowerLOAD(SDValue Op, SelectionDAG &DAG) const {
   SDValue BasePtr = LD->getBasePtr();
   MachineMemOperand *MMO = LD->getMemOperand();
 
-  SDValue NewLD = DAG.getExtLoad(ISD::EXTLOAD, dl, getPointerTy(), Chain,
-                                 BasePtr, MVT::i8, MMO);
+  SDValue NewLD =
+      DAG.getExtLoad(ISD::EXTLOAD, dl, getPointerTy(DAG.getDataLayout()), Chain,
+                     BasePtr, MVT::i8, MMO);
   SDValue Result = DAG.getNode(ISD::TRUNCATE, dl, MVT::i1, NewLD);
 
   SDValue Ops[] = { Result, SDValue(NewLD.getNode(), 1) };
@@ -5740,7 +5955,8 @@ SDValue PPCTargetLowering::LowerSTORE(SDValue Op, SelectionDAG &DAG) const {
   SDValue Value = ST->getValue();
   MachineMemOperand *MMO = ST->getMemOperand();
 
-  Value = DAG.getNode(ISD::ZERO_EXTEND, dl, getPointerTy(), Value);
+  Value = DAG.getNode(ISD::ZERO_EXTEND, dl, getPointerTy(DAG.getDataLayout()),
+                      Value);
   return DAG.getTruncStore(Chain, dl, Value, BasePtr, MVT::i8, MMO);
 }
 
@@ -5905,7 +6121,7 @@ void PPCTargetLowering::LowerFP_TO_INTForReuse(SDValue Op, ReuseLoadInfo &RLI,
   // add in a bias.
   if (Op.getValueType() == MVT::i32 && !i32Stack) {
     FIPtr = DAG.getNode(ISD::ADD, dl, FIPtr.getValueType(), FIPtr,
-                        DAG.getConstant(4, FIPtr.getValueType()));
+                        DAG.getConstant(4, dl, FIPtr.getValueType()));
     MPI = MPI.getWithOffset(4);
   }
 
@@ -6077,7 +6293,7 @@ SDValue PPCTargetLowering::LowerINT_TO_FP(SDValue Op,
     // This can be done with an fma and the 0.5 constant: (V+1.0)*0.5 = 0.5*V+0.5
     Value = DAG.getNode(PPCISD::QBFLT, dl, MVT::v4f64, Value);
   
-    SDValue FPHalfs = DAG.getConstantFP(0.5, MVT::f64);
+    SDValue FPHalfs = DAG.getConstantFP(0.5, dl, MVT::f64);
     FPHalfs = DAG.getNode(ISD::BUILD_VECTOR, dl, MVT::v4f64,
                           FPHalfs, FPHalfs, FPHalfs, FPHalfs);
   
@@ -6085,7 +6301,8 @@ SDValue PPCTargetLowering::LowerINT_TO_FP(SDValue Op,
 
     if (Op.getValueType() != MVT::v4f64)
       Value = DAG.getNode(ISD::FP_ROUND, dl,
-                          Op.getValueType(), Value, DAG.getIntPtrConstant(1));
+                          Op.getValueType(), Value,
+                          DAG.getIntPtrConstant(1, dl));
     return Value;
   }
 
@@ -6095,8 +6312,8 @@ SDValue PPCTargetLowering::LowerINT_TO_FP(SDValue Op,
 
   if (Op.getOperand(0).getValueType() == MVT::i1)
     return DAG.getNode(ISD::SELECT, dl, Op.getValueType(), Op.getOperand(0),
-                       DAG.getConstantFP(1.0, Op.getValueType()),
-                       DAG.getConstantFP(0.0, Op.getValueType()));
+                       DAG.getConstantFP(1.0, dl, Op.getValueType()),
+                       DAG.getConstantFP(0.0, dl, Op.getValueType()));
 
   // If we have direct moves, we can do all the conversion, skip the store/load
   // however, without FPCVT we can't do most conversions.
@@ -6140,12 +6357,12 @@ SDValue PPCTargetLowering::LowerINT_TO_FP(SDValue Op,
       // bit 12 (value 2048) is set instead, so that the final rounding
       // to single-precision gets the correct result.
       SDValue Round = DAG.getNode(ISD::AND, dl, MVT::i64,
-                                  SINT, DAG.getConstant(2047, MVT::i64));
+                                  SINT, DAG.getConstant(2047, dl, MVT::i64));
       Round = DAG.getNode(ISD::ADD, dl, MVT::i64,
-                          Round, DAG.getConstant(2047, MVT::i64));
+                          Round, DAG.getConstant(2047, dl, MVT::i64));
       Round = DAG.getNode(ISD::OR, dl, MVT::i64, Round, SINT);
       Round = DAG.getNode(ISD::AND, dl, MVT::i64,
-                          Round, DAG.getConstant(-2048, MVT::i64));
+                          Round, DAG.getConstant(-2048, dl, MVT::i64));
 
       // However, we cannot use that value unconditionally: if the magnitude
       // of the input value is small, the bit-twiddling we did above might
@@ -6156,11 +6373,11 @@ SDValue PPCTargetLowering::LowerINT_TO_FP(SDValue Op,
       // bits are all sign-bit copies, and use the rounded value computed
       // above otherwise.
       SDValue Cond = DAG.getNode(ISD::SRA, dl, MVT::i64,
-                                 SINT, DAG.getConstant(53, MVT::i32));
+                                 SINT, DAG.getConstant(53, dl, MVT::i32));
       Cond = DAG.getNode(ISD::ADD, dl, MVT::i64,
-                         Cond, DAG.getConstant(1, MVT::i64));
+                         Cond, DAG.getConstant(1, dl, MVT::i64));
       Cond = DAG.getSetCC(dl, MVT::i32,
-                          Cond, DAG.getConstant(1, MVT::i64), ISD::SETUGT);
+                          Cond, DAG.getConstant(1, dl, MVT::i64), ISD::SETUGT);
 
       SINT = DAG.getNode(ISD::SELECT, dl, MVT::i64, Cond, Round, SINT);
     }
@@ -6200,7 +6417,7 @@ SDValue PPCTargetLowering::LowerINT_TO_FP(SDValue Op,
                  SINT.getOpcode() == ISD::ZERO_EXTEND)) &&
                SINT.getOperand(0).getValueType() == MVT::i32) {
       MachineFrameInfo *FrameInfo = MF.getFrameInfo();
-      EVT PtrVT = DAG.getTargetLoweringInfo().getPointerTy();
+      EVT PtrVT = DAG.getTargetLoweringInfo().getPointerTy(DAG.getDataLayout());
 
       int FrameIdx = FrameInfo->CreateStackObject(4, 4, false);
       SDValue FIdx = DAG.getFrameIndex(FrameIdx, PtrVT);
@@ -6233,7 +6450,7 @@ SDValue PPCTargetLowering::LowerINT_TO_FP(SDValue Op,
 
     if (Op.getValueType() == MVT::f32 && !Subtarget.hasFPCVT())
       FP = DAG.getNode(ISD::FP_ROUND, dl,
-                       MVT::f32, FP, DAG.getIntPtrConstant(0));
+                       MVT::f32, FP, DAG.getIntPtrConstant(0, dl));
     return FP;
   }
 
@@ -6245,7 +6462,7 @@ SDValue PPCTargetLowering::LowerINT_TO_FP(SDValue Op,
   // then lfd it and fcfid it.
   MachineFunction &MF = DAG.getMachineFunction();
   MachineFrameInfo *FrameInfo = MF.getFrameInfo();
-  EVT PtrVT = DAG.getTargetLoweringInfo().getPointerTy();
+  EVT PtrVT = DAG.getTargetLoweringInfo().getPointerTy(MF.getDataLayout());
 
   SDValue Ld;
   if (Subtarget.hasLFIWAX() || Subtarget.hasFPCVT()) {
@@ -6303,7 +6520,8 @@ SDValue PPCTargetLowering::LowerINT_TO_FP(SDValue Op,
   // FCFID it and return it.
   SDValue FP = DAG.getNode(FCFOp, dl, FCFTy, Ld);
   if (Op.getValueType() == MVT::f32 && !Subtarget.hasFPCVT())
-    FP = DAG.getNode(ISD::FP_ROUND, dl, MVT::f32, FP, DAG.getIntPtrConstant(0));
+    FP = DAG.getNode(ISD::FP_ROUND, dl, MVT::f32, FP,
+                     DAG.getIntPtrConstant(0, dl));
   return FP;
 }
 
@@ -6331,7 +6549,7 @@ SDValue PPCTargetLowering::LowerFLT_ROUNDS_(SDValue Op,
 
   MachineFunction &MF = DAG.getMachineFunction();
   EVT VT = Op.getValueType();
-  EVT PtrVT = DAG.getTargetLoweringInfo().getPointerTy();
+  EVT PtrVT = DAG.getTargetLoweringInfo().getPointerTy(MF.getDataLayout());
 
   // Save FP Control Word to register
   EVT NodeTys[] = {
@@ -6347,7 +6565,7 @@ SDValue PPCTargetLowering::LowerFLT_ROUNDS_(SDValue Op,
                                StackSlot, MachinePointerInfo(), false, false,0);
 
   // Load FP Control Word from low 32 bits of stack slot.
-  SDValue Four = DAG.getConstant(4, PtrVT);
+  SDValue Four = DAG.getConstant(4, dl, PtrVT);
   SDValue Addr = DAG.getNode(ISD::ADD, dl, PtrVT, StackSlot, Four);
   SDValue CWD = DAG.getLoad(MVT::i32, dl, Store, Addr, MachinePointerInfo(),
                             false, false, false, 0);
@@ -6355,14 +6573,14 @@ SDValue PPCTargetLowering::LowerFLT_ROUNDS_(SDValue Op,
   // Transform as necessary
   SDValue CWD1 =
     DAG.getNode(ISD::AND, dl, MVT::i32,
-                CWD, DAG.getConstant(3, MVT::i32));
+                CWD, DAG.getConstant(3, dl, MVT::i32));
   SDValue CWD2 =
     DAG.getNode(ISD::SRL, dl, MVT::i32,
                 DAG.getNode(ISD::AND, dl, MVT::i32,
                             DAG.getNode(ISD::XOR, dl, MVT::i32,
-                                        CWD, DAG.getConstant(3, MVT::i32)),
-                            DAG.getConstant(3, MVT::i32)),
-                DAG.getConstant(1, MVT::i32));
+                                        CWD, DAG.getConstant(3, dl, MVT::i32)),
+                            DAG.getConstant(3, dl, MVT::i32)),
+                DAG.getConstant(1, dl, MVT::i32));
 
   SDValue RetVal =
     DAG.getNode(ISD::XOR, dl, MVT::i32, CWD1, CWD2);
@@ -6387,12 +6605,12 @@ SDValue PPCTargetLowering::LowerSHL_PARTS(SDValue Op, SelectionDAG &DAG) const {
   EVT AmtVT = Amt.getValueType();
 
   SDValue Tmp1 = DAG.getNode(ISD::SUB, dl, AmtVT,
-                             DAG.getConstant(BitWidth, AmtVT), Amt);
+                             DAG.getConstant(BitWidth, dl, AmtVT), Amt);
   SDValue Tmp2 = DAG.getNode(PPCISD::SHL, dl, VT, Hi, Amt);
   SDValue Tmp3 = DAG.getNode(PPCISD::SRL, dl, VT, Lo, Tmp1);
   SDValue Tmp4 = DAG.getNode(ISD::OR , dl, VT, Tmp2, Tmp3);
   SDValue Tmp5 = DAG.getNode(ISD::ADD, dl, AmtVT, Amt,
-                             DAG.getConstant(-BitWidth, AmtVT));
+                             DAG.getConstant(-BitWidth, dl, AmtVT));
   SDValue Tmp6 = DAG.getNode(PPCISD::SHL, dl, VT, Lo, Tmp5);
   SDValue OutHi = DAG.getNode(ISD::OR, dl, VT, Tmp4, Tmp6);
   SDValue OutLo = DAG.getNode(PPCISD::SHL, dl, VT, Lo, Amt);
@@ -6416,12 +6634,12 @@ SDValue PPCTargetLowering::LowerSRL_PARTS(SDValue Op, SelectionDAG &DAG) const {
   EVT AmtVT = Amt.getValueType();
 
   SDValue Tmp1 = DAG.getNode(ISD::SUB, dl, AmtVT,
-                             DAG.getConstant(BitWidth, AmtVT), Amt);
+                             DAG.getConstant(BitWidth, dl, AmtVT), Amt);
   SDValue Tmp2 = DAG.getNode(PPCISD::SRL, dl, VT, Lo, Amt);
   SDValue Tmp3 = DAG.getNode(PPCISD::SHL, dl, VT, Hi, Tmp1);
   SDValue Tmp4 = DAG.getNode(ISD::OR, dl, VT, Tmp2, Tmp3);
   SDValue Tmp5 = DAG.getNode(ISD::ADD, dl, AmtVT, Amt,
-                             DAG.getConstant(-BitWidth, AmtVT));
+                             DAG.getConstant(-BitWidth, dl, AmtVT));
   SDValue Tmp6 = DAG.getNode(PPCISD::SRL, dl, VT, Hi, Tmp5);
   SDValue OutLo = DAG.getNode(ISD::OR, dl, VT, Tmp4, Tmp6);
   SDValue OutHi = DAG.getNode(PPCISD::SRL, dl, VT, Hi, Amt);
@@ -6444,15 +6662,15 @@ SDValue PPCTargetLowering::LowerSRA_PARTS(SDValue Op, SelectionDAG &DAG) const {
   EVT AmtVT = Amt.getValueType();
 
   SDValue Tmp1 = DAG.getNode(ISD::SUB, dl, AmtVT,
-                             DAG.getConstant(BitWidth, AmtVT), Amt);
+                             DAG.getConstant(BitWidth, dl, AmtVT), Amt);
   SDValue Tmp2 = DAG.getNode(PPCISD::SRL, dl, VT, Lo, Amt);
   SDValue Tmp3 = DAG.getNode(PPCISD::SHL, dl, VT, Hi, Tmp1);
   SDValue Tmp4 = DAG.getNode(ISD::OR, dl, VT, Tmp2, Tmp3);
   SDValue Tmp5 = DAG.getNode(ISD::ADD, dl, AmtVT, Amt,
-                             DAG.getConstant(-BitWidth, AmtVT));
+                             DAG.getConstant(-BitWidth, dl, AmtVT));
   SDValue Tmp6 = DAG.getNode(PPCISD::SRA, dl, VT, Hi, Tmp5);
   SDValue OutHi = DAG.getNode(PPCISD::SRA, dl, VT, Hi, Amt);
-  SDValue OutLo = DAG.getSelectCC(dl, Tmp5, DAG.getConstant(0, AmtVT),
+  SDValue OutLo = DAG.getSelectCC(dl, Tmp5, DAG.getConstant(0, dl, AmtVT),
                                   Tmp4, Tmp6, ISD::SETLE);
   SDValue OutOps[] = { OutLo, OutHi };
   return DAG.getMergeValues(OutOps, dl);
@@ -6481,7 +6699,7 @@ static SDValue BuildSplatI(int Val, unsigned SplatSize, EVT VT,
   EVT CanonicalVT = VTys[SplatSize-1];
 
   // Build a canonical splat for this value.
-  SDValue Elt = DAG.getConstant(Val, MVT::i32);
+  SDValue Elt = DAG.getConstant(Val, dl, MVT::i32);
   SmallVector<SDValue, 8> Ops;
   Ops.assign(CanonicalVT.getVectorNumElements(), Elt);
   SDValue Res = DAG.getNode(ISD::BUILD_VECTOR, dl, CanonicalVT, Ops);
@@ -6495,7 +6713,7 @@ static SDValue BuildIntrinsicOp(unsigned IID, SDValue Op,
                                 EVT DestVT = MVT::Other) {
   if (DestVT == MVT::Other) DestVT = Op.getValueType();
   return DAG.getNode(ISD::INTRINSIC_WO_CHAIN, dl, DestVT,
-                     DAG.getConstant(IID, MVT::i32), Op);
+                     DAG.getConstant(IID, dl, MVT::i32), Op);
 }
 
 /// BuildIntrinsicOp - Return a binary operator intrinsic node with the
@@ -6505,7 +6723,7 @@ static SDValue BuildIntrinsicOp(unsigned IID, SDValue LHS, SDValue RHS,
                                 EVT DestVT = MVT::Other) {
   if (DestVT == MVT::Other) DestVT = LHS.getValueType();
   return DAG.getNode(ISD::INTRINSIC_WO_CHAIN, dl, DestVT,
-                     DAG.getConstant(IID, MVT::i32), LHS, RHS);
+                     DAG.getConstant(IID, dl, MVT::i32), LHS, RHS);
 }
 
 /// BuildIntrinsicOp - Return a ternary operator intrinsic node with the
@@ -6515,7 +6733,7 @@ static SDValue BuildIntrinsicOp(unsigned IID, SDValue Op0, SDValue Op1,
                                 SDLoc dl, EVT DestVT = MVT::Other) {
   if (DestVT == MVT::Other) DestVT = Op0.getValueType();
   return DAG.getNode(ISD::INTRINSIC_WO_CHAIN, dl, DestVT,
-                     DAG.getConstant(IID, MVT::i32), Op0, Op1, Op2);
+                     DAG.getConstant(IID, dl, MVT::i32), Op0, Op1, Op2);
 }
 
 
@@ -6552,7 +6770,7 @@ SDValue PPCTargetLowering::LowerBUILD_VECTOR(SDValue Op,
     MachineFrameInfo *FrameInfo = DAG.getMachineFunction().getFrameInfo();
     int FrameIdx = FrameInfo->CreateStackObject(16, 16, false);
     MachinePointerInfo PtrInfo = MachinePointerInfo::getFixedStack(FrameIdx);
-    EVT PtrVT = getPointerTy();
+    EVT PtrVT = getPointerTy(DAG.getDataLayout());
     SDValue FIdx = DAG.getFrameIndex(FrameIdx, PtrVT);
 
     assert(BVN->getNumOperands() == 4 &&
@@ -6585,9 +6803,9 @@ SDValue PPCTargetLowering::LowerBUILD_VECTOR(SDValue Op,
       }
 
       Constant *CP = ConstantVector::get(CV);
-      SDValue CPIdx = DAG.getConstantPool(CP, getPointerTy(),
-                      16 /* alignment */);
- 
+      SDValue CPIdx = DAG.getConstantPool(CP, getPointerTy(DAG.getDataLayout()),
+                                          16 /* alignment */);
+
       SmallVector<SDValue, 2> Ops;
       Ops.push_back(DAG.getEntryNode());
       Ops.push_back(CPIdx);
@@ -6607,7 +6825,7 @@ SDValue PPCTargetLowering::LowerBUILD_VECTOR(SDValue Op,
       if (BVN->getOperand(i).getOpcode() == ISD::UNDEF) continue;
 
       unsigned Offset = 4*i;
-      SDValue Idx = DAG.getConstant(Offset, FIdx.getValueType());
+      SDValue Idx = DAG.getConstant(Offset, dl, FIdx.getValueType());
       Idx = DAG.getNode(ISD::ADD, dl, FIdx.getValueType(), FIdx, Idx);
 
       unsigned StoreSize = BVN->getOperand(i).getValueType().getStoreSize();
@@ -6641,7 +6859,7 @@ SDValue PPCTargetLowering::LowerBUILD_VECTOR(SDValue Op,
 
     SmallVector<SDValue, 2> Ops;
     Ops.push_back(StoreChain);
-    Ops.push_back(DAG.getConstant(Intrinsic::ppc_qpx_qvlfiwz, MVT::i32));
+    Ops.push_back(DAG.getConstant(Intrinsic::ppc_qpx_qvlfiwz, dl, MVT::i32));
     Ops.push_back(FIdx);
 
     SmallVector<EVT, 2> ValueVTs;
@@ -6652,10 +6870,10 @@ SDValue PPCTargetLowering::LowerBUILD_VECTOR(SDValue Op,
     SDValue LoadedVect = DAG.getMemIntrinsicNode(ISD::INTRINSIC_W_CHAIN,
       dl, VTs, Ops, MVT::v4i32, PtrInfo);
     LoadedVect = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, dl, MVT::v4f64,
-      DAG.getConstant(Intrinsic::ppc_qpx_qvfcfidu, MVT::i32),
+      DAG.getConstant(Intrinsic::ppc_qpx_qvfcfidu, dl, MVT::i32),
       LoadedVect);
 
-    SDValue FPZeros = DAG.getConstantFP(0.0, MVT::f64);
+    SDValue FPZeros = DAG.getConstantFP(0.0, dl, MVT::f64);
     FPZeros = DAG.getNode(ISD::BUILD_VECTOR, dl, MVT::v4f64,
                           FPZeros, FPZeros, FPZeros, FPZeros);
 
@@ -6685,7 +6903,7 @@ SDValue PPCTargetLowering::LowerBUILD_VECTOR(SDValue Op,
   if (SplatBits == 0) {
     // Canonicalize all zero vectors to be v4i32.
     if (Op.getValueType() != MVT::v4i32 || HasAnyUndefs) {
-      SDValue Z = DAG.getConstant(0, MVT::i32);
+      SDValue Z = DAG.getConstant(0, dl, MVT::i32);
       Z = DAG.getNode(ISD::BUILD_VECTOR, dl, MVT::v4i32, Z, Z, Z, Z);
       Op = DAG.getNode(ISD::BITCAST, dl, Op.getValueType(), Z);
     }
@@ -6712,10 +6930,10 @@ SDValue PPCTargetLowering::LowerBUILD_VECTOR(SDValue Op,
     // To avoid having these optimizations undone by constant folding,
     // we convert to a pseudo that will be expanded later into one of
     // the above forms.
-    SDValue Elt = DAG.getConstant(SextVal, MVT::i32);
+    SDValue Elt = DAG.getConstant(SextVal, dl, MVT::i32);
     EVT VT = (SplatSize == 1 ? MVT::v16i8 :
               (SplatSize == 2 ? MVT::v8i16 : MVT::v4i32));
-    SDValue EltSize = DAG.getConstant(SplatSize, MVT::i32);
+    SDValue EltSize = DAG.getConstant(SplatSize, dl, MVT::i32);
     SDValue RetVal = DAG.getNode(PPCISD::VADD_SPLAT, dl, VT, Elt, EltSize);
     if (VT == Op.getValueType())
       return RetVal;
@@ -6802,17 +7020,20 @@ SDValue PPCTargetLowering::LowerBUILD_VECTOR(SDValue Op,
     // t = vsplti c, result = vsldoi t, t, 1
     if (SextVal == (int)(((unsigned)i << 8) | (i < 0 ? 0xFF : 0))) {
       SDValue T = BuildSplatI(i, SplatSize, MVT::v16i8, DAG, dl);
-      return BuildVSLDOI(T, T, 1, Op.getValueType(), DAG, dl);
+      unsigned Amt = Subtarget.isLittleEndian() ? 15 : 1;
+      return BuildVSLDOI(T, T, Amt, Op.getValueType(), DAG, dl);
     }
     // t = vsplti c, result = vsldoi t, t, 2
     if (SextVal == (int)(((unsigned)i << 16) | (i < 0 ? 0xFFFF : 0))) {
       SDValue T = BuildSplatI(i, SplatSize, MVT::v16i8, DAG, dl);
-      return BuildVSLDOI(T, T, 2, Op.getValueType(), DAG, dl);
+      unsigned Amt = Subtarget.isLittleEndian() ? 14 : 2;
+      return BuildVSLDOI(T, T, Amt, Op.getValueType(), DAG, dl);
     }
     // t = vsplti c, result = vsldoi t, t, 3
     if (SextVal == (int)(((unsigned)i << 24) | (i < 0 ? 0xFFFFFF : 0))) {
       SDValue T = BuildSplatI(i, SplatSize, MVT::v16i8, DAG, dl);
-      return BuildVSLDOI(T, T, 3, Op.getValueType(), DAG, dl);
+      unsigned Amt = Subtarget.isLittleEndian() ? 13 : 3;
+      return BuildVSLDOI(T, T, Amt, Op.getValueType(), DAG, dl);
     }
   }
 
@@ -6918,7 +7139,7 @@ SDValue PPCTargetLowering::LowerVECTOR_SHUFFLE(SDValue Op,
     int AlignIdx = PPC::isQVALIGNIShuffleMask(SVOp);
     if (AlignIdx != -1) {
       return DAG.getNode(PPCISD::QVALIGNI, dl, VT, V1, V2,
-                         DAG.getConstant(AlignIdx, MVT::i32));
+                         DAG.getConstant(AlignIdx, dl, MVT::i32));
     } else if (SVOp->isSplat()) {
       int SplatIdx = SVOp->getSplatIndex();
       if (SplatIdx >= 4) {
@@ -6930,7 +7151,7 @@ SDValue PPCTargetLowering::LowerVECTOR_SHUFFLE(SDValue Op,
       // nothing to do.
 
       return DAG.getNode(PPCISD::QVESPLATI, dl, VT, V1,
-                         DAG.getConstant(SplatIdx, MVT::i32));
+                         DAG.getConstant(SplatIdx, dl, MVT::i32));
     }
 
     // Lower this into a qvgpci/qvfperm pair.
@@ -6944,7 +7165,7 @@ SDValue PPCTargetLowering::LowerVECTOR_SHUFFLE(SDValue Op,
     }
 
     SDValue V3 = DAG.getNode(PPCISD::QVGPCI, dl, MVT::v4f64,
-                             DAG.getConstant(idx, MVT::i32));
+                             DAG.getConstant(idx, dl, MVT::i32));
     return DAG.getNode(PPCISD::QVFPERM, dl, VT, V1, V2, V3);
   }
 
@@ -6957,13 +7178,16 @@ SDValue PPCTargetLowering::LowerVECTOR_SHUFFLE(SDValue Op,
         PPC::isSplatShuffleMask(SVOp, 4) ||
         PPC::isVPKUWUMShuffleMask(SVOp, 1, DAG) ||
         PPC::isVPKUHUMShuffleMask(SVOp, 1, DAG) ||
+        PPC::isVPKUDUMShuffleMask(SVOp, 1, DAG) ||
         PPC::isVSLDOIShuffleMask(SVOp, 1, DAG) != -1 ||
         PPC::isVMRGLShuffleMask(SVOp, 1, 1, DAG) ||
         PPC::isVMRGLShuffleMask(SVOp, 2, 1, DAG) ||
         PPC::isVMRGLShuffleMask(SVOp, 4, 1, DAG) ||
         PPC::isVMRGHShuffleMask(SVOp, 1, 1, DAG) ||
         PPC::isVMRGHShuffleMask(SVOp, 2, 1, DAG) ||
-        PPC::isVMRGHShuffleMask(SVOp, 4, 1, DAG)) {
+        PPC::isVMRGHShuffleMask(SVOp, 4, 1, DAG) ||
+        PPC::isVMRGEOShuffleMask(SVOp, true, 1, DAG)   ||
+        PPC::isVMRGEOShuffleMask(SVOp, false, 1, DAG)) {
       return Op;
     }
   }
@@ -6974,13 +7198,16 @@ SDValue PPCTargetLowering::LowerVECTOR_SHUFFLE(SDValue Op,
   unsigned int ShuffleKind = isLittleEndian ? 2 : 0;
   if (PPC::isVPKUWUMShuffleMask(SVOp, ShuffleKind, DAG) ||
       PPC::isVPKUHUMShuffleMask(SVOp, ShuffleKind, DAG) ||
+      PPC::isVPKUDUMShuffleMask(SVOp, ShuffleKind, DAG) ||
       PPC::isVSLDOIShuffleMask(SVOp, ShuffleKind, DAG) != -1 ||
       PPC::isVMRGLShuffleMask(SVOp, 1, ShuffleKind, DAG) ||
       PPC::isVMRGLShuffleMask(SVOp, 2, ShuffleKind, DAG) ||
       PPC::isVMRGLShuffleMask(SVOp, 4, ShuffleKind, DAG) ||
       PPC::isVMRGHShuffleMask(SVOp, 1, ShuffleKind, DAG) ||
       PPC::isVMRGHShuffleMask(SVOp, 2, ShuffleKind, DAG) ||
-      PPC::isVMRGHShuffleMask(SVOp, 4, ShuffleKind, DAG))
+      PPC::isVMRGHShuffleMask(SVOp, 4, ShuffleKind, DAG) ||
+      PPC::isVMRGEOShuffleMask(SVOp, true, ShuffleKind, DAG)             ||
+      PPC::isVMRGEOShuffleMask(SVOp, false, ShuffleKind, DAG))
     return Op;
 
   // Check to see if this is a shuffle of 4-byte values.  If so, we can use our
@@ -7059,10 +7286,10 @@ SDValue PPCTargetLowering::LowerVECTOR_SHUFFLE(SDValue Op,
 
     for (unsigned j = 0; j != BytesPerElement; ++j)
       if (isLittleEndian)
-        ResultMask.push_back(DAG.getConstant(31 - (SrcElt*BytesPerElement+j),
-                                             MVT::i32));
+        ResultMask.push_back(DAG.getConstant(31 - (SrcElt*BytesPerElement + j),
+                                             dl, MVT::i32));
       else
-        ResultMask.push_back(DAG.getConstant(SrcElt*BytesPerElement+j,
+        ResultMask.push_back(DAG.getConstant(SrcElt*BytesPerElement + j, dl,
                                              MVT::i32));
   }
 
@@ -7190,7 +7417,7 @@ SDValue PPCTargetLowering::LowerINTRINSIC_WO_CHAIN(SDValue Op,
   if (!isDot) {
     SDValue Tmp = DAG.getNode(PPCISD::VCMP, dl, Op.getOperand(2).getValueType(),
                               Op.getOperand(1), Op.getOperand(2),
-                              DAG.getConstant(CompareOpc, MVT::i32));
+                              DAG.getConstant(CompareOpc, dl, MVT::i32));
     return DAG.getNode(ISD::BITCAST, dl, Op.getValueType(), Tmp);
   }
 
@@ -7198,7 +7425,7 @@ SDValue PPCTargetLowering::LowerINTRINSIC_WO_CHAIN(SDValue Op,
   SDValue Ops[] = {
     Op.getOperand(2),  // LHS
     Op.getOperand(3),  // RHS
-    DAG.getConstant(CompareOpc, MVT::i32)
+    DAG.getConstant(CompareOpc, dl, MVT::i32)
   };
   EVT VTs[] = { Op.getOperand(2).getValueType(), MVT::Glue };
   SDValue CompNode = DAG.getNode(PPCISD::VCMPo, dl, VTs, Ops);
@@ -7230,15 +7457,15 @@ SDValue PPCTargetLowering::LowerINTRINSIC_WO_CHAIN(SDValue Op,
 
   // Shift the bit into the low position.
   Flags = DAG.getNode(ISD::SRL, dl, MVT::i32, Flags,
-                      DAG.getConstant(8-(3-BitNo), MVT::i32));
+                      DAG.getConstant(8 - (3 - BitNo), dl, MVT::i32));
   // Isolate the bit.
   Flags = DAG.getNode(ISD::AND, dl, MVT::i32, Flags,
-                      DAG.getConstant(1, MVT::i32));
+                      DAG.getConstant(1, dl, MVT::i32));
 
   // If we are supposed to, toggle the bit.
   if (InvertBit)
     Flags = DAG.getNode(ISD::XOR, dl, MVT::i32, Flags,
-                        DAG.getConstant(1, MVT::i32));
+                        DAG.getConstant(1, dl, MVT::i32));
   return Flags;
 }
 
@@ -7272,7 +7499,7 @@ SDValue PPCTargetLowering::LowerSCALAR_TO_VECTOR(SDValue Op,
   // Create a stack slot that is 16-byte aligned.
   MachineFrameInfo *FrameInfo = DAG.getMachineFunction().getFrameInfo();
   int FrameIdx = FrameInfo->CreateStackObject(16, 16, false);
-  EVT PtrVT = getPointerTy();
+  EVT PtrVT = getPointerTy(DAG.getDataLayout());
   SDValue FIdx = DAG.getFrameIndex(FrameIdx, PtrVT);
 
   // Store the input value into Value#0 of the stack slot.
@@ -7304,7 +7531,7 @@ SDValue PPCTargetLowering::LowerEXTRACT_VECTOR_ELT(SDValue Op,
 
   // FIXME: We can make this an f32 vector, but the BUILD_VECTOR code needs to
   // understand how to form the extending load.
-  SDValue FPHalfs = DAG.getConstantFP(0.5, MVT::f64);
+  SDValue FPHalfs = DAG.getConstantFP(0.5, dl, MVT::f64);
   FPHalfs = DAG.getNode(ISD::BUILD_VECTOR, dl, MVT::v4f64,
                         FPHalfs, FPHalfs, FPHalfs, FPHalfs);
 
@@ -7312,19 +7539,19 @@ SDValue PPCTargetLowering::LowerEXTRACT_VECTOR_ELT(SDValue Op,
 
   // Now convert to an integer and store.
   Value = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, dl, MVT::v4f64,
-    DAG.getConstant(Intrinsic::ppc_qpx_qvfctiwu, MVT::i32),
+    DAG.getConstant(Intrinsic::ppc_qpx_qvfctiwu, dl, MVT::i32),
     Value);
 
   MachineFrameInfo *FrameInfo = DAG.getMachineFunction().getFrameInfo();
   int FrameIdx = FrameInfo->CreateStackObject(16, 16, false);
   MachinePointerInfo PtrInfo = MachinePointerInfo::getFixedStack(FrameIdx);
-  EVT PtrVT = getPointerTy();
+  EVT PtrVT = getPointerTy(DAG.getDataLayout());
   SDValue FIdx = DAG.getFrameIndex(FrameIdx, PtrVT);
 
   SDValue StoreChain = DAG.getEntryNode();
   SmallVector<SDValue, 2> Ops;
   Ops.push_back(StoreChain);
-  Ops.push_back(DAG.getConstant(Intrinsic::ppc_qpx_qvstfiw, MVT::i32));
+  Ops.push_back(DAG.getConstant(Intrinsic::ppc_qpx_qvstfiw, dl, MVT::i32));
   Ops.push_back(Value);
   Ops.push_back(FIdx);
 
@@ -7337,7 +7564,7 @@ SDValue PPCTargetLowering::LowerEXTRACT_VECTOR_ELT(SDValue Op,
 
   // Extract the value requested.
   unsigned Offset = 4*cast<ConstantSDNode>(N->getOperand(1))->getZExtValue();
-  SDValue Idx = DAG.getConstant(Offset, FIdx.getValueType());
+  SDValue Idx = DAG.getConstant(Offset, dl, FIdx.getValueType());
   Idx = DAG.getNode(ISD::ADD, dl, FIdx.getValueType(), FIdx, Idx);
 
   SDValue IntVal = DAG.getLoad(MVT::i32, dl, StoreChain, Idx,
@@ -7401,12 +7628,13 @@ SDValue PPCTargetLowering::LowerVectorLoad(SDValue Op,
       LoadChains.push_back(Load.getValue(1));
 
       BasePtr = DAG.getNode(ISD::ADD, dl, BasePtr.getValueType(), BasePtr,
-                            DAG.getConstant(Stride, BasePtr.getValueType()));
+                            DAG.getConstant(Stride, dl,
+                                            BasePtr.getValueType()));
     }
 
     SDValue TF =  DAG.getNode(ISD::TokenFactor, dl, MVT::Other, LoadChains);
     SDValue Value = DAG.getNode(ISD::BUILD_VECTOR, dl,
-                                   Op.getValueType(), Vals);
+                                Op.getValueType(), Vals);
 
     if (LN->isIndexed()) {
       SDValue RetOps[] = { Value, Vals[0].getValue(1), TF };
@@ -7425,7 +7653,7 @@ SDValue PPCTargetLowering::LowerVectorLoad(SDValue Op,
 
   SmallVector<SDValue, 4> VectElmts, VectElmtChains;
   for (unsigned i = 0; i < 4; ++i) {
-    SDValue Idx = DAG.getConstant(i, BasePtr.getValueType());
+    SDValue Idx = DAG.getConstant(i, dl, BasePtr.getValueType());
     Idx = DAG.getNode(ISD::ADD, dl, BasePtr.getValueType(), BasePtr, Idx);
 
     VectElmts.push_back(DAG.getExtLoad(ISD::EXTLOAD,
@@ -7469,9 +7697,9 @@ SDValue PPCTargetLowering::LowerVectorStore(SDValue Op,
 
     SmallVector<SDValue, 8> Stores;
     for (unsigned Idx = 0; Idx < 4; ++Idx) {
-      SDValue Ex =
-        DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, ScalarVT, Value,
-                    DAG.getConstant(Idx, getVectorIdxTy()));
+      SDValue Ex = DAG.getNode(
+          ISD::EXTRACT_VECTOR_ELT, dl, ScalarVT, Value,
+          DAG.getConstant(Idx, dl, getVectorIdxTy(DAG.getDataLayout())));
       SDValue Store;
       if (ScalarVT != ScalarMemVT)
         Store =
@@ -7494,7 +7722,8 @@ SDValue PPCTargetLowering::LowerVectorStore(SDValue Op,
       }
 
       BasePtr = DAG.getNode(ISD::ADD, dl, BasePtr.getValueType(), BasePtr,
-                            DAG.getConstant(Stride, BasePtr.getValueType()));
+                            DAG.getConstant(Stride, dl,
+                                            BasePtr.getValueType()));
       Stores.push_back(Store);
     }
 
@@ -7518,7 +7747,7 @@ SDValue PPCTargetLowering::LowerVectorStore(SDValue Op,
 
   // FIXME: We can make this an f32 vector, but the BUILD_VECTOR code needs to
   // understand how to form the extending load.
-  SDValue FPHalfs = DAG.getConstantFP(0.5, MVT::f64);
+  SDValue FPHalfs = DAG.getConstantFP(0.5, dl, MVT::f64);
   FPHalfs = DAG.getNode(ISD::BUILD_VECTOR, dl, MVT::v4f64,
                         FPHalfs, FPHalfs, FPHalfs, FPHalfs);
 
@@ -7526,18 +7755,18 @@ SDValue PPCTargetLowering::LowerVectorStore(SDValue Op,
 
   // Now convert to an integer and store.
   Value = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, dl, MVT::v4f64,
-    DAG.getConstant(Intrinsic::ppc_qpx_qvfctiwu, MVT::i32),
+    DAG.getConstant(Intrinsic::ppc_qpx_qvfctiwu, dl, MVT::i32),
     Value);
 
   MachineFrameInfo *FrameInfo = DAG.getMachineFunction().getFrameInfo();
   int FrameIdx = FrameInfo->CreateStackObject(16, 16, false);
   MachinePointerInfo PtrInfo = MachinePointerInfo::getFixedStack(FrameIdx);
-  EVT PtrVT = getPointerTy();
+  EVT PtrVT = getPointerTy(DAG.getDataLayout());
   SDValue FIdx = DAG.getFrameIndex(FrameIdx, PtrVT);
 
   SmallVector<SDValue, 2> Ops;
   Ops.push_back(StoreChain);
-  Ops.push_back(DAG.getConstant(Intrinsic::ppc_qpx_qvstfiw, MVT::i32));
+  Ops.push_back(DAG.getConstant(Intrinsic::ppc_qpx_qvstfiw, dl, MVT::i32));
   Ops.push_back(Value);
   Ops.push_back(FIdx);
 
@@ -7552,7 +7781,7 @@ SDValue PPCTargetLowering::LowerVectorStore(SDValue Op,
   SmallVector<SDValue, 4> Loads, LoadChains;
   for (unsigned i = 0; i < 4; ++i) {
     unsigned Offset = 4*i;
-    SDValue Idx = DAG.getConstant(Offset, FIdx.getValueType());
+    SDValue Idx = DAG.getConstant(Offset, dl, FIdx.getValueType());
     Idx = DAG.getNode(ISD::ADD, dl, FIdx.getValueType(), FIdx, Idx);
 
     Loads.push_back(DAG.getLoad(MVT::i32, dl, StoreChain, Idx,
@@ -7565,7 +7794,7 @@ SDValue PPCTargetLowering::LowerVectorStore(SDValue Op,
 
   SmallVector<SDValue, 4> Stores;
   for (unsigned i = 0; i < 4; ++i) {
-    SDValue Idx = DAG.getConstant(i, BasePtr.getValueType());
+    SDValue Idx = DAG.getConstant(i, dl, BasePtr.getValueType());
     Idx = DAG.getNode(ISD::ADD, dl, BasePtr.getValueType(), BasePtr, Idx);
 
     Stores.push_back(DAG.getTruncStore(StoreChain, dl, Loads[i], Idx,
@@ -7737,7 +7966,8 @@ void PPCTargetLowering::ReplaceNodeResults(SDNode *N,
 
     assert(N->getValueType(0) == MVT::i1 &&
            "Unexpected result type for CTR decrement intrinsic");
-    EVT SVT = getSetCCResultType(*DAG.getContext(), N->getValueType(0));
+    EVT SVT = getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(),
+                                 N->getValueType(0));
     SDVTList VTs = DAG.getVTList(SVT, MVT::Other);
     SDValue NewInt = DAG.getNode(N->getOpcode(), dl, VTs, N->getOperand(0),
                                  N->getOperand(1)); 
@@ -7765,10 +7995,10 @@ void PPCTargetLowering::ReplaceNodeResults(SDNode *N,
     assert(N->getOperand(0).getValueType() == MVT::ppcf128);
     SDValue Lo = DAG.getNode(ISD::EXTRACT_ELEMENT, dl,
                              MVT::f64, N->getOperand(0),
-                             DAG.getIntPtrConstant(0));
+                             DAG.getIntPtrConstant(0, dl));
     SDValue Hi = DAG.getNode(ISD::EXTRACT_ELEMENT, dl,
                              MVT::f64, N->getOperand(0),
-                             DAG.getIntPtrConstant(1));
+                             DAG.getIntPtrConstant(1, dl));
 
     // Add the two halves of the long double in round-to-zero mode.
     SDValue FPreg = DAG.getNode(PPCISD::FADDRTZ, dl, MVT::f64, Lo, Hi);
@@ -7797,7 +8027,7 @@ void PPCTargetLowering::ReplaceNodeResults(SDNode *N,
 static Instruction* callIntrinsic(IRBuilder<> &Builder, Intrinsic::ID Id) {
   Module *M = Builder.GetInsertBlock()->getParent()->getParent();
   Function *Func = Intrinsic::getDeclaration(M, Id);
-  return Builder.CreateCall(Func);
+  return Builder.CreateCall(Func, {});
 }
 
 // The mappings for emitLeading/TrailingFence is taken from
@@ -7807,10 +8037,9 @@ Instruction* PPCTargetLowering::emitLeadingFence(IRBuilder<> &Builder,
                                          bool IsLoad) const {
   if (Ord == SequentiallyConsistent)
     return callIntrinsic(Builder, Intrinsic::ppc_sync);
-  else if (isAtLeastRelease(Ord))
+  if (isAtLeastRelease(Ord))
     return callIntrinsic(Builder, Intrinsic::ppc_lwsync);
-  else
-    return nullptr;
+  return nullptr;
 }
 
 Instruction* PPCTargetLowering::emitTrailingFence(IRBuilder<> &Builder,
@@ -7822,8 +8051,7 @@ Instruction* PPCTargetLowering::emitTrailingFence(IRBuilder<> &Builder,
   // See http://www.cl.cam.ac.uk/~pes20/cpp/cpp0xmappings.html and
   // http://www.rdrop.com/users/paulmck/scalability/paper/N2745r.2011.03.04a.html
   // and http://www.cl.cam.ac.uk/~pes20/cppppc/ for justification.
-  else
-    return nullptr;
+  return nullptr;
 }
 
 MachineBasicBlock *
@@ -8067,7 +8295,7 @@ PPCTargetLowering::emitEHSjLjSetJmp(MachineInstr *MI,
   unsigned mainDstReg = MRI.createVirtualRegister(RC);
   unsigned restoreDstReg = MRI.createVirtualRegister(RC);
 
-  MVT PVT = getPointerTy();
+  MVT PVT = getPointerTy(MF->getDataLayout());
   assert((PVT == MVT::i64 || PVT == MVT::i32) &&
          "Invalid Pointer Size!");
   // For v = setjmp(buf), we generate
@@ -8205,7 +8433,7 @@ PPCTargetLowering::emitEHSjLjLongJmp(MachineInstr *MI,
   MachineInstr::mmo_iterator MMOBegin = MI->memoperands_begin();
   MachineInstr::mmo_iterator MMOEnd = MI->memoperands_end();
 
-  MVT PVT = getPointerTy();
+  MVT PVT = getPointerTy(MF->getDataLayout());
   assert((PVT == MVT::i64 || PVT == MVT::i32) &&
          "Invalid Pointer Size!");
 
@@ -8363,6 +8591,7 @@ PPCTargetLowering::EmitInstrWithCustomInserter(MachineInstr *MI,
              MI->getOpcode() == PPC::SELECT_CC_QBRC ||
              MI->getOpcode() == PPC::SELECT_CC_VRRC ||
              MI->getOpcode() == PPC::SELECT_CC_VSFRC ||
+             MI->getOpcode() == PPC::SELECT_CC_VSSRC ||
              MI->getOpcode() == PPC::SELECT_CC_VSRC ||
              MI->getOpcode() == PPC::SELECT_I4 ||
              MI->getOpcode() == PPC::SELECT_I8 ||
@@ -8373,6 +8602,7 @@ PPCTargetLowering::EmitInstrWithCustomInserter(MachineInstr *MI,
              MI->getOpcode() == PPC::SELECT_QBRC ||
              MI->getOpcode() == PPC::SELECT_VRRC ||
              MI->getOpcode() == PPC::SELECT_VSFRC ||
+             MI->getOpcode() == PPC::SELECT_VSSRC ||
              MI->getOpcode() == PPC::SELECT_VSRC) {
     // The incoming instruction knows the destination vreg to set, the
     // condition code register to branch on, the true/false values to
@@ -8409,6 +8639,7 @@ PPCTargetLowering::EmitInstrWithCustomInserter(MachineInstr *MI,
         MI->getOpcode() == PPC::SELECT_QBRC ||
         MI->getOpcode() == PPC::SELECT_VRRC ||
         MI->getOpcode() == PPC::SELECT_VSFRC ||
+        MI->getOpcode() == PPC::SELECT_VSSRC ||
         MI->getOpcode() == PPC::SELECT_VSRC) {
       BuildMI(BB, dl, TII->get(PPC::BC))
         .addReg(MI->getOperand(1).getReg()).addMBB(sinkMBB);
@@ -8848,6 +9079,19 @@ PPCTargetLowering::EmitInstrWithCustomInserter(MachineInstr *MI,
 // Target Optimization Hooks
 //===----------------------------------------------------------------------===//
 
+static std::string getRecipOp(const char *Base, EVT VT) {
+  std::string RecipOp(Base);
+  if (VT.getScalarType() == MVT::f64)
+    RecipOp += "d";
+  else
+    RecipOp += "f";
+
+  if (VT.isVector())
+    RecipOp = "vec-" + RecipOp;
+
+  return RecipOp;
+}
+
 SDValue PPCTargetLowering::getRsqrtEstimate(SDValue Operand,
                                             DAGCombinerInfo &DCI,
                                             unsigned &RefinementSteps,
@@ -8859,13 +9103,12 @@ SDValue PPCTargetLowering::getRsqrtEstimate(SDValue Operand,
       (VT == MVT::v2f64 && Subtarget.hasVSX()) ||
       (VT == MVT::v4f32 && Subtarget.hasQPX()) ||
       (VT == MVT::v4f64 && Subtarget.hasQPX())) {
-    // Convergence is quadratic, so we essentially double the number of digits
-    // correct after every iteration. For both FRE and FRSQRTE, the minimum
-    // architected relative accuracy is 2^-5. When hasRecipPrec(), this is
-    // 2^-14. IEEE float has 23 digits and double has 52 digits.
-    RefinementSteps = Subtarget.hasRecipPrec() ? 1 : 3;
-    if (VT.getScalarType() == MVT::f64)
-      ++RefinementSteps;
+    TargetRecip Recips = DCI.DAG.getTarget().Options.Reciprocals;
+    std::string RecipOp = getRecipOp("sqrt", VT);
+    if (!Recips.isEnabled(RecipOp))
+      return SDValue();
+
+    RefinementSteps = Recips.getRefinementSteps(RecipOp);
     UseOneConstNR = true;
     return DCI.DAG.getNode(PPCISD::FRSQRTE, SDLoc(Operand), VT, Operand);
   }
@@ -8882,19 +9125,18 @@ SDValue PPCTargetLowering::getRecipEstimate(SDValue Operand,
       (VT == MVT::v2f64 && Subtarget.hasVSX()) ||
       (VT == MVT::v4f32 && Subtarget.hasQPX()) ||
       (VT == MVT::v4f64 && Subtarget.hasQPX())) {
-    // Convergence is quadratic, so we essentially double the number of digits
-    // correct after every iteration. For both FRE and FRSQRTE, the minimum
-    // architected relative accuracy is 2^-5. When hasRecipPrec(), this is
-    // 2^-14. IEEE float has 23 digits and double has 52 digits.
-    RefinementSteps = Subtarget.hasRecipPrec() ? 1 : 3;
-    if (VT.getScalarType() == MVT::f64)
-      ++RefinementSteps;
+    TargetRecip Recips = DCI.DAG.getTarget().Options.Reciprocals;
+    std::string RecipOp = getRecipOp("div", VT);
+    if (!Recips.isEnabled(RecipOp))
+      return SDValue();
+
+    RefinementSteps = Recips.getRefinementSteps(RecipOp);
     return DCI.DAG.getNode(PPCISD::FRE, SDLoc(Operand), VT, Operand);
   }
   return SDValue();
 }
 
-bool PPCTargetLowering::combineRepeatedFPDivisors(unsigned NumUsers) const {
+unsigned PPCTargetLowering::combineRepeatedFPDivisors() const {
   // Note: This functionality is used only when unsafe-fp-math is enabled, and
   // on cores with reciprocal estimates (which are used when unsafe-fp-math is
   // enabled for division), this functionality is redundant with the default
@@ -8907,12 +9149,12 @@ bool PPCTargetLowering::combineRepeatedFPDivisors(unsigned NumUsers) const {
   // one FP pipeline) for three or more FDIVs (for generic OOO cores).
   switch (Subtarget.getDarwinDirective()) {
   default:
-    return NumUsers > 2;
+    return 3;
   case PPC::DIR_440:
   case PPC::DIR_A2:
   case PPC::DIR_E500mc:
   case PPC::DIR_E5500:
-    return NumUsers > 1;
+    return 2;
   }
 }
 
@@ -9666,13 +9908,13 @@ SDValue PPCTargetLowering::DAGCombineExtBoolTrunc(SDNode *N,
     return DAG.getNode(ISD::AND, dl, N->getValueType(0), N->getOperand(0),
                        DAG.getConstant(APInt::getLowBitsSet(
                                          N->getValueSizeInBits(0), PromBits),
-                                       N->getValueType(0)));
+                                       dl, N->getValueType(0)));
 
   assert(N->getOpcode() == ISD::SIGN_EXTEND &&
          "Invalid extension type");
-  EVT ShiftAmountTy = getShiftAmountTy(N->getValueType(0));
+  EVT ShiftAmountTy = getShiftAmountTy(N->getValueType(0), DAG.getDataLayout());
   SDValue ShiftCst =
-    DAG.getConstant(N->getValueSizeInBits(0)-PromBits, ShiftAmountTy);
+    DAG.getConstant(N->getValueSizeInBits(0) - PromBits, dl, ShiftAmountTy);
   return DAG.getNode(ISD::SRA, dl, N->getValueType(0), 
                      DAG.getNode(ISD::SHL, dl, N->getValueType(0),
                                  N->getOperand(0), ShiftCst), ShiftCst);
@@ -9738,7 +9980,7 @@ SDValue PPCTargetLowering::combineFPToIntToFP(SDNode *N,
 
     if (Op.getValueType() == MVT::f32 && !Subtarget.hasFPCVT()) {
       FP = DAG.getNode(ISD::FP_ROUND, dl,
-                       MVT::f32, FP, DAG.getIntPtrConstant(0));
+                       MVT::f32, FP, DAG.getIntPtrConstant(0, dl));
       DCI.AddToWorklist(FP.getNode());
     }
 
@@ -9776,7 +10018,9 @@ SDValue PPCTargetLowering::expandVSXLoadForLE(SDNode *N,
   case ISD::INTRINSIC_W_CHAIN: {
     MemIntrinsicSDNode *Intrin = cast<MemIntrinsicSDNode>(N);
     Chain = Intrin->getChain();
-    Base = Intrin->getBasePtr();
+    // Similarly to the store case below, Intrin->getBasePtr() doesn't get
+    // us what we want. Get operand 2 instead.
+    Base = Intrin->getOperand(2);
     MMO = Intrin->getMemOperand();
     break;
   }
@@ -9959,9 +10203,9 @@ SDValue PPCTargetLowering::PerformDAGCombine(SDNode *N,
 
     EVT MemVT = LD->getMemoryVT();
     Type *Ty = MemVT.getTypeForEVT(*DAG.getContext());
-    unsigned ABIAlignment = getDataLayout()->getABITypeAlignment(Ty);
+    unsigned ABIAlignment = DAG.getDataLayout().getABITypeAlignment(Ty);
     Type *STy = MemVT.getScalarType().getTypeForEVT(*DAG.getContext());
-    unsigned ScalarABIAlignment = getDataLayout()->getABITypeAlignment(STy);
+    unsigned ScalarABIAlignment = DAG.getDataLayout().getABITypeAlignment(STy);
     if (LD->isUnindexed() && VT.isVector() &&
         ((Subtarget.hasAltivec() && ISD::isNON_EXTLoad(N) &&
           // P8 and later hardware should just use LOAD.
@@ -10033,7 +10277,8 @@ SDValue PPCTargetLowering::PerformDAGCombine(SDNode *N,
                                 2*MemVT.getStoreSize()-1);
 
       // Create the new base load.
-      SDValue LDXIntID = DAG.getTargetConstant(IntrLD, getPointerTy());
+      SDValue LDXIntID =
+          DAG.getTargetConstant(IntrLD, dl, getPointerTy(MF.getDataLayout()));
       SDValue BaseLoadOps[] = { Chain, LDXIntID, Ptr };
       SDValue BaseLoad =
         DAG.getMemIntrinsicNode(ISD::INTRINSIC_W_CHAIN, dl,
@@ -10057,7 +10302,8 @@ SDValue PPCTargetLowering::PerformDAGCombine(SDNode *N,
       if (!findConsecutiveLoad(LD, DAG))
         --IncValue;
 
-      SDValue Increment = DAG.getConstant(IncValue, getPointerTy());
+      SDValue Increment =
+          DAG.getConstant(IncValue, dl, getPointerTy(MF.getDataLayout()));
       Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr, Increment);
 
       MachineMemOperand *ExtraMMO =
@@ -10089,7 +10335,7 @@ SDValue PPCTargetLowering::PerformDAGCombine(SDNode *N,
         Perm = Subtarget.hasAltivec() ?
                  DAG.getNode(ISD::BITCAST, dl, VT, Perm) :
                  DAG.getNode(ISD::FP_ROUND, dl, VT, Perm, // QPX
-                               DAG.getTargetConstant(1, MVT::i64));
+                               DAG.getTargetConstant(1, dl, MVT::i64));
                                // second argument is 1 because this rounding
                                // is always exact.
 
@@ -10358,7 +10604,7 @@ SDValue PPCTargetLowering::PerformDAGCombine(SDNode *N,
       SDValue Ops[] = {
         LHS.getOperand(2),  // LHS of compare
         LHS.getOperand(3),  // RHS of compare
-        DAG.getConstant(CompareOpc, MVT::i32)
+        DAG.getConstant(CompareOpc, dl, MVT::i32)
       };
       EVT VTs[] = { LHS.getOperand(2).getValueType(), MVT::Glue };
       SDValue CompNode = DAG.getNode(PPCISD::VCMPo, dl, VTs, Ops);
@@ -10382,7 +10628,7 @@ SDValue PPCTargetLowering::PerformDAGCombine(SDNode *N,
       }
 
       return DAG.getNode(PPCISD::COND_BRANCH, dl, MVT::Other, N->getOperand(0),
-                         DAG.getConstant(CompOpc, MVT::i32),
+                         DAG.getConstant(CompOpc, dl, MVT::i32),
                          DAG.getRegister(PPC::CR6, MVT::i32),
                          N->getOperand(4), CompNode.getValue(1));
     }
@@ -10410,14 +10656,14 @@ PPCTargetLowering::BuildSDIVPow2(SDNode *N, const APInt &Divisor,
 
   bool IsNegPow2 = (-Divisor).isPowerOf2();
   unsigned Lg2 = (IsNegPow2 ? -Divisor : Divisor).countTrailingZeros();
-  SDValue ShiftAmt = DAG.getConstant(Lg2, VT);
+  SDValue ShiftAmt = DAG.getConstant(Lg2, DL, VT);
 
   SDValue Op = DAG.getNode(PPCISD::SRA_ADDZE, DL, VT, N0, ShiftAmt);
   if (Created)
     Created->push_back(Op.getNode());
 
   if (IsNegPow2) {
-    Op = DAG.getNode(ISD::SUB, DL, VT, DAG.getConstant(0, VT), Op);
+    Op = DAG.getNode(ISD::SUB, DL, VT, DAG.getConstant(0, DL, VT), Op);
     if (Created)
       Created->push_back(Op.getNode());
   }
@@ -10505,7 +10751,7 @@ unsigned PPCTargetLowering::getPrefLoopAlignment(MachineLoop *ML) const {
 /// getConstraintType - Given a constraint, return the type of
 /// constraint it is for this target.
 PPCTargetLowering::ConstraintType
-PPCTargetLowering::getConstraintType(const std::string &Constraint) const {
+PPCTargetLowering::getConstraintType(StringRef Constraint) const {
   if (Constraint.size() == 1) {
     switch (Constraint[0]) {
     default: break;
@@ -10590,7 +10836,7 @@ PPCTargetLowering::getSingleConstraintMatchWeight(
 
 std::pair<unsigned, const TargetRegisterClass *>
 PPCTargetLowering::getRegForInlineAsmConstraint(const TargetRegisterInfo *TRI,
-                                                const std::string &Constraint,
+                                                StringRef Constraint,
                                                 MVT VT) const {
   if (Constraint.size() == 1) {
     // GCC RS6000 Constraint Letters
@@ -10628,7 +10874,10 @@ PPCTargetLowering::getRegForInlineAsmConstraint(const TargetRegisterInfo *TRI,
              Constraint == "wf") {
     return std::make_pair(0U, &PPC::VSRCRegClass);
   } else if (Constraint == "ws") {
-    return std::make_pair(0U, &PPC::VSFRCRegClass);
+    if (VT == MVT::f32)
+      return std::make_pair(0U, &PPC::VSSRCRegClass);
+    else
+      return std::make_pair(0U, &PPC::VSFRCRegClass);
   }
 
   std::pair<unsigned, const TargetRegisterClass *> R =
@@ -10680,6 +10929,7 @@ void PPCTargetLowering::LowerAsmOperandForConstraint(SDValue Op,
   case 'P': {
     ConstantSDNode *CST = dyn_cast<ConstantSDNode>(Op);
     if (!CST) return; // Must be an immediate to match.
+    SDLoc dl(Op);
     int64_t Value = CST->getSExtValue();
     EVT TCVT = MVT::i64; // All constants taken to be 64 bits so that negative
                          // numbers are printed as such.
@@ -10687,35 +10937,35 @@ void PPCTargetLowering::LowerAsmOperandForConstraint(SDValue Op,
     default: llvm_unreachable("Unknown constraint letter!");
     case 'I':  // "I" is a signed 16-bit constant.
       if (isInt<16>(Value))
-        Result = DAG.getTargetConstant(Value, TCVT);
+        Result = DAG.getTargetConstant(Value, dl, TCVT);
       break;
     case 'J':  // "J" is a constant with only the high-order 16 bits nonzero.
       if (isShiftedUInt<16, 16>(Value))
-        Result = DAG.getTargetConstant(Value, TCVT);
+        Result = DAG.getTargetConstant(Value, dl, TCVT);
       break;
     case 'L':  // "L" is a signed 16-bit constant shifted left 16 bits.
       if (isShiftedInt<16, 16>(Value))
-        Result = DAG.getTargetConstant(Value, TCVT);
+        Result = DAG.getTargetConstant(Value, dl, TCVT);
       break;
     case 'K':  // "K" is a constant with only the low-order 16 bits nonzero.
       if (isUInt<16>(Value))
-        Result = DAG.getTargetConstant(Value, TCVT);
+        Result = DAG.getTargetConstant(Value, dl, TCVT);
       break;
     case 'M':  // "M" is a constant that is greater than 31.
       if (Value > 31)
-        Result = DAG.getTargetConstant(Value, TCVT);
+        Result = DAG.getTargetConstant(Value, dl, TCVT);
       break;
     case 'N':  // "N" is a positive constant that is an exact power of two.
       if (Value > 0 && isPowerOf2_64(Value))
-        Result = DAG.getTargetConstant(Value, TCVT);
+        Result = DAG.getTargetConstant(Value, dl, TCVT);
       break;
     case 'O':  // "O" is the constant zero.
       if (Value == 0)
-        Result = DAG.getTargetConstant(Value, TCVT);
+        Result = DAG.getTargetConstant(Value, dl, TCVT);
       break;
     case 'P':  // "P" is a constant whose negation is a signed 16-bit constant.
       if (isInt<16>(-Value))
-        Result = DAG.getTargetConstant(Value, TCVT);
+        Result = DAG.getTargetConstant(Value, dl, TCVT);
       break;
     }
     break;
@@ -10733,8 +10983,9 @@ void PPCTargetLowering::LowerAsmOperandForConstraint(SDValue Op,
 
 // isLegalAddressingMode - Return true if the addressing mode represented
 // by AM is legal for this target, for a load/store of the specified type.
-bool PPCTargetLowering::isLegalAddressingMode(const AddrMode &AM,
-                                              Type *Ty) const {
+bool PPCTargetLowering::isLegalAddressingMode(const DataLayout &DL,
+                                              const AddrMode &AM, Type *Ty,
+                                              unsigned AS) const {
   // PPC does not allow r+i addressing modes for vectors!
   if (Ty->isVectorTy() && AM.BaseOffs != 0)
     return false;
@@ -10786,22 +11037,22 @@ SDValue PPCTargetLowering::LowerRETURNADDR(SDValue Op,
   PPCFunctionInfo *FuncInfo = MF.getInfo<PPCFunctionInfo>();
   FuncInfo->setLRStoreRequired();
   bool isPPC64 = Subtarget.isPPC64();
+  auto PtrVT = getPointerTy(MF.getDataLayout());
 
   if (Depth > 0) {
     SDValue FrameAddr = LowerFRAMEADDR(Op, DAG);
     SDValue Offset =
-        DAG.getConstant(Subtarget.getFrameLowering()->getReturnSaveOffset(),
+        DAG.getConstant(Subtarget.getFrameLowering()->getReturnSaveOffset(), dl,
                         isPPC64 ? MVT::i64 : MVT::i32);
-    return DAG.getLoad(getPointerTy(), dl, DAG.getEntryNode(),
-                       DAG.getNode(ISD::ADD, dl, getPointerTy(),
-                                   FrameAddr, Offset),
+    return DAG.getLoad(PtrVT, dl, DAG.getEntryNode(),
+                       DAG.getNode(ISD::ADD, dl, PtrVT, FrameAddr, Offset),
                        MachinePointerInfo(), false, false, false, 0);
   }
 
   // Just load the return address off the stack.
   SDValue RetAddrFI = getReturnAddrFrameIndex(DAG);
-  return DAG.getLoad(getPointerTy(), dl, DAG.getEntryNode(),
-                     RetAddrFI, MachinePointerInfo(), false, false, false, 0);
+  return DAG.getLoad(PtrVT, dl, DAG.getEntryNode(), RetAddrFI,
+                     MachinePointerInfo(), false, false, false, 0);
 }
 
 SDValue PPCTargetLowering::LowerFRAMEADDR(SDValue Op,
@@ -10809,13 +11060,13 @@ SDValue PPCTargetLowering::LowerFRAMEADDR(SDValue Op,
   SDLoc dl(Op);
   unsigned Depth = cast<ConstantSDNode>(Op.getOperand(0))->getZExtValue();
 
-  EVT PtrVT = DAG.getTargetLoweringInfo().getPointerTy();
-  bool isPPC64 = PtrVT == MVT::i64;
-
   MachineFunction &MF = DAG.getMachineFunction();
   MachineFrameInfo *MFI = MF.getFrameInfo();
   MFI->setFrameAddressIsTaken(true);
 
+  EVT PtrVT = DAG.getTargetLoweringInfo().getPointerTy(MF.getDataLayout());
+  bool isPPC64 = PtrVT == MVT::i64;
+
   // Naked functions never have a frame pointer, and so we use r1. For all
   // other functions, this decision must be delayed until during PEI.
   unsigned FrameReg;
@@ -10835,8 +11086,8 @@ SDValue PPCTargetLowering::LowerFRAMEADDR(SDValue Op,
 
 // FIXME? Maybe this could be a TableGen attribute on some registers and
 // this table could be generated automatically from RegInfo.
-unsigned PPCTargetLowering::getRegisterByName(const char* RegName,
-                                              EVT VT) const {
+unsigned PPCTargetLowering::getRegisterByName(const char* RegName, EVT VT,
+                                              SelectionDAG &DAG) const {
   bool isPPC64 = Subtarget.isPPC64();
   bool isDarwinABI = Subtarget.isDarwinABI();
 
diff --git a/lib/Target/PowerPC/PPCISelLowering.h b/lib/Target/PowerPC/PPCISelLowering.h
index 7e2ebd40d42..cbd5add8488 100644
--- a/lib/Target/PowerPC/PPCISelLowering.h
+++ b/lib/Target/PowerPC/PPCISelLowering.h
@@ -24,7 +24,7 @@
 
 namespace llvm {
   namespace PPCISD {
-    enum NodeType {
+    enum NodeType : unsigned {
       // Start the numbering where the builtin ops and target ops leave off.
       FIRST_NUMBER = ISD::BUILTIN_OP_END,
 
@@ -275,6 +275,16 @@ namespace llvm {
       /// operand identifies the operating system entry point.
       SC,
 
+      /// CHAIN = CLRBHRB CHAIN - Clear branch history rolling buffer.
+      CLRBHRB,
+
+      /// GPRC, CHAIN = MFBHRBE CHAIN, Entry, Dummy - Move from branch
+      /// history rolling buffer entry.
+      MFBHRBE,
+
+      /// CHAIN = RFEBB CHAIN, State - Return from event-based branch.
+      RFEBB,
+
       /// VSRC, CHAIN = XXSWAPD CHAIN, VSRC - Occurs only for little
       /// endian.  Maps to an xxswapd instruction that corrects an lxvd2x
       /// or stxvd2x instruction.  The chain is necessary because the
@@ -357,6 +367,11 @@ namespace llvm {
     bool isVPKUWUMShuffleMask(ShuffleVectorSDNode *N, unsigned ShuffleKind,
                               SelectionDAG &DAG);
 
+    /// isVPKUDUMShuffleMask - Return true if this is the shuffle mask for a
+    /// VPKUDUM instruction.
+    bool isVPKUDUMShuffleMask(ShuffleVectorSDNode *N, unsigned ShuffleKind,
+                              SelectionDAG &DAG);
+
     /// isVMRGLShuffleMask - Return true if this is a shuffle mask suitable for
     /// a VRGL* instruction with the specified unit size (1,2 or 4 bytes).
     bool isVMRGLShuffleMask(ShuffleVectorSDNode *N, unsigned UnitSize,
@@ -367,6 +382,11 @@ namespace llvm {
     bool isVMRGHShuffleMask(ShuffleVectorSDNode *N, unsigned UnitSize,
                             unsigned ShuffleKind, SelectionDAG &DAG);
 
+    /// isVMRGEOShuffleMask - Return true if this is a shuffle mask suitable for
+    /// a VMRGEW or VMRGOW instruction
+    bool isVMRGEOShuffleMask(ShuffleVectorSDNode *N, bool CheckEven,
+                             unsigned ShuffleKind, SelectionDAG &DAG);
+  
     /// isVSLDOIShuffleMask - If this is a vsldoi shuffle mask, return the
     /// shift amount, otherwise return -1.
     int isVSLDOIShuffleMask(SDNode *N, unsigned ShuffleKind,
@@ -403,7 +423,9 @@ namespace llvm {
     /// DAG node.
     const char *getTargetNodeName(unsigned Opcode) const override;
 
-    MVT getScalarShiftAmountTy(EVT LHSTy) const override { return MVT::i32; }
+    MVT getScalarShiftAmountTy(const DataLayout &, EVT) const override {
+      return MVT::i32;
+    }
 
     bool isCheapToSpeculateCttz() const override {
       return true;
@@ -414,7 +436,8 @@ namespace llvm {
     }
 
     /// getSetCCResultType - Return the ISD::SETCC ValueType
-    EVT getSetCCResultType(LLVMContext &Context, EVT VT) const override;
+    EVT getSetCCResultType(const DataLayout &DL, LLVMContext &Context,
+                           EVT VT) const override;
 
     /// Return true if target always beneficiates from combining into FMA for a
     /// given value type. This must typically return false on targets where FMA
@@ -467,7 +490,8 @@ namespace llvm {
     SDValue BuildSDIVPow2(SDNode *N, const APInt &Divisor, SelectionDAG &DAG,
                           std::vector<SDNode *> *Created) const override;
 
-    unsigned getRegisterByName(const char* RegName, EVT VT) const override;
+    unsigned getRegisterByName(const char* RegName, EVT VT,
+                               SelectionDAG &DAG) const override;
 
     void computeKnownBitsForTargetNode(const SDValue Op,
                                        APInt &KnownZero,
@@ -499,8 +523,7 @@ namespace llvm {
     MachineBasicBlock *emitEHSjLjLongJmp(MachineInstr *MI,
                                          MachineBasicBlock *MBB) const;
 
-    ConstraintType
-    getConstraintType(const std::string &Constraint) const override;
+    ConstraintType getConstraintType(StringRef Constraint) const override;
 
     /// Examine constraint string and operand type and determine a weight value.
     /// The operand object must already have been set up with the operand type.
@@ -509,13 +532,13 @@ namespace llvm {
 
     std::pair<unsigned, const TargetRegisterClass *>
     getRegForInlineAsmConstraint(const TargetRegisterInfo *TRI,
-                                 const std::string &Constraint,
-                                 MVT VT) const override;
+                                 StringRef Constraint, MVT VT) const override;
 
     /// getByValTypeAlignment - Return the desired alignment for ByVal aggregate
     /// function arguments in the caller parameter area.  This is the actual
     /// alignment, not its logarithm.
-    unsigned getByValTypeAlignment(Type *Ty) const override;
+    unsigned getByValTypeAlignment(Type *Ty,
+                                   const DataLayout &DL) const override;
 
     /// LowerAsmOperandForConstraint - Lower the specified operand into the Ops
     /// vector.  If it is invalid, don't add anything to Ops.
@@ -524,8 +547,8 @@ namespace llvm {
                                       std::vector<SDValue> &Ops,
                                       SelectionDAG &DAG) const override;
 
-    unsigned getInlineAsmMemConstraint(
-        const std::string &ConstraintCode) const override {
+    unsigned
+    getInlineAsmMemConstraint(StringRef ConstraintCode) const override {
       if (ConstraintCode == "es")
         return InlineAsm::Constraint_es;
       else if (ConstraintCode == "o")
@@ -541,7 +564,8 @@ namespace llvm {
 
     /// isLegalAddressingMode - Return true if the addressing mode represented
     /// by AM is legal for this target, for a load/store of the specified type.
-    bool isLegalAddressingMode(const AddrMode &AM, Type *Ty) const override;
+    bool isLegalAddressingMode(const DataLayout &DL, const AddrMode &AM,
+                               Type *Ty, unsigned AS) const override;
 
     /// isLegalICmpImmediate - Return true if the specified immediate is legal
     /// icmp immediate, that is the target has icmp instructions which can
@@ -724,7 +748,7 @@ namespace llvm {
                             SDLoc dl, SelectionDAG &DAG,
                             SmallVectorImpl<SDValue> &InVals) const;
     SDValue FinishCall(CallingConv::ID CallConv, SDLoc dl, bool isTailCall,
-                       bool isVarArg, bool IsPatchPoint,
+                       bool isVarArg, bool IsPatchPoint, bool hasNest,
                        SelectionDAG &DAG,
                        SmallVector<std::pair<unsigned, SDValue>, 8>
                          &RegsToPass,
@@ -829,7 +853,7 @@ namespace llvm {
                              bool &UseOneConstNR) const override;
     SDValue getRecipEstimate(SDValue Operand, DAGCombinerInfo &DCI,
                              unsigned &RefinementSteps) const override;
-    bool combineRepeatedFPDivisors(unsigned NumUsers) const override;
+    unsigned combineRepeatedFPDivisors() const override;
 
     CCAssignFn *useFastISelCCs(unsigned Flag) const;
   };
diff --git a/lib/Target/PowerPC/PPCInstr64Bit.td b/lib/Target/PowerPC/PPCInstr64Bit.td
index d1d67cbba44..d62833037db 100644
--- a/lib/Target/PowerPC/PPCInstr64Bit.td
+++ b/lib/Target/PowerPC/PPCInstr64Bit.td
@@ -56,22 +56,23 @@ def tlscall : Operand<i64> {
 
 def SHL64 : SDNodeXForm<imm, [{
   // Transformation function: 63 - imm
-  return getI32Imm(63 - N->getZExtValue());
+  return getI32Imm(63 - N->getZExtValue(), SDLoc(N));
 }]>;
 
 def SRL64 : SDNodeXForm<imm, [{
   // Transformation function: 64 - imm
-  return N->getZExtValue() ? getI32Imm(64 - N->getZExtValue()) : getI32Imm(0);
+  return N->getZExtValue() ? getI32Imm(64 - N->getZExtValue(), SDLoc(N))
+                           : getI32Imm(0, SDLoc(N));
 }]>;
 
 def HI32_48 : SDNodeXForm<imm, [{
   // Transformation function: shift the immediate value down into the low bits.
-  return getI32Imm((unsigned short)(N->getZExtValue() >> 32));
+  return getI32Imm((unsigned short)(N->getZExtValue() >> 32, SDLoc(N)));
 }]>;
 
 def HI48_64 : SDNodeXForm<imm, [{
   // Transformation function: shift the immediate value down into the low bits.
-  return getI32Imm((unsigned short)(N->getZExtValue() >> 48));
+  return getI32Imm((unsigned short)(N->getZExtValue() >> 48, SDLoc(N)));
 }]>;
 
 
diff --git a/lib/Target/PowerPC/PPCInstrAltivec.td b/lib/Target/PowerPC/PPCInstrAltivec.td
index 123808bed3e..cb0271fe8d0 100644
--- a/lib/Target/PowerPC/PPCInstrAltivec.td
+++ b/lib/Target/PowerPC/PPCInstrAltivec.td
@@ -11,6 +11,21 @@
 //
 //===----------------------------------------------------------------------===//
 
+// *********************************** NOTE ***********************************
+// ** For POWER8 Little Endian, the VSX swap optimization relies on knowing  **
+// ** which VMX and VSX instructions are lane-sensitive and which are not.   **
+// ** A lane-sensitive instruction relies, implicitly or explicitly, on      **
+// ** whether lanes are numbered from left to right.  An instruction like    **
+// ** VADDFP is not lane-sensitive, because each lane of the result vector   **
+// ** relies only on the corresponding lane of the source vectors.  However, **
+// ** an instruction like VMULESB is lane-sensitive, because "even" and      **
+// ** "odd" lanes are different for big-endian and little-endian numbering.  **
+// **                                                                        **
+// ** When adding new VMX and VSX instructions, please consider whether they **
+// ** are lane-sensitive.  If so, they must be added to a switch statement   **
+// ** in PPCVSXSwapRemoval::gatherVectorInstructions().                      **
+// ****************************************************************************
+
 //===----------------------------------------------------------------------===//
 // Altivec transformation functions and pattern fragments.
 //
@@ -28,6 +43,10 @@ def vpkuwum_shuffle : PatFrag<(ops node:$lhs, node:$rhs),
                               (vector_shuffle node:$lhs, node:$rhs), [{
   return PPC::isVPKUWUMShuffleMask(cast<ShuffleVectorSDNode>(N), 0, *CurDAG);
 }]>;
+def vpkudum_shuffle : PatFrag<(ops node:$lhs, node:$rhs),
+                              (vector_shuffle node:$lhs, node:$rhs), [{
+  return PPC::isVPKUDUMShuffleMask(cast<ShuffleVectorSDNode>(N), 0, *CurDAG);
+}]>;
 def vpkuhum_unary_shuffle : PatFrag<(ops node:$lhs, node:$rhs),
                                     (vector_shuffle node:$lhs, node:$rhs), [{
   return PPC::isVPKUHUMShuffleMask(cast<ShuffleVectorSDNode>(N), 1, *CurDAG);
@@ -36,6 +55,10 @@ def vpkuwum_unary_shuffle : PatFrag<(ops node:$lhs, node:$rhs),
                                     (vector_shuffle node:$lhs, node:$rhs), [{
   return PPC::isVPKUWUMShuffleMask(cast<ShuffleVectorSDNode>(N), 1, *CurDAG);
 }]>;
+def vpkudum_unary_shuffle : PatFrag<(ops node:$lhs, node:$rhs),
+                                    (vector_shuffle node:$lhs, node:$rhs), [{
+  return PPC::isVPKUDUMShuffleMask(cast<ShuffleVectorSDNode>(N), 1, *CurDAG);
+}]>;
 
 // These fragments are provided for little-endian, where the inputs must be
 // swapped for correct semantics.
@@ -47,6 +70,10 @@ def vpkuwum_swapped_shuffle : PatFrag<(ops node:$lhs, node:$rhs),
                                       (vector_shuffle node:$lhs, node:$rhs), [{
   return PPC::isVPKUWUMShuffleMask(cast<ShuffleVectorSDNode>(N), 2, *CurDAG);
 }]>;
+def vpkudum_swapped_shuffle : PatFrag<(ops node:$lhs, node:$rhs),
+                                      (vector_shuffle node:$lhs, node:$rhs), [{
+  return PPC::isVPKUDUMShuffleMask(cast<ShuffleVectorSDNode>(N), 2, *CurDAG);
+}]>;
 
 def vmrglb_shuffle : PatFrag<(ops node:$lhs, node:$rhs),
                              (vector_shuffle (v16i8 node:$lhs), node:$rhs), [{
@@ -128,8 +155,35 @@ def vmrghw_swapped_shuffle : PatFrag<(ops node:$lhs, node:$rhs),
 }]>;
 
 
+def vmrgew_shuffle : PatFrag<(ops node:$lhs, node:$rhs),
+                             (vector_shuffle node:$lhs, node:$rhs), [{
+  return PPC::isVMRGEOShuffleMask(cast<ShuffleVectorSDNode>(N), true, 0, *CurDAG);
+}]>;
+def vmrgow_shuffle : PatFrag<(ops node:$lhs, node:$rhs),
+                             (vector_shuffle node:$lhs, node:$rhs), [{
+  return PPC::isVMRGEOShuffleMask(cast<ShuffleVectorSDNode>(N), false, 0, *CurDAG);
+}]>;
+def vmrgew_unary_shuffle : PatFrag<(ops node:$lhs, node:$rhs),
+                                   (vector_shuffle node:$lhs, node:$rhs), [{
+  return PPC::isVMRGEOShuffleMask(cast<ShuffleVectorSDNode>(N), true, 1, *CurDAG);
+}]>;
+def vmrgow_unary_shuffle : PatFrag<(ops node:$lhs, node:$rhs),
+                                   (vector_shuffle node:$lhs, node:$rhs), [{
+  return PPC::isVMRGEOShuffleMask(cast<ShuffleVectorSDNode>(N), false, 1, *CurDAG);
+}]>;
+def vmrgew_swapped_shuffle : PatFrag<(ops node:$lhs, node:$rhs),
+                                     (vector_shuffle node:$lhs, node:$rhs), [{
+  return PPC::isVMRGEOShuffleMask(cast<ShuffleVectorSDNode>(N), true, 2, *CurDAG);
+}]>;
+def vmrgow_swapped_shuffle : PatFrag<(ops node:$lhs, node:$rhs),
+                                     (vector_shuffle node:$lhs, node:$rhs), [{
+  return PPC::isVMRGEOShuffleMask(cast<ShuffleVectorSDNode>(N), false, 2, *CurDAG);
+}]>;
+
+
+
 def VSLDOI_get_imm : SDNodeXForm<vector_shuffle, [{
-  return getI32Imm(PPC::isVSLDOIShuffleMask(N, 0, *CurDAG));
+  return getI32Imm(PPC::isVSLDOIShuffleMask(N, 0, *CurDAG), SDLoc(N));
 }]>;
 def vsldoi_shuffle : PatFrag<(ops node:$lhs, node:$rhs),
                              (vector_shuffle node:$lhs, node:$rhs), [{
@@ -140,7 +194,7 @@ def vsldoi_shuffle : PatFrag<(ops node:$lhs, node:$rhs),
 /// VSLDOI_unary* - These are used to match vsldoi(X,X), which is turned into
 /// vector_shuffle(X,undef,mask) by the dag combiner.
 def VSLDOI_unary_get_imm : SDNodeXForm<vector_shuffle, [{
-  return getI32Imm(PPC::isVSLDOIShuffleMask(N, 1, *CurDAG));
+  return getI32Imm(PPC::isVSLDOIShuffleMask(N, 1, *CurDAG), SDLoc(N));
 }]>;
 def vsldoi_unary_shuffle : PatFrag<(ops node:$lhs, node:$rhs),
                                    (vector_shuffle node:$lhs, node:$rhs), [{
@@ -151,7 +205,7 @@ def vsldoi_unary_shuffle : PatFrag<(ops node:$lhs, node:$rhs),
 /// VSLDOI_swapped* - These fragments are provided for little-endian, where
 /// the inputs must be swapped for correct semantics.
 def VSLDOI_swapped_get_imm : SDNodeXForm<vector_shuffle, [{
-  return getI32Imm(PPC::isVSLDOIShuffleMask(N, 2, *CurDAG));
+  return getI32Imm(PPC::isVSLDOIShuffleMask(N, 2, *CurDAG), SDLoc(N));
 }]>;
 def vsldoi_swapped_shuffle : PatFrag<(ops node:$lhs, node:$rhs),
                                      (vector_shuffle node:$lhs, node:$rhs), [{
@@ -161,21 +215,21 @@ def vsldoi_swapped_shuffle : PatFrag<(ops node:$lhs, node:$rhs),
 
 // VSPLT*_get_imm xform function: convert vector_shuffle mask to VSPLT* imm.
 def VSPLTB_get_imm : SDNodeXForm<vector_shuffle, [{
-  return getI32Imm(PPC::getVSPLTImmediate(N, 1, *CurDAG));
+  return getI32Imm(PPC::getVSPLTImmediate(N, 1, *CurDAG), SDLoc(N));
 }]>;
 def vspltb_shuffle : PatFrag<(ops node:$lhs, node:$rhs),
                              (vector_shuffle node:$lhs, node:$rhs), [{
   return PPC::isSplatShuffleMask(cast<ShuffleVectorSDNode>(N), 1);
 }], VSPLTB_get_imm>;
 def VSPLTH_get_imm : SDNodeXForm<vector_shuffle, [{
-  return getI32Imm(PPC::getVSPLTImmediate(N, 2, *CurDAG));
+  return getI32Imm(PPC::getVSPLTImmediate(N, 2, *CurDAG), SDLoc(N));
 }]>;
 def vsplth_shuffle : PatFrag<(ops node:$lhs, node:$rhs),
                              (vector_shuffle node:$lhs, node:$rhs), [{
   return PPC::isSplatShuffleMask(cast<ShuffleVectorSDNode>(N), 2);
 }], VSPLTH_get_imm>;
 def VSPLTW_get_imm : SDNodeXForm<vector_shuffle, [{
-  return getI32Imm(PPC::getVSPLTImmediate(N, 4, *CurDAG));
+  return getI32Imm(PPC::getVSPLTImmediate(N, 4, *CurDAG), SDLoc(N));
 }]>;
 def vspltw_shuffle : PatFrag<(ops node:$lhs, node:$rhs),
                              (vector_shuffle node:$lhs, node:$rhs), [{
@@ -802,26 +856,37 @@ def : Pat<(v16i8 (bitconvert (v8i16 VRRC:$src))), (v16i8 VRRC:$src)>;
 def : Pat<(v16i8 (bitconvert (v4i32 VRRC:$src))), (v16i8 VRRC:$src)>;
 def : Pat<(v16i8 (bitconvert (v4f32 VRRC:$src))), (v16i8 VRRC:$src)>;
 def : Pat<(v16i8 (bitconvert (v2i64 VRRC:$src))), (v16i8 VRRC:$src)>;
+def : Pat<(v16i8 (bitconvert (v1i128 VRRC:$src))), (v16i8 VRRC:$src)>;
 
 def : Pat<(v8i16 (bitconvert (v16i8 VRRC:$src))), (v8i16 VRRC:$src)>;
 def : Pat<(v8i16 (bitconvert (v4i32 VRRC:$src))), (v8i16 VRRC:$src)>;
 def : Pat<(v8i16 (bitconvert (v4f32 VRRC:$src))), (v8i16 VRRC:$src)>;
 def : Pat<(v8i16 (bitconvert (v2i64 VRRC:$src))), (v8i16 VRRC:$src)>;
+def : Pat<(v8i16 (bitconvert (v1i128 VRRC:$src))), (v8i16 VRRC:$src)>;
 
 def : Pat<(v4i32 (bitconvert (v16i8 VRRC:$src))), (v4i32 VRRC:$src)>;
 def : Pat<(v4i32 (bitconvert (v8i16 VRRC:$src))), (v4i32 VRRC:$src)>;
 def : Pat<(v4i32 (bitconvert (v4f32 VRRC:$src))), (v4i32 VRRC:$src)>;
 def : Pat<(v4i32 (bitconvert (v2i64 VRRC:$src))), (v4i32 VRRC:$src)>;
+def : Pat<(v4i32 (bitconvert (v1i128 VRRC:$src))), (v4i32 VRRC:$src)>;
 
 def : Pat<(v4f32 (bitconvert (v16i8 VRRC:$src))), (v4f32 VRRC:$src)>;
 def : Pat<(v4f32 (bitconvert (v8i16 VRRC:$src))), (v4f32 VRRC:$src)>;
 def : Pat<(v4f32 (bitconvert (v4i32 VRRC:$src))), (v4f32 VRRC:$src)>;
 def : Pat<(v4f32 (bitconvert (v2i64 VRRC:$src))), (v4f32 VRRC:$src)>;
+def : Pat<(v4f32 (bitconvert (v1i128 VRRC:$src))), (v4f32 VRRC:$src)>;
 
 def : Pat<(v2i64 (bitconvert (v16i8 VRRC:$src))), (v2i64 VRRC:$src)>;
 def : Pat<(v2i64 (bitconvert (v8i16 VRRC:$src))), (v2i64 VRRC:$src)>;
 def : Pat<(v2i64 (bitconvert (v4i32 VRRC:$src))), (v2i64 VRRC:$src)>;
 def : Pat<(v2i64 (bitconvert (v4f32 VRRC:$src))), (v2i64 VRRC:$src)>;
+def : Pat<(v2i64 (bitconvert (v1i128 VRRC:$src))), (v2i64 VRRC:$src)>;
+
+def : Pat<(v1i128 (bitconvert (v16i8 VRRC:$src))), (v1i128 VRRC:$src)>;
+def : Pat<(v1i128 (bitconvert (v8i16 VRRC:$src))), (v1i128 VRRC:$src)>;
+def : Pat<(v1i128 (bitconvert (v4i32 VRRC:$src))), (v1i128 VRRC:$src)>;
+def : Pat<(v1i128 (bitconvert (v4f32 VRRC:$src))), (v1i128 VRRC:$src)>;
+def : Pat<(v1i128 (bitconvert (v2i64 VRRC:$src))), (v1i128 VRRC:$src)>;
 
 // Shuffles.
 
@@ -970,6 +1035,29 @@ def VMINSD : VX1_Int_Ty<962, "vminsd", int_ppc_altivec_vminsd, v2i64>;
 def VMINUD : VX1_Int_Ty<706, "vminud", int_ppc_altivec_vminud, v2i64>;
 } // isCommutable
 
+// Vector merge 
+def VMRGEW : VXForm_1<1932, (outs vrrc:$vD), (ins vrrc:$vA, vrrc:$vB),
+                      "vmrgew $vD, $vA, $vB", IIC_VecFP,
+                      [(set v16i8:$vD, (vmrgew_shuffle v16i8:$vA, v16i8:$vB))]>;
+def VMRGOW : VXForm_1<1676, (outs vrrc:$vD), (ins vrrc:$vA, vrrc:$vB),
+                      "vmrgow $vD, $vA, $vB", IIC_VecFP,
+                      [(set v16i8:$vD, (vmrgow_shuffle v16i8:$vA, v16i8:$vB))]>;
+
+// Match vmrgew(x,x) and vmrgow(x,x)
+def:Pat<(vmrgew_unary_shuffle v16i8:$vA, undef),
+        (VMRGEW $vA, $vA)>;
+def:Pat<(vmrgow_unary_shuffle v16i8:$vA, undef),
+        (VMRGOW $vA, $vA)>;
+
+// Match vmrgew(y,x) and vmrgow(y,x), i.e., swapped operands.  These fragments
+// are matched for little-endian, where the inputs must be swapped for correct
+// semantics.w
+def:Pat<(vmrgew_swapped_shuffle v16i8:$vA, v16i8:$vB),
+        (VMRGEW $vB, $vA)>;
+def:Pat<(vmrgow_swapped_shuffle v16i8:$vA, v16i8:$vB),
+        (VMRGOW $vB, $vA)>;
+
+
 // Vector shifts
 def VRLD : VX1_Int_Ty<196, "vrld", int_ppc_altivec_vrld, v2i64>;
 def VSLD : VXForm_1<1476, (outs vrrc:$vD), (ins vrrc:$vA, vrrc:$vB),
@@ -987,12 +1075,29 @@ let isCommutable = 1 in {
 def VADDUDM : VXForm_1<192, (outs vrrc:$vD), (ins vrrc:$vA, vrrc:$vB),
                        "vaddudm $vD, $vA, $vB", IIC_VecGeneral,
                        [(set v2i64:$vD, (add v2i64:$vA, v2i64:$vB))]>;
+def VADDUQM : VXForm_1<256, (outs vrrc:$vD), (ins vrrc:$vA, vrrc:$vB),
+                       "vadduqm $vD, $vA, $vB", IIC_VecGeneral,
+                       [(set v1i128:$vD, (add v1i128:$vA, v1i128:$vB))]>;
 } // isCommutable
 
+// Vector Quadword Add
+def VADDEUQM : VA1a_Int_Ty<60, "vaddeuqm", int_ppc_altivec_vaddeuqm, v1i128>;
+def VADDCUQ  : VX1_Int_Ty<320, "vaddcuq", int_ppc_altivec_vaddcuq, v1i128>;
+def VADDECUQ : VA1a_Int_Ty<61, "vaddecuq", int_ppc_altivec_vaddecuq, v1i128>;
+
+// Vector Doubleword Subtract
 def VSUBUDM : VXForm_1<1216, (outs vrrc:$vD), (ins vrrc:$vA, vrrc:$vB),
                        "vsubudm $vD, $vA, $vB", IIC_VecGeneral,
                        [(set v2i64:$vD, (sub v2i64:$vA, v2i64:$vB))]>;
 
+// Vector Quadword Subtract
+def VSUBUQM : VXForm_1<1280, (outs vrrc:$vD), (ins vrrc:$vA, vrrc:$vB),
+                       "vsubuqm $vD, $vA, $vB", IIC_VecGeneral,
+                       [(set v1i128:$vD, (sub v1i128:$vA, v1i128:$vB))]>;
+def VSUBEUQM : VA1a_Int_Ty<62, "vsubeuqm", int_ppc_altivec_vsubeuqm, v1i128>;
+def VSUBCUQ  : VX1_Int_Ty<1344, "vsubcuq", int_ppc_altivec_vsubcuq, v1i128>;
+def VSUBECUQ : VA1a_Int_Ty<63, "vsubecuq", int_ppc_altivec_vsubecuq, v1i128>;
+
 // Count Leading Zeros
 def VCLZB : VXForm_2<1794, (outs vrrc:$vD), (ins vrrc:$vB),
                      "vclzb $vD, $vB", IIC_VecGeneral,
@@ -1065,6 +1170,31 @@ def VPMSUMD : VX1_Int_Ty<1224, "vpmsumd",
 def VPERMXOR : VA1a_Int_Ty<45, "vpermxor",
                          int_ppc_altivec_crypto_vpermxor, v16i8>;
 
+// Vector doubleword integer pack and unpack.
+def VPKSDSS : VX1_Int_Ty2<1486, "vpksdss", int_ppc_altivec_vpksdss,
+                          v4i32, v2i64>;
+def VPKSDUS : VX1_Int_Ty2<1358, "vpksdus", int_ppc_altivec_vpksdus,
+                          v4i32, v2i64>;
+def VPKUDUM : VXForm_1<1102, (outs vrrc:$vD), (ins vrrc:$vA, vrrc:$vB),
+                       "vpkudum $vD, $vA, $vB", IIC_VecFP,
+                       [(set v16i8:$vD,
+                         (vpkudum_shuffle v16i8:$vA, v16i8:$vB))]>;
+def VPKUDUS : VX1_Int_Ty2<1230, "vpkudus", int_ppc_altivec_vpkudus,
+                          v4i32, v2i64>;
+def VUPKHSW : VX2_Int_Ty2<1614, "vupkhsw", int_ppc_altivec_vupkhsw,
+                          v2i64, v4i32>;
+def VUPKLSW : VX2_Int_Ty2<1742, "vupklsw", int_ppc_altivec_vupklsw,
+                          v2i64, v4i32>;
+
+// Shuffle patterns for unary and swapped (LE) vector pack modulo.
+def:Pat<(vpkudum_unary_shuffle v16i8:$vA, undef),
+        (VPKUDUM $vA, $vA)>;
+def:Pat<(vpkudum_swapped_shuffle v16i8:$vA, v16i8:$vB),
+        (VPKUDUM $vB, $vA)>;
+
+def VGBBD : VX2_Int_Ty2<1292, "vgbbd", int_ppc_altivec_vgbbd, v16i8, v16i8>;
+def VBPERMQ : VX1_Int_Ty2<1356, "vbpermq", int_ppc_altivec_vbpermq,
+                          v2i64, v16i8>;
 } // end HasP8Altivec
 
 // Crypto instructions (from builtins)
diff --git a/lib/Target/PowerPC/PPCInstrFormats.td b/lib/Target/PowerPC/PPCInstrFormats.td
index 43c2158d713..4e03ed27653 100644
--- a/lib/Target/PowerPC/PPCInstrFormats.td
+++ b/lib/Target/PowerPC/PPCInstrFormats.td
@@ -936,6 +936,21 @@ class DCB_Form<bits<10> xo, bits<5> immfield, dag OOL, dag IOL, string asmstr,
   let Inst{31}    = 0;
 }
 
+class DCB_Form_hint<bits<10> xo, dag OOL, dag IOL, string asmstr,
+                    InstrItinClass itin, list<dag> pattern>
+  : I<31, OOL, IOL, asmstr, itin> {
+  bits<5> TH;
+  bits<5> A;
+  bits<5> B;
+
+  let Pattern = pattern;
+
+  let Inst{6-10}  = TH;
+  let Inst{11-15} = A;
+  let Inst{16-20} = B;
+  let Inst{21-30} = xo;
+  let Inst{31}    = 0;
+}
 
 // DSS_Form - Form X instruction, used for altivec dss* instructions.
 class DSS_Form<bits<1> T, bits<10> xo, dag OOL, dag IOL, string asmstr,
@@ -1065,6 +1080,19 @@ class XLForm_4<bits<6> opcode, bits<10> xo, dag OOL, dag IOL, string asmstr,
   let Inst{31}    = RC;
 }
 
+class XLForm_S<bits<6> opcode, bits<10> xo, dag OOL, dag IOL, string asmstr,
+               InstrItinClass itin, list<dag> pattern>
+    : I<opcode, OOL, IOL, asmstr, itin> {
+  bits<1> S;
+  
+  let Pattern = pattern;
+  
+  let Inst{6-19}  = 0;
+  let Inst{20}    = S;
+  let Inst{21-30} = xo;
+  let Inst{31}    = 0;
+}
+
 class XLForm_2_and_DSForm_1<bits<6> opcode1, bits<10> xo1, bit lk,
                             bits<6> opcode2, bits<2> xo2,
                             dag OOL, dag IOL, string asmstr,
@@ -1143,6 +1171,19 @@ class XFXForm_3<bits<6> opcode, bits<10> xo, dag OOL, dag IOL, string asmstr,
   let Inst{31}    = 0;
 }
 
+class XFXForm_3p<bits<6> opcode, bits<10> xo, dag OOL, dag IOL, string asmstr,
+                 InstrItinClass itin, list<dag> pattern>
+         : I<opcode, OOL, IOL, asmstr, itin> {
+  bits<5>  RT;
+  bits<10> Entry;
+  let Pattern = pattern;
+
+  let Inst{6-10}  = RT;
+  let Inst{11-20} = Entry;
+  let Inst{21-30} = xo;
+  let Inst{31}    = 0;
+}
+
 class XFXForm_5<bits<6> opcode, bits<10> xo, dag OOL, dag IOL, string asmstr,
                 InstrItinClass itin> 
   : I<opcode, OOL, IOL, asmstr, itin> {
diff --git a/lib/Target/PowerPC/PPCInstrHTM.td b/lib/Target/PowerPC/PPCInstrHTM.td
index 20e6a628632..6c4e2129087 100644
--- a/lib/Target/PowerPC/PPCInstrHTM.td
+++ b/lib/Target/PowerPC/PPCInstrHTM.td
@@ -17,7 +17,7 @@
 def HasHTM : Predicate<"PPCSubTarget->hasHTM()">;
 
 def HTM_get_imm : SDNodeXForm<imm, [{
-  return getI32Imm (N->getZExtValue());
+  return getI32Imm (N->getZExtValue(), SDLoc(N));
 }]>;
 
 let hasSideEffects = 1, usesCustomInserter = 1  in {
diff --git a/lib/Target/PowerPC/PPCInstrInfo.cpp b/lib/Target/PowerPC/PPCInstrInfo.cpp
index c9c2949dc6c..b302d41f203 100644
--- a/lib/Target/PowerPC/PPCInstrInfo.cpp
+++ b/lib/Target/PowerPC/PPCInstrInfo.cpp
@@ -31,6 +31,7 @@
 #include "llvm/CodeGen/SlotIndexes.h"
 #include "llvm/CodeGen/StackMaps.h"
 #include "llvm/MC/MCAsmInfo.h"
+#include "llvm/MC/MCInst.h"
 #include "llvm/Support/CommandLine.h"
 #include "llvm/Support/Debug.h"
 #include "llvm/Support/ErrorHandling.h"
@@ -56,6 +57,10 @@ static cl::opt<bool> VSXSelfCopyCrash("crash-on-ppc-vsx-self-copy",
 cl::desc("Causes the backend to crash instead of generating a nop VSX copy"),
 cl::Hidden);
 
+static cl::opt<bool>
+UseOldLatencyCalc("ppc-old-latency-calc", cl::Hidden,
+  cl::desc("Use the old (incorrect) instruction latency calculation"));
+
 // Pin the vtable to this file.
 void PPCInstrInfo::anchor() {}
 
@@ -102,6 +107,35 @@ PPCInstrInfo::CreateTargetPostRAHazardRecognizer(const InstrItineraryData *II,
   return new ScoreboardHazardRecognizer(II, DAG);
 }
 
+unsigned PPCInstrInfo::getInstrLatency(const InstrItineraryData *ItinData,
+                                       const MachineInstr *MI,
+                                       unsigned *PredCost) const {
+  if (!ItinData || UseOldLatencyCalc)
+    return PPCGenInstrInfo::getInstrLatency(ItinData, MI, PredCost);
+
+  // The default implementation of getInstrLatency calls getStageLatency, but
+  // getStageLatency does not do the right thing for us. While we have
+  // itinerary, most cores are fully pipelined, and so the itineraries only
+  // express the first part of the pipeline, not every stage. Instead, we need
+  // to use the listed output operand cycle number (using operand 0 here, which
+  // is an output).
+
+  unsigned Latency = 1;
+  unsigned DefClass = MI->getDesc().getSchedClass();
+  for (unsigned i = 0, e = MI->getNumOperands(); i != e; ++i) {
+    const MachineOperand &MO = MI->getOperand(i);
+    if (!MO.isReg() || !MO.isDef() || MO.isImplicit())
+      continue;
+
+    int Cycle = ItinData->getOperandCycle(DefClass, i);
+    if (Cycle < 0)
+      continue;
+
+    Latency = std::max(Latency, (unsigned) Cycle);
+  }
+
+  return Latency;
+}
 
 int PPCInstrInfo::getOperandLatency(const InstrItineraryData *ItinData,
                                     const MachineInstr *DefMI, unsigned DefIdx,
@@ -110,6 +144,9 @@ int PPCInstrInfo::getOperandLatency(const InstrItineraryData *ItinData,
   int Latency = PPCGenInstrInfo::getOperandLatency(ItinData, DefMI, DefIdx,
                                                    UseMI, UseIdx);
 
+  if (!DefMI->getParent())
+    return Latency;
+
   const MachineOperand &DefMO = DefMI->getOperand(DefIdx);
   unsigned Reg = DefMO.getReg();
 
@@ -152,6 +189,265 @@ int PPCInstrInfo::getOperandLatency(const InstrItineraryData *ItinData,
   return Latency;
 }
 
+static bool hasVirtualRegDefsInBasicBlock(const MachineInstr &Inst,
+                                          const MachineBasicBlock *MBB) {
+  const MachineOperand &Op1 = Inst.getOperand(1);
+  const MachineOperand &Op2 = Inst.getOperand(2);
+  const MachineRegisterInfo &MRI = MBB->getParent()->getRegInfo();
+
+  // We need virtual register definitions.
+  MachineInstr *MI1 = nullptr;
+  MachineInstr *MI2 = nullptr;
+  if (Op1.isReg() && TargetRegisterInfo::isVirtualRegister(Op1.getReg()))
+    MI1 = MRI.getUniqueVRegDef(Op1.getReg());
+  if (Op2.isReg() && TargetRegisterInfo::isVirtualRegister(Op2.getReg()))
+    MI2 = MRI.getUniqueVRegDef(Op2.getReg());
+
+  // And they need to be in the trace (otherwise, they won't have a depth).
+  if (MI1 && MI2 && MI1->getParent() == MBB && MI2->getParent() == MBB)
+    return true;
+
+  return false;
+}
+
+static bool hasReassocSibling(const MachineInstr &Inst, bool &Commuted) {
+  const MachineBasicBlock *MBB = Inst.getParent();
+  const MachineRegisterInfo &MRI = MBB->getParent()->getRegInfo();
+  MachineInstr *MI1 = MRI.getUniqueVRegDef(Inst.getOperand(1).getReg());
+  MachineInstr *MI2 = MRI.getUniqueVRegDef(Inst.getOperand(2).getReg());
+  unsigned AssocOpcode = Inst.getOpcode();
+
+  // If only one operand has the same opcode and it's the second source operand,
+  // the operands must be commuted.
+  Commuted = MI1->getOpcode() != AssocOpcode && MI2->getOpcode() == AssocOpcode;
+  if (Commuted)
+    std::swap(MI1, MI2);
+
+  // 1. The previous instruction must be the same type as Inst.
+  // 2. The previous instruction must have virtual register definitions for its
+  //    operands in the same basic block as Inst.
+  // 3. The previous instruction's result must only be used by Inst.
+  if (MI1->getOpcode() == AssocOpcode &&
+      hasVirtualRegDefsInBasicBlock(*MI1, MBB) &&
+      MRI.hasOneNonDBGUse(MI1->getOperand(0).getReg()))
+    return true;
+
+  return false;
+}
+
+// This function does not list all associative and commutative operations, but
+// only those worth feeding through the machine combiner in an attempt to
+// reduce the critical path. Mostly, this means floating-point operations,
+// because they have high latencies (compared to other operations, such and
+// and/or, which are also associative and commutative, but have low latencies).
+//
+// The concept is that these operations can benefit from this kind of
+// transformation:
+//
+// A = ? op ?
+// B = A op X
+// C = B op Y
+// -->
+// A = ? op ?
+// B = X op Y
+// C = A op B
+//
+// breaking the dependency between A and B, allowing them to be executed in
+// parallel (or back-to-back in a pipeline) instead of depending on each other.
+static bool isAssociativeAndCommutative(unsigned Opcode) {
+  switch (Opcode) {
+  // FP Add:
+  case PPC::FADD:
+  case PPC::FADDS:
+  // FP Multiply:
+  case PPC::FMUL:
+  case PPC::FMULS:
+  // Altivec Add:
+  case PPC::VADDFP:
+  // VSX Add:
+  case PPC::XSADDDP:
+  case PPC::XVADDDP:
+  case PPC::XVADDSP:
+  case PPC::XSADDSP:
+  // VSX Multiply:
+  case PPC::XSMULDP:
+  case PPC::XVMULDP:
+  case PPC::XVMULSP:
+  case PPC::XSMULSP:
+  // QPX Add:
+  case PPC::QVFADD:
+  case PPC::QVFADDS:
+  case PPC::QVFADDSs:
+  // QPX Multiply:
+  case PPC::QVFMUL:
+  case PPC::QVFMULS:
+  case PPC::QVFMULSs:
+    return true;
+  default:
+    return false;
+  }
+}
+
+/// Return true if the input instruction is part of a chain of dependent ops
+/// that are suitable for reassociation, otherwise return false.
+/// If the instruction's operands must be commuted to have a previous
+/// instruction of the same type define the first source operand, Commuted will
+/// be set to true.
+static bool isReassocCandidate(const MachineInstr &Inst, bool &Commuted) {
+  // 1. The operation must be associative and commutative.
+  // 2. The instruction must have virtual register definitions for its
+  //    operands in the same basic block.
+  // 3. The instruction must have a reassociable sibling.
+  if (isAssociativeAndCommutative(Inst.getOpcode()) &&
+      hasVirtualRegDefsInBasicBlock(Inst, Inst.getParent()) &&
+      hasReassocSibling(Inst, Commuted))
+    return true;
+
+  return false;
+}
+
+bool PPCInstrInfo::getMachineCombinerPatterns(MachineInstr &Root,
+        SmallVectorImpl<MachineCombinerPattern::MC_PATTERN> &Patterns) const {
+  // Using the machine combiner in this way is potentially expensive, so
+  // restrict to when aggressive optimizations are desired.
+  if (Subtarget.getTargetMachine().getOptLevel() != CodeGenOpt::Aggressive)
+    return false;
+
+  // FP reassociation is only legal when we don't need strict IEEE semantics.
+  if (!Root.getParent()->getParent()->getTarget().Options.UnsafeFPMath)
+    return false;
+
+  // Look for this reassociation pattern:
+  //   B = A op X (Prev)
+  //   C = B op Y (Root)
+
+  // FIXME: We should also match FMA operations here, where we consider the
+  // 'part' of the FMA, either the addition or the multiplication, paired with
+  // an actual addition or multiplication.
+
+  bool Commute;
+  if (isReassocCandidate(Root, Commute)) {
+    // We found a sequence of instructions that may be suitable for a
+    // reassociation of operands to increase ILP. Specify each commutation
+    // possibility for the Prev instruction in the sequence and let the
+    // machine combiner decide if changing the operands is worthwhile.
+    if (Commute) {
+      Patterns.push_back(MachineCombinerPattern::MC_REASSOC_AX_YB);
+      Patterns.push_back(MachineCombinerPattern::MC_REASSOC_XA_YB);
+    } else {
+      Patterns.push_back(MachineCombinerPattern::MC_REASSOC_AX_BY);
+      Patterns.push_back(MachineCombinerPattern::MC_REASSOC_XA_BY);
+    }
+    return true;
+  }
+
+  return false;
+}
+
+/// Attempt the following reassociation to reduce critical path length:
+///   B = A op X (Prev)
+///   C = B op Y (Root)
+///   ===>
+///   B = X op Y
+///   C = A op B
+static void reassociateOps(MachineInstr &Root, MachineInstr &Prev,
+                           MachineCombinerPattern::MC_PATTERN Pattern,
+                           SmallVectorImpl<MachineInstr *> &InsInstrs,
+                           SmallVectorImpl<MachineInstr *> &DelInstrs,
+                           DenseMap<unsigned, unsigned> &InstrIdxForVirtReg) {
+  MachineFunction *MF = Root.getParent()->getParent();
+  MachineRegisterInfo &MRI = MF->getRegInfo();
+  const TargetInstrInfo *TII = MF->getSubtarget().getInstrInfo();
+  const TargetRegisterInfo *TRI = MF->getSubtarget().getRegisterInfo();
+  const TargetRegisterClass *RC = Root.getRegClassConstraint(0, TII, TRI);
+
+  // This array encodes the operand index for each parameter because the
+  // operands may be commuted. Each row corresponds to a pattern value,
+  // and each column specifies the index of A, B, X, Y.
+  unsigned OpIdx[4][4] = {
+    { 1, 1, 2, 2 },
+    { 1, 2, 2, 1 },
+    { 2, 1, 1, 2 },
+    { 2, 2, 1, 1 }
+  };
+
+  MachineOperand &OpA = Prev.getOperand(OpIdx[Pattern][0]);
+  MachineOperand &OpB = Root.getOperand(OpIdx[Pattern][1]);
+  MachineOperand &OpX = Prev.getOperand(OpIdx[Pattern][2]);
+  MachineOperand &OpY = Root.getOperand(OpIdx[Pattern][3]);
+  MachineOperand &OpC = Root.getOperand(0);
+
+  unsigned RegA = OpA.getReg();
+  unsigned RegB = OpB.getReg();
+  unsigned RegX = OpX.getReg();
+  unsigned RegY = OpY.getReg();
+  unsigned RegC = OpC.getReg();
+
+  if (TargetRegisterInfo::isVirtualRegister(RegA))
+    MRI.constrainRegClass(RegA, RC);
+  if (TargetRegisterInfo::isVirtualRegister(RegB))
+    MRI.constrainRegClass(RegB, RC);
+  if (TargetRegisterInfo::isVirtualRegister(RegX))
+    MRI.constrainRegClass(RegX, RC);
+  if (TargetRegisterInfo::isVirtualRegister(RegY))
+    MRI.constrainRegClass(RegY, RC);
+  if (TargetRegisterInfo::isVirtualRegister(RegC))
+    MRI.constrainRegClass(RegC, RC);
+
+  // Create a new virtual register for the result of (X op Y) instead of
+  // recycling RegB because the MachineCombiner's computation of the critical
+  // path requires a new register definition rather than an existing one.
+  unsigned NewVR = MRI.createVirtualRegister(RC);
+  InstrIdxForVirtReg.insert(std::make_pair(NewVR, 0));
+
+  unsigned Opcode = Root.getOpcode();
+  bool KillA = OpA.isKill();
+  bool KillX = OpX.isKill();
+  bool KillY = OpY.isKill();
+
+  // Create new instructions for insertion.
+  MachineInstrBuilder MIB1 =
+    BuildMI(*MF, Prev.getDebugLoc(), TII->get(Opcode), NewVR)
+      .addReg(RegX, getKillRegState(KillX))
+      .addReg(RegY, getKillRegState(KillY));
+  InsInstrs.push_back(MIB1);
+
+  MachineInstrBuilder MIB2 =
+    BuildMI(*MF, Root.getDebugLoc(), TII->get(Opcode), RegC)
+      .addReg(RegA, getKillRegState(KillA))
+      .addReg(NewVR, getKillRegState(true));
+  InsInstrs.push_back(MIB2);
+
+  // Record old instructions for deletion.
+  DelInstrs.push_back(&Prev);
+  DelInstrs.push_back(&Root);
+}
+
+void PPCInstrInfo::genAlternativeCodeSequence(
+    MachineInstr &Root,
+    MachineCombinerPattern::MC_PATTERN Pattern,
+    SmallVectorImpl<MachineInstr *> &InsInstrs,
+    SmallVectorImpl<MachineInstr *> &DelInstrs,
+    DenseMap<unsigned, unsigned> &InstIdxForVirtReg) const {
+  MachineRegisterInfo &MRI = Root.getParent()->getParent()->getRegInfo();
+
+  // Select the previous instruction in the sequence based on the input pattern.
+  MachineInstr *Prev = nullptr;
+  switch (Pattern) {
+    case MachineCombinerPattern::MC_REASSOC_AX_BY:
+    case MachineCombinerPattern::MC_REASSOC_XA_BY:
+      Prev = MRI.getUniqueVRegDef(Root.getOperand(1).getReg());
+      break;
+    case MachineCombinerPattern::MC_REASSOC_AX_YB:
+    case MachineCombinerPattern::MC_REASSOC_XA_YB:
+      Prev = MRI.getUniqueVRegDef(Root.getOperand(2).getReg());
+  }
+  assert(Prev && "Unknown pattern for machine combiner");
+
+  reassociateOps(Root, *Prev, Pattern, InsInstrs, DelInstrs, InstIdxForVirtReg);
+  return;
+}
+
 // Detect 32 -> 64-bit extensions where we may reuse the low sub-register.
 bool PPCInstrInfo::isCoalescableExtInstr(const MachineInstr &MI,
                                          unsigned &SrcReg, unsigned &DstReg,
@@ -351,15 +647,10 @@ bool PPCInstrInfo::AnalyzeBranch(MachineBasicBlock &MBB,MachineBasicBlock *&TBB,
   bool isPPC64 = Subtarget.isPPC64();
 
   // If the block has no terminators, it just falls into the block after it.
-  MachineBasicBlock::iterator I = MBB.end();
-  if (I == MBB.begin())
+  MachineBasicBlock::iterator I = MBB.getLastNonDebugInstr();
+  if (I == MBB.end())
     return false;
-  --I;
-  while (I->isDebugValue()) {
-    if (I == MBB.begin())
-      return false;
-    --I;
-  }
+
   if (!isUnpredicatedTerminator(I))
     return false;
 
@@ -512,14 +803,10 @@ bool PPCInstrInfo::AnalyzeBranch(MachineBasicBlock &MBB,MachineBasicBlock *&TBB,
 }
 
 unsigned PPCInstrInfo::RemoveBranch(MachineBasicBlock &MBB) const {
-  MachineBasicBlock::iterator I = MBB.end();
-  if (I == MBB.begin()) return 0;
-  --I;
-  while (I->isDebugValue()) {
-    if (I == MBB.begin())
-      return 0;
-    --I;
-  }
+  MachineBasicBlock::iterator I = MBB.getLastNonDebugInstr();
+  if (I == MBB.end())
+    return 0;
+
   if (I->getOpcode() != PPC::B && I->getOpcode() != PPC::BCC &&
       I->getOpcode() != PPC::BC && I->getOpcode() != PPC::BCn &&
       I->getOpcode() != PPC::BDNZ8 && I->getOpcode() != PPC::BDNZ &&
@@ -547,7 +834,7 @@ unsigned PPCInstrInfo::RemoveBranch(MachineBasicBlock &MBB) const {
 unsigned
 PPCInstrInfo::InsertBranch(MachineBasicBlock &MBB, MachineBasicBlock *TBB,
                            MachineBasicBlock *FBB,
-                           const SmallVectorImpl<MachineOperand> &Cond,
+                           ArrayRef<MachineOperand> Cond,
                            DebugLoc DL) const {
   // Shouldn't be a fall through.
   assert(TBB && "InsertBranch must not be told to insert a fallthrough");
@@ -592,7 +879,7 @@ PPCInstrInfo::InsertBranch(MachineBasicBlock &MBB, MachineBasicBlock *TBB,
 
 // Select analysis.
 bool PPCInstrInfo::canInsertSelect(const MachineBasicBlock &MBB,
-                const SmallVectorImpl<MachineOperand> &Cond,
+                ArrayRef<MachineOperand> Cond,
                 unsigned TrueReg, unsigned FalseReg,
                 int &CondCycles, int &TrueCycles, int &FalseCycles) const {
   if (!Subtarget.hasISEL())
@@ -633,8 +920,7 @@ bool PPCInstrInfo::canInsertSelect(const MachineBasicBlock &MBB,
 
 void PPCInstrInfo::insertSelect(MachineBasicBlock &MBB,
                                 MachineBasicBlock::iterator MI, DebugLoc dl,
-                                unsigned DestReg,
-                                const SmallVectorImpl<MachineOperand> &Cond,
+                                unsigned DestReg, ArrayRef<MachineOperand> Cond,
                                 unsigned TrueReg, unsigned FalseReg) const {
   assert(Cond.size() == 2 &&
          "PPC branch conditions have two components!");
@@ -815,7 +1101,8 @@ void PPCInstrInfo::copyPhysReg(MachineBasicBlock &MBB,
     // copies are generated, they are close enough to some use that the
     // lower-latency form is preferable.
     Opc = PPC::XXLOR;
-  else if (PPC::VSFRCRegClass.contains(DestReg, SrcReg))
+  else if (PPC::VSFRCRegClass.contains(DestReg, SrcReg) ||
+           PPC::VSSRCRegClass.contains(DestReg, SrcReg))
     Opc = PPC::XXLORf;
   else if (PPC::QFRCRegClass.contains(DestReg, SrcReg))
     Opc = PPC::QVFMR;
@@ -900,6 +1187,12 @@ PPCInstrInfo::StoreRegToStackSlot(MachineFunction &MF,
                                                getKillRegState(isKill)),
                                        FrameIdx));
     NonRI = true;
+  } else if (PPC::VSSRCRegClass.hasSubClassEq(RC)) {
+    NewMIs.push_back(addFrameReference(BuildMI(MF, DL, get(PPC::STXSSPX))
+                                       .addReg(SrcReg,
+                                               getKillRegState(isKill)),
+                                       FrameIdx));
+    NonRI = true;
   } else if (PPC::VRSAVERCRegClass.hasSubClassEq(RC)) {
     assert(Subtarget.isDarwin() &&
            "VRSAVE only needs spill/restore on Darwin");
@@ -1013,6 +1306,10 @@ PPCInstrInfo::LoadRegFromStackSlot(MachineFunction &MF, DebugLoc DL,
     NewMIs.push_back(addFrameReference(BuildMI(MF, DL, get(PPC::LXSDX), DestReg),
                                        FrameIdx));
     NonRI = true;
+  } else if (PPC::VSSRCRegClass.hasSubClassEq(RC)) {
+    NewMIs.push_back(addFrameReference(BuildMI(MF, DL, get(PPC::LXSSPX), DestReg),
+                                       FrameIdx));
+    NonRI = true;
   } else if (PPC::VRSAVERCRegClass.hasSubClassEq(RC)) {
     assert(Subtarget.isDarwin() &&
            "VRSAVE only needs spill/restore on Darwin");
@@ -1201,9 +1498,8 @@ bool PPCInstrInfo::isUnpredicatedTerminator(const MachineInstr *MI) const {
   return !isPredicated(MI);
 }
 
-bool PPCInstrInfo::PredicateInstruction(
-                     MachineInstr *MI,
-                     const SmallVectorImpl<MachineOperand> &Pred) const {
+bool PPCInstrInfo::PredicateInstruction(MachineInstr *MI,
+                                        ArrayRef<MachineOperand> Pred) const {
   unsigned OpC = MI->getOpcode();
   if (OpC == PPC::BLR || OpC == PPC::BLR8) {
     if (Pred[1].getReg() == PPC::CTR8 || Pred[1].getReg() == PPC::CTR) {
@@ -1294,9 +1590,8 @@ bool PPCInstrInfo::PredicateInstruction(
   return false;
 }
 
-bool PPCInstrInfo::SubsumesPredicate(
-                     const SmallVectorImpl<MachineOperand> &Pred1,
-                     const SmallVectorImpl<MachineOperand> &Pred2) const {
+bool PPCInstrInfo::SubsumesPredicate(ArrayRef<MachineOperand> Pred1,
+                                     ArrayRef<MachineOperand> Pred2) const {
   assert(Pred1.size() == 2 && "Invalid PPC first predicate");
   assert(Pred2.size() == 2 && "Invalid PPC second predicate");
 
diff --git a/lib/Target/PowerPC/PPCInstrInfo.h b/lib/Target/PowerPC/PPCInstrInfo.h
index 7fd076a7d1c..0dab27c6792 100644
--- a/lib/Target/PowerPC/PPCInstrInfo.h
+++ b/lib/Target/PowerPC/PPCInstrInfo.h
@@ -63,6 +63,19 @@ enum PPC970_Unit {
 };
 } // end namespace PPCII
 
+namespace MachineCombinerPattern {
+enum MC_PATTERN : int {
+  // These are commutative variants for reassociating a computation chain
+  // of the form:
+  //   B = A op X (Prev)
+  //   C = B op Y (Root)
+  MC_REASSOC_AX_BY = 0,
+  MC_REASSOC_AX_YB = 1,
+  MC_REASSOC_XA_BY = 2,
+  MC_REASSOC_XA_YB = 3,
+};
+} // end namespace MachineCombinerPattern
+
 class PPCSubtarget;
 class PPCInstrInfo : public PPCGenInstrInfo {
   PPCSubtarget &Subtarget;
@@ -95,6 +108,10 @@ class PPCInstrInfo : public PPCGenInstrInfo {
   CreateTargetPostRAHazardRecognizer(const InstrItineraryData *II,
                                      const ScheduleDAG *DAG) const override;
 
+  unsigned getInstrLatency(const InstrItineraryData *ItinData,
+                           const MachineInstr *MI,
+                           unsigned *PredCost = nullptr) const override;
+
   int getOperandLatency(const InstrItineraryData *ItinData,
                         const MachineInstr *DefMI, unsigned DefIdx,
                         const MachineInstr *UseMI,
@@ -106,7 +123,7 @@ class PPCInstrInfo : public PPCGenInstrInfo {
                                               UseNode, UseIdx);
   }
 
-  bool hasLowDefLatency(const InstrItineraryData *ItinData,
+  bool hasLowDefLatency(const TargetSchedModel &SchedModel,
                         const MachineInstr *DefMI,
                         unsigned DefIdx) const override {
     // Machine LICM should hoist all instructions in low-register-pressure
@@ -115,6 +132,25 @@ class PPCInstrInfo : public PPCGenInstrInfo {
     return false;
   }
 
+  bool useMachineCombiner() const override {
+    return true;
+  }
+  
+  /// Return true when there is potentially a faster code sequence
+  /// for an instruction chain ending in <Root>. All potential patterns are
+  /// output in the <Pattern> array.
+  bool getMachineCombinerPatterns(
+      MachineInstr &Root,
+      SmallVectorImpl<MachineCombinerPattern::MC_PATTERN> &P) const override;
+  
+  /// When getMachineCombinerPatterns() finds a pattern, this function generates
+  /// the instructions that could replace the original code sequence.
+  void genAlternativeCodeSequence(
+          MachineInstr &Root, MachineCombinerPattern::MC_PATTERN P,
+          SmallVectorImpl<MachineInstr *> &InsInstrs,
+          SmallVectorImpl<MachineInstr *> &DelInstrs,
+          DenseMap<unsigned, unsigned> &InstrIdxForVirtReg) const override;
+
   bool isCoalescableExtInstr(const MachineInstr &MI,
                              unsigned &SrcReg, unsigned &DstReg,
                              unsigned &SubIdx) const override;
@@ -141,18 +177,14 @@ class PPCInstrInfo : public PPCGenInstrInfo {
                      bool AllowModify) const override;
   unsigned RemoveBranch(MachineBasicBlock &MBB) const override;
   unsigned InsertBranch(MachineBasicBlock &MBB, MachineBasicBlock *TBB,
-                        MachineBasicBlock *FBB,
-                        const SmallVectorImpl<MachineOperand> &Cond,
+                        MachineBasicBlock *FBB, ArrayRef<MachineOperand> Cond,
                         DebugLoc DL) const override;
 
   // Select analysis.
-  bool canInsertSelect(const MachineBasicBlock&,
-                       const SmallVectorImpl<MachineOperand> &Cond,
-                       unsigned, unsigned, int&, int&, int&) const override;
-  void insertSelect(MachineBasicBlock &MBB,
-                    MachineBasicBlock::iterator MI, DebugLoc DL,
-                    unsigned DstReg,
-                    const SmallVectorImpl<MachineOperand> &Cond,
+  bool canInsertSelect(const MachineBasicBlock &, ArrayRef<MachineOperand> Cond,
+                       unsigned, unsigned, int &, int &, int &) const override;
+  void insertSelect(MachineBasicBlock &MBB, MachineBasicBlock::iterator MI,
+                    DebugLoc DL, unsigned DstReg, ArrayRef<MachineOperand> Cond,
                     unsigned TrueReg, unsigned FalseReg) const override;
 
   void copyPhysReg(MachineBasicBlock &MBB,
@@ -211,10 +243,10 @@ class PPCInstrInfo : public PPCGenInstrInfo {
   bool isUnpredicatedTerminator(const MachineInstr *MI) const override;
 
   bool PredicateInstruction(MachineInstr *MI,
-                    const SmallVectorImpl<MachineOperand> &Pred) const override;
+                            ArrayRef<MachineOperand> Pred) const override;
 
-  bool SubsumesPredicate(const SmallVectorImpl<MachineOperand> &Pred1,
-                   const SmallVectorImpl<MachineOperand> &Pred2) const override;
+  bool SubsumesPredicate(ArrayRef<MachineOperand> Pred1,
+                         ArrayRef<MachineOperand> Pred2) const override;
 
   bool DefinesPredicate(MachineInstr *MI,
                         std::vector<MachineOperand> &Pred) const override;
diff --git a/lib/Target/PowerPC/PPCInstrInfo.td b/lib/Target/PowerPC/PPCInstrInfo.td
index 8aecb65b3fb..b50124db1ea 100644
--- a/lib/Target/PowerPC/PPCInstrInfo.td
+++ b/lib/Target/PowerPC/PPCInstrInfo.td
@@ -201,6 +201,12 @@ def SDT_PPCsc     : SDTypeProfile<0, 1, [SDTCisInt<0>]>;
 def PPCsc         : SDNode<"PPCISD::SC", SDT_PPCsc,
                            [SDNPHasChain, SDNPSideEffect]>;
 
+def PPCclrbhrb    : SDNode<"PPCISD::CLRBHRB", SDTNone,
+                           [SDNPHasChain, SDNPSideEffect]>;
+def PPCmfbhrbe    : SDNode<"PPCISD::MFBHRBE", SDTIntBinOp, [SDNPHasChain]>;
+def PPCrfebb      : SDNode<"PPCISD::RFEBB", SDT_PPCsc,
+                           [SDNPHasChain, SDNPSideEffect]>;
+
 def PPCvcmp       : SDNode<"PPCISD::VCMP" , SDT_PPCvcmp, []>;
 def PPCvcmp_o     : SDNode<"PPCISD::VCMPo", SDT_PPCvcmp, [SDNPOutGlue]>;
 
@@ -228,41 +234,42 @@ def PPCdynalloc   : SDNode<"PPCISD::DYNALLOC", SDTDynOp, [SDNPHasChain]>;
 
 def SHL32 : SDNodeXForm<imm, [{
   // Transformation function: 31 - imm
-  return getI32Imm(31 - N->getZExtValue());
+  return getI32Imm(31 - N->getZExtValue(), SDLoc(N));
 }]>;
 
 def SRL32 : SDNodeXForm<imm, [{
   // Transformation function: 32 - imm
-  return N->getZExtValue() ? getI32Imm(32 - N->getZExtValue()) : getI32Imm(0);
+  return N->getZExtValue() ? getI32Imm(32 - N->getZExtValue(), SDLoc(N))
+                           : getI32Imm(0, SDLoc(N));
 }]>;
 
 def LO16 : SDNodeXForm<imm, [{
   // Transformation function: get the low 16 bits.
-  return getI32Imm((unsigned short)N->getZExtValue());
+  return getI32Imm((unsigned short)N->getZExtValue(), SDLoc(N));
 }]>;
 
 def HI16 : SDNodeXForm<imm, [{
   // Transformation function: shift the immediate value down into the low bits.
-  return getI32Imm((unsigned)N->getZExtValue() >> 16);
+  return getI32Imm((unsigned)N->getZExtValue() >> 16, SDLoc(N));
 }]>;
 
 def HA16 : SDNodeXForm<imm, [{
   // Transformation function: shift the immediate value down into the low bits.
   signed int Val = N->getZExtValue();
-  return getI32Imm((Val - (signed short)Val) >> 16);
+  return getI32Imm((Val - (signed short)Val) >> 16, SDLoc(N));
 }]>;
 def MB : SDNodeXForm<imm, [{
   // Transformation function: get the start bit of a mask
   unsigned mb = 0, me;
   (void)isRunOfOnes((unsigned)N->getZExtValue(), mb, me);
-  return getI32Imm(mb);
+  return getI32Imm(mb, SDLoc(N));
 }]>;
 
 def ME : SDNodeXForm<imm, [{
   // Transformation function: get the end bit of a mask
   unsigned mb, me = 0;
   (void)isRunOfOnes((unsigned)N->getZExtValue(), mb, me);
-  return getI32Imm(me);
+  return getI32Imm(me, SDLoc(N));
 }]>;
 def maskimm32 : PatLeaf<(imm), [{
   // maskImm predicate - True if immediate is a run of ones.
@@ -498,6 +505,15 @@ def u6imm   : Operand<i32> {
   let ParserMatchClass = PPCU6ImmAsmOperand;
   let DecoderMethod = "decodeUImmOperand<6>";
 }
+def PPCU10ImmAsmOperand : AsmOperandClass {
+  let Name = "U10Imm"; let PredicateMethod = "isU10Imm";
+  let RenderMethod = "addImmOperands";
+}
+def u10imm  : Operand<i32> {
+  let PrintMethod = "printU10ImmOperand";
+  let ParserMatchClass = PPCU10ImmAsmOperand;
+  let DecoderMethod = "decodeUImmOperand<10>";
+}
 def PPCU12ImmAsmOperand : AsmOperandClass {
   let Name = "U12Imm"; let PredicateMethod = "isU12Imm";
   let RenderMethod = "addImmOperands";
@@ -1356,6 +1372,24 @@ let PPC970_Unit = 7 in {
                       "sc $lev", IIC_BrB, [(PPCsc (i32 imm:$lev))]>;
 }
 
+// Branch history rolling buffer.
+def CLRBHRB : XForm_0<31, 430, (outs), (ins), "clrbhrb", IIC_BrB,
+                      [(PPCclrbhrb)]>,
+                      PPC970_DGroup_Single;
+// The $dmy argument used for MFBHRBE is not needed; however, including
+// it avoids automatic generation of PPCFastISel::fastEmit_i(), which
+// interferes with necessary special handling (see PPCFastISel.cpp).
+def MFBHRBE : XFXForm_3p<31, 302, (outs gprc:$rD),
+                         (ins u10imm:$imm, u10imm:$dmy),
+                         "mfbhrbe $rD, $imm", IIC_BrB,
+                         [(set i32:$rD,
+                               (PPCmfbhrbe imm:$imm, imm:$dmy))]>,
+                         PPC970_DGroup_First;
+
+def RFEBB : XLForm_S<19, 146, (outs), (ins u1imm:$imm), "rfebb $imm",
+                     IIC_BrB, [(PPCrfebb (i32 imm:$imm))]>,
+                     PPC970_DGroup_Single;
+
 // DCB* instructions.
 def DCBA   : DCB_Form<758, 0, (outs), (ins memrr:$dst), "dcba $dst",
                       IIC_LdStDCBF, [(int_ppc_dcba xoaddr:$dst)]>,
@@ -1369,12 +1403,6 @@ def DCBI   : DCB_Form<470, 0, (outs), (ins memrr:$dst), "dcbi $dst",
 def DCBST  : DCB_Form<54, 0, (outs), (ins memrr:$dst), "dcbst $dst",
                       IIC_LdStDCBF, [(int_ppc_dcbst xoaddr:$dst)]>,
                       PPC970_DGroup_Single;
-def DCBT   : DCB_Form<278, 0, (outs), (ins memrr:$dst), "dcbt $dst",
-                      IIC_LdStDCBF, [(int_ppc_dcbt xoaddr:$dst)]>,
-                      PPC970_DGroup_Single;
-def DCBTST : DCB_Form<246, 0, (outs), (ins memrr:$dst), "dcbtst $dst",
-                      IIC_LdStDCBF, [(int_ppc_dcbtst xoaddr:$dst)]>,
-                      PPC970_DGroup_Single;
 def DCBZ   : DCB_Form<1014, 0, (outs), (ins memrr:$dst), "dcbz $dst",
                       IIC_LdStDCBF, [(int_ppc_dcbz xoaddr:$dst)]>,
                       PPC970_DGroup_Single;
@@ -1382,13 +1410,27 @@ def DCBZL  : DCB_Form<1014, 1, (outs), (ins memrr:$dst), "dcbzl $dst",
                       IIC_LdStDCBF, [(int_ppc_dcbzl xoaddr:$dst)]>,
                       PPC970_DGroup_Single;
 
+let hasSideEffects = 0, mayLoad = 1, mayStore = 1 in {
+def DCBT   : DCB_Form_hint<278, (outs), (ins u5imm:$TH, memrr:$dst),
+                      "dcbt $dst, $TH", IIC_LdStDCBF, []>,
+                      PPC970_DGroup_Single;
+def DCBTST : DCB_Form_hint<246, (outs), (ins u5imm:$TH, memrr:$dst),
+                      "dcbtst $dst, $TH", IIC_LdStDCBF, []>,
+                      PPC970_DGroup_Single;
+} // hasSideEffects = 0
+
 def ICBT  : XForm_icbt<31, 22, (outs), (ins u4imm:$CT, memrr:$src),
                        "icbt $CT, $src", IIC_LdStLoad>, Requires<[HasICBT]>;
 
+def : Pat<(int_ppc_dcbt xoaddr:$dst),
+          (DCBT 0, xoaddr:$dst)>;
+def : Pat<(int_ppc_dcbtst xoaddr:$dst),
+          (DCBTST 0, xoaddr:$dst)>;
+
 def : Pat<(prefetch xoaddr:$dst, (i32 0), imm, (i32 1)),
-          (DCBT xoaddr:$dst)>;   // data prefetch for loads
+          (DCBT 0, xoaddr:$dst)>;   // data prefetch for loads
 def : Pat<(prefetch xoaddr:$dst, (i32 1), imm, (i32 1)),
-          (DCBTST xoaddr:$dst)>; // data prefetch for stores
+          (DCBTST 0, xoaddr:$dst)>; // data prefetch for stores
 def : Pat<(prefetch xoaddr:$dst, (i32 0), imm, (i32 0)),
           (ICBT 0, xoaddr:$dst)>, Requires<[HasICBT]>; // inst prefetch (for read)
 
@@ -2183,7 +2225,7 @@ def MTSPR : XFXForm_1<31, 467, (outs), (ins i32imm:$SPR, gprc:$RT),
                       "mtspr $SPR, $RT", IIC_SprMTSPR>;
 
 def MFTB : XFXForm_1<31, 371, (outs gprc:$RT), (ins i32imm:$SPR),
-                     "mftb $RT, $SPR", IIC_SprMFTB>, Deprecated<DeprecatedMFTB>;
+                     "mftb $RT, $SPR", IIC_SprMFTB>;
 
 // A pseudo-instruction used to implement the read of the 64-bit cycle counter
 // on a 32-bit target.
@@ -3476,7 +3518,7 @@ class PPCAsmPseudo<string asm, dag iops>
 def : InstAlias<"sc", (SC 0)>;
 
 def : InstAlias<"sync", (SYNC 0)>, Requires<[HasSYNC]>;
-def : InstAlias<"msync", (SYNC 0)>, Requires<[HasSYNC]>;
+def : InstAlias<"msync", (SYNC 0), 0>, Requires<[HasSYNC]>;
 def : InstAlias<"lwsync", (SYNC 1)>, Requires<[HasSYNC]>;
 def : InstAlias<"ptesync", (SYNC 2)>, Requires<[HasSYNC]>;
 
@@ -3486,6 +3528,17 @@ def : InstAlias<"waitimpl", (WAIT 2)>;
 
 def : InstAlias<"mbar", (MBAR 0)>, Requires<[IsBookE]>;
 
+def DCBTx   : PPCAsmPseudo<"dcbt $dst", (ins memrr:$dst)>;
+def DCBTSTx : PPCAsmPseudo<"dcbtst $dst", (ins memrr:$dst)>;
+
+def DCBTCT : PPCAsmPseudo<"dcbtct $dst, $TH", (ins memrr:$dst, u5imm:$TH)>;
+def DCBTDS : PPCAsmPseudo<"dcbtds $dst, $TH", (ins memrr:$dst, u5imm:$TH)>;
+def DCBTT  : PPCAsmPseudo<"dcbtt $dst", (ins memrr:$dst)>;
+
+def DCBTSTCT : PPCAsmPseudo<"dcbtstct $dst, $TH", (ins memrr:$dst, u5imm:$TH)>;
+def DCBTSTDS : PPCAsmPseudo<"dcbtstds $dst, $TH", (ins memrr:$dst, u5imm:$TH)>;
+def DCBTSTT  : PPCAsmPseudo<"dcbtstt $dst", (ins memrr:$dst)>;
+
 def : InstAlias<"crset $bx", (CREQV crbitrc:$bx, crbitrc:$bx, crbitrc:$bx)>;
 def : InstAlias<"crclr $bx", (CRXOR crbitrc:$bx, crbitrc:$bx, crbitrc:$bx)>;
 def : InstAlias<"crmove $bx, $by", (CROR crbitrc:$bx, crbitrc:$by, crbitrc:$by)>;
diff --git a/lib/Target/PowerPC/PPCInstrVSX.td b/lib/Target/PowerPC/PPCInstrVSX.td
index a98e58f18ca..20c95fe888e 100644
--- a/lib/Target/PowerPC/PPCInstrVSX.td
+++ b/lib/Target/PowerPC/PPCInstrVSX.td
@@ -11,6 +11,21 @@
 //
 //===----------------------------------------------------------------------===//
 
+// *********************************** NOTE ***********************************
+// ** For POWER8 Little Endian, the VSX swap optimization relies on knowing  **
+// ** which VMX and VSX instructions are lane-sensitive and which are not.   **
+// ** A lane-sensitive instruction relies, implicitly or explicitly, on      **
+// ** whether lanes are numbered from left to right.  An instruction like    **
+// ** VADDFP is not lane-sensitive, because each lane of the result vector   **
+// ** relies only on the corresponding lane of the source vectors.  However, **
+// ** an instruction like VMULESB is lane-sensitive, because "even" and      **
+// ** "odd" lanes are different for big-endian and little-endian numbering.  **
+// **                                                                        **
+// ** When adding new VMX and VSX instructions, please consider whether they **
+// ** are lane-sensitive.  If so, they must be added to a switch statement   **
+// ** in PPCVSXSwapRemoval::gatherVectorInstructions().                      **
+// ****************************************************************************
+
 def PPCRegVSRCAsmOperand : AsmOperandClass {
   let Name = "RegVSRC"; let PredicateMethod = "isVSRegNumber";
 }
@@ -25,6 +40,13 @@ def vsfrc : RegisterOperand<VSFRC> {
   let ParserMatchClass = PPCRegVSFRCAsmOperand;
 }
 
+def PPCRegVSSRCAsmOperand : AsmOperandClass {
+  let Name = "RegVSSRC"; let PredicateMethod = "isVSRegNumber";
+}
+def vssrc : RegisterOperand<VSSRC> {
+  let ParserMatchClass = PPCRegVSSRCAsmOperand;
+}
+
 // Little-endian-specific nodes.
 def SDT_PPClxvd2x : SDTypeProfile<1, 1, [
   SDTCisVT<0, v2f64>, SDTCisPtrTy<1>
@@ -88,7 +110,7 @@ let Uses = [RM] in {
                          (outs vsrc:$XT), (ins memrr:$src),
                          "lxvw4x $XT, $src", IIC_LdStLFD,
                          [(set v4i32:$XT, (int_ppc_vsx_lxvw4x xoaddr:$src))]>;
-  }
+  } // mayLoad
 
   // Store indexed instructions
   let mayStore = 1 in {
@@ -106,7 +128,8 @@ let Uses = [RM] in {
                          (outs), (ins vsrc:$XT, memrr:$dst),
                          "stxvw4x $XT, $dst", IIC_LdStSTFD,
                          [(store v4i32:$XT, xoaddr:$dst)]>;
-  }
+
+  } // mayStore
 
   // Add/Mul Instructions
   let isCommutable = 1 in {
@@ -434,22 +457,34 @@ let Uses = [RM] in {
 
   defm XVCMPEQDP : XX3Form_Rcr<60, 99,
                              (outs vsrc:$XT), (ins vsrc:$XA, vsrc:$XB),
-                             "xvcmpeqdp", "$XT, $XA, $XB", IIC_VecFPCompare, []>;
+                             "xvcmpeqdp", "$XT, $XA, $XB", IIC_VecFPCompare,
+                             [(set v2i64:$XT,
+                                (int_ppc_vsx_xvcmpeqdp v2f64:$XA, v2f64:$XB))]>;
   defm XVCMPEQSP : XX3Form_Rcr<60, 67,
                              (outs vsrc:$XT), (ins vsrc:$XA, vsrc:$XB),
-                             "xvcmpeqsp", "$XT, $XA, $XB", IIC_VecFPCompare, []>;
+                             "xvcmpeqsp", "$XT, $XA, $XB", IIC_VecFPCompare,
+                             [(set v4i32:$XT,
+                                (int_ppc_vsx_xvcmpeqsp v4f32:$XA, v4f32:$XB))]>;
   defm XVCMPGEDP : XX3Form_Rcr<60, 115,
                              (outs vsrc:$XT), (ins vsrc:$XA, vsrc:$XB),
-                             "xvcmpgedp", "$XT, $XA, $XB", IIC_VecFPCompare, []>;
+                             "xvcmpgedp", "$XT, $XA, $XB", IIC_VecFPCompare,
+                             [(set v2i64:$XT,
+                                (int_ppc_vsx_xvcmpgedp v2f64:$XA, v2f64:$XB))]>;
   defm XVCMPGESP : XX3Form_Rcr<60, 83,
                              (outs vsrc:$XT), (ins vsrc:$XA, vsrc:$XB),
-                             "xvcmpgesp", "$XT, $XA, $XB", IIC_VecFPCompare, []>;
+                             "xvcmpgesp", "$XT, $XA, $XB", IIC_VecFPCompare,
+                             [(set v4i32:$XT,
+                                (int_ppc_vsx_xvcmpgesp v4f32:$XA, v4f32:$XB))]>;
   defm XVCMPGTDP : XX3Form_Rcr<60, 107,
                              (outs vsrc:$XT), (ins vsrc:$XA, vsrc:$XB),
-                             "xvcmpgtdp", "$XT, $XA, $XB", IIC_VecFPCompare, []>;
+                             "xvcmpgtdp", "$XT, $XA, $XB", IIC_VecFPCompare,
+                             [(set v2i64:$XT,
+                                (int_ppc_vsx_xvcmpgtdp v2f64:$XA, v2f64:$XB))]>;
   defm XVCMPGTSP : XX3Form_Rcr<60, 75,
                              (outs vsrc:$XT), (ins vsrc:$XA, vsrc:$XB),
-                             "xvcmpgtsp", "$XT, $XA, $XB", IIC_VecFPCompare, []>;
+                             "xvcmpgtsp", "$XT, $XA, $XB", IIC_VecFPCompare,
+                             [(set v4i32:$XT,
+                                (int_ppc_vsx_xvcmpgtsp v4f32:$XA, v4f32:$XB))]>;
 
   // Move Instructions
   def XSABSDP : XX2Form<60, 345,
@@ -776,6 +811,15 @@ let usesCustomInserter = 1,    // Expanded after instruction selection.
                            "#SELECT_VSFRC",
                            [(set f64:$dst,
                                  (select i1:$cond, f64:$T, f64:$F))]>;
+  def SELECT_CC_VSSRC: Pseudo<(outs f4rc:$dst),
+                              (ins crrc:$cond, f4rc:$T, f4rc:$F,
+                               i32imm:$BROPC), "#SELECT_CC_VSSRC",
+                              []>;
+  def SELECT_VSSRC: Pseudo<(outs f4rc:$dst),
+                           (ins crbitrc:$cond, f4rc:$T, f4rc:$F),
+                           "#SELECT_VSSRC",
+                           [(set f32:$dst,
+                                 (select i1:$cond, f32:$T, f32:$F))]>;
 } // usesCustomInserter
 } // AddedComplexity
 
@@ -875,6 +919,11 @@ def : Pat<(v2f64 (bitconvert v2i64:$A)),
 def : Pat<(v2i64 (bitconvert v2f64:$A)),
           (COPY_TO_REGCLASS $A, VRRC)>;
 
+def : Pat<(v2f64 (bitconvert v1i128:$A)),
+          (COPY_TO_REGCLASS $A, VRRC)>;
+def : Pat<(v1i128 (bitconvert v2f64:$A)),
+          (COPY_TO_REGCLASS $A, VRRC)>;
+
 // sign extension patterns
 // To extend "in place" from v2i32 to v2i64, we have input data like:
 // | undef | i32 | undef | i32 |
@@ -940,6 +989,18 @@ def : Pat<(int_ppc_vsx_xvdivsp v4f32:$A, v4f32:$B),
 def : Pat<(int_ppc_vsx_xvdivdp v2f64:$A, v2f64:$B),
           (XVDIVDP $A, $B)>;
 
+// Reciprocal estimate
+def : Pat<(int_ppc_vsx_xvresp v4f32:$A),
+          (XVRESP $A)>;
+def : Pat<(int_ppc_vsx_xvredp v2f64:$A),
+          (XVREDP $A)>;
+
+// Recip. square root estimate
+def : Pat<(int_ppc_vsx_xvrsqrtesp v4f32:$A),
+          (XVRSQRTESP $A)>;
+def : Pat<(int_ppc_vsx_xvrsqrtedp v2f64:$A),
+          (XVRSQRTEDP $A)>;
+
 } // AddedComplexity
 } // HasVSX
 
@@ -952,41 +1013,194 @@ def HasP8Vector : Predicate<"PPCSubTarget->hasP8Vector()">;
 def HasDirectMove : Predicate<"PPCSubTarget->hasDirectMove()">;
 let Predicates = [HasP8Vector] in {
 let AddedComplexity = 400 in { // Prefer VSX patterns over non-VSX patterns.
-let isCommutable = 1 in {
-  def XXLEQV : XX3Form<60, 186,
-                       (outs vsrc:$XT), (ins vsrc:$XA, vsrc:$XB),
-                       "xxleqv $XT, $XA, $XB", IIC_VecGeneral,
-                       [(set v4i32:$XT, (vnot_ppc (xor v4i32:$XA, v4i32:$XB)))]>;
-  def XXLNAND : XX3Form<60, 178,
-                        (outs vsrc:$XT), (ins vsrc:$XA, vsrc:$XB),
-                        "xxlnand $XT, $XA, $XB", IIC_VecGeneral,
-                        [(set v4i32:$XT, (vnot_ppc (and v4i32:$XA,
+  let isCommutable = 1 in {
+    def XXLEQV : XX3Form<60, 186,
+                         (outs vsrc:$XT), (ins vsrc:$XA, vsrc:$XB),
+                         "xxleqv $XT, $XA, $XB", IIC_VecGeneral,
+                         [(set v4i32:$XT, (vnot_ppc (xor v4i32:$XA, v4i32:$XB)))]>;
+    def XXLNAND : XX3Form<60, 178,
+                          (outs vsrc:$XT), (ins vsrc:$XA, vsrc:$XB),
+                          "xxlnand $XT, $XA, $XB", IIC_VecGeneral,
+                          [(set v4i32:$XT, (vnot_ppc (and v4i32:$XA,
                                                     v4i32:$XB)))]>;
   } // isCommutable
-def XXLORC : XX3Form<60, 170,
-                     (outs vsrc:$XT), (ins vsrc:$XA, vsrc:$XB),
-                     "xxlorc $XT, $XA, $XB", IIC_VecGeneral,
-                     [(set v4i32:$XT, (or v4i32:$XA, (vnot_ppc v4i32:$XB)))]>;
-} // AddedComplexity = 500
+
+  def : Pat<(int_ppc_vsx_xxleqv v4i32:$A, v4i32:$B),
+            (XXLEQV $A, $B)>;
+
+  def XXLORC : XX3Form<60, 170,
+                       (outs vsrc:$XT), (ins vsrc:$XA, vsrc:$XB),
+                       "xxlorc $XT, $XA, $XB", IIC_VecGeneral,
+                       [(set v4i32:$XT, (or v4i32:$XA, (vnot_ppc v4i32:$XB)))]>;
+
+  // VSX scalar loads introduced in ISA 2.07
+  let mayLoad = 1 in {
+    def LXSSPX : XX1Form<31, 524, (outs vssrc:$XT), (ins memrr:$src),
+                         "lxsspx $XT, $src", IIC_LdStLFD,
+                         [(set f32:$XT, (load xoaddr:$src))]>;
+    def LXSIWAX : XX1Form<31, 76, (outs vsfrc:$XT), (ins memrr:$src),
+                          "lxsiwax $XT, $src", IIC_LdStLFD,
+                          [(set f64:$XT, (PPClfiwax xoaddr:$src))]>;
+    def LXSIWZX : XX1Form<31, 12, (outs vsfrc:$XT), (ins memrr:$src),
+                          "lxsiwzx $XT, $src", IIC_LdStLFD,
+                          [(set f64:$XT, (PPClfiwzx xoaddr:$src))]>;
+  } // mayLoad
+
+  // VSX scalar stores introduced in ISA 2.07
+  let mayStore = 1 in {
+    def STXSSPX : XX1Form<31, 652, (outs), (ins vssrc:$XT, memrr:$dst),
+                          "stxsspx $XT, $dst", IIC_LdStSTFD,
+                          [(store f32:$XT, xoaddr:$dst)]>;
+    def STXSIWX : XX1Form<31, 140, (outs), (ins vsfrc:$XT, memrr:$dst),
+                          "stxsiwx $XT, $dst", IIC_LdStSTFD,
+                          [(PPCstfiwx f64:$XT, xoaddr:$dst)]>;
+  } // mayStore
+
+  def : Pat<(f64 (extloadf32 xoaddr:$src)),
+            (COPY_TO_REGCLASS (LXSSPX xoaddr:$src), VSFRC)>;
+  def : Pat<(f64 (fextend f32:$src)),
+            (COPY_TO_REGCLASS $src, VSFRC)>;
+  def : Pat<(f32 (selectcc i1:$lhs, i1:$rhs, f32:$tval, f32:$fval, SETLT)),
+            (SELECT_VSSRC (CRANDC $rhs, $lhs), $tval, $fval)>;
+  def : Pat<(f32 (selectcc i1:$lhs, i1:$rhs, f32:$tval, f32:$fval, SETLE)),
+            (SELECT_VSSRC (CRORC  $rhs, $lhs), $tval, $fval)>;
+  def : Pat<(f32 (selectcc i1:$lhs, i1:$rhs, f32:$tval, f32:$fval, SETEQ)),
+            (SELECT_VSSRC (CREQV $lhs, $rhs), $tval, $fval)>;
+  def : Pat<(f32 (selectcc i1:$lhs, i1:$rhs, f32:$tval, f32:$fval, SETGE)),
+            (SELECT_VSSRC (CRORC  $lhs, $rhs), $tval, $fval)>;
+  def : Pat<(f32 (selectcc i1:$lhs, i1:$rhs, f32:$tval, f32:$fval, SETGT)),
+            (SELECT_VSSRC (CRANDC $lhs, $rhs), $tval, $fval)>;
+  def : Pat<(f32 (selectcc i1:$lhs, i1:$rhs, f32:$tval, f32:$fval, SETNE)),
+          (SELECT_VSSRC (CRXOR $lhs, $rhs), $tval, $fval)>;
+
+  // VSX Elementary Scalar FP arithmetic (SP)
+  let isCommutable = 1 in {
+    def XSADDSP : XX3Form<60, 0,
+                          (outs vssrc:$XT), (ins vssrc:$XA, vssrc:$XB),
+                          "xsaddsp $XT, $XA, $XB", IIC_VecFP,
+                          [(set f32:$XT, (fadd f32:$XA, f32:$XB))]>;
+    def XSMULSP : XX3Form<60, 16,
+                          (outs vssrc:$XT), (ins vssrc:$XA, vssrc:$XB),
+                          "xsmulsp $XT, $XA, $XB", IIC_VecFP,
+                          [(set f32:$XT, (fmul f32:$XA, f32:$XB))]>;
+  } // isCommutable
+
+  def XSDIVSP : XX3Form<60, 24,
+                        (outs vssrc:$XT), (ins vssrc:$XA, vssrc:$XB),
+                        "xsdivsp $XT, $XA, $XB", IIC_FPDivS,
+                        [(set f32:$XT, (fdiv f32:$XA, f32:$XB))]>;
+  def XSRESP : XX2Form<60, 26,
+                        (outs vssrc:$XT), (ins vssrc:$XB),
+                        "xsresp $XT, $XB", IIC_VecFP,
+                        [(set f32:$XT, (PPCfre f32:$XB))]>;
+  def XSSQRTSP : XX2Form<60, 11,
+                        (outs vssrc:$XT), (ins vssrc:$XB),
+                        "xssqrtsp $XT, $XB", IIC_FPSqrtS,
+                        [(set f32:$XT, (fsqrt f32:$XB))]>;
+  def XSRSQRTESP : XX2Form<60, 10,
+                           (outs vssrc:$XT), (ins vssrc:$XB),
+                           "xsrsqrtesp $XT, $XB", IIC_VecFP,
+                           [(set f32:$XT, (PPCfrsqrte f32:$XB))]>;
+  def XSSUBSP : XX3Form<60, 8,
+                        (outs vssrc:$XT), (ins vssrc:$XA, vssrc:$XB),
+                        "xssubsp $XT, $XA, $XB", IIC_VecFP,
+                        [(set f32:$XT, (fsub f32:$XA, f32:$XB))]>;
+
+  // FMA Instructions
+  let BaseName = "XSMADDASP" in {
+  let isCommutable = 1 in
+  def XSMADDASP : XX3Form<60, 1,
+                          (outs vssrc:$XT),
+                          (ins vssrc:$XTi, vssrc:$XA, vssrc:$XB),
+                          "xsmaddasp $XT, $XA, $XB", IIC_VecFP,
+                          [(set f32:$XT, (fma f32:$XA, f32:$XB, f32:$XTi))]>,
+                          RegConstraint<"$XTi = $XT">, NoEncode<"$XTi">,
+                          AltVSXFMARel;
+  let IsVSXFMAAlt = 1 in
+  def XSMADDMSP : XX3Form<60, 9,
+                          (outs vssrc:$XT),
+                          (ins vssrc:$XTi, vssrc:$XA, vssrc:$XB),
+                          "xsmaddmsp $XT, $XA, $XB", IIC_VecFP, []>,
+                          RegConstraint<"$XTi = $XT">, NoEncode<"$XTi">,
+                          AltVSXFMARel;
+  }
+
+  let BaseName = "XSMSUBASP" in {
+  let isCommutable = 1 in
+  def XSMSUBASP : XX3Form<60, 17,
+                          (outs vssrc:$XT),
+                          (ins vssrc:$XTi, vssrc:$XA, vssrc:$XB),
+                          "xsmsubasp $XT, $XA, $XB", IIC_VecFP,
+                          [(set f32:$XT, (fma f32:$XA, f32:$XB,
+                                              (fneg f32:$XTi)))]>,
+                          RegConstraint<"$XTi = $XT">, NoEncode<"$XTi">,
+                          AltVSXFMARel;
+  let IsVSXFMAAlt = 1 in
+  def XSMSUBMSP : XX3Form<60, 25,
+                          (outs vssrc:$XT),
+                          (ins vssrc:$XTi, vssrc:$XA, vssrc:$XB),
+                          "xsmsubmsp $XT, $XA, $XB", IIC_VecFP, []>,
+                          RegConstraint<"$XTi = $XT">, NoEncode<"$XTi">,
+                          AltVSXFMARel;
+  }
+
+  let BaseName = "XSNMADDASP" in {
+  let isCommutable = 1 in
+  def XSNMADDASP : XX3Form<60, 129,
+                          (outs vssrc:$XT),
+                          (ins vssrc:$XTi, vssrc:$XA, vssrc:$XB),
+                          "xsnmaddasp $XT, $XA, $XB", IIC_VecFP,
+                          [(set f32:$XT, (fneg (fma f32:$XA, f32:$XB,
+                                                    f32:$XTi)))]>,
+                          RegConstraint<"$XTi = $XT">, NoEncode<"$XTi">,
+                          AltVSXFMARel;
+  let IsVSXFMAAlt = 1 in
+  def XSNMADDMSP : XX3Form<60, 137,
+                          (outs vssrc:$XT),
+                          (ins vssrc:$XTi, vssrc:$XA, vssrc:$XB),
+                          "xsnmaddmsp $XT, $XA, $XB", IIC_VecFP, []>,
+                          RegConstraint<"$XTi = $XT">, NoEncode<"$XTi">,
+                          AltVSXFMARel;
+  }
+
+  let BaseName = "XSNMSUBASP" in {
+  let isCommutable = 1 in
+  def XSNMSUBASP : XX3Form<60, 145,
+                          (outs vssrc:$XT),
+                          (ins vssrc:$XTi, vssrc:$XA, vssrc:$XB),
+                          "xsnmsubasp $XT, $XA, $XB", IIC_VecFP,
+                          [(set f32:$XT, (fneg (fma f32:$XA, f32:$XB,
+                                                    (fneg f32:$XTi))))]>,
+                          RegConstraint<"$XTi = $XT">, NoEncode<"$XTi">,
+                          AltVSXFMARel;
+  let IsVSXFMAAlt = 1 in
+  def XSNMSUBMSP : XX3Form<60, 153,
+                          (outs vssrc:$XT),
+                          (ins vssrc:$XTi, vssrc:$XA, vssrc:$XB),
+                          "xsnmsubmsp $XT, $XA, $XB", IIC_VecFP, []>,
+                          RegConstraint<"$XTi = $XT">, NoEncode<"$XTi">,
+                          AltVSXFMARel;
+  }
+} // AddedComplexity = 400
 } // HasP8Vector
 
 let Predicates = [HasDirectMove, HasVSX] in {
-// VSX direct move instructions
-def MFVSRD : XX1_RS6_RD5_XO<31, 51, (outs g8rc:$rA), (ins vsfrc:$XT),
-                            "mfvsrd $rA, $XT", IIC_VecGeneral,
-                            [(set i64:$rA, (PPCmfvsr f64:$XT))]>,
-    Requires<[In64BitMode]>;
-def MFVSRWZ : XX1_RS6_RD5_XO<31, 115, (outs gprc:$rA), (ins vsfrc:$XT),
-                             "mfvsrwz $rA, $XT", IIC_VecGeneral,
-                             [(set i32:$rA, (PPCmfvsr f64:$XT))]>;
-def MTVSRD : XX1_RS6_RD5_XO<31, 179, (outs vsfrc:$XT), (ins g8rc:$rA),
-                            "mtvsrd $XT, $rA", IIC_VecGeneral,
-                            [(set f64:$XT, (PPCmtvsra i64:$rA))]>,
-    Requires<[In64BitMode]>;
-def MTVSRWA : XX1_RS6_RD5_XO<31, 211, (outs vsfrc:$XT), (ins gprc:$rA),
-                             "mtvsrwa $XT, $rA", IIC_VecGeneral,
-                             [(set f64:$XT, (PPCmtvsra i32:$rA))]>;
-def MTVSRWZ : XX1_RS6_RD5_XO<31, 243, (outs vsfrc:$XT), (ins gprc:$rA),
-                             "mtvsrwz $XT, $rA", IIC_VecGeneral,
-                             [(set f64:$XT, (PPCmtvsrz i32:$rA))]>;
+  // VSX direct move instructions
+  def MFVSRD : XX1_RS6_RD5_XO<31, 51, (outs g8rc:$rA), (ins vsfrc:$XT),
+                              "mfvsrd $rA, $XT", IIC_VecGeneral,
+                              [(set i64:$rA, (PPCmfvsr f64:$XT))]>,
+      Requires<[In64BitMode]>;
+  def MFVSRWZ : XX1_RS6_RD5_XO<31, 115, (outs gprc:$rA), (ins vsfrc:$XT),
+                               "mfvsrwz $rA, $XT", IIC_VecGeneral,
+                               [(set i32:$rA, (PPCmfvsr f64:$XT))]>;
+  def MTVSRD : XX1_RS6_RD5_XO<31, 179, (outs vsfrc:$XT), (ins g8rc:$rA),
+                              "mtvsrd $XT, $rA", IIC_VecGeneral,
+                              [(set f64:$XT, (PPCmtvsra i64:$rA))]>,
+      Requires<[In64BitMode]>;
+  def MTVSRWA : XX1_RS6_RD5_XO<31, 211, (outs vsfrc:$XT), (ins gprc:$rA),
+                               "mtvsrwa $XT, $rA", IIC_VecGeneral,
+                               [(set f64:$XT, (PPCmtvsra i32:$rA))]>;
+  def MTVSRWZ : XX1_RS6_RD5_XO<31, 243, (outs vsfrc:$XT), (ins gprc:$rA),
+                               "mtvsrwz $XT, $rA", IIC_VecGeneral,
+                               [(set f64:$XT, (PPCmtvsrz i32:$rA))]>;
 } // HasDirectMove, HasVSX
diff --git a/lib/Target/PowerPC/PPCLoopDataPrefetch.cpp b/lib/Target/PowerPC/PPCLoopDataPrefetch.cpp
index 2947c6617bd..b4e1c099f19 100644
--- a/lib/Target/PowerPC/PPCLoopDataPrefetch.cpp
+++ b/lib/Target/PowerPC/PPCLoopDataPrefetch.cpp
@@ -217,9 +217,11 @@ bool PPCLoopDataPrefetch::runOnLoop(Loop *L) {
       Module *M = (*I)->getParent()->getParent();
       Type *I32 = Type::getInt32Ty((*I)->getContext());
       Value *PrefetchFunc = Intrinsic::getDeclaration(M, Intrinsic::prefetch);
-      Builder.CreateCall4(PrefetchFunc, PrefPtrValue,
-        ConstantInt::get(I32, MemI->mayReadFromMemory() ? 0 : 1),
-        ConstantInt::get(I32, 3), ConstantInt::get(I32, 1));
+      Builder.CreateCall(
+          PrefetchFunc,
+          {PrefPtrValue,
+           ConstantInt::get(I32, MemI->mayReadFromMemory() ? 0 : 1),
+           ConstantInt::get(I32, 3), ConstantInt::get(I32, 1)});
 
       MadeChange = true;
     }
diff --git a/lib/Target/PowerPC/PPCMCInstLower.cpp b/lib/Target/PowerPC/PPCMCInstLower.cpp
index 6df89fe2c7d..44a692d4bb4 100644
--- a/lib/Target/PowerPC/PPCMCInstLower.cpp
+++ b/lib/Target/PowerPC/PPCMCInstLower.cpp
@@ -38,9 +38,9 @@ static MachineModuleInfoMachO &getMachOMMI(AsmPrinter &AP) {
 static MCSymbol *GetSymbolFromOperand(const MachineOperand &MO, AsmPrinter &AP){
   const TargetMachine &TM = AP.TM;
   Mangler *Mang = AP.Mang;
-  const DataLayout *DL = TM.getDataLayout();
+  const DataLayout &DL = AP.getDataLayout();
   MCContext &Ctx = AP.OutContext;
-  bool isDarwin = Triple(TM.getTargetTriple()).isOSDarwin();
+  bool isDarwin = TM.getTargetTriple().isOSDarwin();
 
   SmallString<128> Name;
   StringRef Suffix;
@@ -51,13 +51,13 @@ static MCSymbol *GetSymbolFromOperand(const MachineOperand &MO, AsmPrinter &AP){
     Suffix = "$non_lazy_ptr";
 
   if (!Suffix.empty())
-    Name += DL->getPrivateGlobalPrefix();
+    Name += DL.getPrivateGlobalPrefix();
 
   unsigned PrefixLen = Name.size();
 
   if (!MO.isGlobal()) {
     assert(MO.isSymbol() && "Isn't a symbol reference");
-    Mang->getNameWithPrefix(Name, MO.getSymbolName());
+    Mangler::getNameWithPrefix(Name, MO.getSymbolName(), DL);
   } else {
     const GlobalValue *GV = MO.getGlobal();
     TM.getNameWithPrefix(Name, GV, *Mang);
@@ -66,7 +66,7 @@ static MCSymbol *GetSymbolFromOperand(const MachineOperand &MO, AsmPrinter &AP){
   unsigned OrigLen = Name.size() - PrefixLen;
 
   Name += Suffix;
-  MCSymbol *Sym = Ctx.GetOrCreateSymbol(Name);
+  MCSymbol *Sym = Ctx.getOrCreateSymbol(Name);
   StringRef OrigName = StringRef(Name).substr(PrefixLen, OrigLen);
 
   // If the target flags on the operand changes the name of the symbol, do that
@@ -85,7 +85,7 @@ static MCSymbol *GetSymbolFromOperand(const MachineOperand &MO, AsmPrinter &AP){
     } else {
       StubSym =
       MachineModuleInfoImpl::
-      StubValueTy(Ctx.GetOrCreateSymbol(OrigName), false);
+      StubValueTy(Ctx.getOrCreateSymbol(OrigName), false);
     }
     return Sym;
   }
@@ -142,32 +142,32 @@ static MCOperand GetSymbolRef(const MachineOperand &MO, const MCSymbol *Symbol,
   if (MO.getTargetFlags() == PPCII::MO_PLT_OR_STUB && !isDarwin)
     RefKind = MCSymbolRefExpr::VK_PLT;
 
-  const MCExpr *Expr = MCSymbolRefExpr::Create(Symbol, RefKind, Ctx);
+  const MCExpr *Expr = MCSymbolRefExpr::create(Symbol, RefKind, Ctx);
 
   if (!MO.isJTI() && MO.getOffset())
-    Expr = MCBinaryExpr::CreateAdd(Expr,
-                                   MCConstantExpr::Create(MO.getOffset(), Ctx),
+    Expr = MCBinaryExpr::createAdd(Expr,
+                                   MCConstantExpr::create(MO.getOffset(), Ctx),
                                    Ctx);
 
   // Subtract off the PIC base if required.
   if (MO.getTargetFlags() & PPCII::MO_PIC_FLAG) {
     const MachineFunction *MF = MO.getParent()->getParent()->getParent();
     
-    const MCExpr *PB = MCSymbolRefExpr::Create(MF->getPICBaseSymbol(), Ctx);
-    Expr = MCBinaryExpr::CreateSub(Expr, PB, Ctx);
+    const MCExpr *PB = MCSymbolRefExpr::create(MF->getPICBaseSymbol(), Ctx);
+    Expr = MCBinaryExpr::createSub(Expr, PB, Ctx);
   }
 
   // Add ha16() / lo16() markers if required.
   switch (access) {
     case PPCII::MO_LO:
-      Expr = PPCMCExpr::CreateLo(Expr, isDarwin, Ctx);
+      Expr = PPCMCExpr::createLo(Expr, isDarwin, Ctx);
       break;
     case PPCII::MO_HA:
-      Expr = PPCMCExpr::CreateHa(Expr, isDarwin, Ctx);
+      Expr = PPCMCExpr::createHa(Expr, isDarwin, Ctx);
       break;
   }
 
-  return MCOperand::CreateExpr(Expr);
+  return MCOperand::createExpr(Expr);
 }
 
 void llvm::LowerPPCMachineInstrToMCInst(const MachineInstr *MI, MCInst &OutMI,
@@ -187,13 +187,13 @@ void llvm::LowerPPCMachineInstrToMCInst(const MachineInstr *MI, MCInst &OutMI,
       assert(MO.getReg() > PPC::NoRegister &&
              MO.getReg() < PPC::NUM_TARGET_REGS &&
              "Invalid register for this target!");
-      MCOp = MCOperand::CreateReg(MO.getReg());
+      MCOp = MCOperand::createReg(MO.getReg());
       break;
     case MachineOperand::MO_Immediate:
-      MCOp = MCOperand::CreateImm(MO.getImm());
+      MCOp = MCOperand::createImm(MO.getImm());
       break;
     case MachineOperand::MO_MachineBasicBlock:
-      MCOp = MCOperand::CreateExpr(MCSymbolRefExpr::Create(
+      MCOp = MCOperand::createExpr(MCSymbolRefExpr::create(
                                       MO.getMBB()->getSymbol(), AP.OutContext));
       break;
     case MachineOperand::MO_GlobalAddress:
diff --git a/lib/Target/PowerPC/PPCMachineFunctionInfo.cpp b/lib/Target/PowerPC/PPCMachineFunctionInfo.cpp
index dd896a9a908..95f163153c7 100644
--- a/lib/Target/PowerPC/PPCMachineFunctionInfo.cpp
+++ b/lib/Target/PowerPC/PPCMachineFunctionInfo.cpp
@@ -18,8 +18,8 @@ using namespace llvm;
 void PPCFunctionInfo::anchor() { }
 
 MCSymbol *PPCFunctionInfo::getPICOffsetSymbol() const {
-  const DataLayout *DL = MF.getTarget().getDataLayout();
-  return MF.getContext().GetOrCreateSymbol(Twine(DL->getPrivateGlobalPrefix()) +
+  const DataLayout &DL = MF.getDataLayout();
+  return MF.getContext().getOrCreateSymbol(Twine(DL.getPrivateGlobalPrefix()) +
                                            Twine(MF.getFunctionNumber()) +
                                            "$poff");
 }
diff --git a/lib/Target/PowerPC/PPCRegisterInfo.cpp b/lib/Target/PowerPC/PPCRegisterInfo.cpp
index 0e568d3278e..6f364bc0577 100644
--- a/lib/Target/PowerPC/PPCRegisterInfo.cpp
+++ b/lib/Target/PowerPC/PPCRegisterInfo.cpp
@@ -165,8 +165,7 @@ void PPCRegisterInfo::adjustStackMapLiveOutMask(uint32_t *Mask) const {
 BitVector PPCRegisterInfo::getReservedRegs(const MachineFunction &MF) const {
   BitVector Reserved(getNumRegs());
   const PPCSubtarget &Subtarget = MF.getSubtarget<PPCSubtarget>();
-  const PPCFrameLowering *PPCFI =
-      static_cast<const PPCFrameLowering *>(Subtarget.getFrameLowering());
+  const PPCFrameLowering *TFI = getFrameLowering(MF);
 
   // The ZERO register is not really a register, but the representation of r0
   // when used in instructions that treat r0 as the constant 0.
@@ -209,7 +208,7 @@ BitVector PPCRegisterInfo::getReservedRegs(const MachineFunction &MF) const {
     Reserved.set(PPC::X1);
     Reserved.set(PPC::X13);
 
-    if (PPCFI->needsFP(MF))
+    if (TFI->needsFP(MF))
       Reserved.set(PPC::X31);
 
     if (hasBasePointer(MF))
@@ -230,7 +229,7 @@ BitVector PPCRegisterInfo::getReservedRegs(const MachineFunction &MF) const {
     }
   }
 
-  if (PPCFI->needsFP(MF))
+  if (TFI->needsFP(MF))
     Reserved.set(PPC::R31);
 
   if (hasBasePointer(MF)) {
@@ -256,8 +255,7 @@ BitVector PPCRegisterInfo::getReservedRegs(const MachineFunction &MF) const {
 
 unsigned PPCRegisterInfo::getRegPressureLimit(const TargetRegisterClass *RC,
                                               MachineFunction &MF) const {
-  const PPCSubtarget &Subtarget = MF.getSubtarget<PPCSubtarget>();
-  const TargetFrameLowering *TFI = Subtarget.getFrameLowering();
+  const PPCFrameLowering *TFI = getFrameLowering(MF);
   const unsigned DefaultSafety = 1;
 
   switch (RC->getID()) {
@@ -282,6 +280,7 @@ unsigned PPCRegisterInfo::getRegPressureLimit(const TargetRegisterClass *RC,
     return 32 - DefaultSafety;
   case PPC::VSRCRegClassID:
   case PPC::VSFRCRegClassID:
+  case PPC::VSSRCRegClassID:
     return 64 - DefaultSafety;
   case PPC::CRRCRegClassID:
     return 8 - DefaultSafety;
@@ -300,6 +299,8 @@ PPCRegisterInfo::getLargestLegalSuperClass(const TargetRegisterClass *RC,
       return &PPC::VSFRCRegClass;
     else if (RC == &PPC::VRRCRegClass)
       return &PPC::VSRCRegClass;
+    else if (RC == &PPC::F4RCRegClass && Subtarget.hasP8Vector())
+      return &PPC::VSSRCRegClass;
   }
 
   return TargetRegisterInfo::getLargestLegalSuperClass(RC, MF);
@@ -338,7 +339,8 @@ void PPCRegisterInfo::lowerDynamicAlloc(MachineBasicBlock::iterator II) const {
   unsigned FrameSize = MFI->getStackSize();
   
   // Get stack alignments.
-  unsigned TargetAlign = Subtarget.getFrameLowering()->getStackAlignment();
+  const PPCFrameLowering *TFI = getFrameLowering(MF);
+  unsigned TargetAlign = TFI->getStackAlignment();
   unsigned MaxAlign = MFI->getMaxAlignment();
   assert((maxCallFrameSize & (MaxAlign-1)) == 0 &&
          "Maximum call-frame size not sufficiently aligned");
@@ -861,8 +863,7 @@ PPCRegisterInfo::eliminateFrameIndex(MachineBasicBlock::iterator II,
 }
 
 unsigned PPCRegisterInfo::getFrameRegister(const MachineFunction &MF) const {
-  const PPCSubtarget &Subtarget = MF.getSubtarget<PPCSubtarget>();
-  const TargetFrameLowering *TFI = Subtarget.getFrameLowering();
+  const PPCFrameLowering *TFI = getFrameLowering(MF);
 
   if (!TM.isPPC64())
     return TFI->hasFP(MF) ? PPC::R31 : PPC::R1;
@@ -897,24 +898,6 @@ bool PPCRegisterInfo::hasBasePointer(const MachineFunction &MF) const {
   return needsStackRealignment(MF);
 }
 
-bool PPCRegisterInfo::canRealignStack(const MachineFunction &MF) const {
-  if (MF.getFunction()->hasFnAttribute("no-realign-stack"))
-    return false;
-
-  return true;
-}
-
-bool PPCRegisterInfo::needsStackRealignment(const MachineFunction &MF) const {
-  const PPCSubtarget &Subtarget = MF.getSubtarget<PPCSubtarget>();
-  const MachineFrameInfo *MFI = MF.getFrameInfo();
-  const Function *F = MF.getFunction();
-  unsigned StackAlign = Subtarget.getFrameLowering()->getStackAlignment();
-  bool requiresRealignment = ((MFI->getMaxAlignment() > StackAlign) ||
-                              F->hasFnAttribute(Attribute::StackAlignment));
-
-  return requiresRealignment && canRealignStack(MF);
-}
-
 /// Returns true if the instruction's frame index
 /// reference would be better served by a base register other than FP
 /// or SP. Used by LocalStackFrameAllocation to determine which frame index
@@ -943,11 +926,8 @@ needsFrameBaseReg(MachineInstr *MI, int64_t Offset) const {
 
   MachineBasicBlock &MBB = *MI->getParent();
   MachineFunction &MF = *MBB.getParent();
-  const PPCSubtarget &Subtarget = MF.getSubtarget<PPCSubtarget>();
-  const PPCFrameLowering *PPCFI =
-      static_cast<const PPCFrameLowering *>(Subtarget.getFrameLowering());
-  unsigned StackEst =
-    PPCFI->determineFrameLayout(MF, false, true);
+  const PPCFrameLowering *TFI = getFrameLowering(MF);
+  unsigned StackEst = TFI->determineFrameLayout(MF, false, true);
 
   // If we likely don't need a stack frame, then we probably don't need a
   // virtual base register either.
@@ -1031,4 +1011,3 @@ bool PPCRegisterInfo::isFrameOffsetLegal(const MachineInstr *MI,
          MI->getOpcode() == TargetOpcode::PATCHPOINT ||
          (isInt<16>(Offset) && (!usesIXAddr(*MI) || (Offset & 3) == 0));
 }
-
diff --git a/lib/Target/PowerPC/PPCRegisterInfo.h b/lib/Target/PowerPC/PPCRegisterInfo.h
index d304e1d8b5e..cd25b0e3ec3 100644
--- a/lib/Target/PowerPC/PPCRegisterInfo.h
+++ b/lib/Target/PowerPC/PPCRegisterInfo.h
@@ -136,8 +136,6 @@ class PPCRegisterInfo : public PPCGenRegisterInfo {
   // Base pointer (stack realignment) support.
   unsigned getBaseRegister(const MachineFunction &MF) const;
   bool hasBasePointer(const MachineFunction &MF) const;
-  bool canRealignStack(const MachineFunction &MF) const;
-  bool needsStackRealignment(const MachineFunction &MF) const override;
 };
 
 } // end namespace llvm
diff --git a/lib/Target/PowerPC/PPCRegisterInfo.td b/lib/Target/PowerPC/PPCRegisterInfo.td
index 6ca68ed49b7..e5f363c443c 100644
--- a/lib/Target/PowerPC/PPCRegisterInfo.td
+++ b/lib/Target/PowerPC/PPCRegisterInfo.td
@@ -288,7 +288,7 @@ def F8RC : RegisterClass<"PPC", [f64], 64, (add (sequence "F%u", 0, 13),
                                                 (sequence "F%u", 31, 14))>;
 def F4RC : RegisterClass<"PPC", [f32], 32, (add F8RC)>;
 
-def VRRC : RegisterClass<"PPC", [v16i8,v8i16,v4i32,v2i64,v4f32], 128,
+def VRRC : RegisterClass<"PPC", [v16i8,v8i16,v4i32,v2i64,v1i128,v4f32], 128,
                          (add V2, V3, V4, V5, V0, V1, V6, V7, V8, V9, V10, V11,
                              V12, V13, V14, V15, V16, V17, V18, V19, V31, V30,
                              V29, V28, V27, V26, V25, V24, V23, V22, V21, V20)>;
@@ -316,6 +316,9 @@ def VFRC :  RegisterClass<"PPC", [f64], 64,
                                VF22, VF21, VF20)>;
 def VSFRC : RegisterClass<"PPC", [f64], 64, (add F8RC, VFRC)>;
 
+// Register class for single precision scalars in VSX registers
+def VSSRC : RegisterClass<"PPC", [f32], 32, (add VSFRC)>;
+
 // For QPX
 def QFRC : RegisterClass<"PPC", [v4f64], 256, (add (sequence "QF%u", 0, 13),
                                                 (sequence "QF%u", 31, 14))>;
diff --git a/lib/Target/PowerPC/PPCScheduleP7.td b/lib/Target/PowerPC/PPCScheduleP7.td
index 635d154d10b..267f5672618 100644
--- a/lib/Target/PowerPC/PPCScheduleP7.td
+++ b/lib/Target/PowerPC/PPCScheduleP7.td
@@ -315,6 +315,10 @@ def P7Itineraries : ProcessorItineraries<
                                                   P7_DU3, P7_DU4], 0>,
                                    InstrStage<1, [P7_VS1, P7_VS2]>],
                                   [5, 1, 1]>,
+  InstrItinData<IIC_FPAddSub    , [InstrStage<1, [P7_DU1, P7_DU2,
+                                                  P7_DU3, P7_DU4], 0>,
+                                   InstrStage<1, [P7_VS1, P7_VS2]>],
+                                  [5, 1, 1]>,
   InstrItinData<IIC_FPCompare   , [InstrStage<1, [P7_DU1, P7_DU2,
                                                   P7_DU3, P7_DU4], 0>,
                                    InstrStage<1, [P7_VS1, P7_VS2]>],
diff --git a/lib/Target/PowerPC/PPCScheduleP8.td b/lib/Target/PowerPC/PPCScheduleP8.td
index 020739baec3..69e6d05c660 100644
--- a/lib/Target/PowerPC/PPCScheduleP8.td
+++ b/lib/Target/PowerPC/PPCScheduleP8.td
@@ -323,6 +323,10 @@ def P8Itineraries : ProcessorItineraries<
                                                   P8_DU4, P8_DU5, P8_DU6], 0>,
                                    InstrStage<1, [P8_FPU1, P8_FPU2]>],
                                   [5, 1, 1]>,
+  InstrItinData<IIC_FPAddSub    , [InstrStage<1, [P8_DU1, P8_DU2, P8_DU3,
+                                                  P8_DU4, P8_DU5, P8_DU6], 0>,
+                                   InstrStage<1, [P8_FPU1, P8_FPU2]>],
+                                  [5, 1, 1]>,
   InstrItinData<IIC_FPCompare   , [InstrStage<1, [P8_DU1, P8_DU2, P8_DU3,
                                                   P8_DU4, P8_DU5, P8_DU6], 0>,
                                    InstrStage<1, [P8_FPU1, P8_FPU2]>],
diff --git a/lib/Target/PowerPC/PPCSelectionDAGInfo.cpp b/lib/Target/PowerPC/PPCSelectionDAGInfo.cpp
deleted file mode 100644
index dc167421476..00000000000
--- a/lib/Target/PowerPC/PPCSelectionDAGInfo.cpp
+++ /dev/null
@@ -1,22 +0,0 @@
-//===-- PPCSelectionDAGInfo.cpp - PowerPC SelectionDAG Info ---------------===//
-//
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
-//
-//===----------------------------------------------------------------------===//
-//
-// This file implements the PPCSelectionDAGInfo class.
-//
-//===----------------------------------------------------------------------===//
-
-#include "PPCTargetMachine.h"
-using namespace llvm;
-
-#define DEBUG_TYPE "powerpc-selectiondag-info"
-
-PPCSelectionDAGInfo::PPCSelectionDAGInfo(const DataLayout *DL)
-    : TargetSelectionDAGInfo(DL) {}
-
-PPCSelectionDAGInfo::~PPCSelectionDAGInfo() {}
diff --git a/lib/Target/PowerPC/PPCSelectionDAGInfo.h b/lib/Target/PowerPC/PPCSelectionDAGInfo.h
deleted file mode 100644
index 2c1378d5670..00000000000
--- a/lib/Target/PowerPC/PPCSelectionDAGInfo.h
+++ /dev/null
@@ -1,31 +0,0 @@
-//===-- PPCSelectionDAGInfo.h - PowerPC SelectionDAG Info -------*- C++ -*-===//
-//
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
-//
-//===----------------------------------------------------------------------===//
-//
-// This file defines the PowerPC subclass for TargetSelectionDAGInfo.
-//
-//===----------------------------------------------------------------------===//
-
-#ifndef LLVM_LIB_TARGET_POWERPC_PPCSELECTIONDAGINFO_H
-#define LLVM_LIB_TARGET_POWERPC_PPCSELECTIONDAGINFO_H
-
-#include "llvm/Target/TargetSelectionDAGInfo.h"
-
-namespace llvm {
-
-class PPCTargetMachine;
-
-class PPCSelectionDAGInfo : public TargetSelectionDAGInfo {
-public:
-  explicit PPCSelectionDAGInfo(const DataLayout *DL);
-  ~PPCSelectionDAGInfo();
-};
-
-}
-
-#endif
diff --git a/lib/Target/PowerPC/PPCSubtarget.cpp b/lib/Target/PowerPC/PPCSubtarget.cpp
index f313b0a6f17..58daccae90f 100644
--- a/lib/Target/PowerPC/PPCSubtarget.cpp
+++ b/lib/Target/PowerPC/PPCSubtarget.cpp
@@ -47,13 +47,13 @@ PPCSubtarget &PPCSubtarget::initializeSubtargetDependencies(StringRef CPU,
   return *this;
 }
 
-PPCSubtarget::PPCSubtarget(const std::string &TT, const std::string &CPU,
+PPCSubtarget::PPCSubtarget(const Triple &TT, const std::string &CPU,
                            const std::string &FS, const PPCTargetMachine &TM)
     : PPCGenSubtargetInfo(TT, CPU, FS), TargetTriple(TT),
       IsPPC64(TargetTriple.getArch() == Triple::ppc64 ||
               TargetTriple.getArch() == Triple::ppc64le),
       TM(TM), FrameLowering(initializeSubtargetDependencies(CPU, FS)),
-      InstrInfo(*this), TLInfo(TM, *this), TSInfo(TM.getDataLayout()) {}
+      InstrInfo(*this), TLInfo(TM, *this) {}
 
 void PPCSubtarget::initializeEnvironment() {
   StackAlignment = 16;
@@ -91,7 +91,7 @@ void PPCSubtarget::initializeEnvironment() {
   IsPPC4xx = false;
   IsPPC6xx = false;
   IsE500 = false;
-  DeprecatedMFTB = false;
+  FeatureMFTB = false;
   DeprecatedDST = false;
   HasLazyResolverStubs = false;
   HasICBT = false;
@@ -175,7 +175,7 @@ bool PPCSubtarget::enableMachineScheduler() const {
 }
 
 // This overrides the PostRAScheduler bit in the SchedModel for each CPU.
-bool PPCSubtarget::enablePostMachineScheduler() const { return true; }
+bool PPCSubtarget::enablePostRAScheduler() const { return true; }
 
 PPCGenSubtargetInfo::AntiDepBreakMode PPCSubtarget::getAntiDepBreakMode() const {
   return TargetSubtargetInfo::ANTIDEP_ALL;
diff --git a/lib/Target/PowerPC/PPCSubtarget.h b/lib/Target/PowerPC/PPCSubtarget.h
index 8d955088634..0616c1f6560 100644
--- a/lib/Target/PowerPC/PPCSubtarget.h
+++ b/lib/Target/PowerPC/PPCSubtarget.h
@@ -17,10 +17,10 @@
 #include "PPCFrameLowering.h"
 #include "PPCISelLowering.h"
 #include "PPCInstrInfo.h"
-#include "PPCSelectionDAGInfo.h"
 #include "llvm/ADT/Triple.h"
 #include "llvm/IR/DataLayout.h"
 #include "llvm/MC/MCInstrItineraries.h"
+#include "llvm/Target/TargetSelectionDAGInfo.h"
 #include "llvm/Target/TargetSubtargetInfo.h"
 #include <string>
 
@@ -110,7 +110,7 @@ class PPCSubtarget : public PPCGenSubtargetInfo {
   bool IsE500;
   bool IsPPC4xx;
   bool IsPPC6xx;
-  bool DeprecatedMFTB;
+  bool FeatureMFTB;
   bool DeprecatedDST;
   bool HasLazyResolverStubs;
   bool IsLittleEndian;
@@ -129,14 +129,14 @@ class PPCSubtarget : public PPCGenSubtargetInfo {
   PPCFrameLowering FrameLowering;
   PPCInstrInfo InstrInfo;
   PPCTargetLowering TLInfo;
-  PPCSelectionDAGInfo TSInfo;
+  TargetSelectionDAGInfo TSInfo;
 
 public:
   /// This constructor initializes the data members to match that
   /// of the specified triple.
   ///
-  PPCSubtarget(const std::string &TT, const std::string &CPU,
-               const std::string &FS, const PPCTargetMachine &TM);
+  PPCSubtarget(const Triple &TT, const std::string &CPU, const std::string &FS,
+               const PPCTargetMachine &TM);
 
   /// ParseSubtargetFeatures - Parses features string setting specified
   /// subtarget options.  Definition of function is auto generated by tblgen.
@@ -164,7 +164,7 @@ class PPCSubtarget : public PPCGenSubtargetInfo {
   const PPCTargetLowering *getTargetLowering() const override {
     return &TLInfo;
   }
-  const PPCSelectionDAGInfo *getSelectionDAGInfo() const override {
+  const TargetSelectionDAGInfo *getSelectionDAGInfo() const override {
     return &TSInfo;
   }
   const PPCRegisterInfo *getRegisterInfo() const override {
@@ -237,7 +237,7 @@ class PPCSubtarget : public PPCGenSubtargetInfo {
   bool isPPC4xx() const { return IsPPC4xx; }
   bool isPPC6xx() const { return IsPPC6xx; }
   bool isE500() const { return IsE500; }
-  bool isDeprecatedMFTB() const { return DeprecatedMFTB; }
+  bool isFeatureMFTB() const { return FeatureMFTB; }
   bool isDeprecatedDST() const { return DeprecatedDST; }
   bool hasICBT() const { return HasICBT; }
   bool hasInvariantFunctionDescriptors() const {
@@ -274,7 +274,7 @@ class PPCSubtarget : public PPCGenSubtargetInfo {
   // Scheduling customization.
   bool enableMachineScheduler() const override;
   // This overrides the PostRAScheduler bit in the SchedModel for each CPU.
-  bool enablePostMachineScheduler() const override;
+  bool enablePostRAScheduler() const override;
   AntiDepBreakMode getAntiDepBreakMode() const override;
   void getCriticalPathRCs(RegClassVector &CriticalPathRCs) const override;
 
diff --git a/lib/Target/PowerPC/PPCTLSDynamicCall.cpp b/lib/Target/PowerPC/PPCTLSDynamicCall.cpp
index 270fc712be1..2dc0d825c80 100644
--- a/lib/Target/PowerPC/PPCTLSDynamicCall.cpp
+++ b/lib/Target/PowerPC/PPCTLSDynamicCall.cpp
@@ -55,14 +55,16 @@ namespace {
       bool Is64Bit = MBB.getParent()->getSubtarget<PPCSubtarget>().isPPC64();
 
       for (MachineBasicBlock::iterator I = MBB.begin(), IE = MBB.end();
-           I != IE; ++I) {
+           I != IE;) {
         MachineInstr *MI = I;
 
         if (MI->getOpcode() != PPC::ADDItlsgdLADDR &&
             MI->getOpcode() != PPC::ADDItlsldLADDR &&
             MI->getOpcode() != PPC::ADDItlsgdLADDR32 &&
-            MI->getOpcode() != PPC::ADDItlsldLADDR32)
+            MI->getOpcode() != PPC::ADDItlsldLADDR32) {
+          ++I;
           continue;
+        }
 
         DEBUG(dbgs() << "TLS Dynamic Call Fixup:\n    " << *MI;);
 
diff --git a/lib/Target/PowerPC/PPCTOCRegDeps.cpp b/lib/Target/PowerPC/PPCTOCRegDeps.cpp
new file mode 100644
index 00000000000..bf165c9edc6
--- /dev/null
+++ b/lib/Target/PowerPC/PPCTOCRegDeps.cpp
@@ -0,0 +1,156 @@
+//===-- PPCTOCRegDeps.cpp - Add Extra TOC Register Dependencies -----------===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// When resolving an address using the ELF ABI TOC pointer, two relocations are
+// generally required: one for the high part and one for the low part. Only
+// the high part generally explicitly depends on r2 (the TOC pointer). And, so,
+// we might produce code like this:
+//
+// .Ltmp526:
+//         addis 3, 2, .LC12@toc@ha
+// .Ltmp1628:
+//         std 2, 40(1)
+//         ld 5, 0(27)
+//         ld 2, 8(27)
+//         ld 11, 16(27)
+//         ld 3, .LC12@toc@l(3)
+//         rldicl 4, 4, 0, 32
+//         mtctr 5
+//         bctrl
+//         ld 2, 40(1)
+//
+// And there is nothing wrong with this code, as such, but there is a linker bug
+// in binutils (https://sourceware.org/bugzilla/show_bug.cgi?id=18414) that will
+// misoptimize this code sequence to this:
+//         nop
+//         std     r2,40(r1)
+//         ld      r5,0(r27)
+//         ld      r2,8(r27)
+//         ld      r11,16(r27)
+//         ld      r3,-32472(r2)
+//         clrldi  r4,r4,32
+//         mtctr   r5
+//         bctrl
+//         ld      r2,40(r1)
+// because the linker does not know (and does not check) that the value in r2
+// changed in between the instruction using the .LC12@toc@ha (TOC-relative)
+// relocation and the instruction using the .LC12@toc@l(3) relocation.
+// Because it finds these instructions using the relocations (and not by
+// scanning the instructions), it has been asserted that there is no good way
+// to detect the change of r2 in between. As a result, this bug may never be
+// fixed (i.e. it may become part of the definition of the ABI). GCC was
+// updated to add extra dependencies on r2 to instructions using the @toc@l
+// relocations to avoid this problem, and we'll do the same here.
+//
+// This is done as a separate pass because:
+//  1. These extra r2 dependencies are not really properties of the
+//     instructions, but rather due to a linker bug, and maybe one day we'll be
+//     able to get rid of them when targeting linkers without this bug (and,
+//     thus, keeping the logic centralized here will make that
+//     straightforward).
+//  2. There are ISel-level peephole optimizations that propagate the @toc@l
+//     relocations to some user instructions, and so the exta dependencies do
+//     not apply only to a fixed set of instructions (without undesirable
+//     definition replication).
+//
+//===----------------------------------------------------------------------===//
+
+#include "PPCInstrInfo.h"
+#include "MCTargetDesc/PPCPredicates.h"
+#include "PPC.h"
+#include "PPCInstrBuilder.h"
+#include "PPCMachineFunctionInfo.h"
+#include "PPCTargetMachine.h"
+#include "llvm/ADT/STLExtras.h"
+#include "llvm/ADT/Statistic.h"
+#include "llvm/CodeGen/MachineFrameInfo.h"
+#include "llvm/CodeGen/MachineFunctionPass.h"
+#include "llvm/CodeGen/MachineInstr.h"
+#include "llvm/CodeGen/MachineRegisterInfo.h"
+#include "llvm/MC/MCAsmInfo.h"
+#include "llvm/Support/CommandLine.h"
+#include "llvm/Support/Debug.h"
+#include "llvm/Support/ErrorHandling.h"
+#include "llvm/Support/TargetRegistry.h"
+#include "llvm/Support/raw_ostream.h"
+
+using namespace llvm;
+
+#define DEBUG_TYPE "ppc-toc-reg-deps"
+
+namespace llvm {
+  void initializePPCTOCRegDepsPass(PassRegistry&);
+}
+
+namespace {
+  // PPCTOCRegDeps pass - For simple functions without epilogue code, move
+  // returns up, and create conditional returns, to avoid unnecessary
+  // branch-to-blr sequences.
+  struct PPCTOCRegDeps : public MachineFunctionPass {
+    static char ID;
+    PPCTOCRegDeps() : MachineFunctionPass(ID) {
+      initializePPCTOCRegDepsPass(*PassRegistry::getPassRegistry());
+    }
+
+protected:
+    bool hasTOCLoReloc(const MachineInstr &MI) {
+      if (MI.getOpcode() == PPC::LDtocL ||
+          MI.getOpcode() == PPC::ADDItocL)
+        return true;
+
+      for (const MachineOperand &MO : MI.operands()) {
+        if ((MO.getTargetFlags() & PPCII::MO_ACCESS_MASK) == PPCII::MO_TOC_LO)
+          return true;
+      }
+
+      return false;
+    }
+
+    bool processBlock(MachineBasicBlock &MBB) {
+      bool Changed = false;
+
+      for (auto &MI : MBB) {
+        if (!hasTOCLoReloc(MI))
+          continue;
+
+        MI.addOperand(MachineOperand::CreateReg(PPC::X2,
+                                                false  /*IsDef*/,
+                                                true  /*IsImp*/));
+        Changed = true;
+      }
+
+      return Changed;
+    }
+
+public:
+    bool runOnMachineFunction(MachineFunction &MF) override {
+      bool Changed = false;
+
+      for (MachineFunction::iterator I = MF.begin(); I != MF.end();) {
+        MachineBasicBlock &B = *I++;
+        if (processBlock(B))
+          Changed = true;
+      }
+
+      return Changed;
+    }
+
+    void getAnalysisUsage(AnalysisUsage &AU) const override {
+      MachineFunctionPass::getAnalysisUsage(AU);
+    }
+  };
+}
+
+INITIALIZE_PASS(PPCTOCRegDeps, DEBUG_TYPE,
+                "PowerPC TOC Register Dependencies", false, false)
+
+char PPCTOCRegDeps::ID = 0;
+FunctionPass*
+llvm::createPPCTOCRegDepsPass() { return new PPCTOCRegDeps(); }
+
diff --git a/lib/Target/PowerPC/PPCTargetMachine.cpp b/lib/Target/PowerPC/PPCTargetMachine.cpp
index 7267529ef81..b602edb1f83 100644
--- a/lib/Target/PowerPC/PPCTargetMachine.cpp
+++ b/lib/Target/PowerPC/PPCTargetMachine.cpp
@@ -38,6 +38,10 @@ static cl::opt<bool>
 VSXFMAMutateEarly("schedule-ppc-vsx-fma-mutation-early",
   cl::Hidden, cl::desc("Schedule VSX FMA instruction mutation early"));
 
+static cl::
+opt<bool> DisableVSXSwapRemoval("disable-ppc-vsx-swap-removal", cl::Hidden,
+                                cl::desc("Disable VSX Swap Removal for PPC"));
+
 static cl::opt<bool>
 EnableGEPOpt("ppc-gep-opt", cl::Hidden,
              cl::desc("Enable optimizations on complex GEPs"),
@@ -48,6 +52,16 @@ EnablePrefetch("enable-ppc-prefetching",
                   cl::desc("disable software prefetching on PPC"),
                   cl::init(false), cl::Hidden);
 
+static cl::opt<bool>
+EnableExtraTOCRegDeps("enable-ppc-extra-toc-reg-deps",
+                      cl::desc("Add extra TOC register dependencies"),
+                      cl::init(true), cl::Hidden);
+
+static cl::opt<bool>
+EnableMachineCombinerPass("ppc-machine-combiner",
+                          cl::desc("Enable the machine combiner pass"),
+                          cl::init(true), cl::Hidden);
+
 extern "C" void LLVMInitializePowerPCTarget() {
   // Register the targets
   RegisterTargetMachine<PPC32TargetMachine> A(ThePPC32Target);
@@ -89,13 +103,12 @@ static std::string getDataLayoutString(const Triple &T) {
   return Ret;
 }
 
-static std::string computeFSAdditions(StringRef FS, CodeGenOpt::Level OL, StringRef TT) {
+static std::string computeFSAdditions(StringRef FS, CodeGenOpt::Level OL,
+                                      const Triple &TT) {
   std::string FullFS = FS;
-  Triple TargetTriple(TT);
 
   // Make sure 64-bit features are available when CPUname is generic
-  if (TargetTriple.getArch() == Triple::ppc64 ||
-      TargetTriple.getArch() == Triple::ppc64le) {
+  if (TT.getArch() == Triple::ppc64 || TT.getArch() == Triple::ppc64le) {
     if (!FullFS.empty())
       FullFS = "+64bit," + FullFS;
     else
@@ -156,14 +169,34 @@ static PPCTargetMachine::PPCABI computeTargetABI(const Triple &TT,
 // with what are (currently) non-function specific overrides as it goes into the
 // LLVMTargetMachine constructor and then using the stored value in the
 // Subtarget constructor below it.
-PPCTargetMachine::PPCTargetMachine(const Target &T, StringRef TT, StringRef CPU,
-                                   StringRef FS, const TargetOptions &Options,
+PPCTargetMachine::PPCTargetMachine(const Target &T, const Triple &TT,
+                                   StringRef CPU, StringRef FS,
+                                   const TargetOptions &Options,
                                    Reloc::Model RM, CodeModel::Model CM,
                                    CodeGenOpt::Level OL)
-    : LLVMTargetMachine(T, getDataLayoutString(Triple(TT)), TT, CPU,
+    : LLVMTargetMachine(T, getDataLayoutString(TT), TT, CPU,
                         computeFSAdditions(FS, OL, TT), Options, RM, CM, OL),
-      TLOF(createTLOF(Triple(getTargetTriple()))),
-      TargetABI(computeTargetABI(Triple(TT), Options)) {
+      TLOF(createTLOF(getTargetTriple())),
+      TargetABI(computeTargetABI(TT, Options)),
+      Subtarget(TargetTriple, CPU, computeFSAdditions(FS, OL, TT), *this) {
+
+  // For the estimates, convergence is quadratic, so we essentially double the
+  // number of digits correct after every iteration. For both FRE and FRSQRTE,
+  // the minimum architected relative accuracy is 2^-5. When hasRecipPrec(),
+  // this is 2^-14. IEEE float has 23 digits and double has 52 digits.
+  unsigned RefinementSteps = Subtarget.hasRecipPrec() ? 1 : 3,
+           RefinementSteps64 = RefinementSteps + 1;
+
+  this->Options.Reciprocals.setDefaults("sqrtf", true, RefinementSteps);
+  this->Options.Reciprocals.setDefaults("vec-sqrtf", true, RefinementSteps);
+  this->Options.Reciprocals.setDefaults("divf", true, RefinementSteps);
+  this->Options.Reciprocals.setDefaults("vec-divf", true, RefinementSteps);
+
+  this->Options.Reciprocals.setDefaults("sqrtd", true, RefinementSteps64);
+  this->Options.Reciprocals.setDefaults("vec-sqrtd", true, RefinementSteps64);
+  this->Options.Reciprocals.setDefaults("divd", true, RefinementSteps64);
+  this->Options.Reciprocals.setDefaults("vec-divd", true, RefinementSteps64);
+
   initAsmInfo();
 }
 
@@ -171,23 +204,21 @@ PPCTargetMachine::~PPCTargetMachine() {}
 
 void PPC32TargetMachine::anchor() { }
 
-PPC32TargetMachine::PPC32TargetMachine(const Target &T, StringRef TT,
+PPC32TargetMachine::PPC32TargetMachine(const Target &T, const Triple &TT,
                                        StringRef CPU, StringRef FS,
                                        const TargetOptions &Options,
                                        Reloc::Model RM, CodeModel::Model CM,
                                        CodeGenOpt::Level OL)
-  : PPCTargetMachine(T, TT, CPU, FS, Options, RM, CM, OL) {
-}
+    : PPCTargetMachine(T, TT, CPU, FS, Options, RM, CM, OL) {}
 
 void PPC64TargetMachine::anchor() { }
 
-PPC64TargetMachine::PPC64TargetMachine(const Target &T, StringRef TT,
-                                       StringRef CPU,  StringRef FS,
+PPC64TargetMachine::PPC64TargetMachine(const Target &T, const Triple &TT,
+                                       StringRef CPU, StringRef FS,
                                        const TargetOptions &Options,
                                        Reloc::Model RM, CodeModel::Model CM,
                                        CodeGenOpt::Level OL)
-  : PPCTargetMachine(T, TT, CPU, FS, Options, RM, CM, OL) {
-}
+    : PPCTargetMachine(T, TT, CPU, FS, Options, RM, CM, OL) {}
 
 const PPCSubtarget *
 PPCTargetMachine::getSubtargetImpl(const Function &F) const {
@@ -239,6 +270,7 @@ class PPCPassConfig : public TargetPassConfig {
   bool addPreISel() override;
   bool addILPOpts() override;
   bool addInstSelector() override;
+  void addMachineSSAOptimization() override;
   void addPreRegAlloc() override;
   void addPreSched2() override;
   void addPreEmitPass() override;
@@ -254,9 +286,8 @@ void PPCPassConfig::addIRPasses() {
 
   // For the BG/Q (or if explicitly requested), add explicit data prefetch
   // intrinsics.
-  bool UsePrefetching =
-    Triple(TM->getTargetTriple()).getVendor() == Triple::BGQ &&           
-    getOptLevel() != CodeGenOpt::None;
+  bool UsePrefetching = TM->getTargetTriple().getVendor() == Triple::BGQ &&
+                        getOptLevel() != CodeGenOpt::None;
   if (EnablePrefetch.getNumOccurrences() > 0)
     UsePrefetching = EnablePrefetch;
   if (UsePrefetching)
@@ -290,6 +321,10 @@ bool PPCPassConfig::addPreISel() {
 
 bool PPCPassConfig::addILPOpts() {
   addPass(&EarlyIfConverterID);
+
+  if (EnableMachineCombinerPass)
+    addPass(&MachineCombinerID);
+
   return true;
 }
 
@@ -306,12 +341,23 @@ bool PPCPassConfig::addInstSelector() {
   return false;
 }
 
+void PPCPassConfig::addMachineSSAOptimization() {
+  TargetPassConfig::addMachineSSAOptimization();
+  // For little endian, remove where possible the vector swap instructions
+  // introduced at code generation to normalize vector element order.
+  if (TM->getTargetTriple().getArch() == Triple::ppc64le &&
+      !DisableVSXSwapRemoval)
+    addPass(createPPCVSXSwapRemovalPass());
+}
+
 void PPCPassConfig::addPreRegAlloc() {
   initializePPCVSXFMAMutatePass(*PassRegistry::getPassRegistry());
   insertPass(VSXFMAMutateEarly ? &RegisterCoalescerID : &MachineSchedulerID,
              &PPCVSXFMAMutateID);
   if (getPPCTargetMachine().getRelocationModel() == Reloc::PIC_)
     addPass(createPPCTLSDynamicCallPass());
+  if (EnableExtraTOCRegDeps)
+    addPass(createPPCTOCRegDepsPass());
 }
 
 void PPCPassConfig::addPreSched2() {
diff --git a/lib/Target/PowerPC/PPCTargetMachine.h b/lib/Target/PowerPC/PPCTargetMachine.h
index 7a490588989..6496339519a 100644
--- a/lib/Target/PowerPC/PPCTargetMachine.h
+++ b/lib/Target/PowerPC/PPCTargetMachine.h
@@ -29,11 +29,13 @@ class PPCTargetMachine : public LLVMTargetMachine {
 private:
   std::unique_ptr<TargetLoweringObjectFile> TLOF;
   PPCABI TargetABI;
+  PPCSubtarget Subtarget;
+
   mutable StringMap<std::unique_ptr<PPCSubtarget>> SubtargetMap;
 
 public:
-  PPCTargetMachine(const Target &T, StringRef TT, StringRef CPU, StringRef FS,
-                   const TargetOptions &Options, Reloc::Model RM,
+  PPCTargetMachine(const Target &T, const Triple &TT, StringRef CPU,
+                   StringRef FS, const TargetOptions &Options, Reloc::Model RM,
                    CodeModel::Model CM, CodeGenOpt::Level OL);
 
   ~PPCTargetMachine() override;
@@ -50,7 +52,7 @@ class PPCTargetMachine : public LLVMTargetMachine {
   }
   bool isELFv2ABI() const { return TargetABI == PPC_ABI_ELFv2; }
   bool isPPC64() const {
-    Triple TT(getTargetTriple());
+    const Triple &TT = getTargetTriple();
     return (TT.getArch() == Triple::ppc64 || TT.getArch() == Triple::ppc64le);
   };
 };
@@ -60,8 +62,8 @@ class PPCTargetMachine : public LLVMTargetMachine {
 class PPC32TargetMachine : public PPCTargetMachine {
   virtual void anchor();
 public:
-  PPC32TargetMachine(const Target &T, StringRef TT,
-                     StringRef CPU, StringRef FS, const TargetOptions &Options,
+  PPC32TargetMachine(const Target &T, const Triple &TT, StringRef CPU,
+                     StringRef FS, const TargetOptions &Options,
                      Reloc::Model RM, CodeModel::Model CM,
                      CodeGenOpt::Level OL);
 };
@@ -71,8 +73,8 @@ class PPC32TargetMachine : public PPCTargetMachine {
 class PPC64TargetMachine : public PPCTargetMachine {
   virtual void anchor();
 public:
-  PPC64TargetMachine(const Target &T, StringRef TT,
-                     StringRef CPU, StringRef FS, const TargetOptions &Options,
+  PPC64TargetMachine(const Target &T, const Triple &TT, StringRef CPU,
+                     StringRef FS, const TargetOptions &Options,
                      Reloc::Model RM, CodeModel::Model CM,
                      CodeGenOpt::Level OL);
 };
diff --git a/lib/Target/PowerPC/PPCTargetObjectFile.cpp b/lib/Target/PowerPC/PPCTargetObjectFile.cpp
index 2903cc192aa..9ee5db938b6 100644
--- a/lib/Target/PowerPC/PPCTargetObjectFile.cpp
+++ b/lib/Target/PowerPC/PPCTargetObjectFile.cpp
@@ -22,7 +22,7 @@ Initialize(MCContext &Ctx, const TargetMachine &TM) {
   InitializeELF(TM.Options.UseInitArray);
 }
 
-const MCSection *PPC64LinuxTargetObjectFile::SelectSectionForGlobal(
+MCSection *PPC64LinuxTargetObjectFile::SelectSectionForGlobal(
     const GlobalValue *GV, SectionKind Kind, Mangler &Mang,
     const TargetMachine &TM) const {
   // Here override ReadOnlySection to DataRelROSection for PPC64 SVR4 ABI
@@ -55,9 +55,9 @@ const MCSection *PPC64LinuxTargetObjectFile::SelectSectionForGlobal(
 const MCExpr *PPC64LinuxTargetObjectFile::
 getDebugThreadLocalSymbol(const MCSymbol *Sym) const {
   const MCExpr *Expr =
-    MCSymbolRefExpr::Create(Sym, MCSymbolRefExpr::VK_PPC_DTPREL, getContext());
-  return MCBinaryExpr::CreateAdd(Expr,
-                                 MCConstantExpr::Create(0x8000, getContext()),
+    MCSymbolRefExpr::create(Sym, MCSymbolRefExpr::VK_PPC_DTPREL, getContext());
+  return MCBinaryExpr::createAdd(Expr,
+                                 MCConstantExpr::create(0x8000, getContext()),
                                  getContext());
 }
 
diff --git a/lib/Target/PowerPC/PPCTargetObjectFile.h b/lib/Target/PowerPC/PPCTargetObjectFile.h
index cd84da22275..d248791f2ca 100644
--- a/lib/Target/PowerPC/PPCTargetObjectFile.h
+++ b/lib/Target/PowerPC/PPCTargetObjectFile.h
@@ -22,9 +22,9 @@ namespace llvm {
 
     void Initialize(MCContext &Ctx, const TargetMachine &TM) override;
 
-    const MCSection *SelectSectionForGlobal(const GlobalValue *GV,
-                                        SectionKind Kind, Mangler &Mang,
-                                        const TargetMachine &TM) const override;
+    MCSection *SelectSectionForGlobal(const GlobalValue *GV, SectionKind Kind,
+                                      Mangler &Mang,
+                                      const TargetMachine &TM) const override;
 
     /// \brief Describe a TLS variable address within debug info.
     const MCExpr *getDebugThreadLocalSymbol(const MCSymbol *Sym) const override;
diff --git a/lib/Target/PowerPC/PPCTargetStreamer.h b/lib/Target/PowerPC/PPCTargetStreamer.h
index 8aaf5e18890..dbe7617d354 100644
--- a/lib/Target/PowerPC/PPCTargetStreamer.h
+++ b/lib/Target/PowerPC/PPCTargetStreamer.h
@@ -20,7 +20,7 @@ class PPCTargetStreamer : public MCTargetStreamer {
   virtual void emitTCEntry(const MCSymbol &S) = 0;
   virtual void emitMachine(StringRef CPU) = 0;
   virtual void emitAbiVersion(int AbiVersion) = 0;
-  virtual void emitLocalEntry(MCSymbol *S, const MCExpr *LocalOffset) = 0;
+  virtual void emitLocalEntry(MCSymbolELF *S, const MCExpr *LocalOffset) = 0;
 };
 }
 
diff --git a/lib/Target/PowerPC/PPCTargetTransformInfo.cpp b/lib/Target/PowerPC/PPCTargetTransformInfo.cpp
index b46acd47f31..e21c2b77f4d 100644
--- a/lib/Target/PowerPC/PPCTargetTransformInfo.cpp
+++ b/lib/Target/PowerPC/PPCTargetTransformInfo.cpp
@@ -187,6 +187,10 @@ void PPCTTIImpl::getUnrollingPreferences(Loop *L,
     // The A2 is in-order with a deep pipeline, and concatenation unrolling
     // helps expose latency-hiding opportunities to the instruction scheduler.
     UP.Partial = UP.Runtime = true;
+
+    // We unroll a lot on the A2 (hundreds of instructions), and the benefits
+    // often outweigh the cost of a division to compute the trip count.
+    UP.AllowExpensiveTripCount = true;
   }
 
   BaseT::getUnrollingPreferences(L, UP);
@@ -215,7 +219,7 @@ unsigned PPCTTIImpl::getRegisterBitWidth(bool Vector) {
 
 }
 
-unsigned PPCTTIImpl::getMaxInterleaveFactor() {
+unsigned PPCTTIImpl::getMaxInterleaveFactor(unsigned VF) {
   unsigned Directive = ST->getDarwinDirective();
   // The 440 has no SIMD support, but floating-point instructions
   // have a 5-cycle latency, so unroll by 5x for latency hiding.
@@ -313,7 +317,7 @@ unsigned PPCTTIImpl::getMemoryOpCost(unsigned Opcode, Type *Src,
                                      unsigned Alignment,
                                      unsigned AddressSpace) {
   // Legalize the type.
-  std::pair<unsigned, MVT> LT = TLI->getTypeLegalizationCost(Src);
+  std::pair<unsigned, MVT> LT = TLI->getTypeLegalizationCost(DL, Src);
   assert((Opcode == Instruction::Load || Opcode == Instruction::Store) &&
          "Invalid Opcode");
 
diff --git a/lib/Target/PowerPC/PPCTargetTransformInfo.h b/lib/Target/PowerPC/PPCTargetTransformInfo.h
index 21acea1a36d..368bef93f0d 100644
--- a/lib/Target/PowerPC/PPCTargetTransformInfo.h
+++ b/lib/Target/PowerPC/PPCTargetTransformInfo.h
@@ -38,7 +38,8 @@ class PPCTTIImpl : public BasicTTIImplBase<PPCTTIImpl> {
 
 public:
   explicit PPCTTIImpl(const PPCTargetMachine *TM, Function &F)
-      : BaseT(TM), ST(TM->getSubtargetImpl(F)), TLI(ST->getTargetLowering()) {}
+      : BaseT(TM, F.getParent()->getDataLayout()), ST(TM->getSubtargetImpl(F)),
+        TLI(ST->getTargetLowering()) {}
 
   // Provide value semantics. MSVC requires that we spell all of these out.
   PPCTTIImpl(const PPCTTIImpl &Arg)
@@ -46,18 +47,6 @@ class PPCTTIImpl : public BasicTTIImplBase<PPCTTIImpl> {
   PPCTTIImpl(PPCTTIImpl &&Arg)
       : BaseT(std::move(static_cast<BaseT &>(Arg))), ST(std::move(Arg.ST)),
         TLI(std::move(Arg.TLI)) {}
-  PPCTTIImpl &operator=(const PPCTTIImpl &RHS) {
-    BaseT::operator=(static_cast<const BaseT &>(RHS));
-    ST = RHS.ST;
-    TLI = RHS.TLI;
-    return *this;
-  }
-  PPCTTIImpl &operator=(PPCTTIImpl &&RHS) {
-    BaseT::operator=(std::move(static_cast<BaseT &>(RHS)));
-    ST = std::move(RHS.ST);
-    TLI = std::move(RHS.TLI);
-    return *this;
-  }
 
   /// \name Scalar TTI Implementations
   /// @{
@@ -81,7 +70,7 @@ class PPCTTIImpl : public BasicTTIImplBase<PPCTTIImpl> {
   bool enableAggressiveInterleaving(bool LoopHasReductions);
   unsigned getNumberOfRegisters(bool Vector);
   unsigned getRegisterBitWidth(bool Vector);
-  unsigned getMaxInterleaveFactor();
+  unsigned getMaxInterleaveFactor(unsigned VF);
   unsigned getArithmeticInstrCost(
       unsigned Opcode, Type *Ty,
       TTI::OperandValueKind Opd1Info = TTI::OK_AnyValue,
diff --git a/lib/Target/PowerPC/PPCVSXFMAMutate.cpp b/lib/Target/PowerPC/PPCVSXFMAMutate.cpp
index f352fa647ac..974857556dc 100644
--- a/lib/Target/PowerPC/PPCVSXFMAMutate.cpp
+++ b/lib/Target/PowerPC/PPCVSXFMAMutate.cpp
@@ -136,6 +136,16 @@ namespace {
         // source of the copy, it must still be live here.  We can't use
         // interval testing for a physical register, so as long as we're
         // walking the MIs we may as well test liveness here.
+        //
+        // FIXME: There is a case that occurs in practice, like this:
+        //   %vreg9<def> = COPY %F1; VSSRC:%vreg9
+        //   ...
+        //   %vreg6<def> = COPY %vreg9; VSSRC:%vreg6,%vreg9
+        //   %vreg7<def> = COPY %vreg9; VSSRC:%vreg7,%vreg9
+        //   %vreg9<def,tied1> = XSMADDASP %vreg9<tied0>, %vreg1, %vreg4; VSSRC:
+        //   %vreg6<def,tied1> = XSMADDASP %vreg6<tied0>, %vreg1, %vreg2; VSSRC:
+        //   %vreg7<def,tied1> = XSMADDASP %vreg7<tied0>, %vreg1, %vreg3; VSSRC:
+        // which prevents an otherwise-profitable transformation.
         bool OtherUsers = false, KillsAddendSrc = false;
         for (auto J = std::prev(I), JE = MachineBasicBlock::iterator(AddendMI);
              J != JE; --J) {
@@ -179,7 +189,6 @@ namespace {
 
         // Transform: (O2 * O3) + O1 -> (O2 * O1) + O3.
 
-        unsigned AddReg = AddendMI->getOperand(1).getReg();
         unsigned KilledProdReg = MI->getOperand(KilledProdOp).getReg();
         unsigned OtherProdReg  = MI->getOperand(OtherProdOp).getReg();
 
@@ -210,7 +219,7 @@ namespace {
 
         MI->getOperand(0).setReg(KilledProdReg);
         MI->getOperand(1).setReg(KilledProdReg);
-        MI->getOperand(3).setReg(AddReg);
+        MI->getOperand(3).setReg(AddendSrcReg);
         MI->getOperand(2).setReg(OtherProdReg);
 
         MI->getOperand(0).setSubReg(KilledProdSubReg);
@@ -268,6 +277,20 @@ namespace {
         }
         DEBUG(dbgs() << "  extended: " << NewFMAInt << '\n');
 
+        // Extend the live interval of the addend source (it might end at the
+        // copy to be removed, or somewhere in between there and here). This
+        // is necessary only if it is a physical register.
+        if (!TargetRegisterInfo::isVirtualRegister(AddendSrcReg))
+          for (MCRegUnitIterator Units(AddendSrcReg, TRI); Units.isValid();
+               ++Units) {
+            unsigned Unit = *Units;
+
+            LiveRange &AddendSrcRange = LIS->getRegUnit(Unit);
+            AddendSrcRange.extendInBlock(LIS->getMBBStartIdx(&MBB),
+                                         FMAIdx.getRegSlot());
+            DEBUG(dbgs() << "  extended: " << AddendSrcRange << '\n');
+          }
+
         FMAInt.removeValNo(FMAValNo);
         DEBUG(dbgs() << "  trimmed:  " << FMAInt << '\n');
 
diff --git a/lib/Target/PowerPC/PPCVSXSwapRemoval.cpp b/lib/Target/PowerPC/PPCVSXSwapRemoval.cpp
new file mode 100644
index 00000000000..d9504710b3f
--- /dev/null
+++ b/lib/Target/PowerPC/PPCVSXSwapRemoval.cpp
@@ -0,0 +1,1003 @@
+//===----------- PPCVSXSwapRemoval.cpp - Remove VSX LE Swaps -------------===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===---------------------------------------------------------------------===//
+//
+// This pass analyzes vector computations and removes unnecessary
+// doubleword swaps (xxswapd instructions).  This pass is performed
+// only for little-endian VSX code generation.
+//
+// For this specific case, loads and stores of v4i32, v4f32, v2i64,
+// and v2f64 vectors are inefficient.  These are implemented using
+// the lxvd2x and stxvd2x instructions, which invert the order of
+// doublewords in a vector register.  Thus code generation inserts
+// an xxswapd after each such load, and prior to each such store.
+//
+// The extra xxswapd instructions reduce performance.  The purpose
+// of this pass is to reduce the number of xxswapd instructions
+// required for correctness.
+//
+// The primary insight is that much code that operates on vectors
+// does not care about the relative order of elements in a register,
+// so long as the correct memory order is preserved.  If we have a
+// computation where all input values are provided by lxvd2x/xxswapd,
+// all outputs are stored using xxswapd/lxvd2x, and all intermediate
+// computations are lane-insensitive (independent of element order),
+// then all the xxswapd instructions associated with the loads and
+// stores may be removed without changing observable semantics.
+//
+// This pass uses standard equivalence class infrastructure to create
+// maximal webs of computations fitting the above description.  Each
+// such web is then optimized by removing its unnecessary xxswapd
+// instructions.
+//
+// There are some lane-sensitive operations for which we can still
+// permit the optimization, provided we modify those operations
+// accordingly.  Such operations are identified as using "special
+// handling" within this module.
+//
+//===---------------------------------------------------------------------===//
+
+#include "PPCInstrInfo.h"
+#include "PPC.h"
+#include "PPCInstrBuilder.h"
+#include "PPCTargetMachine.h"
+#include "llvm/ADT/DenseMap.h"
+#include "llvm/ADT/EquivalenceClasses.h"
+#include "llvm/CodeGen/MachineFunctionPass.h"
+#include "llvm/CodeGen/MachineInstrBuilder.h"
+#include "llvm/CodeGen/MachineRegisterInfo.h"
+#include "llvm/Support/Debug.h"
+#include "llvm/Support/Format.h"
+#include "llvm/Support/raw_ostream.h"
+
+using namespace llvm;
+
+#define DEBUG_TYPE "ppc-vsx-swaps"
+
+namespace llvm {
+  void initializePPCVSXSwapRemovalPass(PassRegistry&);
+}
+
+namespace {
+
+// A PPCVSXSwapEntry is created for each machine instruction that
+// is relevant to a vector computation.
+struct PPCVSXSwapEntry {
+  // Pointer to the instruction.
+  MachineInstr *VSEMI;
+
+  // Unique ID (position in the swap vector).
+  int VSEId;
+
+  // Attributes of this node.
+  unsigned int IsLoad : 1;
+  unsigned int IsStore : 1;
+  unsigned int IsSwap : 1;
+  unsigned int MentionsPhysVR : 1;
+  unsigned int IsSwappable : 1;
+  unsigned int MentionsPartialVR : 1;
+  unsigned int SpecialHandling : 3;
+  unsigned int WebRejected : 1;
+  unsigned int WillRemove : 1;
+};
+
+enum SHValues {
+  SH_NONE = 0,
+  SH_EXTRACT,
+  SH_INSERT,
+  SH_NOSWAP_LD,
+  SH_NOSWAP_ST,
+  SH_SPLAT,
+  SH_XXPERMDI,
+  SH_COPYWIDEN
+};
+
+struct PPCVSXSwapRemoval : public MachineFunctionPass {
+
+  static char ID;
+  const PPCInstrInfo *TII;
+  MachineFunction *MF;
+  MachineRegisterInfo *MRI;
+
+  // Swap entries are allocated in a vector for better performance.
+  std::vector<PPCVSXSwapEntry> SwapVector;
+
+  // A mapping is maintained between machine instructions and
+  // their swap entries.  The key is the address of the MI.
+  DenseMap<MachineInstr*, int> SwapMap;
+
+  // Equivalence classes are used to gather webs of related computation.
+  // Swap entries are represented by their VSEId fields.
+  EquivalenceClasses<int> *EC;
+
+  PPCVSXSwapRemoval() : MachineFunctionPass(ID) {
+    initializePPCVSXSwapRemovalPass(*PassRegistry::getPassRegistry());
+  }
+
+private:
+  // Initialize data structures.
+  void initialize(MachineFunction &MFParm);
+
+  // Walk the machine instructions to gather vector usage information.
+  // Return true iff vector mentions are present.
+  bool gatherVectorInstructions();
+
+  // Add an entry to the swap vector and swap map.
+  int addSwapEntry(MachineInstr *MI, PPCVSXSwapEntry &SwapEntry);
+
+  // Hunt backwards through COPY and SUBREG_TO_REG chains for a
+  // source register.  VecIdx indicates the swap vector entry to
+  // mark as mentioning a physical register if the search leads
+  // to one.
+  unsigned lookThruCopyLike(unsigned SrcReg, unsigned VecIdx);
+
+  // Generate equivalence classes for related computations (webs).
+  void formWebs();
+
+  // Analyze webs and determine those that cannot be optimized.
+  void recordUnoptimizableWebs();
+
+  // Record which swap instructions can be safely removed.
+  void markSwapsForRemoval();
+
+  // Remove swaps and update other instructions requiring special
+  // handling.  Return true iff any changes are made.
+  bool removeSwaps();
+
+  // Insert a swap instruction from SrcReg to DstReg at the given
+  // InsertPoint.
+  void insertSwap(MachineInstr *MI, MachineBasicBlock::iterator InsertPoint,
+                  unsigned DstReg, unsigned SrcReg);
+
+  // Update instructions requiring special handling.
+  void handleSpecialSwappables(int EntryIdx);
+
+  // Dump a description of the entries in the swap vector.
+  void dumpSwapVector();
+
+  // Return true iff the given register is in the given class.
+  bool isRegInClass(unsigned Reg, const TargetRegisterClass *RC) {
+    if (TargetRegisterInfo::isVirtualRegister(Reg))
+      return RC->hasSubClassEq(MRI->getRegClass(Reg));
+    if (RC->contains(Reg))
+      return true;
+    return false;
+  }
+
+  // Return true iff the given register is a full vector register.
+  bool isVecReg(unsigned Reg) {
+    return (isRegInClass(Reg, &PPC::VSRCRegClass) ||
+            isRegInClass(Reg, &PPC::VRRCRegClass));
+  }
+
+  // Return true iff the given register is a partial vector register.
+  bool isScalarVecReg(unsigned Reg) {
+    return (isRegInClass(Reg, &PPC::VSFRCRegClass) ||
+            isRegInClass(Reg, &PPC::VSSRCRegClass));
+  }
+
+  // Return true iff the given register mentions all or part of a
+  // vector register.  Also sets Partial to true if the mention
+  // is for just the floating-point register overlap of the register.
+  bool isAnyVecReg(unsigned Reg, bool &Partial) {
+    if (isScalarVecReg(Reg))
+      Partial = true;
+    return isScalarVecReg(Reg) || isVecReg(Reg);
+  }
+
+public:
+  // Main entry point for this pass.
+  bool runOnMachineFunction(MachineFunction &MF) override {
+    // If we don't have VSX on the subtarget, don't do anything.
+    const PPCSubtarget &STI = MF.getSubtarget<PPCSubtarget>();
+    if (!STI.hasVSX())
+      return false;
+
+    bool Changed = false;
+    initialize(MF);
+
+    if (gatherVectorInstructions()) {
+      formWebs();
+      recordUnoptimizableWebs();
+      markSwapsForRemoval();
+      Changed = removeSwaps();
+    }
+
+    // FIXME: See the allocation of EC in initialize().
+    delete EC;
+    return Changed;
+  }
+};
+
+// Initialize data structures for this pass.  In particular, clear the
+// swap vector and allocate the equivalence class mapping before
+// processing each function.
+void PPCVSXSwapRemoval::initialize(MachineFunction &MFParm) {
+  MF = &MFParm;
+  MRI = &MF->getRegInfo();
+  TII = static_cast<const PPCInstrInfo*>(MF->getSubtarget().getInstrInfo());
+
+  // An initial vector size of 256 appears to work well in practice.
+  // Small/medium functions with vector content tend not to incur a
+  // reallocation at this size.  Three of the vector tests in
+  // projects/test-suite reallocate, which seems like a reasonable rate.
+  const int InitialVectorSize(256);
+  SwapVector.clear();
+  SwapVector.reserve(InitialVectorSize);
+
+  // FIXME: Currently we allocate EC each time because we don't have
+  // access to the set representation on which to call clear().  Should
+  // consider adding a clear() method to the EquivalenceClasses class.
+  EC = new EquivalenceClasses<int>;
+}
+
+// Create an entry in the swap vector for each instruction that mentions
+// a full vector register, recording various characteristics of the
+// instructions there.
+bool PPCVSXSwapRemoval::gatherVectorInstructions() {
+  bool RelevantFunction = false;
+
+  for (MachineBasicBlock &MBB : *MF) {
+    for (MachineInstr &MI : MBB) {
+
+      bool RelevantInstr = false;
+      bool Partial = false;
+
+      for (const MachineOperand &MO : MI.operands()) {
+        if (!MO.isReg())
+          continue;
+        unsigned Reg = MO.getReg();
+        if (isAnyVecReg(Reg, Partial)) {
+          RelevantInstr = true;
+          break;
+        }
+      }
+
+      if (!RelevantInstr)
+        continue;
+
+      RelevantFunction = true;
+
+      // Create a SwapEntry initialized to zeros, then fill in the
+      // instruction and ID fields before pushing it to the back
+      // of the swap vector.
+      PPCVSXSwapEntry SwapEntry{};
+      int VecIdx = addSwapEntry(&MI, SwapEntry);
+
+      switch(MI.getOpcode()) {
+      default:
+        // Unless noted otherwise, an instruction is considered
+        // safe for the optimization.  There are a large number of
+        // such true-SIMD instructions (all vector math, logical,
+        // select, compare, etc.).  However, if the instruction
+        // mentions a partial vector register and does not have
+        // special handling defined, it is not swappable.
+        if (Partial)
+          SwapVector[VecIdx].MentionsPartialVR = 1;
+        else
+          SwapVector[VecIdx].IsSwappable = 1;
+        break;
+      case PPC::XXPERMDI: {
+        // This is a swap if it is of the form XXPERMDI t, s, s, 2.
+        // Unfortunately, MachineCSE ignores COPY and SUBREG_TO_REG, so we
+        // can also see XXPERMDI t, SUBREG_TO_REG(s), SUBREG_TO_REG(s), 2,
+        // for example.  We have to look through chains of COPY and
+        // SUBREG_TO_REG to find the real source value for comparison.
+        // If the real source value is a physical register, then mark the
+        // XXPERMDI as mentioning a physical register.
+        int immed = MI.getOperand(3).getImm();
+        if (immed == 2) {
+          unsigned trueReg1 = lookThruCopyLike(MI.getOperand(1).getReg(),
+                                               VecIdx);
+          unsigned trueReg2 = lookThruCopyLike(MI.getOperand(2).getReg(),
+                                               VecIdx);
+          if (trueReg1 == trueReg2)
+            SwapVector[VecIdx].IsSwap = 1;
+          else {
+            // We can still handle these if the two registers are not
+            // identical, by adjusting the form of the XXPERMDI.
+            SwapVector[VecIdx].IsSwappable = 1;
+            SwapVector[VecIdx].SpecialHandling = SHValues::SH_XXPERMDI;
+          }
+        // This is a doubleword splat if it is of the form
+        // XXPERMDI t, s, s, 0 or XXPERMDI t, s, s, 3.  As above we
+        // must look through chains of copy-likes to find the source
+        // register.  We turn off the marking for mention of a physical
+        // register, because splatting it is safe; the optimization
+        // will not swap the value in the physical register.  Whether
+        // or not the two input registers are identical, we can handle
+        // these by adjusting the form of the XXPERMDI.
+        } else if (immed == 0 || immed == 3) {
+
+          SwapVector[VecIdx].IsSwappable = 1;
+          SwapVector[VecIdx].SpecialHandling = SHValues::SH_XXPERMDI;
+
+          unsigned trueReg1 = lookThruCopyLike(MI.getOperand(1).getReg(),
+                                               VecIdx);
+          unsigned trueReg2 = lookThruCopyLike(MI.getOperand(2).getReg(),
+                                               VecIdx);
+          if (trueReg1 == trueReg2)
+            SwapVector[VecIdx].MentionsPhysVR = 0;
+
+        } else {
+          // We can still handle these by adjusting the form of the XXPERMDI.
+          SwapVector[VecIdx].IsSwappable = 1;
+          SwapVector[VecIdx].SpecialHandling = SHValues::SH_XXPERMDI;
+        }
+        break;
+      }
+      case PPC::LVX:
+        // Non-permuting loads are currently unsafe.  We can use special
+        // handling for this in the future.  By not marking these as
+        // IsSwap, we ensure computations containing them will be rejected
+        // for now.
+        SwapVector[VecIdx].IsLoad = 1;
+        break;
+      case PPC::LXVD2X:
+      case PPC::LXVW4X:
+        // Permuting loads are marked as both load and swap, and are
+        // safe for optimization.
+        SwapVector[VecIdx].IsLoad = 1;
+        SwapVector[VecIdx].IsSwap = 1;
+        break;
+      case PPC::LXSDX:
+      case PPC::LXSSPX:
+        // A load of a floating-point value into the high-order half of
+        // a vector register is safe, provided that we introduce a swap
+        // following the load, which will be done by the SUBREG_TO_REG
+        // support.  So just mark these as safe.
+        SwapVector[VecIdx].IsLoad = 1;
+        SwapVector[VecIdx].IsSwappable = 1;
+        break;
+      case PPC::STVX:
+        // Non-permuting stores are currently unsafe.  We can use special
+        // handling for this in the future.  By not marking these as
+        // IsSwap, we ensure computations containing them will be rejected
+        // for now.
+        SwapVector[VecIdx].IsStore = 1;
+        break;
+      case PPC::STXVD2X:
+      case PPC::STXVW4X:
+        // Permuting stores are marked as both store and swap, and are
+        // safe for optimization.
+        SwapVector[VecIdx].IsStore = 1;
+        SwapVector[VecIdx].IsSwap = 1;
+        break;
+      case PPC::COPY:
+        // These are fine provided they are moving between full vector
+        // register classes.
+        if (isVecReg(MI.getOperand(0).getReg()) &&
+            isVecReg(MI.getOperand(1).getReg()))
+          SwapVector[VecIdx].IsSwappable = 1;
+        // If we have a copy from one scalar floating-point register
+        // to another, we can accept this even if it is a physical
+        // register.  The only way this gets involved is if it feeds
+        // a SUBREG_TO_REG, which is handled by introducing a swap.
+        else if (isScalarVecReg(MI.getOperand(0).getReg()) &&
+                 isScalarVecReg(MI.getOperand(1).getReg()))
+          SwapVector[VecIdx].IsSwappable = 1;
+        break;
+      case PPC::SUBREG_TO_REG: {
+        // These are fine provided they are moving between full vector
+        // register classes.  If they are moving from a scalar
+        // floating-point class to a vector class, we can handle those
+        // as well, provided we introduce a swap.  It is generally the
+        // case that we will introduce fewer swaps than we remove, but
+        // (FIXME) a cost model could be used.  However, introduced
+        // swaps could potentially be CSEd, so this is not trivial.
+        if (isVecReg(MI.getOperand(0).getReg()) &&
+            isVecReg(MI.getOperand(2).getReg()))
+          SwapVector[VecIdx].IsSwappable = 1;
+        else if (isVecReg(MI.getOperand(0).getReg()) &&
+                 isScalarVecReg(MI.getOperand(2).getReg())) {
+          SwapVector[VecIdx].IsSwappable = 1;
+          SwapVector[VecIdx].SpecialHandling = SHValues::SH_COPYWIDEN;
+        }
+        break;
+      }
+      case PPC::VSPLTB:
+      case PPC::VSPLTH:
+      case PPC::VSPLTW:
+        // Splats are lane-sensitive, but we can use special handling
+        // to adjust the source lane for the splat.  This is not yet
+        // implemented.  When it is, we need to uncomment the following:
+        SwapVector[VecIdx].IsSwappable = 1;
+        SwapVector[VecIdx].SpecialHandling = SHValues::SH_SPLAT;
+        break;
+      // The presence of the following lane-sensitive operations in a
+      // web will kill the optimization, at least for now.  For these
+      // we do nothing, causing the optimization to fail.
+      // FIXME: Some of these could be permitted with special handling,
+      // and will be phased in as time permits.
+      // FIXME: There is no simple and maintainable way to express a set
+      // of opcodes having a common attribute in TableGen.  Should this
+      // change, this is a prime candidate to use such a mechanism.
+      case PPC::INLINEASM:
+      case PPC::EXTRACT_SUBREG:
+      case PPC::INSERT_SUBREG:
+      case PPC::COPY_TO_REGCLASS:
+      case PPC::LVEBX:
+      case PPC::LVEHX:
+      case PPC::LVEWX:
+      case PPC::LVSL:
+      case PPC::LVSR:
+      case PPC::LVXL:
+      case PPC::STVEBX:
+      case PPC::STVEHX:
+      case PPC::STVEWX:
+      case PPC::STVXL:
+        // We can handle STXSDX and STXSSPX similarly to LXSDX and LXSSPX,
+        // by adding special handling for narrowing copies as well as
+        // widening ones.  However, I've experimented with this, and in
+        // practice we currently do not appear to use STXSDX fed by 
+        // a narrowing copy from a full vector register.  Since I can't
+        // generate any useful test cases, I've left this alone for now.
+      case PPC::STXSDX:
+      case PPC::STXSSPX:
+      case PPC::VCIPHER:
+      case PPC::VCIPHERLAST:
+      case PPC::VMRGHB:
+      case PPC::VMRGHH:
+      case PPC::VMRGHW:
+      case PPC::VMRGLB:
+      case PPC::VMRGLH:
+      case PPC::VMRGLW:
+      case PPC::VMULESB:
+      case PPC::VMULESH:
+      case PPC::VMULESW:
+      case PPC::VMULEUB:
+      case PPC::VMULEUH:
+      case PPC::VMULEUW:
+      case PPC::VMULOSB:
+      case PPC::VMULOSH:
+      case PPC::VMULOSW:
+      case PPC::VMULOUB:
+      case PPC::VMULOUH:
+      case PPC::VMULOUW:
+      case PPC::VNCIPHER:
+      case PPC::VNCIPHERLAST:
+      case PPC::VPERM:
+      case PPC::VPERMXOR:
+      case PPC::VPKPX:
+      case PPC::VPKSHSS:
+      case PPC::VPKSHUS:
+      case PPC::VPKSDSS:
+      case PPC::VPKSDUS:
+      case PPC::VPKSWSS:
+      case PPC::VPKSWUS:
+      case PPC::VPKUDUM:
+      case PPC::VPKUDUS:
+      case PPC::VPKUHUM:
+      case PPC::VPKUHUS:
+      case PPC::VPKUWUM:
+      case PPC::VPKUWUS:
+      case PPC::VPMSUMB:
+      case PPC::VPMSUMD:
+      case PPC::VPMSUMH:
+      case PPC::VPMSUMW:
+      case PPC::VRLB:
+      case PPC::VRLD:
+      case PPC::VRLH:
+      case PPC::VRLW:
+      case PPC::VSBOX:
+      case PPC::VSHASIGMAD:
+      case PPC::VSHASIGMAW:
+      case PPC::VSL:
+      case PPC::VSLDOI:
+      case PPC::VSLO:
+      case PPC::VSR:
+      case PPC::VSRO:
+      case PPC::VSUM2SWS:
+      case PPC::VSUM4SBS:
+      case PPC::VSUM4SHS:
+      case PPC::VSUM4UBS:
+      case PPC::VSUMSWS:
+      case PPC::VUPKHPX:
+      case PPC::VUPKHSB:
+      case PPC::VUPKHSH:
+      case PPC::VUPKHSW:
+      case PPC::VUPKLPX:
+      case PPC::VUPKLSB:
+      case PPC::VUPKLSH:
+      case PPC::VUPKLSW:
+      case PPC::XXMRGHW:
+      case PPC::XXMRGLW:
+      // XXSLDWI could be replaced by a general permute with one of three
+      // permute control vectors (for shift values 1, 2, 3).  However,
+      // VPERM has a more restrictive register class.
+      case PPC::XXSLDWI:
+      case PPC::XXSPLTW:
+        break;
+      }
+    }
+  }
+
+  if (RelevantFunction) {
+    DEBUG(dbgs() << "Swap vector when first built\n\n");
+    dumpSwapVector();
+  }
+
+  return RelevantFunction;
+}
+
+// Add an entry to the swap vector and swap map, and make a
+// singleton equivalence class for the entry.
+int PPCVSXSwapRemoval::addSwapEntry(MachineInstr *MI,
+                                  PPCVSXSwapEntry& SwapEntry) {
+  SwapEntry.VSEMI = MI;
+  SwapEntry.VSEId = SwapVector.size();
+  SwapVector.push_back(SwapEntry);
+  EC->insert(SwapEntry.VSEId);
+  SwapMap[MI] = SwapEntry.VSEId;
+  return SwapEntry.VSEId;
+}
+
+// This is used to find the "true" source register for an
+// XXPERMDI instruction, since MachineCSE does not handle the
+// "copy-like" operations (Copy and SubregToReg).  Returns
+// the original SrcReg unless it is the target of a copy-like
+// operation, in which case we chain backwards through all
+// such operations to the ultimate source register.  If a
+// physical register is encountered, we stop the search and
+// flag the swap entry indicated by VecIdx (the original
+// XXPERMDI) as mentioning a physical register.
+unsigned PPCVSXSwapRemoval::lookThruCopyLike(unsigned SrcReg,
+                                             unsigned VecIdx) {
+  MachineInstr *MI = MRI->getVRegDef(SrcReg);
+  if (!MI->isCopyLike())
+    return SrcReg;
+
+  unsigned CopySrcReg;
+  if (MI->isCopy())
+    CopySrcReg = MI->getOperand(1).getReg();
+  else {
+    assert(MI->isSubregToReg() && "bad opcode for lookThruCopyLike");
+    CopySrcReg = MI->getOperand(2).getReg();
+  }
+
+  if (!TargetRegisterInfo::isVirtualRegister(CopySrcReg)) {
+    if (!isScalarVecReg(CopySrcReg))
+      SwapVector[VecIdx].MentionsPhysVR = 1;
+    return CopySrcReg;
+  }
+
+  return lookThruCopyLike(CopySrcReg, VecIdx);
+}
+
+// Generate equivalence classes for related computations (webs) by
+// def-use relationships of virtual registers.  Mention of a physical
+// register terminates the generation of equivalence classes as this
+// indicates a use of a parameter, definition of a return value, use
+// of a value returned from a call, or definition of a parameter to a
+// call.  Computations with physical register mentions are flagged
+// as such so their containing webs will not be optimized.
+void PPCVSXSwapRemoval::formWebs() {
+
+  DEBUG(dbgs() << "\n*** Forming webs for swap removal ***\n\n");
+
+  for (unsigned EntryIdx = 0; EntryIdx < SwapVector.size(); ++EntryIdx) {
+
+    MachineInstr *MI = SwapVector[EntryIdx].VSEMI;
+
+    DEBUG(dbgs() << "\n" << SwapVector[EntryIdx].VSEId << " ");
+    DEBUG(MI->dump());
+
+    // It's sufficient to walk vector uses and join them to their unique
+    // definitions.  In addition, check full vector register operands
+    // for physical regs.  We exclude partial-vector register operands
+    // because we can handle them if copied to a full vector.
+    for (const MachineOperand &MO : MI->operands()) {
+      if (!MO.isReg())
+        continue;
+
+      unsigned Reg = MO.getReg();
+      if (!isVecReg(Reg) && !isScalarVecReg(Reg))
+        continue;
+
+      if (!TargetRegisterInfo::isVirtualRegister(Reg)) {
+        if (!(MI->isCopy() && isScalarVecReg(Reg)))
+          SwapVector[EntryIdx].MentionsPhysVR = 1;
+        continue;
+      }
+
+      if (!MO.isUse())
+        continue;
+
+      MachineInstr* DefMI = MRI->getVRegDef(Reg);
+      assert(SwapMap.find(DefMI) != SwapMap.end() &&
+             "Inconsistency: def of vector reg not found in swap map!");
+      int DefIdx = SwapMap[DefMI];
+      (void)EC->unionSets(SwapVector[DefIdx].VSEId,
+                          SwapVector[EntryIdx].VSEId);
+
+      DEBUG(dbgs() << format("Unioning %d with %d\n", SwapVector[DefIdx].VSEId,
+                             SwapVector[EntryIdx].VSEId));
+      DEBUG(dbgs() << "  Def: ");
+      DEBUG(DefMI->dump());
+    }
+  }
+}
+
+// Walk the swap vector entries looking for conditions that prevent their
+// containing computations from being optimized.  When such conditions are
+// found, mark the representative of the computation's equivalence class
+// as rejected.
+void PPCVSXSwapRemoval::recordUnoptimizableWebs() {
+
+  DEBUG(dbgs() << "\n*** Rejecting webs for swap removal ***\n\n");
+
+  for (unsigned EntryIdx = 0; EntryIdx < SwapVector.size(); ++EntryIdx) {
+    int Repr = EC->getLeaderValue(SwapVector[EntryIdx].VSEId);
+
+    // If representative is already rejected, don't waste further time.
+    if (SwapVector[Repr].WebRejected)
+      continue;
+
+    // Reject webs containing mentions of physical or partial registers, or
+    // containing operations that we don't know how to handle in a lane-
+    // permuted region.
+    if (SwapVector[EntryIdx].MentionsPhysVR ||
+        SwapVector[EntryIdx].MentionsPartialVR ||
+        !(SwapVector[EntryIdx].IsSwappable || SwapVector[EntryIdx].IsSwap)) {
+
+      SwapVector[Repr].WebRejected = 1;
+
+      DEBUG(dbgs() <<
+            format("Web %d rejected for physreg, partial reg, or not "
+                   "swap[pable]\n", Repr));
+      DEBUG(dbgs() << "  in " << EntryIdx << ": ");
+      DEBUG(SwapVector[EntryIdx].VSEMI->dump());
+      DEBUG(dbgs() << "\n");
+    }
+
+    // Reject webs than contain swapping loads that feed something other
+    // than a swap instruction.
+    else if (SwapVector[EntryIdx].IsLoad && SwapVector[EntryIdx].IsSwap) {
+      MachineInstr *MI = SwapVector[EntryIdx].VSEMI;
+      unsigned DefReg = MI->getOperand(0).getReg();
+
+      // We skip debug instructions in the analysis.  (Note that debug
+      // location information is still maintained by this optimization
+      // because it remains on the LXVD2X and STXVD2X instructions after
+      // the XXPERMDIs are removed.)
+      for (MachineInstr &UseMI : MRI->use_nodbg_instructions(DefReg)) {
+        int UseIdx = SwapMap[&UseMI];
+
+        if (!SwapVector[UseIdx].IsSwap || SwapVector[UseIdx].IsLoad ||
+            SwapVector[UseIdx].IsStore) {
+
+          SwapVector[Repr].WebRejected = 1;
+
+          DEBUG(dbgs() <<
+                format("Web %d rejected for load not feeding swap\n", Repr));
+          DEBUG(dbgs() << "  def " << EntryIdx << ": ");
+          DEBUG(MI->dump());
+          DEBUG(dbgs() << "  use " << UseIdx << ": ");
+          DEBUG(UseMI.dump());
+          DEBUG(dbgs() << "\n");
+        }
+      }
+
+    // Reject webs that contain swapping stores that are fed by something
+    // other than a swap instruction.
+    } else if (SwapVector[EntryIdx].IsStore && SwapVector[EntryIdx].IsSwap) {
+      MachineInstr *MI = SwapVector[EntryIdx].VSEMI;
+      unsigned UseReg = MI->getOperand(0).getReg();
+      MachineInstr *DefMI = MRI->getVRegDef(UseReg);
+      int DefIdx = SwapMap[DefMI];
+
+      if (!SwapVector[DefIdx].IsSwap || SwapVector[DefIdx].IsLoad ||
+          SwapVector[DefIdx].IsStore) {
+
+        SwapVector[Repr].WebRejected = 1;
+
+        DEBUG(dbgs() <<
+              format("Web %d rejected for store not fed by swap\n", Repr));
+        DEBUG(dbgs() << "  def " << DefIdx << ": ");
+        DEBUG(DefMI->dump());
+        DEBUG(dbgs() << "  use " << EntryIdx << ": ");
+        DEBUG(MI->dump());
+        DEBUG(dbgs() << "\n");
+      }
+    }
+  }
+
+  DEBUG(dbgs() << "Swap vector after web analysis:\n\n");
+  dumpSwapVector();
+}
+
+// Walk the swap vector entries looking for swaps fed by permuting loads
+// and swaps that feed permuting stores.  If the containing computation
+// has not been marked rejected, mark each such swap for removal.
+// (Removal is delayed in case optimization has disturbed the pattern,
+// such that multiple loads feed the same swap, etc.)
+void PPCVSXSwapRemoval::markSwapsForRemoval() {
+
+  DEBUG(dbgs() << "\n*** Marking swaps for removal ***\n\n");
+
+  for (unsigned EntryIdx = 0; EntryIdx < SwapVector.size(); ++EntryIdx) {
+
+    if (SwapVector[EntryIdx].IsLoad && SwapVector[EntryIdx].IsSwap) {
+      int Repr = EC->getLeaderValue(SwapVector[EntryIdx].VSEId);
+
+      if (!SwapVector[Repr].WebRejected) {
+        MachineInstr *MI = SwapVector[EntryIdx].VSEMI;
+        unsigned DefReg = MI->getOperand(0).getReg();
+
+        for (MachineInstr &UseMI : MRI->use_nodbg_instructions(DefReg)) {
+          int UseIdx = SwapMap[&UseMI];
+          SwapVector[UseIdx].WillRemove = 1;
+
+          DEBUG(dbgs() << "Marking swap fed by load for removal: ");
+          DEBUG(UseMI.dump());
+        }
+      }
+
+    } else if (SwapVector[EntryIdx].IsStore && SwapVector[EntryIdx].IsSwap) {
+      int Repr = EC->getLeaderValue(SwapVector[EntryIdx].VSEId);
+
+      if (!SwapVector[Repr].WebRejected) {
+        MachineInstr *MI = SwapVector[EntryIdx].VSEMI;
+        unsigned UseReg = MI->getOperand(0).getReg();
+        MachineInstr *DefMI = MRI->getVRegDef(UseReg);
+        int DefIdx = SwapMap[DefMI];
+        SwapVector[DefIdx].WillRemove = 1;
+
+        DEBUG(dbgs() << "Marking swap feeding store for removal: ");
+        DEBUG(DefMI->dump());
+      }
+
+    } else if (SwapVector[EntryIdx].IsSwappable &&
+               SwapVector[EntryIdx].SpecialHandling != 0) {
+      int Repr = EC->getLeaderValue(SwapVector[EntryIdx].VSEId);
+
+      if (!SwapVector[Repr].WebRejected)
+        handleSpecialSwappables(EntryIdx);
+    }
+  }
+}
+
+// Create an xxswapd instruction and insert it prior to the given point.
+// MI is used to determine basic block and debug loc information.
+// FIXME: When inserting a swap, we should check whether SrcReg is
+// defined by another swap:  SrcReg = XXPERMDI Reg, Reg, 2;  If so,
+// then instead we should generate a copy from Reg to DstReg.
+void PPCVSXSwapRemoval::insertSwap(MachineInstr *MI,
+                                   MachineBasicBlock::iterator InsertPoint,
+                                   unsigned DstReg, unsigned SrcReg) {
+  BuildMI(*MI->getParent(), InsertPoint, MI->getDebugLoc(),
+          TII->get(PPC::XXPERMDI), DstReg)
+    .addReg(SrcReg)
+    .addReg(SrcReg)
+    .addImm(2);
+}
+
+// The identified swap entry requires special handling to allow its
+// containing computation to be optimized.  Perform that handling
+// here.
+// FIXME: Additional opportunities will be phased in with subsequent
+// patches.
+void PPCVSXSwapRemoval::handleSpecialSwappables(int EntryIdx) {
+  switch (SwapVector[EntryIdx].SpecialHandling) {
+
+  default:
+    assert(false && "Unexpected special handling type");
+    break;
+
+  // For splats based on an index into a vector, add N/2 modulo N
+  // to the index, where N is the number of vector elements.
+  case SHValues::SH_SPLAT: {
+    MachineInstr *MI = SwapVector[EntryIdx].VSEMI;
+    unsigned NElts;
+
+    DEBUG(dbgs() << "Changing splat: ");
+    DEBUG(MI->dump());
+
+    switch (MI->getOpcode()) {
+    default:
+      assert(false && "Unexpected splat opcode");
+    case PPC::VSPLTB: NElts = 16; break;
+    case PPC::VSPLTH: NElts = 8;  break;
+    case PPC::VSPLTW: NElts = 4;  break;
+    }
+
+    unsigned EltNo = MI->getOperand(1).getImm();
+    EltNo = (EltNo + NElts / 2) % NElts;
+    MI->getOperand(1).setImm(EltNo);
+
+    DEBUG(dbgs() << "  Into: ");
+    DEBUG(MI->dump());
+    break;
+  }
+
+  // For an XXPERMDI that isn't handled otherwise, we need to
+  // reverse the order of the operands.  If the selector operand
+  // has a value of 0 or 3, we need to change it to 3 or 0,
+  // respectively.  Otherwise we should leave it alone.  (This
+  // is equivalent to reversing the two bits of the selector
+  // operand and complementing the result.)
+  case SHValues::SH_XXPERMDI: {
+    MachineInstr *MI = SwapVector[EntryIdx].VSEMI;
+
+    DEBUG(dbgs() << "Changing XXPERMDI: ");
+    DEBUG(MI->dump());
+
+    unsigned Selector = MI->getOperand(3).getImm();
+    if (Selector == 0 || Selector == 3)
+      Selector = 3 - Selector;
+    MI->getOperand(3).setImm(Selector);
+
+    unsigned Reg1 = MI->getOperand(1).getReg();
+    unsigned Reg2 = MI->getOperand(2).getReg();
+    MI->getOperand(1).setReg(Reg2);
+    MI->getOperand(2).setReg(Reg1);
+
+    DEBUG(dbgs() << "  Into: ");
+    DEBUG(MI->dump());
+    break;
+  }
+
+  // For a copy from a scalar floating-point register to a vector
+  // register, removing swaps will leave the copied value in the
+  // wrong lane.  Insert a swap following the copy to fix this.
+  case SHValues::SH_COPYWIDEN: {
+    MachineInstr *MI = SwapVector[EntryIdx].VSEMI;
+
+    DEBUG(dbgs() << "Changing SUBREG_TO_REG: ");
+    DEBUG(MI->dump());
+
+    unsigned DstReg = MI->getOperand(0).getReg();
+    const TargetRegisterClass *DstRC = MRI->getRegClass(DstReg);
+    unsigned NewVReg = MRI->createVirtualRegister(DstRC);
+
+    MI->getOperand(0).setReg(NewVReg);
+    DEBUG(dbgs() << "  Into: ");
+    DEBUG(MI->dump());
+
+    MachineBasicBlock::iterator InsertPoint = MI->getNextNode();
+
+    // Note that an XXPERMDI requires a VSRC, so if the SUBREG_TO_REG
+    // is copying to a VRRC, we need to be careful to avoid a register
+    // assignment problem.  In this case we must copy from VRRC to VSRC
+    // prior to the swap, and from VSRC to VRRC following the swap.
+    // Coalescing will usually remove all this mess.
+    if (DstRC == &PPC::VRRCRegClass) {
+      unsigned VSRCTmp1 = MRI->createVirtualRegister(&PPC::VSRCRegClass);
+      unsigned VSRCTmp2 = MRI->createVirtualRegister(&PPC::VSRCRegClass);
+
+      BuildMI(*MI->getParent(), InsertPoint, MI->getDebugLoc(),
+              TII->get(PPC::COPY), VSRCTmp1)
+        .addReg(NewVReg);
+      DEBUG(MI->getNextNode()->dump());
+
+      insertSwap(MI, InsertPoint, VSRCTmp2, VSRCTmp1);
+      DEBUG(MI->getNextNode()->getNextNode()->dump());
+
+      BuildMI(*MI->getParent(), InsertPoint, MI->getDebugLoc(),
+              TII->get(PPC::COPY), DstReg)
+        .addReg(VSRCTmp2);
+      DEBUG(MI->getNextNode()->getNextNode()->getNextNode()->dump());
+
+    } else {
+      insertSwap(MI, InsertPoint, DstReg, NewVReg);
+      DEBUG(MI->getNextNode()->dump());
+    }
+    break;
+  }
+  }
+}
+
+// Walk the swap vector and replace each entry marked for removal with
+// a copy operation.
+bool PPCVSXSwapRemoval::removeSwaps() {
+
+  DEBUG(dbgs() << "\n*** Removing swaps ***\n\n");
+
+  bool Changed = false;
+
+  for (unsigned EntryIdx = 0; EntryIdx < SwapVector.size(); ++EntryIdx) {
+    if (SwapVector[EntryIdx].WillRemove) {
+      Changed = true;
+      MachineInstr *MI = SwapVector[EntryIdx].VSEMI;
+      MachineBasicBlock *MBB = MI->getParent();
+      BuildMI(*MBB, MI, MI->getDebugLoc(),
+              TII->get(TargetOpcode::COPY), MI->getOperand(0).getReg())
+        .addOperand(MI->getOperand(1));
+
+      DEBUG(dbgs() << format("Replaced %d with copy: ",
+                             SwapVector[EntryIdx].VSEId));
+      DEBUG(MI->dump());
+
+      MI->eraseFromParent();
+    }
+  }
+
+  return Changed;
+}
+
+// For debug purposes, dump the contents of the swap vector.
+void PPCVSXSwapRemoval::dumpSwapVector() {
+
+  for (unsigned EntryIdx = 0; EntryIdx < SwapVector.size(); ++EntryIdx) {
+
+    MachineInstr *MI = SwapVector[EntryIdx].VSEMI;
+    int ID = SwapVector[EntryIdx].VSEId;
+
+    DEBUG(dbgs() << format("%6d", ID));
+    DEBUG(dbgs() << format("%6d", EC->getLeaderValue(ID)));
+    DEBUG(dbgs() << format(" BB#%3d", MI->getParent()->getNumber()));
+    DEBUG(dbgs() << format("  %14s  ", TII->getName(MI->getOpcode())));
+
+    if (SwapVector[EntryIdx].IsLoad)
+      DEBUG(dbgs() << "load ");
+    if (SwapVector[EntryIdx].IsStore)
+      DEBUG(dbgs() << "store ");
+    if (SwapVector[EntryIdx].IsSwap)
+      DEBUG(dbgs() << "swap ");
+    if (SwapVector[EntryIdx].MentionsPhysVR)
+      DEBUG(dbgs() << "physreg ");
+    if (SwapVector[EntryIdx].MentionsPartialVR)
+      DEBUG(dbgs() << "partialreg ");
+
+    if (SwapVector[EntryIdx].IsSwappable) {
+      DEBUG(dbgs() << "swappable ");
+      switch(SwapVector[EntryIdx].SpecialHandling) {
+      default:
+        DEBUG(dbgs() << "special:**unknown**");
+        break;
+      case SH_NONE:
+        break;
+      case SH_EXTRACT:
+        DEBUG(dbgs() << "special:extract ");
+        break;
+      case SH_INSERT:
+        DEBUG(dbgs() << "special:insert ");
+        break;
+      case SH_NOSWAP_LD:
+        DEBUG(dbgs() << "special:load ");
+        break;
+      case SH_NOSWAP_ST:
+        DEBUG(dbgs() << "special:store ");
+        break;
+      case SH_SPLAT:
+        DEBUG(dbgs() << "special:splat ");
+        break;
+      case SH_XXPERMDI:
+        DEBUG(dbgs() << "special:xxpermdi ");
+        break;
+      case SH_COPYWIDEN:
+        DEBUG(dbgs() << "special:copywiden ");
+        break;
+      }
+    }
+
+    if (SwapVector[EntryIdx].WebRejected)
+      DEBUG(dbgs() << "rejected ");
+    if (SwapVector[EntryIdx].WillRemove)
+      DEBUG(dbgs() << "remove ");
+
+    DEBUG(dbgs() << "\n");
+
+    // For no-asserts builds.
+    (void)MI;
+    (void)ID;
+  }
+
+  DEBUG(dbgs() << "\n");
+}
+
+} // end default namespace
+
+INITIALIZE_PASS_BEGIN(PPCVSXSwapRemoval, DEBUG_TYPE,
+                      "PowerPC VSX Swap Removal", false, false)
+INITIALIZE_PASS_END(PPCVSXSwapRemoval, DEBUG_TYPE,
+                    "PowerPC VSX Swap Removal", false, false)
+
+char PPCVSXSwapRemoval::ID = 0;
+FunctionPass*
+llvm::createPPCVSXSwapRemovalPass() { return new PPCVSXSwapRemoval(); }
diff --git a/lib/Target/PowerPC/README_ALTIVEC.txt b/lib/Target/PowerPC/README_ALTIVEC.txt
index 1d5b0922a9f..f70ebd82bd5 100644
--- a/lib/Target/PowerPC/README_ALTIVEC.txt
+++ b/lib/Target/PowerPC/README_ALTIVEC.txt
@@ -313,3 +313,31 @@ Opportunies to use instructions from PPCInstrVSX.td during code gen
 Related to this: we currently do not generate the lxvw4x instruction for either
 v4f32 or v4i32, probably because adding a dag pattern to the recognizer requires
 a single target type.  This should probably be addressed in the PPCISelDAGToDAG logic.
+
+//===----------------------------------------------------------------------===//
+
+Currently EXTRACT_VECTOR_ELT and INSERT_VECTOR_ELT are type-legal only
+for v2f64 with VSX available.  We should create custom lowering
+support for the other vector types.  Without this support, we generate
+sequences with load-hit-store hazards.
+
+v4f32 can be supported with VSX by shifting the correct element into
+big-endian lane 0, using xscvspdpn to produce a double-precision
+representation of the single-precision value in big-endian
+double-precision lane 0, and reinterpreting lane 0 as an FPR or
+vector-scalar register.
+
+v2i64 can be supported with VSX and P8Vector in the same manner as
+v2f64, followed by a direct move to a GPR.
+
+v4i32 can be supported with VSX and P8Vector by shifting the correct
+element into big-endian lane 1, using a direct move to a GPR, and
+sign-extending the 32-bit result to 64 bits.
+
+v8i16 can be supported with VSX and P8Vector by shifting the correct
+element into big-endian lane 3, using a direct move to a GPR, and
+sign-extending the 16-bit result to 64 bits.
+
+v16i8 can be supported with VSX and P8Vector by shifting the correct
+element into big-endian lane 7, using a direct move to a GPR, and
+sign-extending the 8-bit result to 64 bits.
diff --git a/lib/Target/PowerPC/TargetInfo/PowerPCTargetInfo.cpp b/lib/Target/PowerPC/TargetInfo/PowerPCTargetInfo.cpp
index 5727dbc4170..5b2fe19837e 100644
--- a/lib/Target/PowerPC/TargetInfo/PowerPCTargetInfo.cpp
+++ b/lib/Target/PowerPC/TargetInfo/PowerPCTargetInfo.cpp
@@ -14,7 +14,7 @@ using namespace llvm;
 
 Target llvm::ThePPC32Target, llvm::ThePPC64Target, llvm::ThePPC64LETarget;
 
-extern "C" void LLVMInitializePowerPCTargetInfo() { 
+extern "C" void LLVMInitializePowerPCTargetInfo() {
   RegisterTarget<Triple::ppc, /*HasJIT=*/true>
     X(ThePPC32Target, "ppc32", "PowerPC 32");
 
diff --git a/lib/Target/R600/AMDGPU.td b/lib/Target/R600/AMDGPU.td
deleted file mode 100644
index 2eb805e814f..00000000000
--- a/lib/Target/R600/AMDGPU.td
+++ /dev/null
@@ -1,231 +0,0 @@
-//===-- AMDIL.td - AMDIL Tablegen files --*- tablegen -*-------------------===//
-//
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
-//
-//==-----------------------------------------------------------------------===//
-
-include "llvm/Target/Target.td"
-
-//===----------------------------------------------------------------------===//
-// Subtarget Features
-//===----------------------------------------------------------------------===//
-
-// Debugging Features
-
-def FeatureDumpCode : SubtargetFeature <"DumpCode",
-        "DumpCode",
-        "true",
-        "Dump MachineInstrs in the CodeEmitter">;
-
-def FeatureIRStructurizer : SubtargetFeature <"disable-irstructurizer",
-        "EnableIRStructurizer",
-        "false",
-        "Disable IR Structurizer">;
-
-def FeaturePromoteAlloca : SubtargetFeature <"promote-alloca",
-        "EnablePromoteAlloca",
-        "true",
-        "Enable promote alloca pass">;
-
-// Target features
-
-def FeatureIfCvt : SubtargetFeature <"disable-ifcvt",
-        "EnableIfCvt",
-        "false",
-        "Disable the if conversion pass">;
-
-def FeatureFP64 : SubtargetFeature<"fp64",
-        "FP64",
-        "true",
-        "Enable double precision operations">;
-
-def FeatureFP64Denormals : SubtargetFeature<"fp64-denormals",
-        "FP64Denormals",
-        "true",
-        "Enable double precision denormal handling",
-        [FeatureFP64]>;
-
-def FeatureFastFMAF32 : SubtargetFeature<"fast-fmaf",
-        "FastFMAF32",
-        "true",
-        "Assuming f32 fma is at least as fast as mul + add",
-        []>;
-
-// Some instructions do not support denormals despite this flag. Using
-// fp32 denormals also causes instructions to run at the double
-// precision rate for the device.
-def FeatureFP32Denormals : SubtargetFeature<"fp32-denormals",
-        "FP32Denormals",
-        "true",
-        "Enable single precision denormal handling">;
-
-def Feature64BitPtr : SubtargetFeature<"64BitPtr",
-        "Is64bit",
-        "true",
-        "Specify if 64-bit addressing should be used">;
-
-def FeatureR600ALUInst : SubtargetFeature<"R600ALUInst",
-        "R600ALUInst",
-        "false",
-        "Older version of ALU instructions encoding">;
-
-def FeatureVertexCache : SubtargetFeature<"HasVertexCache",
-        "HasVertexCache",
-        "true",
-        "Specify use of dedicated vertex cache">;
-
-def FeatureCaymanISA : SubtargetFeature<"caymanISA",
-        "CaymanISA",
-        "true",
-        "Use Cayman ISA">;
-
-def FeatureCFALUBug : SubtargetFeature<"cfalubug",
-        "CFALUBug",
-        "true",
-        "GPU has CF_ALU bug">;
-
-// XXX - This should probably be removed once enabled by default
-def FeatureEnableLoadStoreOpt : SubtargetFeature <"load-store-opt",
-        "EnableLoadStoreOpt",
-        "true",
-        "Enable SI load/store optimizer pass">;
-
-def FeatureFlatAddressSpace : SubtargetFeature<"flat-address-space",
-        "FlatAddressSpace",
-        "true",
-        "Support flat address space">;
-
-def FeatureVGPRSpilling : SubtargetFeature<"vgpr-spilling",
-        "EnableVGPRSpilling",
-        "true",
-        "Enable spilling of VGPRs to scratch memory">;
-
-def FeatureSGPRInitBug : SubtargetFeature<"sgpr-init-bug",
-        "SGPRInitBug",
-        "true",
-        "VI SGPR initilization bug requiring a fixed SGPR allocation size">;
-
-class SubtargetFeatureFetchLimit <string Value> :
-                          SubtargetFeature <"fetch"#Value,
-        "TexVTXClauseSize",
-        Value,
-        "Limit the maximum number of fetches in a clause to "#Value>;
-
-def FeatureFetchLimit8 : SubtargetFeatureFetchLimit <"8">;
-def FeatureFetchLimit16 : SubtargetFeatureFetchLimit <"16">;
-
-class SubtargetFeatureWavefrontSize <int Value> : SubtargetFeature<
-        "wavefrontsize"#Value,
-        "WavefrontSize",
-        !cast<string>(Value),
-        "The number of threads per wavefront">;
-
-def FeatureWavefrontSize16 : SubtargetFeatureWavefrontSize<16>;
-def FeatureWavefrontSize32 : SubtargetFeatureWavefrontSize<32>;
-def FeatureWavefrontSize64 : SubtargetFeatureWavefrontSize<64>;
-
-class SubtargetFeatureLocalMemorySize <int Value> : SubtargetFeature<
-        "localmemorysize"#Value,
-        "LocalMemorySize",
-        !cast<string>(Value),
-        "The size of local memory in bytes">;
-
-def FeatureGCN : SubtargetFeature<"gcn",
-        "IsGCN",
-        "true",
-        "GCN or newer GPU">;
-
-def FeatureGCN1Encoding : SubtargetFeature<"gcn1-encoding",
-        "GCN1Encoding",
-        "true",
-        "Encoding format for SI and CI">;
-
-def FeatureGCN3Encoding : SubtargetFeature<"gcn3-encoding",
-        "GCN3Encoding",
-        "true",
-        "Encoding format for VI">;
-class SubtargetFeatureGeneration <string Value,
-                                  list<SubtargetFeature> Implies> :
-        SubtargetFeature <Value, "Gen", "AMDGPUSubtarget::"#Value,
-                          Value#" GPU generation", Implies>;
-
-def FeatureLocalMemorySize0 : SubtargetFeatureLocalMemorySize<0>;
-def FeatureLocalMemorySize32768 : SubtargetFeatureLocalMemorySize<32768>;
-def FeatureLocalMemorySize65536 : SubtargetFeatureLocalMemorySize<65536>;
-
-def FeatureR600 : SubtargetFeatureGeneration<"R600",
-        [FeatureR600ALUInst, FeatureFetchLimit8, FeatureLocalMemorySize0]>;
-
-def FeatureR700 : SubtargetFeatureGeneration<"R700",
-        [FeatureFetchLimit16, FeatureLocalMemorySize0]>;
-
-def FeatureEvergreen : SubtargetFeatureGeneration<"EVERGREEN",
-        [FeatureFetchLimit16, FeatureLocalMemorySize32768]>;
-
-def FeatureNorthernIslands : SubtargetFeatureGeneration<"NORTHERN_ISLANDS",
-        [FeatureFetchLimit16, FeatureWavefrontSize64,
-         FeatureLocalMemorySize32768]
->;
-
-def FeatureSouthernIslands : SubtargetFeatureGeneration<"SOUTHERN_ISLANDS",
-        [Feature64BitPtr, FeatureFP64, FeatureLocalMemorySize32768,
-         FeatureWavefrontSize64, FeatureGCN, FeatureGCN1Encoding]>;
-
-def FeatureSeaIslands : SubtargetFeatureGeneration<"SEA_ISLANDS",
-        [Feature64BitPtr, FeatureFP64, FeatureLocalMemorySize65536,
-         FeatureWavefrontSize64, FeatureGCN, FeatureFlatAddressSpace,
-         FeatureGCN1Encoding]>;
-
-def FeatureVolcanicIslands : SubtargetFeatureGeneration<"VOLCANIC_ISLANDS",
-        [Feature64BitPtr, FeatureFP64, FeatureLocalMemorySize65536,
-         FeatureWavefrontSize64, FeatureFlatAddressSpace, FeatureGCN,
-         FeatureGCN3Encoding]>;
-
-//===----------------------------------------------------------------------===//
-
-def AMDGPUInstrInfo : InstrInfo {
-  let guessInstructionProperties = 1;
-  let noNamedPositionallyEncodedOperands = 1;
-}
-
-def AMDGPUAsmParser : AsmParser {
-  // Some of the R600 registers have the same name, so this crashes.
-  // For example T0_XYZW and T0_XY both have the asm name T0.
-  let ShouldEmitMatchRegisterName = 0;
-}
-
-def AMDGPU : Target {
-  // Pull in Instruction Info:
-  let InstructionSet = AMDGPUInstrInfo;
-  let AssemblyParsers = [AMDGPUAsmParser];
-}
-
-// Dummy Instruction itineraries for pseudo instructions
-def ALU_NULL : FuncUnit;
-def NullALU : InstrItinClass;
-
-//===----------------------------------------------------------------------===//
-// Predicate helper class
-//===----------------------------------------------------------------------===//
-
-class PredicateControl {
-  Predicate SubtargetPredicate;
-  list<Predicate> AssemblerPredicates = [];
-  list<Predicate> OtherPredicates = [];
-  list<Predicate> Predicates = !listconcat([SubtargetPredicate],
-                                            AssemblerPredicates,
-                                            OtherPredicates);
-}
-
-// Include AMDGPU TD files
-include "R600Schedule.td"
-include "SISchedule.td"
-include "Processors.td"
-include "AMDGPUInstrInfo.td"
-include "AMDGPUIntrinsics.td"
-include "AMDGPURegisterInfo.td"
-include "AMDGPUInstructions.td"
-include "AMDGPUCallingConv.td"
diff --git a/lib/Target/R600/AMDGPUAsmPrinter.cpp b/lib/Target/R600/AMDGPUAsmPrinter.cpp
deleted file mode 100644
index b3480b4b78c..00000000000
--- a/lib/Target/R600/AMDGPUAsmPrinter.cpp
+++ /dev/null
@@ -1,598 +0,0 @@
-//===-- AMDGPUAsmPrinter.cpp - AMDGPU Assebly printer  --------------------===//
-//
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
-//
-//===----------------------------------------------------------------------===//
-//
-/// \file
-///
-/// The AMDGPUAsmPrinter is used to print both assembly string and also binary
-/// code.  When passed an MCAsmStreamer it prints assembly and when passed
-/// an MCObjectStreamer it outputs binary code.
-//
-//===----------------------------------------------------------------------===//
-//
-
-#include "AMDGPUAsmPrinter.h"
-#include "InstPrinter/AMDGPUInstPrinter.h"
-#include "AMDGPU.h"
-#include "AMDKernelCodeT.h"
-#include "AMDGPUSubtarget.h"
-#include "R600Defines.h"
-#include "R600MachineFunctionInfo.h"
-#include "R600RegisterInfo.h"
-#include "SIDefines.h"
-#include "SIMachineFunctionInfo.h"
-#include "SIRegisterInfo.h"
-#include "llvm/CodeGen/MachineFrameInfo.h"
-#include "llvm/MC/MCContext.h"
-#include "llvm/MC/MCSectionELF.h"
-#include "llvm/MC/MCStreamer.h"
-#include "llvm/Support/ELF.h"
-#include "llvm/Support/MathExtras.h"
-#include "llvm/Support/TargetRegistry.h"
-#include "llvm/Target/TargetLoweringObjectFile.h"
-
-using namespace llvm;
-
-// TODO: This should get the default rounding mode from the kernel. We just set
-// the default here, but this could change if the OpenCL rounding mode pragmas
-// are used.
-//
-// The denormal mode here should match what is reported by the OpenCL runtime
-// for the CL_FP_DENORM bit from CL_DEVICE_{HALF|SINGLE|DOUBLE}_FP_CONFIG, but
-// can also be override to flush with the -cl-denorms-are-zero compiler flag.
-//
-// AMD OpenCL only sets flush none and reports CL_FP_DENORM for double
-// precision, and leaves single precision to flush all and does not report
-// CL_FP_DENORM for CL_DEVICE_SINGLE_FP_CONFIG. Mesa's OpenCL currently reports
-// CL_FP_DENORM for both.
-//
-// FIXME: It seems some instructions do not support single precision denormals
-// regardless of the mode (exp_*_f32, rcp_*_f32, rsq_*_f32, rsq_*f32, sqrt_f32,
-// and sin_f32, cos_f32 on most parts).
-
-// We want to use these instructions, and using fp32 denormals also causes
-// instructions to run at the double precision rate for the device so it's
-// probably best to just report no single precision denormals.
-static uint32_t getFPMode(const MachineFunction &F) {
-  const AMDGPUSubtarget& ST = F.getSubtarget<AMDGPUSubtarget>();
-  // TODO: Is there any real use for the flush in only / flush out only modes?
-
-  uint32_t FP32Denormals =
-    ST.hasFP32Denormals() ? FP_DENORM_FLUSH_NONE : FP_DENORM_FLUSH_IN_FLUSH_OUT;
-
-  uint32_t FP64Denormals =
-    ST.hasFP64Denormals() ? FP_DENORM_FLUSH_NONE : FP_DENORM_FLUSH_IN_FLUSH_OUT;
-
-  return FP_ROUND_MODE_SP(FP_ROUND_ROUND_TO_NEAREST) |
-         FP_ROUND_MODE_DP(FP_ROUND_ROUND_TO_NEAREST) |
-         FP_DENORM_MODE_SP(FP32Denormals) |
-         FP_DENORM_MODE_DP(FP64Denormals);
-}
-
-static AsmPrinter *
-createAMDGPUAsmPrinterPass(TargetMachine &tm,
-                           std::unique_ptr<MCStreamer> &&Streamer) {
-  return new AMDGPUAsmPrinter(tm, std::move(Streamer));
-}
-
-extern "C" void LLVMInitializeR600AsmPrinter() {
-  TargetRegistry::RegisterAsmPrinter(TheAMDGPUTarget, createAMDGPUAsmPrinterPass);
-  TargetRegistry::RegisterAsmPrinter(TheGCNTarget, createAMDGPUAsmPrinterPass);
-}
-
-AMDGPUAsmPrinter::AMDGPUAsmPrinter(TargetMachine &TM,
-                                   std::unique_ptr<MCStreamer> Streamer)
-    : AsmPrinter(TM, std::move(Streamer)) {}
-
-void AMDGPUAsmPrinter::EmitEndOfAsmFile(Module &M) {
-
-  // This label is used to mark the end of the .text section.
-  const TargetLoweringObjectFile &TLOF = getObjFileLowering();
-  OutStreamer.SwitchSection(TLOF.getTextSection());
-  MCSymbol *EndOfTextLabel =
-      OutContext.GetOrCreateSymbol(StringRef(END_OF_TEXT_LABEL_NAME));
-  OutStreamer.EmitLabel(EndOfTextLabel);
-}
-
-bool AMDGPUAsmPrinter::runOnMachineFunction(MachineFunction &MF) {
-
-  // The starting address of all shader programs must be 256 bytes aligned.
-  MF.setAlignment(8);
-
-  SetupMachineFunction(MF);
-
-  MCContext &Context = getObjFileLowering().getContext();
-  const MCSectionELF *ConfigSection =
-      Context.getELFSection(".AMDGPU.config", ELF::SHT_PROGBITS, 0);
-  OutStreamer.SwitchSection(ConfigSection);
-
-  const AMDGPUSubtarget &STM = MF.getSubtarget<AMDGPUSubtarget>();
-  SIProgramInfo KernelInfo;
-  if (STM.isAmdHsaOS()) {
-    getSIProgramInfo(KernelInfo, MF);
-    EmitAmdKernelCodeT(MF, KernelInfo);
-    OutStreamer.EmitCodeAlignment(2 << (MF.getAlignment() - 1));
-  } else if (STM.getGeneration() >= AMDGPUSubtarget::SOUTHERN_ISLANDS) {
-    getSIProgramInfo(KernelInfo, MF);
-    EmitProgramInfoSI(MF, KernelInfo);
-  } else {
-    EmitProgramInfoR600(MF);
-  }
-
-  DisasmLines.clear();
-  HexLines.clear();
-  DisasmLineMaxLen = 0;
-
-  EmitFunctionBody();
-
-  if (isVerbose()) {
-    const MCSectionELF *CommentSection =
-        Context.getELFSection(".AMDGPU.csdata", ELF::SHT_PROGBITS, 0);
-    OutStreamer.SwitchSection(CommentSection);
-
-    if (STM.getGeneration() >= AMDGPUSubtarget::SOUTHERN_ISLANDS) {
-      OutStreamer.emitRawComment(" Kernel info:", false);
-      OutStreamer.emitRawComment(" codeLenInByte = " + Twine(KernelInfo.CodeLen),
-                                 false);
-      OutStreamer.emitRawComment(" NumSgprs: " + Twine(KernelInfo.NumSGPR),
-                                 false);
-      OutStreamer.emitRawComment(" NumVgprs: " + Twine(KernelInfo.NumVGPR),
-                                 false);
-      OutStreamer.emitRawComment(" FloatMode: " + Twine(KernelInfo.FloatMode),
-                                 false);
-      OutStreamer.emitRawComment(" IeeeMode: " + Twine(KernelInfo.IEEEMode),
-                                 false);
-      OutStreamer.emitRawComment(" ScratchSize: " + Twine(KernelInfo.ScratchSize),
-                                 false);
-    } else {
-      R600MachineFunctionInfo *MFI = MF.getInfo<R600MachineFunctionInfo>();
-      OutStreamer.emitRawComment(
-        Twine("SQ_PGM_RESOURCES:STACK_SIZE = " + Twine(MFI->StackSize)));
-    }
-  }
-
-  if (STM.dumpCode()) {
-
-    OutStreamer.SwitchSection(
-        Context.getELFSection(".AMDGPU.disasm", ELF::SHT_NOTE, 0));
-
-    for (size_t i = 0; i < DisasmLines.size(); ++i) {
-      std::string Comment(DisasmLineMaxLen - DisasmLines[i].size(), ' ');
-      Comment += " ; " + HexLines[i] + "\n";
-
-      OutStreamer.EmitBytes(StringRef(DisasmLines[i]));
-      OutStreamer.EmitBytes(StringRef(Comment));
-    }
-  }
-
-  return false;
-}
-
-void AMDGPUAsmPrinter::EmitProgramInfoR600(const MachineFunction &MF) {
-  unsigned MaxGPR = 0;
-  bool killPixel = false;
-  const AMDGPUSubtarget &STM = MF.getSubtarget<AMDGPUSubtarget>();
-  const R600RegisterInfo *RI =
-      static_cast<const R600RegisterInfo *>(STM.getRegisterInfo());
-  const R600MachineFunctionInfo *MFI = MF.getInfo<R600MachineFunctionInfo>();
-
-  for (const MachineBasicBlock &MBB : MF) {
-    for (const MachineInstr &MI : MBB) {
-      if (MI.getOpcode() == AMDGPU::KILLGT)
-        killPixel = true;
-      unsigned numOperands = MI.getNumOperands();
-      for (unsigned op_idx = 0; op_idx < numOperands; op_idx++) {
-        const MachineOperand &MO = MI.getOperand(op_idx);
-        if (!MO.isReg())
-          continue;
-        unsigned HWReg = RI->getEncodingValue(MO.getReg()) & 0xff;
-
-        // Register with value > 127 aren't GPR
-        if (HWReg > 127)
-          continue;
-        MaxGPR = std::max(MaxGPR, HWReg);
-      }
-    }
-  }
-
-  unsigned RsrcReg;
-  if (STM.getGeneration() >= AMDGPUSubtarget::EVERGREEN) {
-    // Evergreen / Northern Islands
-    switch (MFI->getShaderType()) {
-    default: // Fall through
-    case ShaderType::COMPUTE:  RsrcReg = R_0288D4_SQ_PGM_RESOURCES_LS; break;
-    case ShaderType::GEOMETRY: RsrcReg = R_028878_SQ_PGM_RESOURCES_GS; break;
-    case ShaderType::PIXEL:    RsrcReg = R_028844_SQ_PGM_RESOURCES_PS; break;
-    case ShaderType::VERTEX:   RsrcReg = R_028860_SQ_PGM_RESOURCES_VS; break;
-    }
-  } else {
-    // R600 / R700
-    switch (MFI->getShaderType()) {
-    default: // Fall through
-    case ShaderType::GEOMETRY: // Fall through
-    case ShaderType::COMPUTE:  // Fall through
-    case ShaderType::VERTEX:   RsrcReg = R_028868_SQ_PGM_RESOURCES_VS; break;
-    case ShaderType::PIXEL:    RsrcReg = R_028850_SQ_PGM_RESOURCES_PS; break;
-    }
-  }
-
-  OutStreamer.EmitIntValue(RsrcReg, 4);
-  OutStreamer.EmitIntValue(S_NUM_GPRS(MaxGPR + 1) |
-                           S_STACK_SIZE(MFI->StackSize), 4);
-  OutStreamer.EmitIntValue(R_02880C_DB_SHADER_CONTROL, 4);
-  OutStreamer.EmitIntValue(S_02880C_KILL_ENABLE(killPixel), 4);
-
-  if (MFI->getShaderType() == ShaderType::COMPUTE) {
-    OutStreamer.EmitIntValue(R_0288E8_SQ_LDS_ALLOC, 4);
-    OutStreamer.EmitIntValue(RoundUpToAlignment(MFI->LDSSize, 4) >> 2, 4);
-  }
-}
-
-void AMDGPUAsmPrinter::getSIProgramInfo(SIProgramInfo &ProgInfo,
-                                        const MachineFunction &MF) const {
-  const AMDGPUSubtarget &STM = MF.getSubtarget<AMDGPUSubtarget>();
-  const SIMachineFunctionInfo *MFI = MF.getInfo<SIMachineFunctionInfo>();
-  uint64_t CodeSize = 0;
-  unsigned MaxSGPR = 0;
-  unsigned MaxVGPR = 0;
-  bool VCCUsed = false;
-  bool FlatUsed = false;
-  const SIRegisterInfo *RI =
-      static_cast<const SIRegisterInfo *>(STM.getRegisterInfo());
-
-  for (const MachineBasicBlock &MBB : MF) {
-    for (const MachineInstr &MI : MBB) {
-      // TODO: CodeSize should account for multiple functions.
-      CodeSize += MI.getDesc().Size;
-
-      unsigned numOperands = MI.getNumOperands();
-      for (unsigned op_idx = 0; op_idx < numOperands; op_idx++) {
-        const MachineOperand &MO = MI.getOperand(op_idx);
-        unsigned width = 0;
-        bool isSGPR = false;
-
-        if (!MO.isReg()) {
-          continue;
-        }
-        unsigned reg = MO.getReg();
-        if (reg == AMDGPU::VCC || reg == AMDGPU::VCC_LO ||
-	    reg == AMDGPU::VCC_HI) {
-          VCCUsed = true;
-          continue;
-        } else if (reg == AMDGPU::FLAT_SCR ||
-                   reg == AMDGPU::FLAT_SCR_LO ||
-                   reg == AMDGPU::FLAT_SCR_HI) {
-          FlatUsed = true;
-          continue;
-        }
-
-        switch (reg) {
-        default: break;
-        case AMDGPU::SCC:
-        case AMDGPU::EXEC:
-        case AMDGPU::M0:
-          continue;
-        }
-
-        if (AMDGPU::SReg_32RegClass.contains(reg)) {
-          isSGPR = true;
-          width = 1;
-        } else if (AMDGPU::VGPR_32RegClass.contains(reg)) {
-          isSGPR = false;
-          width = 1;
-        } else if (AMDGPU::SReg_64RegClass.contains(reg)) {
-          isSGPR = true;
-          width = 2;
-        } else if (AMDGPU::VReg_64RegClass.contains(reg)) {
-          isSGPR = false;
-          width = 2;
-        } else if (AMDGPU::VReg_96RegClass.contains(reg)) {
-          isSGPR = false;
-          width = 3;
-        } else if (AMDGPU::SReg_128RegClass.contains(reg)) {
-          isSGPR = true;
-          width = 4;
-        } else if (AMDGPU::VReg_128RegClass.contains(reg)) {
-          isSGPR = false;
-          width = 4;
-        } else if (AMDGPU::SReg_256RegClass.contains(reg)) {
-          isSGPR = true;
-          width = 8;
-        } else if (AMDGPU::VReg_256RegClass.contains(reg)) {
-          isSGPR = false;
-          width = 8;
-        } else if (AMDGPU::SReg_512RegClass.contains(reg)) {
-          isSGPR = true;
-          width = 16;
-        } else if (AMDGPU::VReg_512RegClass.contains(reg)) {
-          isSGPR = false;
-          width = 16;
-        } else {
-          llvm_unreachable("Unknown register class");
-        }
-        unsigned hwReg = RI->getEncodingValue(reg) & 0xff;
-        unsigned maxUsed = hwReg + width - 1;
-        if (isSGPR) {
-          MaxSGPR = maxUsed > MaxSGPR ? maxUsed : MaxSGPR;
-        } else {
-          MaxVGPR = maxUsed > MaxVGPR ? maxUsed : MaxVGPR;
-        }
-      }
-    }
-  }
-
-  if (VCCUsed)
-    MaxSGPR += 2;
-
-  if (FlatUsed)
-    MaxSGPR += 2;
-
-  // We found the maximum register index. They start at 0, so add one to get the
-  // number of registers.
-  ProgInfo.NumVGPR = MaxVGPR + 1;
-  ProgInfo.NumSGPR = MaxSGPR + 1;
-
-  if (STM.hasSGPRInitBug()) {
-    if (ProgInfo.NumSGPR > AMDGPUSubtarget::FIXED_SGPR_COUNT_FOR_INIT_BUG)
-      llvm_unreachable("Too many SGPRs used with the SGPR init bug");
-
-    ProgInfo.NumSGPR = AMDGPUSubtarget::FIXED_SGPR_COUNT_FOR_INIT_BUG;
-  }
-
-  ProgInfo.VGPRBlocks = (ProgInfo.NumVGPR - 1) / 4;
-  ProgInfo.SGPRBlocks = (ProgInfo.NumSGPR - 1) / 8;
-  // Set the value to initialize FP_ROUND and FP_DENORM parts of the mode
-  // register.
-  ProgInfo.FloatMode = getFPMode(MF);
-
-  // XXX: Not quite sure what this does, but sc seems to unset this.
-  ProgInfo.IEEEMode = 0;
-
-  // Do not clamp NAN to 0.
-  ProgInfo.DX10Clamp = 0;
-
-  const MachineFrameInfo *FrameInfo = MF.getFrameInfo();
-  ProgInfo.ScratchSize = FrameInfo->estimateStackSize(MF);
-
-  ProgInfo.FlatUsed = FlatUsed;
-  ProgInfo.VCCUsed = VCCUsed;
-  ProgInfo.CodeLen = CodeSize;
-
-  unsigned LDSAlignShift;
-  if (STM.getGeneration() < AMDGPUSubtarget::SEA_ISLANDS) {
-    // LDS is allocated in 64 dword blocks.
-    LDSAlignShift = 8;
-  } else {
-    // LDS is allocated in 128 dword blocks.
-    LDSAlignShift = 9;
-  }
-
-  unsigned LDSSpillSize = MFI->LDSWaveSpillSize *
-                          MFI->getMaximumWorkGroupSize(MF);
-
-  ProgInfo.LDSSize = MFI->LDSSize + LDSSpillSize;
-  ProgInfo.LDSBlocks =
-     RoundUpToAlignment(ProgInfo.LDSSize, 1 << LDSAlignShift) >> LDSAlignShift;
-
-  // Scratch is allocated in 256 dword blocks.
-  unsigned ScratchAlignShift = 10;
-  // We need to program the hardware with the amount of scratch memory that
-  // is used by the entire wave.  ProgInfo.ScratchSize is the amount of
-  // scratch memory used per thread.
-  ProgInfo.ScratchBlocks =
-    RoundUpToAlignment(ProgInfo.ScratchSize * STM.getWavefrontSize(),
-                       1 << ScratchAlignShift) >> ScratchAlignShift;
-
-  ProgInfo.ComputePGMRSrc1 =
-      S_00B848_VGPRS(ProgInfo.VGPRBlocks) |
-      S_00B848_SGPRS(ProgInfo.SGPRBlocks) |
-      S_00B848_PRIORITY(ProgInfo.Priority) |
-      S_00B848_FLOAT_MODE(ProgInfo.FloatMode) |
-      S_00B848_PRIV(ProgInfo.Priv) |
-      S_00B848_DX10_CLAMP(ProgInfo.DX10Clamp) |
-      S_00B848_IEEE_MODE(ProgInfo.DebugMode) |
-      S_00B848_IEEE_MODE(ProgInfo.IEEEMode);
-
-  ProgInfo.ComputePGMRSrc2 =
-      S_00B84C_SCRATCH_EN(ProgInfo.ScratchBlocks > 0) |
-      S_00B84C_USER_SGPR(MFI->NumUserSGPRs) |
-      S_00B84C_TGID_X_EN(1) |
-      S_00B84C_TGID_Y_EN(1) |
-      S_00B84C_TGID_Z_EN(1) |
-      S_00B84C_TG_SIZE_EN(1) |
-      S_00B84C_TIDIG_COMP_CNT(2) |
-      S_00B84C_LDS_SIZE(ProgInfo.LDSBlocks);
-}
-
-static unsigned getRsrcReg(unsigned ShaderType) {
-  switch (ShaderType) {
-  default: // Fall through
-  case ShaderType::COMPUTE:  return R_00B848_COMPUTE_PGM_RSRC1;
-  case ShaderType::GEOMETRY: return R_00B228_SPI_SHADER_PGM_RSRC1_GS;
-  case ShaderType::PIXEL:    return R_00B028_SPI_SHADER_PGM_RSRC1_PS;
-  case ShaderType::VERTEX:   return R_00B128_SPI_SHADER_PGM_RSRC1_VS;
-  }
-}
-
-void AMDGPUAsmPrinter::EmitProgramInfoSI(const MachineFunction &MF,
-                                         const SIProgramInfo &KernelInfo) {
-  const AMDGPUSubtarget &STM = MF.getSubtarget<AMDGPUSubtarget>();
-  const SIMachineFunctionInfo *MFI = MF.getInfo<SIMachineFunctionInfo>();
-  unsigned RsrcReg = getRsrcReg(MFI->getShaderType());
-
-  if (MFI->getShaderType() == ShaderType::COMPUTE) {
-    OutStreamer.EmitIntValue(R_00B848_COMPUTE_PGM_RSRC1, 4);
-
-    OutStreamer.EmitIntValue(KernelInfo.ComputePGMRSrc1, 4);
-
-    OutStreamer.EmitIntValue(R_00B84C_COMPUTE_PGM_RSRC2, 4);
-    OutStreamer.EmitIntValue(KernelInfo.ComputePGMRSrc2, 4);
-
-    OutStreamer.EmitIntValue(R_00B860_COMPUTE_TMPRING_SIZE, 4);
-    OutStreamer.EmitIntValue(S_00B860_WAVESIZE(KernelInfo.ScratchBlocks), 4);
-
-    // TODO: Should probably note flat usage somewhere. SC emits a "FlatPtr32 =
-    // 0" comment but I don't see a corresponding field in the register spec.
-  } else {
-    OutStreamer.EmitIntValue(RsrcReg, 4);
-    OutStreamer.EmitIntValue(S_00B028_VGPRS(KernelInfo.VGPRBlocks) |
-                             S_00B028_SGPRS(KernelInfo.SGPRBlocks), 4);
-    if (STM.isVGPRSpillingEnabled(MFI)) {
-      OutStreamer.EmitIntValue(R_0286E8_SPI_TMPRING_SIZE, 4);
-      OutStreamer.EmitIntValue(S_0286E8_WAVESIZE(KernelInfo.ScratchBlocks), 4);
-    }
-  }
-
-  if (MFI->getShaderType() == ShaderType::PIXEL) {
-    OutStreamer.EmitIntValue(R_00B02C_SPI_SHADER_PGM_RSRC2_PS, 4);
-    OutStreamer.EmitIntValue(S_00B02C_EXTRA_LDS_SIZE(KernelInfo.LDSBlocks), 4);
-    OutStreamer.EmitIntValue(R_0286CC_SPI_PS_INPUT_ENA, 4);
-    OutStreamer.EmitIntValue(MFI->PSInputAddr, 4);
-  }
-}
-
-void AMDGPUAsmPrinter::EmitAmdKernelCodeT(const MachineFunction &MF,
-                                        const SIProgramInfo &KernelInfo) const {
-  const SIMachineFunctionInfo *MFI = MF.getInfo<SIMachineFunctionInfo>();
-  const AMDGPUSubtarget &STM = MF.getSubtarget<AMDGPUSubtarget>();
-  amd_kernel_code_t header;
-
-  memset(&header, 0, sizeof(header));
-
-  header.amd_code_version_major = AMD_CODE_VERSION_MAJOR;
-  header.amd_code_version_minor = AMD_CODE_VERSION_MINOR;
-
-  header.struct_byte_size = sizeof(amd_kernel_code_t);
-
-  header.target_chip = STM.getAmdKernelCodeChipID();
-
-  header.kernel_code_entry_byte_offset = (1ULL << MF.getAlignment());
-
-  header.compute_pgm_resource_registers =
-      KernelInfo.ComputePGMRSrc1 |
-      (KernelInfo.ComputePGMRSrc2 << 32);
-
-  // Code Properties:
-  header.code_properties = AMD_CODE_PROPERTY_ENABLE_SGPR_KERNARG_SEGMENT_PTR |
-                           AMD_CODE_PROPERTY_IS_PTR64;
-
-  if (KernelInfo.FlatUsed)
-    header.code_properties |= AMD_CODE_PROPERTY_ENABLE_SGPR_FLAT_SCRATCH_INIT;
-
-  if (KernelInfo.ScratchBlocks)
-    header.code_properties |= AMD_CODE_PROPERTY_ENABLE_SGPR_PRIVATE_SEGMENT_SIZE;
-
-  header.workitem_private_segment_byte_size = KernelInfo.ScratchSize;
-  header.workgroup_group_segment_byte_size = KernelInfo.LDSSize;
-
-  // MFI->ABIArgOffset is the number of bytes for the kernel arguments
-  // plus 36.  36 is the number of bytes reserved at the begining of the
-  // input buffer to store work-group size information.
-  // FIXME: We should be adding the size of the implicit arguments
-  // to this value.
-  header.kernarg_segment_byte_size = MFI->ABIArgOffset;
-
-  header.wavefront_sgpr_count = KernelInfo.NumSGPR;
-  header.workitem_vgpr_count = KernelInfo.NumVGPR;
-
-  // FIXME: What values do I put for these alignments
-  header.kernarg_segment_alignment = 0;
-  header.group_segment_alignment = 0;
-  header.private_segment_alignment = 0;
-
-  header.code_type = 1; // HSA_EXT_CODE_KERNEL
-
-  header.wavefront_size = STM.getWavefrontSize();
-
-  const MCSectionELF *VersionSection =
-      OutContext.getELFSection(".hsa.version", ELF::SHT_PROGBITS, 0);
-  OutStreamer.SwitchSection(VersionSection);
-  OutStreamer.EmitBytes(Twine("HSA Code Unit:" +
-                        Twine(header.hsail_version_major) + "." +
-                        Twine(header.hsail_version_minor) + ":" +
-                        "AMD:" +
-                        Twine(header.amd_code_version_major) + "." +
-                        Twine(header.amd_code_version_minor) +  ":" +
-                        "GFX8.1:0").str());
-
-  OutStreamer.SwitchSection(getObjFileLowering().getTextSection());
-
-  if (isVerbose()) {
-    OutStreamer.emitRawComment("amd_code_version_major = " +
-                               Twine(header.amd_code_version_major), false);
-    OutStreamer.emitRawComment("amd_code_version_minor = " +
-                               Twine(header.amd_code_version_minor), false);
-    OutStreamer.emitRawComment("struct_byte_size = " +
-                               Twine(header.struct_byte_size), false);
-    OutStreamer.emitRawComment("target_chip = " +
-                               Twine(header.target_chip), false);
-    OutStreamer.emitRawComment(" compute_pgm_rsrc1: " +
-                               Twine::utohexstr(KernelInfo.ComputePGMRSrc1), false);
-    OutStreamer.emitRawComment(" compute_pgm_rsrc2: " +
-                               Twine::utohexstr(KernelInfo.ComputePGMRSrc2), false);
-    OutStreamer.emitRawComment("enable_sgpr_private_segment_buffer = " +
-      Twine((bool)(header.code_properties &
-                   AMD_CODE_PROPERTY_ENABLE_SGPR_PRIVATE_SEGMENT_SIZE)), false);
-    OutStreamer.emitRawComment("enable_sgpr_kernarg_segment_ptr = " +
-      Twine((bool)(header.code_properties &
-                   AMD_CODE_PROPERTY_ENABLE_SGPR_KERNARG_SEGMENT_PTR)), false);
-    OutStreamer.emitRawComment("private_element_size = 2 ", false);
-    OutStreamer.emitRawComment("is_ptr64 = " +
-        Twine((bool)(header.code_properties & AMD_CODE_PROPERTY_IS_PTR64)), false);
-    OutStreamer.emitRawComment("workitem_private_segment_byte_size = " +
-                               Twine(header.workitem_private_segment_byte_size),
-                               false);
-    OutStreamer.emitRawComment("workgroup_group_segment_byte_size = " +
-                               Twine(header.workgroup_group_segment_byte_size),
-                               false);
-    OutStreamer.emitRawComment("gds_segment_byte_size = " +
-                               Twine(header.gds_segment_byte_size), false);
-    OutStreamer.emitRawComment("kernarg_segment_byte_size = " +
-                               Twine(header.kernarg_segment_byte_size), false);
-    OutStreamer.emitRawComment("wavefront_sgpr_count = " +
-                               Twine(header.wavefront_sgpr_count), false);
-    OutStreamer.emitRawComment("workitem_vgpr_count = " +
-                               Twine(header.workitem_vgpr_count), false);
-    OutStreamer.emitRawComment("code_type = " + Twine(header.code_type), false);
-    OutStreamer.emitRawComment("wavefront_size = " +
-                               Twine((int)header.wavefront_size), false);
-    OutStreamer.emitRawComment("optimization_level = " +
-                               Twine(header.optimization_level), false);
-    OutStreamer.emitRawComment("hsail_profile = " +
-                               Twine(header.hsail_profile), false);
-    OutStreamer.emitRawComment("hsail_machine_model = " +
-                               Twine(header.hsail_machine_model), false);
-    OutStreamer.emitRawComment("hsail_version_major = " +
-                               Twine(header.hsail_version_major), false);
-    OutStreamer.emitRawComment("hsail_version_minor = " +
-                               Twine(header.hsail_version_minor), false);
-  }
-
-  OutStreamer.EmitBytes(StringRef((char*)&header, sizeof(header)));
-}
-
-bool AMDGPUAsmPrinter::PrintAsmOperand(const MachineInstr *MI, unsigned OpNo,
-                                       unsigned AsmVariant,
-                                       const char *ExtraCode, raw_ostream &O) {
-  if (ExtraCode && ExtraCode[0]) {
-    if (ExtraCode[1] != 0)
-      return true; // Unknown modifier.
-
-    switch (ExtraCode[0]) {
-    default:
-      // See if this is a generic print operand
-      return AsmPrinter::PrintAsmOperand(MI, OpNo, AsmVariant, ExtraCode, O);
-    case 'r':
-      break;
-    }
-  }
-
-  AMDGPUInstPrinter::printRegOperand(MI->getOperand(OpNo).getReg(), O,
-                   *TM.getSubtargetImpl(*MF->getFunction())->getRegisterInfo());
-  return false;
-}
diff --git a/lib/Target/R600/AsmParser/AMDGPUAsmParser.cpp b/lib/Target/R600/AsmParser/AMDGPUAsmParser.cpp
deleted file mode 100644
index aaf9b325e66..00000000000
--- a/lib/Target/R600/AsmParser/AMDGPUAsmParser.cpp
+++ /dev/null
@@ -1,1275 +0,0 @@
-//===-- AMDGPUAsmParser.cpp - Parse SI asm to MCInst instructions ----------===//
-//
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
-//
-//===----------------------------------------------------------------------===//
-
-#include "MCTargetDesc/AMDGPUMCTargetDesc.h"
-#include "SIDefines.h"
-#include "llvm/ADT/APFloat.h"
-#include "llvm/ADT/SmallString.h"
-#include "llvm/ADT/SmallVector.h"
-#include "llvm/ADT/STLExtras.h"
-#include "llvm/ADT/StringSwitch.h"
-#include "llvm/ADT/Twine.h"
-#include "llvm/MC/MCContext.h"
-#include "llvm/MC/MCExpr.h"
-#include "llvm/MC/MCInst.h"
-#include "llvm/MC/MCInstrInfo.h"
-#include "llvm/MC/MCParser/MCAsmLexer.h"
-#include "llvm/MC/MCParser/MCAsmParser.h"
-#include "llvm/MC/MCParser/MCParsedAsmOperand.h"
-#include "llvm/MC/MCRegisterInfo.h"
-#include "llvm/MC/MCStreamer.h"
-#include "llvm/MC/MCSubtargetInfo.h"
-#include "llvm/MC/MCTargetAsmParser.h"
-#include "llvm/Support/SourceMgr.h"
-#include "llvm/Support/TargetRegistry.h"
-#include "llvm/Support/raw_ostream.h"
-#include "llvm/Support/Debug.h"
-
-using namespace llvm;
-
-namespace {
-
-struct OptionalOperand;
-
-class AMDGPUOperand : public MCParsedAsmOperand {
-  enum KindTy {
-    Token,
-    Immediate,
-    Register,
-    Expression
-  } Kind;
-
-  SMLoc StartLoc, EndLoc;
-
-public:
-  AMDGPUOperand(enum KindTy K) : MCParsedAsmOperand(), Kind(K) {}
-
-  MCContext *Ctx;
-
-  enum ImmTy {
-    ImmTyNone,
-    ImmTyDSOffset0,
-    ImmTyDSOffset1,
-    ImmTyGDS,
-    ImmTyOffset,
-    ImmTyGLC,
-    ImmTySLC,
-    ImmTyTFE,
-    ImmTyClamp,
-    ImmTyOMod
-  };
-
-  struct TokOp {
-    const char *Data;
-    unsigned Length;
-  };
-
-  struct ImmOp {
-    bool IsFPImm;
-    ImmTy Type;
-    int64_t Val;
-  };
-
-  struct RegOp {
-    unsigned RegNo;
-    int Modifiers;
-    const MCRegisterInfo *TRI;
-  };
-
-  union {
-    TokOp Tok;
-    ImmOp Imm;
-    RegOp Reg;
-    const MCExpr *Expr;
-  };
-
-  void addImmOperands(MCInst &Inst, unsigned N) const {
-    Inst.addOperand(MCOperand::CreateImm(getImm()));
-  }
-
-  StringRef getToken() const {
-    return StringRef(Tok.Data, Tok.Length);
-  }
-
-  void addRegOperands(MCInst &Inst, unsigned N) const {
-    Inst.addOperand(MCOperand::CreateReg(getReg()));
-  }
-
-  void addRegOrImmOperands(MCInst &Inst, unsigned N) const {
-    if (isReg())
-      addRegOperands(Inst, N);
-    else
-      addImmOperands(Inst, N);
-  }
-
-  void addRegWithInputModsOperands(MCInst &Inst, unsigned N) const {
-    Inst.addOperand(MCOperand::CreateImm(Reg.Modifiers));
-    addRegOperands(Inst, N);
-  }
-
-  void addSoppBrTargetOperands(MCInst &Inst, unsigned N) const {
-    if (isImm())
-      addImmOperands(Inst, N);
-    else {
-      assert(isExpr());
-      Inst.addOperand(MCOperand::CreateExpr(Expr));
-    }
-  }
-
-  bool defaultTokenHasSuffix() const {
-    StringRef Token(Tok.Data, Tok.Length);
-
-    return Token.endswith("_e32") || Token.endswith("_e64");
-  }
-
-  bool isToken() const override {
-    return Kind == Token;
-  }
-
-  bool isImm() const override {
-    return Kind == Immediate;
-  }
-
-  bool isInlineImm() const {
-    float F = BitsToFloat(Imm.Val);
-    // TODO: Add 0.5pi for VI
-    return isImm() && ((Imm.Val <= 64 && Imm.Val >= -16) ||
-           (F == 0.0 || F == 0.5 || F == -0.5 || F == 1.0 || F == -1.0 ||
-           F == 2.0 || F == -2.0 || F == 4.0 || F == -4.0));
-  }
-
-  bool isDSOffset0() const {
-    assert(isImm());
-    return Imm.Type == ImmTyDSOffset0;
-  }
-
-  bool isDSOffset1() const {
-    assert(isImm());
-    return Imm.Type == ImmTyDSOffset1;
-  }
-
-  int64_t getImm() const {
-    return Imm.Val;
-  }
-
-  enum ImmTy getImmTy() const {
-    assert(isImm());
-    return Imm.Type;
-  }
-
-  bool isReg() const override {
-    return Kind == Register && Reg.Modifiers == -1;
-  }
-
-  bool isRegWithInputMods() const {
-    return Kind == Register && Reg.Modifiers != -1;
-  }
-
-  void setModifiers(unsigned Mods) {
-    assert(isReg());
-    Reg.Modifiers = Mods;
-  }
-
-  unsigned getReg() const override {
-    return Reg.RegNo;
-  }
-
-  bool isRegOrImm() const {
-    return isReg() || isImm();
-  }
-
-  bool isRegClass(unsigned RCID) const {
-    return Reg.TRI->getRegClass(RCID).contains(getReg());
-  }
-
-  bool isSCSrc32() const {
-    return isInlineImm() || (isReg() && isRegClass(AMDGPU::SReg_32RegClassID));
-  }
-
-  bool isSSrc32() const {
-    return isImm() || (isReg() && isRegClass(AMDGPU::SReg_32RegClassID));
-  }
-
-  bool isSSrc64() const {
-    return isImm() || isInlineImm() ||
-           (isReg() && isRegClass(AMDGPU::SReg_64RegClassID));
-  }
-
-  bool isVCSrc32() const {
-    return isInlineImm() || (isReg() && isRegClass(AMDGPU::VS_32RegClassID));
-  }
-
-  bool isVCSrc64() const {
-    return isInlineImm() || (isReg() && isRegClass(AMDGPU::VS_64RegClassID));
-  }
-
-  bool isVSrc32() const {
-    return isImm() || (isReg() && isRegClass(AMDGPU::VS_32RegClassID));
-  }
-
-  bool isVSrc64() const {
-    return isImm() || (isReg() && isRegClass(AMDGPU::VS_64RegClassID));
-  }
-
-  bool isMem() const override {
-    return false;
-  }
-
-  bool isExpr() const {
-    return Kind == Expression;
-  }
-
-  bool isSoppBrTarget() const {
-    return isExpr() || isImm();
-  }
-
-  SMLoc getStartLoc() const override {
-    return StartLoc;
-  }
-
-  SMLoc getEndLoc() const override {
-    return EndLoc;
-  }
-
-  void print(raw_ostream &OS) const override { }
-
-  static std::unique_ptr<AMDGPUOperand> CreateImm(int64_t Val, SMLoc Loc,
-                                                  enum ImmTy Type = ImmTyNone,
-                                                  bool IsFPImm = false) {
-    auto Op = llvm::make_unique<AMDGPUOperand>(Immediate);
-    Op->Imm.Val = Val;
-    Op->Imm.IsFPImm = IsFPImm;
-    Op->Imm.Type = Type;
-    Op->StartLoc = Loc;
-    Op->EndLoc = Loc;
-    return Op;
-  }
-
-  static std::unique_ptr<AMDGPUOperand> CreateToken(StringRef Str, SMLoc Loc,
-                                           bool HasExplicitEncodingSize = true) {
-    auto Res = llvm::make_unique<AMDGPUOperand>(Token);
-    Res->Tok.Data = Str.data();
-    Res->Tok.Length = Str.size();
-    Res->StartLoc = Loc;
-    Res->EndLoc = Loc;
-    return Res;
-  }
-
-  static std::unique_ptr<AMDGPUOperand> CreateReg(unsigned RegNo, SMLoc S,
-                                                  SMLoc E,
-                                                  const MCRegisterInfo *TRI) {
-    auto Op = llvm::make_unique<AMDGPUOperand>(Register);
-    Op->Reg.RegNo = RegNo;
-    Op->Reg.TRI = TRI;
-    Op->Reg.Modifiers = -1;
-    Op->StartLoc = S;
-    Op->EndLoc = E;
-    return Op;
-  }
-
-  static std::unique_ptr<AMDGPUOperand> CreateExpr(const class MCExpr *Expr, SMLoc S) {
-    auto Op = llvm::make_unique<AMDGPUOperand>(Expression);
-    Op->Expr = Expr;
-    Op->StartLoc = S;
-    Op->EndLoc = S;
-    return Op;
-  }
-
-  bool isDSOffset() const;
-  bool isDSOffset01() const;
-  bool isSWaitCnt() const;
-  bool isMubufOffset() const;
-};
-
-class AMDGPUAsmParser : public MCTargetAsmParser {
-  MCSubtargetInfo &STI;
-  const MCInstrInfo &MII;
-  MCAsmParser &Parser;
-
-  unsigned ForcedEncodingSize;
-  /// @name Auto-generated Match Functions
-  /// {
-
-#define GET_ASSEMBLER_HEADER
-#include "AMDGPUGenAsmMatcher.inc"
-
-  /// }
-
-public:
-  AMDGPUAsmParser(MCSubtargetInfo &STI, MCAsmParser &_Parser,
-               const MCInstrInfo &MII,
-               const MCTargetOptions &Options)
-      : MCTargetAsmParser(), STI(STI), MII(MII), Parser(_Parser),
-        ForcedEncodingSize(0){
-
-    if (!STI.getFeatureBits()) {
-      // Set default features.
-      STI.ToggleFeature("SOUTHERN_ISLANDS");
-    }
-
-    setAvailableFeatures(ComputeAvailableFeatures(STI.getFeatureBits()));
-  }
-
-  unsigned getForcedEncodingSize() const {
-    return ForcedEncodingSize;
-  }
-
-  void setForcedEncodingSize(unsigned Size) {
-    ForcedEncodingSize = Size;
-  }
-
-  bool ParseRegister(unsigned &RegNo, SMLoc &StartLoc, SMLoc &EndLoc) override;
-  unsigned checkTargetMatchPredicate(MCInst &Inst) override;
-  bool MatchAndEmitInstruction(SMLoc IDLoc, unsigned &Opcode,
-                               OperandVector &Operands, MCStreamer &Out,
-                               uint64_t &ErrorInfo,
-                               bool MatchingInlineAsm) override;
-  bool ParseDirective(AsmToken DirectiveID) override;
-  OperandMatchResultTy parseOperand(OperandVector &Operands, StringRef Mnemonic);
-  bool ParseInstruction(ParseInstructionInfo &Info, StringRef Name,
-                        SMLoc NameLoc, OperandVector &Operands) override;
-
-  OperandMatchResultTy parseIntWithPrefix(const char *Prefix, int64_t &Int,
-                                          int64_t Default = 0);
-  OperandMatchResultTy parseIntWithPrefix(const char *Prefix,
-                                          OperandVector &Operands,
-                                          enum AMDGPUOperand::ImmTy ImmTy =
-                                                      AMDGPUOperand::ImmTyNone);
-  OperandMatchResultTy parseNamedBit(const char *Name, OperandVector &Operands,
-                                     enum AMDGPUOperand::ImmTy ImmTy =
-                                                      AMDGPUOperand::ImmTyNone);
-  OperandMatchResultTy parseOptionalOps(
-                                   const ArrayRef<OptionalOperand> &OptionalOps,
-                                   OperandVector &Operands);
-
-
-  void cvtDSOffset01(MCInst &Inst, const OperandVector &Operands);
-  void cvtDS(MCInst &Inst, const OperandVector &Operands);
-  OperandMatchResultTy parseDSOptionalOps(OperandVector &Operands);
-  OperandMatchResultTy parseDSOff01OptionalOps(OperandVector &Operands);
-  OperandMatchResultTy parseDSOffsetOptional(OperandVector &Operands);
-
-  bool parseCnt(int64_t &IntVal);
-  OperandMatchResultTy parseSWaitCntOps(OperandVector &Operands);
-  OperandMatchResultTy parseSOppBrTarget(OperandVector &Operands);
-
-  void cvtMubuf(MCInst &Inst, const OperandVector &Operands);
-  OperandMatchResultTy parseOffset(OperandVector &Operands);
-  OperandMatchResultTy parseMubufOptionalOps(OperandVector &Operands);
-  OperandMatchResultTy parseGLC(OperandVector &Operands);
-  OperandMatchResultTy parseSLC(OperandVector &Operands);
-  OperandMatchResultTy parseTFE(OperandVector &Operands);
-
-  OperandMatchResultTy parseDMask(OperandVector &Operands);
-  OperandMatchResultTy parseUNorm(OperandVector &Operands);
-  OperandMatchResultTy parseR128(OperandVector &Operands);
-
-  void cvtVOP3(MCInst &Inst, const OperandVector &Operands);
-  OperandMatchResultTy parseVOP3OptionalOps(OperandVector &Operands);
-};
-
-struct OptionalOperand {
-  const char *Name;
-  AMDGPUOperand::ImmTy Type;
-  bool IsBit;
-  int64_t Default;
-  bool (*ConvertResult)(int64_t&);
-};
-
-}
-
-static unsigned getRegClass(bool IsVgpr, unsigned RegWidth) {
-  if (IsVgpr) {
-    switch (RegWidth) {
-      default: llvm_unreachable("Unknown register width");
-      case 1: return AMDGPU::VGPR_32RegClassID;
-      case 2: return AMDGPU::VReg_64RegClassID;
-      case 3: return AMDGPU::VReg_96RegClassID;
-      case 4: return AMDGPU::VReg_128RegClassID;
-      case 8: return AMDGPU::VReg_256RegClassID;
-      case 16: return AMDGPU::VReg_512RegClassID;
-    }
-  }
-
-  switch (RegWidth) {
-    default: llvm_unreachable("Unknown register width");
-    case 1: return AMDGPU::SGPR_32RegClassID;
-    case 2: return AMDGPU::SGPR_64RegClassID;
-    case 4: return AMDGPU::SReg_128RegClassID;
-    case 8: return AMDGPU::SReg_256RegClassID;
-    case 16: return AMDGPU::SReg_512RegClassID;
-  }
-}
-
-static unsigned getRegForName(const StringRef &RegName) {
-
-  return StringSwitch<unsigned>(RegName)
-    .Case("exec", AMDGPU::EXEC)
-    .Case("vcc", AMDGPU::VCC)
-    .Case("flat_scr", AMDGPU::FLAT_SCR)
-    .Case("m0", AMDGPU::M0)
-    .Case("scc", AMDGPU::SCC)
-    .Case("flat_scr_lo", AMDGPU::FLAT_SCR_LO)
-    .Case("flat_scr_hi", AMDGPU::FLAT_SCR_HI)
-    .Case("vcc_lo", AMDGPU::VCC_LO)
-    .Case("vcc_hi", AMDGPU::VCC_HI)
-    .Case("exec_lo", AMDGPU::EXEC_LO)
-    .Case("exec_hi", AMDGPU::EXEC_HI)
-    .Default(0);
-}
-
-bool AMDGPUAsmParser::ParseRegister(unsigned &RegNo, SMLoc &StartLoc, SMLoc &EndLoc) {
-  const AsmToken Tok = Parser.getTok();
-  StartLoc = Tok.getLoc();
-  EndLoc = Tok.getEndLoc();
-  const StringRef &RegName = Tok.getString();
-  RegNo = getRegForName(RegName);
-
-  if (RegNo) {
-    Parser.Lex();
-    return false;
-  }
-
-  // Match vgprs and sgprs
-  if (RegName[0] != 's' && RegName[0] != 'v')
-    return true;
-
-  bool IsVgpr = RegName[0] == 'v';
-  unsigned RegWidth;
-  unsigned RegIndexInClass;
-  if (RegName.size() > 1) {
-    // We have a 32-bit register
-    RegWidth = 1;
-    if (RegName.substr(1).getAsInteger(10, RegIndexInClass))
-      return true;
-    Parser.Lex();
-  } else {
-    // We have a register greater than 32-bits.
-
-    int64_t RegLo, RegHi;
-    Parser.Lex();
-    if (getLexer().isNot(AsmToken::LBrac))
-      return true;
-
-    Parser.Lex();
-    if (getParser().parseAbsoluteExpression(RegLo))
-      return true;
-
-    if (getLexer().isNot(AsmToken::Colon))
-      return true;
-
-    Parser.Lex();
-    if (getParser().parseAbsoluteExpression(RegHi))
-      return true;
-
-    if (getLexer().isNot(AsmToken::RBrac))
-      return true;
-
-    Parser.Lex();
-    RegWidth = (RegHi - RegLo) + 1;
-    if (IsVgpr) {
-      // VGPR registers aren't aligned.
-      RegIndexInClass = RegLo;
-    } else {
-      // SGPR registers are aligned.  Max alignment is 4 dwords.
-      RegIndexInClass = RegLo / std::min(RegWidth, 4u);
-    }
-  }
-
-  const MCRegisterInfo *TRC = getContext().getRegisterInfo();
-  unsigned RC = getRegClass(IsVgpr, RegWidth);
-  if (RegIndexInClass > TRC->getRegClass(RC).getNumRegs())
-    return true;
-  RegNo = TRC->getRegClass(RC).getRegister(RegIndexInClass);
-  return false;
-}
-
-unsigned AMDGPUAsmParser::checkTargetMatchPredicate(MCInst &Inst) {
-
-  uint64_t TSFlags = MII.get(Inst.getOpcode()).TSFlags;
-
-  if ((getForcedEncodingSize() == 32 && (TSFlags & SIInstrFlags::VOP3)) ||
-      (getForcedEncodingSize() == 64 && !(TSFlags & SIInstrFlags::VOP3)))
-    return Match_InvalidOperand;
-
-  return Match_Success;
-}
-
-
-bool AMDGPUAsmParser::MatchAndEmitInstruction(SMLoc IDLoc, unsigned &Opcode,
-                                              OperandVector &Operands,
-                                              MCStreamer &Out,
-                                              uint64_t &ErrorInfo,
-                                              bool MatchingInlineAsm) {
-  MCInst Inst;
-
-  switch (MatchInstructionImpl(Operands, Inst, ErrorInfo, MatchingInlineAsm)) {
-    default: break;
-    case Match_Success:
-      Inst.setLoc(IDLoc);
-      Out.EmitInstruction(Inst, STI);
-      return false;
-    case Match_MissingFeature:
-      return Error(IDLoc, "missing feature");
-
-    case Match_MnemonicFail:
-      return Error(IDLoc, "unrecognized instruction mnemonic");
-
-    case Match_InvalidOperand: {
-      SMLoc ErrorLoc = IDLoc;
-      if (ErrorInfo != ~0ULL) {
-        if (ErrorInfo >= Operands.size()) {
-          return Error(IDLoc, "too few operands for instruction");
-        }
-
-        ErrorLoc = ((AMDGPUOperand &)*Operands[ErrorInfo]).getStartLoc();
-        if (ErrorLoc == SMLoc())
-          ErrorLoc = IDLoc;
-      }
-      return Error(ErrorLoc, "invalid operand for instruction");
-    }
-  }
-  llvm_unreachable("Implement any new match types added!");
-}
-
-bool AMDGPUAsmParser::ParseDirective(AsmToken DirectiveID) {
-  return true;
-}
-
-static bool operandsHaveModifiers(const OperandVector &Operands) {
-
-  for (unsigned i = 0, e = Operands.size(); i != e; ++i) {
-    const AMDGPUOperand &Op = ((AMDGPUOperand&)*Operands[i]);
-    if (Op.isRegWithInputMods())
-      return true;
-    if (Op.isImm() && (Op.getImmTy() == AMDGPUOperand::ImmTyOMod ||
-                       Op.getImmTy() == AMDGPUOperand::ImmTyClamp))
-      return true;
-  }
-  return false;
-}
-
-AMDGPUAsmParser::OperandMatchResultTy
-AMDGPUAsmParser::parseOperand(OperandVector &Operands, StringRef Mnemonic) {
-
-  // Try to parse with a custom parser
-  OperandMatchResultTy ResTy = MatchOperandParserImpl(Operands, Mnemonic);
-
-  // If we successfully parsed the operand or if there as an error parsing,
-  // we are done.
-  //
-  // If we are parsing after we reach EndOfStatement then this means we
-  // are appending default values to the Operands list.  This is only done
-  // by custom parser, so we shouldn't continue on to the generic parsing.
-  if (ResTy == MatchOperand_Success || ResTy == MatchOperand_ParseFail ||
-      getLexer().is(AsmToken::EndOfStatement))
-    return ResTy;
-
-  bool Negate = false, Abs = false;
-  if (getLexer().getKind()== AsmToken::Minus) {
-    Parser.Lex();
-    Negate = true;
-  }
-
-  if (getLexer().getKind() == AsmToken::Pipe) {
-    Parser.Lex();
-    Abs = true;
-  }
-
-  switch(getLexer().getKind()) {
-    case AsmToken::Integer: {
-      SMLoc S = Parser.getTok().getLoc();
-      int64_t IntVal;
-      if (getParser().parseAbsoluteExpression(IntVal))
-        return MatchOperand_ParseFail;
-      APInt IntVal32(32, IntVal);
-      if (IntVal32.getSExtValue() != IntVal) {
-        Error(S, "invalid immediate: only 32-bit values are legal");
-        return MatchOperand_ParseFail;
-      }
-
-      IntVal = IntVal32.getSExtValue();
-      if (Negate)
-        IntVal *= -1;
-      Operands.push_back(AMDGPUOperand::CreateImm(IntVal, S));
-      return MatchOperand_Success;
-    }
-    case AsmToken::Real: {
-      // FIXME: We should emit an error if a double precisions floating-point
-      // value is used.  I'm not sure the best way to detect this.
-      SMLoc S = Parser.getTok().getLoc();
-      int64_t IntVal;
-      if (getParser().parseAbsoluteExpression(IntVal))
-        return MatchOperand_ParseFail;
-
-      APFloat F((float)BitsToDouble(IntVal));
-      if (Negate)
-        F.changeSign();
-      Operands.push_back(
-          AMDGPUOperand::CreateImm(F.bitcastToAPInt().getZExtValue(), S));
-      return MatchOperand_Success;
-    }
-    case AsmToken::Identifier: {
-      SMLoc S, E;
-      unsigned RegNo;
-      if (!ParseRegister(RegNo, S, E)) {
-
-        bool HasModifiers = operandsHaveModifiers(Operands);
-        unsigned Modifiers = 0;
-
-        if (Negate)
-          Modifiers |= 0x1;
-
-        if (Abs) {
-          if (getLexer().getKind() != AsmToken::Pipe)
-            return MatchOperand_ParseFail;
-          Parser.Lex();
-          Modifiers |= 0x2;
-        }
-
-        if (Modifiers && !HasModifiers) {
-          // We are adding a modifier to src1 or src2 and previous sources
-          // don't have modifiers, so we need to go back and empty modifers
-          // for each previous source.
-          for (unsigned PrevRegIdx = Operands.size() - 1; PrevRegIdx > 1;
-               --PrevRegIdx) {
-
-            AMDGPUOperand &RegOp = ((AMDGPUOperand&)*Operands[PrevRegIdx]);
-            RegOp.setModifiers(0);
-          }
-        }
-
-
-        Operands.push_back(AMDGPUOperand::CreateReg(
-            RegNo, S, E, getContext().getRegisterInfo()));
-
-        if (HasModifiers || Modifiers) {
-          AMDGPUOperand &RegOp = ((AMDGPUOperand&)*Operands[Operands.size() - 1]);
-          RegOp.setModifiers(Modifiers);
-
-        }
-     }  else {
-      Operands.push_back(AMDGPUOperand::CreateToken(Parser.getTok().getString(),
-                                                    S));
-      Parser.Lex();
-     }
-     return MatchOperand_Success;
-    }
-    default:
-      return MatchOperand_NoMatch;
-  }
-}
-
-bool AMDGPUAsmParser::ParseInstruction(ParseInstructionInfo &Info,
-                                       StringRef Name,
-                                       SMLoc NameLoc, OperandVector &Operands) {
-
-  // Clear any forced encodings from the previous instruction.
-  setForcedEncodingSize(0);
-
-  if (Name.endswith("_e64"))
-    setForcedEncodingSize(64);
-  else if (Name.endswith("_e32"))
-    setForcedEncodingSize(32);
-
-  // Add the instruction mnemonic
-  Operands.push_back(AMDGPUOperand::CreateToken(Name, NameLoc));
-
-  while (!getLexer().is(AsmToken::EndOfStatement)) {
-    AMDGPUAsmParser::OperandMatchResultTy Res = parseOperand(Operands, Name);
-
-    // Eat the comma or space if there is one.
-    if (getLexer().is(AsmToken::Comma))
-      Parser.Lex();
-
-    switch (Res) {
-      case MatchOperand_Success: break;
-      case MatchOperand_ParseFail: return Error(getLexer().getLoc(),
-                                                "failed parsing operand.");
-      case MatchOperand_NoMatch: return Error(getLexer().getLoc(),
-                                              "not a valid operand.");
-    }
-  }
-
-  // Once we reach end of statement, continue parsing so we can add default
-  // values for optional arguments.
-  AMDGPUAsmParser::OperandMatchResultTy Res;
-  while ((Res = parseOperand(Operands, Name)) != MatchOperand_NoMatch) {
-    if (Res != MatchOperand_Success)
-      return Error(getLexer().getLoc(), "failed parsing operand.");
-  }
-  return false;
-}
-
-//===----------------------------------------------------------------------===//
-// Utility functions
-//===----------------------------------------------------------------------===//
-
-AMDGPUAsmParser::OperandMatchResultTy
-AMDGPUAsmParser::parseIntWithPrefix(const char *Prefix, int64_t &Int,
-                                    int64_t Default) {
-
-  // We are at the end of the statement, and this is a default argument, so
-  // use a default value.
-  if (getLexer().is(AsmToken::EndOfStatement)) {
-    Int = Default;
-    return MatchOperand_Success;
-  }
-
-  switch(getLexer().getKind()) {
-    default: return MatchOperand_NoMatch;
-    case AsmToken::Identifier: {
-      StringRef OffsetName = Parser.getTok().getString();
-      if (!OffsetName.equals(Prefix))
-        return MatchOperand_NoMatch;
-
-      Parser.Lex();
-      if (getLexer().isNot(AsmToken::Colon))
-        return MatchOperand_ParseFail;
-
-      Parser.Lex();
-      if (getLexer().isNot(AsmToken::Integer))
-        return MatchOperand_ParseFail;
-
-      if (getParser().parseAbsoluteExpression(Int))
-        return MatchOperand_ParseFail;
-      break;
-    }
-  }
-  return MatchOperand_Success;
-}
-
-AMDGPUAsmParser::OperandMatchResultTy
-AMDGPUAsmParser::parseIntWithPrefix(const char *Prefix, OperandVector &Operands,
-                                    enum AMDGPUOperand::ImmTy ImmTy) {
-
-  SMLoc S = Parser.getTok().getLoc();
-  int64_t Offset = 0;
-
-  AMDGPUAsmParser::OperandMatchResultTy Res = parseIntWithPrefix(Prefix, Offset);
-  if (Res != MatchOperand_Success)
-    return Res;
-
-  Operands.push_back(AMDGPUOperand::CreateImm(Offset, S, ImmTy));
-  return MatchOperand_Success;
-}
-
-AMDGPUAsmParser::OperandMatchResultTy
-AMDGPUAsmParser::parseNamedBit(const char *Name, OperandVector &Operands,
-                               enum AMDGPUOperand::ImmTy ImmTy) {
-  int64_t Bit = 0;
-  SMLoc S = Parser.getTok().getLoc();
-
-  // We are at the end of the statement, and this is a default argument, so
-  // use a default value.
-  if (getLexer().isNot(AsmToken::EndOfStatement)) {
-    switch(getLexer().getKind()) {
-      case AsmToken::Identifier: {
-        StringRef Tok = Parser.getTok().getString();
-        if (Tok == Name) {
-          Bit = 1;
-          Parser.Lex();
-        } else if (Tok.startswith("no") && Tok.endswith(Name)) {
-          Bit = 0;
-          Parser.Lex();
-        } else {
-          return MatchOperand_NoMatch;
-        }
-        break;
-      }
-      default:
-        return MatchOperand_NoMatch;
-    }
-  }
-
-  Operands.push_back(AMDGPUOperand::CreateImm(Bit, S, ImmTy));
-  return MatchOperand_Success;
-}
-
-static bool operandsHasOptionalOp(const OperandVector &Operands,
-                                  const OptionalOperand &OOp) {
-  for (unsigned i = 0; i < Operands.size(); i++) {
-    const AMDGPUOperand &ParsedOp = ((const AMDGPUOperand &)*Operands[i]);
-    if ((ParsedOp.isImm() && ParsedOp.getImmTy() == OOp.Type) ||
-        (ParsedOp.isToken() && ParsedOp.getToken() == OOp.Name))
-      return true;
-
-  }
-  return false;
-}
-
-AMDGPUAsmParser::OperandMatchResultTy
-AMDGPUAsmParser::parseOptionalOps(const ArrayRef<OptionalOperand> &OptionalOps,
-                                   OperandVector &Operands) {
-  SMLoc S = Parser.getTok().getLoc();
-  for (const OptionalOperand &Op : OptionalOps) {
-    if (operandsHasOptionalOp(Operands, Op))
-      continue;
-    AMDGPUAsmParser::OperandMatchResultTy Res;
-    int64_t Value;
-    if (Op.IsBit) {
-      Res = parseNamedBit(Op.Name, Operands, Op.Type);
-      if (Res == MatchOperand_NoMatch)
-        continue;
-      return Res;
-    }
-
-    Res = parseIntWithPrefix(Op.Name, Value, Op.Default);
-
-    if (Res == MatchOperand_NoMatch)
-      continue;
-
-    if (Res != MatchOperand_Success)
-      return Res;
-
-    if (Op.ConvertResult && !Op.ConvertResult(Value)) {
-      return MatchOperand_ParseFail;
-    }
-
-    Operands.push_back(AMDGPUOperand::CreateImm(Value, S, Op.Type));
-    return MatchOperand_Success;
-  }
-  return MatchOperand_NoMatch;
-}
-
-//===----------------------------------------------------------------------===//
-// ds
-//===----------------------------------------------------------------------===//
-
-static const OptionalOperand DSOptionalOps [] = {
-  {"offset",  AMDGPUOperand::ImmTyOffset, false, 0, nullptr},
-  {"gds",     AMDGPUOperand::ImmTyGDS, true, 0, nullptr}
-};
-
-static const OptionalOperand DSOptionalOpsOff01 [] = {
-  {"offset0", AMDGPUOperand::ImmTyDSOffset0, false, 0, nullptr},
-  {"offset1", AMDGPUOperand::ImmTyDSOffset1, false, 0, nullptr},
-  {"gds",     AMDGPUOperand::ImmTyGDS, true, 0, nullptr}
-};
-
-AMDGPUAsmParser::OperandMatchResultTy
-AMDGPUAsmParser::parseDSOptionalOps(OperandVector &Operands) {
-  return parseOptionalOps(DSOptionalOps, Operands);
-}
-AMDGPUAsmParser::OperandMatchResultTy
-AMDGPUAsmParser::parseDSOff01OptionalOps(OperandVector &Operands) {
-  return parseOptionalOps(DSOptionalOpsOff01, Operands);
-}
-
-AMDGPUAsmParser::OperandMatchResultTy
-AMDGPUAsmParser::parseDSOffsetOptional(OperandVector &Operands) {
-  SMLoc S = Parser.getTok().getLoc();
-  AMDGPUAsmParser::OperandMatchResultTy Res =
-    parseIntWithPrefix("offset", Operands, AMDGPUOperand::ImmTyOffset);
-  if (Res == MatchOperand_NoMatch) {
-    Operands.push_back(AMDGPUOperand::CreateImm(0, S,
-                       AMDGPUOperand::ImmTyOffset));
-    Res = MatchOperand_Success;
-  }
-  return Res;
-}
-
-bool AMDGPUOperand::isDSOffset() const {
-  return isImm() && isUInt<16>(getImm());
-}
-
-bool AMDGPUOperand::isDSOffset01() const {
-  return isImm() && isUInt<8>(getImm());
-}
-
-void AMDGPUAsmParser::cvtDSOffset01(MCInst &Inst,
-                                    const OperandVector &Operands) {
-
-  std::map<enum AMDGPUOperand::ImmTy, unsigned> OptionalIdx;
-
-  for (unsigned i = 1, e = Operands.size(); i != e; ++i) {
-    AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[i]);
-
-    // Add the register arguments
-    if (Op.isReg()) {
-      Op.addRegOperands(Inst, 1);
-      continue;
-    }
-
-    // Handle optional arguments
-    OptionalIdx[Op.getImmTy()] = i;
-  }
-
-  unsigned Offset0Idx = OptionalIdx[AMDGPUOperand::ImmTyDSOffset0];
-  unsigned Offset1Idx = OptionalIdx[AMDGPUOperand::ImmTyDSOffset1];
-  unsigned GDSIdx = OptionalIdx[AMDGPUOperand::ImmTyGDS];
-
-  ((AMDGPUOperand &)*Operands[Offset0Idx]).addImmOperands(Inst, 1); // offset0
-  ((AMDGPUOperand &)*Operands[Offset1Idx]).addImmOperands(Inst, 1); // offset1
-  ((AMDGPUOperand &)*Operands[GDSIdx]).addImmOperands(Inst, 1); // gds
-  Inst.addOperand(MCOperand::CreateReg(AMDGPU::M0)); // m0
-}
-
-void AMDGPUAsmParser::cvtDS(MCInst &Inst, const OperandVector &Operands) {
-
-  std::map<enum AMDGPUOperand::ImmTy, unsigned> OptionalIdx;
-  bool GDSOnly = false;
-
-  for (unsigned i = 1, e = Operands.size(); i != e; ++i) {
-    AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[i]);
-
-    // Add the register arguments
-    if (Op.isReg()) {
-      Op.addRegOperands(Inst, 1);
-      continue;
-    }
-
-    if (Op.isToken() && Op.getToken() == "gds") {
-      GDSOnly = true;
-      continue;
-    }
-
-    // Handle optional arguments
-    OptionalIdx[Op.getImmTy()] = i;
-  }
-
-  unsigned OffsetIdx = OptionalIdx[AMDGPUOperand::ImmTyOffset];
-  ((AMDGPUOperand &)*Operands[OffsetIdx]).addImmOperands(Inst, 1); // offset
-
-  if (!GDSOnly) {
-    unsigned GDSIdx = OptionalIdx[AMDGPUOperand::ImmTyGDS];
-    ((AMDGPUOperand &)*Operands[GDSIdx]).addImmOperands(Inst, 1); // gds
-  }
-  Inst.addOperand(MCOperand::CreateReg(AMDGPU::M0)); // m0
-}
-
-
-//===----------------------------------------------------------------------===//
-// s_waitcnt
-//===----------------------------------------------------------------------===//
-
-bool AMDGPUAsmParser::parseCnt(int64_t &IntVal) {
-  StringRef CntName = Parser.getTok().getString();
-  int64_t CntVal;
-
-  Parser.Lex();
-  if (getLexer().isNot(AsmToken::LParen))
-    return true;
-
-  Parser.Lex();
-  if (getLexer().isNot(AsmToken::Integer))
-    return true;
-
-  if (getParser().parseAbsoluteExpression(CntVal))
-    return true;
-
-  if (getLexer().isNot(AsmToken::RParen))
-    return true;
-
-  Parser.Lex();
-  if (getLexer().is(AsmToken::Amp) || getLexer().is(AsmToken::Comma))
-    Parser.Lex();
-
-  int CntShift;
-  int CntMask;
-
-  if (CntName == "vmcnt") {
-    CntMask = 0xf;
-    CntShift = 0;
-  } else if (CntName == "expcnt") {
-    CntMask = 0x7;
-    CntShift = 4;
-  } else if (CntName == "lgkmcnt") {
-    CntMask = 0x7;
-    CntShift = 8;
-  } else {
-    return true;
-  }
-
-  IntVal &= ~(CntMask << CntShift);
-  IntVal |= (CntVal << CntShift);
-  return false;
-}
-
-AMDGPUAsmParser::OperandMatchResultTy
-AMDGPUAsmParser::parseSWaitCntOps(OperandVector &Operands) {
-  // Disable all counters by default.
-  // vmcnt   [3:0]
-  // expcnt  [6:4]
-  // lgkmcnt [10:8]
-  int64_t CntVal = 0x77f;
-  SMLoc S = Parser.getTok().getLoc();
-
-  switch(getLexer().getKind()) {
-    default: return MatchOperand_ParseFail;
-    case AsmToken::Integer:
-      // The operand can be an integer value.
-      if (getParser().parseAbsoluteExpression(CntVal))
-        return MatchOperand_ParseFail;
-      break;
-
-    case AsmToken::Identifier:
-      do {
-        if (parseCnt(CntVal))
-          return MatchOperand_ParseFail;
-      } while(getLexer().isNot(AsmToken::EndOfStatement));
-      break;
-  }
-  Operands.push_back(AMDGPUOperand::CreateImm(CntVal, S));
-  return MatchOperand_Success;
-}
-
-bool AMDGPUOperand::isSWaitCnt() const {
-  return isImm();
-}
-
-//===----------------------------------------------------------------------===//
-// sopp branch targets
-//===----------------------------------------------------------------------===//
-
-AMDGPUAsmParser::OperandMatchResultTy
-AMDGPUAsmParser::parseSOppBrTarget(OperandVector &Operands) {
-  SMLoc S = Parser.getTok().getLoc();
-
-  switch (getLexer().getKind()) {
-    default: return MatchOperand_ParseFail;
-    case AsmToken::Integer: {
-      int64_t Imm;
-      if (getParser().parseAbsoluteExpression(Imm))
-        return MatchOperand_ParseFail;
-      Operands.push_back(AMDGPUOperand::CreateImm(Imm, S));
-      return MatchOperand_Success;
-    }
-
-    case AsmToken::Identifier:
-      Operands.push_back(AMDGPUOperand::CreateExpr(
-          MCSymbolRefExpr::Create(getContext().GetOrCreateSymbol(
-                                  Parser.getTok().getString()), getContext()), S));
-      Parser.Lex();
-      return MatchOperand_Success;
-  }
-}
-
-//===----------------------------------------------------------------------===//
-// mubuf
-//===----------------------------------------------------------------------===//
-
-static const OptionalOperand MubufOptionalOps [] = {
-  {"offset", AMDGPUOperand::ImmTyOffset, false, 0, nullptr},
-  {"glc",    AMDGPUOperand::ImmTyGLC, true, 0, nullptr},
-  {"slc",    AMDGPUOperand::ImmTySLC, true, 0, nullptr},
-  {"tfe",    AMDGPUOperand::ImmTyTFE, true, 0, nullptr}
-};
-
-AMDGPUAsmParser::OperandMatchResultTy
-AMDGPUAsmParser::parseMubufOptionalOps(OperandVector &Operands) {
-  return parseOptionalOps(MubufOptionalOps, Operands);
-}
-
-AMDGPUAsmParser::OperandMatchResultTy
-AMDGPUAsmParser::parseOffset(OperandVector &Operands) {
-  return parseIntWithPrefix("offset", Operands);
-}
-
-AMDGPUAsmParser::OperandMatchResultTy
-AMDGPUAsmParser::parseGLC(OperandVector &Operands) {
-  return parseNamedBit("glc", Operands);
-}
-
-AMDGPUAsmParser::OperandMatchResultTy
-AMDGPUAsmParser::parseSLC(OperandVector &Operands) {
-  return parseNamedBit("slc", Operands);
-}
-
-AMDGPUAsmParser::OperandMatchResultTy
-AMDGPUAsmParser::parseTFE(OperandVector &Operands) {
-  return parseNamedBit("tfe", Operands);
-}
-
-bool AMDGPUOperand::isMubufOffset() const {
-  return isImm() && isUInt<12>(getImm());
-}
-
-void AMDGPUAsmParser::cvtMubuf(MCInst &Inst,
-                               const OperandVector &Operands) {
-  std::map<enum AMDGPUOperand::ImmTy, unsigned> OptionalIdx;
-
-  for (unsigned i = 1, e = Operands.size(); i != e; ++i) {
-    AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[i]);
-
-    // Add the register arguments
-    if (Op.isReg()) {
-      Op.addRegOperands(Inst, 1);
-      continue;
-    }
-
-    // Handle the case where soffset is an immediate
-    if (Op.isImm() && Op.getImmTy() == AMDGPUOperand::ImmTyNone) {
-      Op.addImmOperands(Inst, 1);
-      continue;
-    }
-
-    // Handle tokens like 'offen' which are sometimes hard-coded into the
-    // asm string.  There are no MCInst operands for these.
-    if (Op.isToken()) {
-      continue;
-    }
-    assert(Op.isImm());
-
-    // Handle optional arguments
-    OptionalIdx[Op.getImmTy()] = i;
-  }
-
-  assert(OptionalIdx.size() == 4);
-
-  unsigned OffsetIdx = OptionalIdx[AMDGPUOperand::ImmTyOffset];
-  unsigned GLCIdx = OptionalIdx[AMDGPUOperand::ImmTyGLC];
-  unsigned SLCIdx = OptionalIdx[AMDGPUOperand::ImmTySLC];
-  unsigned TFEIdx = OptionalIdx[AMDGPUOperand::ImmTyTFE];
-
-  ((AMDGPUOperand &)*Operands[OffsetIdx]).addImmOperands(Inst, 1);
-  ((AMDGPUOperand &)*Operands[GLCIdx]).addImmOperands(Inst, 1);
-  ((AMDGPUOperand &)*Operands[SLCIdx]).addImmOperands(Inst, 1);
-  ((AMDGPUOperand &)*Operands[TFEIdx]).addImmOperands(Inst, 1);
-}
-
-//===----------------------------------------------------------------------===//
-// mimg
-//===----------------------------------------------------------------------===//
-
-AMDGPUAsmParser::OperandMatchResultTy
-AMDGPUAsmParser::parseDMask(OperandVector &Operands) {
-  return parseIntWithPrefix("dmask", Operands);
-}
-
-AMDGPUAsmParser::OperandMatchResultTy
-AMDGPUAsmParser::parseUNorm(OperandVector &Operands) {
-  return parseNamedBit("unorm", Operands);
-}
-
-AMDGPUAsmParser::OperandMatchResultTy
-AMDGPUAsmParser::parseR128(OperandVector &Operands) {
-  return parseNamedBit("r128", Operands);
-}
-
-//===----------------------------------------------------------------------===//
-// vop3
-//===----------------------------------------------------------------------===//
-
-static bool ConvertOmodMul(int64_t &Mul) {
-  if (Mul != 1 && Mul != 2 && Mul != 4)
-    return false;
-
-  Mul >>= 1;
-  return true;
-}
-
-static bool ConvertOmodDiv(int64_t &Div) {
-  if (Div == 1) {
-    Div = 0;
-    return true;
-  }
-
-  if (Div == 2) {
-    Div = 3;
-    return true;
-  }
-
-  return false;
-}
-
-static const OptionalOperand VOP3OptionalOps [] = {
-  {"clamp", AMDGPUOperand::ImmTyClamp, true, 0, nullptr},
-  {"mul",   AMDGPUOperand::ImmTyOMod, false, 1, ConvertOmodMul},
-  {"div",   AMDGPUOperand::ImmTyOMod, false, 1, ConvertOmodDiv},
-};
-
-static bool isVOP3(OperandVector &Operands) {
-  if (operandsHaveModifiers(Operands))
-    return true;
-
-  AMDGPUOperand &DstOp = ((AMDGPUOperand&)*Operands[1]);
-
-  if (DstOp.isReg() && DstOp.isRegClass(AMDGPU::SGPR_64RegClassID))
-    return true;
-
-  if (Operands.size() >= 5)
-    return true;
-
-  if (Operands.size() > 3) {
-    AMDGPUOperand &Src1Op = ((AMDGPUOperand&)*Operands[3]);
-    if (Src1Op.getReg() && (Src1Op.isRegClass(AMDGPU::SReg_32RegClassID) ||
-                            Src1Op.isRegClass(AMDGPU::SReg_64RegClassID)))
-      return true;
-  }
-  return false;
-}
-
-AMDGPUAsmParser::OperandMatchResultTy
-AMDGPUAsmParser::parseVOP3OptionalOps(OperandVector &Operands) {
-
-  // The value returned by this function may change after parsing
-  // an operand so store the original value here.
-  bool HasModifiers = operandsHaveModifiers(Operands);
-
-  bool IsVOP3 = isVOP3(Operands);
-  if (HasModifiers || IsVOP3 ||
-      getLexer().isNot(AsmToken::EndOfStatement) ||
-      getForcedEncodingSize() == 64) {
-
-    AMDGPUAsmParser::OperandMatchResultTy Res =
-        parseOptionalOps(VOP3OptionalOps, Operands);
-
-    if (!HasModifiers && Res == MatchOperand_Success) {
-      // We have added a modifier operation, so we need to make sure all
-      // previous register operands have modifiers
-      for (unsigned i = 2, e = Operands.size(); i != e; ++i) {
-        AMDGPUOperand &Op = ((AMDGPUOperand&)*Operands[i]);
-        if (Op.isReg())
-          Op.setModifiers(0);
-      }
-    }
-    return Res;
-  }
-  return MatchOperand_NoMatch;
-}
-
-void AMDGPUAsmParser::cvtVOP3(MCInst &Inst, const OperandVector &Operands) {
-  ((AMDGPUOperand &)*Operands[1]).addRegOperands(Inst, 1);
-  unsigned i = 2;
-
-  std::map<enum AMDGPUOperand::ImmTy, unsigned> OptionalIdx;
-
-  if (operandsHaveModifiers(Operands)) {
-    for (unsigned e = Operands.size(); i != e; ++i) {
-      AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[i]);
-
-      if (Op.isRegWithInputMods()) {
-        ((AMDGPUOperand &)*Operands[i]).addRegWithInputModsOperands(Inst, 2);
-        continue;
-      }
-      OptionalIdx[Op.getImmTy()] = i;
-    }
-
-    unsigned ClampIdx = OptionalIdx[AMDGPUOperand::ImmTyClamp];
-    unsigned OModIdx = OptionalIdx[AMDGPUOperand::ImmTyOMod];
-
-    ((AMDGPUOperand &)*Operands[ClampIdx]).addImmOperands(Inst, 1);
-    ((AMDGPUOperand &)*Operands[OModIdx]).addImmOperands(Inst, 1);
-  } else {
-    for (unsigned e = Operands.size(); i != e; ++i)
-      ((AMDGPUOperand &)*Operands[i]).addRegOrImmOperands(Inst, 1);
-  }
-}
-
-/// Force static initialization.
-extern "C" void LLVMInitializeR600AsmParser() {
-  RegisterMCAsmParser<AMDGPUAsmParser> A(TheAMDGPUTarget);
-  RegisterMCAsmParser<AMDGPUAsmParser> B(TheGCNTarget);
-}
-
-#define GET_REGISTER_MATCHER
-#define GET_MATCHER_IMPLEMENTATION
-#include "AMDGPUGenAsmMatcher.inc"
-
diff --git a/lib/Target/R600/AsmParser/CMakeLists.txt b/lib/Target/R600/AsmParser/CMakeLists.txt
deleted file mode 100644
index 1b42af73740..00000000000
--- a/lib/Target/R600/AsmParser/CMakeLists.txt
+++ /dev/null
@@ -1,3 +0,0 @@
-add_llvm_library(LLVMR600AsmParser
-  AMDGPUAsmParser.cpp
-  )
diff --git a/lib/Target/R600/AsmParser/LLVMBuild.txt b/lib/Target/R600/AsmParser/LLVMBuild.txt
deleted file mode 100644
index 940e4cee6df..00000000000
--- a/lib/Target/R600/AsmParser/LLVMBuild.txt
+++ /dev/null
@@ -1,23 +0,0 @@
-;===- ./lib/Target/R600/AsmParser/LLVMBuild.txt -------------*- Conf -*--===;
-;
-;                     The LLVM Compiler Infrastructure
-;
-; This file is distributed under the University of Illinois Open Source
-; License. See LICENSE.TXT for details.
-;
-;===------------------------------------------------------------------------===;
-;
-; This is an LLVMBuild description file for the components in this subdirectory.
-;
-; For more information on the LLVMBuild system, please see:
-;
-;   http://llvm.org/docs/LLVMBuild.html
-;
-;===------------------------------------------------------------------------===;
-
-[component_0]
-type = Library
-name = R600AsmParser
-parent = R600
-required_libraries = MC MCParser R600Desc R600Info Support
-add_to_library_groups = R600
diff --git a/lib/Target/R600/AsmParser/Makefile b/lib/Target/R600/AsmParser/Makefile
deleted file mode 100644
index e6689b54b6b..00000000000
--- a/lib/Target/R600/AsmParser/Makefile
+++ /dev/null
@@ -1,15 +0,0 @@
-##===- lib/Target/R600/AsmParser/Makefile ----------------*- Makefile -*-===##
-#
-#                     The LLVM Compiler Infrastructure
-#
-# This file is distributed under the University of Illinois Open Source
-# License. See LICENSE.TXT for details.
-#
-##===----------------------------------------------------------------------===##
-LEVEL = ../../../..
-LIBRARYNAME = LLVMR600AsmParser
-
-# Hack: we need to include 'main' R600 target directory to grab private headers
-CPP.Flags += -I$(PROJ_OBJ_DIR)/.. -I$(PROJ_SRC_DIR)/..
-
-include $(LEVEL)/Makefile.common
diff --git a/lib/Target/R600/CIInstructions.td b/lib/Target/R600/CIInstructions.td
deleted file mode 100644
index 3ac7af83015..00000000000
--- a/lib/Target/R600/CIInstructions.td
+++ /dev/null
@@ -1,42 +0,0 @@
-//===-- CIInstructions.td - CI Instruction Defintions ---------------------===//
-//
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
-//
-//===----------------------------------------------------------------------===//
-// Instruction definitions for CI and newer.
-//===----------------------------------------------------------------------===//
-
-
-def isCIVI : Predicate <
-  "Subtarget->getGeneration() == AMDGPUSubtarget::SEA_ISLANDS || "
-  "Subtarget->getGeneration() == AMDGPUSubtarget::VOLCANIC_ISLANDS"
->;
-
-//===----------------------------------------------------------------------===//
-// VOP1 Instructions
-//===----------------------------------------------------------------------===//
-
-let SubtargetPredicate = isCIVI in {
-
-defm V_TRUNC_F64 : VOP1Inst <vop1<0x17>, "v_trunc_f64",
-  VOP_F64_F64, ftrunc
->;
-defm V_CEIL_F64 : VOP1Inst <vop1<0x18>, "v_ceil_f64",
-  VOP_F64_F64, fceil
->;
-defm V_FLOOR_F64 : VOP1Inst <vop1<0x1A>, "v_floor_f64",
-  VOP_F64_F64, ffloor
->;
-defm V_RNDNE_F64 : VOP1Inst <vop1<0x19>, "v_rndne_f64",
-  VOP_F64_F64, frint
->;
-defm V_LOG_LEGACY_F32 : VOP1Inst <vop1<0x45, 0x4c>, "v_log_legacy_f32",
-  VOP_F32_F32
->;
-defm V_EXP_LEGACY_F32 : VOP1Inst <vop1<0x46, 0x4b>, "v_exp_legacy_f32",
-  VOP_F32_F32
->;
-} // End SubtargetPredicate = isCIVI
diff --git a/lib/Target/R600/CMakeLists.txt b/lib/Target/R600/CMakeLists.txt
deleted file mode 100644
index 5a4bae2f93c..00000000000
--- a/lib/Target/R600/CMakeLists.txt
+++ /dev/null
@@ -1,63 +0,0 @@
-set(LLVM_TARGET_DEFINITIONS AMDGPU.td)
-
-tablegen(LLVM AMDGPUGenRegisterInfo.inc -gen-register-info)
-tablegen(LLVM AMDGPUGenInstrInfo.inc -gen-instr-info)
-tablegen(LLVM AMDGPUGenDAGISel.inc -gen-dag-isel)
-tablegen(LLVM AMDGPUGenCallingConv.inc -gen-callingconv)
-tablegen(LLVM AMDGPUGenSubtargetInfo.inc -gen-subtarget)
-tablegen(LLVM AMDGPUGenIntrinsics.inc -gen-tgt-intrinsic)
-tablegen(LLVM AMDGPUGenMCCodeEmitter.inc -gen-emitter)
-tablegen(LLVM AMDGPUGenDFAPacketizer.inc -gen-dfa-packetizer)
-tablegen(LLVM AMDGPUGenAsmWriter.inc -gen-asm-writer)
-tablegen(LLVM AMDGPUGenAsmMatcher.inc -gen-asm-matcher)
-add_public_tablegen_target(AMDGPUCommonTableGen)
-
-add_llvm_target(R600CodeGen
-  AMDILCFGStructurizer.cpp
-  AMDGPUAlwaysInlinePass.cpp
-  AMDGPUAsmPrinter.cpp
-  AMDGPUFrameLowering.cpp
-  AMDGPUIntrinsicInfo.cpp
-  AMDGPUISelDAGToDAG.cpp
-  AMDGPUMCInstLower.cpp
-  AMDGPUMachineFunction.cpp
-  AMDGPUSubtarget.cpp
-  AMDGPUTargetMachine.cpp
-  AMDGPUTargetTransformInfo.cpp
-  AMDGPUISelLowering.cpp
-  AMDGPUInstrInfo.cpp
-  AMDGPUPromoteAlloca.cpp
-  AMDGPURegisterInfo.cpp
-  R600ClauseMergePass.cpp
-  R600ControlFlowFinalizer.cpp
-  R600EmitClauseMarkers.cpp
-  R600ExpandSpecialInstrs.cpp
-  R600InstrInfo.cpp
-  R600ISelLowering.cpp
-  R600MachineFunctionInfo.cpp
-  R600MachineScheduler.cpp
-  R600OptimizeVectorRegisters.cpp
-  R600Packetizer.cpp
-  R600RegisterInfo.cpp
-  R600TextureIntrinsicsReplacer.cpp
-  SIAnnotateControlFlow.cpp
-  SIFixSGPRCopies.cpp
-  SIFixSGPRLiveRanges.cpp
-  SIFoldOperands.cpp
-  SIInsertWaits.cpp
-  SIInstrInfo.cpp
-  SIISelLowering.cpp
-  SILoadStoreOptimizer.cpp
-  SILowerControlFlow.cpp
-  SILowerI1Copies.cpp
-  SIMachineFunctionInfo.cpp
-  SIPrepareScratchRegs.cpp
-  SIRegisterInfo.cpp
-  SIShrinkInstructions.cpp
-  SITypeRewriter.cpp
-  )
-
-add_subdirectory(AsmParser)
-add_subdirectory(InstPrinter)
-add_subdirectory(TargetInfo)
-add_subdirectory(MCTargetDesc)
diff --git a/lib/Target/R600/InstPrinter/CMakeLists.txt b/lib/Target/R600/InstPrinter/CMakeLists.txt
deleted file mode 100644
index dcd87037fab..00000000000
--- a/lib/Target/R600/InstPrinter/CMakeLists.txt
+++ /dev/null
@@ -1,3 +0,0 @@
-add_llvm_library(LLVMR600AsmPrinter
-  AMDGPUInstPrinter.cpp
-  )
diff --git a/lib/Target/R600/InstPrinter/LLVMBuild.txt b/lib/Target/R600/InstPrinter/LLVMBuild.txt
deleted file mode 100644
index ec0be89f104..00000000000
--- a/lib/Target/R600/InstPrinter/LLVMBuild.txt
+++ /dev/null
@@ -1,24 +0,0 @@
-;===- ./lib/Target/R600/InstPrinter/LLVMBuild.txt -----------*- Conf -*--===;
-;
-;                     The LLVM Compiler Infrastructure
-;
-; This file is distributed under the University of Illinois Open Source
-; License. See LICENSE.TXT for details.
-;
-;===------------------------------------------------------------------------===;
-;
-; This is an LLVMBuild description file for the components in this subdirectory.
-;
-; For more information on the LLVMBuild system, please see:
-;
-;   http://llvm.org/docs/LLVMBuild.html
-;
-;===------------------------------------------------------------------------===;
-
-[component_0]
-type = Library
-name = R600AsmPrinter
-parent = R600
-required_libraries = MC Support
-add_to_library_groups = R600
-
diff --git a/lib/Target/R600/InstPrinter/Makefile b/lib/Target/R600/InstPrinter/Makefile
deleted file mode 100644
index a794cc1124e..00000000000
--- a/lib/Target/R600/InstPrinter/Makefile
+++ /dev/null
@@ -1,15 +0,0 @@
-#===- lib/Target/R600/AsmPrinter/Makefile ------------------*- Makefile -*-===##
-#
-#                     The LLVM Compiler Infrastructure
-#
-# This file is distributed under the University of Illinois Open Source
-# License. See LICENSE.TXT for details.
-#
-##===----------------------------------------------------------------------===##
-LEVEL = ../../../..
-LIBRARYNAME = LLVMR600AsmPrinter
-
-# Hack: we need to include 'main' x86 target directory to grab private headers
-CPP.Flags += -I$(PROJ_OBJ_DIR)/.. -I$(PROJ_SRC_DIR)/..
-
-include $(LEVEL)/Makefile.common
diff --git a/lib/Target/R600/LLVMBuild.txt b/lib/Target/R600/LLVMBuild.txt
deleted file mode 100644
index f3f254fdcba..00000000000
--- a/lib/Target/R600/LLVMBuild.txt
+++ /dev/null
@@ -1,33 +0,0 @@
-;===- ./lib/Target/AMDIL/LLVMBuild.txt -------------------------*- Conf -*--===;
-;
-;                     The LLVM Compiler Infrastructure
-;
-; This file is distributed under the University of Illinois Open Source
-; License. See LICENSE.TXT for details.
-;
-;===------------------------------------------------------------------------===;
-;
-; This is an LLVMBuild description file for the components in this subdirectory.
-;
-; For more information on the LLVMBuild system, please see:
-;
-;   http://llvm.org/docs/LLVMBuild.html
-;
-;===------------------------------------------------------------------------===;
-
-[common]
-subdirectories = AsmParser InstPrinter MCTargetDesc TargetInfo
-
-[component_0]
-type = TargetGroup
-name = R600
-parent = Target
-has_asmparser = 1
-has_asmprinter = 1
-
-[component_1]
-type = Library
-name = R600CodeGen
-parent = R600
-required_libraries = Analysis AsmPrinter CodeGen Core IPO MC R600AsmParser R600AsmPrinter R600Desc R600Info Scalar SelectionDAG Support Target TransformUtils
-add_to_library_groups = R600
diff --git a/lib/Target/R600/MCTargetDesc/AMDGPUAsmBackend.cpp b/lib/Target/R600/MCTargetDesc/AMDGPUAsmBackend.cpp
deleted file mode 100644
index f33e692ac3c..00000000000
--- a/lib/Target/R600/MCTargetDesc/AMDGPUAsmBackend.cpp
+++ /dev/null
@@ -1,146 +0,0 @@
-//===-- AMDGPUAsmBackend.cpp - AMDGPU Assembler Backend -------------------===//
-//
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
-//
-/// \file
-//===----------------------------------------------------------------------===//
-
-#include "MCTargetDesc/AMDGPUMCTargetDesc.h"
-#include "MCTargetDesc/AMDGPUFixupKinds.h"
-#include "llvm/ADT/StringRef.h"
-#include "llvm/MC/MCAsmBackend.h"
-#include "llvm/MC/MCAssembler.h"
-#include "llvm/MC/MCFixupKindInfo.h"
-#include "llvm/MC/MCObjectWriter.h"
-#include "llvm/MC/MCValue.h"
-#include "llvm/Support/TargetRegistry.h"
-
-using namespace llvm;
-
-namespace {
-
-class AMDGPUMCObjectWriter : public MCObjectWriter {
-public:
-  AMDGPUMCObjectWriter(raw_pwrite_stream &OS) : MCObjectWriter(OS, true) {}
-  void ExecutePostLayoutBinding(MCAssembler &Asm,
-                                const MCAsmLayout &Layout) override {
-    //XXX: Implement if necessary.
-  }
-  void RecordRelocation(MCAssembler &Asm, const MCAsmLayout &Layout,
-                        const MCFragment *Fragment, const MCFixup &Fixup,
-                        MCValue Target, bool &IsPCRel,
-                        uint64_t &FixedValue) override {
-    assert(!"Not implemented");
-  }
-
-  void WriteObject(MCAssembler &Asm, const MCAsmLayout &Layout) override;
-
-};
-
-class AMDGPUAsmBackend : public MCAsmBackend {
-public:
-  AMDGPUAsmBackend(const Target &T)
-    : MCAsmBackend() {}
-
-  unsigned getNumFixupKinds() const override { return AMDGPU::NumTargetFixupKinds; };
-  void applyFixup(const MCFixup &Fixup, char *Data, unsigned DataSize,
-                  uint64_t Value, bool IsPCRel) const override;
-  bool fixupNeedsRelaxation(const MCFixup &Fixup, uint64_t Value,
-                            const MCRelaxableFragment *DF,
-                            const MCAsmLayout &Layout) const override {
-    return false;
-  }
-  void relaxInstruction(const MCInst &Inst, MCInst &Res) const override {
-    assert(!"Not implemented");
-  }
-  bool mayNeedRelaxation(const MCInst &Inst) const override { return false; }
-  bool writeNopData(uint64_t Count, MCObjectWriter *OW) const override;
-
-  const MCFixupKindInfo &getFixupKindInfo(MCFixupKind Kind) const override;
-};
-
-} //End anonymous namespace
-
-void AMDGPUMCObjectWriter::WriteObject(MCAssembler &Asm,
-                                       const MCAsmLayout &Layout) {
-  for (MCAssembler::iterator I = Asm.begin(), E = Asm.end(); I != E; ++I) {
-    Asm.writeSectionData(I, Layout);
-  }
-}
-
-void AMDGPUAsmBackend::applyFixup(const MCFixup &Fixup, char *Data,
-                                  unsigned DataSize, uint64_t Value,
-                                  bool IsPCRel) const {
-
-  switch ((unsigned)Fixup.getKind()) {
-    default: llvm_unreachable("Unknown fixup kind");
-    case AMDGPU::fixup_si_sopp_br: {
-      uint16_t *Dst = (uint16_t*)(Data + Fixup.getOffset());
-      *Dst = (Value - 4) / 4;
-      break;
-    }
-
-    case AMDGPU::fixup_si_rodata: {
-      uint32_t *Dst = (uint32_t*)(Data + Fixup.getOffset());
-      *Dst = Value;
-      break;
-    }
-
-    case AMDGPU::fixup_si_end_of_text: {
-      uint32_t *Dst = (uint32_t*)(Data + Fixup.getOffset());
-      // The value points to the last instruction in the text section, so we
-      // need to add 4 bytes to get to the start of the constants.
-      *Dst = Value + 4;
-      break;
-    }
-  }
-}
-
-const MCFixupKindInfo &AMDGPUAsmBackend::getFixupKindInfo(
-                                                       MCFixupKind Kind) const {
-  const static MCFixupKindInfo Infos[AMDGPU::NumTargetFixupKinds] = {
-    // name                   offset bits  flags
-    { "fixup_si_sopp_br",     0,     16,   MCFixupKindInfo::FKF_IsPCRel },
-    { "fixup_si_rodata",      0,     32,   0 },
-    { "fixup_si_end_of_text", 0,     32,   MCFixupKindInfo::FKF_IsPCRel }
-  };
-
-  if (Kind < FirstTargetFixupKind)
-    return MCAsmBackend::getFixupKindInfo(Kind);
-
-  return Infos[Kind - FirstTargetFixupKind];
-}
-
-bool AMDGPUAsmBackend::writeNopData(uint64_t Count, MCObjectWriter *OW) const {
-  for (unsigned i = 0; i < Count; ++i)
-    OW->Write8(0);
-
-  return true;
-}
-
-//===----------------------------------------------------------------------===//
-// ELFAMDGPUAsmBackend class
-//===----------------------------------------------------------------------===//
-
-namespace {
-
-class ELFAMDGPUAsmBackend : public AMDGPUAsmBackend {
-public:
-  ELFAMDGPUAsmBackend(const Target &T) : AMDGPUAsmBackend(T) { }
-
-  MCObjectWriter *createObjectWriter(raw_pwrite_stream &OS) const override {
-    return createAMDGPUELFObjectWriter(OS);
-  }
-};
-
-} // end anonymous namespace
-
-MCAsmBackend *llvm::createAMDGPUAsmBackend(const Target &T,
-                                           const MCRegisterInfo &MRI,
-                                           StringRef TT,
-                                           StringRef CPU) {
-  return new ELFAMDGPUAsmBackend(T);
-}
diff --git a/lib/Target/R600/MCTargetDesc/AMDGPUELFObjectWriter.cpp b/lib/Target/R600/MCTargetDesc/AMDGPUELFObjectWriter.cpp
deleted file mode 100644
index 59f45ff02d8..00000000000
--- a/lib/Target/R600/MCTargetDesc/AMDGPUELFObjectWriter.cpp
+++ /dev/null
@@ -1,39 +0,0 @@
-//===-- AMDGPUELFObjectWriter.cpp - AMDGPU ELF Writer ----------------------==//
-//
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
-//
-/// \file
-//===----------------------------------------------------------------------===//
-
-#include "AMDGPUMCTargetDesc.h"
-#include "llvm/MC/MCELFObjectWriter.h"
-#include "llvm/MC/MCFixup.h"
-
-using namespace llvm;
-
-namespace {
-
-class AMDGPUELFObjectWriter : public MCELFObjectTargetWriter {
-public:
-  AMDGPUELFObjectWriter();
-protected:
-  unsigned GetRelocType(const MCValue &Target, const MCFixup &Fixup,
-                        bool IsPCRel) const override {
-    return Fixup.getKind();
-  }
-
-};
-
-
-} // End anonymous namespace
-
-AMDGPUELFObjectWriter::AMDGPUELFObjectWriter()
-  : MCELFObjectTargetWriter(false, 0, 0, false) { }
-
-MCObjectWriter *llvm::createAMDGPUELFObjectWriter(raw_pwrite_stream &OS) {
-  MCELFObjectTargetWriter *MOTW = new AMDGPUELFObjectWriter();
-  return createELFObjectWriter(MOTW, OS, true);
-}
diff --git a/lib/Target/R600/MCTargetDesc/AMDGPUMCTargetDesc.cpp b/lib/Target/R600/MCTargetDesc/AMDGPUMCTargetDesc.cpp
deleted file mode 100644
index 7b280a41558..00000000000
--- a/lib/Target/R600/MCTargetDesc/AMDGPUMCTargetDesc.cpp
+++ /dev/null
@@ -1,90 +0,0 @@
-//===-- AMDGPUMCTargetDesc.cpp - AMDGPU Target Descriptions ---------------===//
-//
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
-//
-//===----------------------------------------------------------------------===//
-//
-/// \file
-/// \brief This file provides AMDGPU specific target descriptions.
-//
-//===----------------------------------------------------------------------===//
-
-#include "AMDGPUMCTargetDesc.h"
-#include "AMDGPUMCAsmInfo.h"
-#include "InstPrinter/AMDGPUInstPrinter.h"
-#include "SIDefines.h"
-#include "llvm/MC/MCCodeGenInfo.h"
-#include "llvm/MC/MCContext.h"
-#include "llvm/MC/MCInstrInfo.h"
-#include "llvm/MC/MCRegisterInfo.h"
-#include "llvm/MC/MCStreamer.h"
-#include "llvm/MC/MCSubtargetInfo.h"
-#include "llvm/MC/MachineLocation.h"
-#include "llvm/Support/ErrorHandling.h"
-#include "llvm/Support/TargetRegistry.h"
-
-using namespace llvm;
-
-#define GET_INSTRINFO_MC_DESC
-#include "AMDGPUGenInstrInfo.inc"
-
-#define GET_SUBTARGETINFO_MC_DESC
-#include "AMDGPUGenSubtargetInfo.inc"
-
-#define GET_REGINFO_MC_DESC
-#include "AMDGPUGenRegisterInfo.inc"
-
-static MCInstrInfo *createAMDGPUMCInstrInfo() {
-  MCInstrInfo *X = new MCInstrInfo();
-  InitAMDGPUMCInstrInfo(X);
-  return X;
-}
-
-static MCRegisterInfo *createAMDGPUMCRegisterInfo(StringRef TT) {
-  MCRegisterInfo *X = new MCRegisterInfo();
-  InitAMDGPUMCRegisterInfo(X, 0);
-  return X;
-}
-
-static MCSubtargetInfo *createAMDGPUMCSubtargetInfo(StringRef TT, StringRef CPU,
-                                                   StringRef FS) {
-  MCSubtargetInfo * X = new MCSubtargetInfo();
-  InitAMDGPUMCSubtargetInfo(X, TT, CPU, FS);
-  return X;
-}
-
-static MCCodeGenInfo *createAMDGPUMCCodeGenInfo(StringRef TT, Reloc::Model RM,
-                                               CodeModel::Model CM,
-                                               CodeGenOpt::Level OL) {
-  MCCodeGenInfo *X = new MCCodeGenInfo();
-  X->InitMCCodeGenInfo(RM, CM, OL);
-  return X;
-}
-
-static MCInstPrinter *createAMDGPUMCInstPrinter(const Triple &T,
-                                                unsigned SyntaxVariant,
-                                                const MCAsmInfo &MAI,
-                                                const MCInstrInfo &MII,
-                                                const MCRegisterInfo &MRI) {
-  return new AMDGPUInstPrinter(MAI, MII, MRI);
-}
-
-extern "C" void LLVMInitializeR600TargetMC() {
-  for (Target *T : {&TheAMDGPUTarget, &TheGCNTarget}) {
-    RegisterMCAsmInfo<AMDGPUMCAsmInfo> X(*T);
-
-    TargetRegistry::RegisterMCCodeGenInfo(*T, createAMDGPUMCCodeGenInfo);
-    TargetRegistry::RegisterMCInstrInfo(*T, createAMDGPUMCInstrInfo);
-    TargetRegistry::RegisterMCRegInfo(*T, createAMDGPUMCRegisterInfo);
-    TargetRegistry::RegisterMCSubtargetInfo(*T, createAMDGPUMCSubtargetInfo);
-    TargetRegistry::RegisterMCInstPrinter(*T, createAMDGPUMCInstPrinter);
-    TargetRegistry::RegisterMCAsmBackend(*T, createAMDGPUAsmBackend);
-  }
-
-  TargetRegistry::RegisterMCCodeEmitter(TheAMDGPUTarget,
-                                        createR600MCCodeEmitter);
-  TargetRegistry::RegisterMCCodeEmitter(TheGCNTarget, createSIMCCodeEmitter);
-}
diff --git a/lib/Target/R600/MCTargetDesc/CMakeLists.txt b/lib/Target/R600/MCTargetDesc/CMakeLists.txt
deleted file mode 100644
index 801c9054937..00000000000
--- a/lib/Target/R600/MCTargetDesc/CMakeLists.txt
+++ /dev/null
@@ -1,10 +0,0 @@
-
-add_llvm_library(LLVMR600Desc
-  AMDGPUAsmBackend.cpp
-  AMDGPUELFObjectWriter.cpp
-  AMDGPUMCCodeEmitter.cpp
-  AMDGPUMCTargetDesc.cpp
-  AMDGPUMCAsmInfo.cpp
-  R600MCCodeEmitter.cpp
-  SIMCCodeEmitter.cpp
-  )
diff --git a/lib/Target/R600/MCTargetDesc/LLVMBuild.txt b/lib/Target/R600/MCTargetDesc/LLVMBuild.txt
deleted file mode 100644
index 74b8ca09ae1..00000000000
--- a/lib/Target/R600/MCTargetDesc/LLVMBuild.txt
+++ /dev/null
@@ -1,23 +0,0 @@
-;===- ./lib/Target/R600/MCTargetDesc/LLVMBuild.txt -------------*- Conf -*--===;
-;
-;                     The LLVM Compiler Infrastructure
-;
-; This file is distributed under the University of Illinois Open Source
-; License. See LICENSE.TXT for details.
-;
-;===------------------------------------------------------------------------===;
-;
-; This is an LLVMBuild description file for the components in this subdirectory.
-;
-; For more information on the LLVMBuild system, please see:
-;
-;   http://llvm.org/docs/LLVMBuild.html
-;
-;===------------------------------------------------------------------------===;
-
-[component_0]
-type = Library
-name = R600Desc
-parent = R600
-required_libraries = MC R600AsmPrinter R600Info Support
-add_to_library_groups = R600
diff --git a/lib/Target/R600/MCTargetDesc/Makefile b/lib/Target/R600/MCTargetDesc/Makefile
deleted file mode 100644
index 8894a7607f4..00000000000
--- a/lib/Target/R600/MCTargetDesc/Makefile
+++ /dev/null
@@ -1,16 +0,0 @@
-##===- lib/Target/AMDGPU/TargetDesc/Makefile ----------------*- Makefile -*-===##
-#
-#                     The LLVM Compiler Infrastructure
-#
-# This file is distributed under the University of Illinois Open Source
-# License. See LICENSE.TXT for details.
-#
-##===----------------------------------------------------------------------===##
-
-LEVEL = ../../../..
-LIBRARYNAME = LLVMR600Desc
-
-# Hack: we need to include 'main' target directory to grab private headers
-CPP.Flags += -I$(PROJ_OBJ_DIR)/.. -I$(PROJ_SRC_DIR)/..
-
-include $(LEVEL)/Makefile.common
diff --git a/lib/Target/R600/Makefile b/lib/Target/R600/Makefile
deleted file mode 100644
index 64a7c8c045c..00000000000
--- a/lib/Target/R600/Makefile
+++ /dev/null
@@ -1,23 +0,0 @@
-##===- lib/Target/R600/Makefile ---------------------------*- Makefile -*-===##
-#
-#                     The LLVM Compiler Infrastructure
-#
-# This file is distributed under the University of Illinois Open Source
-# License. See LICENSE.TXT for details.
-#
-##===----------------------------------------------------------------------===##
-
-LEVEL = ../../..
-LIBRARYNAME = LLVMR600CodeGen
-TARGET = AMDGPU
-
-# Make sure that tblgen is run, first thing.
-BUILT_SOURCES = AMDGPUGenRegisterInfo.inc AMDGPUGenInstrInfo.inc \
-		AMDGPUGenDAGISel.inc  AMDGPUGenSubtargetInfo.inc \
-		AMDGPUGenMCCodeEmitter.inc AMDGPUGenCallingConv.inc \
-		AMDGPUGenIntrinsics.inc AMDGPUGenDFAPacketizer.inc \
-		AMDGPUGenAsmWriter.inc AMDGPUGenAsmMatcher.inc
-
-DIRS = AsmParser InstPrinter TargetInfo MCTargetDesc
-
-include $(LEVEL)/Makefile.common
diff --git a/lib/Target/R600/SIPrepareScratchRegs.cpp b/lib/Target/R600/SIPrepareScratchRegs.cpp
deleted file mode 100644
index 0a57a5bc201..00000000000
--- a/lib/Target/R600/SIPrepareScratchRegs.cpp
+++ /dev/null
@@ -1,208 +0,0 @@
-//===-- SIPrepareScratchRegs.cpp - Use predicates for control flow --------===//
-//
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
-//
-//===----------------------------------------------------------------------===//
-//
-/// \file
-///
-/// This pass loads scratch pointer and scratch offset into a register or a
-/// frame index which can be used anywhere in the program.  These values will
-/// be used for spilling VGPRs.
-///
-//===----------------------------------------------------------------------===//
-
-#include "AMDGPU.h"
-#include "AMDGPUSubtarget.h"
-#include "SIDefines.h"
-#include "SIInstrInfo.h"
-#include "SIMachineFunctionInfo.h"
-#include "llvm/CodeGen/MachineFrameInfo.h"
-#include "llvm/CodeGen/MachineFunction.h"
-#include "llvm/CodeGen/MachineFunctionPass.h"
-#include "llvm/CodeGen/MachineInstrBuilder.h"
-#include "llvm/CodeGen/MachineRegisterInfo.h"
-#include "llvm/CodeGen/RegisterScavenging.h"
-#include "llvm/IR/Function.h"
-#include "llvm/IR/LLVMContext.h"
-
-using namespace llvm;
-
-namespace {
-
-class SIPrepareScratchRegs : public MachineFunctionPass {
-
-private:
-  static char ID;
-
-public:
-  SIPrepareScratchRegs() : MachineFunctionPass(ID) { }
-
-  bool runOnMachineFunction(MachineFunction &MF) override;
-
-  const char *getPassName() const override {
-    return "SI prepare scratch registers";
-  }
-
-};
-
-} // End anonymous namespace
-
-char SIPrepareScratchRegs::ID = 0;
-
-FunctionPass *llvm::createSIPrepareScratchRegs() {
-  return new SIPrepareScratchRegs();
-}
-
-bool SIPrepareScratchRegs::runOnMachineFunction(MachineFunction &MF) {
-  SIMachineFunctionInfo *MFI = MF.getInfo<SIMachineFunctionInfo>();
-  const SIInstrInfo *TII =
-      static_cast<const SIInstrInfo *>(MF.getSubtarget().getInstrInfo());
-  const SIRegisterInfo *TRI = &TII->getRegisterInfo();
-  MachineRegisterInfo &MRI = MF.getRegInfo();
-  MachineFrameInfo *FrameInfo = MF.getFrameInfo();
-  MachineBasicBlock *Entry = MF.begin();
-  MachineBasicBlock::iterator I = Entry->begin();
-  DebugLoc DL = I->getDebugLoc();
-
-  // FIXME: If we don't have enough VGPRs for SGPR spilling we will need to
-  // run this pass.
-  if (!MFI->hasSpilledVGPRs())
-    return false;
-
-  unsigned ScratchPtrPreloadReg =
-      TRI->getPreloadedValue(MF, SIRegisterInfo::SCRATCH_PTR);
-  unsigned ScratchOffsetPreloadReg =
-      TRI->getPreloadedValue(MF, SIRegisterInfo::SCRATCH_WAVE_OFFSET);
-
-  if (!Entry->isLiveIn(ScratchPtrPreloadReg))
-    Entry->addLiveIn(ScratchPtrPreloadReg);
-
-  if (!Entry->isLiveIn(ScratchOffsetPreloadReg))
-    Entry->addLiveIn(ScratchOffsetPreloadReg);
-
-  // Load the scratch offset.
-  unsigned ScratchOffsetReg =
-      TRI->findUnusedRegister(MRI, &AMDGPU::SGPR_32RegClass);
-  int ScratchOffsetFI = -1;
-
-  if (ScratchOffsetReg != AMDGPU::NoRegister) {
-    // Found an SGPR to use
-    MRI.setPhysRegUsed(ScratchOffsetReg);
-    BuildMI(*Entry, I, DL, TII->get(AMDGPU::S_MOV_B32), ScratchOffsetReg)
-            .addReg(ScratchOffsetPreloadReg);
-  } else {
-    // No SGPR is available, we must spill.
-    ScratchOffsetFI = FrameInfo->CreateSpillStackObject(4,4);
-    BuildMI(*Entry, I, DL, TII->get(AMDGPU::SI_SPILL_S32_SAVE))
-            .addReg(ScratchOffsetPreloadReg)
-            .addFrameIndex(ScratchOffsetFI)
-            .addReg(AMDGPU::SGPR0_SGPR1_SGPR2_SGPR3, RegState::Undef)
-            .addReg(AMDGPU::SGPR0, RegState::Undef);
-  }
-
-
-  // Now that we have the scratch pointer and offset values, we need to
-  // add them to all the SI_SPILL_V* instructions.
-
-  RegScavenger RS;
-  unsigned ScratchRsrcFI = FrameInfo->CreateSpillStackObject(16, 4);
-  RS.addScavengingFrameIndex(ScratchRsrcFI);
-
-  for (MachineFunction::iterator BI = MF.begin(), BE = MF.end();
-       BI != BE; ++BI) {
-
-    MachineBasicBlock &MBB = *BI;
-    // Add the scratch offset reg as a live-in so that the register scavenger
-    // doesn't re-use it.
-    if (!MBB.isLiveIn(ScratchOffsetReg) &&
-        ScratchOffsetReg != AMDGPU::NoRegister)
-      MBB.addLiveIn(ScratchOffsetReg);
-    RS.enterBasicBlock(&MBB);
-
-    for (MachineBasicBlock::iterator I = MBB.begin(), E = MBB.end();
-         I != E; ++I) {
-      MachineInstr &MI = *I;
-      RS.forward(I);
-      DebugLoc DL = MI.getDebugLoc();
-      switch(MI.getOpcode()) {
-        default: break;
-        case AMDGPU::SI_SPILL_V512_SAVE:
-        case AMDGPU::SI_SPILL_V256_SAVE:
-        case AMDGPU::SI_SPILL_V128_SAVE:
-        case AMDGPU::SI_SPILL_V96_SAVE:
-        case AMDGPU::SI_SPILL_V64_SAVE:
-        case AMDGPU::SI_SPILL_V32_SAVE:
-        case AMDGPU::SI_SPILL_V32_RESTORE:
-        case AMDGPU::SI_SPILL_V64_RESTORE:
-        case AMDGPU::SI_SPILL_V128_RESTORE:
-        case AMDGPU::SI_SPILL_V256_RESTORE:
-        case AMDGPU::SI_SPILL_V512_RESTORE:
-
-          // Scratch resource
-          unsigned ScratchRsrcReg =
-              RS.scavengeRegister(&AMDGPU::SReg_128RegClass, 0);
-
-          uint64_t Rsrc = AMDGPU::RSRC_DATA_FORMAT | AMDGPU::RSRC_TID_ENABLE |
-                          0xffffffff; // Size
-
-          unsigned Rsrc0 = TRI->getSubReg(ScratchRsrcReg, AMDGPU::sub0);
-          unsigned Rsrc1 = TRI->getSubReg(ScratchRsrcReg, AMDGPU::sub1);
-          unsigned Rsrc2 = TRI->getSubReg(ScratchRsrcReg, AMDGPU::sub2);
-          unsigned Rsrc3 = TRI->getSubReg(ScratchRsrcReg, AMDGPU::sub3);
-
-          BuildMI(MBB, I, DL, TII->get(AMDGPU::S_MOV_B32), Rsrc0)
-                  .addExternalSymbol("SCRATCH_RSRC_DWORD0")
-                  .addReg(ScratchRsrcReg, RegState::ImplicitDefine);
-
-          BuildMI(MBB, I, DL, TII->get(AMDGPU::S_MOV_B32), Rsrc1)
-                  .addExternalSymbol("SCRATCH_RSRC_DWORD1")
-                  .addReg(ScratchRsrcReg, RegState::ImplicitDefine);
-
-          BuildMI(MBB, I, DL, TII->get(AMDGPU::S_MOV_B32), Rsrc2)
-                  .addImm(Rsrc & 0xffffffff)
-                  .addReg(ScratchRsrcReg, RegState::ImplicitDefine);
-
-          BuildMI(MBB, I, DL, TII->get(AMDGPU::S_MOV_B32), Rsrc3)
-                  .addImm(Rsrc >> 32)
-                  .addReg(ScratchRsrcReg, RegState::ImplicitDefine);
-
-          // Scratch Offset
-          if (ScratchOffsetReg == AMDGPU::NoRegister) {
-            ScratchOffsetReg = RS.scavengeRegister(&AMDGPU::SGPR_32RegClass, 0);
-            BuildMI(MBB, I, DL, TII->get(AMDGPU::SI_SPILL_S32_RESTORE),
-                    ScratchOffsetReg)
-                    .addFrameIndex(ScratchOffsetFI)
-                    .addReg(AMDGPU::SGPR0_SGPR1_SGPR2_SGPR3, RegState::Undef)
-                    .addReg(AMDGPU::SGPR0, RegState::Undef);
-          } else if (!MBB.isLiveIn(ScratchOffsetReg)) {
-            MBB.addLiveIn(ScratchOffsetReg);
-          }
-
-          if (ScratchRsrcReg == AMDGPU::NoRegister ||
-              ScratchOffsetReg == AMDGPU::NoRegister) {
-            LLVMContext &Ctx = MF.getFunction()->getContext();
-            Ctx.emitError("ran out of SGPRs for spilling VGPRs");
-            ScratchRsrcReg = AMDGPU::SGPR0;
-            ScratchOffsetReg = AMDGPU::SGPR0;
-          }
-          MI.getOperand(2).setReg(ScratchRsrcReg);
-          MI.getOperand(2).setIsKill(true);
-          MI.getOperand(2).setIsUndef(false);
-          MI.getOperand(3).setReg(ScratchOffsetReg);
-          MI.getOperand(3).setIsUndef(false);
-          MI.getOperand(3).setIsKill(false);
-          MI.addOperand(MachineOperand::CreateReg(Rsrc0, false, true, true));
-          MI.addOperand(MachineOperand::CreateReg(Rsrc1, false, true, true));
-          MI.addOperand(MachineOperand::CreateReg(Rsrc2, false, true, true));
-          MI.addOperand(MachineOperand::CreateReg(Rsrc3, false, true, true));
-
-          break;
-      }
-    }
-  }
-  return true;
-}
diff --git a/lib/Target/R600/TargetInfo/CMakeLists.txt b/lib/Target/R600/TargetInfo/CMakeLists.txt
deleted file mode 100644
index c3bd26c7a89..00000000000
--- a/lib/Target/R600/TargetInfo/CMakeLists.txt
+++ /dev/null
@@ -1,3 +0,0 @@
-add_llvm_library(LLVMR600Info
-  AMDGPUTargetInfo.cpp
-  )
diff --git a/lib/Target/R600/TargetInfo/LLVMBuild.txt b/lib/Target/R600/TargetInfo/LLVMBuild.txt
deleted file mode 100644
index c3d3cf51cc8..00000000000
--- a/lib/Target/R600/TargetInfo/LLVMBuild.txt
+++ /dev/null
@@ -1,23 +0,0 @@
-;===- ./lib/Target/R600/TargetInfo/LLVMBuild.txt --------------*- Conf -*--===;
-;
-;                     The LLVM Compiler Infrastructure
-;
-; This file is distributed under the University of Illinois Open Source
-; License. See LICENSE.TXT for details.
-;
-;===------------------------------------------------------------------------===;
-;
-; This is an LLVMBuild description file for the components in this subdirectory.
-;
-; For more information on the LLVMBuild system, please see:
-;
-;   http://llvm.org/docs/LLVMBuild.html
-;
-;===------------------------------------------------------------------------===;
-
-[component_0]
-type = Library
-name = R600Info
-parent = R600
-required_libraries = Support
-add_to_library_groups = R600
diff --git a/lib/Target/R600/TargetInfo/Makefile b/lib/Target/R600/TargetInfo/Makefile
deleted file mode 100644
index b8ac4e78230..00000000000
--- a/lib/Target/R600/TargetInfo/Makefile
+++ /dev/null
@@ -1,15 +0,0 @@
-##===- lib/Target/AMDGPU/TargetInfo/Makefile ----------------*- Makefile -*-===##
-#
-#                     The LLVM Compiler Infrastructure
-#
-# This file is distributed under the University of Illinois Open Source
-# License. See LICENSE.TXT for details.
-#
-##===----------------------------------------------------------------------===##
-LEVEL = ../../../..
-LIBRARYNAME = LLVMR600Info
-
-# Hack: we need to include 'main' target directory to grab private headers
-CPPFLAGS = -I$(PROJ_OBJ_DIR)/.. -I$(PROJ_SRC_DIR)/..
-
-include $(LEVEL)/Makefile.common
diff --git a/lib/Target/R600/VIInstructions.td b/lib/Target/R600/VIInstructions.td
deleted file mode 100644
index 4a6e933783b..00000000000
--- a/lib/Target/R600/VIInstructions.td
+++ /dev/null
@@ -1,25 +0,0 @@
-//===-- VIInstructions.td - VI Instruction Defintions ---------------------===//
-//
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
-//
-//===----------------------------------------------------------------------===//
-// Instruction definitions for VI and newer.
-//===----------------------------------------------------------------------===//
-
-
-//===----------------------------------------------------------------------===//
-// SMEM Patterns
-//===----------------------------------------------------------------------===//
-
-let Predicates = [isVI] in {
-
-// 1. Offset as 20bit DWORD immediate
-def : Pat <
-  (SIload_constant v4i32:$sbase, IMM20bit:$offset),
-  (S_BUFFER_LOAD_DWORD_IMM $sbase, (as_i32imm $offset))
->;
-
-} // End Predicates = [isVI]
diff --git a/lib/Target/Sparc/AsmParser/SparcAsmParser.cpp b/lib/Target/Sparc/AsmParser/SparcAsmParser.cpp
index 551189c13c7..fc56bf9ad8e 100644
--- a/lib/Target/Sparc/AsmParser/SparcAsmParser.cpp
+++ b/lib/Target/Sparc/AsmParser/SparcAsmParser.cpp
@@ -76,12 +76,18 @@ class SparcAsmParser : public MCTargetAsmParser {
   bool matchSparcAsmModifiers(const MCExpr *&EVal, SMLoc &EndLoc);
   bool parseDirectiveWord(unsigned Size, SMLoc L);
 
-  bool is64Bit() const { return STI.getTargetTriple().startswith("sparcv9"); }
+  bool is64Bit() const {
+    return STI.getTargetTriple().getArchName().startswith("sparcv9");
+  }
+
+  void expandSET(MCInst &Inst, SMLoc IDLoc,
+                 SmallVectorImpl<MCInst> &Instructions);
+
 public:
   SparcAsmParser(MCSubtargetInfo &sti, MCAsmParser &parser,
                 const MCInstrInfo &MII,
                 const MCTargetOptions &Options)
-      : MCTargetAsmParser(), STI(sti), Parser(parser) {
+      : MCTargetAsmParser(Options), STI(sti), Parser(parser) {
     // Initialize the set of available features.
     setAvailableFeatures(ComputeAvailableFeatures(STI.getFeatureBits()));
   }
@@ -124,6 +130,15 @@ class SparcAsmParser : public MCTargetAsmParser {
     Sparc::Q8,  Sparc::Q9,  Sparc::Q10, Sparc::Q11,
     Sparc::Q12, Sparc::Q13, Sparc::Q14, Sparc::Q15 };
 
+  static unsigned ASRRegs[32] = {
+    SP::Y,     SP::ASR1,  SP::ASR2,  SP::ASR3,
+    SP::ASR4,  SP::ASR5,  SP::ASR6, SP::ASR7,
+    SP::ASR8,  SP::ASR9,  SP::ASR10, SP::ASR11,
+    SP::ASR12, SP::ASR13, SP::ASR14, SP::ASR15,
+    SP::ASR16, SP::ASR17, SP::ASR18, SP::ASR19,
+    SP::ASR20, SP::ASR21, SP::ASR22, SP::ASR23,
+    SP::ASR24, SP::ASR25, SP::ASR26, SP::ASR27,
+    SP::ASR28, SP::ASR29, SP::ASR30, SP::ASR31};
 
 /// SparcOperand - Instances of this class represent a parsed Sparc machine
 /// instruction.
@@ -135,9 +150,9 @@ class SparcOperand : public MCParsedAsmOperand {
     rk_FloatReg,
     rk_DoubleReg,
     rk_QuadReg,
-    rk_CCReg,
-    rk_Y
+    rk_Special,
   };
+
 private:
   enum KindTy {
     k_Token,
@@ -250,7 +265,7 @@ class SparcOperand : public MCParsedAsmOperand {
 
   void addRegOperands(MCInst &Inst, unsigned N) const {
     assert(N == 1 && "Invalid number of operands!");
-    Inst.addOperand(MCOperand::CreateReg(getReg()));
+    Inst.addOperand(MCOperand::createReg(getReg()));
   }
 
   void addImmOperands(MCInst &Inst, unsigned N) const {
@@ -262,26 +277,26 @@ class SparcOperand : public MCParsedAsmOperand {
   void addExpr(MCInst &Inst, const MCExpr *Expr) const{
     // Add as immediate when possible.  Null MCExpr = 0.
     if (!Expr)
-      Inst.addOperand(MCOperand::CreateImm(0));
+      Inst.addOperand(MCOperand::createImm(0));
     else if (const MCConstantExpr *CE = dyn_cast<MCConstantExpr>(Expr))
-      Inst.addOperand(MCOperand::CreateImm(CE->getValue()));
+      Inst.addOperand(MCOperand::createImm(CE->getValue()));
     else
-      Inst.addOperand(MCOperand::CreateExpr(Expr));
+      Inst.addOperand(MCOperand::createExpr(Expr));
   }
 
   void addMEMrrOperands(MCInst &Inst, unsigned N) const {
     assert(N == 2 && "Invalid number of operands!");
 
-    Inst.addOperand(MCOperand::CreateReg(getMemBase()));
+    Inst.addOperand(MCOperand::createReg(getMemBase()));
 
     assert(getMemOffsetReg() != 0 && "Invalid offset");
-    Inst.addOperand(MCOperand::CreateReg(getMemOffsetReg()));
+    Inst.addOperand(MCOperand::createReg(getMemOffsetReg()));
   }
 
   void addMEMriOperands(MCInst &Inst, unsigned N) const {
     assert(N == 2 && "Invalid number of operands!");
 
-    Inst.addOperand(MCOperand::CreateReg(getMemBase()));
+    Inst.addOperand(MCOperand::createReg(getMemBase()));
 
     const MCExpr *Expr = getMemOff();
     addExpr(Inst, Expr);
@@ -360,11 +375,11 @@ class SparcOperand : public MCParsedAsmOperand {
   }
 
   static std::unique_ptr<SparcOperand>
-  CreateMEMri(unsigned Base, const MCExpr *Off, SMLoc S, SMLoc E) {
-    auto Op = make_unique<SparcOperand>(k_MemoryImm);
+  CreateMEMr(unsigned Base, SMLoc S, SMLoc E) {
+    auto Op = make_unique<SparcOperand>(k_MemoryReg);
     Op->Mem.Base = Base;
-    Op->Mem.OffsetReg = 0;
-    Op->Mem.Off = Off;
+    Op->Mem.OffsetReg = Sparc::G0;  // always 0
+    Op->Mem.Off = nullptr;
     Op->StartLoc = S;
     Op->EndLoc = E;
     return Op;
@@ -383,6 +398,49 @@ class SparcOperand : public MCParsedAsmOperand {
 
 } // end namespace
 
+void SparcAsmParser::expandSET(MCInst &Inst, SMLoc IDLoc,
+                               SmallVectorImpl<MCInst> &Instructions) {
+  MCOperand MCRegOp = Inst.getOperand(0);
+  MCOperand MCValOp = Inst.getOperand(1);
+  assert(MCRegOp.isReg());
+  assert(MCValOp.isImm() || MCValOp.isExpr());
+
+  // the imm operand can be either an expression or an immediate.
+  bool IsImm = Inst.getOperand(1).isImm();
+  uint64_t ImmValue = IsImm ? MCValOp.getImm() : 0;
+  const MCExpr *ValExpr;
+  if (IsImm)
+    ValExpr = MCConstantExpr::create(ImmValue, getContext());
+  else
+    ValExpr = MCValOp.getExpr();
+
+  MCOperand PrevReg = MCOperand::createReg(Sparc::G0);
+
+  if (!IsImm || (ImmValue & ~0x1fff)) {
+    MCInst TmpInst;
+    const MCExpr *Expr =
+        SparcMCExpr::create(SparcMCExpr::VK_Sparc_HI, ValExpr, getContext());
+    TmpInst.setLoc(IDLoc);
+    TmpInst.setOpcode(SP::SETHIi);
+    TmpInst.addOperand(MCRegOp);
+    TmpInst.addOperand(MCOperand::createExpr(Expr));
+    Instructions.push_back(TmpInst);
+    PrevReg = MCRegOp;
+  }
+
+  if (!IsImm || ((ImmValue & 0x1fff) != 0 || ImmValue == 0)) {
+    MCInst TmpInst;
+    const MCExpr *Expr =
+        SparcMCExpr::create(SparcMCExpr::VK_Sparc_LO, ValExpr, getContext());
+    TmpInst.setLoc(IDLoc);
+    TmpInst.setOpcode(SP::ORri);
+    TmpInst.addOperand(MCRegOp);
+    TmpInst.addOperand(PrevReg);
+    TmpInst.addOperand(MCOperand::createExpr(Expr));
+    Instructions.push_back(TmpInst);
+  }
+}
+
 bool SparcAsmParser::MatchAndEmitInstruction(SMLoc IDLoc, unsigned &Opcode,
                                              OperandVector &Operands,
                                              MCStreamer &Out,
@@ -394,8 +452,19 @@ bool SparcAsmParser::MatchAndEmitInstruction(SMLoc IDLoc, unsigned &Opcode,
                                               MatchingInlineAsm);
   switch (MatchResult) {
   case Match_Success: {
-    Inst.setLoc(IDLoc);
-    Out.EmitInstruction(Inst, STI);
+    switch (Inst.getOpcode()) {
+    default:
+      Inst.setLoc(IDLoc);
+      Instructions.push_back(Inst);
+      break;
+    case SP::SET:
+      expandSET(Inst, IDLoc, Instructions);
+      break;
+    }
+
+    for (const MCInst &I : Instructions) {
+      Out.EmitInstruction(I, STI);
+    }
     return false;
   }
 
@@ -556,7 +625,7 @@ SparcAsmParser::parseMEMOperand(OperandVector &Operands) {
   case AsmToken::Comma:
   case AsmToken::RBrac:
   case AsmToken::EndOfStatement:
-    Operands.push_back(SparcOperand::CreateMEMri(BaseReg, nullptr, S, E));
+    Operands.push_back(SparcOperand::CreateMEMr(BaseReg, S, E));
     return MatchOperand_Success;
 
   case AsmToken:: Plus:
@@ -622,6 +691,15 @@ SparcAsmParser::parseOperand(OperandVector &Operands, StringRef Mnemonic) {
     Operands.push_back(SparcOperand::CreateToken("]",
                                                  Parser.getTok().getLoc()));
     Parser.Lex(); // Eat the ]
+
+    // Parse an optional address-space identifier after the address.
+    if (getLexer().is(AsmToken::Integer)) {
+      std::unique_ptr<SparcOperand> Op;
+      ResTy = parseSparcAsmOperand(Op, false);
+      if (ResTy != MatchOperand_Success || !Op)
+        return MatchOperand_ParseFail;
+      Operands.push_back(std::move(Op));
+    }
     return MatchOperand_Success;
   }
 
@@ -661,10 +739,15 @@ SparcAsmParser::parseSparcAsmOperand(std::unique_ptr<SparcOperand> &Op,
       default:
         Op = SparcOperand::CreateReg(RegNo, RegKind, S, E);
         break;
-      case Sparc::Y:
-        Op = SparcOperand::CreateToken("%y", S);
+      case Sparc::PSR:
+        Op = SparcOperand::CreateToken("%psr", S);
+        break;
+      case Sparc::WIM:
+        Op = SparcOperand::CreateToken("%wim", S);
+        break;
+      case Sparc::TBR:
+        Op = SparcOperand::CreateToken("%tbr", S);
         break;
-
       case Sparc::ICC:
         if (name == "xcc")
           Op = SparcOperand::CreateToken("%xcc", S);
@@ -682,6 +765,7 @@ SparcAsmParser::parseSparcAsmOperand(std::unique_ptr<SparcOperand> &Op,
 
   case AsmToken::Minus:
   case AsmToken::Integer:
+  case AsmToken::LParen:
     if (!getParser().parseExpression(EVal, E))
       Op = SparcOperand::CreateImm(EVal, S, E);
     break;
@@ -690,13 +774,13 @@ SparcAsmParser::parseSparcAsmOperand(std::unique_ptr<SparcOperand> &Op,
     StringRef Identifier;
     if (!getParser().parseIdentifier(Identifier)) {
       E = SMLoc::getFromPointer(Parser.getTok().getLoc().getPointer() - 1);
-      MCSymbol *Sym = getContext().GetOrCreateSymbol(Identifier);
+      MCSymbol *Sym = getContext().getOrCreateSymbol(Identifier);
 
-      const MCExpr *Res = MCSymbolRefExpr::Create(Sym, MCSymbolRefExpr::VK_None,
+      const MCExpr *Res = MCSymbolRefExpr::create(Sym, MCSymbolRefExpr::VK_None,
                                                   getContext());
       if (isCall &&
           getContext().getObjectFileInfo()->getRelocM() == Reloc::PIC_)
-        Res = SparcMCExpr::Create(SparcMCExpr::VK_Sparc_WPLT30, Res,
+        Res = SparcMCExpr::create(SparcMCExpr::VK_Sparc_WPLT30, Res,
                                   getContext());
       Op = SparcOperand::CreateImm(Res, S, E);
     }
@@ -752,20 +836,46 @@ bool SparcAsmParser::matchRegisterName(const AsmToken &Tok,
 
     if (name.equals("y")) {
       RegNo = Sparc::Y;
-      RegKind = SparcOperand::rk_Y;
+      RegKind = SparcOperand::rk_Special;
+      return true;
+    }
+
+    if (name.substr(0, 3).equals_lower("asr")
+        && !name.substr(3).getAsInteger(10, intVal)
+        && intVal > 0 && intVal < 32) {
+      RegNo = ASRRegs[intVal];
+      RegKind = SparcOperand::rk_Special;
       return true;
     }
 
     if (name.equals("icc")) {
       RegNo = Sparc::ICC;
-      RegKind = SparcOperand::rk_CCReg;
+      RegKind = SparcOperand::rk_Special;
+      return true;
+    }
+
+    if (name.equals("psr")) {
+      RegNo = Sparc::PSR;
+      RegKind = SparcOperand::rk_Special;
+      return true;
+    }
+
+    if (name.equals("wim")) {
+      RegNo = Sparc::WIM;
+      RegKind = SparcOperand::rk_Special;
+      return true;
+    }
+
+    if (name.equals("tbr")) {
+      RegNo = Sparc::TBR;
+      RegKind = SparcOperand::rk_Special;
       return true;
     }
 
     if (name.equals("xcc")) {
       // FIXME:: check 64bit.
       RegNo = Sparc::ICC;
-      RegKind = SparcOperand::rk_CCReg;
+      RegKind = SparcOperand::rk_Special;
       return true;
     }
 
@@ -775,7 +885,7 @@ bool SparcAsmParser::matchRegisterName(const AsmToken &Tok,
         && intVal < 4) {
       // FIXME: check 64bit and  handle %fcc1 - %fcc3
       RegNo = Sparc::FCC0 + intVal;
-      RegKind = SparcOperand::rk_CCReg;
+      RegKind = SparcOperand::rk_Special;
       return true;
     }
 
@@ -837,6 +947,8 @@ bool SparcAsmParser::matchRegisterName(const AsmToken &Tok,
   return false;
 }
 
+// Determine if an expression contains a reference to the symbol
+// "_GLOBAL_OFFSET_TABLE_".
 static bool hasGOTReference(const MCExpr *Expr) {
   switch (Expr->getKind()) {
   case MCExpr::Target:
@@ -888,6 +1000,13 @@ bool SparcAsmParser::matchSparcAsmModifiers(const MCExpr *&EVal,
 
   bool isPIC = getContext().getObjectFileInfo()->getRelocM() == Reloc::PIC_;
 
+  // Ugly: if a sparc assembly expression says "%hi(...)" but the
+  // expression within contains _GLOBAL_OFFSET_TABLE_, it REALLY means
+  // %pc22. Same with %lo -> %pc10. Worse, if it doesn't contain that,
+  // the meaning depends on whether the assembler was invoked with
+  // -KPIC or not: if so, it really means %got22/%got10; if not, it
+  // actually means what it said! Sigh, historical mistakes...
+
   switch(VK) {
   default: break;
   case SparcMCExpr::VK_Sparc_LO:
@@ -902,14 +1021,14 @@ bool SparcAsmParser::matchSparcAsmModifiers(const MCExpr *&EVal,
     break;
   }
 
-  EVal = SparcMCExpr::Create(VK, subExpr, getContext());
+  EVal = SparcMCExpr::create(VK, subExpr, getContext());
   return true;
 }
 
-
 extern "C" void LLVMInitializeSparcAsmParser() {
   RegisterMCAsmParser<SparcAsmParser> A(TheSparcTarget);
   RegisterMCAsmParser<SparcAsmParser> B(TheSparcV9Target);
+  RegisterMCAsmParser<SparcAsmParser> C(TheSparcelTarget);
 }
 
 #define GET_REGISTER_MATCHER
diff --git a/lib/Target/Sparc/CMakeLists.txt b/lib/Target/Sparc/CMakeLists.txt
index c486411f9a1..5b7bfdd2802 100644
--- a/lib/Target/Sparc/CMakeLists.txt
+++ b/lib/Target/Sparc/CMakeLists.txt
@@ -22,7 +22,6 @@ add_llvm_target(SparcCodeGen
   SparcRegisterInfo.cpp
   SparcSubtarget.cpp
   SparcTargetMachine.cpp
-  SparcSelectionDAGInfo.cpp
   SparcMCInstLower.cpp
   SparcTargetObjectFile.cpp
   )
diff --git a/lib/Target/Sparc/Disassembler/SparcDisassembler.cpp b/lib/Target/Sparc/Disassembler/SparcDisassembler.cpp
index 8bc4ca98161..3e56b9e9b88 100644
--- a/lib/Target/Sparc/Disassembler/SparcDisassembler.cpp
+++ b/lib/Target/Sparc/Disassembler/SparcDisassembler.cpp
@@ -16,6 +16,9 @@
 #include "SparcSubtarget.h"
 #include "llvm/MC/MCDisassembler.h"
 #include "llvm/MC/MCFixedLenDisassembler.h"
+#include "llvm/MC/MCInst.h"
+#include "llvm/MC/MCContext.h"
+#include "llvm/MC/MCAsmInfo.h"
 #include "llvm/Support/TargetRegistry.h"
 
 using namespace llvm;
@@ -38,17 +41,15 @@ class SparcDisassembler : public MCDisassembler {
                               raw_ostream &VStream,
                               raw_ostream &CStream) const override;
 };
-
 }
 
 namespace llvm {
-  extern Target TheSparcTarget, TheSparcV9Target;
+extern Target TheSparcTarget, TheSparcV9Target, TheSparcelTarget;
 }
 
-static MCDisassembler *createSparcDisassembler(
-                       const Target &T,
-                       const MCSubtargetInfo &STI,
-                       MCContext &Ctx) {
+static MCDisassembler *createSparcDisassembler(const Target &T,
+                                               const MCSubtargetInfo &STI,
+                                               MCContext &Ctx) {
   return new SparcDisassembler(STI, Ctx);
 }
 
@@ -59,10 +60,10 @@ extern "C" void LLVMInitializeSparcDisassembler() {
                                          createSparcDisassembler);
   TargetRegistry::RegisterMCDisassembler(TheSparcV9Target,
                                          createSparcDisassembler);
+  TargetRegistry::RegisterMCDisassembler(TheSparcelTarget,
+                                         createSparcDisassembler);
 }
 
-
-
 static const unsigned IntRegDecoderTable[] = {
   SP::G0,  SP::G1,  SP::G2,  SP::G3,
   SP::G4,  SP::G5,  SP::G6,  SP::G7,
@@ -106,6 +107,16 @@ static const unsigned QFPRegDecoderTable[] = {
 static const unsigned FCCRegDecoderTable[] = {
   SP::FCC0, SP::FCC1, SP::FCC2, SP::FCC3 };
 
+static const unsigned ASRRegDecoderTable[] = {
+  SP::Y,     SP::ASR1,  SP::ASR2,  SP::ASR3,
+  SP::ASR4,  SP::ASR5,  SP::ASR6,  SP::ASR7,
+  SP::ASR8,  SP::ASR9,  SP::ASR10, SP::ASR11,
+  SP::ASR12, SP::ASR13, SP::ASR14, SP::ASR15,
+  SP::ASR16, SP::ASR17, SP::ASR18, SP::ASR19,
+  SP::ASR20, SP::ASR21, SP::ASR22, SP::ASR23,
+  SP::ASR24, SP::ASR25, SP::ASR26, SP::ASR27,
+  SP::ASR28, SP::ASR29, SP::ASR30, SP::ASR31};
+
 static DecodeStatus DecodeIntRegsRegisterClass(MCInst &Inst,
                                                unsigned RegNo,
                                                uint64_t Address,
@@ -113,7 +124,7 @@ static DecodeStatus DecodeIntRegsRegisterClass(MCInst &Inst,
   if (RegNo > 31)
     return MCDisassembler::Fail;
   unsigned Reg = IntRegDecoderTable[RegNo];
-  Inst.addOperand(MCOperand::CreateReg(Reg));
+  Inst.addOperand(MCOperand::createReg(Reg));
   return MCDisassembler::Success;
 }
 
@@ -124,7 +135,7 @@ static DecodeStatus DecodeI64RegsRegisterClass(MCInst &Inst,
   if (RegNo > 31)
     return MCDisassembler::Fail;
   unsigned Reg = IntRegDecoderTable[RegNo];
-  Inst.addOperand(MCOperand::CreateReg(Reg));
+  Inst.addOperand(MCOperand::createReg(Reg));
   return MCDisassembler::Success;
 }
 
@@ -136,7 +147,7 @@ static DecodeStatus DecodeFPRegsRegisterClass(MCInst &Inst,
   if (RegNo > 31)
     return MCDisassembler::Fail;
   unsigned Reg = FPRegDecoderTable[RegNo];
-  Inst.addOperand(MCOperand::CreateReg(Reg));
+  Inst.addOperand(MCOperand::createReg(Reg));
   return MCDisassembler::Success;
 }
 
@@ -148,7 +159,7 @@ static DecodeStatus DecodeDFPRegsRegisterClass(MCInst &Inst,
   if (RegNo > 31)
     return MCDisassembler::Fail;
   unsigned Reg = DFPRegDecoderTable[RegNo];
-  Inst.addOperand(MCOperand::CreateReg(Reg));
+  Inst.addOperand(MCOperand::createReg(Reg));
   return MCDisassembler::Success;
 }
 
@@ -163,7 +174,7 @@ static DecodeStatus DecodeQFPRegsRegisterClass(MCInst &Inst,
   unsigned Reg = QFPRegDecoderTable[RegNo];
   if (Reg == ~0U)
     return MCDisassembler::Fail;
-  Inst.addOperand(MCOperand::CreateReg(Reg));
+  Inst.addOperand(MCOperand::createReg(Reg));
   return MCDisassembler::Success;
 }
 
@@ -172,7 +183,16 @@ static DecodeStatus DecodeFCCRegsRegisterClass(MCInst &Inst, unsigned RegNo,
                                                const void *Decoder) {
   if (RegNo > 3)
     return MCDisassembler::Fail;
-  Inst.addOperand(MCOperand::CreateReg(FCCRegDecoderTable[RegNo]));
+  Inst.addOperand(MCOperand::createReg(FCCRegDecoderTable[RegNo]));
+  return MCDisassembler::Success;
+}
+
+static DecodeStatus DecodeASRRegsRegisterClass(MCInst &Inst, unsigned RegNo,
+                                               uint64_t Address,
+                                               const void *Decoder) {
+  if (RegNo > 31)
+    return MCDisassembler::Fail;
+  Inst.addOperand(MCOperand::createReg(ASRRegDecoderTable[RegNo]));
   return MCDisassembler::Success;
 }
 
@@ -208,16 +228,19 @@ static DecodeStatus DecodeSWAP(MCInst &Inst, unsigned insn, uint64_t Address,
 
 /// Read four bytes from the ArrayRef and return 32 bit word.
 static DecodeStatus readInstruction32(ArrayRef<uint8_t> Bytes, uint64_t Address,
-                                      uint64_t &Size, uint32_t &Insn) {
+                                      uint64_t &Size, uint32_t &Insn,
+                                      bool IsLittleEndian) {
   // We want to read exactly 4 Bytes of data.
   if (Bytes.size() < 4) {
     Size = 0;
     return MCDisassembler::Fail;
   }
 
-  // Encoded as a big-endian 32-bit word in the stream.
-  Insn =
-      (Bytes[3] << 0) | (Bytes[2] << 8) | (Bytes[1] << 16) | (Bytes[0] << 24);
+  Insn = IsLittleEndian
+             ? (Bytes[0] << 0) | (Bytes[1] << 8) | (Bytes[2] << 16) |
+                   (Bytes[3] << 24)
+             : (Bytes[3] << 0) | (Bytes[2] << 8) | (Bytes[1] << 16) |
+                   (Bytes[0] << 24);
 
   return MCDisassembler::Success;
 }
@@ -228,12 +251,12 @@ DecodeStatus SparcDisassembler::getInstruction(MCInst &Instr, uint64_t &Size,
                                                raw_ostream &VStream,
                                                raw_ostream &CStream) const {
   uint32_t Insn;
-
-  DecodeStatus Result = readInstruction32(Bytes, Address, Size, Insn);
+  bool isLittleEndian = getContext().getAsmInfo()->isLittleEndian();
+  DecodeStatus Result =
+      readInstruction32(Bytes, Address, Size, Insn, isLittleEndian);
   if (Result == MCDisassembler::Fail)
     return MCDisassembler::Fail;
 
-
   // Calling the auto-generated decoder function.
   Result =
       decodeInstruction(DecoderTableSparc32, Instr, Insn, Address, this, STI);
@@ -256,6 +279,8 @@ static DecodeStatus DecodeMem(MCInst &MI, unsigned insn, uint64_t Address,
   unsigned rd = fieldFromInstruction(insn, 25, 5);
   unsigned rs1 = fieldFromInstruction(insn, 14, 5);
   bool isImm = fieldFromInstruction(insn, 13, 1);
+  bool hasAsi = fieldFromInstruction(insn, 23, 1); // (in op3 field)
+  unsigned asi = fieldFromInstruction(insn, 5, 8);
   unsigned rs2 = 0;
   unsigned simm13 = 0;
   if (isImm)
@@ -277,13 +302,16 @@ static DecodeStatus DecodeMem(MCInst &MI, unsigned insn, uint64_t Address,
 
   // Decode imm|rs2.
   if (isImm)
-    MI.addOperand(MCOperand::CreateImm(simm13));
+    MI.addOperand(MCOperand::createImm(simm13));
   else {
     status = DecodeIntRegsRegisterClass(MI, rs2, Address, Decoder);
     if (status != MCDisassembler::Success)
       return status;
   }
 
+  if (hasAsi)
+    MI.addOperand(MCOperand::createImm(asi));
+
   if (!isLoad) {
     status = DecodeRD(MI, rd, Address, Decoder);
     if (status != MCDisassembler::Success)
@@ -355,14 +383,14 @@ static DecodeStatus DecodeCall(MCInst &MI, unsigned insn,
   tgt <<= 2;
   if (!tryAddingSymbolicOperand(tgt+Address, false, Address,
                                 0, 30, MI, Decoder))
-    MI.addOperand(MCOperand::CreateImm(tgt));
+    MI.addOperand(MCOperand::createImm(tgt));
   return MCDisassembler::Success;
 }
 
 static DecodeStatus DecodeSIMM13(MCInst &MI, unsigned insn,
                                  uint64_t Address, const void *Decoder) {
   unsigned tgt = SignExtend32<13>(fieldFromInstruction(insn, 0, 13));
-  MI.addOperand(MCOperand::CreateImm(tgt));
+  MI.addOperand(MCOperand::createImm(tgt));
   return MCDisassembler::Success;
 }
 
@@ -391,7 +419,7 @@ static DecodeStatus DecodeJMPL(MCInst &MI, unsigned insn, uint64_t Address,
 
   // Decode RS1 | SIMM13.
   if (isImm)
-    MI.addOperand(MCOperand::CreateImm(simm13));
+    MI.addOperand(MCOperand::createImm(simm13));
   else {
     status = DecodeIntRegsRegisterClass(MI, rs2, Address, Decoder);
     if (status != MCDisassembler::Success)
@@ -419,7 +447,7 @@ static DecodeStatus DecodeReturn(MCInst &MI, unsigned insn, uint64_t Address,
 
   // Decode RS2 | SIMM13.
   if (isImm)
-    MI.addOperand(MCOperand::CreateImm(simm13));
+    MI.addOperand(MCOperand::createImm(simm13));
   else {
     status = DecodeIntRegsRegisterClass(MI, rs2, Address, Decoder);
     if (status != MCDisassembler::Success)
@@ -434,6 +462,8 @@ static DecodeStatus DecodeSWAP(MCInst &MI, unsigned insn, uint64_t Address,
   unsigned rd = fieldFromInstruction(insn, 25, 5);
   unsigned rs1 = fieldFromInstruction(insn, 14, 5);
   unsigned isImm = fieldFromInstruction(insn, 13, 1);
+  bool hasAsi = fieldFromInstruction(insn, 23, 1); // (in op3 field)
+  unsigned asi = fieldFromInstruction(insn, 5, 8);
   unsigned rs2 = 0;
   unsigned simm13 = 0;
   if (isImm)
@@ -453,11 +483,15 @@ static DecodeStatus DecodeSWAP(MCInst &MI, unsigned insn, uint64_t Address,
 
   // Decode RS1 | SIMM13.
   if (isImm)
-    MI.addOperand(MCOperand::CreateImm(simm13));
+    MI.addOperand(MCOperand::createImm(simm13));
   else {
     status = DecodeIntRegsRegisterClass(MI, rs2, Address, Decoder);
     if (status != MCDisassembler::Success)
       return status;
   }
+
+  if (hasAsi)
+    MI.addOperand(MCOperand::createImm(asi));
+
   return MCDisassembler::Success;
 }
diff --git a/lib/Target/Sparc/InstPrinter/SparcInstPrinter.cpp b/lib/Target/Sparc/InstPrinter/SparcInstPrinter.cpp
index b6eebb07d08..5d714fe4da9 100644
--- a/lib/Target/Sparc/InstPrinter/SparcInstPrinter.cpp
+++ b/lib/Target/Sparc/InstPrinter/SparcInstPrinter.cpp
@@ -35,7 +35,7 @@ namespace Sparc {
 #include "SparcGenAsmWriter.inc"
 
 bool SparcInstPrinter::isV9(const MCSubtargetInfo &STI) const {
-  return (STI.getFeatureBits() & Sparc::FeatureV9) != 0;
+  return (STI.getFeatureBits()[Sparc::FeatureV9]) != 0;
 }
 
 void SparcInstPrinter::printRegName(raw_ostream &OS, unsigned RegNo) const
@@ -120,7 +120,7 @@ void SparcInstPrinter::printOperand(const MCInst *MI, int opNum,
   }
 
   assert(MO.isExpr() && "Unknown operand kind in printOperand");
-  MO.getExpr()->print(O);
+  MO.getExpr()->print(O, &MAI);
 }
 
 void SparcInstPrinter::printMemOperand(const MCInst *MI, int opNum,
diff --git a/lib/Target/Sparc/MCTargetDesc/SparcAsmBackend.cpp b/lib/Target/Sparc/MCTargetDesc/SparcAsmBackend.cpp
index 4abb6b8d25d..d1d7aaa07ea 100644
--- a/lib/Target/Sparc/MCTargetDesc/SparcAsmBackend.cpp
+++ b/lib/Target/Sparc/MCTargetDesc/SparcAsmBackend.cpp
@@ -98,16 +98,23 @@ static unsigned adjustFixupValue(unsigned Kind, uint64_t Value) {
 
 namespace {
   class SparcAsmBackend : public MCAsmBackend {
+  protected:
     const Target &TheTarget;
+    bool IsLittleEndian;
+    bool Is64Bit;
+
   public:
-    SparcAsmBackend(const Target &T) : MCAsmBackend(), TheTarget(T) {}
+    SparcAsmBackend(const Target &T)
+        : MCAsmBackend(), TheTarget(T),
+          IsLittleEndian(StringRef(TheTarget.getName()) == "sparcel"),
+          Is64Bit(StringRef(TheTarget.getName()) == "sparcv9") {}
 
     unsigned getNumFixupKinds() const override {
       return Sparc::NumTargetFixupKinds;
     }
 
     const MCFixupKindInfo &getFixupKindInfo(MCFixupKind Kind) const override {
-      const static MCFixupKindInfo Infos[Sparc::NumTargetFixupKinds] = {
+      const static MCFixupKindInfo InfosBE[Sparc::NumTargetFixupKinds] = {
         // name                    offset bits  flags
         { "fixup_sparc_call30",     2,     30,  MCFixupKindInfo::FKF_IsPCRel },
         { "fixup_sparc_br22",      10,     22,  MCFixupKindInfo::FKF_IsPCRel },
@@ -146,12 +153,54 @@ namespace {
         { "fixup_sparc_tls_le_lox10",   0,  0,  0 }
       };
 
+      const static MCFixupKindInfo InfosLE[Sparc::NumTargetFixupKinds] = {
+        // name                    offset bits  flags
+        { "fixup_sparc_call30",     0,     30,  MCFixupKindInfo::FKF_IsPCRel },
+        { "fixup_sparc_br22",       0,     22,  MCFixupKindInfo::FKF_IsPCRel },
+        { "fixup_sparc_br19",       0,     19,  MCFixupKindInfo::FKF_IsPCRel },
+        { "fixup_sparc_br16_2",    20,      2,  MCFixupKindInfo::FKF_IsPCRel },
+        { "fixup_sparc_br16_14",    0,     14,  MCFixupKindInfo::FKF_IsPCRel },
+        { "fixup_sparc_hi22",       0,     22,  0 },
+        { "fixup_sparc_lo10",       0,     10,  0 },
+        { "fixup_sparc_h44",        0,     22,  0 },
+        { "fixup_sparc_m44",        0,     10,  0 },
+        { "fixup_sparc_l44",        0,     12,  0 },
+        { "fixup_sparc_hh",         0,     22,  0 },
+        { "fixup_sparc_hm",         0,     10,  0 },
+        { "fixup_sparc_pc22",       0,     22,  MCFixupKindInfo::FKF_IsPCRel },
+        { "fixup_sparc_pc10",       0,     10,  MCFixupKindInfo::FKF_IsPCRel },
+        { "fixup_sparc_got22",      0,     22,  0 },
+        { "fixup_sparc_got10",      0,     10,  0 },
+        { "fixup_sparc_wplt30",      0,     30,  MCFixupKindInfo::FKF_IsPCRel },
+        { "fixup_sparc_tls_gd_hi22",    0, 22,  0 },
+        { "fixup_sparc_tls_gd_lo10",    0, 10,  0 },
+        { "fixup_sparc_tls_gd_add",     0,  0,  0 },
+        { "fixup_sparc_tls_gd_call",    0,  0,  0 },
+        { "fixup_sparc_tls_ldm_hi22",   0, 22,  0 },
+        { "fixup_sparc_tls_ldm_lo10",   0, 10,  0 },
+        { "fixup_sparc_tls_ldm_add",    0,  0,  0 },
+        { "fixup_sparc_tls_ldm_call",   0,  0,  0 },
+        { "fixup_sparc_tls_ldo_hix22",  0, 22,  0 },
+        { "fixup_sparc_tls_ldo_lox10",  0, 10,  0 },
+        { "fixup_sparc_tls_ldo_add",    0,  0,  0 },
+        { "fixup_sparc_tls_ie_hi22",    0, 22,  0 },
+        { "fixup_sparc_tls_ie_lo10",    0, 10,  0 },
+        { "fixup_sparc_tls_ie_ld",      0,  0,  0 },
+        { "fixup_sparc_tls_ie_ldx",     0,  0,  0 },
+        { "fixup_sparc_tls_ie_add",     0,  0,  0 },
+        { "fixup_sparc_tls_le_hix22",   0,  0,  0 },
+        { "fixup_sparc_tls_le_lox10",   0,  0,  0 }
+      };
+
       if (Kind < FirstTargetFixupKind)
         return MCAsmBackend::getFixupKindInfo(Kind);
 
       assert(unsigned(Kind - FirstTargetFixupKind) < getNumFixupKinds() &&
              "Invalid kind!");
-      return Infos[Kind - FirstTargetFixupKind];
+      if (IsLittleEndian)
+        return InfosLE[Kind - FirstTargetFixupKind];
+
+      return InfosBE[Kind - FirstTargetFixupKind];
     }
 
     void processFixupValue(const MCAssembler &Asm, const MCAsmLayout &Layout,
@@ -211,15 +260,10 @@ namespace {
 
       uint64_t NumNops = Count / 4;
       for (uint64_t i = 0; i != NumNops; ++i)
-        OW->Write32(0x01000000);
+        OW->write32(0x01000000);
 
       return true;
     }
-
-    bool is64Bit() const {
-      StringRef name = TheTarget.getName();
-      return name == "sparcv9";
-    }
   };
 
   class ELFSparcAsmBackend : public SparcAsmBackend {
@@ -239,23 +283,22 @@ namespace {
       // For each byte of the fragment that the fixup touches, mask in the bits
       // from the fixup value. The Value has been "split up" into the
       // appropriate bitfields above.
-      for (unsigned i = 0; i != 4; ++i)
-        Data[Offset + i] |= uint8_t((Value >> ((4 - i - 1)*8)) & 0xff);
-
+      for (unsigned i = 0; i != 4; ++i) {
+        unsigned Idx = IsLittleEndian ? i : 3 - i;
+        Data[Offset + Idx] |= uint8_t((Value >> (i * 8)) & 0xff);
+      }
     }
 
     MCObjectWriter *createObjectWriter(raw_pwrite_stream &OS) const override {
       uint8_t OSABI = MCELFObjectTargetWriter::getOSABI(OSType);
-      return createSparcELFObjectWriter(OS, is64Bit(), OSABI);
+      return createSparcELFObjectWriter(OS, Is64Bit, IsLittleEndian, OSABI);
     }
   };
 
 } // end anonymous namespace
 
-
 MCAsmBackend *llvm::createSparcAsmBackend(const Target &T,
                                           const MCRegisterInfo &MRI,
-                                          StringRef TT,
-                                          StringRef CPU) {
-  return new ELFSparcAsmBackend(T, Triple(TT).getOS());
+                                          const Triple &TT, StringRef CPU) {
+  return new ELFSparcAsmBackend(T, TT.getOS());
 }
diff --git a/lib/Target/Sparc/MCTargetDesc/SparcELFObjectWriter.cpp b/lib/Target/Sparc/MCTargetDesc/SparcELFObjectWriter.cpp
index 98ba7e6a631..0be60fd7a05 100644
--- a/lib/Target/Sparc/MCTargetDesc/SparcELFObjectWriter.cpp
+++ b/lib/Target/Sparc/MCTargetDesc/SparcELFObjectWriter.cpp
@@ -31,6 +31,10 @@ namespace {
   protected:
     unsigned GetRelocType(const MCValue &Target, const MCFixup &Fixup,
                           bool IsPCRel) const override;
+
+    bool needsRelocateWithSymbol(const MCSymbol &Sym,
+                                 unsigned Type) const override;
+
   };
 }
 
@@ -105,8 +109,31 @@ unsigned SparcELFObjectWriter::GetRelocType(const MCValue &Target,
   return ELF::R_SPARC_NONE;
 }
 
+bool SparcELFObjectWriter::needsRelocateWithSymbol(const MCSymbol &Sym,
+                                                 unsigned Type) const {
+  switch (Type) {
+    default:
+      return false;
+
+    // All relocations that use a GOT need a symbol, not an offset, as
+    // the offset of the symbol within the section is irrelevant to
+    // where the GOT entry is. Don't need to list all the TLS entries,
+    // as they're all marked as requiring a symbol anyways.
+    case ELF::R_SPARC_GOT10:
+    case ELF::R_SPARC_GOT13:
+    case ELF::R_SPARC_GOT22:
+    case ELF::R_SPARC_GOTDATA_HIX22:
+    case ELF::R_SPARC_GOTDATA_LOX10:
+    case ELF::R_SPARC_GOTDATA_OP_HIX22:
+    case ELF::R_SPARC_GOTDATA_OP_LOX10:
+      return true;
+  }
+}
+
 MCObjectWriter *llvm::createSparcELFObjectWriter(raw_pwrite_stream &OS,
-                                                 bool Is64Bit, uint8_t OSABI) {
+                                                 bool Is64Bit,
+                                                 bool IsLittleEndian,
+                                                 uint8_t OSABI) {
   MCELFObjectTargetWriter *MOTW = new SparcELFObjectWriter(Is64Bit, OSABI);
-  return createELFObjectWriter(MOTW, OS,  /*IsLittleEndian=*/false);
+  return createELFObjectWriter(MOTW, OS, IsLittleEndian);
 }
diff --git a/lib/Target/Sparc/MCTargetDesc/SparcMCAsmInfo.cpp b/lib/Target/Sparc/MCTargetDesc/SparcMCAsmInfo.cpp
index 6767e4b224f..280c6d7937b 100644
--- a/lib/Target/Sparc/MCTargetDesc/SparcMCAsmInfo.cpp
+++ b/lib/Target/Sparc/MCTargetDesc/SparcMCAsmInfo.cpp
@@ -18,12 +18,11 @@
 
 using namespace llvm;
 
-void SparcELFMCAsmInfo::anchor() { }
+void SparcELFMCAsmInfo::anchor() {}
 
-SparcELFMCAsmInfo::SparcELFMCAsmInfo(StringRef TT) {
-  IsLittleEndian = false;
-  Triple TheTriple(TT);
+SparcELFMCAsmInfo::SparcELFMCAsmInfo(const Triple &TheTriple) {
   bool isV9 = (TheTriple.getArch() == Triple::sparcv9);
+  IsLittleEndian = (TheTriple.getArch() == Triple::sparcel);
 
   if (isV9) {
     PointerSize = CalleeSaveStackSlotSize = 8;
@@ -51,8 +50,8 @@ SparcELFMCAsmInfo::getExprForPersonalitySymbol(const MCSymbol *Sym,
                                                MCStreamer &Streamer) const {
   if (Encoding & dwarf::DW_EH_PE_pcrel) {
     MCContext &Ctx = Streamer.getContext();
-    return SparcMCExpr::Create(SparcMCExpr::VK_Sparc_R_DISP32,
-                               MCSymbolRefExpr::Create(Sym, Ctx), Ctx);
+    return SparcMCExpr::create(SparcMCExpr::VK_Sparc_R_DISP32,
+                               MCSymbolRefExpr::create(Sym, Ctx), Ctx);
   }
 
   return MCAsmInfo::getExprForPersonalitySymbol(Sym, Encoding, Streamer);
@@ -64,8 +63,8 @@ SparcELFMCAsmInfo::getExprForFDESymbol(const MCSymbol *Sym,
                                        MCStreamer &Streamer) const {
   if (Encoding & dwarf::DW_EH_PE_pcrel) {
     MCContext &Ctx = Streamer.getContext();
-    return SparcMCExpr::Create(SparcMCExpr::VK_Sparc_R_DISP32,
-                               MCSymbolRefExpr::Create(Sym, Ctx), Ctx);
+    return SparcMCExpr::create(SparcMCExpr::VK_Sparc_R_DISP32,
+                               MCSymbolRefExpr::create(Sym, Ctx), Ctx);
   }
   return MCAsmInfo::getExprForFDESymbol(Sym, Encoding, Streamer);
 }
diff --git a/lib/Target/Sparc/MCTargetDesc/SparcMCAsmInfo.h b/lib/Target/Sparc/MCTargetDesc/SparcMCAsmInfo.h
index 84de55145b6..12386f14443 100644
--- a/lib/Target/Sparc/MCTargetDesc/SparcMCAsmInfo.h
+++ b/lib/Target/Sparc/MCTargetDesc/SparcMCAsmInfo.h
@@ -17,12 +17,12 @@
 #include "llvm/MC/MCAsmInfoELF.h"
 
 namespace llvm {
-class StringRef;
+class Triple;
 
 class SparcELFMCAsmInfo : public MCAsmInfoELF {
   void anchor() override;
 public:
-  explicit SparcELFMCAsmInfo(StringRef TT);
+  explicit SparcELFMCAsmInfo(const Triple &TheTriple);
   const MCExpr*
   getExprForPersonalitySymbol(const MCSymbol *Sym, unsigned Encoding,
                               MCStreamer &Streamer) const override;
diff --git a/lib/Target/Sparc/MCTargetDesc/SparcMCCodeEmitter.cpp b/lib/Target/Sparc/MCTargetDesc/SparcMCCodeEmitter.cpp
index b447ab3a75f..9171d4dc9c0 100644
--- a/lib/Target/Sparc/MCTargetDesc/SparcMCCodeEmitter.cpp
+++ b/lib/Target/Sparc/MCTargetDesc/SparcMCCodeEmitter.cpp
@@ -21,6 +21,7 @@
 #include "llvm/MC/MCInst.h"
 #include "llvm/MC/MCRegisterInfo.h"
 #include "llvm/MC/MCSymbol.h"
+#include "llvm/MC/MCAsmInfo.h"
 #include "llvm/Support/raw_ostream.h"
 
 using namespace llvm;
@@ -40,7 +41,7 @@ class SparcMCCodeEmitter : public MCCodeEmitter {
 
   ~SparcMCCodeEmitter() override {}
 
-  void EncodeInstruction(const MCInst &MI, raw_ostream &OS,
+  void encodeInstruction(const MCInst &MI, raw_ostream &OS,
                          SmallVectorImpl<MCFixup> &Fixups,
                          const MCSubtargetInfo &STI) const override;
 
@@ -78,16 +79,17 @@ MCCodeEmitter *llvm::createSparcMCCodeEmitter(const MCInstrInfo &MCII,
   return new SparcMCCodeEmitter(Ctx);
 }
 
-void SparcMCCodeEmitter::
-EncodeInstruction(const MCInst &MI, raw_ostream &OS,
-                  SmallVectorImpl<MCFixup> &Fixups,
-                  const MCSubtargetInfo &STI) const {
+void SparcMCCodeEmitter::encodeInstruction(const MCInst &MI, raw_ostream &OS,
+                                           SmallVectorImpl<MCFixup> &Fixups,
+                                           const MCSubtargetInfo &STI) const {
   unsigned Bits = getBinaryCodeForInstr(MI, Fixups, STI);
 
-  // Output the constant in big endian byte order.
-  for (unsigned i = 0; i != 4; ++i) {
-    OS << (char)(Bits >> 24);
-    Bits <<= 8;
+  if (Ctx.getAsmInfo()->isLittleEndian()) {
+    // Output the bits in little-endian byte order.
+    support::endian::Writer<support::little>(OS).write<uint32_t>(Bits);
+  } else {
+    // Output the bits in big-endian byte order.
+    support::endian::Writer<support::big>(OS).write<uint32_t>(Bits);
   }
   unsigned tlsOpNo = 0;
   switch (MI.getOpcode()) {
@@ -124,12 +126,12 @@ getMachineOpValue(const MCInst &MI, const MCOperand &MO,
   const MCExpr *Expr = MO.getExpr();
   if (const SparcMCExpr *SExpr = dyn_cast<SparcMCExpr>(Expr)) {
     MCFixupKind Kind = (MCFixupKind)SExpr->getFixupKind();
-    Fixups.push_back(MCFixup::Create(0, Expr, Kind));
+    Fixups.push_back(MCFixup::create(0, Expr, Kind));
     return 0;
   }
 
   int64_t Res;
-  if (Expr->EvaluateAsAbsolute(Res))
+  if (Expr->evaluateAsAbsolute(Res))
     return Res;
 
   llvm_unreachable("Unhandled expression!");
@@ -146,7 +148,7 @@ getCallTargetOpValue(const MCInst &MI, unsigned OpNo,
 
   if (MI.getOpcode() == SP::TLS_CALL) {
     // No fixups for __tls_get_addr. Will emit for fixups for tls_symbol in
-    // EncodeInstruction.
+    // encodeInstruction.
 #ifndef NDEBUG
     // Verify that the callee is actually __tls_get_addr.
     const SparcMCExpr *SExpr = dyn_cast<SparcMCExpr>(MO.getExpr());
@@ -166,7 +168,7 @@ getCallTargetOpValue(const MCInst &MI, unsigned OpNo,
       fixupKind = (MCFixupKind)Sparc::fixup_sparc_wplt30;
   }
 
-  Fixups.push_back(MCFixup::Create(0, MO.getExpr(), fixupKind));
+  Fixups.push_back(MCFixup::create(0, MO.getExpr(), fixupKind));
 
   return 0;
 }
@@ -179,7 +181,7 @@ getBranchTargetOpValue(const MCInst &MI, unsigned OpNo,
   if (MO.isReg() || MO.isImm())
     return getMachineOpValue(MI, MO, Fixups, STI);
 
-  Fixups.push_back(MCFixup::Create(0, MO.getExpr(),
+  Fixups.push_back(MCFixup::create(0, MO.getExpr(),
                                    (MCFixupKind)Sparc::fixup_sparc_br22));
   return 0;
 }
@@ -192,7 +194,7 @@ getBranchPredTargetOpValue(const MCInst &MI, unsigned OpNo,
   if (MO.isReg() || MO.isImm())
     return getMachineOpValue(MI, MO, Fixups, STI);
 
-  Fixups.push_back(MCFixup::Create(0, MO.getExpr(),
+  Fixups.push_back(MCFixup::create(0, MO.getExpr(),
                                    (MCFixupKind)Sparc::fixup_sparc_br19));
   return 0;
 }
@@ -204,9 +206,9 @@ getBranchOnRegTargetOpValue(const MCInst &MI, unsigned OpNo,
   if (MO.isReg() || MO.isImm())
     return getMachineOpValue(MI, MO, Fixups, STI);
 
-  Fixups.push_back(MCFixup::Create(0, MO.getExpr(),
+  Fixups.push_back(MCFixup::create(0, MO.getExpr(),
                                    (MCFixupKind)Sparc::fixup_sparc_br16_2));
-  Fixups.push_back(MCFixup::Create(0, MO.getExpr(),
+  Fixups.push_back(MCFixup::create(0, MO.getExpr(),
                                    (MCFixupKind)Sparc::fixup_sparc_br16_14));
 
   return 0;
diff --git a/lib/Target/Sparc/MCTargetDesc/SparcMCExpr.cpp b/lib/Target/Sparc/MCTargetDesc/SparcMCExpr.cpp
index d97e3a25c5a..e85a8cd5e33 100644
--- a/lib/Target/Sparc/MCTargetDesc/SparcMCExpr.cpp
+++ b/lib/Target/Sparc/MCTargetDesc/SparcMCExpr.cpp
@@ -15,9 +15,8 @@
 #include "SparcMCExpr.h"
 #include "llvm/MC/MCAssembler.h"
 #include "llvm/MC/MCContext.h"
-#include "llvm/MC/MCELF.h"
 #include "llvm/MC/MCObjectStreamer.h"
-#include "llvm/MC/MCSymbol.h"
+#include "llvm/MC/MCSymbolELF.h"
 #include "llvm/Object/ELF.h"
 
 
@@ -26,20 +25,17 @@ using namespace llvm;
 #define DEBUG_TYPE "sparcmcexpr"
 
 const SparcMCExpr*
-SparcMCExpr::Create(VariantKind Kind, const MCExpr *Expr,
+SparcMCExpr::create(VariantKind Kind, const MCExpr *Expr,
                       MCContext &Ctx) {
     return new (Ctx) SparcMCExpr(Kind, Expr);
 }
 
-
-
-void SparcMCExpr::PrintImpl(raw_ostream &OS) const
-{
+void SparcMCExpr::printImpl(raw_ostream &OS, const MCAsmInfo *MAI) const {
 
   bool closeParen = printVariantKind(OS, Kind);
 
   const MCExpr *Expr = getSubExpr();
-  Expr->print(OS);
+  Expr->print(OS, MAI);
 
   if (closeParen)
     OS << ')';
@@ -160,10 +156,10 @@ Sparc::Fixups SparcMCExpr::getFixupKind(SparcMCExpr::VariantKind Kind) {
 }
 
 bool
-SparcMCExpr::EvaluateAsRelocatableImpl(MCValue &Res,
+SparcMCExpr::evaluateAsRelocatableImpl(MCValue &Res,
                                        const MCAsmLayout *Layout,
                                        const MCFixup *Fixup) const {
-  return getSubExpr()->EvaluateAsRelocatable(Res, Layout, Fixup);
+  return getSubExpr()->evaluateAsRelocatable(Res, Layout, Fixup);
 }
 
 static void fixELFSymbolsInTLSFixupsImpl(const MCExpr *Expr, MCAssembler &Asm) {
@@ -184,8 +180,7 @@ static void fixELFSymbolsInTLSFixupsImpl(const MCExpr *Expr, MCAssembler &Asm) {
 
   case MCExpr::SymbolRef: {
     const MCSymbolRefExpr &SymRef = *cast<MCSymbolRefExpr>(Expr);
-    MCSymbolData &SD = Asm.getOrCreateSymbolData(SymRef.getSymbol());
-    MCELF::SetType(SD, ELF::STT_TLS);
+    cast<MCSymbolELF>(SymRef.getSymbol()).setType(ELF::STT_TLS);
     break;
   }
 
diff --git a/lib/Target/Sparc/MCTargetDesc/SparcMCExpr.h b/lib/Target/Sparc/MCTargetDesc/SparcMCExpr.h
index 3a6f508da72..d08ad86dbe0 100644
--- a/lib/Target/Sparc/MCTargetDesc/SparcMCExpr.h
+++ b/lib/Target/Sparc/MCTargetDesc/SparcMCExpr.h
@@ -69,7 +69,7 @@ class SparcMCExpr : public MCTargetExpr {
   /// @name Construction
   /// @{
 
-  static const SparcMCExpr *Create(VariantKind Kind, const MCExpr *Expr,
+  static const SparcMCExpr *create(VariantKind Kind, const MCExpr *Expr,
                                  MCContext &Ctx);
   /// @}
   /// @name Accessors
@@ -85,13 +85,13 @@ class SparcMCExpr : public MCTargetExpr {
   Sparc::Fixups getFixupKind() const { return getFixupKind(Kind); }
 
   /// @}
-  void PrintImpl(raw_ostream &OS) const override;
-  bool EvaluateAsRelocatableImpl(MCValue &Res,
+  void printImpl(raw_ostream &OS, const MCAsmInfo *MAI) const override;
+  bool evaluateAsRelocatableImpl(MCValue &Res,
                                  const MCAsmLayout *Layout,
                                  const MCFixup *Fixup) const override;
   void visitUsedExpr(MCStreamer &Streamer) const override;
-  const MCSection *FindAssociatedSection() const override {
-    return getSubExpr()->FindAssociatedSection();
+  MCSection *findAssociatedSection() const override {
+    return getSubExpr()->findAssociatedSection();
   }
 
   void fixELFSymbolsInTLSFixups(MCAssembler &Asm) const override;
diff --git a/lib/Target/Sparc/MCTargetDesc/SparcMCTargetDesc.cpp b/lib/Target/Sparc/MCTargetDesc/SparcMCTargetDesc.cpp
index 78954047fc9..9113e4a46b9 100644
--- a/lib/Target/Sparc/MCTargetDesc/SparcMCTargetDesc.cpp
+++ b/lib/Target/Sparc/MCTargetDesc/SparcMCTargetDesc.cpp
@@ -34,7 +34,7 @@ using namespace llvm;
 #include "SparcGenRegisterInfo.inc"
 
 static MCAsmInfo *createSparcMCAsmInfo(const MCRegisterInfo &MRI,
-                                       StringRef TT) {
+                                       const Triple &TT) {
   MCAsmInfo *MAI = new SparcELFMCAsmInfo(TT);
   unsigned Reg = MRI.getDwarfRegNum(SP::O6, true);
   MCCFIInstruction Inst = MCCFIInstruction::createDefCfa(nullptr, Reg, 0);
@@ -43,7 +43,7 @@ static MCAsmInfo *createSparcMCAsmInfo(const MCRegisterInfo &MRI,
 }
 
 static MCAsmInfo *createSparcV9MCAsmInfo(const MCRegisterInfo &MRI,
-                                       StringRef TT) {
+                                         const Triple &TT) {
   MCAsmInfo *MAI = new SparcELFMCAsmInfo(TT);
   unsigned Reg = MRI.getDwarfRegNum(SP::O6, true);
   MCCFIInstruction Inst = MCCFIInstruction::createDefCfa(nullptr, Reg, 2047);
@@ -57,20 +57,17 @@ static MCInstrInfo *createSparcMCInstrInfo() {
   return X;
 }
 
-static MCRegisterInfo *createSparcMCRegisterInfo(StringRef TT) {
+static MCRegisterInfo *createSparcMCRegisterInfo(const Triple &TT) {
   MCRegisterInfo *X = new MCRegisterInfo();
   InitSparcMCRegisterInfo(X, SP::O7);
   return X;
 }
 
-static MCSubtargetInfo *createSparcMCSubtargetInfo(StringRef TT, StringRef CPU,
-                                                   StringRef FS) {
-  MCSubtargetInfo *X = new MCSubtargetInfo();
-  Triple TheTriple(TT);
+static MCSubtargetInfo *
+createSparcMCSubtargetInfo(const Triple &TT, StringRef CPU, StringRef FS) {
   if (CPU.empty())
-    CPU = (TheTriple.getArch() == Triple::sparcv9) ? "v9" : "v8";
-  InitSparcMCSubtargetInfo(X, TT, CPU, FS);
-  return X;
+    CPU = (TT.getArch() == Triple::sparcv9) ? "v9" : "v8";
+  return createSparcMCSubtargetInfoImpl(TT, CPU, FS);
 }
 
 // Code models. Some only make sense for 64-bit code.
@@ -84,7 +81,8 @@ static MCSubtargetInfo *createSparcMCSubtargetInfo(StringRef TT, StringRef CPU,
 //
 // All code models require that the text segment is smaller than 2GB.
 
-static MCCodeGenInfo *createSparcMCCodeGenInfo(StringRef TT, Reloc::Model RM,
+static MCCodeGenInfo *createSparcMCCodeGenInfo(const Triple &TT,
+                                               Reloc::Model RM,
                                                CodeModel::Model CM,
                                                CodeGenOpt::Level OL) {
   MCCodeGenInfo *X = new MCCodeGenInfo();
@@ -97,11 +95,12 @@ static MCCodeGenInfo *createSparcMCCodeGenInfo(StringRef TT, Reloc::Model RM,
   case CodeModel::JITDefault: CM = CodeModel::Small; break;
   }
 
-  X->InitMCCodeGenInfo(RM, CM, OL);
+  X->initMCCodeGenInfo(RM, CM, OL);
   return X;
 }
 
-static MCCodeGenInfo *createSparcV9MCCodeGenInfo(StringRef TT, Reloc::Model RM,
+static MCCodeGenInfo *createSparcV9MCCodeGenInfo(const Triple &TT,
+                                                 Reloc::Model RM,
                                                  CodeModel::Model CM,
                                                  CodeGenOpt::Level OL) {
   MCCodeGenInfo *X = new MCCodeGenInfo();
@@ -118,7 +117,7 @@ static MCCodeGenInfo *createSparcV9MCCodeGenInfo(StringRef TT, Reloc::Model RM,
     break;
   }
 
-  X->InitMCCodeGenInfo(RM, CM, OL);
+  X->initMCCodeGenInfo(RM, CM, OL);
   return X;
 }
 
@@ -146,8 +145,9 @@ extern "C" void LLVMInitializeSparcTargetMC() {
   // Register the MC asm info.
   RegisterMCAsmInfoFn X(TheSparcTarget, createSparcMCAsmInfo);
   RegisterMCAsmInfoFn Y(TheSparcV9Target, createSparcV9MCAsmInfo);
+  RegisterMCAsmInfoFn Z(TheSparcelTarget, createSparcMCAsmInfo);
 
-  for (Target *T : {&TheSparcTarget, &TheSparcV9Target}) {
+  for (Target *T : {&TheSparcTarget, &TheSparcV9Target, &TheSparcelTarget}) {
     // Register the MC instruction info.
     TargetRegistry::RegisterMCInstrInfo(*T, createSparcMCInstrInfo);
 
@@ -176,8 +176,9 @@ extern "C" void LLVMInitializeSparcTargetMC() {
 
   // Register the MC codegen info.
   TargetRegistry::RegisterMCCodeGenInfo(TheSparcTarget,
-                                       createSparcMCCodeGenInfo);
+                                        createSparcMCCodeGenInfo);
   TargetRegistry::RegisterMCCodeGenInfo(TheSparcV9Target,
-                                       createSparcV9MCCodeGenInfo);
-
+                                        createSparcV9MCCodeGenInfo);
+  TargetRegistry::RegisterMCCodeGenInfo(TheSparcelTarget,
+                                        createSparcMCCodeGenInfo);
 }
diff --git a/lib/Target/Sparc/MCTargetDesc/SparcMCTargetDesc.h b/lib/Target/Sparc/MCTargetDesc/SparcMCTargetDesc.h
index 5f38b12403e..a9c9f15454e 100644
--- a/lib/Target/Sparc/MCTargetDesc/SparcMCTargetDesc.h
+++ b/lib/Target/Sparc/MCTargetDesc/SparcMCTargetDesc.h
@@ -25,22 +25,22 @@ class MCObjectWriter;
 class MCRegisterInfo;
 class MCSubtargetInfo;
 class Target;
+class Triple;
 class StringRef;
 class raw_pwrite_stream;
 class raw_ostream;
 
 extern Target TheSparcTarget;
 extern Target TheSparcV9Target;
+extern Target TheSparcelTarget;
 
 MCCodeEmitter *createSparcMCCodeEmitter(const MCInstrInfo &MCII,
                                         const MCRegisterInfo &MRI,
                                         MCContext &Ctx);
-MCAsmBackend *createSparcAsmBackend(const Target &T,
-                                    const MCRegisterInfo &MRI,
-                                    StringRef TT,
-                                    StringRef CPU);
+MCAsmBackend *createSparcAsmBackend(const Target &T, const MCRegisterInfo &MRI,
+                                    const Triple &TT, StringRef CPU);
 MCObjectWriter *createSparcELFObjectWriter(raw_pwrite_stream &OS, bool Is64Bit,
-                                           uint8_t OSABI);
+                                           bool IsLIttleEndian, uint8_t OSABI);
 } // End llvm namespace
 
 // Defines symbolic names for Sparc registers.  This defines a mapping from
diff --git a/lib/Target/Sparc/SparcAsmPrinter.cpp b/lib/Target/Sparc/SparcAsmPrinter.cpp
index 56290e2cf01..3afaa63ba42 100644
--- a/lib/Target/Sparc/SparcAsmPrinter.cpp
+++ b/lib/Target/Sparc/SparcAsmPrinter.cpp
@@ -40,7 +40,7 @@ namespace {
   class SparcAsmPrinter : public AsmPrinter {
     SparcTargetStreamer &getTargetStreamer() {
       return static_cast<SparcTargetStreamer &>(
-          *OutStreamer.getTargetStreamer());
+          *OutStreamer->getTargetStreamer());
     }
   public:
     explicit SparcAsmPrinter(TargetMachine &TM,
@@ -78,10 +78,10 @@ namespace {
 
 static MCOperand createSparcMCOperand(SparcMCExpr::VariantKind Kind,
                                       MCSymbol *Sym, MCContext &OutContext) {
-  const MCSymbolRefExpr *MCSym = MCSymbolRefExpr::Create(Sym,
+  const MCSymbolRefExpr *MCSym = MCSymbolRefExpr::create(Sym,
                                                          OutContext);
-  const SparcMCExpr *expr = SparcMCExpr::Create(Kind, MCSym, OutContext);
-  return MCOperand::CreateExpr(expr);
+  const SparcMCExpr *expr = SparcMCExpr::create(Kind, MCSym, OutContext);
+  return MCOperand::createExpr(expr);
 
 }
 static MCOperand createPCXCallOP(MCSymbol *Label,
@@ -94,17 +94,17 @@ static MCOperand createPCXRelExprOp(SparcMCExpr::VariantKind Kind,
                                     MCSymbol *CurLabel,
                                     MCContext &OutContext)
 {
-  const MCSymbolRefExpr *GOT = MCSymbolRefExpr::Create(GOTLabel, OutContext);
-  const MCSymbolRefExpr *Start = MCSymbolRefExpr::Create(StartLabel,
+  const MCSymbolRefExpr *GOT = MCSymbolRefExpr::create(GOTLabel, OutContext);
+  const MCSymbolRefExpr *Start = MCSymbolRefExpr::create(StartLabel,
                                                          OutContext);
-  const MCSymbolRefExpr *Cur = MCSymbolRefExpr::Create(CurLabel,
+  const MCSymbolRefExpr *Cur = MCSymbolRefExpr::create(CurLabel,
                                                        OutContext);
 
-  const MCBinaryExpr *Sub = MCBinaryExpr::CreateSub(Cur, Start, OutContext);
-  const MCBinaryExpr *Add = MCBinaryExpr::CreateAdd(GOT, Sub, OutContext);
-  const SparcMCExpr *expr = SparcMCExpr::Create(Kind,
+  const MCBinaryExpr *Sub = MCBinaryExpr::createSub(Cur, Start, OutContext);
+  const MCBinaryExpr *Add = MCBinaryExpr::createAdd(GOT, Sub, OutContext);
+  const SparcMCExpr *expr = SparcMCExpr::create(Kind,
                                                 Add, OutContext);
-  return MCOperand::CreateExpr(expr);
+  return MCOperand::createExpr(expr);
 }
 
 static void EmitCall(MCStreamer &OutStreamer,
@@ -176,13 +176,13 @@ void SparcAsmPrinter::LowerGETPCXAndEmitMCInsts(const MachineInstr *MI,
                                                 const MCSubtargetInfo &STI)
 {
   MCSymbol *GOTLabel   =
-    OutContext.GetOrCreateSymbol(Twine("_GLOBAL_OFFSET_TABLE_"));
+    OutContext.getOrCreateSymbol(Twine("_GLOBAL_OFFSET_TABLE_"));
 
   const MachineOperand &MO = MI->getOperand(0);
   assert(MO.getReg() != SP::O7 &&
          "%o7 is assigned as destination for getpcx!");
 
-  MCOperand MCRegOP = MCOperand::CreateReg(MO.getReg());
+  MCOperand MCRegOP = MCOperand::createReg(MO.getReg());
 
 
   if (TM.getRelocationModel() != Reloc::PIC_) {
@@ -191,45 +191,45 @@ void SparcAsmPrinter::LowerGETPCXAndEmitMCInsts(const MachineInstr *MI,
     default:
       llvm_unreachable("Unsupported absolute code model");
     case CodeModel::Small:
-      EmitHiLo(OutStreamer, GOTLabel,
+      EmitHiLo(*OutStreamer, GOTLabel,
                SparcMCExpr::VK_Sparc_HI, SparcMCExpr::VK_Sparc_LO,
                MCRegOP, OutContext, STI);
       break;
     case CodeModel::Medium: {
-      EmitHiLo(OutStreamer, GOTLabel,
+      EmitHiLo(*OutStreamer, GOTLabel,
                SparcMCExpr::VK_Sparc_H44, SparcMCExpr::VK_Sparc_M44,
                MCRegOP, OutContext, STI);
-      MCOperand imm = MCOperand::CreateExpr(MCConstantExpr::Create(12,
+      MCOperand imm = MCOperand::createExpr(MCConstantExpr::create(12,
                                                                    OutContext));
-      EmitSHL(OutStreamer, MCRegOP, imm, MCRegOP, STI);
+      EmitSHL(*OutStreamer, MCRegOP, imm, MCRegOP, STI);
       MCOperand lo = createSparcMCOperand(SparcMCExpr::VK_Sparc_L44,
                                           GOTLabel, OutContext);
-      EmitOR(OutStreamer, MCRegOP, lo, MCRegOP, STI);
+      EmitOR(*OutStreamer, MCRegOP, lo, MCRegOP, STI);
       break;
     }
     case CodeModel::Large: {
-      EmitHiLo(OutStreamer, GOTLabel,
+      EmitHiLo(*OutStreamer, GOTLabel,
                SparcMCExpr::VK_Sparc_HH, SparcMCExpr::VK_Sparc_HM,
                MCRegOP, OutContext, STI);
-      MCOperand imm = MCOperand::CreateExpr(MCConstantExpr::Create(32,
+      MCOperand imm = MCOperand::createExpr(MCConstantExpr::create(32,
                                                                    OutContext));
-      EmitSHL(OutStreamer, MCRegOP, imm, MCRegOP, STI);
+      EmitSHL(*OutStreamer, MCRegOP, imm, MCRegOP, STI);
       // Use register %o7 to load the lower 32 bits.
-      MCOperand RegO7 = MCOperand::CreateReg(SP::O7);
-      EmitHiLo(OutStreamer, GOTLabel,
+      MCOperand RegO7 = MCOperand::createReg(SP::O7);
+      EmitHiLo(*OutStreamer, GOTLabel,
                SparcMCExpr::VK_Sparc_HI, SparcMCExpr::VK_Sparc_LO,
                RegO7, OutContext, STI);
-      EmitADD(OutStreamer, MCRegOP, RegO7, MCRegOP, STI);
+      EmitADD(*OutStreamer, MCRegOP, RegO7, MCRegOP, STI);
     }
     }
     return;
   }
 
-  MCSymbol *StartLabel = OutContext.CreateTempSymbol();
-  MCSymbol *EndLabel   = OutContext.CreateTempSymbol();
-  MCSymbol *SethiLabel = OutContext.CreateTempSymbol();
+  MCSymbol *StartLabel = OutContext.createTempSymbol();
+  MCSymbol *EndLabel   = OutContext.createTempSymbol();
+  MCSymbol *SethiLabel = OutContext.createTempSymbol();
 
-  MCOperand RegO7   = MCOperand::CreateReg(SP::O7);
+  MCOperand RegO7   = MCOperand::createReg(SP::O7);
 
   // <StartLabel>:
   //   call <EndLabel>
@@ -239,20 +239,20 @@ void SparcAsmPrinter::LowerGETPCXAndEmitMCInsts(const MachineInstr *MI,
   //   or  <MO>, %lo(_GLOBAL_OFFSET_TABLE_+(<EndLabel>-<StartLabel>))), <MO>
   //   add <MO>, %o7, <MO>
 
-  OutStreamer.EmitLabel(StartLabel);
+  OutStreamer->EmitLabel(StartLabel);
   MCOperand Callee =  createPCXCallOP(EndLabel, OutContext);
-  EmitCall(OutStreamer, Callee, STI);
-  OutStreamer.EmitLabel(SethiLabel);
+  EmitCall(*OutStreamer, Callee, STI);
+  OutStreamer->EmitLabel(SethiLabel);
   MCOperand hiImm = createPCXRelExprOp(SparcMCExpr::VK_Sparc_PC22,
                                        GOTLabel, StartLabel, SethiLabel,
                                        OutContext);
-  EmitSETHI(OutStreamer, hiImm, MCRegOP, STI);
-  OutStreamer.EmitLabel(EndLabel);
+  EmitSETHI(*OutStreamer, hiImm, MCRegOP, STI);
+  OutStreamer->EmitLabel(EndLabel);
   MCOperand loImm = createPCXRelExprOp(SparcMCExpr::VK_Sparc_PC10,
                                        GOTLabel, StartLabel, EndLabel,
                                        OutContext);
-  EmitOR(OutStreamer, MCRegOP, loImm, MCRegOP, STI);
-  EmitADD(OutStreamer, MCRegOP, RegO7, MCRegOP, STI);
+  EmitOR(*OutStreamer, MCRegOP, loImm, MCRegOP, STI);
+  EmitADD(*OutStreamer, MCRegOP, RegO7, MCRegOP, STI);
 }
 
 void SparcAsmPrinter::EmitInstruction(const MachineInstr *MI)
@@ -272,7 +272,7 @@ void SparcAsmPrinter::EmitInstruction(const MachineInstr *MI)
   do {
     MCInst TmpInst;
     LowerSparcMachineInstrToMCInst(I, TmpInst, *this);
-    EmitToStreamer(OutStreamer, TmpInst);
+    EmitToStreamer(*OutStreamer, TmpInst);
   } while ((++I != E) && I->isInsideBundle()); // Delay slot check.
 }
 
@@ -296,7 +296,7 @@ void SparcAsmPrinter::EmitFunctionBodyStart() {
 
 void SparcAsmPrinter::printOperand(const MachineInstr *MI, int opNum,
                                    raw_ostream &O) {
-  const DataLayout *DL = TM.getDataLayout();
+  const DataLayout &DL = getDataLayout();
   const MachineOperand &MO = MI->getOperand (opNum);
   SparcMCExpr::VariantKind TF = (SparcMCExpr::VariantKind) MO.getTargetFlags();
 
@@ -361,10 +361,10 @@ void SparcAsmPrinter::printOperand(const MachineInstr *MI, int opNum,
     O << (int)MO.getImm();
     break;
   case MachineOperand::MO_MachineBasicBlock:
-    O << *MO.getMBB()->getSymbol();
+    MO.getMBB()->getSymbol()->print(O, MAI);
     return;
   case MachineOperand::MO_GlobalAddress:
-    O << *getSymbol(MO.getGlobal());
+    getSymbol(MO.getGlobal())->print(O, MAI);
     break;
   case MachineOperand::MO_BlockAddress:
     O <<  GetBlockAddressSymbol(MO.getBlockAddress())->getName();
@@ -373,7 +373,7 @@ void SparcAsmPrinter::printOperand(const MachineInstr *MI, int opNum,
     O << MO.getSymbolName();
     break;
   case MachineOperand::MO_ConstantPoolIndex:
-    O << DL->getPrivateGlobalPrefix() << "CPI" << getFunctionNumber() << "_"
+    O << DL.getPrivateGlobalPrefix() << "CPI" << getFunctionNumber() << "_"
       << MO.getIndex();
     break;
   default:
@@ -445,4 +445,5 @@ bool SparcAsmPrinter::PrintAsmMemoryOperand(const MachineInstr *MI,
 extern "C" void LLVMInitializeSparcAsmPrinter() {
   RegisterAsmPrinter<SparcAsmPrinter> X(TheSparcTarget);
   RegisterAsmPrinter<SparcAsmPrinter> Y(TheSparcV9Target);
+  RegisterAsmPrinter<SparcAsmPrinter> Z(TheSparcelTarget);
 }
diff --git a/lib/Target/Sparc/SparcFrameLowering.cpp b/lib/Target/Sparc/SparcFrameLowering.cpp
index a065d3a0803..8fa10dcae11 100644
--- a/lib/Target/Sparc/SparcFrameLowering.cpp
+++ b/lib/Target/Sparc/SparcFrameLowering.cpp
@@ -82,10 +82,11 @@ void SparcFrameLowering::emitSPAdjustment(MachineFunction &MF,
     .addReg(SP::O6).addReg(SP::G1);
 }
 
-void SparcFrameLowering::emitPrologue(MachineFunction &MF) const {
+void SparcFrameLowering::emitPrologue(MachineFunction &MF,
+                                      MachineBasicBlock &MBB) const {
   SparcMachineFunctionInfo *FuncInfo = MF.getInfo<SparcMachineFunctionInfo>();
 
-  MachineBasicBlock &MBB = MF.front();
+  assert(&MF.front() == &MBB && "Shrink-wrapping not yet supported");
   MachineFrameInfo *MFI = MF.getFrameInfo();
   const SparcInstrInfo &TII =
       *static_cast<const SparcInstrInfo *>(MF.getSubtarget().getInstrInfo());
@@ -189,11 +190,11 @@ static bool LLVM_ATTRIBUTE_UNUSED verifyLeafProcRegUse(MachineRegisterInfo *MRI)
 {
 
   for (unsigned reg = SP::I0; reg <= SP::I7; ++reg)
-    if (MRI->isPhysRegUsed(reg))
+    if (!MRI->reg_nodbg_empty(reg))
       return false;
 
   for (unsigned reg = SP::L0; reg <= SP::L7; ++reg)
-    if (MRI->isPhysRegUsed(reg))
+    if (!MRI->reg_nodbg_empty(reg))
       return false;
 
   return true;
@@ -205,10 +206,10 @@ bool SparcFrameLowering::isLeafProc(MachineFunction &MF) const
   MachineRegisterInfo &MRI = MF.getRegInfo();
   MachineFrameInfo    *MFI = MF.getFrameInfo();
 
-  return !(MFI->hasCalls()              // has calls
-           || MRI.isPhysRegUsed(SP::L0) // Too many registers needed
-           || MRI.isPhysRegUsed(SP::O6) // %SP is used
-           || hasFP(MF));               // need %FP
+  return !(MFI->hasCalls()                 // has calls
+           || !MRI.reg_nodbg_empty(SP::L0) // Too many registers needed
+           || !MRI.reg_nodbg_empty(SP::O6) // %SP is used
+           || hasFP(MF));                  // need %FP
 }
 
 void SparcFrameLowering::remapRegsForLeafProc(MachineFunction &MF) const {
@@ -217,16 +218,13 @@ void SparcFrameLowering::remapRegsForLeafProc(MachineFunction &MF) const {
 
   // Remap %i[0-7] to %o[0-7].
   for (unsigned reg = SP::I0; reg <= SP::I7; ++reg) {
-    if (!MRI.isPhysRegUsed(reg))
+    if (MRI.reg_nodbg_empty(reg))
       continue;
     unsigned mapped_reg = (reg - SP::I0 + SP::O0);
-    assert(!MRI.isPhysRegUsed(mapped_reg));
+    assert(MRI.reg_nodbg_empty(mapped_reg));
 
     // Replace I register with O register.
     MRI.replaceRegWith(reg, mapped_reg);
-
-    // Mark the reg unused.
-    MRI.setPhysRegUnused(reg);
   }
 
   // Rewrite MBB's Live-ins.
@@ -246,9 +244,10 @@ void SparcFrameLowering::remapRegsForLeafProc(MachineFunction &MF) const {
 #endif
 }
 
-void SparcFrameLowering::processFunctionBeforeCalleeSavedScan
-                  (MachineFunction &MF, RegScavenger *RS) const {
-
+void SparcFrameLowering::determineCalleeSaves(MachineFunction &MF,
+                                              BitVector &SavedRegs,
+                                              RegScavenger *RS) const {
+  TargetFrameLowering::determineCalleeSaves(MF, SavedRegs, RS);
   if (!DisableLeafProc && isLeafProc(MF)) {
     SparcMachineFunctionInfo *MFI = MF.getInfo<SparcMachineFunctionInfo>();
     MFI->setLeafProc(true);
diff --git a/lib/Target/Sparc/SparcFrameLowering.h b/lib/Target/Sparc/SparcFrameLowering.h
index 9e53994e3c3..29fc7b7ba03 100644
--- a/lib/Target/Sparc/SparcFrameLowering.h
+++ b/lib/Target/Sparc/SparcFrameLowering.h
@@ -26,7 +26,7 @@ class SparcFrameLowering : public TargetFrameLowering {
 
   /// emitProlog/emitEpilog - These methods insert prolog and epilog code into
   /// the function.
-  void emitPrologue(MachineFunction &MF) const override;
+  void emitPrologue(MachineFunction &MF, MachineBasicBlock &MBB) const override;
   void emitEpilogue(MachineFunction &MF, MachineBasicBlock &MBB) const override;
 
   void
@@ -36,8 +36,8 @@ class SparcFrameLowering : public TargetFrameLowering {
 
   bool hasReservedCallFrame(const MachineFunction &MF) const override;
   bool hasFP(const MachineFunction &MF) const override;
-  void processFunctionBeforeCalleeSavedScan(MachineFunction &MF,
-                                     RegScavenger *RS = nullptr) const override;
+  void determineCalleeSaves(MachineFunction &MF, BitVector &SavedRegs,
+                            RegScavenger *RS = nullptr) const override;
 
 private:
   // Remap input registers to output registers for leaf procedure.
diff --git a/lib/Target/Sparc/SparcISelDAGToDAG.cpp b/lib/Target/Sparc/SparcISelDAGToDAG.cpp
index 1cf5ccfdef6..340b72e7940 100644
--- a/lib/Target/Sparc/SparcISelDAGToDAG.cpp
+++ b/lib/Target/Sparc/SparcISelDAGToDAG.cpp
@@ -67,14 +67,17 @@ class SparcDAGToDAGISel : public SelectionDAGISel {
 
 SDNode* SparcDAGToDAGISel::getGlobalBaseReg() {
   unsigned GlobalBaseReg = Subtarget->getInstrInfo()->getGlobalBaseReg(MF);
-  return CurDAG->getRegister(GlobalBaseReg, TLI->getPointerTy()).getNode();
+  return CurDAG->getRegister(GlobalBaseReg,
+                             TLI->getPointerTy(CurDAG->getDataLayout()))
+      .getNode();
 }
 
 bool SparcDAGToDAGISel::SelectADDRri(SDValue Addr,
                                      SDValue &Base, SDValue &Offset) {
   if (FrameIndexSDNode *FIN = dyn_cast<FrameIndexSDNode>(Addr)) {
-    Base = CurDAG->getTargetFrameIndex(FIN->getIndex(), TLI->getPointerTy());
-    Offset = CurDAG->getTargetConstant(0, MVT::i32);
+    Base = CurDAG->getTargetFrameIndex(
+        FIN->getIndex(), TLI->getPointerTy(CurDAG->getDataLayout()));
+    Offset = CurDAG->getTargetConstant(0, SDLoc(Addr), MVT::i32);
     return true;
   }
   if (Addr.getOpcode() == ISD::TargetExternalSymbol ||
@@ -88,12 +91,13 @@ bool SparcDAGToDAGISel::SelectADDRri(SDValue Addr,
         if (FrameIndexSDNode *FIN =
                 dyn_cast<FrameIndexSDNode>(Addr.getOperand(0))) {
           // Constant offset from frame ref.
-          Base =
-              CurDAG->getTargetFrameIndex(FIN->getIndex(), TLI->getPointerTy());
+          Base = CurDAG->getTargetFrameIndex(
+              FIN->getIndex(), TLI->getPointerTy(CurDAG->getDataLayout()));
         } else {
           Base = Addr.getOperand(0);
         }
-        Offset = CurDAG->getTargetConstant(CN->getZExtValue(), MVT::i32);
+        Offset = CurDAG->getTargetConstant(CN->getZExtValue(), SDLoc(Addr),
+                                           MVT::i32);
         return true;
       }
     }
@@ -109,7 +113,7 @@ bool SparcDAGToDAGISel::SelectADDRri(SDValue Addr,
     }
   }
   Base = Addr;
-  Offset = CurDAG->getTargetConstant(0, MVT::i32);
+  Offset = CurDAG->getTargetConstant(0, SDLoc(Addr), MVT::i32);
   return true;
 }
 
@@ -133,7 +137,7 @@ bool SparcDAGToDAGISel::SelectADDRrr(SDValue Addr, SDValue &R1, SDValue &R2) {
   }
 
   R1 = Addr;
-  R2 = CurDAG->getRegister(SP::G0, TLI->getPointerTy());
+  R2 = CurDAG->getRegister(SP::G0, TLI->getPointerTy(CurDAG->getDataLayout()));
   return true;
 }
 
@@ -162,12 +166,14 @@ SDNode *SparcDAGToDAGISel::Select(SDNode *N) {
     SDValue TopPart;
     if (N->getOpcode() == ISD::SDIV) {
       TopPart = SDValue(CurDAG->getMachineNode(SP::SRAri, dl, MVT::i32, DivLHS,
-                                   CurDAG->getTargetConstant(31, MVT::i32)), 0);
+                                   CurDAG->getTargetConstant(31, dl, MVT::i32)),
+                        0);
     } else {
       TopPart = CurDAG->getRegister(SP::G0, MVT::i32);
     }
-    TopPart = SDValue(CurDAG->getMachineNode(SP::WRYrr, dl, MVT::Glue, TopPart,
-                                     CurDAG->getRegister(SP::G0, MVT::i32)), 0);
+    TopPart = CurDAG->getCopyToReg(CurDAG->getEntryNode(), dl, SP::Y, TopPart,
+                                   SDValue())
+                  .getValue(1);
 
     // FIXME: Handle div by immediate.
     unsigned Opcode = N->getOpcode() == ISD::SDIV ? SP::SDIVrr : SP::UDIVrr;
@@ -180,10 +186,11 @@ SDNode *SparcDAGToDAGISel::Select(SDNode *N) {
     SDValue MulLHS = N->getOperand(0);
     SDValue MulRHS = N->getOperand(1);
     unsigned Opcode = N->getOpcode() == ISD::MULHU ? SP::UMULrr : SP::SMULrr;
-    SDNode *Mul = CurDAG->getMachineNode(Opcode, dl, MVT::i32, MVT::Glue,
-                                         MulLHS, MulRHS);
-    // The high part is in the Y register.
-    return CurDAG->SelectNodeTo(N, SP::RDY, MVT::i32, SDValue(Mul, 1));
+    SDNode *Mul =
+        CurDAG->getMachineNode(Opcode, dl, MVT::i32, MVT::i32, MulLHS, MulRHS);
+    SDValue ResultHigh = SDValue(Mul, 1);
+    ReplaceUses(SDValue(N, 0), ResultHigh);
+    return nullptr;
   }
   }
 
diff --git a/lib/Target/Sparc/SparcISelLowering.cpp b/lib/Target/Sparc/SparcISelLowering.cpp
index 5b964af1959..c7c5d27dab6 100644
--- a/lib/Target/Sparc/SparcISelLowering.cpp
+++ b/lib/Target/Sparc/SparcISelLowering.cpp
@@ -221,15 +221,16 @@ SparcTargetLowering::LowerReturn_32(SDValue Chain,
     unsigned Reg = SFI->getSRetReturnReg();
     if (!Reg)
       llvm_unreachable("sret virtual register not created in the entry block");
-    SDValue Val = DAG.getCopyFromReg(Chain, DL, Reg, getPointerTy());
+    auto PtrVT = getPointerTy(DAG.getDataLayout());
+    SDValue Val = DAG.getCopyFromReg(Chain, DL, Reg, PtrVT);
     Chain = DAG.getCopyToReg(Chain, DL, SP::I0, Val, Flag);
     Flag = Chain.getValue(1);
-    RetOps.push_back(DAG.getRegister(SP::I0, getPointerTy()));
+    RetOps.push_back(DAG.getRegister(SP::I0, PtrVT));
     RetAddrOffset = 12; // CallInst + Delay Slot + Unimp
   }
 
   RetOps[0] = Chain;  // Update chain.
-  RetOps[1] = DAG.getConstant(RetAddrOffset, MVT::i32);
+  RetOps[1] = DAG.getConstant(RetAddrOffset, DL, MVT::i32);
 
   // Add the flag if we have it.
   if (Flag.getNode())
@@ -261,7 +262,7 @@ SparcTargetLowering::LowerReturn_64(SDValue Chain,
 
   // The second operand on the return instruction is the return address offset.
   // The return address is always %i7+8 with the 64-bit ABI.
-  RetOps.push_back(DAG.getConstant(8, MVT::i32));
+  RetOps.push_back(DAG.getConstant(8, DL, MVT::i32));
 
   // Copy the result values into the output registers.
   for (unsigned i = 0; i != RVLocs.size(); ++i) {
@@ -289,7 +290,7 @@ SparcTargetLowering::LowerReturn_64(SDValue Chain,
     // in the high bits of the register.
     if (VA.getValVT() == MVT::i32 && VA.needsCustom()) {
       OutVal = DAG.getNode(ISD::SHL, DL, MVT::i64, OutVal,
-                           DAG.getConstant(32, MVT::i32));
+                           DAG.getConstant(32, DL, MVT::i32));
 
       // The next value may go in the low bits of the same register.
       // Handle both at once.
@@ -418,6 +419,7 @@ LowerFormalArguments_32(SDValue Chain,
     assert(VA.isMemLoc());
 
     unsigned Offset = VA.getLocMemOffset()+StackOffset;
+    auto PtrVT = getPointerTy(DAG.getDataLayout());
 
     if (VA.needsCustom()) {
       assert(VA.getValVT() == MVT::f64);
@@ -426,7 +428,7 @@ LowerFormalArguments_32(SDValue Chain,
         int FI = MF.getFrameInfo()->CreateFixedObject(8,
                                                       Offset,
                                                       true);
-        SDValue FIPtr = DAG.getFrameIndex(FI, getPointerTy());
+        SDValue FIPtr = DAG.getFrameIndex(FI, PtrVT);
         SDValue Load = DAG.getLoad(VA.getValVT(), dl, Chain, FIPtr,
                                    MachinePointerInfo(),
                                    false,false, false, 0);
@@ -437,14 +439,14 @@ LowerFormalArguments_32(SDValue Chain,
       int FI = MF.getFrameInfo()->CreateFixedObject(4,
                                                     Offset,
                                                     true);
-      SDValue FIPtr = DAG.getFrameIndex(FI, getPointerTy());
+      SDValue FIPtr = DAG.getFrameIndex(FI, PtrVT);
       SDValue HiVal = DAG.getLoad(MVT::i32, dl, Chain, FIPtr,
                                   MachinePointerInfo(),
                                   false, false, false, 0);
       int FI2 = MF.getFrameInfo()->CreateFixedObject(4,
                                                      Offset+4,
                                                      true);
-      SDValue FIPtr2 = DAG.getFrameIndex(FI2, getPointerTy());
+      SDValue FIPtr2 = DAG.getFrameIndex(FI2, PtrVT);
 
       SDValue LoVal = DAG.getLoad(MVT::i32, dl, Chain, FIPtr2,
                                   MachinePointerInfo(),
@@ -460,7 +462,7 @@ LowerFormalArguments_32(SDValue Chain,
     int FI = MF.getFrameInfo()->CreateFixedObject(4,
                                                   Offset,
                                                   true);
-    SDValue FIPtr = DAG.getFrameIndex(FI, getPointerTy());
+    SDValue FIPtr = DAG.getFrameIndex(FI, PtrVT);
     SDValue Load ;
     if (VA.getValVT() == MVT::i32 || VA.getValVT() == MVT::f32) {
       Load = DAG.getLoad(VA.getValVT(), dl, Chain, FIPtr,
@@ -471,7 +473,7 @@ LowerFormalArguments_32(SDValue Chain,
       // Sparc is big endian, so add an offset based on the ObjectVT.
       unsigned Offset = 4-std::max(1U, VA.getValVT().getSizeInBits()/8);
       FIPtr = DAG.getNode(ISD::ADD, dl, MVT::i32, FIPtr,
-                          DAG.getConstant(Offset, MVT::i32));
+                          DAG.getConstant(Offset, dl, MVT::i32));
       Load = DAG.getExtLoad(LoadOp, dl, MVT::i32, Chain, FIPtr,
                             MachinePointerInfo(),
                             VA.getValVT(), false, false, false,0);
@@ -570,7 +572,7 @@ LowerFormalArguments_64(SDValue Chain,
       // Get the high bits for i32 struct elements.
       if (VA.getValVT() == MVT::i32 && VA.needsCustom())
         Arg = DAG.getNode(ISD::SRL, DL, VA.getLocVT(), Arg,
-                          DAG.getConstant(32, MVT::i32));
+                          DAG.getConstant(32, DL, MVT::i32));
 
       // The caller promoted the argument, so insert an Assert?ext SDNode so we
       // won't promote the value again in this function.
@@ -607,10 +609,10 @@ LowerFormalArguments_64(SDValue Chain,
     if (VA.isExtInLoc())
       Offset += 8 - ValSize;
     int FI = MF.getFrameInfo()->CreateFixedObject(ValSize, Offset, true);
-    InVals.push_back(DAG.getLoad(VA.getValVT(), DL, Chain,
-                                 DAG.getFrameIndex(FI, getPointerTy()),
-                                 MachinePointerInfo::getFixedStack(FI),
-                                 false, false, false, 0));
+    InVals.push_back(DAG.getLoad(
+        VA.getValVT(), DL, Chain,
+        DAG.getFrameIndex(FI, getPointerTy(MF.getDataLayout())),
+        MachinePointerInfo::getFixedStack(FI), false, false, false, 0));
   }
 
   if (!IsVarArg)
@@ -637,10 +639,10 @@ LowerFormalArguments_64(SDValue Chain,
     unsigned VReg = MF.addLiveIn(SP::I0 + ArgOffset/8, &SP::I64RegsRegClass);
     SDValue VArg = DAG.getCopyFromReg(Chain, DL, VReg, MVT::i64);
     int FI = MF.getFrameInfo()->CreateFixedObject(8, ArgOffset + ArgArea, true);
-    OutChains.push_back(DAG.getStore(Chain, DL, VArg,
-                                     DAG.getFrameIndex(FI, getPointerTy()),
-                                     MachinePointerInfo::getFixedStack(FI),
-                                     false, false, 0));
+    auto PtrVT = getPointerTy(MF.getDataLayout());
+    OutChains.push_back(
+        DAG.getStore(Chain, DL, VArg, DAG.getFrameIndex(FI, PtrVT),
+                     MachinePointerInfo::getFixedStack(FI), false, false, 0));
   }
 
   if (!OutChains.empty())
@@ -722,8 +724,8 @@ SparcTargetLowering::LowerCall_32(TargetLowering::CallLoweringInfo &CLI,
     unsigned Align = Flags.getByValAlign();
 
     int FI = MFI->CreateStackObject(Size, Align, false);
-    SDValue FIPtr = DAG.getFrameIndex(FI, getPointerTy());
-    SDValue SizeNode = DAG.getConstant(Size, MVT::i32);
+    SDValue FIPtr = DAG.getFrameIndex(FI, getPointerTy(DAG.getDataLayout()));
+    SDValue SizeNode = DAG.getConstant(Size, dl, MVT::i32);
 
     Chain = DAG.getMemcpy(Chain, dl, FIPtr, Arg, SizeNode, Align,
                           false,        // isVolatile,
@@ -733,7 +735,7 @@ SparcTargetLowering::LowerCall_32(TargetLowering::CallLoweringInfo &CLI,
     ByValArgs.push_back(FIPtr);
   }
 
-  Chain = DAG.getCALLSEQ_START(Chain, DAG.getIntPtrConstant(ArgsSize, true),
+  Chain = DAG.getCALLSEQ_START(Chain, DAG.getIntPtrConstant(ArgsSize, dl, true),
                                dl);
 
   SmallVector<std::pair<unsigned, SDValue>, 8> RegsToPass;
@@ -776,7 +778,7 @@ SparcTargetLowering::LowerCall_32(TargetLowering::CallLoweringInfo &CLI,
       assert(VA.needsCustom());
       // store SRet argument in %sp+64
       SDValue StackPtr = DAG.getRegister(SP::O6, MVT::i32);
-      SDValue PtrOff = DAG.getIntPtrConstant(64);
+      SDValue PtrOff = DAG.getIntPtrConstant(64, dl);
       PtrOff = DAG.getNode(ISD::ADD, dl, MVT::i32, StackPtr, PtrOff);
       MemOpChains.push_back(DAG.getStore(Chain, dl, Arg, PtrOff,
                                          MachinePointerInfo(),
@@ -793,7 +795,7 @@ SparcTargetLowering::LowerCall_32(TargetLowering::CallLoweringInfo &CLI,
         // if it is double-word aligned, just store.
         if (Offset % 8 == 0) {
           SDValue StackPtr = DAG.getRegister(SP::O6, MVT::i32);
-          SDValue PtrOff = DAG.getIntPtrConstant(Offset);
+          SDValue PtrOff = DAG.getIntPtrConstant(Offset, dl);
           PtrOff = DAG.getNode(ISD::ADD, dl, MVT::i32, StackPtr, PtrOff);
           MemOpChains.push_back(DAG.getStore(Chain, dl, Arg, PtrOff,
                                              MachinePointerInfo(),
@@ -811,7 +813,7 @@ SparcTargetLowering::LowerCall_32(TargetLowering::CallLoweringInfo &CLI,
                                MachinePointerInfo(), false, false, false, 0);
       // Increment the pointer to the other half.
       StackPtr = DAG.getNode(ISD::ADD, dl, StackPtr.getValueType(), StackPtr,
-                             DAG.getIntPtrConstant(4));
+                             DAG.getIntPtrConstant(4, dl));
       // Load the low part.
       SDValue Lo = DAG.getLoad(MVT::i32, dl, Store, StackPtr,
                                MachinePointerInfo(), false, false, false, 0);
@@ -826,7 +828,7 @@ SparcTargetLowering::LowerCall_32(TargetLowering::CallLoweringInfo &CLI,
           // Store the low part in stack.
           unsigned Offset = NextVA.getLocMemOffset() + StackOffset;
           SDValue StackPtr = DAG.getRegister(SP::O6, MVT::i32);
-          SDValue PtrOff = DAG.getIntPtrConstant(Offset);
+          SDValue PtrOff = DAG.getIntPtrConstant(Offset, dl);
           PtrOff = DAG.getNode(ISD::ADD, dl, MVT::i32, StackPtr, PtrOff);
           MemOpChains.push_back(DAG.getStore(Chain, dl, Lo, PtrOff,
                                              MachinePointerInfo(),
@@ -836,13 +838,13 @@ SparcTargetLowering::LowerCall_32(TargetLowering::CallLoweringInfo &CLI,
         unsigned Offset = VA.getLocMemOffset() + StackOffset;
         // Store the high part.
         SDValue StackPtr = DAG.getRegister(SP::O6, MVT::i32);
-        SDValue PtrOff = DAG.getIntPtrConstant(Offset);
+        SDValue PtrOff = DAG.getIntPtrConstant(Offset, dl);
         PtrOff = DAG.getNode(ISD::ADD, dl, MVT::i32, StackPtr, PtrOff);
         MemOpChains.push_back(DAG.getStore(Chain, dl, Hi, PtrOff,
                                            MachinePointerInfo(),
                                            false, false, 0));
         // Store the low part.
-        PtrOff = DAG.getIntPtrConstant(Offset+4);
+        PtrOff = DAG.getIntPtrConstant(Offset + 4, dl);
         PtrOff = DAG.getNode(ISD::ADD, dl, MVT::i32, StackPtr, PtrOff);
         MemOpChains.push_back(DAG.getStore(Chain, dl, Lo, PtrOff,
                                            MachinePointerInfo(),
@@ -867,7 +869,8 @@ SparcTargetLowering::LowerCall_32(TargetLowering::CallLoweringInfo &CLI,
 
     // Create a store off the stack pointer for this argument.
     SDValue StackPtr = DAG.getRegister(SP::O6, MVT::i32);
-    SDValue PtrOff = DAG.getIntPtrConstant(VA.getLocMemOffset()+StackOffset);
+    SDValue PtrOff = DAG.getIntPtrConstant(VA.getLocMemOffset() + StackOffset,
+                                           dl);
     PtrOff = DAG.getNode(ISD::ADD, dl, MVT::i32, StackPtr, PtrOff);
     MemOpChains.push_back(DAG.getStore(Chain, dl, Arg, PtrOff,
                                        MachinePointerInfo(),
@@ -909,7 +912,7 @@ SparcTargetLowering::LowerCall_32(TargetLowering::CallLoweringInfo &CLI,
   Ops.push_back(Chain);
   Ops.push_back(Callee);
   if (hasStructRetAttr)
-    Ops.push_back(DAG.getTargetConstant(SRetArgSize, MVT::i32));
+    Ops.push_back(DAG.getTargetConstant(SRetArgSize, dl, MVT::i32));
   for (unsigned i = 0, e = RegsToPass.size(); i != e; ++i)
     Ops.push_back(DAG.getRegister(toCallerWindow(RegsToPass[i].first),
                                   RegsToPass[i].second.getValueType()));
@@ -929,8 +932,8 @@ SparcTargetLowering::LowerCall_32(TargetLowering::CallLoweringInfo &CLI,
   Chain = DAG.getNode(SPISD::CALL, dl, NodeTys, Ops);
   InFlag = Chain.getValue(1);
 
-  Chain = DAG.getCALLSEQ_END(Chain, DAG.getIntPtrConstant(ArgsSize, true),
-                             DAG.getIntPtrConstant(0, true), InFlag, dl);
+  Chain = DAG.getCALLSEQ_END(Chain, DAG.getIntPtrConstant(ArgsSize, dl, true),
+                             DAG.getIntPtrConstant(0, dl, true), InFlag, dl);
   InFlag = Chain.getValue(1);
 
   // Assign locations to each value returned by this call.
@@ -992,7 +995,7 @@ SparcTargetLowering::getSRetArgSize(SelectionDAG &DAG, SDValue Callee) const
 
   PointerType *Ty = cast<PointerType>(CalleeFn->arg_begin()->getType());
   Type *ElementTy = Ty->getElementType();
-  return getDataLayout()->getTypeAllocSize(ElementTy);
+  return DAG.getDataLayout().getTypeAllocSize(ElementTy);
 }
 
 
@@ -1056,6 +1059,7 @@ SparcTargetLowering::LowerCall_64(TargetLowering::CallLoweringInfo &CLI,
   SelectionDAG &DAG = CLI.DAG;
   SDLoc DL = CLI.DL;
   SDValue Chain = CLI.Chain;
+  auto PtrVT = getPointerTy(DAG.getDataLayout());
 
   // Sparc target does not yet support tail call optimization.
   CLI.IsTailCall = false;
@@ -1082,7 +1086,7 @@ SparcTargetLowering::LowerCall_64(TargetLowering::CallLoweringInfo &CLI,
   // Adjust the stack pointer to make room for the arguments.
   // FIXME: Use hasReservedCallFrame to avoid %sp adjustments around all calls
   // with more than 6 arguments.
-  Chain = DAG.getCALLSEQ_START(Chain, DAG.getIntPtrConstant(ArgsSize, true),
+  Chain = DAG.getCALLSEQ_START(Chain, DAG.getIntPtrConstant(ArgsSize, DL, true),
                                DL);
 
   // Collect the set of registers to pass to the function and their values.
@@ -1129,13 +1133,11 @@ SparcTargetLowering::LowerCall_64(TargetLowering::CallLoweringInfo &CLI,
         // Store and reload into the interger register reg and reg+1.
         unsigned Offset = 8 * (VA.getLocReg() - SP::I0);
         unsigned StackOffset = Offset + Subtarget->getStackPointerBias() + 128;
-        SDValue StackPtr = DAG.getRegister(SP::O6, getPointerTy());
-        SDValue HiPtrOff = DAG.getIntPtrConstant(StackOffset);
-        HiPtrOff         = DAG.getNode(ISD::ADD, DL, getPointerTy(), StackPtr,
-                                       HiPtrOff);
-        SDValue LoPtrOff = DAG.getIntPtrConstant(StackOffset + 8);
-        LoPtrOff         = DAG.getNode(ISD::ADD, DL, getPointerTy(), StackPtr,
-                                       LoPtrOff);
+        SDValue StackPtr = DAG.getRegister(SP::O6, PtrVT);
+        SDValue HiPtrOff = DAG.getIntPtrConstant(StackOffset, DL);
+        HiPtrOff = DAG.getNode(ISD::ADD, DL, PtrVT, StackPtr, HiPtrOff);
+        SDValue LoPtrOff = DAG.getIntPtrConstant(StackOffset + 8, DL);
+        LoPtrOff = DAG.getNode(ISD::ADD, DL, PtrVT, StackPtr, LoPtrOff);
 
         // Store to %sp+BIAS+128+Offset
         SDValue Store = DAG.getStore(Chain, DL, Arg, HiPtrOff,
@@ -1159,7 +1161,7 @@ SparcTargetLowering::LowerCall_64(TargetLowering::CallLoweringInfo &CLI,
       // passed in the high bits of the register.
       if (VA.getValVT() == MVT::i32 && VA.needsCustom()) {
         Arg = DAG.getNode(ISD::SHL, DL, MVT::i64, Arg,
-                          DAG.getConstant(32, MVT::i32));
+                          DAG.getConstant(32, DL, MVT::i32));
 
         // The next value may go in the low bits of the same register.
         // Handle both at once.
@@ -1179,13 +1181,13 @@ SparcTargetLowering::LowerCall_64(TargetLowering::CallLoweringInfo &CLI,
     assert(VA.isMemLoc());
 
     // Create a store off the stack pointer for this argument.
-    SDValue StackPtr = DAG.getRegister(SP::O6, getPointerTy());
+    SDValue StackPtr = DAG.getRegister(SP::O6, PtrVT);
     // The argument area starts at %fp+BIAS+128 in the callee frame,
     // %sp+BIAS+128 in ours.
     SDValue PtrOff = DAG.getIntPtrConstant(VA.getLocMemOffset() +
                                            Subtarget->getStackPointerBias() +
-                                           128);
-    PtrOff = DAG.getNode(ISD::ADD, DL, getPointerTy(), StackPtr, PtrOff);
+                                           128, DL);
+    PtrOff = DAG.getNode(ISD::ADD, DL, PtrVT, StackPtr, PtrOff);
     MemOpChains.push_back(DAG.getStore(Chain, DL, Arg, PtrOff,
                                        MachinePointerInfo(),
                                        false, false, 0));
@@ -1214,10 +1216,9 @@ SparcTargetLowering::LowerCall_64(TargetLowering::CallLoweringInfo &CLI,
   unsigned TF = ((getTargetMachine().getRelocationModel() == Reloc::PIC_)
                  ? SparcMCExpr::VK_Sparc_WPLT30 : 0);
   if (GlobalAddressSDNode *G = dyn_cast<GlobalAddressSDNode>(Callee))
-    Callee = DAG.getTargetGlobalAddress(G->getGlobal(), DL, getPointerTy(), 0,
-                                        TF);
+    Callee = DAG.getTargetGlobalAddress(G->getGlobal(), DL, PtrVT, 0, TF);
   else if (ExternalSymbolSDNode *E = dyn_cast<ExternalSymbolSDNode>(Callee))
-    Callee = DAG.getTargetExternalSymbol(E->getSymbol(), getPointerTy(), TF);
+    Callee = DAG.getTargetExternalSymbol(E->getSymbol(), PtrVT, TF);
 
   // Build the operands for the call instruction itself.
   SmallVector<SDValue, 8> Ops;
@@ -1247,8 +1248,8 @@ SparcTargetLowering::LowerCall_64(TargetLowering::CallLoweringInfo &CLI,
   InGlue = Chain.getValue(1);
 
   // Revert the stack pointer immediately after the call.
-  Chain = DAG.getCALLSEQ_END(Chain, DAG.getIntPtrConstant(ArgsSize, true),
-                             DAG.getIntPtrConstant(0, true), InGlue, DL);
+  Chain = DAG.getCALLSEQ_END(Chain, DAG.getIntPtrConstant(ArgsSize, DL, true),
+                             DAG.getIntPtrConstant(0, DL, true), InGlue, DL);
   InGlue = Chain.getValue(1);
 
   // Now extract the return values. This is more or less the same as
@@ -1289,7 +1290,7 @@ SparcTargetLowering::LowerCall_64(TargetLowering::CallLoweringInfo &CLI,
     // Get the high bits for i32 struct elements.
     if (VA.getValVT() == MVT::i32 && VA.needsCustom())
       RV = DAG.getNode(ISD::SRL, DL, VA.getLocVT(), RV,
-                       DAG.getConstant(32, MVT::i32));
+                       DAG.getConstant(32, DL, MVT::i32));
 
     // The callee promoted the return value, so insert an Assert?ext SDNode so
     // we won't promote the value again in this function.
@@ -1369,6 +1370,8 @@ static SPCC::CondCodes FPCondCCodeToFCC(ISD::CondCode CC) {
 SparcTargetLowering::SparcTargetLowering(TargetMachine &TM,
                                          const SparcSubtarget &STI)
     : TargetLowering(TM), Subtarget(&STI) {
+  MVT PtrVT = MVT::getIntegerVT(8 * TM.getPointerSize());
+
   // Set up the register classes.
   addRegisterClass(MVT::i32, &SP::IntRegsRegClass);
   addRegisterClass(MVT::f32, &SP::FPRegsRegClass);
@@ -1393,10 +1396,10 @@ SparcTargetLowering::SparcTargetLowering(TargetMachine &TM,
   setTruncStoreAction(MVT::f128, MVT::f64, Expand);
 
   // Custom legalize GlobalAddress nodes into LO/HI parts.
-  setOperationAction(ISD::GlobalAddress, getPointerTy(), Custom);
-  setOperationAction(ISD::GlobalTLSAddress, getPointerTy(), Custom);
-  setOperationAction(ISD::ConstantPool, getPointerTy(), Custom);
-  setOperationAction(ISD::BlockAddress, getPointerTy(), Custom);
+  setOperationAction(ISD::GlobalAddress, PtrVT, Custom);
+  setOperationAction(ISD::GlobalTLSAddress, PtrVT, Custom);
+  setOperationAction(ISD::ConstantPool, PtrVT, Custom);
+  setOperationAction(ISD::BlockAddress, PtrVT, Custom);
 
   // Sparc doesn't have sext_inreg, replace them with shl/sra
   setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::i16, Expand);
@@ -1676,8 +1679,8 @@ SparcTargetLowering::SparcTargetLowering(TargetMachine &TM,
 }
 
 const char *SparcTargetLowering::getTargetNodeName(unsigned Opcode) const {
-  switch (Opcode) {
-  default: return nullptr;
+  switch ((SPISD::NodeType)Opcode) {
+  case SPISD::FIRST_NUMBER: break;
   case SPISD::CMPICC:     return "SPISD::CMPICC";
   case SPISD::CMPFCC:     return "SPISD::CMPFCC";
   case SPISD::BRICC:      return "SPISD::BRICC";
@@ -1700,9 +1703,11 @@ const char *SparcTargetLowering::getTargetNodeName(unsigned Opcode) const {
   case SPISD::TLS_LD:     return "SPISD::TLS_LD";
   case SPISD::TLS_CALL:   return "SPISD::TLS_CALL";
   }
+  return nullptr;
 }
 
-EVT SparcTargetLowering::getSetCCResultType(LLVMContext &, EVT VT) const {
+EVT SparcTargetLowering::getSetCCResultType(const DataLayout &, LLVMContext &,
+                                            EVT VT) const {
   if (!VT.isVector())
     return MVT::i32;
   return VT.changeVectorElementTypeToInteger();
@@ -1802,7 +1807,7 @@ SDValue SparcTargetLowering::makeHiLoPair(SDValue Op,
 // or ExternalSymbol SDNode.
 SDValue SparcTargetLowering::makeAddress(SDValue Op, SelectionDAG &DAG) const {
   SDLoc DL(Op);
-  EVT VT = getPointerTy();
+  EVT VT = getPointerTy(DAG.getDataLayout());
 
   // Handle PIC mode first.
   if (getTargetMachine().getRelocationModel() == Reloc::PIC_) {
@@ -1831,7 +1836,7 @@ SDValue SparcTargetLowering::makeAddress(SDValue Op, SelectionDAG &DAG) const {
     // abs44.
     SDValue H44 = makeHiLoPair(Op, SparcMCExpr::VK_Sparc_H44,
                                SparcMCExpr::VK_Sparc_M44, DAG);
-    H44 = DAG.getNode(ISD::SHL, DL, VT, H44, DAG.getConstant(12, MVT::i32));
+    H44 = DAG.getNode(ISD::SHL, DL, VT, H44, DAG.getConstant(12, DL, MVT::i32));
     SDValue L44 = withTargetFlags(Op, SparcMCExpr::VK_Sparc_L44, DAG);
     L44 = DAG.getNode(SPISD::Lo, DL, VT, L44);
     return DAG.getNode(ISD::ADD, DL, VT, H44, L44);
@@ -1840,7 +1845,7 @@ SDValue SparcTargetLowering::makeAddress(SDValue Op, SelectionDAG &DAG) const {
     // abs64.
     SDValue Hi = makeHiLoPair(Op, SparcMCExpr::VK_Sparc_HH,
                               SparcMCExpr::VK_Sparc_HM, DAG);
-    Hi = DAG.getNode(ISD::SHL, DL, VT, Hi, DAG.getConstant(32, MVT::i32));
+    Hi = DAG.getNode(ISD::SHL, DL, VT, Hi, DAG.getConstant(32, DL, MVT::i32));
     SDValue Lo = makeHiLoPair(Op, SparcMCExpr::VK_Sparc_HI,
                               SparcMCExpr::VK_Sparc_LO, DAG);
     return DAG.getNode(ISD::ADD, DL, VT, Hi, Lo);
@@ -1867,9 +1872,12 @@ SDValue SparcTargetLowering::LowerGlobalTLSAddress(SDValue Op,
                                                    SelectionDAG &DAG) const {
 
   GlobalAddressSDNode *GA = cast<GlobalAddressSDNode>(Op);
+  if (DAG.getTarget().Options.EmulatedTLS)
+    return LowerToTLSEmulatedModel(GA, DAG);
+
   SDLoc DL(GA);
   const GlobalValue *GV = GA->getGlobal();
-  EVT PtrVT = getPointerTy();
+  EVT PtrVT = getPointerTy(DAG.getDataLayout());
 
   TLSModel::Model model = getTargetMachine().getTLSModel(GV);
 
@@ -1895,7 +1903,7 @@ SDValue SparcTargetLowering::LowerGlobalTLSAddress(SDValue Op,
     SDValue Chain = DAG.getEntryNode();
     SDValue InFlag;
 
-    Chain = DAG.getCALLSEQ_START(Chain, DAG.getIntPtrConstant(1, true), DL);
+    Chain = DAG.getCALLSEQ_START(Chain, DAG.getIntPtrConstant(1, DL, true), DL);
     Chain = DAG.getCopyToReg(Chain, DL, SP::O0, Argument, InFlag);
     InFlag = Chain.getValue(1);
     SDValue Callee = DAG.getTargetExternalSymbol("__tls_get_addr", PtrVT);
@@ -1914,8 +1922,8 @@ SDValue SparcTargetLowering::LowerGlobalTLSAddress(SDValue Op,
     Ops.push_back(InFlag);
     Chain = DAG.getNode(SPISD::TLS_CALL, DL, NodeTys, Ops);
     InFlag = Chain.getValue(1);
-    Chain = DAG.getCALLSEQ_END(Chain, DAG.getIntPtrConstant(1, true),
-                               DAG.getIntPtrConstant(0, true), InFlag, DL);
+    Chain = DAG.getCALLSEQ_END(Chain, DAG.getIntPtrConstant(1, DL, true),
+                               DAG.getIntPtrConstant(0, DL, true), InFlag, DL);
     InFlag = Chain.getValue(1);
     SDValue Ret = DAG.getCopyFromReg(Chain, DL, SP::O0, PtrVT, InFlag);
 
@@ -1981,7 +1989,7 @@ SparcTargetLowering::LowerF128_LibCallArg(SDValue Chain, ArgListTy &Args,
   if (ArgTy->isFP128Ty()) {
     // Create a stack object and pass the pointer to the library function.
     int FI = MFI->CreateStackObject(16, 8, false);
-    SDValue FIPtr = DAG.getFrameIndex(FI, getPointerTy());
+    SDValue FIPtr = DAG.getFrameIndex(FI, getPointerTy(DAG.getDataLayout()));
     Chain = DAG.getStore(Chain,
                          DL,
                          Entry.Node,
@@ -2006,8 +2014,9 @@ SparcTargetLowering::LowerF128Op(SDValue Op, SelectionDAG &DAG,
   ArgListTy Args;
 
   MachineFrameInfo *MFI = DAG.getMachineFunction().getFrameInfo();
+  auto PtrVT = getPointerTy(DAG.getDataLayout());
 
-  SDValue Callee = DAG.getExternalSymbol(LibFuncName, getPointerTy());
+  SDValue Callee = DAG.getExternalSymbol(LibFuncName, PtrVT);
   Type *RetTy = Op.getValueType().getTypeForEVT(*DAG.getContext());
   Type *RetTyABI = RetTy;
   SDValue Chain = DAG.getEntryNode();
@@ -2017,7 +2026,7 @@ SparcTargetLowering::LowerF128Op(SDValue Op, SelectionDAG &DAG,
     // Create a Stack Object to receive the return value of type f128.
     ArgListEntry Entry;
     int RetFI = MFI->CreateStackObject(16, 8, false);
-    RetPtr = DAG.getFrameIndex(RetFI, getPointerTy());
+    RetPtr = DAG.getFrameIndex(RetFI, PtrVT);
     Entry.Node = RetPtr;
     Entry.Ty   = PointerType::getUnqual(RetTy);
     if (!Subtarget->is64Bit())
@@ -2080,7 +2089,8 @@ SparcTargetLowering::LowerF128Compare(SDValue LHS, SDValue RHS,
   case SPCC::FCC_UE : LibCall = is64Bit? "_Qp_cmp" : "_Q_cmp"; break;
   }
 
-  SDValue Callee = DAG.getExternalSymbol(LibCall, getPointerTy());
+  auto PtrVT = getPointerTy(DAG.getDataLayout());
+  SDValue Callee = DAG.getExternalSymbol(LibCall, PtrVT);
   Type *RetTy = Type::getInt32Ty(*DAG.getContext());
   ArgListTy Args;
   SDValue Chain = DAG.getEntryNode();
@@ -2098,54 +2108,54 @@ SparcTargetLowering::LowerF128Compare(SDValue LHS, SDValue RHS,
 
   switch(SPCC) {
   default: {
-    SDValue RHS = DAG.getTargetConstant(0, Result.getValueType());
+    SDValue RHS = DAG.getTargetConstant(0, DL, Result.getValueType());
     SPCC = SPCC::ICC_NE;
     return DAG.getNode(SPISD::CMPICC, DL, MVT::Glue, Result, RHS);
   }
   case SPCC::FCC_UL : {
-    SDValue Mask   = DAG.getTargetConstant(1, Result.getValueType());
+    SDValue Mask   = DAG.getTargetConstant(1, DL, Result.getValueType());
     Result = DAG.getNode(ISD::AND, DL, Result.getValueType(), Result, Mask);
-    SDValue RHS    = DAG.getTargetConstant(0, Result.getValueType());
+    SDValue RHS    = DAG.getTargetConstant(0, DL, Result.getValueType());
     SPCC = SPCC::ICC_NE;
     return DAG.getNode(SPISD::CMPICC, DL, MVT::Glue, Result, RHS);
   }
   case SPCC::FCC_ULE: {
-    SDValue RHS = DAG.getTargetConstant(2, Result.getValueType());
+    SDValue RHS = DAG.getTargetConstant(2, DL, Result.getValueType());
     SPCC = SPCC::ICC_NE;
     return DAG.getNode(SPISD::CMPICC, DL, MVT::Glue, Result, RHS);
   }
   case SPCC::FCC_UG :  {
-    SDValue RHS = DAG.getTargetConstant(1, Result.getValueType());
+    SDValue RHS = DAG.getTargetConstant(1, DL, Result.getValueType());
     SPCC = SPCC::ICC_G;
     return DAG.getNode(SPISD::CMPICC, DL, MVT::Glue, Result, RHS);
   }
   case SPCC::FCC_UGE: {
-    SDValue RHS = DAG.getTargetConstant(1, Result.getValueType());
+    SDValue RHS = DAG.getTargetConstant(1, DL, Result.getValueType());
     SPCC = SPCC::ICC_NE;
     return DAG.getNode(SPISD::CMPICC, DL, MVT::Glue, Result, RHS);
   }
 
   case SPCC::FCC_U  :  {
-    SDValue RHS = DAG.getTargetConstant(3, Result.getValueType());
+    SDValue RHS = DAG.getTargetConstant(3, DL, Result.getValueType());
     SPCC = SPCC::ICC_E;
     return DAG.getNode(SPISD::CMPICC, DL, MVT::Glue, Result, RHS);
   }
   case SPCC::FCC_O  :  {
-    SDValue RHS = DAG.getTargetConstant(3, Result.getValueType());
+    SDValue RHS = DAG.getTargetConstant(3, DL, Result.getValueType());
     SPCC = SPCC::ICC_NE;
     return DAG.getNode(SPISD::CMPICC, DL, MVT::Glue, Result, RHS);
   }
   case SPCC::FCC_LG :  {
-    SDValue Mask   = DAG.getTargetConstant(3, Result.getValueType());
+    SDValue Mask   = DAG.getTargetConstant(3, DL, Result.getValueType());
     Result = DAG.getNode(ISD::AND, DL, Result.getValueType(), Result, Mask);
-    SDValue RHS    = DAG.getTargetConstant(0, Result.getValueType());
+    SDValue RHS    = DAG.getTargetConstant(0, DL, Result.getValueType());
     SPCC = SPCC::ICC_NE;
     return DAG.getNode(SPISD::CMPICC, DL, MVT::Glue, Result, RHS);
   }
   case SPCC::FCC_UE : {
-    SDValue Mask   = DAG.getTargetConstant(3, Result.getValueType());
+    SDValue Mask   = DAG.getTargetConstant(3, DL, Result.getValueType());
     Result = DAG.getNode(ISD::AND, DL, Result.getValueType(), Result, Mask);
-    SDValue RHS    = DAG.getTargetConstant(0, Result.getValueType());
+    SDValue RHS    = DAG.getTargetConstant(0, DL, Result.getValueType());
     SPCC = SPCC::ICC_E;
     return DAG.getNode(SPISD::CMPICC, DL, MVT::Glue, Result, RHS);
   }
@@ -2317,7 +2327,7 @@ static SDValue LowerBR_CC(SDValue Op, SelectionDAG &DAG,
     }
   }
   return DAG.getNode(Opc, dl, MVT::Other, Chain, Dest,
-                     DAG.getConstant(SPCC, MVT::i32), CompareFlag);
+                     DAG.getConstant(SPCC, dl, MVT::i32), CompareFlag);
 }
 
 static SDValue LowerSELECT_CC(SDValue Op, SelectionDAG &DAG,
@@ -2353,13 +2363,14 @@ static SDValue LowerSELECT_CC(SDValue Op, SelectionDAG &DAG,
     }
   }
   return DAG.getNode(Opc, dl, TrueVal.getValueType(), TrueVal, FalseVal,
-                     DAG.getConstant(SPCC, MVT::i32), CompareFlag);
+                     DAG.getConstant(SPCC, dl, MVT::i32), CompareFlag);
 }
 
 static SDValue LowerVASTART(SDValue Op, SelectionDAG &DAG,
                             const SparcTargetLowering &TLI) {
   MachineFunction &MF = DAG.getMachineFunction();
   SparcMachineFunctionInfo *FuncInfo = MF.getInfo<SparcMachineFunctionInfo>();
+  auto PtrVT = TLI.getPointerTy(DAG.getDataLayout());
 
   // Need frame address to find the address of VarArgsFrameIndex.
   MF.getFrameInfo()->setFrameAddressIsTaken(true);
@@ -2368,9 +2379,8 @@ static SDValue LowerVASTART(SDValue Op, SelectionDAG &DAG,
   // memory location argument.
   SDLoc DL(Op);
   SDValue Offset =
-    DAG.getNode(ISD::ADD, DL, TLI.getPointerTy(),
-                DAG.getRegister(SP::I6, TLI.getPointerTy()),
-                DAG.getIntPtrConstant(FuncInfo->getVarArgsFrameOffset()));
+      DAG.getNode(ISD::ADD, DL, PtrVT, DAG.getRegister(SP::I6, PtrVT),
+                  DAG.getIntPtrConstant(FuncInfo->getVarArgsFrameOffset(), DL));
   const Value *SV = cast<SrcValueSDNode>(Op.getOperand(2))->getValue();
   return DAG.getStore(Op.getOperand(0), DL, Offset, Op.getOperand(1),
                       MachinePointerInfo(SV), false, false, 0);
@@ -2388,7 +2398,8 @@ static SDValue LowerVAARG(SDValue Op, SelectionDAG &DAG) {
                                MachinePointerInfo(SV), false, false, false, 0);
   // Increment the pointer, VAList, to the next vaarg.
   SDValue NextPtr = DAG.getNode(ISD::ADD, DL, PtrVT, VAList,
-                                DAG.getIntPtrConstant(VT.getSizeInBits()/8));
+                                DAG.getIntPtrConstant(VT.getSizeInBits()/8,
+                                                      DL));
   // Store the incremented VAList to the legalized pointer.
   InChain = DAG.getStore(VAList.getValue(1), DL, NextPtr,
                          VAListPtr, MachinePointerInfo(SV), false, false, 0);
@@ -2417,7 +2428,7 @@ static SDValue LowerDYNAMIC_STACKALLOC(SDValue Op, SelectionDAG &DAG,
   regSpillArea += Subtarget->getStackPointerBias();
 
   SDValue NewVal = DAG.getNode(ISD::ADD, dl, VT, NewSP,
-                               DAG.getConstant(regSpillArea, VT));
+                               DAG.getConstant(regSpillArea, dl, VT));
   SDValue Ops[2] = { NewVal, Chain };
   return DAG.getMergeValues(Ops, dl);
 }
@@ -2446,7 +2457,7 @@ static SDValue getFRAMEADDR(uint64_t depth, SDValue Op, SelectionDAG &DAG,
     FrameAddr = DAG.getCopyFromReg(DAG.getEntryNode(), dl, FrameReg, VT);
     if (Subtarget->is64Bit())
       FrameAddr = DAG.getNode(ISD::ADD, dl, VT, FrameAddr,
-                              DAG.getIntPtrConstant(stackBias));
+                              DAG.getIntPtrConstant(stackBias, dl));
     return FrameAddr;
   }
 
@@ -2458,13 +2469,13 @@ static SDValue getFRAMEADDR(uint64_t depth, SDValue Op, SelectionDAG &DAG,
 
   while (depth--) {
     SDValue Ptr = DAG.getNode(ISD::ADD, dl, VT, FrameAddr,
-                              DAG.getIntPtrConstant(Offset));
+                              DAG.getIntPtrConstant(Offset, dl));
     FrameAddr = DAG.getLoad(VT, dl, Chain, Ptr, MachinePointerInfo(),
                             false, false, false, 0);
   }
   if (Subtarget->is64Bit())
     FrameAddr = DAG.getNode(ISD::ADD, dl, VT, FrameAddr,
-                            DAG.getIntPtrConstant(stackBias));
+                            DAG.getIntPtrConstant(stackBias, dl));
   return FrameAddr;
 }
 
@@ -2494,8 +2505,8 @@ static SDValue LowerRETURNADDR(SDValue Op, SelectionDAG &DAG,
 
   SDValue RetAddr;
   if (depth == 0) {
-    unsigned RetReg = MF.addLiveIn(SP::I7,
-                                   TLI.getRegClassFor(TLI.getPointerTy()));
+    auto PtrVT = TLI.getPointerTy(DAG.getDataLayout());
+    unsigned RetReg = MF.addLiveIn(SP::I7, TLI.getRegClassFor(PtrVT));
     RetAddr = DAG.getCopyFromReg(DAG.getEntryNode(), dl, RetReg, VT);
     return RetAddr;
   }
@@ -2507,7 +2518,7 @@ static SDValue LowerRETURNADDR(SDValue Op, SelectionDAG &DAG,
   SDValue Ptr = DAG.getNode(ISD::ADD,
                             dl, VT,
                             FrameAddr,
-                            DAG.getIntPtrConstant(Offset));
+                            DAG.getIntPtrConstant(Offset, dl));
   RetAddr = DAG.getLoad(VT, dl, DAG.getEntryNode(), Ptr,
                         MachinePointerInfo(), false, false, false, 0);
 
@@ -2563,7 +2574,7 @@ static SDValue LowerF128Load(SDValue Op, SelectionDAG &DAG)
   EVT addrVT = LdNode->getBasePtr().getValueType();
   SDValue LoPtr = DAG.getNode(ISD::ADD, dl, addrVT,
                               LdNode->getBasePtr(),
-                              DAG.getConstant(8, addrVT));
+                              DAG.getConstant(8, dl, addrVT));
   SDValue Lo64 = DAG.getLoad(MVT::f64,
                              dl,
                              LdNode->getChain(),
@@ -2571,8 +2582,8 @@ static SDValue LowerF128Load(SDValue Op, SelectionDAG &DAG)
                              LdNode->getPointerInfo(),
                              false, false, false, alignment);
 
-  SDValue SubRegEven = DAG.getTargetConstant(SP::sub_even64, MVT::i32);
-  SDValue SubRegOdd  = DAG.getTargetConstant(SP::sub_odd64, MVT::i32);
+  SDValue SubRegEven = DAG.getTargetConstant(SP::sub_even64, dl, MVT::i32);
+  SDValue SubRegOdd  = DAG.getTargetConstant(SP::sub_odd64, dl, MVT::i32);
 
   SDNode *InFP128 = DAG.getMachineNode(TargetOpcode::IMPLICIT_DEF,
                                        dl, MVT::f128);
@@ -2599,8 +2610,8 @@ static SDValue LowerF128Store(SDValue Op, SelectionDAG &DAG) {
   StoreSDNode *StNode = dyn_cast<StoreSDNode>(Op.getNode());
   assert(StNode && StNode->getOffset().getOpcode() == ISD::UNDEF
          && "Unexpected node type");
-  SDValue SubRegEven = DAG.getTargetConstant(SP::sub_even64, MVT::i32);
-  SDValue SubRegOdd  = DAG.getTargetConstant(SP::sub_odd64, MVT::i32);
+  SDValue SubRegEven = DAG.getTargetConstant(SP::sub_even64, dl, MVT::i32);
+  SDValue SubRegOdd  = DAG.getTargetConstant(SP::sub_odd64, dl, MVT::i32);
 
   SDNode *Hi64 = DAG.getMachineNode(TargetOpcode::EXTRACT_SUBREG,
                                     dl,
@@ -2627,7 +2638,7 @@ static SDValue LowerF128Store(SDValue Op, SelectionDAG &DAG) {
   EVT addrVT = StNode->getBasePtr().getValueType();
   SDValue LoPtr = DAG.getNode(ISD::ADD, dl, addrVT,
                               StNode->getBasePtr(),
-                              DAG.getConstant(8, addrVT));
+                              DAG.getConstant(8, dl, addrVT));
   OutChains[1] = DAG.getStore(StNode->getChain(),
                              dl,
                              SDValue(Lo64, 0),
@@ -2678,13 +2689,13 @@ static SDValue LowerADDC_ADDE_SUBC_SUBE(SDValue Op, SelectionDAG &DAG) {
   SDValue Src1 = Op.getOperand(0);
   SDValue Src1Lo = DAG.getNode(ISD::TRUNCATE, dl, MVT::i32, Src1);
   SDValue Src1Hi = DAG.getNode(ISD::SRL, dl, MVT::i64, Src1,
-                               DAG.getConstant(32, MVT::i64));
+                               DAG.getConstant(32, dl, MVT::i64));
   Src1Hi = DAG.getNode(ISD::TRUNCATE, dl, MVT::i32, Src1Hi);
 
   SDValue Src2 = Op.getOperand(1);
   SDValue Src2Lo = DAG.getNode(ISD::TRUNCATE, dl, MVT::i32, Src2);
   SDValue Src2Hi = DAG.getNode(ISD::SRL, dl, MVT::i64, Src2,
-                               DAG.getConstant(32, MVT::i64));
+                               DAG.getConstant(32, dl, MVT::i64));
   Src2Hi = DAG.getNode(ISD::TRUNCATE, dl, MVT::i32, Src2Hi);
 
 
@@ -2711,7 +2722,7 @@ static SDValue LowerADDC_ADDE_SUBC_SUBE(SDValue Op, SelectionDAG &DAG) {
   Lo = DAG.getNode(ISD::ZERO_EXTEND, dl, MVT::i64, Lo);
   Hi = DAG.getNode(ISD::ZERO_EXTEND, dl, MVT::i64, Hi);
   Hi = DAG.getNode(ISD::SHL, dl, MVT::i64, Hi,
-                   DAG.getConstant(32, MVT::i64));
+                   DAG.getConstant(32, dl, MVT::i64));
 
   SDValue Dst = DAG.getNode(ISD::OR, dl, MVT::i64, Hi, Lo);
   SDValue Ops[2] = { Dst, Carry };
@@ -2735,7 +2746,7 @@ static SDValue LowerUMULO_SMULO(SDValue Op, SelectionDAG &DAG,
   if (LHS.getValueType() != VT)
     return Op;
 
-  SDValue ShiftAmt = DAG.getConstant(63, VT);
+  SDValue ShiftAmt = DAG.getConstant(63, dl, VT);
 
   SDValue RHS = Op.getOperand(1);
   SDValue HiLHS = DAG.getNode(ISD::SRA, dl, VT, LHS, ShiftAmt);
@@ -2746,14 +2757,14 @@ static SDValue LowerUMULO_SMULO(SDValue Op, SelectionDAG &DAG,
                                       RTLIB::MUL_I128, WideVT,
                                       Args, 4, isSigned, dl).first;
   SDValue BottomHalf = DAG.getNode(ISD::EXTRACT_ELEMENT, dl, VT,
-                                   MulResult, DAG.getIntPtrConstant(0));
+                                   MulResult, DAG.getIntPtrConstant(0, dl));
   SDValue TopHalf = DAG.getNode(ISD::EXTRACT_ELEMENT, dl, VT,
-                                MulResult, DAG.getIntPtrConstant(1));
+                                MulResult, DAG.getIntPtrConstant(1, dl));
   if (isSigned) {
     SDValue Tmp1 = DAG.getNode(ISD::SRA, dl, VT, BottomHalf, ShiftAmt);
     TopHalf = DAG.getSetCC(dl, MVT::i32, TopHalf, Tmp1, ISD::SETNE);
   } else {
-    TopHalf = DAG.getSetCC(dl, MVT::i32, TopHalf, DAG.getConstant(0, VT),
+    TopHalf = DAG.getSetCC(dl, MVT::i32, TopHalf, DAG.getConstant(0, dl, VT),
                            ISD::SETNE);
   }
   // MulResult is a node with an illegal type. Because such things are not
@@ -3062,7 +3073,7 @@ SparcTargetLowering::expandAtomicRMW(MachineInstr *MI,
 /// getConstraintType - Given a constraint letter, return the type of
 /// constraint it is for this target.
 SparcTargetLowering::ConstraintType
-SparcTargetLowering::getConstraintType(const std::string &Constraint) const {
+SparcTargetLowering::getConstraintType(StringRef Constraint) const {
   if (Constraint.size() == 1) {
     switch (Constraint[0]) {
     default:  break;
@@ -3119,7 +3130,8 @@ LowerAsmOperandForConstraint(SDValue Op,
   case 'I':
     if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op)) {
       if (isInt<13>(C->getSExtValue())) {
-        Result = DAG.getTargetConstant(C->getSExtValue(), Op.getValueType());
+        Result = DAG.getTargetConstant(C->getSExtValue(), SDLoc(Op),
+                                       Op.getValueType());
         break;
       }
       return;
@@ -3135,7 +3147,7 @@ LowerAsmOperandForConstraint(SDValue Op,
 
 std::pair<unsigned, const TargetRegisterClass *>
 SparcTargetLowering::getRegForInlineAsmConstraint(const TargetRegisterInfo *TRI,
-                                                  const std::string &Constraint,
+                                                  StringRef Constraint,
                                                   MVT VT) const {
   if (Constraint.size() == 1) {
     switch (Constraint[0]) {
diff --git a/lib/Target/Sparc/SparcISelLowering.h b/lib/Target/Sparc/SparcISelLowering.h
index 87153261e1e..bbc91a493c9 100644
--- a/lib/Target/Sparc/SparcISelLowering.h
+++ b/lib/Target/Sparc/SparcISelLowering.h
@@ -22,7 +22,7 @@ namespace llvm {
   class SparcSubtarget;
 
   namespace SPISD {
-    enum {
+    enum NodeType : unsigned {
       FIRST_NUMBER = ISD::BUILTIN_OP_END,
       CMPICC,      // Compare two GPR operands, set icc+xcc.
       CMPFCC,      // Compare two FP operands, set fcc.
@@ -72,7 +72,7 @@ namespace llvm {
 
     const char *getTargetNodeName(unsigned Opcode) const override;
 
-    ConstraintType getConstraintType(const std::string &Constraint) const override;
+    ConstraintType getConstraintType(StringRef Constraint) const override;
     ConstraintWeight
     getSingleConstraintMatchWeight(AsmOperandInfo &info,
                                    const char *constraint) const override;
@@ -82,14 +82,16 @@ namespace llvm {
                                       SelectionDAG &DAG) const override;
     std::pair<unsigned, const TargetRegisterClass *>
     getRegForInlineAsmConstraint(const TargetRegisterInfo *TRI,
-                                 const std::string &Constraint,
-                                 MVT VT) const override;
+                                 StringRef Constraint, MVT VT) const override;
 
     bool isOffsetFoldingLegal(const GlobalAddressSDNode *GA) const override;
-    MVT getScalarShiftAmountTy(EVT LHSTy) const override { return MVT::i32; }
+    MVT getScalarShiftAmountTy(const DataLayout &, EVT) const override {
+      return MVT::i32;
+    }
 
     /// getSetCCResultType - Return the ISD::SETCC ValueType
-    EVT getSetCCResultType(LLVMContext &Context, EVT VT) const override;
+    EVT getSetCCResultType(const DataLayout &DL, LLVMContext &Context,
+                           EVT VT) const override;
 
     SDValue
       LowerFormalArguments(SDValue Chain,
diff --git a/lib/Target/Sparc/SparcInstr64Bit.td b/lib/Target/Sparc/SparcInstr64Bit.td
index 54d824064fb..419e8ccb102 100644
--- a/lib/Target/Sparc/SparcInstr64Bit.td
+++ b/lib/Target/Sparc/SparcInstr64Bit.td
@@ -63,7 +63,7 @@ defm SRAX : F3_S<"srax", 0b100111, 1, sra, i64, I64Regs>;
 
 // The ALU instructions want their simm13 operands as i32 immediates.
 def as_i32imm : SDNodeXForm<imm, [{
-  return CurDAG->getTargetConstant(N->getSExtValue(), MVT::i32);
+  return CurDAG->getTargetConstant(N->getSExtValue(), SDLoc(N), MVT::i32);
 }]>;
 def : Pat<(i64 simm13:$val), (ORri (i64 G0), (as_i32imm $val))>;
 def : Pat<(i64 SETHIimm:$val), (SETHIi (HI22 $val))>;
@@ -83,11 +83,12 @@ def nimm33 : PatLeaf<(imm), [{
 // Bits 10-31 inverted. Same as assembler's %hix.
 def HIX22 : SDNodeXForm<imm, [{
   uint64_t Val = (~N->getZExtValue() >> 10) & ((1u << 22) - 1);
-  return CurDAG->getTargetConstant(Val, MVT::i32);
+  return CurDAG->getTargetConstant(Val, SDLoc(N), MVT::i32);
 }]>;
 // Bits 0-9 with ones in bits 10-31. Same as assembler's %lox.
 def LOX10 : SDNodeXForm<imm, [{
-  return CurDAG->getTargetConstant(~(~N->getZExtValue() & 0x3ff), MVT::i32);
+  return CurDAG->getTargetConstant(~(~N->getZExtValue() & 0x3ff), SDLoc(N),
+                                   MVT::i32);
 }]>;
 def : Pat<(i64 nimm33:$val), (XORri (SETHIi (HIX22 $val)), (LOX10 $val))>,
       Requires<[Is64Bit]>;
@@ -121,12 +122,12 @@ def : Pat<(i64 nimm33:$val), (XORri (SETHIi (HIX22 $val)), (LOX10 $val))>,
 // Bits 42-63, same as assembler's %hh.
 def HH22 : SDNodeXForm<imm, [{
   uint64_t Val = (N->getZExtValue() >> 42) & ((1u << 22) - 1);
-  return CurDAG->getTargetConstant(Val, MVT::i32);
+  return CurDAG->getTargetConstant(Val, SDLoc(N), MVT::i32);
 }]>;
 // Bits 32-41, same as assembler's %hm.
 def HM10 : SDNodeXForm<imm, [{
   uint64_t Val = (N->getZExtValue() >> 32) & ((1u << 10) - 1);
-  return CurDAG->getTargetConstant(Val, MVT::i32);
+  return CurDAG->getTargetConstant(Val, SDLoc(N), MVT::i32);
 }]>;
 def : Pat<(i64 imm:$val),
           (ORrr (SLLXri (ORri (SETHIi (HH22 $val)), (HM10 $val)), (i32 32)),
@@ -485,8 +486,8 @@ def SETHIXi : F2_1<0b100,
 }
 
 // ATOMICS.
-let Predicates = [Is64Bit], Constraints = "$swap = $rd" in {
-  def CASXrr: F3_1_asi<3, 0b111110, 0b10000000,
+let Predicates = [Is64Bit], Constraints = "$swap = $rd", asi = 0b10000000 in {
+  def CASXrr: F3_1_asi<3, 0b111110,
                 (outs I64Regs:$rd), (ins I64Regs:$rs1, I64Regs:$rs2,
                                      I64Regs:$swap),
                  "casx [$rs1], $rs2, $rd",
diff --git a/lib/Target/Sparc/SparcInstrAliases.td b/lib/Target/Sparc/SparcInstrAliases.td
index d36f67b9420..25cc652dbd9 100644
--- a/lib/Target/Sparc/SparcInstrAliases.td
+++ b/lib/Target/Sparc/SparcInstrAliases.td
@@ -245,9 +245,12 @@ multiclass fp_cond_alias<string cond, int condVal> {
 }
 
 defm : int_cond_alias<"a",    0b1000>;
+defm : int_cond_alias<"",     0b1000>; // same as a; gnu asm, not in manual
 defm : int_cond_alias<"n",    0b0000>;
 defm : int_cond_alias<"ne",   0b1001>;
+defm : int_cond_alias<"nz",   0b1001>; // same as ne
 defm : int_cond_alias<"e",    0b0001>;
+defm : int_cond_alias<"z",    0b0001>; // same as e
 defm : int_cond_alias<"g",    0b1010>;
 defm : int_cond_alias<"le",   0b0010>;
 defm : int_cond_alias<"ge",   0b1011>;
@@ -255,13 +258,16 @@ defm : int_cond_alias<"l",    0b0011>;
 defm : int_cond_alias<"gu",   0b1100>;
 defm : int_cond_alias<"leu",  0b0100>;
 defm : int_cond_alias<"cc",   0b1101>;
+defm : int_cond_alias<"geu",  0b1101>; // same as cc
 defm : int_cond_alias<"cs",   0b0101>;
+defm : int_cond_alias<"lu",   0b0101>; // same as cs
 defm : int_cond_alias<"pos",  0b1110>;
 defm : int_cond_alias<"neg",  0b0110>;
 defm : int_cond_alias<"vc",   0b1111>;
 defm : int_cond_alias<"vs",   0b0111>;
 
 defm : fp_cond_alias<"a",     0b0000>;
+defm : fp_cond_alias<"",      0b0000>; // same as a; gnu asm, not in manual
 defm : fp_cond_alias<"n",     0b1000>;
 defm : fp_cond_alias<"u",     0b0111>;
 defm : fp_cond_alias<"g",     0b0110>;
@@ -270,7 +276,9 @@ defm : fp_cond_alias<"l",     0b0100>;
 defm : fp_cond_alias<"ul",    0b0011>;
 defm : fp_cond_alias<"lg",    0b0010>;
 defm : fp_cond_alias<"ne",    0b0001>;
+defm : fp_cond_alias<"nz",    0b0001>; // same as ne
 defm : fp_cond_alias<"e",     0b1001>;
+defm : fp_cond_alias<"z",     0b1001>; // same as e
 defm : fp_cond_alias<"ue",    0b1010>;
 defm : fp_cond_alias<"ge",    0b1011>;
 defm : fp_cond_alias<"uge",   0b1100>;
@@ -278,7 +286,16 @@ defm : fp_cond_alias<"le",    0b1101>;
 defm : fp_cond_alias<"ule",   0b1110>;
 defm : fp_cond_alias<"o",     0b1111>;
 
-// Instruction aliases for JMPL.
+// Section A.3 Synthetic Instructions
+
+// Most are marked as Emit=0, so that they are not used for disassembly. This is
+// an aesthetic issue, but the chosen policy is to typically prefer using the
+// non-alias form, except for the most obvious and clarifying aliases: cmp, jmp,
+// call, tst, ret, retl.
+
+// Note: cmp is handled in SparcInstrInfo.
+//       jmp/call/ret/retl have special case handling for output in
+//       SparcInstPrinter.cpp
 
 // jmp addr -> jmpl addr, %g0
 def : InstAlias<"jmp $addr", (JMPLrr G0, MEMrr:$addr), 0>;
@@ -288,21 +305,130 @@ def : InstAlias<"jmp $addr", (JMPLri G0, MEMri:$addr), 0>;
 def : InstAlias<"call $addr", (JMPLrr O7, MEMrr:$addr), 0>;
 def : InstAlias<"call $addr", (JMPLri O7, MEMri:$addr), 0>;
 
-// retl -> RETL 8
-def : InstAlias<"retl", (RETL 8)>;
+// tst reg -> orcc %g0, reg, %g0
+def : InstAlias<"tst $rs2", (ORCCrr G0, IntRegs:$rs2, G0)>;
 
-// ret -> RET 8
+// ret -> jmpl %i7+8, %g0 (aka RET 8)
 def : InstAlias<"ret", (RET 8)>;
 
-// mov reg, rd -> or %g0, reg, rd
-def : InstAlias<"mov $rs2, $rd", (ORrr IntRegs:$rd, G0, IntRegs:$rs2)>;
-
-// mov simm13, rd -> or %g0, simm13, rd
-def : InstAlias<"mov $simm13, $rd", (ORri IntRegs:$rd, G0, i32imm:$simm13)>;
+// retl -> jmpl %o7+8, %g0 (aka RETL 8)
+def : InstAlias<"retl", (RETL 8)>;
 
 // restore -> restore %g0, %g0, %g0
 def : InstAlias<"restore", (RESTORErr G0, G0, G0)>;
 
+// save -> restore %g0, %g0, %g0
+def : InstAlias<"save", (SAVErr G0, G0, G0)>;
+
+// set value, rd
+// (turns into a sequence of sethi+or, depending on the value)
+// def : InstAlias<"set $val, $rd", (ORri IntRegs:$rd, (SETHIi (HI22 imm:$val)), (LO10 imm:$val))>;
+def SET : AsmPseudoInst<(outs IntRegs:$rd), (ins i32imm:$val), "set $val, $rd">;
+
+// not rd -> xnor rd, %g0, rd
+def : InstAlias<"not $rd", (XNORrr IntRegs:$rd, IntRegs:$rd, G0), 0>;
+
+// not reg, rd -> xnor reg, %g0, rd
+def : InstAlias<"not $rs1, $rd", (XNORrr IntRegs:$rd, IntRegs:$rs1, G0), 0>;
+
+// neg rd -> sub %g0, rd, rd
+def : InstAlias<"neg $rd", (SUBrr IntRegs:$rd, G0, IntRegs:$rd), 0>;
+
+// neg reg, rd -> sub %g0, reg, rd
+def : InstAlias<"neg $rs2, $rd", (SUBrr IntRegs:$rd, G0, IntRegs:$rs2), 0>;
+
+// inc rd -> add rd, 1, rd
+def : InstAlias<"inc $rd", (ADDri IntRegs:$rd, IntRegs:$rd, 1), 0>;
+
+// inc simm13, rd -> add rd, simm13, rd
+def : InstAlias<"inc $simm13, $rd", (ADDri IntRegs:$rd, IntRegs:$rd, i32imm:$simm13), 0>;
+
+// inccc rd -> addcc rd, 1, rd
+def : InstAlias<"inccc $rd", (ADDCCri IntRegs:$rd, IntRegs:$rd, 1), 0>;
+
+// inccc simm13, rd -> addcc rd, simm13, rd
+def : InstAlias<"inccc $simm13, $rd", (ADDCCri IntRegs:$rd, IntRegs:$rd, i32imm:$simm13), 0>;
+
+// dec rd -> sub rd, 1, rd
+def : InstAlias<"dec $rd", (SUBri IntRegs:$rd, IntRegs:$rd, 1), 0>;
+
+// dec simm13, rd -> sub rd, simm13, rd
+def : InstAlias<"dec $simm13, $rd", (SUBri IntRegs:$rd, IntRegs:$rd, i32imm:$simm13), 0>;
+
+// deccc rd -> subcc rd, 1, rd
+def : InstAlias<"deccc $rd", (SUBCCri IntRegs:$rd, IntRegs:$rd, 1), 0>;
+
+// deccc simm13, rd -> subcc rd, simm13, rd
+def : InstAlias<"deccc $simm13, $rd", (SUBCCri IntRegs:$rd, IntRegs:$rd, i32imm:$simm13), 0>;
+
+// btst reg_or_imm, reg -> andcc reg,reg_or_imm,%g0
+def : InstAlias<"btst $rs2, $rs1", (ANDCCrr G0, IntRegs:$rs1, IntRegs:$rs2), 0>;
+def : InstAlias<"btst $simm13, $rs1", (ANDCCri G0, IntRegs:$rs1, i32imm:$simm13), 0>;
+
+// bset reg_or_imm, rd -> or rd,reg_or_imm,rd
+def : InstAlias<"bset $rs2, $rd", (ORrr IntRegs:$rd, IntRegs:$rd, IntRegs:$rs2), 0>;
+def : InstAlias<"bset $simm13, $rd", (ORri IntRegs:$rd, IntRegs:$rd, i32imm:$simm13), 0>;
+
+// bclr reg_or_imm, rd -> andn rd,reg_or_imm,rd
+def : InstAlias<"bclr $rs2, $rd", (ANDNrr IntRegs:$rd, IntRegs:$rd, IntRegs:$rs2), 0>;
+def : InstAlias<"bclr $simm13, $rd", (ANDNri IntRegs:$rd, IntRegs:$rd, i32imm:$simm13), 0>;
+
+// btog reg_or_imm, rd -> xor rd,reg_or_imm,rd
+def : InstAlias<"btog $rs2, $rd", (XORrr IntRegs:$rd, IntRegs:$rd, IntRegs:$rs2), 0>;
+def : InstAlias<"btog $simm13, $rd", (XORri IntRegs:$rd, IntRegs:$rd, i32imm:$simm13), 0>;
+
+
+// clr rd -> or %g0, %g0, rd
+def : InstAlias<"clr $rd", (ORrr IntRegs:$rd, G0, G0), 0>;
+
+// clr{b,h,} [addr] -> st{b,h,} %g0, [addr]
+def : InstAlias<"clrb [$addr]", (STBrr MEMrr:$addr, G0), 0>;
+def : InstAlias<"clrb [$addr]", (STBri MEMri:$addr, G0), 0>;
+def : InstAlias<"clrh [$addr]", (STHrr MEMrr:$addr, G0), 0>;
+def : InstAlias<"clrh [$addr]", (STHri MEMri:$addr, G0), 0>;
+def : InstAlias<"clr [$addr]", (STrr MEMrr:$addr, G0), 0>;
+def : InstAlias<"clr [$addr]", (STri MEMri:$addr, G0), 0>;
+
+
+// mov reg_or_imm, rd -> or %g0, reg_or_imm, rd
+def : InstAlias<"mov $rs2, $rd", (ORrr IntRegs:$rd, G0, IntRegs:$rs2)>;
+def : InstAlias<"mov $simm13, $rd", (ORri IntRegs:$rd, G0, i32imm:$simm13)>;
+
+// mov specialreg, rd -> rd specialreg, rd
+def : InstAlias<"mov $asr, $rd", (RDASR IntRegs:$rd, ASRRegs:$asr), 0>;
+def : InstAlias<"mov %psr, $rd", (RDPSR IntRegs:$rd), 0>;
+def : InstAlias<"mov %wim, $rd", (RDWIM IntRegs:$rd), 0>;
+def : InstAlias<"mov %tbr, $rd", (RDTBR IntRegs:$rd), 0>;
+
+// mov reg_or_imm, specialreg -> wr %g0, reg_or_imm, specialreg
+def : InstAlias<"mov $rs2, $asr", (WRASRrr ASRRegs:$asr, G0, IntRegs:$rs2), 0>;
+def : InstAlias<"mov $simm13, $asr", (WRASRri ASRRegs:$asr, G0, i32imm:$simm13), 0>;
+def : InstAlias<"mov $rs2, %psr", (WRPSRrr G0, IntRegs:$rs2), 0>;
+def : InstAlias<"mov $simm13, %psr", (WRPSRri G0, i32imm:$simm13), 0>;
+def : InstAlias<"mov $rs2, %wim", (WRWIMrr G0, IntRegs:$rs2), 0>;
+def : InstAlias<"mov $simm13, %wim", (WRWIMri G0, i32imm:$simm13), 0>;
+def : InstAlias<"mov $rs2, %tbr", (WRTBRrr G0, IntRegs:$rs2), 0>;
+def : InstAlias<"mov $simm13, %tbr", (WRTBRri G0, i32imm:$simm13), 0>;
+
+// End of Section A.3
+
+// wr reg_or_imm, specialreg -> wr %g0, reg_or_imm, specialreg
+// (aka: omit the first arg when it's g0. This is not in the manual, but is
+// supported by gnu and solaris as)
+def : InstAlias<"wr $rs2, $asr", (WRASRrr ASRRegs:$asr, G0, IntRegs:$rs2), 0>;
+def : InstAlias<"wr $simm13, $asr", (WRASRri ASRRegs:$asr, G0, i32imm:$simm13), 0>;
+def : InstAlias<"wr $rs2, %psr", (WRPSRrr G0, IntRegs:$rs2), 0>;
+def : InstAlias<"wr $simm13, %psr", (WRPSRri G0, i32imm:$simm13), 0>;
+def : InstAlias<"wr $rs2, %wim", (WRWIMrr G0, IntRegs:$rs2), 0>;
+def : InstAlias<"wr $simm13, %wim", (WRWIMri G0, i32imm:$simm13), 0>;
+def : InstAlias<"wr $rs2, %tbr", (WRTBRrr G0, IntRegs:$rs2), 0>;
+def : InstAlias<"wr $simm13, %tbr", (WRTBRri G0, i32imm:$simm13), 0>;
+
+
+// flush -> flush %g0
+def : InstAlias<"flush", (FLUSH), 0>;
+
+
 def : MnemonicAlias<"return", "rett">, Requires<[HasV9]>;
 
 def : MnemonicAlias<"addc", "addx">, Requires<[HasV9]>;
@@ -323,3 +449,4 @@ def : InstAlias<"fcmped $rs1, $rs2", (V9FCMPED FCC0, DFPRegs:$rs1,
 def : InstAlias<"fcmpeq $rs1, $rs2", (V9FCMPEQ FCC0, QFPRegs:$rs1,
                                                      QFPRegs:$rs2)>,
                 Requires<[HasHardQuad]>;
+
diff --git a/lib/Target/Sparc/SparcInstrFormats.td b/lib/Target/Sparc/SparcInstrFormats.td
index 3b5e2389932..74ccf551e47 100644
--- a/lib/Target/Sparc/SparcInstrFormats.td
+++ b/lib/Target/Sparc/SparcInstrFormats.td
@@ -113,8 +113,9 @@ class F3<dag outs, dag ins, string asmstr, list<dag> pattern>
 
 // Specific F3 classes: SparcV8 manual, page 44
 //
-class F3_1_asi<bits<2> opVal, bits<6> op3val, bits<8> asi, dag outs, dag ins,
+class F3_1_asi<bits<2> opVal, bits<6> op3val, dag outs, dag ins,
            string asmstr, list<dag> pattern> : F3<outs, ins, asmstr, pattern> {
+  bits<8> asi;
   bits<5> rs2;
 
   let op         = opVal;
@@ -126,8 +127,10 @@ class F3_1_asi<bits<2> opVal, bits<6> op3val, bits<8> asi, dag outs, dag ins,
 }
 
 class F3_1<bits<2> opVal, bits<6> op3val, dag outs, dag ins, string asmstr,
-       list<dag> pattern> : F3_1_asi<opVal, op3val, 0, outs, ins,
-                                                     asmstr, pattern>;
+       list<dag> pattern> : F3_1_asi<opVal, op3val, outs, ins,
+                                                     asmstr, pattern> {
+  let asi = 0;
+}
 
 class F3_2<bits<2> opVal, bits<6> op3val, dag outs, dag ins,
            string asmstr, list<dag> pattern> : F3<outs, ins, asmstr, pattern> {
@@ -328,3 +331,11 @@ class TRAPSPri<bits<6> op3Val, dag outs, dag ins, string asmstr,
    let Inst{10-8} = 0;
    let Inst{7-0}  = imm;
 }
+
+// Pseudo-instructions for alternate assembly syntax (never used by codegen).
+// These are aliases that require C++ handling to convert to the target
+// instruction, while InstAliases can be handled directly by tblgen.
+class AsmPseudoInst<dag outs, dag ins, string asm>
+  : InstSP<outs, ins, asm, []> {
+  let isPseudo = 1;
+}
diff --git a/lib/Target/Sparc/SparcInstrInfo.cpp b/lib/Target/Sparc/SparcInstrInfo.cpp
index 4b70f1619b1..6167c532db8 100644
--- a/lib/Target/Sparc/SparcInstrInfo.cpp
+++ b/lib/Target/Sparc/SparcInstrInfo.cpp
@@ -229,7 +229,7 @@ bool SparcInstrInfo::AnalyzeBranch(MachineBasicBlock &MBB,
 unsigned
 SparcInstrInfo::InsertBranch(MachineBasicBlock &MBB,MachineBasicBlock *TBB,
                              MachineBasicBlock *FBB,
-                             const SmallVectorImpl<MachineOperand> &Cond,
+                             ArrayRef<MachineOperand> Cond,
                              DebugLoc DL) const {
   assert(TBB && "InsertBranch must not be told to insert a fallthrough");
   assert((Cond.size() == 1 || Cond.size() == 0) &&
@@ -324,6 +324,15 @@ void SparcInstrInfo::copyPhysReg(MachineBasicBlock &MBB,
       numSubRegs = 4;
       movOpc     = SP::FMOVS;
     }
+  } else if (SP::ASRRegsRegClass.contains(DestReg) &&
+             SP::IntRegsRegClass.contains(SrcReg)) {
+    BuildMI(MBB, I, DL, get(SP::WRASRrr), DestReg)
+        .addReg(SP::G0)
+        .addReg(SrcReg, getKillRegState(KillSrc));
+  } else if (SP::IntRegsRegClass.contains(DestReg) &&
+             SP::ASRRegsRegClass.contains(SrcReg)) {
+    BuildMI(MBB, I, DL, get(SP::RDASR), DestReg)
+        .addReg(SrcReg, getKillRegState(KillSrc));
   } else
     llvm_unreachable("Impossible reg-to-reg copy");
 
diff --git a/lib/Target/Sparc/SparcInstrInfo.h b/lib/Target/Sparc/SparcInstrInfo.h
index 6e084189807..15673f134d8 100644
--- a/lib/Target/Sparc/SparcInstrInfo.h
+++ b/lib/Target/Sparc/SparcInstrInfo.h
@@ -73,8 +73,7 @@ class SparcInstrInfo : public SparcGenInstrInfo {
   unsigned RemoveBranch(MachineBasicBlock &MBB) const override;
 
   unsigned InsertBranch(MachineBasicBlock &MBB, MachineBasicBlock *TBB,
-                        MachineBasicBlock *FBB,
-                        const SmallVectorImpl<MachineOperand> &Cond,
+                        MachineBasicBlock *FBB, ArrayRef<MachineOperand> Cond,
                         DebugLoc DL) const override;
 
   void copyPhysReg(MachineBasicBlock &MBB,
diff --git a/lib/Target/Sparc/SparcInstrInfo.td b/lib/Target/Sparc/SparcInstrInfo.td
index 17daecac9e0..3b9e048ea8b 100644
--- a/lib/Target/Sparc/SparcInstrInfo.td
+++ b/lib/Target/Sparc/SparcInstrInfo.td
@@ -64,13 +64,14 @@ def simm11  : PatLeaf<(imm), [{ return isInt<11>(N->getSExtValue()); }]>;
 def simm13  : PatLeaf<(imm), [{ return isInt<13>(N->getSExtValue()); }]>;
 
 def LO10 : SDNodeXForm<imm, [{
-  return CurDAG->getTargetConstant((unsigned)N->getZExtValue() & 1023,
+  return CurDAG->getTargetConstant((unsigned)N->getZExtValue() & 1023, SDLoc(N),
                                    MVT::i32);
 }]>;
 
 def HI22 : SDNodeXForm<imm, [{
   // Transformation function: shift the immediate value down into the low bits.
-  return CurDAG->getTargetConstant((unsigned)N->getZExtValue() >> 10, MVT::i32);
+  return CurDAG->getTargetConstant((unsigned)N->getZExtValue() >> 10, SDLoc(N),
+                                   MVT::i32);
 }]>;
 
 def SETHIimm : PatLeaf<(imm), [{
@@ -282,6 +283,17 @@ multiclass Load<string OpcStr, bits<6> Op3Val, SDPatternOperator OpNode,
                  [(set Ty:$dst, (OpNode ADDRri:$addr))]>;
 }
 
+// LoadA multiclass - As above, but also define alternate address space variant
+multiclass LoadA<string OpcStr, bits<6> Op3Val, bits<6> LoadAOp3Val,
+                 SDPatternOperator OpNode, RegisterClass RC, ValueType Ty> :
+             Load<OpcStr, Op3Val, OpNode, RC, Ty> {
+  // TODO: The LD*Arr instructions are currently asm only; hooking up
+  // CodeGen's address spaces to use these is a future task.
+  def Arr  : F3_1_asi<3, LoadAOp3Val, (outs RC:$dst), (ins MEMrr:$addr, i8imm:$asi),
+                !strconcat(OpcStr, "a [$addr] $asi, $dst"),
+                []>;
+}
+
 // Store multiclass - Define both Reg+Reg/Reg+Imm patterns in one shot.
 multiclass Store<string OpcStr, bits<6> Op3Val, SDPatternOperator OpNode,
            RegisterClass RC, ValueType Ty> {
@@ -295,6 +307,16 @@ multiclass Store<string OpcStr, bits<6> Op3Val, SDPatternOperator OpNode,
                  [(OpNode Ty:$rd, ADDRri:$addr)]>;
 }
 
+multiclass StoreA<string OpcStr, bits<6> Op3Val, bits<6> StoreAOp3Val,
+                  SDPatternOperator OpNode, RegisterClass RC, ValueType Ty> :
+             Store<OpcStr, Op3Val, OpNode, RC, Ty> {
+  // TODO: The ST*Arr instructions are currently asm only; hooking up
+  // CodeGen's address spaces to use these is a future task.
+  def Arr  : F3_1_asi<3, StoreAOp3Val, (outs), (ins MEMrr:$addr, RC:$rd, i8imm:$asi),
+                  !strconcat(OpcStr, "a $rd, [$addr] $asi"),
+                  []>;
+}
+
 //===----------------------------------------------------------------------===//
 // Instructions
 //===----------------------------------------------------------------------===//
@@ -331,13 +353,6 @@ let hasSideEffects = 1, mayStore = 1 in {
                    [(flushw)]>;
 }
 
-let isBarrier = 1, isTerminator = 1, rd = 0b01000, rs1 = 0, simm13 = 5 in
-  def TA5 : F3_2<0b10, 0b111010, (outs), (ins), "ta 5", [(trap)]>;
-
-let rd = 0 in
-  def UNIMP : F2_1<0b000, (outs), (ins i32imm:$imm22),
-                  "unimp $imm22", []>;
-
 // SELECT_CC_* - Used to implement the SELECT_CC DAG operation.  Expanded after
 // instruction selection into a branch sequence.  This has to handle all
 // permutations of selection between i32/f32/f64 on ICC and FCC.
@@ -384,43 +399,13 @@ let usesCustomInserter = 1, Uses = [FCC0] in {
             [(set f128:$dst, (SPselectfcc f128:$T, f128:$F, imm:$Cond))]>;
 }
 
-// JMPL Instruction.
-let isTerminator = 1, hasDelaySlot = 1, isBarrier = 1,
-    DecoderMethod = "DecodeJMPL" in {
-  def JMPLrr: F3_1<2, 0b111000, (outs IntRegs:$dst), (ins MEMrr:$addr),
-                  "jmpl $addr, $dst", []>;
-  def JMPLri: F3_2<2, 0b111000, (outs IntRegs:$dst), (ins MEMri:$addr),
-                  "jmpl $addr, $dst", []>;
-}
-
-// Section A.3 - Synthetic Instructions, p. 85
-// special cases of JMPL:
-let isReturn = 1, isTerminator = 1, hasDelaySlot = 1, isBarrier = 1,
-    isCodeGenOnly = 1 in {
-  let rd = 0, rs1 = 15 in
-    def RETL: F3_2<2, 0b111000, (outs), (ins i32imm:$val),
-                   "jmp %o7+$val", [(retflag simm13:$val)]>;
-
-  let rd = 0, rs1 = 31 in
-    def RET: F3_2<2, 0b111000, (outs), (ins i32imm:$val),
-                  "jmp %i7+$val", []>;
-}
-
-let isReturn = 1, isTerminator = 1, hasDelaySlot = 1,
-     isBarrier = 1, rd = 0, DecoderMethod = "DecodeReturn" in {
-  def RETTrr : F3_1<2, 0b111001, (outs), (ins MEMrr:$addr),
-                       "rett $addr", []>;
-  def RETTri : F3_2<2, 0b111001, (outs), (ins MEMri:$addr),
-                       "rett $addr", []>;
-}
-
 // Section B.1 - Load Integer Instructions, p. 90
 let DecoderMethod = "DecodeLoadInt" in {
-  defm LDSB : Load<"ldsb", 0b001001, sextloadi8,  IntRegs, i32>;
-  defm LDSH : Load<"ldsh", 0b001010, sextloadi16, IntRegs, i32>;
-  defm LDUB : Load<"ldub", 0b000001, zextloadi8,  IntRegs, i32>;
-  defm LDUH : Load<"lduh", 0b000010, zextloadi16, IntRegs, i32>;
-  defm LD   : Load<"ld",   0b000000, load,        IntRegs, i32>;
+  defm LDSB : LoadA<"ldsb", 0b001001, 0b011001, sextloadi8,  IntRegs, i32>;
+  defm LDSH : LoadA<"ldsh", 0b001010, 0b011010, sextloadi16, IntRegs, i32>;
+  defm LDUB : LoadA<"ldub", 0b000001, 0b010001, zextloadi8,  IntRegs, i32>;
+  defm LDUH : LoadA<"lduh", 0b000010, 0b010010, zextloadi16, IntRegs, i32>;
+  defm LD   : LoadA<"ld",   0b000000, 0b010000, load,        IntRegs, i32>;
 }
 
 // Section B.2 - Load Floating-point Instructions, p. 92
@@ -434,9 +419,9 @@ let DecoderMethod = "DecodeLoadQFP" in
 
 // Section B.4 - Store Integer Instructions, p. 95
 let DecoderMethod = "DecodeStoreInt" in {
-  defm STB   : Store<"stb", 0b000101, truncstorei8,  IntRegs, i32>;
-  defm STH   : Store<"sth", 0b000110, truncstorei16, IntRegs, i32>;
-  defm ST    : Store<"st",  0b000100, store,         IntRegs, i32>;
+  defm STB   : StoreA<"stb", 0b000101, 0b010101, truncstorei8,  IntRegs, i32>;
+  defm STH   : StoreA<"sth", 0b000110, 0b010110, truncstorei16, IntRegs, i32>;
+  defm ST    : StoreA<"st",  0b000100, 0b010100, store,         IntRegs, i32>;
 }
 
 // Section B.5 - Store Floating-point Instructions, p. 97
@@ -448,6 +433,24 @@ let DecoderMethod = "DecodeStoreQFP" in
   defm STQF  : Store<"stq", 0b100110, store,         QFPRegs, f128>,
                Requires<[HasV9, HasHardQuad]>;
 
+// Section B.8 - SWAP Register with Memory Instruction
+// (Atomic swap)
+let Constraints = "$val = $dst", DecoderMethod = "DecodeSWAP" in {
+  def SWAPrr : F3_1<3, 0b001111,
+                 (outs IntRegs:$dst), (ins MEMrr:$addr, IntRegs:$val),
+                 "swap [$addr], $dst",
+                 [(set i32:$dst, (atomic_swap_32 ADDRrr:$addr, i32:$val))]>;
+  def SWAPri : F3_2<3, 0b001111,
+                 (outs IntRegs:$dst), (ins MEMri:$addr, IntRegs:$val),
+                 "swap [$addr], $dst",
+                 [(set i32:$dst, (atomic_swap_32 ADDRri:$addr, i32:$val))]>;
+  def SWAPArr : F3_1_asi<3, 0b011111,
+                 (outs IntRegs:$dst), (ins MEMrr:$addr, i8imm:$asi, IntRegs:$val),
+                 "swapa [$addr] $asi, $dst",
+                 [/*FIXME: pattern?*/]>;
+}
+
+
 // Section B.9 - SETHI Instruction, p. 104
 def SETHIi: F2_1<0b100,
                  (outs IntRegs:$rd), (ins i32imm:$imm22),
@@ -533,6 +536,7 @@ let Defs = [ICC] in
 let Uses = [ICC] in
   defm SUBC   : F3_12np <"subx", 0b001100>;
 
+// cmp (from Section A.3) is a specialized alias for subcc
 let Defs = [ICC], rd = 0 in {
   def CMPrr   : F3_1<2, 0b010100,
                      (outs), (ins IntRegs:$rs1, IntRegs:$rs2),
@@ -556,12 +560,12 @@ let Defs = [Y, ICC] in {
 }
 
 // Section B.19 - Divide Instructions, p. 115
-let Defs = [Y] in {
+let Uses = [Y], Defs = [Y] in {
   defm UDIV : F3_12np<"udiv", 0b001110>;
   defm SDIV : F3_12np<"sdiv", 0b001111>;
 }
 
-let Defs = [Y, ICC] in {
+let Uses = [Y], Defs = [Y, ICC] in {
   defm UDIVCC : F3_12np<"udivcc", 0b011110>;
   defm SDIVCC : F3_12np<"sdivcc", 0b011111>;
 }
@@ -703,21 +707,144 @@ let Uses = [O6],
   }
 }
 
+// Section B.25 - Jump and Link Instruction
+
+// JMPL Instruction.
+let isTerminator = 1, hasDelaySlot = 1, isBarrier = 1,
+    DecoderMethod = "DecodeJMPL" in {
+  def JMPLrr: F3_1<2, 0b111000, (outs IntRegs:$dst), (ins MEMrr:$addr),
+                  "jmpl $addr, $dst", []>;
+  def JMPLri: F3_2<2, 0b111000, (outs IntRegs:$dst), (ins MEMri:$addr),
+                  "jmpl $addr, $dst", []>;
+}
+
+// Section A.3 - Synthetic Instructions, p. 85
+// special cases of JMPL:
+let isReturn = 1, isTerminator = 1, hasDelaySlot = 1, isBarrier = 1,
+    isCodeGenOnly = 1 in {
+  let rd = 0, rs1 = 15 in
+    def RETL: F3_2<2, 0b111000, (outs), (ins i32imm:$val),
+                   "jmp %o7+$val", [(retflag simm13:$val)]>;
+
+  let rd = 0, rs1 = 31 in
+    def RET: F3_2<2, 0b111000, (outs), (ins i32imm:$val),
+                  "jmp %i7+$val", []>;
+}
+
+// Section B.26 - Return from Trap Instruction
+let isReturn = 1, isTerminator = 1, hasDelaySlot = 1,
+     isBarrier = 1, rd = 0, DecoderMethod = "DecodeReturn" in {
+  def RETTrr : F3_1<2, 0b111001, (outs), (ins MEMrr:$addr),
+                       "rett $addr", []>;
+  def RETTri : F3_2<2, 0b111001, (outs), (ins MEMri:$addr),
+                       "rett $addr", []>;
+}
+
+
+// Section B.27 - Trap on Integer Condition Codes Instruction
+multiclass TRAP<string regStr> {
+  def rr : TRAPSPrr<0b111010, (outs), (ins IntRegs:$rs1, IntRegs:$rs2,
+                                       CCOp:$cond),
+              !strconcat(!strconcat("t$cond ", regStr), ", $rs1 + $rs2"), []>;
+  def ri : TRAPSPri<0b111010, (outs), (ins IntRegs:$rs1, i32imm:$imm,
+                                      CCOp:$cond),
+              !strconcat(!strconcat("t$cond ", regStr), ", $rs1 + $imm"), []>;
+}
+
+let hasSideEffects = 1, Uses = [ICC], cc = 0b00 in
+  defm TICC : TRAP<"%icc">;
+
+let isBarrier = 1, isTerminator = 1, rd = 0b01000, rs1 = 0, simm13 = 5 in
+  def TA5 : F3_2<0b10, 0b111010, (outs), (ins), "ta 5", [(trap)]>;
+
 // Section B.28 - Read State Register Instructions
-let Uses = [Y], rs1 = 0, rs2 = 0 in
-  def RDY : F3_1<2, 0b101000,
-                 (outs IntRegs:$dst), (ins),
-                 "rd %y, $dst", []>;
+let rs2 = 0 in
+  def RDASR : F3_1<2, 0b101000,
+                 (outs IntRegs:$rd), (ins ASRRegs:$rs1),
+                 "rd $rs1, $rd", []>;
+
+// PSR, WIM, and TBR don't exist on the SparcV9, only the V8.
+let Predicates = [HasNoV9] in {
+  let rs2 = 0, rs1 = 0, Uses=[PSR] in
+    def RDPSR : F3_1<2, 0b101001,
+		     (outs IntRegs:$rd), (ins),
+		     "rd %psr, $rd", []>;
+
+  let rs2 = 0, rs1 = 0, Uses=[WIM] in
+    def RDWIM : F3_1<2, 0b101010,
+		     (outs IntRegs:$rd), (ins),
+		     "rd %wim, $rd", []>;
+
+  let rs2 = 0, rs1 = 0, Uses=[TBR] in
+    def RDTBR : F3_1<2, 0b101011,
+		     (outs IntRegs:$rd), (ins),
+		     "rd %tbr, $rd", []>;
+}
 
 // Section B.29 - Write State Register Instructions
-let Defs = [Y], rd = 0 in {
-  def WRYrr : F3_1<2, 0b110000,
-                   (outs), (ins IntRegs:$rs1, IntRegs:$rs2),
-                   "wr $rs1, $rs2, %y", []>;
-  def WRYri : F3_2<2, 0b110000,
-                   (outs), (ins IntRegs:$rs1, simm13Op:$simm13),
-                   "wr $rs1, $simm13, %y", []>;
+def WRASRrr : F3_1<2, 0b110000,
+                 (outs ASRRegs:$rd), (ins IntRegs:$rs1, IntRegs:$rs2),
+                 "wr $rs1, $rs2, $rd", []>;
+def WRASRri : F3_2<2, 0b110000,
+                 (outs ASRRegs:$rd), (ins IntRegs:$rs1, simm13Op:$simm13),
+                 "wr $rs1, $simm13, $rd", []>;
+
+// PSR, WIM, and TBR don't exist on the SparcV9, only the V8.
+let Predicates = [HasNoV9] in {
+  let Defs = [PSR], rd=0 in {
+    def WRPSRrr : F3_1<2, 0b110001,
+		     (outs), (ins IntRegs:$rs1, IntRegs:$rs2),
+		     "wr $rs1, $rs2, %psr", []>;
+    def WRPSRri : F3_2<2, 0b110001,
+		     (outs), (ins IntRegs:$rs1, simm13Op:$simm13),
+		     "wr $rs1, $simm13, %psr", []>;
+  }
+
+  let Defs = [WIM], rd=0 in {
+    def WRWIMrr : F3_1<2, 0b110010,
+		     (outs), (ins IntRegs:$rs1, IntRegs:$rs2),
+		     "wr $rs1, $rs2, %wim", []>;
+    def WRWIMri : F3_2<2, 0b110010,
+		     (outs), (ins IntRegs:$rs1, simm13Op:$simm13),
+		     "wr $rs1, $simm13, %wim", []>;
+  }
+
+  let Defs = [TBR], rd=0 in {
+    def WRTBRrr : F3_1<2, 0b110011,
+		     (outs), (ins IntRegs:$rs1, IntRegs:$rs2),
+		     "wr $rs1, $rs2, %tbr", []>;
+    def WRTBRri : F3_2<2, 0b110011,
+		     (outs), (ins IntRegs:$rs1, simm13Op:$simm13),
+		     "wr $rs1, $simm13, %tbr", []>;
+  }
 }
+
+// Section B.30 - STBAR Instruction
+let hasSideEffects = 1, rd = 0, rs1 = 0b01111, rs2 = 0 in
+  def STBAR : F3_1<2, 0b101000, (outs), (ins), "stbar", []>;
+
+
+// Section B.31 - Unimplmented Instruction
+let rd = 0 in
+  def UNIMP : F2_1<0b000, (outs), (ins i32imm:$imm22),
+                  "unimp $imm22", []>;
+
+// Section B.32 - Flush Instruction Memory
+let rd = 0 in {
+  def FLUSHrr : F3_1<2, 0b111011, (outs), (ins MEMrr:$addr),
+                       "flush $addr", []>;
+  def FLUSHri : F3_2<2, 0b111011, (outs), (ins MEMri:$addr),
+                       "flush $addr", []>;
+
+  // The no-arg FLUSH is only here for the benefit of the InstAlias
+  // "flush", which cannot seem to use FLUSHrr, due to the inability
+  // to construct a MEMrr with fixed G0 registers.
+  let rs1 = 0, rs2 = 0 in
+    def FLUSH   : F3_1<2, 0b111011, (outs), (ins), "flush %g0", []>;
+}
+
+// Section B.33 - Floating-point Operate (FPop) Instructions
+
 // Convert Integer to Floating-point Instructions, p. 141
 def FITOS : F3_3u<2, 0b110100, 0b011000100,
                  (outs FPRegs:$rd), (ins FPRegs:$rs2),
@@ -1099,27 +1226,18 @@ let rs1 = 0 in
 def : Pat<(ctpop i32:$src),
           (POPCrr (SRLri $src, 0))>;
 
-// Atomic swap.
-let hasSideEffects =1, rd = 0, rs1 = 0b01111, rs2 = 0 in
-  def STBAR : F3_1<2, 0b101000, (outs), (ins), "stbar", []>;
-
 let Predicates = [HasV9], hasSideEffects = 1, rd = 0, rs1 = 0b01111 in
  def MEMBARi : F3_2<2, 0b101000, (outs), (ins simm13Op:$simm13),
                     "membar $simm13", []>;
 
-let Constraints = "$val = $dst", DecoderMethod = "DecodeSWAP" in {
-  def SWAPrr : F3_1<3, 0b001111,
-                 (outs IntRegs:$dst), (ins MEMrr:$addr, IntRegs:$val),
-                 "swap [$addr], $dst",
-                 [(set i32:$dst, (atomic_swap_32 ADDRrr:$addr, i32:$val))]>;
-  def SWAPri : F3_2<3, 0b001111,
-                 (outs IntRegs:$dst), (ins MEMri:$addr, IntRegs:$val),
-                 "swap [$addr], $dst",
-                 [(set i32:$dst, (atomic_swap_32 ADDRri:$addr, i32:$val))]>;
-}
-
-let Predicates = [HasV9], Constraints = "$swap = $rd" in
-  def CASrr: F3_1_asi<3, 0b111100, 0b10000000,
+// TODO: Should add a CASArr variant. In fact, the CAS instruction,
+// unlike other instructions, only comes in a form which requires an
+// ASI be provided. The ASI value hardcoded here is ASI_PRIMARY, the
+// default unprivileged ASI for SparcV9.  (Also of note: some modern
+// SparcV8 implementations provide CASA as an extension, but require
+// the use of SparcV8's default ASI, 0xA ("User Data") instead.)
+let Predicates = [HasV9], Constraints = "$swap = $rd", asi = 0b10000000 in
+  def CASrr: F3_1_asi<3, 0b111100,
                 (outs IntRegs:$rd), (ins IntRegs:$rs1, IntRegs:$rs2,
                                      IntRegs:$swap),
                  "cas [$rs1], $rs2, $rd",
@@ -1136,18 +1254,6 @@ let hasSideEffects = 1 in {
 }
 }
 
-multiclass TRAP<string regStr> {
-  def rr : TRAPSPrr<0b111010, (outs), (ins IntRegs:$rs1, IntRegs:$rs2,
-                                       CCOp:$cond),
-              !strconcat(!strconcat("t$cond ", regStr), ", $rs1 + $rs2"), []>;
-  def ri : TRAPSPri<0b111010, (outs), (ins IntRegs:$rs1, i32imm:$imm,
-                                      CCOp:$cond),
-              !strconcat(!strconcat("t$cond ", regStr), ", $rs1 + $imm"), []>;
-}
-
-let hasSideEffects = 1, Uses = [ICC], cc = 0b00 in
-  defm TICC : TRAP<"%icc">;
-
 //===----------------------------------------------------------------------===//
 // Non-Instruction Patterns
 //===----------------------------------------------------------------------===//
diff --git a/lib/Target/Sparc/SparcMCInstLower.cpp b/lib/Target/Sparc/SparcMCInstLower.cpp
index 9e94d2c3140..b084d0021ba 100644
--- a/lib/Target/Sparc/SparcMCInstLower.cpp
+++ b/lib/Target/Sparc/SparcMCInstLower.cpp
@@ -59,11 +59,11 @@ static MCOperand LowerSymbolOperand(const MachineInstr *MI,
     break;
   }
 
-  const MCSymbolRefExpr *MCSym = MCSymbolRefExpr::Create(Symbol,
+  const MCSymbolRefExpr *MCSym = MCSymbolRefExpr::create(Symbol,
                                                          AP.OutContext);
-  const SparcMCExpr *expr = SparcMCExpr::Create(Kind, MCSym,
+  const SparcMCExpr *expr = SparcMCExpr::create(Kind, MCSym,
                                                 AP.OutContext);
-  return MCOperand::CreateExpr(expr);
+  return MCOperand::createExpr(expr);
 }
 
 static MCOperand LowerOperand(const MachineInstr *MI,
@@ -74,10 +74,10 @@ static MCOperand LowerOperand(const MachineInstr *MI,
   case MachineOperand::MO_Register:
     if (MO.isImplicit())
       break;
-    return MCOperand::CreateReg(MO.getReg());
+    return MCOperand::createReg(MO.getReg());
 
   case MachineOperand::MO_Immediate:
-    return MCOperand::CreateImm(MO.getImm());
+    return MCOperand::createImm(MO.getImm());
 
   case MachineOperand::MO_MachineBasicBlock:
   case MachineOperand::MO_GlobalAddress:
diff --git a/lib/Target/Sparc/SparcRegisterInfo.td b/lib/Target/Sparc/SparcRegisterInfo.td
index 2cadff1ef7b..db8a7e86962 100644
--- a/lib/Target/Sparc/SparcRegisterInfo.td
+++ b/lib/Target/Sparc/SparcRegisterInfo.td
@@ -56,6 +56,43 @@ foreach I = 0-3 in
 
 // Y register
 def Y : SparcCtrlReg<0, "Y">, DwarfRegNum<[64]>;
+// Ancillary state registers (implementation defined)
+def ASR1 : SparcCtrlReg<1, "ASR1">;
+def ASR2 : SparcCtrlReg<2, "ASR2">;
+def ASR3 : SparcCtrlReg<3, "ASR3">;
+def ASR4 : SparcCtrlReg<4, "ASR4">;
+def ASR5 : SparcCtrlReg<5, "ASR5">;
+def ASR6 : SparcCtrlReg<6, "ASR6">;
+def ASR7 : SparcCtrlReg<7, "ASR7">;
+def ASR8 : SparcCtrlReg<8, "ASR8">;
+def ASR9 : SparcCtrlReg<9, "ASR9">;
+def ASR10 : SparcCtrlReg<10, "ASR10">;
+def ASR11 : SparcCtrlReg<11, "ASR11">;
+def ASR12 : SparcCtrlReg<12, "ASR12">;
+def ASR13 : SparcCtrlReg<13, "ASR13">;
+def ASR14 : SparcCtrlReg<14, "ASR14">;
+def ASR15 : SparcCtrlReg<15, "ASR15">;
+def ASR16 : SparcCtrlReg<16, "ASR16">;
+def ASR17 : SparcCtrlReg<17, "ASR17">;
+def ASR18 : SparcCtrlReg<18, "ASR18">;
+def ASR19 : SparcCtrlReg<19, "ASR19">;
+def ASR20 : SparcCtrlReg<20, "ASR20">;
+def ASR21 : SparcCtrlReg<21, "ASR21">;
+def ASR22 : SparcCtrlReg<22, "ASR22">;
+def ASR23 : SparcCtrlReg<23, "ASR23">;
+def ASR24 : SparcCtrlReg<24, "ASR24">;
+def ASR25 : SparcCtrlReg<25, "ASR25">;
+def ASR26 : SparcCtrlReg<26, "ASR26">;
+def ASR27 : SparcCtrlReg<27, "ASR27">;
+def ASR28 : SparcCtrlReg<28, "ASR28">;
+def ASR29 : SparcCtrlReg<29, "ASR29">;
+def ASR30 : SparcCtrlReg<30, "ASR30">;
+def ASR31 : SparcCtrlReg<31, "ASR31">;
+
+// Note that PSR, WIM, and TBR don't exist on the SparcV9, only the V8.
+def PSR : SparcCtrlReg<0, "PSR">;
+def WIM : SparcCtrlReg<0, "WIM">;
+def TBR : SparcCtrlReg<0, "TBR">;
 
 // Integer registers
 def G0 : Ri< 0, "G0">, DwarfRegNum<[0]>;
@@ -209,3 +246,9 @@ def QFPRegs : RegisterClass<"SP", [f128], 128, (sequence "Q%u", 0, 15)>;
 
 // Floating point control register classes.
 def FCCRegs : RegisterClass<"SP", [i1], 1, (sequence "FCC%u", 0, 3)>;
+
+// Ancillary state registers
+def ASRRegs : RegisterClass<"SP", [i32], 32,
+                            (add Y, (sequence "ASR%u", 1, 31))> {
+  let isAllocatable = 0;
+}
diff --git a/lib/Target/Sparc/SparcSelectionDAGInfo.cpp b/lib/Target/Sparc/SparcSelectionDAGInfo.cpp
deleted file mode 100644
index a308fc5e739..00000000000
--- a/lib/Target/Sparc/SparcSelectionDAGInfo.cpp
+++ /dev/null
@@ -1,24 +0,0 @@
-//===-- SparcSelectionDAGInfo.cpp - Sparc SelectionDAG Info ---------------===//
-//
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
-//
-//===----------------------------------------------------------------------===//
-//
-// This file implements the SparcSelectionDAGInfo class.
-//
-//===----------------------------------------------------------------------===//
-
-#include "SparcSelectionDAGInfo.h"
-using namespace llvm;
-
-#define DEBUG_TYPE "sparc-selectiondag-info"
-
-SparcSelectionDAGInfo::SparcSelectionDAGInfo(const DataLayout &DL)
-  : TargetSelectionDAGInfo(&DL) {
-}
-
-SparcSelectionDAGInfo::~SparcSelectionDAGInfo() {
-}
diff --git a/lib/Target/Sparc/SparcSelectionDAGInfo.h b/lib/Target/Sparc/SparcSelectionDAGInfo.h
deleted file mode 100644
index 6818291b30b..00000000000
--- a/lib/Target/Sparc/SparcSelectionDAGInfo.h
+++ /dev/null
@@ -1,31 +0,0 @@
-//===-- SparcSelectionDAGInfo.h - Sparc SelectionDAG Info -------*- C++ -*-===//
-//
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
-//
-//===----------------------------------------------------------------------===//
-//
-// This file defines the Sparc subclass for TargetSelectionDAGInfo.
-//
-//===----------------------------------------------------------------------===//
-
-#ifndef LLVM_LIB_TARGET_SPARC_SPARCSELECTIONDAGINFO_H
-#define LLVM_LIB_TARGET_SPARC_SPARCSELECTIONDAGINFO_H
-
-#include "llvm/Target/TargetSelectionDAGInfo.h"
-
-namespace llvm {
-
-class SparcTargetMachine;
-
-class SparcSelectionDAGInfo : public TargetSelectionDAGInfo {
-public:
-  explicit SparcSelectionDAGInfo(const DataLayout &DL);
-  ~SparcSelectionDAGInfo() override;
-};
-
-}
-
-#endif
diff --git a/lib/Target/Sparc/SparcSubtarget.cpp b/lib/Target/Sparc/SparcSubtarget.cpp
index ce1105f2d72..d69da409e42 100644
--- a/lib/Target/Sparc/SparcSubtarget.cpp
+++ b/lib/Target/Sparc/SparcSubtarget.cpp
@@ -49,12 +49,12 @@ SparcSubtarget &SparcSubtarget::initializeSubtargetDependencies(StringRef CPU,
   return *this;
 }
 
-SparcSubtarget::SparcSubtarget(const std::string &TT, const std::string &CPU,
+SparcSubtarget::SparcSubtarget(const Triple &TT, const std::string &CPU,
                                const std::string &FS, TargetMachine &TM,
                                bool is64Bit)
     : SparcGenSubtargetInfo(TT, CPU, FS), Is64Bit(is64Bit),
       InstrInfo(initializeSubtargetDependencies(CPU, FS)), TLInfo(TM, *this),
-      TSInfo(*TM.getDataLayout()), FrameLowering(*this) {}
+      FrameLowering(*this) {}
 
 int SparcSubtarget::getAdjustedFrameSize(int frameSize) const {
 
diff --git a/lib/Target/Sparc/SparcSubtarget.h b/lib/Target/Sparc/SparcSubtarget.h
index e6cf460b85c..9d21911d88f 100644
--- a/lib/Target/Sparc/SparcSubtarget.h
+++ b/lib/Target/Sparc/SparcSubtarget.h
@@ -17,9 +17,9 @@
 #include "SparcFrameLowering.h"
 #include "SparcInstrInfo.h"
 #include "SparcISelLowering.h"
-#include "SparcSelectionDAGInfo.h"
 #include "llvm/IR/DataLayout.h"
 #include "llvm/Target/TargetFrameLowering.h"
+#include "llvm/Target/TargetSelectionDAGInfo.h"
 #include "llvm/Target/TargetSubtargetInfo.h"
 #include <string>
 
@@ -39,11 +39,11 @@ class SparcSubtarget : public SparcGenSubtargetInfo {
   bool UsePopc;
   SparcInstrInfo InstrInfo;
   SparcTargetLowering TLInfo;
-  SparcSelectionDAGInfo TSInfo;
+  TargetSelectionDAGInfo TSInfo;
   SparcFrameLowering FrameLowering;
 
 public:
-  SparcSubtarget(const std::string &TT, const std::string &CPU,
+  SparcSubtarget(const Triple &TT, const std::string &CPU,
                  const std::string &FS, TargetMachine &TM, bool is64bit);
 
   const SparcInstrInfo *getInstrInfo() const override { return &InstrInfo; }
@@ -56,7 +56,7 @@ class SparcSubtarget : public SparcGenSubtargetInfo {
   const SparcTargetLowering *getTargetLowering() const override {
     return &TLInfo;
   }
-  const SparcSelectionDAGInfo *getSelectionDAGInfo() const override {
+  const TargetSelectionDAGInfo *getSelectionDAGInfo() const override {
     return &TSInfo;
   }
 
diff --git a/lib/Target/Sparc/SparcTargetMachine.cpp b/lib/Target/Sparc/SparcTargetMachine.cpp
index 6979a176232..725d7f047c4 100644
--- a/lib/Target/Sparc/SparcTargetMachine.cpp
+++ b/lib/Target/Sparc/SparcTargetMachine.cpp
@@ -22,11 +22,13 @@ extern "C" void LLVMInitializeSparcTarget() {
   // Register the target.
   RegisterTargetMachine<SparcV8TargetMachine> X(TheSparcTarget);
   RegisterTargetMachine<SparcV9TargetMachine> Y(TheSparcV9Target);
+  RegisterTargetMachine<SparcelTargetMachine> Z(TheSparcelTarget);
 }
 
-static std::string computeDataLayout(bool is64Bit) {
-  // Sparc is big endian.
-  std::string Ret = "E-m:e";
+static std::string computeDataLayout(const Triple &T, bool is64Bit) {
+  // Sparc is typically big endian, but some are little.
+  std::string Ret = T.getArch() == Triple::sparcel ? "e" : "E";
+  Ret += "-m:e";
 
   // Some ABIs have 32bit pointers.
   if (!is64Bit)
@@ -52,13 +54,13 @@ static std::string computeDataLayout(bool is64Bit) {
 
 /// SparcTargetMachine ctor - Create an ILP32 architecture model
 ///
-SparcTargetMachine::SparcTargetMachine(const Target &T, StringRef TT,
+SparcTargetMachine::SparcTargetMachine(const Target &T, const Triple &TT,
                                        StringRef CPU, StringRef FS,
                                        const TargetOptions &Options,
                                        Reloc::Model RM, CodeModel::Model CM,
                                        CodeGenOpt::Level OL, bool is64bit)
-    : LLVMTargetMachine(T, computeDataLayout(is64bit), TT, CPU, FS, Options, RM,
-                        CM, OL),
+    : LLVMTargetMachine(T, computeDataLayout(TT, is64bit), TT, CPU, FS, Options,
+                        RM, CM, OL),
       TLOF(make_unique<SparcELFTargetObjectFile>()),
       Subtarget(TT, CPU, FS, *this, is64bit) {
   initAsmInfo();
@@ -104,24 +106,27 @@ void SparcPassConfig::addPreEmitPass(){
 
 void SparcV8TargetMachine::anchor() { }
 
-SparcV8TargetMachine::SparcV8TargetMachine(const Target &T,
-                                           StringRef TT, StringRef CPU,
-                                           StringRef FS,
+SparcV8TargetMachine::SparcV8TargetMachine(const Target &T, const Triple &TT,
+                                           StringRef CPU, StringRef FS,
                                            const TargetOptions &Options,
-                                           Reloc::Model RM,
-                                           CodeModel::Model CM,
+                                           Reloc::Model RM, CodeModel::Model CM,
                                            CodeGenOpt::Level OL)
-  : SparcTargetMachine(T, TT, CPU, FS, Options, RM, CM, OL, false) {
-}
+    : SparcTargetMachine(T, TT, CPU, FS, Options, RM, CM, OL, false) {}
 
 void SparcV9TargetMachine::anchor() { }
 
-SparcV9TargetMachine::SparcV9TargetMachine(const Target &T,
-                                           StringRef TT,  StringRef CPU,
-                                           StringRef FS,
+SparcV9TargetMachine::SparcV9TargetMachine(const Target &T, const Triple &TT,
+                                           StringRef CPU, StringRef FS,
                                            const TargetOptions &Options,
-                                           Reloc::Model RM,
-                                           CodeModel::Model CM,
+                                           Reloc::Model RM, CodeModel::Model CM,
                                            CodeGenOpt::Level OL)
-  : SparcTargetMachine(T, TT, CPU, FS, Options, RM, CM, OL, true) {
-}
+    : SparcTargetMachine(T, TT, CPU, FS, Options, RM, CM, OL, true) {}
+
+void SparcelTargetMachine::anchor() {}
+
+SparcelTargetMachine::SparcelTargetMachine(const Target &T, const Triple &TT,
+                                           StringRef CPU, StringRef FS,
+                                           const TargetOptions &Options,
+                                           Reloc::Model RM, CodeModel::Model CM,
+                                           CodeGenOpt::Level OL)
+    : SparcTargetMachine(T, TT, CPU, FS, Options, RM, CM, OL, false) {}
diff --git a/lib/Target/Sparc/SparcTargetMachine.h b/lib/Target/Sparc/SparcTargetMachine.h
index 30a8ebf75d9..903c2d15629 100644
--- a/lib/Target/Sparc/SparcTargetMachine.h
+++ b/lib/Target/Sparc/SparcTargetMachine.h
@@ -24,10 +24,10 @@ class SparcTargetMachine : public LLVMTargetMachine {
   std::unique_ptr<TargetLoweringObjectFile> TLOF;
   SparcSubtarget Subtarget;
 public:
-  SparcTargetMachine(const Target &T, StringRef TT,
-                     StringRef CPU, StringRef FS, const TargetOptions &Options,
-                     Reloc::Model RM, CodeModel::Model CM,
-                     CodeGenOpt::Level OL, bool is64bit);
+  SparcTargetMachine(const Target &T, const Triple &TT, StringRef CPU,
+                     StringRef FS, const TargetOptions &Options,
+                     Reloc::Model RM, CodeModel::Model CM, CodeGenOpt::Level OL,
+                     bool is64bit);
   ~SparcTargetMachine() override;
 
   const SparcSubtarget *getSubtargetImpl(const Function &) const override {
@@ -46,9 +46,8 @@ class SparcTargetMachine : public LLVMTargetMachine {
 class SparcV8TargetMachine : public SparcTargetMachine {
   virtual void anchor();
 public:
-  SparcV8TargetMachine(const Target &T, StringRef TT,
-                       StringRef CPU, StringRef FS,
-                       const TargetOptions &Options,
+  SparcV8TargetMachine(const Target &T, const Triple &TT, StringRef CPU,
+                       StringRef FS, const TargetOptions &Options,
                        Reloc::Model RM, CodeModel::Model CM,
                        CodeGenOpt::Level OL);
 };
@@ -58,9 +57,18 @@ class SparcV8TargetMachine : public SparcTargetMachine {
 class SparcV9TargetMachine : public SparcTargetMachine {
   virtual void anchor();
 public:
-  SparcV9TargetMachine(const Target &T, StringRef TT,
-                       StringRef CPU, StringRef FS,
-                       const TargetOptions &Options,
+  SparcV9TargetMachine(const Target &T, const Triple &TT, StringRef CPU,
+                       StringRef FS, const TargetOptions &Options,
+                       Reloc::Model RM, CodeModel::Model CM,
+                       CodeGenOpt::Level OL);
+};
+
+class SparcelTargetMachine : public SparcTargetMachine {
+  virtual void anchor();
+
+public:
+  SparcelTargetMachine(const Target &T, const Triple &TT, StringRef CPU,
+                       StringRef FS, const TargetOptions &Options,
                        Reloc::Model RM, CodeModel::Model CM,
                        CodeGenOpt::Level OL);
 };
diff --git a/lib/Target/Sparc/SparcTargetObjectFile.cpp b/lib/Target/Sparc/SparcTargetObjectFile.cpp
index 32b2240f87e..412e124f9a2 100644
--- a/lib/Target/Sparc/SparcTargetObjectFile.cpp
+++ b/lib/Target/Sparc/SparcTargetObjectFile.cpp
@@ -34,8 +34,8 @@ const MCExpr *SparcELFTargetObjectFile::getTTypeGlobalReference(
     }
 
     MCContext &Ctx = getContext();
-    return SparcMCExpr::Create(SparcMCExpr::VK_Sparc_R_DISP32,
-                               MCSymbolRefExpr::Create(SSym, Ctx), Ctx);
+    return SparcMCExpr::create(SparcMCExpr::VK_Sparc_R_DISP32,
+                               MCSymbolRefExpr::create(SSym, Ctx), Ctx);
   }
 
   return TargetLoweringObjectFileELF::getTTypeGlobalReference(
diff --git a/lib/Target/Sparc/TargetInfo/SparcTargetInfo.cpp b/lib/Target/Sparc/TargetInfo/SparcTargetInfo.cpp
index 4eea16370ea..ab1c6beee4e 100644
--- a/lib/Target/Sparc/TargetInfo/SparcTargetInfo.cpp
+++ b/lib/Target/Sparc/TargetInfo/SparcTargetInfo.cpp
@@ -14,10 +14,13 @@ using namespace llvm;
 
 Target llvm::TheSparcTarget;
 Target llvm::TheSparcV9Target;
+Target llvm::TheSparcelTarget;
 
 extern "C" void LLVMInitializeSparcTargetInfo() {
-  RegisterTarget<Triple::sparc, /*HasJIT=*/ true>
-    X(TheSparcTarget, "sparc", "Sparc");
-  RegisterTarget<Triple::sparcv9, /*HasJIT=*/ true>
-    Y(TheSparcV9Target, "sparcv9", "Sparc V9");
+  RegisterTarget<Triple::sparc, /*HasJIT=*/true> X(TheSparcTarget, "sparc",
+                                                   "Sparc");
+  RegisterTarget<Triple::sparcv9, /*HasJIT=*/true> Y(TheSparcV9Target,
+                                                     "sparcv9", "Sparc V9");
+  RegisterTarget<Triple::sparcel, /*HasJIT=*/true> Z(TheSparcelTarget,
+                                                     "sparcel", "Sparc LE");
 }
diff --git a/lib/Target/SystemZ/AsmParser/SystemZAsmParser.cpp b/lib/Target/SystemZ/AsmParser/SystemZAsmParser.cpp
index 9181ff7a837..0b59786ae22 100644
--- a/lib/Target/SystemZ/AsmParser/SystemZAsmParser.cpp
+++ b/lib/Target/SystemZ/AsmParser/SystemZAsmParser.cpp
@@ -39,13 +39,17 @@ enum RegisterKind {
   ADDR64Reg,
   FP32Reg,
   FP64Reg,
-  FP128Reg
+  FP128Reg,
+  VR32Reg,
+  VR64Reg,
+  VR128Reg
 };
 
 enum MemoryKind {
   BDMem,
   BDXMem,
-  BDLMem
+  BDLMem,
+  BDVMem
 };
 
 class SystemZOperand : public MCParsedAsmOperand {
@@ -85,14 +89,14 @@ class SystemZOperand : public MCParsedAsmOperand {
   };
 
   // Base + Disp + Index, where Base and Index are LLVM registers or 0.
-  // RegKind says what type the registers have (ADDR32Reg or ADDR64Reg).
-  // Length is the operand length for D(L,B)-style operands, otherwise
-  // it is null.
+  // MemKind says what type of memory this is and RegKind says what type
+  // the base register has (ADDR32Reg or ADDR64Reg).  Length is the operand
+  // length for D(L,B)-style operands, otherwise it is null.
   struct MemOp {
-    unsigned Base : 8;
-    unsigned Index : 8;
-    unsigned RegKind : 8;
-    unsigned Unused : 8;
+    unsigned Base : 12;
+    unsigned Index : 12;
+    unsigned MemKind : 4;
+    unsigned RegKind : 4;
     const MCExpr *Disp;
     const MCExpr *Length;
   };
@@ -116,11 +120,11 @@ class SystemZOperand : public MCParsedAsmOperand {
   void addExpr(MCInst &Inst, const MCExpr *Expr) const {
     // Add as immediates when possible.  Null MCExpr = 0.
     if (!Expr)
-      Inst.addOperand(MCOperand::CreateImm(0));
+      Inst.addOperand(MCOperand::createImm(0));
     else if (auto *CE = dyn_cast<MCConstantExpr>(Expr))
-      Inst.addOperand(MCOperand::CreateImm(CE->getValue()));
+      Inst.addOperand(MCOperand::createImm(CE->getValue()));
     else
-      Inst.addOperand(MCOperand::CreateExpr(Expr));
+      Inst.addOperand(MCOperand::createExpr(Expr));
   }
 
 public:
@@ -158,10 +162,11 @@ class SystemZOperand : public MCParsedAsmOperand {
     return Op;
   }
   static std::unique_ptr<SystemZOperand>
-  createMem(RegisterKind RegKind, unsigned Base, const MCExpr *Disp,
-            unsigned Index, const MCExpr *Length, SMLoc StartLoc,
-            SMLoc EndLoc) {
+  createMem(MemoryKind MemKind, RegisterKind RegKind, unsigned Base,
+            const MCExpr *Disp, unsigned Index, const MCExpr *Length,
+            SMLoc StartLoc, SMLoc EndLoc) {
     auto Op = make_unique<SystemZOperand>(KindMem, StartLoc, EndLoc);
+    Op->Mem.MemKind = MemKind;
     Op->Mem.RegKind = RegKind;
     Op->Mem.Base = Base;
     Op->Mem.Index = Index;
@@ -226,20 +231,31 @@ class SystemZOperand : public MCParsedAsmOperand {
   bool isMem() const override {
     return Kind == KindMem;
   }
-  bool isMem(RegisterKind RegKind, MemoryKind MemKind) const {
+  bool isMem(MemoryKind MemKind) const {
     return (Kind == KindMem &&
-            Mem.RegKind == RegKind &&
-            (MemKind == BDXMem || !Mem.Index) &&
-            (MemKind == BDLMem) == (Mem.Length != nullptr));
+            (Mem.MemKind == MemKind ||
+             // A BDMem can be treated as a BDXMem in which the index
+             // register field is 0.
+             (Mem.MemKind == BDMem && MemKind == BDXMem)));
   }
-  bool isMemDisp12(RegisterKind RegKind, MemoryKind MemKind) const {
-    return isMem(RegKind, MemKind) && inRange(Mem.Disp, 0, 0xfff);
+  bool isMem(MemoryKind MemKind, RegisterKind RegKind) const {
+    return isMem(MemKind) && Mem.RegKind == RegKind;
   }
-  bool isMemDisp20(RegisterKind RegKind, MemoryKind MemKind) const {
-    return isMem(RegKind, MemKind) && inRange(Mem.Disp, -524288, 524287);
+  bool isMemDisp12(MemoryKind MemKind, RegisterKind RegKind) const {
+    return isMem(MemKind, RegKind) && inRange(Mem.Disp, 0, 0xfff);
+  }
+  bool isMemDisp20(MemoryKind MemKind, RegisterKind RegKind) const {
+    return isMem(MemKind, RegKind) && inRange(Mem.Disp, -524288, 524287);
   }
   bool isMemDisp12Len8(RegisterKind RegKind) const {
-    return isMemDisp12(RegKind, BDLMem) && inRange(Mem.Length, 1, 0x100);
+    return isMemDisp12(BDLMem, RegKind) && inRange(Mem.Length, 1, 0x100);
+  }
+  void addBDVAddrOperands(MCInst &Inst, unsigned N) const {
+    assert(N == 3 && "Invalid number of operands");
+    assert(isMem(BDVMem) && "Invalid operand type");
+    Inst.addOperand(MCOperand::createReg(Mem.Base));
+    addExpr(Inst, Mem.Disp);
+    Inst.addOperand(MCOperand::createReg(Mem.Index));
   }
 
   // Override MCParsedAsmOperand.
@@ -251,12 +267,12 @@ class SystemZOperand : public MCParsedAsmOperand {
   // to an instruction.
   void addRegOperands(MCInst &Inst, unsigned N) const {
     assert(N == 1 && "Invalid number of operands");
-    Inst.addOperand(MCOperand::CreateReg(getReg()));
+    Inst.addOperand(MCOperand::createReg(getReg()));
   }
   void addAccessRegOperands(MCInst &Inst, unsigned N) const {
     assert(N == 1 && "Invalid number of operands");
     assert(Kind == KindAccessReg && "Invalid operand type");
-    Inst.addOperand(MCOperand::CreateImm(AccessReg));
+    Inst.addOperand(MCOperand::createImm(AccessReg));
   }
   void addImmOperands(MCInst &Inst, unsigned N) const {
     assert(N == 1 && "Invalid number of operands");
@@ -264,21 +280,21 @@ class SystemZOperand : public MCParsedAsmOperand {
   }
   void addBDAddrOperands(MCInst &Inst, unsigned N) const {
     assert(N == 2 && "Invalid number of operands");
-    assert(Kind == KindMem && Mem.Index == 0 && "Invalid operand type");
-    Inst.addOperand(MCOperand::CreateReg(Mem.Base));
+    assert(isMem(BDMem) && "Invalid operand type");
+    Inst.addOperand(MCOperand::createReg(Mem.Base));
     addExpr(Inst, Mem.Disp);
   }
   void addBDXAddrOperands(MCInst &Inst, unsigned N) const {
     assert(N == 3 && "Invalid number of operands");
-    assert(Kind == KindMem && "Invalid operand type");
-    Inst.addOperand(MCOperand::CreateReg(Mem.Base));
+    assert(isMem(BDXMem) && "Invalid operand type");
+    Inst.addOperand(MCOperand::createReg(Mem.Base));
     addExpr(Inst, Mem.Disp);
-    Inst.addOperand(MCOperand::CreateReg(Mem.Index));
+    Inst.addOperand(MCOperand::createReg(Mem.Index));
   }
   void addBDLAddrOperands(MCInst &Inst, unsigned N) const {
     assert(N == 3 && "Invalid number of operands");
-    assert(Kind == KindMem && "Invalid operand type");
-    Inst.addOperand(MCOperand::CreateReg(Mem.Base));
+    assert(isMem(BDLMem) && "Invalid operand type");
+    Inst.addOperand(MCOperand::createReg(Mem.Base));
     addExpr(Inst, Mem.Disp);
     addExpr(Inst, Mem.Length);
   }
@@ -302,17 +318,26 @@ class SystemZOperand : public MCParsedAsmOperand {
   bool isFP32() const { return isReg(FP32Reg); }
   bool isFP64() const { return isReg(FP64Reg); }
   bool isFP128() const { return isReg(FP128Reg); }
-  bool isBDAddr32Disp12() const { return isMemDisp12(ADDR32Reg, BDMem); }
-  bool isBDAddr32Disp20() const { return isMemDisp20(ADDR32Reg, BDMem); }
-  bool isBDAddr64Disp12() const { return isMemDisp12(ADDR64Reg, BDMem); }
-  bool isBDAddr64Disp20() const { return isMemDisp20(ADDR64Reg, BDMem); }
-  bool isBDXAddr64Disp12() const { return isMemDisp12(ADDR64Reg, BDXMem); }
-  bool isBDXAddr64Disp20() const { return isMemDisp20(ADDR64Reg, BDXMem); }
+  bool isVR32() const { return isReg(VR32Reg); }
+  bool isVR64() const { return isReg(VR64Reg); }
+  bool isVF128() const { return false; }
+  bool isVR128() const { return isReg(VR128Reg); }
+  bool isBDAddr32Disp12() const { return isMemDisp12(BDMem, ADDR32Reg); }
+  bool isBDAddr32Disp20() const { return isMemDisp20(BDMem, ADDR32Reg); }
+  bool isBDAddr64Disp12() const { return isMemDisp12(BDMem, ADDR64Reg); }
+  bool isBDAddr64Disp20() const { return isMemDisp20(BDMem, ADDR64Reg); }
+  bool isBDXAddr64Disp12() const { return isMemDisp12(BDXMem, ADDR64Reg); }
+  bool isBDXAddr64Disp20() const { return isMemDisp20(BDXMem, ADDR64Reg); }
   bool isBDLAddr64Disp12Len8() const { return isMemDisp12Len8(ADDR64Reg); }
+  bool isBDVAddr64Disp12() const { return isMemDisp12(BDVMem, ADDR64Reg); }
+  bool isU1Imm() const { return isImm(0, 1); }
+  bool isU2Imm() const { return isImm(0, 3); }
+  bool isU3Imm() const { return isImm(0, 7); }
   bool isU4Imm() const { return isImm(0, 15); }
   bool isU6Imm() const { return isImm(0, 63); }
   bool isU8Imm() const { return isImm(0, 255); }
   bool isS8Imm() const { return isImm(-128, 127); }
+  bool isU12Imm() const { return isImm(0, 4095); }
   bool isU16Imm() const { return isImm(0, 65535); }
   bool isS16Imm() const { return isImm(-32768, 32767); }
   bool isU32Imm() const { return isImm(0, (1LL << 32) - 1); }
@@ -329,6 +354,7 @@ class SystemZAsmParser : public MCTargetAsmParser {
   enum RegisterGroup {
     RegGR,
     RegFP,
+    RegV,
     RegAccess
   };
   struct Register {
@@ -347,12 +373,12 @@ class SystemZAsmParser : public MCTargetAsmParser {
                                      RegisterKind Kind);
 
   bool parseAddress(unsigned &Base, const MCExpr *&Disp,
-                    unsigned &Index, const MCExpr *&Length,
+                    unsigned &Index, bool &IsVector, const MCExpr *&Length,
                     const unsigned *Regs, RegisterKind RegKind);
 
   OperandMatchResultTy parseAddress(OperandVector &Operands,
-                                    const unsigned *Regs, RegisterKind RegKind,
-                                    MemoryKind MemKind);
+                                    MemoryKind MemKind, const unsigned *Regs,
+                                    RegisterKind RegKind);
 
   OperandMatchResultTy parsePCRel(OperandVector &Operands, int64_t MinVal,
                                   int64_t MaxVal, bool AllowTLS);
@@ -363,7 +389,7 @@ class SystemZAsmParser : public MCTargetAsmParser {
   SystemZAsmParser(MCSubtargetInfo &sti, MCAsmParser &parser,
                    const MCInstrInfo &MII,
                    const MCTargetOptions &Options)
-      : MCTargetAsmParser(), STI(sti), Parser(parser) {
+      : MCTargetAsmParser(Options), STI(sti), Parser(parser) {
     MCAsmParserExtension::Initialize(Parser);
 
     // Initialize the set of available features.
@@ -414,17 +440,32 @@ class SystemZAsmParser : public MCTargetAsmParser {
   OperandMatchResultTy parseFP128(OperandVector &Operands) {
     return parseRegister(Operands, RegFP, SystemZMC::FP128Regs, FP128Reg);
   }
+  OperandMatchResultTy parseVR32(OperandVector &Operands) {
+    return parseRegister(Operands, RegV, SystemZMC::VR32Regs, VR32Reg);
+  }
+  OperandMatchResultTy parseVR64(OperandVector &Operands) {
+    return parseRegister(Operands, RegV, SystemZMC::VR64Regs, VR64Reg);
+  }
+  OperandMatchResultTy parseVF128(OperandVector &Operands) {
+    llvm_unreachable("Shouldn't be used as an operand");
+  }
+  OperandMatchResultTy parseVR128(OperandVector &Operands) {
+    return parseRegister(Operands, RegV, SystemZMC::VR128Regs, VR128Reg);
+  }
   OperandMatchResultTy parseBDAddr32(OperandVector &Operands) {
-    return parseAddress(Operands, SystemZMC::GR32Regs, ADDR32Reg, BDMem);
+    return parseAddress(Operands, BDMem, SystemZMC::GR32Regs, ADDR32Reg);
   }
   OperandMatchResultTy parseBDAddr64(OperandVector &Operands) {
-    return parseAddress(Operands, SystemZMC::GR64Regs, ADDR64Reg, BDMem);
+    return parseAddress(Operands, BDMem, SystemZMC::GR64Regs, ADDR64Reg);
   }
   OperandMatchResultTy parseBDXAddr64(OperandVector &Operands) {
-    return parseAddress(Operands, SystemZMC::GR64Regs, ADDR64Reg, BDXMem);
+    return parseAddress(Operands, BDXMem, SystemZMC::GR64Regs, ADDR64Reg);
   }
   OperandMatchResultTy parseBDLAddr64(OperandVector &Operands) {
-    return parseAddress(Operands, SystemZMC::GR64Regs, ADDR64Reg, BDLMem);
+    return parseAddress(Operands, BDLMem, SystemZMC::GR64Regs, ADDR64Reg);
+  }
+  OperandMatchResultTy parseBDVAddr64(OperandVector &Operands) {
+    return parseAddress(Operands, BDVMem, SystemZMC::GR64Regs, ADDR64Reg);
   }
   OperandMatchResultTy parseAccessReg(OperandVector &Operands);
   OperandMatchResultTy parsePCRel16(OperandVector &Operands) {
@@ -479,6 +520,8 @@ bool SystemZAsmParser::parseRegister(Register &Reg) {
     Reg.Group = RegGR;
   else if (Prefix == 'f' && Reg.Num < 16)
     Reg.Group = RegFP;
+  else if (Prefix == 'v' && Reg.Num < 32)
+    Reg.Group = RegV;
   else if (Prefix == 'a' && Reg.Num < 16)
     Reg.Group = RegAccess;
   else
@@ -529,8 +572,8 @@ SystemZAsmParser::parseRegister(OperandVector &Operands, RegisterGroup Group,
 // Regs maps asm register numbers to LLVM register numbers and RegKind
 // says what kind of address register we're using (ADDR32Reg or ADDR64Reg).
 bool SystemZAsmParser::parseAddress(unsigned &Base, const MCExpr *&Disp,
-                                    unsigned &Index, const MCExpr *&Length,
-                                    const unsigned *Regs,
+                                    unsigned &Index, bool &IsVector,
+                                    const MCExpr *&Length, const unsigned *Regs,
                                     RegisterKind RegKind) {
   // Parse the displacement, which must always be present.
   if (getParser().parseExpression(Disp))
@@ -539,6 +582,7 @@ bool SystemZAsmParser::parseAddress(unsigned &Base, const MCExpr *&Disp,
   // Parse the optional base and index.
   Index = 0;
   Base = 0;
+  IsVector = false;
   Length = nullptr;
   if (getLexer().is(AsmToken::LParen)) {
     Parser.Lex();
@@ -546,12 +590,23 @@ bool SystemZAsmParser::parseAddress(unsigned &Base, const MCExpr *&Disp,
     if (getLexer().is(AsmToken::Percent)) {
       // Parse the first register and decide whether it's a base or an index.
       Register Reg;
-      if (parseRegister(Reg, RegGR, Regs, RegKind))
+      if (parseRegister(Reg))
         return true;
-      if (getLexer().is(AsmToken::Comma))
-        Index = Reg.Num;
-      else
-        Base = Reg.Num;
+      if (Reg.Group == RegV) {
+        // A vector index register.  The base register is optional.
+        IsVector = true;
+        Index = SystemZMC::VR128Regs[Reg.Num];
+      } else if (Reg.Group == RegGR) {
+        if (Reg.Num == 0)
+          return Error(Reg.StartLoc, "%r0 used in an address");
+        // If the are two registers, the first one is the index and the
+        // second is the base.
+        if (getLexer().is(AsmToken::Comma))
+          Index = Regs[Reg.Num];
+        else
+          Base = Regs[Reg.Num];
+      } else
+        return Error(Reg.StartLoc, "invalid address register");
     } else {
       // Parse the length.
       if (getParser().parseExpression(Length))
@@ -578,37 +633,46 @@ bool SystemZAsmParser::parseAddress(unsigned &Base, const MCExpr *&Disp,
 // Parse a memory operand and add it to Operands.  The other arguments
 // are as above.
 SystemZAsmParser::OperandMatchResultTy
-SystemZAsmParser::parseAddress(OperandVector &Operands, const unsigned *Regs,
-                               RegisterKind RegKind, MemoryKind MemKind) {
+SystemZAsmParser::parseAddress(OperandVector &Operands, MemoryKind MemKind,
+                               const unsigned *Regs, RegisterKind RegKind) {
   SMLoc StartLoc = Parser.getTok().getLoc();
   unsigned Base, Index;
+  bool IsVector;
   const MCExpr *Disp;
   const MCExpr *Length;
-  if (parseAddress(Base, Disp, Index, Length, Regs, RegKind))
+  if (parseAddress(Base, Disp, Index, IsVector, Length, Regs, RegKind))
     return MatchOperand_ParseFail;
 
-  if (Index && MemKind != BDXMem)
-    {
-      Error(StartLoc, "invalid use of indexed addressing");
-      return MatchOperand_ParseFail;
-    }
+  if (IsVector && MemKind != BDVMem) {
+    Error(StartLoc, "invalid use of vector addressing");
+    return MatchOperand_ParseFail;
+  }
 
-  if (Length && MemKind != BDLMem)
-    {
-      Error(StartLoc, "invalid use of length addressing");
-      return MatchOperand_ParseFail;
-    }
+  if (!IsVector && MemKind == BDVMem) {
+    Error(StartLoc, "vector index required in address");
+    return MatchOperand_ParseFail;
+  }
 
-  if (!Length && MemKind == BDLMem)
-    {
-      Error(StartLoc, "missing length in address");
-      return MatchOperand_ParseFail;
-    }
+  if (Index && MemKind != BDXMem && MemKind != BDVMem) {
+    Error(StartLoc, "invalid use of indexed addressing");
+    return MatchOperand_ParseFail;
+  }
+
+  if (Length && MemKind != BDLMem) {
+    Error(StartLoc, "invalid use of length addressing");
+    return MatchOperand_ParseFail;
+  }
+
+  if (!Length && MemKind == BDLMem) {
+    Error(StartLoc, "missing length in address");
+    return MatchOperand_ParseFail;
+  }
 
   SMLoc EndLoc =
     SMLoc::getFromPointer(Parser.getTok().getLoc().getPointer() - 1);
-  Operands.push_back(SystemZOperand::createMem(RegKind, Base, Disp, Index,
-                                               Length, StartLoc, EndLoc));
+  Operands.push_back(SystemZOperand::createMem(MemKind, RegKind, Base, Disp,
+                                               Index, Length, StartLoc,
+                                               EndLoc));
   return MatchOperand_Success;
 }
 
@@ -625,6 +689,8 @@ bool SystemZAsmParser::ParseRegister(unsigned &RegNo, SMLoc &StartLoc,
     RegNo = SystemZMC::GR64Regs[Reg.Num];
   else if (Reg.Group == RegFP)
     RegNo = SystemZMC::FP64Regs[Reg.Num];
+  else if (Reg.Group == RegV)
+    RegNo = SystemZMC::VR128Regs[Reg.Num];
   else
     // FIXME: Access registers aren't modelled as LLVM registers yet.
     return Error(Reg.StartLoc, "invalid operand for instruction");
@@ -697,8 +763,10 @@ bool SystemZAsmParser::parseOperand(OperandVector &Operands,
   // so we treat any plain expression as an immediate.
   SMLoc StartLoc = Parser.getTok().getLoc();
   unsigned Base, Index;
+  bool IsVector;
   const MCExpr *Expr, *Length;
-  if (parseAddress(Base, Expr, Index, Length, SystemZMC::GR64Regs, ADDR64Reg))
+  if (parseAddress(Base, Expr, Index, IsVector, Length, SystemZMC::GR64Regs,
+                   ADDR64Reg))
     return true;
 
   SMLoc EndLoc =
@@ -795,11 +863,11 @@ SystemZAsmParser::parsePCRel(OperandVector &Operands, int64_t MinVal,
       Error(StartLoc, "offset out of range");
       return MatchOperand_ParseFail;
     }
-    MCSymbol *Sym = Ctx.CreateTempSymbol();
+    MCSymbol *Sym = Ctx.createTempSymbol();
     Out.EmitLabel(Sym);
-    const MCExpr *Base = MCSymbolRefExpr::Create(Sym, MCSymbolRefExpr::VK_None,
+    const MCExpr *Base = MCSymbolRefExpr::create(Sym, MCSymbolRefExpr::VK_None,
                                                  Ctx);
-    Expr = Value == 0 ? Base : MCBinaryExpr::CreateAdd(Base, Expr, Ctx);
+    Expr = Value == 0 ? Base : MCBinaryExpr::createAdd(Base, Expr, Ctx);
   }
 
   // Optionally match :tls_gdcall: or :tls_ldcall: followed by a TLS symbol.
@@ -836,7 +904,7 @@ SystemZAsmParser::parsePCRel(OperandVector &Operands, int64_t MinVal,
     }
 
     StringRef Identifier = Parser.getTok().getString();
-    Sym = MCSymbolRefExpr::Create(Ctx.GetOrCreateSymbol(Identifier),
+    Sym = MCSymbolRefExpr::create(Ctx.getOrCreateSymbol(Identifier),
                                   Kind, Ctx);
     Parser.Lex();
   }
diff --git a/lib/Target/SystemZ/Disassembler/SystemZDisassembler.cpp b/lib/Target/SystemZ/Disassembler/SystemZDisassembler.cpp
index 84400f828cb..bf67b75d533 100644
--- a/lib/Target/SystemZ/Disassembler/SystemZDisassembler.cpp
+++ b/lib/Target/SystemZ/Disassembler/SystemZDisassembler.cpp
@@ -47,74 +47,94 @@ extern "C" void LLVMInitializeSystemZDisassembler() {
 }
 
 static DecodeStatus decodeRegisterClass(MCInst &Inst, uint64_t RegNo,
-                                        const unsigned *Regs) {
-  assert(RegNo < 16 && "Invalid register");
+                                        const unsigned *Regs, unsigned Size) {
+  assert(RegNo < Size && "Invalid register");
   RegNo = Regs[RegNo];
   if (RegNo == 0)
     return MCDisassembler::Fail;
-  Inst.addOperand(MCOperand::CreateReg(RegNo));
+  Inst.addOperand(MCOperand::createReg(RegNo));
   return MCDisassembler::Success;
 }
 
 static DecodeStatus DecodeGR32BitRegisterClass(MCInst &Inst, uint64_t RegNo,
                                                uint64_t Address,
                                                const void *Decoder) {
-  return decodeRegisterClass(Inst, RegNo, SystemZMC::GR32Regs);
+  return decodeRegisterClass(Inst, RegNo, SystemZMC::GR32Regs, 16);
 }
 
 static DecodeStatus DecodeGRH32BitRegisterClass(MCInst &Inst, uint64_t RegNo,
                                                 uint64_t Address,
                                                 const void *Decoder) {
-  return decodeRegisterClass(Inst, RegNo, SystemZMC::GRH32Regs);
+  return decodeRegisterClass(Inst, RegNo, SystemZMC::GRH32Regs, 16);
 }
 
 static DecodeStatus DecodeGR64BitRegisterClass(MCInst &Inst, uint64_t RegNo,
                                                uint64_t Address,
                                                const void *Decoder) {
-  return decodeRegisterClass(Inst, RegNo, SystemZMC::GR64Regs);
+  return decodeRegisterClass(Inst, RegNo, SystemZMC::GR64Regs, 16);
 }
 
 static DecodeStatus DecodeGR128BitRegisterClass(MCInst &Inst, uint64_t RegNo,
                                                 uint64_t Address,
                                                 const void *Decoder) {
-  return decodeRegisterClass(Inst, RegNo, SystemZMC::GR128Regs);
+  return decodeRegisterClass(Inst, RegNo, SystemZMC::GR128Regs, 16);
 }
 
 static DecodeStatus DecodeADDR64BitRegisterClass(MCInst &Inst, uint64_t RegNo,
                                                  uint64_t Address,
                                                  const void *Decoder) {
-  return decodeRegisterClass(Inst, RegNo, SystemZMC::GR64Regs);
+  return decodeRegisterClass(Inst, RegNo, SystemZMC::GR64Regs, 16);
 }
 
 static DecodeStatus DecodeFP32BitRegisterClass(MCInst &Inst, uint64_t RegNo,
                                                uint64_t Address,
                                                const void *Decoder) {
-  return decodeRegisterClass(Inst, RegNo, SystemZMC::FP32Regs);
+  return decodeRegisterClass(Inst, RegNo, SystemZMC::FP32Regs, 16);
 }
 
 static DecodeStatus DecodeFP64BitRegisterClass(MCInst &Inst, uint64_t RegNo,
                                                uint64_t Address,
                                                const void *Decoder) {
-  return decodeRegisterClass(Inst, RegNo, SystemZMC::FP64Regs);
+  return decodeRegisterClass(Inst, RegNo, SystemZMC::FP64Regs, 16);
 }
 
 static DecodeStatus DecodeFP128BitRegisterClass(MCInst &Inst, uint64_t RegNo,
                                                 uint64_t Address,
                                                 const void *Decoder) {
-  return decodeRegisterClass(Inst, RegNo, SystemZMC::FP128Regs);
+  return decodeRegisterClass(Inst, RegNo, SystemZMC::FP128Regs, 16);
+}
+
+static DecodeStatus DecodeVR32BitRegisterClass(MCInst &Inst, uint64_t RegNo,
+                                               uint64_t Address,
+                                               const void *Decoder) {
+  return decodeRegisterClass(Inst, RegNo, SystemZMC::VR32Regs, 32);
+}
+
+static DecodeStatus DecodeVR64BitRegisterClass(MCInst &Inst, uint64_t RegNo,
+                                               uint64_t Address,
+                                               const void *Decoder) {
+  return decodeRegisterClass(Inst, RegNo, SystemZMC::VR64Regs, 32);
+}
+
+static DecodeStatus DecodeVR128BitRegisterClass(MCInst &Inst, uint64_t RegNo,
+                                                uint64_t Address,
+                                                const void *Decoder) {
+  return decodeRegisterClass(Inst, RegNo, SystemZMC::VR128Regs, 32);
 }
 
 template<unsigned N>
 static DecodeStatus decodeUImmOperand(MCInst &Inst, uint64_t Imm) {
-  assert(isUInt<N>(Imm) && "Invalid immediate");
-  Inst.addOperand(MCOperand::CreateImm(Imm));
+  if (!isUInt<N>(Imm))
+    return MCDisassembler::Fail;
+  Inst.addOperand(MCOperand::createImm(Imm));
   return MCDisassembler::Success;
 }
 
 template<unsigned N>
 static DecodeStatus decodeSImmOperand(MCInst &Inst, uint64_t Imm) {
-  assert(isUInt<N>(Imm) && "Invalid immediate");
-  Inst.addOperand(MCOperand::CreateImm(SignExtend64<N>(Imm)));
+  if (!isUInt<N>(Imm))
+    return MCDisassembler::Fail;
+  Inst.addOperand(MCOperand::createImm(SignExtend64<N>(Imm)));
   return MCDisassembler::Success;
 }
 
@@ -124,6 +144,21 @@ static DecodeStatus decodeAccessRegOperand(MCInst &Inst, uint64_t Imm,
   return decodeUImmOperand<4>(Inst, Imm);
 }
 
+static DecodeStatus decodeU1ImmOperand(MCInst &Inst, uint64_t Imm,
+                                       uint64_t Address, const void *Decoder) {
+  return decodeUImmOperand<1>(Inst, Imm);
+}
+
+static DecodeStatus decodeU2ImmOperand(MCInst &Inst, uint64_t Imm,
+                                       uint64_t Address, const void *Decoder) {
+  return decodeUImmOperand<2>(Inst, Imm);
+}
+
+static DecodeStatus decodeU3ImmOperand(MCInst &Inst, uint64_t Imm,
+                                       uint64_t Address, const void *Decoder) {
+  return decodeUImmOperand<3>(Inst, Imm);
+}
+
 static DecodeStatus decodeU4ImmOperand(MCInst &Inst, uint64_t Imm,
                                        uint64_t Address, const void *Decoder) {
   return decodeUImmOperand<4>(Inst, Imm);
@@ -139,6 +174,11 @@ static DecodeStatus decodeU8ImmOperand(MCInst &Inst, uint64_t Imm,
   return decodeUImmOperand<8>(Inst, Imm);
 }
 
+static DecodeStatus decodeU12ImmOperand(MCInst &Inst, uint64_t Imm,
+                                        uint64_t Address, const void *Decoder) {
+  return decodeUImmOperand<12>(Inst, Imm);
+}
+
 static DecodeStatus decodeU16ImmOperand(MCInst &Inst, uint64_t Imm,
                                         uint64_t Address, const void *Decoder) {
   return decodeUImmOperand<16>(Inst, Imm);
@@ -168,7 +208,7 @@ template<unsigned N>
 static DecodeStatus decodePCDBLOperand(MCInst &Inst, uint64_t Imm,
                                        uint64_t Address) {
   assert(isUInt<N>(Imm) && "Invalid PC-relative offset");
-  Inst.addOperand(MCOperand::CreateImm(SignExtend64<N>(Imm) * 2 + Address));
+  Inst.addOperand(MCOperand::createImm(SignExtend64<N>(Imm) * 2 + Address));
   return MCDisassembler::Success;
 }
 
@@ -189,8 +229,8 @@ static DecodeStatus decodeBDAddr12Operand(MCInst &Inst, uint64_t Field,
   uint64_t Base = Field >> 12;
   uint64_t Disp = Field & 0xfff;
   assert(Base < 16 && "Invalid BDAddr12");
-  Inst.addOperand(MCOperand::CreateReg(Base == 0 ? 0 : Regs[Base]));
-  Inst.addOperand(MCOperand::CreateImm(Disp));
+  Inst.addOperand(MCOperand::createReg(Base == 0 ? 0 : Regs[Base]));
+  Inst.addOperand(MCOperand::createImm(Disp));
   return MCDisassembler::Success;
 }
 
@@ -199,8 +239,8 @@ static DecodeStatus decodeBDAddr20Operand(MCInst &Inst, uint64_t Field,
   uint64_t Base = Field >> 20;
   uint64_t Disp = ((Field << 12) & 0xff000) | ((Field >> 8) & 0xfff);
   assert(Base < 16 && "Invalid BDAddr20");
-  Inst.addOperand(MCOperand::CreateReg(Base == 0 ? 0 : Regs[Base]));
-  Inst.addOperand(MCOperand::CreateImm(SignExtend64<20>(Disp)));
+  Inst.addOperand(MCOperand::createReg(Base == 0 ? 0 : Regs[Base]));
+  Inst.addOperand(MCOperand::createImm(SignExtend64<20>(Disp)));
   return MCDisassembler::Success;
 }
 
@@ -210,9 +250,9 @@ static DecodeStatus decodeBDXAddr12Operand(MCInst &Inst, uint64_t Field,
   uint64_t Base = (Field >> 12) & 0xf;
   uint64_t Disp = Field & 0xfff;
   assert(Index < 16 && "Invalid BDXAddr12");
-  Inst.addOperand(MCOperand::CreateReg(Base == 0 ? 0 : Regs[Base]));
-  Inst.addOperand(MCOperand::CreateImm(Disp));
-  Inst.addOperand(MCOperand::CreateReg(Index == 0 ? 0 : Regs[Index]));
+  Inst.addOperand(MCOperand::createReg(Base == 0 ? 0 : Regs[Base]));
+  Inst.addOperand(MCOperand::createImm(Disp));
+  Inst.addOperand(MCOperand::createReg(Index == 0 ? 0 : Regs[Index]));
   return MCDisassembler::Success;
 }
 
@@ -222,9 +262,9 @@ static DecodeStatus decodeBDXAddr20Operand(MCInst &Inst, uint64_t Field,
   uint64_t Base = (Field >> 20) & 0xf;
   uint64_t Disp = ((Field & 0xfff00) >> 8) | ((Field & 0xff) << 12);
   assert(Index < 16 && "Invalid BDXAddr20");
-  Inst.addOperand(MCOperand::CreateReg(Base == 0 ? 0 : Regs[Base]));
-  Inst.addOperand(MCOperand::CreateImm(SignExtend64<20>(Disp)));
-  Inst.addOperand(MCOperand::CreateReg(Index == 0 ? 0 : Regs[Index]));
+  Inst.addOperand(MCOperand::createReg(Base == 0 ? 0 : Regs[Base]));
+  Inst.addOperand(MCOperand::createImm(SignExtend64<20>(Disp)));
+  Inst.addOperand(MCOperand::createReg(Index == 0 ? 0 : Regs[Index]));
   return MCDisassembler::Success;
 }
 
@@ -234,9 +274,21 @@ static DecodeStatus decodeBDLAddr12Len8Operand(MCInst &Inst, uint64_t Field,
   uint64_t Base = (Field >> 12) & 0xf;
   uint64_t Disp = Field & 0xfff;
   assert(Length < 256 && "Invalid BDLAddr12Len8");
-  Inst.addOperand(MCOperand::CreateReg(Base == 0 ? 0 : Regs[Base]));
-  Inst.addOperand(MCOperand::CreateImm(Disp));
-  Inst.addOperand(MCOperand::CreateImm(Length + 1));
+  Inst.addOperand(MCOperand::createReg(Base == 0 ? 0 : Regs[Base]));
+  Inst.addOperand(MCOperand::createImm(Disp));
+  Inst.addOperand(MCOperand::createImm(Length + 1));
+  return MCDisassembler::Success;
+}
+
+static DecodeStatus decodeBDVAddr12Operand(MCInst &Inst, uint64_t Field,
+                                           const unsigned *Regs) {
+  uint64_t Index = Field >> 16;
+  uint64_t Base = (Field >> 12) & 0xf;
+  uint64_t Disp = Field & 0xfff;
+  assert(Index < 32 && "Invalid BDVAddr12");
+  Inst.addOperand(MCOperand::createReg(Base == 0 ? 0 : Regs[Base]));
+  Inst.addOperand(MCOperand::createImm(Disp));
+  Inst.addOperand(MCOperand::createReg(SystemZMC::VR128Regs[Index]));
   return MCDisassembler::Success;
 }
 
@@ -283,6 +335,12 @@ static DecodeStatus decodeBDLAddr64Disp12Len8Operand(MCInst &Inst,
   return decodeBDLAddr12Len8Operand(Inst, Field, SystemZMC::GR64Regs);
 }
 
+static DecodeStatus decodeBDVAddr64Disp12Operand(MCInst &Inst, uint64_t Field,
+                                                 uint64_t Address,
+                                                 const void *Decoder) {
+  return decodeBDVAddr12Operand(Inst, Field, SystemZMC::GR64Regs);
+}
+
 #include "SystemZGenDisassemblerTables.inc"
 
 DecodeStatus SystemZDisassembler::getInstruction(MCInst &MI, uint64_t &Size,
diff --git a/lib/Target/SystemZ/InstPrinter/SystemZInstPrinter.cpp b/lib/Target/SystemZ/InstPrinter/SystemZInstPrinter.cpp
index cf1ee54df38..059ae3f7fb0 100644
--- a/lib/Target/SystemZ/InstPrinter/SystemZInstPrinter.cpp
+++ b/lib/Target/SystemZ/InstPrinter/SystemZInstPrinter.cpp
@@ -9,8 +9,10 @@
 
 #include "SystemZInstPrinter.h"
 #include "llvm/MC/MCExpr.h"
+#include "llvm/MC/MCInst.h"
 #include "llvm/MC/MCInstrInfo.h"
 #include "llvm/MC/MCSymbol.h"
+#include "llvm/Support/ErrorHandling.h"
 #include "llvm/Support/raw_ostream.h"
 
 using namespace llvm;
@@ -22,22 +24,27 @@ using namespace llvm;
 void SystemZInstPrinter::printAddress(unsigned Base, int64_t Disp,
                                       unsigned Index, raw_ostream &O) {
   O << Disp;
-  if (Base) {
+  if (Base || Index) {
     O << '(';
-    if (Index)
-      O << '%' << getRegisterName(Index) << ',';
-    O << '%' << getRegisterName(Base) << ')';
-  } else
-    assert(!Index && "Shouldn't have an index without a base");
+    if (Index) {
+      O << '%' << getRegisterName(Index);
+      if (Base)
+        O << ',';
+    }
+    if (Base)
+      O << '%' << getRegisterName(Base);
+    O << ')';
+  }
 }
 
-void SystemZInstPrinter::printOperand(const MCOperand &MO, raw_ostream &O) {
+void SystemZInstPrinter::printOperand(const MCOperand &MO, const MCAsmInfo *MAI,
+                                      raw_ostream &O) {
   if (MO.isReg())
     O << '%' << getRegisterName(MO.getReg());
   else if (MO.isImm())
     O << MO.getImm();
   else if (MO.isExpr())
-    O << *MO.getExpr();
+    MO.getExpr()->print(O, MAI);
   else
     llvm_unreachable("Invalid operand");
 }
@@ -53,60 +60,78 @@ void SystemZInstPrinter::printRegName(raw_ostream &O, unsigned RegNo) const {
   O << '%' << getRegisterName(RegNo);
 }
 
-void SystemZInstPrinter::printU4ImmOperand(const MCInst *MI, int OpNum,
-                                           raw_ostream &O) {
+template<unsigned N>
+void printUImmOperand(const MCInst *MI, int OpNum, raw_ostream &O) {
   int64_t Value = MI->getOperand(OpNum).getImm();
-  assert(isUInt<4>(Value) && "Invalid u4imm argument");
+  assert(isUInt<N>(Value) && "Invalid uimm argument");
   O << Value;
 }
 
-void SystemZInstPrinter::printU6ImmOperand(const MCInst *MI, int OpNum,
-                                           raw_ostream &O) {
+template<unsigned N>
+void printSImmOperand(const MCInst *MI, int OpNum, raw_ostream &O) {
   int64_t Value = MI->getOperand(OpNum).getImm();
-  assert(isUInt<6>(Value) && "Invalid u6imm argument");
+  assert(isInt<N>(Value) && "Invalid simm argument");
   O << Value;
 }
 
+void SystemZInstPrinter::printU1ImmOperand(const MCInst *MI, int OpNum,
+                                           raw_ostream &O) {
+  printUImmOperand<1>(MI, OpNum, O);
+}
+
+void SystemZInstPrinter::printU2ImmOperand(const MCInst *MI, int OpNum,
+                                           raw_ostream &O) {
+  printUImmOperand<2>(MI, OpNum, O);
+}
+
+void SystemZInstPrinter::printU3ImmOperand(const MCInst *MI, int OpNum,
+                                           raw_ostream &O) {
+  printUImmOperand<3>(MI, OpNum, O);
+}
+
+void SystemZInstPrinter::printU4ImmOperand(const MCInst *MI, int OpNum,
+                                           raw_ostream &O) {
+  printUImmOperand<4>(MI, OpNum, O);
+}
+
+void SystemZInstPrinter::printU6ImmOperand(const MCInst *MI, int OpNum,
+                                           raw_ostream &O) {
+  printUImmOperand<6>(MI, OpNum, O);
+}
+
 void SystemZInstPrinter::printS8ImmOperand(const MCInst *MI, int OpNum,
                                            raw_ostream &O) {
-  int64_t Value = MI->getOperand(OpNum).getImm();
-  assert(isInt<8>(Value) && "Invalid s8imm argument");
-  O << Value;
+  printSImmOperand<8>(MI, OpNum, O);
 }
 
 void SystemZInstPrinter::printU8ImmOperand(const MCInst *MI, int OpNum,
                                            raw_ostream &O) {
-  int64_t Value = MI->getOperand(OpNum).getImm();
-  assert(isUInt<8>(Value) && "Invalid u8imm argument");
-  O << Value;
+  printUImmOperand<8>(MI, OpNum, O);
+}
+
+void SystemZInstPrinter::printU12ImmOperand(const MCInst *MI, int OpNum,
+                                            raw_ostream &O) {
+  printUImmOperand<12>(MI, OpNum, O);
 }
 
 void SystemZInstPrinter::printS16ImmOperand(const MCInst *MI, int OpNum,
                                             raw_ostream &O) {
-  int64_t Value = MI->getOperand(OpNum).getImm();
-  assert(isInt<16>(Value) && "Invalid s16imm argument");
-  O << Value;
+  printSImmOperand<16>(MI, OpNum, O);
 }
 
 void SystemZInstPrinter::printU16ImmOperand(const MCInst *MI, int OpNum,
                                             raw_ostream &O) {
-  int64_t Value = MI->getOperand(OpNum).getImm();
-  assert(isUInt<16>(Value) && "Invalid u16imm argument");
-  O << Value;
+  printUImmOperand<16>(MI, OpNum, O);
 }
 
 void SystemZInstPrinter::printS32ImmOperand(const MCInst *MI, int OpNum,
                                             raw_ostream &O) {
-  int64_t Value = MI->getOperand(OpNum).getImm();
-  assert(isInt<32>(Value) && "Invalid s32imm argument");
-  O << Value;
+  printSImmOperand<32>(MI, OpNum, O);
 }
 
 void SystemZInstPrinter::printU32ImmOperand(const MCInst *MI, int OpNum,
                                             raw_ostream &O) {
-  int64_t Value = MI->getOperand(OpNum).getImm();
-  assert(isUInt<32>(Value) && "Invalid u32imm argument");
-  O << Value;
+  printUImmOperand<32>(MI, OpNum, O);
 }
 
 void SystemZInstPrinter::printAccessRegOperand(const MCInst *MI, int OpNum,
@@ -123,7 +148,7 @@ void SystemZInstPrinter::printPCRelOperand(const MCInst *MI, int OpNum,
     O << "0x";
     O.write_hex(MO.getImm());
   } else
-    O << *MO.getExpr();
+    MO.getExpr()->print(O, &MAI);
 }
 
 void SystemZInstPrinter::printPCRelTLSOperand(const MCInst *MI, int OpNum,
@@ -151,7 +176,7 @@ void SystemZInstPrinter::printPCRelTLSOperand(const MCInst *MI, int OpNum,
 
 void SystemZInstPrinter::printOperand(const MCInst *MI, int OpNum,
                                       raw_ostream &O) {
-  printOperand(MI->getOperand(OpNum), O);
+  printOperand(MI->getOperand(OpNum), &MAI, O);
 }
 
 void SystemZInstPrinter::printBDAddrOperand(const MCInst *MI, int OpNum,
@@ -178,6 +203,13 @@ void SystemZInstPrinter::printBDLAddrOperand(const MCInst *MI, int OpNum,
   O << ')';
 }
 
+void SystemZInstPrinter::printBDVAddrOperand(const MCInst *MI, int OpNum,
+                                             raw_ostream &O) {
+  printAddress(MI->getOperand(OpNum).getReg(),
+               MI->getOperand(OpNum + 1).getImm(),
+               MI->getOperand(OpNum + 2).getReg(), O);
+}
+
 void SystemZInstPrinter::printCond4Operand(const MCInst *MI, int OpNum,
                                            raw_ostream &O) {
   static const char *const CondNames[] = {
diff --git a/lib/Target/SystemZ/InstPrinter/SystemZInstPrinter.h b/lib/Target/SystemZ/InstPrinter/SystemZInstPrinter.h
index 6f56c7be3c3..ba55e686f3e 100644
--- a/lib/Target/SystemZ/InstPrinter/SystemZInstPrinter.h
+++ b/lib/Target/SystemZ/InstPrinter/SystemZInstPrinter.h
@@ -35,7 +35,8 @@ class SystemZInstPrinter : public MCInstPrinter {
                            raw_ostream &O);
 
   // Print the given operand.
-  static void printOperand(const MCOperand &MO, raw_ostream &O);
+  static void printOperand(const MCOperand &MO, const MCAsmInfo *MAI,
+                           raw_ostream &O);
 
   // Override MCInstPrinter.
   void printRegName(raw_ostream &O, unsigned RegNo) const override;
@@ -48,10 +49,15 @@ class SystemZInstPrinter : public MCInstPrinter {
   void printBDAddrOperand(const MCInst *MI, int OpNum, raw_ostream &O);
   void printBDXAddrOperand(const MCInst *MI, int OpNum, raw_ostream &O);
   void printBDLAddrOperand(const MCInst *MI, int OpNum, raw_ostream &O);
+  void printBDVAddrOperand(const MCInst *MI, int OpNum, raw_ostream &O);
+  void printU1ImmOperand(const MCInst *MI, int OpNum, raw_ostream &O);
+  void printU2ImmOperand(const MCInst *MI, int OpNum, raw_ostream &O);
+  void printU3ImmOperand(const MCInst *MI, int OpNum, raw_ostream &O);
   void printU4ImmOperand(const MCInst *MI, int OpNum, raw_ostream &O);
   void printU6ImmOperand(const MCInst *MI, int OpNum, raw_ostream &O);
   void printS8ImmOperand(const MCInst *MI, int OpNum, raw_ostream &O);
   void printU8ImmOperand(const MCInst *MI, int OpNum, raw_ostream &O);
+  void printU12ImmOperand(const MCInst *MI, int OpNum, raw_ostream &O);
   void printS16ImmOperand(const MCInst *MI, int OpNum, raw_ostream &O);
   void printU16ImmOperand(const MCInst *MI, int OpNum, raw_ostream &O);
   void printS32ImmOperand(const MCInst *MI, int OpNum, raw_ostream &O);
diff --git a/lib/Target/SystemZ/MCTargetDesc/SystemZMCAsmBackend.cpp b/lib/Target/SystemZ/MCTargetDesc/SystemZMCAsmBackend.cpp
index 1c3887ab545..57eebe19c04 100644
--- a/lib/Target/SystemZ/MCTargetDesc/SystemZMCAsmBackend.cpp
+++ b/lib/Target/SystemZ/MCTargetDesc/SystemZMCAsmBackend.cpp
@@ -105,13 +105,13 @@ void SystemZMCAsmBackend::applyFixup(const MCFixup &Fixup, char *Data,
 bool SystemZMCAsmBackend::writeNopData(uint64_t Count,
                                        MCObjectWriter *OW) const {
   for (uint64_t I = 0; I != Count; ++I)
-    OW->Write8(7);
+    OW->write8(7);
   return true;
 }
 
 MCAsmBackend *llvm::createSystemZMCAsmBackend(const Target &T,
                                               const MCRegisterInfo &MRI,
-                                              StringRef TT, StringRef CPU) {
-  uint8_t OSABI = MCELFObjectTargetWriter::getOSABI(Triple(TT).getOS());
+                                              const Triple &TT, StringRef CPU) {
+  uint8_t OSABI = MCELFObjectTargetWriter::getOSABI(TT.getOS());
   return new SystemZMCAsmBackend(OSABI);
 }
diff --git a/lib/Target/SystemZ/MCTargetDesc/SystemZMCAsmInfo.cpp b/lib/Target/SystemZ/MCTargetDesc/SystemZMCAsmInfo.cpp
index 0161d6263e7..b17977d41be 100644
--- a/lib/Target/SystemZ/MCTargetDesc/SystemZMCAsmInfo.cpp
+++ b/lib/Target/SystemZ/MCTargetDesc/SystemZMCAsmInfo.cpp
@@ -13,7 +13,7 @@
 
 using namespace llvm;
 
-SystemZMCAsmInfo::SystemZMCAsmInfo(StringRef TT) {
+SystemZMCAsmInfo::SystemZMCAsmInfo(const Triple &TT) {
   PointerSize = 8;
   CalleeSaveStackSlotSize = 8;
   IsLittleEndian = false;
diff --git a/lib/Target/SystemZ/MCTargetDesc/SystemZMCAsmInfo.h b/lib/Target/SystemZ/MCTargetDesc/SystemZMCAsmInfo.h
index 19b5b4b0972..800f8923206 100644
--- a/lib/Target/SystemZ/MCTargetDesc/SystemZMCAsmInfo.h
+++ b/lib/Target/SystemZ/MCTargetDesc/SystemZMCAsmInfo.h
@@ -14,11 +14,11 @@
 #include "llvm/Support/Compiler.h"
 
 namespace llvm {
-class StringRef;
+class Triple;
 
 class SystemZMCAsmInfo : public MCAsmInfoELF {
 public:
-  explicit SystemZMCAsmInfo(StringRef TT);
+  explicit SystemZMCAsmInfo(const Triple &TT);
 };
 
 } // end namespace llvm
diff --git a/lib/Target/SystemZ/MCTargetDesc/SystemZMCCodeEmitter.cpp b/lib/Target/SystemZ/MCTargetDesc/SystemZMCCodeEmitter.cpp
index 8dd70b93351..fd52a2ebf2f 100644
--- a/lib/Target/SystemZ/MCTargetDesc/SystemZMCCodeEmitter.cpp
+++ b/lib/Target/SystemZ/MCTargetDesc/SystemZMCCodeEmitter.cpp
@@ -16,7 +16,9 @@
 #include "llvm/MC/MCCodeEmitter.h"
 #include "llvm/MC/MCContext.h"
 #include "llvm/MC/MCExpr.h"
+#include "llvm/MC/MCInst.h"
 #include "llvm/MC/MCInstrInfo.h"
+#include "llvm/MC/MCRegisterInfo.h"
 
 using namespace llvm;
 
@@ -35,7 +37,7 @@ class SystemZMCCodeEmitter : public MCCodeEmitter {
   ~SystemZMCCodeEmitter() override {}
 
   // OVerride MCCodeEmitter.
-  void EncodeInstruction(const MCInst &MI, raw_ostream &OS,
+  void encodeInstruction(const MCInst &MI, raw_ostream &OS,
                          SmallVectorImpl<MCFixup> &Fixups,
                          const MCSubtargetInfo &STI) const override;
 
@@ -70,6 +72,9 @@ class SystemZMCCodeEmitter : public MCCodeEmitter {
   uint64_t getBDLAddr12Len8Encoding(const MCInst &MI, unsigned OpNum,
                                     SmallVectorImpl<MCFixup> &Fixups,
                                     const MCSubtargetInfo &STI) const;
+  uint64_t getBDVAddr12Encoding(const MCInst &MI, unsigned OpNum,
+                                SmallVectorImpl<MCFixup> &Fixups,
+                                const MCSubtargetInfo &STI) const;
 
   // Operand OpNum of MI needs a PC-relative fixup of kind Kind at
   // Offset bytes from the start of MI.  Add the fixup to Fixups
@@ -115,7 +120,7 @@ MCCodeEmitter *llvm::createSystemZMCCodeEmitter(const MCInstrInfo &MCII,
 }
 
 void SystemZMCCodeEmitter::
-EncodeInstruction(const MCInst &MI, raw_ostream &OS,
+encodeInstruction(const MCInst &MI, raw_ostream &OS,
                   SmallVectorImpl<MCFixup> &Fixups,
                   const MCSubtargetInfo &STI) const {
   uint64_t Bits = getBinaryCodeForInstr(MI, Fixups, STI);
@@ -193,6 +198,17 @@ getBDLAddr12Len8Encoding(const MCInst &MI, unsigned OpNum,
   return (Len << 16) | (Base << 12) | Disp;
 }
 
+uint64_t SystemZMCCodeEmitter::
+getBDVAddr12Encoding(const MCInst &MI, unsigned OpNum,
+                     SmallVectorImpl<MCFixup> &Fixups,
+                     const MCSubtargetInfo &STI) const {
+  uint64_t Base = getMachineOpValue(MI, MI.getOperand(OpNum), Fixups, STI);
+  uint64_t Disp = getMachineOpValue(MI, MI.getOperand(OpNum + 1), Fixups, STI);
+  uint64_t Index = getMachineOpValue(MI, MI.getOperand(OpNum + 2), Fixups, STI);
+  assert(isUInt<4>(Base) && isUInt<12>(Disp) && isUInt<5>(Index));
+  return (Index << 16) | (Base << 12) | Disp;
+}
+
 uint64_t
 SystemZMCCodeEmitter::getPCRelEncoding(const MCInst &MI, unsigned OpNum,
                                        SmallVectorImpl<MCFixup> &Fixups,
@@ -201,7 +217,7 @@ SystemZMCCodeEmitter::getPCRelEncoding(const MCInst &MI, unsigned OpNum,
   const MCOperand &MO = MI.getOperand(OpNum);
   const MCExpr *Expr;
   if (MO.isImm())
-    Expr = MCConstantExpr::Create(MO.getImm() + Offset, Ctx);
+    Expr = MCConstantExpr::create(MO.getImm() + Offset, Ctx);
   else {
     Expr = MO.getExpr();
     if (Offset) {
@@ -209,16 +225,16 @@ SystemZMCCodeEmitter::getPCRelEncoding(const MCInst &MI, unsigned OpNum,
       // is relative to the operand field itself, which is Offset bytes
       // into MI.  Add Offset to the relocation value to cancel out
       // this difference.
-      const MCExpr *OffsetExpr = MCConstantExpr::Create(Offset, Ctx);
-      Expr = MCBinaryExpr::CreateAdd(Expr, OffsetExpr, Ctx);
+      const MCExpr *OffsetExpr = MCConstantExpr::create(Offset, Ctx);
+      Expr = MCBinaryExpr::createAdd(Expr, OffsetExpr, Ctx);
     }
   }
-  Fixups.push_back(MCFixup::Create(Offset, Expr, (MCFixupKind)Kind));
+  Fixups.push_back(MCFixup::create(Offset, Expr, (MCFixupKind)Kind));
 
   // Output the fixup for the TLS marker if present.
   if (AllowTLS && OpNum + 1 < MI.getNumOperands()) {
     const MCOperand &MOTLS = MI.getOperand(OpNum + 1);
-    Fixups.push_back(MCFixup::Create(0, MOTLS.getExpr(),
+    Fixups.push_back(MCFixup::create(0, MOTLS.getExpr(),
                                      (MCFixupKind)SystemZ::FK_390_TLS_CALL));
   }
   return 0;
diff --git a/lib/Target/SystemZ/MCTargetDesc/SystemZMCTargetDesc.cpp b/lib/Target/SystemZ/MCTargetDesc/SystemZMCTargetDesc.cpp
index ea56fb18d49..5fefa315a4c 100644
--- a/lib/Target/SystemZ/MCTargetDesc/SystemZMCTargetDesc.cpp
+++ b/lib/Target/SystemZ/MCTargetDesc/SystemZMCTargetDesc.cpp
@@ -12,6 +12,7 @@
 #include "SystemZMCAsmInfo.h"
 #include "llvm/MC/MCCodeGenInfo.h"
 #include "llvm/MC/MCInstrInfo.h"
+#include "llvm/MC/MCRegisterInfo.h"
 #include "llvm/MC/MCStreamer.h"
 #include "llvm/MC/MCSubtargetInfo.h"
 #include "llvm/Support/TargetRegistry.h"
@@ -76,6 +77,39 @@ const unsigned SystemZMC::FP128Regs[16] = {
   SystemZ::F12Q, SystemZ::F13Q, 0, 0
 };
 
+const unsigned SystemZMC::VR32Regs[32] = {
+  SystemZ::F0S, SystemZ::F1S, SystemZ::F2S, SystemZ::F3S,
+  SystemZ::F4S, SystemZ::F5S, SystemZ::F6S, SystemZ::F7S,
+  SystemZ::F8S, SystemZ::F9S, SystemZ::F10S, SystemZ::F11S,
+  SystemZ::F12S, SystemZ::F13S, SystemZ::F14S, SystemZ::F15S,
+  SystemZ::F16S, SystemZ::F17S, SystemZ::F18S, SystemZ::F19S,
+  SystemZ::F20S, SystemZ::F21S, SystemZ::F22S, SystemZ::F23S,
+  SystemZ::F24S, SystemZ::F25S, SystemZ::F26S, SystemZ::F27S,
+  SystemZ::F28S, SystemZ::F29S, SystemZ::F30S, SystemZ::F31S
+};
+
+const unsigned SystemZMC::VR64Regs[32] = {
+  SystemZ::F0D, SystemZ::F1D, SystemZ::F2D, SystemZ::F3D,
+  SystemZ::F4D, SystemZ::F5D, SystemZ::F6D, SystemZ::F7D,
+  SystemZ::F8D, SystemZ::F9D, SystemZ::F10D, SystemZ::F11D,
+  SystemZ::F12D, SystemZ::F13D, SystemZ::F14D, SystemZ::F15D,
+  SystemZ::F16D, SystemZ::F17D, SystemZ::F18D, SystemZ::F19D,
+  SystemZ::F20D, SystemZ::F21D, SystemZ::F22D, SystemZ::F23D,
+  SystemZ::F24D, SystemZ::F25D, SystemZ::F26D, SystemZ::F27D,
+  SystemZ::F28D, SystemZ::F29D, SystemZ::F30D, SystemZ::F31D
+};
+
+const unsigned SystemZMC::VR128Regs[32] = {
+  SystemZ::V0, SystemZ::V1, SystemZ::V2, SystemZ::V3,
+  SystemZ::V4, SystemZ::V5, SystemZ::V6, SystemZ::V7,
+  SystemZ::V8, SystemZ::V9, SystemZ::V10, SystemZ::V11,
+  SystemZ::V12, SystemZ::V13, SystemZ::V14, SystemZ::V15,
+  SystemZ::V16, SystemZ::V17, SystemZ::V18, SystemZ::V19,
+  SystemZ::V20, SystemZ::V21, SystemZ::V22, SystemZ::V23,
+  SystemZ::V24, SystemZ::V25, SystemZ::V26, SystemZ::V27,
+  SystemZ::V28, SystemZ::V29, SystemZ::V30, SystemZ::V31
+};
+
 unsigned SystemZMC::getFirstReg(unsigned Reg) {
   static unsigned Map[SystemZ::NUM_TARGET_REGS];
   static bool Initialized = false;
@@ -85,17 +119,20 @@ unsigned SystemZMC::getFirstReg(unsigned Reg) {
       Map[GRH32Regs[I]] = I;
       Map[GR64Regs[I]] = I;
       Map[GR128Regs[I]] = I;
-      Map[FP32Regs[I]] = I;
-      Map[FP64Regs[I]] = I;
       Map[FP128Regs[I]] = I;
     }
+    for (unsigned I = 0; I < 32; ++I) {
+      Map[VR32Regs[I]] = I;
+      Map[VR64Regs[I]] = I;
+      Map[VR128Regs[I]] = I;
+    }
   }
   assert(Reg < SystemZ::NUM_TARGET_REGS);
   return Map[Reg];
 }
 
 static MCAsmInfo *createSystemZMCAsmInfo(const MCRegisterInfo &MRI,
-                                         StringRef TT) {
+                                         const Triple &TT) {
   MCAsmInfo *MAI = new SystemZMCAsmInfo(TT);
   MCCFIInstruction Inst =
       MCCFIInstruction::createDefCfa(nullptr,
@@ -111,21 +148,19 @@ static MCInstrInfo *createSystemZMCInstrInfo() {
   return X;
 }
 
-static MCRegisterInfo *createSystemZMCRegisterInfo(StringRef TT) {
+static MCRegisterInfo *createSystemZMCRegisterInfo(const Triple &TT) {
   MCRegisterInfo *X = new MCRegisterInfo();
   InitSystemZMCRegisterInfo(X, SystemZ::R14D);
   return X;
 }
 
-static MCSubtargetInfo *createSystemZMCSubtargetInfo(StringRef TT,
-                                                     StringRef CPU,
-                                                     StringRef FS) {
-  MCSubtargetInfo *X = new MCSubtargetInfo();
-  InitSystemZMCSubtargetInfo(X, TT, CPU, FS);
-  return X;
+static MCSubtargetInfo *
+createSystemZMCSubtargetInfo(const Triple &TT, StringRef CPU, StringRef FS) {
+  return createSystemZMCSubtargetInfoImpl(TT, CPU, FS);
 }
 
-static MCCodeGenInfo *createSystemZMCCodeGenInfo(StringRef TT, Reloc::Model RM,
+static MCCodeGenInfo *createSystemZMCCodeGenInfo(const Triple &TT,
+                                                 Reloc::Model RM,
                                                  CodeModel::Model CM,
                                                  CodeGenOpt::Level OL) {
   MCCodeGenInfo *X = new MCCodeGenInfo();
@@ -168,7 +203,7 @@ static MCCodeGenInfo *createSystemZMCCodeGenInfo(StringRef TT, Reloc::Model RM,
     CM = CodeModel::Small;
   else if (CM == CodeModel::JITDefault)
     CM = RM == Reloc::PIC_ ? CodeModel::Small : CodeModel::Medium;
-  X->InitMCCodeGenInfo(RM, CM, OL);
+  X->initMCCodeGenInfo(RM, CM, OL);
   return X;
 }
 
diff --git a/lib/Target/SystemZ/MCTargetDesc/SystemZMCTargetDesc.h b/lib/Target/SystemZ/MCTargetDesc/SystemZMCTargetDesc.h
index 2b2647bae94..0db48fe5a10 100644
--- a/lib/Target/SystemZ/MCTargetDesc/SystemZMCTargetDesc.h
+++ b/lib/Target/SystemZ/MCTargetDesc/SystemZMCTargetDesc.h
@@ -23,6 +23,7 @@ class MCRegisterInfo;
 class MCSubtargetInfo;
 class StringRef;
 class Target;
+class Triple;
 class raw_pwrite_stream;
 class raw_ostream;
 
@@ -49,6 +50,9 @@ extern const unsigned GR128Regs[16];
 extern const unsigned FP32Regs[16];
 extern const unsigned FP64Regs[16];
 extern const unsigned FP128Regs[16];
+extern const unsigned VR32Regs[32];
+extern const unsigned VR64Regs[32];
+extern const unsigned VR128Regs[32];
 
 // Return the 0-based number of the first architectural register that
 // contains the given LLVM register.   E.g. R1D -> 1.
@@ -68,6 +72,11 @@ inline unsigned getRegAsGR32(unsigned Reg) {
 inline unsigned getRegAsGRH32(unsigned Reg) {
   return GRH32Regs[getFirstReg(Reg)];
 }
+
+// Return the given register as a VR128.
+inline unsigned getRegAsVR128(unsigned Reg) {
+  return VR128Regs[getFirstReg(Reg)];
+}
 } // end namespace SystemZMC
 
 MCCodeEmitter *createSystemZMCCodeEmitter(const MCInstrInfo &MCII,
@@ -76,7 +85,7 @@ MCCodeEmitter *createSystemZMCCodeEmitter(const MCInstrInfo &MCII,
 
 MCAsmBackend *createSystemZMCAsmBackend(const Target &T,
                                         const MCRegisterInfo &MRI,
-                                        StringRef TT, StringRef CPU);
+                                        const Triple &TT, StringRef CPU);
 
 MCObjectWriter *createSystemZObjectWriter(raw_pwrite_stream &OS, uint8_t OSABI);
 } // end namespace llvm
diff --git a/lib/Target/SystemZ/SystemZ.h b/lib/Target/SystemZ/SystemZ.h
index b3a7310f04b..cafe2c5948c 100644
--- a/lib/Target/SystemZ/SystemZ.h
+++ b/lib/Target/SystemZ/SystemZ.h
@@ -80,6 +80,13 @@ const unsigned CCMASK_TEND_TX   = CCMASK_0;
 const unsigned CCMASK_TEND_NOTX = CCMASK_2;
 const unsigned CCMASK_TEND      = CCMASK_TEND_TX | CCMASK_TEND_NOTX;
 
+// Condition-code mask assignments for vector comparisons (and similar
+// operations).
+const unsigned CCMASK_VCMP_ALL       = CCMASK_0;
+const unsigned CCMASK_VCMP_MIXED     = CCMASK_1;
+const unsigned CCMASK_VCMP_NONE      = CCMASK_3;
+const unsigned CCMASK_VCMP           = CCMASK_0 | CCMASK_1 | CCMASK_3;
+
 // The position of the low CC bit in an IPM result.
 const unsigned IPM_CC = 28;
 
@@ -87,6 +94,13 @@ const unsigned IPM_CC = 28;
 const unsigned PFD_READ  = 1;
 const unsigned PFD_WRITE = 2;
 
+// Number of bits in a vector register.
+const unsigned VectorBits = 128;
+
+// Number of bytes in a vector register (and consequently the number of
+// bytes in a general permute vector).
+const unsigned VectorBytes = VectorBits / 8;
+
 // Return true if Val fits an LLILL operand.
 static inline bool isImmLL(uint64_t Val) {
   return (Val & ~0x000000000000ffffULL) == 0;
diff --git a/lib/Target/SystemZ/SystemZ.td b/lib/Target/SystemZ/SystemZ.td
index 5f829034902..d4d636d3479 100644
--- a/lib/Target/SystemZ/SystemZ.td
+++ b/lib/Target/SystemZ/SystemZ.td
@@ -40,6 +40,7 @@ include "SystemZOperands.td"
 include "SystemZPatterns.td"
 include "SystemZInstrFormats.td"
 include "SystemZInstrInfo.td"
+include "SystemZInstrVector.td"
 include "SystemZInstrFP.td"
 
 def SystemZInstrInfo : InstrInfo {}
diff --git a/lib/Target/SystemZ/SystemZAsmPrinter.cpp b/lib/Target/SystemZ/SystemZAsmPrinter.cpp
index 2524733218f..75273114d62 100644
--- a/lib/Target/SystemZ/SystemZAsmPrinter.cpp
+++ b/lib/Target/SystemZ/SystemZAsmPrinter.cpp
@@ -68,18 +68,39 @@ static MCInst lowerRIEfLow(const MachineInstr *MI, unsigned Opcode) {
 
 static const MCSymbolRefExpr *getTLSGetOffset(MCContext &Context) {
   StringRef Name = "__tls_get_offset";
-  return MCSymbolRefExpr::Create(Context.GetOrCreateSymbol(Name),
+  return MCSymbolRefExpr::create(Context.getOrCreateSymbol(Name),
                                  MCSymbolRefExpr::VK_PLT,
                                  Context);
 }
 
 static const MCSymbolRefExpr *getGlobalOffsetTable(MCContext &Context) {
   StringRef Name = "_GLOBAL_OFFSET_TABLE_";
-  return MCSymbolRefExpr::Create(Context.GetOrCreateSymbol(Name),
+  return MCSymbolRefExpr::create(Context.getOrCreateSymbol(Name),
                                  MCSymbolRefExpr::VK_None,
                                  Context);
 }
 
+// MI loads the high part of a vector from memory.  Return an instruction
+// that uses replicating vector load Opcode to do the same thing.
+static MCInst lowerSubvectorLoad(const MachineInstr *MI, unsigned Opcode) {
+  return MCInstBuilder(Opcode)
+    .addReg(SystemZMC::getRegAsVR128(MI->getOperand(0).getReg()))
+    .addReg(MI->getOperand(1).getReg())
+    .addImm(MI->getOperand(2).getImm())
+    .addReg(MI->getOperand(3).getReg());
+}
+
+// MI stores the high part of a vector to memory.  Return an instruction
+// that uses elemental vector store Opcode to do the same thing.
+static MCInst lowerSubvectorStore(const MachineInstr *MI, unsigned Opcode) {
+  return MCInstBuilder(Opcode)
+    .addReg(SystemZMC::getRegAsVR128(MI->getOperand(0).getReg()))
+    .addReg(MI->getOperand(1).getReg())
+    .addImm(MI->getOperand(2).getImm())
+    .addReg(MI->getOperand(3).getReg())
+    .addImm(0);
+}
+
 void SystemZAsmPrinter::EmitInstruction(const MachineInstr *MI) {
   SystemZMCInstLower Lower(MF->getContext(), *this);
   MCInst LoweredMI;
@@ -151,6 +172,51 @@ void SystemZAsmPrinter::EmitInstruction(const MachineInstr *MI) {
     LoweredMI = lowerRIEfLow(MI, SystemZ::RISBLG);
     break;
 
+  case SystemZ::VLVGP32:
+    LoweredMI = MCInstBuilder(SystemZ::VLVGP)
+      .addReg(MI->getOperand(0).getReg())
+      .addReg(SystemZMC::getRegAsGR64(MI->getOperand(1).getReg()))
+      .addReg(SystemZMC::getRegAsGR64(MI->getOperand(2).getReg()));
+    break;
+
+  case SystemZ::VLR32:
+  case SystemZ::VLR64:
+    LoweredMI = MCInstBuilder(SystemZ::VLR)
+      .addReg(SystemZMC::getRegAsVR128(MI->getOperand(0).getReg()))
+      .addReg(SystemZMC::getRegAsVR128(MI->getOperand(1).getReg()));
+    break;
+
+  case SystemZ::VL32:
+    LoweredMI = lowerSubvectorLoad(MI, SystemZ::VLREPF);
+    break;
+
+  case SystemZ::VL64:
+    LoweredMI = lowerSubvectorLoad(MI, SystemZ::VLREPG);
+    break;
+
+  case SystemZ::VST32:
+    LoweredMI = lowerSubvectorStore(MI, SystemZ::VSTEF);
+    break;
+
+  case SystemZ::VST64:
+    LoweredMI = lowerSubvectorStore(MI, SystemZ::VSTEG);
+    break;
+
+  case SystemZ::LFER:
+    LoweredMI = MCInstBuilder(SystemZ::VLGVF)
+      .addReg(SystemZMC::getRegAsGR64(MI->getOperand(0).getReg()))
+      .addReg(SystemZMC::getRegAsVR128(MI->getOperand(1).getReg()))
+      .addReg(0).addImm(0);
+    break;
+
+  case SystemZ::LEFR:
+    LoweredMI = MCInstBuilder(SystemZ::VLVGF)
+      .addReg(SystemZMC::getRegAsVR128(MI->getOperand(0).getReg()))
+      .addReg(SystemZMC::getRegAsVR128(MI->getOperand(0).getReg()))
+      .addReg(MI->getOperand(1).getReg())
+      .addReg(0).addImm(0);
+    break;
+
 #define LOWER_LOW(NAME)                                                 \
   case SystemZ::NAME##64: LoweredMI = lowerRILow(MI, SystemZ::NAME); break
 
@@ -198,7 +264,7 @@ void SystemZAsmPrinter::EmitInstruction(const MachineInstr *MI) {
     Lower.lower(MI, LoweredMI);
     break;
   }
-  EmitToStreamer(OutStreamer, LoweredMI);
+  EmitToStreamer(*OutStreamer, LoweredMI);
 }
 
 // Convert a SystemZ-specific constant pool modifier into the associated
@@ -219,12 +285,12 @@ EmitMachineConstantPoolValue(MachineConstantPoolValue *MCPV) {
   auto *ZCPV = static_cast<SystemZConstantPoolValue*>(MCPV);
 
   const MCExpr *Expr =
-    MCSymbolRefExpr::Create(getSymbol(ZCPV->getGlobalValue()),
+    MCSymbolRefExpr::create(getSymbol(ZCPV->getGlobalValue()),
                             getModifierVariantKind(ZCPV->getModifier()),
                             OutContext);
-  uint64_t Size = TM.getDataLayout()->getTypeAllocSize(ZCPV->getType());
+  uint64_t Size = getDataLayout().getTypeAllocSize(ZCPV->getType());
 
-  OutStreamer.EmitValue(Expr, Size);
+  OutStreamer->EmitValue(Expr, Size);
 }
 
 bool SystemZAsmPrinter::PrintAsmOperand(const MachineInstr *MI,
@@ -239,7 +305,7 @@ bool SystemZAsmPrinter::PrintAsmOperand(const MachineInstr *MI,
   } else {
     SystemZMCInstLower Lower(MF->getContext(), *this);
     MCOperand MO(Lower.lowerOperand(MI->getOperand(OpNo)));
-    SystemZInstPrinter::printOperand(MO, OS);
+    SystemZInstPrinter::printOperand(MO, MAI, OS);
   }
   return false;
 }
diff --git a/lib/Target/SystemZ/SystemZCallingConv.h b/lib/Target/SystemZ/SystemZCallingConv.h
index 71605ac1126..bff0706618a 100644
--- a/lib/Target/SystemZ/SystemZCallingConv.h
+++ b/lib/Target/SystemZ/SystemZCallingConv.h
@@ -10,6 +10,9 @@
 #ifndef LLVM_LIB_TARGET_SYSTEMZ_SYSTEMZCALLINGCONV_H
 #define LLVM_LIB_TARGET_SYSTEMZ_SYSTEMZCALLINGCONV_H
 
+#include "llvm/ADT/SmallVector.h"
+#include "llvm/CodeGen/CallingConvLower.h"
+
 namespace llvm {
 namespace SystemZ {
   const unsigned NumArgGPRs = 5;
@@ -18,6 +21,64 @@ namespace SystemZ {
   const unsigned NumArgFPRs = 4;
   extern const unsigned ArgFPRs[NumArgFPRs];
 } // end namespace SystemZ
+
+class SystemZCCState : public CCState {
+private:
+  /// Records whether the value was a fixed argument.
+  /// See ISD::OutputArg::IsFixed.
+  SmallVector<bool, 4> ArgIsFixed;
+
+  /// Records whether the value was widened from a short vector type.
+  SmallVector<bool, 4> ArgIsShortVector;
+
+  // Check whether ArgVT is a short vector type.
+  bool IsShortVectorType(EVT ArgVT) {
+    return ArgVT.isVector() && ArgVT.getStoreSize() <= 8;
+  }
+
+public:
+  SystemZCCState(CallingConv::ID CC, bool isVarArg, MachineFunction &MF,
+                 SmallVectorImpl<CCValAssign> &locs, LLVMContext &C)
+      : CCState(CC, isVarArg, MF, locs, C) {}
+
+  void AnalyzeFormalArguments(const SmallVectorImpl<ISD::InputArg> &Ins,
+                              CCAssignFn Fn) {
+    // Formal arguments are always fixed.
+    ArgIsFixed.clear();
+    for (unsigned i = 0; i < Ins.size(); ++i)
+      ArgIsFixed.push_back(true);
+    // Record whether the call operand was a short vector.
+    ArgIsShortVector.clear();
+    for (unsigned i = 0; i < Ins.size(); ++i)
+      ArgIsShortVector.push_back(IsShortVectorType(Ins[i].ArgVT));
+
+    CCState::AnalyzeFormalArguments(Ins, Fn);
+  }
+
+  void AnalyzeCallOperands(const SmallVectorImpl<ISD::OutputArg> &Outs,
+                           CCAssignFn Fn) {
+    // Record whether the call operand was a fixed argument.
+    ArgIsFixed.clear();
+    for (unsigned i = 0; i < Outs.size(); ++i)
+      ArgIsFixed.push_back(Outs[i].IsFixed);
+    // Record whether the call operand was a short vector.
+    ArgIsShortVector.clear();
+    for (unsigned i = 0; i < Outs.size(); ++i)
+      ArgIsShortVector.push_back(IsShortVectorType(Outs[i].ArgVT));
+
+    CCState::AnalyzeCallOperands(Outs, Fn);
+  }
+
+  // This version of AnalyzeCallOperands in the base class is not usable
+  // since we must provide a means of accessing ISD::OutputArg::IsFixed.
+  void AnalyzeCallOperands(const SmallVectorImpl<MVT> &Outs,
+                           SmallVectorImpl<ISD::ArgFlagsTy> &Flags,
+                           CCAssignFn Fn) = delete;
+
+  bool IsFixed(unsigned ValNo) { return ArgIsFixed[ValNo]; }
+  bool IsShortVector(unsigned ValNo) { return ArgIsShortVector[ValNo]; }
+};
+
 } // end namespace llvm
 
 #endif
diff --git a/lib/Target/SystemZ/SystemZCallingConv.td b/lib/Target/SystemZ/SystemZCallingConv.td
index fb0d1d8a3fe..be8f00b57ad 100644
--- a/lib/Target/SystemZ/SystemZCallingConv.td
+++ b/lib/Target/SystemZ/SystemZCallingConv.td
@@ -12,6 +12,20 @@
 class CCIfExtend<CCAction A>
   : CCIf<"ArgFlags.isSExt() || ArgFlags.isZExt()", A>;
 
+class CCIfSubtarget<string F, CCAction A>
+  : CCIf<!strconcat("static_cast<const SystemZSubtarget&>"
+                    "(State.getMachineFunction().getSubtarget()).", F),
+         A>;
+
+// Match if this specific argument is a fixed (i.e. named) argument.
+class CCIfFixed<CCAction A>
+    : CCIf<"static_cast<SystemZCCState *>(&State)->IsFixed(ValNo)", A>;
+
+// Match if this specific argument was widened from a short vector type.
+class CCIfShortVector<CCAction A>
+    : CCIf<"static_cast<SystemZCCState *>(&State)->IsShortVector(ValNo)", A>;
+
+
 //===----------------------------------------------------------------------===//
 // z/Linux return value calling convention
 //===----------------------------------------------------------------------===//
@@ -31,7 +45,14 @@ def RetCC_SystemZ : CallingConv<[
   // doesn't care about the ABI.  All floating-point argument registers
   // are call-clobbered, so we can use all of them here.
   CCIfType<[f32], CCAssignToReg<[F0S, F2S, F4S, F6S]>>,
-  CCIfType<[f64], CCAssignToReg<[F0D, F2D, F4D, F6D]>>
+  CCIfType<[f64], CCAssignToReg<[F0D, F2D, F4D, F6D]>>,
+
+  // Similarly for vectors, with V24 being the ABI-compliant choice.
+  // Sub-128 vectors are returned in the same way, but they're widened
+  // to one of these types during type legalization.
+  CCIfSubtarget<"hasVector()",
+    CCIfType<[v16i8, v8i16, v4i32, v2i64, v4f32, v2f64],
+             CCAssignToReg<[V24, V26, V28, V30, V25, V27, V29, V31]>>>
 
   // ABI-compliant code returns long double by reference, but that conversion
   // is left to higher-level code.  Perhaps we could add an f128 definition
@@ -60,6 +81,25 @@ def CC_SystemZ : CallingConv<[
   CCIfType<[f32], CCAssignToReg<[F0S, F2S, F4S, F6S]>>,
   CCIfType<[f64], CCAssignToReg<[F0D, F2D, F4D, F6D]>>,
 
+  // The first 8 named vector arguments are passed in V24-V31.  Sub-128 vectors
+  // are passed in the same way, but they're widened to one of these types
+  // during type legalization.
+  CCIfSubtarget<"hasVector()",
+    CCIfType<[v16i8, v8i16, v4i32, v2i64, v4f32, v2f64],
+             CCIfFixed<CCAssignToReg<[V24, V26, V28, V30,
+                                      V25, V27, V29, V31]>>>>,
+
+  // However, sub-128 vectors which need to go on the stack occupy just a
+  // single 8-byte-aligned 8-byte stack slot.  Pass as i64.
+  CCIfSubtarget<"hasVector()",
+    CCIfType<[v16i8, v8i16, v4i32, v2i64, v4f32, v2f64],
+             CCIfShortVector<CCBitConvertToType<i64>>>>,
+
+  // Other vector arguments are passed in 8-byte-aligned 16-byte stack slots.
+  CCIfSubtarget<"hasVector()",
+    CCIfType<[v16i8, v8i16, v4i32, v2i64, v4f32, v2f64],
+             CCAssignToStack<16, 8>>>,
+
   // Other arguments are passed in 8-byte-aligned 8-byte stack slots.
   CCIfType<[i32, i64, f32, f64], CCAssignToStack<8, 8>>
 ]>;
diff --git a/lib/Target/SystemZ/SystemZFrameLowering.cpp b/lib/Target/SystemZ/SystemZFrameLowering.cpp
index eff4ae3baf3..397de472a6e 100644
--- a/lib/Target/SystemZ/SystemZFrameLowering.cpp
+++ b/lib/Target/SystemZ/SystemZFrameLowering.cpp
@@ -61,11 +61,12 @@ SystemZFrameLowering::getCalleeSavedSpillSlots(unsigned &NumEntries) const {
   return SpillOffsetTable;
 }
 
-void SystemZFrameLowering::
-processFunctionBeforeCalleeSavedScan(MachineFunction &MF,
-                                     RegScavenger *RS) const {
+void SystemZFrameLowering::determineCalleeSaves(MachineFunction &MF,
+                                                BitVector &SavedRegs,
+                                                RegScavenger *RS) const {
+  TargetFrameLowering::determineCalleeSaves(MF, SavedRegs, RS);
+
   MachineFrameInfo *MFFrame = MF.getFrameInfo();
-  MachineRegisterInfo &MRI = MF.getRegInfo();
   const TargetRegisterInfo *TRI = MF.getSubtarget().getRegisterInfo();
   bool HasFP = hasFP(MF);
   SystemZMachineFunctionInfo *MFI = MF.getInfo<SystemZMachineFunctionInfo>();
@@ -77,17 +78,17 @@ processFunctionBeforeCalleeSavedScan(MachineFunction &MF,
   // argument register R6D.
   if (IsVarArg)
     for (unsigned I = MFI->getVarArgsFirstGPR(); I < SystemZ::NumArgGPRs; ++I)
-      MRI.setPhysRegUsed(SystemZ::ArgGPRs[I]);
+      SavedRegs.set(SystemZ::ArgGPRs[I]);
 
   // If the function requires a frame pointer, record that the hard
   // frame pointer will be clobbered.
   if (HasFP)
-    MRI.setPhysRegUsed(SystemZ::R11D);
+    SavedRegs.set(SystemZ::R11D);
 
   // If the function calls other functions, record that the return
   // address register will be clobbered.
   if (MFFrame->hasCalls())
-    MRI.setPhysRegUsed(SystemZ::R14D);
+    SavedRegs.set(SystemZ::R14D);
 
   // If we are saving GPRs other than the stack pointer, we might as well
   // save and restore the stack pointer at the same time, via STMG and LMG.
@@ -96,8 +97,8 @@ processFunctionBeforeCalleeSavedScan(MachineFunction &MF,
   const MCPhysReg *CSRegs = TRI->getCalleeSavedRegs(&MF);
   for (unsigned I = 0; CSRegs[I]; ++I) {
     unsigned Reg = CSRegs[I];
-    if (SystemZ::GR64BitRegClass.contains(Reg) && MRI.isPhysRegUsed(Reg)) {
-      MRI.setPhysRegUsed(SystemZ::R15D);
+    if (SystemZ::GR64BitRegClass.contains(Reg) && SavedRegs.test(Reg)) {
+      SavedRegs.set(SystemZ::R15D);
       break;
     }
   }
@@ -309,8 +310,9 @@ static void emitIncrement(MachineBasicBlock &MBB,
   }
 }
 
-void SystemZFrameLowering::emitPrologue(MachineFunction &MF) const {
-  MachineBasicBlock &MBB = MF.front();
+void SystemZFrameLowering::emitPrologue(MachineFunction &MF,
+                                        MachineBasicBlock &MBB) const {
+  assert(&MF.front() == &MBB && "Shrink-wrapping not yet supported");
   MachineFrameInfo *MFFrame = MF.getFrameInfo();
   auto *ZII =
       static_cast<const SystemZInstrInfo *>(MF.getSubtarget().getInstrInfo());
diff --git a/lib/Target/SystemZ/SystemZFrameLowering.h b/lib/Target/SystemZ/SystemZFrameLowering.h
index cefa56fd74e..5ade757f17f 100644
--- a/lib/Target/SystemZ/SystemZFrameLowering.h
+++ b/lib/Target/SystemZ/SystemZFrameLowering.h
@@ -27,8 +27,8 @@ class SystemZFrameLowering : public TargetFrameLowering {
   bool isFPCloseToIncomingSP() const override { return false; }
   const SpillSlot *getCalleeSavedSpillSlots(unsigned &NumEntries) const
     override;
-  void processFunctionBeforeCalleeSavedScan(MachineFunction &MF,
-                                            RegScavenger *RS) const override;
+  void determineCalleeSaves(MachineFunction &MF, BitVector &SavedRegs,
+                            RegScavenger *RS) const override;
   bool spillCalleeSavedRegisters(MachineBasicBlock &MBB,
                                  MachineBasicBlock::iterator MBBI,
                                  const std::vector<CalleeSavedInfo> &CSI,
@@ -40,7 +40,7 @@ class SystemZFrameLowering : public TargetFrameLowering {
     override;
   void processFunctionBeforeFrameFinalized(MachineFunction &MF,
                                            RegScavenger *RS) const override;
-  void emitPrologue(MachineFunction &MF) const override;
+  void emitPrologue(MachineFunction &MF, MachineBasicBlock &MBB) const override;
   void emitEpilogue(MachineFunction &MF, MachineBasicBlock &MBB) const override;
   bool hasFP(const MachineFunction &MF) const override;
   int getFrameIndexOffset(const MachineFunction &MF, int FI) const override;
diff --git a/lib/Target/SystemZ/SystemZISelDAGToDAG.cpp b/lib/Target/SystemZ/SystemZISelDAGToDAG.cpp
index 1a58b531c03..75fd37f01a1 100644
--- a/lib/Target/SystemZ/SystemZISelDAGToDAG.cpp
+++ b/lib/Target/SystemZ/SystemZISelDAGToDAG.cpp
@@ -96,7 +96,10 @@ struct SystemZAddressingMode {
 
 // Return a mask with Count low bits set.
 static uint64_t allOnes(unsigned int Count) {
-  return Count == 0 ? 0 : (uint64_t(1) << (Count - 1) << 1) - 1;
+  assert(Count <= 64);
+  if (Count > 63)
+    return UINT64_MAX;
+  return (uint64_t(1) << Count) - 1;
 }
 
 // Represents operands 2 to 5 of the ROTATE AND ... SELECTED BITS operation
@@ -131,7 +134,7 @@ class SystemZDAGToDAGISel : public SelectionDAGISel {
 
   // Used by SystemZOperands.td to create integer constants.
   inline SDValue getImm(const SDNode *Node, uint64_t Imm) const {
-    return CurDAG->getTargetConstant(Imm, Node->getValueType(0));
+    return CurDAG->getTargetConstant(Imm, SDLoc(Node), Node->getValueType(0));
   }
 
   const SystemZTargetMachine &getTargetMachine() const {
@@ -255,6 +258,13 @@ class SystemZDAGToDAGISel : public SelectionDAGISel {
                          Addr, Base, Disp, Index);
   }
 
+  // Try to match Addr as an address with a base, 12-bit displacement
+  // and index, where the index is element Elem of a vector.
+  // Return true on success, storing the base, displacement and vector
+  // in Base, Disp and Index respectively.
+  bool selectBDVAddr12Only(SDValue Addr, SDValue Elem, SDValue &Base,
+                           SDValue &Disp, SDValue &Index) const;
+
   // Check whether (or Op (and X InsertMask)) is effectively an insertion
   // of X into bits InsertMask of some Y != Op.  Return true if so and
   // set Op to that Y.
@@ -292,6 +302,12 @@ class SystemZDAGToDAGISel : public SelectionDAGISel {
   SDNode *splitLargeImmediate(unsigned Opcode, SDNode *Node, SDValue Op0,
                               uint64_t UpperVal, uint64_t LowerVal);
 
+  // Try to use gather instruction Opcode to implement vector insertion N.
+  SDNode *tryGather(SDNode *N, unsigned Opcode);
+
+  // Try to use scatter instruction Opcode to implement store Store.
+  SDNode *tryScatter(StoreSDNode *Store, unsigned Opcode);
+
   // Return true if Load and Store are loads and stores of the same size
   // and are guaranteed not to overlap.  Such operations can be implemented
   // using block (SS-format) instructions.
@@ -596,7 +612,7 @@ void SystemZDAGToDAGISel::getAddressOperands(const SystemZAddressingMode &AM,
   }
 
   // Lower the displacement to a TargetConstant.
-  Disp = CurDAG->getTargetConstant(AM.Disp, VT);
+  Disp = CurDAG->getTargetConstant(AM.Disp, SDLoc(Base), VT);
 }
 
 void SystemZDAGToDAGISel::getAddressOperands(const SystemZAddressingMode &AM,
@@ -645,6 +661,30 @@ bool SystemZDAGToDAGISel::selectBDXAddr(SystemZAddressingMode::AddrForm Form,
   return true;
 }
 
+bool SystemZDAGToDAGISel::selectBDVAddr12Only(SDValue Addr, SDValue Elem,
+                                              SDValue &Base,
+                                              SDValue &Disp,
+                                              SDValue &Index) const {
+  SDValue Regs[2];
+  if (selectBDXAddr12Only(Addr, Regs[0], Disp, Regs[1]) &&
+      Regs[0].getNode() && Regs[1].getNode()) {
+    for (unsigned int I = 0; I < 2; ++I) {
+      Base = Regs[I];
+      Index = Regs[1 - I];
+      // We can't tell here whether the index vector has the right type
+      // for the access; the caller needs to do that instead.
+      if (Index.getOpcode() == ISD::ZERO_EXTEND)
+        Index = Index.getOperand(0);
+      if (Index.getOpcode() == ISD::EXTRACT_VECTOR_ELT &&
+          Index.getOperand(1) == Elem) {
+        Index = Index.getOperand(0);
+        return true;
+      }
+    }
+  }
+  return false;
+}
+
 bool SystemZDAGToDAGISel::detectOrAndInsertion(SDValue &Op,
                                                uint64_t InsertMask) const {
   // We're only interested in cases where the insertion is into some operand
@@ -864,7 +904,10 @@ SDValue SystemZDAGToDAGISel::convertTo(SDLoc DL, EVT VT, SDValue N) const {
 }
 
 SDNode *SystemZDAGToDAGISel::tryRISBGZero(SDNode *N) {
+  SDLoc DL(N);
   EVT VT = N->getValueType(0);
+  if (!VT.isInteger() || VT.getSizeInBits() > 64)
+    return nullptr;
   RxSBGOperands RISBG(SystemZ::RISBG, SDValue(N, 0));
   unsigned Count = 0;
   while (expandRxSBG(RISBG))
@@ -889,7 +932,7 @@ SDNode *SystemZDAGToDAGISel::tryRISBGZero(SDNode *N) {
       // Force the new mask into the DAG, since it may include known-one bits.
       auto *MaskN = cast<ConstantSDNode>(N->getOperand(1).getNode());
       if (MaskN->getZExtValue() != RISBG.Mask) {
-        SDValue NewMask = CurDAG->getConstant(RISBG.Mask, VT);
+        SDValue NewMask = CurDAG->getConstant(RISBG.Mask, DL, VT);
         N = CurDAG->UpdateNodeOperands(N, N->getOperand(0), NewMask);
         return SelectCode(N);
       }
@@ -909,17 +952,21 @@ SDNode *SystemZDAGToDAGISel::tryRISBGZero(SDNode *N) {
     RISBG.End &= 31;
   }
   SDValue Ops[5] = {
-    getUNDEF(SDLoc(N), OpcodeVT),
-    convertTo(SDLoc(N), OpcodeVT, RISBG.Input),
-    CurDAG->getTargetConstant(RISBG.Start, MVT::i32),
-    CurDAG->getTargetConstant(RISBG.End | 128, MVT::i32),
-    CurDAG->getTargetConstant(RISBG.Rotate, MVT::i32)
+    getUNDEF(DL, OpcodeVT),
+    convertTo(DL, OpcodeVT, RISBG.Input),
+    CurDAG->getTargetConstant(RISBG.Start, DL, MVT::i32),
+    CurDAG->getTargetConstant(RISBG.End | 128, DL, MVT::i32),
+    CurDAG->getTargetConstant(RISBG.Rotate, DL, MVT::i32)
   };
-  N = CurDAG->getMachineNode(Opcode, SDLoc(N), OpcodeVT, Ops);
-  return convertTo(SDLoc(N), VT, SDValue(N, 0)).getNode();
+  N = CurDAG->getMachineNode(Opcode, DL, OpcodeVT, Ops);
+  return convertTo(DL, VT, SDValue(N, 0)).getNode();
 }
 
 SDNode *SystemZDAGToDAGISel::tryRxSBG(SDNode *N, unsigned Opcode) {
+  SDLoc DL(N);
+  EVT VT = N->getValueType(0);
+  if (!VT.isInteger() || VT.getSizeInBits() > 64)
+    return nullptr;
   // Try treating each operand of N as the second operand of the RxSBG
   // and see which goes deepest.
   RxSBGOperands RxSBG[] = {
@@ -955,16 +1002,15 @@ SDNode *SystemZDAGToDAGISel::tryRxSBG(SDNode *N, unsigned Opcode) {
       Opcode = SystemZ::RISBGN;
   }
 
-  EVT VT = N->getValueType(0);
   SDValue Ops[5] = {
-    convertTo(SDLoc(N), MVT::i64, Op0),
-    convertTo(SDLoc(N), MVT::i64, RxSBG[I].Input),
-    CurDAG->getTargetConstant(RxSBG[I].Start, MVT::i32),
-    CurDAG->getTargetConstant(RxSBG[I].End, MVT::i32),
-    CurDAG->getTargetConstant(RxSBG[I].Rotate, MVT::i32)
+    convertTo(DL, MVT::i64, Op0),
+    convertTo(DL, MVT::i64, RxSBG[I].Input),
+    CurDAG->getTargetConstant(RxSBG[I].Start, DL, MVT::i32),
+    CurDAG->getTargetConstant(RxSBG[I].End, DL, MVT::i32),
+    CurDAG->getTargetConstant(RxSBG[I].Rotate, DL, MVT::i32)
   };
-  N = CurDAG->getMachineNode(Opcode, SDLoc(N), MVT::i64, Ops);
-  return convertTo(SDLoc(N), VT, SDValue(N, 0)).getNode();
+  N = CurDAG->getMachineNode(Opcode, DL, MVT::i64, Ops);
+  return convertTo(DL, VT, SDValue(N, 0)).getNode();
 }
 
 SDNode *SystemZDAGToDAGISel::splitLargeImmediate(unsigned Opcode, SDNode *Node,
@@ -972,16 +1018,81 @@ SDNode *SystemZDAGToDAGISel::splitLargeImmediate(unsigned Opcode, SDNode *Node,
                                                  uint64_t LowerVal) {
   EVT VT = Node->getValueType(0);
   SDLoc DL(Node);
-  SDValue Upper = CurDAG->getConstant(UpperVal, VT);
+  SDValue Upper = CurDAG->getConstant(UpperVal, DL, VT);
   if (Op0.getNode())
     Upper = CurDAG->getNode(Opcode, DL, VT, Op0, Upper);
   Upper = SDValue(Select(Upper.getNode()), 0);
 
-  SDValue Lower = CurDAG->getConstant(LowerVal, VT);
+  SDValue Lower = CurDAG->getConstant(LowerVal, DL, VT);
   SDValue Or = CurDAG->getNode(Opcode, DL, VT, Upper, Lower);
   return Or.getNode();
 }
 
+SDNode *SystemZDAGToDAGISel::tryGather(SDNode *N, unsigned Opcode) {
+  SDValue ElemV = N->getOperand(2);
+  auto *ElemN = dyn_cast<ConstantSDNode>(ElemV);
+  if (!ElemN)
+    return 0;
+
+  unsigned Elem = ElemN->getZExtValue();
+  EVT VT = N->getValueType(0);
+  if (Elem >= VT.getVectorNumElements())
+    return 0;
+
+  auto *Load = dyn_cast<LoadSDNode>(N->getOperand(1));
+  if (!Load || !Load->hasOneUse())
+    return 0;
+  if (Load->getMemoryVT().getSizeInBits() !=
+      Load->getValueType(0).getSizeInBits())
+    return 0;
+
+  SDValue Base, Disp, Index;
+  if (!selectBDVAddr12Only(Load->getBasePtr(), ElemV, Base, Disp, Index) ||
+      Index.getValueType() != VT.changeVectorElementTypeToInteger())
+    return 0;
+
+  SDLoc DL(Load);
+  SDValue Ops[] = {
+    N->getOperand(0), Base, Disp, Index,
+    CurDAG->getTargetConstant(Elem, DL, MVT::i32), Load->getChain()
+  };
+  SDNode *Res = CurDAG->getMachineNode(Opcode, DL, VT, MVT::Other, Ops);
+  ReplaceUses(SDValue(Load, 1), SDValue(Res, 1));
+  return Res;
+}
+
+SDNode *SystemZDAGToDAGISel::tryScatter(StoreSDNode *Store, unsigned Opcode) {
+  SDValue Value = Store->getValue();
+  if (Value.getOpcode() != ISD::EXTRACT_VECTOR_ELT)
+    return 0;
+  if (Store->getMemoryVT().getSizeInBits() !=
+      Value.getValueType().getSizeInBits())
+    return 0;
+
+  SDValue ElemV = Value.getOperand(1);
+  auto *ElemN = dyn_cast<ConstantSDNode>(ElemV);
+  if (!ElemN)
+    return 0;
+
+  SDValue Vec = Value.getOperand(0);
+  EVT VT = Vec.getValueType();
+  unsigned Elem = ElemN->getZExtValue();
+  if (Elem >= VT.getVectorNumElements())
+    return 0;
+
+  SDValue Base, Disp, Index;
+  if (!selectBDVAddr12Only(Store->getBasePtr(), ElemV, Base, Disp, Index) ||
+      Index.getValueType() != VT.changeVectorElementTypeToInteger())
+    return 0;
+
+  SDLoc DL(Store);
+  SDValue Ops[] = {
+    Vec, Base, Disp, Index, CurDAG->getTargetConstant(Elem, DL, MVT::i32),
+    Store->getChain()
+  };
+  return CurDAG->getMachineNode(Opcode, DL, MVT::Other, Ops);
+}
+
 bool SystemZDAGToDAGISel::canUseBlockOperation(StoreSDNode *Store,
                                                LoadSDNode *Load) const {
   // Check that the two memory operands have the same size.
@@ -1009,8 +1120,8 @@ bool SystemZDAGToDAGISel::canUseBlockOperation(StoreSDNode *Store,
   if (V1 == V2 && End1 == End2)
     return false;
 
-  return !AA->alias(AliasAnalysis::Location(V1, End1, Load->getAAInfo()),
-                    AliasAnalysis::Location(V2, End2, Store->getAAInfo()));
+  return !AA->alias(MemoryLocation(V1, End1, Load->getAAInfo()),
+                    MemoryLocation(V2, End2, Store->getAAInfo()));
 }
 
 bool SystemZDAGToDAGISel::storeLoadCanUseMVC(SDNode *N) const {
@@ -1111,13 +1222,33 @@ SDNode *SystemZDAGToDAGISel::Select(SDNode *Node) {
       uint64_t ConstCCMask =
         cast<ConstantSDNode>(CCMask.getNode())->getZExtValue();
       // Invert the condition.
-      CCMask = CurDAG->getConstant(ConstCCValid ^ ConstCCMask,
+      CCMask = CurDAG->getConstant(ConstCCValid ^ ConstCCMask, SDLoc(Node),
                                    CCMask.getValueType());
       SDValue Op4 = Node->getOperand(4);
       Node = CurDAG->UpdateNodeOperands(Node, Op1, Op0, CCValid, CCMask, Op4);
     }
     break;
   }
+
+  case ISD::INSERT_VECTOR_ELT: {
+    EVT VT = Node->getValueType(0);
+    unsigned ElemBitSize = VT.getVectorElementType().getSizeInBits();
+    if (ElemBitSize == 32)
+      ResNode = tryGather(Node, SystemZ::VGEF);
+    else if (ElemBitSize == 64)
+      ResNode = tryGather(Node, SystemZ::VGEG);
+    break;
+  }
+
+  case ISD::STORE: {
+    auto *Store = cast<StoreSDNode>(Node);
+    unsigned ElemBitSize = Store->getValue().getValueType().getSizeInBits();
+    if (ElemBitSize == 32)
+      ResNode = tryScatter(Store, SystemZ::VSCEF);
+    else if (ElemBitSize == 64)
+      ResNode = tryScatter(Store, SystemZ::VSCEG);
+    break;
+  }
   }
 
   // Select the default instruction
diff --git a/lib/Target/SystemZ/SystemZISelLowering.cpp b/lib/Target/SystemZ/SystemZISelLowering.cpp
index 21882cb5cea..dfe42345173 100644
--- a/lib/Target/SystemZ/SystemZISelLowering.cpp
+++ b/lib/Target/SystemZ/SystemZISelLowering.cpp
@@ -81,21 +81,35 @@ static MachineOperand earlyUseOperand(MachineOperand Op) {
   return Op;
 }
 
-SystemZTargetLowering::SystemZTargetLowering(const TargetMachine &tm,
+SystemZTargetLowering::SystemZTargetLowering(const TargetMachine &TM,
                                              const SystemZSubtarget &STI)
-    : TargetLowering(tm), Subtarget(STI) {
-  MVT PtrVT = getPointerTy();
+    : TargetLowering(TM), Subtarget(STI) {
+  MVT PtrVT = MVT::getIntegerVT(8 * TM.getPointerSize());
 
   // Set up the register classes.
   if (Subtarget.hasHighWord())
     addRegisterClass(MVT::i32, &SystemZ::GRX32BitRegClass);
   else
     addRegisterClass(MVT::i32, &SystemZ::GR32BitRegClass);
-  addRegisterClass(MVT::i64,  &SystemZ::GR64BitRegClass);
-  addRegisterClass(MVT::f32,  &SystemZ::FP32BitRegClass);
-  addRegisterClass(MVT::f64,  &SystemZ::FP64BitRegClass);
+  addRegisterClass(MVT::i64, &SystemZ::GR64BitRegClass);
+  if (Subtarget.hasVector()) {
+    addRegisterClass(MVT::f32, &SystemZ::VR32BitRegClass);
+    addRegisterClass(MVT::f64, &SystemZ::VR64BitRegClass);
+  } else {
+    addRegisterClass(MVT::f32, &SystemZ::FP32BitRegClass);
+    addRegisterClass(MVT::f64, &SystemZ::FP64BitRegClass);
+  }
   addRegisterClass(MVT::f128, &SystemZ::FP128BitRegClass);
 
+  if (Subtarget.hasVector()) {
+    addRegisterClass(MVT::v16i8, &SystemZ::VR128BitRegClass);
+    addRegisterClass(MVT::v8i16, &SystemZ::VR128BitRegClass);
+    addRegisterClass(MVT::v4i32, &SystemZ::VR128BitRegClass);
+    addRegisterClass(MVT::v2i64, &SystemZ::VR128BitRegClass);
+    addRegisterClass(MVT::v4f32, &SystemZ::VR128BitRegClass);
+    addRegisterClass(MVT::v2f64, &SystemZ::VR128BitRegClass);
+  }
+
   // Compute derived properties from the register classes
   computeRegisterProperties(Subtarget.getRegisterInfo());
 
@@ -111,7 +125,7 @@ SystemZTargetLowering::SystemZTargetLowering(const TargetMachine &tm,
   setSchedulingPreference(Sched::RegPressure);
 
   setBooleanContents(ZeroOrOneBooleanContent);
-  setBooleanVectorContents(ZeroOrOneBooleanContent); // FIXME: Is this correct?
+  setBooleanVectorContents(ZeroOrNegativeOneBooleanContent);
 
   // Instructions are strings of 2-byte aligned 2-byte values.
   setMinFunctionAlignment(2);
@@ -250,6 +264,90 @@ SystemZTargetLowering::SystemZTargetLowering(const TargetMachine &tm,
   // Handle prefetches with PFD or PFDRL.
   setOperationAction(ISD::PREFETCH, MVT::Other, Custom);
 
+  for (MVT VT : MVT::vector_valuetypes()) {
+    // Assume by default that all vector operations need to be expanded.
+    for (unsigned Opcode = 0; Opcode < ISD::BUILTIN_OP_END; ++Opcode)
+      if (getOperationAction(Opcode, VT) == Legal)
+        setOperationAction(Opcode, VT, Expand);
+
+    // Likewise all truncating stores and extending loads.
+    for (MVT InnerVT : MVT::vector_valuetypes()) {
+      setTruncStoreAction(VT, InnerVT, Expand);
+      setLoadExtAction(ISD::SEXTLOAD, VT, InnerVT, Expand);
+      setLoadExtAction(ISD::ZEXTLOAD, VT, InnerVT, Expand);
+      setLoadExtAction(ISD::EXTLOAD, VT, InnerVT, Expand);
+    }
+
+    if (isTypeLegal(VT)) {
+      // These operations are legal for anything that can be stored in a
+      // vector register, even if there is no native support for the format
+      // as such.  In particular, we can do these for v4f32 even though there
+      // are no specific instructions for that format.
+      setOperationAction(ISD::LOAD, VT, Legal);
+      setOperationAction(ISD::STORE, VT, Legal);
+      setOperationAction(ISD::VSELECT, VT, Legal);
+      setOperationAction(ISD::BITCAST, VT, Legal);
+      setOperationAction(ISD::UNDEF, VT, Legal);
+
+      // Likewise, except that we need to replace the nodes with something
+      // more specific.
+      setOperationAction(ISD::BUILD_VECTOR, VT, Custom);
+      setOperationAction(ISD::VECTOR_SHUFFLE, VT, Custom);
+    }
+  }
+
+  // Handle integer vector types.
+  for (MVT VT : MVT::integer_vector_valuetypes()) {
+    if (isTypeLegal(VT)) {
+      // These operations have direct equivalents.
+      setOperationAction(ISD::EXTRACT_VECTOR_ELT, VT, Legal);
+      setOperationAction(ISD::INSERT_VECTOR_ELT, VT, Legal);
+      setOperationAction(ISD::ADD, VT, Legal);
+      setOperationAction(ISD::SUB, VT, Legal);
+      if (VT != MVT::v2i64)
+        setOperationAction(ISD::MUL, VT, Legal);
+      setOperationAction(ISD::AND, VT, Legal);
+      setOperationAction(ISD::OR, VT, Legal);
+      setOperationAction(ISD::XOR, VT, Legal);
+      setOperationAction(ISD::CTPOP, VT, Custom);
+      setOperationAction(ISD::CTTZ, VT, Legal);
+      setOperationAction(ISD::CTLZ, VT, Legal);
+      setOperationAction(ISD::CTTZ_ZERO_UNDEF, VT, Custom);
+      setOperationAction(ISD::CTLZ_ZERO_UNDEF, VT, Custom);
+
+      // Convert a GPR scalar to a vector by inserting it into element 0.
+      setOperationAction(ISD::SCALAR_TO_VECTOR, VT, Custom);
+
+      // Use a series of unpacks for extensions.
+      setOperationAction(ISD::SIGN_EXTEND_VECTOR_INREG, VT, Custom);
+      setOperationAction(ISD::ZERO_EXTEND_VECTOR_INREG, VT, Custom);
+
+      // Detect shifts by a scalar amount and convert them into
+      // V*_BY_SCALAR.
+      setOperationAction(ISD::SHL, VT, Custom);
+      setOperationAction(ISD::SRA, VT, Custom);
+      setOperationAction(ISD::SRL, VT, Custom);
+
+      // At present ROTL isn't matched by DAGCombiner.  ROTR should be
+      // converted into ROTL.
+      setOperationAction(ISD::ROTL, VT, Expand);
+      setOperationAction(ISD::ROTR, VT, Expand);
+
+      // Map SETCCs onto one of VCE, VCH or VCHL, swapping the operands
+      // and inverting the result as necessary.
+      setOperationAction(ISD::SETCC, VT, Custom);
+    }
+  }
+
+  if (Subtarget.hasVector()) {
+    // There should be no need to check for float types other than v2f64
+    // since <2 x f32> isn't a legal type.
+    setOperationAction(ISD::FP_TO_SINT, MVT::v2i64, Legal);
+    setOperationAction(ISD::FP_TO_UINT, MVT::v2i64, Legal);
+    setOperationAction(ISD::SINT_TO_FP, MVT::v2i64, Legal);
+    setOperationAction(ISD::UINT_TO_FP, MVT::v2i64, Legal);
+  }
+
   // Handle floating-point types.
   for (unsigned I = MVT::FIRST_FP_VALUETYPE;
        I <= MVT::LAST_FP_VALUETYPE;
@@ -275,6 +373,36 @@ SystemZTargetLowering::SystemZTargetLowering(const TargetMachine &tm,
     }
   }
 
+  // Handle floating-point vector types.
+  if (Subtarget.hasVector()) {
+    // Scalar-to-vector conversion is just a subreg.
+    setOperationAction(ISD::SCALAR_TO_VECTOR, MVT::v4f32, Legal);
+    setOperationAction(ISD::SCALAR_TO_VECTOR, MVT::v2f64, Legal);
+
+    // Some insertions and extractions can be done directly but others
+    // need to go via integers.
+    setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v4f32, Custom);
+    setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v2f64, Custom);
+    setOperationAction(ISD::EXTRACT_VECTOR_ELT, MVT::v4f32, Custom);
+    setOperationAction(ISD::EXTRACT_VECTOR_ELT, MVT::v2f64, Custom);
+
+    // These operations have direct equivalents.
+    setOperationAction(ISD::FADD, MVT::v2f64, Legal);
+    setOperationAction(ISD::FNEG, MVT::v2f64, Legal);
+    setOperationAction(ISD::FSUB, MVT::v2f64, Legal);
+    setOperationAction(ISD::FMUL, MVT::v2f64, Legal);
+    setOperationAction(ISD::FMA, MVT::v2f64, Legal);
+    setOperationAction(ISD::FDIV, MVT::v2f64, Legal);
+    setOperationAction(ISD::FABS, MVT::v2f64, Legal);
+    setOperationAction(ISD::FSQRT, MVT::v2f64, Legal);
+    setOperationAction(ISD::FRINT, MVT::v2f64, Legal);
+    setOperationAction(ISD::FNEARBYINT, MVT::v2f64, Legal);
+    setOperationAction(ISD::FFLOOR, MVT::v2f64, Legal);
+    setOperationAction(ISD::FCEIL, MVT::v2f64, Legal);
+    setOperationAction(ISD::FTRUNC, MVT::v2f64, Legal);
+    setOperationAction(ISD::FROUND, MVT::v2f64, Legal);
+  }
+
   // We have fused multiply-addition for f32 and f64 but not f128.
   setOperationAction(ISD::FMA, MVT::f32,  Legal);
   setOperationAction(ISD::FMA, MVT::f64,  Legal);
@@ -293,8 +421,10 @@ SystemZTargetLowering::SystemZTargetLowering(const TargetMachine &tm,
 
   // We have 64-bit FPR<->GPR moves, but need special handling for
   // 32-bit forms.
-  setOperationAction(ISD::BITCAST, MVT::i32, Custom);
-  setOperationAction(ISD::BITCAST, MVT::f32, Custom);
+  if (!Subtarget.hasVector()) {
+    setOperationAction(ISD::BITCAST, MVT::i32, Custom);
+    setOperationAction(ISD::BITCAST, MVT::f32, Custom);
+  }
 
   // VASTART and VACOPY need to deal with the SystemZ-specific varargs
   // structure, but VAEND is a no-op.
@@ -304,9 +434,13 @@ SystemZTargetLowering::SystemZTargetLowering(const TargetMachine &tm,
 
   // Codes for which we want to perform some z-specific combinations.
   setTargetDAGCombine(ISD::SIGN_EXTEND);
+  setTargetDAGCombine(ISD::STORE);
+  setTargetDAGCombine(ISD::EXTRACT_VECTOR_ELT);
+  setTargetDAGCombine(ISD::FP_ROUND);
 
   // Handle intrinsics.
   setOperationAction(ISD::INTRINSIC_W_CHAIN, MVT::Other, Custom);
+  setOperationAction(ISD::INTRINSIC_WO_CHAIN, MVT::Other, Custom);
 
   // We want to use MVC in preference to even a single load/store pair.
   MaxStoresPerMemcpy = 0;
@@ -321,7 +455,8 @@ SystemZTargetLowering::SystemZTargetLowering(const TargetMachine &tm,
   MaxStoresPerMemsetOptSize = 0;
 }
 
-EVT SystemZTargetLowering::getSetCCResultType(LLVMContext &, EVT VT) const {
+EVT SystemZTargetLowering::getSetCCResultType(const DataLayout &DL,
+                                              LLVMContext &, EVT VT) const {
   if (!VT.isVector())
     return MVT::i32;
   return VT.changeVectorElementTypeToInteger();
@@ -372,9 +507,10 @@ bool SystemZTargetLowering::allowsMisalignedMemoryAccesses(EVT VT,
     *Fast = true;
   return true;
 }
-  
-bool SystemZTargetLowering::isLegalAddressingMode(const AddrMode &AM,
-                                                  Type *Ty) const {
+
+bool SystemZTargetLowering::isLegalAddressingMode(const DataLayout &DL,
+                                                  const AddrMode &AM, Type *Ty,
+                                                  unsigned AS) const {
   // Punt on globals for now, although they can be used in limited
   // RELATIVE LONG cases.
   if (AM.BaseGV)
@@ -409,7 +545,7 @@ bool SystemZTargetLowering::isTruncateFree(EVT FromVT, EVT ToVT) const {
 //===----------------------------------------------------------------------===//
 
 TargetLowering::ConstraintType
-SystemZTargetLowering::getConstraintType(const std::string &Constraint) const {
+SystemZTargetLowering::getConstraintType(StringRef Constraint) const {
   if (Constraint.size() == 1) {
     switch (Constraint[0]) {
     case 'a': // Address register
@@ -506,13 +642,14 @@ getSingleConstraintMatchWeight(AsmOperandInfo &info,
 // has already been verified.  MC is the class associated with "t" and
 // Map maps 0-based register numbers to LLVM register numbers.
 static std::pair<unsigned, const TargetRegisterClass *>
-parseRegisterNumber(const std::string &Constraint,
-                    const TargetRegisterClass *RC, const unsigned *Map) {
+parseRegisterNumber(StringRef Constraint, const TargetRegisterClass *RC,
+                    const unsigned *Map) {
   assert(*(Constraint.end()-1) == '}' && "Missing '}'");
   if (isdigit(Constraint[2])) {
-    std::string Suffix(Constraint.data() + 2, Constraint.size() - 2);
-    unsigned Index = atoi(Suffix.c_str());
-    if (Index < 16 && Map[Index])
+    unsigned Index;
+    bool Failed =
+        Constraint.slice(2, Constraint.size() - 1).getAsInteger(10, Index);
+    if (!Failed && Index < 16 && Map[Index])
       return std::make_pair(Map[Index], RC);
   }
   return std::make_pair(0U, nullptr);
@@ -520,8 +657,7 @@ parseRegisterNumber(const std::string &Constraint,
 
 std::pair<unsigned, const TargetRegisterClass *>
 SystemZTargetLowering::getRegForInlineAsmConstraint(
-    const TargetRegisterInfo *TRI, const std::string &Constraint,
-    MVT VT) const {
+    const TargetRegisterInfo *TRI, StringRef Constraint, MVT VT) const {
   if (Constraint.size() == 1) {
     // GCC Constraint Letters
     switch (Constraint[0]) {
@@ -552,7 +688,7 @@ SystemZTargetLowering::getRegForInlineAsmConstraint(
       return std::make_pair(0U, &SystemZ::FP32BitRegClass);
     }
   }
-  if (Constraint[0] == '{') {
+  if (Constraint.size() > 0 && Constraint[0] == '{') {
     // We need to override the default register parsing for GPRs and FPRs
     // because the interpretation depends on VT.  The internal names of
     // the registers are also different from the external names
@@ -591,35 +727,35 @@ LowerAsmOperandForConstraint(SDValue Op, std::string &Constraint,
     case 'I': // Unsigned 8-bit constant
       if (auto *C = dyn_cast<ConstantSDNode>(Op))
         if (isUInt<8>(C->getZExtValue()))
-          Ops.push_back(DAG.getTargetConstant(C->getZExtValue(),
+          Ops.push_back(DAG.getTargetConstant(C->getZExtValue(), SDLoc(Op),
                                               Op.getValueType()));
       return;
 
     case 'J': // Unsigned 12-bit constant
       if (auto *C = dyn_cast<ConstantSDNode>(Op))
         if (isUInt<12>(C->getZExtValue()))
-          Ops.push_back(DAG.getTargetConstant(C->getZExtValue(),
+          Ops.push_back(DAG.getTargetConstant(C->getZExtValue(), SDLoc(Op),
                                               Op.getValueType()));
       return;
 
     case 'K': // Signed 16-bit constant
       if (auto *C = dyn_cast<ConstantSDNode>(Op))
         if (isInt<16>(C->getSExtValue()))
-          Ops.push_back(DAG.getTargetConstant(C->getSExtValue(),
+          Ops.push_back(DAG.getTargetConstant(C->getSExtValue(), SDLoc(Op),
                                               Op.getValueType()));
       return;
 
     case 'L': // Signed 20-bit displacement (on all targets we support)
       if (auto *C = dyn_cast<ConstantSDNode>(Op))
         if (isInt<20>(C->getSExtValue()))
-          Ops.push_back(DAG.getTargetConstant(C->getSExtValue(),
+          Ops.push_back(DAG.getTargetConstant(C->getSExtValue(), SDLoc(Op),
                                               Op.getValueType()));
       return;
 
     case 'M': // 0x7fffffff
       if (auto *C = dyn_cast<ConstantSDNode>(Op))
         if (C->getZExtValue() == 0x7fffffff)
-          Ops.push_back(DAG.getTargetConstant(C->getZExtValue(),
+          Ops.push_back(DAG.getTargetConstant(C->getZExtValue(), SDLoc(Op),
                                               Op.getValueType()));
       return;
     }
@@ -644,6 +780,24 @@ bool SystemZTargetLowering::mayBeEmittedAsTailCall(CallInst *CI) const {
   return true;
 }
 
+// We do not yet support 128-bit single-element vector types.  If the user
+// attempts to use such types as function argument or return type, prefer
+// to error out instead of emitting code violating the ABI.
+static void VerifyVectorType(MVT VT, EVT ArgVT) {
+  if (ArgVT.isVector() && !VT.isVector())
+    report_fatal_error("Unsupported vector argument or return type");
+}
+
+static void VerifyVectorTypes(const SmallVectorImpl<ISD::InputArg> &Ins) {
+  for (unsigned i = 0; i < Ins.size(); ++i)
+    VerifyVectorType(Ins[i].VT, Ins[i].ArgVT);
+}
+
+static void VerifyVectorTypes(const SmallVectorImpl<ISD::OutputArg> &Outs) {
+  for (unsigned i = 0; i < Outs.size(); ++i)
+    VerifyVectorType(Outs[i].VT, Outs[i].ArgVT);
+}
+
 // Value is a value that has been passed to us in the location described by VA
 // (and so has type VA.getLocVT()).  Convert Value to VA.getValVT(), chaining
 // any loads onto Chain.
@@ -664,7 +818,15 @@ static SDValue convertLocVTToValVT(SelectionDAG &DAG, SDLoc DL,
   else if (VA.getLocInfo() == CCValAssign::Indirect)
     Value = DAG.getLoad(VA.getValVT(), DL, Chain, Value,
                         MachinePointerInfo(), false, false, false, 0);
-  else
+  else if (VA.getLocInfo() == CCValAssign::BCvt) {
+    // If this is a short vector argument loaded from the stack,
+    // extend from i64 to full vector size and then bitcast.
+    assert(VA.getLocVT() == MVT::i64);
+    assert(VA.getValVT().isVector());
+    Value = DAG.getNode(ISD::BUILD_VECTOR, DL, MVT::v2i64,
+                        Value, DAG.getUNDEF(MVT::i64));
+    Value = DAG.getNode(ISD::BITCAST, DL, VA.getValVT(), Value);
+  } else
     assert(VA.getLocInfo() == CCValAssign::Full && "Unsupported getLocInfo");
   return Value;
 }
@@ -681,6 +843,14 @@ static SDValue convertValVTToLocVT(SelectionDAG &DAG, SDLoc DL,
     return DAG.getNode(ISD::ZERO_EXTEND, DL, VA.getLocVT(), Value);
   case CCValAssign::AExt:
     return DAG.getNode(ISD::ANY_EXTEND, DL, VA.getLocVT(), Value);
+  case CCValAssign::BCvt:
+    // If this is a short vector argument to be stored to the stack,
+    // bitcast to v2i64 and then extract first element.
+    assert(VA.getLocVT() == MVT::i64);
+    assert(VA.getValVT().isVector());
+    Value = DAG.getNode(ISD::BITCAST, DL, MVT::v2i64, Value);
+    return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, VA.getLocVT(), Value,
+                       DAG.getConstant(0, DL, MVT::i32));
   case CCValAssign::Full:
     return Value;
   default:
@@ -701,9 +871,13 @@ LowerFormalArguments(SDValue Chain, CallingConv::ID CallConv, bool IsVarArg,
   auto *TFL =
       static_cast<const SystemZFrameLowering *>(Subtarget.getFrameLowering());
 
+  // Detect unsupported vector argument types.
+  if (Subtarget.hasVector())
+    VerifyVectorTypes(Ins);
+
   // Assign locations to all of the incoming arguments.
   SmallVector<CCValAssign, 16> ArgLocs;
-  CCState CCInfo(CallConv, IsVarArg, MF, ArgLocs, *DAG.getContext());
+  SystemZCCState CCInfo(CallConv, IsVarArg, MF, ArgLocs, *DAG.getContext());
   CCInfo.AnalyzeFormalArguments(Ins, CC_SystemZ);
 
   unsigned NumFixedGPRs = 0;
@@ -735,6 +909,14 @@ LowerFormalArguments(SDValue Chain, CallingConv::ID CallConv, bool IsVarArg,
         NumFixedFPRs += 1;
         RC = &SystemZ::FP64BitRegClass;
         break;
+      case MVT::v16i8:
+      case MVT::v8i16:
+      case MVT::v4i32:
+      case MVT::v2i64:
+      case MVT::v4f32:
+      case MVT::v2f64:
+        RC = &SystemZ::VR128BitRegClass;
+        break;
       }
 
       unsigned VReg = MRI.createVirtualRegister(RC);
@@ -750,10 +932,11 @@ LowerFormalArguments(SDValue Chain, CallingConv::ID CallConv, bool IsVarArg,
       // Create the SelectionDAG nodes corresponding to a load
       // from this parameter.  Unpromoted ints and floats are
       // passed as right-justified 8-byte values.
-      EVT PtrVT = getPointerTy();
+      EVT PtrVT = getPointerTy(DAG.getDataLayout());
       SDValue FIN = DAG.getFrameIndex(FI, PtrVT);
       if (VA.getLocVT() == MVT::i32 || VA.getLocVT() == MVT::f32)
-        FIN = DAG.getNode(ISD::ADD, DL, PtrVT, FIN, DAG.getIntPtrConstant(4));
+        FIN = DAG.getNode(ISD::ADD, DL, PtrVT, FIN,
+                          DAG.getIntPtrConstant(4, DL));
       ArgValue = DAG.getLoad(LocVT, DL, Chain, FIN,
                              MachinePointerInfo::getFixedStack(FI),
                              false, false, false, 0);
@@ -787,7 +970,7 @@ LowerFormalArguments(SDValue Chain, CallingConv::ID CallConv, bool IsVarArg,
       for (unsigned I = NumFixedFPRs; I < SystemZ::NumArgFPRs; ++I) {
         unsigned Offset = TFL->getRegSpillOffset(SystemZ::ArgFPRs[I]);
         int FI = MFI->CreateFixedObject(8, RegSaveOffset + Offset, true);
-        SDValue FIN = DAG.getFrameIndex(FI, getPointerTy());
+        SDValue FIN = DAG.getFrameIndex(FI, getPointerTy(DAG.getDataLayout()));
         unsigned VReg = MF.addLiveIn(SystemZ::ArgFPRs[I],
                                      &SystemZ::FP64BitRegClass);
         SDValue ArgValue = DAG.getCopyFromReg(Chain, DL, VReg, MVT::f64);
@@ -837,11 +1020,17 @@ SystemZTargetLowering::LowerCall(CallLoweringInfo &CLI,
   CallingConv::ID CallConv = CLI.CallConv;
   bool IsVarArg = CLI.IsVarArg;
   MachineFunction &MF = DAG.getMachineFunction();
-  EVT PtrVT = getPointerTy();
+  EVT PtrVT = getPointerTy(MF.getDataLayout());
+
+  // Detect unsupported vector argument and return types.
+  if (Subtarget.hasVector()) {
+    VerifyVectorTypes(Outs);
+    VerifyVectorTypes(Ins);
+  }
 
   // Analyze the operands of the call, assigning locations to each operand.
   SmallVector<CCValAssign, 16> ArgLocs;
-  CCState ArgCCInfo(CallConv, IsVarArg, MF, ArgLocs, *DAG.getContext());
+  SystemZCCState ArgCCInfo(CallConv, IsVarArg, MF, ArgLocs, *DAG.getContext());
   ArgCCInfo.AnalyzeCallOperands(Outs, CC_SystemZ);
 
   // We don't support GuaranteedTailCallOpt, only automatically-detected
@@ -854,7 +1043,8 @@ SystemZTargetLowering::LowerCall(CallLoweringInfo &CLI,
 
   // Mark the start of the call.
   if (!IsTailCall)
-    Chain = DAG.getCALLSEQ_START(Chain, DAG.getConstant(NumBytes, PtrVT, true),
+    Chain = DAG.getCALLSEQ_START(Chain,
+                                 DAG.getConstant(NumBytes, DL, PtrVT, true),
                                  DL);
 
   // Copy argument values to their designated locations.
@@ -890,7 +1080,7 @@ SystemZTargetLowering::LowerCall(CallLoweringInfo &CLI,
       if (VA.getLocVT() == MVT::i32 || VA.getLocVT() == MVT::f32)
         Offset += 4;
       SDValue Address = DAG.getNode(ISD::ADD, DL, PtrVT, StackPtr,
-                                    DAG.getIntPtrConstant(Offset));
+                                    DAG.getIntPtrConstant(Offset, DL));
 
       // Emit the store.
       MemOpChains.push_back(DAG.getStore(Chain, DL, ArgValue, Address,
@@ -956,8 +1146,8 @@ SystemZTargetLowering::LowerCall(CallLoweringInfo &CLI,
 
   // Mark the end of the call, which is glued to the call itself.
   Chain = DAG.getCALLSEQ_END(Chain,
-                             DAG.getConstant(NumBytes, PtrVT, true),
-                             DAG.getConstant(0, PtrVT, true),
+                             DAG.getConstant(NumBytes, DL, PtrVT, true),
+                             DAG.getConstant(0, DL, PtrVT, true),
                              Glue, DL);
   Glue = Chain.getValue(1);
 
@@ -992,6 +1182,10 @@ SystemZTargetLowering::LowerReturn(SDValue Chain,
                                    SDLoc DL, SelectionDAG &DAG) const {
   MachineFunction &MF = DAG.getMachineFunction();
 
+  // Detect unsupported vector return types.
+  if (Subtarget.hasVector())
+    VerifyVectorTypes(Outs);
+
   // Assign locations to each returned value.
   SmallVector<CCValAssign, 16> RetLocs;
   CCState RetCCInfo(CallConv, IsVarArg, MF, RetLocs, *DAG.getContext());
@@ -1062,6 +1256,143 @@ static bool isIntrinsicWithCCAndChain(SDValue Op, unsigned &Opcode,
   }
 }
 
+// Return true if Op is an intrinsic node without chain that returns the
+// CC value as its final argument.  Provide the associated SystemZISD
+// opcode and the mask of valid CC values if so.
+static bool isIntrinsicWithCC(SDValue Op, unsigned &Opcode, unsigned &CCValid) {
+  unsigned Id = cast<ConstantSDNode>(Op.getOperand(0))->getZExtValue();
+  switch (Id) {
+  case Intrinsic::s390_vpkshs:
+  case Intrinsic::s390_vpksfs:
+  case Intrinsic::s390_vpksgs:
+    Opcode = SystemZISD::PACKS_CC;
+    CCValid = SystemZ::CCMASK_VCMP;
+    return true;
+
+  case Intrinsic::s390_vpklshs:
+  case Intrinsic::s390_vpklsfs:
+  case Intrinsic::s390_vpklsgs:
+    Opcode = SystemZISD::PACKLS_CC;
+    CCValid = SystemZ::CCMASK_VCMP;
+    return true;
+
+  case Intrinsic::s390_vceqbs:
+  case Intrinsic::s390_vceqhs:
+  case Intrinsic::s390_vceqfs:
+  case Intrinsic::s390_vceqgs:
+    Opcode = SystemZISD::VICMPES;
+    CCValid = SystemZ::CCMASK_VCMP;
+    return true;
+
+  case Intrinsic::s390_vchbs:
+  case Intrinsic::s390_vchhs:
+  case Intrinsic::s390_vchfs:
+  case Intrinsic::s390_vchgs:
+    Opcode = SystemZISD::VICMPHS;
+    CCValid = SystemZ::CCMASK_VCMP;
+    return true;
+
+  case Intrinsic::s390_vchlbs:
+  case Intrinsic::s390_vchlhs:
+  case Intrinsic::s390_vchlfs:
+  case Intrinsic::s390_vchlgs:
+    Opcode = SystemZISD::VICMPHLS;
+    CCValid = SystemZ::CCMASK_VCMP;
+    return true;
+
+  case Intrinsic::s390_vtm:
+    Opcode = SystemZISD::VTM;
+    CCValid = SystemZ::CCMASK_VCMP;
+    return true;
+
+  case Intrinsic::s390_vfaebs:
+  case Intrinsic::s390_vfaehs:
+  case Intrinsic::s390_vfaefs:
+    Opcode = SystemZISD::VFAE_CC;
+    CCValid = SystemZ::CCMASK_ANY;
+    return true;
+
+  case Intrinsic::s390_vfaezbs:
+  case Intrinsic::s390_vfaezhs:
+  case Intrinsic::s390_vfaezfs:
+    Opcode = SystemZISD::VFAEZ_CC;
+    CCValid = SystemZ::CCMASK_ANY;
+    return true;
+
+  case Intrinsic::s390_vfeebs:
+  case Intrinsic::s390_vfeehs:
+  case Intrinsic::s390_vfeefs:
+    Opcode = SystemZISD::VFEE_CC;
+    CCValid = SystemZ::CCMASK_ANY;
+    return true;
+
+  case Intrinsic::s390_vfeezbs:
+  case Intrinsic::s390_vfeezhs:
+  case Intrinsic::s390_vfeezfs:
+    Opcode = SystemZISD::VFEEZ_CC;
+    CCValid = SystemZ::CCMASK_ANY;
+    return true;
+
+  case Intrinsic::s390_vfenebs:
+  case Intrinsic::s390_vfenehs:
+  case Intrinsic::s390_vfenefs:
+    Opcode = SystemZISD::VFENE_CC;
+    CCValid = SystemZ::CCMASK_ANY;
+    return true;
+
+  case Intrinsic::s390_vfenezbs:
+  case Intrinsic::s390_vfenezhs:
+  case Intrinsic::s390_vfenezfs:
+    Opcode = SystemZISD::VFENEZ_CC;
+    CCValid = SystemZ::CCMASK_ANY;
+    return true;
+
+  case Intrinsic::s390_vistrbs:
+  case Intrinsic::s390_vistrhs:
+  case Intrinsic::s390_vistrfs:
+    Opcode = SystemZISD::VISTR_CC;
+    CCValid = SystemZ::CCMASK_0 | SystemZ::CCMASK_3;
+    return true;
+
+  case Intrinsic::s390_vstrcbs:
+  case Intrinsic::s390_vstrchs:
+  case Intrinsic::s390_vstrcfs:
+    Opcode = SystemZISD::VSTRC_CC;
+    CCValid = SystemZ::CCMASK_ANY;
+    return true;
+
+  case Intrinsic::s390_vstrczbs:
+  case Intrinsic::s390_vstrczhs:
+  case Intrinsic::s390_vstrczfs:
+    Opcode = SystemZISD::VSTRCZ_CC;
+    CCValid = SystemZ::CCMASK_ANY;
+    return true;
+
+  case Intrinsic::s390_vfcedbs:
+    Opcode = SystemZISD::VFCMPES;
+    CCValid = SystemZ::CCMASK_VCMP;
+    return true;
+
+  case Intrinsic::s390_vfchdbs:
+    Opcode = SystemZISD::VFCMPHS;
+    CCValid = SystemZ::CCMASK_VCMP;
+    return true;
+
+  case Intrinsic::s390_vfchedbs:
+    Opcode = SystemZISD::VFCMPHES;
+    CCValid = SystemZ::CCMASK_VCMP;
+    return true;
+
+  case Intrinsic::s390_vftcidb:
+    Opcode = SystemZISD::VFTCI;
+    CCValid = SystemZ::CCMASK_VCMP;
+    return true;
+
+  default:
+    return false;
+  }
+}
+
 // Emit an intrinsic with chain with a glued value instead of its CC result.
 static SDValue emitIntrinsicWithChainAndGlue(SelectionDAG &DAG, SDValue Op,
                                              unsigned Opcode) {
@@ -1082,6 +1413,23 @@ static SDValue emitIntrinsicWithChainAndGlue(SelectionDAG &DAG, SDValue Op,
   return Intr;
 }
 
+// Emit an intrinsic with a glued value instead of its CC result.
+static SDValue emitIntrinsicWithGlue(SelectionDAG &DAG, SDValue Op,
+                                     unsigned Opcode) {
+  // Copy all operands except the intrinsic ID.
+  unsigned NumOps = Op.getNumOperands();
+  SmallVector<SDValue, 6> Ops;
+  Ops.reserve(NumOps - 1);
+  for (unsigned I = 1; I < NumOps; ++I)
+    Ops.push_back(Op.getOperand(I));
+
+  if (Op->getNumValues() == 1)
+    return DAG.getNode(Opcode, SDLoc(Op), MVT::Glue, Ops);
+  assert(Op->getNumValues() == 2 && "Expected exactly one non-CC result");
+  SDVTList RawVTs = DAG.getVTList(Op->getValueType(0), MVT::Glue);
+  return DAG.getNode(Opcode, SDLoc(Op), RawVTs, Ops);
+}
+
 // CC is a comparison that will be implemented using an integer or
 // floating-point comparison.  Return the condition code mask for
 // a branch on true.  In the integer case, CCMASK_CMP_UO is set for
@@ -1179,7 +1527,7 @@ static IPMConversion getIPMConversion(unsigned CCValid, unsigned CCMask) {
 
 // If C can be converted to a comparison against zero, adjust the operands
 // as necessary.
-static void adjustZeroCmp(SelectionDAG &DAG, Comparison &C) {
+static void adjustZeroCmp(SelectionDAG &DAG, SDLoc DL, Comparison &C) {
   if (C.ICmpType == SystemZICMP::UnsignedOnly)
     return;
 
@@ -1193,13 +1541,13 @@ static void adjustZeroCmp(SelectionDAG &DAG, Comparison &C) {
       (Value == 1 && C.CCMask == SystemZ::CCMASK_CMP_LT) ||
       (Value == 1 && C.CCMask == SystemZ::CCMASK_CMP_GE)) {
     C.CCMask ^= SystemZ::CCMASK_CMP_EQ;
-    C.Op1 = DAG.getConstant(0, C.Op1.getValueType());
+    C.Op1 = DAG.getConstant(0, DL, C.Op1.getValueType());
   }
 }
 
 // If a comparison described by C is suitable for CLI(Y), CHHSI or CLHHSI,
 // adjust the operands as necessary.
-static void adjustSubwordCmp(SelectionDAG &DAG, Comparison &C) {
+static void adjustSubwordCmp(SelectionDAG &DAG, SDLoc DL, Comparison &C) {
   // For us to make any changes, it must a comparison between a single-use
   // load and a constant.
   if (!C.Op0.hasOneUse() ||
@@ -1264,7 +1612,7 @@ static void adjustSubwordCmp(SelectionDAG &DAG, Comparison &C) {
   // Make sure that the second operand is an i32 with the right value.
   if (C.Op1.getValueType() != MVT::i32 ||
       Value != ConstOp1->getZExtValue())
-    C.Op1 = DAG.getConstant(Value, MVT::i32);
+    C.Op1 = DAG.getConstant(Value, DL, MVT::i32);
 }
 
 // Return true if Op is either an unextended load, or a load suitable
@@ -1360,7 +1708,7 @@ static unsigned reverseCCMask(unsigned CCMask) {
 // Check whether C tests for equality between X and Y and whether X - Y
 // or Y - X is also computed.  In that case it's better to compare the
 // result of the subtraction against zero.
-static void adjustForSubtraction(SelectionDAG &DAG, Comparison &C) {
+static void adjustForSubtraction(SelectionDAG &DAG, SDLoc DL, Comparison &C) {
   if (C.CCMask == SystemZ::CCMASK_CMP_EQ ||
       C.CCMask == SystemZ::CCMASK_CMP_NE) {
     for (auto I = C.Op0->use_begin(), E = C.Op0->use_end(); I != E; ++I) {
@@ -1369,7 +1717,7 @@ static void adjustForSubtraction(SelectionDAG &DAG, Comparison &C) {
           ((N->getOperand(0) == C.Op0 && N->getOperand(1) == C.Op1) ||
            (N->getOperand(0) == C.Op1 && N->getOperand(1) == C.Op0))) {
         C.Op0 = SDValue(N, 0);
-        C.Op1 = DAG.getConstant(0, N->getValueType(0));
+        C.Op1 = DAG.getConstant(0, DL, N->getValueType(0));
         return;
       }
     }
@@ -1425,7 +1773,7 @@ static void adjustForLTGFR(Comparison &C) {
 // If C compares the truncation of an extending load, try to compare
 // the untruncated value instead.  This exposes more opportunities to
 // reuse CC.
-static void adjustICmpTruncate(SelectionDAG &DAG, Comparison &C) {
+static void adjustICmpTruncate(SelectionDAG &DAG, SDLoc DL, Comparison &C) {
   if (C.Op0.getOpcode() == ISD::TRUNCATE &&
       C.Op0.getOperand(0).getOpcode() == ISD::LOAD &&
       C.Op1.getOpcode() == ISD::Constant &&
@@ -1437,7 +1785,7 @@ static void adjustICmpTruncate(SelectionDAG &DAG, Comparison &C) {
       if ((Type == ISD::ZEXTLOAD && C.ICmpType != SystemZICMP::SignedOnly) ||
           (Type == ISD::SEXTLOAD && C.ICmpType != SystemZICMP::UnsignedOnly)) {
         C.Op0 = C.Op0.getOperand(0);
-        C.Op1 = DAG.getConstant(0, C.Op0.getValueType());
+        C.Op1 = DAG.getConstant(0, DL, C.Op0.getValueType());
       }
     }
   }
@@ -1556,7 +1904,7 @@ static unsigned getTestUnderMaskCond(unsigned BitSize, unsigned CCMask,
 
 // See whether C can be implemented as a TEST UNDER MASK instruction.
 // Update the arguments with the TM version if so.
-static void adjustForTestUnderMask(SelectionDAG &DAG, Comparison &C) {
+static void adjustForTestUnderMask(SelectionDAG &DAG, SDLoc DL, Comparison &C) {
   // Check that we have a comparison with a constant.
   auto *ConstOp1 = dyn_cast<ConstantSDNode>(C.Op1);
   if (!ConstOp1)
@@ -1634,7 +1982,7 @@ static void adjustForTestUnderMask(SelectionDAG &DAG, Comparison &C) {
   if (Mask && Mask->getZExtValue() == MaskVal)
     C.Op1 = SDValue(Mask, 0);
   else
-    C.Op1 = DAG.getConstant(MaskVal, C.Op0.getValueType());
+    C.Op1 = DAG.getConstant(MaskVal, DL, C.Op0.getValueType());
   C.CCValid = SystemZ::CCMASK_TM;
   C.CCMask = NewCCMask;
 }
@@ -1658,17 +2006,17 @@ static Comparison getIntrinsicCmp(SelectionDAG &DAG, unsigned Opcode,
   else if (Cond == ISD::SETLT || Cond == ISD::SETULT)
     // bits above bit 3 for CC==0 (always false), bits above bit 0 for CC==3,
     // always true for CC>3.
-    C.CCMask = CC < 4 ? -1 << (4 - CC) : -1;
+    C.CCMask = CC < 4 ? ~0U << (4 - CC) : -1;
   else if (Cond == ISD::SETGE || Cond == ISD::SETUGE)
     // ...and the inverse of that.
-    C.CCMask = CC < 4 ? ~(-1 << (4 - CC)) : 0;
+    C.CCMask = CC < 4 ? ~(~0U << (4 - CC)) : 0;
   else if (Cond == ISD::SETLE || Cond == ISD::SETULE)
     // bit 3 and above for CC==0, bit 0 and above for CC==3 (always true),
     // always true for CC>3.
-    C.CCMask = CC < 4 ? -1 << (3 - CC) : -1;
+    C.CCMask = CC < 4 ? ~0U << (3 - CC) : -1;
   else if (Cond == ISD::SETGT || Cond == ISD::SETUGT)
     // ...and the inverse of that.
-    C.CCMask = CC < 4 ? ~(-1 << (3 - CC)) : 0;
+    C.CCMask = CC < 4 ? ~(~0U << (3 - CC)) : 0;
   else
     llvm_unreachable("Unexpected integer comparison type");
   C.CCMask &= CCValid;
@@ -1677,7 +2025,7 @@ static Comparison getIntrinsicCmp(SelectionDAG &DAG, unsigned Opcode,
 
 // Decide how to implement a comparison of type Cond between CmpOp0 with CmpOp1.
 static Comparison getCmp(SelectionDAG &DAG, SDValue CmpOp0, SDValue CmpOp1,
-                         ISD::CondCode Cond) {
+                         ISD::CondCode Cond, SDLoc DL) {
   if (CmpOp1.getOpcode() == ISD::Constant) {
     uint64_t Constant = cast<ConstantSDNode>(CmpOp1)->getZExtValue();
     unsigned Opcode, CCValid;
@@ -1685,6 +2033,10 @@ static Comparison getCmp(SelectionDAG &DAG, SDValue CmpOp0, SDValue CmpOp1,
         CmpOp0.getResNo() == 0 && CmpOp0->hasNUsesOfValue(1, 0) &&
         isIntrinsicWithCCAndChain(CmpOp0, Opcode, CCValid))
       return getIntrinsicCmp(DAG, Opcode, CmpOp0, CCValid, Constant, Cond);
+    if (CmpOp0.getOpcode() == ISD::INTRINSIC_WO_CHAIN &&
+        CmpOp0.getResNo() == CmpOp0->getNumValues() - 1 &&
+        isIntrinsicWithCC(CmpOp0, Opcode, CCValid))
+      return getIntrinsicCmp(DAG, Opcode, CmpOp0, CCValid, Constant, Cond);
   }
   Comparison C(CmpOp0, CmpOp1);
   C.CCMask = CCMaskForCondCode(Cond);
@@ -1709,11 +2061,11 @@ static Comparison getCmp(SelectionDAG &DAG, SDValue CmpOp0, SDValue CmpOp1,
     else
       C.ICmpType = SystemZICMP::SignedOnly;
     C.CCMask &= ~SystemZ::CCMASK_CMP_UO;
-    adjustZeroCmp(DAG, C);
-    adjustSubwordCmp(DAG, C);
-    adjustForSubtraction(DAG, C);
+    adjustZeroCmp(DAG, DL, C);
+    adjustSubwordCmp(DAG, DL, C);
+    adjustForSubtraction(DAG, DL, C);
     adjustForLTGFR(C);
-    adjustICmpTruncate(DAG, C);
+    adjustICmpTruncate(DAG, DL, C);
   }
 
   if (shouldSwapCmpOperands(C)) {
@@ -1721,7 +2073,7 @@ static Comparison getCmp(SelectionDAG &DAG, SDValue CmpOp0, SDValue CmpOp1,
     C.CCMask = reverseCCMask(C.CCMask);
   }
 
-  adjustForTestUnderMask(DAG, C);
+  adjustForTestUnderMask(DAG, DL, C);
   return C;
 }
 
@@ -1733,6 +2085,9 @@ static SDValue emitCmp(SelectionDAG &DAG, SDLoc DL, Comparison &C) {
     case ISD::INTRINSIC_W_CHAIN:
       Op = emitIntrinsicWithChainAndGlue(DAG, C.Op0, C.Opcode);
       break;
+    case ISD::INTRINSIC_WO_CHAIN:
+      Op = emitIntrinsicWithGlue(DAG, C.Op0, C.Opcode);
+      break;
     default:
       llvm_unreachable("Invalid comparison operands");
     }
@@ -1740,12 +2095,12 @@ static SDValue emitCmp(SelectionDAG &DAG, SDLoc DL, Comparison &C) {
   }
   if (C.Opcode == SystemZISD::ICMP)
     return DAG.getNode(SystemZISD::ICMP, DL, MVT::Glue, C.Op0, C.Op1,
-                       DAG.getConstant(C.ICmpType, MVT::i32));
+                       DAG.getConstant(C.ICmpType, DL, MVT::i32));
   if (C.Opcode == SystemZISD::TM) {
     bool RegisterOnly = (bool(C.CCMask & SystemZ::CCMASK_TM_MIXED_MSB_0) !=
                          bool(C.CCMask & SystemZ::CCMASK_TM_MIXED_MSB_1));
     return DAG.getNode(SystemZISD::TM, DL, MVT::Glue, C.Op0, C.Op1,
-                       DAG.getConstant(RegisterOnly, MVT::i32));
+                       DAG.getConstant(RegisterOnly, DL, MVT::i32));
   }
   return DAG.getNode(C.Opcode, DL, MVT::Glue, C.Op0, C.Op1);
 }
@@ -1759,7 +2114,8 @@ static void lowerMUL_LOHI32(SelectionDAG &DAG, SDLoc DL,
   Op0 = DAG.getNode(Extend, DL, MVT::i64, Op0);
   Op1 = DAG.getNode(Extend, DL, MVT::i64, Op1);
   SDValue Mul = DAG.getNode(ISD::MUL, DL, MVT::i64, Op0, Op1);
-  Hi = DAG.getNode(ISD::SRL, DL, MVT::i64, Mul, DAG.getConstant(32, MVT::i64));
+  Hi = DAG.getNode(ISD::SRL, DL, MVT::i64, Mul,
+                   DAG.getConstant(32, DL, MVT::i64));
   Hi = DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, Hi);
   Lo = DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, Mul);
 }
@@ -1791,29 +2147,159 @@ static SDValue emitSETCC(SelectionDAG &DAG, SDLoc DL, SDValue Glue,
 
   if (Conversion.XORValue)
     Result = DAG.getNode(ISD::XOR, DL, MVT::i32, Result,
-                         DAG.getConstant(Conversion.XORValue, MVT::i32));
+                         DAG.getConstant(Conversion.XORValue, DL, MVT::i32));
 
   if (Conversion.AddValue)
     Result = DAG.getNode(ISD::ADD, DL, MVT::i32, Result,
-                         DAG.getConstant(Conversion.AddValue, MVT::i32));
+                         DAG.getConstant(Conversion.AddValue, DL, MVT::i32));
 
   // The SHR/AND sequence should get optimized to an RISBG.
   Result = DAG.getNode(ISD::SRL, DL, MVT::i32, Result,
-                       DAG.getConstant(Conversion.Bit, MVT::i32));
+                       DAG.getConstant(Conversion.Bit, DL, MVT::i32));
   if (Conversion.Bit != 31)
     Result = DAG.getNode(ISD::AND, DL, MVT::i32, Result,
-                         DAG.getConstant(1, MVT::i32));
+                         DAG.getConstant(1, DL, MVT::i32));
   return Result;
 }
 
+// Return the SystemISD vector comparison operation for CC, or 0 if it cannot
+// be done directly.  IsFP is true if CC is for a floating-point rather than
+// integer comparison.
+static unsigned getVectorComparison(ISD::CondCode CC, bool IsFP) {
+  switch (CC) {
+  case ISD::SETOEQ:
+  case ISD::SETEQ:
+    return IsFP ? SystemZISD::VFCMPE : SystemZISD::VICMPE;
+
+  case ISD::SETOGE:
+  case ISD::SETGE:
+    return IsFP ? SystemZISD::VFCMPHE : static_cast<SystemZISD::NodeType>(0);
+
+  case ISD::SETOGT:
+  case ISD::SETGT:
+    return IsFP ? SystemZISD::VFCMPH : SystemZISD::VICMPH;
+
+  case ISD::SETUGT:
+    return IsFP ? static_cast<SystemZISD::NodeType>(0) : SystemZISD::VICMPHL;
+
+  default:
+    return 0;
+  }
+}
+
+// Return the SystemZISD vector comparison operation for CC or its inverse,
+// or 0 if neither can be done directly.  Indicate in Invert whether the
+// result is for the inverse of CC.  IsFP is true if CC is for a
+// floating-point rather than integer comparison.
+static unsigned getVectorComparisonOrInvert(ISD::CondCode CC, bool IsFP,
+                                            bool &Invert) {
+  if (unsigned Opcode = getVectorComparison(CC, IsFP)) {
+    Invert = false;
+    return Opcode;
+  }
+
+  CC = ISD::getSetCCInverse(CC, !IsFP);
+  if (unsigned Opcode = getVectorComparison(CC, IsFP)) {
+    Invert = true;
+    return Opcode;
+  }
+
+  return 0;
+}
+
+// Return a v2f64 that contains the extended form of elements Start and Start+1
+// of v4f32 value Op.
+static SDValue expandV4F32ToV2F64(SelectionDAG &DAG, int Start, SDLoc DL,
+                                  SDValue Op) {
+  int Mask[] = { Start, -1, Start + 1, -1 };
+  Op = DAG.getVectorShuffle(MVT::v4f32, DL, Op, DAG.getUNDEF(MVT::v4f32), Mask);
+  return DAG.getNode(SystemZISD::VEXTEND, DL, MVT::v2f64, Op);
+}
+
+// Build a comparison of vectors CmpOp0 and CmpOp1 using opcode Opcode,
+// producing a result of type VT.
+static SDValue getVectorCmp(SelectionDAG &DAG, unsigned Opcode, SDLoc DL,
+                            EVT VT, SDValue CmpOp0, SDValue CmpOp1) {
+  // There is no hardware support for v4f32, so extend the vector into
+  // two v2f64s and compare those.
+  if (CmpOp0.getValueType() == MVT::v4f32) {
+    SDValue H0 = expandV4F32ToV2F64(DAG, 0, DL, CmpOp0);
+    SDValue L0 = expandV4F32ToV2F64(DAG, 2, DL, CmpOp0);
+    SDValue H1 = expandV4F32ToV2F64(DAG, 0, DL, CmpOp1);
+    SDValue L1 = expandV4F32ToV2F64(DAG, 2, DL, CmpOp1);
+    SDValue HRes = DAG.getNode(Opcode, DL, MVT::v2i64, H0, H1);
+    SDValue LRes = DAG.getNode(Opcode, DL, MVT::v2i64, L0, L1);
+    return DAG.getNode(SystemZISD::PACK, DL, VT, HRes, LRes);
+  }
+  return DAG.getNode(Opcode, DL, VT, CmpOp0, CmpOp1);
+}
+
+// Lower a vector comparison of type CC between CmpOp0 and CmpOp1, producing
+// an integer mask of type VT.
+static SDValue lowerVectorSETCC(SelectionDAG &DAG, SDLoc DL, EVT VT,
+                                ISD::CondCode CC, SDValue CmpOp0,
+                                SDValue CmpOp1) {
+  bool IsFP = CmpOp0.getValueType().isFloatingPoint();
+  bool Invert = false;
+  SDValue Cmp;
+  switch (CC) {
+    // Handle tests for order using (or (ogt y x) (oge x y)).
+  case ISD::SETUO:
+    Invert = true;
+  case ISD::SETO: {
+    assert(IsFP && "Unexpected integer comparison");
+    SDValue LT = getVectorCmp(DAG, SystemZISD::VFCMPH, DL, VT, CmpOp1, CmpOp0);
+    SDValue GE = getVectorCmp(DAG, SystemZISD::VFCMPHE, DL, VT, CmpOp0, CmpOp1);
+    Cmp = DAG.getNode(ISD::OR, DL, VT, LT, GE);
+    break;
+  }
+
+    // Handle <> tests using (or (ogt y x) (ogt x y)).
+  case ISD::SETUEQ:
+    Invert = true;
+  case ISD::SETONE: {
+    assert(IsFP && "Unexpected integer comparison");
+    SDValue LT = getVectorCmp(DAG, SystemZISD::VFCMPH, DL, VT, CmpOp1, CmpOp0);
+    SDValue GT = getVectorCmp(DAG, SystemZISD::VFCMPH, DL, VT, CmpOp0, CmpOp1);
+    Cmp = DAG.getNode(ISD::OR, DL, VT, LT, GT);
+    break;
+  }
+
+    // Otherwise a single comparison is enough.  It doesn't really
+    // matter whether we try the inversion or the swap first, since
+    // there are no cases where both work.
+  default:
+    if (unsigned Opcode = getVectorComparisonOrInvert(CC, IsFP, Invert))
+      Cmp = getVectorCmp(DAG, Opcode, DL, VT, CmpOp0, CmpOp1);
+    else {
+      CC = ISD::getSetCCSwappedOperands(CC);
+      if (unsigned Opcode = getVectorComparisonOrInvert(CC, IsFP, Invert))
+        Cmp = getVectorCmp(DAG, Opcode, DL, VT, CmpOp1, CmpOp0);
+      else
+        llvm_unreachable("Unhandled comparison");
+    }
+    break;
+  }
+  if (Invert) {
+    SDValue Mask = DAG.getNode(SystemZISD::BYTE_MASK, DL, MVT::v16i8,
+                               DAG.getConstant(65535, DL, MVT::i32));
+    Mask = DAG.getNode(ISD::BITCAST, DL, VT, Mask);
+    Cmp = DAG.getNode(ISD::XOR, DL, VT, Cmp, Mask);
+  }
+  return Cmp;
+}
+
 SDValue SystemZTargetLowering::lowerSETCC(SDValue Op,
                                           SelectionDAG &DAG) const {
   SDValue CmpOp0   = Op.getOperand(0);
   SDValue CmpOp1   = Op.getOperand(1);
   ISD::CondCode CC = cast<CondCodeSDNode>(Op.getOperand(2))->get();
   SDLoc DL(Op);
+  EVT VT = Op.getValueType();
+  if (VT.isVector())
+    return lowerVectorSETCC(DAG, DL, VT, CC, CmpOp0, CmpOp1);
 
-  Comparison C(getCmp(DAG, CmpOp0, CmpOp1, CC));
+  Comparison C(getCmp(DAG, CmpOp0, CmpOp1, CC, DL));
   SDValue Glue = emitCmp(DAG, DL, C);
   return emitSETCC(DAG, DL, Glue, C.CCValid, C.CCMask);
 }
@@ -1825,11 +2311,11 @@ SDValue SystemZTargetLowering::lowerBR_CC(SDValue Op, SelectionDAG &DAG) const {
   SDValue Dest     = Op.getOperand(4);
   SDLoc DL(Op);
 
-  Comparison C(getCmp(DAG, CmpOp0, CmpOp1, CC));
+  Comparison C(getCmp(DAG, CmpOp0, CmpOp1, CC, DL));
   SDValue Glue = emitCmp(DAG, DL, C);
   return DAG.getNode(SystemZISD::BR_CCMASK, DL, Op.getValueType(),
-                     Op.getOperand(0), DAG.getConstant(C.CCValid, MVT::i32),
-                     DAG.getConstant(C.CCMask, MVT::i32), Dest, Glue);
+                     Op.getOperand(0), DAG.getConstant(C.CCValid, DL, MVT::i32),
+                     DAG.getConstant(C.CCMask, DL, MVT::i32), Dest, Glue);
 }
 
 // Return true if Pos is CmpOp and Neg is the negative of CmpOp,
@@ -1850,7 +2336,7 @@ static SDValue getAbsolute(SelectionDAG &DAG, SDLoc DL, SDValue Op,
   Op = DAG.getNode(SystemZISD::IABS, DL, Op.getValueType(), Op);
   if (IsNegative)
     Op = DAG.getNode(ISD::SUB, DL, Op.getValueType(),
-                     DAG.getConstant(0, Op.getValueType()), Op);
+                     DAG.getConstant(0, DL, Op.getValueType()), Op);
   return Op;
 }
 
@@ -1863,7 +2349,7 @@ SDValue SystemZTargetLowering::lowerSELECT_CC(SDValue Op,
   ISD::CondCode CC = cast<CondCodeSDNode>(Op.getOperand(4))->get();
   SDLoc DL(Op);
 
-  Comparison C(getCmp(DAG, CmpOp0, CmpOp1, CC));
+  Comparison C(getCmp(DAG, CmpOp0, CmpOp1, CC, DL));
 
   // Check for absolute and negative-absolute selections, including those
   // where the comparison value is sign-extended (for LPGFR and LNGFR).
@@ -1898,14 +2384,14 @@ SDValue SystemZTargetLowering::lowerSELECT_CC(SDValue Op,
       if (!is32Bit(VT))
         Result = DAG.getNode(ISD::ANY_EXTEND, DL, VT, Result);
       // Sign-extend from the low bit.
-      SDValue ShAmt = DAG.getConstant(VT.getSizeInBits() - 1, MVT::i32);
+      SDValue ShAmt = DAG.getConstant(VT.getSizeInBits() - 1, DL, MVT::i32);
       SDValue Shl = DAG.getNode(ISD::SHL, DL, VT, Result, ShAmt);
       return DAG.getNode(ISD::SRA, DL, VT, Shl, ShAmt);
     }
   }
 
-  SDValue Ops[] = {TrueOp, FalseOp, DAG.getConstant(C.CCValid, MVT::i32),
-                   DAG.getConstant(C.CCMask, MVT::i32), Glue};
+  SDValue Ops[] = {TrueOp, FalseOp, DAG.getConstant(C.CCValid, DL, MVT::i32),
+                   DAG.getConstant(C.CCMask, DL, MVT::i32), Glue};
 
   SDVTList VTs = DAG.getVTList(Op.getValueType(), MVT::Glue);
   return DAG.getNode(SystemZISD::SELECT_CCMASK, DL, VTs, Ops);
@@ -1916,7 +2402,7 @@ SDValue SystemZTargetLowering::lowerGlobalAddress(GlobalAddressSDNode *Node,
   SDLoc DL(Node);
   const GlobalValue *GV = Node->getGlobal();
   int64_t Offset = Node->getOffset();
-  EVT PtrVT = getPointerTy();
+  EVT PtrVT = getPointerTy(DAG.getDataLayout());
   Reloc::Model RM = DAG.getTarget().getRelocationModel();
   CodeModel::Model CM = DAG.getTarget().getCodeModel();
 
@@ -1945,7 +2431,7 @@ SDValue SystemZTargetLowering::lowerGlobalAddress(GlobalAddressSDNode *Node,
   // addition for it.
   if (Offset != 0)
     Result = DAG.getNode(ISD::ADD, DL, PtrVT, Result,
-                         DAG.getConstant(Offset, PtrVT));
+                         DAG.getConstant(Offset, DL, PtrVT));
 
   return Result;
 }
@@ -1955,7 +2441,7 @@ SDValue SystemZTargetLowering::lowerTLSGetOffset(GlobalAddressSDNode *Node,
                                                  unsigned Opcode,
                                                  SDValue GOTOffset) const {
   SDLoc DL(Node);
-  EVT PtrVT = getPointerTy();
+  EVT PtrVT = getPointerTy(DAG.getDataLayout());
   SDValue Chain = DAG.getEntryNode();
   SDValue Glue;
 
@@ -1999,24 +2485,26 @@ SDValue SystemZTargetLowering::lowerTLSGetOffset(GlobalAddressSDNode *Node,
 
 SDValue SystemZTargetLowering::lowerGlobalTLSAddress(GlobalAddressSDNode *Node,
 						     SelectionDAG &DAG) const {
+  if (DAG.getTarget().Options.EmulatedTLS)
+    return LowerToTLSEmulatedModel(Node, DAG);
   SDLoc DL(Node);
   const GlobalValue *GV = Node->getGlobal();
-  EVT PtrVT = getPointerTy();
+  EVT PtrVT = getPointerTy(DAG.getDataLayout());
   TLSModel::Model model = DAG.getTarget().getTLSModel(GV);
 
   // The high part of the thread pointer is in access register 0.
   SDValue TPHi = DAG.getNode(SystemZISD::EXTRACT_ACCESS, DL, MVT::i32,
-                             DAG.getConstant(0, MVT::i32));
+                             DAG.getConstant(0, DL, MVT::i32));
   TPHi = DAG.getNode(ISD::ANY_EXTEND, DL, PtrVT, TPHi);
 
   // The low part of the thread pointer is in access register 1.
   SDValue TPLo = DAG.getNode(SystemZISD::EXTRACT_ACCESS, DL, MVT::i32,
-                             DAG.getConstant(1, MVT::i32));
+                             DAG.getConstant(1, DL, MVT::i32));
   TPLo = DAG.getNode(ISD::ZERO_EXTEND, DL, PtrVT, TPLo);
 
   // Merge them into a single 64-bit address.
   SDValue TPHiShifted = DAG.getNode(ISD::SHL, DL, PtrVT, TPHi,
-				    DAG.getConstant(32, PtrVT));
+                                    DAG.getConstant(32, DL, PtrVT));
   SDValue TP = DAG.getNode(ISD::OR, DL, PtrVT, TPHiShifted, TPLo);
 
   // Get the offset of GA from the thread pointer, based on the TLS model.
@@ -2102,7 +2590,7 @@ SDValue SystemZTargetLowering::lowerBlockAddress(BlockAddressSDNode *Node,
   SDLoc DL(Node);
   const BlockAddress *BA = Node->getBlockAddress();
   int64_t Offset = Node->getOffset();
-  EVT PtrVT = getPointerTy();
+  EVT PtrVT = getPointerTy(DAG.getDataLayout());
 
   SDValue Result = DAG.getTargetBlockAddress(BA, PtrVT, Offset);
   Result = DAG.getNode(SystemZISD::PCREL_WRAPPER, DL, PtrVT, Result);
@@ -2112,7 +2600,7 @@ SDValue SystemZTargetLowering::lowerBlockAddress(BlockAddressSDNode *Node,
 SDValue SystemZTargetLowering::lowerJumpTable(JumpTableSDNode *JT,
                                               SelectionDAG &DAG) const {
   SDLoc DL(JT);
-  EVT PtrVT = getPointerTy();
+  EVT PtrVT = getPointerTy(DAG.getDataLayout());
   SDValue Result = DAG.getTargetJumpTable(JT->getIndex(), PtrVT);
 
   // Use LARL to load the address of the table.
@@ -2122,7 +2610,7 @@ SDValue SystemZTargetLowering::lowerJumpTable(JumpTableSDNode *JT,
 SDValue SystemZTargetLowering::lowerConstantPool(ConstantPoolSDNode *CP,
                                                  SelectionDAG &DAG) const {
   SDLoc DL(CP);
-  EVT PtrVT = getPointerTy();
+  EVT PtrVT = getPointerTy(DAG.getDataLayout());
 
   SDValue Result;
   if (CP->isMachineConstantPoolEntry())
@@ -2143,6 +2631,13 @@ SDValue SystemZTargetLowering::lowerBITCAST(SDValue Op,
   EVT InVT = In.getValueType();
   EVT ResVT = Op.getValueType();
 
+  // Convert loads directly.  This is normally done by DAGCombiner,
+  // but we need this case for bitcasts that are created during lowering
+  // and which are then lowered themselves.
+  if (auto *LoadN = dyn_cast<LoadSDNode>(In))
+    return DAG.getLoad(ResVT, DL, LoadN->getChain(), LoadN->getBasePtr(),
+                       LoadN->getMemOperand());
+
   if (InVT == MVT::i32 && ResVT == MVT::f32) {
     SDValue In64;
     if (Subtarget.hasHighWord()) {
@@ -2153,22 +2648,22 @@ SDValue SystemZTargetLowering::lowerBITCAST(SDValue Op,
     } else {
       In64 = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, In);
       In64 = DAG.getNode(ISD::SHL, DL, MVT::i64, In64,
-                         DAG.getConstant(32, MVT::i64));
+                         DAG.getConstant(32, DL, MVT::i64));
     }
     SDValue Out64 = DAG.getNode(ISD::BITCAST, DL, MVT::f64, In64);
-    return DAG.getTargetExtractSubreg(SystemZ::subreg_h32,
+    return DAG.getTargetExtractSubreg(SystemZ::subreg_r32,
                                       DL, MVT::f32, Out64);
   }
   if (InVT == MVT::f32 && ResVT == MVT::i32) {
     SDNode *U64 = DAG.getMachineNode(TargetOpcode::IMPLICIT_DEF, DL, MVT::f64);
-    SDValue In64 = DAG.getTargetInsertSubreg(SystemZ::subreg_h32, DL,
+    SDValue In64 = DAG.getTargetInsertSubreg(SystemZ::subreg_r32, DL,
                                              MVT::f64, SDValue(U64, 0), In);
     SDValue Out64 = DAG.getNode(ISD::BITCAST, DL, MVT::i64, In64);
     if (Subtarget.hasHighWord())
       return DAG.getTargetExtractSubreg(SystemZ::subreg_h32, DL,
                                         MVT::i32, Out64);
     SDValue Shift = DAG.getNode(ISD::SRL, DL, MVT::i64, Out64,
-                                DAG.getConstant(32, MVT::i64));
+                                DAG.getConstant(32, DL, MVT::i64));
     return DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, Shift);
   }
   llvm_unreachable("Unexpected bitcast combination");
@@ -2179,7 +2674,7 @@ SDValue SystemZTargetLowering::lowerVASTART(SDValue Op,
   MachineFunction &MF = DAG.getMachineFunction();
   SystemZMachineFunctionInfo *FuncInfo =
     MF.getInfo<SystemZMachineFunctionInfo>();
-  EVT PtrVT = getPointerTy();
+  EVT PtrVT = getPointerTy(DAG.getDataLayout());
 
   SDValue Chain   = Op.getOperand(0);
   SDValue Addr    = Op.getOperand(1);
@@ -2189,8 +2684,8 @@ SDValue SystemZTargetLowering::lowerVASTART(SDValue Op,
   // The initial values of each field.
   const unsigned NumFields = 4;
   SDValue Fields[NumFields] = {
-    DAG.getConstant(FuncInfo->getVarArgsFirstGPR(), PtrVT),
-    DAG.getConstant(FuncInfo->getVarArgsFirstFPR(), PtrVT),
+    DAG.getConstant(FuncInfo->getVarArgsFirstGPR(), DL, PtrVT),
+    DAG.getConstant(FuncInfo->getVarArgsFirstFPR(), DL, PtrVT),
     DAG.getFrameIndex(FuncInfo->getVarArgsFrameIndex(), PtrVT),
     DAG.getFrameIndex(FuncInfo->getRegSaveFrameIndex(), PtrVT)
   };
@@ -2202,7 +2697,7 @@ SDValue SystemZTargetLowering::lowerVASTART(SDValue Op,
     SDValue FieldAddr = Addr;
     if (Offset != 0)
       FieldAddr = DAG.getNode(ISD::ADD, DL, PtrVT, FieldAddr,
-                              DAG.getIntPtrConstant(Offset));
+                              DAG.getIntPtrConstant(Offset, DL));
     MemOps[I] = DAG.getStore(Chain, DL, Fields[I], FieldAddr,
                              MachinePointerInfo(SV, Offset),
                              false, false, 0);
@@ -2220,7 +2715,7 @@ SDValue SystemZTargetLowering::lowerVACOPY(SDValue Op,
   const Value *SrcSV = cast<SrcValueSDNode>(Op.getOperand(4))->getValue();
   SDLoc DL(Op);
 
-  return DAG.getMemcpy(Chain, DL, DstPtr, SrcPtr, DAG.getIntPtrConstant(32),
+  return DAG.getMemcpy(Chain, DL, DstPtr, SrcPtr, DAG.getIntPtrConstant(32, DL),
                        /*Align*/8, /*isVolatile*/false, /*AlwaysInline*/false,
                        /*isTailCall*/false,
                        MachinePointerInfo(DstSV), MachinePointerInfo(SrcSV));
@@ -2277,7 +2772,7 @@ SDValue SystemZTargetLowering::lowerSMUL_LOHI(SDValue Op,
     // multiplication:
     //
     //   (ll * rl) - (((lh & rl) + (ll & rh)) << 64)
-    SDValue C63 = DAG.getConstant(63, MVT::i64);
+    SDValue C63 = DAG.getConstant(63, DL, MVT::i64);
     SDValue LL = Op.getOperand(0);
     SDValue RL = Op.getOperand(1);
     SDValue LH = DAG.getNode(ISD::SRA, DL, VT, LL, C63);
@@ -2418,18 +2913,52 @@ SDValue SystemZTargetLowering::lowerOR(SDValue Op, SelectionDAG &DAG) const {
 SDValue SystemZTargetLowering::lowerCTPOP(SDValue Op,
                                           SelectionDAG &DAG) const {
   EVT VT = Op.getValueType();
-  int64_t OrigBitSize = VT.getSizeInBits();
   SDLoc DL(Op);
+  Op = Op.getOperand(0);
+
+  // Handle vector types via VPOPCT.
+  if (VT.isVector()) {
+    Op = DAG.getNode(ISD::BITCAST, DL, MVT::v16i8, Op);
+    Op = DAG.getNode(SystemZISD::POPCNT, DL, MVT::v16i8, Op);
+    switch (VT.getVectorElementType().getSizeInBits()) {
+    case 8:
+      break;
+    case 16: {
+      Op = DAG.getNode(ISD::BITCAST, DL, VT, Op);
+      SDValue Shift = DAG.getConstant(8, DL, MVT::i32);
+      SDValue Tmp = DAG.getNode(SystemZISD::VSHL_BY_SCALAR, DL, VT, Op, Shift);
+      Op = DAG.getNode(ISD::ADD, DL, VT, Op, Tmp);
+      Op = DAG.getNode(SystemZISD::VSRL_BY_SCALAR, DL, VT, Op, Shift);
+      break;
+    }
+    case 32: {
+      SDValue Tmp = DAG.getNode(SystemZISD::BYTE_MASK, DL, MVT::v16i8,
+                                DAG.getConstant(0, DL, MVT::i32));
+      Op = DAG.getNode(SystemZISD::VSUM, DL, VT, Op, Tmp);
+      break;
+    }
+    case 64: {
+      SDValue Tmp = DAG.getNode(SystemZISD::BYTE_MASK, DL, MVT::v16i8,
+                                DAG.getConstant(0, DL, MVT::i32));
+      Op = DAG.getNode(SystemZISD::VSUM, DL, MVT::v4i32, Op, Tmp);
+      Op = DAG.getNode(SystemZISD::VSUM, DL, VT, Op, Tmp);
+      break;
+    }
+    default:
+      llvm_unreachable("Unexpected type");
+    }
+    return Op;
+  }
 
   // Get the known-zero mask for the operand.
-  Op = Op.getOperand(0);
   APInt KnownZero, KnownOne;
   DAG.computeKnownBits(Op, KnownZero, KnownOne);
   unsigned NumSignificantBits = (~KnownZero).getActiveBits();
   if (NumSignificantBits == 0)
-    return DAG.getConstant(0, VT);
+    return DAG.getConstant(0, DL, VT);
 
   // Skip known-zero high parts of the operand.
+  int64_t OrigBitSize = VT.getSizeInBits();
   int64_t BitSize = (int64_t)1 << Log2_32_Ceil(NumSignificantBits);
   BitSize = std::min(BitSize, OrigBitSize);
 
@@ -2441,16 +2970,17 @@ SDValue SystemZTargetLowering::lowerCTPOP(SDValue Op,
   // Add up per-byte counts in a binary tree.  All bits of Op at
   // position larger than BitSize remain zero throughout.
   for (int64_t I = BitSize / 2; I >= 8; I = I / 2) {
-    SDValue Tmp = DAG.getNode(ISD::SHL, DL, VT, Op, DAG.getConstant(I, VT));
+    SDValue Tmp = DAG.getNode(ISD::SHL, DL, VT, Op, DAG.getConstant(I, DL, VT));
     if (BitSize != OrigBitSize)
       Tmp = DAG.getNode(ISD::AND, DL, VT, Tmp,
-                        DAG.getConstant(((uint64_t)1 << BitSize) - 1, VT));
+                        DAG.getConstant(((uint64_t)1 << BitSize) - 1, DL, VT));
     Op = DAG.getNode(ISD::ADD, DL, VT, Op, Tmp);
   }
 
   // Extract overall result from high byte.
   if (BitSize > 8)
-    Op = DAG.getNode(ISD::SRL, DL, VT, Op, DAG.getConstant(BitSize - 8, VT));
+    Op = DAG.getNode(ISD::SRL, DL, VT, Op,
+                     DAG.getConstant(BitSize - 8, DL, VT));
 
   return Op;
 }
@@ -2501,23 +3031,23 @@ SDValue SystemZTargetLowering::lowerATOMIC_LOAD_OP(SDValue Op,
   if (Opcode == SystemZISD::ATOMIC_LOADW_SUB)
     if (auto *Const = dyn_cast<ConstantSDNode>(Src2)) {
       Opcode = SystemZISD::ATOMIC_LOADW_ADD;
-      Src2 = DAG.getConstant(-Const->getSExtValue(), Src2.getValueType());
+      Src2 = DAG.getConstant(-Const->getSExtValue(), DL, Src2.getValueType());
     }
 
   // Get the address of the containing word.
   SDValue AlignedAddr = DAG.getNode(ISD::AND, DL, PtrVT, Addr,
-                                    DAG.getConstant(-4, PtrVT));
+                                    DAG.getConstant(-4, DL, PtrVT));
 
   // Get the number of bits that the word must be rotated left in order
   // to bring the field to the top bits of a GR32.
   SDValue BitShift = DAG.getNode(ISD::SHL, DL, PtrVT, Addr,
-                                 DAG.getConstant(3, PtrVT));
+                                 DAG.getConstant(3, DL, PtrVT));
   BitShift = DAG.getNode(ISD::TRUNCATE, DL, WideVT, BitShift);
 
   // Get the complementing shift amount, for rotating a field in the top
   // bits back to its proper position.
   SDValue NegBitShift = DAG.getNode(ISD::SUB, DL, WideVT,
-                                    DAG.getConstant(0, WideVT), BitShift);
+                                    DAG.getConstant(0, DL, WideVT), BitShift);
 
   // Extend the source operand to 32 bits and prepare it for the inner loop.
   // ATOMIC_SWAPW uses RISBG to rotate the field left, but all other
@@ -2526,23 +3056,23 @@ SDValue SystemZTargetLowering::lowerATOMIC_LOAD_OP(SDValue Op,
   // bits must be set, while for other opcodes they should be left clear.
   if (Opcode != SystemZISD::ATOMIC_SWAPW)
     Src2 = DAG.getNode(ISD::SHL, DL, WideVT, Src2,
-                       DAG.getConstant(32 - BitSize, WideVT));
+                       DAG.getConstant(32 - BitSize, DL, WideVT));
   if (Opcode == SystemZISD::ATOMIC_LOADW_AND ||
       Opcode == SystemZISD::ATOMIC_LOADW_NAND)
     Src2 = DAG.getNode(ISD::OR, DL, WideVT, Src2,
-                       DAG.getConstant(uint32_t(-1) >> BitSize, WideVT));
+                       DAG.getConstant(uint32_t(-1) >> BitSize, DL, WideVT));
 
   // Construct the ATOMIC_LOADW_* node.
   SDVTList VTList = DAG.getVTList(WideVT, MVT::Other);
   SDValue Ops[] = { ChainIn, AlignedAddr, Src2, BitShift, NegBitShift,
-                    DAG.getConstant(BitSize, WideVT) };
+                    DAG.getConstant(BitSize, DL, WideVT) };
   SDValue AtomicOp = DAG.getMemIntrinsicNode(Opcode, DL, VTList, Ops,
                                              NarrowVT, MMO);
 
   // Rotate the result of the final CS so that the field is in the lower
   // bits of a GR32, then truncate it.
   SDValue ResultShift = DAG.getNode(ISD::ADD, DL, WideVT, BitShift,
-                                    DAG.getConstant(BitSize, WideVT));
+                                    DAG.getConstant(BitSize, DL, WideVT));
   SDValue Result = DAG.getNode(ISD::ROTL, DL, WideVT, AtomicOp, ResultShift);
 
   SDValue RetOps[2] = { Result, AtomicOp.getValue(1) };
@@ -2568,10 +3098,10 @@ SDValue SystemZTargetLowering::lowerATOMIC_LOAD_SUB(SDValue Op,
       // available or the negative value is in the range of A(G)FHI.
       int64_t Value = (-Op2->getAPIntValue()).getSExtValue();
       if (isInt<32>(Value) || Subtarget.hasInterlockedAccess1())
-        NegSrc2 = DAG.getConstant(Value, MemVT);
+        NegSrc2 = DAG.getConstant(Value, DL, MemVT);
     } else if (Subtarget.hasInterlockedAccess1())
       // Use LAA(G) if available.
-      NegSrc2 = DAG.getNode(ISD::SUB, DL, MemVT, DAG.getConstant(0, MemVT),
+      NegSrc2 = DAG.getNode(ISD::SUB, DL, MemVT, DAG.getConstant(0, DL, MemVT),
                             Src2);
 
     if (NegSrc2.getNode())
@@ -2610,23 +3140,23 @@ SDValue SystemZTargetLowering::lowerATOMIC_CMP_SWAP(SDValue Op,
 
   // Get the address of the containing word.
   SDValue AlignedAddr = DAG.getNode(ISD::AND, DL, PtrVT, Addr,
-                                    DAG.getConstant(-4, PtrVT));
+                                    DAG.getConstant(-4, DL, PtrVT));
 
   // Get the number of bits that the word must be rotated left in order
   // to bring the field to the top bits of a GR32.
   SDValue BitShift = DAG.getNode(ISD::SHL, DL, PtrVT, Addr,
-                                 DAG.getConstant(3, PtrVT));
+                                 DAG.getConstant(3, DL, PtrVT));
   BitShift = DAG.getNode(ISD::TRUNCATE, DL, WideVT, BitShift);
 
   // Get the complementing shift amount, for rotating a field in the top
   // bits back to its proper position.
   SDValue NegBitShift = DAG.getNode(ISD::SUB, DL, WideVT,
-                                    DAG.getConstant(0, WideVT), BitShift);
+                                    DAG.getConstant(0, DL, WideVT), BitShift);
 
   // Construct the ATOMIC_CMP_SWAPW node.
   SDVTList VTList = DAG.getVTList(WideVT, MVT::Other);
   SDValue Ops[] = { ChainIn, AlignedAddr, CmpVal, SwapVal, BitShift,
-                    NegBitShift, DAG.getConstant(BitSize, WideVT) };
+                    NegBitShift, DAG.getConstant(BitSize, DL, WideVT) };
   SDValue AtomicOp = DAG.getMemIntrinsicNode(SystemZISD::ATOMIC_CMP_SWAPW, DL,
                                              VTList, Ops, NarrowVT, MMO);
   return AtomicOp;
@@ -2655,15 +3185,16 @@ SDValue SystemZTargetLowering::lowerPREFETCH(SDValue Op,
     // Just preserve the chain.
     return Op.getOperand(0);
 
+  SDLoc DL(Op);
   bool IsWrite = cast<ConstantSDNode>(Op.getOperand(2))->getZExtValue();
   unsigned Code = IsWrite ? SystemZ::PFD_WRITE : SystemZ::PFD_READ;
   auto *Node = cast<MemIntrinsicSDNode>(Op.getNode());
   SDValue Ops[] = {
     Op.getOperand(0),
-    DAG.getConstant(Code, MVT::i32),
+    DAG.getConstant(Code, DL, MVT::i32),
     Op.getOperand(1)
   };
-  return DAG.getMemIntrinsicNode(SystemZISD::PREFETCH, SDLoc(Op),
+  return DAG.getMemIntrinsicNode(SystemZISD::PREFETCH, DL,
                                  Node->getVTList(), Ops,
                                  Node->getMemoryVT(), Node->getMemOperand());
 }
@@ -2671,10 +3202,11 @@ SDValue SystemZTargetLowering::lowerPREFETCH(SDValue Op,
 // Return an i32 that contains the value of CC immediately after After,
 // whose final operand must be MVT::Glue.
 static SDValue getCCResult(SelectionDAG &DAG, SDNode *After) {
+  SDLoc DL(After);
   SDValue Glue = SDValue(After, After->getNumValues() - 1);
-  SDValue IPM = DAG.getNode(SystemZISD::IPM, SDLoc(After), MVT::i32, Glue);
-  return DAG.getNode(ISD::SRL, SDLoc(After), MVT::i32, IPM,
-                     DAG.getConstant(SystemZ::IPM_CC, MVT::i32));
+  SDValue IPM = DAG.getNode(SystemZISD::IPM, DL, MVT::i32, Glue);
+  return DAG.getNode(ISD::SRL, DL, MVT::i32, IPM,
+                     DAG.getConstant(SystemZ::IPM_CC, DL, MVT::i32));
 }
 
 SDValue
@@ -2692,6 +3224,1045 @@ SystemZTargetLowering::lowerINTRINSIC_W_CHAIN(SDValue Op,
   return SDValue();
 }
 
+SDValue
+SystemZTargetLowering::lowerINTRINSIC_WO_CHAIN(SDValue Op,
+                                               SelectionDAG &DAG) const {
+  unsigned Opcode, CCValid;
+  if (isIntrinsicWithCC(Op, Opcode, CCValid)) {
+    SDValue Glued = emitIntrinsicWithGlue(DAG, Op, Opcode);
+    SDValue CC = getCCResult(DAG, Glued.getNode());
+    if (Op->getNumValues() == 1)
+      return CC;
+    assert(Op->getNumValues() == 2 && "Expected a CC and non-CC result");
+    return DAG.getNode(ISD::MERGE_VALUES, SDLoc(Op), Op->getVTList(),
+		    Glued, CC);
+  }
+
+  unsigned Id = cast<ConstantSDNode>(Op.getOperand(0))->getZExtValue();
+  switch (Id) {
+  case Intrinsic::s390_vpdi:
+    return DAG.getNode(SystemZISD::PERMUTE_DWORDS, SDLoc(Op), Op.getValueType(),
+                       Op.getOperand(1), Op.getOperand(2), Op.getOperand(3));
+
+  case Intrinsic::s390_vperm:
+    return DAG.getNode(SystemZISD::PERMUTE, SDLoc(Op), Op.getValueType(),
+                       Op.getOperand(1), Op.getOperand(2), Op.getOperand(3));
+
+  case Intrinsic::s390_vuphb:
+  case Intrinsic::s390_vuphh:
+  case Intrinsic::s390_vuphf:
+    return DAG.getNode(SystemZISD::UNPACK_HIGH, SDLoc(Op), Op.getValueType(),
+                       Op.getOperand(1));
+
+  case Intrinsic::s390_vuplhb:
+  case Intrinsic::s390_vuplhh:
+  case Intrinsic::s390_vuplhf:
+    return DAG.getNode(SystemZISD::UNPACKL_HIGH, SDLoc(Op), Op.getValueType(),
+                       Op.getOperand(1));
+
+  case Intrinsic::s390_vuplb:
+  case Intrinsic::s390_vuplhw:
+  case Intrinsic::s390_vuplf:
+    return DAG.getNode(SystemZISD::UNPACK_LOW, SDLoc(Op), Op.getValueType(),
+                       Op.getOperand(1));
+
+  case Intrinsic::s390_vupllb:
+  case Intrinsic::s390_vupllh:
+  case Intrinsic::s390_vupllf:
+    return DAG.getNode(SystemZISD::UNPACKL_LOW, SDLoc(Op), Op.getValueType(),
+                       Op.getOperand(1));
+
+  case Intrinsic::s390_vsumb:
+  case Intrinsic::s390_vsumh:
+  case Intrinsic::s390_vsumgh:
+  case Intrinsic::s390_vsumgf:
+  case Intrinsic::s390_vsumqf:
+  case Intrinsic::s390_vsumqg:
+    return DAG.getNode(SystemZISD::VSUM, SDLoc(Op), Op.getValueType(),
+                       Op.getOperand(1), Op.getOperand(2));
+  }
+
+  return SDValue();
+}
+
+namespace {
+// Says that SystemZISD operation Opcode can be used to perform the equivalent
+// of a VPERM with permute vector Bytes.  If Opcode takes three operands,
+// Operand is the constant third operand, otherwise it is the number of
+// bytes in each element of the result.
+struct Permute {
+  unsigned Opcode;
+  unsigned Operand;
+  unsigned char Bytes[SystemZ::VectorBytes];
+};
+}
+
+static const Permute PermuteForms[] = {
+  // VMRHG
+  { SystemZISD::MERGE_HIGH, 8,
+    { 0, 1, 2, 3, 4, 5, 6, 7, 16, 17, 18, 19, 20, 21, 22, 23 } },
+  // VMRHF
+  { SystemZISD::MERGE_HIGH, 4,
+    { 0, 1, 2, 3, 16, 17, 18, 19, 4, 5, 6, 7, 20, 21, 22, 23 } },
+  // VMRHH
+  { SystemZISD::MERGE_HIGH, 2,
+    { 0, 1, 16, 17, 2, 3, 18, 19, 4, 5, 20, 21, 6, 7, 22, 23 } },
+  // VMRHB
+  { SystemZISD::MERGE_HIGH, 1,
+    { 0, 16, 1, 17, 2, 18, 3, 19, 4, 20, 5, 21, 6, 22, 7, 23 } },
+  // VMRLG
+  { SystemZISD::MERGE_LOW, 8,
+    { 8, 9, 10, 11, 12, 13, 14, 15, 24, 25, 26, 27, 28, 29, 30, 31 } },
+  // VMRLF
+  { SystemZISD::MERGE_LOW, 4,
+    { 8, 9, 10, 11, 24, 25, 26, 27, 12, 13, 14, 15, 28, 29, 30, 31 } },
+  // VMRLH
+  { SystemZISD::MERGE_LOW, 2,
+    { 8, 9, 24, 25, 10, 11, 26, 27, 12, 13, 28, 29, 14, 15, 30, 31 } },
+  // VMRLB
+  { SystemZISD::MERGE_LOW, 1,
+    { 8, 24, 9, 25, 10, 26, 11, 27, 12, 28, 13, 29, 14, 30, 15, 31 } },
+  // VPKG
+  { SystemZISD::PACK, 4,
+    { 4, 5, 6, 7, 12, 13, 14, 15, 20, 21, 22, 23, 28, 29, 30, 31 } },
+  // VPKF
+  { SystemZISD::PACK, 2,
+    { 2, 3, 6, 7, 10, 11, 14, 15, 18, 19, 22, 23, 26, 27, 30, 31 } },
+  // VPKH
+  { SystemZISD::PACK, 1,
+    { 1, 3, 5, 7, 9, 11, 13, 15, 17, 19, 21, 23, 25, 27, 29, 31 } },
+  // VPDI V1, V2, 4  (low half of V1, high half of V2)
+  { SystemZISD::PERMUTE_DWORDS, 4,
+    { 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23 } },
+  // VPDI V1, V2, 1  (high half of V1, low half of V2)
+  { SystemZISD::PERMUTE_DWORDS, 1,
+    { 0, 1, 2, 3, 4, 5, 6, 7, 24, 25, 26, 27, 28, 29, 30, 31 } }
+};
+
+// Called after matching a vector shuffle against a particular pattern.
+// Both the original shuffle and the pattern have two vector operands.
+// OpNos[0] is the operand of the original shuffle that should be used for
+// operand 0 of the pattern, or -1 if operand 0 of the pattern can be anything.
+// OpNos[1] is the same for operand 1 of the pattern.  Resolve these -1s and
+// set OpNo0 and OpNo1 to the shuffle operands that should actually be used
+// for operands 0 and 1 of the pattern.
+static bool chooseShuffleOpNos(int *OpNos, unsigned &OpNo0, unsigned &OpNo1) {
+  if (OpNos[0] < 0) {
+    if (OpNos[1] < 0)
+      return false;
+    OpNo0 = OpNo1 = OpNos[1];
+  } else if (OpNos[1] < 0) {
+    OpNo0 = OpNo1 = OpNos[0];
+  } else {
+    OpNo0 = OpNos[0];
+    OpNo1 = OpNos[1];
+  }
+  return true;
+}
+
+// Bytes is a VPERM-like permute vector, except that -1 is used for
+// undefined bytes.  Return true if the VPERM can be implemented using P.
+// When returning true set OpNo0 to the VPERM operand that should be
+// used for operand 0 of P and likewise OpNo1 for operand 1 of P.
+//
+// For example, if swapping the VPERM operands allows P to match, OpNo0
+// will be 1 and OpNo1 will be 0.  If instead Bytes only refers to one
+// operand, but rewriting it to use two duplicated operands allows it to
+// match P, then OpNo0 and OpNo1 will be the same.
+static bool matchPermute(const SmallVectorImpl<int> &Bytes, const Permute &P,
+                         unsigned &OpNo0, unsigned &OpNo1) {
+  int OpNos[] = { -1, -1 };
+  for (unsigned I = 0; I < SystemZ::VectorBytes; ++I) {
+    int Elt = Bytes[I];
+    if (Elt >= 0) {
+      // Make sure that the two permute vectors use the same suboperand
+      // byte number.  Only the operand numbers (the high bits) are
+      // allowed to differ.
+      if ((Elt ^ P.Bytes[I]) & (SystemZ::VectorBytes - 1))
+        return false;
+      int ModelOpNo = P.Bytes[I] / SystemZ::VectorBytes;
+      int RealOpNo = unsigned(Elt) / SystemZ::VectorBytes;
+      // Make sure that the operand mappings are consistent with previous
+      // elements.
+      if (OpNos[ModelOpNo] == 1 - RealOpNo)
+        return false;
+      OpNos[ModelOpNo] = RealOpNo;
+    }
+  }
+  return chooseShuffleOpNos(OpNos, OpNo0, OpNo1);
+}
+
+// As above, but search for a matching permute.
+static const Permute *matchPermute(const SmallVectorImpl<int> &Bytes,
+                                   unsigned &OpNo0, unsigned &OpNo1) {
+  for (auto &P : PermuteForms)
+    if (matchPermute(Bytes, P, OpNo0, OpNo1))
+      return &P;
+  return nullptr;
+}
+
+// Bytes is a VPERM-like permute vector, except that -1 is used for
+// undefined bytes.  This permute is an operand of an outer permute.
+// See whether redistributing the -1 bytes gives a shuffle that can be
+// implemented using P.  If so, set Transform to a VPERM-like permute vector
+// that, when applied to the result of P, gives the original permute in Bytes.
+static bool matchDoublePermute(const SmallVectorImpl<int> &Bytes,
+                               const Permute &P,
+                               SmallVectorImpl<int> &Transform) {
+  unsigned To = 0;
+  for (unsigned From = 0; From < SystemZ::VectorBytes; ++From) {
+    int Elt = Bytes[From];
+    if (Elt < 0)
+      // Byte number From of the result is undefined.
+      Transform[From] = -1;
+    else {
+      while (P.Bytes[To] != Elt) {
+        To += 1;
+        if (To == SystemZ::VectorBytes)
+          return false;
+      }
+      Transform[From] = To;
+    }
+  }
+  return true;
+}
+
+// As above, but search for a matching permute.
+static const Permute *matchDoublePermute(const SmallVectorImpl<int> &Bytes,
+                                         SmallVectorImpl<int> &Transform) {
+  for (auto &P : PermuteForms)
+    if (matchDoublePermute(Bytes, P, Transform))
+      return &P;
+  return nullptr;
+}
+
+// Convert the mask of the given VECTOR_SHUFFLE into a byte-level mask,
+// as if it had type vNi8.
+static void getVPermMask(ShuffleVectorSDNode *VSN,
+                         SmallVectorImpl<int> &Bytes) {
+  EVT VT = VSN->getValueType(0);
+  unsigned NumElements = VT.getVectorNumElements();
+  unsigned BytesPerElement = VT.getVectorElementType().getStoreSize();
+  Bytes.resize(NumElements * BytesPerElement, -1);
+  for (unsigned I = 0; I < NumElements; ++I) {
+    int Index = VSN->getMaskElt(I);
+    if (Index >= 0)
+      for (unsigned J = 0; J < BytesPerElement; ++J)
+        Bytes[I * BytesPerElement + J] = Index * BytesPerElement + J;
+  }
+}
+
+// Bytes is a VPERM-like permute vector, except that -1 is used for
+// undefined bytes.  See whether bytes [Start, Start + BytesPerElement) of
+// the result come from a contiguous sequence of bytes from one input.
+// Set Base to the selector for the first byte if so.
+static bool getShuffleInput(const SmallVectorImpl<int> &Bytes, unsigned Start,
+                            unsigned BytesPerElement, int &Base) {
+  Base = -1;
+  for (unsigned I = 0; I < BytesPerElement; ++I) {
+    if (Bytes[Start + I] >= 0) {
+      unsigned Elem = Bytes[Start + I];
+      if (Base < 0) {
+        Base = Elem - I;
+        // Make sure the bytes would come from one input operand.
+        if (unsigned(Base) % Bytes.size() + BytesPerElement > Bytes.size())
+          return false;
+      } else if (unsigned(Base) != Elem - I)
+        return false;
+    }
+  }
+  return true;
+}
+
+// Bytes is a VPERM-like permute vector, except that -1 is used for
+// undefined bytes.  Return true if it can be performed using VSLDI.
+// When returning true, set StartIndex to the shift amount and OpNo0
+// and OpNo1 to the VPERM operands that should be used as the first
+// and second shift operand respectively.
+static bool isShlDoublePermute(const SmallVectorImpl<int> &Bytes,
+                               unsigned &StartIndex, unsigned &OpNo0,
+                               unsigned &OpNo1) {
+  int OpNos[] = { -1, -1 };
+  int Shift = -1;
+  for (unsigned I = 0; I < 16; ++I) {
+    int Index = Bytes[I];
+    if (Index >= 0) {
+      int ExpectedShift = (Index - I) % SystemZ::VectorBytes;
+      int ModelOpNo = unsigned(ExpectedShift + I) / SystemZ::VectorBytes;
+      int RealOpNo = unsigned(Index) / SystemZ::VectorBytes;
+      if (Shift < 0)
+        Shift = ExpectedShift;
+      else if (Shift != ExpectedShift)
+        return false;
+      // Make sure that the operand mappings are consistent with previous
+      // elements.
+      if (OpNos[ModelOpNo] == 1 - RealOpNo)
+        return false;
+      OpNos[ModelOpNo] = RealOpNo;
+    }
+  }
+  StartIndex = Shift;
+  return chooseShuffleOpNos(OpNos, OpNo0, OpNo1);
+}
+
+// Create a node that performs P on operands Op0 and Op1, casting the
+// operands to the appropriate type.  The type of the result is determined by P.
+static SDValue getPermuteNode(SelectionDAG &DAG, SDLoc DL,
+                              const Permute &P, SDValue Op0, SDValue Op1) {
+  // VPDI (PERMUTE_DWORDS) always operates on v2i64s.  The input
+  // elements of a PACK are twice as wide as the outputs.
+  unsigned InBytes = (P.Opcode == SystemZISD::PERMUTE_DWORDS ? 8 :
+                      P.Opcode == SystemZISD::PACK ? P.Operand * 2 :
+                      P.Operand);
+  // Cast both operands to the appropriate type.
+  MVT InVT = MVT::getVectorVT(MVT::getIntegerVT(InBytes * 8),
+                              SystemZ::VectorBytes / InBytes);
+  Op0 = DAG.getNode(ISD::BITCAST, DL, InVT, Op0);
+  Op1 = DAG.getNode(ISD::BITCAST, DL, InVT, Op1);
+  SDValue Op;
+  if (P.Opcode == SystemZISD::PERMUTE_DWORDS) {
+    SDValue Op2 = DAG.getConstant(P.Operand, DL, MVT::i32);
+    Op = DAG.getNode(SystemZISD::PERMUTE_DWORDS, DL, InVT, Op0, Op1, Op2);
+  } else if (P.Opcode == SystemZISD::PACK) {
+    MVT OutVT = MVT::getVectorVT(MVT::getIntegerVT(P.Operand * 8),
+                                 SystemZ::VectorBytes / P.Operand);
+    Op = DAG.getNode(SystemZISD::PACK, DL, OutVT, Op0, Op1);
+  } else {
+    Op = DAG.getNode(P.Opcode, DL, InVT, Op0, Op1);
+  }
+  return Op;
+}
+
+// Bytes is a VPERM-like permute vector, except that -1 is used for
+// undefined bytes.  Implement it on operands Ops[0] and Ops[1] using
+// VSLDI or VPERM.
+static SDValue getGeneralPermuteNode(SelectionDAG &DAG, SDLoc DL, SDValue *Ops,
+                                     const SmallVectorImpl<int> &Bytes) {
+  for (unsigned I = 0; I < 2; ++I)
+    Ops[I] = DAG.getNode(ISD::BITCAST, DL, MVT::v16i8, Ops[I]);
+
+  // First see whether VSLDI can be used.
+  unsigned StartIndex, OpNo0, OpNo1;
+  if (isShlDoublePermute(Bytes, StartIndex, OpNo0, OpNo1))
+    return DAG.getNode(SystemZISD::SHL_DOUBLE, DL, MVT::v16i8, Ops[OpNo0],
+                       Ops[OpNo1], DAG.getConstant(StartIndex, DL, MVT::i32));
+
+  // Fall back on VPERM.  Construct an SDNode for the permute vector.
+  SDValue IndexNodes[SystemZ::VectorBytes];
+  for (unsigned I = 0; I < SystemZ::VectorBytes; ++I)
+    if (Bytes[I] >= 0)
+      IndexNodes[I] = DAG.getConstant(Bytes[I], DL, MVT::i32);
+    else
+      IndexNodes[I] = DAG.getUNDEF(MVT::i32);
+  SDValue Op2 = DAG.getNode(ISD::BUILD_VECTOR, DL, MVT::v16i8, IndexNodes);
+  return DAG.getNode(SystemZISD::PERMUTE, DL, MVT::v16i8, Ops[0], Ops[1], Op2);
+}
+
+namespace {
+// Describes a general N-operand vector shuffle.
+struct GeneralShuffle {
+  GeneralShuffle(EVT vt) : VT(vt) {}
+  void addUndef();
+  void add(SDValue, unsigned);
+  SDValue getNode(SelectionDAG &, SDLoc);
+
+  // The operands of the shuffle.
+  SmallVector<SDValue, SystemZ::VectorBytes> Ops;
+
+  // Index I is -1 if byte I of the result is undefined.  Otherwise the
+  // result comes from byte Bytes[I] % SystemZ::VectorBytes of operand
+  // Bytes[I] / SystemZ::VectorBytes.
+  SmallVector<int, SystemZ::VectorBytes> Bytes;
+
+  // The type of the shuffle result.
+  EVT VT;
+};
+}
+
+// Add an extra undefined element to the shuffle.
+void GeneralShuffle::addUndef() {
+  unsigned BytesPerElement = VT.getVectorElementType().getStoreSize();
+  for (unsigned I = 0; I < BytesPerElement; ++I)
+    Bytes.push_back(-1);
+}
+
+// Add an extra element to the shuffle, taking it from element Elem of Op.
+// A null Op indicates a vector input whose value will be calculated later;
+// there is at most one such input per shuffle and it always has the same
+// type as the result.
+void GeneralShuffle::add(SDValue Op, unsigned Elem) {
+  unsigned BytesPerElement = VT.getVectorElementType().getStoreSize();
+
+  // The source vector can have wider elements than the result,
+  // either through an explicit TRUNCATE or because of type legalization.
+  // We want the least significant part.
+  EVT FromVT = Op.getNode() ? Op.getValueType() : VT;
+  unsigned FromBytesPerElement = FromVT.getVectorElementType().getStoreSize();
+  assert(FromBytesPerElement >= BytesPerElement &&
+         "Invalid EXTRACT_VECTOR_ELT");
+  unsigned Byte = ((Elem * FromBytesPerElement) % SystemZ::VectorBytes +
+                   (FromBytesPerElement - BytesPerElement));
+
+  // Look through things like shuffles and bitcasts.
+  while (Op.getNode()) {
+    if (Op.getOpcode() == ISD::BITCAST)
+      Op = Op.getOperand(0);
+    else if (Op.getOpcode() == ISD::VECTOR_SHUFFLE && Op.hasOneUse()) {
+      // See whether the bytes we need come from a contiguous part of one
+      // operand.
+      SmallVector<int, SystemZ::VectorBytes> OpBytes;
+      getVPermMask(cast<ShuffleVectorSDNode>(Op), OpBytes);
+      int NewByte;
+      if (!getShuffleInput(OpBytes, Byte, BytesPerElement, NewByte))
+        break;
+      if (NewByte < 0) {
+        addUndef();
+        return;
+      }
+      Op = Op.getOperand(unsigned(NewByte) / SystemZ::VectorBytes);
+      Byte = unsigned(NewByte) % SystemZ::VectorBytes;
+    } else if (Op.getOpcode() == ISD::UNDEF) {
+      addUndef();
+      return;
+    } else
+      break;
+  }
+
+  // Make sure that the source of the extraction is in Ops.
+  unsigned OpNo = 0;
+  for (; OpNo < Ops.size(); ++OpNo)
+    if (Ops[OpNo] == Op)
+      break;
+  if (OpNo == Ops.size())
+    Ops.push_back(Op);
+
+  // Add the element to Bytes.
+  unsigned Base = OpNo * SystemZ::VectorBytes + Byte;
+  for (unsigned I = 0; I < BytesPerElement; ++I)
+    Bytes.push_back(Base + I);
+}
+
+// Return SDNodes for the completed shuffle.
+SDValue GeneralShuffle::getNode(SelectionDAG &DAG, SDLoc DL) {
+  assert(Bytes.size() == SystemZ::VectorBytes && "Incomplete vector");
+
+  if (Ops.size() == 0)
+    return DAG.getUNDEF(VT);
+
+  // Make sure that there are at least two shuffle operands.
+  if (Ops.size() == 1)
+    Ops.push_back(DAG.getUNDEF(MVT::v16i8));
+
+  // Create a tree of shuffles, deferring root node until after the loop.
+  // Try to redistribute the undefined elements of non-root nodes so that
+  // the non-root shuffles match something like a pack or merge, then adjust
+  // the parent node's permute vector to compensate for the new order.
+  // Among other things, this copes with vectors like <2 x i16> that were
+  // padded with undefined elements during type legalization.
+  //
+  // In the best case this redistribution will lead to the whole tree
+  // using packs and merges.  It should rarely be a loss in other cases.
+  unsigned Stride = 1;
+  for (; Stride * 2 < Ops.size(); Stride *= 2) {
+    for (unsigned I = 0; I < Ops.size() - Stride; I += Stride * 2) {
+      SDValue SubOps[] = { Ops[I], Ops[I + Stride] };
+
+      // Create a mask for just these two operands.
+      SmallVector<int, SystemZ::VectorBytes> NewBytes(SystemZ::VectorBytes);
+      for (unsigned J = 0; J < SystemZ::VectorBytes; ++J) {
+        unsigned OpNo = unsigned(Bytes[J]) / SystemZ::VectorBytes;
+        unsigned Byte = unsigned(Bytes[J]) % SystemZ::VectorBytes;
+        if (OpNo == I)
+          NewBytes[J] = Byte;
+        else if (OpNo == I + Stride)
+          NewBytes[J] = SystemZ::VectorBytes + Byte;
+        else
+          NewBytes[J] = -1;
+      }
+      // See if it would be better to reorganize NewMask to avoid using VPERM.
+      SmallVector<int, SystemZ::VectorBytes> NewBytesMap(SystemZ::VectorBytes);
+      if (const Permute *P = matchDoublePermute(NewBytes, NewBytesMap)) {
+        Ops[I] = getPermuteNode(DAG, DL, *P, SubOps[0], SubOps[1]);
+        // Applying NewBytesMap to Ops[I] gets back to NewBytes.
+        for (unsigned J = 0; J < SystemZ::VectorBytes; ++J) {
+          if (NewBytes[J] >= 0) {
+            assert(unsigned(NewBytesMap[J]) < SystemZ::VectorBytes &&
+                   "Invalid double permute");
+            Bytes[J] = I * SystemZ::VectorBytes + NewBytesMap[J];
+          } else
+            assert(NewBytesMap[J] < 0 && "Invalid double permute");
+        }
+      } else {
+        // Just use NewBytes on the operands.
+        Ops[I] = getGeneralPermuteNode(DAG, DL, SubOps, NewBytes);
+        for (unsigned J = 0; J < SystemZ::VectorBytes; ++J)
+          if (NewBytes[J] >= 0)
+            Bytes[J] = I * SystemZ::VectorBytes + J;
+      }
+    }
+  }
+
+  // Now we just have 2 inputs.  Put the second operand in Ops[1].
+  if (Stride > 1) {
+    Ops[1] = Ops[Stride];
+    for (unsigned I = 0; I < SystemZ::VectorBytes; ++I)
+      if (Bytes[I] >= int(SystemZ::VectorBytes))
+        Bytes[I] -= (Stride - 1) * SystemZ::VectorBytes;
+  }
+
+  // Look for an instruction that can do the permute without resorting
+  // to VPERM.
+  unsigned OpNo0, OpNo1;
+  SDValue Op;
+  if (const Permute *P = matchPermute(Bytes, OpNo0, OpNo1))
+    Op = getPermuteNode(DAG, DL, *P, Ops[OpNo0], Ops[OpNo1]);
+  else
+    Op = getGeneralPermuteNode(DAG, DL, &Ops[0], Bytes);
+  return DAG.getNode(ISD::BITCAST, DL, VT, Op);
+}
+
+// Return true if the given BUILD_VECTOR is a scalar-to-vector conversion.
+static bool isScalarToVector(SDValue Op) {
+  for (unsigned I = 1, E = Op.getNumOperands(); I != E; ++I)
+    if (Op.getOperand(I).getOpcode() != ISD::UNDEF)
+      return false;
+  return true;
+}
+
+// Return a vector of type VT that contains Value in the first element.
+// The other elements don't matter.
+static SDValue buildScalarToVector(SelectionDAG &DAG, SDLoc DL, EVT VT,
+                                   SDValue Value) {
+  // If we have a constant, replicate it to all elements and let the
+  // BUILD_VECTOR lowering take care of it.
+  if (Value.getOpcode() == ISD::Constant ||
+      Value.getOpcode() == ISD::ConstantFP) {
+    SmallVector<SDValue, 16> Ops(VT.getVectorNumElements(), Value);
+    return DAG.getNode(ISD::BUILD_VECTOR, DL, VT, Ops);
+  }
+  if (Value.getOpcode() == ISD::UNDEF)
+    return DAG.getUNDEF(VT);
+  return DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, VT, Value);
+}
+
+// Return a vector of type VT in which Op0 is in element 0 and Op1 is in
+// element 1.  Used for cases in which replication is cheap.
+static SDValue buildMergeScalars(SelectionDAG &DAG, SDLoc DL, EVT VT,
+                                 SDValue Op0, SDValue Op1) {
+  if (Op0.getOpcode() == ISD::UNDEF) {
+    if (Op1.getOpcode() == ISD::UNDEF)
+      return DAG.getUNDEF(VT);
+    return DAG.getNode(SystemZISD::REPLICATE, DL, VT, Op1);
+  }
+  if (Op1.getOpcode() == ISD::UNDEF)
+    return DAG.getNode(SystemZISD::REPLICATE, DL, VT, Op0);
+  return DAG.getNode(SystemZISD::MERGE_HIGH, DL, VT,
+                     buildScalarToVector(DAG, DL, VT, Op0),
+                     buildScalarToVector(DAG, DL, VT, Op1));
+}
+
+// Extend GPR scalars Op0 and Op1 to doublewords and return a v2i64
+// vector for them.
+static SDValue joinDwords(SelectionDAG &DAG, SDLoc DL, SDValue Op0,
+                          SDValue Op1) {
+  if (Op0.getOpcode() == ISD::UNDEF && Op1.getOpcode() == ISD::UNDEF)
+    return DAG.getUNDEF(MVT::v2i64);
+  // If one of the two inputs is undefined then replicate the other one,
+  // in order to avoid using another register unnecessarily.
+  if (Op0.getOpcode() == ISD::UNDEF)
+    Op0 = Op1 = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, Op1);
+  else if (Op1.getOpcode() == ISD::UNDEF)
+    Op0 = Op1 = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, Op0);
+  else {
+    Op0 = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, Op0);
+    Op1 = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, Op1);
+  }
+  return DAG.getNode(SystemZISD::JOIN_DWORDS, DL, MVT::v2i64, Op0, Op1);
+}
+
+// Try to represent constant BUILD_VECTOR node BVN using a
+// SystemZISD::BYTE_MASK-style mask.  Store the mask value in Mask
+// on success.
+static bool tryBuildVectorByteMask(BuildVectorSDNode *BVN, uint64_t &Mask) {
+  EVT ElemVT = BVN->getValueType(0).getVectorElementType();
+  unsigned BytesPerElement = ElemVT.getStoreSize();
+  for (unsigned I = 0, E = BVN->getNumOperands(); I != E; ++I) {
+    SDValue Op = BVN->getOperand(I);
+    if (Op.getOpcode() != ISD::UNDEF) {
+      uint64_t Value;
+      if (Op.getOpcode() == ISD::Constant)
+        Value = dyn_cast<ConstantSDNode>(Op)->getZExtValue();
+      else if (Op.getOpcode() == ISD::ConstantFP)
+        Value = (dyn_cast<ConstantFPSDNode>(Op)->getValueAPF().bitcastToAPInt()
+                 .getZExtValue());
+      else
+        return false;
+      for (unsigned J = 0; J < BytesPerElement; ++J) {
+        uint64_t Byte = (Value >> (J * 8)) & 0xff;
+        if (Byte == 0xff)
+          Mask |= 1ULL << ((E - I - 1) * BytesPerElement + J);
+        else if (Byte != 0)
+          return false;
+      }
+    }
+  }
+  return true;
+}
+
+// Try to load a vector constant in which BitsPerElement-bit value Value
+// is replicated to fill the vector.  VT is the type of the resulting
+// constant, which may have elements of a different size from BitsPerElement.
+// Return the SDValue of the constant on success, otherwise return
+// an empty value.
+static SDValue tryBuildVectorReplicate(SelectionDAG &DAG,
+                                       const SystemZInstrInfo *TII,
+                                       SDLoc DL, EVT VT, uint64_t Value,
+                                       unsigned BitsPerElement) {
+  // Signed 16-bit values can be replicated using VREPI.
+  int64_t SignedValue = SignExtend64(Value, BitsPerElement);
+  if (isInt<16>(SignedValue)) {
+    MVT VecVT = MVT::getVectorVT(MVT::getIntegerVT(BitsPerElement),
+                                 SystemZ::VectorBits / BitsPerElement);
+    SDValue Op = DAG.getNode(SystemZISD::REPLICATE, DL, VecVT,
+                             DAG.getConstant(SignedValue, DL, MVT::i32));
+    return DAG.getNode(ISD::BITCAST, DL, VT, Op);
+  }
+  // See whether rotating the constant left some N places gives a value that
+  // is one less than a power of 2 (i.e. all zeros followed by all ones).
+  // If so we can use VGM.
+  unsigned Start, End;
+  if (TII->isRxSBGMask(Value, BitsPerElement, Start, End)) {
+    // isRxSBGMask returns the bit numbers for a full 64-bit value,
+    // with 0 denoting 1 << 63 and 63 denoting 1.  Convert them to
+    // bit numbers for an BitsPerElement value, so that 0 denotes
+    // 1 << (BitsPerElement-1).
+    Start -= 64 - BitsPerElement;
+    End -= 64 - BitsPerElement;
+    MVT VecVT = MVT::getVectorVT(MVT::getIntegerVT(BitsPerElement),
+                                 SystemZ::VectorBits / BitsPerElement);
+    SDValue Op = DAG.getNode(SystemZISD::ROTATE_MASK, DL, VecVT,
+                             DAG.getConstant(Start, DL, MVT::i32),
+                             DAG.getConstant(End, DL, MVT::i32));
+    return DAG.getNode(ISD::BITCAST, DL, VT, Op);
+  }
+  return SDValue();
+}
+
+// If a BUILD_VECTOR contains some EXTRACT_VECTOR_ELTs, it's usually
+// better to use VECTOR_SHUFFLEs on them, only using BUILD_VECTOR for
+// the non-EXTRACT_VECTOR_ELT elements.  See if the given BUILD_VECTOR
+// would benefit from this representation and return it if so.
+static SDValue tryBuildVectorShuffle(SelectionDAG &DAG,
+                                     BuildVectorSDNode *BVN) {
+  EVT VT = BVN->getValueType(0);
+  unsigned NumElements = VT.getVectorNumElements();
+
+  // Represent the BUILD_VECTOR as an N-operand VECTOR_SHUFFLE-like operation
+  // on byte vectors.  If there are non-EXTRACT_VECTOR_ELT elements that still
+  // need a BUILD_VECTOR, add an additional placeholder operand for that
+  // BUILD_VECTOR and store its operands in ResidueOps.
+  GeneralShuffle GS(VT);
+  SmallVector<SDValue, SystemZ::VectorBytes> ResidueOps;
+  bool FoundOne = false;
+  for (unsigned I = 0; I < NumElements; ++I) {
+    SDValue Op = BVN->getOperand(I);
+    if (Op.getOpcode() == ISD::TRUNCATE)
+      Op = Op.getOperand(0);
+    if (Op.getOpcode() == ISD::EXTRACT_VECTOR_ELT &&
+        Op.getOperand(1).getOpcode() == ISD::Constant) {
+      unsigned Elem = cast<ConstantSDNode>(Op.getOperand(1))->getZExtValue();
+      GS.add(Op.getOperand(0), Elem);
+      FoundOne = true;
+    } else if (Op.getOpcode() == ISD::UNDEF) {
+      GS.addUndef();
+    } else {
+      GS.add(SDValue(), ResidueOps.size());
+      ResidueOps.push_back(Op);
+    }
+  }
+
+  // Nothing to do if there are no EXTRACT_VECTOR_ELTs.
+  if (!FoundOne)
+    return SDValue();
+
+  // Create the BUILD_VECTOR for the remaining elements, if any.
+  if (!ResidueOps.empty()) {
+    while (ResidueOps.size() < NumElements)
+      ResidueOps.push_back(DAG.getUNDEF(VT.getVectorElementType()));
+    for (auto &Op : GS.Ops) {
+      if (!Op.getNode()) {
+        Op = DAG.getNode(ISD::BUILD_VECTOR, SDLoc(BVN), VT, ResidueOps);
+        break;
+      }
+    }
+  }
+  return GS.getNode(DAG, SDLoc(BVN));
+}
+
+// Combine GPR scalar values Elems into a vector of type VT.
+static SDValue buildVector(SelectionDAG &DAG, SDLoc DL, EVT VT,
+                           SmallVectorImpl<SDValue> &Elems) {
+  // See whether there is a single replicated value.
+  SDValue Single;
+  unsigned int NumElements = Elems.size();
+  unsigned int Count = 0;
+  for (auto Elem : Elems) {
+    if (Elem.getOpcode() != ISD::UNDEF) {
+      if (!Single.getNode())
+        Single = Elem;
+      else if (Elem != Single) {
+        Single = SDValue();
+        break;
+      }
+      Count += 1;
+    }
+  }
+  // There are three cases here:
+  //
+  // - if the only defined element is a loaded one, the best sequence
+  //   is a replicating load.
+  //
+  // - otherwise, if the only defined element is an i64 value, we will
+  //   end up with the same VLVGP sequence regardless of whether we short-cut
+  //   for replication or fall through to the later code.
+  //
+  // - otherwise, if the only defined element is an i32 or smaller value,
+  //   we would need 2 instructions to replicate it: VLVGP followed by VREPx.
+  //   This is only a win if the single defined element is used more than once.
+  //   In other cases we're better off using a single VLVGx.
+  if (Single.getNode() && (Count > 1 || Single.getOpcode() == ISD::LOAD))
+    return DAG.getNode(SystemZISD::REPLICATE, DL, VT, Single);
+
+  // The best way of building a v2i64 from two i64s is to use VLVGP.
+  if (VT == MVT::v2i64)
+    return joinDwords(DAG, DL, Elems[0], Elems[1]);
+
+  // Use a 64-bit merge high to combine two doubles.
+  if (VT == MVT::v2f64)
+    return buildMergeScalars(DAG, DL, VT, Elems[0], Elems[1]);
+
+  // Build v4f32 values directly from the FPRs:
+  //
+  //   <Axxx> <Bxxx> <Cxxxx> <Dxxx>
+  //         V              V         VMRHF
+  //      <ABxx>         <CDxx>
+  //                V                 VMRHG
+  //              <ABCD>
+  if (VT == MVT::v4f32) {
+    SDValue Op01 = buildMergeScalars(DAG, DL, VT, Elems[0], Elems[1]);
+    SDValue Op23 = buildMergeScalars(DAG, DL, VT, Elems[2], Elems[3]);
+    // Avoid unnecessary undefs by reusing the other operand.
+    if (Op01.getOpcode() == ISD::UNDEF)
+      Op01 = Op23;
+    else if (Op23.getOpcode() == ISD::UNDEF)
+      Op23 = Op01;
+    // Merging identical replications is a no-op.
+    if (Op01.getOpcode() == SystemZISD::REPLICATE && Op01 == Op23)
+      return Op01;
+    Op01 = DAG.getNode(ISD::BITCAST, DL, MVT::v2i64, Op01);
+    Op23 = DAG.getNode(ISD::BITCAST, DL, MVT::v2i64, Op23);
+    SDValue Op = DAG.getNode(SystemZISD::MERGE_HIGH,
+                             DL, MVT::v2i64, Op01, Op23);
+    return DAG.getNode(ISD::BITCAST, DL, VT, Op);
+  }
+
+  // Collect the constant terms.
+  SmallVector<SDValue, SystemZ::VectorBytes> Constants(NumElements, SDValue());
+  SmallVector<bool, SystemZ::VectorBytes> Done(NumElements, false);
+
+  unsigned NumConstants = 0;
+  for (unsigned I = 0; I < NumElements; ++I) {
+    SDValue Elem = Elems[I];
+    if (Elem.getOpcode() == ISD::Constant ||
+        Elem.getOpcode() == ISD::ConstantFP) {
+      NumConstants += 1;
+      Constants[I] = Elem;
+      Done[I] = true;
+    }
+  }
+  // If there was at least one constant, fill in the other elements of
+  // Constants with undefs to get a full vector constant and use that
+  // as the starting point.
+  SDValue Result;
+  if (NumConstants > 0) {
+    for (unsigned I = 0; I < NumElements; ++I)
+      if (!Constants[I].getNode())
+        Constants[I] = DAG.getUNDEF(Elems[I].getValueType());
+    Result = DAG.getNode(ISD::BUILD_VECTOR, DL, VT, Constants);
+  } else {
+    // Otherwise try to use VLVGP to start the sequence in order to
+    // avoid a false dependency on any previous contents of the vector
+    // register.  This only makes sense if one of the associated elements
+    // is defined.
+    unsigned I1 = NumElements / 2 - 1;
+    unsigned I2 = NumElements - 1;
+    bool Def1 = (Elems[I1].getOpcode() != ISD::UNDEF);
+    bool Def2 = (Elems[I2].getOpcode() != ISD::UNDEF);
+    if (Def1 || Def2) {
+      SDValue Elem1 = Elems[Def1 ? I1 : I2];
+      SDValue Elem2 = Elems[Def2 ? I2 : I1];
+      Result = DAG.getNode(ISD::BITCAST, DL, VT,
+                           joinDwords(DAG, DL, Elem1, Elem2));
+      Done[I1] = true;
+      Done[I2] = true;
+    } else
+      Result = DAG.getUNDEF(VT);
+  }
+
+  // Use VLVGx to insert the other elements.
+  for (unsigned I = 0; I < NumElements; ++I)
+    if (!Done[I] && Elems[I].getOpcode() != ISD::UNDEF)
+      Result = DAG.getNode(ISD::INSERT_VECTOR_ELT, DL, VT, Result, Elems[I],
+                           DAG.getConstant(I, DL, MVT::i32));
+  return Result;
+}
+
+SDValue SystemZTargetLowering::lowerBUILD_VECTOR(SDValue Op,
+                                                 SelectionDAG &DAG) const {
+  const SystemZInstrInfo *TII =
+    static_cast<const SystemZInstrInfo *>(Subtarget.getInstrInfo());
+  auto *BVN = cast<BuildVectorSDNode>(Op.getNode());
+  SDLoc DL(Op);
+  EVT VT = Op.getValueType();
+
+  if (BVN->isConstant()) {
+    // Try using VECTOR GENERATE BYTE MASK.  This is the architecturally-
+    // preferred way of creating all-zero and all-one vectors so give it
+    // priority over other methods below.
+    uint64_t Mask = 0;
+    if (tryBuildVectorByteMask(BVN, Mask)) {
+      SDValue Op = DAG.getNode(SystemZISD::BYTE_MASK, DL, MVT::v16i8,
+                               DAG.getConstant(Mask, DL, MVT::i32));
+      return DAG.getNode(ISD::BITCAST, DL, VT, Op);
+    }
+
+    // Try using some form of replication.
+    APInt SplatBits, SplatUndef;
+    unsigned SplatBitSize;
+    bool HasAnyUndefs;
+    if (BVN->isConstantSplat(SplatBits, SplatUndef, SplatBitSize, HasAnyUndefs,
+                             8, true) &&
+        SplatBitSize <= 64) {
+      // First try assuming that any undefined bits above the highest set bit
+      // and below the lowest set bit are 1s.  This increases the likelihood of
+      // being able to use a sign-extended element value in VECTOR REPLICATE
+      // IMMEDIATE or a wraparound mask in VECTOR GENERATE MASK.
+      uint64_t SplatBitsZ = SplatBits.getZExtValue();
+      uint64_t SplatUndefZ = SplatUndef.getZExtValue();
+      uint64_t Lower = (SplatUndefZ
+                        & ((uint64_t(1) << findFirstSet(SplatBitsZ)) - 1));
+      uint64_t Upper = (SplatUndefZ
+                        & ~((uint64_t(1) << findLastSet(SplatBitsZ)) - 1));
+      uint64_t Value = SplatBitsZ | Upper | Lower;
+      SDValue Op = tryBuildVectorReplicate(DAG, TII, DL, VT, Value,
+                                           SplatBitSize);
+      if (Op.getNode())
+        return Op;
+
+      // Now try assuming that any undefined bits between the first and
+      // last defined set bits are set.  This increases the chances of
+      // using a non-wraparound mask.
+      uint64_t Middle = SplatUndefZ & ~Upper & ~Lower;
+      Value = SplatBitsZ | Middle;
+      Op = tryBuildVectorReplicate(DAG, TII, DL, VT, Value, SplatBitSize);
+      if (Op.getNode())
+        return Op;
+    }
+
+    // Fall back to loading it from memory.
+    return SDValue();
+  }
+
+  // See if we should use shuffles to construct the vector from other vectors.
+  SDValue Res = tryBuildVectorShuffle(DAG, BVN);
+  if (Res.getNode())
+    return Res;
+
+  // Detect SCALAR_TO_VECTOR conversions.
+  if (isOperationLegal(ISD::SCALAR_TO_VECTOR, VT) && isScalarToVector(Op))
+    return buildScalarToVector(DAG, DL, VT, Op.getOperand(0));
+
+  // Otherwise use buildVector to build the vector up from GPRs.
+  unsigned NumElements = Op.getNumOperands();
+  SmallVector<SDValue, SystemZ::VectorBytes> Ops(NumElements);
+  for (unsigned I = 0; I < NumElements; ++I)
+    Ops[I] = Op.getOperand(I);
+  return buildVector(DAG, DL, VT, Ops);
+}
+
+SDValue SystemZTargetLowering::lowerVECTOR_SHUFFLE(SDValue Op,
+                                                   SelectionDAG &DAG) const {
+  auto *VSN = cast<ShuffleVectorSDNode>(Op.getNode());
+  SDLoc DL(Op);
+  EVT VT = Op.getValueType();
+  unsigned NumElements = VT.getVectorNumElements();
+
+  if (VSN->isSplat()) {
+    SDValue Op0 = Op.getOperand(0);
+    unsigned Index = VSN->getSplatIndex();
+    assert(Index < VT.getVectorNumElements() &&
+           "Splat index should be defined and in first operand");
+    // See whether the value we're splatting is directly available as a scalar.
+    if ((Index == 0 && Op0.getOpcode() == ISD::SCALAR_TO_VECTOR) ||
+        Op0.getOpcode() == ISD::BUILD_VECTOR)
+      return DAG.getNode(SystemZISD::REPLICATE, DL, VT, Op0.getOperand(Index));
+    // Otherwise keep it as a vector-to-vector operation.
+    return DAG.getNode(SystemZISD::SPLAT, DL, VT, Op.getOperand(0),
+                       DAG.getConstant(Index, DL, MVT::i32));
+  }
+
+  GeneralShuffle GS(VT);
+  for (unsigned I = 0; I < NumElements; ++I) {
+    int Elt = VSN->getMaskElt(I);
+    if (Elt < 0)
+      GS.addUndef();
+    else
+      GS.add(Op.getOperand(unsigned(Elt) / NumElements),
+             unsigned(Elt) % NumElements);
+  }
+  return GS.getNode(DAG, SDLoc(VSN));
+}
+
+SDValue SystemZTargetLowering::lowerSCALAR_TO_VECTOR(SDValue Op,
+                                                     SelectionDAG &DAG) const {
+  SDLoc DL(Op);
+  // Just insert the scalar into element 0 of an undefined vector.
+  return DAG.getNode(ISD::INSERT_VECTOR_ELT, DL,
+                     Op.getValueType(), DAG.getUNDEF(Op.getValueType()),
+                     Op.getOperand(0), DAG.getConstant(0, DL, MVT::i32));
+}
+
+SDValue SystemZTargetLowering::lowerINSERT_VECTOR_ELT(SDValue Op,
+                                                      SelectionDAG &DAG) const {
+  // Handle insertions of floating-point values.
+  SDLoc DL(Op);
+  SDValue Op0 = Op.getOperand(0);
+  SDValue Op1 = Op.getOperand(1);
+  SDValue Op2 = Op.getOperand(2);
+  EVT VT = Op.getValueType();
+
+  // Insertions into constant indices of a v2f64 can be done using VPDI.
+  // However, if the inserted value is a bitcast or a constant then it's
+  // better to use GPRs, as below.
+  if (VT == MVT::v2f64 &&
+      Op1.getOpcode() != ISD::BITCAST &&
+      Op1.getOpcode() != ISD::ConstantFP &&
+      Op2.getOpcode() == ISD::Constant) {
+    uint64_t Index = dyn_cast<ConstantSDNode>(Op2)->getZExtValue();
+    unsigned Mask = VT.getVectorNumElements() - 1;
+    if (Index <= Mask)
+      return Op;
+  }
+
+  // Otherwise bitcast to the equivalent integer form and insert via a GPR.
+  MVT IntVT = MVT::getIntegerVT(VT.getVectorElementType().getSizeInBits());
+  MVT IntVecVT = MVT::getVectorVT(IntVT, VT.getVectorNumElements());
+  SDValue Res = DAG.getNode(ISD::INSERT_VECTOR_ELT, DL, IntVecVT,
+                            DAG.getNode(ISD::BITCAST, DL, IntVecVT, Op0),
+                            DAG.getNode(ISD::BITCAST, DL, IntVT, Op1), Op2);
+  return DAG.getNode(ISD::BITCAST, DL, VT, Res);
+}
+
+SDValue
+SystemZTargetLowering::lowerEXTRACT_VECTOR_ELT(SDValue Op,
+                                               SelectionDAG &DAG) const {
+  // Handle extractions of floating-point values.
+  SDLoc DL(Op);
+  SDValue Op0 = Op.getOperand(0);
+  SDValue Op1 = Op.getOperand(1);
+  EVT VT = Op.getValueType();
+  EVT VecVT = Op0.getValueType();
+
+  // Extractions of constant indices can be done directly.
+  if (auto *CIndexN = dyn_cast<ConstantSDNode>(Op1)) {
+    uint64_t Index = CIndexN->getZExtValue();
+    unsigned Mask = VecVT.getVectorNumElements() - 1;
+    if (Index <= Mask)
+      return Op;
+  }
+
+  // Otherwise bitcast to the equivalent integer form and extract via a GPR.
+  MVT IntVT = MVT::getIntegerVT(VT.getSizeInBits());
+  MVT IntVecVT = MVT::getVectorVT(IntVT, VecVT.getVectorNumElements());
+  SDValue Res = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, IntVT,
+                            DAG.getNode(ISD::BITCAST, DL, IntVecVT, Op0), Op1);
+  return DAG.getNode(ISD::BITCAST, DL, VT, Res);
+}
+
+SDValue
+SystemZTargetLowering::lowerExtendVectorInreg(SDValue Op, SelectionDAG &DAG,
+					      unsigned UnpackHigh) const {
+  SDValue PackedOp = Op.getOperand(0);
+  EVT OutVT = Op.getValueType();
+  EVT InVT = PackedOp.getValueType();
+  unsigned ToBits = OutVT.getVectorElementType().getSizeInBits();
+  unsigned FromBits = InVT.getVectorElementType().getSizeInBits();
+  do {
+    FromBits *= 2;
+    EVT OutVT = MVT::getVectorVT(MVT::getIntegerVT(FromBits),
+                                 SystemZ::VectorBits / FromBits);
+    PackedOp = DAG.getNode(UnpackHigh, SDLoc(PackedOp), OutVT, PackedOp);
+  } while (FromBits != ToBits);
+  return PackedOp;
+}
+
+SDValue SystemZTargetLowering::lowerShift(SDValue Op, SelectionDAG &DAG,
+                                          unsigned ByScalar) const {
+  // Look for cases where a vector shift can use the *_BY_SCALAR form.
+  SDValue Op0 = Op.getOperand(0);
+  SDValue Op1 = Op.getOperand(1);
+  SDLoc DL(Op);
+  EVT VT = Op.getValueType();
+  unsigned ElemBitSize = VT.getVectorElementType().getSizeInBits();
+
+  // See whether the shift vector is a splat represented as BUILD_VECTOR.
+  if (auto *BVN = dyn_cast<BuildVectorSDNode>(Op1)) {
+    APInt SplatBits, SplatUndef;
+    unsigned SplatBitSize;
+    bool HasAnyUndefs;
+    // Check for constant splats.  Use ElemBitSize as the minimum element
+    // width and reject splats that need wider elements.
+    if (BVN->isConstantSplat(SplatBits, SplatUndef, SplatBitSize, HasAnyUndefs,
+                             ElemBitSize, true) &&
+        SplatBitSize == ElemBitSize) {
+      SDValue Shift = DAG.getConstant(SplatBits.getZExtValue() & 0xfff,
+                                      DL, MVT::i32);
+      return DAG.getNode(ByScalar, DL, VT, Op0, Shift);
+    }
+    // Check for variable splats.
+    BitVector UndefElements;
+    SDValue Splat = BVN->getSplatValue(&UndefElements);
+    if (Splat) {
+      // Since i32 is the smallest legal type, we either need a no-op
+      // or a truncation.
+      SDValue Shift = DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, Splat);
+      return DAG.getNode(ByScalar, DL, VT, Op0, Shift);
+    }
+  }
+
+  // See whether the shift vector is a splat represented as SHUFFLE_VECTOR,
+  // and the shift amount is directly available in a GPR.
+  if (auto *VSN = dyn_cast<ShuffleVectorSDNode>(Op1)) {
+    if (VSN->isSplat()) {
+      SDValue VSNOp0 = VSN->getOperand(0);
+      unsigned Index = VSN->getSplatIndex();
+      assert(Index < VT.getVectorNumElements() &&
+             "Splat index should be defined and in first operand");
+      if ((Index == 0 && VSNOp0.getOpcode() == ISD::SCALAR_TO_VECTOR) ||
+          VSNOp0.getOpcode() == ISD::BUILD_VECTOR) {
+        // Since i32 is the smallest legal type, we either need a no-op
+        // or a truncation.
+        SDValue Shift = DAG.getNode(ISD::TRUNCATE, DL, MVT::i32,
+                                    VSNOp0.getOperand(Index));
+        return DAG.getNode(ByScalar, DL, VT, Op0, Shift);
+      }
+    }
+  }
+
+  // Otherwise just treat the current form as legal.
+  return Op;
+}
+
 SDValue SystemZTargetLowering::LowerOperation(SDValue Op,
                                               SelectionDAG &DAG) const {
   switch (Op.getOpcode()) {
@@ -2731,6 +4302,12 @@ SDValue SystemZTargetLowering::LowerOperation(SDValue Op,
     return lowerOR(Op, DAG);
   case ISD::CTPOP:
     return lowerCTPOP(Op, DAG);
+  case ISD::CTLZ_ZERO_UNDEF:
+    return DAG.getNode(ISD::CTLZ, SDLoc(Op),
+                       Op.getValueType(), Op.getOperand(0));
+  case ISD::CTTZ_ZERO_UNDEF:
+    return DAG.getNode(ISD::CTTZ, SDLoc(Op),
+                       Op.getValueType(), Op.getOperand(0));
   case ISD::ATOMIC_SWAP:
     return lowerATOMIC_LOAD_OP(Op, DAG, SystemZISD::ATOMIC_SWAPW);
   case ISD::ATOMIC_STORE:
@@ -2767,6 +4344,28 @@ SDValue SystemZTargetLowering::LowerOperation(SDValue Op,
     return lowerPREFETCH(Op, DAG);
   case ISD::INTRINSIC_W_CHAIN:
     return lowerINTRINSIC_W_CHAIN(Op, DAG);
+  case ISD::INTRINSIC_WO_CHAIN:
+    return lowerINTRINSIC_WO_CHAIN(Op, DAG);
+  case ISD::BUILD_VECTOR:
+    return lowerBUILD_VECTOR(Op, DAG);
+  case ISD::VECTOR_SHUFFLE:
+    return lowerVECTOR_SHUFFLE(Op, DAG);
+  case ISD::SCALAR_TO_VECTOR:
+    return lowerSCALAR_TO_VECTOR(Op, DAG);
+  case ISD::INSERT_VECTOR_ELT:
+    return lowerINSERT_VECTOR_ELT(Op, DAG);
+  case ISD::EXTRACT_VECTOR_ELT:
+    return lowerEXTRACT_VECTOR_ELT(Op, DAG);
+  case ISD::SIGN_EXTEND_VECTOR_INREG:
+    return lowerExtendVectorInreg(Op, DAG, SystemZISD::UNPACK_HIGH);
+  case ISD::ZERO_EXTEND_VECTOR_INREG:
+    return lowerExtendVectorInreg(Op, DAG, SystemZISD::UNPACKL_HIGH);
+  case ISD::SHL:
+    return lowerShift(Op, DAG, SystemZISD::VSHL_BY_SCALAR);
+  case ISD::SRL:
+    return lowerShift(Op, DAG, SystemZISD::VSRL_BY_SCALAR);
+  case ISD::SRA:
+    return lowerShift(Op, DAG, SystemZISD::VSRA_BY_SCALAR);
   default:
     llvm_unreachable("Unexpected node to lower");
   }
@@ -2774,10 +4373,13 @@ SDValue SystemZTargetLowering::LowerOperation(SDValue Op,
 
 const char *SystemZTargetLowering::getTargetNodeName(unsigned Opcode) const {
 #define OPCODE(NAME) case SystemZISD::NAME: return "SystemZISD::" #NAME
-  switch (Opcode) {
+  switch ((SystemZISD::NodeType)Opcode) {
+    case SystemZISD::FIRST_NUMBER: break;
     OPCODE(RET_FLAG);
     OPCODE(CALL);
     OPCODE(SIBCALL);
+    OPCODE(TLS_GDCALL);
+    OPCODE(TLS_LDCALL);
     OPCODE(PCREL_WRAPPER);
     OPCODE(PCREL_OFFSET);
     OPCODE(IABS);
@@ -2788,7 +4390,9 @@ const char *SystemZTargetLowering::getTargetNodeName(unsigned Opcode) const {
     OPCODE(SELECT_CCMASK);
     OPCODE(ADJDYNALLOC);
     OPCODE(EXTRACT_ACCESS);
+    OPCODE(POPCNT);
     OPCODE(UMUL_LOHI64);
+    OPCODE(SDIVREM32);
     OPCODE(SDIVREM64);
     OPCODE(UDIVREM32);
     OPCODE(UDIVREM64);
@@ -2802,14 +4406,60 @@ const char *SystemZTargetLowering::getTargetNodeName(unsigned Opcode) const {
     OPCODE(XC_LOOP);
     OPCODE(CLC);
     OPCODE(CLC_LOOP);
-    OPCODE(STRCMP);
     OPCODE(STPCPY);
+    OPCODE(STRCMP);
     OPCODE(SEARCH_STRING);
     OPCODE(IPM);
     OPCODE(SERIALIZE);
     OPCODE(TBEGIN);
     OPCODE(TBEGIN_NOFLOAT);
     OPCODE(TEND);
+    OPCODE(BYTE_MASK);
+    OPCODE(ROTATE_MASK);
+    OPCODE(REPLICATE);
+    OPCODE(JOIN_DWORDS);
+    OPCODE(SPLAT);
+    OPCODE(MERGE_HIGH);
+    OPCODE(MERGE_LOW);
+    OPCODE(SHL_DOUBLE);
+    OPCODE(PERMUTE_DWORDS);
+    OPCODE(PERMUTE);
+    OPCODE(PACK);
+    OPCODE(PACKS_CC);
+    OPCODE(PACKLS_CC);
+    OPCODE(UNPACK_HIGH);
+    OPCODE(UNPACKL_HIGH);
+    OPCODE(UNPACK_LOW);
+    OPCODE(UNPACKL_LOW);
+    OPCODE(VSHL_BY_SCALAR);
+    OPCODE(VSRL_BY_SCALAR);
+    OPCODE(VSRA_BY_SCALAR);
+    OPCODE(VSUM);
+    OPCODE(VICMPE);
+    OPCODE(VICMPH);
+    OPCODE(VICMPHL);
+    OPCODE(VICMPES);
+    OPCODE(VICMPHS);
+    OPCODE(VICMPHLS);
+    OPCODE(VFCMPE);
+    OPCODE(VFCMPH);
+    OPCODE(VFCMPHE);
+    OPCODE(VFCMPES);
+    OPCODE(VFCMPHS);
+    OPCODE(VFCMPHES);
+    OPCODE(VFTCI);
+    OPCODE(VEXTEND);
+    OPCODE(VROUND);
+    OPCODE(VTM);
+    OPCODE(VFAE_CC);
+    OPCODE(VFAEZ_CC);
+    OPCODE(VFEE_CC);
+    OPCODE(VFEEZ_CC);
+    OPCODE(VFENE_CC);
+    OPCODE(VFENEZ_CC);
+    OPCODE(VISTR_CC);
+    OPCODE(VSTRC_CC);
+    OPCODE(VSTRCZ_CC);
     OPCODE(ATOMIC_SWAPW);
     OPCODE(ATOMIC_LOADW_ADD);
     OPCODE(ATOMIC_LOADW_SUB);
@@ -2828,6 +4478,157 @@ const char *SystemZTargetLowering::getTargetNodeName(unsigned Opcode) const {
 #undef OPCODE
 }
 
+// Return true if VT is a vector whose elements are a whole number of bytes
+// in width.
+static bool canTreatAsByteVector(EVT VT) {
+  return VT.isVector() && VT.getVectorElementType().getSizeInBits() % 8 == 0;
+}
+
+// Try to simplify an EXTRACT_VECTOR_ELT from a vector of type VecVT
+// producing a result of type ResVT.  Op is a possibly bitcast version
+// of the input vector and Index is the index (based on type VecVT) that
+// should be extracted.  Return the new extraction if a simplification
+// was possible or if Force is true.
+SDValue SystemZTargetLowering::combineExtract(SDLoc DL, EVT ResVT, EVT VecVT,
+                                              SDValue Op, unsigned Index,
+                                              DAGCombinerInfo &DCI,
+                                              bool Force) const {
+  SelectionDAG &DAG = DCI.DAG;
+
+  // The number of bytes being extracted.
+  unsigned BytesPerElement = VecVT.getVectorElementType().getStoreSize();
+
+  for (;;) {
+    unsigned Opcode = Op.getOpcode();
+    if (Opcode == ISD::BITCAST)
+      // Look through bitcasts.
+      Op = Op.getOperand(0);
+    else if (Opcode == ISD::VECTOR_SHUFFLE &&
+             canTreatAsByteVector(Op.getValueType())) {
+      // Get a VPERM-like permute mask and see whether the bytes covered
+      // by the extracted element are a contiguous sequence from one
+      // source operand.
+      SmallVector<int, SystemZ::VectorBytes> Bytes;
+      getVPermMask(cast<ShuffleVectorSDNode>(Op), Bytes);
+      int First;
+      if (!getShuffleInput(Bytes, Index * BytesPerElement,
+                           BytesPerElement, First))
+        break;
+      if (First < 0)
+        return DAG.getUNDEF(ResVT);
+      // Make sure the contiguous sequence starts at a multiple of the
+      // original element size.
+      unsigned Byte = unsigned(First) % Bytes.size();
+      if (Byte % BytesPerElement != 0)
+        break;
+      // We can get the extracted value directly from an input.
+      Index = Byte / BytesPerElement;
+      Op = Op.getOperand(unsigned(First) / Bytes.size());
+      Force = true;
+    } else if (Opcode == ISD::BUILD_VECTOR &&
+               canTreatAsByteVector(Op.getValueType())) {
+      // We can only optimize this case if the BUILD_VECTOR elements are
+      // at least as wide as the extracted value.
+      EVT OpVT = Op.getValueType();
+      unsigned OpBytesPerElement = OpVT.getVectorElementType().getStoreSize();
+      if (OpBytesPerElement < BytesPerElement)
+        break;
+      // Make sure that the least-significant bit of the extracted value
+      // is the least significant bit of an input.
+      unsigned End = (Index + 1) * BytesPerElement;
+      if (End % OpBytesPerElement != 0)
+        break;
+      // We're extracting the low part of one operand of the BUILD_VECTOR.
+      Op = Op.getOperand(End / OpBytesPerElement - 1);
+      if (!Op.getValueType().isInteger()) {
+        EVT VT = MVT::getIntegerVT(Op.getValueType().getSizeInBits());
+        Op = DAG.getNode(ISD::BITCAST, DL, VT, Op);
+        DCI.AddToWorklist(Op.getNode());
+      }
+      EVT VT = MVT::getIntegerVT(ResVT.getSizeInBits());
+      Op = DAG.getNode(ISD::TRUNCATE, DL, VT, Op);
+      if (VT != ResVT) {
+        DCI.AddToWorklist(Op.getNode());
+        Op = DAG.getNode(ISD::BITCAST, DL, ResVT, Op);
+      }
+      return Op;
+    } else if ((Opcode == ISD::SIGN_EXTEND_VECTOR_INREG ||
+		Opcode == ISD::ZERO_EXTEND_VECTOR_INREG ||
+		Opcode == ISD::ANY_EXTEND_VECTOR_INREG) &&
+	       canTreatAsByteVector(Op.getValueType()) &&
+               canTreatAsByteVector(Op.getOperand(0).getValueType())) {
+      // Make sure that only the unextended bits are significant.
+      EVT ExtVT = Op.getValueType();
+      EVT OpVT = Op.getOperand(0).getValueType();
+      unsigned ExtBytesPerElement = ExtVT.getVectorElementType().getStoreSize();
+      unsigned OpBytesPerElement = OpVT.getVectorElementType().getStoreSize();
+      unsigned Byte = Index * BytesPerElement;
+      unsigned SubByte = Byte % ExtBytesPerElement;
+      unsigned MinSubByte = ExtBytesPerElement - OpBytesPerElement;
+      if (SubByte < MinSubByte ||
+	  SubByte + BytesPerElement > ExtBytesPerElement)
+	break;
+      // Get the byte offset of the unextended element
+      Byte = Byte / ExtBytesPerElement * OpBytesPerElement;
+      // ...then add the byte offset relative to that element.
+      Byte += SubByte - MinSubByte;
+      if (Byte % BytesPerElement != 0)
+	break;
+      Op = Op.getOperand(0);
+      Index = Byte / BytesPerElement;
+      Force = true;
+    } else
+      break;
+  }
+  if (Force) {
+    if (Op.getValueType() != VecVT) {
+      Op = DAG.getNode(ISD::BITCAST, DL, VecVT, Op);
+      DCI.AddToWorklist(Op.getNode());
+    }
+    return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, ResVT, Op,
+                       DAG.getConstant(Index, DL, MVT::i32));
+  }
+  return SDValue();
+}
+
+// Optimize vector operations in scalar value Op on the basis that Op
+// is truncated to TruncVT.
+SDValue
+SystemZTargetLowering::combineTruncateExtract(SDLoc DL, EVT TruncVT, SDValue Op,
+                                              DAGCombinerInfo &DCI) const {
+  // If we have (trunc (extract_vector_elt X, Y)), try to turn it into
+  // (extract_vector_elt (bitcast X), Y'), where (bitcast X) has elements
+  // of type TruncVT.
+  if (Op.getOpcode() == ISD::EXTRACT_VECTOR_ELT &&
+      TruncVT.getSizeInBits() % 8 == 0) {
+    SDValue Vec = Op.getOperand(0);
+    EVT VecVT = Vec.getValueType();
+    if (canTreatAsByteVector(VecVT)) {
+      if (auto *IndexN = dyn_cast<ConstantSDNode>(Op.getOperand(1))) {
+        unsigned BytesPerElement = VecVT.getVectorElementType().getStoreSize();
+        unsigned TruncBytes = TruncVT.getStoreSize();
+        if (BytesPerElement % TruncBytes == 0) {
+          // Calculate the value of Y' in the above description.  We are
+          // splitting the original elements into Scale equal-sized pieces
+          // and for truncation purposes want the last (least-significant)
+          // of these pieces for IndexN.  This is easiest to do by calculating
+          // the start index of the following element and then subtracting 1.
+          unsigned Scale = BytesPerElement / TruncBytes;
+          unsigned NewIndex = (IndexN->getZExtValue() + 1) * Scale - 1;
+
+          // Defer the creation of the bitcast from X to combineExtract,
+          // which might be able to optimize the extraction.
+          VecVT = MVT::getVectorVT(MVT::getIntegerVT(TruncBytes * 8),
+                                   VecVT.getStoreSize() / TruncBytes);
+          EVT ResVT = (TruncBytes < 4 ? MVT::i32 : TruncVT);
+          return combineExtract(DL, ResVT, VecVT, Vec, NewIndex, DCI, true);
+        }
+      }
+    }
+  }
+  return SDValue();
+}
+
 SDValue SystemZTargetLowering::PerformDAGCombine(SDNode *N,
                                                  DAGCombinerInfo &DCI) const {
   SelectionDAG &DAG = DCI.DAG;
@@ -2851,9 +4652,118 @@ SDValue SystemZTargetLowering::PerformDAGCombine(SDNode *N,
           SDValue Ext = DAG.getNode(ISD::ANY_EXTEND, SDLoc(Inner), VT,
                                     Inner.getOperand(0));
           SDValue Shl = DAG.getNode(ISD::SHL, SDLoc(Inner), VT, Ext,
-                                    DAG.getConstant(NewShlAmt, ShiftVT));
+                                    DAG.getConstant(NewShlAmt, SDLoc(Inner),
+                                                    ShiftVT));
           return DAG.getNode(ISD::SRA, SDLoc(N0), VT, Shl,
-                             DAG.getConstant(NewSraAmt, ShiftVT));
+                             DAG.getConstant(NewSraAmt, SDLoc(N0), ShiftVT));
+        }
+      }
+    }
+  }
+  if (Opcode == SystemZISD::MERGE_HIGH ||
+      Opcode == SystemZISD::MERGE_LOW) {
+    SDValue Op0 = N->getOperand(0);
+    SDValue Op1 = N->getOperand(1);
+    if (Op0.getOpcode() == ISD::BITCAST)
+      Op0 = Op0.getOperand(0);
+    if (Op0.getOpcode() == SystemZISD::BYTE_MASK &&
+        cast<ConstantSDNode>(Op0.getOperand(0))->getZExtValue() == 0) {
+      // (z_merge_* 0, 0) -> 0.  This is mostly useful for using VLLEZF
+      // for v4f32.
+      if (Op1 == N->getOperand(0))
+        return Op1;
+      // (z_merge_? 0, X) -> (z_unpackl_? 0, X).
+      EVT VT = Op1.getValueType();
+      unsigned ElemBytes = VT.getVectorElementType().getStoreSize();
+      if (ElemBytes <= 4) {
+        Opcode = (Opcode == SystemZISD::MERGE_HIGH ?
+                  SystemZISD::UNPACKL_HIGH : SystemZISD::UNPACKL_LOW);
+        EVT InVT = VT.changeVectorElementTypeToInteger();
+        EVT OutVT = MVT::getVectorVT(MVT::getIntegerVT(ElemBytes * 16),
+                                     SystemZ::VectorBytes / ElemBytes / 2);
+        if (VT != InVT) {
+          Op1 = DAG.getNode(ISD::BITCAST, SDLoc(N), InVT, Op1);
+          DCI.AddToWorklist(Op1.getNode());
+        }
+        SDValue Op = DAG.getNode(Opcode, SDLoc(N), OutVT, Op1);
+        DCI.AddToWorklist(Op.getNode());
+        return DAG.getNode(ISD::BITCAST, SDLoc(N), VT, Op);
+      }
+    }
+  }
+  // If we have (truncstoreiN (extract_vector_elt X, Y), Z) then it is better
+  // for the extraction to be done on a vMiN value, so that we can use VSTE.
+  // If X has wider elements then convert it to:
+  // (truncstoreiN (extract_vector_elt (bitcast X), Y2), Z).
+  if (Opcode == ISD::STORE) {
+    auto *SN = cast<StoreSDNode>(N);
+    EVT MemVT = SN->getMemoryVT();
+    if (MemVT.isInteger()) {
+      SDValue Value = combineTruncateExtract(SDLoc(N), MemVT,
+                                             SN->getValue(), DCI);
+      if (Value.getNode()) {
+        DCI.AddToWorklist(Value.getNode());
+
+        // Rewrite the store with the new form of stored value.
+        return DAG.getTruncStore(SN->getChain(), SDLoc(SN), Value,
+                                 SN->getBasePtr(), SN->getMemoryVT(),
+                                 SN->getMemOperand());
+      }
+    }
+  }
+  // Try to simplify a vector extraction.
+  if (Opcode == ISD::EXTRACT_VECTOR_ELT) {
+    if (auto *IndexN = dyn_cast<ConstantSDNode>(N->getOperand(1))) {
+      SDValue Op0 = N->getOperand(0);
+      EVT VecVT = Op0.getValueType();
+      return combineExtract(SDLoc(N), N->getValueType(0), VecVT, Op0,
+                            IndexN->getZExtValue(), DCI, false);
+    }
+  }
+  // (join_dwords X, X) == (replicate X)
+  if (Opcode == SystemZISD::JOIN_DWORDS &&
+      N->getOperand(0) == N->getOperand(1))
+    return DAG.getNode(SystemZISD::REPLICATE, SDLoc(N), N->getValueType(0),
+                       N->getOperand(0));
+  // (fround (extract_vector_elt X 0))
+  // (fround (extract_vector_elt X 1)) ->
+  // (extract_vector_elt (VROUND X) 0)
+  // (extract_vector_elt (VROUND X) 1)
+  //
+  // This is a special case since the target doesn't really support v2f32s.
+  if (Opcode == ISD::FP_ROUND) {
+    SDValue Op0 = N->getOperand(0);
+    if (N->getValueType(0) == MVT::f32 &&
+        Op0.hasOneUse() &&
+        Op0.getOpcode() == ISD::EXTRACT_VECTOR_ELT &&
+        Op0.getOperand(0).getValueType() == MVT::v2f64 &&
+        Op0.getOperand(1).getOpcode() == ISD::Constant &&
+        cast<ConstantSDNode>(Op0.getOperand(1))->getZExtValue() == 0) {
+      SDValue Vec = Op0.getOperand(0);
+      for (auto *U : Vec->uses()) {
+        if (U != Op0.getNode() &&
+            U->hasOneUse() &&
+            U->getOpcode() == ISD::EXTRACT_VECTOR_ELT &&
+            U->getOperand(0) == Vec &&
+            U->getOperand(1).getOpcode() == ISD::Constant &&
+            cast<ConstantSDNode>(U->getOperand(1))->getZExtValue() == 1) {
+          SDValue OtherRound = SDValue(*U->use_begin(), 0);
+          if (OtherRound.getOpcode() == ISD::FP_ROUND &&
+              OtherRound.getOperand(0) == SDValue(U, 0) &&
+              OtherRound.getValueType() == MVT::f32) {
+            SDValue VRound = DAG.getNode(SystemZISD::VROUND, SDLoc(N),
+                                         MVT::v4f32, Vec);
+            DCI.AddToWorklist(VRound.getNode());
+            SDValue Extract1 =
+              DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SDLoc(U), MVT::f32,
+                          VRound, DAG.getConstant(2, SDLoc(U), MVT::i32));
+            DCI.AddToWorklist(Extract1.getNode());
+            DAG.ReplaceAllUsesOfValueWith(OtherRound, Extract1);
+            SDValue Extract0 =
+              DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SDLoc(Op0), MVT::f32,
+                          VRound, DAG.getConstant(0, SDLoc(Op0), MVT::i32));
+            return Extract0;
+          }
         }
       }
     }
@@ -3670,11 +5580,18 @@ SystemZTargetLowering::emitTransactionBegin(MachineInstr *MI,
     }
   }
 
-  // Add FPR clobbers.
+  // Add FPR/VR clobbers.
   if (!NoFloat && (Control & 4) != 0) {
-    for (int I = 0; I < 16; I++) {
-      unsigned Reg = SystemZMC::FP64Regs[I];
-      MI->addOperand(MachineOperand::CreateReg(Reg, true, true));
+    if (Subtarget.hasVector()) {
+      for (int I = 0; I < 32; I++) {
+        unsigned Reg = SystemZMC::VR128Regs[I];
+        MI->addOperand(MachineOperand::CreateReg(Reg, true, true));
+      }
+    } else {
+      for (int I = 0; I < 16; I++) {
+        unsigned Reg = SystemZMC::FP64Regs[I];
+        MI->addOperand(MachineOperand::CreateReg(Reg, true, true));
+      }
     }
   }
 
diff --git a/lib/Target/SystemZ/SystemZISelLowering.h b/lib/Target/SystemZ/SystemZISelLowering.h
index 56d7ef45568..949b67f114e 100644
--- a/lib/Target/SystemZ/SystemZISelLowering.h
+++ b/lib/Target/SystemZ/SystemZISelLowering.h
@@ -22,7 +22,7 @@
 
 namespace llvm {
 namespace SystemZISD {
-enum {
+enum NodeType : unsigned {
   FIRST_NUMBER = ISD::BUILTIN_OP_END,
 
   // Return with a flag operand.  Operand 0 is the chain operand.
@@ -155,6 +155,126 @@ enum {
   // Transaction end.  Just the chain operand.  Returns chain and glue.
   TEND,
 
+  // Create a vector constant by filling byte N of the result with bit
+  // 15-N of the single operand.
+  BYTE_MASK,
+
+  // Create a vector constant by replicating an element-sized RISBG-style mask.
+  // The first operand specifies the starting set bit and the second operand
+  // specifies the ending set bit.  Both operands count from the MSB of the
+  // element.
+  ROTATE_MASK,
+
+  // Replicate a GPR scalar value into all elements of a vector.
+  REPLICATE,
+
+  // Create a vector from two i64 GPRs.
+  JOIN_DWORDS,
+
+  // Replicate one element of a vector into all elements.  The first operand
+  // is the vector and the second is the index of the element to replicate.
+  SPLAT,
+
+  // Interleave elements from the high half of operand 0 and the high half
+  // of operand 1.
+  MERGE_HIGH,
+
+  // Likewise for the low halves.
+  MERGE_LOW,
+
+  // Concatenate the vectors in the first two operands, shift them left
+  // by the third operand, and take the first half of the result.
+  SHL_DOUBLE,
+
+  // Take one element of the first v2i64 operand and the one element of
+  // the second v2i64 operand and concatenate them to form a v2i64 result.
+  // The third operand is a 4-bit value of the form 0A0B, where A and B
+  // are the element selectors for the first operand and second operands
+  // respectively.
+  PERMUTE_DWORDS,
+
+  // Perform a general vector permute on vector operands 0 and 1.
+  // Each byte of operand 2 controls the corresponding byte of the result,
+  // in the same way as a byte-level VECTOR_SHUFFLE mask.
+  PERMUTE,
+
+  // Pack vector operands 0 and 1 into a single vector with half-sized elements.
+  PACK,
+
+  // Likewise, but saturate the result and set CC.  PACKS_CC does signed
+  // saturation and PACKLS_CC does unsigned saturation.
+  PACKS_CC,
+  PACKLS_CC,
+
+  // Unpack the first half of vector operand 0 into double-sized elements.
+  // UNPACK_HIGH sign-extends and UNPACKL_HIGH zero-extends.
+  UNPACK_HIGH,
+  UNPACKL_HIGH,
+
+  // Likewise for the second half.
+  UNPACK_LOW,
+  UNPACKL_LOW,
+
+  // Shift each element of vector operand 0 by the number of bits specified
+  // by scalar operand 1.
+  VSHL_BY_SCALAR,
+  VSRL_BY_SCALAR,
+  VSRA_BY_SCALAR,
+
+  // For each element of the output type, sum across all sub-elements of
+  // operand 0 belonging to the corresponding element, and add in the
+  // rightmost sub-element of the corresponding element of operand 1.
+  VSUM,
+
+  // Compare integer vector operands 0 and 1 to produce the usual 0/-1
+  // vector result.  VICMPE is for equality, VICMPH for "signed greater than"
+  // and VICMPHL for "unsigned greater than".
+  VICMPE,
+  VICMPH,
+  VICMPHL,
+
+  // Likewise, but also set the condition codes on the result.
+  VICMPES,
+  VICMPHS,
+  VICMPHLS,
+
+  // Compare floating-point vector operands 0 and 1 to preoduce the usual 0/-1
+  // vector result.  VFCMPE is for "ordered and equal", VFCMPH for "ordered and
+  // greater than" and VFCMPHE for "ordered and greater than or equal to".
+  VFCMPE,
+  VFCMPH,
+  VFCMPHE,
+
+  // Likewise, but also set the condition codes on the result.
+  VFCMPES,
+  VFCMPHS,
+  VFCMPHES,
+
+  // Test floating-point data class for vectors.
+  VFTCI,
+
+  // Extend the even f32 elements of vector operand 0 to produce a vector
+  // of f64 elements.
+  VEXTEND,
+
+  // Round the f64 elements of vector operand 0 to f32s and store them in the
+  // even elements of the result.
+  VROUND,
+
+  // AND the two vector operands together and set CC based on the result.
+  VTM,
+
+  // String operations that set CC as a side-effect.
+  VFAE_CC,
+  VFAEZ_CC,
+  VFEE_CC,
+  VFEEZ_CC,
+  VFENE_CC,
+  VFENEZ_CC,
+  VISTR_CC,
+  VSTRC_CC,
+  VSTRCZ_CC,
+
   // Wrappers around the inner loop of an 8- or 16-bit ATOMIC_SWAP or
   // ATOMIC_LOAD_<op>.
   //
@@ -219,15 +339,39 @@ class SystemZTargetLowering : public TargetLowering {
                                  const SystemZSubtarget &STI);
 
   // Override TargetLowering.
-  MVT getScalarShiftAmountTy(EVT LHSTy) const override {
+  MVT getScalarShiftAmountTy(const DataLayout &, EVT) const override {
     return MVT::i32;
   }
-  EVT getSetCCResultType(LLVMContext &, EVT) const override;
+  MVT getVectorIdxTy(const DataLayout &DL) const override {
+    // Only the lower 12 bits of an element index are used, so we don't
+    // want to clobber the upper 32 bits of a GPR unnecessarily.
+    return MVT::i32;
+  }
+  TargetLoweringBase::LegalizeTypeAction getPreferredVectorAction(EVT VT)
+    const override {
+    // Widen subvectors to the full width rather than promoting integer
+    // elements.  This is better because:
+    //
+    // (a) it means that we can handle the ABI for passing and returning
+    //     sub-128 vectors without having to handle them as legal types.
+    //
+    // (b) we don't have instructions to extend on load and truncate on store,
+    //     so promoting the integers is less efficient.
+    //
+    // (c) there are no multiplication instructions for the widest integer
+    //     type (v2i64).
+    if (VT.getVectorElementType().getSizeInBits() % 8 == 0)
+      return TypeWidenVector;
+    return TargetLoweringBase::getPreferredVectorAction(VT);
+  }
+  EVT getSetCCResultType(const DataLayout &DL, LLVMContext &,
+                         EVT) const override;
   bool isFMAFasterThanFMulAndFAdd(EVT VT) const override;
   bool isFPImmLegal(const APFloat &Imm, EVT VT) const override;
   bool isLegalICmpImmediate(int64_t Imm) const override;
   bool isLegalAddImmediate(int64_t Imm) const override;
-  bool isLegalAddressingMode(const AddrMode &AM, Type *Ty) const override;
+  bool isLegalAddressingMode(const DataLayout &DL, const AddrMode &AM, Type *Ty,
+                             unsigned AS) const override;
   bool allowsMisalignedMemoryAccesses(EVT VT, unsigned AS,
                                       unsigned Align,
                                       bool *Fast) const override;
@@ -236,10 +380,9 @@ class SystemZTargetLowering : public TargetLowering {
   const char *getTargetNodeName(unsigned Opcode) const override;
   std::pair<unsigned, const TargetRegisterClass *>
   getRegForInlineAsmConstraint(const TargetRegisterInfo *TRI,
-                               const std::string &Constraint,
-                               MVT VT) const override;
+                               StringRef Constraint, MVT VT) const override;
   TargetLowering::ConstraintType
-    getConstraintType(const std::string &Constraint) const override;
+  getConstraintType(StringRef Constraint) const override;
   TargetLowering::ConstraintWeight
     getSingleConstraintMatchWeight(AsmOperandInfo &info,
                                    const char *constraint) const override;
@@ -248,8 +391,7 @@ class SystemZTargetLowering : public TargetLowering {
                                     std::vector<SDValue> &Ops,
                                     SelectionDAG &DAG) const override;
 
-  unsigned getInlineAsmMemConstraint(
-      const std::string &ConstraintCode) const override {
+  unsigned getInlineAsmMemConstraint(StringRef ConstraintCode) const override {
     if (ConstraintCode.size() == 1) {
       switch(ConstraintCode[0]) {
       default:
@@ -328,6 +470,21 @@ class SystemZTargetLowering : public TargetLowering {
   SDValue lowerSTACKRESTORE(SDValue Op, SelectionDAG &DAG) const;
   SDValue lowerPREFETCH(SDValue Op, SelectionDAG &DAG) const;
   SDValue lowerINTRINSIC_W_CHAIN(SDValue Op, SelectionDAG &DAG) const;
+  SDValue lowerINTRINSIC_WO_CHAIN(SDValue Op, SelectionDAG &DAG) const;
+  SDValue lowerBUILD_VECTOR(SDValue Op, SelectionDAG &DAG) const;
+  SDValue lowerVECTOR_SHUFFLE(SDValue Op, SelectionDAG &DAG) const;
+  SDValue lowerSCALAR_TO_VECTOR(SDValue Op, SelectionDAG &DAG) const;
+  SDValue lowerINSERT_VECTOR_ELT(SDValue Op, SelectionDAG &DAG) const;
+  SDValue lowerEXTRACT_VECTOR_ELT(SDValue Op, SelectionDAG &DAG) const;
+  SDValue lowerExtendVectorInreg(SDValue Op, SelectionDAG &DAG,
+				 unsigned UnpackHigh) const;
+  SDValue lowerShift(SDValue Op, SelectionDAG &DAG, unsigned ByScalar) const;
+
+  SDValue combineExtract(SDLoc DL, EVT ElemVT, EVT VecVT, SDValue OrigOp,
+                         unsigned Index, DAGCombinerInfo &DCI,
+                         bool Force) const;
+  SDValue combineTruncateExtract(SDLoc DL, EVT TruncVT, SDValue Op,
+                                 DAGCombinerInfo &DCI) const;
 
   // If the last instruction before MBBI in MBB was some form of COMPARE,
   // try to replace it with a COMPARE AND BRANCH just before MBBI.
diff --git a/lib/Target/SystemZ/SystemZInstrFP.td b/lib/Target/SystemZ/SystemZInstrFP.td
index 4a5582fbf4e..27fbd7df288 100644
--- a/lib/Target/SystemZ/SystemZInstrFP.td
+++ b/lib/Target/SystemZ/SystemZInstrFP.td
@@ -46,9 +46,14 @@ let Defs = [CC], CCValues = 0xF, CompareZeroCCMask = 0xF in {
   defm LTDBR : LoadAndTestRRE<"ltdb", 0xB312, FP64>;
   defm LTXBR : LoadAndTestRRE<"ltxb", 0xB342, FP128>;
 }
-defm : CompareZeroFP<LTEBRCompare, FP32>;
-defm : CompareZeroFP<LTDBRCompare, FP64>;
-defm : CompareZeroFP<LTXBRCompare, FP128>;
+// Note that the comparison against zero operation is not available if we
+// have vector support, since load-and-test instructions will partially
+// clobber the target (vector) register.
+let Predicates = [FeatureNoVector] in {
+  defm : CompareZeroFP<LTEBRCompare, FP32>;
+  defm : CompareZeroFP<LTDBRCompare, FP64>;
+  defm : CompareZeroFP<LTXBRCompare, FP128>;
+}
 
 // Moves between 64-bit integer and floating-point registers.
 def LGDR : UnaryRRE<"lgd", 0xB3CD, bitconvert, GR64, FP64>;
@@ -98,6 +103,9 @@ let canFoldAsLoad = 1, SimpleBDXLoad = 1 in {
   defm LE : UnaryRXPair<"le", 0x78, 0xED64, load, FP32, 4>;
   defm LD : UnaryRXPair<"ld", 0x68, 0xED65, load, FP64, 8>;
 
+  // For z13 we prefer LDE over LE to avoid partial register dependencies.
+  def LDE32 : UnaryRXE<"lde", 0xED24, null_frag, FP32, 4>;
+
   // These instructions are split after register allocation, so we don't
   // want a custom inserter.
   let Has20BitOffset = 1, HasIndex = 1, Is128Bit = 1 in {
@@ -141,7 +149,7 @@ def LDXBRA : UnaryRRF4<"ldxbra", 0xB345, FP128, FP128>,
              Requires<[FeatureFPExtension]>;
 
 def : Pat<(f32 (fround FP128:$src)),
-          (EXTRACT_SUBREG (LEXBR FP128:$src), subreg_hh32)>;
+          (EXTRACT_SUBREG (LEXBR FP128:$src), subreg_hr32)>;
 def : Pat<(f64 (fround FP128:$src)),
           (EXTRACT_SUBREG (LDXBR FP128:$src), subreg_h64)>;
 
@@ -345,13 +353,13 @@ def MDB  : BinaryRXE<"mdb",  0xED1C, fmul, FP64, load, 8>;
 def MDEBR : BinaryRRE<"mdeb", 0xB30C, null_frag, FP64, FP32>;
 def : Pat<(fmul (f64 (fextend FP32:$src1)), (f64 (fextend FP32:$src2))),
           (MDEBR (INSERT_SUBREG (f64 (IMPLICIT_DEF)),
-                                FP32:$src1, subreg_h32), FP32:$src2)>;
+                                FP32:$src1, subreg_r32), FP32:$src2)>;
 
 // f64 multiplication of an FP32 register and an f32 memory.
 def MDEB : BinaryRXE<"mdeb", 0xED0C, null_frag, FP64, load, 4>;
 def : Pat<(fmul (f64 (fextend FP32:$src1)),
                 (f64 (extloadf32 bdxaddr12only:$addr))),
-          (MDEB (INSERT_SUBREG (f64 (IMPLICIT_DEF)), FP32:$src1, subreg_h32),
+          (MDEB (INSERT_SUBREG (f64 (IMPLICIT_DEF)), FP32:$src1, subreg_r32),
                 bdxaddr12only:$addr)>;
 
 // f128 multiplication of two FP64 registers.
diff --git a/lib/Target/SystemZ/SystemZInstrFormats.td b/lib/Target/SystemZ/SystemZInstrFormats.td
index 2d3c9e27e73..71eb9986499 100644
--- a/lib/Target/SystemZ/SystemZInstrFormats.td
+++ b/lib/Target/SystemZ/SystemZInstrFormats.td
@@ -142,10 +142,13 @@ def getThreeOperandOpcode : InstrMapping {
 // Formats are specified using operand field declarations of the form:
 //
 //   bits<4> Rn   : register input or output for operand n
+//   bits<5> Vn   : vector register input or output for operand n
 //   bits<m> In   : immediate value of width m for operand n
 //   bits<4> BDn  : address operand n, which has a base and a displacement
 //   bits<m> XBDn : address operand n, which has an index, a base and a
 //                  displacement
+//   bits<m> VBDn : address operand n, which has a vector index, a base and a
+//                  displacement
 //   bits<4> Xn   : index register for address operand n
 //   bits<4> Mn   : mode value for operand n
 //
@@ -339,11 +342,13 @@ class InstRXE<bits<16> op, dag outs, dag ins, string asmstr, list<dag> pattern>
 
   bits<4> R1;
   bits<20> XBD2;
+  bits<4> M3;
 
   let Inst{47-40} = op{15-8};
   let Inst{39-36} = R1;
   let Inst{35-16} = XBD2;
-  let Inst{15-8}  = 0;
+  let Inst{15-12} = M3;
+  let Inst{11-8}  = 0;
   let Inst{7-0}   = op{7-0};
 
   let HasIndex = 1;
@@ -484,6 +489,382 @@ class InstS<bits<16> op, dag outs, dag ins, string asmstr, list<dag> pattern>
   let Inst{15-0}  = BD2;
 }
 
+class InstVRIa<bits<16> op, dag outs, dag ins, string asmstr, list<dag> pattern>
+  : InstSystemZ<6, outs, ins, asmstr, pattern> {
+  field bits<48> Inst;
+  field bits<48> SoftFail = 0;
+
+  bits<5> V1;
+  bits<16> I2;
+  bits<4> M3;
+
+  let Inst{47-40} = op{15-8};
+  let Inst{39-36} = V1{3-0};
+  let Inst{35-32} = 0;
+  let Inst{31-16} = I2;
+  let Inst{15-12} = M3;
+  let Inst{11}    = V1{4};
+  let Inst{10-8}  = 0;
+  let Inst{7-0}   = op{7-0};
+}
+
+class InstVRIb<bits<16> op, dag outs, dag ins, string asmstr, list<dag> pattern>
+  : InstSystemZ<6, outs, ins, asmstr, pattern> {
+  field bits<48> Inst;
+  field bits<48> SoftFail = 0;
+
+  bits<5> V1;
+  bits<8> I2;
+  bits<8> I3;
+  bits<4> M4;
+
+  let Inst{47-40} = op{15-8};
+  let Inst{39-36} = V1{3-0};
+  let Inst{35-32} = 0;
+  let Inst{31-24} = I2;
+  let Inst{23-16} = I3;
+  let Inst{15-12} = M4;
+  let Inst{11}    = V1{4};
+  let Inst{10-8}  = 0;
+  let Inst{7-0}   = op{7-0};
+}
+
+class InstVRIc<bits<16> op, dag outs, dag ins, string asmstr, list<dag> pattern>
+  : InstSystemZ<6, outs, ins, asmstr, pattern> {
+  field bits<48> Inst;
+  field bits<48> SoftFail = 0;
+
+  bits<5> V1;
+  bits<5> V3;
+  bits<16> I2;
+  bits<4> M4;
+
+  let Inst{47-40} = op{15-8};
+  let Inst{39-36} = V1{3-0};
+  let Inst{35-32} = V3{3-0};
+  let Inst{31-16} = I2;
+  let Inst{15-12} = M4;
+  let Inst{11}    = V1{4};
+  let Inst{10}    = V3{4};
+  let Inst{9-8}   = 0;
+  let Inst{7-0}   = op{7-0};
+}
+
+class InstVRId<bits<16> op, dag outs, dag ins, string asmstr, list<dag> pattern>
+  : InstSystemZ<6, outs, ins, asmstr, pattern> {
+  field bits<48> Inst;
+  field bits<48> SoftFail = 0;
+
+  bits<5> V1;
+  bits<5> V2;
+  bits<5> V3;
+  bits<8> I4;
+  bits<4> M5;
+
+  let Inst{47-40} = op{15-8};
+  let Inst{39-36} = V1{3-0};
+  let Inst{35-32} = V2{3-0};
+  let Inst{31-28} = V3{3-0};
+  let Inst{27-24} = 0;
+  let Inst{23-16} = I4;
+  let Inst{15-12} = M5;
+  let Inst{11}    = V1{4};
+  let Inst{10}    = V2{4};
+  let Inst{9}     = V3{4};
+  let Inst{8}     = 0;
+  let Inst{7-0}   = op{7-0};
+}
+
+class InstVRIe<bits<16> op, dag outs, dag ins, string asmstr, list<dag> pattern>
+  : InstSystemZ<6, outs, ins, asmstr, pattern> {
+  field bits<48> Inst;
+  field bits<48> SoftFail = 0;
+
+  bits<5> V1;
+  bits<5> V2;
+  bits<12> I3;
+  bits<4> M4;
+  bits<4> M5;
+
+  let Inst{47-40} = op{15-8};
+  let Inst{39-36} = V1{3-0};
+  let Inst{35-32} = V2{3-0};
+  let Inst{31-20} = I3;
+  let Inst{19-16} = M5;
+  let Inst{15-12} = M4;
+  let Inst{11}    = V1{4};
+  let Inst{10}    = V2{4};
+  let Inst{9-8}   = 0;
+  let Inst{7-0}   = op{7-0};
+}
+
+// Depending on the instruction mnemonic, certain bits may be or-ed into
+// the M4 value provided as explicit operand.  These are passed as m4or.
+class InstVRRa<bits<16> op, dag outs, dag ins, string asmstr, list<dag> pattern,
+               bits<4> m4or = 0>
+  : InstSystemZ<6, outs, ins, asmstr, pattern> {
+  field bits<48> Inst;
+  field bits<48> SoftFail = 0;
+
+  bits<5> V1;
+  bits<5> V2;
+  bits<4> M3;
+  bits<4> M4;
+  bits<4> M5;
+
+  let Inst{47-40} = op{15-8};
+  let Inst{39-36} = V1{3-0};
+  let Inst{35-32} = V2{3-0};
+  let Inst{31-24} = 0;
+  let Inst{23-20} = M5;
+  let Inst{19}    = !if (!eq (m4or{3}, 1), 1, M4{3});
+  let Inst{18}    = !if (!eq (m4or{2}, 1), 1, M4{2});
+  let Inst{17}    = !if (!eq (m4or{1}, 1), 1, M4{1});
+  let Inst{16}    = !if (!eq (m4or{0}, 1), 1, M4{0});
+  let Inst{15-12} = M3;
+  let Inst{11}    = V1{4};
+  let Inst{10}    = V2{4};
+  let Inst{9-8}   = 0;
+  let Inst{7-0}   = op{7-0};
+}
+
+// Depending on the instruction mnemonic, certain bits may be or-ed into
+// the M5 value provided as explicit operand.  These are passed as m5or.
+class InstVRRb<bits<16> op, dag outs, dag ins, string asmstr, list<dag> pattern,
+               bits<4> m5or = 0>
+  : InstSystemZ<6, outs, ins, asmstr, pattern> {
+  field bits<48> Inst;
+  field bits<48> SoftFail = 0;
+
+  bits<5> V1;
+  bits<5> V2;
+  bits<5> V3;
+  bits<4> M4;
+  bits<4> M5;
+
+  let Inst{47-40} = op{15-8};
+  let Inst{39-36} = V1{3-0};
+  let Inst{35-32} = V2{3-0};
+  let Inst{31-28} = V3{3-0};
+  let Inst{27-24} = 0;
+  let Inst{23}    = !if (!eq (m5or{3}, 1), 1, M5{3});
+  let Inst{22}    = !if (!eq (m5or{2}, 1), 1, M5{2});
+  let Inst{21}    = !if (!eq (m5or{1}, 1), 1, M5{1});
+  let Inst{20}    = !if (!eq (m5or{0}, 1), 1, M5{0});
+  let Inst{19-16} = 0;
+  let Inst{15-12} = M4;
+  let Inst{11}    = V1{4};
+  let Inst{10}    = V2{4};
+  let Inst{9}     = V3{4};
+  let Inst{8}     = 0;
+  let Inst{7-0}   = op{7-0};
+}
+
+class InstVRRc<bits<16> op, dag outs, dag ins, string asmstr, list<dag> pattern>
+  : InstSystemZ<6, outs, ins, asmstr, pattern> {
+  field bits<48> Inst;
+  field bits<48> SoftFail = 0;
+
+  bits<5> V1;
+  bits<5> V2;
+  bits<5> V3;
+  bits<4> M4;
+  bits<4> M5;
+  bits<4> M6;
+
+  let Inst{47-40} = op{15-8};
+  let Inst{39-36} = V1{3-0};
+  let Inst{35-32} = V2{3-0};
+  let Inst{31-28} = V3{3-0};
+  let Inst{27-24} = 0;
+  let Inst{23-20} = M6;
+  let Inst{19-16} = M5;
+  let Inst{15-12} = M4;
+  let Inst{11}    = V1{4};
+  let Inst{10}    = V2{4};
+  let Inst{9}     = V3{4};
+  let Inst{8}     = 0;
+  let Inst{7-0}   = op{7-0};
+}
+
+// Depending on the instruction mnemonic, certain bits may be or-ed into
+// the M6 value provided as explicit operand.  These are passed as m6or.
+class InstVRRd<bits<16> op, dag outs, dag ins, string asmstr, list<dag> pattern,
+               bits<4> m6or = 0>
+  : InstSystemZ<6, outs, ins, asmstr, pattern> {
+  field bits<48> Inst;
+  field bits<48> SoftFail = 0;
+
+  bits<5> V1;
+  bits<5> V2;
+  bits<5> V3;
+  bits<5> V4;
+  bits<4> M5;
+  bits<4> M6;
+
+  let Inst{47-40} = op{15-8};
+  let Inst{39-36} = V1{3-0};
+  let Inst{35-32} = V2{3-0};
+  let Inst{31-28} = V3{3-0};
+  let Inst{27-24} = M5;
+  let Inst{23}    = !if (!eq (m6or{3}, 1), 1, M6{3});
+  let Inst{22}    = !if (!eq (m6or{2}, 1), 1, M6{2});
+  let Inst{21}    = !if (!eq (m6or{1}, 1), 1, M6{1});
+  let Inst{20}    = !if (!eq (m6or{0}, 1), 1, M6{0});
+  let Inst{19-16} = 0;
+  let Inst{15-12} = V4{3-0};
+  let Inst{11}    = V1{4};
+  let Inst{10}    = V2{4};
+  let Inst{9}     = V3{4};
+  let Inst{8}     = V4{4};
+  let Inst{7-0}   = op{7-0};
+}
+
+class InstVRRe<bits<16> op, dag outs, dag ins, string asmstr, list<dag> pattern>
+  : InstSystemZ<6, outs, ins, asmstr, pattern> {
+  field bits<48> Inst;
+  field bits<48> SoftFail = 0;
+
+  bits<5> V1;
+  bits<5> V2;
+  bits<5> V3;
+  bits<5> V4;
+  bits<4> M5;
+  bits<4> M6;
+
+  let Inst{47-40} = op{15-8};
+  let Inst{39-36} = V1{3-0};
+  let Inst{35-32} = V2{3-0};
+  let Inst{31-28} = V3{3-0};
+  let Inst{27-24} = M6;
+  let Inst{23-20} = 0;
+  let Inst{19-16} = M5;
+  let Inst{15-12} = V4{3-0};
+  let Inst{11}    = V1{4};
+  let Inst{10}    = V2{4};
+  let Inst{9}     = V3{4};
+  let Inst{8}     = V4{4};
+  let Inst{7-0}   = op{7-0};
+}
+
+class InstVRRf<bits<16> op, dag outs, dag ins, string asmstr, list<dag> pattern>
+  : InstSystemZ<6, outs, ins, asmstr, pattern> {
+  field bits<48> Inst;
+  field bits<48> SoftFail = 0;
+
+  bits<5> V1;
+  bits<4> R2;
+  bits<4> R3;
+
+  let Inst{47-40} = op{15-8};
+  let Inst{39-36} = V1{3-0};
+  let Inst{35-32} = R2;
+  let Inst{31-28} = R3;
+  let Inst{27-12} = 0;
+  let Inst{11}    = V1{4};
+  let Inst{10-8}  = 0;
+  let Inst{7-0}   = op{7-0};
+}
+
+class InstVRSa<bits<16> op, dag outs, dag ins, string asmstr, list<dag> pattern>
+  : InstSystemZ<6, outs, ins, asmstr, pattern> {
+  field bits<48> Inst;
+  field bits<48> SoftFail = 0;
+
+  bits<5> V1;
+  bits<16> BD2;
+  bits<5> V3;
+  bits<4> M4;
+
+  let Inst{47-40} = op{15-8};
+  let Inst{39-36} = V1{3-0};
+  let Inst{35-32} = V3{3-0};
+  let Inst{31-16} = BD2;
+  let Inst{15-12} = M4;
+  let Inst{11}    = V1{4};
+  let Inst{10}    = V3{4};
+  let Inst{9-8}   = 0;
+  let Inst{7-0}   = op{7-0};
+}
+
+class InstVRSb<bits<16> op, dag outs, dag ins, string asmstr, list<dag> pattern>
+  : InstSystemZ<6, outs, ins, asmstr, pattern> {
+  field bits<48> Inst;
+  field bits<48> SoftFail = 0;
+
+  bits<5> V1;
+  bits<16> BD2;
+  bits<4> R3;
+  bits<4> M4;
+
+  let Inst{47-40} = op{15-8};
+  let Inst{39-36} = V1{3-0};
+  let Inst{35-32} = R3;
+  let Inst{31-16} = BD2;
+  let Inst{15-12} = M4;
+  let Inst{11}    = V1{4};
+  let Inst{10-8}  = 0;
+  let Inst{7-0}   = op{7-0};
+}
+
+class InstVRSc<bits<16> op, dag outs, dag ins, string asmstr, list<dag> pattern>
+  : InstSystemZ<6, outs, ins, asmstr, pattern> {
+  field bits<48> Inst;
+  field bits<48> SoftFail = 0;
+
+  bits<4> R1;
+  bits<16> BD2;
+  bits<5> V3;
+  bits<4> M4;
+
+  let Inst{47-40} = op{15-8};
+  let Inst{39-36} = R1;
+  let Inst{35-32} = V3{3-0};
+  let Inst{31-16} = BD2;
+  let Inst{15-12} = M4;
+  let Inst{11}    = 0;
+  let Inst{10}    = V3{4};
+  let Inst{9-8}   = 0;
+  let Inst{7-0}   = op{7-0};
+}
+
+class InstVRV<bits<16> op, dag outs, dag ins, string asmstr, list<dag> pattern>
+  : InstSystemZ<6, outs, ins, asmstr, pattern> {
+  field bits<48> Inst;
+  field bits<48> SoftFail = 0;
+
+  bits<5> V1;
+  bits<21> VBD2;
+  bits<4> M3;
+
+  let Inst{47-40} = op{15-8};
+  let Inst{39-36} = V1{3-0};
+  let Inst{35-16} = VBD2{19-0};
+  let Inst{15-12} = M3;
+  let Inst{11}    = V1{4};
+  let Inst{10}    = VBD2{20};
+  let Inst{9-8}   = 0;
+  let Inst{7-0}   = op{7-0};
+}
+
+class InstVRX<bits<16> op, dag outs, dag ins, string asmstr, list<dag> pattern>
+  : InstSystemZ<6, outs, ins, asmstr, pattern> {
+  field bits<48> Inst;
+  field bits<48> SoftFail = 0;
+
+  bits<5> V1;
+  bits<20> XBD2;
+  bits<4> M3;
+
+  let Inst{47-40} = op{15-8};
+  let Inst{39-36} = V1{3-0};
+  let Inst{35-16} = XBD2;
+  let Inst{15-12} = M3;
+  let Inst{11}    = V1{4};
+  let Inst{10-8}  = 0;
+  let Inst{7-0}   = op{7-0};
+}
+
 //===----------------------------------------------------------------------===//
 // Instruction definitions with semantics
 //===----------------------------------------------------------------------===//
@@ -503,12 +884,6 @@ class InstS<bits<16> op, dag outs, dag ins, string asmstr, list<dag> pattern>
 //     form of the source register in the destination register and
 //     branches on the result.
 //
-//   Store:
-//     One register or immediate input operand and one address input operand.
-//     The instruction stores the first operand to the address.
-//
-//     This category is used for both pure and truncating stores.
-//
 //   LoadMultiple:
 //     One address input operand and two explicit output operands.
 //     The instruction loads a range of registers from the address,
@@ -521,18 +896,35 @@ class InstS<bits<16> op, dag outs, dag ins, string asmstr, list<dag> pattern>
 //     with the explicit operands giving the first and last register
 //     to store.  Other stored registers are added as implicit uses.
 //
+//   StoreLength:
+//     One value operand, one length operand and one address operand.
+//     The instruction stores the value operand to the address but
+//     doesn't write more than the number of bytes specified by the
+//     length operand.
+//
 //   Unary:
 //     One register output operand and one input operand.
 //
+//   Store:
+//     One address operand and one other input operand.  The instruction
+//     stores to the address.
+//
 //   Binary:
 //     One register output operand and two input operands.
 //
+//   StoreBinary:
+//     One address operand and two other input operands.  The instruction
+//     stores to the address.
+//
 //   Compare:
 //     Two input operands and an implicit CC output operand.
 //
 //   Ternary:
 //     One register output operand and three input operands.
 //
+//   Quaternary:
+//     One register output operand and four input operands.
+//
 //   LoadAndOp:
 //     One output operand and two input operands, one of which is an address.
 //     The instruction both reads from and writes to the address.
@@ -567,6 +959,12 @@ class InherentRRE<string mnemonic, bits<16> opcode, RegisterOperand cls,
   let R2 = 0;
 }
 
+class InherentVRIa<string mnemonic, bits<16> opcode, bits<16> value>
+  : InstVRIa<opcode, (outs VR128:$V1), (ins), mnemonic#"\t$V1", []> {
+  let I2 = value;
+  let M3 = 0;
+}
+
 class BranchUnaryRI<string mnemonic, bits<12> opcode, RegisterOperand cls>
   : InstRI<opcode, (outs cls:$R1), (ins cls:$R1src, brtarget16:$I2),
            mnemonic##"\t$R1, $I2", []> {
@@ -582,6 +980,13 @@ class LoadMultipleRSY<string mnemonic, bits<16> opcode, RegisterOperand cls>
   let mayLoad = 1;
 }
 
+class LoadMultipleVRSa<string mnemonic, bits<16> opcode>
+  : InstVRSa<opcode, (outs VR128:$V1, VR128:$V3), (ins bdaddr12only:$BD2),
+             mnemonic#"\t$V1, $V3, $BD2", []> {
+  let M4 = 0;
+  let mayLoad = 1;
+}
+
 class StoreRILPC<string mnemonic, bits<12> opcode, SDPatternOperator operator,
                  RegisterOperand cls>
   : InstRIL<opcode, (outs), (ins cls:$R1, pcrel32:$I2),
@@ -630,12 +1035,39 @@ multiclass StoreRXPair<string mnemonic, bits<8> rxOpcode, bits<16> rxyOpcode,
   }
 }
 
+class StoreVRX<string mnemonic, bits<16> opcode, SDPatternOperator operator,
+               TypedReg tr, bits<5> bytes, bits<4> type = 0>
+  : InstVRX<opcode, (outs), (ins tr.op:$V1, bdxaddr12only:$XBD2),
+            mnemonic#"\t$V1, $XBD2",
+            [(set tr.op:$V1, (tr.vt (operator bdxaddr12only:$XBD2)))]> {
+  let M3 = type;
+  let mayStore = 1;
+  let AccessBytes = bytes;
+}
+
+class StoreLengthVRSb<string mnemonic, bits<16> opcode,
+                      SDPatternOperator operator, bits<5> bytes>
+  : InstVRSb<opcode, (outs), (ins VR128:$V1, GR32:$R3, bdaddr12only:$BD2),
+             mnemonic#"\t$V1, $R3, $BD2",
+             [(operator VR128:$V1, GR32:$R3, bdaddr12only:$BD2)]> {
+  let M4 = 0;
+  let mayStore = 1;
+  let AccessBytes = bytes;
+}
+
 class StoreMultipleRSY<string mnemonic, bits<16> opcode, RegisterOperand cls>
   : InstRSY<opcode, (outs), (ins cls:$R1, cls:$R3, bdaddr20only:$BD2),
             mnemonic#"\t$R1, $R3, $BD2", []> {
   let mayStore = 1;
 }
 
+class StoreMultipleVRSa<string mnemonic, bits<16> opcode>
+  : InstVRSa<opcode, (outs), (ins VR128:$V1, VR128:$V3, bdaddr12only:$BD2),
+             mnemonic#"\t$V1, $V3, $BD2", []> {
+  let M4 = 0;
+  let mayStore = 1;
+}
+
 // StoreSI* instructions are used to store an integer to memory, but the
 // addresses are more restricted than for normal stores.  If we are in the
 // situation of having to force either the address into a register or the
@@ -868,6 +1300,7 @@ class UnaryRXE<string mnemonic, bits<16> opcode, SDPatternOperator operator,
   let OpType = "mem";
   let mayLoad = 1;
   let AccessBytes = bytes;
+  let M3 = 0;
 }
 
 class UnaryRXY<string mnemonic, bits<16> opcode, SDPatternOperator operator,
@@ -894,6 +1327,46 @@ multiclass UnaryRXPair<string mnemonic, bits<8> rxOpcode, bits<16> rxyOpcode,
   }
 }
 
+class UnaryVRIa<string mnemonic, bits<16> opcode, SDPatternOperator operator,
+                TypedReg tr, Immediate imm, bits<4> type = 0>
+  : InstVRIa<opcode, (outs tr.op:$V1), (ins imm:$I2),
+             mnemonic#"\t$V1, $I2",
+             [(set tr.op:$V1, (tr.vt (operator imm:$I2)))]> {
+  let M3 = type;
+}
+
+class UnaryVRRa<string mnemonic, bits<16> opcode, SDPatternOperator operator,
+                TypedReg tr1, TypedReg tr2, bits<4> type = 0, bits<4> m4 = 0,
+                bits<4> m5 = 0>
+  : InstVRRa<opcode, (outs tr1.op:$V1), (ins tr2.op:$V2),
+             mnemonic#"\t$V1, $V2",
+             [(set tr1.op:$V1, (tr1.vt (operator (tr2.vt tr2.op:$V2))))]> {
+  let M3 = type;
+  let M4 = m4;
+  let M5 = m5;
+}
+
+multiclass UnaryVRRaSPair<string mnemonic, bits<16> opcode,
+                          SDPatternOperator operator,
+                          SDPatternOperator operator_cc, TypedReg tr1,
+                          TypedReg tr2, bits<4> type, bits<4> modifier = 0,
+                          bits<4> modifier_cc = 1> {
+  def "" : UnaryVRRa<mnemonic, opcode, operator, tr1, tr2, type, 0, modifier>;
+  let Defs = [CC] in
+    def S : UnaryVRRa<mnemonic##"s", opcode, operator_cc, tr1, tr2, type, 0,
+                      modifier_cc>;
+}
+
+class UnaryVRX<string mnemonic, bits<16> opcode, SDPatternOperator operator,
+               TypedReg tr, bits<5> bytes, bits<4> type = 0>
+  : InstVRX<opcode, (outs tr.op:$V1), (ins bdxaddr12only:$XBD2),
+            mnemonic#"\t$V1, $XBD2",
+            [(set tr.op:$V1, (tr.vt (operator bdxaddr12only:$XBD2)))]> {
+  let M3 = type;
+  let mayLoad = 1;
+  let AccessBytes = bytes;
+}
+
 class BinaryRR<string mnemonic, bits<8> opcode, SDPatternOperator operator,
                RegisterOperand cls1, RegisterOperand cls2>
   : InstRR<opcode, (outs cls1:$R1), (ins cls1:$R1src, cls2:$R2),
@@ -1047,6 +1520,7 @@ class BinaryRXE<string mnemonic, bits<16> opcode, SDPatternOperator operator,
   let DisableEncoding = "$R1src";
   let mayLoad = 1;
   let AccessBytes = bytes;
+  let M3 = 0;
 }
 
 class BinaryRXY<string mnemonic, bits<16> opcode, SDPatternOperator operator,
@@ -1105,6 +1579,148 @@ multiclass BinarySIPair<string mnemonic, bits<8> siOpcode,
   }
 }
 
+class BinaryVRIb<string mnemonic, bits<16> opcode, SDPatternOperator operator,
+                 TypedReg tr, bits<4> type>
+  : InstVRIb<opcode, (outs tr.op:$V1), (ins imm32zx8:$I2, imm32zx8:$I3),
+             mnemonic#"\t$V1, $I2, $I3",
+             [(set tr.op:$V1, (tr.vt (operator imm32zx8:$I2, imm32zx8:$I3)))]> {
+  let M4 = type;
+}
+
+class BinaryVRIc<string mnemonic, bits<16> opcode, SDPatternOperator operator,
+                 TypedReg tr1, TypedReg tr2, bits<4> type>
+  : InstVRIc<opcode, (outs tr1.op:$V1), (ins tr2.op:$V3, imm32zx16:$I2),
+             mnemonic#"\t$V1, $V3, $I2",
+             [(set tr1.op:$V1, (tr1.vt (operator (tr2.vt tr2.op:$V3),
+                                                 imm32zx16:$I2)))]> {
+  let M4 = type;
+}
+
+class BinaryVRIe<string mnemonic, bits<16> opcode, SDPatternOperator operator,
+                 TypedReg tr1, TypedReg tr2, bits<4> type, bits<4> m5>
+  : InstVRIe<opcode, (outs tr1.op:$V1), (ins tr2.op:$V2, imm32zx12:$I3),
+             mnemonic#"\t$V1, $V2, $I3",
+             [(set tr1.op:$V1, (tr1.vt (operator (tr2.vt tr2.op:$V2),
+                                                 imm32zx12:$I3)))]> {
+  let M4 = type;
+  let M5 = m5;
+}
+
+class BinaryVRRa<string mnemonic, bits<16> opcode>
+  : InstVRRa<opcode, (outs VR128:$V1), (ins VR128:$V2, imm32zx4:$M3),
+             mnemonic#"\t$V1, $V2, $M3", []> {
+  let M4 = 0;
+  let M5 = 0;
+}
+
+class BinaryVRRb<string mnemonic, bits<16> opcode, SDPatternOperator operator,
+                 TypedReg tr1, TypedReg tr2, bits<4> type = 0,
+                 bits<4> modifier = 0>
+  : InstVRRb<opcode, (outs tr1.op:$V1), (ins tr2.op:$V2, tr2.op:$V3),
+             mnemonic#"\t$V1, $V2, $V3",
+             [(set tr1.op:$V1, (tr1.vt (operator (tr2.vt tr2.op:$V2),
+                                                 (tr2.vt tr2.op:$V3))))]> {
+  let M4 = type;
+  let M5 = modifier;
+}
+
+// Declare a pair of instructions, one which sets CC and one which doesn't.
+// The CC-setting form ends with "S" and sets the low bit of M5.
+multiclass BinaryVRRbSPair<string mnemonic, bits<16> opcode,
+                           SDPatternOperator operator,
+                           SDPatternOperator operator_cc, TypedReg tr1,
+                           TypedReg tr2, bits<4> type,
+                           bits<4> modifier = 0, bits<4> modifier_cc = 1> {
+  def "" : BinaryVRRb<mnemonic, opcode, operator, tr1, tr2, type, modifier>;
+  let Defs = [CC] in
+    def S : BinaryVRRb<mnemonic##"s", opcode, operator_cc, tr1, tr2, type,
+                       modifier_cc>;
+}
+
+class BinaryVRRc<string mnemonic, bits<16> opcode, SDPatternOperator operator,
+                 TypedReg tr1, TypedReg tr2, bits<4> type = 0, bits<4> m5 = 0,
+                 bits<4> m6 = 0>
+  : InstVRRc<opcode, (outs tr1.op:$V1), (ins tr2.op:$V2, tr2.op:$V3),
+             mnemonic#"\t$V1, $V2, $V3",
+             [(set tr1.op:$V1, (tr1.vt (operator (tr2.vt tr2.op:$V2),
+                                                 (tr2.vt tr2.op:$V3))))]> {
+  let M4 = type;
+  let M5 = m5;
+  let M6 = m6;
+}
+
+multiclass BinaryVRRcSPair<string mnemonic, bits<16> opcode,
+                           SDPatternOperator operator,
+                           SDPatternOperator operator_cc, TypedReg tr1,
+                           TypedReg tr2, bits<4> type, bits<4> m5,
+                           bits<4> modifier = 0, bits<4> modifier_cc = 1> {
+  def "" : BinaryVRRc<mnemonic, opcode, operator, tr1, tr2, type, m5, modifier>;
+  let Defs = [CC] in
+    def S : BinaryVRRc<mnemonic##"s", opcode, operator_cc, tr1, tr2, type,
+                       m5, modifier_cc>;
+}
+
+class BinaryVRRf<string mnemonic, bits<16> opcode, SDPatternOperator operator,
+                 TypedReg tr>
+  : InstVRRf<opcode, (outs tr.op:$V1), (ins GR64:$R2, GR64:$R3),
+             mnemonic#"\t$V1, $R2, $R3",
+             [(set tr.op:$V1, (tr.vt (operator GR64:$R2, GR64:$R3)))]>;
+
+class BinaryVRSa<string mnemonic, bits<16> opcode, SDPatternOperator operator,
+                 TypedReg tr1, TypedReg tr2, bits<4> type>
+  : InstVRSa<opcode, (outs tr1.op:$V1), (ins tr2.op:$V3, shift12only:$BD2),
+             mnemonic#"\t$V1, $V3, $BD2",
+             [(set tr1.op:$V1, (tr1.vt (operator (tr2.vt tr2.op:$V3),
+                                                 shift12only:$BD2)))]> {
+  let M4 = type;
+}
+
+class BinaryVRSb<string mnemonic, bits<16> opcode, SDPatternOperator operator,
+                 bits<5> bytes>
+  : InstVRSb<opcode, (outs VR128:$V1), (ins GR32:$R3, bdaddr12only:$BD2),
+             mnemonic#"\t$V1, $R3, $BD2",
+             [(set VR128:$V1, (operator GR32:$R3, bdaddr12only:$BD2))]> {
+  let M4 = 0;
+  let mayLoad = 1;
+  let AccessBytes = bytes;
+}
+
+class BinaryVRSc<string mnemonic, bits<16> opcode, SDPatternOperator operator,
+                 TypedReg tr, bits<4> type>
+  : InstVRSc<opcode, (outs GR64:$R1), (ins tr.op:$V3, shift12only:$BD2),
+           mnemonic#"\t$R1, $V3, $BD2",
+           [(set GR64:$R1, (operator (tr.vt tr.op:$V3), shift12only:$BD2))]> {
+  let M4 = type;
+}
+
+class BinaryVRX<string mnemonic, bits<16> opcode, SDPatternOperator operator,
+                TypedReg tr, bits<5> bytes>
+  : InstVRX<opcode, (outs VR128:$V1), (ins bdxaddr12only:$XBD2, imm32zx4:$M3),
+            mnemonic#"\t$V1, $XBD2, $M3",
+            [(set tr.op:$V1, (tr.vt (operator bdxaddr12only:$XBD2,
+                                              imm32zx4:$M3)))]> {
+  let mayLoad = 1;
+  let AccessBytes = bytes;
+}
+
+class StoreBinaryVRV<string mnemonic, bits<16> opcode, bits<5> bytes,
+                     Immediate index>
+  : InstVRV<opcode, (outs), (ins VR128:$V1, bdvaddr12only:$VBD2, index:$M3),
+            mnemonic#"\t$V1, $VBD2, $M3", []> {
+  let mayStore = 1;
+  let AccessBytes = bytes;
+}
+
+class StoreBinaryVRX<string mnemonic, bits<16> opcode,
+                     SDPatternOperator operator, TypedReg tr, bits<5> bytes,
+                     Immediate index>
+  : InstVRX<opcode, (outs), (ins tr.op:$V1, bdxaddr12only:$XBD2, index:$M3),
+            mnemonic#"\t$V1, $XBD2, $M3",
+            [(operator (tr.vt tr.op:$V1), bdxaddr12only:$XBD2, index:$M3)]> {
+  let mayStore = 1;
+  let AccessBytes = bytes;
+}
+
 class CompareRR<string mnemonic, bits<8> opcode, SDPatternOperator operator,
                 RegisterOperand cls1, RegisterOperand cls2>
   : InstRR<opcode, (outs), (ins cls1:$R1, cls2:$R2),
@@ -1177,6 +1793,7 @@ class CompareRXE<string mnemonic, bits<16> opcode, SDPatternOperator operator,
   let isCompare = 1;
   let mayLoad = 1;
   let AccessBytes = bytes;
+  let M3 = 0;
 }
 
 class CompareRXY<string mnemonic, bits<16> opcode, SDPatternOperator operator,
@@ -1246,6 +1863,17 @@ multiclass CompareSIPair<string mnemonic, bits<8> siOpcode, bits<16> siyOpcode,
   }
 }
 
+class CompareVRRa<string mnemonic, bits<16> opcode, SDPatternOperator operator,
+                  TypedReg tr, bits<4> type>
+  : InstVRRa<opcode, (outs), (ins tr.op:$V1, tr.op:$V2),
+             mnemonic#"\t$V1, $V2",
+             [(operator (tr.vt tr.op:$V1), (tr.vt tr.op:$V2))]> {
+  let isCompare = 1;
+  let M3 = type;
+  let M4 = 0;
+  let M5 = 0;
+}
+
 class TernaryRRD<string mnemonic, bits<16> opcode,
                  SDPatternOperator operator, RegisterOperand cls>
   : InstRRD<opcode, (outs cls:$R1), (ins cls:$R1src, cls:$R3, cls:$R2),
@@ -1272,6 +1900,188 @@ class TernaryRXF<string mnemonic, bits<16> opcode, SDPatternOperator operator,
   let AccessBytes = bytes;
 }
 
+class TernaryVRIa<string mnemonic, bits<16> opcode, SDPatternOperator operator,
+                  TypedReg tr1, TypedReg tr2, Immediate imm, Immediate index>
+  : InstVRIa<opcode, (outs tr1.op:$V1), (ins tr2.op:$V1src, imm:$I2, index:$M3),
+             mnemonic#"\t$V1, $I2, $M3",
+             [(set tr1.op:$V1, (tr1.vt (operator (tr2.vt tr2.op:$V1src),
+                                                 imm:$I2, index:$M3)))]> {
+  let Constraints = "$V1 = $V1src";
+  let DisableEncoding = "$V1src";
+}
+
+class TernaryVRId<string mnemonic, bits<16> opcode, SDPatternOperator operator,
+                  TypedReg tr1, TypedReg tr2, bits<4> type>
+  : InstVRId<opcode, (outs tr1.op:$V1),
+             (ins tr2.op:$V2, tr2.op:$V3, imm32zx8:$I4),
+             mnemonic#"\t$V1, $V2, $V3, $I4",
+             [(set tr1.op:$V1, (tr1.vt (operator (tr2.vt tr2.op:$V2),
+                                                 (tr2.vt tr2.op:$V3),
+                                                 imm32zx8:$I4)))]> {
+  let M5 = type;
+}
+
+class TernaryVRRa<string mnemonic, bits<16> opcode, SDPatternOperator operator,
+                  TypedReg tr1, TypedReg tr2, bits<4> type, bits<4> m4or>
+  : InstVRRa<opcode, (outs tr1.op:$V1),
+             (ins tr2.op:$V2, imm32zx4:$M4, imm32zx4:$M5),
+             mnemonic#"\t$V1, $V2, $M4, $M5",
+             [(set tr1.op:$V1, (tr1.vt (operator (tr2.vt tr2.op:$V2),
+                                                 imm32zx4:$M4,
+                                                 imm32zx4:$M5)))],
+             m4or> {
+  let M3 = type;
+}
+
+class TernaryVRRb<string mnemonic, bits<16> opcode, SDPatternOperator operator,
+                  TypedReg tr1, TypedReg tr2, bits<4> type,
+                  SDPatternOperator m5mask, bits<4> m5or>
+  : InstVRRb<opcode, (outs tr1.op:$V1),
+             (ins tr2.op:$V2, tr2.op:$V3, m5mask:$M5),
+             mnemonic#"\t$V1, $V2, $V3, $M5",
+             [(set tr1.op:$V1, (tr1.vt (operator (tr2.vt tr2.op:$V2),
+                                                 (tr2.vt tr2.op:$V3),
+                                                 m5mask:$M5)))],
+             m5or> {
+  let M4 = type;
+}
+
+multiclass TernaryVRRbSPair<string mnemonic, bits<16> opcode,
+                            SDPatternOperator operator,
+                            SDPatternOperator operator_cc, TypedReg tr1,
+                            TypedReg tr2, bits<4> type, bits<4> m5or> {
+  def "" : TernaryVRRb<mnemonic, opcode, operator, tr1, tr2, type,
+                       imm32zx4even, !and (m5or, 14)>;
+  def : InstAlias<mnemonic#"\t$V1, $V2, $V3",
+                  (!cast<Instruction>(NAME) tr1.op:$V1, tr2.op:$V2,
+                                            tr2.op:$V3, 0)>;
+  let Defs = [CC] in
+    def S : TernaryVRRb<mnemonic##"s", opcode, operator_cc, tr1, tr2, type,
+                        imm32zx4even, !add(!and (m5or, 14), 1)>;
+  def : InstAlias<mnemonic#"s\t$V1, $V2, $V3",
+                  (!cast<Instruction>(NAME#"S") tr1.op:$V1, tr2.op:$V2,
+                                                tr2.op:$V3, 0)>;
+}
+
+class TernaryVRRc<string mnemonic, bits<16> opcode, SDPatternOperator operator,
+                  TypedReg tr1, TypedReg tr2>
+  : InstVRRc<opcode, (outs tr1.op:$V1),
+             (ins tr2.op:$V2, tr2.op:$V3, imm32zx4:$M4),
+             mnemonic#"\t$V1, $V2, $V3, $M4",
+             [(set tr1.op:$V1, (tr1.vt (operator (tr2.vt tr2.op:$V2),
+                                                 (tr2.vt tr2.op:$V3),
+                                                 imm32zx4:$M4)))]> {
+  let M5 = 0;
+  let M6 = 0;
+}
+
+class TernaryVRRd<string mnemonic, bits<16> opcode, SDPatternOperator operator,
+                  TypedReg tr1, TypedReg tr2, bits<4> type = 0>
+  : InstVRRd<opcode, (outs tr1.op:$V1),
+             (ins tr2.op:$V2, tr2.op:$V3, tr1.op:$V4),
+             mnemonic#"\t$V1, $V2, $V3, $V4",
+             [(set tr1.op:$V1, (tr1.vt (operator (tr2.vt tr2.op:$V2),
+                                                 (tr2.vt tr2.op:$V3),
+                                                 (tr1.vt tr1.op:$V4))))]> {
+  let M5 = type;
+  let M6 = 0;
+}
+
+class TernaryVRRe<string mnemonic, bits<16> opcode, SDPatternOperator operator,
+                  TypedReg tr1, TypedReg tr2, bits<4> m5 = 0, bits<4> type = 0>
+  : InstVRRe<opcode, (outs tr1.op:$V1),
+             (ins tr2.op:$V2, tr2.op:$V3, tr1.op:$V4),
+             mnemonic#"\t$V1, $V2, $V3, $V4",
+             [(set tr1.op:$V1, (tr1.vt (operator (tr2.vt tr2.op:$V2),
+                                                 (tr2.vt tr2.op:$V3),
+                                                 (tr1.vt tr1.op:$V4))))]> {
+  let M5 = m5;
+  let M6 = type;
+}
+
+class TernaryVRSb<string mnemonic, bits<16> opcode, SDPatternOperator operator,
+                  TypedReg tr1, TypedReg tr2, RegisterOperand cls, bits<4> type>
+  : InstVRSb<opcode, (outs tr1.op:$V1),
+             (ins tr2.op:$V1src, cls:$R3, shift12only:$BD2),
+             mnemonic#"\t$V1, $R3, $BD2",
+             [(set tr1.op:$V1, (tr1.vt (operator (tr2.vt tr2.op:$V1src),
+                                                 cls:$R3,
+                                                 shift12only:$BD2)))]> {
+  let Constraints = "$V1 = $V1src";
+  let DisableEncoding = "$V1src";
+  let M4 = type;
+}
+
+class TernaryVRV<string mnemonic, bits<16> opcode, bits<5> bytes,
+                 Immediate index>
+  : InstVRV<opcode, (outs VR128:$V1),
+           (ins VR128:$V1src, bdvaddr12only:$VBD2, index:$M3),
+           mnemonic#"\t$V1, $VBD2, $M3", []> {
+  let Constraints = "$V1 = $V1src";
+  let DisableEncoding = "$V1src";
+  let mayLoad = 1;
+  let AccessBytes = bytes;
+}
+
+class TernaryVRX<string mnemonic, bits<16> opcode, SDPatternOperator operator,
+                 TypedReg tr1, TypedReg tr2, bits<5> bytes, Immediate index>
+  : InstVRX<opcode, (outs tr1.op:$V1),
+           (ins tr2.op:$V1src, bdxaddr12only:$XBD2, index:$M3),
+           mnemonic#"\t$V1, $XBD2, $M3",
+           [(set tr1.op:$V1, (tr1.vt (operator (tr2.vt tr2.op:$V1src),
+                                               bdxaddr12only:$XBD2,
+                                               index:$M3)))]> {
+  let Constraints = "$V1 = $V1src";
+  let DisableEncoding = "$V1src";
+  let mayLoad = 1;
+  let AccessBytes = bytes;
+}
+
+class QuaternaryVRId<string mnemonic, bits<16> opcode, SDPatternOperator operator,
+                     TypedReg tr1, TypedReg tr2, bits<4> type>
+  : InstVRId<opcode, (outs tr1.op:$V1),
+             (ins tr2.op:$V1src, tr2.op:$V2, tr2.op:$V3, imm32zx8:$I4),
+             mnemonic#"\t$V1, $V2, $V3, $I4",
+             [(set tr1.op:$V1, (tr1.vt (operator (tr2.vt tr2.op:$V1src),
+                                                 (tr2.vt tr2.op:$V2),
+                                                 (tr2.vt tr2.op:$V3),
+                                                 imm32zx8:$I4)))]> {
+  let Constraints = "$V1 = $V1src";
+  let DisableEncoding = "$V1src";
+  let M5 = type;
+}
+
+class QuaternaryVRRd<string mnemonic, bits<16> opcode,
+                     SDPatternOperator operator, TypedReg tr1, TypedReg tr2,
+                     bits<4> type, SDPatternOperator m6mask, bits<4> m6or>
+  : InstVRRd<opcode, (outs tr1.op:$V1),
+             (ins tr2.op:$V2, tr2.op:$V3, tr2.op:$V4, m6mask:$M6),
+             mnemonic#"\t$V1, $V2, $V3, $V4, $M6",
+             [(set tr1.op:$V1, (tr1.vt (operator (tr2.vt tr2.op:$V2),
+                                                 (tr2.vt tr2.op:$V3),
+                                                 (tr2.vt tr2.op:$V4),
+                                                 m6mask:$M6)))],
+             m6or> {
+  let M5 = type;
+}
+
+multiclass QuaternaryVRRdSPair<string mnemonic, bits<16> opcode,
+                               SDPatternOperator operator,
+                               SDPatternOperator operator_cc, TypedReg tr1,
+                               TypedReg tr2, bits<4> type, bits<4> m6or> {
+  def "" : QuaternaryVRRd<mnemonic, opcode, operator, tr1, tr2, type,
+                          imm32zx4even, !and (m6or, 14)>;
+  def : InstAlias<mnemonic#"\t$V1, $V2, $V3, $V4",
+                  (!cast<Instruction>(NAME) tr1.op:$V1, tr2.op:$V2,
+                                            tr2.op:$V3, tr2.op:$V4, 0)>;
+  let Defs = [CC] in
+    def S : QuaternaryVRRd<mnemonic##"s", opcode, operator_cc, tr1, tr2, type,
+                           imm32zx4even, !add (!and (m6or, 14), 1)>;
+  def : InstAlias<mnemonic#"s\t$V1, $V2, $V3, $V4",
+                  (!cast<Instruction>(NAME#"S") tr1.op:$V1, tr2.op:$V2,
+                                                tr2.op:$V3, tr2.op:$V4, 0)>;
+}
+
 class LoadAndOpRSY<string mnemonic, bits<16> opcode, SDPatternOperator operator,
                   RegisterOperand cls, AddressingMode mode = bdaddr20only>
   : InstRSY<opcode, (outs cls:$R1), (ins cls:$R3, mode:$BD2),
@@ -1341,10 +2151,13 @@ class PrefetchRILPC<string mnemonic, bits<12> opcode,
 
 // A floating-point load-and test operation.  Create both a normal unary
 // operation and one that acts as a comparison against zero.
+// Note that the comparison against zero operation is not available if we
+// have vector support, since load-and-test instructions will partially
+// clobber the target (vector) register.
 multiclass LoadAndTestRRE<string mnemonic, bits<16> opcode,
                           RegisterOperand cls> {
   def "" : UnaryRRE<mnemonic, opcode, null_frag, cls, cls>;
-  let isCodeGenOnly = 1 in
+  let isCodeGenOnly = 1, Predicates = [FeatureNoVector] in
     def Compare : CompareRRE<mnemonic, opcode, null_frag, cls, cls>;
 }
 
@@ -1588,6 +2401,26 @@ class Alias<int size, dag outs, dag ins, list<dag> pattern>
   let isCodeGenOnly = 1;
 }
 
+class UnaryAliasVRS<RegisterOperand cls1, RegisterOperand cls2>
+ : Alias<6, (outs cls1:$src1), (ins cls2:$src2), []>;
+
+// An alias of a UnaryVRR*, but with different register sizes.
+class UnaryAliasVRR<SDPatternOperator operator, TypedReg tr1, TypedReg tr2>
+  : Alias<6, (outs tr1.op:$V1), (ins tr2.op:$V2),
+          [(set tr1.op:$V1, (tr1.vt (operator (tr2.vt tr2.op:$V2))))]>;
+
+// An alias of a UnaryVRX, but with different register sizes.
+class UnaryAliasVRX<SDPatternOperator operator, TypedReg tr,
+                    AddressingMode mode = bdxaddr12only>
+  : Alias<6, (outs tr.op:$V1), (ins mode:$XBD2),
+          [(set tr.op:$V1, (tr.vt (operator mode:$XBD2)))]>;
+
+// An alias of a StoreVRX, but with different register sizes.
+class StoreAliasVRX<SDPatternOperator operator, TypedReg tr,
+                    AddressingMode mode = bdxaddr12only>
+  : Alias<6, (outs), (ins tr.op:$V1, mode:$XBD2),
+          [(operator (tr.vt tr.op:$V1), mode:$XBD2)]>;
+
 // An alias of a BinaryRI, but with different register sizes.
 class BinaryAliasRI<SDPatternOperator operator, RegisterOperand cls,
                     Immediate imm>
@@ -1604,6 +2437,10 @@ class BinaryAliasRIL<SDPatternOperator operator, RegisterOperand cls,
   let Constraints = "$R1 = $R1src";
 }
 
+// An alias of a BinaryVRRf, but with different register sizes.
+class BinaryAliasVRRf<RegisterOperand cls>
+  : Alias<6, (outs VR128:$V1), (ins cls:$R2, cls:$R3), []>;
+
 // An alias of a CompareRI, but with different register sizes.
 class CompareAliasRI<SDPatternOperator operator, RegisterOperand cls,
                      Immediate imm>
diff --git a/lib/Target/SystemZ/SystemZInstrInfo.cpp b/lib/Target/SystemZ/SystemZInstrInfo.cpp
index 3a028594fa4..5d4a34f7131 100644
--- a/lib/Target/SystemZ/SystemZInstrInfo.cpp
+++ b/lib/Target/SystemZ/SystemZInstrInfo.cpp
@@ -362,7 +362,7 @@ ReverseBranchCondition(SmallVectorImpl<MachineOperand> &Cond) const {
 unsigned
 SystemZInstrInfo::InsertBranch(MachineBasicBlock &MBB, MachineBasicBlock *TBB,
                                MachineBasicBlock *FBB,
-                               const SmallVectorImpl<MachineOperand> &Cond,
+                               ArrayRef<MachineOperand> Cond,
                                DebugLoc DL) const {
   // In this function we output 32-bit branches, which should always
   // have enough range.  They can be shortened and relaxed by later code
@@ -423,7 +423,7 @@ static MachineInstr *getDef(unsigned Reg,
 }
 
 // Return true if MI is a shift of type Opcode by Imm bits.
-static bool isShift(MachineInstr *MI, int Opcode, int64_t Imm) {
+static bool isShift(MachineInstr *MI, unsigned Opcode, int64_t Imm) {
   return (MI->getOpcode() == Opcode &&
           !MI->getOperand(2).getReg() &&
           MI->getOperand(3).getImm() == Imm);
@@ -530,8 +530,7 @@ isProfitableToIfCvt(MachineBasicBlock &TMBB,
 }
 
 bool SystemZInstrInfo::
-PredicateInstruction(MachineInstr *MI,
-                     const SmallVectorImpl<MachineOperand> &Pred) const {
+PredicateInstruction(MachineInstr *MI, ArrayRef<MachineOperand> Pred) const {
   assert(Pred.size() == 2 && "Invalid condition");
   unsigned CCValid = Pred[0].getImm();
   unsigned CCMask = Pred[1].getImm();
@@ -578,6 +577,12 @@ SystemZInstrInfo::copyPhysReg(MachineBasicBlock &MBB,
     Opcode = SystemZ::LDR;
   else if (SystemZ::FP128BitRegClass.contains(DestReg, SrcReg))
     Opcode = SystemZ::LXR;
+  else if (SystemZ::VR32BitRegClass.contains(DestReg, SrcReg))
+    Opcode = SystemZ::VLR32;
+  else if (SystemZ::VR64BitRegClass.contains(DestReg, SrcReg))
+    Opcode = SystemZ::VLR64;
+  else if (SystemZ::VR128BitRegClass.contains(DestReg, SrcReg))
+    Opcode = SystemZ::VLR;
   else
     llvm_unreachable("Impossible reg-to-reg copy");
 
@@ -746,10 +751,9 @@ SystemZInstrInfo::convertToThreeAddress(MachineFunction::iterator &MFI,
   return nullptr;
 }
 
-MachineInstr *SystemZInstrInfo::foldMemoryOperandImpl(MachineFunction &MF,
-                                                      MachineInstr *MI,
-                                                      ArrayRef<unsigned> Ops,
-                                                      int FrameIndex) const {
+MachineInstr *SystemZInstrInfo::foldMemoryOperandImpl(
+    MachineFunction &MF, MachineInstr *MI, ArrayRef<unsigned> Ops,
+    MachineBasicBlock::iterator InsertPt, int FrameIndex) const {
   const MachineFrameInfo *MFI = MF.getFrameInfo();
   unsigned Size = MFI->getObjectSize(FrameIndex);
   unsigned Opcode = MI->getOpcode();
@@ -759,9 +763,11 @@ MachineInstr *SystemZInstrInfo::foldMemoryOperandImpl(MachineFunction &MF,
         isInt<8>(MI->getOperand(2).getImm()) &&
         !MI->getOperand(3).getReg()) {
       // LA(Y) %reg, CONST(%reg) -> AGSI %mem, CONST
-      return BuildMI(MF, MI->getDebugLoc(), get(SystemZ::AGSI))
-        .addFrameIndex(FrameIndex).addImm(0)
-        .addImm(MI->getOperand(2).getImm());
+      return BuildMI(*InsertPt->getParent(), InsertPt, MI->getDebugLoc(),
+                     get(SystemZ::AGSI))
+          .addFrameIndex(FrameIndex)
+          .addImm(0)
+          .addImm(MI->getOperand(2).getImm());
     }
     return nullptr;
   }
@@ -780,9 +786,11 @@ MachineInstr *SystemZInstrInfo::foldMemoryOperandImpl(MachineFunction &MF,
       isInt<8>(MI->getOperand(2).getImm())) {
     // A(G)HI %reg, CONST -> A(G)SI %mem, CONST
     Opcode = (Opcode == SystemZ::AHI ? SystemZ::ASI : SystemZ::AGSI);
-    return BuildMI(MF, MI->getDebugLoc(), get(Opcode))
-      .addFrameIndex(FrameIndex).addImm(0)
-      .addImm(MI->getOperand(2).getImm());
+    return BuildMI(*InsertPt->getParent(), InsertPt, MI->getDebugLoc(),
+                   get(Opcode))
+        .addFrameIndex(FrameIndex)
+        .addImm(0)
+        .addImm(MI->getOperand(2).getImm());
   }
 
   if (Opcode == SystemZ::LGDR || Opcode == SystemZ::LDGR) {
@@ -792,17 +800,23 @@ MachineInstr *SystemZInstrInfo::foldMemoryOperandImpl(MachineFunction &MF,
     // source register instead.
     if (OpNum == 0) {
       unsigned StoreOpcode = Op1IsGPR ? SystemZ::STG : SystemZ::STD;
-      return BuildMI(MF, MI->getDebugLoc(), get(StoreOpcode))
-        .addOperand(MI->getOperand(1)).addFrameIndex(FrameIndex)
-        .addImm(0).addReg(0);
+      return BuildMI(*InsertPt->getParent(), InsertPt, MI->getDebugLoc(),
+                     get(StoreOpcode))
+          .addOperand(MI->getOperand(1))
+          .addFrameIndex(FrameIndex)
+          .addImm(0)
+          .addReg(0);
     }
     // If we're spilling the source of an LDGR or LGDR, load the
     // destination register instead.
     if (OpNum == 1) {
       unsigned LoadOpcode = Op0IsGPR ? SystemZ::LG : SystemZ::LD;
       unsigned Dest = MI->getOperand(0).getReg();
-      return BuildMI(MF, MI->getDebugLoc(), get(LoadOpcode), Dest)
-        .addFrameIndex(FrameIndex).addImm(0).addReg(0);
+      return BuildMI(*InsertPt->getParent(), InsertPt, MI->getDebugLoc(),
+                     get(LoadOpcode), Dest)
+          .addFrameIndex(FrameIndex)
+          .addImm(0)
+          .addReg(0);
     }
   }
 
@@ -824,17 +838,25 @@ MachineInstr *SystemZInstrInfo::foldMemoryOperandImpl(MachineFunction &MF,
     if (MMO->getSize() == Size && !MMO->isVolatile()) {
       // Handle conversion of loads.
       if (isSimpleBD12Move(MI, SystemZII::SimpleBDXLoad)) {
-        return BuildMI(MF, MI->getDebugLoc(), get(SystemZ::MVC))
-          .addFrameIndex(FrameIndex).addImm(0).addImm(Size)
-          .addOperand(MI->getOperand(1)).addImm(MI->getOperand(2).getImm())
-          .addMemOperand(MMO);
+        return BuildMI(*InsertPt->getParent(), InsertPt, MI->getDebugLoc(),
+                       get(SystemZ::MVC))
+            .addFrameIndex(FrameIndex)
+            .addImm(0)
+            .addImm(Size)
+            .addOperand(MI->getOperand(1))
+            .addImm(MI->getOperand(2).getImm())
+            .addMemOperand(MMO);
       }
       // Handle conversion of stores.
       if (isSimpleBD12Move(MI, SystemZII::SimpleBDXStore)) {
-        return BuildMI(MF, MI->getDebugLoc(), get(SystemZ::MVC))
-          .addOperand(MI->getOperand(1)).addImm(MI->getOperand(2).getImm())
-          .addImm(Size).addFrameIndex(FrameIndex).addImm(0)
-          .addMemOperand(MMO);
+        return BuildMI(*InsertPt->getParent(), InsertPt, MI->getDebugLoc(),
+                       get(SystemZ::MVC))
+            .addOperand(MI->getOperand(1))
+            .addImm(MI->getOperand(2).getImm())
+            .addImm(Size)
+            .addFrameIndex(FrameIndex)
+            .addImm(0)
+            .addMemOperand(MMO);
       }
     }
   }
@@ -850,7 +872,8 @@ MachineInstr *SystemZInstrInfo::foldMemoryOperandImpl(MachineFunction &MF,
       assert(AccessBytes != 0 && "Size of access should be known");
       assert(AccessBytes <= Size && "Access outside the frame index");
       uint64_t Offset = Size - AccessBytes;
-      MachineInstrBuilder MIB = BuildMI(MF, MI->getDebugLoc(), get(MemOpcode));
+      MachineInstrBuilder MIB = BuildMI(*InsertPt->getParent(), InsertPt,
+                                        MI->getDebugLoc(), get(MemOpcode));
       for (unsigned I = 0; I < OpNum; ++I)
         MIB.addOperand(MI->getOperand(I));
       MIB.addFrameIndex(FrameIndex).addImm(Offset);
@@ -863,10 +886,9 @@ MachineInstr *SystemZInstrInfo::foldMemoryOperandImpl(MachineFunction &MF,
   return nullptr;
 }
 
-MachineInstr *
-SystemZInstrInfo::foldMemoryOperandImpl(MachineFunction &MF, MachineInstr *MI,
-                                        ArrayRef<unsigned> Ops,
-                                        MachineInstr *LoadMI) const {
+MachineInstr *SystemZInstrInfo::foldMemoryOperandImpl(
+    MachineFunction &MF, MachineInstr *MI, ArrayRef<unsigned> Ops,
+    MachineBasicBlock::iterator InsertPt, MachineInstr *LoadMI) const {
   return nullptr;
 }
 
@@ -1116,6 +1138,16 @@ void SystemZInstrInfo::getLoadStoreOpcodes(const TargetRegisterClass *RC,
   } else if (RC == &SystemZ::FP128BitRegClass) {
     LoadOpcode = SystemZ::LX;
     StoreOpcode = SystemZ::STX;
+  } else if (RC == &SystemZ::VR32BitRegClass) {
+    LoadOpcode = SystemZ::VL32;
+    StoreOpcode = SystemZ::VST32;
+  } else if (RC == &SystemZ::VR64BitRegClass) {
+    LoadOpcode = SystemZ::VL64;
+    StoreOpcode = SystemZ::VST64;
+  } else if (RC == &SystemZ::VF128BitRegClass ||
+             RC == &SystemZ::VR128BitRegClass) {
+    LoadOpcode = SystemZ::VL;
+    StoreOpcode = SystemZ::VST;
   } else
     llvm_unreachable("Unsupported regclass to load or store");
 }
@@ -1185,6 +1217,7 @@ static bool isStringOfOnes(uint64_t Mask, unsigned &LSB, unsigned &Length) {
 bool SystemZInstrInfo::isRxSBGMask(uint64_t Mask, unsigned BitSize,
                                    unsigned &Start, unsigned &End) const {
   // Reject trivial all-zero masks.
+  Mask &= allOnes(BitSize);
   if (Mask == 0)
     return false;
 
diff --git a/lib/Target/SystemZ/SystemZInstrInfo.h b/lib/Target/SystemZ/SystemZInstrInfo.h
index b55810b253f..31c9db20958 100644
--- a/lib/Target/SystemZ/SystemZInstrInfo.h
+++ b/lib/Target/SystemZ/SystemZInstrInfo.h
@@ -149,8 +149,7 @@ class SystemZInstrInfo : public SystemZGenInstrInfo {
                      bool AllowModify) const override;
   unsigned RemoveBranch(MachineBasicBlock &MBB) const override;
   unsigned InsertBranch(MachineBasicBlock &MBB, MachineBasicBlock *TBB,
-                        MachineBasicBlock *FBB,
-                        const SmallVectorImpl<MachineOperand> &Cond,
+                        MachineBasicBlock *FBB, ArrayRef<MachineOperand> Cond,
                         DebugLoc DL) const override;
   bool analyzeCompare(const MachineInstr *MI, unsigned &SrcReg,
                       unsigned &SrcReg2, int &Mask, int &Value) const override;
@@ -167,8 +166,7 @@ class SystemZInstrInfo : public SystemZGenInstrInfo {
                            unsigned NumCyclesF, unsigned ExtraPredCyclesF,
                            const BranchProbability &Probability) const override;
   bool PredicateInstruction(MachineInstr *MI,
-                            const SmallVectorImpl<MachineOperand> &Pred) const
-    override;
+                            ArrayRef<MachineOperand> Pred) const override;
   void copyPhysReg(MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI,
                    DebugLoc DL, unsigned DestReg, unsigned SrcReg,
                    bool KillSrc) const override;
@@ -187,9 +185,11 @@ class SystemZInstrInfo : public SystemZGenInstrInfo {
                                       LiveVariables *LV) const override;
   MachineInstr *foldMemoryOperandImpl(MachineFunction &MF, MachineInstr *MI,
                                       ArrayRef<unsigned> Ops,
+                                      MachineBasicBlock::iterator InsertPt,
                                       int FrameIndex) const override;
   MachineInstr *foldMemoryOperandImpl(MachineFunction &MF, MachineInstr *MI,
                                       ArrayRef<unsigned> Ops,
+                                      MachineBasicBlock::iterator InsertPt,
                                       MachineInstr *LoadMI) const override;
   bool expandPostRAPseudo(MachineBasicBlock::iterator MBBI) const override;
   bool ReverseBranchCondition(SmallVectorImpl<MachineOperand> &Cond) const
diff --git a/lib/Target/SystemZ/SystemZInstrVector.td b/lib/Target/SystemZ/SystemZInstrVector.td
new file mode 100644
index 00000000000..c101e43ada3
--- /dev/null
+++ b/lib/Target/SystemZ/SystemZInstrVector.td
@@ -0,0 +1,1097 @@
+//==- SystemZInstrVector.td - SystemZ Vector instructions ------*- tblgen-*-==//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+
+//===----------------------------------------------------------------------===//
+// Move instructions
+//===----------------------------------------------------------------------===//
+
+let Predicates = [FeatureVector] in {
+  // Register move.
+  def VLR : UnaryVRRa<"vlr", 0xE756, null_frag, v128any, v128any>;
+  def VLR32 : UnaryAliasVRR<null_frag, v32eb, v32eb>;
+  def VLR64 : UnaryAliasVRR<null_frag, v64db, v64db>;
+
+  // Load GR from VR element.
+  def VLGVB : BinaryVRSc<"vlgvb", 0xE721, null_frag, v128b, 0>;
+  def VLGVH : BinaryVRSc<"vlgvh", 0xE721, null_frag, v128h, 1>;
+  def VLGVF : BinaryVRSc<"vlgvf", 0xE721, null_frag, v128f, 2>;
+  def VLGVG : BinaryVRSc<"vlgvg", 0xE721, z_vector_extract, v128g, 3>;
+
+  // Load VR element from GR.
+  def VLVGB : TernaryVRSb<"vlvgb", 0xE722, z_vector_insert,
+                          v128b, v128b, GR32, 0>;
+  def VLVGH : TernaryVRSb<"vlvgh", 0xE722, z_vector_insert,
+                          v128h, v128h, GR32, 1>;
+  def VLVGF : TernaryVRSb<"vlvgf", 0xE722, z_vector_insert,
+                          v128f, v128f, GR32, 2>;
+  def VLVGG : TernaryVRSb<"vlvgg", 0xE722, z_vector_insert,
+                          v128g, v128g, GR64, 3>;
+
+  // Load VR from GRs disjoint.
+  def VLVGP : BinaryVRRf<"vlvgp", 0xE762, z_join_dwords, v128g>;
+  def VLVGP32 : BinaryAliasVRRf<GR32>;
+}
+
+// Extractions always assign to the full GR64, even if the element would
+// fit in the lower 32 bits.  Sub-i64 extracts therefore need to take a
+// subreg of the result.
+class VectorExtractSubreg<ValueType type, Instruction insn>
+  : Pat<(i32 (z_vector_extract (type VR128:$vec), shift12only:$index)),
+        (EXTRACT_SUBREG (insn VR128:$vec, shift12only:$index), subreg_l32)>;
+
+def : VectorExtractSubreg<v16i8, VLGVB>;
+def : VectorExtractSubreg<v8i16, VLGVH>;
+def : VectorExtractSubreg<v4i32, VLGVF>;
+
+//===----------------------------------------------------------------------===//
+// Immediate instructions
+//===----------------------------------------------------------------------===//
+
+let Predicates = [FeatureVector] in {
+  // Generate byte mask.
+  def VZERO : InherentVRIa<"vzero", 0xE744, 0>;
+  def VONE  : InherentVRIa<"vone", 0xE744, 0xffff>;
+  def VGBM  : UnaryVRIa<"vgbm", 0xE744, z_byte_mask, v128b, imm32zx16>;
+
+  // Generate mask.
+  def VGMB : BinaryVRIb<"vgmb", 0xE746, z_rotate_mask, v128b, 0>;
+  def VGMH : BinaryVRIb<"vgmh", 0xE746, z_rotate_mask, v128h, 1>;
+  def VGMF : BinaryVRIb<"vgmf", 0xE746, z_rotate_mask, v128f, 2>;
+  def VGMG : BinaryVRIb<"vgmg", 0xE746, z_rotate_mask, v128g, 3>;
+
+  // Load element immediate.
+  //
+  // We want these instructions to be used ahead of VLVG* where possible.
+  // However, VLVG* takes a variable BD-format index whereas VLEI takes
+  // a plain immediate index.  This means that VLVG* has an extra "base"
+  // register operand and is 3 units more complex.  Bumping the complexity
+  // of the VLEI* instructions by 4 means that they are strictly better
+  // than VLVG* in cases where both forms match.
+  let AddedComplexity = 4 in {
+    def VLEIB : TernaryVRIa<"vleib", 0xE740, z_vector_insert,
+                            v128b, v128b, imm32sx16trunc, imm32zx4>;
+    def VLEIH : TernaryVRIa<"vleih", 0xE741, z_vector_insert,
+                            v128h, v128h, imm32sx16trunc, imm32zx3>;
+    def VLEIF : TernaryVRIa<"vleif", 0xE743, z_vector_insert,
+                            v128f, v128f, imm32sx16, imm32zx2>;
+    def VLEIG : TernaryVRIa<"vleig", 0xE742, z_vector_insert,
+                            v128g, v128g, imm64sx16, imm32zx1>;
+  }
+
+  // Replicate immediate.
+  def VREPIB : UnaryVRIa<"vrepib", 0xE745, z_replicate, v128b, imm32sx16, 0>;
+  def VREPIH : UnaryVRIa<"vrepih", 0xE745, z_replicate, v128h, imm32sx16, 1>;
+  def VREPIF : UnaryVRIa<"vrepif", 0xE745, z_replicate, v128f, imm32sx16, 2>;
+  def VREPIG : UnaryVRIa<"vrepig", 0xE745, z_replicate, v128g, imm32sx16, 3>;
+}
+
+//===----------------------------------------------------------------------===//
+// Loads
+//===----------------------------------------------------------------------===//
+
+let Predicates = [FeatureVector] in {
+  // Load.
+  def VL : UnaryVRX<"vl", 0xE706, null_frag, v128any, 16>;
+
+  // Load to block boundary.  The number of loaded bytes is only known
+  // at run time.  The instruction is really polymorphic, but v128b matches
+  // the return type of the associated intrinsic.
+  def VLBB : BinaryVRX<"vlbb", 0xE707, int_s390_vlbb, v128b, 0>;
+
+  // Load count to block boundary.
+  let Defs = [CC] in
+    def LCBB : InstRXE<0xE727, (outs GR32:$R1),
+                               (ins bdxaddr12only:$XBD2, imm32zx4:$M3),
+                       "lcbb\t$R1, $XBD2, $M3",
+                       [(set GR32:$R1, (int_s390_lcbb bdxaddr12only:$XBD2,
+                                                      imm32zx4:$M3))]>;
+
+  // Load with length.  The number of loaded bytes is only known at run time.
+  def VLL : BinaryVRSb<"vll", 0xE737, int_s390_vll, 0>;
+
+  // Load multiple.
+  def VLM : LoadMultipleVRSa<"vlm", 0xE736>;
+
+  // Load and replicate
+  def VLREPB : UnaryVRX<"vlrepb", 0xE705, z_replicate_loadi8,  v128b, 1, 0>;
+  def VLREPH : UnaryVRX<"vlreph", 0xE705, z_replicate_loadi16, v128h, 2, 1>;
+  def VLREPF : UnaryVRX<"vlrepf", 0xE705, z_replicate_loadi32, v128f, 4, 2>;
+  def VLREPG : UnaryVRX<"vlrepg", 0xE705, z_replicate_loadi64, v128g, 8, 3>;
+  def : Pat<(v4f32 (z_replicate_loadf32 bdxaddr12only:$addr)),
+            (VLREPF bdxaddr12only:$addr)>;
+  def : Pat<(v2f64 (z_replicate_loadf64 bdxaddr12only:$addr)),
+            (VLREPG bdxaddr12only:$addr)>;
+
+  // Use VLREP to load subvectors.  These patterns use "12pair" because
+  // LEY and LDY offer full 20-bit displacement fields.  It's often better
+  // to use those instructions rather than force a 20-bit displacement
+  // into a GPR temporary.
+  def VL32 : UnaryAliasVRX<load, v32eb, bdxaddr12pair>;
+  def VL64 : UnaryAliasVRX<load, v64db, bdxaddr12pair>;
+
+  // Load logical element and zero.
+  def VLLEZB : UnaryVRX<"vllezb", 0xE704, z_vllezi8,  v128b, 1, 0>;
+  def VLLEZH : UnaryVRX<"vllezh", 0xE704, z_vllezi16, v128h, 2, 1>;
+  def VLLEZF : UnaryVRX<"vllezf", 0xE704, z_vllezi32, v128f, 4, 2>;
+  def VLLEZG : UnaryVRX<"vllezg", 0xE704, z_vllezi64, v128g, 8, 3>;
+  def : Pat<(v4f32 (z_vllezf32 bdxaddr12only:$addr)),
+            (VLLEZF bdxaddr12only:$addr)>;
+  def : Pat<(v2f64 (z_vllezf64 bdxaddr12only:$addr)),
+            (VLLEZG bdxaddr12only:$addr)>;
+
+  // Load element.
+  def VLEB : TernaryVRX<"vleb", 0xE700, z_vlei8,  v128b, v128b, 1, imm32zx4>;
+  def VLEH : TernaryVRX<"vleh", 0xE701, z_vlei16, v128h, v128h, 2, imm32zx3>;
+  def VLEF : TernaryVRX<"vlef", 0xE703, z_vlei32, v128f, v128f, 4, imm32zx2>;
+  def VLEG : TernaryVRX<"vleg", 0xE702, z_vlei64, v128g, v128g, 8, imm32zx1>;
+  def : Pat<(z_vlef32 (v4f32 VR128:$val), bdxaddr12only:$addr, imm32zx2:$index),
+            (VLEF VR128:$val, bdxaddr12only:$addr, imm32zx2:$index)>;
+  def : Pat<(z_vlef64 (v2f64 VR128:$val), bdxaddr12only:$addr, imm32zx1:$index),
+            (VLEG VR128:$val, bdxaddr12only:$addr, imm32zx1:$index)>;
+
+  // Gather element.
+  def VGEF : TernaryVRV<"vgef", 0xE713, 4, imm32zx2>;
+  def VGEG : TernaryVRV<"vgeg", 0xE712, 8, imm32zx1>;
+}
+
+// Use replicating loads if we're inserting a single element into an
+// undefined vector.  This avoids a false dependency on the previous
+// register contents.
+multiclass ReplicatePeephole<Instruction vlrep, ValueType vectype,
+                             SDPatternOperator load, ValueType scalartype> {
+  def : Pat<(vectype (z_vector_insert
+                      (undef), (scalartype (load bdxaddr12only:$addr)), 0)),
+            (vlrep bdxaddr12only:$addr)>;
+  def : Pat<(vectype (scalar_to_vector
+                      (scalartype (load bdxaddr12only:$addr)))),
+            (vlrep bdxaddr12only:$addr)>;
+}
+defm : ReplicatePeephole<VLREPB, v16i8, anyextloadi8, i32>;
+defm : ReplicatePeephole<VLREPH, v8i16, anyextloadi16, i32>;
+defm : ReplicatePeephole<VLREPF, v4i32, load, i32>;
+defm : ReplicatePeephole<VLREPG, v2i64, load, i64>;
+defm : ReplicatePeephole<VLREPF, v4f32, load, f32>;
+defm : ReplicatePeephole<VLREPG, v2f64, load, f64>;
+
+//===----------------------------------------------------------------------===//
+// Stores
+//===----------------------------------------------------------------------===//
+
+let Predicates = [FeatureVector] in {
+  // Store.
+  def VST : StoreVRX<"vst", 0xE70E, null_frag, v128any, 16>;
+
+  // Store with length.  The number of stored bytes is only known at run time.
+  def VSTL : StoreLengthVRSb<"vstl", 0xE73F, int_s390_vstl, 0>;
+
+  // Store multiple.
+  def VSTM : StoreMultipleVRSa<"vstm", 0xE73E>;
+
+  // Store element.
+  def VSTEB : StoreBinaryVRX<"vsteb", 0xE708, z_vstei8,  v128b, 1, imm32zx4>;
+  def VSTEH : StoreBinaryVRX<"vsteh", 0xE709, z_vstei16, v128h, 2, imm32zx3>;
+  def VSTEF : StoreBinaryVRX<"vstef", 0xE70B, z_vstei32, v128f, 4, imm32zx2>;
+  def VSTEG : StoreBinaryVRX<"vsteg", 0xE70A, z_vstei64, v128g, 8, imm32zx1>;
+  def : Pat<(z_vstef32 (v4f32 VR128:$val), bdxaddr12only:$addr,
+                       imm32zx2:$index),
+            (VSTEF VR128:$val, bdxaddr12only:$addr, imm32zx2:$index)>;
+  def : Pat<(z_vstef64 (v2f64 VR128:$val), bdxaddr12only:$addr,
+                       imm32zx1:$index),
+            (VSTEG VR128:$val, bdxaddr12only:$addr, imm32zx1:$index)>;
+
+  // Use VSTE to store subvectors.  These patterns use "12pair" because
+  // STEY and STDY offer full 20-bit displacement fields.  It's often better
+  // to use those instructions rather than force a 20-bit displacement
+  // into a GPR temporary.
+  def VST32 : StoreAliasVRX<store, v32eb, bdxaddr12pair>;
+  def VST64 : StoreAliasVRX<store, v64db, bdxaddr12pair>;
+
+  // Scatter element.
+  def VSCEF : StoreBinaryVRV<"vscef", 0xE71B, 4, imm32zx2>;
+  def VSCEG : StoreBinaryVRV<"vsceg", 0xE71A, 8, imm32zx1>;
+}
+
+//===----------------------------------------------------------------------===//
+// Selects and permutes
+//===----------------------------------------------------------------------===//
+
+let Predicates = [FeatureVector] in {
+  // Merge high.
+  def VMRHB : BinaryVRRc<"vmrhb", 0xE761, z_merge_high, v128b, v128b, 0>;
+  def VMRHH : BinaryVRRc<"vmrhh", 0xE761, z_merge_high, v128h, v128h, 1>;
+  def VMRHF : BinaryVRRc<"vmrhf", 0xE761, z_merge_high, v128f, v128f, 2>;
+  def VMRHG : BinaryVRRc<"vmrhg", 0xE761, z_merge_high, v128g, v128g, 3>;
+  def : BinaryRRWithType<VMRHF, VR128, z_merge_high, v4f32>;
+  def : BinaryRRWithType<VMRHG, VR128, z_merge_high, v2f64>;
+
+  // Merge low.
+  def VMRLB : BinaryVRRc<"vmrlb", 0xE760, z_merge_low, v128b, v128b, 0>;
+  def VMRLH : BinaryVRRc<"vmrlh", 0xE760, z_merge_low, v128h, v128h, 1>;
+  def VMRLF : BinaryVRRc<"vmrlf", 0xE760, z_merge_low, v128f, v128f, 2>;
+  def VMRLG : BinaryVRRc<"vmrlg", 0xE760, z_merge_low, v128g, v128g, 3>;
+  def : BinaryRRWithType<VMRLF, VR128, z_merge_low, v4f32>;
+  def : BinaryRRWithType<VMRLG, VR128, z_merge_low, v2f64>;
+
+  // Permute.
+  def VPERM : TernaryVRRe<"vperm", 0xE78C, z_permute, v128b, v128b>;
+
+  // Permute doubleword immediate.
+  def VPDI : TernaryVRRc<"vpdi", 0xE784, z_permute_dwords, v128g, v128g>;
+
+  // Replicate.
+  def VREPB : BinaryVRIc<"vrepb", 0xE74D, z_splat, v128b, v128b, 0>;
+  def VREPH : BinaryVRIc<"vreph", 0xE74D, z_splat, v128h, v128h, 1>;
+  def VREPF : BinaryVRIc<"vrepf", 0xE74D, z_splat, v128f, v128f, 2>;
+  def VREPG : BinaryVRIc<"vrepg", 0xE74D, z_splat, v128g, v128g, 3>;
+  def : Pat<(v4f32 (z_splat VR128:$vec, imm32zx16:$index)),
+            (VREPF VR128:$vec, imm32zx16:$index)>;
+  def : Pat<(v2f64 (z_splat VR128:$vec, imm32zx16:$index)),
+            (VREPG VR128:$vec, imm32zx16:$index)>;
+
+  // Select.
+  def VSEL : TernaryVRRe<"vsel", 0xE78D, null_frag, v128any, v128any>;
+}
+
+//===----------------------------------------------------------------------===//
+// Widening and narrowing
+//===----------------------------------------------------------------------===//
+
+let Predicates = [FeatureVector] in {
+  // Pack
+  def VPKH : BinaryVRRc<"vpkh", 0xE794, z_pack, v128b, v128h, 1>;
+  def VPKF : BinaryVRRc<"vpkf", 0xE794, z_pack, v128h, v128f, 2>;
+  def VPKG : BinaryVRRc<"vpkg", 0xE794, z_pack, v128f, v128g, 3>;
+
+  // Pack saturate.
+  defm VPKSH : BinaryVRRbSPair<"vpksh", 0xE797, int_s390_vpksh, z_packs_cc,
+                               v128b, v128h, 1>;
+  defm VPKSF : BinaryVRRbSPair<"vpksf", 0xE797, int_s390_vpksf, z_packs_cc,
+                               v128h, v128f, 2>;
+  defm VPKSG : BinaryVRRbSPair<"vpksg", 0xE797, int_s390_vpksg, z_packs_cc,
+                               v128f, v128g, 3>;
+
+  // Pack saturate logical.
+  defm VPKLSH : BinaryVRRbSPair<"vpklsh", 0xE795, int_s390_vpklsh, z_packls_cc,
+                                v128b, v128h, 1>;
+  defm VPKLSF : BinaryVRRbSPair<"vpklsf", 0xE795, int_s390_vpklsf, z_packls_cc,
+                                v128h, v128f, 2>;
+  defm VPKLSG : BinaryVRRbSPair<"vpklsg", 0xE795, int_s390_vpklsg, z_packls_cc,
+                                v128f, v128g, 3>;
+
+  // Sign-extend to doubleword.
+  def VSEGB : UnaryVRRa<"vsegb", 0xE75F, z_vsei8,  v128g, v128g, 0>;
+  def VSEGH : UnaryVRRa<"vsegh", 0xE75F, z_vsei16, v128g, v128g, 1>;
+  def VSEGF : UnaryVRRa<"vsegf", 0xE75F, z_vsei32, v128g, v128g, 2>;
+  def : Pat<(z_vsei8_by_parts  (v16i8 VR128:$src)), (VSEGB VR128:$src)>;
+  def : Pat<(z_vsei16_by_parts (v8i16 VR128:$src)), (VSEGH VR128:$src)>;
+  def : Pat<(z_vsei32_by_parts (v4i32 VR128:$src)), (VSEGF VR128:$src)>;
+
+  // Unpack high.
+  def VUPHB : UnaryVRRa<"vuphb", 0xE7D7, z_unpack_high, v128h, v128b, 0>;
+  def VUPHH : UnaryVRRa<"vuphh", 0xE7D7, z_unpack_high, v128f, v128h, 1>;
+  def VUPHF : UnaryVRRa<"vuphf", 0xE7D7, z_unpack_high, v128g, v128f, 2>;
+
+  // Unpack logical high.
+  def VUPLHB : UnaryVRRa<"vuplhb", 0xE7D5, z_unpackl_high, v128h, v128b, 0>;
+  def VUPLHH : UnaryVRRa<"vuplhh", 0xE7D5, z_unpackl_high, v128f, v128h, 1>;
+  def VUPLHF : UnaryVRRa<"vuplhf", 0xE7D5, z_unpackl_high, v128g, v128f, 2>;
+
+  // Unpack low.
+  def VUPLB  : UnaryVRRa<"vuplb",  0xE7D6, z_unpack_low, v128h, v128b, 0>;
+  def VUPLHW : UnaryVRRa<"vuplhw", 0xE7D6, z_unpack_low, v128f, v128h, 1>;
+  def VUPLF  : UnaryVRRa<"vuplf",  0xE7D6, z_unpack_low, v128g, v128f, 2>;
+
+  // Unpack logical low.
+  def VUPLLB : UnaryVRRa<"vupllb", 0xE7D4, z_unpackl_low, v128h, v128b, 0>;
+  def VUPLLH : UnaryVRRa<"vupllh", 0xE7D4, z_unpackl_low, v128f, v128h, 1>;
+  def VUPLLF : UnaryVRRa<"vupllf", 0xE7D4, z_unpackl_low, v128g, v128f, 2>;
+}
+
+//===----------------------------------------------------------------------===//
+// Instantiating generic operations for specific types.
+//===----------------------------------------------------------------------===//
+
+multiclass GenericVectorOps<ValueType type, ValueType inttype> {
+  let Predicates = [FeatureVector] in {
+    def : Pat<(type (load bdxaddr12only:$addr)),
+              (VL bdxaddr12only:$addr)>;
+    def : Pat<(store (type VR128:$src), bdxaddr12only:$addr),
+              (VST VR128:$src, bdxaddr12only:$addr)>;
+    def : Pat<(type (vselect (inttype VR128:$x), VR128:$y, VR128:$z)),
+              (VSEL VR128:$y, VR128:$z, VR128:$x)>;
+    def : Pat<(type (vselect (inttype (z_vnot VR128:$x)), VR128:$y, VR128:$z)),
+              (VSEL VR128:$z, VR128:$y, VR128:$x)>;
+  }
+}
+
+defm : GenericVectorOps<v16i8, v16i8>;
+defm : GenericVectorOps<v8i16, v8i16>;
+defm : GenericVectorOps<v4i32, v4i32>;
+defm : GenericVectorOps<v2i64, v2i64>;
+defm : GenericVectorOps<v4f32, v4i32>;
+defm : GenericVectorOps<v2f64, v2i64>;
+
+//===----------------------------------------------------------------------===//
+// Integer arithmetic
+//===----------------------------------------------------------------------===//
+
+let Predicates = [FeatureVector] in {
+  // Add.
+  def VAB : BinaryVRRc<"vab", 0xE7F3, add, v128b, v128b, 0>;
+  def VAH : BinaryVRRc<"vah", 0xE7F3, add, v128h, v128h, 1>;
+  def VAF : BinaryVRRc<"vaf", 0xE7F3, add, v128f, v128f, 2>;
+  def VAG : BinaryVRRc<"vag", 0xE7F3, add, v128g, v128g, 3>;
+  def VAQ : BinaryVRRc<"vaq", 0xE7F3, int_s390_vaq, v128q, v128q, 4>;
+
+  // Add compute carry.
+  def VACCB : BinaryVRRc<"vaccb", 0xE7F1, int_s390_vaccb, v128b, v128b, 0>;
+  def VACCH : BinaryVRRc<"vacch", 0xE7F1, int_s390_vacch, v128h, v128h, 1>;
+  def VACCF : BinaryVRRc<"vaccf", 0xE7F1, int_s390_vaccf, v128f, v128f, 2>;
+  def VACCG : BinaryVRRc<"vaccg", 0xE7F1, int_s390_vaccg, v128g, v128g, 3>;
+  def VACCQ : BinaryVRRc<"vaccq", 0xE7F1, int_s390_vaccq, v128q, v128q, 4>;
+
+  // Add with carry.
+  def VACQ : TernaryVRRd<"vacq", 0xE7BB, int_s390_vacq, v128q, v128q, 4>;
+
+  // Add with carry compute carry.
+  def VACCCQ : TernaryVRRd<"vacccq", 0xE7B9, int_s390_vacccq, v128q, v128q, 4>;
+
+  // And.
+  def VN : BinaryVRRc<"vn", 0xE768, null_frag, v128any, v128any>;
+
+  // And with complement.
+  def VNC : BinaryVRRc<"vnc", 0xE769, null_frag, v128any, v128any>;
+
+  // Average.
+  def VAVGB : BinaryVRRc<"vavgb", 0xE7F2, int_s390_vavgb, v128b, v128b, 0>;
+  def VAVGH : BinaryVRRc<"vavgh", 0xE7F2, int_s390_vavgh, v128h, v128h, 1>;
+  def VAVGF : BinaryVRRc<"vavgf", 0xE7F2, int_s390_vavgf, v128f, v128f, 2>;
+  def VAVGG : BinaryVRRc<"vavgg", 0xE7F2, int_s390_vavgg, v128g, v128g, 3>;
+
+  // Average logical.
+  def VAVGLB : BinaryVRRc<"vavglb", 0xE7F0, int_s390_vavglb, v128b, v128b, 0>;
+  def VAVGLH : BinaryVRRc<"vavglh", 0xE7F0, int_s390_vavglh, v128h, v128h, 1>;
+  def VAVGLF : BinaryVRRc<"vavglf", 0xE7F0, int_s390_vavglf, v128f, v128f, 2>;
+  def VAVGLG : BinaryVRRc<"vavglg", 0xE7F0, int_s390_vavglg, v128g, v128g, 3>;
+
+  // Checksum.
+  def VCKSM : BinaryVRRc<"vcksm", 0xE766, int_s390_vcksm, v128f, v128f>;
+
+  // Count leading zeros.
+  def VCLZB : UnaryVRRa<"vclzb", 0xE753, ctlz, v128b, v128b, 0>;
+  def VCLZH : UnaryVRRa<"vclzh", 0xE753, ctlz, v128h, v128h, 1>;
+  def VCLZF : UnaryVRRa<"vclzf", 0xE753, ctlz, v128f, v128f, 2>;
+  def VCLZG : UnaryVRRa<"vclzg", 0xE753, ctlz, v128g, v128g, 3>;
+
+  // Count trailing zeros.
+  def VCTZB : UnaryVRRa<"vctzb", 0xE752, cttz, v128b, v128b, 0>;
+  def VCTZH : UnaryVRRa<"vctzh", 0xE752, cttz, v128h, v128h, 1>;
+  def VCTZF : UnaryVRRa<"vctzf", 0xE752, cttz, v128f, v128f, 2>;
+  def VCTZG : UnaryVRRa<"vctzg", 0xE752, cttz, v128g, v128g, 3>;
+
+  // Exclusive or.
+  def VX : BinaryVRRc<"vx", 0xE76D, null_frag, v128any, v128any>;
+
+  // Galois field multiply sum.
+  def VGFMB : BinaryVRRc<"vgfmb", 0xE7B4, int_s390_vgfmb, v128h, v128b, 0>;
+  def VGFMH : BinaryVRRc<"vgfmh", 0xE7B4, int_s390_vgfmh, v128f, v128h, 1>;
+  def VGFMF : BinaryVRRc<"vgfmf", 0xE7B4, int_s390_vgfmf, v128g, v128f, 2>;
+  def VGFMG : BinaryVRRc<"vgfmg", 0xE7B4, int_s390_vgfmg, v128q, v128g, 3>;
+
+  // Galois field multiply sum and accumulate.
+  def VGFMAB : TernaryVRRd<"vgfmab", 0xE7BC, int_s390_vgfmab, v128h, v128b, 0>;
+  def VGFMAH : TernaryVRRd<"vgfmah", 0xE7BC, int_s390_vgfmah, v128f, v128h, 1>;
+  def VGFMAF : TernaryVRRd<"vgfmaf", 0xE7BC, int_s390_vgfmaf, v128g, v128f, 2>;
+  def VGFMAG : TernaryVRRd<"vgfmag", 0xE7BC, int_s390_vgfmag, v128q, v128g, 3>;
+
+  // Load complement.
+  def VLCB : UnaryVRRa<"vlcb", 0xE7DE, z_vneg, v128b, v128b, 0>;
+  def VLCH : UnaryVRRa<"vlch", 0xE7DE, z_vneg, v128h, v128h, 1>;
+  def VLCF : UnaryVRRa<"vlcf", 0xE7DE, z_vneg, v128f, v128f, 2>;
+  def VLCG : UnaryVRRa<"vlcg", 0xE7DE, z_vneg, v128g, v128g, 3>;
+
+  // Load positive.
+  def VLPB : UnaryVRRa<"vlpb", 0xE7DF, z_viabs8,  v128b, v128b, 0>;
+  def VLPH : UnaryVRRa<"vlph", 0xE7DF, z_viabs16, v128h, v128h, 1>;
+  def VLPF : UnaryVRRa<"vlpf", 0xE7DF, z_viabs32, v128f, v128f, 2>;
+  def VLPG : UnaryVRRa<"vlpg", 0xE7DF, z_viabs64, v128g, v128g, 3>;
+
+  // Maximum.
+  def VMXB : BinaryVRRc<"vmxb", 0xE7FF, null_frag, v128b, v128b, 0>;
+  def VMXH : BinaryVRRc<"vmxh", 0xE7FF, null_frag, v128h, v128h, 1>;
+  def VMXF : BinaryVRRc<"vmxf", 0xE7FF, null_frag, v128f, v128f, 2>;
+  def VMXG : BinaryVRRc<"vmxg", 0xE7FF, null_frag, v128g, v128g, 3>;
+
+  // Maximum logical.
+  def VMXLB : BinaryVRRc<"vmxlb", 0xE7FD, null_frag, v128b, v128b, 0>;
+  def VMXLH : BinaryVRRc<"vmxlh", 0xE7FD, null_frag, v128h, v128h, 1>;
+  def VMXLF : BinaryVRRc<"vmxlf", 0xE7FD, null_frag, v128f, v128f, 2>;
+  def VMXLG : BinaryVRRc<"vmxlg", 0xE7FD, null_frag, v128g, v128g, 3>;
+
+  // Minimum.
+  def VMNB : BinaryVRRc<"vmnb", 0xE7FE, null_frag, v128b, v128b, 0>;
+  def VMNH : BinaryVRRc<"vmnh", 0xE7FE, null_frag, v128h, v128h, 1>;
+  def VMNF : BinaryVRRc<"vmnf", 0xE7FE, null_frag, v128f, v128f, 2>;
+  def VMNG : BinaryVRRc<"vmng", 0xE7FE, null_frag, v128g, v128g, 3>;
+
+  // Minimum logical.
+  def VMNLB : BinaryVRRc<"vmnlb", 0xE7FC, null_frag, v128b, v128b, 0>;
+  def VMNLH : BinaryVRRc<"vmnlh", 0xE7FC, null_frag, v128h, v128h, 1>;
+  def VMNLF : BinaryVRRc<"vmnlf", 0xE7FC, null_frag, v128f, v128f, 2>;
+  def VMNLG : BinaryVRRc<"vmnlg", 0xE7FC, null_frag, v128g, v128g, 3>;
+
+  // Multiply and add low.
+  def VMALB  : TernaryVRRd<"vmalb",  0xE7AA, z_muladd, v128b, v128b, 0>;
+  def VMALHW : TernaryVRRd<"vmalhw", 0xE7AA, z_muladd, v128h, v128h, 1>;
+  def VMALF  : TernaryVRRd<"vmalf",  0xE7AA, z_muladd, v128f, v128f, 2>;
+
+  // Multiply and add high.
+  def VMAHB : TernaryVRRd<"vmahb", 0xE7AB, int_s390_vmahb, v128b, v128b, 0>;
+  def VMAHH : TernaryVRRd<"vmahh", 0xE7AB, int_s390_vmahh, v128h, v128h, 1>;
+  def VMAHF : TernaryVRRd<"vmahf", 0xE7AB, int_s390_vmahf, v128f, v128f, 2>;
+
+  // Multiply and add logical high.
+  def VMALHB : TernaryVRRd<"vmalhb", 0xE7A9, int_s390_vmalhb, v128b, v128b, 0>;
+  def VMALHH : TernaryVRRd<"vmalhh", 0xE7A9, int_s390_vmalhh, v128h, v128h, 1>;
+  def VMALHF : TernaryVRRd<"vmalhf", 0xE7A9, int_s390_vmalhf, v128f, v128f, 2>;
+
+  // Multiply and add even.
+  def VMAEB : TernaryVRRd<"vmaeb", 0xE7AE, int_s390_vmaeb, v128h, v128b, 0>;
+  def VMAEH : TernaryVRRd<"vmaeh", 0xE7AE, int_s390_vmaeh, v128f, v128h, 1>;
+  def VMAEF : TernaryVRRd<"vmaef", 0xE7AE, int_s390_vmaef, v128g, v128f, 2>;
+
+  // Multiply and add logical even.
+  def VMALEB : TernaryVRRd<"vmaleb", 0xE7AC, int_s390_vmaleb, v128h, v128b, 0>;
+  def VMALEH : TernaryVRRd<"vmaleh", 0xE7AC, int_s390_vmaleh, v128f, v128h, 1>;
+  def VMALEF : TernaryVRRd<"vmalef", 0xE7AC, int_s390_vmalef, v128g, v128f, 2>;
+
+  // Multiply and add odd.
+  def VMAOB : TernaryVRRd<"vmaob", 0xE7AF, int_s390_vmaob, v128h, v128b, 0>;
+  def VMAOH : TernaryVRRd<"vmaoh", 0xE7AF, int_s390_vmaoh, v128f, v128h, 1>;
+  def VMAOF : TernaryVRRd<"vmaof", 0xE7AF, int_s390_vmaof, v128g, v128f, 2>;
+
+  // Multiply and add logical odd.
+  def VMALOB : TernaryVRRd<"vmalob", 0xE7AD, int_s390_vmalob, v128h, v128b, 0>;
+  def VMALOH : TernaryVRRd<"vmaloh", 0xE7AD, int_s390_vmaloh, v128f, v128h, 1>;
+  def VMALOF : TernaryVRRd<"vmalof", 0xE7AD, int_s390_vmalof, v128g, v128f, 2>;
+
+  // Multiply high.
+  def VMHB : BinaryVRRc<"vmhb", 0xE7A3, int_s390_vmhb, v128b, v128b, 0>;
+  def VMHH : BinaryVRRc<"vmhh", 0xE7A3, int_s390_vmhh, v128h, v128h, 1>;
+  def VMHF : BinaryVRRc<"vmhf", 0xE7A3, int_s390_vmhf, v128f, v128f, 2>;
+
+  // Multiply logical high.
+  def VMLHB : BinaryVRRc<"vmlhb", 0xE7A1, int_s390_vmlhb, v128b, v128b, 0>;
+  def VMLHH : BinaryVRRc<"vmlhh", 0xE7A1, int_s390_vmlhh, v128h, v128h, 1>;
+  def VMLHF : BinaryVRRc<"vmlhf", 0xE7A1, int_s390_vmlhf, v128f, v128f, 2>;
+
+  // Multiply low.
+  def VMLB  : BinaryVRRc<"vmlb",  0xE7A2, mul, v128b, v128b, 0>;
+  def VMLHW : BinaryVRRc<"vmlhw", 0xE7A2, mul, v128h, v128h, 1>;
+  def VMLF  : BinaryVRRc<"vmlf",  0xE7A2, mul, v128f, v128f, 2>;
+
+  // Multiply even.
+  def VMEB : BinaryVRRc<"vmeb", 0xE7A6, int_s390_vmeb, v128h, v128b, 0>;
+  def VMEH : BinaryVRRc<"vmeh", 0xE7A6, int_s390_vmeh, v128f, v128h, 1>;
+  def VMEF : BinaryVRRc<"vmef", 0xE7A6, int_s390_vmef, v128g, v128f, 2>;
+
+  // Multiply logical even.
+  def VMLEB : BinaryVRRc<"vmleb", 0xE7A4, int_s390_vmleb, v128h, v128b, 0>;
+  def VMLEH : BinaryVRRc<"vmleh", 0xE7A4, int_s390_vmleh, v128f, v128h, 1>;
+  def VMLEF : BinaryVRRc<"vmlef", 0xE7A4, int_s390_vmlef, v128g, v128f, 2>;
+
+  // Multiply odd.
+  def VMOB : BinaryVRRc<"vmob", 0xE7A7, int_s390_vmob, v128h, v128b, 0>;
+  def VMOH : BinaryVRRc<"vmoh", 0xE7A7, int_s390_vmoh, v128f, v128h, 1>;
+  def VMOF : BinaryVRRc<"vmof", 0xE7A7, int_s390_vmof, v128g, v128f, 2>;
+
+  // Multiply logical odd.
+  def VMLOB : BinaryVRRc<"vmlob", 0xE7A5, int_s390_vmlob, v128h, v128b, 0>;
+  def VMLOH : BinaryVRRc<"vmloh", 0xE7A5, int_s390_vmloh, v128f, v128h, 1>;
+  def VMLOF : BinaryVRRc<"vmlof", 0xE7A5, int_s390_vmlof, v128g, v128f, 2>;
+
+  // Nor.
+  def VNO : BinaryVRRc<"vno", 0xE76B, null_frag, v128any, v128any>;
+
+  // Or.
+  def VO : BinaryVRRc<"vo", 0xE76A, null_frag, v128any, v128any>;
+
+  // Population count.
+  def VPOPCT : BinaryVRRa<"vpopct", 0xE750>;
+  def : Pat<(v16i8 (z_popcnt VR128:$x)), (VPOPCT VR128:$x, 0)>;
+
+  // Element rotate left logical (with vector shift amount).
+  def VERLLVB : BinaryVRRc<"verllvb", 0xE773, int_s390_verllvb,
+                           v128b, v128b, 0>;
+  def VERLLVH : BinaryVRRc<"verllvh", 0xE773, int_s390_verllvh,
+                           v128h, v128h, 1>;
+  def VERLLVF : BinaryVRRc<"verllvf", 0xE773, int_s390_verllvf,
+                           v128f, v128f, 2>;
+  def VERLLVG : BinaryVRRc<"verllvg", 0xE773, int_s390_verllvg,
+                           v128g, v128g, 3>;
+
+  // Element rotate left logical (with scalar shift amount).
+  def VERLLB : BinaryVRSa<"verllb", 0xE733, int_s390_verllb, v128b, v128b, 0>;
+  def VERLLH : BinaryVRSa<"verllh", 0xE733, int_s390_verllh, v128h, v128h, 1>;
+  def VERLLF : BinaryVRSa<"verllf", 0xE733, int_s390_verllf, v128f, v128f, 2>;
+  def VERLLG : BinaryVRSa<"verllg", 0xE733, int_s390_verllg, v128g, v128g, 3>;
+
+  // Element rotate and insert under mask.
+  def VERIMB : QuaternaryVRId<"verimb", 0xE772, int_s390_verimb, v128b, v128b, 0>;
+  def VERIMH : QuaternaryVRId<"verimh", 0xE772, int_s390_verimh, v128h, v128h, 1>;
+  def VERIMF : QuaternaryVRId<"verimf", 0xE772, int_s390_verimf, v128f, v128f, 2>;
+  def VERIMG : QuaternaryVRId<"verimg", 0xE772, int_s390_verimg, v128g, v128g, 3>;
+
+  // Element shift left (with vector shift amount).
+  def VESLVB : BinaryVRRc<"veslvb", 0xE770, z_vshl, v128b, v128b, 0>;
+  def VESLVH : BinaryVRRc<"veslvh", 0xE770, z_vshl, v128h, v128h, 1>;
+  def VESLVF : BinaryVRRc<"veslvf", 0xE770, z_vshl, v128f, v128f, 2>;
+  def VESLVG : BinaryVRRc<"veslvg", 0xE770, z_vshl, v128g, v128g, 3>;
+
+  // Element shift left (with scalar shift amount).
+  def VESLB : BinaryVRSa<"veslb", 0xE730, z_vshl_by_scalar, v128b, v128b, 0>;
+  def VESLH : BinaryVRSa<"veslh", 0xE730, z_vshl_by_scalar, v128h, v128h, 1>;
+  def VESLF : BinaryVRSa<"veslf", 0xE730, z_vshl_by_scalar, v128f, v128f, 2>;
+  def VESLG : BinaryVRSa<"veslg", 0xE730, z_vshl_by_scalar, v128g, v128g, 3>;
+
+  // Element shift right arithmetic (with vector shift amount).
+  def VESRAVB : BinaryVRRc<"vesravb", 0xE77A, z_vsra, v128b, v128b, 0>;
+  def VESRAVH : BinaryVRRc<"vesravh", 0xE77A, z_vsra, v128h, v128h, 1>;
+  def VESRAVF : BinaryVRRc<"vesravf", 0xE77A, z_vsra, v128f, v128f, 2>;
+  def VESRAVG : BinaryVRRc<"vesravg", 0xE77A, z_vsra, v128g, v128g, 3>;
+
+  // Element shift right arithmetic (with scalar shift amount).
+  def VESRAB : BinaryVRSa<"vesrab", 0xE73A, z_vsra_by_scalar, v128b, v128b, 0>;
+  def VESRAH : BinaryVRSa<"vesrah", 0xE73A, z_vsra_by_scalar, v128h, v128h, 1>;
+  def VESRAF : BinaryVRSa<"vesraf", 0xE73A, z_vsra_by_scalar, v128f, v128f, 2>;
+  def VESRAG : BinaryVRSa<"vesrag", 0xE73A, z_vsra_by_scalar, v128g, v128g, 3>;
+
+  // Element shift right logical (with vector shift amount).
+  def VESRLVB : BinaryVRRc<"vesrlvb", 0xE778, z_vsrl, v128b, v128b, 0>;
+  def VESRLVH : BinaryVRRc<"vesrlvh", 0xE778, z_vsrl, v128h, v128h, 1>;
+  def VESRLVF : BinaryVRRc<"vesrlvf", 0xE778, z_vsrl, v128f, v128f, 2>;
+  def VESRLVG : BinaryVRRc<"vesrlvg", 0xE778, z_vsrl, v128g, v128g, 3>;
+
+  // Element shift right logical (with scalar shift amount).
+  def VESRLB : BinaryVRSa<"vesrlb", 0xE738, z_vsrl_by_scalar, v128b, v128b, 0>;
+  def VESRLH : BinaryVRSa<"vesrlh", 0xE738, z_vsrl_by_scalar, v128h, v128h, 1>;
+  def VESRLF : BinaryVRSa<"vesrlf", 0xE738, z_vsrl_by_scalar, v128f, v128f, 2>;
+  def VESRLG : BinaryVRSa<"vesrlg", 0xE738, z_vsrl_by_scalar, v128g, v128g, 3>;
+
+  // Shift left.
+  def VSL : BinaryVRRc<"vsl", 0xE774, int_s390_vsl, v128b, v128b>;
+
+  // Shift left by byte.
+  def VSLB : BinaryVRRc<"vslb", 0xE775, int_s390_vslb, v128b, v128b>;
+
+  // Shift left double by byte.
+  def VSLDB : TernaryVRId<"vsldb", 0xE777, z_shl_double, v128b, v128b, 0>;
+  def : Pat<(int_s390_vsldb VR128:$x, VR128:$y, imm32zx8:$z),
+            (VSLDB VR128:$x, VR128:$y, imm32zx8:$z)>;
+
+  // Shift right arithmetic.
+  def VSRA : BinaryVRRc<"vsra", 0xE77E, int_s390_vsra, v128b, v128b>;
+
+  // Shift right arithmetic by byte.
+  def VSRAB : BinaryVRRc<"vsrab", 0xE77F, int_s390_vsrab, v128b, v128b>;
+
+  // Shift right logical.
+  def VSRL : BinaryVRRc<"vsrl", 0xE77C, int_s390_vsrl, v128b, v128b>;
+
+  // Shift right logical by byte.
+  def VSRLB : BinaryVRRc<"vsrlb", 0xE77D, int_s390_vsrlb, v128b, v128b>;
+
+  // Subtract.
+  def VSB : BinaryVRRc<"vsb", 0xE7F7, sub, v128b, v128b, 0>;
+  def VSH : BinaryVRRc<"vsh", 0xE7F7, sub, v128h, v128h, 1>;
+  def VSF : BinaryVRRc<"vsf", 0xE7F7, sub, v128f, v128f, 2>;
+  def VSG : BinaryVRRc<"vsg", 0xE7F7, sub, v128g, v128g, 3>;
+  def VSQ : BinaryVRRc<"vsq", 0xE7F7, int_s390_vsq, v128q, v128q, 4>;
+
+  // Subtract compute borrow indication.
+  def VSCBIB : BinaryVRRc<"vscbib", 0xE7F5, int_s390_vscbib, v128b, v128b, 0>;
+  def VSCBIH : BinaryVRRc<"vscbih", 0xE7F5, int_s390_vscbih, v128h, v128h, 1>;
+  def VSCBIF : BinaryVRRc<"vscbif", 0xE7F5, int_s390_vscbif, v128f, v128f, 2>;
+  def VSCBIG : BinaryVRRc<"vscbig", 0xE7F5, int_s390_vscbig, v128g, v128g, 3>;
+  def VSCBIQ : BinaryVRRc<"vscbiq", 0xE7F5, int_s390_vscbiq, v128q, v128q, 4>;
+
+  // Subtract with borrow indication.
+  def VSBIQ : TernaryVRRd<"vsbiq", 0xE7BF, int_s390_vsbiq, v128q, v128q, 4>;
+
+  // Subtract with borrow compute borrow indication.
+  def VSBCBIQ : TernaryVRRd<"vsbcbiq", 0xE7BD, int_s390_vsbcbiq,
+                            v128q, v128q, 4>;
+
+  // Sum across doubleword.
+  def VSUMGH : BinaryVRRc<"vsumgh", 0xE765, z_vsum, v128g, v128h, 1>;
+  def VSUMGF : BinaryVRRc<"vsumgf", 0xE765, z_vsum, v128g, v128f, 2>;
+
+  // Sum across quadword.
+  def VSUMQF : BinaryVRRc<"vsumqf", 0xE767, z_vsum, v128q, v128f, 2>;
+  def VSUMQG : BinaryVRRc<"vsumqg", 0xE767, z_vsum, v128q, v128g, 3>;
+
+  // Sum across word.
+  def VSUMB : BinaryVRRc<"vsumb", 0xE764, z_vsum, v128f, v128b, 0>;
+  def VSUMH : BinaryVRRc<"vsumh", 0xE764, z_vsum, v128f, v128h, 1>;
+}
+
+// Instantiate the bitwise ops for type TYPE.
+multiclass BitwiseVectorOps<ValueType type> {
+  let Predicates = [FeatureVector] in {
+    def : Pat<(type (and VR128:$x, VR128:$y)), (VN VR128:$x, VR128:$y)>;
+    def : Pat<(type (and VR128:$x, (z_vnot VR128:$y))),
+              (VNC VR128:$x, VR128:$y)>;
+    def : Pat<(type (or VR128:$x, VR128:$y)), (VO VR128:$x, VR128:$y)>;
+    def : Pat<(type (xor VR128:$x, VR128:$y)), (VX VR128:$x, VR128:$y)>;
+    def : Pat<(type (or (and VR128:$x, VR128:$z),
+                        (and VR128:$y, (z_vnot VR128:$z)))),
+              (VSEL VR128:$x, VR128:$y, VR128:$z)>;
+    def : Pat<(type (z_vnot (or VR128:$x, VR128:$y))),
+              (VNO VR128:$x, VR128:$y)>;
+    def : Pat<(type (z_vnot VR128:$x)), (VNO VR128:$x, VR128:$x)>;
+  }
+}
+
+defm : BitwiseVectorOps<v16i8>;
+defm : BitwiseVectorOps<v8i16>;
+defm : BitwiseVectorOps<v4i32>;
+defm : BitwiseVectorOps<v2i64>;
+
+// Instantiate additional patterns for absolute-related expressions on
+// type TYPE.  LC is the negate instruction for TYPE and LP is the absolute
+// instruction.
+multiclass IntegerAbsoluteVectorOps<ValueType type, Instruction lc,
+                                    Instruction lp, int shift> {
+  let Predicates = [FeatureVector] in {
+    def : Pat<(type (vselect (type (z_vicmph_zero VR128:$x)),
+                             (z_vneg VR128:$x), VR128:$x)),
+              (lc (lp VR128:$x))>;
+    def : Pat<(type (vselect (type (z_vnot (z_vicmph_zero VR128:$x))),
+                             VR128:$x, (z_vneg VR128:$x))),
+              (lc (lp VR128:$x))>;
+    def : Pat<(type (vselect (type (z_vicmpl_zero VR128:$x)),
+                             VR128:$x, (z_vneg VR128:$x))),
+              (lc (lp VR128:$x))>;
+    def : Pat<(type (vselect (type (z_vnot (z_vicmpl_zero VR128:$x))),
+                             (z_vneg VR128:$x), VR128:$x)),
+              (lc (lp VR128:$x))>;
+    def : Pat<(type (or (and (z_vsra_by_scalar VR128:$x, (i32 shift)),
+                             (z_vneg VR128:$x)),
+                        (and (z_vnot (z_vsra_by_scalar VR128:$x, (i32 shift))),
+                             VR128:$x))),
+              (lp VR128:$x)>;
+    def : Pat<(type (or (and (z_vsra_by_scalar VR128:$x, (i32 shift)),
+                             VR128:$x),
+                        (and (z_vnot (z_vsra_by_scalar VR128:$x, (i32 shift))),
+                             (z_vneg VR128:$x)))),
+              (lc (lp VR128:$x))>;
+  }
+}
+
+defm : IntegerAbsoluteVectorOps<v16i8, VLCB, VLPB, 7>;
+defm : IntegerAbsoluteVectorOps<v8i16, VLCH, VLPH, 15>;
+defm : IntegerAbsoluteVectorOps<v4i32, VLCF, VLPF, 31>;
+defm : IntegerAbsoluteVectorOps<v2i64, VLCG, VLPG, 63>;
+
+// Instantiate minimum- and maximum-related patterns for TYPE.  CMPH is the
+// signed or unsigned "set if greater than" comparison instruction and
+// MIN and MAX are the associated minimum and maximum instructions.
+multiclass IntegerMinMaxVectorOps<ValueType type, SDPatternOperator cmph,
+                                  Instruction min, Instruction max> {
+  let Predicates = [FeatureVector] in {
+    def : Pat<(type (vselect (cmph VR128:$x, VR128:$y), VR128:$x, VR128:$y)),
+              (max VR128:$x, VR128:$y)>;
+    def : Pat<(type (vselect (cmph VR128:$x, VR128:$y), VR128:$y, VR128:$x)),
+              (min VR128:$x, VR128:$y)>;
+    def : Pat<(type (vselect (z_vnot (cmph VR128:$x, VR128:$y)),
+                             VR128:$x, VR128:$y)),
+              (min VR128:$x, VR128:$y)>;
+    def : Pat<(type (vselect (z_vnot (cmph VR128:$x, VR128:$y)),
+                             VR128:$y, VR128:$x)),
+              (max VR128:$x, VR128:$y)>;
+  }
+}
+
+// Signed min/max.
+defm : IntegerMinMaxVectorOps<v16i8, z_vicmph, VMNB, VMXB>;
+defm : IntegerMinMaxVectorOps<v8i16, z_vicmph, VMNH, VMXH>;
+defm : IntegerMinMaxVectorOps<v4i32, z_vicmph, VMNF, VMXF>;
+defm : IntegerMinMaxVectorOps<v2i64, z_vicmph, VMNG, VMXG>;
+
+// Unsigned min/max.
+defm : IntegerMinMaxVectorOps<v16i8, z_vicmphl, VMNLB, VMXLB>;
+defm : IntegerMinMaxVectorOps<v8i16, z_vicmphl, VMNLH, VMXLH>;
+defm : IntegerMinMaxVectorOps<v4i32, z_vicmphl, VMNLF, VMXLF>;
+defm : IntegerMinMaxVectorOps<v2i64, z_vicmphl, VMNLG, VMXLG>;
+
+//===----------------------------------------------------------------------===//
+// Integer comparison
+//===----------------------------------------------------------------------===//
+
+let Predicates = [FeatureVector] in {
+  // Element compare.
+  let Defs = [CC] in {
+    def VECB : CompareVRRa<"vecb", 0xE7DB, null_frag, v128b, 0>;
+    def VECH : CompareVRRa<"vech", 0xE7DB, null_frag, v128h, 1>;
+    def VECF : CompareVRRa<"vecf", 0xE7DB, null_frag, v128f, 2>;
+    def VECG : CompareVRRa<"vecg", 0xE7DB, null_frag, v128g, 3>;
+  }
+
+  // Element compare logical.
+  let Defs = [CC] in {
+    def VECLB : CompareVRRa<"veclb", 0xE7D9, null_frag, v128b, 0>;
+    def VECLH : CompareVRRa<"veclh", 0xE7D9, null_frag, v128h, 1>;
+    def VECLF : CompareVRRa<"veclf", 0xE7D9, null_frag, v128f, 2>;
+    def VECLG : CompareVRRa<"veclg", 0xE7D9, null_frag, v128g, 3>;
+  }
+
+  // Compare equal.
+  defm VCEQB : BinaryVRRbSPair<"vceqb", 0xE7F8, z_vicmpe, z_vicmpes,
+                               v128b, v128b, 0>;
+  defm VCEQH : BinaryVRRbSPair<"vceqh", 0xE7F8, z_vicmpe, z_vicmpes,
+                               v128h, v128h, 1>;
+  defm VCEQF : BinaryVRRbSPair<"vceqf", 0xE7F8, z_vicmpe, z_vicmpes,
+                               v128f, v128f, 2>;
+  defm VCEQG : BinaryVRRbSPair<"vceqg", 0xE7F8, z_vicmpe, z_vicmpes,
+                               v128g, v128g, 3>;
+
+  // Compare high.
+  defm VCHB : BinaryVRRbSPair<"vchb", 0xE7FB, z_vicmph, z_vicmphs,
+                              v128b, v128b, 0>;
+  defm VCHH : BinaryVRRbSPair<"vchh", 0xE7FB, z_vicmph, z_vicmphs,
+                              v128h, v128h, 1>;
+  defm VCHF : BinaryVRRbSPair<"vchf", 0xE7FB, z_vicmph, z_vicmphs,
+                              v128f, v128f, 2>;
+  defm VCHG : BinaryVRRbSPair<"vchg", 0xE7FB, z_vicmph, z_vicmphs,
+                              v128g, v128g, 3>;
+
+  // Compare high logical.
+  defm VCHLB : BinaryVRRbSPair<"vchlb", 0xE7F9, z_vicmphl, z_vicmphls,
+                               v128b, v128b, 0>;
+  defm VCHLH : BinaryVRRbSPair<"vchlh", 0xE7F9, z_vicmphl, z_vicmphls,
+                               v128h, v128h, 1>;
+  defm VCHLF : BinaryVRRbSPair<"vchlf", 0xE7F9, z_vicmphl, z_vicmphls,
+                               v128f, v128f, 2>;
+  defm VCHLG : BinaryVRRbSPair<"vchlg", 0xE7F9, z_vicmphl, z_vicmphls,
+                               v128g, v128g, 3>;
+
+  // Test under mask.
+  let Defs = [CC] in
+    def VTM : CompareVRRa<"vtm", 0xE7D8, z_vtm, v128b, 0>;
+}
+
+//===----------------------------------------------------------------------===//
+// Floating-point arithmetic
+//===----------------------------------------------------------------------===//
+
+// See comments in SystemZInstrFP.td for the suppression flags and
+// rounding modes.
+multiclass VectorRounding<Instruction insn, TypedReg tr> {
+  def : FPConversion<insn, frint,      tr, tr, 0, 0>;
+  def : FPConversion<insn, fnearbyint, tr, tr, 4, 0>;
+  def : FPConversion<insn, ffloor,     tr, tr, 4, 7>;
+  def : FPConversion<insn, fceil,      tr, tr, 4, 6>;
+  def : FPConversion<insn, ftrunc,     tr, tr, 4, 5>;
+  def : FPConversion<insn, frnd,       tr, tr, 4, 1>;
+}
+
+let Predicates = [FeatureVector] in {
+  // Add.
+  def VFADB : BinaryVRRc<"vfadb", 0xE7E3, fadd, v128db, v128db, 3, 0>;
+  def WFADB : BinaryVRRc<"wfadb", 0xE7E3, fadd, v64db, v64db, 3, 8>;
+
+  // Convert from fixed 64-bit.
+  def VCDGB : TernaryVRRa<"vcdgb", 0xE7C3, null_frag, v128db, v128g, 3, 0>;
+  def WCDGB : TernaryVRRa<"wcdgb", 0xE7C3, null_frag, v64db, v64g, 3, 8>;
+  def : FPConversion<VCDGB, sint_to_fp, v128db, v128g, 0, 0>;
+
+  // Convert from logical 64-bit.
+  def VCDLGB : TernaryVRRa<"vcdlgb", 0xE7C1, null_frag, v128db, v128g, 3, 0>;
+  def WCDLGB : TernaryVRRa<"wcdlgb", 0xE7C1, null_frag, v64db, v64g, 3, 8>;
+  def : FPConversion<VCDLGB, uint_to_fp, v128db, v128g, 0, 0>;
+
+  // Convert to fixed 64-bit.
+  def VCGDB : TernaryVRRa<"vcgdb", 0xE7C2, null_frag, v128g, v128db, 3, 0>;
+  def WCGDB : TernaryVRRa<"wcgdb", 0xE7C2, null_frag, v64g, v64db, 3, 8>;
+  // Rounding mode should agree with SystemZInstrFP.td.
+  def : FPConversion<VCGDB, fp_to_sint, v128g, v128db, 0, 5>;
+
+  // Convert to logical 64-bit.
+  def VCLGDB : TernaryVRRa<"vclgdb", 0xE7C0, null_frag, v128g, v128db, 3, 0>;
+  def WCLGDB : TernaryVRRa<"wclgdb", 0xE7C0, null_frag, v64g, v64db, 3, 8>;
+  // Rounding mode should agree with SystemZInstrFP.td.
+  def : FPConversion<VCLGDB, fp_to_uint, v128g, v128db, 0, 5>;
+
+  // Divide.
+  def VFDDB : BinaryVRRc<"vfddb", 0xE7E5, fdiv, v128db, v128db, 3, 0>;
+  def WFDDB : BinaryVRRc<"wfddb", 0xE7E5, fdiv, v64db, v64db, 3, 8>;
+
+  // Load FP integer.
+  def VFIDB : TernaryVRRa<"vfidb", 0xE7C7, int_s390_vfidb, v128db, v128db, 3, 0>;
+  def WFIDB : TernaryVRRa<"wfidb", 0xE7C7, null_frag, v64db, v64db, 3, 8>;
+  defm : VectorRounding<VFIDB, v128db>;
+  defm : VectorRounding<WFIDB, v64db>;
+
+  // Load lengthened.
+  def VLDEB : UnaryVRRa<"vldeb", 0xE7C4, z_vextend, v128db, v128eb, 2, 0>;
+  def WLDEB : UnaryVRRa<"wldeb", 0xE7C4, fextend, v64db, v32eb, 2, 8>;
+
+  // Load rounded,
+  def VLEDB : TernaryVRRa<"vledb", 0xE7C5, null_frag, v128eb, v128db, 3, 0>;
+  def WLEDB : TernaryVRRa<"wledb", 0xE7C5, null_frag, v32eb, v64db, 3, 8>;
+  def : Pat<(v4f32 (z_vround (v2f64 VR128:$src))), (VLEDB VR128:$src, 0, 0)>;
+  def : FPConversion<WLEDB, fround, v32eb, v64db, 0, 0>;
+
+  // Multiply.
+  def VFMDB : BinaryVRRc<"vfmdb", 0xE7E7, fmul, v128db, v128db, 3, 0>;
+  def WFMDB : BinaryVRRc<"wfmdb", 0xE7E7, fmul, v64db, v64db, 3, 8>;
+
+  // Multiply and add.
+  def VFMADB : TernaryVRRe<"vfmadb", 0xE78F, fma, v128db, v128db, 0, 3>;
+  def WFMADB : TernaryVRRe<"wfmadb", 0xE78F, fma, v64db, v64db, 8, 3>;
+
+  // Multiply and subtract.
+  def VFMSDB : TernaryVRRe<"vfmsdb", 0xE78E, fms, v128db, v128db, 0, 3>;
+  def WFMSDB : TernaryVRRe<"wfmsdb", 0xE78E, fms, v64db, v64db, 8, 3>;
+
+  // Load complement,
+  def VFLCDB : UnaryVRRa<"vflcdb", 0xE7CC, fneg, v128db, v128db, 3, 0, 0>;
+  def WFLCDB : UnaryVRRa<"wflcdb", 0xE7CC, fneg, v64db, v64db, 3, 8, 0>;
+
+  // Load negative.
+  def VFLNDB : UnaryVRRa<"vflndb", 0xE7CC, fnabs, v128db, v128db, 3, 0, 1>;
+  def WFLNDB : UnaryVRRa<"wflndb", 0xE7CC, fnabs, v64db, v64db, 3, 8, 1>;
+
+  // Load positive.
+  def VFLPDB : UnaryVRRa<"vflpdb", 0xE7CC, fabs, v128db, v128db, 3, 0, 2>;
+  def WFLPDB : UnaryVRRa<"wflpdb", 0xE7CC, fabs, v64db, v64db, 3, 8, 2>;
+
+  // Square root.
+  def VFSQDB : UnaryVRRa<"vfsqdb", 0xE7CE, fsqrt, v128db, v128db, 3, 0>;
+  def WFSQDB : UnaryVRRa<"wfsqdb", 0xE7CE, fsqrt, v64db, v64db, 3, 8>;
+
+  // Subtract.
+  def VFSDB : BinaryVRRc<"vfsdb", 0xE7E2, fsub, v128db, v128db, 3, 0>;
+  def WFSDB : BinaryVRRc<"wfsdb", 0xE7E2, fsub, v64db, v64db, 3, 8>;
+
+  // Test data class immediate.
+  let Defs = [CC] in {
+    def VFTCIDB : BinaryVRIe<"vftcidb", 0xE74A, z_vftci, v128g, v128db, 3, 0>;
+    def WFTCIDB : BinaryVRIe<"wftcidb", 0xE74A, null_frag, v64g, v64db, 3, 8>;
+  }
+}
+
+//===----------------------------------------------------------------------===//
+// Floating-point comparison
+//===----------------------------------------------------------------------===//
+
+let Predicates = [FeatureVector] in {
+  // Compare scalar.
+  let Defs = [CC] in
+    def WFCDB : CompareVRRa<"wfcdb", 0xE7CB, z_fcmp, v64db, 3>;
+
+  // Compare and signal scalar.
+  let Defs = [CC] in
+    def WFKDB : CompareVRRa<"wfkdb", 0xE7CA, null_frag, v64db, 3>;
+
+  // Compare equal.
+  defm VFCEDB : BinaryVRRcSPair<"vfcedb", 0xE7E8, z_vfcmpe, z_vfcmpes,
+                                v128g, v128db, 3, 0>;
+  defm WFCEDB : BinaryVRRcSPair<"wfcedb", 0xE7E8, null_frag, null_frag,
+                                v64g, v64db, 3, 8>;
+
+  // Compare high.
+  defm VFCHDB : BinaryVRRcSPair<"vfchdb", 0xE7EB, z_vfcmph, z_vfcmphs,
+                                v128g, v128db, 3, 0>;
+  defm WFCHDB : BinaryVRRcSPair<"wfchdb", 0xE7EB, null_frag, null_frag,
+                                v64g, v64db, 3, 8>;
+
+  // Compare high or equal.
+  defm VFCHEDB : BinaryVRRcSPair<"vfchedb", 0xE7EA, z_vfcmphe, z_vfcmphes,
+                                 v128g, v128db, 3, 0>;
+  defm WFCHEDB : BinaryVRRcSPair<"wfchedb", 0xE7EA, null_frag, null_frag,
+                                 v64g, v64db, 3, 8>;
+}
+
+//===----------------------------------------------------------------------===//
+// Conversions
+//===----------------------------------------------------------------------===//
+
+def : Pat<(v16i8 (bitconvert (v8i16 VR128:$src))), (v16i8 VR128:$src)>;
+def : Pat<(v16i8 (bitconvert (v4i32 VR128:$src))), (v16i8 VR128:$src)>;
+def : Pat<(v16i8 (bitconvert (v2i64 VR128:$src))), (v16i8 VR128:$src)>;
+def : Pat<(v16i8 (bitconvert (v4f32 VR128:$src))), (v16i8 VR128:$src)>;
+def : Pat<(v16i8 (bitconvert (v2f64 VR128:$src))), (v16i8 VR128:$src)>;
+
+def : Pat<(v8i16 (bitconvert (v16i8 VR128:$src))), (v8i16 VR128:$src)>;
+def : Pat<(v8i16 (bitconvert (v4i32 VR128:$src))), (v8i16 VR128:$src)>;
+def : Pat<(v8i16 (bitconvert (v2i64 VR128:$src))), (v8i16 VR128:$src)>;
+def : Pat<(v8i16 (bitconvert (v4f32 VR128:$src))), (v8i16 VR128:$src)>;
+def : Pat<(v8i16 (bitconvert (v2f64 VR128:$src))), (v8i16 VR128:$src)>;
+
+def : Pat<(v4i32 (bitconvert (v16i8 VR128:$src))), (v4i32 VR128:$src)>;
+def : Pat<(v4i32 (bitconvert (v8i16 VR128:$src))), (v4i32 VR128:$src)>;
+def : Pat<(v4i32 (bitconvert (v2i64 VR128:$src))), (v4i32 VR128:$src)>;
+def : Pat<(v4i32 (bitconvert (v4f32 VR128:$src))), (v4i32 VR128:$src)>;
+def : Pat<(v4i32 (bitconvert (v2f64 VR128:$src))), (v4i32 VR128:$src)>;
+
+def : Pat<(v2i64 (bitconvert (v16i8 VR128:$src))), (v2i64 VR128:$src)>;
+def : Pat<(v2i64 (bitconvert (v8i16 VR128:$src))), (v2i64 VR128:$src)>;
+def : Pat<(v2i64 (bitconvert (v4i32 VR128:$src))), (v2i64 VR128:$src)>;
+def : Pat<(v2i64 (bitconvert (v4f32 VR128:$src))), (v2i64 VR128:$src)>;
+def : Pat<(v2i64 (bitconvert (v2f64 VR128:$src))), (v2i64 VR128:$src)>;
+
+def : Pat<(v4f32 (bitconvert (v16i8 VR128:$src))), (v4f32 VR128:$src)>;
+def : Pat<(v4f32 (bitconvert (v8i16 VR128:$src))), (v4f32 VR128:$src)>;
+def : Pat<(v4f32 (bitconvert (v4i32 VR128:$src))), (v4f32 VR128:$src)>;
+def : Pat<(v4f32 (bitconvert (v2i64 VR128:$src))), (v4f32 VR128:$src)>;
+def : Pat<(v4f32 (bitconvert (v2f64 VR128:$src))), (v4f32 VR128:$src)>;
+
+def : Pat<(v2f64 (bitconvert (v16i8 VR128:$src))), (v2f64 VR128:$src)>;
+def : Pat<(v2f64 (bitconvert (v8i16 VR128:$src))), (v2f64 VR128:$src)>;
+def : Pat<(v2f64 (bitconvert (v4i32 VR128:$src))), (v2f64 VR128:$src)>;
+def : Pat<(v2f64 (bitconvert (v2i64 VR128:$src))), (v2f64 VR128:$src)>;
+def : Pat<(v2f64 (bitconvert (v4f32 VR128:$src))), (v2f64 VR128:$src)>;
+
+//===----------------------------------------------------------------------===//
+// Replicating scalars
+//===----------------------------------------------------------------------===//
+
+// Define patterns for replicating a scalar GR32 into a vector of type TYPE.
+// INDEX is 8 minus the element size in bytes.
+class VectorReplicateScalar<ValueType type, Instruction insn, bits<16> index>
+  : Pat<(type (z_replicate GR32:$scalar)),
+        (insn (VLVGP32 GR32:$scalar, GR32:$scalar), index)>;
+
+def : VectorReplicateScalar<v16i8, VREPB, 7>;
+def : VectorReplicateScalar<v8i16, VREPH, 3>;
+def : VectorReplicateScalar<v4i32, VREPF, 1>;
+
+// i64 replications are just a single isntruction.
+def : Pat<(v2i64 (z_replicate GR64:$scalar)),
+          (VLVGP GR64:$scalar, GR64:$scalar)>;
+
+//===----------------------------------------------------------------------===//
+// Floating-point insertion and extraction
+//===----------------------------------------------------------------------===//
+
+// Moving 32-bit values between GPRs and FPRs can be done using VLVGF
+// and VLGVF.
+def LEFR : UnaryAliasVRS<VR32, GR32>;
+def LFER : UnaryAliasVRS<GR64, VR32>;
+def : Pat<(f32 (bitconvert (i32 GR32:$src))), (LEFR GR32:$src)>;
+def : Pat<(i32 (bitconvert (f32 VR32:$src))),
+          (EXTRACT_SUBREG (LFER VR32:$src), subreg_l32)>;
+
+// Floating-point values are stored in element 0 of the corresponding
+// vector register.  Scalar to vector conversion is just a subreg and
+// scalar replication can just replicate element 0 of the vector register.
+multiclass ScalarToVectorFP<Instruction vrep, ValueType vt, RegisterOperand cls,
+                            SubRegIndex subreg> {
+  def : Pat<(vt (scalar_to_vector cls:$scalar)),
+            (INSERT_SUBREG (vt (IMPLICIT_DEF)), cls:$scalar, subreg)>;
+  def : Pat<(vt (z_replicate cls:$scalar)),
+            (vrep (INSERT_SUBREG (vt (IMPLICIT_DEF)), cls:$scalar,
+                                 subreg), 0)>;
+}
+defm : ScalarToVectorFP<VREPF, v4f32, FP32, subreg_r32>;
+defm : ScalarToVectorFP<VREPG, v2f64, FP64, subreg_r64>;
+
+// Match v2f64 insertions.  The AddedComplexity counters the 3 added by
+// TableGen for the base register operand in VLVG-based integer insertions
+// and ensures that this version is strictly better.
+let AddedComplexity = 4 in {
+  def : Pat<(z_vector_insert (v2f64 VR128:$vec), FP64:$elt, 0),
+            (VPDI (INSERT_SUBREG (v2f64 (IMPLICIT_DEF)), FP64:$elt,
+                                 subreg_r64), VR128:$vec, 1)>;
+  def : Pat<(z_vector_insert (v2f64 VR128:$vec), FP64:$elt, 1),
+            (VPDI VR128:$vec, (INSERT_SUBREG (v2f64 (IMPLICIT_DEF)), FP64:$elt,
+                                             subreg_r64), 0)>;
+}
+
+// We extract floating-point element X by replicating (for elements other
+// than 0) and then taking a high subreg.  The AddedComplexity counters the
+// 3 added by TableGen for the base register operand in VLGV-based integer
+// extractions and ensures that this version is strictly better.
+let AddedComplexity = 4 in {
+  def : Pat<(f32 (z_vector_extract (v4f32 VR128:$vec), 0)),
+            (EXTRACT_SUBREG VR128:$vec, subreg_r32)>;
+  def : Pat<(f32 (z_vector_extract (v4f32 VR128:$vec), imm32zx2:$index)),
+            (EXTRACT_SUBREG (VREPF VR128:$vec, imm32zx2:$index), subreg_r32)>;
+
+  def : Pat<(f64 (z_vector_extract (v2f64 VR128:$vec), 0)),
+            (EXTRACT_SUBREG VR128:$vec, subreg_r64)>;
+  def : Pat<(f64 (z_vector_extract (v2f64 VR128:$vec), imm32zx1:$index)),
+            (EXTRACT_SUBREG (VREPG VR128:$vec, imm32zx1:$index), subreg_r64)>;
+}
+
+//===----------------------------------------------------------------------===//
+// String instructions
+//===----------------------------------------------------------------------===//
+
+let Predicates = [FeatureVector] in {
+  defm VFAEB : TernaryVRRbSPair<"vfaeb", 0xE782, int_s390_vfaeb, z_vfae_cc,
+                                v128b, v128b, 0, 0>;
+  defm VFAEH : TernaryVRRbSPair<"vfaeh", 0xE782, int_s390_vfaeh, z_vfae_cc,
+                                v128h, v128h, 1, 0>;
+  defm VFAEF : TernaryVRRbSPair<"vfaef", 0xE782, int_s390_vfaef, z_vfae_cc,
+                                v128f, v128f, 2, 0>;
+  defm VFAEZB : TernaryVRRbSPair<"vfaezb", 0xE782, int_s390_vfaezb, z_vfaez_cc,
+                                 v128b, v128b, 0, 2>;
+  defm VFAEZH : TernaryVRRbSPair<"vfaezh", 0xE782, int_s390_vfaezh, z_vfaez_cc,
+                                 v128h, v128h, 1, 2>;
+  defm VFAEZF : TernaryVRRbSPair<"vfaezf", 0xE782, int_s390_vfaezf, z_vfaez_cc,
+                                 v128f, v128f, 2, 2>;
+
+  defm VFEEB : BinaryVRRbSPair<"vfeeb", 0xE780, int_s390_vfeeb, z_vfee_cc,
+                               v128b, v128b, 0, 0, 1>;
+  defm VFEEH : BinaryVRRbSPair<"vfeeh", 0xE780, int_s390_vfeeh, z_vfee_cc,
+                               v128h, v128h, 1, 0, 1>;
+  defm VFEEF : BinaryVRRbSPair<"vfeef", 0xE780, int_s390_vfeef, z_vfee_cc,
+                               v128f, v128f, 2, 0, 1>;
+  defm VFEEZB : BinaryVRRbSPair<"vfeezb", 0xE780, int_s390_vfeezb, z_vfeez_cc,
+                                v128b, v128b, 0, 2, 3>;
+  defm VFEEZH : BinaryVRRbSPair<"vfeezh", 0xE780, int_s390_vfeezh, z_vfeez_cc,
+                                v128h, v128h, 1, 2, 3>;
+  defm VFEEZF : BinaryVRRbSPair<"vfeezf", 0xE780, int_s390_vfeezf, z_vfeez_cc,
+                                v128f, v128f, 2, 2, 3>;
+
+  defm VFENEB : BinaryVRRbSPair<"vfeneb", 0xE781, int_s390_vfeneb, z_vfene_cc,
+                                v128b, v128b, 0, 0, 1>;
+  defm VFENEH : BinaryVRRbSPair<"vfeneh", 0xE781, int_s390_vfeneh, z_vfene_cc,
+                                v128h, v128h, 1, 0, 1>;
+  defm VFENEF : BinaryVRRbSPair<"vfenef", 0xE781, int_s390_vfenef, z_vfene_cc,
+                                v128f, v128f, 2, 0, 1>;
+  defm VFENEZB : BinaryVRRbSPair<"vfenezb", 0xE781, int_s390_vfenezb,
+                                 z_vfenez_cc, v128b, v128b, 0, 2, 3>;
+  defm VFENEZH : BinaryVRRbSPair<"vfenezh", 0xE781, int_s390_vfenezh,
+                                 z_vfenez_cc, v128h, v128h, 1, 2, 3>;
+  defm VFENEZF : BinaryVRRbSPair<"vfenezf", 0xE781, int_s390_vfenezf,
+                                 z_vfenez_cc, v128f, v128f, 2, 2, 3>;
+
+  defm VISTRB : UnaryVRRaSPair<"vistrb", 0xE75C, int_s390_vistrb, z_vistr_cc,
+                               v128b, v128b, 0>;
+  defm VISTRH : UnaryVRRaSPair<"vistrh", 0xE75C, int_s390_vistrh, z_vistr_cc,
+                               v128h, v128h, 1>;
+  defm VISTRF : UnaryVRRaSPair<"vistrf", 0xE75C, int_s390_vistrf, z_vistr_cc,
+                               v128f, v128f, 2>;
+
+  defm VSTRCB : QuaternaryVRRdSPair<"vstrcb", 0xE78A, int_s390_vstrcb,
+                                    z_vstrc_cc, v128b, v128b, 0, 0>;
+  defm VSTRCH : QuaternaryVRRdSPair<"vstrch", 0xE78A, int_s390_vstrch,
+                                    z_vstrc_cc, v128h, v128h, 1, 0>;
+  defm VSTRCF : QuaternaryVRRdSPair<"vstrcf", 0xE78A, int_s390_vstrcf,
+                                    z_vstrc_cc, v128f, v128f, 2, 0>;
+  defm VSTRCZB : QuaternaryVRRdSPair<"vstrczb", 0xE78A, int_s390_vstrczb,
+                                     z_vstrcz_cc, v128b, v128b, 0, 2>;
+  defm VSTRCZH : QuaternaryVRRdSPair<"vstrczh", 0xE78A, int_s390_vstrczh,
+                                     z_vstrcz_cc, v128h, v128h, 1, 2>;
+  defm VSTRCZF : QuaternaryVRRdSPair<"vstrczf", 0xE78A, int_s390_vstrczf,
+                                     z_vstrcz_cc, v128f, v128f, 2, 2>;
+}
diff --git a/lib/Target/SystemZ/SystemZMCInstLower.cpp b/lib/Target/SystemZ/SystemZMCInstLower.cpp
index 6bb96f1b40f..2655e4866b2 100644
--- a/lib/Target/SystemZ/SystemZMCInstLower.cpp
+++ b/lib/Target/SystemZ/SystemZMCInstLower.cpp
@@ -11,6 +11,7 @@
 #include "SystemZAsmPrinter.h"
 #include "llvm/IR/Mangler.h"
 #include "llvm/MC/MCExpr.h"
+#include "llvm/MC/MCInst.h"
 #include "llvm/MC/MCStreamer.h"
 
 using namespace llvm;
@@ -67,11 +68,11 @@ SystemZMCInstLower::getExpr(const MachineOperand &MO,
   default:
     llvm_unreachable("unknown operand type");
   }
-  const MCExpr *Expr = MCSymbolRefExpr::Create(Symbol, Kind, Ctx);
+  const MCExpr *Expr = MCSymbolRefExpr::create(Symbol, Kind, Ctx);
   if (HasOffset)
     if (int64_t Offset = MO.getOffset()) {
-      const MCExpr *OffsetExpr = MCConstantExpr::Create(Offset, Ctx);
-      Expr = MCBinaryExpr::CreateAdd(Expr, OffsetExpr, Ctx);
+      const MCExpr *OffsetExpr = MCConstantExpr::create(Offset, Ctx);
+      Expr = MCBinaryExpr::createAdd(Expr, OffsetExpr, Ctx);
     }
   return Expr;
 }
@@ -79,14 +80,14 @@ SystemZMCInstLower::getExpr(const MachineOperand &MO,
 MCOperand SystemZMCInstLower::lowerOperand(const MachineOperand &MO) const {
   switch (MO.getType()) {
   case MachineOperand::MO_Register:
-    return MCOperand::CreateReg(MO.getReg());
+    return MCOperand::createReg(MO.getReg());
 
   case MachineOperand::MO_Immediate:
-    return MCOperand::CreateImm(MO.getImm());
+    return MCOperand::createImm(MO.getImm());
 
   default: {
     MCSymbolRefExpr::VariantKind Kind = getVariantKind(MO.getTargetFlags());
-    return MCOperand::CreateExpr(getExpr(MO, Kind));
+    return MCOperand::createExpr(getExpr(MO, Kind));
   }
   }
 }
diff --git a/lib/Target/SystemZ/SystemZMachineFunctionInfo.cpp b/lib/Target/SystemZ/SystemZMachineFunctionInfo.cpp
index 00572d0b9d7..1a7c0d7f687 100644
--- a/lib/Target/SystemZ/SystemZMachineFunctionInfo.cpp
+++ b/lib/Target/SystemZ/SystemZMachineFunctionInfo.cpp
@@ -1,4 +1,4 @@
-//== SystemZMachineFuctionInfo.cpp - SystemZ machine function info-*- C++ -*-=//
+//=== SystemZMachineFunctionInfo.cpp - SystemZ machine function info ------===//
 //
 //                     The LLVM Compiler Infrastructure
 //
diff --git a/lib/Target/SystemZ/SystemZMachineFunctionInfo.h b/lib/Target/SystemZ/SystemZMachineFunctionInfo.h
index 34fc36d6bf6..f4a517bd54d 100644
--- a/lib/Target/SystemZ/SystemZMachineFunctionInfo.h
+++ b/lib/Target/SystemZ/SystemZMachineFunctionInfo.h
@@ -1,4 +1,4 @@
-//==- SystemZMachineFuctionInfo.h - SystemZ machine function info -*- C++ -*-=//
+//=== SystemZMachineFunctionInfo.h - SystemZ machine function info -*- C++ -*-//
 //
 //                     The LLVM Compiler Infrastructure
 //
diff --git a/lib/Target/SystemZ/SystemZOperands.td b/lib/Target/SystemZ/SystemZOperands.td
index 1b5b7d7bf32..9af90d492cf 100644
--- a/lib/Target/SystemZ/SystemZOperands.td
+++ b/lib/Target/SystemZ/SystemZOperands.td
@@ -78,6 +78,22 @@ class AddressAsmOperand<string format, string bitsize, string dispsize,
   let RenderMethod = "add"##format##"Operands";
 }
 
+// Constructs an instruction operand for an addressing mode.  FORMAT,
+// BITSIZE, DISPSIZE and LENGTH are the parameters to an associated
+// AddressAsmOperand.  OPERANDS is a list of individual operands
+// (base register, displacement, etc.).
+class AddressOperand<string bitsize, string dispsize, string length,
+                     string format, dag operands>
+  : Operand<!cast<ValueType>("i"##bitsize)> {
+  let PrintMethod = "print"##format##"Operand";
+  let EncoderMethod = "get"##format##dispsize##length##"Encoding";
+  let DecoderMethod =
+    "decode"##format##bitsize##"Disp"##dispsize##length##"Operand";
+  let MIOperandInfo = operands;
+  let ParserMatchClass =
+    !cast<AddressAsmOperand>(format##bitsize##"Disp"##dispsize##length);
+}
+
 // Constructs both a DAG pattern and instruction operand for an addressing mode.
 // FORMAT, BITSIZE, DISPSIZE and LENGTH are the parameters to an associated
 // AddressAsmOperand.  OPERANDS is a list of NUMOPS individual operands
@@ -93,15 +109,7 @@ class AddressingMode<string seltype, string bitsize, string dispsize,
   : ComplexPattern<!cast<ValueType>("i"##bitsize), numops,
                    "select"##seltype##dispsize##suffix##length,
                    [add, sub, or, frameindex, z_adjdynalloc]>,
-    Operand<!cast<ValueType>("i"##bitsize)> {
-  let PrintMethod = "print"##format##"Operand";
-  let EncoderMethod = "get"##format##dispsize##length##"Encoding";
-  let DecoderMethod =
-    "decode"##format##bitsize##"Disp"##dispsize##length##"Operand";
-  let MIOperandInfo = operands;
-  let ParserMatchClass =
-    !cast<AddressAsmOperand>(format##bitsize##"Disp"##dispsize##length);
-}
+    AddressOperand<bitsize, dispsize, length, format, operands>;
 
 // An addressing mode with a base and displacement but no index.
 class BDMode<string type, string bitsize, string dispsize, string suffix>
@@ -125,6 +133,13 @@ class BDLMode<string type, string bitsize, string dispsize, string suffix,
                         !cast<Immediate>("disp"##dispsize##"imm"##bitsize),
                         !cast<Immediate>("imm"##bitsize))>;
 
+// An addressing mode with a base, displacement and a vector index.
+class BDVMode<string bitsize, string dispsize>
+  : AddressOperand<bitsize, dispsize, "", "BDVAddr",
+                   (ops !cast<RegisterOperand>("ADDR"##bitsize),
+                        !cast<Immediate>("disp"##dispsize##"imm"##bitsize),
+                        !cast<RegisterOperand>("VR128"))>;
+
 //===----------------------------------------------------------------------===//
 // Extracting immediate operands from nodes
 // These all create MVT::i64 nodes to ensure the value is not sign-extended
@@ -134,82 +149,105 @@ class BDLMode<string type, string bitsize, string dispsize, string suffix,
 // Bits 0-15 (counting from the lsb).
 def LL16 : SDNodeXForm<imm, [{
   uint64_t Value = N->getZExtValue() & 0x000000000000FFFFULL;
-  return CurDAG->getTargetConstant(Value, MVT::i64);
+  return CurDAG->getTargetConstant(Value, SDLoc(N), MVT::i64);
 }]>;
 
 // Bits 16-31 (counting from the lsb).
 def LH16 : SDNodeXForm<imm, [{
   uint64_t Value = (N->getZExtValue() & 0x00000000FFFF0000ULL) >> 16;
-  return CurDAG->getTargetConstant(Value, MVT::i64);
+  return CurDAG->getTargetConstant(Value, SDLoc(N), MVT::i64);
 }]>;
 
 // Bits 32-47 (counting from the lsb).
 def HL16 : SDNodeXForm<imm, [{
   uint64_t Value = (N->getZExtValue() & 0x0000FFFF00000000ULL) >> 32;
-  return CurDAG->getTargetConstant(Value, MVT::i64);
+  return CurDAG->getTargetConstant(Value, SDLoc(N), MVT::i64);
 }]>;
 
 // Bits 48-63 (counting from the lsb).
 def HH16 : SDNodeXForm<imm, [{
   uint64_t Value = (N->getZExtValue() & 0xFFFF000000000000ULL) >> 48;
-  return CurDAG->getTargetConstant(Value, MVT::i64);
+  return CurDAG->getTargetConstant(Value, SDLoc(N), MVT::i64);
 }]>;
 
 // Low 32 bits.
 def LF32 : SDNodeXForm<imm, [{
   uint64_t Value = N->getZExtValue() & 0x00000000FFFFFFFFULL;
-  return CurDAG->getTargetConstant(Value, MVT::i64);
+  return CurDAG->getTargetConstant(Value, SDLoc(N), MVT::i64);
 }]>;
 
 // High 32 bits.
 def HF32 : SDNodeXForm<imm, [{
   uint64_t Value = N->getZExtValue() >> 32;
-  return CurDAG->getTargetConstant(Value, MVT::i64);
+  return CurDAG->getTargetConstant(Value, SDLoc(N), MVT::i64);
 }]>;
 
 // Truncate an immediate to a 8-bit signed quantity.
 def SIMM8 : SDNodeXForm<imm, [{
-  return CurDAG->getTargetConstant(int8_t(N->getZExtValue()), MVT::i64);
+  return CurDAG->getTargetConstant(int8_t(N->getZExtValue()), SDLoc(N),
+                                   MVT::i64);
 }]>;
 
 // Truncate an immediate to a 8-bit unsigned quantity.
 def UIMM8 : SDNodeXForm<imm, [{
-  return CurDAG->getTargetConstant(uint8_t(N->getZExtValue()), MVT::i64);
+  return CurDAG->getTargetConstant(uint8_t(N->getZExtValue()), SDLoc(N),
+                                   MVT::i64);
+}]>;
+
+// Truncate an immediate to a 8-bit unsigned quantity and mask off low bit.
+def UIMM8EVEN : SDNodeXForm<imm, [{
+  return CurDAG->getTargetConstant(N->getZExtValue() & 0xfe, SDLoc(N),
+                                   MVT::i64);
+}]>;
+
+// Truncate an immediate to a 12-bit unsigned quantity.
+def UIMM12 : SDNodeXForm<imm, [{
+  return CurDAG->getTargetConstant(N->getZExtValue() & 0xfff, SDLoc(N),
+                                   MVT::i64);
 }]>;
 
 // Truncate an immediate to a 16-bit signed quantity.
 def SIMM16 : SDNodeXForm<imm, [{
-  return CurDAG->getTargetConstant(int16_t(N->getZExtValue()), MVT::i64);
+  return CurDAG->getTargetConstant(int16_t(N->getZExtValue()), SDLoc(N),
+                                   MVT::i64);
 }]>;
 
 // Truncate an immediate to a 16-bit unsigned quantity.
 def UIMM16 : SDNodeXForm<imm, [{
-  return CurDAG->getTargetConstant(uint16_t(N->getZExtValue()), MVT::i64);
+  return CurDAG->getTargetConstant(uint16_t(N->getZExtValue()), SDLoc(N),
+                                   MVT::i64);
 }]>;
 
 // Truncate an immediate to a 32-bit signed quantity.
 def SIMM32 : SDNodeXForm<imm, [{
-  return CurDAG->getTargetConstant(int32_t(N->getZExtValue()), MVT::i64);
+  return CurDAG->getTargetConstant(int32_t(N->getZExtValue()), SDLoc(N),
+                                   MVT::i64);
 }]>;
 
 // Truncate an immediate to a 32-bit unsigned quantity.
 def UIMM32 : SDNodeXForm<imm, [{
-  return CurDAG->getTargetConstant(uint32_t(N->getZExtValue()), MVT::i64);
+  return CurDAG->getTargetConstant(uint32_t(N->getZExtValue()), SDLoc(N),
+                                   MVT::i64);
 }]>;
 
 // Negate and then truncate an immediate to a 32-bit unsigned quantity.
 def NEGIMM32 : SDNodeXForm<imm, [{
-  return CurDAG->getTargetConstant(uint32_t(-N->getZExtValue()), MVT::i64);
+  return CurDAG->getTargetConstant(uint32_t(-N->getZExtValue()), SDLoc(N),
+                                   MVT::i64);
 }]>;
 
 //===----------------------------------------------------------------------===//
 // Immediate asm operands.
 //===----------------------------------------------------------------------===//
 
+def U1Imm  : ImmediateAsmOperand<"U1Imm">;
+def U2Imm  : ImmediateAsmOperand<"U2Imm">;
+def U3Imm  : ImmediateAsmOperand<"U3Imm">;
 def U4Imm  : ImmediateAsmOperand<"U4Imm">;
 def U6Imm  : ImmediateAsmOperand<"U6Imm">;
 def S8Imm  : ImmediateAsmOperand<"S8Imm">;
 def U8Imm  : ImmediateAsmOperand<"U8Imm">;
+def U12Imm : ImmediateAsmOperand<"U12Imm">;
 def S16Imm : ImmediateAsmOperand<"S16Imm">;
 def U16Imm : ImmediateAsmOperand<"U16Imm">;
 def S32Imm : ImmediateAsmOperand<"S32Imm">;
@@ -240,10 +278,28 @@ def imm32lh16c : Immediate<i32, [{
 }], LH16, "U16Imm">;
 
 // Short immediates
+def imm32zx1 : Immediate<i32, [{
+  return isUInt<1>(N->getZExtValue());
+}], NOOP_SDNodeXForm, "U1Imm">;
+
+def imm32zx2 : Immediate<i32, [{
+  return isUInt<2>(N->getZExtValue());
+}], NOOP_SDNodeXForm, "U2Imm">;
+
+def imm32zx3 : Immediate<i32, [{
+  return isUInt<3>(N->getZExtValue());
+}], NOOP_SDNodeXForm, "U3Imm">;
+
 def imm32zx4 : Immediate<i32, [{
   return isUInt<4>(N->getZExtValue());
 }], NOOP_SDNodeXForm, "U4Imm">;
 
+// Note: this enforces an even value during code generation only.
+// When used from the assembler, any 4-bit value is allowed.
+def imm32zx4even : Immediate<i32, [{
+  return isUInt<4>(N->getZExtValue());
+}], UIMM8EVEN, "U4Imm">;
+
 def imm32zx6 : Immediate<i32, [{
   return isUInt<6>(N->getZExtValue());
 }], NOOP_SDNodeXForm, "U6Imm">;
@@ -258,6 +314,10 @@ def imm32zx8 : Immediate<i32, [{
 
 def imm32zx8trunc : Immediate<i32, [{}], UIMM8, "U8Imm">;
 
+def imm32zx12 : Immediate<i32, [{
+  return isUInt<12>(N->getZExtValue());
+}], UIMM12, "U12Imm">;
+
 def imm32sx16 : Immediate<i32, [{
   return isInt<16>(N->getSExtValue());
 }], SIMM16, "S16Imm">;
@@ -438,6 +498,7 @@ def BDAddr64Disp20      : AddressAsmOperand<"BDAddr",   "64", "20">;
 def BDXAddr64Disp12     : AddressAsmOperand<"BDXAddr",  "64", "12">;
 def BDXAddr64Disp20     : AddressAsmOperand<"BDXAddr",  "64", "20">;
 def BDLAddr64Disp12Len8 : AddressAsmOperand<"BDLAddr",  "64", "12", "Len8">;
+def BDVAddr64Disp12     : AddressAsmOperand<"BDVAddr",  "64", "12">;
 
 // DAG patterns and operands for addressing modes.  Each mode has
 // the form <type><range><group>[<len>] where:
@@ -450,6 +511,7 @@ def BDLAddr64Disp12Len8 : AddressAsmOperand<"BDLAddr",  "64", "12", "Len8">;
 //   laaddr   : like bdxaddr, but used for Load Address operations
 //   dynalloc : base + displacement + index + ADJDYNALLOC
 //   bdladdr  : base + displacement with a length field
+//   bdvaddr  : base + displacement with a vector index
 //
 // <range> is one of:
 //   12       : the displacement is an unsigned 12-bit value
@@ -482,6 +544,7 @@ def dynalloc12only    : BDXMode<"DynAlloc", "64", "12", "Only">;
 def laaddr12pair      : BDXMode<"LAAddr",   "64", "12", "Pair">;
 def laaddr20pair      : BDXMode<"LAAddr",   "64", "20", "Pair">;
 def bdladdr12onlylen8 : BDLMode<"BDLAddr",  "64", "12", "Only", "8">;
+def bdvaddr12only     : BDVMode<            "64", "12">;
 
 //===----------------------------------------------------------------------===//
 // Miscellaneous
diff --git a/lib/Target/SystemZ/SystemZOperators.td b/lib/Target/SystemZ/SystemZOperators.td
index 3151052ecf5..3c95a1e11b4 100644
--- a/lib/Target/SystemZ/SystemZOperators.td
+++ b/lib/Target/SystemZ/SystemZOperators.td
@@ -82,6 +82,61 @@ def SDT_ZPrefetch           : SDTypeProfile<0, 2,
 def SDT_ZTBegin             : SDTypeProfile<0, 2,
                                             [SDTCisPtrTy<0>,
                                              SDTCisVT<1, i32>]>;
+def SDT_ZInsertVectorElt    : SDTypeProfile<1, 3,
+                                            [SDTCisVec<0>,
+                                             SDTCisSameAs<0, 1>,
+                                             SDTCisVT<3, i32>]>;
+def SDT_ZExtractVectorElt   : SDTypeProfile<1, 2,
+                                            [SDTCisVec<1>,
+                                             SDTCisVT<2, i32>]>;
+def SDT_ZReplicate          : SDTypeProfile<1, 1,
+                                            [SDTCisVec<0>]>;
+def SDT_ZVecUnaryConv       : SDTypeProfile<1, 1,
+                                            [SDTCisVec<0>,
+                                             SDTCisVec<1>]>;
+def SDT_ZVecUnary           : SDTypeProfile<1, 1,
+                                            [SDTCisVec<0>,
+                                             SDTCisSameAs<0, 1>]>;
+def SDT_ZVecBinary          : SDTypeProfile<1, 2,
+                                            [SDTCisVec<0>,
+                                             SDTCisSameAs<0, 1>,
+                                             SDTCisSameAs<0, 2>]>;
+def SDT_ZVecBinaryInt       : SDTypeProfile<1, 2,
+                                            [SDTCisVec<0>,
+                                             SDTCisSameAs<0, 1>,
+                                             SDTCisVT<2, i32>]>;
+def SDT_ZVecBinaryConv      : SDTypeProfile<1, 2,
+                                            [SDTCisVec<0>,
+                                             SDTCisVec<1>,
+                                             SDTCisSameAs<1, 2>]>;
+def SDT_ZVecBinaryConvInt   : SDTypeProfile<1, 2,
+                                            [SDTCisVec<0>,
+                                             SDTCisVec<1>,
+                                             SDTCisVT<2, i32>]>;
+def SDT_ZRotateMask         : SDTypeProfile<1, 2,
+                                            [SDTCisVec<0>,
+                                             SDTCisVT<1, i32>,
+                                             SDTCisVT<2, i32>]>;
+def SDT_ZJoinDwords         : SDTypeProfile<1, 2,
+                                            [SDTCisVT<0, v2i64>,
+                                             SDTCisVT<1, i64>,
+                                             SDTCisVT<2, i64>]>;
+def SDT_ZVecTernary         : SDTypeProfile<1, 3,
+                                            [SDTCisVec<0>,
+                                             SDTCisSameAs<0, 1>,
+                                             SDTCisSameAs<0, 2>,
+                                             SDTCisSameAs<0, 3>]>;
+def SDT_ZVecTernaryInt      : SDTypeProfile<1, 3,
+                                            [SDTCisVec<0>,
+                                             SDTCisSameAs<0, 1>,
+                                             SDTCisSameAs<0, 2>,
+                                             SDTCisVT<3, i32>]>;
+def SDT_ZVecQuaternaryInt   : SDTypeProfile<1, 4,
+                                            [SDTCisVec<0>,
+                                             SDTCisSameAs<0, 1>,
+                                             SDTCisSameAs<0, 2>,
+                                             SDTCisSameAs<0, 3>,
+                                             SDTCisVT<4, i32>]>;
 
 //===----------------------------------------------------------------------===//
 // Node definitions
@@ -134,6 +189,80 @@ def z_udivrem64         : SDNode<"SystemZISD::UDIVREM64", SDT_ZGR128Binary64>;
 def z_serialize         : SDNode<"SystemZISD::SERIALIZE", SDTNone,
                                  [SDNPHasChain, SDNPMayStore]>;
 
+// Defined because the index is an i32 rather than a pointer.
+def z_vector_insert     : SDNode<"ISD::INSERT_VECTOR_ELT",
+                                 SDT_ZInsertVectorElt>;
+def z_vector_extract    : SDNode<"ISD::EXTRACT_VECTOR_ELT",
+                                 SDT_ZExtractVectorElt>;
+def z_byte_mask         : SDNode<"SystemZISD::BYTE_MASK", SDT_ZReplicate>;
+def z_rotate_mask       : SDNode<"SystemZISD::ROTATE_MASK", SDT_ZRotateMask>;
+def z_replicate         : SDNode<"SystemZISD::REPLICATE", SDT_ZReplicate>;
+def z_join_dwords       : SDNode<"SystemZISD::JOIN_DWORDS", SDT_ZJoinDwords>;
+def z_splat             : SDNode<"SystemZISD::SPLAT", SDT_ZVecBinaryInt>;
+def z_merge_high        : SDNode<"SystemZISD::MERGE_HIGH", SDT_ZVecBinary>;
+def z_merge_low         : SDNode<"SystemZISD::MERGE_LOW", SDT_ZVecBinary>;
+def z_shl_double        : SDNode<"SystemZISD::SHL_DOUBLE", SDT_ZVecTernaryInt>;
+def z_permute_dwords    : SDNode<"SystemZISD::PERMUTE_DWORDS",
+                                 SDT_ZVecTernaryInt>;
+def z_permute           : SDNode<"SystemZISD::PERMUTE", SDT_ZVecTernary>;
+def z_pack              : SDNode<"SystemZISD::PACK", SDT_ZVecBinaryConv>;
+def z_packs_cc          : SDNode<"SystemZISD::PACKS_CC", SDT_ZVecBinaryConv,
+                                 [SDNPOutGlue]>;
+def z_packls_cc         : SDNode<"SystemZISD::PACKLS_CC", SDT_ZVecBinaryConv,
+                                 [SDNPOutGlue]>;
+def z_unpack_high       : SDNode<"SystemZISD::UNPACK_HIGH", SDT_ZVecUnaryConv>;
+def z_unpackl_high      : SDNode<"SystemZISD::UNPACKL_HIGH", SDT_ZVecUnaryConv>;
+def z_unpack_low        : SDNode<"SystemZISD::UNPACK_LOW", SDT_ZVecUnaryConv>;
+def z_unpackl_low       : SDNode<"SystemZISD::UNPACKL_LOW", SDT_ZVecUnaryConv>;
+def z_vshl_by_scalar    : SDNode<"SystemZISD::VSHL_BY_SCALAR",
+                                 SDT_ZVecBinaryInt>;
+def z_vsrl_by_scalar    : SDNode<"SystemZISD::VSRL_BY_SCALAR",
+                                 SDT_ZVecBinaryInt>;
+def z_vsra_by_scalar    : SDNode<"SystemZISD::VSRA_BY_SCALAR",
+                                 SDT_ZVecBinaryInt>;
+def z_vsum              : SDNode<"SystemZISD::VSUM", SDT_ZVecBinaryConv>;
+def z_vicmpe            : SDNode<"SystemZISD::VICMPE", SDT_ZVecBinary>;
+def z_vicmph            : SDNode<"SystemZISD::VICMPH", SDT_ZVecBinary>;
+def z_vicmphl           : SDNode<"SystemZISD::VICMPHL", SDT_ZVecBinary>;
+def z_vicmpes           : SDNode<"SystemZISD::VICMPES", SDT_ZVecBinary,
+                                 [SDNPOutGlue]>;
+def z_vicmphs           : SDNode<"SystemZISD::VICMPHS", SDT_ZVecBinary,
+                                 [SDNPOutGlue]>;
+def z_vicmphls          : SDNode<"SystemZISD::VICMPHLS", SDT_ZVecBinary,
+                                 [SDNPOutGlue]>;
+def z_vfcmpe            : SDNode<"SystemZISD::VFCMPE", SDT_ZVecBinaryConv>;
+def z_vfcmph            : SDNode<"SystemZISD::VFCMPH", SDT_ZVecBinaryConv>;
+def z_vfcmphe           : SDNode<"SystemZISD::VFCMPHE", SDT_ZVecBinaryConv>;
+def z_vfcmpes           : SDNode<"SystemZISD::VFCMPES", SDT_ZVecBinaryConv,
+                                 [SDNPOutGlue]>;
+def z_vfcmphs           : SDNode<"SystemZISD::VFCMPHS", SDT_ZVecBinaryConv,
+                                 [SDNPOutGlue]>;
+def z_vfcmphes          : SDNode<"SystemZISD::VFCMPHES", SDT_ZVecBinaryConv,
+                                 [SDNPOutGlue]>;
+def z_vextend           : SDNode<"SystemZISD::VEXTEND", SDT_ZVecUnaryConv>;
+def z_vround            : SDNode<"SystemZISD::VROUND", SDT_ZVecUnaryConv>;
+def z_vtm               : SDNode<"SystemZISD::VTM", SDT_ZCmp, [SDNPOutGlue]>;
+def z_vfae_cc           : SDNode<"SystemZISD::VFAE_CC", SDT_ZVecTernaryInt,
+                                 [SDNPOutGlue]>;
+def z_vfaez_cc          : SDNode<"SystemZISD::VFAEZ_CC", SDT_ZVecTernaryInt,
+                                 [SDNPOutGlue]>;
+def z_vfee_cc           : SDNode<"SystemZISD::VFEE_CC", SDT_ZVecBinary,
+                                 [SDNPOutGlue]>;
+def z_vfeez_cc          : SDNode<"SystemZISD::VFEEZ_CC", SDT_ZVecBinary,
+                                 [SDNPOutGlue]>;
+def z_vfene_cc          : SDNode<"SystemZISD::VFENE_CC", SDT_ZVecBinary,
+                                 [SDNPOutGlue]>;
+def z_vfenez_cc         : SDNode<"SystemZISD::VFENEZ_CC", SDT_ZVecBinary,
+                                 [SDNPOutGlue]>;
+def z_vistr_cc          : SDNode<"SystemZISD::VISTR_CC", SDT_ZVecUnary,
+                                 [SDNPOutGlue]>;
+def z_vstrc_cc          : SDNode<"SystemZISD::VSTRC_CC", SDT_ZVecQuaternaryInt,
+                                 [SDNPOutGlue]>;
+def z_vstrcz_cc         : SDNode<"SystemZISD::VSTRCZ_CC",
+                                 SDT_ZVecQuaternaryInt, [SDNPOutGlue]>;
+def z_vftci             : SDNode<"SystemZISD::VFTCI", SDT_ZVecBinaryConvInt,
+                                 [SDNPOutGlue]>;
+
 class AtomicWOp<string name, SDTypeProfile profile = SDT_ZAtomicLoadBinaryW>
   : SDNode<"SystemZISD::"##name, profile,
            [SDNPHasChain, SDNPMayStore, SDNPMayLoad, SDNPMemOperand]>;
@@ -192,6 +321,10 @@ def z_tbegin_nofloat    : SDNode<"SystemZISD::TBEGIN_NOFLOAT", SDT_ZTBegin,
 def z_tend              : SDNode<"SystemZISD::TEND", SDTNone,
                                  [SDNPHasChain, SDNPOutGlue, SDNPSideEffect]>;
 
+def z_vshl              : SDNode<"ISD::SHL", SDT_ZVecBinary>;
+def z_vsra              : SDNode<"ISD::SRA", SDT_ZVecBinary>;
+def z_vsrl              : SDNode<"ISD::SRL", SDT_ZVecBinary>;
+
 //===----------------------------------------------------------------------===//
 // Pattern fragments
 //===----------------------------------------------------------------------===//
@@ -215,11 +348,21 @@ def sext8  : PatFrag<(ops node:$src), (sext_inreg node:$src, i8)>;
 def sext16 : PatFrag<(ops node:$src), (sext_inreg node:$src, i16)>;
 def sext32 : PatFrag<(ops node:$src), (sext (i32 node:$src))>;
 
+// Match extensions of an i32 to an i64, followed by an in-register sign
+// extension from a sub-i32 value.
+def sext8dbl : PatFrag<(ops node:$src), (sext8 (anyext node:$src))>;
+def sext16dbl : PatFrag<(ops node:$src), (sext16 (anyext node:$src))>;
+
 // Register zero-extend operations.  Sub-32-bit values are represented as i32s.
 def zext8  : PatFrag<(ops node:$src), (and node:$src, 0xff)>;
 def zext16 : PatFrag<(ops node:$src), (and node:$src, 0xffff)>;
 def zext32 : PatFrag<(ops node:$src), (zext (i32 node:$src))>;
 
+// Match extensions of an i32 to an i64, followed by an AND of the low
+// i8 or i16 part.
+def zext8dbl : PatFrag<(ops node:$src), (zext8 (anyext node:$src))>;
+def zext16dbl : PatFrag<(ops node:$src), (zext16 (anyext node:$src))>;
+
 // Typed floating-point loads.
 def loadf32 : PatFrag<(ops node:$src), (f32 (load node:$src))>;
 def loadf64 : PatFrag<(ops node:$src), (f64 (load node:$src))>;
@@ -383,6 +526,14 @@ def z_iabs64 : PatFrag<(ops node:$src),
 def z_inegabs32 : PatFrag<(ops node:$src), (ineg (z_iabs32 node:$src))>;
 def z_inegabs64 : PatFrag<(ops node:$src), (ineg (z_iabs64 node:$src))>;
 
+// Integer multiply-and-add
+def z_muladd : PatFrag<(ops node:$src1, node:$src2, node:$src3),
+                       (add (mul node:$src1, node:$src2), node:$src3)>;
+
+// Fused multiply-subtract, using the natural operand order.
+def fms : PatFrag<(ops node:$src1, node:$src2, node:$src3),
+                  (fma node:$src1, node:$src2, (fneg node:$src3))>;
+
 // Fused multiply-add and multiply-subtract, but with the order of the
 // operands matching SystemZ's MA and MS instructions.
 def z_fma : PatFrag<(ops node:$src1, node:$src2, node:$src3),
@@ -403,3 +554,110 @@ class loadu<SDPatternOperator operator, SDPatternOperator load = load>
 class storeu<SDPatternOperator operator, SDPatternOperator store = store>
   : PatFrag<(ops node:$value, node:$addr),
             (store (operator node:$value), node:$addr)>;
+
+// Vector representation of all-zeros and all-ones.
+def z_vzero : PatFrag<(ops), (bitconvert (v16i8 (z_byte_mask (i32 0))))>;
+def z_vones : PatFrag<(ops), (bitconvert (v16i8 (z_byte_mask (i32 65535))))>;
+
+// Load a scalar and replicate it in all elements of a vector.
+class z_replicate_load<ValueType scalartype, SDPatternOperator load>
+  : PatFrag<(ops node:$addr),
+            (z_replicate (scalartype (load node:$addr)))>;
+def z_replicate_loadi8  : z_replicate_load<i32, anyextloadi8>;
+def z_replicate_loadi16 : z_replicate_load<i32, anyextloadi16>;
+def z_replicate_loadi32 : z_replicate_load<i32, load>;
+def z_replicate_loadi64 : z_replicate_load<i64, load>;
+def z_replicate_loadf32 : z_replicate_load<f32, load>;
+def z_replicate_loadf64 : z_replicate_load<f64, load>;
+
+// Load a scalar and insert it into a single element of a vector.
+class z_vle<ValueType scalartype, SDPatternOperator load>
+  : PatFrag<(ops node:$vec, node:$addr, node:$index),
+            (z_vector_insert node:$vec, (scalartype (load node:$addr)),
+                             node:$index)>;
+def z_vlei8  : z_vle<i32, anyextloadi8>;
+def z_vlei16 : z_vle<i32, anyextloadi16>;
+def z_vlei32 : z_vle<i32, load>;
+def z_vlei64 : z_vle<i64, load>;
+def z_vlef32 : z_vle<f32, load>;
+def z_vlef64 : z_vle<f64, load>;
+
+// Load a scalar and insert it into the low element of the high i64 of a
+// zeroed vector.
+class z_vllez<ValueType scalartype, SDPatternOperator load, int index>
+  : PatFrag<(ops node:$addr),
+            (z_vector_insert (z_vzero),
+                             (scalartype (load node:$addr)), (i32 index))>;
+def z_vllezi8  : z_vllez<i32, anyextloadi8, 7>;
+def z_vllezi16 : z_vllez<i32, anyextloadi16, 3>;
+def z_vllezi32 : z_vllez<i32, load, 1>;
+def z_vllezi64 : PatFrag<(ops node:$addr),
+                         (z_join_dwords (i64 (load node:$addr)), (i64 0))>;
+// We use high merges to form a v4f32 from four f32s.  Propagating zero
+// into all elements but index 1 gives this expression.
+def z_vllezf32 : PatFrag<(ops node:$addr),
+                         (bitconvert
+                          (z_merge_high
+                           (v2i64
+                            (z_unpackl_high
+                             (v4i32
+                              (bitconvert
+                               (v4f32 (scalar_to_vector
+                                       (f32 (load node:$addr)))))))),
+                           (v2i64 (z_vzero))))>;
+def z_vllezf64 : PatFrag<(ops node:$addr),
+                         (z_merge_high
+                          (scalar_to_vector (f64 (load node:$addr))),
+                          (z_vzero))>;
+
+// Store one element of a vector.
+class z_vste<ValueType scalartype, SDPatternOperator store>
+  : PatFrag<(ops node:$vec, node:$addr, node:$index),
+            (store (scalartype (z_vector_extract node:$vec, node:$index)),
+                   node:$addr)>;
+def z_vstei8  : z_vste<i32, truncstorei8>;
+def z_vstei16 : z_vste<i32, truncstorei16>;
+def z_vstei32 : z_vste<i32, store>;
+def z_vstei64 : z_vste<i64, store>;
+def z_vstef32 : z_vste<f32, store>;
+def z_vstef64 : z_vste<f64, store>;
+
+// Arithmetic negation on vectors.
+def z_vneg : PatFrag<(ops node:$x), (sub (z_vzero), node:$x)>;
+
+// Bitwise negation on vectors.
+def z_vnot : PatFrag<(ops node:$x), (xor node:$x, (z_vones))>;
+
+// Signed "integer greater than zero" on vectors.
+def z_vicmph_zero : PatFrag<(ops node:$x), (z_vicmph node:$x, (z_vzero))>;
+
+// Signed "integer less than zero" on vectors.
+def z_vicmpl_zero : PatFrag<(ops node:$x), (z_vicmph (z_vzero), node:$x)>;
+
+// Integer absolute on vectors.
+class z_viabs<int shift>
+  : PatFrag<(ops node:$src),
+            (xor (add node:$src, (z_vsra_by_scalar node:$src, (i32 shift))),
+                 (z_vsra_by_scalar node:$src, (i32 shift)))>;
+def z_viabs8  : z_viabs<7>;
+def z_viabs16 : z_viabs<15>;
+def z_viabs32 : z_viabs<31>;
+def z_viabs64 : z_viabs<63>;
+
+// Sign-extend the i64 elements of a vector.
+class z_vse<int shift>
+  : PatFrag<(ops node:$src),
+            (z_vsra_by_scalar (z_vshl_by_scalar node:$src, shift), shift)>;
+def z_vsei8  : z_vse<56>;
+def z_vsei16 : z_vse<48>;
+def z_vsei32 : z_vse<32>;
+
+// ...and again with the extensions being done on individual i64 scalars.
+class z_vse_by_parts<SDPatternOperator operator, int index1, int index2>
+  : PatFrag<(ops node:$src),
+            (z_join_dwords
+             (operator (z_vector_extract node:$src, index1)),
+             (operator (z_vector_extract node:$src, index2)))>;
+def z_vsei8_by_parts  : z_vse_by_parts<sext8dbl, 7, 15>;
+def z_vsei16_by_parts : z_vse_by_parts<sext16dbl, 3, 7>;
+def z_vsei32_by_parts : z_vse_by_parts<sext32, 1, 3>;
diff --git a/lib/Target/SystemZ/SystemZPatterns.td b/lib/Target/SystemZ/SystemZPatterns.td
index e307f8a888e..16a7ed784d7 100644
--- a/lib/Target/SystemZ/SystemZPatterns.td
+++ b/lib/Target/SystemZ/SystemZPatterns.td
@@ -153,3 +153,17 @@ multiclass CompareZeroFP<Instruction insn, RegisterOperand cls> {
   // The sign of the zero makes no difference.
   def : Pat<(z_fcmp cls:$reg, (fpimmneg0)), (insn cls:$reg, cls:$reg)>;
 }
+
+// Use INSN for performing binary operation OPERATION of type VT
+// on registers of class CLS.
+class BinaryRRWithType<Instruction insn, RegisterOperand cls,
+                       SDPatternOperator operator, ValueType vt>
+  : Pat<(vt (operator cls:$x, cls:$y)), (insn cls:$x, cls:$y)>;
+
+// Use INSN to perform conversion operation OPERATOR, with the input being
+// TR2 and the output being TR1.  SUPPRESS is 4 to suppress inexact conditions
+// and 0 to allow them.  MODE is the rounding mode to use.
+class FPConversion<Instruction insn, SDPatternOperator operator, TypedReg tr1,
+                   TypedReg tr2, bits<3> suppress, bits<4> mode>
+  : Pat<(tr1.vt (operator (tr2.vt tr2.op:$vec))),
+        (insn tr2.op:$vec, suppress, mode)>;
diff --git a/lib/Target/SystemZ/SystemZProcessors.td b/lib/Target/SystemZ/SystemZProcessors.td
index 15614c970cd..32fbe5ae9ef 100644
--- a/lib/Target/SystemZ/SystemZProcessors.td
+++ b/lib/Target/SystemZ/SystemZProcessors.td
@@ -70,6 +70,12 @@ def FeatureProcessorAssist : SystemZFeature<
   "Assume that the processor-assist facility is installed"
 >;
 
+def FeatureVector : SystemZFeature<
+  "vector", "Vector",
+  "Assume that the vectory facility is installed"
+>;
+def FeatureNoVector : SystemZMissingFeature<"Vector">;
+
 def : Processor<"generic", NoItineraries, []>;
 def : Processor<"z10", NoItineraries, []>;
 def : Processor<"z196", NoItineraries,
@@ -82,3 +88,9 @@ def : Processor<"zEC12", NoItineraries,
                  FeatureFastSerialization, FeatureInterlockedAccess1,
                  FeatureMiscellaneousExtensions,
                  FeatureTransactionalExecution, FeatureProcessorAssist]>;
+def : Processor<"z13", NoItineraries,
+                [FeatureDistinctOps, FeatureLoadStoreOnCond, FeatureHighWord,
+                 FeatureFPExtension, FeaturePopulationCount,
+                 FeatureFastSerialization, FeatureInterlockedAccess1,
+                 FeatureTransactionalExecution, FeatureProcessorAssist,
+                 FeatureVector]>;
diff --git a/lib/Target/SystemZ/SystemZRegisterInfo.cpp b/lib/Target/SystemZ/SystemZRegisterInfo.cpp
index 7cabea962e9..dc7bd25d7ed 100644
--- a/lib/Target/SystemZ/SystemZRegisterInfo.cpp
+++ b/lib/Target/SystemZ/SystemZRegisterInfo.cpp
@@ -36,7 +36,7 @@ SystemZRegisterInfo::getCallPreservedMask(const MachineFunction &MF,
 BitVector
 SystemZRegisterInfo::getReservedRegs(const MachineFunction &MF) const {
   BitVector Reserved(getNumRegs());
-  const TargetFrameLowering *TFI = MF.getSubtarget().getFrameLowering();
+  const SystemZFrameLowering *TFI = getFrameLowering(MF);
 
   if (TFI->hasFP(MF)) {
     // R11D is the frame pointer.  Reserve all aliases.
@@ -64,7 +64,7 @@ SystemZRegisterInfo::eliminateFrameIndex(MachineBasicBlock::iterator MI,
   MachineFunction &MF = *MBB.getParent();
   auto *TII =
       static_cast<const SystemZInstrInfo *>(MF.getSubtarget().getInstrInfo());
-  const TargetFrameLowering *TFI = MF.getSubtarget().getFrameLowering();
+  const SystemZFrameLowering *TFI = getFrameLowering(MF);
   DebugLoc DL = MI->getDebugLoc();
 
   // Decompose the frame index into a base and offset.
@@ -135,6 +135,6 @@ SystemZRegisterInfo::eliminateFrameIndex(MachineBasicBlock::iterator MI,
 
 unsigned
 SystemZRegisterInfo::getFrameRegister(const MachineFunction &MF) const {
-  const TargetFrameLowering *TFI = MF.getSubtarget().getFrameLowering();
+  const SystemZFrameLowering *TFI = getFrameLowering(MF);
   return TFI->hasFP(MF) ? SystemZ::R11D : SystemZ::R15D;
 }
diff --git a/lib/Target/SystemZ/SystemZRegisterInfo.td b/lib/Target/SystemZ/SystemZRegisterInfo.td
index 47ac20dae78..85aa0a62cc7 100644
--- a/lib/Target/SystemZ/SystemZRegisterInfo.td
+++ b/lib/Target/SystemZ/SystemZRegisterInfo.td
@@ -25,20 +25,24 @@ def subreg_l32   : SubRegIndex<32, 0>;  // Also acts as subreg_ll32.
 def subreg_h32   : SubRegIndex<32, 32>; // Also acts as subreg_lh32.
 def subreg_l64   : SubRegIndex<64, 0>;
 def subreg_h64   : SubRegIndex<64, 64>;
+def subreg_r32   : SubRegIndex<32, 32>; // Reinterpret a wider reg as 32 bits.
+def subreg_r64   : SubRegIndex<64, 64>; // Reinterpret a wider reg as 64 bits.
 def subreg_hh32  : ComposedSubRegIndex<subreg_h64, subreg_h32>;
 def subreg_hl32  : ComposedSubRegIndex<subreg_h64, subreg_l32>;
+def subreg_hr32  : ComposedSubRegIndex<subreg_h64, subreg_r32>;
 }
 
-// Define a register class that contains values of type TYPE and an
+// Define a register class that contains values of types TYPES and an
 // associated operand called NAME.  SIZE is the size and alignment
 // of the registers and REGLIST is the list of individual registers.
-multiclass SystemZRegClass<string name, ValueType type, int size, dag regList> {
+multiclass SystemZRegClass<string name, list<ValueType> types, int size,
+                           dag regList> {
   def AsmOperand : AsmOperandClass {
     let Name = name;
     let ParserMethod = "parse"##name;
     let RenderMethod = "addRegOperands";
   }
-  def Bit : RegisterClass<"SystemZ", [type], size, regList> {
+  def Bit : RegisterClass<"SystemZ", types, size, regList> {
     let Size = size;
   }
   def "" : RegisterOperand<!cast<RegisterClass>(name##"Bit")> {
@@ -84,16 +88,19 @@ foreach I = [0, 2, 4, 6, 8, 10, 12, 14] in {
 
 /// Allocate the callee-saved R6-R13 backwards. That way they can be saved
 /// together with R14 and R15 in one prolog instruction.
-defm GR32  : SystemZRegClass<"GR32",  i32, 32, (add (sequence "R%uL",  0, 5),
-                                                    (sequence "R%uL", 15, 6))>;
-defm GRH32 : SystemZRegClass<"GRH32", i32, 32, (add (sequence "R%uH",  0, 5),
-                                                    (sequence "R%uH", 15, 6))>;
-defm GR64  : SystemZRegClass<"GR64",  i64, 64, (add (sequence "R%uD",  0, 5),
-                                                    (sequence "R%uD", 15, 6))>;
+defm GR32  : SystemZRegClass<"GR32",  [i32], 32,
+                             (add (sequence "R%uL",  0, 5),
+                                  (sequence "R%uL", 15, 6))>;
+defm GRH32 : SystemZRegClass<"GRH32", [i32], 32,
+                             (add (sequence "R%uH",  0, 5),
+                                  (sequence "R%uH", 15, 6))>;
+defm GR64  : SystemZRegClass<"GR64",  [i64], 64,
+                             (add (sequence "R%uD",  0, 5),
+                                  (sequence "R%uD", 15, 6))>;
 
 // Combine the low and high GR32s into a single class.  This can only be
 // used for virtual registers if the high-word facility is available.
-defm GRX32 : SystemZRegClass<"GRX32", i32, 32,
+defm GRX32 : SystemZRegClass<"GRX32", [i32], 32,
                              (add (sequence "R%uL",  0, 5),
                                   (sequence "R%uH",  0, 5),
                                   R15L, R15H, R14L, R14H, R13L, R13H,
@@ -102,18 +109,17 @@ defm GRX32 : SystemZRegClass<"GRX32", i32, 32,
 
 // The architecture doesn't really have any i128 support, so model the
 // register pairs as untyped instead.
-defm GR128 : SystemZRegClass<"GR128", untyped, 128, (add R0Q, R2Q, R4Q,
-                                                         R12Q, R10Q, R8Q, R6Q,
-                                                         R14Q)>;
+defm GR128 : SystemZRegClass<"GR128", [untyped], 128,
+                             (add R0Q, R2Q, R4Q, R12Q, R10Q, R8Q, R6Q, R14Q)>;
 
 // Base and index registers.  Everything except R0, which in an address
 // context evaluates as 0.
-defm ADDR32 : SystemZRegClass<"ADDR32", i32, 32, (sub GR32Bit, R0L)>;
-defm ADDR64 : SystemZRegClass<"ADDR64", i64, 64, (sub GR64Bit, R0D)>;
+defm ADDR32 : SystemZRegClass<"ADDR32", [i32], 32, (sub GR32Bit, R0L)>;
+defm ADDR64 : SystemZRegClass<"ADDR64", [i64], 64, (sub GR64Bit, R0D)>;
 
 // Not used directly, but needs to exist for ADDR32 and ADDR64 subregs
 // of a GR128.
-defm ADDR128 : SystemZRegClass<"ADDR128", untyped, 128, (sub GR128Bit, R0Q)>;
+defm ADDR128 : SystemZRegClass<"ADDR128", [untyped], 128, (sub GR128Bit, R0Q)>;
 
 //===----------------------------------------------------------------------===//
 // Floating-point registers
@@ -142,16 +148,36 @@ def F11Dwarf : DwarfMapping<29>;
 def F13Dwarf : DwarfMapping<30>;
 def F15Dwarf : DwarfMapping<31>;
 
-// Lower 32 bits of one of the 16 64-bit floating-point registers
+def F16Dwarf : DwarfMapping<68>;
+def F18Dwarf : DwarfMapping<69>;
+def F20Dwarf : DwarfMapping<70>;
+def F22Dwarf : DwarfMapping<71>;
+
+def F17Dwarf : DwarfMapping<72>;
+def F19Dwarf : DwarfMapping<73>;
+def F21Dwarf : DwarfMapping<74>;
+def F23Dwarf : DwarfMapping<75>;
+
+def F24Dwarf : DwarfMapping<76>;
+def F26Dwarf : DwarfMapping<77>;
+def F28Dwarf : DwarfMapping<78>;
+def F30Dwarf : DwarfMapping<79>;
+
+def F25Dwarf : DwarfMapping<80>;
+def F27Dwarf : DwarfMapping<81>;
+def F29Dwarf : DwarfMapping<82>;
+def F31Dwarf : DwarfMapping<83>;
+
+// Upper 32 bits of one of the floating-point registers
 class FPR32<bits<16> num, string n> : SystemZReg<n> {
   let HWEncoding = num;
 }
 
-// One of the 16 64-bit floating-point registers
-class FPR64<bits<16> num, string n, FPR32 low>
- : SystemZRegWithSubregs<n, [low]> {
+// One of the floating-point registers.
+class FPR64<bits<16> num, string n, FPR32 high>
+ : SystemZRegWithSubregs<n, [high]> {
   let HWEncoding = num;
-  let SubRegIndices = [subreg_h32];
+  let SubRegIndices = [subreg_r32];
 }
 
 // 8 pairs of FPR64s, with a one-register gap inbetween.
@@ -161,12 +187,17 @@ class FPR128<bits<16> num, string n, FPR64 low, FPR64 high>
   let SubRegIndices = [subreg_l64, subreg_h64];
 }
 
-// Floating-point registers
+// Floating-point registers.  Registers 16-31 require the vector facility.
 foreach I = 0-15 in {
   def F#I#S : FPR32<I, "f"#I>;
   def F#I#D : FPR64<I, "f"#I, !cast<FPR32>("F"#I#"S")>,
               DwarfRegNum<[!cast<DwarfMapping>("F"#I#"Dwarf").Id]>;
 }
+foreach I = 16-31 in {
+  def F#I#S : FPR32<I, "v"#I>;
+  def F#I#D : FPR64<I, "v"#I, !cast<FPR32>("F"#I#"S")>,
+              DwarfRegNum<[!cast<DwarfMapping>("F"#I#"Dwarf").Id]>;
+}
 
 foreach I = [0, 1, 4, 5, 8, 9, 12, 13] in {
   def F#I#Q  : FPR128<I, "f"#I, !cast<FPR64>("F"#!add(I, 2)#"D"),
@@ -175,10 +206,74 @@ foreach I = [0, 1, 4, 5, 8, 9, 12, 13] in {
 
 // There's no store-multiple instruction for FPRs, so we're not fussy
 // about the order in which call-saved registers are allocated.
-defm FP32  : SystemZRegClass<"FP32", f32, 32, (sequence "F%uS", 0, 15)>;
-defm FP64  : SystemZRegClass<"FP64", f64, 64, (sequence "F%uD", 0, 15)>;
-defm FP128 : SystemZRegClass<"FP128", f128, 128, (add F0Q, F1Q, F4Q, F5Q,
-                                                      F8Q, F9Q, F12Q, F13Q)>;
+defm FP32  : SystemZRegClass<"FP32", [f32], 32, (sequence "F%uS", 0, 15)>;
+defm FP64  : SystemZRegClass<"FP64", [f64], 64, (sequence "F%uD", 0, 15)>;
+defm FP128 : SystemZRegClass<"FP128", [f128], 128,
+                             (add F0Q, F1Q, F4Q, F5Q, F8Q, F9Q, F12Q, F13Q)>;
+
+//===----------------------------------------------------------------------===//
+// Vector registers
+//===----------------------------------------------------------------------===//
+
+// A full 128-bit vector register, with an FPR64 as its high part.
+class VR128<bits<16> num, string n, FPR64 high>
+  : SystemZRegWithSubregs<n, [high]> {
+  let HWEncoding = num;
+  let SubRegIndices = [subreg_r64];
+}
+
+// Full vector registers.
+foreach I = 0-31 in {
+  def V#I : VR128<I, "v"#I, !cast<FPR64>("F"#I#"D")>,
+            DwarfRegNum<[!cast<DwarfMapping>("F"#I#"Dwarf").Id]>;
+}
+
+// Class used to store 32-bit values in the first element of a vector
+// register.  f32 scalars are used for the WLEDB and WLDEB instructions.
+defm VR32 : SystemZRegClass<"VR32", [f32, v4i8, v2i16], 32,
+                            (add (sequence "F%uS", 0, 7),
+                                 (sequence "F%uS", 16, 31),
+                                 (sequence "F%uS", 8, 15))>;
+
+// Class used to store 64-bit values in the upper half of a vector register.
+// The vector facility also includes scalar f64 instructions that operate
+// on the full vector register set.
+defm VR64 : SystemZRegClass<"VR64", [f64, v8i8, v4i16, v2i32, v2f32], 64,
+                            (add (sequence "F%uD", 0, 7),
+                                 (sequence "F%uD", 16, 31),
+                                 (sequence "F%uD", 8, 15))>;
+
+// The subset of vector registers that can be used for floating-point
+// operations too.
+defm VF128 : SystemZRegClass<"VF128",
+                             [v16i8, v8i16, v4i32, v2i64, v4f32, v2f64], 128,
+                             (sequence "V%u", 0, 15)>;
+
+// All vector registers.
+defm VR128 : SystemZRegClass<"VR128",
+                             [v16i8, v8i16, v4i32, v2i64, v4f32, v2f64], 128,
+                             (add (sequence "V%u", 0, 7),
+                                  (sequence "V%u", 16, 31),
+                                  (sequence "V%u", 8, 15))>;
+
+// Attaches a ValueType to a register operand, to make the instruction
+// definitions easier.
+class TypedReg<ValueType vtin, RegisterOperand opin> {
+  ValueType vt = vtin;
+  RegisterOperand op = opin;
+}
+
+def v32eb   : TypedReg<f32,     VR32>;
+def v64g    : TypedReg<i64,     VR64>;
+def v64db   : TypedReg<f64,     VR64>;
+def v128b   : TypedReg<v16i8,   VR128>;
+def v128h   : TypedReg<v8i16,   VR128>;
+def v128f   : TypedReg<v4i32,   VR128>;
+def v128g   : TypedReg<v2i64,   VR128>;
+def v128q   : TypedReg<v16i8,   VR128>;
+def v128eb  : TypedReg<v4f32,   VR128>;
+def v128db  : TypedReg<v2f64,   VR128>;
+def v128any : TypedReg<untyped, VR128>;
 
 //===----------------------------------------------------------------------===//
 // Other registers
diff --git a/lib/Target/SystemZ/SystemZSelectionDAGInfo.cpp b/lib/Target/SystemZ/SystemZSelectionDAGInfo.cpp
index 12fc1981d7d..178aa381731 100644
--- a/lib/Target/SystemZ/SystemZSelectionDAGInfo.cpp
+++ b/lib/Target/SystemZ/SystemZSelectionDAGInfo.cpp
@@ -18,12 +18,6 @@ using namespace llvm;
 
 #define DEBUG_TYPE "systemz-selectiondag-info"
 
-SystemZSelectionDAGInfo::SystemZSelectionDAGInfo(const DataLayout &DL)
-    : TargetSelectionDAGInfo(&DL) {}
-
-SystemZSelectionDAGInfo::~SystemZSelectionDAGInfo() {
-}
-
 // Decide whether it is best to use a loop or straight-line code for
 // a block operation of Size bytes with source address Src and destination
 // address Dest.  Sequence is the opcode to use for straight-line code
@@ -46,10 +40,10 @@ static SDValue emitMemMem(SelectionDAG &DAG, SDLoc DL, unsigned Sequence,
   // number of straight-line MVCs as 6 * 256 - 1.
   if (Size > 6 * 256)
     return DAG.getNode(Loop, DL, MVT::Other, Chain, Dst, Src,
-                       DAG.getConstant(Size, PtrVT),
-                       DAG.getConstant(Size / 256, PtrVT));
+                       DAG.getConstant(Size, DL, PtrVT),
+                       DAG.getConstant(Size / 256, DL, PtrVT));
   return DAG.getNode(Sequence, DL, MVT::Other, Chain, Dst, Src,
-                     DAG.getConstant(Size, PtrVT));
+                     DAG.getConstant(Size, DL, PtrVT));
 }
 
 SDValue SystemZSelectionDAGInfo::
@@ -78,7 +72,8 @@ static SDValue memsetStore(SelectionDAG &DAG, SDLoc DL, SDValue Chain,
   for (unsigned I = 1; I < Size; ++I)
     StoreVal |= ByteVal << (I * 8);
   return DAG.getStore(Chain, DL,
-                      DAG.getConstant(StoreVal, MVT::getIntegerVT(Size * 8)),
+                      DAG.getConstant(StoreVal, DL,
+                                      MVT::getIntegerVT(Size * 8)),
                       Dst, DstPtrInfo, false, false, Align);
 }
 
@@ -112,7 +107,7 @@ EmitTargetCodeForMemset(SelectionDAG &DAG, SDLoc DL, SDValue Chain,
         if (Size2 == 0)
           return Chain1;
         Dst = DAG.getNode(ISD::ADD, DL, PtrVT, Dst,
-                          DAG.getConstant(Size1, PtrVT));
+                          DAG.getConstant(Size1, DL, PtrVT));
         DstPtrInfo = DstPtrInfo.getWithOffset(Size1);
         SDValue Chain2 = memsetStore(DAG, DL, Chain, Dst, ByteVal, Size2,
                                      std::min(Align, Size1), DstPtrInfo);
@@ -126,7 +121,7 @@ EmitTargetCodeForMemset(SelectionDAG &DAG, SDLoc DL, SDValue Chain,
         if (Bytes == 1)
           return Chain1;
         SDValue Dst2 = DAG.getNode(ISD::ADD, DL, PtrVT, Dst,
-                                   DAG.getConstant(1, PtrVT));
+                                   DAG.getConstant(1, DL, PtrVT));
         SDValue Chain2 = DAG.getStore(Chain, DL, Byte, Dst2,
                                       DstPtrInfo.getWithOffset(1),
                                       false, false, 1);
@@ -146,7 +141,7 @@ EmitTargetCodeForMemset(SelectionDAG &DAG, SDLoc DL, SDValue Chain,
     Chain = DAG.getStore(Chain, DL, Byte, Dst, DstPtrInfo,
                          false, false, Align);
     SDValue DstPlus1 = DAG.getNode(ISD::ADD, DL, PtrVT, Dst,
-                                   DAG.getConstant(1, PtrVT));
+                                   DAG.getConstant(1, DL, PtrVT));
     return emitMemMem(DAG, DL, SystemZISD::MVC, SystemZISD::MVC_LOOP,
                       Chain, DstPlus1, Dst, Bytes - 1);
   }
@@ -169,10 +164,10 @@ static SDValue emitCLC(SelectionDAG &DAG, SDLoc DL, SDValue Chain,
   // needs 2 branches, whereas a straight-line sequence would need 3 or more.
   if (Size > 3 * 256)
     return DAG.getNode(SystemZISD::CLC_LOOP, DL, VTs, Chain, Src1, Src2,
-                       DAG.getConstant(Size, PtrVT),
-                       DAG.getConstant(Size / 256, PtrVT));
+                       DAG.getConstant(Size, DL, PtrVT),
+                       DAG.getConstant(Size / 256, DL, PtrVT));
   return DAG.getNode(SystemZISD::CLC, DL, VTs, Chain, Src1, Src2,
-                     DAG.getConstant(Size, PtrVT));
+                     DAG.getConstant(Size, DL, PtrVT));
 }
 
 // Convert the current CC value into an integer that is 0 if CC == 0,
@@ -182,9 +177,9 @@ static SDValue emitCLC(SelectionDAG &DAG, SDLoc DL, SDValue Chain,
 static SDValue addIPMSequence(SDLoc DL, SDValue Glue, SelectionDAG &DAG) {
   SDValue IPM = DAG.getNode(SystemZISD::IPM, DL, MVT::i32, Glue);
   SDValue SRL = DAG.getNode(ISD::SRL, DL, MVT::i32, IPM,
-                            DAG.getConstant(SystemZ::IPM_CC, MVT::i32));
+                            DAG.getConstant(SystemZ::IPM_CC, DL, MVT::i32));
   SDValue ROTL = DAG.getNode(ISD::ROTL, DL, MVT::i32, SRL,
-                             DAG.getConstant(31, MVT::i32));
+                             DAG.getConstant(31, DL, MVT::i32));
   return ROTL;
 }
 
@@ -213,7 +208,7 @@ EmitTargetCodeForMemchr(SelectionDAG &DAG, SDLoc DL, SDValue Chain,
   Length = DAG.getZExtOrTrunc(Length, DL, PtrVT);
   Char = DAG.getZExtOrTrunc(Char, DL, MVT::i32);
   Char = DAG.getNode(ISD::AND, DL, MVT::i32, Char,
-                     DAG.getConstant(255, MVT::i32));
+                     DAG.getConstant(255, DL, MVT::i32));
   SDValue Limit = DAG.getNode(ISD::ADD, DL, PtrVT, Src, Length);
   SDValue End = DAG.getNode(SystemZISD::SEARCH_STRING, DL, VTs, Chain,
                             Limit, Src, Char);
@@ -222,9 +217,10 @@ EmitTargetCodeForMemchr(SelectionDAG &DAG, SDLoc DL, SDValue Chain,
 
   // Now select between End and null, depending on whether the character
   // was found.
-  SDValue Ops[] = {End, DAG.getConstant(0, PtrVT),
-                   DAG.getConstant(SystemZ::CCMASK_SRST, MVT::i32),
-                   DAG.getConstant(SystemZ::CCMASK_SRST_FOUND, MVT::i32), Glue};
+  SDValue Ops[] = {End, DAG.getConstant(0, DL, PtrVT),
+                   DAG.getConstant(SystemZ::CCMASK_SRST, DL, MVT::i32),
+                   DAG.getConstant(SystemZ::CCMASK_SRST_FOUND, DL, MVT::i32),
+                   Glue};
   VTs = DAG.getVTList(PtrVT, MVT::Glue);
   End = DAG.getNode(SystemZISD::SELECT_CCMASK, DL, VTs, Ops);
   return std::make_pair(End, Chain);
@@ -237,7 +233,7 @@ EmitTargetCodeForStrcpy(SelectionDAG &DAG, SDLoc DL, SDValue Chain,
                         MachinePointerInfo SrcPtrInfo, bool isStpcpy) const {
   SDVTList VTs = DAG.getVTList(Dest.getValueType(), MVT::Other);
   SDValue EndDest = DAG.getNode(SystemZISD::STPCPY, DL, VTs, Chain, Dest, Src,
-                                DAG.getConstant(0, MVT::i32));
+                                DAG.getConstant(0, DL, MVT::i32));
   return std::make_pair(isStpcpy ? EndDest : Dest, EndDest.getValue(1));
 }
 
@@ -248,7 +244,7 @@ EmitTargetCodeForStrcmp(SelectionDAG &DAG, SDLoc DL, SDValue Chain,
                         MachinePointerInfo Op2PtrInfo) const {
   SDVTList VTs = DAG.getVTList(Src1.getValueType(), MVT::Other, MVT::Glue);
   SDValue Unused = DAG.getNode(SystemZISD::STRCMP, DL, VTs, Chain, Src1, Src2,
-                               DAG.getConstant(0, MVT::i32));
+                               DAG.getConstant(0, DL, MVT::i32));
   Chain = Unused.getValue(1);
   SDValue Glue = Chain.getValue(2);
   return std::make_pair(addIPMSequence(DL, Glue, DAG), Chain);
@@ -265,7 +261,7 @@ static std::pair<SDValue, SDValue> getBoundedStrlen(SelectionDAG &DAG, SDLoc DL,
   EVT PtrVT = Src.getValueType();
   SDVTList VTs = DAG.getVTList(PtrVT, MVT::Other, MVT::Glue);
   SDValue End = DAG.getNode(SystemZISD::SEARCH_STRING, DL, VTs, Chain,
-                            Limit, Src, DAG.getConstant(0, MVT::i32));
+                            Limit, Src, DAG.getConstant(0, DL, MVT::i32));
   Chain = End.getValue(1);
   SDValue Len = DAG.getNode(ISD::SUB, DL, PtrVT, End, Src);
   return std::make_pair(Len, Chain);
@@ -275,7 +271,7 @@ std::pair<SDValue, SDValue> SystemZSelectionDAGInfo::
 EmitTargetCodeForStrlen(SelectionDAG &DAG, SDLoc DL, SDValue Chain,
                         SDValue Src, MachinePointerInfo SrcPtrInfo) const {
   EVT PtrVT = Src.getValueType();
-  return getBoundedStrlen(DAG, DL, Chain, Src, DAG.getConstant(0, PtrVT));
+  return getBoundedStrlen(DAG, DL, Chain, Src, DAG.getConstant(0, DL, PtrVT));
 }
 
 std::pair<SDValue, SDValue> SystemZSelectionDAGInfo::
diff --git a/lib/Target/SystemZ/SystemZSelectionDAGInfo.h b/lib/Target/SystemZ/SystemZSelectionDAGInfo.h
index a257d6b5549..246fa3e5e65 100644
--- a/lib/Target/SystemZ/SystemZSelectionDAGInfo.h
+++ b/lib/Target/SystemZ/SystemZSelectionDAGInfo.h
@@ -22,8 +22,7 @@ class SystemZTargetMachine;
 
 class SystemZSelectionDAGInfo : public TargetSelectionDAGInfo {
 public:
-  explicit SystemZSelectionDAGInfo(const DataLayout &DL);
-  ~SystemZSelectionDAGInfo();
+  explicit SystemZSelectionDAGInfo() = default;
 
   SDValue EmitTargetCodeForMemcpy(SelectionDAG &DAG, SDLoc DL, SDValue Chain,
                                   SDValue Dst, SDValue Src,
diff --git a/lib/Target/SystemZ/SystemZShortenInst.cpp b/lib/Target/SystemZ/SystemZShortenInst.cpp
index ec7a8c40d18..d1a17c5500d 100644
--- a/lib/Target/SystemZ/SystemZShortenInst.cpp
+++ b/lib/Target/SystemZ/SystemZShortenInst.cpp
@@ -15,6 +15,7 @@
 
 #include "SystemZTargetMachine.h"
 #include "llvm/CodeGen/MachineFunctionPass.h"
+#include "llvm/CodeGen/MachineInstrBuilder.h"
 
 using namespace llvm;
 
@@ -36,6 +37,10 @@ class SystemZShortenInst : public MachineFunctionPass {
 private:
   bool shortenIIF(MachineInstr &MI, unsigned *GPRMap, unsigned LiveOther,
                   unsigned LLIxL, unsigned LLIxH);
+  bool shortenOn0(MachineInstr &MI, unsigned Opcode);
+  bool shortenOn01(MachineInstr &MI, unsigned Opcode);
+  bool shortenOn001(MachineInstr &MI, unsigned Opcode);
+  bool shortenFPConv(MachineInstr &MI, unsigned Opcode);
 
   const SystemZInstrInfo *TII;
 
@@ -97,6 +102,64 @@ bool SystemZShortenInst::shortenIIF(MachineInstr &MI, unsigned *GPRMap,
   return false;
 }
 
+// Change MI's opcode to Opcode if register operand 0 has a 4-bit encoding.
+bool SystemZShortenInst::shortenOn0(MachineInstr &MI, unsigned Opcode) {
+  if (SystemZMC::getFirstReg(MI.getOperand(0).getReg()) < 16) {
+    MI.setDesc(TII->get(Opcode));
+    return true;
+  }
+  return false;
+}
+
+// Change MI's opcode to Opcode if register operands 0 and 1 have a
+// 4-bit encoding.
+bool SystemZShortenInst::shortenOn01(MachineInstr &MI, unsigned Opcode) {
+  if (SystemZMC::getFirstReg(MI.getOperand(0).getReg()) < 16 &&
+      SystemZMC::getFirstReg(MI.getOperand(1).getReg()) < 16) {
+    MI.setDesc(TII->get(Opcode));
+    return true;
+  }
+  return false;
+}
+
+// Change MI's opcode to Opcode if register operands 0, 1 and 2 have a
+// 4-bit encoding and if operands 0 and 1 are tied.
+bool SystemZShortenInst::shortenOn001(MachineInstr &MI, unsigned Opcode) {
+  if (SystemZMC::getFirstReg(MI.getOperand(0).getReg()) < 16 &&
+      MI.getOperand(1).getReg() == MI.getOperand(0).getReg() &&
+      SystemZMC::getFirstReg(MI.getOperand(2).getReg()) < 16) {
+    MI.setDesc(TII->get(Opcode));
+    return true;
+  }
+  return false;
+}
+
+// MI is a vector-style conversion instruction with the operand order:
+// destination, source, exact-suppress, rounding-mode.  If both registers
+// have a 4-bit encoding then change it to Opcode, which has operand order:
+// destination, rouding-mode, source, exact-suppress.
+bool SystemZShortenInst::shortenFPConv(MachineInstr &MI, unsigned Opcode) {
+  if (SystemZMC::getFirstReg(MI.getOperand(0).getReg()) < 16 &&
+      SystemZMC::getFirstReg(MI.getOperand(1).getReg()) < 16) {
+    MachineOperand Dest(MI.getOperand(0));
+    MachineOperand Src(MI.getOperand(1));
+    MachineOperand Suppress(MI.getOperand(2));
+    MachineOperand Mode(MI.getOperand(3));
+    MI.RemoveOperand(3);
+    MI.RemoveOperand(2);
+    MI.RemoveOperand(1);
+    MI.RemoveOperand(0);
+    MI.setDesc(TII->get(Opcode));
+    MachineInstrBuilder(*MI.getParent()->getParent(), &MI)
+      .addOperand(Dest)
+      .addOperand(Mode)
+      .addOperand(Src)
+      .addOperand(Suppress);
+    return true;
+  }
+  return false;
+}
+
 // Process all instructions in MBB.  Return true if something changed.
 bool SystemZShortenInst::processBlock(MachineBasicBlock &MBB) {
   bool Changed = false;
@@ -117,13 +180,83 @@ bool SystemZShortenInst::processBlock(MachineBasicBlock &MBB) {
   // Iterate backwards through the block looking for instructions to change.
   for (auto MBBI = MBB.rbegin(), MBBE = MBB.rend(); MBBI != MBBE; ++MBBI) {
     MachineInstr &MI = *MBBI;
-    unsigned Opcode = MI.getOpcode();
-    if (Opcode == SystemZ::IILF)
+    switch (MI.getOpcode()) {
+    case SystemZ::IILF:
       Changed |= shortenIIF(MI, LowGPRs, LiveHigh, SystemZ::LLILL,
                             SystemZ::LLILH);
-    else if (Opcode == SystemZ::IIHF)
+      break;
+
+    case SystemZ::IIHF:
       Changed |= shortenIIF(MI, HighGPRs, LiveLow, SystemZ::LLIHL,
                             SystemZ::LLIHH);
+      break;
+
+    case SystemZ::WFADB:
+      Changed |= shortenOn001(MI, SystemZ::ADBR);
+      break;
+
+    case SystemZ::WFDDB:
+      Changed |= shortenOn001(MI, SystemZ::DDBR);
+      break;
+
+    case SystemZ::WFIDB:
+      Changed |= shortenFPConv(MI, SystemZ::FIDBRA);
+      break;
+
+    case SystemZ::WLDEB:
+      Changed |= shortenOn01(MI, SystemZ::LDEBR);
+      break;
+
+    case SystemZ::WLEDB:
+      Changed |= shortenFPConv(MI, SystemZ::LEDBRA);
+      break;
+
+    case SystemZ::WFMDB:
+      Changed |= shortenOn001(MI, SystemZ::MDBR);
+      break;
+
+    case SystemZ::WFLCDB:
+      Changed |= shortenOn01(MI, SystemZ::LCDBR);
+      break;
+
+    case SystemZ::WFLNDB:
+      Changed |= shortenOn01(MI, SystemZ::LNDBR);
+      break;
+
+    case SystemZ::WFLPDB:
+      Changed |= shortenOn01(MI, SystemZ::LPDBR);
+      break;
+
+    case SystemZ::WFSQDB:
+      Changed |= shortenOn01(MI, SystemZ::SQDBR);
+      break;
+
+    case SystemZ::WFSDB:
+      Changed |= shortenOn001(MI, SystemZ::SDBR);
+      break;
+
+    case SystemZ::WFCDB:
+      Changed |= shortenOn01(MI, SystemZ::CDBR);
+      break;
+
+    case SystemZ::VL32:
+      // For z13 we prefer LDE over LE to avoid partial register dependencies.
+      Changed |= shortenOn0(MI, SystemZ::LDE32);
+      break;
+
+    case SystemZ::VST32:
+      Changed |= shortenOn0(MI, SystemZ::STE);
+      break;
+
+    case SystemZ::VL64:
+      Changed |= shortenOn0(MI, SystemZ::LD);
+      break;
+
+    case SystemZ::VST64:
+      Changed |= shortenOn0(MI, SystemZ::STD);
+      break;
+    }
+
     unsigned UsedLow = 0;
     unsigned UsedHigh = 0;
     for (auto MOI = MI.operands_begin(), MOE = MI.operands_end();
diff --git a/lib/Target/SystemZ/SystemZSubtarget.cpp b/lib/Target/SystemZ/SystemZSubtarget.cpp
index de725ae6123..0b49fcdd8f7 100644
--- a/lib/Target/SystemZ/SystemZSubtarget.cpp
+++ b/lib/Target/SystemZ/SystemZSubtarget.cpp
@@ -32,8 +32,7 @@ SystemZSubtarget::initializeSubtargetDependencies(StringRef CPU, StringRef FS) {
   return *this;
 }
 
-SystemZSubtarget::SystemZSubtarget(const std::string &TT,
-                                   const std::string &CPU,
+SystemZSubtarget::SystemZSubtarget(const Triple &TT, const std::string &CPU,
                                    const std::string &FS,
                                    const TargetMachine &TM)
     : SystemZGenSubtargetInfo(TT, CPU, FS), HasDistinctOps(false),
@@ -41,8 +40,9 @@ SystemZSubtarget::SystemZSubtarget(const std::string &TT,
       HasPopulationCount(false), HasFastSerialization(false),
       HasInterlockedAccess1(false), HasMiscellaneousExtensions(false),
       HasTransactionalExecution(false), HasProcessorAssist(false),
-      TargetTriple(TT), InstrInfo(initializeSubtargetDependencies(CPU, FS)),
-      TLInfo(TM, *this), TSInfo(*TM.getDataLayout()), FrameLowering() {}
+      HasVector(false), TargetTriple(TT),
+      InstrInfo(initializeSubtargetDependencies(CPU, FS)), TLInfo(TM, *this),
+      TSInfo(), FrameLowering() {}
 
 // Return true if GV binds locally under reloc model RM.
 static bool bindsLocally(const GlobalValue *GV, Reloc::Model RM) {
diff --git a/lib/Target/SystemZ/SystemZSubtarget.h b/lib/Target/SystemZ/SystemZSubtarget.h
index c99e552abb1..f7eaf01cb77 100644
--- a/lib/Target/SystemZ/SystemZSubtarget.h
+++ b/lib/Target/SystemZ/SystemZSubtarget.h
@@ -44,6 +44,7 @@ class SystemZSubtarget : public SystemZGenSubtargetInfo {
   bool HasMiscellaneousExtensions;
   bool HasTransactionalExecution;
   bool HasProcessorAssist;
+  bool HasVector;
 
 private:
   Triple TargetTriple;
@@ -55,7 +56,7 @@ class SystemZSubtarget : public SystemZGenSubtargetInfo {
   SystemZSubtarget &initializeSubtargetDependencies(StringRef CPU,
                                                     StringRef FS);
 public:
-  SystemZSubtarget(const std::string &TT, const std::string &CPU,
+  SystemZSubtarget(const Triple &TT, const std::string &CPU,
                    const std::string &FS, const TargetMachine &TM);
 
   const TargetFrameLowering *getFrameLowering() const override {
@@ -110,6 +111,9 @@ class SystemZSubtarget : public SystemZGenSubtargetInfo {
   // Return true if the target has the processor-assist facility.
   bool hasProcessorAssist() const { return HasProcessorAssist; }
 
+  // Return true if the target has the vector facility.
+  bool hasVector() const { return HasVector; }
+
   // Return true if GV can be accessed using LARL for reloc model RM
   // and code model CM.
   bool isPC32DBLSymbol(const GlobalValue *GV, Reloc::Model RM,
diff --git a/lib/Target/SystemZ/SystemZTargetMachine.cpp b/lib/Target/SystemZ/SystemZTargetMachine.cpp
index b2f8175579f..00cbbd10a81 100644
--- a/lib/Target/SystemZ/SystemZTargetMachine.cpp
+++ b/lib/Target/SystemZ/SystemZTargetMachine.cpp
@@ -21,16 +21,70 @@ extern "C" void LLVMInitializeSystemZTarget() {
   RegisterTargetMachine<SystemZTargetMachine> X(TheSystemZTarget);
 }
 
-SystemZTargetMachine::SystemZTargetMachine(const Target &T, StringRef TT,
+// Determine whether we use the vector ABI.
+static bool UsesVectorABI(StringRef CPU, StringRef FS) {
+  // We use the vector ABI whenever the vector facility is avaiable.
+  // This is the case by default if CPU is z13 or later, and can be
+  // overridden via "[+-]vector" feature string elements.
+  bool VectorABI = true;
+  if (CPU.empty() || CPU == "generic" ||
+      CPU == "z10" || CPU == "z196" || CPU == "zEC12")
+    VectorABI = false;
+
+  SmallVector<StringRef, 3> Features;
+  FS.split(Features, ",", -1, false /* KeepEmpty */);
+  for (auto &Feature : Features) {
+    if (Feature == "vector" || Feature == "+vector")
+      VectorABI = true;
+    if (Feature == "-vector")
+      VectorABI = false;
+  }
+
+  return VectorABI;
+}
+
+static std::string computeDataLayout(const Triple &TT, StringRef CPU,
+                                     StringRef FS) {
+  bool VectorABI = UsesVectorABI(CPU, FS);
+  std::string Ret = "";
+
+  // Big endian.
+  Ret += "E";
+
+  // Data mangling.
+  Ret += DataLayout::getManglingComponent(TT);
+
+  // Make sure that global data has at least 16 bits of alignment by
+  // default, so that we can refer to it using LARL.  We don't have any
+  // special requirements for stack variables though.
+  Ret += "-i1:8:16-i8:8:16";
+
+  // 64-bit integers are naturally aligned.
+  Ret += "-i64:64";
+
+  // 128-bit floats are aligned only to 64 bits.
+  Ret += "-f128:64";
+
+  // When using the vector ABI, 128-bit vectors are also aligned to 64 bits.
+  if (VectorABI)
+    Ret += "-v128:64";
+
+  // We prefer 16 bits of aligned for all globals; see above.
+  Ret += "-a:8:16";
+
+  // Integer registers are 32 or 64 bits.
+  Ret += "-n32:64";
+
+  return Ret;
+}
+
+SystemZTargetMachine::SystemZTargetMachine(const Target &T, const Triple &TT,
                                            StringRef CPU, StringRef FS,
                                            const TargetOptions &Options,
                                            Reloc::Model RM, CodeModel::Model CM,
                                            CodeGenOpt::Level OL)
-    // Make sure that global data has at least 16 bits of alignment by
-    // default, so that we can refer to it using LARL.  We don't have any
-    // special requirements for stack variables though.
-    : LLVMTargetMachine(T, "E-m:e-i1:8:16-i8:8:16-i64:64-f128:64-a:8:16-n32:64",
-                        TT, CPU, FS, Options, RM, CM, OL),
+    : LLVMTargetMachine(T, computeDataLayout(TT, CPU, FS), TT, CPU, FS, Options,
+                        RM, CM, OL),
       TLOF(make_unique<TargetLoweringObjectFileELF>()),
       Subtarget(TT, CPU, FS, *this) {
   initAsmInfo();
diff --git a/lib/Target/SystemZ/SystemZTargetMachine.h b/lib/Target/SystemZ/SystemZTargetMachine.h
index 5ded07c1efb..0a81e1f9fdf 100644
--- a/lib/Target/SystemZ/SystemZTargetMachine.h
+++ b/lib/Target/SystemZ/SystemZTargetMachine.h
@@ -27,7 +27,7 @@ class SystemZTargetMachine : public LLVMTargetMachine {
   SystemZSubtarget        Subtarget;
 
 public:
-  SystemZTargetMachine(const Target &T, StringRef TT, StringRef CPU,
+  SystemZTargetMachine(const Target &T, const Triple &TT, StringRef CPU,
                        StringRef FS, const TargetOptions &Options,
                        Reloc::Model RM, CodeModel::Model CM,
                        CodeGenOpt::Level OL);
diff --git a/lib/Target/SystemZ/SystemZTargetTransformInfo.cpp b/lib/Target/SystemZ/SystemZTargetTransformInfo.cpp
index 3337f6388bd..5a87df1976c 100644
--- a/lib/Target/SystemZ/SystemZTargetTransformInfo.cpp
+++ b/lib/Target/SystemZ/SystemZTargetTransformInfo.cpp
@@ -238,3 +238,21 @@ SystemZTTIImpl::getPopcntSupport(unsigned TyWidth) {
   return TTI::PSK_Software;
 }
 
+unsigned SystemZTTIImpl::getNumberOfRegisters(bool Vector) {
+  if (!Vector)
+    // Discount the stack pointer.  Also leave out %r0, since it can't
+    // be used in an address.
+    return 14;
+  if (ST->hasVector())
+    return 32;
+  return 0;
+}
+
+unsigned SystemZTTIImpl::getRegisterBitWidth(bool Vector) {
+  if (!Vector)
+    return 64;
+  if (ST->hasVector())
+    return 128;
+  return 0;
+}
+
diff --git a/lib/Target/SystemZ/SystemZTargetTransformInfo.h b/lib/Target/SystemZ/SystemZTargetTransformInfo.h
index d4989130679..4b80973ed87 100644
--- a/lib/Target/SystemZ/SystemZTargetTransformInfo.h
+++ b/lib/Target/SystemZ/SystemZTargetTransformInfo.h
@@ -29,7 +29,8 @@ class SystemZTTIImpl : public BasicTTIImplBase<SystemZTTIImpl> {
 
 public:
   explicit SystemZTTIImpl(const SystemZTargetMachine *TM, Function &F)
-    : BaseT(TM), ST(TM->getSubtargetImpl(F)), TLI(ST->getTargetLowering()) {}
+      : BaseT(TM, F.getParent()->getDataLayout()), ST(TM->getSubtargetImpl(F)),
+        TLI(ST->getTargetLowering()) {}
 
   // Provide value semantics. MSVC requires that we spell all of these out.
   SystemZTTIImpl(const SystemZTTIImpl &Arg)
@@ -37,18 +38,6 @@ class SystemZTTIImpl : public BasicTTIImplBase<SystemZTTIImpl> {
   SystemZTTIImpl(SystemZTTIImpl &&Arg)
       : BaseT(std::move(static_cast<BaseT &>(Arg))), ST(std::move(Arg.ST)),
         TLI(std::move(Arg.TLI)) {}
-  SystemZTTIImpl &operator=(const SystemZTTIImpl &RHS) {
-    BaseT::operator=(static_cast<const BaseT &>(RHS));
-    ST = RHS.ST;
-    TLI = RHS.TLI;
-    return *this;
-  }
-  SystemZTTIImpl &operator=(SystemZTTIImpl &&RHS) {
-    BaseT::operator=(std::move(static_cast<BaseT &>(RHS)));
-    ST = std::move(RHS.ST);
-    TLI = std::move(RHS.TLI);
-    return *this;
-  }
 
   /// \name Scalar TTI Implementations
   /// @{
@@ -63,6 +52,14 @@ class SystemZTTIImpl : public BasicTTIImplBase<SystemZTTIImpl> {
   TTI::PopcntSupportKind getPopcntSupport(unsigned TyWidth);
 
   /// @}
+
+  /// \name Vector TTI Implementations
+  /// @{
+
+  unsigned getNumberOfRegisters(bool Vector);
+  unsigned getRegisterBitWidth(bool Vector);
+
+  /// @}
 };
 
 } // end namespace llvm
diff --git a/lib/Target/TargetLoweringObjectFile.cpp b/lib/Target/TargetLoweringObjectFile.cpp
index db543f33392..856d92dcede 100644
--- a/lib/Target/TargetLoweringObjectFile.cpp
+++ b/lib/Target/TargetLoweringObjectFile.cpp
@@ -43,9 +43,8 @@ using namespace llvm;
 void TargetLoweringObjectFile::Initialize(MCContext &ctx,
                                           const TargetMachine &TM) {
   Ctx = &ctx;
-  DL = TM.getDataLayout();
-  InitMCObjectFileInfo(TM.getTargetTriple(),
-                       TM.getRelocationModel(), TM.getCodeModel(), *Ctx);
+  InitMCObjectFileInfo(TM.getTargetTriple(), TM.getRelocationModel(),
+                       TM.getCodeModel(), *Ctx);
 }
 
 TargetLoweringObjectFile::~TargetLoweringObjectFile() {
@@ -107,10 +106,10 @@ MCSymbol *TargetLoweringObjectFile::getSymbolWithGlobalValueBase(
   assert(!Suffix.empty());
 
   SmallString<60> NameStr;
-  NameStr += DL->getPrivateGlobalPrefix();
+  NameStr += GV->getParent()->getDataLayout().getPrivateGlobalPrefix();
   TM.getNameWithPrefix(NameStr, GV, Mang);
   NameStr.append(Suffix.begin(), Suffix.end());
-  return Ctx->GetOrCreateSymbol(NameStr);
+  return Ctx->getOrCreateSymbol(NameStr);
 }
 
 MCSymbol *TargetLoweringObjectFile::getCFIPersonalitySymbol(
@@ -120,7 +119,7 @@ MCSymbol *TargetLoweringObjectFile::getCFIPersonalitySymbol(
 }
 
 void TargetLoweringObjectFile::emitPersonalityValue(MCStreamer &Streamer,
-                                                    const TargetMachine &TM,
+                                                    const DataLayout &,
                                                     const MCSymbol *Sym) const {
 }
 
@@ -200,7 +199,7 @@ SectionKind TargetLoweringObjectFile::getKindForGlobal(const GlobalValue *GV,
       // Otherwise, just drop it into a mergable constant section.  If we have
       // a section for this size, use it, otherwise use the arbitrary sized
       // mergable section.
-      switch (TM.getDataLayout()->getTypeAllocSize(C->getType())) {
+      switch (GV->getParent()->getDataLayout().getTypeAllocSize(C->getType())) {
       case 4:  return SectionKind::getMergeableConst4();
       case 8:  return SectionKind::getMergeableConst8();
       case 16: return SectionKind::getMergeableConst16();
@@ -255,12 +254,13 @@ SectionKind TargetLoweringObjectFile::getKindForGlobal(const GlobalValue *GV,
   llvm_unreachable("Invalid relocation");
 }
 
-/// SectionForGlobal - This method computes the appropriate section to emit
-/// the specified global variable or function definition.  This should not
-/// be passed external (or available externally) globals.
-const MCSection *TargetLoweringObjectFile::
-SectionForGlobal(const GlobalValue *GV, SectionKind Kind, Mangler &Mang,
-                 const TargetMachine &TM) const {
+/// This method computes the appropriate section to emit the specified global
+/// variable or function definition.  This should not be passed external (or
+/// available externally) globals.
+MCSection *
+TargetLoweringObjectFile::SectionForGlobal(const GlobalValue *GV,
+                                           SectionKind Kind, Mangler &Mang,
+                                           const TargetMachine &TM) const {
   // Select section name.
   if (GV->hasSection())
     return getExplicitSectionGlobal(GV, Kind, Mang, TM);
@@ -270,9 +270,10 @@ SectionForGlobal(const GlobalValue *GV, SectionKind Kind, Mangler &Mang,
   return SelectSectionForGlobal(GV, Kind, Mang, TM);
 }
 
-const MCSection *TargetLoweringObjectFile::getSectionForJumpTable(
+MCSection *TargetLoweringObjectFile::getSectionForJumpTable(
     const Function &F, Mangler &Mang, const TargetMachine &TM) const {
-  return getSectionForConstant(SectionKind::getReadOnly(), /*C=*/nullptr);
+  return getSectionForConstant(F.getParent()->getDataLayout(),
+                               SectionKind::getReadOnly(), /*C=*/nullptr);
 }
 
 bool TargetLoweringObjectFile::shouldPutJumpTableInFunctionSection(
@@ -293,12 +294,10 @@ bool TargetLoweringObjectFile::shouldPutJumpTableInFunctionSection(
   return false;
 }
 
-/// getSectionForConstant - Given a mergable constant with the
-/// specified size and relocation information, return a section that it
-/// should be placed in.
-const MCSection *
-TargetLoweringObjectFile::getSectionForConstant(SectionKind Kind,
-                                                const Constant *C) const {
+/// Given a mergable constant with the specified size and relocation
+/// information, return a section that it should be placed in.
+MCSection *TargetLoweringObjectFile::getSectionForConstant(
+    const DataLayout &DL, SectionKind Kind, const Constant *C) const {
   if (Kind.isReadOnly() && ReadOnlySection != nullptr)
     return ReadOnlySection;
 
@@ -313,7 +312,7 @@ const MCExpr *TargetLoweringObjectFile::getTTypeGlobalReference(
     const TargetMachine &TM, MachineModuleInfo *MMI,
     MCStreamer &Streamer) const {
   const MCSymbolRefExpr *Ref =
-      MCSymbolRefExpr::Create(TM.getSymbol(GV, Mang), getContext());
+      MCSymbolRefExpr::create(TM.getSymbol(GV, Mang), getContext());
 
   return getTTypeReference(Ref, Encoding, Streamer);
 }
@@ -330,10 +329,10 @@ getTTypeReference(const MCSymbolRefExpr *Sym, unsigned Encoding,
   case dwarf::DW_EH_PE_pcrel: {
     // Emit a label to the streamer for the current position.  This gives us
     // .-foo addressing.
-    MCSymbol *PCSym = getContext().CreateTempSymbol();
+    MCSymbol *PCSym = getContext().createTempSymbol();
     Streamer.EmitLabel(PCSym);
-    const MCExpr *PC = MCSymbolRefExpr::Create(PCSym, getContext());
-    return MCBinaryExpr::CreateSub(Sym, PC, getContext());
+    const MCExpr *PC = MCSymbolRefExpr::create(PCSym, getContext());
+    return MCBinaryExpr::createSub(Sym, PC, getContext());
   }
   }
 }
@@ -341,7 +340,7 @@ getTTypeReference(const MCSymbolRefExpr *Sym, unsigned Encoding,
 const MCExpr *TargetLoweringObjectFile::getDebugThreadLocalSymbol(const MCSymbol *Sym) const {
   // FIXME: It's not clear what, if any, default this should have - perhaps a
   // null return could mean 'no location' & we should just do that here.
-  return MCSymbolRefExpr::Create(Sym, *Ctx);
+  return MCSymbolRefExpr::create(Sym, *Ctx);
 }
 
 void TargetLoweringObjectFile::getNameWithPrefix(
diff --git a/lib/Target/TargetMachine.cpp b/lib/Target/TargetMachine.cpp
index 9e26b9fbb8c..83174c20c8e 100644
--- a/lib/Target/TargetMachine.cpp
+++ b/lib/Target/TargetMachine.cpp
@@ -34,31 +34,11 @@
 using namespace llvm;
 
 //---------------------------------------------------------------------------
-
-// @LOCALMOD-BEGIN
-namespace llvm {
-  bool TLSUseCall;
-}
-
-// Use a function call to get the thread pointer for TLS accesses,
-// instead of using inline code.
-static cl::opt<bool, true>
-EnableTLSUseCall("mtls-use-call",
-  cl::desc("Use a function call to get the thread pointer for TLS accesses."),
-  cl::location(TLSUseCall),
-  cl::init(false));
-
-static cl::opt<bool>
-  ForceTLSNonPIC("force-tls-non-pic",
-                 cl::desc("Force TLS to use non-PIC models"),
-                 cl::init(false));
-// @LOCALMOD-END
-
 // TargetMachine Class
 //
 
 TargetMachine::TargetMachine(const Target &T, StringRef DataLayoutString,
-                             StringRef TT, StringRef CPU, StringRef FS,
+                             const Triple &TT, StringRef CPU, StringRef FS,
                              const TargetOptions &Options)
     : TheTarget(T), DL(DataLayoutString), TargetTriple(TT), TargetCPU(CPU),
       TargetFS(FS), CodeGenInfo(nullptr), AsmInfo(nullptr), MRI(nullptr),
@@ -74,6 +54,11 @@ TargetMachine::~TargetMachine() {
 }
 
 /// \brief Reset the target options based on the function's attributes.
+// FIXME: This function needs to go away for a number of reasons:
+// a) global state on the TargetMachine is terrible in general,
+// b) there's no default state here to keep,
+// c) these target options should be passed only on the function
+//    and not on the TargetMachine (via TargetOptions) at all.
 void TargetMachine::resetTargetOptions(const Function &F) const {
 #define RESET_OPTION(X, Y)                                                     \
   do {                                                                         \
@@ -81,15 +66,10 @@ void TargetMachine::resetTargetOptions(const Function &F) const {
       Options.X = (F.getFnAttribute(Y).getValueAsString() == "true");          \
   } while (0)
 
-  RESET_OPTION(NoFramePointerElim, "no-frame-pointer-elim");
   RESET_OPTION(LessPreciseFPMADOption, "less-precise-fpmad");
   RESET_OPTION(UnsafeFPMath, "unsafe-fp-math");
   RESET_OPTION(NoInfsFPMath, "no-infs-fp-math");
   RESET_OPTION(NoNaNsFPMath, "no-nans-fp-math");
-  RESET_OPTION(UseSoftFloat, "use-soft-float");
-  RESET_OPTION(DisableTailCalls, "disable-tail-calls");
-
-  Options.MCOptions.SanitizeAddress = F.hasFnAttribute(Attribute::SanitizeAddress);
 }
 
 /// getRelocationModel - Returns the code generation relocation model. The
@@ -136,8 +116,7 @@ TLSModel::Model TargetMachine::getTLSModel(const GlobalValue *GV) const {
   bool isHidden = GV->hasHiddenVisibility();
 
   TLSModel::Model Model;
-  if (isPIC && !isPIE &&
-      !ForceTLSNonPIC) { // @LOCALMOD
+  if (isPIC && !isPIE) {
     if (isLocal || isHidden)
       Model = TLSModel::LocalDynamic;
     else
@@ -171,8 +150,9 @@ void TargetMachine::setOptLevel(CodeGenOpt::Level Level) const {
 }
 
 TargetIRAnalysis TargetMachine::getTargetIRAnalysis() {
-  return TargetIRAnalysis(
-      [this](Function &) { return TargetTransformInfo(getDataLayout()); });
+  return TargetIRAnalysis([this](Function &F) {
+    return TargetTransformInfo(F.getParent()->getDataLayout());
+  });
 }
 
 static bool canUsePrivateLabel(const MCAsmInfo &AsmInfo,
@@ -205,8 +185,8 @@ void TargetMachine::getNameWithPrefix(SmallVectorImpl<char> &Name,
 }
 
 MCSymbol *TargetMachine::getSymbol(const GlobalValue *GV, Mangler &Mang) const {
-  SmallString<60> NameStr;
+  SmallString<128> NameStr;
   getNameWithPrefix(NameStr, GV, Mang);
   const TargetLoweringObjectFile *TLOF = getObjFileLowering();
-  return TLOF->getContext().GetOrCreateSymbol(NameStr);
+  return TLOF->getContext().getOrCreateSymbol(NameStr);
 }
diff --git a/lib/Target/TargetMachineC.cpp b/lib/Target/TargetMachineC.cpp
index 1a5bf163bd9..5913822de23 100644
--- a/lib/Target/TargetMachineC.cpp
+++ b/lib/Target/TargetMachineC.cpp
@@ -32,26 +32,36 @@
 
 using namespace llvm;
 
-inline TargetMachine *unwrap(LLVMTargetMachineRef P) {
-  return reinterpret_cast<TargetMachine*>(P);
+
+// The TargetMachine uses to offer access to a DataLayout member. This is reflected
+// in the C API. For backward compatibility reason, this structure allows to keep
+// a DataLayout member accessible to C client that have a handle to a
+// LLVMTargetMachineRef.
+struct LLVMOpaqueTargetMachine {
+  std::unique_ptr<TargetMachine> Machine;
+  DataLayout DL;
+};
+
+
+static TargetMachine *unwrap(LLVMTargetMachineRef P) {
+  return P->Machine.get();
 }
-inline Target *unwrap(LLVMTargetRef P) {
+static Target *unwrap(LLVMTargetRef P) {
   return reinterpret_cast<Target*>(P);
 }
-inline LLVMTargetMachineRef wrap(const TargetMachine *P) {
-  return
-    reinterpret_cast<LLVMTargetMachineRef>(const_cast<TargetMachine*>(P));
+static LLVMTargetMachineRef wrap(const TargetMachine *P) {
+  return new LLVMOpaqueTargetMachine{ std::unique_ptr<TargetMachine>(const_cast<TargetMachine*>(P)),  P->createDataLayout() };
 }
-inline LLVMTargetRef wrap(const Target * P) {
+static LLVMTargetRef wrap(const Target * P) {
   return reinterpret_cast<LLVMTargetRef>(const_cast<Target*>(P));
 }
 
 LLVMTargetRef LLVMGetFirstTarget() {
-  if(TargetRegistry::begin() == TargetRegistry::end()) {
+  if (TargetRegistry::targets().begin() == TargetRegistry::targets().end()) {
     return nullptr;
   }
 
-  const Target* target = &*TargetRegistry::begin();
+  const Target *target = &*TargetRegistry::targets().begin();
   return wrap(target);
 }
 LLVMTargetRef LLVMGetNextTarget(LLVMTargetRef T) {
@@ -60,13 +70,10 @@ LLVMTargetRef LLVMGetNextTarget(LLVMTargetRef T) {
 
 LLVMTargetRef LLVMGetTargetFromName(const char *Name) {
   StringRef NameRef = Name;
-  for (TargetRegistry::iterator IT = TargetRegistry::begin(),
-                                IE = TargetRegistry::end(); IT != IE; ++IT) {
-    if (IT->getName() == NameRef)
-      return wrap(&*IT);
-  }
-  
-  return nullptr;
+  auto I = std::find_if(
+      TargetRegistry::targets().begin(), TargetRegistry::targets().end(),
+      [&](const Target &T) { return T.getName() == NameRef; });
+  return I != TargetRegistry::targets().end() ? wrap(&*I) : nullptr;
 }
 
 LLVMBool LLVMGetTargetFromTriple(const char* TripleStr, LLVMTargetRef *T,
@@ -150,7 +157,7 @@ LLVMTargetMachineRef LLVMCreateTargetMachine(LLVMTargetRef T,
 
 
 void LLVMDisposeTargetMachine(LLVMTargetMachineRef T) {
-  delete unwrap(T);
+  delete T;
 }
 
 LLVMTargetRef LLVMGetTargetMachineTarget(LLVMTargetMachineRef T) {
@@ -159,7 +166,7 @@ LLVMTargetRef LLVMGetTargetMachineTarget(LLVMTargetMachineRef T) {
 }
 
 char* LLVMGetTargetMachineTriple(LLVMTargetMachineRef T) {
-  std::string StringRep = unwrap(T)->getTargetTriple();
+  std::string StringRep = unwrap(T)->getTargetTriple().str();
   return strdup(StringRep.c_str());
 }
 
@@ -173,8 +180,9 @@ char* LLVMGetTargetMachineFeatureString(LLVMTargetMachineRef T) {
   return strdup(StringRep.c_str());
 }
 
+/// @deprecated: see "struct LLVMOpaqueTargetMachine" description above
 LLVMTargetDataRef LLVMGetTargetMachineData(LLVMTargetMachineRef T) {
-  return wrap(unwrap(T)->getDataLayout());
+  return wrap(&T->DL);
 }
 
 void LLVMSetTargetMachineAsmVerbosity(LLVMTargetMachineRef T,
@@ -193,14 +201,7 @@ static LLVMBool LLVMTargetMachineEmit(LLVMTargetMachineRef T, LLVMModuleRef M,
 
   std::string error;
 
-  const DataLayout *td = TM->getDataLayout();
-
-  if (!td) {
-    error = "No DataLayout in TargetMachine";
-    *ErrorMessage = strdup(error.c_str());
-    return true;
-  }
-  Mod->setDataLayout(*td);
+  Mod->setDataLayout(TM->createDataLayout());
 
   TargetMachine::CodeGenFileType ft;
   switch (codegen) {
diff --git a/lib/Target/TargetRecip.cpp b/lib/Target/TargetRecip.cpp
new file mode 100644
index 00000000000..42bc487fe6d
--- /dev/null
+++ b/lib/Target/TargetRecip.cpp
@@ -0,0 +1,225 @@
+//===-------------------------- TargetRecip.cpp ---------------------------===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This class is used to customize machine-specific reciprocal estimate code
+// generation in a target-independent way.
+// If a target does not support operations in this specification, then code
+// generation will default to using supported operations.
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/ADT/StringRef.h"
+#include "llvm/ADT/STLExtras.h"
+#include "llvm/Support/ErrorHandling.h"
+#include "llvm/Target/TargetRecip.h"
+#include <map>
+
+using namespace llvm;
+
+// These are the names of the individual reciprocal operations. These are
+// the key strings for queries and command-line inputs.
+// In addition, the command-line interface recognizes the global parameters
+// "all", "none", and "default".
+static const char *RecipOps[] = {
+  "divd",
+  "divf",
+  "vec-divd",
+  "vec-divf",
+  "sqrtd",
+  "sqrtf",
+  "vec-sqrtd",
+  "vec-sqrtf",
+};
+
+// The uninitialized state is needed for the enabled settings and refinement
+// steps because custom settings may arrive via the command-line before target
+// defaults are set.
+TargetRecip::TargetRecip() {
+  unsigned NumStrings = llvm::array_lengthof(RecipOps);
+  for (unsigned i = 0; i < NumStrings; ++i)
+    RecipMap.insert(std::make_pair(RecipOps[i], RecipParams()));
+}
+
+static bool parseRefinementStep(const StringRef &In, size_t &Position,
+                                uint8_t &Value) {
+  const char RefStepToken = ':';
+  Position = In.find(RefStepToken);
+  if (Position == StringRef::npos)
+    return false;
+
+  StringRef RefStepString = In.substr(Position + 1);
+  // Allow exactly one numeric character for the additional refinement
+  // step parameter.
+  if (RefStepString.size() == 1) {
+    char RefStepChar = RefStepString[0];
+    if (RefStepChar >= '0' && RefStepChar <= '9') {
+      Value = RefStepChar - '0';
+      return true;
+    }
+  }
+  report_fatal_error("Invalid refinement step for -recip.");
+}
+
+bool TargetRecip::parseGlobalParams(const std::string &Arg) {
+  StringRef ArgSub = Arg;
+
+  // Look for an optional setting of the number of refinement steps needed
+  // for this type of reciprocal operation.
+  size_t RefPos;
+  uint8_t RefSteps;
+  StringRef RefStepString;
+  if (parseRefinementStep(ArgSub, RefPos, RefSteps)) {
+    // Split the string for further processing.
+    RefStepString = ArgSub.substr(RefPos + 1);
+    ArgSub = ArgSub.substr(0, RefPos);
+  }
+  bool Enable;
+  bool UseDefaults;
+  if (ArgSub == "all") {
+    UseDefaults = false;
+    Enable = true;
+  } else if (ArgSub == "none") {
+    UseDefaults = false;
+    Enable = false;
+  } else if (ArgSub == "default") {
+    UseDefaults = true;
+  } else {
+    // Any other string is invalid or an individual setting.
+    return false;
+  }
+
+  // All enable values will be initialized to target defaults if 'default' was
+  // specified.
+  if (!UseDefaults)
+    for (auto &KV : RecipMap)
+      KV.second.Enabled = Enable;
+
+  // Custom refinement count was specified with all, none, or default.
+  if (!RefStepString.empty())
+    for (auto &KV : RecipMap)
+      KV.second.RefinementSteps = RefSteps;
+  
+  return true;
+}
+
+void TargetRecip::parseIndividualParams(const std::vector<std::string> &Args) {
+  static const char DisabledPrefix = '!';
+  unsigned NumArgs = Args.size();
+
+  for (unsigned i = 0; i != NumArgs; ++i) {
+    StringRef Val = Args[i];
+    
+    bool IsDisabled = Val[0] == DisabledPrefix;
+    // Ignore the disablement token for string matching.
+    if (IsDisabled)
+      Val = Val.substr(1);
+    
+    size_t RefPos;
+    uint8_t RefSteps;
+    StringRef RefStepString;
+    if (parseRefinementStep(Val, RefPos, RefSteps)) {
+      // Split the string for further processing.
+      RefStepString = Val.substr(RefPos + 1);
+      Val = Val.substr(0, RefPos);
+    }
+
+    RecipIter Iter = RecipMap.find(Val);
+    if (Iter == RecipMap.end()) {
+      // Try again specifying float suffix.
+      Iter = RecipMap.find(Val.str() + 'f');
+      if (Iter == RecipMap.end()) {
+        Iter = RecipMap.find(Val.str() + 'd');
+        assert(Iter == RecipMap.end() && "Float entry missing from map");
+        report_fatal_error("Invalid option for -recip.");
+      }
+      
+      // The option was specified without a float or double suffix.
+      if (RecipMap[Val.str() + 'd'].Enabled != Uninitialized) {
+        // Make sure that the double entry was not already specified.
+        // The float entry will be checked below.
+        report_fatal_error("Duplicate option for -recip.");
+      }
+    }
+    
+    if (Iter->second.Enabled != Uninitialized)
+      report_fatal_error("Duplicate option for -recip.");
+    
+    // Mark the matched option as found. Do not allow duplicate specifiers.
+    Iter->second.Enabled = !IsDisabled;
+    if (!RefStepString.empty())
+      Iter->second.RefinementSteps = RefSteps;
+    
+    // If the precision was not specified, the double entry is also initialized.
+    if (Val.back() != 'f' && Val.back() != 'd') {
+      RecipMap[Val.str() + 'd'].Enabled = !IsDisabled;
+      if (!RefStepString.empty())
+        RecipMap[Val.str() + 'd'].RefinementSteps = RefSteps;
+    }
+  }
+}
+
+TargetRecip::TargetRecip(const std::vector<std::string> &Args) :
+  TargetRecip() {
+  unsigned NumArgs = Args.size();
+
+  // Check if "all", "default", or "none" was specified.
+  if (NumArgs == 1 && parseGlobalParams(Args[0]))
+    return;
+ 
+  parseIndividualParams(Args);
+}
+
+bool TargetRecip::isEnabled(const StringRef &Key) const {
+  ConstRecipIter Iter = RecipMap.find(Key);
+  assert(Iter != RecipMap.end() && "Unknown name for reciprocal map");
+  assert(Iter->second.Enabled != Uninitialized &&
+         "Enablement setting was not initialized");
+  return Iter->second.Enabled;
+}
+
+unsigned TargetRecip::getRefinementSteps(const StringRef &Key) const {
+  ConstRecipIter Iter = RecipMap.find(Key);
+  assert(Iter != RecipMap.end() && "Unknown name for reciprocal map");
+  assert(Iter->second.RefinementSteps != Uninitialized &&
+         "Refinement step setting was not initialized");
+  return Iter->second.RefinementSteps;
+}
+
+/// Custom settings (previously initialized values) override target defaults.
+void TargetRecip::setDefaults(const StringRef &Key, bool Enable,
+                              unsigned RefSteps) {
+  if (Key == "all") {
+    for (auto &KV : RecipMap) {
+      RecipParams &RP = KV.second;
+      if (RP.Enabled == Uninitialized)
+        RP.Enabled = Enable;
+      if (RP.RefinementSteps == Uninitialized)
+        RP.RefinementSteps = RefSteps;
+    }
+  } else {
+    RecipParams &RP = RecipMap[Key];
+    if (RP.Enabled == Uninitialized)
+      RP.Enabled = Enable;
+    if (RP.RefinementSteps == Uninitialized)
+      RP.RefinementSteps = RefSteps;
+  }
+}
+
+bool TargetRecip::operator==(const TargetRecip &Other) const {
+  for (const auto &KV : RecipMap) {
+    const StringRef &Op = KV.first;
+    const RecipParams &RP = KV.second;
+    const RecipParams &OtherRP = Other.RecipMap.find(Op)->second;
+    if (RP.RefinementSteps != OtherRP.RefinementSteps)
+      return false;
+    if (RP.Enabled != OtherRP.Enabled)
+      return false;
+  }
+  return true;
+}
diff --git a/lib/Target/TargetSubtargetInfo.cpp b/lib/Target/TargetSubtargetInfo.cpp
index b2bb59ea28c..6a61fcdf0f8 100644
--- a/lib/Target/TargetSubtargetInfo.cpp
+++ b/lib/Target/TargetSubtargetInfo.cpp
@@ -19,7 +19,14 @@ using namespace llvm;
 //---------------------------------------------------------------------------
 // TargetSubtargetInfo Class
 //
-TargetSubtargetInfo::TargetSubtargetInfo() {}
+TargetSubtargetInfo::TargetSubtargetInfo(
+    const Triple &TT, StringRef CPU, StringRef FS,
+    ArrayRef<SubtargetFeatureKV> PF, ArrayRef<SubtargetFeatureKV> PD,
+    const SubtargetInfoKV *ProcSched, const MCWriteProcResEntry *WPR,
+    const MCWriteLatencyEntry *WL, const MCReadAdvanceEntry *RA,
+    const InstrStage *IS, const unsigned *OC, const unsigned *FP)
+    : MCSubtargetInfo(TT, CPU, FS, PF, PD, ProcSched, WPR, WL, RA, IS, OC, FP) {
+}
 
 TargetSubtargetInfo::~TargetSubtargetInfo() {}
 
@@ -40,7 +47,7 @@ bool TargetSubtargetInfo::enableRALocalReassignment(
   return true;
 }
 
-bool TargetSubtargetInfo::enablePostMachineScheduler() const {
+bool TargetSubtargetInfo::enablePostRAScheduler() const {
   return getSchedModel().PostRAScheduler;
 }
 
diff --git a/lib/Target/WebAssembly/CMakeLists.txt b/lib/Target/WebAssembly/CMakeLists.txt
new file mode 100644
index 00000000000..a1099c535b1
--- /dev/null
+++ b/lib/Target/WebAssembly/CMakeLists.txt
@@ -0,0 +1,29 @@
+set(LLVM_TARGET_DEFINITIONS WebAssembly.td)
+
+tablegen(LLVM WebAssemblyGenAsmWriter.inc -gen-asm-writer)
+tablegen(LLVM WebAssemblyGenDAGISel.inc -gen-dag-isel)
+tablegen(LLVM WebAssemblyGenInstrInfo.inc -gen-instr-info)
+tablegen(LLVM WebAssemblyGenMCCodeEmitter.inc -gen-emitter)
+tablegen(LLVM WebAssemblyGenRegisterInfo.inc -gen-register-info)
+tablegen(LLVM WebAssemblyGenSubtargetInfo.inc -gen-subtarget)
+add_public_tablegen_target(WebAssemblyCommonTableGen)
+
+add_llvm_target(WebAssemblyCodeGen
+  WebAssemblyAsmPrinter.cpp
+  WebAssemblyFrameLowering.cpp
+  WebAssemblyISelDAGToDAG.cpp
+  WebAssemblyISelLowering.cpp
+  WebAssemblyInstrInfo.cpp
+  WebAssemblyMachineFunctionInfo.cpp
+  WebAssemblyRegisterInfo.cpp
+  WebAssemblySelectionDAGInfo.cpp
+  WebAssemblySubtarget.cpp
+  WebAssemblyTargetMachine.cpp
+  WebAssemblyTargetTransformInfo.cpp
+)
+
+add_dependencies(LLVMWebAssemblyCodeGen intrinsics_gen)
+
+add_subdirectory(InstPrinter)
+add_subdirectory(TargetInfo)
+add_subdirectory(MCTargetDesc)
diff --git a/lib/Target/WebAssembly/InstPrinter/CMakeLists.txt b/lib/Target/WebAssembly/InstPrinter/CMakeLists.txt
new file mode 100644
index 00000000000..5394b67d2b8
--- /dev/null
+++ b/lib/Target/WebAssembly/InstPrinter/CMakeLists.txt
@@ -0,0 +1,3 @@
+add_llvm_library(LLVMWebAssemblyAsmPrinter
+  WebAssemblyInstPrinter.cpp
+  )
diff --git a/lib/Target/WebAssembly/InstPrinter/LLVMBuild.txt b/lib/Target/WebAssembly/InstPrinter/LLVMBuild.txt
new file mode 100644
index 00000000000..54df6d65570
--- /dev/null
+++ b/lib/Target/WebAssembly/InstPrinter/LLVMBuild.txt
@@ -0,0 +1,23 @@
+;===- ./lib/Target/WebAssembly/InstPrinter/LLVMBuild.txt -------*- Conf -*--===;
+;
+;                     The LLVM Compiler Infrastructure
+;
+; This file is distributed under the University of Illinois Open Source
+; License. See LICENSE.TXT for details.
+;
+;===------------------------------------------------------------------------===;
+;
+; This is an LLVMBuild description file for the components in this subdirectory.
+;
+; For more information on the LLVMBuild system, please see:
+;
+;   http://llvm.org/docs/LLVMBuild.html
+;
+;===------------------------------------------------------------------------===;
+
+[component_0]
+type = Library
+name = WebAssemblyAsmPrinter
+parent = WebAssembly
+required_libraries = MC Support
+add_to_library_groups = WebAssembly
diff --git a/lib/Target/WebAssembly/InstPrinter/Makefile b/lib/Target/WebAssembly/InstPrinter/Makefile
new file mode 100644
index 00000000000..87534379f79
--- /dev/null
+++ b/lib/Target/WebAssembly/InstPrinter/Makefile
@@ -0,0 +1,16 @@
+##===- lib/Target/WebAssembly/AsmPrinter/Makefile ----------*- Makefile -*-===##
+#
+#                     The LLVM Compiler Infrastructure
+#
+# This file is distributed under the University of Illinois Open Source
+# License. See LICENSE.TXT for details.
+#
+##===----------------------------------------------------------------------===##
+
+LEVEL = ../../../..
+LIBRARYNAME = LLVMWebAssemblyAsmPrinter
+
+# Hack: we need to include 'main' wasm target directory to grab private headers
+CPP.Flags += -I$(PROJ_OBJ_DIR)/.. -I$(PROJ_SRC_DIR)/..
+
+include $(LEVEL)/Makefile.common
diff --git a/lib/Target/WebAssembly/InstPrinter/WebAssemblyInstPrinter.cpp b/lib/Target/WebAssembly/InstPrinter/WebAssemblyInstPrinter.cpp
new file mode 100644
index 00000000000..0c5792c6b51
--- /dev/null
+++ b/lib/Target/WebAssembly/InstPrinter/WebAssemblyInstPrinter.cpp
@@ -0,0 +1,46 @@
+//=- WebAssemblyInstPrinter.cpp - WebAssembly assembly instruction printing -=//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+///
+/// \file
+/// \brief Print MCInst instructions to wasm format.
+///
+//===----------------------------------------------------------------------===//
+
+#include "InstPrinter/WebAssemblyInstPrinter.h"
+#include "WebAssembly.h"
+#include "llvm/MC/MCExpr.h"
+#include "llvm/MC/MCInst.h"
+#include "llvm/MC/MCInstrInfo.h"
+#include "llvm/MC/MCSubtargetInfo.h"
+#include "llvm/MC/MCSymbol.h"
+#include "llvm/Support/ErrorHandling.h"
+#include "llvm/Support/FormattedStream.h"
+#include <cctype>
+using namespace llvm;
+
+#define DEBUG_TYPE "asm-printer"
+
+#include "WebAssemblyGenAsmWriter.inc"
+
+WebAssemblyInstPrinter::WebAssemblyInstPrinter(const MCAsmInfo &MAI,
+                                               const MCInstrInfo &MII,
+                                               const MCRegisterInfo &MRI)
+    : MCInstPrinter(MAI, MII, MRI) {}
+
+void WebAssemblyInstPrinter::printRegName(raw_ostream &OS,
+                                          unsigned RegNo) const {
+  OS << getRegisterName(RegNo);
+}
+
+void WebAssemblyInstPrinter::printInst(const MCInst *MI, raw_ostream &OS,
+                                       StringRef Annot,
+                                       const MCSubtargetInfo &STI) {
+  printInstruction(MI, OS);
+  printAnnotation(OS, Annot);
+}
diff --git a/lib/Target/WebAssembly/InstPrinter/WebAssemblyInstPrinter.h b/lib/Target/WebAssembly/InstPrinter/WebAssemblyInstPrinter.h
new file mode 100644
index 00000000000..4c54a525577
--- /dev/null
+++ b/lib/Target/WebAssembly/InstPrinter/WebAssemblyInstPrinter.h
@@ -0,0 +1,42 @@
+// WebAssemblyInstPrinter.h - Print wasm MCInst to assembly syntax -*- C++ -*-//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+///
+/// \file
+/// \brief This class prints an WebAssembly MCInst to wasm file syntax.
+///
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIB_TARGET_WEBASSEMBLY_INSTPRINTER_WEBASSEMBLYINSTPRINTER_H
+#define LLVM_LIB_TARGET_WEBASSEMBLY_INSTPRINTER_WEBASSEMBLYINSTPRINTER_H
+
+#include "llvm/MC/MCInstPrinter.h"
+#include "llvm/Support/raw_ostream.h"
+
+namespace llvm {
+
+class MCOperand;
+class MCSubtargetInfo;
+
+class WebAssemblyInstPrinter : public MCInstPrinter {
+public:
+  WebAssemblyInstPrinter(const MCAsmInfo &MAI, const MCInstrInfo &MII,
+                         const MCRegisterInfo &MRI);
+
+  void printRegName(raw_ostream &OS, unsigned RegNo) const override;
+  void printInst(const MCInst *MI, raw_ostream &OS, StringRef Annot,
+                 const MCSubtargetInfo &STI) override;
+
+  // Autogenerated by tblgen.
+  void printInstruction(const MCInst *MI, raw_ostream &O);
+  static const char *getRegisterName(unsigned RegNo);
+};
+
+} // end namespace llvm
+
+#endif
diff --git a/lib/Target/WebAssembly/LLVMBuild.txt b/lib/Target/WebAssembly/LLVMBuild.txt
new file mode 100644
index 00000000000..04ef9c4e4bc
--- /dev/null
+++ b/lib/Target/WebAssembly/LLVMBuild.txt
@@ -0,0 +1,32 @@
+;===- ./lib/Target/WebAssembly/LLVMBuild.txt -------------------*- Conf -*--===;
+;
+;                     The LLVM Compiler Infrastructure
+;
+; This file is distributed under the University of Illinois Open Source
+; License. See LICENSE.TXT for details.
+;
+;===------------------------------------------------------------------------===;
+;
+; This is an LLVMBuild description file for the components in this subdirectory.
+;
+; For more information on the LLVMBuild system, please see:
+;
+;   http://llvm.org/docs/LLVMBuild.html
+;
+;===------------------------------------------------------------------------===;
+
+[common]
+subdirectories = InstPrinter MCTargetDesc TargetInfo
+
+[component_0]
+type = TargetGroup
+name = WebAssembly
+parent = Target
+has_asmprinter = 1
+
+[component_1]
+type = Library
+name = WebAssemblyCodeGen
+parent = WebAssembly
+required_libraries = Analysis AsmPrinter CodeGen Core MC Scalar SelectionDAG Support Target WebAssemblyDesc WebAssemblyInfo
+add_to_library_groups = WebAssembly
diff --git a/lib/Target/WebAssembly/MCTargetDesc/CMakeLists.txt b/lib/Target/WebAssembly/MCTargetDesc/CMakeLists.txt
new file mode 100644
index 00000000000..ccc0f0d7ccb
--- /dev/null
+++ b/lib/Target/WebAssembly/MCTargetDesc/CMakeLists.txt
@@ -0,0 +1,4 @@
+add_llvm_library(LLVMWebAssemblyDesc
+  WebAssemblyMCAsmInfo.cpp
+  WebAssemblyMCTargetDesc.cpp
+)
diff --git a/lib/Target/WebAssembly/MCTargetDesc/LLVMBuild.txt b/lib/Target/WebAssembly/MCTargetDesc/LLVMBuild.txt
new file mode 100644
index 00000000000..ce7cb5dd4da
--- /dev/null
+++ b/lib/Target/WebAssembly/MCTargetDesc/LLVMBuild.txt
@@ -0,0 +1,23 @@
+;===- ./lib/Target/WebAssembly/MCTargetDesc/LLVMBuild.txt ------*- Conf -*--===;
+;
+;                     The LLVM Compiler Infrastructure
+;
+; This file is distributed under the University of Illinois Open Source
+; License. See LICENSE.TXT for details.
+;
+;===------------------------------------------------------------------------===;
+;
+; This is an LLVMBuild description file for the components in this subdirectory.
+;
+; For more information on the LLVMBuild system, please see:
+;
+;   http://llvm.org/docs/LLVMBuild.html
+;
+;===------------------------------------------------------------------------===;
+
+[component_0]
+type = Library
+name = WebAssemblyDesc
+parent = WebAssembly
+required_libraries = MC Support WebAssemblyAsmPrinter WebAssemblyInfo
+add_to_library_groups = WebAssembly
diff --git a/lib/Target/WebAssembly/MCTargetDesc/Makefile b/lib/Target/WebAssembly/MCTargetDesc/Makefile
new file mode 100644
index 00000000000..11dcb4ff607
--- /dev/null
+++ b/lib/Target/WebAssembly/MCTargetDesc/Makefile
@@ -0,0 +1,16 @@
+##===- lib/Target/WebAssembly/TargetDesc/Makefile ----------*- Makefile -*-===##
+#
+#                     The LLVM Compiler Infrastructure
+#
+# This file is distributed under the University of Illinois Open Source
+# License. See LICENSE.TXT for details.
+#
+##===----------------------------------------------------------------------===##
+
+LEVEL = ../../../..
+LIBRARYNAME = LLVMWebAssemblyDesc
+
+# Hack: we need to include 'main' target directory to grab private headers
+CPP.Flags += -I$(PROJ_OBJ_DIR)/.. -I$(PROJ_SRC_DIR)/..
+
+include $(LEVEL)/Makefile.common
diff --git a/lib/Target/WebAssembly/MCTargetDesc/WebAssemblyMCAsmInfo.cpp b/lib/Target/WebAssembly/MCTargetDesc/WebAssemblyMCAsmInfo.cpp
new file mode 100644
index 00000000000..c3ab48fd405
--- /dev/null
+++ b/lib/Target/WebAssembly/MCTargetDesc/WebAssemblyMCAsmInfo.cpp
@@ -0,0 +1,56 @@
+//===-- WebAssemblyMCAsmInfo.cpp - WebAssembly asm properties -------------===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+///
+/// \file
+/// \brief This file contains the declarations of the WebAssemblyMCAsmInfo
+/// properties.
+///
+//===----------------------------------------------------------------------===//
+
+#include "WebAssemblyMCAsmInfo.h"
+#include "llvm/ADT/Triple.h"
+#include "llvm/Support/CommandLine.h"
+using namespace llvm;
+
+#define DEBUG_TYPE "wasm-mc-asm-info"
+
+WebAssemblyMCAsmInfo::~WebAssemblyMCAsmInfo() {}
+
+WebAssemblyMCAsmInfo::WebAssemblyMCAsmInfo(const Triple &T) {
+  PointerSize = CalleeSaveStackSlotSize = T.isArch64Bit() ? 8 : 4;
+
+  // TODO: What should MaxInstLength be?
+
+  // The s-expression format of WebAssembly uses LISP-style comments.
+  CommentString = ";";
+
+  PrivateGlobalPrefix = "";
+  PrivateLabelPrefix = "";
+
+  UseDataRegionDirectives = true;
+
+  Data8bitsDirective = "\t.int8\t";
+  Data16bitsDirective = "\t.int16\t";
+  Data32bitsDirective = "\t.int32\t";
+  Data64bitsDirective = "\t.int64\t";
+
+  AlignmentIsInBytes = false;
+  COMMDirectiveAlignmentIsInBytes = false;
+  LCOMMDirectiveAlignmentType = LCOMM::Log2Alignment;
+
+  HasDotTypeDotSizeDirective = false;
+  HasSingleParameterDotFile = false;
+
+  SupportsDebugInformation = true;
+
+  // For now, WebAssembly does not support exceptions.
+  ExceptionsType = ExceptionHandling::None;
+
+  // TODO: UseIntegratedAssembler?
+}
diff --git a/lib/Target/WebAssembly/MCTargetDesc/WebAssemblyMCAsmInfo.h b/lib/Target/WebAssembly/MCTargetDesc/WebAssemblyMCAsmInfo.h
new file mode 100644
index 00000000000..d2b8fb7748f
--- /dev/null
+++ b/lib/Target/WebAssembly/MCTargetDesc/WebAssemblyMCAsmInfo.h
@@ -0,0 +1,32 @@
+//===-- WebAssemblyMCAsmInfo.h - WebAssembly asm properties -----*- C++ -*-===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+///
+/// \file
+/// \brief This file contains the declaration of the WebAssemblyMCAsmInfo class.
+///
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIB_TARGET_WEBASSEMBLY_MCTARGETDESC_WEBASSEMBLYMCASMINFO_H
+#define LLVM_LIB_TARGET_WEBASSEMBLY_MCTARGETDESC_WEBASSEMBLYMCASMINFO_H
+
+#include "llvm/MC/MCAsmInfo.h"
+
+namespace llvm {
+
+class Triple;
+
+class WebAssemblyMCAsmInfo final : public MCAsmInfo {
+public:
+  explicit WebAssemblyMCAsmInfo(const Triple &T);
+  ~WebAssemblyMCAsmInfo() override;
+};
+
+} // end namespace llvm
+
+#endif
diff --git a/lib/Target/WebAssembly/MCTargetDesc/WebAssemblyMCTargetDesc.cpp b/lib/Target/WebAssembly/MCTargetDesc/WebAssemblyMCTargetDesc.cpp
new file mode 100644
index 00000000000..0644fd80bdf
--- /dev/null
+++ b/lib/Target/WebAssembly/MCTargetDesc/WebAssemblyMCTargetDesc.cpp
@@ -0,0 +1,60 @@
+//===-- WebAssemblyMCTargetDesc.cpp - WebAssembly Target Descriptions -----===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+///
+/// \file
+/// \brief This file provides WebAssembly-specific target descriptions.
+///
+//===----------------------------------------------------------------------===//
+
+#include "WebAssemblyMCTargetDesc.h"
+#include "InstPrinter/WebAssemblyInstPrinter.h"
+#include "WebAssemblyMCAsmInfo.h"
+#include "llvm/MC/MCCodeGenInfo.h"
+#include "llvm/MC/MCInstrInfo.h"
+#include "llvm/MC/MCRegisterInfo.h"
+#include "llvm/MC/MCStreamer.h"
+#include "llvm/MC/MCSubtargetInfo.h"
+#include "llvm/Support/ErrorHandling.h"
+#include "llvm/Support/TargetRegistry.h"
+using namespace llvm;
+
+#define DEBUG_TYPE "wasm-mc-target-desc"
+
+#define GET_INSTRINFO_MC_DESC
+#include "WebAssemblyGenInstrInfo.inc"
+
+#define GET_SUBTARGETINFO_MC_DESC
+#include "WebAssemblyGenSubtargetInfo.inc"
+
+#define GET_REGINFO_MC_DESC
+#include "WebAssemblyGenRegisterInfo.inc"
+
+static MCAsmInfo *createWebAssemblyMCAsmInfo(const MCRegisterInfo &MRI,
+                                             const Triple &TT) {
+  return new WebAssemblyMCAsmInfo(TT);
+}
+
+static MCInstPrinter *
+createWebAssemblyMCInstPrinter(const Triple &T, unsigned SyntaxVariant,
+                               const MCAsmInfo &MAI, const MCInstrInfo &MII,
+                               const MCRegisterInfo &MRI) {
+  assert(SyntaxVariant == 0);
+  return new WebAssemblyInstPrinter(MAI, MII, MRI);
+}
+
+// Force static initialization.
+extern "C" void LLVMInitializeWebAssemblyTargetMC() {
+  for (Target *T : {&TheWebAssemblyTarget32, &TheWebAssemblyTarget64}) {
+    // Register the MC asm info.
+    RegisterMCAsmInfoFn X(*T, createWebAssemblyMCAsmInfo);
+
+    // Register the MCInstPrinter.
+    TargetRegistry::RegisterMCInstPrinter(*T, createWebAssemblyMCInstPrinter);
+  }
+}
diff --git a/lib/Target/WebAssembly/MCTargetDesc/WebAssemblyMCTargetDesc.h b/lib/Target/WebAssembly/MCTargetDesc/WebAssemblyMCTargetDesc.h
new file mode 100644
index 00000000000..bb788dce03c
--- /dev/null
+++ b/lib/Target/WebAssembly/MCTargetDesc/WebAssemblyMCTargetDesc.h
@@ -0,0 +1,61 @@
+//==- WebAssemblyMCTargetDesc.h - WebAssembly Target Descriptions -*- C++ -*-=//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+///
+/// \file
+/// \brief This file provides WebAssembly-specific target descriptions.
+///
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIB_TARGET_WEBASSEMBLY_MCTARGETDESC_WEBASSEMBLYMCTARGETDESC_H
+#define LLVM_LIB_TARGET_WEBASSEMBLY_MCTARGETDESC_WEBASSEMBLYMCTARGETDESC_H
+
+#include "llvm/Support/DataTypes.h"
+#include <string>
+
+namespace llvm {
+
+class formatted_raw_ostream;
+class MCAsmBackend;
+class MCCodeEmitter;
+class MCContext;
+class MCInstrInfo;
+class MCRegisterInfo;
+class MCObjectWriter;
+class MCStreamer;
+class MCSubtargetInfo;
+class MCTargetStreamer;
+class StringRef;
+class Target;
+class Triple;
+class raw_ostream;
+
+extern Target TheWebAssemblyTarget32;
+extern Target TheWebAssemblyTarget64;
+
+MCAsmBackend *createWebAssemblyAsmBackend(const Target &T,
+                                          const MCRegisterInfo &MRI,
+                                          StringRef TT, StringRef CPU);
+
+} // end namespace llvm
+
+// Defines symbolic names for WebAssembly registers. This defines a mapping from
+// register name to register number.
+//
+#define GET_REGINFO_ENUM
+#include "WebAssemblyGenRegisterInfo.inc"
+
+// Defines symbolic names for the WebAssembly instructions.
+//
+#define GET_INSTRINFO_ENUM
+#include "WebAssemblyGenInstrInfo.inc"
+
+#define GET_SUBTARGETINFO_ENUM
+#include "WebAssemblyGenSubtargetInfo.inc"
+
+#endif
diff --git a/lib/Target/WebAssembly/Makefile b/lib/Target/WebAssembly/Makefile
new file mode 100644
index 00000000000..a58e82d0c49
--- /dev/null
+++ b/lib/Target/WebAssembly/Makefile
@@ -0,0 +1,25 @@
+##===- lib/Target/WebAssembly/Makefile ---------------------*- Makefile -*-===##
+#
+#                     The LLVM Compiler Infrastructure
+#
+# This file is distributed under the University of Illinois Open Source
+# License. See LICENSE.TXT for details.
+#
+##===----------------------------------------------------------------------===##
+
+LEVEL = ../../..
+LIBRARYNAME = LLVMWebAssemblyCodeGen
+TARGET = WebAssembly
+
+# Make sure that tblgen is run, first thing.
+BUILT_SOURCES = \
+	WebAssemblyGenAsmWriter.inc \
+	WebAssemblyGenDAGISel.inc \
+	WebAssemblyGenInstrInfo.inc \
+	WebAssemblyGenMCCodeEmitter.inc \
+	WebAssemblyGenRegisterInfo.inc \
+	WebAssemblyGenSubtargetInfo.inc
+
+DIRS = InstPrinter TargetInfo MCTargetDesc
+
+include $(LEVEL)/Makefile.common
diff --git a/lib/Target/WebAssembly/README.txt b/lib/Target/WebAssembly/README.txt
new file mode 100644
index 00000000000..63e02c45589
--- /dev/null
+++ b/lib/Target/WebAssembly/README.txt
@@ -0,0 +1,26 @@
+//===-- README.txt - Notes for WebAssembly code gen -----------------------===//
+
+This WebAssembly backend is presently in a very early stage of development.
+The code should build and not break anything else, but don't expect a lot more
+at this point.
+
+For more information on WebAssembly itself, see the design documents:
+  * https://github.com/WebAssembly/design/blob/master/README.md
+
+The following documents contain some information on the planned semantics and
+binary encoding of WebAssembly itself:
+  * https://github.com/WebAssembly/design/blob/master/AstSemantics.md
+  * https://github.com/WebAssembly/design/blob/master/BinaryEncoding.md
+
+Interesting work that remains to be done:
+* Write a pass to restructurize irreducible control flow. This needs to be done
+  before register allocation to be efficient, because it may duplicate basic
+  blocks and WebAssembly performs register allocation at a whole-function
+  level. Note that LLVM's GPU code has such a pass, but it linearizes control
+  flow (e.g. both sides of branches execute and are masked) which is undesirable
+  for WebAssembly.
+* Basic relooper to expose control flow as an AST.
+* Figure out how to properly use MC for virtual ISAs. This may require some
+  refactoring of MC.
+
+//===---------------------------------------------------------------------===//
diff --git a/lib/Target/WebAssembly/TargetInfo/CMakeLists.txt b/lib/Target/WebAssembly/TargetInfo/CMakeLists.txt
new file mode 100644
index 00000000000..ef6e4d2b617
--- /dev/null
+++ b/lib/Target/WebAssembly/TargetInfo/CMakeLists.txt
@@ -0,0 +1,7 @@
+include_directories( ${CMAKE_CURRENT_BINARY_DIR}/.. ${CMAKE_CURRENT_SOURCE_DIR}/.. )
+
+add_llvm_library(LLVMWebAssemblyInfo
+  WebAssemblyTargetInfo.cpp
+  )
+
+add_dependencies(LLVMWebAssemblyInfo WebAssemblyCommonTableGen)
diff --git a/lib/Target/WebAssembly/TargetInfo/LLVMBuild.txt b/lib/Target/WebAssembly/TargetInfo/LLVMBuild.txt
new file mode 100644
index 00000000000..f4da9239bbe
--- /dev/null
+++ b/lib/Target/WebAssembly/TargetInfo/LLVMBuild.txt
@@ -0,0 +1,23 @@
+;===- ./lib/Target/WebAssembly/TargetInfo/LLVMBuild.txt --------*- Conf -*--===;
+;
+;                     The LLVM Compiler Infrastructure
+;
+; This file is distributed under the University of Illinois Open Source
+; License. See LICENSE.TXT for details.
+;
+;===------------------------------------------------------------------------===;
+;
+; This is an LLVMBuild description file for the components in this subdirectory.
+;
+; For more information on the LLVMBuild system, please see:
+;
+;   http://llvm.org/docs/LLVMBuild.html
+;
+;===------------------------------------------------------------------------===;
+
+[component_0]
+type = Library
+name = WebAssemblyInfo
+parent = WebAssembly
+required_libraries = Support
+add_to_library_groups = WebAssembly
diff --git a/lib/Target/WebAssembly/TargetInfo/Makefile b/lib/Target/WebAssembly/TargetInfo/Makefile
new file mode 100644
index 00000000000..b021eb6d945
--- /dev/null
+++ b/lib/Target/WebAssembly/TargetInfo/Makefile
@@ -0,0 +1,15 @@
+##===- lib/Target/WebAssembly/TargetInfo/Makefile ----------*- Makefile -*-===##
+#
+#                     The LLVM Compiler Infrastructure
+#
+# This file is distributed under the University of Illinois Open Source
+# License. See LICENSE.TXT for details.
+#
+##===----------------------------------------------------------------------===##
+LEVEL = ../../../..
+LIBRARYNAME = LLVMWebAssemblyInfo
+
+# Hack: we need to include 'main' target directory to grab private headers
+CPPFLAGS = -I$(PROJ_OBJ_DIR)/.. -I$(PROJ_SRC_DIR)/..
+
+include $(LEVEL)/Makefile.common
diff --git a/lib/Target/WebAssembly/TargetInfo/WebAssemblyTargetInfo.cpp b/lib/Target/WebAssembly/TargetInfo/WebAssemblyTargetInfo.cpp
new file mode 100644
index 00000000000..ddb1eb1d189
--- /dev/null
+++ b/lib/Target/WebAssembly/TargetInfo/WebAssemblyTargetInfo.cpp
@@ -0,0 +1,30 @@
+//===-- WebAssemblyTargetInfo.cpp - WebAssembly Target Implementation -----===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+///
+/// \file
+/// \brief This file registers the WebAssembly target.
+///
+//===----------------------------------------------------------------------===//
+
+#include "MCTargetDesc/WebAssemblyMCTargetDesc.h"
+#include "llvm/ADT/Triple.h"
+#include "llvm/Support/TargetRegistry.h"
+using namespace llvm;
+
+#define DEBUG_TYPE "wasm-target-info"
+
+Target llvm::TheWebAssemblyTarget32;
+Target llvm::TheWebAssemblyTarget64;
+
+extern "C" void LLVMInitializeWebAssemblyTargetInfo() {
+  RegisterTarget<Triple::wasm32> X(TheWebAssemblyTarget32, "wasm32",
+                                   "WebAssembly 32-bit");
+  RegisterTarget<Triple::wasm64> Y(TheWebAssemblyTarget64, "wasm64",
+                                   "WebAssembly 64-bit");
+}
diff --git a/lib/Target/WebAssembly/WebAssembly.h b/lib/Target/WebAssembly/WebAssembly.h
new file mode 100644
index 00000000000..3ff19d46f43
--- /dev/null
+++ b/lib/Target/WebAssembly/WebAssembly.h
@@ -0,0 +1,31 @@
+//===-- WebAssembly.h - Top-level interface for WebAssembly  ----*- C++ -*-===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+///
+/// \file
+/// \brief This file contains the entry points for global functions defined in
+/// the LLVM WebAssembly back-end.
+///
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIB_TARGET_WEBASSEMBLY_WEBASSEMBLY_H
+#define LLVM_LIB_TARGET_WEBASSEMBLY_WEBASSEMBLY_H
+
+#include "llvm/Support/CodeGen.h"
+
+namespace llvm {
+
+class WebAssemblyTargetMachine;
+class FunctionPass;
+
+FunctionPass *createWebAssemblyISelDag(WebAssemblyTargetMachine &TM,
+                                       CodeGenOpt::Level OptLevel);
+
+} // end namespace llvm
+
+#endif
diff --git a/lib/Target/WebAssembly/WebAssembly.td b/lib/Target/WebAssembly/WebAssembly.td
new file mode 100644
index 00000000000..53dd9290348
--- /dev/null
+++ b/lib/Target/WebAssembly/WebAssembly.td
@@ -0,0 +1,65 @@
+//- WebAssembly.td - Describe the WebAssembly Target Machine --*- tablegen -*-//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This is a target description file for the WebAssembly architecture, which is
+// also known as "wasm".
+//
+//===----------------------------------------------------------------------===//
+
+//===----------------------------------------------------------------------===//
+// Target-independent interfaces which we are implementing
+//===----------------------------------------------------------------------===//
+
+include "llvm/Target/Target.td"
+
+//===----------------------------------------------------------------------===//
+// WebAssembly Subtarget features.
+//===----------------------------------------------------------------------===//
+
+def FeatureSIMD128 : SubtargetFeature<"simd128", "HasSIMD128", "false",
+                                      "Enable 128-bit SIMD">;
+
+//===----------------------------------------------------------------------===//
+// Architectures.
+//===----------------------------------------------------------------------===//
+
+//===----------------------------------------------------------------------===//
+// Register File Description
+//===----------------------------------------------------------------------===//
+
+include "WebAssemblyRegisterInfo.td"
+
+//===----------------------------------------------------------------------===//
+// Instruction Descriptions
+//===----------------------------------------------------------------------===//
+
+include "WebAssemblyInstrInfo.td"
+
+def WebAssemblyInstrInfo : InstrInfo;
+
+//===----------------------------------------------------------------------===//
+// WebAssembly Processors supported.
+//===----------------------------------------------------------------------===//
+
+// Minimal Viable Product.
+def : ProcessorModel<"mvp", NoSchedModel, []>;
+
+// Generic processor: latest stable version.
+def : ProcessorModel<"generic", NoSchedModel, []>;
+
+// Latest and greatest experimental version of WebAssembly. Bugs included!
+def : ProcessorModel<"bleeding-edge", NoSchedModel, [FeatureSIMD128]>;
+
+//===----------------------------------------------------------------------===//
+// Target Declaration
+//===----------------------------------------------------------------------===//
+
+def WebAssembly : Target {
+  let InstructionSet = WebAssemblyInstrInfo;
+}
diff --git a/lib/Target/WebAssembly/WebAssemblyAsmPrinter.cpp b/lib/Target/WebAssembly/WebAssemblyAsmPrinter.cpp
new file mode 100644
index 00000000000..e0e9a3b14bc
--- /dev/null
+++ b/lib/Target/WebAssembly/WebAssemblyAsmPrinter.cpp
@@ -0,0 +1,125 @@
+//===-- WebAssemblyAsmPrinter.cpp - WebAssembly LLVM assembly writer ------===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+///
+/// \file
+/// \brief This file contains a printer that converts from our internal
+/// representation of machine-dependent LLVM code to the WebAssembly assembly
+/// language.
+///
+//===----------------------------------------------------------------------===//
+
+#include "WebAssembly.h"
+#include "WebAssemblyMachineFunctionInfo.h"
+#include "WebAssemblyRegisterInfo.h"
+#include "WebAssemblySubtarget.h"
+#include "InstPrinter/WebAssemblyInstPrinter.h"
+#include "MCTargetDesc/WebAssemblyMCTargetDesc.h"
+
+#include "llvm/ADT/SmallString.h"
+#include "llvm/CodeGen/AsmPrinter.h"
+#include "llvm/CodeGen/MachineInstr.h"
+#include "llvm/IR/DataLayout.h"
+#include "llvm/IR/DebugInfo.h"
+#include "llvm/MC/MCStreamer.h"
+#include "llvm/Support/Debug.h"
+#include "llvm/Support/TargetRegistry.h"
+#include "llvm/Support/raw_ostream.h"
+
+using namespace llvm;
+
+#define DEBUG_TYPE "asm-printer"
+
+namespace {
+
+class WebAssemblyAsmPrinter final : public AsmPrinter {
+  const WebAssemblyInstrInfo *TII;
+
+public:
+  WebAssemblyAsmPrinter(TargetMachine &TM, std::unique_ptr<MCStreamer> Streamer)
+      : AsmPrinter(TM, std::move(Streamer)), TII(nullptr) {}
+
+private:
+  const char *getPassName() const override {
+    return "WebAssembly Assembly Printer";
+  }
+
+  //===------------------------------------------------------------------===//
+  // MachineFunctionPass Implementation.
+  //===------------------------------------------------------------------===//
+
+  void getAnalysisUsage(AnalysisUsage &AU) const override {
+    AsmPrinter::getAnalysisUsage(AU);
+  }
+
+  bool runOnMachineFunction(MachineFunction &MF) override {
+    TII = static_cast<const WebAssemblyInstrInfo *>(
+        MF.getSubtarget().getInstrInfo());
+    return AsmPrinter::runOnMachineFunction(MF);
+  }
+
+  //===------------------------------------------------------------------===//
+  // AsmPrinter Implementation.
+  //===------------------------------------------------------------------===//
+
+  void EmitInstruction(const MachineInstr *MI) override;
+};
+
+} // end anonymous namespace
+
+//===----------------------------------------------------------------------===//
+
+void WebAssemblyAsmPrinter::EmitInstruction(const MachineInstr *MI) {
+  SmallString<128> Str;
+  raw_svector_ostream OS(Str);
+
+  unsigned NumDefs = MI->getDesc().getNumDefs();
+  assert(NumDefs <= 1 &&
+         "Instructions with multiple result values not implemented");
+
+  if (NumDefs != 0) {
+    const MachineOperand &MO = MI->getOperand(0);
+    unsigned Reg = MO.getReg();
+    OS << "(setlocal @" << TargetRegisterInfo::virtReg2Index(Reg) << ' ';
+  }
+
+  OS << '(';
+
+  bool PrintOperands = true;
+  switch (MI->getOpcode()) {
+  case WebAssembly::ARGUMENT:
+    OS << "argument " << MI->getOperand(1).getImm();
+    PrintOperands = false;
+    break;
+  default:
+    OS << TII->getName(MI->getOpcode());
+    break;
+  }
+
+  if (PrintOperands)
+    for (const MachineOperand &MO : MI->uses()) {
+      if (MO.isReg() && MO.isImplicit())
+        continue;
+      unsigned Reg = MO.getReg();
+      OS << " @" << TargetRegisterInfo::virtReg2Index(Reg);
+    }
+  OS << ')';
+
+  if (NumDefs != 0)
+    OS << ')';
+
+  OS << '\n';
+
+  OutStreamer->EmitRawText(OS.str());
+}
+
+// Force static initialization.
+extern "C" void LLVMInitializeWebAssemblyAsmPrinter() {
+  RegisterAsmPrinter<WebAssemblyAsmPrinter> X(TheWebAssemblyTarget32);
+  RegisterAsmPrinter<WebAssemblyAsmPrinter> Y(TheWebAssemblyTarget64);
+}
diff --git a/lib/Target/WebAssembly/WebAssemblyFrameLowering.cpp b/lib/Target/WebAssembly/WebAssemblyFrameLowering.cpp
new file mode 100644
index 00000000000..fec94554833
--- /dev/null
+++ b/lib/Target/WebAssembly/WebAssemblyFrameLowering.cpp
@@ -0,0 +1,74 @@
+//===-- WebAssemblyFrameLowering.cpp - WebAssembly Frame Lowering ----------==//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+///
+/// \file
+/// \brief This file contains the WebAssembly implementation of
+/// TargetFrameLowering class.
+///
+/// On WebAssembly, there aren't a lot of things to do here. There are no
+/// callee-saved registers to save, and no spill slots.
+///
+/// The stack grows downward.
+///
+//===----------------------------------------------------------------------===//
+
+#include "WebAssemblyFrameLowering.h"
+#include "MCTargetDesc/WebAssemblyMCTargetDesc.h"
+#include "WebAssemblyInstrInfo.h"
+#include "WebAssemblyMachineFunctionInfo.h"
+#include "WebAssemblySubtarget.h"
+#include "WebAssemblyTargetMachine.h"
+#include "llvm/CodeGen/MachineFrameInfo.h"
+#include "llvm/CodeGen/MachineFunction.h"
+#include "llvm/CodeGen/MachineInstrBuilder.h"
+#include "llvm/CodeGen/MachineModuleInfo.h"
+#include "llvm/CodeGen/MachineRegisterInfo.h"
+#include "llvm/Support/Debug.h"
+using namespace llvm;
+
+#define DEBUG_TYPE "wasm-frame-info"
+
+// TODO: Implement a red zone?
+
+/// Return true if the specified function should have a dedicated frame pointer
+/// register.
+bool WebAssemblyFrameLowering::hasFP(const MachineFunction &MF) const {
+  const MachineFrameInfo *MFI = MF.getFrameInfo();
+  const auto *RegInfo = static_cast<const WebAssemblyRegisterInfo *>(
+      MF.getSubtarget().getRegisterInfo());
+  return MFI->hasCalls() || MFI->hasVarSizedObjects() ||
+         MFI->isFrameAddressTaken() || MFI->hasStackMap() ||
+         MFI->hasPatchPoint() || RegInfo->needsStackRealignment(MF);
+}
+
+/// Under normal circumstances, when a frame pointer is not required, we reserve
+/// argument space for call sites in the function immediately on entry to the
+/// current function. This eliminates the need for add/sub sp brackets around
+/// call sites. Returns true if the call frame is included as part of the stack
+/// frame.
+bool WebAssemblyFrameLowering::hasReservedCallFrame(
+    const MachineFunction &MF) const {
+  return !MF.getFrameInfo()->hasVarSizedObjects();
+}
+
+void WebAssemblyFrameLowering::eliminateCallFramePseudoInstr(
+    MachineFunction &MF, MachineBasicBlock &MBB,
+    MachineBasicBlock::iterator I) const {
+  llvm_unreachable("TODO: implement eliminateCallFramePseudoInstr");
+}
+
+void WebAssemblyFrameLowering::emitPrologue(MachineFunction &MF,
+                                            MachineBasicBlock &MBB) const {
+  // FIXME: Implement WebAssemblyFrameLowering::emitPrologue.
+}
+
+void WebAssemblyFrameLowering::emitEpilogue(MachineFunction &MF,
+                                            MachineBasicBlock &MBB) const {
+  llvm_unreachable("TODO: implement emitEpilogue");
+}
diff --git a/lib/Target/WebAssembly/WebAssemblyFrameLowering.h b/lib/Target/WebAssembly/WebAssemblyFrameLowering.h
new file mode 100644
index 00000000000..5f4708fe77e
--- /dev/null
+++ b/lib/Target/WebAssembly/WebAssemblyFrameLowering.h
@@ -0,0 +1,45 @@
+// WebAssemblyFrameLowering.h - TargetFrameLowering for WebAssembly -*- C++ -*-/
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+///
+/// \file
+/// \brief This class implements WebAssembly-specific bits of
+/// TargetFrameLowering class.
+///
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIB_TARGET_WEBASSEMBLY_WEBASSEMBLYFRAMELOWERING_H
+#define LLVM_LIB_TARGET_WEBASSEMBLY_WEBASSEMBLYFRAMELOWERING_H
+
+#include "llvm/Target/TargetFrameLowering.h"
+
+namespace llvm {
+
+class WebAssemblyFrameLowering final : public TargetFrameLowering {
+public:
+  WebAssemblyFrameLowering()
+      : TargetFrameLowering(StackGrowsDown, /*StackAlignment=*/16,
+                            /*LocalAreaOffset=*/0,
+                            /*TransientStackAlignment=*/16,
+                            /*StackRealignable=*/true) {}
+
+  void
+  eliminateCallFramePseudoInstr(MachineFunction &MF, MachineBasicBlock &MBB,
+                                MachineBasicBlock::iterator I) const override;
+
+  /// These methods insert prolog and epilog code into the function.
+  void emitPrologue(MachineFunction &MF, MachineBasicBlock &MBB) const override;
+  void emitEpilogue(MachineFunction &MF, MachineBasicBlock &MBB) const override;
+
+  bool hasFP(const MachineFunction &MF) const override;
+  bool hasReservedCallFrame(const MachineFunction &MF) const override;
+};
+
+} // end namespace llvm
+
+#endif
diff --git a/lib/Target/WebAssembly/WebAssemblyISelDAGToDAG.cpp b/lib/Target/WebAssembly/WebAssemblyISelDAGToDAG.cpp
new file mode 100644
index 00000000000..58a23756bdc
--- /dev/null
+++ b/lib/Target/WebAssembly/WebAssemblyISelDAGToDAG.cpp
@@ -0,0 +1,109 @@
+//- WebAssemblyISelDAGToDAG.cpp - A dag to dag inst selector for WebAssembly -//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+///
+/// \file
+/// \brief This file defines an instruction selector for the WebAssembly target.
+///
+//===----------------------------------------------------------------------===//
+
+#include "WebAssembly.h"
+#include "MCTargetDesc/WebAssemblyMCTargetDesc.h"
+#include "WebAssemblyTargetMachine.h"
+#include "llvm/CodeGen/SelectionDAGISel.h"
+#include "llvm/IR/Function.h" // To access function attributes.
+#include "llvm/Support/Debug.h"
+#include "llvm/Support/MathExtras.h"
+#include "llvm/Support/raw_ostream.h"
+using namespace llvm;
+
+#define DEBUG_TYPE "wasm-isel"
+
+//===--------------------------------------------------------------------===//
+/// WebAssembly-specific code to select WebAssembly machine instructions for
+/// SelectionDAG operations.
+///
+namespace {
+class WebAssemblyDAGToDAGISel final : public SelectionDAGISel {
+  /// Keep a pointer to the WebAssemblySubtarget around so that we can make the
+  /// right decision when generating code for different targets.
+  const WebAssemblySubtarget *Subtarget;
+
+  bool ForCodeSize;
+
+public:
+  WebAssemblyDAGToDAGISel(WebAssemblyTargetMachine &tm,
+                          CodeGenOpt::Level OptLevel)
+      : SelectionDAGISel(tm, OptLevel), Subtarget(nullptr), ForCodeSize(false) {
+  }
+
+  const char *getPassName() const override {
+    return "WebAssembly Instruction Selection";
+  }
+
+  bool runOnMachineFunction(MachineFunction &MF) override {
+    ForCodeSize =
+        MF.getFunction()->hasFnAttribute(Attribute::OptimizeForSize) ||
+        MF.getFunction()->hasFnAttribute(Attribute::MinSize);
+    Subtarget = &MF.getSubtarget<WebAssemblySubtarget>();
+    return SelectionDAGISel::runOnMachineFunction(MF);
+  }
+
+  SDNode *Select(SDNode *Node) override;
+
+// Include the pieces autogenerated from the target description.
+#include "WebAssemblyGenDAGISel.inc"
+
+private:
+  // add select functions here...
+};
+} // end anonymous namespace
+
+SDNode *WebAssemblyDAGToDAGISel::Select(SDNode *Node) {
+  // Dump information about the Node being selected.
+  DEBUG(errs() << "Selecting: ");
+  DEBUG(Node->dump(CurDAG));
+  DEBUG(errs() << "\n");
+
+  // If we have a custom node, we already have selected!
+  if (Node->isMachineOpcode()) {
+    DEBUG(errs() << "== "; Node->dump(CurDAG); errs() << "\n");
+    Node->setNodeId(-1);
+    return nullptr;
+  }
+
+  // Few custom selection stuff.
+  SDNode *ResNode = nullptr;
+  EVT VT = Node->getValueType(0);
+
+  switch (Node->getOpcode()) {
+  default:
+    break;
+    // FIXME: Implement WebAssembly-specific selection.
+    (void)VT;
+  }
+
+  // Select the default instruction.
+  ResNode = SelectCode(Node);
+
+  DEBUG(errs() << "=> ");
+  if (ResNode == nullptr || ResNode == Node)
+    DEBUG(Node->dump(CurDAG));
+  else
+    DEBUG(ResNode->dump(CurDAG));
+  DEBUG(errs() << "\n");
+
+  return ResNode;
+}
+
+/// This pass converts a legalized DAG into a WebAssembly-specific DAG, ready
+/// for instruction scheduling.
+FunctionPass *llvm::createWebAssemblyISelDag(WebAssemblyTargetMachine &TM,
+                                             CodeGenOpt::Level OptLevel) {
+  return new WebAssemblyDAGToDAGISel(TM, OptLevel);
+}
diff --git a/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp b/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp
new file mode 100644
index 00000000000..90a4078b407
--- /dev/null
+++ b/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp
@@ -0,0 +1,218 @@
+//=- WebAssemblyISelLowering.cpp - WebAssembly DAG Lowering Implementation -==//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+///
+/// \file
+/// \brief This file implements the WebAssemblyTargetLowering class.
+///
+//===----------------------------------------------------------------------===//
+
+#include "WebAssemblyISelLowering.h"
+#include "MCTargetDesc/WebAssemblyMCTargetDesc.h"
+#include "WebAssemblyMachineFunctionInfo.h"
+#include "WebAssemblySubtarget.h"
+#include "WebAssemblyTargetMachine.h"
+#include "WebAssemblyTargetObjectFile.h"
+#include "llvm/CodeGen/Analysis.h"
+#include "llvm/CodeGen/MachineRegisterInfo.h"
+#include "llvm/CodeGen/SelectionDAG.h"
+#include "llvm/IR/DiagnosticInfo.h"
+#include "llvm/IR/DiagnosticPrinter.h"
+#include "llvm/IR/Function.h"
+#include "llvm/IR/Intrinsics.h"
+#include "llvm/Support/CommandLine.h"
+#include "llvm/Support/Debug.h"
+#include "llvm/Support/ErrorHandling.h"
+#include "llvm/Support/raw_ostream.h"
+#include "llvm/Target/TargetOptions.h"
+
+using namespace llvm;
+
+#define DEBUG_TYPE "wasm-lower"
+
+namespace {
+// Diagnostic information for unimplemented or unsupported feature reporting.
+// FIXME copied from BPF and AMDGPU.
+class DiagnosticInfoUnsupported : public DiagnosticInfo {
+private:
+  // Debug location where this diagnostic is triggered.
+  DebugLoc DLoc;
+  const Twine &Description;
+  const Function &Fn;
+  SDValue Value;
+
+  static int KindID;
+
+  static int getKindID() {
+    if (KindID == 0)
+      KindID = llvm::getNextAvailablePluginDiagnosticKind();
+    return KindID;
+  }
+
+public:
+  DiagnosticInfoUnsupported(SDLoc DLoc, const Function &Fn, const Twine &Desc,
+                            SDValue Value)
+      : DiagnosticInfo(getKindID(), DS_Error), DLoc(DLoc.getDebugLoc()),
+        Description(Desc), Fn(Fn), Value(Value) {}
+
+  void print(DiagnosticPrinter &DP) const override {
+    std::string Str;
+    raw_string_ostream OS(Str);
+
+    if (DLoc) {
+      auto DIL = DLoc.get();
+      StringRef Filename = DIL->getFilename();
+      unsigned Line = DIL->getLine();
+      unsigned Column = DIL->getColumn();
+      OS << Filename << ':' << Line << ':' << Column << ' ';
+    }
+
+    OS << "in function " << Fn.getName() << ' ' << *Fn.getFunctionType() << '\n'
+       << Description;
+    if (Value)
+      Value->print(OS);
+    OS << '\n';
+    OS.flush();
+    DP << Str;
+  }
+
+  static bool classof(const DiagnosticInfo *DI) {
+    return DI->getKind() == getKindID();
+  }
+};
+
+int DiagnosticInfoUnsupported::KindID = 0;
+} // end anonymous namespace
+
+WebAssemblyTargetLowering::WebAssemblyTargetLowering(
+    const TargetMachine &TM, const WebAssemblySubtarget &STI)
+    : TargetLowering(TM), Subtarget(&STI) {
+  // WebAssembly does not produce floating-point exceptions on normal floating
+  // point operations.
+  setHasFloatingPointExceptions(false);
+  // We don't know the microarchitecture here, so just reduce register pressure.
+  setSchedulingPreference(Sched::RegPressure);
+  // Tell ISel that we have a stack pointer.
+  setStackPointerRegisterToSaveRestore(
+      Subtarget->hasAddr64() ? WebAssembly::SP64 : WebAssembly::SP32);
+  // Set up the register classes.
+  addRegisterClass(MVT::i32, &WebAssembly::Int32RegClass);
+  addRegisterClass(MVT::i64, &WebAssembly::Int64RegClass);
+  addRegisterClass(MVT::f32, &WebAssembly::Float32RegClass);
+  addRegisterClass(MVT::f64, &WebAssembly::Float64RegClass);
+  // Compute derived properties from the register classes.
+  computeRegisterProperties(Subtarget->getRegisterInfo());
+
+  // FIXME: setOperationAction...
+}
+
+//===----------------------------------------------------------------------===//
+// WebAssembly Lowering private implementation.
+//===----------------------------------------------------------------------===//
+
+//===----------------------------------------------------------------------===//
+// Lowering Code
+//===----------------------------------------------------------------------===//
+
+static void fail(SDLoc DL, SelectionDAG &DAG, const char *msg) {
+  MachineFunction &MF = DAG.getMachineFunction();
+  DAG.getContext()->diagnose(
+      DiagnosticInfoUnsupported(DL, *MF.getFunction(), msg, SDValue()));
+}
+
+bool WebAssemblyTargetLowering::CanLowerReturn(
+    CallingConv::ID CallConv, MachineFunction &MF, bool IsVarArg,
+    const SmallVectorImpl<ISD::OutputArg> &Outs, LLVMContext &Context) const {
+  // WebAssembly can't currently handle returning tuples.
+  return Outs.size() <= 1;
+}
+
+SDValue WebAssemblyTargetLowering::LowerReturn(
+    SDValue Chain, CallingConv::ID CallConv, bool IsVarArg,
+    const SmallVectorImpl<ISD::OutputArg> &Outs,
+    const SmallVectorImpl<SDValue> &OutVals, SDLoc DL,
+    SelectionDAG &DAG) const {
+
+  assert(Outs.size() <= 1 && "WebAssembly can only return up to one value");
+  if (CallConv != CallingConv::C)
+    fail(DL, DAG, "WebAssembly doesn't support non-C calling conventions");
+  if (IsVarArg)
+    fail(DL, DAG, "WebAssembly doesn't support varargs yet");
+
+  SmallVector<SDValue, 4> RetOps(1, Chain);
+  RetOps.append(OutVals.begin(), OutVals.end());
+  const SDValue Ops[] = {Chain, OutVals.front()};
+  Chain = DAG.getNode(WebAssemblyISD::RETURN, DL, MVT::Other, Ops);
+
+  return Chain;
+}
+
+SDValue WebAssemblyTargetLowering::LowerFormalArguments(
+    SDValue Chain, CallingConv::ID CallConv, bool IsVarArg,
+    const SmallVectorImpl<ISD::InputArg> &Ins, SDLoc DL, SelectionDAG &DAG,
+    SmallVectorImpl<SDValue> &InVals) const {
+  MachineFunction &MF = DAG.getMachineFunction();
+
+  if (CallConv != CallingConv::C)
+    fail(DL, DAG, "WebAssembly doesn't support non-C calling conventions");
+  if (IsVarArg)
+    fail(DL, DAG, "WebAssembly doesn't support varargs yet");
+  if (MF.getFunction()->hasStructRetAttr())
+    fail(DL, DAG, "WebAssembly doesn't support struct return yet");
+
+  unsigned ArgNo = 0;
+  for (const ISD::InputArg &In : Ins) {
+    if (In.Flags.isZExt())
+      fail(DL, DAG, "WebAssembly hasn't implemented zext arguments");
+    if (In.Flags.isSExt())
+      fail(DL, DAG, "WebAssembly hasn't implemented sext arguments");
+    if (In.Flags.isInReg())
+      fail(DL, DAG, "WebAssembly hasn't implemented inreg arguments");
+    if (In.Flags.isSRet())
+      fail(DL, DAG, "WebAssembly hasn't implemented sret arguments");
+    if (In.Flags.isByVal())
+      fail(DL, DAG, "WebAssembly hasn't implemented byval arguments");
+    if (In.Flags.isInAlloca())
+      fail(DL, DAG, "WebAssembly hasn't implemented inalloca arguments");
+    if (In.Flags.isNest())
+      fail(DL, DAG, "WebAssembly hasn't implemented nest arguments");
+    if (In.Flags.isReturned())
+      fail(DL, DAG, "WebAssembly hasn't implemented returned arguments");
+    if (In.Flags.isInConsecutiveRegs())
+      fail(DL, DAG, "WebAssembly hasn't implemented cons regs arguments");
+    if (In.Flags.isInConsecutiveRegsLast())
+      fail(DL, DAG, "WebAssembly hasn't implemented cons regs last arguments");
+    if (In.Flags.isSplit())
+      fail(DL, DAG, "WebAssembly hasn't implemented split arguments");
+    if (In.VT != MVT::i32)
+      fail(DL, DAG, "WebAssembly hasn't implemented non-i32 arguments");
+    // FIXME Do something with In.getOrigAlign()?
+    InVals.push_back(
+        In.Used
+            ? DAG.getNode(WebAssemblyISD::ARGUMENT, DL, In.VT,
+                          DAG.getTargetConstant(ArgNo, DL, MVT::i32))
+            : DAG.getNode(ISD::UNDEF, DL, In.VT));
+    ++ArgNo;
+  }
+
+  return Chain;
+}
+
+//===----------------------------------------------------------------------===//
+//  Other Lowering Code
+//===----------------------------------------------------------------------===//
+
+//===----------------------------------------------------------------------===//
+//                          WebAssembly Optimization Hooks
+//===----------------------------------------------------------------------===//
+
+MCSection *WebAssemblyTargetObjectFile::SelectSectionForGlobal(
+    const GlobalValue *GV, SectionKind Kind, Mangler &Mang,
+    const TargetMachine &TM) const {
+  return getDataSection();
+}
diff --git a/lib/Target/WebAssembly/WebAssemblyISelLowering.h b/lib/Target/WebAssembly/WebAssemblyISelLowering.h
new file mode 100644
index 00000000000..3e8252604d6
--- /dev/null
+++ b/lib/Target/WebAssembly/WebAssemblyISelLowering.h
@@ -0,0 +1,67 @@
+//- WebAssemblyISelLowering.h - WebAssembly DAG Lowering Interface -*- C++ -*-//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+///
+/// \file
+/// \brief This file defines the interfaces that WebAssembly uses to lower LLVM
+/// code into a selection DAG.
+///
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIB_TARGET_WEBASSEMBLY_WEBASSEMBLYISELLOWERING_H
+#define LLVM_LIB_TARGET_WEBASSEMBLY_WEBASSEMBLYISELLOWERING_H
+
+#include "llvm/Target/TargetLowering.h"
+
+namespace llvm {
+
+namespace WebAssemblyISD {
+
+enum {
+  FIRST_NUMBER = ISD::BUILTIN_OP_END,
+  RETURN,
+  ARGUMENT,
+
+  // add memory opcodes starting at ISD::FIRST_TARGET_MEMORY_OPCODE here...
+};
+
+} // end namespace WebAssemblyISD
+
+class WebAssemblySubtarget;
+class WebAssemblyTargetMachine;
+
+class WebAssemblyTargetLowering final : public TargetLowering {
+public:
+  WebAssemblyTargetLowering(const TargetMachine &TM,
+                            const WebAssemblySubtarget &STI);
+
+private:
+  /// Keep a pointer to the WebAssemblySubtarget around so that we can make the
+  /// right decision when generating code for different targets.
+  const WebAssemblySubtarget *Subtarget;
+
+  bool CanLowerReturn(CallingConv::ID CallConv, MachineFunction &MF,
+                      bool isVarArg,
+                      const SmallVectorImpl<ISD::OutputArg> &Outs,
+                      LLVMContext &Context) const override;
+
+  SDValue LowerReturn(SDValue Chain, CallingConv::ID CallConv, bool isVarArg,
+                      const SmallVectorImpl<ISD::OutputArg> &Outs,
+                      const SmallVectorImpl<SDValue> &OutVals, SDLoc dl,
+                      SelectionDAG &DAG) const override;
+
+  SDValue LowerFormalArguments(SDValue Chain, CallingConv::ID CallConv,
+                               bool IsVarArg,
+                               const SmallVectorImpl<ISD::InputArg> &Ins,
+                               SDLoc DL, SelectionDAG &DAG,
+                               SmallVectorImpl<SDValue> &InVals) const override;
+};
+
+} // end namespace llvm
+
+#endif
diff --git a/lib/Target/WebAssembly/WebAssemblyInstrAtomics.td b/lib/Target/WebAssembly/WebAssemblyInstrAtomics.td
new file mode 100644
index 00000000000..64415658ed8
--- /dev/null
+++ b/lib/Target/WebAssembly/WebAssemblyInstrAtomics.td
@@ -0,0 +1,47 @@
+// WebAssemblyInstrAtomics.td-WebAssembly Atomic codegen support-*- tablegen -*-
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+///
+/// \file
+/// \brief WebAssembly Atomic operand code-gen constructs.
+///
+//===----------------------------------------------------------------------===//
+
+// TODO: Implement atomic instructions.
+
+//===----------------------------------------------------------------------===//
+// Atomic fences
+//===----------------------------------------------------------------------===//
+
+// TODO: add atomic fences here...
+
+//===----------------------------------------------------------------------===//
+// Atomic loads
+//===----------------------------------------------------------------------===//
+
+// TODO: add atomic loads here...
+
+//===----------------------------------------------------------------------===//
+// Atomic stores
+//===----------------------------------------------------------------------===//
+
+// TODO: add atomic stores here...
+
+//===----------------------------------------------------------------------===//
+// Low-level exclusive operations
+//===----------------------------------------------------------------------===//
+
+// TODO: add exclusive operations here...
+
+// Load-exclusives.
+
+// Store-exclusives.
+
+// Store-release-exclusives.
+
+// And clear exclusive.
diff --git a/lib/Target/WebAssembly/WebAssemblyInstrCall.td b/lib/Target/WebAssembly/WebAssemblyInstrCall.td
new file mode 100644
index 00000000000..6b5b6cd5417
--- /dev/null
+++ b/lib/Target/WebAssembly/WebAssemblyInstrCall.td
@@ -0,0 +1,21 @@
+//===- WebAssemblyInstrCall.td-WebAssembly Call codegen support -*- tablegen -*-
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+///
+/// \file
+/// \brief WebAssembly Call operand code-gen constructs.
+///
+//===----------------------------------------------------------------------===//
+
+/*
+ * TODO(jfb): Add the following.
+ *
+ * call_direct: call function directly
+ * call_indirect: call function indirectly
+ * addressof: obtain a function pointer value for a given function
+ */
diff --git a/lib/Target/WebAssembly/WebAssemblyInstrControl.td b/lib/Target/WebAssembly/WebAssemblyInstrControl.td
new file mode 100644
index 00000000000..59e6e3b6711
--- /dev/null
+++ b/lib/Target/WebAssembly/WebAssemblyInstrControl.td
@@ -0,0 +1,33 @@
+//===- WebAssemblyInstrControl.td-WebAssembly control-flow ------*- tablegen -*-
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+///
+/// \file
+/// \brief WebAssembly control-flow code-gen constructs.
+///
+//===----------------------------------------------------------------------===//
+
+/*
+ * TODO(jfb): Add the following.
+ *
+ * block: a fixed-length sequence of statements
+ * if: if statement
+ * do_while: do while statement, basically a loop with a conditional branch
+ * forever: infinite loop statement (like while (1)), basically an unconditional
+ *          branch (back to the top of the loop)
+ * continue: continue to start of nested loop
+ * break: break to end from nested loop or block
+ * switch: switch statement with fallthrough
+ */
+
+let hasSideEffects = 1, isReturn = 1, isTerminator = 1, hasCtrlDep = 1,
+    isBarrier = 1 in {
+//FIXME return more than just int32.
+def RETURN : I<(outs), (ins Int32:$val), [(WebAssemblyreturn Int32:$val)]>;
+} // hasSideEffects = 1, isReturn = 1, isTerminator = 1, hasCtrlDep = 1,
+  // isBarrier = 1
diff --git a/lib/Target/WebAssembly/WebAssemblyInstrConv.td b/lib/Target/WebAssembly/WebAssemblyInstrConv.td
new file mode 100644
index 00000000000..3fa29061b1d
--- /dev/null
+++ b/lib/Target/WebAssembly/WebAssemblyInstrConv.td
@@ -0,0 +1,44 @@
+//===-- WebAssemblyInstrConv.td-WebAssembly Conversion support -*- tablegen -*-=
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+///
+/// \file
+/// \brief WebAssembly datatype conversions, truncations, reinterpretations,
+/// promotions, and demotions operand code-gen constructs.
+///
+//===----------------------------------------------------------------------===//
+
+/*
+ * TODO(jfb): Add the following.
+ *
+ * int32.wrap[int64]: wrap a 64-bit integer to a 32-bit integer
+ * int32.trunc_signed[float32]: truncate a 32-bit float to a signed 32-bit integer
+ * int32.trunc_signed[float64]: truncate a 64-bit float to a signed 32-bit integer
+ * int32.trunc_unsigned[float32]: truncate a 32-bit float to an unsigned 32-bit integer
+ * int32.trunc_unsigned[float64]: truncate a 64-bit float to an unsigned 32-bit integer
+ * int32.reinterpret[float32]: reinterpret the bits of a 32-bit float as a 32-bit integer
+ * int64.extend_signed[int32]: extend a signed 32-bit integer to a 64-bit integer
+ * int64.extend_unsigned[int32]: extend an unsigned 32-bit integer to a 64-bit integer
+ * int64.trunc_signed[float32]: truncate a 32-bit float to a signed 64-bit integer
+ * int64.trunc_signed[float64]: truncate a 64-bit float to a signed 64-bit integer
+ * int64.trunc_unsigned[float32]: truncate a 32-bit float to an unsigned 64-bit integer
+ * int64.trunc_unsigned[float64]: truncate a 64-bit float to an unsigned 64-bit integer
+ * int64.reinterpret[float64]: reinterpret the bits of a 64-bit float as a 64-bit integer
+ * float32.demote[float64]: demote a 64-bit float to a 32-bit float
+ * float32.cvt_signed[int32]: convert a signed 32-bit integer to a 32-bit float
+ * float32.cvt_signed[int64]: convert a signed 64-bit integer to a 32-bit float
+ * float32.cvt_unsigned[int32]: convert an unsigned 32-bit integer to a 32-bit float
+ * float32.cvt_unsigned[int64]: convert an unsigned 64-bit integer to a 32-bit float
+ * float32.reinterpret[int32]: reinterpret the bits of a 32-bit integer as a 32-bit float
+ * float64.promote[float32]: promote a 32-bit float to a 64-bit float
+ * float64.cvt_signed[int32]: convert a signed 32-bit integer to a 64-bit float
+ * float64.cvt_signed[int64]: convert a signed 64-bit integer to a 64-bit float
+ * float64.cvt_unsigned[int32]: convert an unsigned 32-bit integer to a 64-bit float
+ * float64.cvt_unsigned[int64]: convert an unsigned 64-bit integer to a 64-bit float
+ * float64.reinterpret[int64]: reinterpret the bits of a 64-bit integer as a 64-bit float
+ */
diff --git a/lib/Target/WebAssembly/WebAssemblyInstrFloat.td b/lib/Target/WebAssembly/WebAssemblyInstrFloat.td
new file mode 100644
index 00000000000..30ef6339d65
--- /dev/null
+++ b/lib/Target/WebAssembly/WebAssemblyInstrFloat.td
@@ -0,0 +1,44 @@
+// WebAssemblyInstrFloat.td-WebAssembly Float codegen support ---*- tablegen -*-
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+///
+/// \file
+/// \brief WebAssembly Floating-point operand code-gen constructs.
+///
+//===----------------------------------------------------------------------===//
+
+defm FADD : BinaryFP<fadd>;
+defm FSUB : BinaryFP<fsub>;
+defm FMUL : BinaryFP<fmul>;
+defm FDIV : BinaryFP<fdiv>;
+defm FABS : UnaryFP<fabs>;
+defm FNEG : UnaryFP<fneg>;
+defm COPYSIGN : BinaryFP<fcopysign>;
+defm CEIL : UnaryFP<fceil>;
+defm FLOOR : UnaryFP<ffloor>;
+defm TRUNC : UnaryFP<ftrunc>;
+defm NEARESTINT : UnaryFP<fnearbyint>;
+
+/*
+ * TODO(jfb): Add the following for 32-bit and 64-bit.
+ *
+ * float32.eq: compare equal
+ * float32.lt: less than
+ * float32.le: less than or equal
+ * float32.gt: greater than
+ * float32.ge: greater than or equal
+ */
+
+defm SQRT : UnaryFP<fsqrt>;
+
+/*
+ * TODO(jfb): Add the following for 32-bit and 64-bit.
+ *
+ * float32.min: minimum (binary operator); if either operand is NaN, returns NaN
+ * float32.max: maximum (binary operator); if either operand is NaN, returns NaN
+ */
diff --git a/lib/Target/WebAssembly/WebAssemblyInstrFormats.td b/lib/Target/WebAssembly/WebAssemblyInstrFormats.td
new file mode 100644
index 00000000000..513c36fa2ec
--- /dev/null
+++ b/lib/Target/WebAssembly/WebAssemblyInstrFormats.td
@@ -0,0 +1,55 @@
+// WebAssemblyInstrFormats.td - WebAssembly Instruction Formats -*- tblgen -*-//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+///
+/// \file
+/// \brief WebAssembly instruction format definitions.
+///
+//===----------------------------------------------------------------------===//
+
+// WebAssembly Instruction Format
+class WebAssemblyInst<string cstr> : Instruction {
+  field bits<0> Inst; // Instruction encoding.
+  let Namespace   = "WebAssembly";
+  let Pattern     = [];
+  let Constraints = cstr;
+}
+
+// Normal instructions
+class I<dag oops, dag iops, list<dag> pattern, string cstr = "">
+    : WebAssemblyInst<cstr> {
+  dag OutOperandList = oops;
+  dag InOperandList  = iops;
+  let Pattern        = pattern;
+}
+
+// Unary and binary instructions, for the local types that WebAssembly supports.
+multiclass UnaryInt<SDNode node> {
+  def _I32 : I<(outs Int32:$dst), (ins Int32:$src),
+               [(set Int32:$dst, (node Int32:$src))]>;
+  def _I64 : I<(outs Int64:$dst), (ins Int64:$src),
+               [(set Int64:$dst, (node Int64:$src))]>;
+}
+multiclass BinaryInt<SDNode node> {
+  def _I32 : I<(outs Int32:$dst), (ins Int32:$lhs, Int32:$rhs),
+               [(set Int32:$dst, (node Int32:$lhs, Int32:$rhs))]>;
+  def _I64 : I<(outs Int64:$dst), (ins Int64:$lhs, Int64:$rhs),
+               [(set Int64:$dst, (node Int64:$lhs, Int64:$rhs))]>;
+}
+multiclass UnaryFP<SDNode node> {
+  def _F32 : I<(outs Float32:$dst), (ins Float32:$src),
+               [(set Float32:$dst, (node Float32:$src))]>;
+  def _F64 : I<(outs Float64:$dst), (ins Float64:$src),
+               [(set Float64:$dst, (node Float64:$src))]>;
+}
+multiclass BinaryFP<SDNode node> {
+  def _F32 : I<(outs Float32:$dst), (ins Float32:$lhs, Float32:$rhs),
+               [(set Float32:$dst, (node Float32:$lhs, Float32:$rhs))]>;
+  def _F64 : I<(outs Float64:$dst), (ins Float64:$lhs, Float64:$rhs),
+               [(set Float64:$dst, (node Float64:$lhs, Float64:$rhs))]>;
+}
diff --git a/lib/Target/WebAssembly/WebAssemblyInstrInfo.cpp b/lib/Target/WebAssembly/WebAssemblyInstrInfo.cpp
new file mode 100644
index 00000000000..d4c65a4b98c
--- /dev/null
+++ b/lib/Target/WebAssembly/WebAssemblyInstrInfo.cpp
@@ -0,0 +1,31 @@
+//===-- WebAssemblyInstrInfo.cpp - WebAssembly Instruction Information ----===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+///
+/// \file
+/// \brief This file contains the WebAssembly implementation of the
+/// TargetInstrInfo class.
+///
+//===----------------------------------------------------------------------===//
+
+#include "WebAssemblyInstrInfo.h"
+#include "MCTargetDesc/WebAssemblyMCTargetDesc.h"
+#include "WebAssemblySubtarget.h"
+#include "llvm/CodeGen/MachineFrameInfo.h"
+#include "llvm/CodeGen/MachineInstrBuilder.h"
+#include "llvm/CodeGen/MachineMemOperand.h"
+#include "llvm/CodeGen/MachineRegisterInfo.h"
+using namespace llvm;
+
+#define DEBUG_TYPE "wasm-instr-info"
+
+#define GET_INSTRINFO_CTOR_DTOR
+#include "WebAssemblyGenInstrInfo.inc"
+
+WebAssemblyInstrInfo::WebAssemblyInstrInfo(const WebAssemblySubtarget &STI)
+    : RI(STI.getTargetTriple()) {}
diff --git a/lib/Target/WebAssembly/WebAssemblyInstrInfo.h b/lib/Target/WebAssembly/WebAssemblyInstrInfo.h
new file mode 100644
index 00000000000..29fdcb0e6cb
--- /dev/null
+++ b/lib/Target/WebAssembly/WebAssemblyInstrInfo.h
@@ -0,0 +1,40 @@
+//=- WebAssemblyInstrInfo.h - WebAssembly Instruction Information -*- C++ -*-=//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+///
+/// \file
+/// \brief This file contains the WebAssembly implementation of the
+/// TargetInstrInfo class.
+///
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIB_TARGET_WEBASSEMBLY_WEBASSEMBLYINSTRINFO_H
+#define LLVM_LIB_TARGET_WEBASSEMBLY_WEBASSEMBLYINSTRINFO_H
+
+#include "WebAssemblyRegisterInfo.h"
+#include "llvm/Target/TargetInstrInfo.h"
+
+#define GET_INSTRINFO_HEADER
+#include "WebAssemblyGenInstrInfo.inc"
+
+namespace llvm {
+
+class WebAssemblySubtarget;
+
+class WebAssemblyInstrInfo final : public WebAssemblyGenInstrInfo {
+  const WebAssemblyRegisterInfo RI;
+
+public:
+  explicit WebAssemblyInstrInfo(const WebAssemblySubtarget &STI);
+
+  const WebAssemblyRegisterInfo &getRegisterInfo() const { return RI; }
+};
+
+} // end namespace llvm
+
+#endif
diff --git a/lib/Target/WebAssembly/WebAssemblyInstrInfo.td b/lib/Target/WebAssembly/WebAssemblyInstrInfo.td
new file mode 100644
index 00000000000..6084d14874a
--- /dev/null
+++ b/lib/Target/WebAssembly/WebAssemblyInstrInfo.td
@@ -0,0 +1,72 @@
+// WebAssemblyInstrInfo.td-Describe the WebAssembly Instructions-*- tablegen -*-
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+///
+/// \file
+/// \brief WebAssembly Instruction definitions.
+///
+//===----------------------------------------------------------------------===//
+
+//===----------------------------------------------------------------------===//
+// WebAssembly Instruction Predicate Definitions.
+//===----------------------------------------------------------------------===//
+
+def HasAddr32 : Predicate<"!Subtarget->hasAddr64()">;
+def HasAddr64 : Predicate<"Subtarget->hasAddr64()">;
+def HasSIMD128 : Predicate<"Subtarget->hasSIMD128()">,
+                           AssemblerPredicate<"FeatureSIMD128", "simd128">;
+
+//===----------------------------------------------------------------------===//
+// WebAssembly-specific DAG Node Types.
+//===----------------------------------------------------------------------===//
+
+def SDT_WebAssemblyArgument : SDTypeProfile<1, 1, [SDTCisVT<1, i32>]>;
+def SDT_WebAssemblyReturn   : SDTypeProfile<0, 1, []>;
+
+//===----------------------------------------------------------------------===//
+// WebAssembly-specific DAG Nodes.
+//===----------------------------------------------------------------------===//
+
+def WebAssemblyargument : SDNode<"WebAssemblyISD::ARGUMENT",
+                                 SDT_WebAssemblyArgument>;
+def WebAssemblyreturn   : SDNode<"WebAssemblyISD::RETURN",
+                                 SDT_WebAssemblyReturn,
+                                 [SDNPHasChain, SDNPSideEffect]>;
+
+//===----------------------------------------------------------------------===//
+// WebAssembly-specific Operands.
+//===----------------------------------------------------------------------===//
+
+/*
+ * TODO(jfb): Add the following.
+ *
+ * get_local: read the current value of a local variable
+ * set_local: set the current value of a local variable
+*/
+
+//===----------------------------------------------------------------------===//
+// WebAssembly Instruction Format Definitions.
+//===----------------------------------------------------------------------===//
+
+include "WebAssemblyInstrFormats.td"
+
+def ARGUMENT : I<(outs Int32:$res), (ins i32imm:$argno),
+                 [(set Int32:$res, (WebAssemblyargument timm:$argno))]>;
+
+//===----------------------------------------------------------------------===//
+// Additional sets of instructions.
+//===----------------------------------------------------------------------===//
+
+include "WebAssemblyInstrMemory.td"
+include "WebAssemblyInstrCall.td"
+include "WebAssemblyInstrControl.td"
+include "WebAssemblyInstrInteger.td"
+include "WebAssemblyInstrFloat.td"
+include "WebAssemblyInstrConv.td"
+include "WebAssemblyInstrAtomics.td"
+include "WebAssemblyInstrSIMD.td"
diff --git a/lib/Target/WebAssembly/WebAssemblyInstrInteger.td b/lib/Target/WebAssembly/WebAssemblyInstrInteger.td
new file mode 100644
index 00000000000..5f60fe81b1a
--- /dev/null
+++ b/lib/Target/WebAssembly/WebAssemblyInstrInteger.td
@@ -0,0 +1,45 @@
+// WebAssemblyInstrInteger.td-WebAssembly Integer codegen -------*- tablegen -*-
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+///
+/// \file
+/// \brief WebAssembly Integer operand code-gen constructs.
+///
+//===----------------------------------------------------------------------===//
+
+defm ADD : BinaryInt<add>;
+defm SUB : BinaryInt<sub>;
+defm MUL : BinaryInt<mul>;
+defm SDIV : BinaryInt<sdiv>;
+defm UDIV : BinaryInt<udiv>;
+defm SREM : BinaryInt<srem>;
+defm UREM : BinaryInt<urem>;
+defm AND : BinaryInt<and>;
+defm IOR : BinaryInt<or>;
+defm XOR : BinaryInt<xor>;
+defm SHL : BinaryInt<shl>;
+defm SHR : BinaryInt<srl>;
+defm SAR : BinaryInt<sra>;
+
+/*
+ * TODO(jfb): Add the following for 32-bit and 64-bit.
+ *
+ * int32.eq: signed-less compare equal
+ * int32.slt: signed less than
+ * int32.sle: signed less than or equal
+ * int32.ult: unsigned less than
+ * int32.ule: unsigned less than or equal
+ * int32.sgt: signed greater than
+ * int32.sge: signed greater than or equal
+ * int32.ugt: unsigned greater than
+ * int32.uge: unsigned greater than or equal
+ */
+
+defm CLZ : UnaryInt<ctlz>;
+defm CTZ : UnaryInt<cttz>;
+defm POPCNT : UnaryInt<ctpop>;
diff --git a/lib/Target/WebAssembly/WebAssemblyInstrMemory.td b/lib/Target/WebAssembly/WebAssemblyInstrMemory.td
new file mode 100644
index 00000000000..5ab40e826ca
--- /dev/null
+++ b/lib/Target/WebAssembly/WebAssemblyInstrMemory.td
@@ -0,0 +1,46 @@
+// WebAssemblyInstrMemory.td-WebAssembly Memory codegen support -*- tablegen -*-
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+///
+/// \file
+/// \brief WebAssembly Memory operand code-gen constructs.
+///
+//===----------------------------------------------------------------------===//
+
+/*
+ * TODO(jfb): Add the following.
+ * Each has optional alignment and immediate byte offset.
+ *
+ * int32.load_sx[int8]: sign-extend to int32
+ * int32.load_sx[int16]: sign-extend to int32
+ * int32.load_zx[int8]: zero-extend to int32
+ * int32.load_zx[int16]: zero-extend to int32
+ * int32.load[int32]: (no conversion)
+ * int64.load_sx[int8]: sign-extend to int64
+ * int64.load_sx[int16]: sign-extend to int64
+ * int64.load_sx[int32]: sign-extend to int64
+ * int64.load_zx[int8]: zero-extend to int64
+ * int64.load_zx[int16]: zero-extend to int64
+ * int64.load_zx[int32]: zero-extend to int64
+ * int64.load[int64]: (no conversion)
+ * float32.load[float32]: (no conversion)
+ * float64.load[float64]: (no conversion)
+ * 
+ * int32.store[int8]: wrap int32 to int8
+ * int32.store[int16]: wrap int32 to int16
+ * int32.store[int32]: (no conversion)
+ * int64.store[int8]: wrap int64 to int8
+ * int64.store[int16]: wrap int64 to int16
+ * int64.store[int32]: wrap int64 to int32
+ * int64.store[int64]: (no conversion)
+ * float32.store[float32]: (no conversion)
+ * float64.store[float64]: (no conversion)
+ * 
+ * load_global: load the value of a given global variable
+ * store_global: store a given value to a given global variable
+ */
diff --git a/lib/Target/WebAssembly/WebAssemblyInstrSIMD.td b/lib/Target/WebAssembly/WebAssemblyInstrSIMD.td
new file mode 100644
index 00000000000..3e29906219d
--- /dev/null
+++ b/lib/Target/WebAssembly/WebAssemblyInstrSIMD.td
@@ -0,0 +1,16 @@
+// WebAssemblyInstrSIMD.td - WebAssembly SIMD codegen support -*- tablegen -*-//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+///
+/// \file
+/// \brief WebAssembly SIMD operand code-gen constructs.
+///
+//===----------------------------------------------------------------------===//
+
+// TODO: Implement SIMD instructions.
+// Note: use Requires<[HasSIMD128]>.
diff --git a/lib/Target/WebAssembly/WebAssemblyMachineFunctionInfo.cpp b/lib/Target/WebAssembly/WebAssemblyMachineFunctionInfo.cpp
new file mode 100644
index 00000000000..542d984b900
--- /dev/null
+++ b/lib/Target/WebAssembly/WebAssemblyMachineFunctionInfo.cpp
@@ -0,0 +1,19 @@
+//=- WebAssemblyMachineFunctionInfo.cpp - WebAssembly Machine Function Info -=//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+///
+/// \file
+/// \brief This file implements WebAssembly-specific per-machine-function
+/// information.
+///
+//===----------------------------------------------------------------------===//
+
+#include "WebAssemblyMachineFunctionInfo.h"
+using namespace llvm;
+
+WebAssemblyFunctionInfo::~WebAssemblyFunctionInfo() {}
diff --git a/lib/Target/WebAssembly/WebAssemblyMachineFunctionInfo.h b/lib/Target/WebAssembly/WebAssemblyMachineFunctionInfo.h
new file mode 100644
index 00000000000..22ef48116af
--- /dev/null
+++ b/lib/Target/WebAssembly/WebAssemblyMachineFunctionInfo.h
@@ -0,0 +1,37 @@
+// WebAssemblyMachineFunctionInfo.h-WebAssembly machine function info-*- C++ -*-
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+///
+/// \file
+/// \brief This file declares WebAssembly-specific per-machine-function
+/// information.
+///
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIB_TARGET_WEBASSEMBLY_WEBASSEMBLYMACHINEFUNCTIONINFO_H
+#define LLVM_LIB_TARGET_WEBASSEMBLY_WEBASSEMBLYMACHINEFUNCTIONINFO_H
+
+#include "WebAssemblyRegisterInfo.h"
+#include "llvm/CodeGen/MachineFunction.h"
+#include "llvm/CodeGen/MachineRegisterInfo.h"
+
+namespace llvm {
+
+/// This class is derived from MachineFunctionInfo and contains private
+/// WebAssembly-specific information for each MachineFunction.
+class WebAssemblyFunctionInfo final : public MachineFunctionInfo {
+  MachineFunction &MF;
+
+public:
+  explicit WebAssemblyFunctionInfo(MachineFunction &MF) : MF(MF) {}
+  ~WebAssemblyFunctionInfo() override;
+};
+
+} // end namespace llvm
+
+#endif
diff --git a/lib/Target/WebAssembly/WebAssemblyRegisterInfo.cpp b/lib/Target/WebAssembly/WebAssemblyRegisterInfo.cpp
new file mode 100644
index 00000000000..8a300373421
--- /dev/null
+++ b/lib/Target/WebAssembly/WebAssemblyRegisterInfo.cpp
@@ -0,0 +1,68 @@
+//===-- WebAssemblyRegisterInfo.cpp - WebAssembly Register Information ----===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+///
+/// \file
+/// \brief This file contains the WebAssembly implementation of the
+/// TargetRegisterInfo class.
+///
+//===----------------------------------------------------------------------===//
+
+#include "WebAssemblyRegisterInfo.h"
+#include "MCTargetDesc/WebAssemblyMCTargetDesc.h"
+#include "WebAssemblyFrameLowering.h"
+#include "WebAssemblyInstrInfo.h"
+#include "WebAssemblyMachineFunctionInfo.h"
+#include "WebAssemblySubtarget.h"
+#include "llvm/CodeGen/MachineFrameInfo.h"
+#include "llvm/CodeGen/MachineInstrBuilder.h"
+#include "llvm/CodeGen/MachineRegisterInfo.h"
+#include "llvm/IR/Function.h"
+#include "llvm/Support/raw_ostream.h"
+#include "llvm/Target/TargetFrameLowering.h"
+#include "llvm/Target/TargetOptions.h"
+using namespace llvm;
+
+#define DEBUG_TYPE "wasm-reg-info"
+
+#define GET_REGINFO_TARGET_DESC
+#include "WebAssemblyGenRegisterInfo.inc"
+
+WebAssemblyRegisterInfo::WebAssemblyRegisterInfo(const Triple &TT)
+    : WebAssemblyGenRegisterInfo(0), TT(TT) {}
+
+const MCPhysReg *
+WebAssemblyRegisterInfo::getCalleeSavedRegs(const MachineFunction *) const {
+  static const MCPhysReg CalleeSavedRegs[] = {0};
+  return CalleeSavedRegs;
+}
+
+BitVector
+WebAssemblyRegisterInfo::getReservedRegs(const MachineFunction &MF) const {
+  BitVector Reserved(getNumRegs());
+  for (auto Reg : {WebAssembly::SP32, WebAssembly::SP64, WebAssembly::FP32,
+                   WebAssembly::FP64})
+    Reserved.set(Reg);
+  return Reserved;
+}
+
+void WebAssemblyRegisterInfo::eliminateFrameIndex(
+    MachineBasicBlock::iterator II, int SPAdj, unsigned FIOperandNum,
+    RegScavenger *RS) const {
+  llvm_unreachable("WebAssemblyRegisterInfo::eliminateFrameIndex"); // FIXME
+}
+
+unsigned
+WebAssemblyRegisterInfo::getFrameRegister(const MachineFunction &MF) const {
+  static const unsigned Regs[2][2] = {
+      /*            !isArch64Bit       isArch64Bit      */
+      /* !hasFP */ {WebAssembly::SP32, WebAssembly::SP64},
+      /*  hasFP */ {WebAssembly::FP32, WebAssembly::FP64}};
+  const WebAssemblyFrameLowering *TFI = getFrameLowering(MF);
+  return Regs[TFI->hasFP(MF)][TT.isArch64Bit()];
+}
diff --git a/lib/Target/WebAssembly/WebAssemblyRegisterInfo.h b/lib/Target/WebAssembly/WebAssemblyRegisterInfo.h
new file mode 100644
index 00000000000..bd148531716
--- /dev/null
+++ b/lib/Target/WebAssembly/WebAssemblyRegisterInfo.h
@@ -0,0 +1,48 @@
+// WebAssemblyRegisterInfo.h - WebAssembly Register Information Impl -*- C++ -*-
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+///
+/// \file
+/// \brief This file contains the WebAssembly implementation of the
+/// WebAssemblyRegisterInfo class.
+///
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIB_TARGET_WEBASSEMBLY_WEBASSEMBLYREGISTERINFO_H
+#define LLVM_LIB_TARGET_WEBASSEMBLY_WEBASSEMBLYREGISTERINFO_H
+
+#define GET_REGINFO_HEADER
+#include "WebAssemblyGenRegisterInfo.inc"
+
+namespace llvm {
+
+class MachineFunction;
+class RegScavenger;
+class TargetRegisterClass;
+class Triple;
+
+class WebAssemblyRegisterInfo final : public WebAssemblyGenRegisterInfo {
+  const Triple &TT;
+
+public:
+  explicit WebAssemblyRegisterInfo(const Triple &TT);
+
+  // Code Generation virtual methods.
+  const MCPhysReg *getCalleeSavedRegs(const MachineFunction *MF) const override;
+  BitVector getReservedRegs(const MachineFunction &MF) const override;
+  void eliminateFrameIndex(MachineBasicBlock::iterator MI, int SPAdj,
+                           unsigned FIOperandNum,
+                           RegScavenger *RS = nullptr) const override;
+
+  // Debug information queries.
+  unsigned getFrameRegister(const MachineFunction &MF) const override;
+};
+
+} // end namespace llvm
+
+#endif
diff --git a/lib/Target/WebAssembly/WebAssemblyRegisterInfo.td b/lib/Target/WebAssembly/WebAssemblyRegisterInfo.td
new file mode 100644
index 00000000000..2ba42eb94a4
--- /dev/null
+++ b/lib/Target/WebAssembly/WebAssemblyRegisterInfo.td
@@ -0,0 +1,54 @@
+//WebAssemblyRegisterInfo.td-Describe the WebAssembly Registers -*- tablegen -*-
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+///
+/// \file
+/// \brief This file describes the WebAssembly register classes and some nominal
+/// physical registers.
+///
+//===----------------------------------------------------------------------===//
+
+class WebAssemblyReg<string n> : Register<n> {
+  let Namespace = "WebAssembly";
+}
+
+class WebAssemblyRegClass<list<ValueType> regTypes, int alignment, dag regList>
+     : RegisterClass<"WebAssembly", regTypes, alignment, regList>;
+
+//===----------------------------------------------------------------------===//
+// Registers
+//===----------------------------------------------------------------------===//
+
+// Special registers used as the frame and stack pointer.
+//
+// WebAssembly may someday supports mixed 32-bit and 64-bit heaps in the same
+// application, which requires separate width FP and SP.
+def FP32 : WebAssemblyReg<"%FP32">;
+def FP64 : WebAssemblyReg<"%FP64">;
+def SP32 : WebAssemblyReg<"%SP32">;
+def SP64 : WebAssemblyReg<"%SP64">;
+
+// TODO(jfb) The following comes from NVPTX. Is it really needed, or can we do
+//           away with it? Try deleting once the backend works.
+// WebAssembly uses virtual registers, but the backend defines a few physical
+// registers here to keep SDAG and the MachineInstr layers happy.
+foreach i = 0-4 in {
+  def I#i : WebAssemblyReg<"%i."#i>; // i32
+  def L#i : WebAssemblyReg<"%l."#i>; // i64
+  def F#i : WebAssemblyReg<"%f."#i>; // f32
+  def D#i : WebAssemblyReg<"%d."#i>; // f64
+}
+
+//===----------------------------------------------------------------------===//
+//  Register classes
+//===----------------------------------------------------------------------===//
+
+def Int32 : WebAssemblyRegClass<[i32], 32, (add (sequence "I%u", 0, 4), SP32)>;
+def Int64 : WebAssemblyRegClass<[i64], 64, (add (sequence "L%u", 0, 4), SP64)>;
+def Float32 : WebAssemblyRegClass<[f32], 32, (add (sequence "F%u", 0, 4))>;
+def Float64 : WebAssemblyRegClass<[f64], 64, (add (sequence "D%u", 0, 4))>;
diff --git a/lib/Target/WebAssembly/WebAssemblySelectionDAGInfo.cpp b/lib/Target/WebAssembly/WebAssemblySelectionDAGInfo.cpp
new file mode 100644
index 00000000000..fae9c610051
--- /dev/null
+++ b/lib/Target/WebAssembly/WebAssemblySelectionDAGInfo.cpp
@@ -0,0 +1,20 @@
+//===-- WebAssemblySelectionDAGInfo.cpp - WebAssembly SelectionDAG Info ---===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+///
+/// \file
+/// \brief This file implements the WebAssemblySelectionDAGInfo class.
+///
+//===----------------------------------------------------------------------===//
+
+#include "WebAssemblyTargetMachine.h"
+using namespace llvm;
+
+#define DEBUG_TYPE "wasm-selectiondag-info"
+
+WebAssemblySelectionDAGInfo::~WebAssemblySelectionDAGInfo() {}
diff --git a/lib/Target/WebAssembly/WebAssemblySelectionDAGInfo.h b/lib/Target/WebAssembly/WebAssemblySelectionDAGInfo.h
new file mode 100644
index 00000000000..13d96671276
--- /dev/null
+++ b/lib/Target/WebAssembly/WebAssemblySelectionDAGInfo.h
@@ -0,0 +1,30 @@
+//=- WebAssemblySelectionDAGInfo.h - WebAssembly SelectionDAG Info -*- C++ -*-//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+///
+/// \file
+/// \brief This file defines the WebAssembly subclass for
+/// TargetSelectionDAGInfo.
+///
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIB_TARGET_WEBASSEMBLY_WEBASSEMBLYSELECTIONDAGINFO_H
+#define LLVM_LIB_TARGET_WEBASSEMBLY_WEBASSEMBLYSELECTIONDAGINFO_H
+
+#include "llvm/Target/TargetSelectionDAGInfo.h"
+
+namespace llvm {
+
+class WebAssemblySelectionDAGInfo final : public TargetSelectionDAGInfo {
+public:
+  ~WebAssemblySelectionDAGInfo() override;
+};
+
+} // end namespace llvm
+
+#endif
diff --git a/lib/Target/WebAssembly/WebAssemblySubtarget.cpp b/lib/Target/WebAssembly/WebAssemblySubtarget.cpp
new file mode 100644
index 00000000000..3d9e7aacbfb
--- /dev/null
+++ b/lib/Target/WebAssembly/WebAssemblySubtarget.cpp
@@ -0,0 +1,48 @@
+//===-- WebAssemblySubtarget.cpp - WebAssembly Subtarget Information ------===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+///
+/// \file
+/// \brief This file implements the WebAssembly-specific subclass of
+/// TargetSubtarget.
+///
+//===----------------------------------------------------------------------===//
+
+#include "WebAssemblyInstrInfo.h"
+#include "MCTargetDesc/WebAssemblyMCTargetDesc.h"
+#include "WebAssemblySubtarget.h"
+#include "llvm/Support/TargetRegistry.h"
+using namespace llvm;
+
+#define DEBUG_TYPE "wasm-subtarget"
+
+#define GET_SUBTARGETINFO_CTOR
+#define GET_SUBTARGETINFO_TARGET_DESC
+#include "WebAssemblyGenSubtargetInfo.inc"
+
+WebAssemblySubtarget &
+WebAssemblySubtarget::initializeSubtargetDependencies(StringRef FS) {
+  // Determine default and user-specified characteristics
+
+  if (CPUString.empty())
+    CPUString = "generic";
+
+  ParseSubtargetFeatures(CPUString, FS);
+  return *this;
+}
+
+WebAssemblySubtarget::WebAssemblySubtarget(const Triple &TT,
+                                           const std::string &CPU,
+                                           const std::string &FS,
+                                           const TargetMachine &TM)
+    : WebAssemblyGenSubtargetInfo(TT, CPU, FS), HasSIMD128(false),
+      CPUString(CPU), TargetTriple(TT), FrameLowering(),
+      InstrInfo(initializeSubtargetDependencies(FS)), TSInfo(),
+      TLInfo(TM, *this) {}
+
+bool WebAssemblySubtarget::enableMachineScheduler() const { return true; }
diff --git a/lib/Target/WebAssembly/WebAssemblySubtarget.h b/lib/Target/WebAssembly/WebAssemblySubtarget.h
new file mode 100644
index 00000000000..9b17300e497
--- /dev/null
+++ b/lib/Target/WebAssembly/WebAssemblySubtarget.h
@@ -0,0 +1,85 @@
+//=- WebAssemblySubtarget.h - Define Subtarget for the WebAssembly -*- C++ -*-//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+///
+/// \file
+/// \brief This file declares the WebAssembly-specific subclass of
+/// TargetSubtarget.
+///
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIB_TARGET_WEBASSEMBLY_WEBASSEMBLYSUBTARGET_H
+#define LLVM_LIB_TARGET_WEBASSEMBLY_WEBASSEMBLYSUBTARGET_H
+
+#include "WebAssemblyFrameLowering.h"
+#include "WebAssemblyISelLowering.h"
+#include "WebAssemblyInstrInfo.h"
+#include "WebAssemblySelectionDAGInfo.h"
+#include "llvm/Target/TargetSubtargetInfo.h"
+#include <string>
+
+#define GET_SUBTARGETINFO_HEADER
+#include "WebAssemblyGenSubtargetInfo.inc"
+
+namespace llvm {
+
+class WebAssemblySubtarget final : public WebAssemblyGenSubtargetInfo {
+  bool HasSIMD128;
+
+  /// String name of used CPU.
+  std::string CPUString;
+
+  /// What processor and OS we're targeting.
+  Triple TargetTriple;
+
+  WebAssemblyFrameLowering FrameLowering;
+  WebAssemblyInstrInfo InstrInfo;
+  WebAssemblySelectionDAGInfo TSInfo;
+  WebAssemblyTargetLowering TLInfo;
+
+  /// Initializes using CPUString and the passed in feature string so that we
+  /// can use initializer lists for subtarget initialization.
+  WebAssemblySubtarget &initializeSubtargetDependencies(StringRef FS);
+
+public:
+  /// This constructor initializes the data members to match that
+  /// of the specified triple.
+  WebAssemblySubtarget(const Triple &TT, const std::string &CPU,
+                       const std::string &FS, const TargetMachine &TM);
+
+  const WebAssemblySelectionDAGInfo *getSelectionDAGInfo() const override {
+    return &TSInfo;
+  }
+  const WebAssemblyFrameLowering *getFrameLowering() const override {
+    return &FrameLowering;
+  }
+  const WebAssemblyTargetLowering *getTargetLowering() const override {
+    return &TLInfo;
+  }
+  const WebAssemblyInstrInfo *getInstrInfo() const override {
+    return &InstrInfo;
+  }
+  const WebAssemblyRegisterInfo *getRegisterInfo() const override {
+    return &getInstrInfo()->getRegisterInfo();
+  }
+  const Triple &getTargetTriple() const { return TargetTriple; }
+  bool enableMachineScheduler() const override;
+  bool useAA() const override { return true; }
+
+  // Predicates used by WebAssemblyInstrInfo.td.
+  bool hasAddr64() const { return TargetTriple.isArch64Bit(); }
+  bool hasSIMD128() const { return HasSIMD128; }
+
+  /// Parses features string setting specified subtarget options. Definition of
+  /// function is auto generated by tblgen.
+  void ParseSubtargetFeatures(StringRef CPU, StringRef FS);
+};
+
+} // end namespace llvm
+
+#endif
diff --git a/lib/Target/WebAssembly/WebAssemblyTargetMachine.cpp b/lib/Target/WebAssembly/WebAssemblyTargetMachine.cpp
new file mode 100644
index 00000000000..bac635d76fa
--- /dev/null
+++ b/lib/Target/WebAssembly/WebAssemblyTargetMachine.cpp
@@ -0,0 +1,181 @@
+//===- WebAssemblyTargetMachine.cpp - Define TargetMachine for WebAssembly -==//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+///
+/// \file
+/// \brief This file defines the WebAssembly-specific subclass of TargetMachine.
+///
+//===----------------------------------------------------------------------===//
+
+#include "WebAssembly.h"
+#include "MCTargetDesc/WebAssemblyMCTargetDesc.h"
+#include "WebAssemblyTargetMachine.h"
+#include "WebAssemblyTargetObjectFile.h"
+#include "WebAssemblyTargetTransformInfo.h"
+#include "llvm/CodeGen/MachineFunctionPass.h"
+#include "llvm/CodeGen/Passes.h"
+#include "llvm/CodeGen/RegAllocRegistry.h"
+#include "llvm/IR/Function.h"
+#include "llvm/Support/CommandLine.h"
+#include "llvm/Support/TargetRegistry.h"
+#include "llvm/Target/TargetOptions.h"
+#include "llvm/Transforms/Scalar.h"
+using namespace llvm;
+
+#define DEBUG_TYPE "wasm"
+
+extern "C" void LLVMInitializeWebAssemblyTarget() {
+  // Register the target.
+  RegisterTargetMachine<WebAssemblyTargetMachine> X(TheWebAssemblyTarget32);
+  RegisterTargetMachine<WebAssemblyTargetMachine> Y(TheWebAssemblyTarget64);
+}
+
+//===----------------------------------------------------------------------===//
+// WebAssembly Lowering public interface.
+//===----------------------------------------------------------------------===//
+
+/// Create an WebAssembly architecture model.
+///
+WebAssemblyTargetMachine::WebAssemblyTargetMachine(
+    const Target &T, const Triple &TT, StringRef CPU, StringRef FS,
+    const TargetOptions &Options, Reloc::Model RM, CodeModel::Model CM,
+    CodeGenOpt::Level OL)
+    : LLVMTargetMachine(T, TT.isArch64Bit()
+                               ? "e-p:64:64-i64:64-v128:8:128-n32:64-S128"
+                               : "e-p:32:32-i64:64-v128:8:128-n32:64-S128",
+                        TT, CPU, FS, Options, RM, CM, OL),
+      TLOF(make_unique<WebAssemblyTargetObjectFile>()) {
+  initAsmInfo();
+
+  // We need a reducible CFG, so disable some optimizations which tend to
+  // introduce irreducibility.
+  setRequiresStructuredCFG(true);
+}
+
+WebAssemblyTargetMachine::~WebAssemblyTargetMachine() {}
+
+const WebAssemblySubtarget *
+WebAssemblyTargetMachine::getSubtargetImpl(const Function &F) const {
+  Attribute CPUAttr = F.getFnAttribute("target-cpu");
+  Attribute FSAttr = F.getFnAttribute("target-features");
+
+  std::string CPU = !CPUAttr.hasAttribute(Attribute::None)
+                        ? CPUAttr.getValueAsString().str()
+                        : TargetCPU;
+  std::string FS = !FSAttr.hasAttribute(Attribute::None)
+                       ? FSAttr.getValueAsString().str()
+                       : TargetFS;
+
+  auto &I = SubtargetMap[CPU + FS];
+  if (!I) {
+    // This needs to be done before we create a new subtarget since any
+    // creation will depend on the TM and the code generation flags on the
+    // function that reside in TargetOptions.
+    resetTargetOptions(F);
+    I = make_unique<WebAssemblySubtarget>(TargetTriple, CPU, FS, *this);
+  }
+  return I.get();
+}
+
+namespace {
+/// WebAssembly Code Generator Pass Configuration Options.
+class WebAssemblyPassConfig final : public TargetPassConfig {
+public:
+  WebAssemblyPassConfig(WebAssemblyTargetMachine *TM, PassManagerBase &PM)
+      : TargetPassConfig(TM, PM) {}
+
+  WebAssemblyTargetMachine &getWebAssemblyTargetMachine() const {
+    return getTM<WebAssemblyTargetMachine>();
+  }
+
+  FunctionPass *createTargetRegisterAllocator(bool) override;
+  void addFastRegAlloc(FunctionPass *RegAllocPass) override;
+  void addOptimizedRegAlloc(FunctionPass *RegAllocPass) override;
+
+  void addIRPasses() override;
+  bool addPreISel() override;
+  bool addInstSelector() override;
+  bool addILPOpts() override;
+  void addPreRegAlloc() override;
+  void addRegAllocPasses(bool Optimized);
+  void addPostRegAlloc() override;
+  void addPreSched2() override;
+  void addPreEmitPass() override;
+};
+} // end anonymous namespace
+
+TargetIRAnalysis WebAssemblyTargetMachine::getTargetIRAnalysis() {
+  return TargetIRAnalysis([this](Function &F) {
+    return TargetTransformInfo(WebAssemblyTTIImpl(this, F));
+  });
+}
+
+TargetPassConfig *
+WebAssemblyTargetMachine::createPassConfig(PassManagerBase &PM) {
+  return new WebAssemblyPassConfig(this, PM);
+}
+
+FunctionPass *WebAssemblyPassConfig::createTargetRegisterAllocator(bool) {
+  return nullptr; // No reg alloc
+}
+
+void WebAssemblyPassConfig::addFastRegAlloc(FunctionPass *RegAllocPass) {
+  assert(!RegAllocPass && "WebAssembly uses no regalloc!");
+  addRegAllocPasses(false);
+}
+
+void WebAssemblyPassConfig::addOptimizedRegAlloc(FunctionPass *RegAllocPass) {
+  assert(!RegAllocPass && "WebAssembly uses no regalloc!");
+  addRegAllocPasses(true);
+}
+
+//===----------------------------------------------------------------------===//
+// The following functions are called from lib/CodeGen/Passes.cpp to modify
+// the CodeGen pass sequence.
+//===----------------------------------------------------------------------===//
+
+void WebAssemblyPassConfig::addIRPasses() {
+  // FIXME: the default for this option is currently POSIX, whereas
+  // WebAssembly's MVP should default to Single.
+  if (TM->Options.ThreadModel == ThreadModel::Single)
+    addPass(createLowerAtomicPass());
+  else
+    // Expand some atomic operations. WebAssemblyTargetLowering has hooks which
+    // control specifically what gets lowered.
+    addPass(createAtomicExpandPass(TM));
+
+  TargetPassConfig::addIRPasses();
+}
+
+bool WebAssemblyPassConfig::addPreISel() { return false; }
+
+bool WebAssemblyPassConfig::addInstSelector() {
+  addPass(
+      createWebAssemblyISelDag(getWebAssemblyTargetMachine(), getOptLevel()));
+  return false;
+}
+
+bool WebAssemblyPassConfig::addILPOpts() { return true; }
+
+void WebAssemblyPassConfig::addPreRegAlloc() {}
+
+void WebAssemblyPassConfig::addRegAllocPasses(bool Optimized) {}
+
+void WebAssemblyPassConfig::addPostRegAlloc() {
+  // FIXME: the following passes dislike virtual registers. Disable them for now
+  //        so that basic tests can pass. Future patches will remedy this.
+  //
+  // Fails with: Regalloc must assign all vregs.
+  disablePass(&PrologEpilogCodeInserterID);
+  // Fails with: should be run after register allocation.
+  disablePass(&MachineCopyPropagationID);
+}
+
+void WebAssemblyPassConfig::addPreSched2() {}
+
+void WebAssemblyPassConfig::addPreEmitPass() {}
diff --git a/lib/Target/WebAssembly/WebAssemblyTargetMachine.h b/lib/Target/WebAssembly/WebAssemblyTargetMachine.h
new file mode 100644
index 00000000000..3226edcdc61
--- /dev/null
+++ b/lib/Target/WebAssembly/WebAssemblyTargetMachine.h
@@ -0,0 +1,51 @@
+// WebAssemblyTargetMachine.h - Define TargetMachine for WebAssembly -*- C++ -*-
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+///
+/// \file
+/// \brief This file declares the WebAssembly-specific subclass of
+/// TargetMachine.
+///
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIB_TARGET_WEBASSEMBLY_WEBASSEMBLYTARGETMACHINE_H
+#define LLVM_LIB_TARGET_WEBASSEMBLY_WEBASSEMBLYTARGETMACHINE_H
+
+#include "WebAssemblySubtarget.h"
+#include "llvm/Target/TargetMachine.h"
+
+namespace llvm {
+
+class WebAssemblyTargetMachine final : public LLVMTargetMachine {
+  std::unique_ptr<TargetLoweringObjectFile> TLOF;
+  mutable StringMap<std::unique_ptr<WebAssemblySubtarget>> SubtargetMap;
+
+public:
+  WebAssemblyTargetMachine(const Target &T, const Triple &TT, StringRef CPU,
+                           StringRef FS, const TargetOptions &Options,
+                           Reloc::Model RM, CodeModel::Model CM,
+                           CodeGenOpt::Level OL);
+
+  ~WebAssemblyTargetMachine() override;
+  const WebAssemblySubtarget *
+  getSubtargetImpl(const Function &F) const override;
+
+  // Pass Pipeline Configuration
+  TargetPassConfig *createPassConfig(PassManagerBase &PM) override;
+
+  TargetLoweringObjectFile *getObjFileLowering() const override {
+    return TLOF.get();
+  }
+
+  /// \brief Get the TargetIRAnalysis for this target.
+  TargetIRAnalysis getTargetIRAnalysis() override;
+};
+
+} // end namespace llvm
+
+#endif
diff --git a/lib/Target/WebAssembly/WebAssemblyTargetObjectFile.h b/lib/Target/WebAssembly/WebAssemblyTargetObjectFile.h
new file mode 100644
index 00000000000..b5382340a61
--- /dev/null
+++ b/lib/Target/WebAssembly/WebAssemblyTargetObjectFile.h
@@ -0,0 +1,67 @@
+//===-- WebAssemblyTargetObjectFile.h - WebAssembly Object Info -*- C++ -*-===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+///
+/// \file
+/// \brief This file declares the WebAssembly-specific subclass of
+/// TargetLoweringObjectFile.
+///
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIB_TARGET_WEBASSEMBLY_WEBASSEMBLYTARGETOBJECTFILE_H
+#define LLVM_LIB_TARGET_WEBASSEMBLY_WEBASSEMBLYTARGETOBJECTFILE_H
+
+#include "llvm/Target/TargetLoweringObjectFile.h"
+
+namespace llvm {
+
+class GlobalVariable;
+
+class WebAssemblyTargetObjectFile final : public TargetLoweringObjectFile {
+public:
+  WebAssemblyTargetObjectFile() {
+    TextSection = nullptr;
+    DataSection = nullptr;
+    BSSSection = nullptr;
+    ReadOnlySection = nullptr;
+
+    StaticCtorSection = nullptr;
+    StaticDtorSection = nullptr;
+    LSDASection = nullptr;
+    EHFrameSection = nullptr;
+    DwarfAbbrevSection = nullptr;
+    DwarfInfoSection = nullptr;
+    DwarfLineSection = nullptr;
+    DwarfFrameSection = nullptr;
+    DwarfPubTypesSection = nullptr;
+    DwarfDebugInlineSection = nullptr;
+    DwarfStrSection = nullptr;
+    DwarfLocSection = nullptr;
+    DwarfARangesSection = nullptr;
+    DwarfRangesSection = nullptr;
+  }
+
+  MCSection *getSectionForConstant(const DataLayout &DL, SectionKind Kind,
+                                   const Constant *C) const override {
+    return ReadOnlySection;
+  }
+
+  MCSection *getExplicitSectionGlobal(const GlobalValue *GV, SectionKind Kind,
+                                      Mangler &Mang,
+                                      const TargetMachine &TM) const override {
+    return DataSection;
+  }
+
+  MCSection *SelectSectionForGlobal(const GlobalValue *GV, SectionKind Kind,
+                                    Mangler &Mang,
+                                    const TargetMachine &TM) const override;
+};
+
+} // end namespace llvm
+
+#endif
diff --git a/lib/Target/WebAssembly/WebAssemblyTargetTransformInfo.cpp b/lib/Target/WebAssembly/WebAssemblyTargetTransformInfo.cpp
new file mode 100644
index 00000000000..fa88ed526df
--- /dev/null
+++ b/lib/Target/WebAssembly/WebAssemblyTargetTransformInfo.cpp
@@ -0,0 +1,28 @@
+//===-- WebAssemblyTargetTransformInfo.cpp - WebAssembly-specific TTI -----===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+///
+/// \file
+/// \brief This file defines the WebAssembly-specific TargetTransformInfo
+/// implementation.
+///
+//===----------------------------------------------------------------------===//
+
+#include "WebAssemblyTargetTransformInfo.h"
+#include "llvm/Support/Debug.h"
+#include "llvm/Target/CostTable.h"
+using namespace llvm;
+
+#define DEBUG_TYPE "wasmtti"
+
+TargetTransformInfo::PopcntSupportKind
+WebAssemblyTTIImpl::getPopcntSupport(unsigned TyWidth) {
+  assert(isPowerOf2_32(TyWidth) && "Ty width must be power of 2");
+  // TODO: Make Math.popcount32 happen in WebAssembly.
+  return TTI::PSK_Software;
+}
diff --git a/lib/Target/WebAssembly/WebAssemblyTargetTransformInfo.h b/lib/Target/WebAssembly/WebAssemblyTargetTransformInfo.h
new file mode 100644
index 00000000000..7ffb6047b96
--- /dev/null
+++ b/lib/Target/WebAssembly/WebAssemblyTargetTransformInfo.h
@@ -0,0 +1,71 @@
+//==- WebAssemblyTargetTransformInfo.h - WebAssembly-specific TTI -*- C++ -*-=//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+///
+/// \file
+/// \brief This file a TargetTransformInfo::Concept conforming object specific
+/// to the WebAssembly target machine.
+///
+/// It uses the target's detailed information to provide more precise answers to
+/// certain TTI queries, while letting the target independent and default TTI
+/// implementations handle the rest.
+///
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIB_TARGET_WEBASSEMBLY_WEBASSEMBLYTARGETTRANSFORMINFO_H
+#define LLVM_LIB_TARGET_WEBASSEMBLY_WEBASSEMBLYTARGETTRANSFORMINFO_H
+
+#include "WebAssemblyTargetMachine.h"
+#include "llvm/CodeGen/BasicTTIImpl.h"
+#include <algorithm>
+
+namespace llvm {
+
+class WebAssemblyTTIImpl final : public BasicTTIImplBase<WebAssemblyTTIImpl> {
+  typedef BasicTTIImplBase<WebAssemblyTTIImpl> BaseT;
+  typedef TargetTransformInfo TTI;
+  friend BaseT;
+
+  const WebAssemblySubtarget *ST;
+  const WebAssemblyTargetLowering *TLI;
+
+  const WebAssemblySubtarget *getST() const { return ST; }
+  const WebAssemblyTargetLowering *getTLI() const { return TLI; }
+
+public:
+  WebAssemblyTTIImpl(const WebAssemblyTargetMachine *TM, Function &F)
+      : BaseT(TM, F.getParent()->getDataLayout()), ST(TM->getSubtargetImpl(F)),
+        TLI(ST->getTargetLowering()) {}
+
+  // Provide value semantics. MSVC requires that we spell all of these out.
+  WebAssemblyTTIImpl(const WebAssemblyTTIImpl &Arg)
+      : BaseT(static_cast<const BaseT &>(Arg)), ST(Arg.ST), TLI(Arg.TLI) {}
+  WebAssemblyTTIImpl(WebAssemblyTTIImpl &&Arg)
+      : BaseT(std::move(static_cast<BaseT &>(Arg))), ST(std::move(Arg.ST)),
+        TLI(std::move(Arg.TLI)) {}
+
+  /// \name Scalar TTI Implementations
+  /// @{
+
+  // TODO: Implement more Scalar TTI for WebAssembly
+
+  TTI::PopcntSupportKind getPopcntSupport(unsigned TyWidth);
+
+  /// @}
+
+  /// \name Vector TTI Implementations
+  /// @{
+
+  // TODO: Implement Vector TTI for WebAssembly
+
+  /// @}
+};
+
+} // end namespace llvm
+
+#endif
diff --git a/lib/Target/X86/AsmParser/X86AsmInstrumentation.cpp b/lib/Target/X86/AsmParser/X86AsmInstrumentation.cpp
index 543af8e4e77..9eee4a0f3d8 100644
--- a/lib/Target/X86/AsmParser/X86AsmInstrumentation.cpp
+++ b/lib/Target/X86/AsmParser/X86AsmInstrumentation.cpp
@@ -14,7 +14,6 @@
 #include "llvm/ADT/StringExtras.h"
 #include "llvm/ADT/Triple.h"
 #include "llvm/CodeGen/MachineValueType.h"
-#include "llvm/IR/Function.h"
 #include "llvm/MC/MCAsmInfo.h"
 #include "llvm/MC/MCContext.h"
 #include "llvm/MC/MCInst.h"
@@ -246,7 +245,7 @@ class X86AddressSanitizer : public X86AsmInstrumentation {
     assert(VT == MVT::i32 || VT == MVT::i64);
     MCInst Inst;
     Inst.setOpcode(VT == MVT::i32 ? X86::LEA32r : X86::LEA64r);
-    Inst.addOperand(MCOperand::CreateReg(getX86SubSuperRegister(Reg, VT)));
+    Inst.addOperand(MCOperand::createReg(getX86SubSuperRegister(Reg, VT)));
     Op.addMemOperands(Inst, 5);
     EmitInstruction(Out, Inst);
   }
@@ -262,13 +261,13 @@ class X86AddressSanitizer : public X86AsmInstrumentation {
                                               MCContext &Ctx, int64_t *Residue);
 
   bool is64BitMode() const {
-    return (STI.getFeatureBits() & X86::Mode64Bit) != 0;
+    return STI.getFeatureBits()[X86::Mode64Bit];
   }
   bool is32BitMode() const {
-    return (STI.getFeatureBits() & X86::Mode32Bit) != 0;
+    return STI.getFeatureBits()[X86::Mode32Bit];
   }
   bool is16BitMode() const {
-    return (STI.getFeatureBits() & X86::Mode16Bit) != 0;
+    return STI.getFeatureBits()[X86::Mode16Bit];
   }
 
   unsigned getPointerWidth() {
@@ -316,7 +315,7 @@ void X86AddressSanitizer::InstrumentMOVSBase(unsigned DstReg, unsigned SrcReg,
 
   // Test (%SrcReg)
   {
-    const MCExpr *Disp = MCConstantExpr::Create(0, Ctx);
+    const MCExpr *Disp = MCConstantExpr::create(0, Ctx);
     std::unique_ptr<X86Operand> Op(X86Operand::CreateMem(
         getPointerWidth(), 0, Disp, SrcReg, 0, AccessSize, SMLoc(), SMLoc()));
     InstrumentMemOperand(*Op, AccessSize, false /* IsWrite */, RegCtx, Ctx,
@@ -325,7 +324,7 @@ void X86AddressSanitizer::InstrumentMOVSBase(unsigned DstReg, unsigned SrcReg,
 
   // Test -1(%SrcReg, %CntReg, AccessSize)
   {
-    const MCExpr *Disp = MCConstantExpr::Create(-1, Ctx);
+    const MCExpr *Disp = MCConstantExpr::create(-1, Ctx);
     std::unique_ptr<X86Operand> Op(X86Operand::CreateMem(
         getPointerWidth(), 0, Disp, SrcReg, CntReg, AccessSize, SMLoc(),
         SMLoc()));
@@ -335,7 +334,7 @@ void X86AddressSanitizer::InstrumentMOVSBase(unsigned DstReg, unsigned SrcReg,
 
   // Test (%DstReg)
   {
-    const MCExpr *Disp = MCConstantExpr::Create(0, Ctx);
+    const MCExpr *Disp = MCConstantExpr::create(0, Ctx);
     std::unique_ptr<X86Operand> Op(X86Operand::CreateMem(
         getPointerWidth(), 0, Disp, DstReg, 0, AccessSize, SMLoc(), SMLoc()));
     InstrumentMemOperand(*Op, AccessSize, true /* IsWrite */, RegCtx, Ctx, Out);
@@ -343,7 +342,7 @@ void X86AddressSanitizer::InstrumentMOVSBase(unsigned DstReg, unsigned SrcReg,
 
   // Test -1(%DstReg, %CntReg, AccessSize)
   {
-    const MCExpr *Disp = MCConstantExpr::Create(-1, Ctx);
+    const MCExpr *Disp = MCConstantExpr::create(-1, Ctx);
     std::unique_ptr<X86Operand> Op(X86Operand::CreateMem(
         getPointerWidth(), 0, Disp, DstReg, CntReg, AccessSize, SMLoc(),
         SMLoc()));
@@ -462,7 +461,7 @@ void X86AddressSanitizer::ComputeMemOperandAddress(X86Operand &Op,
 
   while (Residue != 0) {
     const MCConstantExpr *Disp =
-        MCConstantExpr::Create(ApplyDisplacementBounds(Residue), Ctx);
+        MCConstantExpr::create(ApplyDisplacementBounds(Residue), Ctx);
     std::unique_ptr<X86Operand> DispOp =
         X86Operand::CreateMem(getPointerWidth(), 0, Disp, Reg, 0, 1, SMLoc(),
                               SMLoc());
@@ -494,7 +493,7 @@ X86AddressSanitizer::AddDisplacement(X86Operand &Op, int64_t Displacement,
   CheckDisplacementBounds(NewDisplacement);
 
   *Residue = Displacement - NewDisplacement;
-  const MCExpr *Disp = MCConstantExpr::Create(NewDisplacement, Ctx);
+  const MCExpr *Disp = MCConstantExpr::create(NewDisplacement, Ctx);
   return X86Operand::CreateMem(Op.getMemModeSize(), Op.getMemSegReg(), Disp,
                                Op.getMemBaseReg(), Op.getMemIndexReg(),
                                Op.getMemScale(), SMLoc(), SMLoc());
@@ -614,9 +613,9 @@ class X86AddressSanitizer32 : public X86AddressSanitizer {
         Out, MCInstBuilder(X86::PUSH32r).addReg(RegCtx.AddressReg(MVT::i32)));
 
     const std::string &Fn = FuncName(AccessSize, IsWrite);
-    MCSymbol *FnSym = Ctx.GetOrCreateSymbol(StringRef(Fn));
+    MCSymbol *FnSym = Ctx.getOrCreateSymbol(StringRef(Fn));
     const MCSymbolRefExpr *FnExpr =
-        MCSymbolRefExpr::Create(FnSym, MCSymbolRefExpr::VK_PLT, Ctx);
+        MCSymbolRefExpr::create(FnSym, MCSymbolRefExpr::VK_PLT, Ctx);
     EmitInstruction(Out, MCInstBuilder(X86::CALLpcrel32).addExpr(FnExpr));
   }
 };
@@ -643,8 +642,8 @@ void X86AddressSanitizer32::InstrumentMemOperandSmall(
   {
     MCInst Inst;
     Inst.setOpcode(X86::MOV8rm);
-    Inst.addOperand(MCOperand::CreateReg(ShadowRegI8));
-    const MCExpr *Disp = MCConstantExpr::Create(kShadowOffset, Ctx);
+    Inst.addOperand(MCOperand::createReg(ShadowRegI8));
+    const MCExpr *Disp = MCConstantExpr::create(kShadowOffset, Ctx);
     std::unique_ptr<X86Operand> Op(
         X86Operand::CreateMem(getPointerWidth(), 0, Disp, ShadowRegI32, 0, 1,
                               SMLoc(), SMLoc()));
@@ -654,8 +653,8 @@ void X86AddressSanitizer32::InstrumentMemOperandSmall(
 
   EmitInstruction(
       Out, MCInstBuilder(X86::TEST8rr).addReg(ShadowRegI8).addReg(ShadowRegI8));
-  MCSymbol *DoneSym = Ctx.CreateTempSymbol();
-  const MCExpr *DoneExpr = MCSymbolRefExpr::Create(DoneSym, Ctx);
+  MCSymbol *DoneSym = Ctx.createTempSymbol();
+  const MCExpr *DoneExpr = MCSymbolRefExpr::create(DoneSym, Ctx);
   EmitInstruction(Out, MCInstBuilder(X86::JE_1).addExpr(DoneExpr));
 
   EmitInstruction(Out, MCInstBuilder(X86::MOV32rr).addReg(ScratchRegI32).addReg(
@@ -670,7 +669,7 @@ void X86AddressSanitizer32::InstrumentMemOperandSmall(
   case 1:
     break;
   case 2: {
-    const MCExpr *Disp = MCConstantExpr::Create(1, Ctx);
+    const MCExpr *Disp = MCConstantExpr::create(1, Ctx);
     std::unique_ptr<X86Operand> Op(
         X86Operand::CreateMem(getPointerWidth(), 0, Disp, ScratchRegI32, 0, 1,
                               SMLoc(), SMLoc()));
@@ -721,16 +720,16 @@ void X86AddressSanitizer32::InstrumentMemOperandLarge(
       Inst.setOpcode(X86::CMP16mi);
       break;
     }
-    const MCExpr *Disp = MCConstantExpr::Create(kShadowOffset, Ctx);
+    const MCExpr *Disp = MCConstantExpr::create(kShadowOffset, Ctx);
     std::unique_ptr<X86Operand> Op(
         X86Operand::CreateMem(getPointerWidth(), 0, Disp, ShadowRegI32, 0, 1,
                               SMLoc(), SMLoc()));
     Op->addMemOperands(Inst, 5);
-    Inst.addOperand(MCOperand::CreateImm(0));
+    Inst.addOperand(MCOperand::createImm(0));
     EmitInstruction(Out, Inst);
   }
-  MCSymbol *DoneSym = Ctx.CreateTempSymbol();
-  const MCExpr *DoneExpr = MCSymbolRefExpr::Create(DoneSym, Ctx);
+  MCSymbol *DoneSym = Ctx.createTempSymbol();
+  const MCExpr *DoneExpr = MCSymbolRefExpr::create(DoneSym, Ctx);
   EmitInstruction(Out, MCInstBuilder(X86::JE_1).addExpr(DoneExpr));
 
   EmitCallAsanReport(AccessSize, IsWrite, Ctx, Out, RegCtx);
@@ -743,8 +742,8 @@ void X86AddressSanitizer32::InstrumentMOVSImpl(unsigned AccessSize,
   StoreFlags(Out);
 
   // No need to test when ECX is equals to zero.
-  MCSymbol *DoneSym = Ctx.CreateTempSymbol();
-  const MCExpr *DoneExpr = MCSymbolRefExpr::Create(DoneSym, Ctx);
+  MCSymbol *DoneSym = Ctx.createTempSymbol();
+  const MCExpr *DoneExpr = MCSymbolRefExpr::create(DoneSym, Ctx);
   EmitInstruction(
       Out, MCInstBuilder(X86::TEST32rr).addReg(X86::ECX).addReg(X86::ECX));
   EmitInstruction(Out, MCInstBuilder(X86::JE_1).addExpr(DoneExpr));
@@ -861,7 +860,7 @@ class X86AddressSanitizer64 : public X86AddressSanitizer {
 
 private:
   void EmitAdjustRSP(MCContext &Ctx, MCStreamer &Out, long Offset) {
-    const MCExpr *Disp = MCConstantExpr::Create(Offset, Ctx);
+    const MCExpr *Disp = MCConstantExpr::create(Offset, Ctx);
     std::unique_ptr<X86Operand> Op(
         X86Operand::CreateMem(getPointerWidth(), 0, Disp, X86::RSP, 0, 1,
                               SMLoc(), SMLoc()));
@@ -884,9 +883,9 @@ class X86AddressSanitizer64 : public X86AddressSanitizer {
                                RegCtx.AddressReg(MVT::i64)));
     }
     const std::string &Fn = FuncName(AccessSize, IsWrite);
-    MCSymbol *FnSym = Ctx.GetOrCreateSymbol(StringRef(Fn));
+    MCSymbol *FnSym = Ctx.getOrCreateSymbol(StringRef(Fn));
     const MCSymbolRefExpr *FnExpr =
-        MCSymbolRefExpr::Create(FnSym, MCSymbolRefExpr::VK_PLT, Ctx);
+        MCSymbolRefExpr::create(FnSym, MCSymbolRefExpr::VK_PLT, Ctx);
     EmitInstruction(Out, MCInstBuilder(X86::CALL64pcrel32).addExpr(FnExpr));
   }
 };
@@ -914,8 +913,8 @@ void X86AddressSanitizer64::InstrumentMemOperandSmall(
   {
     MCInst Inst;
     Inst.setOpcode(X86::MOV8rm);
-    Inst.addOperand(MCOperand::CreateReg(ShadowRegI8));
-    const MCExpr *Disp = MCConstantExpr::Create(kShadowOffset, Ctx);
+    Inst.addOperand(MCOperand::createReg(ShadowRegI8));
+    const MCExpr *Disp = MCConstantExpr::create(kShadowOffset, Ctx);
     std::unique_ptr<X86Operand> Op(
         X86Operand::CreateMem(getPointerWidth(), 0, Disp, ShadowRegI64, 0, 1,
                               SMLoc(), SMLoc()));
@@ -925,8 +924,8 @@ void X86AddressSanitizer64::InstrumentMemOperandSmall(
 
   EmitInstruction(
       Out, MCInstBuilder(X86::TEST8rr).addReg(ShadowRegI8).addReg(ShadowRegI8));
-  MCSymbol *DoneSym = Ctx.CreateTempSymbol();
-  const MCExpr *DoneExpr = MCSymbolRefExpr::Create(DoneSym, Ctx);
+  MCSymbol *DoneSym = Ctx.createTempSymbol();
+  const MCExpr *DoneExpr = MCSymbolRefExpr::create(DoneSym, Ctx);
   EmitInstruction(Out, MCInstBuilder(X86::JE_1).addExpr(DoneExpr));
 
   EmitInstruction(Out, MCInstBuilder(X86::MOV32rr).addReg(ScratchRegI32).addReg(
@@ -941,7 +940,7 @@ void X86AddressSanitizer64::InstrumentMemOperandSmall(
   case 1:
     break;
   case 2: {
-    const MCExpr *Disp = MCConstantExpr::Create(1, Ctx);
+    const MCExpr *Disp = MCConstantExpr::create(1, Ctx);
     std::unique_ptr<X86Operand> Op(
         X86Operand::CreateMem(getPointerWidth(), 0, Disp, ScratchRegI32, 0, 1,
                               SMLoc(), SMLoc()));
@@ -992,17 +991,17 @@ void X86AddressSanitizer64::InstrumentMemOperandLarge(
       Inst.setOpcode(X86::CMP16mi);
       break;
     }
-    const MCExpr *Disp = MCConstantExpr::Create(kShadowOffset, Ctx);
+    const MCExpr *Disp = MCConstantExpr::create(kShadowOffset, Ctx);
     std::unique_ptr<X86Operand> Op(
         X86Operand::CreateMem(getPointerWidth(), 0, Disp, ShadowRegI64, 0, 1,
                               SMLoc(), SMLoc()));
     Op->addMemOperands(Inst, 5);
-    Inst.addOperand(MCOperand::CreateImm(0));
+    Inst.addOperand(MCOperand::createImm(0));
     EmitInstruction(Out, Inst);
   }
 
-  MCSymbol *DoneSym = Ctx.CreateTempSymbol();
-  const MCExpr *DoneExpr = MCSymbolRefExpr::Create(DoneSym, Ctx);
+  MCSymbol *DoneSym = Ctx.createTempSymbol();
+  const MCExpr *DoneExpr = MCSymbolRefExpr::create(DoneSym, Ctx);
   EmitInstruction(Out, MCInstBuilder(X86::JE_1).addExpr(DoneExpr));
 
   EmitCallAsanReport(AccessSize, IsWrite, Ctx, Out, RegCtx);
@@ -1015,8 +1014,8 @@ void X86AddressSanitizer64::InstrumentMOVSImpl(unsigned AccessSize,
   StoreFlags(Out);
 
   // No need to test when RCX is equals to zero.
-  MCSymbol *DoneSym = Ctx.CreateTempSymbol();
-  const MCExpr *DoneExpr = MCSymbolRefExpr::Create(DoneSym, Ctx);
+  MCSymbol *DoneSym = Ctx.createTempSymbol();
+  const MCExpr *DoneExpr = MCSymbolRefExpr::create(DoneSym, Ctx);
   EmitInstruction(
       Out, MCInstBuilder(X86::TEST64rr).addReg(X86::RCX).addReg(X86::RCX));
   EmitInstruction(Out, MCInstBuilder(X86::JE_1).addExpr(DoneExpr));
@@ -1073,9 +1072,9 @@ CreateX86AsmInstrumentation(const MCTargetOptions &MCOptions,
   const bool hasCompilerRTSupport = T.isOSLinux();
   if (ClAsanInstrumentAssembly && hasCompilerRTSupport &&
       MCOptions.SanitizeAddress) {
-    if ((STI.getFeatureBits() & X86::Mode32Bit) != 0)
+    if (STI.getFeatureBits()[X86::Mode32Bit] != 0)
       return new X86AddressSanitizer32(STI);
-    if ((STI.getFeatureBits() & X86::Mode64Bit) != 0)
+    if (STI.getFeatureBits()[X86::Mode64Bit] != 0)
       return new X86AddressSanitizer64(STI);
   }
   return new X86AsmInstrumentation(STI);
diff --git a/lib/Target/X86/AsmParser/X86AsmParser.cpp b/lib/Target/X86/AsmParser/X86AsmParser.cpp
index f4d5ed5db39..637629118ac 100644
--- a/lib/Target/X86/AsmParser/X86AsmParser.cpp
+++ b/lib/Target/X86/AsmParser/X86AsmParser.cpp
@@ -42,15 +42,16 @@ namespace {
 
 static const char OpPrecedence[] = {
   0, // IC_OR
-  1, // IC_AND
-  2, // IC_LSHIFT
-  2, // IC_RSHIFT
-  3, // IC_PLUS
-  3, // IC_MINUS
-  4, // IC_MULTIPLY
-  4, // IC_DIVIDE
-  5, // IC_RPAREN
-  6, // IC_LPAREN
+  1, // IC_XOR
+  2, // IC_AND
+  3, // IC_LSHIFT
+  3, // IC_RSHIFT
+  4, // IC_PLUS
+  4, // IC_MINUS
+  5, // IC_MULTIPLY
+  5, // IC_DIVIDE
+  6, // IC_RPAREN
+  7, // IC_LPAREN
   0, // IC_IMM
   0  // IC_REGISTER
 };
@@ -70,6 +71,7 @@ class X86AsmParser : public MCTargetAsmParser {
 
   enum InfixCalculatorTok {
     IC_OR = 0,
+    IC_XOR,
     IC_AND,
     IC_LSHIFT,
     IC_RSHIFT,
@@ -204,6 +206,12 @@ class X86AsmParser : public MCTargetAsmParser {
             Val = Op1.second | Op2.second;
             OperandStack.push_back(std::make_pair(IC_IMM, Val));
             break;
+          case IC_XOR:
+            assert(Op1.first == IC_IMM && Op2.first == IC_IMM &&
+              "Xor operation with an immediate and a register!");
+            Val = Op1.second ^ Op2.second;
+            OperandStack.push_back(std::make_pair(IC_IMM, Val));
+            break;
           case IC_AND:
             assert (Op1.first == IC_IMM && Op2.first == IC_IMM &&
                     "And operation with an immediate and a register!");
@@ -232,6 +240,7 @@ class X86AsmParser : public MCTargetAsmParser {
 
   enum IntelExprState {
     IES_OR,
+    IES_XOR,
     IES_AND,
     IES_LSHIFT,
     IES_RSHIFT,
@@ -297,6 +306,21 @@ class X86AsmParser : public MCTargetAsmParser {
       }
       PrevState = CurrState;
     }
+    void onXor() {
+      IntelExprState CurrState = State;
+      switch (State) {
+      default:
+        State = IES_ERROR;
+        break;
+      case IES_INTEGER:
+      case IES_RPAREN:
+      case IES_REGISTER:
+        State = IES_XOR;
+        IC.pushOperator(IC_XOR);
+        break;
+      }
+      PrevState = CurrState;
+    }
     void onAnd() {
       IntelExprState CurrState = State;
       switch (State) {
@@ -473,6 +497,7 @@ class X86AsmParser : public MCTargetAsmParser {
       case IES_MINUS:
       case IES_NOT:
       case IES_OR:
+      case IES_XOR:
       case IES_AND:
       case IES_LSHIFT:
       case IES_RSHIFT:
@@ -496,7 +521,7 @@ class X86AsmParser : public MCTargetAsmParser {
                     PrevState == IES_LSHIFT || PrevState == IES_RSHIFT ||
                     PrevState == IES_MULTIPLY || PrevState == IES_DIVIDE ||
                     PrevState == IES_LPAREN || PrevState == IES_LBRAC ||
-                    PrevState == IES_NOT) &&
+                    PrevState == IES_NOT || PrevState == IES_XOR) &&
                    CurrState == IES_MINUS) {
           // Unary minus.  No need to pop the minus operand because it was never
           // pushed.
@@ -506,7 +531,7 @@ class X86AsmParser : public MCTargetAsmParser {
                     PrevState == IES_LSHIFT || PrevState == IES_RSHIFT ||
                     PrevState == IES_MULTIPLY || PrevState == IES_DIVIDE ||
                     PrevState == IES_LPAREN || PrevState == IES_LBRAC ||
-                    PrevState == IES_NOT) &&
+                    PrevState == IES_NOT || PrevState == IES_XOR) &&
                    CurrState == IES_NOT) {
           // Unary not.  No need to pop the not operand because it was never
           // pushed.
@@ -593,6 +618,7 @@ class X86AsmParser : public MCTargetAsmParser {
       case IES_MINUS:
       case IES_NOT:
       case IES_OR:
+      case IES_XOR:
       case IES_AND:
       case IES_LSHIFT:
       case IES_RSHIFT:
@@ -605,7 +631,7 @@ class X86AsmParser : public MCTargetAsmParser {
             PrevState == IES_LSHIFT || PrevState == IES_RSHIFT ||
             PrevState == IES_MULTIPLY || PrevState == IES_DIVIDE ||
             PrevState == IES_LPAREN || PrevState == IES_LBRAC ||
-            PrevState == IES_NOT) &&
+            PrevState == IES_NOT || PrevState == IES_XOR) &&
             (CurrState == IES_MINUS || CurrState == IES_NOT)) {
           State = IES_ERROR;
           break;
@@ -655,6 +681,9 @@ class X86AsmParser : public MCTargetAsmParser {
 
   std::unique_ptr<X86Operand> DefaultMemSIOperand(SMLoc Loc);
   std::unique_ptr<X86Operand> DefaultMemDIOperand(SMLoc Loc);
+  void AddDefaultSrcDestOperands(
+      OperandVector& Operands, std::unique_ptr<llvm::MCParsedAsmOperand> &&Src,
+      std::unique_ptr<llvm::MCParsedAsmOperand> &&Dst);
   std::unique_ptr<X86Operand> ParseOperand();
   std::unique_ptr<X86Operand> ParseATTOperand();
   std::unique_ptr<X86Operand> ParseIntelOperand();
@@ -729,23 +758,24 @@ class X86AsmParser : public MCTargetAsmParser {
 
   bool is64BitMode() const {
     // FIXME: Can tablegen auto-generate this?
-    return (STI.getFeatureBits() & X86::Mode64Bit) != 0;
+    return STI.getFeatureBits()[X86::Mode64Bit];
   }
   bool is32BitMode() const {
     // FIXME: Can tablegen auto-generate this?
-    return (STI.getFeatureBits() & X86::Mode32Bit) != 0;
+    return STI.getFeatureBits()[X86::Mode32Bit];
   }
   bool is16BitMode() const {
     // FIXME: Can tablegen auto-generate this?
-    return (STI.getFeatureBits() & X86::Mode16Bit) != 0;
+    return STI.getFeatureBits()[X86::Mode16Bit];
   }
-  void SwitchMode(uint64_t mode) {
-    uint64_t oldMode = STI.getFeatureBits() &
-        (X86::Mode64Bit | X86::Mode32Bit | X86::Mode16Bit);
-    unsigned FB = ComputeAvailableFeatures(STI.ToggleFeature(oldMode | mode));
+  void SwitchMode(unsigned mode) {
+    FeatureBitset AllModes({X86::Mode64Bit, X86::Mode32Bit, X86::Mode16Bit});
+    FeatureBitset OldMode = STI.getFeatureBits() & AllModes;
+    unsigned FB = ComputeAvailableFeatures(
+      STI.ToggleFeature(OldMode.flip(mode)));
     setAvailableFeatures(FB);
-    assert(mode == (STI.getFeatureBits() &
-                    (X86::Mode64Bit | X86::Mode32Bit | X86::Mode16Bit)));
+    
+    assert(FeatureBitset({mode}) == (STI.getFeatureBits() & AllModes));
   }
 
   unsigned getPointerWidth() {
@@ -770,7 +800,7 @@ class X86AsmParser : public MCTargetAsmParser {
 public:
   X86AsmParser(MCSubtargetInfo &sti, MCAsmParser &Parser,
                const MCInstrInfo &mii, const MCTargetOptions &Options)
-      : MCTargetAsmParser(), STI(sti), MII(mii), InstInfo(nullptr) {
+      : MCTargetAsmParser(Options), STI(sti), MII(mii), InstInfo(nullptr) {
 
     // Initialize the set of available features.
     setAvailableFeatures(ComputeAvailableFeatures(STI.getFeatureBits()));
@@ -882,6 +912,11 @@ bool X86AsmParser::ParseRegister(unsigned &RegNo,
   if (RegNo == 0)
     RegNo = MatchRegisterName(Tok.getString().lower());
 
+  // The "flags" register cannot be referenced directly.
+  // Treat it as an identifier instead.
+  if (isParsingInlineAsm() && isParsingIntelSyntax() && RegNo == X86::EFLAGS)
+    RegNo = 0;
+
   if (!is64BitMode()) {
     // FIXME: This should be done using Requires<Not64BitMode> and
     // Requires<In64BitMode> so "eiz" usage in 64-bit instructions can be also
@@ -972,7 +1007,7 @@ void X86AsmParser::SetFrameRegister(unsigned RegNo) {
 std::unique_ptr<X86Operand> X86AsmParser::DefaultMemSIOperand(SMLoc Loc) {
   unsigned basereg =
     is64BitMode() ? X86::RSI : (is32BitMode() ? X86::ESI : X86::SI);
-  const MCExpr *Disp = MCConstantExpr::Create(0, getContext());
+  const MCExpr *Disp = MCConstantExpr::create(0, getContext());
   return X86Operand::CreateMem(getPointerWidth(), /*SegReg=*/0, Disp,
                                /*BaseReg=*/basereg, /*IndexReg=*/0, /*Scale=*/1,
                                Loc, Loc, 0);
@@ -981,12 +1016,25 @@ std::unique_ptr<X86Operand> X86AsmParser::DefaultMemSIOperand(SMLoc Loc) {
 std::unique_ptr<X86Operand> X86AsmParser::DefaultMemDIOperand(SMLoc Loc) {
   unsigned basereg =
     is64BitMode() ? X86::RDI : (is32BitMode() ? X86::EDI : X86::DI);
-  const MCExpr *Disp = MCConstantExpr::Create(0, getContext());
+  const MCExpr *Disp = MCConstantExpr::create(0, getContext());
   return X86Operand::CreateMem(getPointerWidth(), /*SegReg=*/0, Disp,
                                /*BaseReg=*/basereg, /*IndexReg=*/0, /*Scale=*/1,
                                Loc, Loc, 0);
 }
 
+void X86AsmParser::AddDefaultSrcDestOperands(
+    OperandVector& Operands, std::unique_ptr<llvm::MCParsedAsmOperand> &&Src,
+    std::unique_ptr<llvm::MCParsedAsmOperand> &&Dst) {
+  if (isParsingIntelSyntax()) {
+    Operands.push_back(std::move(Dst));
+    Operands.push_back(std::move(Src));
+  }
+  else {
+    Operands.push_back(std::move(Src));
+    Operands.push_back(std::move(Dst));
+  }
+}
+
 std::unique_ptr<X86Operand> X86AsmParser::ParseOperand() {
   if (isParsingIntelSyntax())
     return ParseIntelOperand();
@@ -1001,6 +1049,7 @@ static unsigned getIntelMemOperandSize(StringRef OpStr) {
     .Cases("DWORD", "dword", 32)
     .Cases("QWORD", "qword", 64)
     .Cases("XWORD", "xword", 80)
+    .Cases("TBYTE", "tbyte", 80)
     .Cases("XMMWORD", "xmmword", 128)
     .Cases("YMMWORD", "ymmword", 256)
     .Cases("ZMMWORD", "zmmword", 512)
@@ -1191,10 +1240,10 @@ bool X86AsmParser::ParseIntelExpression(IntelExprStateMachine &SM, SMLoc &End) {
         StringRef IDVal = getTok().getString();
         if (IDVal == "f" || IDVal == "b") {
           MCSymbol *Sym =
-              getContext().GetDirectionalLocalSymbol(IntVal, IDVal == "b");
+              getContext().getDirectionalLocalSymbol(IntVal, IDVal == "b");
           MCSymbolRefExpr::VariantKind Variant = MCSymbolRefExpr::VK_None;
           const MCExpr *Val =
-	    MCSymbolRefExpr::Create(Sym, Variant, getContext());
+	    MCSymbolRefExpr::create(Sym, Variant, getContext());
           if (IDVal == "b" && Sym->isUndefined())
             return Error(Loc, "invalid reference to undefined symbol");
           StringRef Identifier = Sym->getName();
@@ -1216,6 +1265,7 @@ bool X86AsmParser::ParseIntelExpression(IntelExprStateMachine &SM, SMLoc &End) {
     case AsmToken::Star:    SM.onStar(); break;
     case AsmToken::Slash:   SM.onDivide(); break;
     case AsmToken::Pipe:    SM.onOr(); break;
+    case AsmToken::Caret:   SM.onXor(); break;
     case AsmToken::Amp:     SM.onAnd(); break;
     case AsmToken::LessLess:
                             SM.onLShift(); break;
@@ -1264,9 +1314,9 @@ X86AsmParser::ParseIntelBracExpression(unsigned SegReg, SMLoc Start,
   }
 
   if (SM.getImm() || !Disp) {
-    const MCExpr *Imm = MCConstantExpr::Create(SM.getImm(), getContext());
+    const MCExpr *Imm = MCConstantExpr::create(SM.getImm(), getContext());
     if (Disp)
-      Disp = MCBinaryExpr::CreateAdd(Disp, Imm, getContext());
+      Disp = MCBinaryExpr::createAdd(Disp, Imm, getContext());
     else
       Disp = Imm;  // An immediate displacement only.
   }
@@ -1351,9 +1401,9 @@ bool X86AsmParser::ParseIntelIdentifier(const MCExpr *&Val,
   }
 
   // Create the symbol reference.
-  MCSymbol *Sym = getContext().GetOrCreateSymbol(Identifier);
+  MCSymbol *Sym = getContext().getOrCreateSymbol(Identifier);
   MCSymbolRefExpr::VariantKind Variant = MCSymbolRefExpr::VK_None;
-  Val = MCSymbolRefExpr::Create(Sym, Variant, getParser().getContext());
+  Val = MCSymbolRefExpr::create(Sym, Variant, getParser().getContext());
   return false;
 }
 
@@ -1381,7 +1431,7 @@ X86AsmParser::ParseIntelSegmentOverride(unsigned SegReg, SMLoc Start,
       // An immediate following a 'segment register', 'colon' token sequence can
       // be followed by a bracketed expression.  If it isn't we know we have our
       // final segment override.
-      const MCExpr *Disp = MCConstantExpr::Create(ImmDisp, getContext());
+      const MCExpr *Disp = MCConstantExpr::create(ImmDisp, getContext());
       return X86Operand::CreateMem(getPointerWidth(), SegReg, Disp,
                                    /*BaseReg=*/0, /*IndexReg=*/0, /*Scale=*/1,
                                    Start, ImmDispToken.getEndLoc(), Size);
@@ -1414,7 +1464,8 @@ std::unique_ptr<X86Operand>
 X86AsmParser::ParseRoundingModeOp(SMLoc Start, SMLoc End) {
   MCAsmParser &Parser = getParser();
   const AsmToken &Tok = Parser.getTok();
-  consumeToken(); // Eat "{"
+  // Eat "{" and mark the current place.
+  const SMLoc consumedToken = consumeToken();
   if (Tok.getIdentifier().startswith("r")){
     int rndMode = StringSwitch<int>(Tok.getIdentifier())
       .Case("rn", X86::STATIC_ROUNDING::TO_NEAREST_INT)
@@ -1433,9 +1484,16 @@ X86AsmParser::ParseRoundingModeOp(SMLoc Start, SMLoc End) {
       return ErrorOperand(Tok.getLoc(), "Expected } at this point");
     Parser.Lex();  // Eat "}"
     const MCExpr *RndModeOp =
-      MCConstantExpr::Create(rndMode, Parser.getContext());
+      MCConstantExpr::create(rndMode, Parser.getContext());
     return X86Operand::CreateImm(RndModeOp, Start, End);
   }
+  if(Tok.getIdentifier().equals("sae")){
+    Parser.Lex();  // Eat the sae
+    if (!getLexer().is(AsmToken::RCurly))
+      return ErrorOperand(Tok.getLoc(), "Expected } at this point");
+    Parser.Lex();  // Eat "}"
+    return X86Operand::CreateToken("{sae}", consumedToken);
+  }
   return ErrorOperand(Tok.getLoc(), "unknown token in expression");
 }
 /// ParseIntelMemOperand - Parse intel style memory operand.
@@ -1490,7 +1548,7 @@ std::unique_ptr<X86Operand> X86AsmParser::ParseIntelMemOperand(int64_t ImmDisp,
     return nullptr;
   }
 
-  const MCExpr *Disp = MCConstantExpr::Create(SM.getImm(), getContext());
+  const MCExpr *Disp = MCConstantExpr::create(SM.getImm(), getContext());
   // BaseReg is non-zero to avoid assertions.  In the context of inline asm,
   // we're pointing to a local variable in memory, so the base register is
   // really the frame or stack pointer.
@@ -1540,7 +1598,7 @@ bool X86AsmParser::ParseIntelDotOperator(const MCExpr *Disp,
                                                 Val));
   }
 
-  NewDisp = MCConstantExpr::Create(OrigDispVal + DotDispVal, getContext());
+  NewDisp = MCConstantExpr::create(OrigDispVal + DotDispVal, getContext());
   return false;
 }
 
@@ -1614,7 +1672,7 @@ std::unique_ptr<X86Operand> X86AsmParser::ParseIntelOperator(unsigned OpKind) {
   unsigned Len = End.getPointer() - TypeLoc.getPointer();
   InstInfo->AsmRewrites->push_back(AsmRewrite(AOK_Imm, TypeLoc, Len, CVal));
 
-  const MCExpr *Imm = MCConstantExpr::Create(CVal, getContext());
+  const MCExpr *Imm = MCConstantExpr::create(CVal, getContext());
   return X86Operand::CreateImm(Imm, Start, End);
 }
 
@@ -1674,7 +1732,7 @@ std::unique_ptr<X86Operand> X86AsmParser::ParseIntelOperand() {
         return X86Operand::CreateMem(getPointerWidth(), SM.getSym(), Start, End,
                                      Size);
 
-      const MCExpr *ImmExpr = MCConstantExpr::Create(Imm, getContext());
+      const MCExpr *ImmExpr = MCConstantExpr::create(Imm, getContext());
       return X86Operand::CreateImm(ImmExpr, Start, End);
     }
 
@@ -1688,7 +1746,7 @@ std::unique_ptr<X86Operand> X86AsmParser::ParseIntelOperand() {
   }
 
   // rounding mode token
-  if (STI.getFeatureBits() & X86::FeatureAVX512 &&
+  if (STI.getFeatureBits()[X86::FeatureAVX512] &&
       getLexer().is(AsmToken::LCurly))
     return ParseRoundingModeOp(Start, End);
 
@@ -1746,7 +1804,7 @@ std::unique_ptr<X86Operand> X86AsmParser::ParseATTOperand() {
   }
   case AsmToken::LCurly:{
     SMLoc Start = Parser.getTok().getLoc(), End;
-    if (STI.getFeatureBits() & X86::FeatureAVX512)
+    if (STI.getFeatureBits()[X86::FeatureAVX512])
       return ParseRoundingModeOp(Start, End);
     return ErrorOperand(Start, "unknown token in expression");
   }
@@ -1756,7 +1814,7 @@ std::unique_ptr<X86Operand> X86AsmParser::ParseATTOperand() {
 bool X86AsmParser::HandleAVX512Operand(OperandVector &Operands,
                                        const MCParsedAsmOperand &Op) {
   MCAsmParser &Parser = getParser();
-  if(STI.getFeatureBits() & X86::FeatureAVX512) {
+  if(STI.getFeatureBits()[X86::FeatureAVX512]) {
     if (getLexer().is(AsmToken::LCurly)) {
       // Eat "{" and mark the current place.
       const SMLoc consumedToken = consumeToken();
@@ -1832,7 +1890,7 @@ std::unique_ptr<X86Operand> X86AsmParser::ParseMemOperand(unsigned SegReg,
   // of a memory operand with a missing displacement "(%ebx)" or "(,%eax)".  The
   // only way to do this without lookahead is to eat the '(' and see what is
   // after it.
-  const MCExpr *Disp = MCConstantExpr::Create(0, getParser().getContext());
+  const MCExpr *Disp = MCConstantExpr::create(0, getParser().getContext());
   if (getLexer().isNot(AsmToken::LParen)) {
     SMLoc ExprEnd;
     if (getParser().parseExpression(Disp, ExprEnd)) return nullptr;
@@ -2052,7 +2110,7 @@ bool X86AsmParser::ParseInstruction(ParseInstructionInfo &Info, StringRef Name,
       Operands.push_back(X86Operand::CreateToken(PatchedName.slice(0, CCIdx),
                                                  NameLoc));
 
-      const MCExpr *ImmOp = MCConstantExpr::Create(ComparisonCode,
+      const MCExpr *ImmOp = MCConstantExpr::create(ComparisonCode,
                                                    getParser().getContext());
       Operands.push_back(X86Operand::CreateImm(ImmOp, NameLoc, NameLoc));
 
@@ -2079,7 +2137,7 @@ bool X86AsmParser::ParseInstruction(ParseInstructionInfo &Info, StringRef Name,
     if (ComparisonCode != ~0U && (ComparisonCode != 0 || CCIdx == 2)) {
       Operands.push_back(X86Operand::CreateToken("vpcmp", NameLoc));
 
-      const MCExpr *ImmOp = MCConstantExpr::Create(ComparisonCode,
+      const MCExpr *ImmOp = MCConstantExpr::create(ComparisonCode,
                                                    getParser().getContext());
       Operands.push_back(X86Operand::CreateImm(ImmOp, NameLoc, NameLoc));
 
@@ -2106,7 +2164,7 @@ bool X86AsmParser::ParseInstruction(ParseInstructionInfo &Info, StringRef Name,
     if (ComparisonCode != ~0U) {
       Operands.push_back(X86Operand::CreateToken("vpcom", NameLoc));
 
-      const MCExpr *ImmOp = MCConstantExpr::Create(ComparisonCode,
+      const MCExpr *ImmOp = MCConstantExpr::create(ComparisonCode,
                                                    getParser().getContext());
       Operands.push_back(X86Operand::CreateImm(ImmOp, NameLoc, NameLoc));
 
@@ -2192,26 +2250,18 @@ bool X86AsmParser::ParseInstruction(ParseInstructionInfo &Info, StringRef Name,
   if (Name.startswith("ins") && Operands.size() == 1 &&
       (Name == "insb" || Name == "insw" || Name == "insl" ||
        Name == "insd" )) {
-    if (isParsingIntelSyntax()) {
-      Operands.push_back(X86Operand::CreateReg(X86::DX, NameLoc, NameLoc));
-      Operands.push_back(DefaultMemDIOperand(NameLoc));
-    } else {
-      Operands.push_back(X86Operand::CreateReg(X86::DX, NameLoc, NameLoc));
-      Operands.push_back(DefaultMemDIOperand(NameLoc));
-    }
+    AddDefaultSrcDestOperands(Operands, 
+                              X86Operand::CreateReg(X86::DX, NameLoc, NameLoc),
+                              DefaultMemDIOperand(NameLoc));
   }
 
   // Append default arguments to "outs[bwld]"
   if (Name.startswith("outs") && Operands.size() == 1 &&
       (Name == "outsb" || Name == "outsw" || Name == "outsl" ||
        Name == "outsd" )) {
-    if (isParsingIntelSyntax()) {
-      Operands.push_back(DefaultMemSIOperand(NameLoc));
-      Operands.push_back(X86Operand::CreateReg(X86::DX, NameLoc, NameLoc));
-    } else {
-      Operands.push_back(DefaultMemSIOperand(NameLoc));
-      Operands.push_back(X86Operand::CreateReg(X86::DX, NameLoc, NameLoc));
-    }
+    AddDefaultSrcDestOperands(Operands,
+                              DefaultMemSIOperand(NameLoc),
+                              X86Operand::CreateReg(X86::DX, NameLoc, NameLoc));
   }
 
   // Transform "lods[bwlq]" into "lods[bwlq] ($SIREG)" for appropriate
@@ -2243,13 +2293,9 @@ bool X86AsmParser::ParseInstruction(ParseInstructionInfo &Info, StringRef Name,
       (Name == "cmps" || Name == "cmpsb" || Name == "cmpsw" ||
        Name == "cmpsl" || Name == "cmpsd" || Name == "cmpsq")) {
     if (Operands.size() == 1) {
-      if (isParsingIntelSyntax()) {
-        Operands.push_back(DefaultMemSIOperand(NameLoc));
-        Operands.push_back(DefaultMemDIOperand(NameLoc));
-      } else {
-        Operands.push_back(DefaultMemDIOperand(NameLoc));
-        Operands.push_back(DefaultMemSIOperand(NameLoc));
-      }
+      AddDefaultSrcDestOperands(Operands,
+                                DefaultMemDIOperand(NameLoc),
+                                DefaultMemSIOperand(NameLoc));
     } else if (Operands.size() == 3) {
       X86Operand &Op = (X86Operand &)*Operands[1];
       X86Operand &Op2 = (X86Operand &)*Operands[2];
@@ -2269,13 +2315,9 @@ bool X86AsmParser::ParseInstruction(ParseInstructionInfo &Info, StringRef Name,
     if (Operands.size() == 1) {
       if (Name == "movsd")
         Operands.back() = X86Operand::CreateToken("movsl", NameLoc);
-      if (isParsingIntelSyntax()) {
-        Operands.push_back(DefaultMemDIOperand(NameLoc));
-        Operands.push_back(DefaultMemSIOperand(NameLoc));
-      } else {
-        Operands.push_back(DefaultMemSIOperand(NameLoc));
-        Operands.push_back(DefaultMemDIOperand(NameLoc));
-      }
+      AddDefaultSrcDestOperands(Operands,
+                                DefaultMemSIOperand(NameLoc),
+                                DefaultMemDIOperand(NameLoc));
     } else if (Operands.size() == 3) {
       X86Operand &Op = (X86Operand &)*Operands[1];
       X86Operand &Op2 = (X86Operand &)*Operands[2];
@@ -2310,11 +2352,12 @@ bool X86AsmParser::ParseInstruction(ParseInstructionInfo &Info, StringRef Name,
   // instalias with an immediate operand yet.
   if (Name == "int" && Operands.size() == 2) {
     X86Operand &Op1 = static_cast<X86Operand &>(*Operands[1]);
-    if (Op1.isImm() && isa<MCConstantExpr>(Op1.getImm()) &&
-        cast<MCConstantExpr>(Op1.getImm())->getValue() == 3) {
-      Operands.erase(Operands.begin() + 1);
-      static_cast<X86Operand &>(*Operands[0]).setTokenValue("int3");
-    }
+    if (Op1.isImm())
+      if (auto *CE = dyn_cast<MCConstantExpr>(Op1.getImm()))
+        if (CE->getValue() == 3) {
+          Operands.erase(Operands.begin() + 1);
+          static_cast<X86Operand &>(*Operands[0]).setTokenValue("int3");
+        }
   }
 
   return false;
@@ -2325,8 +2368,8 @@ static bool convertToSExti8(MCInst &Inst, unsigned Opcode, unsigned Reg,
   MCInst TmpInst;
   TmpInst.setOpcode(Opcode);
   if (!isCmp)
-    TmpInst.addOperand(MCOperand::CreateReg(Reg));
-  TmpInst.addOperand(MCOperand::CreateReg(Reg));
+    TmpInst.addOperand(MCOperand::createReg(Reg));
+  TmpInst.addOperand(MCOperand::createReg(Reg));
   TmpInst.addOperand(Inst.getOperand(0));
   Inst = TmpInst;
   return true;
@@ -2366,7 +2409,7 @@ bool X86AsmParser::validateInstruction(MCInst &Inst, const OperandVector &Ops) {
     X86Operand &Op = static_cast<X86Operand &>(*Ops[1]);
     assert(Op.isImm() && "expected immediate");
     int64_t Res;
-    if (!Op.getImm()->EvaluateAsAbsolute(Res) || Res > 255) {
+    if (!Op.getImm()->evaluateAsAbsolute(Res) || Res > 255) {
       Error(Op.getStartLoc(), "interrupt vector must be in range [0-255]");
       return false;
     }
@@ -2862,18 +2905,7 @@ bool X86AsmParser::ParseDirective(AsmToken DirectiveID) {
                                            "a '%' prefix in .intel_syntax");
     }
     return false;
-  // @LOCALMOD-START
-  } else if (IDVal.startswith(".dwarf_addr_size")) {
-    // For compatibility with NaCl gas, which needs to override the size of
-    // DWARF address values on x86-64 (normally it assumes that the address size
-    // matches the ELF class, which is not currently true for x86-64 NaCl).
-    // Require the size to be 4, since that matches the module address size.
-    if (Parser.getTok().getString() != "4")
-      return Error(DirectiveID.getLoc(), ".dwarf_addr_size must be 4");
-    Parser.Lex();
-    return false;
   }
-  // @LOCALMOD-END
   return true;
 }
 
diff --git a/lib/Target/X86/AsmParser/X86Operand.h b/lib/Target/X86/AsmParser/X86Operand.h
index 94dbedb4c8a..7ec02408ffa 100644
--- a/lib/Target/X86/AsmParser/X86Operand.h
+++ b/lib/Target/X86/AsmParser/X86Operand.h
@@ -12,8 +12,11 @@
 
 #include "X86AsmParserCommon.h"
 #include "llvm/MC/MCExpr.h"
+#include "llvm/MC/MCInst.h"
+#include "llvm/MC/MCRegisterInfo.h"
 #include "llvm/MC/MCParser/MCParsedAsmOperand.h"
 #include "llvm/ADT/STLExtras.h"
+#include "MCTargetDesc/X86MCTargetDesc.h"
 
 namespace llvm {
 
@@ -235,18 +238,34 @@ struct X86Operand : public MCParsedAsmOperand {
     return Kind == Memory && (!Mem.Size || Mem.Size == 32) &&
       getMemIndexReg() >= X86::XMM0 && getMemIndexReg() <= X86::XMM15;
   }
+  bool isMemVX32X() const {
+    return Kind == Memory && (!Mem.Size || Mem.Size == 32) &&
+      getMemIndexReg() >= X86::XMM0 && getMemIndexReg() <= X86::XMM31;
+  }
   bool isMemVY32() const {
     return Kind == Memory && (!Mem.Size || Mem.Size == 32) &&
       getMemIndexReg() >= X86::YMM0 && getMemIndexReg() <= X86::YMM15;
   }
+  bool isMemVY32X() const {
+    return Kind == Memory && (!Mem.Size || Mem.Size == 32) &&
+      getMemIndexReg() >= X86::YMM0 && getMemIndexReg() <= X86::YMM31;
+  }
   bool isMemVX64() const {
     return Kind == Memory && (!Mem.Size || Mem.Size == 64) &&
       getMemIndexReg() >= X86::XMM0 && getMemIndexReg() <= X86::XMM15;
   }
+  bool isMemVX64X() const {
+    return Kind == Memory && (!Mem.Size || Mem.Size == 64) &&
+      getMemIndexReg() >= X86::XMM0 && getMemIndexReg() <= X86::XMM31;
+  }
   bool isMemVY64() const {
     return Kind == Memory && (!Mem.Size || Mem.Size == 64) &&
       getMemIndexReg() >= X86::YMM0 && getMemIndexReg() <= X86::YMM15;
   }
+  bool isMemVY64X() const {
+    return Kind == Memory && (!Mem.Size || Mem.Size == 64) &&
+      getMemIndexReg() >= X86::YMM0 && getMemIndexReg() <= X86::YMM31;
+  }
   bool isMemVZ32() const {
     return Kind == Memory && (!Mem.Size || Mem.Size == 32) &&
       getMemIndexReg() >= X86::ZMM0 && getMemIndexReg() <= X86::ZMM31;
@@ -357,14 +376,14 @@ struct X86Operand : public MCParsedAsmOperand {
   void addExpr(MCInst &Inst, const MCExpr *Expr) const {
     // Add as immediates when possible.
     if (const MCConstantExpr *CE = dyn_cast<MCConstantExpr>(Expr))
-      Inst.addOperand(MCOperand::CreateImm(CE->getValue()));
+      Inst.addOperand(MCOperand::createImm(CE->getValue()));
     else
-      Inst.addOperand(MCOperand::CreateExpr(Expr));
+      Inst.addOperand(MCOperand::createExpr(Expr));
   }
 
   void addRegOperands(MCInst &Inst, unsigned N) const {
     assert(N == 1 && "Invalid number of operands!");
-    Inst.addOperand(MCOperand::CreateReg(getReg()));
+    Inst.addOperand(MCOperand::createReg(getReg()));
   }
 
   static unsigned getGR32FromGR64(unsigned RegNo) {
@@ -395,7 +414,7 @@ struct X86Operand : public MCParsedAsmOperand {
     unsigned RegNo = getReg();
     if (X86MCRegisterClasses[X86::GR64RegClassID].contains(RegNo))
       RegNo = getGR32FromGR64(RegNo);
-    Inst.addOperand(MCOperand::CreateReg(RegNo));
+    Inst.addOperand(MCOperand::createReg(RegNo));
   }
   void addAVX512RCOperands(MCInst &Inst, unsigned N) const {
     assert(N == 1 && "Invalid number of operands!");
@@ -408,40 +427,40 @@ struct X86Operand : public MCParsedAsmOperand {
 
   void addMemOperands(MCInst &Inst, unsigned N) const {
     assert((N == 5) && "Invalid number of operands!");
-    Inst.addOperand(MCOperand::CreateReg(getMemBaseReg()));
-    Inst.addOperand(MCOperand::CreateImm(getMemScale()));
-    Inst.addOperand(MCOperand::CreateReg(getMemIndexReg()));
+    Inst.addOperand(MCOperand::createReg(getMemBaseReg()));
+    Inst.addOperand(MCOperand::createImm(getMemScale()));
+    Inst.addOperand(MCOperand::createReg(getMemIndexReg()));
     addExpr(Inst, getMemDisp());
-    Inst.addOperand(MCOperand::CreateReg(getMemSegReg()));
+    Inst.addOperand(MCOperand::createReg(getMemSegReg()));
   }
 
   void addAbsMemOperands(MCInst &Inst, unsigned N) const {
     assert((N == 1) && "Invalid number of operands!");
     // Add as immediates when possible.
     if (const MCConstantExpr *CE = dyn_cast<MCConstantExpr>(getMemDisp()))
-      Inst.addOperand(MCOperand::CreateImm(CE->getValue()));
+      Inst.addOperand(MCOperand::createImm(CE->getValue()));
     else
-      Inst.addOperand(MCOperand::CreateExpr(getMemDisp()));
+      Inst.addOperand(MCOperand::createExpr(getMemDisp()));
   }
 
   void addSrcIdxOperands(MCInst &Inst, unsigned N) const {
     assert((N == 2) && "Invalid number of operands!");
-    Inst.addOperand(MCOperand::CreateReg(getMemBaseReg()));
-    Inst.addOperand(MCOperand::CreateReg(getMemSegReg()));
+    Inst.addOperand(MCOperand::createReg(getMemBaseReg()));
+    Inst.addOperand(MCOperand::createReg(getMemSegReg()));
   }
   void addDstIdxOperands(MCInst &Inst, unsigned N) const {
     assert((N == 1) && "Invalid number of operands!");
-    Inst.addOperand(MCOperand::CreateReg(getMemBaseReg()));
+    Inst.addOperand(MCOperand::createReg(getMemBaseReg()));
   }
 
   void addMemOffsOperands(MCInst &Inst, unsigned N) const {
     assert((N == 2) && "Invalid number of operands!");
     // Add as immediates when possible.
     if (const MCConstantExpr *CE = dyn_cast<MCConstantExpr>(getMemDisp()))
-      Inst.addOperand(MCOperand::CreateImm(CE->getValue()));
+      Inst.addOperand(MCOperand::createImm(CE->getValue()));
     else
-      Inst.addOperand(MCOperand::CreateExpr(getMemDisp()));
-    Inst.addOperand(MCOperand::CreateReg(getMemSegReg()));
+      Inst.addOperand(MCOperand::createExpr(getMemDisp()));
+    Inst.addOperand(MCOperand::createReg(getMemSegReg()));
   }
 
   static std::unique_ptr<X86Operand> CreateToken(StringRef Str, SMLoc Loc) {
diff --git a/lib/Target/X86/CMakeLists.txt b/lib/Target/X86/CMakeLists.txt
index 6aa78511636..fba2c280d20 100644
--- a/lib/Target/X86/CMakeLists.txt
+++ b/lib/Target/X86/CMakeLists.txt
@@ -15,6 +15,7 @@ add_public_tablegen_target(X86CommonTableGen)
 set(sources
   X86AsmPrinter.cpp
   X86CallFrameOptimization.cpp
+  X86ExpandPseudo.cpp
   X86FastISel.cpp
   X86FloatingPoint.cpp
   X86FrameLowering.cpp
@@ -23,7 +24,6 @@ set(sources
   X86InstrInfo.cpp
   X86MCInstLower.cpp
   X86MachineFunctionInfo.cpp
-  X86NaClRewritePass.cpp
   X86PadShortFunction.cpp
   X86RegisterInfo.cpp
   X86SelectionDAGInfo.cpp
@@ -33,6 +33,7 @@ set(sources
   X86TargetTransformInfo.cpp
   X86VZeroUpper.cpp
   X86FixupLEAs.cpp
+  X86WinEHState.cpp
   )
 
 if( CMAKE_CL_64 )
diff --git a/lib/Target/X86/Disassembler/X86Disassembler.cpp b/lib/Target/X86/Disassembler/X86Disassembler.cpp
index e8c547517b0..cfc3ee2fb08 100644
--- a/lib/Target/X86/Disassembler/X86Disassembler.cpp
+++ b/lib/Target/X86/Disassembler/X86Disassembler.cpp
@@ -80,20 +80,19 @@ X86GenericDisassembler::X86GenericDisassembler(
                                          MCContext &Ctx,
                                          std::unique_ptr<const MCInstrInfo> MII)
   : MCDisassembler(STI, Ctx), MII(std::move(MII)) {
-  switch (STI.getFeatureBits() &
-          (X86::Mode16Bit | X86::Mode32Bit | X86::Mode64Bit)) {
-  case X86::Mode16Bit:
+  const FeatureBitset &FB = STI.getFeatureBits();
+  if (FB[X86::Mode16Bit]) {
     fMode = MODE_16BIT;
-    break;
-  case X86::Mode32Bit:
+    return;
+  } else if (FB[X86::Mode32Bit]) {
     fMode = MODE_32BIT;
-    break;
-  case X86::Mode64Bit:
+    return;
+  } else if (FB[X86::Mode64Bit]) {
     fMode = MODE_64BIT;
-    break;
-  default:
-    llvm_unreachable("Invalid CPU mode");
+    return;
   }
+
+  llvm_unreachable("Invalid CPU mode");
 }
 
 struct Region {
@@ -180,7 +179,7 @@ static void translateRegister(MCInst &mcInst, Reg reg) {
 #undef ENTRY
 
   uint8_t llvmRegnum = llvmRegnums[reg];
-  mcInst.addOperand(MCOperand::CreateReg(llvmRegnum));
+  mcInst.addOperand(MCOperand::createReg(llvmRegnum));
 }
 
 /// tryAddingSymbolicOperand - trys to add a symbolic operand in place of the
@@ -248,11 +247,11 @@ static bool translateSrcIndex(MCInst &mcInst, InternalInstruction &insn) {
     assert(insn.mode == MODE_16BIT);
     baseRegNo = insn.prefixPresent[0x67] ? X86::ESI : X86::SI;
   }
-  MCOperand baseReg = MCOperand::CreateReg(baseRegNo);
+  MCOperand baseReg = MCOperand::createReg(baseRegNo);
   mcInst.addOperand(baseReg);
 
   MCOperand segmentReg;
-  segmentReg = MCOperand::CreateReg(segmentRegnums[insn.segmentOverride]);
+  segmentReg = MCOperand::createReg(segmentRegnums[insn.segmentOverride]);
   mcInst.addOperand(segmentReg);
   return false;
 }
@@ -273,7 +272,7 @@ static bool translateDstIndex(MCInst &mcInst, InternalInstruction &insn) {
     assert(insn.mode == MODE_16BIT);
     baseRegNo = insn.prefixPresent[0x67] ? X86::EDI : X86::DI;
   }
-  MCOperand baseReg = MCOperand::CreateReg(baseRegNo);
+  MCOperand baseReg = MCOperand::createReg(baseRegNo);
   mcInst.addOperand(baseReg);
   return false;
 }
@@ -539,20 +538,28 @@ static void translateImmediate(MCInst &mcInst, uint64_t immediate,
   case TYPE_XMM32:
   case TYPE_XMM64:
   case TYPE_XMM128:
-    mcInst.addOperand(MCOperand::CreateReg(X86::XMM0 + (immediate >> 4)));
+    mcInst.addOperand(MCOperand::createReg(X86::XMM0 + (immediate >> 4)));
     return;
   case TYPE_XMM256:
-    mcInst.addOperand(MCOperand::CreateReg(X86::YMM0 + (immediate >> 4)));
+    mcInst.addOperand(MCOperand::createReg(X86::YMM0 + (immediate >> 4)));
     return;
   case TYPE_XMM512:
-    mcInst.addOperand(MCOperand::CreateReg(X86::ZMM0 + (immediate >> 4)));
+    mcInst.addOperand(MCOperand::createReg(X86::ZMM0 + (immediate >> 4)));
     return;
+  case TYPE_BNDR:
+    mcInst.addOperand(MCOperand::createReg(X86::BND0 + (immediate >> 4)));
   case TYPE_REL8:
     isBranch = true;
     pcrel = insn.startLocation + insn.immediateOffset + insn.immediateSize;
-    if(immediate & 0x80)
+    if (immediate & 0x80)
       immediate |= ~(0xffull);
     break;
+  case TYPE_REL16:
+    isBranch = true;
+    pcrel = insn.startLocation + insn.immediateOffset + insn.immediateSize;
+    if (immediate & 0x8000)
+      immediate |= ~(0xffffull);
+    break;
   case TYPE_REL32:
   case TYPE_REL64:
     isBranch = true;
@@ -568,12 +575,12 @@ static void translateImmediate(MCInst &mcInst, uint64_t immediate,
   if(!tryAddingSymbolicOperand(immediate + pcrel, isBranch, insn.startLocation,
                                insn.immediateOffset, insn.immediateSize,
                                mcInst, Dis))
-    mcInst.addOperand(MCOperand::CreateImm(immediate));
+    mcInst.addOperand(MCOperand::createImm(immediate));
 
   if (type == TYPE_MOFFS8 || type == TYPE_MOFFS16 ||
       type == TYPE_MOFFS32 || type == TYPE_MOFFS64) {
     MCOperand segmentReg;
-    segmentReg = MCOperand::CreateReg(segmentRegnums[insn.segmentOverride]);
+    segmentReg = MCOperand::createReg(segmentRegnums[insn.segmentOverride]);
     mcInst.addOperand(segmentReg);
   }
 }
@@ -606,7 +613,7 @@ static bool translateRMRegister(MCInst &mcInst,
     return true;
 #define ENTRY(x)                                                      \
   case EA_REG_##x:                                                    \
-    mcInst.addOperand(MCOperand::CreateReg(X86::x)); break;
+    mcInst.addOperand(MCOperand::createReg(X86::x)); break;
   ALL_REGS
 #undef ENTRY
   }
@@ -651,12 +658,12 @@ static bool translateRMMemory(MCInst &mcInst, InternalInstruction &insn,
         return true;
 #define ENTRY(x)                                          \
       case SIB_BASE_##x:                                  \
-        baseReg = MCOperand::CreateReg(X86::x); break;
+        baseReg = MCOperand::createReg(X86::x); break;
       ALL_SIB_BASES
 #undef ENTRY
       }
     } else {
-      baseReg = MCOperand::CreateReg(0);
+      baseReg = MCOperand::createReg(0);
     }
 
     // Check whether we are handling VSIB addressing mode for GATHER.
@@ -706,7 +713,7 @@ static bool translateRMMemory(MCInst &mcInst, InternalInstruction &insn,
         return true;
 #define ENTRY(x)                                          \
       case SIB_INDEX_##x:                                 \
-        indexReg = MCOperand::CreateReg(X86::x); break;
+        indexReg = MCOperand::createReg(X86::x); break;
       EA_BASES_32BIT
       EA_BASES_64BIT
       REGS_XMM
@@ -715,10 +722,10 @@ static bool translateRMMemory(MCInst &mcInst, InternalInstruction &insn,
 #undef ENTRY
       }
     } else {
-      indexReg = MCOperand::CreateReg(0);
+      indexReg = MCOperand::createReg(0);
     }
 
-    scaleAmount = MCOperand::CreateImm(insn.sibScale);
+    scaleAmount = MCOperand::createImm(insn.sibScale);
   } else {
     switch (insn.eaBase) {
     case EA_BASE_NONE:
@@ -732,31 +739,31 @@ static bool translateRMMemory(MCInst &mcInst, InternalInstruction &insn,
         tryAddingPcLoadReferenceComment(insn.startLocation +
                                         insn.displacementOffset,
                                         insn.displacement + pcrel, Dis);
-        baseReg = MCOperand::CreateReg(X86::RIP); // Section 2.2.1.6
+        baseReg = MCOperand::createReg(X86::RIP); // Section 2.2.1.6
       }
       else
-        baseReg = MCOperand::CreateReg(0);
+        baseReg = MCOperand::createReg(0);
 
-      indexReg = MCOperand::CreateReg(0);
+      indexReg = MCOperand::createReg(0);
       break;
     case EA_BASE_BX_SI:
-      baseReg = MCOperand::CreateReg(X86::BX);
-      indexReg = MCOperand::CreateReg(X86::SI);
+      baseReg = MCOperand::createReg(X86::BX);
+      indexReg = MCOperand::createReg(X86::SI);
       break;
     case EA_BASE_BX_DI:
-      baseReg = MCOperand::CreateReg(X86::BX);
-      indexReg = MCOperand::CreateReg(X86::DI);
+      baseReg = MCOperand::createReg(X86::BX);
+      indexReg = MCOperand::createReg(X86::DI);
       break;
     case EA_BASE_BP_SI:
-      baseReg = MCOperand::CreateReg(X86::BP);
-      indexReg = MCOperand::CreateReg(X86::SI);
+      baseReg = MCOperand::createReg(X86::BP);
+      indexReg = MCOperand::createReg(X86::SI);
       break;
     case EA_BASE_BP_DI:
-      baseReg = MCOperand::CreateReg(X86::BP);
-      indexReg = MCOperand::CreateReg(X86::DI);
+      baseReg = MCOperand::createReg(X86::BP);
+      indexReg = MCOperand::createReg(X86::DI);
       break;
     default:
-      indexReg = MCOperand::CreateReg(0);
+      indexReg = MCOperand::createReg(0);
       switch (insn.eaBase) {
       default:
         debug("Unexpected eaBase");
@@ -767,7 +774,7 @@ static bool translateRMMemory(MCInst &mcInst, InternalInstruction &insn,
         //   placeholders to keep the compiler happy.
 #define ENTRY(x)                                        \
       case EA_BASE_##x:                                 \
-        baseReg = MCOperand::CreateReg(X86::x); break;
+        baseReg = MCOperand::createReg(X86::x); break;
       ALL_EA_BASES
 #undef ENTRY
 #define ENTRY(x) case EA_REG_##x:
@@ -779,12 +786,12 @@ static bool translateRMMemory(MCInst &mcInst, InternalInstruction &insn,
       }
     }
 
-    scaleAmount = MCOperand::CreateImm(1);
+    scaleAmount = MCOperand::createImm(1);
   }
 
-  displacement = MCOperand::CreateImm(insn.displacement);
+  displacement = MCOperand::createImm(insn.displacement);
 
-  segmentReg = MCOperand::CreateReg(segmentRegnums[insn.segmentOverride]);
+  segmentReg = MCOperand::createReg(segmentRegnums[insn.segmentOverride]);
 
   mcInst.addOperand(baseReg);
   mcInst.addOperand(scaleAmount);
@@ -828,6 +835,7 @@ static bool translateRM(MCInst &mcInst, const OperandSpecifier &operand,
   case TYPE_VK16:
   case TYPE_DEBUGREG:
   case TYPE_CONTROLREG:
+  case TYPE_BNDR:
     return translateRMRegister(mcInst, insn);
   case TYPE_M:
   case TYPE_M8:
@@ -856,7 +864,7 @@ static bool translateRM(MCInst &mcInst, const OperandSpecifier &operand,
 /// @param stackPos     - The stack position to translate.
 static void translateFPRegister(MCInst &mcInst,
                                 uint8_t stackPos) {
-  mcInst.addOperand(MCOperand::CreateReg(X86::ST0 + stackPos));
+  mcInst.addOperand(MCOperand::createReg(X86::ST0 + stackPos));
 }
 
 /// translateMaskRegister - Translates a 3-bit mask register number to
@@ -872,7 +880,7 @@ static bool translateMaskRegister(MCInst &mcInst,
     return true;
   }
 
-  mcInst.addOperand(MCOperand::CreateReg(X86::K0 + maskRegNum));
+  mcInst.addOperand(MCOperand::createReg(X86::K0 + maskRegNum));
   return false;
 }
 
diff --git a/lib/Target/X86/Disassembler/X86DisassemblerDecoder.cpp b/lib/Target/X86/Disassembler/X86DisassemblerDecoder.cpp
index 7c9e0127388..f73fa75f888 100644
--- a/lib/Target/X86/Disassembler/X86DisassemblerDecoder.cpp
+++ b/lib/Target/X86/Disassembler/X86DisassemblerDecoder.cpp
@@ -1165,35 +1165,30 @@ static int readSIB(struct InternalInstruction* insn) {
     return -1;
 
   index = indexFromSIB(insn->sib) | (xFromREX(insn->rexPrefix) << 3);
+
+  // FIXME: The fifth bit (bit index 4) is only to be used for instructions
+  // that understand VSIB indexing. ORing the bit in here is mildy dangerous
+  // because performing math on an 'enum SIBIndex' can produce garbage.
+  // Excluding the "none" value, it should cover 6 spaces of register names:
+  //   - 16 possibilities for 16-bit GPR starting at SIB_INDEX_BX_SI
+  //   - 16 possibilities for 32-bit GPR starting at SIB_INDEX_EAX
+  //   - 16 possibilities for 64-bit GPR starting at SIB_INDEX_RAX
+  //   - 32 possibilities for each of XMM, YMM, ZMM registers
+  // When sibIndexBase gets assigned SIB_INDEX_RAX as it does in 64-bit mode,
+  // summing in a fully decoded index between 0 and 31 can end up with a value
+  // that looks like something in the low half of the XMM range.
+  // translateRMMemory() tries to reverse the damage, with only partial success,
+  // as evidenced by known bugs in "test/MC/Disassembler/X86/x86-64.txt"
   if (insn->vectorExtensionType == TYPE_EVEX)
     index |= v2FromEVEX4of4(insn->vectorExtensionPrefix[3]) << 4;
 
-  switch (index) {
-  case 0x4:
+  if (index == 0x4) {
     insn->sibIndex = SIB_INDEX_NONE;
-    break;
-  default:
+  } else {
     insn->sibIndex = (SIBIndex)(sibIndexBase + index);
-    if (insn->sibIndex == SIB_INDEX_sib ||
-        insn->sibIndex == SIB_INDEX_sib64)
-      insn->sibIndex = SIB_INDEX_NONE;
-    break;
   }
 
-  switch (scaleFromSIB(insn->sib)) {
-  case 0:
-    insn->sibScale = 1;
-    break;
-  case 1:
-    insn->sibScale = 2;
-    break;
-  case 2:
-    insn->sibScale = 4;
-    break;
-  case 3:
-    insn->sibScale = 8;
-    break;
-  }
+  insn->sibScale = 1 << scaleFromSIB(insn->sib);
 
   base = baseFromSIB(insn->sib) | (bFromREX(insn->rexPrefix) << 3);
 
@@ -1366,16 +1361,17 @@ static int readModRM(struct InternalInstruction* insn) {
     switch (mod) {
     case 0x0:
       insn->eaDisplacement = EA_DISP_NONE; /* readSIB may override this */
-      switch (rm) {
-      case 0x14:
-      case 0x4:
-      case 0xc:   /* in case REXW.b is set */
+      // In determining whether RIP-relative mode is used (rm=5),
+      // or whether a SIB byte is present (rm=4),
+      // the extension bits (REX.b and EVEX.x) are ignored.
+      switch (rm & 7) {
+      case 0x4: // SIB byte is present
         insn->eaBase = (insn->addressSize == 4 ?
                         EA_BASE_sib : EA_BASE_sib64);
         if (readSIB(insn) || readDisplacement(insn))
           return -1;
         break;
-      case 0x5:
+      case 0x5: // RIP-relative
         insn->eaBase = EA_BASE_NONE;
         insn->eaDisplacement = EA_DISP_32;
         if (readDisplacement(insn))
@@ -1391,10 +1387,8 @@ static int readModRM(struct InternalInstruction* insn) {
       /* FALLTHROUGH */
     case 0x2:
       insn->eaDisplacement = (mod == 0x1 ? EA_DISP_8 : EA_DISP_32);
-      switch (rm) {
-      case 0x14:
-      case 0x4:
-      case 0xc:   /* in case REXW.b is set */
+      switch (rm & 7) {
+      case 0x4: // SIB byte is present
         insn->eaBase = EA_BASE_sib;
         if (readSIB(insn) || readDisplacement(insn))
           return -1;
diff --git a/lib/Target/X86/Disassembler/X86DisassemblerDecoderCommon.h b/lib/Target/X86/Disassembler/X86DisassemblerDecoderCommon.h
index 9e650500139..301db72feaf 100644
--- a/lib/Target/X86/Disassembler/X86DisassemblerDecoderCommon.h
+++ b/lib/Target/X86/Disassembler/X86DisassemblerDecoderCommon.h
@@ -460,6 +460,7 @@ enum OperandEncoding {
   ENUM_ENTRY(TYPE_SEGMENTREG, "Segment register operand")                      \
   ENUM_ENTRY(TYPE_DEBUGREG,   "Debug register operand")                        \
   ENUM_ENTRY(TYPE_CONTROLREG, "Control register operand")                      \
+  ENUM_ENTRY(TYPE_BNDR,       "MPX bounds register")                           \
                                                                                \
   ENUM_ENTRY(TYPE_Mv,         "Memory operand of operand size")                \
   ENUM_ENTRY(TYPE_Rv,         "Register operand of operand size")              \
diff --git a/lib/Target/X86/InstPrinter/X86ATTInstPrinter.cpp b/lib/Target/X86/InstPrinter/X86ATTInstPrinter.cpp
index f265f1dee9d..ea727e6e82f 100644
--- a/lib/Target/X86/InstPrinter/X86ATTInstPrinter.cpp
+++ b/lib/Target/X86/InstPrinter/X86ATTInstPrinter.cpp
@@ -48,7 +48,7 @@ void X86ATTInstPrinter::printInst(const MCInst *MI, raw_ostream &OS,
         EmitAnyX86InstComments(MI, *CommentStream, getRegisterName);
 
   if (TSFlags & X86II::LOCK)
-    OS << "\tlock\n";
+    OS << "\tlock\t";
 
   // Output CALLpcrel32 as "callq" in 64-bit mode.
   // In Intel annotation it's always emitted as "call".
@@ -57,7 +57,7 @@ void X86ATTInstPrinter::printInst(const MCInst *MI, raw_ostream &OS,
   // InstrInfo.td as soon as Requires clause is supported properly
   // for InstAlias.
   if (MI->getOpcode() == X86::CALLpcrel32 &&
-      (STI.getFeatureBits() & X86::Mode64Bit) != 0) {
+      (STI.getFeatureBits()[X86::Mode64Bit])) {
     OS << "\tcallq\t";
     printPCRelImm(MI, 0, OS);
   }
@@ -150,11 +150,11 @@ void X86ATTInstPrinter::printPCRelImm(const MCInst *MI, unsigned OpNo,
     // that address in hex.
     const MCConstantExpr *BranchTarget = dyn_cast<MCConstantExpr>(Op.getExpr());
     int64_t Address;
-    if (BranchTarget && BranchTarget->EvaluateAsAbsolute(Address)) {
+    if (BranchTarget && BranchTarget->evaluateAsAbsolute(Address)) {
       O << formatHex((uint64_t)Address);
     } else {
       // Otherwise, just print the expression.
-      O << *Op.getExpr();
+      Op.getExpr()->print(O, &MAI);
     }
   }
 }
@@ -178,7 +178,9 @@ void X86ATTInstPrinter::printOperand(const MCInst *MI, unsigned OpNo,
 
   } else {
     assert(Op.isExpr() && "unknown operand kind in printOperand");
-    O << markup("<imm:") << '$' << *Op.getExpr() << markup(">");
+    O << markup("<imm:") << '$';
+    Op.getExpr()->print(O, &MAI);
+    O << markup(">");
   }
 }
 
@@ -203,7 +205,7 @@ void X86ATTInstPrinter::printMemReference(const MCInst *MI, unsigned Op,
       O << formatImm(DispVal);
   } else {
     assert(DispSpec.isExpr() && "non-immediate displacement for LEA?");
-    O << *DispSpec.getExpr();
+    DispSpec.getExpr()->print(O, &MAI);
   }
 
   if (IndexReg.getReg() || BaseReg.getReg()) {
@@ -273,7 +275,7 @@ void X86ATTInstPrinter::printMemOffset(const MCInst *MI, unsigned Op,
     O << formatImm(DispSpec.getImm());
   } else {
     assert(DispSpec.isExpr() && "non-immediate displacement?");
-    O << *DispSpec.getExpr();
+    DispSpec.getExpr()->print(O, &MAI);
   }
 
   O << markup(">");
diff --git a/lib/Target/X86/InstPrinter/X86InstComments.cpp b/lib/Target/X86/InstPrinter/X86InstComments.cpp
index 10a1482d416..91b144a4482 100644
--- a/lib/Target/X86/InstPrinter/X86InstComments.cpp
+++ b/lib/Target/X86/InstPrinter/X86InstComments.cpp
@@ -1,982 +1,1004 @@
-//===-- X86InstComments.cpp - Generate verbose-asm comments for instrs ----===//
-//
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
-//
-//===----------------------------------------------------------------------===//
-//
-// This defines functionality used to emit comments about X86 instructions to
-// an output stream for -fverbose-asm.
-//
-//===----------------------------------------------------------------------===//
-
-#include "X86InstComments.h"
-#include "MCTargetDesc/X86MCTargetDesc.h"
-#include "Utils/X86ShuffleDecode.h"
-#include "llvm/MC/MCInst.h"
-#include "llvm/CodeGen/MachineValueType.h"
-#include "llvm/Support/raw_ostream.h"
-
-using namespace llvm;
-
-/// \brief Extracts the src/dst types for a given zero extension instruction.
-/// \note While the number of elements in DstVT type correct, the
-/// number in the SrcVT type is expanded to fill the src xmm register and the
-/// upper elements may not be included in the dst xmm/ymm register.
-static void getZeroExtensionTypes(const MCInst *MI, MVT &SrcVT, MVT &DstVT) {
-  switch (MI->getOpcode()) {
-  default:
-    llvm_unreachable("Unknown zero extension instruction");
-  // i8 zero extension
-  case X86::PMOVZXBWrm:
-  case X86::PMOVZXBWrr:
-  case X86::VPMOVZXBWrm:
-  case X86::VPMOVZXBWrr:
-    SrcVT = MVT::v16i8;
-    DstVT = MVT::v8i16;
-    break;
-  case X86::VPMOVZXBWYrm:
-  case X86::VPMOVZXBWYrr:
-    SrcVT = MVT::v16i8;
-    DstVT = MVT::v16i16;
-    break;
-  case X86::PMOVZXBDrm:
-  case X86::PMOVZXBDrr:
-  case X86::VPMOVZXBDrm:
-  case X86::VPMOVZXBDrr:
-    SrcVT = MVT::v16i8;
-    DstVT = MVT::v4i32;
-    break;
-  case X86::VPMOVZXBDYrm:
-  case X86::VPMOVZXBDYrr:
-    SrcVT = MVT::v16i8;
-    DstVT = MVT::v8i32;
-    break;
-  case X86::PMOVZXBQrm:
-  case X86::PMOVZXBQrr:
-  case X86::VPMOVZXBQrm:
-  case X86::VPMOVZXBQrr:
-    SrcVT = MVT::v16i8;
-    DstVT = MVT::v2i64;
-    break;
-  case X86::VPMOVZXBQYrm:
-  case X86::VPMOVZXBQYrr:
-    SrcVT = MVT::v16i8;
-    DstVT = MVT::v4i64;
-    break;
-  // i16 zero extension
-  case X86::PMOVZXWDrm:
-  case X86::PMOVZXWDrr:
-  case X86::VPMOVZXWDrm:
-  case X86::VPMOVZXWDrr:
-    SrcVT = MVT::v8i16;
-    DstVT = MVT::v4i32;
-    break;
-  case X86::VPMOVZXWDYrm:
-  case X86::VPMOVZXWDYrr:
-    SrcVT = MVT::v8i16;
-    DstVT = MVT::v8i32;
-    break;
-  case X86::PMOVZXWQrm:
-  case X86::PMOVZXWQrr:
-  case X86::VPMOVZXWQrm:
-  case X86::VPMOVZXWQrr:
-    SrcVT = MVT::v8i16;
-    DstVT = MVT::v2i64;
-    break;
-  case X86::VPMOVZXWQYrm:
-  case X86::VPMOVZXWQYrr:
-    SrcVT = MVT::v8i16;
-    DstVT = MVT::v4i64;
-    break;
-  // i32 zero extension
-  case X86::PMOVZXDQrm:
-  case X86::PMOVZXDQrr:
-  case X86::VPMOVZXDQrm:
-  case X86::VPMOVZXDQrr:
-    SrcVT = MVT::v4i32;
-    DstVT = MVT::v2i64;
-    break;
-  case X86::VPMOVZXDQYrm:
-  case X86::VPMOVZXDQYrr:
-    SrcVT = MVT::v4i32;
-    DstVT = MVT::v4i64;
-    break;
-  }
-}
-
-//===----------------------------------------------------------------------===//
-// Top Level Entrypoint
-//===----------------------------------------------------------------------===//
-
-/// EmitAnyX86InstComments - This function decodes x86 instructions and prints
-/// newline terminated strings to the specified string if desired.  This
-/// information is shown in disassembly dumps when verbose assembly is enabled.
-bool llvm::EmitAnyX86InstComments(const MCInst *MI, raw_ostream &OS,
-                                  const char *(*getRegName)(unsigned)) {
-  // If this is a shuffle operation, the switch should fill in this state.
-  SmallVector<int, 8> ShuffleMask;
-  const char *DestName = nullptr, *Src1Name = nullptr, *Src2Name = nullptr;
-
-  switch (MI->getOpcode()) {
-  default:
-    // Not an instruction for which we can decode comments.
-    return false;
-
-  case X86::BLENDPDrri:
-  case X86::VBLENDPDrri:
-    Src2Name = getRegName(MI->getOperand(2).getReg());
-    // FALL THROUGH.
-  case X86::BLENDPDrmi:
-  case X86::VBLENDPDrmi:
-    if(MI->getOperand(MI->getNumOperands()-1).isImm())
-      DecodeBLENDMask(MVT::v2f64,
-                      MI->getOperand(MI->getNumOperands()-1).getImm(),
-                      ShuffleMask);
-    Src1Name = getRegName(MI->getOperand(1).getReg());
-    DestName = getRegName(MI->getOperand(0).getReg());
-    break;
-  case X86::VBLENDPDYrri:
-    Src2Name = getRegName(MI->getOperand(2).getReg());
-    // FALL THROUGH.
-  case X86::VBLENDPDYrmi:
-    if(MI->getOperand(MI->getNumOperands()-1).isImm())
-      DecodeBLENDMask(MVT::v4f64,
-                      MI->getOperand(MI->getNumOperands()-1).getImm(),
-                      ShuffleMask);
-    Src1Name = getRegName(MI->getOperand(1).getReg());
-    DestName = getRegName(MI->getOperand(0).getReg());
-    break;
-
-  case X86::BLENDPSrri:
-  case X86::VBLENDPSrri:
-    Src2Name = getRegName(MI->getOperand(2).getReg());
-    // FALL THROUGH.
-  case X86::BLENDPSrmi:
-  case X86::VBLENDPSrmi:
-    if(MI->getOperand(MI->getNumOperands()-1).isImm())
-      DecodeBLENDMask(MVT::v4f32,
-                      MI->getOperand(MI->getNumOperands()-1).getImm(),
-                      ShuffleMask);
-    Src1Name = getRegName(MI->getOperand(1).getReg());
-    DestName = getRegName(MI->getOperand(0).getReg());
-    break;
-  case X86::VBLENDPSYrri:
-    Src2Name = getRegName(MI->getOperand(2).getReg());
-    // FALL THROUGH.
-  case X86::VBLENDPSYrmi:
-    if(MI->getOperand(MI->getNumOperands()-1).isImm())
-      DecodeBLENDMask(MVT::v8f32,
-                      MI->getOperand(MI->getNumOperands()-1).getImm(),
-                      ShuffleMask);
-    Src1Name = getRegName(MI->getOperand(1).getReg());
-    DestName = getRegName(MI->getOperand(0).getReg());
-    break;
-
-  case X86::PBLENDWrri:
-  case X86::VPBLENDWrri:
-    Src2Name = getRegName(MI->getOperand(2).getReg());
-    // FALL THROUGH.
-  case X86::PBLENDWrmi:
-  case X86::VPBLENDWrmi:
-    if(MI->getOperand(MI->getNumOperands()-1).isImm())
-      DecodeBLENDMask(MVT::v8i16,
-                      MI->getOperand(MI->getNumOperands()-1).getImm(),
-                      ShuffleMask);
-    Src1Name = getRegName(MI->getOperand(1).getReg());
-    DestName = getRegName(MI->getOperand(0).getReg());
-    break;
-  case X86::VPBLENDWYrri:
-    Src2Name = getRegName(MI->getOperand(2).getReg());
-    // FALL THROUGH.
-  case X86::VPBLENDWYrmi:
-    if(MI->getOperand(MI->getNumOperands()-1).isImm())
-      DecodeBLENDMask(MVT::v16i16,
-                      MI->getOperand(MI->getNumOperands()-1).getImm(),
-                      ShuffleMask);
-    Src1Name = getRegName(MI->getOperand(1).getReg());
-    DestName = getRegName(MI->getOperand(0).getReg());
-    break;
-
-  case X86::VPBLENDDrri:
-    Src2Name = getRegName(MI->getOperand(2).getReg());
-    // FALL THROUGH.
-  case X86::VPBLENDDrmi:
-    if(MI->getOperand(MI->getNumOperands()-1).isImm())
-      DecodeBLENDMask(MVT::v4i32,
-                      MI->getOperand(MI->getNumOperands()-1).getImm(),
-                      ShuffleMask);
-    Src1Name = getRegName(MI->getOperand(1).getReg());
-    DestName = getRegName(MI->getOperand(0).getReg());
-    break;
-
-  case X86::VPBLENDDYrri:
-    Src2Name = getRegName(MI->getOperand(2).getReg());
-    // FALL THROUGH.
-  case X86::VPBLENDDYrmi:
-    if(MI->getOperand(MI->getNumOperands()-1).isImm())
-      DecodeBLENDMask(MVT::v8i32,
-                      MI->getOperand(MI->getNumOperands()-1).getImm(),
-                      ShuffleMask);
-    Src1Name = getRegName(MI->getOperand(1).getReg());
-    DestName = getRegName(MI->getOperand(0).getReg());
-    break;
-
-  case X86::INSERTPSrr:
-  case X86::VINSERTPSrr:
-    Src2Name = getRegName(MI->getOperand(2).getReg());
-    // FALL THROUGH.
-  case X86::INSERTPSrm:
-  case X86::VINSERTPSrm:
-    DestName = getRegName(MI->getOperand(0).getReg());
-    Src1Name = getRegName(MI->getOperand(1).getReg());
-    if(MI->getOperand(MI->getNumOperands()-1).isImm())
-      DecodeINSERTPSMask(MI->getOperand(MI->getNumOperands()-1).getImm(),
-                         ShuffleMask);
-    break;
-
-  case X86::MOVLHPSrr:
-  case X86::VMOVLHPSrr:
-    Src2Name = getRegName(MI->getOperand(2).getReg());
-    Src1Name = getRegName(MI->getOperand(1).getReg());
-    DestName = getRegName(MI->getOperand(0).getReg());
-    DecodeMOVLHPSMask(2, ShuffleMask);
-    break;
-
-  case X86::MOVHLPSrr:
-  case X86::VMOVHLPSrr:
-    Src2Name = getRegName(MI->getOperand(2).getReg());
-    Src1Name = getRegName(MI->getOperand(1).getReg());
-    DestName = getRegName(MI->getOperand(0).getReg());
-    DecodeMOVHLPSMask(2, ShuffleMask);
-    break;
-
-  case X86::MOVSLDUPrr:
-  case X86::VMOVSLDUPrr:
-    Src1Name = getRegName(MI->getOperand(1).getReg());
-    // FALL THROUGH.
-  case X86::MOVSLDUPrm:
-  case X86::VMOVSLDUPrm:
-    DestName = getRegName(MI->getOperand(0).getReg());
-    DecodeMOVSLDUPMask(MVT::v4f32, ShuffleMask);
-    break;
-
-  case X86::VMOVSHDUPYrr:
-    Src1Name = getRegName(MI->getOperand(1).getReg());
-    // FALL THROUGH.
-  case X86::VMOVSHDUPYrm:
-    DestName = getRegName(MI->getOperand(0).getReg());
-    DecodeMOVSHDUPMask(MVT::v8f32, ShuffleMask);
-    break;
-
-  case X86::VMOVSLDUPYrr:
-    Src1Name = getRegName(MI->getOperand(1).getReg());
-    // FALL THROUGH.
-  case X86::VMOVSLDUPYrm:
-    DestName = getRegName(MI->getOperand(0).getReg());
-    DecodeMOVSLDUPMask(MVT::v8f32, ShuffleMask);
-    break;
-
-  case X86::MOVSHDUPrr:
-  case X86::VMOVSHDUPrr:
-    Src1Name = getRegName(MI->getOperand(1).getReg());
-    // FALL THROUGH.
-  case X86::MOVSHDUPrm:
-  case X86::VMOVSHDUPrm:
-    DestName = getRegName(MI->getOperand(0).getReg());
-    DecodeMOVSHDUPMask(MVT::v4f32, ShuffleMask);
-    break;
-
-  case X86::VMOVDDUPYrr:
-    Src1Name = getRegName(MI->getOperand(1).getReg());
-    // FALL THROUGH.
-  case X86::VMOVDDUPYrm:
-    DestName = getRegName(MI->getOperand(0).getReg());
-    DecodeMOVDDUPMask(MVT::v4f64, ShuffleMask);
-    break;
-
-  case X86::MOVDDUPrr:
-  case X86::VMOVDDUPrr:
-    Src1Name = getRegName(MI->getOperand(1).getReg());
-    // FALL THROUGH.
-  case X86::MOVDDUPrm:
-  case X86::VMOVDDUPrm:
-    DestName = getRegName(MI->getOperand(0).getReg());
-    DecodeMOVDDUPMask(MVT::v2f64, ShuffleMask);
-    break;
-
-  case X86::PSLLDQri:
-  case X86::VPSLLDQri:
-    Src1Name = getRegName(MI->getOperand(1).getReg());
-    DestName = getRegName(MI->getOperand(0).getReg());
-    if(MI->getOperand(MI->getNumOperands()-1).isImm())
-      DecodePSLLDQMask(MVT::v16i8,
-                       MI->getOperand(MI->getNumOperands()-1).getImm(),
-                       ShuffleMask);
-    break;
-
-  case X86::VPSLLDQYri:
-    Src1Name = getRegName(MI->getOperand(1).getReg());
-    DestName = getRegName(MI->getOperand(0).getReg());
-    if(MI->getOperand(MI->getNumOperands()-1).isImm())
-      DecodePSLLDQMask(MVT::v32i8,
-                       MI->getOperand(MI->getNumOperands()-1).getImm(),
-                       ShuffleMask);
-    break;
-
-  case X86::PSRLDQri:
-  case X86::VPSRLDQri:
-    Src1Name = getRegName(MI->getOperand(1).getReg());
-    DestName = getRegName(MI->getOperand(0).getReg());
-    if(MI->getOperand(MI->getNumOperands()-1).isImm())
-      DecodePSRLDQMask(MVT::v16i8,
-                       MI->getOperand(MI->getNumOperands()-1).getImm(),
-                       ShuffleMask);
-    break;
-
-  case X86::VPSRLDQYri:
-    Src1Name = getRegName(MI->getOperand(1).getReg());
-    DestName = getRegName(MI->getOperand(0).getReg());
-    if(MI->getOperand(MI->getNumOperands()-1).isImm())
-      DecodePSRLDQMask(MVT::v32i8,
-                       MI->getOperand(MI->getNumOperands()-1).getImm(),
-                       ShuffleMask);
-    break;
-
-  case X86::PALIGNR128rr:
-  case X86::VPALIGNR128rr:
-    Src1Name = getRegName(MI->getOperand(2).getReg());
-    // FALL THROUGH.
-  case X86::PALIGNR128rm:
-  case X86::VPALIGNR128rm:
-    Src2Name = getRegName(MI->getOperand(1).getReg());
-    DestName = getRegName(MI->getOperand(0).getReg());
-    if(MI->getOperand(MI->getNumOperands()-1).isImm())
-      DecodePALIGNRMask(MVT::v16i8,
-                        MI->getOperand(MI->getNumOperands()-1).getImm(),
-                        ShuffleMask);
-    break;
-  case X86::VPALIGNR256rr:
-    Src1Name = getRegName(MI->getOperand(2).getReg());
-    // FALL THROUGH.
-  case X86::VPALIGNR256rm:
-    Src2Name = getRegName(MI->getOperand(1).getReg());
-    DestName = getRegName(MI->getOperand(0).getReg());
-    if(MI->getOperand(MI->getNumOperands()-1).isImm())
-      DecodePALIGNRMask(MVT::v32i8,
-                        MI->getOperand(MI->getNumOperands()-1).getImm(),
-                        ShuffleMask);
-    break;
-
-  case X86::PSHUFDri:
-  case X86::VPSHUFDri:
-    Src1Name = getRegName(MI->getOperand(1).getReg());
-    // FALL THROUGH.
-  case X86::PSHUFDmi:
-  case X86::VPSHUFDmi:
-    DestName = getRegName(MI->getOperand(0).getReg());
-    if(MI->getOperand(MI->getNumOperands()-1).isImm())
-      DecodePSHUFMask(MVT::v4i32,
-                      MI->getOperand(MI->getNumOperands()-1).getImm(),
-                      ShuffleMask);
-    break;
-  case X86::VPSHUFDYri:
-    Src1Name = getRegName(MI->getOperand(1).getReg());
-    // FALL THROUGH.
-  case X86::VPSHUFDYmi:
-    DestName = getRegName(MI->getOperand(0).getReg());
-    if(MI->getOperand(MI->getNumOperands()-1).isImm())
-      DecodePSHUFMask(MVT::v8i32,
-                      MI->getOperand(MI->getNumOperands()-1).getImm(),
-                      ShuffleMask);
-    break;
-
-
-  case X86::PSHUFHWri:
-  case X86::VPSHUFHWri:
-    Src1Name = getRegName(MI->getOperand(1).getReg());
-    // FALL THROUGH.
-  case X86::PSHUFHWmi:
-  case X86::VPSHUFHWmi:
-    DestName = getRegName(MI->getOperand(0).getReg());
-    if(MI->getOperand(MI->getNumOperands()-1).isImm())
-      DecodePSHUFHWMask(MVT::v8i16,
-                        MI->getOperand(MI->getNumOperands()-1).getImm(),
-                        ShuffleMask);
-    break;
-  case X86::VPSHUFHWYri:
-    Src1Name = getRegName(MI->getOperand(1).getReg());
-    // FALL THROUGH.
-  case X86::VPSHUFHWYmi:
-    DestName = getRegName(MI->getOperand(0).getReg());
-    if(MI->getOperand(MI->getNumOperands()-1).isImm())
-      DecodePSHUFHWMask(MVT::v16i16,
-                        MI->getOperand(MI->getNumOperands()-1).getImm(),
-                        ShuffleMask);
-    break;
-  case X86::PSHUFLWri:
-  case X86::VPSHUFLWri:
-    Src1Name = getRegName(MI->getOperand(1).getReg());
-    // FALL THROUGH.
-  case X86::PSHUFLWmi:
-  case X86::VPSHUFLWmi:
-    DestName = getRegName(MI->getOperand(0).getReg());
-    if(MI->getOperand(MI->getNumOperands()-1).isImm())
-      DecodePSHUFLWMask(MVT::v8i16,
-                        MI->getOperand(MI->getNumOperands()-1).getImm(),
-                        ShuffleMask);
-    break;
-  case X86::VPSHUFLWYri:
-    Src1Name = getRegName(MI->getOperand(1).getReg());
-    // FALL THROUGH.
-  case X86::VPSHUFLWYmi:
-    DestName = getRegName(MI->getOperand(0).getReg());
-    if(MI->getOperand(MI->getNumOperands()-1).isImm())
-      DecodePSHUFLWMask(MVT::v16i16,
-                        MI->getOperand(MI->getNumOperands()-1).getImm(),
-                        ShuffleMask);
-    break;
-
-  case X86::PUNPCKHBWrr:
-  case X86::VPUNPCKHBWrr:
-    Src2Name = getRegName(MI->getOperand(2).getReg());
-    // FALL THROUGH.
-  case X86::PUNPCKHBWrm:
-  case X86::VPUNPCKHBWrm:
-    Src1Name = getRegName(MI->getOperand(1).getReg());
-    DestName = getRegName(MI->getOperand(0).getReg());
-    DecodeUNPCKHMask(MVT::v16i8, ShuffleMask);
-    break;
-  case X86::VPUNPCKHBWYrr:
-    Src2Name = getRegName(MI->getOperand(2).getReg());
-    // FALL THROUGH.
-  case X86::VPUNPCKHBWYrm:
-    Src1Name = getRegName(MI->getOperand(1).getReg());
-    DestName = getRegName(MI->getOperand(0).getReg());
-    DecodeUNPCKHMask(MVT::v32i8, ShuffleMask);
-    break;
-  case X86::PUNPCKHWDrr:
-  case X86::VPUNPCKHWDrr:
-    Src2Name = getRegName(MI->getOperand(2).getReg());
-    // FALL THROUGH.
-  case X86::PUNPCKHWDrm:
-  case X86::VPUNPCKHWDrm:
-    Src1Name = getRegName(MI->getOperand(1).getReg());
-    DestName = getRegName(MI->getOperand(0).getReg());
-    DecodeUNPCKHMask(MVT::v8i16, ShuffleMask);
-    break;
-  case X86::VPUNPCKHWDYrr:
-    Src2Name = getRegName(MI->getOperand(2).getReg());
-    // FALL THROUGH.
-  case X86::VPUNPCKHWDYrm:
-    Src1Name = getRegName(MI->getOperand(1).getReg());
-    DestName = getRegName(MI->getOperand(0).getReg());
-    DecodeUNPCKHMask(MVT::v16i16, ShuffleMask);
-    break;
-  case X86::PUNPCKHDQrr:
-  case X86::VPUNPCKHDQrr:
-    Src2Name = getRegName(MI->getOperand(2).getReg());
-    // FALL THROUGH.
-  case X86::PUNPCKHDQrm:
-  case X86::VPUNPCKHDQrm:
-    Src1Name = getRegName(MI->getOperand(1).getReg());
-    DestName = getRegName(MI->getOperand(0).getReg());
-    DecodeUNPCKHMask(MVT::v4i32, ShuffleMask);
-    break;
-  case X86::VPUNPCKHDQYrr:
-    Src2Name = getRegName(MI->getOperand(2).getReg());
-    // FALL THROUGH.
-  case X86::VPUNPCKHDQYrm:
-    Src1Name = getRegName(MI->getOperand(1).getReg());
-    DestName = getRegName(MI->getOperand(0).getReg());
-    DecodeUNPCKHMask(MVT::v8i32, ShuffleMask);
-    break;
-  case X86::VPUNPCKHDQZrr:
-    Src2Name = getRegName(MI->getOperand(2).getReg());
-    // FALL THROUGH.
-  case X86::VPUNPCKHDQZrm:
-    Src1Name = getRegName(MI->getOperand(1).getReg());
-    DestName = getRegName(MI->getOperand(0).getReg());
-    DecodeUNPCKHMask(MVT::v16i32, ShuffleMask);
-    break;
-  case X86::PUNPCKHQDQrr:
-  case X86::VPUNPCKHQDQrr:
-    Src2Name = getRegName(MI->getOperand(2).getReg());
-    // FALL THROUGH.
-  case X86::PUNPCKHQDQrm:
-  case X86::VPUNPCKHQDQrm:
-    Src1Name = getRegName(MI->getOperand(1).getReg());
-    DestName = getRegName(MI->getOperand(0).getReg());
-    DecodeUNPCKHMask(MVT::v2i64, ShuffleMask);
-    break;
-  case X86::VPUNPCKHQDQYrr:
-    Src2Name = getRegName(MI->getOperand(2).getReg());
-    // FALL THROUGH.
-  case X86::VPUNPCKHQDQYrm:
-    Src1Name = getRegName(MI->getOperand(1).getReg());
-    DestName = getRegName(MI->getOperand(0).getReg());
-    DecodeUNPCKHMask(MVT::v4i64, ShuffleMask);
-    break;
-  case X86::VPUNPCKHQDQZrr:
-    Src2Name = getRegName(MI->getOperand(2).getReg());
-    // FALL THROUGH.
-  case X86::VPUNPCKHQDQZrm:
-    Src1Name = getRegName(MI->getOperand(1).getReg());
-    DestName = getRegName(MI->getOperand(0).getReg());
-    DecodeUNPCKHMask(MVT::v8i64, ShuffleMask);
-    break;
-
-  case X86::PUNPCKLBWrr:
-  case X86::VPUNPCKLBWrr:
-    Src2Name = getRegName(MI->getOperand(2).getReg());
-    // FALL THROUGH.
-  case X86::PUNPCKLBWrm:
-  case X86::VPUNPCKLBWrm:
-    Src1Name = getRegName(MI->getOperand(1).getReg());
-    DestName = getRegName(MI->getOperand(0).getReg());
-    DecodeUNPCKLMask(MVT::v16i8, ShuffleMask);
-    break;
-  case X86::VPUNPCKLBWYrr:
-    Src2Name = getRegName(MI->getOperand(2).getReg());
-    // FALL THROUGH.
-  case X86::VPUNPCKLBWYrm:
-    Src1Name = getRegName(MI->getOperand(1).getReg());
-    DestName = getRegName(MI->getOperand(0).getReg());
-    DecodeUNPCKLMask(MVT::v32i8, ShuffleMask);
-    break;
-  case X86::PUNPCKLWDrr:
-  case X86::VPUNPCKLWDrr:
-    Src2Name = getRegName(MI->getOperand(2).getReg());
-    // FALL THROUGH.
-  case X86::PUNPCKLWDrm:
-  case X86::VPUNPCKLWDrm:
-    Src1Name = getRegName(MI->getOperand(1).getReg());
-    DestName = getRegName(MI->getOperand(0).getReg());
-    DecodeUNPCKLMask(MVT::v8i16, ShuffleMask);
-    break;
-  case X86::VPUNPCKLWDYrr:
-    Src2Name = getRegName(MI->getOperand(2).getReg());
-    // FALL THROUGH.
-  case X86::VPUNPCKLWDYrm:
-    Src1Name = getRegName(MI->getOperand(1).getReg());
-    DestName = getRegName(MI->getOperand(0).getReg());
-    DecodeUNPCKLMask(MVT::v16i16, ShuffleMask);
-    break;
-  case X86::PUNPCKLDQrr:
-  case X86::VPUNPCKLDQrr:
-    Src2Name = getRegName(MI->getOperand(2).getReg());
-    // FALL THROUGH.
-  case X86::PUNPCKLDQrm:
-  case X86::VPUNPCKLDQrm:
-    Src1Name = getRegName(MI->getOperand(1).getReg());
-    DestName = getRegName(MI->getOperand(0).getReg());
-    DecodeUNPCKLMask(MVT::v4i32, ShuffleMask);
-    break;
-  case X86::VPUNPCKLDQYrr:
-    Src2Name = getRegName(MI->getOperand(2).getReg());
-    // FALL THROUGH.
-  case X86::VPUNPCKLDQYrm:
-    Src1Name = getRegName(MI->getOperand(1).getReg());
-    DestName = getRegName(MI->getOperand(0).getReg());
-    DecodeUNPCKLMask(MVT::v8i32, ShuffleMask);
-    break;
-  case X86::VPUNPCKLDQZrr:
-    Src2Name = getRegName(MI->getOperand(2).getReg());
-    // FALL THROUGH.
-  case X86::VPUNPCKLDQZrm:
-    Src1Name = getRegName(MI->getOperand(1).getReg());
-    DestName = getRegName(MI->getOperand(0).getReg());
-    DecodeUNPCKLMask(MVT::v16i32, ShuffleMask);
-    break;
-  case X86::PUNPCKLQDQrr:
-  case X86::VPUNPCKLQDQrr:
-    Src2Name = getRegName(MI->getOperand(2).getReg());
-    // FALL THROUGH.
-  case X86::PUNPCKLQDQrm:
-  case X86::VPUNPCKLQDQrm:
-    Src1Name = getRegName(MI->getOperand(1).getReg());
-    DestName = getRegName(MI->getOperand(0).getReg());
-    DecodeUNPCKLMask(MVT::v2i64, ShuffleMask);
-    break;
-  case X86::VPUNPCKLQDQYrr:
-    Src2Name = getRegName(MI->getOperand(2).getReg());
-    // FALL THROUGH.
-  case X86::VPUNPCKLQDQYrm:
-    Src1Name = getRegName(MI->getOperand(1).getReg());
-    DestName = getRegName(MI->getOperand(0).getReg());
-    DecodeUNPCKLMask(MVT::v4i64, ShuffleMask);
-    break;
-  case X86::VPUNPCKLQDQZrr:
-    Src2Name = getRegName(MI->getOperand(2).getReg());
-    // FALL THROUGH.
-  case X86::VPUNPCKLQDQZrm:
-    Src1Name = getRegName(MI->getOperand(1).getReg());
-    DestName = getRegName(MI->getOperand(0).getReg());
-    DecodeUNPCKLMask(MVT::v8i64, ShuffleMask);
-    break;
-
-  case X86::SHUFPDrri:
-  case X86::VSHUFPDrri:
-    Src2Name = getRegName(MI->getOperand(2).getReg());
-    // FALL THROUGH.
-  case X86::SHUFPDrmi:
-  case X86::VSHUFPDrmi:
-    if(MI->getOperand(MI->getNumOperands()-1).isImm())
-      DecodeSHUFPMask(MVT::v2f64,
-                      MI->getOperand(MI->getNumOperands()-1).getImm(),
-                      ShuffleMask);
-    Src1Name = getRegName(MI->getOperand(1).getReg());
-    DestName = getRegName(MI->getOperand(0).getReg());
-    break;
-  case X86::VSHUFPDYrri:
-    Src2Name = getRegName(MI->getOperand(2).getReg());
-    // FALL THROUGH.
-  case X86::VSHUFPDYrmi:
-    if(MI->getOperand(MI->getNumOperands()-1).isImm())
-      DecodeSHUFPMask(MVT::v4f64,
-                      MI->getOperand(MI->getNumOperands()-1).getImm(),
-                      ShuffleMask);
-    Src1Name = getRegName(MI->getOperand(1).getReg());
-    DestName = getRegName(MI->getOperand(0).getReg());
-    break;
-
-  case X86::SHUFPSrri:
-  case X86::VSHUFPSrri:
-    Src2Name = getRegName(MI->getOperand(2).getReg());
-    // FALL THROUGH.
-  case X86::SHUFPSrmi:
-  case X86::VSHUFPSrmi:
-    if(MI->getOperand(MI->getNumOperands()-1).isImm())
-      DecodeSHUFPMask(MVT::v4f32,
-                      MI->getOperand(MI->getNumOperands()-1).getImm(),
-                      ShuffleMask);
-    Src1Name = getRegName(MI->getOperand(1).getReg());
-    DestName = getRegName(MI->getOperand(0).getReg());
-    break;
-  case X86::VSHUFPSYrri:
-    Src2Name = getRegName(MI->getOperand(2).getReg());
-    // FALL THROUGH.
-  case X86::VSHUFPSYrmi:
-    if(MI->getOperand(MI->getNumOperands()-1).isImm())
-      DecodeSHUFPMask(MVT::v8f32,
-                      MI->getOperand(MI->getNumOperands()-1).getImm(),
-                      ShuffleMask);
-    Src1Name = getRegName(MI->getOperand(1).getReg());
-    DestName = getRegName(MI->getOperand(0).getReg());
-    break;
-
-  case X86::UNPCKLPDrr:
-  case X86::VUNPCKLPDrr:
-    Src2Name = getRegName(MI->getOperand(2).getReg());
-    // FALL THROUGH.
-  case X86::UNPCKLPDrm:
-  case X86::VUNPCKLPDrm:
-    DecodeUNPCKLMask(MVT::v2f64, ShuffleMask);
-    Src1Name = getRegName(MI->getOperand(1).getReg());
-    DestName = getRegName(MI->getOperand(0).getReg());
-    break;
-  case X86::VUNPCKLPDYrr:
-    Src2Name = getRegName(MI->getOperand(2).getReg());
-    // FALL THROUGH.
-  case X86::VUNPCKLPDYrm:
-    DecodeUNPCKLMask(MVT::v4f64, ShuffleMask);
-    Src1Name = getRegName(MI->getOperand(1).getReg());
-    DestName = getRegName(MI->getOperand(0).getReg());
-    break;
-  case X86::VUNPCKLPDZrr:
-    Src2Name = getRegName(MI->getOperand(2).getReg());
-    // FALL THROUGH.
-  case X86::VUNPCKLPDZrm:
-    DecodeUNPCKLMask(MVT::v8f64, ShuffleMask);
-    Src1Name = getRegName(MI->getOperand(1).getReg());
-    DestName = getRegName(MI->getOperand(0).getReg());
-    break;
-  case X86::UNPCKLPSrr:
-  case X86::VUNPCKLPSrr:
-    Src2Name = getRegName(MI->getOperand(2).getReg());
-    // FALL THROUGH.
-  case X86::UNPCKLPSrm:
-  case X86::VUNPCKLPSrm:
-    DecodeUNPCKLMask(MVT::v4f32, ShuffleMask);
-    Src1Name = getRegName(MI->getOperand(1).getReg());
-    DestName = getRegName(MI->getOperand(0).getReg());
-    break;
-  case X86::VUNPCKLPSYrr:
-    Src2Name = getRegName(MI->getOperand(2).getReg());
-    // FALL THROUGH.
-  case X86::VUNPCKLPSYrm:
-    DecodeUNPCKLMask(MVT::v8f32, ShuffleMask);
-    Src1Name = getRegName(MI->getOperand(1).getReg());
-    DestName = getRegName(MI->getOperand(0).getReg());
-    break;
-  case X86::VUNPCKLPSZrr:
-    Src2Name = getRegName(MI->getOperand(2).getReg());
-    // FALL THROUGH.
-  case X86::VUNPCKLPSZrm:
-    DecodeUNPCKLMask(MVT::v16f32, ShuffleMask);
-    Src1Name = getRegName(MI->getOperand(1).getReg());
-    DestName = getRegName(MI->getOperand(0).getReg());
-    break;
-  case X86::UNPCKHPDrr:
-  case X86::VUNPCKHPDrr:
-    Src2Name = getRegName(MI->getOperand(2).getReg());
-    // FALL THROUGH.
-  case X86::UNPCKHPDrm:
-  case X86::VUNPCKHPDrm:
-    DecodeUNPCKHMask(MVT::v2f64, ShuffleMask);
-    Src1Name = getRegName(MI->getOperand(1).getReg());
-    DestName = getRegName(MI->getOperand(0).getReg());
-    break;
-  case X86::VUNPCKHPDYrr:
-    Src2Name = getRegName(MI->getOperand(2).getReg());
-    // FALL THROUGH.
-  case X86::VUNPCKHPDYrm:
-    DecodeUNPCKHMask(MVT::v4f64, ShuffleMask);
-    Src1Name = getRegName(MI->getOperand(1).getReg());
-    DestName = getRegName(MI->getOperand(0).getReg());
-    break;
-  case X86::VUNPCKHPDZrr:
-    Src2Name = getRegName(MI->getOperand(2).getReg());
-    // FALL THROUGH.
-  case X86::VUNPCKHPDZrm:
-    DecodeUNPCKHMask(MVT::v8f64, ShuffleMask);
-    Src1Name = getRegName(MI->getOperand(1).getReg());
-    DestName = getRegName(MI->getOperand(0).getReg());
-    break;
-  case X86::UNPCKHPSrr:
-  case X86::VUNPCKHPSrr:
-    Src2Name = getRegName(MI->getOperand(2).getReg());
-    // FALL THROUGH.
-  case X86::UNPCKHPSrm:
-  case X86::VUNPCKHPSrm:
-    DecodeUNPCKHMask(MVT::v4f32, ShuffleMask);
-    Src1Name = getRegName(MI->getOperand(1).getReg());
-    DestName = getRegName(MI->getOperand(0).getReg());
-    break;
-  case X86::VUNPCKHPSYrr:
-    Src2Name = getRegName(MI->getOperand(2).getReg());
-    // FALL THROUGH.
-  case X86::VUNPCKHPSYrm:
-    DecodeUNPCKHMask(MVT::v8f32, ShuffleMask);
-    Src1Name = getRegName(MI->getOperand(1).getReg());
-    DestName = getRegName(MI->getOperand(0).getReg());
-    break;
-  case X86::VUNPCKHPSZrr:
-    Src2Name = getRegName(MI->getOperand(2).getReg());
-    // FALL THROUGH.
-  case X86::VUNPCKHPSZrm:
-    DecodeUNPCKHMask(MVT::v16f32, ShuffleMask);
-    Src1Name = getRegName(MI->getOperand(1).getReg());
-    DestName = getRegName(MI->getOperand(0).getReg());
-    break;
-  case X86::VPERMILPSri:
-    Src1Name = getRegName(MI->getOperand(1).getReg());
-    // FALL THROUGH.
-  case X86::VPERMILPSmi:
-    if(MI->getOperand(MI->getNumOperands()-1).isImm())
-      DecodePSHUFMask(MVT::v4f32,
-                      MI->getOperand(MI->getNumOperands()-1).getImm(),
-                      ShuffleMask);
-    DestName = getRegName(MI->getOperand(0).getReg());
-    break;
-  case X86::VPERMILPSYri:
-    Src1Name = getRegName(MI->getOperand(1).getReg());
-    // FALL THROUGH.
-  case X86::VPERMILPSYmi:
-    if(MI->getOperand(MI->getNumOperands()-1).isImm())
-      DecodePSHUFMask(MVT::v8f32,
-                      MI->getOperand(MI->getNumOperands()-1).getImm(),
-                      ShuffleMask);
-    DestName = getRegName(MI->getOperand(0).getReg());
-    break;
-  case X86::VPERMILPDri:
-    Src1Name = getRegName(MI->getOperand(1).getReg());
-    // FALL THROUGH.
-  case X86::VPERMILPDmi:
-    if(MI->getOperand(MI->getNumOperands()-1).isImm())
-      DecodePSHUFMask(MVT::v2f64,
-                      MI->getOperand(MI->getNumOperands()-1).getImm(),
-                      ShuffleMask);
-    DestName = getRegName(MI->getOperand(0).getReg());
-    break;
-  case X86::VPERMILPDYri:
-    Src1Name = getRegName(MI->getOperand(1).getReg());
-    // FALL THROUGH.
-  case X86::VPERMILPDYmi:
-    if(MI->getOperand(MI->getNumOperands()-1).isImm())
-      DecodePSHUFMask(MVT::v4f64,
-                      MI->getOperand(MI->getNumOperands()-1).getImm(),
-                      ShuffleMask);
-    DestName = getRegName(MI->getOperand(0).getReg());
-    break;
-  case X86::VPERM2F128rr:
-  case X86::VPERM2I128rr:
-    Src2Name = getRegName(MI->getOperand(2).getReg());
-    // FALL THROUGH.
-  case X86::VPERM2F128rm:
-  case X86::VPERM2I128rm:
-    // For instruction comments purpose, assume the 256-bit vector is v4i64.
-    if(MI->getOperand(MI->getNumOperands()-1).isImm())
-      DecodeVPERM2X128Mask(MVT::v4i64,
-                           MI->getOperand(MI->getNumOperands()-1).getImm(),
-                           ShuffleMask);
-    Src1Name = getRegName(MI->getOperand(1).getReg());
-    DestName = getRegName(MI->getOperand(0).getReg());
-    break;
-  case X86::VPERMQYri:
-  case X86::VPERMPDYri:
-    Src1Name = getRegName(MI->getOperand(1).getReg());
-    // FALL THROUGH.
-  case X86::VPERMQYmi:
-  case X86::VPERMPDYmi:
-    if(MI->getOperand(MI->getNumOperands()-1).isImm())
-      DecodeVPERMMask(MI->getOperand(MI->getNumOperands()-1).getImm(),
-                      ShuffleMask);
-    DestName = getRegName(MI->getOperand(0).getReg());
-    break;
-
-  case X86::MOVSDrr:
-  case X86::VMOVSDrr:
-    Src2Name = getRegName(MI->getOperand(2).getReg());
-    Src1Name = getRegName(MI->getOperand(1).getReg());
-    // FALL THROUGH.
-  case X86::MOVSDrm:
-  case X86::VMOVSDrm:
-    DecodeScalarMoveMask(MVT::v2f64, nullptr == Src2Name, ShuffleMask);
-    DestName = getRegName(MI->getOperand(0).getReg());
-    break;
-  case X86::MOVSSrr:
-  case X86::VMOVSSrr:
-    Src2Name = getRegName(MI->getOperand(2).getReg());
-    Src1Name = getRegName(MI->getOperand(1).getReg());
-    // FALL THROUGH.
-  case X86::MOVSSrm:
-  case X86::VMOVSSrm:
-    DecodeScalarMoveMask(MVT::v4f32, nullptr == Src2Name, ShuffleMask);
-    DestName = getRegName(MI->getOperand(0).getReg());
-    break;
-
-  case X86::MOVPQI2QIrr:
-  case X86::MOVZPQILo2PQIrr:
-  case X86::VMOVPQI2QIrr:
-  case X86::VMOVZPQILo2PQIrr:
-    Src1Name = getRegName(MI->getOperand(1).getReg());
-  // FALL THROUGH.
-  case X86::MOVQI2PQIrm:
-  case X86::MOVZQI2PQIrm:
-  case X86::MOVZPQILo2PQIrm:
-  case X86::VMOVQI2PQIrm:
-  case X86::VMOVZQI2PQIrm:
-  case X86::VMOVZPQILo2PQIrm:
-    DecodeZeroMoveLowMask(MVT::v2i64, ShuffleMask);
-    DestName = getRegName(MI->getOperand(0).getReg());
-    break;
-  case X86::MOVDI2PDIrm:
-  case X86::VMOVDI2PDIrm:
-    DecodeZeroMoveLowMask(MVT::v4i32, ShuffleMask);
-    DestName = getRegName(MI->getOperand(0).getReg());
-    break;
-
-  case X86::PMOVZXBWrr:
-  case X86::PMOVZXBDrr:
-  case X86::PMOVZXBQrr:
-  case X86::PMOVZXWDrr:
-  case X86::PMOVZXWQrr:
-  case X86::PMOVZXDQrr:
-  case X86::VPMOVZXBWrr:
-  case X86::VPMOVZXBDrr:
-  case X86::VPMOVZXBQrr:
-  case X86::VPMOVZXWDrr:
-  case X86::VPMOVZXWQrr:
-  case X86::VPMOVZXDQrr:
-  case X86::VPMOVZXBWYrr:
-  case X86::VPMOVZXBDYrr:
-  case X86::VPMOVZXBQYrr:
-  case X86::VPMOVZXWDYrr:
-  case X86::VPMOVZXWQYrr:
-  case X86::VPMOVZXDQYrr:
-    Src1Name = getRegName(MI->getOperand(1).getReg());
-  // FALL THROUGH.
-  case X86::PMOVZXBWrm:
-  case X86::PMOVZXBDrm:
-  case X86::PMOVZXBQrm:
-  case X86::PMOVZXWDrm:
-  case X86::PMOVZXWQrm:
-  case X86::PMOVZXDQrm:
-  case X86::VPMOVZXBWrm:
-  case X86::VPMOVZXBDrm:
-  case X86::VPMOVZXBQrm:
-  case X86::VPMOVZXWDrm:
-  case X86::VPMOVZXWQrm:
-  case X86::VPMOVZXDQrm:
-  case X86::VPMOVZXBWYrm:
-  case X86::VPMOVZXBDYrm:
-  case X86::VPMOVZXBQYrm:
-  case X86::VPMOVZXWDYrm:
-  case X86::VPMOVZXWQYrm:
-  case X86::VPMOVZXDQYrm: {
-    MVT SrcVT, DstVT;
-    getZeroExtensionTypes(MI, SrcVT, DstVT);
-    DecodeZeroExtendMask(SrcVT, DstVT, ShuffleMask);
-    DestName = getRegName(MI->getOperand(0).getReg());
-  } break;
-  }
-
-  // The only comments we decode are shuffles, so give up if we were unable to
-  // decode a shuffle mask.
-  if (ShuffleMask.empty())
-    return false;
-
-  if (!DestName) DestName = Src1Name;
-  OS << (DestName ? DestName : "mem") << " = ";
-
-  // If the two sources are the same, canonicalize the input elements to be
-  // from the first src so that we get larger element spans.
-  if (Src1Name == Src2Name) {
-    for (unsigned i = 0, e = ShuffleMask.size(); i != e; ++i) {
-      if ((int)ShuffleMask[i] >= 0 && // Not sentinel.
-          ShuffleMask[i] >= (int)e)        // From second mask.
-        ShuffleMask[i] -= e;
-    }
-  }
-
-  // The shuffle mask specifies which elements of the src1/src2 fill in the
-  // destination, with a few sentinel values.  Loop through and print them
-  // out.
-  for (unsigned i = 0, e = ShuffleMask.size(); i != e; ++i) {
-    if (i != 0)
-      OS << ',';
-    if (ShuffleMask[i] == SM_SentinelZero) {
-      OS << "zero";
-      continue;
-    }
-
-    // Otherwise, it must come from src1 or src2.  Print the span of elements
-    // that comes from this src.
-    bool isSrc1 = ShuffleMask[i] < (int)ShuffleMask.size();
-    const char *SrcName = isSrc1 ? Src1Name : Src2Name;
-    OS << (SrcName ? SrcName : "mem") << '[';
-    bool IsFirst = true;
-    while (i != e && (int)ShuffleMask[i] != SM_SentinelZero &&
-           (ShuffleMask[i] < (int)ShuffleMask.size()) == isSrc1) {
-      if (!IsFirst)
-        OS << ',';
-      else
-        IsFirst = false;
-      if (ShuffleMask[i] == SM_SentinelUndef)
-        OS << "u";
-      else
-        OS << ShuffleMask[i] % ShuffleMask.size();
-      ++i;
-    }
-    OS << ']';
-    --i;  // For loop increments element #.
-  }
-  //MI->print(OS, 0);
-  OS << "\n";
-
-  // We successfully added a comment to this instruction.
-  return true;
-}
+//===-- X86InstComments.cpp - Generate verbose-asm comments for instrs ----===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This defines functionality used to emit comments about X86 instructions to
+// an output stream for -fverbose-asm.
+//
+//===----------------------------------------------------------------------===//
+
+#include "X86InstComments.h"
+#include "MCTargetDesc/X86MCTargetDesc.h"
+#include "Utils/X86ShuffleDecode.h"
+#include "llvm/MC/MCInst.h"
+#include "llvm/CodeGen/MachineValueType.h"
+#include "llvm/Support/raw_ostream.h"
+
+using namespace llvm;
+
+/// \brief Extracts the src/dst types for a given zero extension instruction.
+/// \note While the number of elements in DstVT type correct, the
+/// number in the SrcVT type is expanded to fill the src xmm register and the
+/// upper elements may not be included in the dst xmm/ymm register.
+static void getZeroExtensionTypes(const MCInst *MI, MVT &SrcVT, MVT &DstVT) {
+  switch (MI->getOpcode()) {
+  default:
+    llvm_unreachable("Unknown zero extension instruction");
+  // i8 zero extension
+  case X86::PMOVZXBWrm:
+  case X86::PMOVZXBWrr:
+  case X86::VPMOVZXBWrm:
+  case X86::VPMOVZXBWrr:
+    SrcVT = MVT::v16i8;
+    DstVT = MVT::v8i16;
+    break;
+  case X86::VPMOVZXBWYrm:
+  case X86::VPMOVZXBWYrr:
+    SrcVT = MVT::v16i8;
+    DstVT = MVT::v16i16;
+    break;
+  case X86::PMOVZXBDrm:
+  case X86::PMOVZXBDrr:
+  case X86::VPMOVZXBDrm:
+  case X86::VPMOVZXBDrr:
+    SrcVT = MVT::v16i8;
+    DstVT = MVT::v4i32;
+    break;
+  case X86::VPMOVZXBDYrm:
+  case X86::VPMOVZXBDYrr:
+    SrcVT = MVT::v16i8;
+    DstVT = MVT::v8i32;
+    break;
+  case X86::PMOVZXBQrm:
+  case X86::PMOVZXBQrr:
+  case X86::VPMOVZXBQrm:
+  case X86::VPMOVZXBQrr:
+    SrcVT = MVT::v16i8;
+    DstVT = MVT::v2i64;
+    break;
+  case X86::VPMOVZXBQYrm:
+  case X86::VPMOVZXBQYrr:
+    SrcVT = MVT::v16i8;
+    DstVT = MVT::v4i64;
+    break;
+  // i16 zero extension
+  case X86::PMOVZXWDrm:
+  case X86::PMOVZXWDrr:
+  case X86::VPMOVZXWDrm:
+  case X86::VPMOVZXWDrr:
+    SrcVT = MVT::v8i16;
+    DstVT = MVT::v4i32;
+    break;
+  case X86::VPMOVZXWDYrm:
+  case X86::VPMOVZXWDYrr:
+    SrcVT = MVT::v8i16;
+    DstVT = MVT::v8i32;
+    break;
+  case X86::PMOVZXWQrm:
+  case X86::PMOVZXWQrr:
+  case X86::VPMOVZXWQrm:
+  case X86::VPMOVZXWQrr:
+    SrcVT = MVT::v8i16;
+    DstVT = MVT::v2i64;
+    break;
+  case X86::VPMOVZXWQYrm:
+  case X86::VPMOVZXWQYrr:
+    SrcVT = MVT::v8i16;
+    DstVT = MVT::v4i64;
+    break;
+  // i32 zero extension
+  case X86::PMOVZXDQrm:
+  case X86::PMOVZXDQrr:
+  case X86::VPMOVZXDQrm:
+  case X86::VPMOVZXDQrr:
+    SrcVT = MVT::v4i32;
+    DstVT = MVT::v2i64;
+    break;
+  case X86::VPMOVZXDQYrm:
+  case X86::VPMOVZXDQYrr:
+    SrcVT = MVT::v4i32;
+    DstVT = MVT::v4i64;
+    break;
+  }
+}
+
+//===----------------------------------------------------------------------===//
+// Top Level Entrypoint
+//===----------------------------------------------------------------------===//
+
+/// EmitAnyX86InstComments - This function decodes x86 instructions and prints
+/// newline terminated strings to the specified string if desired.  This
+/// information is shown in disassembly dumps when verbose assembly is enabled.
+bool llvm::EmitAnyX86InstComments(const MCInst *MI, raw_ostream &OS,
+                                  const char *(*getRegName)(unsigned)) {
+  // If this is a shuffle operation, the switch should fill in this state.
+  SmallVector<int, 8> ShuffleMask;
+  const char *DestName = nullptr, *Src1Name = nullptr, *Src2Name = nullptr;
+
+  switch (MI->getOpcode()) {
+  default:
+    // Not an instruction for which we can decode comments.
+    return false;
+
+  case X86::BLENDPDrri:
+  case X86::VBLENDPDrri:
+    Src2Name = getRegName(MI->getOperand(2).getReg());
+    // FALL THROUGH.
+  case X86::BLENDPDrmi:
+  case X86::VBLENDPDrmi:
+    if (MI->getOperand(MI->getNumOperands() - 1).isImm())
+      DecodeBLENDMask(MVT::v2f64,
+                      MI->getOperand(MI->getNumOperands() - 1).getImm(),
+                      ShuffleMask);
+    Src1Name = getRegName(MI->getOperand(1).getReg());
+    DestName = getRegName(MI->getOperand(0).getReg());
+    break;
+  case X86::VBLENDPDYrri:
+    Src2Name = getRegName(MI->getOperand(2).getReg());
+    // FALL THROUGH.
+  case X86::VBLENDPDYrmi:
+    if (MI->getOperand(MI->getNumOperands() - 1).isImm())
+      DecodeBLENDMask(MVT::v4f64,
+                      MI->getOperand(MI->getNumOperands() - 1).getImm(),
+                      ShuffleMask);
+    Src1Name = getRegName(MI->getOperand(1).getReg());
+    DestName = getRegName(MI->getOperand(0).getReg());
+    break;
+
+  case X86::BLENDPSrri:
+  case X86::VBLENDPSrri:
+    Src2Name = getRegName(MI->getOperand(2).getReg());
+    // FALL THROUGH.
+  case X86::BLENDPSrmi:
+  case X86::VBLENDPSrmi:
+    if (MI->getOperand(MI->getNumOperands() - 1).isImm())
+      DecodeBLENDMask(MVT::v4f32,
+                      MI->getOperand(MI->getNumOperands() - 1).getImm(),
+                      ShuffleMask);
+    Src1Name = getRegName(MI->getOperand(1).getReg());
+    DestName = getRegName(MI->getOperand(0).getReg());
+    break;
+  case X86::VBLENDPSYrri:
+    Src2Name = getRegName(MI->getOperand(2).getReg());
+    // FALL THROUGH.
+  case X86::VBLENDPSYrmi:
+    if (MI->getOperand(MI->getNumOperands() - 1).isImm())
+      DecodeBLENDMask(MVT::v8f32,
+                      MI->getOperand(MI->getNumOperands() - 1).getImm(),
+                      ShuffleMask);
+    Src1Name = getRegName(MI->getOperand(1).getReg());
+    DestName = getRegName(MI->getOperand(0).getReg());
+    break;
+
+  case X86::PBLENDWrri:
+  case X86::VPBLENDWrri:
+    Src2Name = getRegName(MI->getOperand(2).getReg());
+    // FALL THROUGH.
+  case X86::PBLENDWrmi:
+  case X86::VPBLENDWrmi:
+    if (MI->getOperand(MI->getNumOperands() - 1).isImm())
+      DecodeBLENDMask(MVT::v8i16,
+                      MI->getOperand(MI->getNumOperands() - 1).getImm(),
+                      ShuffleMask);
+    Src1Name = getRegName(MI->getOperand(1).getReg());
+    DestName = getRegName(MI->getOperand(0).getReg());
+    break;
+  case X86::VPBLENDWYrri:
+    Src2Name = getRegName(MI->getOperand(2).getReg());
+    // FALL THROUGH.
+  case X86::VPBLENDWYrmi:
+    if (MI->getOperand(MI->getNumOperands() - 1).isImm())
+      DecodeBLENDMask(MVT::v16i16,
+                      MI->getOperand(MI->getNumOperands() - 1).getImm(),
+                      ShuffleMask);
+    Src1Name = getRegName(MI->getOperand(1).getReg());
+    DestName = getRegName(MI->getOperand(0).getReg());
+    break;
+
+  case X86::VPBLENDDrri:
+    Src2Name = getRegName(MI->getOperand(2).getReg());
+    // FALL THROUGH.
+  case X86::VPBLENDDrmi:
+    if (MI->getOperand(MI->getNumOperands() - 1).isImm())
+      DecodeBLENDMask(MVT::v4i32,
+                      MI->getOperand(MI->getNumOperands() - 1).getImm(),
+                      ShuffleMask);
+    Src1Name = getRegName(MI->getOperand(1).getReg());
+    DestName = getRegName(MI->getOperand(0).getReg());
+    break;
+
+  case X86::VPBLENDDYrri:
+    Src2Name = getRegName(MI->getOperand(2).getReg());
+    // FALL THROUGH.
+  case X86::VPBLENDDYrmi:
+    if (MI->getOperand(MI->getNumOperands() - 1).isImm())
+      DecodeBLENDMask(MVT::v8i32,
+                      MI->getOperand(MI->getNumOperands() - 1).getImm(),
+                      ShuffleMask);
+    Src1Name = getRegName(MI->getOperand(1).getReg());
+    DestName = getRegName(MI->getOperand(0).getReg());
+    break;
+
+  case X86::INSERTPSrr:
+  case X86::VINSERTPSrr:
+    Src2Name = getRegName(MI->getOperand(2).getReg());
+    // FALL THROUGH.
+  case X86::INSERTPSrm:
+  case X86::VINSERTPSrm:
+    DestName = getRegName(MI->getOperand(0).getReg());
+    Src1Name = getRegName(MI->getOperand(1).getReg());
+    if (MI->getOperand(MI->getNumOperands() - 1).isImm())
+      DecodeINSERTPSMask(MI->getOperand(MI->getNumOperands() - 1).getImm(),
+                         ShuffleMask);
+    break;
+
+  case X86::MOVLHPSrr:
+  case X86::VMOVLHPSrr:
+    Src2Name = getRegName(MI->getOperand(2).getReg());
+    Src1Name = getRegName(MI->getOperand(1).getReg());
+    DestName = getRegName(MI->getOperand(0).getReg());
+    DecodeMOVLHPSMask(2, ShuffleMask);
+    break;
+
+  case X86::MOVHLPSrr:
+  case X86::VMOVHLPSrr:
+    Src2Name = getRegName(MI->getOperand(2).getReg());
+    Src1Name = getRegName(MI->getOperand(1).getReg());
+    DestName = getRegName(MI->getOperand(0).getReg());
+    DecodeMOVHLPSMask(2, ShuffleMask);
+    break;
+
+  case X86::MOVSLDUPrr:
+  case X86::VMOVSLDUPrr:
+    Src1Name = getRegName(MI->getOperand(1).getReg());
+    // FALL THROUGH.
+  case X86::MOVSLDUPrm:
+  case X86::VMOVSLDUPrm:
+    DestName = getRegName(MI->getOperand(0).getReg());
+    DecodeMOVSLDUPMask(MVT::v4f32, ShuffleMask);
+    break;
+
+  case X86::VMOVSHDUPYrr:
+    Src1Name = getRegName(MI->getOperand(1).getReg());
+    // FALL THROUGH.
+  case X86::VMOVSHDUPYrm:
+    DestName = getRegName(MI->getOperand(0).getReg());
+    DecodeMOVSHDUPMask(MVT::v8f32, ShuffleMask);
+    break;
+
+  case X86::VMOVSLDUPYrr:
+    Src1Name = getRegName(MI->getOperand(1).getReg());
+    // FALL THROUGH.
+  case X86::VMOVSLDUPYrm:
+    DestName = getRegName(MI->getOperand(0).getReg());
+    DecodeMOVSLDUPMask(MVT::v8f32, ShuffleMask);
+    break;
+
+  case X86::MOVSHDUPrr:
+  case X86::VMOVSHDUPrr:
+    Src1Name = getRegName(MI->getOperand(1).getReg());
+    // FALL THROUGH.
+  case X86::MOVSHDUPrm:
+  case X86::VMOVSHDUPrm:
+    DestName = getRegName(MI->getOperand(0).getReg());
+    DecodeMOVSHDUPMask(MVT::v4f32, ShuffleMask);
+    break;
+
+  case X86::VMOVDDUPYrr:
+    Src1Name = getRegName(MI->getOperand(1).getReg());
+    // FALL THROUGH.
+  case X86::VMOVDDUPYrm:
+    DestName = getRegName(MI->getOperand(0).getReg());
+    DecodeMOVDDUPMask(MVT::v4f64, ShuffleMask);
+    break;
+
+  case X86::MOVDDUPrr:
+  case X86::VMOVDDUPrr:
+    Src1Name = getRegName(MI->getOperand(1).getReg());
+    // FALL THROUGH.
+  case X86::MOVDDUPrm:
+  case X86::VMOVDDUPrm:
+    DestName = getRegName(MI->getOperand(0).getReg());
+    DecodeMOVDDUPMask(MVT::v2f64, ShuffleMask);
+    break;
+
+  case X86::PSLLDQri:
+  case X86::VPSLLDQri:
+    Src1Name = getRegName(MI->getOperand(1).getReg());
+    DestName = getRegName(MI->getOperand(0).getReg());
+    if (MI->getOperand(MI->getNumOperands() - 1).isImm())
+      DecodePSLLDQMask(MVT::v16i8,
+                       MI->getOperand(MI->getNumOperands() - 1).getImm(),
+                       ShuffleMask);
+    break;
+
+  case X86::VPSLLDQYri:
+    Src1Name = getRegName(MI->getOperand(1).getReg());
+    DestName = getRegName(MI->getOperand(0).getReg());
+    if (MI->getOperand(MI->getNumOperands() - 1).isImm())
+      DecodePSLLDQMask(MVT::v32i8,
+                       MI->getOperand(MI->getNumOperands() - 1).getImm(),
+                       ShuffleMask);
+    break;
+
+  case X86::PSRLDQri:
+  case X86::VPSRLDQri:
+    Src1Name = getRegName(MI->getOperand(1).getReg());
+    DestName = getRegName(MI->getOperand(0).getReg());
+    if (MI->getOperand(MI->getNumOperands() - 1).isImm())
+      DecodePSRLDQMask(MVT::v16i8,
+                       MI->getOperand(MI->getNumOperands() - 1).getImm(),
+                       ShuffleMask);
+    break;
+
+  case X86::VPSRLDQYri:
+    Src1Name = getRegName(MI->getOperand(1).getReg());
+    DestName = getRegName(MI->getOperand(0).getReg());
+    if (MI->getOperand(MI->getNumOperands() - 1).isImm())
+      DecodePSRLDQMask(MVT::v32i8,
+                       MI->getOperand(MI->getNumOperands() - 1).getImm(),
+                       ShuffleMask);
+    break;
+
+  case X86::PALIGNR128rr:
+  case X86::VPALIGNR128rr:
+    Src1Name = getRegName(MI->getOperand(2).getReg());
+    // FALL THROUGH.
+  case X86::PALIGNR128rm:
+  case X86::VPALIGNR128rm:
+    Src2Name = getRegName(MI->getOperand(1).getReg());
+    DestName = getRegName(MI->getOperand(0).getReg());
+    if (MI->getOperand(MI->getNumOperands() - 1).isImm())
+      DecodePALIGNRMask(MVT::v16i8,
+                        MI->getOperand(MI->getNumOperands() - 1).getImm(),
+                        ShuffleMask);
+    break;
+  case X86::VPALIGNR256rr:
+    Src1Name = getRegName(MI->getOperand(2).getReg());
+    // FALL THROUGH.
+  case X86::VPALIGNR256rm:
+    Src2Name = getRegName(MI->getOperand(1).getReg());
+    DestName = getRegName(MI->getOperand(0).getReg());
+    if (MI->getOperand(MI->getNumOperands() - 1).isImm())
+      DecodePALIGNRMask(MVT::v32i8,
+                        MI->getOperand(MI->getNumOperands() - 1).getImm(),
+                        ShuffleMask);
+    break;
+
+  case X86::PSHUFDri:
+  case X86::VPSHUFDri:
+    Src1Name = getRegName(MI->getOperand(1).getReg());
+    // FALL THROUGH.
+  case X86::PSHUFDmi:
+  case X86::VPSHUFDmi:
+    DestName = getRegName(MI->getOperand(0).getReg());
+    if (MI->getOperand(MI->getNumOperands() - 1).isImm())
+      DecodePSHUFMask(MVT::v4i32,
+                      MI->getOperand(MI->getNumOperands() - 1).getImm(),
+                      ShuffleMask);
+    break;
+  case X86::VPSHUFDYri:
+    Src1Name = getRegName(MI->getOperand(1).getReg());
+    // FALL THROUGH.
+  case X86::VPSHUFDYmi:
+    DestName = getRegName(MI->getOperand(0).getReg());
+    if (MI->getOperand(MI->getNumOperands() - 1).isImm())
+      DecodePSHUFMask(MVT::v8i32,
+                      MI->getOperand(MI->getNumOperands() - 1).getImm(),
+                      ShuffleMask);
+    break;
+
+  case X86::PSHUFHWri:
+  case X86::VPSHUFHWri:
+    Src1Name = getRegName(MI->getOperand(1).getReg());
+    // FALL THROUGH.
+  case X86::PSHUFHWmi:
+  case X86::VPSHUFHWmi:
+    DestName = getRegName(MI->getOperand(0).getReg());
+    if (MI->getOperand(MI->getNumOperands() - 1).isImm())
+      DecodePSHUFHWMask(MVT::v8i16,
+                        MI->getOperand(MI->getNumOperands() - 1).getImm(),
+                        ShuffleMask);
+    break;
+  case X86::VPSHUFHWYri:
+    Src1Name = getRegName(MI->getOperand(1).getReg());
+    // FALL THROUGH.
+  case X86::VPSHUFHWYmi:
+    DestName = getRegName(MI->getOperand(0).getReg());
+    if (MI->getOperand(MI->getNumOperands() - 1).isImm())
+      DecodePSHUFHWMask(MVT::v16i16,
+                        MI->getOperand(MI->getNumOperands() - 1).getImm(),
+                        ShuffleMask);
+    break;
+  case X86::PSHUFLWri:
+  case X86::VPSHUFLWri:
+    Src1Name = getRegName(MI->getOperand(1).getReg());
+    // FALL THROUGH.
+  case X86::PSHUFLWmi:
+  case X86::VPSHUFLWmi:
+    DestName = getRegName(MI->getOperand(0).getReg());
+    if (MI->getOperand(MI->getNumOperands() - 1).isImm())
+      DecodePSHUFLWMask(MVT::v8i16,
+                        MI->getOperand(MI->getNumOperands() - 1).getImm(),
+                        ShuffleMask);
+    break;
+  case X86::VPSHUFLWYri:
+    Src1Name = getRegName(MI->getOperand(1).getReg());
+    // FALL THROUGH.
+  case X86::VPSHUFLWYmi:
+    DestName = getRegName(MI->getOperand(0).getReg());
+    if (MI->getOperand(MI->getNumOperands() - 1).isImm())
+      DecodePSHUFLWMask(MVT::v16i16,
+                        MI->getOperand(MI->getNumOperands() - 1).getImm(),
+                        ShuffleMask);
+    break;
+
+  case X86::PUNPCKHBWrr:
+  case X86::VPUNPCKHBWrr:
+    Src2Name = getRegName(MI->getOperand(2).getReg());
+    // FALL THROUGH.
+  case X86::PUNPCKHBWrm:
+  case X86::VPUNPCKHBWrm:
+    Src1Name = getRegName(MI->getOperand(1).getReg());
+    DestName = getRegName(MI->getOperand(0).getReg());
+    DecodeUNPCKHMask(MVT::v16i8, ShuffleMask);
+    break;
+  case X86::VPUNPCKHBWYrr:
+    Src2Name = getRegName(MI->getOperand(2).getReg());
+    // FALL THROUGH.
+  case X86::VPUNPCKHBWYrm:
+    Src1Name = getRegName(MI->getOperand(1).getReg());
+    DestName = getRegName(MI->getOperand(0).getReg());
+    DecodeUNPCKHMask(MVT::v32i8, ShuffleMask);
+    break;
+  case X86::PUNPCKHWDrr:
+  case X86::VPUNPCKHWDrr:
+    Src2Name = getRegName(MI->getOperand(2).getReg());
+    // FALL THROUGH.
+  case X86::PUNPCKHWDrm:
+  case X86::VPUNPCKHWDrm:
+    Src1Name = getRegName(MI->getOperand(1).getReg());
+    DestName = getRegName(MI->getOperand(0).getReg());
+    DecodeUNPCKHMask(MVT::v8i16, ShuffleMask);
+    break;
+  case X86::VPUNPCKHWDYrr:
+    Src2Name = getRegName(MI->getOperand(2).getReg());
+    // FALL THROUGH.
+  case X86::VPUNPCKHWDYrm:
+    Src1Name = getRegName(MI->getOperand(1).getReg());
+    DestName = getRegName(MI->getOperand(0).getReg());
+    DecodeUNPCKHMask(MVT::v16i16, ShuffleMask);
+    break;
+  case X86::PUNPCKHDQrr:
+  case X86::VPUNPCKHDQrr:
+    Src2Name = getRegName(MI->getOperand(2).getReg());
+    // FALL THROUGH.
+  case X86::PUNPCKHDQrm:
+  case X86::VPUNPCKHDQrm:
+    Src1Name = getRegName(MI->getOperand(1).getReg());
+    DestName = getRegName(MI->getOperand(0).getReg());
+    DecodeUNPCKHMask(MVT::v4i32, ShuffleMask);
+    break;
+  case X86::VPUNPCKHDQYrr:
+    Src2Name = getRegName(MI->getOperand(2).getReg());
+    // FALL THROUGH.
+  case X86::VPUNPCKHDQYrm:
+    Src1Name = getRegName(MI->getOperand(1).getReg());
+    DestName = getRegName(MI->getOperand(0).getReg());
+    DecodeUNPCKHMask(MVT::v8i32, ShuffleMask);
+    break;
+  case X86::VPUNPCKHDQZrr:
+    Src2Name = getRegName(MI->getOperand(2).getReg());
+    // FALL THROUGH.
+  case X86::VPUNPCKHDQZrm:
+    Src1Name = getRegName(MI->getOperand(1).getReg());
+    DestName = getRegName(MI->getOperand(0).getReg());
+    DecodeUNPCKHMask(MVT::v16i32, ShuffleMask);
+    break;
+  case X86::PUNPCKHQDQrr:
+  case X86::VPUNPCKHQDQrr:
+    Src2Name = getRegName(MI->getOperand(2).getReg());
+    // FALL THROUGH.
+  case X86::PUNPCKHQDQrm:
+  case X86::VPUNPCKHQDQrm:
+    Src1Name = getRegName(MI->getOperand(1).getReg());
+    DestName = getRegName(MI->getOperand(0).getReg());
+    DecodeUNPCKHMask(MVT::v2i64, ShuffleMask);
+    break;
+  case X86::VPUNPCKHQDQYrr:
+    Src2Name = getRegName(MI->getOperand(2).getReg());
+    // FALL THROUGH.
+  case X86::VPUNPCKHQDQYrm:
+    Src1Name = getRegName(MI->getOperand(1).getReg());
+    DestName = getRegName(MI->getOperand(0).getReg());
+    DecodeUNPCKHMask(MVT::v4i64, ShuffleMask);
+    break;
+  case X86::VPUNPCKHQDQZrr:
+    Src2Name = getRegName(MI->getOperand(2).getReg());
+    // FALL THROUGH.
+  case X86::VPUNPCKHQDQZrm:
+    Src1Name = getRegName(MI->getOperand(1).getReg());
+    DestName = getRegName(MI->getOperand(0).getReg());
+    DecodeUNPCKHMask(MVT::v8i64, ShuffleMask);
+    break;
+
+  case X86::PUNPCKLBWrr:
+  case X86::VPUNPCKLBWrr:
+    Src2Name = getRegName(MI->getOperand(2).getReg());
+    // FALL THROUGH.
+  case X86::PUNPCKLBWrm:
+  case X86::VPUNPCKLBWrm:
+    Src1Name = getRegName(MI->getOperand(1).getReg());
+    DestName = getRegName(MI->getOperand(0).getReg());
+    DecodeUNPCKLMask(MVT::v16i8, ShuffleMask);
+    break;
+  case X86::VPUNPCKLBWYrr:
+    Src2Name = getRegName(MI->getOperand(2).getReg());
+    // FALL THROUGH.
+  case X86::VPUNPCKLBWYrm:
+    Src1Name = getRegName(MI->getOperand(1).getReg());
+    DestName = getRegName(MI->getOperand(0).getReg());
+    DecodeUNPCKLMask(MVT::v32i8, ShuffleMask);
+    break;
+  case X86::PUNPCKLWDrr:
+  case X86::VPUNPCKLWDrr:
+    Src2Name = getRegName(MI->getOperand(2).getReg());
+    // FALL THROUGH.
+  case X86::PUNPCKLWDrm:
+  case X86::VPUNPCKLWDrm:
+    Src1Name = getRegName(MI->getOperand(1).getReg());
+    DestName = getRegName(MI->getOperand(0).getReg());
+    DecodeUNPCKLMask(MVT::v8i16, ShuffleMask);
+    break;
+  case X86::VPUNPCKLWDYrr:
+    Src2Name = getRegName(MI->getOperand(2).getReg());
+    // FALL THROUGH.
+  case X86::VPUNPCKLWDYrm:
+    Src1Name = getRegName(MI->getOperand(1).getReg());
+    DestName = getRegName(MI->getOperand(0).getReg());
+    DecodeUNPCKLMask(MVT::v16i16, ShuffleMask);
+    break;
+  case X86::PUNPCKLDQrr:
+  case X86::VPUNPCKLDQrr:
+    Src2Name = getRegName(MI->getOperand(2).getReg());
+    // FALL THROUGH.
+  case X86::PUNPCKLDQrm:
+  case X86::VPUNPCKLDQrm:
+    Src1Name = getRegName(MI->getOperand(1).getReg());
+    DestName = getRegName(MI->getOperand(0).getReg());
+    DecodeUNPCKLMask(MVT::v4i32, ShuffleMask);
+    break;
+  case X86::VPUNPCKLDQYrr:
+    Src2Name = getRegName(MI->getOperand(2).getReg());
+    // FALL THROUGH.
+  case X86::VPUNPCKLDQYrm:
+    Src1Name = getRegName(MI->getOperand(1).getReg());
+    DestName = getRegName(MI->getOperand(0).getReg());
+    DecodeUNPCKLMask(MVT::v8i32, ShuffleMask);
+    break;
+  case X86::VPUNPCKLDQZrr:
+    Src2Name = getRegName(MI->getOperand(2).getReg());
+    // FALL THROUGH.
+  case X86::VPUNPCKLDQZrm:
+    Src1Name = getRegName(MI->getOperand(1).getReg());
+    DestName = getRegName(MI->getOperand(0).getReg());
+    DecodeUNPCKLMask(MVT::v16i32, ShuffleMask);
+    break;
+  case X86::PUNPCKLQDQrr:
+  case X86::VPUNPCKLQDQrr:
+    Src2Name = getRegName(MI->getOperand(2).getReg());
+    // FALL THROUGH.
+  case X86::PUNPCKLQDQrm:
+  case X86::VPUNPCKLQDQrm:
+    Src1Name = getRegName(MI->getOperand(1).getReg());
+    DestName = getRegName(MI->getOperand(0).getReg());
+    DecodeUNPCKLMask(MVT::v2i64, ShuffleMask);
+    break;
+  case X86::VPUNPCKLQDQYrr:
+    Src2Name = getRegName(MI->getOperand(2).getReg());
+    // FALL THROUGH.
+  case X86::VPUNPCKLQDQYrm:
+    Src1Name = getRegName(MI->getOperand(1).getReg());
+    DestName = getRegName(MI->getOperand(0).getReg());
+    DecodeUNPCKLMask(MVT::v4i64, ShuffleMask);
+    break;
+  case X86::VPUNPCKLQDQZrr:
+    Src2Name = getRegName(MI->getOperand(2).getReg());
+    // FALL THROUGH.
+  case X86::VPUNPCKLQDQZrm:
+    Src1Name = getRegName(MI->getOperand(1).getReg());
+    DestName = getRegName(MI->getOperand(0).getReg());
+    DecodeUNPCKLMask(MVT::v8i64, ShuffleMask);
+    break;
+
+  case X86::SHUFPDrri:
+  case X86::VSHUFPDrri:
+    Src2Name = getRegName(MI->getOperand(2).getReg());
+    // FALL THROUGH.
+  case X86::SHUFPDrmi:
+  case X86::VSHUFPDrmi:
+    if (MI->getOperand(MI->getNumOperands() - 1).isImm())
+      DecodeSHUFPMask(MVT::v2f64,
+                      MI->getOperand(MI->getNumOperands() - 1).getImm(),
+                      ShuffleMask);
+    Src1Name = getRegName(MI->getOperand(1).getReg());
+    DestName = getRegName(MI->getOperand(0).getReg());
+    break;
+  case X86::VSHUFPDYrri:
+    Src2Name = getRegName(MI->getOperand(2).getReg());
+    // FALL THROUGH.
+  case X86::VSHUFPDYrmi:
+    if (MI->getOperand(MI->getNumOperands() - 1).isImm())
+      DecodeSHUFPMask(MVT::v4f64,
+                      MI->getOperand(MI->getNumOperands() - 1).getImm(),
+                      ShuffleMask);
+    Src1Name = getRegName(MI->getOperand(1).getReg());
+    DestName = getRegName(MI->getOperand(0).getReg());
+    break;
+
+  case X86::SHUFPSrri:
+  case X86::VSHUFPSrri:
+    Src2Name = getRegName(MI->getOperand(2).getReg());
+    // FALL THROUGH.
+  case X86::SHUFPSrmi:
+  case X86::VSHUFPSrmi:
+    if (MI->getOperand(MI->getNumOperands() - 1).isImm())
+      DecodeSHUFPMask(MVT::v4f32,
+                      MI->getOperand(MI->getNumOperands() - 1).getImm(),
+                      ShuffleMask);
+    Src1Name = getRegName(MI->getOperand(1).getReg());
+    DestName = getRegName(MI->getOperand(0).getReg());
+    break;
+  case X86::VSHUFPSYrri:
+    Src2Name = getRegName(MI->getOperand(2).getReg());
+    // FALL THROUGH.
+  case X86::VSHUFPSYrmi:
+    if (MI->getOperand(MI->getNumOperands() - 1).isImm())
+      DecodeSHUFPMask(MVT::v8f32,
+                      MI->getOperand(MI->getNumOperands() - 1).getImm(),
+                      ShuffleMask);
+    Src1Name = getRegName(MI->getOperand(1).getReg());
+    DestName = getRegName(MI->getOperand(0).getReg());
+    break;
+
+  case X86::UNPCKLPDrr:
+  case X86::VUNPCKLPDrr:
+    Src2Name = getRegName(MI->getOperand(2).getReg());
+    // FALL THROUGH.
+  case X86::UNPCKLPDrm:
+  case X86::VUNPCKLPDrm:
+    DecodeUNPCKLMask(MVT::v2f64, ShuffleMask);
+    Src1Name = getRegName(MI->getOperand(1).getReg());
+    DestName = getRegName(MI->getOperand(0).getReg());
+    break;
+  case X86::VUNPCKLPDYrr:
+    Src2Name = getRegName(MI->getOperand(2).getReg());
+    // FALL THROUGH.
+  case X86::VUNPCKLPDYrm:
+    DecodeUNPCKLMask(MVT::v4f64, ShuffleMask);
+    Src1Name = getRegName(MI->getOperand(1).getReg());
+    DestName = getRegName(MI->getOperand(0).getReg());
+    break;
+  case X86::VUNPCKLPDZrr:
+    Src2Name = getRegName(MI->getOperand(2).getReg());
+    // FALL THROUGH.
+  case X86::VUNPCKLPDZrm:
+    DecodeUNPCKLMask(MVT::v8f64, ShuffleMask);
+    Src1Name = getRegName(MI->getOperand(1).getReg());
+    DestName = getRegName(MI->getOperand(0).getReg());
+    break;
+  case X86::UNPCKLPSrr:
+  case X86::VUNPCKLPSrr:
+    Src2Name = getRegName(MI->getOperand(2).getReg());
+    // FALL THROUGH.
+  case X86::UNPCKLPSrm:
+  case X86::VUNPCKLPSrm:
+    DecodeUNPCKLMask(MVT::v4f32, ShuffleMask);
+    Src1Name = getRegName(MI->getOperand(1).getReg());
+    DestName = getRegName(MI->getOperand(0).getReg());
+    break;
+  case X86::VUNPCKLPSYrr:
+    Src2Name = getRegName(MI->getOperand(2).getReg());
+    // FALL THROUGH.
+  case X86::VUNPCKLPSYrm:
+    DecodeUNPCKLMask(MVT::v8f32, ShuffleMask);
+    Src1Name = getRegName(MI->getOperand(1).getReg());
+    DestName = getRegName(MI->getOperand(0).getReg());
+    break;
+  case X86::VUNPCKLPSZrr:
+    Src2Name = getRegName(MI->getOperand(2).getReg());
+    // FALL THROUGH.
+  case X86::VUNPCKLPSZrm:
+    DecodeUNPCKLMask(MVT::v16f32, ShuffleMask);
+    Src1Name = getRegName(MI->getOperand(1).getReg());
+    DestName = getRegName(MI->getOperand(0).getReg());
+    break;
+  case X86::UNPCKHPDrr:
+  case X86::VUNPCKHPDrr:
+    Src2Name = getRegName(MI->getOperand(2).getReg());
+    // FALL THROUGH.
+  case X86::UNPCKHPDrm:
+  case X86::VUNPCKHPDrm:
+    DecodeUNPCKHMask(MVT::v2f64, ShuffleMask);
+    Src1Name = getRegName(MI->getOperand(1).getReg());
+    DestName = getRegName(MI->getOperand(0).getReg());
+    break;
+  case X86::VUNPCKHPDYrr:
+    Src2Name = getRegName(MI->getOperand(2).getReg());
+    // FALL THROUGH.
+  case X86::VUNPCKHPDYrm:
+    DecodeUNPCKHMask(MVT::v4f64, ShuffleMask);
+    Src1Name = getRegName(MI->getOperand(1).getReg());
+    DestName = getRegName(MI->getOperand(0).getReg());
+    break;
+  case X86::VUNPCKHPDZrr:
+    Src2Name = getRegName(MI->getOperand(2).getReg());
+    // FALL THROUGH.
+  case X86::VUNPCKHPDZrm:
+    DecodeUNPCKHMask(MVT::v8f64, ShuffleMask);
+    Src1Name = getRegName(MI->getOperand(1).getReg());
+    DestName = getRegName(MI->getOperand(0).getReg());
+    break;
+  case X86::UNPCKHPSrr:
+  case X86::VUNPCKHPSrr:
+    Src2Name = getRegName(MI->getOperand(2).getReg());
+    // FALL THROUGH.
+  case X86::UNPCKHPSrm:
+  case X86::VUNPCKHPSrm:
+    DecodeUNPCKHMask(MVT::v4f32, ShuffleMask);
+    Src1Name = getRegName(MI->getOperand(1).getReg());
+    DestName = getRegName(MI->getOperand(0).getReg());
+    break;
+  case X86::VUNPCKHPSYrr:
+    Src2Name = getRegName(MI->getOperand(2).getReg());
+    // FALL THROUGH.
+  case X86::VUNPCKHPSYrm:
+    DecodeUNPCKHMask(MVT::v8f32, ShuffleMask);
+    Src1Name = getRegName(MI->getOperand(1).getReg());
+    DestName = getRegName(MI->getOperand(0).getReg());
+    break;
+  case X86::VUNPCKHPSZrr:
+    Src2Name = getRegName(MI->getOperand(2).getReg());
+    // FALL THROUGH.
+  case X86::VUNPCKHPSZrm:
+    DecodeUNPCKHMask(MVT::v16f32, ShuffleMask);
+    Src1Name = getRegName(MI->getOperand(1).getReg());
+    DestName = getRegName(MI->getOperand(0).getReg());
+    break;
+  case X86::VPERMILPSri:
+    Src1Name = getRegName(MI->getOperand(1).getReg());
+    // FALL THROUGH.
+  case X86::VPERMILPSmi:
+    if (MI->getOperand(MI->getNumOperands() - 1).isImm())
+      DecodePSHUFMask(MVT::v4f32,
+                      MI->getOperand(MI->getNumOperands() - 1).getImm(),
+                      ShuffleMask);
+    DestName = getRegName(MI->getOperand(0).getReg());
+    break;
+  case X86::VPERMILPSYri:
+    Src1Name = getRegName(MI->getOperand(1).getReg());
+    // FALL THROUGH.
+  case X86::VPERMILPSYmi:
+    if (MI->getOperand(MI->getNumOperands() - 1).isImm())
+      DecodePSHUFMask(MVT::v8f32,
+                      MI->getOperand(MI->getNumOperands() - 1).getImm(),
+                      ShuffleMask);
+    DestName = getRegName(MI->getOperand(0).getReg());
+    break;
+  case X86::VPERMILPDri:
+    Src1Name = getRegName(MI->getOperand(1).getReg());
+    // FALL THROUGH.
+  case X86::VPERMILPDmi:
+    if (MI->getOperand(MI->getNumOperands() - 1).isImm())
+      DecodePSHUFMask(MVT::v2f64,
+                      MI->getOperand(MI->getNumOperands() - 1).getImm(),
+                      ShuffleMask);
+    DestName = getRegName(MI->getOperand(0).getReg());
+    break;
+  case X86::VPERMILPDYri:
+    Src1Name = getRegName(MI->getOperand(1).getReg());
+    // FALL THROUGH.
+  case X86::VPERMILPDYmi:
+    if (MI->getOperand(MI->getNumOperands() - 1).isImm())
+      DecodePSHUFMask(MVT::v4f64,
+                      MI->getOperand(MI->getNumOperands() - 1).getImm(),
+                      ShuffleMask);
+    DestName = getRegName(MI->getOperand(0).getReg());
+    break;
+  case X86::VPERM2F128rr:
+  case X86::VPERM2I128rr:
+    Src2Name = getRegName(MI->getOperand(2).getReg());
+    // FALL THROUGH.
+  case X86::VPERM2F128rm:
+  case X86::VPERM2I128rm:
+    // For instruction comments purpose, assume the 256-bit vector is v4i64.
+    if (MI->getOperand(MI->getNumOperands() - 1).isImm())
+      DecodeVPERM2X128Mask(MVT::v4i64,
+                           MI->getOperand(MI->getNumOperands() - 1).getImm(),
+                           ShuffleMask);
+    Src1Name = getRegName(MI->getOperand(1).getReg());
+    DestName = getRegName(MI->getOperand(0).getReg());
+    break;
+  case X86::VPERMQYri:
+  case X86::VPERMPDYri:
+    Src1Name = getRegName(MI->getOperand(1).getReg());
+    // FALL THROUGH.
+  case X86::VPERMQYmi:
+  case X86::VPERMPDYmi:
+    if (MI->getOperand(MI->getNumOperands() - 1).isImm())
+      DecodeVPERMMask(MI->getOperand(MI->getNumOperands() - 1).getImm(),
+                      ShuffleMask);
+    DestName = getRegName(MI->getOperand(0).getReg());
+    break;
+
+  case X86::MOVSDrr:
+  case X86::VMOVSDrr:
+    Src2Name = getRegName(MI->getOperand(2).getReg());
+    Src1Name = getRegName(MI->getOperand(1).getReg());
+    // FALL THROUGH.
+  case X86::MOVSDrm:
+  case X86::VMOVSDrm:
+    DecodeScalarMoveMask(MVT::v2f64, nullptr == Src2Name, ShuffleMask);
+    DestName = getRegName(MI->getOperand(0).getReg());
+    break;
+  case X86::MOVSSrr:
+  case X86::VMOVSSrr:
+    Src2Name = getRegName(MI->getOperand(2).getReg());
+    Src1Name = getRegName(MI->getOperand(1).getReg());
+    // FALL THROUGH.
+  case X86::MOVSSrm:
+  case X86::VMOVSSrm:
+    DecodeScalarMoveMask(MVT::v4f32, nullptr == Src2Name, ShuffleMask);
+    DestName = getRegName(MI->getOperand(0).getReg());
+    break;
+
+  case X86::MOVPQI2QIrr:
+  case X86::MOVZPQILo2PQIrr:
+  case X86::VMOVPQI2QIrr:
+  case X86::VMOVZPQILo2PQIrr:
+    Src1Name = getRegName(MI->getOperand(1).getReg());
+  // FALL THROUGH.
+  case X86::MOVQI2PQIrm:
+  case X86::MOVZQI2PQIrm:
+  case X86::MOVZPQILo2PQIrm:
+  case X86::VMOVQI2PQIrm:
+  case X86::VMOVZQI2PQIrm:
+  case X86::VMOVZPQILo2PQIrm:
+    DecodeZeroMoveLowMask(MVT::v2i64, ShuffleMask);
+    DestName = getRegName(MI->getOperand(0).getReg());
+    break;
+  case X86::MOVDI2PDIrm:
+  case X86::VMOVDI2PDIrm:
+    DecodeZeroMoveLowMask(MVT::v4i32, ShuffleMask);
+    DestName = getRegName(MI->getOperand(0).getReg());
+    break;
+
+  case X86::EXTRQI:
+    if (MI->getOperand(2).isImm() &&
+        MI->getOperand(3).isImm())
+      DecodeEXTRQIMask(MI->getOperand(2).getImm(),
+                       MI->getOperand(3).getImm(),
+                       ShuffleMask);
+
+    DestName = getRegName(MI->getOperand(0).getReg());
+    Src1Name = getRegName(MI->getOperand(1).getReg());
+    break;
+
+  case X86::INSERTQI:
+    if (MI->getOperand(3).isImm() &&
+        MI->getOperand(4).isImm())
+      DecodeINSERTQIMask(MI->getOperand(3).getImm(),
+                         MI->getOperand(4).getImm(),
+                         ShuffleMask);
+
+    DestName = getRegName(MI->getOperand(0).getReg());
+    Src1Name = getRegName(MI->getOperand(1).getReg());
+    Src2Name = getRegName(MI->getOperand(2).getReg());
+    break;
+
+  case X86::PMOVZXBWrr:
+  case X86::PMOVZXBDrr:
+  case X86::PMOVZXBQrr:
+  case X86::PMOVZXWDrr:
+  case X86::PMOVZXWQrr:
+  case X86::PMOVZXDQrr:
+  case X86::VPMOVZXBWrr:
+  case X86::VPMOVZXBDrr:
+  case X86::VPMOVZXBQrr:
+  case X86::VPMOVZXWDrr:
+  case X86::VPMOVZXWQrr:
+  case X86::VPMOVZXDQrr:
+  case X86::VPMOVZXBWYrr:
+  case X86::VPMOVZXBDYrr:
+  case X86::VPMOVZXBQYrr:
+  case X86::VPMOVZXWDYrr:
+  case X86::VPMOVZXWQYrr:
+  case X86::VPMOVZXDQYrr:
+    Src1Name = getRegName(MI->getOperand(1).getReg());
+  // FALL THROUGH.
+  case X86::PMOVZXBWrm:
+  case X86::PMOVZXBDrm:
+  case X86::PMOVZXBQrm:
+  case X86::PMOVZXWDrm:
+  case X86::PMOVZXWQrm:
+  case X86::PMOVZXDQrm:
+  case X86::VPMOVZXBWrm:
+  case X86::VPMOVZXBDrm:
+  case X86::VPMOVZXBQrm:
+  case X86::VPMOVZXWDrm:
+  case X86::VPMOVZXWQrm:
+  case X86::VPMOVZXDQrm:
+  case X86::VPMOVZXBWYrm:
+  case X86::VPMOVZXBDYrm:
+  case X86::VPMOVZXBQYrm:
+  case X86::VPMOVZXWDYrm:
+  case X86::VPMOVZXWQYrm:
+  case X86::VPMOVZXDQYrm: {
+    MVT SrcVT, DstVT;
+    getZeroExtensionTypes(MI, SrcVT, DstVT);
+    DecodeZeroExtendMask(SrcVT, DstVT, ShuffleMask);
+    DestName = getRegName(MI->getOperand(0).getReg());
+  } break;
+  }
+
+  // The only comments we decode are shuffles, so give up if we were unable to
+  // decode a shuffle mask.
+  if (ShuffleMask.empty())
+    return false;
+
+  if (!DestName) DestName = Src1Name;
+  OS << (DestName ? DestName : "mem") << " = ";
+
+  // If the two sources are the same, canonicalize the input elements to be
+  // from the first src so that we get larger element spans.
+  if (Src1Name == Src2Name) {
+    for (unsigned i = 0, e = ShuffleMask.size(); i != e; ++i) {
+      if ((int)ShuffleMask[i] >= 0 && // Not sentinel.
+          ShuffleMask[i] >= (int)e)   // From second mask.
+        ShuffleMask[i] -= e;
+    }
+  }
+
+  // The shuffle mask specifies which elements of the src1/src2 fill in the
+  // destination, with a few sentinel values.  Loop through and print them
+  // out.
+  for (unsigned i = 0, e = ShuffleMask.size(); i != e; ++i) {
+    if (i != 0)
+      OS << ',';
+    if (ShuffleMask[i] == SM_SentinelZero) {
+      OS << "zero";
+      continue;
+    }
+
+    // Otherwise, it must come from src1 or src2.  Print the span of elements
+    // that comes from this src.
+    bool isSrc1 = ShuffleMask[i] < (int)ShuffleMask.size();
+    const char *SrcName = isSrc1 ? Src1Name : Src2Name;
+    OS << (SrcName ? SrcName : "mem") << '[';
+    bool IsFirst = true;
+    while (i != e && (int)ShuffleMask[i] != SM_SentinelZero &&
+           (ShuffleMask[i] < (int)ShuffleMask.size()) == isSrc1) {
+      if (!IsFirst)
+        OS << ',';
+      else
+        IsFirst = false;
+      if (ShuffleMask[i] == SM_SentinelUndef)
+        OS << "u";
+      else
+        OS << ShuffleMask[i] % ShuffleMask.size();
+      ++i;
+    }
+    OS << ']';
+    --i; // For loop increments element #.
+  }
+  //MI->print(OS, 0);
+  OS << "\n";
+
+  // We successfully added a comment to this instruction.
+  return true;
+}
diff --git a/lib/Target/X86/InstPrinter/X86IntelInstPrinter.cpp b/lib/Target/X86/InstPrinter/X86IntelInstPrinter.cpp
index 4d92dafa938..879378fc7a9 100644
--- a/lib/Target/X86/InstPrinter/X86IntelInstPrinter.cpp
+++ b/lib/Target/X86/InstPrinter/X86IntelInstPrinter.cpp
@@ -131,12 +131,12 @@ void X86IntelInstPrinter::printPCRelImm(const MCInst *MI, unsigned OpNo,
     // that address in hex.
     const MCConstantExpr *BranchTarget = dyn_cast<MCConstantExpr>(Op.getExpr());
     int64_t Address;
-    if (BranchTarget && BranchTarget->EvaluateAsAbsolute(Address)) {
+    if (BranchTarget && BranchTarget->evaluateAsAbsolute(Address)) {
       O << formatHex((uint64_t)Address);
     }
     else {
       // Otherwise, just print the expression.
-      O << *Op.getExpr();
+      Op.getExpr()->print(O, &MAI);
     }
   }
 }
@@ -150,7 +150,7 @@ void X86IntelInstPrinter::printOperand(const MCInst *MI, unsigned OpNo,
     O << formatImm((int64_t)Op.getImm());
   } else {
     assert(Op.isExpr() && "unknown operand kind in printOperand");
-    O << *Op.getExpr();
+    Op.getExpr()->print(O, &MAI);
   }
 }
 
@@ -187,7 +187,7 @@ void X86IntelInstPrinter::printMemReference(const MCInst *MI, unsigned Op,
   if (!DispSpec.isImm()) {
     if (NeedPlus) O << " + ";
     assert(DispSpec.isExpr() && "non-immediate displacement for LEA?");
-    O << *DispSpec.getExpr();
+    DispSpec.getExpr()->print(O, &MAI);
   } else {
     int64_t DispVal = DispSpec.getImm();
     if (DispVal || (!IndexReg.getReg() && !BaseReg.getReg())) {
@@ -245,7 +245,7 @@ void X86IntelInstPrinter::printMemOffset(const MCInst *MI, unsigned Op,
     O << formatImm(DispSpec.getImm());
   } else {
     assert(DispSpec.isExpr() && "non-immediate displacement?");
-    O << *DispSpec.getExpr();
+    DispSpec.getExpr()->print(O, &MAI);
   }
 
   O << ']';
diff --git a/lib/Target/X86/MCTargetDesc/CMakeLists.txt b/lib/Target/X86/MCTargetDesc/CMakeLists.txt
index d4a872ae120..129c28d804e 100644
--- a/lib/Target/X86/MCTargetDesc/CMakeLists.txt
+++ b/lib/Target/X86/MCTargetDesc/CMakeLists.txt
@@ -3,8 +3,6 @@ add_llvm_library(LLVMX86Desc
   X86MCTargetDesc.cpp
   X86MCAsmInfo.cpp
   X86MCCodeEmitter.cpp
-  X86MCNaCl.cpp # LOCALMOD
-  X86MCNaClExpander.cpp
   X86MachObjectWriter.cpp
   X86ELFObjectWriter.cpp
   X86WinCOFFStreamer.cpp
diff --git a/lib/Target/X86/MCTargetDesc/X86AsmBackend.cpp b/lib/Target/X86/MCTargetDesc/X86AsmBackend.cpp
index 2be7f3ab580..629802f5dc5 100644
--- a/lib/Target/X86/MCTargetDesc/X86AsmBackend.cpp
+++ b/lib/Target/X86/MCTargetDesc/X86AsmBackend.cpp
@@ -9,14 +9,15 @@
 
 #include "MCTargetDesc/X86BaseInfo.h"
 #include "MCTargetDesc/X86FixupKinds.h"
-#include "MCTargetDesc/X86MCNaCl.h" // @LOCALMOD
 #include "llvm/ADT/StringSwitch.h"
 #include "llvm/MC/MCAsmBackend.h"
 #include "llvm/MC/MCELFObjectWriter.h"
 #include "llvm/MC/MCExpr.h"
 #include "llvm/MC/MCFixupKindInfo.h"
+#include "llvm/MC/MCInst.h"
 #include "llvm/MC/MCMachObjectWriter.h"
 #include "llvm/MC/MCObjectWriter.h"
+#include "llvm/MC/MCRegisterInfo.h"
 #include "llvm/MC/MCSectionCOFF.h"
 #include "llvm/MC/MCSectionELF.h"
 #include "llvm/MC/MCSectionMachO.h"
@@ -28,13 +29,6 @@
 #include "llvm/Support/raw_ostream.h"
 using namespace llvm;
 
-// Option to allow disabling arithmetic relaxation to workaround PR9807, which
-// is useful when running bitwise comparison experiments on Darwin. We should be
-// able to remove this once PR9807 is resolved.
-static cl::opt<bool>
-MCDisableArithRelaxation("mc-x86-disable-arith-relaxation",
-         cl::desc("Disable relaxation of arithmetic instruction for X86"));
-
 static unsigned getFixupKindLog2Size(unsigned Kind) {
   switch (Kind) {
   default:
@@ -113,19 +107,12 @@ class X86AsmBackend : public MCAsmBackend {
     assert(Fixup.getOffset() + Size <= DataSize &&
            "Invalid fixup offset!");
 
-    // @LOCALMOD-BEGIN
-    // This check breaks negative addends on x86-32.  It makes x86-32
-    // behaviour inconsistent with x86-64 and ARM.
-    // See: https://code.google.com/p/nativeclient/issues/detail?id=3548
-#if 0
     // Check that uppper bits are either all zeros or all ones.
     // Specifically ignore overflow/underflow as long as the leakage is
     // limited to the lower bits. This is to remain compatible with
     // other assemblers.
     assert(isIntN(Size * 8 + 1, Value) &&
            "Value does not fit in the Fixup field");
-#endif
-    // @LOCALMOD-END
 
     for (unsigned i = 0; i != Size; ++i)
       Data[Fixup.getOffset() + i] = uint8_t(Value >> (i * 8));
@@ -233,7 +220,6 @@ static unsigned getRelaxedOpcodeArith(unsigned Op) {
   case X86::PUSH32i8:  return X86::PUSHi32;
   case X86::PUSH16i8:  return X86::PUSHi16;
   case X86::PUSH64i8:  return X86::PUSH64i32;
-  case X86::PUSH64i16: return X86::PUSH64i32;
   }
 }
 
@@ -249,29 +235,18 @@ bool X86AsmBackend::mayNeedRelaxation(const MCInst &Inst) const {
   if (getRelaxedOpcodeBranch(Inst.getOpcode()) != Inst.getOpcode())
     return true;
 
-  if (MCDisableArithRelaxation)
-    return false;
-
   // Check if this instruction is ever relaxable.
   if (getRelaxedOpcodeArith(Inst.getOpcode()) == Inst.getOpcode())
     return false;
 
 
-  // Check if it has an expression and is not RIP relative.
-  bool hasExp = false;
-  bool hasRIP = false;
-  for (unsigned i = 0; i < Inst.getNumOperands(); ++i) {
-    const MCOperand &Op = Inst.getOperand(i);
-    if (Op.isExpr())
-      hasExp = true;
-
-    if (Op.isReg() && Op.getReg() == X86::RIP)
-      hasRIP = true;
-  }
+  // Check if the relaxable operand has an expression. For the current set of
+  // relaxable instructions, the relaxable operand is always the last operand.
+  unsigned RelaxableOp = Inst.getNumOperands() - 1;
+  if (Inst.getOperand(RelaxableOp).isExpr())
+    return true;
 
-  // FIXME: Why exactly do we need the !hasRIP? Is it just a limitation on
-  // how we do relaxations?
-  return hasExp && !hasRIP;
+  return false;
 }
 
 bool X86AsmBackend::fixupNeedsRelaxation(const MCFixup &Fixup,
@@ -332,7 +307,7 @@ bool X86AsmBackend::writeNopData(uint64_t Count, MCObjectWriter *OW) const {
   // FIXME: We could generated something better than plain 0x90.
   if (!HasNopl) {
     for (uint64_t i = 0; i < Count; ++i)
-      OW->Write8(0x90);
+      OW->write8(0x90);
     return true;
   }
 
@@ -342,10 +317,10 @@ bool X86AsmBackend::writeNopData(uint64_t Count, MCObjectWriter *OW) const {
     const uint8_t ThisNopLength = (uint8_t) std::min(Count, MaxNopLength);
     const uint8_t Prefixes = ThisNopLength <= 10 ? 0 : ThisNopLength - 10;
     for (uint8_t i = 0; i < Prefixes; i++)
-      OW->Write8(0x66);
+      OW->write8(0x66);
     const uint8_t Rest = ThisNopLength - Prefixes;
     for (uint8_t i = 0; i < Rest; i++)
-      OW->Write8(Nops[Rest - 1][i]);
+      OW->write8(Nops[Rest - 1][i]);
     Count -= ThisNopLength;
   } while (Count != 0);
 
@@ -394,46 +369,6 @@ class ELFX86_64AsmBackend : public ELFX86AsmBackend {
   }
 };
 
-// @LOCALMOD-BEGIN
-class NaClX86_32AsmBackend : public ELFX86_32AsmBackend {
-public:
-  NaClX86_32AsmBackend(const Target &T, uint8_t OSABI, StringRef CPU)
-      : ELFX86_32AsmBackend(T, OSABI, CPU),
-        STI(X86_MC::createX86MCSubtargetInfo("i386-unknown-nacl", CPU, "")) {
-    State.PrefixSaved = 0;
-    State.EmitRaw = false;
-  }
-  ~NaClX86_32AsmBackend() override {}
-
-  bool CustomExpandInst(const MCInst &Inst, MCStreamer &Out) override {
-    return CustomExpandInstNaClX86(*STI, Inst, Out, State);
-  }
-
-private:
-  std::unique_ptr<MCSubtargetInfo> STI;
-  X86MCNaClSFIState State;
-};
-
-class NaClX86_64AsmBackend : public ELFX86_64AsmBackend {
-public:
-  NaClX86_64AsmBackend(const Target &T, uint8_t OSABI, StringRef CPU)
-      : ELFX86_64AsmBackend(T, OSABI, CPU),
-        STI(X86_MC::createX86MCSubtargetInfo("x86_64-unknown-nacl", CPU, "")) {
-    State.PrefixSaved = 0;
-    State.EmitRaw = false;
-  }
-  ~NaClX86_64AsmBackend() override {}
-
-  bool CustomExpandInst(const MCInst &Inst, MCStreamer &Out) override {
-    return CustomExpandInstNaClX86(*STI, Inst, Out, State);
-  }
-
-private:
-  std::unique_ptr<MCSubtargetInfo> STI;
-  X86MCNaClSFIState State;
-};
-// @LOCALMOD-END
-
 class WindowsX86AsmBackend : public X86AsmBackend {
   bool Is64Bit;
 
@@ -836,10 +771,8 @@ class DarwinX86_64AsmBackend : public DarwinX86AsmBackend {
 
 MCAsmBackend *llvm::createX86_32AsmBackend(const Target &T,
                                            const MCRegisterInfo &MRI,
-                                           StringRef TT,
+                                           const Triple &TheTriple,
                                            StringRef CPU) {
-  Triple TheTriple(TT);
-
   if (TheTriple.isOSBinFormatMachO())
     return new DarwinX86_32AsmBackend(T, MRI, CPU);
 
@@ -847,19 +780,13 @@ MCAsmBackend *llvm::createX86_32AsmBackend(const Target &T,
     return new WindowsX86AsmBackend(T, false, CPU);
 
   uint8_t OSABI = MCELFObjectTargetWriter::getOSABI(TheTriple.getOS());
-  // @LOCALMOD-BEGIN
-  if (TheTriple.isOSNaCl())
-    return new NaClX86_32AsmBackend(T, OSABI, CPU);
-  // @LOCALMOD-END
   return new ELFX86_32AsmBackend(T, OSABI, CPU);
 }
 
 MCAsmBackend *llvm::createX86_64AsmBackend(const Target &T,
                                            const MCRegisterInfo &MRI,
-                                           StringRef TT,
+                                           const Triple &TheTriple,
                                            StringRef CPU) {
-  Triple TheTriple(TT);
-
   if (TheTriple.isOSBinFormatMachO()) {
     MachO::CPUSubTypeX86 CS =
         StringSwitch<MachO::CPUSubTypeX86>(TheTriple.getArchName())
@@ -873,10 +800,6 @@ MCAsmBackend *llvm::createX86_64AsmBackend(const Target &T,
 
   uint8_t OSABI = MCELFObjectTargetWriter::getOSABI(TheTriple.getOS());
 
-  // @LOCALMOD-BEGIN
-  if (TheTriple.isOSNaCl())
-    return new NaClX86_64AsmBackend(T, OSABI, CPU);
-  // @LOCALMOD-END
   if (TheTriple.getEnvironment() == Triple::GNUX32)
     return new ELFX86_X32AsmBackend(T, OSABI, CPU);
   return new ELFX86_64AsmBackend(T, OSABI, CPU);
diff --git a/lib/Target/X86/MCTargetDesc/X86BaseInfo.h b/lib/Target/X86/MCTargetDesc/X86BaseInfo.h
index 0b9c8eee092..f0d00b0c1bc 100644
--- a/lib/Target/X86/MCTargetDesc/X86BaseInfo.h
+++ b/lib/Target/X86/MCTargetDesc/X86BaseInfo.h
@@ -267,7 +267,7 @@ namespace X86II {
     /// register DI/EDI/ESI.
     RawFrmDst      = 9,
 
-    /// RawFrmSrc - This form is for instructions that use the the source index
+    /// RawFrmSrc - This form is for instructions that use the source index
     /// register SI/ESI/ERI with a possible segment override, and also the
     /// destination index register DI/ESI/RDI.
     RawFrmDstSrc   = 10,
@@ -317,7 +317,7 @@ namespace X86II {
     MRM_F4 = 84, MRM_F5 = 85, MRM_F6 = 86, MRM_F7 = 87,
     MRM_F8 = 88, MRM_F9 = 89, MRM_FA = 90, MRM_FB = 91,
     MRM_FC = 92, MRM_FD = 93, MRM_FE = 94, MRM_FF = 95,
-    CustomFrm = 126, // @LOCALMOD
+
     FormMask       = 127,
 
     //===------------------------------------------------------------------===//
@@ -664,7 +664,6 @@ namespace X86II {
 
     switch (TSFlags & X86II::FormMask) {
     default: llvm_unreachable("Unknown FormMask value in getMemoryOperandNo!");
-    case X86II::CustomFrm: // @LOCALMOD
     case X86II::Pseudo:
     case X86II::RawFrm:
     case X86II::AddRegFrm:
diff --git a/lib/Target/X86/MCTargetDesc/X86ELFObjectWriter.cpp b/lib/Target/X86/MCTargetDesc/X86ELFObjectWriter.cpp
index 45088835cfb..a33468dc476 100644
--- a/lib/Target/X86/MCTargetDesc/X86ELFObjectWriter.cpp
+++ b/lib/Target/X86/MCTargetDesc/X86ELFObjectWriter.cpp
@@ -66,6 +66,7 @@ static X86_64RelType getType64(unsigned Kind,
   case X86::reloc_riprel_4byte:
   case X86::reloc_riprel_4byte_movq_load:
     return RT64_32;
+  case FK_PCRel_2:
   case FK_Data_2:
     return RT64_16;
   case FK_PCRel_1:
diff --git a/lib/Target/X86/MCTargetDesc/X86ELFRelocationInfo.cpp b/lib/Target/X86/MCTargetDesc/X86ELFRelocationInfo.cpp
index 10b83f40fc3..ddb764facdb 100644
--- a/lib/Target/X86/MCTargetDesc/X86ELFRelocationInfo.cpp
+++ b/lib/Target/X86/MCTargetDesc/X86ELFRelocationInfo.cpp
@@ -26,18 +26,24 @@ class X86_64ELFRelocationInfo : public MCRelocationInfo {
   X86_64ELFRelocationInfo(MCContext &Ctx) : MCRelocationInfo(Ctx) {}
 
   const MCExpr *createExprForRelocation(RelocationRef Rel) override {
-    uint64_t RelType; Rel.getType(RelType);
-    symbol_iterator SymI = Rel.getSymbol();
+    uint64_t RelType = Rel.getType();
+    elf_symbol_iterator SymI = Rel.getSymbol();
 
-    StringRef SymName; SymI->getName(SymName);
-    uint64_t  SymAddr; SymI->getAddress(SymAddr);
-    uint64_t  SymSize; SymI->getSize(SymSize);
-    int64_t  Addend;  getELFRelocationAddend(Rel, Addend);
+    ErrorOr<StringRef> SymNameOrErr = SymI->getName();
+    if (std::error_code EC = SymNameOrErr.getError())
+      report_fatal_error(EC.message());
+    StringRef SymName = *SymNameOrErr;
 
-    MCSymbol *Sym = Ctx.GetOrCreateSymbol(SymName);
+    ErrorOr<uint64_t> SymAddr = SymI->getAddress();
+    if (std::error_code EC = SymAddr.getError())
+      report_fatal_error(EC.message());
+    uint64_t SymSize = SymI->getSize();
+    int64_t Addend = *ELFRelocationRef(Rel).getAddend();
+
+    MCSymbol *Sym = Ctx.getOrCreateSymbol(SymName);
     // FIXME: check that the value is actually the same.
     if (!Sym->isVariable())
-      Sym->setVariableValue(MCConstantExpr::Create(SymAddr, Ctx));
+      Sym->setVariableValue(MCConstantExpr::create(*SymAddr, Ctx));
 
     const MCExpr *Expr = nullptr;
     // If hasAddend is true, then we need to add Addend (r_addend) to Expr.
@@ -76,7 +82,7 @@ class X86_64ELFRelocationInfo : public MCRelocationInfo {
     case R_X86_64_PC64:
       // S + A - P (P/pcrel is implicit)
       hasAddend = true;
-      Expr = MCSymbolRefExpr::Create(Sym, Ctx);
+      Expr = MCSymbolRefExpr::create(Sym, Ctx);
       break;
     case R_X86_64_GOT32:
     case R_X86_64_GOT64:
@@ -85,27 +91,27 @@ class X86_64ELFRelocationInfo : public MCRelocationInfo {
     case R_X86_64_GOTPLT64:
       // G + A
       hasAddend = true;
-      Expr = MCSymbolRefExpr::Create(Sym, MCSymbolRefExpr::VK_GOT, Ctx);
+      Expr = MCSymbolRefExpr::create(Sym, MCSymbolRefExpr::VK_GOT, Ctx);
       break;
     case R_X86_64_PLT32:
       // L + A - P -> S@PLT + A
       hasAddend = true;
-      Expr = MCSymbolRefExpr::Create(Sym, MCSymbolRefExpr::VK_PLT, Ctx);
+      Expr = MCSymbolRefExpr::create(Sym, MCSymbolRefExpr::VK_PLT, Ctx);
       break;
     case R_X86_64_GLOB_DAT:
     case R_X86_64_JUMP_SLOT:
       // S
-      Expr = MCSymbolRefExpr::Create(Sym, Ctx);
+      Expr = MCSymbolRefExpr::create(Sym, Ctx);
       break;
     case R_X86_64_GOTPCREL:
     case R_X86_64_GOTPCREL64:
       // G + GOT + A - P -> S@GOTPCREL + A
       hasAddend = true;
-      Expr = MCSymbolRefExpr::Create(Sym, MCSymbolRefExpr::VK_GOTPCREL, Ctx);
+      Expr = MCSymbolRefExpr::create(Sym, MCSymbolRefExpr::VK_GOTPCREL, Ctx);
       break;
     case R_X86_64_GOTOFF64:
       // S + A - GOT
-      Expr = MCSymbolRefExpr::Create(Sym, MCSymbolRefExpr::VK_GOTOFF, Ctx);
+      Expr = MCSymbolRefExpr::create(Sym, MCSymbolRefExpr::VK_GOTOFF, Ctx);
       break;
     case R_X86_64_PLTOFF64:
       // L + A - GOT
@@ -113,15 +119,15 @@ class X86_64ELFRelocationInfo : public MCRelocationInfo {
     case R_X86_64_SIZE32:
     case R_X86_64_SIZE64:
       // Z + A
-      Expr = MCConstantExpr::Create(SymSize, Ctx);
+      Expr = MCConstantExpr::create(SymSize, Ctx);
       break;
     default:
-      Expr = MCSymbolRefExpr::Create(Sym, Ctx);
+      Expr = MCSymbolRefExpr::create(Sym, Ctx);
       break;
     }
     if (Expr && hasAddend && Addend != 0)
-      Expr = MCBinaryExpr::CreateAdd(Expr,
-                                     MCConstantExpr::Create(Addend, Ctx),
+      Expr = MCBinaryExpr::createAdd(Expr,
+                                     MCConstantExpr::create(Addend, Ctx),
                                      Ctx);
     return Expr;
   }
diff --git a/lib/Target/X86/MCTargetDesc/X86MCAsmInfo.cpp b/lib/Target/X86/MCTargetDesc/X86MCAsmInfo.cpp
index 97ddcb92a0b..fc0b0f89e23 100644
--- a/lib/Target/X86/MCTargetDesc/X86MCAsmInfo.cpp
+++ b/lib/Target/X86/MCTargetDesc/X86MCAsmInfo.cpp
@@ -90,15 +90,10 @@ X86ELFMCAsmInfo::X86ELFMCAsmInfo(const Triple &T) {
   bool is64Bit = T.getArch() == Triple::x86_64;
   bool isX32 = T.getEnvironment() == Triple::GNUX32;
 
-  // @LOCALMOD-BEGIN(eliben)
-  // Until Nacl implies x32, we add &&!isNaCl in the PointerSize condition
-  bool isNaCl = T.isOSNaCl();
-
   // For ELF, x86-64 pointer size depends on the ABI.
   // For x86-64 without the x32 ABI, pointer size is 8. For x86 and for x86-64
   // with the x32 ABI, pointer size remains the default 4.
-  PointerSize = (is64Bit && !isX32 && !isNaCl) ? 8 : 4;
-  // @LOCALMOD-END
+  PointerSize = (is64Bit && !isX32) ? 8 : 4;
 
   // OTOH, stack slot size is always 8 for x86-64, even with the x32 ABI.
   CalleeSaveStackSlotSize = is64Bit ? 8 : 4;
@@ -124,9 +119,9 @@ X86_64MCAsmInfoDarwin::getExprForPersonalitySymbol(const MCSymbol *Sym,
                                                    MCStreamer &Streamer) const {
   MCContext &Context = Streamer.getContext();
   const MCExpr *Res =
-    MCSymbolRefExpr::Create(Sym, MCSymbolRefExpr::VK_GOTPCREL, Context);
-  const MCExpr *Four = MCConstantExpr::Create(4, Context);
-  return MCBinaryExpr::CreateAdd(Res, Four, Context);
+    MCSymbolRefExpr::create(Sym, MCSymbolRefExpr::VK_GOTPCREL, Context);
+  const MCExpr *Four = MCConstantExpr::create(4, Context);
+  return MCBinaryExpr::createAdd(Res, Four, Context);
 }
 
 void X86MCAsmInfoMicrosoft::anchor() { }
@@ -137,9 +132,15 @@ X86MCAsmInfoMicrosoft::X86MCAsmInfoMicrosoft(const Triple &Triple) {
     PrivateLabelPrefix = ".L";
     PointerSize = 8;
     WinEHEncodingType = WinEH::EncodingType::Itanium;
-    ExceptionsType = ExceptionHandling::WinEH;
+  } else {
+    // 32-bit X86 doesn't use CFI, so this isn't a real encoding type. It's just
+    // a place holder that the Windows EHStreamer looks for to suppress CFI
+    // output. In particular, usesWindowsCFI() returns false.
+    WinEHEncodingType = WinEH::EncodingType::X86;
   }
 
+  ExceptionsType = ExceptionHandling::WinEH;
+
   AssemblerDialect = AsmWriterFlavor;
 
   TextAlignFillValue = 0x90;
diff --git a/lib/Target/X86/MCTargetDesc/X86MCCodeEmitter.cpp b/lib/Target/X86/MCTargetDesc/X86MCCodeEmitter.cpp
index 0b6288c676e..10c434c8b1b 100644
--- a/lib/Target/X86/MCTargetDesc/X86MCCodeEmitter.cpp
+++ b/lib/Target/X86/MCTargetDesc/X86MCCodeEmitter.cpp
@@ -42,15 +42,15 @@ class X86MCCodeEmitter : public MCCodeEmitter {
   ~X86MCCodeEmitter() override {}
 
   bool is64BitMode(const MCSubtargetInfo &STI) const {
-    return (STI.getFeatureBits() & X86::Mode64Bit) != 0;
+    return STI.getFeatureBits()[X86::Mode64Bit];
   }
 
   bool is32BitMode(const MCSubtargetInfo &STI) const {
-    return (STI.getFeatureBits() & X86::Mode32Bit) != 0;
+    return STI.getFeatureBits()[X86::Mode32Bit];
   }
 
   bool is16BitMode(const MCSubtargetInfo &STI) const {
-    return (STI.getFeatureBits() & X86::Mode16Bit) != 0;
+    return STI.getFeatureBits()[X86::Mode16Bit];
   }
 
   /// Is16BitMemOperand - Return true if the specified instruction has
@@ -149,7 +149,7 @@ class X86MCCodeEmitter : public MCCodeEmitter {
                         SmallVectorImpl<MCFixup> &Fixups,
                         const MCSubtargetInfo &STI) const;
 
-  void EncodeInstruction(const MCInst &MI, raw_ostream &OS,
+  void encodeInstruction(const MCInst &MI, raw_ostream &OS,
                          SmallVectorImpl<MCFixup> &Fixups,
                          const MCSubtargetInfo &STI) const override;
 
@@ -304,7 +304,7 @@ EmitImmediate(const MCOperand &DispOp, SMLoc Loc, unsigned Size,
       EmitConstant(DispOp.getImm()+ImmOffset, Size, CurByte, OS);
       return;
     }
-    Expr = MCConstantExpr::Create(DispOp.getImm(), Ctx);
+    Expr = MCConstantExpr::create(DispOp.getImm(), Ctx);
   } else {
     Expr = DispOp.getExpr();
   }
@@ -351,11 +351,11 @@ EmitImmediate(const MCOperand &DispOp, SMLoc Loc, unsigned Size,
     ImmOffset -= 1;
 
   if (ImmOffset)
-    Expr = MCBinaryExpr::CreateAdd(Expr, MCConstantExpr::Create(ImmOffset, Ctx),
+    Expr = MCBinaryExpr::createAdd(Expr, MCConstantExpr::create(ImmOffset, Ctx),
                                    Ctx);
 
   // Emit a symbolic constant as a fixup and 4 zeros.
-  Fixups.push_back(MCFixup::Create(CurByte, Expr, FixupKind, Loc));
+  Fixups.push_back(MCFixup::create(CurByte, Expr, FixupKind, Loc));
   EmitConstant(0, Size, CurByte, OS);
 }
 
@@ -1152,7 +1152,7 @@ void X86MCCodeEmitter::EmitOpcodePrefix(uint64_t TSFlags, unsigned &CurByte,
 }
 
 void X86MCCodeEmitter::
-EncodeInstruction(const MCInst &MI, raw_ostream &OS,
+encodeInstruction(const MCInst &MI, raw_ostream &OS,
                   SmallVectorImpl<MCFixup> &Fixups,
                   const MCSubtargetInfo &STI) const {
   unsigned Opcode = MI.getOpcode();
@@ -1233,10 +1233,6 @@ EncodeInstruction(const MCInst &MI, raw_ostream &OS,
   switch (TSFlags & X86II::FormMask) {
   default: errs() << "FORM: " << (TSFlags & X86II::FormMask) << "\n";
     llvm_unreachable("Unknown FormMask value in X86MCCodeEmitter!");
-  // @LOCALMOD-BEGIN
-  case X86II::CustomFrm:
-    llvm_unreachable("CustomFrm instruction shouldn't be emitted");
-  // @LOCALMOD-END
   case X86II::Pseudo:
     llvm_unreachable("Pseudo instruction shouldn't be emitted");
   case X86II::RawFrmDstSrc: {
@@ -1479,7 +1475,7 @@ EncodeInstruction(const MCInst &MI, raw_ostream &OS,
           RegNum |= Val;
         }
       }
-      EmitImmediate(MCOperand::CreateImm(RegNum), MI.getLoc(), 1, FK_Data_1,
+      EmitImmediate(MCOperand::createImm(RegNum), MI.getLoc(), 1, FK_Data_1,
                     CurByte, OS, Fixups);
     } else {
       EmitImmediate(MI.getOperand(CurOp++), MI.getLoc(),
diff --git a/lib/Target/X86/MCTargetDesc/X86MCNaCl.cpp b/lib/Target/X86/MCTargetDesc/X86MCNaCl.cpp
deleted file mode 100644
index 2450d5744a0..00000000000
--- a/lib/Target/X86/MCTargetDesc/X86MCNaCl.cpp
+++ /dev/null
@@ -1,832 +0,0 @@
-//=== X86MCNaCl.cpp - Expansion of NaCl pseudo-instructions      --*- C++ -*-=//
-//
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
-//
-//===----------------------------------------------------------------------===//
-//
-//===----------------------------------------------------------------------===//
-#define DEBUG_TYPE "x86-sandboxing"
-
-#include "MCTargetDesc/X86MCTargetDesc.h"
-#include "MCTargetDesc/X86BaseInfo.h"
-#include "MCTargetDesc/X86MCNaCl.h"
-#include "X86NaClDecls.h"
-#include "llvm/MC/MCAsmInfo.h"
-#include "llvm/MC/MCContext.h"
-#include "llvm/MC/MCExpr.h"
-#include "llvm/MC/MCInstrInfo.h"
-#include "llvm/MC/MCRegisterInfo.h"
-#include "llvm/MC/MCInst.h"
-#include "llvm/MC/MCObjectFileInfo.h"
-#include "llvm/MC/MCStreamer.h"
-#include "llvm/CodeGen/ValueTypes.h"
-#include "llvm/Support/CommandLine.h"
-#include "llvm/Support/Debug.h"
-
-using namespace llvm;
-
-cl::opt<bool> FlagUseZeroBasedSandbox("sfi-zero-based-sandbox",
-                                      cl::desc("Use a zero-based sandbox model"
-                                               " for the NaCl SFI."),
-                                      cl::init(false));
-// This flag can be set to false to test the performance impact of
-// hiding the sandbox base.
-cl::opt<bool> FlagHideSandboxBase("sfi-hide-sandbox-base",
-                                  cl::desc("Prevent 64-bit NaCl sandbox"
-                                           " pointers from being written to"
-                                           " the stack. [default=true]"),
-                                  cl::init(true));
-
-const int kNaClX86InstructionBundleSize = 32;
-
-// See the notes below where these functions are defined.
-namespace {
-unsigned getX86SubSuperRegister_(unsigned Reg, EVT VT, bool High=false);
-unsigned DemoteRegTo32_(unsigned RegIn);
-} // namespace
-
-static void PushReturnAddress(const llvm::MCSubtargetInfo &STI,
-                              MCContext &Context, MCStreamer &Out,
-                              MCSymbol *RetTarget) {
-  const MCExpr *RetTargetExpr = MCSymbolRefExpr::Create(RetTarget, Context);
-  if (Context.getObjectFileInfo()->getRelocM() == Reloc::PIC_) {
-    // Calculate return_addr
-    // The return address should not be calculated into R11 because if the push
-    // instruction ends up at the start of a bundle, an attacker could arrange
-    // an indirect jump to it, which would push the full jump target
-    // (which itself was calculated into r11) onto the stack.
-    MCInst LEAInst;
-    LEAInst.setOpcode(X86::LEA64_32r);
-    LEAInst.addOperand(MCOperand::CreateReg(X86::R10D)); // DestReg
-    LEAInst.addOperand(MCOperand::CreateReg(X86::RIP)); // BaseReg
-    LEAInst.addOperand(MCOperand::CreateImm(1)); // Scale
-    LEAInst.addOperand(MCOperand::CreateReg(0)); // IndexReg
-    LEAInst.addOperand(MCOperand::CreateExpr(RetTargetExpr)); // Offset
-    LEAInst.addOperand(MCOperand::CreateReg(0)); // SegmentReg
-    Out.EmitInstruction(LEAInst, STI);
-    // push return_addr
-    MCInst PUSHInst;
-    PUSHInst.setOpcode(X86::PUSH64r);
-    PUSHInst.addOperand(MCOperand::CreateReg(X86::R10));
-    Out.EmitInstruction(PUSHInst, STI);
-  } else {
-    // push return_addr
-    MCInst PUSHInst;
-    PUSHInst.setOpcode(X86::PUSH64i32);
-    PUSHInst.addOperand(MCOperand::CreateExpr(RetTargetExpr));
-    Out.EmitInstruction(PUSHInst, STI);
-  }
-}
-
-static void EmitDirectCall(const llvm::MCSubtargetInfo &STI,
-                           const MCOperand &Op, bool Is64Bit, MCStreamer &Out) {
-  const bool HideSandboxBase =
-      (FlagHideSandboxBase && Is64Bit && !FlagUseZeroBasedSandbox);
-  if (HideSandboxBase) {
-    // For NaCl64, the sequence
-    //   call target
-    //   return_addr:
-    // is changed to
-    //   push return_addr
-    //   jmp target
-    //   .align 32
-    //   return_addr:
-    // This avoids exposing the sandbox base address via the return
-    // address on the stack.
-
-    // When generating PIC code, calculate the return address manually:
-    //  leal return_addr(%rip), %r10d
-    //  push %r10
-    //  jmp target
-    //  .align 32
-    //  return_addr:
-
-    MCContext &Context = Out.getContext();
-
-    // Generate a label for the return address.
-    MCSymbol *RetTarget = Context.createTempSymbol("DirectCallRetAddr", true);
-
-    PushReturnAddress(STI, Context, Out, RetTarget);
-
-    // jmp target
-    MCInst JMPInst;
-    JMPInst.setOpcode(X86::JMP_4);
-    JMPInst.addOperand(Op);
-    Out.EmitInstruction(JMPInst, STI);
-
-    Out.EmitCodeAlignment(kNaClX86InstructionBundleSize);
-    Out.EmitLabel(RetTarget);
-  } else {
-    Out.EmitBundleLock(true);
-
-    MCInst CALLInst;
-    CALLInst.setOpcode(Is64Bit ? X86::CALL64pcrel32 : X86::CALLpcrel32);
-    CALLInst.addOperand(Op);
-    Out.EmitInstruction(CALLInst, STI);
-    Out.EmitBundleUnlock();
-  }
-}
-
-static void EmitIndirectBranch(const llvm::MCSubtargetInfo &STI,
-                               const MCOperand &Op, bool Is64Bit, bool IsCall,
-                               MCStreamer &Out) {
-  const bool HideSandboxBase =
-      (FlagHideSandboxBase && Is64Bit && !FlagUseZeroBasedSandbox);
-  const int JmpMask = -kNaClX86InstructionBundleSize;
-  unsigned Reg32 = Op.getReg();
-
-  // For NaCl64, the sequence
-  //   jmp *%rXX
-  // is changed to
-  //   mov %rXX,%r11d
-  //   and $0xffffffe0,%r11d
-  //   add %r15,%r11
-  //   jmpq *%r11
-  //
-  // And the sequence
-  //   call *%rXX
-  //   return_addr:
-  // is changed to
-  //   mov %rXX,%r11d
-  //   push return_addr
-  //   and $0xffffffe0,%r11d
-  //   add %r15,%r11
-  //   jmpq *%r11
-  //   .align 32
-  //   return_addr:
-  //
-  // This avoids exposing the sandbox base address via the return
-  // address on the stack.
-
-  // When generating PIC code for calls, calculate the return address manually:
-  //   mov %rXX,%r11d
-  //   leal return_addr(%rip), %r10d
-  //   pushq %r10
-  //   and $0xffffffe0,%r11d
-  //   add %r15,%r11
-  //   jmpq *%r11
-  //   .align 32
-  //   return_addr:
-
-  MCSymbol *RetTarget = NULL;
-
-  // For NaCl64, force an assignment of the branch target into r11,
-  // and subsequently use r11 as the ultimate branch target, so that
-  // only r11 (which will never be written to memory) exposes the
-  // sandbox base address.  But avoid a redundant assignment if the
-  // original branch target is already r11 or r11d.
-  const unsigned SafeReg32 = X86::R11D;
-  const unsigned SafeReg64 = X86::R11;
-  if (HideSandboxBase) {
-    // In some cases, EmitIndirectBranch() is called with a 32-bit
-    // register Op (e.g. r11d), and in other cases a 64-bit register
-    // (e.g. r11), so we need to test both variants to avoid a
-    // redundant assignment.  TODO(stichnot): Make callers consistent
-    // on 32 vs 64 bit register.
-    if ((Reg32 != SafeReg32) && (Reg32 != SafeReg64)) {
-      MCInst MOVInst;
-      MOVInst.setOpcode(X86::MOV32rr);
-      MOVInst.addOperand(MCOperand::CreateReg(SafeReg32));
-      MOVInst.addOperand(MCOperand::CreateReg(Reg32));
-      Out.EmitInstruction(MOVInst, STI);
-      Reg32 = SafeReg32;
-    }
-    if (IsCall) {
-      MCContext &Context = Out.getContext();
-      // Generate a label for the return address.
-      RetTarget = Context.createTempSymbol("IndirectCallRetAddr", true);
-      // Explicitly push the (32-bit) return address for a NaCl64 call
-      // instruction.
-      PushReturnAddress(STI, Context, Out, RetTarget);
-    }
-  }
-  const unsigned Reg64 = getX86SubSuperRegister_(Reg32, MVT::i64);
-
-  const bool WillEmitCallInst = IsCall && !HideSandboxBase;
-  Out.EmitBundleLock(WillEmitCallInst);
-
-  MCInst ANDInst;
-  ANDInst.setOpcode(X86::AND32ri8);
-  ANDInst.addOperand(MCOperand::CreateReg(Reg32));
-  ANDInst.addOperand(MCOperand::CreateReg(Reg32));
-  ANDInst.addOperand(MCOperand::CreateImm(JmpMask));
-  Out.EmitInstruction(ANDInst, STI);
-
-  if (Is64Bit && !FlagUseZeroBasedSandbox) {
-    MCInst InstADD;
-    InstADD.setOpcode(X86::ADD64rr);
-    InstADD.addOperand(MCOperand::CreateReg(Reg64));
-    InstADD.addOperand(MCOperand::CreateReg(Reg64));
-    InstADD.addOperand(MCOperand::CreateReg(X86::R15));
-    Out.EmitInstruction(InstADD, STI);
-  }
-
-  if (WillEmitCallInst) {
-    // callq *%rXX
-    MCInst CALLInst;
-    CALLInst.setOpcode(Is64Bit ? X86::CALL64r : X86::CALL32r);
-    CALLInst.addOperand(MCOperand::CreateReg(Is64Bit ? Reg64 : Reg32));
-    Out.EmitInstruction(CALLInst, STI);
-  } else {
-    // jmpq *%rXX   -or-   jmpq *%r11
-    MCInst JMPInst;
-    JMPInst.setOpcode(Is64Bit ? X86::JMP64r : X86::JMP32r);
-    JMPInst.addOperand(MCOperand::CreateReg(Is64Bit ? Reg64 : Reg32));
-    Out.EmitInstruction(JMPInst, STI);
-  }
-  Out.EmitBundleUnlock();
-  if (RetTarget) {
-    Out.EmitCodeAlignment(kNaClX86InstructionBundleSize);
-    Out.EmitLabel(RetTarget);
-  }
-}
-
-static void EmitRet(const llvm::MCSubtargetInfo &STI, const MCOperand *AmtOp,
-                    bool Is64Bit, MCStreamer &Out) {
-  // For NaCl64 returns, follow the convention of using r11 to hold
-  // the target of an indirect jump to avoid potentially leaking the
-  // sandbox base address.
-  const bool HideSandboxBase = (FlagHideSandboxBase &&
-                                Is64Bit && !FlagUseZeroBasedSandbox);
-  // For NaCl64 sandbox hiding, use r11 to hold the branch target.
-  // Otherwise, use rcx/ecx for fewer instruction bytes (no REX
-  // prefix).
-  const unsigned RegTarget = HideSandboxBase ? X86::R11 :
-    (Is64Bit ? X86::RCX : X86::ECX);
-  MCInst POPInst;
-  POPInst.setOpcode(Is64Bit ? X86::POP64r : X86::POP32r);
-  POPInst.addOperand(MCOperand::CreateReg(RegTarget));
-  Out.EmitInstruction(POPInst, STI);
-
-  if (AmtOp) {
-    assert(!Is64Bit);
-    MCInst ADDInst;
-    unsigned ADDReg = X86::ESP;
-    ADDInst.setOpcode(X86::ADD32ri);
-    ADDInst.addOperand(MCOperand::CreateReg(ADDReg));
-    ADDInst.addOperand(MCOperand::CreateReg(ADDReg));
-    ADDInst.addOperand(*AmtOp);
-    Out.EmitInstruction(ADDInst, STI);
-  }
-
-  EmitIndirectBranch(STI, MCOperand::CreateReg(RegTarget), Is64Bit, false, Out);
-}
-
-// Fix a register after being truncated to 32-bits.
-static void EmitRegFix(const llvm::MCSubtargetInfo &STI, unsigned Reg64,
-                       MCStreamer &Out) {
-  // lea (%rsp, %r15, 1), %rsp
-  // We do not need to add the R15 base for the zero-based sandbox model
-  if (!FlagUseZeroBasedSandbox) {
-    MCInst Tmp;
-    Tmp.setOpcode(X86::LEA64r);
-    Tmp.addOperand(MCOperand::CreateReg(Reg64));    // DestReg
-    Tmp.addOperand(MCOperand::CreateReg(Reg64));    // BaseReg
-    Tmp.addOperand(MCOperand::CreateImm(1));        // Scale
-    Tmp.addOperand(MCOperand::CreateReg(X86::R15)); // IndexReg
-    Tmp.addOperand(MCOperand::CreateImm(0));        // Offset
-    Tmp.addOperand(MCOperand::CreateReg(0));        // SegmentReg
-    Out.EmitInstruction(Tmp, STI);
-  }
-}
-
-static void EmitSPArith(const llvm::MCSubtargetInfo &STI, unsigned Opc,
-                        const MCOperand &ImmOp, MCStreamer &Out) {
-  Out.EmitBundleLock(false);
-
-  MCInst Tmp;
-  Tmp.setOpcode(Opc);
-  Tmp.addOperand(MCOperand::CreateReg(X86::ESP));
-  Tmp.addOperand(MCOperand::CreateReg(X86::ESP));
-  Tmp.addOperand(ImmOp);
-  Out.EmitInstruction(Tmp, STI);
-
-  EmitRegFix(STI, X86::RSP, Out);
-  Out.EmitBundleUnlock();
-}
-
-static void EmitSPAdj(const llvm::MCSubtargetInfo &STI, const MCOperand &ImmOp,
-                      MCStreamer &Out) {
-  Out.EmitBundleLock(false);
-
-  MCInst Tmp;
-  Tmp.setOpcode(X86::LEA64_32r);
-  Tmp.addOperand(MCOperand::CreateReg(X86::RSP)); // DestReg
-  Tmp.addOperand(MCOperand::CreateReg(X86::RBP)); // BaseReg
-  Tmp.addOperand(MCOperand::CreateImm(1));        // Scale
-  Tmp.addOperand(MCOperand::CreateReg(0));        // IndexReg
-  Tmp.addOperand(ImmOp);                          // Offset
-  Tmp.addOperand(MCOperand::CreateReg(0));        // SegmentReg
-  Out.EmitInstruction(Tmp, STI);
-
-  EmitRegFix(STI, X86::RSP, Out);
-  Out.EmitBundleUnlock();
-}
-
-static void EmitPrefix(const llvm::MCSubtargetInfo &STI, unsigned Opc,
-                       MCStreamer &Out, X86MCNaClSFIState &State) {
-  MCInst PrefixInst;
-  PrefixInst.setOpcode(Opc);
-  State.EmitRaw = true;
-  Out.EmitInstruction(PrefixInst, STI);
-  State.EmitRaw = false;
-}
-
-static void EmitMoveRegReg(const llvm::MCSubtargetInfo &STI, bool Is64Bit,
-                           unsigned ToReg, unsigned FromReg, MCStreamer &Out) {
-  MCInst Move;
-  Move.setOpcode(Is64Bit ? X86::MOV64rr : X86::MOV32rr);
-  Move.addOperand(MCOperand::CreateReg(ToReg));
-  Move.addOperand(MCOperand::CreateReg(FromReg));
-  Out.EmitInstruction(Move, STI);
-}
-
-static void EmitRegTruncate(const llvm::MCSubtargetInfo &STI, unsigned Reg64,
-                            MCStreamer &Out) {
-  unsigned Reg32 = getX86SubSuperRegister_(Reg64, MVT::i32);
-  EmitMoveRegReg(STI, false, Reg32, Reg32, Out);
-}
-
-static void HandleMemoryRefTruncation(const llvm::MCSubtargetInfo &STI,
-                                      MCInst *Inst, unsigned IndexOpPosition,
-                                      MCStreamer &Out) {
-  unsigned IndexReg = Inst->getOperand(IndexOpPosition).getReg();
-  if (FlagUseZeroBasedSandbox) {
-    // With the zero-based sandbox, we use a 32-bit register on the index
-    Inst->getOperand(IndexOpPosition).setReg(DemoteRegTo32_(IndexReg));
-  } else {
-    EmitRegTruncate(STI, IndexReg, Out);
-  }
-}
-
-static void ShortenMemoryRef(MCInst *Inst, unsigned IndexOpPosition) {
-  unsigned ImmOpPosition = IndexOpPosition - 1;
-  unsigned BaseOpPosition = IndexOpPosition - 2;
-  unsigned IndexReg = Inst->getOperand(IndexOpPosition).getReg();
-  // For the SIB byte, if the scale is 1 and the base is 0, then
-  // an equivalent setup moves index to base, and index to 0.  The
-  // equivalent setup is optimized to remove the SIB byte in
-  // X86MCCodeEmitter.cpp.
-  if (Inst->getOperand(ImmOpPosition).getImm() == 1 &&
-      Inst->getOperand(BaseOpPosition).getReg() == 0) {
-    Inst->getOperand(BaseOpPosition).setReg(IndexReg);
-    Inst->getOperand(IndexOpPosition).setReg(0);
-  }
-}
-
-static void EmitLoad(const llvm::MCSubtargetInfo &STI, bool Is64Bit,
-                     unsigned DestReg, unsigned BaseReg, unsigned Scale,
-                     unsigned IndexReg, unsigned Offset, unsigned SegmentReg,
-                     MCStreamer &Out) {
-  // Load DestReg from address BaseReg + Scale * IndexReg + Offset
-  MCInst Load;
-  Load.setOpcode(Is64Bit ? X86::MOV64rm : X86::MOV32rm);
-  Load.addOperand(MCOperand::CreateReg(DestReg));
-  Load.addOperand(MCOperand::CreateReg(BaseReg));
-  Load.addOperand(MCOperand::CreateImm(Scale));
-  Load.addOperand(MCOperand::CreateReg(IndexReg));
-  Load.addOperand(MCOperand::CreateImm(Offset));
-  Load.addOperand(MCOperand::CreateReg(SegmentReg));
-  Out.EmitInstruction(Load, STI);
-}
-
-static bool SandboxMemoryRef(MCInst *Inst,
-                             unsigned *IndexOpPosition) {
-  for (unsigned i = 0, last = Inst->getNumOperands(); i < last; i++) {
-    if (!Inst->getOperand(i).isReg() ||
-        Inst->getOperand(i).getReg() != X86::PSEUDO_NACL_SEG) {
-      continue;
-    }
-    // Return the index register that will need to be truncated.
-    // The order of operands on a memory reference is always:
-    // (BaseReg, ScaleImm, IndexReg, DisplacementImm, SegmentReg),
-    // So if we found a match for a segment register value, we know that
-    // the index register is exactly two operands prior.
-    *IndexOpPosition = i - 2;
-
-    // Remove the PSEUDO_NACL_SEG annotation.
-    Inst->getOperand(i).setReg(0);
-    return true;
-  }
-  return false;
-}
-
-static void EmitREST(const llvm::MCSubtargetInfo &STI, const MCInst &Inst,
-                     unsigned Reg32, bool IsMem, MCStreamer &Out) {
-  unsigned Reg64 = getX86SubSuperRegister_(Reg32, MVT::i64);
-  Out.EmitBundleLock(false);
-  if (!IsMem) {
-    EmitMoveRegReg(STI, false, Reg32, Inst.getOperand(0).getReg(), Out);
-  } else {
-    unsigned IndexOpPosition;
-    MCInst SandboxedInst = Inst;
-    if (SandboxMemoryRef(&SandboxedInst, &IndexOpPosition)) {
-      HandleMemoryRefTruncation(STI, &SandboxedInst, IndexOpPosition, Out);
-      ShortenMemoryRef(&SandboxedInst, IndexOpPosition);
-    }
-    EmitLoad(STI, false, Reg32,
-             SandboxedInst.getOperand(0).getReg(), // BaseReg
-             SandboxedInst.getOperand(1).getImm(), // Scale
-             SandboxedInst.getOperand(2).getReg(), // IndexReg
-             SandboxedInst.getOperand(3).getImm(), // Offset
-             SandboxedInst.getOperand(4).getReg(), // SegmentReg
-             Out);
-  }
-
-  EmitRegFix(STI, Reg64, Out);
-  Out.EmitBundleUnlock();
-}
-
-
-namespace {
-// RAII holder for the recursion guard.
-class EmitRawState {
- public:
-  EmitRawState(X86MCNaClSFIState &S) : State(S) {
-    State.EmitRaw = true;
-  }
-  ~EmitRawState() {
-    State.EmitRaw = false;
-  }
- private:
-  X86MCNaClSFIState &State;
-};
-}
-
-namespace llvm {
-// CustomExpandInstNaClX86 -
-//   If Inst is a NaCl pseudo instruction, emits the substitute
-//   expansion to the MCStreamer and returns true.
-//   Otherwise, returns false.
-//
-//   NOTE: Each time this function calls Out.EmitInstruction(), it will be
-//   called again recursively to rewrite the new instruction being emitted.
-//   Care must be taken to ensure that this does not result in an infinite
-//   loop. Also, global state must be managed carefully so that it is
-//   consistent during recursive calls.
-//
-//   We need global state to keep track of the explicit prefix (PREFIX_*)
-//   instructions. Unfortunately, the assembly parser prefers to generate
-//   these instead of combined instructions. At this time, having only
-//   one explicit prefix is supported.
-bool CustomExpandInstNaClX86(const llvm::MCSubtargetInfo &STI,
-                             const MCInst &Inst, MCStreamer &Out,
-                             X86MCNaClSFIState &State) {
-  // If we are emitting to .s, only sandbox pseudos not supported by gas.
-  if (Out.hasRawTextSupport()) {
-    if (!(Inst.getOpcode() == X86::NACL_ANDSPi8 ||
-          Inst.getOpcode() == X86::NACL_ANDSPi32))
-      return false;
-  }
-  // If we make a call to EmitInstruction, we will be called recursively. In
-  // this case we just want the raw instruction to be emitted instead of
-  // handling the instruction here.
-  if (State.EmitRaw == true) {
-    return false;
-  }
-  EmitRawState E(State);
-  unsigned Opc = Inst.getOpcode();
-  DEBUG(dbgs() << "CustomExpandInstNaClX86("; Inst.dump(); dbgs() << ")\n");
-  switch (Opc) {
-  case X86::LOCK_PREFIX:
-  case X86::REP_PREFIX:
-  case X86::REPNE_PREFIX:
-  case X86::REX64_PREFIX:
-    // Ugly hack because LLVM AsmParser is not smart enough to combine
-    // prefixes back into the instruction they modify.
-    assert(State.PrefixSaved == 0);
-    State.PrefixSaved = Opc;
-    return true;
-  case X86::CALLpcrel32:
-    assert(State.PrefixSaved == 0);
-    EmitDirectCall(STI, Inst.getOperand(0), false, Out);
-    return true;
-  case X86::CALL64pcrel32:
-  case X86::NACL_CALL64d:
-    assert(State.PrefixSaved == 0);
-    EmitDirectCall(STI, Inst.getOperand(0), true, Out);
-    return true;
-  case X86::NACL_CALL32r:
-    assert(State.PrefixSaved == 0);
-    EmitIndirectBranch(STI, Inst.getOperand(0), false, true, Out);
-    return true;
-  case X86::NACL_CALL64r:
-    assert(State.PrefixSaved == 0);
-    EmitIndirectBranch(STI, Inst.getOperand(0), true, true, Out);
-    return true;
-  case X86::NACL_JMP32r:
-    assert(State.PrefixSaved == 0);
-    EmitIndirectBranch(STI, Inst.getOperand(0), false, false, Out);
-    return true;
-  case X86::NACL_JMP64r:
-  case X86::NACL_JMP64z:
-    assert(State.PrefixSaved == 0);
-    EmitIndirectBranch(STI, Inst.getOperand(0), true, false, Out);
-    return true;
-  case X86::NACL_RET32:
-    assert(State.PrefixSaved == 0);
-    EmitRet(STI, NULL, false, Out);
-    return true;
-  case X86::NACL_RET64:
-    assert(State.PrefixSaved == 0);
-    EmitRet(STI, NULL, true, Out);
-    return true;
-  case X86::NACL_RETI32:
-    assert(State.PrefixSaved == 0);
-    EmitRet(STI, &Inst.getOperand(0), false, Out);
-    return true;
-  case X86::NACL_ASPi8:
-    assert(State.PrefixSaved == 0);
-    EmitSPArith(STI, X86::ADD32ri8, Inst.getOperand(0), Out);
-    return true;
-  case X86::NACL_ASPi32:
-    assert(State.PrefixSaved == 0);
-    EmitSPArith(STI, X86::ADD32ri, Inst.getOperand(0), Out);
-    return true;
-  case X86::NACL_SSPi8:
-    assert(State.PrefixSaved == 0);
-    EmitSPArith(STI, X86::SUB32ri8, Inst.getOperand(0), Out);
-    return true;
-  case X86::NACL_SSPi32:
-    assert(State.PrefixSaved == 0);
-    EmitSPArith(STI, X86::SUB32ri, Inst.getOperand(0), Out);
-    return true;
-  case X86::NACL_ANDSPi8:
-    assert(State.PrefixSaved == 0);
-    EmitSPArith(STI, X86::AND32ri8, Inst.getOperand(0), Out);
-    return true;
-  case X86::NACL_ANDSPi32:
-    assert(State.PrefixSaved == 0);
-    EmitSPArith(STI, X86::AND32ri, Inst.getOperand(0), Out);
-    return true;
-  case X86::NACL_SPADJi32:
-    assert(State.PrefixSaved == 0);
-    EmitSPAdj(STI, Inst.getOperand(0), Out);
-    return true;
-  case X86::NACL_RESTBPm:
-    assert(State.PrefixSaved == 0);
-    EmitREST(STI, Inst, X86::EBP, true, Out);
-    return true;
-  case X86::NACL_RESTBPr:
-  case X86::NACL_RESTBPrz:
-    assert(State.PrefixSaved == 0);
-    EmitREST(STI, Inst, X86::EBP, false, Out);
-    return true;
-  case X86::NACL_RESTSPm:
-    assert(State.PrefixSaved == 0);
-    EmitREST(STI, Inst, X86::ESP, true, Out);
-    return true;
-  case X86::NACL_RESTSPr:
-  case X86::NACL_RESTSPrz:
-    assert(State.PrefixSaved == 0);
-    EmitREST(STI, Inst, X86::ESP, false, Out);
-    return true;
-  }
-
-  unsigned IndexOpPosition;
-  MCInst SandboxedInst = Inst;
-  // If we need to sandbox a memory reference and we have a saved prefix,
-  // use a single bundle-lock/unlock for the whole sequence of
-  // added_truncating_inst + prefix + mem_ref_inst.
-  if (SandboxMemoryRef(&SandboxedInst, &IndexOpPosition)) {
-    unsigned PrefixLocal = State.PrefixSaved;
-    State.PrefixSaved = 0;
-
-    if (PrefixLocal || !FlagUseZeroBasedSandbox)
-      Out.EmitBundleLock(false);
-
-    HandleMemoryRefTruncation(STI, &SandboxedInst, IndexOpPosition, Out);
-    ShortenMemoryRef(&SandboxedInst, IndexOpPosition);
-
-    if (PrefixLocal)
-      EmitPrefix(STI, PrefixLocal, Out, State);
-    Out.EmitInstruction(SandboxedInst, STI);
-
-    if (PrefixLocal || !FlagUseZeroBasedSandbox)
-      Out.EmitBundleUnlock();
-    return true;
-  }
-
-  // If the special case above doesn't apply, but there is still a saved prefix,
-  // then the saved prefix should be bundled-locked with Inst, so that it cannot
-  // be separated by bundle padding.
-  if (State.PrefixSaved) {
-    unsigned PrefixLocal = State.PrefixSaved;
-    State.PrefixSaved = 0;
-    Out.EmitBundleLock(false);
-    EmitPrefix(STI, PrefixLocal, Out, State);
-    Out.EmitInstruction(Inst, STI);
-    Out.EmitBundleUnlock();
-    return true;
-  }
-  return false;
-}
-
-} // namespace llvm
-
-
-
-
-// @@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@
-//
-// This is an exact copy of getX86SubSuperRegister from X86RegisterInfo.h
-// We cannot use the original because it is part of libLLVMX86CodeGen,
-// which cannot be a dependency of this module (libLLVMX86Desc).
-//
-// However, in all likelyhood, the real getX86SubSuperRegister will
-// eventually be moved to MCTargetDesc, and then this copy can be
-// removed.
-
-namespace {
-unsigned getX86SubSuperRegister_(unsigned Reg, EVT VT, bool High) {
-  switch (VT.getSimpleVT().SimpleTy) {
-  default: return Reg;
-  case MVT::i8:
-    if (High) {
-      switch (Reg) {
-      default: return 0;
-      case X86::AH: case X86::AL: case X86::AX: case X86::EAX: case X86::RAX:
-        return X86::AH;
-      case X86::DH: case X86::DL: case X86::DX: case X86::EDX: case X86::RDX:
-        return X86::DH;
-      case X86::CH: case X86::CL: case X86::CX: case X86::ECX: case X86::RCX:
-        return X86::CH;
-      case X86::BH: case X86::BL: case X86::BX: case X86::EBX: case X86::RBX:
-        return X86::BH;
-      }
-    } else {
-      switch (Reg) {
-      default: return 0;
-      case X86::AH: case X86::AL: case X86::AX: case X86::EAX: case X86::RAX:
-        return X86::AL;
-      case X86::DH: case X86::DL: case X86::DX: case X86::EDX: case X86::RDX:
-        return X86::DL;
-      case X86::CH: case X86::CL: case X86::CX: case X86::ECX: case X86::RCX:
-        return X86::CL;
-      case X86::BH: case X86::BL: case X86::BX: case X86::EBX: case X86::RBX:
-        return X86::BL;
-      case X86::SIL: case X86::SI: case X86::ESI: case X86::RSI:
-        return X86::SIL;
-      case X86::DIL: case X86::DI: case X86::EDI: case X86::RDI:
-        return X86::DIL;
-      case X86::BPL: case X86::BP: case X86::EBP: case X86::RBP:
-        return X86::BPL;
-      case X86::SPL: case X86::SP: case X86::ESP: case X86::RSP:
-        return X86::SPL;
-      case X86::R8B: case X86::R8W: case X86::R8D: case X86::R8:
-        return X86::R8B;
-      case X86::R9B: case X86::R9W: case X86::R9D: case X86::R9:
-        return X86::R9B;
-      case X86::R10B: case X86::R10W: case X86::R10D: case X86::R10:
-        return X86::R10B;
-      case X86::R11B: case X86::R11W: case X86::R11D: case X86::R11:
-        return X86::R11B;
-      case X86::R12B: case X86::R12W: case X86::R12D: case X86::R12:
-        return X86::R12B;
-      case X86::R13B: case X86::R13W: case X86::R13D: case X86::R13:
-        return X86::R13B;
-      case X86::R14B: case X86::R14W: case X86::R14D: case X86::R14:
-        return X86::R14B;
-      case X86::R15B: case X86::R15W: case X86::R15D: case X86::R15:
-        return X86::R15B;
-      }
-    }
-  case MVT::i16:
-    switch (Reg) {
-    default: return Reg;
-    case X86::AH: case X86::AL: case X86::AX: case X86::EAX: case X86::RAX:
-      return X86::AX;
-    case X86::DH: case X86::DL: case X86::DX: case X86::EDX: case X86::RDX:
-      return X86::DX;
-    case X86::CH: case X86::CL: case X86::CX: case X86::ECX: case X86::RCX:
-      return X86::CX;
-    case X86::BH: case X86::BL: case X86::BX: case X86::EBX: case X86::RBX:
-      return X86::BX;
-    case X86::SIL: case X86::SI: case X86::ESI: case X86::RSI:
-      return X86::SI;
-    case X86::DIL: case X86::DI: case X86::EDI: case X86::RDI:
-      return X86::DI;
-    case X86::BPL: case X86::BP: case X86::EBP: case X86::RBP:
-      return X86::BP;
-    case X86::SPL: case X86::SP: case X86::ESP: case X86::RSP:
-      return X86::SP;
-    case X86::R8B: case X86::R8W: case X86::R8D: case X86::R8:
-      return X86::R8W;
-    case X86::R9B: case X86::R9W: case X86::R9D: case X86::R9:
-      return X86::R9W;
-    case X86::R10B: case X86::R10W: case X86::R10D: case X86::R10:
-      return X86::R10W;
-    case X86::R11B: case X86::R11W: case X86::R11D: case X86::R11:
-      return X86::R11W;
-    case X86::R12B: case X86::R12W: case X86::R12D: case X86::R12:
-      return X86::R12W;
-    case X86::R13B: case X86::R13W: case X86::R13D: case X86::R13:
-      return X86::R13W;
-    case X86::R14B: case X86::R14W: case X86::R14D: case X86::R14:
-      return X86::R14W;
-    case X86::R15B: case X86::R15W: case X86::R15D: case X86::R15:
-      return X86::R15W;
-    }
-  case MVT::i32:
-    switch (Reg) {
-    default: return Reg;
-    case X86::AH: case X86::AL: case X86::AX: case X86::EAX: case X86::RAX:
-      return X86::EAX;
-    case X86::DH: case X86::DL: case X86::DX: case X86::EDX: case X86::RDX:
-      return X86::EDX;
-    case X86::CH: case X86::CL: case X86::CX: case X86::ECX: case X86::RCX:
-      return X86::ECX;
-    case X86::BH: case X86::BL: case X86::BX: case X86::EBX: case X86::RBX:
-      return X86::EBX;
-    case X86::SIL: case X86::SI: case X86::ESI: case X86::RSI:
-      return X86::ESI;
-    case X86::DIL: case X86::DI: case X86::EDI: case X86::RDI:
-      return X86::EDI;
-    case X86::BPL: case X86::BP: case X86::EBP: case X86::RBP:
-      return X86::EBP;
-    case X86::SPL: case X86::SP: case X86::ESP: case X86::RSP:
-      return X86::ESP;
-    case X86::R8B: case X86::R8W: case X86::R8D: case X86::R8:
-      return X86::R8D;
-    case X86::R9B: case X86::R9W: case X86::R9D: case X86::R9:
-      return X86::R9D;
-    case X86::R10B: case X86::R10W: case X86::R10D: case X86::R10:
-      return X86::R10D;
-    case X86::R11B: case X86::R11W: case X86::R11D: case X86::R11:
-      return X86::R11D;
-    case X86::R12B: case X86::R12W: case X86::R12D: case X86::R12:
-      return X86::R12D;
-    case X86::R13B: case X86::R13W: case X86::R13D: case X86::R13:
-      return X86::R13D;
-    case X86::R14B: case X86::R14W: case X86::R14D: case X86::R14:
-      return X86::R14D;
-    case X86::R15B: case X86::R15W: case X86::R15D: case X86::R15:
-      return X86::R15D;
-    }
-  case MVT::i64:
-    switch (Reg) {
-    default: return Reg;
-    case X86::AH: case X86::AL: case X86::AX: case X86::EAX: case X86::RAX:
-      return X86::RAX;
-    case X86::DH: case X86::DL: case X86::DX: case X86::EDX: case X86::RDX:
-      return X86::RDX;
-    case X86::CH: case X86::CL: case X86::CX: case X86::ECX: case X86::RCX:
-      return X86::RCX;
-    case X86::BH: case X86::BL: case X86::BX: case X86::EBX: case X86::RBX:
-      return X86::RBX;
-    case X86::SIL: case X86::SI: case X86::ESI: case X86::RSI:
-      return X86::RSI;
-    case X86::DIL: case X86::DI: case X86::EDI: case X86::RDI:
-      return X86::RDI;
-    case X86::BPL: case X86::BP: case X86::EBP: case X86::RBP:
-      return X86::RBP;
-    case X86::SPL: case X86::SP: case X86::ESP: case X86::RSP:
-      return X86::RSP;
-    case X86::R8B: case X86::R8W: case X86::R8D: case X86::R8:
-      return X86::R8;
-    case X86::R9B: case X86::R9W: case X86::R9D: case X86::R9:
-      return X86::R9;
-    case X86::R10B: case X86::R10W: case X86::R10D: case X86::R10:
-      return X86::R10;
-    case X86::R11B: case X86::R11W: case X86::R11D: case X86::R11:
-      return X86::R11;
-    case X86::R12B: case X86::R12W: case X86::R12D: case X86::R12:
-      return X86::R12;
-    case X86::R13B: case X86::R13W: case X86::R13D: case X86::R13:
-      return X86::R13;
-    case X86::R14B: case X86::R14W: case X86::R14D: case X86::R14:
-      return X86::R14;
-    case X86::R15B: case X86::R15W: case X86::R15D: case X86::R15:
-      return X86::R15;
-    }
-  }
-
-  return Reg;
-}
-
-// This is a copy of DemoteRegTo32 from X86NaClRewritePass.cpp.
-// We cannot use the original because it uses part of libLLVMX86CodeGen,
-// which cannot be a dependency of this module (libLLVMX86Desc).
-// Note that this function calls getX86SubSuperRegister_, which is
-// also a copied function for the same reason.
-
-unsigned DemoteRegTo32_(unsigned RegIn) {
-  if (RegIn == 0)
-    return 0;
-  unsigned RegOut = getX86SubSuperRegister_(RegIn, MVT::i32, false);
-  assert(RegOut != 0);
-  return RegOut;
-}
-} //namespace
-// @@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@
-
-unsigned getReg32(unsigned Reg) {
-  return getX86SubSuperRegister_(Reg, MVT::i32, false);
-}
-
-unsigned getReg64(unsigned Reg) {
-  return getX86SubSuperRegister_(Reg, MVT::i64, false);
-}
diff --git a/lib/Target/X86/MCTargetDesc/X86MCNaCl.h b/lib/Target/X86/MCTargetDesc/X86MCNaCl.h
deleted file mode 100644
index bfe61c23288..00000000000
--- a/lib/Target/X86/MCTargetDesc/X86MCNaCl.h
+++ /dev/null
@@ -1,24 +0,0 @@
-//===-- X86MCNaCl.h - Prototype for CustomExpandInstNaClX86   ---*- C++ -*-===//
-//
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
-//
-//===----------------------------------------------------------------------===//
-
-#ifndef X86MCNACL_H
-#define X86MCNACL_H
-
-namespace llvm {
-class MCInst;
-class MCStreamer;
-struct X86MCNaClSFIState {
-  unsigned PrefixSaved;
-  bool EmitRaw;
-};
-bool CustomExpandInstNaClX86(const MCSubtargetInfo &STI, const MCInst &Inst,
-                             MCStreamer &Out, X86MCNaClSFIState &State);
-}
-
-#endif
diff --git a/lib/Target/X86/MCTargetDesc/X86MCNaClExpander.cpp b/lib/Target/X86/MCTargetDesc/X86MCNaClExpander.cpp
deleted file mode 100644
index 6b6314b61ee..00000000000
--- a/lib/Target/X86/MCTargetDesc/X86MCNaClExpander.cpp
+++ /dev/null
@@ -1,174 +0,0 @@
-//===- X86MCNaClExpander.cpp ------------------------------------*- C++ -*-===//
-//
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
-//
-//===----------------------------------------------------------------------===//
-//
-// This file implements the X86MCNaClExpander class, the X86 specific
-// subclass of MCNaClExpander.
-//
-//===----------------------------------------------------------------------===//
-#include "X86MCNaClExpander.h"
-#include "X86BaseInfo.h"
-
-#include "llvm/MC/MCInst.h"
-#include "llvm/MC/MCInstrDesc.h"
-#include "llvm/MC/MCInstrInfo.h"
-#include "llvm/MC/MCNaClExpander.h"
-#include "llvm/MC/MCRegisterInfo.h"
-#include "llvm/MC/MCStreamer.h"
-
-using namespace llvm;
-
-static const int kBundleSize = 32;
-
-unsigned getReg32(unsigned Reg);
-
-void X86::X86MCNaClExpander::expandIndirectBranch(const MCInst &Inst,
-                                                  MCStreamer &Out,
-                                                  const MCSubtargetInfo &STI) {
-  bool ThroughMemory = false, isCall = false;
-  switch (Inst.getOpcode()) {
-  case X86::CALL16m:
-  case X86::CALL32m:
-    ThroughMemory = true;
-  case X86::CALL16r:
-  case X86::CALL32r:
-    isCall = true;
-    break;
-  case X86::JMP16m:
-  case X86::JMP32m:
-    ThroughMemory = true;
-  case X86::JMP16r:
-  case X86::JMP32r:
-    break;
-  default:
-    llvm_unreachable("invalid indirect jmp/call");
-  }
-
-  MCOperand Target;
-  if (ThroughMemory) {
-    if (numScratchRegs() == 0) {
-      Error(Inst, "No scratch registers specified");
-      exit(1);
-    }
-
-    Target = MCOperand::CreateReg(getReg32(getScratchReg(0)));
-
-    MCInst Mov;
-    Mov.setOpcode(X86::MOV32rm);
-    Mov.addOperand(Target);
-    Mov.addOperand(Inst.getOperand(0)); // Base
-    Mov.addOperand(Inst.getOperand(1)); // Scale
-    Mov.addOperand(Inst.getOperand(2)); // Index
-    Mov.addOperand(Inst.getOperand(3)); // Offset
-    Mov.addOperand(Inst.getOperand(4)); // Segment
-    Out.EmitInstruction(Mov, STI);
-  } else {
-    Target = MCOperand::CreateReg(getReg32(Inst.getOperand(0).getReg()));
-  }
-
-  Out.EmitBundleLock(isCall);
-
-  MCInst And;
-  And.setOpcode(X86::AND32ri8);
-  And.addOperand(Target);
-  And.addOperand(Target);
-  And.addOperand(MCOperand::CreateImm(-kBundleSize));
-  Out.EmitInstruction(And, STI);
-
-  MCInst Branch;
-  Branch.setOpcode(isCall ? X86::CALL32r : X86::JMP32r);
-  Branch.addOperand(Target);
-  Out.EmitInstruction(Branch, STI);
-
-  Out.EmitBundleUnlock();
-}
-
-void X86::X86MCNaClExpander::expandReturn(const MCInst &Inst, MCStreamer &Out,
-                                          const MCSubtargetInfo &STI) {
-  if (numScratchRegs() == 0) {
-    Error(Inst, "No scratch registers specified.");
-    exit(1);
-  }
-
-  MCOperand ScratchReg = MCOperand::CreateReg(getReg32(getScratchReg(0)));
-  MCInst Pop;
-  Pop.setOpcode(X86::POP32r);
-  Pop.addOperand(ScratchReg);
-  Out.EmitInstruction(Pop, STI);
-
-  if (Inst.getNumOperands() > 0) {
-    assert(Inst.getOpcode() == X86::RETIL);
-    MCInst Add;
-    Add.setOpcode(X86::ADD32ri);
-    Add.addOperand(MCOperand::CreateReg(X86::ESP));
-    Add.addOperand(MCOperand::CreateReg(X86::ESP));
-    Add.addOperand(Inst.getOperand(0));
-    Out.EmitInstruction(Add, STI);
-  }
-
-  MCInst Jmp;
-  Jmp.setOpcode(X86::JMP32r);
-  Jmp.addOperand(ScratchReg);
-  expandIndirectBranch(Jmp, Out, STI);
-}
-
-static bool isPrefix(const MCInst &Inst) {
-  switch (Inst.getOpcode()) {
-  case X86::LOCK_PREFIX:
-  case X86::REP_PREFIX:
-  case X86::REPNE_PREFIX:
-  case X86::REX64_PREFIX:
-    return true;
-  default:
-    return false;
-  }
-}
-
-void X86::X86MCNaClExpander::emitPrefixes(MCStreamer &Out,
-                                          const MCSubtargetInfo &STI) {
-  for (const MCInst &Prefix : Prefixes)
-    Out.EmitInstruction(Prefix, STI);
-  Prefixes.clear();
-}
-
-void X86::X86MCNaClExpander::doExpandInst(const MCInst &Inst, MCStreamer &Out,
-                                          const MCSubtargetInfo &STI) {
-  if (isPrefix(Inst)) {
-    Prefixes.push_back(Inst);
-  } else {
-    switch (Inst.getOpcode()) {
-    case X86::CALL16r:
-    case X86::CALL32r:
-    case X86::CALL16m:
-    case X86::CALL32m:
-    case X86::JMP16r:
-    case X86::JMP32r:
-    case X86::JMP16m:
-    case X86::JMP32m:
-      return expandIndirectBranch(Inst, Out, STI);
-    case X86::RETL:
-    case X86::RETIL:
-      return expandReturn(Inst, Out, STI);
-    default:
-      emitPrefixes(Out, STI);
-      Out.EmitInstruction(Inst, STI);
-    }
-  }
-}
-
-bool X86::X86MCNaClExpander::expandInst(const MCInst &Inst, MCStreamer &Out,
-                                        const MCSubtargetInfo &STI) {
-  if (Guard)
-    return false;
-  Guard = true;
-
-  doExpandInst(Inst, Out, STI);
-
-  Guard = false;
-  return true;
-}
diff --git a/lib/Target/X86/MCTargetDesc/X86MCNaClExpander.h b/lib/Target/X86/MCTargetDesc/X86MCNaClExpander.h
deleted file mode 100644
index d6d3e7a0907..00000000000
--- a/lib/Target/X86/MCTargetDesc/X86MCNaClExpander.h
+++ /dev/null
@@ -1,54 +0,0 @@
-//===- X86MCNaClExpander.h --------------------------------------*- C++ -*-===//
-//
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
-//
-//===----------------------------------------------------------------------===//
-//
-// This file declares the X86MCNaClExpander class, the X86 specific
-// subclass of MCNaClExpander.
-//
-//===----------------------------------------------------------------------===//
-#ifndef LLVM_MC_X86MCNACLEXPANDER_H
-#define LLVM_MC_X86MCNACLEXPANDER_H
-
-#include "llvm/ADT/SmallVector.h"
-#include "llvm/MC/MCInst.h"
-#include "llvm/MC/MCInstrInfo.h"
-#include "llvm/MC/MCNaClExpander.h"
-#include "llvm/MC/MCRegisterInfo.h"
-
-namespace llvm {
-class MCContext;
-class MCStreamer;
-class MCSubtargetInfo;
-
-namespace X86 {
-class X86MCNaClExpander : public MCNaClExpander {
-public:
-  X86MCNaClExpander(const MCContext &Ctx, std::unique_ptr<MCRegisterInfo> &&RI,
-                    std::unique_ptr<MCInstrInfo> &&II)
-      : MCNaClExpander(Ctx, std::move(RI), std::move(II)) {}
-
-  bool expandInst(const MCInst &Inst, MCStreamer &Out,
-                  const MCSubtargetInfo &STI) override;
-private:
-  bool Guard = false; // recursion guard
-  SmallVector<MCInst, 4> Prefixes;
-
-  void emitPrefixes(MCStreamer &Out, const MCSubtargetInfo &STI);
-
-  void expandIndirectBranch(const MCInst &Inst, MCStreamer &Out,
-                            const MCSubtargetInfo &STI);
-
-  void expandReturn(const MCInst &Inst, MCStreamer &Out,
-                    const MCSubtargetInfo &STI);
-
-  void doExpandInst(const MCInst &Inst, MCStreamer &Out,
-                    const MCSubtargetInfo &STI);
-};
-}
-}
-#endif
diff --git a/lib/Target/X86/MCTargetDesc/X86MCTargetDesc.cpp b/lib/Target/X86/MCTargetDesc/X86MCTargetDesc.cpp
index 6ea4ab37086..83b4091d766 100644
--- a/lib/Target/X86/MCTargetDesc/X86MCTargetDesc.cpp
+++ b/lib/Target/X86/MCTargetDesc/X86MCTargetDesc.cpp
@@ -15,7 +15,6 @@
 #include "InstPrinter/X86ATTInstPrinter.h"
 #include "InstPrinter/X86IntelInstPrinter.h"
 #include "X86MCAsmInfo.h"
-#include "X86MCNaClExpander.h" // @LOCALMOD
 #include "llvm/ADT/Triple.h"
 #include "llvm/MC/MCCodeGenInfo.h"
 #include "llvm/MC/MCInstrAnalysis.h"
@@ -43,12 +42,11 @@ using namespace llvm;
 #define GET_SUBTARGETINFO_MC_DESC
 #include "X86GenSubtargetInfo.inc"
 
-std::string X86_MC::ParseX86Triple(StringRef TT) {
-  Triple TheTriple(TT);
+std::string X86_MC::ParseX86Triple(const Triple &TT) {
   std::string FS;
-  if (TheTriple.getArch() == Triple::x86_64)
+  if (TT.getArch() == Triple::x86_64)
     FS = "+64bit-mode,-32bit-mode,-16bit-mode";
-  else if (TheTriple.getEnvironment() != Triple::CODE16)
+  else if (TT.getEnvironment() != Triple::CODE16)
     FS = "-64bit-mode,+32bit-mode,-16bit-mode";
   else
     FS = "-64bit-mode,-32bit-mode,+16bit-mode";
@@ -56,7 +54,7 @@ std::string X86_MC::ParseX86Triple(StringRef TT) {
   return FS;
 }
 
-unsigned X86_MC::getDwarfRegFlavour(Triple TT, bool isEH) {
+unsigned X86_MC::getDwarfRegFlavour(const Triple &TT, bool isEH) {
   if (TT.getArch() == Triple::x86_64)
     return DWARFFlavour::X86_64;
 
@@ -76,8 +74,8 @@ void X86_MC::InitLLVM2SEHRegisterMapping(MCRegisterInfo *MRI) {
   }
 }
 
-MCSubtargetInfo *X86_MC::createX86MCSubtargetInfo(StringRef TT, StringRef CPU,
-                                                  StringRef FS) {
+MCSubtargetInfo *X86_MC::createX86MCSubtargetInfo(const Triple &TT,
+                                                  StringRef CPU, StringRef FS) {
   std::string ArchFS = X86_MC::ParseX86Triple(TT);
   if (!FS.empty()) {
     if (!ArchFS.empty())
@@ -90,9 +88,7 @@ MCSubtargetInfo *X86_MC::createX86MCSubtargetInfo(StringRef TT, StringRef CPU,
   if (CPUName.empty())
     CPUName = "generic";
 
-  MCSubtargetInfo *X = new MCSubtargetInfo();
-  InitX86MCSubtargetInfo(X, TT, CPUName, ArchFS);
-  return X;
+  return createX86MCSubtargetInfoImpl(TT, CPUName, ArchFS);
 }
 
 static MCInstrInfo *createX86MCInstrInfo() {
@@ -101,23 +97,20 @@ static MCInstrInfo *createX86MCInstrInfo() {
   return X;
 }
 
-static MCRegisterInfo *createX86MCRegisterInfo(StringRef TT) {
-  Triple TheTriple(TT);
-  unsigned RA = (TheTriple.getArch() == Triple::x86_64)
-    ? X86::RIP     // Should have dwarf #16.
-    : X86::EIP;    // Should have dwarf #8.
+static MCRegisterInfo *createX86MCRegisterInfo(const Triple &TT) {
+  unsigned RA = (TT.getArch() == Triple::x86_64)
+                    ? X86::RIP  // Should have dwarf #16.
+                    : X86::EIP; // Should have dwarf #8.
 
   MCRegisterInfo *X = new MCRegisterInfo();
-  InitX86MCRegisterInfo(X, RA,
-                        X86_MC::getDwarfRegFlavour(TheTriple, false),
-                        X86_MC::getDwarfRegFlavour(TheTriple, true),
-                        RA);
+  InitX86MCRegisterInfo(X, RA, X86_MC::getDwarfRegFlavour(TT, false),
+                        X86_MC::getDwarfRegFlavour(TT, true), RA);
   X86_MC::InitLLVM2SEHRegisterMapping(X);
   return X;
 }
 
-static MCAsmInfo *createX86MCAsmInfo(const MCRegisterInfo &MRI, StringRef TT) {
-  Triple TheTriple(TT);
+static MCAsmInfo *createX86MCAsmInfo(const MCRegisterInfo &MRI,
+                                     const Triple &TheTriple) {
   bool is64Bit = TheTriple.getArch() == Triple::x86_64;
 
   MCAsmInfo *MAI;
@@ -158,24 +151,23 @@ static MCAsmInfo *createX86MCAsmInfo(const MCRegisterInfo &MRI, StringRef TT) {
   return MAI;
 }
 
-static MCCodeGenInfo *createX86MCCodeGenInfo(StringRef TT, Reloc::Model RM,
+static MCCodeGenInfo *createX86MCCodeGenInfo(const Triple &TT, Reloc::Model RM,
                                              CodeModel::Model CM,
                                              CodeGenOpt::Level OL) {
   MCCodeGenInfo *X = new MCCodeGenInfo();
 
-  Triple T(TT);
-  bool is64Bit = T.getArch() == Triple::x86_64;
+  bool is64Bit = TT.getArch() == Triple::x86_64;
 
   if (RM == Reloc::Default) {
     // Darwin defaults to PIC in 64 bit mode and dynamic-no-pic in 32 bit mode.
     // Win64 requires rip-rel addressing, thus we force it to PIC. Otherwise we
     // use static relocation model by default.
-    if (T.isOSDarwin()) {
+    if (TT.isOSDarwin()) {
       if (is64Bit)
         RM = Reloc::PIC_;
       else
         RM = Reloc::DynamicNoPIC;
-    } else if (T.isOSWindows() && is64Bit)
+    } else if (TT.isOSWindows() && is64Bit)
       RM = Reloc::PIC_;
     else
       RM = Reloc::Static;
@@ -188,13 +180,13 @@ static MCCodeGenInfo *createX86MCCodeGenInfo(StringRef TT, Reloc::Model RM,
   if (RM == Reloc::DynamicNoPIC) {
     if (is64Bit)
       RM = Reloc::PIC_;
-    else if (!T.isOSDarwin())
+    else if (!TT.isOSDarwin())
       RM = Reloc::Static;
   }
 
   // If we are on Darwin, disallow static relocation model in X86-64 mode, since
   // the Mach-O file format doesn't support it.
-  if (RM == Reloc::Static && T.isOSDarwin() && is64Bit)
+  if (RM == Reloc::Static && TT.isOSDarwin() && is64Bit)
     RM = Reloc::PIC_;
 
   // For static codegen, if we're not already set, use Small codegen.
@@ -204,7 +196,7 @@ static MCCodeGenInfo *createX86MCCodeGenInfo(StringRef TT, Reloc::Model RM,
     // 64-bit JIT places everything in the same buffer except external funcs.
     CM = is64Bit ? CodeModel::Large : CodeModel::Small;
 
-  X->InitMCCodeGenInfo(RM, CM, OL);
+  X->initMCCodeGenInfo(RM, CM, OL);
   return X;
 }
 
@@ -220,31 +212,20 @@ static MCInstPrinter *createX86MCInstPrinter(const Triple &T,
   return nullptr;
 }
 
-static MCRelocationInfo *createX86MCRelocationInfo(StringRef TT,
+static MCRelocationInfo *createX86MCRelocationInfo(const Triple &TheTriple,
                                                    MCContext &Ctx) {
-  Triple TheTriple(TT);
   if (TheTriple.isOSBinFormatMachO() && TheTriple.getArch() == Triple::x86_64)
     return createX86_64MachORelocationInfo(Ctx);
   else if (TheTriple.isOSBinFormatELF())
     return createX86_64ELFRelocationInfo(Ctx);
   // Default to the stock relocation info.
-  return llvm::createMCRelocationInfo(TT, Ctx);
+  return llvm::createMCRelocationInfo(TheTriple, Ctx);
 }
 
 static MCInstrAnalysis *createX86MCInstrAnalysis(const MCInstrInfo *Info) {
   return new MCInstrAnalysis(Info);
 }
 
-// @LOCALMOD-START
-static void createX86MCNaClExpander(MCStreamer &S,
-                                    std::unique_ptr<MCRegisterInfo> &&RegInfo,
-                                    std::unique_ptr<MCInstrInfo> &&InstInfo) {
-  auto *Exp = new X86::X86MCNaClExpander(S.getContext(), std::move(RegInfo),
-                                         std::move(InstInfo));
-  S.setNaClExpander(Exp);
-}
-// @LOCALMOD-END
-
 // Force static initialization.
 extern "C" void LLVMInitializeX86TargetMC() {
   for (Target *T : {&TheX86_32Target, &TheX86_64Target}) {
@@ -269,16 +250,15 @@ extern "C" void LLVMInitializeX86TargetMC() {
 
     // Register the code emitter.
     TargetRegistry::RegisterMCCodeEmitter(*T, createX86MCCodeEmitter);
-#ifndef __native_client__
+
     // Register the object streamer.
     TargetRegistry::RegisterCOFFStreamer(*T, createX86WinCOFFStreamer);
-#endif
+
     // Register the MCInstPrinter.
     TargetRegistry::RegisterMCInstPrinter(*T, createX86MCInstPrinter);
 
     // Register the MC relocation info.
     TargetRegistry::RegisterMCRelocationInfo(*T, createX86MCRelocationInfo);
-
   }
 
   // Register the asm backend.
@@ -286,9 +266,4 @@ extern "C" void LLVMInitializeX86TargetMC() {
                                        createX86_32AsmBackend);
   TargetRegistry::RegisterMCAsmBackend(TheX86_64Target,
                                        createX86_64AsmBackend);
-
-  // @LOCALMOD-START
-  TargetRegistry::RegisterMCNaClExpander(TheX86_32Target,
-                                         createX86MCNaClExpander);
-  // @LOCALMOD-END
 }
diff --git a/lib/Target/X86/MCTargetDesc/X86MCTargetDesc.h b/lib/Target/X86/MCTargetDesc/X86MCTargetDesc.h
index dcdae1dbc46..6221baba179 100644
--- a/lib/Target/X86/MCTargetDesc/X86MCTargetDesc.h
+++ b/lib/Target/X86/MCTargetDesc/X86MCTargetDesc.h
@@ -52,16 +52,16 @@ namespace N86 {
 }
 
 namespace X86_MC {
-  std::string ParseX86Triple(StringRef TT);
+std::string ParseX86Triple(const Triple &TT);
 
-  unsigned getDwarfRegFlavour(Triple TT, bool isEH);
+unsigned getDwarfRegFlavour(const Triple &TT, bool isEH);
 
-  void InitLLVM2SEHRegisterMapping(MCRegisterInfo *MRI);
+void InitLLVM2SEHRegisterMapping(MCRegisterInfo *MRI);
 
-  /// Create a X86 MCSubtargetInfo instance. This is exposed so Asm parser, etc.
-  /// do not need to go through TargetRegistry.
-  MCSubtargetInfo *createX86MCSubtargetInfo(StringRef TT, StringRef CPU,
-                                            StringRef FS);
+/// Create a X86 MCSubtargetInfo instance. This is exposed so Asm parser, etc.
+/// do not need to go through TargetRegistry.
+MCSubtargetInfo *createX86MCSubtargetInfo(const Triple &TT, StringRef CPU,
+                                          StringRef FS);
 }
 
 MCCodeEmitter *createX86MCCodeEmitter(const MCInstrInfo &MCII,
@@ -69,9 +69,9 @@ MCCodeEmitter *createX86MCCodeEmitter(const MCInstrInfo &MCII,
                                       MCContext &Ctx);
 
 MCAsmBackend *createX86_32AsmBackend(const Target &T, const MCRegisterInfo &MRI,
-                                     StringRef TT, StringRef CPU);
+                                     const Triple &TT, StringRef CPU);
 MCAsmBackend *createX86_64AsmBackend(const Target &T, const MCRegisterInfo &MRI,
-                                     StringRef TT, StringRef CPU);
+                                     const Triple &TT, StringRef CPU);
 
 /// Construct an X86 Windows COFF machine code streamer which will generate
 /// PE/COFF format object files.
diff --git a/lib/Target/X86/MCTargetDesc/X86MachORelocationInfo.cpp b/lib/Target/X86/MCTargetDesc/X86MachORelocationInfo.cpp
index 81749fcc6b4..9bfe999424f 100644
--- a/lib/Target/X86/MCTargetDesc/X86MachORelocationInfo.cpp
+++ b/lib/Target/X86/MCTargetDesc/X86MachORelocationInfo.cpp
@@ -25,47 +25,50 @@ class X86_64MachORelocationInfo : public MCRelocationInfo {
   X86_64MachORelocationInfo(MCContext &Ctx) : MCRelocationInfo(Ctx) {}
 
   const MCExpr *createExprForRelocation(RelocationRef Rel) override {
-    const MachOObjectFile *Obj = cast<MachOObjectFile>(Rel.getObjectFile());
+    const MachOObjectFile *Obj = cast<MachOObjectFile>(Rel.getObject());
 
-    uint64_t RelType; Rel.getType(RelType);
+    uint64_t RelType = Rel.getType();
     symbol_iterator SymI = Rel.getSymbol();
 
-    StringRef SymName; SymI->getName(SymName);
-    uint64_t  SymAddr; SymI->getAddress(SymAddr);
+    ErrorOr<StringRef> SymNameOrErr = SymI->getName();
+    if (std::error_code EC = SymNameOrErr.getError())
+      report_fatal_error(EC.message());
+    StringRef SymName = *SymNameOrErr;
+    uint64_t SymAddr = SymI->getValue();
 
     any_relocation_info RE = Obj->getRelocation(Rel.getRawDataRefImpl());
     bool isPCRel = Obj->getAnyRelocationPCRel(RE);
 
-    MCSymbol *Sym = Ctx.GetOrCreateSymbol(SymName);
+    MCSymbol *Sym = Ctx.getOrCreateSymbol(SymName);
     // FIXME: check that the value is actually the same.
     if (!Sym->isVariable())
-      Sym->setVariableValue(MCConstantExpr::Create(SymAddr, Ctx));
+      Sym->setVariableValue(MCConstantExpr::create(SymAddr, Ctx));
     const MCExpr *Expr = nullptr;
 
     switch(RelType) {
     case X86_64_RELOC_TLV:
-      Expr = MCSymbolRefExpr::Create(Sym, MCSymbolRefExpr::VK_TLVP, Ctx);
+      Expr = MCSymbolRefExpr::create(Sym, MCSymbolRefExpr::VK_TLVP, Ctx);
       break;
     case X86_64_RELOC_SIGNED_4:
-      Expr = MCBinaryExpr::CreateAdd(MCSymbolRefExpr::Create(Sym, Ctx),
-                                     MCConstantExpr::Create(4, Ctx),
+      Expr = MCBinaryExpr::createAdd(MCSymbolRefExpr::create(Sym, Ctx),
+                                     MCConstantExpr::create(4, Ctx),
                                      Ctx);
       break;
     case X86_64_RELOC_SIGNED_2:
-      Expr = MCBinaryExpr::CreateAdd(MCSymbolRefExpr::Create(Sym, Ctx),
-                                     MCConstantExpr::Create(2, Ctx),
+      Expr = MCBinaryExpr::createAdd(MCSymbolRefExpr::create(Sym, Ctx),
+                                     MCConstantExpr::create(2, Ctx),
                                      Ctx);
       break;
     case X86_64_RELOC_SIGNED_1:
-      Expr = MCBinaryExpr::CreateAdd(MCSymbolRefExpr::Create(Sym, Ctx),
-                                     MCConstantExpr::Create(1, Ctx),
+      Expr = MCBinaryExpr::createAdd(MCSymbolRefExpr::create(Sym, Ctx),
+                                     MCConstantExpr::create(1, Ctx),
                                      Ctx);
       break;
     case X86_64_RELOC_GOT_LOAD:
-      Expr = MCSymbolRefExpr::Create(Sym, MCSymbolRefExpr::VK_GOTPCREL, Ctx);
+      Expr = MCSymbolRefExpr::create(Sym, MCSymbolRefExpr::VK_GOTPCREL, Ctx);
       break;
     case X86_64_RELOC_GOT:
-      Expr = MCSymbolRefExpr::Create(Sym, isPCRel ?
+      Expr = MCSymbolRefExpr::create(Sym, isPCRel ?
                                      MCSymbolRefExpr::VK_GOTPCREL :
                                      MCSymbolRefExpr::VK_GOT,
                                      Ctx);
@@ -84,25 +87,25 @@ class X86_64MachORelocationInfo : public MCRelocationInfo {
           report_fatal_error("Expected X86_64_RELOC_UNSIGNED after "
                              "X86_64_RELOC_SUBTRACTOR.");
 
-        const MCExpr *LHS = MCSymbolRefExpr::Create(Sym, Ctx);
+        const MCExpr *LHS = MCSymbolRefExpr::create(Sym, Ctx);
 
         symbol_iterator RSymI = Rel.getSymbol();
-        uint64_t RSymAddr;
-        RSymI->getAddress(RSymAddr);
-        StringRef RSymName;
-        RSymI->getName(RSymName);
+        uint64_t RSymAddr = RSymI->getValue();
+        ErrorOr<StringRef> RSymName = RSymI->getName();
+        if (std::error_code EC = RSymName.getError())
+          report_fatal_error(EC.message());
 
-        MCSymbol *RSym = Ctx.GetOrCreateSymbol(RSymName);
+        MCSymbol *RSym = Ctx.getOrCreateSymbol(*RSymName);
         if (!RSym->isVariable())
-          RSym->setVariableValue(MCConstantExpr::Create(RSymAddr, Ctx));
+          RSym->setVariableValue(MCConstantExpr::create(RSymAddr, Ctx));
 
-        const MCExpr *RHS = MCSymbolRefExpr::Create(RSym, Ctx);
+        const MCExpr *RHS = MCSymbolRefExpr::create(RSym, Ctx);
 
-        Expr = MCBinaryExpr::CreateSub(LHS, RHS, Ctx);
+        Expr = MCBinaryExpr::createSub(LHS, RHS, Ctx);
         break;
       }
     default:
-      Expr = MCSymbolRefExpr::Create(Sym, Ctx);
+      Expr = MCSymbolRefExpr::create(Sym, Ctx);
       break;
     }
     return Expr;
diff --git a/lib/Target/X86/MCTargetDesc/X86MachObjectWriter.cpp b/lib/Target/X86/MCTargetDesc/X86MachObjectWriter.cpp
index 38539cd7bee..9e801fc8f19 100644
--- a/lib/Target/X86/MCTargetDesc/X86MachObjectWriter.cpp
+++ b/lib/Target/X86/MCTargetDesc/X86MachObjectWriter.cpp
@@ -25,7 +25,7 @@ using namespace llvm;
 
 namespace {
 class X86MachObjectWriter : public MCMachObjectTargetWriter {
-  bool RecordScatteredRelocation(MachObjectWriter *Writer,
+  bool recordScatteredRelocation(MachObjectWriter *Writer,
                                  const MCAssembler &Asm,
                                  const MCAsmLayout &Layout,
                                  const MCFragment *Fragment,
@@ -33,7 +33,7 @@ class X86MachObjectWriter : public MCMachObjectTargetWriter {
                                  MCValue Target,
                                  unsigned Log2Size,
                                  uint64_t &FixedValue);
-  void RecordTLVPRelocation(MachObjectWriter *Writer,
+  void recordTLVPRelocation(MachObjectWriter *Writer,
                             const MCAssembler &Asm,
                             const MCAsmLayout &Layout,
                             const MCFragment *Fragment,
@@ -54,12 +54,10 @@ class X86MachObjectWriter : public MCMachObjectTargetWriter {
                               MCValue Target, uint64_t &FixedValue);
 
 public:
-  X86MachObjectWriter(bool Is64Bit, uint32_t CPUType,
-                      uint32_t CPUSubtype)
-    : MCMachObjectTargetWriter(Is64Bit, CPUType, CPUSubtype,
-                               /*UseAggressiveSymbolFolding=*/Is64Bit) {}
+  X86MachObjectWriter(bool Is64Bit, uint32_t CPUType, uint32_t CPUSubtype)
+      : MCMachObjectTargetWriter(Is64Bit, CPUType, CPUSubtype) {}
 
-  void RecordRelocation(MachObjectWriter *Writer, MCAssembler &Asm,
+  void recordRelocation(MachObjectWriter *Writer, MCAssembler &Asm,
                         const MCAsmLayout &Layout, const MCFragment *Fragment,
                         const MCFixup &Fixup, MCValue Target,
                         uint64_t &FixedValue) override {
@@ -113,7 +111,7 @@ void X86MachObjectWriter::RecordX86_64Relocation(
   unsigned Index = 0;
   unsigned IsExtern = 0;
   unsigned Type = 0;
-  const MCSymbolData *RelSymbol = nullptr;
+  const MCSymbol *RelSymbol = nullptr;
 
   Value = Target.getConstant();
 
@@ -141,15 +139,13 @@ void X86MachObjectWriter::RecordX86_64Relocation(
   } else if (Target.getSymB()) { // A - B + constant
     const MCSymbol *A = &Target.getSymA()->getSymbol();
     if (A->isTemporary())
-      A = &A->AliasedSymbol();
-    const MCSymbolData &A_SD = Asm.getSymbolData(*A);
-    const MCSymbolData *A_Base = Asm.getAtom(&A_SD);
+      A = &Writer->findAliasedSymbol(*A);
+    const MCSymbol *A_Base = Asm.getAtom(*A);
 
     const MCSymbol *B = &Target.getSymB()->getSymbol();
     if (B->isTemporary())
-      B = &B->AliasedSymbol();
-    const MCSymbolData &B_SD = Asm.getSymbolData(*B);
-    const MCSymbolData *B_Base = Asm.getAtom(&B_SD);
+      B = &Writer->findAliasedSymbol(*B);
+    const MCSymbol *B_Base = Asm.getAtom(*B);
 
     // Neither symbol can be modified.
     if (Target.getSymA()->getKind() != MCSymbolRefExpr::VK_None ||
@@ -179,18 +175,18 @@ void X86MachObjectWriter::RecordX86_64Relocation(
     // non-relocatable expression.
     if (A->isUndefined() || B->isUndefined()) {
       StringRef Name = A->isUndefined() ? A->getName() : B->getName();
-      Asm.getContext().FatalError(Fixup.getLoc(),
+      Asm.getContext().reportFatalError(Fixup.getLoc(),
         "unsupported relocation with subtraction expression, symbol '" +
         Name + "' can not be undefined in a subtraction expression");
     }
 
-    Value += Writer->getSymbolAddress(&A_SD, Layout) -
-      (!A_Base ? 0 : Writer->getSymbolAddress(A_Base, Layout));
-    Value -= Writer->getSymbolAddress(&B_SD, Layout) -
-      (!B_Base ? 0 : Writer->getSymbolAddress(B_Base, Layout));
+    Value += Writer->getSymbolAddress(*A, Layout) -
+             (!A_Base ? 0 : Writer->getSymbolAddress(*A_Base, Layout));
+    Value -= Writer->getSymbolAddress(*B, Layout) -
+             (!B_Base ? 0 : Writer->getSymbolAddress(*B_Base, Layout));
 
     if (!A_Base)
-      Index = A_SD.getFragment()->getParent()->getOrdinal() + 1;
+      Index = A->getFragment()->getParent()->getOrdinal() + 1;
     Type = MachO::X86_64_RELOC_UNSIGNED;
 
     MachO::any_relocation_info MRE;
@@ -202,25 +198,24 @@ void X86MachObjectWriter::RecordX86_64Relocation(
     if (B_Base)
       RelSymbol = B_Base;
     else
-      Index = B_SD.getFragment()->getParent()->getOrdinal() + 1;
+      Index = B->getFragment()->getParent()->getOrdinal() + 1;
     Type = MachO::X86_64_RELOC_SUBTRACTOR;
   } else {
     const MCSymbol *Symbol = &Target.getSymA()->getSymbol();
     if (Symbol->isTemporary() && Value) {
       const MCSection &Sec = Symbol->getSection();
       if (!Asm.getContext().getAsmInfo()->isSectionAtomizableBySymbols(Sec))
-        Asm.addLocalUsedInReloc(*Symbol);
+        Symbol->setUsedInReloc();
     }
-    const MCSymbolData &SD = Asm.getSymbolData(*Symbol);
-    RelSymbol = Asm.getAtom(&SD);
+    RelSymbol = Asm.getAtom(*Symbol);
 
     // Relocations inside debug sections always use local relocations when
     // possible. This seems to be done because the debugger doesn't fully
     // understand x86_64 relocation entries, and expects to find values that
     // have already been fixed up.
     if (Symbol->isInSection()) {
-      const MCSectionMachO &Section = static_cast<const MCSectionMachO&>(
-        Fragment->getParent()->getSection());
+      const MCSectionMachO &Section =
+          static_cast<const MCSectionMachO &>(*Fragment->getParent());
       if (Section.hasAttribute(MachO::S_ATTR_DEBUG))
         RelSymbol = nullptr;
     }
@@ -230,20 +225,20 @@ void X86MachObjectWriter::RecordX86_64Relocation(
     // non-local symbol).
     if (RelSymbol) {
       // Add the local offset, if needed.
-      if (RelSymbol != &SD)
-        Value +=
-            Layout.getSymbolOffset(&SD) - Layout.getSymbolOffset(RelSymbol);
+      if (RelSymbol != Symbol)
+        Value += Layout.getSymbolOffset(*Symbol) -
+                 Layout.getSymbolOffset(*RelSymbol);
     } else if (Symbol->isInSection() && !Symbol->isVariable()) {
       // The index is the section ordinal (1-based).
-      Index = SD.getFragment()->getParent()->getOrdinal() + 1;
-      Value += Writer->getSymbolAddress(&SD, Layout);
+      Index = Symbol->getFragment()->getParent()->getOrdinal() + 1;
+      Value += Writer->getSymbolAddress(*Symbol, Layout);
 
       if (IsPCRel)
         Value -= FixupAddress + (1 << Log2Size);
     } else if (Symbol->isVariable()) {
       const MCExpr *Value = Symbol->getVariableValue();
       int64_t Res;
-      bool isAbs = Value->EvaluateAsAbsolute(Res, Layout,
+      bool isAbs = Value->evaluateAsAbsolute(Res, Layout,
                                              Writer->getSectionAddressMap());
       if (isAbs) {
         FixedValue = Res;
@@ -339,7 +334,7 @@ void X86MachObjectWriter::RecordX86_64Relocation(
   Writer->addRelocation(RelSymbol, Fragment->getParent(), MRE);
 }
 
-bool X86MachObjectWriter::RecordScatteredRelocation(MachObjectWriter *Writer,
+bool X86MachObjectWriter::recordScatteredRelocation(MachObjectWriter *Writer,
                                                     const MCAssembler &Asm,
                                                     const MCAsmLayout &Layout,
                                                     const MCFragment *Fragment,
@@ -354,22 +349,21 @@ bool X86MachObjectWriter::RecordScatteredRelocation(MachObjectWriter *Writer,
 
   // See <reloc.h>.
   const MCSymbol *A = &Target.getSymA()->getSymbol();
-  const MCSymbolData *A_SD = &Asm.getSymbolData(*A);
 
-  if (!A_SD->getFragment())
+  if (!A->getFragment())
     report_fatal_error("symbol '" + A->getName() +
                        "' can not be undefined in a subtraction expression",
                        false);
 
-  uint32_t Value = Writer->getSymbolAddress(A_SD, Layout);
-  uint64_t SecAddr = Writer->getSectionAddress(A_SD->getFragment()->getParent());
+  uint32_t Value = Writer->getSymbolAddress(*A, Layout);
+  uint64_t SecAddr = Writer->getSectionAddress(A->getFragment()->getParent());
   FixedValue += SecAddr;
   uint32_t Value2 = 0;
 
   if (const MCSymbolRefExpr *B = Target.getSymB()) {
-    const MCSymbolData *B_SD = &Asm.getSymbolData(B->getSymbol());
+    const MCSymbol *SB = &B->getSymbol();
 
-    if (!B_SD->getFragment())
+    if (!SB->getFragment())
       report_fatal_error("symbol '" + B->getSymbol().getName() +
                          "' can not be undefined in a subtraction expression",
                          false);
@@ -379,10 +373,10 @@ bool X86MachObjectWriter::RecordScatteredRelocation(MachObjectWriter *Writer,
     // Note that there is no longer any semantic difference between these two
     // relocation types from the linkers point of view, this is done solely for
     // pedantic compatibility with 'as'.
-    Type = A_SD->isExternal() ? (unsigned)MachO::GENERIC_RELOC_SECTDIFF :
-      (unsigned)MachO::GENERIC_RELOC_LOCAL_SECTDIFF;
-    Value2 = Writer->getSymbolAddress(B_SD, Layout);
-    FixedValue -= Writer->getSectionAddress(B_SD->getFragment()->getParent());
+    Type = A->isExternal() ? (unsigned)MachO::GENERIC_RELOC_SECTDIFF
+                           : (unsigned)MachO::GENERIC_RELOC_LOCAL_SECTDIFF;
+    Value2 = Writer->getSymbolAddress(B->getSymbol(), Layout);
+    FixedValue -= Writer->getSectionAddress(SB->getFragment()->getParent());
   }
 
   // Relocations are written out in reverse order, so the PAIR comes first.
@@ -393,7 +387,7 @@ bool X86MachObjectWriter::RecordScatteredRelocation(MachObjectWriter *Writer,
     if (FixupOffset > 0xffffff) {
       char Buffer[32];
       format("0x%x", FixupOffset).print(Buffer, sizeof(Buffer));
-      Asm.getContext().FatalError(Fixup.getLoc(),
+      Asm.getContext().reportFatalError(Fixup.getLoc(),
                          Twine("Section too large, can't encode "
                                 "r_address (") + Buffer +
                          ") into 24 bits of scattered "
@@ -434,7 +428,7 @@ bool X86MachObjectWriter::RecordScatteredRelocation(MachObjectWriter *Writer,
   return true;
 }
 
-void X86MachObjectWriter::RecordTLVPRelocation(MachObjectWriter *Writer,
+void X86MachObjectWriter::recordTLVPRelocation(MachObjectWriter *Writer,
                                                const MCAssembler &Asm,
                                                const MCAsmLayout &Layout,
                                                const MCFragment *Fragment,
@@ -449,9 +443,6 @@ void X86MachObjectWriter::RecordTLVPRelocation(MachObjectWriter *Writer,
   uint32_t Value = Layout.getFragmentOffset(Fragment)+Fixup.getOffset();
   unsigned IsPCRel = 0;
 
-  // Get the symbol data.
-  const MCSymbolData *SD_A = &Asm.getSymbolData(Target.getSymA()->getSymbol());
-
   // We're only going to have a second symbol in pic mode and it'll be a
   // subtraction from the picbase. For 32-bit pic the addend is the difference
   // between the picbase and the next address.  For 32-bit static the addend is
@@ -460,11 +451,11 @@ void X86MachObjectWriter::RecordTLVPRelocation(MachObjectWriter *Writer,
     // If this is a subtraction then we're pcrel.
     uint32_t FixupAddress =
       Writer->getFragmentAddress(Fragment, Layout) + Fixup.getOffset();
-    const MCSymbolData *SD_B =
-        &Asm.getSymbolData(Target.getSymB()->getSymbol());
     IsPCRel = 1;
-    FixedValue = (FixupAddress - Writer->getSymbolAddress(SD_B, Layout) +
-                  Target.getConstant());
+    FixedValue =
+        FixupAddress -
+        Writer->getSymbolAddress(Target.getSymB()->getSymbol(), Layout) +
+        Target.getConstant();
     FixedValue += 1ULL << Log2Size;
   } else {
     FixedValue = 0;
@@ -475,7 +466,8 @@ void X86MachObjectWriter::RecordTLVPRelocation(MachObjectWriter *Writer,
   MRE.r_word0 = Value;
   MRE.r_word1 =
       (IsPCRel << 24) | (Log2Size << 25) | (MachO::GENERIC_RELOC_TLV << 28);
-  Writer->addRelocation(SD_A, Fragment->getParent(), MRE);
+  Writer->addRelocation(&Target.getSymA()->getSymbol(), Fragment->getParent(),
+                        MRE);
 }
 
 void X86MachObjectWriter::RecordX86Relocation(MachObjectWriter *Writer,
@@ -491,7 +483,7 @@ void X86MachObjectWriter::RecordX86Relocation(MachObjectWriter *Writer,
   // If this is a 32-bit TLVP reloc it's handled a bit differently.
   if (Target.getSymA() &&
       Target.getSymA()->getKind() == MCSymbolRefExpr::VK_TLVP) {
-    RecordTLVPRelocation(Writer, Asm, Layout, Fragment, Fixup, Target,
+    recordTLVPRelocation(Writer, Asm, Layout, Fragment, Fixup, Target,
                          FixedValue);
     return;
   }
@@ -500,15 +492,15 @@ void X86MachObjectWriter::RecordX86Relocation(MachObjectWriter *Writer,
   // scattered relocation entry. Differences always require scattered
   // relocations.
   if (Target.getSymB()) {
-    RecordScatteredRelocation(Writer, Asm, Layout, Fragment, Fixup,
+    recordScatteredRelocation(Writer, Asm, Layout, Fragment, Fixup,
                               Target, Log2Size, FixedValue);
     return;
   }
 
   // Get the symbol data, if any.
-  const MCSymbolData *SD = nullptr;
+  const MCSymbol *A = nullptr;
   if (Target.getSymA())
-    SD = &Asm.getSymbolData(Target.getSymA()->getSymbol());
+    A = &Target.getSymA()->getSymbol();
 
   // If this is an internal relocation with an offset, it also needs a scattered
   // relocation entry.
@@ -516,18 +508,18 @@ void X86MachObjectWriter::RecordX86Relocation(MachObjectWriter *Writer,
   if (IsPCRel)
     Offset += 1 << Log2Size;
   // Try to record the scattered relocation if needed. Fall back to non
-  // scattered if necessary (see comments in RecordScatteredRelocation()
+  // scattered if necessary (see comments in recordScatteredRelocation()
   // for details).
-  if (Offset && SD && !Writer->doesSymbolRequireExternRelocation(SD) &&
-      RecordScatteredRelocation(Writer, Asm, Layout, Fragment, Fixup,
-                                Target, Log2Size, FixedValue))
+  if (Offset && A && !Writer->doesSymbolRequireExternRelocation(*A) &&
+      recordScatteredRelocation(Writer, Asm, Layout, Fragment, Fixup, Target,
+                                Log2Size, FixedValue))
     return;
 
   // See <reloc.h>.
   uint32_t FixupOffset = Layout.getFragmentOffset(Fragment)+Fixup.getOffset();
   unsigned Index = 0;
   unsigned Type = 0;
-  const MCSymbolData *RelSymbol = nullptr;
+  const MCSymbol *RelSymbol = nullptr;
 
   if (Target.isAbsolute()) { // constant
     // SymbolNum of 0 indicates the absolute section.
@@ -537,29 +529,28 @@ void X86MachObjectWriter::RecordX86Relocation(MachObjectWriter *Writer,
     Type = MachO::GENERIC_RELOC_VANILLA;
   } else {
     // Resolve constant variables.
-    if (SD->getSymbol().isVariable()) {
+    if (A->isVariable()) {
       int64_t Res;
-      if (SD->getSymbol().getVariableValue()->EvaluateAsAbsolute(
-            Res, Layout, Writer->getSectionAddressMap())) {
+      if (A->getVariableValue()->evaluateAsAbsolute(
+              Res, Layout, Writer->getSectionAddressMap())) {
         FixedValue = Res;
         return;
       }
     }
 
     // Check whether we need an external or internal relocation.
-    if (Writer->doesSymbolRequireExternRelocation(SD)) {
-      RelSymbol = SD;
+    if (Writer->doesSymbolRequireExternRelocation(*A)) {
+      RelSymbol = A;
       // For external relocations, make sure to offset the fixup value to
       // compensate for the addend of the symbol address, if it was
       // undefined. This occurs with weak definitions, for example.
-      if (!SD->getSymbol().isUndefined())
-        FixedValue -= Layout.getSymbolOffset(SD);
+      if (!A->isUndefined())
+        FixedValue -= Layout.getSymbolOffset(*A);
     } else {
       // The index is the section ordinal (1-based).
-      const MCSectionData &SymSD = Asm.getSectionData(
-        SD->getSymbol().getSection());
-      Index = SymSD.getOrdinal() + 1;
-      FixedValue += Writer->getSectionAddress(&SymSD);
+      const MCSection &Sec = A->getSection();
+      Index = Sec.getOrdinal() + 1;
+      FixedValue += Writer->getSectionAddress(&Sec);
     }
     if (IsPCRel)
       FixedValue -= Writer->getSectionAddress(Fragment->getParent());
diff --git a/lib/Target/X86/Utils/X86ShuffleDecode.cpp b/lib/Target/X86/Utils/X86ShuffleDecode.cpp
index a7101e4febb..cae865a4081 100644
--- a/lib/Target/X86/Utils/X86ShuffleDecode.cpp
+++ b/lib/Target/X86/Utils/X86ShuffleDecode.cpp
@@ -1,434 +1,506 @@
-//===-- X86ShuffleDecode.cpp - X86 shuffle decode logic -------------------===//
-//
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
-//
-//===----------------------------------------------------------------------===//
-//
-// Define several functions to decode x86 specific shuffle semantics into a
-// generic vector mask.
-//
-//===----------------------------------------------------------------------===//
-
-#include "X86ShuffleDecode.h"
-#include "llvm/IR/Constants.h"
-#include "llvm/CodeGen/MachineValueType.h"
-
-//===----------------------------------------------------------------------===//
-//  Vector Mask Decoding
-//===----------------------------------------------------------------------===//
-
-namespace llvm {
-
-void DecodeINSERTPSMask(unsigned Imm, SmallVectorImpl<int> &ShuffleMask) {
-  // Defaults the copying the dest value.
-  ShuffleMask.push_back(0);
-  ShuffleMask.push_back(1);
-  ShuffleMask.push_back(2);
-  ShuffleMask.push_back(3);
-
-  // Decode the immediate.
-  unsigned ZMask = Imm & 15;
-  unsigned CountD = (Imm >> 4) & 3;
-  unsigned CountS = (Imm >> 6) & 3;
-
-  // CountS selects which input element to use.
-  unsigned InVal = 4+CountS;
-  // CountD specifies which element of destination to update.
-  ShuffleMask[CountD] = InVal;
-  // ZMask zaps values, potentially overriding the CountD elt.
-  if (ZMask & 1) ShuffleMask[0] = SM_SentinelZero;
-  if (ZMask & 2) ShuffleMask[1] = SM_SentinelZero;
-  if (ZMask & 4) ShuffleMask[2] = SM_SentinelZero;
-  if (ZMask & 8) ShuffleMask[3] = SM_SentinelZero;
-}
-
-// <3,1> or <6,7,2,3>
-void DecodeMOVHLPSMask(unsigned NElts, SmallVectorImpl<int> &ShuffleMask) {
-  for (unsigned i = NElts/2; i != NElts; ++i)
-    ShuffleMask.push_back(NElts+i);
-
-  for (unsigned i = NElts/2; i != NElts; ++i)
-    ShuffleMask.push_back(i);
-}
-
-// <0,2> or <0,1,4,5>
-void DecodeMOVLHPSMask(unsigned NElts, SmallVectorImpl<int> &ShuffleMask) {
-  for (unsigned i = 0; i != NElts/2; ++i)
-    ShuffleMask.push_back(i);
-
-  for (unsigned i = 0; i != NElts/2; ++i)
-    ShuffleMask.push_back(NElts+i);
-}
-
-void DecodeMOVSLDUPMask(MVT VT, SmallVectorImpl<int> &ShuffleMask) {
-  unsigned NumElts = VT.getVectorNumElements();
-  for (int i = 0, e = NumElts / 2; i < e; ++i) {
-    ShuffleMask.push_back(2 * i);
-    ShuffleMask.push_back(2 * i);
-  }
-}
-
-void DecodeMOVSHDUPMask(MVT VT, SmallVectorImpl<int> &ShuffleMask) {
-  unsigned NumElts = VT.getVectorNumElements();
-  for (int i = 0, e = NumElts / 2; i < e; ++i) {
-    ShuffleMask.push_back(2 * i + 1);
-    ShuffleMask.push_back(2 * i + 1);
-  }
-}
-
-void DecodeMOVDDUPMask(MVT VT, SmallVectorImpl<int> &ShuffleMask) {
-  unsigned VectorSizeInBits = VT.getSizeInBits();
-  unsigned ScalarSizeInBits = VT.getScalarSizeInBits();
-  unsigned NumElts = VT.getVectorNumElements();
-  unsigned NumLanes = VectorSizeInBits / 128;
-  unsigned NumLaneElts = NumElts / NumLanes;
-  unsigned NumLaneSubElts = 64 / ScalarSizeInBits;
-
-  for (unsigned l = 0; l < NumElts; l += NumLaneElts)
-    for (unsigned i = 0; i < NumLaneElts; i += NumLaneSubElts)
-      for (unsigned s = 0; s != NumLaneSubElts; s++)
-        ShuffleMask.push_back(l + s);
-}
-
-void DecodePSLLDQMask(MVT VT, unsigned Imm, SmallVectorImpl<int> &ShuffleMask) {
-  unsigned VectorSizeInBits = VT.getSizeInBits();
-  unsigned NumElts = VectorSizeInBits / 8;
-  unsigned NumLanes = VectorSizeInBits / 128;
-  unsigned NumLaneElts = NumElts / NumLanes;
-
-  for (unsigned l = 0; l < NumElts; l += NumLaneElts)
-    for (unsigned i = 0; i < NumLaneElts; ++i) {
-      int M = SM_SentinelZero;
-      if (i >= Imm) M = i - Imm + l;
-      ShuffleMask.push_back(M);
-    }
-}
-
-void DecodePSRLDQMask(MVT VT, unsigned Imm, SmallVectorImpl<int> &ShuffleMask) {
-  unsigned VectorSizeInBits = VT.getSizeInBits();
-  unsigned NumElts = VectorSizeInBits / 8;
-  unsigned NumLanes = VectorSizeInBits / 128;
-  unsigned NumLaneElts = NumElts / NumLanes;
-
-  for (unsigned l = 0; l < NumElts; l += NumLaneElts)
-    for (unsigned i = 0; i < NumLaneElts; ++i) {
-      unsigned Base = i + Imm;
-      int M = Base + l;
-      if (Base >= NumLaneElts) M = SM_SentinelZero;
-      ShuffleMask.push_back(M);
-    }
-}
-
-void DecodePALIGNRMask(MVT VT, unsigned Imm,
-                       SmallVectorImpl<int> &ShuffleMask) {
-  unsigned NumElts = VT.getVectorNumElements();
-  unsigned Offset = Imm * (VT.getVectorElementType().getSizeInBits() / 8);
-
-  unsigned NumLanes = VT.getSizeInBits() / 128;
-  unsigned NumLaneElts = NumElts / NumLanes;
-
-  for (unsigned l = 0; l != NumElts; l += NumLaneElts) {
-    for (unsigned i = 0; i != NumLaneElts; ++i) {
-      unsigned Base = i + Offset;
-      // if i+offset is out of this lane then we actually need the other source
-      if (Base >= NumLaneElts) Base += NumElts - NumLaneElts;
-      ShuffleMask.push_back(Base + l);
-    }
-  }
-}
-
-/// DecodePSHUFMask - This decodes the shuffle masks for pshufd, and vpermilp*.
-/// VT indicates the type of the vector allowing it to handle different
-/// datatypes and vector widths.
-void DecodePSHUFMask(MVT VT, unsigned Imm, SmallVectorImpl<int> &ShuffleMask) {
-  unsigned NumElts = VT.getVectorNumElements();
-
-  unsigned NumLanes = VT.getSizeInBits() / 128;
-  unsigned NumLaneElts = NumElts / NumLanes;
-
-  unsigned NewImm = Imm;
-  for (unsigned l = 0; l != NumElts; l += NumLaneElts) {
-    for (unsigned i = 0; i != NumLaneElts; ++i) {
-      ShuffleMask.push_back(NewImm % NumLaneElts + l);
-      NewImm /= NumLaneElts;
-    }
-    if (NumLaneElts == 4) NewImm = Imm; // reload imm
-  }
-}
-
-void DecodePSHUFHWMask(MVT VT, unsigned Imm,
-                       SmallVectorImpl<int> &ShuffleMask) {
-  unsigned NumElts = VT.getVectorNumElements();
-
-  for (unsigned l = 0; l != NumElts; l += 8) {
-    unsigned NewImm = Imm;
-    for (unsigned i = 0, e = 4; i != e; ++i) {
-      ShuffleMask.push_back(l + i);
-    }
-    for (unsigned i = 4, e = 8; i != e; ++i) {
-      ShuffleMask.push_back(l + 4 + (NewImm & 3));
-      NewImm >>= 2;
-    }
-  }
-}
-
-void DecodePSHUFLWMask(MVT VT, unsigned Imm,
-                       SmallVectorImpl<int> &ShuffleMask) {
-  unsigned NumElts = VT.getVectorNumElements();
-
-  for (unsigned l = 0; l != NumElts; l += 8) {
-    unsigned NewImm = Imm;
-    for (unsigned i = 0, e = 4; i != e; ++i) {
-      ShuffleMask.push_back(l + (NewImm & 3));
-      NewImm >>= 2;
-    }
-    for (unsigned i = 4, e = 8; i != e; ++i) {
-      ShuffleMask.push_back(l + i);
-    }
-  }
-}
-
-/// DecodeSHUFPMask - This decodes the shuffle masks for shufp*. VT indicates
-/// the type of the vector allowing it to handle different datatypes and vector
-/// widths.
-void DecodeSHUFPMask(MVT VT, unsigned Imm, SmallVectorImpl<int> &ShuffleMask) {
-  unsigned NumElts = VT.getVectorNumElements();
-
-  unsigned NumLanes = VT.getSizeInBits() / 128;
-  unsigned NumLaneElts = NumElts / NumLanes;
-
-  unsigned NewImm = Imm;
-  for (unsigned l = 0; l != NumElts; l += NumLaneElts) {
-    // each half of a lane comes from different source
-    for (unsigned s = 0; s != NumElts*2; s += NumElts) {
-      for (unsigned i = 0; i != NumLaneElts/2; ++i) {
-        ShuffleMask.push_back(NewImm % NumLaneElts + s + l);
-        NewImm /= NumLaneElts;
-      }
-    }
-    if (NumLaneElts == 4) NewImm = Imm; // reload imm
-  }
-}
-
-/// DecodeUNPCKHMask - This decodes the shuffle masks for unpckhps/unpckhpd
-/// and punpckh*. VT indicates the type of the vector allowing it to handle
-/// different datatypes and vector widths.
-void DecodeUNPCKHMask(MVT VT, SmallVectorImpl<int> &ShuffleMask) {
-  unsigned NumElts = VT.getVectorNumElements();
-
-  // Handle 128 and 256-bit vector lengths. AVX defines UNPCK* to operate
-  // independently on 128-bit lanes.
-  unsigned NumLanes = VT.getSizeInBits() / 128;
-  if (NumLanes == 0 ) NumLanes = 1;  // Handle MMX
-  unsigned NumLaneElts = NumElts / NumLanes;
-
-  for (unsigned l = 0; l != NumElts; l += NumLaneElts) {
-    for (unsigned i = l + NumLaneElts/2, e = l + NumLaneElts; i != e; ++i) {
-      ShuffleMask.push_back(i);          // Reads from dest/src1
-      ShuffleMask.push_back(i+NumElts);  // Reads from src/src2
-    }
-  }
-}
-
-/// DecodeUNPCKLMask - This decodes the shuffle masks for unpcklps/unpcklpd
-/// and punpckl*. VT indicates the type of the vector allowing it to handle
-/// different datatypes and vector widths.
-void DecodeUNPCKLMask(MVT VT, SmallVectorImpl<int> &ShuffleMask) {
-  unsigned NumElts = VT.getVectorNumElements();
-
-  // Handle 128 and 256-bit vector lengths. AVX defines UNPCK* to operate
-  // independently on 128-bit lanes.
-  unsigned NumLanes = VT.getSizeInBits() / 128;
-  if (NumLanes == 0 ) NumLanes = 1;  // Handle MMX
-  unsigned NumLaneElts = NumElts / NumLanes;
-
-  for (unsigned l = 0; l != NumElts; l += NumLaneElts) {
-    for (unsigned i = l, e = l + NumLaneElts/2; i != e; ++i) {
-      ShuffleMask.push_back(i);          // Reads from dest/src1
-      ShuffleMask.push_back(i+NumElts);  // Reads from src/src2
-    }
-  }
-}
-
-void DecodeVPERM2X128Mask(MVT VT, unsigned Imm,
-                          SmallVectorImpl<int> &ShuffleMask) {
-  if (Imm & 0x88)
-    return; // Not a shuffle
-
-  unsigned HalfSize = VT.getVectorNumElements()/2;
-
-  for (unsigned l = 0; l != 2; ++l) {
-    unsigned HalfBegin = ((Imm >> (l*4)) & 0x3) * HalfSize;
-    for (unsigned i = HalfBegin, e = HalfBegin+HalfSize; i != e; ++i)
-      ShuffleMask.push_back(i);
-  }
-}
-
-void DecodePSHUFBMask(const Constant *C, SmallVectorImpl<int> &ShuffleMask) {
-  Type *MaskTy = C->getType();
-  // It is not an error for the PSHUFB mask to not be a vector of i8 because the
-  // constant pool uniques constants by their bit representation.
-  // e.g. the following take up the same space in the constant pool:
-  //   i128 -170141183420855150465331762880109871104
-  //
-  //   <2 x i64> <i64 -9223372034707292160, i64 -9223372034707292160>
-  //
-  //   <4 x i32> <i32 -2147483648, i32 -2147483648,
-  //              i32 -2147483648, i32 -2147483648>
-
-  unsigned MaskTySize = MaskTy->getPrimitiveSizeInBits();
-
-  if (MaskTySize != 128 && MaskTySize != 256) // FIXME: Add support for AVX-512.
-    return;
-
-  // This is a straightforward byte vector.
-  if (MaskTy->isVectorTy() && MaskTy->getVectorElementType()->isIntegerTy(8)) {
-    int NumElements = MaskTy->getVectorNumElements();
-    ShuffleMask.reserve(NumElements);
-
-    for (int i = 0; i < NumElements; ++i) {
-      // For AVX vectors with 32 bytes the base of the shuffle is the 16-byte
-      // lane of the vector we're inside.
-      int Base = i < 16 ? 0 : 16;
-      Constant *COp = C->getAggregateElement(i);
-      if (!COp) {
-        ShuffleMask.clear();
-        return;
-      } else if (isa<UndefValue>(COp)) {
-        ShuffleMask.push_back(SM_SentinelUndef);
-        continue;
-      }
-      uint64_t Element = cast<ConstantInt>(COp)->getZExtValue();
-      // If the high bit (7) of the byte is set, the element is zeroed.
-      if (Element & (1 << 7))
-        ShuffleMask.push_back(SM_SentinelZero);
-      else {
-        // Only the least significant 4 bits of the byte are used.
-        int Index = Base + (Element & 0xf);
-        ShuffleMask.push_back(Index);
-      }
-    }
-  }
-  // TODO: Handle funny-looking vectors too.
-}
-
-void DecodePSHUFBMask(ArrayRef<uint64_t> RawMask,
-                      SmallVectorImpl<int> &ShuffleMask) {
-  for (int i = 0, e = RawMask.size(); i < e; ++i) {
-    uint64_t M = RawMask[i];
-    if (M == (uint64_t)SM_SentinelUndef) {
-      ShuffleMask.push_back(M);
-      continue;
-    }
-    // For AVX vectors with 32 bytes the base of the shuffle is the half of
-    // the vector we're inside.
-    int Base = i < 16 ? 0 : 16;
-    // If the high bit (7) of the byte is set, the element is zeroed.
-    if (M & (1 << 7))
-      ShuffleMask.push_back(SM_SentinelZero);
-    else {
-      // Only the least significant 4 bits of the byte are used.
-      int Index = Base + (M & 0xf);
-      ShuffleMask.push_back(Index);
-    }
-  }
-}
-
-void DecodeBLENDMask(MVT VT, unsigned Imm, SmallVectorImpl<int> &ShuffleMask) {
-  int ElementBits = VT.getScalarSizeInBits();
-  int NumElements = VT.getVectorNumElements();
-  for (int i = 0; i < NumElements; ++i) {
-    // If there are more than 8 elements in the vector, then any immediate blend
-    // mask applies to each 128-bit lane. There can never be more than
-    // 8 elements in a 128-bit lane with an immediate blend.
-    int Bit = NumElements > 8 ? i % (128 / ElementBits) : i;
-    assert(Bit < 8 &&
-           "Immediate blends only operate over 8 elements at a time!");
-    ShuffleMask.push_back(((Imm >> Bit) & 1) ? NumElements + i : i);
-  }
-}
-
-/// DecodeVPERMMask - this decodes the shuffle masks for VPERMQ/VPERMPD.
-/// No VT provided since it only works on 256-bit, 4 element vectors.
-void DecodeVPERMMask(unsigned Imm, SmallVectorImpl<int> &ShuffleMask) {
-  for (unsigned i = 0; i != 4; ++i) {
-    ShuffleMask.push_back((Imm >> (2*i)) & 3);
-  }
-}
-
-void DecodeVPERMILPMask(const Constant *C, SmallVectorImpl<int> &ShuffleMask) {
-  Type *MaskTy = C->getType();
-  assert(MaskTy->isVectorTy() && "Expected a vector constant mask!");
-  assert(MaskTy->getVectorElementType()->isIntegerTy() &&
-         "Expected integer constant mask elements!");
-  int ElementBits = MaskTy->getScalarSizeInBits();
-  int NumElements = MaskTy->getVectorNumElements();
-  assert((NumElements == 2 || NumElements == 4 || NumElements == 8) &&
-         "Unexpected number of vector elements.");
-  ShuffleMask.reserve(NumElements);
-  if (auto *CDS = dyn_cast<ConstantDataSequential>(C)) {
-    assert((unsigned)NumElements == CDS->getNumElements() &&
-           "Constant mask has a different number of elements!");
-
-    for (int i = 0; i < NumElements; ++i) {
-      int Base = (i * ElementBits / 128) * (128 / ElementBits);
-      uint64_t Element = CDS->getElementAsInteger(i);
-      // Only the least significant 2 bits of the integer are used.
-      int Index = Base + (Element & 0x3);
-      ShuffleMask.push_back(Index);
-    }
-  } else if (auto *CV = dyn_cast<ConstantVector>(C)) {
-    assert((unsigned)NumElements == C->getNumOperands() &&
-           "Constant mask has a different number of elements!");
-
-    for (int i = 0; i < NumElements; ++i) {
-      int Base = (i * ElementBits / 128) * (128 / ElementBits);
-      Constant *COp = CV->getOperand(i);
-      if (isa<UndefValue>(COp)) {
-        ShuffleMask.push_back(SM_SentinelUndef);
-        continue;
-      }
-      uint64_t Element = cast<ConstantInt>(COp)->getZExtValue();
-      // Only the least significant 2 bits of the integer are used.
-      int Index = Base + (Element & 0x3);
-      ShuffleMask.push_back(Index);
-    }
-  }
-}
-
-void DecodeZeroExtendMask(MVT SrcVT, MVT DstVT, SmallVectorImpl<int> &Mask) {
-  unsigned NumDstElts = DstVT.getVectorNumElements();
-  unsigned SrcScalarBits = SrcVT.getScalarSizeInBits();
-  unsigned DstScalarBits = DstVT.getScalarSizeInBits();
-  unsigned Scale = DstScalarBits / SrcScalarBits;
-  assert(SrcScalarBits < DstScalarBits &&
-         "Expected zero extension mask to increase scalar size");
-  assert(SrcVT.getVectorNumElements() >= NumDstElts &&
-         "Too many zero extension lanes");
-
-  for (unsigned i = 0; i != NumDstElts; i++) {
-    Mask.push_back(i);
-    for (unsigned j = 1; j != Scale; j++)
-      Mask.push_back(SM_SentinelZero);
-  }
-}
-
-void DecodeZeroMoveLowMask(MVT VT, SmallVectorImpl<int> &ShuffleMask) {
-  unsigned NumElts = VT.getVectorNumElements();
-  ShuffleMask.push_back(0);
-  for (unsigned i = 1; i < NumElts; i++)
-    ShuffleMask.push_back(SM_SentinelZero);
-}
-
-void DecodeScalarMoveMask(MVT VT, bool IsLoad, SmallVectorImpl<int> &Mask) {
-  // First element comes from the first element of second source.
-  // Remaining elements: Load zero extends / Move copies from first source.
-  unsigned NumElts = VT.getVectorNumElements();
-  Mask.push_back(NumElts);
-  for (unsigned i = 1; i < NumElts; i++)
-    Mask.push_back(IsLoad ? static_cast<int>(SM_SentinelZero) : i);
-}
-} // llvm namespace
+//===-- X86ShuffleDecode.cpp - X86 shuffle decode logic -------------------===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// Define several functions to decode x86 specific shuffle semantics into a
+// generic vector mask.
+//
+//===----------------------------------------------------------------------===//
+
+#include "X86ShuffleDecode.h"
+#include "llvm/IR/Constants.h"
+#include "llvm/CodeGen/MachineValueType.h"
+
+//===----------------------------------------------------------------------===//
+//  Vector Mask Decoding
+//===----------------------------------------------------------------------===//
+
+namespace llvm {
+
+void DecodeINSERTPSMask(unsigned Imm, SmallVectorImpl<int> &ShuffleMask) {
+  // Defaults the copying the dest value.
+  ShuffleMask.push_back(0);
+  ShuffleMask.push_back(1);
+  ShuffleMask.push_back(2);
+  ShuffleMask.push_back(3);
+
+  // Decode the immediate.
+  unsigned ZMask = Imm & 15;
+  unsigned CountD = (Imm >> 4) & 3;
+  unsigned CountS = (Imm >> 6) & 3;
+
+  // CountS selects which input element to use.
+  unsigned InVal = 4 + CountS;
+  // CountD specifies which element of destination to update.
+  ShuffleMask[CountD] = InVal;
+  // ZMask zaps values, potentially overriding the CountD elt.
+  if (ZMask & 1) ShuffleMask[0] = SM_SentinelZero;
+  if (ZMask & 2) ShuffleMask[1] = SM_SentinelZero;
+  if (ZMask & 4) ShuffleMask[2] = SM_SentinelZero;
+  if (ZMask & 8) ShuffleMask[3] = SM_SentinelZero;
+}
+
+// <3,1> or <6,7,2,3>
+void DecodeMOVHLPSMask(unsigned NElts, SmallVectorImpl<int> &ShuffleMask) {
+  for (unsigned i = NElts / 2; i != NElts; ++i)
+    ShuffleMask.push_back(NElts + i);
+
+  for (unsigned i = NElts / 2; i != NElts; ++i)
+    ShuffleMask.push_back(i);
+}
+
+// <0,2> or <0,1,4,5>
+void DecodeMOVLHPSMask(unsigned NElts, SmallVectorImpl<int> &ShuffleMask) {
+  for (unsigned i = 0; i != NElts / 2; ++i)
+    ShuffleMask.push_back(i);
+
+  for (unsigned i = 0; i != NElts / 2; ++i)
+    ShuffleMask.push_back(NElts + i);
+}
+
+void DecodeMOVSLDUPMask(MVT VT, SmallVectorImpl<int> &ShuffleMask) {
+  unsigned NumElts = VT.getVectorNumElements();
+  for (int i = 0, e = NumElts / 2; i < e; ++i) {
+    ShuffleMask.push_back(2 * i);
+    ShuffleMask.push_back(2 * i);
+  }
+}
+
+void DecodeMOVSHDUPMask(MVT VT, SmallVectorImpl<int> &ShuffleMask) {
+  unsigned NumElts = VT.getVectorNumElements();
+  for (int i = 0, e = NumElts / 2; i < e; ++i) {
+    ShuffleMask.push_back(2 * i + 1);
+    ShuffleMask.push_back(2 * i + 1);
+  }
+}
+
+void DecodeMOVDDUPMask(MVT VT, SmallVectorImpl<int> &ShuffleMask) {
+  unsigned VectorSizeInBits = VT.getSizeInBits();
+  unsigned ScalarSizeInBits = VT.getScalarSizeInBits();
+  unsigned NumElts = VT.getVectorNumElements();
+  unsigned NumLanes = VectorSizeInBits / 128;
+  unsigned NumLaneElts = NumElts / NumLanes;
+  unsigned NumLaneSubElts = 64 / ScalarSizeInBits;
+
+  for (unsigned l = 0; l < NumElts; l += NumLaneElts)
+    for (unsigned i = 0; i < NumLaneElts; i += NumLaneSubElts)
+      for (unsigned s = 0; s != NumLaneSubElts; s++)
+        ShuffleMask.push_back(l + s);
+}
+
+void DecodePSLLDQMask(MVT VT, unsigned Imm, SmallVectorImpl<int> &ShuffleMask) {
+  unsigned VectorSizeInBits = VT.getSizeInBits();
+  unsigned NumElts = VectorSizeInBits / 8;
+  unsigned NumLanes = VectorSizeInBits / 128;
+  unsigned NumLaneElts = NumElts / NumLanes;
+
+  for (unsigned l = 0; l < NumElts; l += NumLaneElts)
+    for (unsigned i = 0; i < NumLaneElts; ++i) {
+      int M = SM_SentinelZero;
+      if (i >= Imm) M = i - Imm + l;
+      ShuffleMask.push_back(M);
+    }
+}
+
+void DecodePSRLDQMask(MVT VT, unsigned Imm, SmallVectorImpl<int> &ShuffleMask) {
+  unsigned VectorSizeInBits = VT.getSizeInBits();
+  unsigned NumElts = VectorSizeInBits / 8;
+  unsigned NumLanes = VectorSizeInBits / 128;
+  unsigned NumLaneElts = NumElts / NumLanes;
+
+  for (unsigned l = 0; l < NumElts; l += NumLaneElts)
+    for (unsigned i = 0; i < NumLaneElts; ++i) {
+      unsigned Base = i + Imm;
+      int M = Base + l;
+      if (Base >= NumLaneElts) M = SM_SentinelZero;
+      ShuffleMask.push_back(M);
+    }
+}
+
+void DecodePALIGNRMask(MVT VT, unsigned Imm,
+                       SmallVectorImpl<int> &ShuffleMask) {
+  unsigned NumElts = VT.getVectorNumElements();
+  unsigned Offset = Imm * (VT.getVectorElementType().getSizeInBits() / 8);
+
+  unsigned NumLanes = VT.getSizeInBits() / 128;
+  unsigned NumLaneElts = NumElts / NumLanes;
+
+  for (unsigned l = 0; l != NumElts; l += NumLaneElts) {
+    for (unsigned i = 0; i != NumLaneElts; ++i) {
+      unsigned Base = i + Offset;
+      // if i+offset is out of this lane then we actually need the other source
+      if (Base >= NumLaneElts) Base += NumElts - NumLaneElts;
+      ShuffleMask.push_back(Base + l);
+    }
+  }
+}
+
+/// DecodePSHUFMask - This decodes the shuffle masks for pshufd, and vpermilp*.
+/// VT indicates the type of the vector allowing it to handle different
+/// datatypes and vector widths.
+void DecodePSHUFMask(MVT VT, unsigned Imm, SmallVectorImpl<int> &ShuffleMask) {
+  unsigned NumElts = VT.getVectorNumElements();
+
+  unsigned NumLanes = VT.getSizeInBits() / 128;
+  unsigned NumLaneElts = NumElts / NumLanes;
+
+  unsigned NewImm = Imm;
+  for (unsigned l = 0; l != NumElts; l += NumLaneElts) {
+    for (unsigned i = 0; i != NumLaneElts; ++i) {
+      ShuffleMask.push_back(NewImm % NumLaneElts + l);
+      NewImm /= NumLaneElts;
+    }
+    if (NumLaneElts == 4) NewImm = Imm; // reload imm
+  }
+}
+
+void DecodePSHUFHWMask(MVT VT, unsigned Imm,
+                       SmallVectorImpl<int> &ShuffleMask) {
+  unsigned NumElts = VT.getVectorNumElements();
+
+  for (unsigned l = 0; l != NumElts; l += 8) {
+    unsigned NewImm = Imm;
+    for (unsigned i = 0, e = 4; i != e; ++i) {
+      ShuffleMask.push_back(l + i);
+    }
+    for (unsigned i = 4, e = 8; i != e; ++i) {
+      ShuffleMask.push_back(l + 4 + (NewImm & 3));
+      NewImm >>= 2;
+    }
+  }
+}
+
+void DecodePSHUFLWMask(MVT VT, unsigned Imm,
+                       SmallVectorImpl<int> &ShuffleMask) {
+  unsigned NumElts = VT.getVectorNumElements();
+
+  for (unsigned l = 0; l != NumElts; l += 8) {
+    unsigned NewImm = Imm;
+    for (unsigned i = 0, e = 4; i != e; ++i) {
+      ShuffleMask.push_back(l + (NewImm & 3));
+      NewImm >>= 2;
+    }
+    for (unsigned i = 4, e = 8; i != e; ++i) {
+      ShuffleMask.push_back(l + i);
+    }
+  }
+}
+
+/// DecodeSHUFPMask - This decodes the shuffle masks for shufp*. VT indicates
+/// the type of the vector allowing it to handle different datatypes and vector
+/// widths.
+void DecodeSHUFPMask(MVT VT, unsigned Imm, SmallVectorImpl<int> &ShuffleMask) {
+  unsigned NumElts = VT.getVectorNumElements();
+
+  unsigned NumLanes = VT.getSizeInBits() / 128;
+  unsigned NumLaneElts = NumElts / NumLanes;
+
+  unsigned NewImm = Imm;
+  for (unsigned l = 0; l != NumElts; l += NumLaneElts) {
+    // each half of a lane comes from different source
+    for (unsigned s = 0; s != NumElts * 2; s += NumElts) {
+      for (unsigned i = 0; i != NumLaneElts / 2; ++i) {
+        ShuffleMask.push_back(NewImm % NumLaneElts + s + l);
+        NewImm /= NumLaneElts;
+      }
+    }
+    if (NumLaneElts == 4) NewImm = Imm; // reload imm
+  }
+}
+
+/// DecodeUNPCKHMask - This decodes the shuffle masks for unpckhps/unpckhpd
+/// and punpckh*. VT indicates the type of the vector allowing it to handle
+/// different datatypes and vector widths.
+void DecodeUNPCKHMask(MVT VT, SmallVectorImpl<int> &ShuffleMask) {
+  unsigned NumElts = VT.getVectorNumElements();
+
+  // Handle 128 and 256-bit vector lengths. AVX defines UNPCK* to operate
+  // independently on 128-bit lanes.
+  unsigned NumLanes = VT.getSizeInBits() / 128;
+  if (NumLanes == 0 ) NumLanes = 1;  // Handle MMX
+  unsigned NumLaneElts = NumElts / NumLanes;
+
+  for (unsigned l = 0; l != NumElts; l += NumLaneElts) {
+    for (unsigned i = l + NumLaneElts / 2, e = l + NumLaneElts; i != e; ++i) {
+      ShuffleMask.push_back(i);           // Reads from dest/src1
+      ShuffleMask.push_back(i + NumElts); // Reads from src/src2
+    }
+  }
+}
+
+/// DecodeUNPCKLMask - This decodes the shuffle masks for unpcklps/unpcklpd
+/// and punpckl*. VT indicates the type of the vector allowing it to handle
+/// different datatypes and vector widths.
+void DecodeUNPCKLMask(MVT VT, SmallVectorImpl<int> &ShuffleMask) {
+  unsigned NumElts = VT.getVectorNumElements();
+
+  // Handle 128 and 256-bit vector lengths. AVX defines UNPCK* to operate
+  // independently on 128-bit lanes.
+  unsigned NumLanes = VT.getSizeInBits() / 128;
+  if (NumLanes == 0 ) NumLanes = 1;  // Handle MMX
+  unsigned NumLaneElts = NumElts / NumLanes;
+
+  for (unsigned l = 0; l != NumElts; l += NumLaneElts) {
+    for (unsigned i = l, e = l + NumLaneElts / 2; i != e; ++i) {
+      ShuffleMask.push_back(i);           // Reads from dest/src1
+      ShuffleMask.push_back(i + NumElts); // Reads from src/src2
+    }
+  }
+}
+
+void DecodeVPERM2X128Mask(MVT VT, unsigned Imm,
+                          SmallVectorImpl<int> &ShuffleMask) {
+  unsigned HalfSize = VT.getVectorNumElements() / 2;
+
+  for (unsigned l = 0; l != 2; ++l) {
+    unsigned HalfMask = Imm >> (l * 4);
+    unsigned HalfBegin = (HalfMask & 0x3) * HalfSize;
+    for (unsigned i = HalfBegin, e = HalfBegin + HalfSize; i != e; ++i)
+      ShuffleMask.push_back(HalfMask & 8 ? SM_SentinelZero : (int)i);
+  }
+}
+
+void DecodePSHUFBMask(const Constant *C, SmallVectorImpl<int> &ShuffleMask) {
+  Type *MaskTy = C->getType();
+  // It is not an error for the PSHUFB mask to not be a vector of i8 because the
+  // constant pool uniques constants by their bit representation.
+  // e.g. the following take up the same space in the constant pool:
+  //   i128 -170141183420855150465331762880109871104
+  //
+  //   <2 x i64> <i64 -9223372034707292160, i64 -9223372034707292160>
+  //
+  //   <4 x i32> <i32 -2147483648, i32 -2147483648,
+  //              i32 -2147483648, i32 -2147483648>
+
+  unsigned MaskTySize = MaskTy->getPrimitiveSizeInBits();
+
+  if (MaskTySize != 128 && MaskTySize != 256) // FIXME: Add support for AVX-512.
+    return;
+
+  // This is a straightforward byte vector.
+  if (MaskTy->isVectorTy() && MaskTy->getVectorElementType()->isIntegerTy(8)) {
+    int NumElements = MaskTy->getVectorNumElements();
+    ShuffleMask.reserve(NumElements);
+
+    for (int i = 0; i < NumElements; ++i) {
+      // For AVX vectors with 32 bytes the base of the shuffle is the 16-byte
+      // lane of the vector we're inside.
+      int Base = i < 16 ? 0 : 16;
+      Constant *COp = C->getAggregateElement(i);
+      if (!COp) {
+        ShuffleMask.clear();
+        return;
+      } else if (isa<UndefValue>(COp)) {
+        ShuffleMask.push_back(SM_SentinelUndef);
+        continue;
+      }
+      uint64_t Element = cast<ConstantInt>(COp)->getZExtValue();
+      // If the high bit (7) of the byte is set, the element is zeroed.
+      if (Element & (1 << 7))
+        ShuffleMask.push_back(SM_SentinelZero);
+      else {
+        // Only the least significant 4 bits of the byte are used.
+        int Index = Base + (Element & 0xf);
+        ShuffleMask.push_back(Index);
+      }
+    }
+  }
+  // TODO: Handle funny-looking vectors too.
+}
+
+void DecodePSHUFBMask(ArrayRef<uint64_t> RawMask,
+                      SmallVectorImpl<int> &ShuffleMask) {
+  for (int i = 0, e = RawMask.size(); i < e; ++i) {
+    uint64_t M = RawMask[i];
+    if (M == (uint64_t)SM_SentinelUndef) {
+      ShuffleMask.push_back(M);
+      continue;
+    }
+    // For AVX vectors with 32 bytes the base of the shuffle is the half of
+    // the vector we're inside.
+    int Base = i < 16 ? 0 : 16;
+    // If the high bit (7) of the byte is set, the element is zeroed.
+    if (M & (1 << 7))
+      ShuffleMask.push_back(SM_SentinelZero);
+    else {
+      // Only the least significant 4 bits of the byte are used.
+      int Index = Base + (M & 0xf);
+      ShuffleMask.push_back(Index);
+    }
+  }
+}
+
+void DecodeBLENDMask(MVT VT, unsigned Imm, SmallVectorImpl<int> &ShuffleMask) {
+  int ElementBits = VT.getScalarSizeInBits();
+  int NumElements = VT.getVectorNumElements();
+  for (int i = 0; i < NumElements; ++i) {
+    // If there are more than 8 elements in the vector, then any immediate blend
+    // mask applies to each 128-bit lane. There can never be more than
+    // 8 elements in a 128-bit lane with an immediate blend.
+    int Bit = NumElements > 8 ? i % (128 / ElementBits) : i;
+    assert(Bit < 8 &&
+           "Immediate blends only operate over 8 elements at a time!");
+    ShuffleMask.push_back(((Imm >> Bit) & 1) ? NumElements + i : i);
+  }
+}
+
+/// DecodeVPERMMask - this decodes the shuffle masks for VPERMQ/VPERMPD.
+/// No VT provided since it only works on 256-bit, 4 element vectors.
+void DecodeVPERMMask(unsigned Imm, SmallVectorImpl<int> &ShuffleMask) {
+  for (unsigned i = 0; i != 4; ++i) {
+    ShuffleMask.push_back((Imm >> (2 * i)) & 3);
+  }
+}
+
+void DecodeVPERMILPMask(const Constant *C, SmallVectorImpl<int> &ShuffleMask) {
+  Type *MaskTy = C->getType();
+  assert(MaskTy->isVectorTy() && "Expected a vector constant mask!");
+  assert(MaskTy->getVectorElementType()->isIntegerTy() &&
+         "Expected integer constant mask elements!");
+  int ElementBits = MaskTy->getScalarSizeInBits();
+  int NumElements = MaskTy->getVectorNumElements();
+  assert((NumElements == 2 || NumElements == 4 || NumElements == 8) &&
+         "Unexpected number of vector elements.");
+  ShuffleMask.reserve(NumElements);
+  if (auto *CDS = dyn_cast<ConstantDataSequential>(C)) {
+    assert((unsigned)NumElements == CDS->getNumElements() &&
+           "Constant mask has a different number of elements!");
+
+    for (int i = 0; i < NumElements; ++i) {
+      int Base = (i * ElementBits / 128) * (128 / ElementBits);
+      uint64_t Element = CDS->getElementAsInteger(i);
+      // Only the least significant 2 bits of the integer are used.
+      int Index = Base + (Element & 0x3);
+      ShuffleMask.push_back(Index);
+    }
+  } else if (auto *CV = dyn_cast<ConstantVector>(C)) {
+    assert((unsigned)NumElements == C->getNumOperands() &&
+           "Constant mask has a different number of elements!");
+
+    for (int i = 0; i < NumElements; ++i) {
+      int Base = (i * ElementBits / 128) * (128 / ElementBits);
+      Constant *COp = CV->getOperand(i);
+      if (isa<UndefValue>(COp)) {
+        ShuffleMask.push_back(SM_SentinelUndef);
+        continue;
+      }
+      uint64_t Element = cast<ConstantInt>(COp)->getZExtValue();
+      // Only the least significant 2 bits of the integer are used.
+      int Index = Base + (Element & 0x3);
+      ShuffleMask.push_back(Index);
+    }
+  }
+}
+
+void DecodeZeroExtendMask(MVT SrcVT, MVT DstVT, SmallVectorImpl<int> &Mask) {
+  unsigned NumDstElts = DstVT.getVectorNumElements();
+  unsigned SrcScalarBits = SrcVT.getScalarSizeInBits();
+  unsigned DstScalarBits = DstVT.getScalarSizeInBits();
+  unsigned Scale = DstScalarBits / SrcScalarBits;
+  assert(SrcScalarBits < DstScalarBits &&
+         "Expected zero extension mask to increase scalar size");
+  assert(SrcVT.getVectorNumElements() >= NumDstElts &&
+         "Too many zero extension lanes");
+
+  for (unsigned i = 0; i != NumDstElts; i++) {
+    Mask.push_back(i);
+    for (unsigned j = 1; j != Scale; j++)
+      Mask.push_back(SM_SentinelZero);
+  }
+}
+
+void DecodeZeroMoveLowMask(MVT VT, SmallVectorImpl<int> &ShuffleMask) {
+  unsigned NumElts = VT.getVectorNumElements();
+  ShuffleMask.push_back(0);
+  for (unsigned i = 1; i < NumElts; i++)
+    ShuffleMask.push_back(SM_SentinelZero);
+}
+
+void DecodeScalarMoveMask(MVT VT, bool IsLoad, SmallVectorImpl<int> &Mask) {
+  // First element comes from the first element of second source.
+  // Remaining elements: Load zero extends / Move copies from first source.
+  unsigned NumElts = VT.getVectorNumElements();
+  Mask.push_back(NumElts);
+  for (unsigned i = 1; i < NumElts; i++)
+    Mask.push_back(IsLoad ? static_cast<int>(SM_SentinelZero) : i);
+}
+
+void DecodeEXTRQIMask(int Len, int Idx,
+                      SmallVectorImpl<int> &ShuffleMask) {
+  // Only the bottom 6 bits are valid for each immediate.
+  Len &= 0x3F;
+  Idx &= 0x3F;
+
+  // We can only decode this bit extraction instruction as a shuffle if both the
+  // length and index work with whole bytes.
+  if (0 != (Len % 8) || 0 != (Idx % 8))
+    return;
+
+  // A length of zero is equivalent to a bit length of 64.
+  if (Len == 0)
+    Len = 64;
+
+  // If the length + index exceeds the bottom 64 bits the result is undefined.
+  if ((Len + Idx) > 64) {
+    ShuffleMask.append(16, SM_SentinelUndef);
+    return;
+  }
+
+  // Convert index and index to work with bytes.
+  Len /= 8;
+  Idx /= 8;
+
+  // EXTRQ: Extract Len bytes starting from Idx. Zero pad the remaining bytes
+  // of the lower 64-bits. The upper 64-bits are undefined.
+  for (int i = 0; i != Len; ++i)
+    ShuffleMask.push_back(i + Idx);
+  for (int i = Len; i != 8; ++i)
+    ShuffleMask.push_back(SM_SentinelZero);
+  for (int i = 8; i != 16; ++i)
+    ShuffleMask.push_back(SM_SentinelUndef);
+}
+
+void DecodeINSERTQIMask(int Len, int Idx,
+                        SmallVectorImpl<int> &ShuffleMask) {
+  // Only the bottom 6 bits are valid for each immediate.
+  Len &= 0x3F;
+  Idx &= 0x3F;
+
+  // We can only decode this bit insertion instruction as a shuffle if both the
+  // length and index work with whole bytes.
+  if (0 != (Len % 8) || 0 != (Idx % 8))
+    return;
+
+  // A length of zero is equivalent to a bit length of 64.
+  if (Len == 0)
+    Len = 64;
+
+  // If the length + index exceeds the bottom 64 bits the result is undefined.
+  if ((Len + Idx) > 64) {
+    ShuffleMask.append(16, SM_SentinelUndef);
+    return;
+  }
+
+  // Convert index and index to work with bytes.
+  Len /= 8;
+  Idx /= 8;
+
+  // INSERTQ: Extract lowest Len bytes from lower half of second source and
+  // insert over first source starting at Idx byte. The upper 64-bits are
+  // undefined.
+  for (int i = 0; i != Idx; ++i)
+    ShuffleMask.push_back(i);
+  for (int i = 0; i != Len; ++i)
+    ShuffleMask.push_back(i + 16);
+  for (int i = Idx + Len; i != 8; ++i)
+    ShuffleMask.push_back(i);
+  for (int i = 8; i != 16; ++i)
+    ShuffleMask.push_back(SM_SentinelUndef);
+}
+
+} // llvm namespace
diff --git a/lib/Target/X86/Utils/X86ShuffleDecode.h b/lib/Target/X86/Utils/X86ShuffleDecode.h
index 5c9a8cf3752..3d10d18e860 100644
--- a/lib/Target/X86/Utils/X86ShuffleDecode.h
+++ b/lib/Target/X86/Utils/X86ShuffleDecode.h
@@ -1,105 +1,113 @@
-//===-- X86ShuffleDecode.h - X86 shuffle decode logic -----------*-C++-*---===//
-//
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
-//
-//===----------------------------------------------------------------------===//
-//
-// Define several functions to decode x86 specific shuffle semantics into a
-// generic vector mask.
-//
-//===----------------------------------------------------------------------===//
-
-#ifndef LLVM_LIB_TARGET_X86_UTILS_X86SHUFFLEDECODE_H
-#define LLVM_LIB_TARGET_X86_UTILS_X86SHUFFLEDECODE_H
-
-#include "llvm/ADT/SmallVector.h"
-#include "llvm/ADT/ArrayRef.h"
-
-//===----------------------------------------------------------------------===//
-//  Vector Mask Decoding
-//===----------------------------------------------------------------------===//
-
-namespace llvm {
-class Constant;
-class MVT;
-
-enum { SM_SentinelUndef = -1, SM_SentinelZero = -2 };
-
-void DecodeINSERTPSMask(unsigned Imm, SmallVectorImpl<int> &ShuffleMask);
-
-// <3,1> or <6,7,2,3>
-void DecodeMOVHLPSMask(unsigned NElts, SmallVectorImpl<int> &ShuffleMask);
-
-// <0,2> or <0,1,4,5>
-void DecodeMOVLHPSMask(unsigned NElts, SmallVectorImpl<int> &ShuffleMask);
-
-void DecodeMOVSLDUPMask(MVT VT, SmallVectorImpl<int> &ShuffleMask);
-
-void DecodeMOVSHDUPMask(MVT VT, SmallVectorImpl<int> &ShuffleMask);
-
-void DecodeMOVDDUPMask(MVT VT, SmallVectorImpl<int> &ShuffleMask);
-
-void DecodePSLLDQMask(MVT VT, unsigned Imm, SmallVectorImpl<int> &ShuffleMask);
-
-void DecodePSRLDQMask(MVT VT, unsigned Imm, SmallVectorImpl<int> &ShuffleMask);
-
-void DecodePALIGNRMask(MVT VT, unsigned Imm, SmallVectorImpl<int> &ShuffleMask);
-
-void DecodePSHUFMask(MVT VT, unsigned Imm, SmallVectorImpl<int> &ShuffleMask);
-
-void DecodePSHUFHWMask(MVT VT, unsigned Imm, SmallVectorImpl<int> &ShuffleMask);
-
-void DecodePSHUFLWMask(MVT, unsigned Imm, SmallVectorImpl<int> &ShuffleMask);
-
-/// DecodeSHUFPMask - This decodes the shuffle masks for shufp*. VT indicates
-/// the type of the vector allowing it to handle different datatypes and vector
-/// widths.
-void DecodeSHUFPMask(MVT VT, unsigned Imm, SmallVectorImpl<int> &ShuffleMask);
-
-/// DecodeUNPCKHMask - This decodes the shuffle masks for unpckhps/unpckhpd
-/// and punpckh*. VT indicates the type of the vector allowing it to handle
-/// different datatypes and vector widths.
-void DecodeUNPCKHMask(MVT VT, SmallVectorImpl<int> &ShuffleMask);
-
-/// DecodeUNPCKLMask - This decodes the shuffle masks for unpcklps/unpcklpd
-/// and punpckl*. VT indicates the type of the vector allowing it to handle
-/// different datatypes and vector widths.
-void DecodeUNPCKLMask(MVT VT, SmallVectorImpl<int> &ShuffleMask);
-
-/// \brief Decode a PSHUFB mask from an IR-level vector constant.
-void DecodePSHUFBMask(const Constant *C, SmallVectorImpl<int> &ShuffleMask);
-
-/// \brief Decode a PSHUFB mask from a raw array of constants such as from
-/// BUILD_VECTOR.
-void DecodePSHUFBMask(ArrayRef<uint64_t> RawMask,
-                      SmallVectorImpl<int> &ShuffleMask);
-
-/// \brief Decode a BLEND immediate mask into a shuffle mask.
-void DecodeBLENDMask(MVT VT, unsigned Imm, SmallVectorImpl<int> &ShuffleMask);
-
-void DecodeVPERM2X128Mask(MVT VT, unsigned Imm,
-                          SmallVectorImpl<int> &ShuffleMask);
-
-/// DecodeVPERMMask - this decodes the shuffle masks for VPERMQ/VPERMPD.
-/// No VT provided since it only works on 256-bit, 4 element vectors.
-void DecodeVPERMMask(unsigned Imm, SmallVectorImpl<int> &ShuffleMask);
-
-/// \brief Decode a VPERMILP variable mask from an IR-level vector constant.
-void DecodeVPERMILPMask(const Constant *C, SmallVectorImpl<int> &ShuffleMask);
-
-/// \brief Decode a zero extension instruction as a shuffle mask.
-void DecodeZeroExtendMask(MVT SrcVT, MVT DstVT,
-                          SmallVectorImpl<int> &ShuffleMask);
-
-/// \brief Decode a move lower and zero upper instruction as a shuffle mask.
-void DecodeZeroMoveLowMask(MVT VT, SmallVectorImpl<int> &ShuffleMask);
-
-/// \brief Decode a scalar float move instruction as a shuffle mask.
-void DecodeScalarMoveMask(MVT VT, bool IsLoad,
-                          SmallVectorImpl<int> &ShuffleMask);
-} // llvm namespace
-
-#endif
+//===-- X86ShuffleDecode.h - X86 shuffle decode logic -----------*-C++-*---===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// Define several functions to decode x86 specific shuffle semantics into a
+// generic vector mask.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIB_TARGET_X86_UTILS_X86SHUFFLEDECODE_H
+#define LLVM_LIB_TARGET_X86_UTILS_X86SHUFFLEDECODE_H
+
+#include "llvm/ADT/SmallVector.h"
+#include "llvm/ADT/ArrayRef.h"
+
+//===----------------------------------------------------------------------===//
+//  Vector Mask Decoding
+//===----------------------------------------------------------------------===//
+
+namespace llvm {
+class Constant;
+class MVT;
+
+enum { SM_SentinelUndef = -1, SM_SentinelZero = -2 };
+
+void DecodeINSERTPSMask(unsigned Imm, SmallVectorImpl<int> &ShuffleMask);
+
+// <3,1> or <6,7,2,3>
+void DecodeMOVHLPSMask(unsigned NElts, SmallVectorImpl<int> &ShuffleMask);
+
+// <0,2> or <0,1,4,5>
+void DecodeMOVLHPSMask(unsigned NElts, SmallVectorImpl<int> &ShuffleMask);
+
+void DecodeMOVSLDUPMask(MVT VT, SmallVectorImpl<int> &ShuffleMask);
+
+void DecodeMOVSHDUPMask(MVT VT, SmallVectorImpl<int> &ShuffleMask);
+
+void DecodeMOVDDUPMask(MVT VT, SmallVectorImpl<int> &ShuffleMask);
+
+void DecodePSLLDQMask(MVT VT, unsigned Imm, SmallVectorImpl<int> &ShuffleMask);
+
+void DecodePSRLDQMask(MVT VT, unsigned Imm, SmallVectorImpl<int> &ShuffleMask);
+
+void DecodePALIGNRMask(MVT VT, unsigned Imm, SmallVectorImpl<int> &ShuffleMask);
+
+void DecodePSHUFMask(MVT VT, unsigned Imm, SmallVectorImpl<int> &ShuffleMask);
+
+void DecodePSHUFHWMask(MVT VT, unsigned Imm, SmallVectorImpl<int> &ShuffleMask);
+
+void DecodePSHUFLWMask(MVT, unsigned Imm, SmallVectorImpl<int> &ShuffleMask);
+
+/// DecodeSHUFPMask - This decodes the shuffle masks for shufp*. VT indicates
+/// the type of the vector allowing it to handle different datatypes and vector
+/// widths.
+void DecodeSHUFPMask(MVT VT, unsigned Imm, SmallVectorImpl<int> &ShuffleMask);
+
+/// DecodeUNPCKHMask - This decodes the shuffle masks for unpckhps/unpckhpd
+/// and punpckh*. VT indicates the type of the vector allowing it to handle
+/// different datatypes and vector widths.
+void DecodeUNPCKHMask(MVT VT, SmallVectorImpl<int> &ShuffleMask);
+
+/// DecodeUNPCKLMask - This decodes the shuffle masks for unpcklps/unpcklpd
+/// and punpckl*. VT indicates the type of the vector allowing it to handle
+/// different datatypes and vector widths.
+void DecodeUNPCKLMask(MVT VT, SmallVectorImpl<int> &ShuffleMask);
+
+/// \brief Decode a PSHUFB mask from an IR-level vector constant.
+void DecodePSHUFBMask(const Constant *C, SmallVectorImpl<int> &ShuffleMask);
+
+/// \brief Decode a PSHUFB mask from a raw array of constants such as from
+/// BUILD_VECTOR.
+void DecodePSHUFBMask(ArrayRef<uint64_t> RawMask,
+                      SmallVectorImpl<int> &ShuffleMask);
+
+/// \brief Decode a BLEND immediate mask into a shuffle mask.
+void DecodeBLENDMask(MVT VT, unsigned Imm, SmallVectorImpl<int> &ShuffleMask);
+
+void DecodeVPERM2X128Mask(MVT VT, unsigned Imm,
+                          SmallVectorImpl<int> &ShuffleMask);
+
+/// DecodeVPERMMask - this decodes the shuffle masks for VPERMQ/VPERMPD.
+/// No VT provided since it only works on 256-bit, 4 element vectors.
+void DecodeVPERMMask(unsigned Imm, SmallVectorImpl<int> &ShuffleMask);
+
+/// \brief Decode a VPERMILP variable mask from an IR-level vector constant.
+void DecodeVPERMILPMask(const Constant *C, SmallVectorImpl<int> &ShuffleMask);
+
+/// \brief Decode a zero extension instruction as a shuffle mask.
+void DecodeZeroExtendMask(MVT SrcVT, MVT DstVT,
+                          SmallVectorImpl<int> &ShuffleMask);
+
+/// \brief Decode a move lower and zero upper instruction as a shuffle mask.
+void DecodeZeroMoveLowMask(MVT VT, SmallVectorImpl<int> &ShuffleMask);
+
+/// \brief Decode a scalar float move instruction as a shuffle mask.
+void DecodeScalarMoveMask(MVT VT, bool IsLoad,
+                          SmallVectorImpl<int> &ShuffleMask);
+
+/// \brief Decode a SSE4A EXTRQ instruction as a v16i8 shuffle mask.
+void DecodeEXTRQIMask(int Len, int Idx,
+                      SmallVectorImpl<int> &ShuffleMask);
+
+/// \brief Decode a SSE4A INSERTQ instruction as a v16i8 shuffle mask.
+void DecodeINSERTQIMask(int Len, int Idx,
+                        SmallVectorImpl<int> &ShuffleMask);
+} // llvm namespace
+
+#endif
diff --git a/lib/Target/X86/X86.h b/lib/Target/X86/X86.h
index 17553ee9289..8403ae6101d 100644
--- a/lib/Target/X86/X86.h
+++ b/lib/Target/X86/X86.h
@@ -44,10 +44,6 @@ FunctionPass *createCleanupLocalDynamicTLSPass();
 ///
 FunctionPass *createX86FloatingPointStackifierPass();
 
-// @LOCALMOD-BEGIN - Creates a pass to make instructions follow NaCl SFI rules.
-FunctionPass* createX86NaClRewritePass();
-// @LOCALMOD-END
-
 /// createX86IssueVZeroUpperPass - This pass inserts AVX vzeroupper instructions
 /// before each call to avoid transition penalty between functions encoded with
 /// AVX and SSE.
@@ -73,6 +69,17 @@ FunctionPass *createX86FixupLEAs();
 /// esp-relative movs with pushes.
 FunctionPass *createX86CallFrameOptimization();
 
+/// createX86WinEHStatePass - Return an IR pass that inserts EH registration
+/// stack objects and explicit EH state updates. This pass must run after EH
+/// preparation, which does Windows-specific but architecture-neutral
+/// preparation.
+FunctionPass *createX86WinEHStatePass();
+
+/// Return a Machine IR pass that expands X86-specific pseudo
+/// instructions into a sequence of actual instructions. This pass
+/// must run after prologue/epilogue insertion and before lowering
+/// the MachineInstr to MC.
+FunctionPass *createX86ExpandPseudoPass();
 } // End llvm namespace
 
 #endif
diff --git a/lib/Target/X86/X86.td b/lib/Target/X86/X86.td
index d13f1551d20..2965cefe3d3 100644
--- a/lib/Target/X86/X86.td
+++ b/lib/Target/X86/X86.td
@@ -168,6 +168,8 @@ def FeaturePRFCHW  : SubtargetFeature<"prfchw", "HasPRFCHW", "true",
                                       "Support PRFCHW instructions">;
 def FeatureRDSEED  : SubtargetFeature<"rdseed", "HasRDSEED", "true",
                                       "Support RDSEED instruction">;
+def FeatureMPX     : SubtargetFeature<"mpx", "HasMPX", "true",
+                                      "Support MPX instructions">;
 def FeatureLeaForSP : SubtargetFeature<"lea-sp", "UseLeaForSP", "true",
                                      "Use LEA for adjusting the stack pointer">;
 def FeatureSlowDivide32 : SubtargetFeature<"idivl-to-divb",
@@ -179,6 +181,11 @@ def FeatureSlowDivide64 : SubtargetFeature<"idivq-to-divw",
 def FeaturePadShortFunctions : SubtargetFeature<"pad-short-functions",
                                      "PadShortFunctions", "true",
                                      "Pad short functions">;
+// TODO: This feature ought to be renamed.
+// What it really refers to are CPUs for which certain instructions
+// (which ones besides the example below?) are microcoded.
+// The best examples of this are the memory forms of CALL and PUSH
+// instructions, which should be avoided in favor of a MOV + register CALL/PUSH.
 def FeatureCallRegIndirect : SubtargetFeature<"call-reg-indirect",
                                      "CallRegIndirect", "true",
                                      "Call register indirect">;
@@ -188,10 +195,9 @@ def FeatureSlowLEA : SubtargetFeature<"slow-lea", "SlowLEA", "true",
                                    "LEA instruction with certain arguments is slow">;
 def FeatureSlowIncDec : SubtargetFeature<"slow-incdec", "SlowIncDec", "true",
                                    "INC and DEC instructions are slower than ADD and SUB">;
-def FeatureUseSqrtEst : SubtargetFeature<"use-sqrt-est", "UseSqrtEst", "true",
-                            "Use RSQRT* to optimize square root calculations">;
-def FeatureUseRecipEst : SubtargetFeature<"use-recip-est", "UseReciprocalEst",
-                          "true", "Use RCP* to optimize division calculations">;
+def FeatureSoftFloat
+    : SubtargetFeature<"soft-float", "UseSoftFloat", "true",
+                       "Use software floating point features.">;
 
 //===----------------------------------------------------------------------===//
 // X86 processors supported.
@@ -377,7 +383,7 @@ class KnightsLandingProc<string Name> : ProcessorModel<Name, HaswellModel,
                       FeatureAES, FeaturePCLMUL, FeatureRDRAND, FeatureF16C,
                       FeatureFSGSBase, FeatureMOVBE, FeatureLZCNT, FeatureBMI,
                       FeatureBMI2, FeatureFMA, FeatureRTM, FeatureHLE,
-                      FeatureSlowIncDec]>;
+                      FeatureSlowIncDec, FeatureMPX]>;
 def : KnightsLandingProc<"knl">;
 
 // FIXME: define SKX model
@@ -388,7 +394,7 @@ class SkylakeProc<string Name> : ProcessorModel<Name, HaswellModel,
                       FeatureAES, FeaturePCLMUL, FeatureRDRAND, FeatureF16C,
                       FeatureFSGSBase, FeatureMOVBE, FeatureLZCNT, FeatureBMI,
                       FeatureBMI2, FeatureFMA, FeatureRTM, FeatureHLE,
-                      FeatureSlowIncDec]>;
+                      FeatureSlowIncDec, FeatureMPX]>;
 def : SkylakeProc<"skylake">;
 def : SkylakeProc<"skx">; // Legacy alias.
 
@@ -441,7 +447,7 @@ def : ProcessorModel<"btver2", BtVer2Model,
                       FeaturePRFCHW, FeatureAES, FeaturePCLMUL,
                       FeatureBMI, FeatureF16C, FeatureMOVBE,
                       FeatureLZCNT, FeaturePOPCNT, FeatureFastUAMem,
-                      FeatureSlowSHLD, FeatureUseSqrtEst, FeatureUseRecipEst]>;
+                      FeatureSlowSHLD]>;
 
 // TODO: We should probably add 'FeatureFastUAMem' to all of the AMD chips.
 
diff --git a/lib/Target/X86/X86AsmPrinter.cpp b/lib/Target/X86/X86AsmPrinter.cpp
index 6a875c67ced..9fe849a04ee 100644
--- a/lib/Target/X86/X86AsmPrinter.cpp
+++ b/lib/Target/X86/X86AsmPrinter.cpp
@@ -30,7 +30,6 @@
 #include "llvm/MC/MCAsmInfo.h"
 #include "llvm/MC/MCContext.h"
 #include "llvm/MC/MCExpr.h"
-#include "llvm/MC/MCNaCl.h" // @LOCALMOD
 #include "llvm/MC/MCSectionCOFF.h"
 #include "llvm/MC/MCSectionMachO.h"
 #include "llvm/MC/MCStreamer.h"
@@ -56,12 +55,12 @@ bool X86AsmPrinter::runOnMachineFunction(MachineFunction &MF) {
 
   if (Subtarget->isTargetCOFF()) {
     bool Intrn = MF.getFunction()->hasInternalLinkage();
-    OutStreamer.BeginCOFFSymbolDef(CurrentFnSym);
-    OutStreamer.EmitCOFFSymbolStorageClass(Intrn ? COFF::IMAGE_SYM_CLASS_STATIC
-                                              : COFF::IMAGE_SYM_CLASS_EXTERNAL);
-    OutStreamer.EmitCOFFSymbolType(COFF::IMAGE_SYM_DTYPE_FUNCTION
+    OutStreamer->BeginCOFFSymbolDef(CurrentFnSym);
+    OutStreamer->EmitCOFFSymbolStorageClass(Intrn ? COFF::IMAGE_SYM_CLASS_STATIC
+                                            : COFF::IMAGE_SYM_CLASS_EXTERNAL);
+    OutStreamer->EmitCOFFSymbolType(COFF::IMAGE_SYM_DTYPE_FUNCTION
                                                << COFF::SCT_COMPLEX_TYPE_SHIFT);
-    OutStreamer.EndCOFFSymbolDef();
+    OutStreamer->EndCOFFSymbolDef();
   }
 
   // Emit the rest of the function body.
@@ -71,35 +70,6 @@ bool X86AsmPrinter::runOnMachineFunction(MachineFunction &MF) {
   return false;
 }
 
-// @LOCALMOD-BEGIN
-bool X86AsmPrinter::UseReadOnlyJumpTables() const {
-  return Subtarget->isTargetNaCl();
-}
-
-unsigned X86AsmPrinter::GetTargetBasicBlockAlign() const {
-  if (Subtarget->isTargetNaCl())
-    return 5;
-  return 0;
-}
-
-unsigned X86AsmPrinter::GetTargetLabelAlign(const MachineInstr *MI) const {
-  if (Subtarget->isTargetNaCl()) {
-    switch (MI->getOpcode()) {
-      default: return 0;
-      // These labels may indicate an indirect entry point that is
-      // externally reachable and hence must be bundle aligned.
-      // Note: these labels appear to be always at basic block beginnings
-      // so it may be possible to simply set the MBB alignment.
-      // However, it is unclear whether this always holds.
-      case TargetOpcode::EH_LABEL:
-      case TargetOpcode::GC_LABEL:
-        return 5;
-    }
-  }
-  return 0;
-}
-// @LOCALMOD-END
-
 /// printSymbolOperand - Print a raw symbol reference operand.  This handles
 /// jump tables, constant pools, global address and external symbols, all of
 /// which print to a label with various suffixes for relocation types etc.
@@ -108,7 +78,7 @@ static void printSymbolOperand(X86AsmPrinter &P, const MachineOperand &MO,
   switch (MO.getType()) {
   default: llvm_unreachable("unknown symbol type!");
   case MachineOperand::MO_ConstantPoolIndex:
-    O << *P.GetCPISymbol(MO.getIndex());
+    P.GetCPISymbol(MO.getIndex())->print(O, P.MAI);
     P.printOffset(MO.getOffset(), O);
     break;
   case MachineOperand::MO_GlobalAddress: {
@@ -127,7 +97,7 @@ static void printSymbolOperand(X86AsmPrinter &P, const MachineOperand &MO,
     // Handle dllimport linkage.
     if (MO.getTargetFlags() == X86II::MO_DLLIMPORT)
       GVSym =
-          P.OutContext.GetOrCreateSymbol(Twine("__imp_") + GVSym->getName());
+          P.OutContext.getOrCreateSymbol(Twine("__imp_") + GVSym->getName());
 
     if (MO.getTargetFlags() == X86II::MO_DARWIN_NONLAZY ||
         MO.getTargetFlags() == X86II::MO_DARWIN_NONLAZY_PIC_BASE) {
@@ -157,9 +127,12 @@ static void printSymbolOperand(X86AsmPrinter &P, const MachineOperand &MO,
     // If the name begins with a dollar-sign, enclose it in parens.  We do this
     // to avoid having it look like an integer immediate to the assembler.
     if (GVSym->getName()[0] != '$')
-      O << *GVSym;
-    else
-      O << '(' << *GVSym << ')';
+      GVSym->print(O, P.MAI);
+    else {
+      O << '(';
+      GVSym->print(O, P.MAI);
+      O << ')';
+    }
     P.printOffset(MO.getOffset(), O);
     break;
   }
@@ -176,12 +149,15 @@ static void printSymbolOperand(X86AsmPrinter &P, const MachineOperand &MO,
     // These affect the name of the symbol, not any suffix.
     break;
   case X86II::MO_GOT_ABSOLUTE_ADDRESS:
-    O << " + [.-" << *P.MF->getPICBaseSymbol() << ']';
+    O << " + [.-";
+    P.MF->getPICBaseSymbol()->print(O, P.MAI);
+    O << ']';
     break;
   case X86II::MO_PIC_BASE_OFFSET:
   case X86II::MO_DARWIN_NONLAZY_PIC_BASE:
   case X86II::MO_DARWIN_HIDDEN_NONLAZY_PIC_BASE:
-    O << '-' << *P.MF->getPICBaseSymbol();
+    O << '-';
+    P.MF->getPICBaseSymbol()->print(O, P.MAI);
     break;
   case X86II::MO_TLSGD:     O << "@TLSGD";     break;
   case X86II::MO_TLSLD:     O << "@TLSLD";     break;
@@ -198,7 +174,8 @@ static void printSymbolOperand(X86AsmPrinter &P, const MachineOperand &MO,
   case X86II::MO_PLT:       O << "@PLT";       break;
   case X86II::MO_TLVP:      O << "@TLVP";      break;
   case X86II::MO_TLVP_PIC_BASE:
-    O << "@TLVP" << '-' << *P.MF->getPICBaseSymbol();
+    O << "@TLVP" << '-';
+    P.MF->getPICBaseSymbol()->print(O, P.MAI);
     break;
   case X86II::MO_SECREL:    O << "@SECREL32";  break;
   }
@@ -534,35 +511,31 @@ bool X86AsmPrinter::PrintAsmMemoryOperand(const MachineInstr *MI,
 }
 
 void X86AsmPrinter::EmitStartOfAsmFile(Module &M) {
-  Triple TT(TM.getTargetTriple());
+  const Triple &TT = TM.getTargetTriple();
 
   if (TT.isOSBinFormatMachO())
-    OutStreamer.SwitchSection(getObjFileLowering().getTextSection());
+    OutStreamer->SwitchSection(getObjFileLowering().getTextSection());
 
   if (TT.isOSBinFormatCOFF()) {
     // Emit an absolute @feat.00 symbol.  This appears to be some kind of
     // compiler features bitfield read by link.exe.
     if (TT.getArch() == Triple::x86) {
-      MCSymbol *S = MMI->getContext().GetOrCreateSymbol(StringRef("@feat.00"));
-      OutStreamer.BeginCOFFSymbolDef(S);
-      OutStreamer.EmitCOFFSymbolStorageClass(COFF::IMAGE_SYM_CLASS_STATIC);
-      OutStreamer.EmitCOFFSymbolType(COFF::IMAGE_SYM_DTYPE_NULL);
-      OutStreamer.EndCOFFSymbolDef();
+      MCSymbol *S = MMI->getContext().getOrCreateSymbol(StringRef("@feat.00"));
+      OutStreamer->BeginCOFFSymbolDef(S);
+      OutStreamer->EmitCOFFSymbolStorageClass(COFF::IMAGE_SYM_CLASS_STATIC);
+      OutStreamer->EmitCOFFSymbolType(COFF::IMAGE_SYM_DTYPE_NULL);
+      OutStreamer->EndCOFFSymbolDef();
       // According to the PE-COFF spec, the LSB of this value marks the object
       // for "registered SEH".  This means that all SEH handler entry points
       // must be registered in .sxdata.  Use of any unregistered handlers will
       // cause the process to terminate immediately.  LLVM does not know how to
       // register any SEH handlers, so its object files should be safe.
-      OutStreamer.EmitSymbolAttribute(S, MCSA_Global);
-      OutStreamer.EmitAssignment(
-          S, MCConstantExpr::Create(int64_t(1), MMI->getContext()));
+      OutStreamer->EmitSymbolAttribute(S, MCSA_Global);
+      OutStreamer->EmitAssignment(
+          S, MCConstantExpr::create(int64_t(1), MMI->getContext()));
     }
   }
-
-  // @LOCALMOD-BEGIN
-  if (TT.isOSNaCl())
-    initializeNaClMCStreamer(OutStreamer, OutContext, TT);
-  // @LOCALMOD-END
+  OutStreamer->EmitSyntaxDirective();
 }
 
 static void
@@ -584,7 +557,7 @@ emitNonLazySymbolPointer(MCStreamer &OutStreamer, MCSymbol *StubLabel,
     // using NLPs; however, sometimes the types are local to the file.
     // We need to fill in the value for the NLP in those cases.
     OutStreamer.EmitValue(
-        MCSymbolRefExpr::Create(MCSym.getPointer(), OutStreamer.getContext()),
+        MCSymbolRefExpr::create(MCSym.getPointer(), OutStreamer.getContext()),
         4 /*size*/);
 }
 
@@ -593,13 +566,14 @@ MCSymbol *X86AsmPrinter::GetCPISymbol(unsigned CPID) const {
     const MachineConstantPoolEntry &CPE =
         MF->getConstantPool()->getConstants()[CPID];
     if (!CPE.isMachineConstantPoolEntry()) {
-      SectionKind Kind = CPE.getSectionKind(TM.getDataLayout());
+      const DataLayout &DL = MF->getDataLayout();
+      SectionKind Kind = CPE.getSectionKind(&DL);
       const Constant *C = CPE.Val.ConstVal;
       if (const MCSectionCOFF *S = dyn_cast<MCSectionCOFF>(
-            getObjFileLowering().getSectionForConstant(Kind, C))) {
+              getObjFileLowering().getSectionForConstant(DL, Kind, C))) {
         if (MCSymbol *Sym = S->getCOMDATSymbol()) {
           if (Sym->isUndefined())
-            OutStreamer.EmitSymbolAttribute(Sym, MCSA_Global);
+            OutStreamer->EmitSymbolAttribute(Sym, MCSA_Global);
           return Sym;
         }
       }
@@ -609,36 +583,8 @@ MCSymbol *X86AsmPrinter::GetCPISymbol(unsigned CPID) const {
   return AsmPrinter::GetCPISymbol(CPID);
 }
 
-void X86AsmPrinter::GenerateExportDirective(const MCSymbol *Sym, bool IsData) {
-  SmallString<128> Directive;
-  raw_svector_ostream OS(Directive);
-  StringRef Name = Sym->getName();
-  Triple TT(TM.getTargetTriple());
-
-  if (TT.isKnownWindowsMSVCEnvironment())
-    OS << " /EXPORT:";
-  else
-    OS << " -export:";
-
-  if ((TT.isWindowsGNUEnvironment() || TT.isWindowsCygwinEnvironment()) &&
-      (Name[0] == getDataLayout().getGlobalPrefix()))
-    Name = Name.drop_front();
-
-  OS << Name;
-
-  if (IsData) {
-    if (TT.isKnownWindowsMSVCEnvironment())
-      OS << ",DATA";
-    else
-      OS << ",data";
-  }
-
-  OS.flush();
-  OutStreamer.EmitBytes(Directive);
-}
-
 void X86AsmPrinter::EmitEndOfAsmFile(Module &M) {
-  Triple TT(TM.getTargetTriple());
+  const Triple &TT = TM.getTargetTriple();
 
   if (TT.isOSBinFormatMachO()) {
     // All darwin targets use mach-o.
@@ -650,115 +596,104 @@ void X86AsmPrinter::EmitEndOfAsmFile(Module &M) {
 
     Stubs = MMIMacho.GetFnStubList();
     if (!Stubs.empty()) {
-      const MCSection *TheSection =
-        OutContext.getMachOSection("__IMPORT", "__jump_table",
-                                   MachO::S_SYMBOL_STUBS |
-                                   MachO::S_ATTR_SELF_MODIFYING_CODE |
-                                   MachO::S_ATTR_PURE_INSTRUCTIONS,
-                                   5, SectionKind::getMetadata());
-      OutStreamer.SwitchSection(TheSection);
+      MCSection *TheSection = OutContext.getMachOSection(
+          "__IMPORT", "__jump_table",
+          MachO::S_SYMBOL_STUBS | MachO::S_ATTR_SELF_MODIFYING_CODE |
+              MachO::S_ATTR_PURE_INSTRUCTIONS,
+          5, SectionKind::getMetadata());
+      OutStreamer->SwitchSection(TheSection);
 
       for (const auto &Stub : Stubs) {
         // L_foo$stub:
-        OutStreamer.EmitLabel(Stub.first);
+        OutStreamer->EmitLabel(Stub.first);
         //   .indirect_symbol _foo
-        OutStreamer.EmitSymbolAttribute(Stub.second.getPointer(),
-                                        MCSA_IndirectSymbol);
+        OutStreamer->EmitSymbolAttribute(Stub.second.getPointer(),
+                                         MCSA_IndirectSymbol);
         // hlt; hlt; hlt; hlt; hlt     hlt = 0xf4.
         const char HltInsts[] = "\xf4\xf4\xf4\xf4\xf4";
-        OutStreamer.EmitBytes(StringRef(HltInsts, 5));
+        OutStreamer->EmitBytes(StringRef(HltInsts, 5));
       }
 
       Stubs.clear();
-      OutStreamer.AddBlankLine();
+      OutStreamer->AddBlankLine();
     }
 
     // Output stubs for external and common global variables.
     Stubs = MMIMacho.GetGVStubList();
     if (!Stubs.empty()) {
-      const MCSection *TheSection =
-        OutContext.getMachOSection("__IMPORT", "__pointers",
-                                   MachO::S_NON_LAZY_SYMBOL_POINTERS,
-                                   SectionKind::getMetadata());
-      OutStreamer.SwitchSection(TheSection);
+      MCSection *TheSection = OutContext.getMachOSection(
+          "__IMPORT", "__pointers", MachO::S_NON_LAZY_SYMBOL_POINTERS,
+          SectionKind::getMetadata());
+      OutStreamer->SwitchSection(TheSection);
 
       for (auto &Stub : Stubs)
-        emitNonLazySymbolPointer(OutStreamer, Stub.first, Stub.second);
+        emitNonLazySymbolPointer(*OutStreamer, Stub.first, Stub.second);
 
       Stubs.clear();
-      OutStreamer.AddBlankLine();
+      OutStreamer->AddBlankLine();
     }
 
     Stubs = MMIMacho.GetHiddenGVStubList();
     if (!Stubs.empty()) {
-      const MCSection *TheSection =
-        OutContext.getMachOSection("__IMPORT", "__pointers",
-                                   MachO::S_NON_LAZY_SYMBOL_POINTERS,
-                                   SectionKind::getMetadata());
-      OutStreamer.SwitchSection(TheSection);
+      MCSection *TheSection = OutContext.getMachOSection(
+          "__IMPORT", "__pointers", MachO::S_NON_LAZY_SYMBOL_POINTERS,
+          SectionKind::getMetadata());
+      OutStreamer->SwitchSection(TheSection);
 
       for (auto &Stub : Stubs)
-        emitNonLazySymbolPointer(OutStreamer, Stub.first, Stub.second);
+        emitNonLazySymbolPointer(*OutStreamer, Stub.first, Stub.second);
 
       Stubs.clear();
-      OutStreamer.AddBlankLine();
+      OutStreamer->AddBlankLine();
     }
 
     SM.serializeToStackMapSection();
+    FM.serializeToFaultMapSection();
 
     // Funny Darwin hack: This flag tells the linker that no global symbols
     // contain code that falls through to other global symbols (e.g. the obvious
     // implementation of multiple entry points).  If this doesn't occur, the
     // linker can safely perform dead code stripping.  Since LLVM never
     // generates code that does this, it is always safe to set.
-    OutStreamer.EmitAssemblerFlag(MCAF_SubsectionsViaSymbols);
+    OutStreamer->EmitAssemblerFlag(MCAF_SubsectionsViaSymbols);
   }
 
   if (TT.isKnownWindowsMSVCEnvironment() && MMI->usesVAFloatArgument()) {
     StringRef SymbolName =
         (TT.getArch() == Triple::x86_64) ? "_fltused" : "__fltused";
-    MCSymbol *S = MMI->getContext().GetOrCreateSymbol(SymbolName);
-    OutStreamer.EmitSymbolAttribute(S, MCSA_Global);
+    MCSymbol *S = MMI->getContext().getOrCreateSymbol(SymbolName);
+    OutStreamer->EmitSymbolAttribute(S, MCSA_Global);
   }
 
   if (TT.isOSBinFormatCOFF()) {
-    // Necessary for dllexport support
-    std::vector<const MCSymbol*> DLLExportedFns, DLLExportedGlobals;
+    const TargetLoweringObjectFileCOFF &TLOFCOFF =
+        static_cast<const TargetLoweringObjectFileCOFF&>(getObjFileLowering());
 
-    for (const auto &Function : M)
-      if (Function.hasDLLExportStorageClass() && !Function.isDeclaration())
-        DLLExportedFns.push_back(getSymbol(&Function));
+    std::string Flags;
+    raw_string_ostream FlagsOS(Flags);
 
+    for (const auto &Function : M)
+      TLOFCOFF.emitLinkerFlagsForGlobal(FlagsOS, &Function, *Mang);
     for (const auto &Global : M.globals())
-      if (Global.hasDLLExportStorageClass() && !Global.isDeclaration())
-        DLLExportedGlobals.push_back(getSymbol(&Global));
+      TLOFCOFF.emitLinkerFlagsForGlobal(FlagsOS, &Global, *Mang);
+    for (const auto &Alias : M.aliases())
+      TLOFCOFF.emitLinkerFlagsForGlobal(FlagsOS, &Alias, *Mang);
 
-    for (const auto &Alias : M.aliases()) {
-      if (!Alias.hasDLLExportStorageClass())
-        continue;
+    FlagsOS.flush();
 
-      if (Alias.getType()->getElementType()->isFunctionTy())
-        DLLExportedFns.push_back(getSymbol(&Alias));
-      else
-        DLLExportedGlobals.push_back(getSymbol(&Alias));
+    // Output collected flags.
+    if (!Flags.empty()) {
+      OutStreamer->SwitchSection(TLOFCOFF.getDrectveSection());
+      OutStreamer->EmitBytes(Flags);
     }
 
-    // Output linker support code for dllexported globals on windows.
-    if (!DLLExportedGlobals.empty() || !DLLExportedFns.empty()) {
-      const TargetLoweringObjectFileCOFF &TLOFCOFF =
-        static_cast<const TargetLoweringObjectFileCOFF&>(getObjFileLowering());
-
-      OutStreamer.SwitchSection(TLOFCOFF.getDrectveSection());
-
-      for (auto & Symbol : DLLExportedGlobals)
-        GenerateExportDirective(Symbol, /*IsData=*/true);
-      for (auto & Symbol : DLLExportedFns)
-        GenerateExportDirective(Symbol, /*IsData=*/false);
-    }
+    SM.serializeToStackMapSection();
   }
 
-  if (TT.isOSBinFormatELF())
+  if (TT.isOSBinFormatELF()) {
     SM.serializeToStackMapSection();
+    FM.serializeToFaultMapSection();
+  }
 }
 
 //===----------------------------------------------------------------------===//
diff --git a/lib/Target/X86/X86AsmPrinter.h b/lib/Target/X86/X86AsmPrinter.h
index d63c2bb138f..7f5d127c68d 100644
--- a/lib/Target/X86/X86AsmPrinter.h
+++ b/lib/Target/X86/X86AsmPrinter.h
@@ -12,8 +12,8 @@
 
 #include "X86Subtarget.h"
 #include "llvm/CodeGen/AsmPrinter.h"
+#include "llvm/CodeGen/FaultMaps.h"
 #include "llvm/CodeGen/StackMaps.h"
-#include "llvm/IR/Module.h"
 #include "llvm/Target/TargetMachine.h"
 
 // Implemented in X86MCInstLower.cpp
@@ -28,8 +28,7 @@ class MCSymbol;
 class LLVM_LIBRARY_VISIBILITY X86AsmPrinter : public AsmPrinter {
   const X86Subtarget *Subtarget;
   StackMaps SM;
-
-  void GenerateExportDirective(const MCSymbol *Sym, bool IsData);
+  FaultMaps FM;
 
   // This utility class tracks the length of a stackmap instruction's 'shadow'.
   // It is used by the X86AsmPrinter to ensure that the stackmap shadow
@@ -82,14 +81,17 @@ class LLVM_LIBRARY_VISIBILITY X86AsmPrinter : public AsmPrinter {
 
   void InsertStackMapShadows(MachineFunction &MF);
   void LowerSTACKMAP(const MachineInstr &MI);
-  void LowerPATCHPOINT(const MachineInstr &MI);
+  void LowerPATCHPOINT(const MachineInstr &MI, X86MCInstLower &MCIL);
+  void LowerSTATEPOINT(const MachineInstr &MI, X86MCInstLower &MCIL);
+  void LowerFAULTING_LOAD_OP(const MachineInstr &MI, X86MCInstLower &MCIL);
 
   void LowerTlsAddr(X86MCInstLower &MCInstLowering, const MachineInstr &MI);
 
  public:
    explicit X86AsmPrinter(TargetMachine &TM,
                           std::unique_ptr<MCStreamer> Streamer)
-       : AsmPrinter(TM, std::move(Streamer)), SM(*this), SMShadowTracker(TM) {}
+       : AsmPrinter(TM, std::move(Streamer)), SM(*this), FM(*this),
+         SMShadowTracker(TM) {}
 
   const char *getPassName() const override {
     return "X86 Assembly / Object Emitter";
@@ -103,15 +105,8 @@ class LLVM_LIBRARY_VISIBILITY X86AsmPrinter : public AsmPrinter {
 
   void EmitInstruction(const MachineInstr *MI) override;
 
-  bool UseReadOnlyJumpTables() const override; // @LOCALMOD
-
-  unsigned GetTargetBasicBlockAlign() const override; // @LOCLAMOD
-
-  unsigned
-  GetTargetLabelAlign(const MachineInstr *MI) const override; //@LOCALMOD
-
   void EmitBasicBlockEnd(const MachineBasicBlock &MBB) override {
-    SMShadowTracker.emitShadowPadding(OutStreamer, getSubtargetInfo());
+    SMShadowTracker.emitShadowPadding(*OutStreamer, getSubtargetInfo());
   }
 
   bool PrintAsmOperand(const MachineInstr *MI, unsigned OpNo,
@@ -125,14 +120,7 @@ class LLVM_LIBRARY_VISIBILITY X86AsmPrinter : public AsmPrinter {
   MCSymbol *GetCPISymbol(unsigned CPID) const override;
 
   bool doInitialization(Module &M) override {
-    // @LOCALMOD-BEGIN -- disable ShadowMapShadowTracker for NaCl for now.
-    // This requires knowing the size of instructions, and with NaCl pseudos
-    // being expanded late by the streamer, it isn't going to be counting
-    // the expanded instruction. It might not be counting the bundle padding
-    // correctly either.
-    if (!Triple(M.getTargetTriple()).isOSNaCl())
-      SMShadowTracker.reset(0);
-    // @LOCALMOD-END
+    SMShadowTracker.reset(0);
     SM.reset();
     return AsmPrinter::doInitialization(M);
   }
diff --git a/lib/Target/X86/X86CallFrameOptimization.cpp b/lib/Target/X86/X86CallFrameOptimization.cpp
index 5e8d37431e1..323c5bfd408 100644
--- a/lib/Target/X86/X86CallFrameOptimization.cpp
+++ b/lib/Target/X86/X86CallFrameOptimization.cpp
@@ -54,7 +54,7 @@ class X86CallFrameOptimization : public MachineFunctionPass {
   struct CallContext {
     CallContext()
         : Call(nullptr), SPCopy(nullptr), ExpectedDist(0),
-          MovVector(4, nullptr), NoStackParams(false), UsePush(false){};
+          MovVector(4, nullptr), NoStackParams(false), UsePush(false){}
 
     // Actuall call instruction
     MachineInstr *Call;
@@ -78,7 +78,7 @@ class X86CallFrameOptimization : public MachineFunctionPass {
   typedef DenseMap<MachineInstr *, CallContext> ContextMap;
 
   bool isLegal(MachineFunction &MF);
-  
+
   bool isProfitable(MachineFunction &MF, ContextMap &CallSeqMap);
 
   void collectCallInfo(MachineFunction &MF, MachineBasicBlock &MBB,
@@ -90,6 +90,13 @@ class X86CallFrameOptimization : public MachineFunctionPass {
   MachineInstr *canFoldIntoRegPush(MachineBasicBlock::iterator FrameSetup,
                                    unsigned Reg);
 
+  enum InstClassification { Convert, Skip, Exit };
+
+  InstClassification classifyInstruction(MachineBasicBlock &MBB,
+                                         MachineBasicBlock::iterator MI,
+                                         const X86RegisterInfo &RegInfo,
+                                         DenseSet<unsigned int> &UsedRegs);
+
   const char *getPassName() const override { return "X86 Optimize Call Frame"; }
 
   const TargetInstrInfo *TII;
@@ -105,7 +112,7 @@ FunctionPass *llvm::createX86CallFrameOptimization() {
   return new X86CallFrameOptimization();
 }
 
-// This checks whether the transformation is legal. 
+// This checks whether the transformation is legal.
 // Also returns false in cases where it's potentially legal, but
 // we don't even want to try.
 bool X86CallFrameOptimization::isLegal(MachineFunction &MF) {
@@ -128,8 +135,8 @@ bool X86CallFrameOptimization::isLegal(MachineFunction &MF) {
   // This is bad, and breaks SP adjustment.
   // So, check that all of the frames in the function are closed inside
   // the same block, and, for good measure, that there are no nested frames.
-  int FrameSetupOpcode = TII->getCallFrameSetupOpcode();
-  int FrameDestroyOpcode = TII->getCallFrameDestroyOpcode();
+  unsigned FrameSetupOpcode = TII->getCallFrameSetupOpcode();
+  unsigned FrameDestroyOpcode = TII->getCallFrameDestroyOpcode();
   for (MachineBasicBlock &BB : MF) {
     bool InsideFrameSequence = false;
     for (MachineInstr &MI : BB) {
@@ -170,9 +177,8 @@ bool X86CallFrameOptimization::isProfitable(MachineFunction &MF,
   if (!OptForSize)
     return false;
 
-
   unsigned StackAlign = TFL->getStackAlignment();
-  
+
   int64_t Advantage = 0;
   for (auto CC : CallSeqMap) {
     // Call sites where no parameters are passed on the stack
@@ -205,7 +211,6 @@ bool X86CallFrameOptimization::isProfitable(MachineFunction &MF,
   return (Advantage >= 0);
 }
 
-
 bool X86CallFrameOptimization::runOnMachineFunction(MachineFunction &MF) {
   TII = MF.getSubtarget().getInstrInfo();
   TFL = MF.getSubtarget().getFrameLowering();
@@ -214,7 +219,7 @@ bool X86CallFrameOptimization::runOnMachineFunction(MachineFunction &MF) {
   if (!isLegal(MF))
     return false;
 
-  int FrameSetupOpcode = TII->getCallFrameSetupOpcode();
+  unsigned FrameSetupOpcode = TII->getCallFrameSetupOpcode();
 
   bool Changed = false;
 
@@ -237,6 +242,64 @@ bool X86CallFrameOptimization::runOnMachineFunction(MachineFunction &MF) {
   return Changed;
 }
 
+X86CallFrameOptimization::InstClassification
+X86CallFrameOptimization::classifyInstruction(
+    MachineBasicBlock &MBB, MachineBasicBlock::iterator MI,
+    const X86RegisterInfo &RegInfo, DenseSet<unsigned int> &UsedRegs) {
+  if (MI == MBB.end())
+    return Exit;
+
+  // The instructions we actually care about are movs onto the stack
+  int Opcode = MI->getOpcode();
+  if (Opcode == X86::MOV32mi || Opcode == X86::MOV32mr)
+    return Convert;
+
+  // Not all calling conventions have only stack MOVs between the stack
+  // adjust and the call.
+
+  // We want to tolerate other instructions, to cover more cases.
+  // In particular:
+  // a) PCrel calls, where we expect an additional COPY of the basereg.
+  // b) Passing frame-index addresses.
+  // c) Calling conventions that have inreg parameters. These generate
+  //    both copies and movs into registers.
+  // To avoid creating lots of special cases, allow any instruction
+  // that does not write into memory, does not def or use the stack
+  // pointer, and does not def any register that was used by a preceding
+  // push.
+  // (Reading from memory is allowed, even if referenced through a
+  // frame index, since these will get adjusted properly in PEI)
+
+  // The reason for the last condition is that the pushes can't replace
+  // the movs in place, because the order must be reversed.
+  // So if we have a MOV32mr that uses EDX, then an instruction that defs
+  // EDX, and then the call, after the transformation the push will use
+  // the modified version of EDX, and not the original one.
+  // Since we are still in SSA form at this point, we only need to
+  // make sure we don't clobber any *physical* registers that were
+  // used by an earlier mov that will become a push.
+
+  if (MI->isCall() || MI->mayStore())
+    return Exit;
+
+  for (const MachineOperand &MO : MI->operands()) {
+    if (!MO.isReg())
+      continue;
+    unsigned int Reg = MO.getReg();
+    if (!RegInfo.isPhysicalRegister(Reg))
+      continue;
+    if (RegInfo.regsOverlap(Reg, RegInfo.getStackRegister()))
+      return Exit;
+    if (MO.isDef()) {
+      for (unsigned int U : UsedRegs)
+        if (RegInfo.regsOverlap(Reg, U))
+          return Exit;
+    }
+  }
+
+  return Skip;
+}
+
 void X86CallFrameOptimization::collectCallInfo(MachineFunction &MF,
                                                MachineBasicBlock &MBB,
                                                MachineBasicBlock::iterator I,
@@ -246,7 +309,7 @@ void X86CallFrameOptimization::collectCallInfo(MachineFunction &MF,
   const X86RegisterInfo &RegInfo = *static_cast<const X86RegisterInfo *>(
                                        MF.getSubtarget().getRegisterInfo());
   unsigned StackPtr = RegInfo.getStackRegister();
-  int FrameDestroyOpcode = TII->getCallFrameDestroyOpcode();
+  unsigned FrameDestroyOpcode = TII->getCallFrameDestroyOpcode();
 
   // We expect to enter this at the beginning of a call sequence
   assert(I->getOpcode() == TII->getCallFrameSetupOpcode());
@@ -254,8 +317,8 @@ void X86CallFrameOptimization::collectCallInfo(MachineFunction &MF,
 
   // How much do we adjust the stack? This puts an upper bound on
   // the number of parameters actually passed on it.
-  unsigned int MaxAdjust = FrameSetup->getOperand(0).getImm() / 4;  
-  
+  unsigned int MaxAdjust = FrameSetup->getOperand(0).getImm() / 4;
+
   // A zero adjustment means no stack parameters
   if (!MaxAdjust) {
     Context.NoStackParams = true;
@@ -284,11 +347,17 @@ void X86CallFrameOptimization::collectCallInfo(MachineFunction &MF,
   if (MaxAdjust > 4)
     Context.MovVector.resize(MaxAdjust, nullptr);
 
-  do {
-    int Opcode = I->getOpcode();
-    if (Opcode != X86::MOV32mi && Opcode != X86::MOV32mr)
-      break;
+  InstClassification Classification;
+  DenseSet<unsigned int> UsedRegs;
 
+  while ((Classification = classifyInstruction(MBB, I, RegInfo, UsedRegs)) !=
+         Exit) {
+    if (Classification == Skip) {
+      ++I;
+      continue;
+    }
+
+    // We know the instruction is a MOV32mi/MOV32mr.
     // We only want movs of the form:
     // movl imm/r32, k(%esp)
     // If we run into something else, bail.
@@ -323,24 +392,20 @@ void X86CallFrameOptimization::collectCallInfo(MachineFunction &MF,
       return;
     Context.MovVector[StackDisp] = I;
 
-    ++I;
-  } while (I != MBB.end());
-
-  // We now expect the end of the sequence - a call and a stack adjust.
-  if (I == MBB.end())
-    return;
+    for (const MachineOperand &MO : I->uses()) {
+      if (!MO.isReg())
+        continue;
+      unsigned int Reg = MO.getReg();
+      if (RegInfo.isPhysicalRegister(Reg))
+        UsedRegs.insert(Reg);
+    }
 
-  // For PCrel calls, we expect an additional COPY of the basereg.
-  // If we find one, skip it.
-  if (I->isCopy()) {
-    if (I->getOperand(1).getReg() ==
-        MF.getInfo<X86MachineFunctionInfo>()->getGlobalBaseReg())
-      ++I;
-    else
-      return;
+    ++I;
   }
 
-  if (!I->isCall())
+  // We now expect the end of the sequence. If we stopped early,
+  // or reached the end of the block without finding a call, bail.
+  if (I == MBB.end() || !I->isCall())
     return;
 
   Context.Call = I;
diff --git a/lib/Target/X86/X86CallingConv.td b/lib/Target/X86/X86CallingConv.td
index f7b17eaa2f3..8f88888f5ce 100644
--- a/lib/Target/X86/X86CallingConv.td
+++ b/lib/Target/X86/X86CallingConv.td
@@ -34,11 +34,22 @@ def RetCC_X86Common : CallingConv<[
   //
   // For code that doesn't care about the ABI, we allow returning more than two
   // integer values in registers.
+  CCIfType<[i1],  CCPromoteToType<i8>>,
   CCIfType<[i8] , CCAssignToReg<[AL, DL, CL]>>,
   CCIfType<[i16], CCAssignToReg<[AX, DX, CX]>>,
   CCIfType<[i32], CCAssignToReg<[EAX, EDX, ECX]>>,
   CCIfType<[i64], CCAssignToReg<[RAX, RDX, RCX]>>,
 
+  // Boolean vectors of AVX-512 are returned in SIMD registers.
+  // The call from AVX to AVX-512 function should work,
+  // since the boolean types in AVX/AVX2 are promoted by default.
+  CCIfType<[v2i1],  CCPromoteToType<v2i64>>,
+  CCIfType<[v4i1],  CCPromoteToType<v4i32>>,
+  CCIfType<[v8i1],  CCPromoteToType<v8i16>>,
+  CCIfType<[v16i1], CCPromoteToType<v16i8>>,
+  CCIfType<[v32i1], CCPromoteToType<v32i8>>,
+  CCIfType<[v64i1], CCPromoteToType<v64i8>>,
+
   // Vector types are returned in XMM0 and XMM1, when they fit.  XMM2 and XMM3
   // can only be used by ABI non-compliant code. If the target doesn't have XMM
   // registers, it won't have vector types.
@@ -241,8 +252,8 @@ def CC_X86_64_C : CallingConv<[
   // Handles byval parameters.
   CCIfByVal<CCPassByVal<8, 8>>,
 
-  // Promote i8/i16 arguments to i32.
-  CCIfType<[i8, i16], CCPromoteToType<i32>>,
+  // Promote i1/i8/i16 arguments to i32.
+  CCIfType<[i1, i8, i16], CCPromoteToType<i32>>,
 
   // The 'nest' parameter, if any, is passed in R10.
   CCIfNest<CCIfSubtarget<"isTarget64BitILP32()", CCAssignToReg<[R10D]>>>,
@@ -258,6 +269,16 @@ def CC_X86_64_C : CallingConv<[
             CCIfSubtarget<"hasSSE2()",
             CCPromoteToType<v2i64>>>>,
 
+  // Boolean vectors of AVX-512 are passed in SIMD registers.
+  // The call from AVX to AVX-512 function should work,
+  // since the boolean types in AVX/AVX2 are promoted by default.
+  CCIfType<[v2i1],  CCPromoteToType<v2i64>>,
+  CCIfType<[v4i1],  CCPromoteToType<v4i32>>,
+  CCIfType<[v8i1],  CCPromoteToType<v8i16>>,
+  CCIfType<[v16i1], CCPromoteToType<v16i8>>,
+  CCIfType<[v32i1], CCPromoteToType<v32i8>>,
+  CCIfType<[v64i1], CCPromoteToType<v64i8>>,
+
   // The first 8 FP/Vector arguments are passed in XMM registers.
   CCIfType<[f32, f64, v16i8, v8i16, v4i32, v2i64, v4f32, v2f64],
             CCIfSubtarget<"hasSSE1()",
@@ -303,8 +324,8 @@ def CC_X86_Win64_C : CallingConv<[
   // FIXME: Handle byval stuff.
   // FIXME: Handle varargs.
 
-  // Promote i8/i16 arguments to i32.
-  CCIfType<[i8, i16], CCPromoteToType<i32>>,
+  // Promote i1/i8/i16 arguments to i32.
+  CCIfType<[i1, i8, i16], CCPromoteToType<i32>>,
 
   // The 'nest' parameter, if any, is passed in R10.
   CCIfNest<CCAssignToReg<[R10]>>,
@@ -424,9 +445,61 @@ def CC_X86_64_AnyReg : CallingConv<[
 // X86 C Calling Convention
 //===----------------------------------------------------------------------===//
 
+/// CC_X86_32_Vector_Common - In all X86-32 calling conventions, extra vector
+/// values are spilled on the stack.
+def CC_X86_32_Vector_Common : CallingConv<[
+  // Other SSE vectors get 16-byte stack slots that are 16-byte aligned.
+  CCIfType<[v16i8, v8i16, v4i32, v2i64, v4f32, v2f64], CCAssignToStack<16, 16>>,
+
+  // 256-bit AVX vectors get 32-byte stack slots that are 32-byte aligned.
+  CCIfType<[v32i8, v16i16, v8i32, v4i64, v8f32, v4f64],
+           CCAssignToStack<32, 32>>,
+
+  // 512-bit AVX 512-bit vectors get 64-byte stack slots that are 64-byte aligned.
+  CCIfType<[v64i8, v32i16, v16i32, v8i64, v16f32, v8f64],
+           CCAssignToStack<64, 64>>
+]>;
+
+// CC_X86_32_Vector_Standard - The first 3 vector arguments are passed in
+// vector registers
+def CC_X86_32_Vector_Standard : CallingConv<[
+  // SSE vector arguments are passed in XMM registers.
+  CCIfNotVarArg<CCIfType<[v16i8, v8i16, v4i32, v2i64, v4f32, v2f64],
+                CCAssignToReg<[XMM0, XMM1, XMM2]>>>,
+
+  // AVX 256-bit vector arguments are passed in YMM registers.
+  CCIfNotVarArg<CCIfType<[v32i8, v16i16, v8i32, v4i64, v8f32, v4f64],
+                CCIfSubtarget<"hasFp256()",
+                CCAssignToReg<[YMM0, YMM1, YMM2]>>>>,
+
+  // AVX 512-bit vector arguments are passed in ZMM registers.
+  CCIfNotVarArg<CCIfType<[v64i8, v32i16, v16i32, v8i64, v16f32, v8f64],
+                CCAssignToReg<[ZMM0, ZMM1, ZMM2]>>>,
+
+  CCDelegateTo<CC_X86_32_Vector_Common>
+]>;
+
+// CC_X86_32_Vector_Darwin - The first 4 vector arguments are passed in
+// vector registers.
+def CC_X86_32_Vector_Darwin : CallingConv<[
+  // SSE vector arguments are passed in XMM registers.
+  CCIfNotVarArg<CCIfType<[v16i8, v8i16, v4i32, v2i64, v4f32, v2f64],
+                CCAssignToReg<[XMM0, XMM1, XMM2, XMM3]>>>,
+
+  // AVX 256-bit vector arguments are passed in YMM registers.
+  CCIfNotVarArg<CCIfType<[v32i8, v16i16, v8i32, v4i64, v8f32, v4f64],
+                CCIfSubtarget<"hasFp256()",
+                CCAssignToReg<[YMM0, YMM1, YMM2, YMM3]>>>>,
+
+  // AVX 512-bit vector arguments are passed in ZMM registers.
+  CCIfNotVarArg<CCIfType<[v64i8, v32i16, v16i32, v8i64, v16f32, v8f64],
+                CCAssignToReg<[ZMM0, ZMM1, ZMM2, ZMM3]>>>,
+
+  CCDelegateTo<CC_X86_32_Vector_Common>
+]>;
+
 /// CC_X86_32_Common - In all X86-32 calling conventions, extra integers and FP
-/// values are spilled on the stack, and the first 4 vector values go in XMM
-/// regs.
+/// values are spilled on the stack.
 def CC_X86_32_Common : CallingConv<[
   // Handles byval parameters.
   CCIfByVal<CCPassByVal<4, 4>>,
@@ -452,37 +525,30 @@ def CC_X86_32_Common : CallingConv<[
   // Long doubles get slots whose size depends on the subtarget.
   CCIfType<[f80], CCAssignToStack<0, 4>>,
 
-  // The first 4 SSE vector arguments are passed in XMM registers.
-  CCIfNotVarArg<CCIfType<[v16i8, v8i16, v4i32, v2i64, v4f32, v2f64],
-                CCAssignToReg<[XMM0, XMM1, XMM2, XMM3]>>>,
-
-  // The first 4 AVX 256-bit vector arguments are passed in YMM registers.
-  CCIfNotVarArg<CCIfType<[v32i8, v16i16, v8i32, v4i64, v8f32, v4f64],
-                CCIfSubtarget<"hasFp256()",
-                CCAssignToReg<[YMM0, YMM1, YMM2, YMM3]>>>>,
-
-  // The first 4 AVX 512-bit vector arguments are passed in ZMM registers.
-  CCIfNotVarArg<CCIfType<[v64i8, v32i16, v16i32, v8i64, v16f32, v8f64],
-                CCAssignToReg<[ZMM0, ZMM1, ZMM2, ZMM3]>>>,
-
-  // Other SSE vectors get 16-byte stack slots that are 16-byte aligned.
-  CCIfType<[v16i8, v8i16, v4i32, v2i64, v4f32, v2f64], CCAssignToStack<16, 16>>,
-
-  // 256-bit AVX vectors get 32-byte stack slots that are 32-byte aligned.
-  CCIfType<[v32i8, v16i16, v8i32, v4i64, v8f32, v4f64],
-           CCAssignToStack<32, 32>>,
-
-  // 512-bit AVX 512-bit vectors get 64-byte stack slots that are 64-byte aligned.
-  CCIfType<[v64i8, v32i16, v16i32, v8i64, v16f32, v8f64],
-           CCAssignToStack<64, 64>>,
+  // Boolean vectors of AVX-512 are passed in SIMD registers.
+  // The call from AVX to AVX-512 function should work,
+  // since the boolean types in AVX/AVX2 are promoted by default.
+  CCIfType<[v2i1],  CCPromoteToType<v2i64>>,
+  CCIfType<[v4i1],  CCPromoteToType<v4i32>>,
+  CCIfType<[v8i1],  CCPromoteToType<v8i16>>,
+  CCIfType<[v16i1], CCPromoteToType<v16i8>>,
+  CCIfType<[v32i1], CCPromoteToType<v32i8>>,
+  CCIfType<[v64i1], CCPromoteToType<v64i8>>,
 
   // __m64 vectors get 8-byte stack slots that are 4-byte aligned. They are
   // passed in the parameter area.
-  CCIfType<[x86mmx], CCAssignToStack<8, 4>>]>;
+  CCIfType<[x86mmx], CCAssignToStack<8, 4>>,
+
+  // Darwin passes vectors in a form that differs from the i386 psABI
+  CCIfSubtarget<"isTargetDarwin()", CCDelegateTo<CC_X86_32_Vector_Darwin>>,
+
+  // Otherwise, drop to 'normal' X86-32 CC
+  CCDelegateTo<CC_X86_32_Vector_Standard>
+]>;
 
 def CC_X86_32_C : CallingConv<[
-  // Promote i8/i16 arguments to i32.
-  CCIfType<[i8, i16], CCPromoteToType<i32>>,
+  // Promote i1/i8/i16 arguments to i32.
+  CCIfType<[i1, i8, i16], CCPromoteToType<i32>>,
 
   // The 'nest' parameter, if any, is passed in ECX.
   CCIfNest<CCAssignToReg<[ECX]>>,
@@ -496,8 +562,8 @@ def CC_X86_32_C : CallingConv<[
 ]>;
 
 def CC_X86_32_FastCall : CallingConv<[
-  // Promote i8/i16 arguments to i32.
-  CCIfType<[i8, i16], CCPromoteToType<i32>>,
+  // Promote i1/i8/i16 arguments to i32.
+  CCIfType<[i1, i8, i16], CCPromoteToType<i32>>,
 
   // The 'nest' parameter, if any, is passed in EAX.
   CCIfNest<CCAssignToReg<[EAX]>>,
@@ -542,15 +608,15 @@ def CC_X86_32_ThisCall_Common : CallingConv<[
 ]>;
 
 def CC_X86_32_ThisCall_Mingw : CallingConv<[
-  // Promote i8/i16 arguments to i32.
-  CCIfType<[i8, i16], CCPromoteToType<i32>>,
+  // Promote i1/i8/i16 arguments to i32.
+  CCIfType<[i1, i8, i16], CCPromoteToType<i32>>,
 
   CCDelegateTo<CC_X86_32_ThisCall_Common>
 ]>;
 
 def CC_X86_32_ThisCall_Win : CallingConv<[
-  // Promote i8/i16 arguments to i32.
-  CCIfType<[i8, i16], CCPromoteToType<i32>>,
+  // Promote i1/i8/i16 arguments to i32.
+  CCIfType<[i1, i8, i16], CCPromoteToType<i32>>,
 
   // Pass sret arguments indirectly through stack.
   CCIfSRet<CCAssignToStack<4, 4>>,
@@ -569,8 +635,8 @@ def CC_X86_32_FastCC : CallingConv<[
   // puts arguments in registers.
   CCIfByVal<CCPassByVal<4, 4>>,
 
-  // Promote i8/i16 arguments to i32.
-  CCIfType<[i8, i16], CCPromoteToType<i32>>,
+  // Promote i1/i8/i16 arguments to i32.
+  CCIfType<[i1, i8, i16], CCPromoteToType<i32>>,
 
   // The 'nest' parameter, if any, is passed in EAX.
   CCIfNest<CCAssignToReg<[EAX]>>,
@@ -738,9 +804,3 @@ def CSR_64_Intel_OCL_BI_AVX    : CalleeSavedRegs<(add CSR_64,
 def CSR_64_Intel_OCL_BI_AVX512 : CalleeSavedRegs<(add RBX, RDI, RSI, R14, R15,
                                                   (sequence "ZMM%u", 16, 31),
                                                   K4, K5, K6, K7)>;
-
-// @LOCALMOD-BEGIN
-// NaCl x86-64 (R15 cannot be modified):
-def CSR_NaCl64 : CalleeSavedRegs<(add RBX, R12, R13, R14, RBP)>;
-def CSR_NaCl64EHRet : CalleeSavedRegs<(add RAX, RDX, CSR_NaCl64)>;
-// @LOCALMOD-END
diff --git a/lib/Target/X86/X86ExpandPseudo.cpp b/lib/Target/X86/X86ExpandPseudo.cpp
new file mode 100644
index 00000000000..6a5a28e546f
--- /dev/null
+++ b/lib/Target/X86/X86ExpandPseudo.cpp
@@ -0,0 +1,179 @@
+//===------- X86ExpandPseudo.cpp - Expand pseudo instructions -------------===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This file contains a pass that expands pseudo instructions into target
+// instructions to allow proper scheduling, if-conversion, other late
+// optimizations, or simply the encoding of the instructions.
+//
+//===----------------------------------------------------------------------===//
+
+#include "X86.h"
+#include "X86FrameLowering.h"
+#include "X86InstrBuilder.h"
+#include "X86InstrInfo.h"
+#include "X86MachineFunctionInfo.h"
+#include "X86Subtarget.h"
+#include "llvm/CodeGen/Passes.h" // For IDs of passes that are preserved.
+#include "llvm/CodeGen/MachineFunctionPass.h"
+#include "llvm/CodeGen/MachineInstrBuilder.h"
+#include "llvm/IR/GlobalValue.h"
+using namespace llvm;
+
+#define DEBUG_TYPE "x86-pseudo"
+
+namespace {
+class X86ExpandPseudo : public MachineFunctionPass {
+public:
+  static char ID;
+  X86ExpandPseudo() : MachineFunctionPass(ID) {}
+
+  void getAnalysisUsage(AnalysisUsage &AU) const override {
+    AU.setPreservesCFG();
+    AU.addPreservedID(MachineLoopInfoID);
+    AU.addPreservedID(MachineDominatorsID);
+    MachineFunctionPass::getAnalysisUsage(AU);
+  }
+
+  const X86Subtarget *STI;
+  const X86InstrInfo *TII;
+  const X86RegisterInfo *TRI;
+  const X86FrameLowering *X86FL;
+
+  bool runOnMachineFunction(MachineFunction &Fn) override;
+
+  const char *getPassName() const override {
+    return "X86 pseudo instruction expansion pass";
+  }
+
+private:
+  bool ExpandMI(MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI);
+  bool ExpandMBB(MachineBasicBlock &MBB);
+};
+char X86ExpandPseudo::ID = 0;
+} // End anonymous namespace.
+
+/// If \p MBBI is a pseudo instruction, this method expands
+/// it to the corresponding (sequence of) actual instruction(s).
+/// \returns true if \p MBBI has been expanded.
+bool X86ExpandPseudo::ExpandMI(MachineBasicBlock &MBB,
+                               MachineBasicBlock::iterator MBBI) {
+  MachineInstr &MI = *MBBI;
+  unsigned Opcode = MI.getOpcode();
+  DebugLoc DL = MBBI->getDebugLoc();
+  switch (Opcode) {
+  default:
+    return false;
+  case X86::TCRETURNdi:
+  case X86::TCRETURNri:
+  case X86::TCRETURNmi:
+  case X86::TCRETURNdi64:
+  case X86::TCRETURNri64:
+  case X86::TCRETURNmi64: {
+    bool isMem = Opcode == X86::TCRETURNmi || Opcode == X86::TCRETURNmi64;
+    MachineOperand &JumpTarget = MBBI->getOperand(0);
+    MachineOperand &StackAdjust = MBBI->getOperand(isMem ? 5 : 1);
+    assert(StackAdjust.isImm() && "Expecting immediate value.");
+
+    // Adjust stack pointer.
+    int StackAdj = StackAdjust.getImm();
+
+    if (StackAdj) {
+      // Check for possible merge with preceding ADD instruction.
+      StackAdj += X86FL->mergeSPUpdates(MBB, MBBI, true);
+      X86FL->emitSPUpdate(MBB, MBBI, StackAdj, /*InEpilogue=*/true);
+    }
+
+    // Jump to label or value in register.
+    bool IsWin64 = STI->isTargetWin64();
+    if (Opcode == X86::TCRETURNdi || Opcode == X86::TCRETURNdi64) {
+      unsigned Op = (Opcode == X86::TCRETURNdi)
+                        ? X86::TAILJMPd
+                        : (IsWin64 ? X86::TAILJMPd64_REX : X86::TAILJMPd64);
+      MachineInstrBuilder MIB = BuildMI(MBB, MBBI, DL, TII->get(Op));
+      if (JumpTarget.isGlobal())
+        MIB.addGlobalAddress(JumpTarget.getGlobal(), JumpTarget.getOffset(),
+                             JumpTarget.getTargetFlags());
+      else {
+        assert(JumpTarget.isSymbol());
+        MIB.addExternalSymbol(JumpTarget.getSymbolName(),
+                              JumpTarget.getTargetFlags());
+      }
+    } else if (Opcode == X86::TCRETURNmi || Opcode == X86::TCRETURNmi64) {
+      unsigned Op = (Opcode == X86::TCRETURNmi)
+                        ? X86::TAILJMPm
+                        : (IsWin64 ? X86::TAILJMPm64_REX : X86::TAILJMPm64);
+      MachineInstrBuilder MIB = BuildMI(MBB, MBBI, DL, TII->get(Op));
+      for (unsigned i = 0; i != 5; ++i)
+        MIB.addOperand(MBBI->getOperand(i));
+    } else if (Opcode == X86::TCRETURNri64) {
+      BuildMI(MBB, MBBI, DL,
+              TII->get(IsWin64 ? X86::TAILJMPr64_REX : X86::TAILJMPr64))
+          .addReg(JumpTarget.getReg(), RegState::Kill);
+    } else {
+      BuildMI(MBB, MBBI, DL, TII->get(X86::TAILJMPr))
+          .addReg(JumpTarget.getReg(), RegState::Kill);
+    }
+
+    MachineInstr *NewMI = std::prev(MBBI);
+    NewMI->copyImplicitOps(*MBBI->getParent()->getParent(), MBBI);
+
+    // Delete the pseudo instruction TCRETURN.
+    MBB.erase(MBBI);
+
+    return true;
+  }
+  case X86::EH_RETURN:
+  case X86::EH_RETURN64: {
+    MachineOperand &DestAddr = MBBI->getOperand(0);
+    assert(DestAddr.isReg() && "Offset should be in register!");
+    const bool Uses64BitFramePtr =
+        STI->isTarget64BitLP64() || STI->isTargetNaCl64();
+    unsigned StackPtr = TRI->getStackRegister();
+    BuildMI(MBB, MBBI, DL,
+            TII->get(Uses64BitFramePtr ? X86::MOV64rr : X86::MOV32rr), StackPtr)
+        .addReg(DestAddr.getReg());
+    // The EH_RETURN pseudo is really removed during the MC Lowering.
+    return true;
+  }
+  }
+  llvm_unreachable("Previous switch has a fallthrough?");
+}
+
+/// Expand all pseudo instructions contained in \p MBB.
+/// \returns true if any expansion occurred for \p MBB.
+bool X86ExpandPseudo::ExpandMBB(MachineBasicBlock &MBB) {
+  bool Modified = false;
+
+  // MBBI may be invalidated by the expansion.
+  MachineBasicBlock::iterator MBBI = MBB.begin(), E = MBB.end();
+  while (MBBI != E) {
+    MachineBasicBlock::iterator NMBBI = std::next(MBBI);
+    Modified |= ExpandMI(MBB, MBBI);
+    MBBI = NMBBI;
+  }
+
+  return Modified;
+}
+
+bool X86ExpandPseudo::runOnMachineFunction(MachineFunction &MF) {
+  STI = &static_cast<const X86Subtarget &>(MF.getSubtarget());
+  TII = STI->getInstrInfo();
+  TRI = STI->getRegisterInfo();
+  X86FL = STI->getFrameLowering();
+
+  bool Modified = false;
+  for (MachineBasicBlock &MBB : MF)
+    Modified |= ExpandMBB(MBB);
+  return Modified;
+}
+
+/// Returns an instance of the pseudo instruction expansion pass.
+FunctionPass *llvm::createX86ExpandPseudoPass() {
+  return new X86ExpandPseudo();
+}
diff --git a/lib/Target/X86/X86FastISel.cpp b/lib/Target/X86/X86FastISel.cpp
index 3ffc8a89263..b4319c8bb04 100644
--- a/lib/Target/X86/X86FastISel.cpp
+++ b/lib/Target/X86/X86FastISel.cpp
@@ -38,16 +38,11 @@
 #include "llvm/IR/IntrinsicInst.h"
 #include "llvm/IR/Operator.h"
 #include "llvm/MC/MCAsmInfo.h"
+#include "llvm/MC/MCSymbol.h"
 #include "llvm/Support/ErrorHandling.h"
 #include "llvm/Target/TargetOptions.h"
-#include "llvm/ADT/Statistic.h" // @LOCALMOD
 using namespace llvm;
 
-// @LOCALMOD-BEGIN
-#define DEBUG_TYPE "isel"
-STATISTIC(NumFastIselNaClFailures, "Number of instructions fast isel failed on for NaCl illegality");
-// @LOCALMOD-END
-
 namespace {
 
 class X86FastISel final : public FastISel {
@@ -89,13 +84,13 @@ class X86FastISel final : public FastISel {
 private:
   bool X86FastEmitCompare(const Value *LHS, const Value *RHS, EVT VT, DebugLoc DL);
 
-  bool X86FastEmitLoad(EVT VT, const X86AddressMode &AM, MachineMemOperand *MMO,
+  bool X86FastEmitLoad(EVT VT, X86AddressMode &AM, MachineMemOperand *MMO,
                        unsigned &ResultReg, unsigned Alignment = 1);
 
-  bool X86FastEmitStore(EVT VT, const Value *Val, const X86AddressMode &AM,
+  bool X86FastEmitStore(EVT VT, const Value *Val, X86AddressMode &AM,
                         MachineMemOperand *MMO = nullptr, bool Aligned = false);
   bool X86FastEmitStore(EVT VT, unsigned ValReg, bool ValIsKill,
-                        const X86AddressMode &AM,
+                        X86AddressMode &AM,
                         MachineMemOperand *MMO = nullptr, bool Aligned = false);
 
   bool X86FastEmitExtend(ISD::NodeType Opc, EVT DstVT, unsigned Src, EVT SrcVT,
@@ -171,6 +166,9 @@ class X86FastISel final : public FastISel {
 
   bool foldX86XALUIntrinsic(X86::CondCode &CC, const Instruction *I,
                             const Value *Cond);
+
+  const MachineInstrBuilder &addFullAddress(const MachineInstrBuilder &MIB,
+                                            X86AddressMode &AM);
 };
 
 } // end anonymous namespace.
@@ -248,6 +246,20 @@ getX86SSEConditionCode(CmpInst::Predicate Predicate) {
   return std::make_pair(CC, NeedSwap);
 }
 
+/// \brief Adds a complex addressing mode to the given machine instr builder.
+/// Note, this will constrain the index register.  If its not possible to
+/// constrain the given index register, then a new one will be created.  The
+/// IndexReg field of the addressing mode will be updated to match in this case.
+const MachineInstrBuilder &
+X86FastISel::addFullAddress(const MachineInstrBuilder &MIB,
+                            X86AddressMode &AM) {
+  // First constrain the index register.  It needs to be a GR64_NOSP.
+  AM.IndexReg = constrainOperandRegClass(MIB->getDesc(), AM.IndexReg,
+                                         MIB->getNumOperands() +
+                                         X86::AddrIndexReg);
+  return ::addFullAddress(MIB, AM);
+}
+
 /// \brief Check if it is possible to fold the condition from the XALU intrinsic
 /// into the user. The condition code will only be updated on success.
 bool X86FastISel::foldX86XALUIntrinsic(X86::CondCode &CC, const Instruction *I,
@@ -305,7 +317,7 @@ bool X86FastISel::foldX86XALUIntrinsic(X86::CondCode &CC, const Instruction *I,
 }
 
 bool X86FastISel::isTypeLegal(Type *Ty, MVT &VT, bool AllowI1) {
-  EVT evt = TLI.getValueType(Ty, /*HandleUnknown=*/true);
+  EVT evt = TLI.getValueType(DL, Ty, /*HandleUnknown=*/true);
   if (evt == MVT::Other || !evt.isSimple())
     // Unhandled type. Halt "fast" selection and bail.
     return false;
@@ -332,7 +344,7 @@ bool X86FastISel::isTypeLegal(Type *Ty, MVT &VT, bool AllowI1) {
 /// X86FastEmitLoad - Emit a machine instruction to load a value of type VT.
 /// The address is either pre-computed, i.e. Ptr, or a GlobalAddress, i.e. GV.
 /// Return true and the result register by reference if it is possible.
-bool X86FastISel::X86FastEmitLoad(EVT VT, const X86AddressMode &AM,
+bool X86FastISel::X86FastEmitLoad(EVT VT, X86AddressMode &AM,
                                   MachineMemOperand *MMO, unsigned &ResultReg,
                                   unsigned Alignment) {
   // Get opcode and regclass of the output for the given load instruction.
@@ -419,7 +431,7 @@ bool X86FastISel::X86FastEmitLoad(EVT VT, const X86AddressMode &AM,
 /// and a displacement offset, or a GlobalAddress,
 /// i.e. V. Return true if it is possible.
 bool X86FastISel::X86FastEmitStore(EVT VT, unsigned ValReg, bool ValIsKill,
-                                   const X86AddressMode &AM,
+                                   X86AddressMode &AM,
                                    MachineMemOperand *MMO, bool Aligned) {
   // Get opcode and regclass of the output for the given store instruction.
   unsigned Opc = 0;
@@ -480,7 +492,7 @@ bool X86FastISel::X86FastEmitStore(EVT VT, unsigned ValReg, bool ValIsKill,
 }
 
 bool X86FastISel::X86FastEmitStore(EVT VT, const Value *Val,
-                                   const X86AddressMode &AM,
+                                   X86AddressMode &AM,
                                    MachineMemOperand *MMO, bool Aligned) {
   // Handle 'null' like i32/i64 0.
   if (isa<ConstantPointerNull>(Val))
@@ -537,85 +549,6 @@ bool X86FastISel::X86FastEmitExtend(ISD::NodeType Opc, EVT DstVT,
   return true;
 }
 
-/// @LOCALMOD-BEGIN
-/// isLegalAddressingModeForNaCl - Determine if the addressing mode is
-/// legal for NaCl translation.  If not, the caller is expected to
-/// reject the instruction for fast-ISel code generation.
-///
-/// The logic for the test is translated from the corresponding logic
-/// in X86DAGToDAGISel::LegalizeAddressingModeForNaCl().  It can't be
-/// used directly due to the X86AddressMode vs X86ISelAddressMode
-/// types.  As such, any changes to isLegalAddressingModeForNaCl() and
-/// X86DAGToDAGISel::LegalizeAddressingModeForNaCl() need to be
-/// synchronized.  The original conditions are indicated in comments.
-static bool isLegalAddressingModeForNaCl(const X86Subtarget *Subtarget,
-                                         const X86AddressMode &AM) {
-  if (Subtarget->isTargetNaCl64()) {
-    // Return true (i.e., is legal) if the equivalent of
-    // X86ISelAddressMode::isRIPRelative() is true.
-    if (AM.BaseType == X86AddressMode::RegBase &&
-        AM.Base.Reg == X86::RIP)
-      return true;
-
-    // Check for the equivalent of
-    // (!AM.hasBaseOrIndexReg() &&
-    //  !AM.hasSymbolicDisplacement() &&
-    //  AM.Disp < 0)
-    if (!((AM.BaseType == X86AddressMode::RegBase && AM.Base.Reg) ||
-          AM.IndexReg) &&
-        !AM.GV &&
-        AM.Disp < 0) {
-      ++NumFastIselNaClFailures;
-      return false;
-    }
-
-    // At this point in the LegalizeAddressingModeForNaCl() code, it
-    // normalizes an addressing mode with a base register and no index
-    // register into an equivalent mode with an index register and no
-    // base register.  Since we don't modify AM, we may have to check
-    // both the base and index register fields in the remainder of the
-    // tests.
-
-    // Check for the equivalent of
-    // ((AM.BaseType == X86ISelAddressMode::FrameIndexBase || AM.GV || AM.CP) &&
-    //   AM.IndexReg.getNode() &&
-    //   AM.Disp > 0)
-    // Note: X86AddressMode doesn't have a CP analogue
-    if ((AM.BaseType == X86AddressMode::FrameIndexBase || AM.GV) &&
-        ((AM.BaseType == X86AddressMode::RegBase && AM.Base.Reg) ||
-         AM.IndexReg) &&
-        AM.Disp > 0) {
-      ++NumFastIselNaClFailures;
-      return false;
-    }
-
-    // Check for the equivalent of
-    // ((AM.BaseType == X86ISelAddressMode::RegBase) &&
-    //  AM.Base_Reg.getNode() &&
-    //  AM.IndexReg.getNode())
-    if ((AM.BaseType == X86AddressMode::RegBase) &&
-        AM.Base.Reg &&
-        AM.IndexReg) {
-      ++NumFastIselNaClFailures;
-      return false;
-    }
-
-    // See X86DAGToDAGISel::FoldOffsetIntoAddress().
-    // Check for the equivalent of
-    // ((AM.BaseType == X86ISelAddressMode::RegBase ||
-    //   AM.BaseType == X86ISelAddressMode::FrameIndexBase) &&
-    //  (Val > 65535 || Val < -65536))
-    if ((AM.BaseType == X86AddressMode::RegBase ||
-         AM.BaseType == X86AddressMode::FrameIndexBase) &&
-        (AM.Disp > 65535 || AM.Disp < -65536)) {
-      ++NumFastIselNaClFailures;
-      return false;
-    }
-  }
-
-  return true;
-}
-
 bool X86FastISel::handleConstantAddresses(const Value *V, X86AddressMode &AM) {
   // Handle constant address.
   if (const GlobalValue *GV = dyn_cast<GlobalValue>(V)) {
@@ -675,20 +608,12 @@ bool X86FastISel::handleConstantAddresses(const Value *V, X86AddressMode &AM) {
         // Prepare for inserting code in the local-value area.
         SavePoint SaveInsertPt = enterLocalValueArea();
 
-        if (TLI.getPointerTy() == MVT::i64) {
+        if (TLI.getPointerTy(DL) == MVT::i64) {
           Opc = X86::MOV64rm;
           RC  = &X86::GR64RegClass;
 
           if (Subtarget->isPICStyleRIPRel())
             StubAM.Base.Reg = X86::RIP;
-        // @LOCALMOD-BEGIN
-        } else if (Subtarget->isTargetNaCl64()) {
-          Opc = X86::MOV32rm;
-          RC  = &X86::GR32RegClass;
-
-          if (Subtarget->isPICStyleRIPRel())
-            StubAM.Base.Reg = X86::RIP;
-        // @LOCALMOD-END
         } else {
           Opc = X86::MOV32rm;
           RC  = &X86::GR32RegClass;
@@ -716,21 +641,6 @@ bool X86FastISel::handleConstantAddresses(const Value *V, X86AddressMode &AM) {
 
   // If all else fails, try to materialize the value in a register.
   if (!AM.GV || !Subtarget->isPICStyleRIPRel()) {
-    // @LOCALMOD-START
-    if (Subtarget->isTargetNaCl64()) {
-      // We are about use a register in an addressing mode. However, x86-64
-      // NaCl does not allow arbitrary r+r addressing. One of the regs must
-      // be %r15 (inserted by the NaClRewritePass). Check that we will only
-      // end up with one reg defined after this.
-      if ((AM.Base.Reg == 0) && (AM.IndexReg == 0)) {
-        // Put into index register so that the NaCl rewrite pass will
-        // convert this to a 64-bit address.
-        AM.IndexReg = getRegForValue(V);
-        return AM.IndexReg != 0;
-      }
-      return false;
-    }
-    // @LOCALMOD-END
     if (AM.Base.Reg == 0) {
       AM.Base.Reg = getRegForValue(V);
       return AM.Base.Reg != 0;
@@ -747,16 +657,6 @@ bool X86FastISel::handleConstantAddresses(const Value *V, X86AddressMode &AM) {
 
 /// X86SelectAddress - Attempt to fill in an address from the given value.
 ///
-/// @LOCALMOD-BEGIN
-/// All "return v;" statements must be converted to
-/// "return (v) && isLegalAddressingModeForNaCl(Subtarget, AM);"
-/// except that "return false;" can of course be left unchanged.
-///
-/// Since X86SelectAddress() recursively builds up the AM result
-/// object, there is a risk that an intermediate result could be
-/// rejected in a situation where the final result was in fact legal,
-/// though it is hard to imagine this happening.
-/// @LOCALMOD-END
 bool X86FastISel::X86SelectAddress(const Value *V, X86AddressMode &AM) {
   SmallVector<const Value *, 32> GEPs;
 redo_gep:
@@ -790,13 +690,14 @@ bool X86FastISel::X86SelectAddress(const Value *V, X86AddressMode &AM) {
 
   case Instruction::IntToPtr:
     // Look past no-op inttoptrs.
-    if (TLI.getValueType(U->getOperand(0)->getType()) == TLI.getPointerTy())
+    if (TLI.getValueType(DL, U->getOperand(0)->getType()) ==
+        TLI.getPointerTy(DL))
       return X86SelectAddress(U->getOperand(0), AM);
     break;
 
   case Instruction::PtrToInt:
     // Look past no-op ptrtoints.
-    if (TLI.getValueType(U->getType()) == TLI.getPointerTy())
+    if (TLI.getValueType(DL, U->getType()) == TLI.getPointerTy(DL))
       return X86SelectAddress(U->getOperand(0), AM);
     break;
 
@@ -808,7 +709,7 @@ bool X86FastISel::X86SelectAddress(const Value *V, X86AddressMode &AM) {
     if (SI != FuncInfo.StaticAllocaMap.end()) {
       AM.BaseType = X86AddressMode::FrameIndexBase;
       AM.Base.FrameIndex = SI->second;
-      return isLegalAddressingModeForNaCl(Subtarget, AM); // @LOCALMOD
+      return true;
     }
     break;
   }
@@ -894,7 +795,7 @@ bool X86FastISel::X86SelectAddress(const Value *V, X86AddressMode &AM) {
       V = GEP;
       goto redo_gep;
     } else if (X86SelectAddress(U->getOperand(0), AM)) {
-      return isLegalAddressingModeForNaCl(Subtarget, AM); // @LOCALMOD
+      return true;
     }
 
     // If we couldn't merge the gep value into this addr mode, revert back to
@@ -904,7 +805,7 @@ bool X86FastISel::X86SelectAddress(const Value *V, X86AddressMode &AM) {
     for (SmallVectorImpl<const Value *>::reverse_iterator
            I = GEPs.rbegin(), E = GEPs.rend(); I != E; ++I)
       if (handleConstantAddresses(*I, AM))
-        return isLegalAddressingModeForNaCl(Subtarget, AM); // @LOCALMOD
+        return true;
 
     return false;
   unsupported_gep:
@@ -913,11 +814,7 @@ bool X86FastISel::X86SelectAddress(const Value *V, X86AddressMode &AM) {
   }
   }
 
-  // @LOCALMOD-START
-  if (handleConstantAddresses(V, AM))
-    return isLegalAddressingModeForNaCl(Subtarget, AM);
-  return false;
-  // @LOCALMOD-END
+  return handleConstantAddresses(V, AM);
 }
 
 /// X86SelectCallAddress - Attempt to fill in an address from the given value.
@@ -970,14 +867,14 @@ bool X86FastISel::X86SelectCallAddress(const Value *V, X86AddressMode &AM) {
   case Instruction::IntToPtr:
     // Look past no-op inttoptrs if its operand is in the same BB.
     if (InMBB &&
-        TLI.getValueType(U->getOperand(0)->getType()) == TLI.getPointerTy())
+        TLI.getValueType(DL, U->getOperand(0)->getType()) ==
+            TLI.getPointerTy(DL))
       return X86SelectCallAddress(U->getOperand(0), AM);
     break;
 
   case Instruction::PtrToInt:
     // Look past no-op ptrtoints if its operand is in the same BB.
-    if (InMBB &&
-        TLI.getValueType(U->getType()) == TLI.getPointerTy())
+    if (InMBB && TLI.getValueType(DL, U->getType()) == TLI.getPointerTy(DL))
       return X86SelectCallAddress(U->getOperand(0), AM);
     break;
   }
@@ -1104,7 +1001,7 @@ bool X86FastISel::X86SelectRet(const Instruction *I) {
 
   if (Ret->getNumOperands() > 0) {
     SmallVector<ISD::OutputArg, 4> Outs;
-    GetReturnInfo(F.getReturnType(), F.getAttributes(), Outs, TLI);
+    GetReturnInfo(F.getReturnType(), F.getAttributes(), Outs, TLI, DL);
 
     // Analyze operands of the call, assigning locations to each operand.
     SmallVector<CCValAssign, 16> ValLocs;
@@ -1135,7 +1032,7 @@ bool X86FastISel::X86SelectRet(const Instruction *I) {
       return false;
 
     unsigned SrcReg = Reg + VA.getValNo();
-    EVT SrcVT = TLI.getValueType(RV->getType());
+    EVT SrcVT = TLI.getValueType(DL, RV->getType());
     EVT DstVT = VA.getValVT();
     // Special handling for extended integers.
     if (SrcVT != DstVT) {
@@ -1181,15 +1078,10 @@ bool X86FastISel::X86SelectRet(const Instruction *I) {
     unsigned Reg = X86MFInfo->getSRetReturnReg();
     assert(Reg &&
            "SRetReturnReg should have been set in LowerFormalArguments()!");
-    // @LOCALMOD-BEGIN -- Ensure that the register classes match.
-    // At this point, SRetReturnReg is EDI, because PointerTy() for NaCl
-    // is i32.  We then copy to EAX instead of RAX.  Alternatively, we could
-    // have zero-extended EDI to RDI then copy to RAX, but this has a smaller
-    // encoding (2 bytes vs 3 bytes).
-    unsigned CopyTo = Subtarget->isTarget64BitLP64() ? X86::RAX : X86::EAX;
+    unsigned RetReg = Subtarget->is64Bit() ? X86::RAX : X86::EAX;
     BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc,
-            TII.get(TargetOpcode::COPY), CopyTo).addReg(Reg);
-    // @LOCALMOD-END
+            TII.get(TargetOpcode::COPY), RetReg).addReg(Reg);
+    RetRegs.push_back(RetReg);
   }
 
   // Now emit the RET.
@@ -1409,7 +1301,7 @@ bool X86FastISel::X86SelectCmp(const Instruction *I) {
 }
 
 bool X86FastISel::X86SelectZExt(const Instruction *I) {
-  EVT DstVT = TLI.getValueType(I->getType());
+  EVT DstVT = TLI.getValueType(DL, I->getType());
   if (!TLI.isTypeLegal(DstVT))
     return false;
 
@@ -1418,7 +1310,7 @@ bool X86FastISel::X86SelectZExt(const Instruction *I) {
     return false;
 
   // Handle zero-extension from i1 to i8, which is common.
-  MVT SrcVT = TLI.getSimpleValueType(I->getOperand(0)->getType());
+  MVT SrcVT = TLI.getSimpleValueType(DL, I->getOperand(0)->getType());
   if (SrcVT.SimpleTy == MVT::i1) {
     // Set the high bits to zero.
     ResultReg = fastEmitZExtFromI1(MVT::i8, ResultReg, /*TODO: Kill=*/false);
@@ -1471,7 +1363,7 @@ bool X86FastISel::X86SelectBranch(const Instruction *I) {
   X86::CondCode CC;
   if (const CmpInst *CI = dyn_cast<CmpInst>(BI->getCondition())) {
     if (CI->hasOneUse() && CI->getParent() == I->getParent()) {
-      EVT VT = TLI.getValueType(CI->getOperand(0)->getType());
+      EVT VT = TLI.getValueType(DL, CI->getOperand(0)->getType());
 
       // Try to optimize or fold the cmp.
       CmpInst::Predicate Predicate = optimizeCmpPredicate(CI);
@@ -1911,7 +1803,7 @@ bool X86FastISel::X86FastEmitCMoveSelect(MVT RetVT, const Instruction *I) {
     if (NeedSwap)
       std::swap(CmpLHS, CmpRHS);
 
-    EVT CmpVT = TLI.getValueType(CmpLHS->getType());
+    EVT CmpVT = TLI.getValueType(DL, CmpLHS->getType());
     // Emit a compare of the LHS and RHS, setting the flags.
     if (!X86FastEmitCompare(CmpLHS, CmpRHS, CmpVT, CI->getDebugLoc()))
       return false;
@@ -2113,7 +2005,7 @@ bool X86FastISel::X86FastEmitPseudoSelect(MVT RetVT, const Instruction *I) {
     if (NeedSwap)
       std::swap(CmpLHS, CmpRHS);
 
-    EVT CmpVT = TLI.getValueType(CmpLHS->getType());
+    EVT CmpVT = TLI.getValueType(DL, CmpLHS->getType());
     if (!X86FastEmitCompare(CmpLHS, CmpRHS, CmpVT, CI->getDebugLoc()))
       return false;
   } else {
@@ -2192,6 +2084,12 @@ bool X86FastISel::X86SelectSelect(const Instruction *I) {
 }
 
 bool X86FastISel::X86SelectSIToFP(const Instruction *I) {
+  // The target-independent selection algorithm in FastISel already knows how
+  // to select a SINT_TO_FP if the target is SSE but not AVX.
+  // Early exit if the subtarget doesn't have AVX.
+  if (!Subtarget->hasAVX())
+    return false;
+
   if (!I->getOperand(0)->getType()->isIntegerTy(32))
     return false;
 
@@ -2214,11 +2112,6 @@ bool X86FastISel::X86SelectSIToFP(const Instruction *I) {
   } else
     return false;
 
-  // The target-independent selection algorithm in FastISel already knows how
-  // to select a SINT_TO_FP if the target is SSE but not AVX. This code is only
-  // reachable if the subtarget has AVX.
-  assert(Subtarget->hasAVX() && "Expected a subtarget with AVX!");
-
   unsigned ImplicitDefReg = createResultReg(RC);
   BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc,
           TII.get(TargetOpcode::IMPLICIT_DEF), ImplicitDefReg);
@@ -2274,8 +2167,8 @@ bool X86FastISel::X86SelectFPTrunc(const Instruction *I) {
 }
 
 bool X86FastISel::X86SelectTrunc(const Instruction *I) {
-  EVT SrcVT = TLI.getValueType(I->getOperand(0)->getType());
-  EVT DstVT = TLI.getValueType(I->getType());
+  EVT SrcVT = TLI.getValueType(DL, I->getOperand(0)->getType());
+  EVT DstVT = TLI.getValueType(DL, I->getType());
 
   // This code only handles truncation to byte.
   if (DstVT != MVT::i8 && DstVT != MVT::i1)
@@ -2294,6 +2187,7 @@ bool X86FastISel::X86SelectTrunc(const Instruction *I) {
     return true;
   }
 
+  bool KillInputReg = false;
   if (!Subtarget->is64Bit()) {
     // If we're on x86-32; we can't extract an i8 from a general register.
     // First issue a copy to GR16_ABCD or GR32_ABCD.
@@ -2303,11 +2197,12 @@ bool X86FastISel::X86SelectTrunc(const Instruction *I) {
     BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc,
             TII.get(TargetOpcode::COPY), CopyReg).addReg(InputReg);
     InputReg = CopyReg;
+    KillInputReg = true;
   }
 
   // Issue an extract_subreg.
   unsigned ResultReg = fastEmitInst_extractsubreg(MVT::i8,
-                                                  InputReg, /*Kill=*/true,
+                                                  InputReg, KillInputReg,
                                                   X86::sub_8bit);
   if (!ResultReg)
     return false;
@@ -2361,7 +2256,7 @@ bool X86FastISel::fastLowerIntrinsicCall(const IntrinsicInst *II) {
   default: return false;
   case Intrinsic::convert_from_fp16:
   case Intrinsic::convert_to_fp16: {
-    if (TM.Options.UseSoftFloat || !Subtarget->hasF16C())
+    if (Subtarget->useSoftFloat() || !Subtarget->hasF16C())
       return false;
 
     const Value *Op = II->getArgOperand(0);
@@ -2522,7 +2417,7 @@ bool X86FastISel::fastLowerIntrinsicCall(const IntrinsicInst *II) {
   }
   case Intrinsic::stackprotector: {
     // Emit code to store the stack guard onto the stack.
-    EVT PtrTy = TLI.getPointerTy();
+    EVT PtrTy = TLI.getPointerTy(DL);
 
     const Value *Op1 = II->getArgOperand(0); // The guard's value.
     const AllocaInst *Slot = cast<AllocaInst>(II->getArgOperand(1));
@@ -2841,7 +2736,7 @@ bool X86FastISel::fastLowerArguments() {
     if (ArgTy->isStructTy() || ArgTy->isArrayTy() || ArgTy->isVectorTy())
       return false;
 
-    EVT ArgVT = TLI.getValueType(ArgTy);
+    EVT ArgVT = TLI.getValueType(DL, ArgTy);
     if (!ArgVT.isSimple()) return false;
     switch (ArgVT.getSimpleVT().SimpleTy) {
     default: return false;
@@ -2878,7 +2773,7 @@ bool X86FastISel::fastLowerArguments() {
   unsigned GPRIdx = 0;
   unsigned FPRIdx = 0;
   for (auto const &Arg : F->args()) {
-    MVT VT = TLI.getSimpleValueType(Arg.getType());
+    MVT VT = TLI.getSimpleValueType(DL, Arg.getType());
     const TargetRegisterClass *RC = TLI.getRegClassFor(VT);
     unsigned SrcReg;
     switch (VT.SimpleTy) {
@@ -2928,7 +2823,7 @@ bool X86FastISel::fastLowerCall(CallLoweringInfo &CLI) {
   bool &IsTailCall    = CLI.IsTailCall;
   bool IsVarArg       = CLI.IsVarArg;
   const Value *Callee = CLI.Callee;
-  const char *SymName = CLI.SymName;
+  MCSymbol *Symbol = CLI.Symbol;
 
   bool Is64Bit        = Subtarget->is64Bit();
   bool IsWin64        = Subtarget->isCallingConvWin64(CC);
@@ -3194,32 +3089,13 @@ bool X86FastISel::fastLowerCall(CallLoweringInfo &CLI) {
   MachineInstrBuilder MIB;
   if (CalleeOp) {
     // Register-indirect call.
-    unsigned CallOpc;
-    if (Subtarget->is64Bit()) {
-      // @LOCALMOD-BEGIN: zero extend the register like in the non-O0 case:
-      // http://reviews.llvm.org/D5355
-      // TODO: upstream this for x32 also (add a test).
-      if (Subtarget->isTarget64BitILP32()) {
-        unsigned CalleeOp32 = CalleeOp;
-        CalleeOp = createResultReg(&X86::GR64RegClass);
-        BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc,
-                TII.get(TargetOpcode::SUBREG_TO_REG), CalleeOp)
-            .addImm(0).addReg(CalleeOp32).addImm(X86::sub_32bit);
-      }
-      // @LOCALMOD-END
-      CallOpc = X86::CALL64r;
-    } else
-      CallOpc = X86::CALL32r;
+    unsigned CallOpc = Is64Bit ? X86::CALL64r : X86::CALL32r;
     MIB = BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, TII.get(CallOpc))
       .addReg(CalleeOp);
   } else {
     // Direct call.
     assert(GV && "Not a direct call");
-    unsigned CallOpc;
-    if (Subtarget->is64Bit())
-      CallOpc = Subtarget->isTargetNaCl() ? X86::NACL_CG_CALL64pcrel32 : X86::CALL64pcrel32; // @LOCALMOD
-    else
-      CallOpc = X86::CALLpcrel32;
+    unsigned CallOpc = Is64Bit ? X86::CALL64pcrel32 : X86::CALLpcrel32;
 
     // See if we need any target-specific flags on the GV operand.
     unsigned char OpFlags = 0;
@@ -3233,7 +3109,7 @@ bool X86FastISel::fastLowerCall(CallLoweringInfo &CLI) {
         GV->hasDefaultVisibility() && !GV->hasLocalLinkage()) {
       OpFlags = X86II::MO_PLT;
     } else if (Subtarget->isPICStyleStubAny() &&
-               (GV->isDeclaration() || GV->isWeakForLinker()) &&
+               !GV->isStrongDefinitionForLinker() &&
                (!Subtarget->getTargetTriple().isMacOSX() ||
                 Subtarget->getTargetTriple().isMacOSXVersionLT(10, 5))) {
       // PC-relative references to external symbols should go through $stub,
@@ -3243,8 +3119,8 @@ bool X86FastISel::fastLowerCall(CallLoweringInfo &CLI) {
     }
 
     MIB = BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, TII.get(CallOpc));
-    if (SymName)
-      MIB.addExternalSymbol(SymName, OpFlags);
+    if (Symbol)
+      MIB.addSym(Symbol, OpFlags);
     else
       MIB.addGlobalAddress(GV, 0, OpFlags);
   }
@@ -3365,8 +3241,8 @@ X86FastISel::fastSelectInstruction(const Instruction *I)  {
     return X86SelectSIToFP(I);
   case Instruction::IntToPtr: // Deliberate fall-through.
   case Instruction::PtrToInt: {
-    EVT SrcVT = TLI.getValueType(I->getOperand(0)->getType());
-    EVT DstVT = TLI.getValueType(I->getType());
+    EVT SrcVT = TLI.getValueType(DL, I->getOperand(0)->getType());
+    EVT DstVT = TLI.getValueType(DL, I->getType());
     if (DstVT.bitsGT(SrcVT))
       return X86SelectZExt(I);
     if (DstVT.bitsLT(SrcVT))
@@ -3509,7 +3385,7 @@ unsigned X86FastISel::X86MaterializeFP(const ConstantFP *CFP, MVT VT) {
     addDirectMem(MIB, AddrReg);
     MachineMemOperand *MMO = FuncInfo.MF->getMachineMemOperand(
         MachinePointerInfo::getConstantPool(), MachineMemOperand::MOLoad,
-        TM.getDataLayout()->getPointerSize(), Align);
+        DL.getPointerSize(), Align);
     MIB->addMemOperand(*FuncInfo.MF, MMO);
     return ResultReg;
   }
@@ -3536,17 +3412,17 @@ unsigned X86FastISel::X86MaterializeGV(const GlobalValue *GV, MVT VT) {
 
     unsigned ResultReg = createResultReg(TLI.getRegClassFor(VT));
     if (TM.getRelocationModel() == Reloc::Static &&
-        TLI.getPointerTy() == MVT::i64) {
+        TLI.getPointerTy(DL) == MVT::i64) {
       // The displacement code could be more than 32 bits away so we need to use
       // an instruction with a 64 bit immediate
       BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, TII.get(X86::MOV64ri),
               ResultReg)
         .addGlobalAddress(GV);
     } else {
-      unsigned Opc = TLI.getPointerTy() == MVT::i32
-                     ? (Subtarget->isTarget64BitILP32()
-                        ? X86::LEA64_32r : X86::LEA32r)
-                     : X86::LEA64r;
+      unsigned Opc =
+          TLI.getPointerTy(DL) == MVT::i32
+              ? (Subtarget->isTarget64BitILP32() ? X86::LEA64_32r : X86::LEA32r)
+              : X86::LEA64r;
       addFullAddress(BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc,
                              TII.get(Opc), ResultReg), AM);
     }
@@ -3556,7 +3432,7 @@ unsigned X86FastISel::X86MaterializeGV(const GlobalValue *GV, MVT VT) {
 }
 
 unsigned X86FastISel::fastMaterializeConstant(const Constant *C) {
-  EVT CEVT = TLI.getValueType(C->getType(), true);
+  EVT CEVT = TLI.getValueType(DL, C->getType(), true);
 
   // Only handle simple types.
   if (!CEVT.isSimple())
@@ -3588,11 +3464,11 @@ unsigned X86FastISel::fastMaterializeAlloca(const AllocaInst *C) {
   X86AddressMode AM;
   if (!X86SelectAddress(C, AM))
     return 0;
-  unsigned Opc = TLI.getPointerTy() == MVT::i32
-                 ? (Subtarget->isTarget64BitILP32()
-                    ? X86::LEA64_32r : X86::LEA32r)
-                 : X86::LEA64r;
-  const TargetRegisterClass* RC = TLI.getRegClassFor(TLI.getPointerTy());
+  unsigned Opc =
+      TLI.getPointerTy(DL) == MVT::i32
+          ? (Subtarget->isTarget64BitILP32() ? X86::LEA64_32r : X86::LEA32r)
+          : X86::LEA64r;
+  const TargetRegisterClass *RC = TLI.getRegClassFor(TLI.getPointerTy(DL));
   unsigned ResultReg = createResultReg(RC);
   addFullAddress(BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc,
                          TII.get(Opc), ResultReg), AM);
@@ -3656,14 +3532,32 @@ bool X86FastISel::tryToFoldLoadIntoMI(MachineInstr *MI, unsigned OpNo,
   SmallVector<MachineOperand, 8> AddrOps;
   AM.getFullAddress(AddrOps);
 
-  MachineInstr *Result =
-    XII.foldMemoryOperandImpl(*FuncInfo.MF, MI, OpNo, AddrOps,
-                              Size, Alignment, /*AllowCommute=*/true);
+  MachineInstr *Result = XII.foldMemoryOperandImpl(
+      *FuncInfo.MF, MI, OpNo, AddrOps, FuncInfo.InsertPt, Size, Alignment,
+      /*AllowCommute=*/true);
   if (!Result)
     return false;
 
+  // The index register could be in the wrong register class.  Unfortunately,
+  // foldMemoryOperandImpl could have commuted the instruction so its not enough
+  // to just look at OpNo + the offset to the index reg.  We actually need to
+  // scan the instruction to find the index reg and see if its the correct reg
+  // class.
+  unsigned OperandNo = 0;
+  for (MachineInstr::mop_iterator I = Result->operands_begin(),
+       E = Result->operands_end(); I != E; ++I, ++OperandNo) {
+    MachineOperand &MO = *I;
+    if (!MO.isReg() || MO.isDef() || MO.getReg() != AM.IndexReg)
+      continue;
+    // Found the index reg, now try to rewrite it.
+    unsigned IndexReg = constrainOperandRegClass(Result->getDesc(),
+                                                 MO.getReg(), OperandNo);
+    if (IndexReg == MO.getReg())
+      continue;
+    MO.setReg(IndexReg);
+  }
+
   Result->addMemOperand(*FuncInfo.MF, createMachineMemOperandFor(LI));
-  FuncInfo.MBB->insert(FuncInfo.InsertPt, Result);
   MI->eraseFromParent();
   return true;
 }
diff --git a/lib/Target/X86/X86FixupLEAs.cpp b/lib/Target/X86/X86FixupLEAs.cpp
index b39c5aba30b..5eb4faeedff 100644
--- a/lib/Target/X86/X86FixupLEAs.cpp
+++ b/lib/Target/X86/X86FixupLEAs.cpp
@@ -44,7 +44,7 @@ class FixupLEAPass : public MachineFunctionPass {
   /// \brief Given a machine register, look for the instruction
   /// which writes it in the current basic block. If found,
   /// try to replace it with an equivalent LEA instruction.
-  /// If replacement succeeds, then also process the the newly created
+  /// If replacement succeeds, then also process the newly created
   /// instruction.
   void seekLEAFixup(MachineOperand &p, MachineBasicBlock::iterator &I,
                     MachineFunction::iterator MFI);
diff --git a/lib/Target/X86/X86FloatingPoint.cpp b/lib/Target/X86/X86FloatingPoint.cpp
index 3b0bd03095a..36a8cdbab55 100644
--- a/lib/Target/X86/X86FloatingPoint.cpp
+++ b/lib/Target/X86/X86FloatingPoint.cpp
@@ -301,8 +301,9 @@ bool FPS::runOnMachineFunction(MachineFunction &MF) {
   bool FPIsUsed = false;
 
   static_assert(X86::FP6 == X86::FP0+6, "Register enums aren't sorted right!");
+  const MachineRegisterInfo &MRI = MF.getRegInfo();
   for (unsigned i = 0; i <= 6; ++i)
-    if (MF.getRegInfo().isPhysRegUsed(X86::FP0+i)) {
+    if (!MRI.reg_nodbg_empty(X86::FP0 + i)) {
       FPIsUsed = true;
       break;
     }
@@ -1530,7 +1531,7 @@ void FPS::handleSpecialFP(MachineBasicBlock::iterator &Inst) {
     if (Op.isKill())
       moveToTop(FPReg, Inst);
     else
-      duplicateToTop(FPReg, FPReg, Inst);
+      duplicateToTop(FPReg, ScratchFPReg, Inst);
 
     // Emit the call. This will pop the operand.
     BuildMI(*MBB, Inst, MI->getDebugLoc(), TII->get(X86::CALLpcrel32))
diff --git a/lib/Target/X86/X86FrameLowering.cpp b/lib/Target/X86/X86FrameLowering.cpp
index afd46191c2d..b8858362e74 100644
--- a/lib/Target/X86/X86FrameLowering.cpp
+++ b/lib/Target/X86/X86FrameLowering.cpp
@@ -15,7 +15,6 @@
 #include "X86InstrBuilder.h"
 #include "X86InstrInfo.h"
 #include "X86MachineFunctionInfo.h"
-#include "X86NaClDecls.h" // @LOCALMOD
 #include "X86Subtarget.h"
 #include "X86TargetMachine.h"
 #include "llvm/ADT/SmallSet.h"
@@ -28,15 +27,25 @@
 #include "llvm/IR/Function.h"
 #include "llvm/MC/MCAsmInfo.h"
 #include "llvm/MC/MCSymbol.h"
-#include "llvm/Support/CommandLine.h"
 #include "llvm/Target/TargetOptions.h"
 #include "llvm/Support/Debug.h"
 #include <cstdlib>
 
 using namespace llvm;
 
-// FIXME: completely move here.
-extern cl::opt<bool> ForceStackAlign;
+X86FrameLowering::X86FrameLowering(const X86Subtarget &STI,
+                                   unsigned StackAlignOverride)
+    : TargetFrameLowering(StackGrowsDown, StackAlignOverride,
+                          STI.is64Bit() ? -8 : -4),
+      STI(STI), TII(*STI.getInstrInfo()), TRI(STI.getRegisterInfo()) {
+  // Cache a bunch of frame-related predicates for this subtarget.
+  SlotSize = TRI->getSlotSize();
+  Is64Bit = STI.is64Bit();
+  IsLP64 = STI.isTarget64BitLP64();
+  // standard x86_64 and NaCl use 64-bit frame/stack pointers, x32 - 32-bit.
+  Uses64BitFramePtr = STI.isTarget64BitLP64() || STI.isTargetNaCl64();
+  StackPtr = TRI->getStackRegister();
+}
 
 bool X86FrameLowering::hasReservedCallFrame(const MachineFunction &MF) const {
   return !MF.getFrameInfo()->hasVarSizedObjects() &&
@@ -49,11 +58,9 @@ bool X86FrameLowering::hasReservedCallFrame(const MachineFunction &MF) const {
 /// Use a more nuanced condition.
 bool
 X86FrameLowering::canSimplifyCallFramePseudos(const MachineFunction &MF) const {
-  const X86RegisterInfo *TRI = static_cast<const X86RegisterInfo *>
-                               (MF.getSubtarget().getRegisterInfo());
   return hasReservedCallFrame(MF) ||
-         (hasFP(MF) && !TRI->needsStackRealignment(MF))
-         || TRI->hasBasePointer(MF);
+         (hasFP(MF) && !TRI->needsStackRealignment(MF)) ||
+         TRI->hasBasePointer(MF);
 }
 
 // needsFrameIndexResolution - Do we need to perform FI resolution for
@@ -75,12 +82,11 @@ X86FrameLowering::needsFrameIndexResolution(const MachineFunction &MF) const {
 bool X86FrameLowering::hasFP(const MachineFunction &MF) const {
   const MachineFrameInfo *MFI = MF.getFrameInfo();
   const MachineModuleInfo &MMI = MF.getMMI();
-  const TargetRegisterInfo *RegInfo = MF.getSubtarget().getRegisterInfo();
 
   return (MF.getTarget().Options.DisableFramePointerElim(MF) ||
-          RegInfo->needsStackRealignment(MF) ||
+          TRI->needsStackRealignment(MF) ||
           MFI->hasVarSizedObjects() ||
-          MFI->isFrameAddressTaken() || MFI->hasInlineAsmWithSPAdjust() ||
+          MFI->isFrameAddressTaken() || MFI->hasOpaqueSPAdjustment() ||
           MF.getInfo<X86MachineFunctionInfo>()->getForceFramePointer() ||
           MMI.callsUnwindInit() || MMI.callsEHReturn() ||
           MFI->hasStackMap() || MFI->hasPatchPoint());
@@ -138,7 +144,7 @@ static unsigned getLEArOpcode(unsigned IsLP64) {
 /// to this register without worry about clobbering it.
 static unsigned findDeadCallerSavedReg(MachineBasicBlock &MBB,
                                        MachineBasicBlock::iterator &MBBI,
-                                       const TargetRegisterInfo &TRI,
+                                       const TargetRegisterInfo *TRI,
                                        bool Is64Bit) {
   const MachineFunction *MF = MBB.getParent();
   const Function *F = MF->getFunction();
@@ -166,7 +172,6 @@ static unsigned findDeadCallerSavedReg(MachineBasicBlock &MBB,
   case X86::TCRETURNmi:
   case X86::TCRETURNdi64:
   case X86::TCRETURNri64:
-  case X86::NACL_CG_TCRETURNdi64: // @LOCALMOD
   case X86::TCRETURNmi64:
   case X86::EH_RETURN:
   case X86::EH_RETURN64: {
@@ -178,7 +183,7 @@ static unsigned findDeadCallerSavedReg(MachineBasicBlock &MBB,
       unsigned Reg = MO.getReg();
       if (!Reg)
         continue;
-      for (MCRegAliasIterator AI(Reg, &TRI, true); AI.isValid(); ++AI)
+      for (MCRegAliasIterator AI(Reg, TRI, true); AI.isValid(); ++AI)
         Uses.insert(*AI);
     }
 
@@ -205,22 +210,36 @@ static bool isEAXLiveIn(MachineFunction &MF) {
   return false;
 }
 
+/// Check whether or not the terminators of \p MBB needs to read EFLAGS.
+static bool terminatorsNeedFlagsAsInput(const MachineBasicBlock &MBB) {
+  for (const MachineInstr &MI : MBB.terminators()) {
+    bool BreakNext = false;
+    for (const MachineOperand &MO : MI.operands()) {
+      if (!MO.isReg())
+        continue;
+      unsigned Reg = MO.getReg();
+      if (Reg != X86::EFLAGS)
+        continue;
+
+      // This terminator needs an eflag that is not defined
+      // by a previous terminator.
+      if (!MO.isDef())
+        return true;
+      BreakNext = true;
+    }
+    if (BreakNext)
+      break;
+  }
+  return false;
+}
+
 /// emitSPUpdate - Emit a series of instructions to increment / decrement the
 /// stack pointer by a constant value.
-static
-void emitSPUpdate(MachineBasicBlock &MBB, MachineBasicBlock::iterator &MBBI,
-                  unsigned StackPtr, int64_t NumBytes,
-                  bool Is64BitTarget, bool Is64BitStackPtr, bool UseLEA,
-                  const TargetInstrInfo &TII, const TargetRegisterInfo &TRI) {
+void X86FrameLowering::emitSPUpdate(MachineBasicBlock &MBB,
+                                    MachineBasicBlock::iterator &MBBI,
+                                    int64_t NumBytes, bool InEpilogue) const {
   bool isSub = NumBytes < 0;
   uint64_t Offset = isSub ? -NumBytes : NumBytes;
-  unsigned Opc;
-  if (UseLEA)
-    Opc = getLEArOpcode(Is64BitStackPtr);
-  else
-    Opc = isSub
-      ? getSUBriOpcode(Is64BitStackPtr, Offset)
-      : getADDriOpcode(Is64BitStackPtr, Offset);
 
   uint64_t Chunk = (1LL << 31) - 1;
   DebugLoc DL = MBB.findDebugLoc(MBBI);
@@ -232,17 +251,17 @@ void emitSPUpdate(MachineBasicBlock &MBB, MachineBasicBlock::iterator &MBBI,
       unsigned Reg = 0;
 
       if (isSub && !isEAXLiveIn(*MBB.getParent()))
-        Reg = (unsigned)(Is64BitTarget ? X86::RAX : X86::EAX);
+        Reg = (unsigned)(Is64Bit ? X86::RAX : X86::EAX);
       else
-        Reg = findDeadCallerSavedReg(MBB, MBBI, TRI, Is64BitTarget);
+        Reg = findDeadCallerSavedReg(MBB, MBBI, TRI, Is64Bit);
 
       if (Reg) {
-        Opc = Is64BitTarget ? X86::MOV64ri : X86::MOV32ri;
+        unsigned Opc = Is64Bit ? X86::MOV64ri : X86::MOV32ri;
         BuildMI(MBB, MBBI, DL, TII.get(Opc), Reg)
           .addImm(Offset);
         Opc = isSub
-          ? getSUBrrOpcode(Is64BitTarget)
-          : getADDrrOpcode(Is64BitTarget);
+          ? getSUBrrOpcode(Is64Bit)
+          : getADDrrOpcode(Is64Bit);
         MachineInstr *MI = BuildMI(MBB, MBBI, DL, TII.get(Opc), StackPtr)
           .addReg(StackPtr)
           .addReg(Reg);
@@ -253,15 +272,15 @@ void emitSPUpdate(MachineBasicBlock &MBB, MachineBasicBlock::iterator &MBBI,
     }
 
     uint64_t ThisVal = std::min(Offset, Chunk);
-    if (ThisVal == (Is64BitTarget ? 8 : 4)) {
+    if (ThisVal == (Is64Bit ? 8 : 4)) {
       // Use push / pop instead.
       unsigned Reg = isSub
-        ? (unsigned)(Is64BitTarget ? X86::RAX : X86::EAX)
-        : findDeadCallerSavedReg(MBB, MBBI, TRI, Is64BitTarget);
+        ? (unsigned)(Is64Bit ? X86::RAX : X86::EAX)
+        : findDeadCallerSavedReg(MBB, MBBI, TRI, Is64Bit);
       if (Reg) {
-        Opc = isSub
-          ? (Is64BitTarget ? X86::PUSH64r : X86::PUSH32r)
-          : (Is64BitTarget ? X86::POP64r  : X86::POP32r);
+        unsigned Opc = isSub
+          ? (Is64Bit ? X86::PUSH64r : X86::PUSH32r)
+          : (Is64Bit ? X86::POP64r  : X86::POP32r);
         MachineInstr *MI = BuildMI(MBB, MBBI, DL, TII.get(Opc))
           .addReg(Reg, getDefRegState(!isSub) | getUndefRegState(isSub));
         if (isSub)
@@ -271,56 +290,62 @@ void emitSPUpdate(MachineBasicBlock &MBB, MachineBasicBlock::iterator &MBBI,
       }
     }
 
-    MachineInstr *MI = nullptr;
-
-    if (UseLEA) {
-      MI =  addRegOffset(BuildMI(MBB, MBBI, DL, TII.get(Opc), StackPtr),
-                          StackPtr, false, isSub ? -ThisVal : ThisVal);
-    } else {
-      MI = BuildMI(MBB, MBBI, DL, TII.get(Opc), StackPtr)
-            .addReg(StackPtr)
-            .addImm(ThisVal);
-      MI->getOperand(3).setIsDead(); // The EFLAGS implicit def is dead.
-    }
-
+    MachineInstrBuilder MI = BuildStackAdjustment(
+        MBB, MBBI, DL, isSub ? -ThisVal : ThisVal, InEpilogue);
     if (isSub)
-      MI->setFlag(MachineInstr::FrameSetup);
+      MI.setMIFlag(MachineInstr::FrameSetup);
 
     Offset -= ThisVal;
   }
 }
 
-/// mergeSPUpdatesUp - Merge two stack-manipulating instructions upper iterator.
-static
-void mergeSPUpdatesUp(MachineBasicBlock &MBB, MachineBasicBlock::iterator &MBBI,
-                      unsigned StackPtr, uint64_t *NumBytes = nullptr) {
-  if (MBBI == MBB.begin()) return;
+MachineInstrBuilder X86FrameLowering::BuildStackAdjustment(
+    MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, DebugLoc DL,
+    int64_t Offset, bool InEpilogue) const {
+  assert(Offset != 0 && "zero offset stack adjustment requested");
 
-  MachineBasicBlock::iterator PI = std::prev(MBBI);
-  unsigned Opc = PI->getOpcode();
-  if ((Opc == X86::ADD64ri32 || Opc == X86::ADD64ri8 ||
-       Opc == X86::ADD32ri || Opc == X86::ADD32ri8 ||
-       Opc == X86::LEA32r || Opc == X86::LEA64_32r) &&
-      PI->getOperand(0).getReg() == StackPtr) {
-    if (NumBytes)
-      *NumBytes += PI->getOperand(2).getImm();
-    MBB.erase(PI);
-  } else if ((Opc == X86::SUB64ri32 || Opc == X86::SUB64ri8 ||
-              Opc == X86::SUB32ri || Opc == X86::SUB32ri8) &&
-             PI->getOperand(0).getReg() == StackPtr) {
-    if (NumBytes)
-      *NumBytes -= PI->getOperand(2).getImm();
-    MBB.erase(PI);
+  // On Atom, using LEA to adjust SP is preferred, but using it in the epilogue
+  // is tricky.
+  bool UseLEA;
+  if (!InEpilogue) {
+    UseLEA = STI.useLeaForSP();
+  } else {
+    // If we can use LEA for SP but we shouldn't, check that none
+    // of the terminators uses the eflags. Otherwise we will insert
+    // a ADD that will redefine the eflags and break the condition.
+    // Alternatively, we could move the ADD, but this may not be possible
+    // and is an optimization anyway.
+    UseLEA = canUseLEAForSPInEpilogue(*MBB.getParent());
+    if (UseLEA && !STI.useLeaForSP())
+      UseLEA = terminatorsNeedFlagsAsInput(MBB);
+    // If that assert breaks, that means we do not do the right thing
+    // in canUseAsEpilogue.
+    assert((UseLEA || !terminatorsNeedFlagsAsInput(MBB)) &&
+           "We shouldn't have allowed this insertion point");
+  }
+
+  MachineInstrBuilder MI;
+  if (UseLEA) {
+    MI = addRegOffset(BuildMI(MBB, MBBI, DL,
+                              TII.get(getLEArOpcode(Uses64BitFramePtr)),
+                              StackPtr),
+                      StackPtr, false, Offset);
+  } else {
+    bool IsSub = Offset < 0;
+    uint64_t AbsOffset = IsSub ? -Offset : Offset;
+    unsigned Opc = IsSub ? getSUBriOpcode(Uses64BitFramePtr, AbsOffset)
+                         : getADDriOpcode(Uses64BitFramePtr, AbsOffset);
+    MI = BuildMI(MBB, MBBI, DL, TII.get(Opc), StackPtr)
+             .addReg(StackPtr)
+             .addImm(AbsOffset);
+    MI->getOperand(3).setIsDead(); // The EFLAGS implicit def is dead.
   }
+  return MI;
 }
 
-/// mergeSPUpdates - Checks the instruction before/after the passed
-/// instruction. If it is an ADD/SUB/LEA instruction it is deleted argument and
-/// the stack adjustment is returned as a positive value for ADD/LEA and a
-/// negative for SUB.
-static int mergeSPUpdates(MachineBasicBlock &MBB,
-                          MachineBasicBlock::iterator &MBBI, unsigned StackPtr,
-                          bool doMergeWithPrevious) {
+int X86FrameLowering::mergeSPUpdates(MachineBasicBlock &MBB,
+                                     MachineBasicBlock::iterator &MBBI,
+                                     bool doMergeWithPrevious) const {
   if ((doMergeWithPrevious && MBBI == MBB.begin()) ||
       (!doMergeWithPrevious && MBBI == MBB.end()))
     return 0;
@@ -349,6 +374,15 @@ static int mergeSPUpdates(MachineBasicBlock &MBB,
   return Offset;
 }
 
+void X86FrameLowering::BuildCFI(MachineBasicBlock &MBB,
+                                MachineBasicBlock::iterator MBBI, DebugLoc DL,
+                                MCCFIInstruction CFIInst) const {
+  MachineFunction &MF = *MBB.getParent();
+  unsigned CFIIndex = MF.getMMI().addFrameInst(CFIInst);
+  BuildMI(MBB, MBBI, DL, TII.get(TargetOpcode::CFI_INSTRUCTION))
+      .addCFIIndex(CFIIndex);
+}
+
 void
 X86FrameLowering::emitCalleeSavedFrameMoves(MachineBasicBlock &MBB,
                                             MachineBasicBlock::iterator MBBI,
@@ -357,7 +391,6 @@ X86FrameLowering::emitCalleeSavedFrameMoves(MachineBasicBlock &MBB,
   MachineFrameInfo *MFI = MF.getFrameInfo();
   MachineModuleInfo &MMI = MF.getMMI();
   const MCRegisterInfo *MRI = MMI.getContext().getRegisterInfo();
-  const TargetInstrInfo &TII = *MF.getSubtarget().getInstrInfo();
 
   // Add callee saved registers to move list.
   const std::vector<CalleeSavedInfo> &CSI = MFI->getCalleeSavedInfo();
@@ -370,11 +403,8 @@ X86FrameLowering::emitCalleeSavedFrameMoves(MachineBasicBlock &MBB,
     unsigned Reg = I->getReg();
 
     unsigned DwarfReg = MRI->getDwarfRegNum(Reg, true);
-    unsigned CFIIndex =
-        MMI.addFrameInst(MCCFIInstruction::createOffset(nullptr, DwarfReg,
-                                                        Offset));
-    BuildMI(MBB, MBBI, DL, TII.get(TargetOpcode::CFI_INSTRUCTION))
-        .addCFIIndex(CFIIndex);
+    BuildCFI(MBB, MBBI, DL,
+             MCCFIInstruction::createOffset(nullptr, DwarfReg, Offset));
   }
 }
 
@@ -398,10 +428,7 @@ static bool usesTheStack(const MachineFunction &MF) {
 void X86FrameLowering::emitStackProbeCall(MachineFunction &MF,
                                           MachineBasicBlock &MBB,
                                           MachineBasicBlock::iterator MBBI,
-                                          DebugLoc DL) {
-  const X86Subtarget &STI = MF.getSubtarget<X86Subtarget>();
-  const TargetInstrInfo &TII = *STI.getInstrInfo();
-  bool Is64Bit = STI.is64Bit();
+                                          DebugLoc DL) const {
   bool IsLargeCodeModel = MF.getTarget().getCodeModel() == CodeModel::Large;
 
   unsigned CallOp;
@@ -467,13 +494,10 @@ static unsigned calculateSetFPREG(uint64_t SPAdjust) {
 // info, we need to know the ABI stack alignment as well in case we
 // have a call out.  Otherwise just make sure we have some alignment - we'll
 // go with the minimum SlotSize.
-static uint64_t calculateMaxStackAlign(const MachineFunction &MF) {
+uint64_t X86FrameLowering::calculateMaxStackAlign(const MachineFunction &MF) const {
   const MachineFrameInfo *MFI = MF.getFrameInfo();
   uint64_t MaxAlign = MFI->getMaxAlignment(); // Desired stack alignment.
-  const X86Subtarget &STI = MF.getSubtarget<X86Subtarget>();
-  const X86RegisterInfo *RegInfo = STI.getRegisterInfo();
-  unsigned SlotSize = RegInfo->getSlotSize();
-  unsigned StackAlign = STI.getFrameLowering()->getStackAlignment();
+  unsigned StackAlign = getStackAlignment();
   if (ForceStackAlign) {
     if (MFI->hasCalls())
       MaxAlign = (StackAlign > MaxAlign) ? StackAlign : MaxAlign;
@@ -483,6 +507,22 @@ static uint64_t calculateMaxStackAlign(const MachineFunction &MF) {
   return MaxAlign;
 }
 
+void X86FrameLowering::BuildStackAlignAND(MachineBasicBlock &MBB,
+                                          MachineBasicBlock::iterator MBBI,
+                                          DebugLoc DL,
+                                          uint64_t MaxAlign) const {
+  uint64_t Val = -MaxAlign;
+  MachineInstr *MI =
+      BuildMI(MBB, MBBI, DL, TII.get(getANDriOpcode(Uses64BitFramePtr, Val)),
+              StackPtr)
+          .addReg(StackPtr)
+          .addImm(Val)
+          .setMIFlag(MachineInstr::FrameSetup);
+
+  // The EFLAGS implicit def is dead.
+  MI->getOperand(3).setIsDead();
+}
+
 /// emitPrologue - Push callee-saved registers onto the stack, which
 /// automatically adjust the stack pointer. Adjust the stack pointer to allocate
 /// space for local variables. Also emit labels used by the exception handler to
@@ -567,42 +607,34 @@ static uint64_t calculateMaxStackAlign(const MachineFunction &MF) {
   - for 32-bit code, substitute %e?? registers for %r??
 */
 
-void X86FrameLowering::emitPrologue(MachineFunction &MF) const {
-  MachineBasicBlock &MBB = MF.front(); // Prologue goes in entry BB.
+void X86FrameLowering::emitPrologue(MachineFunction &MF,
+                                    MachineBasicBlock &MBB) const {
+  assert(&STI == &MF.getSubtarget<X86Subtarget>() &&
+         "MF used frame lowering for wrong subtarget");
   MachineBasicBlock::iterator MBBI = MBB.begin();
   MachineFrameInfo *MFI = MF.getFrameInfo();
   const Function *Fn = MF.getFunction();
-  const X86Subtarget &STI = MF.getSubtarget<X86Subtarget>();
-  const X86RegisterInfo *RegInfo = STI.getRegisterInfo();
-  const TargetInstrInfo &TII = *STI.getInstrInfo();
   MachineModuleInfo &MMI = MF.getMMI();
   X86MachineFunctionInfo *X86FI = MF.getInfo<X86MachineFunctionInfo>();
   uint64_t MaxAlign = calculateMaxStackAlign(MF); // Desired stack alignment.
   uint64_t StackSize = MFI->getStackSize();    // Number of bytes to allocate.
   bool HasFP = hasFP(MF);
-  bool Is64Bit = STI.is64Bit();
-  // standard x86_64 and NaCl use 64-bit frame/stack pointers, x32 - 32-bit.
-  const bool Uses64BitFramePtr = STI.isTarget64BitLP64() || STI.isTargetNaCl64();
-  bool IsWin64 = STI.isCallingConvWin64(Fn->getCallingConv());
-  // Not necessarily synonymous with IsWin64.
-  bool IsWinEH = MF.getTarget().getMCAsmInfo()->usesWindowsCFI();
-  bool NeedsWinEH = IsWinEH && Fn->needsUnwindTableEntry();
+  bool IsWin64CC = STI.isCallingConvWin64(Fn->getCallingConv());
+  bool IsWin64Prologue = MF.getTarget().getMCAsmInfo()->usesWindowsCFI();
+  bool NeedsWinCFI = IsWin64Prologue && Fn->needsUnwindTableEntry();
   bool NeedsDwarfCFI =
-      !IsWinEH && (MMI.hasDebugInfo() || Fn->needsUnwindTableEntry());
-  bool UseLEA = STI.useLeaForSP();
-  unsigned SlotSize = RegInfo->getSlotSize();
-  unsigned FramePtr = RegInfo->getFrameRegister(MF);
+      !IsWin64Prologue && (MMI.hasDebugInfo() || Fn->needsUnwindTableEntry());
+  unsigned FramePtr = TRI->getFrameRegister(MF);
   const unsigned MachineFramePtr =
       STI.isTarget64BitILP32()
           ? getX86SubSuperRegister(FramePtr, MVT::i64, false)
           : FramePtr;
-  unsigned StackPtr = RegInfo->getStackRegister();
-  unsigned BasePtr = RegInfo->getBaseRegister();
+  unsigned BasePtr = TRI->getBaseRegister();
   DebugLoc DL;
 
   // Add RETADDR move area to callee saved frame size.
   int TailCallReturnAddrDelta = X86FI->getTCReturnAddrDelta();
-  if (TailCallReturnAddrDelta && IsWinEH)
+  if (TailCallReturnAddrDelta && IsWin64Prologue)
     report_fatal_error("Can't handle guaranteed tail call under win64 yet");
 
   if (TailCallReturnAddrDelta < 0)
@@ -625,10 +657,10 @@ void X86FrameLowering::emitPrologue(MachineFunction &MF) const {
   // stack pointer (we fit in the Red Zone). We also check that we don't
   // push and pop from the stack.
   if (Is64Bit && !Fn->hasFnAttribute(Attribute::NoRedZone) &&
-      !RegInfo->needsStackRealignment(MF) &&
+      !TRI->needsStackRealignment(MF) &&
       !MFI->hasVarSizedObjects() && // No dynamic alloca.
       !MFI->adjustsStack() &&       // No calls.
-      !IsWin64 &&                   // Win64 has no Red Zone
+      !IsWin64CC &&                 // Win64 has no Red Zone
       !usesTheStack(MF) &&          // Don't push and pop.
       !MF.shouldSplitStack()) {     // Regular stack
     uint64_t MinSize = X86FI->getCalleeSavedFrameSize();
@@ -641,14 +673,9 @@ void X86FrameLowering::emitPrologue(MachineFunction &MF) const {
   // applies to tail call optimized functions where the callee argument stack
   // size is bigger than the callers.
   if (TailCallReturnAddrDelta < 0) {
-    MachineInstr *MI =
-      BuildMI(MBB, MBBI, DL,
-              TII.get(getSUBriOpcode(Uses64BitFramePtr, -TailCallReturnAddrDelta)),
-              StackPtr)
-        .addReg(StackPtr)
-        .addImm(-TailCallReturnAddrDelta)
+    BuildStackAdjustment(MBB, MBBI, DL, TailCallReturnAddrDelta,
+                         /*InEpilogue=*/false)
         .setMIFlag(MachineInstr::FrameSetup);
-    MI->getOperand(3).setIsDead(); // The EFLAGS implicit def is dead.
   }
 
   // Mapping for machine moves:
@@ -678,7 +705,7 @@ void X86FrameLowering::emitPrologue(MachineFunction &MF) const {
     NumBytes = FrameSize - X86FI->getCalleeSavedFrameSize();
 
     // Callee-saved registers are pushed on stack before the stack is realigned.
-    if (RegInfo->needsStackRealignment(MF) && !IsWinEH)
+    if (TRI->needsStackRealignment(MF) && !IsWin64Prologue)
       NumBytes = RoundUpToAlignment(NumBytes, MaxAlign);
 
     // Get the offset of the stack slot for the EBP register, which is
@@ -687,76 +714,30 @@ void X86FrameLowering::emitPrologue(MachineFunction &MF) const {
     MFI->setOffsetAdjustment(-NumBytes);
 
     // Save EBP/RBP into the appropriate stack slot.
-    // @LOCALMOD-BEGIN
-    unsigned RegToPush = MachineFramePtr;
-    const bool HideSandboxBase = (FlagHideSandboxBase &&
-                                  STI.isTargetNaCl64() &&
-                                  !FlagUseZeroBasedSandbox);
-    if (HideSandboxBase) {
-      // Hide the sandbox base address by masking off the upper 32
-      // bits of the pushed/saved RBP on the stack, using:
-      //   mov %ebp, %r10d
-      //   push %r10
-      // instead of:
-      //   push %rbp
-      // Additionally, we can use rax instead of r10 when it is not a
-      // varargs function and therefore rax is available, saving one
-      // byte of REX prefix per instruction.
-      // Note that the epilog already adds R15 when restoring RBP.
-
-      // mov %ebp, %r10d
-      if (Fn->isVarArg()) {
-        // Note: This use of r10 in the prolog can't be used with the
-        // gcc "nest" attribute, due to its use of r10.  Example:
-        // target triple = "x86_64-pc-linux-gnu"
-        // define i64 @func(i64 nest %arg) {
-        //   ret i64 %arg
-        // }
-        //
-        // $ clang -m64 llvm_nest_attr.ll -S -o -
-        // ...
-        // func:
-        //     movq    %r10, %rax
-        //     ret
-        RegToPush = X86::R10;
-      } else {
-        RegToPush = X86::RAX;
-      }
-      BuildMI(MBB, MBBI, DL, TII.get(X86::MOV32rr),
-              getX86SubSuperRegister(RegToPush, MVT::i32, false))
-        .addReg(getX86SubSuperRegister(FramePtr, MVT::i32, false))
-        .setMIFlag(MachineInstr::FrameSetup);
-    }
-    // @LOCALMOD-END
     BuildMI(MBB, MBBI, DL, TII.get(Is64Bit ? X86::PUSH64r : X86::PUSH32r))
-      .addReg(RegToPush, RegState::Kill) // @LOCALMOD
+      .addReg(MachineFramePtr, RegState::Kill)
       .setMIFlag(MachineInstr::FrameSetup);
 
     if (NeedsDwarfCFI) {
       // Mark the place where EBP/RBP was saved.
       // Define the current CFA rule to use the provided offset.
       assert(StackSize);
-      unsigned CFIIndex = MMI.addFrameInst(
-          MCCFIInstruction::createDefCfaOffset(nullptr, 2 * stackGrowth));
-      BuildMI(MBB, MBBI, DL, TII.get(TargetOpcode::CFI_INSTRUCTION))
-          .addCFIIndex(CFIIndex);
+      BuildCFI(MBB, MBBI, DL,
+               MCCFIInstruction::createDefCfaOffset(nullptr, 2 * stackGrowth));
 
       // Change the rule for the FramePtr to be an "offset" rule.
-      unsigned DwarfFramePtr = RegInfo->getDwarfRegNum(MachineFramePtr, true);
-      CFIIndex = MMI.addFrameInst(
-          MCCFIInstruction::createOffset(nullptr,
-                                         DwarfFramePtr, 2 * stackGrowth));
-      BuildMI(MBB, MBBI, DL, TII.get(TargetOpcode::CFI_INSTRUCTION))
-          .addCFIIndex(CFIIndex);
+      unsigned DwarfFramePtr = TRI->getDwarfRegNum(MachineFramePtr, true);
+      BuildCFI(MBB, MBBI, DL, MCCFIInstruction::createOffset(
+                                  nullptr, DwarfFramePtr, 2 * stackGrowth));
     }
 
-    if (NeedsWinEH) {
+    if (NeedsWinCFI) {
       BuildMI(MBB, MBBI, DL, TII.get(X86::SEH_PushReg))
           .addImm(FramePtr)
           .setMIFlag(MachineInstr::FrameSetup);
     }
 
-    if (!IsWinEH) {
+    if (!IsWin64Prologue) {
       // Update EBP with the new base value.
       BuildMI(MBB, MBBI, DL,
               TII.get(Uses64BitFramePtr ? X86::MOV64rr : X86::MOV32rr),
@@ -768,11 +749,9 @@ void X86FrameLowering::emitPrologue(MachineFunction &MF) const {
     if (NeedsDwarfCFI) {
       // Mark effective beginning of when frame pointer becomes valid.
       // Define the current CFA to use the EBP/RBP register.
-      unsigned DwarfFramePtr = RegInfo->getDwarfRegNum(MachineFramePtr, true);
-      unsigned CFIIndex = MMI.addFrameInst(
-          MCCFIInstruction::createDefCfaRegister(nullptr, DwarfFramePtr));
-      BuildMI(MBB, MBBI, DL, TII.get(TargetOpcode::CFI_INSTRUCTION))
-          .addCFIIndex(CFIIndex);
+      unsigned DwarfFramePtr = TRI->getDwarfRegNum(MachineFramePtr, true);
+      BuildCFI(MBB, MBBI, DL,
+               MCCFIInstruction::createDefCfaRegister(nullptr, DwarfFramePtr));
     }
 
     // Mark the FramePtr as live-in in every block.
@@ -787,6 +766,7 @@ void X86FrameLowering::emitPrologue(MachineFunction &MF) const {
   int StackOffset = 2 * stackGrowth;
 
   while (MBBI != MBB.end() &&
+         MBBI->getFlag(MachineInstr::FrameSetup) &&
          (MBBI->getOpcode() == X86::PUSH32r ||
           MBBI->getOpcode() == X86::PUSH64r)) {
     PushedRegs = true;
@@ -797,14 +777,12 @@ void X86FrameLowering::emitPrologue(MachineFunction &MF) const {
       // Mark callee-saved push instruction.
       // Define the current CFA rule to use the provided offset.
       assert(StackSize);
-      unsigned CFIIndex = MMI.addFrameInst(
-          MCCFIInstruction::createDefCfaOffset(nullptr, StackOffset));
-      BuildMI(MBB, MBBI, DL, TII.get(TargetOpcode::CFI_INSTRUCTION))
-          .addCFIIndex(CFIIndex);
+      BuildCFI(MBB, MBBI, DL,
+               MCCFIInstruction::createDefCfaOffset(nullptr, StackOffset));
       StackOffset += stackGrowth;
     }
 
-    if (NeedsWinEH) {
+    if (NeedsWinCFI) {
       BuildMI(MBB, MBBI, DL, TII.get(X86::SEH_PushReg)).addImm(Reg).setMIFlag(
           MachineInstr::FrameSetup);
     }
@@ -813,24 +791,15 @@ void X86FrameLowering::emitPrologue(MachineFunction &MF) const {
   // Realign stack after we pushed callee-saved registers (so that we'll be
   // able to calculate their offsets from the frame pointer).
   // Don't do this for Win64, it needs to realign the stack after the prologue.
-  if (!IsWinEH && RegInfo->needsStackRealignment(MF)) {
+  if (!IsWin64Prologue && TRI->needsStackRealignment(MF)) {
     assert(HasFP && "There should be a frame pointer if stack is realigned.");
-    uint64_t Val = -MaxAlign;
-    MachineInstr *MI =
-        BuildMI(MBB, MBBI, DL, TII.get(getANDriOpcode(Uses64BitFramePtr, Val)),
-                StackPtr)
-            .addReg(StackPtr)
-            .addImm(Val)
-            .setMIFlag(MachineInstr::FrameSetup);
-
-    // The EFLAGS implicit def is dead.
-    MI->getOperand(3).setIsDead();
+    BuildStackAlignAND(MBB, MBBI, DL, MaxAlign);
   }
 
   // If there is an SUB32ri of ESP immediately before this instruction, merge
   // the two. This can be the case when tail call elimination is enabled and
   // the callee has more arguments then the caller.
-  NumBytes -= mergeSPUpdates(MBB, MBBI, StackPtr, true);
+  NumBytes -= mergeSPUpdates(MBB, MBBI, true);
 
   // Adjust stack pointer: ESP -= numbytes.
 
@@ -843,7 +812,7 @@ void X86FrameLowering::emitPrologue(MachineFunction &MF) const {
   // increments is necessary to ensure that the guard pages used by the OS
   // virtual memory manager are allocated in correct sequence.
   uint64_t AlignedNumBytes = NumBytes;
-  if (IsWinEH && RegInfo->needsStackRealignment(MF))
+  if (IsWin64Prologue && TRI->needsStackRealignment(MF))
     AlignedNumBytes = RoundUpToAlignment(AlignedNumBytes, MaxAlign);
   if (AlignedNumBytes >= StackProbeSize && UseStackProbe) {
     // Check whether EAX is livein for this function.
@@ -904,17 +873,16 @@ void X86FrameLowering::emitPrologue(MachineFunction &MF) const {
       MBB.insert(MBBI, MI);
     }
   } else if (NumBytes) {
-    emitSPUpdate(MBB, MBBI, StackPtr, -(int64_t)NumBytes, Is64Bit, Uses64BitFramePtr,
-                 UseLEA, TII, *RegInfo);
+    emitSPUpdate(MBB, MBBI, -(int64_t)NumBytes, /*InEpilogue=*/false);
   }
 
-  if (NeedsWinEH && NumBytes)
+  if (NeedsWinCFI && NumBytes)
     BuildMI(MBB, MBBI, DL, TII.get(X86::SEH_StackAlloc))
         .addImm(NumBytes)
         .setMIFlag(MachineInstr::FrameSetup);
 
   int SEHFrameOffset = 0;
-  if (IsWinEH && HasFP) {
+  if (IsWin64Prologue && HasFP) {
     SEHFrameOffset = calculateSetFPREG(NumBytes);
     if (SEHFrameOffset)
       addRegOffset(BuildMI(MBB, MBBI, DL, TII.get(X86::LEA64r), FramePtr),
@@ -922,7 +890,7 @@ void X86FrameLowering::emitPrologue(MachineFunction &MF) const {
     else
       BuildMI(MBB, MBBI, DL, TII.get(X86::MOV64rr), FramePtr).addReg(StackPtr);
 
-    if (NeedsWinEH)
+    if (NeedsWinCFI)
       BuildMI(MBB, MBBI, DL, TII.get(X86::SEH_SetFrame))
           .addImm(FramePtr)
           .addImm(SEHFrameOffset)
@@ -933,7 +901,7 @@ void X86FrameLowering::emitPrologue(MachineFunction &MF) const {
     const MachineInstr *FrameInstr = &*MBBI;
     ++MBBI;
 
-    if (NeedsWinEH) {
+    if (NeedsWinCFI) {
       int FI;
       if (unsigned Reg = TII.isStoreToStackSlot(FrameInstr, FI)) {
         if (X86::FR64RegClass.contains(Reg)) {
@@ -949,45 +917,49 @@ void X86FrameLowering::emitPrologue(MachineFunction &MF) const {
     }
   }
 
-  if (NeedsWinEH)
+  if (NeedsWinCFI)
     BuildMI(MBB, MBBI, DL, TII.get(X86::SEH_EndPrologue))
         .setMIFlag(MachineInstr::FrameSetup);
 
   // Realign stack after we spilled callee-saved registers (so that we'll be
   // able to calculate their offsets from the frame pointer).
   // Win64 requires aligning the stack after the prologue.
-  if (IsWinEH && RegInfo->needsStackRealignment(MF)) {
+  if (IsWin64Prologue && TRI->needsStackRealignment(MF)) {
     assert(HasFP && "There should be a frame pointer if stack is realigned.");
-    uint64_t Val = -MaxAlign;
-    MachineInstr *MI =
-        BuildMI(MBB, MBBI, DL, TII.get(getANDriOpcode(Uses64BitFramePtr, Val)),
-                StackPtr)
-            .addReg(StackPtr)
-            .addImm(Val)
-            .setMIFlag(MachineInstr::FrameSetup);
-
-    // The EFLAGS implicit def is dead.
-    MI->getOperand(3).setIsDead();
+    BuildStackAlignAND(MBB, MBBI, DL, MaxAlign);
   }
 
   // If we need a base pointer, set it up here. It's whatever the value
   // of the stack pointer is at this point. Any variable size objects
   // will be allocated after this, so we can still use the base pointer
   // to reference locals.
-  if (RegInfo->hasBasePointer(MF)) {
+  if (TRI->hasBasePointer(MF)) {
     // Update the base pointer with the current stack pointer.
     unsigned Opc = Uses64BitFramePtr ? X86::MOV64rr : X86::MOV32rr;
     BuildMI(MBB, MBBI, DL, TII.get(Opc), BasePtr)
       .addReg(StackPtr)
       .setMIFlag(MachineInstr::FrameSetup);
     if (X86FI->getRestoreBasePointer()) {
-      // Stash value of base pointer.  Saving RSP instead of EBP shortens dependence chain.
+      // Stash value of base pointer.  Saving RSP instead of EBP shortens
+      // dependence chain. Used by SjLj EH.
       unsigned Opm = Uses64BitFramePtr ? X86::MOV64mr : X86::MOV32mr;
       addRegOffset(BuildMI(MBB, MBBI, DL, TII.get(Opm)),
                    FramePtr, true, X86FI->getRestoreBasePointerOffset())
         .addReg(StackPtr)
         .setMIFlag(MachineInstr::FrameSetup);
     }
+
+    if (X86FI->getHasSEHFramePtrSave()) {
+      // Stash the value of the frame pointer relative to the base pointer for
+      // Win32 EH. This supports Win32 EH, which does the inverse of the above:
+      // it recovers the frame pointer from the base pointer rather than the
+      // other way around.
+      unsigned Opm = Uses64BitFramePtr ? X86::MOV64mr : X86::MOV32mr;
+      addRegOffset(BuildMI(MBB, MBBI, DL, TII.get(Opm)), BasePtr, true,
+                   getFrameIndexOffset(MF, X86FI->getSEHFramePtrSaveIndex()))
+          .addReg(FramePtr)
+          .setMIFlag(MachineInstr::FrameSetup);
+    }
   }
 
   if (((!HasFP && NumBytes) || PushedRegs) && NeedsDwarfCFI) {
@@ -995,12 +967,8 @@ void X86FrameLowering::emitPrologue(MachineFunction &MF) const {
     if (!HasFP && NumBytes) {
       // Define the current CFA rule to use the provided offset.
       assert(StackSize);
-      unsigned CFIIndex = MMI.addFrameInst(
-          MCCFIInstruction::createDefCfaOffset(nullptr,
-                                               -StackSize + stackGrowth));
-
-      BuildMI(MBB, MBBI, DL, TII.get(TargetOpcode::CFI_INSTRUCTION))
-          .addCFIIndex(CFIIndex);
+      BuildCFI(MBB, MBBI, DL, MCCFIInstruction::createDefCfaOffset(
+                                  nullptr, -StackSize + stackGrowth));
     }
 
     // Emit DWARF info specifying the offsets of the callee-saved registers.
@@ -1009,64 +977,35 @@ void X86FrameLowering::emitPrologue(MachineFunction &MF) const {
   }
 }
 
+bool X86FrameLowering::canUseLEAForSPInEpilogue(
+    const MachineFunction &MF) const {
+  // We can't use LEA instructions for adjusting the stack pointer if this is a
+  // leaf function in the Win64 ABI.  Only ADD instructions may be used to
+  // deallocate the stack.
+  // This means that we can use LEA for SP in two situations:
+  // 1. We *aren't* using the Win64 ABI which means we are free to use LEA.
+  // 2. We *have* a frame pointer which means we are permitted to use LEA.
+  return !MF.getTarget().getMCAsmInfo()->usesWindowsCFI() || hasFP(MF);
+}
+
 void X86FrameLowering::emitEpilogue(MachineFunction &MF,
                                     MachineBasicBlock &MBB) const {
   const MachineFrameInfo *MFI = MF.getFrameInfo();
   X86MachineFunctionInfo *X86FI = MF.getInfo<X86MachineFunctionInfo>();
-  const X86Subtarget &STI = MF.getSubtarget<X86Subtarget>();
-  const X86RegisterInfo *RegInfo = STI.getRegisterInfo();
-  const TargetInstrInfo &TII = *STI.getInstrInfo();
-  MachineBasicBlock::iterator MBBI = MBB.getLastNonDebugInstr();
-  assert(MBBI != MBB.end() && "Returning block has no instructions");
-  unsigned RetOpcode = MBBI->getOpcode();
-  DebugLoc DL = MBBI->getDebugLoc();
-  bool Is64Bit = STI.is64Bit();
+  MachineBasicBlock::iterator MBBI = MBB.getFirstTerminator();
+  DebugLoc DL;
+  if (MBBI != MBB.end())
+    DL = MBBI->getDebugLoc();
   // standard x86_64 and NaCl use 64-bit frame/stack pointers, x32 - 32-bit.
-  const bool Uses64BitFramePtr = STI.isTarget64BitLP64() || STI.isTargetNaCl64();
-  bool HasFP = hasFP(MF);
   const bool Is64BitILP32 = STI.isTarget64BitILP32();
-  unsigned SlotSize = RegInfo->getSlotSize();
-  unsigned FramePtr = RegInfo->getFrameRegister(MF);
+  unsigned FramePtr = TRI->getFrameRegister(MF);
   unsigned MachineFramePtr =
       Is64BitILP32 ? getX86SubSuperRegister(FramePtr, MVT::i64, false)
                    : FramePtr;
-  unsigned StackPtr = RegInfo->getStackRegister();
-
-  bool IsWinEH = MF.getTarget().getMCAsmInfo()->usesWindowsCFI();
-  bool NeedsWinEH = IsWinEH && MF.getFunction()->needsUnwindTableEntry();
-  bool UseLEAForSP = false;
 
-  // We can't use LEA instructions for adjusting the stack pointer if this is a
-  // leaf function in the Win64 ABI.  Only ADD instructions may be used to
-  // deallocate the stack.
-  if (STI.useLeaForSP()) {
-    if (!IsWinEH) {
-      // We *aren't* using the Win64 ABI which means we are free to use LEA.
-      UseLEAForSP = true;
-    } else if (HasFP) {
-      // We *have* a frame pointer which means we are permitted to use LEA.
-      UseLEAForSP = true;
-    }
-  }
-
-  switch (RetOpcode) {
-  default:
-    llvm_unreachable("Can only insert epilogue into returning blocks");
-  case X86::RETQ:
-  case X86::RETL:
-  case X86::RETIL:
-  case X86::RETIQ:
-  case X86::TCRETURNdi:
-  case X86::TCRETURNri:
-  case X86::TCRETURNmi:
-  case X86::TCRETURNdi64:
-  case X86::TCRETURNri64:
-  case X86::TCRETURNmi64:
-  case X86::NACL_CG_TCRETURNdi64: // @LOCALMOD
-  case X86::EH_RETURN:
-  case X86::EH_RETURN64:
-    break;  // These are ok
-  }
+  bool IsWin64Prologue = MF.getTarget().getMCAsmInfo()->usesWindowsCFI();
+  bool NeedsWinCFI =
+      IsWin64Prologue && MF.getFunction()->needsUnwindTableEntry();
 
   // Get the number of bytes to allocate from the FrameInfo.
   uint64_t StackSize = MFI->getStackSize();
@@ -1081,7 +1020,7 @@ void X86FrameLowering::emitEpilogue(MachineFunction &MF,
 
     // Callee-saved registers were pushed on stack before the stack was
     // realigned.
-    if (RegInfo->needsStackRealignment(MF) && !IsWinEH)
+    if (TRI->needsStackRealignment(MF) && !IsWin64Prologue)
       NumBytes = RoundUpToAlignment(FrameSize, MaxAlign);
 
     // Pop EBP.
@@ -1105,21 +1044,23 @@ void X86FrameLowering::emitEpilogue(MachineFunction &MF,
   }
   MachineBasicBlock::iterator FirstCSPop = MBBI;
 
-  DL = MBBI->getDebugLoc();
+  if (MBBI != MBB.end())
+    DL = MBBI->getDebugLoc();
 
   // If there is an ADD32ri or SUB32ri of ESP immediately before this
   // instruction, merge the two instructions.
   if (NumBytes || MFI->hasVarSizedObjects())
-    mergeSPUpdatesUp(MBB, MBBI, StackPtr, &NumBytes);
+    NumBytes += mergeSPUpdates(MBB, MBBI, true);
 
   // If dynamic alloca is used, then reset esp to point to the last callee-saved
   // slot before popping them off! Same applies for the case, when stack was
   // realigned.
-  if (RegInfo->needsStackRealignment(MF) || MFI->hasVarSizedObjects()) {
-    if (RegInfo->needsStackRealignment(MF))
+  if (TRI->needsStackRealignment(MF) || MFI->hasVarSizedObjects()) {
+    if (TRI->needsStackRealignment(MF))
       MBBI = FirstCSPop;
     unsigned SEHFrameOffset = calculateSetFPREG(SEHStackAllocAmt);
-    uint64_t LEAAmount = IsWinEH ? SEHStackAllocAmt - SEHFrameOffset : -CSSize;
+    uint64_t LEAAmount =
+        IsWin64Prologue ? SEHStackAllocAmt - SEHFrameOffset : -CSSize;
 
     // There are only two legal forms of epilogue:
     // - add SEHAllocationSize, %rsp
@@ -1141,8 +1082,7 @@ void X86FrameLowering::emitEpilogue(MachineFunction &MF,
     }
   } else if (NumBytes) {
     // Adjust stack pointer back: ESP += numbytes.
-    emitSPUpdate(MBB, MBBI, StackPtr, NumBytes, Is64Bit, Uses64BitFramePtr,
-                 UseLEAForSP, TII, *RegInfo);
+    emitSPUpdate(MBB, MBBI, NumBytes, /*InEpilogue=*/true);
     --MBBI;
   }
 
@@ -1152,113 +1092,23 @@ void X86FrameLowering::emitEpilogue(MachineFunction &MF,
   // into the epilogue.  To cope with that, we insert an epilogue marker here,
   // then replace it with a 'nop' if it ends up immediately after a CALL in the
   // final emitted code.
-  if (NeedsWinEH)
+  if (NeedsWinCFI)
     BuildMI(MBB, MBBI, DL, TII.get(X86::SEH_Epilogue));
 
-  // We're returning from function via eh_return.
-  if (RetOpcode == X86::EH_RETURN || RetOpcode == X86::EH_RETURN64) {
-    MBBI = MBB.getLastNonDebugInstr();
-    MachineOperand &DestAddr  = MBBI->getOperand(0);
-    assert(DestAddr.isReg() && "Offset should be in register!");
-    BuildMI(MBB, MBBI, DL,
-            TII.get(Uses64BitFramePtr ? X86::MOV64rr : X86::MOV32rr),
-            StackPtr).addReg(DestAddr.getReg());
-  } else if (RetOpcode == X86::TCRETURNri || RetOpcode == X86::TCRETURNdi ||
-             RetOpcode == X86::TCRETURNmi ||
-             RetOpcode == X86::TCRETURNri64 || RetOpcode == X86::TCRETURNdi64 ||
-             RetOpcode == X86::NACL_CG_TCRETURNdi64 || // @LOCALMOD
-             RetOpcode == X86::TCRETURNmi64) {
-    bool isMem = RetOpcode == X86::TCRETURNmi || RetOpcode == X86::TCRETURNmi64;
-    // Tail call return: adjust the stack pointer and jump to callee.
-    MBBI = MBB.getLastNonDebugInstr();
-    MachineOperand &JumpTarget = MBBI->getOperand(0);
-    MachineOperand &StackAdjust = MBBI->getOperand(isMem ? 5 : 1);
-    assert(StackAdjust.isImm() && "Expecting immediate value.");
-
-    // Adjust stack pointer.
-    int StackAdj = StackAdjust.getImm();
-    int MaxTCDelta = X86FI->getTCReturnAddrDelta();
-    int Offset = 0;
-    assert(MaxTCDelta <= 0 && "MaxTCDelta should never be positive");
-
-    // Incoporate the retaddr area.
-    Offset = StackAdj-MaxTCDelta;
-    assert(Offset >= 0 && "Offset should never be negative");
-
-    if (Offset) {
-      // Check for possible merge with preceding ADD instruction.
-      Offset += mergeSPUpdates(MBB, MBBI, StackPtr, true);
-      emitSPUpdate(MBB, MBBI, StackPtr, Offset, Is64Bit, Uses64BitFramePtr,
-                   UseLEAForSP, TII, *RegInfo);
-    }
-
-    // Jump to label or value in register.
-    bool IsWin64 = STI.isTargetWin64();
-    if (RetOpcode == X86::TCRETURNdi || RetOpcode == X86::TCRETURNdi64 ||
-        RetOpcode == X86::NACL_CG_TCRETURNdi64) { // @LOCALMOD
-      // @LOCALMOD-BEGIN
-      unsigned Op;
-      switch (RetOpcode) {
-      case X86::TCRETURNdi: 
-        Op = X86::TAILJMPd;
-        break;
-      case X86::TCRETURNdi64: 
-        Op = (IsWin64 ? X86::TAILJMPd64_REX : X86::TAILJMPd64);
-        break;
-      case X86::NACL_CG_TCRETURNdi64:
-        Op = X86::NACL_CG_TAILJMPd64;
-        break;
-      default: llvm_unreachable("Unexpected return opcode");
-      }
-      // @LOCALMOD-END
-      MachineInstrBuilder MIB = BuildMI(MBB, MBBI, DL, TII.get(Op));
-      if (JumpTarget.isGlobal())
-        MIB.addGlobalAddress(JumpTarget.getGlobal(), JumpTarget.getOffset(),
-                             JumpTarget.getTargetFlags());
-      else {
-        assert(JumpTarget.isSymbol());
-        MIB.addExternalSymbol(JumpTarget.getSymbolName(),
-                              JumpTarget.getTargetFlags());
-      }
-    } else if (RetOpcode == X86::TCRETURNmi || RetOpcode == X86::TCRETURNmi64) {
-      unsigned Op = (RetOpcode == X86::TCRETURNmi)
-                        ? X86::TAILJMPm
-                        : (IsWin64 ? X86::TAILJMPm64_REX : X86::TAILJMPm64);
-      MachineInstrBuilder MIB = BuildMI(MBB, MBBI, DL, TII.get(Op));
-      for (unsigned i = 0; i != 5; ++i)
-        MIB.addOperand(MBBI->getOperand(i));
-    } else if (RetOpcode == X86::TCRETURNri64) {
-      BuildMI(MBB, MBBI, DL,
-              TII.get(IsWin64 ? X86::TAILJMPr64_REX : X86::TAILJMPr64))
-          .addReg(JumpTarget.getReg(), RegState::Kill);
-    } else {
-      BuildMI(MBB, MBBI, DL, TII.get(X86::TAILJMPr)).
-        addReg(JumpTarget.getReg(), RegState::Kill);
-    }
-
-    MachineInstr *NewMI = std::prev(MBBI);
-    NewMI->copyImplicitOps(MF, MBBI);
-
-    // Delete the pseudo instruction TCRETURN.
-    MBB.erase(MBBI);
-  } else if ((RetOpcode == X86::RETQ || RetOpcode == X86::RETL ||
-              RetOpcode == X86::RETIQ || RetOpcode == X86::RETIL) &&
-             (X86FI->getTCReturnAddrDelta() < 0)) {
-    // Add the return addr area delta back since we are not tail calling.
-    int delta = -1*X86FI->getTCReturnAddrDelta();
-    MBBI = MBB.getLastNonDebugInstr();
+  // Add the return addr area delta back since we are not tail calling.
+  int Offset = -1 * X86FI->getTCReturnAddrDelta();
+  assert(Offset >= 0 && "TCDelta should never be positive");
+  if (Offset) {
+    MBBI = MBB.getFirstTerminator();
 
     // Check for possible merge with preceding ADD instruction.
-    delta += mergeSPUpdates(MBB, MBBI, StackPtr, true);
-    emitSPUpdate(MBB, MBBI, StackPtr, delta, Is64Bit, Uses64BitFramePtr,
-                 UseLEAForSP, TII, *RegInfo);
+    Offset += mergeSPUpdates(MBB, MBBI, true);
+    emitSPUpdate(MBB, MBBI, Offset, /*InEpilogue=*/true);
   }
 }
 
 int X86FrameLowering::getFrameIndexOffset(const MachineFunction &MF,
                                           int FI) const {
-  const X86RegisterInfo *RegInfo =
-      MF.getSubtarget<X86Subtarget>().getRegisterInfo();
   const MachineFrameInfo *MFI = MF.getFrameInfo();
   // Offset will hold the offset from the stack pointer at function entry to the
   // object.
@@ -1268,12 +1118,11 @@ int X86FrameLowering::getFrameIndexOffset(const MachineFunction &MF,
   const X86MachineFunctionInfo *X86FI = MF.getInfo<X86MachineFunctionInfo>();
   unsigned CSSize = X86FI->getCalleeSavedFrameSize();
   uint64_t StackSize = MFI->getStackSize();
-  unsigned SlotSize = RegInfo->getSlotSize();
   bool HasFP = hasFP(MF);
-  bool IsWinEH = MF.getTarget().getMCAsmInfo()->usesWindowsCFI();
+  bool IsWin64Prologue = MF.getTarget().getMCAsmInfo()->usesWindowsCFI();
   int64_t FPDelta = 0;
 
-  if (IsWinEH) {
+  if (IsWin64Prologue) {
     assert(!MFI->hasCalls() || (StackSize % 16) == 8);
 
     // Calculate required stack adjustment.
@@ -1297,7 +1146,7 @@ int X86FrameLowering::getFrameIndexOffset(const MachineFunction &MF,
   }
 
 
-  if (RegInfo->hasBasePointer(MF)) {
+  if (TRI->hasBasePointer(MF)) {
     assert(HasFP && "VLAs and dynamic stack realign, but no FP?!");
     if (FI < 0) {
       // Skip the saved EBP.
@@ -1306,7 +1155,7 @@ int X86FrameLowering::getFrameIndexOffset(const MachineFunction &MF,
       assert((-(Offset + StackSize)) % MFI->getObjectAlignment(FI) == 0);
       return Offset + StackSize;
     }
-  } else if (RegInfo->needsStackRealignment(MF)) {
+  } else if (TRI->needsStackRealignment(MF)) {
     if (FI < 0) {
       // Skip the saved EBP.
       return Offset + SlotSize + FPDelta;
@@ -1333,17 +1182,15 @@ int X86FrameLowering::getFrameIndexOffset(const MachineFunction &MF,
 
 int X86FrameLowering::getFrameIndexReference(const MachineFunction &MF, int FI,
                                              unsigned &FrameReg) const {
-  const X86RegisterInfo *RegInfo =
-      MF.getSubtarget<X86Subtarget>().getRegisterInfo();
   // We can't calculate offset from frame pointer if the stack is realigned,
   // so enforce usage of stack/base pointer.  The base pointer is used when we
   // have dynamic allocas in addition to dynamic realignment.
-  if (RegInfo->hasBasePointer(MF))
-    FrameReg = RegInfo->getBaseRegister();
-  else if (RegInfo->needsStackRealignment(MF))
-    FrameReg = RegInfo->getStackRegister();
+  if (TRI->hasBasePointer(MF))
+    FrameReg = TRI->getBaseRegister();
+  else if (TRI->needsStackRealignment(MF))
+    FrameReg = TRI->getStackRegister();
   else
-    FrameReg = RegInfo->getFrameRegister(MF);
+    FrameReg = TRI->getFrameRegister(MF);
   return getFrameIndexOffset(MF, FI);
 }
 
@@ -1354,8 +1201,6 @@ int X86FrameLowering::getFrameIndexOffsetFromSP(const MachineFunction &MF, int F
   const uint64_t StackSize = MFI->getStackSize();
   {
 #ifndef NDEBUG
-    const X86RegisterInfo *RegInfo =
-        MF.getSubtarget<X86Subtarget>().getRegisterInfo();
     // Note: LLVM arranges the stack as:
     // Args > Saved RetPC (<--FP) > CSRs > dynamic alignment (<--BP)
     //      > "Stack Slots" (<--SP)
@@ -1367,7 +1212,7 @@ int X86FrameLowering::getFrameIndexOffsetFromSP(const MachineFunction &MF, int F
     // frame).  As a result, THE RESULT OF THIS CALL IS MEANINGLESS FOR CSRs
     // AND FixedObjects IFF needsStackRealignment or hasVarSizedObject.
 
-    assert(!RegInfo->hasBasePointer(MF) && "we don't handle this case");
+    assert(!TRI->hasBasePointer(MF) && "we don't handle this case");
 
     // We don't handle tail calls, and shouldn't be seeing them
     // either.
@@ -1412,11 +1257,9 @@ int X86FrameLowering::getFrameIndexOffsetFromSP(const MachineFunction &MF, int F
 int X86FrameLowering::getFrameIndexReferenceFromSP(const MachineFunction &MF,
                                                    int FI,
                                                    unsigned &FrameReg) const {
-  const X86RegisterInfo *RegInfo =
-      MF.getSubtarget<X86Subtarget>().getRegisterInfo();
-  assert(!RegInfo->hasBasePointer(MF) && "we don't handle this case");
+  assert(!TRI->hasBasePointer(MF) && "we don't handle this case");
 
-  FrameReg = RegInfo->getStackRegister();
+  FrameReg = TRI->getStackRegister();
   return getFrameIndexOffsetFromSP(MF, FI);
 }
 
@@ -1424,9 +1267,6 @@ bool X86FrameLowering::assignCalleeSavedSpillSlots(
     MachineFunction &MF, const TargetRegisterInfo *TRI,
     std::vector<CalleeSavedInfo> &CSI) const {
   MachineFrameInfo *MFI = MF.getFrameInfo();
-  const X86RegisterInfo *RegInfo =
-      MF.getSubtarget<X86Subtarget>().getRegisterInfo();
-  unsigned SlotSize = RegInfo->getSlotSize();
   X86MachineFunctionInfo *X86FI = MF.getInfo<X86MachineFunctionInfo>();
 
   unsigned CalleeSavedFrameSize = 0;
@@ -1440,7 +1280,7 @@ bool X86FrameLowering::assignCalleeSavedSpillSlots(
     // Since emitPrologue and emitEpilogue will handle spilling and restoring of
     // the frame register, we can delete it from CSI list and not have to worry
     // about avoiding it later.
-    unsigned FPReg = RegInfo->getFrameRegister(MF);
+    unsigned FPReg = TRI->getFrameRegister(MF);
     for (unsigned i = 0; i < CSI.size(); ++i) {
       if (TRI->regsOverlap(CSI[i].getReg(),FPReg)) {
         CSI.erase(CSI.begin() + i);
@@ -1471,7 +1311,7 @@ bool X86FrameLowering::assignCalleeSavedSpillSlots(
     if (X86::GR64RegClass.contains(Reg) || X86::GR32RegClass.contains(Reg))
       continue;
 
-    const TargetRegisterClass *RC = RegInfo->getMinimalPhysRegClass(Reg);
+    const TargetRegisterClass *RC = TRI->getMinimalPhysRegClass(Reg);
     // ensure alignment
     SpillSlotOffset -= std::abs(SpillSlotOffset) % RC->getAlignment();
     // spill into slot
@@ -1491,10 +1331,6 @@ bool X86FrameLowering::spillCalleeSavedRegisters(
     const TargetRegisterInfo *TRI) const {
   DebugLoc DL = MBB.findDebugLoc(MI);
 
-  MachineFunction &MF = *MBB.getParent();
-  const X86Subtarget &STI = MF.getSubtarget<X86Subtarget>();
-  const TargetInstrInfo &TII = *STI.getInstrInfo();
-
   // Push GPRs. It increases frame size.
   unsigned Opc = STI.is64Bit() ? X86::PUSH64r : X86::PUSH32r;
   for (unsigned i = CSI.size(); i != 0; --i) {
@@ -1538,10 +1374,6 @@ bool X86FrameLowering::restoreCalleeSavedRegisters(MachineBasicBlock &MBB,
 
   DebugLoc DL = MBB.findDebugLoc(MI);
 
-  MachineFunction &MF = *MBB.getParent();
-  const X86Subtarget &STI = MF.getSubtarget<X86Subtarget>();
-  const TargetInstrInfo &TII = *STI.getInstrInfo();
-
   // Reload XMMs from stack frame.
   for (unsigned i = 0, e = CSI.size(); i != e; ++i) {
     unsigned Reg = CSI[i].getReg();
@@ -1566,13 +1398,12 @@ bool X86FrameLowering::restoreCalleeSavedRegisters(MachineBasicBlock &MBB,
   return true;
 }
 
-void
-X86FrameLowering::processFunctionBeforeCalleeSavedScan(MachineFunction &MF,
-                                                       RegScavenger *RS) const {
+void X86FrameLowering::determineCalleeSaves(MachineFunction &MF,
+                                            BitVector &SavedRegs,
+                                            RegScavenger *RS) const {
+  TargetFrameLowering::determineCalleeSaves(MF, SavedRegs, RS);
+
   MachineFrameInfo *MFI = MF.getFrameInfo();
-  const X86RegisterInfo *RegInfo =
-      MF.getSubtarget<X86Subtarget>().getRegisterInfo();
-  unsigned SlotSize = RegInfo->getSlotSize();
 
   X86MachineFunctionInfo *X86FI = MF.getInfo<X86MachineFunctionInfo>();
   int64_t TailCallReturnAddrDelta = X86FI->getTCReturnAddrDelta();
@@ -1592,8 +1423,8 @@ X86FrameLowering::processFunctionBeforeCalleeSavedScan(MachineFunction &MF,
   }
 
   // Spill the BasePtr if it's used.
-  if (RegInfo->hasBasePointer(MF))
-    MF.getRegInfo().setPhysRegUsed(RegInfo->getBaseRegister());
+  if (TRI->hasBasePointer(MF))
+    SavedRegs.set(TRI->getBaseRegister());
 }
 
 static bool
@@ -1648,15 +1479,10 @@ GetScratchRegister(bool Is64Bit, bool IsLP64, const MachineFunction &MF, bool Pr
 // limit.
 static const uint64_t kSplitStackAvailable = 256;
 
-void
-X86FrameLowering::adjustForSegmentedStacks(MachineFunction &MF) const {
-  MachineBasicBlock &prologueMBB = MF.front();
+void X86FrameLowering::adjustForSegmentedStacks(
+    MachineFunction &MF, MachineBasicBlock &PrologueMBB) const {
   MachineFrameInfo *MFI = MF.getFrameInfo();
-  const X86Subtarget &STI = MF.getSubtarget<X86Subtarget>();
-  const TargetInstrInfo &TII = *STI.getInstrInfo();
   uint64_t StackSize;
-  bool Is64Bit = STI.is64Bit();
-  const bool IsLP64 = STI.isTarget64BitLP64();
   unsigned TlsReg, TlsOffset;
   DebugLoc DL;
 
@@ -1692,8 +1518,9 @@ X86FrameLowering::adjustForSegmentedStacks(MachineFunction &MF) const {
   // The MOV R10, RAX needs to be in a different block, since the RET we emit in
   // allocMBB needs to be last (terminating) instruction.
 
-  for (MachineBasicBlock::livein_iterator i = prologueMBB.livein_begin(),
-         e = prologueMBB.livein_end(); i != e; i++) {
+  for (MachineBasicBlock::livein_iterator i = PrologueMBB.livein_begin(),
+                                          e = PrologueMBB.livein_end();
+       i != e; i++) {
     allocMBB->addLiveIn(*i);
     checkMBB->addLiveIn(*i);
   }
@@ -1807,7 +1634,7 @@ X86FrameLowering::adjustForSegmentedStacks(MachineFunction &MF) const {
 
   // This jump is taken if SP >= (Stacklet Limit + Stack Space required).
   // It jumps to normal execution of the function body.
-  BuildMI(checkMBB, DL, TII.get(X86::JA_1)).addMBB(&prologueMBB);
+  BuildMI(checkMBB, DL, TII.get(X86::JA_1)).addMBB(&PrologueMBB);
 
   // On 32 bit we first push the arguments size and then the frame size. On 64
   // bit, we pass the stack frame size in r10 and the argument size in r11.
@@ -1828,8 +1655,6 @@ X86FrameLowering::adjustForSegmentedStacks(MachineFunction &MF) const {
       .addImm(StackSize);
     BuildMI(allocMBB, DL, TII.get(MOVri), Reg11)
       .addImm(X86FI->getArgumentStackSize());
-    MF.getRegInfo().setPhysRegUsed(Reg10);
-    MF.getRegInfo().setPhysRegUsed(Reg11);
   } else {
     BuildMI(allocMBB, DL, TII.get(X86::PUSHi32))
       .addImm(X86FI->getArgumentStackSize());
@@ -1874,10 +1699,10 @@ X86FrameLowering::adjustForSegmentedStacks(MachineFunction &MF) const {
   else
     BuildMI(allocMBB, DL, TII.get(X86::MORESTACK_RET));
 
-  allocMBB->addSuccessor(&prologueMBB);
+  allocMBB->addSuccessor(&PrologueMBB);
 
   checkMBB->addSuccessor(allocMBB);
-  checkMBB->addSuccessor(&prologueMBB);
+  checkMBB->addSuccessor(&PrologueMBB);
 
 #ifdef XDEBUG
   MF.verify();
@@ -1899,13 +1724,9 @@ X86FrameLowering::adjustForSegmentedStacks(MachineFunction &MF) const {
 ///       call inc_stack   # doubles the stack space
 ///       temp0 = sp - MaxStack
 ///       if( temp0 < SP_LIMIT(P) ) goto IncStack else goto OldStart
-void X86FrameLowering::adjustForHiPEPrologue(MachineFunction &MF) const {
-  const X86Subtarget &STI = MF.getSubtarget<X86Subtarget>();
-  const TargetInstrInfo &TII = *STI.getInstrInfo();
+void X86FrameLowering::adjustForHiPEPrologue(
+    MachineFunction &MF, MachineBasicBlock &PrologueMBB) const {
   MachineFrameInfo *MFI = MF.getFrameInfo();
-  const unsigned SlotSize = STI.getRegisterInfo()->getSlotSize();
-  const bool Is64Bit = STI.is64Bit();
-  const bool IsLP64 = STI.isTarget64BitLP64();
   DebugLoc DL;
   // HiPE-specific values
   const unsigned HipeLeafWords = 24;
@@ -1968,12 +1789,12 @@ void X86FrameLowering::adjustForHiPEPrologue(MachineFunction &MF) const {
   // If the stack frame needed is larger than the guaranteed then runtime checks
   // and calls to "inc_stack_0" BIF should be inserted in the assembly prologue.
   if (MaxStack > Guaranteed) {
-    MachineBasicBlock &prologueMBB = MF.front();
     MachineBasicBlock *stackCheckMBB = MF.CreateMachineBasicBlock();
     MachineBasicBlock *incStackMBB = MF.CreateMachineBasicBlock();
 
-    for (MachineBasicBlock::livein_iterator I = prologueMBB.livein_begin(),
-           E = prologueMBB.livein_end(); I != E; I++) {
+    for (MachineBasicBlock::livein_iterator I = PrologueMBB.livein_begin(),
+                                            E = PrologueMBB.livein_end();
+         I != E; I++) {
       stackCheckMBB->addLiveIn(*I);
       incStackMBB->addLiveIn(*I);
     }
@@ -2009,7 +1830,7 @@ void X86FrameLowering::adjustForHiPEPrologue(MachineFunction &MF) const {
     // SPLimitOffset is in a fixed heap location (pointed by BP).
     addRegOffset(BuildMI(stackCheckMBB, DL, TII.get(CMPop))
                  .addReg(ScratchReg), PReg, false, SPLimitOffset);
-    BuildMI(stackCheckMBB, DL, TII.get(X86::JAE_1)).addMBB(&prologueMBB);
+    BuildMI(stackCheckMBB, DL, TII.get(X86::JAE_1)).addMBB(&PrologueMBB);
 
     // Create new MBB for IncStack:
     BuildMI(incStackMBB, DL, TII.get(CALLop)).
@@ -2020,9 +1841,9 @@ void X86FrameLowering::adjustForHiPEPrologue(MachineFunction &MF) const {
                  .addReg(ScratchReg), PReg, false, SPLimitOffset);
     BuildMI(incStackMBB, DL, TII.get(X86::JLE_1)).addMBB(incStackMBB);
 
-    stackCheckMBB->addSuccessor(&prologueMBB, 99);
+    stackCheckMBB->addSuccessor(&PrologueMBB, 99);
     stackCheckMBB->addSuccessor(incStackMBB, 1);
-    incStackMBB->addSuccessor(&prologueMBB, 99);
+    incStackMBB->addSuccessor(&PrologueMBB, 99);
     incStackMBB->addSuccessor(incStackMBB, 1);
   }
 #ifdef XDEBUG
@@ -2033,18 +1854,9 @@ void X86FrameLowering::adjustForHiPEPrologue(MachineFunction &MF) const {
 void X86FrameLowering::
 eliminateCallFramePseudoInstr(MachineFunction &MF, MachineBasicBlock &MBB,
                               MachineBasicBlock::iterator I) const {
-  const X86Subtarget &STI = MF.getSubtarget<X86Subtarget>();
-  const TargetInstrInfo &TII = *STI.getInstrInfo();
-  const X86RegisterInfo &RegInfo = *STI.getRegisterInfo();
-  unsigned StackPtr = RegInfo.getStackRegister();
   bool reserveCallFrame = hasReservedCallFrame(MF);
-  int Opcode = I->getOpcode();
+  unsigned Opcode = I->getOpcode();
   bool isDestroy = Opcode == TII.getCallFrameDestroyOpcode();
-  // @LOCALMOD-BEGIN
-  // standard x86_64 and NaCl use 64-bit frame/stack pointers, x32 - 32-bit.
-  const bool Uses64BitStackPtr =
-      STI.isTarget64BitLP64() || STI.isTargetNaCl64();
-  // @LOCALMOD-END
   DebugLoc DL = I->getDebugLoc();
   uint64_t Amount = !reserveCallFrame ? I->getOperand(0).getImm() : 0;
   uint64_t InternalAmt = (isDestroy || Amount) ? I->getOperand(1).getImm() : 0;
@@ -2063,54 +1875,41 @@ eliminateCallFramePseudoInstr(MachineFunction &MF, MachineBasicBlock &MBB,
     unsigned StackAlign = getStackAlignment();
     Amount = RoundUpToAlignment(Amount, StackAlign);
 
-    MachineInstr *New = nullptr;
-
     // Factor out the amount that gets handled inside the sequence
     // (Pushes of argument for frame setup, callee pops for frame destroy)
     Amount -= InternalAmt;
 
     if (Amount) {
-      if (Opcode == TII.getCallFrameSetupOpcode()) {
-        New = BuildMI(MF, DL, TII.get(getSUBriOpcode(Uses64BitStackPtr, Amount)), StackPtr)
-          .addReg(StackPtr).addImm(Amount);
-      } else {
-        assert(Opcode == TII.getCallFrameDestroyOpcode());
-
-        unsigned Opc = getADDriOpcode(Uses64BitStackPtr, Amount);
-        New = BuildMI(MF, DL, TII.get(Opc), StackPtr)
-          .addReg(StackPtr).addImm(Amount);
-      }
-    }
-
-    if (New) {
-      // The EFLAGS implicit def is dead.
-      New->getOperand(3).setIsDead();
-
-      // Replace the pseudo instruction with a new instruction.
-      MBB.insert(I, New);
+      // Add Amount to SP to destroy a frame, and subtract to setup.
+      int Offset = isDestroy ? Amount : -Amount;
+      BuildStackAdjustment(MBB, I, DL, Offset, /*InEpilogue=*/false);
     }
-
     return;
   }
 
-  if (Opcode == TII.getCallFrameDestroyOpcode() && InternalAmt) {
+  if (isDestroy && InternalAmt) {
     // If we are performing frame pointer elimination and if the callee pops
     // something off the stack pointer, add it back.  We do this until we have
     // more advanced stack pointer tracking ability.
-    // @LOCALMOD
-    unsigned Opc = getSUBriOpcode(Uses64BitStackPtr, InternalAmt); //MERGETODO(dschuff): is this correct, or should it be IsLP64?
-    MachineInstr *New = BuildMI(MF, DL, TII.get(Opc), StackPtr)
-      .addReg(StackPtr).addImm(InternalAmt);
-
-    // The EFLAGS implicit def is dead.
-    New->getOperand(3).setIsDead();
-
     // We are not tracking the stack pointer adjustment by the callee, so make
     // sure we restore the stack pointer immediately after the call, there may
     // be spill code inserted between the CALL and ADJCALLSTACKUP instructions.
     MachineBasicBlock::iterator B = MBB.begin();
     while (I != B && !std::prev(I)->isCall())
       --I;
-    MBB.insert(I, New);
+    BuildStackAdjustment(MBB, I, DL, -InternalAmt, /*InEpilogue=*/false);
   }
 }
+
+bool X86FrameLowering::canUseAsEpilogue(const MachineBasicBlock &MBB) const {
+  assert(MBB.getParent() && "Block is not attached to a function!");
+
+  if (canUseLEAForSPInEpilogue(*MBB.getParent()))
+    return true;
+
+  // If we cannot use LEA to adjust SP, we may need to use ADD, which
+  // clobbers the EFLAGS. Check that none of the terminators reads the
+  // EFLAGS, and if one uses it, conservatively assume this is not
+  // safe to insert the epilogue here.
+  return !terminatorsNeedFlagsAsInput(MBB);
+}
diff --git a/lib/Target/X86/X86FrameLowering.h b/lib/Target/X86/X86FrameLowering.h
index 542bbbc79ae..495cfcd1c3f 100644
--- a/lib/Target/X86/X86FrameLowering.h
+++ b/lib/Target/X86/X86FrameLowering.h
@@ -18,16 +18,40 @@
 
 namespace llvm {
 
+class MachineInstrBuilder;
+class MCCFIInstruction;
+class X86Subtarget;
+class X86RegisterInfo;
+
 class X86FrameLowering : public TargetFrameLowering {
 public:
-  explicit X86FrameLowering(StackDirection D, unsigned StackAl, int LAO)
-    : TargetFrameLowering(StackGrowsDown, StackAl, LAO) {}
+  X86FrameLowering(const X86Subtarget &STI, unsigned StackAlignOverride);
+
+  // Cached subtarget predicates.
+
+  const X86Subtarget &STI;
+  const TargetInstrInfo &TII;
+  const X86RegisterInfo *TRI;
+
+  unsigned SlotSize;
+
+  /// Is64Bit implies that x86_64 instructions are available.
+  bool Is64Bit;
+
+  bool IsLP64;
+
+  /// True if the 64-bit frame or stack pointer should be used. True for most
+  /// 64-bit targets with the exception of x32. If this is false, 32-bit
+  /// instruction operands should be used to manipulate StackPtr and FramePtr.
+  bool Uses64BitFramePtr;
+
+  unsigned StackPtr;
 
   /// Emit a call to the target's stack probe function. This is required for all
   /// large stack allocations on Windows. The caller is required to materialize
   /// the number of bytes to probe in RAX/EAX.
-  static void emitStackProbeCall(MachineFunction &MF, MachineBasicBlock &MBB,
-                                 MachineBasicBlock::iterator MBBI, DebugLoc DL);
+  void emitStackProbeCall(MachineFunction &MF, MachineBasicBlock &MBB,
+                          MachineBasicBlock::iterator MBBI, DebugLoc DL) const;
 
   void emitCalleeSavedFrameMoves(MachineBasicBlock &MBB,
                                  MachineBasicBlock::iterator MBBI,
@@ -35,15 +59,17 @@ class X86FrameLowering : public TargetFrameLowering {
 
   /// emitProlog/emitEpilog - These methods insert prolog and epilog code into
   /// the function.
-  void emitPrologue(MachineFunction &MF) const override;
+  void emitPrologue(MachineFunction &MF, MachineBasicBlock &MBB) const override;
   void emitEpilogue(MachineFunction &MF, MachineBasicBlock &MBB) const override;
 
-  void adjustForSegmentedStacks(MachineFunction &MF) const override;
+  void adjustForSegmentedStacks(MachineFunction &MF,
+                                MachineBasicBlock &PrologueMBB) const override;
 
-  void adjustForHiPEPrologue(MachineFunction &MF) const override;
+  void adjustForHiPEPrologue(MachineFunction &MF,
+                             MachineBasicBlock &PrologueMBB) const override;
 
-  void processFunctionBeforeCalleeSavedScan(MachineFunction &MF,
-                                     RegScavenger *RS = nullptr) const override;
+  void determineCalleeSaves(MachineFunction &MF, BitVector &SavedRegs,
+                            RegScavenger *RS = nullptr) const override;
 
   bool
   assignCalleeSavedSpillSlots(MachineFunction &MF,
@@ -77,6 +103,28 @@ class X86FrameLowering : public TargetFrameLowering {
                                  MachineBasicBlock &MBB,
                                  MachineBasicBlock::iterator MI) const override;
 
+  /// Check the instruction before/after the passed instruction. If
+  /// it is an ADD/SUB/LEA instruction it is deleted argument and the
+  /// stack adjustment is returned as a positive value for ADD/LEA and
+  /// a negative for SUB.
+  int mergeSPUpdates(MachineBasicBlock &MBB, MachineBasicBlock::iterator &MBBI,
+                     bool doMergeWithPrevious) const;
+
+  /// Emit a series of instructions to increment / decrement the stack
+  /// pointer by a constant value.
+  void emitSPUpdate(MachineBasicBlock &MBB, MachineBasicBlock::iterator &MBBI,
+                    int64_t NumBytes, bool InEpilogue) const;
+
+  /// Check that LEA can be used on SP in an epilogue sequence for \p MF.
+  bool canUseLEAForSPInEpilogue(const MachineFunction &MF) const;
+
+  /// Check whether or not the given \p MBB can be used as a epilogue
+  /// for the target.
+  /// The epilogue will be inserted before the first terminator of that block.
+  /// This method is used by the shrink-wrapping pass to decide if
+  /// \p MBB will be correctly handled by the target.
+  bool canUseAsEpilogue(const MachineBasicBlock &MBB) const override;
+
 private:
   /// convertArgMovsToPushes - This method tries to convert a call sequence
   /// that uses sub and mov instructions to put the argument onto the stack
@@ -86,6 +134,23 @@ class X86FrameLowering : public TargetFrameLowering {
                               MachineBasicBlock &MBB,
                               MachineBasicBlock::iterator I, 
                               uint64_t Amount) const;
+
+  uint64_t calculateMaxStackAlign(const MachineFunction &MF) const;
+
+  /// Wraps up getting a CFI index and building a MachineInstr for it.
+  void BuildCFI(MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI,
+                DebugLoc DL, MCCFIInstruction CFIInst) const;
+
+  /// Aligns the stack pointer by ANDing it with -MaxAlign.
+  void BuildStackAlignAND(MachineBasicBlock &MBB,
+                          MachineBasicBlock::iterator MBBI, DebugLoc DL,
+                          uint64_t MaxAlign) const;
+
+  /// Adjusts the stack pointer using LEA, SUB, or ADD.
+  MachineInstrBuilder BuildStackAdjustment(MachineBasicBlock &MBB,
+                                           MachineBasicBlock::iterator MBBI,
+                                           DebugLoc DL, int64_t Offset,
+                                           bool InEpilogue) const;
 };
 
 } // End llvm namespace
diff --git a/lib/Target/X86/X86ISelDAGToDAG.cpp b/lib/Target/X86/X86ISelDAGToDAG.cpp
index 21b899dae5a..d5351d25d6e 100644
--- a/lib/Target/X86/X86ISelDAGToDAG.cpp
+++ b/lib/Target/X86/X86ISelDAGToDAG.cpp
@@ -67,38 +67,20 @@ namespace {
     const Constant *CP;
     const BlockAddress *BlockAddr;
     const char *ES;
+    MCSymbol *MCSym;
     int JT;
     unsigned Align;    // CP alignment.
     unsigned char SymbolFlags;  // X86II::MO_*
 
     X86ISelAddressMode()
-      : BaseType(RegBase), Base_FrameIndex(0), Scale(1), IndexReg(), Disp(0),
-        Segment(), GV(nullptr), CP(nullptr), BlockAddr(nullptr), ES(nullptr),
-        JT(-1), Align(0), SymbolFlags(X86II::MO_NO_FLAG) {
-    }
+        : BaseType(RegBase), Base_FrameIndex(0), Scale(1), IndexReg(), Disp(0),
+          Segment(), GV(nullptr), CP(nullptr), BlockAddr(nullptr), ES(nullptr),
+          MCSym(nullptr), JT(-1), Align(0), SymbolFlags(X86II::MO_NO_FLAG) {}
 
     bool hasSymbolicDisplacement() const {
       return GV != nullptr || CP != nullptr || ES != nullptr ||
-             JT != -1 || BlockAddr != nullptr;
-    }
-
-    // @LOCALMOD-BEGIN
-    /// clearSymbolicDisplacement - Remove all sources of symbolic
-    /// constant displacement from the addressing mode.  This is
-    /// needed when the symbolic constant is pulled out of the address
-    /// computation, e.g. when
-    ///   ... DISP(%r15,%rax,1) ...
-    /// is replaced with
-    ///   lea DISP(%rax),%tmp
-    ///   ... (%r15,%tmp,1) ...
-    void clearSymbolicDisplacement() {
-      GV = 0;
-      CP = 0;
-      BlockAddr = 0;
-      ES = 0;
-      JT = -1;
+             MCSym != nullptr || JT != -1 || BlockAddr != nullptr;
     }
-    // @LOCALMOD-END
 
     bool hasBaseOrIndexReg() const {
       return BaseType == FrameIndexBase ||
@@ -152,6 +134,11 @@ namespace {
         dbgs() << ES;
       else
         dbgs() << "nul";
+      dbgs() << " MCSym ";
+      if (MCSym)
+        dbgs() << MCSym;
+      else
+        dbgs() << "nul";
       dbgs() << " JT" << JT << " Align" << Align << '\n';
     }
 #endif
@@ -222,6 +209,9 @@ namespace {
     bool SelectAddr(SDNode *Parent, SDValue N, SDValue &Base,
                     SDValue &Scale, SDValue &Index, SDValue &Disp,
                     SDValue &Segment);
+    bool SelectVectorAddr(SDNode *Parent, SDValue N, SDValue &Base,
+                          SDValue &Scale, SDValue &Index, SDValue &Disp,
+                          SDValue &Segment);
     bool SelectMOV64Imm32(SDValue N, SDValue &Imm);
     bool SelectLEAAddr(SDValue N, SDValue &Base,
                        SDValue &Scale, SDValue &Index, SDValue &Disp,
@@ -237,10 +227,6 @@ namespace {
                              SDValue &Index, SDValue &Disp,
                              SDValue &Segment,
                              SDValue &NodeWithChain);
-    // @LOCALMOD-BEGIN
-    void LegalizeAddressingModeForNaCl(SDValue N, X86ISelAddressMode &AM);
-    // @LOCALMOD-END
-
 
     bool TryFoldLoad(SDNode *P, SDValue N,
                      SDValue &Base, SDValue &Scale,
@@ -255,19 +241,16 @@ namespace {
 
     void EmitSpecialCodeForMain();
 
-    inline void getAddressOperands(X86ISelAddressMode &AM, SDValue &Base,
-                                   SDValue &Scale, SDValue &Index,
-                                   SDValue &Disp, SDValue &Segment) {
-
-      // @LOCALMOD-BEGIN: NaCl64 pointers are 32-bit, but memory operands
-      // are 64-bit (adds r15).
-      EVT MemOpVT = Subtarget->is64Bit() ? MVT::i64 : MVT::i32;
+    inline void getAddressOperands(X86ISelAddressMode &AM, SDLoc DL,
+                                   SDValue &Base, SDValue &Scale,
+                                   SDValue &Index, SDValue &Disp,
+                                   SDValue &Segment) {
       Base = (AM.BaseType == X86ISelAddressMode::FrameIndexBase)
-                 ? CurDAG->getTargetFrameIndex(AM.Base_FrameIndex,
-                                               MemOpVT)
+                 ? CurDAG->getTargetFrameIndex(
+                       AM.Base_FrameIndex,
+                       TLI->getPointerTy(CurDAG->getDataLayout()))
                  : AM.Base_Reg;
-      // @LOCALMOD-END
-      Scale = getI8Imm(AM.Scale);
+      Scale = getI8Imm(AM.Scale, DL);
       Index = AM.IndexReg;
       // These are 32-bit even in 64-bit mode since RIP relative offset
       // is 32-bit.
@@ -281,6 +264,10 @@ namespace {
       else if (AM.ES) {
         assert(!AM.Disp && "Non-zero displacement is ignored with ES.");
         Disp = CurDAG->getTargetExternalSymbol(AM.ES, MVT::i32, AM.SymbolFlags);
+      } else if (AM.MCSym) {
+        assert(!AM.Disp && "Non-zero displacement is ignored with MCSym.");
+        assert(AM.SymbolFlags == 0 && "oo");
+        Disp = CurDAG->getMCSymbol(AM.MCSym, MVT::i32);
       } else if (AM.JT != -1) {
         assert(!AM.Disp && "Non-zero displacement is ignored with JT.");
         Disp = CurDAG->getTargetJumpTable(AM.JT, MVT::i32, AM.SymbolFlags);
@@ -288,7 +275,7 @@ namespace {
         Disp = CurDAG->getTargetBlockAddress(AM.BlockAddr, MVT::i32, AM.Disp,
                                              AM.SymbolFlags);
       else
-        Disp = CurDAG->getTargetConstant(AM.Disp, MVT::i32);
+        Disp = CurDAG->getTargetConstant(AM.Disp, DL, MVT::i32);
 
       if (AM.Segment.getNode())
         Segment = AM.Segment;
@@ -298,14 +285,14 @@ namespace {
 
     /// getI8Imm - Return a target constant with the specified value, of type
     /// i8.
-    inline SDValue getI8Imm(unsigned Imm) {
-      return CurDAG->getTargetConstant(Imm, MVT::i8);
+    inline SDValue getI8Imm(unsigned Imm, SDLoc DL) {
+      return CurDAG->getTargetConstant(Imm, DL, MVT::i8);
     }
 
     /// getI32Imm - Return a target constant with the specified value, of type
     /// i32.
-    inline SDValue getI32Imm(unsigned Imm) {
-      return CurDAG->getTargetConstant(Imm, MVT::i32);
+    inline SDValue getI32Imm(unsigned Imm, SDLoc DL) {
+      return CurDAG->getTargetConstant(Imm, DL, MVT::i32);
     }
 
     /// getGlobalBaseReg - Return an SDNode that returns the value of
@@ -326,20 +313,6 @@ namespace {
       return Subtarget->getInstrInfo();
     }
 
-    // @LOCALMOD-START
-    // Determine when an addressing mode with a base register and an index
-    // register can be selected (or not). The NaCl 64-bit sandbox could end
-    // up adding an r15 base register for memory accesses late (after register
-    // allocation), so having both a base and index register selected
-    // now will make it hard to modify the address and base off of r15 later.
-    // This only applies to memory accesses and not lea (distinguished by
-    // selectingMemOp).
-    bool selectingMemOp;
-    bool RestrictUseOfBaseReg() {
-      return selectingMemOp && Subtarget->isTargetNaCl64();
-    }
-    // @LOCALMOD-END
-
     /// \brief Address-mode matching performs shift-of-and to and-of-shift
     /// reassociation in order to expose more scaled addressing
     /// opportunities.
@@ -496,7 +469,8 @@ void X86DAGToDAGISel::PreprocessISelDAG() {
     SDNode *N = I++;  // Preincrement iterator to avoid invalidation issues.
 
     if (OptLevel != CodeGenOpt::None &&
-        !Subtarget->isTargetNaCl() &&   // @LOCALMOD: We can't fold load/call
+        // Only does this when target favors doesn't favor register indirect
+        // call.
         ((N->getOpcode() == X86ISD::CALL && !Subtarget->callRegIndirect()) ||
          (N->getOpcode() == X86ISD::TC_RETURN &&
           // Only does this if load can be folded into TC_RETURN.
@@ -608,11 +582,12 @@ void X86DAGToDAGISel::PreprocessISelDAG() {
 void X86DAGToDAGISel::EmitSpecialCodeForMain() {
   if (Subtarget->isTargetCygMing()) {
     TargetLowering::ArgListTy Args;
+    auto &DL = CurDAG->getDataLayout();
 
     TargetLowering::CallLoweringInfo CLI(*CurDAG);
     CLI.setChain(CurDAG->getRoot())
         .setCallee(CallingConv::C, Type::getVoidTy(*CurDAG->getContext()),
-                   CurDAG->getExternalSymbol("__main", TLI->getPointerTy()),
+                   CurDAG->getExternalSymbol("__main", TLI->getPointerTy(DL)),
                    std::move(Args), 0);
     const TargetLowering &TLI = CurDAG->getTargetLoweringInfo();
     std::pair<SDValue, SDValue> Result = TLI.LowerCallTo(CLI);
@@ -639,6 +614,9 @@ static bool isDispSafeForFrameIndex(int64_t Val) {
 
 bool X86DAGToDAGISel::FoldOffsetIntoAddress(uint64_t Offset,
                                             X86ISelAddressMode &AM) {
+  // Cannot combine ExternalSymbol displacements with integer offsets.
+  if (Offset != 0 && (AM.ES || AM.MCSym))
+    return true;
   int64_t Val = AM.Disp + Offset;
   CodeModel::Model M = TM.getCodeModel();
   if (Subtarget->is64Bit()) {
@@ -650,18 +628,6 @@ bool X86DAGToDAGISel::FoldOffsetIntoAddress(uint64_t Offset,
     if (AM.BaseType == X86ISelAddressMode::FrameIndexBase &&
         !isDispSafeForFrameIndex(Val))
       return true;
-    // LOCALMOD-BEGIN
-    // Do not fold large offsets into displacements.
-    // Various constant folding and address-mode selections can result in
-    // 32-bit operations (e.g. from GEP) getting folded into the displacement
-    // and often results in a negative value in the index register
-    // (see also LegalizeAddressModeForNaCl)
-    else if (Subtarget->isTargetNaCl64() &&
-             (AM.BaseType == X86ISelAddressMode::RegBase ||
-              AM.BaseType == X86ISelAddressMode::FrameIndexBase) &&
-             (Val > 65535 || Val < -65536) && selectingMemOp)
-      return true;
-    // LOCALMOD-END
   }
   AM.Disp = Val;
   return false;
@@ -671,14 +637,6 @@ bool X86DAGToDAGISel::FoldOffsetIntoAddress(uint64_t Offset,
 bool X86DAGToDAGISel::MatchLoadInAddress(LoadSDNode *N, X86ISelAddressMode &AM){
   SDValue Address = N->getOperand(1);
 
-  // @LOCALMOD-START
-  // Disable this tls access optimization in Native Client, since
-  // gs:0 (or fs:0 on X86-64) does not exactly contain its own address.
-  if (Subtarget->isTargetNaCl()) {
-    return true;
-  }
-  // @LOCALMOD-END
-    
   // load gs:0 -> GS segment register.
   // load fs:0 -> FS segment register.
   //
@@ -743,6 +701,8 @@ bool X86DAGToDAGISel::MatchWrapper(SDValue N, X86ISelAddressMode &AM) {
     } else if (ExternalSymbolSDNode *S = dyn_cast<ExternalSymbolSDNode>(N0)) {
       AM.ES = S->getSymbol();
       AM.SymbolFlags = S->getTargetFlags();
+    } else if (auto *S = dyn_cast<MCSymbolSDNode>(N0)) {
+      AM.MCSym = S->getMCSymbol();
     } else if (JumpTableSDNode *J = dyn_cast<JumpTableSDNode>(N0)) {
       AM.JT = J->getIndex();
       AM.SymbolFlags = J->getTargetFlags();
@@ -781,6 +741,8 @@ bool X86DAGToDAGISel::MatchWrapper(SDValue N, X86ISelAddressMode &AM) {
     } else if (ExternalSymbolSDNode *S = dyn_cast<ExternalSymbolSDNode>(N0)) {
       AM.ES = S->getSymbol();
       AM.SymbolFlags = S->getTargetFlags();
+    } else if (auto *S = dyn_cast<MCSymbolSDNode>(N0)) {
+      AM.MCSym = S->getMCSymbol();
     } else if (JumpTableSDNode *J = dyn_cast<JumpTableSDNode>(N0)) {
       AM.JT = J->getIndex();
       AM.SymbolFlags = J->getTargetFlags();
@@ -803,8 +765,6 @@ bool X86DAGToDAGISel::MatchAddress(SDValue N, X86ISelAddressMode &AM) {
   if (MatchAddressRecursively(N, AM, 0))
     return true;
 
-
-  if (!RestrictUseOfBaseReg()) {   // @LOCALMOD
   // Post-processing: Convert lea(,%reg,2) to lea(%reg,%reg), which has
   // a smaller encoding and avoids a scaled-index.
   if (AM.Scale == 2 &&
@@ -813,8 +773,7 @@ bool X86DAGToDAGISel::MatchAddress(SDValue N, X86ISelAddressMode &AM) {
     AM.Base_Reg = AM.IndexReg;
     AM.Scale = 1;
   }
-  } // @LOCALMOD
-  
+
   // Post-processing: Convert foo to foo(%rip), even in non-PIC mode,
   // because it has a smaller encoding.
   // TODO: Which other code models can use this?
@@ -864,11 +823,11 @@ static bool FoldMaskAndShiftToExtract(SelectionDAG &DAG, SDValue N,
 
   MVT VT = N.getSimpleValueType();
   SDLoc DL(N);
-  SDValue Eight = DAG.getConstant(8, MVT::i8);
-  SDValue NewMask = DAG.getConstant(0xff, VT);
+  SDValue Eight = DAG.getConstant(8, DL, MVT::i8);
+  SDValue NewMask = DAG.getConstant(0xff, DL, VT);
   SDValue Srl = DAG.getNode(ISD::SRL, DL, VT, X, Eight);
   SDValue And = DAG.getNode(ISD::AND, DL, VT, Srl, NewMask);
-  SDValue ShlCount = DAG.getConstant(ScaleLog, MVT::i8);
+  SDValue ShlCount = DAG.getConstant(ScaleLog, DL, MVT::i8);
   SDValue Shl = DAG.getNode(ISD::SHL, DL, VT, And, ShlCount);
 
   // Insert the new nodes into the topological ordering. We must do this in
@@ -912,7 +871,7 @@ static bool FoldMaskedShiftToScaledMask(SelectionDAG &DAG, SDValue N,
 
   MVT VT = N.getSimpleValueType();
   SDLoc DL(N);
-  SDValue NewMask = DAG.getConstant(Mask >> ShiftAmt, VT);
+  SDValue NewMask = DAG.getConstant(Mask >> ShiftAmt, DL, VT);
   SDValue NewAnd = DAG.getNode(ISD::AND, DL, VT, X, NewMask);
   SDValue NewShift = DAG.getNode(ISD::SHL, DL, VT, NewAnd, Shift.getOperand(1));
 
@@ -1018,9 +977,9 @@ static bool FoldMaskAndShiftToScale(SelectionDAG &DAG, SDValue N,
     X = NewX;
   }
   SDLoc DL(N);
-  SDValue NewSRLAmt = DAG.getConstant(ShiftAmt + AMShiftAmt, MVT::i8);
+  SDValue NewSRLAmt = DAG.getConstant(ShiftAmt + AMShiftAmt, DL, MVT::i8);
   SDValue NewSRL = DAG.getNode(ISD::SRL, DL, VT, X, NewSRLAmt);
-  SDValue NewSHLAmt = DAG.getConstant(AMShiftAmt, MVT::i8);
+  SDValue NewSHLAmt = DAG.getConstant(AMShiftAmt, DL, MVT::i8);
   SDValue NewSHL = DAG.getNode(ISD::SHL, DL, VT, NewSRL, NewSHLAmt);
 
   // Insert the new nodes into the topological ordering. We must do this in
@@ -1057,7 +1016,8 @@ bool X86DAGToDAGISel::MatchAddressRecursively(SDValue N, X86ISelAddressMode &AM,
     // FIXME: JumpTable and ExternalSymbol address currently don't like
     // displacements.  It isn't very important, but this should be fixed for
     // consistency.
-    if (!AM.ES && AM.JT != -1) return true;
+    if (!(AM.ES || AM.MCSym) && AM.JT != -1)
+      return true;
 
     if (ConstantSDNode *Cst = dyn_cast<ConstantSDNode>(N))
       if (!FoldOffsetIntoAddress(Cst->getSExtValue(), AM))
@@ -1067,13 +1027,13 @@ bool X86DAGToDAGISel::MatchAddressRecursively(SDValue N, X86ISelAddressMode &AM,
 
   switch (N.getOpcode()) {
   default: break;
-  case ISD::FRAME_ALLOC_RECOVER: {
-    if (!AM.hasSymbolicDisplacement())
-      if (const auto *ESNode = dyn_cast<ExternalSymbolSDNode>(N.getOperand(0)))
-        if (ESNode->getOpcode() == ISD::TargetExternalSymbol) {
-          AM.ES = ESNode->getSymbol();
-          return false;
-        }
+  case ISD::LOCAL_RECOVER: {
+    if (!AM.hasSymbolicDisplacement() && AM.Disp == 0)
+      if (const auto *ESNode = dyn_cast<MCSymbolSDNode>(N.getOperand(0))) {
+        // Use the symbol and don't prefix it.
+        AM.MCSym = ESNode->getMCSymbol();
+        return false;
+      }
     break;
   }
   case ISD::Constant: {
@@ -1105,8 +1065,6 @@ bool X86DAGToDAGISel::MatchAddressRecursively(SDValue N, X86ISelAddressMode &AM,
     break;
 
   case ISD::SHL:
-    if (Subtarget->isTargetNaCl64() && selectingMemOp) // @LOCALMOD
-        break;
     if (AM.IndexReg.getNode() != nullptr || AM.Scale != 1)
       break;
 
@@ -1173,8 +1131,6 @@ bool X86DAGToDAGISel::MatchAddressRecursively(SDValue N, X86ISelAddressMode &AM,
     // FALL THROUGH
   case ISD::MUL:
   case X86ISD::MUL_IMM:
-    // @LOCALMOD
-    if (!RestrictUseOfBaseReg()) {
     // X*[3,5,9] -> X+X*[2,4,8]
     if (AM.BaseType == X86ISelAddressMode::RegBase &&
         AM.Base_Reg.getNode() == nullptr &&
@@ -1207,7 +1163,6 @@ bool X86DAGToDAGISel::MatchAddressRecursively(SDValue N, X86ISelAddressMode &AM,
           return false;
         }
     }
-    } // @LOCALMOD
     break;
 
   case ISD::SUB: {
@@ -1266,7 +1221,7 @@ bool X86DAGToDAGISel::MatchAddressRecursively(SDValue N, X86ISelAddressMode &AM,
     }
 
     // Ok, the transformation is legal and appears profitable. Go for it.
-    SDValue Zero = CurDAG->getConstant(0, N.getValueType());
+    SDValue Zero = CurDAG->getConstant(0, dl, N.getValueType());
     SDValue Neg = CurDAG->getNode(ISD::SUB, dl, N.getValueType(), Zero, RHS);
     AM.IndexReg = Neg;
     AM.Scale = 1;
@@ -1294,7 +1249,6 @@ bool X86DAGToDAGISel::MatchAddressRecursively(SDValue N, X86ISelAddressMode &AM,
       return false;
     AM = Backup;
 
-    if (!RestrictUseOfBaseReg()) { // @LOCALMOD
     // If we couldn't fold both operands into the address at the same time,
     // see if we can just put each operand into a register and fold at least
     // the add.
@@ -1307,7 +1261,6 @@ bool X86DAGToDAGISel::MatchAddressRecursively(SDValue N, X86ISelAddressMode &AM,
       AM.Scale = 1;
       return false;
     }
-    } // @LOCALMOD
     N = Handle.getValue();
     break;
   }
@@ -1367,15 +1320,7 @@ bool X86DAGToDAGISel::MatchAddressRecursively(SDValue N, X86ISelAddressMode &AM,
 /// MatchAddressBase - Helper for MatchAddress. Add the specified node to the
 /// specified addressing mode without any further recursion.
 bool X86DAGToDAGISel::MatchAddressBase(SDValue N, X86ISelAddressMode &AM) {
-  if (RestrictUseOfBaseReg()) { // @LOCALMOD
-    if (AM.IndexReg.getNode() == 0) {
-      AM.IndexReg = N;
-      AM.Scale = 1;
-      return false;
-    }
-    return true;
-  } // @LOCALMOD
-// Is the base register already occupied?
+  // Is the base register already occupied?
   if (AM.BaseType != X86ISelAddressMode::RegBase || AM.Base_Reg.getNode()) {
     // If so, check to see if the scale index register is set.
     if (!AM.IndexReg.getNode()) {
@@ -1394,6 +1339,42 @@ bool X86DAGToDAGISel::MatchAddressBase(SDValue N, X86ISelAddressMode &AM) {
   return false;
 }
 
+bool X86DAGToDAGISel::SelectVectorAddr(SDNode *Parent, SDValue N, SDValue &Base,
+                                      SDValue &Scale, SDValue &Index,
+                                      SDValue &Disp, SDValue &Segment) {
+
+  MaskedGatherScatterSDNode *Mgs = dyn_cast<MaskedGatherScatterSDNode>(Parent);
+  if (!Mgs)
+    return false;
+  X86ISelAddressMode AM;
+  unsigned AddrSpace = Mgs->getPointerInfo().getAddrSpace();
+  // AddrSpace 256 -> GS, 257 -> FS.
+  if (AddrSpace == 256)
+    AM.Segment = CurDAG->getRegister(X86::GS, MVT::i16);
+  if (AddrSpace == 257)
+    AM.Segment = CurDAG->getRegister(X86::FS, MVT::i16);
+
+  SDLoc DL(N);
+  Base = Mgs->getBasePtr();
+  Index = Mgs->getIndex();
+  unsigned ScalarSize = Mgs->getValue().getValueType().getScalarSizeInBits();
+  Scale = getI8Imm(ScalarSize/8, DL);
+
+  // If Base is 0, the whole address is in index and the Scale is 1
+  if (isa<ConstantSDNode>(Base)) {
+    assert(dyn_cast<ConstantSDNode>(Base)->isNullValue() &&
+           "Unexpected base in gather/scatter");
+    Scale = getI8Imm(1, DL);
+    Base = CurDAG->getRegister(0, MVT::i32);
+  }
+  if (AM.Segment.getNode())
+    Segment = AM.Segment;
+  else
+    Segment = CurDAG->getRegister(0, MVT::i32);
+  Disp = CurDAG->getTargetConstant(0, DL, MVT::i32);
+  return true;
+}
+
 /// SelectAddr - returns true if it is able pattern match an addressing mode.
 /// It returns the operands which make up the maximal addressing mode it can
 /// match by reference.
@@ -1405,8 +1386,6 @@ bool X86DAGToDAGISel::SelectAddr(SDNode *Parent, SDValue N, SDValue &Base,
                                  SDValue &Scale, SDValue &Index,
                                  SDValue &Disp, SDValue &Segment) {
   X86ISelAddressMode AM;
-  // @LOCALMOD
-  selectingMemOp = true;
 
   if (Parent &&
       // This list of opcodes are all the nodes that have an "addr:$ptr" operand
@@ -1428,14 +1407,7 @@ bool X86DAGToDAGISel::SelectAddr(SDNode *Parent, SDValue N, SDValue &Base,
   if (MatchAddress(N, AM))
     return false;
 
-  // @LOCALMOD-START
-  if (Subtarget->isTargetNaCl64()) {
-      LegalizeAddressingModeForNaCl(N, AM);
-  }
-  // @LOCALMOD-END
-
-  MVT VT = Subtarget->is64Bit() ? MVT::i64 : MVT::i32; // @LOCALMOD
-
+  MVT VT = N.getSimpleValueType();
   if (AM.BaseType == X86ISelAddressMode::RegBase) {
     if (!AM.Base_Reg.getNode())
       AM.Base_Reg = CurDAG->getRegister(0, VT);
@@ -1444,28 +1416,7 @@ bool X86DAGToDAGISel::SelectAddr(SDNode *Parent, SDValue N, SDValue &Base,
   if (!AM.IndexReg.getNode())
     AM.IndexReg = CurDAG->getRegister(0, VT);
 
-  getAddressOperands(AM, Base, Scale, Index, Disp, Segment);
-
-  // @LOCALMOD-BEGIN
-  // For Native Client 64-bit, zero-extend 32-bit pointers
-  // to 64-bits for memory operations.  Most of the time, this
-  // won't generate any additional instructions because the backend
-  // knows that operations on 32-bit registers implicitly zero-extends.
-  // If we don't do this, there are a few corner cases where LLVM might
-  // assume the upper bits won't be modified or used, but since we
-  // always clear the upper bits, this is not a good assumption.
-  // http://code.google.com/p/nativeclient/issues/detail?id=1564
-  if (Subtarget->isTargetNaCl64()) {
-    assert(Base.getValueType() == MVT::i64 && "Unexpected base operand size");
-
-    if (Index.getValueType() != MVT::i64) {
-      Index = CurDAG->getZExtOrTrunc(Index, SDLoc(Index), MVT::i64);
-      // Insert the new node into the topological ordering.
-      InsertDAGNode(*CurDAG, Parent ? SDValue(Parent, 0) : N, Index);
-    }
-  }
-  // @LOCALMOD-END
-
+  getAddressOperands(AM, SDLoc(N), Base, Scale, Index, Disp, Segment);
   return true;
 }
 
@@ -1521,7 +1472,7 @@ bool X86DAGToDAGISel::SelectMOV64Imm32(SDValue N, SDValue &Imm) {
     if ((uint32_t)ImmVal != (uint64_t)ImmVal)
       return false;
 
-    Imm = CurDAG->getTargetConstant(ImmVal, MVT::i64);
+    Imm = CurDAG->getTargetConstant(ImmVal, SDLoc(N), MVT::i64);
     return true;
   }
 
@@ -1536,6 +1487,7 @@ bool X86DAGToDAGISel::SelectMOV64Imm32(SDValue N, SDValue &Imm) {
       N->getOpcode() != ISD::TargetJumpTable &&
       N->getOpcode() != ISD::TargetGlobalAddress &&
       N->getOpcode() != ISD::TargetExternalSymbol &&
+      N->getOpcode() != ISD::MCSymbol &&
       N->getOpcode() != ISD::TargetBlockAddress)
     return false;
 
@@ -1553,19 +1505,13 @@ bool X86DAGToDAGISel::SelectLEA64_32Addr(SDValue N, SDValue &Base,
   RegisterSDNode *RN = dyn_cast<RegisterSDNode>(Base);
   if (RN && RN->getReg() == 0)
     Base = CurDAG->getRegister(0, MVT::i64);
-  // @LOCALMOD-BEGIN: -- the commit in http://reviews.llvm.org/D4929
-  // to address: http://llvm.org/bugs/show_bug.cgi?id=20016
-  // break's NaCl's test/NaCl/X86/nacl64-addrmodes.ll
-  // revert to checking N.
-  //  else if (Base.getValueType() == MVT::i32 && !dyn_cast<FrameIndexSDNode>(Base)) {
-  else if (Base.getValueType() == MVT::i32 && !dyn_cast<FrameIndexSDNode>(N)) {
-    // @LOCALMOD-END
+  else if (Base.getValueType() == MVT::i32 && !dyn_cast<FrameIndexSDNode>(Base)) {
     // Base could already be %rip, particularly in the x32 ABI.
     Base = SDValue(CurDAG->getMachineNode(
                        TargetOpcode::SUBREG_TO_REG, DL, MVT::i64,
-                       CurDAG->getTargetConstant(0, MVT::i64),
+                       CurDAG->getTargetConstant(0, DL, MVT::i64),
                        Base,
-                       CurDAG->getTargetConstant(X86::sub_32bit, MVT::i32)),
+                       CurDAG->getTargetConstant(X86::sub_32bit, DL, MVT::i32)),
                    0);
   }
 
@@ -1577,9 +1523,10 @@ bool X86DAGToDAGISel::SelectLEA64_32Addr(SDValue N, SDValue &Base,
            "Expect to be extending 32-bit registers for use in LEA");
     Index = SDValue(CurDAG->getMachineNode(
                         TargetOpcode::SUBREG_TO_REG, DL, MVT::i64,
-                        CurDAG->getTargetConstant(0, MVT::i64),
+                        CurDAG->getTargetConstant(0, DL, MVT::i64),
                         Index,
-                        CurDAG->getTargetConstant(X86::sub_32bit, MVT::i32)),
+                        CurDAG->getTargetConstant(X86::sub_32bit, DL,
+                                                  MVT::i32)),
                     0);
   }
 
@@ -1599,8 +1546,6 @@ bool X86DAGToDAGISel::SelectLEAAddr(SDValue N,
   SDValue Copy = AM.Segment;
   SDValue T = CurDAG->getRegister(0, MVT::i32);
   AM.Segment = T;
-  // @LOCALMOD
-  selectingMemOp = false;
   if (MatchAddress(N, AM))
     return false;
   assert (T == AM.Segment);
@@ -1647,7 +1592,7 @@ bool X86DAGToDAGISel::SelectLEAAddr(SDValue N,
   if (Complexity <= 2)
     return false;
 
-  getAddressOperands(AM, Base, Scale, Index, Disp, Segment);
+  getAddressOperands(AM, SDLoc(N), Base, Scale, Index, Disp, Segment);
   return true;
 }
 
@@ -1664,15 +1609,14 @@ bool X86DAGToDAGISel::SelectTLSADDRAddr(SDValue N, SDValue &Base,
   AM.Base_Reg = CurDAG->getRegister(0, N.getValueType());
   AM.SymbolFlags = GA->getTargetFlags();
 
-  if (N.getValueType() == MVT::i32 && 
-      !Subtarget->isTargetNaCl64()) {   // @LOCALMOD
+  if (N.getValueType() == MVT::i32) {
     AM.Scale = 1;
     AM.IndexReg = CurDAG->getRegister(X86::EBX, MVT::i32);
   } else {
     AM.IndexReg = CurDAG->getRegister(0, MVT::i64);
   }
 
-  getAddressOperands(AM, Base, Scale, Index, Disp, Segment);
+  getAddressOperands(AM, SDLoc(N), Base, Scale, Index, Disp, Segment);
   return true;
 }
 
@@ -1690,178 +1634,14 @@ bool X86DAGToDAGISel::TryFoldLoad(SDNode *P, SDValue N,
                     N.getOperand(1), Base, Scale, Index, Disp, Segment);
 }
 
-// @LOCALMOD-BEGIN
-// LegalizeAddressingModeForNaCl - NaCl specific addressing fixes.  This ensures
-// two addressing mode invariants.
-//
-//   case 1. Addressing using only a displacement (constant address references)
-//   is only legal when the displacement is positive.  This is because, when
-//   later we replace
-//     movl 0xffffffff, %eax
-//   by
-//     movl 0xffffffff(%r15), %eax
-//   the displacement becomes a negative offset from %r15, making this a
-//   reference to the guard region below %r15 rather than to %r15 + 4GB - 1,
-//   as the programmer expected.  To handle these cases we pull negative
-//   displacements out whenever there is no base or index register in the
-//   addressing mode.  I.e., the above becomes
-//     movl $0xffffffff, %ebx
-//     movl %rbx, %rbx
-//     movl (%r15, %rbx, 1), %eax
-//
-//   case 2. Because NaCl needs to zero the top 32-bits of the index, we can't
-//   allow the index register to be negative. However, if we are using a base
-//   frame index, global address or the constant pool, and AM.Disp > 0, then
-//   negative values of "index" may be expected to legally occur.
-//   To avoid this, we fold the displacement (and scale) back into the
-//   index. This results in a LEA before the current instruction.
-//   Unfortunately, this may add a requirement for an additional register.
-//
-//   For example, this sandboxed code is broken if %eax is negative:
-//
-//     movl %eax,%eax
-//     incl -30(%rbp,%rax,4)
-//
-//   Instead, we now generate:
-//     leal -30(%rbp,%rax,4), %tmp
-//     movl %tmp,%tmp
-//     incl (%r15,%tmp,1)
-//
-//  TODO(espindola): This might not be complete since the matcher can select
-//  any dag node to go in the index. This is also not how the rest of the
-//  matcher logic works, if the matcher selects something, it must be
-//  valid and not depend on further patching. A more desirable fix is
-//  probably to update the matching code to avoid assigning a register
-//  to a value that we cannot prove is positive.
-//
-//  Note: Any changes to the testing logic need to be synchronized
-//  with the implementation of isLegalAddressingModeForNaCl() in
-//  X86FastISel.cpp.
-void X86DAGToDAGISel::LegalizeAddressingModeForNaCl(SDValue N,
-                                                    X86ISelAddressMode &AM) {
-
-
-  // RIP-relative addressing is always fine.
-  if (AM.isRIPRelative())
-    return;
-
-  SDLoc dl(N);
-  // Case 1 above:
-  if (!AM.hasBaseOrIndexReg() && !AM.hasSymbolicDisplacement() && AM.Disp < 0) {
-    SDValue Imm = CurDAG->getTargetConstant(AM.Disp, MVT::i32);
-    SDValue MovNode =
-      SDValue(CurDAG->getMachineNode(X86::MOV32ri, dl, MVT::i32, Imm), 0);
-    AM.IndexReg = MovNode;
-    AM.Disp = 0;
-    InsertDAGNode(*CurDAG, N, MovNode);
-    return;
-  }
-
-  // MatchAddress wants to use the base register when there's only
-  // one register and no scale. We need to use the index register instead.
-  if (AM.BaseType == X86ISelAddressMode::RegBase &&
-      AM.Base_Reg.getNode() &&
-      !AM.IndexReg.getNode()) {
-    AM.IndexReg = AM.Base_Reg;
-    AM.setBaseReg(SDValue());
-  }
-
-  // Case 2 above comprises two sub-cases:
-  // sub-case 1: Prevent negative indexes
-
-  // This is relevant only if there is an index register involved.
-  if (!AM.IndexReg.getNode())
-    return;
-
-  // There are two situations to deal with.  The first is as described
-  // above, which is essentially a potentially negative index into an
-  // interior pointer to a stack-allocated structure.  The second is a
-  // potentially negative index into an interior pointer to a global
-  // array.  In this case, the global array will be a symbolic
-  // displacement.  In theory, we could recognize that it is an
-  // interior pointer only when the concrete displacement AM.Disp is
-  // nonzero, but there is at least one test case (aha.c in the LLVM
-  // test suite) that incorrectly uses a negative index to dereference
-  // a global array, so we conservatively apply the translation to all
-  // global dereferences.
-  bool HasSymbolic = AM.hasSymbolicDisplacement();
-  bool NeedsFixing1 = HasSymbolic ||
-    (AM.BaseType == X86ISelAddressMode::FrameIndexBase && AM.Disp > 0);
-
-  // sub-case 2: Both index and base registers are being used.  The
-  // test for index register being used is done above, so we only need
-  // to test for a base register being used.
-  bool NeedsFixing2 =
-    (AM.BaseType == X86ISelAddressMode::RegBase) && AM.Base_Reg.getNode();
-
-  if (!NeedsFixing1 && !NeedsFixing2)
-    return;
-
-  static const unsigned LogTable[] = { ~0u, 0, 1, ~0u, 2, ~0u, ~0u, ~0u, 3 };
-  assert(AM.Scale < sizeof(LogTable)/sizeof(LogTable[0]));
-  unsigned ScaleLog = LogTable[AM.Scale];
-  assert(ScaleLog <= 3);
-  SmallVector<SDNode*, 8> NewNodes;
-
-  SDValue NewIndex = AM.IndexReg;
-  if (ScaleLog > 0) {
-    SDValue ShlCount = CurDAG->getConstant(ScaleLog, MVT::i8);
-    NewNodes.push_back(ShlCount.getNode());
-    SDValue ShlNode = CurDAG->getNode(ISD::SHL, dl, N.getValueType(),
-                                      NewIndex, ShlCount);
-    NewNodes.push_back(ShlNode.getNode());
-    NewIndex = ShlNode;
-  }
-  if (AM.Disp > 0 || HasSymbolic) {
-    // If the addressing mode has a potentially problematic
-    // displacement, we pull it out into a new instruction node.  If
-    // it contains a symbolic displacement, we need to express it as a
-    // Wrapper node to make LLVM work.
-    SDValue Base, Scale, Index, Disp, Segment;
-    getAddressOperands(AM, Base, Scale, Index, Disp, Segment);
-    SDValue DispNode;
-    if (HasSymbolic)
-      DispNode = CurDAG->getNode(X86ISD::Wrapper, dl, N.getValueType(), Disp);
-    else
-      DispNode = CurDAG->getConstant(AM.Disp, N.getValueType());
-    NewNodes.push_back(DispNode.getNode());
-
-    SDValue AddNode = CurDAG->getNode(ISD::ADD, dl, N.getValueType(),
-                                  NewIndex, DispNode);
-    NewNodes.push_back(AddNode.getNode());
-    NewIndex = AddNode;
-  }
-
-  if (NeedsFixing2) {
-    SDValue AddBase = CurDAG->getNode(ISD::ADD, dl, N.getValueType(),
-                                      NewIndex, AM.Base_Reg);
-    NewNodes.push_back(AddBase.getNode());
-    NewIndex = AddBase;
-    AM.setBaseReg(SDValue());
-  }
-  AM.Disp = 0;
-  AM.clearSymbolicDisplacement();
-  AM.Scale = 1;
-  AM.IndexReg = NewIndex;
-
-  // Insert the new nodes into the topological ordering.
-  for (unsigned i=0; i < NewNodes.size(); i++) {
-    if (NewNodes[i]->getNodeId() == -1 ||
-        NewNodes[i]->getNodeId() > N.getNode()->getNodeId()) {
-      CurDAG->RepositionNode(N.getNode(), NewNodes[i]);
-      NewNodes[i]->setNodeId(N.getNode()->getNodeId());
-    }
-  }
-}
-// @LOCALMOD-END
-
 /// getGlobalBaseReg - Return an SDNode that returns the value of
 /// the global base register. Output instructions required to
 /// initialize the global base register, if necessary.
 ///
 SDNode *X86DAGToDAGISel::getGlobalBaseReg() {
   unsigned GlobalBaseReg = getInstrInfo()->getGlobalBaseReg(MF);
-  return CurDAG->getRegister(GlobalBaseReg, TLI->getPointerTy()).getNode();
+  auto &DL = MF->getDataLayout();
+  return CurDAG->getRegister(GlobalBaseReg, TLI->getPointerTy(DL)).getNode();
 }
 
 /// Atomic opcode table
@@ -2007,7 +1787,7 @@ static SDValue getAtomicLoadArithTargetConstant(SelectionDAG *CurDAG,
     // an immediate operand to sub. However, it still fits in 32 bits for the
     // add (since it is not negated) so we can return target-constant.
     if (CNVal == INT32_MIN)
-      return CurDAG->getTargetConstant(CNVal, NVT);
+      return CurDAG->getTargetConstant(CNVal, dl, NVT);
     // For atomic-load-add, we could do some optimizations.
     if (Op == ADD) {
       // Translate to INC/DEC if ADD by 1 or -1.
@@ -2022,7 +1802,7 @@ static SDValue getAtomicLoadArithTargetConstant(SelectionDAG *CurDAG,
         CNVal = -CNVal;
       }
     }
-    return CurDAG->getTargetConstant(CNVal, NVT);
+    return CurDAG->getTargetConstant(CNVal, dl, NVT);
   }
 
   // If the value operand is single-used, try to optimize it.
@@ -2335,12 +2115,14 @@ SDNode *X86DAGToDAGISel::SelectGather(SDNode *Node, unsigned Opc) {
   SDVTList VTs = CurDAG->getVTList(VSrc.getValueType(), VSrc.getValueType(),
                                    MVT::Other);
 
+  SDLoc DL(Node);
+
   // Memory Operands: Base, Scale, Index, Disp, Segment
-  SDValue Disp = CurDAG->getTargetConstant(0, MVT::i32);
+  SDValue Disp = CurDAG->getTargetConstant(0, DL, MVT::i32);
   SDValue Segment = CurDAG->getRegister(0, MVT::i32);
-  const SDValue Ops[] = { VSrc, Base, getI8Imm(Scale->getSExtValue()), VIdx,
+  const SDValue Ops[] = { VSrc, Base, getI8Imm(Scale->getSExtValue(), DL), VIdx,
                           Disp, Segment, VMask, Chain};
-  SDNode *ResNode = CurDAG->getMachineNode(Opc, SDLoc(Node), VTs, Ops);
+  SDNode *ResNode = CurDAG->getMachineNode(Opc, DL, VTs, Ops);
   // Node has 2 outputs: VDst and MVT::Other.
   // ResNode has 3 outputs: VDst, VMask_wb, and MVT::Other.
   // We replace VDst of Node with VDst of ResNode, and Other of Node with Other
@@ -2514,13 +2296,13 @@ SDNode *X86DAGToDAGISel::Select(SDNode *Node) {
     }
 
     // Emit the smaller op and the shift.
-    SDValue NewCst = CurDAG->getTargetConstant(Val >> ShlVal, CstVT);
+    SDValue NewCst = CurDAG->getTargetConstant(Val >> ShlVal, dl, CstVT);
     SDNode *New = CurDAG->getMachineNode(Op, dl, NVT, N0->getOperand(0),NewCst);
     if (ShlVal == 1)
       return CurDAG->SelectNodeTo(Node, AddOp, NVT, SDValue(New, 0),
                                   SDValue(New, 0));
     return CurDAG->SelectNodeTo(Node, ShlOp, NVT, SDValue(New, 0),
-                                getI8Imm(ShlVal));
+                                getI8Imm(ShlVal, dl));
   }
   case X86ISD::UMUL8:
   case X86ISD::SMUL8: {
@@ -2684,7 +2466,8 @@ SDNode *X86DAGToDAGISel::Select(SDNode *Node) {
       // Shift AX down 8 bits.
       Result = SDValue(CurDAG->getMachineNode(X86::SHR16ri, dl, MVT::i16,
                                               Result,
-                                     CurDAG->getTargetConstant(8, MVT::i8)), 0);
+                                     CurDAG->getTargetConstant(8, dl, MVT::i8)),
+                       0);
       // Then truncate it down to i8.
       ReplaceUses(SDValue(Node, 1),
         CurDAG->getTargetExtractSubreg(X86::sub_8bit, dl, MVT::i8, Result));
@@ -2804,7 +2587,8 @@ SDNode *X86DAGToDAGISel::Select(SDNode *Node) {
           ClrNode =
               SDValue(CurDAG->getMachineNode(
                           TargetOpcode::EXTRACT_SUBREG, dl, MVT::i16, ClrNode,
-                          CurDAG->getTargetConstant(X86::sub_16bit, MVT::i32)),
+                          CurDAG->getTargetConstant(X86::sub_16bit, dl,
+                                                    MVT::i32)),
                       0);
           break;
         case MVT::i32:
@@ -2813,8 +2597,9 @@ SDNode *X86DAGToDAGISel::Select(SDNode *Node) {
           ClrNode =
               SDValue(CurDAG->getMachineNode(
                           TargetOpcode::SUBREG_TO_REG, dl, MVT::i64,
-                          CurDAG->getTargetConstant(0, MVT::i64), ClrNode,
-                          CurDAG->getTargetConstant(X86::sub_32bit, MVT::i32)),
+                          CurDAG->getTargetConstant(0, dl, MVT::i64), ClrNode,
+                          CurDAG->getTargetConstant(X86::sub_32bit, dl,
+                                                    MVT::i32)),
                       0);
           break;
         default:
@@ -2866,8 +2651,9 @@ SDNode *X86DAGToDAGISel::Select(SDNode *Node) {
           Result =
               SDValue(CurDAG->getMachineNode(
                           TargetOpcode::SUBREG_TO_REG, dl, MVT::i64,
-                          CurDAG->getTargetConstant(0, MVT::i64), Result,
-                          CurDAG->getTargetConstant(X86::sub_32bit, MVT::i32)),
+                          CurDAG->getTargetConstant(0, dl, MVT::i64), Result,
+                          CurDAG->getTargetConstant(X86::sub_32bit, dl,
+                                                    MVT::i32)),
                       0);
         }
       } else {
@@ -2924,7 +2710,7 @@ SDNode *X86DAGToDAGISel::Select(SDNode *Node) {
       if ((C->getZExtValue() & ~UINT64_C(0xff)) == 0 &&
           (!(C->getZExtValue() & 0x80) ||
            HasNoSignedComparisonUses(Node))) {
-        SDValue Imm = CurDAG->getTargetConstant(C->getZExtValue(), MVT::i8);
+        SDValue Imm = CurDAG->getTargetConstant(C->getZExtValue(), dl, MVT::i8);
         SDValue Reg = N0.getNode()->getOperand(0);
 
         // On x86-32, only the ABCD registers have 8-bit subregisters.
@@ -2935,7 +2721,7 @@ SDNode *X86DAGToDAGISel::Select(SDNode *Node) {
           case MVT::i16: TRC = &X86::GR16_ABCDRegClass; break;
           default: llvm_unreachable("Unsupported TEST operand type!");
           }
-          SDValue RC = CurDAG->getTargetConstant(TRC->getID(), MVT::i32);
+          SDValue RC = CurDAG->getTargetConstant(TRC->getID(), dl, MVT::i32);
           Reg = SDValue(CurDAG->getMachineNode(X86::COPY_TO_REGCLASS, dl,
                                                Reg.getValueType(), Reg, RC), 0);
         }
@@ -2960,7 +2746,7 @@ SDNode *X86DAGToDAGISel::Select(SDNode *Node) {
            HasNoSignedComparisonUses(Node))) {
         // Shift the immediate right by 8 bits.
         SDValue ShiftedImm = CurDAG->getTargetConstant(C->getZExtValue() >> 8,
-                                                       MVT::i8);
+                                                       dl, MVT::i8);
         SDValue Reg = N0.getNode()->getOperand(0);
 
         // Put the value in an ABCD register.
@@ -2971,7 +2757,7 @@ SDNode *X86DAGToDAGISel::Select(SDNode *Node) {
         case MVT::i16: TRC = &X86::GR16_ABCDRegClass; break;
         default: llvm_unreachable("Unsupported TEST operand type!");
         }
-        SDValue RC = CurDAG->getTargetConstant(TRC->getID(), MVT::i32);
+        SDValue RC = CurDAG->getTargetConstant(TRC->getID(), dl, MVT::i32);
         Reg = SDValue(CurDAG->getMachineNode(X86::COPY_TO_REGCLASS, dl,
                                              Reg.getValueType(), Reg, RC), 0);
 
@@ -2996,7 +2782,8 @@ SDNode *X86DAGToDAGISel::Select(SDNode *Node) {
           N0.getValueType() != MVT::i16 &&
           (!(C->getZExtValue() & 0x8000) ||
            HasNoSignedComparisonUses(Node))) {
-        SDValue Imm = CurDAG->getTargetConstant(C->getZExtValue(), MVT::i16);
+        SDValue Imm = CurDAG->getTargetConstant(C->getZExtValue(), dl,
+                                                MVT::i16);
         SDValue Reg = N0.getNode()->getOperand(0);
 
         // Extract the 16-bit subregister.
@@ -3018,7 +2805,8 @@ SDNode *X86DAGToDAGISel::Select(SDNode *Node) {
           N0.getValueType() == MVT::i64 &&
           (!(C->getZExtValue() & 0x80000000) ||
            HasNoSignedComparisonUses(Node))) {
-        SDValue Imm = CurDAG->getTargetConstant(C->getZExtValue(), MVT::i32);
+        SDValue Imm = CurDAG->getTargetConstant(C->getZExtValue(), dl,
+                                                MVT::i32);
         SDValue Reg = N0.getNode()->getOperand(0);
 
         // Extract the 32-bit subregister.
@@ -3105,10 +2893,16 @@ SelectInlineAsmMemoryOperand(const SDValue &Op, unsigned ConstraintID,
                              std::vector<SDValue> &OutOps) {
   SDValue Op0, Op1, Op2, Op3, Op4;
   switch (ConstraintID) {
+  default:
+    llvm_unreachable("Unexpected asm memory constraint");
+  case InlineAsm::Constraint_i:
+    // FIXME: It seems strange that 'i' is needed here since it's supposed to
+    //        be an immediate and not a memory constraint.
+    // Fallthrough.
   case InlineAsm::Constraint_o: // offsetable        ??
   case InlineAsm::Constraint_v: // not offsetable    ??
-  default: return true;
   case InlineAsm::Constraint_m: // memory
+  case InlineAsm::Constraint_X:
     if (!SelectAddr(nullptr, Op, Op0, Op1, Op2, Op3, Op4))
       return true;
     break;
diff --git a/lib/Target/X86/X86ISelLowering.cpp b/lib/Target/X86/X86ISelLowering.cpp
index 5415d6127f0..1788edcaf78 100644
--- a/lib/Target/X86/X86ISelLowering.cpp
+++ b/lib/Target/X86/X86ISelLowering.cpp
@@ -18,7 +18,6 @@
 #include "X86FrameLowering.h"
 #include "X86InstrBuilder.h"
 #include "X86MachineFunctionInfo.h"
-#include "X86NaClDecls.h" // @LOCALMOD
 #include "X86TargetMachine.h"
 #include "X86TargetObjectFile.h"
 #include "llvm/ADT/SmallBitVector.h"
@@ -68,26 +67,12 @@ static cl::opt<bool> ExperimentalVectorWideningLegalization(
              "rather than promotion."),
     cl::Hidden);
 
-static cl::opt<int> ReciprocalEstimateRefinementSteps(
-    "x86-recip-refinement-steps", cl::init(1),
-    cl::desc("Specify the number of Newton-Raphson iterations applied to the "
-             "result of the hardware reciprocal estimate instruction."),
-    cl::NotHidden);
-
-// Forward declarations.
-static SDValue getMOVL(SelectionDAG &DAG, SDLoc dl, EVT VT, SDValue V1,
-                       SDValue V2);
-
 X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM,
                                      const X86Subtarget &STI)
     : TargetLowering(TM), Subtarget(&STI) {
   X86ScalarSSEf64 = Subtarget->hasSSE2();
   X86ScalarSSEf32 = Subtarget->hasSSE1();
-  // @LOCALMOD-START
-  X86StackPtr = Subtarget->isTarget64BitLP64() ? X86::RSP : X86::ESP;
-  // @LOCALMOD-END
-
-  TD = getDataLayout();
+  MVT PtrVT = MVT::getIntegerVT(8 * TM.getPointerSize());
 
   // Set up the TargetLowering object.
   static const MVT IntVTs[] = { MVT::i8, MVT::i16, MVT::i32, MVT::i64 };
@@ -107,8 +92,7 @@ X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM,
   else
     setSchedulingPreference(Sched::RegPressure);
   const X86RegisterInfo *RegInfo = Subtarget->getRegisterInfo();
-  (void)RegInfo;                                     // @LOCALMOD
-  setStackPointerRegisterToSaveRestore(X86StackPtr); // @LOCALMOD
+  setStackPointerRegisterToSaveRestore(RegInfo->getStackRegister());
 
   // Bypass expensive divides on Atom when compiling with O2.
   if (TM.getOptLevel() >= CodeGenOpt::Default) {
@@ -189,7 +173,7 @@ X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM,
   if (Subtarget->is64Bit()) {
     setOperationAction(ISD::UINT_TO_FP     , MVT::i32  , Promote);
     setOperationAction(ISD::UINT_TO_FP     , MVT::i64  , Custom);
-  } else if (!TM.Options.UseSoftFloat) {
+  } else if (!Subtarget->useSoftFloat()) {
     // We have an algorithm for SSE2->double, and we turn this into a
     // 64-bit FILD followed by conditional FADD for other targets.
     setOperationAction(ISD::UINT_TO_FP     , MVT::i64  , Custom);
@@ -203,7 +187,7 @@ X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM,
   setOperationAction(ISD::SINT_TO_FP       , MVT::i1   , Promote);
   setOperationAction(ISD::SINT_TO_FP       , MVT::i8   , Promote);
 
-  if (!TM.Options.UseSoftFloat) {
+  if (!Subtarget->useSoftFloat()) {
     // SSE has no i16 to fp conversion, only i32
     if (X86ScalarSSEf32) {
       setOperationAction(ISD::SINT_TO_FP     , MVT::i16  , Promote);
@@ -246,7 +230,7 @@ X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM,
   if (Subtarget->is64Bit()) {
     setOperationAction(ISD::FP_TO_UINT     , MVT::i64  , Expand);
     setOperationAction(ISD::FP_TO_UINT     , MVT::i32  , Promote);
-  } else if (!TM.Options.UseSoftFloat) {
+  } else if (!Subtarget->useSoftFloat()) {
     // Since AVX is a superset of SSE3, only check for SSE here.
     if (Subtarget->hasSSE1() && !Subtarget->hasSSE3())
       // Expand FP_TO_UINT into a select.
@@ -374,7 +358,7 @@ X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM,
   // Special handling for half-precision floating point conversions.
   // If we don't have F16C support, then lower half float conversions
   // into library calls.
-  if (TM.Options.UseSoftFloat || !Subtarget->hasF16C()) {
+  if (Subtarget->useSoftFloat() || !Subtarget->hasF16C()) {
     setOperationAction(ISD::FP16_TO_FP, MVT::f32, Expand);
     setOperationAction(ISD::FP_TO_FP16, MVT::f32, Expand);
   }
@@ -485,7 +469,7 @@ X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM,
     setOperationAction(ISD::EH_LABEL, MVT::Other, Expand);
   }
 
-  if (Subtarget->has64BitPointers()) {     // @LOCALMOD
+  if (Subtarget->is64Bit()) {
     setExceptionPointerRegister(X86::RAX);
     setExceptionSelectorRegister(X86::RDX);
   } else {
@@ -517,9 +501,13 @@ X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM,
   setOperationAction(ISD::STACKSAVE,          MVT::Other, Expand);
   setOperationAction(ISD::STACKRESTORE,       MVT::Other, Expand);
 
-  setOperationAction(ISD::DYNAMIC_STACKALLOC, getPointerTy(), Custom);
+  setOperationAction(ISD::DYNAMIC_STACKALLOC, PtrVT, Custom);
+
+  // GC_TRANSITION_START and GC_TRANSITION_END need custom lowering.
+  setOperationAction(ISD::GC_TRANSITION_START, MVT::Other, Custom);
+  setOperationAction(ISD::GC_TRANSITION_END, MVT::Other, Custom);
 
-  if (!TM.Options.UseSoftFloat && X86ScalarSSEf64) {
+  if (!Subtarget->useSoftFloat() && X86ScalarSSEf64) {
     // f32 and f64 use SSE.
     // Set up the FP register classes.
     addRegisterClass(MVT::f32, &X86::FR32RegClass);
@@ -553,7 +541,7 @@ X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM,
     // cases we handle.
     addLegalFPImmediate(APFloat(+0.0)); // xorpd
     addLegalFPImmediate(APFloat(+0.0f)); // xorps
-  } else if (!TM.Options.UseSoftFloat && X86ScalarSSEf32) {
+  } else if (!Subtarget->useSoftFloat() && X86ScalarSSEf32) {
     // Use SSE for f32, x87 for f64.
     // Set up the FP register classes.
     addRegisterClass(MVT::f32, &X86::FR32RegClass);
@@ -588,7 +576,7 @@ X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM,
       setOperationAction(ISD::FCOS   , MVT::f64, Expand);
       setOperationAction(ISD::FSINCOS, MVT::f64, Expand);
     }
-  } else if (!TM.Options.UseSoftFloat) {
+  } else if (!Subtarget->useSoftFloat()) {
     // f32 and f64 in x87.
     // Set up the FP register classes.
     addRegisterClass(MVT::f64, &X86::RFP64RegClass);
@@ -622,7 +610,7 @@ X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM,
   setOperationAction(ISD::FMA, MVT::f32, Expand);
 
   // Long double always uses X87.
-  if (!TM.Options.UseSoftFloat) {
+  if (!Subtarget->useSoftFloat()) {
     addRegisterClass(MVT::f80, &X86::RFP80RegClass);
     setOperationAction(ISD::UNDEF,     MVT::f80, Expand);
     setOperationAction(ISD::FCOPYSIGN, MVT::f80, Expand);
@@ -752,12 +740,17 @@ X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM,
       // them legal.
       if (VT.getVectorElementType() == MVT::i1)
         setLoadExtAction(ISD::EXTLOAD, InnerVT, VT, Expand);
+
+      // EXTLOAD for MVT::f16 vectors is not legal because f16 vectors are
+      // split/scalarized right now.
+      if (VT.getVectorElementType() == MVT::f16)
+        setLoadExtAction(ISD::EXTLOAD, InnerVT, VT, Expand);
     }
   }
 
   // FIXME: In order to prevent SSE instructions being expanded to MMX ones
   // with -msoft-float, disable use of MMX as well.
-  if (!TM.Options.UseSoftFloat && Subtarget->hasMMX()) {
+  if (!Subtarget->useSoftFloat() && Subtarget->hasMMX()) {
     addRegisterClass(MVT::x86mmx, &X86::VR64RegClass);
     // No operations on x86mmx supported, everything uses intrinsics.
   }
@@ -775,7 +768,7 @@ X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM,
   }
   setOperationAction(ISD::INSERT_VECTOR_ELT,  MVT::v1i64, Expand);
 
-  if (!TM.Options.UseSoftFloat && Subtarget->hasSSE1()) {
+  if (!Subtarget->useSoftFloat() && Subtarget->hasSSE1()) {
     addRegisterClass(MVT::v4f32, &X86::VR128RegClass);
 
     setOperationAction(ISD::FADD,               MVT::v4f32, Legal);
@@ -794,7 +787,7 @@ X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM,
     setOperationAction(ISD::UINT_TO_FP,         MVT::v4i32, Custom);
   }
 
-  if (!TM.Options.UseSoftFloat && Subtarget->hasSSE2()) {
+  if (!Subtarget->useSoftFloat() && Subtarget->hasSSE2()) {
     addRegisterClass(MVT::v2f64, &X86::VR128RegClass);
 
     // FIXME: Unfortunately, -soft-float and -no-implicit-float mean XMM
@@ -808,6 +801,7 @@ X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM,
     setOperationAction(ISD::ADD,                MVT::v8i16, Legal);
     setOperationAction(ISD::ADD,                MVT::v4i32, Legal);
     setOperationAction(ISD::ADD,                MVT::v2i64, Legal);
+    setOperationAction(ISD::MUL,                MVT::v16i8, Custom);
     setOperationAction(ISD::MUL,                MVT::v4i32, Custom);
     setOperationAction(ISD::MUL,                MVT::v2i64, Custom);
     setOperationAction(ISD::UMUL_LOHI,          MVT::v4i32, Custom);
@@ -827,6 +821,11 @@ X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM,
     setOperationAction(ISD::FNEG,               MVT::v2f64, Custom);
     setOperationAction(ISD::FABS,               MVT::v2f64, Custom);
 
+    setOperationAction(ISD::SMAX,               MVT::v8i16, Legal);
+    setOperationAction(ISD::UMAX,               MVT::v16i8, Legal);
+    setOperationAction(ISD::SMIN,               MVT::v8i16, Legal);
+    setOperationAction(ISD::UMIN,               MVT::v16i8, Legal);
+
     setOperationAction(ISD::SETCC,              MVT::v2i64, Custom);
     setOperationAction(ISD::SETCC,              MVT::v16i8, Custom);
     setOperationAction(ISD::SETCC,              MVT::v8i16, Custom);
@@ -838,13 +837,10 @@ X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM,
     setOperationAction(ISD::INSERT_VECTOR_ELT,  MVT::v4i32, Custom);
     setOperationAction(ISD::INSERT_VECTOR_ELT,  MVT::v4f32, Custom);
 
-    // Only provide customized ctpop vector bit twiddling for vector types we
-    // know to perform better than using the popcnt instructions on each vector
-    // element. If popcnt isn't supported, always provide the custom version.
-    if (!Subtarget->hasPOPCNT()) {
-      setOperationAction(ISD::CTPOP,            MVT::v4i32, Custom);
-      setOperationAction(ISD::CTPOP,            MVT::v2i64, Custom);
-    }
+    setOperationAction(ISD::CTPOP,              MVT::v16i8, Custom);
+    setOperationAction(ISD::CTPOP,              MVT::v8i16, Custom);
+    setOperationAction(ISD::CTPOP,              MVT::v4i32, Custom);
+    setOperationAction(ISD::CTPOP,              MVT::v2i64, Custom);
 
     // Custom lower build_vector, vector_shuffle, and extract_vector_elt.
     for (int i = MVT::v16i8; i != MVT::v2i64; ++i) {
@@ -920,6 +916,8 @@ X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM,
     setOperationAction(ISD::FP_TO_SINT,         MVT::v4i32, Legal);
     setOperationAction(ISD::SINT_TO_FP,         MVT::v4i32, Legal);
 
+    setOperationAction(ISD::SINT_TO_FP,         MVT::v2i32, Custom);
+
     setOperationAction(ISD::UINT_TO_FP,         MVT::v4i8,  Custom);
     setOperationAction(ISD::UINT_TO_FP,         MVT::v4i16, Custom);
     // As there is no 64-bit GPR available, we need build a special custom
@@ -938,7 +936,7 @@ X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM,
     setOperationAction(ISD::BITCAST,            MVT::v8i8,  Custom);
   }
 
-  if (!TM.Options.UseSoftFloat && Subtarget->hasSSE41()) {
+  if (!Subtarget->useSoftFloat() && Subtarget->hasSSE41()) {
     for (MVT RoundedTy : {MVT::f32, MVT::f64, MVT::v4f32, MVT::v2f64}) {
       setOperationAction(ISD::FFLOOR,           RoundedTy,  Legal);
       setOperationAction(ISD::FCEIL,            RoundedTy,  Legal);
@@ -947,6 +945,15 @@ X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM,
       setOperationAction(ISD::FNEARBYINT,       RoundedTy,  Legal);
     }
 
+    setOperationAction(ISD::SMAX,               MVT::v16i8, Legal);
+    setOperationAction(ISD::SMAX,               MVT::v4i32, Legal);
+    setOperationAction(ISD::UMAX,               MVT::v8i16, Legal);
+    setOperationAction(ISD::UMAX,               MVT::v4i32, Legal);
+    setOperationAction(ISD::SMIN,               MVT::v16i8, Legal);
+    setOperationAction(ISD::SMIN,               MVT::v4i32, Legal);
+    setOperationAction(ISD::UMIN,               MVT::v8i16, Legal);
+    setOperationAction(ISD::UMIN,               MVT::v4i32, Legal);
+
     // FIXME: Do we need to handle scalar-to-vector here?
     setOperationAction(ISD::MUL,                MVT::v4i32, Legal);
 
@@ -1000,6 +1007,10 @@ X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM,
   }
 
   if (Subtarget->hasSSE2()) {
+    setOperationAction(ISD::SIGN_EXTEND_VECTOR_INREG, MVT::v2i64, Custom);
+    setOperationAction(ISD::SIGN_EXTEND_VECTOR_INREG, MVT::v4i32, Custom);
+    setOperationAction(ISD::SIGN_EXTEND_VECTOR_INREG, MVT::v8i16, Custom);
+
     setOperationAction(ISD::SRL,               MVT::v8i16, Custom);
     setOperationAction(ISD::SRL,               MVT::v16i8, Custom);
 
@@ -1017,10 +1028,11 @@ X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM,
     setOperationAction(ISD::SHL,               MVT::v2i64, Custom);
     setOperationAction(ISD::SHL,               MVT::v4i32, Custom);
 
+    setOperationAction(ISD::SRA,               MVT::v2i64, Custom);
     setOperationAction(ISD::SRA,               MVT::v4i32, Custom);
   }
 
-  if (!TM.Options.UseSoftFloat && Subtarget->hasFp256()) {
+  if (!Subtarget->useSoftFloat() && Subtarget->hasFp256()) {
     addRegisterClass(MVT::v32i8,  &X86::VR256RegClass);
     addRegisterClass(MVT::v16i16, &X86::VR256RegClass);
     addRegisterClass(MVT::v8i32,  &X86::VR256RegClass);
@@ -1105,7 +1117,12 @@ X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM,
     setOperationAction(ISD::TRUNCATE,          MVT::v8i16, Custom);
     setOperationAction(ISD::TRUNCATE,          MVT::v4i32, Custom);
 
-    if (Subtarget->hasFMA() || Subtarget->hasFMA4()) {
+    setOperationAction(ISD::CTPOP,             MVT::v32i8, Custom);
+    setOperationAction(ISD::CTPOP,             MVT::v16i16, Custom);
+    setOperationAction(ISD::CTPOP,             MVT::v8i32, Custom);
+    setOperationAction(ISD::CTPOP,             MVT::v4i64, Custom);
+
+    if (Subtarget->hasFMA() || Subtarget->hasFMA4() || Subtarget->hasAVX512()) {
       setOperationAction(ISD::FMA,             MVT::v8f32, Legal);
       setOperationAction(ISD::FMA,             MVT::v4f64, Legal);
       setOperationAction(ISD::FMA,             MVT::v4f32, Legal);
@@ -1128,27 +1145,30 @@ X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM,
       setOperationAction(ISD::MUL,             MVT::v4i64, Custom);
       setOperationAction(ISD::MUL,             MVT::v8i32, Legal);
       setOperationAction(ISD::MUL,             MVT::v16i16, Legal);
-      // Don't lower v32i8 because there is no 128-bit byte mul
+      setOperationAction(ISD::MUL,             MVT::v32i8, Custom);
 
       setOperationAction(ISD::UMUL_LOHI,       MVT::v8i32, Custom);
       setOperationAction(ISD::SMUL_LOHI,       MVT::v8i32, Custom);
       setOperationAction(ISD::MULHU,           MVT::v16i16, Legal);
       setOperationAction(ISD::MULHS,           MVT::v16i16, Legal);
 
+      setOperationAction(ISD::SMAX,            MVT::v32i8,  Legal);
+      setOperationAction(ISD::SMAX,            MVT::v16i16, Legal);
+      setOperationAction(ISD::SMAX,            MVT::v8i32,  Legal);
+      setOperationAction(ISD::UMAX,            MVT::v32i8,  Legal);
+      setOperationAction(ISD::UMAX,            MVT::v16i16, Legal);
+      setOperationAction(ISD::UMAX,            MVT::v8i32,  Legal);
+      setOperationAction(ISD::SMIN,            MVT::v32i8,  Legal);
+      setOperationAction(ISD::SMIN,            MVT::v16i16, Legal);
+      setOperationAction(ISD::SMIN,            MVT::v8i32,  Legal);
+      setOperationAction(ISD::UMIN,            MVT::v32i8,  Legal);
+      setOperationAction(ISD::UMIN,            MVT::v16i16, Legal);
+      setOperationAction(ISD::UMIN,            MVT::v8i32,  Legal);
+
       // The custom lowering for UINT_TO_FP for v8i32 becomes interesting
       // when we have a 256bit-wide blend with immediate.
       setOperationAction(ISD::UINT_TO_FP, MVT::v8i32, Custom);
 
-      // Only provide customized ctpop vector bit twiddling for vector types we
-      // know to perform better than using the popcnt instructions on each
-      // vector element. If popcnt isn't supported, always provide the custom
-      // version.
-      if (!Subtarget->hasPOPCNT())
-        setOperationAction(ISD::CTPOP,           MVT::v4i64, Custom);
-
-      // Custom CTPOP always performs better on natively supported v8i32
-      setOperationAction(ISD::CTPOP,             MVT::v8i32, Custom);
-
       // AVX2 also has wider vector sign/zero extending loads, VPMOV[SZ]X
       setLoadExtAction(ISD::SEXTLOAD, MVT::v16i16, MVT::v16i8, Legal);
       setLoadExtAction(ISD::SEXTLOAD, MVT::v8i32,  MVT::v8i8,  Legal);
@@ -1177,7 +1197,7 @@ X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM,
       setOperationAction(ISD::MUL,             MVT::v4i64, Custom);
       setOperationAction(ISD::MUL,             MVT::v8i32, Custom);
       setOperationAction(ISD::MUL,             MVT::v16i16, Custom);
-      // Don't lower v32i8 because there is no 128-bit byte mul
+      setOperationAction(ISD::MUL,             MVT::v32i8, Custom);
     }
 
     // In the customized shift lowering, the legal cases in AVX2 will be
@@ -1188,6 +1208,7 @@ X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM,
     setOperationAction(ISD::SHL,               MVT::v4i64, Custom);
     setOperationAction(ISD::SHL,               MVT::v8i32, Custom);
 
+    setOperationAction(ISD::SRA,               MVT::v4i64, Custom);
     setOperationAction(ISD::SRA,               MVT::v8i32, Custom);
 
     // Custom lower several nodes for 256-bit types.
@@ -1240,7 +1261,7 @@ X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM,
     }
   }
 
-  if (!TM.Options.UseSoftFloat && Subtarget->hasAVX512()) {
+  if (!Subtarget->useSoftFloat() && Subtarget->hasAVX512()) {
     addRegisterClass(MVT::v16i32, &X86::VR512RegClass);
     addRegisterClass(MVT::v16f32, &X86::VR512RegClass);
     addRegisterClass(MVT::v8i64,  &X86::VR512RegClass);
@@ -1253,11 +1274,27 @@ X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM,
     for (MVT VT : MVT::fp_vector_valuetypes())
       setLoadExtAction(ISD::EXTLOAD, VT, MVT::v8f32, Legal);
 
+    setLoadExtAction(ISD::ZEXTLOAD, MVT::v16i32, MVT::v16i8, Legal);
+    setLoadExtAction(ISD::SEXTLOAD, MVT::v16i32, MVT::v16i8, Legal);
+    setLoadExtAction(ISD::ZEXTLOAD, MVT::v16i32, MVT::v16i16, Legal);
+    setLoadExtAction(ISD::SEXTLOAD, MVT::v16i32, MVT::v16i16, Legal);
+    setLoadExtAction(ISD::ZEXTLOAD, MVT::v32i16, MVT::v32i8, Legal);
+    setLoadExtAction(ISD::SEXTLOAD, MVT::v32i16, MVT::v32i8, Legal);
+    setLoadExtAction(ISD::ZEXTLOAD, MVT::v8i64,  MVT::v8i8,  Legal);
+    setLoadExtAction(ISD::SEXTLOAD, MVT::v8i64,  MVT::v8i8,  Legal);
+    setLoadExtAction(ISD::ZEXTLOAD, MVT::v8i64,  MVT::v8i16,  Legal);
+    setLoadExtAction(ISD::SEXTLOAD, MVT::v8i64,  MVT::v8i16,  Legal);
+    setLoadExtAction(ISD::ZEXTLOAD, MVT::v8i64,  MVT::v8i32,  Legal);
+    setLoadExtAction(ISD::SEXTLOAD, MVT::v8i64,  MVT::v8i32,  Legal);
+
     setOperationAction(ISD::BR_CC,              MVT::i1,    Expand);
     setOperationAction(ISD::SETCC,              MVT::i1,    Custom);
     setOperationAction(ISD::XOR,                MVT::i1,    Legal);
     setOperationAction(ISD::OR,                 MVT::i1,    Legal);
     setOperationAction(ISD::AND,                MVT::i1,    Legal);
+    setOperationAction(ISD::SUB,                MVT::i1,    Custom);
+    setOperationAction(ISD::ADD,                MVT::i1,    Custom);
+    setOperationAction(ISD::MUL,                MVT::i1,    Custom);
     setOperationAction(ISD::LOAD,               MVT::v16f32, Legal);
     setOperationAction(ISD::LOAD,               MVT::v8f64, Legal);
     setOperationAction(ISD::LOAD,               MVT::v8i64, Legal);
@@ -1302,23 +1339,77 @@ X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM,
     setOperationAction(ISD::UINT_TO_FP,         MVT::v16i32, Legal);
     setOperationAction(ISD::UINT_TO_FP,         MVT::v8i32, Legal);
     setOperationAction(ISD::UINT_TO_FP,         MVT::v4i32, Legal);
+    setOperationAction(ISD::UINT_TO_FP,         MVT::v16i8, Custom);
+    setOperationAction(ISD::UINT_TO_FP,         MVT::v16i16, Custom);
     setOperationAction(ISD::FP_ROUND,           MVT::v8f32, Legal);
     setOperationAction(ISD::FP_EXTEND,          MVT::v8f32, Legal);
 
+    setTruncStoreAction(MVT::v8i64,   MVT::v8i8,   Legal);
+    setTruncStoreAction(MVT::v8i64,   MVT::v8i16,  Legal);
+    setTruncStoreAction(MVT::v8i64,   MVT::v8i32,  Legal);
+    setTruncStoreAction(MVT::v16i32,  MVT::v16i8,  Legal);
+    setTruncStoreAction(MVT::v16i32,  MVT::v16i16, Legal);
+    if (Subtarget->hasVLX()){
+      setTruncStoreAction(MVT::v4i64, MVT::v4i8,  Legal);
+      setTruncStoreAction(MVT::v4i64, MVT::v4i16, Legal);
+      setTruncStoreAction(MVT::v4i64, MVT::v4i32, Legal);
+      setTruncStoreAction(MVT::v8i32, MVT::v8i8,  Legal);
+      setTruncStoreAction(MVT::v8i32, MVT::v8i16, Legal);
+
+      setTruncStoreAction(MVT::v2i64, MVT::v2i8,  Legal);
+      setTruncStoreAction(MVT::v2i64, MVT::v2i16, Legal);
+      setTruncStoreAction(MVT::v2i64, MVT::v2i32, Legal);
+      setTruncStoreAction(MVT::v4i32, MVT::v4i8,  Legal);
+      setTruncStoreAction(MVT::v4i32, MVT::v4i16, Legal);
+    }
     setOperationAction(ISD::TRUNCATE,           MVT::i1, Custom);
     setOperationAction(ISD::TRUNCATE,           MVT::v16i8, Custom);
     setOperationAction(ISD::TRUNCATE,           MVT::v8i32, Custom);
+    if (Subtarget->hasDQI()) {
+      setOperationAction(ISD::TRUNCATE,         MVT::v2i1, Custom);
+      setOperationAction(ISD::TRUNCATE,         MVT::v4i1, Custom);
+
+      setOperationAction(ISD::SINT_TO_FP,       MVT::v8i64, Legal);
+      setOperationAction(ISD::UINT_TO_FP,       MVT::v8i64, Legal);
+      setOperationAction(ISD::FP_TO_SINT,       MVT::v8i64, Legal);
+      setOperationAction(ISD::FP_TO_UINT,       MVT::v8i64, Legal);
+      if (Subtarget->hasVLX()) {
+        setOperationAction(ISD::SINT_TO_FP,    MVT::v4i64, Legal);
+        setOperationAction(ISD::SINT_TO_FP,    MVT::v2i64, Legal);
+        setOperationAction(ISD::UINT_TO_FP,    MVT::v4i64, Legal);
+        setOperationAction(ISD::UINT_TO_FP,    MVT::v2i64, Legal);
+        setOperationAction(ISD::FP_TO_SINT,    MVT::v4i64, Legal);
+        setOperationAction(ISD::FP_TO_SINT,    MVT::v2i64, Legal);
+        setOperationAction(ISD::FP_TO_UINT,    MVT::v4i64, Legal);
+        setOperationAction(ISD::FP_TO_UINT,    MVT::v2i64, Legal);
+      }
+    }
+    if (Subtarget->hasVLX()) {
+      setOperationAction(ISD::SINT_TO_FP,       MVT::v8i32, Legal);
+      setOperationAction(ISD::UINT_TO_FP,       MVT::v8i32, Legal);
+      setOperationAction(ISD::FP_TO_SINT,       MVT::v8i32, Legal);
+      setOperationAction(ISD::FP_TO_UINT,       MVT::v8i32, Legal);
+      setOperationAction(ISD::SINT_TO_FP,       MVT::v4i32, Legal);
+      setOperationAction(ISD::UINT_TO_FP,       MVT::v4i32, Legal);
+      setOperationAction(ISD::FP_TO_SINT,       MVT::v4i32, Legal);
+      setOperationAction(ISD::FP_TO_UINT,       MVT::v4i32, Legal);
+    }
     setOperationAction(ISD::TRUNCATE,           MVT::v8i1, Custom);
     setOperationAction(ISD::TRUNCATE,           MVT::v16i1, Custom);
     setOperationAction(ISD::TRUNCATE,           MVT::v16i16, Custom);
     setOperationAction(ISD::ZERO_EXTEND,        MVT::v16i32, Custom);
     setOperationAction(ISD::ZERO_EXTEND,        MVT::v8i64, Custom);
+    setOperationAction(ISD::ANY_EXTEND,         MVT::v16i32, Custom);
+    setOperationAction(ISD::ANY_EXTEND,         MVT::v8i64, Custom);
     setOperationAction(ISD::SIGN_EXTEND,        MVT::v16i32, Custom);
     setOperationAction(ISD::SIGN_EXTEND,        MVT::v8i64, Custom);
     setOperationAction(ISD::SIGN_EXTEND,        MVT::v16i8, Custom);
     setOperationAction(ISD::SIGN_EXTEND,        MVT::v8i16, Custom);
     setOperationAction(ISD::SIGN_EXTEND,        MVT::v16i16, Custom);
-
+    if (Subtarget->hasDQI()) {
+      setOperationAction(ISD::SIGN_EXTEND,        MVT::v4i32, Custom);
+      setOperationAction(ISD::SIGN_EXTEND,        MVT::v2i64, Custom);
+    }
     setOperationAction(ISD::FFLOOR,             MVT::v16f32, Legal);
     setOperationAction(ISD::FFLOOR,             MVT::v8f64, Legal);
     setOperationAction(ISD::FCEIL,              MVT::v16f32, Legal);
@@ -1350,6 +1441,17 @@ X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM,
     setOperationAction(ISD::SELECT,             MVT::v8f64, Custom);
     setOperationAction(ISD::SELECT,             MVT::v8i64, Custom);
     setOperationAction(ISD::SELECT,             MVT::v16f32, Custom);
+    setOperationAction(ISD::SELECT,             MVT::v16i1, Custom);
+    setOperationAction(ISD::SELECT,             MVT::v8i1,  Custom);
+
+    setOperationAction(ISD::SMAX,               MVT::v16i32, Legal);
+    setOperationAction(ISD::SMAX,               MVT::v8i64, Legal);
+    setOperationAction(ISD::UMAX,               MVT::v16i32, Legal);
+    setOperationAction(ISD::UMAX,               MVT::v8i64, Legal);
+    setOperationAction(ISD::SMIN,               MVT::v16i32, Legal);
+    setOperationAction(ISD::SMIN,               MVT::v8i64, Legal);
+    setOperationAction(ISD::UMIN,               MVT::v16i32, Legal);
+    setOperationAction(ISD::UMIN,               MVT::v8i64, Legal);
 
     setOperationAction(ISD::ADD,                MVT::v8i64, Legal);
     setOperationAction(ISD::ADD,                MVT::v16i32, Legal);
@@ -1379,10 +1481,23 @@ X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM,
       setOperationAction(ISD::CTLZ,             MVT::v8i64, Legal);
       setOperationAction(ISD::CTLZ,             MVT::v16i32, Legal);
     }
-
+    if (Subtarget->hasDQI()) {
+      setOperationAction(ISD::MUL,             MVT::v2i64, Legal);
+      setOperationAction(ISD::MUL,             MVT::v4i64, Legal);
+      setOperationAction(ISD::MUL,             MVT::v8i64, Legal);
+    }
     // Custom lower several nodes.
     for (MVT VT : MVT::vector_valuetypes()) {
       unsigned EltSize = VT.getVectorElementType().getSizeInBits();
+      if (EltSize == 1) {
+        setOperationAction(ISD::AND, VT, Legal);
+        setOperationAction(ISD::OR,  VT, Legal);
+        setOperationAction(ISD::XOR,  VT, Legal);
+      }
+      if (EltSize >= 32 && VT.getSizeInBits() <= 512) {
+        setOperationAction(ISD::MGATHER,  VT, Custom);
+        setOperationAction(ISD::MSCATTER, VT, Custom);
+      }
       // Extract subvector is special because the value type
       // (result) is 256/128-bit but the source is 512-bit wide.
       if (VT.is128BitVector() || VT.is256BitVector()) {
@@ -1395,7 +1510,7 @@ X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM,
       if (!VT.is512BitVector())
         continue;
 
-      if ( EltSize >= 32) {
+      if (EltSize >= 32) {
         setOperationAction(ISD::VECTOR_SHUFFLE,      VT, Custom);
         setOperationAction(ISD::INSERT_VECTOR_ELT,   VT, Custom);
         setOperationAction(ISD::BUILD_VECTOR,        VT, Custom);
@@ -1419,7 +1534,7 @@ X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM,
     }
   }// has  AVX-512
 
-  if (!TM.Options.UseSoftFloat && Subtarget->hasBWI()) {
+  if (!Subtarget->useSoftFloat() && Subtarget->hasBWI()) {
     addRegisterClass(MVT::v32i16, &X86::VR512RegClass);
     addRegisterClass(MVT::v64i8,  &X86::VR512RegClass);
 
@@ -1435,10 +1550,41 @@ X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM,
     setOperationAction(ISD::SUB,                MVT::v32i16, Legal);
     setOperationAction(ISD::SUB,                MVT::v64i8, Legal);
     setOperationAction(ISD::MUL,                MVT::v32i16, Legal);
+    setOperationAction(ISD::MULHS,              MVT::v32i16, Legal);
+    setOperationAction(ISD::MULHU,              MVT::v32i16, Legal);
     setOperationAction(ISD::CONCAT_VECTORS,     MVT::v32i1, Custom);
     setOperationAction(ISD::CONCAT_VECTORS,     MVT::v64i1, Custom);
     setOperationAction(ISD::INSERT_SUBVECTOR,   MVT::v32i1, Custom);
     setOperationAction(ISD::INSERT_SUBVECTOR,   MVT::v64i1, Custom);
+    setOperationAction(ISD::SELECT,             MVT::v32i1, Custom);
+    setOperationAction(ISD::SELECT,             MVT::v64i1, Custom);
+    setOperationAction(ISD::SIGN_EXTEND,        MVT::v32i8, Custom);
+    setOperationAction(ISD::ZERO_EXTEND,        MVT::v32i8, Custom);
+    setOperationAction(ISD::SIGN_EXTEND,        MVT::v32i16, Custom);
+    setOperationAction(ISD::ZERO_EXTEND,        MVT::v32i16, Custom);
+    setOperationAction(ISD::SIGN_EXTEND,        MVT::v64i8, Custom);
+    setOperationAction(ISD::ZERO_EXTEND,        MVT::v64i8, Custom);
+    setOperationAction(ISD::INSERT_VECTOR_ELT,  MVT::v32i1, Custom);
+    setOperationAction(ISD::INSERT_VECTOR_ELT,  MVT::v64i1, Custom);
+    setOperationAction(ISD::VSELECT,            MVT::v32i16, Legal);
+    setOperationAction(ISD::VSELECT,            MVT::v64i8, Legal);
+    setOperationAction(ISD::TRUNCATE,           MVT::v32i1, Custom);
+    setOperationAction(ISD::TRUNCATE,           MVT::v64i1, Custom);
+    setOperationAction(ISD::TRUNCATE,           MVT::v32i8, Custom);
+
+    setOperationAction(ISD::SMAX,               MVT::v64i8, Legal);
+    setOperationAction(ISD::SMAX,               MVT::v32i16, Legal);
+    setOperationAction(ISD::UMAX,               MVT::v64i8, Legal);
+    setOperationAction(ISD::UMAX,               MVT::v32i16, Legal);
+    setOperationAction(ISD::SMIN,               MVT::v64i8, Legal);
+    setOperationAction(ISD::SMIN,               MVT::v32i16, Legal);
+    setOperationAction(ISD::UMIN,               MVT::v64i8, Legal);
+    setOperationAction(ISD::UMIN,               MVT::v32i16, Legal);
+
+    setTruncStoreAction(MVT::v32i16,  MVT::v32i8, Legal);
+    setTruncStoreAction(MVT::v16i16,  MVT::v16i8, Legal);
+    if (Subtarget->hasVLX())
+      setTruncStoreAction(MVT::v8i16,   MVT::v8i8,  Legal);
 
     for (int i = MVT::v32i8; i != MVT::v8i64; ++i) {
       const MVT VT = (MVT::SimpleValueType)i;
@@ -1456,7 +1602,7 @@ X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM,
     }
   }
 
-  if (!TM.Options.UseSoftFloat && Subtarget->hasVLX()) {
+  if (!Subtarget->useSoftFloat() && Subtarget->hasVLX()) {
     addRegisterClass(MVT::v4i1,   &X86::VK4RegClass);
     addRegisterClass(MVT::v2i1,   &X86::VK2RegClass);
 
@@ -1466,6 +1612,10 @@ X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM,
     setOperationAction(ISD::CONCAT_VECTORS,     MVT::v8i1, Custom);
     setOperationAction(ISD::INSERT_SUBVECTOR,   MVT::v8i1, Custom);
     setOperationAction(ISD::INSERT_SUBVECTOR,   MVT::v4i1, Custom);
+    setOperationAction(ISD::SELECT,             MVT::v4i1, Custom);
+    setOperationAction(ISD::SELECT,             MVT::v2i1, Custom);
+    setOperationAction(ISD::BUILD_VECTOR,       MVT::v4i1, Custom);
+    setOperationAction(ISD::BUILD_VECTOR,       MVT::v2i1, Custom);
 
     setOperationAction(ISD::AND,                MVT::v8i32, Legal);
     setOperationAction(ISD::OR,                 MVT::v8i32, Legal);
@@ -1473,6 +1623,17 @@ X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM,
     setOperationAction(ISD::AND,                MVT::v4i32, Legal);
     setOperationAction(ISD::OR,                 MVT::v4i32, Legal);
     setOperationAction(ISD::XOR,                MVT::v4i32, Legal);
+    setOperationAction(ISD::SRA,                MVT::v2i64, Custom);
+    setOperationAction(ISD::SRA,                MVT::v4i64, Custom);
+
+    setOperationAction(ISD::SMAX,               MVT::v2i64, Legal);
+    setOperationAction(ISD::SMAX,               MVT::v4i64, Legal);
+    setOperationAction(ISD::UMAX,               MVT::v2i64, Legal);
+    setOperationAction(ISD::UMAX,               MVT::v4i64, Legal);
+    setOperationAction(ISD::SMIN,               MVT::v2i64, Legal);
+    setOperationAction(ISD::SMIN,               MVT::v4i64, Legal);
+    setOperationAction(ISD::UMIN,               MVT::v2i64, Legal);
+    setOperationAction(ISD::UMIN,               MVT::v4i64, Legal);
   }
 
   // We want to custom lower some of our intrinsics.
@@ -1552,8 +1713,8 @@ X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM,
   setTargetDAGCombine(ISD::ANY_EXTEND);
   setTargetDAGCombine(ISD::SIGN_EXTEND);
   setTargetDAGCombine(ISD::SIGN_EXTEND_INREG);
-  setTargetDAGCombine(ISD::TRUNCATE);
   setTargetDAGCombine(ISD::SINT_TO_FP);
+  setTargetDAGCombine(ISD::UINT_TO_FP);
   setTargetDAGCombine(ISD::SETCC);
   setTargetDAGCombine(ISD::INTRINSIC_WO_CHAIN);
   setTargetDAGCombine(ISD::BUILD_VECTOR);
@@ -1562,14 +1723,12 @@ X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM,
 
   computeRegisterProperties(Subtarget->getRegisterInfo());
 
-  // On Darwin, -Os means optimize for size without hurting performance,
-  // do not reduce the limit.
   MaxStoresPerMemset = 16; // For @llvm.memset -> sequence of stores
-  MaxStoresPerMemsetOptSize = Subtarget->isTargetDarwin() ? 16 : 8;
+  MaxStoresPerMemsetOptSize = 8;
   MaxStoresPerMemcpy = 8; // For @llvm.memcpy -> sequence of stores
-  MaxStoresPerMemcpyOptSize = Subtarget->isTargetDarwin() ? 8 : 4;
+  MaxStoresPerMemcpyOptSize = 4;
   MaxStoresPerMemmove = 8; // For @llvm.memmove -> sequence of stores
-  MaxStoresPerMemmoveOptSize = Subtarget->isTargetDarwin() ? 8 : 4;
+  MaxStoresPerMemmoveOptSize = 4;
   setPrefLoopAlignment(4); // 2^4 bytes.
 
   // Predictable cmov don't hurt on atom because it's in-order.
@@ -1595,7 +1754,8 @@ X86TargetLowering::getPreferredVectorAction(EVT VT) const {
   return TargetLoweringBase::getPreferredVectorAction(VT);
 }
 
-EVT X86TargetLowering::getSetCCResultType(LLVMContext &, EVT VT) const {
+EVT X86TargetLowering::getSetCCResultType(const DataLayout &DL, LLVMContext &,
+                                          EVT VT) const {
   if (!VT.isVector())
     return Subtarget->hasAVX512() ? MVT::i1: MVT::i8;
 
@@ -1652,9 +1812,9 @@ static void getMaxByValAlign(Type *Ty, unsigned &MaxAlign) {
     if (EltAlign > MaxAlign)
       MaxAlign = EltAlign;
   } else if (StructType *STy = dyn_cast<StructType>(Ty)) {
-    for (unsigned i = 0, e = STy->getNumElements(); i != e; ++i) {
+    for (auto *EltTy : STy->elements()) {
       unsigned EltAlign = 0;
-      getMaxByValAlign(STy->getElementType(i), EltAlign);
+      getMaxByValAlign(EltTy, EltAlign);
       if (EltAlign > MaxAlign)
         MaxAlign = EltAlign;
       if (MaxAlign == 16)
@@ -1667,10 +1827,11 @@ static void getMaxByValAlign(Type *Ty, unsigned &MaxAlign) {
 /// function arguments in the caller parameter area. For X86, aggregates
 /// that contain SSE vectors are placed at 16-byte boundaries while the rest
 /// are at 4-byte boundaries.
-unsigned X86TargetLowering::getByValTypeAlignment(Type *Ty) const {
+unsigned X86TargetLowering::getByValTypeAlignment(Type *Ty,
+                                                  const DataLayout &DL) const {
   if (Subtarget->is64Bit()) {
     // Max of 8 and alignment of type.
-    unsigned TyAlign = TD->getABITypeAlignment(Ty);
+    unsigned TyAlign = DL.getABITypeAlignment(Ty);
     if (TyAlign > 8)
       return TyAlign;
     return 8;
@@ -1761,6 +1922,10 @@ unsigned X86TargetLowering::getJumpTableEncoding() const {
   return TargetLowering::getJumpTableEncoding();
 }
 
+bool X86TargetLowering::useSoftFloat() const {
+  return Subtarget->useSoftFloat();
+}
+
 const MCExpr *
 X86TargetLowering::LowerCustomJumpTableEntry(const MachineJumpTableInfo *MJTI,
                                              const MachineBasicBlock *MBB,
@@ -1769,7 +1934,7 @@ X86TargetLowering::LowerCustomJumpTableEntry(const MachineJumpTableInfo *MJTI,
          Subtarget->isPICStyleGOT());
   // In 32-bit ELF systems, our jump table entries are formed with @GOTOFF
   // entries.
-  return MCSymbolRefExpr::Create(MBB->getSymbol(),
+  return MCSymbolRefExpr::create(MBB->getSymbol(),
                                  MCSymbolRefExpr::VK_GOTOFF, Ctx);
 }
 
@@ -1779,7 +1944,8 @@ SDValue X86TargetLowering::getPICJumpTableRelocBase(SDValue Table,
   if (!Subtarget->is64Bit())
     // This doesn't have SDLoc associated with it, but is not really the
     // same as a Register.
-    return DAG.getNode(X86ISD::GlobalBaseReg, SDLoc(), getPointerTy());
+    return DAG.getNode(X86ISD::GlobalBaseReg, SDLoc(),
+                       getPointerTy(DAG.getDataLayout()));
   return Table;
 }
 
@@ -1793,7 +1959,7 @@ getPICJumpTableRelocBaseExpr(const MachineFunction *MF, unsigned JTI,
     return TargetLowering::getPICJumpTableRelocBaseExpr(MF, JTI, Ctx);
 
   // Otherwise, the reference is relative to the PIC base.
-  return MCSymbolRefExpr::Create(MF->getPICBaseSymbol(), Ctx);
+  return MCSymbolRefExpr::create(MF->getPICBaseSymbol(), Ctx);
 }
 
 std::pair<const TargetRegisterClass *, uint8_t>
@@ -1886,7 +2052,7 @@ X86TargetLowering::LowerReturn(SDValue Chain,
   SmallVector<SDValue, 6> RetOps;
   RetOps.push_back(Chain); // Operand #0 = Chain (updated below)
   // Operand #1 = Bytes To Pop
-  RetOps.push_back(DAG.getTargetConstant(FuncInfo->getBytesToPopOnReturn(),
+  RetOps.push_back(DAG.getTargetConstant(FuncInfo->getBytesToPopOnReturn(), dl,
                    MVT::i16));
 
   // Copy the result values into the output registers.
@@ -1901,10 +2067,14 @@ X86TargetLowering::LowerReturn(SDValue Chain,
       ValToCopy = DAG.getNode(ISD::SIGN_EXTEND, dl, VA.getLocVT(), ValToCopy);
     else if (VA.getLocInfo() == CCValAssign::ZExt)
       ValToCopy = DAG.getNode(ISD::ZERO_EXTEND, dl, VA.getLocVT(), ValToCopy);
-    else if (VA.getLocInfo() == CCValAssign::AExt)
-      ValToCopy = DAG.getNode(ISD::ANY_EXTEND, dl, VA.getLocVT(), ValToCopy);
+    else if (VA.getLocInfo() == CCValAssign::AExt) {
+      if (ValVT.isVector() && ValVT.getScalarType() == MVT::i1)
+        ValToCopy = DAG.getNode(ISD::SIGN_EXTEND, dl, VA.getLocVT(), ValToCopy);
+      else
+        ValToCopy = DAG.getNode(ISD::ANY_EXTEND, dl, VA.getLocVT(), ValToCopy);
+    }
     else if (VA.getLocInfo() == CCValAssign::BCvt)
-      ValToCopy = DAG.getNode(ISD::BITCAST, dl, VA.getLocVT(), ValToCopy);
+      ValToCopy = DAG.getBitcast(VA.getLocVT(), ValToCopy);
 
     assert(VA.getLocInfo() != CCValAssign::FPExt &&
            "Unexpected FP-extend for return value.");
@@ -1941,13 +2111,13 @@ X86TargetLowering::LowerReturn(SDValue Chain,
     if (Subtarget->is64Bit()) {
       if (ValVT == MVT::x86mmx) {
         if (VA.getLocReg() == X86::XMM0 || VA.getLocReg() == X86::XMM1) {
-          ValToCopy = DAG.getNode(ISD::BITCAST, dl, MVT::i64, ValToCopy);
+          ValToCopy = DAG.getBitcast(MVT::i64, ValToCopy);
           ValToCopy = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, MVT::v2i64,
                                   ValToCopy);
           // If we don't have SSE2 available, convert to v4f32 so the generated
           // register is legal.
           if (!Subtarget->hasSSE2())
-            ValToCopy = DAG.getNode(ISD::BITCAST, dl, MVT::v4f32,ValToCopy);
+            ValToCopy = DAG.getBitcast(MVT::v4f32, ValToCopy);
         }
       }
     }
@@ -1957,9 +2127,8 @@ X86TargetLowering::LowerReturn(SDValue Chain,
     RetOps.push_back(DAG.getRegister(VA.getLocReg(), VA.getLocVT()));
   }
 
-  // The x86-64 ABIs require that for returning structs by value we copy
+  // All x86 ABIs require that for returning structs by value we copy
   // the sret argument into %rax/%eax (depending on ABI) for the return.
-  // Win32 requires us to put the sret argument to %eax as well.
   // We saved the argument into a virtual register in the entry block,
   // so now we copy the value out and into %rax/%eax.
   //
@@ -1968,28 +2137,18 @@ X86TargetLowering::LowerReturn(SDValue Chain,
   // false, then an sret argument may be implicitly inserted in the SelDAG. In
   // either case FuncInfo->setSRetReturnReg() will have been called.
   if (unsigned SRetReg = FuncInfo->getSRetReturnReg()) {
-    assert((Subtarget->is64Bit() || Subtarget->isTargetKnownWindowsMSVC()) &&
-           "No need for an sret register");
-    SDValue Val = DAG.getCopyFromReg(Chain, dl, SRetReg, getPointerTy());
+    SDValue Val = DAG.getCopyFromReg(Chain, dl, SRetReg,
+                                     getPointerTy(MF.getDataLayout()));
 
     unsigned RetValReg
         = (Subtarget->is64Bit() && !Subtarget->isTarget64BitILP32()) ?
           X86::RAX : X86::EAX;
-
-    // @LOCALMOD-BEGIN
-    if (Subtarget->isTargetNaCl()) {
-      RetValReg = Subtarget->isTarget64BitILP32() ? X86::EAX : X86::RAX;
-      // NaCl 64 uses 32-bit pointers, so there might be some zero-ext needed.
-      SDValue Zext = DAG.getZExtOrTrunc(Val, dl, MVT::i64);
-      Chain = DAG.getCopyToReg(Chain, dl, X86::RAX, Zext, Flag);
-    } else {
-      Chain = DAG.getCopyToReg(Chain, dl, RetValReg, Val, Flag);
-    }
-    // @LOCALMOD-END
+    Chain = DAG.getCopyToReg(Chain, dl, RetValReg, Val, Flag);
     Flag = Chain.getValue(1);
 
     // RAX/EAX now acts like a return value.
-    RetOps.push_back(DAG.getRegister(RetValReg, getPointerTy()));
+    RetOps.push_back(
+        DAG.getRegister(RetValReg, getPointerTy(DAG.getDataLayout())));
   }
 
   RetOps[0] = Chain;  // Update chain.
@@ -2074,7 +2233,7 @@ X86TargetLowering::LowerCallResult(SDValue Chain, SDValue InFlag,
   // Copy all of the result registers out of their specified physreg.
   for (unsigned i = 0, e = RVLocs.size(); i != e; ++i) {
     CCValAssign &VA = RVLocs[i];
-    EVT CopyVT = VA.getValVT();
+    EVT CopyVT = VA.getLocVT();
 
     // If this is x86-64, and we disabled SSE, we can't return FP values
     if ((CopyVT == MVT::f32 || CopyVT == MVT::f64) &&
@@ -2084,18 +2243,24 @@ X86TargetLowering::LowerCallResult(SDValue Chain, SDValue InFlag,
 
     // If we prefer to use the value in xmm registers, copy it out as f80 and
     // use a truncate to move it from fp stack reg to xmm reg.
+    bool RoundAfterCopy = false;
     if ((VA.getLocReg() == X86::FP0 || VA.getLocReg() == X86::FP1) &&
-        isScalarFPTypeInSSEReg(VA.getValVT()))
+        isScalarFPTypeInSSEReg(VA.getValVT())) {
       CopyVT = MVT::f80;
+      RoundAfterCopy = (CopyVT != VA.getLocVT());
+    }
 
     Chain = DAG.getCopyFromReg(Chain, dl, VA.getLocReg(),
                                CopyVT, InFlag).getValue(1);
     SDValue Val = Chain.getValue(0);
 
-    if (CopyVT != VA.getValVT())
+    if (RoundAfterCopy)
       Val = DAG.getNode(ISD::FP_ROUND, dl, VA.getValVT(), Val,
                         // This truncation won't change the value.
-                        DAG.getIntPtrConstant(1));
+                        DAG.getIntPtrConstant(1, dl));
+
+    if (VA.isExtInLoc() && VA.getValVT().getScalarType() == MVT::i1)
+      Val = DAG.getNode(ISD::TRUNCATE, dl, VA.getValVT(), Val);
 
     InFlag = Chain.getValue(2);
     InVals.push_back(Val);
@@ -2155,7 +2320,7 @@ static SDValue
 CreateCopyOfByValArgument(SDValue Src, SDValue Dst, SDValue Chain,
                           ISD::ArgFlagsTy Flags, SelectionDAG &DAG,
                           SDLoc dl) {
-  SDValue SizeNode = DAG.getConstant(Flags.getByValSize(), MVT::i32);
+  SDValue SizeNode = DAG.getConstant(Flags.getByValSize(), dl, MVT::i32);
 
   return DAG.getMemcpy(Chain, dl, Dst, Src, SizeNode, Flags.getByValAlign(),
                        /*isVolatile*/false, /*AlwaysInline=*/true,
@@ -2177,7 +2342,9 @@ static bool IsCCallConvention(CallingConv::ID CC) {
 }
 
 bool X86TargetLowering::mayBeEmittedAsTailCall(CallInst *CI) const {
-  if (!CI->isTailCall() || getTargetMachine().Options.DisableTailCalls)
+  auto Attr =
+      CI->getParent()->getParent()->getFnAttribute("disable-tail-calls");
+  if (!CI->isTailCall() || Attr.getValueAsString() == "true")
     return false;
 
   CallSite CS(CI);
@@ -2212,7 +2379,10 @@ X86TargetLowering::LowerMemArgument(SDValue Chain,
 
   // If value is passed by pointer we have address passed instead of the value
   // itself.
-  if (VA.getLocInfo() == CCValAssign::Indirect)
+  bool ExtendedInMem = VA.isExtInLoc() &&
+    VA.getValVT().getScalarType() == MVT::i1;
+
+  if (VA.getLocInfo() == CCValAssign::Indirect || ExtendedInMem)
     ValVT = VA.getLocVT();
   else
     ValVT = VA.getValVT();
@@ -2225,14 +2395,16 @@ X86TargetLowering::LowerMemArgument(SDValue Chain,
     unsigned Bytes = Flags.getByValSize();
     if (Bytes == 0) Bytes = 1; // Don't create zero-sized stack objects.
     int FI = MFI->CreateFixedObject(Bytes, VA.getLocMemOffset(), isImmutable);
-    return DAG.getFrameIndex(FI, getPointerTy());
+    return DAG.getFrameIndex(FI, getPointerTy(DAG.getDataLayout()));
   } else {
     int FI = MFI->CreateFixedObject(ValVT.getSizeInBits()/8,
                                     VA.getLocMemOffset(), isImmutable);
-    SDValue FIN = DAG.getFrameIndex(FI, getPointerTy());
-    return DAG.getLoad(ValVT, dl, Chain, FIN,
-                       MachinePointerInfo::getFixedStack(FI),
-                       false, false, false, 0);
+    SDValue FIN = DAG.getFrameIndex(FI, getPointerTy(DAG.getDataLayout()));
+    SDValue Val =  DAG.getLoad(ValVT, dl, Chain, FIN,
+                               MachinePointerInfo::getFixedStack(FI),
+                               false, false, false, 0);
+    return ExtendedInMem ?
+      DAG.getNode(ISD::TRUNCATE, dl, VA.getValVT(), Val) : Val;
   }
 }
 
@@ -2269,10 +2441,10 @@ static ArrayRef<MCPhysReg> get64BitArgumentXMMs(MachineFunction &MF,
 
   const Function *Fn = MF.getFunction();
   bool NoImplicitFloatOps = Fn->hasFnAttribute(Attribute::NoImplicitFloat);
-  assert(!(MF.getTarget().Options.UseSoftFloat && NoImplicitFloatOps) &&
+  bool isSoftFloat = Subtarget->useSoftFloat();
+  assert(!(isSoftFloat && NoImplicitFloatOps) &&
          "SSE register cannot be used when SSE is disabled!");
-  if (MF.getTarget().Options.UseSoftFloat || NoImplicitFloatOps ||
-      !Subtarget->hasSSE1())
+  if (isSoftFloat || NoImplicitFloatOps || !Subtarget->hasSSE1())
     // Kernel mode asks for SSE to be disabled, so there are no XMM argument
     // registers.
     return None;
@@ -2376,11 +2548,11 @@ X86TargetLowering::LowerFormalArguments(SDValue Chain,
         ArgValue = DAG.getNode(ISD::AssertZext, dl, RegVT, ArgValue,
                                DAG.getValueType(VA.getValVT()));
       else if (VA.getLocInfo() == CCValAssign::BCvt)
-        ArgValue = DAG.getNode(ISD::BITCAST, dl, VA.getValVT(), ArgValue);
+        ArgValue = DAG.getBitcast(VA.getValVT(), ArgValue);
 
       if (VA.isExtInLoc()) {
         // Handle MMX values passed in XMM regs.
-        if (RegVT.isVector())
+        if (RegVT.isVector() && VA.getValVT().getScalarType() != MVT::i1)
           ArgValue = DAG.getNode(X86ISD::MOVDQ2Q, dl, VA.getValVT(), ArgValue);
         else
           ArgValue = DAG.getNode(ISD::TRUNCATE, dl, VA.getValVT(), ArgValue);
@@ -2398,24 +2570,21 @@ X86TargetLowering::LowerFormalArguments(SDValue Chain,
     InVals.push_back(ArgValue);
   }
 
-  if (Subtarget->is64Bit() || Subtarget->isTargetKnownWindowsMSVC()) {
-    for (unsigned i = 0, e = ArgLocs.size(); i != e; ++i) {
-      // The x86-64 ABIs require that for returning structs by value we copy
-      // the sret argument into %rax/%eax (depending on ABI) for the return.
-      // Win32 requires us to put the sret argument to %eax as well.
-      // Save the argument into a virtual register so that we can access it
-      // from the return points.
-      if (Ins[i].Flags.isSRet()) {
-        unsigned Reg = FuncInfo->getSRetReturnReg();
-        if (!Reg) {
-          MVT PtrTy = getPointerTy();
-          Reg = MF.getRegInfo().createVirtualRegister(getRegClassFor(PtrTy));
-          FuncInfo->setSRetReturnReg(Reg);
-        }
-        SDValue Copy = DAG.getCopyToReg(DAG.getEntryNode(), dl, Reg, InVals[i]);
-        Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Copy, Chain);
-        break;
+  for (unsigned i = 0, e = ArgLocs.size(); i != e; ++i) {
+    // All x86 ABIs require that for returning structs by value we copy the
+    // sret argument into %rax/%eax (depending on ABI) for the return. Save
+    // the argument into a virtual register so that we can access it from the
+    // return points.
+    if (Ins[i].Flags.isSRet()) {
+      unsigned Reg = FuncInfo->getSRetReturnReg();
+      if (!Reg) {
+        MVT PtrTy = getPointerTy(DAG.getDataLayout());
+        Reg = MF.getRegInfo().createVirtualRegister(getRegClassFor(PtrTy));
+        FuncInfo->setSRetReturnReg(Reg);
       }
+      SDValue Copy = DAG.getCopyToReg(DAG.getEntryNode(), dl, Reg, InVals[i]);
+      Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Copy, Chain);
+      break;
     }
   }
 
@@ -2437,13 +2606,13 @@ X86TargetLowering::LowerFormalArguments(SDValue Chain,
 
   MachineModuleInfo &MMI = MF.getMMI();
   const Function *WinEHParent = nullptr;
-  if (IsWin64 && MMI.hasWinEHFuncInfo(Fn))
+  if (MMI.hasWinEHFuncInfo(Fn))
     WinEHParent = MMI.getWinEHParent(Fn);
   bool IsWinEHOutlined = WinEHParent && WinEHParent != Fn;
   bool IsWinEHParent = WinEHParent && WinEHParent == Fn;
 
   // Figure out if XMM registers are in use.
-  assert(!(MF.getTarget().Options.UseSoftFloat &&
+  assert(!(Subtarget->useSoftFloat() &&
            Fn->hasFnAttribute(Attribute::NoImplicitFloat)) &&
          "SSE register cannot be used when SSE is disabled!");
 
@@ -2499,11 +2668,11 @@ X86TargetLowering::LowerFormalArguments(SDValue Chain,
     // Store the integer parameter registers.
     SmallVector<SDValue, 8> MemOps;
     SDValue RSFIN = DAG.getFrameIndex(FuncInfo->getRegSaveFrameIndex(),
-                                      getPointerTy());
+                                      getPointerTy(DAG.getDataLayout()));
     unsigned Offset = FuncInfo->getVarArgsGPOffset();
     for (SDValue Val : LiveGPRs) {
-      SDValue FIN = DAG.getNode(ISD::ADD, dl, getPointerTy(), RSFIN,
-                                DAG.getIntPtrConstant(Offset));
+      SDValue FIN = DAG.getNode(ISD::ADD, dl, getPointerTy(DAG.getDataLayout()),
+                                RSFIN, DAG.getIntPtrConstant(Offset, dl));
       SDValue Store =
         DAG.getStore(Val.getValue(1), dl, Val, FIN,
                      MachinePointerInfo::getFixedStack(
@@ -2519,9 +2688,9 @@ X86TargetLowering::LowerFormalArguments(SDValue Chain,
       SaveXMMOps.push_back(Chain);
       SaveXMMOps.push_back(ALVal);
       SaveXMMOps.push_back(DAG.getIntPtrConstant(
-                             FuncInfo->getRegSaveFrameIndex()));
+                             FuncInfo->getRegSaveFrameIndex(), dl));
       SaveXMMOps.push_back(DAG.getIntPtrConstant(
-                             FuncInfo->getVarArgsFPOffset()));
+                             FuncInfo->getVarArgsFPOffset(), dl));
       SaveXMMOps.insert(SaveXMMOps.end(), LiveXMMRegs.begin(),
                         LiveXMMRegs.end());
       MemOps.push_back(DAG.getNode(X86ISD::VASTART_SAVE_XMM_REGS, dl,
@@ -2530,7 +2699,7 @@ X86TargetLowering::LowerFormalArguments(SDValue Chain,
 
     if (!MemOps.empty())
       Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, MemOps);
-  } else if (IsWinEHOutlined) {
+  } else if (IsWin64 && IsWinEHOutlined) {
     // Get to the caller-allocated home save location.  Add 8 to account
     // for the return address.
     int HomeOffset = TFI.getOffsetOfLocalArea() + 8;
@@ -2543,8 +2712,8 @@ X86TargetLowering::LowerFormalArguments(SDValue Chain,
 
     // Store the second integer parameter (rdx) into rsp+16 relative to the
     // stack pointer at the entry of the function.
-    SDValue RSFIN =
-        DAG.getFrameIndex(FuncInfo->getRegSaveFrameIndex(), getPointerTy());
+    SDValue RSFIN = DAG.getFrameIndex(FuncInfo->getRegSaveFrameIndex(),
+                                      getPointerTy(DAG.getDataLayout()));
     unsigned GPR = MF.addLiveIn(X86::RDX, &X86::GR64RegClass);
     SDValue Val = DAG.getCopyFromReg(Chain, dl, GPR, MVT::i64);
     Chain = DAG.getStore(
@@ -2618,14 +2787,21 @@ X86TargetLowering::LowerFormalArguments(SDValue Chain,
   FuncInfo->setArgumentStackSize(StackSize);
 
   if (IsWinEHParent) {
-    int UnwindHelpFI = MFI->CreateStackObject(8, 8, /*isSS=*/false);
-    SDValue StackSlot = DAG.getFrameIndex(UnwindHelpFI, MVT::i64);
-    MMI.getWinEHFuncInfo(MF.getFunction()).UnwindHelpFrameIdx = UnwindHelpFI;
-    SDValue Neg2 = DAG.getConstant(-2, MVT::i64);
-    Chain = DAG.getStore(Chain, dl, Neg2, StackSlot,
-                         MachinePointerInfo::getFixedStack(UnwindHelpFI),
-                         /*isVolatile=*/true,
-                         /*isNonTemporal=*/false, /*Alignment=*/0);
+    if (Is64Bit) {
+      int UnwindHelpFI = MFI->CreateStackObject(8, 8, /*isSS=*/false);
+      SDValue StackSlot = DAG.getFrameIndex(UnwindHelpFI, MVT::i64);
+      MMI.getWinEHFuncInfo(MF.getFunction()).UnwindHelpFrameIdx = UnwindHelpFI;
+      SDValue Neg2 = DAG.getConstant(-2, dl, MVT::i64);
+      Chain = DAG.getStore(Chain, dl, Neg2, StackSlot,
+                           MachinePointerInfo::getFixedStack(UnwindHelpFI),
+                           /*isVolatile=*/true,
+                           /*isNonTemporal=*/false, /*Alignment=*/0);
+    } else {
+      // Functions using Win32 EH are considered to have opaque SP adjustments
+      // to force local variables to be addressed from the frame or base
+      // pointers.
+      MFI->setHasOpaqueSPAdjustment(true);
+    }
   }
 
   return Chain;
@@ -2638,8 +2814,9 @@ X86TargetLowering::LowerMemOpCallTo(SDValue Chain,
                                     const CCValAssign &VA,
                                     ISD::ArgFlagsTy Flags) const {
   unsigned LocMemOffset = VA.getLocMemOffset();
-  SDValue PtrOff = DAG.getIntPtrConstant(LocMemOffset);
-  PtrOff = DAG.getNode(ISD::ADD, dl, getPointerTy(), StackPtr, PtrOff);
+  SDValue PtrOff = DAG.getIntPtrConstant(LocMemOffset, dl);
+  PtrOff = DAG.getNode(ISD::ADD, dl, getPointerTy(DAG.getDataLayout()),
+                       StackPtr, PtrOff);
   if (Flags.isByVal())
     return CreateCopyOfByValArgument(Arg, PtrOff, Chain, Flags, DAG, dl);
 
@@ -2656,7 +2833,7 @@ X86TargetLowering::EmitTailCallLoadRetAddr(SelectionDAG &DAG,
                                            bool IsTailCall, bool Is64Bit,
                                            int FPDiff, SDLoc dl) const {
   // Adjust the Return address stack slot.
-  EVT VT = getPointerTy();
+  EVT VT = getPointerTy(DAG.getDataLayout());
   OutRetAddr = getReturnAddressFrameIndex(DAG);
 
   // Load the "old" Return address.
@@ -2684,6 +2861,18 @@ static SDValue EmitTailCallStoreRetAddr(SelectionDAG &DAG, MachineFunction &MF,
   return Chain;
 }
 
+/// Returns a vector_shuffle mask for an movs{s|d}, movd
+/// operation of specified width.
+static SDValue getMOVL(SelectionDAG &DAG, SDLoc dl, EVT VT, SDValue V1,
+                       SDValue V2) {
+  unsigned NumElems = VT.getVectorNumElements();
+  SmallVector<int, 8> Mask;
+  Mask.push_back(NumElems);
+  for (unsigned i = 1; i != NumElems; ++i)
+    Mask.push_back(i);
+  return DAG.getVectorShuffle(VT, dl, V1, V2, &Mask[0]);
+}
+
 SDValue
 X86TargetLowering::LowerCall(TargetLowering::CallLoweringInfo &CLI,
                              SmallVectorImpl<SDValue> &InVals) const {
@@ -2704,10 +2893,24 @@ X86TargetLowering::LowerCall(TargetLowering::CallLoweringInfo &CLI,
   StructReturnType SR = callIsStructReturn(Outs);
   bool IsSibcall      = false;
   X86MachineFunctionInfo *X86Info = MF.getInfo<X86MachineFunctionInfo>();
+  auto Attr = MF.getFunction()->getFnAttribute("disable-tail-calls");
 
-  if (MF.getTarget().Options.DisableTailCalls)
+  if (Attr.getValueAsString() == "true")
     isTailCall = false;
 
+  if (Subtarget->isPICStyleGOT() &&
+      !MF.getTarget().Options.GuaranteedTailCallOpt) {
+    // If we are using a GOT, disable tail calls to external symbols with
+    // default visibility. Tail calling such a symbol requires using a GOT
+    // relocation, which forces early binding of the symbol. This breaks code
+    // that require lazy function symbol resolution. Using musttail or
+    // GuaranteedTailCallOpt will override this.
+    GlobalAddressSDNode *G = dyn_cast<GlobalAddressSDNode>(Callee);
+    if (!G || (!G->getGlobal()->hasLocalLinkage() &&
+               G->getGlobal()->hasDefaultVisibility()))
+      isTailCall = false;
+  }
+
   bool IsMustTail = CLI.CS && CLI.CS->isMustTailCall();
   if (IsMustTail) {
     // Force this to be a tail call.  The verifier rules are enough to ensure
@@ -2784,7 +2987,7 @@ X86TargetLowering::LowerCall(TargetLowering::CallLoweringInfo &CLI,
 
   if (!IsSibcall)
     Chain = DAG.getCALLSEQ_START(
-        Chain, DAG.getIntPtrConstant(NumBytesToPush, true), dl);
+        Chain, DAG.getIntPtrConstant(NumBytesToPush, dl, true), dl);
 
   SDValue RetAddrFrIdx;
   // Load return address for tail calls.
@@ -2821,16 +3024,19 @@ X86TargetLowering::LowerCall(TargetLowering::CallLoweringInfo &CLI,
       Arg = DAG.getNode(ISD::ZERO_EXTEND, dl, RegVT, Arg);
       break;
     case CCValAssign::AExt:
-      if (RegVT.is128BitVector()) {
+      if (Arg.getValueType().isVector() &&
+          Arg.getValueType().getScalarType() == MVT::i1)
+        Arg = DAG.getNode(ISD::SIGN_EXTEND, dl, RegVT, Arg);
+      else if (RegVT.is128BitVector()) {
         // Special case: passing MMX values in XMM registers.
-        Arg = DAG.getNode(ISD::BITCAST, dl, MVT::i64, Arg);
+        Arg = DAG.getBitcast(MVT::i64, Arg);
         Arg = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, MVT::v2i64, Arg);
         Arg = getMOVL(DAG, dl, MVT::v2i64, DAG.getUNDEF(MVT::v2i64), Arg);
       } else
         Arg = DAG.getNode(ISD::ANY_EXTEND, dl, RegVT, Arg);
       break;
     case CCValAssign::BCvt:
-      Arg = DAG.getNode(ISD::BITCAST, dl, RegVT, Arg);
+      Arg = DAG.getBitcast(RegVT, Arg);
       break;
     case CCValAssign::Indirect: {
       // Store the argument.
@@ -2862,8 +3068,8 @@ X86TargetLowering::LowerCall(TargetLowering::CallLoweringInfo &CLI,
     } else if (!IsSibcall && (!isTailCall || isByVal)) {
       assert(VA.isMemLoc());
       if (!StackPtr.getNode())
-        StackPtr = DAG.getCopyFromReg(Chain, dl, X86StackPtr, // @LOCALMOD
-                                      getPointerTy());
+        StackPtr = DAG.getCopyFromReg(Chain, dl, RegInfo->getStackRegister(),
+                                      getPointerTy(DAG.getDataLayout()));
       MemOpChains.push_back(LowerMemOpCallTo(Chain, StackPtr, Arg,
                                              dl, DAG, VA, Flags));
     }
@@ -2876,8 +3082,9 @@ X86TargetLowering::LowerCall(TargetLowering::CallLoweringInfo &CLI,
     // ELF / PIC requires GOT in the EBX register before function calls via PLT
     // GOT pointer.
     if (!isTailCall) {
-      RegsToPass.push_back(std::make_pair(unsigned(X86::EBX),
-               DAG.getNode(X86ISD::GlobalBaseReg, SDLoc(), getPointerTy())));
+      RegsToPass.push_back(std::make_pair(
+          unsigned(X86::EBX), DAG.getNode(X86ISD::GlobalBaseReg, SDLoc(),
+                                          getPointerTy(DAG.getDataLayout()))));
     } else {
       // If we are tail calling and generating PIC/GOT style code load the
       // address of the callee into ECX. The value in ecx is used as target of
@@ -2889,8 +3096,8 @@ X86TargetLowering::LowerCall(TargetLowering::CallLoweringInfo &CLI,
 
       // Note: The actual moving to ECX is done further down.
       GlobalAddressSDNode *G = dyn_cast<GlobalAddressSDNode>(Callee);
-      if (G && !G->getGlobal()->hasHiddenVisibility() &&
-          !G->getGlobal()->hasProtectedVisibility())
+      if (G && !G->getGlobal()->hasLocalLinkage() &&
+          G->getGlobal()->hasDefaultVisibility())
         Callee = LowerGlobalAddress(Callee, DAG);
       else if (isa<ExternalSymbolSDNode>(Callee))
         Callee = LowerExternalSymbol(Callee, DAG);
@@ -2916,7 +3123,8 @@ X86TargetLowering::LowerCall(TargetLowering::CallLoweringInfo &CLI,
            && "SSE registers cannot be used when SSE is disabled");
 
     RegsToPass.push_back(std::make_pair(unsigned(X86::AL),
-                                        DAG.getConstant(NumXMMRegs, MVT::i8)));
+                                        DAG.getConstant(NumXMMRegs, dl,
+                                                        MVT::i8)));
   }
 
   if (isVarArg && IsMustTail) {
@@ -2956,16 +3164,16 @@ X86TargetLowering::LowerCall(TargetLowering::CallLoweringInfo &CLI,
       int32_t Offset = VA.getLocMemOffset()+FPDiff;
       uint32_t OpSize = (VA.getLocVT().getSizeInBits()+7)/8;
       FI = MF.getFrameInfo()->CreateFixedObject(OpSize, Offset, true);
-      FIN = DAG.getFrameIndex(FI, getPointerTy());
+      FIN = DAG.getFrameIndex(FI, getPointerTy(DAG.getDataLayout()));
 
       if (Flags.isByVal()) {
         // Copy relative to framepointer.
-        SDValue Source = DAG.getIntPtrConstant(VA.getLocMemOffset());
+        SDValue Source = DAG.getIntPtrConstant(VA.getLocMemOffset(), dl);
         if (!StackPtr.getNode())
-          StackPtr = DAG.getCopyFromReg(Chain, dl,
-                                        X86StackPtr, // @LOCALMOD
-                                        getPointerTy());
-        Source = DAG.getNode(ISD::ADD, dl, getPointerTy(), StackPtr, Source);
+          StackPtr = DAG.getCopyFromReg(Chain, dl, RegInfo->getStackRegister(),
+                                        getPointerTy(DAG.getDataLayout()));
+        Source = DAG.getNode(ISD::ADD, dl, getPointerTy(DAG.getDataLayout()),
+                             StackPtr, Source);
 
         MemOpChains2.push_back(CreateCopyOfByValArgument(Source, FIN,
                                                          ArgChain,
@@ -2984,8 +3192,8 @@ X86TargetLowering::LowerCall(TargetLowering::CallLoweringInfo &CLI,
 
     // Store the return address to the appropriate stack slot.
     Chain = EmitTailCallStoreRetAddr(DAG, MF, Chain, RetAddrFrIdx,
-                                     getPointerTy(), RegInfo->getSlotSize(),
-                                     FPDiff, dl);
+                                     getPointerTy(DAG.getDataLayout()),
+                                     RegInfo->getSlotSize(), FPDiff, dl);
   }
 
   // Build a sequence of copy-to-reg nodes chained together with token chain
@@ -3026,7 +3234,7 @@ X86TargetLowering::LowerCall(TargetLowering::CallLoweringInfo &CLI,
           GV->hasDefaultVisibility() && !GV->hasLocalLinkage()) {
         OpFlags = X86II::MO_PLT;
       } else if (Subtarget->isPICStyleStubAny() &&
-                 (GV->isDeclaration() || GV->isWeakForLinker()) &&
+                 !GV->isStrongDefinitionForLinker() &&
                  (!Subtarget->getTargetTriple().isMacOSX() ||
                   Subtarget->getTargetTriple().isMacOSXVersionLT(10, 5))) {
         // PC-relative references to external symbols should go through $stub,
@@ -3043,17 +3251,18 @@ X86TargetLowering::LowerCall(TargetLowering::CallLoweringInfo &CLI,
         ExtraLoad = true;
       }
 
-      Callee = DAG.getTargetGlobalAddress(GV, dl, getPointerTy(),
-                                          G->getOffset(), OpFlags);
+      Callee = DAG.getTargetGlobalAddress(
+          GV, dl, getPointerTy(DAG.getDataLayout()), G->getOffset(), OpFlags);
 
       // Add a wrapper if needed.
       if (WrapperKind != ISD::DELETED_NODE)
-        Callee = DAG.getNode(X86ISD::WrapperRIP, dl, getPointerTy(), Callee);
+        Callee = DAG.getNode(X86ISD::WrapperRIP, dl,
+                             getPointerTy(DAG.getDataLayout()), Callee);
       // Add extra indirection if needed.
       if (ExtraLoad)
-        Callee = DAG.getLoad(getPointerTy(), dl, DAG.getEntryNode(), Callee,
-                             MachinePointerInfo::getGOT(),
-                             false, false, false, 0);
+        Callee = DAG.getLoad(
+            getPointerTy(DAG.getDataLayout()), dl, DAG.getEntryNode(), Callee,
+            MachinePointerInfo::getGOT(), false, false, false, 0);
     }
   } else if (ExternalSymbolSDNode *S = dyn_cast<ExternalSymbolSDNode>(Callee)) {
     unsigned char OpFlags = 0;
@@ -3072,8 +3281,8 @@ X86TargetLowering::LowerCall(TargetLowering::CallLoweringInfo &CLI,
       OpFlags = X86II::MO_DARWIN_STUB;
     }
 
-    Callee = DAG.getTargetExternalSymbol(S->getSymbol(), getPointerTy(),
-                                         OpFlags);
+    Callee = DAG.getTargetExternalSymbol(
+        S->getSymbol(), getPointerTy(DAG.getDataLayout()), OpFlags);
   } else if (Subtarget->isTarget64BitILP32() &&
              Callee->getValueType(0) == MVT::i32) {
     // Zero-extend the 32-bit Callee address into a 64-bit according to x32 ABI
@@ -3086,8 +3295,8 @@ X86TargetLowering::LowerCall(TargetLowering::CallLoweringInfo &CLI,
 
   if (!IsSibcall && isTailCall) {
     Chain = DAG.getCALLSEQ_END(Chain,
-                               DAG.getIntPtrConstant(NumBytesToPop, true),
-                               DAG.getIntPtrConstant(0, true), InFlag, dl);
+                               DAG.getIntPtrConstant(NumBytesToPop, dl, true),
+                               DAG.getIntPtrConstant(0, dl, true), InFlag, dl);
     InFlag = Chain.getValue(1);
   }
 
@@ -3095,7 +3304,7 @@ X86TargetLowering::LowerCall(TargetLowering::CallLoweringInfo &CLI,
   Ops.push_back(Callee);
 
   if (isTailCall)
-    Ops.push_back(DAG.getConstant(FPDiff, MVT::i32));
+    Ops.push_back(DAG.getConstant(FPDiff, dl, MVT::i32));
 
   // Add argument registers to the end of the list so that they are known live
   // into the call.
@@ -3104,9 +3313,24 @@ X86TargetLowering::LowerCall(TargetLowering::CallLoweringInfo &CLI,
                                   RegsToPass[i].second.getValueType()));
 
   // Add a register mask operand representing the call-preserved registers.
-  const TargetRegisterInfo *TRI = Subtarget->getRegisterInfo();
-  const uint32_t *Mask = TRI->getCallPreservedMask(MF, CallConv);
+  const uint32_t *Mask = RegInfo->getCallPreservedMask(MF, CallConv);
   assert(Mask && "Missing call preserved mask for calling convention");
+
+  // If this is an invoke in a 32-bit function using an MSVC personality, assume
+  // the function clobbers all registers. If an exception is thrown, the runtime
+  // will not restore CSRs.
+  // FIXME: Model this more precisely so that we can register allocate across
+  // the normal edge and spill and fill across the exceptional edge.
+  if (!Is64Bit && CLI.CS && CLI.CS->isInvoke()) {
+    const Function *CallerFn = MF.getFunction();
+    EHPersonality Pers =
+        CallerFn->hasPersonalityFn()
+            ? classifyEHPersonality(CallerFn->getPersonalityFn())
+            : EHPersonality::Unknown;
+    if (isMSVCEHPersonality(Pers))
+      Mask = RegInfo->getNoPreservedMask();
+  }
+
   Ops.push_back(DAG.getRegisterMask(Mask));
 
   if (InFlag.getNode())
@@ -3119,6 +3343,7 @@ X86TargetLowering::LowerCall(TargetLowering::CallLoweringInfo &CLI,
     // This isn't right, although it's probably harmless on x86; liveouts
     // should be computed from returns not tail calls.  Consider a void
     // function making a tail call to a function returning int.
+    MF.getFrameInfo()->setHasTailCall();
     return DAG.getNode(X86ISD::TC_RETURN, dl, NodeTys, Ops);
   }
 
@@ -3144,8 +3369,8 @@ X86TargetLowering::LowerCall(TargetLowering::CallLoweringInfo &CLI,
   // Returns a flag for retval copy to use.
   if (!IsSibcall) {
     Chain = DAG.getCALLSEQ_END(Chain,
-                               DAG.getIntPtrConstant(NumBytesToPop, true),
-                               DAG.getIntPtrConstant(NumBytesForCalleeToPop,
+                               DAG.getIntPtrConstant(NumBytesToPop, dl, true),
+                               DAG.getIntPtrConstant(NumBytesForCalleeToPop, dl,
                                                      true),
                                InFlag, dl);
     InFlag = Chain.getValue(1);
@@ -3188,8 +3413,8 @@ X86TargetLowering::LowerCall(TargetLowering::CallLoweringInfo &CLI,
 //    EDI
 //    local1 ..
 
-/// GetAlignedArgumentStackSize - Make the stack size align e.g 16n + 12 aligned
-/// for a 16 byte align requirement.
+/// Make the stack size align e.g 16n + 12 aligned for a 16-byte align
+/// requirement.
 unsigned
 X86TargetLowering::GetAlignedArgumentStackSize(unsigned StackSize,
                                                SelectionDAG& DAG) const {
@@ -3210,9 +3435,8 @@ X86TargetLowering::GetAlignedArgumentStackSize(unsigned StackSize,
   return Offset;
 }
 
-/// MatchingStackOffset - Return true if the given stack call argument is
-/// already available in the same position (relatively) of the caller's
-/// incoming argument stack.
+/// Return true if the given stack call argument is already available in the
+/// same position (relatively) of the caller's incoming argument stack.
 static
 bool MatchingStackOffset(SDValue Arg, unsigned Offset, ISD::ArgFlagsTy Flags,
                          MachineFrameInfo *MFI, const MachineRegisterInfo *MRI,
@@ -3265,9 +3489,8 @@ bool MatchingStackOffset(SDValue Arg, unsigned Offset, ISD::ArgFlagsTy Flags,
   return Offset == MFI->getObjectOffset(FI) && Bytes == MFI->getObjectSize(FI);
 }
 
-/// IsEligibleForTailCallOptimization - Check whether the call is eligible
-/// for tail call optimization. Targets which want to do tail call
-/// optimization should implement this function.
+/// Check whether the call is eligible for tail call optimization. Targets
+/// that want to do tail call optimization should implement this function.
 bool
 X86TargetLowering::IsEligibleForTailCallOptimization(SDValue Callee,
                                                      CallingConv::ID CalleeCC,
@@ -3349,23 +3572,6 @@ X86TargetLowering::IsEligibleForTailCallOptimization(SDValue Callee,
     for (unsigned i = 0, e = ArgLocs.size(); i != e; ++i)
       if (!ArgLocs[i].isRegLoc())
         return false;
-    // @LOCALMOD-BEGIN
-    // Do not optimize vararg calls with 6 arguments. TCrets use a reduced
-    // subset of registers because they cannot use callee-saves. When using
-    // sandbox base hiding (with r11 reserved) there are not enough registers
-    // because all of the available registers are used for arguments (%al is
-    // also used for the float/vector register count).
-    // Another option to fix this would be to replace r11 with r10 in the
-    // GR64_TC register class if we decide we don't care about supporting "nest"
-    // parameters. It might even be technically possible to actually use r11
-    // because for tcreturn it's used as a jump target and not read, but I don't
-    // know if reserved regs can be set differently for different purposes like
-    // that.
-    if (Subtarget->isTargetNaCl64() &&
-        FlagHideSandboxBase &&
-        ArgLocs.size() > 5)
-      return false;
-    // @LOCALMOD-END
   }
 
   // If the call result is in ST0 / ST1, it needs to be popped off the x87
@@ -3549,7 +3755,8 @@ static SDValue getTargetShuffleNode(unsigned Opc, SDLoc dl, EVT VT,
   case X86ISD::PSHUFLW:
   case X86ISD::VPERMILPI:
   case X86ISD::VPERMI:
-    return DAG.getNode(Opc, dl, VT, V1, DAG.getConstant(TargetMask, MVT::i8));
+    return DAG.getNode(Opc, dl, VT, V1,
+                       DAG.getConstant(TargetMask, dl, MVT::i8));
   }
 }
 
@@ -3585,8 +3792,7 @@ SDValue X86TargetLowering::getReturnAddressFrameIndex(SelectionDAG &DAG) const {
     FuncInfo->setRAIndex(ReturnAddrIndex);
   }
 
-  return DAG.getFrameIndex(ReturnAddrIndex, // @LOCALMOD
-                           Subtarget->is64Bit() ? MVT::i64 : MVT::i32);
+  return DAG.getFrameIndex(ReturnAddrIndex, getPointerTy(DAG.getDataLayout()));
 }
 
 bool X86::isOffsetSuitableForCodeModel(int64_t Offset, CodeModel::Model M,
@@ -3619,8 +3825,8 @@ bool X86::isOffsetSuitableForCodeModel(int64_t Offset, CodeModel::Model M,
   return false;
 }
 
-/// isCalleePop - Determines whether the callee is required to pop its
-/// own arguments. Callee pop is necessary to support tail calls.
+/// Determines whether the callee is required to pop its own arguments.
+/// Callee pop is necessary to support tail calls.
 bool X86::isCalleePop(CallingConv::ID CallingConv,
                       bool is64Bit, bool IsVarArg, bool TailCallOpt) {
   switch (CallingConv) {
@@ -3657,16 +3863,16 @@ static bool isX86CCUnsigned(unsigned X86CC) {
   llvm_unreachable("covered switch fell through?!");
 }
 
-/// TranslateX86CC - do a one to one translation of a ISD::CondCode to the X86
-/// specific condition code, returning the condition code and the LHS/RHS of the
+/// Do a one-to-one translation of a ISD::CondCode to the X86-specific
+/// condition code, returning the condition code and the LHS/RHS of the
 /// comparison to make.
-static unsigned TranslateX86CC(ISD::CondCode SetCCOpcode, bool isFP,
+static unsigned TranslateX86CC(ISD::CondCode SetCCOpcode, SDLoc DL, bool isFP,
                                SDValue &LHS, SDValue &RHS, SelectionDAG &DAG) {
   if (!isFP) {
     if (ConstantSDNode *RHSC = dyn_cast<ConstantSDNode>(RHS)) {
       if (SetCCOpcode == ISD::SETGT && RHSC->isAllOnesValue()) {
         // X > -1   -> X == 0, jump !sign.
-        RHS = DAG.getConstant(0, RHS.getValueType());
+        RHS = DAG.getConstant(0, DL, RHS.getValueType());
         return X86::COND_NS;
       }
       if (SetCCOpcode == ISD::SETLT && RHSC->isNullValue()) {
@@ -3675,7 +3881,7 @@ static unsigned TranslateX86CC(ISD::CondCode SetCCOpcode, bool isFP,
       }
       if (SetCCOpcode == ISD::SETLT && RHSC->getZExtValue() == 1) {
         // X < 1   -> X <= 0
-        RHS = DAG.getConstant(0, RHS.getValueType());
+        RHS = DAG.getConstant(0, DL, RHS.getValueType());
         return X86::COND_LE;
       }
     }
@@ -3745,8 +3951,8 @@ static unsigned TranslateX86CC(ISD::CondCode SetCCOpcode, bool isFP,
   }
 }
 
-/// hasFPCMov - is there a floating point cmov for the specific X86 condition
-/// code. Current x86 isa includes the following FP cmov instructions:
+/// Is there a floating point cmov for the specific X86 condition code?
+/// Current x86 isa includes the following FP cmov instructions:
 /// fcmovb, fcomvbe, fcomve, fcmovu, fcmovae, fcmova, fcmovne, fcmovnu.
 static bool hasFPCMov(unsigned X86CC) {
   switch (X86CC) {
@@ -3764,7 +3970,7 @@ static bool hasFPCMov(unsigned X86CC) {
   }
 }
 
-/// isFPImmLegal - Returns true if the target can instruction select the
+/// Returns true if the target can instruction select the
 /// specified FP immediate natively. If false, the legalizer will
 /// materialize the FP immediate as a load from a constant pool.
 bool X86TargetLowering::isFPImmLegal(const APFloat &Imm, EVT VT) const {
@@ -3817,19 +4023,27 @@ bool X86TargetLowering::isCheapToSpeculateCtlz() const {
   return Subtarget->hasLZCNT();
 }
 
-/// isUndefOrInRange - Return true if Val is undef or if its value falls within
-/// the specified range (L, H].
+/// Return true if every element in Mask, beginning
+/// from position Pos and ending in Pos+Size is undef.
+static bool isUndefInRange(ArrayRef<int> Mask, unsigned Pos, unsigned Size) {
+  for (unsigned i = Pos, e = Pos + Size; i != e; ++i)
+    if (0 <= Mask[i])
+      return false;
+  return true;
+}
+
+/// Return true if Val is undef or if its value falls within the
+/// specified range (L, H].
 static bool isUndefOrInRange(int Val, int Low, int Hi) {
   return (Val < 0) || (Val >= Low && Val < Hi);
 }
 
-/// isUndefOrEqual - Val is either less than zero (undef) or equal to the
-/// specified value.
+/// Val is either less than zero (undef) or equal to the specified value.
 static bool isUndefOrEqual(int Val, int CmpVal) {
   return (Val < 0 || Val == CmpVal);
 }
 
-/// isSequentialOrUndefInRange - Return true if every element in Mask, beginning
+/// Return true if every element in Mask, beginning
 /// from position Pos and ending in Pos+Size, falls within the specified
 /// sequential range (Low, Low+Size]. or is undef.
 static bool isSequentialOrUndefInRange(ArrayRef<int> Mask,
@@ -3840,9 +4054,8 @@ static bool isSequentialOrUndefInRange(ArrayRef<int> Mask,
   return true;
 }
 
-/// isVEXTRACTIndex - Return true if the specified
-/// EXTRACT_SUBVECTOR operand specifies a vector extract that is
-/// suitable for instruction that extract 128 or 256 bit vectors
+/// Return true if the specified EXTRACT_SUBVECTOR operand specifies a vector
+/// extract that is suitable for instruction that extract 128 or 256 bit vectors
 static bool isVEXTRACTIndex(SDNode *N, unsigned vecWidth) {
   assert((vecWidth == 128 || vecWidth == 256) && "Unexpected vector width");
   if (!isa<ConstantSDNode>(N->getOperand(1).getNode()))
@@ -3859,7 +4072,7 @@ static bool isVEXTRACTIndex(SDNode *N, unsigned vecWidth) {
   return Result;
 }
 
-/// isVINSERTIndex - Return true if the specified INSERT_SUBVECTOR
+/// Return true if the specified INSERT_SUBVECTOR
 /// operand specifies a subvector insert that is suitable for input to
 /// insertion of 128 or 256-bit subvectors
 static bool isVINSERTIndex(SDNode *N, unsigned vecWidth) {
@@ -3923,42 +4136,37 @@ static unsigned getInsertVINSERTImmediate(SDNode *N, unsigned vecWidth) {
   return Index / NumElemsPerChunk;
 }
 
-/// getExtractVEXTRACT128Immediate - Return the appropriate immediate
-/// to extract the specified EXTRACT_SUBVECTOR index with VEXTRACTF128
-/// and VINSERTI128 instructions.
+/// Return the appropriate immediate to extract the specified
+/// EXTRACT_SUBVECTOR index with VEXTRACTF128 and VINSERTI128 instructions.
 unsigned X86::getExtractVEXTRACT128Immediate(SDNode *N) {
   return getExtractVEXTRACTImmediate(N, 128);
 }
 
-/// getExtractVEXTRACT256Immediate - Return the appropriate immediate
-/// to extract the specified EXTRACT_SUBVECTOR index with VEXTRACTF64x4
-/// and VINSERTI64x4 instructions.
+/// Return the appropriate immediate to extract the specified
+/// EXTRACT_SUBVECTOR index with VEXTRACTF64x4 and VINSERTI64x4 instructions.
 unsigned X86::getExtractVEXTRACT256Immediate(SDNode *N) {
   return getExtractVEXTRACTImmediate(N, 256);
 }
 
-/// getInsertVINSERT128Immediate - Return the appropriate immediate
-/// to insert at the specified INSERT_SUBVECTOR index with VINSERTF128
-/// and VINSERTI128 instructions.
+/// Return the appropriate immediate to insert at the specified
+/// INSERT_SUBVECTOR index with VINSERTF128 and VINSERTI128 instructions.
 unsigned X86::getInsertVINSERT128Immediate(SDNode *N) {
   return getInsertVINSERTImmediate(N, 128);
 }
 
-/// getInsertVINSERT256Immediate - Return the appropriate immediate
-/// to insert at the specified INSERT_SUBVECTOR index with VINSERTF46x4
-/// and VINSERTI64x4 instructions.
+/// Return the appropriate immediate to insert at the specified
+/// INSERT_SUBVECTOR index with VINSERTF46x4 and VINSERTI64x4 instructions.
 unsigned X86::getInsertVINSERT256Immediate(SDNode *N) {
   return getInsertVINSERTImmediate(N, 256);
 }
 
-/// isZero - Returns true if Elt is a constant integer zero
+/// Returns true if Elt is a constant integer zero
 static bool isZero(SDValue V) {
   ConstantSDNode *C = dyn_cast<ConstantSDNode>(V);
   return C && C->isNullValue();
 }
 
-/// isZeroNode - Returns true if Elt is a constant zero or a floating point
-/// constant +0.0.
+/// Returns true if Elt is a constant zero or a floating point constant +0.0.
 bool X86::isZeroNode(SDValue Elt) {
   if (isZero(Elt))
     return true;
@@ -3967,8 +4175,7 @@ bool X86::isZeroNode(SDValue Elt) {
   return false;
 }
 
-/// getZeroVector - Returns a vector of specified type with all zero elements.
-///
+/// Returns a vector of specified type with all zero elements.
 static SDValue getZeroVector(EVT VT, const X86Subtarget *Subtarget,
                              SelectionDAG &DAG, SDLoc dl) {
   assert(VT.isVector() && "Expected a vector type");
@@ -3978,26 +4185,26 @@ static SDValue getZeroVector(EVT VT, const X86Subtarget *Subtarget,
   SDValue Vec;
   if (VT.is128BitVector()) {  // SSE
     if (Subtarget->hasSSE2()) {  // SSE2
-      SDValue Cst = DAG.getConstant(0, MVT::i32);
+      SDValue Cst = DAG.getConstant(0, dl, MVT::i32);
       Vec = DAG.getNode(ISD::BUILD_VECTOR, dl, MVT::v4i32, Cst, Cst, Cst, Cst);
     } else { // SSE1
-      SDValue Cst = DAG.getConstantFP(+0.0, MVT::f32);
+      SDValue Cst = DAG.getConstantFP(+0.0, dl, MVT::f32);
       Vec = DAG.getNode(ISD::BUILD_VECTOR, dl, MVT::v4f32, Cst, Cst, Cst, Cst);
     }
   } else if (VT.is256BitVector()) { // AVX
     if (Subtarget->hasInt256()) { // AVX2
-      SDValue Cst = DAG.getConstant(0, MVT::i32);
+      SDValue Cst = DAG.getConstant(0, dl, MVT::i32);
       SDValue Ops[] = { Cst, Cst, Cst, Cst, Cst, Cst, Cst, Cst };
       Vec = DAG.getNode(ISD::BUILD_VECTOR, dl, MVT::v8i32, Ops);
     } else {
       // 256-bit logic and arithmetic instructions in AVX are all
       // floating-point, no support for integer ops. Emit fp zeroed vectors.
-      SDValue Cst = DAG.getConstantFP(+0.0, MVT::f32);
+      SDValue Cst = DAG.getConstantFP(+0.0, dl, MVT::f32);
       SDValue Ops[] = { Cst, Cst, Cst, Cst, Cst, Cst, Cst, Cst };
       Vec = DAG.getNode(ISD::BUILD_VECTOR, dl, MVT::v8f32, Ops);
     }
   } else if (VT.is512BitVector()) { // AVX-512
-      SDValue Cst = DAG.getConstant(0, MVT::i32);
+      SDValue Cst = DAG.getConstant(0, dl, MVT::i32);
       SDValue Ops[] = { Cst, Cst, Cst, Cst, Cst, Cst, Cst, Cst,
                         Cst, Cst, Cst, Cst, Cst, Cst, Cst, Cst };
       Vec = DAG.getNode(ISD::BUILD_VECTOR, dl, MVT::v16i32, Ops);
@@ -4007,13 +4214,13 @@ static SDValue getZeroVector(EVT VT, const X86Subtarget *Subtarget,
             && "Unexpected vector type");
     assert((Subtarget->hasVLX() || VT.getVectorNumElements() >= 8)
             && "Unexpected vector type");
-    SDValue Cst = DAG.getConstant(0, MVT::i1);
+    SDValue Cst = DAG.getConstant(0, dl, MVT::i1);
     SmallVector<SDValue, 64> Ops(VT.getVectorNumElements(), Cst);
     return DAG.getNode(ISD::BUILD_VECTOR, dl, VT, Ops);
   } else
     llvm_unreachable("Unexpected vector type");
 
-  return DAG.getNode(ISD::BITCAST, dl, VT, Vec);
+  return DAG.getBitcast(VT, Vec);
 }
 
 static SDValue ExtractSubVector(SDValue Vec, unsigned IdxVal,
@@ -4045,7 +4252,7 @@ static SDValue ExtractSubVector(SDValue Vec, unsigned IdxVal,
                        makeArrayRef(Vec->op_begin() + NormalizedIdxVal,
                                     ElemsPerChunk));
 
-  SDValue VecIdx = DAG.getIntPtrConstant(NormalizedIdxVal);
+  SDValue VecIdx = DAG.getIntPtrConstant(NormalizedIdxVal, dl);
   return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, ResultVT, Vec, VecIdx);
 }
 
@@ -4089,7 +4296,7 @@ static SDValue InsertSubVector(SDValue Result, SDValue Vec,
   unsigned NormalizedIdxVal = (((IdxVal * ElVT.getSizeInBits())/vectorWidth)
                                * ElemsPerChunk);
 
-  SDValue VecIdx = DAG.getIntPtrConstant(NormalizedIdxVal);
+  SDValue VecIdx = DAG.getIntPtrConstant(NormalizedIdxVal, dl);
   return DAG.getNode(ISD::INSERT_SUBVECTOR, dl, ResultVT, Result, Vec, VecIdx);
 }
 
@@ -4111,7 +4318,7 @@ static SDValue Insert128BitVector(SDValue Result, SDValue Vec, unsigned IdxVal,
   if (IdxVal == 0 && Result.getValueType().is256BitVector() &&
       Result.getOpcode() != ISD::UNDEF) {
     EVT ResultVT = Result.getValueType();
-    SDValue ZeroIndex = DAG.getIntPtrConstant(0);
+    SDValue ZeroIndex = DAG.getIntPtrConstant(0, dl);
     SDValue Undef = DAG.getUNDEF(ResultVT);
     SDValue Vec256 = DAG.getNode(ISD::INSERT_SUBVECTOR, dl, ResultVT, Undef,
                                  Vec, ZeroIndex);
@@ -4123,7 +4330,7 @@ static SDValue Insert128BitVector(SDValue Result, SDValue Vec, unsigned IdxVal,
       unsigned ScalarSize = ScalarType.getSizeInBits();
       assert((ScalarSize == 64 || ScalarSize == 32) && "Unknown float type");
       unsigned MaskVal = (ScalarSize == 64) ? 0x03 : 0x0f;
-      SDValue Mask = DAG.getConstant(MaskVal, MVT::i8);
+      SDValue Mask = DAG.getConstant(MaskVal, dl, MVT::i8);
       return DAG.getNode(X86ISD::BLENDI, dl, ResultVT, Result, Vec256, Mask);
     }
 
@@ -4139,10 +4346,10 @@ static SDValue Insert128BitVector(SDValue Result, SDValue Vec, unsigned IdxVal,
     // will be created by InsertSubVector().
     MVT CastVT = Subtarget.hasAVX2() ? MVT::v8i32 : MVT::v8f32;
 
-    SDValue Mask = DAG.getConstant(0x0f, MVT::i8);
-    Vec256 = DAG.getNode(ISD::BITCAST, dl, CastVT, Vec256);
+    SDValue Mask = DAG.getConstant(0x0f, dl, MVT::i8);
+    Vec256 = DAG.getBitcast(CastVT, Vec256);
     Vec256 = DAG.getNode(X86ISD::BLENDI, dl, CastVT, Result, Vec256, Mask);
-    return DAG.getNode(ISD::BITCAST, dl, ResultVT, Vec256);
+    return DAG.getBitcast(ResultVT, Vec256);
   }
 
   return InsertSubVector(Result, Vec, IdxVal, DAG, dl, 128);
@@ -4172,7 +4379,7 @@ static SDValue Concat256BitVectors(SDValue V1, SDValue V2, EVT VT,
   return Insert256BitVector(V, V2, NumElems/2, DAG, dl);
 }
 
-/// getOnesVector - Returns a vector of specified type with all bits set.
+/// Returns a vector of specified type with all bits set.
 /// Always build ones vectors as <4 x i32> or <8 x i32>. For 256-bit types with
 /// no AVX2 supprt, use two <4 x i32> inserted in a <8 x i32> appropriately.
 /// Then bitcast to their original type, ensuring they get CSE'd.
@@ -4180,7 +4387,7 @@ static SDValue getOnesVector(MVT VT, bool HasInt256, SelectionDAG &DAG,
                              SDLoc dl) {
   assert(VT.isVector() && "Expected a vector type");
 
-  SDValue Cst = DAG.getConstant(~0U, MVT::i32);
+  SDValue Cst = DAG.getConstant(~0U, dl, MVT::i32);
   SDValue Vec;
   if (VT.is256BitVector()) {
     if (HasInt256) { // AVX2
@@ -4195,22 +4402,10 @@ static SDValue getOnesVector(MVT VT, bool HasInt256, SelectionDAG &DAG,
   } else
     llvm_unreachable("Unexpected vector type");
 
-  return DAG.getNode(ISD::BITCAST, dl, VT, Vec);
-}
-
-/// getMOVLMask - Returns a vector_shuffle mask for an movs{s|d}, movd
-/// operation of specified width.
-static SDValue getMOVL(SelectionDAG &DAG, SDLoc dl, EVT VT, SDValue V1,
-                       SDValue V2) {
-  unsigned NumElems = VT.getVectorNumElements();
-  SmallVector<int, 8> Mask;
-  Mask.push_back(NumElems);
-  for (unsigned i = 1; i != NumElems; ++i)
-    Mask.push_back(i);
-  return DAG.getVectorShuffle(VT, dl, V1, V2, &Mask[0]);
+  return DAG.getBitcast(VT, Vec);
 }
 
-/// getUnpackl - Returns a vector_shuffle node for an unpackl operation.
+/// Returns a vector_shuffle node for an unpackl operation.
 static SDValue getUnpackl(SelectionDAG &DAG, SDLoc dl, MVT VT, SDValue V1,
                           SDValue V2) {
   unsigned NumElems = VT.getVectorNumElements();
@@ -4222,7 +4417,7 @@ static SDValue getUnpackl(SelectionDAG &DAG, SDLoc dl, MVT VT, SDValue V1,
   return DAG.getVectorShuffle(VT, dl, V1, V2, &Mask[0]);
 }
 
-/// getUnpackh - Returns a vector_shuffle node for an unpackh operation.
+/// Returns a vector_shuffle node for an unpackh operation.
 static SDValue getUnpackh(SelectionDAG &DAG, SDLoc dl, MVT VT, SDValue V1,
                           SDValue V2) {
   unsigned NumElems = VT.getVectorNumElements();
@@ -4234,10 +4429,10 @@ static SDValue getUnpackh(SelectionDAG &DAG, SDLoc dl, MVT VT, SDValue V1,
   return DAG.getVectorShuffle(VT, dl, V1, V2, &Mask[0]);
 }
 
-/// getShuffleVectorZeroOrUndef - Return a vector_shuffle of the specified
-/// vector of zero or undef vector.  This produces a shuffle where the low
-/// element of V2 is swizzled into the zero/undef vector, landing at element
-/// Idx.  This produces a shuffle mask like 4,1,2,3 (idx=0) or  0,1,2,4 (idx=3).
+/// Return a vector_shuffle of the specified vector of zero or undef vector.
+/// This produces a shuffle where the low element of V2 is swizzled into the
+/// zero/undef vector, landing at element Idx.
+/// This produces a shuffle mask like 4,1,2,3 (idx=0) or  0,1,2,4 (idx=3).
 static SDValue getShuffleVectorZeroOrUndef(SDValue V2, unsigned Idx,
                                            bool IsZero,
                                            const X86Subtarget *Subtarget,
@@ -4253,11 +4448,12 @@ static SDValue getShuffleVectorZeroOrUndef(SDValue V2, unsigned Idx,
   return DAG.getVectorShuffle(VT, SDLoc(V2), V1, V2, &MaskVec[0]);
 }
 
-/// getTargetShuffleMask - Calculates the shuffle mask corresponding to the
-/// target specific opcode. Returns true if the Mask could be calculated. Sets
-/// IsUnary to true if only uses one source. Note that this will set IsUnary for
-/// shuffles which use a single input multiple times, and in those cases it will
+/// Calculates the shuffle mask corresponding to the target-specific opcode.
+/// Returns true if the Mask could be calculated. Sets IsUnary to true if only
+/// uses one source. Note that this will set IsUnary for shuffles which use a
+/// single input multiple times, and in those cases it will
 /// adjust the mask to only have indices within that single input.
+/// FIXME: Add support for Decode*Mask functions that return SM_SentinelZero.
 static bool getTargetShuffleMask(SDNode *N, MVT VT,
                                  SmallVectorImpl<int> &Mask, bool &IsUnary) {
   unsigned NumElems = VT.getVectorNumElements();
@@ -4357,7 +4553,8 @@ static bool getTargetShuffleMask(SDNode *N, MVT VT,
       return false;
 
     SDValue Ptr = MaskLoad->getBasePtr();
-    if (Ptr->getOpcode() == X86ISD::Wrapper)
+    if (Ptr->getOpcode() == X86ISD::Wrapper ||
+        Ptr->getOpcode() == X86ISD::WrapperRIP)
       Ptr = Ptr->getOperand(0);
 
     auto *MaskCP = dyn_cast<ConstantPoolSDNode>(Ptr);
@@ -4386,6 +4583,10 @@ static bool getTargetShuffleMask(SDNode *N, MVT VT,
     ImmN = N->getOperand(N->getNumOperands()-1);
     DecodeVPERM2X128Mask(VT, cast<ConstantSDNode>(ImmN)->getZExtValue(), Mask);
     if (Mask.empty()) return false;
+    // Mask only contains negative index if an element is zero.
+    if (std::any_of(Mask.begin(), Mask.end(),
+                    [](int M){ return M == SM_SentinelZero; }))
+      return false;
     break;
   case X86ISD::MOVSLDUP:
     DecodeMOVSLDUPMask(VT, Mask);
@@ -4418,7 +4619,7 @@ static bool getTargetShuffleMask(SDNode *N, MVT VT,
   return true;
 }
 
-/// getShuffleScalarElt - Returns the scalar element that will make up the ith
+/// Returns the scalar element that will make up the ith
 /// element of the result of the vector shuffle.
 static SDValue getShuffleScalarElt(SDNode *N, unsigned Index, SelectionDAG &DAG,
                                    unsigned Depth) {
@@ -4482,8 +4683,7 @@ static SDValue getShuffleScalarElt(SDNode *N, unsigned Index, SelectionDAG &DAG,
   return SDValue();
 }
 
-/// LowerBuildVectorv16i8 - Custom lower build_vector of v16i8.
-///
+/// Custom lower build_vector of v16i8.
 static SDValue LowerBuildVectorv16i8(SDValue Op, unsigned NonZeros,
                                        unsigned NumNonZero, unsigned NumZero,
                                        SelectionDAG &DAG,
@@ -4510,7 +4710,7 @@ static SDValue LowerBuildVectorv16i8(SDValue Op, unsigned NonZeros,
         }
         V = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl,
                         MVT::v16i8, V, Op.getOperand(i),
-                        DAG.getIntPtrConstant(i));
+                        DAG.getIntPtrConstant(i, dl));
       }
     }
 
@@ -4538,7 +4738,7 @@ static SDValue LowerBuildVectorv16i8(SDValue Op, unsigned NonZeros,
       if (ThisIsNonZero) {
         ThisElt = DAG.getNode(ISD::ZERO_EXTEND, dl, MVT::i16, Op.getOperand(i));
         ThisElt = DAG.getNode(ISD::SHL, dl, MVT::i16,
-                              ThisElt, DAG.getConstant(8, MVT::i8));
+                              ThisElt, DAG.getConstant(8, dl, MVT::i8));
         if (LastIsNonZero)
           ThisElt = DAG.getNode(ISD::OR, dl, MVT::i16, ThisElt, LastElt);
       } else
@@ -4546,15 +4746,14 @@ static SDValue LowerBuildVectorv16i8(SDValue Op, unsigned NonZeros,
 
       if (ThisElt.getNode())
         V = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, MVT::v8i16, V, ThisElt,
-                        DAG.getIntPtrConstant(i/2));
+                        DAG.getIntPtrConstant(i/2, dl));
     }
   }
 
-  return DAG.getNode(ISD::BITCAST, dl, MVT::v16i8, V);
+  return DAG.getBitcast(MVT::v16i8, V);
 }
 
-/// LowerBuildVectorv8i16 - Custom lower build_vector of v8i16.
-///
+/// Custom lower build_vector of v8i16.
 static SDValue LowerBuildVectorv8i16(SDValue Op, unsigned NonZeros,
                                      unsigned NumNonZero, unsigned NumZero,
                                      SelectionDAG &DAG,
@@ -4578,14 +4777,14 @@ static SDValue LowerBuildVectorv8i16(SDValue Op, unsigned NonZeros,
       }
       V = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl,
                       MVT::v8i16, V, Op.getOperand(i),
-                      DAG.getIntPtrConstant(i));
+                      DAG.getIntPtrConstant(i, dl));
     }
   }
 
   return V;
 }
 
-/// LowerBuildVectorv4x32 - Custom lower build_vector of v4i32 or v4f32.
+/// Custom lower build_vector of v4i32 or v4f32.
 static SDValue LowerBuildVectorv4x32(SDValue Op, SelectionDAG &DAG,
                                      const X86Subtarget *Subtarget,
                                      const TargetLowering &TLI) {
@@ -4685,9 +4884,10 @@ static SDValue LowerBuildVectorv4x32(SDValue Op, SelectionDAG &DAG,
 
   unsigned InsertPSMask = EltMaskIdx << 6 | EltIdx << 4 | ZMask;
   assert((InsertPSMask & ~0xFFu) == 0 && "Invalid mask!");
-  SDValue Result = DAG.getNode(X86ISD::INSERTPS, SDLoc(Op), MVT::v4f32, V1, V2,
-                               DAG.getIntPtrConstant(InsertPSMask));
-  return DAG.getNode(ISD::BITCAST, SDLoc(Op), VT, Result);
+  SDLoc DL(Op);
+  SDValue Result = DAG.getNode(X86ISD::INSERTPS, DL, MVT::v4f32, V1, V2,
+                               DAG.getIntPtrConstant(InsertPSMask, DL));
+  return DAG.getBitcast(VT, Result);
 }
 
 /// Return a vector logical shift node.
@@ -4697,12 +4897,11 @@ static SDValue getVShift(bool isLeft, EVT VT, SDValue SrcOp,
   assert(VT.is128BitVector() && "Unknown type for VShift");
   MVT ShVT = MVT::v2i64;
   unsigned Opc = isLeft ? X86ISD::VSHLDQ : X86ISD::VSRLDQ;
-  SrcOp = DAG.getNode(ISD::BITCAST, dl, ShVT, SrcOp);
-  MVT ScalarShiftTy = TLI.getScalarShiftAmountTy(SrcOp.getValueType());
+  SrcOp = DAG.getBitcast(ShVT, SrcOp);
+  MVT ScalarShiftTy = TLI.getScalarShiftAmountTy(DAG.getDataLayout(), VT);
   assert(NumBits % 8 == 0 && "Only support byte sized shifts");
-  SDValue ShiftVal = DAG.getConstant(NumBits/8, ScalarShiftTy);
-  return DAG.getNode(ISD::BITCAST, dl, VT,
-                     DAG.getNode(Opc, dl, ShVT, SrcOp, ShiftVal));
+  SDValue ShiftVal = DAG.getConstant(NumBits/8, dl, ScalarShiftTy);
+  return DAG.getBitcast(VT, DAG.getNode(Opc, dl, ShVT, SrcOp, ShiftVal));
 }
 
 static SDValue
@@ -4757,9 +4956,11 @@ LowerAsSplatVectorLoad(SDValue SrcOp, MVT VT, SDLoc dl, SelectionDAG &DAG) {
     if ((Offset % RequiredAlign) & 3)
       return SDValue();
     int64_t StartOffset = Offset & ~(RequiredAlign-1);
-    if (StartOffset)
-      Ptr = DAG.getNode(ISD::ADD, SDLoc(Ptr), Ptr.getValueType(),
-                        Ptr,DAG.getConstant(StartOffset, Ptr.getValueType()));
+    if (StartOffset) {
+      SDLoc DL(Ptr);
+      Ptr = DAG.getNode(ISD::ADD, DL, Ptr.getValueType(), Ptr,
+                        DAG.getConstant(StartOffset, DL, Ptr.getValueType()));
+    }
 
     int EltNo = (Offset - StartOffset) >> 2;
     unsigned NumElems = VT.getVectorNumElements();
@@ -4885,7 +5086,7 @@ static SDValue EltsFromConsecutiveLoads(EVT VT, ArrayRef<SDValue> Elts,
                              SDValue(ResNode.getNode(), 1));
     }
 
-    return DAG.getNode(ISD::BITCAST, DL, VT, ResNode);
+    return DAG.getBitcast(VT, ResNode);
   }
   return SDValue();
 }
@@ -5015,7 +5216,8 @@ static SDValue LowerVectorBroadcast(SDValue Op, const X86Subtarget* Subtarget,
       assert(C && "Invalid constant type");
 
       const TargetLowering &TLI = DAG.getTargetLoweringInfo();
-      SDValue CP = DAG.getConstantPool(C, TLI.getPointerTy());
+      SDValue CP =
+          DAG.getConstantPool(C, TLI.getPointerTy(DAG.getDataLayout()));
       unsigned Alignment = cast<ConstantPoolSDNode>(CP)->getAlignment();
       Ld = DAG.getLoad(CVT, dl, DAG.getEntryNode(), CP,
                        MachinePointerInfo::getConstantPool(),
@@ -5152,95 +5354,117 @@ static SDValue buildFromShuffleMostly(SDValue Op, SelectionDAG &DAG) {
   for (unsigned i = 0, e = InsertIndices.size(); i != e; ++i) {
     unsigned Idx = InsertIndices[i];
     NV = DAG.getNode(ISD::INSERT_VECTOR_ELT, DL, VT, NV, Op.getOperand(Idx),
-                     DAG.getIntPtrConstant(Idx));
+                     DAG.getIntPtrConstant(Idx, DL));
   }
 
   return NV;
 }
 
+static SDValue ConvertI1VectorToInteger(SDValue Op, SelectionDAG &DAG) {
+  assert(ISD::isBuildVectorOfConstantSDNodes(Op.getNode()) &&
+         Op.getScalarValueSizeInBits() == 1 &&
+         "Can not convert non-constant vector");
+  uint64_t Immediate = 0;
+  for (unsigned idx = 0, e = Op.getNumOperands(); idx < e; ++idx) {
+    SDValue In = Op.getOperand(idx);
+    if (In.getOpcode() != ISD::UNDEF)
+      Immediate |= cast<ConstantSDNode>(In)->getZExtValue() << idx;
+  }
+  SDLoc dl(Op);
+  MVT VT =
+   MVT::getIntegerVT(std::max((int)Op.getValueType().getSizeInBits(), 8));
+  return DAG.getConstant(Immediate, dl, VT);
+}
 // Lower BUILD_VECTOR operation for v8i1 and v16i1 types.
 SDValue
 X86TargetLowering::LowerBUILD_VECTORvXi1(SDValue Op, SelectionDAG &DAG) const {
 
   MVT VT = Op.getSimpleValueType();
-  assert((VT.getVectorElementType() == MVT::i1) && (VT.getSizeInBits() <= 16) &&
+  assert((VT.getVectorElementType() == MVT::i1) &&
          "Unexpected type in LowerBUILD_VECTORvXi1!");
 
   SDLoc dl(Op);
   if (ISD::isBuildVectorAllZeros(Op.getNode())) {
-    SDValue Cst = DAG.getTargetConstant(0, MVT::i1);
+    SDValue Cst = DAG.getTargetConstant(0, dl, MVT::i1);
     SmallVector<SDValue, 16> Ops(VT.getVectorNumElements(), Cst);
     return DAG.getNode(ISD::BUILD_VECTOR, dl, VT, Ops);
   }
 
   if (ISD::isBuildVectorAllOnes(Op.getNode())) {
-    SDValue Cst = DAG.getTargetConstant(1, MVT::i1);
+    SDValue Cst = DAG.getTargetConstant(1, dl, MVT::i1);
     SmallVector<SDValue, 16> Ops(VT.getVectorNumElements(), Cst);
     return DAG.getNode(ISD::BUILD_VECTOR, dl, VT, Ops);
   }
 
-  bool AllContants = true;
+  if (ISD::isBuildVectorOfConstantSDNodes(Op.getNode())) {
+    SDValue Imm = ConvertI1VectorToInteger(Op, DAG);
+    if (Imm.getValueSizeInBits() == VT.getSizeInBits())
+      return DAG.getBitcast(VT, Imm);
+    SDValue ExtVec = DAG.getBitcast(MVT::v8i1, Imm);
+    return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, VT, ExtVec,
+                        DAG.getIntPtrConstant(0, dl));
+  }
+
+  // Vector has one or more non-const elements
   uint64_t Immediate = 0;
-  int NonConstIdx = -1;
+  SmallVector<unsigned, 16> NonConstIdx;
   bool IsSplat = true;
-  unsigned NumNonConsts = 0;
-  unsigned NumConsts = 0;
+  bool HasConstElts = false;
+  int SplatIdx = -1;
   for (unsigned idx = 0, e = Op.getNumOperands(); idx < e; ++idx) {
     SDValue In = Op.getOperand(idx);
     if (In.getOpcode() == ISD::UNDEF)
       continue;
-    if (!isa<ConstantSDNode>(In)) {
-      AllContants = false;
-      NonConstIdx = idx;
-      NumNonConsts++;
-    } else {
-      NumConsts++;
-      if (cast<ConstantSDNode>(In)->getZExtValue())
-      Immediate |= (1ULL << idx);
+    if (!isa<ConstantSDNode>(In))
+      NonConstIdx.push_back(idx);
+    else {
+      Immediate |= cast<ConstantSDNode>(In)->getZExtValue() << idx;
+      HasConstElts = true;
     }
-    if (In != Op.getOperand(0))
+    if (SplatIdx == -1)
+      SplatIdx = idx;
+    else if (In != Op.getOperand(SplatIdx))
       IsSplat = false;
   }
 
-  if (AllContants) {
-    SDValue FullMask = DAG.getNode(ISD::BITCAST, dl, MVT::v16i1,
-      DAG.getConstant(Immediate, MVT::i16));
-    return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, VT, FullMask,
-                       DAG.getIntPtrConstant(0));
-  }
-
-  if (NumNonConsts == 1 && NonConstIdx != 0) {
-    SDValue DstVec;
-    if (NumConsts) {
-      SDValue VecAsImm = DAG.getConstant(Immediate,
-                                         MVT::getIntegerVT(VT.getSizeInBits()));
-      DstVec = DAG.getNode(ISD::BITCAST, dl, VT, VecAsImm);
-    }
-    else
-      DstVec = DAG.getUNDEF(VT);
-    return DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, VT, DstVec,
-                       Op.getOperand(NonConstIdx),
-                       DAG.getIntPtrConstant(NonConstIdx));
-  }
-  if (!IsSplat && (NonConstIdx != 0))
-    llvm_unreachable("Unsupported BUILD_VECTOR operation");
-  MVT SelectVT = (VT == MVT::v16i1)? MVT::i16 : MVT::i8;
-  SDValue Select;
+  // for splat use " (select i1 splat_elt, all-ones, all-zeroes)"
   if (IsSplat)
-    Select = DAG.getNode(ISD::SELECT, dl, SelectVT, Op.getOperand(0),
-                          DAG.getConstant(-1, SelectVT),
-                          DAG.getConstant(0, SelectVT));
+    return DAG.getNode(ISD::SELECT, dl, VT, Op.getOperand(SplatIdx),
+                       DAG.getConstant(1, dl, VT),
+                       DAG.getConstant(0, dl, VT));
+
+  // insert elements one by one
+  SDValue DstVec;
+  SDValue Imm;
+  if (Immediate) {
+    MVT ImmVT = MVT::getIntegerVT(std::max((int)VT.getSizeInBits(), 8));
+    Imm = DAG.getConstant(Immediate, dl, ImmVT);
+  }
+  else if (HasConstElts)
+    Imm = DAG.getConstant(0, dl, VT);
   else
-    Select = DAG.getNode(ISD::SELECT, dl, SelectVT, Op.getOperand(0),
-                         DAG.getConstant((Immediate | 1), SelectVT),
-                         DAG.getConstant(Immediate, SelectVT));
-  return DAG.getNode(ISD::BITCAST, dl, VT, Select);
+    Imm = DAG.getUNDEF(VT);
+  if (Imm.getValueSizeInBits() == VT.getSizeInBits())
+    DstVec = DAG.getBitcast(VT, Imm);
+  else {
+    SDValue ExtVec = DAG.getBitcast(MVT::v8i1, Imm);
+    DstVec = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, VT, ExtVec,
+                         DAG.getIntPtrConstant(0, dl));
+  }
+
+  for (unsigned i = 0; i < NonConstIdx.size(); ++i) {
+    unsigned InsertIdx = NonConstIdx[i];
+    DstVec = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, VT, DstVec,
+                         Op.getOperand(InsertIdx),
+                         DAG.getIntPtrConstant(InsertIdx, dl));
+  }
+  return DstVec;
 }
 
 /// \brief Return true if \p N implements a horizontal binop and return the
 /// operands for the horizontal binop into V0 and V1.
 ///
-/// This is a helper function of PerformBUILD_VECTORCombine.
+/// This is a helper function of LowerToHorizontalOp().
 /// This function checks that the build_vector \p N in input implements a
 /// horizontal operation. Parameter \p Opcode defines the kind of horizontal
 /// operation to match.
@@ -5303,11 +5527,17 @@ static bool isHorizontalBinOp(const BuildVectorSDNode *N, unsigned Opcode,
     unsigned I1 = cast<ConstantSDNode>(Op1.getOperand(1))->getZExtValue();
 
     if (i * 2 < NumElts) {
-      if (V0.getOpcode() == ISD::UNDEF)
+      if (V0.getOpcode() == ISD::UNDEF) {
         V0 = Op0.getOperand(0);
+        if (V0.getValueType() != VT)
+          return false;
+      }
     } else {
-      if (V1.getOpcode() == ISD::UNDEF)
+      if (V1.getOpcode() == ISD::UNDEF) {
         V1 = Op0.getOperand(0);
+        if (V1.getValueType() != VT)
+          return false;
+      }
       if (i * 2 == NumElts)
         ExpectedVExtractIdx = BaseIdx;
     }
@@ -5331,7 +5561,7 @@ static bool isHorizontalBinOp(const BuildVectorSDNode *N, unsigned Opcode,
 /// \brief Emit a sequence of two 128-bit horizontal add/sub followed by
 /// a concat_vector.
 ///
-/// This is a helper function of PerformBUILD_VECTORCombine.
+/// This is a helper function of LowerToHorizontalOp().
 /// This function expects two 256-bit vectors called V0 and V1.
 /// At first, each vector is split into two separate 128-bit vectors.
 /// Then, the resulting 128-bit vectors are used to implement two
@@ -5351,7 +5581,7 @@ static bool isHorizontalBinOp(const BuildVectorSDNode *N, unsigned Opcode,
 ///
 /// Otherwise, the first horizontal binop dag node takes as input the lower
 /// 128-bit of V0 and the lower 128-bit of V1, and the second horizontal binop
-/// dag node takes the the upper 128-bit of V0 and the upper 128-bit of V1.
+/// dag node takes the upper 128-bit of V0 and the upper 128-bit of V1.
 ///   Example:
 ///     HADD V0_LO, V1_LO
 ///     HADD V0_HI, V1_HI
@@ -5397,12 +5627,16 @@ static SDValue ExpandHorizontalBinOp(const SDValue &V0, const SDValue &V1,
   return DAG.getNode(ISD::CONCAT_VECTORS, DL, VT, LO, HI);
 }
 
-/// \brief Try to fold a build_vector that performs an 'addsub' into the
-/// sequence of 'vadd + vsub + blendi'.
-static SDValue matchAddSub(const BuildVectorSDNode *BV, SelectionDAG &DAG,
-                           const X86Subtarget *Subtarget) {
-  SDLoc DL(BV);
+/// Try to fold a build_vector that performs an 'addsub' to an X86ISD::ADDSUB
+/// node.
+static SDValue LowerToAddSub(const BuildVectorSDNode *BV,
+                             const X86Subtarget *Subtarget, SelectionDAG &DAG) {
   EVT VT = BV->getValueType(0);
+  if ((!Subtarget->hasSSE3() || (VT != MVT::v4f32 && VT != MVT::v2f64)) &&
+      (!Subtarget->hasAVX() || (VT != MVT::v8f32 && VT != MVT::v4f64)))
+    return SDValue();
+
+  SDLoc DL(BV);
   unsigned NumElts = VT.getVectorNumElements();
   SDValue InVec0 = DAG.getUNDEF(VT);
   SDValue InVec1 = DAG.getUNDEF(VT);
@@ -5457,10 +5691,16 @@ static SDValue matchAddSub(const BuildVectorSDNode *BV, SelectionDAG &DAG,
       SubFound = true;
 
     // Update InVec0 and InVec1.
-    if (InVec0.getOpcode() == ISD::UNDEF)
+    if (InVec0.getOpcode() == ISD::UNDEF) {
       InVec0 = Op0.getOperand(0);
-    if (InVec1.getOpcode() == ISD::UNDEF)
+      if (InVec0.getValueType() != VT)
+        return SDValue();
+    }
+    if (InVec1.getOpcode() == ISD::UNDEF) {
       InVec1 = Op1.getOperand(0);
+      if (InVec1.getValueType() != VT)
+        return SDValue();
+    }
 
     // Make sure that operands in input to each add/sub node always
     // come from a same pair of vectors.
@@ -5490,23 +5730,12 @@ static SDValue matchAddSub(const BuildVectorSDNode *BV, SelectionDAG &DAG,
   return SDValue();
 }
 
-static SDValue PerformBUILD_VECTORCombine(SDNode *N, SelectionDAG &DAG,
-                                          const X86Subtarget *Subtarget) {
-  SDLoc DL(N);
-  EVT VT = N->getValueType(0);
+/// Lower BUILD_VECTOR to a horizontal add/sub operation if possible.
+static SDValue LowerToHorizontalOp(const BuildVectorSDNode *BV,
+                                   const X86Subtarget *Subtarget,
+                                   SelectionDAG &DAG) {
+  EVT VT = BV->getValueType(0);
   unsigned NumElts = VT.getVectorNumElements();
-  BuildVectorSDNode *BV = cast<BuildVectorSDNode>(N);
-  SDValue InVec0, InVec1;
-
-  // Try to match an ADDSUB.
-  if ((Subtarget->hasSSE3() && (VT == MVT::v4f32 || VT == MVT::v2f64)) ||
-      (Subtarget->hasAVX() && (VT == MVT::v8f32 || VT == MVT::v4f64))) {
-    SDValue Value = matchAddSub(BV, DAG, Subtarget);
-    if (Value.getNode())
-      return Value;
-  }
-
-  // Try to match horizontal ADD/SUB.
   unsigned NumUndefsLO = 0;
   unsigned NumUndefsHI = 0;
   unsigned Half = NumElts/2;
@@ -5525,6 +5754,8 @@ static SDValue PerformBUILD_VECTORCombine(SDNode *N, SelectionDAG &DAG,
   if (NumUndefsLO + NumUndefsHI + 1 >= NumElts)
     return SDValue();
 
+  SDLoc DL(BV);
+  SDValue InVec0, InVec1;
   if ((VT == MVT::v4f32 || VT == MVT::v2f64) && Subtarget->hasSSE3()) {
     // Try to match an SSE3 float HADD/HSUB.
     if (isHorizontalBinOp(BV, ISD::FADD, DAG, 0, NumElts, InVec0, InVec1))
@@ -5669,6 +5900,11 @@ X86TargetLowering::LowerBUILD_VECTOR(SDValue Op, SelectionDAG &DAG) const {
       return getOnesVector(VT, Subtarget->hasInt256(), DAG, dl);
   }
 
+  BuildVectorSDNode *BV = cast<BuildVectorSDNode>(Op.getNode());
+  if (SDValue AddSub = LowerToAddSub(BV, Subtarget, DAG))
+    return AddSub;
+  if (SDValue HorizontalOp = LowerToHorizontalOp(BV, Subtarget, DAG))
+    return HorizontalOp;
   if (SDValue Broadcast = LowerVectorBroadcast(Op, Subtarget, DAG))
     return Broadcast;
 
@@ -5720,9 +5956,8 @@ X86TargetLowering::LowerBUILD_VECTOR(SDValue Op, SelectionDAG &DAG) const {
         // convert it to a vector with movd (S2V+shuffle to zero extend).
         Item = DAG.getNode(ISD::TRUNCATE, dl, MVT::i32, Item);
         Item = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VecVT, Item);
-        return DAG.getNode(
-            ISD::BITCAST, dl, VT,
-            getShuffleVectorZeroOrUndef(Item, Idx * 2, true, Subtarget, DAG));
+        return DAG.getBitcast(VT, getShuffleVectorZeroOrUndef(
+                                      Item, Idx * 2, true, Subtarget, DAG));
       }
     }
 
@@ -5739,7 +5974,7 @@ X86TargetLowering::LowerBUILD_VECTOR(SDValue Op, SelectionDAG &DAG) const {
         if (VT.is512BitVector()) {
           SDValue ZeroVec = getZeroVector(VT, Subtarget, DAG, dl);
           return DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, VT, ZeroVec,
-                             Item, DAG.getIntPtrConstant(0));
+                             Item, DAG.getIntPtrConstant(0, dl));
         }
         assert((VT.is128BitVector() || VT.is256BitVector()) &&
                "Expected an SSE value type!");
@@ -5768,7 +6003,7 @@ X86TargetLowering::LowerBUILD_VECTOR(SDValue Op, SelectionDAG &DAG) const {
           Item = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, MVT::v4i32, Item);
           Item = getShuffleVectorZeroOrUndef(Item, 0, true, Subtarget, DAG);
         }
-        return DAG.getNode(ISD::BITCAST, dl, VT, Item);
+        return DAG.getBitcast(VT, Item);
       }
     }
 
@@ -5933,7 +6168,7 @@ X86TargetLowering::LowerBUILD_VECTOR(SDValue Op, SelectionDAG &DAG) const {
       for (unsigned i = 1; i < NumElems; ++i) {
         if (Op.getOperand(i).getOpcode() == ISD::UNDEF) continue;
         Result = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, VT, Result,
-                             Op.getOperand(i), DAG.getIntPtrConstant(i));
+                             Op.getOperand(i), DAG.getIntPtrConstant(i, dl));
       }
       return Result;
     }
@@ -5973,7 +6208,7 @@ X86TargetLowering::LowerBUILD_VECTOR(SDValue Op, SelectionDAG &DAG) const {
   return SDValue();
 }
 
-// LowerAVXCONCAT_VECTORS - 256-bit AVX can use the vinsertf128 instruction
+// 256-bit AVX can use the vinsertf128 instruction
 // to create 256-bit vectors from two other 128-bit ones.
 static SDValue LowerAVXCONCAT_VECTORS(SDValue Op, SelectionDAG &DAG) {
   SDLoc dl(Op);
@@ -6031,10 +6266,10 @@ static SDValue LowerCONCAT_VECTORSvXi1(SDValue Op,
   if (IsZeroV1 && IsZeroV2)
     return getZeroVector(ResVT, Subtarget, DAG, dl);
 
-  SDValue ZeroIdx = DAG.getIntPtrConstant(0);
+  SDValue ZeroIdx = DAG.getIntPtrConstant(0, dl);
   SDValue Undef = DAG.getUNDEF(ResVT);
   unsigned NumElems = ResVT.getVectorNumElements();
-  SDValue ShiftBits = DAG.getConstant(NumElems/2, MVT::i8);
+  SDValue ShiftBits = DAG.getConstant(NumElems/2, dl, MVT::i8);
 
   V2 = DAG.getNode(ISD::INSERT_SUBVECTOR, dl, ResVT, Undef, V2, ZeroIdx);
   V2 = DAG.getNode(X86ISD::VSHLI, dl, ResVT, V2, ShiftBits);
@@ -6202,7 +6437,7 @@ static bool isShuffleEquivalent(SDValue V1, SDValue V2, ArrayRef<int> Mask,
 /// example.
 ///
 /// NB: We rely heavily on "undef" masks preserving the input lane.
-static SDValue getV4X86ShuffleImm8ForMask(ArrayRef<int> Mask,
+static SDValue getV4X86ShuffleImm8ForMask(ArrayRef<int> Mask, SDLoc DL,
                                           SelectionDAG &DAG) {
   assert(Mask.size() == 4 && "Only 4-lane shuffle masks");
   assert(Mask[0] >= -1 && Mask[0] < 4 && "Out of bound mask element!");
@@ -6215,7 +6450,93 @@ static SDValue getV4X86ShuffleImm8ForMask(ArrayRef<int> Mask,
   Imm |= (Mask[1] == -1 ? 1 : Mask[1]) << 2;
   Imm |= (Mask[2] == -1 ? 2 : Mask[2]) << 4;
   Imm |= (Mask[3] == -1 ? 3 : Mask[3]) << 6;
-  return DAG.getConstant(Imm, MVT::i8);
+  return DAG.getConstant(Imm, DL, MVT::i8);
+}
+
+/// \brief Compute whether each element of a shuffle is zeroable.
+///
+/// A "zeroable" vector shuffle element is one which can be lowered to zero.
+/// Either it is an undef element in the shuffle mask, the element of the input
+/// referenced is undef, or the element of the input referenced is known to be
+/// zero. Many x86 shuffles can zero lanes cheaply and we often want to handle
+/// as many lanes with this technique as possible to simplify the remaining
+/// shuffle.
+static SmallBitVector computeZeroableShuffleElements(ArrayRef<int> Mask,
+                                                     SDValue V1, SDValue V2) {
+  SmallBitVector Zeroable(Mask.size(), false);
+
+  while (V1.getOpcode() == ISD::BITCAST)
+    V1 = V1->getOperand(0);
+  while (V2.getOpcode() == ISD::BITCAST)
+    V2 = V2->getOperand(0);
+
+  bool V1IsZero = ISD::isBuildVectorAllZeros(V1.getNode());
+  bool V2IsZero = ISD::isBuildVectorAllZeros(V2.getNode());
+
+  for (int i = 0, Size = Mask.size(); i < Size; ++i) {
+    int M = Mask[i];
+    // Handle the easy cases.
+    if (M < 0 || (M >= 0 && M < Size && V1IsZero) || (M >= Size && V2IsZero)) {
+      Zeroable[i] = true;
+      continue;
+    }
+
+    // If this is an index into a build_vector node (which has the same number
+    // of elements), dig out the input value and use it.
+    SDValue V = M < Size ? V1 : V2;
+    if (V.getOpcode() != ISD::BUILD_VECTOR || Size != (int)V.getNumOperands())
+      continue;
+
+    SDValue Input = V.getOperand(M % Size);
+    // The UNDEF opcode check really should be dead code here, but not quite
+    // worth asserting on (it isn't invalid, just unexpected).
+    if (Input.getOpcode() == ISD::UNDEF || X86::isZeroNode(Input))
+      Zeroable[i] = true;
+  }
+
+  return Zeroable;
+}
+
+/// \brief Try to emit a bitmask instruction for a shuffle.
+///
+/// This handles cases where we can model a blend exactly as a bitmask due to
+/// one of the inputs being zeroable.
+static SDValue lowerVectorShuffleAsBitMask(SDLoc DL, MVT VT, SDValue V1,
+                                           SDValue V2, ArrayRef<int> Mask,
+                                           SelectionDAG &DAG) {
+  MVT EltVT = VT.getScalarType();
+  int NumEltBits = EltVT.getSizeInBits();
+  MVT IntEltVT = MVT::getIntegerVT(NumEltBits);
+  SDValue Zero = DAG.getConstant(0, DL, IntEltVT);
+  SDValue AllOnes = DAG.getConstant(APInt::getAllOnesValue(NumEltBits), DL,
+                                    IntEltVT);
+  if (EltVT.isFloatingPoint()) {
+    Zero = DAG.getBitcast(EltVT, Zero);
+    AllOnes = DAG.getBitcast(EltVT, AllOnes);
+  }
+  SmallVector<SDValue, 16> VMaskOps(Mask.size(), Zero);
+  SmallBitVector Zeroable = computeZeroableShuffleElements(Mask, V1, V2);
+  SDValue V;
+  for (int i = 0, Size = Mask.size(); i < Size; ++i) {
+    if (Zeroable[i])
+      continue;
+    if (Mask[i] % Size != i)
+      return SDValue(); // Not a blend.
+    if (!V)
+      V = Mask[i] < Size ? V1 : V2;
+    else if (V != (Mask[i] < Size ? V1 : V2))
+      return SDValue(); // Can only let one input through the mask.
+
+    VMaskOps[i] = AllOnes;
+  }
+  if (!V)
+    return SDValue(); // No non-zeroable elements!
+
+  SDValue VMask = DAG.getNode(ISD::BUILD_VECTOR, DL, VT, VMaskOps);
+  V = DAG.getNode(VT.isFloatingPoint()
+                  ? (unsigned) X86ISD::FAND : (unsigned) ISD::AND,
+                  DL, VT, V, VMask);
+  return V;
 }
 
 /// \brief Try to emit a blend instruction for a shuffle using bit math.
@@ -6229,8 +6550,9 @@ static SDValue lowerVectorShuffleAsBitBlend(SDLoc DL, MVT VT, SDValue V1,
   assert(VT.isInteger() && "Only supports integer vector types!");
   MVT EltVT = VT.getScalarType();
   int NumEltBits = EltVT.getSizeInBits();
-  SDValue Zero = DAG.getConstant(0, EltVT);
-  SDValue AllOnes = DAG.getConstant(APInt::getAllOnesValue(NumEltBits), EltVT);
+  SDValue Zero = DAG.getConstant(0, DL, EltVT);
+  SDValue AllOnes = DAG.getConstant(APInt::getAllOnesValue(NumEltBits), DL,
+                                    EltVT);
   SmallVector<SDValue, 16> MaskOps;
   for (int i = 0, Size = Mask.size(); i < Size; ++i) {
     if (Mask[i] != -1 && Mask[i] != i && Mask[i] != i + Size)
@@ -6242,10 +6564,9 @@ static SDValue lowerVectorShuffleAsBitBlend(SDLoc DL, MVT VT, SDValue V1,
   V1 = DAG.getNode(ISD::AND, DL, VT, V1, V1Mask);
   // We have to cast V2 around.
   MVT MaskVT = MVT::getVectorVT(MVT::i64, VT.getSizeInBits() / 64);
-  V2 = DAG.getNode(ISD::BITCAST, DL, VT,
-                   DAG.getNode(X86ISD::ANDNP, DL, MaskVT,
-                               DAG.getNode(ISD::BITCAST, DL, MaskVT, V1Mask),
-                               DAG.getNode(ISD::BITCAST, DL, MaskVT, V2)));
+  V2 = DAG.getBitcast(VT, DAG.getNode(X86ISD::ANDNP, DL, MaskVT,
+                                      DAG.getBitcast(MaskVT, V1Mask),
+                                      DAG.getBitcast(MaskVT, V2)));
   return DAG.getNode(ISD::OR, DL, VT, V1, V2);
 }
 
@@ -6276,7 +6597,7 @@ static SDValue lowerVectorShuffleAsBlend(SDLoc DL, MVT VT, SDValue V1,
   case MVT::v4f64:
   case MVT::v8f32:
     return DAG.getNode(X86ISD::BLENDI, DL, VT, V1, V2,
-                       DAG.getConstant(BlendMask, MVT::i8));
+                       DAG.getConstant(BlendMask, DL, MVT::i8));
 
   case MVT::v4i64:
   case MVT::v8i32:
@@ -6296,11 +6617,11 @@ static SDValue lowerVectorShuffleAsBlend(SDLoc DL, MVT VT, SDValue V1,
             BlendMask |= 1u << (i * Scale + j);
 
       MVT BlendVT = VT.getSizeInBits() > 128 ? MVT::v8i32 : MVT::v4i32;
-      V1 = DAG.getNode(ISD::BITCAST, DL, BlendVT, V1);
-      V2 = DAG.getNode(ISD::BITCAST, DL, BlendVT, V2);
-      return DAG.getNode(ISD::BITCAST, DL, VT,
-                         DAG.getNode(X86ISD::BLENDI, DL, BlendVT, V1, V2,
-                                     DAG.getConstant(BlendMask, MVT::i8)));
+      V1 = DAG.getBitcast(BlendVT, V1);
+      V2 = DAG.getBitcast(BlendVT, V2);
+      return DAG.getBitcast(
+          VT, DAG.getNode(X86ISD::BLENDI, DL, BlendVT, V1, V2,
+                          DAG.getConstant(BlendMask, DL, MVT::i8)));
     }
     // FALLTHROUGH
   case MVT::v8i16: {
@@ -6313,11 +6634,11 @@ static SDValue lowerVectorShuffleAsBlend(SDLoc DL, MVT VT, SDValue V1,
         for (int j = 0; j < Scale; ++j)
           BlendMask |= 1u << (i * Scale + j);
 
-    V1 = DAG.getNode(ISD::BITCAST, DL, MVT::v8i16, V1);
-    V2 = DAG.getNode(ISD::BITCAST, DL, MVT::v8i16, V2);
-    return DAG.getNode(ISD::BITCAST, DL, VT,
-                       DAG.getNode(X86ISD::BLENDI, DL, MVT::v8i16, V1, V2,
-                                   DAG.getConstant(BlendMask, MVT::i8)));
+    V1 = DAG.getBitcast(MVT::v8i16, V1);
+    V2 = DAG.getBitcast(MVT::v8i16, V2);
+    return DAG.getBitcast(VT,
+                          DAG.getNode(X86ISD::BLENDI, DL, MVT::v8i16, V1, V2,
+                                      DAG.getConstant(BlendMask, DL, MVT::i8)));
   }
 
   case MVT::v16i16: {
@@ -6331,7 +6652,7 @@ static SDValue lowerVectorShuffleAsBlend(SDLoc DL, MVT VT, SDValue V1,
         if (RepeatedMask[i] >= 16)
           BlendMask |= 1u << i;
       return DAG.getNode(X86ISD::BLENDI, DL, MVT::v16i16, V1, V2,
-                         DAG.getConstant(BlendMask, MVT::i8));
+                         DAG.getConstant(BlendMask, DL, MVT::i8));
     }
   }
     // FALLTHROUGH
@@ -6340,6 +6661,10 @@ static SDValue lowerVectorShuffleAsBlend(SDLoc DL, MVT VT, SDValue V1,
     assert((VT.getSizeInBits() == 128 || Subtarget->hasAVX2()) &&
            "256-bit byte-blends require AVX2 support!");
 
+    // Attempt to lower to a bitmask if we can. VPAND is faster than VPBLENDVB.
+    if (SDValue Masked = lowerVectorShuffleAsBitMask(DL, VT, V1, V2, Mask, DAG))
+      return Masked;
+
     // Scale the blend by the number of bytes per element.
     int Scale = VT.getScalarSizeInBits() / 8;
 
@@ -6363,15 +6688,15 @@ static SDValue lowerVectorShuffleAsBlend(SDLoc DL, MVT VT, SDValue V1,
       for (int j = 0; j < Scale; ++j)
         VSELECTMask.push_back(
             Mask[i] < 0 ? DAG.getUNDEF(MVT::i8)
-                        : DAG.getConstant(Mask[i] < Size ? -1 : 0, MVT::i8));
+                        : DAG.getConstant(Mask[i] < Size ? -1 : 0, DL,
+                                          MVT::i8));
 
-    V1 = DAG.getNode(ISD::BITCAST, DL, BlendVT, V1);
-    V2 = DAG.getNode(ISD::BITCAST, DL, BlendVT, V2);
-    return DAG.getNode(
-        ISD::BITCAST, DL, VT,
-        DAG.getNode(ISD::VSELECT, DL, BlendVT,
-                    DAG.getNode(ISD::BUILD_VECTOR, DL, BlendVT, VSELECTMask),
-                    V1, V2));
+    V1 = DAG.getBitcast(BlendVT, V1);
+    V2 = DAG.getBitcast(BlendVT, V2);
+    return DAG.getBitcast(VT, DAG.getNode(ISD::VSELECT, DL, BlendVT,
+                                          DAG.getNode(ISD::BUILD_VECTOR, DL,
+                                                      BlendVT, VSELECTMask),
+                                          V1, V2));
   }
 
   default:
@@ -6552,12 +6877,12 @@ static SDValue lowerVectorShuffleAsByteRotate(SDLoc DL, MVT VT, SDValue V1,
   if (Subtarget->hasSSSE3()) {
     // Cast the inputs to i8 vector of correct length to match PALIGNR.
     MVT AlignVT = MVT::getVectorVT(MVT::i8, 16 * NumLanes);
-    Lo = DAG.getNode(ISD::BITCAST, DL, AlignVT, Lo);
-    Hi = DAG.getNode(ISD::BITCAST, DL, AlignVT, Hi);
+    Lo = DAG.getBitcast(AlignVT, Lo);
+    Hi = DAG.getBitcast(AlignVT, Hi);
 
-    return DAG.getNode(ISD::BITCAST, DL, VT,
-                       DAG.getNode(X86ISD::PALIGNR, DL, AlignVT, Hi, Lo,
-                                   DAG.getConstant(Rotation * Scale, MVT::i8)));
+    return DAG.getBitcast(
+        VT, DAG.getNode(X86ISD::PALIGNR, DL, AlignVT, Hi, Lo,
+                        DAG.getConstant(Rotation * Scale, DL, MVT::i8)));
   }
 
   assert(VT.getSizeInBits() == 128 &&
@@ -6570,129 +6895,44 @@ static SDValue lowerVectorShuffleAsByteRotate(SDLoc DL, MVT VT, SDValue V1,
   int HiByteShift = Rotation * Scale;
 
   // Cast the inputs to v2i64 to match PSLLDQ/PSRLDQ.
-  Lo = DAG.getNode(ISD::BITCAST, DL, MVT::v2i64, Lo);
-  Hi = DAG.getNode(ISD::BITCAST, DL, MVT::v2i64, Hi);
+  Lo = DAG.getBitcast(MVT::v2i64, Lo);
+  Hi = DAG.getBitcast(MVT::v2i64, Hi);
 
   SDValue LoShift = DAG.getNode(X86ISD::VSHLDQ, DL, MVT::v2i64, Lo,
-                                DAG.getConstant(LoByteShift, MVT::i8));
+                                DAG.getConstant(LoByteShift, DL, MVT::i8));
   SDValue HiShift = DAG.getNode(X86ISD::VSRLDQ, DL, MVT::v2i64, Hi,
-                                DAG.getConstant(HiByteShift, MVT::i8));
-  return DAG.getNode(ISD::BITCAST, DL, VT,
-                     DAG.getNode(ISD::OR, DL, MVT::v2i64, LoShift, HiShift));
+                                DAG.getConstant(HiByteShift, DL, MVT::i8));
+  return DAG.getBitcast(VT,
+                        DAG.getNode(ISD::OR, DL, MVT::v2i64, LoShift, HiShift));
 }
 
-/// \brief Compute whether each element of a shuffle is zeroable.
+/// \brief Try to lower a vector shuffle as a bit shift (shifts in zeros).
 ///
-/// A "zeroable" vector shuffle element is one which can be lowered to zero.
-/// Either it is an undef element in the shuffle mask, the element of the input
-/// referenced is undef, or the element of the input referenced is known to be
-/// zero. Many x86 shuffles can zero lanes cheaply and we often want to handle
-/// as many lanes with this technique as possible to simplify the remaining
-/// shuffle.
-static SmallBitVector computeZeroableShuffleElements(ArrayRef<int> Mask,
-                                                     SDValue V1, SDValue V2) {
-  SmallBitVector Zeroable(Mask.size(), false);
-
-  while (V1.getOpcode() == ISD::BITCAST)
-    V1 = V1->getOperand(0);
-  while (V2.getOpcode() == ISD::BITCAST)
-    V2 = V2->getOperand(0);
-
-  bool V1IsZero = ISD::isBuildVectorAllZeros(V1.getNode());
-  bool V2IsZero = ISD::isBuildVectorAllZeros(V2.getNode());
-
-  for (int i = 0, Size = Mask.size(); i < Size; ++i) {
-    int M = Mask[i];
-    // Handle the easy cases.
-    if (M < 0 || (M >= 0 && M < Size && V1IsZero) || (M >= Size && V2IsZero)) {
-      Zeroable[i] = true;
-      continue;
-    }
-
-    // If this is an index into a build_vector node (which has the same number
-    // of elements), dig out the input value and use it.
-    SDValue V = M < Size ? V1 : V2;
-    if (V.getOpcode() != ISD::BUILD_VECTOR || Size != (int)V.getNumOperands())
-      continue;
-
-    SDValue Input = V.getOperand(M % Size);
-    // The UNDEF opcode check really should be dead code here, but not quite
-    // worth asserting on (it isn't invalid, just unexpected).
-    if (Input.getOpcode() == ISD::UNDEF || X86::isZeroNode(Input))
-      Zeroable[i] = true;
-  }
-
-  return Zeroable;
-}
-
-/// \brief Try to emit a bitmask instruction for a shuffle.
-///
-/// This handles cases where we can model a blend exactly as a bitmask due to
-/// one of the inputs being zeroable.
-static SDValue lowerVectorShuffleAsBitMask(SDLoc DL, MVT VT, SDValue V1,
-                                           SDValue V2, ArrayRef<int> Mask,
-                                           SelectionDAG &DAG) {
-  MVT EltVT = VT.getScalarType();
-  int NumEltBits = EltVT.getSizeInBits();
-  MVT IntEltVT = MVT::getIntegerVT(NumEltBits);
-  SDValue Zero = DAG.getConstant(0, IntEltVT);
-  SDValue AllOnes = DAG.getConstant(APInt::getAllOnesValue(NumEltBits), IntEltVT);
-  if (EltVT.isFloatingPoint()) {
-    Zero = DAG.getNode(ISD::BITCAST, DL, EltVT, Zero);
-    AllOnes = DAG.getNode(ISD::BITCAST, DL, EltVT, AllOnes);
-  }
-  SmallVector<SDValue, 16> VMaskOps(Mask.size(), Zero);
-  SmallBitVector Zeroable = computeZeroableShuffleElements(Mask, V1, V2);
-  SDValue V;
-  for (int i = 0, Size = Mask.size(); i < Size; ++i) {
-    if (Zeroable[i])
-      continue;
-    if (Mask[i] % Size != i)
-      return SDValue(); // Not a blend.
-    if (!V)
-      V = Mask[i] < Size ? V1 : V2;
-    else if (V != (Mask[i] < Size ? V1 : V2))
-      return SDValue(); // Can only let one input through the mask.
-
-    VMaskOps[i] = AllOnes;
-  }
-  if (!V)
-    return SDValue(); // No non-zeroable elements!
-
-  SDValue VMask = DAG.getNode(ISD::BUILD_VECTOR, DL, VT, VMaskOps);
-  V = DAG.getNode(VT.isFloatingPoint()
-                  ? (unsigned) X86ISD::FAND : (unsigned) ISD::AND,
-                  DL, VT, V, VMask);
-  return V;
-}
-
-/// \brief Try to lower a vector shuffle as a bit shift (shifts in zeros).
-///
-/// Attempts to match a shuffle mask against the PSLL(W/D/Q/DQ) and
-/// PSRL(W/D/Q/DQ) SSE2 and AVX2 logical bit-shift instructions. The function
-/// matches elements from one of the input vectors shuffled to the left or
-/// right with zeroable elements 'shifted in'. It handles both the strictly
-/// bit-wise element shifts and the byte shift across an entire 128-bit double
-/// quad word lane.
-///
-/// PSHL : (little-endian) left bit shift.
-/// [ zz, 0, zz,  2 ]
-/// [ -1, 4, zz, -1 ]
-/// PSRL : (little-endian) right bit shift.
-/// [  1, zz,  3, zz]
-/// [ -1, -1,  7, zz]
-/// PSLLDQ : (little-endian) left byte shift
-/// [ zz,  0,  1,  2,  3,  4,  5,  6]
-/// [ zz, zz, -1, -1,  2,  3,  4, -1]
-/// [ zz, zz, zz, zz, zz, zz, -1,  1]
-/// PSRLDQ : (little-endian) right byte shift
-/// [  5, 6,  7, zz, zz, zz, zz, zz]
-/// [ -1, 5,  6,  7, zz, zz, zz, zz]
-/// [  1, 2, -1, -1, -1, -1, zz, zz]
-static SDValue lowerVectorShuffleAsShift(SDLoc DL, MVT VT, SDValue V1,
-                                         SDValue V2, ArrayRef<int> Mask,
-                                         SelectionDAG &DAG) {
-  SmallBitVector Zeroable = computeZeroableShuffleElements(Mask, V1, V2);
+/// Attempts to match a shuffle mask against the PSLL(W/D/Q/DQ) and
+/// PSRL(W/D/Q/DQ) SSE2 and AVX2 logical bit-shift instructions. The function
+/// matches elements from one of the input vectors shuffled to the left or
+/// right with zeroable elements 'shifted in'. It handles both the strictly
+/// bit-wise element shifts and the byte shift across an entire 128-bit double
+/// quad word lane.
+///
+/// PSHL : (little-endian) left bit shift.
+/// [ zz, 0, zz,  2 ]
+/// [ -1, 4, zz, -1 ]
+/// PSRL : (little-endian) right bit shift.
+/// [  1, zz,  3, zz]
+/// [ -1, -1,  7, zz]
+/// PSLLDQ : (little-endian) left byte shift
+/// [ zz,  0,  1,  2,  3,  4,  5,  6]
+/// [ zz, zz, -1, -1,  2,  3,  4, -1]
+/// [ zz, zz, zz, zz, zz, zz, -1,  1]
+/// PSRLDQ : (little-endian) right byte shift
+/// [  5, 6,  7, zz, zz, zz, zz, zz]
+/// [ -1, 5,  6,  7, zz, zz, zz, zz]
+/// [  1, 2, -1, -1, -1, -1, zz, zz]
+static SDValue lowerVectorShuffleAsShift(SDLoc DL, MVT VT, SDValue V1,
+                                         SDValue V2, ArrayRef<int> Mask,
+                                         SelectionDAG &DAG) {
+  SmallBitVector Zeroable = computeZeroableShuffleElements(Mask, V1, V2);
 
   int Size = Mask.size();
   assert(Size == (int)VT.getVectorNumElements() && "Unexpected mask size");
@@ -6731,10 +6971,11 @@ static SDValue lowerVectorShuffleAsShift(SDLoc DL, MVT VT, SDValue V1,
     MVT ShiftVT = MVT::getVectorVT(ShiftSVT, Size / Scale);
     assert(DAG.getTargetLoweringInfo().isTypeLegal(ShiftVT) &&
            "Illegal integer vector type");
-    V = DAG.getNode(ISD::BITCAST, DL, ShiftVT, V);
+    V = DAG.getBitcast(ShiftVT, V);
 
-    V = DAG.getNode(OpCode, DL, ShiftVT, V, DAG.getConstant(ShiftAmt, MVT::i8));
-    return DAG.getNode(ISD::BITCAST, DL, VT, V);
+    V = DAG.getNode(OpCode, DL, ShiftVT, V,
+                    DAG.getConstant(ShiftAmt, DL, MVT::i8));
+    return DAG.getBitcast(VT, V);
   };
 
   // SSE/AVX supports logical shifts up to 64-bit integers - so we can just
@@ -6755,6 +6996,136 @@ static SDValue lowerVectorShuffleAsShift(SDLoc DL, MVT VT, SDValue V1,
   return SDValue();
 }
 
+/// \brief Try to lower a vector shuffle using SSE4a EXTRQ/INSERTQ.
+static SDValue lowerVectorShuffleWithSSE4A(SDLoc DL, MVT VT, SDValue V1,
+                                           SDValue V2, ArrayRef<int> Mask,
+                                           SelectionDAG &DAG) {
+  SmallBitVector Zeroable = computeZeroableShuffleElements(Mask, V1, V2);
+  assert(!Zeroable.all() && "Fully zeroable shuffle mask");
+
+  int Size = Mask.size();
+  int HalfSize = Size / 2;
+  assert(Size == (int)VT.getVectorNumElements() && "Unexpected mask size");
+
+  // Upper half must be undefined.
+  if (!isUndefInRange(Mask, HalfSize, HalfSize))
+    return SDValue();
+
+  // EXTRQ: Extract Len elements from lower half of source, starting at Idx.
+  // Remainder of lower half result is zero and upper half is all undef.
+  auto LowerAsEXTRQ = [&]() {
+    // Determine the extraction length from the part of the
+    // lower half that isn't zeroable.
+    int Len = HalfSize;
+    for (; Len >= 0; --Len)
+      if (!Zeroable[Len - 1])
+        break;
+    assert(Len > 0 && "Zeroable shuffle mask");
+
+    // Attempt to match first Len sequential elements from the lower half.
+    SDValue Src;
+    int Idx = -1;
+    for (int i = 0; i != Len; ++i) {
+      int M = Mask[i];
+      if (M < 0)
+        continue;
+      SDValue &V = (M < Size ? V1 : V2);
+      M = M % Size;
+
+      // All mask elements must be in the lower half.
+      if (M > HalfSize)
+        return SDValue();
+
+      if (Idx < 0 || (Src == V && Idx == (M - i))) {
+        Src = V;
+        Idx = M - i;
+        continue;
+      }
+      return SDValue();
+    }
+
+    if (Idx < 0)
+      return SDValue();
+
+    assert((Idx + Len) <= HalfSize && "Illegal extraction mask");
+    int BitLen = (Len * VT.getScalarSizeInBits()) & 0x3f;
+    int BitIdx = (Idx * VT.getScalarSizeInBits()) & 0x3f;
+    return DAG.getNode(X86ISD::EXTRQI, DL, VT, Src,
+                       DAG.getConstant(BitLen, DL, MVT::i8),
+                       DAG.getConstant(BitIdx, DL, MVT::i8));
+  };
+
+  if (SDValue ExtrQ = LowerAsEXTRQ())
+    return ExtrQ;
+
+  // INSERTQ: Extract lowest Len elements from lower half of second source and
+  // insert over first source, starting at Idx.
+  // { A[0], .., A[Idx-1], B[0], .., B[Len-1], A[Idx+Len], .., UNDEF, ... }
+  auto LowerAsInsertQ = [&]() {
+    for (int Idx = 0; Idx != HalfSize; ++Idx) {
+      SDValue Base;
+
+      // Attempt to match first source from mask before insertion point.
+      if (isUndefInRange(Mask, 0, Idx)) {
+        /* EMPTY */
+      } else if (isSequentialOrUndefInRange(Mask, 0, Idx, 0)) {
+        Base = V1;
+      } else if (isSequentialOrUndefInRange(Mask, 0, Idx, Size)) {
+        Base = V2;
+      } else {
+        continue;
+      }
+
+      // Extend the extraction length looking to match both the insertion of
+      // the second source and the remaining elements of the first.
+      for (int Hi = Idx + 1; Hi <= HalfSize; ++Hi) {
+        SDValue Insert;
+        int Len = Hi - Idx;
+
+        // Match insertion.
+        if (isSequentialOrUndefInRange(Mask, Idx, Len, 0)) {
+          Insert = V1;
+        } else if (isSequentialOrUndefInRange(Mask, Idx, Len, Size)) {
+          Insert = V2;
+        } else {
+          continue;
+        }
+
+        // Match the remaining elements of the lower half.
+        if (isUndefInRange(Mask, Hi, HalfSize - Hi)) {
+          /* EMPTY */
+        } else if ((!Base || (Base == V1)) &&
+                   isSequentialOrUndefInRange(Mask, Hi, HalfSize - Hi, Hi)) {
+          Base = V1;
+        } else if ((!Base || (Base == V2)) &&
+                   isSequentialOrUndefInRange(Mask, Hi, HalfSize - Hi,
+                                              Size + Hi)) {
+          Base = V2;
+        } else {
+          continue;
+        }
+
+        // We may not have a base (first source) - this can safely be undefined.
+        if (!Base)
+          Base = DAG.getUNDEF(VT);
+
+        int BitLen = (Len * VT.getScalarSizeInBits()) & 0x3f;
+        int BitIdx = (Idx * VT.getScalarSizeInBits()) & 0x3f;
+        return DAG.getNode(X86ISD::INSERTQI, DL, VT, Base, Insert,
+                           DAG.getConstant(BitLen, DL, MVT::i8),
+                           DAG.getConstant(BitIdx, DL, MVT::i8));
+      }
+    }
+
+    return SDValue();
+  };
+
+  if (SDValue InsertQ = LowerAsInsertQ())
+    return InsertQ;
+
+  return SDValue();
+}
+
 /// \brief Lower a vector shuffle as a zero or any extension.
 ///
 /// Given a specific number of elements, element bit width, and extension
@@ -6762,7 +7133,7 @@ static SDValue lowerVectorShuffleAsShift(SDLoc DL, MVT VT, SDValue V1,
 /// features of the subtarget.
 static SDValue lowerVectorShuffleAsSpecificZeroOrAnyExtend(
     SDLoc DL, MVT VT, int Scale, bool AnyExt, SDValue InputV,
-    const X86Subtarget *Subtarget, SelectionDAG &DAG) {
+    ArrayRef<int> Mask, const X86Subtarget *Subtarget, SelectionDAG &DAG) {
   assert(Scale > 1 && "Need a scale to extend.");
   int NumElements = VT.getVectorNumElements();
   int EltBits = VT.getScalarSizeInBits();
@@ -6775,31 +7146,50 @@ static SDValue lowerVectorShuffleAsSpecificZeroOrAnyExtend(
   if (Subtarget->hasSSE41()) {
     MVT ExtVT = MVT::getVectorVT(MVT::getIntegerVT(EltBits * Scale),
                                  NumElements / Scale);
-    return DAG.getNode(ISD::BITCAST, DL, VT,
-                       DAG.getNode(X86ISD::VZEXT, DL, ExtVT, InputV));
+    return DAG.getBitcast(VT, DAG.getNode(X86ISD::VZEXT, DL, ExtVT, InputV));
   }
 
   // For any extends we can cheat for larger element sizes and use shuffle
   // instructions that can fold with a load and/or copy.
   if (AnyExt && EltBits == 32) {
     int PSHUFDMask[4] = {0, -1, 1, -1};
-    return DAG.getNode(
-        ISD::BITCAST, DL, VT,
-        DAG.getNode(X86ISD::PSHUFD, DL, MVT::v4i32,
-                    DAG.getNode(ISD::BITCAST, DL, MVT::v4i32, InputV),
-                    getV4X86ShuffleImm8ForMask(PSHUFDMask, DAG)));
+    return DAG.getBitcast(
+        VT, DAG.getNode(X86ISD::PSHUFD, DL, MVT::v4i32,
+                        DAG.getBitcast(MVT::v4i32, InputV),
+                        getV4X86ShuffleImm8ForMask(PSHUFDMask, DL, DAG)));
   }
   if (AnyExt && EltBits == 16 && Scale > 2) {
     int PSHUFDMask[4] = {0, -1, 0, -1};
     InputV = DAG.getNode(X86ISD::PSHUFD, DL, MVT::v4i32,
-                         DAG.getNode(ISD::BITCAST, DL, MVT::v4i32, InputV),
-                         getV4X86ShuffleImm8ForMask(PSHUFDMask, DAG));
+                         DAG.getBitcast(MVT::v4i32, InputV),
+                         getV4X86ShuffleImm8ForMask(PSHUFDMask, DL, DAG));
     int PSHUFHWMask[4] = {1, -1, -1, -1};
-    return DAG.getNode(
-        ISD::BITCAST, DL, VT,
-        DAG.getNode(X86ISD::PSHUFHW, DL, MVT::v8i16,
-                    DAG.getNode(ISD::BITCAST, DL, MVT::v8i16, InputV),
-                    getV4X86ShuffleImm8ForMask(PSHUFHWMask, DAG)));
+    return DAG.getBitcast(
+        VT, DAG.getNode(X86ISD::PSHUFHW, DL, MVT::v8i16,
+                        DAG.getBitcast(MVT::v8i16, InputV),
+                        getV4X86ShuffleImm8ForMask(PSHUFHWMask, DL, DAG)));
+  }
+
+  // The SSE4A EXTRQ instruction can efficiently extend the first 2 lanes
+  // to 64-bits.
+  if ((Scale * EltBits) == 64 && EltBits < 32 && Subtarget->hasSSE4A()) {
+    assert(NumElements == (int)Mask.size() && "Unexpected shuffle mask size!");
+    assert(VT.getSizeInBits() == 128 && "Unexpected vector width!");
+
+    SDValue Lo = DAG.getNode(ISD::BITCAST, DL, MVT::v2i64,
+                             DAG.getNode(X86ISD::EXTRQI, DL, VT, InputV,
+                                         DAG.getConstant(EltBits, DL, MVT::i8),
+                                         DAG.getConstant(0, DL, MVT::i8)));
+    if (isUndefInRange(Mask, NumElements/2, NumElements/2))
+      return DAG.getNode(ISD::BITCAST, DL, VT, Lo);
+
+    SDValue Hi =
+        DAG.getNode(ISD::BITCAST, DL, MVT::v2i64,
+                    DAG.getNode(X86ISD::EXTRQI, DL, VT, InputV,
+                                DAG.getConstant(EltBits, DL, MVT::i8),
+                                DAG.getConstant(EltBits, DL, MVT::i8)));
+    return DAG.getNode(ISD::BITCAST, DL, VT,
+                       DAG.getNode(X86ISD::UNPCKL, DL, MVT::v2i64, Lo, Hi));
   }
 
   // If this would require more than 2 unpack instructions to expand, use
@@ -6810,12 +7200,12 @@ static SDValue lowerVectorShuffleAsSpecificZeroOrAnyExtend(
     SDValue PSHUFBMask[16];
     for (int i = 0; i < 16; ++i)
       PSHUFBMask[i] =
-          DAG.getConstant((i % Scale == 0) ? i / Scale : 0x80, MVT::i8);
-    InputV = DAG.getNode(ISD::BITCAST, DL, MVT::v16i8, InputV);
-    return DAG.getNode(ISD::BITCAST, DL, VT,
-                       DAG.getNode(X86ISD::PSHUFB, DL, MVT::v16i8, InputV,
-                                   DAG.getNode(ISD::BUILD_VECTOR, DL,
-                                               MVT::v16i8, PSHUFBMask)));
+          DAG.getConstant((i % Scale == 0) ? i / Scale : 0x80, DL, MVT::i8);
+    InputV = DAG.getBitcast(MVT::v16i8, InputV);
+    return DAG.getBitcast(VT,
+                          DAG.getNode(X86ISD::PSHUFB, DL, MVT::v16i8, InputV,
+                                      DAG.getNode(ISD::BUILD_VECTOR, DL,
+                                                  MVT::v16i8, PSHUFBMask)));
   }
 
   // Otherwise emit a sequence of unpacks.
@@ -6823,13 +7213,13 @@ static SDValue lowerVectorShuffleAsSpecificZeroOrAnyExtend(
     MVT InputVT = MVT::getVectorVT(MVT::getIntegerVT(EltBits), NumElements);
     SDValue Ext = AnyExt ? DAG.getUNDEF(InputVT)
                          : getZeroVector(InputVT, Subtarget, DAG, DL);
-    InputV = DAG.getNode(ISD::BITCAST, DL, InputVT, InputV);
+    InputV = DAG.getBitcast(InputVT, InputV);
     InputV = DAG.getNode(X86ISD::UNPCKL, DL, InputVT, InputV, Ext);
     Scale /= 2;
     EltBits *= 2;
     NumElements /= 2;
   } while (Scale > 1);
-  return DAG.getNode(ISD::BITCAST, DL, VT, InputV);
+  return DAG.getBitcast(VT, InputV);
 }
 
 /// \brief Try to lower a vector shuffle as a zero extension on any microarch.
@@ -6892,7 +7282,7 @@ static SDValue lowerVectorShuffleAsZeroOrAnyExtend(
       return SDValue();
 
     return lowerVectorShuffleAsSpecificZeroOrAnyExtend(
-        DL, VT, Scale, AnyExt, InputV, Subtarget, DAG);
+        DL, VT, Scale, AnyExt, InputV, Mask, Subtarget, DAG);
   };
 
   // The widest scale possible for extending is to a 64-bit integer.
@@ -6927,9 +7317,9 @@ static SDValue lowerVectorShuffleAsZeroOrAnyExtend(
   };
 
   if (SDValue V = CanZExtLowHalf()) {
-    V = DAG.getNode(ISD::BITCAST, DL, MVT::v2i64, V);
+    V = DAG.getBitcast(MVT::v2i64, V);
     V = DAG.getNode(X86ISD::VZEXT_MOVL, DL, MVT::v2i64, V);
-    return DAG.getNode(ISD::BITCAST, DL, VT, V);
+    return DAG.getBitcast(VT, V);
   }
 
   // No viable ext lowering found.
@@ -6952,8 +7342,13 @@ static SDValue getScalarValueForVectorElement(SDValue V, int Idx,
     return SDValue();
 
   if (V.getOpcode() == ISD::BUILD_VECTOR ||
-      (Idx == 0 && V.getOpcode() == ISD::SCALAR_TO_VECTOR))
-    return DAG.getNode(ISD::BITCAST, SDLoc(V), EltVT, V.getOperand(Idx));
+      (Idx == 0 && V.getOpcode() == ISD::SCALAR_TO_VECTOR)) {
+    // Ensure the scalar operand is the same size as the destination.
+    // FIXME: Add support for scalar truncation where possible.
+    SDValue S = V.getOperand(Idx);
+    if (EltVT.getSizeInBits() == S.getSimpleValueType().getSizeInBits())
+      return DAG.getNode(ISD::BITCAST, SDLoc(V), EltVT, S);
+  }
 
   return SDValue();
 }
@@ -6995,10 +7390,11 @@ static SDValue lowerVectorShuffleAsElementInsertion(
   // all the smarts here sunk into that routine. However, the current
   // lowering of BUILD_VECTOR makes that nearly impossible until the old
   // vector shuffle lowering is dead.
-  if (SDValue V2S = getScalarValueForVectorElement(
-          V2, Mask[V2Index] - Mask.size(), DAG)) {
+  SDValue V2S = getScalarValueForVectorElement(V2, Mask[V2Index] - Mask.size(),
+                                               DAG);
+  if (V2S && DAG.getTargetLoweringInfo().isTypeLegal(V2S.getValueType())) {
     // We need to zext the scalar if it is smaller than an i32.
-    V2S = DAG.getNode(ISD::BITCAST, DL, EltVT, V2S);
+    V2S = DAG.getBitcast(EltVT, V2S);
     if (EltVT == MVT::i8 || EltVT == MVT::i16) {
       // Using zext to expand a narrow element won't work for non-zero
       // insertions.
@@ -7047,7 +7443,7 @@ static SDValue lowerVectorShuffleAsElementInsertion(
 
   V2 = DAG.getNode(X86ISD::VZEXT_MOVL, DL, ExtVT, V2);
   if (ExtVT != VT)
-    V2 = DAG.getNode(ISD::BITCAST, DL, VT, V2);
+    V2 = DAG.getBitcast(VT, V2);
 
   if (V2Index != 0) {
     // If we have 4 or fewer lanes we can cheaply shuffle the element into
@@ -7059,13 +7455,13 @@ static SDValue lowerVectorShuffleAsElementInsertion(
       V2Shuffle[V2Index] = 0;
       V2 = DAG.getVectorShuffle(VT, DL, V2, DAG.getUNDEF(VT), V2Shuffle);
     } else {
-      V2 = DAG.getNode(ISD::BITCAST, DL, MVT::v2i64, V2);
+      V2 = DAG.getBitcast(MVT::v2i64, V2);
       V2 = DAG.getNode(
           X86ISD::VSHLDQ, DL, MVT::v2i64, V2,
-          DAG.getConstant(
-              V2Index * EltVT.getSizeInBits()/8,
-              DAG.getTargetLoweringInfo().getScalarShiftAmountTy(MVT::v2i64)));
-      V2 = DAG.getNode(ISD::BITCAST, DL, VT, V2);
+          DAG.getConstant(V2Index * EltVT.getSizeInBits() / 8, DL,
+                          DAG.getTargetLoweringInfo().getScalarShiftAmountTy(
+                              DAG.getDataLayout(), VT)));
+      V2 = DAG.getBitcast(VT, V2);
     }
   }
   return V2;
@@ -7223,7 +7619,7 @@ static SDValue lowerVectorShuffleAsInsertPS(SDValue Op, SDValue V1, SDValue V2,
   // Insert the V2 element into the desired position.
   SDLoc DL(Op);
   return DAG.getNode(X86ISD::INSERTPS, DL, MVT::v4f32, V1, V2,
-                     DAG.getConstant(InsertPSMask, MVT::i8));
+                     DAG.getConstant(InsertPSMask, DL, MVT::i8));
 }
 
 /// \brief Try to lower a shuffle as a permute of the inputs followed by an
@@ -7288,13 +7684,13 @@ static SDValue lowerVectorShuffleAsUnpack(SDLoc DL, MVT VT, SDValue V1,
     V2 = DAG.getVectorShuffle(VT, DL, V2, DAG.getUNDEF(VT), V2Mask);
 
     // Cast the inputs to the type we will use to unpack them.
-    V1 = DAG.getNode(ISD::BITCAST, DL, UnpackVT, V1);
-    V2 = DAG.getNode(ISD::BITCAST, DL, UnpackVT, V2);
+    V1 = DAG.getBitcast(UnpackVT, V1);
+    V2 = DAG.getBitcast(UnpackVT, V2);
 
     // Unpack the inputs and cast the result back to the desired type.
-    return DAG.getNode(ISD::BITCAST, DL, VT,
-                       DAG.getNode(UnpackLo ? X86ISD::UNPCKL : X86ISD::UNPCKH,
-                                   DL, UnpackVT, V1, V2));
+    return DAG.getBitcast(
+        VT, DAG.getNode(UnpackLo ? X86ISD::UNPCKL : X86ISD::UNPCKH, DL,
+                        UnpackVT, V1, V2));
   };
 
   // We try each unpack from the largest to the smallest to try and find one
@@ -7373,11 +7769,11 @@ static SDValue lowerV2F64VectorShuffle(SDValue Op, SDValue V1, SDValue V2,
       // If we have AVX, we can use VPERMILPS which will allow folding a load
       // into the shuffle.
       return DAG.getNode(X86ISD::VPERMILPI, DL, MVT::v2f64, V1,
-                         DAG.getConstant(SHUFPDMask, MVT::i8));
+                         DAG.getConstant(SHUFPDMask, DL, MVT::i8));
     }
 
-    return DAG.getNode(X86ISD::SHUFP, SDLoc(Op), MVT::v2f64, V1, V1,
-                       DAG.getConstant(SHUFPDMask, MVT::i8));
+    return DAG.getNode(X86ISD::SHUFP, DL, MVT::v2f64, V1, V1,
+                       DAG.getConstant(SHUFPDMask, DL, MVT::i8));
   }
   assert(Mask[0] >= 0 && Mask[0] < 2 && "Non-canonicalized blend!");
   assert(Mask[1] >= 2 && "Non-canonicalized blend!");
@@ -7420,8 +7816,8 @@ static SDValue lowerV2F64VectorShuffle(SDValue Op, SDValue V1, SDValue V2,
     return DAG.getNode(X86ISD::UNPCKH, DL, MVT::v2f64, V1, V2);
 
   unsigned SHUFPDMask = (Mask[0] == 1) | (((Mask[1] - 2) == 1) << 1);
-  return DAG.getNode(X86ISD::SHUFP, SDLoc(Op), MVT::v2f64, V1, V2,
-                     DAG.getConstant(SHUFPDMask, MVT::i8));
+  return DAG.getNode(X86ISD::SHUFP, DL, MVT::v2f64, V1, V2,
+                     DAG.getConstant(SHUFPDMask, DL, MVT::i8));
 }
 
 /// \brief Handle lowering of 2-lane 64-bit integer shuffles.
@@ -7450,14 +7846,14 @@ static SDValue lowerV2I64VectorShuffle(SDValue Op, SDValue V1, SDValue V2,
     // Straight shuffle of a single input vector. For everything from SSE2
     // onward this has a single fast instruction with no scary immediates.
     // We have to map the mask as it is actually a v4i32 shuffle instruction.
-    V1 = DAG.getNode(ISD::BITCAST, DL, MVT::v4i32, V1);
+    V1 = DAG.getBitcast(MVT::v4i32, V1);
     int WidenedMask[4] = {
         std::max(Mask[0], 0) * 2, std::max(Mask[0], 0) * 2 + 1,
         std::max(Mask[1], 0) * 2, std::max(Mask[1], 0) * 2 + 1};
-    return DAG.getNode(
-        ISD::BITCAST, DL, MVT::v2i64,
-        DAG.getNode(X86ISD::PSHUFD, SDLoc(Op), MVT::v4i32, V1,
-                    getV4X86ShuffleImm8ForMask(WidenedMask, DAG)));
+    return DAG.getBitcast(
+        MVT::v2i64,
+        DAG.getNode(X86ISD::PSHUFD, DL, MVT::v4i32, V1,
+                    getV4X86ShuffleImm8ForMask(WidenedMask, DL, DAG)));
   }
   assert(Mask[0] != -1 && "No undef lanes in multi-input v2 shuffles!");
   assert(Mask[1] != -1 && "No undef lanes in multi-input v2 shuffles!");
@@ -7476,12 +7872,12 @@ static SDValue lowerV2I64VectorShuffle(SDValue Op, SDValue V1, SDValue V2,
   };
   if (SDValue V1Pack = GetPackNode(V1))
     if (SDValue V2Pack = GetPackNode(V2))
-      return DAG.getNode(ISD::BITCAST, DL, MVT::v2i64,
-                         DAG.getNode(X86ISD::PACKUS, DL, MVT::v16i8,
-                                     Mask[0] == 0 ? V1Pack.getOperand(0)
-                                                  : V1Pack.getOperand(1),
-                                     Mask[1] == 2 ? V2Pack.getOperand(0)
-                                                  : V2Pack.getOperand(1)));
+      return DAG.getBitcast(MVT::v2i64,
+                            DAG.getNode(X86ISD::PACKUS, DL, MVT::v16i8,
+                                        Mask[0] == 0 ? V1Pack.getOperand(0)
+                                                     : V1Pack.getOperand(1),
+                                        Mask[1] == 2 ? V2Pack.getOperand(0)
+                                                     : V2Pack.getOperand(1)));
 
   // Try to use shift instructions.
   if (SDValue Shift =
@@ -7531,10 +7927,10 @@ static SDValue lowerV2I64VectorShuffle(SDValue Op, SDValue V1, SDValue V2,
   // incur 2 cycles of stall for integer vectors on Nehalem and older chips.
   // However, all the alternatives are still more cycles and newer chips don't
   // have this problem. It would be really nice if x86 had better shuffles here.
-  V1 = DAG.getNode(ISD::BITCAST, DL, MVT::v2f64, V1);
-  V2 = DAG.getNode(ISD::BITCAST, DL, MVT::v2f64, V2);
-  return DAG.getNode(ISD::BITCAST, DL, MVT::v2i64,
-                     DAG.getVectorShuffle(MVT::v2f64, DL, V1, V2, Mask));
+  V1 = DAG.getBitcast(MVT::v2f64, V1);
+  V2 = DAG.getBitcast(MVT::v2f64, V2);
+  return DAG.getBitcast(MVT::v2i64,
+                        DAG.getVectorShuffle(MVT::v2f64, DL, V1, V2, Mask));
 }
 
 /// \brief Test whether this can be lowered with a single SHUFPS instruction.
@@ -7591,7 +7987,7 @@ static SDValue lowerVectorShuffleWithSHUFPS(SDLoc DL, MVT VT,
       int V1Index = V2AdjIndex;
       int BlendMask[4] = {Mask[V2Index] - 4, 0, Mask[V1Index], 0};
       V2 = DAG.getNode(X86ISD::SHUFP, DL, VT, V2, V1,
-                       getV4X86ShuffleImm8ForMask(BlendMask, DAG));
+                       getV4X86ShuffleImm8ForMask(BlendMask, DL, DAG));
 
       // Now proceed to reconstruct the final blend as we have the necessary
       // high or low half formed.
@@ -7630,7 +8026,7 @@ static SDValue lowerVectorShuffleWithSHUFPS(SDLoc DL, MVT VT,
                           (Mask[0] >= 4 ? Mask[0] : Mask[1]) - 4,
                           (Mask[2] >= 4 ? Mask[2] : Mask[3]) - 4};
       V1 = DAG.getNode(X86ISD::SHUFP, DL, VT, V1, V2,
-                       getV4X86ShuffleImm8ForMask(BlendMask, DAG));
+                       getV4X86ShuffleImm8ForMask(BlendMask, DL, DAG));
 
       // Now we do a normal shuffle of V1 by giving V1 as both operands to
       // a blend.
@@ -7642,7 +8038,7 @@ static SDValue lowerVectorShuffleWithSHUFPS(SDLoc DL, MVT VT,
     }
   }
   return DAG.getNode(X86ISD::SHUFP, DL, VT, LowV, HighV,
-                     getV4X86ShuffleImm8ForMask(NewMask, DAG));
+                     getV4X86ShuffleImm8ForMask(NewMask, DL, DAG));
 }
 
 /// \brief Lower 4-lane 32-bit floating point shuffles.
@@ -7682,13 +8078,13 @@ static SDValue lowerV4F32VectorShuffle(SDValue Op, SDValue V1, SDValue V2,
       // If we have AVX, we can use VPERMILPS which will allow folding a load
       // into the shuffle.
       return DAG.getNode(X86ISD::VPERMILPI, DL, MVT::v4f32, V1,
-                         getV4X86ShuffleImm8ForMask(Mask, DAG));
+                         getV4X86ShuffleImm8ForMask(Mask, DL, DAG));
     }
 
     // Otherwise, use a straight shuffle of a single input vector. We pass the
     // input vector to both operands to simulate this with a SHUFPS.
     return DAG.getNode(X86ISD::SHUFP, DL, MVT::v4f32, V1, V1,
-                       getV4X86ShuffleImm8ForMask(Mask, DAG));
+                       getV4X86ShuffleImm8ForMask(Mask, DL, DAG));
   }
 
   // There are special ways we can lower some single-element blends. However, we
@@ -7774,7 +8170,7 @@ static SDValue lowerV4I32VectorShuffle(SDValue Op, SDValue V1, SDValue V2,
       Mask = UnpackHiMask;
 
     return DAG.getNode(X86ISD::PSHUFD, DL, MVT::v4i32, V1,
-                       getV4X86ShuffleImm8ForMask(Mask, DAG));
+                       getV4X86ShuffleImm8ForMask(Mask, DL, DAG));
   }
 
   // Try to use shift instructions.
@@ -7833,11 +8229,10 @@ static SDValue lowerV4I32VectorShuffle(SDValue Op, SDValue V1, SDValue V2,
   // up the inputs, bypassing domain shift penalties that we would encur if we
   // directly used PSHUFD on Nehalem and older. For newer chips, this isn't
   // relevant.
-  return DAG.getNode(ISD::BITCAST, DL, MVT::v4i32,
-                     DAG.getVectorShuffle(
-                         MVT::v4f32, DL,
-                         DAG.getNode(ISD::BITCAST, DL, MVT::v4f32, V1),
-                         DAG.getNode(ISD::BITCAST, DL, MVT::v4f32, V2), Mask));
+  return DAG.getBitcast(
+      MVT::v4i32,
+      DAG.getVectorShuffle(MVT::v4f32, DL, DAG.getBitcast(MVT::v4f32, V1),
+                           DAG.getBitcast(MVT::v4f32, V2), Mask));
 }
 
 /// \brief Lowering of single-input v8i16 shuffles is the cornerstone of SSE2
@@ -7991,7 +8386,7 @@ static SDValue lowerV8I16GeneralSingleInputVectorShuffle(
           std::swap(PSHUFHalfMask[FixFreeIdx % 4], PSHUFHalfMask[FixIdx % 4]);
           V = DAG.getNode(FixIdx < 4 ? X86ISD::PSHUFLW : X86ISD::PSHUFHW, DL,
                           MVT::v8i16, V,
-                          getV4X86ShuffleImm8ForMask(PSHUFHalfMask, DAG));
+                          getV4X86ShuffleImm8ForMask(PSHUFHalfMask, DL, DAG));
 
           for (int &M : Mask)
             if (M != -1 && M == FixIdx)
@@ -8015,10 +8410,10 @@ static SDValue lowerV8I16GeneralSingleInputVectorShuffle(
     int PSHUFDMask[] = {0, 1, 2, 3};
     PSHUFDMask[ADWord] = BDWord;
     PSHUFDMask[BDWord] = ADWord;
-    V = DAG.getNode(ISD::BITCAST, DL, VT,
-                    DAG.getNode(X86ISD::PSHUFD, DL, PSHUFDVT,
-                                DAG.getNode(ISD::BITCAST, DL, PSHUFDVT, V),
-                                getV4X86ShuffleImm8ForMask(PSHUFDMask, DAG)));
+    V = DAG.getBitcast(
+        VT,
+        DAG.getNode(X86ISD::PSHUFD, DL, PSHUFDVT, DAG.getBitcast(PSHUFDVT, V),
+                    getV4X86ShuffleImm8ForMask(PSHUFDMask, DL, DAG)));
 
     // Adjust the mask to match the new locations of A and B.
     for (int &M : Mask)
@@ -8254,15 +8649,15 @@ static SDValue lowerV8I16GeneralSingleInputVectorShuffle(
   // target half.
   if (!isNoopShuffleMask(PSHUFLMask))
     V = DAG.getNode(X86ISD::PSHUFLW, DL, VT, V,
-                    getV4X86ShuffleImm8ForMask(PSHUFLMask, DAG));
+                    getV4X86ShuffleImm8ForMask(PSHUFLMask, DL, DAG));
   if (!isNoopShuffleMask(PSHUFHMask))
     V = DAG.getNode(X86ISD::PSHUFHW, DL, VT, V,
-                    getV4X86ShuffleImm8ForMask(PSHUFHMask, DAG));
+                    getV4X86ShuffleImm8ForMask(PSHUFHMask, DL, DAG));
   if (!isNoopShuffleMask(PSHUFDMask))
-    V = DAG.getNode(ISD::BITCAST, DL, VT,
-                    DAG.getNode(X86ISD::PSHUFD, DL, PSHUFDVT,
-                                DAG.getNode(ISD::BITCAST, DL, PSHUFDVT, V),
-                                getV4X86ShuffleImm8ForMask(PSHUFDMask, DAG)));
+    V = DAG.getBitcast(
+        VT,
+        DAG.getNode(X86ISD::PSHUFD, DL, PSHUFDVT, DAG.getBitcast(PSHUFDVT, V),
+                    getV4X86ShuffleImm8ForMask(PSHUFDMask, DL, DAG)));
 
   // At this point, each half should contain all its inputs, and we can then
   // just shuffle them into their final position.
@@ -8276,7 +8671,7 @@ static SDValue lowerV8I16GeneralSingleInputVectorShuffle(
   // Do a half shuffle for the low mask.
   if (!isNoopShuffleMask(LoMask))
     V = DAG.getNode(X86ISD::PSHUFLW, DL, VT, V,
-                    getV4X86ShuffleImm8ForMask(LoMask, DAG));
+                    getV4X86ShuffleImm8ForMask(LoMask, DL, DAG));
 
   // Do a half shuffle with the high mask after shifting its values down.
   for (int &M : HiMask)
@@ -8284,7 +8679,7 @@ static SDValue lowerV8I16GeneralSingleInputVectorShuffle(
       M -= 4;
   if (!isNoopShuffleMask(HiMask))
     V = DAG.getNode(X86ISD::PSHUFHW, DL, VT, V,
-                    getV4X86ShuffleImm8ForMask(HiMask, DAG));
+                    getV4X86ShuffleImm8ForMask(HiMask, DL, DAG));
 
   return V;
 }
@@ -8314,8 +8709,8 @@ static SDValue lowerVectorShuffleAsPSHUFB(SDLoc DL, MVT VT, SDValue V1,
                       : (Mask[i / Scale] - Size) * Scale + i % Scale;
       if (Zeroable[i / Scale])
         V1Idx = V2Idx = ZeroMask;
-      V1Mask[i] = DAG.getConstant(V1Idx, MVT::i8);
-      V2Mask[i] = DAG.getConstant(V2Idx, MVT::i8);
+      V1Mask[i] = DAG.getConstant(V1Idx, DL, MVT::i8);
+      V2Mask[i] = DAG.getConstant(V2Idx, DL, MVT::i8);
       V1InUse |= (ZeroMask != V1Idx);
       V2InUse |= (ZeroMask != V2Idx);
     }
@@ -8323,11 +8718,11 @@ static SDValue lowerVectorShuffleAsPSHUFB(SDLoc DL, MVT VT, SDValue V1,
 
   if (V1InUse)
     V1 = DAG.getNode(X86ISD::PSHUFB, DL, MVT::v16i8,
-                     DAG.getNode(ISD::BITCAST, DL, MVT::v16i8, V1),
+                     DAG.getBitcast(MVT::v16i8, V1),
                      DAG.getNode(ISD::BUILD_VECTOR, DL, MVT::v16i8, V1Mask));
   if (V2InUse)
     V2 = DAG.getNode(X86ISD::PSHUFB, DL, MVT::v16i8,
-                     DAG.getNode(ISD::BITCAST, DL, MVT::v16i8, V2),
+                     DAG.getBitcast(MVT::v16i8, V2),
                      DAG.getNode(ISD::BUILD_VECTOR, DL, MVT::v16i8, V2Mask));
 
   // If we need shuffled inputs from both, blend the two.
@@ -8338,7 +8733,7 @@ static SDValue lowerVectorShuffleAsPSHUFB(SDLoc DL, MVT VT, SDValue V1,
     V = V1InUse ? V1 : V2;
 
   // Cast the result back to the correct type.
-  return DAG.getNode(ISD::BITCAST, DL, VT, V);
+  return DAG.getBitcast(VT, V);
 }
 
 /// \brief Generic lowering of 8-lane i16 shuffles.
@@ -8415,6 +8810,11 @@ static SDValue lowerV8I16VectorShuffle(SDValue Op, SDValue V1, SDValue V2,
           lowerVectorShuffleAsShift(DL, MVT::v8i16, V1, V2, Mask, DAG))
     return Shift;
 
+  // See if we can use SSE4A Extraction / Insertion.
+  if (Subtarget->hasSSE4A())
+    if (SDValue V = lowerVectorShuffleWithSSE4A(DL, MVT::v8i16, V1, V2, Mask, DAG))
+      return V;
+
   // There are special ways we can lower some single-element blends.
   if (NumV2Inputs == 1)
     if (SDValue V = lowerVectorShuffleAsElementInsertion(DL, MVT::v8i16, V1, V2,
@@ -8567,6 +8967,11 @@ static SDValue lowerV16I8VectorShuffle(SDValue Op, SDValue V1, SDValue V2,
           DL, MVT::v16i8, V1, V2, Mask, Subtarget, DAG))
     return ZExt;
 
+  // See if we can use SSE4A Extraction / Insertion.
+  if (Subtarget->hasSSE4A())
+    if (SDValue V = lowerVectorShuffleWithSSE4A(DL, MVT::v16i8, V1, V2, Mask, DAG))
+      return V;
+
   int NumV2Elements =
       std::count_if(Mask.begin(), Mask.end(), [](int M) { return M >= 16; });
 
@@ -8639,10 +9044,9 @@ static SDValue lowerV16I8VectorShuffle(SDValue Op, SDValue V1, SDValue V2,
         // Update the lane map based on the mapping we ended up with.
         LaneMap[MovingInputs[i]] = 2 * j + MovingInputs[i] % 2;
       }
-      V1 = DAG.getNode(
-          ISD::BITCAST, DL, MVT::v16i8,
-          DAG.getVectorShuffle(MVT::v8i16, DL,
-                               DAG.getNode(ISD::BITCAST, DL, MVT::v8i16, V1),
+      V1 = DAG.getBitcast(
+          MVT::v16i8,
+          DAG.getVectorShuffle(MVT::v8i16, DL, DAG.getBitcast(MVT::v8i16, V1),
                                DAG.getUNDEF(MVT::v8i16), PreDupI16Shuffle));
 
       // Unpack the bytes to form the i16s that will be shuffled into place.
@@ -8660,16 +9064,19 @@ static SDValue lowerV16I8VectorShuffle(SDValue Op, SDValue V1, SDValue V2,
             assert(PostDupI16Shuffle[i / 2] == MappedMask &&
                    "Conflicting entrties in the original shuffle!");
         }
-      return DAG.getNode(
-          ISD::BITCAST, DL, MVT::v16i8,
-          DAG.getVectorShuffle(MVT::v8i16, DL,
-                               DAG.getNode(ISD::BITCAST, DL, MVT::v8i16, V1),
+      return DAG.getBitcast(
+          MVT::v16i8,
+          DAG.getVectorShuffle(MVT::v8i16, DL, DAG.getBitcast(MVT::v8i16, V1),
                                DAG.getUNDEF(MVT::v8i16), PostDupI16Shuffle));
     };
     if (SDValue V = tryToWidenViaDuplication())
       return V;
   }
 
+  if (SDValue Masked =
+          lowerVectorShuffleAsBitMask(DL, MVT::v16i8, V1, V2, Mask, DAG))
+    return Masked;
+
   // Use dedicated unpack instructions for masks that match their pattern.
   if (isShuffleEquivalent(V1, V2, Mask, {// Low half.
                                          0, 16, 1, 17, 2, 18, 3, 19,
@@ -8756,19 +9163,18 @@ static SDValue lowerV16I8VectorShuffle(SDValue Op, SDValue V1, SDValue V2,
     // We use the mask type to pick which bytes are preserved based on how many
     // elements are dropped.
     MVT MaskVTs[] = { MVT::v8i16, MVT::v4i32, MVT::v2i64 };
-    SDValue ByteClearMask =
-        DAG.getNode(ISD::BITCAST, DL, MVT::v16i8,
-                    DAG.getConstant(0xFF, MaskVTs[NumEvenDrops - 1]));
+    SDValue ByteClearMask = DAG.getBitcast(
+        MVT::v16i8, DAG.getConstant(0xFF, DL, MaskVTs[NumEvenDrops - 1]));
     V1 = DAG.getNode(ISD::AND, DL, MVT::v16i8, V1, ByteClearMask);
     if (!IsSingleInput)
       V2 = DAG.getNode(ISD::AND, DL, MVT::v16i8, V2, ByteClearMask);
 
     // Now pack things back together.
-    V1 = DAG.getNode(ISD::BITCAST, DL, MVT::v8i16, V1);
-    V2 = IsSingleInput ? V1 : DAG.getNode(ISD::BITCAST, DL, MVT::v8i16, V2);
+    V1 = DAG.getBitcast(MVT::v8i16, V1);
+    V2 = IsSingleInput ? V1 : DAG.getBitcast(MVT::v8i16, V2);
     SDValue Result = DAG.getNode(X86ISD::PACKUS, DL, MVT::v16i8, V1, V2);
     for (int i = 1; i < NumEvenDrops; ++i) {
-      Result = DAG.getNode(ISD::BITCAST, DL, MVT::v8i16, Result);
+      Result = DAG.getBitcast(MVT::v8i16, Result);
       Result = DAG.getNode(X86ISD::PACKUS, DL, MVT::v16i8, Result, Result);
     }
 
@@ -8802,9 +9208,9 @@ static SDValue lowerV16I8VectorShuffle(SDValue Op, SDValue V1, SDValue V2,
       std::none_of(std::begin(HiBlendMask), std::end(HiBlendMask),
                    [](int M) { return M >= 0 && M % 2 == 1; })) {
     // Use a mask to drop the high bytes.
-    VLoHalf = DAG.getNode(ISD::BITCAST, DL, MVT::v8i16, V);
+    VLoHalf = DAG.getBitcast(MVT::v8i16, V);
     VLoHalf = DAG.getNode(ISD::AND, DL, MVT::v8i16, VLoHalf,
-                     DAG.getConstant(0x00FF, MVT::v8i16));
+                     DAG.getConstant(0x00FF, DL, MVT::v8i16));
 
     // This will be a single vector shuffle instead of a blend so nuke VHiHalf.
     VHiHalf = DAG.getUNDEF(MVT::v8i16);
@@ -8819,10 +9225,10 @@ static SDValue lowerV16I8VectorShuffle(SDValue Op, SDValue V1, SDValue V2,
   } else {
     // Otherwise just unpack the low half of V into VLoHalf and the high half into
     // VHiHalf so that we can blend them as i16s.
-    VLoHalf = DAG.getNode(ISD::BITCAST, DL, MVT::v8i16,
-                     DAG.getNode(X86ISD::UNPCKL, DL, MVT::v16i8, V, Zero));
-    VHiHalf = DAG.getNode(ISD::BITCAST, DL, MVT::v8i16,
-                     DAG.getNode(X86ISD::UNPCKH, DL, MVT::v16i8, V, Zero));
+    VLoHalf = DAG.getBitcast(
+        MVT::v8i16, DAG.getNode(X86ISD::UNPCKL, DL, MVT::v16i8, V, Zero));
+    VHiHalf = DAG.getBitcast(
+        MVT::v8i16, DAG.getNode(X86ISD::UNPCKH, DL, MVT::v16i8, V, Zero));
   }
 
   SDValue LoV = DAG.getVectorShuffle(MVT::v8i16, DL, VLoHalf, VHiHalf, LoBlendMask);
@@ -8950,9 +9356,9 @@ static SDValue splitAndLowerVectorShuffle(SDLoc DL, MVT VT, SDValue V1,
     auto *BV = dyn_cast<BuildVectorSDNode>(V);
     if (!BV) {
       LoV = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, OrigSplitVT, V,
-                        DAG.getIntPtrConstant(0));
+                        DAG.getIntPtrConstant(0, DL));
       HiV = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, OrigSplitVT, V,
-                        DAG.getIntPtrConstant(OrigSplitNumElements));
+                        DAG.getIntPtrConstant(OrigSplitNumElements, DL));
     } else {
 
       SmallVector<SDValue, 16> LoOps, HiOps;
@@ -8963,8 +9369,8 @@ static SDValue splitAndLowerVectorShuffle(SDLoc DL, MVT VT, SDValue V1,
       LoV = DAG.getNode(ISD::BUILD_VECTOR, DL, OrigSplitVT, LoOps);
       HiV = DAG.getNode(ISD::BUILD_VECTOR, DL, OrigSplitVT, HiOps);
     }
-    return std::make_pair(DAG.getNode(ISD::BITCAST, DL, SplitVT, LoV),
-                          DAG.getNode(ISD::BITCAST, DL, SplitVT, HiV));
+    return std::make_pair(DAG.getBitcast(SplitVT, LoV),
+                          DAG.getBitcast(SplitVT, HiV));
   };
 
   SDValue LoV1, HiV1, LoV2, HiV2;
@@ -9141,7 +9547,7 @@ static SDValue lowerVectorShuffleAsLanePermuteAndBlend(SDLoc DL, MVT VT,
     // allow folding it into a memory operand.
     unsigned PERMMask = 3 | 2 << 4;
     SDValue Flipped = DAG.getNode(X86ISD::VPERM2X128, DL, VT, DAG.getUNDEF(VT),
-                                  V1, DAG.getConstant(PERMMask, MVT::i8));
+                                  V1, DAG.getConstant(PERMMask, DL, MVT::i8));
     return DAG.getVectorShuffle(VT, DL, V1, Flipped, FlippedBlendMask);
   }
 
@@ -9178,9 +9584,10 @@ static SDValue lowerV2X128VectorShuffle(SDLoc DL, MVT VT, SDValue V1,
       MVT SubVT = MVT::getVectorVT(VT.getVectorElementType(),
                                    VT.getVectorNumElements() / 2);
       SDValue LoV = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, SubVT, V1,
-                                DAG.getIntPtrConstant(0));
+                                DAG.getIntPtrConstant(0, DL));
       SDValue HiV = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, SubVT,
-                                OnlyUsesV1 ? V1 : V2, DAG.getIntPtrConstant(0));
+                                OnlyUsesV1 ? V1 : V2,
+                                DAG.getIntPtrConstant(0, DL));
       return DAG.getNode(ISD::CONCAT_VECTORS, DL, VT, LoV, HiV);
     }
   }
@@ -9229,7 +9636,7 @@ static SDValue lowerV2X128VectorShuffle(SDLoc DL, MVT VT, SDValue V1,
   }
 
   return DAG.getNode(X86ISD::VPERM2X128, DL, VT, V1, V2,
-                     DAG.getConstant(PermMask, MVT::i8));
+                     DAG.getConstant(PermMask, DL, MVT::i8));
 }
 
 /// \brief Lower a vector shuffle by first fixing the 128-bit lanes and then
@@ -9296,12 +9703,12 @@ static SDValue lowerVectorShuffleByMerging128BitLanes(
       LaneMask[2 * i + 1] = 2*Lanes[i] + 1;
     }
 
-  V1 = DAG.getNode(ISD::BITCAST, DL, LaneVT, V1);
-  V2 = DAG.getNode(ISD::BITCAST, DL, LaneVT, V2);
+  V1 = DAG.getBitcast(LaneVT, V1);
+  V2 = DAG.getBitcast(LaneVT, V2);
   SDValue LaneShuffle = DAG.getVectorShuffle(LaneVT, DL, V1, V2, LaneMask);
 
   // Cast it back to the type we actually want.
-  LaneShuffle = DAG.getNode(ISD::BITCAST, DL, VT, LaneShuffle);
+  LaneShuffle = DAG.getBitcast(VT, LaneShuffle);
 
   // Now do a simple shuffle that isn't lane crossing.
   SmallVector<int, 8> NewMask;
@@ -9330,6 +9737,37 @@ static bool isShuffleMaskInputInPlace(int Input, ArrayRef<int> Mask) {
   return true;
 }
 
+static SDValue lowerVectorShuffleWithSHUFPD(SDLoc DL, MVT VT,
+                                            ArrayRef<int> Mask, SDValue V1,
+                                            SDValue V2, SelectionDAG &DAG) {
+
+  // Mask for V8F64: 0/1,  8/9,  2/3,  10/11, 4/5, ..
+  // Mask for V4F64; 0/1,  4/5,  2/3,  6/7..
+  assert(VT.getScalarSizeInBits() == 64 && "Unexpected data type for VSHUFPD");
+  int NumElts = VT.getVectorNumElements();
+  bool ShufpdMask = true;
+  bool CommutableMask = true;
+  unsigned Immediate = 0;
+  for (int i = 0; i < NumElts; ++i) {
+    if (Mask[i] < 0)
+      continue;
+    int Val = (i & 6) + NumElts * (i & 1);
+    int CommutVal = (i & 0xe) + NumElts * ((i & 1)^1);
+    if (Mask[i] < Val ||  Mask[i] > Val + 1)
+      ShufpdMask = false;
+    if (Mask[i] < CommutVal ||  Mask[i] > CommutVal + 1)
+      CommutableMask = false;
+    Immediate |= (Mask[i] % 2) << i;
+  }
+  if (ShufpdMask)
+    return DAG.getNode(X86ISD::SHUFP, DL, VT, V1, V2,
+                       DAG.getConstant(Immediate, DL, MVT::i8));
+  if (CommutableMask)
+    return DAG.getNode(X86ISD::SHUFP, DL, VT, V2, V1,
+                       DAG.getConstant(Immediate, DL, MVT::i8));
+  return SDValue();
+}
+
 /// \brief Handle lowering of 4-lane 64-bit floating point shuffles.
 ///
 /// Also ends up handling lowering of 4-lane 64-bit integer shuffles when AVX2
@@ -9365,13 +9803,13 @@ static SDValue lowerV4F64VectorShuffle(SDValue Op, SDValue V1, SDValue V2,
       unsigned VPERMILPMask = (Mask[0] == 1) | ((Mask[1] == 1) << 1) |
                               ((Mask[2] == 3) << 2) | ((Mask[3] == 3) << 3);
       return DAG.getNode(X86ISD::VPERMILPI, DL, MVT::v4f64, V1,
-                         DAG.getConstant(VPERMILPMask, MVT::i8));
+                         DAG.getConstant(VPERMILPMask, DL, MVT::i8));
     }
 
     // With AVX2 we have direct support for this permutation.
     if (Subtarget->hasAVX2())
       return DAG.getNode(X86ISD::VPERMI, DL, MVT::v4f64, V1,
-                         getV4X86ShuffleImm8ForMask(Mask, DAG));
+                         getV4X86ShuffleImm8ForMask(Mask, DL, DAG));
 
     // Otherwise, fall back.
     return lowerVectorShuffleAsLanePermuteAndBlend(DL, MVT::v4f64, V1, V2, Mask,
@@ -9394,24 +9832,9 @@ static SDValue lowerV4F64VectorShuffle(SDValue Op, SDValue V1, SDValue V2,
     return Blend;
 
   // Check if the blend happens to exactly fit that of SHUFPD.
-  if ((Mask[0] == -1 || Mask[0] < 2) &&
-      (Mask[1] == -1 || (Mask[1] >= 4 && Mask[1] < 6)) &&
-      (Mask[2] == -1 || (Mask[2] >= 2 && Mask[2] < 4)) &&
-      (Mask[3] == -1 || Mask[3] >= 6)) {
-    unsigned SHUFPDMask = (Mask[0] == 1) | ((Mask[1] == 5) << 1) |
-                          ((Mask[2] == 3) << 2) | ((Mask[3] == 7) << 3);
-    return DAG.getNode(X86ISD::SHUFP, DL, MVT::v4f64, V1, V2,
-                       DAG.getConstant(SHUFPDMask, MVT::i8));
-  }
-  if ((Mask[0] == -1 || (Mask[0] >= 4 && Mask[0] < 6)) &&
-      (Mask[1] == -1 || Mask[1] < 2) &&
-      (Mask[2] == -1 || Mask[2] >= 6) &&
-      (Mask[3] == -1 || (Mask[3] >= 2 && Mask[3] < 4))) {
-    unsigned SHUFPDMask = (Mask[0] == 5) | ((Mask[1] == 1) << 1) |
-                          ((Mask[2] == 7) << 2) | ((Mask[3] == 3) << 3);
-    return DAG.getNode(X86ISD::SHUFP, DL, MVT::v4f64, V2, V1,
-                       DAG.getConstant(SHUFPDMask, MVT::i8));
-  }
+  if (SDValue Op =
+      lowerVectorShuffleWithSHUFPD(DL, MVT::v4f64, Mask, V1, V2, DAG))
+    return Op;
 
   // Try to simplify this by merging 128-bit lanes to enable a lane-based
   // shuffle. However, if we have AVX2 and either inputs are already in place,
@@ -9473,11 +9896,11 @@ static SDValue lowerV4I64VectorShuffle(SDValue Op, SDValue V1, SDValue V2,
           PSHUFDMask[2 * i] = 2 * RepeatedMask[i];
           PSHUFDMask[2 * i + 1] = 2 * RepeatedMask[i] + 1;
         }
-      return DAG.getNode(
-          ISD::BITCAST, DL, MVT::v4i64,
+      return DAG.getBitcast(
+          MVT::v4i64,
           DAG.getNode(X86ISD::PSHUFD, DL, MVT::v8i32,
-                      DAG.getNode(ISD::BITCAST, DL, MVT::v8i32, V1),
-                      getV4X86ShuffleImm8ForMask(PSHUFDMask, DAG)));
+                      DAG.getBitcast(MVT::v8i32, V1),
+                      getV4X86ShuffleImm8ForMask(PSHUFDMask, DL, DAG)));
     }
   }
 
@@ -9485,7 +9908,7 @@ static SDValue lowerV4I64VectorShuffle(SDValue Op, SDValue V1, SDValue V2,
   // lanes.
   if (isSingleInputShuffleMask(Mask))
     return DAG.getNode(X86ISD::VPERMI, DL, MVT::v4i64, V1,
-                       getV4X86ShuffleImm8ForMask(Mask, DAG));
+                       getV4X86ShuffleImm8ForMask(Mask, DL, DAG));
 
   // Try to use shift instructions.
   if (SDValue Shift =
@@ -9555,7 +9978,7 @@ static SDValue lowerV8F32VectorShuffle(SDValue Op, SDValue V1, SDValue V2,
 
     if (isSingleInputShuffleMask(Mask))
       return DAG.getNode(X86ISD::VPERMILPI, DL, MVT::v8f32, V1,
-                         getV4X86ShuffleImm8ForMask(RepeatedMask, DAG));
+                         getV4X86ShuffleImm8ForMask(RepeatedMask, DL, DAG));
 
     // Use dedicated unpack instructions for masks that match their pattern.
     if (isShuffleEquivalent(V1, V2, Mask, {0, 8, 1, 9, 4, 12, 5, 13}))
@@ -9582,18 +10005,18 @@ static SDValue lowerV8F32VectorShuffle(SDValue Op, SDValue V1, SDValue V2,
     SDValue VPermMask[8];
     for (int i = 0; i < 8; ++i)
       VPermMask[i] = Mask[i] < 0 ? DAG.getUNDEF(MVT::i32)
-                                 : DAG.getConstant(Mask[i], MVT::i32);
+                                 : DAG.getConstant(Mask[i], DL, MVT::i32);
     if (!is128BitLaneCrossingShuffleMask(MVT::v8f32, Mask))
       return DAG.getNode(
           X86ISD::VPERMILPV, DL, MVT::v8f32, V1,
           DAG.getNode(ISD::BUILD_VECTOR, DL, MVT::v8i32, VPermMask));
 
     if (Subtarget->hasAVX2())
-      return DAG.getNode(X86ISD::VPERMV, DL, MVT::v8f32,
-                         DAG.getNode(ISD::BITCAST, DL, MVT::v8f32,
-                                     DAG.getNode(ISD::BUILD_VECTOR, DL,
+      return DAG.getNode(
+          X86ISD::VPERMV, DL, MVT::v8f32,
+          DAG.getBitcast(MVT::v8f32, DAG.getNode(ISD::BUILD_VECTOR, DL,
                                                  MVT::v8i32, VPermMask)),
-                         V1);
+          V1);
 
     // Otherwise, fall back.
     return lowerVectorShuffleAsLanePermuteAndBlend(DL, MVT::v8f32, V1, V2, Mask,
@@ -9655,7 +10078,7 @@ static SDValue lowerV8I32VectorShuffle(SDValue Op, SDValue V1, SDValue V2,
     assert(RepeatedMask.size() == 4 && "Unexpected repeated mask size!");
     if (isSingleInputShuffleMask(Mask))
       return DAG.getNode(X86ISD::PSHUFD, DL, MVT::v8i32, V1,
-                         getV4X86ShuffleImm8ForMask(RepeatedMask, DAG));
+                         getV4X86ShuffleImm8ForMask(RepeatedMask, DL, DAG));
 
     // Use dedicated unpack instructions for masks that match their pattern.
     if (isShuffleEquivalent(V1, V2, Mask, {0, 8, 1, 9, 4, 12, 5, 13}))
@@ -9683,7 +10106,7 @@ static SDValue lowerV8I32VectorShuffle(SDValue Op, SDValue V1, SDValue V2,
     SDValue VPermMask[8];
     for (int i = 0; i < 8; ++i)
       VPermMask[i] = Mask[i] < 0 ? DAG.getUNDEF(MVT::i32)
-                                 : DAG.getConstant(Mask[i], MVT::i32);
+                                 : DAG.getConstant(Mask[i], DL, MVT::i32);
     return DAG.getNode(
         X86ISD::VPERMV, DL, MVT::v8i32,
         DAG.getNode(ISD::BUILD_VECTOR, DL, MVT::v8i32, VPermMask), V1);
@@ -9780,15 +10203,14 @@ static SDValue lowerV16I16VectorShuffle(SDValue Op, SDValue V1, SDValue V2,
 
       int M = i < 8 ? Mask[i] : Mask[i] - 8;
       assert(M >= 0 && M < 8 && "Invalid single-input mask!");
-      PSHUFBMask[2 * i] = DAG.getConstant(2 * M, MVT::i8);
-      PSHUFBMask[2 * i + 1] = DAG.getConstant(2 * M + 1, MVT::i8);
+      PSHUFBMask[2 * i] = DAG.getConstant(2 * M, DL, MVT::i8);
+      PSHUFBMask[2 * i + 1] = DAG.getConstant(2 * M + 1, DL, MVT::i8);
     }
-    return DAG.getNode(
-        ISD::BITCAST, DL, MVT::v16i16,
-        DAG.getNode(
-            X86ISD::PSHUFB, DL, MVT::v32i8,
-            DAG.getNode(ISD::BITCAST, DL, MVT::v32i8, V1),
-            DAG.getNode(ISD::BUILD_VECTOR, DL, MVT::v32i8, PSHUFBMask)));
+    return DAG.getBitcast(MVT::v16i16,
+                          DAG.getNode(X86ISD::PSHUFB, DL, MVT::v32i8,
+                                      DAG.getBitcast(MVT::v32i8, V1),
+                                      DAG.getNode(ISD::BUILD_VECTOR, DL,
+                                                  MVT::v32i8, PSHUFBMask)));
   }
 
   // Try to simplify this by merging 128-bit lanes to enable a lane-based
@@ -9872,7 +10294,8 @@ static SDValue lowerV32I8VectorShuffle(SDValue Op, SDValue V1, SDValue V2,
       PSHUFBMask[i] =
           Mask[i] < 0
               ? DAG.getUNDEF(MVT::i8)
-              : DAG.getConstant(Mask[i] < 16 ? Mask[i] : Mask[i] - 16, MVT::i8);
+              : DAG.getConstant(Mask[i] < 16 ? Mask[i] : Mask[i] - 16, DL,
+                                MVT::i8);
 
     return DAG.getNode(
         X86ISD::PSHUFB, DL, MVT::v32i8, V1,
@@ -9907,7 +10330,7 @@ static SDValue lower256BitVectorShuffle(SDValue Op, SDValue V1, SDValue V2,
   int NumV2Elements = std::count_if(Mask.begin(), Mask.end(), [NumElts](int M) {
     return M >= NumElts;
   });
-  
+
   if (NumV2Elements == 1 && Mask[0] >= NumElts)
     if (SDValue Insertion = lowerVectorShuffleAsElementInsertion(
                               DL, VT, V1, V2, Mask, Subtarget, DAG))
@@ -9927,10 +10350,9 @@ static SDValue lower256BitVectorShuffle(SDValue Op, SDValue V1, SDValue V2,
 
     MVT FpVT = MVT::getVectorVT(MVT::getFloatingPointVT(ElementBits),
                                 VT.getVectorNumElements());
-    V1 = DAG.getNode(ISD::BITCAST, DL, FpVT, V1);
-    V2 = DAG.getNode(ISD::BITCAST, DL, FpVT, V2);
-    return DAG.getNode(ISD::BITCAST, DL, VT,
-                       DAG.getVectorShuffle(FpVT, DL, V1, V2, Mask));
+    V1 = DAG.getBitcast(FpVT, V1);
+    V2 = DAG.getBitcast(FpVT, V2);
+    return DAG.getBitcast(VT, DAG.getVectorShuffle(FpVT, DL, V1, V2, Mask));
   }
 
   switch (VT.SimpleTy) {
@@ -10199,10 +10621,10 @@ static SDValue lowerVectorShuffle(SDValue Op, const X86Subtarget *Subtarget,
     // Make sure that the new vector type is legal. For example, v2f64 isn't
     // legal on SSE1.
     if (DAG.getTargetLoweringInfo().isTypeLegal(NewVT)) {
-      V1 = DAG.getNode(ISD::BITCAST, dl, NewVT, V1);
-      V2 = DAG.getNode(ISD::BITCAST, dl, NewVT, V2);
-      return DAG.getNode(ISD::BITCAST, dl, VT,
-                         DAG.getVectorShuffle(NewVT, dl, V1, V2, WidenedMask));
+      V1 = DAG.getBitcast(NewVT, V1);
+      V2 = DAG.getBitcast(NewVT, V2);
+      return DAG.getBitcast(
+          VT, DAG.getVectorShuffle(NewVT, dl, V1, V2, WidenedMask));
     }
   }
 
@@ -10397,12 +10819,11 @@ static SDValue LowerEXTRACT_VECTOR_ELT_SSE4(SDValue Op, SelectionDAG &DAG) {
     unsigned Idx = cast<ConstantSDNode>(Op.getOperand(1))->getZExtValue();
     // If Idx is 0, it's cheaper to do a move instead of a pextrw.
     if (Idx == 0)
-      return DAG.getNode(ISD::TRUNCATE, dl, MVT::i16,
-                         DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::i32,
-                                     DAG.getNode(ISD::BITCAST, dl,
-                                                 MVT::v4i32,
-                                                 Op.getOperand(0)),
-                                     Op.getOperand(1)));
+      return DAG.getNode(
+          ISD::TRUNCATE, dl, MVT::i16,
+          DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::i32,
+                      DAG.getBitcast(MVT::v4i32, Op.getOperand(0)),
+                      Op.getOperand(1)));
     SDValue Extract = DAG.getNode(X86ISD::PEXTRW, dl, MVT::i32,
                                   Op.getOperand(0), Op.getOperand(1));
     SDValue Assert  = DAG.getNode(ISD::AssertZext, dl, MVT::i32, Extract,
@@ -10426,10 +10847,9 @@ static SDValue LowerEXTRACT_VECTOR_ELT_SSE4(SDValue Op, SelectionDAG &DAG) {
          User->getValueType(0) != MVT::i32))
       return SDValue();
     SDValue Extract = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::i32,
-                                  DAG.getNode(ISD::BITCAST, dl, MVT::v4i32,
-                                              Op.getOperand(0)),
-                                              Op.getOperand(1));
-    return DAG.getNode(ISD::BITCAST, dl, MVT::f32, Extract);
+                                  DAG.getBitcast(MVT::v4i32, Op.getOperand(0)),
+                                  Op.getOperand(1));
+    return DAG.getBitcast(MVT::f32, Extract);
   }
 
   if (VT == MVT::i32 || VT == MVT::i64) {
@@ -10470,11 +10890,11 @@ X86TargetLowering::ExtractBitFromMaskVector(SDValue Op, SelectionDAG &DAG) const
     rc = getRegClassFor(MVT::v16i1);
   unsigned MaxSift = rc->getSize()*8 - 1;
   Vec = DAG.getNode(X86ISD::VSHLI, dl, VecVT, Vec,
-                    DAG.getConstant(MaxSift - IdxVal, MVT::i8));
+                    DAG.getConstant(MaxSift - IdxVal, dl, MVT::i8));
   Vec = DAG.getNode(X86ISD::VSRLI, dl, VecVT, Vec,
-                    DAG.getConstant(MaxSift, MVT::i8));
+                    DAG.getConstant(MaxSift, dl, MVT::i8));
   return DAG.getNode(X86ISD::VEXTRACT, dl, MVT::i1, Vec,
-                       DAG.getIntPtrConstant(0));
+                       DAG.getIntPtrConstant(0, dl));
 }
 
 SDValue
@@ -10499,12 +10919,13 @@ X86TargetLowering::LowerEXTRACT_VECTOR_ELT(SDValue Op,
                                     MaskEltVT.getSizeInBits());
 
       Idx = DAG.getZExtOrTrunc(Idx, dl, MaskEltVT);
+      auto PtrVT = getPointerTy(DAG.getDataLayout());
       SDValue Mask = DAG.getNode(X86ISD::VINSERT, dl, MaskVT,
-                                getZeroVector(MaskVT, Subtarget, DAG, dl),
-                                Idx, DAG.getConstant(0, getPointerTy()));
+                                 getZeroVector(MaskVT, Subtarget, DAG, dl), Idx,
+                                 DAG.getConstant(0, dl, PtrVT));
       SDValue Perm = DAG.getNode(X86ISD::VPERMV, dl, VecVT, Mask, Vec);
-      return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, Op.getValueType(),
-                        Perm, DAG.getConstant(0, getPointerTy()));
+      return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, Op.getValueType(), Perm,
+                         DAG.getConstant(0, dl, PtrVT));
     }
     return SDValue();
   }
@@ -10524,16 +10945,14 @@ X86TargetLowering::LowerEXTRACT_VECTOR_ELT(SDValue Op,
     //  IdxVal -= NumElems/2;
     IdxVal -= (IdxVal/ElemsPerChunk)*ElemsPerChunk;
     return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, Op.getValueType(), Vec,
-                       DAG.getConstant(IdxVal, MVT::i32));
+                       DAG.getConstant(IdxVal, dl, MVT::i32));
   }
 
   assert(VecVT.is128BitVector() && "Unexpected vector length");
 
-  if (Subtarget->hasSSE41()) {
-    SDValue Res = LowerEXTRACT_VECTOR_ELT_SSE4(Op, DAG);
-    if (Res.getNode())
+  if (Subtarget->hasSSE41())
+    if (SDValue Res = LowerEXTRACT_VECTOR_ELT_SSE4(Op, DAG))
       return Res;
-  }
 
   MVT VT = Op.getSimpleValueType();
   // TODO: handle v16i8.
@@ -10543,8 +10962,7 @@ X86TargetLowering::LowerEXTRACT_VECTOR_ELT(SDValue Op,
     if (Idx == 0)
       return DAG.getNode(ISD::TRUNCATE, dl, MVT::i16,
                          DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::i32,
-                                     DAG.getNode(ISD::BITCAST, dl,
-                                                 MVT::v4i32, Vec),
+                                     DAG.getBitcast(MVT::v4i32, Vec),
                                      Op.getOperand(1)));
     // Transform it so it match pextrw which produces a 32-bit result.
     MVT EltVT = MVT::i32;
@@ -10566,7 +10984,7 @@ X86TargetLowering::LowerEXTRACT_VECTOR_ELT(SDValue Op,
     SDValue Vec = DAG.getVectorShuffle(VVT, dl, Op.getOperand(0),
                                        DAG.getUNDEF(VVT), Mask);
     return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, VT, Vec,
-                       DAG.getIntPtrConstant(0));
+                       DAG.getIntPtrConstant(0, dl));
   }
 
   if (VT.getSizeInBits() == 64) {
@@ -10585,7 +11003,7 @@ X86TargetLowering::LowerEXTRACT_VECTOR_ELT(SDValue Op,
     SDValue Vec = DAG.getVectorShuffle(VVT, dl, Op.getOperand(0),
                                        DAG.getUNDEF(VVT), Mask);
     return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, VT, Vec,
-                       DAG.getIntPtrConstant(0));
+                       DAG.getIntPtrConstant(0, dl));
   }
 
   return SDValue();
@@ -10614,15 +11032,11 @@ X86TargetLowering::InsertBitToMaskVector(SDValue Op, SelectionDAG &DAG) const {
 
   unsigned IdxVal = cast<ConstantSDNode>(Idx)->getZExtValue();
   SDValue EltInVec = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VecVT, Elt);
+  if (IdxVal)
+    EltInVec = DAG.getNode(X86ISD::VSHLI, dl, VecVT, EltInVec,
+                           DAG.getConstant(IdxVal, dl, MVT::i8));
   if (Vec.getOpcode() == ISD::UNDEF)
-    return DAG.getNode(X86ISD::VSHLI, dl, VecVT, EltInVec,
-                       DAG.getConstant(IdxVal, MVT::i8));
-  const TargetRegisterClass* rc = getRegClassFor(VecVT);
-  unsigned MaxSift = rc->getSize()*8 - 1;
-  EltInVec = DAG.getNode(X86ISD::VSHLI, dl, VecVT, EltInVec,
-                    DAG.getConstant(MaxSift, MVT::i8));
-  EltInVec = DAG.getNode(X86ISD::VSRLI, dl, VecVT, EltInVec,
-                    DAG.getConstant(MaxSift - IdxVal, MVT::i8));
+    return EltInVec;
   return DAG.getNode(ISD::OR, dl, VecVT, Vec, EltInVec);
 }
 
@@ -10655,11 +11069,11 @@ SDValue X86TargetLowering::LowerINSERT_VECTOR_ELT(SDValue Op,
       if ((Subtarget->hasAVX() && (EltVT == MVT::f64 || EltVT == MVT::f32)) ||
           (Subtarget->hasAVX2() && EltVT == MVT::i32)) {
         SDValue N1Vec = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, N1);
-        N2 = DAG.getIntPtrConstant(1);
+        N2 = DAG.getIntPtrConstant(1, dl);
         return DAG.getNode(X86ISD::BLENDI, dl, VT, N0, N1Vec, N2);
       }
     }
-    
+
     // Get the desired 128-bit vector chunk.
     SDValue V = Extract128BitVector(N0, IdxVal, DAG, dl);
 
@@ -10668,7 +11082,7 @@ SDValue X86TargetLowering::LowerINSERT_VECTOR_ELT(SDValue Op,
     unsigned IdxIn128 = IdxVal - (IdxVal / NumEltsIn128) * NumEltsIn128;
 
     V = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, V.getValueType(), V, N1,
-                    DAG.getConstant(IdxIn128, MVT::i32));
+                    DAG.getConstant(IdxIn128, dl, MVT::i32));
 
     // Insert the changed part back into the bigger vector
     return Insert128BitVector(N0, V, IdxVal, DAG, dl);
@@ -10690,7 +11104,7 @@ SDValue X86TargetLowering::LowerINSERT_VECTOR_ELT(SDValue Op,
       if (N1.getValueType() != MVT::i32)
         N1 = DAG.getNode(ISD::ANY_EXTEND, dl, MVT::i32, N1);
       if (N2.getValueType() != MVT::i32)
-        N2 = DAG.getIntPtrConstant(IdxVal);
+        N2 = DAG.getIntPtrConstant(IdxVal, dl);
       return DAG.getNode(Opc, dl, VT, N0, N1, N2);
     }
 
@@ -10714,11 +11128,11 @@ SDValue X86TargetLowering::LowerINSERT_VECTOR_ELT(SDValue Op,
         // But if optimizing for size and there's a load folding opportunity,
         // generate insertps because blendps does not have a 32-bit memory
         // operand form.
-        N2 = DAG.getIntPtrConstant(1);
+        N2 = DAG.getIntPtrConstant(1, dl);
         N1 = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, MVT::v4f32, N1);
         return DAG.getNode(X86ISD::BLENDI, dl, VT, N0, N1, N2);
       }
-      N2 = DAG.getIntPtrConstant(IdxVal << 4);
+      N2 = DAG.getIntPtrConstant(IdxVal << 4, dl);
       // Create this as a scalar to vector..
       N1 = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, MVT::v4f32, N1);
       return DAG.getNode(X86ISD::INSERTPS, dl, VT, N0, N1, N2);
@@ -10739,7 +11153,7 @@ SDValue X86TargetLowering::LowerINSERT_VECTOR_ELT(SDValue Op,
     if (N1.getValueType() != MVT::i32)
       N1 = DAG.getNode(ISD::ANY_EXTEND, dl, MVT::i32, N1);
     if (N2.getValueType() != MVT::i32)
-      N2 = DAG.getIntPtrConstant(IdxVal);
+      N2 = DAG.getIntPtrConstant(IdxVal, dl);
     return DAG.getNode(X86ISD::PINSRW, dl, VT, N0, N1, N2);
   }
   return SDValue();
@@ -10769,8 +11183,8 @@ static SDValue LowerSCALAR_TO_VECTOR(SDValue Op, SelectionDAG &DAG) {
 
   SDValue AnyExt = DAG.getNode(ISD::ANY_EXTEND, dl, MVT::i32, Op.getOperand(0));
   assert(OpVT.is128BitVector() && "Expected an SSE type!");
-  return DAG.getNode(ISD::BITCAST, dl, OpVT,
-                     DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, MVT::v4i32,AnyExt));
+  return DAG.getBitcast(
+      OpVT, DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, MVT::v4i32, AnyExt));
 }
 
 // Lower a node with an EXTRACT_SUBVECTOR opcode.  This may result in
@@ -10831,9 +11245,8 @@ static SDValue LowerINSERT_SUBVECTOR(SDValue Op, const X86Subtarget *Subtarget,
     if (auto *Idx2 = dyn_cast<ConstantSDNode>(Vec.getOperand(2))) {
       if (Idx2->getZExtValue() == 0) {
         SDValue Ops[] = { SubVec2, SubVec };
-        SDValue LD = EltsFromConsecutiveLoads(OpVT, Ops, dl, DAG, false);
-        if (LD.getNode())
-          return LD;
+        if (SDValue Ld = EltsFromConsecutiveLoads(OpVT, Ops, dl, DAG, false))
+          return Ld;
       }
     }
   }
@@ -10848,10 +11261,10 @@ static SDValue LowerINSERT_SUBVECTOR(SDValue Op, const X86Subtarget *Subtarget,
   if (OpVT.getVectorElementType() == MVT::i1) {
     if (IdxVal == 0  && Vec.getOpcode() == ISD::UNDEF) // the operation is legal
       return Op;
-    SDValue ZeroIdx = DAG.getIntPtrConstant(0);
+    SDValue ZeroIdx = DAG.getIntPtrConstant(0, dl);
     SDValue Undef = DAG.getUNDEF(OpVT);
     unsigned NumElems = OpVT.getVectorNumElements();
-    SDValue ShiftBits = DAG.getConstant(NumElems/2, MVT::i8);
+    SDValue ShiftBits = DAG.getConstant(NumElems/2, dl, MVT::i8);
 
     if (IdxVal == OpVT.getVectorNumElements() / 2) {
       // Zero upper bits of the Vec
@@ -10903,17 +11316,16 @@ X86TargetLowering::LowerConstantPool(SDValue Op, SelectionDAG &DAG) const {
   else if (Subtarget->isPICStyleStubPIC())
     OpFlag = X86II::MO_PIC_BASE_OFFSET;
 
-  SDValue Result = DAG.getTargetConstantPool(CP->getConstVal(), getPointerTy(),
-                                             CP->getAlignment(),
-                                             CP->getOffset(), OpFlag);
+  auto PtrVT = getPointerTy(DAG.getDataLayout());
+  SDValue Result = DAG.getTargetConstantPool(
+      CP->getConstVal(), PtrVT, CP->getAlignment(), CP->getOffset(), OpFlag);
   SDLoc DL(CP);
-  Result = DAG.getNode(WrapperKind, DL, getPointerTy(), Result);
+  Result = DAG.getNode(WrapperKind, DL, PtrVT, Result);
   // With PIC, the address is actually $g + Offset.
   if (OpFlag) {
-    Result = DAG.getNode(ISD::ADD, DL, getPointerTy(),
-                         DAG.getNode(X86ISD::GlobalBaseReg,
-                                     SDLoc(), getPointerTy()),
-                         Result);
+    Result =
+        DAG.getNode(ISD::ADD, DL, PtrVT,
+                    DAG.getNode(X86ISD::GlobalBaseReg, SDLoc(), PtrVT), Result);
   }
 
   return Result;
@@ -10936,17 +11348,16 @@ SDValue X86TargetLowering::LowerJumpTable(SDValue Op, SelectionDAG &DAG) const {
   else if (Subtarget->isPICStyleStubPIC())
     OpFlag = X86II::MO_PIC_BASE_OFFSET;
 
-  SDValue Result = DAG.getTargetJumpTable(JT->getIndex(), getPointerTy(),
-                                          OpFlag);
+  auto PtrVT = getPointerTy(DAG.getDataLayout());
+  SDValue Result = DAG.getTargetJumpTable(JT->getIndex(), PtrVT, OpFlag);
   SDLoc DL(JT);
-  Result = DAG.getNode(WrapperKind, DL, getPointerTy(), Result);
+  Result = DAG.getNode(WrapperKind, DL, PtrVT, Result);
 
   // With PIC, the address is actually $g + Offset.
   if (OpFlag)
-    Result = DAG.getNode(ISD::ADD, DL, getPointerTy(),
-                         DAG.getNode(X86ISD::GlobalBaseReg,
-                                     SDLoc(), getPointerTy()),
-                         Result);
+    Result =
+        DAG.getNode(ISD::ADD, DL, PtrVT,
+                    DAG.getNode(X86ISD::GlobalBaseReg, SDLoc(), PtrVT), Result);
 
   return Result;
 }
@@ -10974,24 +11385,24 @@ X86TargetLowering::LowerExternalSymbol(SDValue Op, SelectionDAG &DAG) const {
     OpFlag = X86II::MO_DARWIN_NONLAZY;
   }
 
-  SDValue Result = DAG.getTargetExternalSymbol(Sym, getPointerTy(), OpFlag);
+  auto PtrVT = getPointerTy(DAG.getDataLayout());
+  SDValue Result = DAG.getTargetExternalSymbol(Sym, PtrVT, OpFlag);
 
   SDLoc DL(Op);
-  Result = DAG.getNode(WrapperKind, DL, getPointerTy(), Result);
+  Result = DAG.getNode(WrapperKind, DL, PtrVT, Result);
 
   // With PIC, the address is actually $g + Offset.
   if (DAG.getTarget().getRelocationModel() == Reloc::PIC_ &&
       !Subtarget->is64Bit()) {
-    Result = DAG.getNode(ISD::ADD, DL, getPointerTy(),
-                         DAG.getNode(X86ISD::GlobalBaseReg,
-                                     SDLoc(), getPointerTy()),
-                         Result);
+    Result =
+        DAG.getNode(ISD::ADD, DL, PtrVT,
+                    DAG.getNode(X86ISD::GlobalBaseReg, SDLoc(), PtrVT), Result);
   }
 
   // For symbols that require a load from a stub to get the address, emit the
   // load.
   if (isGlobalStubReference(OpFlag))
-    Result = DAG.getLoad(getPointerTy(), DL, DAG.getEntryNode(), Result,
+    Result = DAG.getLoad(PtrVT, DL, DAG.getEntryNode(), Result,
                          MachinePointerInfo::getGOT(), false, false, false, 0);
 
   return Result;
@@ -11006,20 +11417,19 @@ X86TargetLowering::LowerBlockAddress(SDValue Op, SelectionDAG &DAG) const {
   const BlockAddress *BA = cast<BlockAddressSDNode>(Op)->getBlockAddress();
   int64_t Offset = cast<BlockAddressSDNode>(Op)->getOffset();
   SDLoc dl(Op);
-  SDValue Result = DAG.getTargetBlockAddress(BA, getPointerTy(), Offset,
-                                             OpFlags);
+  auto PtrVT = getPointerTy(DAG.getDataLayout());
+  SDValue Result = DAG.getTargetBlockAddress(BA, PtrVT, Offset, OpFlags);
 
   if (Subtarget->isPICStyleRIPRel() &&
       (M == CodeModel::Small || M == CodeModel::Kernel))
-    Result = DAG.getNode(X86ISD::WrapperRIP, dl, getPointerTy(), Result);
+    Result = DAG.getNode(X86ISD::WrapperRIP, dl, PtrVT, Result);
   else
-    Result = DAG.getNode(X86ISD::Wrapper, dl, getPointerTy(), Result);
+    Result = DAG.getNode(X86ISD::Wrapper, dl, PtrVT, Result);
 
   // With PIC, the address is actually $g + Offset.
   if (isGlobalRelativeToPICBase(OpFlags)) {
-    Result = DAG.getNode(ISD::ADD, dl, getPointerTy(),
-                         DAG.getNode(X86ISD::GlobalBaseReg, dl, getPointerTy()),
-                         Result);
+    Result = DAG.getNode(ISD::ADD, dl, PtrVT,
+                         DAG.getNode(X86ISD::GlobalBaseReg, dl, PtrVT), Result);
   }
 
   return Result;
@@ -11033,40 +11443,40 @@ X86TargetLowering::LowerGlobalAddress(const GlobalValue *GV, SDLoc dl,
   unsigned char OpFlags =
       Subtarget->ClassifyGlobalReference(GV, DAG.getTarget());
   CodeModel::Model M = DAG.getTarget().getCodeModel();
+  auto PtrVT = getPointerTy(DAG.getDataLayout());
   SDValue Result;
   if (OpFlags == X86II::MO_NO_FLAG &&
       X86::isOffsetSuitableForCodeModel(Offset, M)) {
     // A direct static reference to a global.
-    Result = DAG.getTargetGlobalAddress(GV, dl, getPointerTy(), Offset);
+    Result = DAG.getTargetGlobalAddress(GV, dl, PtrVT, Offset);
     Offset = 0;
   } else {
-    Result = DAG.getTargetGlobalAddress(GV, dl, getPointerTy(), 0, OpFlags);
+    Result = DAG.getTargetGlobalAddress(GV, dl, PtrVT, 0, OpFlags);
   }
 
   if (Subtarget->isPICStyleRIPRel() &&
       (M == CodeModel::Small || M == CodeModel::Kernel))
-    Result = DAG.getNode(X86ISD::WrapperRIP, dl, getPointerTy(), Result);
+    Result = DAG.getNode(X86ISD::WrapperRIP, dl, PtrVT, Result);
   else
-    Result = DAG.getNode(X86ISD::Wrapper, dl, getPointerTy(), Result);
+    Result = DAG.getNode(X86ISD::Wrapper, dl, PtrVT, Result);
 
   // With PIC, the address is actually $g + Offset.
   if (isGlobalRelativeToPICBase(OpFlags)) {
-    Result = DAG.getNode(ISD::ADD, dl, getPointerTy(),
-                         DAG.getNode(X86ISD::GlobalBaseReg, dl, getPointerTy()),
-                         Result);
+    Result = DAG.getNode(ISD::ADD, dl, PtrVT,
+                         DAG.getNode(X86ISD::GlobalBaseReg, dl, PtrVT), Result);
   }
 
   // For globals that require a load from a stub to get the address, emit the
   // load.
   if (isGlobalStubReference(OpFlags))
-    Result = DAG.getLoad(getPointerTy(), dl, DAG.getEntryNode(), Result,
+    Result = DAG.getLoad(PtrVT, dl, DAG.getEntryNode(), Result,
                          MachinePointerInfo::getGOT(), false, false, false, 0);
 
   // If there was a non-zero offset that we didn't fold, create an explicit
   // addition for it.
   if (Offset != 0)
-    Result = DAG.getNode(ISD::ADD, dl, getPointerTy(), Result,
-                         DAG.getConstant(Offset, getPointerTy()));
+    Result = DAG.getNode(ISD::ADD, dl, PtrVT, Result,
+                         DAG.getConstant(Offset, dl, PtrVT));
 
   return Result;
 }
@@ -11081,8 +11491,7 @@ X86TargetLowering::LowerGlobalAddress(SDValue Op, SelectionDAG &DAG) const {
 static SDValue
 GetTLSADDR(SelectionDAG &DAG, SDValue Chain, GlobalAddressSDNode *GA,
            SDValue *InFlag, const EVT PtrVT, unsigned ReturnReg,
-           unsigned char OperandFlags, bool LocalDynamic = false,
-           unsigned Opcode = ISD::DELETED_NODE) { // @LOCALMOD
+           unsigned char OperandFlags, bool LocalDynamic = false) {
   MachineFrameInfo *MFI = DAG.getMachineFunction().getFrameInfo();
   SDVTList NodeTys = DAG.getVTList(MVT::Other, MVT::Glue);
   SDLoc dl(GA);
@@ -11091,15 +11500,8 @@ GetTLSADDR(SelectionDAG &DAG, SDValue Chain, GlobalAddressSDNode *GA,
                                            GA->getOffset(),
                                            OperandFlags);
 
-  // @LOCALMOD - changed type for casting
-  unsigned CallType = LocalDynamic ? X86ISD::TLSBASEADDR
-                                   : X86ISD::TLSADDR;
-
-  // @LOCALMOD-START
-  // If Opcode was explicitly overridden, use it as the call type.
-  if (Opcode != ISD::DELETED_NODE)
-    CallType = Opcode;
-  // @LOCALMOD-END
+  X86ISD::NodeType CallType = LocalDynamic ? X86ISD::TLSBASEADDR
+                                           : X86ISD::TLSADDR;
 
   if (InFlag) {
     SDValue Ops[] = { Chain,  TGA, *InFlag };
@@ -11139,52 +11541,6 @@ LowerToTLSGeneralDynamicModel64(GlobalAddressSDNode *GA, SelectionDAG &DAG,
                     X86::RAX, X86II::MO_TLSGD);
 }
 
-// Lower ISD::GlobalTLSAddress using the "initial exec" or "local exec" model.
-static SDValue
-LowerToTLSExecCall(GlobalAddressSDNode *GA, SelectionDAG &DAG,
-                   const EVT PtrVT, TLSModel::Model model, bool is64Bit) {
-
-  // See: http://code.google.com/p/nativeclient/issues/detail?id=1685
-  unsigned char TargetFlag;
-  unsigned Opcode;
-  if (model == TLSModel::LocalExec) {
-    TargetFlag = is64Bit ? X86II::MO_TPOFF : X86II::MO_NTPOFF;
-    Opcode = X86ISD::TLSADDR_LE;
-  } else if (model == TLSModel::InitialExec) {
-    TargetFlag = is64Bit ? X86II::MO_GOTTPOFF : X86II::MO_INDNTPOFF;
-    Opcode = X86ISD::TLSADDR_IE;
-  } else {
-    llvm_unreachable("Unknown TLS model");
-  }
-
-  return GetTLSADDR(DAG, DAG.getEntryNode(), GA, NULL, PtrVT,
-                    X86::EAX, // PtrVT is 32-bit.
-                    TargetFlag, false, Opcode);
-}
-
-// @LOCALMOD-START
-// Lower TLS accesses to a function call, rather than use segment registers.
-// Lower ISD::GlobalTLSAddress for NaCl 64 bit.
-static SDValue
-LowerToTLSNaCl64(GlobalAddressSDNode *GA, SelectionDAG &DAG,
-                 const EVT PtrVT, TLSModel::Model model) {
-
-  // See: http://code.google.com/p/nativeclient/issues/detail?id=1685
-  unsigned char TargetFlag;
-  unsigned Opcode;
-  if (model == TLSModel::GeneralDynamic || model == TLSModel::LocalDynamic) {
-    TargetFlag = X86II::MO_TLSGD;
-    Opcode = X86ISD::TLSADDR;
-  } else {
-    return LowerToTLSExecCall(GA, DAG, PtrVT, model, true);
-  }
-
-  return GetTLSADDR(DAG, DAG.getEntryNode(), GA, NULL, PtrVT,
-                    X86::EAX, // PtrVT is 32-bit.
-                    TargetFlag, false, Opcode);
-}
-// @LOCALMOD-END
-
 static SDValue LowerToTLSLocalDynamicModel(GlobalAddressSDNode *GA,
                                            SelectionDAG &DAG,
                                            const EVT PtrVT,
@@ -11235,7 +11591,7 @@ static SDValue LowerToTLSExecModel(GlobalAddressSDNode *GA, SelectionDAG &DAG,
                                                          is64Bit ? 257 : 256));
 
   SDValue ThreadPointer =
-      DAG.getLoad(PtrVT, dl, DAG.getEntryNode(), DAG.getIntPtrConstant(0),
+      DAG.getLoad(PtrVT, dl, DAG.getEntryNode(), DAG.getIntPtrConstant(0, dl),
                   MachinePointerInfo(Ptr), false, false, false, 0);
 
   unsigned char OperandFlags = 0;
@@ -11284,35 +11640,25 @@ X86TargetLowering::LowerGlobalTLSAddress(SDValue Op, SelectionDAG &DAG) const {
 
   GlobalAddressSDNode *GA = cast<GlobalAddressSDNode>(Op);
   const GlobalValue *GV = GA->getGlobal();
+  auto PtrVT = getPointerTy(DAG.getDataLayout());
 
   if (Subtarget->isTargetELF()) {
+    if (DAG.getTarget().Options.EmulatedTLS)
+      return LowerToTLSEmulatedModel(GA, DAG);
     TLSModel::Model model = DAG.getTarget().getTLSModel(GV);
-
-    // @LOCALMOD-START
-    if (Subtarget->isTargetNaCl64())
-      return LowerToTLSNaCl64(GA, DAG, getPointerTy(), model);
-    // @LOCALMOD-END
-
     switch (model) {
       case TLSModel::GeneralDynamic:
         if (Subtarget->is64Bit())
-          return LowerToTLSGeneralDynamicModel64(GA, DAG, getPointerTy());
-        return LowerToTLSGeneralDynamicModel32(GA, DAG, getPointerTy());
+          return LowerToTLSGeneralDynamicModel64(GA, DAG, PtrVT);
+        return LowerToTLSGeneralDynamicModel32(GA, DAG, PtrVT);
       case TLSModel::LocalDynamic:
-        return LowerToTLSLocalDynamicModel(GA, DAG, getPointerTy(),
+        return LowerToTLSLocalDynamicModel(GA, DAG, PtrVT,
                                            Subtarget->is64Bit());
       case TLSModel::InitialExec:
       case TLSModel::LocalExec:
-        // @LOCALMOD-START
-        if (llvm::TLSUseCall && Subtarget->isTargetNaCl()) {
-          return LowerToTLSExecCall(GA, DAG, getPointerTy(), model,
-                                    Subtarget->is64Bit());
-        } else {
-        return LowerToTLSExecModel(
-            GA, DAG, getPointerTy(), model, Subtarget->is64Bit(),
-            DAG.getTarget().getRelocationModel() == Reloc::PIC_);
-        }
-        // @LOCALMOD-END
+        return LowerToTLSExecModel(GA, DAG, PtrVT, model, Subtarget->is64Bit(),
+                                   DAG.getTarget().getRelocationModel() ==
+                                       Reloc::PIC_);
     }
     llvm_unreachable("Unknown TLS model.");
   }
@@ -11335,13 +11681,12 @@ X86TargetLowering::LowerGlobalTLSAddress(SDValue Op, SelectionDAG &DAG) const {
     SDValue Result = DAG.getTargetGlobalAddress(GA->getGlobal(), DL,
                                                 GA->getValueType(0),
                                                 GA->getOffset(), OpFlag);
-    SDValue Offset = DAG.getNode(WrapperKind, DL, getPointerTy(), Result);
+    SDValue Offset = DAG.getNode(WrapperKind, DL, PtrVT, Result);
 
     // With PIC32, the address is actually $g + Offset.
     if (PIC32)
-      Offset = DAG.getNode(ISD::ADD, DL, getPointerTy(),
-                           DAG.getNode(X86ISD::GlobalBaseReg,
-                                       SDLoc(), getPointerTy()),
+      Offset = DAG.getNode(ISD::ADD, DL, PtrVT,
+                           DAG.getNode(X86ISD::GlobalBaseReg, SDLoc(), PtrVT),
                            Offset);
 
     // Lowering the machine isd will make sure everything is in the right
@@ -11358,8 +11703,7 @@ X86TargetLowering::LowerGlobalTLSAddress(SDValue Op, SelectionDAG &DAG) const {
     // And our return value (tls address) is in the standard call return value
     // location.
     unsigned Reg = Subtarget->is64Bit() ? X86::RAX : X86::EAX;
-    return DAG.getCopyFromReg(Chain, DL, Reg, getPointerTy(),
-                              Chain.getValue(1));
+    return DAG.getCopyFromReg(Chain, DL, Reg, PtrVT, Chain.getValue(1));
   }
 
   if (Subtarget->isTargetKnownWindowsMSVC() ||
@@ -11387,44 +11731,50 @@ X86TargetLowering::LowerGlobalTLSAddress(SDValue Op, SelectionDAG &DAG) const {
                                         : Type::getInt32PtrTy(*DAG.getContext(),
                                                               257));
 
-    SDValue TlsArray =
-        Subtarget->is64Bit()
-            ? DAG.getIntPtrConstant(0x58)
-            : (Subtarget->isTargetWindowsGNU()
-                   ? DAG.getIntPtrConstant(0x2C)
-                   : DAG.getExternalSymbol("_tls_array", getPointerTy()));
+    SDValue TlsArray = Subtarget->is64Bit()
+                           ? DAG.getIntPtrConstant(0x58, dl)
+                           : (Subtarget->isTargetWindowsGNU()
+                                  ? DAG.getIntPtrConstant(0x2C, dl)
+                                  : DAG.getExternalSymbol("_tls_array", PtrVT));
 
     SDValue ThreadPointer =
-        DAG.getLoad(getPointerTy(), dl, Chain, TlsArray,
-                    MachinePointerInfo(Ptr), false, false, false, 0);
+        DAG.getLoad(PtrVT, dl, Chain, TlsArray, MachinePointerInfo(Ptr), false,
+                    false, false, 0);
 
-    // Load the _tls_index variable
-    SDValue IDX = DAG.getExternalSymbol("_tls_index", getPointerTy());
-    if (Subtarget->is64Bit())
-      IDX = DAG.getExtLoad(ISD::ZEXTLOAD, dl, getPointerTy(), Chain,
-                           IDX, MachinePointerInfo(), MVT::i32,
-                           false, false, false, 0);
-    else
-      IDX = DAG.getLoad(getPointerTy(), dl, Chain, IDX, MachinePointerInfo(),
-                        false, false, false, 0);
+    SDValue res;
+    if (GV->getThreadLocalMode() == GlobalVariable::LocalExecTLSModel) {
+      res = ThreadPointer;
+    } else {
+      // Load the _tls_index variable
+      SDValue IDX = DAG.getExternalSymbol("_tls_index", PtrVT);
+      if (Subtarget->is64Bit())
+        IDX = DAG.getExtLoad(ISD::ZEXTLOAD, dl, PtrVT, Chain, IDX,
+                             MachinePointerInfo(), MVT::i32, false, false,
+                             false, 0);
+      else
+        IDX = DAG.getLoad(PtrVT, dl, Chain, IDX, MachinePointerInfo(), false,
+                          false, false, 0);
 
-    SDValue Scale = DAG.getConstant(Log2_64_Ceil(TD->getPointerSize()),
-                                    getPointerTy());
-    IDX = DAG.getNode(ISD::SHL, dl, getPointerTy(), IDX, Scale);
+      auto &DL = DAG.getDataLayout();
+      SDValue Scale =
+          DAG.getConstant(Log2_64_Ceil(DL.getPointerSize()), dl, PtrVT);
+      IDX = DAG.getNode(ISD::SHL, dl, PtrVT, IDX, Scale);
 
-    SDValue res = DAG.getNode(ISD::ADD, dl, getPointerTy(), ThreadPointer, IDX);
-    res = DAG.getLoad(getPointerTy(), dl, Chain, res, MachinePointerInfo(),
-                      false, false, false, 0);
+      res = DAG.getNode(ISD::ADD, dl, PtrVT, ThreadPointer, IDX);
+    }
+
+    res = DAG.getLoad(PtrVT, dl, Chain, res, MachinePointerInfo(), false, false,
+                      false, 0);
 
     // Get the offset of start of .tls section
     SDValue TGA = DAG.getTargetGlobalAddress(GA->getGlobal(), dl,
                                              GA->getValueType(0),
                                              GA->getOffset(), X86II::MO_SECREL);
-    SDValue Offset = DAG.getNode(X86ISD::Wrapper, dl, getPointerTy(), TGA);
+    SDValue Offset = DAG.getNode(X86ISD::Wrapper, dl, PtrVT, TGA);
 
     // The address of the thread local variable is the add of the thread
     // pointer with the offset of the variable.
-    return DAG.getNode(ISD::ADD, dl, getPointerTy(), res, Offset);
+    return DAG.getNode(ISD::ADD, dl, PtrVT, res, Offset);
   }
 
   llvm_unreachable("TLS not implemented for this target.");
@@ -11445,10 +11795,10 @@ static SDValue LowerShiftParts(SDValue Op, SelectionDAG &DAG) {
   // generic ISD nodes haven't. Insert an AND to be safe, it's optimized away
   // during isel.
   SDValue SafeShAmt = DAG.getNode(ISD::AND, dl, MVT::i8, ShAmt,
-                                  DAG.getConstant(VTBits - 1, MVT::i8));
+                                  DAG.getConstant(VTBits - 1, dl, MVT::i8));
   SDValue Tmp1 = isSRA ? DAG.getNode(ISD::SRA, dl, VT, ShOpHi,
-                                     DAG.getConstant(VTBits - 1, MVT::i8))
-                       : DAG.getConstant(0, VT);
+                                     DAG.getConstant(VTBits - 1, dl, MVT::i8))
+                       : DAG.getConstant(0, dl, VT);
 
   SDValue Tmp2, Tmp3;
   if (Op.getOpcode() == ISD::SHL_PARTS) {
@@ -11463,12 +11813,12 @@ static SDValue LowerShiftParts(SDValue Op, SelectionDAG &DAG) {
   // rely on the results of shld/shrd. Insert a test and select the appropriate
   // values for large shift amounts.
   SDValue AndNode = DAG.getNode(ISD::AND, dl, MVT::i8, ShAmt,
-                                DAG.getConstant(VTBits, MVT::i8));
+                                DAG.getConstant(VTBits, dl, MVT::i8));
   SDValue Cond = DAG.getNode(X86ISD::CMP, dl, MVT::i32,
-                             AndNode, DAG.getConstant(0, MVT::i8));
+                             AndNode, DAG.getConstant(0, dl, MVT::i8));
 
   SDValue Hi, Lo;
-  SDValue CC = DAG.getConstant(X86::COND_NE, MVT::i8);
+  SDValue CC = DAG.getConstant(X86::COND_NE, dl, MVT::i8);
   SDValue Ops0[4] = { Tmp2, Tmp3, CC, Cond };
   SDValue Ops1[4] = { Tmp3, Tmp1, CC, Cond };
 
@@ -11486,15 +11836,21 @@ static SDValue LowerShiftParts(SDValue Op, SelectionDAG &DAG) {
 
 SDValue X86TargetLowering::LowerSINT_TO_FP(SDValue Op,
                                            SelectionDAG &DAG) const {
-  MVT SrcVT = Op.getOperand(0).getSimpleValueType();
+  SDValue Src = Op.getOperand(0);
+  MVT SrcVT = Src.getSimpleValueType();
+  MVT VT = Op.getSimpleValueType();
   SDLoc dl(Op);
 
   if (SrcVT.isVector()) {
+    if (SrcVT == MVT::v2i32 && VT == MVT::v2f64) {
+      return DAG.getNode(X86ISD::CVTDQ2PD, dl, VT,
+                         DAG.getNode(ISD::CONCAT_VECTORS, dl, MVT::v4i32, Src,
+                         DAG.getUNDEF(SrcVT)));
+    }
     if (SrcVT.getVectorElementType() == MVT::i1) {
       MVT IntegerVT = MVT::getVectorVT(MVT::i32, SrcVT.getVectorNumElements());
       return DAG.getNode(ISD::SINT_TO_FP, dl, Op.getValueType(),
-                         DAG.getNode(ISD::SIGN_EXTEND, dl, IntegerVT,
-                                     Op.getOperand(0)));
+                         DAG.getNode(ISD::SIGN_EXTEND, dl, IntegerVT, Src));
     }
     return SDValue();
   }
@@ -11513,8 +11869,9 @@ SDValue X86TargetLowering::LowerSINT_TO_FP(SDValue Op,
 
   unsigned Size = SrcVT.getSizeInBits()/8;
   MachineFunction &MF = DAG.getMachineFunction();
+  auto PtrVT = getPointerTy(MF.getDataLayout());
   int SSFI = MF.getFrameInfo()->CreateStackObject(Size, Size, false);
-  SDValue StackSlot = DAG.getFrameIndex(SSFI, getPointerTy());
+  SDValue StackSlot = DAG.getFrameIndex(SSFI, PtrVT);
   SDValue Chain = DAG.getStore(DAG.getEntryNode(), dl, Op.getOperand(0),
                                StackSlot,
                                MachinePointerInfo::getFixedStack(SSFI),
@@ -11563,7 +11920,8 @@ SDValue X86TargetLowering::BuildFILD(SDValue Op, EVT SrcVT, SDValue Chain,
     MachineFunction &MF = DAG.getMachineFunction();
     unsigned SSFISize = Op.getValueType().getSizeInBits()/8;
     int SSFI = MF.getFrameInfo()->CreateStackObject(SSFISize, SSFISize, false);
-    SDValue StackSlot = DAG.getFrameIndex(SSFI, getPointerTy());
+    auto PtrVT = getPointerTy(MF.getDataLayout());
+    SDValue StackSlot = DAG.getFrameIndex(SSFI, PtrVT);
     Tys = DAG.getVTList(MVT::Other);
     SDValue Ops[] = {
       Chain, Result, StackSlot, DAG.getValueType(Op.getValueType()), InFlag
@@ -11605,7 +11963,8 @@ SDValue X86TargetLowering::LowerUINT_TO_FP_i64(SDValue Op,
   // Build some magic constants.
   static const uint32_t CV0[] = { 0x43300000, 0x45300000, 0, 0 };
   Constant *C0 = ConstantDataVector::get(*Context, CV0);
-  SDValue CPIdx0 = DAG.getConstantPool(C0, getPointerTy(), 16);
+  auto PtrVT = getPointerTy(DAG.getDataLayout());
+  SDValue CPIdx0 = DAG.getConstantPool(C0, PtrVT, 16);
 
   SmallVector<Constant*,2> CV1;
   CV1.push_back(
@@ -11615,7 +11974,7 @@ SDValue X86TargetLowering::LowerUINT_TO_FP_i64(SDValue Op,
     ConstantFP::get(*Context, APFloat(APFloat::IEEEdouble,
                                       APInt(64, 0x4530000000000000ULL))));
   Constant *C1 = ConstantVector::get(CV1);
-  SDValue CPIdx1 = DAG.getConstantPool(C1, getPointerTy(), 16);
+  SDValue CPIdx1 = DAG.getConstantPool(C1, PtrVT, 16);
 
   // Load the 64-bit value into an XMM register.
   SDValue XR1 = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, MVT::v2i64,
@@ -11623,14 +11982,13 @@ SDValue X86TargetLowering::LowerUINT_TO_FP_i64(SDValue Op,
   SDValue CLod0 = DAG.getLoad(MVT::v4i32, dl, DAG.getEntryNode(), CPIdx0,
                               MachinePointerInfo::getConstantPool(),
                               false, false, false, 16);
-  SDValue Unpck1 = getUnpackl(DAG, dl, MVT::v4i32,
-                              DAG.getNode(ISD::BITCAST, dl, MVT::v4i32, XR1),
-                              CLod0);
+  SDValue Unpck1 =
+      getUnpackl(DAG, dl, MVT::v4i32, DAG.getBitcast(MVT::v4i32, XR1), CLod0);
 
   SDValue CLod1 = DAG.getLoad(MVT::v2f64, dl, CLod0.getValue(1), CPIdx1,
                               MachinePointerInfo::getConstantPool(),
                               false, false, false, 16);
-  SDValue XR2F = DAG.getNode(ISD::BITCAST, dl, MVT::v2f64, Unpck1);
+  SDValue XR2F = DAG.getBitcast(MVT::v2f64, Unpck1);
   SDValue Sub = DAG.getNode(ISD::FSUB, dl, MVT::v2f64, XR2F, CLod1);
   SDValue Result;
 
@@ -11638,16 +11996,15 @@ SDValue X86TargetLowering::LowerUINT_TO_FP_i64(SDValue Op,
     // FIXME: The 'haddpd' instruction may be slower than 'movhlps + addsd'.
     Result = DAG.getNode(X86ISD::FHADD, dl, MVT::v2f64, Sub, Sub);
   } else {
-    SDValue S2F = DAG.getNode(ISD::BITCAST, dl, MVT::v4i32, Sub);
+    SDValue S2F = DAG.getBitcast(MVT::v4i32, Sub);
     SDValue Shuffle = getTargetShuffleNode(X86ISD::PSHUFD, dl, MVT::v4i32,
                                            S2F, 0x4E, DAG);
     Result = DAG.getNode(ISD::FADD, dl, MVT::v2f64,
-                         DAG.getNode(ISD::BITCAST, dl, MVT::v2f64, Shuffle),
-                         Sub);
+                         DAG.getBitcast(MVT::v2f64, Shuffle), Sub);
   }
 
   return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::f64, Result,
-                     DAG.getIntPtrConstant(0));
+                     DAG.getIntPtrConstant(0, dl));
 }
 
 // LowerUINT_TO_FP_i32 - 32-bit unsigned integer to float expansion.
@@ -11655,7 +12012,7 @@ SDValue X86TargetLowering::LowerUINT_TO_FP_i32(SDValue Op,
                                                SelectionDAG &DAG) const {
   SDLoc dl(Op);
   // FP constant to bias correct the final result.
-  SDValue Bias = DAG.getConstantFP(BitsToDouble(0x4330000000000000ULL),
+  SDValue Bias = DAG.getConstantFP(BitsToDouble(0x4330000000000000ULL), dl,
                                    MVT::f64);
 
   // Load the 32-bit value into an XMM register.
@@ -11666,20 +12023,19 @@ SDValue X86TargetLowering::LowerUINT_TO_FP_i32(SDValue Op,
   Load = getShuffleVectorZeroOrUndef(Load, 0, true, Subtarget, DAG);
 
   Load = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::f64,
-                     DAG.getNode(ISD::BITCAST, dl, MVT::v2f64, Load),
-                     DAG.getIntPtrConstant(0));
+                     DAG.getBitcast(MVT::v2f64, Load),
+                     DAG.getIntPtrConstant(0, dl));
 
   // Or the load with the bias.
-  SDValue Or = DAG.getNode(ISD::OR, dl, MVT::v2i64,
-                           DAG.getNode(ISD::BITCAST, dl, MVT::v2i64,
-                                       DAG.getNode(ISD::SCALAR_TO_VECTOR, dl,
-                                                   MVT::v2f64, Load)),
-                           DAG.getNode(ISD::BITCAST, dl, MVT::v2i64,
-                                       DAG.getNode(ISD::SCALAR_TO_VECTOR, dl,
-                                                   MVT::v2f64, Bias)));
-  Or = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::f64,
-                   DAG.getNode(ISD::BITCAST, dl, MVT::v2f64, Or),
-                   DAG.getIntPtrConstant(0));
+  SDValue Or = DAG.getNode(
+      ISD::OR, dl, MVT::v2i64,
+      DAG.getBitcast(MVT::v2i64,
+                     DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, MVT::v2f64, Load)),
+      DAG.getBitcast(MVT::v2i64,
+                     DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, MVT::v2f64, Bias)));
+  Or =
+      DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::f64,
+                  DAG.getBitcast(MVT::v2f64, Or), DAG.getIntPtrConstant(0, dl));
 
   // Subtract the bias.
   SDValue Sub = DAG.getNode(ISD::FSUB, dl, MVT::f64, Or, Bias);
@@ -11689,7 +12045,7 @@ SDValue X86TargetLowering::LowerUINT_TO_FP_i32(SDValue Op,
 
   if (DestVT.bitsLT(MVT::f64))
     return DAG.getNode(ISD::FP_ROUND, dl, DestVT, Sub,
-                       DAG.getIntPtrConstant(0));
+                       DAG.getIntPtrConstant(0, dl));
   if (DestVT.bitsGT(MVT::f64))
     return DAG.getNode(ISD::FP_EXTEND, dl, DestVT, Sub);
 
@@ -11734,20 +12090,20 @@ static SDValue lowerUINT_TO_FP_vXi32(SDValue Op, SelectionDAG &DAG,
   // -- v >> 16
 
   // Create the splat vector for 0x4b000000.
-  SDValue CstLow = DAG.getConstant(0x4b000000, MVT::i32);
+  SDValue CstLow = DAG.getConstant(0x4b000000, DL, MVT::i32);
   SDValue CstLowArray[] = {CstLow, CstLow, CstLow, CstLow,
                            CstLow, CstLow, CstLow, CstLow};
   SDValue VecCstLow = DAG.getNode(ISD::BUILD_VECTOR, DL, VecIntVT,
                                   makeArrayRef(&CstLowArray[0], NumElts));
   // Create the splat vector for 0x53000000.
-  SDValue CstHigh = DAG.getConstant(0x53000000, MVT::i32);
+  SDValue CstHigh = DAG.getConstant(0x53000000, DL, MVT::i32);
   SDValue CstHighArray[] = {CstHigh, CstHigh, CstHigh, CstHigh,
                             CstHigh, CstHigh, CstHigh, CstHigh};
   SDValue VecCstHigh = DAG.getNode(ISD::BUILD_VECTOR, DL, VecIntVT,
                                    makeArrayRef(&CstHighArray[0], NumElts));
 
   // Create the right shift.
-  SDValue CstShift = DAG.getConstant(16, MVT::i32);
+  SDValue CstShift = DAG.getConstant(16, DL, MVT::i32);
   SDValue CstShiftArray[] = {CstShift, CstShift, CstShift, CstShift,
                              CstShift, CstShift, CstShift, CstShift};
   SDValue VecCstShift = DAG.getNode(ISD::BUILD_VECTOR, DL, VecIntVT,
@@ -11758,25 +12114,22 @@ static SDValue lowerUINT_TO_FP_vXi32(SDValue Op, SelectionDAG &DAG,
   if (Subtarget.hasSSE41()) {
     EVT VecI16VT = Is128 ? MVT::v8i16 : MVT::v16i16;
     //     uint4 lo = _mm_blend_epi16( v, (uint4) 0x4b000000, 0xaa);
-    SDValue VecCstLowBitcast =
-        DAG.getNode(ISD::BITCAST, DL, VecI16VT, VecCstLow);
-    SDValue VecBitcast = DAG.getNode(ISD::BITCAST, DL, VecI16VT, V);
+    SDValue VecCstLowBitcast = DAG.getBitcast(VecI16VT, VecCstLow);
+    SDValue VecBitcast = DAG.getBitcast(VecI16VT, V);
     // Low will be bitcasted right away, so do not bother bitcasting back to its
     // original type.
     Low = DAG.getNode(X86ISD::BLENDI, DL, VecI16VT, VecBitcast,
-                      VecCstLowBitcast, DAG.getConstant(0xaa, MVT::i32));
+                      VecCstLowBitcast, DAG.getConstant(0xaa, DL, MVT::i32));
     //     uint4 hi = _mm_blend_epi16( _mm_srli_epi32(v,16),
     //                                 (uint4) 0x53000000, 0xaa);
-    SDValue VecCstHighBitcast =
-        DAG.getNode(ISD::BITCAST, DL, VecI16VT, VecCstHigh);
-    SDValue VecShiftBitcast =
-        DAG.getNode(ISD::BITCAST, DL, VecI16VT, HighShift);
+    SDValue VecCstHighBitcast = DAG.getBitcast(VecI16VT, VecCstHigh);
+    SDValue VecShiftBitcast = DAG.getBitcast(VecI16VT, HighShift);
     // High will be bitcasted right away, so do not bother bitcasting back to
     // its original type.
     High = DAG.getNode(X86ISD::BLENDI, DL, VecI16VT, VecShiftBitcast,
-                       VecCstHighBitcast, DAG.getConstant(0xaa, MVT::i32));
+                       VecCstHighBitcast, DAG.getConstant(0xaa, DL, MVT::i32));
   } else {
-    SDValue CstMask = DAG.getConstant(0xffff, MVT::i32);
+    SDValue CstMask = DAG.getConstant(0xffff, DL, MVT::i32);
     SDValue VecCstMask = DAG.getNode(ISD::BUILD_VECTOR, DL, VecIntVT, CstMask,
                                      CstMask, CstMask, CstMask);
     //     uint4 lo = (v & (uint4) 0xffff) | (uint4) 0x4b000000;
@@ -11789,18 +12142,18 @@ static SDValue lowerUINT_TO_FP_vXi32(SDValue Op, SelectionDAG &DAG,
 
   // Create the vector constant for -(0x1.0p39f + 0x1.0p23f).
   SDValue CstFAdd = DAG.getConstantFP(
-      APFloat(APFloat::IEEEsingle, APInt(32, 0xD3000080)), MVT::f32);
+      APFloat(APFloat::IEEEsingle, APInt(32, 0xD3000080)), DL, MVT::f32);
   SDValue CstFAddArray[] = {CstFAdd, CstFAdd, CstFAdd, CstFAdd,
                             CstFAdd, CstFAdd, CstFAdd, CstFAdd};
   SDValue VecCstFAdd = DAG.getNode(ISD::BUILD_VECTOR, DL, VecFloatVT,
                                    makeArrayRef(&CstFAddArray[0], NumElts));
 
   //     float4 fhi = (float4) hi - (0x1.0p39f + 0x1.0p23f);
-  SDValue HighBitcast = DAG.getNode(ISD::BITCAST, DL, VecFloatVT, High);
+  SDValue HighBitcast = DAG.getBitcast(VecFloatVT, High);
   SDValue FHigh =
       DAG.getNode(ISD::FADD, DL, VecFloatVT, HighBitcast, VecCstFAdd);
   //     return (float4) lo + fhi;
-  SDValue LowBitcast = DAG.getNode(ISD::BITCAST, DL, VecFloatVT, Low);
+  SDValue LowBitcast = DAG.getBitcast(VecFloatVT, Low);
   return DAG.getNode(ISD::FADD, DL, VecFloatVT, LowBitcast, FHigh);
 }
 
@@ -11824,6 +12177,11 @@ SDValue X86TargetLowering::lowerUINT_TO_FP_vec(SDValue Op,
   case MVT::v4i32:
   case MVT::v8i32:
     return lowerUINT_TO_FP_vXi32(Op, DAG, *Subtarget);
+  case MVT::v16i8:
+  case MVT::v16i16:
+    if (Subtarget->hasAVX512())
+      return DAG.getNode(ISD::UINT_TO_FP, dl, Op.getValueType(),
+                         DAG.getNode(ISD::ZERO_EXTEND, dl, MVT::v16i32, N0));
   }
   llvm_unreachable(nullptr);
 }
@@ -11832,6 +12190,7 @@ SDValue X86TargetLowering::LowerUINT_TO_FP(SDValue Op,
                                            SelectionDAG &DAG) const {
   SDValue N0 = Op.getOperand(0);
   SDLoc dl(Op);
+  auto PtrVT = getPointerTy(DAG.getDataLayout());
 
   if (Op.getValueType().isVector())
     return lowerUINT_TO_FP_vec(Op, DAG);
@@ -11854,13 +12213,12 @@ SDValue X86TargetLowering::LowerUINT_TO_FP(SDValue Op,
   // Make a 64-bit buffer, and use it to build an FILD.
   SDValue StackSlot = DAG.CreateStackTemporary(MVT::i64);
   if (SrcVT == MVT::i32) {
-    SDValue WordOff = DAG.getConstant(4, getPointerTy());
-    SDValue OffsetSlot = DAG.getNode(ISD::ADD, dl,
-                                     getPointerTy(), StackSlot, WordOff);
+    SDValue WordOff = DAG.getConstant(4, dl, PtrVT);
+    SDValue OffsetSlot = DAG.getNode(ISD::ADD, dl, PtrVT, StackSlot, WordOff);
     SDValue Store1 = DAG.getStore(DAG.getEntryNode(), dl, Op.getOperand(0),
                                   StackSlot, MachinePointerInfo(),
                                   false, false, 0);
-    SDValue Store2 = DAG.getStore(Store1, dl, DAG.getConstant(0, MVT::i32),
+    SDValue Store2 = DAG.getStore(Store1, dl, DAG.getConstant(0, dl, MVT::i32),
                                   OffsetSlot, MachinePointerInfo(),
                                   false, false, 0);
     SDValue Fild = BuildFILD(Op, MVT::i64, Store2, StackSlot, DAG);
@@ -11890,22 +12248,20 @@ SDValue X86TargetLowering::LowerUINT_TO_FP(SDValue Op,
   APInt FF(32, 0x5F800000ULL);
 
   // Check whether the sign bit is set.
-  SDValue SignSet = DAG.getSetCC(dl,
-                                 getSetCCResultType(*DAG.getContext(), MVT::i64),
-                                 Op.getOperand(0), DAG.getConstant(0, MVT::i64),
-                                 ISD::SETLT);
+  SDValue SignSet = DAG.getSetCC(
+      dl, getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), MVT::i64),
+      Op.getOperand(0), DAG.getConstant(0, dl, MVT::i64), ISD::SETLT);
 
   // Build a 64 bit pair (0, FF) in the constant pool, with FF in the lo bits.
   SDValue FudgePtr = DAG.getConstantPool(
-                             ConstantInt::get(*DAG.getContext(), FF.zext(64)),
-                                         getPointerTy());
+      ConstantInt::get(*DAG.getContext(), FF.zext(64)), PtrVT);
 
   // Get a pointer to FF if the sign bit was set, or to 0 otherwise.
-  SDValue Zero = DAG.getIntPtrConstant(0);
-  SDValue Four = DAG.getIntPtrConstant(4);
+  SDValue Zero = DAG.getIntPtrConstant(0, dl);
+  SDValue Four = DAG.getIntPtrConstant(4, dl);
   SDValue Offset = DAG.getNode(ISD::SELECT, dl, Zero.getValueType(), SignSet,
                                Zero, Four);
-  FudgePtr = DAG.getNode(ISD::ADD, dl, getPointerTy(), FudgePtr, Offset);
+  FudgePtr = DAG.getNode(ISD::ADD, dl, PtrVT, FudgePtr, Offset);
 
   // Load the value out, extending it from f32 to f80.
   // FIXME: Avoid the extend by constructing the right constant pool?
@@ -11914,7 +12270,8 @@ SDValue X86TargetLowering::LowerUINT_TO_FP(SDValue Op,
                                  MVT::f32, false, false, false, 4);
   // Extend everything to 80 bits to force it to be done on x87.
   SDValue Add = DAG.getNode(ISD::FADD, dl, MVT::f80, Fild, Fudge);
-  return DAG.getNode(ISD::FP_ROUND, dl, DstVT, Add, DAG.getIntPtrConstant(0));
+  return DAG.getNode(ISD::FP_ROUND, dl, DstVT, Add,
+                     DAG.getIntPtrConstant(0, dl));
 }
 
 std::pair<SDValue,SDValue>
@@ -11923,6 +12280,7 @@ X86TargetLowering:: FP_TO_INTHelper(SDValue Op, SelectionDAG &DAG,
   SDLoc DL(Op);
 
   EVT DstTy = Op.getValueType();
+  auto PtrVT = getPointerTy(DAG.getDataLayout());
 
   if (!IsSigned && !isIntegerTypeFTOL(DstTy)) {
     assert(DstTy == MVT::i32 && "Unexpected FP_TO_UINT");
@@ -11947,7 +12305,7 @@ X86TargetLowering:: FP_TO_INTHelper(SDValue Op, SelectionDAG &DAG,
   MachineFunction &MF = DAG.getMachineFunction();
   unsigned MemSize = DstTy.getSizeInBits()/8;
   int SSFI = MF.getFrameInfo()->CreateStackObject(MemSize, MemSize, false);
-  SDValue StackSlot = DAG.getFrameIndex(SSFI, getPointerTy());
+  SDValue StackSlot = DAG.getFrameIndex(SSFI, PtrVT);
 
   unsigned Opc;
   if (!IsSigned && isIntegerTypeFTOL(DstTy))
@@ -11981,7 +12339,7 @@ X86TargetLowering:: FP_TO_INTHelper(SDValue Op, SelectionDAG &DAG,
     Value = DAG.getMemIntrinsicNode(X86ISD::FLD, DL, Tys, Ops, DstTy, MMO);
     Chain = Value.getValue(1);
     SSFI = MF.getFrameInfo()->CreateStackObject(MemSize, MemSize, false);
-    StackSlot = DAG.getFrameIndex(SSFI, getPointerTy());
+    StackSlot = DAG.getFrameIndex(SSFI, PtrVT);
   }
 
   MachineMemOperand *MMO =
@@ -12017,6 +12375,9 @@ static SDValue LowerAVXExtend(SDValue Op, SelectionDAG &DAG,
   MVT InVT = In.getSimpleValueType();
   SDLoc dl(Op);
 
+  if (VT.is512BitVector() || InVT.getScalarType() == MVT::i1)
+    return DAG.getNode(ISD::ZERO_EXTEND, dl, VT, In);
+
   // Optimize vectors in AVX mode:
   //
   //   v8i16 -> v8i32
@@ -12047,50 +12408,43 @@ static SDValue LowerAVXExtend(SDValue Op, SelectionDAG &DAG,
   MVT HVT = MVT::getVectorVT(VT.getVectorElementType(),
                              VT.getVectorNumElements()/2);
 
-  OpLo = DAG.getNode(ISD::BITCAST, dl, HVT, OpLo);
-  OpHi = DAG.getNode(ISD::BITCAST, dl, HVT, OpHi);
+  OpLo = DAG.getBitcast(HVT, OpLo);
+  OpHi = DAG.getBitcast(HVT, OpHi);
 
   return DAG.getNode(ISD::CONCAT_VECTORS, dl, VT, OpLo, OpHi);
 }
 
 static  SDValue LowerZERO_EXTEND_AVX512(SDValue Op,
-                                        SelectionDAG &DAG) {
+                  const X86Subtarget *Subtarget, SelectionDAG &DAG) {
   MVT VT = Op->getSimpleValueType(0);
   SDValue In = Op->getOperand(0);
   MVT InVT = In.getSimpleValueType();
   SDLoc DL(Op);
   unsigned int NumElts = VT.getVectorNumElements();
-  if (NumElts != 8 && NumElts != 16)
+  if (NumElts != 8 && NumElts != 16 && !Subtarget->hasBWI())
     return SDValue();
 
   if (VT.is512BitVector() && InVT.getVectorElementType() != MVT::i1)
     return DAG.getNode(X86ISD::VZEXT, DL, VT, In);
 
-  EVT ExtVT = (NumElts == 8)? MVT::v8i64 : MVT::v16i32;
-  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
-  // Now we have only mask extension
   assert(InVT.getVectorElementType() == MVT::i1);
-  SDValue Cst = DAG.getTargetConstant(1, ExtVT.getScalarType());
-  const Constant *C = cast<ConstantSDNode>(Cst)->getConstantIntValue();
-  SDValue CP = DAG.getConstantPool(C, TLI.getPointerTy());
-  unsigned Alignment = cast<ConstantPoolSDNode>(CP)->getAlignment();
-  SDValue Ld = DAG.getLoad(Cst.getValueType(), DL, DAG.getEntryNode(), CP,
-                           MachinePointerInfo::getConstantPool(),
-                           false, false, false, Alignment);
-
-  SDValue Brcst = DAG.getNode(X86ISD::VBROADCASTM, DL, ExtVT, In, Ld);
+  MVT ExtVT = NumElts == 8 ? MVT::v8i64 : MVT::v16i32;
+  SDValue One =
+   DAG.getConstant(APInt(ExtVT.getScalarSizeInBits(), 1), DL, ExtVT);
+  SDValue Zero =
+   DAG.getConstant(APInt::getNullValue(ExtVT.getScalarSizeInBits()), DL, ExtVT);
+
+  SDValue V = DAG.getNode(ISD::VSELECT, DL, ExtVT, In, One, Zero);
   if (VT.is512BitVector())
-    return Brcst;
-  return DAG.getNode(X86ISD::VTRUNC, DL, VT, Brcst);
+    return V;
+  return DAG.getNode(X86ISD::VTRUNC, DL, VT, V);
 }
 
 static SDValue LowerANY_EXTEND(SDValue Op, const X86Subtarget *Subtarget,
                                SelectionDAG &DAG) {
-  if (Subtarget->hasFp256()) {
-    SDValue Res = LowerAVXExtend(Op, DAG, Subtarget);
-    if (Res.getNode())
+  if (Subtarget->hasFp256())
+    if (SDValue Res = LowerAVXExtend(Op, DAG, Subtarget))
       return Res;
-  }
 
   return SDValue();
 }
@@ -12103,13 +12457,11 @@ static SDValue LowerZERO_EXTEND(SDValue Op, const X86Subtarget *Subtarget,
   MVT SVT = In.getSimpleValueType();
 
   if (VT.is512BitVector() || SVT.getVectorElementType() == MVT::i1)
-    return LowerZERO_EXTEND_AVX512(Op, DAG);
+    return LowerZERO_EXTEND_AVX512(Op, Subtarget, DAG);
 
-  if (Subtarget->hasFp256()) {
-    SDValue Res = LowerAVXExtend(Op, DAG, Subtarget);
-    if (Res.getNode())
+  if (Subtarget->hasFp256())
+    if (SDValue Res = LowerAVXExtend(Op, DAG, Subtarget))
       return Res;
-  }
 
   assert(!VT.is256BitVector() || !SVT.is128BitVector() ||
          VT.getVectorNumElements() != SVT.getVectorNumElements());
@@ -12133,10 +12485,25 @@ SDValue X86TargetLowering::LowerTRUNCATE(SDValue Op, SelectionDAG &DAG) const {
   assert(VT.getVectorNumElements() == InVT.getVectorNumElements() &&
          "Invalid TRUNCATE operation");
 
-  if (InVT.is512BitVector() || VT.getVectorElementType() == MVT::i1) {
-    if (VT.getVectorElementType().getSizeInBits() >=8)
-      return DAG.getNode(X86ISD::VTRUNC, DL, VT, In);
-
+  // move vector to mask - truncate solution for SKX
+  if (VT.getVectorElementType() == MVT::i1) {
+    if (InVT.is512BitVector() && InVT.getScalarSizeInBits() <= 16 &&
+        Subtarget->hasBWI())
+      return Op; // legal, will go to VPMOVB2M, VPMOVW2M
+    if ((InVT.is256BitVector() || InVT.is128BitVector())
+        && InVT.getScalarSizeInBits() <= 16 &&
+        Subtarget->hasBWI() && Subtarget->hasVLX())
+      return Op; // legal, will go to VPMOVB2M, VPMOVW2M
+    if (InVT.is512BitVector() && InVT.getScalarSizeInBits() >= 32 &&
+        Subtarget->hasDQI())
+      return Op; // legal, will go to VPMOVD2M, VPMOVQ2M
+    if ((InVT.is256BitVector() || InVT.is128BitVector())
+        && InVT.getScalarSizeInBits() >= 32 &&
+        Subtarget->hasDQI() && Subtarget->hasVLX())
+      return Op; // legal, will go to VPMOVB2M, VPMOVQ2M
+  }
+
+  if (VT.getVectorElementType() == MVT::i1) {
     assert(VT.getVectorElementType() == MVT::i1 && "Unexpected vector type");
     unsigned NumElts = InVT.getVectorNumElements();
     assert ((NumElts == 8 || NumElts == 16) && "Unexpected vector type");
@@ -12146,35 +12513,34 @@ SDValue X86TargetLowering::LowerTRUNCATE(SDValue Op, SelectionDAG &DAG) const {
       InVT = ExtVT;
     }
 
-    SDValue Cst = DAG.getTargetConstant(1, InVT.getVectorElementType());
-    const Constant *C = cast<ConstantSDNode>(Cst)->getConstantIntValue();
-    SDValue CP = DAG.getConstantPool(C, getPointerTy());
-    unsigned Alignment = cast<ConstantPoolSDNode>(CP)->getAlignment();
-    SDValue Ld = DAG.getLoad(Cst.getValueType(), DL, DAG.getEntryNode(), CP,
-                           MachinePointerInfo::getConstantPool(),
-                           false, false, false, Alignment);
-    SDValue OneV = DAG.getNode(X86ISD::VBROADCAST, DL, InVT, Ld);
+    SDValue OneV =
+     DAG.getConstant(APInt::getSignBit(InVT.getScalarSizeInBits()), DL, InVT);
     SDValue And = DAG.getNode(ISD::AND, DL, InVT, OneV, In);
     return DAG.getNode(X86ISD::TESTM, DL, VT, And, And);
   }
 
+  // vpmovqb/w/d, vpmovdb/w, vpmovwb
+  if (((!InVT.is512BitVector() && Subtarget->hasVLX()) || InVT.is512BitVector()) &&
+      (InVT.getVectorElementType() != MVT::i16 || Subtarget->hasBWI()))
+    return DAG.getNode(X86ISD::VTRUNC, DL, VT, In);
+
   if ((VT == MVT::v4i32) && (InVT == MVT::v4i64)) {
     // On AVX2, v4i64 -> v4i32 becomes VPERMD.
     if (Subtarget->hasInt256()) {
       static const int ShufMask[] = {0, 2, 4, 6, -1, -1, -1, -1};
-      In = DAG.getNode(ISD::BITCAST, DL, MVT::v8i32, In);
+      In = DAG.getBitcast(MVT::v8i32, In);
       In = DAG.getVectorShuffle(MVT::v8i32, DL, In, DAG.getUNDEF(MVT::v8i32),
                                 ShufMask);
       return DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, VT, In,
-                         DAG.getIntPtrConstant(0));
+                         DAG.getIntPtrConstant(0, DL));
     }
 
     SDValue OpLo = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, MVT::v2i64, In,
-                               DAG.getIntPtrConstant(0));
+                               DAG.getIntPtrConstant(0, DL));
     SDValue OpHi = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, MVT::v2i64, In,
-                               DAG.getIntPtrConstant(2));
-    OpLo = DAG.getNode(ISD::BITCAST, DL, MVT::v4i32, OpLo);
-    OpHi = DAG.getNode(ISD::BITCAST, DL, MVT::v4i32, OpHi);
+                               DAG.getIntPtrConstant(2, DL));
+    OpLo = DAG.getBitcast(MVT::v4i32, OpLo);
+    OpHi = DAG.getBitcast(MVT::v4i32, OpHi);
     static const int ShufMask[] = {0, 2, 4, 6};
     return DAG.getVectorShuffle(VT, DL, OpLo, OpHi, ShufMask);
   }
@@ -12182,41 +12548,41 @@ SDValue X86TargetLowering::LowerTRUNCATE(SDValue Op, SelectionDAG &DAG) const {
   if ((VT == MVT::v8i16) && (InVT == MVT::v8i32)) {
     // On AVX2, v8i32 -> v8i16 becomed PSHUFB.
     if (Subtarget->hasInt256()) {
-      In = DAG.getNode(ISD::BITCAST, DL, MVT::v32i8, In);
+      In = DAG.getBitcast(MVT::v32i8, In);
 
       SmallVector<SDValue,32> pshufbMask;
       for (unsigned i = 0; i < 2; ++i) {
-        pshufbMask.push_back(DAG.getConstant(0x0, MVT::i8));
-        pshufbMask.push_back(DAG.getConstant(0x1, MVT::i8));
-        pshufbMask.push_back(DAG.getConstant(0x4, MVT::i8));
-        pshufbMask.push_back(DAG.getConstant(0x5, MVT::i8));
-        pshufbMask.push_back(DAG.getConstant(0x8, MVT::i8));
-        pshufbMask.push_back(DAG.getConstant(0x9, MVT::i8));
-        pshufbMask.push_back(DAG.getConstant(0xc, MVT::i8));
-        pshufbMask.push_back(DAG.getConstant(0xd, MVT::i8));
+        pshufbMask.push_back(DAG.getConstant(0x0, DL, MVT::i8));
+        pshufbMask.push_back(DAG.getConstant(0x1, DL, MVT::i8));
+        pshufbMask.push_back(DAG.getConstant(0x4, DL, MVT::i8));
+        pshufbMask.push_back(DAG.getConstant(0x5, DL, MVT::i8));
+        pshufbMask.push_back(DAG.getConstant(0x8, DL, MVT::i8));
+        pshufbMask.push_back(DAG.getConstant(0x9, DL, MVT::i8));
+        pshufbMask.push_back(DAG.getConstant(0xc, DL, MVT::i8));
+        pshufbMask.push_back(DAG.getConstant(0xd, DL, MVT::i8));
         for (unsigned j = 0; j < 8; ++j)
-          pshufbMask.push_back(DAG.getConstant(0x80, MVT::i8));
+          pshufbMask.push_back(DAG.getConstant(0x80, DL, MVT::i8));
       }
       SDValue BV = DAG.getNode(ISD::BUILD_VECTOR, DL, MVT::v32i8, pshufbMask);
       In = DAG.getNode(X86ISD::PSHUFB, DL, MVT::v32i8, In, BV);
-      In = DAG.getNode(ISD::BITCAST, DL, MVT::v4i64, In);
+      In = DAG.getBitcast(MVT::v4i64, In);
 
       static const int ShufMask[] = {0,  2,  -1,  -1};
       In = DAG.getVectorShuffle(MVT::v4i64, DL,  In, DAG.getUNDEF(MVT::v4i64),
                                 &ShufMask[0]);
       In = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, MVT::v2i64, In,
-                       DAG.getIntPtrConstant(0));
-      return DAG.getNode(ISD::BITCAST, DL, VT, In);
+                       DAG.getIntPtrConstant(0, DL));
+      return DAG.getBitcast(VT, In);
     }
 
     SDValue OpLo = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, MVT::v4i32, In,
-                               DAG.getIntPtrConstant(0));
+                               DAG.getIntPtrConstant(0, DL));
 
     SDValue OpHi = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, MVT::v4i32, In,
-                               DAG.getIntPtrConstant(4));
+                               DAG.getIntPtrConstant(4, DL));
 
-    OpLo = DAG.getNode(ISD::BITCAST, DL, MVT::v16i8, OpLo);
-    OpHi = DAG.getNode(ISD::BITCAST, DL, MVT::v16i8, OpHi);
+    OpLo = DAG.getBitcast(MVT::v16i8, OpLo);
+    OpHi = DAG.getBitcast(MVT::v16i8, OpHi);
 
     // The PSHUFB mask:
     static const int ShufMask1[] = {0,  1,  4,  5,  8,  9, 12, 13,
@@ -12226,13 +12592,13 @@ SDValue X86TargetLowering::LowerTRUNCATE(SDValue Op, SelectionDAG &DAG) const {
     OpLo = DAG.getVectorShuffle(MVT::v16i8, DL, OpLo, Undef, ShufMask1);
     OpHi = DAG.getVectorShuffle(MVT::v16i8, DL, OpHi, Undef, ShufMask1);
 
-    OpLo = DAG.getNode(ISD::BITCAST, DL, MVT::v4i32, OpLo);
-    OpHi = DAG.getNode(ISD::BITCAST, DL, MVT::v4i32, OpHi);
+    OpLo = DAG.getBitcast(MVT::v4i32, OpLo);
+    OpHi = DAG.getBitcast(MVT::v4i32, OpHi);
 
     // The MOVLHPS Mask:
     static const int ShufMask2[] = {0, 1, 4, 5};
     SDValue res = DAG.getVectorShuffle(MVT::v4i32, DL, OpLo, OpHi, ShufMask2);
-    return DAG.getNode(ISD::BITCAST, DL, MVT::v8i16, res);
+    return DAG.getBitcast(MVT::v8i16, res);
   }
 
   // Handle truncation of V256 to V128 using shuffles.
@@ -12248,11 +12614,10 @@ SDValue X86TargetLowering::LowerTRUNCATE(SDValue Op, SelectionDAG &DAG) const {
   // Prepare truncation shuffle mask
   for (unsigned i = 0; i != NumElems; ++i)
     MaskVec[i] = i * 2;
-  SDValue V = DAG.getVectorShuffle(NVT, DL,
-                                   DAG.getNode(ISD::BITCAST, DL, NVT, In),
+  SDValue V = DAG.getVectorShuffle(NVT, DL, DAG.getBitcast(NVT, In),
                                    DAG.getUNDEF(NVT), &MaskVec[0]);
   return DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, VT, V,
-                     DAG.getIntPtrConstant(0));
+                     DAG.getIntPtrConstant(0, DL));
 }
 
 SDValue X86TargetLowering::LowerFP_TO_SINT(SDValue Op,
@@ -12320,24 +12685,29 @@ static SDValue LowerFABSorFNEG(SDValue Op, SelectionDAG &DAG) {
       if (User->getOpcode() == ISD::FNEG)
         return Op;
 
-  SDValue Op0 = Op.getOperand(0);
-  bool IsFNABS = !IsFABS && (Op0.getOpcode() == ISD::FABS);
-
   SDLoc dl(Op);
   MVT VT = Op.getSimpleValueType();
-  // Assume scalar op for initialization; update for vector if needed.
-  // Note that there are no scalar bitwise logical SSE/AVX instructions, so we
-  // generate a 16-byte vector constant and logic op even for the scalar case.
-  // Using a 16-byte mask allows folding the load of the mask with
-  // the logic op, so it can save (~4 bytes) on code size.
-  MVT EltVT = VT;
-  unsigned NumElts = VT == MVT::f64 ? 2 : 4;
+
   // FIXME: Use function attribute "OptimizeForSize" and/or CodeGenOpt::Level to
   // decide if we should generate a 16-byte constant mask when we only need 4 or
   // 8 bytes for the scalar case.
+
+  MVT LogicVT;
+  MVT EltVT;
+  unsigned NumElts;
+  
   if (VT.isVector()) {
+    LogicVT = VT;
     EltVT = VT.getVectorElementType();
     NumElts = VT.getVectorNumElements();
+  } else {
+    // There are no scalar bitwise logical SSE/AVX instructions, so we
+    // generate a 16-byte vector constant and logic op even for the scalar case.
+    // Using a 16-byte mask allows folding the load of the mask with
+    // the logic op, so it can save (~4 bytes) on code size.
+    LogicVT = (VT == MVT::f64) ? MVT::v2f64 : MVT::v4f32;
+    EltVT = VT;
+    NumElts = (VT == MVT::f64) ? 2 : 4;
   }
 
   unsigned EltBits = EltVT.getSizeInBits();
@@ -12348,29 +12718,27 @@ static SDValue LowerFABSorFNEG(SDValue Op, SelectionDAG &DAG) {
   Constant *C = ConstantInt::get(*Context, MaskElt);
   C = ConstantVector::getSplat(NumElts, C);
   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
-  SDValue CPIdx = DAG.getConstantPool(C, TLI.getPointerTy());
+  SDValue CPIdx = DAG.getConstantPool(C, TLI.getPointerTy(DAG.getDataLayout()));
   unsigned Alignment = cast<ConstantPoolSDNode>(CPIdx)->getAlignment();
-  SDValue Mask = DAG.getLoad(VT, dl, DAG.getEntryNode(), CPIdx,
+  SDValue Mask = DAG.getLoad(LogicVT, dl, DAG.getEntryNode(), CPIdx,
                              MachinePointerInfo::getConstantPool(),
                              false, false, false, Alignment);
 
-  if (VT.isVector()) {
-    // For a vector, cast operands to a vector type, perform the logic op,
-    // and cast the result back to the original value type.
-    MVT VecVT = MVT::getVectorVT(MVT::i64, VT.getSizeInBits() / 64);
-    SDValue MaskCasted = DAG.getNode(ISD::BITCAST, dl, VecVT, Mask);
-    SDValue Operand = IsFNABS ?
-      DAG.getNode(ISD::BITCAST, dl, VecVT, Op0.getOperand(0)) :
-      DAG.getNode(ISD::BITCAST, dl, VecVT, Op0);
-    unsigned BitOp = IsFABS ? ISD::AND : IsFNABS ? ISD::OR : ISD::XOR;
-    return DAG.getNode(ISD::BITCAST, dl, VT,
-                       DAG.getNode(BitOp, dl, VecVT, Operand, MaskCasted));
-  }
-
-  // If not vector, then scalar.
-  unsigned BitOp = IsFABS ? X86ISD::FAND : IsFNABS ? X86ISD::FOR : X86ISD::FXOR;
+  SDValue Op0 = Op.getOperand(0);
+  bool IsFNABS = !IsFABS && (Op0.getOpcode() == ISD::FABS);
+  unsigned LogicOp =
+    IsFABS ? X86ISD::FAND : IsFNABS ? X86ISD::FOR : X86ISD::FXOR;
   SDValue Operand = IsFNABS ? Op0.getOperand(0) : Op0;
-  return DAG.getNode(BitOp, dl, VT, Operand, Mask);
+
+  if (VT.isVector())
+    return DAG.getNode(LogicOp, dl, LogicVT, Operand, Mask);
+
+  // For the scalar case extend to a 128-bit vector, perform the logic op,
+  // and extract the scalar result back out.
+  Operand = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, LogicVT, Operand);
+  SDValue LogicNode = DAG.getNode(LogicOp, dl, LogicVT, Operand, Mask);
+  return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, VT, LogicNode,
+                     DAG.getIntPtrConstant(0, dl));
 }
 
 static SDValue LowerFCOPYSIGN(SDValue Op, SelectionDAG &DAG) {
@@ -12389,7 +12757,7 @@ static SDValue LowerFCOPYSIGN(SDValue Op, SelectionDAG &DAG) {
   }
   // And if it is bigger, shrink it first.
   if (SrcVT.bitsGT(VT)) {
-    Op1 = DAG.getNode(ISD::FP_ROUND, dl, VT, Op1, DAG.getIntPtrConstant(1));
+    Op1 = DAG.getNode(ISD::FP_ROUND, dl, VT, Op1, DAG.getIntPtrConstant(1, dl));
     SrcVT = VT;
   }
 
@@ -12408,11 +12776,18 @@ static SDValue LowerFCOPYSIGN(SDValue Op, SelectionDAG &DAG) {
   CV[0] = ConstantFP::get(*Context,
                           APFloat(Sem, APInt::getHighBitsSet(SizeInBits, 1)));
   Constant *C = ConstantVector::get(CV);
-  SDValue CPIdx = DAG.getConstantPool(C, TLI.getPointerTy(), 16);
-  SDValue Mask1 = DAG.getLoad(SrcVT, dl, DAG.getEntryNode(), CPIdx,
+  auto PtrVT = TLI.getPointerTy(DAG.getDataLayout());
+  SDValue CPIdx = DAG.getConstantPool(C, PtrVT, 16);
+
+  // Perform all logic operations as 16-byte vectors because there are no
+  // scalar FP logic instructions in SSE. This allows load folding of the
+  // constants into the logic instructions.
+  MVT LogicVT = (VT == MVT::f64) ? MVT::v2f64 : MVT::v4f32;
+  SDValue Mask1 = DAG.getLoad(LogicVT, dl, DAG.getEntryNode(), CPIdx,
                               MachinePointerInfo::getConstantPool(),
                               false, false, false, 16);
-  SDValue SignBit = DAG.getNode(X86ISD::FAND, dl, SrcVT, Op1, Mask1);
+  Op1 = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, LogicVT, Op1);
+  SDValue SignBit = DAG.getNode(X86ISD::FAND, dl, LogicVT, Op1, Mask1);
 
   // Next, clear the sign bit from the first operand (magnitude).
   // If it's a constant, we can clear it here.
@@ -12420,7 +12795,8 @@ static SDValue LowerFCOPYSIGN(SDValue Op, SelectionDAG &DAG) {
     APFloat APF = Op0CN->getValueAPF();
     // If the magnitude is a positive zero, the sign bit alone is enough.
     if (APF.isPosZero())
-      return SignBit;
+      return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, SrcVT, SignBit,
+                         DAG.getIntPtrConstant(0, dl));
     APF.clearSign();
     CV[0] = ConstantFP::get(*Context, APF);
   } else {
@@ -12429,16 +12805,19 @@ static SDValue LowerFCOPYSIGN(SDValue Op, SelectionDAG &DAG) {
         APFloat(Sem, APInt::getLowBitsSet(SizeInBits, SizeInBits - 1)));
   }
   C = ConstantVector::get(CV);
-  CPIdx = DAG.getConstantPool(C, TLI.getPointerTy(), 16);
-  SDValue Val = DAG.getLoad(VT, dl, DAG.getEntryNode(), CPIdx,
+  CPIdx = DAG.getConstantPool(C, PtrVT, 16);
+  SDValue Val = DAG.getLoad(LogicVT, dl, DAG.getEntryNode(), CPIdx,
                             MachinePointerInfo::getConstantPool(),
                             false, false, false, 16);
   // If the magnitude operand wasn't a constant, we need to AND out the sign.
-  if (!isa<ConstantFPSDNode>(Op0))
-    Val = DAG.getNode(X86ISD::FAND, dl, VT, Op0, Val);
-
+  if (!isa<ConstantFPSDNode>(Op0)) {
+    Op0 = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, LogicVT, Op0);
+    Val = DAG.getNode(X86ISD::FAND, dl, LogicVT, Op0, Val);
+  }
   // OR the magnitude value with the sign bit.
-  return DAG.getNode(X86ISD::FOR, dl, VT, Val, SignBit);
+  Val = DAG.getNode(X86ISD::FOR, dl, LogicVT, Val, SignBit);
+  return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, SrcVT, Val,
+                     DAG.getIntPtrConstant(0, dl));
 }
 
 static SDValue LowerFGETSIGN(SDValue Op, SelectionDAG &DAG) {
@@ -12448,8 +12827,8 @@ static SDValue LowerFGETSIGN(SDValue Op, SelectionDAG &DAG) {
 
   // Lower ISD::FGETSIGN to (AND (X86ISD::FGETSIGNx86 ...) 1).
   SDValue xFGETSIGN = DAG.getNode(X86ISD::FGETSIGNx86, dl, VT, N0,
-                                  DAG.getConstant(1, VT));
-  return DAG.getNode(ISD::AND, dl, VT, xFGETSIGN, DAG.getConstant(1, VT));
+                                  DAG.getConstant(1, dl, VT));
+  return DAG.getNode(ISD::AND, dl, VT, xFGETSIGN, DAG.getConstant(1, dl, VT));
 }
 
 // Check whether an OR'd tree is PTEST-able.
@@ -12529,7 +12908,7 @@ static SDValue LowerVectorAllZeroTest(SDValue Op, const X86Subtarget *Subtarget,
 
   // Cast all vectors into TestVT for PTEST.
   for (unsigned i = 0, e = VecIns.size(); i < e; ++i)
-    VecIns[i] = DAG.getNode(ISD::BITCAST, DL, TestVT, VecIns[i]);
+    VecIns[i] = DAG.getBitcast(TestVT, VecIns[i]);
 
   // If more than one full vectors are evaluated, OR them first before PTEST.
   for (unsigned Slot = 0, e = VecIns.size(); e - Slot > 1; Slot += 2, e += 1) {
@@ -12570,25 +12949,8 @@ SDValue X86TargetLowering::EmitTest(SDValue Op, unsigned X86CC, SDLoc dl,
   if (Op.getValueType() == MVT::i1) {
     SDValue ExtOp = DAG.getNode(ISD::ZERO_EXTEND, dl, MVT::i8, Op);
     return DAG.getNode(X86ISD::CMP, dl, MVT::i32, ExtOp,
-                       DAG.getConstant(0, MVT::i8));
-  }
-  // @LOCALMOD-BEGIN
-  // This function only peeks at the data dependencies of the DAG to find
-  // an arith op that also defines EFLAGS.  However, function calls may
-  // clobber EFLAGS and the data dependencies do not show that.
-  // When that occurs, EFLAGS must be copied via PUSHF and POPF.
-  // The problem is that NaCl does not allow PUSHF and POPF.
-  // We could try to detect such clobbers for NaCl, but for now, we
-  // keep this code simple, and bail out for NaCl.  A further
-  // PeepholeOptimizer pass can do a similar optimization
-  // (see optimizeCompareInstr in X86InstrInfo.cpp), so it's not *so*
-  // bad.  This function also converts "add op, -1" to DEC, which can
-  // help fold load/stores:
-  //    (store m, (add (load m), -1)) -> (dec m)
-  // So we lose out on that.
-  // BUG=http://code.google.com/p/nativeclient/issues/detail?id=2711
-  bool ConservativeForNaCl = Subtarget->isTargetNaCl();
-
+                       DAG.getConstant(0, dl, MVT::i8));
+  }
   // CF and OF aren't always set the way we want. Determine which
   // of these we need.
   bool NeedCF = false;
@@ -12610,9 +12972,8 @@ SDValue X86TargetLowering::EmitTest(SDValue Op, unsigned X86CC, SDLoc dl,
     case ISD::SUB:
     case ISD::MUL:
     case ISD::SHL: {
-      const BinaryWithFlagsSDNode *BinNode =
-          cast<BinaryWithFlagsSDNode>(Op.getNode());
-      if (BinNode->hasNoSignedWrap())
+      const auto *BinNode = cast<BinaryWithFlagsSDNode>(Op.getNode());
+      if (BinNode->Flags.hasNoSignedWrap())
         break;
     }
     default:
@@ -12625,15 +12986,14 @@ SDValue X86TargetLowering::EmitTest(SDValue Op, unsigned X86CC, SDLoc dl,
   // See if we can use the EFLAGS value from the operand instead of
   // doing a separate TEST. TEST always sets OF and CF to 0, so unless
   // we prove that the arithmetic won't overflow, we can't use OF or CF.
-  if (Op.getResNo() != 0 || NeedOF || NeedCF || ConservativeForNaCl) {
+  if (Op.getResNo() != 0 || NeedOF || NeedCF) {
     // Emit a CMP with 0, which is the TEST pattern.
     //if (Op.getValueType() == MVT::i1)
     //  return DAG.getNode(X86ISD::CMP, dl, MVT::i1, Op,
     //                     DAG.getConstant(0, MVT::i1));
     return DAG.getNode(X86ISD::CMP, dl, MVT::i32, Op,
-                       DAG.getConstant(0, Op.getValueType()));
+                       DAG.getConstant(0, dl, Op.getValueType()));
   }
-  // @LOCALMOD-END
   unsigned Opcode = 0;
   unsigned NumOperands = 0;
 
@@ -12720,7 +13080,7 @@ SDValue X86TargetLowering::EmitTest(SDValue Op, unsigned X86CC, SDLoc dl,
       if (!Mask.isSignedIntN(32)) // Avoid large immediates.
         break;
       SDValue New = DAG.getNode(ISD::AND, dl, VT, Op->getOperand(0),
-                                DAG.getConstant(Mask, VT));
+                                DAG.getConstant(Mask, dl, VT));
       DAG.ReplaceAllUsesWith(Op, New);
       Op = New;
     }
@@ -12806,7 +13166,7 @@ SDValue X86TargetLowering::EmitTest(SDValue Op, unsigned X86CC, SDLoc dl,
   if (Opcode == 0)
     // Emit a CMP with 0, which is the TEST pattern.
     return DAG.getNode(X86ISD::CMP, dl, MVT::i32, Op,
-                       DAG.getConstant(0, Op.getValueType()));
+                       DAG.getConstant(0, dl, Op.getValueType()));
 
   SDVTList VTs = DAG.getVTList(Op.getValueType(), MVT::i32);
   SmallVector<SDValue, 4> Ops(Op->op_begin(), Op->op_begin() + NumOperands);
@@ -12871,7 +13231,7 @@ SDValue X86TargetLowering::ConvertCmpIfNecessary(SDValue Cmp,
   SDValue TruncFPSW = DAG.getNode(ISD::TRUNCATE, dl, MVT::i16, Cmp);
   SDValue FNStSW = DAG.getNode(X86ISD::FNSTSW16r, dl, MVT::i16, TruncFPSW);
   SDValue Srl = DAG.getNode(ISD::SRL, dl, MVT::i16, FNStSW,
-                            DAG.getConstant(8, MVT::i8));
+                            DAG.getConstant(8, dl, MVT::i8));
   SDValue TruncSrl = DAG.getNode(ISD::TRUNCATE, dl, MVT::i8, Srl);
   return DAG.getNode(X86ISD::SAHF, dl, MVT::i32, TruncSrl);
 }
@@ -12882,44 +13242,40 @@ SDValue X86TargetLowering::getRsqrtEstimate(SDValue Op,
                                             DAGCombinerInfo &DCI,
                                             unsigned &RefinementSteps,
                                             bool &UseOneConstNR) const {
-  // FIXME: We should use instruction latency models to calculate the cost of
-  // each potential sequence, but this is very hard to do reliably because
-  // at least Intel's Core* chips have variable timing based on the number of
-  // significant digits in the divisor and/or sqrt operand.
-  if (!Subtarget->useSqrtEst())
-    return SDValue();
-
   EVT VT = Op.getValueType();
+  const char *RecipOp;
 
-  // SSE1 has rsqrtss and rsqrtps.
+  // SSE1 has rsqrtss and rsqrtps. AVX adds a 256-bit variant for rsqrtps.
   // TODO: Add support for AVX512 (v16f32).
   // It is likely not profitable to do this for f64 because a double-precision
   // rsqrt estimate with refinement on x86 prior to FMA requires at least 16
   // instructions: convert to single, rsqrtss, convert back to double, refine
   // (3 steps = at least 13 insts). If an 'rsqrtsd' variant was added to the ISA
   // along with FMA, this could be a throughput win.
-  if ((Subtarget->hasSSE1() && (VT == MVT::f32 || VT == MVT::v4f32)) ||
-      (Subtarget->hasAVX() && VT == MVT::v8f32)) {
-    RefinementSteps = 1;
-    UseOneConstNR = false;
-    return DCI.DAG.getNode(X86ISD::FRSQRT, SDLoc(Op), VT, Op);
-  }
-  return SDValue();
-}
+  if (VT == MVT::f32 && Subtarget->hasSSE1())
+    RecipOp = "sqrtf";
+  else if ((VT == MVT::v4f32 && Subtarget->hasSSE1()) ||
+           (VT == MVT::v8f32 && Subtarget->hasAVX()))
+    RecipOp = "vec-sqrtf";
+  else
+    return SDValue();
+
+  TargetRecip Recips = DCI.DAG.getTarget().Options.Reciprocals;
+  if (!Recips.isEnabled(RecipOp))
+    return SDValue();
+
+  RefinementSteps = Recips.getRefinementSteps(RecipOp);
+  UseOneConstNR = false;
+  return DCI.DAG.getNode(X86ISD::FRSQRT, SDLoc(Op), VT, Op);
+}
 
 /// The minimum architected relative accuracy is 2^-12. We need one
 /// Newton-Raphson step to have a good float result (24 bits of precision).
 SDValue X86TargetLowering::getRecipEstimate(SDValue Op,
                                             DAGCombinerInfo &DCI,
                                             unsigned &RefinementSteps) const {
-  // FIXME: We should use instruction latency models to calculate the cost of
-  // each potential sequence, but this is very hard to do reliably because
-  // at least Intel's Core* chips have variable timing based on the number of
-  // significant digits in the divisor.
-  if (!Subtarget->useReciprocalEst())
-    return SDValue();
-
   EVT VT = Op.getValueType();
+  const char *RecipOp;
 
   // SSE1 has rcpss and rcpps. AVX adds a 256-bit variant for rcpps.
   // TODO: Add support for AVX512 (v16f32).
@@ -12928,12 +13284,20 @@ SDValue X86TargetLowering::getRecipEstimate(SDValue Op,
   // 15 instructions: convert to single, rcpss, convert back to double, refine
   // (3 steps = 12 insts). If an 'rcpsd' variant was added to the ISA
   // along with FMA, this could be a throughput win.
-  if ((Subtarget->hasSSE1() && (VT == MVT::f32 || VT == MVT::v4f32)) ||
-      (Subtarget->hasAVX() && VT == MVT::v8f32)) {
-    RefinementSteps = ReciprocalEstimateRefinementSteps;
-    return DCI.DAG.getNode(X86ISD::FRCP, SDLoc(Op), VT, Op);
-  }
-  return SDValue();
+  if (VT == MVT::f32 && Subtarget->hasSSE1())
+    RecipOp = "divf";
+  else if ((VT == MVT::v4f32 && Subtarget->hasSSE1()) ||
+           (VT == MVT::v8f32 && Subtarget->hasAVX()))
+    RecipOp = "vec-divf";
+  else
+    return SDValue();
+
+  TargetRecip Recips = DCI.DAG.getTarget().Options.Reciprocals;
+  if (!Recips.isEnabled(RecipOp))
+    return SDValue();
+
+  RefinementSteps = Recips.getRefinementSteps(RecipOp);
+  return DCI.DAG.getNode(X86ISD::FRCP, SDLoc(Op), VT, Op);
 }
 
 /// If we have at least two divisions that use the same divisor, convert to
@@ -12942,8 +13306,8 @@ SDValue X86TargetLowering::getRecipEstimate(SDValue Op,
 /// This is because we still need one division to calculate the reciprocal and
 /// then we need two multiplies by that reciprocal as replacements for the
 /// original divisions.
-bool X86TargetLowering::combineRepeatedFPDivisors(unsigned NumUsers) const {
-  return NumUsers > 1;
+unsigned X86TargetLowering::combineRepeatedFPDivisors() const {
+  return 2;
 }
 
 static bool isAllOnes(SDValue V) {
@@ -12955,10 +13319,6 @@ static bool isAllOnes(SDValue V) {
 /// if it's possible.
 SDValue X86TargetLowering::LowerToBT(SDValue And, ISD::CondCode CC,
                                      SDLoc dl, SelectionDAG &DAG) const {
-   // @LOCALMOD: NaCl validator rejects BT, BTS, and BTC.
-  if (Subtarget->isTargetNaCl())
-    return SDValue();
-  
   SDValue Op0 = And.getOperand(0);
   SDValue Op1 = And.getOperand(1);
   if (Op0.getOpcode() == ISD::TRUNCATE)
@@ -12998,7 +13358,7 @@ SDValue X86TargetLowering::LowerToBT(SDValue And, ISD::CondCode CC,
     // Use BT if the immediate can't be encoded in a TEST instruction.
     if (!isUInt<32>(AndRHSVal) && isPowerOf2_64(AndRHSVal)) {
       LHS = AndLHS;
-      RHS = DAG.getConstant(Log2_64_Ceil(AndRHSVal), LHS.getValueType());
+      RHS = DAG.getConstant(Log2_64_Ceil(AndRHSVal), dl, LHS.getValueType());
     }
   }
 
@@ -13020,7 +13380,7 @@ SDValue X86TargetLowering::LowerToBT(SDValue And, ISD::CondCode CC,
     SDValue BT = DAG.getNode(X86ISD::BT, dl, MVT::i32, LHS, RHS);
     X86::CondCode Cond = CC == ISD::SETEQ ? X86::COND_AE : X86::COND_B;
     return DAG.getNode(X86ISD::SETCC, dl, MVT::i8,
-                       DAG.getConstant(Cond, MVT::i8), BT);
+                       DAG.getConstant(Cond, dl, MVT::i8), BT);
   }
 
   return SDValue();
@@ -13101,6 +13461,49 @@ static SDValue Lower256IntVSETCC(SDValue Op, SelectionDAG &DAG) {
                      DAG.getNode(Op.getOpcode(), dl, NewVT, LHS2, RHS2, CC));
 }
 
+static SDValue LowerBoolVSETCC_AVX512(SDValue Op, SelectionDAG &DAG) {
+  SDValue Op0 = Op.getOperand(0);
+  SDValue Op1 = Op.getOperand(1);
+  SDValue CC = Op.getOperand(2);
+  MVT VT = Op.getSimpleValueType();
+  SDLoc dl(Op);
+
+  assert(Op0.getValueType().getVectorElementType() == MVT::i1 &&
+         "Unexpected type for boolean compare operation");
+  ISD::CondCode SetCCOpcode = cast<CondCodeSDNode>(CC)->get();
+  SDValue NotOp0 = DAG.getNode(ISD::XOR, dl, VT, Op0,
+                               DAG.getConstant(-1, dl, VT));
+  SDValue NotOp1 = DAG.getNode(ISD::XOR, dl, VT, Op1,
+                               DAG.getConstant(-1, dl, VT));
+  switch (SetCCOpcode) {
+  default: llvm_unreachable("Unexpected SETCC condition");
+  case ISD::SETEQ:
+    // (x == y) -> ~(x ^ y)
+    return DAG.getNode(ISD::XOR, dl, VT,
+                       DAG.getNode(ISD::XOR, dl, VT, Op0, Op1),
+                       DAG.getConstant(-1, dl, VT));
+  case ISD::SETNE:
+    // (x != y) -> (x ^ y)
+    return DAG.getNode(ISD::XOR, dl, VT, Op0, Op1);
+  case ISD::SETUGT:
+  case ISD::SETGT:
+    // (x > y) -> (x & ~y)
+    return DAG.getNode(ISD::AND, dl, VT, Op0, NotOp1);
+  case ISD::SETULT:
+  case ISD::SETLT:
+    // (x < y) -> (~x & y)
+    return DAG.getNode(ISD::AND, dl, VT, NotOp0, Op1);
+  case ISD::SETULE:
+  case ISD::SETLE:
+    // (x <= y) -> (~x | y)
+    return DAG.getNode(ISD::OR, dl, VT, NotOp0, Op1);
+  case ISD::SETUGE:
+  case ISD::SETGE:
+    // (x >=y) -> (x | ~y)
+    return DAG.getNode(ISD::OR, dl, VT, Op0, NotOp1);
+  }
+}
+
 static SDValue LowerIntVSETCC_AVX512(SDValue Op, SelectionDAG &DAG,
                                      const X86Subtarget *Subtarget) {
   SDValue Op0 = Op.getOperand(0);
@@ -13138,7 +13541,7 @@ static SDValue LowerIntVSETCC_AVX512(SDValue Op, SelectionDAG &DAG,
     return DAG.getNode(Opc, dl, VT, Op0, Op1);
   Opc = Unsigned ? X86ISD::CMPMU: X86ISD::CMPM;
   return DAG.getNode(Opc, dl, VT, Op0, Op1,
-                     DAG.getConstant(SSECC, MVT::i8));
+                     DAG.getConstant(SSECC, dl, MVT::i8));
 }
 
 /// \brief Try to turn a VSETULT into a VSETULE by modifying its second
@@ -13165,7 +13568,7 @@ static SDValue ChangeVSETULTtoVSETULE(SDLoc dl, SDValue Op1, SelectionDAG &DAG)
     if (Val == 0)
       return SDValue();
 
-    ULTOp1.push_back(DAG.getConstant(Val - 1, EVT));
+    ULTOp1.push_back(DAG.getConstant(Val - 1, dl, EVT));
   }
 
   return DAG.getNode(ISD::BUILD_VECTOR, dl, VT, ULTOp1);
@@ -13205,22 +13608,25 @@ static SDValue LowerVSETCC(SDValue Op, const X86Subtarget *Subtarget,
       }
 
       SDValue Cmp0 = DAG.getNode(Opc, dl, VT, Op0, Op1,
-                                 DAG.getConstant(CC0, MVT::i8));
+                                 DAG.getConstant(CC0, dl, MVT::i8));
       SDValue Cmp1 = DAG.getNode(Opc, dl, VT, Op0, Op1,
-                                 DAG.getConstant(CC1, MVT::i8));
+                                 DAG.getConstant(CC1, dl, MVT::i8));
       return DAG.getNode(CombineOpc, dl, VT, Cmp0, Cmp1);
     }
     // Handle all other FP comparisons here.
     return DAG.getNode(Opc, dl, VT, Op0, Op1,
-                       DAG.getConstant(SSECC, MVT::i8));
+                       DAG.getConstant(SSECC, dl, MVT::i8));
   }
 
   // Break 256-bit integer vector compare into smaller ones.
   if (VT.is256BitVector() && !Subtarget->hasInt256())
     return Lower256IntVSETCC(Op, DAG);
 
-  bool MaskResult = (VT.getVectorElementType() == MVT::i1);
   EVT OpVT = Op1.getValueType();
+  if (OpVT.getVectorElementType() == MVT::i1)
+    return LowerBoolVSETCC_AVX512(Op, DAG);
+
+  bool MaskResult = (VT.getVectorElementType() == MVT::i1);
   if (Subtarget->hasAVX512()) {
     if (Op1.getValueType().is512BitVector() ||
         (Subtarget->hasBWI() && Subtarget->hasVLX()) ||
@@ -13271,8 +13677,8 @@ static SDValue LowerVSETCC(SDValue Op, const X86Subtarget *Subtarget,
   if (hasMinMax) {
     switch (SetCCOpcode) {
     default: break;
-    case ISD::SETULE: Opc = X86ISD::UMIN; MinMax = true; break;
-    case ISD::SETUGE: Opc = X86ISD::UMAX; MinMax = true; break;
+    case ISD::SETULE: Opc = ISD::UMIN; MinMax = true; break;
+    case ISD::SETUGE: Opc = ISD::UMAX; MinMax = true; break;
     }
 
     if (MinMax) { Swap = false; Invert = false; FlipSigns = false; }
@@ -13322,18 +13728,18 @@ static SDValue LowerVSETCC(SDValue Op, const X86Subtarget *Subtarget,
       assert(Subtarget->hasSSE2() && "Don't know how to lower!");
 
       // First cast everything to the right type.
-      Op0 = DAG.getNode(ISD::BITCAST, dl, MVT::v4i32, Op0);
-      Op1 = DAG.getNode(ISD::BITCAST, dl, MVT::v4i32, Op1);
+      Op0 = DAG.getBitcast(MVT::v4i32, Op0);
+      Op1 = DAG.getBitcast(MVT::v4i32, Op1);
 
       // Since SSE has no unsigned integer comparisons, we need to flip the sign
       // bits of the inputs before performing those operations. The lower
       // compare is always unsigned.
       SDValue SB;
       if (FlipSigns) {
-        SB = DAG.getConstant(0x80000000U, MVT::v4i32);
+        SB = DAG.getConstant(0x80000000U, dl, MVT::v4i32);
       } else {
-        SDValue Sign = DAG.getConstant(0x80000000U, MVT::i32);
-        SDValue Zero = DAG.getConstant(0x00000000U, MVT::i32);
+        SDValue Sign = DAG.getConstant(0x80000000U, dl, MVT::i32);
+        SDValue Zero = DAG.getConstant(0x00000000U, dl, MVT::i32);
         SB = DAG.getNode(ISD::BUILD_VECTOR, dl, MVT::v4i32,
                          Sign, Zero, Sign, Zero);
       }
@@ -13357,7 +13763,7 @@ static SDValue LowerVSETCC(SDValue Op, const X86Subtarget *Subtarget,
       if (Invert)
         Result = DAG.getNOT(dl, Result, MVT::v4i32);
 
-      return DAG.getNode(ISD::BITCAST, dl, VT, Result);
+      return DAG.getBitcast(VT, Result);
     }
 
     if (Opc == X86ISD::PCMPEQ && !Subtarget->hasSSE41()) {
@@ -13366,8 +13772,8 @@ static SDValue LowerVSETCC(SDValue Op, const X86Subtarget *Subtarget,
       assert(Subtarget->hasSSE2() && !FlipSigns && "Don't know how to lower!");
 
       // First cast everything to the right type.
-      Op0 = DAG.getNode(ISD::BITCAST, dl, MVT::v4i32, Op0);
-      Op1 = DAG.getNode(ISD::BITCAST, dl, MVT::v4i32, Op1);
+      Op0 = DAG.getBitcast(MVT::v4i32, Op0);
+      Op1 = DAG.getBitcast(MVT::v4i32, Op1);
 
       // Do the compare.
       SDValue Result = DAG.getNode(Opc, dl, MVT::v4i32, Op0, Op1);
@@ -13380,7 +13786,7 @@ static SDValue LowerVSETCC(SDValue Op, const X86Subtarget *Subtarget,
       if (Invert)
         Result = DAG.getNOT(dl, Result, MVT::v4i32);
 
-      return DAG.getNode(ISD::BITCAST, dl, VT, Result);
+      return DAG.getBitcast(VT, Result);
     }
   }
 
@@ -13388,7 +13794,8 @@ static SDValue LowerVSETCC(SDValue Op, const X86Subtarget *Subtarget,
   // bits of the inputs before performing those operations.
   if (FlipSigns) {
     EVT EltVT = VT.getVectorElementType();
-    SDValue SB = DAG.getConstant(APInt::getSignBit(EltVT.getSizeInBits()), VT);
+    SDValue SB = DAG.getConstant(APInt::getSignBit(EltVT.getSizeInBits()), dl,
+                                 VT);
     Op0 = DAG.getNode(ISD::XOR, dl, VT, Op0, SB);
     Op1 = DAG.getNode(ISD::XOR, dl, VT, Op1, SB);
   }
@@ -13456,7 +13863,7 @@ SDValue X86TargetLowering::LowerSETCC(SDValue Op, SelectionDAG &DAG) const {
 
       CCode = X86::GetOppositeBranchCondition(CCode);
       SDValue SetCC = DAG.getNode(X86ISD::SETCC, dl, MVT::i8,
-                                  DAG.getConstant(CCode, MVT::i8),
+                                  DAG.getConstant(CCode, dl, MVT::i8),
                                   Op0.getOperand(1));
       if (VT == MVT::i1)
         return DAG.getNode(ISD::TRUNCATE, dl, MVT::i1, SetCC);
@@ -13468,18 +13875,18 @@ SDValue X86TargetLowering::LowerSETCC(SDValue Op, SelectionDAG &DAG) const {
       (CC == ISD::SETEQ || CC == ISD::SETNE)) {
 
     ISD::CondCode NewCC = ISD::getSetCCInverse(CC, true);
-    return DAG.getSetCC(dl, VT, Op0, DAG.getConstant(0, MVT::i1), NewCC);
+    return DAG.getSetCC(dl, VT, Op0, DAG.getConstant(0, dl, MVT::i1), NewCC);
   }
 
   bool isFP = Op1.getSimpleValueType().isFloatingPoint();
-  unsigned X86CC = TranslateX86CC(CC, isFP, Op0, Op1, DAG);
+  unsigned X86CC = TranslateX86CC(CC, dl, isFP, Op0, Op1, DAG);
   if (X86CC == X86::COND_INVALID)
     return SDValue();
 
   SDValue EFLAGS = EmitCmp(Op0, Op1, X86CC, dl, DAG);
   EFLAGS = ConvertCmpIfNecessary(EFLAGS, DAG);
   SDValue SetCC = DAG.getNode(X86ISD::SETCC, dl, MVT::i8,
-                              DAG.getConstant(X86CC, MVT::i8), EFLAGS);
+                              DAG.getConstant(X86CC, dl, MVT::i8), EFLAGS);
   if (VT == MVT::i1)
     return DAG.getNode(ISD::TRUNCATE, dl, MVT::i1, SetCC);
   return SetCC;
@@ -13544,12 +13951,12 @@ SDValue X86TargetLowering::LowerSELECT(SDValue Op, SelectionDAG &DAG) const {
     if (SSECC != 8) {
       if (Subtarget->hasAVX512()) {
         SDValue Cmp = DAG.getNode(X86ISD::FSETCC, DL, MVT::i1, CondOp0, CondOp1,
-                                  DAG.getConstant(SSECC, MVT::i8));
+                                  DAG.getConstant(SSECC, DL, MVT::i8));
         return DAG.getNode(X86ISD::SELECT, DL, VT, Cmp, Op1, Op2);
       }
 
       SDValue Cmp = DAG.getNode(X86ISD::FSETCC, DL, VT, CondOp0, CondOp1,
-                                DAG.getConstant(SSECC, MVT::i8));
+                                DAG.getConstant(SSECC, DL, MVT::i8));
 
       // If we have AVX, we can use a variable vector select (VBLENDV) instead
       // of 3 logic instructions for size savings and potentially speed.
@@ -13576,12 +13983,12 @@ SDValue X86TargetLowering::LowerSELECT(SDValue Op, SelectionDAG &DAG) const {
         SDValue VCmp = DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, VecVT, Cmp);
 
         EVT VCmpVT = VT == MVT::f32 ? MVT::v4i32 : MVT::v2i64;
-        VCmp = DAG.getNode(ISD::BITCAST, DL, VCmpVT, VCmp);
+        VCmp = DAG.getBitcast(VCmpVT, VCmp);
 
         SDValue VSel = DAG.getNode(ISD::VSELECT, DL, VecVT, VCmp, VOp1, VOp2);
 
         return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT,
-                           VSel, DAG.getIntPtrConstant(0));
+                           VSel, DAG.getIntPtrConstant(0, DL));
       }
       SDValue AndN = DAG.getNode(X86ISD::FANDN, DL, VT, Cmp, Op2);
       SDValue And = DAG.getNode(X86ISD::FAND, DL, VT, Cmp, Op1);
@@ -13589,6 +13996,40 @@ SDValue X86TargetLowering::LowerSELECT(SDValue Op, SelectionDAG &DAG) const {
     }
   }
 
+  if (VT.isVector() && VT.getScalarType() == MVT::i1) {
+    SDValue Op1Scalar;
+    if (ISD::isBuildVectorOfConstantSDNodes(Op1.getNode()))
+      Op1Scalar = ConvertI1VectorToInteger(Op1, DAG);
+    else if (Op1.getOpcode() == ISD::BITCAST && Op1.getOperand(0))
+      Op1Scalar = Op1.getOperand(0);
+    SDValue Op2Scalar;
+    if (ISD::isBuildVectorOfConstantSDNodes(Op2.getNode()))
+      Op2Scalar = ConvertI1VectorToInteger(Op2, DAG);
+    else if (Op2.getOpcode() == ISD::BITCAST && Op2.getOperand(0))
+      Op2Scalar = Op2.getOperand(0);
+    if (Op1Scalar.getNode() && Op2Scalar.getNode()) {
+      SDValue newSelect = DAG.getNode(ISD::SELECT, DL,
+                                      Op1Scalar.getValueType(),
+                                      Cond, Op1Scalar, Op2Scalar);
+      if (newSelect.getValueSizeInBits() == VT.getSizeInBits())
+        return DAG.getBitcast(VT, newSelect);
+      SDValue ExtVec = DAG.getBitcast(MVT::v8i1, newSelect);
+      return DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, VT, ExtVec,
+                         DAG.getIntPtrConstant(0, DL));
+    }
+  }
+
+  if (VT == MVT::v4i1 || VT == MVT::v2i1) {
+    SDValue zeroConst = DAG.getIntPtrConstant(0, DL);
+    Op1 = DAG.getNode(ISD::INSERT_SUBVECTOR, DL, MVT::v8i1,
+                      DAG.getUNDEF(MVT::v8i1), Op1, zeroConst);
+    Op2 = DAG.getNode(ISD::INSERT_SUBVECTOR, DL, MVT::v8i1,
+                      DAG.getUNDEF(MVT::v8i1), Op2, zeroConst);
+    SDValue newSelect = DAG.getNode(ISD::SELECT, DL, MVT::v8i1,
+                                    Cond, Op1, Op2);
+    return DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, VT, newSelect, zeroConst);
+  }
+
   if (Cond.getOpcode() == ISD::SETCC) {
     SDValue NewCond = LowerSETCC(Cond, DAG);
     if (NewCond.getNode())
@@ -13619,21 +14060,22 @@ SDValue X86TargetLowering::LowerSELECT(SDValue Op, SelectionDAG &DAG) const {
             (isAllOnes(Op1) == (CondCode == X86::COND_NE))) {
           SDVTList VTs = DAG.getVTList(CmpOp0.getValueType(), MVT::i32);
           SDValue Neg = DAG.getNode(X86ISD::SUB, DL, VTs,
-                                    DAG.getConstant(0, CmpOp0.getValueType()),
+                                    DAG.getConstant(0, DL,
+                                                    CmpOp0.getValueType()),
                                     CmpOp0);
           SDValue Res = DAG.getNode(X86ISD::SETCC_CARRY, DL, Op.getValueType(),
-                                    DAG.getConstant(X86::COND_B, MVT::i8),
+                                    DAG.getConstant(X86::COND_B, DL, MVT::i8),
                                     SDValue(Neg.getNode(), 1));
           return Res;
         }
 
       Cmp = DAG.getNode(X86ISD::CMP, DL, MVT::i32,
-                        CmpOp0, DAG.getConstant(1, CmpOp0.getValueType()));
+                        CmpOp0, DAG.getConstant(1, DL, CmpOp0.getValueType()));
       Cmp = ConvertCmpIfNecessary(Cmp, DAG);
 
       SDValue Res =   // Res = 0 or -1.
         DAG.getNode(X86ISD::SETCC_CARRY, DL, Op.getValueType(),
-                    DAG.getConstant(X86::COND_B, MVT::i8), Cmp);
+                    DAG.getConstant(X86::COND_B, DL, MVT::i8), Cmp);
 
       if (isAllOnes(Op1) != (CondCode == X86::COND_E))
         Res = DAG.getNOT(DL, Res, Res.getValueType());
@@ -13705,14 +14147,14 @@ SDValue X86TargetLowering::LowerSELECT(SDValue Op, SelectionDAG &DAG) const {
     else
       Cond = X86Op.getValue(1);
 
-    CC = DAG.getConstant(X86Cond, MVT::i8);
+    CC = DAG.getConstant(X86Cond, DL, MVT::i8);
     addTest = false;
   }
 
   if (addTest) {
-    // Look pass the truncate if the high bits are known zero.
+    // Look past the truncate if the high bits are known zero.
     if (isTruncWithZeroHighBitsInput(Cond, DAG))
-        Cond = Cond.getOperand(0);
+      Cond = Cond.getOperand(0);
 
     // We know the result of AND is compared against zero. Try to match
     // it to BT.
@@ -13727,7 +14169,7 @@ SDValue X86TargetLowering::LowerSELECT(SDValue Op, SelectionDAG &DAG) const {
   }
 
   if (addTest) {
-    CC = DAG.getConstant(X86::COND_NE, MVT::i8);
+    CC = DAG.getConstant(X86::COND_NE, DL, MVT::i8);
     Cond = EmitTest(Cond, X86::COND_NE, DL, DAG);
   }
 
@@ -13742,7 +14184,8 @@ SDValue X86TargetLowering::LowerSELECT(SDValue Op, SelectionDAG &DAG) const {
     if ((CondCode == X86::COND_AE || CondCode == X86::COND_B) &&
         (isAllOnes(Op1) || isAllOnes(Op2)) && (isZero(Op1) || isZero(Op2))) {
       SDValue Res = DAG.getNode(X86ISD::SETCC_CARRY, DL, Op.getValueType(),
-                                DAG.getConstant(X86::COND_B, MVT::i8), Cond);
+                                DAG.getConstant(X86::COND_B, DL, MVT::i8),
+                                Cond);
       if (isAllOnes(Op1) != (CondCode == X86::COND_B))
         return DAG.getNOT(DL, Res, Res.getValueType());
       return Res;
@@ -13771,7 +14214,8 @@ SDValue X86TargetLowering::LowerSELECT(SDValue Op, SelectionDAG &DAG) const {
   return DAG.getNode(X86ISD::CMOV, DL, VTs, Ops);
 }
 
-static SDValue LowerSIGN_EXTEND_AVX512(SDValue Op, const X86Subtarget *Subtarget,
+static SDValue LowerSIGN_EXTEND_AVX512(SDValue Op,
+                                       const X86Subtarget *Subtarget,
                                        SelectionDAG &DAG) {
   MVT VT = Op->getSimpleValueType(0);
   SDValue In = Op->getOperand(0);
@@ -13797,7 +14241,7 @@ static SDValue LowerSIGN_EXTEND_AVX512(SDValue Op, const X86Subtarget *Subtarget
 
   unsigned int NumElts = VT.getVectorNumElements();
 
-  if (NumElts != 8 && NumElts != 16)
+  if (NumElts != 8 && NumElts != 16 && !Subtarget->hasBWI())
     return SDValue();
 
   if (VT.is512BitVector() && InVT.getVectorElementType() != MVT::i1) {
@@ -13806,22 +14250,74 @@ static SDValue LowerSIGN_EXTEND_AVX512(SDValue Op, const X86Subtarget *Subtarget
     return DAG.getNode(X86ISD::VSEXT, dl, VT, In);
   }
 
-  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
   assert (InVT.getVectorElementType() == MVT::i1 && "Unexpected vector type");
+  MVT ExtVT = NumElts == 8 ? MVT::v8i64 : MVT::v16i32;
+  SDValue NegOne =
+   DAG.getConstant(APInt::getAllOnesValue(ExtVT.getScalarSizeInBits()), dl,
+                   ExtVT);
+  SDValue Zero =
+   DAG.getConstant(APInt::getNullValue(ExtVT.getScalarSizeInBits()), dl, ExtVT);
+
+  SDValue V = DAG.getNode(ISD::VSELECT, dl, ExtVT, In, NegOne, Zero);
+  if (VT.is512BitVector())
+    return V;
+  return DAG.getNode(X86ISD::VTRUNC, dl, VT, V);
+}
 
-  MVT ExtVT = (NumElts == 8) ? MVT::v8i64 : MVT::v16i32;
-  Constant *C = ConstantInt::get(*DAG.getContext(),
-    APInt::getAllOnesValue(ExtVT.getScalarType().getSizeInBits()));
+static SDValue LowerSIGN_EXTEND_VECTOR_INREG(SDValue Op,
+                                             const X86Subtarget *Subtarget,
+                                             SelectionDAG &DAG) {
+  SDValue In = Op->getOperand(0);
+  MVT VT = Op->getSimpleValueType(0);
+  MVT InVT = In.getSimpleValueType();
+  assert(VT.getSizeInBits() == InVT.getSizeInBits());
 
-  SDValue CP = DAG.getConstantPool(C, TLI.getPointerTy());
-  unsigned Alignment = cast<ConstantPoolSDNode>(CP)->getAlignment();
-  SDValue Ld = DAG.getLoad(ExtVT.getScalarType(), dl, DAG.getEntryNode(), CP,
-                          MachinePointerInfo::getConstantPool(),
-                          false, false, false, Alignment);
-  SDValue Brcst = DAG.getNode(X86ISD::VBROADCASTM, dl, ExtVT, In, Ld);
-  if (VT.is512BitVector())
-    return Brcst;
-  return DAG.getNode(X86ISD::VTRUNC, dl, VT, Brcst);
+  MVT InSVT = InVT.getScalarType();
+  assert(VT.getScalarType().getScalarSizeInBits() > InSVT.getScalarSizeInBits());
+
+  if (VT != MVT::v2i64 && VT != MVT::v4i32 && VT != MVT::v8i16)
+    return SDValue();
+  if (InSVT != MVT::i32 && InSVT != MVT::i16 && InSVT != MVT::i8)
+    return SDValue();
+
+  SDLoc dl(Op);
+
+  // SSE41 targets can use the pmovsx* instructions directly.
+  if (Subtarget->hasSSE41())
+    return DAG.getNode(X86ISD::VSEXT, dl, VT, In);
+
+  // pre-SSE41 targets unpack lower lanes and then sign-extend using SRAI.
+  SDValue Curr = In;
+  MVT CurrVT = InVT;
+
+  // As SRAI is only available on i16/i32 types, we expand only up to i32
+  // and handle i64 separately.
+  while (CurrVT != VT && CurrVT.getScalarType() != MVT::i32) {
+    Curr = DAG.getNode(X86ISD::UNPCKL, dl, CurrVT, DAG.getUNDEF(CurrVT), Curr);
+    MVT CurrSVT = MVT::getIntegerVT(CurrVT.getScalarSizeInBits() * 2);
+    CurrVT = MVT::getVectorVT(CurrSVT, CurrVT.getVectorNumElements() / 2);
+    Curr = DAG.getBitcast(CurrVT, Curr);
+  }
+
+  SDValue SignExt = Curr;
+  if (CurrVT != InVT) {
+    unsigned SignExtShift =
+        CurrVT.getScalarSizeInBits() - InSVT.getScalarSizeInBits();
+    SignExt = DAG.getNode(X86ISD::VSRAI, dl, CurrVT, Curr,
+                          DAG.getConstant(SignExtShift, dl, MVT::i8));
+  }
+
+  if (CurrVT == VT)
+    return SignExt;
+
+  if (VT == MVT::v2i64 && CurrVT == MVT::v4i32) {
+    SDValue Sign = DAG.getNode(X86ISD::VSRAI, dl, CurrVT, Curr,
+                               DAG.getConstant(31, dl, MVT::i8));
+    SDValue Ext = DAG.getVectorShuffle(CurrVT, dl, SignExt, Sign, {0, 4, 1, 5});
+    return DAG.getBitcast(VT, Ext);
+  }
+
+  return SDValue();
 }
 
 static SDValue LowerSIGN_EXTEND(SDValue Op, const X86Subtarget *Subtarget,
@@ -13978,8 +14474,8 @@ static SDValue LowerExtendedLoad(SDValue Op, const X86Subtarget *Subtarget,
          "Can only lower sext loads with a single scalar load!");
 
   unsigned loadRegZize = RegSz;
-  if (Ext == ISD::SEXTLOAD && RegSz == 256)
-    loadRegZize /= 2;
+  if (Ext == ISD::SEXTLOAD && RegSz >= 256)
+    loadRegZize = 128;
 
   // Represent our vector as a sequence of elements which are the
   // largest scalar that we can load.
@@ -14001,8 +14497,8 @@ static SDValue LowerExtendedLoad(SDValue Op, const X86Subtarget *Subtarget,
 
   SmallVector<SDValue, 8> Chains;
   SDValue Ptr = Ld->getBasePtr();
-  SDValue Increment =
-      DAG.getConstant(SclrLoadTy.getSizeInBits() / 8, TLI.getPointerTy());
+  SDValue Increment = DAG.getConstant(SclrLoadTy.getSizeInBits() / 8, dl,
+                                      TLI.getPointerTy(DAG.getDataLayout()));
   SDValue Res = DAG.getUNDEF(LoadUnitVecVT);
 
   for (unsigned i = 0; i < NumLoads; ++i) {
@@ -14018,7 +14514,7 @@ static SDValue LowerExtendedLoad(SDValue Op, const X86Subtarget *Subtarget,
       Res = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, LoadUnitVecVT, ScalarLoad);
     else
       Res = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, LoadUnitVecVT, Res,
-                        ScalarLoad, DAG.getIntPtrConstant(i));
+                        ScalarLoad, DAG.getIntPtrConstant(i, dl));
 
     Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr, Increment);
   }
@@ -14027,7 +14523,7 @@ static SDValue LowerExtendedLoad(SDValue Op, const X86Subtarget *Subtarget,
 
   // Bitcast the loaded value to a vector of the original element type, in
   // the size of the target vector type.
-  SDValue SlicedVec = DAG.getNode(ISD::BITCAST, dl, WideVecVT, Res);
+  SDValue SlicedVec = DAG.getBitcast(WideVecVT, Res);
   unsigned SizeRatio = RegSz / MemSz;
 
   if (Ext == ISD::SEXTLOAD) {
@@ -14052,13 +14548,14 @@ static SDValue LowerExtendedLoad(SDValue Op, const X86Subtarget *Subtarget,
     SDValue Shuff = DAG.getVectorShuffle(
         WideVecVT, dl, SlicedVec, DAG.getUNDEF(WideVecVT), &ShuffleVec[0]);
 
-    Shuff = DAG.getNode(ISD::BITCAST, dl, RegVT, Shuff);
+    Shuff = DAG.getBitcast(RegVT, Shuff);
 
     // Build the arithmetic shift.
     unsigned Amt = RegVT.getVectorElementType().getSizeInBits() -
                    MemVT.getVectorElementType().getSizeInBits();
     Shuff =
-        DAG.getNode(ISD::SRA, dl, RegVT, Shuff, DAG.getConstant(Amt, RegVT));
+        DAG.getNode(ISD::SRA, dl, RegVT, Shuff,
+                    DAG.getConstant(Amt, dl, RegVT));
 
     DAG.ReplaceAllUsesOfValueWith(SDValue(Ld, 1), TF);
     return Shuff;
@@ -14073,7 +14570,7 @@ static SDValue LowerExtendedLoad(SDValue Op, const X86Subtarget *Subtarget,
                                        DAG.getUNDEF(WideVecVT), &ShuffleVec[0]);
 
   // Bitcast to the requested type.
-  Shuff = DAG.getNode(ISD::BITCAST, dl, RegVT, Shuff);
+  Shuff = DAG.getBitcast(RegVT, Shuff);
   DAG.ReplaceAllUsesOfValueWith(SDValue(Ld, 1), TF);
   return Shuff;
 }
@@ -14225,7 +14722,7 @@ SDValue X86TargetLowering::LowerBRCOND(SDValue Op, SelectionDAG &DAG) const {
     else
       Cond = X86Op.getValue(1);
 
-    CC = DAG.getConstant(X86Cond, MVT::i8);
+    CC = DAG.getConstant(X86Cond, dl, MVT::i8);
     addTest = false;
   } else {
     unsigned CondOpc;
@@ -14256,7 +14753,7 @@ SDValue X86TargetLowering::LowerBRCOND(SDValue Op, SelectionDAG &DAG) const {
           X86::CondCode CCode =
             (X86::CondCode)Cond.getOperand(0).getConstantOperandVal(0);
           CCode = X86::GetOppositeBranchCondition(CCode);
-          CC = DAG.getConstant(CCode, MVT::i8);
+          CC = DAG.getConstant(CCode, dl, MVT::i8);
           SDNode *User = *Op.getNode()->use_begin();
           // Look for an unconditional branch following this conditional branch.
           // We need this because we need to reverse the successors in order
@@ -14274,7 +14771,7 @@ SDValue X86TargetLowering::LowerBRCOND(SDValue Op, SelectionDAG &DAG) const {
             X86::CondCode CCode =
               (X86::CondCode)Cond.getOperand(1).getConstantOperandVal(0);
             CCode = X86::GetOppositeBranchCondition(CCode);
-            CC = DAG.getConstant(CCode, MVT::i8);
+            CC = DAG.getConstant(CCode, dl, MVT::i8);
             Cond = Cmp;
             addTest = false;
           }
@@ -14287,7 +14784,7 @@ SDValue X86TargetLowering::LowerBRCOND(SDValue Op, SelectionDAG &DAG) const {
       X86::CondCode CCode =
         (X86::CondCode)Cond.getOperand(0).getConstantOperandVal(0);
       CCode = X86::GetOppositeBranchCondition(CCode);
-      CC = DAG.getConstant(CCode, MVT::i8);
+      CC = DAG.getConstant(CCode, dl, MVT::i8);
       Cond = Cond.getOperand(0).getOperand(1);
       addTest = false;
     } else if (Cond.getOpcode() == ISD::SETCC &&
@@ -14313,10 +14810,10 @@ SDValue X86TargetLowering::LowerBRCOND(SDValue Op, SelectionDAG &DAG) const {
           SDValue Cmp = DAG.getNode(X86ISD::CMP, dl, MVT::i32,
                                     Cond.getOperand(0), Cond.getOperand(1));
           Cmp = ConvertCmpIfNecessary(Cmp, DAG);
-          CC = DAG.getConstant(X86::COND_NE, MVT::i8);
+          CC = DAG.getConstant(X86::COND_NE, dl, MVT::i8);
           Chain = DAG.getNode(X86ISD::BRCOND, dl, Op.getValueType(),
                               Chain, Dest, CC, Cmp);
-          CC = DAG.getConstant(X86::COND_P, MVT::i8);
+          CC = DAG.getConstant(X86::COND_P, dl, MVT::i8);
           Cond = Cmp;
           addTest = false;
         }
@@ -14343,10 +14840,10 @@ SDValue X86TargetLowering::LowerBRCOND(SDValue Op, SelectionDAG &DAG) const {
           SDValue Cmp = DAG.getNode(X86ISD::CMP, dl, MVT::i32,
                                     Cond.getOperand(0), Cond.getOperand(1));
           Cmp = ConvertCmpIfNecessary(Cmp, DAG);
-          CC = DAG.getConstant(X86::COND_NE, MVT::i8);
+          CC = DAG.getConstant(X86::COND_NE, dl, MVT::i8);
           Chain = DAG.getNode(X86ISD::BRCOND, dl, Op.getValueType(),
                               Chain, Dest, CC, Cmp);
-          CC = DAG.getConstant(X86::COND_NP, MVT::i8);
+          CC = DAG.getConstant(X86::COND_NP, dl, MVT::i8);
           Cond = Cmp;
           addTest = false;
           Dest = FalseBB;
@@ -14374,7 +14871,7 @@ SDValue X86TargetLowering::LowerBRCOND(SDValue Op, SelectionDAG &DAG) const {
 
   if (addTest) {
     X86::CondCode X86Cond = Inverted ? X86::COND_E : X86::COND_NE;
-    CC = DAG.getConstant(X86Cond, MVT::i8);
+    CC = DAG.getConstant(X86Cond, dl, MVT::i8);
     Cond = EmitTest(Cond, X86Cond, dl, DAG);
   }
   Cond = ConvertCmpIfNecessary(Cond, DAG);
@@ -14411,7 +14908,7 @@ X86TargetLowering::LowerDYNAMIC_STACKALLOC(SDValue Op,
 
     // Chain the dynamic stack allocation so that it doesn't modify the stack
     // pointer when other instructions are using the stack.
-    Chain = DAG.getCALLSEQ_START(Chain, DAG.getIntPtrConstant(0, true),
+    Chain = DAG.getCALLSEQ_START(Chain, DAG.getIntPtrConstant(0, dl, true),
         SDLoc(Node));
 
     SDValue Size = Tmp2.getOperand(1);
@@ -14423,11 +14920,11 @@ X86TargetLowering::LowerDYNAMIC_STACKALLOC(SDValue Op,
     Tmp1 = DAG.getNode(ISD::SUB, dl, VT, SP, Size); // Value
     if (Align > StackAlign)
       Tmp1 = DAG.getNode(ISD::AND, dl, VT, Tmp1,
-          DAG.getConstant(-(uint64_t)Align, VT));
+          DAG.getConstant(-(uint64_t)Align, dl, VT));
     Chain = DAG.getCopyToReg(Chain, dl, SPReg, Tmp1); // Output chain
 
-    Tmp2 = DAG.getCALLSEQ_END(Chain, DAG.getIntPtrConstant(0, true),
-        DAG.getIntPtrConstant(0, true), SDValue(),
+    Tmp2 = DAG.getCALLSEQ_END(Chain, DAG.getIntPtrConstant(0, dl, true),
+        DAG.getIntPtrConstant(0, dl, true), SDValue(),
         SDLoc(Node));
 
     SDValue Ops[2] = { Tmp1, Tmp2 };
@@ -14441,7 +14938,7 @@ X86TargetLowering::LowerDYNAMIC_STACKALLOC(SDValue Op,
   EVT VT = Op.getNode()->getValueType(0);
 
   bool Is64Bit = Subtarget->is64Bit();
-  EVT SPTy = getPointerTy();
+  MVT SPTy = getPointerTy(DAG.getDataLayout());
 
   if (SplitStack) {
     MachineRegisterInfo &MRI = MF.getRegInfo();
@@ -14458,8 +14955,7 @@ X86TargetLowering::LowerDYNAMIC_STACKALLOC(SDValue Op,
                              "have nested arguments.");
     }
 
-    const TargetRegisterClass *AddrRegClass =
-      getRegClassFor(getPointerTy());
+    const TargetRegisterClass *AddrRegClass = getRegClassFor(SPTy);
     unsigned Vreg = MRI.createVirtualRegister(AddrRegClass);
     Chain = DAG.getCopyToReg(Chain, dl, Vreg, Size);
     SDValue Value = DAG.getNode(X86ISD::SEG_ALLOCA, dl, SPTy, Chain,
@@ -14477,14 +14973,13 @@ X86TargetLowering::LowerDYNAMIC_STACKALLOC(SDValue Op,
     Chain = DAG.getNode(X86ISD::WIN_ALLOCA, dl, NodeTys, Chain, Flag);
 
     const X86RegisterInfo *RegInfo = Subtarget->getRegisterInfo();
-    (void)RegInfo;                // @LOCALMOD
-    unsigned SPReg = X86StackPtr; // @LOCALMOD
+    unsigned SPReg = RegInfo->getStackRegister();
     SDValue SP = DAG.getCopyFromReg(Chain, dl, SPReg, SPTy);
     Chain = SP.getValue(1);
 
     if (Align) {
       SP = DAG.getNode(ISD::AND, dl, VT, SP.getValue(0),
-                       DAG.getConstant(-(uint64_t)Align, VT));
+                       DAG.getConstant(-(uint64_t)Align, dl, VT));
       Chain = DAG.getCopyToReg(Chain, dl, SPReg, SP);
     }
 
@@ -14495,6 +14990,7 @@ X86TargetLowering::LowerDYNAMIC_STACKALLOC(SDValue Op,
 
 SDValue X86TargetLowering::LowerVASTART(SDValue Op, SelectionDAG &DAG) const {
   MachineFunction &MF = DAG.getMachineFunction();
+  auto PtrVT = getPointerTy(MF.getDataLayout());
   X86MachineFunctionInfo *FuncInfo = MF.getInfo<X86MachineFunctionInfo>();
 
   const Value *SV = cast<SrcValueSDNode>(Op.getOperand(2))->getValue();
@@ -14503,8 +14999,7 @@ SDValue X86TargetLowering::LowerVASTART(SDValue Op, SelectionDAG &DAG) const {
   if (!Subtarget->is64Bit() || Subtarget->isTargetWin64()) {
     // vastart just stores the address of the VarArgsFrameIndex slot into the
     // memory location argument.
-    SDValue FR = DAG.getFrameIndex(FuncInfo->getVarArgsFrameIndex(),
-                                   getPointerTy());
+    SDValue FR = DAG.getFrameIndex(FuncInfo->getVarArgsFrameIndex(), PtrVT);
     return DAG.getStore(Op.getOperand(0), DL, FR, Op.getOperand(1),
                         MachinePointerInfo(SV), false, false, 0);
   }
@@ -14514,43 +15009,36 @@ SDValue X86TargetLowering::LowerVASTART(SDValue Op, SelectionDAG &DAG) const {
   //   fp_offset         (48 - 48 + 8 * 16)
   //   overflow_arg_area (point to parameters coming in memory).
   //   reg_save_area
-  unsigned PointerSize = TD->getPointerSize(0); // @LOCALMOD
   SmallVector<SDValue, 8> MemOps;
   SDValue FIN = Op.getOperand(1);
   // Store gp_offset
   SDValue Store = DAG.getStore(Op.getOperand(0), DL,
                                DAG.getConstant(FuncInfo->getVarArgsGPOffset(),
-                                               MVT::i32),
+                                               DL, MVT::i32),
                                FIN, MachinePointerInfo(SV), false, false, 0);
   MemOps.push_back(Store);
 
   // Store fp_offset
-  FIN = DAG.getNode(ISD::ADD, DL, getPointerTy(),
-                    FIN, DAG.getIntPtrConstant(4));
+  FIN = DAG.getNode(ISD::ADD, DL, PtrVT, FIN, DAG.getIntPtrConstant(4, DL));
   Store = DAG.getStore(Op.getOperand(0), DL,
-                       DAG.getConstant(FuncInfo->getVarArgsFPOffset(),
+                       DAG.getConstant(FuncInfo->getVarArgsFPOffset(), DL,
                                        MVT::i32),
                        FIN, MachinePointerInfo(SV, 4), false, false, 0);
   MemOps.push_back(Store);
 
   // Store ptr to overflow_arg_area
-  FIN = DAG.getNode(ISD::ADD, DL, getPointerTy(),
-                    FIN, DAG.getIntPtrConstant(4));
-  SDValue OVFIN = DAG.getFrameIndex(FuncInfo->getVarArgsFrameIndex(),
-                                    getPointerTy());
+  FIN = DAG.getNode(ISD::ADD, DL, PtrVT, FIN, DAG.getIntPtrConstant(4, DL));
+  SDValue OVFIN = DAG.getFrameIndex(FuncInfo->getVarArgsFrameIndex(), PtrVT);
   Store = DAG.getStore(Op.getOperand(0), DL, OVFIN, FIN,
                        MachinePointerInfo(SV, 8),
                        false, false, 0);
   MemOps.push_back(Store);
 
   // Store ptr to reg_save_area.
-  FIN = DAG.getNode(ISD::ADD, DL, getPointerTy(),
-                    FIN, DAG.getIntPtrConstant(PointerSize)); // @LOCALMOD
-  SDValue RSFIN = DAG.getFrameIndex(FuncInfo->getRegSaveFrameIndex(),
-                                    getPointerTy());
+  FIN = DAG.getNode(ISD::ADD, DL, PtrVT, FIN, DAG.getIntPtrConstant(8, DL));
+  SDValue RSFIN = DAG.getFrameIndex(FuncInfo->getRegSaveFrameIndex(), PtrVT);
   Store = DAG.getStore(Op.getOperand(0), DL, RSFIN, FIN,
-                       MachinePointerInfo(SV, 8+PointerSize), // @LOCALMOD
-                       false, false, 0);
+                       MachinePointerInfo(SV, 16), false, false, 0);
   MemOps.push_back(Store);
   return DAG.getNode(ISD::TokenFactor, DL, MVT::Other, MemOps);
 }
@@ -14559,8 +15047,7 @@ SDValue X86TargetLowering::LowerVAARG(SDValue Op, SelectionDAG &DAG) const {
   assert(Subtarget->is64Bit() &&
          "LowerVAARG only handles 64-bit va_arg!");
   assert((Subtarget->isTargetLinux() ||
-          Subtarget->isTargetDarwin() ||
-          Subtarget->isTargetNaCl()) && // @LOCALMOD
+          Subtarget->isTargetDarwin()) &&
           "Unhandled target in LowerVAARG");
   assert(Op.getNode()->getNumOperands() == 4);
   SDValue Chain = Op.getOperand(0);
@@ -14571,7 +15058,7 @@ SDValue X86TargetLowering::LowerVAARG(SDValue Op, SelectionDAG &DAG) const {
 
   EVT ArgVT = Op.getNode()->getValueType(0);
   Type *ArgTy = ArgVT.getTypeForEVT(*DAG.getContext());
-  uint32_t ArgSize = getDataLayout()->getTypeAllocSize(ArgTy);
+  uint32_t ArgSize = DAG.getDataLayout().getTypeAllocSize(ArgTy);
   uint8_t ArgMode;
 
   // Decide which area this value should be read from.
@@ -14589,7 +15076,7 @@ SDValue X86TargetLowering::LowerVAARG(SDValue Op, SelectionDAG &DAG) const {
 
   if (ArgMode == 2) {
     // Sanity Check: Make sure using fp_offset makes sense.
-    assert(!DAG.getTarget().Options.UseSoftFloat &&
+    assert(!Subtarget->useSoftFloat() &&
            !(DAG.getMachineFunction().getFunction()->hasFnAttribute(
                Attribute::NoImplicitFloat)) &&
            Subtarget->hasSSE1());
@@ -14597,10 +15084,10 @@ SDValue X86TargetLowering::LowerVAARG(SDValue Op, SelectionDAG &DAG) const {
 
   // Insert VAARG_64 node into the DAG
   // VAARG_64 returns two values: Variable Argument Address, Chain
-  SDValue InstOps[] = {Chain, SrcPtr, DAG.getConstant(ArgSize, MVT::i32),
-                       DAG.getConstant(ArgMode, MVT::i8),
-                       DAG.getConstant(Align, MVT::i32)};
-  SDVTList VTs = DAG.getVTList(getPointerTy(), MVT::Other);
+  SDValue InstOps[] = {Chain, SrcPtr, DAG.getConstant(ArgSize, dl, MVT::i32),
+                       DAG.getConstant(ArgMode, dl, MVT::i8),
+                       DAG.getConstant(Align, dl, MVT::i32)};
+  SDVTList VTs = DAG.getVTList(getPointerTy(DAG.getDataLayout()), MVT::Other);
   SDValue VAARG = DAG.getMemIntrinsicNode(X86ISD::VAARG_64, dl,
                                           VTs, InstOps, MVT::i64,
                                           MachinePointerInfo(SV),
@@ -14630,15 +15117,7 @@ static SDValue LowerVACOPY(SDValue Op, const X86Subtarget *Subtarget,
   SDLoc DL(Op);
 
   return DAG.getMemcpy(Chain, DL, DstPtr, SrcPtr,
-                       // @LOCALMOD-START
-                       // Size is actually 8 + 2 * pointer size and align
-                       // is the pointer ABI alignment but we don't have a
-                       // pointer to TD in this static function
-                       DAG.getIntPtrConstant(Subtarget->has64BitPointers() ?
-                                             24 : 16),
-                       Subtarget->has64BitPointers() ? 8 : 4,
-                       /*isVolatile*/false,
-                       // @LOCALMOD-END
+                       DAG.getIntPtrConstant(24, DL), 8, /*isVolatile*/false,
                        false, false,
                        MachinePointerInfo(DstSV), MachinePointerInfo(SrcSV));
 }
@@ -14659,7 +15138,7 @@ static SDValue getTargetVShiftByConstNode(unsigned Opc, SDLoc dl, MVT VT,
     if (Opc == X86ISD::VSRAI)
       ShiftAmt = ElementType.getSizeInBits() - 1;
     else
-      return DAG.getConstant(0, VT);
+      return DAG.getConstant(0, dl, VT);
   }
 
   assert((Opc == X86ISD::VSHLI || Opc == X86ISD::VSRLI || Opc == X86ISD::VSRAI)
@@ -14684,7 +15163,7 @@ static SDValue getTargetVShiftByConstNode(unsigned Opc, SDLoc dl, MVT VT,
         }
         ND = cast<ConstantSDNode>(CurrentOp);
         const APInt &C = ND->getAPIntValue();
-        Elts.push_back(DAG.getConstant(C.shl(ShiftAmt), ElementType));
+        Elts.push_back(DAG.getConstant(C.shl(ShiftAmt), dl, ElementType));
       }
       break;
     case X86ISD::VSRLI:
@@ -14696,7 +15175,7 @@ static SDValue getTargetVShiftByConstNode(unsigned Opc, SDLoc dl, MVT VT,
         }
         ND = cast<ConstantSDNode>(CurrentOp);
         const APInt &C = ND->getAPIntValue();
-        Elts.push_back(DAG.getConstant(C.lshr(ShiftAmt), ElementType));
+        Elts.push_back(DAG.getConstant(C.lshr(ShiftAmt), dl, ElementType));
       }
       break;
     case X86ISD::VSRAI:
@@ -14708,7 +15187,7 @@ static SDValue getTargetVShiftByConstNode(unsigned Opc, SDLoc dl, MVT VT,
         }
         ND = cast<ConstantSDNode>(CurrentOp);
         const APInt &C = ND->getAPIntValue();
-        Elts.push_back(DAG.getConstant(C.ashr(ShiftAmt), ElementType));
+        Elts.push_back(DAG.getConstant(C.ashr(ShiftAmt), dl, ElementType));
       }
       break;
     }
@@ -14716,7 +15195,8 @@ static SDValue getTargetVShiftByConstNode(unsigned Opc, SDLoc dl, MVT VT,
     return DAG.getNode(ISD::BUILD_VECTOR, dl, VT, Elts);
   }
 
-  return DAG.getNode(Opc, dl, VT, SrcOp, DAG.getConstant(ShiftAmt, MVT::i8));
+  return DAG.getNode(Opc, dl, VT, SrcOp,
+                     DAG.getConstant(ShiftAmt, dl, MVT::i8));
 }
 
 // getTargetVShiftNode - Handle vector element shifts where the shift amount
@@ -14754,7 +15234,7 @@ static SDValue getTargetVShiftNode(unsigned Opc, SDLoc dl, MVT VT,
     SmallVector<SDValue, 4> ShOps;
     ShOps.push_back(ShAmt);
     if (SVT == MVT::i32) {
-      ShOps.push_back(DAG.getConstant(0, SVT));
+      ShOps.push_back(DAG.getConstant(0, dl, SVT));
       ShOps.push_back(DAG.getUNDEF(SVT));
     }
     ShOps.push_back(DAG.getUNDEF(SVT));
@@ -14768,13 +15248,13 @@ static SDValue getTargetVShiftNode(unsigned Opc, SDLoc dl, MVT VT,
   MVT EltVT = VT.getVectorElementType();
   EVT ShVT = MVT::getVectorVT(EltVT, 128/EltVT.getSizeInBits());
 
-  ShAmt = DAG.getNode(ISD::BITCAST, dl, ShVT, ShAmt);
+  ShAmt = DAG.getBitcast(ShVT, ShAmt);
   return DAG.getNode(Opc, dl, VT, SrcOp, ShAmt);
 }
 
 /// \brief Return (and \p Op, \p Mask) for compare instructions or
 /// (vselect \p Mask, \p Op, \p PreservedSrc) for others along with the
-/// necessary casting for \p Mask when lowering masking intrinsics.
+/// necessary casting or extending for \p Mask when lowering masking intrinsics
 static SDValue getVectorMaskingNode(SDValue Op, SDValue Mask,
                                     SDValue PreservedSrc,
                                     const X86Subtarget *Subtarget,
@@ -14782,8 +15262,8 @@ static SDValue getVectorMaskingNode(SDValue Op, SDValue Mask,
     EVT VT = Op.getValueType();
     EVT MaskVT = EVT::getVectorVT(*DAG.getContext(),
                                   MVT::i1, VT.getVectorNumElements());
-    EVT BitcastVT = EVT::getVectorVT(*DAG.getContext(), MVT::i1,
-                                     Mask.getValueType().getSizeInBits());
+    SDValue VMask = SDValue();
+    unsigned OpcodeSelect = ISD::VSELECT;
     SDLoc dl(Op);
 
     assert(MaskVT.isSimple() && "invalid mask type");
@@ -14791,11 +15271,20 @@ static SDValue getVectorMaskingNode(SDValue Op, SDValue Mask,
     if (isAllOnes(Mask))
       return Op;
 
-    // In case when MaskVT equals v2i1 or v4i1, low 2 or 4 elements
-    // are extracted by EXTRACT_SUBVECTOR.
-    SDValue VMask = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, MaskVT,
-                              DAG.getNode(ISD::BITCAST, dl, BitcastVT, Mask),
-                              DAG.getIntPtrConstant(0));
+    if (MaskVT.bitsGT(Mask.getValueType())) {
+      EVT newMaskVT =  EVT::getIntegerVT(*DAG.getContext(),
+                                         MaskVT.getSizeInBits());
+      VMask = DAG.getBitcast(MaskVT,
+                             DAG.getNode(ISD::ANY_EXTEND, dl, newMaskVT, Mask));
+    } else {
+      EVT BitcastVT = EVT::getVectorVT(*DAG.getContext(), MVT::i1,
+                                       Mask.getValueType().getSizeInBits());
+      // In case when MaskVT equals v2i1 or v4i1, low 2 or 4 elements
+      // are extracted by EXTRACT_SUBVECTOR.
+      VMask = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, MaskVT,
+                          DAG.getBitcast(BitcastVT, Mask),
+                          DAG.getIntPtrConstant(0, dl));
+    }
 
     switch (Op.getOpcode()) {
       default: break;
@@ -14804,10 +15293,18 @@ static SDValue getVectorMaskingNode(SDValue Op, SDValue Mask,
       case X86ISD::CMPM:
       case X86ISD::CMPMU:
         return DAG.getNode(ISD::AND, dl, VT, Op, VMask);
+      case X86ISD::VTRUNC:
+      case X86ISD::VTRUNCS:
+      case X86ISD::VTRUNCUS:
+        // We can't use ISD::VSELECT here because it is not always "Legal"
+        // for the destination type. For example vpmovqb require only AVX512
+        // and vselect that can operate on byte element type require BWI
+        OpcodeSelect = X86ISD::SELECT;
+        break;
     }
     if (PreservedSrc.getOpcode() == ISD::UNDEF)
       PreservedSrc = getZeroVector(VT, Subtarget, DAG, dl);
-    return DAG.getNode(ISD::VSELECT, dl, VT, VMask, Op, PreservedSrc);
+    return DAG.getNode(OpcodeSelect, dl, VT, VMask, Op, PreservedSrc);
 }
 
 /// \brief Creates an SDNode for a predicated scalar operation.
@@ -14834,6 +15331,60 @@ static SDValue getScalarMaskingNode(SDValue Op, SDValue Mask,
     return DAG.getNode(X86ISD::SELECT, dl, VT, IMask, Op, PreservedSrc);
 }
 
+static int getSEHRegistrationNodeSize(const Function *Fn) {
+  if (!Fn->hasPersonalityFn())
+    report_fatal_error(
+        "querying registration node size for function without personality");
+  // The RegNodeSize is 6 32-bit words for SEH and 4 for C++ EH. See
+  // WinEHStatePass for the full struct definition.
+  switch (classifyEHPersonality(Fn->getPersonalityFn())) {
+  case EHPersonality::MSVC_X86SEH: return 24;
+  case EHPersonality::MSVC_CXX: return 16;
+  default: break;
+  }
+  report_fatal_error("can only recover FP for MSVC EH personality functions");
+}
+
+/// When the 32-bit MSVC runtime transfers control to us, either to an outlined
+/// function or when returning to a parent frame after catching an exception, we
+/// recover the parent frame pointer by doing arithmetic on the incoming EBP.
+/// Here's the math:
+///   RegNodeBase = EntryEBP - RegNodeSize
+///   ParentFP = RegNodeBase - RegNodeFrameOffset
+/// Subtracting RegNodeSize takes us to the offset of the registration node, and
+/// subtracting the offset (negative on x86) takes us back to the parent FP.
+static SDValue recoverFramePointer(SelectionDAG &DAG, const Function *Fn,
+                                   SDValue EntryEBP) {
+  MachineFunction &MF = DAG.getMachineFunction();
+  SDLoc dl;
+
+  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
+  MVT PtrVT = TLI.getPointerTy(DAG.getDataLayout());
+
+  // It's possible that the parent function no longer has a personality function
+  // if the exceptional code was optimized away, in which case we just return
+  // the incoming EBP.
+  if (!Fn->hasPersonalityFn())
+    return EntryEBP;
+
+  int RegNodeSize = getSEHRegistrationNodeSize(Fn);
+
+  // Get an MCSymbol that will ultimately resolve to the frame offset of the EH
+  // registration.
+  MCSymbol *OffsetSym =
+      MF.getMMI().getContext().getOrCreateParentFrameOffsetSymbol(
+          GlobalValue::getRealLinkageName(Fn->getName()));
+  SDValue OffsetSymVal = DAG.getMCSymbol(OffsetSym, PtrVT);
+  SDValue RegNodeFrameOffset =
+      DAG.getNode(ISD::LOCAL_RECOVER, dl, PtrVT, OffsetSymVal);
+
+  // RegNodeBase = EntryEBP - RegNodeSize
+  // ParentFP = RegNodeBase - RegNodeFrameOffset
+  SDValue RegNodeBase = DAG.getNode(ISD::SUB, dl, PtrVT, EntryEBP,
+                                    DAG.getConstant(RegNodeSize, dl, PtrVT));
+  return DAG.getNode(ISD::SUB, dl, PtrVT, RegNodeBase, RegNodeFrameOffset);
+}
+
 static SDValue LowerINTRINSIC_WO_CHAIN(SDValue Op, const X86Subtarget *Subtarget,
                                        SelectionDAG &DAG) {
   SDLoc dl(Op);
@@ -14850,14 +15401,53 @@ static SDValue LowerINTRINSIC_WO_CHAIN(SDValue Op, const X86Subtarget *Subtarget
     case INTR_TYPE_3OP:
       return DAG.getNode(IntrData->Opc0, dl, Op.getValueType(), Op.getOperand(1),
         Op.getOperand(2), Op.getOperand(3));
+    case INTR_TYPE_4OP:
+      return DAG.getNode(IntrData->Opc0, dl, Op.getValueType(), Op.getOperand(1),
+        Op.getOperand(2), Op.getOperand(3), Op.getOperand(4));
     case INTR_TYPE_1OP_MASK_RM: {
       SDValue Src = Op.getOperand(1);
-      SDValue Src0 = Op.getOperand(2);
+      SDValue PassThru = Op.getOperand(2);
       SDValue Mask = Op.getOperand(3);
-      SDValue RoundingMode = Op.getOperand(4);
+      SDValue RoundingMode;
+      // We allways add rounding mode to the Node.
+      // If the rounding mode is not specified, we add the 
+      // "current direction" mode.
+      if (Op.getNumOperands() == 4)
+        RoundingMode =
+          DAG.getConstant(X86::STATIC_ROUNDING::CUR_DIRECTION, dl, MVT::i32);
+      else
+        RoundingMode = Op.getOperand(4);
+      unsigned IntrWithRoundingModeOpcode = IntrData->Opc1;
+      if (IntrWithRoundingModeOpcode != 0)
+        if (cast<ConstantSDNode>(RoundingMode)->getZExtValue() !=
+            X86::STATIC_ROUNDING::CUR_DIRECTION)
+          return getVectorMaskingNode(DAG.getNode(IntrWithRoundingModeOpcode,
+                                      dl, Op.getValueType(), Src, RoundingMode),
+                                      Mask, PassThru, Subtarget, DAG);
       return getVectorMaskingNode(DAG.getNode(IntrData->Opc0, dl, VT, Src,
                                               RoundingMode),
-                                  Mask, Src0, Subtarget, DAG);
+                                  Mask, PassThru, Subtarget, DAG);
+    }
+    case INTR_TYPE_1OP_MASK: {
+      SDValue Src = Op.getOperand(1);
+      SDValue PassThru = Op.getOperand(2);
+      SDValue Mask = Op.getOperand(3);
+      // We add rounding mode to the Node when
+      //   - RM Opcode is specified and
+      //   - RM is not "current direction".
+      unsigned IntrWithRoundingModeOpcode = IntrData->Opc1;
+      if (IntrWithRoundingModeOpcode != 0) {
+        SDValue Rnd = Op.getOperand(4);
+        unsigned Round = cast<ConstantSDNode>(Rnd)->getZExtValue();
+        if (Round != X86::STATIC_ROUNDING::CUR_DIRECTION) {
+          return getVectorMaskingNode(DAG.getNode(IntrWithRoundingModeOpcode,
+                                      dl, Op.getValueType(),
+                                      Src, Rnd),
+                                      Mask, PassThru, Subtarget, DAG);
+        }
+      }
+      return getVectorMaskingNode(DAG.getNode(IntrData->Opc0, dl, VT, Src),
+                                  Mask, PassThru, Subtarget, DAG);
     }
     case INTR_TYPE_SCALAR_MASK_RM: {
       SDValue Src1 = Op.getOperand(1);
@@ -14865,11 +15455,12 @@ static SDValue LowerINTRINSIC_WO_CHAIN(SDValue Op, const X86Subtarget *Subtarget
       SDValue Src0 = Op.getOperand(3);
       SDValue Mask = Op.getOperand(4);
       // There are 2 kinds of intrinsics in this group:
-      // (1) With supress-all-exceptions (sae) - 6 operands
+      // (1) With supress-all-exceptions (sae) or rounding mode- 6 operands
       // (2) With rounding mode and sae - 7 operands.
       if (Op.getNumOperands() == 6) {
         SDValue Sae  = Op.getOperand(5);
-        return getScalarMaskingNode(DAG.getNode(IntrData->Opc0, dl, VT, Src1, Src2,
+        unsigned Opc = IntrData->Opc1 ? IntrData->Opc1 : IntrData->Opc0;
+        return getScalarMaskingNode(DAG.getNode(Opc, dl, VT, Src1, Src2,
                                                 Sae),
                                     Mask, Src0, Subtarget, DAG);
       }
@@ -14903,11 +15494,85 @@ static SDValue LowerINTRINSIC_WO_CHAIN(SDValue Op, const X86Subtarget *Subtarget
                                               Src1,Src2),
                                   Mask, PassThru, Subtarget, DAG);
     }
+    case INTR_TYPE_2OP_MASK_RM: {
+      SDValue Src1 = Op.getOperand(1);
+      SDValue Src2 = Op.getOperand(2);
+      SDValue PassThru = Op.getOperand(3);
+      SDValue Mask = Op.getOperand(4);
+      // We specify 2 possible modes for intrinsics, with/without rounding modes.
+      // First, we check if the intrinsic have rounding mode (6 operands),
+      // if not, we set rounding mode to "current".
+      SDValue Rnd;
+      if (Op.getNumOperands() == 6)
+        Rnd = Op.getOperand(5);
+      else
+        Rnd = DAG.getConstant(X86::STATIC_ROUNDING::CUR_DIRECTION, dl, MVT::i32);
+      return getVectorMaskingNode(DAG.getNode(IntrData->Opc0, dl, VT,
+                                              Src1, Src2, Rnd),
+                                  Mask, PassThru, Subtarget, DAG);
+    }
+    case INTR_TYPE_3OP_MASK_RM: {
+      SDValue Src1 = Op.getOperand(1);
+      SDValue Src2 = Op.getOperand(2);
+      SDValue Imm = Op.getOperand(3);
+      SDValue PassThru = Op.getOperand(4);
+      SDValue Mask = Op.getOperand(5);
+      // We specify 2 possible modes for intrinsics, with/without rounding modes.
+      // First, we check if the intrinsic have rounding mode (7 operands),
+      // if not, we set rounding mode to "current".
+      SDValue Rnd;
+      if (Op.getNumOperands() == 7)
+        Rnd = Op.getOperand(6);
+      else
+        Rnd = DAG.getConstant(X86::STATIC_ROUNDING::CUR_DIRECTION, dl, MVT::i32);
+      return getVectorMaskingNode(DAG.getNode(IntrData->Opc0, dl, VT,
+        Src1, Src2, Imm, Rnd),
+        Mask, PassThru, Subtarget, DAG);
+    }
+    case INTR_TYPE_3OP_MASK: {
+      SDValue Src1 = Op.getOperand(1);
+      SDValue Src2 = Op.getOperand(2);
+      SDValue Src3 = Op.getOperand(3);
+      SDValue PassThru = Op.getOperand(4);
+      SDValue Mask = Op.getOperand(5);
+      // We specify 2 possible opcodes for intrinsics with rounding modes.
+      // First, we check if the intrinsic may have non-default rounding mode,
+      // (IntrData->Opc1 != 0), then we check the rounding mode operand.
+      unsigned IntrWithRoundingModeOpcode = IntrData->Opc1;
+      if (IntrWithRoundingModeOpcode != 0) {
+        SDValue Rnd = Op.getOperand(6);
+        unsigned Round = cast<ConstantSDNode>(Rnd)->getZExtValue();
+        if (Round != X86::STATIC_ROUNDING::CUR_DIRECTION) {
+          return getVectorMaskingNode(DAG.getNode(IntrWithRoundingModeOpcode,
+                                      dl, Op.getValueType(),
+                                      Src1, Src2, Src3, Rnd),
+                                      Mask, PassThru, Subtarget, DAG);
+        }
+      }
+      return getVectorMaskingNode(DAG.getNode(IntrData->Opc0, dl, VT,
+                                              Src1, Src2, Src3),
+                                  Mask, PassThru, Subtarget, DAG);
+    }
+    case VPERM_3OP_MASKZ:
+    case VPERM_3OP_MASK:
+    case FMA_OP_MASK3:
+    case FMA_OP_MASKZ:
     case FMA_OP_MASK: {
       SDValue Src1 = Op.getOperand(1);
       SDValue Src2 = Op.getOperand(2);
       SDValue Src3 = Op.getOperand(3);
       SDValue Mask = Op.getOperand(4);
+      EVT VT = Op.getValueType();
+      SDValue PassThru = SDValue();
+
+      // set PassThru element
+      if (IntrData->Type == VPERM_3OP_MASKZ || IntrData->Type == FMA_OP_MASKZ)
+        PassThru = getZeroVector(VT, Subtarget, DAG, dl);
+      else if (IntrData->Type == FMA_OP_MASK3)
+        PassThru = Src3;
+      else
+        PassThru = Src1;
+
       // We specify 2 possible opcodes for intrinsics with rounding modes.
       // First, we check if the intrinsic may have non-default rounding mode,
       // (IntrData->Opc1 != 0), then we check the rounding mode operand.
@@ -14919,12 +15584,12 @@ static SDValue LowerINTRINSIC_WO_CHAIN(SDValue Op, const X86Subtarget *Subtarget
           return getVectorMaskingNode(DAG.getNode(IntrWithRoundingModeOpcode,
                                                   dl, Op.getValueType(),
                                                   Src1, Src2, Src3, Rnd),
-                                      Mask, Src1, Subtarget, DAG);
+                                      Mask, PassThru, Subtarget, DAG);
       }
       return getVectorMaskingNode(DAG.getNode(IntrData->Opc0,
                                               dl, Op.getValueType(),
                                               Src1, Src2, Src3),
-                                  Mask, Src1, Subtarget, DAG);
+                                  Mask, PassThru, Subtarget, DAG);
     }
     case CMP_MASK:
     case CMP_MASK_CC: {
@@ -14945,30 +15610,46 @@ static SDValue LowerINTRINSIC_WO_CHAIN(SDValue Op, const X86Subtarget *Subtarget
                                        Mask.getValueType().getSizeInBits());
       SDValue Cmp;
       if (IntrData->Type == CMP_MASK_CC) {
-        Cmp = DAG.getNode(IntrData->Opc0, dl, MaskVT, Op.getOperand(1),
-                    Op.getOperand(2), Op.getOperand(3));
+        SDValue CC = Op.getOperand(3);
+        CC = DAG.getNode(ISD::TRUNCATE, dl, MVT::i8, CC);
+        // We specify 2 possible opcodes for intrinsics with rounding modes.
+        // First, we check if the intrinsic may have non-default rounding mode,
+        // (IntrData->Opc1 != 0), then we check the rounding mode operand.
+        if (IntrData->Opc1 != 0) {
+          SDValue Rnd = Op.getOperand(5);
+          if (cast<ConstantSDNode>(Rnd)->getZExtValue() !=
+              X86::STATIC_ROUNDING::CUR_DIRECTION)
+            Cmp = DAG.getNode(IntrData->Opc1, dl, MaskVT, Op.getOperand(1),
+                              Op.getOperand(2), CC, Rnd);
+        }
+        //default rounding mode
+        if(!Cmp.getNode())
+            Cmp = DAG.getNode(IntrData->Opc0, dl, MaskVT, Op.getOperand(1),
+                              Op.getOperand(2), CC);
+
       } else {
         assert(IntrData->Type == CMP_MASK && "Unexpected intrinsic type!");
         Cmp = DAG.getNode(IntrData->Opc0, dl, MaskVT, Op.getOperand(1),
-                    Op.getOperand(2));
+                          Op.getOperand(2));
       }
       SDValue CmpMask = getVectorMaskingNode(Cmp, Mask,
-                                             DAG.getTargetConstant(0, MaskVT),
+                                             DAG.getTargetConstant(0, dl,
+                                                                   MaskVT),
                                              Subtarget, DAG);
       SDValue Res = DAG.getNode(ISD::INSERT_SUBVECTOR, dl, BitcastVT,
                                 DAG.getUNDEF(BitcastVT), CmpMask,
-                                DAG.getIntPtrConstant(0));
-      return DAG.getNode(ISD::BITCAST, dl, Op.getValueType(), Res);
+                                DAG.getIntPtrConstant(0, dl));
+      return DAG.getBitcast(Op.getValueType(), Res);
     }
     case COMI: { // Comparison intrinsics
       ISD::CondCode CC = (ISD::CondCode)IntrData->Opc1;
       SDValue LHS = Op.getOperand(1);
       SDValue RHS = Op.getOperand(2);
-      unsigned X86CC = TranslateX86CC(CC, true, LHS, RHS, DAG);
+      unsigned X86CC = TranslateX86CC(CC, dl, true, LHS, RHS, DAG);
       assert(X86CC != X86::COND_INVALID && "Unexpected illegal condition!");
       SDValue Cond = DAG.getNode(IntrData->Opc0, dl, MVT::i32, LHS, RHS);
       SDValue SetCC = DAG.getNode(X86ISD::SETCC, dl, MVT::i8,
-                                  DAG.getConstant(X86CC, MVT::i8), Cond);
+                                  DAG.getConstant(X86CC, dl, MVT::i8), Cond);
       return DAG.getNode(ISD::ZERO_EXTEND, dl, MVT::i32, SetCC);
     }
     case VSHIFT:
@@ -14987,18 +15668,10 @@ static SDValue LowerINTRINSIC_WO_CHAIN(SDValue Op, const X86Subtarget *Subtarget
       SDValue PassThru = Op.getOperand(2);
       if (isAllOnes(Mask)) // return data as is
         return Op.getOperand(1);
-      EVT VT = Op.getValueType();
-      EVT MaskVT = EVT::getVectorVT(*DAG.getContext(), MVT::i1,
-                                    VT.getVectorNumElements());
-      EVT BitcastVT = EVT::getVectorVT(*DAG.getContext(), MVT::i1,
-                                       Mask.getValueType().getSizeInBits());
-      SDLoc dl(Op);
-      SDValue VMask = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, MaskVT,
-                                  DAG.getNode(ISD::BITCAST, dl, BitcastVT, Mask),
-                                  DAG.getIntPtrConstant(0));
 
-      return DAG.getNode(IntrData->Opc0, dl, VT, VMask, DataToCompress,
-                         PassThru);
+      return getVectorMaskingNode(DAG.getNode(IntrData->Opc0, dl, VT,
+                                              DataToCompress),
+                                  Mask, PassThru, Subtarget, DAG);
     }
     case BLEND: {
       SDValue Mask = Op.getOperand(3);
@@ -15009,8 +15682,8 @@ static SDValue LowerINTRINSIC_WO_CHAIN(SDValue Op, const X86Subtarget *Subtarget
                                        Mask.getValueType().getSizeInBits());
       SDLoc dl(Op);
       SDValue VMask = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, MaskVT,
-                                  DAG.getNode(ISD::BITCAST, dl, BitcastVT, Mask),
-                                  DAG.getIntPtrConstant(0));
+                                  DAG.getBitcast(BitcastVT, Mask),
+                                  DAG.getIntPtrConstant(0, dl));
       return DAG.getNode(IntrData->Opc0, dl, VT, VMask, Op.getOperand(1),
                          Op.getOperand(2));
     }
@@ -15028,35 +15701,6 @@ static SDValue LowerINTRINSIC_WO_CHAIN(SDValue Op, const X86Subtarget *Subtarget
     // but second operand for node/instruction.
     return DAG.getNode(X86ISD::VPERMV, dl, Op.getValueType(),
                        Op.getOperand(2), Op.getOperand(1));
-  // @LOCALMOD-BEGIN
-  case Intrinsic::nacl_read_tp: {
-    EVT PtrVT = DAG.getTargetLoweringInfo().getPointerTy();
-    if (Subtarget->is64Bit() || llvm::TLSUseCall) {
-      // Call __nacl_read_tp() to get the thread pointer.
-      unsigned PtrSize = PtrVT.getSizeInBits();
-      IntegerType *RetTy = Type::getIntNTy(*DAG.getContext(), PtrSize);
-      SDValue ReadTpFunction = DAG.getExternalSymbol("__nacl_read_tp", PtrVT);
-      X86TargetLowering::ArgListTy Args;
-      const TargetLowering &TLI = DAG.getTargetLoweringInfo();
-      return TLI.LowerCallTo(TargetLowering::CallLoweringInfo(DAG)
-                             .setChain(DAG.getEntryNode())
-                             .setCallee(CallingConv::C, RetTy, ReadTpFunction,
-                                        std::move(Args), 0)).first;
-    } else {
-      // Get %gs:0, which contains the thread pointer on x86-32.
-      return DAG.getNode(X86ISD::THREAD_POINTER_FROM_GS, dl, PtrVT);
-    }
-  }
-  // @LOCALMOD-END
-  case Intrinsic::x86_avx512_mask_valign_q_512:
-  case Intrinsic::x86_avx512_mask_valign_d_512:
-    // Vector source operands are swapped.
-    return getVectorMaskingNode(DAG.getNode(X86ISD::VALIGN, dl,
-                                            Op.getValueType(), Op.getOperand(2),
-                                            Op.getOperand(1),
-                                            Op.getOperand(3)),
-                                Op.getOperand(5), Op.getOperand(4),
-                                Subtarget, DAG);
 
   // ptest and testp intrinsics. The intrinsic these come from are designed to
   // return an integer value, not just an instruction so lower it to the ptest
@@ -15119,16 +15763,16 @@ static SDValue LowerINTRINSIC_WO_CHAIN(SDValue Op, const X86Subtarget *Subtarget
     SDValue RHS = Op.getOperand(2);
     unsigned TestOpc = IsTestPacked ? X86ISD::TESTP : X86ISD::PTEST;
     SDValue Test = DAG.getNode(TestOpc, dl, MVT::i32, LHS, RHS);
-    SDValue CC = DAG.getConstant(X86CC, MVT::i8);
+    SDValue CC = DAG.getConstant(X86CC, dl, MVT::i8);
     SDValue SetCC = DAG.getNode(X86ISD::SETCC, dl, MVT::i8, CC, Test);
     return DAG.getNode(ISD::ZERO_EXTEND, dl, MVT::i32, SetCC);
   }
   case Intrinsic::x86_avx512_kortestz_w:
   case Intrinsic::x86_avx512_kortestc_w: {
     unsigned X86CC = (IntNo == Intrinsic::x86_avx512_kortestz_w)? X86::COND_E: X86::COND_B;
-    SDValue LHS = DAG.getNode(ISD::BITCAST, dl, MVT::v16i1, Op.getOperand(1));
-    SDValue RHS = DAG.getNode(ISD::BITCAST, dl, MVT::v16i1, Op.getOperand(2));
-    SDValue CC = DAG.getConstant(X86CC, MVT::i8);
+    SDValue LHS = DAG.getBitcast(MVT::v16i1, Op.getOperand(1));
+    SDValue RHS = DAG.getBitcast(MVT::v16i1, Op.getOperand(2));
+    SDValue CC = DAG.getConstant(X86CC, dl, MVT::i8);
     SDValue Test = DAG.getNode(X86ISD::KORTEST, dl, MVT::i32, LHS, RHS);
     SDValue SetCC = DAG.getNode(X86ISD::SETCC, dl, MVT::i1, CC, Test);
     return DAG.getNode(ISD::ZERO_EXTEND, dl, MVT::i32, SetCC);
@@ -15193,7 +15837,7 @@ static SDValue LowerINTRINSIC_WO_CHAIN(SDValue Op, const X86Subtarget *Subtarget
     SDVTList VTs = DAG.getVTList(Op.getValueType(), MVT::i32);
     SDValue PCMP = DAG.getNode(Opcode, dl, VTs, NewOps);
     SDValue SetCC = DAG.getNode(X86ISD::SETCC, dl, MVT::i8,
-                                DAG.getConstant(X86CC, MVT::i8),
+                                DAG.getConstant(X86CC, dl, MVT::i8),
                                 SDValue(PCMP.getNode(), 1));
     return DAG.getNode(ISD::ZERO_EXTEND, dl, MVT::i32, SetCC);
   }
@@ -15210,6 +15854,44 @@ static SDValue LowerINTRINSIC_WO_CHAIN(SDValue Op, const X86Subtarget *Subtarget
     SDVTList VTs = DAG.getVTList(Op.getValueType(), MVT::i32);
     return DAG.getNode(Opcode, dl, VTs, NewOps);
   }
+
+  case Intrinsic::x86_seh_lsda: {
+    // Compute the symbol for the LSDA. We know it'll get emitted later.
+    MachineFunction &MF = DAG.getMachineFunction();
+    SDValue Op1 = Op.getOperand(1);
+    auto *Fn = cast<Function>(cast<GlobalAddressSDNode>(Op1)->getGlobal());
+    MCSymbol *LSDASym = MF.getMMI().getContext().getOrCreateLSDASymbol(
+        GlobalValue::getRealLinkageName(Fn->getName()));
+
+    // Generate a simple absolute symbol reference. This intrinsic is only
+    // supported on 32-bit Windows, which isn't PIC.
+    SDValue Result = DAG.getMCSymbol(LSDASym, VT);
+    return DAG.getNode(X86ISD::Wrapper, dl, VT, Result);
+  }
+
+  case Intrinsic::x86_seh_recoverfp: {
+    SDValue FnOp = Op.getOperand(1);
+    SDValue IncomingFPOp = Op.getOperand(2);
+    GlobalAddressSDNode *GSD = dyn_cast<GlobalAddressSDNode>(FnOp);
+    auto *Fn = dyn_cast_or_null<Function>(GSD ? GSD->getGlobal() : nullptr);
+    if (!Fn)
+      report_fatal_error(
+          "llvm.x86.seh.recoverfp must take a function as the first argument");
+    return recoverFramePointer(DAG, Fn, IncomingFPOp);
+  }
+
+  case Intrinsic::localaddress: {
+    // Returns one of the stack, base, or frame pointer registers, depending on
+    // which is used to reference local variables.
+    MachineFunction &MF = DAG.getMachineFunction();
+    const X86RegisterInfo *RegInfo = Subtarget->getRegisterInfo();
+    unsigned Reg;
+    if (RegInfo->hasBasePointer(MF))
+      Reg = RegInfo->getBaseRegister();
+    else // This function handles the SP or FP case.
+      Reg = RegInfo->getPtrSizedFrameRegister(MF);
+    return DAG.getCopyFromReg(DAG.getEntryNode(), dl, Reg, VT);
+  }
   }
 }
 
@@ -15219,18 +15901,31 @@ static SDValue getGatherNode(unsigned Opc, SDValue Op, SelectionDAG &DAG,
                               const X86Subtarget * Subtarget) {
   SDLoc dl(Op);
   ConstantSDNode *C = dyn_cast<ConstantSDNode>(ScaleOp);
-  assert(C && "Invalid scale type");
-  SDValue Scale = DAG.getTargetConstant(C->getZExtValue(), MVT::i8);
+  if (!C)
+    llvm_unreachable("Invalid scale type");
+  unsigned ScaleVal = C->getZExtValue();
+  if (ScaleVal > 2 && ScaleVal != 4 && ScaleVal != 8)
+    llvm_unreachable("Valid scale values are 1, 2, 4, 8");
+
+  SDValue Scale = DAG.getTargetConstant(C->getZExtValue(), dl, MVT::i8);
   EVT MaskVT = MVT::getVectorVT(MVT::i1,
                              Index.getSimpleValueType().getVectorNumElements());
   SDValue MaskInReg;
   ConstantSDNode *MaskC = dyn_cast<ConstantSDNode>(Mask);
   if (MaskC)
-    MaskInReg = DAG.getTargetConstant(MaskC->getSExtValue(), MaskVT);
-  else
-    MaskInReg = DAG.getNode(ISD::BITCAST, dl, MaskVT, Mask);
+    MaskInReg = DAG.getTargetConstant(MaskC->getSExtValue(), dl, MaskVT);
+  else {
+    EVT BitcastVT = EVT::getVectorVT(*DAG.getContext(), MVT::i1,
+                                     Mask.getValueType().getSizeInBits());
+
+    // In case when MaskVT equals v2i1 or v4i1, low 2 or 4 elements
+    // are extracted by EXTRACT_SUBVECTOR.
+    MaskInReg = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, MaskVT,
+                            DAG.getBitcast(BitcastVT, Mask),
+                            DAG.getIntPtrConstant(0, dl));
+  }
   SDVTList VTs = DAG.getVTList(Op.getValueType(), MaskVT, MVT::Other);
-  SDValue Disp = DAG.getTargetConstant(0, MVT::i32);
+  SDValue Disp = DAG.getTargetConstant(0, dl, MVT::i32);
   SDValue Segment = DAG.getRegister(0, MVT::i32);
   if (Src.getOpcode() == ISD::UNDEF)
     Src = getZeroVector(Op.getValueType(), Subtarget, DAG, dl);
@@ -15245,18 +15940,31 @@ static SDValue getScatterNode(unsigned Opc, SDValue Op, SelectionDAG &DAG,
                                SDValue Index, SDValue ScaleOp, SDValue Chain) {
   SDLoc dl(Op);
   ConstantSDNode *C = dyn_cast<ConstantSDNode>(ScaleOp);
-  assert(C && "Invalid scale type");
-  SDValue Scale = DAG.getTargetConstant(C->getZExtValue(), MVT::i8);
-  SDValue Disp = DAG.getTargetConstant(0, MVT::i32);
+  if (!C)
+    llvm_unreachable("Invalid scale type");
+  unsigned ScaleVal = C->getZExtValue();
+  if (ScaleVal > 2 && ScaleVal != 4 && ScaleVal != 8)
+    llvm_unreachable("Valid scale values are 1, 2, 4, 8");
+
+  SDValue Scale = DAG.getTargetConstant(C->getZExtValue(), dl, MVT::i8);
+  SDValue Disp = DAG.getTargetConstant(0, dl, MVT::i32);
   SDValue Segment = DAG.getRegister(0, MVT::i32);
   EVT MaskVT = MVT::getVectorVT(MVT::i1,
                              Index.getSimpleValueType().getVectorNumElements());
   SDValue MaskInReg;
   ConstantSDNode *MaskC = dyn_cast<ConstantSDNode>(Mask);
   if (MaskC)
-    MaskInReg = DAG.getTargetConstant(MaskC->getSExtValue(), MaskVT);
-  else
-    MaskInReg = DAG.getNode(ISD::BITCAST, dl, MaskVT, Mask);
+    MaskInReg = DAG.getTargetConstant(MaskC->getSExtValue(), dl, MaskVT);
+  else {
+    EVT BitcastVT = EVT::getVectorVT(*DAG.getContext(), MVT::i1,
+                                     Mask.getValueType().getSizeInBits());
+
+    // In case when MaskVT equals v2i1 or v4i1, low 2 or 4 elements
+    // are extracted by EXTRACT_SUBVECTOR.
+    MaskInReg = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, MaskVT,
+                            DAG.getBitcast(BitcastVT, Mask),
+                            DAG.getIntPtrConstant(0, dl));
+  }
   SDVTList VTs = DAG.getVTList(MaskVT, MVT::Other);
   SDValue Ops[] = {Base, Scale, Index, Disp, Segment, MaskInReg, Src, Chain};
   SDNode *Res = DAG.getMachineNode(Opc, dl, VTs, Ops);
@@ -15269,17 +15977,17 @@ static SDValue getPrefetchNode(unsigned Opc, SDValue Op, SelectionDAG &DAG,
   SDLoc dl(Op);
   ConstantSDNode *C = dyn_cast<ConstantSDNode>(ScaleOp);
   assert(C && "Invalid scale type");
-  SDValue Scale = DAG.getTargetConstant(C->getZExtValue(), MVT::i8);
-  SDValue Disp = DAG.getTargetConstant(0, MVT::i32);
+  SDValue Scale = DAG.getTargetConstant(C->getZExtValue(), dl, MVT::i8);
+  SDValue Disp = DAG.getTargetConstant(0, dl, MVT::i32);
   SDValue Segment = DAG.getRegister(0, MVT::i32);
   EVT MaskVT =
     MVT::getVectorVT(MVT::i1, Index.getSimpleValueType().getVectorNumElements());
   SDValue MaskInReg;
   ConstantSDNode *MaskC = dyn_cast<ConstantSDNode>(Mask);
   if (MaskC)
-    MaskInReg = DAG.getTargetConstant(MaskC->getSExtValue(), MaskVT);
+    MaskInReg = DAG.getTargetConstant(MaskC->getSExtValue(), dl, MaskVT);
   else
-    MaskInReg = DAG.getNode(ISD::BITCAST, dl, MaskVT, Mask);
+    MaskInReg = DAG.getBitcast(MaskVT, Mask);
   //SDVTList VTs = DAG.getVTList(MVT::Other);
   SDValue Ops[] = {MaskInReg, Base, Scale, Index, Disp, Segment, Chain};
   SDNode *Res = DAG.getMachineNode(Opc, dl, MVT::Other, Ops);
@@ -15318,7 +16026,7 @@ static void getReadPerformanceCounter(SDNode *N, SDLoc DL,
     // The EAX register is loaded with the low-order 32 bits. The EDX register
     // is loaded with the supported high-order bits of the counter.
     SDValue Tmp = DAG.getNode(ISD::SHL, DL, MVT::i64, HI,
-                              DAG.getConstant(32, MVT::i8));
+                              DAG.getConstant(32, DL, MVT::i8));
     Results.push_back(DAG.getNode(ISD::OR, DL, MVT::i64, LO, Tmp));
     Results.push_back(Chain);
     return;
@@ -15372,7 +16080,7 @@ static void getReadTimeStampCounter(SDNode *N, SDLoc DL, unsigned Opcode,
     // The EDX register is loaded with the high-order 32 bits of the MSR, and
     // the EAX register is loaded with the low-order 32 bits.
     SDValue Tmp = DAG.getNode(ISD::SHL, DL, MVT::i64, HI,
-                              DAG.getConstant(32, MVT::i8));
+                              DAG.getConstant(32, DL, MVT::i8));
     Results.push_back(DAG.getNode(ISD::OR, DL, MVT::i64, LO, Tmp));
     Results.push_back(Chain);
     return;
@@ -15394,14 +16102,115 @@ static SDValue LowerREADCYCLECOUNTER(SDValue Op, const X86Subtarget *Subtarget,
   return DAG.getMergeValues(Results, DL);
 }
 
+static SDValue LowerSEHRESTOREFRAME(SDValue Op, const X86Subtarget *Subtarget,
+                                    SelectionDAG &DAG) {
+  MachineFunction &MF = DAG.getMachineFunction();
+  const Function *Fn = MF.getFunction();
+  SDLoc dl(Op);
+  SDValue Chain = Op.getOperand(0);
+
+  assert(Subtarget->getFrameLowering()->hasFP(MF) &&
+         "using llvm.x86.seh.restoreframe requires a frame pointer");
+
+  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
+  MVT VT = TLI.getPointerTy(DAG.getDataLayout());
+
+  const X86RegisterInfo *RegInfo = Subtarget->getRegisterInfo();
+  unsigned FrameReg =
+      RegInfo->getPtrSizedFrameRegister(DAG.getMachineFunction());
+  unsigned SPReg = RegInfo->getStackRegister();
+  unsigned SlotSize = RegInfo->getSlotSize();
+
+  // Get incoming EBP.
+  SDValue IncomingEBP =
+      DAG.getCopyFromReg(Chain, dl, FrameReg, VT);
+
+  // SP is saved in the first field of every registration node, so load
+  // [EBP-RegNodeSize] into SP.
+  int RegNodeSize = getSEHRegistrationNodeSize(Fn);
+  SDValue SPAddr = DAG.getNode(ISD::ADD, dl, VT, IncomingEBP,
+                               DAG.getConstant(-RegNodeSize, dl, VT));
+  SDValue NewSP =
+      DAG.getLoad(VT, dl, Chain, SPAddr, MachinePointerInfo(), false, false,
+                  false, VT.getScalarSizeInBits() / 8);
+  Chain = DAG.getCopyToReg(Chain, dl, SPReg, NewSP);
+
+  if (!RegInfo->needsStackRealignment(MF)) {
+    // Adjust EBP to point back to the original frame position.
+    SDValue NewFP = recoverFramePointer(DAG, Fn, IncomingEBP);
+    Chain = DAG.getCopyToReg(Chain, dl, FrameReg, NewFP);
+  } else {
+    assert(RegInfo->hasBasePointer(MF) &&
+           "functions with Win32 EH must use frame or base pointer register");
+
+    // Reload the base pointer (ESI) with the adjusted incoming EBP.
+    SDValue NewBP = recoverFramePointer(DAG, Fn, IncomingEBP);
+    Chain = DAG.getCopyToReg(Chain, dl, RegInfo->getBaseRegister(), NewBP);
+
+    // Reload the spilled EBP value, now that the stack and base pointers are
+    // set up.
+    X86MachineFunctionInfo *X86FI = MF.getInfo<X86MachineFunctionInfo>();
+    X86FI->setHasSEHFramePtrSave(true);
+    int FI = MF.getFrameInfo()->CreateSpillStackObject(SlotSize, SlotSize);
+    X86FI->setSEHFramePtrSaveIndex(FI);
+    SDValue NewFP = DAG.getLoad(VT, dl, Chain, DAG.getFrameIndex(FI, VT),
+                                MachinePointerInfo(), false, false, false,
+                                VT.getScalarSizeInBits() / 8);
+    Chain = DAG.getCopyToReg(NewFP, dl, FrameReg, NewFP);
+  }
+
+  return Chain;
+}
+
+/// \brief Lower intrinsics for TRUNCATE_TO_MEM case
+/// return truncate Store/MaskedStore Node
+static SDValue LowerINTRINSIC_TRUNCATE_TO_MEM(const SDValue & Op,
+                                               SelectionDAG &DAG,
+                                               MVT ElementType) {
+  SDLoc dl(Op);
+  SDValue Mask = Op.getOperand(4);
+  SDValue DataToTruncate = Op.getOperand(3);
+  SDValue Addr = Op.getOperand(2);
+  SDValue Chain = Op.getOperand(0);
+
+  EVT VT  = DataToTruncate.getValueType();
+  EVT SVT = EVT::getVectorVT(*DAG.getContext(),
+                             ElementType, VT.getVectorNumElements());
+
+  if (isAllOnes(Mask)) // return just a truncate store
+    return DAG.getTruncStore(Chain, dl, DataToTruncate, Addr,
+                             MachinePointerInfo(), SVT, false, false,
+                             SVT.getScalarSizeInBits()/8);
+
+  EVT MaskVT = EVT::getVectorVT(*DAG.getContext(),
+                                MVT::i1, VT.getVectorNumElements());
+  EVT BitcastVT = EVT::getVectorVT(*DAG.getContext(), MVT::i1,
+                                   Mask.getValueType().getSizeInBits());
+  // In case when MaskVT equals v2i1 or v4i1, low 2 or 4 elements
+  // are extracted by EXTRACT_SUBVECTOR.
+  SDValue VMask = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, MaskVT,
+                              DAG.getBitcast(BitcastVT, Mask),
+                              DAG.getIntPtrConstant(0, dl));
+
+  MachineMemOperand *MMO = DAG.getMachineFunction().
+    getMachineMemOperand(MachinePointerInfo(),
+                         MachineMemOperand::MOStore, SVT.getStoreSize(),
+                         SVT.getScalarSizeInBits()/8);
+
+  return DAG.getMaskedStore(Chain, dl, DataToTruncate, Addr,
+                            VMask, SVT, MMO, true);
+}
 
 static SDValue LowerINTRINSIC_W_CHAIN(SDValue Op, const X86Subtarget *Subtarget,
                                       SelectionDAG &DAG) {
   unsigned IntNo = cast<ConstantSDNode>(Op.getOperand(1))->getZExtValue();
 
   const IntrinsicData* IntrData = getIntrinsicWithChain(IntNo);
-  if (!IntrData)
+  if (!IntrData) {
+    if (IntNo == llvm::Intrinsic::x86_seh_restoreframe)
+      return LowerSEHRESTOREFRAME(Op, Subtarget, DAG);
     return SDValue();
+  }
 
   SDLoc dl(Op);
   switch(IntrData->Type) {
@@ -15417,8 +16226,8 @@ static SDValue LowerINTRINSIC_W_CHAIN(SDValue Op, const X86Subtarget *Subtarget,
     // If the value returned by RDRAND/RDSEED was valid (CF=1), return 1.
     // Otherwise return the value from Rand, which is always 0, casted to i32.
     SDValue Ops[] = { DAG.getZExtOrTrunc(Result, dl, Op->getValueType(1)),
-                      DAG.getConstant(1, Op->getValueType(1)),
-                      DAG.getConstant(X86::COND_B, MVT::i32),
+                      DAG.getConstant(1, dl, Op->getValueType(1)),
+                      DAG.getConstant(X86::COND_B, dl, MVT::i32),
                       SDValue(Result.getNode(), 1) };
     SDValue isValid = DAG.getNode(X86ISD::CMOV, dl,
                                   DAG.getVTList(Op->getValueType(1), MVT::Glue),
@@ -15436,8 +16245,8 @@ static SDValue LowerINTRINSIC_W_CHAIN(SDValue Op, const X86Subtarget *Subtarget,
     SDValue Index = Op.getOperand(4);
     SDValue Mask  = Op.getOperand(5);
     SDValue Scale = Op.getOperand(6);
-    return getGatherNode(IntrData->Opc0, Op, DAG, Src, Mask, Base, Index, Scale, Chain,
-                          Subtarget);
+    return getGatherNode(IntrData->Opc0, Op, DAG, Src, Mask, Base, Index, Scale,
+                         Chain, Subtarget);
   }
   case SCATTER: {
   //scatter(base, mask, index, v1, scale);
@@ -15447,7 +16256,8 @@ static SDValue LowerINTRINSIC_W_CHAIN(SDValue Op, const X86Subtarget *Subtarget,
     SDValue Index = Op.getOperand(4);
     SDValue Src   = Op.getOperand(5);
     SDValue Scale = Op.getOperand(6);
-    return getScatterNode(IntrData->Opc0, Op, DAG, Src, Mask, Base, Index, Scale, Chain);
+    return getScatterNode(IntrData->Opc0, Op, DAG, Src, Mask, Base, Index,
+                          Scale, Chain);
   }
   case PREFETCH: {
     SDValue Hint = Op.getOperand(6);
@@ -15464,7 +16274,8 @@ static SDValue LowerINTRINSIC_W_CHAIN(SDValue Op, const X86Subtarget *Subtarget,
   // Read Time Stamp Counter (RDTSC) and Processor ID (RDTSCP).
   case RDTSC: {
     SmallVector<SDValue, 2> Results;
-    getReadTimeStampCounter(Op.getNode(), dl, IntrData->Opc0, DAG, Subtarget, Results);
+    getReadTimeStampCounter(Op.getNode(), dl, IntrData->Opc0, DAG, Subtarget,
+                            Results);
     return DAG.getMergeValues(Results, dl);
   }
   // Read Performance Monitoring Counters.
@@ -15478,7 +16289,7 @@ static SDValue LowerINTRINSIC_W_CHAIN(SDValue Op, const X86Subtarget *Subtarget,
     SDVTList VTs = DAG.getVTList(Op->getValueType(0), MVT::Other);
     SDValue InTrans = DAG.getNode(IntrData->Opc0, dl, VTs, Op.getOperand(0));
     SDValue SetCC = DAG.getNode(X86ISD::SETCC, dl, MVT::i8,
-                                DAG.getConstant(X86::COND_NE, MVT::i8),
+                                DAG.getConstant(X86::COND_NE, dl, MVT::i8),
                                 InTrans);
     SDValue Ret = DAG.getNode(ISD::ZERO_EXTEND, dl, Op->getValueType(0), SetCC);
     return DAG.getNode(ISD::MERGE_VALUES, dl, Op->getVTList(),
@@ -15490,14 +16301,14 @@ static SDValue LowerINTRINSIC_W_CHAIN(SDValue Op, const X86Subtarget *Subtarget,
     SDVTList CFVTs = DAG.getVTList(Op->getValueType(0), MVT::Other);
     SDVTList VTs = DAG.getVTList(Op.getOperand(3)->getValueType(0), MVT::Other);
     SDValue GenCF = DAG.getNode(X86ISD::ADD, dl, CFVTs, Op.getOperand(2),
-                                DAG.getConstant(-1, MVT::i8));
+                                DAG.getConstant(-1, dl, MVT::i8));
     SDValue Res = DAG.getNode(IntrData->Opc0, dl, VTs, Op.getOperand(3),
                               Op.getOperand(4), GenCF.getValue(1));
     SDValue Store = DAG.getStore(Op.getOperand(0), dl, Res.getValue(0),
                                  Op.getOperand(5), MachinePointerInfo(),
                                  false, false, 0);
     SDValue SetCC = DAG.getNode(X86ISD::SETCC, dl, MVT::i8,
-                                DAG.getConstant(X86::COND_B, MVT::i8),
+                                DAG.getConstant(X86::COND_B, dl, MVT::i8),
                                 Res.getValue(1));
     Results.push_back(SetCC);
     Results.push_back(Store);
@@ -15510,49 +16321,44 @@ static SDValue LowerINTRINSIC_W_CHAIN(SDValue Op, const X86Subtarget *Subtarget,
     SDValue Addr = Op.getOperand(2);
     SDValue Chain = Op.getOperand(0);
 
+    EVT VT = DataToCompress.getValueType();
     if (isAllOnes(Mask)) // return just a store
       return DAG.getStore(Chain, dl, DataToCompress, Addr,
-                          MachinePointerInfo(), false, false, 0);
-
-    EVT VT = DataToCompress.getValueType();
-    EVT MaskVT = EVT::getVectorVT(*DAG.getContext(), MVT::i1,
-                                  VT.getVectorNumElements());
-    EVT BitcastVT = EVT::getVectorVT(*DAG.getContext(), MVT::i1,
-                                     Mask.getValueType().getSizeInBits());
-    SDValue VMask = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, MaskVT,
-                                DAG.getNode(ISD::BITCAST, dl, BitcastVT, Mask),
-                                DAG.getIntPtrConstant(0));
+                          MachinePointerInfo(), false, false,
+                          VT.getScalarSizeInBits()/8);
 
-    SDValue Compressed =  DAG.getNode(IntrData->Opc0, dl, VT, VMask,
-                                      DataToCompress, DAG.getUNDEF(VT));
+    SDValue Compressed =
+      getVectorMaskingNode(DAG.getNode(IntrData->Opc0, dl, VT, DataToCompress),
+                           Mask, DAG.getUNDEF(VT), Subtarget, DAG);
     return DAG.getStore(Chain, dl, Compressed, Addr,
-                        MachinePointerInfo(), false, false, 0);
-  }
+                        MachinePointerInfo(), false, false,
+                        VT.getScalarSizeInBits()/8);
+  }
+  case TRUNCATE_TO_MEM_VI8:
+    return LowerINTRINSIC_TRUNCATE_TO_MEM(Op, DAG, MVT::i8);
+  case TRUNCATE_TO_MEM_VI16:
+    return LowerINTRINSIC_TRUNCATE_TO_MEM(Op, DAG, MVT::i16);
+  case TRUNCATE_TO_MEM_VI32:
+    return LowerINTRINSIC_TRUNCATE_TO_MEM(Op, DAG, MVT::i32);
   case EXPAND_FROM_MEM: {
     SDLoc dl(Op);
     SDValue Mask = Op.getOperand(4);
-    SDValue PathThru = Op.getOperand(3);
+    SDValue PassThru = Op.getOperand(3);
     SDValue Addr = Op.getOperand(2);
     SDValue Chain = Op.getOperand(0);
     EVT VT = Op.getValueType();
 
     if (isAllOnes(Mask)) // return just a load
       return DAG.getLoad(VT, dl, Chain, Addr, MachinePointerInfo(), false, false,
-                         false, 0);
-    EVT MaskVT = EVT::getVectorVT(*DAG.getContext(), MVT::i1,
-                                  VT.getVectorNumElements());
-    EVT BitcastVT = EVT::getVectorVT(*DAG.getContext(), MVT::i1,
-                                     Mask.getValueType().getSizeInBits());
-    SDValue VMask = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, MaskVT,
-                                DAG.getNode(ISD::BITCAST, dl, BitcastVT, Mask),
-                                DAG.getIntPtrConstant(0));
+                         false, VT.getScalarSizeInBits()/8);
 
     SDValue DataToExpand = DAG.getLoad(VT, dl, Chain, Addr, MachinePointerInfo(),
-                                   false, false, false, 0);
+                                       false, false, false,
+                                       VT.getScalarSizeInBits()/8);
 
     SDValue Results[] = {
-        DAG.getNode(IntrData->Opc0, dl, VT, VMask, DataToExpand, PathThru),
-        Chain};
+      getVectorMaskingNode(DAG.getNode(IntrData->Opc0, dl, VT, DataToExpand),
+                           Mask, PassThru, Subtarget, DAG), Chain};
     return DAG.getMergeValues(Results, dl);
   }
   }
@@ -15568,12 +16374,12 @@ SDValue X86TargetLowering::LowerRETURNADDR(SDValue Op,
 
   unsigned Depth = cast<ConstantSDNode>(Op.getOperand(0))->getZExtValue();
   SDLoc dl(Op);
-  EVT PtrVT = getPointerTy();
+  EVT PtrVT = getPointerTy(DAG.getDataLayout());
 
   if (Depth > 0) {
     SDValue FrameAddr = LowerFRAMEADDR(Op, DAG);
     const X86RegisterInfo *RegInfo = Subtarget->getRegisterInfo();
-    SDValue Offset = DAG.getConstant(RegInfo->getSlotSize(), PtrVT);
+    SDValue Offset = DAG.getConstant(RegInfo->getSlotSize(), dl, PtrVT);
     return DAG.getLoad(PtrVT, dl, DAG.getEntryNode(),
                        DAG.getNode(ISD::ADD, dl, PtrVT,
                                    FrameAddr, Offset),
@@ -15604,7 +16410,7 @@ SDValue X86TargetLowering::LowerFRAMEADDR(SDValue Op, SelectionDAG &DAG) const {
       // Set up a frame object for the return address.
       unsigned SlotSize = RegInfo->getSlotSize();
       FrameAddrIndex = MF.getFrameInfo()->CreateFixedObject(
-          SlotSize, /*Offset=*/INT64_MIN, /*IsImmutable=*/false);
+          SlotSize, /*Offset=*/0, /*IsImmutable=*/false);
       FuncInfo->setFAIndex(FrameAddrIndex);
     }
     return DAG.getFrameIndex(FrameAddrIndex, VT);
@@ -15627,21 +16433,43 @@ SDValue X86TargetLowering::LowerFRAMEADDR(SDValue Op, SelectionDAG &DAG) const {
 
 // FIXME? Maybe this could be a TableGen attribute on some registers and
 // this table could be generated automatically from RegInfo.
-unsigned X86TargetLowering::getRegisterByName(const char* RegName,
-                                              EVT VT) const {
+unsigned X86TargetLowering::getRegisterByName(const char* RegName, EVT VT,
+                                              SelectionDAG &DAG) const {
+  const TargetFrameLowering &TFI = *Subtarget->getFrameLowering();
+  const MachineFunction &MF = DAG.getMachineFunction();
+
   unsigned Reg = StringSwitch<unsigned>(RegName)
                        .Case("esp", X86::ESP)
                        .Case("rsp", X86::RSP)
+                       .Case("ebp", X86::EBP)
+                       .Case("rbp", X86::RBP)
                        .Default(0);
+
+  if (Reg == X86::EBP || Reg == X86::RBP) {
+    if (!TFI.hasFP(MF))
+      report_fatal_error("register " + StringRef(RegName) +
+                         " is allocatable: function has no frame pointer");
+#ifndef NDEBUG
+    else {
+      const X86RegisterInfo *RegInfo = Subtarget->getRegisterInfo();
+      unsigned FrameReg =
+          RegInfo->getPtrSizedFrameRegister(DAG.getMachineFunction());
+      assert((FrameReg == X86::EBP || FrameReg == X86::RBP) &&
+             "Invalid Frame Register!");
+    }
+#endif
+  }
+
   if (Reg)
     return Reg;
+
   report_fatal_error("Invalid register name global variable");
 }
 
 SDValue X86TargetLowering::LowerFRAME_TO_ARGS_OFFSET(SDValue Op,
                                                      SelectionDAG &DAG) const {
   const X86RegisterInfo *RegInfo = Subtarget->getRegisterInfo();
-  return DAG.getIntPtrConstant(2 * RegInfo->getSlotSize());
+  return DAG.getIntPtrConstant(2 * RegInfo->getSlotSize(), SDLoc(Op));
 }
 
 SDValue X86TargetLowering::LowerEH_RETURN(SDValue Op, SelectionDAG &DAG) const {
@@ -15650,25 +16478,18 @@ SDValue X86TargetLowering::LowerEH_RETURN(SDValue Op, SelectionDAG &DAG) const {
   SDValue Handler   = Op.getOperand(2);
   SDLoc dl      (Op);
 
-  // @LOCALMOD-START
-  bool Has64BitPointers = Subtarget->has64BitPointers();
-  EVT PtrVT = getPointerTy();
+  EVT PtrVT = getPointerTy(DAG.getDataLayout());
   const X86RegisterInfo *RegInfo = Subtarget->getRegisterInfo();
-  unsigned FrameReg;
-  if (Subtarget->isTargetNaCl()) {
-    FrameReg = Subtarget->has64BitPointers() ? X86::RBP : X86::EBP;
-  } else {
-    FrameReg = RegInfo->getFrameRegister(DAG.getMachineFunction());
-  }
+  unsigned FrameReg = RegInfo->getFrameRegister(DAG.getMachineFunction());
   assert(((FrameReg == X86::RBP && PtrVT == MVT::i64) ||
           (FrameReg == X86::EBP && PtrVT == MVT::i32)) &&
          "Invalid Frame Register!");
   SDValue Frame = DAG.getCopyFromReg(DAG.getEntryNode(), dl, FrameReg, PtrVT);
-  unsigned StoreAddrReg = Has64BitPointers ? X86::RCX : X86::ECX;
-  // @LOCALMOD-END
+  unsigned StoreAddrReg = (PtrVT == MVT::i64) ? X86::RCX : X86::ECX;
 
   SDValue StoreAddr = DAG.getNode(ISD::ADD, dl, PtrVT, Frame,
-                                 DAG.getIntPtrConstant(RegInfo->getSlotSize()));
+                                 DAG.getIntPtrConstant(RegInfo->getSlotSize(),
+                                                       dl));
   StoreAddr = DAG.getNode(ISD::ADD, dl, PtrVT, StoreAddr, Offset);
   Chain = DAG.getStore(Chain, dl, Handler, StoreAddr, MachinePointerInfo(),
                        false, false, 0);
@@ -15723,12 +16544,12 @@ SDValue X86TargetLowering::LowerINIT_TRAMPOLINE(SDValue Op,
     // Load the pointer to the nested function into R11.
     unsigned OpCode = ((MOV64ri | N86R11) << 8) | REX_WB; // movabsq r11
     SDValue Addr = Trmp;
-    OutChains[0] = DAG.getStore(Root, dl, DAG.getConstant(OpCode, MVT::i16),
+    OutChains[0] = DAG.getStore(Root, dl, DAG.getConstant(OpCode, dl, MVT::i16),
                                 Addr, MachinePointerInfo(TrmpAddr),
                                 false, false, 0);
 
     Addr = DAG.getNode(ISD::ADD, dl, MVT::i64, Trmp,
-                       DAG.getConstant(2, MVT::i64));
+                       DAG.getConstant(2, dl, MVT::i64));
     OutChains[1] = DAG.getStore(Root, dl, FPtr, Addr,
                                 MachinePointerInfo(TrmpAddr, 2),
                                 false, false, 2);
@@ -15737,13 +16558,13 @@ SDValue X86TargetLowering::LowerINIT_TRAMPOLINE(SDValue Op,
     // R10 is specified in X86CallingConv.td
     OpCode = ((MOV64ri | N86R10) << 8) | REX_WB; // movabsq r10
     Addr = DAG.getNode(ISD::ADD, dl, MVT::i64, Trmp,
-                       DAG.getConstant(10, MVT::i64));
-    OutChains[2] = DAG.getStore(Root, dl, DAG.getConstant(OpCode, MVT::i16),
+                       DAG.getConstant(10, dl, MVT::i64));
+    OutChains[2] = DAG.getStore(Root, dl, DAG.getConstant(OpCode, dl, MVT::i16),
                                 Addr, MachinePointerInfo(TrmpAddr, 10),
                                 false, false, 0);
 
     Addr = DAG.getNode(ISD::ADD, dl, MVT::i64, Trmp,
-                       DAG.getConstant(12, MVT::i64));
+                       DAG.getConstant(12, dl, MVT::i64));
     OutChains[3] = DAG.getStore(Root, dl, Nest, Addr,
                                 MachinePointerInfo(TrmpAddr, 12),
                                 false, false, 2);
@@ -15751,16 +16572,16 @@ SDValue X86TargetLowering::LowerINIT_TRAMPOLINE(SDValue Op,
     // Jump to the nested function.
     OpCode = (JMP64r << 8) | REX_WB; // jmpq *...
     Addr = DAG.getNode(ISD::ADD, dl, MVT::i64, Trmp,
-                       DAG.getConstant(20, MVT::i64));
-    OutChains[4] = DAG.getStore(Root, dl, DAG.getConstant(OpCode, MVT::i16),
+                       DAG.getConstant(20, dl, MVT::i64));
+    OutChains[4] = DAG.getStore(Root, dl, DAG.getConstant(OpCode, dl, MVT::i16),
                                 Addr, MachinePointerInfo(TrmpAddr, 20),
                                 false, false, 0);
 
     unsigned char ModRM = N86R11 | (4 << 3) | (3 << 6); // ...r11
     Addr = DAG.getNode(ISD::ADD, dl, MVT::i64, Trmp,
-                       DAG.getConstant(22, MVT::i64));
-    OutChains[5] = DAG.getStore(Root, dl, DAG.getConstant(ModRM, MVT::i8), Addr,
-                                MachinePointerInfo(TrmpAddr, 22),
+                       DAG.getConstant(22, dl, MVT::i64));
+    OutChains[5] = DAG.getStore(Root, dl, DAG.getConstant(ModRM, dl, MVT::i8),
+                                Addr, MachinePointerInfo(TrmpAddr, 22),
                                 false, false, 0);
 
     return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, OutChains);
@@ -15789,9 +16610,11 @@ SDValue X86TargetLowering::LowerINIT_TRAMPOLINE(SDValue Op,
 
         for (FunctionType::param_iterator I = FTy->param_begin(),
              E = FTy->param_end(); I != E; ++I, ++Idx)
-          if (Attrs.hasAttribute(Idx, Attribute::InReg))
+          if (Attrs.hasAttribute(Idx, Attribute::InReg)) {
+            auto &DL = DAG.getDataLayout();
             // FIXME: should only count parameters that are lowered to integers.
-            InRegCount += (TD->getTypeSizeInBits(*I) + 31) / 32;
+            InRegCount += (DL.getTypeSizeInBits(*I) + 31) / 32;
+          }
 
         if (InRegCount > 2) {
           report_fatal_error("Nest register in use - reduce number of inreg"
@@ -15813,32 +16636,32 @@ SDValue X86TargetLowering::LowerINIT_TRAMPOLINE(SDValue Op,
     SDValue Addr, Disp;
 
     Addr = DAG.getNode(ISD::ADD, dl, MVT::i32, Trmp,
-                       DAG.getConstant(10, MVT::i32));
+                       DAG.getConstant(10, dl, MVT::i32));
     Disp = DAG.getNode(ISD::SUB, dl, MVT::i32, FPtr, Addr);
 
     // This is storing the opcode for MOV32ri.
     const unsigned char MOV32ri = 0xB8; // X86::MOV32ri's opcode byte.
     const unsigned char N86Reg = TRI->getEncodingValue(NestReg) & 0x7;
     OutChains[0] = DAG.getStore(Root, dl,
-                                DAG.getConstant(MOV32ri|N86Reg, MVT::i8),
+                                DAG.getConstant(MOV32ri|N86Reg, dl, MVT::i8),
                                 Trmp, MachinePointerInfo(TrmpAddr),
                                 false, false, 0);
 
     Addr = DAG.getNode(ISD::ADD, dl, MVT::i32, Trmp,
-                       DAG.getConstant(1, MVT::i32));
+                       DAG.getConstant(1, dl, MVT::i32));
     OutChains[1] = DAG.getStore(Root, dl, Nest, Addr,
                                 MachinePointerInfo(TrmpAddr, 1),
                                 false, false, 1);
 
     const unsigned char JMP = 0xE9; // jmp <32bit dst> opcode.
     Addr = DAG.getNode(ISD::ADD, dl, MVT::i32, Trmp,
-                       DAG.getConstant(5, MVT::i32));
-    OutChains[2] = DAG.getStore(Root, dl, DAG.getConstant(JMP, MVT::i8), Addr,
-                                MachinePointerInfo(TrmpAddr, 5),
+                       DAG.getConstant(5, dl, MVT::i32));
+    OutChains[2] = DAG.getStore(Root, dl, DAG.getConstant(JMP, dl, MVT::i8),
+                                Addr, MachinePointerInfo(TrmpAddr, 5),
                                 false, false, 1);
 
     Addr = DAG.getNode(ISD::ADD, dl, MVT::i32, Trmp,
-                       DAG.getConstant(6, MVT::i32));
+                       DAG.getConstant(6, dl, MVT::i32));
     OutChains[3] = DAG.getStore(Root, dl, Disp, Addr,
                                 MachinePointerInfo(TrmpAddr, 6),
                                 false, false, 1);
@@ -15876,7 +16699,8 @@ SDValue X86TargetLowering::LowerFLT_ROUNDS_(SDValue Op,
 
   // Save FP Control Word to stack slot
   int SSFI = MF.getFrameInfo()->CreateStackObject(2, StackAlignment, false);
-  SDValue StackSlot = DAG.getFrameIndex(SSFI, getPointerTy());
+  SDValue StackSlot =
+      DAG.getFrameIndex(SSFI, getPointerTy(DAG.getDataLayout()));
 
   MachineMemOperand *MMO =
    MF.getMachineMemOperand(MachinePointerInfo::getFixedStack(SSFI),
@@ -15895,20 +16719,20 @@ SDValue X86TargetLowering::LowerFLT_ROUNDS_(SDValue Op,
   SDValue CWD1 =
     DAG.getNode(ISD::SRL, DL, MVT::i16,
                 DAG.getNode(ISD::AND, DL, MVT::i16,
-                            CWD, DAG.getConstant(0x800, MVT::i16)),
-                DAG.getConstant(11, MVT::i8));
+                            CWD, DAG.getConstant(0x800, DL, MVT::i16)),
+                DAG.getConstant(11, DL, MVT::i8));
   SDValue CWD2 =
     DAG.getNode(ISD::SRL, DL, MVT::i16,
                 DAG.getNode(ISD::AND, DL, MVT::i16,
-                            CWD, DAG.getConstant(0x400, MVT::i16)),
-                DAG.getConstant(9, MVT::i8));
+                            CWD, DAG.getConstant(0x400, DL, MVT::i16)),
+                DAG.getConstant(9, DL, MVT::i8));
 
   SDValue RetVal =
     DAG.getNode(ISD::AND, DL, MVT::i16,
                 DAG.getNode(ISD::ADD, DL, MVT::i16,
                             DAG.getNode(ISD::OR, DL, MVT::i16, CWD1, CWD2),
-                            DAG.getConstant(1, MVT::i16)),
-                DAG.getConstant(3, MVT::i16));
+                            DAG.getConstant(1, DL, MVT::i16)),
+                DAG.getConstant(3, DL, MVT::i16));
 
   return DAG.getNode((VT.getSizeInBits() < 16 ?
                       ISD::TRUNCATE : ISD::ZERO_EXTEND), DL, VT, RetVal);
@@ -15934,14 +16758,15 @@ static SDValue LowerCTLZ(SDValue Op, SelectionDAG &DAG) {
   // If src is zero (i.e. bsr sets ZF), returns NumBits.
   SDValue Ops[] = {
     Op,
-    DAG.getConstant(NumBits+NumBits-1, OpVT),
-    DAG.getConstant(X86::COND_E, MVT::i8),
+    DAG.getConstant(NumBits + NumBits - 1, dl, OpVT),
+    DAG.getConstant(X86::COND_E, dl, MVT::i8),
     Op.getValue(1)
   };
   Op = DAG.getNode(X86ISD::CMOV, dl, OpVT, Ops);
 
   // Finally xor with NumBits-1.
-  Op = DAG.getNode(ISD::XOR, dl, OpVT, Op, DAG.getConstant(NumBits-1, OpVT));
+  Op = DAG.getNode(ISD::XOR, dl, OpVT, Op,
+                   DAG.getConstant(NumBits - 1, dl, OpVT));
 
   if (VT == MVT::i8)
     Op = DAG.getNode(ISD::TRUNCATE, dl, MVT::i8, Op);
@@ -15966,7 +16791,8 @@ static SDValue LowerCTLZ_ZERO_UNDEF(SDValue Op, SelectionDAG &DAG) {
   Op = DAG.getNode(X86ISD::BSR, dl, VTs, Op);
 
   // And xor with NumBits-1.
-  Op = DAG.getNode(ISD::XOR, dl, OpVT, Op, DAG.getConstant(NumBits-1, OpVT));
+  Op = DAG.getNode(ISD::XOR, dl, OpVT, Op,
+                   DAG.getConstant(NumBits - 1, dl, OpVT));
 
   if (VT == MVT::i8)
     Op = DAG.getNode(ISD::TRUNCATE, dl, MVT::i8, Op);
@@ -15986,8 +16812,8 @@ static SDValue LowerCTTZ(SDValue Op, SelectionDAG &DAG) {
   // If src is zero (i.e. bsf sets ZF), returns NumBits.
   SDValue Ops[] = {
     Op,
-    DAG.getConstant(NumBits, VT),
-    DAG.getConstant(X86::COND_E, MVT::i8),
+    DAG.getConstant(NumBits, dl, VT),
+    DAG.getConstant(X86::COND_E, dl, MVT::i8),
     Op.getValue(1)
   };
   return DAG.getNode(X86ISD::CMOV, dl, VT, Ops);
@@ -16023,6 +16849,9 @@ static SDValue Lower256IntArith(SDValue Op, SelectionDAG &DAG) {
 }
 
 static SDValue LowerADD(SDValue Op, SelectionDAG &DAG) {
+  if (Op.getValueType() == MVT::i1)
+    return DAG.getNode(ISD::XOR, SDLoc(Op), Op.getValueType(),
+                       Op.getOperand(0), Op.getOperand(1));
   assert(Op.getSimpleValueType().is256BitVector() &&
          Op.getSimpleValueType().isInteger() &&
          "Only handle AVX 256-bit vector integer operation");
@@ -16030,6 +16859,9 @@ static SDValue LowerADD(SDValue Op, SelectionDAG &DAG) {
 }
 
 static SDValue LowerSUB(SDValue Op, SelectionDAG &DAG) {
+  if (Op.getValueType() == MVT::i1)
+    return DAG.getNode(ISD::XOR, SDLoc(Op), Op.getValueType(),
+                       Op.getOperand(0), Op.getOperand(1));
   assert(Op.getSimpleValueType().is256BitVector() &&
          Op.getSimpleValueType().isInteger() &&
          "Only handle AVX 256-bit vector integer operation");
@@ -16041,6 +16873,9 @@ static SDValue LowerMUL(SDValue Op, const X86Subtarget *Subtarget,
   SDLoc dl(Op);
   MVT VT = Op.getSimpleValueType();
 
+  if (VT == MVT::i1)
+    return DAG.getNode(ISD::AND, dl, VT, Op.getOperand(0), Op.getOperand(1));
+
   // Decompose 256-bit ops into smaller 128-bit ops.
   if (VT.is256BitVector() && !Subtarget->hasInt256())
     return Lower256IntArith(Op, DAG);
@@ -16048,6 +16883,79 @@ static SDValue LowerMUL(SDValue Op, const X86Subtarget *Subtarget,
   SDValue A = Op.getOperand(0);
   SDValue B = Op.getOperand(1);
 
+  // Lower v16i8/v32i8 mul as promotion to v8i16/v16i16 vector
+  // pairs, multiply and truncate.
+  if (VT == MVT::v16i8 || VT == MVT::v32i8) {
+    if (Subtarget->hasInt256()) {
+      if (VT == MVT::v32i8) {
+        MVT SubVT = MVT::getVectorVT(MVT::i8, VT.getVectorNumElements() / 2);
+        SDValue Lo = DAG.getIntPtrConstant(0, dl);
+        SDValue Hi = DAG.getIntPtrConstant(VT.getVectorNumElements() / 2, dl);
+        SDValue ALo = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, SubVT, A, Lo);
+        SDValue BLo = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, SubVT, B, Lo);
+        SDValue AHi = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, SubVT, A, Hi);
+        SDValue BHi = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, SubVT, B, Hi);
+        return DAG.getNode(ISD::CONCAT_VECTORS, dl, VT,
+                           DAG.getNode(ISD::MUL, dl, SubVT, ALo, BLo),
+                           DAG.getNode(ISD::MUL, dl, SubVT, AHi, BHi));
+      }
+
+      MVT ExVT = MVT::getVectorVT(MVT::i16, VT.getVectorNumElements());
+      return DAG.getNode(
+          ISD::TRUNCATE, dl, VT,
+          DAG.getNode(ISD::MUL, dl, ExVT,
+                      DAG.getNode(ISD::SIGN_EXTEND, dl, ExVT, A),
+                      DAG.getNode(ISD::SIGN_EXTEND, dl, ExVT, B)));
+    }
+
+    assert(VT == MVT::v16i8 &&
+           "Pre-AVX2 support only supports v16i8 multiplication");
+    MVT ExVT = MVT::v8i16;
+
+    // Extract the lo parts and sign extend to i16
+    SDValue ALo, BLo;
+    if (Subtarget->hasSSE41()) {
+      ALo = DAG.getNode(X86ISD::VSEXT, dl, ExVT, A);
+      BLo = DAG.getNode(X86ISD::VSEXT, dl, ExVT, B);
+    } else {
+      const int ShufMask[] = {-1, 0, -1, 1, -1, 2, -1, 3,
+                              -1, 4, -1, 5, -1, 6, -1, 7};
+      ALo = DAG.getVectorShuffle(VT, dl, A, A, ShufMask);
+      BLo = DAG.getVectorShuffle(VT, dl, B, B, ShufMask);
+      ALo = DAG.getBitcast(ExVT, ALo);
+      BLo = DAG.getBitcast(ExVT, BLo);
+      ALo = DAG.getNode(ISD::SRA, dl, ExVT, ALo, DAG.getConstant(8, dl, ExVT));
+      BLo = DAG.getNode(ISD::SRA, dl, ExVT, BLo, DAG.getConstant(8, dl, ExVT));
+    }
+
+    // Extract the hi parts and sign extend to i16
+    SDValue AHi, BHi;
+    if (Subtarget->hasSSE41()) {
+      const int ShufMask[] = {8,  9,  10, 11, 12, 13, 14, 15,
+                              -1, -1, -1, -1, -1, -1, -1, -1};
+      AHi = DAG.getVectorShuffle(VT, dl, A, A, ShufMask);
+      BHi = DAG.getVectorShuffle(VT, dl, B, B, ShufMask);
+      AHi = DAG.getNode(X86ISD::VSEXT, dl, ExVT, AHi);
+      BHi = DAG.getNode(X86ISD::VSEXT, dl, ExVT, BHi);
+    } else {
+      const int ShufMask[] = {-1, 8,  -1, 9,  -1, 10, -1, 11,
+                              -1, 12, -1, 13, -1, 14, -1, 15};
+      AHi = DAG.getVectorShuffle(VT, dl, A, A, ShufMask);
+      BHi = DAG.getVectorShuffle(VT, dl, B, B, ShufMask);
+      AHi = DAG.getBitcast(ExVT, AHi);
+      BHi = DAG.getBitcast(ExVT, BHi);
+      AHi = DAG.getNode(ISD::SRA, dl, ExVT, AHi, DAG.getConstant(8, dl, ExVT));
+      BHi = DAG.getNode(ISD::SRA, dl, ExVT, BHi, DAG.getConstant(8, dl, ExVT));
+    }
+
+    // Multiply, mask the lower 8bits of the lo/hi results and pack
+    SDValue RLo = DAG.getNode(ISD::MUL, dl, ExVT, ALo, BLo);
+    SDValue RHi = DAG.getNode(ISD::MUL, dl, ExVT, AHi, BHi);
+    RLo = DAG.getNode(ISD::AND, dl, ExVT, RLo, DAG.getConstant(255, dl, ExVT));
+    RHi = DAG.getNode(ISD::AND, dl, ExVT, RHi, DAG.getConstant(255, dl, ExVT));
+    return DAG.getNode(X86ISD::PACKUS, dl, VT, RLo, RHi);
+  }
+
   // Lower v4i32 mul as 2x shuffle, 2x pmuludq, 2x shuffle.
   if (VT == MVT::v4i32) {
     assert(Subtarget->hasSSE2() && !Subtarget->hasSSE41() &&
@@ -16063,8 +16971,8 @@ static SDValue LowerMUL(SDValue Op, const X86Subtarget *Subtarget,
     // Now multiply odd parts.
     SDValue Odds = DAG.getNode(X86ISD::PMULUDQ, dl, MVT::v2i64, Aodds, Bodds);
 
-    Evens = DAG.getNode(ISD::BITCAST, dl, VT, Evens);
-    Odds = DAG.getNode(ISD::BITCAST, dl, VT, Odds);
+    Evens = DAG.getBitcast(VT, Evens);
+    Odds = DAG.getBitcast(VT, Odds);
 
     // Merge the two vectors back together with a shuffle. This expands into 2
     // shuffles.
@@ -16089,20 +16997,26 @@ static SDValue LowerMUL(SDValue Op, const X86Subtarget *Subtarget,
   SDValue Ahi = getTargetVShiftByConstNode(X86ISD::VSRLI, dl, VT, A, 32, DAG);
   SDValue Bhi = getTargetVShiftByConstNode(X86ISD::VSRLI, dl, VT, B, 32, DAG);
 
+  SDValue AhiBlo = Ahi;
+  SDValue AloBhi = Bhi;
   // Bit cast to 32-bit vectors for MULUDQ
   EVT MulVT = (VT == MVT::v2i64) ? MVT::v4i32 :
                                   (VT == MVT::v4i64) ? MVT::v8i32 : MVT::v16i32;
-  A = DAG.getNode(ISD::BITCAST, dl, MulVT, A);
-  B = DAG.getNode(ISD::BITCAST, dl, MulVT, B);
-  Ahi = DAG.getNode(ISD::BITCAST, dl, MulVT, Ahi);
-  Bhi = DAG.getNode(ISD::BITCAST, dl, MulVT, Bhi);
+  A = DAG.getBitcast(MulVT, A);
+  B = DAG.getBitcast(MulVT, B);
+  Ahi = DAG.getBitcast(MulVT, Ahi);
+  Bhi = DAG.getBitcast(MulVT, Bhi);
 
   SDValue AloBlo = DAG.getNode(X86ISD::PMULUDQ, dl, VT, A, B);
-  SDValue AloBhi = DAG.getNode(X86ISD::PMULUDQ, dl, VT, A, Bhi);
-  SDValue AhiBlo = DAG.getNode(X86ISD::PMULUDQ, dl, VT, Ahi, B);
-
-  AloBhi = getTargetVShiftByConstNode(X86ISD::VSHLI, dl, VT, AloBhi, 32, DAG);
-  AhiBlo = getTargetVShiftByConstNode(X86ISD::VSHLI, dl, VT, AhiBlo, 32, DAG);
+  // After shifting right const values the result may be all-zero.
+  if (!ISD::isBuildVectorAllZeros(Ahi.getNode())) {
+    AhiBlo = DAG.getNode(X86ISD::PMULUDQ, dl, VT, Ahi, B);
+    AhiBlo = getTargetVShiftByConstNode(X86ISD::VSHLI, dl, VT, AhiBlo, 32, DAG);
+  }
+  if (!ISD::isBuildVectorAllZeros(Bhi.getNode())) {
+    AloBhi = DAG.getNode(X86ISD::PMULUDQ, dl, VT, A, Bhi);
+    AloBhi = getTargetVShiftByConstNode(X86ISD::VSHLI, dl, VT, AloBhi, 32, DAG);
+  }
 
   SDValue Res = DAG.getNode(ISD::ADD, dl, VT, AloBlo, AloBhi);
   return DAG.getNode(ISD::ADD, dl, VT, Res, AhiBlo);
@@ -16147,7 +17061,7 @@ SDValue X86TargetLowering::LowerWin64_i128OP(SDValue Op, SelectionDAG &DAG) cons
   }
 
   SDValue Callee = DAG.getExternalSymbol(getLibcallName(LC),
-                                         getPointerTy());
+                                         getPointerTy(DAG.getDataLayout()));
 
   TargetLowering::CallLoweringInfo CLI(DAG);
   CLI.setDebugLoc(dl).setChain(InChain)
@@ -16157,7 +17071,7 @@ SDValue X86TargetLowering::LowerWin64_i128OP(SDValue Op, SelectionDAG &DAG) cons
     .setInRegister().setSExtResult(isSigned).setZExtResult(!isSigned);
 
   std::pair<SDValue, SDValue> CallInfo = LowerCallTo(CLI);
-  return DAG.getNode(ISD::BITCAST, dl, VT, CallInfo.first);
+  return DAG.getBitcast(VT, CallInfo.first);
 }
 
 static SDValue LowerMUL_LOHI(SDValue Op, const X86Subtarget *Subtarget,
@@ -16195,12 +17109,10 @@ static SDValue LowerMUL_LOHI(SDValue Op, const X86Subtarget *Subtarget,
       (!IsSigned || !Subtarget->hasSSE41()) ? X86ISD::PMULUDQ : X86ISD::PMULDQ;
   // PMULUDQ <4 x i32> <a|b|c|d>, <4 x i32> <e|f|g|h>
   // => <2 x i64> <ae|cg>
-  SDValue Mul1 = DAG.getNode(ISD::BITCAST, dl, VT,
-                             DAG.getNode(Opcode, dl, MulVT, Op0, Op1));
+  SDValue Mul1 = DAG.getBitcast(VT, DAG.getNode(Opcode, dl, MulVT, Op0, Op1));
   // PMULUDQ <4 x i32> <b|undef|d|undef>, <4 x i32> <f|undef|h|undef>
   // => <2 x i64> <bf|dh>
-  SDValue Mul2 = DAG.getNode(ISD::BITCAST, dl, VT,
-                             DAG.getNode(Opcode, dl, MulVT, Odd0, Odd1));
+  SDValue Mul2 = DAG.getBitcast(VT, DAG.getNode(Opcode, dl, MulVT, Odd0, Odd1));
 
   // Shuffle it back into the right order.
   SDValue Highs, Lows;
@@ -16219,8 +17131,9 @@ static SDValue LowerMUL_LOHI(SDValue Op, const X86Subtarget *Subtarget,
   // If we have a signed multiply but no PMULDQ fix up the high parts of a
   // unsigned multiply.
   if (IsSigned && !Subtarget->hasSSE41()) {
-    SDValue ShAmt =
-        DAG.getConstant(31, DAG.getTargetLoweringInfo().getShiftAmountTy(VT));
+    SDValue ShAmt = DAG.getConstant(
+        31, dl,
+        DAG.getTargetLoweringInfo().getShiftAmountTy(VT, DAG.getDataLayout()));
     SDValue T1 = DAG.getNode(ISD::AND, dl, VT,
                              DAG.getNode(ISD::SRA, dl, VT, Op0, ShAmt), Op1);
     SDValue T2 = DAG.getNode(ISD::AND, dl, VT,
@@ -16236,6 +17149,53 @@ static SDValue LowerMUL_LOHI(SDValue Op, const X86Subtarget *Subtarget,
   return DAG.getMergeValues(Ops, dl);
 }
 
+// Return true if the required (according to Opcode) shift-imm form is natively
+// supported by the Subtarget
+static bool SupportedVectorShiftWithImm(MVT VT, const X86Subtarget *Subtarget,
+                                        unsigned Opcode) {
+  if (VT.getScalarSizeInBits() < 16)
+    return false;
+
+  if (VT.is512BitVector() &&
+      (VT.getScalarSizeInBits() > 16 || Subtarget->hasBWI()))
+    return true;
+
+  bool LShift = VT.is128BitVector() ||
+    (VT.is256BitVector() && Subtarget->hasInt256());
+
+  bool AShift = LShift && (Subtarget->hasVLX() ||
+    (VT != MVT::v2i64 && VT != MVT::v4i64));
+  return (Opcode == ISD::SRA) ? AShift : LShift;
+}
+
+// The shift amount is a variable, but it is the same for all vector lanes.
+// These instructions are defined together with shift-immediate.
+static
+bool SupportedVectorShiftWithBaseAmnt(MVT VT, const X86Subtarget *Subtarget,
+                                      unsigned Opcode) {
+  return SupportedVectorShiftWithImm(VT, Subtarget, Opcode);
+}
+
+// Return true if the required (according to Opcode) variable-shift form is
+// natively supported by the Subtarget
+static bool SupportedVectorVarShift(MVT VT, const X86Subtarget *Subtarget,
+                                    unsigned Opcode) {
+
+  if (!Subtarget->hasInt256() || VT.getScalarSizeInBits() < 16)
+    return false;
+
+  // vXi16 supported only on AVX-512, BWI
+  if (VT.getScalarSizeInBits() == 16 && !Subtarget->hasBWI())
+    return false;
+
+  if (VT.is512BitVector() || Subtarget->hasVLX())
+    return true;
+
+  bool LShift = VT.is128BitVector() || VT.is256BitVector();
+  bool AShift = LShift &&  VT != MVT::v2i64 && VT != MVT::v4i64;
+  return (Opcode == ISD::SRA) ? AShift : LShift;
+}
+
 static SDValue LowerScalarImmediateShift(SDValue Op, SelectionDAG &DAG,
                                          const X86Subtarget *Subtarget) {
   MVT VT = Op.getSimpleValueType();
@@ -16243,39 +17203,70 @@ static SDValue LowerScalarImmediateShift(SDValue Op, SelectionDAG &DAG,
   SDValue R = Op.getOperand(0);
   SDValue Amt = Op.getOperand(1);
 
+  unsigned X86Opc = (Op.getOpcode() == ISD::SHL) ? X86ISD::VSHLI :
+    (Op.getOpcode() == ISD::SRL) ? X86ISD::VSRLI : X86ISD::VSRAI;
+
+  auto ArithmeticShiftRight64 = [&](uint64_t ShiftAmt) {
+    assert((VT == MVT::v2i64 || VT == MVT::v4i64) && "Unexpected SRA type");
+    MVT ExVT = MVT::getVectorVT(MVT::i32, VT.getVectorNumElements() * 2);
+    SDValue Ex = DAG.getBitcast(ExVT, R);
+
+    if (ShiftAmt >= 32) {
+      // Splat sign to upper i32 dst, and SRA upper i32 src to lower i32.
+      SDValue Upper =
+          getTargetVShiftByConstNode(X86ISD::VSRAI, dl, ExVT, Ex, 31, DAG);
+      SDValue Lower = getTargetVShiftByConstNode(X86ISD::VSRAI, dl, ExVT, Ex,
+                                                 ShiftAmt - 32, DAG);
+      if (VT == MVT::v2i64)
+        Ex = DAG.getVectorShuffle(ExVT, dl, Upper, Lower, {5, 1, 7, 3});
+      if (VT == MVT::v4i64)
+        Ex = DAG.getVectorShuffle(ExVT, dl, Upper, Lower,
+                                  {9, 1, 11, 3, 13, 5, 15, 7});
+    } else {
+      // SRA upper i32, SHL whole i64 and select lower i32.
+      SDValue Upper = getTargetVShiftByConstNode(X86ISD::VSRAI, dl, ExVT, Ex,
+                                                 ShiftAmt, DAG);
+      SDValue Lower =
+          getTargetVShiftByConstNode(X86ISD::VSRLI, dl, VT, R, ShiftAmt, DAG);
+      Lower = DAG.getBitcast(ExVT, Lower);
+      if (VT == MVT::v2i64)
+        Ex = DAG.getVectorShuffle(ExVT, dl, Upper, Lower, {4, 1, 6, 3});
+      if (VT == MVT::v4i64)
+        Ex = DAG.getVectorShuffle(ExVT, dl, Upper, Lower,
+                                  {8, 1, 10, 3, 12, 5, 14, 7});
+    }
+    return DAG.getBitcast(VT, Ex);
+  };
+
   // Optimize shl/srl/sra with constant shift amount.
   if (auto *BVAmt = dyn_cast<BuildVectorSDNode>(Amt)) {
     if (auto *ShiftConst = BVAmt->getConstantSplatNode()) {
       uint64_t ShiftAmt = ShiftConst->getZExtValue();
 
-      if (VT == MVT::v2i64 || VT == MVT::v4i32 || VT == MVT::v8i16 ||
-          (Subtarget->hasInt256() &&
-           (VT == MVT::v4i64 || VT == MVT::v8i32 || VT == MVT::v16i16)) ||
-          (Subtarget->hasAVX512() &&
-           (VT == MVT::v8i64 || VT == MVT::v16i32))) {
-        if (Op.getOpcode() == ISD::SHL)
-          return getTargetVShiftByConstNode(X86ISD::VSHLI, dl, VT, R, ShiftAmt,
-                                            DAG);
-        if (Op.getOpcode() == ISD::SRL)
-          return getTargetVShiftByConstNode(X86ISD::VSRLI, dl, VT, R, ShiftAmt,
-                                            DAG);
-        if (Op.getOpcode() == ISD::SRA && VT != MVT::v2i64 && VT != MVT::v4i64)
-          return getTargetVShiftByConstNode(X86ISD::VSRAI, dl, VT, R, ShiftAmt,
-                                            DAG);
-      }
+      if (SupportedVectorShiftWithImm(VT, Subtarget, Op.getOpcode()))
+        return getTargetVShiftByConstNode(X86Opc, dl, VT, R, ShiftAmt, DAG);
+
+      // i64 SRA needs to be performed as partial shifts.
+      if ((VT == MVT::v2i64 || (Subtarget->hasInt256() && VT == MVT::v4i64)) &&
+          Op.getOpcode() == ISD::SRA)
+        return ArithmeticShiftRight64(ShiftAmt);
 
       if (VT == MVT::v16i8 || (Subtarget->hasInt256() && VT == MVT::v32i8)) {
         unsigned NumElts = VT.getVectorNumElements();
         MVT ShiftVT = MVT::getVectorVT(MVT::i16, NumElts / 2);
 
         if (Op.getOpcode() == ISD::SHL) {
+          // Simple i8 add case
+          if (ShiftAmt == 1)
+            return DAG.getNode(ISD::ADD, dl, VT, R, R);
+
           // Make a large shift.
           SDValue SHL = getTargetVShiftByConstNode(X86ISD::VSHLI, dl, ShiftVT,
                                                    R, ShiftAmt, DAG);
-          SHL = DAG.getNode(ISD::BITCAST, dl, VT, SHL);
+          SHL = DAG.getBitcast(VT, SHL);
           // Zero out the rightmost bits.
           SmallVector<SDValue, 32> V(
-              NumElts, DAG.getConstant(uint8_t(-1U << ShiftAmt), MVT::i8));
+              NumElts, DAG.getConstant(uint8_t(-1U << ShiftAmt), dl, MVT::i8));
           return DAG.getNode(ISD::AND, dl, VT, SHL,
                              DAG.getNode(ISD::BUILD_VECTOR, dl, VT, V));
         }
@@ -16283,24 +17274,25 @@ static SDValue LowerScalarImmediateShift(SDValue Op, SelectionDAG &DAG,
           // Make a large shift.
           SDValue SRL = getTargetVShiftByConstNode(X86ISD::VSRLI, dl, ShiftVT,
                                                    R, ShiftAmt, DAG);
-          SRL = DAG.getNode(ISD::BITCAST, dl, VT, SRL);
+          SRL = DAG.getBitcast(VT, SRL);
           // Zero out the leftmost bits.
           SmallVector<SDValue, 32> V(
-              NumElts, DAG.getConstant(uint8_t(-1U) >> ShiftAmt, MVT::i8));
+              NumElts, DAG.getConstant(uint8_t(-1U) >> ShiftAmt, dl, MVT::i8));
           return DAG.getNode(ISD::AND, dl, VT, SRL,
                              DAG.getNode(ISD::BUILD_VECTOR, dl, VT, V));
         }
         if (Op.getOpcode() == ISD::SRA) {
           if (ShiftAmt == 7) {
-            // R s>> 7  ===  R s< 0
+            // ashr(R, 7)  === cmp_slt(R, 0)
             SDValue Zeros = getZeroVector(VT, Subtarget, DAG, dl);
             return DAG.getNode(X86ISD::PCMPGT, dl, VT, Zeros, R);
           }
 
-          // R s>> a === ((R u>> a) ^ m) - m
+          // ashr(R, Amt) === sub(xor(lshr(R, Amt), Mask), Mask)
           SDValue Res = DAG.getNode(ISD::SRL, dl, VT, R, Amt);
           SmallVector<SDValue, 32> V(NumElts,
-                                     DAG.getConstant(128 >> ShiftAmt, MVT::i8));
+                                     DAG.getConstant(128 >> ShiftAmt, dl,
+                                                     MVT::i8));
           SDValue Mask = DAG.getNode(ISD::BUILD_VECTOR, dl, VT, V);
           Res = DAG.getNode(ISD::XOR, dl, VT, Res, Mask);
           Res = DAG.getNode(ISD::SUB, dl, VT, Res, Mask);
@@ -16313,48 +17305,57 @@ static SDValue LowerScalarImmediateShift(SDValue Op, SelectionDAG &DAG,
 
   // Special case in 32-bit mode, where i64 is expanded into high and low parts.
   if (!Subtarget->is64Bit() &&
-      (VT == MVT::v2i64 || (Subtarget->hasInt256() && VT == MVT::v4i64)) &&
-      Amt.getOpcode() == ISD::BITCAST &&
-      Amt.getOperand(0).getOpcode() == ISD::BUILD_VECTOR) {
+      (VT == MVT::v2i64 || (Subtarget->hasInt256() && VT == MVT::v4i64))) {
+
+    // Peek through any splat that was introduced for i64 shift vectorization.
+    int SplatIndex = -1;
+    if (ShuffleVectorSDNode *SVN = dyn_cast<ShuffleVectorSDNode>(Amt.getNode()))
+      if (SVN->isSplat()) {
+        SplatIndex = SVN->getSplatIndex();
+        Amt = Amt.getOperand(0);
+        assert(SplatIndex < (int)VT.getVectorNumElements() &&
+               "Splat shuffle referencing second operand");
+      }
+
+    if (Amt.getOpcode() != ISD::BITCAST ||
+        Amt.getOperand(0).getOpcode() != ISD::BUILD_VECTOR)
+      return SDValue();
+
     Amt = Amt.getOperand(0);
     unsigned Ratio = Amt.getSimpleValueType().getVectorNumElements() /
                      VT.getVectorNumElements();
     unsigned RatioInLog2 = Log2_32_Ceil(Ratio);
     uint64_t ShiftAmt = 0;
+    unsigned BaseOp = (SplatIndex < 0 ? 0 : SplatIndex * Ratio);
     for (unsigned i = 0; i != Ratio; ++i) {
-      ConstantSDNode *C = dyn_cast<ConstantSDNode>(Amt.getOperand(i));
+      ConstantSDNode *C = dyn_cast<ConstantSDNode>(Amt.getOperand(i + BaseOp));
       if (!C)
         return SDValue();
       // 6 == Log2(64)
       ShiftAmt |= C->getZExtValue() << (i * (1 << (6 - RatioInLog2)));
     }
-    // Check remaining shift amounts.
-    for (unsigned i = Ratio; i != Amt.getNumOperands(); i += Ratio) {
-      uint64_t ShAmt = 0;
-      for (unsigned j = 0; j != Ratio; ++j) {
-        ConstantSDNode *C =
-          dyn_cast<ConstantSDNode>(Amt.getOperand(i + j));
-        if (!C)
+
+    // Check remaining shift amounts (if not a splat).
+    if (SplatIndex < 0) {
+      for (unsigned i = Ratio; i != Amt.getNumOperands(); i += Ratio) {
+        uint64_t ShAmt = 0;
+        for (unsigned j = 0; j != Ratio; ++j) {
+          ConstantSDNode *C = dyn_cast<ConstantSDNode>(Amt.getOperand(i + j));
+          if (!C)
+            return SDValue();
+          // 6 == Log2(64)
+          ShAmt |= C->getZExtValue() << (j * (1 << (6 - RatioInLog2)));
+        }
+        if (ShAmt != ShiftAmt)
           return SDValue();
-        // 6 == Log2(64)
-        ShAmt |= C->getZExtValue() << (j * (1 << (6 - RatioInLog2)));
       }
-      if (ShAmt != ShiftAmt)
-        return SDValue();
-    }
-    switch (Op.getOpcode()) {
-    default:
-      llvm_unreachable("Unknown shift opcode!");
-    case ISD::SHL:
-      return getTargetVShiftByConstNode(X86ISD::VSHLI, dl, VT, R, ShiftAmt,
-                                        DAG);
-    case ISD::SRL:
-      return getTargetVShiftByConstNode(X86ISD::VSRLI, dl, VT, R, ShiftAmt,
-                                        DAG);
-    case ISD::SRA:
-      return getTargetVShiftByConstNode(X86ISD::VSRAI, dl, VT, R, ShiftAmt,
-                                        DAG);
     }
+
+    if (SupportedVectorShiftWithImm(VT, Subtarget, Op.getOpcode()))
+      return getTargetVShiftByConstNode(X86Opc, dl, VT, R, ShiftAmt, DAG);
+
+    if (Op.getOpcode() == ISD::SRA)
+      return ArithmeticShiftRight64(ShiftAmt);
   }
 
   return SDValue();
@@ -16367,12 +17368,13 @@ static SDValue LowerScalarVariableShift(SDValue Op, SelectionDAG &DAG,
   SDValue R = Op.getOperand(0);
   SDValue Amt = Op.getOperand(1);
 
-  if ((VT == MVT::v2i64 && Op.getOpcode() != ISD::SRA) ||
-      VT == MVT::v4i32 || VT == MVT::v8i16 ||
-      (Subtarget->hasInt256() &&
-       ((VT == MVT::v4i64 && Op.getOpcode() != ISD::SRA) ||
-        VT == MVT::v8i32 || VT == MVT::v16i16)) ||
-       (Subtarget->hasAVX512() && (VT == MVT::v8i64 || VT == MVT::v16i32))) {
+  unsigned X86OpcI = (Op.getOpcode() == ISD::SHL) ? X86ISD::VSHLI :
+    (Op.getOpcode() == ISD::SRL) ? X86ISD::VSRLI : X86ISD::VSRAI;
+
+  unsigned X86OpcV = (Op.getOpcode() == ISD::SHL) ? X86ISD::VSHL :
+    (Op.getOpcode() == ISD::SRL) ? X86ISD::VSRL : X86ISD::VSRA;
+
+  if (SupportedVectorShiftWithBaseAmnt(VT, Subtarget, Op.getOpcode())) {
     SDValue BaseShAmt;
     EVT EltVT = VT.getVectorElementType();
 
@@ -16405,7 +17407,7 @@ static SDValue LowerScalarVariableShift(SDValue Op, SelectionDAG &DAG,
         if (!BaseShAmt)
           // Avoid introducing an extract element from a shuffle.
           BaseShAmt = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, EltVT, InVec,
-                                    DAG.getIntPtrConstant(SplatIdx));
+                                  DAG.getIntPtrConstant(SplatIdx, dl));
       }
     }
 
@@ -16416,54 +17418,12 @@ static SDValue LowerScalarVariableShift(SDValue Op, SelectionDAG &DAG,
       else if (EltVT.bitsLT(MVT::i32))
         BaseShAmt = DAG.getNode(ISD::ZERO_EXTEND, dl, MVT::i32, BaseShAmt);
 
-      switch (Op.getOpcode()) {
-      default:
-        llvm_unreachable("Unknown shift opcode!");
-      case ISD::SHL:
-        switch (VT.SimpleTy) {
-        default: return SDValue();
-        case MVT::v2i64:
-        case MVT::v4i32:
-        case MVT::v8i16:
-        case MVT::v4i64:
-        case MVT::v8i32:
-        case MVT::v16i16:
-        case MVT::v16i32:
-        case MVT::v8i64:
-          return getTargetVShiftNode(X86ISD::VSHLI, dl, VT, R, BaseShAmt, DAG);
-        }
-      case ISD::SRA:
-        switch (VT.SimpleTy) {
-        default: return SDValue();
-        case MVT::v4i32:
-        case MVT::v8i16:
-        case MVT::v8i32:
-        case MVT::v16i16:
-        case MVT::v16i32:
-        case MVT::v8i64:
-          return getTargetVShiftNode(X86ISD::VSRAI, dl, VT, R, BaseShAmt, DAG);
-        }
-      case ISD::SRL:
-        switch (VT.SimpleTy) {
-        default: return SDValue();
-        case MVT::v2i64:
-        case MVT::v4i32:
-        case MVT::v8i16:
-        case MVT::v4i64:
-        case MVT::v8i32:
-        case MVT::v16i16:
-        case MVT::v16i32:
-        case MVT::v8i64:
-          return getTargetVShiftNode(X86ISD::VSRLI, dl, VT, R, BaseShAmt, DAG);
-        }
-      }
+      return getTargetVShiftNode(X86OpcI, dl, VT, R, BaseShAmt, DAG);
     }
   }
 
   // Special case in 32-bit mode, where i64 is expanded into high and low parts.
-  if (!Subtarget->is64Bit() &&
-      (VT == MVT::v2i64 || (Subtarget->hasInt256() && VT == MVT::v4i64) ||
-      (Subtarget->hasAVX512() && VT == MVT::v8i64)) &&
+  if (!Subtarget->is64Bit() && VT == MVT::v2i64  &&
       Amt.getOpcode() == ISD::BITCAST &&
       Amt.getOperand(0).getOpcode() == ISD::BUILD_VECTOR) {
     Amt = Amt.getOperand(0);
@@ -16477,18 +17437,10 @@ static SDValue LowerScalarVariableShift(SDValue Op, SelectionDAG &DAG,
         if (Vals[j] != Amt.getOperand(i + j))
           return SDValue();
     }
-    switch (Op.getOpcode()) {
-    default:
-      llvm_unreachable("Unknown shift opcode!");
-    case ISD::SHL:
-      return DAG.getNode(X86ISD::VSHL, dl, VT, R, Op.getOperand(1));
-    case ISD::SRL:
-      return DAG.getNode(X86ISD::VSRL, dl, VT, R, Op.getOperand(1));
-    case ISD::SRA:
-      return DAG.getNode(X86ISD::VSRA, dl, VT, R, Op.getOperand(1));
-    }
-  }
 
+    if (SupportedVectorShiftWithBaseAmnt(VT, Subtarget, Op.getOpcode()))
+      return DAG.getNode(X86OpcV, dl, VT, R, Op.getOperand(1));
+  }
   return SDValue();
 }
 
@@ -16508,23 +17460,9 @@ static SDValue LowerShift(SDValue Op, const X86Subtarget* Subtarget,
   if (SDValue V = LowerScalarVariableShift(Op, DAG, Subtarget))
       return V;
 
-  if (Subtarget->hasAVX512() && (VT == MVT::v16i32 || VT == MVT::v8i64))
+  if (SupportedVectorVarShift(VT, Subtarget, Op.getOpcode()))
     return Op;
 
-  // AVX2 has VPSLLV/VPSRAV/VPSRLV.
-  if (Subtarget->hasInt256()) {
-    if (Op.getOpcode() == ISD::SRL &&
-        (VT == MVT::v2i64 || VT == MVT::v4i32 ||
-         VT == MVT::v4i64 || VT == MVT::v8i32))
-      return Op;
-    if (Op.getOpcode() == ISD::SHL &&
-        (VT == MVT::v2i64 || VT == MVT::v4i32 ||
-         VT == MVT::v4i64 || VT == MVT::v8i32))
-      return Op;
-    if (Op.getOpcode() == ISD::SRA && (VT == MVT::v4i32 || VT == MVT::v8i32))
-      return Op;
-  }
-
   // 2i64 vector logical shifts can efficiently avoid scalarization - do the
   // shifts per-lane and then shuffle the partial results back together.
   if (VT == MVT::v2i64 && Op.getOpcode() != ISD::SRA) {
@@ -16536,6 +17474,19 @@ static SDValue LowerShift(SDValue Op, const X86Subtarget* Subtarget,
     return DAG.getVectorShuffle(VT, dl, R0, R1, {0, 3});
   }
 
+  // i64 vector arithmetic shift can be emulated with the transform:
+  // M = lshr(SIGN_BIT, Amt)
+  // ashr(R, Amt) === sub(xor(lshr(R, Amt), M), M)
+  if ((VT == MVT::v2i64 || (VT == MVT::v4i64 && Subtarget->hasInt256())) &&
+      Op.getOpcode() == ISD::SRA) {
+    SDValue S = DAG.getConstant(APInt::getSignBit(64), dl, VT);
+    SDValue M = DAG.getNode(ISD::SRL, dl, VT, S, Amt);
+    R = DAG.getNode(ISD::SRL, dl, VT, R, Amt);
+    R = DAG.getNode(ISD::XOR, dl, VT, R, M);
+    R = DAG.getNode(ISD::SUB, dl, VT, R, M);
+    return R;
+  }
+
   // If possible, lower this packed shift into a vector multiply instead of
   // expanding it into a sequence of scalar shifts.
   // Do this only if the vector shift count is a constant build_vector.
@@ -16563,7 +17514,7 @@ static SDValue LowerShift(SDValue Op, const X86Subtarget* Subtarget,
         Elts.push_back(DAG.getUNDEF(SVT));
         continue;
       }
-      Elts.push_back(DAG.getConstant(One.shl(ShAmt), SVT));
+      Elts.push_back(DAG.getConstant(One.shl(ShAmt), dl, SVT));
     }
     SDValue BV = DAG.getNode(ISD::BUILD_VECTOR, dl, VT, Elts);
     return DAG.getNode(ISD::MUL, dl, VT, R, BV);
@@ -16571,10 +17522,11 @@ static SDValue LowerShift(SDValue Op, const X86Subtarget* Subtarget,
 
   // Lower SHL with variable shift amount.
   if (VT == MVT::v4i32 && Op->getOpcode() == ISD::SHL) {
-    Op = DAG.getNode(ISD::SHL, dl, VT, Amt, DAG.getConstant(23, VT));
+    Op = DAG.getNode(ISD::SHL, dl, VT, Amt, DAG.getConstant(23, dl, VT));
 
-    Op = DAG.getNode(ISD::ADD, dl, VT, Op, DAG.getConstant(0x3f800000U, VT));
-    Op = DAG.getNode(ISD::BITCAST, dl, MVT::v4f32, Op);
+    Op = DAG.getNode(ISD::ADD, dl, VT, Op,
+                     DAG.getConstant(0x3f800000U, dl, VT));
+    Op = DAG.getBitcast(MVT::v4f32, Op);
     Op = DAG.getNode(ISD::FP_TO_SINT, dl, VT, Op);
     return DAG.getNode(ISD::MUL, dl, VT, Op, R);
   }
@@ -16637,75 +17589,265 @@ static SDValue LowerShift(SDValue Op, const X86Subtarget* Subtarget,
       // Replace this node with two shifts followed by a MOVSS/MOVSD.
       EVT CastVT = MVT::v4i32;
       SDValue Splat1 =
-        DAG.getConstant(cast<ConstantSDNode>(Amt1)->getAPIntValue(), VT);
+        DAG.getConstant(cast<ConstantSDNode>(Amt1)->getAPIntValue(), dl, VT);
       SDValue Shift1 = DAG.getNode(Op->getOpcode(), dl, VT, R, Splat1);
       SDValue Splat2 =
-        DAG.getConstant(cast<ConstantSDNode>(Amt2)->getAPIntValue(), VT);
+        DAG.getConstant(cast<ConstantSDNode>(Amt2)->getAPIntValue(), dl, VT);
       SDValue Shift2 = DAG.getNode(Op->getOpcode(), dl, VT, R, Splat2);
       if (TargetOpcode == X86ISD::MOVSD)
         CastVT = MVT::v2i64;
-      SDValue BitCast1 = DAG.getNode(ISD::BITCAST, dl, CastVT, Shift1);
-      SDValue BitCast2 = DAG.getNode(ISD::BITCAST, dl, CastVT, Shift2);
+      SDValue BitCast1 = DAG.getBitcast(CastVT, Shift1);
+      SDValue BitCast2 = DAG.getBitcast(CastVT, Shift2);
       SDValue Result = getTargetShuffleNode(TargetOpcode, dl, CastVT, BitCast2,
                                             BitCast1, DAG);
-      return DAG.getNode(ISD::BITCAST, dl, VT, Result);
+      return DAG.getBitcast(VT, Result);
     }
   }
 
-  if (VT == MVT::v16i8 && Op->getOpcode() == ISD::SHL) {
-    assert(Subtarget->hasSSE2() && "Need SSE2 for pslli/pcmpeq.");
-
-    // a = a << 5;
-    Op = DAG.getNode(ISD::SHL, dl, VT, Amt, DAG.getConstant(5, VT));
-    Op = DAG.getNode(ISD::BITCAST, dl, VT, Op);
+  // v4i32 Non Uniform Shifts.
+  // If the shift amount is constant we can shift each lane using the SSE2
+  // immediate shifts, else we need to zero-extend each lane to the lower i64
+  // and shift using the SSE2 variable shifts.
+  // The separate results can then be blended together.
+  if (VT == MVT::v4i32) {
+    unsigned Opc = Op.getOpcode();
+    SDValue Amt0, Amt1, Amt2, Amt3;
+    if (ISD::isBuildVectorOfConstantSDNodes(Amt.getNode())) {
+      Amt0 = DAG.getVectorShuffle(VT, dl, Amt, DAG.getUNDEF(VT), {0, 0, 0, 0});
+      Amt1 = DAG.getVectorShuffle(VT, dl, Amt, DAG.getUNDEF(VT), {1, 1, 1, 1});
+      Amt2 = DAG.getVectorShuffle(VT, dl, Amt, DAG.getUNDEF(VT), {2, 2, 2, 2});
+      Amt3 = DAG.getVectorShuffle(VT, dl, Amt, DAG.getUNDEF(VT), {3, 3, 3, 3});
+    } else {
+      // ISD::SHL is handled above but we include it here for completeness.
+      switch (Opc) {
+      default:
+        llvm_unreachable("Unknown target vector shift node");
+      case ISD::SHL:
+        Opc = X86ISD::VSHL;
+        break;
+      case ISD::SRL:
+        Opc = X86ISD::VSRL;
+        break;
+      case ISD::SRA:
+        Opc = X86ISD::VSRA;
+        break;
+      }
+      // The SSE2 shifts use the lower i64 as the same shift amount for
+      // all lanes and the upper i64 is ignored. These shuffle masks
+      // optimally zero-extend each lanes on SSE2/SSE41/AVX targets.
+      SDValue Z = getZeroVector(VT, Subtarget, DAG, dl);
+      Amt0 = DAG.getVectorShuffle(VT, dl, Amt, Z, {0, 4, -1, -1});
+      Amt1 = DAG.getVectorShuffle(VT, dl, Amt, Z, {1, 5, -1, -1});
+      Amt2 = DAG.getVectorShuffle(VT, dl, Amt, Z, {2, 6, -1, -1});
+      Amt3 = DAG.getVectorShuffle(VT, dl, Amt, Z, {3, 7, -1, -1});
+    }
+
+    SDValue R0 = DAG.getNode(Opc, dl, VT, R, Amt0);
+    SDValue R1 = DAG.getNode(Opc, dl, VT, R, Amt1);
+    SDValue R2 = DAG.getNode(Opc, dl, VT, R, Amt2);
+    SDValue R3 = DAG.getNode(Opc, dl, VT, R, Amt3);
+    SDValue R02 = DAG.getVectorShuffle(VT, dl, R0, R2, {0, -1, 6, -1});
+    SDValue R13 = DAG.getVectorShuffle(VT, dl, R1, R3, {-1, 1, -1, 7});
+    return DAG.getVectorShuffle(VT, dl, R02, R13, {0, 5, 2, 7});
+  }
+
+  if (VT == MVT::v16i8 || (VT == MVT::v32i8 && Subtarget->hasInt256())) {
+    MVT ExtVT = MVT::getVectorVT(MVT::i16, VT.getVectorNumElements() / 2);
+    unsigned ShiftOpcode = Op->getOpcode();
+
+    auto SignBitSelect = [&](MVT SelVT, SDValue Sel, SDValue V0, SDValue V1) {
+      // On SSE41 targets we make use of the fact that VSELECT lowers
+      // to PBLENDVB which selects bytes based just on the sign bit.
+      if (Subtarget->hasSSE41()) {
+        V0 = DAG.getBitcast(VT, V0);
+        V1 = DAG.getBitcast(VT, V1);
+        Sel = DAG.getBitcast(VT, Sel);
+        return DAG.getBitcast(SelVT,
+                              DAG.getNode(ISD::VSELECT, dl, VT, Sel, V0, V1));
+      }
+      // On pre-SSE41 targets we test for the sign bit by comparing to
+      // zero - a negative value will set all bits of the lanes to true
+      // and VSELECT uses that in its OR(AND(V0,C),AND(V1,~C)) lowering.
+      SDValue Z = getZeroVector(SelVT, Subtarget, DAG, dl);
+      SDValue C = DAG.getNode(X86ISD::PCMPGT, dl, SelVT, Z, Sel);
+      return DAG.getNode(ISD::VSELECT, dl, SelVT, C, V0, V1);
+    };
 
-    // Turn 'a' into a mask suitable for VSELECT
-    SDValue VSelM = DAG.getConstant(0x80, VT);
-    SDValue OpVSel = DAG.getNode(ISD::AND, dl, VT, VSelM, Op);
-    OpVSel = DAG.getNode(X86ISD::PCMPEQ, dl, VT, OpVSel, VSelM);
+    // Turn 'a' into a mask suitable for VSELECT: a = a << 5;
+    // We can safely do this using i16 shifts as we're only interested in
+    // the 3 lower bits of each byte.
+    Amt = DAG.getBitcast(ExtVT, Amt);
+    Amt = DAG.getNode(ISD::SHL, dl, ExtVT, Amt, DAG.getConstant(5, dl, ExtVT));
+    Amt = DAG.getBitcast(VT, Amt);
 
-    SDValue CM1 = DAG.getConstant(0x0f, VT);
-    SDValue CM2 = DAG.getConstant(0x3f, VT);
+    if (Op->getOpcode() == ISD::SHL || Op->getOpcode() == ISD::SRL) {
+      // r = VSELECT(r, shift(r, 4), a);
+      SDValue M =
+          DAG.getNode(ShiftOpcode, dl, VT, R, DAG.getConstant(4, dl, VT));
+      R = SignBitSelect(VT, Amt, M, R);
 
-    // r = VSELECT(r, psllw(r & (char16)15, 4), a);
-    SDValue M = DAG.getNode(ISD::AND, dl, VT, R, CM1);
-    M = getTargetVShiftByConstNode(X86ISD::VSHLI, dl, MVT::v8i16, M, 4, DAG);
-    M = DAG.getNode(ISD::BITCAST, dl, VT, M);
-    R = DAG.getNode(ISD::VSELECT, dl, VT, OpVSel, M, R);
+      // a += a
+      Amt = DAG.getNode(ISD::ADD, dl, VT, Amt, Amt);
 
-    // a += a
-    Op = DAG.getNode(ISD::ADD, dl, VT, Op, Op);
-    OpVSel = DAG.getNode(ISD::AND, dl, VT, VSelM, Op);
-    OpVSel = DAG.getNode(X86ISD::PCMPEQ, dl, VT, OpVSel, VSelM);
+      // r = VSELECT(r, shift(r, 2), a);
+      M = DAG.getNode(ShiftOpcode, dl, VT, R, DAG.getConstant(2, dl, VT));
+      R = SignBitSelect(VT, Amt, M, R);
 
-    // r = VSELECT(r, psllw(r & (char16)63, 2), a);
-    M = DAG.getNode(ISD::AND, dl, VT, R, CM2);
-    M = getTargetVShiftByConstNode(X86ISD::VSHLI, dl, MVT::v8i16, M, 2, DAG);
-    M = DAG.getNode(ISD::BITCAST, dl, VT, M);
-    R = DAG.getNode(ISD::VSELECT, dl, VT, OpVSel, M, R);
+      // a += a
+      Amt = DAG.getNode(ISD::ADD, dl, VT, Amt, Amt);
 
-    // a += a
-    Op = DAG.getNode(ISD::ADD, dl, VT, Op, Op);
-    OpVSel = DAG.getNode(ISD::AND, dl, VT, VSelM, Op);
-    OpVSel = DAG.getNode(X86ISD::PCMPEQ, dl, VT, OpVSel, VSelM);
+      // return VSELECT(r, shift(r, 1), a);
+      M = DAG.getNode(ShiftOpcode, dl, VT, R, DAG.getConstant(1, dl, VT));
+      R = SignBitSelect(VT, Amt, M, R);
+      return R;
+    }
 
-    // return VSELECT(r, r+r, a);
-    R = DAG.getNode(ISD::VSELECT, dl, VT, OpVSel,
-                    DAG.getNode(ISD::ADD, dl, VT, R, R), R);
-    return R;
+    if (Op->getOpcode() == ISD::SRA) {
+      // For SRA we need to unpack each byte to the higher byte of a i16 vector
+      // so we can correctly sign extend. We don't care what happens to the
+      // lower byte.
+      SDValue ALo = DAG.getNode(X86ISD::UNPCKL, dl, VT, DAG.getUNDEF(VT), Amt);
+      SDValue AHi = DAG.getNode(X86ISD::UNPCKH, dl, VT, DAG.getUNDEF(VT), Amt);
+      SDValue RLo = DAG.getNode(X86ISD::UNPCKL, dl, VT, DAG.getUNDEF(VT), R);
+      SDValue RHi = DAG.getNode(X86ISD::UNPCKH, dl, VT, DAG.getUNDEF(VT), R);
+      ALo = DAG.getBitcast(ExtVT, ALo);
+      AHi = DAG.getBitcast(ExtVT, AHi);
+      RLo = DAG.getBitcast(ExtVT, RLo);
+      RHi = DAG.getBitcast(ExtVT, RHi);
+
+      // r = VSELECT(r, shift(r, 4), a);
+      SDValue MLo = DAG.getNode(ShiftOpcode, dl, ExtVT, RLo,
+                                DAG.getConstant(4, dl, ExtVT));
+      SDValue MHi = DAG.getNode(ShiftOpcode, dl, ExtVT, RHi,
+                                DAG.getConstant(4, dl, ExtVT));
+      RLo = SignBitSelect(ExtVT, ALo, MLo, RLo);
+      RHi = SignBitSelect(ExtVT, AHi, MHi, RHi);
+
+      // a += a
+      ALo = DAG.getNode(ISD::ADD, dl, ExtVT, ALo, ALo);
+      AHi = DAG.getNode(ISD::ADD, dl, ExtVT, AHi, AHi);
+
+      // r = VSELECT(r, shift(r, 2), a);
+      MLo = DAG.getNode(ShiftOpcode, dl, ExtVT, RLo,
+                        DAG.getConstant(2, dl, ExtVT));
+      MHi = DAG.getNode(ShiftOpcode, dl, ExtVT, RHi,
+                        DAG.getConstant(2, dl, ExtVT));
+      RLo = SignBitSelect(ExtVT, ALo, MLo, RLo);
+      RHi = SignBitSelect(ExtVT, AHi, MHi, RHi);
+
+      // a += a
+      ALo = DAG.getNode(ISD::ADD, dl, ExtVT, ALo, ALo);
+      AHi = DAG.getNode(ISD::ADD, dl, ExtVT, AHi, AHi);
+
+      // r = VSELECT(r, shift(r, 1), a);
+      MLo = DAG.getNode(ShiftOpcode, dl, ExtVT, RLo,
+                        DAG.getConstant(1, dl, ExtVT));
+      MHi = DAG.getNode(ShiftOpcode, dl, ExtVT, RHi,
+                        DAG.getConstant(1, dl, ExtVT));
+      RLo = SignBitSelect(ExtVT, ALo, MLo, RLo);
+      RHi = SignBitSelect(ExtVT, AHi, MHi, RHi);
+
+      // Logical shift the result back to the lower byte, leaving a zero upper
+      // byte
+      // meaning that we can safely pack with PACKUSWB.
+      RLo =
+          DAG.getNode(ISD::SRL, dl, ExtVT, RLo, DAG.getConstant(8, dl, ExtVT));
+      RHi =
+          DAG.getNode(ISD::SRL, dl, ExtVT, RHi, DAG.getConstant(8, dl, ExtVT));
+      return DAG.getNode(X86ISD::PACKUS, dl, VT, RLo, RHi);
+    }
   }
 
   // It's worth extending once and using the v8i32 shifts for 16-bit types, but
   // the extra overheads to get from v16i8 to v8i32 make the existing SSE
   // solution better.
   if (Subtarget->hasInt256() && VT == MVT::v8i16) {
-    MVT NewVT = VT == MVT::v8i16 ? MVT::v8i32 : MVT::v16i16;
+    MVT ExtVT = MVT::v8i32;
     unsigned ExtOpc =
         Op.getOpcode() == ISD::SRA ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND;
-    R = DAG.getNode(ExtOpc, dl, NewVT, R);
-    Amt = DAG.getNode(ISD::ANY_EXTEND, dl, NewVT, Amt);
+    R = DAG.getNode(ExtOpc, dl, ExtVT, R);
+    Amt = DAG.getNode(ISD::ANY_EXTEND, dl, ExtVT, Amt);
     return DAG.getNode(ISD::TRUNCATE, dl, VT,
-                       DAG.getNode(Op.getOpcode(), dl, NewVT, R, Amt));
+                       DAG.getNode(Op.getOpcode(), dl, ExtVT, R, Amt));
+  }
+
+  if (Subtarget->hasInt256() && VT == MVT::v16i16) {
+    MVT ExtVT = MVT::v8i32;
+    SDValue Z = getZeroVector(VT, Subtarget, DAG, dl);
+    SDValue ALo = DAG.getNode(X86ISD::UNPCKL, dl, VT, Amt, Z);
+    SDValue AHi = DAG.getNode(X86ISD::UNPCKH, dl, VT, Amt, Z);
+    SDValue RLo = DAG.getNode(X86ISD::UNPCKL, dl, VT, R, R);
+    SDValue RHi = DAG.getNode(X86ISD::UNPCKH, dl, VT, R, R);
+    ALo = DAG.getBitcast(ExtVT, ALo);
+    AHi = DAG.getBitcast(ExtVT, AHi);
+    RLo = DAG.getBitcast(ExtVT, RLo);
+    RHi = DAG.getBitcast(ExtVT, RHi);
+    SDValue Lo = DAG.getNode(Op.getOpcode(), dl, ExtVT, RLo, ALo);
+    SDValue Hi = DAG.getNode(Op.getOpcode(), dl, ExtVT, RHi, AHi);
+    Lo = DAG.getNode(ISD::SRL, dl, ExtVT, Lo, DAG.getConstant(16, dl, ExtVT));
+    Hi = DAG.getNode(ISD::SRL, dl, ExtVT, Hi, DAG.getConstant(16, dl, ExtVT));
+    return DAG.getNode(X86ISD::PACKUS, dl, VT, Lo, Hi);
+  }
+
+  if (VT == MVT::v8i16) {
+    unsigned ShiftOpcode = Op->getOpcode();
+
+    auto SignBitSelect = [&](SDValue Sel, SDValue V0, SDValue V1) {
+      // On SSE41 targets we make use of the fact that VSELECT lowers
+      // to PBLENDVB which selects bytes based just on the sign bit.
+      if (Subtarget->hasSSE41()) {
+        MVT ExtVT = MVT::getVectorVT(MVT::i8, VT.getVectorNumElements() * 2);
+        V0 = DAG.getBitcast(ExtVT, V0);
+        V1 = DAG.getBitcast(ExtVT, V1);
+        Sel = DAG.getBitcast(ExtVT, Sel);
+        return DAG.getBitcast(
+            VT, DAG.getNode(ISD::VSELECT, dl, ExtVT, Sel, V0, V1));
+      }
+      // On pre-SSE41 targets we splat the sign bit - a negative value will
+      // set all bits of the lanes to true and VSELECT uses that in
+      // its OR(AND(V0,C),AND(V1,~C)) lowering.
+      SDValue C =
+          DAG.getNode(ISD::SRA, dl, VT, Sel, DAG.getConstant(15, dl, VT));
+      return DAG.getNode(ISD::VSELECT, dl, VT, C, V0, V1);
+    };
+
+    // Turn 'a' into a mask suitable for VSELECT: a = a << 12;
+    if (Subtarget->hasSSE41()) {
+      // On SSE41 targets we need to replicate the shift mask in both
+      // bytes for PBLENDVB.
+      Amt = DAG.getNode(
+          ISD::OR, dl, VT,
+          DAG.getNode(ISD::SHL, dl, VT, Amt, DAG.getConstant(4, dl, VT)),
+          DAG.getNode(ISD::SHL, dl, VT, Amt, DAG.getConstant(12, dl, VT)));
+    } else {
+      Amt = DAG.getNode(ISD::SHL, dl, VT, Amt, DAG.getConstant(12, dl, VT));
+    }
+
+    // r = VSELECT(r, shift(r, 8), a);
+    SDValue M = DAG.getNode(ShiftOpcode, dl, VT, R, DAG.getConstant(8, dl, VT));
+    R = SignBitSelect(Amt, M, R);
+
+    // a += a
+    Amt = DAG.getNode(ISD::ADD, dl, VT, Amt, Amt);
+
+    // r = VSELECT(r, shift(r, 4), a);
+    M = DAG.getNode(ShiftOpcode, dl, VT, R, DAG.getConstant(4, dl, VT));
+    R = SignBitSelect(Amt, M, R);
+
+    // a += a
+    Amt = DAG.getNode(ISD::ADD, dl, VT, Amt, Amt);
+
+    // r = VSELECT(r, shift(r, 2), a);
+    M = DAG.getNode(ShiftOpcode, dl, VT, R, DAG.getConstant(2, dl, VT));
+    R = SignBitSelect(Amt, M, R);
+
+    // a += a
+    Amt = DAG.getNode(ISD::ADD, dl, VT, Amt, Amt);
+
+    // return VSELECT(r, shift(r, 1), a);
+    M = DAG.getNode(ShiftOpcode, dl, VT, R, DAG.getConstant(1, dl, VT));
+    R = SignBitSelect(Amt, M, R);
+    return R;
   }
 
   // Decompose 256-bit shifts into smaller 128-bit shifts.
@@ -16806,7 +17948,7 @@ static SDValue LowerXALUO(SDValue Op, SelectionDAG &DAG) {
 
     SDValue SetCC =
       DAG.getNode(X86ISD::SETCC, DL, MVT::i8,
-                  DAG.getConstant(X86::COND_O, MVT::i32),
+                  DAG.getConstant(X86::COND_O, DL, MVT::i32),
                   SDValue(Sum.getNode(), 2));
 
     return DAG.getNode(ISD::MERGE_VALUES, DL, N->getVTList(), Sum, SetCC);
@@ -16819,7 +17961,7 @@ static SDValue LowerXALUO(SDValue Op, SelectionDAG &DAG) {
 
   SDValue SetCC =
     DAG.getNode(X86ISD::SETCC, DL, N->getValueType(1),
-                DAG.getConstant(Cond, MVT::i32),
+                DAG.getConstant(Cond, DL, MVT::i32),
                 SDValue(Sum.getNode(), 1));
 
   return DAG.getNode(ISD::MERGE_VALUES, DL, N->getVTList(), Sum, SetCC);
@@ -16931,21 +18073,21 @@ X86TargetLowering::lowerIdempotentRMWIntoFencedLoad(AtomicRMWInst *AI) const {
   // otherwise, we might be able to be more agressive on relaxed idempotent
   // rmw. In practice, they do not look useful, so we don't try to be
   // especially clever.
-  if (SynchScope == SingleThread) {
+  if (SynchScope == SingleThread)
     // FIXME: we could just insert an X86ISD::MEMBARRIER here, except we are at
     // the IR level, so we must wrap it in an intrinsic.
     return nullptr;
-  } else if (hasMFENCE(*Subtarget)) {
-    Function *MFence = llvm::Intrinsic::getDeclaration(M,
-            Intrinsic::x86_sse2_mfence);
-    Builder.CreateCall(MFence);
-  } else {
+
+  if (!hasMFENCE(*Subtarget))
     // FIXME: it might make sense to use a locked operation here but on a
     // different cache-line to prevent cache-line bouncing. In practice it
     // is probably a small win, and x86 processors without mfence are rare
     // enough that we do not bother.
     return nullptr;
-  }
+
+  Function *MFence =
+      llvm::Intrinsic::getDeclaration(M, Intrinsic::x86_sse2_mfence);
+  Builder.CreateCall(MFence, {});
 
   // Finally we can emit the atomic load.
   LoadInst *Loaded = Builder.CreateAlignedLoad(Ptr,
@@ -16971,13 +18113,13 @@ static SDValue LowerATOMIC_FENCE(SDValue Op, const X86Subtarget *Subtarget,
       return DAG.getNode(X86ISD::MFENCE, dl, MVT::Other, Op.getOperand(0));
 
     SDValue Chain = Op.getOperand(0);
-    SDValue Zero = DAG.getConstant(0, MVT::i32);
+    SDValue Zero = DAG.getConstant(0, dl, MVT::i32);
     SDValue Ops[] = {
-      DAG.getRegister(X86::ESP, MVT::i32), // Base
-      DAG.getTargetConstant(1, MVT::i8),   // Scale
-      DAG.getRegister(0, MVT::i32),        // Index
-      DAG.getTargetConstant(0, MVT::i32),  // Disp
-      DAG.getRegister(0, MVT::i32),        // Segment.
+      DAG.getRegister(X86::ESP, MVT::i32),     // Base
+      DAG.getTargetConstant(1, dl, MVT::i8),   // Scale
+      DAG.getRegister(0, MVT::i32),            // Index
+      DAG.getTargetConstant(0, dl, MVT::i32),  // Disp
+      DAG.getRegister(0, MVT::i32),            // Segment.
       Zero,
       Chain
     };
@@ -17010,7 +18152,7 @@ static SDValue LowerCMP_SWAP(SDValue Op, const X86Subtarget *Subtarget,
   SDValue Ops[] = { cpIn.getValue(0),
                     Op.getOperand(1),
                     Op.getOperand(3),
-                    DAG.getTargetConstant(size, MVT::i8),
+                    DAG.getTargetConstant(size, DL, MVT::i8),
                     cpIn.getValue(1) };
   SDVTList Tys = DAG.getVTList(MVT::Other, MVT::Glue);
   MachineMemOperand *MMO = cast<AtomicSDNode>(Op)->getMemOperand();
@@ -17022,7 +18164,8 @@ static SDValue LowerCMP_SWAP(SDValue Op, const X86Subtarget *Subtarget,
   SDValue EFLAGS = DAG.getCopyFromReg(cpOut.getValue(1), DL, X86::EFLAGS,
                                       MVT::i32, cpOut.getValue(2));
   SDValue Success = DAG.getNode(X86ISD::SETCC, DL, Op->getValueType(1),
-                                DAG.getConstant(X86::COND_E, MVT::i8), EFLAGS);
+                                DAG.getConstant(X86::COND_E, DL, MVT::i8),
+                                EFLAGS);
 
   DAG.ReplaceAllUsesOfValueWith(Op.getValue(0), cpOut);
   DAG.ReplaceAllUsesOfValueWith(Op.getValue(1), Success);
@@ -17051,16 +18194,16 @@ static SDValue LowerBITCAST(SDValue Op, const X86Subtarget *Subtarget,
     SmallVector<SDValue, 16> Elts;
     for (unsigned i = 0, e = NumElts; i != e; ++i)
       Elts.push_back(DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, SVT, InVec,
-                                 DAG.getIntPtrConstant(i)));
+                                 DAG.getIntPtrConstant(i, dl)));
 
     // Explicitly mark the extra elements as Undef.
     Elts.append(NumElts, DAG.getUNDEF(SVT));
 
     EVT NewVT = EVT::getVectorVT(*DAG.getContext(), SVT, NumElts * 2);
     SDValue BV = DAG.getNode(ISD::BUILD_VECTOR, dl, NewVT, Elts);
-    SDValue ToV2F64 = DAG.getNode(ISD::BITCAST, dl, MVT::v2f64, BV);
+    SDValue ToV2F64 = DAG.getBitcast(MVT::v2f64, BV);
     return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::f64, ToV2F64,
-                       DAG.getIntPtrConstant(0));
+                       DAG.getIntPtrConstant(0, dl));
   }
 
   assert(Subtarget->is64Bit() && !Subtarget->hasSSE2() &&
@@ -17080,137 +18223,241 @@ static SDValue LowerBITCAST(SDValue Op, const X86Subtarget *Subtarget,
   return SDValue();
 }
 
-static SDValue LowerCTPOP(SDValue Op, const X86Subtarget *Subtarget,
-                          SelectionDAG &DAG) {
-  SDNode *Node = Op.getNode();
-  SDLoc dl(Node);
+/// Compute the horizontal sum of bytes in V for the elements of VT.
+///
+/// Requires V to be a byte vector and VT to be an integer vector type with
+/// wider elements than V's type. The width of the elements of VT determines
+/// how many bytes of V are summed horizontally to produce each element of the
+/// result.
+static SDValue LowerHorizontalByteSum(SDValue V, MVT VT,
+                                      const X86Subtarget *Subtarget,
+                                      SelectionDAG &DAG) {
+  SDLoc DL(V);
+  MVT ByteVecVT = V.getSimpleValueType();
+  MVT EltVT = VT.getVectorElementType();
+  int NumElts = VT.getVectorNumElements();
+  assert(ByteVecVT.getVectorElementType() == MVT::i8 &&
+         "Expected value to have byte element type.");
+  assert(EltVT != MVT::i8 &&
+         "Horizontal byte sum only makes sense for wider elements!");
+  unsigned VecSize = VT.getSizeInBits();
+  assert(ByteVecVT.getSizeInBits() == VecSize && "Cannot change vector size!");
+
+  // PSADBW instruction horizontally add all bytes and leave the result in i64
+  // chunks, thus directly computes the pop count for v2i64 and v4i64.
+  if (EltVT == MVT::i64) {
+    SDValue Zeros = getZeroVector(ByteVecVT, Subtarget, DAG, DL);
+    V = DAG.getNode(X86ISD::PSADBW, DL, ByteVecVT, V, Zeros);
+    return DAG.getBitcast(VT, V);
+  }
+
+  if (EltVT == MVT::i32) {
+    // We unpack the low half and high half into i32s interleaved with zeros so
+    // that we can use PSADBW to horizontally sum them. The most useful part of
+    // this is that it lines up the results of two PSADBW instructions to be
+    // two v2i64 vectors which concatenated are the 4 population counts. We can
+    // then use PACKUSWB to shrink and concatenate them into a v4i32 again.
+    SDValue Zeros = getZeroVector(VT, Subtarget, DAG, DL);
+    SDValue Low = DAG.getNode(X86ISD::UNPCKL, DL, VT, V, Zeros);
+    SDValue High = DAG.getNode(X86ISD::UNPCKH, DL, VT, V, Zeros);
+
+    // Do the horizontal sums into two v2i64s.
+    Zeros = getZeroVector(ByteVecVT, Subtarget, DAG, DL);
+    Low = DAG.getNode(X86ISD::PSADBW, DL, ByteVecVT,
+                      DAG.getBitcast(ByteVecVT, Low), Zeros);
+    High = DAG.getNode(X86ISD::PSADBW, DL, ByteVecVT,
+                       DAG.getBitcast(ByteVecVT, High), Zeros);
+
+    // Merge them together.
+    MVT ShortVecVT = MVT::getVectorVT(MVT::i16, VecSize / 16);
+    V = DAG.getNode(X86ISD::PACKUS, DL, ByteVecVT,
+                    DAG.getBitcast(ShortVecVT, Low),
+                    DAG.getBitcast(ShortVecVT, High));
+
+    return DAG.getBitcast(VT, V);
+  }
+
+  // The only element type left is i16.
+  assert(EltVT == MVT::i16 && "Unknown how to handle type");
+
+  // To obtain pop count for each i16 element starting from the pop count for
+  // i8 elements, shift the i16s left by 8, sum as i8s, and then shift as i16s
+  // right by 8. It is important to shift as i16s as i8 vector shift isn't
+  // directly supported.
+  SmallVector<SDValue, 16> Shifters(NumElts, DAG.getConstant(8, DL, EltVT));
+  SDValue Shifter = DAG.getNode(ISD::BUILD_VECTOR, DL, VT, Shifters);
+  SDValue Shl = DAG.getNode(ISD::SHL, DL, VT, DAG.getBitcast(VT, V), Shifter);
+  V = DAG.getNode(ISD::ADD, DL, ByteVecVT, DAG.getBitcast(ByteVecVT, Shl),
+                  DAG.getBitcast(ByteVecVT, V));
+  return DAG.getNode(ISD::SRL, DL, VT, DAG.getBitcast(VT, V), Shifter);
+}
+
+static SDValue LowerVectorCTPOPInRegLUT(SDValue Op, SDLoc DL,
+                                        const X86Subtarget *Subtarget,
+                                        SelectionDAG &DAG) {
+  MVT VT = Op.getSimpleValueType();
+  MVT EltVT = VT.getVectorElementType();
+  unsigned VecSize = VT.getSizeInBits();
 
-  Op = Op.getOperand(0);
-  EVT VT = Op.getValueType();
-  assert((VT.is128BitVector() || VT.is256BitVector()) &&
-         "CTPOP lowering only implemented for 128/256-bit wide vector types");
+  // Implement a lookup table in register by using an algorithm based on:
+  // http://wm.ite.pl/articles/sse-popcount.html
+  //
+  // The general idea is that every lower byte nibble in the input vector is an
+  // index into a in-register pre-computed pop count table. We then split up the
+  // input vector in two new ones: (1) a vector with only the shifted-right
+  // higher nibbles for each byte and (2) a vector with the lower nibbles (and
+  // masked out higher ones) for each byte. PSHUB is used separately with both
+  // to index the in-register table. Next, both are added and the result is a
+  // i8 vector where each element contains the pop count for input byte.
+  //
+  // To obtain the pop count for elements != i8, we follow up with the same
+  // approach and use additional tricks as described below.
+  //
+  const int LUT[16] = {/* 0 */ 0, /* 1 */ 1, /* 2 */ 1, /* 3 */ 2,
+                       /* 4 */ 1, /* 5 */ 2, /* 6 */ 2, /* 7 */ 3,
+                       /* 8 */ 1, /* 9 */ 2, /* a */ 2, /* b */ 3,
+                       /* c */ 2, /* d */ 3, /* e */ 3, /* f */ 4};
+
+  int NumByteElts = VecSize / 8;
+  MVT ByteVecVT = MVT::getVectorVT(MVT::i8, NumByteElts);
+  SDValue In = DAG.getBitcast(ByteVecVT, Op);
+  SmallVector<SDValue, 16> LUTVec;
+  for (int i = 0; i < NumByteElts; ++i)
+    LUTVec.push_back(DAG.getConstant(LUT[i % 16], DL, MVT::i8));
+  SDValue InRegLUT = DAG.getNode(ISD::BUILD_VECTOR, DL, ByteVecVT, LUTVec);
+  SmallVector<SDValue, 16> Mask0F(NumByteElts,
+                                  DAG.getConstant(0x0F, DL, MVT::i8));
+  SDValue M0F = DAG.getNode(ISD::BUILD_VECTOR, DL, ByteVecVT, Mask0F);
+
+  // High nibbles
+  SmallVector<SDValue, 16> Four(NumByteElts, DAG.getConstant(4, DL, MVT::i8));
+  SDValue FourV = DAG.getNode(ISD::BUILD_VECTOR, DL, ByteVecVT, Four);
+  SDValue HighNibbles = DAG.getNode(ISD::SRL, DL, ByteVecVT, In, FourV);
+
+  // Low nibbles
+  SDValue LowNibbles = DAG.getNode(ISD::AND, DL, ByteVecVT, In, M0F);
+
+  // The input vector is used as the shuffle mask that index elements into the
+  // LUT. After counting low and high nibbles, add the vector to obtain the
+  // final pop count per i8 element.
+  SDValue HighPopCnt =
+      DAG.getNode(X86ISD::PSHUFB, DL, ByteVecVT, InRegLUT, HighNibbles);
+  SDValue LowPopCnt =
+      DAG.getNode(X86ISD::PSHUFB, DL, ByteVecVT, InRegLUT, LowNibbles);
+  SDValue PopCnt = DAG.getNode(ISD::ADD, DL, ByteVecVT, HighPopCnt, LowPopCnt);
 
-  unsigned NumElts = VT.getVectorNumElements();
-  EVT EltVT = VT.getVectorElementType();
-  unsigned Len = EltVT.getSizeInBits();
+  if (EltVT == MVT::i8)
+    return PopCnt;
+
+  return LowerHorizontalByteSum(PopCnt, VT, Subtarget, DAG);
+}
+
+static SDValue LowerVectorCTPOPBitmath(SDValue Op, SDLoc DL,
+                                       const X86Subtarget *Subtarget,
+                                       SelectionDAG &DAG) {
+  MVT VT = Op.getSimpleValueType();
+  assert(VT.is128BitVector() &&
+         "Only 128-bit vector bitmath lowering supported.");
+
+  int VecSize = VT.getSizeInBits();
+  MVT EltVT = VT.getVectorElementType();
+  int Len = EltVT.getSizeInBits();
 
   // This is the vectorized version of the "best" algorithm from
   // http://graphics.stanford.edu/~seander/bithacks.html#CountBitsSetParallel
   // with a minor tweak to use a series of adds + shifts instead of vector
-  // multiplications. Implemented for the v2i64, v4i64, v4i32, v8i32 types:
-  //
-  //  v2i64, v4i64, v4i32 => Only profitable w/ popcnt disabled
-  //  v8i32 => Always profitable
-  //
-  // FIXME: There a couple of possible improvements:
-  //
-  // 1) Support for i8 and i16 vectors (needs measurements if popcnt enabled).
-  // 2) Use strategies from http://wm.ite.pl/articles/sse-popcount.html
-  //
-  assert(EltVT.isInteger() && (Len == 32 || Len == 64) && Len % 8 == 0 &&
-         "CTPOP not implemented for this vector element type.");
+  // multiplications. Implemented for all integer vector types. We only use
+  // this when we don't have SSSE3 which allows a LUT-based lowering that is
+  // much faster, even faster than using native popcnt instructions.
+
+  auto GetShift = [&](unsigned OpCode, SDValue V, int Shifter) {
+    MVT VT = V.getSimpleValueType();
+    SmallVector<SDValue, 32> Shifters(
+        VT.getVectorNumElements(),
+        DAG.getConstant(Shifter, DL, VT.getVectorElementType()));
+    return DAG.getNode(OpCode, DL, VT, V,
+                       DAG.getNode(ISD::BUILD_VECTOR, DL, VT, Shifters));
+  };
+  auto GetMask = [&](SDValue V, APInt Mask) {
+    MVT VT = V.getSimpleValueType();
+    SmallVector<SDValue, 32> Masks(
+        VT.getVectorNumElements(),
+        DAG.getConstant(Mask, DL, VT.getVectorElementType()));
+    return DAG.getNode(ISD::AND, DL, VT, V,
+                       DAG.getNode(ISD::BUILD_VECTOR, DL, VT, Masks));
+  };
 
-  // X86 canonicalize ANDs to vXi64, generate the appropriate bitcasts to avoid
-  // extra legalization.
-  bool NeedsBitcast = EltVT == MVT::i32;
-  MVT BitcastVT = VT.is256BitVector() ? MVT::v4i64 : MVT::v2i64;
+  // We don't want to incur the implicit masks required to SRL vNi8 vectors on
+  // x86, so set the SRL type to have elements at least i16 wide. This is
+  // correct because all of our SRLs are followed immediately by a mask anyways
+  // that handles any bits that sneak into the high bits of the byte elements.
+  MVT SrlVT = Len > 8 ? VT : MVT::getVectorVT(MVT::i16, VecSize / 16);
 
-  SDValue Cst55 = DAG.getConstant(APInt::getSplat(Len, APInt(8, 0x55)), EltVT);
-  SDValue Cst33 = DAG.getConstant(APInt::getSplat(Len, APInt(8, 0x33)), EltVT);
-  SDValue Cst0F = DAG.getConstant(APInt::getSplat(Len, APInt(8, 0x0F)), EltVT);
+  SDValue V = Op;
 
   // v = v - ((v >> 1) & 0x55555555...)
-  SmallVector<SDValue, 8> Ones(NumElts, DAG.getConstant(1, EltVT));
-  SDValue OnesV = DAG.getNode(ISD::BUILD_VECTOR, dl, VT, Ones);
-  SDValue Srl = DAG.getNode(ISD::SRL, dl, VT, Op, OnesV);
-  if (NeedsBitcast)
-    Srl = DAG.getNode(ISD::BITCAST, dl, BitcastVT, Srl);
-
-  SmallVector<SDValue, 8> Mask55(NumElts, Cst55);
-  SDValue M55 = DAG.getNode(ISD::BUILD_VECTOR, dl, VT, Mask55);
-  if (NeedsBitcast)
-    M55 = DAG.getNode(ISD::BITCAST, dl, BitcastVT, M55);
-
-  SDValue And = DAG.getNode(ISD::AND, dl, Srl.getValueType(), Srl, M55);
-  if (VT != And.getValueType())
-    And = DAG.getNode(ISD::BITCAST, dl, VT, And);
-  SDValue Sub = DAG.getNode(ISD::SUB, dl, VT, Op, And);
+  SDValue Srl =
+      DAG.getBitcast(VT, GetShift(ISD::SRL, DAG.getBitcast(SrlVT, V), 1));
+  SDValue And = GetMask(Srl, APInt::getSplat(Len, APInt(8, 0x55)));
+  V = DAG.getNode(ISD::SUB, DL, VT, V, And);
 
   // v = (v & 0x33333333...) + ((v >> 2) & 0x33333333...)
-  SmallVector<SDValue, 8> Mask33(NumElts, Cst33);
-  SDValue M33 = DAG.getNode(ISD::BUILD_VECTOR, dl, VT, Mask33);
-  SmallVector<SDValue, 8> Twos(NumElts, DAG.getConstant(2, EltVT));
-  SDValue TwosV = DAG.getNode(ISD::BUILD_VECTOR, dl, VT, Twos);
+  SDValue AndLHS = GetMask(V, APInt::getSplat(Len, APInt(8, 0x33)));
+  Srl = DAG.getBitcast(VT, GetShift(ISD::SRL, DAG.getBitcast(SrlVT, V), 2));
+  SDValue AndRHS = GetMask(Srl, APInt::getSplat(Len, APInt(8, 0x33)));
+  V = DAG.getNode(ISD::ADD, DL, VT, AndLHS, AndRHS);
 
-  Srl = DAG.getNode(ISD::SRL, dl, VT, Sub, TwosV);
-  if (NeedsBitcast) {
-    Srl = DAG.getNode(ISD::BITCAST, dl, BitcastVT, Srl);
-    M33 = DAG.getNode(ISD::BITCAST, dl, BitcastVT, M33);
-    Sub = DAG.getNode(ISD::BITCAST, dl, BitcastVT, Sub);
-  }
+  // v = (v + (v >> 4)) & 0x0F0F0F0F...
+  Srl = DAG.getBitcast(VT, GetShift(ISD::SRL, DAG.getBitcast(SrlVT, V), 4));
+  SDValue Add = DAG.getNode(ISD::ADD, DL, VT, V, Srl);
+  V = GetMask(Add, APInt::getSplat(Len, APInt(8, 0x0F)));
 
-  SDValue AndRHS = DAG.getNode(ISD::AND, dl, M33.getValueType(), Srl, M33);
-  SDValue AndLHS = DAG.getNode(ISD::AND, dl, M33.getValueType(), Sub, M33);
-  if (VT != AndRHS.getValueType()) {
-    AndRHS = DAG.getNode(ISD::BITCAST, dl, VT, AndRHS);
-    AndLHS = DAG.getNode(ISD::BITCAST, dl, VT, AndLHS);
-  }
-  SDValue Add = DAG.getNode(ISD::ADD, dl, VT, AndLHS, AndRHS);
+  // At this point, V contains the byte-wise population count, and we are
+  // merely doing a horizontal sum if necessary to get the wider element
+  // counts.
+  if (EltVT == MVT::i8)
+    return V;
 
-  // v = (v + (v >> 4)) & 0x0F0F0F0F...
-  SmallVector<SDValue, 8> Fours(NumElts, DAG.getConstant(4, EltVT));
-  SDValue FoursV = DAG.getNode(ISD::BUILD_VECTOR, dl, VT, Fours);
-  Srl = DAG.getNode(ISD::SRL, dl, VT, Add, FoursV);
-  Add = DAG.getNode(ISD::ADD, dl, VT, Add, Srl);
-
-  SmallVector<SDValue, 8> Mask0F(NumElts, Cst0F);
-  SDValue M0F = DAG.getNode(ISD::BUILD_VECTOR, dl, VT, Mask0F);
-  if (NeedsBitcast) {
-    Add = DAG.getNode(ISD::BITCAST, dl, BitcastVT, Add);
-    M0F = DAG.getNode(ISD::BITCAST, dl, BitcastVT, M0F);
-  }
-  And = DAG.getNode(ISD::AND, dl, M0F.getValueType(), Add, M0F);
-  if (VT != And.getValueType())
-    And = DAG.getNode(ISD::BITCAST, dl, VT, And);
-
-  // The algorithm mentioned above uses:
-  //    v = (v * 0x01010101...) >> (Len - 8)
-  //
-  // Change it to use vector adds + vector shifts which yield faster results on
-  // Haswell than using vector integer multiplication.
-  //
-  // For i32 elements:
-  //    v = v + (v >> 8)
-  //    v = v + (v >> 16)
-  //
-  // For i64 elements:
-  //    v = v + (v >> 8)
-  //    v = v + (v >> 16)
-  //    v = v + (v >> 32)
-  //
-  Add = And;
-  SmallVector<SDValue, 8> Csts;
-  for (unsigned i = 8; i <= Len/2; i *= 2) {
-    Csts.assign(NumElts, DAG.getConstant(i, EltVT));
-    SDValue CstsV = DAG.getNode(ISD::BUILD_VECTOR, dl, VT, Csts);
-    Srl = DAG.getNode(ISD::SRL, dl, VT, Add, CstsV);
-    Add = DAG.getNode(ISD::ADD, dl, VT, Add, Srl);
-    Csts.clear();
+  return LowerHorizontalByteSum(
+      DAG.getBitcast(MVT::getVectorVT(MVT::i8, VecSize / 8), V), VT, Subtarget,
+      DAG);
+}
+
+static SDValue LowerVectorCTPOP(SDValue Op, const X86Subtarget *Subtarget,
+                                SelectionDAG &DAG) {
+  MVT VT = Op.getSimpleValueType();
+  // FIXME: Need to add AVX-512 support here!
+  assert((VT.is256BitVector() || VT.is128BitVector()) &&
+         "Unknown CTPOP type to handle");
+  SDLoc DL(Op.getNode());
+  SDValue Op0 = Op.getOperand(0);
+
+  if (!Subtarget->hasSSSE3()) {
+    // We can't use the fast LUT approach, so fall back on vectorized bitmath.
+    assert(VT.is128BitVector() && "Only 128-bit vectors supported in SSE!");
+    return LowerVectorCTPOPBitmath(Op0, DL, Subtarget, DAG);
   }
 
-  // The result is on the least significant 6-bits on i32 and 7-bits on i64.
-  SDValue Cst3F = DAG.getConstant(APInt(Len, Len == 32 ? 0x3F : 0x7F), EltVT);
-  SmallVector<SDValue, 8> Cst3FV(NumElts, Cst3F);
-  SDValue M3F = DAG.getNode(ISD::BUILD_VECTOR, dl, VT, Cst3FV);
-  if (NeedsBitcast) {
-    Add = DAG.getNode(ISD::BITCAST, dl, BitcastVT, Add);
-    M3F = DAG.getNode(ISD::BITCAST, dl, BitcastVT, M3F);
+  if (VT.is256BitVector() && !Subtarget->hasInt256()) {
+    unsigned NumElems = VT.getVectorNumElements();
+
+    // Extract each 128-bit vector, compute pop count and concat the result.
+    SDValue LHS = Extract128BitVector(Op0, 0, DAG, DL);
+    SDValue RHS = Extract128BitVector(Op0, NumElems/2, DAG, DL);
+
+    return DAG.getNode(ISD::CONCAT_VECTORS, DL, VT,
+                       LowerVectorCTPOPInRegLUT(LHS, DL, Subtarget, DAG),
+                       LowerVectorCTPOPInRegLUT(RHS, DL, Subtarget, DAG));
   }
-  And = DAG.getNode(ISD::AND, dl, M3F.getValueType(), Add, M3F);
-  if (VT != And.getValueType())
-    And = DAG.getNode(ISD::BITCAST, dl, VT, And);
 
-  return And;
+  return LowerVectorCTPOPInRegLUT(Op0, DL, Subtarget, DAG);
+}
+
+static SDValue LowerCTPOP(SDValue Op, const X86Subtarget *Subtarget,
+                          SelectionDAG &DAG) {
+  assert(Op.getValueType().isVector() &&
+         "We only do custom lowering for vector population count.");
+  return LowerVectorCTPOP(Op, Subtarget, DAG);
 }
 
 static SDValue LowerLOAD_SUB(SDValue Op, SelectionDAG &DAG) {
@@ -17218,7 +18465,7 @@ static SDValue LowerLOAD_SUB(SDValue Op, SelectionDAG &DAG) {
   SDLoc dl(Node);
   EVT T = Node->getValueType(0);
   SDValue negOp = DAG.getNode(ISD::SUB, dl, T,
-                              DAG.getConstant(0, T), Node->getOperand(2));
+                              DAG.getConstant(0, dl, T), Node->getOperand(2));
   return DAG.getAtomic(ISD::ATOMIC_LOAD_ADD, dl,
                        cast<AtomicSDNode>(Node)->getMemoryVT(),
                        Node->getOperand(0),
@@ -17306,7 +18553,8 @@ static SDValue LowerFSINCOS(SDValue Op, const X86Subtarget *Subtarget,
   // the results are returned via SRet in memory.
   const char *LibcallName =  isF64 ? "__sincos_stret" : "__sincosf_stret";
   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
-  SDValue Callee = DAG.getExternalSymbol(LibcallName, TLI.getPointerTy());
+  SDValue Callee =
+      DAG.getExternalSymbol(LibcallName, TLI.getPointerTy(DAG.getDataLayout()));
 
   Type *RetTy = isF64
     ? (Type*)StructType::get(ArgTy, ArgTy, nullptr)
@@ -17324,13 +18572,105 @@ static SDValue LowerFSINCOS(SDValue Op, const X86Subtarget *Subtarget,
 
   // Returned in bits 0:31 and 32:64 xmm0.
   SDValue SinVal = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, ArgVT,
-                               CallResult.first, DAG.getIntPtrConstant(0));
+                               CallResult.first, DAG.getIntPtrConstant(0, dl));
   SDValue CosVal = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, ArgVT,
-                               CallResult.first, DAG.getIntPtrConstant(1));
+                               CallResult.first, DAG.getIntPtrConstant(1, dl));
   SDVTList Tys = DAG.getVTList(ArgVT, ArgVT);
   return DAG.getNode(ISD::MERGE_VALUES, dl, Tys, SinVal, CosVal);
 }
 
+static SDValue LowerMSCATTER(SDValue Op, const X86Subtarget *Subtarget,
+                             SelectionDAG &DAG) {
+  assert(Subtarget->hasAVX512() &&
+         "MGATHER/MSCATTER are supported on AVX-512 arch only");
+
+  MaskedScatterSDNode *N = cast<MaskedScatterSDNode>(Op.getNode());
+  EVT VT = N->getValue().getValueType();
+  assert(VT.getScalarSizeInBits() >= 32 && "Unsupported scatter op");
+  SDLoc dl(Op);
+
+  // X86 scatter kills mask register, so its type should be added to
+  // the list of return values
+  if (N->getNumValues() == 1) {
+    SDValue Index = N->getIndex();
+    if (!Subtarget->hasVLX() && !VT.is512BitVector() &&
+        !Index.getValueType().is512BitVector())
+      Index = DAG.getNode(ISD::SIGN_EXTEND, dl, MVT::v8i64, Index);
+
+    SDVTList VTs = DAG.getVTList(N->getMask().getValueType(), MVT::Other);
+    SDValue Ops[] = { N->getOperand(0), N->getOperand(1),  N->getOperand(2),
+                      N->getOperand(3), Index };
+
+    SDValue NewScatter = DAG.getMaskedScatter(VTs, VT, dl, Ops, N->getMemOperand());
+    DAG.ReplaceAllUsesWith(Op, SDValue(NewScatter.getNode(), 1));
+    return SDValue(NewScatter.getNode(), 0);
+  }
+  return Op;
+}
+
+static SDValue LowerMGATHER(SDValue Op, const X86Subtarget *Subtarget,
+                            SelectionDAG &DAG) {
+  assert(Subtarget->hasAVX512() &&
+         "MGATHER/MSCATTER are supported on AVX-512 arch only");
+
+  MaskedGatherSDNode *N = cast<MaskedGatherSDNode>(Op.getNode());
+  EVT VT = Op.getValueType();
+  assert(VT.getScalarSizeInBits() >= 32 && "Unsupported gather op");
+  SDLoc dl(Op);
+
+  SDValue Index = N->getIndex();
+  if (!Subtarget->hasVLX() && !VT.is512BitVector() &&
+      !Index.getValueType().is512BitVector()) {
+    Index = DAG.getNode(ISD::SIGN_EXTEND, dl, MVT::v8i64, Index);
+    SDValue Ops[] = { N->getOperand(0), N->getOperand(1),  N->getOperand(2),
+                      N->getOperand(3), Index };
+    DAG.UpdateNodeOperands(N, Ops);
+  }
+  return Op;
+}
+
+SDValue X86TargetLowering::LowerGC_TRANSITION_START(SDValue Op,
+                                                    SelectionDAG &DAG) const {
+  // TODO: Eventually, the lowering of these nodes should be informed by or
+  // deferred to the GC strategy for the function in which they appear. For
+  // now, however, they must be lowered to something. Since they are logically
+  // no-ops in the case of a null GC strategy (or a GC strategy which does not
+  // require special handling for these nodes), lower them as literal NOOPs for
+  // the time being.
+  SmallVector<SDValue, 2> Ops;
+
+  Ops.push_back(Op.getOperand(0));
+  if (Op->getGluedNode())
+    Ops.push_back(Op->getOperand(Op->getNumOperands() - 1));
+
+  SDLoc OpDL(Op);
+  SDVTList VTs = DAG.getVTList(MVT::Other, MVT::Glue);
+  SDValue NOOP(DAG.getMachineNode(X86::NOOP, SDLoc(Op), VTs, Ops), 0);
+
+  return NOOP;
+}
+
+SDValue X86TargetLowering::LowerGC_TRANSITION_END(SDValue Op,
+                                                  SelectionDAG &DAG) const {
+  // TODO: Eventually, the lowering of these nodes should be informed by or
+  // deferred to the GC strategy for the function in which they appear. For
+  // now, however, they must be lowered to something. Since they are logically
+  // no-ops in the case of a null GC strategy (or a GC strategy which does not
+  // require special handling for these nodes), lower them as literal NOOPs for
+  // the time being.
+  SmallVector<SDValue, 2> Ops;
+
+  Ops.push_back(Op.getOperand(0));
+  if (Op->getGluedNode())
+    Ops.push_back(Op->getOperand(Op->getNumOperands() - 1));
+
+  SDLoc OpDL(Op);
+  SDVTList VTs = DAG.getVTList(MVT::Other, MVT::Glue);
+  SDValue NOOP(DAG.getMachineNode(X86::NOOP, SDLoc(Op), VTs, Ops), 0);
+
+  return NOOP;
+}
+
 /// LowerOperation - Provide custom lowering hooks for some operations.
 ///
 SDValue X86TargetLowering::LowerOperation(SDValue Op, SelectionDAG &DAG) const {
@@ -17365,6 +18705,8 @@ SDValue X86TargetLowering::LowerOperation(SDValue Op, SelectionDAG &DAG) const {
   case ISD::ZERO_EXTEND:        return LowerZERO_EXTEND(Op, Subtarget, DAG);
   case ISD::SIGN_EXTEND:        return LowerSIGN_EXTEND(Op, Subtarget, DAG);
   case ISD::ANY_EXTEND:         return LowerANY_EXTEND(Op, Subtarget, DAG);
+  case ISD::SIGN_EXTEND_VECTOR_INREG:
+    return LowerSIGN_EXTEND_VECTOR_INREG(Op, Subtarget, DAG);
   case ISD::FP_TO_SINT:         return LowerFP_TO_SINT(Op, DAG);
   case ISD::FP_TO_UINT:         return LowerFP_TO_UINT(Op, DAG);
   case ISD::FP_EXTEND:          return LowerFP_EXTEND(Op, DAG);
@@ -17418,6 +18760,11 @@ SDValue X86TargetLowering::LowerOperation(SDValue Op, SelectionDAG &DAG) const {
   case ISD::ADD:                return LowerADD(Op, DAG);
   case ISD::SUB:                return LowerSUB(Op, DAG);
   case ISD::FSINCOS:            return LowerFSINCOS(Op, Subtarget, DAG);
+  case ISD::MGATHER:            return LowerMGATHER(Op, Subtarget, DAG);
+  case ISD::MSCATTER:           return LowerMSCATTER(Op, Subtarget, DAG);
+  case ISD::GC_TRANSITION_START:
+                                return LowerGC_TRANSITION_START(Op, DAG);
+  case ISD::GC_TRANSITION_END:  return LowerGC_TRANSITION_END(Op, DAG);
   }
 }
 
@@ -17465,6 +18812,11 @@ void X86TargetLowering::ReplaceNodeResults(SDNode *N,
     return;
   }
   case ISD::FP_TO_SINT:
+    // FP_TO_INT*_IN_MEM is not legal for f16 inputs.  Do not convert
+    // (FP_TO_SINT (load f16)) to FP_TO_INT*.
+    if (N->getOperand(0).getValueType() == MVT::f16)
+      break;
+    // fallthrough
   case ISD::FP_TO_UINT: {
     bool IsSigned = N->getOpcode() == ISD::FP_TO_SINT;
 
@@ -17493,12 +18845,12 @@ void X86TargetLowering::ReplaceNodeResults(SDNode *N,
       return;
     SDValue ZExtIn = DAG.getNode(ISD::ZERO_EXTEND, dl, MVT::v2i64,
                                  N->getOperand(0));
-    SDValue Bias = DAG.getConstantFP(BitsToDouble(0x4330000000000000ULL),
+    SDValue Bias = DAG.getConstantFP(BitsToDouble(0x4330000000000000ULL), dl,
                                      MVT::f64);
     SDValue VBias = DAG.getNode(ISD::BUILD_VECTOR, dl, MVT::v2f64, Bias, Bias);
     SDValue Or = DAG.getNode(ISD::OR, dl, MVT::v2i64, ZExtIn,
-                             DAG.getNode(ISD::BITCAST, dl, MVT::v2i64, VBias));
-    Or = DAG.getNode(ISD::BITCAST, dl, MVT::v2f64, Or);
+                             DAG.getBitcast(MVT::v2i64, VBias));
+    Or = DAG.getBitcast(MVT::v2f64, Or);
     SDValue Sub = DAG.getNode(ISD::FSUB, dl, MVT::v2f64, Or, VBias);
     Results.push_back(DAG.getNode(X86ISD::VFPROUND, dl, MVT::v4f32, Sub));
     return;
@@ -17510,6 +18862,13 @@ void X86TargetLowering::ReplaceNodeResults(SDNode *N,
     Results.push_back(V);
     return;
   }
+  case ISD::FP_EXTEND: {
+    // Right now, only MVT::v2f32 has OperationAction for FP_EXTEND.
+    // No other ValueType for FP_EXTEND should reach this point.
+    assert(N->getValueType(0) == MVT::v2f32 &&
+           "Do not know how to legalize this Node");
+    return;
+  }
   case ISD::INTRINSIC_W_CHAIN: {
     unsigned IntNo = cast<ConstantSDNode>(N->getOperand(1))->getZExtValue();
     switch (IntNo) {
@@ -17536,9 +18895,9 @@ void X86TargetLowering::ReplaceNodeResults(SDNode *N,
     EVT HalfT = Regs64bit ? MVT::i64 : MVT::i32;
     SDValue cpInL, cpInH;
     cpInL = DAG.getNode(ISD::EXTRACT_ELEMENT, dl, HalfT, N->getOperand(2),
-                        DAG.getConstant(0, HalfT));
+                        DAG.getConstant(0, dl, HalfT));
     cpInH = DAG.getNode(ISD::EXTRACT_ELEMENT, dl, HalfT, N->getOperand(2),
-                        DAG.getConstant(1, HalfT));
+                        DAG.getConstant(1, dl, HalfT));
     cpInL = DAG.getCopyToReg(N->getOperand(0), dl,
                              Regs64bit ? X86::RAX : X86::EAX,
                              cpInL, SDValue());
@@ -17547,9 +18906,9 @@ void X86TargetLowering::ReplaceNodeResults(SDNode *N,
                              cpInH, cpInL.getValue(1));
     SDValue swapInL, swapInH;
     swapInL = DAG.getNode(ISD::EXTRACT_ELEMENT, dl, HalfT, N->getOperand(3),
-                          DAG.getConstant(0, HalfT));
+                          DAG.getConstant(0, dl, HalfT));
     swapInH = DAG.getNode(ISD::EXTRACT_ELEMENT, dl, HalfT, N->getOperand(3),
-                          DAG.getConstant(1, HalfT));
+                          DAG.getConstant(1, dl, HalfT));
     swapInL = DAG.getCopyToReg(cpInH.getValue(0), dl,
                                Regs64bit ? X86::RBX : X86::EBX,
                                swapInL, cpInH.getValue(1));
@@ -17576,7 +18935,7 @@ void X86TargetLowering::ReplaceNodeResults(SDNode *N,
                                         MVT::i32, cpOutH.getValue(2));
     SDValue Success =
         DAG.getNode(X86ISD::SETCC, dl, MVT::i8,
-                    DAG.getConstant(X86::COND_E, MVT::i8), EFLAGS);
+                    DAG.getConstant(X86::COND_E, dl, MVT::i8), EFLAGS);
     Success = DAG.getZExtOrTrunc(Success, dl, N->getValueType(1));
 
     Results.push_back(DAG.getNode(ISD::BUILD_PAIR, dl, T, OpsF));
@@ -17614,7 +18973,7 @@ void X86TargetLowering::ReplaceNodeResults(SDNode *N,
     EVT WiderVT = EVT::getVectorVT(*DAG.getContext(), SVT, NumElts * 2);
     SDValue Expanded = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl,
                                    MVT::v2f64, N->getOperand(0));
-    SDValue ToVecInt = DAG.getNode(ISD::BITCAST, dl, WiderVT, Expanded);
+    SDValue ToVecInt = DAG.getBitcast(WiderVT, Expanded);
 
     if (ExperimentalVectorWideningLegalization) {
       // If we are legalizing vectors by widening, we already have the desired
@@ -17626,7 +18985,7 @@ void X86TargetLowering::ReplaceNodeResults(SDNode *N,
     SmallVector<SDValue, 8> Elts;
     for (unsigned i = 0, e = NumElts; i != e; ++i)
       Elts.push_back(DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, SVT,
-                                   ToVecInt, DAG.getIntPtrConstant(i)));
+                                   ToVecInt, DAG.getIntPtrConstant(i, dl)));
 
     Results.push_back(DAG.getNode(ISD::BUILD_VECTOR, dl, DstVT, Elts));
   }
@@ -17634,8 +18993,8 @@ void X86TargetLowering::ReplaceNodeResults(SDNode *N,
 }
 
 const char *X86TargetLowering::getTargetNodeName(unsigned Opcode) const {
-  switch (Opcode) {
-  default: return nullptr;
+  switch ((X86ISD::NodeType)Opcode) {
+  case X86ISD::FIRST_NUMBER:       break;
   case X86ISD::BSF:                return "X86ISD::BSF";
   case X86ISD::BSR:                return "X86ISD::BSR";
   case X86ISD::SHLD:               return "X86ISD::SHLD";
@@ -17644,7 +19003,6 @@ const char *X86TargetLowering::getTargetNodeName(unsigned Opcode) const {
   case X86ISD::FANDN:              return "X86ISD::FANDN";
   case X86ISD::FOR:                return "X86ISD::FOR";
   case X86ISD::FXOR:               return "X86ISD::FXOR";
-  case X86ISD::FSRL:               return "X86ISD::FSRL";
   case X86ISD::FILD:               return "X86ISD::FILD";
   case X86ISD::FILD_FLAG:          return "X86ISD::FILD_FLAG";
   case X86ISD::FP_TO_INT16_IN_MEM: return "X86ISD::FP_TO_INT16_IN_MEM";
@@ -17662,9 +19020,11 @@ const char *X86TargetLowering::getTargetNodeName(unsigned Opcode) const {
   case X86ISD::UCOMI:              return "X86ISD::UCOMI";
   case X86ISD::CMPM:               return "X86ISD::CMPM";
   case X86ISD::CMPMU:              return "X86ISD::CMPMU";
+  case X86ISD::CMPM_RND:           return "X86ISD::CMPM_RND";
   case X86ISD::SETCC:              return "X86ISD::SETCC";
   case X86ISD::SETCC_CARRY:        return "X86ISD::SETCC_CARRY";
   case X86ISD::FSETCC:             return "X86ISD::FSETCC";
+  case X86ISD::FGETSIGNx86:        return "X86ISD::FGETSIGNx86";
   case X86ISD::CMOV:               return "X86ISD::CMOV";
   case X86ISD::BRCOND:             return "X86ISD::BRCOND";
   case X86ISD::RET_FLAG:           return "X86ISD::RET_FLAG";
@@ -17673,31 +19033,37 @@ const char *X86TargetLowering::getTargetNodeName(unsigned Opcode) const {
   case X86ISD::GlobalBaseReg:      return "X86ISD::GlobalBaseReg";
   case X86ISD::Wrapper:            return "X86ISD::Wrapper";
   case X86ISD::WrapperRIP:         return "X86ISD::WrapperRIP";
+  case X86ISD::MOVDQ2Q:            return "X86ISD::MOVDQ2Q";
+  case X86ISD::MMX_MOVD2W:         return "X86ISD::MMX_MOVD2W";
+  case X86ISD::MMX_MOVW2D:         return "X86ISD::MMX_MOVW2D";
   case X86ISD::PEXTRB:             return "X86ISD::PEXTRB";
   case X86ISD::PEXTRW:             return "X86ISD::PEXTRW";
   case X86ISD::INSERTPS:           return "X86ISD::INSERTPS";
   case X86ISD::PINSRB:             return "X86ISD::PINSRB";
   case X86ISD::PINSRW:             return "X86ISD::PINSRW";
+  case X86ISD::MMX_PINSRW:         return "X86ISD::MMX_PINSRW";
   case X86ISD::PSHUFB:             return "X86ISD::PSHUFB";
   case X86ISD::ANDNP:              return "X86ISD::ANDNP";
   case X86ISD::PSIGN:              return "X86ISD::PSIGN";
   case X86ISD::BLENDI:             return "X86ISD::BLENDI";
   case X86ISD::SHRUNKBLEND:        return "X86ISD::SHRUNKBLEND";
+  case X86ISD::ADDUS:              return "X86ISD::ADDUS";
   case X86ISD::SUBUS:              return "X86ISD::SUBUS";
   case X86ISD::HADD:               return "X86ISD::HADD";
   case X86ISD::HSUB:               return "X86ISD::HSUB";
   case X86ISD::FHADD:              return "X86ISD::FHADD";
   case X86ISD::FHSUB:              return "X86ISD::FHSUB";
-  case X86ISD::UMAX:               return "X86ISD::UMAX";
-  case X86ISD::UMIN:               return "X86ISD::UMIN";
-  case X86ISD::SMAX:               return "X86ISD::SMAX";
-  case X86ISD::SMIN:               return "X86ISD::SMIN";
+  case X86ISD::ABS:                return "X86ISD::ABS";
   case X86ISD::FMAX:               return "X86ISD::FMAX";
+  case X86ISD::FMAX_RND:           return "X86ISD::FMAX_RND";
   case X86ISD::FMIN:               return "X86ISD::FMIN";
+  case X86ISD::FMIN_RND:           return "X86ISD::FMIN_RND";
   case X86ISD::FMAXC:              return "X86ISD::FMAXC";
   case X86ISD::FMINC:              return "X86ISD::FMINC";
   case X86ISD::FRSQRT:             return "X86ISD::FRSQRT";
   case X86ISD::FRCP:               return "X86ISD::FRCP";
+  case X86ISD::EXTRQI:             return "X86ISD::EXTRQI";
+  case X86ISD::INSERTQI:           return "X86ISD::INSERTQI";
   case X86ISD::TLSADDR:            return "X86ISD::TLSADDR";
   case X86ISD::TLSBASEADDR:        return "X86ISD::TLSBASEADDR";
   case X86ISD::TLSCALL:            return "X86ISD::TLSCALL";
@@ -17715,10 +19081,13 @@ const char *X86TargetLowering::getTargetNodeName(unsigned Opcode) const {
   case X86ISD::VZEXT:              return "X86ISD::VZEXT";
   case X86ISD::VSEXT:              return "X86ISD::VSEXT";
   case X86ISD::VTRUNC:             return "X86ISD::VTRUNC";
-  case X86ISD::VTRUNCM:            return "X86ISD::VTRUNCM";
+  case X86ISD::VTRUNCS:            return "X86ISD::VTRUNCS";
+  case X86ISD::VTRUNCUS:           return "X86ISD::VTRUNCUS";
   case X86ISD::VINSERT:            return "X86ISD::VINSERT";
   case X86ISD::VFPEXT:             return "X86ISD::VFPEXT";
   case X86ISD::VFPROUND:           return "X86ISD::VFPROUND";
+  case X86ISD::CVTDQ2PD:           return "X86ISD::CVTDQ2PD";
+  case X86ISD::CVTUDQ2PD:          return "X86ISD::CVTUDQ2PD";
   case X86ISD::VSHLDQ:             return "X86ISD::VSHLDQ";
   case X86ISD::VSRLDQ:             return "X86ISD::VSRLDQ";
   case X86ISD::VSHL:               return "X86ISD::VSHL";
@@ -17762,6 +19131,7 @@ const char *X86TargetLowering::getTargetNodeName(unsigned Opcode) const {
   case X86ISD::PSHUFHW:            return "X86ISD::PSHUFHW";
   case X86ISD::PSHUFLW:            return "X86ISD::PSHUFLW";
   case X86ISD::SHUFP:              return "X86ISD::SHUFP";
+  case X86ISD::SHUF128:            return "X86ISD::SHUF128";
   case X86ISD::MOVLHPS:            return "X86ISD::MOVLHPS";
   case X86ISD::MOVLHPD:            return "X86ISD::MOVLHPD";
   case X86ISD::MOVHLPS:            return "X86ISD::MOVHLPS";
@@ -17775,31 +19145,48 @@ const char *X86TargetLowering::getTargetNodeName(unsigned Opcode) const {
   case X86ISD::UNPCKL:             return "X86ISD::UNPCKL";
   case X86ISD::UNPCKH:             return "X86ISD::UNPCKH";
   case X86ISD::VBROADCAST:         return "X86ISD::VBROADCAST";
-  case X86ISD::VBROADCASTM:        return "X86ISD::VBROADCASTM";
+  case X86ISD::SUBV_BROADCAST:     return "X86ISD::SUBV_BROADCAST";
   case X86ISD::VEXTRACT:           return "X86ISD::VEXTRACT";
+  case X86ISD::VPERMILPV:          return "X86ISD::VPERMILPV";
   case X86ISD::VPERMILPI:          return "X86ISD::VPERMILPI";
   case X86ISD::VPERM2X128:         return "X86ISD::VPERM2X128";
   case X86ISD::VPERMV:             return "X86ISD::VPERMV";
   case X86ISD::VPERMV3:            return "X86ISD::VPERMV3";
   case X86ISD::VPERMIV3:           return "X86ISD::VPERMIV3";
   case X86ISD::VPERMI:             return "X86ISD::VPERMI";
+  case X86ISD::VFIXUPIMM:          return "X86ISD::VFIXUPIMM";
+  case X86ISD::VRANGE:             return "X86ISD::VRANGE";
   case X86ISD::PMULUDQ:            return "X86ISD::PMULUDQ";
   case X86ISD::PMULDQ:             return "X86ISD::PMULDQ";
+  case X86ISD::PSADBW:             return "X86ISD::PSADBW";
   case X86ISD::VASTART_SAVE_XMM_REGS: return "X86ISD::VASTART_SAVE_XMM_REGS";
   case X86ISD::VAARG_64:           return "X86ISD::VAARG_64";
   case X86ISD::WIN_ALLOCA:         return "X86ISD::WIN_ALLOCA";
   case X86ISD::MEMBARRIER:         return "X86ISD::MEMBARRIER";
+  case X86ISD::MFENCE:             return "X86ISD::MFENCE";
+  case X86ISD::SFENCE:             return "X86ISD::SFENCE";
+  case X86ISD::LFENCE:             return "X86ISD::LFENCE";
   case X86ISD::SEG_ALLOCA:         return "X86ISD::SEG_ALLOCA";
   case X86ISD::WIN_FTOL:           return "X86ISD::WIN_FTOL";
   case X86ISD::SAHF:               return "X86ISD::SAHF";
   case X86ISD::RDRAND:             return "X86ISD::RDRAND";
   case X86ISD::RDSEED:             return "X86ISD::RDSEED";
+  case X86ISD::VPMADDUBSW:         return "X86ISD::VPMADDUBSW";
+  case X86ISD::VPMADDWD:           return "X86ISD::VPMADDWD";
   case X86ISD::FMADD:              return "X86ISD::FMADD";
   case X86ISD::FMSUB:              return "X86ISD::FMSUB";
   case X86ISD::FNMADD:             return "X86ISD::FNMADD";
   case X86ISD::FNMSUB:             return "X86ISD::FNMSUB";
   case X86ISD::FMADDSUB:           return "X86ISD::FMADDSUB";
   case X86ISD::FMSUBADD:           return "X86ISD::FMSUBADD";
+  case X86ISD::FMADD_RND:          return "X86ISD::FMADD_RND";
+  case X86ISD::FNMADD_RND:         return "X86ISD::FNMADD_RND";
+  case X86ISD::FMSUB_RND:          return "X86ISD::FMSUB_RND";
+  case X86ISD::FNMSUB_RND:         return "X86ISD::FNMSUB_RND";
+  case X86ISD::FMADDSUB_RND:       return "X86ISD::FMADDSUB_RND";
+  case X86ISD::FMSUBADD_RND:       return "X86ISD::FMSUBADD_RND";
+  case X86ISD::VRNDSCALE:          return "X86ISD::VRNDSCALE";
+  case X86ISD::VREDUCE:            return "X86ISD::VREDUCE";
   case X86ISD::PCMPESTRI:          return "X86ISD::PCMPESTRI";
   case X86ISD::PCMPISTRI:          return "X86ISD::PCMPISTRI";
   case X86ISD::XTEST:              return "X86ISD::XTEST";
@@ -17808,18 +19195,32 @@ const char *X86TargetLowering::getTargetNodeName(unsigned Opcode) const {
   case X86ISD::SELECT:             return "X86ISD::SELECT";
   case X86ISD::ADDSUB:             return "X86ISD::ADDSUB";
   case X86ISD::RCP28:              return "X86ISD::RCP28";
+  case X86ISD::EXP2:               return "X86ISD::EXP2";
   case X86ISD::RSQRT28:            return "X86ISD::RSQRT28";
   case X86ISD::FADD_RND:           return "X86ISD::FADD_RND";
   case X86ISD::FSUB_RND:           return "X86ISD::FSUB_RND";
   case X86ISD::FMUL_RND:           return "X86ISD::FMUL_RND";
   case X86ISD::FDIV_RND:           return "X86ISD::FDIV_RND";
+  case X86ISD::FSQRT_RND:          return "X86ISD::FSQRT_RND";
+  case X86ISD::FGETEXP_RND:        return "X86ISD::FGETEXP_RND";
+  case X86ISD::SCALEF:             return "X86ISD::SCALEF";
+  case X86ISD::ADDS:               return "X86ISD::ADDS";
+  case X86ISD::SUBS:               return "X86ISD::SUBS";
+  case X86ISD::AVG:                return "X86ISD::AVG";
+  case X86ISD::MULHRS:             return "X86ISD::MULHRS";
+  case X86ISD::SINT_TO_FP_RND:     return "X86ISD::SINT_TO_FP_RND";
+  case X86ISD::UINT_TO_FP_RND:     return "X86ISD::UINT_TO_FP_RND";
+  case X86ISD::FP_TO_SINT_RND:     return "X86ISD::FP_TO_SINT_RND";
+  case X86ISD::FP_TO_UINT_RND:     return "X86ISD::FP_TO_UINT_RND";
   }
+  return nullptr;
 }
 
 // isLegalAddressingMode - Return true if the addressing mode represented
 // by AM is legal for this target, for a load/store of the specified type.
-bool X86TargetLowering::isLegalAddressingMode(const AddrMode &AM,
-                                              Type *Ty) const {
+bool X86TargetLowering::isLegalAddressingMode(const DataLayout &DL,
+                                              const AddrMode &AM, Type *Ty,
+                                              unsigned AS) const {
   // X86 supports extremely general addressing modes.
   CodeModel::Model M = getTargetMachine().getCodeModel();
   Reloc::Model R = getTargetMachine().getRelocationModel();
@@ -17965,7 +19366,7 @@ bool X86TargetLowering::isVectorLoadExtDesirable(SDValue) const { return true; }
 
 bool
 X86TargetLowering::isFMAFasterThanFMulAndFAdd(EVT VT) const {
-  if (!(Subtarget->hasFMA() || Subtarget->hasFMA4()))
+  if (!(Subtarget->hasFMA() || Subtarget->hasFMA4() || Subtarget->hasAVX512()))
     return false;
 
   VT = VT.getScalarType();
@@ -17999,6 +19400,10 @@ X86TargetLowering::isShuffleMaskLegal(const SmallVectorImpl<int> &M,
   if (!VT.isSimple())
     return false;
 
+  // Not for i1 vectors
+  if (VT.getScalarType() == MVT::i1)
+    return false;
+
   // Very little shuffling can be done for 64-bit vectors right now.
   if (VT.getSizeInBits() == 64)
     return false;
@@ -18206,12 +19611,10 @@ X86TargetLowering::EmitVAARG64WithCustomInserter(MachineInstr *MI,
   MachineInstr::mmo_iterator MMOBegin = MI->memoperands_begin();
   MachineInstr::mmo_iterator MMOEnd = MI->memoperands_end();
 
-  bool IsNaCl = Subtarget->isTargetNaCl(); // @LOCALMOD
   // Machine Information
   const TargetInstrInfo *TII = Subtarget->getInstrInfo();
   MachineRegisterInfo &MRI = MBB->getParent()->getRegInfo();
-  const TargetRegisterClass *AddrRegClass =
-    getRegClassFor(getPointerTy()); // @LOCALMOD
+  const TargetRegisterClass *AddrRegClass = getRegClassFor(MVT::i64);
   const TargetRegisterClass *OffsetRegClass = getRegClassFor(MVT::i32);
   DebugLoc DL = MI->getDebugLoc();
 
@@ -18319,39 +19722,29 @@ X86TargetLowering::EmitVAARG64WithCustomInserter(MachineInstr *MI,
   }
 
   // In offsetMBB, emit code to use the reg_save_area.
-  unsigned Opc; // @LOCALMOD
   if (offsetMBB) {
     assert(OffsetReg != 0);
 
     // Read the reg_save_area address.
     unsigned RegSaveReg = MRI.createVirtualRegister(AddrRegClass);
-    Opc = IsNaCl ? X86::MOV32rm : X86::MOV64rm; // @LOCALMOD
-    BuildMI(offsetMBB, DL, TII->get(Opc), RegSaveReg) // @LOCALMOD
+    BuildMI(offsetMBB, DL, TII->get(X86::MOV64rm), RegSaveReg)
       .addOperand(Base)
       .addOperand(Scale)
       .addOperand(Index)
-      .addDisp(Disp, 8+TD->getPointerSize(0)) // @LOCALMOD
+      .addDisp(Disp, 16)
       .addOperand(Segment)
       .setMemRefs(MMOBegin, MMOEnd);
 
     // Zero-extend the offset
-    // @LOCALMOD-BEGIN
-    unsigned OffsetRegExt;
-    if (IsNaCl) {
-      OffsetRegExt = OffsetReg;
-    } else {
-      OffsetRegExt = MRI.createVirtualRegister(AddrRegClass);
-        BuildMI(offsetMBB, DL, TII->get(X86::SUBREG_TO_REG), OffsetRegExt)
-          .addImm(0)
-          .addReg(OffsetReg)
-          .addImm(X86::sub_32bit);
-    }
-    // @LOCALMOD-END
+    unsigned OffsetReg64 = MRI.createVirtualRegister(AddrRegClass);
+      BuildMI(offsetMBB, DL, TII->get(X86::SUBREG_TO_REG), OffsetReg64)
+        .addImm(0)
+        .addReg(OffsetReg)
+        .addImm(X86::sub_32bit);
 
     // Add the offset to the reg_save_area to get the final address.
-    Opc = IsNaCl ? X86::ADD32rr : X86::ADD64rr; // @LOCALMOD
-    BuildMI(offsetMBB, DL, TII->get(Opc), OffsetDestReg)
-      .addReg(OffsetRegExt) // @LOCALMOD
+    BuildMI(offsetMBB, DL, TII->get(X86::ADD64rr), OffsetDestReg)
+      .addReg(OffsetReg64)
       .addReg(RegSaveReg);
 
     // Compute the offset for the next argument
@@ -18381,8 +19774,7 @@ X86TargetLowering::EmitVAARG64WithCustomInserter(MachineInstr *MI,
 
   // Load the overflow_area address into a register.
   unsigned OverflowAddrReg = MRI.createVirtualRegister(AddrRegClass);
-  Opc = IsNaCl ? X86::MOV32rm : X86::MOV64rm; // @LOCALMOD
-  BuildMI(overflowMBB, DL, TII->get(Opc), OverflowAddrReg)
+  BuildMI(overflowMBB, DL, TII->get(X86::MOV64rm), OverflowAddrReg)
     .addOperand(Base)
     .addOperand(Scale)
     .addOperand(Index)
@@ -18398,13 +19790,11 @@ X86TargetLowering::EmitVAARG64WithCustomInserter(MachineInstr *MI,
     unsigned TmpReg = MRI.createVirtualRegister(AddrRegClass);
 
     // aligned_addr = (addr + (align-1)) & ~(align-1)
-    Opc = IsNaCl ? X86::ADD32ri : X86::ADD64ri32; // @LOCALMOD
-    BuildMI(overflowMBB, DL, TII->get(Opc), TmpReg)
+    BuildMI(overflowMBB, DL, TII->get(X86::ADD64ri32), TmpReg)
       .addReg(OverflowAddrReg)
       .addImm(Align-1);
 
-    Opc = IsNaCl ? X86::AND32ri : X86::AND64ri32; // @LOCALMOD
-    BuildMI(overflowMBB, DL, TII->get(Opc), OverflowDestReg)
+    BuildMI(overflowMBB, DL, TII->get(X86::AND64ri32), OverflowDestReg)
       .addReg(TmpReg)
       .addImm(~(uint64_t)(Align-1));
   } else {
@@ -18415,14 +19805,12 @@ X86TargetLowering::EmitVAARG64WithCustomInserter(MachineInstr *MI,
   // Compute the next overflow address after this argument.
   // (the overflow address should be kept 8-byte aligned)
   unsigned NextAddrReg = MRI.createVirtualRegister(AddrRegClass);
-  Opc = IsNaCl ? X86::ADD32ri : X86::ADD64ri32; // @LOCALMOD
-  BuildMI(overflowMBB, DL, TII->get(Opc), NextAddrReg)
+  BuildMI(overflowMBB, DL, TII->get(X86::ADD64ri32), NextAddrReg)
     .addReg(OverflowDestReg)
     .addImm(ArgSizeA8);
 
   // Store the new overflow address.
-  Opc = IsNaCl ? X86::MOV32mr : X86::MOV64mr; // @LOCALMOD
-  BuildMI(overflowMBB, DL, TII->get(Opc))
+  BuildMI(overflowMBB, DL, TII->get(X86::MOV64mr))
     .addOperand(Base)
     .addOperand(Scale)
     .addOperand(Index)
@@ -18783,7 +20171,7 @@ X86TargetLowering::EmitLoweredSegAlloca(MachineInstr *MI,
 
   MachineRegisterInfo &MRI = MF->getRegInfo();
   const TargetRegisterClass *AddrRegClass =
-    getRegClassFor(getPointerTy());
+      getRegClassFor(getPointerTy(MF->getDataLayout()));
 
   unsigned mallocPtrVReg = MRI.createVirtualRegister(AddrRegClass),
     bumpSPPtrVReg = MRI.createVirtualRegister(AddrRegClass),
@@ -18884,31 +20272,13 @@ X86TargetLowering::EmitLoweredWinAlloca(MachineInstr *MI,
 
   assert(!Subtarget->isTargetMachO());
 
-  X86FrameLowering::emitStackProbeCall(*BB->getParent(), *BB, MI, DL);
+  Subtarget->getFrameLowering()->emitStackProbeCall(*BB->getParent(), *BB, MI,
+                                                    DL);
 
   MI->eraseFromParent();   // The pseudo instruction is gone now.
   return BB;
 }
 
-// @LOCALMOD-BEGIN
-MachineBasicBlock *
-X86TargetLowering::EmitLoweredThreadPointerFromGs(MachineInstr *MI,
-                                                  MachineBasicBlock *BB) const {
-  const TargetInstrInfo *TII = BB->getParent()->getSubtarget().getInstrInfo();
-  DebugLoc DL = MI->getDebugLoc();
-  // This generates "movl %gs:0, %DEST", which fetches the thread
-  // pointer on x86-32.
-  BuildMI(*BB, MI, DL, TII->get(X86::MOV32rm), MI->getOperand(0).getReg())
-    .addReg(/*Base=*/0)
-    .addImm(/*Scale=*/1)
-    .addReg(/*IndexReg=*/0)
-    .addImm(/*Disp=*/0)
-    .addReg(/*Segment=*/X86::GS);
-  MI->eraseFromParent();
-  return BB;
-}
-// @LOCALMOD-END
-
 MachineBasicBlock *
 X86TargetLowering::EmitLoweredTLSCall(MachineInstr *MI,
                                       MachineBasicBlock *BB) const {
@@ -18996,7 +20366,7 @@ X86TargetLowering::emitEHSjLjSetJmp(MachineInstr *MI,
 
   MemOpndSlot = CurOp;
 
-  MVT PVT = getPointerTy();
+  MVT PVT = getPointerTy(MF->getDataLayout());
   assert((PVT == MVT::i64 || PVT == MVT::i32) &&
          "Invalid Pointer Size!");
 
@@ -19128,7 +20498,7 @@ X86TargetLowering::emitEHSjLjLongJmp(MachineInstr *MI,
   MachineInstr::mmo_iterator MMOBegin = MI->memoperands_begin();
   MachineInstr::mmo_iterator MMOEnd = MI->memoperands_end();
 
-  MVT PVT = getPointerTy();
+  MVT PVT = getPointerTy(MF->getDataLayout());
   assert((PVT == MVT::i64 || PVT == MVT::i32) &&
          "Invalid Pointer Size!");
 
@@ -19181,6 +20551,7 @@ X86TargetLowering::emitEHSjLjLongJmp(MachineInstr *MI,
 // Replace 213-type (isel default) FMA3 instructions with 231-type for
 // accumulator loops. Writing back to the accumulator allows the coalescer
 // to remove extra copies in the loop.
+// FIXME: Do this on AVX512.  We don't support 231 variants yet (PR23937).
 MachineBasicBlock *
 X86TargetLowering::emitFMA3Instr(MachineInstr *MI,
                                  MachineBasicBlock *MBB) const {
@@ -19291,10 +20662,6 @@ X86TargetLowering::EmitInstrWithCustomInserter(MachineInstr *MI,
   case X86::SEG_ALLOCA_32:
   case X86::SEG_ALLOCA_64:
     return EmitLoweredSegAlloca(MI, BB);
-  // @LOCALMOD-BEGIN
-  case X86::THREAD_POINTER_FROM_GS:
-    return EmitLoweredThreadPointerFromGs(MI, BB);
-  // @LOCALMOD-END
   case X86::TLSCall_32:
   case X86::TLSCall_64:
     return EmitLoweredTLSCall(MI, BB);
@@ -19315,6 +20682,10 @@ X86TargetLowering::EmitInstrWithCustomInserter(MachineInstr *MI,
   case X86::CMOV_RFP32:
   case X86::CMOV_RFP64:
   case X86::CMOV_RFP80:
+  case X86::CMOV_V8I1:
+  case X86::CMOV_V16I1:
+  case X86::CMOV_V32I1:
+  case X86::CMOV_V64I1:
     return EmitLoweredSelect(MI, BB);
 
   case X86::FP32_TO_INT16_IN_MEM:
@@ -19438,7 +20809,6 @@ X86TargetLowering::EmitInstrWithCustomInserter(MachineInstr *MI,
     return EmitVAStartSaveXMMRegsWithCustomInserter(MI, BB);
 
   case X86::VAARG_64:
-  case X86::NACL_CG_VAARG_64: // @LOCALMOD
     return EmitVAARG64WithCustomInserter(MI, BB);
 
   case X86::EH_SjLj_SetJmp32:
@@ -19687,7 +21057,7 @@ static SDValue PerformShuffleCombine256(SDNode *N, SelectionDAG &DAG,
                                  SDValue(ResNode.getNode(), 1));
         }
 
-        return DAG.getNode(ISD::BITCAST, dl, VT, ResNode);
+        return DAG.getBitcast(VT, ResNode);
       }
     }
 
@@ -19746,7 +21116,7 @@ static bool combineX86ShuffleChain(SDValue Op, SDValue Root, ArrayRef<int> Mask,
 
   // Just remove no-op shuffle masks.
   if (Mask.size() == 1) {
-    DCI.CombineTo(Root.getNode(), DAG.getNode(ISD::BITCAST, DL, RootVT, Input),
+    DCI.CombineTo(Root.getNode(), DAG.getBitcast(RootVT, Input),
                   /*AddTo*/ true);
     return true;
   }
@@ -19782,14 +21152,14 @@ static bool combineX86ShuffleChain(SDValue Op, SDValue Root, ArrayRef<int> Mask,
       }
       if (Depth == 1 && Root->getOpcode() == Shuffle)
         return false; // Nothing to do!
-      Op = DAG.getNode(ISD::BITCAST, DL, ShuffleVT, Input);
+      Op = DAG.getBitcast(ShuffleVT, Input);
       DCI.AddToWorklist(Op.getNode());
       if (Shuffle == X86ISD::MOVDDUP)
         Op = DAG.getNode(Shuffle, DL, ShuffleVT, Op);
       else
         Op = DAG.getNode(Shuffle, DL, ShuffleVT, Op, Op);
       DCI.AddToWorklist(Op.getNode());
-      DCI.CombineTo(Root.getNode(), DAG.getNode(ISD::BITCAST, DL, RootVT, Op),
+      DCI.CombineTo(Root.getNode(), DAG.getBitcast(RootVT, Op),
                     /*AddTo*/ true);
       return true;
     }
@@ -19800,11 +21170,11 @@ static bool combineX86ShuffleChain(SDValue Op, SDValue Root, ArrayRef<int> Mask,
       MVT ShuffleVT = MVT::v4f32;
       if (Depth == 1 && Root->getOpcode() == Shuffle)
         return false; // Nothing to do!
-      Op = DAG.getNode(ISD::BITCAST, DL, ShuffleVT, Input);
+      Op = DAG.getBitcast(ShuffleVT, Input);
       DCI.AddToWorklist(Op.getNode());
       Op = DAG.getNode(Shuffle, DL, ShuffleVT, Op);
       DCI.AddToWorklist(Op.getNode());
-      DCI.CombineTo(Root.getNode(), DAG.getNode(ISD::BITCAST, DL, RootVT, Op),
+      DCI.CombineTo(Root.getNode(), DAG.getBitcast(RootVT, Op),
                     /*AddTo*/ true);
       return true;
     }
@@ -19814,11 +21184,11 @@ static bool combineX86ShuffleChain(SDValue Op, SDValue Root, ArrayRef<int> Mask,
       MVT ShuffleVT = MVT::v4f32;
       if (Depth == 1 && Root->getOpcode() == Shuffle)
         return false; // Nothing to do!
-      Op = DAG.getNode(ISD::BITCAST, DL, ShuffleVT, Input);
+      Op = DAG.getBitcast(ShuffleVT, Input);
       DCI.AddToWorklist(Op.getNode());
       Op = DAG.getNode(Shuffle, DL, ShuffleVT, Op, Op);
       DCI.AddToWorklist(Op.getNode());
-      DCI.CombineTo(Root.getNode(), DAG.getNode(ISD::BITCAST, DL, RootVT, Op),
+      DCI.CombineTo(Root.getNode(), DAG.getBitcast(RootVT, Op),
                     /*AddTo*/ true);
       return true;
     }
@@ -19848,11 +21218,11 @@ static bool combineX86ShuffleChain(SDValue Op, SDValue Root, ArrayRef<int> Mask,
     default:
       llvm_unreachable("Impossible mask size!");
     };
-    Op = DAG.getNode(ISD::BITCAST, DL, ShuffleVT, Input);
+    Op = DAG.getBitcast(ShuffleVT, Input);
     DCI.AddToWorklist(Op.getNode());
     Op = DAG.getNode(Shuffle, DL, ShuffleVT, Op, Op);
     DCI.AddToWorklist(Op.getNode());
-    DCI.CombineTo(Root.getNode(), DAG.getNode(ISD::BITCAST, DL, RootVT, Op),
+    DCI.CombineTo(Root.getNode(), DAG.getBitcast(RootVT, Op),
                   /*AddTo*/ true);
     return true;
   }
@@ -19878,17 +21248,17 @@ static bool combineX86ShuffleChain(SDValue Op, SDValue Root, ArrayRef<int> Mask,
       int M = Mask[i / Ratio] != SM_SentinelZero
                   ? Ratio * Mask[i / Ratio] + i % Ratio
                   : 255;
-      PSHUFBMask.push_back(DAG.getConstant(M, MVT::i8));
+      PSHUFBMask.push_back(DAG.getConstant(M, DL, MVT::i8));
     }
     MVT ByteVT = MVT::getVectorVT(MVT::i8, NumBytes);
-    Op = DAG.getNode(ISD::BITCAST, DL, ByteVT, Input);
+    Op = DAG.getBitcast(ByteVT, Input);
     DCI.AddToWorklist(Op.getNode());
     SDValue PSHUFBMaskOp =
         DAG.getNode(ISD::BUILD_VECTOR, DL, ByteVT, PSHUFBMask);
     DCI.AddToWorklist(PSHUFBMaskOp.getNode());
     Op = DAG.getNode(X86ISD::PSHUFB, DL, ByteVT, Op, PSHUFBMaskOp);
     DCI.AddToWorklist(Op.getNode());
-    DCI.CombineTo(Root.getNode(), DAG.getNode(ISD::BITCAST, DL, RootVT, Op),
+    DCI.CombineTo(Root.getNode(), DAG.getBitcast(RootVT, Op),
                   /*AddTo*/ true);
     return true;
   }
@@ -20060,7 +21430,7 @@ static SmallVector<int, 4> getPSHUFShuffleMask(SDValue N) {
 #ifndef NDEBUG
     for (int i = 1, NumLanes = VT.getSizeInBits() / 128; i < NumLanes; ++i)
       for (int j = 0; j < LaneElts; ++j)
-        assert(Mask[j] == Mask[i * LaneElts + j] - LaneElts &&
+        assert(Mask[j] == Mask[i * LaneElts + j] - (LaneElts * i) &&
                "Mask doesn't repeat in high 128-bit lanes!");
 #endif
     Mask.resize(LaneElts);
@@ -20184,14 +21554,14 @@ combineRedundantDWordShuffle(SDValue N, MutableArrayRef<int> Mask,
   for (int &M : Mask)
     M = VMask[M];
   V = DAG.getNode(V.getOpcode(), DL, V.getValueType(), V.getOperand(0),
-                  getV4X86ShuffleImm8ForMask(Mask, DAG));
+                  getV4X86ShuffleImm8ForMask(Mask, DL, DAG));
 
   // Rebuild the chain around this new shuffle.
   while (!Chain.empty()) {
     SDValue W = Chain.pop_back_val();
 
     if (V.getValueType() != W.getOperand(0).getValueType())
-      V = DAG.getNode(ISD::BITCAST, DL, W.getOperand(0).getValueType(), V);
+      V = DAG.getBitcast(W.getOperand(0).getValueType(), V);
 
     switch (W.getOpcode()) {
     default:
@@ -20210,7 +21580,7 @@ combineRedundantDWordShuffle(SDValue N, MutableArrayRef<int> Mask,
     }
   }
   if (V.getValueType() != N.getValueType())
-    V = DAG.getNode(ISD::BITCAST, DL, N.getValueType(), V);
+    V = DAG.getBitcast(N.getValueType(), V);
 
   // Return the new chain to replace N.
   return V;
@@ -20271,7 +21641,7 @@ static bool combineRedundantHalfShuffle(SDValue N, MutableArrayRef<int> Mask,
   for (int &M : Mask)
     M = VMask[M];
   V = DAG.getNode(V.getOpcode(), DL, MVT::v8i16, V.getOperand(0),
-                  getV4X86ShuffleImm8ForMask(Mask, DAG));
+                  getV4X86ShuffleImm8ForMask(Mask, DL, DAG));
 
   // Check that the shuffles didn't cancel each other out. If not, we need to
   // combine to the new one.
@@ -20327,12 +21697,12 @@ static SDValue PerformTargetShuffleCombine(SDValue N, SelectionDAG &DAG,
       DMask[DOffset + 0] = DOffset + 1;
       DMask[DOffset + 1] = DOffset + 0;
       MVT DVT = MVT::getVectorVT(MVT::i32, VT.getVectorNumElements() / 2);
-      V = DAG.getNode(ISD::BITCAST, DL, DVT, V);
+      V = DAG.getBitcast(DVT, V);
       DCI.AddToWorklist(V.getNode());
       V = DAG.getNode(X86ISD::PSHUFD, DL, DVT, V,
-                      getV4X86ShuffleImm8ForMask(DMask, DAG));
+                      getV4X86ShuffleImm8ForMask(DMask, DL, DAG));
       DCI.AddToWorklist(V.getNode());
-      return DAG.getNode(ISD::BITCAST, DL, VT, V);
+      return DAG.getBitcast(VT, V);
     }
 
     // Look for shuffle patterns which can be implemented as a single unpack.
@@ -20363,7 +21733,7 @@ static SDValue PerformTargetShuffleCombine(SDValue N, SelectionDAG &DAG,
         if (makeArrayRef(MappedMask).equals({0, 0, 1, 1, 2, 2, 3, 3}) ||
             makeArrayRef(MappedMask).equals({4, 4, 5, 5, 6, 6, 7, 7})) {
           // We can replace all three shuffles with an unpack.
-          V = DAG.getNode(ISD::BITCAST, DL, VT, D.getOperand(0));
+          V = DAG.getBitcast(VT, D.getOperand(0));
           DCI.AddToWorklist(V.getNode());
           return DAG.getNode(MappedMask[0] == 0 ? X86ISD::UNPCKL
                                                 : X86ISD::UNPCKH,
@@ -20507,8 +21877,8 @@ static SDValue PerformShuffleCombine(SDNode *N, SelectionDAG &DAG,
         CanFold = SVOp->getMaskElt(i) < 0;
 
       if (CanFold) {
-        SDValue BC00 = DAG.getNode(ISD::BITCAST, dl, VT, BC0.getOperand(0));
-        SDValue BC01 = DAG.getNode(ISD::BITCAST, dl, VT, BC0.getOperand(1));
+        SDValue BC00 = DAG.getBitcast(VT, BC0.getOperand(0));
+        SDValue BC01 = DAG.getBitcast(VT, BC0.getOperand(1));
         SDValue NewBinOp = DAG.getNode(BC0.getOpcode(), dl, VT, BC00, BC01);
         return DAG.getVectorShuffle(VT, dl, NewBinOp, N1, &SVOp->getMask()[0]);
       }
@@ -20522,8 +21892,7 @@ static SDValue PerformShuffleCombine(SDNode *N, SelectionDAG &DAG,
   for (unsigned i = 0, e = VT.getVectorNumElements(); i != e; ++i)
     Elts.push_back(getShuffleScalarElt(N, i, DAG, 0));
 
-  SDValue LD = EltsFromConsecutiveLoads(VT, Elts, dl, DAG, true);
-  if (LD.getNode())
+  if (SDValue LD = EltsFromConsecutiveLoads(VT, Elts, dl, DAG, true))
     return LD;
 
   if (isTargetShuffle(N->getOpcode())) {
@@ -20548,15 +21917,6 @@ static SDValue PerformShuffleCombine(SDNode *N, SelectionDAG &DAG,
   return SDValue();
 }
 
-/// PerformTruncateCombine - Converts truncate operation to
-/// a sequence of vector shuffle operations.
-/// It is possible when we truncate 256-bit vector to 128-bit vector
-static SDValue PerformTruncateCombine(SDNode *N, SelectionDAG &DAG,
-                                      TargetLowering::DAGCombinerInfo &DCI,
-                                      const X86Subtarget *Subtarget)  {
-  return SDValue();
-}
-
 /// XFormVExtractWithShuffleIntoLoad - Check if a vector extract from a target
 /// specific shuffle of a load can be folded into a single element load.
 /// Similar handling for VECTOR_SHUFFLE is performed by DAGCombiner, but
@@ -20579,7 +21939,8 @@ static SDValue XFormVExtractWithShuffleIntoLoad(SDNode *N, SelectionDAG &DAG,
     if (!InVec.hasOneUse())
       return SDValue();
     EVT BCVT = InVec.getOperand(0).getValueType();
-    if (BCVT.getVectorNumElements() != OriginalVT.getVectorNumElements())
+    if (!BCVT.isVector() ||
+        BCVT.getVectorNumElements() != OriginalVT.getVectorNumElements())
       return SDValue();
     InVec = InVec.getOperand(0);
   }
@@ -20632,7 +21993,7 @@ static SDValue XFormVExtractWithShuffleIntoLoad(SDNode *N, SelectionDAG &DAG,
   // alignment is valid.
   unsigned Align = LN0->getAlignment();
   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
-  unsigned NewAlign = TLI.getDataLayout()->getABITypeAlignment(
+  unsigned NewAlign = DAG.getDataLayout().getABITypeAlignment(
       EltVT.getTypeForEVT(*DAG.getContext()));
 
   if (NewAlign > Align || !TLI.isOperationLegalOrCustom(ISD::LOAD, EltVT))
@@ -20648,7 +22009,7 @@ static SDValue XFormVExtractWithShuffleIntoLoad(SDNode *N, SelectionDAG &DAG,
   Shuffle = DAG.getVectorShuffle(CurrentVT, dl,
                                  InVec.getOperand(0), Shuffle,
                                  &ShuffleMask[0]);
-  Shuffle = DAG.getNode(ISD::BITCAST, dl, OriginalVT, Shuffle);
+  Shuffle = DAG.getBitcast(OriginalVT, Shuffle);
   return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, N->getValueType(0), Shuffle,
                      EltNo);
 }
@@ -20679,12 +22040,11 @@ static SDValue PerformBITCASTCombine(SDNode *N, SelectionDAG &DAG) {
 /// use 64-bit extracts and shifts.
 static SDValue PerformEXTRACT_VECTOR_ELTCombine(SDNode *N, SelectionDAG &DAG,
                                          TargetLowering::DAGCombinerInfo &DCI) {
-  SDValue NewOp = XFormVExtractWithShuffleIntoLoad(N, DAG, DCI);
-  if (NewOp.getNode())
+  if (SDValue NewOp = XFormVExtractWithShuffleIntoLoad(N, DAG, DCI))
     return NewOp;
 
   SDValue InputVector = N->getOperand(0);
-
+  SDLoc dl(InputVector);
   // Detect mmx to i32 conversion through a v2i32 elt extract.
   if (InputVector.getOpcode() == ISD::BITCAST && InputVector.hasOneUse() &&
       N->getValueType(0) == MVT::i32 &&
@@ -20709,6 +22069,18 @@ static SDValue PerformEXTRACT_VECTOR_ELTCombine(SDNode *N, SelectionDAG &DAG,
                          MMXSrcOp.getOperand(0));
   }
 
+  EVT VT = N->getValueType(0);
+
+  if (VT == MVT::i1 && dyn_cast<ConstantSDNode>(N->getOperand(1)) &&
+      InputVector.getOpcode() == ISD::BITCAST &&
+      dyn_cast<ConstantSDNode>(InputVector.getOperand(0))) {
+    uint64_t ExtractedElt =
+	  cast<ConstantSDNode>(N->getOperand(1))->getZExtValue();
+    uint64_t InputValue =
+	  cast<ConstantSDNode>(InputVector.getOperand(0))->getZExtValue();
+    uint64_t Res = (InputValue >> ExtractedElt) & 1;
+    return DAG.getConstant(Res, dl, MVT::i1);
+  }
   // Only operate on vectors of 4 elements, where the alternative shuffling
   // gets to be more expensive.
   if (InputVector.getValueType() != MVT::v4i32)
@@ -20754,18 +22126,18 @@ static SDValue PerformEXTRACT_VECTOR_ELTCombine(SDNode *N, SelectionDAG &DAG,
   // otherwise bounce the vector off the cache.
   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
   SDValue Vals[4];
-  SDLoc dl(InputVector);
 
   if (TLI.isOperationLegal(ISD::SRA, MVT::i64)) {
-    SDValue Cst = DAG.getNode(ISD::BITCAST, dl, MVT::v2i64, InputVector);
-    EVT VecIdxTy = DAG.getTargetLoweringInfo().getVectorIdxTy();
+    SDValue Cst = DAG.getBitcast(MVT::v2i64, InputVector);
+    auto &DL = DAG.getDataLayout();
+    EVT VecIdxTy = DAG.getTargetLoweringInfo().getVectorIdxTy(DL);
     SDValue BottomHalf = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::i64, Cst,
-      DAG.getConstant(0, VecIdxTy));
+      DAG.getConstant(0, dl, VecIdxTy));
     SDValue TopHalf = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::i64, Cst,
-      DAG.getConstant(1, VecIdxTy));
+      DAG.getConstant(1, dl, VecIdxTy));
 
-    SDValue ShAmt = DAG.getConstant(32,
-      DAG.getTargetLoweringInfo().getShiftAmountTy(MVT::i64));
+    SDValue ShAmt = DAG.getConstant(
+        32, dl, DAG.getTargetLoweringInfo().getShiftAmountTy(MVT::i64, DL));
     Vals[0] = DAG.getNode(ISD::TRUNCATE, dl, MVT::i32, BottomHalf);
     Vals[1] = DAG.getNode(ISD::TRUNCATE, dl, MVT::i32,
       DAG.getNode(ISD::SRA, dl, MVT::i64, BottomHalf, ShAmt));
@@ -20784,10 +22156,11 @@ static SDValue PerformEXTRACT_VECTOR_ELTCombine(SDNode *N, SelectionDAG &DAG,
     // Replace each use (extract) with a load of the appropriate element.
     for (unsigned i = 0; i < 4; ++i) {
       uint64_t Offset = EltSize * i;
-      SDValue OffsetVal = DAG.getConstant(Offset, TLI.getPointerTy());
+      auto PtrVT = TLI.getPointerTy(DAG.getDataLayout());
+      SDValue OffsetVal = DAG.getConstant(Offset, dl, PtrVT);
 
-      SDValue ScalarAddr = DAG.getNode(ISD::ADD, dl, TLI.getPointerTy(),
-                                       StackPtr, OffsetVal);
+      SDValue ScalarAddr =
+          DAG.getNode(ISD::ADD, dl, PtrVT, StackPtr, OffsetVal);
 
       // Load the scalar.
       Vals[i] = DAG.getLoad(ElementType, dl, Ch,
@@ -20867,16 +22240,16 @@ matchIntegerMINMAX(SDValue Cond, EVT VT, SDValue LHS, SDValue RHS,
     default: break;
     case ISD::SETULT:
     case ISD::SETULE:
-      Opc = hasUnsigned ? X86ISD::UMIN : 0; break;
+      Opc = hasUnsigned ? ISD::UMIN : 0; break;
     case ISD::SETUGT:
     case ISD::SETUGE:
-      Opc = hasUnsigned ? X86ISD::UMAX : 0; break;
+      Opc = hasUnsigned ? ISD::UMAX : 0; break;
     case ISD::SETLT:
     case ISD::SETLE:
-      Opc = hasSigned ? X86ISD::SMIN : 0; break;
+      Opc = hasSigned ? ISD::SMIN : 0; break;
     case ISD::SETGT:
     case ISD::SETGE:
-      Opc = hasSigned ? X86ISD::SMAX : 0; break;
+      Opc = hasSigned ? ISD::SMAX : 0; break;
     }
   // Check for x CC y ? y : x -- a min/max with reversed arms.
   } else if (DAG.isEqualTo(LHS, Cond.getOperand(1)) &&
@@ -20885,16 +22258,16 @@ matchIntegerMINMAX(SDValue Cond, EVT VT, SDValue LHS, SDValue RHS,
     default: break;
     case ISD::SETULT:
     case ISD::SETULE:
-      Opc = hasUnsigned ? X86ISD::UMAX : 0; break;
+      Opc = hasUnsigned ? ISD::UMAX : 0; break;
     case ISD::SETUGT:
     case ISD::SETUGE:
-      Opc = hasUnsigned ? X86ISD::UMIN : 0; break;
+      Opc = hasUnsigned ? ISD::UMIN : 0; break;
     case ISD::SETLT:
     case ISD::SETLE:
-      Opc = hasSigned ? X86ISD::SMAX : 0; break;
+      Opc = hasSigned ? ISD::SMAX : 0; break;
     case ISD::SETGT:
     case ISD::SETGE:
-      Opc = hasSigned ? X86ISD::SMIN : 0; break;
+      Opc = hasSigned ? ISD::SMIN : 0; break;
     }
   }
 
@@ -21145,21 +22518,21 @@ static SDValue PerformSELECTCombine(SDNode *N, SelectionDAG &DAG,
             TrueC->getAPIntValue().isPowerOf2()) {
           if (NeedsCondInvert) // Invert the condition if needed.
             Cond = DAG.getNode(ISD::XOR, DL, Cond.getValueType(), Cond,
-                               DAG.getConstant(1, Cond.getValueType()));
+                               DAG.getConstant(1, DL, Cond.getValueType()));
 
           // Zero extend the condition if needed.
           Cond = DAG.getNode(ISD::ZERO_EXTEND, DL, LHS.getValueType(), Cond);
 
           unsigned ShAmt = TrueC->getAPIntValue().logBase2();
           return DAG.getNode(ISD::SHL, DL, LHS.getValueType(), Cond,
-                             DAG.getConstant(ShAmt, MVT::i8));
+                             DAG.getConstant(ShAmt, DL, MVT::i8));
         }
 
         // Optimize Cond ? cst+1 : cst -> zext(setcc(C)+cst.
         if (FalseC->getAPIntValue()+1 == TrueC->getAPIntValue()) {
           if (NeedsCondInvert) // Invert the condition if needed.
             Cond = DAG.getNode(ISD::XOR, DL, Cond.getValueType(), Cond,
-                               DAG.getConstant(1, Cond.getValueType()));
+                               DAG.getConstant(1, DL, Cond.getValueType()));
 
           // Zero extend the condition if needed.
           Cond = DAG.getNode(ISD::ZERO_EXTEND, DL,
@@ -21194,7 +22567,7 @@ static SDValue PerformSELECTCombine(SDNode *N, SelectionDAG &DAG,
             APInt Diff = TrueC->getAPIntValue()-FalseC->getAPIntValue();
             if (NeedsCondInvert) // Invert the condition if needed.
               Cond = DAG.getNode(ISD::XOR, DL, Cond.getValueType(), Cond,
-                                 DAG.getConstant(1, Cond.getValueType()));
+                                 DAG.getConstant(1, DL, Cond.getValueType()));
 
             // Zero extend the condition if needed.
             Cond = DAG.getNode(ISD::ZERO_EXTEND, DL, FalseC->getValueType(0),
@@ -21202,7 +22575,8 @@ static SDValue PerformSELECTCombine(SDNode *N, SelectionDAG &DAG,
             // Scale the condition by the difference.
             if (Diff != 1)
               Cond = DAG.getNode(ISD::MUL, DL, Cond.getValueType(), Cond,
-                                 DAG.getConstant(Diff, Cond.getValueType()));
+                                 DAG.getConstant(Diff, DL,
+                                                 Cond.getValueType()));
 
             // Add the base if non-zero.
             if (FalseC->getAPIntValue() != 0)
@@ -21290,7 +22664,7 @@ static SDValue PerformSELECTCombine(SDNode *N, SelectionDAG &DAG,
                       (-OpRHSConst->getAPIntValue() - 1))
                 return DAG.getNode(
                     X86ISD::SUBUS, DL, VT, OpLHS,
-                    DAG.getConstant(-OpRHSConst->getAPIntValue(), VT));
+                    DAG.getConstant(-OpRHSConst->getAPIntValue(), DL, VT));
 
           // Another special case: If C was a sign bit, the sub has been
           // canonicalized into a xor.
@@ -21304,7 +22678,7 @@ static SDValue PerformSELECTCombine(SDNode *N, SelectionDAG &DAG,
             // don't rely on particular values of undef lanes.
             return DAG.getNode(
                 X86ISD::SUBUS, DL, VT, OpLHS,
-                DAG.getConstant(OpRHSConst->getAPIntValue(), VT));
+                DAG.getConstant(OpRHSConst->getAPIntValue(), DL, VT));
         }
     }
   }
@@ -21350,7 +22724,8 @@ static SDValue PerformSELECTCombine(SDNode *N, SelectionDAG &DAG,
         // Check if the selector will be produced by CMPP*/PCMP*
         Cond.getOpcode() == ISD::SETCC &&
         // Check if SETCC has already been promoted
-        TLI.getSetCCResultType(*DAG.getContext(), VT) == CondVT) {
+        TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT) ==
+            CondVT) {
       bool TValIsAllZeros = ISD::isBuildVectorAllZeros(LHS.getNode());
       bool FValIsAllOnes = ISD::isBuildVectorAllOnes(RHS.getNode());
 
@@ -21372,13 +22747,13 @@ static SDValue PerformSELECTCombine(SDNode *N, SelectionDAG &DAG,
       if (TValIsAllOnes && FValIsAllZeros)
         Ret = Cond;
       else if (TValIsAllOnes)
-        Ret = DAG.getNode(ISD::OR, DL, CondVT, Cond,
-                          DAG.getNode(ISD::BITCAST, DL, CondVT, RHS));
+        Ret =
+            DAG.getNode(ISD::OR, DL, CondVT, Cond, DAG.getBitcast(CondVT, RHS));
       else if (FValIsAllZeros)
         Ret = DAG.getNode(ISD::AND, DL, CondVT, Cond,
-                          DAG.getNode(ISD::BITCAST, DL, CondVT, LHS));
+                          DAG.getBitcast(CondVT, LHS));
 
-      return DAG.getNode(ISD::BITCAST, DL, VT, Ret);
+      return DAG.getBitcast(VT, Ret);
     }
   }
 
@@ -21395,7 +22770,7 @@ static SDValue PerformSELECTCombine(SDNode *N, SelectionDAG &DAG,
   // know will be matched by LowerVECTOR_SHUFFLEtoBlend.
   if ((N->getOpcode() == ISD::VSELECT ||
        N->getOpcode() == X86ISD::SHRUNKBLEND) &&
-      !DCI.isBeforeLegalize()) {
+      !DCI.isBeforeLegalize() && !VT.is512BitVector()) {
     SDValue Shuffle = transformVSELECTtoBlendVECTOR_SHUFFLE(N, DAG, Subtarget);
     if (Shuffle.getNode())
       return Shuffle;
@@ -21698,7 +23073,7 @@ static SDValue PerformCMOVCombine(SDNode *N, SelectionDAG &DAG,
       // Extra check as FCMOV only supports a subset of X86 cond.
       (FalseOp.getValueType() != MVT::f80 || hasFPCMov(CC))) {
     SDValue Ops[] = { FalseOp, TrueOp,
-                      DAG.getConstant(CC, MVT::i8), Flags };
+                      DAG.getConstant(CC, DL, MVT::i8), Flags };
     return DAG.getNode(X86ISD::CMOV, DL, N->getVTList(), Ops);
   }
 
@@ -21720,14 +23095,14 @@ static SDValue PerformCMOVCombine(SDNode *N, SelectionDAG &DAG,
       // shift amount.
       if (FalseC->getAPIntValue() == 0 && TrueC->getAPIntValue().isPowerOf2()) {
         Cond = DAG.getNode(X86ISD::SETCC, DL, MVT::i8,
-                           DAG.getConstant(CC, MVT::i8), Cond);
+                           DAG.getConstant(CC, DL, MVT::i8), Cond);
 
         // Zero extend the condition if needed.
         Cond = DAG.getNode(ISD::ZERO_EXTEND, DL, TrueC->getValueType(0), Cond);
 
         unsigned ShAmt = TrueC->getAPIntValue().logBase2();
         Cond = DAG.getNode(ISD::SHL, DL, Cond.getValueType(), Cond,
-                           DAG.getConstant(ShAmt, MVT::i8));
+                           DAG.getConstant(ShAmt, DL, MVT::i8));
         if (N->getNumValues() == 2)  // Dead flag value?
           return DCI.CombineTo(N, Cond, SDValue());
         return Cond;
@@ -21737,7 +23112,7 @@ static SDValue PerformCMOVCombine(SDNode *N, SelectionDAG &DAG,
       // for any integer data type, including i8/i16.
       if (FalseC->getAPIntValue()+1 == TrueC->getAPIntValue()) {
         Cond = DAG.getNode(X86ISD::SETCC, DL, MVT::i8,
-                           DAG.getConstant(CC, MVT::i8), Cond);
+                           DAG.getConstant(CC, DL, MVT::i8), Cond);
 
         // Zero extend the condition if needed.
         Cond = DAG.getNode(ISD::ZERO_EXTEND, DL,
@@ -21775,14 +23150,14 @@ static SDValue PerformCMOVCombine(SDNode *N, SelectionDAG &DAG,
         if (isFastMultiplier) {
           APInt Diff = TrueC->getAPIntValue()-FalseC->getAPIntValue();
           Cond = DAG.getNode(X86ISD::SETCC, DL, MVT::i8,
-                             DAG.getConstant(CC, MVT::i8), Cond);
+                             DAG.getConstant(CC, DL, MVT::i8), Cond);
           // Zero extend the condition if needed.
           Cond = DAG.getNode(ISD::ZERO_EXTEND, DL, FalseC->getValueType(0),
                              Cond);
           // Scale the condition by the difference.
           if (Diff != 1)
             Cond = DAG.getNode(ISD::MUL, DL, Cond.getValueType(), Cond,
-                               DAG.getConstant(Diff, Cond.getValueType()));
+                               DAG.getConstant(Diff, DL, Cond.getValueType()));
 
           // Add the base if non-zero.
           if (FalseC->getAPIntValue() != 0)
@@ -21828,7 +23203,7 @@ static SDValue PerformCMOVCombine(SDNode *N, SelectionDAG &DAG,
       if (CC == X86::COND_E &&
           CmpAgainst == dyn_cast<ConstantSDNode>(TrueOp)) {
         SDValue Ops[] = { FalseOp, Cond.getOperand(0),
-                          DAG.getConstant(CC, MVT::i8), Cond };
+                          DAG.getConstant(CC, DL, MVT::i8), Cond };
         return DAG.getNode(X86ISD::CMOV, DL, N->getVTList (), Ops);
       }
     }
@@ -21862,10 +23237,10 @@ static SDValue PerformCMOVCombine(SDNode *N, SelectionDAG &DAG,
         CC1 = X86::GetOppositeBranchCondition(CC1);
       }
 
-      SDValue LOps[] = {FalseOp, TrueOp, DAG.getConstant(CC0, MVT::i8),
+      SDValue LOps[] = {FalseOp, TrueOp, DAG.getConstant(CC0, DL, MVT::i8),
         Flags};
       SDValue LCMOV = DAG.getNode(X86ISD::CMOV, DL, N->getVTList(), LOps);
-      SDValue Ops[] = {LCMOV, TrueOp, DAG.getConstant(CC1, MVT::i8), Flags};
+      SDValue Ops[] = {LCMOV, TrueOp, DAG.getConstant(CC1, DL, MVT::i8), Flags};
       SDValue CMOV = DAG.getNode(X86ISD::CMOV, DL, N->getVTList(), Ops);
       DAG.ReplaceAllUsesOfValueWith(SDValue(N, 1), SDValue(CMOV.getNode(), 1));
       return CMOV;
@@ -21963,8 +23338,9 @@ static SDValue PerformINTRINSIC_WO_CHAINCombine(SDNode *N, SelectionDAG &DAG,
 
     // Replace this packed shift intrinsic with a target independent
     // shift dag node.
-    SDValue Splat = DAG.getConstant(C, VT);
-    return DAG.getNode(ISD::SRA, SDLoc(N), VT, Op0, Splat);
+    SDLoc DL(N);
+    SDValue Splat = DAG.getConstant(C, DL, VT);
+    return DAG.getNode(ISD::SRA, DL, VT, Op0, Splat);
   }
   }
 }
@@ -22014,17 +23390,17 @@ static SDValue PerformMulCombine(SDNode *N, SelectionDAG &DAG,
     SDValue NewMul;
     if (isPowerOf2_64(MulAmt1))
       NewMul = DAG.getNode(ISD::SHL, DL, VT, N->getOperand(0),
-                           DAG.getConstant(Log2_64(MulAmt1), MVT::i8));
+                           DAG.getConstant(Log2_64(MulAmt1), DL, MVT::i8));
     else
       NewMul = DAG.getNode(X86ISD::MUL_IMM, DL, VT, N->getOperand(0),
-                           DAG.getConstant(MulAmt1, VT));
+                           DAG.getConstant(MulAmt1, DL, VT));
 
     if (isPowerOf2_64(MulAmt2))
       NewMul = DAG.getNode(ISD::SHL, DL, VT, NewMul,
-                           DAG.getConstant(Log2_64(MulAmt2), MVT::i8));
+                           DAG.getConstant(Log2_64(MulAmt2), DL, MVT::i8));
     else
       NewMul = DAG.getNode(X86ISD::MUL_IMM, DL, VT, NewMul,
-                           DAG.getConstant(MulAmt2, VT));
+                           DAG.getConstant(MulAmt2, DL, VT));
 
     // Do not add new nodes to DAG combiner worklist.
     DCI.CombineTo(N, NewMul, false);
@@ -22051,9 +23427,11 @@ static SDValue PerformSHLCombine(SDNode *N, SelectionDAG &DAG) {
       APInt Mask = cast<ConstantSDNode>(N0.getOperand(1))->getAPIntValue();
       APInt ShAmt = N1C->getAPIntValue();
       Mask = Mask.shl(ShAmt);
-      if (Mask != 0)
-        return DAG.getNode(ISD::AND, SDLoc(N), VT,
-                           N00, DAG.getConstant(Mask, VT));
+      if (Mask != 0) {
+        SDLoc DL(N);
+        return DAG.getNode(ISD::AND, DL, VT,
+                           N00, DAG.getConstant(Mask, DL, VT));
+      }
     }
   }
 
@@ -22067,7 +23445,7 @@ static SDValue PerformSHLCombine(SDNode *N, SelectionDAG &DAG) {
       // We shift all of the values by one. In many cases we do not have
       // hardware support for this operation. This is better expressed as an ADD
       // of two values.
-      if (N1SplatC->getZExtValue() == 1)
+      if (N1SplatC->getAPIntValue() == 1)
         return DAG.getNode(ISD::ADD, SDLoc(N), VT, N0, N0);
     }
 
@@ -22108,16 +23486,14 @@ static SDValue performShiftToAllZeros(SDNode *N, SelectionDAG &DAG,
 static SDValue PerformShiftCombine(SDNode* N, SelectionDAG &DAG,
                                    TargetLowering::DAGCombinerInfo &DCI,
                                    const X86Subtarget *Subtarget) {
-  if (N->getOpcode() == ISD::SHL) {
-    SDValue V = PerformSHLCombine(N, DAG);
-    if (V.getNode()) return V;
-  }
+  if (N->getOpcode() == ISD::SHL)
+    if (SDValue V = PerformSHLCombine(N, DAG))
+      return V;
 
-  if (N->getOpcode() != ISD::SRA) {
-    // Try to fold this logical shift into a zero vector.
-    SDValue V = performShiftToAllZeros(N, DAG, Subtarget);
-    if (V.getNode()) return V;
-  }
+  // Try to fold this logical shift into a zero vector.
+  if (N->getOpcode() != ISD::SRA)
+    if (SDValue V = performShiftToAllZeros(N, DAG, Subtarget))
+      return V;
 
   return SDValue();
 }
@@ -22183,7 +23559,8 @@ static SDValue CMPEQCombine(SDNode *N, SelectionDAG &DAG,
           unsigned x86cc = (cc0 == X86::COND_E) ? 0 : 4;
           if (Subtarget->hasAVX512()) {
             SDValue FSetCC = DAG.getNode(X86ISD::FSETCC, DL, MVT::i1, CMP00,
-                                         CMP01, DAG.getConstant(x86cc, MVT::i8));
+                                         CMP01,
+                                         DAG.getConstant(x86cc, DL, MVT::i8));
             if (N->getValueType(0) != MVT::i1)
               return DAG.getNode(ISD::ZERO_EXTEND, DL, N->getValueType(0),
                                  FSetCC);
@@ -22191,7 +23568,8 @@ static SDValue CMPEQCombine(SDNode *N, SelectionDAG &DAG,
           }
           SDValue OnesOrZeroesF = DAG.getNode(X86ISD::FSETCC, DL,
                                               CMP00.getValueType(), CMP00, CMP01,
-                                              DAG.getConstant(x86cc, MVT::i8));
+                                              DAG.getConstant(x86cc, DL,
+                                                              MVT::i8));
 
           bool is64BitFP = (CMP00.getValueType() == MVT::f64);
           MVT IntVT = is64BitFP ? MVT::i64 : MVT::i32;
@@ -22204,17 +23582,17 @@ static SDValue CMPEQCombine(SDNode *N, SelectionDAG &DAG,
             // and work with those going forward.
             SDValue Vector64 = DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, MVT::v2f64,
                                            OnesOrZeroesF);
-            SDValue Vector32 = DAG.getNode(ISD::BITCAST, DL, MVT::v4f32,
-                                           Vector64);
+            SDValue Vector32 = DAG.getBitcast(MVT::v4f32, Vector64);
             OnesOrZeroesF = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, MVT::f32,
-                                        Vector32, DAG.getIntPtrConstant(0));
+                                        Vector32, DAG.getIntPtrConstant(0, DL));
             IntVT = MVT::i32;
           }
 
-          SDValue OnesOrZeroesI = DAG.getNode(ISD::BITCAST, DL, IntVT, OnesOrZeroesF);
+          SDValue OnesOrZeroesI = DAG.getBitcast(IntVT, OnesOrZeroesF);
           SDValue ANDed = DAG.getNode(ISD::AND, DL, IntVT, OnesOrZeroesI,
-                                      DAG.getConstant(1, IntVT));
-          SDValue OneBitOfTruth = DAG.getNode(ISD::TRUNCATE, DL, MVT::i8, ANDed);
+                                      DAG.getConstant(1, DL, IntVT));
+          SDValue OneBitOfTruth = DAG.getNode(ISD::TRUNCATE, DL, MVT::i8,
+                                              ANDed);
           return OneBitOfTruth;
         }
       }
@@ -22326,7 +23704,7 @@ static SDValue WidenMaskArithmetic(SDNode *N, SelectionDAG &DAG,
     APInt Mask = APInt::getAllOnesValue(InBits);
     Mask = Mask.zext(VT.getScalarType().getSizeInBits());
     return DAG.getNode(ISD::AND, DL, VT,
-                       Op, DAG.getConstant(Mask, VT));
+                       Op, DAG.getConstant(Mask, DL, VT));
   }
   case ISD::SIGN_EXTEND:
     return DAG.getNode(ISD::SIGN_EXTEND_INREG, DL, VT,
@@ -22422,8 +23800,8 @@ static SDValue VectorZextCombine(SDNode *N, SelectionDAG &DAG,
       Mask.push_back(i / ZextRatio);
 
   SDValue NewShuffle = DAG.getVectorShuffle(Shuffle->getValueType(0), DL,
-    Shuffle->getOperand(0), DAG.getConstant(0, SrcType), Mask);
-  return DAG.getNode(ISD::BITCAST, DL,  N0.getValueType(), NewShuffle);
+    Shuffle->getOperand(0), DAG.getConstant(0, DL, SrcType), Mask);
+  return DAG.getBitcast(N0.getValueType(), NewShuffle);
 }
 
 static SDValue PerformAndCombine(SDNode *N, SelectionDAG &DAG,
@@ -22458,7 +23836,8 @@ static SDValue PerformAndCombine(SDNode *N, SelectionDAG &DAG,
           uint64_t MaskSize = countPopulation(Mask);
           if (Shift + MaskSize <= VT.getSizeInBits())
             return DAG.getNode(X86ISD::BEXTR, DL, VT, N0.getOperand(0),
-                               DAG.getConstant(Shift | (MaskSize << 8), VT));
+                               DAG.getConstant(Shift | (MaskSize << 8), DL,
+                                               VT));
         }
       }
     } // BEXTR
@@ -22494,8 +23873,7 @@ static SDValue PerformOrCombine(SDNode *N, SelectionDAG &DAG,
   if (DCI.isBeforeLegalizeOps())
     return SDValue();
 
-  SDValue R = CMPEQCombine(N, DAG, DCI, Subtarget);
-  if (R.getNode())
+  if (SDValue R = CMPEQCombine(N, DAG, DCI, Subtarget))
     return R;
 
   SDValue N0 = N->getOperand(0);
@@ -22563,7 +23941,7 @@ static SDValue PerformOrCombine(SDNode *N, SelectionDAG &DAG,
         assert((EltBits == 8 || EltBits == 16 || EltBits == 32) &&
                "Unsupported VT for PSIGN");
         Mask = DAG.getNode(X86ISD::PSIGN, DL, MaskVT, X, Mask.getOperand(0));
-        return DAG.getNode(ISD::BITCAST, DL, VT, Mask);
+        return DAG.getBitcast(VT, Mask);
       }
       // PBLENDVB only available on SSE 4.1
       if (!Subtarget->hasSSE41())
@@ -22571,11 +23949,11 @@ static SDValue PerformOrCombine(SDNode *N, SelectionDAG &DAG,
 
       EVT BlendVT = (VT == MVT::v4i64) ? MVT::v32i8 : MVT::v16i8;
 
-      X = DAG.getNode(ISD::BITCAST, DL, BlendVT, X);
-      Y = DAG.getNode(ISD::BITCAST, DL, BlendVT, Y);
-      Mask = DAG.getNode(ISD::BITCAST, DL, BlendVT, Mask);
+      X = DAG.getBitcast(BlendVT, X);
+      Y = DAG.getBitcast(BlendVT, Y);
+      Mask = DAG.getBitcast(BlendVT, Mask);
       Mask = DAG.getNode(ISD::VSELECT, DL, BlendVT, Mask, Y, X);
-      return DAG.getNode(ISD::BITCAST, DL, VT, Mask);
+      return DAG.getBitcast(VT, Mask);
     }
   }
 
@@ -22673,10 +24051,10 @@ static SDValue performIntegerAbsCombine(SDNode *N, SelectionDAG &DAG) {
       if (Y1C->getAPIntValue() == VT.getSizeInBits()-1) {
         // Generate SUB & CMOV.
         SDValue Neg = DAG.getNode(X86ISD::SUB, DL, DAG.getVTList(VT, MVT::i32),
-                                  DAG.getConstant(0, VT), N0.getOperand(0));
+                                  DAG.getConstant(0, DL, VT), N0.getOperand(0));
 
         SDValue Ops[] = { N0.getOperand(0), Neg,
-                          DAG.getConstant(X86::COND_GE, MVT::i8),
+                          DAG.getConstant(X86::COND_GE, DL, MVT::i8),
                           SDValue(Neg.getNode(), 1) };
         return DAG.getNode(X86ISD::CMOV, DL, DAG.getVTList(VT, MVT::Glue), Ops);
       }
@@ -22690,11 +24068,9 @@ static SDValue PerformXorCombine(SDNode *N, SelectionDAG &DAG,
   if (DCI.isBeforeLegalizeOps())
     return SDValue();
 
-  if (Subtarget->hasCMov()) {
-    SDValue RV = performIntegerAbsCombine(N, DAG);
-    if (RV.getNode())
+  if (Subtarget->hasCMov())
+    if (SDValue RV = performIntegerAbsCombine(N, DAG))
       return RV;
-  }
 
   return SDValue();
 }
@@ -22721,7 +24097,8 @@ static SDValue PerformLOADCombine(SDNode *N, SelectionDAG &DAG,
       return SDValue();
 
     SDValue Ptr = Ld->getBasePtr();
-    SDValue Increment = DAG.getConstant(16, TLI.getPointerTy());
+    SDValue Increment =
+        DAG.getConstant(16, dl, TLI.getPointerTy(DAG.getDataLayout()));
 
     EVT HalfVT = EVT::getVectorVT(*DAG.getContext(), MemVT.getScalarType(),
                                   NumElems/2);
@@ -22776,7 +24153,7 @@ static SDValue PerformMLOADCombine(SDNode *N, SelectionDAG &DAG,
   assert(WideVecVT.getSizeInBits() == VT.getSizeInBits());
 
   // Convert Src0 value
-  SDValue WideSrc0 = DAG.getNode(ISD::BITCAST, dl, WideVecVT, Mld->getSrc0());
+  SDValue WideSrc0 = DAG.getBitcast(WideVecVT, Mld->getSrc0());
   if (Mld->getSrc0().getOpcode() != ISD::UNDEF) {
     SmallVector<int, 16> ShuffleVec(NumElems * SizeRatio, -1);
     for (unsigned i = 0; i != NumElems; ++i)
@@ -22793,14 +24170,14 @@ static SDValue PerformMLOADCombine(SDNode *N, SelectionDAG &DAG,
   SDValue Mask = Mld->getMask();
   if (Mask.getValueType() == VT) {
     // Mask and original value have the same type
-    NewMask = DAG.getNode(ISD::BITCAST, dl, WideVecVT, Mask);
+    NewMask = DAG.getBitcast(WideVecVT, Mask);
     SmallVector<int, 16> ShuffleVec(NumElems * SizeRatio, -1);
     for (unsigned i = 0; i != NumElems; ++i)
       ShuffleVec[i] = i * SizeRatio;
     for (unsigned i = NumElems; i != NumElems*SizeRatio; ++i)
       ShuffleVec[i] = NumElems*SizeRatio;
     NewMask = DAG.getVectorShuffle(WideVecVT, dl, NewMask,
-                                   DAG.getConstant(0, WideVecVT),
+                                   DAG.getConstant(0, dl, WideVecVT),
                                    &ShuffleVec[0]);
   }
   else {
@@ -22812,7 +24189,7 @@ static SDValue PerformMLOADCombine(SDNode *N, SelectionDAG &DAG,
 
     unsigned NumConcat = WidenNumElts / MaskNumElts;
     SmallVector<SDValue, 16> Ops(NumConcat);
-    SDValue ZeroVal = DAG.getConstant(0, Mask.getValueType());
+    SDValue ZeroVal = DAG.getConstant(0, dl, Mask.getValueType());
     Ops[0] = Mask;
     for (unsigned i = 1; i != NumConcat; ++i)
       Ops[i] = ZeroVal;
@@ -22844,6 +24221,15 @@ static SDValue PerformMSTORECombine(SDNode *N, SelectionDAG &DAG,
   unsigned FromSz = VT.getVectorElementType().getSizeInBits();
   unsigned ToSz = StVT.getVectorElementType().getSizeInBits();
 
+  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
+
+  // The truncating store is legal in some cases. For example
+  // vpmovqb, vpmovqw, vpmovqd, vpmovdb, vpmovdw
+  // are designated for truncate store.
+  // In this case we don't need any further transformations.
+  if (TLI.isTruncStoreLegal(VT, StVT))
+    return SDValue();
+
   // From, To sizes and ElemCount must be pow of two
   assert (isPowerOf2_32(NumElems * FromSz * ToSz) &&
     "Unexpected size for truncating masked store");
@@ -22861,7 +24247,7 @@ static SDValue PerformMSTORECombine(SDNode *N, SelectionDAG &DAG,
 
   assert(WideVecVT.getSizeInBits() == VT.getSizeInBits());
 
-  SDValue WideVec = DAG.getNode(ISD::BITCAST, dl, WideVecVT, Mst->getValue());
+  SDValue WideVec = DAG.getBitcast(WideVecVT, Mst->getValue());
   SmallVector<int, 16> ShuffleVec(NumElems * SizeRatio, -1);
   for (unsigned i = 0; i != NumElems; ++i)
     ShuffleVec[i] = i * SizeRatio;
@@ -22878,13 +24264,13 @@ static SDValue PerformMSTORECombine(SDNode *N, SelectionDAG &DAG,
   SDValue Mask = Mst->getMask();
   if (Mask.getValueType() == VT) {
     // Mask and original value have the same type
-    NewMask = DAG.getNode(ISD::BITCAST, dl, WideVecVT, Mask);
+    NewMask = DAG.getBitcast(WideVecVT, Mask);
     for (unsigned i = 0; i != NumElems; ++i)
       ShuffleVec[i] = i * SizeRatio;
     for (unsigned i = NumElems; i != NumElems*SizeRatio; ++i)
       ShuffleVec[i] = NumElems*SizeRatio;
     NewMask = DAG.getVectorShuffle(WideVecVT, dl, NewMask,
-                                   DAG.getConstant(0, WideVecVT),
+                                   DAG.getConstant(0, dl, WideVecVT),
                                    &ShuffleVec[0]);
   }
   else {
@@ -22896,7 +24282,7 @@ static SDValue PerformMSTORECombine(SDNode *N, SelectionDAG &DAG,
 
     unsigned NumConcat = WidenNumElts / MaskNumElts;
     SmallVector<SDValue, 16> Ops(NumConcat);
-    SDValue ZeroVal = DAG.getConstant(0, Mask.getValueType());
+    SDValue ZeroVal = DAG.getConstant(0, dl, Mask.getValueType());
     Ops[0] = Mask;
     for (unsigned i = 1; i != NumConcat; ++i)
       Ops[i] = ZeroVal;
@@ -22930,7 +24316,8 @@ static SDValue PerformSTORECombine(SDNode *N, SelectionDAG &DAG,
     SDValue Value0 = Extract128BitVector(StoredVal, 0, DAG, dl);
     SDValue Value1 = Extract128BitVector(StoredVal, NumElems/2, DAG, dl);
 
-    SDValue Stride = DAG.getConstant(16, TLI.getPointerTy());
+    SDValue Stride =
+        DAG.getConstant(16, dl, TLI.getPointerTy(DAG.getDataLayout()));
     SDValue Ptr0 = St->getBasePtr();
     SDValue Ptr1 = DAG.getNode(ISD::ADD, dl, Ptr0.getValueType(), Ptr0, Stride);
 
@@ -22954,6 +24341,13 @@ static SDValue PerformSTORECombine(SDNode *N, SelectionDAG &DAG,
     unsigned FromSz = VT.getVectorElementType().getSizeInBits();
     unsigned ToSz = StVT.getVectorElementType().getSizeInBits();
 
+    // The truncating store is legal in some cases. For example
+    // vpmovqb, vpmovqw, vpmovqd, vpmovdb, vpmovdw
+    // are designated for truncate store.
+    // In this case we don't need any further transformations.
+    if (TLI.isTruncStoreLegal(VT, StVT))
+      return SDValue();
+
     // From, To sizes and ElemCount must be pow of two
     if (!isPowerOf2_32(NumElems * FromSz * ToSz)) return SDValue();
     // We are going to use the original vector elt for storing.
@@ -22970,7 +24364,7 @@ static SDValue PerformSTORECombine(SDNode *N, SelectionDAG &DAG,
 
     assert(WideVecVT.getSizeInBits() == VT.getSizeInBits());
 
-    SDValue WideVec = DAG.getNode(ISD::BITCAST, dl, WideVecVT, St->getValue());
+    SDValue WideVec = DAG.getBitcast(WideVecVT, St->getValue());
     SmallVector<int, 8> ShuffleVec(NumElems * SizeRatio, -1);
     for (unsigned i = 0; i != NumElems; ++i)
       ShuffleVec[i] = i * SizeRatio;
@@ -23001,17 +24395,17 @@ static SDValue PerformSTORECombine(SDNode *N, SelectionDAG &DAG,
     EVT StoreVecVT = EVT::getVectorVT(*DAG.getContext(),
             StoreType, VT.getSizeInBits()/StoreType.getSizeInBits());
     assert(StoreVecVT.getSizeInBits() == VT.getSizeInBits());
-    SDValue ShuffWide = DAG.getNode(ISD::BITCAST, dl, StoreVecVT, Shuff);
+    SDValue ShuffWide = DAG.getBitcast(StoreVecVT, Shuff);
     SmallVector<SDValue, 8> Chains;
-    SDValue Increment = DAG.getConstant(StoreType.getSizeInBits()/8,
-                                        TLI.getPointerTy());
+    SDValue Increment = DAG.getConstant(StoreType.getSizeInBits() / 8, dl,
+                                        TLI.getPointerTy(DAG.getDataLayout()));
     SDValue Ptr = St->getBasePtr();
 
     // Perform one or more big stores into memory.
     for (unsigned i=0, e=(ToSz*NumElems)/StoreType.getSizeInBits(); i!=e; ++i) {
       SDValue SubVec = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl,
                                    StoreType, ShuffWide,
-                                   DAG.getIntPtrConstant(i));
+                                   DAG.getIntPtrConstant(i, dl));
       SDValue Ch = DAG.getStore(St->getChain(), dl, SubVec, Ptr,
                                 St->getPointerInfo(), St->isVolatile(),
                                 St->isNonTemporal(), St->getAlignment());
@@ -23033,8 +24427,8 @@ static SDValue PerformSTORECombine(SDNode *N, SelectionDAG &DAG,
 
   const Function *F = DAG.getMachineFunction().getFunction();
   bool NoImplicitFloatOps = F->hasFnAttribute(Attribute::NoImplicitFloat);
-  bool F64IsLegal = !DAG.getTarget().Options.UseSoftFloat && !NoImplicitFloatOps
-                     && Subtarget->hasSSE2();
+  bool F64IsLegal =
+      !Subtarget->useSoftFloat() && !NoImplicitFloatOps && Subtarget->hasSSE2();
   if ((VT.isVector() ||
        (VT == MVT::i64 && F64IsLegal && !Subtarget->is64Bit())) &&
       isa<LoadSDNode>(St->getValue()) &&
@@ -23095,7 +24489,7 @@ static SDValue PerformSTORECombine(SDNode *N, SelectionDAG &DAG,
     // Otherwise, lower to two pairs of 32-bit loads / stores.
     SDValue LoAddr = Ld->getBasePtr();
     SDValue HiAddr = DAG.getNode(ISD::ADD, LdDL, MVT::i32, LoAddr,
-                                 DAG.getConstant(4, MVT::i32));
+                                 DAG.getConstant(4, LdDL, MVT::i32));
 
     SDValue LoLd = DAG.getLoad(MVT::i32, LdDL, Ld->getChain(), LoAddr,
                                Ld->getPointerInfo(),
@@ -23116,7 +24510,7 @@ static SDValue PerformSTORECombine(SDNode *N, SelectionDAG &DAG,
 
     LoAddr = St->getBasePtr();
     HiAddr = DAG.getNode(ISD::ADD, StDL, MVT::i32, LoAddr,
-                         DAG.getConstant(4, MVT::i32));
+                         DAG.getConstant(4, StDL, MVT::i32));
 
     SDValue LoSt = DAG.getStore(NewChain, StDL, LoLd, LoAddr,
                                 St->getPointerInfo(),
@@ -23129,6 +24523,27 @@ static SDValue PerformSTORECombine(SDNode *N, SelectionDAG &DAG,
                                 MinAlign(St->getAlignment(), 4));
     return DAG.getNode(ISD::TokenFactor, StDL, MVT::Other, LoSt, HiSt);
   }
+
+  // This is similar to the above case, but here we handle a scalar 64-bit
+  // integer store that is extracted from a vector on a 32-bit target.
+  // If we have SSE2, then we can treat it like a floating-point double
+  // to get past legalization. The execution dependencies fixup pass will
+  // choose the optimal machine instruction for the store if this really is
+  // an integer or v2f32 rather than an f64.
+  if (VT == MVT::i64 && F64IsLegal && !Subtarget->is64Bit() &&
+      St->getOperand(1).getOpcode() == ISD::EXTRACT_VECTOR_ELT) {
+    SDValue OldExtract = St->getOperand(1);
+    SDValue ExtOp0 = OldExtract.getOperand(0);
+    unsigned VecSize = ExtOp0.getValueSizeInBits();
+    EVT VecVT = EVT::getVectorVT(*DAG.getContext(), MVT::f64, VecSize / 64);
+    SDValue BitCast = DAG.getBitcast(VecVT, ExtOp0);
+    SDValue NewExtract = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::f64,
+                                     BitCast, OldExtract.getOperand(1));
+    return DAG.getStore(St->getChain(), dl, NewExtract, St->getBasePtr(),
+                        St->getPointerInfo(), St->isVolatile(),
+                        St->isNonTemporal(), St->getAlignment());
+  }
+
   return SDValue();
 }
 
@@ -23427,32 +24842,97 @@ static SDValue PerformSExtCombine(SDNode *N, SelectionDAG &DAG,
                                   const X86Subtarget *Subtarget) {
   SDValue N0 = N->getOperand(0);
   EVT VT = N->getValueType(0);
+  EVT SVT = VT.getScalarType();
+  EVT InVT = N0.getValueType();
+  EVT InSVT = InVT.getScalarType();
+  SDLoc DL(N);
 
   // (i8,i32 sext (sdivrem (i8 x, i8 y)) ->
   // (i8,i32 (sdivrem_sext_hreg (i8 x, i8 y)
   // This exposes the sext to the sdivrem lowering, so that it directly extends
   // from AH (which we otherwise need to do contortions to access).
   if (N0.getOpcode() == ISD::SDIVREM && N0.getResNo() == 1 &&
-      N0.getValueType() == MVT::i8 && VT == MVT::i32) {
-    SDLoc dl(N);
+      InVT == MVT::i8 && VT == MVT::i32) {
     SDVTList NodeTys = DAG.getVTList(MVT::i8, VT);
-    SDValue R = DAG.getNode(X86ISD::SDIVREM8_SEXT_HREG, dl, NodeTys,
+    SDValue R = DAG.getNode(X86ISD::SDIVREM8_SEXT_HREG, DL, NodeTys,
                             N0.getOperand(0), N0.getOperand(1));
     DAG.ReplaceAllUsesOfValueWith(N0.getValue(0), R.getValue(0));
     return R.getValue(1);
   }
 
-  if (!DCI.isBeforeLegalizeOps())
+  if (!DCI.isBeforeLegalizeOps()) {
+    if (InVT == MVT::i1) {
+      SDValue Zero = DAG.getConstant(0, DL, VT);
+      SDValue AllOnes =
+        DAG.getConstant(APInt::getAllOnesValue(VT.getSizeInBits()), DL, VT);
+      return DAG.getNode(ISD::SELECT, DL, VT, N0, AllOnes, Zero);
+    }
     return SDValue();
+  }
+
+  if (VT.isVector() && Subtarget->hasSSE2()) {
+    auto ExtendVecSize = [&DAG](SDLoc DL, SDValue N, unsigned Size) {
+      EVT InVT = N.getValueType();
+      EVT OutVT = EVT::getVectorVT(*DAG.getContext(), InVT.getScalarType(),
+                                   Size / InVT.getScalarSizeInBits());
+      SmallVector<SDValue, 8> Opnds(Size / InVT.getSizeInBits(),
+                                    DAG.getUNDEF(InVT));
+      Opnds[0] = N;
+      return DAG.getNode(ISD::CONCAT_VECTORS, DL, OutVT, Opnds);
+    };
+
+    // If target-size is less than 128-bits, extend to a type that would extend
+    // to 128 bits, extend that and extract the original target vector.
+    if (VT.getSizeInBits() < 128 && !(128 % VT.getSizeInBits()) &&
+        (SVT == MVT::i64 || SVT == MVT::i32 || SVT == MVT::i16) &&
+        (InSVT == MVT::i32 || InSVT == MVT::i16 || InSVT == MVT::i8)) {
+      unsigned Scale = 128 / VT.getSizeInBits();
+      EVT ExVT =
+          EVT::getVectorVT(*DAG.getContext(), SVT, 128 / SVT.getSizeInBits());
+      SDValue Ex = ExtendVecSize(DL, N0, Scale * InVT.getSizeInBits());
+      SDValue SExt = DAG.getNode(ISD::SIGN_EXTEND, DL, ExVT, Ex);
+      return DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, VT, SExt,
+                         DAG.getIntPtrConstant(0, DL));
+    }
+
+    // If target-size is 128-bits, then convert to ISD::SIGN_EXTEND_VECTOR_INREG
+    // which ensures lowering to X86ISD::VSEXT (pmovsx*).
+    if (VT.getSizeInBits() == 128 &&
+        (SVT == MVT::i64 || SVT == MVT::i32 || SVT == MVT::i16) &&
+        (InSVT == MVT::i32 || InSVT == MVT::i16 || InSVT == MVT::i8)) {
+      SDValue ExOp = ExtendVecSize(DL, N0, 128);
+      return DAG.getSignExtendVectorInReg(ExOp, DL, VT);
+    }
+
+    // On pre-AVX2 targets, split into 128-bit nodes of
+    // ISD::SIGN_EXTEND_VECTOR_INREG.
+    if (!Subtarget->hasInt256() && !(VT.getSizeInBits() % 128) &&
+        (SVT == MVT::i64 || SVT == MVT::i32 || SVT == MVT::i16) &&
+        (InSVT == MVT::i32 || InSVT == MVT::i16 || InSVT == MVT::i8)) {
+      unsigned NumVecs = VT.getSizeInBits() / 128;
+      unsigned NumSubElts = 128 / SVT.getSizeInBits();
+      EVT SubVT = EVT::getVectorVT(*DAG.getContext(), SVT, NumSubElts);
+      EVT InSubVT = EVT::getVectorVT(*DAG.getContext(), InSVT, NumSubElts);
+
+      SmallVector<SDValue, 8> Opnds;
+      for (unsigned i = 0, Offset = 0; i != NumVecs;
+           ++i, Offset += NumSubElts) {
+        SDValue SrcVec = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, InSubVT, N0,
+                                     DAG.getIntPtrConstant(Offset, DL));
+        SrcVec = ExtendVecSize(DL, SrcVec, 128);
+        SrcVec = DAG.getSignExtendVectorInReg(SrcVec, DL, SubVT);
+        Opnds.push_back(SrcVec);
+      }
+      return DAG.getNode(ISD::CONCAT_VECTORS, DL, VT, Opnds);
+    }
+  }
 
   if (!Subtarget->hasFp256())
     return SDValue();
 
-  if (VT.isVector() && VT.getSizeInBits() == 256) {
-    SDValue R = WidenMaskArithmetic(N, DAG, DCI, Subtarget);
-    if (R.getNode())
+  if (VT.isVector() && VT.getSizeInBits() == 256)
+    if (SDValue R = WidenMaskArithmetic(N, DAG, DCI, Subtarget))
       return R;
-  }
 
   return SDValue();
 }
@@ -23468,7 +24948,8 @@ static SDValue PerformFMACombine(SDNode *N, SelectionDAG &DAG,
 
   EVT ScalarVT = VT.getScalarType();
   if ((ScalarVT != MVT::f32 && ScalarVT != MVT::f64) ||
-      (!Subtarget->hasFMA() && !Subtarget->hasFMA4()))
+      (!Subtarget->hasFMA() && !Subtarget->hasFMA4() &&
+       !Subtarget->hasAVX512()))
     return SDValue();
 
   SDValue A = N->getOperand(0);
@@ -23519,7 +25000,7 @@ static SDValue PerformZExtCombine(SDNode *N, SelectionDAG &DAG,
       return DAG.getNode(ISD::AND, dl, VT,
                          DAG.getNode(X86ISD::SETCC_CARRY, dl, VT,
                                      N00.getOperand(0), N00.getOperand(1)),
-                         DAG.getConstant(1, VT));
+                         DAG.getConstant(1, dl, VT));
     }
   }
 
@@ -23531,14 +25012,13 @@ static SDValue PerformZExtCombine(SDNode *N, SelectionDAG &DAG,
       return DAG.getNode(ISD::AND, dl, VT,
                          DAG.getNode(X86ISD::SETCC_CARRY, dl, VT,
                                      N00.getOperand(0), N00.getOperand(1)),
-                         DAG.getConstant(1, VT));
+                         DAG.getConstant(1, dl, VT));
     }
   }
-  if (VT.is256BitVector()) {
-    SDValue R = WidenMaskArithmetic(N, DAG, DCI, Subtarget);
-    if (R.getNode())
+
+  if (VT.is256BitVector())
+    if (SDValue R = WidenMaskArithmetic(N, DAG, DCI, Subtarget))
       return R;
-  }
 
   // (i8,i32 zext (udivrem (i8 x, i8 y)) ->
   // (i8,i32 (udivrem_zext_hreg (i8 x, i8 y)
@@ -23570,18 +25050,18 @@ static SDValue PerformISDSETCCCombine(SDNode *N, SelectionDAG &DAG,
   if ((CC == ISD::SETNE || CC == ISD::SETEQ) && LHS.getOpcode() == ISD::SUB)
     if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(LHS.getOperand(0)))
       if (C->getAPIntValue() == 0 && LHS.hasOneUse()) {
-        SDValue addV = DAG.getNode(ISD::ADD, SDLoc(N), LHS.getValueType(), RHS,
+        SDValue addV = DAG.getNode(ISD::ADD, DL, LHS.getValueType(), RHS,
                                    LHS.getOperand(1));
-        return DAG.getSetCC(SDLoc(N), N->getValueType(0), addV,
-                            DAG.getConstant(0, addV.getValueType()), CC);
+        return DAG.getSetCC(DL, N->getValueType(0), addV,
+                            DAG.getConstant(0, DL, addV.getValueType()), CC);
       }
   if ((CC == ISD::SETNE || CC == ISD::SETEQ) && RHS.getOpcode() == ISD::SUB)
     if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(RHS.getOperand(0)))
       if (C->getAPIntValue() == 0 && RHS.hasOneUse()) {
-        SDValue addV = DAG.getNode(ISD::ADD, SDLoc(N), RHS.getValueType(), LHS,
+        SDValue addV = DAG.getNode(ISD::ADD, DL, RHS.getValueType(), LHS,
                                    RHS.getOperand(1));
-        return DAG.getSetCC(SDLoc(N), N->getValueType(0), addV,
-                            DAG.getConstant(0, addV.getValueType()), CC);
+        return DAG.getSetCC(DL, N->getValueType(0), addV,
+                            DAG.getConstant(0, DL, addV.getValueType()), CC);
       }
 
   if (VT.getScalarType() == MVT::i1 &&
@@ -23605,9 +25085,9 @@ static SDValue PerformISDSETCCCombine(SDNode *N, SelectionDAG &DAG,
       assert(VT == LHS.getOperand(0).getValueType() &&
              "Uexpected operand type");
       if (CC == ISD::SETGT)
-        return DAG.getConstant(0, VT);
+        return DAG.getConstant(0, DL, VT);
       if (CC == ISD::SETLE)
-        return DAG.getConstant(1, VT);
+        return DAG.getConstant(1, DL, VT);
       if (CC == ISD::SETEQ || CC == ISD::SETGE)
         return DAG.getNOT(DL, LHS.getOperand(0), VT);
 
@@ -23628,7 +25108,8 @@ static SDValue NarrowVectorLoadToElement(LoadSDNode *Load, unsigned Index,
   SDValue Addr = Load->getOperand(1);
   SDValue NewAddr = DAG.getNode(
       ISD::ADD, dl, Addr.getSimpleValueType(), Addr,
-      DAG.getConstant(Index * EVT.getStoreSize(), Addr.getSimpleValueType()));
+      DAG.getConstant(Index * EVT.getStoreSize(), dl,
+                      Addr.getSimpleValueType()));
 
   SDValue NewLoad =
       DAG.getLoad(EVT, dl, Load->getChain(), NewAddr,
@@ -23684,7 +25165,7 @@ static SDValue PerformBLENDICombine(SDNode *N, SelectionDAG &DAG) {
   if (VT == MVT::v2f64)
     if (auto *Mask = dyn_cast<ConstantSDNode>(N->getOperand(2)))
       if (Mask->getZExtValue() == 2 && !isShuffleFoldableLoad(V0)) {
-        SDValue NewMask = DAG.getConstant(1, MVT::i8);
+        SDValue NewMask = DAG.getConstant(1, DL, MVT::i8);
         return DAG.getNode(X86ISD::BLENDI, DL, VT, V1, V0, NewMask);
       }
 
@@ -23699,12 +25180,14 @@ static SDValue MaterializeSETB(SDLoc DL, SDValue EFLAGS, SelectionDAG &DAG,
   if (VT == MVT::i8)
     return DAG.getNode(ISD::AND, DL, VT,
                        DAG.getNode(X86ISD::SETCC_CARRY, DL, MVT::i8,
-                                   DAG.getConstant(X86::COND_B, MVT::i8), EFLAGS),
-                       DAG.getConstant(1, VT));
+                                   DAG.getConstant(X86::COND_B, DL, MVT::i8),
+                                   EFLAGS),
+                       DAG.getConstant(1, DL, VT));
   assert (VT == MVT::i1 && "Unexpected type for SECCC node");
   return DAG.getNode(ISD::TRUNCATE, DL, MVT::i1,
                      DAG.getNode(X86ISD::SETCC_CARRY, DL, MVT::i8,
-                                 DAG.getConstant(X86::COND_B, MVT::i8), EFLAGS));
+                                 DAG.getConstant(X86::COND_B, DL, MVT::i8),
+                                 EFLAGS));
 }
 
 // Optimize  RES = X86ISD::SETCC CONDCODE, EFLAG_INPUT
@@ -23739,11 +25222,8 @@ static SDValue PerformSETCCCombine(SDNode *N, SelectionDAG &DAG,
   if (CC == X86::COND_B)
     return MaterializeSETB(DL, EFLAGS, DAG, N->getSimpleValueType(0));
 
-  SDValue Flags;
-
-  Flags = checkBoolTestSetCCCombine(EFLAGS, CC);
-  if (Flags.getNode()) {
-    SDValue Cond = DAG.getConstant(CC, MVT::i8);
+  if (SDValue Flags = checkBoolTestSetCCCombine(EFLAGS, CC)) {
+    SDValue Cond = DAG.getConstant(CC, DL, MVT::i8);
     return DAG.getNode(X86ISD::SETCC, DL, N->getVTList(), Cond, Flags);
   }
 
@@ -23761,11 +25241,8 @@ static SDValue PerformBrCondCombine(SDNode *N, SelectionDAG &DAG,
   SDValue EFLAGS = N->getOperand(3);
   X86::CondCode CC = X86::CondCode(N->getConstantOperandVal(2));
 
-  SDValue Flags;
-
-  Flags = checkBoolTestSetCCCombine(EFLAGS, CC);
-  if (Flags.getNode()) {
-    SDValue Cond = DAG.getConstant(CC, MVT::i8);
+  if (SDValue Flags = checkBoolTestSetCCCombine(EFLAGS, CC)) {
+    SDValue Cond = DAG.getConstant(CC, DL, MVT::i8);
     return DAG.getNode(X86ISD::BRCOND, DL, N->getVTList(), Chain, Dest, Cond,
                        Flags);
   }
@@ -23809,46 +25286,79 @@ static SDValue performVectorCompareAndMaskUnaryOpCombine(SDNode *N,
     // DAG.
     SDValue SourceConst = DAG.getNode(N->getOpcode(), DL, VT, SDValue(BV, 0));
     // The AND node needs bitcasts to/from an integer vector type around it.
-    SDValue MaskConst = DAG.getNode(ISD::BITCAST, DL, IntVT, SourceConst);
+    SDValue MaskConst = DAG.getBitcast(IntVT, SourceConst);
     SDValue NewAnd = DAG.getNode(ISD::AND, DL, IntVT,
                                  N->getOperand(0)->getOperand(0), MaskConst);
-    SDValue Res = DAG.getNode(ISD::BITCAST, DL, VT, NewAnd);
+    SDValue Res = DAG.getBitcast(VT, NewAnd);
     return Res;
   }
 
   return SDValue();
 }
 
+static SDValue PerformUINT_TO_FPCombine(SDNode *N, SelectionDAG &DAG,
+                                        const X86Subtarget *Subtarget) {
+  SDValue Op0 = N->getOperand(0);
+  EVT VT = N->getValueType(0);
+  EVT InVT = Op0.getValueType();
+  EVT InSVT = InVT.getScalarType();
+  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
+
+  // UINT_TO_FP(vXi8) -> SINT_TO_FP(ZEXT(vXi8 to vXi32))
+  // UINT_TO_FP(vXi16) -> SINT_TO_FP(ZEXT(vXi16 to vXi32))
+  if (InVT.isVector() && (InSVT == MVT::i8 || InSVT == MVT::i16)) {
+    SDLoc dl(N);
+    EVT DstVT = EVT::getVectorVT(*DAG.getContext(), MVT::i32,
+                                 InVT.getVectorNumElements());
+    SDValue P = DAG.getNode(ISD::ZERO_EXTEND, dl, DstVT, Op0);
+
+    if (TLI.isOperationLegal(ISD::UINT_TO_FP, DstVT))
+      return DAG.getNode(ISD::UINT_TO_FP, dl, VT, P);
+
+    return DAG.getNode(ISD::SINT_TO_FP, dl, VT, P);
+  }
+
+  return SDValue();
+}
+
 static SDValue PerformSINT_TO_FPCombine(SDNode *N, SelectionDAG &DAG,
                                         const X86Subtarget *Subtarget) {
   // First try to optimize away the conversion entirely when it's
   // conditionally from a constant. Vectors only.
-  SDValue Res = performVectorCompareAndMaskUnaryOpCombine(N, DAG);
-  if (Res != SDValue())
+  if (SDValue Res = performVectorCompareAndMaskUnaryOpCombine(N, DAG))
     return Res;
 
   // Now move on to more general possibilities.
   SDValue Op0 = N->getOperand(0);
-  EVT InVT = Op0->getValueType(0);
+  EVT VT = N->getValueType(0);
+  EVT InVT = Op0.getValueType();
+  EVT InSVT = InVT.getScalarType();
 
-  // SINT_TO_FP(v4i8) -> SINT_TO_FP(SEXT(v4i8 to v4i32))
-  if (InVT == MVT::v8i8 || InVT == MVT::v4i8) {
+  // SINT_TO_FP(vXi8) -> SINT_TO_FP(SEXT(vXi8 to vXi32))
+  // SINT_TO_FP(vXi16) -> SINT_TO_FP(SEXT(vXi16 to vXi32))
+  if (InVT.isVector() && (InSVT == MVT::i8 || InSVT == MVT::i16)) {
     SDLoc dl(N);
-    MVT DstVT = InVT == MVT::v4i8 ? MVT::v4i32 : MVT::v8i32;
+    EVT DstVT = EVT::getVectorVT(*DAG.getContext(), MVT::i32,
+                                 InVT.getVectorNumElements());
     SDValue P = DAG.getNode(ISD::SIGN_EXTEND, dl, DstVT, Op0);
-    return DAG.getNode(ISD::SINT_TO_FP, dl, N->getValueType(0), P);
+    return DAG.getNode(ISD::SINT_TO_FP, dl, VT, P);
   }
 
   // Transform (SINT_TO_FP (i64 ...)) into an x87 operation if we have
   // a 32-bit target where SSE doesn't support i64->FP operations.
   if (Op0.getOpcode() == ISD::LOAD) {
     LoadSDNode *Ld = cast<LoadSDNode>(Op0.getNode());
-    EVT VT = Ld->getValueType(0);
-    if (!Ld->isVolatile() && !N->getValueType(0).isVector() &&
+    EVT LdVT = Ld->getValueType(0);
+
+    // This transformation is not supported if the result type is f16
+    if (VT == MVT::f16)
+      return SDValue();
+
+    if (!Ld->isVolatile() && !VT.isVector() &&
         ISD::isNON_EXTLoad(Op0.getNode()) && Op0.hasOneUse() &&
-        !Subtarget->is64Bit() && VT == MVT::i64) {
+        !Subtarget->is64Bit() && LdVT == MVT::i64) {
       SDValue FILDChain = Subtarget->getTargetLowering()->BuildFILD(
-          SDValue(N, 0), Ld->getValueType(0), Ld->getChain(), Op0, DAG);
+          SDValue(N, 0), LdVT, Ld->getChain(), Op0, DAG);
       DAG.ReplaceAllUsesOfValueWith(Op0.getValue(1), FILDChain.getValue(1));
       return FILDChain;
     }
@@ -23869,12 +25379,13 @@ static SDValue PerformADCCombine(SDNode *N, SelectionDAG &DAG,
       SDValue(N, 1).use_empty()) {
     SDLoc DL(N);
     EVT VT = N->getValueType(0);
-    SDValue CarryOut = DAG.getConstant(0, N->getValueType(1));
+    SDValue CarryOut = DAG.getConstant(0, DL, N->getValueType(1));
     SDValue Res1 = DAG.getNode(ISD::AND, DL, VT,
                                DAG.getNode(X86ISD::SETCC_CARRY, DL, VT,
-                                           DAG.getConstant(X86::COND_B,MVT::i8),
+                                           DAG.getConstant(X86::COND_B, DL,
+                                                           MVT::i8),
                                            N->getOperand(2)),
-                               DAG.getConstant(1, VT));
+                               DAG.getConstant(1, DL, VT));
     return DCI.CombineTo(N, Res1, CarryOut);
   }
 
@@ -23909,16 +25420,17 @@ static SDValue OptimizeConditionalInDecrement(SDNode *N, SelectionDAG &DAG) {
 
   SDValue CmpOp0 = Cmp.getOperand(0);
   SDValue NewCmp = DAG.getNode(X86ISD::CMP, DL, MVT::i32, CmpOp0,
-                               DAG.getConstant(1, CmpOp0.getValueType()));
+                               DAG.getConstant(1, DL, CmpOp0.getValueType()));
 
   SDValue OtherVal = N->getOperand(N->getOpcode() == ISD::SUB ? 0 : 1);
   if (CC == X86::COND_NE)
     return DAG.getNode(N->getOpcode() == ISD::SUB ? X86ISD::ADC : X86ISD::SBB,
                        DL, OtherVal.getValueType(), OtherVal,
-                       DAG.getConstant(-1ULL, OtherVal.getValueType()), NewCmp);
+                       DAG.getConstant(-1ULL, DL, OtherVal.getValueType()),
+                       NewCmp);
   return DAG.getNode(N->getOpcode() == ISD::SUB ? X86ISD::SBB : X86ISD::ADC,
                      DL, OtherVal.getValueType(), OtherVal,
-                     DAG.getConstant(0, OtherVal.getValueType()), NewCmp);
+                     DAG.getConstant(0, DL, OtherVal.getValueType()), NewCmp);
 }
 
 /// PerformADDCombine - Do target-specific dag combines on integer adds.
@@ -23954,9 +25466,9 @@ static SDValue PerformSubCombine(SDNode *N, SelectionDAG &DAG,
       EVT VT = Op0.getValueType();
       SDValue NewXor = DAG.getNode(ISD::XOR, SDLoc(Op1), VT,
                                    Op1.getOperand(0),
-                                   DAG.getConstant(~XorC, VT));
+                                   DAG.getConstant(~XorC, SDLoc(Op1), VT));
       return DAG.getNode(ISD::ADD, SDLoc(N), VT, NewXor,
-                         DAG.getConstant(C->getAPIntValue()+1, VT));
+                         DAG.getConstant(C->getAPIntValue() + 1, SDLoc(N), VT));
     }
   }
 
@@ -24005,8 +25517,7 @@ static SDValue performVZEXTCombine(SDNode *N, SelectionDAG &DAG,
     // In this case, the inner vzext is completely dead because we're going to
     // only look at bits inside of the low element. Just do the outer vzext on
     // a bitcast of the input to the inner.
-    return DAG.getNode(X86ISD::VZEXT, DL, VT,
-                       DAG.getNode(ISD::BITCAST, DL, OpVT, V));
+    return DAG.getNode(X86ISD::VZEXT, DL, VT, DAG.getBitcast(OpVT, V));
   }
 
   // Check if we can bypass extracting and re-inserting an element of an input
@@ -24026,9 +25537,9 @@ static SDValue performVZEXTCombine(SDNode *N, SelectionDAG &DAG,
           OrigVT = MVT::getVectorVT(OrigVT.getVectorElementType(),
                                     OrigVT.getVectorNumElements() / Ratio);
           OrigV = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, OrigVT, OrigV,
-                              DAG.getIntPtrConstant(0));
+                              DAG.getIntPtrConstant(0, DL));
         }
-        Op = DAG.getNode(ISD::BITCAST, DL, OpVT, OrigV);
+        Op = DAG.getBitcast(OpVT, OrigV);
         return DAG.getNode(X86ISD::VZEXT, DL, VT, Op);
       }
   }
@@ -24064,6 +25575,7 @@ SDValue X86TargetLowering::PerformDAGCombine(SDNode *N,
   case ISD::STORE:          return PerformSTORECombine(N, DAG, Subtarget);
   case ISD::MSTORE:         return PerformMSTORECombine(N, DAG, Subtarget);
   case ISD::SINT_TO_FP:     return PerformSINT_TO_FPCombine(N, DAG, Subtarget);
+  case ISD::UINT_TO_FP:     return PerformUINT_TO_FPCombine(N, DAG, Subtarget);
   case ISD::FADD:           return PerformFADDCombine(N, DAG, Subtarget);
   case ISD::FSUB:           return PerformFSUBCombine(N, DAG, Subtarget);
   case X86ISD::FXOR:
@@ -24079,7 +25591,6 @@ SDValue X86TargetLowering::PerformDAGCombine(SDNode *N,
   case ISD::SIGN_EXTEND:    return PerformSExtCombine(N, DAG, DCI, Subtarget);
   case ISD::SIGN_EXTEND_INREG:
     return PerformSIGN_EXTEND_INREGCombine(N, DAG, Subtarget);
-  case ISD::TRUNCATE:       return PerformTruncateCombine(N, DAG,DCI,Subtarget);
   case ISD::SETCC:          return PerformISDSETCCCombine(N, DAG, Subtarget);
   case X86ISD::SETCC:       return PerformSETCCCombine(N, DAG, DCI, Subtarget);
   case X86ISD::BRCOND:      return PerformBrCondCombine(N, DAG, DCI, Subtarget);
@@ -24108,7 +25619,6 @@ SDValue X86TargetLowering::PerformDAGCombine(SDNode *N,
     break;
   }
   case X86ISD::BLENDI:    return PerformBLENDICombine(N, DAG);
-  case ISD::BUILD_VECTOR: return PerformBUILD_VECTORCombine(N, DAG, Subtarget);
   }
 
   return SDValue();
@@ -24288,7 +25798,7 @@ bool X86TargetLowering::ExpandInlineAsm(CallInst *CI) const {
         (matchAsm(AsmPieces[0], {"rorw", "$$8,", "${0:w}"}) ||
          matchAsm(AsmPieces[0], {"rolw", "$$8,", "${0:w}"}))) {
       AsmPieces.clear();
-      const std::string &ConstraintsStr = IA->getConstraintString();
+      StringRef ConstraintsStr = IA->getConstraintString();
       SplitString(StringRef(ConstraintsStr).substr(5), AsmPieces, ",");
       array_pod_sort(AsmPieces.begin(), AsmPieces.end());
       if (clobbersFlagRegisters(AsmPieces))
@@ -24302,7 +25812,7 @@ bool X86TargetLowering::ExpandInlineAsm(CallInst *CI) const {
         matchAsm(AsmPieces[1], {"rorl", "$$16,", "$0"}) &&
         matchAsm(AsmPieces[2], {"rorw", "$$8,", "${0:w}"})) {
       AsmPieces.clear();
-      const std::string &ConstraintsStr = IA->getConstraintString();
+      StringRef ConstraintsStr = IA->getConstraintString();
       SplitString(StringRef(ConstraintsStr).substr(5), AsmPieces, ",");
       array_pod_sort(AsmPieces.begin(), AsmPieces.end());
       if (clobbersFlagRegisters(AsmPieces))
@@ -24329,7 +25839,7 @@ bool X86TargetLowering::ExpandInlineAsm(CallInst *CI) const {
 /// getConstraintType - Given a constraint letter, return the type of
 /// constraint it is for this target.
 X86TargetLowering::ConstraintType
-X86TargetLowering::getConstraintType(const std::string &Constraint) const {
+X86TargetLowering::getConstraintType(StringRef Constraint) const {
   if (Constraint.size() == 1) {
     switch (Constraint[0]) {
     case 'R':
@@ -24508,7 +26018,8 @@ void X86TargetLowering::LowerAsmOperandForConstraint(SDValue Op,
   case 'I':
     if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op)) {
       if (C->getZExtValue() <= 31) {
-        Result = DAG.getTargetConstant(C->getZExtValue(), Op.getValueType());
+        Result = DAG.getTargetConstant(C->getZExtValue(), SDLoc(Op),
+                                       Op.getValueType());
         break;
       }
     }
@@ -24516,7 +26027,8 @@ void X86TargetLowering::LowerAsmOperandForConstraint(SDValue Op,
   case 'J':
     if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op)) {
       if (C->getZExtValue() <= 63) {
-        Result = DAG.getTargetConstant(C->getZExtValue(), Op.getValueType());
+        Result = DAG.getTargetConstant(C->getZExtValue(), SDLoc(Op),
+                                       Op.getValueType());
         break;
       }
     }
@@ -24524,7 +26036,8 @@ void X86TargetLowering::LowerAsmOperandForConstraint(SDValue Op,
   case 'K':
     if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op)) {
       if (isInt<8>(C->getSExtValue())) {
-        Result = DAG.getTargetConstant(C->getZExtValue(), Op.getValueType());
+        Result = DAG.getTargetConstant(C->getZExtValue(), SDLoc(Op),
+                                       Op.getValueType());
         break;
       }
     }
@@ -24533,7 +26046,8 @@ void X86TargetLowering::LowerAsmOperandForConstraint(SDValue Op,
     if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op)) {
       if (C->getZExtValue() == 0xff || C->getZExtValue() == 0xffff ||
           (Subtarget->is64Bit() && C->getZExtValue() == 0xffffffff)) {
-        Result = DAG.getTargetConstant(C->getSExtValue(), Op.getValueType());
+        Result = DAG.getTargetConstant(C->getSExtValue(), SDLoc(Op),
+                                       Op.getValueType());
         break;
       }
     }
@@ -24541,7 +26055,8 @@ void X86TargetLowering::LowerAsmOperandForConstraint(SDValue Op,
   case 'M':
     if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op)) {
       if (C->getZExtValue() <= 3) {
-        Result = DAG.getTargetConstant(C->getZExtValue(), Op.getValueType());
+        Result = DAG.getTargetConstant(C->getZExtValue(), SDLoc(Op),
+                                       Op.getValueType());
         break;
       }
     }
@@ -24549,7 +26064,8 @@ void X86TargetLowering::LowerAsmOperandForConstraint(SDValue Op,
   case 'N':
     if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op)) {
       if (C->getZExtValue() <= 255) {
-        Result = DAG.getTargetConstant(C->getZExtValue(), Op.getValueType());
+        Result = DAG.getTargetConstant(C->getZExtValue(), SDLoc(Op),
+                                       Op.getValueType());
         break;
       }
     }
@@ -24557,7 +26073,8 @@ void X86TargetLowering::LowerAsmOperandForConstraint(SDValue Op,
   case 'O':
     if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op)) {
       if (C->getZExtValue() <= 127) {
-        Result = DAG.getTargetConstant(C->getZExtValue(), Op.getValueType());
+        Result = DAG.getTargetConstant(C->getZExtValue(), SDLoc(Op),
+                                       Op.getValueType());
         break;
       }
     }
@@ -24568,7 +26085,7 @@ void X86TargetLowering::LowerAsmOperandForConstraint(SDValue Op,
       if (ConstantInt::isValueValidForType(Type::getInt32Ty(*DAG.getContext()),
                                            C->getSExtValue())) {
         // Widen to 64 bits here to get it sign extended.
-        Result = DAG.getTargetConstant(C->getSExtValue(), MVT::i64);
+        Result = DAG.getTargetConstant(C->getSExtValue(), SDLoc(Op), MVT::i64);
         break;
       }
     // FIXME gcc accepts some relocatable values here too, but only in certain
@@ -24581,7 +26098,8 @@ void X86TargetLowering::LowerAsmOperandForConstraint(SDValue Op,
     if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op)) {
       if (ConstantInt::isValueValidForType(Type::getInt32Ty(*DAG.getContext()),
                                            C->getZExtValue())) {
-        Result = DAG.getTargetConstant(C->getZExtValue(), Op.getValueType());
+        Result = DAG.getTargetConstant(C->getZExtValue(), SDLoc(Op),
+                                       Op.getValueType());
         break;
       }
     }
@@ -24593,7 +26111,7 @@ void X86TargetLowering::LowerAsmOperandForConstraint(SDValue Op,
     // Literal immediates are always ok.
     if (ConstantSDNode *CST = dyn_cast<ConstantSDNode>(Op)) {
       // Widen to 64 bits here to get it sign extended.
-      Result = DAG.getTargetConstant(CST->getSExtValue(), MVT::i64);
+      Result = DAG.getTargetConstant(CST->getSExtValue(), SDLoc(Op), MVT::i64);
       break;
     }
 
@@ -24653,7 +26171,7 @@ void X86TargetLowering::LowerAsmOperandForConstraint(SDValue Op,
 
 std::pair<unsigned, const TargetRegisterClass *>
 X86TargetLowering::getRegForInlineAsmConstraint(const TargetRegisterInfo *TRI,
-                                                const std::string &Constraint,
+                                                StringRef Constraint,
                                                 MVT VT) const {
   // First, see if this is a constraint that directly corresponds to an LLVM
   // register class.
@@ -24803,71 +26321,40 @@ X86TargetLowering::getRegForInlineAsmConstraint(const TargetRegisterInfo *TRI,
   // Otherwise, check to see if this is a register class of the wrong value
   // type.  For example, we want to map "{ax},i32" -> {eax}, we don't want it to
   // turn into {ax},{dx}.
-  if (Res.second->hasType(VT))
+  // MVT::Other is used to specify clobber names.
+  if (Res.second->hasType(VT) || VT == MVT::Other)
     return Res;   // Correct type already, nothing to do.
 
-  // All of the single-register GCC register classes map their values onto
-  // 16-bit register pieces "ax","dx","cx","bx","si","di","bp","sp".  If we
-  // really want an 8-bit or 32-bit register, map to the appropriate register
-  // class and return the appropriate register.
-  if (Res.second == &X86::GR16RegClass) {
-    if (VT == MVT::i8 || VT == MVT::i1) {
-      unsigned DestReg = 0;
-      switch (Res.first) {
-      default: break;
-      case X86::AX: DestReg = X86::AL; break;
-      case X86::DX: DestReg = X86::DL; break;
-      case X86::CX: DestReg = X86::CL; break;
-      case X86::BX: DestReg = X86::BL; break;
-      }
-      if (DestReg) {
-        Res.first = DestReg;
-        Res.second = &X86::GR8RegClass;
-      }
-    } else if (VT == MVT::i32 || VT == MVT::f32) {
-      unsigned DestReg = 0;
-      switch (Res.first) {
-      default: break;
-      case X86::AX: DestReg = X86::EAX; break;
-      case X86::DX: DestReg = X86::EDX; break;
-      case X86::CX: DestReg = X86::ECX; break;
-      case X86::BX: DestReg = X86::EBX; break;
-      case X86::SI: DestReg = X86::ESI; break;
-      case X86::DI: DestReg = X86::EDI; break;
-      case X86::BP: DestReg = X86::EBP; break;
-      case X86::SP: DestReg = X86::ESP; break;
-      }
-      if (DestReg) {
-        Res.first = DestReg;
-        Res.second = &X86::GR32RegClass;
-      }
-    } else if (VT == MVT::i64 || VT == MVT::f64) {
-      unsigned DestReg = 0;
-      switch (Res.first) {
-      default: break;
-      case X86::AX: DestReg = X86::RAX; break;
-      case X86::DX: DestReg = X86::RDX; break;
-      case X86::CX: DestReg = X86::RCX; break;
-      case X86::BX: DestReg = X86::RBX; break;
-      case X86::SI: DestReg = X86::RSI; break;
-      case X86::DI: DestReg = X86::RDI; break;
-      case X86::BP: DestReg = X86::RBP; break;
-      case X86::SP: DestReg = X86::RSP; break;
-      }
-      if (DestReg) {
-        Res.first = DestReg;
-        Res.second = &X86::GR64RegClass;
-      }
-    }
-  } else if (Res.second == &X86::FR32RegClass ||
-             Res.second == &X86::FR64RegClass ||
-             Res.second == &X86::VR128RegClass ||
-             Res.second == &X86::VR256RegClass ||
-             Res.second == &X86::FR32XRegClass ||
-             Res.second == &X86::FR64XRegClass ||
-             Res.second == &X86::VR128XRegClass ||
-             Res.second == &X86::VR256XRegClass ||
-             Res.second == &X86::VR512RegClass) {
+  // Get a matching integer of the correct size. i.e. "ax" with MVT::32 should
+  // return "eax". This should even work for things like getting 64bit integer
+  // registers when given an f64 type.
+  const TargetRegisterClass *Class = Res.second;
+  if (Class == &X86::GR8RegClass || Class == &X86::GR16RegClass ||
+      Class == &X86::GR32RegClass || Class == &X86::GR64RegClass) {
+    unsigned Size = VT.getSizeInBits();
+    MVT::SimpleValueType SimpleTy = Size == 1 || Size == 8 ? MVT::i8
+                                  : Size == 16 ? MVT::i16
+                                  : Size == 32 ? MVT::i32
+                                  : Size == 64 ? MVT::i64
+                                  : MVT::Other;
+    unsigned DestReg = getX86SubSuperRegisterOrZero(Res.first, SimpleTy);
+    if (DestReg > 0) {
+      Res.first = DestReg;
+      Res.second = SimpleTy == MVT::i8 ? &X86::GR8RegClass
+                 : SimpleTy == MVT::i16 ? &X86::GR16RegClass
+                 : SimpleTy == MVT::i32 ? &X86::GR32RegClass
+                 : &X86::GR64RegClass;
+      assert(Res.second->contains(Res.first) && "Register in register class");
+    } else {
+      // No register found/type mismatch.
+      Res.first = 0;
+      Res.second = nullptr;
+    }
+  } else if (Class == &X86::FR32RegClass || Class == &X86::FR64RegClass ||
+             Class == &X86::VR128RegClass || Class == &X86::VR256RegClass ||
+             Class == &X86::FR32XRegClass || Class == &X86::FR64XRegClass ||
+             Class == &X86::VR128XRegClass || Class == &X86::VR256XRegClass ||
+             Class == &X86::VR512RegClass) {
     // Handle references to XMM physical registers that got mapped into the
     // wrong class.  This can happen with constraints like {xmm0} where the
     // target independent register mapper will just pick the first match it can
@@ -24883,13 +26370,19 @@ X86TargetLowering::getRegForInlineAsmConstraint(const TargetRegisterInfo *TRI,
       Res.second = &X86::VR256RegClass;
     else if (X86::VR512RegClass.hasType(VT))
       Res.second = &X86::VR512RegClass;
+    else {
+      // Type mismatch and not a clobber: Return an error;
+      Res.first = 0;
+      Res.second = nullptr;
+    }
   }
 
   return Res;
 }
 
-int X86TargetLowering::getScalingFactorCost(const AddrMode &AM,
-                                            Type *Ty) const {
+int X86TargetLowering::getScalingFactorCost(const DataLayout &DL,
+                                            const AddrMode &AM, Type *Ty,
+                                            unsigned AS) const {
   // Scaling factors are not free at all.
   // An indexed folded instruction, i.e., inst (reg1, reg2, scale),
   // will take 2 allocations in the out of order engine instead of 1
@@ -24908,7 +26401,7 @@ int X86TargetLowering::getScalingFactorCost(const AddrMode &AM,
   // E.g., on Haswell:
   // vmovaps %ymm1, (%r8, %rdi) can use port 2 or 3.
   // vmovaps %ymm1, (%r8) can use port 2, 3, or 7.
-  if (isLegalAddressingMode(AM, Ty))
+  if (isLegalAddressingMode(DL, AM, Ty, AS))
     // Scale represents reg2 * scale, thus account for 1
     // as soon as we use a second register.
     return AM.Scale != 0;
diff --git a/lib/Target/X86/X86ISelLowering.h b/lib/Target/X86/X86ISelLowering.h
index c4b1f017bfc..18f823c6201 100644
--- a/lib/Target/X86/X86ISelLowering.h
+++ b/lib/Target/X86/X86ISelLowering.h
@@ -26,7 +26,7 @@ namespace llvm {
 
   namespace X86ISD {
     // X86 Specific DAG Nodes
-    enum NodeType {
+    enum NodeType : unsigned {
       // Start the numbering where the builtin ops leave off.
       FIRST_NUMBER = ISD::BUILTIN_OP_END,
 
@@ -56,10 +56,6 @@ namespace llvm {
       /// corresponds to X86::ANDNPS or X86::ANDNPD.
       FANDN,
 
-      /// Bitwise logical right shift of floating point values. This
-      /// corresponds to X86::PSRLDQ.
-      FSRL,
-
       /// These operations represent an abstract X86 call
       /// instruction, which includes a bunch of information.  In particular the
       /// operands of these node are:
@@ -184,6 +180,9 @@ namespace llvm {
       /// Shuffle 16 8-bit values within a vector.
       PSHUFB,
 
+      /// Compute Sum of Absolute Differences.
+      PSADBW,
+
       /// Bitwise Logical AND NOT of Packed FP values.
       ANDNP,
 
@@ -200,15 +199,28 @@ namespace llvm {
 
       /// Combined add and sub on an FP vector.
       ADDSUB,
+
       //  FP vector ops with rounding mode.
       FADD_RND,
       FSUB_RND,
       FMUL_RND,
       FDIV_RND,
-      
-      // Integer sub with unsigned saturation.
+      FMAX_RND,
+      FMIN_RND,
+      FSQRT_RND,
+
+      // FP vector get exponent 
+      FGETEXP_RND,
+      // FP Scale
+      SCALEF,
+      // Integer add/sub with unsigned saturation.
+      ADDUS,
       SUBUS,
-
+      // Integer add/sub with signed saturation.
+      ADDS,
+      SUBS,
+      // Unsigned Integer average 
+      AVG,
       /// Integer horizontal add.
       HADD,
 
@@ -221,11 +233,8 @@ namespace llvm {
       /// Floating point horizontal sub.
       FHSUB,
 
-      /// Unsigned integer max and min.
-      UMAX, UMIN,
-
-      /// Signed integer max and min.
-      SMAX, SMIN,
+      // Integer absolute value
+      ABS,
 
       /// Floating point max and min.
       FMAX, FMIN,
@@ -244,16 +253,6 @@ namespace llvm {
       // Thread Local Storage. A call to get the start address
       // of the TLS block for the current module.
       TLSBASEADDR,
-      // @LOCALMOD-BEGIN
-      // TLSADDR_LE - Thread Local Storage. (Local Exec Model)
-      TLSADDR_LE,
-
-      // TLSADDR_IE - Thread Local Storage. (Initial Exec Model)
-      TLSADDR_IE,
-
-      // THREAD_POINTER_FROM_GS - Read thread pointer from %gs:0 on x86-32.
-      THREAD_POINTER_FROM_GS,
-      // @LOCALMOD-END
 
       // Thread Local Storage.  When calling to an OS provided
       // thunk at the address from an earlier relocation.
@@ -283,9 +282,8 @@ namespace llvm {
 
       // Vector integer truncate.
       VTRUNC,
-
-      // Vector integer truncate with mask.
-      VTRUNCM,
+      // Vector integer truncate with unsigned/signed saturation.
+      VTRUNCUS, VTRUNCS,
 
       // Vector FP extend.
       VFPEXT,
@@ -293,6 +291,9 @@ namespace llvm {
       // Vector FP round.
       VFPROUND,
 
+      // Vector signed/unsigned integer to double.
+      CVTDQ2PD, CVTUDQ2PD,
+
       // 128-bit vector logical left / right shift
       VSHLDQ, VSRLDQ,
 
@@ -314,6 +315,8 @@ namespace llvm {
       /// integer signed and unsigned data types.
       CMPM,
       CMPMU,
+      // Vector comparison with rounding mode for FP values
+      CMPM_RND,
 
       // Arithmetic operations with FLAGS results.
       ADD, SUB, ADC, SBB, SMUL,
@@ -357,6 +360,8 @@ namespace llvm {
       PSHUFHW,
       PSHUFLW,
       SHUFP,
+      //Shuffle Packed Values at 128-bit granularity
+      SHUF128,
       MOVDDUP,
       MOVSHDUP,
       MOVSLDUP,
@@ -376,18 +381,33 @@ namespace llvm {
       VPERMIV3,
       VPERMI,
       VPERM2X128,
+      //Fix Up Special Packed Float32/64 values
+      VFIXUPIMM,
+      //Range Restriction Calculation For Packed Pairs of Float32/64 values
+      VRANGE,
+      // Reduce - Perform Reduction Transformation on scalar\packed FP
+      VREDUCE,
+      // RndScale - Round FP Values To Include A Given Number Of Fraction Bits
+      VRNDSCALE,
+      // Broadcast scalar to vector
       VBROADCAST,
-      // masked broadcast
-      VBROADCASTM,
+      // Broadcast subvector to vector
+      SUBV_BROADCAST,
       // Insert/Extract vector element
       VINSERT,
       VEXTRACT,
 
+      /// SSE4A Extraction and Insertion.
+      EXTRQI, INSERTQI,
+
       // Vector multiply packed unsigned doubleword integers
       PMULUDQ,
       // Vector multiply packed signed doubleword integers
       PMULDQ,
-
+      // Vector Multiply Packed UnsignedIntegers with Round and Scale
+      MULHRS,
+      // Multiply and Add Packed Integers
+      VPMADDUBSW, VPMADDWD,
       // FMA nodes
       FMADD,
       FNMADD,
@@ -402,12 +422,18 @@ namespace llvm {
       FNMSUB_RND,
       FMADDSUB_RND,
       FMSUBADD_RND,
-      RNDSCALE,
 
       // Compress and expand
       COMPRESS,
       EXPAND,
 
+      //Convert Unsigned/Integer to Scalar Floating-Point Value
+      //with rounding mode
+      SINT_TO_FP_RND,
+      UINT_TO_FP_RND,
+
+      // Vector float/double to signed/unsigned integer.
+      FP_TO_SINT_RND, FP_TO_UINT_RND,
       // Save xmm argument registers to the stack, according to %al. An operator
       // is needed so that this can be expanded with control flow.
       VASTART_SAVE_XMM_REGS,
@@ -576,8 +602,11 @@ namespace llvm {
                                const X86Subtarget &STI);
 
     unsigned getJumpTableEncoding() const override;
+    bool useSoftFloat() const override;
 
-    MVT getScalarShiftAmountTy(EVT LHSTy) const override { return MVT::i8; }
+    MVT getScalarShiftAmountTy(const DataLayout &, EVT) const override {
+      return MVT::i8;
+    }
 
     const MCExpr *
     LowerCustomJumpTableEntry(const MachineJumpTableInfo *MJTI,
@@ -595,7 +624,8 @@ namespace llvm {
     /// function arguments in the caller parameter area. For X86, aggregates
     /// that contains are placed at 16-byte boundaries while the rest are at
     /// 4-byte boundaries.
-    unsigned getByValTypeAlignment(Type *Ty) const override;
+    unsigned getByValTypeAlignment(Type *Ty,
+                                   const DataLayout &DL) const override;
 
     /// Returns the target specific optimal type for load
     /// and store operations as a result of memset, memcpy, and memmove
@@ -620,9 +650,8 @@ namespace llvm {
     /// legal as the hook is used before type legalization.
     bool isSafeMemOpType(MVT VT) const override;
 
-    /// Returns true if the target allows
-    /// unaligned memory accesses. of the specified type. Returns whether it
-    /// is "fast" by reference in the second argument.
+    /// Returns true if the target allows unaligned memory accesses of the
+    /// specified type. Returns whether it is "fast" in the last argument.
     bool allowsMisalignedMemoryAccesses(EVT VT, unsigned AS, unsigned Align,
                                        bool *Fast) const override;
 
@@ -664,7 +693,8 @@ namespace llvm {
     bool isCheapToSpeculateCtlz() const override;
 
     /// Return the value type to use for ISD::SETCC.
-    EVT getSetCCResultType(LLVMContext &Context, EVT VT) const override;
+    EVT getSetCCResultType(const DataLayout &DL, LLVMContext &Context,
+                           EVT VT) const override;
 
     /// Determine which of the bits specified in Mask are known to be either
     /// zero or one and return them in the KnownZero/KnownOne bitsets.
@@ -686,8 +716,7 @@ namespace llvm {
 
     bool ExpandInlineAsm(CallInst *CI) const override;
 
-    ConstraintType
-      getConstraintType(const std::string &Constraint) const override;
+    ConstraintType getConstraintType(StringRef Constraint) const override;
 
     /// Examine constraint string and operand type and determine a weight value.
     /// The operand object must already have been set up with the operand type.
@@ -705,10 +734,17 @@ namespace llvm {
                                       std::vector<SDValue> &Ops,
                                       SelectionDAG &DAG) const override;
 
-    unsigned getInlineAsmMemConstraint(
-        const std::string &ConstraintCode) const override {
-      // FIXME: Map different constraints differently.
-      return InlineAsm::Constraint_m;
+    unsigned
+    getInlineAsmMemConstraint(StringRef ConstraintCode) const override {
+      if (ConstraintCode == "i")
+        return InlineAsm::Constraint_i;
+      else if (ConstraintCode == "o")
+        return InlineAsm::Constraint_o;
+      else if (ConstraintCode == "v")
+        return InlineAsm::Constraint_v;
+      else if (ConstraintCode == "X")
+        return InlineAsm::Constraint_X;
+      return TargetLowering::getInlineAsmMemConstraint(ConstraintCode);
     }
 
     /// Given a physical register constraint
@@ -717,12 +753,12 @@ namespace llvm {
     /// error, this returns a register number of 0.
     std::pair<unsigned, const TargetRegisterClass *>
     getRegForInlineAsmConstraint(const TargetRegisterInfo *TRI,
-                                 const std::string &Constraint,
-                                 MVT VT) const override;
+                                 StringRef Constraint, MVT VT) const override;
 
     /// Return true if the addressing mode represented
     /// by AM is legal for this target, for a load/store of the specified type.
-    bool isLegalAddressingMode(const AddrMode &AM, Type *Ty) const override;
+    bool isLegalAddressingMode(const DataLayout &DL, const AddrMode &AM,
+                               Type *Ty, unsigned AS) const override;
 
     /// Return true if the specified immediate is legal
     /// icmp immediate, that is the target has icmp instructions which can
@@ -741,7 +777,8 @@ namespace llvm {
     /// of the specified type.
     /// If the AM is supported, the return value must be >= 0.
     /// If the AM is not supported, it returns a negative value.
-    int getScalingFactorCost(const AddrMode &AM, Type *Ty) const override;
+    int getScalingFactorCost(const DataLayout &DL, const AddrMode &AM, Type *Ty,
+                             unsigned AS) const override;
 
     bool isVectorShiftByScalarCheap(Type *Ty) const override;
 
@@ -842,7 +879,8 @@ namespace llvm {
       return nullptr; // nothing to do, move along.
     }
 
-    unsigned getRegisterByName(const char* RegName, EVT VT) const override;
+    unsigned getRegisterByName(const char* RegName, EVT VT,
+                               SelectionDAG &DAG) const override;
 
     /// This method returns a target specific FastISel object,
     /// or null if the target does not support "fast" ISel.
@@ -874,9 +912,6 @@ namespace llvm {
     /// make the right decision when generating code for different targets.
     const X86Subtarget *Subtarget;
     const DataLayout *TD;
-    // @LOCALMOD - This is essentially a revert of r167104
-    /// X86StackPtr - X86 physical register used as stack ptr.
-    unsigned X86StackPtr;
 
     /// Select between SSE or x87 floating point ops.
     /// When SSE is available, use it for f32 operations.
@@ -978,6 +1013,8 @@ namespace llvm {
     SDValue LowerINIT_TRAMPOLINE(SDValue Op, SelectionDAG &DAG) const;
     SDValue LowerFLT_ROUNDS_(SDValue Op, SelectionDAG &DAG) const;
     SDValue LowerWin64_i128OP(SDValue Op, SelectionDAG &DAG) const;
+    SDValue LowerGC_TRANSITION_START(SDValue Op, SelectionDAG &DAG) const;
+    SDValue LowerGC_TRANSITION_END(SDValue Op, SelectionDAG &DAG) const;
 
     SDValue
       LowerFormalArguments(SDValue Chain,
@@ -1049,12 +1086,6 @@ namespace llvm {
     MachineBasicBlock *EmitLoweredSegAlloca(MachineInstr *MI,
                                             MachineBasicBlock *BB) const;
 
-    // @LOCALMOD-BEGIN
-    MachineBasicBlock *EmitLoweredThreadPointerFromGs(
-        MachineInstr *MI,
-        MachineBasicBlock *BB) const;
-    // @LOCALMOD-END
-
     MachineBasicBlock *EmitLoweredTLSCall(MachineInstr *MI,
                                           MachineBasicBlock *BB) const;
 
@@ -1093,7 +1124,7 @@ namespace llvm {
                              unsigned &RefinementSteps) const override;
 
     /// Reassociate floating point divisions into multiply by reciprocal.
-    bool combineRepeatedFPDivisors(unsigned NumUsers) const override;
+    unsigned combineRepeatedFPDivisors() const override;
   };
 
   namespace X86 {
diff --git a/lib/Target/X86/X86InstrAVX512.td b/lib/Target/X86/X86InstrAVX512.td
index 095916233c8..ef33993f6f6 100644
--- a/lib/Target/X86/X86InstrAVX512.td
+++ b/lib/Target/X86/X86InstrAVX512.td
@@ -180,21 +180,20 @@ multiclass AVX512_maskable_custom<bits<8> O, Format F,
                                   list<dag> Pattern,
                                   list<dag> MaskingPattern,
                                   list<dag> ZeroMaskingPattern,
-                                  string Round = "",
                                   string MaskingConstraint = "",
                                   InstrItinClass itin = NoItinerary,
                                   bit IsCommutable = 0> {
   let isCommutable = IsCommutable in
     def NAME: AVX512<O, F, Outs, Ins,
-                       OpcodeStr#"\t{"#AttSrcAsm#", $dst "#Round#"|"#
-                                     "$dst "#Round#", "#IntelSrcAsm#"}",
+                       OpcodeStr#"\t{"#AttSrcAsm#", $dst|"#
+                                     "$dst , "#IntelSrcAsm#"}",
                        Pattern, itin>;
 
   // Prefer over VMOV*rrk Pat<>
   let AddedComplexity = 20 in
     def NAME#k: AVX512<O, F, Outs, MaskingIns,
-                       OpcodeStr#"\t{"#AttSrcAsm#", $dst {${mask}}"#Round#"|"#
-                                     "$dst {${mask}}"#Round#", "#IntelSrcAsm#"}",
+                       OpcodeStr#"\t{"#AttSrcAsm#", $dst {${mask}}|"#
+                                     "$dst {${mask}}, "#IntelSrcAsm#"}",
                        MaskingPattern, itin>,
               EVEX_K {
       // In case of the 3src subclass this is overridden with a let.
@@ -202,8 +201,8 @@ multiclass AVX512_maskable_custom<bits<8> O, Format F,
   }
   let AddedComplexity = 30 in // Prefer over VMOV*rrkz Pat<>
     def NAME#kz: AVX512<O, F, Outs, ZeroMaskingIns,
-                       OpcodeStr#"\t{"#AttSrcAsm#", $dst {${mask}} {z}"#Round#"|"#
-                                     "$dst {${mask}} {z}"#Round#", "#IntelSrcAsm#"}",
+                       OpcodeStr#"\t{"#AttSrcAsm#", $dst {${mask}} {z}|"#
+                                     "$dst {${mask}} {z}, "#IntelSrcAsm#"}",
                        ZeroMaskingPattern,
                        itin>,
               EVEX_KZ;
@@ -217,7 +216,7 @@ multiclass AVX512_maskable_common<bits<8> O, Format F, X86VectorVTInfo _,
                                   string OpcodeStr,
                                   string AttSrcAsm, string IntelSrcAsm,
                                   dag RHS, dag MaskingRHS,
-                                  SDNode Select = vselect, string Round = "",
+                                  SDNode Select = vselect,
                                   string MaskingConstraint = "",
                                   InstrItinClass itin = NoItinerary,
                                   bit IsCommutable = 0> :
@@ -227,7 +226,7 @@ multiclass AVX512_maskable_common<bits<8> O, Format F, X86VectorVTInfo _,
                          [(set _.RC:$dst, MaskingRHS)],
                          [(set _.RC:$dst,
                                (Select _.KRCWM:$mask, RHS, _.ImmAllZerosV))],
-                         Round, MaskingConstraint, NoItinerary, IsCommutable>;
+                         MaskingConstraint, NoItinerary, IsCommutable>;
 
 // This multiclass generates the unconditional/non-masking, the masking and
 // the zero-masking variant of the vector instruction.  In the masking case, the
@@ -235,7 +234,7 @@ multiclass AVX512_maskable_common<bits<8> O, Format F, X86VectorVTInfo _,
 multiclass AVX512_maskable<bits<8> O, Format F, X86VectorVTInfo _,
                            dag Outs, dag Ins, string OpcodeStr,
                            string AttSrcAsm, string IntelSrcAsm,
-                           dag RHS, string Round = "",
+                           dag RHS,
                            InstrItinClass itin = NoItinerary,
                            bit IsCommutable = 0> :
    AVX512_maskable_common<O, F, _, Outs, Ins,
@@ -243,14 +242,14 @@ multiclass AVX512_maskable<bits<8> O, Format F, X86VectorVTInfo _,
                           !con((ins _.KRCWM:$mask), Ins),
                           OpcodeStr, AttSrcAsm, IntelSrcAsm, RHS,
                           (vselect _.KRCWM:$mask, RHS, _.RC:$src0), vselect,
-                          Round, "$src0 = $dst", itin, IsCommutable>;
+                          "$src0 = $dst", itin, IsCommutable>;
 
 // This multiclass generates the unconditional/non-masking, the masking and
 // the zero-masking variant of the scalar instruction.
 multiclass AVX512_maskable_scalar<bits<8> O, Format F, X86VectorVTInfo _,
                            dag Outs, dag Ins, string OpcodeStr,
                            string AttSrcAsm, string IntelSrcAsm,
-                           dag RHS, string Round = "",
+                           dag RHS,
                            InstrItinClass itin = NoItinerary,
                            bit IsCommutable = 0> :
    AVX512_maskable_common<O, F, _, Outs, Ins,
@@ -258,7 +257,7 @@ multiclass AVX512_maskable_scalar<bits<8> O, Format F, X86VectorVTInfo _,
                           !con((ins _.KRCWM:$mask), Ins),
                           OpcodeStr, AttSrcAsm, IntelSrcAsm, RHS,
                           (X86select _.KRCWM:$mask, RHS, _.RC:$src0), X86select,
-                          Round, "$src0 = $dst", itin, IsCommutable>;
+                          "$src0 = $dst", itin, IsCommutable>;
 
 // Similar to AVX512_maskable but in this case one of the source operands
 // ($src1) is already tied to $dst so we just use that for the preserved
@@ -275,6 +274,16 @@ multiclass AVX512_maskable_3src<bits<8> O, Format F, X86VectorVTInfo _,
                           OpcodeStr, AttSrcAsm, IntelSrcAsm, RHS,
                           (vselect _.KRCWM:$mask, RHS, _.RC:$src1)>;
 
+multiclass AVX512_maskable_3src_scalar<bits<8> O, Format F, X86VectorVTInfo _,
+                                     dag Outs, dag NonTiedIns, string OpcodeStr,
+                                     string AttSrcAsm, string IntelSrcAsm,
+                                     dag RHS> :
+   AVX512_maskable_common<O, F, _, Outs,
+                          !con((ins _.RC:$src1), NonTiedIns),
+                          !con((ins _.RC:$src1, _.KRCWM:$mask), NonTiedIns),
+                          !con((ins _.RC:$src1, _.KRCWM:$mask), NonTiedIns),
+                          OpcodeStr, AttSrcAsm, IntelSrcAsm, RHS,
+                          (X86select _.KRCWM:$mask, RHS, _.RC:$src1)>;
 
 multiclass AVX512_maskable_in_asm<bits<8> O, Format F, X86VectorVTInfo _,
                                   dag Outs, dag Ins,
@@ -284,9 +293,65 @@ multiclass AVX512_maskable_in_asm<bits<8> O, Format F, X86VectorVTInfo _,
    AVX512_maskable_custom<O, F, Outs, Ins,
                           !con((ins _.RC:$src0, _.KRCWM:$mask), Ins),
                           !con((ins _.KRCWM:$mask), Ins),
-                          OpcodeStr, AttSrcAsm, IntelSrcAsm, Pattern, [], [], "",
+                          OpcodeStr, AttSrcAsm, IntelSrcAsm, Pattern, [], [],
                           "$src0 = $dst">;
 
+
+// Instruction with mask that puts result in mask register,
+// like "compare" and "vptest"
+multiclass AVX512_maskable_custom_cmp<bits<8> O, Format F,
+                                  dag Outs,
+                                  dag Ins, dag MaskingIns,
+                                  string OpcodeStr,
+                                  string AttSrcAsm, string IntelSrcAsm,
+                                  list<dag> Pattern,
+                                  list<dag> MaskingPattern,
+                                  string Round = "",
+                                  InstrItinClass itin = NoItinerary> {
+    def NAME: AVX512<O, F, Outs, Ins,
+                       OpcodeStr#"\t{"#AttSrcAsm#", $dst "#Round#"|"#
+                                     "$dst "#Round#", "#IntelSrcAsm#"}",
+                       Pattern, itin>;
+
+    def NAME#k: AVX512<O, F, Outs, MaskingIns,
+                       OpcodeStr#"\t{"#Round#AttSrcAsm#", $dst {${mask}}|"#
+                                     "$dst {${mask}}, "#IntelSrcAsm#Round#"}",
+                       MaskingPattern, itin>, EVEX_K;
+}
+
+multiclass AVX512_maskable_common_cmp<bits<8> O, Format F, X86VectorVTInfo _,
+                                  dag Outs,
+                                  dag Ins, dag MaskingIns,
+                                  string OpcodeStr,
+                                  string AttSrcAsm, string IntelSrcAsm,
+                                  dag RHS, dag MaskingRHS,
+                                  string Round = "",
+                                  InstrItinClass itin = NoItinerary> :
+  AVX512_maskable_custom_cmp<O, F, Outs, Ins, MaskingIns, OpcodeStr,
+                         AttSrcAsm, IntelSrcAsm,
+                         [(set _.KRC:$dst, RHS)],
+                         [(set _.KRC:$dst, MaskingRHS)],
+                         Round, NoItinerary>;
+
+multiclass AVX512_maskable_cmp<bits<8> O, Format F, X86VectorVTInfo _,
+                           dag Outs, dag Ins, string OpcodeStr,
+                           string AttSrcAsm, string IntelSrcAsm,
+                           dag RHS, string Round = "",
+                           InstrItinClass itin = NoItinerary> :
+   AVX512_maskable_common_cmp<O, F, _, Outs, Ins,
+                          !con((ins _.KRCWM:$mask), Ins),
+                          OpcodeStr, AttSrcAsm, IntelSrcAsm, RHS,
+                          (and _.KRCWM:$mask, RHS),
+                          Round, itin>;
+
+multiclass AVX512_maskable_cmp_alt<bits<8> O, Format F, X86VectorVTInfo _,
+                           dag Outs, dag Ins, string OpcodeStr,
+                           string AttSrcAsm, string IntelSrcAsm> :
+   AVX512_maskable_custom_cmp<O, F, Outs,
+                             Ins, !con((ins _.KRCWM:$mask),Ins), OpcodeStr,
+                             AttSrcAsm, IntelSrcAsm,
+                             [],[],"", NoItinerary>;
+
 // Bitcasts between 512-bit vector types. Return the original type since
 // no instruction is needed for the conversion
 let Predicates = [HasAVX512] in {
@@ -754,12 +819,8 @@ def : Pat <(v8i64 (X86vzext VK8WM:$mask)),
 
 def : Pat<(v16i32 (X86VBroadcast (i32 GR32:$src))),
         (VPBROADCASTDrZr GR32:$src)>;
-def : Pat<(v16i32 (X86VBroadcastm VK16WM:$mask, (i32 GR32:$src))),
-        (VPBROADCASTDrZrkz VK16WM:$mask, GR32:$src)>;
 def : Pat<(v8i64 (X86VBroadcast (i64 GR64:$src))),
         (VPBROADCASTQrZr GR64:$src)>;
-def : Pat<(v8i64 (X86VBroadcastm VK8WM:$mask, (i64 GR64:$src))),
-        (VPBROADCASTQrZrkz VK8WM:$mask, GR64:$src)>;
 
 def : Pat<(v16i32 (int_x86_avx512_pbroadcastd_i32_512 (i32 GR32:$src))),
         (VPBROADCASTDrZr GR32:$src)>;
@@ -781,24 +842,33 @@ multiclass avx512_int_broadcast_rm<bits<8> opc, string OpcodeStr,
                   !strconcat(OpcodeStr, "\t{$src, $dst|$dst, $src}"),
                   [(set DstRC:$dst,
                     (OpVT (X86VBroadcast (SrcVT VR128X:$src))))]>, EVEX;
-  def krr : AVX5128I<opc, MRMSrcReg, (outs DstRC:$dst), (ins KRC:$mask,
+  def rrk : AVX5128I<opc, MRMSrcReg, (outs DstRC:$dst), (ins KRC:$mask,
+                                                         VR128X:$src),
+                    !strconcat(OpcodeStr,
+                    "\t{$src, ${dst} {${mask}} |${dst} {${mask}}, $src}"),
+                    []>, EVEX, EVEX_K;
+  def rrkz : AVX5128I<opc, MRMSrcReg, (outs DstRC:$dst), (ins KRC:$mask,
                                                          VR128X:$src),
                     !strconcat(OpcodeStr,
                     "\t{$src, ${dst} {${mask}} {z}|${dst} {${mask}} {z}, $src}"),
-                    [(set DstRC:$dst,
-                      (OpVT (X86VBroadcastm KRC:$mask, (SrcVT VR128X:$src))))]>,
-                    EVEX, EVEX_KZ;
+                    []>, EVEX, EVEX_KZ;
   let mayLoad = 1 in {
   def rm : AVX5128I<opc, MRMSrcMem, (outs DstRC:$dst), (ins x86memop:$src),
                   !strconcat(OpcodeStr, "\t{$src, $dst|$dst, $src}"),
                   [(set DstRC:$dst,
                     (OpVT (X86VBroadcast (ld_frag addr:$src))))]>, EVEX;
-  def krm : AVX5128I<opc, MRMSrcMem, (outs DstRC:$dst), (ins KRC:$mask,
+  def rmk : AVX5128I<opc, MRMSrcMem, (outs DstRC:$dst), (ins KRC:$mask,
+                                                         x86memop:$src),
+                  !strconcat(OpcodeStr,
+                      "\t{$src, ${dst} {${mask}}|${dst} {${mask}} , $src}"),
+                  []>, EVEX, EVEX_K;
+  def rmkz : AVX5128I<opc, MRMSrcMem, (outs DstRC:$dst), (ins KRC:$mask,
                                                          x86memop:$src),
                   !strconcat(OpcodeStr,
                       "\t{$src, ${dst} {${mask}} {z}|${dst} {${mask}} {z}, $src}"),
-                  [(set DstRC:$dst, (OpVT (X86VBroadcastm KRC:$mask,
-                                     (ld_frag addr:$src))))]>, EVEX, EVEX_KZ;
+                  [(set DstRC:$dst, (OpVT (vselect KRC:$mask,
+                             (X86VBroadcast (ld_frag addr:$src)), 
+                             (OpVT (bitconvert (v16i32 immAllZerosV))))))]>, EVEX, EVEX_KZ;
   }
 }
 
@@ -809,28 +879,71 @@ defm VPBROADCASTQZ  : avx512_int_broadcast_rm<0x59, "vpbroadcastq", i64mem,
                       loadi64, VR512, v8i64, v2i64, VK8WM>,  EVEX_V512, VEX_W,
                       EVEX_CD8<64, CD8VT1>;
 
-multiclass avx512_int_subvec_broadcast_rm<bits<8> opc, string OpcodeStr,
-                          X86MemOperand x86memop, PatFrag ld_frag,
-                          RegisterClass KRC> {
+multiclass avx512_subvec_broadcast_rm<bits<8> opc, string OpcodeStr,
+                          X86VectorVTInfo _Dst, X86VectorVTInfo _Src> {
   let mayLoad = 1 in {
-  def rm : AVX5128I<opc, MRMSrcMem, (outs VR512:$dst), (ins x86memop:$src),
+  def rm : AVX5128I<opc, MRMSrcMem, (outs _Dst.RC:$dst), (ins _Src.MemOp:$src),
                   !strconcat(OpcodeStr, "\t{$src, $dst|$dst, $src}"),
-                  []>, EVEX;
-  def krm : AVX5128I<opc, MRMSrcMem, (outs VR512:$dst), (ins KRC:$mask,
-                                                         x86memop:$src),
+                  [(set _Dst.RC:$dst, 
+                    (_Dst.VT (X86SubVBroadcast 
+                    (_Src.VT (bitconvert (_Src.LdFrag addr:$src))))))]>, EVEX;
+  def rmk : AVX5128I<opc, MRMSrcMem, (outs _Dst.RC:$dst), (ins _Dst.KRCWM:$mask,
+                                                         _Src.MemOp:$src),
                   !strconcat(OpcodeStr,
-                      "\t{$src, ${dst} {${mask}} {z}|${dst} {${mask}} {z}, $src}"),
+                      "\t{$src, ${dst} {${mask}}|${dst} {${mask}}, $src}"),
+                  []>, EVEX, EVEX_K;
+  def rmkz : AVX5128I<opc, MRMSrcMem, (outs _Dst.RC:$dst), (ins _Dst.KRCWM:$mask,
+                                                         _Src.MemOp:$src),
+                  !strconcat(OpcodeStr,
+                    "\t{$src, ${dst} {${mask}} {z}|${dst} {${mask}} {z}, $src}"),
                   []>, EVEX, EVEX_KZ;
   }
 }
 
-defm VBROADCASTI32X4 : avx512_int_subvec_broadcast_rm<0x5a, "vbroadcasti32x4",
-                       i128mem, loadv2i64, VK16WM>,
+defm VBROADCASTI32X4 : avx512_subvec_broadcast_rm<0x5a, "vbroadcasti32x4",
+                       v16i32_info, v4i32x_info>,
                        EVEX_V512, EVEX_CD8<32, CD8VT4>;
-defm VBROADCASTI64X4 : avx512_int_subvec_broadcast_rm<0x5b, "vbroadcasti64x4",
-                       i256mem, loadv4i64, VK16WM>, VEX_W,
+defm VBROADCASTF32X4 : avx512_subvec_broadcast_rm<0x1a, "vbroadcastf32x4",
+                       v16f32_info, v4f32x_info>,
+                       EVEX_V512, EVEX_CD8<32, CD8VT4>;
+defm VBROADCASTI64X4 : avx512_subvec_broadcast_rm<0x5b, "vbroadcasti64x4",
+                       v8i64_info, v4i64x_info>, VEX_W,
+                       EVEX_V512, EVEX_CD8<64, CD8VT4>;
+defm VBROADCASTF64X4 : avx512_subvec_broadcast_rm<0x1b, "vbroadcastf64x4",
+                       v8f64_info, v4f64x_info>, VEX_W,
                        EVEX_V512, EVEX_CD8<64, CD8VT4>;
 
+let Predicates = [HasVLX] in {
+defm VBROADCASTI32X4Z256 : avx512_subvec_broadcast_rm<0x5a, "vbroadcasti32x4",
+                           v8i32x_info, v4i32x_info>,
+                           EVEX_V256, EVEX_CD8<32, CD8VT4>;
+defm VBROADCASTF32X4Z256 : avx512_subvec_broadcast_rm<0x1a, "vbroadcastf32x4",
+                           v8f32x_info, v4f32x_info>,
+                           EVEX_V256, EVEX_CD8<32, CD8VT4>;
+}
+let Predicates = [HasVLX, HasDQI] in {
+defm VBROADCASTI64X2Z128 : avx512_subvec_broadcast_rm<0x5a, "vbroadcasti64x2",
+                           v4i64x_info, v2i64x_info>, VEX_W,
+                           EVEX_V256, EVEX_CD8<64, CD8VT2>;
+defm VBROADCASTF64X2Z128 : avx512_subvec_broadcast_rm<0x1a, "vbroadcastf64x2",
+                           v4f64x_info, v2f64x_info>, VEX_W,
+                           EVEX_V256, EVEX_CD8<64, CD8VT2>;
+}
+let Predicates = [HasDQI] in {
+defm VBROADCASTI64X2 : avx512_subvec_broadcast_rm<0x5a, "vbroadcasti64x2",
+                       v8i64_info, v2i64x_info>, VEX_W,
+                       EVEX_V512, EVEX_CD8<64, CD8VT2>;
+defm VBROADCASTI32X8 : avx512_subvec_broadcast_rm<0x5b, "vbroadcasti32x8",
+                       v16i32_info, v8i32x_info>,
+                       EVEX_V512, EVEX_CD8<32, CD8VT8>;
+defm VBROADCASTF64X2 : avx512_subvec_broadcast_rm<0x1a, "vbroadcastf64x2",
+                       v8f64_info, v2f64x_info>, VEX_W,
+                       EVEX_V512, EVEX_CD8<64, CD8VT2>;
+defm VBROADCASTF32X8 : avx512_subvec_broadcast_rm<0x1b, "vbroadcastf32x8",
+                       v16f32_info, v8f32x_info>,
+                       EVEX_V512, EVEX_CD8<32, CD8VT8>;
+}
+
 def : Pat<(v16i32 (int_x86_avx512_pbroadcastd_512 (v4i32 VR128X:$src))),
           (VPBROADCASTDZrr VR128X:$src)>;
 def : Pat<(v8i64 (int_x86_avx512_pbroadcastq_512 (v2i64 VR128X:$src))),
@@ -838,13 +951,23 @@ def : Pat<(v8i64 (int_x86_avx512_pbroadcastq_512 (v2i64 VR128X:$src))),
 
 def : Pat<(v16f32 (X86VBroadcast (v16f32 VR512:$src))),
           (VBROADCASTSSZr (EXTRACT_SUBREG (v16f32 VR512:$src), sub_xmm))>;
+def : Pat<(v16f32 (X86VBroadcast (v8f32 VR256X:$src))),
+          (VBROADCASTSSZr (EXTRACT_SUBREG (v8f32 VR256X:$src), sub_xmm))>;
+
 def : Pat<(v8f64 (X86VBroadcast (v8f64 VR512:$src))),
           (VBROADCASTSDZr (EXTRACT_SUBREG (v8f64 VR512:$src), sub_xmm))>;
+def : Pat<(v8f64 (X86VBroadcast (v4f64 VR256X:$src))),
+          (VBROADCASTSDZr (EXTRACT_SUBREG (v4f64 VR256X:$src), sub_xmm))>;
 
 def : Pat<(v16i32 (X86VBroadcast (v16i32 VR512:$src))),
           (VPBROADCASTDZrr (EXTRACT_SUBREG (v16i32 VR512:$src), sub_xmm))>;
+def : Pat<(v16i32 (X86VBroadcast (v8i32 VR256X:$src))),
+          (VPBROADCASTDZrr (EXTRACT_SUBREG (v8i32 VR256X:$src), sub_xmm))>;
+
 def : Pat<(v8i64 (X86VBroadcast (v8i64 VR512:$src))),
           (VPBROADCASTQZrr (EXTRACT_SUBREG (v8i64 VR512:$src), sub_xmm))>;
+def : Pat<(v8i64 (X86VBroadcast (v4i64 VR256X:$src))),
+          (VPBROADCASTQZrr (EXTRACT_SUBREG (v4i64 VR256X:$src), sub_xmm))>;
 
 def : Pat<(v16f32 (int_x86_avx512_vbroadcast_ss_ps_512 (v4f32 VR128X:$src))),
           (VBROADCASTSSZr VR128X:$src)>;
@@ -859,12 +982,6 @@ def : Pat<(v8f64 (X86VBroadcast FR64X:$src)),
           (VBROADCASTSDZr (COPY_TO_REGCLASS FR64X:$src, VR128X))>;
 
 
-let Predicates = [HasAVX512] in {
-def : Pat<(v8i32 (X86VBroadcastm (v8i1 VK8WM:$mask), (loadi32 addr:$src))),
-           (EXTRACT_SUBREG
-              (v16i32 (VPBROADCASTDZkrm (COPY_TO_REGCLASS VK8WM:$mask, VK16WM),
-                       addr:$src)), sub_ymm)>;
-}
 //===----------------------------------------------------------------------===//
 // AVX-512 BROADCAST MASK TO VECTOR REGISTER
 //---
@@ -940,12 +1057,6 @@ multiclass avx512_permil<bits<8> OpcImm, bits<8> OpcVar, X86VectorVTInfo _,
              EVEX_4V;
   }
 }
-
-defm VPERMQZ :    avx512_perm_imm<0x00, "vpermq", X86VPermi, v8i64_info>,
-                  EVEX_V512, VEX_W;
-defm VPERMPDZ :   avx512_perm_imm<0x01, "vpermpd", X86VPermi, v8f64_info>,
-                  EVEX_V512, VEX_W;
-
 defm VPERMILPSZ : avx512_permil<0x04, 0x0C, v16f32_info, v16i32_info>,
                   EVEX_V512;
 defm VPERMILPDZ : avx512_permil<0x05, 0x0D, v8f64_info, v8i64_info>,
@@ -956,150 +1067,88 @@ def : Pat<(v16i32 (X86VPermilpi VR512:$src1, (i8 imm:$imm))),
 def : Pat<(v8i64 (X86VPermilpi VR512:$src1, (i8 imm:$imm))),
           (VPERMILPDZri VR512:$src1, imm:$imm)>;
 
-// -- VPERM - register form --
-multiclass avx512_perm<bits<8> opc, string OpcodeStr, RegisterClass RC,
-                     PatFrag mem_frag, X86MemOperand x86memop, ValueType OpVT> {
-
-  def rr : AVX5128I<opc, MRMSrcReg, (outs RC:$dst),
-                   (ins RC:$src1, RC:$src2),
-                   !strconcat(OpcodeStr,
-                       "\t{$src2, $src1, $dst|$dst, $src1, $src2}"),
-                   [(set RC:$dst,
-                     (OpVT (X86VPermv RC:$src1, RC:$src2)))]>, EVEX_4V;
-
-  def rm : AVX5128I<opc, MRMSrcMem, (outs RC:$dst),
-                   (ins RC:$src1, x86memop:$src2),
-                   !strconcat(OpcodeStr,
-                       "\t{$src2, $src1, $dst|$dst, $src1, $src2}"),
-                   [(set RC:$dst,
-                     (OpVT (X86VPermv RC:$src1, (mem_frag addr:$src2))))]>,
-                     EVEX_4V;
-}
-
-defm VPERMDZ   : avx512_perm<0x36, "vpermd",  VR512,  loadv16i32, i512mem,
-                           v16i32>, EVEX_V512, EVEX_CD8<32, CD8VF>;
-defm VPERMQZ   : avx512_perm<0x36, "vpermq",  VR512,  loadv8i64,  i512mem,
-                           v8i64>,  EVEX_V512, VEX_W, EVEX_CD8<64, CD8VF>;
-let ExeDomain = SSEPackedSingle in
-defm VPERMPSZ  : avx512_perm<0x16, "vpermps", VR512,  loadv16f32, f512mem,
-                           v16f32>, EVEX_V512, EVEX_CD8<32, CD8VF>;
-let ExeDomain = SSEPackedDouble in
-defm VPERMPDZ  : avx512_perm<0x16, "vpermpd", VR512,  loadv8f64, f512mem,
-                           v8f64>, EVEX_V512, VEX_W, EVEX_CD8<64, CD8VF>;
-
 // -- VPERM2I - 3 source operands form --
-multiclass avx512_perm_3src<bits<8> opc, string OpcodeStr, RegisterClass RC,
-                          PatFrag mem_frag, X86MemOperand x86memop,
-                          SDNode OpNode, ValueType OpVT, RegisterClass KRC> {
+multiclass avx512_perm_3src<bits<8> opc, string OpcodeStr,
+                            SDNode OpNode, X86VectorVTInfo _> {
 let Constraints = "$src1 = $dst" in {
-  def rr : AVX5128I<opc, MRMSrcReg, (outs RC:$dst),
-                   (ins RC:$src1, RC:$src2, RC:$src3),
-                   !strconcat(OpcodeStr,
-                       "\t{$src3, $src2, $dst|$dst, $src2, $src3}"),
-                   [(set RC:$dst,
-                     (OpVT (OpNode RC:$src1, RC:$src2, RC:$src3)))]>,
-                    EVEX_4V;
-
-  def rrk : AVX5128I<opc, MRMSrcReg, (outs RC:$dst),
-                   (ins RC:$src1, KRC:$mask, RC:$src2, RC:$src3),
-                   !strconcat(OpcodeStr,
-                       "\t{$src3, $src2, $dst {${mask}}|"
-                       "$dst {${mask}}, $src2, $src3}"),
-                   [(set RC:$dst, (OpVT (vselect KRC:$mask,
-                                           (OpNode RC:$src1, RC:$src2,
-                                              RC:$src3),
-                                           RC:$src1)))]>,
-                    EVEX_4V, EVEX_K;
-
-  let AddedComplexity = 30 in // Prefer over VMOV*rrkz Pat<>
-    def rrkz : AVX5128I<opc, MRMSrcReg, (outs RC:$dst),
-                   (ins RC:$src1, KRC:$mask, RC:$src2, RC:$src3),
-                   !strconcat(OpcodeStr,
-                       "\t{$src3, $src2, $dst {${mask}} {z} |",
-                       "$dst {${mask}} {z}, $src2, $src3}"),
-                   [(set RC:$dst, (OpVT (vselect KRC:$mask,
-                                           (OpNode RC:$src1, RC:$src2,
-                                              RC:$src3),
-                                           (OpVT (bitconvert
-                                              (v16i32 immAllZerosV))))))]>,
-                    EVEX_4V, EVEX_KZ;
+  defm rr: AVX512_maskable_3src<opc, MRMSrcReg, _, (outs _.RC:$dst),
+          (ins _.RC:$src2, _.RC:$src3),
+          OpcodeStr, "$src3, $src2", "$src2, $src3",
+          (_.VT (OpNode _.RC:$src1, _.RC:$src2, _.RC:$src3))>, EVEX_4V,
+         AVX5128IBase;
 
-  def rm : AVX5128I<opc, MRMSrcMem, (outs RC:$dst),
-                   (ins RC:$src1, RC:$src2, x86memop:$src3),
-                   !strconcat(OpcodeStr,
-                    "\t{$src3, $src2, $dst|$dst, $src2, $src3}"),
-                   [(set RC:$dst,
-                     (OpVT (OpNode RC:$src1, RC:$src2,
-                      (mem_frag addr:$src3))))]>, EVEX_4V;
+  let mayLoad = 1 in
+  defm rm: AVX512_maskable_3src<opc, MRMSrcMem, _, (outs _.RC:$dst),
+            (ins _.RC:$src2, _.MemOp:$src3),
+            OpcodeStr, "$src3, $src2", "$src2, $src3",
+            (_.VT (OpNode _.RC:$src1, _.RC:$src2,
+                   (_.VT (bitconvert (_.LdFrag addr:$src3)))))>,
+            EVEX_4V, AVX5128IBase;
+  }
+}
+multiclass avx512_perm_3src_mb<bits<8> opc, string OpcodeStr,
+                               SDNode OpNode, X86VectorVTInfo _> {
+  let mayLoad = 1, Constraints = "$src1 = $dst" in
+  defm rmb: AVX512_maskable_3src<opc, MRMSrcMem, _, (outs _.RC:$dst),
+              (ins _.RC:$src2, _.ScalarMemOp:$src3),
+              OpcodeStr,   !strconcat("${src3}", _.BroadcastStr,", $src2"),
+              !strconcat("$src2, ${src3}", _.BroadcastStr ),
+              (_.VT (OpNode _.RC:$src1,
+               _.RC:$src2,(_.VT (X86VBroadcast (_.ScalarLdFrag addr:$src3)))))>, 
+              AVX5128IBase, EVEX_4V, EVEX_B;
+}
 
-  def rmk : AVX5128I<opc, MRMSrcMem, (outs RC:$dst),
-                   (ins RC:$src1, KRC:$mask, RC:$src2, x86memop:$src3),
-                   !strconcat(OpcodeStr,
-                    "\t{$src3, $src2, $dst {${mask}}|"
-                    "$dst {${mask}}, $src2, $src3}"),
-                   [(set RC:$dst,
-                       (OpVT (vselect KRC:$mask,
-                                      (OpNode RC:$src1, RC:$src2,
-                                         (mem_frag addr:$src3)),
-                                      RC:$src1)))]>,
-                    EVEX_4V, EVEX_K;
-
-  let AddedComplexity = 10 in // Prefer over the rrkz variant
-    def rmkz : AVX5128I<opc, MRMSrcMem, (outs RC:$dst),
-                   (ins RC:$src1, KRC:$mask, RC:$src2, x86memop:$src3),
-                   !strconcat(OpcodeStr,
-                    "\t{$src3, $src2, $dst {${mask}} {z}|"
-                    "$dst {${mask}} {z}, $src2, $src3}"),
-                   [(set RC:$dst,
-                     (OpVT (vselect KRC:$mask,
-                                    (OpNode RC:$src1, RC:$src2,
-                                            (mem_frag addr:$src3)),
-                                    (OpVT (bitconvert
-                                       (v16i32 immAllZerosV))))))]>,
-                    EVEX_4V, EVEX_KZ;
-  }
-}
-defm VPERMI2D  : avx512_perm_3src<0x76, "vpermi2d",  VR512, loadv16i32,
-                                  i512mem, X86VPermiv3, v16i32, VK16WM>,
-                 EVEX_V512, EVEX_CD8<32, CD8VF>;
-defm VPERMI2Q  : avx512_perm_3src<0x76, "vpermi2q",  VR512, loadv8i64,
-                                  i512mem, X86VPermiv3, v8i64, VK8WM>,
-                 EVEX_V512, VEX_W, EVEX_CD8<64, CD8VF>;
-defm VPERMI2PS : avx512_perm_3src<0x77, "vpermi2ps",  VR512, loadv16f32,
-                                  i512mem, X86VPermiv3, v16f32, VK16WM>,
-                 EVEX_V512, EVEX_CD8<32, CD8VF>;
-defm VPERMI2PD : avx512_perm_3src<0x77, "vpermi2pd",  VR512, loadv8f64,
-                                  i512mem, X86VPermiv3, v8f64, VK8WM>,
-                  EVEX_V512, VEX_W, EVEX_CD8<64, CD8VF>;
-
-multiclass avx512_perm_table_3src<bits<8> opc, string Suffix, RegisterClass RC,
-                          PatFrag mem_frag, X86MemOperand x86memop,
-                          SDNode OpNode, ValueType OpVT, RegisterClass KRC,
-                          ValueType MaskVT, RegisterClass MRC> :
-        avx512_perm_3src<opc, "vpermt2"##Suffix, RC, mem_frag, x86memop, OpNode,
-                         OpVT, KRC> {
-  def : Pat<(OpVT (!cast<Intrinsic>("int_x86_avx512_mask_vpermt_"##Suffix##"_512")
-                     VR512:$idx, VR512:$src1, VR512:$src2, -1)),
-            (!cast<Instruction>(NAME#rr) VR512:$src1, VR512:$idx, VR512:$src2)>;
-
-  def : Pat<(OpVT (!cast<Intrinsic>("int_x86_avx512_mask_vpermt_"##Suffix##"_512")
-                     VR512:$idx, VR512:$src1, VR512:$src2, MRC:$mask)),
-            (!cast<Instruction>(NAME#rrk) VR512:$src1,
-              (MaskVT (COPY_TO_REGCLASS MRC:$mask, KRC)), VR512:$idx, VR512:$src2)>;
-}
-
-defm VPERMT2D  : avx512_perm_table_3src<0x7E, "d",  VR512, loadv16i32, i512mem,
-                               X86VPermv3, v16i32, VK16WM, v16i1, GR16>,
-                 EVEX_V512, EVEX_CD8<32, CD8VF>;
-defm VPERMT2Q  : avx512_perm_table_3src<0x7E, "q",  VR512, loadv8i64, i512mem,
-                               X86VPermv3, v8i64, VK8WM, v8i1, GR8>,
-                 EVEX_V512, VEX_W, EVEX_CD8<64, CD8VF>;
-defm VPERMT2PS : avx512_perm_table_3src<0x7F, "ps",  VR512, loadv16f32, i512mem,
-                               X86VPermv3, v16f32, VK16WM, v16i1, GR16>,
-                 EVEX_V512, EVEX_CD8<32, CD8VF>;
-defm VPERMT2PD : avx512_perm_table_3src<0x7F, "pd",  VR512, loadv8f64, i512mem,
-                               X86VPermv3, v8f64, VK8WM, v8i1, GR8>,
-                 EVEX_V512, VEX_W, EVEX_CD8<64, CD8VF>;
+multiclass avx512_perm_3src_sizes<bits<8> opc, string OpcodeStr,
+                                  SDNode OpNode, AVX512VLVectorVTInfo VTInfo> {
+  let Predicates = [HasAVX512] in
+  defm NAME: avx512_perm_3src<opc, OpcodeStr, OpNode, VTInfo.info512>, 
+            avx512_perm_3src_mb<opc, OpcodeStr, OpNode, VTInfo.info512>, EVEX_V512;
+  let Predicates = [HasVLX] in {
+  defm NAME#128: avx512_perm_3src<opc, OpcodeStr, OpNode, VTInfo.info128>, 
+                 avx512_perm_3src_mb<opc, OpcodeStr, OpNode, VTInfo.info128>,
+                 EVEX_V128;
+  defm NAME#256: avx512_perm_3src<opc, OpcodeStr, OpNode, VTInfo.info256>, 
+                 avx512_perm_3src_mb<opc, OpcodeStr, OpNode, VTInfo.info256>,
+                 EVEX_V256;
+  }
+}
+multiclass avx512_perm_3src_sizes_w<bits<8> opc, string OpcodeStr, 
+                                   SDNode OpNode, AVX512VLVectorVTInfo VTInfo> {
+  let Predicates = [HasBWI] in
+  defm NAME: avx512_perm_3src<opc, OpcodeStr, OpNode, VTInfo.info512>, 
+             avx512_perm_3src_mb<opc, OpcodeStr, OpNode, VTInfo.info512>,
+             EVEX_V512;
+  let Predicates = [HasBWI, HasVLX] in {
+  defm NAME#128: avx512_perm_3src<opc, OpcodeStr, OpNode, VTInfo.info128>, 
+                 avx512_perm_3src_mb<opc, OpcodeStr, OpNode, VTInfo.info128>,
+                 EVEX_V128;
+  defm NAME#256: avx512_perm_3src<opc, OpcodeStr, OpNode, VTInfo.info256>, 
+                 avx512_perm_3src_mb<opc, OpcodeStr, OpNode, VTInfo.info256>,
+                 EVEX_V256;
+  }
+}
+defm VPERMI2D  : avx512_perm_3src_sizes<0x76, "vpermi2d", X86VPermiv3,
+                                  avx512vl_i32_info>, EVEX_CD8<32, CD8VF>;
+defm VPERMI2Q  : avx512_perm_3src_sizes<0x76, "vpermi2q", X86VPermiv3,
+                                  avx512vl_i64_info>, VEX_W, EVEX_CD8<64, CD8VF>;
+defm VPERMI2PS : avx512_perm_3src_sizes<0x77, "vpermi2ps", X86VPermiv3,
+                                  avx512vl_f32_info>, EVEX_CD8<32, CD8VF>;
+defm VPERMI2PD : avx512_perm_3src_sizes<0x77, "vpermi2pd", X86VPermiv3,
+                                  avx512vl_f64_info>, VEX_W, EVEX_CD8<64, CD8VF>;
+
+defm VPERMT2D  : avx512_perm_3src_sizes<0x7E, "vpermt2d", X86VPermv3,
+                                  avx512vl_i32_info>, EVEX_CD8<32, CD8VF>;
+defm VPERMT2Q  : avx512_perm_3src_sizes<0x7E, "vpermt2q", X86VPermv3,
+                                  avx512vl_i64_info>, VEX_W, EVEX_CD8<64, CD8VF>;
+defm VPERMT2PS : avx512_perm_3src_sizes<0x7F, "vpermt2ps", X86VPermv3,
+                                  avx512vl_f32_info>, EVEX_CD8<32, CD8VF>;
+defm VPERMT2PD : avx512_perm_3src_sizes<0x7F, "vpermt2pd", X86VPermv3,
+                                  avx512vl_f64_info>, VEX_W, EVEX_CD8<64, CD8VF>;
+
+defm VPERMT2W  : avx512_perm_3src_sizes_w<0x7D, "vpermt2w", X86VPermv3,
+                                  avx512vl_i16_info>, VEX_W, EVEX_CD8<16, CD8VF>;
+defm VPERMI2W  : avx512_perm_3src_sizes_w<0x75, "vpermi2w", X86VPermiv3,
+                                  avx512vl_i16_info>, VEX_W, EVEX_CD8<16, CD8VF>;
 
 //===----------------------------------------------------------------------===//
 // AVX-512 - BLEND using mask
@@ -1543,53 +1592,97 @@ defm VPCMPQ : avx512_icmp_cc_rmb_vl<0x1F, "q", X86cmpm, avx512vl_i64_info,
 defm VPCMPUQ : avx512_icmp_cc_rmb_vl<0x1E, "uq", X86cmpmu, avx512vl_i64_info,
                                      HasAVX512>, VEX_W, EVEX_CD8<64, CD8VF>;
 
-// avx512_cmp_packed - compare packed instructions
-multiclass avx512_cmp_packed<RegisterClass KRC, RegisterClass RC,
-                           X86MemOperand x86memop, ValueType vt,
-                           string suffix, Domain d> {
-  def rri : AVX512PIi8<0xC2, MRMSrcReg,
-             (outs KRC:$dst), (ins RC:$src1, RC:$src2, AVXCC:$cc),
-             !strconcat("vcmp${cc}", suffix,
-                        "\t{$src2, $src1, $dst|$dst, $src1, $src2}"),
-             [(set KRC:$dst, (X86cmpm (vt RC:$src1), (vt RC:$src2), imm:$cc))], d>;
-  let hasSideEffects = 0 in
-  def rrib: AVX512PIi8<0xC2, MRMSrcReg,
-             (outs KRC:$dst), (ins RC:$src1, RC:$src2, AVXCC:$cc),
-     !strconcat("vcmp${cc}", suffix,
-                "\t{{sae}, $src2, $src1, $dst|$dst, $src1, $src2, {sae}}"),
-                [], d>, EVEX_B;
-  def rmi : AVX512PIi8<0xC2, MRMSrcMem,
-             (outs KRC:$dst), (ins RC:$src1, x86memop:$src2, AVXCC:$cc),
-              !strconcat("vcmp${cc}", suffix,
-                         "\t{$src2, $src1, $dst|$dst, $src1, $src2, $cc}"),
-             [(set KRC:$dst,
-              (X86cmpm (vt RC:$src1), (load addr:$src2), imm:$cc))], d>;
+multiclass avx512_vcmp_common<X86VectorVTInfo _> {
 
+  defm  rri  : AVX512_maskable_cmp<0xC2, MRMSrcReg, _,
+                   (outs _.KRC:$dst), (ins _.RC:$src1, _.RC:$src2,AVXCC:$cc),
+                   "vcmp${cc}"#_.Suffix,
+                   "$src2, $src1", "$src1, $src2",
+                   (X86cmpm (_.VT _.RC:$src1),
+                         (_.VT _.RC:$src2),
+                           imm:$cc)>;
+
+  let mayLoad = 1 in {
+    defm  rmi  : AVX512_maskable_cmp<0xC2, MRMSrcMem, _,
+                  (outs _.KRC:$dst),(ins _.RC:$src1, _.MemOp:$src2, AVXCC:$cc),
+                  "vcmp${cc}"#_.Suffix,
+                  "$src2, $src1", "$src1, $src2",
+                  (X86cmpm (_.VT _.RC:$src1),
+                          (_.VT (bitconvert (_.LdFrag addr:$src2))),
+                          imm:$cc)>;
+
+    defm  rmbi : AVX512_maskable_cmp<0xC2, MRMSrcMem, _,
+                  (outs _.KRC:$dst),
+                  (ins _.RC:$src1, _.ScalarMemOp:$src2, AVXCC:$cc),
+                  "vcmp${cc}"#_.Suffix,
+                  "${src2}"##_.BroadcastStr##", $src1",
+                  "$src1, ${src2}"##_.BroadcastStr,
+                  (X86cmpm (_.VT _.RC:$src1),
+                          (_.VT (X86VBroadcast(_.ScalarLdFrag addr:$src2))),
+                          imm:$cc)>,EVEX_B;
+  }
   // Accept explicit immediate argument form instead of comparison code.
   let isAsmParserOnly = 1, hasSideEffects = 0 in {
-    def rri_alt : AVX512PIi8<0xC2, MRMSrcReg,
-               (outs KRC:$dst), (ins RC:$src1, RC:$src2, u8imm:$cc),
-              !strconcat("vcmp", suffix,
-                        "\t{$cc, $src2, $src1, $dst|$dst, $src1, $src2, $cc}"), [], d>;
-    def rrib_alt: AVX512PIi8<0xC2, MRMSrcReg,
-               (outs KRC:$dst), (ins RC:$src1, RC:$src2, u8imm:$cc),
-       !strconcat("vcmp", suffix,
-                  "\t{{sae}, $cc, $src2, $src1, $dst|$dst, $src1, $src2, $cc, {sae}}"),
-                  [], d>, EVEX_B;
-    let mayLoad = 1 in
-    def rmi_alt : AVX512PIi8<0xC2, MRMSrcMem,
-               (outs KRC:$dst), (ins RC:$src1, x86memop:$src2, u8imm:$cc),
-              !strconcat("vcmp", suffix,
-                        "\t{$cc, $src2, $src1, $dst|$dst, $src1, $src2, $cc}"), [], d>;
+    defm  rri_alt : AVX512_maskable_cmp_alt<0xC2, MRMSrcReg, _,
+                         (outs _.KRC:$dst),
+                         (ins _.RC:$src1, _.RC:$src2, u8imm:$cc),
+                         "vcmp"#_.Suffix,
+                         "$cc, $src2, $src1", "$src1, $src2, $cc">;
+
+    let mayLoad = 1 in {
+      defm rmi_alt : AVX512_maskable_cmp_alt<0xC2, MRMSrcMem, _,
+                             (outs _.KRC:$dst),
+                             (ins _.RC:$src1, _.MemOp:$src2, u8imm:$cc),
+                             "vcmp"#_.Suffix,
+                             "$cc, $src2, $src1", "$src1, $src2, $cc">;
+
+      defm  rmbi_alt : AVX512_maskable_cmp_alt<0xC2, MRMSrcMem, _,
+                         (outs _.KRC:$dst),
+                         (ins _.RC:$src1, _.ScalarMemOp:$src2, u8imm:$cc),
+                         "vcmp"#_.Suffix,
+                         "$cc, ${src2}"##_.BroadcastStr##", $src1",
+                         "$src1, ${src2}"##_.BroadcastStr##", $cc">,EVEX_B;
+    }
+ }
+}
+
+multiclass avx512_vcmp_sae<X86VectorVTInfo _> {
+  // comparison code form (VCMP[EQ/LT/LE/...]
+  defm  rrib  : AVX512_maskable_cmp<0xC2, MRMSrcReg, _,
+                     (outs _.KRC:$dst),(ins _.RC:$src1, _.RC:$src2, AVXCC:$cc),
+                     "vcmp${cc}"#_.Suffix,
+                     "{sae}, $src2, $src1", "$src1, $src2,{sae}",
+                     (X86cmpmRnd (_.VT _.RC:$src1),
+                                    (_.VT _.RC:$src2),
+                                    imm:$cc,
+                                (i32 FROUND_NO_EXC))>, EVEX_B;
+
+  let isAsmParserOnly = 1, hasSideEffects = 0 in {
+    defm  rrib_alt  : AVX512_maskable_cmp_alt<0xC2, MRMSrcReg, _,
+                         (outs _.KRC:$dst),
+                         (ins _.RC:$src1, _.RC:$src2, u8imm:$cc),
+                         "vcmp"#_.Suffix,
+                         "$cc,{sae}, $src2, $src1",
+                         "$src1, $src2,{sae}, $cc">, EVEX_B;
+   }
+}
+
+multiclass avx512_vcmp<AVX512VLVectorVTInfo _> {
+  let Predicates = [HasAVX512] in {
+    defm Z    : avx512_vcmp_common<_.info512>,
+                avx512_vcmp_sae<_.info512>, EVEX_V512;
+
+  }
+  let Predicates = [HasAVX512,HasVLX] in {
+   defm Z128 : avx512_vcmp_common<_.info128>, EVEX_V128;
+   defm Z256 : avx512_vcmp_common<_.info256>, EVEX_V256;
   }
 }
 
-defm VCMPPSZ : avx512_cmp_packed<VK16, VR512, f512mem, v16f32,
-               "ps", SSEPackedSingle>, PS, EVEX_4V, EVEX_V512,
-               EVEX_CD8<32, CD8VF>;
-defm VCMPPDZ : avx512_cmp_packed<VK8, VR512, f512mem, v8f64,
-               "pd", SSEPackedDouble>, PD, EVEX_4V, VEX_W, EVEX_V512,
-               EVEX_CD8<64, CD8VF>;
+defm VCMPPD : avx512_vcmp<avx512vl_f64_info>,
+                          AVX512PDIi8Base, EVEX_4V, EVEX_CD8<64, CD8VF>, VEX_W;
+defm VCMPPS : avx512_vcmp<avx512vl_f32_info>,
+                          AVX512PSIi8Base, EVEX_4V, EVEX_CD8<32, CD8VF>;
 
 def : Pat<(v8i1 (X86cmpm (v8f32 VR256X:$src1), (v8f32 VR256X:$src2), imm:$cc)),
           (COPY_TO_REGCLASS (VCMPPSZrri
@@ -1607,30 +1700,7 @@ def : Pat<(v8i1 (X86cmpmu (v8i32 VR256X:$src1), (v8i32 VR256X:$src2), imm:$cc)),
             (v16i32 (SUBREG_TO_REG (i32 0), VR256X:$src2, sub_ymm)),
             imm:$cc), VK8)>;
 
-def : Pat<(i16 (int_x86_avx512_mask_cmp_ps_512 (v16f32 VR512:$src1),
-                (v16f32 VR512:$src2), i8immZExt5:$cc, (i16 -1),
-                 FROUND_NO_EXC)),
-          (COPY_TO_REGCLASS (VCMPPSZrrib VR512:$src1, VR512:$src2,
-                             (I8Imm imm:$cc)), GR16)>;
-
-def : Pat<(i8 (int_x86_avx512_mask_cmp_pd_512 (v8f64 VR512:$src1),
-                (v8f64 VR512:$src2), i8immZExt5:$cc, (i8 -1),
-                 FROUND_NO_EXC)),
-          (COPY_TO_REGCLASS (VCMPPDZrrib VR512:$src1, VR512:$src2,
-                             (I8Imm imm:$cc)), GR8)>;
-
-def : Pat<(i16 (int_x86_avx512_mask_cmp_ps_512 (v16f32 VR512:$src1),
-                (v16f32 VR512:$src2), i8immZExt5:$cc, (i16 -1),
-                FROUND_CURRENT)),
-          (COPY_TO_REGCLASS (VCMPPSZrri VR512:$src1, VR512:$src2,
-                             (I8Imm imm:$cc)), GR16)>;
-
-def : Pat<(i8 (int_x86_avx512_mask_cmp_pd_512 (v8f64 VR512:$src1),
-                (v8f64 VR512:$src2), i8immZExt5:$cc, (i8 -1),
-                 FROUND_CURRENT)),
-          (COPY_TO_REGCLASS (VCMPPDZrri VR512:$src1, VR512:$src2,
-                             (I8Imm imm:$cc)), GR8)>;
-
+//-----------------------------------------------------------------
 // Mask register copy, including
 // - copy between mask registers
 // - load/store mask registers
@@ -1727,7 +1797,9 @@ let Predicates = [HasAVX512] in {
   def : Pat<(store (i16 (bitconvert (v16i1 VK16:$src))), addr:$dst),
             (KMOVWmk addr:$dst, VK16:$src)>;
   def : Pat<(i1 (load addr:$src)),
-            (COPY_TO_REGCLASS (KMOVWkm addr:$src), VK1)>;
+            (COPY_TO_REGCLASS (AND16ri (i16 (SUBREG_TO_REG (i32 0),
+                                              (MOV8rm addr:$src), sub_8bit)),
+                                (i16 1)), VK1)>;
   def : Pat<(v16i1 (bitconvert (i16 (load addr:$src)))),
             (KMOVWkm addr:$src)>;
 }
@@ -1763,6 +1835,8 @@ let Predicates = [HasAVX512] in {
 
   def : Pat<(i32 (zext VK1:$src)),
             (AND32ri (KMOVWrk (COPY_TO_REGCLASS VK1:$src, VK16)), (i32 1))>;
+  def : Pat<(i32 (anyext VK1:$src)),
+            (KMOVWrk (COPY_TO_REGCLASS VK1:$src, VK16))>;
   def : Pat<(i8 (zext VK1:$src)),
             (EXTRACT_SUBREG
              (AND32ri (KMOVWrk
@@ -1788,17 +1862,18 @@ let Predicates = [HasBWI] in {
 
 
 // With AVX-512 only, 8-bit mask is promoted to 16-bit mask.
-let Predicates = [HasAVX512] in {
+let Predicates = [HasAVX512, NoDQI] in {
   // GR from/to 8-bit mask without native support
   def : Pat<(v8i1 (bitconvert (i8 GR8:$src))),
             (COPY_TO_REGCLASS
-              (KMOVWkr (SUBREG_TO_REG (i32 0), GR8:$src, sub_8bit)),
-              VK8)>;
+             (KMOVWkr (MOVZX32rr8 GR8 :$src)), VK8)>;
   def : Pat<(i8 (bitconvert (v8i1 VK8:$src))),
             (EXTRACT_SUBREG
               (KMOVWrk (COPY_TO_REGCLASS VK8:$src, VK16)),
               sub_8bit)>;
+}
 
+let Predicates = [HasAVX512] in {
   def : Pat<(i1 (X86Vextract VK16:$src, (iPTR 0))),
             (COPY_TO_REGCLASS VK16:$src, VK1)>;
   def : Pat<(i1 (X86Vextract VK8:$src, (iPTR 0))),
@@ -1858,18 +1933,21 @@ def : Pat<(xor VK64:$src1, (v64i1 immAllOnesV)), (KNOTQrr VK64:$src1)>;
 let Predicates = [HasAVX512, NoDQI] in {
 def : Pat<(xor VK8:$src1,  (v8i1 immAllOnesV)),
           (COPY_TO_REGCLASS (KNOTWrr (COPY_TO_REGCLASS VK8:$src1, VK16)), VK8)>;
-
 def : Pat<(not VK8:$src),
           (COPY_TO_REGCLASS
             (KNOTWrr (COPY_TO_REGCLASS VK8:$src, VK16)), VK8)>;
 }
+def : Pat<(xor VK4:$src1,  (v4i1 immAllOnesV)),
+          (COPY_TO_REGCLASS (KNOTWrr (COPY_TO_REGCLASS VK4:$src1, VK16)), VK4)>;
+def : Pat<(xor VK2:$src1,  (v2i1 immAllOnesV)),
+          (COPY_TO_REGCLASS (KNOTWrr (COPY_TO_REGCLASS VK2:$src1, VK16)), VK2)>;
 
 // Mask binary operation
 // - KAND, KANDN, KOR, KXNOR, KXOR
 multiclass avx512_mask_binop<bits<8> opc, string OpcodeStr,
                            RegisterClass KRC, SDPatternOperator OpNode,
-                           Predicate prd> {
-  let Predicates = [prd] in
+                           Predicate prd, bit IsCommutable> {
+  let Predicates = [prd], isCommutable = IsCommutable in
     def rr : I<opc, MRMSrcReg, (outs KRC:$dst), (ins KRC:$src1, KRC:$src2),
                !strconcat(OpcodeStr,
                           "\t{$src2, $src1, $dst|$dst, $src1, $src2}"),
@@ -1877,40 +1955,25 @@ multiclass avx512_mask_binop<bits<8> opc, string OpcodeStr,
 }
 
 multiclass avx512_mask_binop_all<bits<8> opc, string OpcodeStr,
-                               SDPatternOperator OpNode> {
+                               SDPatternOperator OpNode, bit IsCommutable> {
   defm B : avx512_mask_binop<opc, !strconcat(OpcodeStr, "b"), VK8, OpNode,
-                             HasDQI>, VEX_4V, VEX_L, PD;
+                             HasDQI, IsCommutable>, VEX_4V, VEX_L, PD;
   defm W : avx512_mask_binop<opc, !strconcat(OpcodeStr, "w"), VK16, OpNode,
-                             HasAVX512>, VEX_4V, VEX_L, PS;
+                             HasAVX512, IsCommutable>, VEX_4V, VEX_L, PS;
   defm D : avx512_mask_binop<opc, !strconcat(OpcodeStr, "d"), VK32, OpNode,
-                             HasBWI>, VEX_4V, VEX_L, VEX_W, PD;
+                             HasBWI, IsCommutable>, VEX_4V, VEX_L, VEX_W, PD;
   defm Q : avx512_mask_binop<opc, !strconcat(OpcodeStr, "q"), VK64, OpNode,
-                             HasBWI>, VEX_4V, VEX_L, VEX_W, PS;
+                             HasBWI, IsCommutable>, VEX_4V, VEX_L, VEX_W, PS;
 }
 
 def andn : PatFrag<(ops node:$i0, node:$i1), (and (not node:$i0), node:$i1)>;
 def xnor : PatFrag<(ops node:$i0, node:$i1), (not (xor node:$i0, node:$i1))>;
 
-let isCommutable = 1 in {
-  defm KAND  : avx512_mask_binop_all<0x41, "kand",  and>;
-  defm KOR   : avx512_mask_binop_all<0x45, "kor",   or>;
-  defm KXNOR : avx512_mask_binop_all<0x46, "kxnor", xnor>;
-  defm KXOR  : avx512_mask_binop_all<0x47, "kxor",  xor>;
-}
-let isCommutable = 0 in
-  defm KANDN : avx512_mask_binop_all<0x42, "kandn", andn>;
-
-def : Pat<(xor VK1:$src1, VK1:$src2),
-     (COPY_TO_REGCLASS (KXORWrr (COPY_TO_REGCLASS VK1:$src1, VK16),
-                                (COPY_TO_REGCLASS VK1:$src2, VK16)), VK1)>;
-
-def : Pat<(or VK1:$src1, VK1:$src2),
-     (COPY_TO_REGCLASS (KORWrr (COPY_TO_REGCLASS VK1:$src1, VK16),
-                               (COPY_TO_REGCLASS VK1:$src2, VK16)), VK1)>;
-
-def : Pat<(and VK1:$src1, VK1:$src2),
-     (COPY_TO_REGCLASS (KANDWrr (COPY_TO_REGCLASS VK1:$src1, VK16),
-                                (COPY_TO_REGCLASS VK1:$src2, VK16)), VK1)>;
+defm KAND  : avx512_mask_binop_all<0x41, "kand",  and,  1>;
+defm KOR   : avx512_mask_binop_all<0x45, "kor",   or,   1>;
+defm KXNOR : avx512_mask_binop_all<0x46, "kxnor", xnor, 1>;
+defm KXOR  : avx512_mask_binop_all<0x47, "kxor",  xor,  1>;
+defm KANDN : avx512_mask_binop_all<0x42, "kandn", andn, 0>;
 
 multiclass avx512_mask_binop_int<string IntName, string InstName> {
   let Predicates = [HasAVX512] in
@@ -1927,13 +1990,28 @@ defm : avx512_mask_binop_int<"kor",   "KOR">;
 defm : avx512_mask_binop_int<"kxnor", "KXNOR">;
 defm : avx512_mask_binop_int<"kxor",  "KXOR">;
 
-// With AVX-512, 8-bit mask is promoted to 16-bit mask.
 multiclass avx512_binop_pat<SDPatternOperator OpNode, Instruction Inst> {
-  let Predicates = [HasAVX512] in
-    def : Pat<(OpNode VK8:$src1, VK8:$src2),
-              (COPY_TO_REGCLASS
-                (Inst (COPY_TO_REGCLASS VK8:$src1, VK16),
-                      (COPY_TO_REGCLASS VK8:$src2, VK16)), VK8)>;
+  // With AVX512F, 8-bit mask is promoted to 16-bit mask,
+  // for the DQI set, this type is legal and KxxxB instruction is used
+  let Predicates = [NoDQI] in
+  def : Pat<(OpNode VK8:$src1, VK8:$src2),
+            (COPY_TO_REGCLASS
+              (Inst (COPY_TO_REGCLASS VK8:$src1, VK16),
+                    (COPY_TO_REGCLASS VK8:$src2, VK16)), VK8)>;
+
+  // All types smaller than 8 bits require conversion anyway
+  def : Pat<(OpNode VK1:$src1, VK1:$src2),
+        (COPY_TO_REGCLASS (Inst
+                           (COPY_TO_REGCLASS VK1:$src1, VK16),
+                           (COPY_TO_REGCLASS VK1:$src2, VK16)), VK1)>;
+  def : Pat<(OpNode VK2:$src1, VK2:$src2),
+        (COPY_TO_REGCLASS (Inst
+                           (COPY_TO_REGCLASS VK2:$src1, VK16),
+                           (COPY_TO_REGCLASS VK2:$src2, VK16)), VK1)>;
+  def : Pat<(OpNode VK4:$src1, VK4:$src2),
+        (COPY_TO_REGCLASS (Inst
+                           (COPY_TO_REGCLASS VK4:$src1, VK16),
+                           (COPY_TO_REGCLASS VK4:$src2, VK16)), VK1)>;
 }
 
 defm : avx512_binop_pat<and,  KANDWrr>;
@@ -1942,6 +2020,32 @@ defm : avx512_binop_pat<or,   KORWrr>;
 defm : avx512_binop_pat<xnor, KXNORWrr>;
 defm : avx512_binop_pat<xor,  KXORWrr>;
 
+def : Pat<(xor (xor VK16:$src1, VK16:$src2), (v16i1 immAllOnesV)),
+          (KXNORWrr VK16:$src1, VK16:$src2)>;
+def : Pat<(xor (xor VK8:$src1, VK8:$src2), (v8i1 immAllOnesV)),
+          (KXNORBrr VK8:$src1, VK8:$src2)>, Requires<[HasDQI]>;
+def : Pat<(xor (xor VK32:$src1, VK32:$src2), (v32i1 immAllOnesV)),
+          (KXNORDrr VK32:$src1, VK32:$src2)>, Requires<[HasBWI]>;
+def : Pat<(xor (xor VK64:$src1, VK64:$src2), (v64i1 immAllOnesV)),
+          (KXNORQrr VK64:$src1, VK64:$src2)>, Requires<[HasBWI]>;
+
+let Predicates = [NoDQI] in
+def : Pat<(xor (xor VK8:$src1, VK8:$src2), (v8i1 immAllOnesV)),
+          (COPY_TO_REGCLASS (KXNORWrr (COPY_TO_REGCLASS VK8:$src1, VK16),
+                             (COPY_TO_REGCLASS VK8:$src2, VK16)), VK8)>;
+
+def : Pat<(xor (xor VK4:$src1, VK4:$src2), (v4i1 immAllOnesV)),
+          (COPY_TO_REGCLASS (KXNORWrr (COPY_TO_REGCLASS VK4:$src1, VK16),
+                             (COPY_TO_REGCLASS VK4:$src2, VK16)), VK4)>;
+
+def : Pat<(xor (xor VK2:$src1, VK2:$src2), (v2i1 immAllOnesV)),
+          (COPY_TO_REGCLASS (KXNORWrr (COPY_TO_REGCLASS VK2:$src1, VK16),
+                             (COPY_TO_REGCLASS VK2:$src2, VK16)), VK2)>;
+
+def : Pat<(xor (xor VK1:$src1, VK1:$src2), (i1 1)),
+          (COPY_TO_REGCLASS (KXNORWrr (COPY_TO_REGCLASS VK1:$src1, VK16),
+                             (COPY_TO_REGCLASS VK1:$src2, VK16)), VK1)>;
+
 // Mask unpacking
 multiclass avx512_mask_unpck<bits<8> opc, string OpcodeStr,
                            RegisterClass KRC> {
@@ -2037,6 +2141,8 @@ multiclass avx512_mask_setop<RegisterClass KRC, ValueType VT, PatFrag Val> {
 multiclass avx512_mask_setop_w<PatFrag Val> {
   defm B : avx512_mask_setop<VK8,   v8i1, Val>;
   defm W : avx512_mask_setop<VK16, v16i1, Val>;
+  defm D : avx512_mask_setop<VK32,  v32i1, Val>;
+  defm Q : avx512_mask_setop<VK64, v64i1, Val>;
 }
 
 defm KSET0 : avx512_mask_setop_w<immAllZerosV>;
@@ -2046,9 +2152,11 @@ defm KSET1 : avx512_mask_setop_w<immAllOnesV>;
 let Predicates = [HasAVX512] in {
   def : Pat<(v8i1 immAllZerosV), (COPY_TO_REGCLASS (KSET0W), VK8)>;
   def : Pat<(v8i1 immAllOnesV),  (COPY_TO_REGCLASS (KSET1W), VK8)>;
+  def : Pat<(v4i1 immAllOnesV),  (COPY_TO_REGCLASS (KSET1W), VK4)>;
+  def : Pat<(v2i1 immAllOnesV),  (COPY_TO_REGCLASS (KSET1W), VK2)>;
   def : Pat<(i1 0), (COPY_TO_REGCLASS (KSET0W), VK1)>;
-  def : Pat<(i1 1), (COPY_TO_REGCLASS (KSET1W), VK1)>;
-  def : Pat<(i1 -1), (COPY_TO_REGCLASS (KSET1W), VK1)>;
+  def : Pat<(i1 1), (COPY_TO_REGCLASS (KSHIFTRWri (KSET1W), (i8 15)), VK1)>;
+  def : Pat<(i1 -1), (COPY_TO_REGCLASS (KSHIFTRWri (KSET1W), (i8 15)), VK1)>;
 }
 def : Pat<(v8i1 (extract_subvector (v16i1 VK16:$src), (iPTR 0))),
           (v8i1 (COPY_TO_REGCLASS VK16:$src, VK8))>;
@@ -2059,6 +2167,12 @@ def : Pat<(v16i1 (insert_subvector undef, (v8i1 VK8:$src), (iPTR 0))),
 def : Pat<(v8i1 (extract_subvector (v16i1 VK16:$src), (iPTR 8))),
           (v8i1 (COPY_TO_REGCLASS (KSHIFTRWri VK16:$src, (i8 8)), VK8))>;
 
+def : Pat<(v32i1 (extract_subvector (v64i1 VK64:$src), (iPTR 0))),
+          (v32i1 (COPY_TO_REGCLASS VK64:$src, VK32))>;
+
+def : Pat<(v32i1 (extract_subvector (v64i1 VK64:$src), (iPTR 32))),
+          (v32i1 (COPY_TO_REGCLASS (KSHIFTRQri VK64:$src, (i8 32)), VK32))>;
+
 let Predicates = [HasVLX] in {
   def : Pat<(v8i1 (insert_subvector undef, (v4i1 VK4:$src), (iPTR 0))),
             (v8i1 (COPY_TO_REGCLASS VK4:$src, VK8))>;
@@ -2312,7 +2426,7 @@ def: Pat<(int_x86_avx512_mask_store_pd_512 addr:$ptr, (v8f64 VR512:$src),
             VR512:$src)>;
 
 let Predicates = [HasAVX512, NoVLX] in {
-def: Pat<(masked_store addr:$ptr, VK8WM:$mask, (v8f32 VR256:$src)),
+def: Pat<(X86mstore addr:$ptr, VK8WM:$mask, (v8f32 VR256:$src)),
          (VMOVUPSZmrk addr:$ptr,
          (v16i1 (COPY_TO_REGCLASS VK8WM:$mask, VK16WM)),
          (INSERT_SUBREG (v16f32 (IMPLICIT_DEF)), VR256:$src, sub_ymm))>;
@@ -2390,7 +2504,7 @@ def : Pat<(v16i32 (vselect VK16WM:$mask, (v16i32 immAllZerosV),
 }
 // NoVLX patterns
 let Predicates = [HasAVX512, NoVLX] in {
-def: Pat<(masked_store addr:$ptr, VK8WM:$mask, (v8i32 VR256:$src)),
+def: Pat<(X86mstore addr:$ptr, VK8WM:$mask, (v8i32 VR256:$src)),
          (VMOVDQU32Zmrk addr:$ptr,
          (v16i1 (COPY_TO_REGCLASS VK8WM:$mask, VK16WM)),
          (INSERT_SUBREG (v16i32 (IMPLICIT_DEF)), VR256:$src, sub_ymm))>;
@@ -2850,19 +2964,19 @@ multiclass avx512_binop_rm<bits<8> opc, string OpcodeStr, SDNode OpNode,
                            X86VectorVTInfo _, OpndItins itins,
                            bit IsCommutable = 0> {
   defm rr : AVX512_maskable<opc, MRMSrcReg, _, (outs _.RC:$dst),
-                    (ins _.RC:$src1, _.RC:$src2), OpcodeStr##_.Suffix,
+                    (ins _.RC:$src1, _.RC:$src2), OpcodeStr,
                     "$src2, $src1", "$src1, $src2",
                     (_.VT (OpNode _.RC:$src1, _.RC:$src2)),
-                    "", itins.rr, IsCommutable>,
+                    itins.rr, IsCommutable>,
             AVX512BIBase, EVEX_4V;
 
   let mayLoad = 1 in
     defm rm : AVX512_maskable<opc, MRMSrcMem, _, (outs _.RC:$dst),
-                    (ins _.RC:$src1, _.MemOp:$src2), OpcodeStr##_.Suffix,
+                    (ins _.RC:$src1, _.MemOp:$src2), OpcodeStr,
                     "$src2, $src1", "$src1, $src2",
                     (_.VT (OpNode _.RC:$src1,
                                   (bitconvert (_.LdFrag addr:$src2)))),
-                    "", itins.rm>,
+                    itins.rm>,
               AVX512BIBase, EVEX_4V;
 }
 
@@ -2872,13 +2986,13 @@ multiclass avx512_binop_rmb<bits<8> opc, string OpcodeStr, SDNode OpNode,
            avx512_binop_rm<opc, OpcodeStr, OpNode, _, itins, IsCommutable> {
   let mayLoad = 1 in
     defm rmb : AVX512_maskable<opc, MRMSrcMem, _, (outs _.RC:$dst),
-                    (ins _.RC:$src1, _.ScalarMemOp:$src2), OpcodeStr##_.Suffix,
+                    (ins _.RC:$src1, _.ScalarMemOp:$src2), OpcodeStr,
                     "${src2}"##_.BroadcastStr##", $src1",
                     "$src1, ${src2}"##_.BroadcastStr,
                     (_.VT (OpNode _.RC:$src1,
                                   (X86VBroadcast
                                       (_.ScalarLdFrag addr:$src2)))),
-                    "", itins.rm>,
+                    itins.rm>,
                AVX512BIBase, EVEX_4V, EVEX_B;
 }
 
@@ -2944,20 +3058,20 @@ multiclass avx512_binop_rm_vl_b<bits<8> opc, string OpcodeStr, SDNode OpNode,
 multiclass avx512_binop_rm_vl_dq<bits<8> opc_d, bits<8> opc_q, string OpcodeStr,
                                  SDNode OpNode, OpndItins itins, Predicate prd,
                                  bit IsCommutable = 0> {
-  defm Q : avx512_binop_rm_vl_q<opc_q, OpcodeStr, OpNode, itins, prd,
+  defm Q : avx512_binop_rm_vl_q<opc_q, OpcodeStr#"q", OpNode, itins, prd,
                                    IsCommutable>;
 
-  defm D : avx512_binop_rm_vl_d<opc_d, OpcodeStr, OpNode, itins, prd,
+  defm D : avx512_binop_rm_vl_d<opc_d, OpcodeStr#"d", OpNode, itins, prd,
                                    IsCommutable>;
 }
 
 multiclass avx512_binop_rm_vl_bw<bits<8> opc_b, bits<8> opc_w, string OpcodeStr,
                                  SDNode OpNode, OpndItins itins, Predicate prd,
                                  bit IsCommutable = 0> {
-  defm W : avx512_binop_rm_vl_w<opc_w, OpcodeStr, OpNode, itins, prd,
+  defm W : avx512_binop_rm_vl_w<opc_w, OpcodeStr#"w", OpNode, itins, prd,
                                    IsCommutable>;
 
-  defm B : avx512_binop_rm_vl_b<opc_b, OpcodeStr, OpNode, itins, prd,
+  defm B : avx512_binop_rm_vl_b<opc_b, OpcodeStr#"b", OpNode, itins, prd,
                                    IsCommutable>;
 }
 
@@ -2980,7 +3094,7 @@ multiclass avx512_binop_rm2<bits<8> opc, string OpcodeStr, OpndItins itins,
                             (_Dst.VT (OpNode 
                                          (_Src.VT _Src.RC:$src1), 
                                          (_Src.VT _Src.RC:$src2))),
-                            "",itins.rr, IsCommutable>, 
+                            itins.rr, IsCommutable>, 
                             AVX512BIBase, EVEX_4V;
   let mayLoad = 1 in {
       defm rm : AVX512_maskable<opc, MRMSrcMem, _Dst, (outs _Dst.RC:$dst),
@@ -2988,7 +3102,7 @@ multiclass avx512_binop_rm2<bits<8> opc, string OpcodeStr, OpndItins itins,
                             "$src2, $src1", "$src1, $src2",
                             (_Dst.VT (OpNode (_Src.VT _Src.RC:$src1),
                                           (bitconvert (_Src.LdFrag addr:$src2)))),
-                            "", itins.rm>,
+                            itins.rm>,
                             AVX512BIBase, EVEX_4V;
 
       defm rmb : AVX512_maskable<opc, MRMSrcMem, _Dst, (outs _Dst.RC:$dst),
@@ -2996,10 +3110,10 @@ multiclass avx512_binop_rm2<bits<8> opc, string OpcodeStr, OpndItins itins,
                         OpcodeStr,
                         "${src2}"##_Dst.BroadcastStr##", $src1",
                          "$src1, ${src2}"##_Dst.BroadcastStr,
-                        (_Dst.VT (OpNode (_Src.VT _Src.RC:$src1), (bc_v16i32 
+                        (_Dst.VT (OpNode (_Src.VT _Src.RC:$src1), (bitconvert 
                                      (_Dst.VT (X86VBroadcast 
                                               (_Dst.ScalarLdFrag addr:$src2)))))),
-                        "", itins.rm>,
+                        itins.rm>,
                         AVX512BIBase, EVEX_4V, EVEX_B;
   }
 }
@@ -3008,160 +3122,166 @@ defm VPADD : avx512_binop_rm_vl_all<0xFC, 0xFD, 0xFE, 0xD4, "vpadd", add,
                                     SSE_INTALU_ITINS_P, 1>;
 defm VPSUB : avx512_binop_rm_vl_all<0xF8, 0xF9, 0xFA, 0xFB, "vpsub", sub,
                                     SSE_INTALU_ITINS_P, 0>;
-defm VPMULLD : avx512_binop_rm_vl_d<0x40, "vpmull", mul,
+defm VPADDS : avx512_binop_rm_vl_bw<0xEC, 0xED, "vpadds", X86adds,
+                                    SSE_INTALU_ITINS_P, HasBWI, 1>;
+defm VPSUBS : avx512_binop_rm_vl_bw<0xE8, 0xE9, "vpsubs", X86subs,
+                                    SSE_INTALU_ITINS_P, HasBWI, 0>;
+defm VPADDUS : avx512_binop_rm_vl_bw<0xDC, 0xDD, "vpaddus", X86addus,
+                                    SSE_INTALU_ITINS_P, HasBWI, 1>;
+defm VPSUBUS : avx512_binop_rm_vl_bw<0xD8, 0xD9, "vpsubus", X86subus,
+                                    SSE_INTALU_ITINS_P, HasBWI, 0>;
+defm VPMULLD : avx512_binop_rm_vl_d<0x40, "vpmulld", mul,
                                    SSE_INTALU_ITINS_P, HasAVX512, 1>, T8PD;
-defm VPMULLW : avx512_binop_rm_vl_w<0xD5, "vpmull", mul,
+defm VPMULLW : avx512_binop_rm_vl_w<0xD5, "vpmullw", mul,
                                    SSE_INTALU_ITINS_P, HasBWI, 1>;
-defm VPMULLQ : avx512_binop_rm_vl_q<0x40, "vpmull", mul,
+defm VPMULLQ : avx512_binop_rm_vl_q<0x40, "vpmullq", mul,
                                    SSE_INTALU_ITINS_P, HasDQI, 1>, T8PD;
+defm VPMULHW : avx512_binop_rm_vl_w<0xE5, "vpmulhw", mulhs, SSE_INTALU_ITINS_P,
+                                    HasBWI, 1>;
+defm VPMULHUW : avx512_binop_rm_vl_w<0xE4, "vpmulhuw", mulhu, SSE_INTMUL_ITINS_P,
+                                    HasBWI, 1>;
+defm VPMULHRSW : avx512_binop_rm_vl_w<0x0B, "vpmulhrsw", X86mulhrs, SSE_INTMUL_ITINS_P,
+                                    HasBWI, 1>, T8PD;
+defm VPAVG : avx512_binop_rm_vl_bw<0xE0, 0xE3, "vpavg", X86avg,
+                                    SSE_INTALU_ITINS_P, HasBWI, 1>;
+                                   
+multiclass avx512_binop_all<bits<8> opc, string OpcodeStr, OpndItins itins,
+                            SDNode OpNode, bit IsCommutable = 0> {
+
+  defm NAME#Z : avx512_binop_rm2<opc, OpcodeStr, itins, OpNode,
+                                 v16i32_info, v8i64_info, IsCommutable>,
+                                EVEX_V512, EVEX_CD8<64, CD8VF>, VEX_W;
+  let Predicates = [HasVLX] in {
+    defm NAME#Z256 : avx512_binop_rm2<opc, OpcodeStr, itins, OpNode,
+                                      v8i32x_info, v4i64x_info, IsCommutable>,
+                                     EVEX_V256, EVEX_CD8<64, CD8VF>, VEX_W;
+    defm NAME#Z128 : avx512_binop_rm2<opc, OpcodeStr, itins, OpNode,
+                                      v4i32x_info, v2i64x_info, IsCommutable>,
+                                     EVEX_V128, EVEX_CD8<64, CD8VF>, VEX_W;
+  }
+}                            
+
+defm VPMULDQ : avx512_binop_all<0x28, "vpmuldq", SSE_INTALU_ITINS_P,
+                   X86pmuldq, 1>,T8PD;
+defm VPMULUDQ : avx512_binop_all<0xF4, "vpmuludq", SSE_INTMUL_ITINS_P,
+                   X86pmuludq, 1>;
+
+multiclass avx512_packs_rmb<bits<8> opc, string OpcodeStr, SDNode OpNode,
+                            X86VectorVTInfo _Src, X86VectorVTInfo _Dst> {
+  let mayLoad = 1 in {
+      defm rmb : AVX512_maskable<opc, MRMSrcMem, _Dst, (outs _Dst.RC:$dst),
+                        (ins _Src.RC:$src1, _Src.ScalarMemOp:$src2), 
+                        OpcodeStr,
+                        "${src2}"##_Src.BroadcastStr##", $src1",
+                         "$src1, ${src2}"##_Src.BroadcastStr,
+                        (_Dst.VT (OpNode (_Src.VT _Src.RC:$src1), (bitconvert 
+                                     (_Src.VT (X86VBroadcast 
+                                              (_Src.ScalarLdFrag addr:$src2))))))>,
+                        EVEX_4V, EVEX_B, EVEX_CD8<_Src.EltSize, CD8VF>;
+  }
+}
+
+multiclass avx512_packs_rm<bits<8> opc, string OpcodeStr, 
+                            SDNode OpNode,X86VectorVTInfo _Src, 
+                            X86VectorVTInfo _Dst> {
+  defm rr : AVX512_maskable<opc, MRMSrcReg, _Dst, (outs _Dst.RC:$dst), 
+                            (ins _Src.RC:$src1, _Src.RC:$src2), OpcodeStr,
+                            "$src2, $src1","$src1, $src2", 
+                            (_Dst.VT (OpNode 
+                                         (_Src.VT _Src.RC:$src1), 
+                                         (_Src.VT _Src.RC:$src2)))>,
+                            EVEX_CD8<_Src.EltSize, CD8VF>, EVEX_4V;
+  let mayLoad = 1 in {
+    defm rm : AVX512_maskable<opc, MRMSrcMem, _Dst, (outs _Dst.RC:$dst),
+                          (ins _Src.RC:$src1, _Src.MemOp:$src2), OpcodeStr,
+                          "$src2, $src1", "$src1, $src2",
+                          (_Dst.VT (OpNode (_Src.VT _Src.RC:$src1),
+                                        (bitconvert (_Src.LdFrag addr:$src2))))>,
+                           EVEX_4V, EVEX_CD8<_Src.EltSize, CD8VF>;
+  }
+}
+
+multiclass avx512_packs_all_i32_i16<bits<8> opc, string OpcodeStr,
+                                    SDNode OpNode> {
+  defm NAME#Z : avx512_packs_rm<opc, OpcodeStr, OpNode, v16i32_info,
+                                 v32i16_info>,
+                avx512_packs_rmb<opc, OpcodeStr, OpNode, v16i32_info,
+                                 v32i16_info>, EVEX_V512;
+  let Predicates = [HasVLX] in {
+    defm NAME#Z256 : avx512_packs_rm<opc, OpcodeStr, OpNode, v8i32x_info,
+                                     v16i16x_info>,
+                     avx512_packs_rmb<opc, OpcodeStr, OpNode, v8i32x_info,
+                                     v16i16x_info>, EVEX_V256;
+    defm NAME#Z128 : avx512_packs_rm<opc, OpcodeStr, OpNode, v4i32x_info,
+                                     v8i16x_info>,
+                     avx512_packs_rmb<opc, OpcodeStr, OpNode, v4i32x_info,
+                                     v8i16x_info>, EVEX_V128;
+  }
+}
+multiclass avx512_packs_all_i16_i8<bits<8> opc, string OpcodeStr,
+                            SDNode OpNode> {
+  defm NAME#Z : avx512_packs_rm<opc, OpcodeStr, OpNode, v32i16_info,
+                                v64i8_info>, EVEX_V512;
+  let Predicates = [HasVLX] in {
+    defm NAME#Z256 : avx512_packs_rm<opc, OpcodeStr, OpNode, v16i16x_info,
+                                    v32i8x_info>, EVEX_V256;
+    defm NAME#Z128 : avx512_packs_rm<opc, OpcodeStr, OpNode, v8i16x_info,
+                                    v16i8x_info>, EVEX_V128;
+  }
+}
 
-defm VPMULDQZ : avx512_binop_rm2<0x28, "vpmuldq", SSE_INTALU_ITINS_P,
-                   X86pmuldq, v16i32_info, v8i64_info, 1>, 
-                   T8PD, EVEX_V512, EVEX_CD8<64, CD8VF>, VEX_W;
+multiclass avx512_vpmadd<bits<8> opc, string OpcodeStr,
+                            SDNode OpNode, AVX512VLVectorVTInfo _Src,
+                            AVX512VLVectorVTInfo _Dst> {
+  defm NAME#Z : avx512_packs_rm<opc, OpcodeStr, OpNode, _Src.info512,
+                                _Dst.info512>, EVEX_V512;
+  let Predicates = [HasVLX] in {
+    defm NAME#Z256 : avx512_packs_rm<opc, OpcodeStr, OpNode, _Src.info256,
+                                     _Dst.info256>, EVEX_V256;
+    defm NAME#Z128 : avx512_packs_rm<opc, OpcodeStr, OpNode, _Src.info128,
+                                     _Dst.info128>, EVEX_V128;
+  }
+}
+
+let Predicates = [HasBWI] in {
+  defm VPACKSSDW : avx512_packs_all_i32_i16<0x6B, "vpackssdw", X86Packss>, PD;
+  defm VPACKUSDW : avx512_packs_all_i32_i16<0x2b, "vpackusdw", X86Packus>, T8PD;
+  defm VPACKSSWB : avx512_packs_all_i16_i8 <0x63, "vpacksswb", X86Packss>, AVX512BIBase, VEX_W;
+  defm VPACKUSWB : avx512_packs_all_i16_i8 <0x67, "vpackuswb", X86Packus>, AVX512BIBase, VEX_W;
 
-defm VPMULUDQZ : avx512_binop_rm2<0xF4, "vpmuludq", SSE_INTMUL_ITINS_P,
-                   X86pmuludq, v16i32_info, v8i64_info, 1>, 
-                   EVEX_V512, EVEX_CD8<64, CD8VF>, VEX_W;
+  defm VPMADDUBSW : avx512_vpmadd<0x04, "vpmaddubsw", X86vpmaddubsw,
+                       avx512vl_i8_info, avx512vl_i16_info>, AVX512BIBase, T8PD;
+  defm VPMADDWD   : avx512_vpmadd<0xF5, "vpmaddwd", X86vpmaddwd,
+                       avx512vl_i16_info, avx512vl_i32_info>, AVX512BIBase;
+}
 
-defm VPMAXSB : avx512_binop_rm_vl_b<0x3C, "vpmaxs", X86smax,
+defm VPMAXSB : avx512_binop_rm_vl_b<0x3C, "vpmaxsb", smax,
                                      SSE_INTALU_ITINS_P, HasBWI, 1>, T8PD;
-defm VPMAXSW : avx512_binop_rm_vl_w<0xEE, "vpmaxs", X86smax,
+defm VPMAXSW : avx512_binop_rm_vl_w<0xEE, "vpmaxsw", smax,
                                      SSE_INTALU_ITINS_P, HasBWI, 1>;
-defm VPMAXS : avx512_binop_rm_vl_dq<0x3D, 0x3D, "vpmaxs", X86smax,
+defm VPMAXS : avx512_binop_rm_vl_dq<0x3D, 0x3D, "vpmaxs", smax,
                                      SSE_INTALU_ITINS_P, HasAVX512, 1>, T8PD;
 
-defm VPMAXUB : avx512_binop_rm_vl_b<0xDE, "vpmaxu", X86umax,
+defm VPMAXUB : avx512_binop_rm_vl_b<0xDE, "vpmaxub", umax,
                                      SSE_INTALU_ITINS_P, HasBWI, 1>;
-defm VPMAXUW : avx512_binop_rm_vl_w<0x3E, "vpmaxu", X86umax,
+defm VPMAXUW : avx512_binop_rm_vl_w<0x3E, "vpmaxuw", umax,
                                      SSE_INTALU_ITINS_P, HasBWI, 1>, T8PD;
-defm VPMAXU : avx512_binop_rm_vl_dq<0x3F, 0x3F, "vpmaxu", X86umax,
+defm VPMAXU : avx512_binop_rm_vl_dq<0x3F, 0x3F, "vpmaxu", umax,
                                      SSE_INTALU_ITINS_P, HasAVX512, 1>, T8PD;
 
-defm VPMINSB : avx512_binop_rm_vl_b<0x38, "vpmins", X86smin,
+defm VPMINSB : avx512_binop_rm_vl_b<0x38, "vpminsb", smin,
                                      SSE_INTALU_ITINS_P, HasBWI, 1>, T8PD;
-defm VPMINSW : avx512_binop_rm_vl_w<0xEA, "vpmins", X86smin,
+defm VPMINSW : avx512_binop_rm_vl_w<0xEA, "vpminsw", smin,
                                      SSE_INTALU_ITINS_P, HasBWI, 1>;
-defm VPMINS : avx512_binop_rm_vl_dq<0x39, 0x39, "vpmins", X86smin,
+defm VPMINS : avx512_binop_rm_vl_dq<0x39, 0x39, "vpmins", smin,
                                      SSE_INTALU_ITINS_P, HasAVX512, 1>, T8PD;
 
-defm VPMINUB : avx512_binop_rm_vl_b<0xDA, "vpminu", X86umin,
+defm VPMINUB : avx512_binop_rm_vl_b<0xDA, "vpminub", umin,
                                      SSE_INTALU_ITINS_P, HasBWI, 1>;
-defm VPMINUW : avx512_binop_rm_vl_w<0x3A, "vpminu", X86umin,
+defm VPMINUW : avx512_binop_rm_vl_w<0x3A, "vpminuw", umin,
                                      SSE_INTALU_ITINS_P, HasBWI, 1>, T8PD;
-defm VPMINU : avx512_binop_rm_vl_dq<0x3B, 0x3B, "vpminu", X86umin,
+defm VPMINU : avx512_binop_rm_vl_dq<0x3B, 0x3B, "vpminu", umin,
                                      SSE_INTALU_ITINS_P, HasAVX512, 1>, T8PD;
-
-def : Pat <(v16i32 (int_x86_avx512_mask_pmaxs_d_512 (v16i32 VR512:$src1),
-                    (v16i32 VR512:$src2), (v16i32 immAllZerosV), (i16 -1))),
-           (VPMAXSDZrr VR512:$src1, VR512:$src2)>;
-def : Pat <(v16i32 (int_x86_avx512_mask_pmaxu_d_512 (v16i32 VR512:$src1),
-                    (v16i32 VR512:$src2), (v16i32 immAllZerosV), (i16 -1))),
-           (VPMAXUDZrr VR512:$src1, VR512:$src2)>;
-def : Pat <(v8i64 (int_x86_avx512_mask_pmaxs_q_512 (v8i64 VR512:$src1),
-                (v8i64 VR512:$src2), (bc_v8i64 (v16i32 immAllZerosV)), (i8 -1))),
-           (VPMAXSQZrr VR512:$src1, VR512:$src2)>;
-def : Pat <(v8i64 (int_x86_avx512_mask_pmaxu_q_512 (v8i64 VR512:$src1),
-                (v8i64 VR512:$src2), (bc_v8i64 (v16i32 immAllZerosV)), (i8 -1))),
-           (VPMAXUQZrr VR512:$src1, VR512:$src2)>;
-def : Pat <(v16i32 (int_x86_avx512_mask_pmins_d_512 (v16i32 VR512:$src1),
-                    (v16i32 VR512:$src2), (v16i32 immAllZerosV), (i16 -1))),
-           (VPMINSDZrr VR512:$src1, VR512:$src2)>;
-def : Pat <(v16i32 (int_x86_avx512_mask_pminu_d_512 (v16i32 VR512:$src1),
-                    (v16i32 VR512:$src2), (v16i32 immAllZerosV), (i16 -1))),
-           (VPMINUDZrr VR512:$src1, VR512:$src2)>;
-def : Pat <(v8i64 (int_x86_avx512_mask_pmins_q_512 (v8i64 VR512:$src1),
-                (v8i64 VR512:$src2), (bc_v8i64 (v16i32 immAllZerosV)), (i8 -1))),
-           (VPMINSQZrr VR512:$src1, VR512:$src2)>;
-def : Pat <(v8i64 (int_x86_avx512_mask_pminu_q_512 (v8i64 VR512:$src1),
-                (v8i64 VR512:$src2), (bc_v8i64 (v16i32 immAllZerosV)), (i8 -1))),
-           (VPMINUQZrr VR512:$src1, VR512:$src2)>;
-//===----------------------------------------------------------------------===//
-// AVX-512 - Unpack Instructions
-//===----------------------------------------------------------------------===//
-
-multiclass avx512_unpack_fp<bits<8> opc, SDNode OpNode, ValueType vt,
-                                   PatFrag mem_frag, RegisterClass RC,
-                                   X86MemOperand x86memop, string asm,
-                                   Domain d> {
-    def rr : AVX512PI<opc, MRMSrcReg,
-                (outs RC:$dst), (ins RC:$src1, RC:$src2),
-                asm, [(set RC:$dst,
-                           (vt (OpNode RC:$src1, RC:$src2)))],
-                           d>, EVEX_4V;
-    def rm : AVX512PI<opc, MRMSrcMem,
-                (outs RC:$dst), (ins RC:$src1, x86memop:$src2),
-                asm, [(set RC:$dst,
-                       (vt (OpNode RC:$src1,
-                            (bitconvert (mem_frag addr:$src2)))))],
-                        d>, EVEX_4V;
-}
-
-defm VUNPCKHPSZ: avx512_unpack_fp<0x15, X86Unpckh, v16f32, loadv8f64,
-      VR512, f512mem, "vunpckhps\t{$src2, $src1, $dst|$dst, $src1, $src2}",
-      SSEPackedSingle>, PS, EVEX_V512, EVEX_CD8<32, CD8VF>;
-defm VUNPCKHPDZ: avx512_unpack_fp<0x15, X86Unpckh, v8f64, loadv8f64,
-      VR512, f512mem, "vunpckhpd\t{$src2, $src1, $dst|$dst, $src1, $src2}",
-      SSEPackedDouble>, PD, EVEX_V512, VEX_W, EVEX_CD8<64, CD8VF>;
-defm VUNPCKLPSZ: avx512_unpack_fp<0x14, X86Unpckl, v16f32, loadv8f64,
-      VR512, f512mem, "vunpcklps\t{$src2, $src1, $dst|$dst, $src1, $src2}",
-      SSEPackedSingle>, PS, EVEX_V512, EVEX_CD8<32, CD8VF>;
-defm VUNPCKLPDZ: avx512_unpack_fp<0x14, X86Unpckl, v8f64, loadv8f64,
-      VR512, f512mem, "vunpcklpd\t{$src2, $src1, $dst|$dst, $src1, $src2}",
-      SSEPackedDouble>, PD, EVEX_V512, VEX_W, EVEX_CD8<64, CD8VF>;
-
-multiclass avx512_unpack_int<bits<8> opc, string OpcodeStr, SDNode OpNode,
-                        ValueType OpVT, RegisterClass RC, PatFrag memop_frag,
-                        X86MemOperand x86memop> {
-  def rr : AVX512BI<opc, MRMSrcReg, (outs RC:$dst),
-       (ins RC:$src1, RC:$src2),
-       !strconcat(OpcodeStr, "\t{$src2, $src1, $dst|$dst, $src1, $src2}"),
-       [(set RC:$dst, (OpVT (OpNode (OpVT RC:$src1), (OpVT RC:$src2))))],
-       IIC_SSE_UNPCK>, EVEX_4V;
-  def rm : AVX512BI<opc, MRMSrcMem, (outs RC:$dst),
-       (ins RC:$src1, x86memop:$src2),
-       !strconcat(OpcodeStr, "\t{$src2, $src1, $dst|$dst, $src1, $src2}"),
-       [(set RC:$dst, (OpVT (OpNode (OpVT RC:$src1),
-                                     (bitconvert (memop_frag addr:$src2)))))],
-                                     IIC_SSE_UNPCK>, EVEX_4V;
-}
-defm VPUNPCKLDQZ  : avx512_unpack_int<0x62, "vpunpckldq", X86Unpckl, v16i32,
-                                VR512, loadv16i32, i512mem>, EVEX_V512,
-                                EVEX_CD8<32, CD8VF>;
-defm VPUNPCKLQDQZ : avx512_unpack_int<0x6C, "vpunpcklqdq", X86Unpckl, v8i64,
-                                VR512, loadv8i64, i512mem>, EVEX_V512,
-                                VEX_W, EVEX_CD8<64, CD8VF>;
-defm VPUNPCKHDQZ  : avx512_unpack_int<0x6A, "vpunpckhdq", X86Unpckh, v16i32,
-                                VR512, loadv16i32, i512mem>, EVEX_V512,
-                                EVEX_CD8<32, CD8VF>;
-defm VPUNPCKHQDQZ : avx512_unpack_int<0x6D, "vpunpckhqdq", X86Unpckh, v8i64,
-                                VR512, loadv8i64, i512mem>, EVEX_V512,
-                                VEX_W, EVEX_CD8<64, CD8VF>;
-//===----------------------------------------------------------------------===//
-// AVX-512 - PSHUFD
-//
-
-multiclass avx512_pshuf_imm<bits<8> opc, string OpcodeStr, RegisterClass RC,
-                         SDNode OpNode, PatFrag mem_frag,
-                         X86MemOperand x86memop, ValueType OpVT> {
-  def ri : AVX512Ii8<opc, MRMSrcReg, (outs RC:$dst),
-                     (ins RC:$src1, u8imm:$src2),
-                     !strconcat(OpcodeStr,
-                         "\t{$src2, $src1, $dst|$dst, $src1, $src2}"),
-                     [(set RC:$dst,
-                       (OpVT (OpNode RC:$src1, (i8 imm:$src2))))]>,
-                     EVEX;
-  def mi : AVX512Ii8<opc, MRMSrcMem, (outs RC:$dst),
-                     (ins x86memop:$src1, u8imm:$src2),
-                     !strconcat(OpcodeStr,
-                         "\t{$src2, $src1, $dst|$dst, $src1, $src2}"),
-                     [(set RC:$dst,
-                       (OpVT (OpNode (mem_frag addr:$src1),
-                              (i8 imm:$src2))))]>, EVEX;
-}
-
-defm VPSHUFDZ : avx512_pshuf_imm<0x70, "vpshufd", VR512, X86PShufd, loadv16i32,
-                      i512mem, v16i32>, PD, EVEX_V512, EVEX_CD8<32, CD8VF>;
-
 //===----------------------------------------------------------------------===//
 // AVX-512  Logical Instructions
 //===----------------------------------------------------------------------===//
@@ -3187,7 +3307,7 @@ multiclass avx512_fp_scalar<bits<8> opc, string OpcodeStr,X86VectorVTInfo _,
                            "$src2, $src1", "$src1, $src2",
                            (VecNode (_.VT _.RC:$src1), (_.VT _.RC:$src2),
                            (i32 FROUND_CURRENT)),
-                           "", itins.rr, IsCommutable>;
+                           itins.rr, IsCommutable>;
 
   defm rm_Int : AVX512_maskable_scalar<opc, MRMSrcMem, _, (outs _.RC:$dst),
                          (ins _.RC:$src1, _.MemOp:$src2), OpcodeStr,
@@ -3195,7 +3315,7 @@ multiclass avx512_fp_scalar<bits<8> opc, string OpcodeStr,X86VectorVTInfo _,
                          (VecNode (_.VT _.RC:$src1),
                           (_.VT (scalar_to_vector (_.ScalarLdFrag addr:$src2))),
                            (i32 FROUND_CURRENT)),
-                         "", itins.rm, IsCommutable>;
+                         itins.rm, IsCommutable>;
   let isCodeGenOnly = 1, isCommutable = IsCommutable,
       Predicates = [HasAVX512] in {
   def rr : I< opc, MRMSrcReg, (outs _.FRC:$dst),
@@ -3212,13 +3332,13 @@ multiclass avx512_fp_scalar<bits<8> opc, string OpcodeStr,X86VectorVTInfo _,
 }
 
 multiclass avx512_fp_scalar_round<bits<8> opc, string OpcodeStr,X86VectorVTInfo _,
-                         SDNode VecNode, OpndItins itins, bit IsCommutable> {
+                         SDNode VecNode, OpndItins itins, bit IsCommutable = 0> {
 
   defm rrb : AVX512_maskable_scalar<opc, MRMSrcReg, _, (outs _.RC:$dst),
                           (ins _.RC:$src1, _.RC:$src2, AVX512RC:$rc), OpcodeStr,
                           "$rc, $src2, $src1", "$src1, $src2, $rc",
                           (VecNode (_.VT _.RC:$src1), (_.VT _.RC:$src2),
-                          (i32 imm:$rc)), "", itins.rr, IsCommutable>,
+                          (i32 imm:$rc)), itins.rr, IsCommutable>,
                           EVEX_B, EVEX_RC;
 }
 multiclass avx512_fp_scalar_sae<bits<8> opc, string OpcodeStr,X86VectorVTInfo _,
@@ -3226,9 +3346,9 @@ multiclass avx512_fp_scalar_sae<bits<8> opc, string OpcodeStr,X86VectorVTInfo _,
 
   defm rrb : AVX512_maskable_scalar<opc, MRMSrcReg, _, (outs _.RC:$dst),
                             (ins _.RC:$src1, _.RC:$src2), OpcodeStr,
-                            "$src2, $src1", "$src1, $src2",
+                            "{sae}, $src2, $src1", "$src1, $src2, {sae}",
                             (VecNode (_.VT _.RC:$src1), (_.VT _.RC:$src2),
-                            (i32 FROUND_NO_EXC)), "{sae}">, EVEX_B;
+                            (i32 FROUND_NO_EXC))>, EVEX_B;
 }
 
 multiclass avx512_binop_s_round<bits<8> opc, string OpcodeStr, SDNode OpNode,
@@ -3289,7 +3409,7 @@ multiclass avx512_fp_packed<bits<8> opc, string OpcodeStr, SDNode OpNode,
 }
 
 multiclass avx512_fp_round_packed<bits<8> opc, string OpcodeStr, SDNode OpNodeRnd,
-                            X86VectorVTInfo _, bit IsCommutable> {
+                            X86VectorVTInfo _> {
   defm rb: AVX512_maskable<opc, MRMSrcReg, _, (outs _.RC:$dst),
                   (ins _.RC:$src1, _.RC:$src2, AVX512RC:$rc), OpcodeStr##_.Suffix,
                   "$rc, $src2, $src1", "$src1, $src2, $rc",
@@ -3297,6 +3417,16 @@ multiclass avx512_fp_round_packed<bits<8> opc, string OpcodeStr, SDNode OpNodeRn
                   EVEX_4V, EVEX_B, EVEX_RC;
 }
 
+
+multiclass avx512_fp_sae_packed<bits<8> opc, string OpcodeStr, SDNode OpNodeRnd,
+                            X86VectorVTInfo _> {
+  defm rb: AVX512_maskable<opc, MRMSrcReg, _, (outs _.RC:$dst),
+                  (ins _.RC:$src1, _.RC:$src2), OpcodeStr##_.Suffix,
+                  "{sae}, $src2, $src1", "$src1, $src2, {sae}",
+                  (_.VT (OpNodeRnd _.RC:$src1, _.RC:$src2, (i32 FROUND_NO_EXC)))>,
+                  EVEX_4V, EVEX_B;
+}
+
 multiclass avx512_fp_binop_p<bits<8> opc, string OpcodeStr, SDNode OpNode, 
                              bit IsCommutable = 0> {
   defm PSZ : avx512_fp_packed<opc, OpcodeStr, OpNode, v16f32_info,
@@ -3324,9 +3454,16 @@ multiclass avx512_fp_binop_p<bits<8> opc, string OpcodeStr, SDNode OpNode,
 }
 
 multiclass avx512_fp_binop_p_round<bits<8> opc, string OpcodeStr, SDNode OpNodeRnd> {
-  defm PSZ : avx512_fp_round_packed<opc, OpcodeStr, OpNodeRnd, v16f32_info, 0>,
+  defm PSZ : avx512_fp_round_packed<opc, OpcodeStr, OpNodeRnd, v16f32_info>,
                               EVEX_V512, PS, EVEX_CD8<32, CD8VF>;
-  defm PDZ : avx512_fp_round_packed<opc, OpcodeStr, OpNodeRnd, v8f64_info, 0>,
+  defm PDZ : avx512_fp_round_packed<opc, OpcodeStr, OpNodeRnd, v8f64_info>,
+                              EVEX_V512, PD, VEX_W,EVEX_CD8<64, CD8VF>;
+}
+
+multiclass avx512_fp_binop_p_sae<bits<8> opc, string OpcodeStr, SDNode OpNodeRnd> {
+  defm PSZ : avx512_fp_sae_packed<opc, OpcodeStr, OpNodeRnd, v16f32_info>,
+                              EVEX_V512, PS, EVEX_CD8<32, CD8VF>;
+  defm PDZ : avx512_fp_sae_packed<opc, OpcodeStr, OpNodeRnd, v8f64_info>,
                               EVEX_V512, PD, VEX_W,EVEX_CD8<64, CD8VF>;
 }
 
@@ -3338,73 +3475,172 @@ defm VSUB : avx512_fp_binop_p<0x5C, "vsub", fsub>,
             avx512_fp_binop_p_round<0x5C, "vsub", X86fsubRnd>;
 defm VDIV : avx512_fp_binop_p<0x5E, "vdiv", fdiv>,
             avx512_fp_binop_p_round<0x5E, "vdiv", X86fdivRnd>;
-defm VMIN : avx512_fp_binop_p<0x5D, "vmin", X86fmin, 1>;
-defm VMAX : avx512_fp_binop_p<0x5F, "vmax", X86fmax, 1>;
-
-def : Pat<(v16f32 (int_x86_avx512_mask_max_ps_512 (v16f32 VR512:$src1),
-                   (v16f32 VR512:$src2), (bc_v16f32 (v16i32 immAllZerosV)),
-                   (i16 -1), FROUND_CURRENT)),
-          (VMAXPSZrr VR512:$src1, VR512:$src2)>;
-
-def : Pat<(v8f64 (int_x86_avx512_mask_max_pd_512 (v8f64 VR512:$src1),
-                   (v8f64 VR512:$src2), (bc_v8f64 (v16i32 immAllZerosV)),
-                   (i8 -1), FROUND_CURRENT)),
-          (VMAXPDZrr VR512:$src1, VR512:$src2)>;
-
-def : Pat<(v16f32 (int_x86_avx512_mask_min_ps_512 (v16f32 VR512:$src1),
-                   (v16f32 VR512:$src2), (bc_v16f32 (v16i32 immAllZerosV)),
-                   (i16 -1), FROUND_CURRENT)),
-          (VMINPSZrr VR512:$src1, VR512:$src2)>;
-
-def : Pat<(v8f64 (int_x86_avx512_mask_min_pd_512 (v8f64 VR512:$src1),
-                   (v8f64 VR512:$src2), (bc_v8f64 (v16i32 immAllZerosV)),
-                   (i8 -1), FROUND_CURRENT)),
-          (VMINPDZrr VR512:$src1, VR512:$src2)>;
-//===----------------------------------------------------------------------===//
-// AVX-512  VPTESTM instructions
-//===----------------------------------------------------------------------===//
-
-multiclass avx512_vptest<bits<8> opc, string OpcodeStr, RegisterClass KRC,
-              RegisterClass RC, X86MemOperand x86memop, PatFrag memop_frag,
-              SDNode OpNode, ValueType vt> {
-  def rr : AVX512PI<opc, MRMSrcReg,
-             (outs KRC:$dst), (ins RC:$src1, RC:$src2),
-             !strconcat(OpcodeStr, "\t{$src2, $src1, $dst|$dst, $src1, $src2}"),
-             [(set KRC:$dst, (OpNode (vt RC:$src1), (vt RC:$src2)))],
-             SSEPackedInt>, EVEX_4V;
-  def rm : AVX512PI<opc, MRMSrcMem,
-             (outs KRC:$dst), (ins RC:$src1, x86memop:$src2),
-             !strconcat(OpcodeStr, "\t{$src2, $src1, $dst|$dst, $src1, $src2}"),
-             [(set KRC:$dst, (OpNode (vt RC:$src1),
-              (bitconvert (memop_frag addr:$src2))))], SSEPackedInt>, EVEX_4V;
+defm VMIN : avx512_fp_binop_p<0x5D, "vmin", X86fmin, 1>,
+            avx512_fp_binop_p_sae<0x5D, "vmin", X86fminRnd>;
+defm VMAX : avx512_fp_binop_p<0x5F, "vmax", X86fmax, 1>,
+            avx512_fp_binop_p_sae<0x5F, "vmax", X86fmaxRnd>;
+let Predicates = [HasDQI] in {
+  defm VAND  : avx512_fp_binop_p<0x54, "vand", X86fand, 1>;
+  defm VANDN : avx512_fp_binop_p<0x55, "vandn", X86fandn, 0>;
+  defm VOR   : avx512_fp_binop_p<0x56, "vor", X86for, 1>;
+  defm VXOR  : avx512_fp_binop_p<0x57, "vxor", X86fxor, 1>;
 }
 
-defm VPTESTMDZ  : avx512_vptest<0x27, "vptestmd", VK16, VR512,  f512mem,
-                              loadv16i32, X86testm, v16i32>, T8PD, EVEX_V512,
-                              EVEX_CD8<32, CD8VF>;
-defm VPTESTMQZ  : avx512_vptest<0x27, "vptestmq", VK8, VR512,  f512mem,
-                              loadv8i64, X86testm, v8i64>, T8PD, EVEX_V512, VEX_W,
-                              EVEX_CD8<64, CD8VF>;
-
-let Predicates = [HasCDI] in {
-defm VPTESTNMDZ  : avx512_vptest<0x27, "vptestnmd", VK16, VR512,  f512mem,
-                              loadv16i32, X86testnm, v16i32>, T8XS, EVEX_V512,
-                              EVEX_CD8<32, CD8VF>;
-defm VPTESTNMQZ  : avx512_vptest<0x27, "vptestnmq", VK8, VR512,  f512mem,
-                              loadv8i64, X86testnm, v8i64>, T8XS, EVEX_V512, VEX_W,
-                              EVEX_CD8<64, CD8VF>;
+multiclass avx512_fp_scalef_p<bits<8> opc, string OpcodeStr, SDNode OpNode,
+                            X86VectorVTInfo _> {
+  defm rr: AVX512_maskable<opc, MRMSrcReg, _, (outs _.RC:$dst),
+                  (ins _.RC:$src1, _.RC:$src2), OpcodeStr##_.Suffix,
+                  "$src2, $src1", "$src1, $src2",
+                  (_.VT (OpNode _.RC:$src1, _.RC:$src2, (i32 FROUND_CURRENT)))>, EVEX_4V;
+  let mayLoad = 1 in {
+    defm rm: AVX512_maskable<opc, MRMSrcMem, _, (outs _.RC:$dst),
+                    (ins _.RC:$src1, _.MemOp:$src2), OpcodeStr##_.Suffix,
+                    "$src2, $src1", "$src1, $src2",
+                    (OpNode _.RC:$src1, (_.LdFrag addr:$src2), (i32 FROUND_CURRENT))>, EVEX_4V;
+    defm rmb: AVX512_maskable<opc, MRMSrcMem, _, (outs _.RC:$dst),
+                     (ins _.RC:$src1, _.ScalarMemOp:$src2), OpcodeStr##_.Suffix,
+                     "${src2}"##_.BroadcastStr##", $src1",
+                     "$src1, ${src2}"##_.BroadcastStr,
+                     (OpNode  _.RC:$src1, (_.VT (X86VBroadcast
+                                                (_.ScalarLdFrag addr:$src2))), (i32 FROUND_CURRENT))>,
+                     EVEX_4V, EVEX_B;
+  }//let mayLoad = 1
 }
 
-def : Pat <(i16 (int_x86_avx512_mask_ptestm_d_512 (v16i32 VR512:$src1),
-                 (v16i32 VR512:$src2), (i16 -1))),
-                 (COPY_TO_REGCLASS (VPTESTMDZrr VR512:$src1, VR512:$src2), GR16)>;
+multiclass avx512_fp_scalef_scalar<bits<8> opc, string OpcodeStr, SDNode OpNode,
+                            X86VectorVTInfo _> {
+  defm rr: AVX512_maskable_scalar<opc, MRMSrcReg, _, (outs _.RC:$dst),
+                  (ins _.RC:$src1, _.RC:$src2), OpcodeStr##_.Suffix,
+                  "$src2, $src1", "$src1, $src2",
+                  (_.VT (OpNode _.RC:$src1, _.RC:$src2, (i32 FROUND_CURRENT)))>;
+  let mayLoad = 1 in {
+    defm rm: AVX512_maskable_scalar<opc, MRMSrcMem, _, (outs _.RC:$dst),
+                    (ins _.RC:$src1, _.MemOp:$src2), OpcodeStr##_.Suffix,
+                    "$src2, $src1", "$src1, $src2",
+                    (OpNode _.RC:$src1, (_.LdFrag addr:$src2), (i32 FROUND_CURRENT))>;
+  }//let mayLoad = 1
+}
 
-def : Pat <(i8 (int_x86_avx512_mask_ptestm_q_512 (v8i64 VR512:$src1),
-                 (v8i64 VR512:$src2), (i8 -1))),
-                 (COPY_TO_REGCLASS (VPTESTMQZrr VR512:$src1, VR512:$src2), GR8)>;
+multiclass avx512_fp_scalef_all<bits<8> opc, bits<8> opcScaler, string OpcodeStr, SDNode OpNode> {
+  defm PSZ : avx512_fp_scalef_p<opc, OpcodeStr, OpNode, v16f32_info>, 
+             avx512_fp_round_packed<opc, OpcodeStr, OpNode, v16f32_info>,
+                              EVEX_V512, EVEX_CD8<32, CD8VF>;
+  defm PDZ : avx512_fp_scalef_p<opc, OpcodeStr, OpNode, v8f64_info>, 
+             avx512_fp_round_packed<opc, OpcodeStr, OpNode, v8f64_info>,
+                              EVEX_V512, VEX_W, EVEX_CD8<64, CD8VF>;
+  defm SSZ128 : avx512_fp_scalef_scalar<opcScaler, OpcodeStr, OpNode, f32x_info>,
+                avx512_fp_scalar_round<opcScaler, OpcodeStr##"ss", f32x_info, OpNode, SSE_ALU_ITINS_S.s>,
+                              EVEX_4V,EVEX_CD8<32, CD8VT1>;
+  defm SDZ128 : avx512_fp_scalef_scalar<opcScaler, OpcodeStr, OpNode, f64x_info>,
+                avx512_fp_scalar_round<opcScaler, OpcodeStr##"sd", f64x_info, OpNode, SSE_ALU_ITINS_S.d>,
+                              EVEX_4V, EVEX_CD8<64, CD8VT1>, VEX_W;
 
-//===----------------------------------------------------------------------===//
-// AVX-512  Shift instructions
+  // Define only if AVX512VL feature is present.
+  let Predicates = [HasVLX] in {
+    defm PSZ128 : avx512_fp_scalef_p<opc, OpcodeStr, OpNode, v4f32x_info>,
+                                   EVEX_V128, EVEX_CD8<32, CD8VF>;
+    defm PSZ256 : avx512_fp_scalef_p<opc, OpcodeStr, OpNode, v8f32x_info>,
+                                   EVEX_V256, EVEX_CD8<32, CD8VF>;
+    defm PDZ128 : avx512_fp_scalef_p<opc, OpcodeStr, OpNode, v2f64x_info>,
+                                   EVEX_V128, VEX_W, EVEX_CD8<64, CD8VF>;
+    defm PDZ256 : avx512_fp_scalef_p<opc, OpcodeStr, OpNode, v4f64x_info>,
+                                   EVEX_V256, VEX_W, EVEX_CD8<64, CD8VF>;
+  }
+}
+defm VSCALEF : avx512_fp_scalef_all<0x2C, 0x2D, "vscalef", X86scalef>, T8PD;
+
+//===----------------------------------------------------------------------===//
+// AVX-512  VPTESTM instructions
+//===----------------------------------------------------------------------===//
+
+multiclass avx512_vptest<bits<8> opc, string OpcodeStr, SDNode OpNode,
+                            X86VectorVTInfo _> {
+  defm rr : AVX512_maskable_cmp<opc, MRMSrcReg, _, (outs _.KRC:$dst),
+                   (ins _.RC:$src1, _.RC:$src2), OpcodeStr,
+                      "$src2, $src1", "$src1, $src2",
+                   (OpNode (_.VT _.RC:$src1), (_.VT _.RC:$src2))>,
+                    EVEX_4V;
+  let mayLoad = 1 in
+  defm rm : AVX512_maskable_cmp<opc, MRMSrcMem, _, (outs _.KRC:$dst),
+                   (ins _.RC:$src1, _.MemOp:$src2), OpcodeStr,
+                       "$src2, $src1", "$src1, $src2",
+                   (OpNode (_.VT _.RC:$src1), 
+                    (_.VT (bitconvert (_.LdFrag addr:$src2))))>,
+                    EVEX_4V,
+                   EVEX_CD8<_.EltSize, CD8VF>;
+}
+
+multiclass avx512_vptest_mb<bits<8> opc, string OpcodeStr, SDNode OpNode,
+                            X86VectorVTInfo _> {
+  let mayLoad = 1 in
+  defm rmb : AVX512_maskable_cmp<opc, MRMSrcMem, _, (outs _.KRC:$dst),
+                    (ins _.RC:$src1, _.ScalarMemOp:$src2), OpcodeStr,
+                    "${src2}"##_.BroadcastStr##", $src1",
+                    "$src1, ${src2}"##_.BroadcastStr,
+                    (OpNode (_.VT _.RC:$src1), (_.VT (X86VBroadcast
+                                                (_.ScalarLdFrag addr:$src2))))>,
+                    EVEX_B, EVEX_4V, EVEX_CD8<_.EltSize, CD8VF>;
+}
+multiclass avx512_vptest_dq_sizes<bits<8> opc, string OpcodeStr, SDNode OpNode,
+                                  AVX512VLVectorVTInfo _> {
+  let Predicates  = [HasAVX512] in
+  defm Z : avx512_vptest<opc, OpcodeStr, OpNode, _.info512>,
+           avx512_vptest_mb<opc, OpcodeStr, OpNode, _.info512>, EVEX_V512;
+
+  let Predicates = [HasAVX512, HasVLX] in {
+  defm Z256 : avx512_vptest<opc, OpcodeStr, OpNode, _.info256>,
+              avx512_vptest_mb<opc, OpcodeStr, OpNode, _.info256>, EVEX_V256;
+  defm Z128 : avx512_vptest<opc, OpcodeStr, OpNode, _.info128>,
+              avx512_vptest_mb<opc, OpcodeStr, OpNode, _.info128>, EVEX_V128;
+  }
+}
+
+multiclass avx512_vptest_dq<bits<8> opc, string OpcodeStr, SDNode OpNode> {
+  defm D : avx512_vptest_dq_sizes<opc, OpcodeStr#"d", OpNode,
+                                 avx512vl_i32_info>;
+  defm Q : avx512_vptest_dq_sizes<opc, OpcodeStr#"q", OpNode,
+                                 avx512vl_i64_info>, VEX_W;
+}
+
+multiclass avx512_vptest_wb<bits<8> opc, string OpcodeStr,
+                                 SDNode OpNode> {
+  let Predicates = [HasBWI] in {
+  defm WZ:    avx512_vptest<opc, OpcodeStr#"w", OpNode, v32i16_info>,
+              EVEX_V512, VEX_W;
+  defm BZ:    avx512_vptest<opc, OpcodeStr#"b", OpNode, v64i8_info>,
+              EVEX_V512;
+  }
+  let Predicates = [HasVLX, HasBWI] in {
+
+  defm WZ256: avx512_vptest<opc, OpcodeStr#"w", OpNode, v16i16x_info>,
+              EVEX_V256, VEX_W;
+  defm WZ128: avx512_vptest<opc, OpcodeStr#"w", OpNode, v8i16x_info>,
+              EVEX_V128, VEX_W;
+  defm BZ256: avx512_vptest<opc, OpcodeStr#"b", OpNode, v32i8x_info>,
+              EVEX_V256;
+  defm BZ128: avx512_vptest<opc, OpcodeStr#"b", OpNode, v16i8x_info>,
+              EVEX_V128;
+  }
+}
+
+multiclass avx512_vptest_all_forms<bits<8> opc_wb, bits<8> opc_dq, string OpcodeStr,
+                                   SDNode OpNode> :
+  avx512_vptest_wb <opc_wb, OpcodeStr, OpNode>,
+  avx512_vptest_dq<opc_dq, OpcodeStr, OpNode>;
+
+defm VPTESTM   : avx512_vptest_all_forms<0x26, 0x27, "vptestm", X86testm>, T8PD;
+defm VPTESTNM  : avx512_vptest_all_forms<0x26, 0x27, "vptestnm", X86testnm>, T8XS;
+
+def : Pat <(i16 (int_x86_avx512_mask_ptestm_d_512 (v16i32 VR512:$src1),
+                 (v16i32 VR512:$src2), (i16 -1))),
+                 (COPY_TO_REGCLASS (VPTESTMDZrr VR512:$src1, VR512:$src2), GR16)>;
+
+def : Pat <(i8 (int_x86_avx512_mask_ptestm_q_512 (v8i64 VR512:$src1),
+                 (v8i64 VR512:$src2), (i8 -1))),
+                 (COPY_TO_REGCLASS (VPTESTMQZrr VR512:$src1, VR512:$src2), GR8)>;
+
+//===----------------------------------------------------------------------===//
+// AVX-512  Shift instructions
 //===----------------------------------------------------------------------===//
 multiclass avx512_shift_rmi<bits<8> opc, Format ImmFormR, Format ImmFormM,
                          string OpcodeStr, SDNode OpNode, X86VectorVTInfo _> {
@@ -3412,14 +3648,14 @@ multiclass avx512_shift_rmi<bits<8> opc, Format ImmFormR, Format ImmFormM,
                    (ins _.RC:$src1, u8imm:$src2), OpcodeStr,
                       "$src2, $src1", "$src1, $src2",
                    (_.VT (OpNode _.RC:$src1, (i8 imm:$src2))),
-                   " ",  SSE_INTSHIFT_ITINS_P.rr>, AVX512BIi8Base, EVEX_4V;
+                   SSE_INTSHIFT_ITINS_P.rr>;
   let mayLoad = 1 in
   defm mi : AVX512_maskable<opc, ImmFormM, _, (outs _.RC:$dst),
                    (ins _.MemOp:$src1, u8imm:$src2), OpcodeStr,
                        "$src2, $src1", "$src1, $src2",
                    (_.VT (OpNode (_.VT (bitconvert (_.LdFrag addr:$src1))),
                           (i8 imm:$src2))),
-                   " ",  SSE_INTSHIFT_ITINS_P.rm>, AVX512BIi8Base, EVEX_4V;
+                   SSE_INTSHIFT_ITINS_P.rm>;
 }
 
 multiclass avx512_shift_rmbi<bits<8> opc, Format ImmFormM,
@@ -3429,7 +3665,7 @@ multiclass avx512_shift_rmbi<bits<8> opc, Format ImmFormM,
                    (ins _.ScalarMemOp:$src1, u8imm:$src2), OpcodeStr,
       "$src2, ${src1}"##_.BroadcastStr, "${src1}"##_.BroadcastStr##", $src2",
      (_.VT (OpNode (X86VBroadcast (_.ScalarLdFrag addr:$src1)), (i8 imm:$src2))),
-     " ",  SSE_INTSHIFT_ITINS_P.rm>, AVX512BIi8Base, EVEX_4V, EVEX_B;
+     SSE_INTSHIFT_ITINS_P.rm>, EVEX_B;
 }
 
 multiclass avx512_shift_rrm<bits<8> opc, string OpcodeStr, SDNode OpNode,
@@ -3439,12 +3675,12 @@ multiclass avx512_shift_rrm<bits<8> opc, string OpcodeStr, SDNode OpNode,
                    (ins _.RC:$src1, VR128X:$src2), OpcodeStr,
                       "$src2, $src1", "$src1, $src2",
                    (_.VT (OpNode _.RC:$src1, (SrcVT VR128X:$src2))),
-                   " ",  SSE_INTSHIFT_ITINS_P.rr>, AVX512BIBase, EVEX_4V;
+                   SSE_INTSHIFT_ITINS_P.rr>, AVX512BIBase, EVEX_4V;
   defm rm : AVX512_maskable<opc, MRMSrcMem, _, (outs _.RC:$dst),
                    (ins _.RC:$src1, i128mem:$src2), OpcodeStr,
                        "$src2, $src1", "$src1, $src2",
                    (_.VT (OpNode _.RC:$src1, (bc_frag (loadv2i64 addr:$src2)))),
-                   " ",  SSE_INTSHIFT_ITINS_P.rm>, AVX512BIBase,
+                   SSE_INTSHIFT_ITINS_P.rm>, AVX512BIBase,
                    EVEX_4V;
 }
 
@@ -3519,16 +3755,16 @@ multiclass avx512_shift_rmi_dq<bits<8> opcd, bits<8> opcq,
 }
 
 defm VPSRL : avx512_shift_rmi_dq<0x72, 0x73, MRM2r, MRM2m, "vpsrl", X86vsrli>,
-             avx512_shift_rmi_w<0x71, MRM2r, MRM2m, "vpsrlw", X86vsrli>;
+             avx512_shift_rmi_w<0x71, MRM2r, MRM2m, "vpsrlw", X86vsrli>, AVX512BIi8Base, EVEX_4V;
 
 defm VPSLL : avx512_shift_rmi_dq<0x72, 0x73, MRM6r, MRM6m, "vpsll", X86vshli>,
-             avx512_shift_rmi_w<0x71, MRM6r, MRM6m, "vpsllw", X86vshli>;
+             avx512_shift_rmi_w<0x71, MRM6r, MRM6m, "vpsllw", X86vshli>, AVX512BIi8Base, EVEX_4V;
 
-defm VPSRA : avx512_shift_rmi_dq<0x72, 0x73, MRM4r, MRM4m, "vpsra", X86vsrai>,
-             avx512_shift_rmi_w<0x71, MRM4r, MRM4m, "vpsraw", X86vsrai>;
+defm VPSRA : avx512_shift_rmi_dq<0x72, 0x72, MRM4r, MRM4m, "vpsra", X86vsrai>,
+             avx512_shift_rmi_w<0x71, MRM4r, MRM4m, "vpsraw", X86vsrai>, AVX512BIi8Base, EVEX_4V;
 
-defm VPROR : avx512_shift_rmi_dq<0x72, 0x72, MRM0r, MRM0m, "vpror", rotr>;
-defm VPROL : avx512_shift_rmi_dq<0x72, 0x72, MRM1r, MRM1m, "vprol", rotl>;
+defm VPROR : avx512_shift_rmi_dq<0x72, 0x72, MRM0r, MRM0m, "vpror", rotr>, AVX512BIi8Base, EVEX_4V;
+defm VPROL : avx512_shift_rmi_dq<0x72, 0x72, MRM1r, MRM1m, "vprol", rotl>, AVX512BIi8Base, EVEX_4V;
 
 defm VPSLL : avx512_shift_types<0xF2, 0xF3, 0xF1, "vpsll", X86vshl>;
 defm VPSRA : avx512_shift_types<0xE2, 0xE2, 0xE1, "vpsra", X86vsra>;
@@ -3543,13 +3779,14 @@ multiclass avx512_var_shift<bits<8> opc, string OpcodeStr, SDNode OpNode,
                    (ins _.RC:$src1, _.RC:$src2), OpcodeStr,
                       "$src2, $src1", "$src1, $src2",
                    (_.VT (OpNode _.RC:$src1, (_.VT _.RC:$src2))),
-                   " ",  SSE_INTSHIFT_ITINS_P.rr>, AVX5128IBase, EVEX_4V;
+                   SSE_INTSHIFT_ITINS_P.rr>, AVX5128IBase, EVEX_4V;
   let mayLoad = 1 in
   defm rm : AVX512_maskable<opc, MRMSrcMem, _, (outs _.RC:$dst),
                    (ins _.RC:$src1, _.MemOp:$src2), OpcodeStr,
                        "$src2, $src1", "$src1, $src2",
-                   (_.VT (OpNode _.RC:$src1, (_.LdFrag addr:$src2))),
-                   " ",  SSE_INTSHIFT_ITINS_P.rm>, AVX5128IBase, EVEX_4V,
+                   (_.VT (OpNode _.RC:$src1,
+                   (_.VT (bitconvert (_.LdFrag addr:$src2))))),
+                   SSE_INTSHIFT_ITINS_P.rm>, AVX5128IBase, EVEX_4V,
                    EVEX_CD8<_.EltSize, CD8VF>;
 }
 
@@ -3562,7 +3799,7 @@ multiclass avx512_var_shift_mb<bits<8> opc, string OpcodeStr, SDNode OpNode,
                     "$src1, ${src2}"##_.BroadcastStr,
                     (_.VT (OpNode _.RC:$src1, (_.VT (X86VBroadcast
                                                 (_.ScalarLdFrag addr:$src2))))),
-                    " ",  SSE_INTSHIFT_ITINS_P.rm>, AVX5128IBase, EVEX_B,
+                    SSE_INTSHIFT_ITINS_P.rm>, AVX5128IBase, EVEX_B,
                     EVEX_4V, EVEX_CD8<_.EltSize, CD8VF>;
 }
 multiclass avx512_var_shift_sizes<bits<8> opc, string OpcodeStr, SDNode OpNode,
@@ -3610,6 +3847,78 @@ defm VPSRLV : avx512_var_shift_types<0x45, "vpsrlv", srl>,
 defm VPRORV : avx512_var_shift_types<0x14, "vprorv", rotr>;
 defm VPROLV : avx512_var_shift_types<0x15, "vprolv", rotl>;
 
+//===-------------------------------------------------------------------===//
+// 1-src variable permutation VPERMW/D/Q
+//===-------------------------------------------------------------------===//
+multiclass avx512_vperm_dq_sizes<bits<8> opc, string OpcodeStr, SDNode OpNode,
+                                  AVX512VLVectorVTInfo _> {
+  let Predicates  = [HasAVX512] in
+  defm Z : avx512_var_shift<opc, OpcodeStr, OpNode, _.info512>,
+           avx512_var_shift_mb<opc, OpcodeStr, OpNode, _.info512>, EVEX_V512;
+
+  let Predicates = [HasAVX512, HasVLX] in
+  defm Z256 : avx512_var_shift<opc, OpcodeStr, OpNode, _.info256>,
+              avx512_var_shift_mb<opc, OpcodeStr, OpNode, _.info256>, EVEX_V256;
+}
+
+multiclass avx512_vpermi_dq_sizes<bits<8> opc, Format ImmFormR, Format ImmFormM,
+                                 string OpcodeStr, SDNode OpNode,
+                                 AVX512VLVectorVTInfo VTInfo> {
+  let Predicates = [HasAVX512] in
+  defm Z:    avx512_shift_rmi<opc, ImmFormR, ImmFormM, OpcodeStr, OpNode,
+                              VTInfo.info512>,
+             avx512_shift_rmbi<opc, ImmFormM, OpcodeStr, OpNode,
+                              VTInfo.info512>, EVEX_V512;
+  let Predicates = [HasAVX512, HasVLX] in
+  defm Z256: avx512_shift_rmi<opc, ImmFormR, ImmFormM, OpcodeStr, OpNode,
+                              VTInfo.info256>,
+             avx512_shift_rmbi<opc, ImmFormM, OpcodeStr, OpNode,
+                              VTInfo.info256>, EVEX_V256;
+}
+
+
+defm VPERM  : avx512_var_shift_w<0x8D, "vpermw", X86VPermv>;
+
+defm VPERMD : avx512_vperm_dq_sizes<0x36, "vpermd", X86VPermv,
+                                    avx512vl_i32_info>;
+defm VPERMQ : avx512_vperm_dq_sizes<0x36, "vpermq", X86VPermv,
+                                    avx512vl_i64_info>, VEX_W;
+defm VPERMPS : avx512_vperm_dq_sizes<0x16, "vpermps", X86VPermv,
+                                    avx512vl_f32_info>;
+defm VPERMPD : avx512_vperm_dq_sizes<0x16, "vpermpd", X86VPermv,
+                                    avx512vl_f64_info>, VEX_W;
+
+defm VPERMQ : avx512_vpermi_dq_sizes<0x00, MRMSrcReg, MRMSrcMem, "vpermq",
+                             X86VPermi, avx512vl_i64_info>,
+                             EVEX, AVX512AIi8Base, EVEX_CD8<64, CD8VF>, VEX_W;
+defm VPERMPD : avx512_vpermi_dq_sizes<0x01, MRMSrcReg, MRMSrcMem, "vpermpd",
+                             X86VPermi, avx512vl_f64_info>,
+                             EVEX, AVX512AIi8Base, EVEX_CD8<64, CD8VF>, VEX_W;
+
+//===----------------------------------------------------------------------===//
+// AVX-512 - VPSHUFD, VPSHUFLW, VPSHUFHW
+//===----------------------------------------------------------------------===//
+
+defm VPSHUFD : avx512_shift_rmi_sizes<0x70, MRMSrcReg, MRMSrcMem, "vpshufd",
+                             X86PShufd, avx512vl_i32_info>, 
+                             EVEX, AVX512BIi8Base, EVEX_CD8<32, CD8VF>;
+defm VPSHUFH : avx512_shift_rmi_w<0x70, MRMSrcReg, MRMSrcMem, "vpshufhw",
+                                  X86PShufhw>, EVEX, AVX512XSIi8Base, VEX_W;
+defm VPSHUFL : avx512_shift_rmi_w<0x70, MRMSrcReg, MRMSrcMem, "vpshuflw",
+                                  X86PShuflw>, EVEX, AVX512XDIi8Base, VEX_W;
+                                  
+multiclass avx512_pshufb_sizes<bits<8> opc, string OpcodeStr, SDNode OpNode> {
+  let Predicates = [HasBWI] in
+  defm Z:    avx512_var_shift<opc, OpcodeStr, OpNode, v64i8_info>, EVEX_V512;
+
+  let Predicates = [HasVLX, HasBWI] in {
+  defm Z256: avx512_var_shift<opc, OpcodeStr, OpNode, v32i8x_info>, EVEX_V256;
+  defm Z128: avx512_var_shift<opc, OpcodeStr, OpNode, v16i8x_info>, EVEX_V128;
+  }
+}
+
+defm VPSHUFB: avx512_pshufb_sizes<0x00, "vpshufb", X86pshufb>;
+
 //===----------------------------------------------------------------------===//
 // AVX-512 - MOVDDUP
 //===----------------------------------------------------------------------===//
@@ -3690,216 +3999,366 @@ let Predicates = [HasAVX512] in {
 //
 
 let Constraints = "$src1 = $dst" in {
-// Omitting the parameter OpNode (= null_frag) disables ISel pattern matching.
-multiclass avx512_fma3p_rm<bits<8> opc, string OpcodeStr, X86VectorVTInfo _,
-                           SDPatternOperator OpNode = null_frag> {
+multiclass avx512_fma3p_213_rm<bits<8> opc, string OpcodeStr, SDNode OpNode,
+                                                            X86VectorVTInfo _> {
   defm r: AVX512_maskable_3src<opc, MRMSrcReg, _, (outs _.RC:$dst),
           (ins _.RC:$src2, _.RC:$src3),
           OpcodeStr, "$src3, $src2", "$src2, $src3",
           (_.VT (OpNode _.RC:$src1, _.RC:$src2, _.RC:$src3))>,
          AVX512FMA3Base;
 
-  let mayLoad = 1 in
-  defm m: AVX512_maskable_3src<opc, MRMSrcMem, _, (outs _.RC:$dst),
+  let mayLoad = 1 in {
+    defm m: AVX512_maskable_3src<opc, MRMSrcMem, _, (outs _.RC:$dst),
             (ins _.RC:$src2, _.MemOp:$src3),
             OpcodeStr, "$src3, $src2", "$src2, $src3",
             (_.VT (OpNode _.RC:$src1, _.RC:$src2, (_.LdFrag addr:$src3)))>,
             AVX512FMA3Base; 
 
-  defm mb: AVX512_maskable_3src<opc, MRMSrcMem, _, (outs _.RC:$dst),
+    defm mb: AVX512_maskable_3src<opc, MRMSrcMem, _, (outs _.RC:$dst),
               (ins _.RC:$src2, _.ScalarMemOp:$src3),
               OpcodeStr,   !strconcat("${src3}", _.BroadcastStr,", $src2"),
               !strconcat("$src2, ${src3}", _.BroadcastStr ),
               (OpNode _.RC:$src1,
-               _.RC:$src2,(_.VT (X86VBroadcast (_.ScalarLdFrag addr:$src3))))>,	
+               _.RC:$src2,(_.VT (X86VBroadcast (_.ScalarLdFrag addr:$src3))))>,
               AVX512FMA3Base, EVEX_B;
- }
-} // Constraints = "$src1 = $dst"
+  }
+}
 
-let Constraints = "$src1 = $dst" in {
-// Omitting the parameter OpNode (= null_frag) disables ISel pattern matching.
-multiclass avx512_fma3_round_rrb<bits<8> opc, string OpcodeStr,
-                                 X86VectorVTInfo _,
-                                 SDPatternOperator OpNode> {
-   defm rb: AVX512_maskable_3src<opc, MRMSrcReg, _, (outs _.RC:$dst),
+multiclass avx512_fma3_213_round<bits<8> opc, string OpcodeStr, SDNode OpNode,
+                                                            X86VectorVTInfo _> {
+  defm rb: AVX512_maskable_3src<opc, MRMSrcReg, _, (outs _.RC:$dst),
           (ins _.RC:$src2, _.RC:$src3, AVX512RC:$rc),
           OpcodeStr, "$rc, $src3, $src2", "$src2, $src3, $rc",
           (_.VT ( OpNode _.RC:$src1, _.RC:$src2, _.RC:$src3, (i32 imm:$rc)))>,
           AVX512FMA3Base, EVEX_B, EVEX_RC;
- }
+}
 } // Constraints = "$src1 = $dst"
 
-multiclass avx512_fma3_round_forms<bits<8> opc213, string OpcodeStr,
-                              X86VectorVTInfo VTI, SDPatternOperator OpNode> {
-  defm v213r : avx512_fma3_round_rrb<opc213, !strconcat(OpcodeStr, "213", VTI.Suffix),
-                              VTI, OpNode>, EVEX_CD8<VTI.EltSize, CD8VF>;
+multiclass avx512_fma3p_213_common<bits<8> opc, string OpcodeStr, SDNode OpNode,
+                                     SDNode OpNodeRnd, AVX512VLVectorVTInfo _> {
+  let Predicates = [HasAVX512] in {
+    defm Z      : avx512_fma3p_213_rm<opc, OpcodeStr, OpNode, _.info512>,
+                  avx512_fma3_213_round<opc, OpcodeStr, OpNodeRnd, _.info512>,
+                      EVEX_V512, EVEX_CD8<_.info512.EltSize, CD8VF>;
+  }
+  let Predicates = [HasVLX, HasAVX512] in {
+    defm Z256 : avx512_fma3p_213_rm<opc, OpcodeStr, OpNode, _.info256>,
+                      EVEX_V256, EVEX_CD8<_.info256.EltSize, CD8VF>;
+    defm Z128 : avx512_fma3p_213_rm<opc, OpcodeStr, OpNode, _.info128>,
+                      EVEX_V128, EVEX_CD8<_.info128.EltSize, CD8VF>;
+  }
 }
 
-multiclass avx512_fma3p_forms<bits<8> opc213, bits<8> opc231,
-                              string OpcodeStr, X86VectorVTInfo VTI,
-                              SDPatternOperator OpNode> {
-  defm v213r : avx512_fma3p_rm<opc213, !strconcat(OpcodeStr, "213", VTI.Suffix),
-                              VTI, OpNode>, EVEX_CD8<VTI.EltSize, CD8VF>;
-  defm v231r : avx512_fma3p_rm<opc231, !strconcat(OpcodeStr, "231", VTI.Suffix),
-                              VTI>, EVEX_CD8<VTI.EltSize, CD8VF>;
+multiclass avx512_fma3p_213_f<bits<8> opc, string OpcodeStr, SDNode OpNode,
+                                                            SDNode OpNodeRnd > {
+    defm PS : avx512_fma3p_213_common<opc, OpcodeStr#"ps", OpNode, OpNodeRnd,
+                                      avx512vl_f32_info>;
+    defm PD : avx512_fma3p_213_common<opc, OpcodeStr#"pd", OpNode, OpNodeRnd,
+                                      avx512vl_f64_info>, VEX_W;
 }
 
-multiclass avx512_fma3p<bits<8> opc213, bits<8> opc231,
-                              string OpcodeStr,
-                              SDPatternOperator OpNode,
-                              SDPatternOperator OpNodeRnd> {
-let ExeDomain = SSEPackedSingle in {
-    defm NAME##PSZ      : avx512_fma3p_forms<opc213, opc231, OpcodeStr,
-                                             v16f32_info, OpNode>,
-                          avx512_fma3_round_forms<opc213, OpcodeStr,
-                                             v16f32_info, OpNodeRnd>, EVEX_V512;
-    defm NAME##PSZ256   : avx512_fma3p_forms<opc213, opc231, OpcodeStr,
-                                             v8f32x_info, OpNode>, EVEX_V256;
-    defm NAME##PSZ128   : avx512_fma3p_forms<opc213, opc231, OpcodeStr,
-                                             v4f32x_info, OpNode>, EVEX_V128;
-  }
-let ExeDomain = SSEPackedDouble in {
-    defm  NAME##PDZ     : avx512_fma3p_forms<opc213, opc231, OpcodeStr,
-                                             v8f64_info, OpNode>,
-                          avx512_fma3_round_forms<opc213, OpcodeStr, v8f64_info,
-                                                  OpNodeRnd>, EVEX_V512, VEX_W;
-    defm  NAME##PDZ256  : avx512_fma3p_forms<opc213, opc231, OpcodeStr,
-                                             v4f64x_info, OpNode>,
-                                             EVEX_V256, VEX_W;
-    defm  NAME##PDZ128  : avx512_fma3p_forms<opc213, opc231, OpcodeStr,
-                                             v2f64x_info, OpNode>,
-                                             EVEX_V128, VEX_W;
-  }
-}
-
-defm VFMADD    : avx512_fma3p<0xA8, 0xB8, "vfmadd", X86Fmadd, X86FmaddRnd>;
-defm VFMSUB    : avx512_fma3p<0xAA, 0xBA, "vfmsub", X86Fmsub, X86FmsubRnd>;
-defm VFMADDSUB : avx512_fma3p<0xA6, 0xB6, "vfmaddsub", X86Fmaddsub, X86FmaddsubRnd>;
-defm VFMSUBADD : avx512_fma3p<0xA7, 0xB7, "vfmsubadd", X86Fmsubadd, X86FmsubaddRnd>;
-defm VFNMADD   : avx512_fma3p<0xAC, 0xBC, "vfnmadd", X86Fnmadd, X86FnmaddRnd>;
-defm VFNMSUB   : avx512_fma3p<0xAE, 0xBE, "vfnmsub", X86Fnmsub, X86FnmsubRnd>;
+defm VFMADD213    : avx512_fma3p_213_f<0xA8, "vfmadd213", X86Fmadd, X86FmaddRnd>;
+defm VFMSUB213    : avx512_fma3p_213_f<0xAA, "vfmsub213", X86Fmsub, X86FmsubRnd>;
+defm VFMADDSUB213 : avx512_fma3p_213_f<0xA6, "vfmaddsub213", X86Fmaddsub, X86FmaddsubRnd>;
+defm VFMSUBADD213 : avx512_fma3p_213_f<0xA7, "vfmsubadd213", X86Fmsubadd, X86FmsubaddRnd>;
+defm VFNMADD213   : avx512_fma3p_213_f<0xAC, "vfnmadd213", X86Fnmadd, X86FnmaddRnd>;
+defm VFNMSUB213   : avx512_fma3p_213_f<0xAE, "vfnmsub213", X86Fnmsub, X86FnmsubRnd>;
+
 
 let Constraints = "$src1 = $dst" in {
-multiclass avx512_fma3p_m132<bits<8> opc, string OpcodeStr, SDNode OpNode,
-                             X86VectorVTInfo _> {
-  let mayLoad = 1 in
-  def m: AVX512FMA3<opc, MRMSrcMem, (outs _.RC:$dst),
-          (ins _.RC:$src1, _.RC:$src3, _.MemOp:$src2),
-          !strconcat(OpcodeStr, "\t{$src2, $src3, $dst|$dst, $src3, $src2}"),
-          [(set _.RC:$dst, (_.VT (OpNode _.RC:$src1, (_.LdFrag addr:$src2),
-                                                    _.RC:$src3)))]>;
-   def mb: AVX512FMA3<opc, MRMSrcMem, (outs _.RC:$dst),
-           (ins _.RC:$src1, _.RC:$src3, _.ScalarMemOp:$src2),
-           !strconcat(OpcodeStr, "\t{${src2}", _.BroadcastStr,
-            ", $src3, $dst|$dst, $src3, ${src2}", _.BroadcastStr, "}"),
-           [(set _.RC:$dst,
-               (OpNode _.RC:$src1, (_.VT (X86VBroadcast
-                                            (_.ScalarLdFrag addr:$src2))),
-                                   _.RC:$src3))]>, EVEX_B;
+multiclass avx512_fma3p_231_rm<bits<8> opc, string OpcodeStr, SDNode OpNode,
+                                                            X86VectorVTInfo _> {
+  defm r: AVX512_maskable_3src<opc, MRMSrcReg, _, (outs _.RC:$dst),
+          (ins _.RC:$src2, _.RC:$src3),
+          OpcodeStr, "$src3, $src2", "$src2, $src3",
+          (_.VT (OpNode _.RC:$src2, _.RC:$src3, _.RC:$src1))>,
+         AVX512FMA3Base;
+
+  let mayLoad = 1 in {
+    defm m: AVX512_maskable_3src<opc, MRMSrcMem, _, (outs _.RC:$dst),
+            (ins _.RC:$src2, _.MemOp:$src3),
+            OpcodeStr, "$src3, $src2", "$src2, $src3",
+            (_.VT (OpNode _.RC:$src2, (_.LdFrag addr:$src3), _.RC:$src1))>,
+           AVX512FMA3Base;
+
+    defm mb: AVX512_maskable_3src<opc, MRMSrcMem, _, (outs _.RC:$dst),
+           (ins _.RC:$src2, _.ScalarMemOp:$src3),
+           OpcodeStr, "${src3}"##_.BroadcastStr##", $src2",
+           "$src2, ${src3}"##_.BroadcastStr,
+           (_.VT (OpNode _.RC:$src2,
+                        (_.VT (X86VBroadcast(_.ScalarLdFrag addr:$src3))),
+                        _.RC:$src1))>, AVX512FMA3Base, EVEX_B;
+  }
+}
+
+multiclass avx512_fma3_231_round<bits<8> opc, string OpcodeStr, SDNode OpNode,
+                                                            X86VectorVTInfo _> {
+  defm rb: AVX512_maskable_3src<opc, MRMSrcReg, _, (outs _.RC:$dst),
+          (ins _.RC:$src2, _.RC:$src3, AVX512RC:$rc),
+          OpcodeStr, "$rc, $src3, $src2", "$src2, $src3, $rc",
+          (_.VT ( OpNode _.RC:$src2, _.RC:$src3, _.RC:$src1, (i32 imm:$rc)))>,
+          AVX512FMA3Base, EVEX_B, EVEX_RC;
 }
 } // Constraints = "$src1 = $dst"
 
-multiclass avx512_fma3p_m132_f<bits<8> opc, string OpcodeStr, SDNode OpNode> {
+multiclass avx512_fma3p_231_common<bits<8> opc, string OpcodeStr, SDNode OpNode,
+                                     SDNode OpNodeRnd, AVX512VLVectorVTInfo _> {
+  let Predicates = [HasAVX512] in {
+    defm Z      : avx512_fma3p_231_rm<opc, OpcodeStr, OpNode, _.info512>,
+                  avx512_fma3_231_round<opc, OpcodeStr, OpNodeRnd, _.info512>,
+                      EVEX_V512, EVEX_CD8<_.info512.EltSize, CD8VF>;
+  }
+  let Predicates = [HasVLX, HasAVX512] in {
+    defm Z256 : avx512_fma3p_231_rm<opc, OpcodeStr, OpNode, _.info256>,
+                      EVEX_V256, EVEX_CD8<_.info256.EltSize, CD8VF>;
+    defm Z128 : avx512_fma3p_231_rm<opc, OpcodeStr, OpNode, _.info128>,
+                      EVEX_V128, EVEX_CD8<_.info128.EltSize, CD8VF>;
+  }
+}
+
+multiclass avx512_fma3p_231_f<bits<8> opc, string OpcodeStr, SDNode OpNode,
+                                                            SDNode OpNodeRnd > {
+    defm PS : avx512_fma3p_231_common<opc, OpcodeStr#"ps", OpNode, OpNodeRnd,
+                                      avx512vl_f32_info>;
+    defm PD : avx512_fma3p_231_common<opc, OpcodeStr#"pd", OpNode, OpNodeRnd,
+                                      avx512vl_f64_info>, VEX_W;
+}
 
-let ExeDomain = SSEPackedSingle in {
-    defm NAME##PSZ      : avx512_fma3p_m132<opc, OpcodeStr##ps,
-                                             OpNode,v16f32_info>, EVEX_V512,
-                                             EVEX_CD8<32, CD8VF>;
-    defm NAME##PSZ256   : avx512_fma3p_m132<opc, OpcodeStr##ps,
-                                             OpNode, v8f32x_info>, EVEX_V256,
-                                             EVEX_CD8<32, CD8VF>;
-    defm NAME##PSZ128   : avx512_fma3p_m132<opc, OpcodeStr##ps,
-                                             OpNode, v4f32x_info>, EVEX_V128,
-                                             EVEX_CD8<32, CD8VF>;
+defm VFMADD231    : avx512_fma3p_231_f<0xB8, "vfmadd231", X86Fmadd, X86FmaddRnd>;
+defm VFMSUB231    : avx512_fma3p_231_f<0xBA, "vfmsub231", X86Fmsub, X86FmsubRnd>;
+defm VFMADDSUB231 : avx512_fma3p_231_f<0xB6, "vfmaddsub231", X86Fmaddsub, X86FmaddsubRnd>;
+defm VFMSUBADD231 : avx512_fma3p_231_f<0xB7, "vfmsubadd231", X86Fmsubadd, X86FmsubaddRnd>;
+defm VFNMADD231   : avx512_fma3p_231_f<0xBC, "vfnmadd231", X86Fnmadd, X86FnmaddRnd>;
+defm VFNMSUB231   : avx512_fma3p_231_f<0xBE, "vfnmsub231", X86Fnmsub, X86FnmsubRnd>;
+
+let Constraints = "$src1 = $dst" in {
+multiclass avx512_fma3p_132_rm<bits<8> opc, string OpcodeStr, SDNode OpNode,
+                                                            X86VectorVTInfo _> {
+  defm r: AVX512_maskable_3src<opc, MRMSrcReg, _, (outs _.RC:$dst),
+          (ins _.RC:$src3, _.RC:$src2),
+          OpcodeStr, "$src2, $src3", "$src3, $src2",
+          (_.VT (OpNode _.RC:$src1, _.RC:$src2, _.RC:$src3))>,
+         AVX512FMA3Base;
+
+  let mayLoad = 1 in {
+    defm m: AVX512_maskable_3src<opc, MRMSrcMem, _, (outs _.RC:$dst),
+            (ins _.RC:$src3, _.MemOp:$src2),
+            OpcodeStr, "$src2, $src3", "$src3, $src2",
+            (_.VT (OpNode _.RC:$src1, (_.LdFrag addr:$src2), _.RC:$src3))>,
+           AVX512FMA3Base;
+
+    defm mb: AVX512_maskable_3src<opc, MRMSrcMem, _, (outs _.RC:$dst),
+           (ins _.RC:$src3, _.ScalarMemOp:$src2),
+           OpcodeStr, "${src2}"##_.BroadcastStr##", $src3",
+           "$src3, ${src2}"##_.BroadcastStr,
+           (_.VT (OpNode _.RC:$src1,
+                        (_.VT (X86VBroadcast(_.ScalarLdFrag addr:$src2))),
+                        _.RC:$src3))>, AVX512FMA3Base, EVEX_B;
   }
-let ExeDomain = SSEPackedDouble in {
-    defm  NAME##PDZ       : avx512_fma3p_m132<opc, OpcodeStr##pd,
-                                           OpNode, v8f64_info>, EVEX_V512,
-                                           VEX_W, EVEX_CD8<32, CD8VF>;
-    defm  NAME##PDZ256    : avx512_fma3p_m132<opc, OpcodeStr##pd,
-                                           OpNode, v4f64x_info>, EVEX_V256,
-                                           VEX_W, EVEX_CD8<32, CD8VF>;
-    defm  NAME##PDZ128    : avx512_fma3p_m132<opc, OpcodeStr##pd,
-                                           OpNode, v2f64x_info>, EVEX_V128,
-                                           VEX_W, EVEX_CD8<32, CD8VF>;
+}
+
+multiclass avx512_fma3_132_round<bits<8> opc, string OpcodeStr, SDNode OpNode,
+                                                            X86VectorVTInfo _> {
+  defm rb: AVX512_maskable_3src<opc, MRMSrcReg, _, (outs _.RC:$dst),
+          (ins _.RC:$src3, _.RC:$src2, AVX512RC:$rc),
+          OpcodeStr, "$rc, $src2, $src3", "$src3, $src2, $rc",
+          (_.VT ( OpNode _.RC:$src1, _.RC:$src2, _.RC:$src3, (i32 imm:$rc)))>,
+          AVX512FMA3Base, EVEX_B, EVEX_RC;
+}
+} // Constraints = "$src1 = $dst"
+
+multiclass avx512_fma3p_132_common<bits<8> opc, string OpcodeStr, SDNode OpNode,
+                                     SDNode OpNodeRnd, AVX512VLVectorVTInfo _> {
+  let Predicates = [HasAVX512] in {
+    defm Z      : avx512_fma3p_132_rm<opc, OpcodeStr, OpNode, _.info512>,
+                  avx512_fma3_132_round<opc, OpcodeStr, OpNodeRnd, _.info512>,
+                      EVEX_V512, EVEX_CD8<_.info512.EltSize, CD8VF>;
   }
+  let Predicates = [HasVLX, HasAVX512] in {
+    defm Z256 : avx512_fma3p_132_rm<opc, OpcodeStr, OpNode, _.info256>,
+                      EVEX_V256, EVEX_CD8<_.info256.EltSize, CD8VF>;
+    defm Z128 : avx512_fma3p_132_rm<opc, OpcodeStr, OpNode, _.info128>,
+                      EVEX_V128, EVEX_CD8<_.info128.EltSize, CD8VF>;
+  }
+}
+
+multiclass avx512_fma3p_132_f<bits<8> opc, string OpcodeStr, SDNode OpNode,
+                                                            SDNode OpNodeRnd > {
+    defm PS : avx512_fma3p_132_common<opc, OpcodeStr#"ps", OpNode, OpNodeRnd,
+                                      avx512vl_f32_info>;
+    defm PD : avx512_fma3p_132_common<opc, OpcodeStr#"pd", OpNode, OpNodeRnd,
+                                      avx512vl_f64_info>, VEX_W;
 }
 
-defm VFMADD132    : avx512_fma3p_m132_f<0x98, "vfmadd132", X86Fmadd>;
-defm VFMSUB132    : avx512_fma3p_m132_f<0x9A, "vfmsub132", X86Fmsub>;
-defm VFMADDSUB132 : avx512_fma3p_m132_f<0x96, "vfmaddsub132", X86Fmaddsub>;
-defm VFMSUBADD132 : avx512_fma3p_m132_f<0x97, "vfmsubadd132", X86Fmsubadd>;
-defm VFNMADD132   : avx512_fma3p_m132_f<0x9C, "vfnmadd132", X86Fnmadd>;
-defm VFNMSUB132   : avx512_fma3p_m132_f<0x9E, "vfnmsub132", X86Fnmsub>;
+defm VFMADD132    : avx512_fma3p_132_f<0x98, "vfmadd132", X86Fmadd, X86FmaddRnd>;
+defm VFMSUB132    : avx512_fma3p_132_f<0x9A, "vfmsub132", X86Fmsub, X86FmsubRnd>;
+defm VFMADDSUB132 : avx512_fma3p_132_f<0x96, "vfmaddsub132", X86Fmaddsub, X86FmaddsubRnd>;
+defm VFMSUBADD132 : avx512_fma3p_132_f<0x97, "vfmsubadd132", X86Fmsubadd, X86FmsubaddRnd>;
+defm VFNMADD132   : avx512_fma3p_132_f<0x9C, "vfnmadd132", X86Fnmadd, X86FnmaddRnd>;
+defm VFNMSUB132   : avx512_fma3p_132_f<0x9E, "vfnmsub132", X86Fnmsub, X86FnmsubRnd>;
 
 // Scalar FMA
 let Constraints = "$src1 = $dst" in {
-multiclass avx512_fma3s_rm<bits<8> opc, string OpcodeStr, SDNode OpNode,
-                 RegisterClass RC, ValueType OpVT,
-                 X86MemOperand x86memop, Operand memop,
-                 PatFrag mem_frag> {
-  let isCommutable = 1 in
-  def r     : AVX512FMA3<opc, MRMSrcReg, (outs RC:$dst),
-                   (ins RC:$src1, RC:$src2, RC:$src3),
-                   !strconcat(OpcodeStr,
-                              "\t{$src3, $src2, $dst|$dst, $src2, $src3}"),
-                   [(set RC:$dst,
-                     (OpVT (OpNode RC:$src2, RC:$src1, RC:$src3)))]>;
+multiclass avx512_fma3s_common<bits<8> opc, string OpcodeStr, X86VectorVTInfo _,
+                               dag RHS_VEC_r, dag RHS_VEC_m, dag RHS_VEC_rb,
+                                                        dag RHS_r, dag RHS_m > {
+  defm r_Int: AVX512_maskable_3src_scalar<opc, MRMSrcReg, _, (outs _.RC:$dst),
+          (ins _.RC:$src2, _.RC:$src3), OpcodeStr,
+          "$src3, $src2", "$src2, $src3", RHS_VEC_r>, AVX512FMA3Base;
+
   let mayLoad = 1 in
-  def m     : AVX512FMA3<opc, MRMSrcMem, (outs RC:$dst),
-                   (ins RC:$src1, RC:$src2, f128mem:$src3),
-                   !strconcat(OpcodeStr,
+    defm m_Int: AVX512_maskable_3src_scalar<opc, MRMSrcMem, _, (outs _.RC:$dst),
+            (ins _.RC:$src2, _.MemOp:$src3), OpcodeStr,
+            "$src3, $src2", "$src2, $src3", RHS_VEC_m>, AVX512FMA3Base;
+
+  defm rb_Int: AVX512_maskable_3src_scalar<opc, MRMSrcReg, _, (outs _.RC:$dst),
+         (ins _.RC:$src2, _.RC:$src3, AVX512RC:$rc),
+         OpcodeStr, "$rc, $src3, $src2", "$src2, $src3, $rc", RHS_VEC_rb>,
+                                       AVX512FMA3Base, EVEX_B, EVEX_RC;
+
+  let isCodeGenOnly = 1 in {
+    def r     : AVX512FMA3<opc, MRMSrcReg, (outs _.FRC:$dst),
+                     (ins _.FRC:$src1, _.FRC:$src2, _.FRC:$src3),
+                     !strconcat(OpcodeStr,
                               "\t{$src3, $src2, $dst|$dst, $src2, $src3}"),
-                   [(set RC:$dst,
-                     (OpVT (OpNode RC:$src2, RC:$src1,
-                            (mem_frag addr:$src3))))]>;
+                     [RHS_r]>;
+    let mayLoad = 1 in
+      def m     : AVX512FMA3<opc, MRMSrcMem, (outs _.FRC:$dst),
+                      (ins _.FRC:$src1, _.FRC:$src2, _.ScalarMemOp:$src3),
+                      !strconcat(OpcodeStr,
+                                 "\t{$src3, $src2, $dst|$dst, $src2, $src3}"),
+                      [RHS_m]>;
+  }// isCodeGenOnly = 1
+}
+}// Constraints = "$src1 = $dst"
+
+multiclass avx512_fma3s_all<bits<8> opc213, bits<8> opc231, bits<8> opc132,
+         string OpcodeStr, SDNode OpNode, SDNode OpNodeRnd, X86VectorVTInfo _ ,
+                                                                  string SUFF> {
+
+  defm NAME#213#SUFF: avx512_fma3s_common<opc213, OpcodeStr#"213"#_.Suffix , _ ,
+                (_.VT (OpNode _.RC:$src2, _.RC:$src1, _.RC:$src3)),
+                (_.VT (OpNode _.RC:$src2, _.RC:$src1,
+                         (_.VT (scalar_to_vector(_.ScalarLdFrag addr:$src3))))),
+                (_.VT ( OpNodeRnd _.RC:$src2, _.RC:$src1, _.RC:$src3,
+                         (i32 imm:$rc))),
+                (set _.FRC:$dst, (_.EltVT (OpNode _.FRC:$src2, _.FRC:$src1,
+                         _.FRC:$src3))),
+                (set _.FRC:$dst, (_.EltVT (OpNode _.FRC:$src2, _.FRC:$src1,
+                         (_.ScalarLdFrag addr:$src3))))>;
+
+  defm NAME#231#SUFF: avx512_fma3s_common<opc231, OpcodeStr#"231"#_.Suffix , _ ,
+                (_.VT (OpNode _.RC:$src2, _.RC:$src3, _.RC:$src1)),
+                (_.VT (OpNode _.RC:$src2,
+                       (_.VT (scalar_to_vector(_.ScalarLdFrag addr:$src3))),
+                              _.RC:$src1)),
+                (_.VT ( OpNodeRnd _.RC:$src2, _.RC:$src3, _.RC:$src1,
+                                  (i32 imm:$rc))),
+                (set _.FRC:$dst, (_.EltVT (OpNode _.FRC:$src2, _.FRC:$src3,
+                                          _.FRC:$src1))),
+                (set _.FRC:$dst, (_.EltVT (OpNode _.FRC:$src2,
+                            (_.ScalarLdFrag addr:$src3), _.FRC:$src1)))>;
+
+  defm NAME#132#SUFF: avx512_fma3s_common<opc132, OpcodeStr#"132"#_.Suffix , _ ,
+                (_.VT (OpNode _.RC:$src1, _.RC:$src3, _.RC:$src2)),
+                (_.VT (OpNode _.RC:$src1,
+                       (_.VT (scalar_to_vector(_.ScalarLdFrag addr:$src3))),
+                              _.RC:$src2)),
+                (_.VT ( OpNodeRnd _.RC:$src1, _.RC:$src3, _.RC:$src2,
+                         (i32 imm:$rc))),
+                (set _.FRC:$dst, (_.EltVT (OpNode _.FRC:$src1, _.FRC:$src3,
+                         _.FRC:$src2))),
+                (set _.FRC:$dst, (_.EltVT (OpNode _.FRC:$src1,
+                          (_.ScalarLdFrag addr:$src3), _.FRC:$src2)))>;
+}
+
+multiclass avx512_fma3s<bits<8> opc213, bits<8> opc231, bits<8> opc132,
+                             string OpcodeStr, SDNode OpNode, SDNode OpNodeRnd>{
+  let Predicates = [HasAVX512] in {
+    defm NAME : avx512_fma3s_all<opc213, opc231, opc132, OpcodeStr, OpNode,
+                                   OpNodeRnd, f32x_info, "SS">,
+                                   EVEX_CD8<32, CD8VT1>, VEX_LIG;
+    defm NAME : avx512_fma3s_all<opc213, opc231, opc132, OpcodeStr, OpNode,
+                                   OpNodeRnd, f64x_info, "SD">,
+                                   EVEX_CD8<64, CD8VT1>, VEX_LIG, VEX_W;
+  }
 }
-} // Constraints = "$src1 = $dst"
 
-defm VFMADDSSZ  : avx512_fma3s_rm<0xA9, "vfmadd213ss", X86Fmadd, FR32X,
-                      f32, f32mem, ssmem, loadf32>, EVEX_CD8<32, CD8VT1>;
-defm VFMADDSDZ  : avx512_fma3s_rm<0xA9, "vfmadd213sd", X86Fmadd, FR64X,
-                      f64, f64mem, sdmem, loadf64>, VEX_W, EVEX_CD8<64, CD8VT1>;
-defm VFMSUBSSZ  : avx512_fma3s_rm<0xAB, "vfmsub213ss", X86Fmsub, FR32X,
-                      f32, f32mem, ssmem, loadf32>, EVEX_CD8<32, CD8VT1>;
-defm VFMSUBSDZ  : avx512_fma3s_rm<0xAB, "vfmsub213sd", X86Fmsub, FR64X,
-                      f64, f64mem, sdmem, loadf64>, VEX_W, EVEX_CD8<64, CD8VT1>;
-defm VFNMADDSSZ  : avx512_fma3s_rm<0xAD, "vfnmadd213ss", X86Fnmadd, FR32X,
-                      f32, f32mem, ssmem, loadf32>, EVEX_CD8<32, CD8VT1>;
-defm VFNMADDSDZ  : avx512_fma3s_rm<0xAD, "vfnmadd213sd", X86Fnmadd, FR64X,
-                      f64, f64mem, sdmem, loadf64>, VEX_W, EVEX_CD8<64, CD8VT1>;
-defm VFNMSUBSSZ  : avx512_fma3s_rm<0xAF, "vfnmsub213ss", X86Fnmsub, FR32X,
-                      f32, f32mem, ssmem, loadf32>, EVEX_CD8<32, CD8VT1>;
-defm VFNMSUBSDZ  : avx512_fma3s_rm<0xAF, "vfnmsub213sd", X86Fnmsub, FR64X,
-                      f64, f64mem, sdmem, loadf64>, VEX_W, EVEX_CD8<64, CD8VT1>;
+defm VFMADD  : avx512_fma3s<0xA9, 0xB9, 0x99, "vfmadd", X86Fmadd, X86FmaddRnd>;
+defm VFMSUB  : avx512_fma3s<0xAB, 0xBB, 0x9B, "vfmsub", X86Fmsub, X86FmsubRnd>;
+defm VFNMADD : avx512_fma3s<0xAD, 0xBD, 0x9D, "vfnmadd", X86Fnmadd, X86FnmaddRnd>;
+defm VFNMSUB : avx512_fma3s<0xAF, 0xBF, 0x9F, "vfnmsub", X86Fnmsub, X86FnmsubRnd>;
 
 //===----------------------------------------------------------------------===//
 // AVX-512  Scalar convert from sign integer to float/double
 //===----------------------------------------------------------------------===//
 
-multiclass avx512_vcvtsi<bits<8> opc, RegisterClass SrcRC, RegisterClass DstRC,
-                          X86MemOperand x86memop, string asm> {
-let hasSideEffects = 0 in {
-  def rr : SI<opc, MRMSrcReg, (outs DstRC:$dst), (ins DstRC:$src1, SrcRC:$src),
+multiclass avx512_vcvtsi<bits<8> opc, SDNode OpNode, RegisterClass SrcRC,
+                    X86VectorVTInfo DstVT, X86MemOperand x86memop,
+                    PatFrag ld_frag, string asm> {
+  let hasSideEffects = 0 in {
+    def rr : SI<opc, MRMSrcReg, (outs DstVT.FRC:$dst),
+              (ins DstVT.FRC:$src1, SrcRC:$src),
               !strconcat(asm,"\t{$src, $src1, $dst|$dst, $src1, $src}"), []>,
               EVEX_4V;
-  let mayLoad = 1 in
-  def rm : SI<opc, MRMSrcMem, (outs DstRC:$dst),
-              (ins DstRC:$src1, x86memop:$src),
+    let mayLoad = 1 in
+      def rm : SI<opc, MRMSrcMem, (outs DstVT.FRC:$dst),
+              (ins DstVT.FRC:$src1, x86memop:$src),
               !strconcat(asm,"\t{$src, $src1, $dst|$dst, $src1, $src}"), []>,
               EVEX_4V;
-} // hasSideEffects = 0
+  } // hasSideEffects = 0
+  let isCodeGenOnly = 1 in {
+    def rr_Int : SI<opc, MRMSrcReg, (outs DstVT.RC:$dst),
+                  (ins DstVT.RC:$src1, SrcRC:$src2),
+                  !strconcat(asm,"\t{$src2, $src1, $dst|$dst, $src1, $src2}"),
+                  [(set DstVT.RC:$dst,
+                        (OpNode (DstVT.VT DstVT.RC:$src1),
+                                 SrcRC:$src2,
+                                 (i32 FROUND_CURRENT)))]>, EVEX_4V;
+
+    def rm_Int : SI<opc, MRMSrcMem, (outs DstVT.RC:$dst),
+                  (ins DstVT.RC:$src1, x86memop:$src2),
+                  !strconcat(asm,"\t{$src2, $src1, $dst|$dst, $src1, $src2}"),
+                  [(set DstVT.RC:$dst,
+                        (OpNode (DstVT.VT DstVT.RC:$src1),
+                                 (ld_frag addr:$src2),
+                                 (i32 FROUND_CURRENT)))]>, EVEX_4V;
+  }//isCodeGenOnly = 1
+}
+
+multiclass avx512_vcvtsi_round<bits<8> opc, SDNode OpNode, RegisterClass SrcRC,
+                    X86VectorVTInfo DstVT, string asm> {
+  def rrb_Int : SI<opc, MRMSrcReg, (outs DstVT.RC:$dst),
+              (ins DstVT.RC:$src1, SrcRC:$src2, AVX512RC:$rc),
+              !strconcat(asm,
+                  "\t{$src2, $rc, $src1, $dst|$dst, $src1, $rc, $src2}"),
+              [(set DstVT.RC:$dst,
+                    (OpNode (DstVT.VT DstVT.RC:$src1),
+                             SrcRC:$src2,
+                             (i32 imm:$rc)))]>, EVEX_4V, EVEX_B, EVEX_RC;
+}
+
+multiclass avx512_vcvtsi_common<bits<8> opc, SDNode OpNode, RegisterClass SrcRC,
+                    X86VectorVTInfo DstVT, X86MemOperand x86memop,
+                    PatFrag ld_frag, string asm> {
+  defm NAME : avx512_vcvtsi_round<opc, OpNode, SrcRC, DstVT, asm>,
+              avx512_vcvtsi<opc, OpNode, SrcRC, DstVT, x86memop, ld_frag, asm>,
+                        VEX_LIG;
 }
 
 let Predicates = [HasAVX512] in {
-defm VCVTSI2SSZ   : avx512_vcvtsi<0x2A, GR32, FR32X, i32mem, "cvtsi2ss{l}">,
-                                  XS, VEX_LIG, EVEX_CD8<32, CD8VT1>;
-defm VCVTSI642SSZ : avx512_vcvtsi<0x2A, GR64, FR32X, i64mem, "cvtsi2ss{q}">,
-                                  XS, VEX_W, VEX_LIG, EVEX_CD8<64, CD8VT1>;
-defm VCVTSI2SDZ   : avx512_vcvtsi<0x2A, GR32, FR64X, i32mem, "cvtsi2sd{l}">,
-                                  XD, VEX_LIG, EVEX_CD8<32, CD8VT1>;
-defm VCVTSI642SDZ : avx512_vcvtsi<0x2A, GR64, FR64X, i64mem, "cvtsi2sd{q}">,
-                                  XD, VEX_W, VEX_LIG, EVEX_CD8<64, CD8VT1>;
+defm VCVTSI2SSZ  : avx512_vcvtsi_common<0x2A, X86SintToFpRnd, GR32,
+                                 v4f32x_info, i32mem, loadi32, "cvtsi2ss{l}">,
+                                 XS, EVEX_CD8<32, CD8VT1>;
+defm VCVTSI642SSZ: avx512_vcvtsi_common<0x2A, X86SintToFpRnd, GR64,
+                                 v4f32x_info, i64mem, loadi64, "cvtsi2ss{q}">,
+                                 XS, VEX_W, EVEX_CD8<64, CD8VT1>;
+defm VCVTSI2SDZ  : avx512_vcvtsi_common<0x2A, X86SintToFpRnd, GR32,
+                                 v2f64x_info, i32mem, loadi32, "cvtsi2sd{l}">,
+                                 XD, EVEX_CD8<32, CD8VT1>;
+defm VCVTSI642SDZ: avx512_vcvtsi_common<0x2A, X86SintToFpRnd, GR64,
+                                 v2f64x_info, i64mem, loadi64, "cvtsi2sd{q}">,
+                                 XD, VEX_W, EVEX_CD8<64, CD8VT1>;
 
 def : Pat<(f32 (sint_to_fp (loadi32 addr:$src))),
           (VCVTSI2SSZrm (f32 (IMPLICIT_DEF)), addr:$src)>;
@@ -3919,14 +4378,18 @@ def : Pat<(f64 (sint_to_fp GR32:$src)),
 def : Pat<(f64 (sint_to_fp GR64:$src)),
           (VCVTSI642SDZrr (f64 (IMPLICIT_DEF)), GR64:$src)>;
 
-defm VCVTUSI2SSZ   : avx512_vcvtsi<0x7B, GR32, FR32X, i32mem, "cvtusi2ss{l}">,
-                                  XS, VEX_LIG, EVEX_CD8<32, CD8VT1>;
-defm VCVTUSI642SSZ : avx512_vcvtsi<0x7B, GR64, FR32X, i64mem, "cvtusi2ss{q}">,
-                                  XS, VEX_W, VEX_LIG, EVEX_CD8<64, CD8VT1>;
-defm VCVTUSI2SDZ   : avx512_vcvtsi<0x7B, GR32, FR64X, i32mem, "cvtusi2sd{l}">,
+defm VCVTUSI2SSZ   : avx512_vcvtsi_common<0x7B, X86UintToFpRnd, GR32,
+                                  v4f32x_info, i32mem, loadi32,
+                                  "cvtusi2ss{l}">, XS, EVEX_CD8<32, CD8VT1>;
+defm VCVTUSI642SSZ : avx512_vcvtsi_common<0x7B, X86UintToFpRnd, GR64,
+                                  v4f32x_info, i64mem, loadi64, "cvtusi2ss{q}">,
+                                  XS, VEX_W, EVEX_CD8<64, CD8VT1>;
+defm VCVTUSI2SDZ   : avx512_vcvtsi<0x7B, X86UintToFpRnd, GR32, v2f64x_info,
+                                  i32mem, loadi32, "cvtusi2sd{l}">,
                                   XD, VEX_LIG, EVEX_CD8<32, CD8VT1>;
-defm VCVTUSI642SDZ : avx512_vcvtsi<0x7B, GR64, FR64X, i64mem, "cvtusi2sd{q}">,
-                                  XD, VEX_W, VEX_LIG, EVEX_CD8<64, CD8VT1>;
+defm VCVTUSI642SDZ : avx512_vcvtsi_common<0x7B, X86UintToFpRnd, GR64,
+                                  v2f64x_info, i64mem, loadi64, "cvtusi2sd{q}">,
+                                  XD, VEX_W, EVEX_CD8<64, CD8VT1>;
 
 def : Pat<(f32 (uint_to_fp (loadi32 addr:$src))),
           (VCVTUSI2SSZrm (f32 (IMPLICIT_DEF)), addr:$src)>;
@@ -4007,18 +4470,9 @@ let isCodeGenOnly = 1 in {
             int_x86_sse2_cvtsi642sd, i64mem, loadi64, "cvtsi2sd{q}",
             SSE_CVT_Scalar, 0>, XD, EVEX_4V, VEX_W;
 
-  defm Int_VCVTUSI2SSZ : sse12_cvt_sint_3addr<0x2A, GR32, VR128X,
-            int_x86_avx512_cvtusi2ss, i32mem, loadi32, "cvtusi2ss{l}",
-            SSE_CVT_Scalar, 0>, XS, EVEX_4V;
-  defm Int_VCVTUSI2SS64Z : sse12_cvt_sint_3addr<0x2A, GR64, VR128X,
-            int_x86_avx512_cvtusi642ss, i64mem, loadi64, "cvtusi2ss{q}",
-            SSE_CVT_Scalar, 0>, XS, EVEX_4V, VEX_W;
   defm Int_VCVTUSI2SDZ : sse12_cvt_sint_3addr<0x2A, GR32, VR128X,
             int_x86_avx512_cvtusi2sd, i32mem, loadi32, "cvtusi2sd{l}",
             SSE_CVT_Scalar, 0>, XD, EVEX_4V;
-  defm Int_VCVTUSI2SD64Z : sse12_cvt_sint_3addr<0x2A, GR64, VR128X,
-            int_x86_avx512_cvtusi642sd, i64mem, loadi64, "cvtusi2sd{q}",
-            SSE_CVT_Scalar, 0>, XD, EVEX_4V, VEX_W;
 } // isCodeGenOnly = 1
 
 // Convert float/double to signed/unsigned int 32/64 with truncation
@@ -4134,117 +4588,389 @@ def : Pat<(extloadf32 addr:$src),
 def : Pat<(f32 (fround FR64X:$src)), (VCVTSD2SSZrr FR64X:$src, FR64X:$src)>,
            Requires<[HasAVX512]>;
 
-multiclass avx512_vcvt_fp_with_rc<bits<8> opc, string asm, RegisterClass SrcRC,
-               RegisterClass DstRC, SDNode OpNode, PatFrag mem_frag,
-               X86MemOperand x86memop, ValueType OpVT, ValueType InVT,
-               Domain d> {
-let hasSideEffects = 0 in {
-  def rr : AVX512PI<opc, MRMSrcReg, (outs DstRC:$dst), (ins SrcRC:$src),
-              !strconcat(asm,"\t{$src, $dst|$dst, $src}"),
-              [(set DstRC:$dst,
-                (OpVT (OpNode (InVT SrcRC:$src))))], d>, EVEX;
-  def rrb : AVX512PI<opc, MRMSrcReg, (outs DstRC:$dst), (ins SrcRC:$src, AVX512RC:$rc),
-              !strconcat(asm,"\t{$rc, $src, $dst|$dst, $src, $rc}"),
-              [], d>, EVEX, EVEX_B, EVEX_RC;
-  let mayLoad = 1 in
-  def rm : AVX512PI<opc, MRMSrcMem, (outs DstRC:$dst), (ins x86memop:$src),
-              !strconcat(asm,"\t{$src, $dst|$dst, $src}"),
-              [(set DstRC:$dst,
-                (OpVT (OpNode (InVT (bitconvert (mem_frag addr:$src))))))], d>, EVEX;
-} // hasSideEffects = 0
+//===----------------------------------------------------------------------===//
+// AVX-512  Vector convert from signed/unsigned integer to float/double
+//          and from float/double to signed/unsigned integer
+//===----------------------------------------------------------------------===//
+
+multiclass avx512_vcvt_fp<bits<8> opc, string OpcodeStr, X86VectorVTInfo _,
+                         X86VectorVTInfo _Src, SDNode OpNode,
+                         string Broadcast = _.BroadcastStr,
+                         string Alias = ""> {
+
+  defm rr : AVX512_maskable<opc, MRMSrcReg, _, (outs _.RC:$dst),
+                         (ins _Src.RC:$src), OpcodeStr, "$src", "$src",
+                         (_.VT (OpNode (_Src.VT _Src.RC:$src)))>, EVEX;
+
+  defm rm : AVX512_maskable<opc, MRMSrcMem, _, (outs _.RC:$dst),
+                         (ins _Src.MemOp:$src), OpcodeStr#Alias, "$src", "$src",
+                         (_.VT (OpNode (_Src.VT
+                             (bitconvert (_Src.LdFrag addr:$src)))))>, EVEX;
+
+  defm rmb : AVX512_maskable<opc, MRMSrcMem, _, (outs _.RC:$dst),
+                         (ins _Src.MemOp:$src), OpcodeStr,
+                         "${src}"##Broadcast, "${src}"##Broadcast,
+                         (_.VT (OpNode (_Src.VT
+                                  (X86VBroadcast (_Src.ScalarLdFrag addr:$src)))
+                            ))>, EVEX, EVEX_B;
+}
+// Coversion with SAE - suppress all exceptions
+multiclass avx512_vcvt_fp_sae<bits<8> opc, string OpcodeStr, X86VectorVTInfo _,
+                         X86VectorVTInfo _Src, SDNode OpNodeRnd> {
+  defm rrb : AVX512_maskable<opc, MRMSrcReg, _, (outs _.RC:$dst),
+                        (ins _Src.RC:$src), OpcodeStr,
+                        "{sae}, $src", "$src, {sae}",
+                        (_.VT (OpNodeRnd (_Src.VT _Src.RC:$src),
+                               (i32 FROUND_NO_EXC)))>,
+                        EVEX, EVEX_B;
+}
+
+// Conversion with rounding control (RC)
+multiclass avx512_vcvt_fp_rc<bits<8> opc, string OpcodeStr, X86VectorVTInfo _,
+                         X86VectorVTInfo _Src, SDNode OpNodeRnd> {
+  defm rrb : AVX512_maskable<opc, MRMSrcReg, _, (outs _.RC:$dst),
+                        (ins _Src.RC:$src, AVX512RC:$rc), OpcodeStr,
+                        "$rc, $src", "$src, $rc",
+                        (_.VT (OpNodeRnd (_Src.VT _Src.RC:$src), (i32 imm:$rc)))>,
+                        EVEX, EVEX_B, EVEX_RC;
+}
+
+// Extend Float to Double
+multiclass avx512_cvtps2pd<bits<8> opc, string OpcodeStr> {
+  let Predicates = [HasAVX512] in {
+    defm Z : avx512_vcvt_fp<opc, OpcodeStr, v8f64_info, v8f32x_info, fextend>,
+             avx512_vcvt_fp_sae<opc, OpcodeStr, v8f64_info, v8f32x_info,
+                                X86vfpextRnd>, EVEX_V512;
+  }
+  let Predicates = [HasVLX] in {
+    defm Z128 : avx512_vcvt_fp<opc, OpcodeStr, v2f64x_info, v4f32x_info,
+                               X86vfpext, "{1to2}">, EVEX_V128;
+    defm Z256 : avx512_vcvt_fp<opc, OpcodeStr, v4f64x_info, v4f32x_info, fextend>,
+                                     EVEX_V256;
+  }
 }
 
-multiclass avx512_vcvt_fp<bits<8> opc, string asm, RegisterClass SrcRC,
-               RegisterClass DstRC, SDNode OpNode, PatFrag mem_frag,
-               X86MemOperand x86memop, ValueType OpVT, ValueType InVT,
-               Domain d> {
-let hasSideEffects = 0 in {
-  def rr : AVX512PI<opc, MRMSrcReg, (outs DstRC:$dst), (ins SrcRC:$src),
-              !strconcat(asm,"\t{$src, $dst|$dst, $src}"),
-              [(set DstRC:$dst,
-                (OpVT (OpNode (InVT SrcRC:$src))))], d>, EVEX;
-  let mayLoad = 1 in
-  def rm : AVX512PI<opc, MRMSrcMem, (outs DstRC:$dst), (ins x86memop:$src),
-              !strconcat(asm,"\t{$src, $dst|$dst, $src}"),
-              [(set DstRC:$dst,
-                (OpVT (OpNode (InVT (bitconvert (mem_frag addr:$src))))))], d>, EVEX;
-} // hasSideEffects = 0
+// Truncate Double to Float
+multiclass avx512_cvtpd2ps<bits<8> opc, string OpcodeStr> {
+  let Predicates = [HasAVX512] in {
+    defm Z : avx512_vcvt_fp<opc, OpcodeStr, v8f32x_info, v8f64_info, fround>,
+             avx512_vcvt_fp_rc<opc, OpcodeStr, v8f32x_info, v8f64_info,
+                               X86vfproundRnd>, EVEX_V512;
+  }
+  let Predicates = [HasVLX] in {
+    defm Z128 : avx512_vcvt_fp<opc, OpcodeStr, v4f32x_info, v2f64x_info,
+                               X86vfpround, "{1to2}", "{x}">, EVEX_V128;
+    defm Z256 : avx512_vcvt_fp<opc, OpcodeStr, v4f32x_info, v4f64x_info, fround,
+                               "{1to4}", "{y}">, EVEX_V256;
+  }
 }
 
-defm VCVTPD2PSZ : avx512_vcvt_fp_with_rc<0x5A, "vcvtpd2ps", VR512, VR256X, fround,
-                                loadv8f64, f512mem, v8f32, v8f64,
-                                SSEPackedSingle>, EVEX_V512, VEX_W, PD,
-                                EVEX_CD8<64, CD8VF>;
+defm VCVTPD2PS : avx512_cvtpd2ps<0x5A, "vcvtpd2ps">,
+                                  VEX_W, PD, EVEX_CD8<64, CD8VF>;
+defm VCVTPS2PD : avx512_cvtps2pd<0x5A, "vcvtps2pd">,
+                                  PS, EVEX_CD8<32, CD8VH>;
 
-defm VCVTPS2PDZ : avx512_vcvt_fp<0x5A, "vcvtps2pd", VR256X, VR512, fextend,
-                                loadv4f64, f256mem, v8f64, v8f32,
-                                SSEPackedDouble>, EVEX_V512, PS,
-                                EVEX_CD8<32, CD8VH>;
 def : Pat<(v8f64 (extloadv8f32 addr:$src)),
             (VCVTPS2PDZrm addr:$src)>;
 
-def : Pat<(v8f32 (int_x86_avx512_mask_cvtpd2ps_512 (v8f64 VR512:$src),
-                   (bc_v8f32(v8i32 immAllZerosV)), (i8 -1), (i32 FROUND_CURRENT))),
-          (VCVTPD2PSZrr VR512:$src)>;
-
-def : Pat<(v8f32 (int_x86_avx512_mask_cvtpd2ps_512 (v8f64 VR512:$src),
-                   (bc_v8f32(v8i32 immAllZerosV)), (i8 -1), imm:$rc)),
-          (VCVTPD2PSZrrb VR512:$src, imm:$rc)>;
+let Predicates = [HasVLX] in {
+  def : Pat<(v4f64 (extloadv4f32 addr:$src)),
+              (VCVTPS2PDZ256rm addr:$src)>;
+}
 
-//===----------------------------------------------------------------------===//
-// AVX-512  Vector convert from sign integer to float/double
-//===----------------------------------------------------------------------===//
+// Convert Signed/Unsigned Doubleword to Double
+multiclass avx512_cvtdq2pd<bits<8> opc, string OpcodeStr, SDNode OpNode,
+                           SDNode OpNode128> {
+  // No rounding in this op
+  let Predicates = [HasAVX512] in
+    defm Z : avx512_vcvt_fp<opc, OpcodeStr, v8f64_info, v8i32x_info, OpNode>,
+                                     EVEX_V512;
 
-defm VCVTDQ2PSZ : avx512_vcvt_fp_with_rc<0x5B, "vcvtdq2ps", VR512, VR512, sint_to_fp,
-                                loadv8i64, i512mem, v16f32, v16i32,
-                                SSEPackedSingle>, EVEX_V512, PS,
-                                EVEX_CD8<32, CD8VF>;
+  let Predicates = [HasVLX] in {
+    defm Z128 : avx512_vcvt_fp<opc, OpcodeStr, v2f64x_info, v4i32x_info,
+                                     OpNode128, "{1to2}">, EVEX_V128;
+    defm Z256 : avx512_vcvt_fp<opc, OpcodeStr, v4f64x_info, v4i32x_info, OpNode>,
+                                     EVEX_V256;
+  }
+}
 
-defm VCVTDQ2PDZ : avx512_vcvt_fp<0xE6, "vcvtdq2pd", VR256X, VR512, sint_to_fp,
-                                loadv4i64, i256mem, v8f64, v8i32,
-                                SSEPackedDouble>, EVEX_V512, XS,
-                                EVEX_CD8<32, CD8VH>;
+// Convert Signed/Unsigned Doubleword to Float
+multiclass avx512_cvtdq2ps<bits<8> opc, string OpcodeStr, SDNode OpNode,
+                           SDNode OpNodeRnd> {
+  let Predicates = [HasAVX512] in
+    defm Z : avx512_vcvt_fp<opc, OpcodeStr, v16f32_info, v16i32_info, OpNode>,
+             avx512_vcvt_fp_rc<opc, OpcodeStr, v16f32_info, v16i32_info,
+                               OpNodeRnd>, EVEX_V512;
 
-defm VCVTTPS2DQZ : avx512_vcvt_fp<0x5B, "vcvttps2dq", VR512, VR512, fp_to_sint,
-                                 loadv16f32, f512mem, v16i32, v16f32,
-                                 SSEPackedSingle>, EVEX_V512, XS,
-                                 EVEX_CD8<32, CD8VF>;
+  let Predicates = [HasVLX] in {
+    defm Z128 : avx512_vcvt_fp<opc, OpcodeStr, v4f32x_info, v4i32x_info, OpNode>,
+                                     EVEX_V128;
+    defm Z256 : avx512_vcvt_fp<opc, OpcodeStr, v8f32x_info, v8i32x_info, OpNode>,
+                                     EVEX_V256;
+  }
+}
 
-defm VCVTTPD2DQZ : avx512_vcvt_fp<0xE6, "vcvttpd2dq", VR512, VR256X, fp_to_sint,
-                                 loadv8f64, f512mem, v8i32, v8f64,
-                                 SSEPackedDouble>, EVEX_V512, PD, VEX_W,
-                                 EVEX_CD8<64, CD8VF>;
+// Convert Float to Signed/Unsigned Doubleword with truncation
+multiclass avx512_cvttps2dq<bits<8> opc, string OpcodeStr,
+                                  SDNode OpNode, SDNode OpNodeRnd> {
+  let Predicates = [HasAVX512] in {
+    defm Z : avx512_vcvt_fp<opc, OpcodeStr, v16i32_info, v16f32_info, OpNode>,
+             avx512_vcvt_fp_sae<opc, OpcodeStr, v16i32_info, v16f32_info,
+                                OpNodeRnd>, EVEX_V512;
+  }
+  let Predicates = [HasVLX] in {
+    defm Z128 : avx512_vcvt_fp<opc, OpcodeStr, v4i32x_info, v4f32x_info, OpNode>,
+                                     EVEX_V128;
+    defm Z256 : avx512_vcvt_fp<opc, OpcodeStr, v8i32x_info, v8f32x_info, OpNode>,
+                                     EVEX_V256;
+  }
+}
 
-defm VCVTTPS2UDQZ : avx512_vcvt_fp<0x78, "vcvttps2udq", VR512, VR512, fp_to_uint,
-                                 loadv16f32, f512mem, v16i32, v16f32,
-                                 SSEPackedSingle>, EVEX_V512, PS,
-                                 EVEX_CD8<32, CD8VF>;
+// Convert Float to Signed/Unsigned Doubleword
+multiclass avx512_cvtps2dq<bits<8> opc, string OpcodeStr,
+                                  SDNode OpNode, SDNode OpNodeRnd> {
+  let Predicates = [HasAVX512] in {
+    defm Z : avx512_vcvt_fp<opc, OpcodeStr, v16i32_info, v16f32_info, OpNode>,
+             avx512_vcvt_fp_rc<opc, OpcodeStr, v16i32_info, v16f32_info,
+                                OpNodeRnd>, EVEX_V512;
+  }
+  let Predicates = [HasVLX] in {
+    defm Z128 : avx512_vcvt_fp<opc, OpcodeStr, v4i32x_info, v4f32x_info, OpNode>,
+                                     EVEX_V128;
+    defm Z256 : avx512_vcvt_fp<opc, OpcodeStr, v8i32x_info, v8f32x_info, OpNode>,
+                                     EVEX_V256;
+  }
+}
 
-// cvttps2udq (src, 0, mask-all-ones, sae-current)
-def : Pat<(v16i32 (int_x86_avx512_mask_cvttps2udq_512 (v16f32 VR512:$src),
-                   (v16i32 immAllZerosV), (i16 -1), FROUND_CURRENT)),
-          (VCVTTPS2UDQZrr VR512:$src)>;
+// Convert Double to Signed/Unsigned Doubleword with truncation
+multiclass avx512_cvttpd2dq<bits<8> opc, string OpcodeStr,
+                                  SDNode OpNode, SDNode OpNodeRnd> {
+  let Predicates = [HasAVX512] in {
+    defm Z : avx512_vcvt_fp<opc, OpcodeStr, v8i32x_info, v8f64_info, OpNode>,
+             avx512_vcvt_fp_sae<opc, OpcodeStr, v8i32x_info, v8f64_info,
+                                OpNodeRnd>, EVEX_V512;
+  }
+  let Predicates = [HasVLX] in {
+    // we need "x"/"y" suffixes in order to distinguish between 128 and 256
+    // memory forms of these instructions in Asm Parcer. They have the same
+    // dest type - 'v4i32x_info'. We also specify the broadcast string explicitly
+    // due to the same reason.
+    defm Z128 : avx512_vcvt_fp<opc, OpcodeStr, v4i32x_info, v2f64x_info, OpNode,
+                               "{1to2}", "{x}">, EVEX_V128;
+    defm Z256 : avx512_vcvt_fp<opc, OpcodeStr, v4i32x_info, v4f64x_info, OpNode,
+                               "{1to4}", "{y}">, EVEX_V256;
+  }
+}
 
-defm VCVTTPD2UDQZ : avx512_vcvt_fp<0x78, "vcvttpd2udq", VR512, VR256X, fp_to_uint,
-                                 loadv8f64, f512mem, v8i32, v8f64,
-                                 SSEPackedDouble>, EVEX_V512, PS, VEX_W,
-                                 EVEX_CD8<64, CD8VF>;
+// Convert Double to Signed/Unsigned Doubleword
+multiclass avx512_cvtpd2dq<bits<8> opc, string OpcodeStr,
+                                  SDNode OpNode, SDNode OpNodeRnd> {
+  let Predicates = [HasAVX512] in {
+    defm Z : avx512_vcvt_fp<opc, OpcodeStr, v8i32x_info, v8f64_info, OpNode>,
+             avx512_vcvt_fp_rc<opc, OpcodeStr, v8i32x_info, v8f64_info,
+                               OpNodeRnd>, EVEX_V512;
+  }
+  let Predicates = [HasVLX] in {
+    // we need "x"/"y" suffixes in order to distinguish between 128 and 256
+    // memory forms of these instructions in Asm Parcer. They have the same
+    // dest type - 'v4i32x_info'. We also specify the broadcast string explicitly
+    // due to the same reason.
+    defm Z128 : avx512_vcvt_fp<opc, OpcodeStr, v4i32x_info, v2f64x_info, OpNode,
+                               "{1to2}", "{x}">, EVEX_V128;
+    defm Z256 : avx512_vcvt_fp<opc, OpcodeStr, v4i32x_info, v4f64x_info, OpNode,
+                               "{1to4}", "{y}">, EVEX_V256;
+  }
+}
 
-// cvttpd2udq (src, 0, mask-all-ones, sae-current)
-def : Pat<(v8i32 (int_x86_avx512_mask_cvttpd2udq_512 (v8f64 VR512:$src),
-                   (v8i32 immAllZerosV), (i8 -1), FROUND_CURRENT)),
-          (VCVTTPD2UDQZrr VR512:$src)>;
+// Convert Double to Signed/Unsigned Quardword
+multiclass avx512_cvtpd2qq<bits<8> opc, string OpcodeStr,
+                                  SDNode OpNode, SDNode OpNodeRnd> {
+  let Predicates = [HasDQI] in {
+    defm Z : avx512_vcvt_fp<opc, OpcodeStr, v8i64_info, v8f64_info, OpNode>,
+             avx512_vcvt_fp_rc<opc, OpcodeStr, v8i64_info, v8f64_info,
+                               OpNodeRnd>, EVEX_V512;
+  }
+  let Predicates = [HasDQI, HasVLX] in {
+    defm Z128 : avx512_vcvt_fp<opc, OpcodeStr, v2i64x_info, v2f64x_info, OpNode>,
+                               EVEX_V128;
+    defm Z256 : avx512_vcvt_fp<opc, OpcodeStr, v4i64x_info, v4f64x_info, OpNode>,
+                               EVEX_V256;
+  }
+}
 
-defm VCVTUDQ2PDZ : avx512_vcvt_fp<0x7A, "vcvtudq2pd", VR256X, VR512, uint_to_fp,
-                                 loadv4i64, f256mem, v8f64, v8i32,
-                                 SSEPackedDouble>, EVEX_V512, XS,
-                                 EVEX_CD8<32, CD8VH>;
+// Convert Double to Signed/Unsigned Quardword with truncation
+multiclass avx512_cvttpd2qq<bits<8> opc, string OpcodeStr,
+                                  SDNode OpNode, SDNode OpNodeRnd> {
+  let Predicates = [HasDQI] in {
+    defm Z : avx512_vcvt_fp<opc, OpcodeStr, v8i64_info, v8f64_info, OpNode>,
+             avx512_vcvt_fp_sae<opc, OpcodeStr, v8i64_info, v8f64_info,
+                               OpNodeRnd>, EVEX_V512;
+  }
+  let Predicates = [HasDQI, HasVLX] in {
+    defm Z128 : avx512_vcvt_fp<opc, OpcodeStr, v2i64x_info, v2f64x_info, OpNode>,
+                               EVEX_V128;
+    defm Z256 : avx512_vcvt_fp<opc, OpcodeStr, v4i64x_info, v4f64x_info, OpNode>,
+                               EVEX_V256;
+  }
+}
 
-defm VCVTUDQ2PSZ : avx512_vcvt_fp_with_rc<0x7A, "vcvtudq2ps", VR512, VR512, uint_to_fp,
-                                 loadv16i32, f512mem, v16f32, v16i32,
-                                 SSEPackedSingle>, EVEX_V512, XD,
-                                 EVEX_CD8<32, CD8VF>;
+// Convert Signed/Unsigned Quardword to Double
+multiclass avx512_cvtqq2pd<bits<8> opc, string OpcodeStr,
+                                  SDNode OpNode, SDNode OpNodeRnd> {
+  let Predicates = [HasDQI] in {
+    defm Z : avx512_vcvt_fp<opc, OpcodeStr, v8f64_info, v8i64_info, OpNode>,
+             avx512_vcvt_fp_rc<opc, OpcodeStr, v8f64_info, v8i64_info,
+                               OpNodeRnd>, EVEX_V512;
+  }
+  let Predicates = [HasDQI, HasVLX] in {
+    defm Z128 : avx512_vcvt_fp<opc, OpcodeStr, v2f64x_info, v2i64x_info, OpNode>,
+                               EVEX_V128;
+    defm Z256 : avx512_vcvt_fp<opc, OpcodeStr, v4f64x_info, v4i64x_info, OpNode>,
+                               EVEX_V256;
+  }
+}
 
+// Convert Float to Signed/Unsigned Quardword
+multiclass avx512_cvtps2qq<bits<8> opc, string OpcodeStr,
+                                  SDNode OpNode, SDNode OpNodeRnd> {
+  let Predicates = [HasDQI] in {
+    defm Z : avx512_vcvt_fp<opc, OpcodeStr, v8i64_info, v8f32x_info, OpNode>,
+             avx512_vcvt_fp_rc<opc, OpcodeStr, v8i64_info, v8f32x_info,
+                               OpNodeRnd>, EVEX_V512;
+  }
+  let Predicates = [HasDQI, HasVLX] in {
+    // Explicitly specified broadcast string, since we take only 2 elements
+    // from v4f32x_info source
+    defm Z128 : avx512_vcvt_fp<opc, OpcodeStr, v2i64x_info, v4f32x_info, OpNode,
+                               "{1to2}">, EVEX_V128;
+    defm Z256 : avx512_vcvt_fp<opc, OpcodeStr, v4i64x_info, v4f32x_info, OpNode>,
+                               EVEX_V256;
+  }
+}
+
+// Convert Float to Signed/Unsigned Quardword with truncation
+multiclass avx512_cvttps2qq<bits<8> opc, string OpcodeStr,
+                                  SDNode OpNode, SDNode OpNodeRnd> {
+  let Predicates = [HasDQI] in {
+    defm Z : avx512_vcvt_fp<opc, OpcodeStr, v8i64_info, v8f32x_info, OpNode>,
+             avx512_vcvt_fp_sae<opc, OpcodeStr, v8i64_info, v8f32x_info,
+                               OpNodeRnd>, EVEX_V512;
+  }
+  let Predicates = [HasDQI, HasVLX] in {
+    // Explicitly specified broadcast string, since we take only 2 elements
+    // from v4f32x_info source
+    defm Z128 : avx512_vcvt_fp<opc, OpcodeStr, v2i64x_info, v4f32x_info, OpNode,
+                               "{1to2}">, EVEX_V128;
+    defm Z256 : avx512_vcvt_fp<opc, OpcodeStr, v4i64x_info, v4f32x_info, OpNode>,
+                               EVEX_V256;
+  }
+}
+
+// Convert Signed/Unsigned Quardword to Float
+multiclass avx512_cvtqq2ps<bits<8> opc, string OpcodeStr,
+                                  SDNode OpNode, SDNode OpNodeRnd> {
+  let Predicates = [HasDQI] in {
+    defm Z : avx512_vcvt_fp<opc, OpcodeStr, v8f32x_info, v8i64_info, OpNode>,
+             avx512_vcvt_fp_rc<opc, OpcodeStr, v8f32x_info, v8i64_info,
+                               OpNodeRnd>, EVEX_V512;
+  }
+  let Predicates = [HasDQI, HasVLX] in {
+    // we need "x"/"y" suffixes in order to distinguish between 128 and 256
+    // memory forms of these instructions in Asm Parcer. They have the same
+    // dest type - 'v4i32x_info'. We also specify the broadcast string explicitly
+    // due to the same reason.
+    defm Z128 : avx512_vcvt_fp<opc, OpcodeStr, v4f32x_info, v2i64x_info, OpNode,
+                               "{1to2}", "{x}">, EVEX_V128;
+    defm Z256 : avx512_vcvt_fp<opc, OpcodeStr, v4f32x_info, v4i64x_info, OpNode,
+                               "{1to4}", "{y}">, EVEX_V256;
+  }
+}
+
+defm VCVTDQ2PD : avx512_cvtdq2pd<0xE6, "vcvtdq2pd", sint_to_fp, X86cvtdq2pd>, XS,
+                                EVEX_CD8<32, CD8VH>;
+
+defm VCVTDQ2PS : avx512_cvtdq2ps<0x5B, "vcvtdq2ps", sint_to_fp,
+                                X86VSintToFpRnd>,
+                                PS, EVEX_CD8<32, CD8VF>;
+
+defm VCVTTPS2DQ : avx512_cvttps2dq<0x5B, "vcvttps2dq", fp_to_sint,
+                                X86VFpToSintRnd>,
+                                XS, EVEX_CD8<32, CD8VF>;
+
+defm VCVTTPD2DQ : avx512_cvttpd2dq<0xE6, "vcvttpd2dq", fp_to_sint,
+                                 X86VFpToSintRnd>,
+                                 PD, VEX_W, EVEX_CD8<64, CD8VF>;
+
+defm VCVTTPS2UDQ : avx512_cvttps2dq<0x78, "vcvttps2udq", fp_to_uint,
+                                 X86VFpToUintRnd>, PS,
+                                 EVEX_CD8<32, CD8VF>;
+
+defm VCVTTPD2UDQ : avx512_cvttpd2dq<0x78, "vcvttpd2udq", fp_to_uint,
+                                 X86VFpToUintRnd>, PS, VEX_W,
+                                 EVEX_CD8<64, CD8VF>;
+
+defm VCVTUDQ2PD : avx512_cvtdq2pd<0x7A, "vcvtudq2pd", uint_to_fp, X86cvtudq2pd>,
+                                 XS, EVEX_CD8<32, CD8VH>;
+
+defm VCVTUDQ2PS : avx512_cvtdq2ps<0x7A, "vcvtudq2ps", uint_to_fp,
+                                 X86VUintToFpRnd>, XD,
+                                 EVEX_CD8<32, CD8VF>;
+
+defm VCVTPS2DQ : avx512_cvtps2dq<0x5B, "vcvtps2dq", X86cvtps2Int,
+                                 X86cvtps2IntRnd>, PD, EVEX_CD8<32, CD8VF>;
+
+defm VCVTPD2DQ : avx512_cvtpd2dq<0xE6, "vcvtpd2dq", X86cvtpd2Int,
+                                 X86cvtpd2IntRnd>, XD, VEX_W,
+                                 EVEX_CD8<64, CD8VF>;
+
+defm VCVTPS2UDQ : avx512_cvtps2dq<0x79, "vcvtps2udq", X86cvtps2UInt,
+                                 X86cvtps2UIntRnd>,
+                                 PS, EVEX_CD8<32, CD8VF>;
+defm VCVTPD2UDQ : avx512_cvtpd2dq<0x79, "vcvtpd2udq", X86cvtpd2UInt,
+                                 X86cvtpd2UIntRnd>, VEX_W,
+                                 PS, EVEX_CD8<64, CD8VF>;
+
+defm VCVTPD2QQ : avx512_cvtpd2qq<0x7B, "vcvtpd2qq", X86cvtpd2Int,
+                                 X86cvtpd2IntRnd>, VEX_W,
+                                 PD, EVEX_CD8<64, CD8VF>;
+
+defm VCVTPS2QQ : avx512_cvtps2qq<0x7B, "vcvtps2qq", X86cvtps2Int,
+                                 X86cvtps2IntRnd>, PD, EVEX_CD8<32, CD8VH>;
+
+defm VCVTPD2UQQ : avx512_cvtpd2qq<0x79, "vcvtpd2uqq", X86cvtpd2UInt,
+                                 X86cvtpd2UIntRnd>, VEX_W,
+                                 PD, EVEX_CD8<64, CD8VF>;
+
+defm VCVTPS2UQQ : avx512_cvtps2qq<0x79, "vcvtps2uqq", X86cvtps2UInt,
+                                 X86cvtps2UIntRnd>, PD, EVEX_CD8<32, CD8VH>;
+
+defm VCVTTPD2QQ : avx512_cvttpd2qq<0x7A, "vcvttpd2qq", fp_to_sint,
+                                 X86VFpToSlongRnd>, VEX_W,
+                                 PD, EVEX_CD8<64, CD8VF>;
+
+defm VCVTTPS2QQ : avx512_cvttps2qq<0x7A, "vcvttps2qq", fp_to_sint,
+                                 X86VFpToSlongRnd>, PD, EVEX_CD8<32, CD8VH>;
+
+defm VCVTTPD2UQQ : avx512_cvttpd2qq<0x78, "vcvttpd2uqq", fp_to_uint,
+                                 X86VFpToUlongRnd>, VEX_W,
+                                 PD, EVEX_CD8<64, CD8VF>;
+
+defm VCVTTPS2UQQ : avx512_cvttps2qq<0x78, "vcvttps2uqq", fp_to_uint,
+                                 X86VFpToUlongRnd>, PD, EVEX_CD8<32, CD8VH>;
+
+defm VCVTQQ2PD : avx512_cvtqq2pd<0xE6, "vcvtqq2pd", sint_to_fp,
+                            X86VSlongToFpRnd>, VEX_W, XS, EVEX_CD8<64, CD8VF>;
+
+defm VCVTUQQ2PD : avx512_cvtqq2pd<0x7A, "vcvtuqq2pd", uint_to_fp,
+                            X86VUlongToFpRnd>, VEX_W, XS, EVEX_CD8<64, CD8VF>;
+
+defm VCVTQQ2PS : avx512_cvtqq2ps<0x5B, "vcvtqq2ps", sint_to_fp,
+                            X86VSlongToFpRnd>, VEX_W, PS, EVEX_CD8<64, CD8VF>;
+
+defm VCVTUQQ2PS : avx512_cvtqq2ps<0x7A, "vcvtuqq2ps", uint_to_fp,
+                            X86VUlongToFpRnd>, VEX_W, XD, EVEX_CD8<64, CD8VF>;
+
+let Predicates = [NoVLX] in {
 def : Pat<(v8i32 (fp_to_uint (v8f32 VR256X:$src1))),
           (EXTRACT_SUBREG (v16i32 (VCVTTPS2UDQZrr
            (v16f32 (SUBREG_TO_REG (i32 0), VR256X:$src1, sub_ymm)))), sub_ymm)>;
@@ -4264,67 +4990,8 @@ def : Pat<(v4f32 (uint_to_fp (v4i32 VR128X:$src1))),
 def : Pat<(v4f64 (uint_to_fp (v4i32 VR128X:$src1))),
           (EXTRACT_SUBREG (v8f64 (VCVTUDQ2PDZrr
            (v8i32 (SUBREG_TO_REG (i32 0), VR128X:$src1, sub_xmm)))), sub_ymm)>;
-
-def : Pat<(v16f32 (int_x86_avx512_mask_cvtdq2ps_512 (v16i32 VR512:$src),
-                   (bc_v16f32 (v16i32 immAllZerosV)), (i16 -1), imm:$rc)),
-          (VCVTDQ2PSZrrb VR512:$src, imm:$rc)>;
-def : Pat<(v8f64 (int_x86_avx512_mask_cvtdq2pd_512 (v8i32 VR256X:$src),
-                   (bc_v8f64 (v16i32 immAllZerosV)), (i8 -1))),
-          (VCVTDQ2PDZrr VR256X:$src)>;
-def : Pat<(v16f32 (int_x86_avx512_mask_cvtudq2ps_512 (v16i32 VR512:$src),
-                   (bc_v16f32 (v16i32 immAllZerosV)), (i16 -1), imm:$rc)),
-          (VCVTUDQ2PSZrrb VR512:$src, imm:$rc)>;
-def : Pat<(v8f64 (int_x86_avx512_mask_cvtudq2pd_512 (v8i32 VR256X:$src),
-                   (bc_v8f64 (v16i32 immAllZerosV)), (i8 -1))),
-          (VCVTUDQ2PDZrr VR256X:$src)>;
-
-multiclass avx512_vcvt_fp2int<bits<8> opc, string asm, RegisterClass SrcRC,
-               RegisterClass DstRC, PatFrag mem_frag,
-               X86MemOperand x86memop, Domain d> {
-let hasSideEffects = 0 in {
-  def rr : AVX512PI<opc, MRMSrcReg, (outs DstRC:$dst), (ins SrcRC:$src),
-              !strconcat(asm,"\t{$src, $dst|$dst, $src}"),
-              [], d>, EVEX;
-  def rrb : AVX512PI<opc, MRMSrcReg, (outs DstRC:$dst), (ins SrcRC:$src, AVX512RC:$rc),
-              !strconcat(asm,"\t{$rc, $src, $dst|$dst, $src, $rc}"),
-              [], d>, EVEX, EVEX_B, EVEX_RC;
-  let mayLoad = 1 in
-  def rm : AVX512PI<opc, MRMSrcMem, (outs DstRC:$dst), (ins x86memop:$src),
-              !strconcat(asm,"\t{$src, $dst|$dst, $src}"),
-              [], d>, EVEX;
-} // hasSideEffects = 0
 }
 
-defm VCVTPS2DQZ : avx512_vcvt_fp2int<0x5B, "vcvtps2dq", VR512, VR512,
-                                 loadv16f32, f512mem, SSEPackedSingle>, PD,
-                                 EVEX_V512, EVEX_CD8<32, CD8VF>;
-defm VCVTPD2DQZ : avx512_vcvt_fp2int<0xE6, "vcvtpd2dq", VR512, VR256X,
-                                 loadv8f64, f512mem, SSEPackedDouble>, XD, VEX_W,
-                                 EVEX_V512, EVEX_CD8<64, CD8VF>;
-
-def : Pat <(v16i32 (int_x86_avx512_mask_cvtps2dq_512 (v16f32 VR512:$src),
-                    (v16i32 immAllZerosV), (i16 -1), imm:$rc)),
-           (VCVTPS2DQZrrb VR512:$src, imm:$rc)>;
-
-def : Pat <(v8i32 (int_x86_avx512_mask_cvtpd2dq_512 (v8f64 VR512:$src),
-                    (v8i32 immAllZerosV), (i8 -1), imm:$rc)),
-           (VCVTPD2DQZrrb VR512:$src, imm:$rc)>;
-
-defm VCVTPS2UDQZ : avx512_vcvt_fp2int<0x79, "vcvtps2udq", VR512, VR512,
-                                 loadv16f32, f512mem, SSEPackedSingle>,
-                                 PS, EVEX_V512, EVEX_CD8<32, CD8VF>;
-defm VCVTPD2UDQZ : avx512_vcvt_fp2int<0x79, "vcvtpd2udq", VR512, VR256X,
-                                 loadv8f64, f512mem, SSEPackedDouble>, VEX_W,
-                                 PS, EVEX_V512, EVEX_CD8<64, CD8VF>;
-
-def : Pat <(v16i32 (int_x86_avx512_mask_cvtps2udq_512 (v16f32 VR512:$src),
-                    (v16i32 immAllZerosV), (i16 -1), imm:$rc)),
-           (VCVTPS2UDQZrrb VR512:$src, imm:$rc)>;
-
-def : Pat <(v8i32 (int_x86_avx512_mask_cvtpd2udq_512 (v8f64 VR512:$src),
-                    (v8i32 immAllZerosV), (i8 -1), imm:$rc)),
-           (VCVTPD2UDQZrrb VR512:$src, imm:$rc)>;
-
 let Predicates = [HasAVX512] in {
   def : Pat<(v8f32 (fround (loadv8f64 addr:$src))),
             (VCVTPD2PSZrm addr:$src)>;
@@ -4520,9 +5187,9 @@ multiclass avx512_fp28_s<bits<8> opc, string OpcodeStr,X86VectorVTInfo _,
 
   defm rb : AVX512_maskable_scalar<opc, MRMSrcReg, _, (outs _.RC:$dst),
                             (ins _.RC:$src1, _.RC:$src2), OpcodeStr,
-                            "$src2, $src1", "$src1, $src2",
+                            "{sae}, $src2, $src1", "$src1, $src2, {sae}",
                             (OpNode (_.VT _.RC:$src1), (_.VT _.RC:$src2),
-                            (i32 FROUND_NO_EXC)), "{sae}">, EVEX_B;
+                            (i32 FROUND_NO_EXC))>, EVEX_B;
 
   defm m : AVX512_maskable_scalar<opc, MRMSrcMem, _, (outs _.RC:$dst),
                          (ins _.RC:$src1, _.MemOp:$src2), OpcodeStr,
@@ -4543,6 +5210,8 @@ let hasSideEffects = 0, Predicates = [HasERI] in {
   defm VRCP28   : avx512_eri_s<0xCB, "vrcp28",   X86rcp28s>,   T8PD, EVEX_4V;
   defm VRSQRT28 : avx512_eri_s<0xCD, "vrsqrt28", X86rsqrt28s>, T8PD, EVEX_4V;
 }
+
+defm VGETEXP   : avx512_eri_s<0x43, "vgetexp", X86fgetexpRnds>, T8PD, EVEX_4V;
 /// avx512_fp28_p rcp28ps, rcp28pd, rsqrt28ps, rsqrt28pd
 
 multiclass avx512_fp28_p<bits<8> opc, string OpcodeStr, X86VectorVTInfo _,
@@ -4552,12 +5221,6 @@ multiclass avx512_fp28_p<bits<8> opc, string OpcodeStr, X86VectorVTInfo _,
                          (ins _.RC:$src), OpcodeStr, "$src", "$src",
                          (OpNode (_.VT _.RC:$src), (i32 FROUND_CURRENT))>;
 
-  defm rb : AVX512_maskable<opc, MRMSrcReg, _, (outs _.RC:$dst),
-                        (ins _.RC:$src), OpcodeStr,
-                        "$src", "$src",
-                        (OpNode (_.VT _.RC:$src), (i32 FROUND_NO_EXC)),
-                        "{sae}">, EVEX_B;
-
   defm m : AVX512_maskable<opc, MRMSrcMem, _, (outs _.RC:$dst),
                          (ins _.MemOp:$src), OpcodeStr, "$src", "$src",
                          (OpNode (_.FloatVT
@@ -4565,24 +5228,58 @@ multiclass avx512_fp28_p<bits<8> opc, string OpcodeStr, X86VectorVTInfo _,
                           (i32 FROUND_CURRENT))>;
 
   defm mb : AVX512_maskable<opc, MRMSrcMem, _, (outs _.RC:$dst),
-                         (ins _.MemOp:$src), OpcodeStr, "$src", "$src",
+                         (ins _.MemOp:$src), OpcodeStr,
+                         "${src}"##_.BroadcastStr, "${src}"##_.BroadcastStr,
                          (OpNode (_.FloatVT
                                   (X86VBroadcast (_.ScalarLdFrag addr:$src))),
                                  (i32 FROUND_CURRENT))>, EVEX_B;
 }
+multiclass avx512_fp28_p_round<bits<8> opc, string OpcodeStr, X86VectorVTInfo _,
+                         SDNode OpNode> {
+  defm rb : AVX512_maskable<opc, MRMSrcReg, _, (outs _.RC:$dst),
+                        (ins _.RC:$src), OpcodeStr,
+                        "{sae}, $src", "$src, {sae}",
+                        (OpNode (_.VT _.RC:$src), (i32 FROUND_NO_EXC))>, EVEX_B;
+}
 
 multiclass  avx512_eri<bits<8> opc, string OpcodeStr, SDNode OpNode> {
    defm PS : avx512_fp28_p<opc, OpcodeStr#"ps", v16f32_info, OpNode>,
-                     EVEX_CD8<32, CD8VF>;
+             avx512_fp28_p_round<opc, OpcodeStr#"ps", v16f32_info, OpNode>,
+             T8PD, EVEX_V512, EVEX_CD8<32, CD8VF>;
    defm PD : avx512_fp28_p<opc, OpcodeStr#"pd", v8f64_info, OpNode>,
-                     VEX_W, EVEX_CD8<32, CD8VF>;
+             avx512_fp28_p_round<opc, OpcodeStr#"pd", v8f64_info, OpNode>,
+             T8PD, EVEX_V512, VEX_W, EVEX_CD8<64, CD8VF>;
 }
 
+multiclass avx512_fp_unaryop_packed<bits<8> opc, string OpcodeStr,
+                                  SDNode OpNode> {
+  // Define only if AVX512VL feature is present.
+  let Predicates = [HasVLX] in {
+    defm PSZ128 : avx512_fp28_p<opc, OpcodeStr#"ps", v4f32x_info, OpNode>,
+                                     EVEX_V128, T8PD, EVEX_CD8<32, CD8VF>;
+    defm PSZ256 : avx512_fp28_p<opc, OpcodeStr#"ps", v8f32x_info, OpNode>,
+                                     EVEX_V256, T8PD, EVEX_CD8<32, CD8VF>;
+    defm PDZ128 : avx512_fp28_p<opc, OpcodeStr#"pd", v2f64x_info, OpNode>,
+                                     EVEX_V128, VEX_W, T8PD, EVEX_CD8<64, CD8VF>;
+    defm PDZ256 : avx512_fp28_p<opc, OpcodeStr#"pd", v4f64x_info, OpNode>,
+                                     EVEX_V256, VEX_W, T8PD, EVEX_CD8<64, CD8VF>;
+  }
+}
 let Predicates = [HasERI], hasSideEffects = 0 in {
 
- defm VRSQRT28 : avx512_eri<0xCC, "vrsqrt28", X86rsqrt28>, EVEX, EVEX_V512, T8PD;
- defm VRCP28   : avx512_eri<0xCA, "vrcp28",   X86rcp28>,   EVEX, EVEX_V512, T8PD;
- defm VEXP2    : avx512_eri<0xC8, "vexp2",    X86exp2>,    EVEX, EVEX_V512, T8PD;
+ defm VRSQRT28 : avx512_eri<0xCC, "vrsqrt28", X86rsqrt28>, EVEX;
+ defm VRCP28   : avx512_eri<0xCA, "vrcp28",   X86rcp28>,   EVEX;
+ defm VEXP2    : avx512_eri<0xC8, "vexp2",    X86exp2>,    EVEX;
+}
+defm VGETEXP   : avx512_eri<0x42, "vgetexp", X86fgetexpRnd>,
+                 avx512_fp_unaryop_packed<0x42, "vgetexp", X86fgetexpRnd> , EVEX;
+
+multiclass avx512_sqrt_packed_round<bits<8> opc, string OpcodeStr,
+                              SDNode OpNodeRnd, X86VectorVTInfo _>{
+  defm rb: AVX512_maskable<opc, MRMSrcReg, _, (outs _.RC:$dst),
+                         (ins _.RC:$src, AVX512RC:$rc), OpcodeStr, "$rc, $src", "$src, $rc",
+                         (_.VT (OpNodeRnd _.RC:$src, (i32 imm:$rc)))>,
+                         EVEX, EVEX_B, EVEX_RC;
 }
 
 multiclass avx512_sqrt_packed<bits<8> opc, string OpcodeStr,
@@ -4691,20 +5388,22 @@ multiclass avx512_sqrt_packed_all<bits<8> opc, string OpcodeStr,
   }
 }
 
-defm VSQRT : avx512_sqrt_packed_all<0x51, "vsqrt", fsqrt>;
+multiclass avx512_sqrt_packed_all_round<bits<8> opc, string OpcodeStr,
+                                          SDNode OpNodeRnd> {
+  defm PSZ : avx512_sqrt_packed_round<opc, !strconcat(OpcodeStr, "ps"), OpNodeRnd,
+                                v16f32_info>, EVEX_V512, PS, EVEX_CD8<32, CD8VF>;
+  defm PDZ : avx512_sqrt_packed_round<opc, !strconcat(OpcodeStr, "pd"), OpNodeRnd,
+                                v8f64_info>, EVEX_V512, VEX_W, PD, EVEX_CD8<64, CD8VF>;
+}
+
+defm VSQRT   : avx512_sqrt_packed_all<0x51, "vsqrt", fsqrt>,
+               avx512_sqrt_packed_all_round<0x51, "vsqrt", X86fsqrtRnd>;
 
 defm VSQRT  : avx512_sqrt_scalar<0x51, "sqrt",
                 int_x86_avx512_sqrt_ss, int_x86_avx512_sqrt_sd,
                 SSE_SQRTSS, SSE_SQRTSD>;
 
 let Predicates = [HasAVX512] in {
-  def : Pat<(v16f32 (int_x86_avx512_sqrt_ps_512 (v16f32 VR512:$src1),
-                    (bc_v16f32 (v16i32 immAllZerosV)), (i16 -1), FROUND_CURRENT)),
-                   (VSQRTPSZr VR512:$src1)>;
-  def : Pat<(v8f64 (int_x86_avx512_sqrt_pd_512 (v8f64 VR512:$src1),
-                    (bc_v8f64 (v16i32 immAllZerosV)), (i8 -1), FROUND_CURRENT)),
-                   (VSQRTPDZr VR512:$src1)>;
-
   def : Pat<(f32 (fsqrt FR32X:$src)),
             (VSQRTSSZr (f32 (IMPLICIT_DEF)), FR32X:$src)>;
   def : Pat<(f32 (fsqrt (load addr:$src))),
@@ -4743,47 +5442,6 @@ let Predicates = [HasAVX512] in {
             (VSQRTSDZm_Int (v2f64 (IMPLICIT_DEF)), sse_load_f64:$src)>;
 }
 
-
-multiclass avx512_rndscale<bits<8> opc, string OpcodeStr,
-                            X86MemOperand x86memop, RegisterClass RC,
-                            PatFrag mem_frag, Domain d> {
-let ExeDomain = d in {
-  // Intrinsic operation, reg.
-  // Vector intrinsic operation, reg
-  def r : AVX512AIi8<opc, MRMSrcReg,
-                    (outs RC:$dst), (ins RC:$src1, i32u8imm:$src2),
-                    !strconcat(OpcodeStr,
-                    "\t{$src2, $src1, $dst|$dst, $src1, $src2}"),
-                    []>, EVEX;
-
-  // Vector intrinsic operation, mem
-  def m : AVX512AIi8<opc, MRMSrcMem,
-                    (outs RC:$dst), (ins x86memop:$src1, i32u8imm:$src2),
-                    !strconcat(OpcodeStr,
-                    "\t{$src2, $src1, $dst|$dst, $src1, $src2}"),
-                    []>, EVEX;
-} // ExeDomain
-}
-
-defm VRNDSCALEPSZ : avx512_rndscale<0x08, "vrndscaleps", f512mem, VR512,
-                                loadv16f32, SSEPackedSingle>, EVEX_V512,
-                                EVEX_CD8<32, CD8VF>;
-
-def : Pat<(v16f32 (int_x86_avx512_mask_rndscale_ps_512 (v16f32 VR512:$src1),
-                   imm:$src2, (v16f32 VR512:$src1), (i16 -1),
-                   FROUND_CURRENT)),
-                   (VRNDSCALEPSZr VR512:$src1, imm:$src2)>;
-
-
-defm VRNDSCALEPDZ : avx512_rndscale<0x09, "vrndscalepd", f512mem, VR512,
-                                loadv8f64, SSEPackedDouble>, EVEX_V512,
-                                VEX_W, EVEX_CD8<64, CD8VF>;
-
-def : Pat<(v8f64 (int_x86_avx512_mask_rndscale_pd_512 (v8f64 VR512:$src1),
-                  imm:$src2, (v8f64 VR512:$src1), (i8 -1),
-                  FROUND_CURRENT)),
-                   (VRNDSCALEPDZr VR512:$src1, imm:$src2)>;
-
 multiclass
 avx512_rndscale_scalar<bits<8> opc, string OpcodeStr, X86VectorVTInfo _> {
 
@@ -4791,20 +5449,20 @@ avx512_rndscale_scalar<bits<8> opc, string OpcodeStr, X86VectorVTInfo _> {
   defm r : AVX512_maskable_scalar<opc, MRMSrcReg, _, (outs _.RC:$dst),
                            (ins _.RC:$src1, _.RC:$src2, i32u8imm:$src3), OpcodeStr,
                            "$src3, $src2, $src1", "$src1, $src2, $src3",
-                           (_.VT (X86RndScale (_.VT _.RC:$src1), (_.VT _.RC:$src2),
+                           (_.VT (X86RndScales (_.VT _.RC:$src1), (_.VT _.RC:$src2),
                             (i32 imm:$src3), (i32 FROUND_CURRENT)))>;
 
   defm rb : AVX512_maskable_scalar<opc, MRMSrcReg, _, (outs _.RC:$dst),
                          (ins _.RC:$src1, _.RC:$src2, i32u8imm:$src3), OpcodeStr,
-                         "$src3, $src2, $src1", "$src1, $src2, $src3",
-                         (_.VT (X86RndScale (_.VT _.RC:$src1), (_.VT _.RC:$src2),
-                         (i32 imm:$src3), (i32 FROUND_NO_EXC))), "{sae}">, EVEX_B;
+                         "$src3, {sae}, $src2, $src1", "$src1, $src2, {sae}, $src3",
+                         (_.VT (X86RndScales (_.VT _.RC:$src1), (_.VT _.RC:$src2),
+                         (i32 imm:$src3), (i32 FROUND_NO_EXC)))>, EVEX_B;
 
   let mayLoad = 1 in
   defm m : AVX512_maskable_scalar<opc, MRMSrcMem, _, (outs _.RC:$dst),
                          (ins _.RC:$src1, _.MemOp:$src2, i32u8imm:$src3), OpcodeStr,
                          "$src3, $src2, $src1", "$src1, $src2, $src3",
-                         (_.VT (X86RndScale (_.VT _.RC:$src1),
+                         (_.VT (X86RndScales (_.VT _.RC:$src1),
                           (_.VT (scalar_to_vector (_.ScalarLdFrag addr:$src2))),
                           (i32 imm:$src3), (i32 FROUND_CURRENT)))>;
   }
@@ -4849,255 +5507,478 @@ defm VRNDSCALESS : avx512_rndscale_scalar<0x0A, "vrndscaless", f32x_info>,
 defm VRNDSCALESD : avx512_rndscale_scalar<0x0B, "vrndscalesd", f64x_info>, VEX_W,
                                 AVX512AIi8Base, EVEX_4V, EVEX_CD8<64, CD8VT1>;
 
-let Predicates = [HasAVX512] in {
-def : Pat<(v16f32 (ffloor VR512:$src)),
-          (VRNDSCALEPSZr VR512:$src, (i32 0x1))>;
-def : Pat<(v16f32 (fnearbyint VR512:$src)),
-          (VRNDSCALEPSZr VR512:$src, (i32 0xC))>;
-def : Pat<(v16f32 (fceil VR512:$src)),
-          (VRNDSCALEPSZr VR512:$src, (i32 0x2))>;
-def : Pat<(v16f32 (frint VR512:$src)),
-          (VRNDSCALEPSZr VR512:$src, (i32 0x4))>;
-def : Pat<(v16f32 (ftrunc VR512:$src)),
-          (VRNDSCALEPSZr VR512:$src, (i32 0x3))>;
-
-def : Pat<(v8f64 (ffloor VR512:$src)),
-          (VRNDSCALEPDZr VR512:$src, (i32 0x1))>;
-def : Pat<(v8f64 (fnearbyint VR512:$src)),
-          (VRNDSCALEPDZr VR512:$src, (i32 0xC))>;
-def : Pat<(v8f64 (fceil VR512:$src)),
-          (VRNDSCALEPDZr VR512:$src, (i32 0x2))>;
-def : Pat<(v8f64 (frint VR512:$src)),
-          (VRNDSCALEPDZr VR512:$src, (i32 0x4))>;
-def : Pat<(v8f64 (ftrunc VR512:$src)),
-          (VRNDSCALEPDZr VR512:$src, (i32 0x3))>;
-}
 //-------------------------------------------------
 // Integer truncate and extend operations
 //-------------------------------------------------
 
-multiclass avx512_trunc_sat<bits<8> opc, string OpcodeStr,
-                          RegisterClass dstRC, RegisterClass srcRC,
-                          RegisterClass KRC, X86MemOperand x86memop> {
-  def rr : AVX512XS8I<opc, MRMDestReg, (outs dstRC:$dst),
-               (ins srcRC:$src),
-               !strconcat(OpcodeStr,"\t{$src, $dst|$dst, $src}"),
+multiclass avx512_trunc_common<bits<8> opc, string OpcodeStr, SDNode OpNode,
+                              X86VectorVTInfo SrcInfo, X86VectorVTInfo DestInfo,
+                              X86MemOperand x86memop> {
+
+  defm rr  : AVX512_maskable<opc, MRMDestReg, DestInfo, (outs DestInfo.RC:$dst),
+                      (ins SrcInfo.RC:$src1), OpcodeStr ,"$src1", "$src1",
+                      (DestInfo.VT (OpNode (SrcInfo.VT SrcInfo.RC:$src1)))>,
+                       EVEX, T8XS;
+
+  // for intrinsic patter match
+  def : Pat<(DestInfo.VT (X86select DestInfo.KRCWM:$mask,
+                           (DestInfo.VT (OpNode (SrcInfo.VT SrcInfo.RC:$src1))),
+                           undef)),
+            (!cast<Instruction>(NAME#SrcInfo.ZSuffix##rrkz) DestInfo.KRCWM:$mask ,
+                                      SrcInfo.RC:$src1)>;
+
+  def : Pat<(DestInfo.VT (X86select DestInfo.KRCWM:$mask,
+                           (DestInfo.VT (OpNode (SrcInfo.VT SrcInfo.RC:$src1))),
+                           DestInfo.ImmAllZerosV)),
+            (!cast<Instruction>(NAME#SrcInfo.ZSuffix##rrkz) DestInfo.KRCWM:$mask ,
+                                      SrcInfo.RC:$src1)>;
+
+  def : Pat<(DestInfo.VT (X86select DestInfo.KRCWM:$mask,
+                           (DestInfo.VT (OpNode (SrcInfo.VT SrcInfo.RC:$src1))),
+                           DestInfo.RC:$src0)),
+            (!cast<Instruction>(NAME#SrcInfo.ZSuffix##rrk) DestInfo.RC:$src0,
+                                      DestInfo.KRCWM:$mask ,
+                                      SrcInfo.RC:$src1)>;
+
+  let mayStore = 1 in {
+    def mr : AVX512XS8I<opc, MRMDestMem, (outs),
+               (ins x86memop:$dst, SrcInfo.RC:$src),
+               OpcodeStr # "\t{$src, $dst |$dst, $src}",
                []>, EVEX;
 
-  def rrk : AVX512XS8I<opc, MRMDestReg, (outs dstRC:$dst),
-               (ins KRC:$mask, srcRC:$src),
-               !strconcat(OpcodeStr,
-                 "\t{$src, ${dst} {${mask}}|${dst} {${mask}}, $src}"),
+    def mrk : AVX512XS8I<opc, MRMDestMem, (outs),
+               (ins x86memop:$dst, SrcInfo.KRCWM:$mask, SrcInfo.RC:$src),
+               OpcodeStr # "\t{$src, $dst {${mask}} |$dst {${mask}}, $src}",
                []>, EVEX, EVEX_K;
+  }//mayStore = 1
+}
 
-  def rrkz : AVX512XS8I<opc, MRMDestReg, (outs dstRC:$dst),
-               (ins KRC:$mask, srcRC:$src),
-               !strconcat(OpcodeStr,
-                 "\t{$src, ${dst} {${mask}} {z}|${dst} {${mask}} {z}, $src}"),
-               []>, EVEX, EVEX_KZ;
+multiclass avx512_trunc_mr_lowering<X86VectorVTInfo SrcInfo,
+                                    X86VectorVTInfo DestInfo,
+                                    PatFrag truncFrag, PatFrag mtruncFrag > {
 
-  def mr : AVX512XS8I<opc, MRMDestMem, (outs), (ins x86memop:$dst, srcRC:$src),
-               !strconcat(OpcodeStr, "\t{$src, $dst|$dst, $src}"),
-               []>, EVEX;
+  def : Pat<(truncFrag (SrcInfo.VT SrcInfo.RC:$src), addr:$dst),
+            (!cast<Instruction>(NAME#SrcInfo.ZSuffix##mr)
+                                    addr:$dst, SrcInfo.RC:$src)>;
 
-  def mrk : AVX512XS8I<opc, MRMDestMem, (outs),
-               (ins x86memop:$dst, KRC:$mask, srcRC:$src),
-               !strconcat(OpcodeStr, "\t{$src, $dst {${mask}}|${dst} {${mask}}, $src}"),
-               []>, EVEX, EVEX_K;
+  def : Pat<(mtruncFrag addr:$dst, SrcInfo.KRCWM:$mask,
+                                               (SrcInfo.VT SrcInfo.RC:$src)),
+            (!cast<Instruction>(NAME#SrcInfo.ZSuffix##mrk)
+                            addr:$dst, SrcInfo.KRCWM:$mask, SrcInfo.RC:$src)>;
+}
+
+multiclass avx512_trunc_sat_mr_lowering<X86VectorVTInfo SrcInfo,
+                                        X86VectorVTInfo DestInfo, string sat > {
 
+  def: Pat<(!cast<Intrinsic>("int_x86_avx512_mask_pmov"#sat#"_"#SrcInfo.Suffix#
+                               DestInfo.Suffix#"_mem_"#SrcInfo.Size)
+                  addr:$ptr, (SrcInfo.VT SrcInfo.RC:$src), SrcInfo.MRC:$mask),
+           (!cast<Instruction>(NAME#SrcInfo.ZSuffix##mrk) addr:$ptr,
+                    (COPY_TO_REGCLASS SrcInfo.MRC:$mask, SrcInfo.KRCWM),
+                    (SrcInfo.VT SrcInfo.RC:$src))>;
+
+  def: Pat<(!cast<Intrinsic>("int_x86_avx512_mask_pmov"#sat#"_"#SrcInfo.Suffix#
+                               DestInfo.Suffix#"_mem_"#SrcInfo.Size)
+                  addr:$ptr, (SrcInfo.VT SrcInfo.RC:$src), -1),
+           (!cast<Instruction>(NAME#SrcInfo.ZSuffix##mr) addr:$ptr,
+                    (SrcInfo.VT SrcInfo.RC:$src))>;
+}
+
+multiclass avx512_trunc<bits<8> opc, string OpcodeStr, SDNode OpNode,
+         AVX512VLVectorVTInfo VTSrcInfo, X86VectorVTInfo DestInfoZ128,
+         X86VectorVTInfo DestInfoZ256, X86VectorVTInfo DestInfoZ,
+         X86MemOperand x86memopZ128, X86MemOperand x86memopZ256,
+         X86MemOperand x86memopZ, PatFrag truncFrag, PatFrag mtruncFrag,
+                                                     Predicate prd = HasAVX512>{
+
+  let Predicates = [HasVLX, prd] in {
+    defm Z128:  avx512_trunc_common<opc, OpcodeStr, OpNode, VTSrcInfo.info128,
+                             DestInfoZ128, x86memopZ128>,
+                avx512_trunc_mr_lowering<VTSrcInfo.info128, DestInfoZ128,
+                             truncFrag, mtruncFrag>, EVEX_V128;
+
+    defm Z256:  avx512_trunc_common<opc, OpcodeStr, OpNode, VTSrcInfo.info256,
+                             DestInfoZ256, x86memopZ256>,
+                avx512_trunc_mr_lowering<VTSrcInfo.info256, DestInfoZ256,
+                             truncFrag, mtruncFrag>, EVEX_V256;
+  }
+  let Predicates = [prd] in
+    defm Z:     avx512_trunc_common<opc, OpcodeStr, OpNode, VTSrcInfo.info512,
+                             DestInfoZ, x86memopZ>,
+                avx512_trunc_mr_lowering<VTSrcInfo.info512, DestInfoZ,
+                             truncFrag, mtruncFrag>, EVEX_V512;
+}
+
+multiclass avx512_trunc_sat<bits<8> opc, string OpcodeStr, SDNode OpNode,
+         AVX512VLVectorVTInfo VTSrcInfo, X86VectorVTInfo DestInfoZ128,
+         X86VectorVTInfo DestInfoZ256, X86VectorVTInfo DestInfoZ,
+         X86MemOperand x86memopZ128, X86MemOperand x86memopZ256,
+         X86MemOperand x86memopZ, string sat, Predicate prd = HasAVX512>{
+
+  let Predicates = [HasVLX, prd] in {
+    defm Z128:  avx512_trunc_common<opc, OpcodeStr, OpNode, VTSrcInfo.info128,
+                             DestInfoZ128, x86memopZ128>,
+                avx512_trunc_sat_mr_lowering<VTSrcInfo.info128, DestInfoZ128,
+                             sat>, EVEX_V128;
+
+    defm Z256:  avx512_trunc_common<opc, OpcodeStr, OpNode, VTSrcInfo.info256,
+                             DestInfoZ256, x86memopZ256>,
+                avx512_trunc_sat_mr_lowering<VTSrcInfo.info256, DestInfoZ256,
+                             sat>, EVEX_V256;
+  }
+  let Predicates = [prd] in
+    defm Z:     avx512_trunc_common<opc, OpcodeStr, OpNode, VTSrcInfo.info512,
+                             DestInfoZ, x86memopZ>,
+                avx512_trunc_sat_mr_lowering<VTSrcInfo.info512, DestInfoZ,
+                             sat>, EVEX_V512;
+}
+
+multiclass avx512_trunc_qb<bits<8> opc, string OpcodeStr, SDNode OpNode> {
+  defm NAME: avx512_trunc<opc, OpcodeStr, OpNode, avx512vl_i64_info,
+               v16i8x_info, v16i8x_info, v16i8x_info, i16mem, i32mem, i64mem,
+               truncstorevi8, masked_truncstorevi8>, EVEX_CD8<8, CD8VO>;
+}
+multiclass avx512_trunc_sat_qb<bits<8> opc, string sat, SDNode OpNode> {
+  defm NAME: avx512_trunc_sat<opc, "vpmov"##sat##"qb", OpNode, avx512vl_i64_info,
+               v16i8x_info, v16i8x_info, v16i8x_info, i16mem, i32mem, i64mem,
+               sat>, EVEX_CD8<8, CD8VO>;
+}
+
+multiclass avx512_trunc_qw<bits<8> opc, string OpcodeStr, SDNode OpNode> {
+  defm NAME: avx512_trunc<opc, OpcodeStr, OpNode, avx512vl_i64_info,
+               v8i16x_info, v8i16x_info, v8i16x_info, i32mem, i64mem, i128mem,
+               truncstorevi16, masked_truncstorevi16>, EVEX_CD8<16, CD8VQ>;
+}
+multiclass avx512_trunc_sat_qw<bits<8> opc, string sat, SDNode OpNode> {
+  defm NAME: avx512_trunc_sat<opc, "vpmov"##sat##"qw", OpNode, avx512vl_i64_info,
+               v8i16x_info, v8i16x_info, v8i16x_info, i32mem, i64mem, i128mem,
+               sat>, EVEX_CD8<16, CD8VQ>;
+}
+
+multiclass avx512_trunc_qd<bits<8> opc, string OpcodeStr, SDNode OpNode> {
+  defm NAME: avx512_trunc<opc, OpcodeStr, OpNode, avx512vl_i64_info,
+               v4i32x_info, v4i32x_info, v8i32x_info, i64mem, i128mem, i256mem,
+               truncstorevi32, masked_truncstorevi32>, EVEX_CD8<32, CD8VH>;
+}
+multiclass avx512_trunc_sat_qd<bits<8> opc, string sat, SDNode OpNode> {
+  defm NAME: avx512_trunc_sat<opc, "vpmov"##sat##"qd", OpNode, avx512vl_i64_info,
+               v4i32x_info, v4i32x_info, v8i32x_info, i64mem, i128mem, i256mem,
+               sat>, EVEX_CD8<32, CD8VH>;
+}
+
+multiclass avx512_trunc_db<bits<8> opc, string OpcodeStr, SDNode OpNode> {
+  defm NAME: avx512_trunc<opc, OpcodeStr, OpNode, avx512vl_i32_info,
+               v16i8x_info, v16i8x_info, v16i8x_info, i32mem, i64mem, i128mem,
+               truncstorevi8, masked_truncstorevi8>, EVEX_CD8<8, CD8VQ>;
+}
+multiclass avx512_trunc_sat_db<bits<8> opc, string sat, SDNode OpNode> {
+  defm NAME: avx512_trunc_sat<opc, "vpmov"##sat##"db", OpNode, avx512vl_i32_info,
+               v16i8x_info, v16i8x_info, v16i8x_info, i32mem, i64mem, i128mem,
+               sat>, EVEX_CD8<8, CD8VQ>;
+}
+
+multiclass avx512_trunc_dw<bits<8> opc, string OpcodeStr, SDNode OpNode> {
+  defm NAME: avx512_trunc<opc, OpcodeStr, OpNode, avx512vl_i32_info,
+              v8i16x_info, v8i16x_info, v16i16x_info, i64mem, i128mem, i256mem,
+              truncstorevi16, masked_truncstorevi16>, EVEX_CD8<16, CD8VH>;
+}
+multiclass avx512_trunc_sat_dw<bits<8> opc, string sat, SDNode OpNode> {
+  defm NAME: avx512_trunc_sat<opc, "vpmov"##sat##"dw", OpNode, avx512vl_i32_info,
+              v8i16x_info, v8i16x_info, v16i16x_info, i64mem, i128mem, i256mem,
+              sat>, EVEX_CD8<16, CD8VH>;
 }
-defm VPMOVQB    : avx512_trunc_sat<0x32, "vpmovqb",   VR128X, VR512, VK8WM,
-                                 i128mem>, EVEX_V512, EVEX_CD8<8, CD8VO>;
-defm VPMOVSQB   : avx512_trunc_sat<0x22, "vpmovsqb",  VR128X, VR512, VK8WM,
-                                 i128mem>, EVEX_V512, EVEX_CD8<8, CD8VO>;
-defm VPMOVUSQB  : avx512_trunc_sat<0x12, "vpmovusqb", VR128X, VR512, VK8WM,
-                                 i128mem>, EVEX_V512, EVEX_CD8<8, CD8VO>;
-defm VPMOVQW    : avx512_trunc_sat<0x34, "vpmovqw",   VR128X, VR512, VK8WM,
-                                 i128mem>, EVEX_V512, EVEX_CD8<16, CD8VQ>;
-defm VPMOVSQW   : avx512_trunc_sat<0x24, "vpmovsqw",  VR128X, VR512, VK8WM,
-                                 i128mem>, EVEX_V512, EVEX_CD8<16, CD8VQ>;
-defm VPMOVUSQW  : avx512_trunc_sat<0x14, "vpmovusqw", VR128X, VR512, VK8WM,
-                                 i128mem>, EVEX_V512, EVEX_CD8<16, CD8VQ>;
-defm VPMOVQD    : avx512_trunc_sat<0x35, "vpmovqd",   VR256X, VR512, VK8WM,
-                                 i256mem>, EVEX_V512, EVEX_CD8<32, CD8VH>;
-defm VPMOVSQD   : avx512_trunc_sat<0x25, "vpmovsqd",  VR256X, VR512, VK8WM,
-                                 i256mem>, EVEX_V512, EVEX_CD8<32, CD8VH>;
-defm VPMOVUSQD  : avx512_trunc_sat<0x15, "vpmovusqd", VR256X, VR512, VK8WM,
-                                 i256mem>, EVEX_V512, EVEX_CD8<32, CD8VH>;
-defm VPMOVDW    : avx512_trunc_sat<0x33, "vpmovdw",   VR256X, VR512, VK16WM,
-                                 i256mem>, EVEX_V512, EVEX_CD8<16, CD8VH>;
-defm VPMOVSDW   : avx512_trunc_sat<0x23, "vpmovsdw",  VR256X, VR512, VK16WM,
-                                 i256mem>, EVEX_V512, EVEX_CD8<16, CD8VH>;
-defm VPMOVUSDW  : avx512_trunc_sat<0x13, "vpmovusdw", VR256X, VR512, VK16WM,
-                                 i256mem>, EVEX_V512, EVEX_CD8<16, CD8VH>;
-defm VPMOVDB    : avx512_trunc_sat<0x31, "vpmovdb",   VR128X, VR512, VK16WM,
-                                 i128mem>, EVEX_V512, EVEX_CD8<8, CD8VQ>;
-defm VPMOVSDB   : avx512_trunc_sat<0x21, "vpmovsdb",  VR128X, VR512, VK16WM,
-                                 i128mem>, EVEX_V512, EVEX_CD8<8, CD8VQ>;
-defm VPMOVUSDB  : avx512_trunc_sat<0x11, "vpmovusdb", VR128X, VR512, VK16WM,
-                                 i128mem>, EVEX_V512, EVEX_CD8<8, CD8VQ>;
-
-def : Pat<(v16i8  (X86vtrunc (v8i64  VR512:$src))), (VPMOVQBrr  VR512:$src)>;
-def : Pat<(v8i16  (X86vtrunc (v8i64  VR512:$src))), (VPMOVQWrr  VR512:$src)>;
-def : Pat<(v16i16 (X86vtrunc (v16i32 VR512:$src))), (VPMOVDWrr  VR512:$src)>;
-def : Pat<(v16i8  (X86vtrunc (v16i32 VR512:$src))), (VPMOVDBrr  VR512:$src)>;
-def : Pat<(v8i32  (X86vtrunc (v8i64  VR512:$src))), (VPMOVQDrr  VR512:$src)>;
-
-def : Pat<(v16i8  (X86vtruncm VK16WM:$mask, (v16i32 VR512:$src))),
-                  (VPMOVDBrrkz VK16WM:$mask, VR512:$src)>;
-def : Pat<(v16i16 (X86vtruncm VK16WM:$mask, (v16i32 VR512:$src))),
-                  (VPMOVDWrrkz VK16WM:$mask, VR512:$src)>;
-def : Pat<(v8i16  (X86vtruncm VK8WM:$mask,  (v8i64 VR512:$src))),
-                  (VPMOVQWrrkz  VK8WM:$mask, VR512:$src)>;
-def : Pat<(v8i32  (X86vtruncm VK8WM:$mask,  (v8i64 VR512:$src))),
-                  (VPMOVQDrrkz  VK8WM:$mask, VR512:$src)>;
-
-
-multiclass avx512_extend<bits<8> opc, string OpcodeStr, RegisterClass KRC,
-                      RegisterClass DstRC, RegisterClass SrcRC, SDNode OpNode,
-                      PatFrag mem_frag, X86MemOperand x86memop,
-                      ValueType OpVT, ValueType InVT> {
-
-  def rr : AVX5128I<opc, MRMSrcReg, (outs DstRC:$dst),
-              (ins SrcRC:$src),
-              !strconcat(OpcodeStr, "\t{$src, $dst|$dst, $src}"),
-              [(set DstRC:$dst, (OpVT (OpNode (InVT SrcRC:$src))))]>, EVEX;
-
-  def rrk : AVX5128I<opc, MRMSrcReg, (outs DstRC:$dst),
-              (ins KRC:$mask, SrcRC:$src),
-              !strconcat(OpcodeStr, "\t{$src, $dst {${mask}} |$dst {${mask}}, $src}"),
-              []>, EVEX, EVEX_K;
-
-  def rrkz : AVX5128I<opc, MRMSrcReg, (outs DstRC:$dst),
-              (ins KRC:$mask, SrcRC:$src),
-              !strconcat(OpcodeStr, "\t{$src, $dst {${mask}} {z}|$dst {${mask}} {z}, $src}"),
-              []>, EVEX, EVEX_KZ;
+
+multiclass avx512_trunc_wb<bits<8> opc, string OpcodeStr, SDNode OpNode> {
+  defm NAME: avx512_trunc<opc, OpcodeStr, OpNode, avx512vl_i16_info,
+              v16i8x_info, v16i8x_info, v32i8x_info, i64mem, i128mem, i256mem,
+              truncstorevi8, masked_truncstorevi8,HasBWI>, EVEX_CD8<16, CD8VH>;
+}
+multiclass avx512_trunc_sat_wb<bits<8> opc, string sat, SDNode OpNode> {
+  defm NAME: avx512_trunc_sat<opc, "vpmov"##sat##"wb", OpNode, avx512vl_i16_info,
+              v16i8x_info, v16i8x_info, v32i8x_info, i64mem, i128mem, i256mem,
+              sat, HasBWI>, EVEX_CD8<16, CD8VH>;
+}
+
+defm VPMOVQB    : avx512_trunc_qb<0x32, "vpmovqb", X86vtrunc>;
+defm VPMOVSQB   : avx512_trunc_sat_qb<0x22, "s",   X86vtruncs>;
+defm VPMOVUSQB  : avx512_trunc_sat_qb<0x12, "us",  X86vtruncus>;
+
+defm VPMOVQW    : avx512_trunc_qw<0x34, "vpmovqw", X86vtrunc>;
+defm VPMOVSQW   : avx512_trunc_sat_qw<0x24, "s",   X86vtruncs>;
+defm VPMOVUSQW  : avx512_trunc_sat_qw<0x14, "us",  X86vtruncus>;
+
+defm VPMOVQD    : avx512_trunc_qd<0x35, "vpmovqd", X86vtrunc>;
+defm VPMOVSQD   : avx512_trunc_sat_qd<0x25, "s",   X86vtruncs>;
+defm VPMOVUSQD  : avx512_trunc_sat_qd<0x15, "us",  X86vtruncus>;
+
+defm VPMOVDB    : avx512_trunc_db<0x31, "vpmovdb", X86vtrunc>;
+defm VPMOVSDB   : avx512_trunc_sat_db<0x21, "s",   X86vtruncs>;
+defm VPMOVUSDB  : avx512_trunc_sat_db<0x11, "us",  X86vtruncus>;
+
+defm VPMOVDW    : avx512_trunc_dw<0x33, "vpmovdw", X86vtrunc>;
+defm VPMOVSDW   : avx512_trunc_sat_dw<0x23, "s",   X86vtruncs>;
+defm VPMOVUSDW  : avx512_trunc_sat_dw<0x13, "us",  X86vtruncus>;
+
+defm VPMOVWB    : avx512_trunc_wb<0x30, "vpmovwb", X86vtrunc>;
+defm VPMOVSWB   : avx512_trunc_sat_wb<0x20, "s",   X86vtruncs>;
+defm VPMOVUSWB  : avx512_trunc_sat_wb<0x10, "us",  X86vtruncus>;
+
+multiclass avx512_extend_common<bits<8> opc, string OpcodeStr,
+                  X86VectorVTInfo DestInfo, X86VectorVTInfo SrcInfo,
+                  X86MemOperand x86memop, PatFrag LdFrag, SDNode OpNode>{
+
+  defm rr   : AVX512_maskable<opc, MRMSrcReg, DestInfo, (outs DestInfo.RC:$dst),
+                    (ins SrcInfo.RC:$src), OpcodeStr ,"$src", "$src",
+                    (DestInfo.VT (OpNode (SrcInfo.VT SrcInfo.RC:$src)))>,
+                  EVEX;
 
   let mayLoad = 1 in {
-    def rm : AVX5128I<opc, MRMSrcMem, (outs DstRC:$dst),
-              (ins x86memop:$src),
-              !strconcat(OpcodeStr,"\t{$src, $dst|$dst, $src}"),
-              [(set DstRC:$dst,
-                (OpVT (OpNode (InVT (bitconvert (mem_frag addr:$src))))))]>,
-              EVEX;
-
-    def rmk : AVX5128I<opc, MRMSrcMem, (outs DstRC:$dst),
-              (ins KRC:$mask, x86memop:$src),
-              !strconcat(OpcodeStr,"\t{$src, $dst {${mask}} |$dst {${mask}}, $src}"),
-              []>,
-              EVEX, EVEX_K;
-
-    def rmkz : AVX5128I<opc, MRMSrcMem, (outs DstRC:$dst),
-              (ins KRC:$mask, x86memop:$src),
-              !strconcat(OpcodeStr,"\t{$src, $dst {${mask}} {z}|$dst {${mask}} {z}, $src}"),
-              []>,
-              EVEX, EVEX_KZ;
-  }
-}
-
-defm VPMOVZXBDZ: avx512_extend<0x31, "vpmovzxbd", VK16WM, VR512, VR128X, X86vzext,
-                             loadv2i64, i128mem, v16i32, v16i8>, EVEX_V512,
-                             EVEX_CD8<8, CD8VQ>;
-defm VPMOVZXBQZ: avx512_extend<0x32, "vpmovzxbq", VK8WM, VR512, VR128X, X86vzext,
-                             loadv2i64, i128mem, v8i64, v16i8>, EVEX_V512,
-                             EVEX_CD8<8, CD8VO>;
-defm VPMOVZXWDZ: avx512_extend<0x33, "vpmovzxwd", VK16WM, VR512, VR256X, X86vzext,
-                             loadv4i64, i256mem, v16i32, v16i16>, EVEX_V512,
-                             EVEX_CD8<16, CD8VH>;
-defm VPMOVZXWQZ: avx512_extend<0x34, "vpmovzxwq", VK8WM, VR512, VR128X, X86vzext,
-                             loadv2i64, i128mem, v8i64, v8i16>, EVEX_V512,
-                             EVEX_CD8<16, CD8VQ>;
-defm VPMOVZXDQZ: avx512_extend<0x35, "vpmovzxdq", VK8WM, VR512, VR256X, X86vzext,
-                             loadv4i64, i256mem, v8i64, v8i32>, EVEX_V512,
-                             EVEX_CD8<32, CD8VH>;
-
-defm VPMOVSXBDZ: avx512_extend<0x21, "vpmovsxbd", VK16WM, VR512, VR128X, X86vsext,
-                             loadv2i64, i128mem, v16i32, v16i8>, EVEX_V512,
-                             EVEX_CD8<8, CD8VQ>;
-defm VPMOVSXBQZ: avx512_extend<0x22, "vpmovsxbq", VK8WM, VR512, VR128X, X86vsext,
-                             loadv2i64, i128mem, v8i64, v16i8>, EVEX_V512,
-                             EVEX_CD8<8, CD8VO>;
-defm VPMOVSXWDZ: avx512_extend<0x23, "vpmovsxwd", VK16WM, VR512, VR256X, X86vsext,
-                             loadv4i64, i256mem, v16i32, v16i16>, EVEX_V512,
-                             EVEX_CD8<16, CD8VH>;
-defm VPMOVSXWQZ: avx512_extend<0x24, "vpmovsxwq", VK8WM, VR512, VR128X, X86vsext,
-                             loadv2i64, i128mem, v8i64, v8i16>, EVEX_V512,
-                             EVEX_CD8<16, CD8VQ>;
-defm VPMOVSXDQZ: avx512_extend<0x25, "vpmovsxdq", VK8WM, VR512, VR256X, X86vsext,
-                             loadv4i64, i256mem, v8i64, v8i32>, EVEX_V512,
-                             EVEX_CD8<32, CD8VH>;
+    defm rm : AVX512_maskable<opc, MRMSrcMem, DestInfo, (outs DestInfo.RC:$dst),
+                    (ins x86memop:$src), OpcodeStr ,"$src", "$src",
+                    (DestInfo.VT (LdFrag addr:$src))>,
+                  EVEX;
+  }
+}
 
-//===----------------------------------------------------------------------===//
-// GATHER - SCATTER Operations
+multiclass avx512_extend_BW<bits<8> opc, string OpcodeStr, SDNode OpNode,
+          string ExtTy,PatFrag LdFrag = !cast<PatFrag>(ExtTy#"extloadvi8")> {
+  let Predicates = [HasVLX, HasBWI] in {
+    defm Z128:  avx512_extend_common<opc, OpcodeStr, v8i16x_info,
+                    v16i8x_info, i64mem, LdFrag, OpNode>,
+                     EVEX_CD8<8, CD8VH>, T8PD, EVEX_V128;
 
-multiclass avx512_gather<bits<8> opc, string OpcodeStr, RegisterClass KRC,
-                       RegisterClass RC, X86MemOperand memop> {
-let mayLoad = 1,
-  Constraints = "@earlyclobber $dst, $src1 = $dst, $mask = $mask_wb" in
-  def rm  : AVX5128I<opc, MRMSrcMem, (outs RC:$dst, KRC:$mask_wb),
-            (ins RC:$src1, KRC:$mask, memop:$src2),
-            !strconcat(OpcodeStr,
-            "\t{$src2, ${dst} {${mask}}|${dst} {${mask}}, $src2}"),
-            []>, EVEX, EVEX_K;
+    defm Z256:  avx512_extend_common<opc, OpcodeStr, v16i16x_info,
+                    v16i8x_info, i128mem, LdFrag, OpNode>,
+                     EVEX_CD8<8, CD8VH>, T8PD, EVEX_V256;
+  }
+  let Predicates = [HasBWI] in {
+    defm Z   :  avx512_extend_common<opc, OpcodeStr, v32i16_info,
+                    v32i8x_info, i256mem, LdFrag, OpNode>,
+                     EVEX_CD8<8, CD8VH>, T8PD, EVEX_V512;
+  }
 }
 
-let ExeDomain = SSEPackedDouble in {
-defm VGATHERDPDZ : avx512_gather<0x92, "vgatherdpd", VK8WM, VR512, vy64xmem>,
-                                 EVEX_V512, VEX_W, EVEX_CD8<64, CD8VT1>;
-defm VGATHERQPDZ : avx512_gather<0x93, "vgatherqpd", VK8WM, VR512, vz64mem>,
-                                 EVEX_V512, VEX_W, EVEX_CD8<64, CD8VT1>;
+multiclass avx512_extend_BD<bits<8> opc, string OpcodeStr, SDNode OpNode,
+          string ExtTy,PatFrag LdFrag = !cast<PatFrag>(ExtTy#"extloadvi8")> {
+  let Predicates = [HasVLX, HasAVX512] in {
+    defm Z128:  avx512_extend_common<opc, OpcodeStr, v4i32x_info,
+                   v16i8x_info, i32mem, LdFrag, OpNode>,
+                         EVEX_CD8<8, CD8VQ>, T8PD, EVEX_V128;
+
+    defm Z256:  avx512_extend_common<opc, OpcodeStr, v8i32x_info,
+                   v16i8x_info, i64mem, LdFrag, OpNode>,
+                         EVEX_CD8<8, CD8VQ>, T8PD, EVEX_V256;
+  }
+  let Predicates = [HasAVX512] in {
+    defm Z   :  avx512_extend_common<opc, OpcodeStr, v16i32_info,
+                   v16i8x_info, i128mem, LdFrag, OpNode>,
+                         EVEX_CD8<8, CD8VQ>, T8PD, EVEX_V512;
+  }
 }
 
-let ExeDomain = SSEPackedSingle in {
-defm VGATHERDPSZ : avx512_gather<0x92, "vgatherdps", VK16WM, VR512, vz32mem>,
-                                 EVEX_V512, EVEX_CD8<32, CD8VT1>;
-defm VGATHERQPSZ : avx512_gather<0x93, "vgatherqps", VK8WM, VR256X, vz64mem>,
-                                 EVEX_V512, EVEX_CD8<32, CD8VT1>;
-}
-
-defm VPGATHERDQZ : avx512_gather<0x90, "vpgatherdq", VK8WM, VR512,  vy64xmem>,
-                                 EVEX_V512, VEX_W, EVEX_CD8<64, CD8VT1>;
-defm VPGATHERDDZ : avx512_gather<0x90, "vpgatherdd", VK16WM, VR512, vz32mem>,
-                                 EVEX_V512, EVEX_CD8<32, CD8VT1>;
-
-defm VPGATHERQQZ : avx512_gather<0x91, "vpgatherqq", VK8WM, VR512,  vz64mem>,
-                                 EVEX_V512, VEX_W, EVEX_CD8<64, CD8VT1>;
-defm VPGATHERQDZ : avx512_gather<0x91, "vpgatherqd", VK8WM, VR256X,  vz64mem>,
-                                 EVEX_V512, EVEX_CD8<32, CD8VT1>;
-
-multiclass avx512_scatter<bits<8> opc, string OpcodeStr, RegisterClass KRC,
-                       RegisterClass RC, X86MemOperand memop> {
-let mayStore = 1, Constraints = "$mask = $mask_wb" in
-  def mr  : AVX5128I<opc, MRMDestMem, (outs KRC:$mask_wb),
-            (ins memop:$dst, KRC:$mask, RC:$src2),
-            !strconcat(OpcodeStr,
+multiclass avx512_extend_BQ<bits<8> opc, string OpcodeStr, SDNode OpNode,
+          string ExtTy,PatFrag LdFrag = !cast<PatFrag>(ExtTy#"extloadvi8")> {
+  let Predicates = [HasVLX, HasAVX512] in {
+    defm Z128:  avx512_extend_common<opc, OpcodeStr, v2i64x_info,
+                   v16i8x_info, i16mem, LdFrag, OpNode>,
+                     EVEX_CD8<8, CD8VO>, T8PD, EVEX_V128;
+
+    defm Z256:  avx512_extend_common<opc, OpcodeStr, v4i64x_info,
+                   v16i8x_info, i32mem, LdFrag, OpNode>,
+                     EVEX_CD8<8, CD8VO>, T8PD, EVEX_V256;
+  }
+  let Predicates = [HasAVX512] in {
+    defm Z   :  avx512_extend_common<opc, OpcodeStr, v8i64_info,
+                   v16i8x_info, i64mem, LdFrag, OpNode>,
+                     EVEX_CD8<8, CD8VO>, T8PD, EVEX_V512;
+  }
+}
+
+multiclass avx512_extend_WD<bits<8> opc, string OpcodeStr, SDNode OpNode,
+         string ExtTy,PatFrag LdFrag = !cast<PatFrag>(ExtTy#"extloadvi16")> {
+  let Predicates = [HasVLX, HasAVX512] in {
+    defm Z128:  avx512_extend_common<opc, OpcodeStr, v4i32x_info,
+                   v8i16x_info, i64mem, LdFrag, OpNode>,
+                     EVEX_CD8<16, CD8VH>, T8PD, EVEX_V128;
+
+    defm Z256:  avx512_extend_common<opc, OpcodeStr, v8i32x_info,
+                   v8i16x_info, i128mem, LdFrag, OpNode>,
+                     EVEX_CD8<16, CD8VH>, T8PD, EVEX_V256;
+  }
+  let Predicates = [HasAVX512] in {
+    defm Z   :  avx512_extend_common<opc, OpcodeStr, v16i32_info,
+                   v16i16x_info, i256mem, LdFrag, OpNode>,
+                     EVEX_CD8<16, CD8VH>, T8PD, EVEX_V512;
+  }
+}
+
+multiclass avx512_extend_WQ<bits<8> opc, string OpcodeStr, SDNode OpNode,
+         string ExtTy,PatFrag LdFrag = !cast<PatFrag>(ExtTy#"extloadvi16")> {
+  let Predicates = [HasVLX, HasAVX512] in {
+    defm Z128:  avx512_extend_common<opc, OpcodeStr, v2i64x_info,
+                   v8i16x_info, i32mem, LdFrag, OpNode>,
+                     EVEX_CD8<16, CD8VQ>, T8PD, EVEX_V128;
+
+    defm Z256:  avx512_extend_common<opc, OpcodeStr, v4i64x_info,
+                   v8i16x_info, i64mem, LdFrag, OpNode>,
+                     EVEX_CD8<16, CD8VQ>, T8PD, EVEX_V256;
+  }
+  let Predicates = [HasAVX512] in {
+    defm Z   :  avx512_extend_common<opc, OpcodeStr, v8i64_info,
+                   v8i16x_info, i128mem, LdFrag, OpNode>,
+                     EVEX_CD8<16, CD8VQ>, T8PD, EVEX_V512;
+  }
+}
+
+multiclass avx512_extend_DQ<bits<8> opc, string OpcodeStr, SDNode OpNode,
+         string ExtTy,PatFrag LdFrag = !cast<PatFrag>(ExtTy#"extloadvi32")> {
+
+  let Predicates = [HasVLX, HasAVX512] in {
+    defm Z128:  avx512_extend_common<opc, OpcodeStr, v2i64x_info,
+                   v4i32x_info, i64mem, LdFrag, OpNode>,
+                     EVEX_CD8<32, CD8VH>, T8PD, EVEX_V128;
+
+    defm Z256:  avx512_extend_common<opc, OpcodeStr, v4i64x_info,
+                   v4i32x_info, i128mem, LdFrag, OpNode>,
+                     EVEX_CD8<32, CD8VH>, T8PD, EVEX_V256;
+  }
+  let Predicates = [HasAVX512] in {
+    defm Z   :  avx512_extend_common<opc, OpcodeStr, v8i64_info,
+                   v8i32x_info, i256mem, LdFrag, OpNode>,
+                     EVEX_CD8<32, CD8VH>, T8PD, EVEX_V512;
+  }
+}
+
+defm VPMOVZXBW : avx512_extend_BW<0x30, "vpmovzxbw", X86vzext, "z">;
+defm VPMOVZXBD : avx512_extend_BD<0x31, "vpmovzxbd", X86vzext, "z">;
+defm VPMOVZXBQ : avx512_extend_BQ<0x32, "vpmovzxbq", X86vzext, "z">;
+defm VPMOVZXWD : avx512_extend_WD<0x33, "vpmovzxwd", X86vzext, "z">;
+defm VPMOVZXWQ : avx512_extend_WQ<0x34, "vpmovzxwq", X86vzext, "z">;
+defm VPMOVZXDQ : avx512_extend_DQ<0x35, "vpmovzxdq", X86vzext, "z">;
+
+
+defm VPMOVSXBW: avx512_extend_BW<0x20, "vpmovsxbw", X86vsext, "s">;
+defm VPMOVSXBD: avx512_extend_BD<0x21, "vpmovsxbd", X86vsext, "s">;
+defm VPMOVSXBQ: avx512_extend_BQ<0x22, "vpmovsxbq", X86vsext, "s">;
+defm VPMOVSXWD: avx512_extend_WD<0x23, "vpmovsxwd", X86vsext, "s">;
+defm VPMOVSXWQ: avx512_extend_WQ<0x24, "vpmovsxwq", X86vsext, "s">;
+defm VPMOVSXDQ: avx512_extend_DQ<0x25, "vpmovsxdq", X86vsext, "s">;
+
+//===----------------------------------------------------------------------===//
+// GATHER - SCATTER Operations
+
+multiclass avx512_gather<bits<8> opc, string OpcodeStr, X86VectorVTInfo _,
+                         X86MemOperand memop, PatFrag GatherNode> {
+  let Constraints = "@earlyclobber $dst, $src1 = $dst, $mask = $mask_wb",
+      ExeDomain = _.ExeDomain in
+  def rm  : AVX5128I<opc, MRMSrcMem, (outs _.RC:$dst, _.KRCWM:$mask_wb),
+            (ins _.RC:$src1, _.KRCWM:$mask, memop:$src2),
+            !strconcat(OpcodeStr#_.Suffix,
             "\t{$src2, ${dst} {${mask}}|${dst} {${mask}}, $src2}"),
-            []>, EVEX, EVEX_K;
+            [(set _.RC:$dst, _.KRCWM:$mask_wb,
+              (GatherNode  (_.VT _.RC:$src1), _.KRCWM:$mask,
+                     vectoraddr:$src2))]>, EVEX, EVEX_K,
+             EVEX_CD8<_.EltSize, CD8VT1>;
+}
+
+multiclass avx512_gather_q_pd<bits<8> dopc, bits<8> qopc,
+                        AVX512VLVectorVTInfo _, string OpcodeStr, string SUFF> {
+  defm NAME##D##SUFF##Z: avx512_gather<dopc, OpcodeStr##"d", _.info512,
+                                      vy32xmem, mgatherv8i32>, EVEX_V512, VEX_W;
+  defm NAME##Q##SUFF##Z: avx512_gather<qopc, OpcodeStr##"q", _.info512,
+                                      vz64mem,  mgatherv8i64>, EVEX_V512, VEX_W;
+let Predicates = [HasVLX] in {
+  defm NAME##D##SUFF##Z256: avx512_gather<dopc, OpcodeStr##"d", _.info256,
+                              vx32xmem, mgatherv4i32>, EVEX_V256, VEX_W;
+  defm NAME##Q##SUFF##Z256: avx512_gather<qopc, OpcodeStr##"q", _.info256,
+                              vy64xmem, mgatherv4i64>, EVEX_V256, VEX_W;
+  defm NAME##D##SUFF##Z128: avx512_gather<dopc, OpcodeStr##"d", _.info128,
+                              vx32xmem, mgatherv4i32>, EVEX_V128, VEX_W;
+  defm NAME##Q##SUFF##Z128: avx512_gather<qopc, OpcodeStr##"q", _.info128,
+                              vx64xmem, mgatherv2i64>, EVEX_V128, VEX_W;
+}
+}
+
+multiclass avx512_gather_d_ps<bits<8> dopc, bits<8> qopc,
+                       AVX512VLVectorVTInfo _, string OpcodeStr, string SUFF> {
+  defm NAME##D##SUFF##Z: avx512_gather<dopc, OpcodeStr##"d", _.info512, vz32mem,
+                                       mgatherv16i32>, EVEX_V512;
+  defm NAME##Q##SUFF##Z: avx512_gather<qopc, OpcodeStr##"q", _.info256, vz64mem,
+                                       mgatherv8i64>, EVEX_V512;
+let Predicates = [HasVLX] in {
+  defm NAME##D##SUFF##Z256: avx512_gather<dopc, OpcodeStr##"d", _.info256,
+                                          vy32xmem, mgatherv8i32>, EVEX_V256;
+  defm NAME##Q##SUFF##Z256: avx512_gather<qopc, OpcodeStr##"q", _.info128,
+                                          vy64xmem, mgatherv4i64>, EVEX_V256;
+  defm NAME##D##SUFF##Z128: avx512_gather<dopc, OpcodeStr##"d", _.info128,
+                                          vx32xmem, mgatherv4i32>, EVEX_V128;
+  defm NAME##Q##SUFF##Z128: avx512_gather<qopc, OpcodeStr##"q", _.info128,
+                                          vx64xmem, mgatherv2i64>, EVEX_V128;
+}
 }
 
-let ExeDomain = SSEPackedDouble in {
-defm VSCATTERDPDZ : avx512_scatter<0xA2, "vscatterdpd", VK8WM, VR512, vy64xmem>,
-                                   EVEX_V512, VEX_W, EVEX_CD8<64, CD8VT1>;
-defm VSCATTERQPDZ : avx512_scatter<0xA3, "vscatterqpd", VK8WM, VR512, vz64mem>,
-                                   EVEX_V512, VEX_W, EVEX_CD8<64, CD8VT1>;
+
+defm VGATHER : avx512_gather_q_pd<0x92, 0x93, avx512vl_f64_info, "vgather", "PD">,
+               avx512_gather_d_ps<0x92, 0x93, avx512vl_f32_info, "vgather", "PS">;
+
+defm VPGATHER : avx512_gather_q_pd<0x90, 0x91, avx512vl_i64_info, "vpgather", "Q">,
+                avx512_gather_d_ps<0x90, 0x91, avx512vl_i32_info, "vpgather", "D">;
+
+multiclass avx512_scatter<bits<8> opc, string OpcodeStr, X86VectorVTInfo _,
+                          X86MemOperand memop, PatFrag ScatterNode> {
+
+let mayStore = 1, Constraints = "$mask = $mask_wb", ExeDomain = _.ExeDomain in
+
+  def mr  : AVX5128I<opc, MRMDestMem, (outs _.KRCWM:$mask_wb),
+            (ins memop:$dst, _.KRCWM:$mask, _.RC:$src),
+            !strconcat(OpcodeStr#_.Suffix,
+            "\t{$src, ${dst} {${mask}}|${dst} {${mask}}, $src}"),
+            [(set _.KRCWM:$mask_wb, (ScatterNode (_.VT _.RC:$src),
+                                     _.KRCWM:$mask,  vectoraddr:$dst))]>,
+            EVEX, EVEX_K, EVEX_CD8<_.EltSize, CD8VT1>;
 }
 
-let ExeDomain = SSEPackedSingle in {
-defm VSCATTERDPSZ : avx512_scatter<0xA2, "vscatterdps", VK16WM, VR512, vz32mem>,
-                                   EVEX_V512, EVEX_CD8<32, CD8VT1>;
-defm VSCATTERQPSZ : avx512_scatter<0xA3, "vscatterqps", VK8WM, VR256X, vz64mem>,
-                                   EVEX_V512, EVEX_CD8<32, CD8VT1>;
+multiclass avx512_scatter_q_pd<bits<8> dopc, bits<8> qopc,
+                        AVX512VLVectorVTInfo _, string OpcodeStr, string SUFF> {
+  defm NAME##D##SUFF##Z: avx512_scatter<dopc, OpcodeStr##"d", _.info512,
+                                      vy32xmem, mscatterv8i32>, EVEX_V512, VEX_W;
+  defm NAME##Q##SUFF##Z: avx512_scatter<qopc, OpcodeStr##"q", _.info512,
+                                      vz64mem,  mscatterv8i64>, EVEX_V512, VEX_W;
+let Predicates = [HasVLX] in {
+  defm NAME##D##SUFF##Z256: avx512_scatter<dopc, OpcodeStr##"d", _.info256,
+                              vx32xmem, mscatterv4i32>, EVEX_V256, VEX_W;
+  defm NAME##Q##SUFF##Z256: avx512_scatter<qopc, OpcodeStr##"q", _.info256,
+                              vy64xmem, mscatterv4i64>, EVEX_V256, VEX_W;
+  defm NAME##D##SUFF##Z128: avx512_scatter<dopc, OpcodeStr##"d", _.info128,
+                              vx32xmem, mscatterv4i32>, EVEX_V128, VEX_W;
+  defm NAME##Q##SUFF##Z128: avx512_scatter<qopc, OpcodeStr##"q", _.info128,
+                              vx64xmem, mscatterv2i64>, EVEX_V128, VEX_W;
+}
+}
+
+multiclass avx512_scatter_d_ps<bits<8> dopc, bits<8> qopc,
+                       AVX512VLVectorVTInfo _, string OpcodeStr, string SUFF> {
+  defm NAME##D##SUFF##Z: avx512_scatter<dopc, OpcodeStr##"d", _.info512, vz32mem,
+                                       mscatterv16i32>, EVEX_V512;
+  defm NAME##Q##SUFF##Z: avx512_scatter<qopc, OpcodeStr##"q", _.info256, vz64mem,
+                                       mscatterv8i64>, EVEX_V512;
+let Predicates = [HasVLX] in {
+  defm NAME##D##SUFF##Z256: avx512_scatter<dopc, OpcodeStr##"d", _.info256,
+                                          vy32xmem, mscatterv8i32>, EVEX_V256;
+  defm NAME##Q##SUFF##Z256: avx512_scatter<qopc, OpcodeStr##"q", _.info128,
+                                          vy64xmem, mscatterv4i64>, EVEX_V256;
+  defm NAME##D##SUFF##Z128: avx512_scatter<dopc, OpcodeStr##"d", _.info128,
+                                          vx32xmem, mscatterv4i32>, EVEX_V128;
+  defm NAME##Q##SUFF##Z128: avx512_scatter<qopc, OpcodeStr##"q", _.info128,
+                                          vx64xmem, mscatterv2i64>, EVEX_V128;
+}
 }
 
-defm VPSCATTERDQZ : avx512_scatter<0xA0, "vpscatterdq", VK8WM, VR512, vy64xmem>,
-                                   EVEX_V512, VEX_W, EVEX_CD8<64, CD8VT1>;
-defm VPSCATTERDDZ : avx512_scatter<0xA0, "vpscatterdd", VK16WM, VR512, vz32mem>,
-                                   EVEX_V512, EVEX_CD8<32, CD8VT1>;
+defm VSCATTER : avx512_scatter_q_pd<0xA2, 0xA3, avx512vl_f64_info, "vscatter", "PD">,
+               avx512_scatter_d_ps<0xA2, 0xA3, avx512vl_f32_info, "vscatter", "PS">;
 
-defm VPSCATTERQQZ : avx512_scatter<0xA1, "vpscatterqq", VK8WM, VR512, vz64mem>,
-                                  EVEX_V512, VEX_W, EVEX_CD8<64, CD8VT1>;
-defm VPSCATTERQDZ : avx512_scatter<0xA1, "vpscatterqd", VK8WM, VR256X, vz64mem>,
-                                  EVEX_V512, EVEX_CD8<32, CD8VT1>;
+defm VPSCATTER : avx512_scatter_q_pd<0xA0, 0xA1, avx512vl_i64_info, "vpscatter", "Q">,
+                avx512_scatter_d_ps<0xA0, 0xA1, avx512vl_i32_info, "vpscatter", "D">;
 
 // prefetch
 multiclass avx512_gather_scatter_prefetch<bits<8> opc, Format F, string OpcodeStr,
@@ -5194,105 +6075,10 @@ def : Pat<(v8i64 (X86Shufp VR512:$src1,
                             (loadv8i64 addr:$src2), (i8 imm:$imm))),
           (VSHUFPDZrmi VR512:$src1, addr:$src2, imm:$imm)>;
 
-multiclass avx512_valign<X86VectorVTInfo _> {
-  defm rri : AVX512_maskable<0x03, MRMSrcReg, _, (outs _.RC:$dst),
-                     (ins _.RC:$src1, _.RC:$src2, u8imm:$src3),
-                     "valign"##_.Suffix,
-                     "$src3, $src2, $src1", "$src1, $src2, $src3",
-                     (_.VT (X86VAlign _.RC:$src2, _.RC:$src1,
-                                      (i8 imm:$src3)))>,
-             AVX512AIi8Base, EVEX_4V;
-
-  // Also match valign of packed floats.
-  def : Pat<(_.FloatVT (X86VAlign _.RC:$src1, _.RC:$src2, (i8 imm:$imm))),
-            (!cast<Instruction>(NAME##rri) _.RC:$src2, _.RC:$src1, imm:$imm)>;
-
-  let mayLoad = 1 in
-  def rmi : AVX512AIi8<0x03, MRMSrcMem, (outs _.RC:$dst),
-                     (ins _.RC:$src1, _.MemOp:$src2, u8imm:$src3),
-                     !strconcat("valign"##_.Suffix,
-                     "\t{$src3, $src2, $src1, $dst|"
-                         "$dst, $src1, $src2, $src3}"),
-                     []>, EVEX_4V;
-}
-defm VALIGND : avx512_valign<v16i32_info>, EVEX_V512, EVEX_CD8<32, CD8VF>;
-defm VALIGNQ : avx512_valign<v8i64_info>, VEX_W, EVEX_V512, EVEX_CD8<64, CD8VF>;
-
 // Helper fragments to match sext vXi1 to vXiY.
 def v16i1sextv16i32  : PatLeaf<(v16i32 (X86vsrai VR512:$src, (i8 31)))>;
 def v8i1sextv8i64  : PatLeaf<(v8i64 (X86vsrai VR512:$src, (i8 63)))>;
 
-multiclass avx512_vpabs<bits<8> opc, string OpcodeStr, ValueType OpVT,
-                        RegisterClass KRC, RegisterClass RC,
-                        X86MemOperand x86memop, X86MemOperand x86scalar_mop,
-                        string BrdcstStr> {
-  def rr : AVX5128I<opc, MRMSrcReg, (outs RC:$dst), (ins RC:$src),
-            !strconcat(OpcodeStr, "\t{$src, $dst|$dst, $src}"),
-            []>, EVEX;
-  def rrk : AVX5128I<opc, MRMSrcReg, (outs RC:$dst), (ins KRC:$mask, RC:$src),
-             !strconcat(OpcodeStr, "\t{$src, $dst {${mask}}|$dst {${mask}}, $src}"),
-             []>, EVEX, EVEX_K;
-  def rrkz : AVX5128I<opc, MRMSrcReg, (outs RC:$dst), (ins KRC:$mask, RC:$src),
-              !strconcat(OpcodeStr,
-                         "\t{$src, $dst {${mask}} {z}|$dst {${mask}} {z}, $src}"),
-              []>, EVEX, EVEX_KZ;
-  let mayLoad = 1 in {
-    def rm : AVX5128I<opc, MRMSrcMem, (outs VR512:$dst),
-              (ins x86memop:$src),
-              !strconcat(OpcodeStr, "\t{$src, $dst|$dst, $src}"),
-              []>, EVEX;
-    def rmk : AVX5128I<opc, MRMSrcMem, (outs VR512:$dst),
-               (ins KRC:$mask, x86memop:$src),
-               !strconcat(OpcodeStr,
-                          "\t{$src, $dst {${mask}}|$dst {${mask}}, $src}"),
-               []>, EVEX, EVEX_K;
-    def rmkz : AVX5128I<opc, MRMSrcMem, (outs VR512:$dst),
-                (ins KRC:$mask, x86memop:$src),
-                !strconcat(OpcodeStr,
-                           "\t{$src, $dst {${mask}} {z}|$dst {${mask}} {z}, $src}"),
-                []>, EVEX, EVEX_KZ;
-    def rmb : AVX5128I<opc, MRMSrcMem, (outs VR512:$dst),
-               (ins x86scalar_mop:$src),
-               !strconcat(OpcodeStr, "\t{${src}", BrdcstStr,
-                          ", $dst|$dst, ${src}", BrdcstStr, "}"),
-               []>, EVEX, EVEX_B;
-    def rmbk : AVX5128I<opc, MRMSrcMem, (outs VR512:$dst),
-                (ins KRC:$mask, x86scalar_mop:$src),
-                !strconcat(OpcodeStr, "\t{${src}", BrdcstStr,
-                           ", $dst {${mask}}|$dst {${mask}}, ${src}", BrdcstStr, "}"),
-                []>, EVEX, EVEX_B, EVEX_K;
-    def rmbkz : AVX5128I<opc, MRMSrcMem, (outs VR512:$dst),
-                 (ins KRC:$mask, x86scalar_mop:$src),
-                 !strconcat(OpcodeStr, "\t{${src}", BrdcstStr,
-                            ", $dst {${mask}} {z}|$dst {${mask}} {z}, ${src}",
-                            BrdcstStr, "}"),
-                 []>, EVEX, EVEX_B, EVEX_KZ;
-  }
-}
-
-defm VPABSDZ : avx512_vpabs<0x1E, "vpabsd", v16i32, VK16WM, VR512,
-                           i512mem, i32mem, "{1to16}">, EVEX_V512,
-                           EVEX_CD8<32, CD8VF>;
-defm VPABSQZ : avx512_vpabs<0x1F, "vpabsq", v8i64, VK8WM, VR512,
-                           i512mem, i64mem, "{1to8}">, EVEX_V512, VEX_W,
-                           EVEX_CD8<64, CD8VF>;
-
-def : Pat<(xor
-          (bc_v16i32 (v16i1sextv16i32)),
-          (bc_v16i32 (add (v16i32 VR512:$src), (v16i1sextv16i32)))),
-          (VPABSDZrr VR512:$src)>;
-def : Pat<(xor
-          (bc_v8i64 (v8i1sextv8i64)),
-          (bc_v8i64 (add (v8i64 VR512:$src), (v8i1sextv8i64)))),
-          (VPABSQZrr VR512:$src)>;
-
-def : Pat<(v16i32 (int_x86_avx512_mask_pabs_d_512 (v16i32 VR512:$src),
-                   (v16i32 immAllZerosV), (i16 -1))),
-          (VPABSDZrr VR512:$src)>;
-def : Pat<(v8i64 (int_x86_avx512_mask_pabs_q_512 (v8i64 VR512:$src),
-                   (bc_v8i64 (v16i32 immAllZerosV)), (i8 -1))),
-          (VPABSQZrr VR512:$src)>;
-
 multiclass avx512_conflict<bits<8> opc, string OpcodeStr,
                         RegisterClass RC, RegisterClass KRC,
                         X86MemOperand x86memop,
@@ -5430,7 +6216,7 @@ def : Pat<(truncstorei1 GR8:$src, addr:$dst),
           (MOV8mr addr:$dst, GR8:$src)>;
 
 multiclass cvt_by_vec_width<bits<8> opc, X86VectorVTInfo Vec, string OpcodeStr > {
-def rr : AVX512XS8I<opc, MRMDestReg, (outs Vec.RC:$dst), (ins Vec.KRC:$src),
+def rr : AVX512XS8I<opc, MRMSrcReg, (outs Vec.RC:$dst), (ins Vec.KRC:$src),
                   !strconcat(OpcodeStr##Vec.Suffix, "\t{$src, $dst|$dst, $src}"),
                   [(set Vec.RC:$dst, (Vec.VT (X86vsext Vec.KRC:$src)))]>, EVEX;
 }
@@ -5459,29 +6245,56 @@ multiclass avx512_convert_mask_to_vector<string OpcodeStr> {
 
 defm VPMOVM2 : avx512_convert_mask_to_vector<"vpmovm2">;
 
+multiclass convert_vector_to_mask_common<bits<8> opc, X86VectorVTInfo _, string OpcodeStr > {
+def rr : AVX512XS8I<opc, MRMSrcReg, (outs _.KRC:$dst), (ins _.RC:$src),
+                  !strconcat(OpcodeStr, "\t{$src, $dst|$dst, $src}"),
+                  [(set _.KRC:$dst, (trunc (_.VT _.RC:$src)))]>, EVEX;
+}
+
+multiclass avx512_convert_vector_to_mask<bits<8> opc, string OpcodeStr,
+                        AVX512VLVectorVTInfo VTInfo, Predicate prd> {
+let Predicates = [prd] in
+  defm Z : convert_vector_to_mask_common <opc, VTInfo.info512, OpcodeStr>,
+   EVEX_V512;
+
+  let Predicates = [prd, HasVLX] in {
+    defm Z256 : convert_vector_to_mask_common<opc, VTInfo.info256, OpcodeStr>,
+     EVEX_V256;
+    defm Z128 : convert_vector_to_mask_common<opc, VTInfo.info128, OpcodeStr>,
+     EVEX_V128;
+  }
+}
+
+defm VPMOVB2M : avx512_convert_vector_to_mask<0x29, "vpmovb2m",
+                                              avx512vl_i8_info, HasBWI>;
+defm VPMOVW2M : avx512_convert_vector_to_mask<0x29, "vpmovw2m",
+                                              avx512vl_i16_info, HasBWI>, VEX_W;
+defm VPMOVD2M : avx512_convert_vector_to_mask<0x39, "vpmovd2m",
+                                              avx512vl_i32_info, HasDQI>;
+defm VPMOVQ2M : avx512_convert_vector_to_mask<0x39, "vpmovq2m",
+                                              avx512vl_i64_info, HasDQI>, VEX_W;
+
 //===----------------------------------------------------------------------===//
 // AVX-512 - COMPRESS and EXPAND
 //
+
 multiclass compress_by_vec_width<bits<8> opc, X86VectorVTInfo _,
                                  string OpcodeStr> {
-  def rrkz : AVX5128I<opc, MRMDestReg, (outs _.RC:$dst),
-              (ins _.KRCWM:$mask, _.RC:$src),
-              OpcodeStr # "\t{$src, $dst {${mask}} {z}|$dst {${mask}} {z}, $src}",
-              [(set _.RC:$dst, (_.VT (X86compress _.KRCWM:$mask, _.RC:$src,
-                                      _.ImmAllZerosV)))]>, EVEX_KZ;
-
-  let Constraints = "$src0 = $dst" in
-  def rrk : AVX5128I<opc, MRMDestReg, (outs _.RC:$dst),
-                    (ins _.RC:$src0, _.KRCWM:$mask, _.RC:$src),
-                    OpcodeStr # "\t{$src, $dst {${mask}} |$dst {${mask}}, $src}",
-                    [(set _.RC:$dst, (_.VT (X86compress _.KRCWM:$mask, _.RC:$src,
-                                            _.RC:$src0)))]>, EVEX_K;
+  defm rr : AVX512_maskable<opc, MRMDestReg, _, (outs _.RC:$dst),
+              (ins _.RC:$src1), OpcodeStr, "$src1", "$src1", 
+              (_.VT (X86compress _.RC:$src1))>, AVX5128IBase;
 
   let mayStore = 1 in {
+  def mr : AVX5128I<opc, MRMDestMem, (outs),
+              (ins _.MemOp:$dst, _.RC:$src),
+              OpcodeStr # "\t{$src, $dst |$dst, $src}",
+              []>, EVEX_CD8<_.EltSize, CD8VT1>;
+
   def mrk : AVX5128I<opc, MRMDestMem, (outs),
               (ins _.MemOp:$dst, _.KRCWM:$mask, _.RC:$src),
               OpcodeStr # "\t{$src, $dst {${mask}} |$dst {${mask}}, $src}",
-              [(store (_.VT (X86compress _.KRCWM:$mask, _.RC:$src, undef)),
+              [(store (_.VT (vselect _.KRCWM:$mask, 
+                             (_.VT (X86compress  _.RC:$src)), _.ImmAllZerosV)),
                 addr:$dst)]>,
               EVEX_K, EVEX_CD8<_.EltSize, CD8VT1>;
   }
@@ -5509,38 +6322,16 @@ defm VCOMPRESSPD : compress_by_elt_width <0x8A, "vcompresspd", avx512vl_f64_info
 // expand
 multiclass expand_by_vec_width<bits<8> opc, X86VectorVTInfo _,
                                  string OpcodeStr> {
-  def rrkz : AVX5128I<opc, MRMSrcReg, (outs _.RC:$dst),
-              (ins _.KRCWM:$mask, _.RC:$src),
-              OpcodeStr # "\t{$src, $dst {${mask}} {z}|$dst {${mask}} {z}, $src}",
-              [(set _.RC:$dst, (_.VT (X86expand _.KRCWM:$mask, (_.VT _.RC:$src),
-                                      _.ImmAllZerosV)))]>, EVEX_KZ;
-
-  let Constraints = "$src0 = $dst" in
-  def rrk : AVX5128I<opc, MRMSrcReg, (outs _.RC:$dst),
-                    (ins _.RC:$src0, _.KRCWM:$mask, _.RC:$src),
-                    OpcodeStr # "\t{$src, $dst {${mask}} |$dst {${mask}}, $src}",
-                    [(set _.RC:$dst, (_.VT (X86expand _.KRCWM:$mask,
-                                      (_.VT _.RC:$src), _.RC:$src0)))]>, EVEX_K;
+  defm rr : AVX512_maskable<opc, MRMSrcReg, _, (outs _.RC:$dst),
+              (ins _.RC:$src1), OpcodeStr, "$src1", "$src1", 
+              (_.VT (X86expand _.RC:$src1))>, AVX5128IBase;
 
-  let mayLoad = 1, Constraints = "$src0 = $dst" in
-  def rmk : AVX5128I<opc, MRMSrcMem, (outs _.RC:$dst),
-              (ins _.RC:$src0, _.KRCWM:$mask, _.MemOp:$src),
-              OpcodeStr # "\t{$src, $dst {${mask}} |$dst {${mask}}, $src}",
-              [(set _.RC:$dst, (_.VT (X86expand _.KRCWM:$mask,
-                                      (_.VT (bitconvert
-                                              (_.LdFrag addr:$src))),
-                                      _.RC:$src0)))]>,
-              EVEX_K, EVEX_CD8<_.EltSize, CD8VT1>;
-  
   let mayLoad = 1 in
-  def rmkz : AVX5128I<opc, MRMSrcMem, (outs _.RC:$dst),
-              (ins _.KRCWM:$mask, _.MemOp:$src),
-              OpcodeStr # "\t{$src, $dst {${mask}} {z}|$dst {${mask}} {z}, $src}",
-              [(set _.RC:$dst, (_.VT (X86expand _.KRCWM:$mask,
-                                      (_.VT (bitconvert (_.LdFrag addr:$src))),
-                                     _.ImmAllZerosV)))]>,
-              EVEX_KZ, EVEX_CD8<_.EltSize, CD8VT1>;
-  
+  defm rm : AVX512_maskable<opc, MRMSrcMem, _, (outs _.RC:$dst),
+              (ins _.MemOp:$src1), OpcodeStr, "$src1", "$src1",
+              (_.VT (X86expand (_.VT (bitconvert
+                                      (_.LdFrag addr:$src1)))))>,
+            AVX5128IBase, EVEX_CD8<_.EltSize, CD8VT1>;
 }
 
 multiclass expand_by_elt_width<bits<8> opc, string OpcodeStr,
@@ -5561,3 +6352,427 @@ defm VEXPANDPS : expand_by_elt_width <0x88, "vexpandps", avx512vl_f32_info>,
                                          EVEX;
 defm VEXPANDPD : expand_by_elt_width <0x88, "vexpandpd", avx512vl_f64_info>,
                                          EVEX, VEX_W;
+
+//handle instruction  reg_vec1 = op(reg_vec,imm)
+//                               op(mem_vec,imm)
+//                               op(broadcast(eltVt),imm)
+//all instruction created with FROUND_CURRENT
+multiclass avx512_unary_fp_packed_imm<bits<8> opc, string OpcodeStr, SDNode OpNode,
+                                                            X86VectorVTInfo _>{
+  defm rri : AVX512_maskable<opc, MRMSrcReg, _, (outs _.RC:$dst),
+                      (ins _.RC:$src1, i32u8imm:$src2),
+                      OpcodeStr##_.Suffix, "$src2, $src1", "$src2, $src2",
+                      (OpNode (_.VT _.RC:$src1),
+                              (i32 imm:$src2),
+                              (i32 FROUND_CURRENT))>;
+  let mayLoad = 1 in {
+    defm rmi : AVX512_maskable<opc, MRMSrcMem, _, (outs _.RC:$dst),
+                      (ins _.MemOp:$src1, i32u8imm:$src2),
+                      OpcodeStr##_.Suffix, "$src2, $src1", "$src1, $src2",
+                      (OpNode (_.VT (bitconvert (_.LdFrag addr:$src1))),
+                              (i32 imm:$src2),
+                              (i32 FROUND_CURRENT))>;
+    defm rmbi : AVX512_maskable<opc, MRMSrcMem, _, (outs _.RC:$dst),
+                      (ins _.ScalarMemOp:$src1, i32u8imm:$src2),
+                      OpcodeStr##_.Suffix, "$src2, ${src1}"##_.BroadcastStr,
+                      "${src1}"##_.BroadcastStr##", $src2",
+                      (OpNode (_.VT (X86VBroadcast(_.ScalarLdFrag addr:$src1))),
+                              (i32 imm:$src2),
+                              (i32 FROUND_CURRENT))>, EVEX_B;
+  }
+}
+
+//handle instruction  reg_vec1 = op(reg_vec2,reg_vec3,imm),{sae}
+multiclass avx512_unary_fp_sae_packed_imm<bits<8> opc, string OpcodeStr,
+                                             SDNode OpNode, X86VectorVTInfo _>{
+  defm rrib : AVX512_maskable<opc, MRMSrcReg, _, (outs _.RC:$dst),
+                      (ins _.RC:$src1, i32u8imm:$src2),
+                      OpcodeStr##_.Suffix, "$src2,{sae}, $src1",
+                      "$src1, {sae}, $src2",
+                      (OpNode (_.VT _.RC:$src1),
+                              (i32 imm:$src2),
+                              (i32 FROUND_NO_EXC))>, EVEX_B;
+}
+
+multiclass avx512_common_unary_fp_sae_packed_imm<string OpcodeStr,
+            AVX512VLVectorVTInfo _, bits<8> opc, SDNode OpNode, Predicate prd>{
+  let Predicates = [prd] in {
+    defm Z    : avx512_unary_fp_packed_imm<opc, OpcodeStr, OpNode, _.info512>,
+                avx512_unary_fp_sae_packed_imm<opc, OpcodeStr, OpNode, _.info512>,
+                                  EVEX_V512;
+  }
+  let Predicates = [prd, HasVLX] in {
+    defm Z128 : avx512_unary_fp_packed_imm<opc, OpcodeStr, OpNode, _.info128>,
+                                  EVEX_V128;
+    defm Z256 : avx512_unary_fp_packed_imm<opc, OpcodeStr, OpNode, _.info256>,
+                                  EVEX_V256;
+  }
+}
+
+//handle instruction  reg_vec1 = op(reg_vec2,reg_vec3,imm)
+//                               op(reg_vec2,mem_vec,imm)
+//                               op(reg_vec2,broadcast(eltVt),imm)
+//all instruction created with FROUND_CURRENT
+multiclass avx512_fp_packed_imm<bits<8> opc, string OpcodeStr, SDNode OpNode,
+                                                            X86VectorVTInfo _>{
+  defm rri : AVX512_maskable<opc, MRMSrcReg, _, (outs _.RC:$dst),
+                      (ins _.RC:$src1, _.RC:$src2, i32u8imm:$src3),
+                      OpcodeStr, "$src3, $src2, $src1", "$src1, $src2, $src3",
+                      (OpNode (_.VT _.RC:$src1),
+                              (_.VT _.RC:$src2),
+                              (i32 imm:$src3),
+                              (i32 FROUND_CURRENT))>;
+  let mayLoad = 1 in {
+    defm rmi : AVX512_maskable<opc, MRMSrcMem, _, (outs _.RC:$dst),
+                      (ins _.RC:$src1, _.MemOp:$src2, i32u8imm:$src3),
+                      OpcodeStr, "$src3, $src2, $src1", "$src1, $src2, $src3",
+                      (OpNode (_.VT _.RC:$src1),
+                              (_.VT (bitconvert (_.LdFrag addr:$src2))),
+                              (i32 imm:$src3),
+                              (i32 FROUND_CURRENT))>;
+    defm rmbi : AVX512_maskable<opc, MRMSrcMem, _, (outs _.RC:$dst),
+                      (ins _.RC:$src1, _.ScalarMemOp:$src2, i32u8imm:$src3),
+                      OpcodeStr, "$src3, ${src2}"##_.BroadcastStr##", $src1",
+                      "$src1, ${src2}"##_.BroadcastStr##", $src3",
+                      (OpNode (_.VT _.RC:$src1),
+                              (_.VT (X86VBroadcast(_.ScalarLdFrag addr:$src2))),
+                              (i32 imm:$src3),
+                              (i32 FROUND_CURRENT))>, EVEX_B;
+  }
+}
+
+//handle instruction  reg_vec1 = op(reg_vec2,reg_vec3,imm)
+//                               op(reg_vec2,mem_vec,imm)
+//                               op(reg_vec2,broadcast(eltVt),imm)
+multiclass avx512_3Op_imm8<bits<8> opc, string OpcodeStr, SDNode OpNode,
+                                                            X86VectorVTInfo _>{
+  defm rri : AVX512_maskable<opc, MRMSrcReg, _, (outs _.RC:$dst),
+                      (ins _.RC:$src1, _.RC:$src2, u8imm:$src3),
+                      OpcodeStr, "$src3, $src2, $src1", "$src1, $src2, $src3",
+                      (OpNode (_.VT _.RC:$src1),
+                              (_.VT _.RC:$src2),
+                              (i8 imm:$src3))>;
+  let mayLoad = 1 in {
+    defm rmi : AVX512_maskable<opc, MRMSrcMem, _, (outs _.RC:$dst),
+                      (ins _.RC:$src1, _.MemOp:$src2, u8imm:$src3),
+                      OpcodeStr, "$src3, $src2, $src1", "$src1, $src2, $src3",
+                      (OpNode (_.VT _.RC:$src1),
+                              (_.VT (bitconvert (_.LdFrag addr:$src2))),
+                              (i8 imm:$src3))>;
+    defm rmbi : AVX512_maskable<opc, MRMSrcMem, _, (outs _.RC:$dst),
+                      (ins _.RC:$src1, _.ScalarMemOp:$src2, u8imm:$src3),
+                      OpcodeStr, "$src3, ${src2}"##_.BroadcastStr##", $src1",
+                      "$src1, ${src2}"##_.BroadcastStr##", $src3",
+                      (OpNode (_.VT _.RC:$src1),
+                              (_.VT (X86VBroadcast(_.ScalarLdFrag addr:$src2))),
+                              (i8 imm:$src3))>, EVEX_B;
+  }
+}
+
+//handle scalar instruction  reg_vec1 = op(reg_vec2,reg_vec3,imm)
+//                                      op(reg_vec2,mem_scalar,imm)
+//all instruction created with FROUND_CURRENT
+multiclass avx512_fp_scalar_imm<bits<8> opc, string OpcodeStr, SDNode OpNode,
+                                                           X86VectorVTInfo _> {
+
+  defm rri : AVX512_maskable_scalar<opc, MRMSrcReg, _, (outs _.RC:$dst),
+                      (ins _.RC:$src1, _.RC:$src2, i32u8imm:$src3),
+                      OpcodeStr, "$src3, $src2, $src1", "$src1, $src2, $src3",
+                      (OpNode (_.VT _.RC:$src1),
+                              (_.VT _.RC:$src2),
+                              (i32 imm:$src3),
+                              (i32 FROUND_CURRENT))>;
+  let mayLoad = 1 in {
+    defm rmi : AVX512_maskable_scalar<opc, MRMSrcMem, _, (outs _.RC:$dst),
+                      (ins _.RC:$src1, _.MemOp:$src2, i32u8imm:$src3),
+                      OpcodeStr, "$src3, $src2, $src1", "$src1, $src2, $src3",
+                      (OpNode (_.VT _.RC:$src1),
+                              (_.VT (scalar_to_vector
+                                        (_.ScalarLdFrag addr:$src2))),
+                              (i32 imm:$src3),
+                              (i32 FROUND_CURRENT))>;
+
+    let isAsmParserOnly = 1 in {
+      defm rmi_alt :AVX512_maskable_in_asm<opc, MRMSrcMem, _, (outs _.FRC:$dst),
+                      (ins _.FRC:$src1, _.ScalarMemOp:$src2, u8imm:$src3),
+                      OpcodeStr, "$src3, $src2, $src1", "$src1, $src2, $src3",
+                      []>;
+    }
+  }
+}
+
+//handle instruction  reg_vec1 = op(reg_vec2,reg_vec3,imm),{sae}
+multiclass avx512_fp_sae_packed_imm<bits<8> opc, string OpcodeStr,
+                                             SDNode OpNode, X86VectorVTInfo _>{
+  defm rrib : AVX512_maskable<opc, MRMSrcReg, _, (outs _.RC:$dst),
+                      (ins _.RC:$src1, _.RC:$src2, i32u8imm:$src3),
+                      OpcodeStr, "$src3,{sae}, $src2, $src1",
+                      "$src1, $src2,{sae}, $src3",
+                      (OpNode (_.VT _.RC:$src1),
+                              (_.VT _.RC:$src2),
+                              (i32 imm:$src3),
+                              (i32 FROUND_NO_EXC))>, EVEX_B;
+}
+//handle scalar instruction  reg_vec1 = op(reg_vec2,reg_vec3,imm),{sae}
+multiclass avx512_fp_sae_scalar_imm<bits<8> opc, string OpcodeStr,
+                                             SDNode OpNode, X86VectorVTInfo _> {
+  defm NAME#rrib : AVX512_maskable_scalar<opc, MRMSrcReg, _, (outs _.RC:$dst),
+                      (ins _.RC:$src1, _.RC:$src2, i32u8imm:$src3),
+                      OpcodeStr, "$src3,{sae}, $src2, $src1",
+                      "$src1, $src2,{sae}, $src3",
+                      (OpNode (_.VT _.RC:$src1),
+                              (_.VT _.RC:$src2),
+                              (i32 imm:$src3),
+                              (i32 FROUND_NO_EXC))>, EVEX_B;
+}
+
+multiclass avx512_common_fp_sae_packed_imm<string OpcodeStr,
+            AVX512VLVectorVTInfo _, bits<8> opc, SDNode OpNode, Predicate prd>{
+  let Predicates = [prd] in {
+    defm Z    : avx512_fp_packed_imm<opc, OpcodeStr, OpNode, _.info512>,
+                avx512_fp_sae_packed_imm<opc, OpcodeStr, OpNode, _.info512>,
+                                  EVEX_V512;
+
+  }
+  let Predicates = [prd, HasVLX] in {
+    defm Z128 : avx512_fp_packed_imm<opc, OpcodeStr, OpNode, _.info128>,
+                                  EVEX_V128;
+    defm Z256 : avx512_fp_packed_imm<opc, OpcodeStr, OpNode, _.info256>,
+                                  EVEX_V256;
+  }
+}
+
+multiclass avx512_common_3Op_imm8<string OpcodeStr, AVX512VLVectorVTInfo _,
+                                bits<8> opc, SDNode OpNode>{
+  let Predicates = [HasAVX512] in {
+    defm Z    : avx512_3Op_imm8<opc, OpcodeStr, OpNode, _.info512>, EVEX_V512;
+  }
+  let Predicates = [HasAVX512, HasVLX] in {
+    defm Z128 : avx512_3Op_imm8<opc, OpcodeStr, OpNode, _.info128>, EVEX_V128;
+    defm Z256 : avx512_3Op_imm8<opc, OpcodeStr, OpNode, _.info256>, EVEX_V256;
+  }
+}
+
+multiclass avx512_common_fp_sae_scalar_imm<string OpcodeStr,
+                  X86VectorVTInfo _, bits<8> opc, SDNode OpNode, Predicate prd>{
+  let Predicates = [prd] in {
+     defm Z128 : avx512_fp_scalar_imm<opc, OpcodeStr, OpNode, _>,
+                 avx512_fp_sae_scalar_imm<opc, OpcodeStr, OpNode, _>;
+  }
+}
+
+multiclass avx512_common_fp_sae_packed_imm_all<string OpcodeStr, bits<8> opcPs,
+             bits<8> opcPd, SDNode OpNode, Predicate prd>{
+  defm PS : avx512_common_unary_fp_sae_packed_imm<OpcodeStr,  avx512vl_f32_info, opcPs, 
+                                            OpNode, prd>, EVEX_CD8<32, CD8VF>;
+  defm PD : avx512_common_unary_fp_sae_packed_imm<OpcodeStr,  avx512vl_f64_info, opcPd, 
+                                            OpNode, prd>,EVEX_CD8<64, CD8VF> , VEX_W;
+}
+
+defm VFIXUPIMMPD : avx512_common_fp_sae_packed_imm<"vfixupimmpd",
+                              avx512vl_f64_info, 0x54, X86VFixupimm, HasAVX512>,
+      AVX512AIi8Base, EVEX_4V, EVEX_CD8<64, CD8VF>, VEX_W;
+defm VFIXUPIMMPS : avx512_common_fp_sae_packed_imm<"vfixupimmps",
+                              avx512vl_f32_info, 0x54, X86VFixupimm, HasAVX512>,
+      AVX512AIi8Base, EVEX_4V, EVEX_CD8<32, CD8VF>;
+
+defm VFIXUPIMMSD: avx512_common_fp_sae_scalar_imm<"vfixupimmsd", f64x_info,
+                                                 0x55, X86VFixupimm, HasAVX512>,
+      AVX512AIi8Base, VEX_LIG, EVEX_4V, EVEX_CD8<64, CD8VT1>, VEX_W;
+defm VFIXUPIMMSS: avx512_common_fp_sae_scalar_imm<"vfixupimmss", f32x_info,
+                                                 0x55, X86VFixupimm, HasAVX512>,
+      AVX512AIi8Base, VEX_LIG, EVEX_4V, EVEX_CD8<32, CD8VT1>;
+
+defm VREDUCE : avx512_common_fp_sae_packed_imm_all<"vreduce", 0x56, 0x56, X86VReduce, HasDQI>,AVX512AIi8Base,EVEX;
+defm VRNDSCALE : avx512_common_fp_sae_packed_imm_all<"vrndscale", 0x08, 0x09, X86VRndScale, HasAVX512>,AVX512AIi8Base, EVEX;
+
+defm VRANGEPD : avx512_common_fp_sae_packed_imm<"vrangepd", avx512vl_f64_info,
+                                                       0x50, X86VRange, HasDQI>,
+      AVX512AIi8Base, EVEX_4V, EVEX_CD8<64, CD8VF>, VEX_W;
+defm VRANGEPS : avx512_common_fp_sae_packed_imm<"vrangeps", avx512vl_f32_info,
+                                                       0x50, X86VRange, HasDQI>,
+      AVX512AIi8Base, EVEX_4V, EVEX_CD8<32, CD8VF>;
+
+defm VRANGESD: avx512_common_fp_sae_scalar_imm<"vrangesd", f64x_info,
+                                                 0x51, X86VRange, HasDQI>,
+      AVX512AIi8Base, VEX_LIG, EVEX_4V, EVEX_CD8<64, CD8VT1>, VEX_W;
+defm VRANGESS: avx512_common_fp_sae_scalar_imm<"vrangess", f32x_info,
+                                                 0x51, X86VRange, HasDQI>,
+      AVX512AIi8Base, VEX_LIG, EVEX_4V, EVEX_CD8<32, CD8VT1>;
+
+defm VREDUCESD: avx512_common_fp_sae_scalar_imm<"vreducesd", f64x_info,
+                                                 0x57, X86Reduces, HasDQI>,
+      AVX512AIi8Base, VEX_LIG, EVEX_4V, EVEX_CD8<64, CD8VT1>, VEX_W;
+defm VREDUCESS: avx512_common_fp_sae_scalar_imm<"vreducess", f32x_info,
+                                                 0x57, X86Reduces, HasDQI>,
+      AVX512AIi8Base, VEX_LIG, EVEX_4V, EVEX_CD8<32, CD8VT1>;
+
+multiclass avx512_shuff_packed_128<string OpcodeStr, AVX512VLVectorVTInfo _,
+                                       bits<8> opc, SDNode OpNode = X86Shuf128>{
+  let Predicates = [HasAVX512] in {
+    defm Z    : avx512_3Op_imm8<opc, OpcodeStr, OpNode, _.info512>, EVEX_V512;
+
+  }
+  let Predicates = [HasAVX512, HasVLX] in {
+     defm Z256 : avx512_3Op_imm8<opc, OpcodeStr, OpNode, _.info256>, EVEX_V256;
+  }
+}
+let Predicates = [HasAVX512] in {
+def : Pat<(v16f32 (ffloor VR512:$src)),
+          (VRNDSCALEPSZrri VR512:$src, (i32 0x1))>;
+def : Pat<(v16f32 (fnearbyint VR512:$src)),
+          (VRNDSCALEPSZrri VR512:$src, (i32 0xC))>;
+def : Pat<(v16f32 (fceil VR512:$src)),
+          (VRNDSCALEPSZrri VR512:$src, (i32 0x2))>;
+def : Pat<(v16f32 (frint VR512:$src)),
+          (VRNDSCALEPSZrri VR512:$src, (i32 0x4))>;
+def : Pat<(v16f32 (ftrunc VR512:$src)),
+          (VRNDSCALEPSZrri VR512:$src, (i32 0x3))>;
+
+def : Pat<(v8f64 (ffloor VR512:$src)),
+          (VRNDSCALEPDZrri VR512:$src, (i32 0x1))>;
+def : Pat<(v8f64 (fnearbyint VR512:$src)),
+          (VRNDSCALEPDZrri VR512:$src, (i32 0xC))>;
+def : Pat<(v8f64 (fceil VR512:$src)),
+          (VRNDSCALEPDZrri VR512:$src, (i32 0x2))>;
+def : Pat<(v8f64 (frint VR512:$src)),
+          (VRNDSCALEPDZrri VR512:$src, (i32 0x4))>;
+def : Pat<(v8f64 (ftrunc VR512:$src)),
+          (VRNDSCALEPDZrri VR512:$src, (i32 0x3))>;
+}
+
+defm VSHUFF32X4 : avx512_shuff_packed_128<"vshuff32x4",avx512vl_f32_info, 0x23>,
+      AVX512AIi8Base, EVEX_4V, EVEX_CD8<32, CD8VF>;
+defm VSHUFF64X2 : avx512_shuff_packed_128<"vshuff64x2",avx512vl_f64_info, 0x23>,
+      AVX512AIi8Base, EVEX_4V, EVEX_CD8<64, CD8VF>, VEX_W;
+defm VSHUFI32X4 : avx512_shuff_packed_128<"vshufi32x4",avx512vl_i32_info, 0x43>,
+      AVX512AIi8Base, EVEX_4V, EVEX_CD8<32, CD8VF>;
+defm VSHUFI64X2 : avx512_shuff_packed_128<"vshufi64x2",avx512vl_i64_info, 0x43>,
+      AVX512AIi8Base, EVEX_4V, EVEX_CD8<64, CD8VF>, VEX_W;
+
+multiclass avx512_valign<string OpcodeStr, AVX512VLVectorVTInfo VTInfo_I,
+                                                AVX512VLVectorVTInfo VTInfo_FP>{
+  defm NAME:       avx512_common_3Op_imm8<OpcodeStr, VTInfo_I, 0x03, X86VAlign>,
+                           AVX512AIi8Base, EVEX_4V;
+  let isCodeGenOnly = 1 in {
+    defm NAME#_FP: avx512_common_3Op_imm8<OpcodeStr, VTInfo_FP, 0x03, X86VAlign>,
+                           AVX512AIi8Base, EVEX_4V;
+  }
+}
+
+defm VALIGND: avx512_valign<"valignd", avx512vl_i32_info, avx512vl_f32_info>,
+                                                  EVEX_CD8<32, CD8VF>;
+defm VALIGNQ: avx512_valign<"valignq", avx512vl_i64_info, avx512vl_f64_info>,
+                                                  EVEX_CD8<64, CD8VF>, VEX_W;
+
+multiclass avx512_unary_rm<bits<8> opc, string OpcodeStr, SDNode OpNode,
+                           X86VectorVTInfo _> {
+  defm rr : AVX512_maskable<opc, MRMSrcReg, _, (outs _.RC:$dst),
+                    (ins _.RC:$src1), OpcodeStr##_.Suffix,
+                    "$src1", "$src1",
+                    (_.VT (OpNode _.RC:$src1))>, EVEX, AVX5128IBase;
+
+  let mayLoad = 1 in
+    defm rm : AVX512_maskable<opc, MRMSrcMem, _, (outs _.RC:$dst),
+                    (ins _.MemOp:$src1), OpcodeStr##_.Suffix,
+                    "$src1", "$src1",
+                    (_.VT (OpNode (bitconvert (_.LdFrag addr:$src1))))>,
+              EVEX, AVX5128IBase, EVEX_CD8<_.EltSize, CD8VF>;
+}
+
+multiclass avx512_unary_rmb<bits<8> opc, string OpcodeStr, SDNode OpNode,
+                            X86VectorVTInfo _> :
+           avx512_unary_rm<opc, OpcodeStr, OpNode, _> {
+  let mayLoad = 1 in
+    defm rmb : AVX512_maskable<opc, MRMSrcMem, _, (outs _.RC:$dst),
+                    (ins _.ScalarMemOp:$src1), OpcodeStr##_.Suffix,
+                    "${src1}"##_.BroadcastStr,
+                    "${src1}"##_.BroadcastStr,
+                    (_.VT (OpNode (X86VBroadcast
+                                      (_.ScalarLdFrag addr:$src1))))>,
+               EVEX, AVX5128IBase, EVEX_B, EVEX_CD8<_.EltSize, CD8VF>;
+}
+
+multiclass avx512_unary_rm_vl<bits<8> opc, string OpcodeStr, SDNode OpNode,
+                              AVX512VLVectorVTInfo VTInfo, Predicate prd> {
+  let Predicates = [prd] in
+    defm Z : avx512_unary_rm<opc, OpcodeStr, OpNode, VTInfo.info512>, EVEX_V512;
+
+  let Predicates = [prd, HasVLX] in {
+    defm Z256 : avx512_unary_rm<opc, OpcodeStr, OpNode, VTInfo.info256>,
+                              EVEX_V256;
+    defm Z128 : avx512_unary_rm<opc, OpcodeStr, OpNode, VTInfo.info128>,
+                              EVEX_V128;
+  }
+}
+
+multiclass avx512_unary_rmb_vl<bits<8> opc, string OpcodeStr, SDNode OpNode,
+                               AVX512VLVectorVTInfo VTInfo, Predicate prd> {
+  let Predicates = [prd] in
+    defm Z : avx512_unary_rmb<opc, OpcodeStr, OpNode, VTInfo.info512>,
+                              EVEX_V512;
+
+  let Predicates = [prd, HasVLX] in {
+    defm Z256 : avx512_unary_rmb<opc, OpcodeStr, OpNode, VTInfo.info256>,
+                                 EVEX_V256;
+    defm Z128 : avx512_unary_rmb<opc, OpcodeStr, OpNode, VTInfo.info128>,
+                                 EVEX_V128;
+  }
+}
+
+multiclass avx512_unary_rm_vl_dq<bits<8> opc_d, bits<8> opc_q, string OpcodeStr,
+                                 SDNode OpNode, Predicate prd> {
+  defm Q : avx512_unary_rmb_vl<opc_q, OpcodeStr, OpNode, avx512vl_i64_info,
+                               prd>, VEX_W;
+  defm D : avx512_unary_rmb_vl<opc_d, OpcodeStr, OpNode, avx512vl_i32_info, prd>;
+}
+
+multiclass avx512_unary_rm_vl_bw<bits<8> opc_b, bits<8> opc_w, string OpcodeStr,
+                                 SDNode OpNode, Predicate prd> {
+  defm W : avx512_unary_rm_vl<opc_w, OpcodeStr, OpNode, avx512vl_i16_info, prd>;
+  defm B : avx512_unary_rm_vl<opc_b, OpcodeStr, OpNode, avx512vl_i8_info, prd>;
+}
+
+multiclass avx512_unary_rm_vl_all<bits<8> opc_b, bits<8> opc_w,
+                                  bits<8> opc_d, bits<8> opc_q,
+                                  string OpcodeStr, SDNode OpNode> {
+  defm NAME : avx512_unary_rm_vl_dq<opc_d, opc_q, OpcodeStr, OpNode,
+                                    HasAVX512>,
+              avx512_unary_rm_vl_bw<opc_b, opc_w, OpcodeStr, OpNode,
+                                    HasBWI>;
+}
+
+defm VPABS : avx512_unary_rm_vl_all<0x1C, 0x1D, 0x1E, 0x1F, "vpabs", X86Abs>;
+
+def : Pat<(xor
+          (bc_v16i32 (v16i1sextv16i32)),
+          (bc_v16i32 (add (v16i32 VR512:$src), (v16i1sextv16i32)))),
+          (VPABSDZrr VR512:$src)>;
+def : Pat<(xor
+          (bc_v8i64 (v8i1sextv8i64)),
+          (bc_v8i64 (add (v8i64 VR512:$src), (v8i1sextv8i64)))),
+          (VPABSQZrr VR512:$src)>;
+
+//===----------------------------------------------------------------------===//
+// AVX-512 - Unpack Instructions
+//===----------------------------------------------------------------------===//
+defm VUNPCKH : avx512_fp_binop_p<0x15, "vunpckh", X86Unpckh>;
+defm VUNPCKL : avx512_fp_binop_p<0x14, "vunpckl", X86Unpckl>;
+
+defm VPUNPCKLBW : avx512_binop_rm_vl_b<0x60, "vpunpcklbw", X86Unpckl,
+                                       SSE_INTALU_ITINS_P, HasBWI>;
+defm VPUNPCKHBW : avx512_binop_rm_vl_b<0x68, "vpunpckhbw", X86Unpckh,
+                                       SSE_INTALU_ITINS_P, HasBWI>;
+defm VPUNPCKLWD : avx512_binop_rm_vl_w<0x61, "vpunpcklwd", X86Unpckl,
+                                       SSE_INTALU_ITINS_P, HasBWI>;
+defm VPUNPCKHWD : avx512_binop_rm_vl_w<0x69, "vpunpckhwd", X86Unpckh,
+                                       SSE_INTALU_ITINS_P, HasBWI>;
+
+defm VPUNPCKLDQ : avx512_binop_rm_vl_d<0x62, "vpunpckldq", X86Unpckl,
+                                       SSE_INTALU_ITINS_P, HasAVX512>;
+defm VPUNPCKHDQ : avx512_binop_rm_vl_d<0x6A, "vpunpckhdq", X86Unpckh,
+                                       SSE_INTALU_ITINS_P, HasAVX512>;
+defm VPUNPCKLQDQ : avx512_binop_rm_vl_q<0x6C, "vpunpcklqdq", X86Unpckl,
+                                       SSE_INTALU_ITINS_P, HasAVX512>;
+defm VPUNPCKHQDQ : avx512_binop_rm_vl_q<0x6D, "vpunpckhqdq", X86Unpckh,
+                                       SSE_INTALU_ITINS_P, HasAVX512>;
diff --git a/lib/Target/X86/X86InstrCMovSetCC.td b/lib/Target/X86/X86InstrCMovSetCC.td
index 315f21308c0..c73c95019f8 100644
--- a/lib/Target/X86/X86InstrCMovSetCC.td
+++ b/lib/Target/X86/X86InstrCMovSetCC.td
@@ -13,7 +13,7 @@
 //===----------------------------------------------------------------------===//
 
 
-// SetCC instructions.
+// CMOV instructions.
 multiclass CMOV<bits<8> opc, string Mnemonic, PatLeaf CondNode> {
   let Uses = [EFLAGS], Predicates = [HasCMov], Constraints = "$src1 = $dst",
       isCommutable = 1, SchedRW = [WriteALU] in {
diff --git a/lib/Target/X86/X86InstrCompiler.td b/lib/Target/X86/X86InstrCompiler.td
index 54ff85dff85..49dc3189f09 100644
--- a/lib/Target/X86/X86InstrCompiler.td
+++ b/lib/Target/X86/X86InstrCompiler.td
@@ -17,12 +17,12 @@
 
 def GetLo32XForm : SDNodeXForm<imm, [{
   // Transformation function: get the low 32 bits.
-  return getI32Imm((unsigned)N->getZExtValue());
+  return getI32Imm((unsigned)N->getZExtValue(), SDLoc(N));
 }]>;
 
 def GetLo8XForm : SDNodeXForm<imm, [{
   // Transformation function: get the low 8 bits.
-  return getI8Imm((uint8_t)N->getZExtValue());
+  return getI8Imm((uint8_t)N->getZExtValue(), SDLoc(N));
 }]>;
 
 
@@ -97,8 +97,8 @@ def VAARG_64 : I<0, Pseudo,
                  "#VAARG_64 $dst, $ap, $size, $mode, $align",
                  [(set GR64:$dst,
                     (X86vaarg64 addr:$ap, imm:$size, imm:$mode, imm:$align)),
-                  (implicit EFLAGS)]>,
-                 Requires<[IsNotNaCl]>;
+                  (implicit EFLAGS)]>;
+
 // Dynamic stack allocation yields a _chkstk or _alloca call for all Windows
 // targets.  These calls are needed to probe the stack when allocating more than
 // 4k bytes in one go. Touching the stack at 4K increments is necessary to
@@ -449,16 +449,6 @@ def TLS_base_addr64 : I<0, Pseudo, (outs), (ins i64mem:$sym),
                   Requires<[In64BitMode]>;
 }
 
-// @LOCALMOD-BEGIN
-// NaCl TLS support
-let usesCustomInserter = 1 in {
-  def THREAD_POINTER_FROM_GS :
-    I<0, Pseudo, (outs GR32:$dst), (ins),
-      "# get thread pointer from %gs:0",
-      [(set GR32:$dst, (X86thread_pointer_from_gs))]>;
-}
-// @LOCALMOD-END
-
 // Darwin TLS Support
 // For i386, the address of the thunk is passed on the stack, on return the
 // address of the variable is in %eax.  %ecx is trashed during the function
@@ -528,6 +518,10 @@ let usesCustomInserter = 1, Uses = [EFLAGS] in {
   defm _V8I64  : CMOVrr_PSEUDO<VR512, v8i64>;
   defm _V8F64  : CMOVrr_PSEUDO<VR512, v8f64>;
   defm _V16F32 : CMOVrr_PSEUDO<VR512, v16f32>;
+  defm _V8I1   : CMOVrr_PSEUDO<VK8,  v8i1>;
+  defm _V16I1  : CMOVrr_PSEUDO<VK16, v16i1>;
+  defm _V32I1  : CMOVrr_PSEUDO<VK32, v32i1>;
+  defm _V64I1  : CMOVrr_PSEUDO<VK64, v64i1>;
 } // usesCustomInserter = 1, Uses = [EFLAGS]
 
 //===----------------------------------------------------------------------===//
@@ -875,6 +869,7 @@ def : Pat<(i32 (X86Wrapper tjumptable  :$dst)), (MOV32ri tjumptable  :$dst)>;
 def : Pat<(i32 (X86Wrapper tglobaltlsaddr:$dst)),(MOV32ri tglobaltlsaddr:$dst)>;
 def : Pat<(i32 (X86Wrapper tglobaladdr :$dst)), (MOV32ri tglobaladdr :$dst)>;
 def : Pat<(i32 (X86Wrapper texternalsym:$dst)), (MOV32ri texternalsym:$dst)>;
+def : Pat<(i32 (X86Wrapper mcsym:$dst)), (MOV32ri mcsym:$dst)>;
 def : Pat<(i32 (X86Wrapper tblockaddress:$dst)), (MOV32ri tblockaddress:$dst)>;
 
 def : Pat<(add GR32:$src1, (X86Wrapper tconstpool:$src2)),
@@ -885,6 +880,8 @@ def : Pat<(add GR32:$src1, (X86Wrapper tglobaladdr :$src2)),
           (ADD32ri GR32:$src1, tglobaladdr:$src2)>;
 def : Pat<(add GR32:$src1, (X86Wrapper texternalsym:$src2)),
           (ADD32ri GR32:$src1, texternalsym:$src2)>;
+def : Pat<(add GR32:$src1, (X86Wrapper mcsym:$src2)),
+          (ADD32ri GR32:$src1, mcsym:$src2)>;
 def : Pat<(add GR32:$src1, (X86Wrapper tblockaddress:$src2)),
           (ADD32ri GR32:$src1, tblockaddress:$src2)>;
 
@@ -892,6 +889,8 @@ def : Pat<(store (i32 (X86Wrapper tglobaladdr:$src)), addr:$dst),
           (MOV32mi addr:$dst, tglobaladdr:$src)>;
 def : Pat<(store (i32 (X86Wrapper texternalsym:$src)), addr:$dst),
           (MOV32mi addr:$dst, texternalsym:$src)>;
+def : Pat<(store (i32 (X86Wrapper mcsym:$src)), addr:$dst),
+          (MOV32mi addr:$dst, mcsym:$src)>;
 def : Pat<(store (i32 (X86Wrapper tblockaddress:$src)), addr:$dst),
           (MOV32mi addr:$dst, tblockaddress:$src)>;
 
@@ -906,6 +905,8 @@ def : Pat<(i64 (X86Wrapper tglobaladdr :$dst)),
           (MOV64ri tglobaladdr :$dst)>, Requires<[FarData]>;
 def : Pat<(i64 (X86Wrapper texternalsym:$dst)),
           (MOV64ri texternalsym:$dst)>, Requires<[FarData]>;
+def : Pat<(i64 (X86Wrapper mcsym:$dst)),
+          (MOV64ri mcsym:$dst)>, Requires<[FarData]>;
 def : Pat<(i64 (X86Wrapper tblockaddress:$dst)),
           (MOV64ri tblockaddress:$dst)>, Requires<[FarData]>;
 
@@ -920,6 +921,8 @@ def : Pat<(i64 (X86Wrapper tglobaladdr :$dst)),
           (MOV64ri32 tglobaladdr :$dst)>, Requires<[KernelCode]>;
 def : Pat<(i64 (X86Wrapper texternalsym:$dst)),
           (MOV64ri32 texternalsym:$dst)>, Requires<[KernelCode]>;
+def : Pat<(i64 (X86Wrapper mcsym:$dst)),
+          (MOV64ri32 mcsym:$dst)>, Requires<[KernelCode]>;
 def : Pat<(i64 (X86Wrapper tblockaddress:$dst)),
           (MOV64ri32 tblockaddress:$dst)>, Requires<[KernelCode]>;
 
@@ -938,12 +941,15 @@ def : Pat<(store (i64 (X86Wrapper tglobaladdr:$src)), addr:$dst),
 def : Pat<(store (i64 (X86Wrapper texternalsym:$src)), addr:$dst),
           (MOV64mi32 addr:$dst, texternalsym:$src)>,
           Requires<[NearData, IsStatic]>;
+def : Pat<(store (i64 (X86Wrapper mcsym:$src)), addr:$dst),
+          (MOV64mi32 addr:$dst, mcsym:$src)>,
+          Requires<[NearData, IsStatic]>;
 def : Pat<(store (i64 (X86Wrapper tblockaddress:$src)), addr:$dst),
           (MOV64mi32 addr:$dst, tblockaddress:$src)>,
           Requires<[NearData, IsStatic]>;
 
-def : Pat<(i32 (X86RecoverFrameAlloc texternalsym:$dst)), (MOV32ri texternalsym:$dst)>;
-def : Pat<(i64 (X86RecoverFrameAlloc texternalsym:$dst)), (MOV64ri texternalsym:$dst)>;
+def : Pat<(i32 (X86RecoverFrameAlloc mcsym:$dst)), (MOV32ri mcsym:$dst)>;
+def : Pat<(i64 (X86RecoverFrameAlloc mcsym:$dst)), (MOV64ri mcsym:$dst)>;
 
 // Calls
 
@@ -959,9 +965,9 @@ def : Pat<(add GR64:$src1, (X86Wrapper tglobaltlsaddr :$dst)),
 // Direct PC relative function call for small code model. 32-bit displacement
 // sign extended to 64-bit.
 def : Pat<(X86call (i64 tglobaladdr:$dst)),
-          (CALL64pcrel32 tglobaladdr:$dst)>, Requires<[IsNotNaCl]>;
+          (CALL64pcrel32 tglobaladdr:$dst)>;
 def : Pat<(X86call (i64 texternalsym:$dst)),
-          (CALL64pcrel32 texternalsym:$dst)>, Requires<[IsNotNaCl]>;
+          (CALL64pcrel32 texternalsym:$dst)>;
 
 // Tailcall stuff. The TCRETURN instructions execute after the epilog, so they
 // can never use callee-saved registers. That is the purpose of the GR64_TC
@@ -990,7 +996,7 @@ def : Pat<(X86tcret ptr_rc_tailcall:$dst, imm:$off),
 // callee-saved register.
 def : Pat<(X86tcret (load addr:$dst), imm:$off),
           (TCRETURNmi addr:$dst, imm:$off)>,
-          Requires<[Not64BitMode, IsNotPIC, IsNotNaCl]>; // @LOCALMOD
+          Requires<[Not64BitMode, IsNotPIC]>;
 
 def : Pat<(X86tcret (i32 tglobaladdr:$dst), imm:$off),
           (TCRETURNdi tglobaladdr:$dst, imm:$off)>,
@@ -1008,7 +1014,7 @@ def : Pat<(X86tcret ptr_rc_tailcall:$dst, imm:$off),
 // There wouldn't be enough scratch registers for base+index.
 def : Pat<(X86tcret_6regs (load addr:$dst), imm:$off),
           (TCRETURNmi64 addr:$dst, imm:$off)>,
-          Requires<[In64BitMode, IsNotNaCl]>;
+          Requires<[In64BitMode]>;
 
 def : Pat<(X86tcret (i64 tglobaladdr:$dst), imm:$off),
           (TCRETURNdi64 tglobaladdr:$dst, imm:$off)>,
@@ -1070,11 +1076,12 @@ defm : CMOVmr<X86_COND_O , CMOVNO16rm, CMOVNO32rm, CMOVNO64rm>;
 defm : CMOVmr<X86_COND_NO, CMOVO16rm , CMOVO32rm , CMOVO64rm>;
 
 // zextload bool -> zextload byte
-def : Pat<(zextloadi8i1  addr:$src), (MOV8rm     addr:$src)>;
-def : Pat<(zextloadi16i1 addr:$src), (MOVZX16rm8 addr:$src)>;
-def : Pat<(zextloadi32i1 addr:$src), (MOVZX32rm8 addr:$src)>;
+def : Pat<(zextloadi8i1  addr:$src), (AND8ri (MOV8rm addr:$src), (i8 1))>;
+def : Pat<(zextloadi16i1 addr:$src), (AND16ri8 (MOVZX16rm8 addr:$src), (i16 1))>;
+def : Pat<(zextloadi32i1 addr:$src), (AND32ri8 (MOVZX32rm8 addr:$src), (i32 1))>;
 def : Pat<(zextloadi64i1 addr:$src),
-          (SUBREG_TO_REG (i64 0), (MOVZX32rm8 addr:$src), sub_32bit)>;
+          (SUBREG_TO_REG (i64 0),
+           (AND32ri8 (MOVZX32rm8 addr:$src), (i32 1)), sub_32bit)>;
 
 // extload bool -> extload byte
 // When extloading from 16-bit and smaller memory locations into 64-bit
@@ -1475,19 +1482,19 @@ def : Pat<(store (i8 (trunc_su (srl_su GR64:$src, (i8 8)))), addr:$dst),
           (MOV8mr_NOREX
             addr:$dst,
             (EXTRACT_SUBREG (i64 (COPY_TO_REGCLASS GR64:$src, GR64_ABCD)),
-                            sub_8bit_hi))>, Requires<[IsNotNaCl]>; // @LOCALMOD
+                            sub_8bit_hi))>;
 def : Pat<(store (i8 (trunc_su (srl_su GR32:$src, (i8 8)))), addr:$dst),
           (MOV8mr_NOREX
             addr:$dst,
             (EXTRACT_SUBREG (i32 (COPY_TO_REGCLASS GR32:$src, GR32_ABCD)),
                             sub_8bit_hi))>,
-      Requires<[In64BitMode, IsNotNaCl]>; // @LOCALMOD
+      Requires<[In64BitMode]>;
 def : Pat<(store (i8 (trunc_su (srl_su GR16:$src, (i8 8)))), addr:$dst),
           (MOV8mr_NOREX
             addr:$dst,
             (EXTRACT_SUBREG (i16 (COPY_TO_REGCLASS GR16:$src, GR16_ABCD)),
                             sub_8bit_hi))>,
-      Requires<[In64BitMode, IsNotNaCl]>; // @LOCALMOD
+      Requires<[In64BitMode]>;
 
 
 // (shl x, 1) ==> (add x, x)
diff --git a/lib/Target/X86/X86InstrControl.td b/lib/Target/X86/X86InstrControl.td
index e87f24f4e6a..4cd5563ce72 100644
--- a/lib/Target/X86/X86InstrControl.td
+++ b/lib/Target/X86/X86InstrControl.td
@@ -105,14 +105,16 @@ let isBranch = 1, isTerminator = 1, hasSideEffects = 0, SchedRW = [WriteJump] in
   // jecxz.
   let Uses = [CX] in
     def JCXZ : Ii8PCRel<0xE3, RawFrm, (outs), (ins brtarget8:$dst),
-                        "jcxz\t$dst", [], IIC_JCXZ>, AdSize16;
+                        "jcxz\t$dst", [], IIC_JCXZ>, AdSize16,
+                        Requires<[Not64BitMode]>;
   let Uses = [ECX] in
     def JECXZ : Ii8PCRel<0xE3, RawFrm, (outs), (ins brtarget8:$dst),
                         "jecxz\t$dst", [], IIC_JCXZ>, AdSize32;
 
   let Uses = [RCX] in
     def JRCXZ : Ii8PCRel<0xE3, RawFrm, (outs), (ins brtarget8:$dst),
-                           "jrcxz\t$dst", [], IIC_JCXZ>, AdSize64;
+                         "jrcxz\t$dst", [], IIC_JCXZ>, AdSize64,
+                         Requires<[In64BitMode]>;
 }
 
 // Indirect branches
@@ -129,7 +131,7 @@ let isBranch = 1, isTerminator = 1, isBarrier = 1, isIndirectBranch = 1 in {
                    OpSize32, Sched<[WriteJump]>;
   def JMP32m     : I<0xFF, MRM4m, (outs), (ins i32mem:$dst), "jmp{l}\t{*}$dst",
                      [(brind (loadi32 addr:$dst))], IIC_JMP_MEM>,
-                   Requires<[Not64BitMode,IsNotNaCl]>, OpSize32, Sched<[WriteJumpLd]>;
+                   Requires<[Not64BitMode]>, OpSize32, Sched<[WriteJumpLd]>;
 
   def JMP64r     : I<0xFF, MRM4r, (outs), (ins GR64:$dst), "jmp{q}\t{*}$dst",
                      [(brind GR64:$dst)], IIC_JMP_REG>, Requires<[In64BitMode]>,
@@ -149,7 +151,7 @@ let isBranch = 1, isTerminator = 1, isBarrier = 1, isIndirectBranch = 1 in {
                             IIC_JMP_FAR_PTR>, OpSize32, Sched<[WriteJump]>;
   }
   def FARJMP64   : RI<0xFF, MRM5m, (outs), (ins opaque80mem:$dst),
-                      "ljmp{q}\t{*}$dst", [], IIC_JMP_FAR_MEM>, Requires<[IsNotNaCl]>,
+                      "ljmp{q}\t{*}$dst", [], IIC_JMP_FAR_MEM>,
                    Sched<[WriteJump]>;
 
   def FARJMP16m  : I<0xFF, MRM5m, (outs), (ins opaque32mem:$dst),
@@ -185,7 +187,6 @@ let isCall = 1 in
       def CALLpcrel16 : Ii16PCRel<0xE8, RawFrm,
                              (outs), (ins i16imm_pcrel:$dst),
                              "call{w}\t$dst", [], IIC_CALL_RI>, OpSize16,
-                      Requires<[IsNotNaCl]>, // @LOCALMOD
                         Sched<[WriteJump]>;
     def CALL16r     : I<0xFF, MRM2r, (outs), (ins GR16:$dst),
                         "call{w}\t{*}$dst", [(X86call GR16:$dst)], IIC_CALL_RI>,
@@ -198,12 +199,10 @@ let isCall = 1 in
     def CALL32r     : I<0xFF, MRM2r, (outs), (ins GR32:$dst),
                         "call{l}\t{*}$dst", [(X86call GR32:$dst)], IIC_CALL_RI>,
                       OpSize32, Requires<[Not64BitMode]>, Sched<[WriteJump]>;
-    // @LOCALMOD to disallow load-folded calls in NaCl (need to sandbox
-    // the destination address first).
     def CALL32m     : I<0xFF, MRM2m, (outs), (ins i32mem:$dst),
                         "call{l}\t{*}$dst", [(X86call (loadi32 addr:$dst))],
                         IIC_CALL_MEM>, OpSize32,
-                      Requires<[Not64BitMode,IsNotNaCl,FavorMemIndirectCall]>,
+                      Requires<[Not64BitMode,FavorMemIndirectCall]>,
                       Sched<[WriteJumpLd]>;
 
     let Predicates = [Not64BitMode] in {
@@ -225,6 +224,7 @@ let isCall = 1 in
                       Sched<[WriteJumpLd]>;
   }
 
+
 // Tail call stuff.
 
 let isCall = 1, isTerminator = 1, isReturn = 1, isBarrier = 1,
@@ -248,7 +248,7 @@ let isCall = 1, isTerminator = 1, isReturn = 1, isBarrier = 1,
                    "", [], IIC_JMP_REG>;  // FIXME: Remove encoding when JIT is dead.
   let mayLoad = 1 in
   def TAILJMPm : I<0xFF, MRM4m, (outs), (ins i32mem_TC:$dst),
-                   "jmp{l}\t{*}$dst", [], IIC_JMP_MEM>, Requires<[IsNotNaCl]>; // @LOCALMOD
+                   "jmp{l}\t{*}$dst", [], IIC_JMP_MEM>;
 }
 
 
@@ -266,18 +266,18 @@ let isCall = 1, Uses = [RSP], SchedRW = [WriteJump] in {
   def CALL64pcrel32 : Ii32PCRel<0xE8, RawFrm,
                         (outs), (ins i64i32imm_pcrel:$dst),
                         "call{q}\t$dst", [], IIC_CALL_RI>, OpSize32,
-                        Requires<[In64BitMode, IsNotNaCl]>; // @LOCALMOD
+                      Requires<[In64BitMode]>;
   def CALL64r       : I<0xFF, MRM2r, (outs), (ins GR64:$dst),
                         "call{q}\t{*}$dst", [(X86call GR64:$dst)],
                         IIC_CALL_RI>,
-                        Requires<[In64BitMode]>;
+                      Requires<[In64BitMode]>;
   def CALL64m       : I<0xFF, MRM2m, (outs), (ins i64mem:$dst),
                         "call{q}\t{*}$dst", [(X86call (loadi64 addr:$dst))],
                         IIC_CALL_MEM>,
-                      Requires<[In64BitMode, IsNotNaCl, FavorMemIndirectCall]>;
+                      Requires<[In64BitMode,FavorMemIndirectCall]>;
 
   def FARCALL64   : RI<0xFF, MRM3m, (outs), (ins opaque80mem:$dst),
-                       "lcall{q}\t{*}$dst", [], IIC_CALL_FAR_MEM>, Requires<[IsNotNaCl]>; // @LOCALMOD
+                       "lcall{q}\t{*}$dst", [], IIC_CALL_FAR_MEM>;
 }
 
 let isCall = 1, isTerminator = 1, isReturn = 1, isBarrier = 1,
@@ -299,8 +299,7 @@ let isCall = 1, isTerminator = 1, isReturn = 1, isBarrier = 1,
 
   let mayLoad = 1 in
   def TAILJMPm64 : I<0xFF, MRM4m, (outs), (ins i64mem_TC:$dst),
-                     "jmp{q}\t{*}$dst", [], IIC_JMP_MEM>,
-                   Requires<[IsNotNaCl]>; // @LOCALMOD
+                     "jmp{q}\t{*}$dst", [], IIC_JMP_MEM>;
 
   // Win64 wants jumps leaving the function to have a REX_W prefix.
   let hasREX_WPrefix = 1 in {
diff --git a/lib/Target/X86/X86InstrFMA.td b/lib/Target/X86/X86InstrFMA.td
index 2993e42443d..7cc3b599a73 100644
--- a/lib/Target/X86/X86InstrFMA.td
+++ b/lib/Target/X86/X86InstrFMA.td
@@ -183,19 +183,24 @@ multiclass fma3s<bits<8> opc132, bits<8> opc213, bits<8> opc231,
   defm SD : fma3s_forms<opc132, opc213, opc231, OpStr, "sd", "PD", IntF64, OpNode,
                         FR64, f64, f64mem, sdmem, loadf64, sse_load_f64>, VEX_W;
 
+// These patterns use the 123 ordering, instead of 213, even though
+// they match the intrinsic to the 213 version of the instruction.
+// This is because src1 is tied to dest, and the scalar intrinsics
+// require the pass-through values to come from the first source
+// operand, not the second.
   def : Pat<(IntF32 VR128:$src1, VR128:$src2, VR128:$src3),
             (COPY_TO_REGCLASS
               (!cast<Instruction>(NAME#"SSr213r")
-                (COPY_TO_REGCLASS $src2, FR32),
                 (COPY_TO_REGCLASS $src1, FR32),
+                (COPY_TO_REGCLASS $src2, FR32),
                 (COPY_TO_REGCLASS $src3, FR32)),
               VR128)>;
 
   def : Pat<(IntF64 VR128:$src1, VR128:$src2, VR128:$src3),
             (COPY_TO_REGCLASS
               (!cast<Instruction>(NAME#"SDr213r")
-                (COPY_TO_REGCLASS $src2, FR64),
                 (COPY_TO_REGCLASS $src1, FR64),
+                (COPY_TO_REGCLASS $src2, FR64),
                 (COPY_TO_REGCLASS $src3, FR64)),
               VR128)>;
 }
diff --git a/lib/Target/X86/X86InstrFPStack.td b/lib/Target/X86/X86InstrFPStack.td
index 6cd5e79a15a..49068e9c37d 100644
--- a/lib/Target/X86/X86InstrFPStack.td
+++ b/lib/Target/X86/X86InstrFPStack.td
@@ -350,21 +350,21 @@ defm CMOVNP : FPCMov<X86_COND_NP>;
 
 let Predicates = [HasCMov] in {
 // These are not factored because there's no clean way to pass DA/DB.
-def CMOVB_F  : FPI<0xDA, MRM0r, (outs RST:$op), (ins),
+def CMOVB_F  : FPI<0xDA, MRM0r, (outs), (ins RST:$op),
                   "fcmovb\t{$op, %st(0)|st(0), $op}">;
-def CMOVBE_F : FPI<0xDA, MRM2r, (outs RST:$op), (ins),
+def CMOVBE_F : FPI<0xDA, MRM2r, (outs), (ins RST:$op),
                   "fcmovbe\t{$op, %st(0)|st(0), $op}">;
-def CMOVE_F  : FPI<0xDA, MRM1r, (outs RST:$op), (ins),
+def CMOVE_F  : FPI<0xDA, MRM1r, (outs), (ins RST:$op),
                   "fcmove\t{$op, %st(0)|st(0), $op}">;
-def CMOVP_F  : FPI<0xDA, MRM3r, (outs RST:$op), (ins),
+def CMOVP_F  : FPI<0xDA, MRM3r, (outs), (ins RST:$op),
                   "fcmovu\t{$op, %st(0)|st(0), $op}">;
-def CMOVNB_F : FPI<0xDB, MRM0r, (outs RST:$op), (ins),
+def CMOVNB_F : FPI<0xDB, MRM0r, (outs), (ins RST:$op),
                   "fcmovnb\t{$op, %st(0)|st(0), $op}">;
-def CMOVNBE_F: FPI<0xDB, MRM2r, (outs RST:$op), (ins),
+def CMOVNBE_F: FPI<0xDB, MRM2r, (outs), (ins RST:$op),
                   "fcmovnbe\t{$op, %st(0)|st(0), $op}">;
-def CMOVNE_F : FPI<0xDB, MRM1r, (outs RST:$op), (ins),
+def CMOVNE_F : FPI<0xDB, MRM1r, (outs), (ins RST:$op),
                   "fcmovne\t{$op, %st(0)|st(0), $op}">;
-def CMOVNP_F : FPI<0xDB, MRM3r, (outs RST:$op), (ins),
+def CMOVNP_F : FPI<0xDB, MRM3r, (outs), (ins RST:$op),
                   "fcmovnu\t{$op, %st(0)|st(0), $op}">;
 } // Predicates = [HasCMov]
 
@@ -633,16 +633,16 @@ def FRNDINT : I<0xD9, MRM_FC, (outs), (ins), "frndint", [], IIC_FRNDINT>;
 def FSCALE : I<0xD9, MRM_FD, (outs), (ins), "fscale", [], IIC_FSCALE>;
 def FCOMPP : I<0xDE, MRM_D9, (outs), (ins), "fcompp", [], IIC_FCOMPP>;
 
-def FXSAVE : I<0xAE, MRM0m, (outs opaque512mem:$dst), (ins),
-               "fxsave\t$dst", [], IIC_FXSAVE>, TB;
-def FXSAVE64 : RI<0xAE, MRM0m, (outs opaque512mem:$dst), (ins),
-                  "fxsave64\t$dst", [], IIC_FXSAVE>, TB,
-                  Requires<[In64BitMode]>;
+def FXSAVE : I<0xAE, MRM0m, (outs), (ins opaque512mem:$dst),
+               "fxsave\t$dst", [(int_x86_fxsave addr:$dst)], IIC_FXSAVE>, TB;
+def FXSAVE64 : RI<0xAE, MRM0m, (outs), (ins opaque512mem:$dst),
+                  "fxsave64\t$dst", [(int_x86_fxsave64 addr:$dst)], 
+                  IIC_FXSAVE>, TB, Requires<[In64BitMode]>;
 def FXRSTOR : I<0xAE, MRM1m, (outs), (ins opaque512mem:$src),
-                "fxrstor\t$src", [], IIC_FXRSTOR>, TB;
+              "fxrstor\t$src", [(int_x86_fxrstor addr:$src)], IIC_FXRSTOR>, TB;
 def FXRSTOR64 : RI<0xAE, MRM1m, (outs), (ins opaque512mem:$src),
-                  "fxrstor64\t$src", [], IIC_FXRSTOR>, TB,
-                  Requires<[In64BitMode]>;
+                   "fxrstor64\t$src", [(int_x86_fxrstor64 addr:$src)],
+                   IIC_FXRSTOR>, TB, Requires<[In64BitMode]>;
 } // SchedRW
 
 //===----------------------------------------------------------------------===//
diff --git a/lib/Target/X86/X86InstrFormats.td b/lib/Target/X86/X86InstrFormats.td
index ca2c093fbb3..e2fa295c023 100644
--- a/lib/Target/X86/X86InstrFormats.td
+++ b/lib/Target/X86/X86InstrFormats.td
@@ -55,7 +55,6 @@ def MRM_F6 : Format<86>; def MRM_F7 : Format<87>; def MRM_F8 : Format<88>;
 def MRM_F9 : Format<89>; def MRM_FA : Format<90>; def MRM_FB : Format<91>;
 def MRM_FC : Format<92>; def MRM_FD : Format<93>; def MRM_FE : Format<94>;
 def MRM_FF : Format<95>;
-def CustomFrm : Format<126>; // @LOCALMOD
 
 // ImmType - This specifies the immediate type used by an instruction. This is
 // part of the ad-hoc solution used to emit machine instruction encodings by our
@@ -443,12 +442,29 @@ class SI<bits<8> o, Format F, dag outs, dag ins, string asm,
                   asm));
 }
 
-// SIi8 - SSE 1 & 2 scalar instructions
+// SI - SSE 1 & 2 scalar intrinsics - vex form available on AVX512
+class SI_Int<bits<8> o, Format F, dag outs, dag ins, string asm,
+         list<dag> pattern, InstrItinClass itin = NoItinerary,
+         Domain d = GenericDomain>
+      : I<o, F, outs, ins, asm, pattern, itin, d> {
+  let Predicates = !if(!eq(OpEnc.Value, EncEVEX.Value), [HasAVX512],
+                   !if(!eq(OpEnc.Value, EncVEX.Value), [HasAVX],
+                   !if(!eq(OpPrefix.Value, XS.Value), [UseSSE1],
+                   !if(!eq(OpPrefix.Value, XD.Value), [UseSSE2],
+                   !if(!eq(OpPrefix.Value, PD.Value), [UseSSE2],
+                   [UseSSE1])))));
+
+  // AVX instructions have a 'v' prefix in the mnemonic
+  let AsmString = !if(!eq(OpEnc.Value, EncEVEX.Value), !strconcat("v", asm),
+                  !if(!eq(OpEnc.Value, EncVEX.Value), !strconcat("v", asm),
+                  asm));
+}
+// SIi8 - SSE 1 & 2 scalar instructions - vex form available on AVX512
 class SIi8<bits<8> o, Format F, dag outs, dag ins, string asm,
            list<dag> pattern, InstrItinClass itin = NoItinerary>
       : Ii8<o, F, outs, ins, asm, pattern, itin> {
   let Predicates = !if(!eq(OpEnc.Value, EncEVEX.Value), [HasAVX512],
-                   !if(!eq(OpEnc.Value, EncVEX.Value), [UseAVX],
+                   !if(!eq(OpEnc.Value, EncVEX.Value), [HasAVX],
                    !if(!eq(OpPrefix.Value, XS.Value), [UseSSE1],
                    [UseSSE2])));
 
@@ -748,6 +764,22 @@ class AVX512BIi8Base : PD {
   Domain ExeDomain = SSEPackedInt;
   ImmType ImmT = Imm8;
 }
+class AVX512XSIi8Base : XS {
+  Domain ExeDomain = SSEPackedInt;
+  ImmType ImmT = Imm8;
+}
+class AVX512XDIi8Base : XD {
+  Domain ExeDomain = SSEPackedInt;
+  ImmType ImmT = Imm8;
+}
+class AVX512PSIi8Base : PS {
+  Domain ExeDomain = SSEPackedSingle;
+  ImmType ImmT = Imm8;
+}
+class AVX512PDIi8Base : PD {
+  Domain ExeDomain = SSEPackedDouble;
+  ImmType ImmT = Imm8;
+}
 class AVX512AIi8<bits<8> o, Format F, dag outs, dag ins, string asm,
               list<dag> pattern, InstrItinClass itin = NoItinerary>
       : Ii8<o, F, outs, ins, asm, pattern, itin, SSEPackedInt>, TAPD,
diff --git a/lib/Target/X86/X86InstrFragmentsSIMD.td b/lib/Target/X86/X86InstrFragmentsSIMD.td
index b75a9f4b2d0..361e1966ae7 100644
--- a/lib/Target/X86/X86InstrFragmentsSIMD.td
+++ b/lib/Target/X86/X86InstrFragmentsSIMD.td
@@ -35,17 +35,10 @@ def bc_mmx  : PatFrag<(ops node:$in), (x86mmx  (bitconvert node:$in))>;
 // SSE specific DAG Nodes.
 //===----------------------------------------------------------------------===//
 
-def SDTX86FPShiftOp : SDTypeProfile<1, 2, [ SDTCisSameAs<0, 1>,
-                                            SDTCisFP<0>, SDTCisInt<2> ]>;
 def SDTX86VFCMP : SDTypeProfile<1, 3, [SDTCisInt<0>, SDTCisSameAs<1, 2>,
                                        SDTCisFP<1>, SDTCisVT<3, i8>,
                                        SDTCisVec<1>]>;
 
-def X86umin    : SDNode<"X86ISD::UMIN",      SDTIntBinOp>;
-def X86umax    : SDNode<"X86ISD::UMAX",      SDTIntBinOp>;
-def X86smin    : SDNode<"X86ISD::SMIN",      SDTIntBinOp>;
-def X86smax    : SDNode<"X86ISD::SMAX",      SDTIntBinOp>;
-
 def X86fmin    : SDNode<"X86ISD::FMIN",      SDTFPBinOp>;
 def X86fmax    : SDNode<"X86ISD::FMAX",      SDTFPBinOp>;
 
@@ -65,7 +58,6 @@ def X86fandn   : SDNode<"X86ISD::FANDN",     SDTFPBinOp,
                         [SDNPCommutative, SDNPAssociative]>;
 def X86frsqrt  : SDNode<"X86ISD::FRSQRT",    SDTFPUnaryOp>;
 def X86frcp    : SDNode<"X86ISD::FRCP",      SDTFPUnaryOp>;
-def X86fsrl    : SDNode<"X86ISD::FSRL",      SDTX86FPShiftOp>;
 def X86fgetsign: SDNode<"X86ISD::FGETSIGNx86",SDTFPToIntOp>;
 def X86fhadd   : SDNode<"X86ISD::FHADD",     SDTFPBinOp>;
 def X86fhsub   : SDNode<"X86ISD::FHSUB",     SDTFPBinOp>;
@@ -75,9 +67,18 @@ def X86comi    : SDNode<"X86ISD::COMI",      SDTX86CmpTest>;
 def X86ucomi   : SDNode<"X86ISD::UCOMI",     SDTX86CmpTest>;
 def X86cmps    : SDNode<"X86ISD::FSETCC",     SDTX86Cmps>;
 //def X86cmpsd   : SDNode<"X86ISD::FSETCCsd",    SDTX86Cmpsd>;
+def X86cvtdq2pd: SDNode<"X86ISD::CVTDQ2PD",
+                 SDTypeProfile<1, 1, [SDTCisVT<0, v2f64>,
+                                      SDTCisVT<1, v4i32>]>>;
+def X86cvtudq2pd: SDNode<"X86ISD::CVTUDQ2PD",
+                 SDTypeProfile<1, 1, [SDTCisVT<0, v2f64>,
+                                      SDTCisVT<1, v4i32>]>>;
 def X86pshufb  : SDNode<"X86ISD::PSHUFB",
                  SDTypeProfile<1, 2, [SDTCisVec<0>, SDTCisSameAs<0,1>,
                                       SDTCisSameAs<0,2>]>>;
+def X86psadbw  : SDNode<"X86ISD::PSADBW",
+                 SDTypeProfile<1, 2, [SDTCisVec<0>, SDTCisSameAs<0,1>,
+                                      SDTCisSameAs<0,2>]>>;
 def X86andnp   : SDNode<"X86ISD::ANDNP",
                  SDTypeProfile<1, 2, [SDTCisVec<0>, SDTCisSameAs<0,1>,
                                       SDTCisSameAs<0,2>]>>;
@@ -113,19 +114,17 @@ def X86vsext   : SDNode<"X86ISD::VSEXT",
                                               SDTCisInt<0>, SDTCisInt<1>,
                                               SDTCisOpSmallerThanOp<1, 0>]>>;
 
-def X86vtrunc   : SDNode<"X86ISD::VTRUNC",
-                         SDTypeProfile<1, 1, [SDTCisVec<0>, SDTCisVec<1>,
-                                              SDTCisInt<0>, SDTCisInt<1>,
-                                              SDTCisOpSmallerThanOp<0, 1>]>>;
+def SDTVtrunc    : SDTypeProfile<1, 1, [SDTCisVec<0>, SDTCisVec<1>,
+                                       SDTCisInt<0>, SDTCisInt<1>,
+                                       SDTCisOpSmallerThanOp<0, 1>]>;
+
+def X86vtrunc    : SDNode<"X86ISD::VTRUNC",   SDTVtrunc>;
+def X86vtruncs   : SDNode<"X86ISD::VTRUNCS",  SDTVtrunc>;
+def X86vtruncus  : SDNode<"X86ISD::VTRUNCUS", SDTVtrunc>;
+
 def X86trunc    : SDNode<"X86ISD::TRUNC",
                          SDTypeProfile<1, 1, [SDTCisInt<0>, SDTCisInt<1>,
                                               SDTCisOpSmallerThanOp<0, 1>]>>;
-
-def X86vtruncm   : SDNode<"X86ISD::VTRUNCM",
-                         SDTypeProfile<1, 2, [SDTCisVec<0>, SDTCisVec<1>,
-                                              SDTCisInt<0>, SDTCisInt<1>,
-                                              SDTCisVec<2>, SDTCisInt<2>,
-                                              SDTCisOpSmallerThanOp<0, 2>]>>;
 def X86vfpext  : SDNode<"X86ISD::VFPEXT",
                         SDTypeProfile<1, 1, [SDTCisVec<0>, SDTCisVec<1>,
                                              SDTCisFP<0>, SDTCisFP<1>,
@@ -147,14 +146,21 @@ def X86pcmpeqm  : SDNode<"X86ISD::PCMPEQM", X86IntCmpMask, [SDNPCommutative]>;
 def X86pcmpgtm  : SDNode<"X86ISD::PCMPGTM", X86IntCmpMask>;
 
 def X86CmpMaskCC :
-      SDTypeProfile<1, 3, [SDTCisVec<0>, SDTCisInt<0>, SDTCisVec<1>,
-                           SDTCisSameAs<1, 2>, SDTCisVT<3, i8>]>;
+      SDTypeProfile<1, 3, [SDTCisVec<0>, SDTCVecEltisVT<0, i1>,
+                       SDTCisVec<1>, SDTCisSameAs<2, 1>,
+                       SDTCisSameNumEltsAs<0, 1>, SDTCisVT<3, i8>]>;
+def X86CmpMaskCCRound :
+      SDTypeProfile<1, 4, [SDTCisVec<0>,SDTCVecEltisVT<0, i1>,
+                       SDTCisVec<1>, SDTCisSameAs<2, 1>,
+                       SDTCisSameNumEltsAs<0, 1>, SDTCisVT<3, i8>,
+                       SDTCisInt<4>]>;
 def X86CmpMaskCCScalar :
       SDTypeProfile<1, 3, [SDTCisInt<0>, SDTCisSameAs<1, 2>, SDTCisVT<3, i8>]>;
 
-def X86cmpm   : SDNode<"X86ISD::CMPM",    X86CmpMaskCC>;
-def X86cmpmu  : SDNode<"X86ISD::CMPMU",   X86CmpMaskCC>;
-def X86cmpms  : SDNode<"X86ISD::FSETCC",  X86CmpMaskCCScalar>;
+def X86cmpm    : SDNode<"X86ISD::CMPM",     X86CmpMaskCC>;
+def X86cmpmRnd : SDNode<"X86ISD::CMPM_RND", X86CmpMaskCCRound>;
+def X86cmpmu   : SDNode<"X86ISD::CMPMU",    X86CmpMaskCC>;
+def X86cmpms   : SDNode<"X86ISD::FSETCC",   X86CmpMaskCCScalar>;
 
 def X86vshl    : SDNode<"X86ISD::VSHL",
                         SDTypeProfile<1, 2, [SDTCisVec<0>, SDTCisSameAs<0,1>,
@@ -173,16 +179,23 @@ def X86vsrai   : SDNode<"X86ISD::VSRAI", SDTIntShiftOp>;
 def SDTX86CmpPTest : SDTypeProfile<1, 2, [SDTCisVT<0, i32>,
                                           SDTCisVec<1>,
                                           SDTCisSameAs<2, 1>]>;
+def X86addus   : SDNode<"X86ISD::ADDUS", SDTIntBinOp>;
 def X86subus   : SDNode<"X86ISD::SUBUS", SDTIntBinOp>;
+def X86adds    : SDNode<"X86ISD::ADDS", SDTIntBinOp>;
+def X86subs    : SDNode<"X86ISD::SUBS", SDTIntBinOp>;
+def X86mulhrs  : SDNode<"X86ISD::MULHRS" , SDTIntBinOp>;
+def X86avg     : SDNode<"X86ISD::AVG" , SDTIntBinOp>;
 def X86ptest   : SDNode<"X86ISD::PTEST", SDTX86CmpPTest>;
 def X86testp   : SDNode<"X86ISD::TESTP", SDTX86CmpPTest>;
 def X86kortest : SDNode<"X86ISD::KORTEST", SDTX86CmpPTest>;
 def X86testm   : SDNode<"X86ISD::TESTM", SDTypeProfile<1, 2, [SDTCisVec<0>,
-                                          SDTCisVec<1>,
-                                          SDTCisSameAs<2, 1>]>>;
+                                          SDTCisVec<1>, SDTCisSameAs<2, 1>,
+                                          SDTCVecEltisVT<0, i1>,
+                                          SDTCisSameNumEltsAs<0, 1>]>>;
 def X86testnm  : SDNode<"X86ISD::TESTNM", SDTypeProfile<1, 2, [SDTCisVec<0>,
-                                          SDTCisVec<1>,
-                                          SDTCisSameAs<2, 1>]>>;
+                                          SDTCisVec<1>, SDTCisSameAs<2, 1>,
+                                          SDTCVecEltisVT<0, i1>,
+                                          SDTCisSameNumEltsAs<0, 1>]>>;
 def X86select  : SDNode<"X86ISD::SELECT"     , SDTSelect>;
 
 def X86pmuludq : SDNode<"X86ISD::PMULUDQ",
@@ -192,6 +205,14 @@ def X86pmuldq  : SDNode<"X86ISD::PMULDQ",
                          SDTypeProfile<1, 2, [SDTCisVec<0>, SDTCisVec<1>,
                                        SDTCisSameAs<1,2>]>>;
 
+def X86extrqi : SDNode<"X86ISD::EXTRQI",
+                  SDTypeProfile<1, 3, [SDTCisVT<0, v2i64>, SDTCisSameAs<0,1>,
+                                       SDTCisVT<2, i8>, SDTCisVT<3, i8>]>>;
+def X86insertqi : SDNode<"X86ISD::INSERTQI",
+                    SDTypeProfile<1, 4, [SDTCisVT<0, v2i64>, SDTCisSameAs<0,1>,
+                                         SDTCisSameAs<1,2>, SDTCisVT<3, i8>,
+                                         SDTCisVT<4, i8>]>>;
+
 // Specific shuffle nodes - At some point ISD::VECTOR_SHUFFLE will always get
 // translated into one of the target nodes below during lowering.
 // Note: this is a work in progress...
@@ -207,6 +228,10 @@ def SDTShuff2OpI : SDTypeProfile<1, 2, [SDTCisVec<0>,
                                  SDTCisSameAs<0,1>, SDTCisInt<2>]>;
 def SDTShuff3OpI : SDTypeProfile<1, 3, [SDTCisVec<0>, SDTCisSameAs<0,1>,
                                  SDTCisSameAs<0,2>, SDTCisInt<3>]>;
+def SDTFPBinOpImmRound: SDTypeProfile<1, 4, [SDTCisVec<0>, SDTCisSameAs<0,1>,
+                             SDTCisSameAs<0,2>, SDTCisInt<3>, SDTCisInt<4>]>;
+def SDTFPUnaryOpImmRound: SDTypeProfile<1, 3, [SDTCisVec<0>, SDTCisSameAs<0,1>,
+                              SDTCisInt<2>, SDTCisInt<3>]>;
 
 def SDTVBroadcast  : SDTypeProfile<1, 1, [SDTCisVec<0>]>;
 def SDTVBroadcastm : SDTypeProfile<1, 2, [SDTCisVec<0>, SDTCisVec<1>]>;
@@ -217,6 +242,9 @@ def SDTBlend : SDTypeProfile<1, 3, [SDTCisVec<0>, SDTCisSameAs<0,1>,
 def SDTFPBinOpRound : SDTypeProfile<1, 3, [      // fadd_round, fmul_round, etc.
   SDTCisSameAs<0, 1>, SDTCisSameAs<0, 2>, SDTCisFP<0>, SDTCisInt<3>]>;
 
+def SDTFPUnaryOpRound : SDTypeProfile<1, 2, [      // fsqrt_round, fgetexp_round, etc.
+  SDTCisSameAs<0, 1>, SDTCisFP<0>, SDTCisInt<2>]>;
+
 def SDTFma : SDTypeProfile<1, 3, [SDTCisSameAs<0,1>,
                            SDTCisSameAs<1,2>, SDTCisSameAs<1,3>]>;
 def SDTFmaRound : SDTypeProfile<1, 4, [SDTCisSameAs<0,1>,
@@ -230,12 +258,14 @@ def STDFp3SrcRm : SDTypeProfile<1, 4, [SDTCisSameAs<0,1>,
 
 def X86PAlignr : SDNode<"X86ISD::PALIGNR", SDTShuff3OpI>;
 def X86VAlign  : SDNode<"X86ISD::VALIGN", SDTShuff3OpI>;
+def X86Abs     : SDNode<"X86ISD::ABS", SDTIntUnaryOp>;
 
 def X86PShufd  : SDNode<"X86ISD::PSHUFD", SDTShuff2OpI>;
 def X86PShufhw : SDNode<"X86ISD::PSHUFHW", SDTShuff2OpI>;
 def X86PShuflw : SDNode<"X86ISD::PSHUFLW", SDTShuff2OpI>;
 
-def X86Shufp : SDNode<"X86ISD::SHUFP", SDTShuff3OpI>;
+def X86Shufp   : SDNode<"X86ISD::SHUFP", SDTShuff3OpI>;
+def X86Shuf128 : SDNode<"X86ISD::SHUF128", SDTShuff3OpI>;
 
 def X86Movddup  : SDNode<"X86ISD::MOVDDUP", SDTShuff1Op>;
 def X86Movshdup : SDNode<"X86ISD::MOVSHDUP", SDTShuff1Op>;
@@ -258,6 +288,9 @@ def X86Packus : SDNode<"X86ISD::PACKUS", SDTPack>;
 def X86Unpckl : SDNode<"X86ISD::UNPCKL", SDTShuff2Op>;
 def X86Unpckh : SDNode<"X86ISD::UNPCKH", SDTShuff2Op>;
 
+def X86vpmaddubsw  : SDNode<"X86ISD::VPMADDUBSW" , SDTPack>;
+def X86vpmaddwd    : SDNode<"X86ISD::VPMADDWD"   , SDTPack>;
+
 def X86VPermilpv  : SDNode<"X86ISD::VPERMILPV", SDTShuff2OpM>;
 def X86VPermilpi  : SDNode<"X86ISD::VPERMILPI", SDTShuff2OpI>;
 def X86VPermv     : SDNode<"X86ISD::VPERMV",    SDTShuff2Op>;
@@ -267,8 +300,15 @@ def X86VPermiv3   : SDNode<"X86ISD::VPERMIV3",  SDTShuff3Op>;
 
 def X86VPerm2x128 : SDNode<"X86ISD::VPERM2X128", SDTShuff3OpI>;
 
+def X86VFixupimm       : SDNode<"X86ISD::VFIXUPIMM", SDTFPBinOpImmRound>;
+def X86VRange          : SDNode<"X86ISD::VRANGE", SDTFPBinOpImmRound>;
+def X86VReduce          : SDNode<"X86ISD::VREDUCE", SDTFPUnaryOpImmRound>;
+def X86VRndScale       : SDNode<"X86ISD::VRNDSCALE", SDTFPUnaryOpImmRound>;
+
+def X86SubVBroadcast : SDNode<"X86ISD::SUBV_BROADCAST",
+                    SDTypeProfile<1, 1, [SDTCisVec<0>, SDTCisVec<1>,
+                                         SDTCisSubVecOfVec<1, 0>]>, []>;
 def X86VBroadcast : SDNode<"X86ISD::VBROADCAST", SDTVBroadcast>;
-def X86VBroadcastm : SDNode<"X86ISD::VBROADCASTM", SDTVBroadcastm>;
 def X86Vinsert   : SDNode<"X86ISD::VINSERT",  SDTypeProfile<1, 3,
                               [SDTCisSameAs<0, 1>, SDTCisPtrTy<3>]>, []>;
 def X86Vextract   : SDNode<"X86ISD::VEXTRACT",  SDTypeProfile<1, 2,
@@ -282,8 +322,12 @@ def X86faddRnd   : SDNode<"X86ISD::FADD_RND",  SDTFPBinOpRound>;
 def X86fsubRnd   : SDNode<"X86ISD::FSUB_RND",  SDTFPBinOpRound>;
 def X86fmulRnd   : SDNode<"X86ISD::FMUL_RND",  SDTFPBinOpRound>;
 def X86fdivRnd   : SDNode<"X86ISD::FDIV_RND",  SDTFPBinOpRound>;
-def X86fmaxRnd   : SDNode<"X86ISD::FMAX",      SDTFPBinOpRound>;
-def X86fminRnd   : SDNode<"X86ISD::FMIN",      SDTFPBinOpRound>;
+def X86fmaxRnd   : SDNode<"X86ISD::FMAX_RND",      SDTFPBinOpRound>;
+def X86scalef    : SDNode<"X86ISD::SCALEF",    SDTFPBinOpRound>;
+def X86fminRnd   : SDNode<"X86ISD::FMIN_RND",      SDTFPBinOpRound>;
+def X86fsqrtRnd     : SDNode<"X86ISD::FSQRT_RND",  SDTFPUnaryOpRound>;
+def X86fgetexpRnd   : SDNode<"X86ISD::FGETEXP_RND", SDTFPUnaryOpRound>;
+def X86fgetexpRnds  : SDNode<"X86ISD::FGETEXP_RND", STDFp2SrcRm>;
 
 def X86Fmadd     : SDNode<"X86ISD::FMADD",     SDTFma>;
 def X86Fnmadd    : SDNode<"X86ISD::FNMADD",    SDTFma>;
@@ -305,7 +349,8 @@ def X86exp2      : SDNode<"X86ISD::EXP2",     STDFp1SrcRm>;
 
 def X86rsqrt28s  : SDNode<"X86ISD::RSQRT28",  STDFp2SrcRm>;
 def X86rcp28s    : SDNode<"X86ISD::RCP28",    STDFp2SrcRm>;
-def X86RndScale  : SDNode<"X86ISD::RNDSCALE", STDFp3SrcRm>;
+def X86RndScales : SDNode<"X86ISD::VRNDSCALE", STDFp3SrcRm>;
+def X86Reduces   : SDNode<"X86ISD::VREDUCE", STDFp3SrcRm>;
 
 def SDT_PCMPISTRI : SDTypeProfile<2, 3, [SDTCisVT<0, i32>, SDTCisVT<1, i32>,
                                          SDTCisVT<2, v16i8>, SDTCisVT<3, v16i8>,
@@ -318,12 +363,78 @@ def SDT_PCMPESTRI : SDTypeProfile<2, 5, [SDTCisVT<0, i32>, SDTCisVT<1, i32>,
 def X86pcmpistri : SDNode<"X86ISD::PCMPISTRI", SDT_PCMPISTRI>;
 def X86pcmpestri : SDNode<"X86ISD::PCMPESTRI", SDT_PCMPESTRI>;
 
-def X86compress: SDNode<"X86ISD::COMPRESS", SDTypeProfile<1, 3,
-                              [SDTCisSameAs<0, 2>, SDTCisSameAs<0, 3>,
-                               SDTCisVec<3>, SDTCisVec<1>, SDTCisInt<1>]>, []>;
-def X86expand  : SDNode<"X86ISD::EXPAND", SDTypeProfile<1, 3,
-                              [SDTCisSameAs<0, 3>,
-                               SDTCisVec<3>, SDTCisVec<1>, SDTCisInt<1>]>, []>;
+def X86compress: SDNode<"X86ISD::COMPRESS", SDTypeProfile<1, 1,
+                              [SDTCisSameAs<0, 1>, SDTCisVec<1>]>, []>;
+def X86expand  : SDNode<"X86ISD::EXPAND", SDTypeProfile<1, 1,
+                              [SDTCisSameAs<0, 1>, SDTCisVec<1>]>, []>;
+
+def SDTintToFPRound: SDTypeProfile<1, 3, [SDTCisVec<0>, SDTCisFP<0>,
+                               SDTCisSameAs<0,1>, SDTCisInt<2>, SDTCisInt<3>]>;
+
+def SDTDoubleToInt: SDTypeProfile<1, 1, [SDTCisVec<0>, SDTCisVec<1>,
+                                         SDTCisInt<0>, SDTCVecEltisVT<1, f64>]>;
+def SDTFloatToInt: SDTypeProfile<1, 1, [SDTCisVec<0>, SDTCisVec<1>,
+                                         SDTCisInt<0>, SDTCVecEltisVT<1, f32>]>;
+
+def SDTDoubleToIntRnd: SDTypeProfile<1, 2, [SDTCisVec<0>, SDTCisVec<1>,
+                                         SDTCisInt<0>, SDTCVecEltisVT<1, f64>]>;
+def SDTFloatToIntRnd: SDTypeProfile<1, 2, [SDTCisVec<0>, SDTCisVec<1>,
+                                         SDTCisInt<0>, SDTCVecEltisVT<1, f32>]>;
+
+def SDTVintToFPRound: SDTypeProfile<1, 2, [SDTCisVec<0>, SDTCisVec<1>,
+                                           SDTCisFP<0>, SDTCVecEltisVT<1, i32>,
+                                           SDTCisInt<2>]>;
+def SDTVlongToFPRound: SDTypeProfile<1, 2, [SDTCisVec<0>, SDTCisVec<1>,
+                                           SDTCisFP<0>, SDTCVecEltisVT<1, i64>,
+                                           SDTCisInt<2>]>;
+
+def SDTVFPToIntRound: SDTypeProfile<1, 2, [SDTCisVec<0>, SDTCisVec<1>,
+                                           SDTCisFP<1>, SDTCVecEltisVT<0, i32>,
+                                           SDTCisInt<2>]>;
+def SDTVFPToLongRound: SDTypeProfile<1, 2, [SDTCisVec<0>, SDTCisVec<1>,
+                                           SDTCisFP<1>, SDTCVecEltisVT<0, i64>,
+                                           SDTCisInt<2>]>;
+
+// Scalar
+def X86SintToFpRnd  : SDNode<"X86ISD::SINT_TO_FP_RND",  SDTintToFPRound>;
+def X86UintToFpRnd  : SDNode<"X86ISD::UINT_TO_FP_RND",  SDTintToFPRound>;
+
+// Vector with rounding mode
+
+// cvtt fp-to-int staff
+def X86VFpToSintRnd   : SDNode<"ISD::FP_TO_SINT",  SDTVFPToIntRound>;
+def X86VFpToUintRnd   : SDNode<"ISD::FP_TO_UINT",  SDTVFPToIntRound>;
+def X86VFpToSlongRnd  : SDNode<"ISD::FP_TO_SINT",  SDTVFPToLongRound>;
+def X86VFpToUlongRnd  : SDNode<"ISD::FP_TO_UINT",  SDTVFPToLongRound>;
+
+def X86VSintToFpRnd   : SDNode<"ISD::SINT_TO_FP",  SDTVintToFPRound>;
+def X86VUintToFpRnd   : SDNode<"ISD::UINT_TO_FP",  SDTVintToFPRound>;
+def X86VSlongToFpRnd  : SDNode<"ISD::SINT_TO_FP",  SDTVlongToFPRound>;
+def X86VUlongToFpRnd  : SDNode<"ISD::UINT_TO_FP",  SDTVlongToFPRound>;
+
+// cvt fp-to-int staff
+def X86cvtps2IntRnd      : SDNode<"X86ISD::FP_TO_SINT_RND",  SDTFloatToIntRnd>;
+def X86cvtps2UIntRnd     : SDNode<"X86ISD::FP_TO_UINT_RND",  SDTFloatToIntRnd>;
+def X86cvtpd2IntRnd      : SDNode<"X86ISD::FP_TO_SINT_RND",  SDTDoubleToIntRnd>;
+def X86cvtpd2UIntRnd     : SDNode<"X86ISD::FP_TO_UINT_RND",  SDTDoubleToIntRnd>;
+
+// Vector without rounding mode
+def X86cvtps2Int      : SDNode<"X86ISD::FP_TO_SINT_RND",  SDTFloatToInt>;
+def X86cvtps2UInt     : SDNode<"X86ISD::FP_TO_UINT_RND",  SDTFloatToInt>;
+def X86cvtpd2Int      : SDNode<"X86ISD::FP_TO_SINT_RND",  SDTDoubleToInt>;
+def X86cvtpd2UInt     : SDNode<"X86ISD::FP_TO_UINT_RND",  SDTDoubleToInt>;
+
+def X86vfpextRnd  : SDNode<"X86ISD::VFPEXT",
+                        SDTypeProfile<1, 2, [SDTCisVec<0>, SDTCisVec<1>,
+                                             SDTCisFP<0>, SDTCisFP<1>,
+                                             SDTCisOpSmallerThanOp<1, 0>,
+                                             SDTCisInt<2>]>>;
+def X86vfproundRnd: SDNode<"X86ISD::VFPROUND",
+                        SDTypeProfile<1, 2, [SDTCisVec<0>, SDTCisVec<1>,
+                                             SDTCisFP<0>, SDTCisFP<1>,
+                                             SDTCVecEltisVT<0, f32>,
+                                             SDTCVecEltisVT<1, f64>,
+                                             SDTCisInt<2>]>>;
 
 //===----------------------------------------------------------------------===//
 // SSE Complex Patterns
@@ -526,6 +637,98 @@ def unalignednontemporalstore : PatFrag<(ops node:$val, node:$ptr),
   return false;
 }]>;
 
+def mgatherv4i32 : PatFrag<(ops node:$src1, node:$src2, node:$src3),
+  (masked_gather node:$src1, node:$src2, node:$src3) , [{
+  if (MaskedGatherSDNode *Mgt = dyn_cast<MaskedGatherSDNode>(N))
+    return (Mgt->getIndex().getValueType() == MVT::v4i32 ||
+            Mgt->getBasePtr().getValueType() == MVT::v4i32);
+  return false;
+}]>;
+
+def mgatherv8i32 : PatFrag<(ops node:$src1, node:$src2, node:$src3),
+  (masked_gather node:$src1, node:$src2, node:$src3) , [{
+  if (MaskedGatherSDNode *Mgt = dyn_cast<MaskedGatherSDNode>(N))
+    return (Mgt->getIndex().getValueType() == MVT::v8i32 ||
+            Mgt->getBasePtr().getValueType() == MVT::v8i32);
+  return false;
+}]>;
+
+def mgatherv2i64 : PatFrag<(ops node:$src1, node:$src2, node:$src3),
+  (masked_gather node:$src1, node:$src2, node:$src3) , [{
+  if (MaskedGatherSDNode *Mgt = dyn_cast<MaskedGatherSDNode>(N))
+    return (Mgt->getIndex().getValueType() == MVT::v2i64 ||
+            Mgt->getBasePtr().getValueType() == MVT::v2i64);
+  return false;
+}]>;
+def mgatherv4i64 : PatFrag<(ops node:$src1, node:$src2, node:$src3),
+  (masked_gather node:$src1, node:$src2, node:$src3) , [{
+  if (MaskedGatherSDNode *Mgt = dyn_cast<MaskedGatherSDNode>(N))
+    return (Mgt->getIndex().getValueType() == MVT::v4i64 ||
+            Mgt->getBasePtr().getValueType() == MVT::v4i64);
+  return false;
+}]>;
+def mgatherv8i64 : PatFrag<(ops node:$src1, node:$src2, node:$src3),
+  (masked_gather node:$src1, node:$src2, node:$src3) , [{
+  if (MaskedGatherSDNode *Mgt = dyn_cast<MaskedGatherSDNode>(N))
+    return (Mgt->getIndex().getValueType() == MVT::v8i64 ||
+            Mgt->getBasePtr().getValueType() == MVT::v8i64);
+  return false;
+}]>;
+def mgatherv16i32 : PatFrag<(ops node:$src1, node:$src2, node:$src3),
+  (masked_gather node:$src1, node:$src2, node:$src3) , [{
+  if (MaskedGatherSDNode *Mgt = dyn_cast<MaskedGatherSDNode>(N))
+    return (Mgt->getIndex().getValueType() == MVT::v16i32 ||
+            Mgt->getBasePtr().getValueType() == MVT::v16i32);
+  return false;
+}]>;
+
+def mscatterv2i64 : PatFrag<(ops node:$src1, node:$src2, node:$src3),
+  (masked_scatter node:$src1, node:$src2, node:$src3) , [{
+  if (MaskedScatterSDNode *Sc = dyn_cast<MaskedScatterSDNode>(N))
+    return (Sc->getIndex().getValueType() == MVT::v2i64 ||
+            Sc->getBasePtr().getValueType() == MVT::v2i64);
+  return false;
+}]>;
+
+def mscatterv4i32 : PatFrag<(ops node:$src1, node:$src2, node:$src3),
+  (masked_scatter node:$src1, node:$src2, node:$src3) , [{
+  if (MaskedScatterSDNode *Sc = dyn_cast<MaskedScatterSDNode>(N))
+    return (Sc->getIndex().getValueType() == MVT::v4i32 ||
+            Sc->getBasePtr().getValueType() == MVT::v4i32);
+  return false;
+}]>;
+
+def mscatterv4i64 : PatFrag<(ops node:$src1, node:$src2, node:$src3),
+  (masked_scatter node:$src1, node:$src2, node:$src3) , [{
+  if (MaskedScatterSDNode *Sc = dyn_cast<MaskedScatterSDNode>(N))
+    return (Sc->getIndex().getValueType() == MVT::v4i64 ||
+            Sc->getBasePtr().getValueType() == MVT::v4i64);
+  return false;
+}]>;
+
+def mscatterv8i32 : PatFrag<(ops node:$src1, node:$src2, node:$src3),
+  (masked_scatter node:$src1, node:$src2, node:$src3) , [{
+  if (MaskedScatterSDNode *Sc = dyn_cast<MaskedScatterSDNode>(N))
+    return (Sc->getIndex().getValueType() == MVT::v8i32 ||
+            Sc->getBasePtr().getValueType() == MVT::v8i32);
+  return false;
+}]>;
+
+def mscatterv8i64 : PatFrag<(ops node:$src1, node:$src2, node:$src3),
+  (masked_scatter node:$src1, node:$src2, node:$src3) , [{
+  if (MaskedScatterSDNode *Sc = dyn_cast<MaskedScatterSDNode>(N))
+    return (Sc->getIndex().getValueType() == MVT::v8i64 ||
+            Sc->getBasePtr().getValueType() == MVT::v8i64);
+  return false;
+}]>;
+def mscatterv16i32 : PatFrag<(ops node:$src1, node:$src2, node:$src3),
+  (masked_scatter node:$src1, node:$src2, node:$src3) , [{
+  if (MaskedScatterSDNode *Sc = dyn_cast<MaskedScatterSDNode>(N))
+    return (Sc->getIndex().getValueType() == MVT::v16i32 ||
+            Sc->getBasePtr().getValueType() == MVT::v16i32);
+  return false;
+}]>;
+
 // 128-bit bitconvert pattern fragments
 def bc_v4f32 : PatFrag<(ops node:$in), (v4f32 (bitconvert node:$in))>;
 def bc_v2f64 : PatFrag<(ops node:$in), (v2f64 (bitconvert node:$in))>;
@@ -564,7 +767,7 @@ def fp32imm0 : PatLeaf<(f32 fpimm), [{
 
 def I8Imm : SDNodeXForm<imm, [{
   // Transformation function: get the low 8 bits.
-  return getI8Imm((uint8_t)N->getZExtValue());
+  return getI8Imm((uint8_t)N->getZExtValue(), SDLoc(N));
 }]>;
 
 def FROUND_NO_EXC : ImmLeaf<i32, [{ return Imm == 8; }]>;
@@ -575,31 +778,31 @@ def FROUND_CURRENT : ImmLeaf<i32, [{
 // BYTE_imm - Transform bit immediates into byte immediates.
 def BYTE_imm  : SDNodeXForm<imm, [{
   // Transformation function: imm >> 3
-  return getI32Imm(N->getZExtValue() >> 3);
+  return getI32Imm(N->getZExtValue() >> 3, SDLoc(N));
 }]>;
 
 // EXTRACT_get_vextract128_imm xform function: convert extract_subvector index
 // to VEXTRACTF128/VEXTRACTI128 imm.
 def EXTRACT_get_vextract128_imm : SDNodeXForm<extract_subvector, [{
-  return getI8Imm(X86::getExtractVEXTRACT128Immediate(N));
+  return getI8Imm(X86::getExtractVEXTRACT128Immediate(N), SDLoc(N));
 }]>;
 
 // INSERT_get_vinsert128_imm xform function: convert insert_subvector index to
 // VINSERTF128/VINSERTI128 imm.
 def INSERT_get_vinsert128_imm : SDNodeXForm<insert_subvector, [{
-  return getI8Imm(X86::getInsertVINSERT128Immediate(N));
+  return getI8Imm(X86::getInsertVINSERT128Immediate(N), SDLoc(N));
 }]>;
 
 // EXTRACT_get_vextract256_imm xform function: convert extract_subvector index
 // to VEXTRACTF64x4 imm.
 def EXTRACT_get_vextract256_imm : SDNodeXForm<extract_subvector, [{
-  return getI8Imm(X86::getExtractVEXTRACT256Immediate(N));
+  return getI8Imm(X86::getExtractVEXTRACT256Immediate(N), SDLoc(N));
 }]>;
 
 // INSERT_get_vinsert256_imm xform function: convert insert_subvector index to
 // VINSERTF64x4 imm.
 def INSERT_get_vinsert256_imm : SDNodeXForm<insert_subvector, [{
-  return getI8Imm(X86::getInsertVINSERT256Immediate(N));
+  return getI8Imm(X86::getInsertVINSERT256Immediate(N), SDLoc(N));
 }]>;
 
 def vextract128_extract : PatFrag<(ops node:$bigvec, node:$index),
@@ -655,29 +858,59 @@ def masked_load_unaligned : PatFrag<(ops node:$src1, node:$src2, node:$src3),
   return isa<MaskedLoadSDNode>(N);
 }]>;
 
+// masked store fragments.
+// X86mstore can't be implemented in core DAG files because some targets
+// doesn't support vector type ( llvm-tblgen will fail)
+def X86mstore : PatFrag<(ops node:$src1, node:$src2, node:$src3),
+                        (masked_store node:$src1, node:$src2, node:$src3), [{
+  return !cast<MaskedStoreSDNode>(N)->isTruncatingStore();
+}]>;
+
 def masked_store_aligned128 : PatFrag<(ops node:$src1, node:$src2, node:$src3),
-                         (masked_store node:$src1, node:$src2, node:$src3), [{
+                         (X86mstore node:$src1, node:$src2, node:$src3), [{
   if (auto *Store = dyn_cast<MaskedStoreSDNode>(N))
     return Store->getAlignment() >= 16;
   return false;
 }]>;
 
 def masked_store_aligned256 : PatFrag<(ops node:$src1, node:$src2, node:$src3),
-                         (masked_store node:$src1, node:$src2, node:$src3), [{
+                         (X86mstore node:$src1, node:$src2, node:$src3), [{
   if (auto *Store = dyn_cast<MaskedStoreSDNode>(N))
     return Store->getAlignment() >= 32;
   return false;
 }]>;
 
 def masked_store_aligned512 : PatFrag<(ops node:$src1, node:$src2, node:$src3),
-                         (masked_store node:$src1, node:$src2, node:$src3), [{
+                         (X86mstore node:$src1, node:$src2, node:$src3), [{
   if (auto *Store = dyn_cast<MaskedStoreSDNode>(N))
     return Store->getAlignment() >= 64;
   return false;
 }]>;
 
 def masked_store_unaligned : PatFrag<(ops node:$src1, node:$src2, node:$src3),
-                         (masked_store node:$src1, node:$src2, node:$src3), [{
+                         (X86mstore node:$src1, node:$src2, node:$src3), [{
   return isa<MaskedStoreSDNode>(N);
 }]>;
 
+// masked truncstore fragments
+// X86mtruncstore can't be implemented in core DAG files because some targets
+// doesn't support vector type ( llvm-tblgen will fail)
+def X86mtruncstore : PatFrag<(ops node:$src1, node:$src2, node:$src3),
+                             (masked_store node:$src1, node:$src2, node:$src3), [{
+    return cast<MaskedStoreSDNode>(N)->isTruncatingStore();
+}]>;
+def masked_truncstorevi8 :
+  PatFrag<(ops node:$src1, node:$src2, node:$src3),
+          (X86mtruncstore node:$src1, node:$src2, node:$src3), [{
+  return cast<MaskedStoreSDNode>(N)->getMemoryVT().getScalarType() == MVT::i8;
+}]>;
+def masked_truncstorevi16 :
+  PatFrag<(ops node:$src1, node:$src2, node:$src3),
+          (X86mtruncstore node:$src1, node:$src2, node:$src3), [{
+  return cast<MaskedStoreSDNode>(N)->getMemoryVT().getScalarType() == MVT::i16;
+}]>;
+def masked_truncstorevi32 :
+  PatFrag<(ops node:$src1, node:$src2, node:$src3),
+          (X86mtruncstore node:$src1, node:$src2, node:$src3), [{
+  return cast<MaskedStoreSDNode>(N)->getMemoryVT().getScalarType() == MVT::i32;
+}]>;
diff --git a/lib/Target/X86/X86InstrInfo.cpp b/lib/Target/X86/X86InstrInfo.cpp
index eefd0be7836..c33580c740c 100644
--- a/lib/Target/X86/X86InstrInfo.cpp
+++ b/lib/Target/X86/X86InstrInfo.cpp
@@ -269,26 +269,19 @@ X86InstrInfo::X86InstrInfo(X86Subtarget &STI)
     { X86::XOR8rr,      X86::XOR8mr,     0 }
   };
 
-  for (unsigned i = 0, e = array_lengthof(MemoryFoldTable2Addr); i != e; ++i) {
-    unsigned RegOp = MemoryFoldTable2Addr[i].RegOp;
-    unsigned MemOp = MemoryFoldTable2Addr[i].MemOp;
-    unsigned Flags = MemoryFoldTable2Addr[i].Flags;
+  for (X86MemoryFoldTableEntry Entry : MemoryFoldTable2Addr) {
     AddTableEntry(RegOp2MemOpTable2Addr, MemOp2RegOpTable,
-                  RegOp, MemOp,
+                  Entry.RegOp, Entry.MemOp,
                   // Index 0, folded load and store, no alignment requirement.
-                  Flags | TB_INDEX_0 | TB_FOLDED_LOAD | TB_FOLDED_STORE);
+                  Entry.Flags | TB_INDEX_0 | TB_FOLDED_LOAD | TB_FOLDED_STORE);
   }
 
-  // @LOCALMOD-BEGIN
-  uint16_t NoForwardForNaCl = STI.isTargetNaCl() ? TB_NO_FORWARD : 0;
-  // @LOCALMOD-END
-
-static const X86MemoryFoldTableEntry MemoryFoldTable0[] = {
+  static const X86MemoryFoldTableEntry MemoryFoldTable0[] = {
     { X86::BT16ri8,     X86::BT16mi8,       TB_FOLDED_LOAD },
     { X86::BT32ri8,     X86::BT32mi8,       TB_FOLDED_LOAD },
     { X86::BT64ri8,     X86::BT64mi8,       TB_FOLDED_LOAD },
-    { X86::CALL32r,     X86::CALL32m,       (uint16_t)(TB_FOLDED_LOAD | NoForwardForNaCl) },
-    { X86::CALL64r,     X86::CALL64m,       (uint16_t)(TB_FOLDED_LOAD | NoForwardForNaCl) },
+    { X86::CALL32r,     X86::CALL32m,       TB_FOLDED_LOAD },
+    { X86::CALL64r,     X86::CALL64m,       TB_FOLDED_LOAD },
     { X86::CMP16ri,     X86::CMP16mi,       TB_FOLDED_LOAD },
     { X86::CMP16ri8,    X86::CMP16mi8,      TB_FOLDED_LOAD },
     { X86::CMP16rr,     X86::CMP16mr,       TB_FOLDED_LOAD },
@@ -313,8 +306,8 @@ static const X86MemoryFoldTableEntry MemoryFoldTable0[] = {
     { X86::IMUL32r,     X86::IMUL32m,       TB_FOLDED_LOAD },
     { X86::IMUL64r,     X86::IMUL64m,       TB_FOLDED_LOAD },
     { X86::IMUL8r,      X86::IMUL8m,        TB_FOLDED_LOAD },
-    { X86::JMP32r,      X86::JMP32m,        (uint16_t)(TB_FOLDED_LOAD | NoForwardForNaCl) },
-    { X86::JMP64r,      X86::JMP64m,        (uint16_t)(TB_FOLDED_LOAD | NoForwardForNaCl) },
+    { X86::JMP32r,      X86::JMP32m,        TB_FOLDED_LOAD },
+    { X86::JMP64r,      X86::JMP64m,        TB_FOLDED_LOAD },
     { X86::MOV16ri,     X86::MOV16mi,       TB_FOLDED_STORE },
     { X86::MOV16rr,     X86::MOV16mr,       TB_FOLDED_STORE },
     { X86::MOV32ri,     X86::MOV32mi,       TB_FOLDED_STORE },
@@ -339,6 +332,9 @@ static const X86MemoryFoldTableEntry MemoryFoldTable0[] = {
     { X86::MUL8r,       X86::MUL8m,         TB_FOLDED_LOAD },
     { X86::PEXTRDrr,    X86::PEXTRDmr,      TB_FOLDED_STORE },
     { X86::PEXTRQrr,    X86::PEXTRQmr,      TB_FOLDED_STORE },
+    { X86::PUSH16r,     X86::PUSH16rmm,     TB_FOLDED_LOAD },
+    { X86::PUSH32r,     X86::PUSH32rmm,     TB_FOLDED_LOAD },
+    { X86::PUSH64r,     X86::PUSH64rmm,     TB_FOLDED_LOAD },
     { X86::SETAEr,      X86::SETAEm,        TB_FOLDED_STORE },
     { X86::SETAr,       X86::SETAm,         TB_FOLDED_STORE },
     { X86::SETBEr,      X86::SETBEm,        TB_FOLDED_STORE },
@@ -355,8 +351,8 @@ static const X86MemoryFoldTableEntry MemoryFoldTable0[] = {
     { X86::SETOr,       X86::SETOm,         TB_FOLDED_STORE },
     { X86::SETPr,       X86::SETPm,         TB_FOLDED_STORE },
     { X86::SETSr,       X86::SETSm,         TB_FOLDED_STORE },
-    { X86::TAILJMPr,    X86::TAILJMPm,      (uint16_t)(TB_FOLDED_LOAD | NoForwardForNaCl) },
-    { X86::TAILJMPr64,  X86::TAILJMPm64,    (uint16_t)(TB_FOLDED_LOAD | NoForwardForNaCl) },
+    { X86::TAILJMPr,    X86::TAILJMPm,      TB_FOLDED_LOAD },
+    { X86::TAILJMPr64,  X86::TAILJMPm64,    TB_FOLDED_LOAD },
     { X86::TAILJMPr64_REX, X86::TAILJMPm64_REX, TB_FOLDED_LOAD },
     { X86::TEST16ri,    X86::TEST16mi,      TB_FOLDED_LOAD },
     { X86::TEST32ri,    X86::TEST32mi,      TB_FOLDED_LOAD },
@@ -428,15 +424,18 @@ static const X86MemoryFoldTableEntry MemoryFoldTable0[] = {
     { X86::VCVTPS2PHYrr,       X86::VCVTPS2PHYmr,     TB_FOLDED_STORE }
   };
 
-  for (unsigned i = 0, e = array_lengthof(MemoryFoldTable0); i != e; ++i) {
-    unsigned RegOp      = MemoryFoldTable0[i].RegOp;
-    unsigned MemOp      = MemoryFoldTable0[i].MemOp;
-    unsigned Flags      = MemoryFoldTable0[i].Flags;
+  for (X86MemoryFoldTableEntry Entry : MemoryFoldTable0) {
     AddTableEntry(RegOp2MemOpTable0, MemOp2RegOpTable,
-                  RegOp, MemOp, TB_INDEX_0 | Flags);
+                  Entry.RegOp, Entry.MemOp, TB_INDEX_0 | Entry.Flags);
   }
 
   static const X86MemoryFoldTableEntry MemoryFoldTable1[] = {
+    { X86::BSF16rr,         X86::BSF16rm,             0 },
+    { X86::BSF32rr,         X86::BSF32rm,             0 },
+    { X86::BSF64rr,         X86::BSF64rm,             0 },
+    { X86::BSR16rr,         X86::BSR16rm,             0 },
+    { X86::BSR32rr,         X86::BSR32rm,             0 },
+    { X86::BSR64rr,         X86::BSR64rm,             0 },
     { X86::CMP16rr,         X86::CMP16rm,             0 },
     { X86::CMP32rr,         X86::CMP32rm,             0 },
     { X86::CMP64rr,         X86::CMP64rm,             0 },
@@ -530,11 +529,11 @@ static const X86MemoryFoldTableEntry MemoryFoldTable0[] = {
     { X86::PSHUFLWri,       X86::PSHUFLWmi,           TB_ALIGN_16 },
     { X86::PTESTrr,         X86::PTESTrm,             TB_ALIGN_16 },
     { X86::RCPPSr,          X86::RCPPSm,              TB_ALIGN_16 },
-    { X86::RCPPSr_Int,      X86::RCPPSm_Int,          TB_ALIGN_16 },
+    { X86::RCPSSr,          X86::RCPSSm,              0 },
+    { X86::RCPSSr_Int,      X86::RCPSSm_Int,          0 },
     { X86::ROUNDPDr,        X86::ROUNDPDm,            TB_ALIGN_16 },
     { X86::ROUNDPSr,        X86::ROUNDPSm,            TB_ALIGN_16 },
     { X86::RSQRTPSr,        X86::RSQRTPSm,            TB_ALIGN_16 },
-    { X86::RSQRTPSr_Int,    X86::RSQRTPSm_Int,        TB_ALIGN_16 },
     { X86::RSQRTSSr,        X86::RSQRTSSm,            0 },
     { X86::RSQRTSSr_Int,    X86::RSQRTSSm_Int,        0 },
     { X86::SQRTPDr,         X86::SQRTPDm,             TB_ALIGN_16 },
@@ -638,11 +637,9 @@ static const X86MemoryFoldTableEntry MemoryFoldTable0[] = {
     { X86::VPSHUFLWri,      X86::VPSHUFLWmi,          0 },
     { X86::VPTESTrr,        X86::VPTESTrm,            0 },
     { X86::VRCPPSr,         X86::VRCPPSm,             0 },
-    { X86::VRCPPSr_Int,     X86::VRCPPSm_Int,         0 },
     { X86::VROUNDPDr,       X86::VROUNDPDm,           0 },
     { X86::VROUNDPSr,       X86::VROUNDPSm,           0 },
     { X86::VRSQRTPSr,       X86::VRSQRTPSm,           0 },
-    { X86::VRSQRTPSr_Int,   X86::VRSQRTPSm_Int,       0 },
     { X86::VSQRTPDr,        X86::VSQRTPDm,            0 },
     { X86::VSQRTPSr,        X86::VSQRTPSm,            0 },
     { X86::VTESTPDrr,       X86::VTESTPDrm,           0 },
@@ -671,11 +668,9 @@ static const X86MemoryFoldTableEntry MemoryFoldTable0[] = {
     { X86::VPERMILPSYri,    X86::VPERMILPSYmi,        0 },
     { X86::VPTESTYrr,       X86::VPTESTYrm,           0 },
     { X86::VRCPPSYr,        X86::VRCPPSYm,            0 },
-    { X86::VRCPPSYr_Int,    X86::VRCPPSYm_Int,        0 },
     { X86::VROUNDYPDr,      X86::VROUNDYPDm,          0 },
     { X86::VROUNDYPSr,      X86::VROUNDYPSm,          0 },
     { X86::VRSQRTPSYr,      X86::VRSQRTPSYm,          0 },
-    { X86::VRSQRTPSYr_Int,  X86::VRSQRTPSYm_Int,      0 },
     { X86::VSQRTPDYr,       X86::VSQRTPDYm,           0 },
     { X86::VSQRTPSYr,       X86::VSQRTPSYm,           0 },
     { X86::VTESTPDYrr,      X86::VTESTPDYrm,          0 },
@@ -864,14 +859,11 @@ static const X86MemoryFoldTableEntry MemoryFoldTable0[] = {
     { X86::VAESKEYGENASSIST128rr, X86::VAESKEYGENASSIST128rm, 0 }
   };
 
-  for (unsigned i = 0, e = array_lengthof(MemoryFoldTable1); i != e; ++i) {
-    unsigned RegOp = MemoryFoldTable1[i].RegOp;
-    unsigned MemOp = MemoryFoldTable1[i].MemOp;
-    unsigned Flags = MemoryFoldTable1[i].Flags;
+  for (X86MemoryFoldTableEntry Entry : MemoryFoldTable1) {
     AddTableEntry(RegOp2MemOpTable1, MemOp2RegOpTable,
-                  RegOp, MemOp,
+                  Entry.RegOp, Entry.MemOp,
                   // Index 1, folded load
-                  Flags | TB_INDEX_1 | TB_FOLDED_LOAD);
+                  Entry.Flags | TB_INDEX_1 | TB_FOLDED_LOAD);
   }
 
   static const X86MemoryFoldTableEntry MemoryFoldTable2[] = {
@@ -967,18 +959,9 @@ static const X86MemoryFoldTableEntry MemoryFoldTable0[] = {
     { X86::DPPDrri,         X86::DPPDrmi,       TB_ALIGN_16 },
     { X86::DPPSrri,         X86::DPPSrmi,       TB_ALIGN_16 },
 
-    // FIXME: We should not be folding Fs* scalar loads into vector
-    // instructions because the vector instructions require vector-sized
-    // loads. Lowering should create vector-sized instructions (the Fv*
-    // variants below) to allow load folding.
-    { X86::FsANDNPDrr,      X86::FsANDNPDrm,    TB_ALIGN_16 },
-    { X86::FsANDNPSrr,      X86::FsANDNPSrm,    TB_ALIGN_16 },
-    { X86::FsANDPDrr,       X86::FsANDPDrm,     TB_ALIGN_16 },
-    { X86::FsANDPSrr,       X86::FsANDPSrm,     TB_ALIGN_16 },
-    { X86::FsORPDrr,        X86::FsORPDrm,      TB_ALIGN_16 },
-    { X86::FsORPSrr,        X86::FsORPSrm,      TB_ALIGN_16 },
-    { X86::FsXORPDrr,       X86::FsXORPDrm,     TB_ALIGN_16 },
-    { X86::FsXORPSrr,       X86::FsXORPSrm,     TB_ALIGN_16 },
+    // Do not fold Fs* scalar logical op loads because there are no scalar
+    // load variants for these instructions. When folded, the load is required
+    // to be 128-bits, so the load size would not match.
 
     { X86::FvANDNPDrr,      X86::FvANDNPDrm,    TB_ALIGN_16 },
     { X86::FvANDNPSrr,      X86::FvANDNPSrm,    TB_ALIGN_16 },
@@ -1118,6 +1101,8 @@ static const X86MemoryFoldTableEntry MemoryFoldTable0[] = {
     { X86::PUNPCKLQDQrr,    X86::PUNPCKLQDQrm,  TB_ALIGN_16 },
     { X86::PUNPCKLWDrr,     X86::PUNPCKLWDrm,   TB_ALIGN_16 },
     { X86::PXORrr,          X86::PXORrm,        TB_ALIGN_16 },
+    { X86::ROUNDSDr,        X86::ROUNDSDm,      0 },
+    { X86::ROUNDSSr,        X86::ROUNDSSm,      0 },
     { X86::SBB32rr,         X86::SBB32rm,       0 },
     { X86::SBB64rr,         X86::SBB64rm,       0 },
     { X86::SHUFPDrri,       X86::SHUFPDrmi,     TB_ALIGN_16 },
@@ -1249,9 +1234,13 @@ static const X86MemoryFoldTableEntry MemoryFoldTable0[] = {
     { X86::VCVTSS2SDrr,       X86::VCVTSS2SDrm,        0 },
     { X86::Int_VCVTSS2SDrr,   X86::Int_VCVTSS2SDrm,    0 },
     { X86::VRCPSSr,           X86::VRCPSSm,            0 },
+    { X86::VRCPSSr_Int,       X86::VRCPSSm_Int,        0 },
     { X86::VRSQRTSSr,         X86::VRSQRTSSm,          0 },
+    { X86::VRSQRTSSr_Int,     X86::VRSQRTSSm_Int,      0 },
     { X86::VSQRTSDr,          X86::VSQRTSDm,           0 },
+    { X86::VSQRTSDr_Int,      X86::VSQRTSDm_Int,       0 },
     { X86::VSQRTSSr,          X86::VSQRTSSm,           0 },
+    { X86::VSQRTSSr_Int,      X86::VSQRTSSm_Int,       0 },
     { X86::VADDPDrr,          X86::VADDPDrm,           0 },
     { X86::VADDPSrr,          X86::VADDPSrm,           0 },
     { X86::VADDSDrr,          X86::VADDSDrm,           0 },
@@ -1410,6 +1399,8 @@ static const X86MemoryFoldTableEntry MemoryFoldTable0[] = {
     { X86::VPUNPCKLQDQrr,     X86::VPUNPCKLQDQrm,      0 },
     { X86::VPUNPCKLWDrr,      X86::VPUNPCKLWDrm,       0 },
     { X86::VPXORrr,           X86::VPXORrm,            0 },
+    { X86::VROUNDSDr,         X86::VROUNDSDm,          0 },
+    { X86::VROUNDSSr,         X86::VROUNDSSm,          0 },
     { X86::VSHUFPDrri,        X86::VSHUFPDrmi,         0 },
     { X86::VSHUFPSrri,        X86::VSHUFPSrmi,         0 },
     { X86::VSUBPDrr,          X86::VSUBPDrm,           0 },
@@ -1575,38 +1566,38 @@ static const X86MemoryFoldTableEntry MemoryFoldTable0[] = {
     { X86::VPXORYrr,          X86::VPXORYrm,           0 },
 
     // FMA4 foldable patterns
-    { X86::VFMADDSS4rr,       X86::VFMADDSS4mr,        0 },
-    { X86::VFMADDSD4rr,       X86::VFMADDSD4mr,        0 },
-    { X86::VFMADDPS4rr,       X86::VFMADDPS4mr,        0 },
-    { X86::VFMADDPD4rr,       X86::VFMADDPD4mr,        0 },
-    { X86::VFMADDPS4rrY,      X86::VFMADDPS4mrY,       0 },
-    { X86::VFMADDPD4rrY,      X86::VFMADDPD4mrY,       0 },
-    { X86::VFNMADDSS4rr,      X86::VFNMADDSS4mr,       0 },
-    { X86::VFNMADDSD4rr,      X86::VFNMADDSD4mr,       0 },
-    { X86::VFNMADDPS4rr,      X86::VFNMADDPS4mr,       0 },
-    { X86::VFNMADDPD4rr,      X86::VFNMADDPD4mr,       0 },
-    { X86::VFNMADDPS4rrY,     X86::VFNMADDPS4mrY,      0 },
-    { X86::VFNMADDPD4rrY,     X86::VFNMADDPD4mrY,      0 },
-    { X86::VFMSUBSS4rr,       X86::VFMSUBSS4mr,        0 },
-    { X86::VFMSUBSD4rr,       X86::VFMSUBSD4mr,        0 },
-    { X86::VFMSUBPS4rr,       X86::VFMSUBPS4mr,        0 },
-    { X86::VFMSUBPD4rr,       X86::VFMSUBPD4mr,        0 },
-    { X86::VFMSUBPS4rrY,      X86::VFMSUBPS4mrY,       0 },
-    { X86::VFMSUBPD4rrY,      X86::VFMSUBPD4mrY,       0 },
-    { X86::VFNMSUBSS4rr,      X86::VFNMSUBSS4mr,       0 },
-    { X86::VFNMSUBSD4rr,      X86::VFNMSUBSD4mr,       0 },
-    { X86::VFNMSUBPS4rr,      X86::VFNMSUBPS4mr,       0 },
-    { X86::VFNMSUBPD4rr,      X86::VFNMSUBPD4mr,       0 },
-    { X86::VFNMSUBPS4rrY,     X86::VFNMSUBPS4mrY,      0 },
-    { X86::VFNMSUBPD4rrY,     X86::VFNMSUBPD4mrY,      0 },
-    { X86::VFMADDSUBPS4rr,    X86::VFMADDSUBPS4mr,     0 },
-    { X86::VFMADDSUBPD4rr,    X86::VFMADDSUBPD4mr,     0 },
-    { X86::VFMADDSUBPS4rrY,   X86::VFMADDSUBPS4mrY,    0 },
-    { X86::VFMADDSUBPD4rrY,   X86::VFMADDSUBPD4mrY,    0 },
-    { X86::VFMSUBADDPS4rr,    X86::VFMSUBADDPS4mr,     0 },
-    { X86::VFMSUBADDPD4rr,    X86::VFMSUBADDPD4mr,     0 },
-    { X86::VFMSUBADDPS4rrY,   X86::VFMSUBADDPS4mrY,    0 },
-    { X86::VFMSUBADDPD4rrY,   X86::VFMSUBADDPD4mrY,    0 },
+    { X86::VFMADDSS4rr,       X86::VFMADDSS4mr,        TB_ALIGN_NONE },
+    { X86::VFMADDSD4rr,       X86::VFMADDSD4mr,        TB_ALIGN_NONE },
+    { X86::VFMADDPS4rr,       X86::VFMADDPS4mr,        TB_ALIGN_NONE },
+    { X86::VFMADDPD4rr,       X86::VFMADDPD4mr,        TB_ALIGN_NONE },
+    { X86::VFMADDPS4rrY,      X86::VFMADDPS4mrY,       TB_ALIGN_NONE },
+    { X86::VFMADDPD4rrY,      X86::VFMADDPD4mrY,       TB_ALIGN_NONE },
+    { X86::VFNMADDSS4rr,      X86::VFNMADDSS4mr,       TB_ALIGN_NONE },
+    { X86::VFNMADDSD4rr,      X86::VFNMADDSD4mr,       TB_ALIGN_NONE },
+    { X86::VFNMADDPS4rr,      X86::VFNMADDPS4mr,       TB_ALIGN_NONE },
+    { X86::VFNMADDPD4rr,      X86::VFNMADDPD4mr,       TB_ALIGN_NONE },
+    { X86::VFNMADDPS4rrY,     X86::VFNMADDPS4mrY,      TB_ALIGN_NONE },
+    { X86::VFNMADDPD4rrY,     X86::VFNMADDPD4mrY,      TB_ALIGN_NONE },
+    { X86::VFMSUBSS4rr,       X86::VFMSUBSS4mr,        TB_ALIGN_NONE },
+    { X86::VFMSUBSD4rr,       X86::VFMSUBSD4mr,        TB_ALIGN_NONE },
+    { X86::VFMSUBPS4rr,       X86::VFMSUBPS4mr,        TB_ALIGN_NONE },
+    { X86::VFMSUBPD4rr,       X86::VFMSUBPD4mr,        TB_ALIGN_NONE },
+    { X86::VFMSUBPS4rrY,      X86::VFMSUBPS4mrY,       TB_ALIGN_NONE },
+    { X86::VFMSUBPD4rrY,      X86::VFMSUBPD4mrY,       TB_ALIGN_NONE },
+    { X86::VFNMSUBSS4rr,      X86::VFNMSUBSS4mr,       TB_ALIGN_NONE },
+    { X86::VFNMSUBSD4rr,      X86::VFNMSUBSD4mr,       TB_ALIGN_NONE },
+    { X86::VFNMSUBPS4rr,      X86::VFNMSUBPS4mr,       TB_ALIGN_NONE },
+    { X86::VFNMSUBPD4rr,      X86::VFNMSUBPD4mr,       TB_ALIGN_NONE },
+    { X86::VFNMSUBPS4rrY,     X86::VFNMSUBPS4mrY,      TB_ALIGN_NONE },
+    { X86::VFNMSUBPD4rrY,     X86::VFNMSUBPD4mrY,      TB_ALIGN_NONE },
+    { X86::VFMADDSUBPS4rr,    X86::VFMADDSUBPS4mr,     TB_ALIGN_NONE },
+    { X86::VFMADDSUBPD4rr,    X86::VFMADDSUBPD4mr,     TB_ALIGN_NONE },
+    { X86::VFMADDSUBPS4rrY,   X86::VFMADDSUBPS4mrY,    TB_ALIGN_NONE },
+    { X86::VFMADDSUBPD4rrY,   X86::VFMADDSUBPD4mrY,    TB_ALIGN_NONE },
+    { X86::VFMSUBADDPS4rr,    X86::VFMSUBADDPS4mr,     TB_ALIGN_NONE },
+    { X86::VFMSUBADDPD4rr,    X86::VFMSUBADDPD4mr,     TB_ALIGN_NONE },
+    { X86::VFMSUBADDPS4rrY,   X86::VFMSUBADDPS4mrY,    TB_ALIGN_NONE },
+    { X86::VFMSUBADDPD4rrY,   X86::VFMSUBADDPD4mrY,    TB_ALIGN_NONE },
 
     // XOP foldable instructions
     { X86::VPCMOVrr,          X86::VPCMOVmr,            0 },
@@ -1694,8 +1685,8 @@ static const X86MemoryFoldTableEntry MemoryFoldTable0[] = {
     { X86::VPSUBQZrr,         X86::VPSUBQZrm,           0 },
     { X86::VSHUFPDZrri,       X86::VSHUFPDZrmi,         0 },
     { X86::VSHUFPSZrri,       X86::VSHUFPSZrmi,         0 },
-    { X86::VALIGNQrri,        X86::VALIGNQrmi,          0 },
-    { X86::VALIGNDrri,        X86::VALIGNDrmi,          0 },
+    { X86::VALIGNQZrri,       X86::VALIGNQZrmi,         0 },
+    { X86::VALIGNDZrri,       X86::VALIGNDZrmi,         0 },
     { X86::VPMULUDQZrr,       X86::VPMULUDQZrm,         0 },
     { X86::VBROADCASTSSZrkz,  X86::VBROADCASTSSZmkz,    TB_NO_REVERSE },
     { X86::VBROADCASTSDZrkz,  X86::VBROADCASTSDZmkz,    TB_NO_REVERSE },
@@ -1731,14 +1722,11 @@ static const X86MemoryFoldTableEntry MemoryFoldTable0[] = {
     { X86::SHA256RNDS2rr,     X86::SHA256RNDS2rm,       TB_ALIGN_16 }
   };
 
-  for (unsigned i = 0, e = array_lengthof(MemoryFoldTable2); i != e; ++i) {
-    unsigned RegOp = MemoryFoldTable2[i].RegOp;
-    unsigned MemOp = MemoryFoldTable2[i].MemOp;
-    unsigned Flags = MemoryFoldTable2[i].Flags;
+  for (X86MemoryFoldTableEntry Entry : MemoryFoldTable2) {
     AddTableEntry(RegOp2MemOpTable2, MemOp2RegOpTable,
-                  RegOp, MemOp,
+                  Entry.RegOp, Entry.MemOp,
                   // Index 2, folded load
-                  Flags | TB_INDEX_2 | TB_FOLDED_LOAD);
+                  Entry.Flags | TB_INDEX_2 | TB_FOLDED_LOAD);
   }
 
   static const X86MemoryFoldTableEntry MemoryFoldTable3[] = {
@@ -1850,38 +1838,38 @@ static const X86MemoryFoldTableEntry MemoryFoldTable0[] = {
     { X86::VFMSUBADDPDr213rY,     X86::VFMSUBADDPDr213mY,     TB_ALIGN_NONE },
 
     // FMA4 foldable patterns
-    { X86::VFMADDSS4rr,           X86::VFMADDSS4rm,           0           },
-    { X86::VFMADDSD4rr,           X86::VFMADDSD4rm,           0           },
-    { X86::VFMADDPS4rr,           X86::VFMADDPS4rm,           TB_ALIGN_16 },
-    { X86::VFMADDPD4rr,           X86::VFMADDPD4rm,           TB_ALIGN_16 },
-    { X86::VFMADDPS4rrY,          X86::VFMADDPS4rmY,          TB_ALIGN_32 },
-    { X86::VFMADDPD4rrY,          X86::VFMADDPD4rmY,          TB_ALIGN_32 },
-    { X86::VFNMADDSS4rr,          X86::VFNMADDSS4rm,          0           },
-    { X86::VFNMADDSD4rr,          X86::VFNMADDSD4rm,          0           },
-    { X86::VFNMADDPS4rr,          X86::VFNMADDPS4rm,          TB_ALIGN_16 },
-    { X86::VFNMADDPD4rr,          X86::VFNMADDPD4rm,          TB_ALIGN_16 },
-    { X86::VFNMADDPS4rrY,         X86::VFNMADDPS4rmY,         TB_ALIGN_32 },
-    { X86::VFNMADDPD4rrY,         X86::VFNMADDPD4rmY,         TB_ALIGN_32 },
-    { X86::VFMSUBSS4rr,           X86::VFMSUBSS4rm,           0           },
-    { X86::VFMSUBSD4rr,           X86::VFMSUBSD4rm,           0           },
-    { X86::VFMSUBPS4rr,           X86::VFMSUBPS4rm,           TB_ALIGN_16 },
-    { X86::VFMSUBPD4rr,           X86::VFMSUBPD4rm,           TB_ALIGN_16 },
-    { X86::VFMSUBPS4rrY,          X86::VFMSUBPS4rmY,          TB_ALIGN_32 },
-    { X86::VFMSUBPD4rrY,          X86::VFMSUBPD4rmY,          TB_ALIGN_32 },
-    { X86::VFNMSUBSS4rr,          X86::VFNMSUBSS4rm,          0           },
-    { X86::VFNMSUBSD4rr,          X86::VFNMSUBSD4rm,          0           },
-    { X86::VFNMSUBPS4rr,          X86::VFNMSUBPS4rm,          TB_ALIGN_16 },
-    { X86::VFNMSUBPD4rr,          X86::VFNMSUBPD4rm,          TB_ALIGN_16 },
-    { X86::VFNMSUBPS4rrY,         X86::VFNMSUBPS4rmY,         TB_ALIGN_32 },
-    { X86::VFNMSUBPD4rrY,         X86::VFNMSUBPD4rmY,         TB_ALIGN_32 },
-    { X86::VFMADDSUBPS4rr,        X86::VFMADDSUBPS4rm,        TB_ALIGN_16 },
-    { X86::VFMADDSUBPD4rr,        X86::VFMADDSUBPD4rm,        TB_ALIGN_16 },
-    { X86::VFMADDSUBPS4rrY,       X86::VFMADDSUBPS4rmY,       TB_ALIGN_32 },
-    { X86::VFMADDSUBPD4rrY,       X86::VFMADDSUBPD4rmY,       TB_ALIGN_32 },
-    { X86::VFMSUBADDPS4rr,        X86::VFMSUBADDPS4rm,        TB_ALIGN_16 },
-    { X86::VFMSUBADDPD4rr,        X86::VFMSUBADDPD4rm,        TB_ALIGN_16 },
-    { X86::VFMSUBADDPS4rrY,       X86::VFMSUBADDPS4rmY,       TB_ALIGN_32 },
-    { X86::VFMSUBADDPD4rrY,       X86::VFMSUBADDPD4rmY,       TB_ALIGN_32 },
+    { X86::VFMADDSS4rr,           X86::VFMADDSS4rm,           TB_ALIGN_NONE },
+    { X86::VFMADDSD4rr,           X86::VFMADDSD4rm,           TB_ALIGN_NONE },
+    { X86::VFMADDPS4rr,           X86::VFMADDPS4rm,           TB_ALIGN_NONE },
+    { X86::VFMADDPD4rr,           X86::VFMADDPD4rm,           TB_ALIGN_NONE },
+    { X86::VFMADDPS4rrY,          X86::VFMADDPS4rmY,          TB_ALIGN_NONE },
+    { X86::VFMADDPD4rrY,          X86::VFMADDPD4rmY,          TB_ALIGN_NONE },
+    { X86::VFNMADDSS4rr,          X86::VFNMADDSS4rm,          TB_ALIGN_NONE },
+    { X86::VFNMADDSD4rr,          X86::VFNMADDSD4rm,          TB_ALIGN_NONE },
+    { X86::VFNMADDPS4rr,          X86::VFNMADDPS4rm,          TB_ALIGN_NONE },
+    { X86::VFNMADDPD4rr,          X86::VFNMADDPD4rm,          TB_ALIGN_NONE },
+    { X86::VFNMADDPS4rrY,         X86::VFNMADDPS4rmY,         TB_ALIGN_NONE },
+    { X86::VFNMADDPD4rrY,         X86::VFNMADDPD4rmY,         TB_ALIGN_NONE },
+    { X86::VFMSUBSS4rr,           X86::VFMSUBSS4rm,           TB_ALIGN_NONE },
+    { X86::VFMSUBSD4rr,           X86::VFMSUBSD4rm,           TB_ALIGN_NONE },
+    { X86::VFMSUBPS4rr,           X86::VFMSUBPS4rm,           TB_ALIGN_NONE },
+    { X86::VFMSUBPD4rr,           X86::VFMSUBPD4rm,           TB_ALIGN_NONE },
+    { X86::VFMSUBPS4rrY,          X86::VFMSUBPS4rmY,          TB_ALIGN_NONE },
+    { X86::VFMSUBPD4rrY,          X86::VFMSUBPD4rmY,          TB_ALIGN_NONE },
+    { X86::VFNMSUBSS4rr,          X86::VFNMSUBSS4rm,          TB_ALIGN_NONE },
+    { X86::VFNMSUBSD4rr,          X86::VFNMSUBSD4rm,          TB_ALIGN_NONE },
+    { X86::VFNMSUBPS4rr,          X86::VFNMSUBPS4rm,          TB_ALIGN_NONE },
+    { X86::VFNMSUBPD4rr,          X86::VFNMSUBPD4rm,          TB_ALIGN_NONE },
+    { X86::VFNMSUBPS4rrY,         X86::VFNMSUBPS4rmY,         TB_ALIGN_NONE },
+    { X86::VFNMSUBPD4rrY,         X86::VFNMSUBPD4rmY,         TB_ALIGN_NONE },
+    { X86::VFMADDSUBPS4rr,        X86::VFMADDSUBPS4rm,        TB_ALIGN_NONE },
+    { X86::VFMADDSUBPD4rr,        X86::VFMADDSUBPD4rm,        TB_ALIGN_NONE },
+    { X86::VFMADDSUBPS4rrY,       X86::VFMADDSUBPS4rmY,       TB_ALIGN_NONE },
+    { X86::VFMADDSUBPD4rrY,       X86::VFMADDSUBPD4rmY,       TB_ALIGN_NONE },
+    { X86::VFMSUBADDPS4rr,        X86::VFMSUBADDPS4rm,        TB_ALIGN_NONE },
+    { X86::VFMSUBADDPD4rr,        X86::VFMSUBADDPD4rm,        TB_ALIGN_NONE },
+    { X86::VFMSUBADDPS4rrY,       X86::VFMSUBADDPS4rmY,       TB_ALIGN_NONE },
+    { X86::VFMSUBADDPD4rrY,       X86::VFMSUBADDPD4rmY,       TB_ALIGN_NONE },
 
     // XOP foldable instructions
     { X86::VPCMOVrr,              X86::VPCMOVrm,              0 },
@@ -1947,14 +1935,11 @@ static const X86MemoryFoldTableEntry MemoryFoldTable0[] = {
     { X86::VMAXPDZ128rrkz,        X86::VMAXPDZ128rmkz,        0 }
   };
 
-  for (unsigned i = 0, e = array_lengthof(MemoryFoldTable3); i != e; ++i) {
-    unsigned RegOp = MemoryFoldTable3[i].RegOp;
-    unsigned MemOp = MemoryFoldTable3[i].MemOp;
-    unsigned Flags = MemoryFoldTable3[i].Flags;
+  for (X86MemoryFoldTableEntry Entry : MemoryFoldTable3) {
     AddTableEntry(RegOp2MemOpTable3, MemOp2RegOpTable,
-                  RegOp, MemOp,
+                  Entry.RegOp, Entry.MemOp,
                   // Index 3, folded load
-                  Flags | TB_INDEX_3 | TB_FOLDED_LOAD);
+                  Entry.Flags | TB_INDEX_3 | TB_FOLDED_LOAD);
   }
 
   static const X86MemoryFoldTableEntry MemoryFoldTable4[] = {
@@ -1999,14 +1984,11 @@ static const X86MemoryFoldTableEntry MemoryFoldTable0[] = {
     { X86::VMAXPDZ128rrk,      X86::VMAXPDZ128rmk,        0 }
   };
 
-  for (unsigned i = 0, e = array_lengthof(MemoryFoldTable4); i != e; ++i) {
-    unsigned RegOp = MemoryFoldTable4[i].RegOp;
-    unsigned MemOp = MemoryFoldTable4[i].MemOp;
-    unsigned Flags = MemoryFoldTable4[i].Flags;
+  for (X86MemoryFoldTableEntry Entry : MemoryFoldTable4) {
     AddTableEntry(RegOp2MemOpTable4, MemOp2RegOpTable,
-                  RegOp, MemOp,
+                  Entry.RegOp, Entry.MemOp,
                   // Index 4, folded load
-                  Flags | TB_INDEX_4 | TB_FOLDED_LOAD);
+                  Entry.Flags | TB_INDEX_4 | TB_FOLDED_LOAD);
   }
 }
 
@@ -3454,11 +3436,11 @@ bool X86InstrInfo::isUnpredicatedTerminator(const MachineInstr *MI) const {
   return !isPredicated(MI);
 }
 
-bool X86InstrInfo::AnalyzeBranch(MachineBasicBlock &MBB,
-                                 MachineBasicBlock *&TBB,
-                                 MachineBasicBlock *&FBB,
-                                 SmallVectorImpl<MachineOperand> &Cond,
-                                 bool AllowModify) const {
+bool X86InstrInfo::AnalyzeBranchImpl(
+    MachineBasicBlock &MBB, MachineBasicBlock *&TBB, MachineBasicBlock *&FBB,
+    SmallVectorImpl<MachineOperand> &Cond,
+    SmallVectorImpl<MachineInstr *> &CondBranches, bool AllowModify) const {
+
   // Start from the bottom of the block and work up, examining the
   // terminator instructions.
   MachineBasicBlock::iterator I = MBB.end();
@@ -3556,6 +3538,7 @@ bool X86InstrInfo::AnalyzeBranch(MachineBasicBlock &MBB,
       FBB = TBB;
       TBB = I->getOperand(0).getMBB();
       Cond.push_back(MachineOperand::CreateImm(BranchCode));
+      CondBranches.push_back(I);
       continue;
     }
 
@@ -3593,11 +3576,90 @@ bool X86InstrInfo::AnalyzeBranch(MachineBasicBlock &MBB,
 
     // Update the MachineOperand.
     Cond[0].setImm(BranchCode);
+    CondBranches.push_back(I);
   }
 
   return false;
 }
 
+bool X86InstrInfo::AnalyzeBranch(MachineBasicBlock &MBB,
+                                 MachineBasicBlock *&TBB,
+                                 MachineBasicBlock *&FBB,
+                                 SmallVectorImpl<MachineOperand> &Cond,
+                                 bool AllowModify) const {
+  SmallVector<MachineInstr *, 4> CondBranches;
+  return AnalyzeBranchImpl(MBB, TBB, FBB, Cond, CondBranches, AllowModify);
+}
+
+bool X86InstrInfo::AnalyzeBranchPredicate(MachineBasicBlock &MBB,
+                                          MachineBranchPredicate &MBP,
+                                          bool AllowModify) const {
+  using namespace std::placeholders;
+
+  SmallVector<MachineOperand, 4> Cond;
+  SmallVector<MachineInstr *, 4> CondBranches;
+  if (AnalyzeBranchImpl(MBB, MBP.TrueDest, MBP.FalseDest, Cond, CondBranches,
+                        AllowModify))
+    return true;
+
+  if (Cond.size() != 1)
+    return true;
+
+  assert(MBP.TrueDest && "expected!");
+
+  if (!MBP.FalseDest)
+    MBP.FalseDest = MBB.getNextNode();
+
+  const TargetRegisterInfo *TRI = &getRegisterInfo();
+
+  MachineInstr *ConditionDef = nullptr;
+  bool SingleUseCondition = true;
+
+  for (auto I = std::next(MBB.rbegin()), E = MBB.rend(); I != E; ++I) {
+    if (I->modifiesRegister(X86::EFLAGS, TRI)) {
+      ConditionDef = &*I;
+      break;
+    }
+
+    if (I->readsRegister(X86::EFLAGS, TRI))
+      SingleUseCondition = false;
+  }
+
+  if (!ConditionDef)
+    return true;
+
+  if (SingleUseCondition) {
+    for (auto *Succ : MBB.successors())
+      if (Succ->isLiveIn(X86::EFLAGS))
+        SingleUseCondition = false;
+  }
+
+  MBP.ConditionDef = ConditionDef;
+  MBP.SingleUseCondition = SingleUseCondition;
+
+  // Currently we only recognize the simple pattern:
+  //
+  //   test %reg, %reg
+  //   je %label
+  //
+  const unsigned TestOpcode =
+      Subtarget.is64Bit() ? X86::TEST64rr : X86::TEST32rr;
+
+  if (ConditionDef->getOpcode() == TestOpcode &&
+      ConditionDef->getNumOperands() == 3 &&
+      ConditionDef->getOperand(0).isIdenticalTo(ConditionDef->getOperand(1)) &&
+      (Cond[0].getImm() == X86::COND_NE || Cond[0].getImm() == X86::COND_E)) {
+    MBP.LHS = ConditionDef->getOperand(0);
+    MBP.RHS = MachineOperand::CreateImm(0);
+    MBP.Predicate = Cond[0].getImm() == X86::COND_NE
+                        ? MachineBranchPredicate::PRED_NE
+                        : MachineBranchPredicate::PRED_EQ;
+    return false;
+  }
+
+  return true;
+}
+
 unsigned X86InstrInfo::RemoveBranch(MachineBasicBlock &MBB) const {
   MachineBasicBlock::iterator I = MBB.end();
   unsigned Count = 0;
@@ -3620,8 +3682,7 @@ unsigned X86InstrInfo::RemoveBranch(MachineBasicBlock &MBB) const {
 
 unsigned
 X86InstrInfo::InsertBranch(MachineBasicBlock &MBB, MachineBasicBlock *TBB,
-                           MachineBasicBlock *FBB,
-                           const SmallVectorImpl<MachineOperand> &Cond,
+                           MachineBasicBlock *FBB, ArrayRef<MachineOperand> Cond,
                            DebugLoc DL) const {
   // Shouldn't be a fall through.
   assert(TBB && "InsertBranch must not be told to insert a fallthrough");
@@ -3669,7 +3730,7 @@ X86InstrInfo::InsertBranch(MachineBasicBlock &MBB, MachineBasicBlock *TBB,
 
 bool X86InstrInfo::
 canInsertSelect(const MachineBasicBlock &MBB,
-                const SmallVectorImpl<MachineOperand> &Cond,
+                ArrayRef<MachineOperand> Cond,
                 unsigned TrueReg, unsigned FalseReg,
                 int &CondCycles, int &TrueCycles, int &FalseCycles) const {
   // Not all subtargets have cmov instructions.
@@ -3706,8 +3767,7 @@ canInsertSelect(const MachineBasicBlock &MBB,
 
 void X86InstrInfo::insertSelect(MachineBasicBlock &MBB,
                                 MachineBasicBlock::iterator I, DebugLoc DL,
-                                unsigned DstReg,
-                                const SmallVectorImpl<MachineOperand> &Cond,
+                                unsigned DstReg, ArrayRef<MachineOperand> Cond,
                                 unsigned TrueReg, unsigned FalseReg) const {
    MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
    assert(Cond.size() == 1 && "Invalid Cond array");
@@ -3740,7 +3800,7 @@ static unsigned CopyToFromAsymmetricReg(unsigned DestReg, unsigned SrcReg,
                                                X86::MOVPQIto64rr);
     if (X86::VR64RegClass.contains(SrcReg))
       // Copy from a VR64 register to a GR64 register.
-      return X86::MOVSDto64rr;
+      return X86::MMX_MOVD64from64rr;
   } else if (X86::GR64RegClass.contains(SrcReg)) {
     // Copy from a GR64 register to a VR128 register.
     if (X86::VR128XRegClass.contains(DestReg))
@@ -3748,7 +3808,7 @@ static unsigned CopyToFromAsymmetricReg(unsigned DestReg, unsigned SrcReg,
                                                X86::MOV64toPQIrr);
     // Copy from a GR64 register to a VR64 register.
     if (X86::VR64RegClass.contains(DestReg))
-      return X86::MOV64toSDrr;
+      return X86::MMX_MOVD64to64rr;
   }
 
   // SrcReg(FR32) -> DestReg(GR32)
@@ -3846,35 +3906,6 @@ void X86InstrInfo::copyPhysReg(MachineBasicBlock &MBB,
   // Moving EFLAGS to / from another register requires a push and a pop.
   // Notice that we have to adjust the stack if we don't want to clobber the
   // first frame index. See X86FrameLowering.cpp - clobbersTheStack.
-
-  // @LOCALMOD-BEGIN
-  //
-  // NaCl's sandbox doesn't allow usage of PUSHF/POPF. Instead use LAHF/SAHF
-  // which write the bottom 8 EFLAGS bits from/to AH.
-  bool FromEFLAGS = SrcReg == X86::EFLAGS;
-  bool ToEFLAGS = DestReg == X86::EFLAGS;
-  int Reg = FromEFLAGS ? DestReg : SrcReg;
-  bool is32 = X86::GR32RegClass.contains(Reg);
-  bool is64 = X86::GR64RegClass.contains(Reg);
-  int Mov = is64 ? X86::MOV64rr : X86::MOV32rr;
-  int Push = is64 ? X86::PUSH64r : X86::PUSH32r;
-  int Pop = is64 ? X86::POP64r : X86::POP32r;
-  int AX = is64 ? X86::RAX : X86::EAX;
-  if ((FromEFLAGS || ToEFLAGS) && (is32 || is64) && Subtarget.isTargetNaCl()) {
-    BuildMI(MBB, MI, DL, get(Push)).addReg(AX);
-    if (FromEFLAGS) {
-      BuildMI(MBB, MI, DL, get(X86::LAHF));
-      BuildMI(MBB, MI, DL, get(Mov), Reg).addReg(AX);
-    }
-    if (ToEFLAGS) {
-      BuildMI(MBB, MI, DL, get(Mov), AX).addReg(Reg, getKillRegState(KillSrc));
-      BuildMI(MBB, MI, DL, get(X86::SAHF));
-    }
-    BuildMI(MBB, MI, DL, get(Pop), AX);
-    return;
-  }
-  // @LOCALMOD-END
-
   if (SrcReg == X86::EFLAGS) {
     if (X86::GR64RegClass.contains(DestReg)) {
       BuildMI(MBB, MI, DL, get(X86::PUSHF64));
@@ -3994,6 +4025,36 @@ static unsigned getLoadStoreRegOpcode(unsigned Reg,
   }
 }
 
+bool X86InstrInfo::getMemOpBaseRegImmOfs(MachineInstr *MemOp, unsigned &BaseReg,
+                                         unsigned &Offset,
+                                         const TargetRegisterInfo *TRI) const {
+  const MCInstrDesc &Desc = MemOp->getDesc();
+  int MemRefBegin = X86II::getMemoryOperandNo(Desc.TSFlags, MemOp->getOpcode());
+  if (MemRefBegin < 0)
+    return false;
+
+  MemRefBegin += X86II::getOperandBias(Desc);
+
+  BaseReg = MemOp->getOperand(MemRefBegin + X86::AddrBaseReg).getReg();
+  if (MemOp->getOperand(MemRefBegin + X86::AddrScaleAmt).getImm() != 1)
+    return false;
+
+  if (MemOp->getOperand(MemRefBegin + X86::AddrIndexReg).getReg() !=
+      X86::NoRegister)
+    return false;
+
+  const MachineOperand &DispMO = MemOp->getOperand(MemRefBegin + X86::AddrDisp);
+
+  // Displacement can be symbolic
+  if (!DispMO.isImm())
+    return false;
+
+  Offset = DispMO.getImm();
+
+  return (MemOp->getOperand(MemRefBegin + X86::AddrIndexReg).getReg() ==
+          X86::NoRegister);
+}
+
 static unsigned getStoreRegOpcode(unsigned SrcReg,
                                   const TargetRegisterClass *RC,
                                   bool isStackAligned,
@@ -4612,7 +4673,7 @@ MachineInstr *X86InstrInfo::optimizeLoadInstr(MachineInstr *MI,
   DefMI = MRI->getVRegDef(FoldAsLoadDefReg);
   assert(DefMI);
   bool SawStore = false;
-  if (!DefMI->isSafeToMove(this, nullptr, SawStore))
+  if (!DefMI->isSafeToMove(nullptr, SawStore))
     return nullptr;
 
   // Collect information about virtual register operands of MI.
@@ -4730,8 +4791,17 @@ bool X86InstrInfo::expandPostRAPseudo(MachineBasicBlock::iterator MI) const {
   return false;
 }
 
+static void addOperands(MachineInstrBuilder &MIB, ArrayRef<MachineOperand> MOs) {
+  unsigned NumAddrOps = MOs.size();
+  for (unsigned i = 0; i != NumAddrOps; ++i)
+    MIB.addOperand(MOs[i]);
+  if (NumAddrOps < 4) // FrameIndex only
+    addOffset(MIB, 0);
+}
+
 static MachineInstr *FuseTwoAddrInst(MachineFunction &MF, unsigned Opcode,
                                      ArrayRef<MachineOperand> MOs,
+                                     MachineBasicBlock::iterator InsertPt,
                                      MachineInstr *MI,
                                      const TargetInstrInfo &TII) {
   // Create the base instruction with the memory operand as the first part.
@@ -4739,11 +4809,7 @@ static MachineInstr *FuseTwoAddrInst(MachineFunction &MF, unsigned Opcode,
   MachineInstr *NewMI = MF.CreateMachineInstr(TII.get(Opcode),
                                               MI->getDebugLoc(), true);
   MachineInstrBuilder MIB(MF, NewMI);
-  unsigned NumAddrOps = MOs.size();
-  for (unsigned i = 0; i != NumAddrOps; ++i)
-    MIB.addOperand(MOs[i]);
-  if (NumAddrOps < 4)  // FrameIndex only
-    addOffset(MIB, 0);
+  addOperands(MIB, MOs);
 
   // Loop over the rest of the ri operands, converting them over.
   unsigned NumOps = MI->getDesc().getNumOperands()-2;
@@ -4755,11 +4821,16 @@ static MachineInstr *FuseTwoAddrInst(MachineFunction &MF, unsigned Opcode,
     MachineOperand &MO = MI->getOperand(i);
     MIB.addOperand(MO);
   }
+
+  MachineBasicBlock *MBB = InsertPt->getParent();
+  MBB->insert(InsertPt, NewMI);
+
   return MIB;
 }
 
 static MachineInstr *FuseInst(MachineFunction &MF, unsigned Opcode,
                               unsigned OpNo, ArrayRef<MachineOperand> MOs,
+                              MachineBasicBlock::iterator InsertPt,
                               MachineInstr *MI, const TargetInstrInfo &TII) {
   // Omit the implicit operands, something BuildMI can't do.
   MachineInstr *NewMI = MF.CreateMachineInstr(TII.get(Opcode),
@@ -4770,47 +4841,45 @@ static MachineInstr *FuseInst(MachineFunction &MF, unsigned Opcode,
     MachineOperand &MO = MI->getOperand(i);
     if (i == OpNo) {
       assert(MO.isReg() && "Expected to fold into reg operand!");
-      unsigned NumAddrOps = MOs.size();
-      for (unsigned i = 0; i != NumAddrOps; ++i)
-        MIB.addOperand(MOs[i]);
-      if (NumAddrOps < 4)  // FrameIndex only
-        addOffset(MIB, 0);
+      addOperands(MIB, MOs);
     } else {
       MIB.addOperand(MO);
     }
   }
+
+  MachineBasicBlock *MBB = InsertPt->getParent();
+  MBB->insert(InsertPt, NewMI);
+
   return MIB;
 }
 
 static MachineInstr *MakeM0Inst(const TargetInstrInfo &TII, unsigned Opcode,
                                 ArrayRef<MachineOperand> MOs,
+                                MachineBasicBlock::iterator InsertPt,
                                 MachineInstr *MI) {
-  MachineFunction &MF = *MI->getParent()->getParent();
-  MachineInstrBuilder MIB = BuildMI(MF, MI->getDebugLoc(), TII.get(Opcode));
-
-  unsigned NumAddrOps = MOs.size();
-  for (unsigned i = 0; i != NumAddrOps; ++i)
-    MIB.addOperand(MOs[i]);
-  if (NumAddrOps < 4)  // FrameIndex only
-    addOffset(MIB, 0);
+  MachineInstrBuilder MIB = BuildMI(*InsertPt->getParent(), InsertPt,
+                                    MI->getDebugLoc(), TII.get(Opcode));
+  addOperands(MIB, MOs);
   return MIB.addImm(0);
 }
 
-MachineInstr *X86InstrInfo::foldMemoryOperandImpl(MachineFunction &MF,
-                                                  MachineInstr *MI,
-                                                  unsigned OpNum,
-                                                  ArrayRef<MachineOperand> MOs,
-                                                  unsigned Size, unsigned Align,
-                                                  bool AllowCommute) const {
+MachineInstr *X86InstrInfo::foldMemoryOperandImpl(
+    MachineFunction &MF, MachineInstr *MI, unsigned OpNum,
+    ArrayRef<MachineOperand> MOs, MachineBasicBlock::iterator InsertPt,
+    unsigned Size, unsigned Align, bool AllowCommute) const {
   const DenseMap<unsigned,
                  std::pair<unsigned,unsigned> > *OpcodeTablePtr = nullptr;
   bool isCallRegIndirect = Subtarget.callRegIndirect();
   bool isTwoAddrFold = false;
 
-  // For CPUs that favor the register form of a call,
-  // do not fold loads into calls.
-  if (isCallRegIndirect &&
-    (MI->getOpcode() == X86::CALL32r || MI->getOpcode() == X86::CALL64r))
+  // For CPUs that favor the register form of a call or push,
+  // do not fold loads into calls or pushes, unless optimizing for size
+  // aggressively.
+  if (isCallRegIndirect && 
+      !MF.getFunction()->hasFnAttribute(Attribute::MinSize) &&
+      (MI->getOpcode() == X86::CALL32r || MI->getOpcode() == X86::CALL64r ||
+       MI->getOpcode() == X86::PUSH16r || MI->getOpcode() == X86::PUSH32r ||
+       MI->getOpcode() == X86::PUSH64r))
     return nullptr;
 
   unsigned NumOps = MI->getDesc().getNumOperands();
@@ -4835,7 +4904,7 @@ MachineInstr *X86InstrInfo::foldMemoryOperandImpl(MachineFunction &MF,
     isTwoAddrFold = true;
   } else if (OpNum == 0) {
     if (MI->getOpcode() == X86::MOV32r0) {
-      NewMI = MakeM0Inst(*this, X86::MOV32mi, MOs, MI);
+      NewMI = MakeM0Inst(*this, X86::MOV32mi, MOs, InsertPt, MI);
       if (NewMI)
         return NewMI;
     }
@@ -4880,9 +4949,9 @@ MachineInstr *X86InstrInfo::foldMemoryOperandImpl(MachineFunction &MF,
       }
 
       if (isTwoAddrFold)
-        NewMI = FuseTwoAddrInst(MF, Opcode, MOs, MI, *this);
+        NewMI = FuseTwoAddrInst(MF, Opcode, MOs, InsertPt, MI, *this);
       else
-        NewMI = FuseInst(MF, Opcode, OpNum, MOs, MI, *this);
+        NewMI = FuseInst(MF, Opcode, OpNum, MOs, InsertPt, MI, *this);
 
       if (NarrowToMOV32rm) {
         // If this is the special case where we use a MOV32rm to load a 32-bit
@@ -4934,8 +5003,9 @@ MachineInstr *X86InstrInfo::foldMemoryOperandImpl(MachineFunction &MF,
         // Attempt to fold with the commuted version of the instruction.
         unsigned CommuteOp =
             (CommuteOpIdx1 == OriginalOpIdx ? CommuteOpIdx2 : CommuteOpIdx1);
-        NewMI = foldMemoryOperandImpl(MF, MI, CommuteOp, MOs, Size, Align,
-                                      /*AllowCommute=*/false);
+        NewMI =
+            foldMemoryOperandImpl(MF, MI, CommuteOp, MOs, InsertPt, Size, Align,
+                                  /*AllowCommute=*/false);
         if (NewMI)
           return NewMI;
 
@@ -5164,10 +5234,9 @@ breakPartialRegDependency(MachineBasicBlock::iterator MI, unsigned OpNum,
   MI->addRegisterKilled(Reg, TRI, true);
 }
 
-MachineInstr *X86InstrInfo::foldMemoryOperandImpl(MachineFunction &MF,
-                                                  MachineInstr *MI,
-                                                  ArrayRef<unsigned> Ops,
-                                                  int FrameIndex) const {
+MachineInstr *X86InstrInfo::foldMemoryOperandImpl(
+    MachineFunction &MF, MachineInstr *MI, ArrayRef<unsigned> Ops,
+    MachineBasicBlock::iterator InsertPt, int FrameIndex) const {
   // Check switch flag
   if (NoFusing) return nullptr;
 
@@ -5206,40 +5275,75 @@ MachineInstr *X86InstrInfo::foldMemoryOperandImpl(MachineFunction &MF,
     return nullptr;
 
   return foldMemoryOperandImpl(MF, MI, Ops[0],
-                               MachineOperand::CreateFI(FrameIndex), Size,
-                               Alignment, /*AllowCommute=*/true);
+                               MachineOperand::CreateFI(FrameIndex), InsertPt,
+                               Size, Alignment, /*AllowCommute=*/true);
 }
 
-static bool isPartialRegisterLoad(const MachineInstr &LoadMI,
-                                  const MachineFunction &MF) {
+/// Check if \p LoadMI is a partial register load that we can't fold into \p MI
+/// because the latter uses contents that wouldn't be defined in the folded
+/// version.  For instance, this transformation isn't legal:
+///   movss (%rdi), %xmm0
+///   addps %xmm0, %xmm0
+/// ->
+///   addps (%rdi), %xmm0
+///
+/// But this one is:
+///   movss (%rdi), %xmm0
+///   addss %xmm0, %xmm0
+/// ->
+///   addss (%rdi), %xmm0
+///
+static bool isNonFoldablePartialRegisterLoad(const MachineInstr &LoadMI,
+                                             const MachineInstr &UserMI,
+                                             const MachineFunction &MF) {
   unsigned Opc = LoadMI.getOpcode();
+  unsigned UserOpc = UserMI.getOpcode();
   unsigned RegSize =
       MF.getRegInfo().getRegClass(LoadMI.getOperand(0).getReg())->getSize();
 
-  if ((Opc == X86::MOVSSrm || Opc == X86::VMOVSSrm) && RegSize > 4)
+  if ((Opc == X86::MOVSSrm || Opc == X86::VMOVSSrm) && RegSize > 4) {
     // These instructions only load 32 bits, we can't fold them if the
-    // destination register is wider than 32 bits (4 bytes).
-    return true;
+    // destination register is wider than 32 bits (4 bytes), and its user
+    // instruction isn't scalar (SS).
+    switch (UserOpc) {
+    case X86::ADDSSrr_Int: case X86::VADDSSrr_Int:
+    case X86::DIVSSrr_Int: case X86::VDIVSSrr_Int:
+    case X86::MULSSrr_Int: case X86::VMULSSrr_Int:
+    case X86::SUBSSrr_Int: case X86::VSUBSSrr_Int:
+      return false;
+    default:
+      return true;
+    }
+  }
 
-  if ((Opc == X86::MOVSDrm || Opc == X86::VMOVSDrm) && RegSize > 8)
+  if ((Opc == X86::MOVSDrm || Opc == X86::VMOVSDrm) && RegSize > 8) {
     // These instructions only load 64 bits, we can't fold them if the
-    // destination register is wider than 64 bits (8 bytes).
-    return true;
+    // destination register is wider than 64 bits (8 bytes), and its user
+    // instruction isn't scalar (SD).
+    switch (UserOpc) {
+    case X86::ADDSDrr_Int: case X86::VADDSDrr_Int:
+    case X86::DIVSDrr_Int: case X86::VDIVSDrr_Int:
+    case X86::MULSDrr_Int: case X86::VMULSDrr_Int:
+    case X86::SUBSDrr_Int: case X86::VSUBSDrr_Int:
+      return false;
+    default:
+      return true;
+    }
+  }
 
   return false;
 }
 
-MachineInstr *X86InstrInfo::foldMemoryOperandImpl(MachineFunction &MF,
-                                                  MachineInstr *MI,
-                                                  ArrayRef<unsigned> Ops,
-                                                  MachineInstr *LoadMI) const {
+MachineInstr *X86InstrInfo::foldMemoryOperandImpl(
+    MachineFunction &MF, MachineInstr *MI, ArrayRef<unsigned> Ops,
+    MachineBasicBlock::iterator InsertPt, MachineInstr *LoadMI) const {
   // If loading from a FrameIndex, fold directly from the FrameIndex.
   unsigned NumOps = LoadMI->getDesc().getNumOperands();
   int FrameIndex;
   if (isLoadFromStackSlot(LoadMI, FrameIndex)) {
-    if (isPartialRegisterLoad(*LoadMI, MF))
+    if (isNonFoldablePartialRegisterLoad(*LoadMI, *MI, MF))
       return nullptr;
-    return foldMemoryOperandImpl(MF, MI, Ops, FrameIndex);
+    return foldMemoryOperandImpl(MF, MI, Ops, InsertPt, FrameIndex);
   }
 
   // Check switch flag
@@ -5350,7 +5454,7 @@ MachineInstr *X86InstrInfo::foldMemoryOperandImpl(MachineFunction &MF,
     break;
   }
   default: {
-    if (isPartialRegisterLoad(*LoadMI, MF))
+    if (isNonFoldablePartialRegisterLoad(*LoadMI, *MI, MF))
       return nullptr;
 
     // Folding a normal load. Just copy the load's address operands.
@@ -5359,66 +5463,10 @@ MachineInstr *X86InstrInfo::foldMemoryOperandImpl(MachineFunction &MF,
     break;
   }
   }
-  return foldMemoryOperandImpl(MF, MI, Ops[0], MOs,
+  return foldMemoryOperandImpl(MF, MI, Ops[0], MOs, InsertPt,
                                /*Size=*/0, Alignment, /*AllowCommute=*/true);
 }
 
-bool X86InstrInfo::canFoldMemoryOperand(const MachineInstr *MI,
-                                        ArrayRef<unsigned> Ops) const {
-  // Check switch flag
-  if (NoFusing) return 0;
-
-  if (Ops.size() == 2 && Ops[0] == 0 && Ops[1] == 1) {
-    switch (MI->getOpcode()) {
-    default: return false;
-    case X86::TEST8rr:
-    case X86::TEST16rr:
-    case X86::TEST32rr:
-    case X86::TEST64rr:
-      return true;
-    case X86::ADD32ri:
-      // FIXME: AsmPrinter doesn't know how to handle
-      // X86II::MO_GOT_ABSOLUTE_ADDRESS after folding.
-      if (MI->getOperand(2).getTargetFlags() == X86II::MO_GOT_ABSOLUTE_ADDRESS)
-        return false;
-      break;
-    }
-  }
-
-  if (Ops.size() != 1)
-    return false;
-
-  unsigned OpNum = Ops[0];
-  unsigned Opc = MI->getOpcode();
-  unsigned NumOps = MI->getDesc().getNumOperands();
-  bool isTwoAddr = NumOps > 1 &&
-    MI->getDesc().getOperandConstraint(1, MCOI::TIED_TO) != -1;
-
-  // Folding a memory location into the two-address part of a two-address
-  // instruction is different than folding it other places.  It requires
-  // replacing the *two* registers with the memory location.
-  const DenseMap<unsigned,
-                 std::pair<unsigned,unsigned> > *OpcodeTablePtr = nullptr;
-  if (isTwoAddr && NumOps >= 2 && OpNum < 2) {
-    OpcodeTablePtr = &RegOp2MemOpTable2Addr;
-  } else if (OpNum == 0) {
-    if (Opc == X86::MOV32r0)
-      return true;
-
-    OpcodeTablePtr = &RegOp2MemOpTable0;
-  } else if (OpNum == 1) {
-    OpcodeTablePtr = &RegOp2MemOpTable1;
-  } else if (OpNum == 2) {
-    OpcodeTablePtr = &RegOp2MemOpTable2;
-  } else if (OpNum == 3) {
-    OpcodeTablePtr = &RegOp2MemOpTable3;
-  }
-
-  if (OpcodeTablePtr && OpcodeTablePtr->count(Opc))
-    return true;
-  return TargetInstrInfo::canFoldMemoryOperand(MI, Ops);
-}
-
 bool X86InstrInfo::unfoldMemoryOperand(MachineFunction &MF, MachineInstr *MI,
                                 unsigned Reg, bool UnfoldLoad, bool UnfoldStore,
                                 SmallVectorImpl<MachineInstr*> &NewMIs) const {
@@ -6048,7 +6096,7 @@ static const uint16_t ReplaceableInstrs[][3] = {
   { X86::VMOVAPSrr,  X86::VMOVAPDrr,  X86::VMOVDQArr  },
   { X86::VMOVUPSmr,  X86::VMOVUPDmr,  X86::VMOVDQUmr  },
   { X86::VMOVUPSrm,  X86::VMOVUPDrm,  X86::VMOVDQUrm  },
-  // TODO: Add the AVX versions of MOVLPSmr
+  { X86::VMOVLPSmr,  X86::VMOVLPDmr,  X86::VMOVPQI2QImr  },
   { X86::VMOVNTPSmr, X86::VMOVNTPDmr, X86::VMOVNTDQmr },
   { X86::VANDNPSrm,  X86::VANDNPDrm,  X86::VPANDNrm   },
   { X86::VANDNPSrr,  X86::VANDNPDrr,  X86::VPANDNrr   },
@@ -6146,7 +6194,7 @@ void X86InstrInfo::getNoopForMachoTarget(MCInst &NopInst) const {
 void X86InstrInfo::getUnconditionalBranch(
     MCInst &Branch, const MCSymbolRefExpr *BranchTarget) const {
   Branch.setOpcode(X86::JMP_1);
-  Branch.addOperand(MCOperand::CreateExpr(BranchTarget));
+  Branch.addOperand(MCOperand::createExpr(BranchTarget));
 }
 
 // This code must remain in sync with getJumpInstrTableEntryBound in this class!
@@ -6243,13 +6291,256 @@ bool X86InstrInfo::isHighLatencyDef(int opc) const {
 }
 
 bool X86InstrInfo::
-hasHighOperandLatency(const InstrItineraryData *ItinData,
+hasHighOperandLatency(const TargetSchedModel &SchedModel,
                       const MachineRegisterInfo *MRI,
                       const MachineInstr *DefMI, unsigned DefIdx,
                       const MachineInstr *UseMI, unsigned UseIdx) const {
   return isHighLatencyDef(DefMI->getOpcode());
 }
 
+static bool hasReassociableOperands(const MachineInstr &Inst,
+                                    const MachineBasicBlock *MBB) {
+  assert((Inst.getNumOperands() == 3 || Inst.getNumOperands() == 4) &&
+         "Reassociation needs binary operators");
+  const MachineOperand &Op1 = Inst.getOperand(1);
+  const MachineOperand &Op2 = Inst.getOperand(2);
+  const MachineRegisterInfo &MRI = MBB->getParent()->getRegInfo();
+
+  // Integer binary math/logic instructions have a third source operand:
+  // the EFLAGS register. That operand must be both defined here and never
+  // used; ie, it must be dead. If the EFLAGS operand is live, then we can
+  // not change anything because rearranging the operands could affect other
+  // instructions that depend on the exact status flags (zero, sign, etc.)
+  // that are set by using these particular operands with this operation.
+  if (Inst.getNumOperands() == 4) {
+    assert(Inst.getOperand(3).isReg() &&
+           Inst.getOperand(3).getReg() == X86::EFLAGS &&
+           "Unexpected operand in reassociable instruction");
+    if (!Inst.getOperand(3).isDead())
+      return false;
+  }
+  
+  // We need virtual register definitions for the operands that we will
+  // reassociate.
+  MachineInstr *MI1 = nullptr;
+  MachineInstr *MI2 = nullptr;
+  if (Op1.isReg() && TargetRegisterInfo::isVirtualRegister(Op1.getReg()))
+    MI1 = MRI.getUniqueVRegDef(Op1.getReg());
+  if (Op2.isReg() && TargetRegisterInfo::isVirtualRegister(Op2.getReg()))
+    MI2 = MRI.getUniqueVRegDef(Op2.getReg());
+
+  // And they need to be in the trace (otherwise, they won't have a depth).
+  if (MI1 && MI2 && MI1->getParent() == MBB && MI2->getParent() == MBB)
+    return true;
+
+  return false;
+}
+
+static bool hasReassociableSibling(const MachineInstr &Inst, bool &Commuted) {
+  const MachineBasicBlock *MBB = Inst.getParent();
+  const MachineRegisterInfo &MRI = MBB->getParent()->getRegInfo();
+  MachineInstr *MI1 = MRI.getUniqueVRegDef(Inst.getOperand(1).getReg());
+  MachineInstr *MI2 = MRI.getUniqueVRegDef(Inst.getOperand(2).getReg());
+  unsigned AssocOpcode = Inst.getOpcode();
+
+  // If only one operand has the same opcode and it's the second source operand,
+  // the operands must be commuted.
+  Commuted = MI1->getOpcode() != AssocOpcode && MI2->getOpcode() == AssocOpcode;
+  if (Commuted)
+    std::swap(MI1, MI2);
+
+  // 1. The previous instruction must be the same type as Inst.
+  // 2. The previous instruction must have virtual register definitions for its
+  //    operands in the same basic block as Inst.
+  // 3. The previous instruction's result must only be used by Inst.
+  if (MI1->getOpcode() == AssocOpcode &&
+      hasReassociableOperands(*MI1, MBB) &&
+      MRI.hasOneNonDBGUse(MI1->getOperand(0).getReg()))
+    return true;
+
+  return false;
+}
+
+// TODO: There are many more machine instruction opcodes to match:
+//       1. Other data types (integer, vectors)
+//       2. Other math / logic operations (and, or)
+static bool isAssociativeAndCommutative(const MachineInstr &Inst) {
+  switch (Inst.getOpcode()) {
+  case X86::IMUL16rr:
+  case X86::IMUL32rr:
+  case X86::IMUL64rr:
+    return true;
+  case X86::ADDSDrr:
+  case X86::ADDSSrr:
+  case X86::VADDSDrr:
+  case X86::VADDSSrr:
+  case X86::MULSDrr:
+  case X86::MULSSrr:
+  case X86::VMULSDrr:
+  case X86::VMULSSrr:
+    return Inst.getParent()->getParent()->getTarget().Options.UnsafeFPMath;
+  default:
+    return false;
+  }
+}
+
+/// Return true if the input instruction is part of a chain of dependent ops
+/// that are suitable for reassociation, otherwise return false.
+/// If the instruction's operands must be commuted to have a previous
+/// instruction of the same type define the first source operand, Commuted will
+/// be set to true.
+static bool isReassociationCandidate(const MachineInstr &Inst, bool &Commuted) {
+  // 1. The operation must be associative and commutative.
+  // 2. The instruction must have virtual register definitions for its
+  //    operands in the same basic block.
+  // 3. The instruction must have a reassociable sibling.
+  if (isAssociativeAndCommutative(Inst) &&
+      hasReassociableOperands(Inst, Inst.getParent()) &&
+      hasReassociableSibling(Inst, Commuted))
+    return true;
+
+  return false;
+}
+
+// FIXME: This has the potential to be expensive (compile time) while not
+// improving the code at all. Some ways to limit the overhead:
+// 1. Track successful transforms; bail out if hit rate gets too low.
+// 2. Only enable at -O3 or some other non-default optimization level.
+// 3. Pre-screen pattern candidates here: if an operand of the previous
+//    instruction is known to not increase the critical path, then don't match
+//    that pattern.
+bool X86InstrInfo::getMachineCombinerPatterns(MachineInstr &Root,
+        SmallVectorImpl<MachineCombinerPattern::MC_PATTERN> &Patterns) const {
+  // TODO: There is nothing x86-specific here except the instruction type.
+  // This logic could be hoisted into the machine combiner pass itself.
+
+  // Look for this reassociation pattern:
+  //   B = A op X (Prev)
+  //   C = B op Y (Root)
+
+  bool Commute;
+  if (isReassociationCandidate(Root, Commute)) {
+    // We found a sequence of instructions that may be suitable for a
+    // reassociation of operands to increase ILP. Specify each commutation
+    // possibility for the Prev instruction in the sequence and let the
+    // machine combiner decide if changing the operands is worthwhile.
+    if (Commute) {
+      Patterns.push_back(MachineCombinerPattern::MC_REASSOC_AX_YB);
+      Patterns.push_back(MachineCombinerPattern::MC_REASSOC_XA_YB);
+    } else {
+      Patterns.push_back(MachineCombinerPattern::MC_REASSOC_AX_BY);
+      Patterns.push_back(MachineCombinerPattern::MC_REASSOC_XA_BY);
+    }
+    return true;
+  }
+
+  return false;
+}
+
+/// Attempt the following reassociation to reduce critical path length:
+///   B = A op X (Prev)
+///   C = B op Y (Root)
+///   ===>
+///   B = X op Y
+///   C = A op B
+static void reassociateOps(MachineInstr &Root, MachineInstr &Prev,
+                           MachineCombinerPattern::MC_PATTERN Pattern,
+                           SmallVectorImpl<MachineInstr *> &InsInstrs,
+                           SmallVectorImpl<MachineInstr *> &DelInstrs,
+                           DenseMap<unsigned, unsigned> &InstrIdxForVirtReg) {
+  MachineFunction *MF = Root.getParent()->getParent();
+  MachineRegisterInfo &MRI = MF->getRegInfo();
+  const TargetInstrInfo *TII = MF->getSubtarget().getInstrInfo();
+  const TargetRegisterInfo *TRI = MF->getSubtarget().getRegisterInfo();
+  const TargetRegisterClass *RC = Root.getRegClassConstraint(0, TII, TRI);
+
+  // This array encodes the operand index for each parameter because the
+  // operands may be commuted. Each row corresponds to a pattern value,
+  // and each column specifies the index of A, B, X, Y.
+  unsigned OpIdx[4][4] = {
+    { 1, 1, 2, 2 },
+    { 1, 2, 2, 1 },
+    { 2, 1, 1, 2 },
+    { 2, 2, 1, 1 }
+  };
+
+  MachineOperand &OpA = Prev.getOperand(OpIdx[Pattern][0]);
+  MachineOperand &OpB = Root.getOperand(OpIdx[Pattern][1]);
+  MachineOperand &OpX = Prev.getOperand(OpIdx[Pattern][2]);
+  MachineOperand &OpY = Root.getOperand(OpIdx[Pattern][3]);
+  MachineOperand &OpC = Root.getOperand(0);
+
+  unsigned RegA = OpA.getReg();
+  unsigned RegB = OpB.getReg();
+  unsigned RegX = OpX.getReg();
+  unsigned RegY = OpY.getReg();
+  unsigned RegC = OpC.getReg();
+
+  if (TargetRegisterInfo::isVirtualRegister(RegA))
+    MRI.constrainRegClass(RegA, RC);
+  if (TargetRegisterInfo::isVirtualRegister(RegB))
+    MRI.constrainRegClass(RegB, RC);
+  if (TargetRegisterInfo::isVirtualRegister(RegX))
+    MRI.constrainRegClass(RegX, RC);
+  if (TargetRegisterInfo::isVirtualRegister(RegY))
+    MRI.constrainRegClass(RegY, RC);
+  if (TargetRegisterInfo::isVirtualRegister(RegC))
+    MRI.constrainRegClass(RegC, RC);
+
+  // Create a new virtual register for the result of (X op Y) instead of
+  // recycling RegB because the MachineCombiner's computation of the critical
+  // path requires a new register definition rather than an existing one.
+  unsigned NewVR = MRI.createVirtualRegister(RC);
+  InstrIdxForVirtReg.insert(std::make_pair(NewVR, 0));
+
+  unsigned Opcode = Root.getOpcode();
+  bool KillA = OpA.isKill();
+  bool KillX = OpX.isKill();
+  bool KillY = OpY.isKill();
+
+  // Create new instructions for insertion.
+  MachineInstrBuilder MIB1 =
+    BuildMI(*MF, Prev.getDebugLoc(), TII->get(Opcode), NewVR)
+      .addReg(RegX, getKillRegState(KillX))
+      .addReg(RegY, getKillRegState(KillY));
+  InsInstrs.push_back(MIB1);
+
+  MachineInstrBuilder MIB2 =
+    BuildMI(*MF, Root.getDebugLoc(), TII->get(Opcode), RegC)
+      .addReg(RegA, getKillRegState(KillA))
+      .addReg(NewVR, getKillRegState(true));
+  InsInstrs.push_back(MIB2);
+
+  // Record old instructions for deletion.
+  DelInstrs.push_back(&Prev);
+  DelInstrs.push_back(&Root);
+}
+
+void X86InstrInfo::genAlternativeCodeSequence(
+    MachineInstr &Root,
+    MachineCombinerPattern::MC_PATTERN Pattern,
+    SmallVectorImpl<MachineInstr *> &InsInstrs,
+    SmallVectorImpl<MachineInstr *> &DelInstrs,
+    DenseMap<unsigned, unsigned> &InstIdxForVirtReg) const {
+  MachineRegisterInfo &MRI = Root.getParent()->getParent()->getRegInfo();
+
+  // Select the previous instruction in the sequence based on the input pattern.
+  MachineInstr *Prev = nullptr;
+  switch (Pattern) {
+    case MachineCombinerPattern::MC_REASSOC_AX_BY:
+    case MachineCombinerPattern::MC_REASSOC_XA_BY:
+      Prev = MRI.getUniqueVRegDef(Root.getOperand(1).getReg());
+      break;
+    case MachineCombinerPattern::MC_REASSOC_AX_YB:
+    case MachineCombinerPattern::MC_REASSOC_XA_YB:
+      Prev = MRI.getUniqueVRegDef(Root.getOperand(2).getReg());
+  }
+  assert(Prev && "Unknown pattern for machine combiner");
+
+  reassociateOps(Root, *Prev, Pattern, InsInstrs, DelInstrs, InstIdxForVirtReg);
+  return;
+}
+
 namespace {
   /// Create Global Base Reg pass. This initializes the PIC
   /// global base register for x86-32.
diff --git a/lib/Target/X86/X86InstrInfo.h b/lib/Target/X86/X86InstrInfo.h
index 0dd8101bbe5..0ef4b088eac 100644
--- a/lib/Target/X86/X86InstrInfo.h
+++ b/lib/Target/X86/X86InstrInfo.h
@@ -26,6 +26,19 @@ namespace llvm {
   class X86RegisterInfo;
   class X86Subtarget;
 
+  namespace MachineCombinerPattern {
+    enum MC_PATTERN : int {
+      // These are commutative variants for reassociating a computation chain
+      // of the form:
+      //   B = A op X (Prev)
+      //   C = B op Y (Root)
+      MC_REASSOC_AX_BY = 0,
+      MC_REASSOC_AX_YB = 1,
+      MC_REASSOC_XA_BY = 2,
+      MC_REASSOC_XA_YB = 3,
+    };
+  } // end namespace MachineCombinerPattern
+
 namespace X86 {
   // X86 specific condition code. These correspond to X86_*_COND in
   // X86InstrInfo.td. They must be kept in synch.
@@ -166,6 +179,12 @@ class X86InstrInfo final : public X86GenInstrInfo {
 
   virtual void anchor();
 
+  bool AnalyzeBranchImpl(MachineBasicBlock &MBB, MachineBasicBlock *&TBB,
+                         MachineBasicBlock *&FBB,
+                         SmallVectorImpl<MachineOperand> &Cond,
+                         SmallVectorImpl<MachineInstr *> &CondBranches,
+                         bool AllowModify) const;
+
 public:
   explicit X86InstrInfo(X86Subtarget &STI);
 
@@ -254,18 +273,23 @@ class X86InstrInfo final : public X86GenInstrInfo {
                      MachineBasicBlock *&FBB,
                      SmallVectorImpl<MachineOperand> &Cond,
                      bool AllowModify) const override;
+
+  bool getMemOpBaseRegImmOfs(MachineInstr *LdSt, unsigned &BaseReg,
+                             unsigned &Offset,
+                             const TargetRegisterInfo *TRI) const override;
+  bool AnalyzeBranchPredicate(MachineBasicBlock &MBB,
+                              TargetInstrInfo::MachineBranchPredicate &MBP,
+                              bool AllowModify = false) const override;
+
   unsigned RemoveBranch(MachineBasicBlock &MBB) const override;
   unsigned InsertBranch(MachineBasicBlock &MBB, MachineBasicBlock *TBB,
-                        MachineBasicBlock *FBB,
-                        const SmallVectorImpl<MachineOperand> &Cond,
+                        MachineBasicBlock *FBB, ArrayRef<MachineOperand> Cond,
                         DebugLoc DL) const override;
-  bool canInsertSelect(const MachineBasicBlock&,
-                       const SmallVectorImpl<MachineOperand> &Cond,
+  bool canInsertSelect(const MachineBasicBlock&, ArrayRef<MachineOperand> Cond,
                        unsigned, unsigned, int&, int&, int&) const override;
   void insertSelect(MachineBasicBlock &MBB,
                     MachineBasicBlock::iterator MI, DebugLoc DL,
-                    unsigned DstReg,
-                    const SmallVectorImpl<MachineOperand> &Cond,
+                    unsigned DstReg, ArrayRef<MachineOperand> Cond,
                     unsigned TrueReg, unsigned FalseReg) const override;
   void copyPhysReg(MachineBasicBlock &MBB,
                    MachineBasicBlock::iterator MI, DebugLoc DL,
@@ -307,6 +331,7 @@ class X86InstrInfo final : public X86GenInstrInfo {
   /// references has been changed.
   MachineInstr *foldMemoryOperandImpl(MachineFunction &MF, MachineInstr *MI,
                                       ArrayRef<unsigned> Ops,
+                                      MachineBasicBlock::iterator InsertPt,
                                       int FrameIndex) const override;
 
   /// foldMemoryOperand - Same as the previous version except it allows folding
@@ -314,13 +339,9 @@ class X86InstrInfo final : public X86GenInstrInfo {
   /// stack slot.
   MachineInstr *foldMemoryOperandImpl(MachineFunction &MF, MachineInstr *MI,
                                       ArrayRef<unsigned> Ops,
+                                      MachineBasicBlock::iterator InsertPt,
                                       MachineInstr *LoadMI) const override;
 
-  /// canFoldMemoryOperand - Returns true if the specified load / store is
-  /// folding is possible.
-  bool canFoldMemoryOperand(const MachineInstr *,
-                            ArrayRef<unsigned>) const override;
-
   /// unfoldMemoryOperand - Separate a single instruction which folded a load or
   /// a store or a load and a store into two or more instruction. If this is
   /// possible, returns true as well as the new instructions by reference.
@@ -407,6 +428,7 @@ class X86InstrInfo final : public X86GenInstrInfo {
   MachineInstr *foldMemoryOperandImpl(MachineFunction &MF, MachineInstr *MI,
                                       unsigned OpNum,
                                       ArrayRef<MachineOperand> MOs,
+                                      MachineBasicBlock::iterator InsertPt,
                                       unsigned Size, unsigned Alignment,
                                       bool AllowCommute) const;
 
@@ -420,12 +442,32 @@ class X86InstrInfo final : public X86GenInstrInfo {
 
   bool isHighLatencyDef(int opc) const override;
 
-  bool hasHighOperandLatency(const InstrItineraryData *ItinData,
+  bool hasHighOperandLatency(const TargetSchedModel &SchedModel,
                              const MachineRegisterInfo *MRI,
                              const MachineInstr *DefMI, unsigned DefIdx,
                              const MachineInstr *UseMI,
                              unsigned UseIdx) const override;
 
+  
+  bool useMachineCombiner() const override {
+    return true;
+  }
+  
+  /// Return true when there is potentially a faster code sequence
+  /// for an instruction chain ending in <Root>. All potential patterns are
+  /// output in the <Pattern> array.
+  bool getMachineCombinerPatterns(
+      MachineInstr &Root,
+      SmallVectorImpl<MachineCombinerPattern::MC_PATTERN> &P) const override;
+  
+  /// When getMachineCombinerPatterns() finds a pattern, this function generates
+  /// the instructions that could replace the original code sequence.
+  void genAlternativeCodeSequence(
+          MachineInstr &Root, MachineCombinerPattern::MC_PATTERN P,
+          SmallVectorImpl<MachineInstr *> &InsInstrs,
+          SmallVectorImpl<MachineInstr *> &DelInstrs,
+          DenseMap<unsigned, unsigned> &InstrIdxForVirtReg) const override;
+
   /// analyzeCompare - For a comparison instruction, return the source registers
   /// in SrcReg and SrcReg2 if having two register operands, and the value it
   /// compares against in CmpValue. Return true if the comparison instruction
diff --git a/lib/Target/X86/X86InstrInfo.td b/lib/Target/X86/X86InstrInfo.td
index d2ace93c346..3fb44172c64 100644
--- a/lib/Target/X86/X86InstrInfo.td
+++ b/lib/Target/X86/X86InstrInfo.td
@@ -104,10 +104,6 @@ def SDT_X86TLSBASEADDR : SDTypeProfile<0, 1, [SDTCisInt<0>]>;
 
 def SDT_X86TLSCALL : SDTypeProfile<0, 1, [SDTCisInt<0>]>;
 
-// @LOCALMOD-BEGIN
-def SDT_X86ThreadPointer : SDTypeProfile<1, 0, [SDTCisPtrTy<0>]>;
-// @LOCALMOD-END
-
 def SDT_X86SEG_ALLOCA : SDTypeProfile<1, 1, [SDTCisVT<0, iPTR>, SDTCisVT<1, iPTR>]>;
 
 def SDT_X86WIN_FTOL : SDTypeProfile<0, 1, [SDTCisFP<0>]>;
@@ -198,7 +194,7 @@ def X86rdpmc   : SDNode<"X86ISD::RDPMC_DAG", SDTX86Void,
 def X86Wrapper    : SDNode<"X86ISD::Wrapper",     SDTX86Wrapper>;
 def X86WrapperRIP : SDNode<"X86ISD::WrapperRIP",  SDTX86Wrapper>;
 
-def X86RecoverFrameAlloc : SDNode<"ISD::FRAME_ALLOC_RECOVER",
+def X86RecoverFrameAlloc : SDNode<"ISD::LOCAL_RECOVER",
                                   SDTypeProfile<1, 1, [SDTCisSameAs<0, 1>,
                                                        SDTCisInt<1>]>>;
 
@@ -208,17 +204,6 @@ def X86tlsaddr : SDNode<"X86ISD::TLSADDR", SDT_X86TLSADDR,
 def X86tlsbaseaddr : SDNode<"X86ISD::TLSBASEADDR", SDT_X86TLSBASEADDR,
                         [SDNPHasChain, SDNPOptInGlue, SDNPOutGlue]>;
 
-// @LOCALMOD-BEGIN
-def X86tlsaddr_le : SDNode<"X86ISD::TLSADDR_LE", SDT_X86TLSADDR,
-                           [SDNPHasChain, SDNPOptInGlue, SDNPOutGlue]>;
-
-def X86tlsaddr_ie : SDNode<"X86ISD::TLSADDR_IE", SDT_X86TLSADDR,
-                           [SDNPHasChain, SDNPOptInGlue, SDNPOutGlue]>;
-
-def X86thread_pointer_from_gs :
-  SDNode<"X86ISD::THREAD_POINTER_FROM_GS", SDT_X86ThreadPointer>;
-// @LOCALMOD-END
-
 def X86ehret : SDNode<"X86ISD::EH_RETURN", SDT_X86EHRET,
                         [SDNPHasChain]>;
 
@@ -297,6 +282,10 @@ let RenderMethod = "addMemOperands" in {
   def X86MemVX64Operand : AsmOperandClass { let Name = "MemVX64"; }
   def X86MemVY64Operand : AsmOperandClass { let Name = "MemVY64"; }
   def X86MemVZ64Operand : AsmOperandClass { let Name = "MemVZ64"; }
+  def X86MemVX32XOperand : AsmOperandClass { let Name = "MemVX32X"; }
+  def X86MemVY32XOperand : AsmOperandClass { let Name = "MemVY32X"; }
+  def X86MemVX64XOperand : AsmOperandClass { let Name = "MemVX64X"; }
+  def X86MemVY64XOperand : AsmOperandClass { let Name = "MemVY64X"; }
 }
 
 def X86AbsMemAsmOperand : AsmOperandClass {
@@ -347,7 +336,11 @@ def vx32mem  : X86VMemOperand<VR128,  "printi32mem", X86MemVX32Operand>;
 def vy32mem  : X86VMemOperand<VR256,  "printi32mem", X86MemVY32Operand>;
 def vx64mem  : X86VMemOperand<VR128,  "printi64mem", X86MemVX64Operand>;
 def vy64mem  : X86VMemOperand<VR256,  "printi64mem", X86MemVY64Operand>;
-def vy64xmem : X86VMemOperand<VR256X, "printi64mem", X86MemVY64Operand>;
+
+def vx32xmem : X86VMemOperand<VR128X, "printi32mem", X86MemVX32XOperand>;
+def vx64xmem : X86VMemOperand<VR128X, "printi32mem", X86MemVX64XOperand>;
+def vy32xmem : X86VMemOperand<VR256X, "printi32mem", X86MemVY32XOperand>;
+def vy64xmem : X86VMemOperand<VR256X, "printi64mem", X86MemVY64XOperand>;
 def vz32mem  : X86VMemOperand<VR512,  "printi32mem", X86MemVZ32Operand>;
 def vz64mem  : X86VMemOperand<VR512,  "printi64mem", X86MemVZ64Operand>;
 
@@ -731,6 +724,8 @@ def tls64addr : ComplexPattern<i64, 5, "SelectTLSADDRAddr",
 def tls64baseaddr : ComplexPattern<i64, 5, "SelectTLSADDRAddr",
                                [tglobaltlsaddr], []>;
 
+def vectoraddr : ComplexPattern<iPTR, 5, "SelectVectorAddr", [],[SDNPWantParent]>;
+
 //===----------------------------------------------------------------------===//
 // X86 Instruction Predicate Definitions.
 def HasCMov      : Predicate<"Subtarget->hasCMov()">;
@@ -772,9 +767,11 @@ def HasDQI       : Predicate<"Subtarget->hasDQI()">,
 def NoDQI        : Predicate<"!Subtarget->hasDQI()">;
 def HasBWI       : Predicate<"Subtarget->hasBWI()">,
                      AssemblerPredicate<"FeatureBWI", "AVX-512 BW ISA">;
+def NoBWI        : Predicate<"!Subtarget->hasBWI()">;
 def HasVLX       : Predicate<"Subtarget->hasVLX()">,
                      AssemblerPredicate<"FeatureVLX", "AVX-512 VL ISA">;
 def NoVLX        : Predicate<"!Subtarget->hasVLX()">;
+def NoVLX_Or_NoBWI : Predicate<"!Subtarget->hasVLX() || !Subtarget->hasBWI()">;
 
 def HasPOPCNT    : Predicate<"Subtarget->hasPOPCNT()">;
 def HasAES       : Predicate<"Subtarget->hasAES()">;
@@ -801,6 +798,7 @@ def HasRDSEED    : Predicate<"Subtarget->hasRDSEED()">;
 def HasPrefetchW : Predicate<"Subtarget->hasPRFCHW()">;
 def FPStackf32   : Predicate<"!Subtarget->hasSSE1()">;
 def FPStackf64   : Predicate<"!Subtarget->hasSSE2()">;
+def HasMPX       : Predicate<"Subtarget->hasMPX()">;
 def HasCmpxchg16b: Predicate<"Subtarget->hasCmpxchg16b()">;
 def Not64BitMode : Predicate<"!Subtarget->is64Bit()">,
                              AssemblerPredicate<"!Mode64Bit", "Not 64-bit mode">;
@@ -819,7 +817,7 @@ def NotWin64     : Predicate<"!Subtarget->isTargetWin64()">;
 def IsPS4        : Predicate<"Subtarget->isTargetPS4()">;
 def NotPS4       : Predicate<"!Subtarget->isTargetPS4()">;
 def IsNaCl       : Predicate<"Subtarget->isTargetNaCl()">;
-def IsNotNaCl    : Predicate<"!Subtarget->isTargetNaCl()">;
+def NotNaCl      : Predicate<"!Subtarget->isTargetNaCl()">;
 def SmallCode    : Predicate<"TM.getCodeModel() == CodeModel::Small">;
 def KernelCode   : Predicate<"TM.getCodeModel() == CodeModel::Kernel">;
 def FarData      : Predicate<"TM.getCodeModel() != CodeModel::Small &&"
@@ -1024,26 +1022,29 @@ def PUSH32r  : I<0x50, AddRegFrm, (outs), (ins GR32:$reg), "push{l}\t$reg",[],
                  IIC_PUSH_REG>, OpSize32, Requires<[Not64BitMode]>;
 def PUSH16rmr: I<0xFF, MRM6r, (outs), (ins GR16:$reg), "push{w}\t$reg",[],
                  IIC_PUSH_REG>, OpSize16;
-def PUSH16rmm: I<0xFF, MRM6m, (outs), (ins i16mem:$src), "push{w}\t$src",[],
-                 IIC_PUSH_MEM>, OpSize16;
 def PUSH32rmr: I<0xFF, MRM6r, (outs), (ins GR32:$reg), "push{l}\t$reg",[],
                  IIC_PUSH_REG>, OpSize32, Requires<[Not64BitMode]>;
-def PUSH32rmm: I<0xFF, MRM6m, (outs), (ins i32mem:$src), "push{l}\t$src",[],
-                 IIC_PUSH_MEM>, OpSize32, Requires<[Not64BitMode]>;
 
 def PUSH16i8 : Ii8<0x6a, RawFrm, (outs), (ins i16i8imm:$imm),
-                   "push{w}\t$imm", [], IIC_PUSH_IMM>, OpSize16,
-                   Requires<[Not64BitMode]>;
+                   "push{w}\t$imm", [], IIC_PUSH_IMM>, OpSize16;
+def PUSHi16  : Ii16<0x68, RawFrm, (outs), (ins i16imm:$imm),
+                   "push{w}\t$imm", [], IIC_PUSH_IMM>, OpSize16;
+
 def PUSH32i8 : Ii8<0x6a, RawFrm, (outs), (ins i32i8imm:$imm),
                    "push{l}\t$imm", [], IIC_PUSH_IMM>, OpSize32,
                    Requires<[Not64BitMode]>;
-def PUSHi16  : Ii16<0x68, RawFrm, (outs), (ins i16imm:$imm),
-                   "push{w}\t$imm", [], IIC_PUSH_IMM>, OpSize16,
-                   Requires<[Not64BitMode]>;
 def PUSHi32  : Ii32<0x68, RawFrm, (outs), (ins i32imm:$imm),
                    "push{l}\t$imm", [], IIC_PUSH_IMM>, OpSize32,
                    Requires<[Not64BitMode]>;
 } // mayStore, SchedRW
+
+let mayLoad = 1, mayStore = 1, SchedRW = [WriteRMW] in {
+def PUSH16rmm: I<0xFF, MRM6m, (outs), (ins i16mem:$src), "push{w}\t$src",[],
+                 IIC_PUSH_MEM>, OpSize16;
+def PUSH32rmm: I<0xFF, MRM6m, (outs), (ins i32mem:$src), "push{l}\t$src",[],
+                 IIC_PUSH_MEM>, OpSize32, Requires<[Not64BitMode]>;
+} // mayLoad, mayStore, SchedRW
+
 }
 
 let Defs = [ESP, EFLAGS], Uses = [ESP], mayLoad = 1, hasSideEffects=0,
@@ -1076,18 +1077,17 @@ def PUSH64r  : I<0x50, AddRegFrm, (outs), (ins GR64:$reg), "push{q}\t$reg", [],
                  IIC_PUSH_REG>, OpSize32, Requires<[In64BitMode]>;
 def PUSH64rmr: I<0xFF, MRM6r, (outs), (ins GR64:$reg), "push{q}\t$reg", [],
                  IIC_PUSH_REG>, OpSize32, Requires<[In64BitMode]>;
+} // mayStore, SchedRW
+let mayLoad = 1, mayStore = 1, SchedRW = [WriteRMW] in {
 def PUSH64rmm: I<0xFF, MRM6m, (outs), (ins i64mem:$src), "push{q}\t$src", [],
                  IIC_PUSH_MEM>, OpSize32, Requires<[In64BitMode]>;
-} // mayStore, SchedRW
+} // mayLoad, mayStore, SchedRW
 }
 
 let Defs = [RSP], Uses = [RSP], hasSideEffects = 0, mayStore = 1,
     SchedRW = [WriteStore] in {
 def PUSH64i8   : Ii8<0x6a, RawFrm, (outs), (ins i64i8imm:$imm),
                     "push{q}\t$imm", [], IIC_PUSH_IMM>, Requires<[In64BitMode]>;
-def PUSH64i16  : Ii16<0x68, RawFrm, (outs), (ins i16imm:$imm),
-                    "push{w}\t$imm", [], IIC_PUSH_IMM>, OpSize16,
-                    Requires<[In64BitMode]>;
 def PUSH64i32  : Ii32S<0x68, RawFrm, (outs), (ins i64i32imm:$imm),
                     "push{q}\t$imm", [], IIC_PUSH_IMM>, OpSize32,
                     Requires<[In64BitMode]>;
@@ -2238,7 +2238,7 @@ let Predicates = [HasBMI2], Defs = [EFLAGS] in {
 
 def CountTrailingOnes : SDNodeXForm<imm, [{
   // Count the trailing ones in the immediate.
-  return getI8Imm(countTrailingOnes(N->getZExtValue()));
+  return getI8Imm(countTrailingOnes(N->getZExtValue()), SDLoc(N));
 }]>;
 
 def BZHIMask : ImmLeaf<i64, [{
@@ -2446,12 +2446,6 @@ def PCOMMIT    : I<0xAE, MRM_F8, (outs), (ins), "pcommit", []>, PD;
 //===----------------------------------------------------------------------===//
 
 include "X86InstrArithmetic.td"
-
-//===----------------------------------------------------------------------===//
-// NaCl support (@LOCALMOD)
-//===----------------------------------------------------------------------===//
-
-include "X86InstrNaCl.td"
 include "X86InstrCMovSetCC.td"
 include "X86InstrExtension.td"
 include "X86InstrControl.td"
@@ -2475,6 +2469,9 @@ include "X86InstrAVX512.td"
 include "X86InstrMMX.td"
 include "X86Instr3DNow.td"
 
+// MPX instructions
+include "X86InstrMPX.td"
+
 include "X86InstrVMX.td"
 include "X86InstrSVM.td"
 
diff --git a/lib/Target/X86/X86InstrMPX.td b/lib/Target/X86/X86InstrMPX.td
new file mode 100644
index 00000000000..cf5e2e38fe5
--- /dev/null
+++ b/lib/Target/X86/X86InstrMPX.td
@@ -0,0 +1,70 @@
+//===-- X86InstrMPX.td - MPX Instruction Set ---------*- tablegen -*-===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This file describes the X86 MPX instruction set, defining the
+// instructions, and properties of the instructions which are needed for code
+// generation, machine code emission, and analysis.
+//
+//===----------------------------------------------------------------------===//
+
+multiclass mpx_bound_make<bits<8> opc, string OpcodeStr> {
+  def 32rm: I<opc, MRMSrcMem, (outs BNDR:$dst), (ins i32mem:$src),
+              OpcodeStr#" \t{$src, $dst|$dst, $src}", []>,
+              Requires<[HasMPX, Not64BitMode]>;
+  def 64rm: RI<opc, MRMSrcMem, (outs BNDR:$dst), (ins i64mem:$src),
+              OpcodeStr#" \t{$src, $dst|$dst, $src}", []>,
+              Requires<[HasMPX, In64BitMode]>;
+}
+
+defm BNDMK : mpx_bound_make<0x1B, "bndmk">, XS;
+
+multiclass mpx_bound_check<bits<8> opc, string OpcodeStr> {
+  def 32rm: I<opc, MRMSrcMem, (outs), (ins  BNDR:$src1, i32mem:$src2),
+              OpcodeStr#" \t{$src2, $src1|$src1, $src2}", []>,
+              Requires<[HasMPX, Not64BitMode]>;
+  def 64rm: RI<opc, MRMSrcMem, (outs), (ins  BNDR:$src1, i64mem:$src2),
+              OpcodeStr#" \t{$src2, $src1|$src1, $src2}", []>,
+              Requires<[HasMPX, In64BitMode]>;
+  def 32rr: I<opc, MRMSrcReg, (outs), (ins  BNDR:$src1, GR32:$src2),
+              OpcodeStr#" \t{$src2, $src1|$src1, $src2}", []>,
+              Requires<[HasMPX, Not64BitMode]>;
+  def 64rr: RI<opc, MRMSrcReg, (outs), (ins  BNDR:$src1, GR64:$src2),
+              OpcodeStr#" \t{$src2, $src1|$src1, $src2}", []>,
+              Requires<[HasMPX, In64BitMode]>;
+}
+defm BNDCL : mpx_bound_check<0x1A, "bndcl">, XS;
+defm BNDCU : mpx_bound_check<0x1A, "bndcu">, XD;
+defm BNDCN : mpx_bound_check<0x1B, "bndcn">, XD;
+
+def BNDMOVRMrr   : I<0x1A, MRMSrcReg, (outs BNDR:$dst), (ins BNDR:$src),
+                    "bndmov \t{$src, $dst|$dst, $src}", []>, PD,
+                    Requires<[HasMPX]>;
+def BNDMOVRM32rm : I<0x1A, MRMSrcMem, (outs BNDR:$dst), (ins i64mem:$src),
+                    "bndmov \t{$src, $dst|$dst, $src}", []>, PD,
+                    Requires<[HasMPX, Not64BitMode]>;
+def BNDMOVRM64rm : RI<0x1A, MRMSrcMem, (outs BNDR:$dst), (ins i128mem:$src),
+                    "bndmov \t{$src, $dst|$dst, $src}", []>, PD,
+                    Requires<[HasMPX, In64BitMode]>;
+
+def BNDMOVMRrr   : I<0x1B, MRMDestReg, (outs BNDR:$dst), (ins BNDR:$src),
+                    "bndmov \t{$src, $dst|$dst, $src}", []>, PD,
+                    Requires<[HasMPX]>;
+def BNDMOVMR32mr : I<0x1B, MRMDestMem, (outs i64mem:$dst), (ins BNDR:$src),
+                    "bndmov \t{$src, $dst|$dst, $src}", []>, PD,
+                    Requires<[HasMPX, Not64BitMode]>;
+def BNDMOVMR64mr : RI<0x1B, MRMDestMem, (outs i128mem:$dst), (ins BNDR:$src),
+                    "bndmov \t{$src, $dst|$dst, $src}", []>, PD,
+                    Requires<[HasMPX, In64BitMode]>;
+
+def BNDSTXmr:      I<0x1B, MRMDestMem, (outs), (ins i64mem:$dst, BNDR:$src),
+                    "bndstx \t{$src, $dst|$dst, $src}", []>, TB,
+                    Requires<[HasMPX]>;
+def BNDLDXrm:      I<0x1A, MRMSrcMem, (outs BNDR:$dst), (ins i64mem:$src),
+                    "bndldx \t{$src, $dst|$dst, $src}", []>, TB,
+                    Requires<[HasMPX]>;
\ No newline at end of file
diff --git a/lib/Target/X86/X86InstrNaCl.td b/lib/Target/X86/X86InstrNaCl.td
deleted file mode 100644
index 2f35a784df1..00000000000
--- a/lib/Target/X86/X86InstrNaCl.td
+++ /dev/null
@@ -1,251 +0,0 @@
-//====- X86InstrNaCl.td - Describe NaCl Instructions ----*- tablegen -*-===//
-//
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
-//
-//===----------------------------------------------------------------------===//
-//
-// This file describes the modifications to the X86 instruction set needed for
-// Native Client code generation.
-//
-//===----------------------------------------------------------------------===//
-
-
-//===----------------------------------------------------------------------===//
-// NaCl specific DAG Nodes.
-//
-
-//===----------------------------------------------------------------------===//
-//
-//                       Native Client Pseudo-Instructions
-//
-// These instructions implement the Native Client pseudo-instructions, such
-// as nacljmp and naclasp.
-//
-// TableGen and MC consider these to be "real" instructions. They can be
-// parsed by the AsmParser and emitted by the AsmStreamer as if they
-// were just regular instructions. They are not marked "Pseudo" because
-// this would imply isCodeGenOnly=1, which would stop them from being
-// parsed by the assembler.
-//
-// These instructions cannot be encoded (written into an object file) by the
-// MCCodeEmitter. Instead, during direct object emission, they get lowered to
-// a sequence of streamer emits. (see X86InstrNaCl.cpp)
-//
-// These instructions should not be used in CodeGen. They have no pattern
-// and lack CodeGen metadata. Instead, the X86NaClRewritePass should
-// generate these instructions after CodeGen is finished.
-//
-//===----------------------------------------------------------------------===//
-
-
-//===----------------------------------------------------------------------===//
-// 32-bit Native Client Pseudo Instructions
-//===----------------------------------------------------------------------===//
-
-class NaClPI32<dag outs, dag ins, string asm>
-  : I<0, CustomFrm, outs, ins, asm, []>, Requires<[IsNaCl, Not64BitMode]>;
-
-let isTerminator = 1, isReturn = 1, isBarrier = 1,
-    hasCtrlDep = 1, FPForm = SpecialFP, isAsmParserOnly = 1 in {
-  def NACL_RET32  : NaClPI32<(outs), (ins), "naclret">;
-  def NACL_RETI32 : NaClPI32<(outs), (ins i16imm:$amt), "naclreti\t$amt">;
-}
-
-let isBranch = 1, isTerminator = 1, isBarrier = 1, isIndirectBranch = 1,
-    isAsmParserOnly = 1 in {
-  def NACL_JMP32r : NaClPI32<(outs), (ins GR32:$dst), "nacljmp\t$dst">;
-}
-
-let isCall = 1, isAsmParserOnly = 1 in {
-  def NACL_CALL32r : NaClPI32<(outs), (ins GR32:$dst),
-                     "naclcall\t$dst">;
-}
-
-
-//===----------------------------------------------------------------------===//
-// 64-bit Native Client Pseudo Instructions
-//===----------------------------------------------------------------------===//
-
-class NaClPI64<dag outs, dag ins, string asm>
-  : I<0, CustomFrm, outs, ins, asm, []>, Requires<[IsNaCl, In64BitMode]>;
-
-let isTerminator = 1, isReturn = 1, isBarrier = 1,
-    hasCtrlDep = 1, FPForm = SpecialFP, isAsmParserOnly = 1 in {
-  def NACL_RET64  : NaClPI64<(outs), (ins), "naclret">;
-}
-
-let isBranch = 1, isTerminator = 1, isBarrier = 1, isIndirectBranch = 1,
-    isAsmParserOnly = 1 in {
-  def NACL_JMP64r : NaClPI64<(outs), (ins GR32:$dst, GR64:$rZP),
-                    "nacljmp\t{$dst, $rZP|$rZP, $dst}">;
-  def NACL_JMP64z : NaClPI64<(outs), (ins GR32:$dst),
-                    "nacljmp\t$dst">;
-}
-
-
-let isCall = 1, isAsmParserOnly = 1 in {
-  def NACL_CALL64d : NaClPI64<(outs), (ins i32imm_pcrel:$dst),
-                     "call{q}\t$dst">;
-  def NACL_CALL64r : NaClPI64<(outs), (ins GR32:$dst, GR64:$rZP),
-                     "naclcall\t$dst,$rZP">;
-}
-
-let Defs = [RSP, EFLAGS], Uses = [RSP], isAsmParserOnly = 1 in {
-  def NACL_ASPi8 : NaClPI64<(outs), (ins i64i8imm:$off, GR64:$rZP),
-                   "naclasp{q}\t{$off, $rZP|$rZP, $off}">;
-
-  def NACL_ASPi32: NaClPI64<(outs), (ins i64i32imm:$off, GR64:$rZP),
-                   "naclasp{q}\t{$off, $rZP|$rZP, $off}">;
-
-  def NACL_SSPi8 : NaClPI64<(outs), (ins i64i8imm:$off, GR64:$rZP),
-                   "naclssp{q}\t{$off, $rZP|$rZP, $off}">;
-
-  def NACL_SSPi32: NaClPI64<(outs), (ins i64i32imm:$off, GR64:$rZP),
-                   "naclssp{q}\t{$off, $rZP|$rZP, $off}">;
-
-  def NACL_ANDSPi8: NaClPI64<(outs), (ins i64i8imm:$off, GR64:$rZP),
-                   "naclandsp{q}\t{$off, $rZP|$rZP, $off}">;
-
-  def NACL_ANDSPi32: NaClPI64<(outs), (ins i64i32imm:$off, GR64:$rZP),
-                   "naclandsp{q}\t{$off, $rZP|$rZP, $off}">;
-}
-
-let Defs = [RSP], Uses = [RBP], isAsmParserOnly = 1 in {
-  def NACL_SPADJi32  : NaClPI64<(outs), (ins i64i32imm:$off, GR64:$rZP),
-                       "naclspadj\t{$off, $rZP|$rZP, $off}">;
-}
-
-let Defs = [RSP], isAsmParserOnly = 1 in {
-  def NACL_RESTSPr   : NaClPI64<(outs), (ins GR32:$src, GR64:$rZP),
-                       "naclrestsp_noflags\t{$src, $rZP|$rZP, $src}">;
-  def NACL_RESTSPm   : NaClPI64<(outs), (ins i32mem:$src, GR64:$rZP),
-                       "naclrestsp_noflags\t{$src, $rZP|$rZP, $src}">;
-  def NACL_RESTSPrz   : NaClPI64<(outs), (ins GR32:$src),
-                       "naclrestsp_noflags\t$src">;
-}
-
-def : MnemonicAlias<"naclrestsp", "naclrestsp_noflags">;
-
-let Defs = [RBP], isAsmParserOnly = 1 in {
-  def NACL_RESTBPr   : NaClPI64<(outs), (ins GR32:$src, GR64:$rZP),
-                       "naclrestbp\t{$src, $rZP|$rZP, $src}">;
-  def NACL_RESTBPm   : NaClPI64<(outs), (ins i32mem:$src, GR64:$rZP),
-                       "naclrestbp\t{$src, $rZP|$rZP, $src}">;
-  def NACL_RESTBPrz   : NaClPI64<(outs), (ins GR32:$src),
-                       "naclrestbp\t$src">;
-}
-
-//===----------------------------------------------------------------------===//
-//
-// Code Generator Instructions (isCodeGenOnly == 1)
-//
-// These instructions exists to make CodeGen work with Native Client's
-// modifications.
-//
-// Many of these instructions exist because of limitations in CodeGen
-// or TableGen, and may become unnecessary in the future.
-//===----------------------------------------------------------------------===//
-
-//===----------------------------------------------------------------------===//
-//
-// CodeGen 64-bit
-//
-//===----------------------------------------------------------------------===//
-
-
-// Because pointers are 32-bit on X86-64 Native Client, we need to
-// produce new versions of the JMP64/CALL64 instructions which can accept
-// addresses which are i32 instead of i64.
-//
-// Indirect calls have been changed to zero extend i32 to i64, so
-// we don't need a NaCl-specific CALL64r. We could possibly do the same for
-// indirect branches (switch jump tables) and direct calls.
-
-let isBranch = 1, isTerminator = 1, isBarrier = 1, isIndirectBranch = 1 in {
-  def NACL_CG_JMP64r     : I<0, Pseudo, (outs), (ins GR32:$dst),
-                           "nacljmp\t$dst",
-                           [(brind GR32:$dst)]>,
-                           Requires<[IsNaCl, In64BitMode]>;
-}
-
-// RSP is marked as a use to prevent stack-pointer assignments that appear
-// immediately before calls from potentially appearing dead. Uses for argument
-// registers are added manually.
-let isCall = 1, Uses = [RSP] in {
-    def NACL_CG_CALL64pcrel32 : I<0, Pseudo, (outs),
-                                (ins i32imm_pcrel:$dst),
-                                "call{q}\t$dst", []>,
-                                Requires<[IsNaCl, In64BitMode]>;
-}
-
-def : Pat<(X86call (i32 tglobaladdr:$dst)),
-          (NACL_CG_CALL64pcrel32 tglobaladdr:$dst)>,
-      Requires<[IsNaCl, In64BitMode]>;
-def : Pat<(X86call (i32 texternalsym:$dst)),
-          (NACL_CG_CALL64pcrel32 texternalsym:$dst)>,
-      Requires<[IsNaCl, In64BitMode]>;
-
-// Tail calls
-// Also needed due to the i64 / i32 pointer problem.
-let isCall = 1, isTerminator = 1, isReturn = 1, isBarrier = 1,
-    isCodeGenOnly = 1 in
-  let Uses = [RSP] in {
-
-  def NACL_CG_TCRETURNdi64 : I<0, Pseudo, (outs),
-                             (ins i32imm_pcrel:$dst, i32imm:$offset),
-                             "#TC_RETURN $dst $offset", []>,
-                          Requires<[IsNaCl, In64BitMode]>;
-
-  def NACL_CG_TAILJMPd64 : I<0, Pseudo, (outs),
-                           (ins i32imm_pcrel:$dst),
-                           "jmp\t$dst  # TAILCALL", []>,
-                           Requires<[IsNaCl, In64BitMode]>;
-}
-
-def : Pat<(X86tcret (i32 tglobaladdr:$dst), imm:$off),
-          (NACL_CG_TCRETURNdi64 tglobaladdr:$dst, imm:$off)>,
-         Requires<[IsNaCl, In64BitMode]>;
-
-def : Pat<(X86tcret (i32 texternalsym:$dst), imm:$off),
-          (NACL_CG_TCRETURNdi64 texternalsym:$dst, imm:$off)>,
-         Requires<[IsNaCl, In64BitMode]>;
-
-// ELF TLS Support
-
-// These are lowered in X86NaClRewritePass.
-let Defs = [RAX, RCX, RDX, RSI, RDI, R8, R9, R10, R11,
-            FP0, FP1, FP2, FP3, FP4, FP5, FP6, ST0, ST1,
-            MM0, MM1, MM2, MM3, MM4, MM5, MM6, MM7,
-            XMM0, XMM1, XMM2, XMM3, XMM4, XMM5, XMM6, XMM7,
-            XMM8, XMM9, XMM10, XMM11, XMM12, XMM13, XMM14, XMM15, EFLAGS],
-    Uses = [RSP] in {
-def NACL_CG_GD_TLS_addr64 : I<0, Pseudo, (outs), (ins i32mem:$sym), "",
-                            [(X86tlsaddr tls32addr:$sym)]>,
-                            Requires<[IsNaCl, In64BitMode]>;
-def NACL_CG_LE_TLS_addr64 : I<0, Pseudo, (outs), (ins i32mem:$sym), "",
-                            [(X86tlsaddr_le tls32addr:$sym)]>,
-                            Requires<[IsNaCl, In64BitMode]>;
-def NACL_CG_IE_TLS_addr64 : I<0, Pseudo, (outs), (ins i32mem:$sym), "",
-                            [(X86tlsaddr_ie tls32addr:$sym)]>,
-                            Requires<[IsNaCl, In64BitMode]>;
-// For mtls-use-call.
-def NACL_CG_LE_TLS_addr32 : I<0, Pseudo, (outs), (ins i32mem:$sym), "",
-                            [(X86tlsaddr_le tls32addr:$sym)]>,
-                            Requires<[IsNaCl, Not64BitMode]>;
-def NACL_CG_IE_TLS_addr32 : I<0, Pseudo, (outs), (ins i32mem:$sym), "",
-                            [(X86tlsaddr_ie tls32addr:$sym)]>,
-                            Requires<[IsNaCl, Not64BitMode]>;
-}
-
-let usesCustomInserter = 1, Defs = [EFLAGS] in
-def NACL_CG_VAARG_64 : I<0, Pseudo,
-                     (outs GR32:$dst),
-                     (ins i8mem:$ap, i32imm:$size, i8imm:$mode, i32imm:$align),
-                     "#NACL_VAARG_64 $dst, $ap, $size, $mode, $align",
-                     [(set GR32:$dst,
-                     (X86vaarg64 addr:$ap, imm:$size, imm:$mode, imm:$align)),
-                     (implicit EFLAGS)]>,
-                     Requires<[IsNaCl, In64BitMode]>;
diff --git a/lib/Target/X86/X86InstrSSE.td b/lib/Target/X86/X86InstrSSE.td
index 65b155c0a9d..1a77d0efdd3 100644
--- a/lib/Target/X86/X86InstrSSE.td
+++ b/lib/Target/X86/X86InstrSSE.td
@@ -241,21 +241,20 @@ def SSE_INTALU_ITINS_BLEND_P : OpndItins<
 /// sse12_fp_scalar - SSE 1 & 2 scalar instructions class
 multiclass sse12_fp_scalar<bits<8> opc, string OpcodeStr, SDNode OpNode,
                            RegisterClass RC, X86MemOperand x86memop,
-                           OpndItins itins,
-                           bit Is2Addr = 1> {
+                           Domain d, OpndItins itins, bit Is2Addr = 1> {
   let isCommutable = 1 in {
     def rr : SI<opc, MRMSrcReg, (outs RC:$dst), (ins RC:$src1, RC:$src2),
        !if(Is2Addr,
            !strconcat(OpcodeStr, "\t{$src2, $dst|$dst, $src2}"),
            !strconcat(OpcodeStr, "\t{$src2, $src1, $dst|$dst, $src1, $src2}")),
-       [(set RC:$dst, (OpNode RC:$src1, RC:$src2))], itins.rr>,
+       [(set RC:$dst, (OpNode RC:$src1, RC:$src2))], itins.rr, d>,
        Sched<[itins.Sched]>;
   }
   def rm : SI<opc, MRMSrcMem, (outs RC:$dst), (ins RC:$src1, x86memop:$src2),
        !if(Is2Addr,
            !strconcat(OpcodeStr, "\t{$src2, $dst|$dst, $src2}"),
            !strconcat(OpcodeStr, "\t{$src2, $src1, $dst|$dst, $src1, $src2}")),
-       [(set RC:$dst, (OpNode RC:$src1, (load addr:$src2)))], itins.rm>,
+       [(set RC:$dst, (OpNode RC:$src1, (load addr:$src2)))], itins.rm, d>,
        Sched<[itins.Sched.Folded, ReadAfterLd]>;
 }
 
@@ -263,24 +262,23 @@ multiclass sse12_fp_scalar<bits<8> opc, string OpcodeStr, SDNode OpNode,
 multiclass sse12_fp_scalar_int<bits<8> opc, string OpcodeStr, RegisterClass RC,
                              string asm, string SSEVer, string FPSizeStr,
                              Operand memopr, ComplexPattern mem_cpat,
-                             OpndItins itins,
-                             bit Is2Addr = 1> {
+                             Domain d, OpndItins itins, bit Is2Addr = 1> {
 let isCodeGenOnly = 1 in {
-  def rr_Int : SI<opc, MRMSrcReg, (outs RC:$dst), (ins RC:$src1, RC:$src2),
+  def rr_Int : SI_Int<opc, MRMSrcReg, (outs RC:$dst), (ins RC:$src1, RC:$src2),
        !if(Is2Addr,
            !strconcat(asm, "\t{$src2, $dst|$dst, $src2}"),
            !strconcat(asm, "\t{$src2, $src1, $dst|$dst, $src1, $src2}")),
        [(set RC:$dst, (!cast<Intrinsic>(
                  !strconcat("int_x86_sse", SSEVer, "_", OpcodeStr, FPSizeStr))
-             RC:$src1, RC:$src2))], itins.rr>,
+             RC:$src1, RC:$src2))], itins.rr, d>,
        Sched<[itins.Sched]>;
-  def rm_Int : SI<opc, MRMSrcMem, (outs RC:$dst), (ins RC:$src1, memopr:$src2),
+  def rm_Int : SI_Int<opc, MRMSrcMem, (outs RC:$dst), (ins RC:$src1, memopr:$src2),
        !if(Is2Addr,
            !strconcat(asm, "\t{$src2, $dst|$dst, $src2}"),
            !strconcat(asm, "\t{$src2, $src1, $dst|$dst, $src1, $src2}")),
        [(set RC:$dst, (!cast<Intrinsic>(!strconcat("int_x86_sse",
                                           SSEVer, "_", OpcodeStr, FPSizeStr))
-             RC:$src1, mem_cpat:$src2))], itins.rm>,
+             RC:$src1, mem_cpat:$src2))], itins.rm, d>,
        Sched<[itins.Sched.Folded, ReadAfterLd]>;
 }
 }
@@ -1853,14 +1851,14 @@ def Int_VCVTSD2SSrr: I<0x5A, MRMSrcReg,
                        "vcvtsd2ss\t{$src2, $src1, $dst|$dst, $src1, $src2}",
                        [(set VR128:$dst,
                          (int_x86_sse2_cvtsd2ss VR128:$src1, VR128:$src2))],
-                       IIC_SSE_CVT_Scalar_RR>, XD, VEX_4V, Requires<[UseAVX]>,
+                       IIC_SSE_CVT_Scalar_RR>, XD, VEX_4V, Requires<[HasAVX]>,
                        Sched<[WriteCvtF2F]>;
 def Int_VCVTSD2SSrm: I<0x5A, MRMSrcReg,
                        (outs VR128:$dst), (ins VR128:$src1, sdmem:$src2),
                        "vcvtsd2ss\t{$src2, $src1, $dst|$dst, $src1, $src2}",
                        [(set VR128:$dst, (int_x86_sse2_cvtsd2ss
                                           VR128:$src1, sse_load_f64:$src2))],
-                       IIC_SSE_CVT_Scalar_RM>, XD, VEX_4V, Requires<[UseAVX]>,
+                       IIC_SSE_CVT_Scalar_RM>, XD, VEX_4V, Requires<[HasAVX]>,
                        Sched<[WriteCvtF2FLd, ReadAfterLd]>;
 
 let Constraints = "$src1 = $dst" in {
@@ -1938,14 +1936,14 @@ def Int_VCVTSS2SDrr: I<0x5A, MRMSrcReg,
                     "vcvtss2sd\t{$src2, $src1, $dst|$dst, $src1, $src2}",
                     [(set VR128:$dst,
                       (int_x86_sse2_cvtss2sd VR128:$src1, VR128:$src2))],
-                    IIC_SSE_CVT_Scalar_RR>, XS, VEX_4V, Requires<[UseAVX]>,
+                    IIC_SSE_CVT_Scalar_RR>, XS, VEX_4V, Requires<[HasAVX]>,
                     Sched<[WriteCvtF2F]>;
 def Int_VCVTSS2SDrm: I<0x5A, MRMSrcMem,
                       (outs VR128:$dst), (ins VR128:$src1, ssmem:$src2),
                     "vcvtss2sd\t{$src2, $src1, $dst|$dst, $src1, $src2}",
                     [(set VR128:$dst,
                       (int_x86_sse2_cvtss2sd VR128:$src1, sse_load_f32:$src2))],
-                    IIC_SSE_CVT_Scalar_RM>, XS, VEX_4V, Requires<[UseAVX]>,
+                    IIC_SSE_CVT_Scalar_RM>, XS, VEX_4V, Requires<[HasAVX]>,
                     Sched<[WriteCvtF2FLd, ReadAfterLd]>;
 let Constraints = "$src1 = $dst" in { // SSE2 instructions with XS prefix
 def Int_CVTSS2SDrr: I<0x5A, MRMSrcReg,
@@ -2075,14 +2073,16 @@ def CVTTPS2DQrm : S2SI<0x5B, MRMSrcMem, (outs VR128:$dst), (ins f128mem:$src),
                        IIC_SSE_CVT_PS_RM>, Sched<[WriteCvtF2ILd]>;
 
 let Predicates = [HasAVX] in {
-  def : Pat<(v4f32 (sint_to_fp (v4i32 VR128:$src))),
+  def : Pat<(int_x86_sse2_cvtdq2ps VR128:$src),
             (VCVTDQ2PSrr VR128:$src)>;
-  def : Pat<(v4f32 (sint_to_fp (bc_v4i32 (loadv2i64 addr:$src)))),
+  def : Pat<(int_x86_sse2_cvtdq2ps (bc_v4i32 (loadv2i64 addr:$src))),
             (VCVTDQ2PSrm addr:$src)>;
+}
 
-  def : Pat<(int_x86_sse2_cvtdq2ps VR128:$src),
+let Predicates = [HasAVX, NoVLX] in {
+  def : Pat<(v4f32 (sint_to_fp (v4i32 VR128:$src))),
             (VCVTDQ2PSrr VR128:$src)>;
-  def : Pat<(int_x86_sse2_cvtdq2ps (bc_v4i32 (loadv2i64 addr:$src))),
+  def : Pat<(v4f32 (sint_to_fp (bc_v4i32 (loadv2i64 addr:$src)))),
             (VCVTDQ2PSrm addr:$src)>;
 
   def : Pat<(v4i32 (fp_to_sint (v4f32 VR128:$src))),
@@ -2151,7 +2151,7 @@ def VCVTTPD2DQYrm : VPDI<0xE6, MRMSrcMem, (outs VR128:$dst), (ins f256mem:$src),
 def : InstAlias<"vcvttpd2dq\t{$src, $dst|$dst, $src}",
                 (VCVTTPD2DQYrr VR128:$dst, VR256:$src), 0>;
 
-let Predicates = [HasAVX] in {
+let Predicates = [HasAVX, NoVLX] in {
   def : Pat<(v4i32 (fp_to_sint (v4f64 VR256:$src))),
             (VCVTTPD2DQYrr VR256:$src)>;
   def : Pat<(v4i32 (fp_to_sint (loadv4f64 addr:$src))),
@@ -2236,14 +2236,27 @@ def CVTDQ2PDrr  : S2SI<0xE6, MRMSrcReg, (outs VR128:$dst), (ins VR128:$src),
                        [(set VR128:$dst, (int_x86_sse2_cvtdq2pd VR128:$src))],
                        IIC_SSE_CVT_PD_RM>, Sched<[WriteCvtI2F]>;
 
-// AVX 256-bit register conversion intrinsics
+// AVX register conversion intrinsics
 let Predicates = [HasAVX] in {
+  def : Pat<(v2f64 (X86cvtdq2pd (v4i32 VR128:$src))),
+            (VCVTDQ2PDrr VR128:$src)>;
+  def : Pat<(v2f64 (X86cvtdq2pd (bc_v4i32 (loadv2i64 addr:$src)))),
+            (VCVTDQ2PDrm addr:$src)>;
+
   def : Pat<(v4f64 (sint_to_fp (v4i32 VR128:$src))),
             (VCVTDQ2PDYrr VR128:$src)>;
   def : Pat<(v4f64 (sint_to_fp (bc_v4i32 (loadv2i64 addr:$src)))),
             (VCVTDQ2PDYrm addr:$src)>;
 } // Predicates = [HasAVX]
 
+// SSE2 register conversion intrinsics
+let Predicates = [HasSSE2] in {
+  def : Pat<(v2f64 (X86cvtdq2pd (v4i32 VR128:$src))),
+            (CVTDQ2PDrr VR128:$src)>;
+  def : Pat<(v2f64 (X86cvtdq2pd (bc_v4i32 (loadv2i64 addr:$src)))),
+            (CVTDQ2PDrm addr:$src)>;
+} // Predicates = [HasSSE2]
+
 // Convert packed double to packed single
 // The assembler can recognize rr 256-bit instructions by seeing a ymm
 // register, but the same isn't true when using memory operands instead.
@@ -2295,7 +2308,9 @@ let Predicates = [HasAVX] in {
             (VCVTDQ2PSYrr VR256:$src)>;
   def : Pat<(int_x86_avx_cvtdq2_ps_256 (bitconvert (loadv4i64 addr:$src))),
             (VCVTDQ2PSYrm addr:$src)>;
+}
 
+let Predicates = [HasAVX, NoVLX] in {
   // Match fround and fextend for 128/256-bit conversions
   def : Pat<(v4f32 (X86vfpround (v2f64 VR128:$src))),
             (VCVTPD2PSrr VR128:$src)>;
@@ -2683,6 +2698,7 @@ multiclass sse12_unpack_interleave<bits<8> opc, SDNode OpNode, ValueType vt,
              Sched<[WriteFShuffleLd, ReadAfterLd]>;
 }
 
+let Predicates = [HasAVX, NoVLX] in {
 defm VUNPCKHPS: sse12_unpack_interleave<0x15, X86Unpckh, v4f32, loadv4f32,
       VR128, f128mem, "unpckhps\t{$src2, $src1, $dst|$dst, $src1, $src2}",
                      SSEPackedSingle>, PS, VEX_4V;
@@ -2708,7 +2724,7 @@ defm VUNPCKLPSY: sse12_unpack_interleave<0x14, X86Unpckl, v8f32, loadv8f32,
 defm VUNPCKLPDY: sse12_unpack_interleave<0x14, X86Unpckl, v4f64, loadv4f64,
       VR256, f256mem, "unpcklpd\t{$src2, $src1, $dst|$dst, $src1, $src2}",
                      SSEPackedDouble>, PD, VEX_4V, VEX_L;
-
+}// Predicates = [HasAVX, NoVLX]
 let Constraints = "$src1 = $dst" in {
   defm UNPCKHPS: sse12_unpack_interleave<0x15, X86Unpckh, v4f32, memopv4f32,
         VR128, f128mem, "unpckhps\t{$src2, $dst|$dst, $src2}",
@@ -2834,8 +2850,8 @@ multiclass PDI_binop_rm<bits<8> opc, string OpcodeStr, SDNode OpNode,
 
 multiclass PDI_binop_all<bits<8> opc, string OpcodeStr, SDNode Opcode,
                          ValueType OpVT128, ValueType OpVT256,
-                         OpndItins itins, bit IsCommutable = 0> {
-let Predicates = [HasAVX, NoVLX] in
+                         OpndItins itins, bit IsCommutable = 0, Predicate prd> {
+let Predicates = [HasAVX, prd] in
   defm V#NAME : PDI_binop_rm<opc, !strconcat("v", OpcodeStr), Opcode, OpVT128,
                     VR128, loadv2i64, i128mem, itins, IsCommutable, 0>, VEX_4V;
 
@@ -2843,7 +2859,7 @@ let Constraints = "$src1 = $dst" in
   defm NAME : PDI_binop_rm<opc, OpcodeStr, Opcode, OpVT128, VR128,
                            memopv2i64, i128mem, itins, IsCommutable, 1>;
 
-let Predicates = [HasAVX2, NoVLX] in
+let Predicates = [HasAVX2, prd] in
   defm V#NAME#Y : PDI_binop_rm<opc, !strconcat("v", OpcodeStr), Opcode,
                                OpVT256, VR256, loadv4i64, i256mem, itins,
                                IsCommutable, 0>, VEX_4V, VEX_L;
@@ -2852,13 +2868,13 @@ let Predicates = [HasAVX2, NoVLX] in
 // These are ordered here for pattern ordering requirements with the fp versions
 
 defm PAND  : PDI_binop_all<0xDB, "pand", and, v2i64, v4i64,
-                           SSE_VEC_BIT_ITINS_P, 1>;
+                           SSE_VEC_BIT_ITINS_P, 1, NoVLX>;
 defm POR   : PDI_binop_all<0xEB, "por", or, v2i64, v4i64,
-                           SSE_VEC_BIT_ITINS_P, 1>;
+                           SSE_VEC_BIT_ITINS_P, 1, NoVLX>;
 defm PXOR  : PDI_binop_all<0xEF, "pxor", xor, v2i64, v4i64,
-                           SSE_VEC_BIT_ITINS_P, 1>;
+                           SSE_VEC_BIT_ITINS_P, 1, NoVLX>;
 defm PANDN : PDI_binop_all<0xDF, "pandn", X86andnp, v2i64, v4i64,
-                           SSE_VEC_BIT_ITINS_P, 0>;
+                           SSE_VEC_BIT_ITINS_P, 0, NoVLX>;
 
 //===----------------------------------------------------------------------===//
 // SSE 1 & 2 - Logical Instructions
@@ -2908,6 +2924,14 @@ multiclass sse12_fp_packed_vector_logical_alias<
   defm V#NAME#PD : sse12_fp_packed<opc, !strconcat(OpcodeStr, "pd"), OpNode,
         VR128, v2f64, f128mem, loadv2f64, SSEPackedDouble, itins, 0>,
         PD, VEX_4V;
+
+  defm V#NAME#PSY : sse12_fp_packed<opc, !strconcat(OpcodeStr, "ps"), OpNode,
+        VR256, v8f32, f256mem, loadv8f32, SSEPackedSingle, itins, 0>,
+        PS, VEX_4V, VEX_L;
+        
+  defm V#NAME#PDY : sse12_fp_packed<opc, !strconcat(OpcodeStr, "pd"), OpNode,
+        VR256, v4f64, f256mem, loadv4f64, SSEPackedDouble, itins, 0>,
+        PD, VEX_4V, VEX_L;
   }
 
   let Constraints = "$src1 = $dst" in {
@@ -3054,15 +3078,19 @@ multiclass basic_sse12_fp_binop_p<bits<8> opc, string OpcodeStr,
 multiclass basic_sse12_fp_binop_s<bits<8> opc, string OpcodeStr, SDNode OpNode,
                                   SizeItins itins> {
   defm V#NAME#SS : sse12_fp_scalar<opc, !strconcat(OpcodeStr, "ss"),
-                         OpNode, FR32, f32mem, itins.s, 0>, XS, VEX_4V, VEX_LIG;
+                         OpNode, FR32, f32mem, SSEPackedSingle, itins.s, 0>,
+                         XS, VEX_4V, VEX_LIG;
   defm V#NAME#SD : sse12_fp_scalar<opc, !strconcat(OpcodeStr, "sd"),
-                         OpNode, FR64, f64mem, itins.d, 0>, XD, VEX_4V, VEX_LIG;
+                         OpNode, FR64, f64mem, SSEPackedDouble, itins.d, 0>,
+                         XD, VEX_4V, VEX_LIG;
 
   let Constraints = "$src1 = $dst" in {
     defm SS : sse12_fp_scalar<opc, !strconcat(OpcodeStr, "ss"),
-                              OpNode, FR32, f32mem, itins.s>, XS;
+                              OpNode, FR32, f32mem, SSEPackedSingle,
+                              itins.s>, XS;
     defm SD : sse12_fp_scalar<opc, !strconcat(OpcodeStr, "sd"),
-                              OpNode, FR64, f64mem, itins.d>, XD;
+                              OpNode, FR64, f64mem, SSEPackedDouble,
+                              itins.d>, XD;
   }
 }
 
@@ -3070,18 +3098,18 @@ multiclass basic_sse12_fp_binop_s_int<bits<8> opc, string OpcodeStr,
                                       SizeItins itins> {
   defm V#NAME#SS : sse12_fp_scalar_int<opc, OpcodeStr, VR128,
                    !strconcat(OpcodeStr, "ss"), "", "_ss", ssmem, sse_load_f32,
-                   itins.s, 0>, XS, VEX_4V, VEX_LIG;
+                   SSEPackedSingle, itins.s, 0>, XS, VEX_4V, VEX_LIG;
   defm V#NAME#SD : sse12_fp_scalar_int<opc, OpcodeStr, VR128,
                    !strconcat(OpcodeStr, "sd"), "2", "_sd", sdmem, sse_load_f64,
-                   itins.d, 0>, XD, VEX_4V, VEX_LIG;
+                   SSEPackedDouble, itins.d, 0>, XD, VEX_4V, VEX_LIG;
 
   let Constraints = "$src1 = $dst" in {
     defm SS : sse12_fp_scalar_int<opc, OpcodeStr, VR128,
                    !strconcat(OpcodeStr, "ss"), "", "_ss", ssmem, sse_load_f32,
-                   itins.s>, XS;
+                   SSEPackedSingle, itins.s>, XS;
     defm SD : sse12_fp_scalar_int<opc, OpcodeStr, VR128,
                    !strconcat(OpcodeStr, "sd"), "2", "_sd", sdmem, sse_load_f64,
-                   itins.d>, XD;
+                   SSEPackedDouble, itins.d>, XD;
   }
 }
 
@@ -3170,16 +3198,9 @@ multiclass scalar_math_f32_patterns<SDNode Op, string OpcPrefix> {
           (Op (v4f32 VR128:$dst), (v4f32 VR128:$src)))),
       (!cast<I>(OpcPrefix#SSrr_Int) v4f32:$dst, v4f32:$src)>;
   }
-  
-  // With SSE 4.1, insertps/blendi are preferred to movsd, so match those too.
-  let Predicates = [UseSSE41] in {
-    // extracted scalar math op with insert via insertps
-    def : Pat<(v4f32 (X86insertps (v4f32 VR128:$dst), (v4f32 (scalar_to_vector
-          (Op (f32 (vector_extract (v4f32 VR128:$dst), (iPTR 0))),
-          FR32:$src))), (iPTR 0))),
-      (!cast<I>(OpcPrefix#SSrr_Int) v4f32:$dst,
-          (COPY_TO_REGCLASS FR32:$src, VR128))>;
 
+  // With SSE 4.1, blendi is preferred to movsd, so match that too.
+  let Predicates = [UseSSE41] in {
     // extracted scalar math op with insert via blend
     def : Pat<(v4f32 (X86Blendi (v4f32 VR128:$dst), (v4f32 (scalar_to_vector
           (Op (f32 (vector_extract (v4f32 VR128:$dst), (iPTR 0))),
@@ -3197,13 +3218,6 @@ multiclass scalar_math_f32_patterns<SDNode Op, string OpcPrefix> {
   // Repeat everything for AVX, except for the movss + scalar combo...
   // because that one shouldn't occur with AVX codegen?
   let Predicates = [HasAVX] in {
-    // extracted scalar math op with insert via insertps
-    def : Pat<(v4f32 (X86insertps (v4f32 VR128:$dst), (v4f32 (scalar_to_vector
-          (Op (f32 (vector_extract (v4f32 VR128:$dst), (iPTR 0))),
-          FR32:$src))), (iPTR 0))),
-      (!cast<I>("V"#OpcPrefix#SSrr_Int) v4f32:$dst,
-          (COPY_TO_REGCLASS FR32:$src, VR128))>;
- 
     // extracted scalar math op with insert via blend
     def : Pat<(v4f32 (X86Blendi (v4f32 VR128:$dst), (v4f32 (scalar_to_vector
           (Op (f32 (vector_extract (v4f32 VR128:$dst), (iPTR 0))),
@@ -3251,7 +3265,7 @@ multiclass scalar_math_f64_patterns<SDNode Op, string OpcPrefix> {
           FR64:$src))), (i8 1))),
       (!cast<I>(OpcPrefix#SDrr_Int) v2f64:$dst,
           (COPY_TO_REGCLASS FR64:$src, VR128))>;
-          
+
     // vector math op with insert via blend
     def : Pat<(v2f64 (X86Blendi (v2f64 VR128:$dst),
           (Op (v2f64 VR128:$dst), (v2f64 VR128:$src)), (i8 1))),
@@ -3345,17 +3359,17 @@ multiclass sse_fp_unop_s<bits<8> opc, string OpcodeStr, RegisterClass RC,
                           ValueType vt, ValueType ScalarVT,
                           X86MemOperand x86memop, Operand vec_memop,
                           ComplexPattern mem_cpat, Intrinsic Intr,
-                          SDNode OpNode, OpndItins itins, Predicate target,
-                          string Suffix> {
+                          SDNode OpNode, Domain d, OpndItins itins,
+                          Predicate target, string Suffix> {
   let hasSideEffects = 0 in {
   def r : I<opc, MRMSrcReg, (outs RC:$dst), (ins RC:$src1),
               !strconcat(OpcodeStr, "\t{$src1, $dst|$dst, $src1}"),
-            [(set RC:$dst, (OpNode RC:$src1))], itins.rr>, Sched<[itins.Sched]>,
+            [(set RC:$dst, (OpNode RC:$src1))], itins.rr, d>, Sched<[itins.Sched]>,
             Requires<[target]>;
   let mayLoad = 1 in
   def m : I<opc, MRMSrcMem, (outs RC:$dst), (ins x86memop:$src1),
             !strconcat(OpcodeStr, "\t{$src1, $dst|$dst, $src1}"),
-            [(set RC:$dst, (OpNode (load addr:$src1)))], itins.rm>,
+            [(set RC:$dst, (OpNode (load addr:$src1)))], itins.rm, d>,
             Sched<[itins.Sched.Folded, ReadAfterLd]>,
             Requires<[target, OptForSize]>;
 
@@ -3378,10 +3392,10 @@ multiclass sse_fp_unop_s<bits<8> opc, string OpcodeStr, RegisterClass RC,
   // because the high elements of the destination are unchanged in SSE.
   def : Pat<(Intr VR128:$src),
             (!cast<Instruction>(NAME#Suffix##r_Int) VR128:$src, VR128:$src)>;
-  def : Pat<(Intr (load addr:$src)), 
+  def : Pat<(Intr (load addr:$src)),
             (vt (COPY_TO_REGCLASS(!cast<Instruction>(NAME#Suffix##m)
                                       addr:$src), VR128))>;
-   def : Pat<(Intr mem_cpat:$src),
+  def : Pat<(Intr mem_cpat:$src),
              (!cast<Instruction>(NAME#Suffix##m_Int)
                     (vt (IMPLICIT_DEF)), mem_cpat:$src)>;
   }
@@ -3391,51 +3405,49 @@ multiclass avx_fp_unop_s<bits<8> opc, string OpcodeStr, RegisterClass RC,
                           ValueType vt, ValueType ScalarVT,
                           X86MemOperand x86memop, Operand vec_memop,
                           ComplexPattern mem_cpat,
-                          Intrinsic Intr, SDNode OpNode, OpndItins itins,
-                          Predicate target, string Suffix> {
+                          Intrinsic Intr, SDNode OpNode, Domain d,
+                          OpndItins itins, string Suffix> {
   let hasSideEffects = 0 in {
   def r : I<opc, MRMSrcReg, (outs RC:$dst), (ins RC:$src1, RC:$src2),
             !strconcat(OpcodeStr, "\t{$src2, $src1, $dst|$dst, $src1, $src2}"),
-            [], itins.rr>, Sched<[itins.Sched]>;
-  let mayLoad = 1 in 
+            [], itins.rr, d>, Sched<[itins.Sched]>;
+  let mayLoad = 1 in
   def m : I<opc, MRMSrcMem, (outs RC:$dst), (ins RC:$src1, x86memop:$src2),
              !strconcat(OpcodeStr, "\t{$src2, $src1, $dst|$dst, $src1, $src2}"),
-            [], itins.rm>, Sched<[itins.Sched.Folded, ReadAfterLd]>;
+            [], itins.rm, d>, Sched<[itins.Sched.Folded, ReadAfterLd]>;
   let isCodeGenOnly = 1 in {
-  // todo: uncomment when all r_Int forms will be added to X86InstrInfo.cpp
-  //def r_Int : I<opc, MRMSrcReg, (outs VR128:$dst), 
-  //              (ins VR128:$src1, VR128:$src2),
-  //           !strconcat(OpcodeStr, "\t{$src2, $src1, $dst|$dst, $src1, $src2}"),
-  //          []>, Sched<[itins.Sched.Folded]>;
+  def r_Int : I<opc, MRMSrcReg, (outs VR128:$dst),
+                (ins VR128:$src1, VR128:$src2),
+             !strconcat(OpcodeStr, "\t{$src2, $src1, $dst|$dst, $src1, $src2}"),
+             []>, Sched<[itins.Sched.Folded]>;
   let mayLoad = 1 in
-  def m_Int : I<opc, MRMSrcMem, (outs VR128:$dst), 
+  def m_Int : I<opc, MRMSrcMem, (outs VR128:$dst),
                 (ins VR128:$src1, vec_memop:$src2),
              !strconcat(OpcodeStr, "\t{$src2, $src1, $dst|$dst, $src1, $src2}"),
-            []>, Sched<[itins.Sched.Folded, ReadAfterLd]>;
+             []>, Sched<[itins.Sched.Folded, ReadAfterLd]>;
   }
   }
 
-  let Predicates = [target] in {
+  let Predicates = [UseAVX] in {
    def : Pat<(OpNode RC:$src),  (!cast<Instruction>("V"#NAME#Suffix##r)
                                 (ScalarVT (IMPLICIT_DEF)), RC:$src)>;
 
-   def : Pat<(vt (OpNode mem_cpat:$src)), 
+   def : Pat<(vt (OpNode mem_cpat:$src)),
              (!cast<Instruction>("V"#NAME#Suffix##m_Int) (vt (IMPLICIT_DEF)),
                                   mem_cpat:$src)>;
 
-   // todo: use r_Int form when it will be ready
-   //def : Pat<(Intr VR128:$src), (!cast<Instruction>("V"#NAME#Suffix##r_Int)
-   //                 (VT (IMPLICIT_DEF)), VR128:$src)>;
+  }
+  let Predicates = [HasAVX] in {
    def : Pat<(Intr VR128:$src),
-             (vt (COPY_TO_REGCLASS(
-             !cast<Instruction>("V"#NAME#Suffix##r) (ScalarVT (IMPLICIT_DEF)), 
-                    (ScalarVT (COPY_TO_REGCLASS VR128:$src, RC))), VR128))>;
+             (!cast<Instruction>("V"#NAME#Suffix##r_Int) (vt (IMPLICIT_DEF)),
+                                 VR128:$src)>;
+
    def : Pat<(Intr mem_cpat:$src),
              (!cast<Instruction>("V"#NAME#Suffix##m_Int)
                     (vt (IMPLICIT_DEF)), mem_cpat:$src)>;
   }
-  let Predicates = [target, OptForSize] in
-  def : Pat<(ScalarVT (OpNode (load addr:$src))), 
+  let Predicates = [UseAVX, OptForSize] in
+  def : Pat<(ScalarVT (OpNode (load addr:$src))),
             (!cast<Instruction>("V"#NAME#Suffix##m) (ScalarVT (IMPLICIT_DEF)),
              addr:$src)>;
 }
@@ -3476,46 +3488,6 @@ let Predicates = [HasAVX] in {
             Sched<[itins.Sched.Folded]>;
 }
 
-/// sse1_fp_unop_p_int - SSE1 intrinsics unops in packed forms.
-multiclass sse1_fp_unop_p_int<bits<8> opc, string OpcodeStr,
-                              Intrinsic V4F32Int, Intrinsic V8F32Int,
-                              OpndItins itins> {
-let isCodeGenOnly = 1 in {
-let Predicates = [HasAVX] in {
-  def V#NAME#PSr_Int : PSI<opc, MRMSrcReg, (outs VR128:$dst), (ins VR128:$src),
-                           !strconcat("v", OpcodeStr,
-                                      "ps\t{$src, $dst|$dst, $src}"),
-                           [(set VR128:$dst, (V4F32Int VR128:$src))],
-                           itins.rr>, VEX, Sched<[itins.Sched]>;
-  def V#NAME#PSm_Int : PSI<opc, MRMSrcMem, (outs VR128:$dst), (ins f128mem:$src),
-                          !strconcat("v", OpcodeStr,
-                          "ps\t{$src, $dst|$dst, $src}"),
-                          [(set VR128:$dst, (V4F32Int (loadv4f32 addr:$src)))],
-                          itins.rm>, VEX, Sched<[itins.Sched.Folded]>;
-  def V#NAME#PSYr_Int : PSI<opc, MRMSrcReg, (outs VR256:$dst), (ins VR256:$src),
-                            !strconcat("v", OpcodeStr,
-                                       "ps\t{$src, $dst|$dst, $src}"),
-                            [(set VR256:$dst, (V8F32Int VR256:$src))],
-                            itins.rr>, VEX, VEX_L, Sched<[itins.Sched]>;
-  def V#NAME#PSYm_Int : PSI<opc, MRMSrcMem, (outs VR256:$dst),
-                          (ins f256mem:$src),
-                          !strconcat("v", OpcodeStr,
-                                    "ps\t{$src, $dst|$dst, $src}"),
-                          [(set VR256:$dst, (V8F32Int (loadv8f32 addr:$src)))],
-                          itins.rm>, VEX, VEX_L, Sched<[itins.Sched.Folded]>;
-}
-
-  def PSr_Int : PSI<opc, MRMSrcReg, (outs VR128:$dst), (ins VR128:$src),
-                    !strconcat(OpcodeStr, "ps\t{$src, $dst|$dst, $src}"),
-                    [(set VR128:$dst, (V4F32Int VR128:$src))],
-                    itins.rr>, Sched<[itins.Sched]>;
-  def PSm_Int : PSI<opc, MRMSrcMem, (outs VR128:$dst), (ins f128mem:$src),
-                    !strconcat(OpcodeStr, "ps\t{$src, $dst|$dst, $src}"),
-                    [(set VR128:$dst, (V4F32Int (memopv4f32 addr:$src)))],
-                    itins.rm>, Sched<[itins.Sched.Folded]>;
-} // isCodeGenOnly = 1
-}
-
 /// sse2_fp_unop_p - SSE2 unops in vector forms.
 multiclass sse2_fp_unop_p<bits<8> opc, string OpcodeStr,
                           SDNode OpNode, OpndItins itins> {
@@ -3557,11 +3529,11 @@ multiclass sse1_fp_unop_s<bits<8> opc, string OpcodeStr, SDNode OpNode,
   defm SS        :  sse_fp_unop_s<opc, OpcodeStr##ss, FR32, v4f32, f32, f32mem,
                       ssmem, sse_load_f32,
                       !cast<Intrinsic>("int_x86_sse_"##OpcodeStr##_ss), OpNode,
-                      itins, UseSSE1, "SS">, XS;
+                      SSEPackedSingle, itins, UseSSE1, "SS">, XS;
   defm V#NAME#SS  : avx_fp_unop_s<opc, "v"#OpcodeStr##ss, FR32, v4f32, f32,
                       f32mem, ssmem, sse_load_f32,
                       !cast<Intrinsic>("int_x86_sse_"##OpcodeStr##_ss), OpNode,
-                      itins, UseAVX, "SS">, XS, VEX_4V, VEX_LIG;
+                      SSEPackedSingle, itins, "SS">, XS, VEX_4V, VEX_LIG;
 }
 
 multiclass sse2_fp_unop_s<bits<8> opc, string OpcodeStr, SDNode OpNode,
@@ -3569,11 +3541,12 @@ multiclass sse2_fp_unop_s<bits<8> opc, string OpcodeStr, SDNode OpNode,
   defm SD         : sse_fp_unop_s<opc, OpcodeStr##sd, FR64, v2f64, f64, f64mem,
                          sdmem, sse_load_f64,
                          !cast<Intrinsic>("int_x86_sse2_"##OpcodeStr##_sd),
-                         OpNode, itins, UseSSE2, "SD">, XD;
+                         OpNode, SSEPackedDouble, itins, UseSSE2, "SD">, XD;
   defm V#NAME#SD  : avx_fp_unop_s<opc, "v"#OpcodeStr##sd, FR64, v2f64, f64,
                          f64mem, sdmem, sse_load_f64,
                          !cast<Intrinsic>("int_x86_sse2_"##OpcodeStr##_sd),
-                         OpNode, itins, UseAVX, "SD">, XD, VEX_4V, VEX_LIG;
+                         OpNode, SSEPackedDouble, itins, "SD">,
+                         XD, VEX_4V, VEX_LIG;
 }
 
 // Square root.
@@ -3585,16 +3558,50 @@ defm SQRT  : sse1_fp_unop_s<0x51, "sqrt", fsqrt, SSE_SQRTSS>,
 // Reciprocal approximations. Note that these typically require refinement
 // in order to obtain suitable precision.
 defm RSQRT : sse1_fp_unop_s<0x52, "rsqrt", X86frsqrt, SSE_RSQRTSS>,
-             sse1_fp_unop_p<0x52, "rsqrt", X86frsqrt, SSE_RSQRTPS>,
-             sse1_fp_unop_p_int<0x52, "rsqrt", int_x86_sse_rsqrt_ps,
-                                int_x86_avx_rsqrt_ps_256, SSE_RSQRTPS>;
+             sse1_fp_unop_p<0x52, "rsqrt", X86frsqrt, SSE_RSQRTPS>;
 defm RCP   : sse1_fp_unop_s<0x53, "rcp", X86frcp, SSE_RCPS>,
-             sse1_fp_unop_p<0x53, "rcp", X86frcp, SSE_RCPP>,
-             sse1_fp_unop_p_int<0x53, "rcp", int_x86_sse_rcp_ps,
-                                int_x86_avx_rcp_ps_256, SSE_RCPP>;
+             sse1_fp_unop_p<0x53, "rcp", X86frcp, SSE_RCPP>;
 
 // There is no f64 version of the reciprocal approximation instructions.
 
+// TODO: We should add *scalar* op patterns for these just like we have for
+// the binops above. If the binop and unop patterns could all be unified
+// that would be even better.
+
+multiclass scalar_unary_math_patterns<Intrinsic Intr, string OpcPrefix,
+                                      SDNode Move, ValueType VT,
+                                      Predicate BasePredicate> {
+  let Predicates = [BasePredicate] in {
+    def : Pat<(VT (Move VT:$dst, (Intr VT:$src))),
+              (!cast<I>(OpcPrefix#r_Int) VT:$dst, VT:$src)>;
+  }
+
+  // With SSE 4.1, blendi is preferred to movs*, so match that too.
+  let Predicates = [UseSSE41] in {
+    def : Pat<(VT (X86Blendi VT:$dst, (Intr VT:$src), (i8 1))),
+              (!cast<I>(OpcPrefix#r_Int) VT:$dst, VT:$src)>;
+  }
+
+  // Repeat for AVX versions of the instructions.
+  let Predicates = [HasAVX] in {
+    def : Pat<(VT (Move VT:$dst, (Intr VT:$src))),
+              (!cast<I>("V"#OpcPrefix#r_Int) VT:$dst, VT:$src)>;
+
+    def : Pat<(VT (X86Blendi VT:$dst, (Intr VT:$src), (i8 1))),
+              (!cast<I>("V"#OpcPrefix#r_Int) VT:$dst, VT:$src)>;
+  }
+}
+
+defm : scalar_unary_math_patterns<int_x86_sse_rcp_ss, "RCPSS", X86Movss,
+                                  v4f32, UseSSE1>;
+defm : scalar_unary_math_patterns<int_x86_sse_rsqrt_ss, "RSQRTSS", X86Movss,
+                                  v4f32, UseSSE1>;
+defm : scalar_unary_math_patterns<int_x86_sse_sqrt_ss, "SQRTSS", X86Movss,
+                                  v4f32, UseSSE1>;
+defm : scalar_unary_math_patterns<int_x86_sse2_sqrt_sd, "SQRTSD", X86Movsd,
+                                  v2f64, UseSSE2>;
+
+
 //===----------------------------------------------------------------------===//
 // SSE 1 & 2 - Non-temporal stores
 //===----------------------------------------------------------------------===//
@@ -4016,39 +4023,39 @@ multiclass PDI_binop_rm2<bits<8> opc, string OpcodeStr, SDNode OpNode,
 } // ExeDomain = SSEPackedInt
 
 defm PADDB   : PDI_binop_all<0xFC, "paddb", add, v16i8, v32i8,
-                             SSE_INTALU_ITINS_P, 1>;
+                             SSE_INTALU_ITINS_P, 1, NoVLX_Or_NoBWI>;
 defm PADDW   : PDI_binop_all<0xFD, "paddw", add, v8i16, v16i16,
-                             SSE_INTALU_ITINS_P, 1>;
+                             SSE_INTALU_ITINS_P, 1, NoVLX_Or_NoBWI>;
 defm PADDD   : PDI_binop_all<0xFE, "paddd", add, v4i32, v8i32,
-                             SSE_INTALU_ITINS_P, 1>;
+                             SSE_INTALU_ITINS_P, 1, NoVLX>;
 defm PADDQ   : PDI_binop_all<0xD4, "paddq", add, v2i64, v4i64,
-                             SSE_INTALUQ_ITINS_P, 1>;
+                             SSE_INTALUQ_ITINS_P, 1, NoVLX>;
 defm PMULLW  : PDI_binop_all<0xD5, "pmullw", mul, v8i16, v16i16,
-                             SSE_INTMUL_ITINS_P, 1>;
+                             SSE_INTMUL_ITINS_P, 1, NoVLX_Or_NoBWI>;
 defm PMULHUW : PDI_binop_all<0xE4, "pmulhuw", mulhu, v8i16, v16i16,
-                             SSE_INTMUL_ITINS_P, 1>;
+                             SSE_INTMUL_ITINS_P, 1, NoVLX_Or_NoBWI>;
 defm PMULHW  : PDI_binop_all<0xE5, "pmulhw", mulhs, v8i16, v16i16,
-                             SSE_INTMUL_ITINS_P, 1>;
+                             SSE_INTMUL_ITINS_P, 1, NoVLX_Or_NoBWI>;
 defm PSUBB   : PDI_binop_all<0xF8, "psubb", sub, v16i8, v32i8,
-                             SSE_INTALU_ITINS_P, 0>;
+                             SSE_INTALU_ITINS_P, 0, NoVLX_Or_NoBWI>;
 defm PSUBW   : PDI_binop_all<0xF9, "psubw", sub, v8i16, v16i16,
-                             SSE_INTALU_ITINS_P, 0>;
+                             SSE_INTALU_ITINS_P, 0, NoVLX_Or_NoBWI>;
 defm PSUBD   : PDI_binop_all<0xFA, "psubd", sub, v4i32, v8i32,
-                             SSE_INTALU_ITINS_P, 0>;
+                             SSE_INTALU_ITINS_P, 0, NoVLX>;
 defm PSUBQ   : PDI_binop_all<0xFB, "psubq", sub, v2i64, v4i64,
-                             SSE_INTALUQ_ITINS_P, 0>;
+                             SSE_INTALUQ_ITINS_P, 0, NoVLX>;
 defm PSUBUSB : PDI_binop_all<0xD8, "psubusb", X86subus, v16i8, v32i8,
-                             SSE_INTALU_ITINS_P, 0>;
+                             SSE_INTALU_ITINS_P, 0, NoVLX_Or_NoBWI>;
 defm PSUBUSW : PDI_binop_all<0xD9, "psubusw", X86subus, v8i16, v16i16,
-                             SSE_INTALU_ITINS_P, 0>;
-defm PMINUB  : PDI_binop_all<0xDA, "pminub", X86umin, v16i8, v32i8,
-                             SSE_INTALU_ITINS_P, 1>;
-defm PMINSW  : PDI_binop_all<0xEA, "pminsw", X86smin, v8i16, v16i16,
-                             SSE_INTALU_ITINS_P, 1>;
-defm PMAXUB  : PDI_binop_all<0xDE, "pmaxub", X86umax, v16i8, v32i8,
-                             SSE_INTALU_ITINS_P, 1>;
-defm PMAXSW  : PDI_binop_all<0xEE, "pmaxsw", X86smax, v8i16, v16i16,
-                             SSE_INTALU_ITINS_P, 1>;
+                             SSE_INTALU_ITINS_P, 0, NoVLX_Or_NoBWI>;
+defm PMINUB  : PDI_binop_all<0xDA, "pminub", umin, v16i8, v32i8,
+                             SSE_INTALU_ITINS_P, 1, NoVLX_Or_NoBWI>;
+defm PMINSW  : PDI_binop_all<0xEA, "pminsw", smin, v8i16, v16i16,
+                             SSE_INTALU_ITINS_P, 1, NoVLX_Or_NoBWI>;
+defm PMAXUB  : PDI_binop_all<0xDE, "pmaxub", umax, v16i8, v32i8,
+                             SSE_INTALU_ITINS_P, 1, NoVLX_Or_NoBWI>;
+defm PMAXSW  : PDI_binop_all<0xEE, "pmaxsw", smax, v8i16, v16i16,
+                             SSE_INTALU_ITINS_P, 1, NoVLX_Or_NoBWI>;
 
 // Intrinsic forms
 defm PSUBSB  : PDI_binop_all_int<0xE8, "psubsb", int_x86_sse2_psubs_b,
@@ -4072,6 +4079,20 @@ defm PAVGW   : PDI_binop_all_int<0xE3, "pavgw", int_x86_sse2_pavg_w,
 defm PSADBW  : PDI_binop_all_int<0xF6, "psadbw", int_x86_sse2_psad_bw,
                                  int_x86_avx2_psad_bw, SSE_PMADD, 1>;
 
+let Predicates = [HasAVX2] in
+  def : Pat<(v32i8 (X86psadbw (v32i8 VR256:$src1),
+                              (v32i8 VR256:$src2))),
+            (VPSADBWYrr VR256:$src2, VR256:$src1)>;
+
+let Predicates = [HasAVX] in
+  def : Pat<(v16i8 (X86psadbw (v16i8 VR128:$src1),
+                              (v16i8 VR128:$src2))),
+            (VPSADBWrr VR128:$src2, VR128:$src1)>;
+
+def : Pat<(v16i8 (X86psadbw (v16i8 VR128:$src1),
+                            (v16i8 VR128:$src2))),
+          (PSADBWrr VR128:$src2, VR128:$src1)>;
+
 let Predicates = [HasAVX] in
 defm VPMULUDQ : PDI_binop_rm2<0xF4, "vpmuludq", X86pmuludq, v2i64, v4i32, VR128,
                               loadv2i64, i128mem, SSE_INTMUL_ITINS_P, 1, 0>,
@@ -4226,32 +4247,22 @@ let ExeDomain = SSEPackedInt, SchedRW = [WriteVecShift], hasSideEffects = 0 in {
 }
 } // Constraints = "$src1 = $dst"
 
-let Predicates = [HasAVX] in {
-  def : Pat<(v2f64 (X86fsrl VR128:$src1, i32immSExt8:$src2)),
-            (VPSRLDQri VR128:$src1, (BYTE_imm imm:$src2))>;
-}
-
-let Predicates = [UseSSE2] in {
-  def : Pat<(v2f64 (X86fsrl VR128:$src1, i32immSExt8:$src2)),
-            (PSRLDQri VR128:$src1, (BYTE_imm imm:$src2))>;
-}
-
 //===---------------------------------------------------------------------===//
 // SSE2 - Packed Integer Comparison Instructions
 //===---------------------------------------------------------------------===//
 
 defm PCMPEQB : PDI_binop_all<0x74, "pcmpeqb", X86pcmpeq, v16i8, v32i8,
-                             SSE_INTALU_ITINS_P, 1>;
+                             SSE_INTALU_ITINS_P, 1, NoVLX_Or_NoBWI>;
 defm PCMPEQW : PDI_binop_all<0x75, "pcmpeqw", X86pcmpeq, v8i16, v16i16,
-                             SSE_INTALU_ITINS_P, 1>;
+                             SSE_INTALU_ITINS_P, 1, NoVLX_Or_NoBWI>;
 defm PCMPEQD : PDI_binop_all<0x76, "pcmpeqd", X86pcmpeq, v4i32, v8i32,
-                             SSE_INTALU_ITINS_P, 1>;
+                             SSE_INTALU_ITINS_P, 1, NoVLX>;
 defm PCMPGTB : PDI_binop_all<0x64, "pcmpgtb", X86pcmpgt, v16i8, v32i8,
-                             SSE_INTALU_ITINS_P, 0>;
+                             SSE_INTALU_ITINS_P, 0, NoVLX_Or_NoBWI>;
 defm PCMPGTW : PDI_binop_all<0x65, "pcmpgtw", X86pcmpgt, v8i16, v16i16,
-                             SSE_INTALU_ITINS_P, 0>;
+                             SSE_INTALU_ITINS_P, 0, NoVLX_Or_NoBWI>;
 defm PCMPGTD : PDI_binop_all<0x66, "pcmpgtd", X86pcmpgt, v4i32, v8i32,
-                             SSE_INTALU_ITINS_P, 0>;
+                             SSE_INTALU_ITINS_P, 0, NoVLX>;
 
 //===---------------------------------------------------------------------===//
 // SSE2 - Packed Integer Shuffle Instructions
@@ -4505,40 +4516,43 @@ multiclass sse2_unpack_y<bits<8> opc, string OpcodeStr, ValueType vt,
       Sched<[WriteShuffleLd, ReadAfterLd]>;
 }
 
-let Predicates = [HasAVX] in {
+
+let Predicates = [HasAVX, NoVLX_Or_NoBWI] in {
   defm VPUNPCKLBW  : sse2_unpack<0x60, "vpunpcklbw", v16i8, X86Unpckl,
                                  bc_v16i8, loadv2i64, 0>, VEX_4V;
   defm VPUNPCKLWD  : sse2_unpack<0x61, "vpunpcklwd", v8i16, X86Unpckl,
                                  bc_v8i16, loadv2i64, 0>, VEX_4V;
-  defm VPUNPCKLDQ  : sse2_unpack<0x62, "vpunpckldq", v4i32, X86Unpckl,
-                                 bc_v4i32, loadv2i64, 0>, VEX_4V;
-  defm VPUNPCKLQDQ : sse2_unpack<0x6C, "vpunpcklqdq", v2i64, X86Unpckl,
-                                 bc_v2i64, loadv2i64, 0>, VEX_4V;
-
   defm VPUNPCKHBW  : sse2_unpack<0x68, "vpunpckhbw", v16i8, X86Unpckh,
                                  bc_v16i8, loadv2i64, 0>, VEX_4V;
   defm VPUNPCKHWD  : sse2_unpack<0x69, "vpunpckhwd", v8i16, X86Unpckh,
                                  bc_v8i16, loadv2i64, 0>, VEX_4V;
+}
+let Predicates = [HasAVX, NoVLX] in {
+  defm VPUNPCKLDQ  : sse2_unpack<0x62, "vpunpckldq", v4i32, X86Unpckl,
+                                 bc_v4i32, loadv2i64, 0>, VEX_4V;
+  defm VPUNPCKLQDQ : sse2_unpack<0x6C, "vpunpcklqdq", v2i64, X86Unpckl,
+                                 bc_v2i64, loadv2i64, 0>, VEX_4V;
   defm VPUNPCKHDQ  : sse2_unpack<0x6A, "vpunpckhdq", v4i32, X86Unpckh,
                                  bc_v4i32, loadv2i64, 0>, VEX_4V;
   defm VPUNPCKHQDQ : sse2_unpack<0x6D, "vpunpckhqdq", v2i64, X86Unpckh,
                                  bc_v2i64, loadv2i64, 0>, VEX_4V;
 }
 
-let Predicates = [HasAVX2] in {
+let Predicates = [HasAVX2, NoVLX_Or_NoBWI] in {
   defm VPUNPCKLBW  : sse2_unpack_y<0x60, "vpunpcklbw", v32i8, X86Unpckl,
                                    bc_v32i8>, VEX_4V, VEX_L;
   defm VPUNPCKLWD  : sse2_unpack_y<0x61, "vpunpcklwd", v16i16, X86Unpckl,
                                    bc_v16i16>, VEX_4V, VEX_L;
-  defm VPUNPCKLDQ  : sse2_unpack_y<0x62, "vpunpckldq", v8i32, X86Unpckl,
-                                   bc_v8i32>, VEX_4V, VEX_L;
-  defm VPUNPCKLQDQ : sse2_unpack_y<0x6C, "vpunpcklqdq", v4i64, X86Unpckl,
-                                   bc_v4i64>, VEX_4V, VEX_L;
-
   defm VPUNPCKHBW  : sse2_unpack_y<0x68, "vpunpckhbw", v32i8, X86Unpckh,
                                    bc_v32i8>, VEX_4V, VEX_L;
   defm VPUNPCKHWD  : sse2_unpack_y<0x69, "vpunpckhwd", v16i16, X86Unpckh,
                                    bc_v16i16>, VEX_4V, VEX_L;
+}
+let Predicates = [HasAVX2, NoVLX] in {
+  defm VPUNPCKLDQ  : sse2_unpack_y<0x62, "vpunpckldq", v8i32, X86Unpckl,
+                                   bc_v8i32>, VEX_4V, VEX_L;
+  defm VPUNPCKLQDQ : sse2_unpack_y<0x6C, "vpunpcklqdq", v4i64, X86Unpckl,
+                                   bc_v4i64>, VEX_4V, VEX_L;
   defm VPUNPCKHDQ  : sse2_unpack_y<0x6A, "vpunpckhdq", v8i32, X86Unpckh,
                                    bc_v8i32>, VEX_4V, VEX_L;
   defm VPUNPCKHQDQ : sse2_unpack_y<0x6D, "vpunpckhqdq", v4i64, X86Unpckh,
@@ -5001,7 +5015,7 @@ def MOVPQI2QIrr : S2I<0xD6, MRMDestReg, (outs VR128:$dst), (ins VR128:$src),
 //===---------------------------------------------------------------------===//
 // Store / copy lower 64-bits of a XMM register.
 //
-let Predicates = [UseAVX] in
+let Predicates = [HasAVX] in
 def : Pat<(int_x86_sse2_storel_dq addr:$dst, VR128:$src),
           (VMOVPQI2QImr addr:$dst, VR128:$src)>;
 let Predicates = [UseSSE2] in
@@ -5220,9 +5234,9 @@ let Predicates = [HasAVX] in {
 
 let Predicates = [UseAVX, OptForSize] in {
   def : Pat<(v2f64 (X86VBroadcast (loadf64 addr:$src))),
-  (VMOVDDUPrm addr:$src)>;
+            (VMOVDDUPrm addr:$src)>;
   def : Pat<(v2i64 (X86VBroadcast (loadi64 addr:$src))),
-  (VMOVDDUPrm addr:$src)>;
+            (VMOVDDUPrm addr:$src)>;
 }
 
 let Predicates = [UseSSE3] in {
@@ -5869,10 +5883,10 @@ multiclass SS41I_pmovx_rm_all<bits<8> opc, string OpcodeStr,
                           OpndItins SSEItins, OpndItins AVXItins,
                           OpndItins AVX2Itins> {
   defm NAME : SS41I_pmovx_rrrm<opc, OpcodeStr, MemOp, VR128, VR128, SSEItins>;
-  let Predicates = [HasAVX] in
+  let Predicates = [HasAVX, NoVLX] in
     defm V#NAME   : SS41I_pmovx_rrrm<opc, !strconcat("v", OpcodeStr), MemOp,
                                      VR128, VR128, AVXItins>, VEX;
-  let Predicates = [HasAVX2] in
+  let Predicates = [HasAVX2, NoVLX] in
     defm V#NAME#Y : SS41I_pmovx_rrrm<opc, !strconcat("v", OpcodeStr), MemYOp,
                                      VR256, VR128, AVX2Itins>, VEX, VEX_L;
 }
@@ -6007,7 +6021,7 @@ multiclass SS41I_pmovx_avx2_patterns<string OpcPrefix, string ExtTy, SDNode ExtO
             (!cast<I>(OpcPrefix#DQYrm) addr:$src)>;
 }
 
-let Predicates = [HasAVX2] in {
+let Predicates = [HasAVX2, NoVLX] in {
   defm : SS41I_pmovx_avx2_patterns<"VPMOVSX", "s", X86vsext>;
   defm : SS41I_pmovx_avx2_patterns<"VPMOVZX", "z", X86vzext>;
 }
@@ -6106,7 +6120,7 @@ multiclass SS41I_pmovx_patterns<string OpcPrefix, string ExtTy,
             (!cast<I>(OpcPrefix#DQrm) addr:$src)>;
 }
 
-let Predicates = [HasAVX] in {
+let Predicates = [HasAVX, NoVLX] in {
   defm : SS41I_pmovx_patterns<"VPMOVSX", "s", X86vsext, extloadi32i16>;
   defm : SS41I_pmovx_patterns<"VPMOVZX", "z", X86vzext, loadi16_anyext>;
 }
@@ -6836,29 +6850,28 @@ multiclass SS48I_binop_rm2<bits<8> opc, string OpcodeStr, SDNode OpNode,
 }
 
 let Predicates = [HasAVX, NoVLX] in {
-  let isCommutable = 0 in
-  defm VPMINSB   : SS48I_binop_rm<0x38, "vpminsb", X86smin, v16i8, VR128,
+  defm VPMINSB   : SS48I_binop_rm<0x38, "vpminsb", smin, v16i8, VR128,
                                   loadv2i64, i128mem, 0, SSE_INTALU_ITINS_P>,
                                   VEX_4V;
-  defm VPMINSD   : SS48I_binop_rm<0x39, "vpminsd", X86smin, v4i32, VR128,
+  defm VPMINSD   : SS48I_binop_rm<0x39, "vpminsd", smin, v4i32, VR128,
                                   loadv2i64, i128mem, 0, SSE_INTALU_ITINS_P>,
                                   VEX_4V;
-  defm VPMINUD   : SS48I_binop_rm<0x3B, "vpminud", X86umin, v4i32, VR128,
+  defm VPMINUD   : SS48I_binop_rm<0x3B, "vpminud", umin, v4i32, VR128,
                                   loadv2i64, i128mem, 0, SSE_INTALU_ITINS_P>,
                                   VEX_4V;
-  defm VPMINUW   : SS48I_binop_rm<0x3A, "vpminuw", X86umin, v8i16, VR128,
+  defm VPMINUW   : SS48I_binop_rm<0x3A, "vpminuw", umin, v8i16, VR128,
                                   loadv2i64, i128mem, 0, SSE_INTALU_ITINS_P>,
                                   VEX_4V;
-  defm VPMAXSB   : SS48I_binop_rm<0x3C, "vpmaxsb", X86smax, v16i8, VR128,
+  defm VPMAXSB   : SS48I_binop_rm<0x3C, "vpmaxsb", smax, v16i8, VR128,
                                   loadv2i64, i128mem, 0, SSE_INTALU_ITINS_P>,
                                   VEX_4V;
-  defm VPMAXSD   : SS48I_binop_rm<0x3D, "vpmaxsd", X86smax, v4i32, VR128,
+  defm VPMAXSD   : SS48I_binop_rm<0x3D, "vpmaxsd", smax, v4i32, VR128,
                                   loadv2i64, i128mem, 0, SSE_INTALU_ITINS_P>,
                                   VEX_4V;
-  defm VPMAXUD   : SS48I_binop_rm<0x3F, "vpmaxud", X86umax, v4i32, VR128,
+  defm VPMAXUD   : SS48I_binop_rm<0x3F, "vpmaxud", umax, v4i32, VR128,
                                   loadv2i64, i128mem, 0, SSE_INTALU_ITINS_P>,
                                   VEX_4V;
-  defm VPMAXUW   : SS48I_binop_rm<0x3E, "vpmaxuw", X86umax, v8i16, VR128,
+  defm VPMAXUW   : SS48I_binop_rm<0x3E, "vpmaxuw", umax, v8i16, VR128,
                                   loadv2i64, i128mem, 0, SSE_INTALU_ITINS_P>,
                                   VEX_4V;
   defm VPMULDQ   : SS48I_binop_rm2<0x28, "vpmuldq", X86pmuldq, v2i64, v4i32,
@@ -6867,29 +6880,28 @@ let Predicates = [HasAVX, NoVLX] in {
 }
 
 let Predicates = [HasAVX2, NoVLX] in {
-  let isCommutable = 0 in
-  defm VPMINSBY  : SS48I_binop_rm<0x38, "vpminsb", X86smin, v32i8, VR256,
+  defm VPMINSBY  : SS48I_binop_rm<0x38, "vpminsb", smin, v32i8, VR256,
                                   loadv4i64, i256mem, 0, SSE_INTALU_ITINS_P>,
                                   VEX_4V, VEX_L;
-  defm VPMINSDY  : SS48I_binop_rm<0x39, "vpminsd", X86smin, v8i32, VR256,
+  defm VPMINSDY  : SS48I_binop_rm<0x39, "vpminsd", smin, v8i32, VR256,
                                   loadv4i64, i256mem, 0, SSE_INTALU_ITINS_P>,
                                   VEX_4V, VEX_L;
-  defm VPMINUDY  : SS48I_binop_rm<0x3B, "vpminud", X86umin, v8i32, VR256,
+  defm VPMINUDY  : SS48I_binop_rm<0x3B, "vpminud", umin, v8i32, VR256,
                                   loadv4i64, i256mem, 0, SSE_INTALU_ITINS_P>,
                                   VEX_4V, VEX_L;
-  defm VPMINUWY  : SS48I_binop_rm<0x3A, "vpminuw", X86umin, v16i16, VR256,
+  defm VPMINUWY  : SS48I_binop_rm<0x3A, "vpminuw", umin, v16i16, VR256,
                                   loadv4i64, i256mem, 0, SSE_INTALU_ITINS_P>,
                                   VEX_4V, VEX_L;
-  defm VPMAXSBY  : SS48I_binop_rm<0x3C, "vpmaxsb", X86smax, v32i8, VR256,
+  defm VPMAXSBY  : SS48I_binop_rm<0x3C, "vpmaxsb", smax, v32i8, VR256,
                                   loadv4i64, i256mem, 0, SSE_INTALU_ITINS_P>,
                                   VEX_4V, VEX_L;
-  defm VPMAXSDY  : SS48I_binop_rm<0x3D, "vpmaxsd", X86smax, v8i32, VR256,
+  defm VPMAXSDY  : SS48I_binop_rm<0x3D, "vpmaxsd", smax, v8i32, VR256,
                                   loadv4i64, i256mem, 0, SSE_INTALU_ITINS_P>,
                                   VEX_4V, VEX_L;
-  defm VPMAXUDY  : SS48I_binop_rm<0x3F, "vpmaxud", X86umax, v8i32, VR256,
+  defm VPMAXUDY  : SS48I_binop_rm<0x3F, "vpmaxud", umax, v8i32, VR256,
                                   loadv4i64, i256mem, 0, SSE_INTALU_ITINS_P>,
                                   VEX_4V, VEX_L;
-  defm VPMAXUWY  : SS48I_binop_rm<0x3E, "vpmaxuw", X86umax, v16i16, VR256,
+  defm VPMAXUWY  : SS48I_binop_rm<0x3E, "vpmaxuw", umax, v16i16, VR256,
                                   loadv4i64, i256mem, 0, SSE_INTALU_ITINS_P>,
                                   VEX_4V, VEX_L;
   defm VPMULDQY : SS48I_binop_rm2<0x28, "vpmuldq", X86pmuldq, v4i64, v8i32,
@@ -6898,22 +6910,21 @@ let Predicates = [HasAVX2, NoVLX] in {
 }
 
 let Constraints = "$src1 = $dst" in {
-  let isCommutable = 0 in
-  defm PMINSB   : SS48I_binop_rm<0x38, "pminsb", X86smin, v16i8, VR128,
+  defm PMINSB   : SS48I_binop_rm<0x38, "pminsb", smin, v16i8, VR128,
                                  memopv2i64, i128mem, 1, SSE_INTALU_ITINS_P>;
-  defm PMINSD   : SS48I_binop_rm<0x39, "pminsd", X86smin, v4i32, VR128,
+  defm PMINSD   : SS48I_binop_rm<0x39, "pminsd", smin, v4i32, VR128,
                                  memopv2i64, i128mem, 1, SSE_INTALU_ITINS_P>;
-  defm PMINUD   : SS48I_binop_rm<0x3B, "pminud", X86umin, v4i32, VR128,
+  defm PMINUD   : SS48I_binop_rm<0x3B, "pminud", umin, v4i32, VR128,
                                  memopv2i64, i128mem, 1, SSE_INTALU_ITINS_P>;
-  defm PMINUW   : SS48I_binop_rm<0x3A, "pminuw", X86umin, v8i16, VR128,
+  defm PMINUW   : SS48I_binop_rm<0x3A, "pminuw", umin, v8i16, VR128,
                                  memopv2i64, i128mem, 1, SSE_INTALU_ITINS_P>;
-  defm PMAXSB   : SS48I_binop_rm<0x3C, "pmaxsb", X86smax, v16i8, VR128,
+  defm PMAXSB   : SS48I_binop_rm<0x3C, "pmaxsb", smax, v16i8, VR128,
                                  memopv2i64, i128mem, 1, SSE_INTALU_ITINS_P>;
-  defm PMAXSD   : SS48I_binop_rm<0x3D, "pmaxsd", X86smax, v4i32, VR128,
+  defm PMAXSD   : SS48I_binop_rm<0x3D, "pmaxsd", smax, v4i32, VR128,
                                  memopv2i64, i128mem, 1, SSE_INTALU_ITINS_P>;
-  defm PMAXUD   : SS48I_binop_rm<0x3F, "pmaxud", X86umax, v4i32, VR128,
+  defm PMAXUD   : SS48I_binop_rm<0x3F, "pmaxud", umax, v4i32, VR128,
                                  memopv2i64, i128mem, 1, SSE_INTALU_ITINS_P>;
-  defm PMAXUW   : SS48I_binop_rm<0x3E, "pmaxuw", X86umax, v8i16, VR128,
+  defm PMAXUW   : SS48I_binop_rm<0x3E, "pmaxuw", umax, v8i16, VR128,
                                  memopv2i64, i128mem, 1, SSE_INTALU_ITINS_P>;
   defm PMULDQ   : SS48I_binop_rm2<0x28, "pmuldq", X86pmuldq, v2i64, v4i32,
                                   VR128, memopv2i64, i128mem,
@@ -7775,7 +7786,7 @@ let Constraints = "$src = $dst" in {
 def EXTRQI : Ii8<0x78, MRMXr, (outs VR128:$dst),
                  (ins VR128:$src, u8imm:$len, u8imm:$idx),
                  "extrq\t{$idx, $len, $src|$src, $len, $idx}",
-                 [(set VR128:$dst, (int_x86_sse4a_extrqi VR128:$src, imm:$len,
+                 [(set VR128:$dst, (X86extrqi VR128:$src, imm:$len,
                                     imm:$idx))]>, PD;
 def EXTRQ  : I<0x79, MRMSrcReg, (outs VR128:$dst),
               (ins VR128:$src, VR128:$mask),
@@ -7786,8 +7797,8 @@ def EXTRQ  : I<0x79, MRMSrcReg, (outs VR128:$dst),
 def INSERTQI : Ii8<0x78, MRMSrcReg, (outs VR128:$dst),
                    (ins VR128:$src, VR128:$src2, u8imm:$len, u8imm:$idx),
                    "insertq\t{$idx, $len, $src2, $src|$src, $src2, $len, $idx}",
-                   [(set VR128:$dst, (int_x86_sse4a_insertqi VR128:$src,
-                                      VR128:$src2, imm:$len, imm:$idx))]>, XD;
+                   [(set VR128:$dst, (X86insertqi VR128:$src, VR128:$src2,
+                                      imm:$len, imm:$idx))]>, XD;
 def INSERTQ  : I<0x79, MRMSrcReg, (outs VR128:$dst),
                  (ins VR128:$src, VR128:$mask),
                  "insertq\t{$mask, $src|$src, $mask}",
@@ -7862,10 +7873,11 @@ def VBROADCASTSDYrr  : avx2_broadcast_reg<0x19, "vbroadcastsd", VR256,
                                       int_x86_avx2_vbroadcast_sd_pd_256,
                                       WriteFShuffle256>, VEX_L;
 
-let Predicates = [HasAVX2] in
-def VBROADCASTI128 : avx_broadcast_no_int<0x5A, "vbroadcasti128", VR256,
-                                          i128mem, v4i64, loadv2i64,
-                                          WriteLoad>, VEX_L;
+let mayLoad = 1, Predicates = [HasAVX2] in
+def VBROADCASTI128 : AVX8I<0x5A, MRMSrcMem, (outs VR256:$dst),
+                           (ins i128mem:$src),
+                           "vbroadcasti128\t{$src, $dst|$dst, $src}", []>,
+                           Sched<[WriteLoad]>, VEX, VEX_L;
 
 let Predicates = [HasAVX] in
 def : Pat<(int_x86_avx_vbroadcastf128_ps_256 addr:$src),
@@ -8240,6 +8252,18 @@ let Predicates = [HasF16C] in {
             (VCVTPH2PSrm addr:$src)>;
   def : Pat<(int_x86_vcvtph2ps_128 (vzload_v2i64 addr:$src)),
             (VCVTPH2PSrm addr:$src)>;
+
+  def : Pat<(store (f64 (vector_extract (bc_v2f64 (v8i16
+                  (int_x86_vcvtps2ph_128 VR128:$src1, i32:$src2))), (iPTR 0))),
+                   addr:$dst),
+                   (VCVTPS2PHmr addr:$dst, VR128:$src1, imm:$src2)>;
+  def : Pat<(store (i64 (vector_extract (bc_v2i64 (v8i16
+                  (int_x86_vcvtps2ph_128 VR128:$src1, i32:$src2))), (iPTR 0))),
+                   addr:$dst),
+                   (VCVTPS2PHmr addr:$dst, VR128:$src1, imm:$src2)>;
+  def : Pat<(store (v8i16 (int_x86_vcvtps2ph_256 VR256:$src1, i32:$src2)),
+                   addr:$dst),
+                   (VCVTPS2PHYmr addr:$dst, VR256:$src1, imm:$src2)>;
 }
 
 // Patterns for  matching conversions from float to half-float and vice versa.
@@ -8475,6 +8499,8 @@ let Predicates = [HasAVX] in {
 
   def : Pat<(v2f64 (X86VBroadcast f64:$src)),
             (VMOVDDUPrr (COPY_TO_REGCLASS FR64:$src, VR128))>;
+  def : Pat<(v2i64 (X86VBroadcast i64:$src)),
+            (VMOVDDUPrr (COPY_TO_REGCLASS GR64:$src, VR128))>;
 }
 
 //===----------------------------------------------------------------------===//
@@ -8704,16 +8730,16 @@ defm VPMASKMOVQ : avx2_pmovmask<"vpmaskmovq",
                                 int_x86_avx2_maskstore_q,
                                 int_x86_avx2_maskstore_q_256>, VEX_W;
 
-def: Pat<(masked_store addr:$ptr, (v8i32 VR256:$mask), (v8f32 VR256:$src)),
+def: Pat<(X86mstore addr:$ptr, (v8i32 VR256:$mask), (v8f32 VR256:$src)),
          (VMASKMOVPSYmr addr:$ptr, VR256:$mask, VR256:$src)>;
 
-def: Pat<(masked_store addr:$ptr, (v8i32 VR256:$mask), (v8i32 VR256:$src)),
+def: Pat<(X86mstore addr:$ptr, (v8i32 VR256:$mask), (v8i32 VR256:$src)),
          (VPMASKMOVDYmr addr:$ptr, VR256:$mask, VR256:$src)>;
 
-def: Pat<(masked_store addr:$ptr, (v4i32 VR128:$mask), (v4f32 VR128:$src)),
+def: Pat<(X86mstore addr:$ptr, (v4i32 VR128:$mask), (v4f32 VR128:$src)),
          (VMASKMOVPSmr addr:$ptr, VR128:$mask, VR128:$src)>;
 
-def: Pat<(masked_store addr:$ptr, (v4i32 VR128:$mask), (v4i32 VR128:$src)),
+def: Pat<(X86mstore addr:$ptr, (v4i32 VR128:$mask), (v4i32 VR128:$src)),
          (VPMASKMOVDmr addr:$ptr, VR128:$mask, VR128:$src)>;
 
 def: Pat<(v8f32 (masked_load addr:$ptr, (v8i32 VR256:$mask), undef)),
@@ -8758,10 +8784,10 @@ def: Pat<(v4i32 (masked_load addr:$ptr, (v4i32 VR128:$mask), (v4i32 VR128:$src0)
          (VBLENDVPSrr VR128:$src0, (VPMASKMOVDrm VR128:$mask, addr:$ptr),
                        VR128:$mask)>;
 
-def: Pat<(masked_store addr:$ptr, (v4i64 VR256:$mask), (v4f64 VR256:$src)),
+def: Pat<(X86mstore addr:$ptr, (v4i64 VR256:$mask), (v4f64 VR256:$src)),
          (VMASKMOVPDYmr addr:$ptr, VR256:$mask, VR256:$src)>;
 
-def: Pat<(masked_store addr:$ptr, (v4i64 VR256:$mask), (v4i64 VR256:$src)),
+def: Pat<(X86mstore addr:$ptr, (v4i64 VR256:$mask), (v4i64 VR256:$src)),
          (VPMASKMOVQYmr addr:$ptr, VR256:$mask, VR256:$src)>;
 
 def: Pat<(v4f64 (masked_load addr:$ptr, (v4i64 VR256:$mask), undef)),
@@ -8786,10 +8812,10 @@ def: Pat<(v4i64 (masked_load addr:$ptr, (v4i64 VR256:$mask), (v4i64 VR256:$src0)
          (VBLENDVPDYrr VR256:$src0, (VPMASKMOVQYrm VR256:$mask, addr:$ptr),
                        VR256:$mask)>;
 
-def: Pat<(masked_store addr:$ptr, (v2i64 VR128:$mask), (v2f64 VR128:$src)),
+def: Pat<(X86mstore addr:$ptr, (v2i64 VR128:$mask), (v2f64 VR128:$src)),
          (VMASKMOVPDmr addr:$ptr, VR128:$mask, VR128:$src)>;
 
-def: Pat<(masked_store addr:$ptr, (v2i64 VR128:$mask), (v2i64 VR128:$src)),
+def: Pat<(X86mstore addr:$ptr, (v2i64 VR128:$mask), (v2i64 VR128:$src)),
          (VPMASKMOVQmr addr:$ptr, VR128:$mask, VR128:$src)>;
 
 def: Pat<(v2f64 (masked_load addr:$ptr, (v2i64 VR128:$mask), undef)),
diff --git a/lib/Target/X86/X86InstrShiftRotate.td b/lib/Target/X86/X86InstrShiftRotate.td
index c706d43c9f5..caecf7001ef 100644
--- a/lib/Target/X86/X86InstrShiftRotate.td
+++ b/lib/Target/X86/X86InstrShiftRotate.td
@@ -850,12 +850,12 @@ def SHRD64mri8 : RIi8<0xAC, MRMDestMem,
 
 def ROT32L2R_imm8  : SDNodeXForm<imm, [{
   // Convert a ROTL shamt to a ROTR shamt on 32-bit integer.
-  return getI8Imm(32 - N->getZExtValue());
+  return getI8Imm(32 - N->getZExtValue(), SDLoc(N));
 }]>;
 
 def ROT64L2R_imm8  : SDNodeXForm<imm, [{
   // Convert a ROTL shamt to a ROTR shamt on 64-bit integer.
-  return getI8Imm(64 - N->getZExtValue());
+  return getI8Imm(64 - N->getZExtValue(), SDLoc(N));
 }]>;
 
 multiclass bmi_rotate<string asm, RegisterClass RC, X86MemOperand x86memop> {
diff --git a/lib/Target/X86/X86IntrinsicsInfo.h b/lib/Target/X86/X86IntrinsicsInfo.h
index 28a3b7b11e0..c065f3d86c5 100644
--- a/lib/Target/X86/X86IntrinsicsInfo.h
+++ b/lib/Target/X86/X86IntrinsicsInfo.h
@@ -1,4 +1,4 @@
-//===-- X86IntinsicsInfo.h - X86 Instrinsics ------------*- C++ -*-===//
+//===-- X86IntrinsicsInfo.h - X86 Intrinsics ------------*- C++ -*-===//
 //
 //                     The LLVM Compiler Infrastructure
 //
@@ -19,10 +19,15 @@ namespace llvm {
 enum IntrinsicType {
   INTR_NO_TYPE,
   GATHER, SCATTER, PREFETCH, RDSEED, RDRAND, RDPMC, RDTSC, XTEST, ADX,
-  INTR_TYPE_1OP, INTR_TYPE_2OP, INTR_TYPE_3OP,
+  INTR_TYPE_1OP, INTR_TYPE_2OP, INTR_TYPE_3OP, INTR_TYPE_4OP,
   CMP_MASK, CMP_MASK_CC, VSHIFT, VSHIFT_MASK, COMI,
-  INTR_TYPE_1OP_MASK_RM, INTR_TYPE_2OP_MASK, FMA_OP_MASK, INTR_TYPE_SCALAR_MASK_RM,
-  COMPRESS_EXPAND_IN_REG, COMPRESS_TO_MEM, EXPAND_FROM_MEM, BLEND
+  INTR_TYPE_1OP_MASK, INTR_TYPE_1OP_MASK_RM,
+  INTR_TYPE_2OP_MASK, INTR_TYPE_2OP_MASK_RM,
+  INTR_TYPE_3OP_MASK, INTR_TYPE_3OP_MASK_RM, FMA_OP_MASK, FMA_OP_MASKZ, FMA_OP_MASK3, VPERM_3OP_MASK,
+  VPERM_3OP_MASKZ,
+  INTR_TYPE_SCALAR_MASK_RM, COMPRESS_EXPAND_IN_REG, COMPRESS_TO_MEM,
+  TRUNCATE_TO_MEM_VI8, TRUNCATE_TO_MEM_VI16, TRUNCATE_TO_MEM_VI32,
+  EXPAND_FROM_MEM, BLEND
 };
 
 struct IntrinsicData {
@@ -53,6 +58,22 @@ static const IntrinsicData IntrinsicsWithChain[] = {
   X86_INTRINSIC_DATA(addcarryx_u32, ADX, X86ISD::ADC, 0),
   X86_INTRINSIC_DATA(addcarryx_u64, ADX, X86ISD::ADC, 0),
 
+  X86_INTRINSIC_DATA(avx512_gather3div2_df, GATHER, X86::VGATHERQPDZ128rm, 0),
+  X86_INTRINSIC_DATA(avx512_gather3div2_di, GATHER, X86::VPGATHERQQZ128rm, 0),
+  X86_INTRINSIC_DATA(avx512_gather3div4_df, GATHER, X86::VGATHERQPDZ256rm, 0),
+  X86_INTRINSIC_DATA(avx512_gather3div4_di, GATHER, X86::VPGATHERQQZ256rm, 0),
+  X86_INTRINSIC_DATA(avx512_gather3div4_sf, GATHER, X86::VGATHERQPSZ128rm, 0),
+  X86_INTRINSIC_DATA(avx512_gather3div4_si, GATHER, X86::VPGATHERQDZ128rm, 0),
+  X86_INTRINSIC_DATA(avx512_gather3div8_sf, GATHER, X86::VGATHERQPSZ256rm, 0),
+  X86_INTRINSIC_DATA(avx512_gather3div8_si, GATHER, X86::VPGATHERQDZ256rm, 0),
+  X86_INTRINSIC_DATA(avx512_gather3siv2_df, GATHER, X86::VGATHERDPDZ128rm, 0),
+  X86_INTRINSIC_DATA(avx512_gather3siv2_di, GATHER, X86::VPGATHERDQZ128rm, 0),
+  X86_INTRINSIC_DATA(avx512_gather3siv4_df, GATHER, X86::VGATHERDPDZ256rm, 0),
+  X86_INTRINSIC_DATA(avx512_gather3siv4_di, GATHER, X86::VPGATHERDQZ256rm, 0),
+  X86_INTRINSIC_DATA(avx512_gather3siv4_sf, GATHER, X86::VGATHERDPSZ128rm, 0),
+  X86_INTRINSIC_DATA(avx512_gather3siv4_si, GATHER, X86::VPGATHERDDZ128rm, 0),
+  X86_INTRINSIC_DATA(avx512_gather3siv8_sf, GATHER, X86::VGATHERDPSZ256rm, 0),
+  X86_INTRINSIC_DATA(avx512_gather3siv8_si, GATHER, X86::VPGATHERDDZ256rm, 0),
   X86_INTRINSIC_DATA(avx512_gather_dpd_512, GATHER, X86::VGATHERDPDZrm, 0),
   X86_INTRINSIC_DATA(avx512_gather_dpi_512, GATHER, X86::VPGATHERDDZrm, 0),
   X86_INTRINSIC_DATA(avx512_gather_dpq_512, GATHER, X86::VPGATHERDQZrm, 0),
@@ -119,6 +140,42 @@ static const IntrinsicData IntrinsicsWithChain[] = {
                      EXPAND_FROM_MEM, X86ISD::EXPAND, 0),
   X86_INTRINSIC_DATA(avx512_mask_expand_load_q_512,
                      EXPAND_FROM_MEM, X86ISD::EXPAND, 0),
+  X86_INTRINSIC_DATA(avx512_mask_pmov_db_mem_128, TRUNCATE_TO_MEM_VI8,
+                     X86ISD::VTRUNC, 0),
+  X86_INTRINSIC_DATA(avx512_mask_pmov_db_mem_256, TRUNCATE_TO_MEM_VI8,
+                     X86ISD::VTRUNC, 0),
+  X86_INTRINSIC_DATA(avx512_mask_pmov_db_mem_512, TRUNCATE_TO_MEM_VI8,
+                     X86ISD::VTRUNC, 0),
+  X86_INTRINSIC_DATA(avx512_mask_pmov_dw_mem_128, TRUNCATE_TO_MEM_VI16,
+                     X86ISD::VTRUNC, 0),
+  X86_INTRINSIC_DATA(avx512_mask_pmov_dw_mem_256, TRUNCATE_TO_MEM_VI16,
+                     X86ISD::VTRUNC, 0),
+  X86_INTRINSIC_DATA(avx512_mask_pmov_dw_mem_512, TRUNCATE_TO_MEM_VI16,
+                     X86ISD::VTRUNC, 0),
+  X86_INTRINSIC_DATA(avx512_mask_pmov_qb_mem_128, TRUNCATE_TO_MEM_VI8,
+                     X86ISD::VTRUNC, 0),
+  X86_INTRINSIC_DATA(avx512_mask_pmov_qb_mem_256, TRUNCATE_TO_MEM_VI8,
+                     X86ISD::VTRUNC, 0),
+  X86_INTRINSIC_DATA(avx512_mask_pmov_qb_mem_512, TRUNCATE_TO_MEM_VI8,
+                     X86ISD::VTRUNC, 0),
+  X86_INTRINSIC_DATA(avx512_mask_pmov_qd_mem_128, TRUNCATE_TO_MEM_VI32,
+                     X86ISD::VTRUNC, 0),
+  X86_INTRINSIC_DATA(avx512_mask_pmov_qd_mem_256, TRUNCATE_TO_MEM_VI32,
+                     X86ISD::VTRUNC, 0),
+  X86_INTRINSIC_DATA(avx512_mask_pmov_qd_mem_512, TRUNCATE_TO_MEM_VI32,
+                     X86ISD::VTRUNC, 0),
+  X86_INTRINSIC_DATA(avx512_mask_pmov_qw_mem_128, TRUNCATE_TO_MEM_VI16,
+                     X86ISD::VTRUNC, 0),
+  X86_INTRINSIC_DATA(avx512_mask_pmov_qw_mem_256, TRUNCATE_TO_MEM_VI16,
+                     X86ISD::VTRUNC, 0),
+  X86_INTRINSIC_DATA(avx512_mask_pmov_qw_mem_512, TRUNCATE_TO_MEM_VI16,
+                     X86ISD::VTRUNC, 0),
+  X86_INTRINSIC_DATA(avx512_mask_pmov_wb_mem_128, TRUNCATE_TO_MEM_VI8,
+                     X86ISD::VTRUNC, 0),
+  X86_INTRINSIC_DATA(avx512_mask_pmov_wb_mem_256, TRUNCATE_TO_MEM_VI8,
+                     X86ISD::VTRUNC, 0),
+  X86_INTRINSIC_DATA(avx512_mask_pmov_wb_mem_512, TRUNCATE_TO_MEM_VI8,
+                     X86ISD::VTRUNC, 0),
   X86_INTRINSIC_DATA(avx512_scatter_dpd_512, SCATTER, X86::VSCATTERDPDZmr, 0),
   X86_INTRINSIC_DATA(avx512_scatter_dpi_512, SCATTER, X86::VPSCATTERDDZmr, 0),
   X86_INTRINSIC_DATA(avx512_scatter_dpq_512, SCATTER, X86::VPSCATTERDQZmr, 0),
@@ -127,15 +184,30 @@ static const IntrinsicData IntrinsicsWithChain[] = {
   X86_INTRINSIC_DATA(avx512_scatter_qpi_512, SCATTER, X86::VPSCATTERQDZmr, 0),
   X86_INTRINSIC_DATA(avx512_scatter_qpq_512, SCATTER, X86::VPSCATTERQQZmr, 0),
   X86_INTRINSIC_DATA(avx512_scatter_qps_512, SCATTER, X86::VSCATTERQPSZmr, 0),
-
-  X86_INTRINSIC_DATA(avx512_scatterpf_dpd_512, PREFETCH,
-                     X86::VSCATTERPF0DPDm, X86::VSCATTERPF1DPDm),
-  X86_INTRINSIC_DATA(avx512_scatterpf_dps_512, PREFETCH,
-                     X86::VSCATTERPF0DPSm, X86::VSCATTERPF1DPSm),
-  X86_INTRINSIC_DATA(avx512_scatterpf_qpd_512, PREFETCH,
-                     X86::VSCATTERPF0QPDm, X86::VSCATTERPF1QPDm),
-  X86_INTRINSIC_DATA(avx512_scatterpf_qps_512, PREFETCH,
-                     X86::VSCATTERPF0QPSm, X86::VSCATTERPF1QPSm),
+  X86_INTRINSIC_DATA(avx512_scatterdiv2_df, SCATTER, X86::VSCATTERQPDZ128mr, 0),
+  X86_INTRINSIC_DATA(avx512_scatterdiv2_di, SCATTER, X86::VPSCATTERQQZ128mr, 0),
+  X86_INTRINSIC_DATA(avx512_scatterdiv4_df, SCATTER, X86::VSCATTERQPDZ256mr, 0),
+  X86_INTRINSIC_DATA(avx512_scatterdiv4_di, SCATTER, X86::VPSCATTERQQZ256mr, 0),
+  X86_INTRINSIC_DATA(avx512_scatterdiv4_sf, SCATTER, X86::VSCATTERQPSZ128mr, 0),
+  X86_INTRINSIC_DATA(avx512_scatterdiv4_si, SCATTER, X86::VPSCATTERQDZ128mr, 0),
+  X86_INTRINSIC_DATA(avx512_scatterdiv8_sf, SCATTER, X86::VSCATTERQPSZ256mr, 0),
+  X86_INTRINSIC_DATA(avx512_scatterdiv8_si, SCATTER, X86::VPSCATTERQDZ256mr, 0),
+  X86_INTRINSIC_DATA(avx512_scatterpf_dpd_512, PREFETCH, X86::VSCATTERPF0DPDm,
+                     X86::VSCATTERPF1DPDm),
+  X86_INTRINSIC_DATA(avx512_scatterpf_dps_512, PREFETCH, X86::VSCATTERPF0DPSm,
+                     X86::VSCATTERPF1DPSm),
+  X86_INTRINSIC_DATA(avx512_scatterpf_qpd_512, PREFETCH, X86::VSCATTERPF0QPDm,
+                     X86::VSCATTERPF1QPDm),
+  X86_INTRINSIC_DATA(avx512_scatterpf_qps_512, PREFETCH, X86::VSCATTERPF0QPSm,
+                     X86::VSCATTERPF1QPSm),
+  X86_INTRINSIC_DATA(avx512_scattersiv2_df, SCATTER, X86::VSCATTERDPDZ128mr, 0),
+  X86_INTRINSIC_DATA(avx512_scattersiv2_di, SCATTER, X86::VPSCATTERDQZ128mr, 0),
+  X86_INTRINSIC_DATA(avx512_scattersiv4_df, SCATTER, X86::VSCATTERDPDZ256mr, 0),
+  X86_INTRINSIC_DATA(avx512_scattersiv4_di, SCATTER, X86::VPSCATTERDQZ256mr, 0),
+  X86_INTRINSIC_DATA(avx512_scattersiv4_sf, SCATTER, X86::VSCATTERDPSZ128mr, 0),
+  X86_INTRINSIC_DATA(avx512_scattersiv4_si, SCATTER, X86::VPSCATTERDDZ128mr, 0),
+  X86_INTRINSIC_DATA(avx512_scattersiv8_sf, SCATTER, X86::VSCATTERDPSZ256mr, 0),
+  X86_INTRINSIC_DATA(avx512_scattersiv8_si, SCATTER, X86::VPSCATTERDDZ256mr, 0),
 
   X86_INTRINSIC_DATA(rdpmc,     RDPMC,  X86ISD::RDPMC_DAG, 0),
   X86_INTRINSIC_DATA(rdrand_16, RDRAND, X86ISD::RDRAND, 0),
@@ -171,81 +243,154 @@ static const IntrinsicData* getIntrinsicWithChain(unsigned IntNo) {
  * the alphabetical order.
  */
 static const IntrinsicData  IntrinsicsWithoutChain[] = {
-  X86_INTRINSIC_DATA(avx2_packssdw,     INTR_TYPE_2OP, X86ISD::PACKSS, 0),
-  X86_INTRINSIC_DATA(avx2_packsswb,     INTR_TYPE_2OP, X86ISD::PACKSS, 0),
-  X86_INTRINSIC_DATA(avx2_packusdw,     INTR_TYPE_2OP, X86ISD::PACKUS, 0),
-  X86_INTRINSIC_DATA(avx2_packuswb,     INTR_TYPE_2OP, X86ISD::PACKUS, 0),
-  X86_INTRINSIC_DATA(avx2_phadd_d,      INTR_TYPE_2OP, X86ISD::HADD, 0),
-  X86_INTRINSIC_DATA(avx2_phadd_w,      INTR_TYPE_2OP, X86ISD::HADD, 0),
-  X86_INTRINSIC_DATA(avx2_phsub_d,      INTR_TYPE_2OP, X86ISD::HSUB, 0),
-  X86_INTRINSIC_DATA(avx2_phsub_w,      INTR_TYPE_2OP, X86ISD::HSUB, 0),
-  X86_INTRINSIC_DATA(avx2_pmaxs_b,      INTR_TYPE_2OP, X86ISD::SMAX, 0),
-  X86_INTRINSIC_DATA(avx2_pmaxs_d,      INTR_TYPE_2OP, X86ISD::SMAX, 0),
-  X86_INTRINSIC_DATA(avx2_pmaxs_w,      INTR_TYPE_2OP, X86ISD::SMAX, 0),
-  X86_INTRINSIC_DATA(avx2_pmaxu_b,      INTR_TYPE_2OP, X86ISD::UMAX, 0),
-  X86_INTRINSIC_DATA(avx2_pmaxu_d,      INTR_TYPE_2OP, X86ISD::UMAX, 0),
-  X86_INTRINSIC_DATA(avx2_pmaxu_w,      INTR_TYPE_2OP, X86ISD::UMAX, 0),
-  X86_INTRINSIC_DATA(avx2_pmins_b,      INTR_TYPE_2OP, X86ISD::SMIN, 0),
-  X86_INTRINSIC_DATA(avx2_pmins_d,      INTR_TYPE_2OP, X86ISD::SMIN, 0),
-  X86_INTRINSIC_DATA(avx2_pmins_w,      INTR_TYPE_2OP, X86ISD::SMIN, 0),
-  X86_INTRINSIC_DATA(avx2_pminu_b,      INTR_TYPE_2OP, X86ISD::UMIN, 0),
-  X86_INTRINSIC_DATA(avx2_pminu_d,      INTR_TYPE_2OP, X86ISD::UMIN, 0),
-  X86_INTRINSIC_DATA(avx2_pminu_w,      INTR_TYPE_2OP, X86ISD::UMIN, 0),
-  X86_INTRINSIC_DATA(avx2_pmovsxbd,     INTR_TYPE_1OP, X86ISD::VSEXT, 0),
-  X86_INTRINSIC_DATA(avx2_pmovsxbq,     INTR_TYPE_1OP, X86ISD::VSEXT, 0),
-  X86_INTRINSIC_DATA(avx2_pmovsxbw,     INTR_TYPE_1OP, X86ISD::VSEXT, 0),
-  X86_INTRINSIC_DATA(avx2_pmovsxdq,     INTR_TYPE_1OP, X86ISD::VSEXT, 0),
-  X86_INTRINSIC_DATA(avx2_pmovsxwd,     INTR_TYPE_1OP, X86ISD::VSEXT, 0),
-  X86_INTRINSIC_DATA(avx2_pmovsxwq,     INTR_TYPE_1OP, X86ISD::VSEXT, 0),
-  X86_INTRINSIC_DATA(avx2_pmovzxbd,     INTR_TYPE_1OP, X86ISD::VZEXT, 0),
-  X86_INTRINSIC_DATA(avx2_pmovzxbq,     INTR_TYPE_1OP, X86ISD::VZEXT, 0),
-  X86_INTRINSIC_DATA(avx2_pmovzxbw,     INTR_TYPE_1OP, X86ISD::VZEXT, 0),
-  X86_INTRINSIC_DATA(avx2_pmovzxdq,     INTR_TYPE_1OP, X86ISD::VZEXT, 0),
-  X86_INTRINSIC_DATA(avx2_pmovzxwd,     INTR_TYPE_1OP, X86ISD::VZEXT, 0),
-  X86_INTRINSIC_DATA(avx2_pmovzxwq,     INTR_TYPE_1OP, X86ISD::VZEXT, 0),
-  X86_INTRINSIC_DATA(avx2_pmul_dq,      INTR_TYPE_2OP, X86ISD::PMULDQ, 0),
-  X86_INTRINSIC_DATA(avx2_pmulh_w,      INTR_TYPE_2OP, ISD::MULHS, 0),
-  X86_INTRINSIC_DATA(avx2_pmulhu_w,     INTR_TYPE_2OP, ISD::MULHU, 0),
-  X86_INTRINSIC_DATA(avx2_pmulu_dq,     INTR_TYPE_2OP, X86ISD::PMULUDQ, 0),
-  X86_INTRINSIC_DATA(avx2_pshuf_b,      INTR_TYPE_2OP, X86ISD::PSHUFB, 0),
-  X86_INTRINSIC_DATA(avx2_psign_b,      INTR_TYPE_2OP, X86ISD::PSIGN, 0),
-  X86_INTRINSIC_DATA(avx2_psign_d,      INTR_TYPE_2OP, X86ISD::PSIGN, 0),
-  X86_INTRINSIC_DATA(avx2_psign_w,      INTR_TYPE_2OP, X86ISD::PSIGN, 0),
-  X86_INTRINSIC_DATA(avx2_psll_d,       INTR_TYPE_2OP, X86ISD::VSHL, 0),
-  X86_INTRINSIC_DATA(avx2_psll_q,       INTR_TYPE_2OP, X86ISD::VSHL, 0),
-  X86_INTRINSIC_DATA(avx2_psll_w,       INTR_TYPE_2OP, X86ISD::VSHL, 0),
-  X86_INTRINSIC_DATA(avx2_pslli_d,      VSHIFT, X86ISD::VSHLI, 0),
-  X86_INTRINSIC_DATA(avx2_pslli_q,      VSHIFT, X86ISD::VSHLI, 0),
-  X86_INTRINSIC_DATA(avx2_pslli_w,      VSHIFT, X86ISD::VSHLI, 0),
-  X86_INTRINSIC_DATA(avx2_psllv_d,      INTR_TYPE_2OP, ISD::SHL, 0),
-  X86_INTRINSIC_DATA(avx2_psllv_d_256,  INTR_TYPE_2OP, ISD::SHL, 0),
-  X86_INTRINSIC_DATA(avx2_psllv_q,      INTR_TYPE_2OP, ISD::SHL, 0),
-  X86_INTRINSIC_DATA(avx2_psllv_q_256,  INTR_TYPE_2OP, ISD::SHL, 0),
-  X86_INTRINSIC_DATA(avx2_psra_d,       INTR_TYPE_2OP, X86ISD::VSRA, 0),
-  X86_INTRINSIC_DATA(avx2_psra_w,       INTR_TYPE_2OP, X86ISD::VSRA, 0),
-  X86_INTRINSIC_DATA(avx2_psrai_d,      VSHIFT, X86ISD::VSRAI, 0),
-  X86_INTRINSIC_DATA(avx2_psrai_w,      VSHIFT, X86ISD::VSRAI, 0),
-  X86_INTRINSIC_DATA(avx2_psrav_d,      INTR_TYPE_2OP, ISD::SRA, 0),
-  X86_INTRINSIC_DATA(avx2_psrav_d_256,  INTR_TYPE_2OP, ISD::SRA, 0),
-  X86_INTRINSIC_DATA(avx2_psrl_d,       INTR_TYPE_2OP, X86ISD::VSRL, 0),
-  X86_INTRINSIC_DATA(avx2_psrl_q,       INTR_TYPE_2OP, X86ISD::VSRL, 0),
-  X86_INTRINSIC_DATA(avx2_psrl_w,       INTR_TYPE_2OP, X86ISD::VSRL, 0),
-  X86_INTRINSIC_DATA(avx2_psrli_d,      VSHIFT, X86ISD::VSRLI, 0),
-  X86_INTRINSIC_DATA(avx2_psrli_q,      VSHIFT, X86ISD::VSRLI, 0),
-  X86_INTRINSIC_DATA(avx2_psrli_w,      VSHIFT, X86ISD::VSRLI, 0),
-  X86_INTRINSIC_DATA(avx2_psrlv_d,      INTR_TYPE_2OP, ISD::SRL, 0),
-  X86_INTRINSIC_DATA(avx2_psrlv_d_256,  INTR_TYPE_2OP, ISD::SRL, 0),
-  X86_INTRINSIC_DATA(avx2_psrlv_q,      INTR_TYPE_2OP, ISD::SRL, 0),
-  X86_INTRINSIC_DATA(avx2_psrlv_q_256,  INTR_TYPE_2OP, ISD::SRL, 0),
-  X86_INTRINSIC_DATA(avx2_psubus_b,     INTR_TYPE_2OP, X86ISD::SUBUS, 0),
-  X86_INTRINSIC_DATA(avx2_psubus_w,     INTR_TYPE_2OP, X86ISD::SUBUS, 0),
-  X86_INTRINSIC_DATA(avx2_vperm2i128,   INTR_TYPE_3OP, X86ISD::VPERM2X128, 0),
-  X86_INTRINSIC_DATA(avx512_exp2_pd,    INTR_TYPE_1OP_MASK_RM,X86ISD::EXP2, 0),
-  X86_INTRINSIC_DATA(avx512_exp2_ps,    INTR_TYPE_1OP_MASK_RM,X86ISD::EXP2, 0),
+  X86_INTRINSIC_DATA(avx2_packssdw, INTR_TYPE_2OP, X86ISD::PACKSS, 0),
+  X86_INTRINSIC_DATA(avx2_packsswb, INTR_TYPE_2OP, X86ISD::PACKSS, 0),
+  X86_INTRINSIC_DATA(avx2_packusdw, INTR_TYPE_2OP, X86ISD::PACKUS, 0),
+  X86_INTRINSIC_DATA(avx2_packuswb, INTR_TYPE_2OP, X86ISD::PACKUS, 0),
+  X86_INTRINSIC_DATA(avx2_phadd_d, INTR_TYPE_2OP, X86ISD::HADD, 0),
+  X86_INTRINSIC_DATA(avx2_phadd_w, INTR_TYPE_2OP, X86ISD::HADD, 0),
+  X86_INTRINSIC_DATA(avx2_phsub_d, INTR_TYPE_2OP, X86ISD::HSUB, 0),
+  X86_INTRINSIC_DATA(avx2_phsub_w, INTR_TYPE_2OP, X86ISD::HSUB, 0),
+  X86_INTRINSIC_DATA(avx2_pmaxs_b, INTR_TYPE_2OP, ISD::SMAX, 0),
+  X86_INTRINSIC_DATA(avx2_pmaxs_d, INTR_TYPE_2OP, ISD::SMAX, 0),
+  X86_INTRINSIC_DATA(avx2_pmaxs_w, INTR_TYPE_2OP, ISD::SMAX, 0),
+  X86_INTRINSIC_DATA(avx2_pmaxu_b, INTR_TYPE_2OP, ISD::UMAX, 0),
+  X86_INTRINSIC_DATA(avx2_pmaxu_d, INTR_TYPE_2OP, ISD::UMAX, 0),
+  X86_INTRINSIC_DATA(avx2_pmaxu_w, INTR_TYPE_2OP, ISD::UMAX, 0),
+  X86_INTRINSIC_DATA(avx2_pmins_b, INTR_TYPE_2OP, ISD::SMIN, 0),
+  X86_INTRINSIC_DATA(avx2_pmins_d, INTR_TYPE_2OP, ISD::SMIN, 0),
+  X86_INTRINSIC_DATA(avx2_pmins_w, INTR_TYPE_2OP, ISD::SMIN, 0),
+  X86_INTRINSIC_DATA(avx2_pminu_b, INTR_TYPE_2OP, ISD::UMIN, 0),
+  X86_INTRINSIC_DATA(avx2_pminu_d, INTR_TYPE_2OP, ISD::UMIN, 0),
+  X86_INTRINSIC_DATA(avx2_pminu_w, INTR_TYPE_2OP, ISD::UMIN, 0),
+  X86_INTRINSIC_DATA(avx2_pmovsxbd, INTR_TYPE_1OP, X86ISD::VSEXT, 0),
+  X86_INTRINSIC_DATA(avx2_pmovsxbq, INTR_TYPE_1OP, X86ISD::VSEXT, 0),
+  X86_INTRINSIC_DATA(avx2_pmovsxbw, INTR_TYPE_1OP, X86ISD::VSEXT, 0),
+  X86_INTRINSIC_DATA(avx2_pmovsxdq, INTR_TYPE_1OP, X86ISD::VSEXT, 0),
+  X86_INTRINSIC_DATA(avx2_pmovsxwd, INTR_TYPE_1OP, X86ISD::VSEXT, 0),
+  X86_INTRINSIC_DATA(avx2_pmovsxwq, INTR_TYPE_1OP, X86ISD::VSEXT, 0),
+  X86_INTRINSIC_DATA(avx2_pmovzxbd, INTR_TYPE_1OP, X86ISD::VZEXT, 0),
+  X86_INTRINSIC_DATA(avx2_pmovzxbq, INTR_TYPE_1OP, X86ISD::VZEXT, 0),
+  X86_INTRINSIC_DATA(avx2_pmovzxbw, INTR_TYPE_1OP, X86ISD::VZEXT, 0),
+  X86_INTRINSIC_DATA(avx2_pmovzxdq, INTR_TYPE_1OP, X86ISD::VZEXT, 0),
+  X86_INTRINSIC_DATA(avx2_pmovzxwd, INTR_TYPE_1OP, X86ISD::VZEXT, 0),
+  X86_INTRINSIC_DATA(avx2_pmovzxwq, INTR_TYPE_1OP, X86ISD::VZEXT, 0),
+  X86_INTRINSIC_DATA(avx2_pmul_dq, INTR_TYPE_2OP, X86ISD::PMULDQ, 0),
+  X86_INTRINSIC_DATA(avx2_pmulh_w, INTR_TYPE_2OP, ISD::MULHS, 0),
+  X86_INTRINSIC_DATA(avx2_pmulhu_w, INTR_TYPE_2OP, ISD::MULHU, 0),
+  X86_INTRINSIC_DATA(avx2_pmulu_dq, INTR_TYPE_2OP, X86ISD::PMULUDQ, 0),
+  X86_INTRINSIC_DATA(avx2_pshuf_b, INTR_TYPE_2OP, X86ISD::PSHUFB, 0),
+  X86_INTRINSIC_DATA(avx2_psign_b, INTR_TYPE_2OP, X86ISD::PSIGN, 0),
+  X86_INTRINSIC_DATA(avx2_psign_d, INTR_TYPE_2OP, X86ISD::PSIGN, 0),
+  X86_INTRINSIC_DATA(avx2_psign_w, INTR_TYPE_2OP, X86ISD::PSIGN, 0),
+  X86_INTRINSIC_DATA(avx2_psll_d, INTR_TYPE_2OP, X86ISD::VSHL, 0),
+  X86_INTRINSIC_DATA(avx2_psll_q, INTR_TYPE_2OP, X86ISD::VSHL, 0),
+  X86_INTRINSIC_DATA(avx2_psll_w, INTR_TYPE_2OP, X86ISD::VSHL, 0),
+  X86_INTRINSIC_DATA(avx2_pslli_d, VSHIFT, X86ISD::VSHLI, 0),
+  X86_INTRINSIC_DATA(avx2_pslli_q, VSHIFT, X86ISD::VSHLI, 0),
+  X86_INTRINSIC_DATA(avx2_pslli_w, VSHIFT, X86ISD::VSHLI, 0),
+  X86_INTRINSIC_DATA(avx2_psllv_d, INTR_TYPE_2OP, ISD::SHL, 0),
+  X86_INTRINSIC_DATA(avx2_psllv_d_256, INTR_TYPE_2OP, ISD::SHL, 0),
+  X86_INTRINSIC_DATA(avx2_psllv_q, INTR_TYPE_2OP, ISD::SHL, 0),
+  X86_INTRINSIC_DATA(avx2_psllv_q_256, INTR_TYPE_2OP, ISD::SHL, 0),
+  X86_INTRINSIC_DATA(avx2_psra_d, INTR_TYPE_2OP, X86ISD::VSRA, 0),
+  X86_INTRINSIC_DATA(avx2_psra_w, INTR_TYPE_2OP, X86ISD::VSRA, 0),
+  X86_INTRINSIC_DATA(avx2_psrai_d, VSHIFT, X86ISD::VSRAI, 0),
+  X86_INTRINSIC_DATA(avx2_psrai_w, VSHIFT, X86ISD::VSRAI, 0),
+  X86_INTRINSIC_DATA(avx2_psrav_d, INTR_TYPE_2OP, ISD::SRA, 0),
+  X86_INTRINSIC_DATA(avx2_psrav_d_256, INTR_TYPE_2OP, ISD::SRA, 0),
+  X86_INTRINSIC_DATA(avx2_psrl_d, INTR_TYPE_2OP, X86ISD::VSRL, 0),
+  X86_INTRINSIC_DATA(avx2_psrl_q, INTR_TYPE_2OP, X86ISD::VSRL, 0),
+  X86_INTRINSIC_DATA(avx2_psrl_w, INTR_TYPE_2OP, X86ISD::VSRL, 0),
+  X86_INTRINSIC_DATA(avx2_psrli_d, VSHIFT, X86ISD::VSRLI, 0),
+  X86_INTRINSIC_DATA(avx2_psrli_q, VSHIFT, X86ISD::VSRLI, 0),
+  X86_INTRINSIC_DATA(avx2_psrli_w, VSHIFT, X86ISD::VSRLI, 0),
+  X86_INTRINSIC_DATA(avx2_psrlv_d, INTR_TYPE_2OP, ISD::SRL, 0),
+  X86_INTRINSIC_DATA(avx2_psrlv_d_256, INTR_TYPE_2OP, ISD::SRL, 0),
+  X86_INTRINSIC_DATA(avx2_psrlv_q, INTR_TYPE_2OP, ISD::SRL, 0),
+  X86_INTRINSIC_DATA(avx2_psrlv_q_256, INTR_TYPE_2OP, ISD::SRL, 0),
+  X86_INTRINSIC_DATA(avx2_psubus_b, INTR_TYPE_2OP, X86ISD::SUBUS, 0),
+  X86_INTRINSIC_DATA(avx2_psubus_w, INTR_TYPE_2OP, X86ISD::SUBUS, 0),
+  X86_INTRINSIC_DATA(avx2_vperm2i128, INTR_TYPE_3OP, X86ISD::VPERM2X128, 0),
+  X86_INTRINSIC_DATA(avx512_cvtsi2sd32,  INTR_TYPE_3OP, X86ISD::SINT_TO_FP_RND, 0),
+  X86_INTRINSIC_DATA(avx512_cvtsi2sd64,  INTR_TYPE_3OP, X86ISD::SINT_TO_FP_RND, 0),
+  X86_INTRINSIC_DATA(avx512_cvtsi2ss32,  INTR_TYPE_3OP, X86ISD::SINT_TO_FP_RND, 0),
+  X86_INTRINSIC_DATA(avx512_cvtsi2ss64,  INTR_TYPE_3OP, X86ISD::SINT_TO_FP_RND, 0),
+  X86_INTRINSIC_DATA(avx512_cvtusi2ss,   INTR_TYPE_3OP, X86ISD::UINT_TO_FP_RND, 0),
+  X86_INTRINSIC_DATA(avx512_cvtusi642sd, INTR_TYPE_3OP, X86ISD::UINT_TO_FP_RND, 0),
+  X86_INTRINSIC_DATA(avx512_cvtusi642ss, INTR_TYPE_3OP, X86ISD::UINT_TO_FP_RND, 0),
+  X86_INTRINSIC_DATA(avx512_exp2_pd, INTR_TYPE_1OP_MASK_RM, X86ISD::EXP2, 0),
+  X86_INTRINSIC_DATA(avx512_exp2_ps, INTR_TYPE_1OP_MASK_RM, X86ISD::EXP2, 0),
+
+  X86_INTRINSIC_DATA(avx512_mask3_vfmadd_pd_128, FMA_OP_MASK3, X86ISD::FMADD, 0),
+  X86_INTRINSIC_DATA(avx512_mask3_vfmadd_pd_256, FMA_OP_MASK3, X86ISD::FMADD, 0),
+  X86_INTRINSIC_DATA(avx512_mask3_vfmadd_pd_512, FMA_OP_MASK3, X86ISD::FMADD,
+                     X86ISD::FMADD_RND),
+  X86_INTRINSIC_DATA(avx512_mask3_vfmadd_ps_128, FMA_OP_MASK3, X86ISD::FMADD, 0),
+  X86_INTRINSIC_DATA(avx512_mask3_vfmadd_ps_256, FMA_OP_MASK3, X86ISD::FMADD, 0),
+  X86_INTRINSIC_DATA(avx512_mask3_vfmadd_ps_512, FMA_OP_MASK3, X86ISD::FMADD,
+                     X86ISD::FMADD_RND),
+
+  X86_INTRINSIC_DATA(avx512_mask3_vfmaddsub_pd_128, FMA_OP_MASK3, X86ISD::FMADDSUB, 0),
+  X86_INTRINSIC_DATA(avx512_mask3_vfmaddsub_pd_256, FMA_OP_MASK3, X86ISD::FMADDSUB, 0),
+  X86_INTRINSIC_DATA(avx512_mask3_vfmaddsub_pd_512, FMA_OP_MASK3, X86ISD::FMADDSUB,
+                     X86ISD::FMADDSUB_RND),
+  X86_INTRINSIC_DATA(avx512_mask3_vfmaddsub_ps_128, FMA_OP_MASK3, X86ISD::FMADDSUB, 0),
+  X86_INTRINSIC_DATA(avx512_mask3_vfmaddsub_ps_256, FMA_OP_MASK3, X86ISD::FMADDSUB, 0),
+  X86_INTRINSIC_DATA(avx512_mask3_vfmaddsub_ps_512, FMA_OP_MASK3, X86ISD::FMADDSUB,
+                     X86ISD::FMADDSUB_RND),
+
+  X86_INTRINSIC_DATA(avx512_mask3_vfmsub_pd_128, FMA_OP_MASK3, X86ISD::FMSUB, 0),
+  X86_INTRINSIC_DATA(avx512_mask3_vfmsub_pd_256, FMA_OP_MASK3, X86ISD::FMSUB, 0),
+  X86_INTRINSIC_DATA(avx512_mask3_vfmsub_pd_512, FMA_OP_MASK3, X86ISD::FMSUB,
+                     X86ISD::FMSUB_RND),
+  X86_INTRINSIC_DATA(avx512_mask3_vfmsub_ps_128, FMA_OP_MASK3, X86ISD::FMSUB, 0),
+  X86_INTRINSIC_DATA(avx512_mask3_vfmsub_ps_256, FMA_OP_MASK3, X86ISD::FMSUB, 0),
+  X86_INTRINSIC_DATA(avx512_mask3_vfmsub_ps_512, FMA_OP_MASK3, X86ISD::FMSUB,
+                     X86ISD::FMSUB_RND),
+
+  X86_INTRINSIC_DATA(avx512_mask3_vfmsubadd_pd_128, FMA_OP_MASK3, X86ISD::FMSUBADD, 0),
+  X86_INTRINSIC_DATA(avx512_mask3_vfmsubadd_pd_256, FMA_OP_MASK3, X86ISD::FMSUBADD, 0),
+  X86_INTRINSIC_DATA(avx512_mask3_vfmsubadd_pd_512, FMA_OP_MASK3, X86ISD::FMSUBADD,
+                     X86ISD::FMSUBADD_RND),
+  X86_INTRINSIC_DATA(avx512_mask3_vfmsubadd_ps_128, FMA_OP_MASK3, X86ISD::FMSUBADD, 0),
+  X86_INTRINSIC_DATA(avx512_mask3_vfmsubadd_ps_256, FMA_OP_MASK3, X86ISD::FMSUBADD, 0),
+  X86_INTRINSIC_DATA(avx512_mask3_vfmsubadd_ps_512, FMA_OP_MASK3, X86ISD::FMSUBADD,
+                     X86ISD::FMSUBADD_RND),
+
+  X86_INTRINSIC_DATA(avx512_mask3_vfnmsub_pd_128, FMA_OP_MASK3, X86ISD::FNMSUB, 0),
+  X86_INTRINSIC_DATA(avx512_mask3_vfnmsub_pd_256, FMA_OP_MASK3, X86ISD::FNMSUB, 0),
+  X86_INTRINSIC_DATA(avx512_mask3_vfnmsub_pd_512, FMA_OP_MASK3, X86ISD::FNMSUB,
+                     X86ISD::FNMSUB_RND),
+  X86_INTRINSIC_DATA(avx512_mask3_vfnmsub_ps_128, FMA_OP_MASK3, X86ISD::FNMSUB, 0),
+  X86_INTRINSIC_DATA(avx512_mask3_vfnmsub_ps_256, FMA_OP_MASK3, X86ISD::FNMSUB, 0),
+  X86_INTRINSIC_DATA(avx512_mask3_vfnmsub_ps_512, FMA_OP_MASK3, X86ISD::FNMSUB,
+                     X86ISD::FNMSUB_RND),
+
+  X86_INTRINSIC_DATA(avx512_mask_add_pd_128, INTR_TYPE_2OP_MASK, ISD::FADD, 0),
+  X86_INTRINSIC_DATA(avx512_mask_add_pd_256, INTR_TYPE_2OP_MASK, ISD::FADD, 0),
   X86_INTRINSIC_DATA(avx512_mask_add_pd_512, INTR_TYPE_2OP_MASK, ISD::FADD,
-                     X86ISD::FADD_RND),
+  X86ISD::FADD_RND),
+  X86_INTRINSIC_DATA(avx512_mask_add_ps_128, INTR_TYPE_2OP_MASK, ISD::FADD, 0),
+  X86_INTRINSIC_DATA(avx512_mask_add_ps_256, INTR_TYPE_2OP_MASK, ISD::FADD, 0),
   X86_INTRINSIC_DATA(avx512_mask_add_ps_512, INTR_TYPE_2OP_MASK, ISD::FADD,
-                     X86ISD::FADD_RND),
+  X86ISD::FADD_RND),
+  X86_INTRINSIC_DATA(avx512_mask_add_sd_round, INTR_TYPE_SCALAR_MASK_RM, ISD::FADD,
+  X86ISD::FADD_RND),
+  X86_INTRINSIC_DATA(avx512_mask_add_ss_round, INTR_TYPE_SCALAR_MASK_RM, ISD::FADD,
+  X86ISD::FADD_RND),
+  X86_INTRINSIC_DATA(avx512_mask_and_pd_128, INTR_TYPE_2OP_MASK, X86ISD::FAND, 0),
+  X86_INTRINSIC_DATA(avx512_mask_and_pd_256, INTR_TYPE_2OP_MASK, X86ISD::FAND, 0),
+  X86_INTRINSIC_DATA(avx512_mask_and_pd_512, INTR_TYPE_2OP_MASK, X86ISD::FAND, 0),
+  X86_INTRINSIC_DATA(avx512_mask_and_ps_128, INTR_TYPE_2OP_MASK, X86ISD::FAND, 0),
+  X86_INTRINSIC_DATA(avx512_mask_and_ps_256, INTR_TYPE_2OP_MASK, X86ISD::FAND, 0),
+  X86_INTRINSIC_DATA(avx512_mask_and_ps_512, INTR_TYPE_2OP_MASK, X86ISD::FAND, 0),
+  X86_INTRINSIC_DATA(avx512_mask_andn_pd_128, INTR_TYPE_2OP_MASK, X86ISD::FANDN, 0),
+  X86_INTRINSIC_DATA(avx512_mask_andn_pd_256, INTR_TYPE_2OP_MASK, X86ISD::FANDN, 0),
+  X86_INTRINSIC_DATA(avx512_mask_andn_pd_512, INTR_TYPE_2OP_MASK, X86ISD::FANDN, 0),
+  X86_INTRINSIC_DATA(avx512_mask_andn_ps_128, INTR_TYPE_2OP_MASK, X86ISD::FANDN, 0),
+  X86_INTRINSIC_DATA(avx512_mask_andn_ps_256, INTR_TYPE_2OP_MASK, X86ISD::FANDN, 0),
+  X86_INTRINSIC_DATA(avx512_mask_andn_ps_512, INTR_TYPE_2OP_MASK, X86ISD::FANDN, 0),
   X86_INTRINSIC_DATA(avx512_mask_blend_b_128,  BLEND, X86ISD::SELECT, 0),
   X86_INTRINSIC_DATA(avx512_mask_blend_b_256,  BLEND, X86ISD::SELECT, 0),
   X86_INTRINSIC_DATA(avx512_mask_blend_b_512,  BLEND, X86ISD::SELECT, 0),
@@ -264,18 +409,26 @@ static const IntrinsicData  IntrinsicsWithoutChain[] = {
   X86_INTRINSIC_DATA(avx512_mask_blend_w_128,  BLEND, X86ISD::SELECT, 0),
   X86_INTRINSIC_DATA(avx512_mask_blend_w_256,  BLEND, X86ISD::SELECT, 0),
   X86_INTRINSIC_DATA(avx512_mask_blend_w_512,  BLEND, X86ISD::SELECT, 0),
-  X86_INTRINSIC_DATA(avx512_mask_cmp_b_128,     CMP_MASK_CC,  X86ISD::CMPM, 0),
-  X86_INTRINSIC_DATA(avx512_mask_cmp_b_256,     CMP_MASK_CC,  X86ISD::CMPM, 0),
-  X86_INTRINSIC_DATA(avx512_mask_cmp_b_512,     CMP_MASK_CC,  X86ISD::CMPM, 0),
-  X86_INTRINSIC_DATA(avx512_mask_cmp_d_128,     CMP_MASK_CC,  X86ISD::CMPM, 0),
-  X86_INTRINSIC_DATA(avx512_mask_cmp_d_256,     CMP_MASK_CC,  X86ISD::CMPM, 0),
-  X86_INTRINSIC_DATA(avx512_mask_cmp_d_512,     CMP_MASK_CC,  X86ISD::CMPM, 0),
-  X86_INTRINSIC_DATA(avx512_mask_cmp_q_128,     CMP_MASK_CC,  X86ISD::CMPM, 0),
-  X86_INTRINSIC_DATA(avx512_mask_cmp_q_256,     CMP_MASK_CC,  X86ISD::CMPM, 0),
-  X86_INTRINSIC_DATA(avx512_mask_cmp_q_512,     CMP_MASK_CC,  X86ISD::CMPM, 0),
-  X86_INTRINSIC_DATA(avx512_mask_cmp_w_128,     CMP_MASK_CC,  X86ISD::CMPM, 0),
-  X86_INTRINSIC_DATA(avx512_mask_cmp_w_256,     CMP_MASK_CC,  X86ISD::CMPM, 0),
-  X86_INTRINSIC_DATA(avx512_mask_cmp_w_512,     CMP_MASK_CC,  X86ISD::CMPM, 0),
+  X86_INTRINSIC_DATA(avx512_mask_cmp_b_128,  CMP_MASK_CC, X86ISD::CMPM, 0),
+  X86_INTRINSIC_DATA(avx512_mask_cmp_b_256,  CMP_MASK_CC, X86ISD::CMPM, 0),
+  X86_INTRINSIC_DATA(avx512_mask_cmp_b_512,  CMP_MASK_CC, X86ISD::CMPM, 0),
+  X86_INTRINSIC_DATA(avx512_mask_cmp_d_128,  CMP_MASK_CC, X86ISD::CMPM, 0),
+  X86_INTRINSIC_DATA(avx512_mask_cmp_d_256,  CMP_MASK_CC, X86ISD::CMPM, 0),
+  X86_INTRINSIC_DATA(avx512_mask_cmp_d_512,  CMP_MASK_CC, X86ISD::CMPM, 0),
+  X86_INTRINSIC_DATA(avx512_mask_cmp_pd_128, CMP_MASK_CC, X86ISD::CMPM, 0),
+  X86_INTRINSIC_DATA(avx512_mask_cmp_pd_256, CMP_MASK_CC, X86ISD::CMPM, 0),
+  X86_INTRINSIC_DATA(avx512_mask_cmp_pd_512, CMP_MASK_CC, X86ISD::CMPM,
+                     X86ISD::CMPM_RND),
+  X86_INTRINSIC_DATA(avx512_mask_cmp_ps_128, CMP_MASK_CC, X86ISD::CMPM, 0),
+  X86_INTRINSIC_DATA(avx512_mask_cmp_ps_256, CMP_MASK_CC, X86ISD::CMPM, 0),
+  X86_INTRINSIC_DATA(avx512_mask_cmp_ps_512, CMP_MASK_CC, X86ISD::CMPM,
+                     X86ISD::CMPM_RND),
+  X86_INTRINSIC_DATA(avx512_mask_cmp_q_128,  CMP_MASK_CC, X86ISD::CMPM, 0),
+  X86_INTRINSIC_DATA(avx512_mask_cmp_q_256,  CMP_MASK_CC, X86ISD::CMPM, 0),
+  X86_INTRINSIC_DATA(avx512_mask_cmp_q_512,  CMP_MASK_CC, X86ISD::CMPM, 0),
+  X86_INTRINSIC_DATA(avx512_mask_cmp_w_128,  CMP_MASK_CC, X86ISD::CMPM, 0),
+  X86_INTRINSIC_DATA(avx512_mask_cmp_w_256,  CMP_MASK_CC, X86ISD::CMPM, 0),
+  X86_INTRINSIC_DATA(avx512_mask_cmp_w_512,  CMP_MASK_CC, X86ISD::CMPM, 0),
   X86_INTRINSIC_DATA(avx512_mask_compress_d_128,  COMPRESS_EXPAND_IN_REG,
                      X86ISD::COMPRESS, 0),
   X86_INTRINSIC_DATA(avx512_mask_compress_d_256,  COMPRESS_EXPAND_IN_REG,
@@ -301,10 +454,175 @@ static const IntrinsicData  IntrinsicsWithoutChain[] = {
   X86_INTRINSIC_DATA(avx512_mask_compress_q_512,  COMPRESS_EXPAND_IN_REG,
                      X86ISD::COMPRESS, 0),
 
+  X86_INTRINSIC_DATA(avx512_mask_cvtdq2pd_128, INTR_TYPE_1OP_MASK,
+                     X86ISD::CVTDQ2PD, 0),
+  X86_INTRINSIC_DATA(avx512_mask_cvtdq2pd_256, INTR_TYPE_1OP_MASK,
+                     ISD::SINT_TO_FP, 0),
+  X86_INTRINSIC_DATA(avx512_mask_cvtdq2pd_512, INTR_TYPE_1OP_MASK,
+                     ISD::SINT_TO_FP, 0), // no rm
+  X86_INTRINSIC_DATA(avx512_mask_cvtdq2ps_128, INTR_TYPE_1OP_MASK,
+                     ISD::SINT_TO_FP, 0),
+  X86_INTRINSIC_DATA(avx512_mask_cvtdq2ps_256, INTR_TYPE_1OP_MASK,
+                     ISD::SINT_TO_FP, 0),
+  X86_INTRINSIC_DATA(avx512_mask_cvtdq2ps_512, INTR_TYPE_1OP_MASK,
+                     ISD::SINT_TO_FP, ISD::SINT_TO_FP), //er
+  X86_INTRINSIC_DATA(avx512_mask_cvtpd2dq_128, INTR_TYPE_1OP_MASK,
+                     X86ISD::FP_TO_SINT_RND, 0),
+  X86_INTRINSIC_DATA(avx512_mask_cvtpd2dq_256, INTR_TYPE_1OP_MASK,
+                    X86ISD::FP_TO_SINT_RND, 0),
+  X86_INTRINSIC_DATA(avx512_mask_cvtpd2dq_512, INTR_TYPE_1OP_MASK,
+                    X86ISD::FP_TO_SINT_RND, X86ISD::FP_TO_SINT_RND),
+  X86_INTRINSIC_DATA(avx512_mask_cvtpd2ps,     INTR_TYPE_1OP_MASK,
+                    X86ISD::VFPROUND, 0),
+  X86_INTRINSIC_DATA(avx512_mask_cvtpd2ps_256, INTR_TYPE_1OP_MASK_RM,
+                     ISD::FP_ROUND, 0),
+  X86_INTRINSIC_DATA(avx512_mask_cvtpd2ps_512, INTR_TYPE_1OP_MASK_RM,
+                     ISD::FP_ROUND, X86ISD::VFPROUND),
+  X86_INTRINSIC_DATA(avx512_mask_cvtpd2qq_128, INTR_TYPE_1OP_MASK,
+                     X86ISD::FP_TO_SINT_RND, 0),
+  X86_INTRINSIC_DATA(avx512_mask_cvtpd2qq_256, INTR_TYPE_1OP_MASK,
+                     X86ISD::FP_TO_SINT_RND, 0),
+  X86_INTRINSIC_DATA(avx512_mask_cvtpd2qq_512, INTR_TYPE_1OP_MASK,
+                     X86ISD::FP_TO_SINT_RND, X86ISD::FP_TO_SINT_RND),
+  X86_INTRINSIC_DATA(avx512_mask_cvtpd2udq_128, INTR_TYPE_1OP_MASK,
+                     X86ISD::FP_TO_UINT_RND, 0),
+  X86_INTRINSIC_DATA(avx512_mask_cvtpd2udq_256, INTR_TYPE_1OP_MASK,
+                     X86ISD::FP_TO_UINT_RND, 0),
+  X86_INTRINSIC_DATA(avx512_mask_cvtpd2udq_512, INTR_TYPE_1OP_MASK,
+                     X86ISD::FP_TO_UINT_RND, X86ISD::FP_TO_UINT_RND),
+  X86_INTRINSIC_DATA(avx512_mask_cvtpd2uqq_128, INTR_TYPE_1OP_MASK,
+                     X86ISD::FP_TO_UINT_RND, 0),
+  X86_INTRINSIC_DATA(avx512_mask_cvtpd2uqq_256, INTR_TYPE_1OP_MASK,
+                     X86ISD::FP_TO_UINT_RND, 0),
+  X86_INTRINSIC_DATA(avx512_mask_cvtpd2uqq_512, INTR_TYPE_1OP_MASK,
+                     X86ISD::FP_TO_UINT_RND, X86ISD::FP_TO_UINT_RND),
+  X86_INTRINSIC_DATA(avx512_mask_cvtps2dq_128, INTR_TYPE_1OP_MASK,
+                     X86ISD::FP_TO_SINT_RND, 0),
+  X86_INTRINSIC_DATA(avx512_mask_cvtps2dq_256, INTR_TYPE_1OP_MASK,
+                     X86ISD::FP_TO_SINT_RND, 0),
+  X86_INTRINSIC_DATA(avx512_mask_cvtps2dq_512, INTR_TYPE_1OP_MASK,
+                     X86ISD::FP_TO_SINT_RND, X86ISD::FP_TO_SINT_RND),
+  X86_INTRINSIC_DATA(avx512_mask_cvtps2pd_128, INTR_TYPE_1OP_MASK,
+                     X86ISD::VFPEXT, 0),
+  X86_INTRINSIC_DATA(avx512_mask_cvtps2pd_256, INTR_TYPE_1OP_MASK,
+                     ISD::FP_EXTEND, 0),
+  X86_INTRINSIC_DATA(avx512_mask_cvtps2pd_512, INTR_TYPE_1OP_MASK,
+                     ISD::FP_EXTEND, X86ISD::VFPEXT),
+  X86_INTRINSIC_DATA(avx512_mask_cvtps2qq_128, INTR_TYPE_1OP_MASK,
+                     X86ISD::FP_TO_SINT_RND, 0),
+  X86_INTRINSIC_DATA(avx512_mask_cvtps2qq_256, INTR_TYPE_1OP_MASK,
+                     X86ISD::FP_TO_SINT_RND, 0),
+  X86_INTRINSIC_DATA(avx512_mask_cvtps2qq_512, INTR_TYPE_1OP_MASK,
+                     X86ISD::FP_TO_SINT_RND, X86ISD::FP_TO_SINT_RND),
+  X86_INTRINSIC_DATA(avx512_mask_cvtps2udq_128, INTR_TYPE_1OP_MASK,
+                     X86ISD::FP_TO_UINT_RND, 0),
+  X86_INTRINSIC_DATA(avx512_mask_cvtps2udq_256, INTR_TYPE_1OP_MASK,
+                     X86ISD::FP_TO_UINT_RND, 0),
+  X86_INTRINSIC_DATA(avx512_mask_cvtps2udq_512, INTR_TYPE_1OP_MASK,
+                     X86ISD::FP_TO_UINT_RND, X86ISD::FP_TO_UINT_RND),
+  X86_INTRINSIC_DATA(avx512_mask_cvtps2uqq_128, INTR_TYPE_1OP_MASK,
+                     X86ISD::FP_TO_UINT_RND, 0),
+  X86_INTRINSIC_DATA(avx512_mask_cvtps2uqq_256, INTR_TYPE_1OP_MASK,
+                     X86ISD::FP_TO_UINT_RND, 0),
+  X86_INTRINSIC_DATA(avx512_mask_cvtps2uqq_512, INTR_TYPE_1OP_MASK,
+                     X86ISD::FP_TO_UINT_RND, X86ISD::FP_TO_UINT_RND),
+  X86_INTRINSIC_DATA(avx512_mask_cvtqq2pd_128, INTR_TYPE_1OP_MASK,
+                     ISD::SINT_TO_FP, 0),
+  X86_INTRINSIC_DATA(avx512_mask_cvtqq2pd_256, INTR_TYPE_1OP_MASK,
+                     ISD::SINT_TO_FP, 0),
+  X86_INTRINSIC_DATA(avx512_mask_cvtqq2pd_512, INTR_TYPE_1OP_MASK,
+                     ISD::SINT_TO_FP, ISD::SINT_TO_FP),
+  X86_INTRINSIC_DATA(avx512_mask_cvtqq2ps_128, INTR_TYPE_1OP_MASK,
+                     ISD::SINT_TO_FP, 0),
+  X86_INTRINSIC_DATA(avx512_mask_cvtqq2ps_256, INTR_TYPE_1OP_MASK,
+                     ISD::SINT_TO_FP, 0),
+  X86_INTRINSIC_DATA(avx512_mask_cvtqq2ps_512, INTR_TYPE_1OP_MASK,
+                     ISD::SINT_TO_FP, ISD::SINT_TO_FP),
+  X86_INTRINSIC_DATA(avx512_mask_cvttpd2dq_128, INTR_TYPE_1OP_MASK,
+                     ISD::FP_TO_SINT, 0),
+  X86_INTRINSIC_DATA(avx512_mask_cvttpd2dq_256, INTR_TYPE_1OP_MASK,
+                     ISD::FP_TO_SINT, 0),
+  X86_INTRINSIC_DATA(avx512_mask_cvttpd2dq_512, INTR_TYPE_1OP_MASK,
+                     ISD::FP_TO_SINT, ISD::FP_TO_SINT),
+  X86_INTRINSIC_DATA(avx512_mask_cvttpd2qq_128, INTR_TYPE_1OP_MASK,
+                     ISD::FP_TO_SINT, 0),
+  X86_INTRINSIC_DATA(avx512_mask_cvttpd2qq_256, INTR_TYPE_1OP_MASK,
+                     ISD::FP_TO_SINT, 0),
+  X86_INTRINSIC_DATA(avx512_mask_cvttpd2qq_512, INTR_TYPE_1OP_MASK,
+                     ISD::FP_TO_SINT, ISD::FP_TO_SINT),
+  X86_INTRINSIC_DATA(avx512_mask_cvttpd2udq_128, INTR_TYPE_1OP_MASK,
+                     ISD::FP_TO_UINT, 0),
+  X86_INTRINSIC_DATA(avx512_mask_cvttpd2udq_256, INTR_TYPE_1OP_MASK,
+                     ISD::FP_TO_UINT, 0),
+  X86_INTRINSIC_DATA(avx512_mask_cvttpd2udq_512, INTR_TYPE_1OP_MASK,
+                     ISD::FP_TO_UINT, ISD::FP_TO_UINT),
+  X86_INTRINSIC_DATA(avx512_mask_cvttpd2uqq_128, INTR_TYPE_1OP_MASK,
+                     ISD::FP_TO_UINT, 0),
+  X86_INTRINSIC_DATA(avx512_mask_cvttpd2uqq_256, INTR_TYPE_1OP_MASK,
+                     ISD::FP_TO_UINT, 0),
+  X86_INTRINSIC_DATA(avx512_mask_cvttpd2uqq_512, INTR_TYPE_1OP_MASK,
+                     ISD::FP_TO_UINT, ISD::FP_TO_UINT),
+  X86_INTRINSIC_DATA(avx512_mask_cvttps2dq_128, INTR_TYPE_1OP_MASK,
+                     ISD::FP_TO_SINT, 0),
+  X86_INTRINSIC_DATA(avx512_mask_cvttps2dq_256, INTR_TYPE_1OP_MASK,
+                     ISD::FP_TO_SINT, 0),
+  X86_INTRINSIC_DATA(avx512_mask_cvttps2dq_512, INTR_TYPE_1OP_MASK,
+                     ISD::FP_TO_SINT, ISD::FP_TO_SINT),
+  X86_INTRINSIC_DATA(avx512_mask_cvttps2qq_128, INTR_TYPE_1OP_MASK,
+                     ISD::FP_TO_SINT, 0),
+  X86_INTRINSIC_DATA(avx512_mask_cvttps2qq_256, INTR_TYPE_1OP_MASK,
+                     ISD::FP_TO_SINT, 0),
+  X86_INTRINSIC_DATA(avx512_mask_cvttps2qq_512, INTR_TYPE_1OP_MASK,
+                     ISD::FP_TO_SINT, ISD::FP_TO_SINT),
+  X86_INTRINSIC_DATA(avx512_mask_cvttps2udq_128, INTR_TYPE_1OP_MASK,
+                     ISD::FP_TO_UINT, 0),
+  X86_INTRINSIC_DATA(avx512_mask_cvttps2udq_256, INTR_TYPE_1OP_MASK,
+                     ISD::FP_TO_UINT, 0),
+  X86_INTRINSIC_DATA(avx512_mask_cvttps2udq_512, INTR_TYPE_1OP_MASK,
+                     ISD::FP_TO_UINT, ISD::FP_TO_UINT),
+  X86_INTRINSIC_DATA(avx512_mask_cvttps2uqq_128, INTR_TYPE_1OP_MASK,
+                     ISD::FP_TO_UINT, 0),
+  X86_INTRINSIC_DATA(avx512_mask_cvttps2uqq_256, INTR_TYPE_1OP_MASK,
+                     ISD::FP_TO_UINT, 0),
+  X86_INTRINSIC_DATA(avx512_mask_cvttps2uqq_512, INTR_TYPE_1OP_MASK,
+                     ISD::FP_TO_UINT, ISD::FP_TO_UINT),
+  X86_INTRINSIC_DATA(avx512_mask_cvtudq2pd_128, INTR_TYPE_1OP_MASK,
+                     X86ISD::CVTUDQ2PD, 0),
+  X86_INTRINSIC_DATA(avx512_mask_cvtudq2pd_256, INTR_TYPE_1OP_MASK,
+                     ISD::UINT_TO_FP, 0),
+  X86_INTRINSIC_DATA(avx512_mask_cvtudq2pd_512, INTR_TYPE_1OP_MASK,
+                     ISD::UINT_TO_FP, 0), // no rm
+  X86_INTRINSIC_DATA(avx512_mask_cvtudq2ps_128, INTR_TYPE_1OP_MASK,
+                     ISD::UINT_TO_FP, 0),
+  X86_INTRINSIC_DATA(avx512_mask_cvtudq2ps_256, INTR_TYPE_1OP_MASK,
+                     ISD::UINT_TO_FP, 0),
+  X86_INTRINSIC_DATA(avx512_mask_cvtudq2ps_512, INTR_TYPE_1OP_MASK,
+                     ISD::UINT_TO_FP, ISD::UINT_TO_FP),
+  X86_INTRINSIC_DATA(avx512_mask_cvtuqq2pd_128, INTR_TYPE_1OP_MASK,
+                     ISD::UINT_TO_FP, 0),
+  X86_INTRINSIC_DATA(avx512_mask_cvtuqq2pd_256, INTR_TYPE_1OP_MASK,
+                     ISD::UINT_TO_FP, 0),
+  X86_INTRINSIC_DATA(avx512_mask_cvtuqq2pd_512, INTR_TYPE_1OP_MASK,
+                     ISD::UINT_TO_FP, ISD::UINT_TO_FP),
+  X86_INTRINSIC_DATA(avx512_mask_cvtuqq2ps_128, INTR_TYPE_1OP_MASK,
+                     ISD::UINT_TO_FP, 0),
+  X86_INTRINSIC_DATA(avx512_mask_cvtuqq2ps_256, INTR_TYPE_1OP_MASK,
+                     ISD::UINT_TO_FP, 0),
+  X86_INTRINSIC_DATA(avx512_mask_cvtuqq2ps_512, INTR_TYPE_1OP_MASK,
+                     ISD::UINT_TO_FP, ISD::UINT_TO_FP),
+
+  X86_INTRINSIC_DATA(avx512_mask_div_pd_128, INTR_TYPE_2OP_MASK, ISD::FDIV, 0),
+  X86_INTRINSIC_DATA(avx512_mask_div_pd_256, INTR_TYPE_2OP_MASK, ISD::FDIV, 0),
   X86_INTRINSIC_DATA(avx512_mask_div_pd_512, INTR_TYPE_2OP_MASK, ISD::FDIV,
                      X86ISD::FDIV_RND),
+  X86_INTRINSIC_DATA(avx512_mask_div_ps_128, INTR_TYPE_2OP_MASK, ISD::FDIV, 0),
+  X86_INTRINSIC_DATA(avx512_mask_div_ps_256, INTR_TYPE_2OP_MASK, ISD::FDIV, 0),
   X86_INTRINSIC_DATA(avx512_mask_div_ps_512, INTR_TYPE_2OP_MASK, ISD::FDIV,
                      X86ISD::FDIV_RND),
+  X86_INTRINSIC_DATA(avx512_mask_div_sd_round, INTR_TYPE_SCALAR_MASK_RM, ISD::FDIV,
+                     X86ISD::FDIV_RND),
+  X86_INTRINSIC_DATA(avx512_mask_div_ss_round, INTR_TYPE_SCALAR_MASK_RM, ISD::FDIV,
+                     X86ISD::FDIV_RND),
   X86_INTRINSIC_DATA(avx512_mask_expand_d_128,  COMPRESS_EXPAND_IN_REG,
                      X86ISD::EXPAND, 0),
   X86_INTRINSIC_DATA(avx512_mask_expand_d_256,  COMPRESS_EXPAND_IN_REG,
@@ -329,15 +647,130 @@ static const IntrinsicData  IntrinsicsWithoutChain[] = {
                      X86ISD::EXPAND, 0),
   X86_INTRINSIC_DATA(avx512_mask_expand_q_512,  COMPRESS_EXPAND_IN_REG,
                      X86ISD::EXPAND, 0),
-  
+  X86_INTRINSIC_DATA(avx512_mask_getexp_pd_128, INTR_TYPE_1OP_MASK_RM,
+                     X86ISD::FGETEXP_RND, 0),
+  X86_INTRINSIC_DATA(avx512_mask_getexp_pd_256, INTR_TYPE_1OP_MASK_RM,
+                     X86ISD::FGETEXP_RND, 0),
+  X86_INTRINSIC_DATA(avx512_mask_getexp_pd_512, INTR_TYPE_1OP_MASK_RM,
+                     X86ISD::FGETEXP_RND, 0),
+  X86_INTRINSIC_DATA(avx512_mask_getexp_ps_128, INTR_TYPE_1OP_MASK_RM,
+                     X86ISD::FGETEXP_RND, 0),
+  X86_INTRINSIC_DATA(avx512_mask_getexp_ps_256, INTR_TYPE_1OP_MASK_RM,
+                     X86ISD::FGETEXP_RND, 0),
+  X86_INTRINSIC_DATA(avx512_mask_getexp_ps_512, INTR_TYPE_1OP_MASK_RM,
+                     X86ISD::FGETEXP_RND, 0),
+  X86_INTRINSIC_DATA(avx512_mask_getexp_sd, INTR_TYPE_SCALAR_MASK_RM,
+                     X86ISD::FGETEXP_RND, 0),
+  X86_INTRINSIC_DATA(avx512_mask_getexp_ss, INTR_TYPE_SCALAR_MASK_RM,
+                    X86ISD::FGETEXP_RND, 0),
+  X86_INTRINSIC_DATA(avx512_mask_max_pd_128, INTR_TYPE_2OP_MASK, X86ISD::FMAX, 0),
+  X86_INTRINSIC_DATA(avx512_mask_max_pd_256, INTR_TYPE_2OP_MASK, X86ISD::FMAX, 0),
+  X86_INTRINSIC_DATA(avx512_mask_max_pd_512, INTR_TYPE_2OP_MASK, X86ISD::FMAX,
+                     X86ISD::FMAX_RND),
+  X86_INTRINSIC_DATA(avx512_mask_max_ps_128, INTR_TYPE_2OP_MASK, X86ISD::FMAX, 0),
+  X86_INTRINSIC_DATA(avx512_mask_max_ps_256, INTR_TYPE_2OP_MASK, X86ISD::FMAX, 0),
+  X86_INTRINSIC_DATA(avx512_mask_max_ps_512, INTR_TYPE_2OP_MASK, X86ISD::FMAX,
+                     X86ISD::FMAX_RND),
+  X86_INTRINSIC_DATA(avx512_mask_max_sd_round, INTR_TYPE_SCALAR_MASK_RM, X86ISD::FMAX,
+  X86ISD::FMAX_RND),
+  X86_INTRINSIC_DATA(avx512_mask_max_ss_round, INTR_TYPE_SCALAR_MASK_RM, X86ISD::FMAX,
+  X86ISD::FMAX_RND),
+  X86_INTRINSIC_DATA(avx512_mask_min_pd_128, INTR_TYPE_2OP_MASK, X86ISD::FMIN, 0),
+  X86_INTRINSIC_DATA(avx512_mask_min_pd_256, INTR_TYPE_2OP_MASK, X86ISD::FMIN, 0),
+  X86_INTRINSIC_DATA(avx512_mask_min_pd_512, INTR_TYPE_2OP_MASK, X86ISD::FMIN,
+                     X86ISD::FMIN_RND),
+  X86_INTRINSIC_DATA(avx512_mask_min_ps_128, INTR_TYPE_2OP_MASK, X86ISD::FMIN, 0),
+  X86_INTRINSIC_DATA(avx512_mask_min_ps_256, INTR_TYPE_2OP_MASK, X86ISD::FMIN, 0),
+  X86_INTRINSIC_DATA(avx512_mask_min_ps_512, INTR_TYPE_2OP_MASK, X86ISD::FMIN,
+                     X86ISD::FMIN_RND),
+  X86_INTRINSIC_DATA(avx512_mask_min_sd_round, INTR_TYPE_SCALAR_MASK_RM, X86ISD::FMIN,
+  X86ISD::FMIN_RND),
+  X86_INTRINSIC_DATA(avx512_mask_min_ss_round, INTR_TYPE_SCALAR_MASK_RM, X86ISD::FMIN,
+  X86ISD::FMIN_RND),
+  X86_INTRINSIC_DATA(avx512_mask_mul_pd_128, INTR_TYPE_2OP_MASK, ISD::FMUL, 0),
+  X86_INTRINSIC_DATA(avx512_mask_mul_pd_256, INTR_TYPE_2OP_MASK, ISD::FMUL, 0),
   X86_INTRINSIC_DATA(avx512_mask_mul_pd_512, INTR_TYPE_2OP_MASK, ISD::FMUL,
                      X86ISD::FMUL_RND),
+  X86_INTRINSIC_DATA(avx512_mask_mul_ps_128, INTR_TYPE_2OP_MASK, ISD::FMUL, 0),
+  X86_INTRINSIC_DATA(avx512_mask_mul_ps_256, INTR_TYPE_2OP_MASK, ISD::FMUL, 0),
   X86_INTRINSIC_DATA(avx512_mask_mul_ps_512, INTR_TYPE_2OP_MASK, ISD::FMUL,
                      X86ISD::FMUL_RND),
+  X86_INTRINSIC_DATA(avx512_mask_mul_sd_round, INTR_TYPE_SCALAR_MASK_RM, ISD::FMUL,
+  X86ISD::FMUL_RND),
+  X86_INTRINSIC_DATA(avx512_mask_mul_ss_round, INTR_TYPE_SCALAR_MASK_RM, ISD::FMUL,
+  X86ISD::FMUL_RND),
+  X86_INTRINSIC_DATA(avx512_mask_or_pd_128, INTR_TYPE_2OP_MASK, X86ISD::FOR, 0),
+  X86_INTRINSIC_DATA(avx512_mask_or_pd_256, INTR_TYPE_2OP_MASK, X86ISD::FOR, 0),
+  X86_INTRINSIC_DATA(avx512_mask_or_pd_512, INTR_TYPE_2OP_MASK, X86ISD::FOR, 0),
+  X86_INTRINSIC_DATA(avx512_mask_or_ps_128, INTR_TYPE_2OP_MASK, X86ISD::FOR, 0),
+  X86_INTRINSIC_DATA(avx512_mask_or_ps_256, INTR_TYPE_2OP_MASK, X86ISD::FOR, 0),
+  X86_INTRINSIC_DATA(avx512_mask_or_ps_512, INTR_TYPE_2OP_MASK, X86ISD::FOR, 0),
+  X86_INTRINSIC_DATA(avx512_mask_pabs_b_128, INTR_TYPE_1OP_MASK, X86ISD::ABS, 0),
+  X86_INTRINSIC_DATA(avx512_mask_pabs_b_256, INTR_TYPE_1OP_MASK, X86ISD::ABS, 0),
+  X86_INTRINSIC_DATA(avx512_mask_pabs_b_512, INTR_TYPE_1OP_MASK, X86ISD::ABS, 0),
+  X86_INTRINSIC_DATA(avx512_mask_pabs_d_128, INTR_TYPE_1OP_MASK, X86ISD::ABS, 0),
+  X86_INTRINSIC_DATA(avx512_mask_pabs_d_256, INTR_TYPE_1OP_MASK, X86ISD::ABS, 0),
+  X86_INTRINSIC_DATA(avx512_mask_pabs_d_512, INTR_TYPE_1OP_MASK, X86ISD::ABS, 0),
+  X86_INTRINSIC_DATA(avx512_mask_pabs_q_128, INTR_TYPE_1OP_MASK, X86ISD::ABS, 0),
+  X86_INTRINSIC_DATA(avx512_mask_pabs_q_256, INTR_TYPE_1OP_MASK, X86ISD::ABS, 0),
+  X86_INTRINSIC_DATA(avx512_mask_pabs_q_512, INTR_TYPE_1OP_MASK, X86ISD::ABS, 0),
+  X86_INTRINSIC_DATA(avx512_mask_pabs_w_128, INTR_TYPE_1OP_MASK, X86ISD::ABS, 0),
+  X86_INTRINSIC_DATA(avx512_mask_pabs_w_256, INTR_TYPE_1OP_MASK, X86ISD::ABS, 0),
+  X86_INTRINSIC_DATA(avx512_mask_pabs_w_512, INTR_TYPE_1OP_MASK, X86ISD::ABS, 0),
+  X86_INTRINSIC_DATA(avx512_mask_packssdw_128, INTR_TYPE_2OP_MASK, X86ISD::PACKSS, 0),
+  X86_INTRINSIC_DATA(avx512_mask_packssdw_256, INTR_TYPE_2OP_MASK, X86ISD::PACKSS, 0),
+  X86_INTRINSIC_DATA(avx512_mask_packssdw_512, INTR_TYPE_2OP_MASK, X86ISD::PACKSS, 0),
+  X86_INTRINSIC_DATA(avx512_mask_packsswb_128, INTR_TYPE_2OP_MASK, X86ISD::PACKSS, 0),
+  X86_INTRINSIC_DATA(avx512_mask_packsswb_256, INTR_TYPE_2OP_MASK, X86ISD::PACKSS, 0),
+  X86_INTRINSIC_DATA(avx512_mask_packsswb_512, INTR_TYPE_2OP_MASK, X86ISD::PACKSS, 0),
+  X86_INTRINSIC_DATA(avx512_mask_packusdw_128, INTR_TYPE_2OP_MASK, X86ISD::PACKUS, 0),
+  X86_INTRINSIC_DATA(avx512_mask_packusdw_256, INTR_TYPE_2OP_MASK, X86ISD::PACKUS, 0),
+  X86_INTRINSIC_DATA(avx512_mask_packusdw_512, INTR_TYPE_2OP_MASK, X86ISD::PACKUS, 0),
+  X86_INTRINSIC_DATA(avx512_mask_packuswb_128, INTR_TYPE_2OP_MASK, X86ISD::PACKUS, 0),
+  X86_INTRINSIC_DATA(avx512_mask_packuswb_256, INTR_TYPE_2OP_MASK, X86ISD::PACKUS, 0),
+  X86_INTRINSIC_DATA(avx512_mask_packuswb_512, INTR_TYPE_2OP_MASK, X86ISD::PACKUS, 0),
+  X86_INTRINSIC_DATA(avx512_mask_padd_b_128, INTR_TYPE_2OP_MASK, ISD::ADD, 0),
+  X86_INTRINSIC_DATA(avx512_mask_padd_b_256, INTR_TYPE_2OP_MASK, ISD::ADD, 0),
+  X86_INTRINSIC_DATA(avx512_mask_padd_b_512, INTR_TYPE_2OP_MASK, ISD::ADD, 0),
+  X86_INTRINSIC_DATA(avx512_mask_padd_d_128, INTR_TYPE_2OP_MASK, ISD::ADD, 0),
+  X86_INTRINSIC_DATA(avx512_mask_padd_d_256, INTR_TYPE_2OP_MASK, ISD::ADD, 0),
   X86_INTRINSIC_DATA(avx512_mask_padd_d_512, INTR_TYPE_2OP_MASK, ISD::ADD, 0),
+  X86_INTRINSIC_DATA(avx512_mask_padd_q_128, INTR_TYPE_2OP_MASK, ISD::ADD, 0),
+  X86_INTRINSIC_DATA(avx512_mask_padd_q_256, INTR_TYPE_2OP_MASK, ISD::ADD, 0),
   X86_INTRINSIC_DATA(avx512_mask_padd_q_512, INTR_TYPE_2OP_MASK, ISD::ADD, 0),
+  X86_INTRINSIC_DATA(avx512_mask_padd_w_128, INTR_TYPE_2OP_MASK, ISD::ADD, 0),
+  X86_INTRINSIC_DATA(avx512_mask_padd_w_256, INTR_TYPE_2OP_MASK, ISD::ADD, 0),
+  X86_INTRINSIC_DATA(avx512_mask_padd_w_512, INTR_TYPE_2OP_MASK, ISD::ADD, 0),
+  X86_INTRINSIC_DATA(avx512_mask_padds_b_128, INTR_TYPE_2OP_MASK, X86ISD::ADDS, 0),
+  X86_INTRINSIC_DATA(avx512_mask_padds_b_256, INTR_TYPE_2OP_MASK, X86ISD::ADDS, 0),
+  X86_INTRINSIC_DATA(avx512_mask_padds_b_512, INTR_TYPE_2OP_MASK, X86ISD::ADDS, 0),
+  X86_INTRINSIC_DATA(avx512_mask_padds_w_128, INTR_TYPE_2OP_MASK, X86ISD::ADDS, 0),
+  X86_INTRINSIC_DATA(avx512_mask_padds_w_256, INTR_TYPE_2OP_MASK, X86ISD::ADDS, 0),
+  X86_INTRINSIC_DATA(avx512_mask_padds_w_512, INTR_TYPE_2OP_MASK, X86ISD::ADDS, 0),
+  X86_INTRINSIC_DATA(avx512_mask_paddus_b_128, INTR_TYPE_2OP_MASK, X86ISD::ADDUS, 0),
+  X86_INTRINSIC_DATA(avx512_mask_paddus_b_256, INTR_TYPE_2OP_MASK, X86ISD::ADDUS, 0),
+  X86_INTRINSIC_DATA(avx512_mask_paddus_b_512, INTR_TYPE_2OP_MASK, X86ISD::ADDUS, 0),
+  X86_INTRINSIC_DATA(avx512_mask_paddus_w_128, INTR_TYPE_2OP_MASK, X86ISD::ADDUS, 0),
+  X86_INTRINSIC_DATA(avx512_mask_paddus_w_256, INTR_TYPE_2OP_MASK, X86ISD::ADDUS, 0),
+  X86_INTRINSIC_DATA(avx512_mask_paddus_w_512, INTR_TYPE_2OP_MASK, X86ISD::ADDUS, 0),
+  X86_INTRINSIC_DATA(avx512_mask_pand_d_128, INTR_TYPE_2OP_MASK, ISD::AND, 0),
+  X86_INTRINSIC_DATA(avx512_mask_pand_d_256, INTR_TYPE_2OP_MASK, ISD::AND, 0),
   X86_INTRINSIC_DATA(avx512_mask_pand_d_512, INTR_TYPE_2OP_MASK, ISD::AND, 0),
+  X86_INTRINSIC_DATA(avx512_mask_pand_q_128, INTR_TYPE_2OP_MASK, ISD::AND, 0),
+  X86_INTRINSIC_DATA(avx512_mask_pand_q_256, INTR_TYPE_2OP_MASK, ISD::AND, 0),
   X86_INTRINSIC_DATA(avx512_mask_pand_q_512, INTR_TYPE_2OP_MASK, ISD::AND, 0),
+  X86_INTRINSIC_DATA(avx512_mask_pandn_d_128, INTR_TYPE_2OP_MASK, X86ISD::ANDNP, 0),
+  X86_INTRINSIC_DATA(avx512_mask_pandn_d_256, INTR_TYPE_2OP_MASK, X86ISD::ANDNP, 0),
+  X86_INTRINSIC_DATA(avx512_mask_pandn_d_512, INTR_TYPE_2OP_MASK, X86ISD::ANDNP, 0),
+  X86_INTRINSIC_DATA(avx512_mask_pandn_q_128, INTR_TYPE_2OP_MASK, X86ISD::ANDNP, 0),
+  X86_INTRINSIC_DATA(avx512_mask_pandn_q_256, INTR_TYPE_2OP_MASK, X86ISD::ANDNP, 0),
+  X86_INTRINSIC_DATA(avx512_mask_pandn_q_512, INTR_TYPE_2OP_MASK, X86ISD::ANDNP, 0),
+  X86_INTRINSIC_DATA(avx512_mask_pavg_b_128, INTR_TYPE_2OP_MASK, X86ISD::AVG, 0),
+  X86_INTRINSIC_DATA(avx512_mask_pavg_b_256, INTR_TYPE_2OP_MASK, X86ISD::AVG, 0),
+  X86_INTRINSIC_DATA(avx512_mask_pavg_b_512, INTR_TYPE_2OP_MASK, X86ISD::AVG, 0),
+  X86_INTRINSIC_DATA(avx512_mask_pavg_w_128, INTR_TYPE_2OP_MASK, X86ISD::AVG, 0),
+  X86_INTRINSIC_DATA(avx512_mask_pavg_w_256, INTR_TYPE_2OP_MASK, X86ISD::AVG, 0),
+  X86_INTRINSIC_DATA(avx512_mask_pavg_w_512, INTR_TYPE_2OP_MASK, X86ISD::AVG, 0),
   X86_INTRINSIC_DATA(avx512_mask_pcmpeq_b_128,  CMP_MASK,  X86ISD::PCMPEQM, 0),
   X86_INTRINSIC_DATA(avx512_mask_pcmpeq_b_256,  CMP_MASK,  X86ISD::PCMPEQM, 0),
   X86_INTRINSIC_DATA(avx512_mask_pcmpeq_b_512,  CMP_MASK,  X86ISD::PCMPEQM, 0),
@@ -362,12 +795,216 @@ static const IntrinsicData  IntrinsicsWithoutChain[] = {
   X86_INTRINSIC_DATA(avx512_mask_pcmpgt_w_128,  CMP_MASK,  X86ISD::PCMPGTM, 0),
   X86_INTRINSIC_DATA(avx512_mask_pcmpgt_w_256,  CMP_MASK,  X86ISD::PCMPGTM, 0),
   X86_INTRINSIC_DATA(avx512_mask_pcmpgt_w_512,  CMP_MASK,  X86ISD::PCMPGTM, 0),
+  X86_INTRINSIC_DATA(avx512_mask_pmaddubs_w_128, INTR_TYPE_2OP_MASK,
+                     X86ISD::VPMADDUBSW, 0),
+  X86_INTRINSIC_DATA(avx512_mask_pmaddubs_w_256, INTR_TYPE_2OP_MASK,
+                     X86ISD::VPMADDUBSW, 0),
+  X86_INTRINSIC_DATA(avx512_mask_pmaddubs_w_512, INTR_TYPE_2OP_MASK,
+                     X86ISD::VPMADDUBSW, 0),
+  X86_INTRINSIC_DATA(avx512_mask_pmaddw_d_128, INTR_TYPE_2OP_MASK,
+                     X86ISD::VPMADDWD, 0),
+  X86_INTRINSIC_DATA(avx512_mask_pmaddw_d_256, INTR_TYPE_2OP_MASK,
+                     X86ISD::VPMADDWD, 0),
+  X86_INTRINSIC_DATA(avx512_mask_pmaddw_d_512, INTR_TYPE_2OP_MASK,
+                     X86ISD::VPMADDWD, 0),
+  X86_INTRINSIC_DATA(avx512_mask_pmaxs_b_128, INTR_TYPE_2OP_MASK, ISD::SMAX, 0),
+  X86_INTRINSIC_DATA(avx512_mask_pmaxs_b_256, INTR_TYPE_2OP_MASK, ISD::SMAX, 0),
+  X86_INTRINSIC_DATA(avx512_mask_pmaxs_b_512, INTR_TYPE_2OP_MASK, ISD::SMAX, 0),
+  X86_INTRINSIC_DATA(avx512_mask_pmaxs_d_128, INTR_TYPE_2OP_MASK, ISD::SMAX, 0),
+  X86_INTRINSIC_DATA(avx512_mask_pmaxs_d_256, INTR_TYPE_2OP_MASK, ISD::SMAX, 0),
+  X86_INTRINSIC_DATA(avx512_mask_pmaxs_d_512, INTR_TYPE_2OP_MASK, ISD::SMAX, 0),
+  X86_INTRINSIC_DATA(avx512_mask_pmaxs_q_128, INTR_TYPE_2OP_MASK, ISD::SMAX, 0),
+  X86_INTRINSIC_DATA(avx512_mask_pmaxs_q_256, INTR_TYPE_2OP_MASK, ISD::SMAX, 0),
+  X86_INTRINSIC_DATA(avx512_mask_pmaxs_q_512, INTR_TYPE_2OP_MASK, ISD::SMAX, 0),
+  X86_INTRINSIC_DATA(avx512_mask_pmaxs_w_128, INTR_TYPE_2OP_MASK, ISD::SMAX, 0),
+  X86_INTRINSIC_DATA(avx512_mask_pmaxs_w_256, INTR_TYPE_2OP_MASK, ISD::SMAX, 0),
+  X86_INTRINSIC_DATA(avx512_mask_pmaxs_w_512, INTR_TYPE_2OP_MASK, ISD::SMAX, 0),
+  X86_INTRINSIC_DATA(avx512_mask_pmaxu_b_128, INTR_TYPE_2OP_MASK, ISD::UMAX, 0),
+  X86_INTRINSIC_DATA(avx512_mask_pmaxu_b_256, INTR_TYPE_2OP_MASK, ISD::UMAX, 0),
+  X86_INTRINSIC_DATA(avx512_mask_pmaxu_b_512, INTR_TYPE_2OP_MASK, ISD::UMAX, 0),
+  X86_INTRINSIC_DATA(avx512_mask_pmaxu_d_128, INTR_TYPE_2OP_MASK, ISD::UMAX, 0),
+  X86_INTRINSIC_DATA(avx512_mask_pmaxu_d_256, INTR_TYPE_2OP_MASK, ISD::UMAX, 0),
+  X86_INTRINSIC_DATA(avx512_mask_pmaxu_d_512, INTR_TYPE_2OP_MASK, ISD::UMAX, 0),
+  X86_INTRINSIC_DATA(avx512_mask_pmaxu_q_128, INTR_TYPE_2OP_MASK, ISD::UMAX, 0),
+  X86_INTRINSIC_DATA(avx512_mask_pmaxu_q_256, INTR_TYPE_2OP_MASK, ISD::UMAX, 0),
+  X86_INTRINSIC_DATA(avx512_mask_pmaxu_q_512, INTR_TYPE_2OP_MASK, ISD::UMAX, 0),
+  X86_INTRINSIC_DATA(avx512_mask_pmaxu_w_128, INTR_TYPE_2OP_MASK, ISD::UMAX, 0),
+  X86_INTRINSIC_DATA(avx512_mask_pmaxu_w_256, INTR_TYPE_2OP_MASK, ISD::UMAX, 0),
+  X86_INTRINSIC_DATA(avx512_mask_pmaxu_w_512, INTR_TYPE_2OP_MASK, ISD::UMAX, 0),
+  X86_INTRINSIC_DATA(avx512_mask_pmins_b_128, INTR_TYPE_2OP_MASK, ISD::SMIN, 0),
+  X86_INTRINSIC_DATA(avx512_mask_pmins_b_256, INTR_TYPE_2OP_MASK, ISD::SMIN, 0),
+  X86_INTRINSIC_DATA(avx512_mask_pmins_b_512, INTR_TYPE_2OP_MASK, ISD::SMIN, 0),
+  X86_INTRINSIC_DATA(avx512_mask_pmins_d_128, INTR_TYPE_2OP_MASK, ISD::SMIN, 0),
+  X86_INTRINSIC_DATA(avx512_mask_pmins_d_256, INTR_TYPE_2OP_MASK, ISD::SMIN, 0),
+  X86_INTRINSIC_DATA(avx512_mask_pmins_d_512, INTR_TYPE_2OP_MASK, ISD::SMIN, 0),
+  X86_INTRINSIC_DATA(avx512_mask_pmins_q_128, INTR_TYPE_2OP_MASK, ISD::SMIN, 0),
+  X86_INTRINSIC_DATA(avx512_mask_pmins_q_256, INTR_TYPE_2OP_MASK, ISD::SMIN, 0),
+  X86_INTRINSIC_DATA(avx512_mask_pmins_q_512, INTR_TYPE_2OP_MASK, ISD::SMIN, 0),
+  X86_INTRINSIC_DATA(avx512_mask_pmins_w_128, INTR_TYPE_2OP_MASK, ISD::SMIN, 0),
+  X86_INTRINSIC_DATA(avx512_mask_pmins_w_256, INTR_TYPE_2OP_MASK, ISD::SMIN, 0),
+  X86_INTRINSIC_DATA(avx512_mask_pmins_w_512, INTR_TYPE_2OP_MASK, ISD::SMIN, 0),
+  X86_INTRINSIC_DATA(avx512_mask_pminu_b_128, INTR_TYPE_2OP_MASK, ISD::UMIN, 0),
+  X86_INTRINSIC_DATA(avx512_mask_pminu_b_256, INTR_TYPE_2OP_MASK, ISD::UMIN, 0),
+  X86_INTRINSIC_DATA(avx512_mask_pminu_b_512, INTR_TYPE_2OP_MASK, ISD::UMIN, 0),
+  X86_INTRINSIC_DATA(avx512_mask_pminu_d_128, INTR_TYPE_2OP_MASK, ISD::UMIN, 0),
+  X86_INTRINSIC_DATA(avx512_mask_pminu_d_256, INTR_TYPE_2OP_MASK, ISD::UMIN, 0),
+  X86_INTRINSIC_DATA(avx512_mask_pminu_d_512, INTR_TYPE_2OP_MASK, ISD::UMIN, 0),
+  X86_INTRINSIC_DATA(avx512_mask_pminu_q_128, INTR_TYPE_2OP_MASK, ISD::UMIN, 0),
+  X86_INTRINSIC_DATA(avx512_mask_pminu_q_256, INTR_TYPE_2OP_MASK, ISD::UMIN, 0),
+  X86_INTRINSIC_DATA(avx512_mask_pminu_q_512, INTR_TYPE_2OP_MASK, ISD::UMIN, 0),
+  X86_INTRINSIC_DATA(avx512_mask_pminu_w_128, INTR_TYPE_2OP_MASK, ISD::UMIN, 0),
+  X86_INTRINSIC_DATA(avx512_mask_pminu_w_256, INTR_TYPE_2OP_MASK, ISD::UMIN, 0),
+  X86_INTRINSIC_DATA(avx512_mask_pminu_w_512, INTR_TYPE_2OP_MASK, ISD::UMIN, 0),
+  X86_INTRINSIC_DATA(avx512_mask_pmov_db_128, INTR_TYPE_1OP_MASK,
+                     X86ISD::VTRUNC, 0),
+  X86_INTRINSIC_DATA(avx512_mask_pmov_db_256, INTR_TYPE_1OP_MASK,
+                     X86ISD::VTRUNC, 0),
+  X86_INTRINSIC_DATA(avx512_mask_pmov_db_512, INTR_TYPE_1OP_MASK,
+                     X86ISD::VTRUNC, 0),
+  X86_INTRINSIC_DATA(avx512_mask_pmov_dw_128, INTR_TYPE_1OP_MASK,
+                     X86ISD::VTRUNC, 0),
+  X86_INTRINSIC_DATA(avx512_mask_pmov_dw_256, INTR_TYPE_1OP_MASK,
+                     X86ISD::VTRUNC, 0),
+  X86_INTRINSIC_DATA(avx512_mask_pmov_dw_512, INTR_TYPE_1OP_MASK,
+                     X86ISD::VTRUNC, 0),
+  X86_INTRINSIC_DATA(avx512_mask_pmov_qb_128, INTR_TYPE_1OP_MASK,
+                     X86ISD::VTRUNC, 0),
+  X86_INTRINSIC_DATA(avx512_mask_pmov_qb_256, INTR_TYPE_1OP_MASK,
+                     X86ISD::VTRUNC, 0),
+  X86_INTRINSIC_DATA(avx512_mask_pmov_qb_512, INTR_TYPE_1OP_MASK,
+                     X86ISD::VTRUNC, 0),
+  X86_INTRINSIC_DATA(avx512_mask_pmov_qd_128, INTR_TYPE_1OP_MASK,
+                     X86ISD::VTRUNC, 0),
+  X86_INTRINSIC_DATA(avx512_mask_pmov_qd_256, INTR_TYPE_1OP_MASK,
+                     X86ISD::VTRUNC, 0),
+  X86_INTRINSIC_DATA(avx512_mask_pmov_qd_512, INTR_TYPE_1OP_MASK,
+                     X86ISD::VTRUNC, 0),
+  X86_INTRINSIC_DATA(avx512_mask_pmov_qw_128, INTR_TYPE_1OP_MASK,
+                     X86ISD::VTRUNC, 0),
+  X86_INTRINSIC_DATA(avx512_mask_pmov_qw_256, INTR_TYPE_1OP_MASK,
+                     X86ISD::VTRUNC, 0),
+  X86_INTRINSIC_DATA(avx512_mask_pmov_qw_512, INTR_TYPE_1OP_MASK,
+                     X86ISD::VTRUNC, 0),
+  X86_INTRINSIC_DATA(avx512_mask_pmov_wb_128, INTR_TYPE_1OP_MASK,
+                     X86ISD::VTRUNC, 0),
+  X86_INTRINSIC_DATA(avx512_mask_pmov_wb_256, INTR_TYPE_1OP_MASK,
+                     X86ISD::VTRUNC, 0),
+  X86_INTRINSIC_DATA(avx512_mask_pmov_wb_512, INTR_TYPE_1OP_MASK,
+                     X86ISD::VTRUNC, 0),
+  X86_INTRINSIC_DATA(avx512_mask_pmovs_db_128, INTR_TYPE_1OP_MASK,
+                     X86ISD::VTRUNCS, 0),
+  X86_INTRINSIC_DATA(avx512_mask_pmovs_db_256, INTR_TYPE_1OP_MASK,
+                     X86ISD::VTRUNCS, 0),
+  X86_INTRINSIC_DATA(avx512_mask_pmovs_db_512, INTR_TYPE_1OP_MASK,
+                     X86ISD::VTRUNCS, 0),
+  X86_INTRINSIC_DATA(avx512_mask_pmovs_dw_128, INTR_TYPE_1OP_MASK,
+                     X86ISD::VTRUNCS, 0),
+  X86_INTRINSIC_DATA(avx512_mask_pmovs_dw_256, INTR_TYPE_1OP_MASK,
+                     X86ISD::VTRUNCS, 0),
+  X86_INTRINSIC_DATA(avx512_mask_pmovs_dw_512, INTR_TYPE_1OP_MASK,
+                     X86ISD::VTRUNCS, 0),
+  X86_INTRINSIC_DATA(avx512_mask_pmovs_qb_128, INTR_TYPE_1OP_MASK,
+                     X86ISD::VTRUNCS, 0),
+  X86_INTRINSIC_DATA(avx512_mask_pmovs_qb_256, INTR_TYPE_1OP_MASK,
+                     X86ISD::VTRUNCS, 0),
+  X86_INTRINSIC_DATA(avx512_mask_pmovs_qb_512, INTR_TYPE_1OP_MASK,
+                     X86ISD::VTRUNCS, 0),
+  X86_INTRINSIC_DATA(avx512_mask_pmovs_qd_128, INTR_TYPE_1OP_MASK,
+                     X86ISD::VTRUNCS, 0),
+  X86_INTRINSIC_DATA(avx512_mask_pmovs_qd_256, INTR_TYPE_1OP_MASK,
+                     X86ISD::VTRUNCS, 0),
+  X86_INTRINSIC_DATA(avx512_mask_pmovs_qd_512, INTR_TYPE_1OP_MASK,
+                     X86ISD::VTRUNCS, 0),
+  X86_INTRINSIC_DATA(avx512_mask_pmovs_qw_128, INTR_TYPE_1OP_MASK,
+                     X86ISD::VTRUNCS, 0),
+  X86_INTRINSIC_DATA(avx512_mask_pmovs_qw_256, INTR_TYPE_1OP_MASK,
+                     X86ISD::VTRUNCS, 0),
+  X86_INTRINSIC_DATA(avx512_mask_pmovs_qw_512, INTR_TYPE_1OP_MASK,
+                     X86ISD::VTRUNCS, 0),
+  X86_INTRINSIC_DATA(avx512_mask_pmovs_wb_128, INTR_TYPE_1OP_MASK,
+                     X86ISD::VTRUNCS, 0),
+  X86_INTRINSIC_DATA(avx512_mask_pmovs_wb_256, INTR_TYPE_1OP_MASK,
+                     X86ISD::VTRUNCS, 0),
+  X86_INTRINSIC_DATA(avx512_mask_pmovs_wb_512, INTR_TYPE_1OP_MASK,
+                     X86ISD::VTRUNCS, 0),
+  X86_INTRINSIC_DATA(avx512_mask_pmovus_db_128, INTR_TYPE_1OP_MASK,
+                     X86ISD::VTRUNCUS, 0),
+  X86_INTRINSIC_DATA(avx512_mask_pmovus_db_256, INTR_TYPE_1OP_MASK,
+                     X86ISD::VTRUNCUS, 0),
+  X86_INTRINSIC_DATA(avx512_mask_pmovus_db_512, INTR_TYPE_1OP_MASK,
+                     X86ISD::VTRUNCUS, 0),
+  X86_INTRINSIC_DATA(avx512_mask_pmovus_dw_128, INTR_TYPE_1OP_MASK,
+                     X86ISD::VTRUNCUS, 0),
+  X86_INTRINSIC_DATA(avx512_mask_pmovus_dw_256, INTR_TYPE_1OP_MASK,
+                     X86ISD::VTRUNCUS, 0),
+  X86_INTRINSIC_DATA(avx512_mask_pmovus_dw_512, INTR_TYPE_1OP_MASK,
+                     X86ISD::VTRUNCUS, 0),
+  X86_INTRINSIC_DATA(avx512_mask_pmovus_qb_128, INTR_TYPE_1OP_MASK,
+                     X86ISD::VTRUNCUS, 0),
+  X86_INTRINSIC_DATA(avx512_mask_pmovus_qb_256, INTR_TYPE_1OP_MASK,
+                     X86ISD::VTRUNCUS, 0),
+  X86_INTRINSIC_DATA(avx512_mask_pmovus_qb_512, INTR_TYPE_1OP_MASK,
+                     X86ISD::VTRUNCUS, 0),
+  X86_INTRINSIC_DATA(avx512_mask_pmovus_qd_128, INTR_TYPE_1OP_MASK,
+                     X86ISD::VTRUNCUS, 0),
+  X86_INTRINSIC_DATA(avx512_mask_pmovus_qd_256, INTR_TYPE_1OP_MASK,
+                     X86ISD::VTRUNCUS, 0),
+  X86_INTRINSIC_DATA(avx512_mask_pmovus_qd_512, INTR_TYPE_1OP_MASK,
+                     X86ISD::VTRUNCUS, 0),
+  X86_INTRINSIC_DATA(avx512_mask_pmovus_qw_128, INTR_TYPE_1OP_MASK,
+                     X86ISD::VTRUNCUS, 0),
+  X86_INTRINSIC_DATA(avx512_mask_pmovus_qw_256, INTR_TYPE_1OP_MASK,
+                     X86ISD::VTRUNCUS, 0),
+  X86_INTRINSIC_DATA(avx512_mask_pmovus_qw_512, INTR_TYPE_1OP_MASK,
+                     X86ISD::VTRUNCUS, 0),
+  X86_INTRINSIC_DATA(avx512_mask_pmovus_wb_128, INTR_TYPE_1OP_MASK,
+                     X86ISD::VTRUNCUS, 0),
+  X86_INTRINSIC_DATA(avx512_mask_pmovus_wb_256, INTR_TYPE_1OP_MASK,
+                     X86ISD::VTRUNCUS, 0),
+  X86_INTRINSIC_DATA(avx512_mask_pmovus_wb_512, INTR_TYPE_1OP_MASK,
+                     X86ISD::VTRUNCUS, 0),
+  X86_INTRINSIC_DATA(avx512_mask_pmul_dq_128, INTR_TYPE_2OP_MASK,
+                     X86ISD::PMULDQ, 0),
+  X86_INTRINSIC_DATA(avx512_mask_pmul_dq_256, INTR_TYPE_2OP_MASK,
+                     X86ISD::PMULDQ, 0),
   X86_INTRINSIC_DATA(avx512_mask_pmul_dq_512, INTR_TYPE_2OP_MASK,
                      X86ISD::PMULDQ, 0),
+  X86_INTRINSIC_DATA(avx512_mask_pmul_hr_sw_128, INTR_TYPE_2OP_MASK, X86ISD::MULHRS, 0),
+  X86_INTRINSIC_DATA(avx512_mask_pmul_hr_sw_256, INTR_TYPE_2OP_MASK, X86ISD::MULHRS, 0),
+  X86_INTRINSIC_DATA(avx512_mask_pmul_hr_sw_512, INTR_TYPE_2OP_MASK, X86ISD::MULHRS, 0),
+  X86_INTRINSIC_DATA(avx512_mask_pmulh_w_128, INTR_TYPE_2OP_MASK, ISD::MULHS, 0),
+  X86_INTRINSIC_DATA(avx512_mask_pmulh_w_256, INTR_TYPE_2OP_MASK, ISD::MULHS, 0),
+  X86_INTRINSIC_DATA(avx512_mask_pmulh_w_512, INTR_TYPE_2OP_MASK, ISD::MULHS, 0),
+  X86_INTRINSIC_DATA(avx512_mask_pmulhu_w_128, INTR_TYPE_2OP_MASK, ISD::MULHU, 0),
+  X86_INTRINSIC_DATA(avx512_mask_pmulhu_w_256, INTR_TYPE_2OP_MASK, ISD::MULHU, 0),
+  X86_INTRINSIC_DATA(avx512_mask_pmulhu_w_512, INTR_TYPE_2OP_MASK, ISD::MULHU, 0),
+  X86_INTRINSIC_DATA(avx512_mask_pmull_d_128, INTR_TYPE_2OP_MASK, ISD::MUL, 0),
+  X86_INTRINSIC_DATA(avx512_mask_pmull_d_256, INTR_TYPE_2OP_MASK, ISD::MUL, 0),
+  X86_INTRINSIC_DATA(avx512_mask_pmull_d_512, INTR_TYPE_2OP_MASK, ISD::MUL, 0),
+  X86_INTRINSIC_DATA(avx512_mask_pmull_q_128, INTR_TYPE_2OP_MASK, ISD::MUL, 0),
+  X86_INTRINSIC_DATA(avx512_mask_pmull_q_256, INTR_TYPE_2OP_MASK, ISD::MUL, 0),
+  X86_INTRINSIC_DATA(avx512_mask_pmull_q_512, INTR_TYPE_2OP_MASK, ISD::MUL, 0),
+  X86_INTRINSIC_DATA(avx512_mask_pmull_w_128, INTR_TYPE_2OP_MASK, ISD::MUL, 0),
+  X86_INTRINSIC_DATA(avx512_mask_pmull_w_256, INTR_TYPE_2OP_MASK, ISD::MUL, 0),
+  X86_INTRINSIC_DATA(avx512_mask_pmull_w_512, INTR_TYPE_2OP_MASK, ISD::MUL, 0),
+  X86_INTRINSIC_DATA(avx512_mask_pmulu_dq_128, INTR_TYPE_2OP_MASK,
+                     X86ISD::PMULUDQ, 0),
+  X86_INTRINSIC_DATA(avx512_mask_pmulu_dq_256, INTR_TYPE_2OP_MASK,
+                     X86ISD::PMULUDQ, 0),
   X86_INTRINSIC_DATA(avx512_mask_pmulu_dq_512, INTR_TYPE_2OP_MASK,
                      X86ISD::PMULUDQ, 0),
+  X86_INTRINSIC_DATA(avx512_mask_por_d_128, INTR_TYPE_2OP_MASK, ISD::OR, 0),
+  X86_INTRINSIC_DATA(avx512_mask_por_d_256, INTR_TYPE_2OP_MASK, ISD::OR, 0),
   X86_INTRINSIC_DATA(avx512_mask_por_d_512, INTR_TYPE_2OP_MASK, ISD::OR, 0),
+  X86_INTRINSIC_DATA(avx512_mask_por_q_128, INTR_TYPE_2OP_MASK, ISD::OR, 0),
+  X86_INTRINSIC_DATA(avx512_mask_por_q_256, INTR_TYPE_2OP_MASK, ISD::OR, 0),
   X86_INTRINSIC_DATA(avx512_mask_por_q_512, INTR_TYPE_2OP_MASK, ISD::OR, 0),
+  X86_INTRINSIC_DATA(avx512_mask_pshuf_b_128, INTR_TYPE_2OP_MASK,
+                     X86ISD::PSHUFB, 0),
+  X86_INTRINSIC_DATA(avx512_mask_pshuf_b_256, INTR_TYPE_2OP_MASK,
+                    X86ISD::PSHUFB, 0),
+  X86_INTRINSIC_DATA(avx512_mask_pshuf_b_512, INTR_TYPE_2OP_MASK,
+                    X86ISD::PSHUFB, 0),
   X86_INTRINSIC_DATA(avx512_mask_psll_d,        INTR_TYPE_2OP_MASK, X86ISD::VSHL, 0),
   X86_INTRINSIC_DATA(avx512_mask_psll_q,        INTR_TYPE_2OP_MASK, X86ISD::VSHL, 0),
   X86_INTRINSIC_DATA(avx512_mask_pslli_d,       VSHIFT_MASK, X86ISD::VSHLI, 0),
@@ -386,18 +1023,146 @@ static const IntrinsicData  IntrinsicsWithoutChain[] = {
   X86_INTRINSIC_DATA(avx512_mask_psrli_q,       VSHIFT_MASK, X86ISD::VSRLI, 0),
   X86_INTRINSIC_DATA(avx512_mask_psrlv_d,       INTR_TYPE_2OP_MASK, ISD::SRL, 0),
   X86_INTRINSIC_DATA(avx512_mask_psrlv_q,       INTR_TYPE_2OP_MASK, ISD::SRL, 0),
+  X86_INTRINSIC_DATA(avx512_mask_psub_b_128, INTR_TYPE_2OP_MASK, ISD::SUB, 0),
+  X86_INTRINSIC_DATA(avx512_mask_psub_b_256, INTR_TYPE_2OP_MASK, ISD::SUB, 0),
+  X86_INTRINSIC_DATA(avx512_mask_psub_b_512, INTR_TYPE_2OP_MASK, ISD::SUB, 0),
+  X86_INTRINSIC_DATA(avx512_mask_psub_d_128, INTR_TYPE_2OP_MASK, ISD::SUB, 0),
+  X86_INTRINSIC_DATA(avx512_mask_psub_d_256, INTR_TYPE_2OP_MASK, ISD::SUB, 0),
   X86_INTRINSIC_DATA(avx512_mask_psub_d_512, INTR_TYPE_2OP_MASK, ISD::SUB, 0),
+  X86_INTRINSIC_DATA(avx512_mask_psub_q_128, INTR_TYPE_2OP_MASK, ISD::SUB, 0),
+  X86_INTRINSIC_DATA(avx512_mask_psub_q_256, INTR_TYPE_2OP_MASK, ISD::SUB, 0),
   X86_INTRINSIC_DATA(avx512_mask_psub_q_512, INTR_TYPE_2OP_MASK, ISD::SUB, 0),
+  X86_INTRINSIC_DATA(avx512_mask_psub_w_128, INTR_TYPE_2OP_MASK, ISD::SUB, 0),
+  X86_INTRINSIC_DATA(avx512_mask_psub_w_256, INTR_TYPE_2OP_MASK, ISD::SUB, 0),
+  X86_INTRINSIC_DATA(avx512_mask_psub_w_512, INTR_TYPE_2OP_MASK, ISD::SUB, 0),
+  X86_INTRINSIC_DATA(avx512_mask_psubs_b_128, INTR_TYPE_2OP_MASK, X86ISD::SUBS, 0),
+  X86_INTRINSIC_DATA(avx512_mask_psubs_b_256, INTR_TYPE_2OP_MASK, X86ISD::SUBS, 0),
+  X86_INTRINSIC_DATA(avx512_mask_psubs_b_512, INTR_TYPE_2OP_MASK, X86ISD::SUBS, 0),
+  X86_INTRINSIC_DATA(avx512_mask_psubs_w_128, INTR_TYPE_2OP_MASK, X86ISD::SUBS, 0),
+  X86_INTRINSIC_DATA(avx512_mask_psubs_w_256, INTR_TYPE_2OP_MASK, X86ISD::SUBS, 0),
+  X86_INTRINSIC_DATA(avx512_mask_psubs_w_512, INTR_TYPE_2OP_MASK, X86ISD::SUBS, 0),
+  X86_INTRINSIC_DATA(avx512_mask_psubus_b_128, INTR_TYPE_2OP_MASK, X86ISD::SUBUS, 0),
+  X86_INTRINSIC_DATA(avx512_mask_psubus_b_256, INTR_TYPE_2OP_MASK, X86ISD::SUBUS, 0),
+  X86_INTRINSIC_DATA(avx512_mask_psubus_b_512, INTR_TYPE_2OP_MASK, X86ISD::SUBUS, 0),
+  X86_INTRINSIC_DATA(avx512_mask_psubus_w_128, INTR_TYPE_2OP_MASK, X86ISD::SUBUS, 0),
+  X86_INTRINSIC_DATA(avx512_mask_psubus_w_256, INTR_TYPE_2OP_MASK, X86ISD::SUBUS, 0),
+  X86_INTRINSIC_DATA(avx512_mask_psubus_w_512, INTR_TYPE_2OP_MASK, X86ISD::SUBUS, 0),
+  X86_INTRINSIC_DATA(avx512_mask_punpckhb_w_128, INTR_TYPE_2OP_MASK,
+                     X86ISD::UNPCKH, 0),
+  X86_INTRINSIC_DATA(avx512_mask_punpckhb_w_256, INTR_TYPE_2OP_MASK,
+                     X86ISD::UNPCKH, 0),
+  X86_INTRINSIC_DATA(avx512_mask_punpckhb_w_512, INTR_TYPE_2OP_MASK,
+                     X86ISD::UNPCKH, 0),
+  X86_INTRINSIC_DATA(avx512_mask_punpckhd_q_128, INTR_TYPE_2OP_MASK,
+                     X86ISD::UNPCKH, 0),
+  X86_INTRINSIC_DATA(avx512_mask_punpckhd_q_256, INTR_TYPE_2OP_MASK,
+                     X86ISD::UNPCKH, 0),
+  X86_INTRINSIC_DATA(avx512_mask_punpckhd_q_512, INTR_TYPE_2OP_MASK,
+                     X86ISD::UNPCKH, 0),
+  X86_INTRINSIC_DATA(avx512_mask_punpckhqd_q_128, INTR_TYPE_2OP_MASK,
+                     X86ISD::UNPCKH, 0),
+  X86_INTRINSIC_DATA(avx512_mask_punpckhqd_q_256, INTR_TYPE_2OP_MASK,
+                     X86ISD::UNPCKH, 0),
+  X86_INTRINSIC_DATA(avx512_mask_punpckhqd_q_512, INTR_TYPE_2OP_MASK,
+                     X86ISD::UNPCKH, 0),
+  X86_INTRINSIC_DATA(avx512_mask_punpckhw_d_128, INTR_TYPE_2OP_MASK,
+                     X86ISD::UNPCKH, 0),
+  X86_INTRINSIC_DATA(avx512_mask_punpckhw_d_256, INTR_TYPE_2OP_MASK,
+                     X86ISD::UNPCKH, 0),
+  X86_INTRINSIC_DATA(avx512_mask_punpckhw_d_512, INTR_TYPE_2OP_MASK,
+                     X86ISD::UNPCKH, 0),
+  X86_INTRINSIC_DATA(avx512_mask_punpcklb_w_128, INTR_TYPE_2OP_MASK,
+                     X86ISD::UNPCKL, 0),
+  X86_INTRINSIC_DATA(avx512_mask_punpcklb_w_256, INTR_TYPE_2OP_MASK,
+                     X86ISD::UNPCKL, 0),
+  X86_INTRINSIC_DATA(avx512_mask_punpcklb_w_512, INTR_TYPE_2OP_MASK,
+                     X86ISD::UNPCKL, 0),
+  X86_INTRINSIC_DATA(avx512_mask_punpckld_q_128, INTR_TYPE_2OP_MASK,
+                     X86ISD::UNPCKL, 0),
+  X86_INTRINSIC_DATA(avx512_mask_punpckld_q_256, INTR_TYPE_2OP_MASK,
+                     X86ISD::UNPCKL, 0),
+  X86_INTRINSIC_DATA(avx512_mask_punpckld_q_512, INTR_TYPE_2OP_MASK,
+                     X86ISD::UNPCKL, 0),
+  X86_INTRINSIC_DATA(avx512_mask_punpcklqd_q_128, INTR_TYPE_2OP_MASK,
+                     X86ISD::UNPCKL, 0),
+  X86_INTRINSIC_DATA(avx512_mask_punpcklqd_q_256, INTR_TYPE_2OP_MASK,
+                     X86ISD::UNPCKL, 0),
+  X86_INTRINSIC_DATA(avx512_mask_punpcklqd_q_512, INTR_TYPE_2OP_MASK,
+                     X86ISD::UNPCKL, 0),
+  X86_INTRINSIC_DATA(avx512_mask_punpcklw_d_128, INTR_TYPE_2OP_MASK,
+                     X86ISD::UNPCKL, 0),
+  X86_INTRINSIC_DATA(avx512_mask_punpcklw_d_256, INTR_TYPE_2OP_MASK,
+                     X86ISD::UNPCKL, 0),
+  X86_INTRINSIC_DATA(avx512_mask_punpcklw_d_512, INTR_TYPE_2OP_MASK,
+                     X86ISD::UNPCKL, 0),
+  X86_INTRINSIC_DATA(avx512_mask_pxor_d_128, INTR_TYPE_2OP_MASK, ISD::XOR, 0),
+  X86_INTRINSIC_DATA(avx512_mask_pxor_d_256, INTR_TYPE_2OP_MASK, ISD::XOR, 0),
   X86_INTRINSIC_DATA(avx512_mask_pxor_d_512, INTR_TYPE_2OP_MASK, ISD::XOR, 0),
+  X86_INTRINSIC_DATA(avx512_mask_pxor_q_128, INTR_TYPE_2OP_MASK, ISD::XOR, 0),
+  X86_INTRINSIC_DATA(avx512_mask_pxor_q_256, INTR_TYPE_2OP_MASK, ISD::XOR, 0),
   X86_INTRINSIC_DATA(avx512_mask_pxor_q_512, INTR_TYPE_2OP_MASK, ISD::XOR, 0),
+  X86_INTRINSIC_DATA(avx512_mask_range_pd_128, INTR_TYPE_3OP_MASK_RM, X86ISD::VRANGE, 0),
+  X86_INTRINSIC_DATA(avx512_mask_range_pd_256, INTR_TYPE_3OP_MASK_RM, X86ISD::VRANGE, 0),
+  X86_INTRINSIC_DATA(avx512_mask_range_pd_512, INTR_TYPE_3OP_MASK_RM, X86ISD::VRANGE, 0),
+  X86_INTRINSIC_DATA(avx512_mask_range_ps_128, INTR_TYPE_3OP_MASK_RM, X86ISD::VRANGE, 0),
+  X86_INTRINSIC_DATA(avx512_mask_range_ps_256, INTR_TYPE_3OP_MASK_RM, X86ISD::VRANGE, 0),
+  X86_INTRINSIC_DATA(avx512_mask_range_ps_512, INTR_TYPE_3OP_MASK_RM, X86ISD::VRANGE, 0),
+  X86_INTRINSIC_DATA(avx512_mask_range_sd, INTR_TYPE_SCALAR_MASK_RM, X86ISD::VRANGE, 0),
+  X86_INTRINSIC_DATA(avx512_mask_range_ss, INTR_TYPE_SCALAR_MASK_RM, X86ISD::VRANGE, 0),
+  X86_INTRINSIC_DATA(avx512_mask_reduce_pd_128, INTR_TYPE_2OP_MASK_RM, X86ISD::VREDUCE, 0),
+  X86_INTRINSIC_DATA(avx512_mask_reduce_pd_256, INTR_TYPE_2OP_MASK_RM, X86ISD::VREDUCE, 0),
+  X86_INTRINSIC_DATA(avx512_mask_reduce_pd_512, INTR_TYPE_2OP_MASK_RM, X86ISD::VREDUCE, 0),
+  X86_INTRINSIC_DATA(avx512_mask_reduce_ps_128, INTR_TYPE_2OP_MASK_RM, X86ISD::VREDUCE, 0),
+  X86_INTRINSIC_DATA(avx512_mask_reduce_ps_256, INTR_TYPE_2OP_MASK_RM, X86ISD::VREDUCE, 0),
+  X86_INTRINSIC_DATA(avx512_mask_reduce_ps_512, INTR_TYPE_2OP_MASK_RM, X86ISD::VREDUCE, 0),
+  X86_INTRINSIC_DATA(avx512_mask_reduce_sd, INTR_TYPE_SCALAR_MASK_RM, X86ISD::VREDUCE, 0),
+  X86_INTRINSIC_DATA(avx512_mask_reduce_ss, INTR_TYPE_SCALAR_MASK_RM, X86ISD::VREDUCE, 0),
+  X86_INTRINSIC_DATA(avx512_mask_rndscale_pd_128, INTR_TYPE_2OP_MASK_RM, X86ISD::VRNDSCALE, 0),
+  X86_INTRINSIC_DATA(avx512_mask_rndscale_pd_256, INTR_TYPE_2OP_MASK_RM, X86ISD::VRNDSCALE, 0),
+  X86_INTRINSIC_DATA(avx512_mask_rndscale_pd_512, INTR_TYPE_2OP_MASK_RM, X86ISD::VRNDSCALE, 0),
+  X86_INTRINSIC_DATA(avx512_mask_rndscale_ps_128, INTR_TYPE_2OP_MASK_RM, X86ISD::VRNDSCALE, 0),
+  X86_INTRINSIC_DATA(avx512_mask_rndscale_ps_256, INTR_TYPE_2OP_MASK_RM, X86ISD::VRNDSCALE, 0),
+  X86_INTRINSIC_DATA(avx512_mask_rndscale_ps_512, INTR_TYPE_2OP_MASK_RM, X86ISD::VRNDSCALE, 0),
   X86_INTRINSIC_DATA(avx512_mask_rndscale_sd,   INTR_TYPE_SCALAR_MASK_RM,
-                     X86ISD::RNDSCALE, 0),
+                     X86ISD::VRNDSCALE, 0),
   X86_INTRINSIC_DATA(avx512_mask_rndscale_ss,   INTR_TYPE_SCALAR_MASK_RM,
-                     X86ISD::RNDSCALE, 0),
+                     X86ISD::VRNDSCALE, 0),
+  X86_INTRINSIC_DATA(avx512_mask_scalef_pd_128, INTR_TYPE_2OP_MASK_RM,
+                     X86ISD::SCALEF, 0),
+  X86_INTRINSIC_DATA(avx512_mask_scalef_pd_256, INTR_TYPE_2OP_MASK_RM,
+                     X86ISD::SCALEF, 0),
+  X86_INTRINSIC_DATA(avx512_mask_scalef_pd_512, INTR_TYPE_2OP_MASK_RM,
+                     X86ISD::SCALEF, 0),
+  X86_INTRINSIC_DATA(avx512_mask_scalef_ps_128, INTR_TYPE_2OP_MASK_RM,
+                     X86ISD::SCALEF, 0),
+  X86_INTRINSIC_DATA(avx512_mask_scalef_ps_256, INTR_TYPE_2OP_MASK_RM,
+                     X86ISD::SCALEF, 0),
+  X86_INTRINSIC_DATA(avx512_mask_scalef_ps_512, INTR_TYPE_2OP_MASK_RM,
+                     X86ISD::SCALEF, 0),
+  X86_INTRINSIC_DATA(avx512_mask_scalef_sd, INTR_TYPE_SCALAR_MASK_RM,
+                     X86ISD::SCALEF, 0),
+  X86_INTRINSIC_DATA(avx512_mask_scalef_ss, INTR_TYPE_SCALAR_MASK_RM,
+                     X86ISD::SCALEF, 0),
+  X86_INTRINSIC_DATA(avx512_mask_sqrt_pd_128, INTR_TYPE_1OP_MASK, ISD::FSQRT, 0),
+  X86_INTRINSIC_DATA(avx512_mask_sqrt_pd_256, INTR_TYPE_1OP_MASK, ISD::FSQRT, 0),
+  X86_INTRINSIC_DATA(avx512_mask_sqrt_pd_512, INTR_TYPE_1OP_MASK_RM, ISD::FSQRT,
+                     X86ISD::FSQRT_RND),
+  X86_INTRINSIC_DATA(avx512_mask_sqrt_ps_128, INTR_TYPE_1OP_MASK, ISD::FSQRT, 0),
+  X86_INTRINSIC_DATA(avx512_mask_sqrt_ps_256, INTR_TYPE_1OP_MASK, ISD::FSQRT, 0),
+  X86_INTRINSIC_DATA(avx512_mask_sqrt_ps_512, INTR_TYPE_1OP_MASK_RM, ISD::FSQRT,
+                     X86ISD::FSQRT_RND),
+  X86_INTRINSIC_DATA(avx512_mask_sub_pd_128, INTR_TYPE_2OP_MASK, ISD::FSUB, 0),
+  X86_INTRINSIC_DATA(avx512_mask_sub_pd_256, INTR_TYPE_2OP_MASK, ISD::FSUB, 0),
   X86_INTRINSIC_DATA(avx512_mask_sub_pd_512, INTR_TYPE_2OP_MASK, ISD::FSUB,
                      X86ISD::FSUB_RND),
+  X86_INTRINSIC_DATA(avx512_mask_sub_ps_128, INTR_TYPE_2OP_MASK, ISD::FSUB, 0),
+  X86_INTRINSIC_DATA(avx512_mask_sub_ps_256, INTR_TYPE_2OP_MASK, ISD::FSUB, 0),
   X86_INTRINSIC_DATA(avx512_mask_sub_ps_512, INTR_TYPE_2OP_MASK, ISD::FSUB,
                      X86ISD::FSUB_RND),
+  X86_INTRINSIC_DATA(avx512_mask_sub_sd_round, INTR_TYPE_SCALAR_MASK_RM, ISD::FSUB,
+  X86ISD::FSUB_RND),
+  X86_INTRINSIC_DATA(avx512_mask_sub_ss_round, INTR_TYPE_SCALAR_MASK_RM, ISD::FSUB,
+  X86ISD::FSUB_RND),
   X86_INTRINSIC_DATA(avx512_mask_ucmp_b_128,    CMP_MASK_CC,  X86ISD::CMPMU, 0),
   X86_INTRINSIC_DATA(avx512_mask_ucmp_b_256,    CMP_MASK_CC,  X86ISD::CMPMU, 0),
   X86_INTRINSIC_DATA(avx512_mask_ucmp_b_512,    CMP_MASK_CC,  X86ISD::CMPMU, 0),
@@ -410,6 +1175,207 @@ static const IntrinsicData  IntrinsicsWithoutChain[] = {
   X86_INTRINSIC_DATA(avx512_mask_ucmp_w_128,    CMP_MASK_CC,  X86ISD::CMPMU, 0),
   X86_INTRINSIC_DATA(avx512_mask_ucmp_w_256,    CMP_MASK_CC,  X86ISD::CMPMU, 0),
   X86_INTRINSIC_DATA(avx512_mask_ucmp_w_512,    CMP_MASK_CC,  X86ISD::CMPMU, 0),
+  X86_INTRINSIC_DATA(avx512_mask_unpckh_pd_128, INTR_TYPE_2OP_MASK,
+                     X86ISD::UNPCKH, 0),
+  X86_INTRINSIC_DATA(avx512_mask_unpckh_pd_256, INTR_TYPE_2OP_MASK,
+                     X86ISD::UNPCKH, 0),
+  X86_INTRINSIC_DATA(avx512_mask_unpckh_pd_512, INTR_TYPE_2OP_MASK,
+                     X86ISD::UNPCKH, 0),
+  X86_INTRINSIC_DATA(avx512_mask_unpckh_ps_128, INTR_TYPE_2OP_MASK,
+                     X86ISD::UNPCKH, 0),
+  X86_INTRINSIC_DATA(avx512_mask_unpckh_ps_256, INTR_TYPE_2OP_MASK,
+                     X86ISD::UNPCKH, 0),
+  X86_INTRINSIC_DATA(avx512_mask_unpckh_ps_512, INTR_TYPE_2OP_MASK,
+                     X86ISD::UNPCKH, 0),
+  X86_INTRINSIC_DATA(avx512_mask_unpckl_pd_128, INTR_TYPE_2OP_MASK,
+                     X86ISD::UNPCKL, 0),
+  X86_INTRINSIC_DATA(avx512_mask_unpckl_pd_256, INTR_TYPE_2OP_MASK,
+                     X86ISD::UNPCKL, 0),
+  X86_INTRINSIC_DATA(avx512_mask_unpckl_pd_512, INTR_TYPE_2OP_MASK,
+                     X86ISD::UNPCKL, 0),
+  X86_INTRINSIC_DATA(avx512_mask_unpckl_ps_128, INTR_TYPE_2OP_MASK,
+                     X86ISD::UNPCKL, 0),
+  X86_INTRINSIC_DATA(avx512_mask_unpckl_ps_256, INTR_TYPE_2OP_MASK,
+                     X86ISD::UNPCKL, 0),
+  X86_INTRINSIC_DATA(avx512_mask_unpckl_ps_512, INTR_TYPE_2OP_MASK,
+                     X86ISD::UNPCKL, 0),
+  X86_INTRINSIC_DATA(avx512_mask_valign_d_512, INTR_TYPE_3OP_MASK, X86ISD::VALIGN, 0),
+  X86_INTRINSIC_DATA(avx512_mask_valign_q_512, INTR_TYPE_3OP_MASK, X86ISD::VALIGN, 0),
+
+  X86_INTRINSIC_DATA(avx512_mask_vfmadd_pd_128, FMA_OP_MASK, X86ISD::FMADD, 0),
+  X86_INTRINSIC_DATA(avx512_mask_vfmadd_pd_256, FMA_OP_MASK, X86ISD::FMADD, 0),
+  X86_INTRINSIC_DATA(avx512_mask_vfmadd_pd_512, FMA_OP_MASK, X86ISD::FMADD,
+                     X86ISD::FMADD_RND),
+  X86_INTRINSIC_DATA(avx512_mask_vfmadd_ps_128, FMA_OP_MASK, X86ISD::FMADD, 0),
+  X86_INTRINSIC_DATA(avx512_mask_vfmadd_ps_256, FMA_OP_MASK, X86ISD::FMADD, 0),
+  X86_INTRINSIC_DATA(avx512_mask_vfmadd_ps_512, FMA_OP_MASK, X86ISD::FMADD,
+                     X86ISD::FMADD_RND),
+
+  X86_INTRINSIC_DATA(avx512_mask_vfmaddsub_pd_128, FMA_OP_MASK, X86ISD::FMADDSUB, 0),
+  X86_INTRINSIC_DATA(avx512_mask_vfmaddsub_pd_256, FMA_OP_MASK, X86ISD::FMADDSUB, 0),
+  X86_INTRINSIC_DATA(avx512_mask_vfmaddsub_pd_512, FMA_OP_MASK, X86ISD::FMADDSUB,
+                     X86ISD::FMADDSUB_RND),
+  X86_INTRINSIC_DATA(avx512_mask_vfmaddsub_ps_128, FMA_OP_MASK, X86ISD::FMADDSUB, 0),
+  X86_INTRINSIC_DATA(avx512_mask_vfmaddsub_ps_256, FMA_OP_MASK, X86ISD::FMADDSUB, 0),
+  X86_INTRINSIC_DATA(avx512_mask_vfmaddsub_ps_512, FMA_OP_MASK, X86ISD::FMADDSUB,
+                     X86ISD::FMADDSUB_RND),
+
+  X86_INTRINSIC_DATA(avx512_mask_vfnmadd_pd_128, FMA_OP_MASK, X86ISD::FNMADD, 0),
+  X86_INTRINSIC_DATA(avx512_mask_vfnmadd_pd_256, FMA_OP_MASK, X86ISD::FNMADD, 0),
+  X86_INTRINSIC_DATA(avx512_mask_vfnmadd_pd_512, FMA_OP_MASK, X86ISD::FNMADD,
+                     X86ISD::FNMADD_RND),
+  X86_INTRINSIC_DATA(avx512_mask_vfnmadd_ps_128, FMA_OP_MASK, X86ISD::FNMADD, 0),
+  X86_INTRINSIC_DATA(avx512_mask_vfnmadd_ps_256, FMA_OP_MASK, X86ISD::FNMADD, 0),
+  X86_INTRINSIC_DATA(avx512_mask_vfnmadd_ps_512, FMA_OP_MASK, X86ISD::FNMADD,
+                     X86ISD::FNMADD_RND),
+
+  X86_INTRINSIC_DATA(avx512_mask_vfnmsub_pd_128, FMA_OP_MASK, X86ISD::FNMSUB, 0),
+  X86_INTRINSIC_DATA(avx512_mask_vfnmsub_pd_256, FMA_OP_MASK, X86ISD::FNMSUB, 0),
+  X86_INTRINSIC_DATA(avx512_mask_vfnmsub_pd_512, FMA_OP_MASK, X86ISD::FNMSUB,
+                     X86ISD::FNMSUB_RND),
+  X86_INTRINSIC_DATA(avx512_mask_vfnmsub_ps_128, FMA_OP_MASK, X86ISD::FNMSUB, 0),
+  X86_INTRINSIC_DATA(avx512_mask_vfnmsub_ps_256, FMA_OP_MASK, X86ISD::FNMSUB, 0),
+  X86_INTRINSIC_DATA(avx512_mask_vfnmsub_ps_512, FMA_OP_MASK, X86ISD::FNMSUB,
+                     X86ISD::FNMSUB_RND),
+
+
+  X86_INTRINSIC_DATA(avx512_mask_vpermi2var_d_128, VPERM_3OP_MASK,
+                    X86ISD::VPERMIV3, 0),
+  X86_INTRINSIC_DATA(avx512_mask_vpermi2var_d_256, VPERM_3OP_MASK,
+                    X86ISD::VPERMIV3, 0),
+  X86_INTRINSIC_DATA(avx512_mask_vpermi2var_d_512, VPERM_3OP_MASK,
+                    X86ISD::VPERMIV3, 0),
+  X86_INTRINSIC_DATA(avx512_mask_vpermi2var_hi_128, VPERM_3OP_MASK,
+                    X86ISD::VPERMIV3, 0),
+  X86_INTRINSIC_DATA(avx512_mask_vpermi2var_hi_256, VPERM_3OP_MASK,
+                    X86ISD::VPERMIV3, 0),
+  X86_INTRINSIC_DATA(avx512_mask_vpermi2var_hi_512, VPERM_3OP_MASK,
+                    X86ISD::VPERMIV3, 0),
+  X86_INTRINSIC_DATA(avx512_mask_vpermi2var_pd_128, VPERM_3OP_MASK,
+                    X86ISD::VPERMIV3, 0),
+  X86_INTRINSIC_DATA(avx512_mask_vpermi2var_pd_256, VPERM_3OP_MASK,
+                    X86ISD::VPERMIV3, 0),
+  X86_INTRINSIC_DATA(avx512_mask_vpermi2var_pd_512, VPERM_3OP_MASK,
+                    X86ISD::VPERMIV3, 0),
+  X86_INTRINSIC_DATA(avx512_mask_vpermi2var_ps_128, VPERM_3OP_MASK,
+                    X86ISD::VPERMIV3, 0),
+  X86_INTRINSIC_DATA(avx512_mask_vpermi2var_ps_256, VPERM_3OP_MASK,
+                    X86ISD::VPERMIV3, 0),
+  X86_INTRINSIC_DATA(avx512_mask_vpermi2var_ps_512, VPERM_3OP_MASK,
+                    X86ISD::VPERMIV3, 0),
+  X86_INTRINSIC_DATA(avx512_mask_vpermi2var_q_128, VPERM_3OP_MASK,
+                    X86ISD::VPERMIV3, 0),
+  X86_INTRINSIC_DATA(avx512_mask_vpermi2var_q_256, VPERM_3OP_MASK,
+                    X86ISD::VPERMIV3, 0),
+  X86_INTRINSIC_DATA(avx512_mask_vpermi2var_q_512, VPERM_3OP_MASK,
+                    X86ISD::VPERMIV3, 0),
+  X86_INTRINSIC_DATA(avx512_mask_vpermt2var_d_128, VPERM_3OP_MASK,
+                    X86ISD::VPERMV3, 0),
+  X86_INTRINSIC_DATA(avx512_mask_vpermt2var_d_128, VPERM_3OP_MASK,
+                    X86ISD::VPERMV3, 0),
+  X86_INTRINSIC_DATA(avx512_mask_vpermt2var_d_256, VPERM_3OP_MASK,
+                    X86ISD::VPERMV3, 0),
+  X86_INTRINSIC_DATA(avx512_mask_vpermt2var_d_256, VPERM_3OP_MASK,
+                    X86ISD::VPERMV3, 0),
+  X86_INTRINSIC_DATA(avx512_mask_vpermt2var_d_512, VPERM_3OP_MASK,
+                    X86ISD::VPERMV3, 0),
+  X86_INTRINSIC_DATA(avx512_mask_vpermt2var_hi_128, VPERM_3OP_MASK,
+                    X86ISD::VPERMV3, 0),
+  X86_INTRINSIC_DATA(avx512_mask_vpermt2var_hi_128, VPERM_3OP_MASK,
+                    X86ISD::VPERMV3, 0),
+  X86_INTRINSIC_DATA(avx512_mask_vpermt2var_hi_256, VPERM_3OP_MASK,
+                    X86ISD::VPERMV3, 0),
+  X86_INTRINSIC_DATA(avx512_mask_vpermt2var_hi_256, VPERM_3OP_MASK,
+                    X86ISD::VPERMV3, 0),
+  X86_INTRINSIC_DATA(avx512_mask_vpermt2var_hi_512, VPERM_3OP_MASK,
+                    X86ISD::VPERMV3, 0),
+  X86_INTRINSIC_DATA(avx512_mask_vpermt2var_hi_512, VPERM_3OP_MASK,
+                    X86ISD::VPERMV3, 0),
+  X86_INTRINSIC_DATA(avx512_mask_vpermt2var_pd_128, VPERM_3OP_MASK,
+                    X86ISD::VPERMV3, 0),
+  X86_INTRINSIC_DATA(avx512_mask_vpermt2var_pd_128, VPERM_3OP_MASK,
+                    X86ISD::VPERMV3, 0),
+  X86_INTRINSIC_DATA(avx512_mask_vpermt2var_pd_256, VPERM_3OP_MASK,
+                    X86ISD::VPERMV3, 0),
+  X86_INTRINSIC_DATA(avx512_mask_vpermt2var_pd_256, VPERM_3OP_MASK,
+                    X86ISD::VPERMV3, 0),
+  X86_INTRINSIC_DATA(avx512_mask_vpermt2var_pd_512, VPERM_3OP_MASK,
+                    X86ISD::VPERMV3, 0),
+  X86_INTRINSIC_DATA(avx512_mask_vpermt2var_ps_128, VPERM_3OP_MASK,
+                    X86ISD::VPERMV3, 0),
+  X86_INTRINSIC_DATA(avx512_mask_vpermt2var_ps_128, VPERM_3OP_MASK,
+                    X86ISD::VPERMV3, 0),
+  X86_INTRINSIC_DATA(avx512_mask_vpermt2var_ps_256, VPERM_3OP_MASK,
+                    X86ISD::VPERMV3, 0),
+  X86_INTRINSIC_DATA(avx512_mask_vpermt2var_ps_256, VPERM_3OP_MASK,
+                    X86ISD::VPERMV3, 0),
+  X86_INTRINSIC_DATA(avx512_mask_vpermt2var_ps_512, VPERM_3OP_MASK,
+                    X86ISD::VPERMV3, 0),
+  X86_INTRINSIC_DATA(avx512_mask_vpermt2var_q_128, VPERM_3OP_MASK,
+                    X86ISD::VPERMV3, 0),
+  X86_INTRINSIC_DATA(avx512_mask_vpermt2var_q_128, VPERM_3OP_MASK,
+                    X86ISD::VPERMV3, 0),
+  X86_INTRINSIC_DATA(avx512_mask_vpermt2var_q_256, VPERM_3OP_MASK,
+                    X86ISD::VPERMV3, 0),
+  X86_INTRINSIC_DATA(avx512_mask_vpermt2var_q_256, VPERM_3OP_MASK,
+                    X86ISD::VPERMV3, 0),
+  X86_INTRINSIC_DATA(avx512_mask_vpermt2var_q_512, VPERM_3OP_MASK,
+                    X86ISD::VPERMV3, 0),
+  X86_INTRINSIC_DATA(avx512_mask_xor_pd_128, INTR_TYPE_2OP_MASK, X86ISD::FXOR, 0),
+  X86_INTRINSIC_DATA(avx512_mask_xor_pd_256, INTR_TYPE_2OP_MASK, X86ISD::FXOR, 0),
+  X86_INTRINSIC_DATA(avx512_mask_xor_pd_512, INTR_TYPE_2OP_MASK, X86ISD::FXOR, 0),
+  X86_INTRINSIC_DATA(avx512_mask_xor_ps_128, INTR_TYPE_2OP_MASK, X86ISD::FXOR, 0),
+  X86_INTRINSIC_DATA(avx512_mask_xor_ps_256, INTR_TYPE_2OP_MASK, X86ISD::FXOR, 0),
+  X86_INTRINSIC_DATA(avx512_mask_xor_ps_512, INTR_TYPE_2OP_MASK, X86ISD::FXOR, 0),
+
+  X86_INTRINSIC_DATA(avx512_maskz_vfmadd_pd_128, FMA_OP_MASKZ, X86ISD::FMADD, 0),
+  X86_INTRINSIC_DATA(avx512_maskz_vfmadd_pd_256, FMA_OP_MASKZ, X86ISD::FMADD, 0),
+  X86_INTRINSIC_DATA(avx512_maskz_vfmadd_pd_512, FMA_OP_MASKZ, X86ISD::FMADD,
+                     X86ISD::FMADD_RND),
+  X86_INTRINSIC_DATA(avx512_maskz_vfmadd_ps_128, FMA_OP_MASKZ, X86ISD::FMADD, 0),
+  X86_INTRINSIC_DATA(avx512_maskz_vfmadd_ps_256, FMA_OP_MASKZ, X86ISD::FMADD, 0),
+  X86_INTRINSIC_DATA(avx512_maskz_vfmadd_ps_512, FMA_OP_MASKZ, X86ISD::FMADD,
+                     X86ISD::FMADD_RND),
+
+  X86_INTRINSIC_DATA(avx512_maskz_vfmaddsub_pd_128, FMA_OP_MASKZ, X86ISD::FMADDSUB, 0),
+  X86_INTRINSIC_DATA(avx512_maskz_vfmaddsub_pd_256, FMA_OP_MASKZ, X86ISD::FMADDSUB, 0),
+  X86_INTRINSIC_DATA(avx512_maskz_vfmaddsub_pd_512, FMA_OP_MASKZ, X86ISD::FMADDSUB,
+                     X86ISD::FMADDSUB_RND),
+  X86_INTRINSIC_DATA(avx512_maskz_vfmaddsub_ps_128, FMA_OP_MASKZ, X86ISD::FMADDSUB, 0),
+  X86_INTRINSIC_DATA(avx512_maskz_vfmaddsub_ps_256, FMA_OP_MASKZ, X86ISD::FMADDSUB, 0),
+  X86_INTRINSIC_DATA(avx512_maskz_vfmaddsub_ps_512, FMA_OP_MASKZ, X86ISD::FMADDSUB,
+                     X86ISD::FMADDSUB_RND),
+
+  X86_INTRINSIC_DATA(avx512_maskz_vpermt2var_d_128, VPERM_3OP_MASKZ,
+                     X86ISD::VPERMV3, 0),
+  X86_INTRINSIC_DATA(avx512_maskz_vpermt2var_d_256, VPERM_3OP_MASKZ,
+                     X86ISD::VPERMV3, 0),
+  X86_INTRINSIC_DATA(avx512_maskz_vpermt2var_d_512, VPERM_3OP_MASKZ,
+                     X86ISD::VPERMV3, 0),
+  X86_INTRINSIC_DATA(avx512_maskz_vpermt2var_hi_128, VPERM_3OP_MASKZ,
+                     X86ISD::VPERMV3, 0),
+  X86_INTRINSIC_DATA(avx512_maskz_vpermt2var_hi_256, VPERM_3OP_MASKZ,
+                     X86ISD::VPERMV3, 0),
+  X86_INTRINSIC_DATA(avx512_maskz_vpermt2var_hi_512, VPERM_3OP_MASKZ,
+                     X86ISD::VPERMV3, 0),
+  X86_INTRINSIC_DATA(avx512_maskz_vpermt2var_pd_128, VPERM_3OP_MASKZ,
+                     X86ISD::VPERMV3, 0),
+  X86_INTRINSIC_DATA(avx512_maskz_vpermt2var_pd_256, VPERM_3OP_MASKZ,
+                     X86ISD::VPERMV3, 0),
+  X86_INTRINSIC_DATA(avx512_maskz_vpermt2var_pd_512, VPERM_3OP_MASKZ,
+                     X86ISD::VPERMV3, 0),
+  X86_INTRINSIC_DATA(avx512_maskz_vpermt2var_ps_128, VPERM_3OP_MASKZ,
+                     X86ISD::VPERMV3, 0),
+  X86_INTRINSIC_DATA(avx512_maskz_vpermt2var_ps_256, VPERM_3OP_MASKZ,
+                     X86ISD::VPERMV3, 0),
+  X86_INTRINSIC_DATA(avx512_maskz_vpermt2var_ps_512, VPERM_3OP_MASKZ,
+                     X86ISD::VPERMV3, 0),
+  X86_INTRINSIC_DATA(avx512_maskz_vpermt2var_q_128, VPERM_3OP_MASKZ,
+                     X86ISD::VPERMV3, 0),
+  X86_INTRINSIC_DATA(avx512_maskz_vpermt2var_q_256, VPERM_3OP_MASKZ,
+                     X86ISD::VPERMV3, 0),
+  X86_INTRINSIC_DATA(avx512_maskz_vpermt2var_q_512, VPERM_3OP_MASKZ,
+                     X86ISD::VPERMV3, 0),
   X86_INTRINSIC_DATA(avx512_rcp28_pd,   INTR_TYPE_1OP_MASK_RM,X86ISD::RCP28, 0),
   X86_INTRINSIC_DATA(avx512_rcp28_ps,   INTR_TYPE_1OP_MASK_RM,X86ISD::RCP28, 0),
   X86_INTRINSIC_DATA(avx512_rcp28_sd,   INTR_TYPE_SCALAR_MASK_RM, X86ISD::RCP28, 0),
@@ -426,59 +1392,13 @@ static const IntrinsicData  IntrinsicsWithoutChain[] = {
   X86_INTRINSIC_DATA(avx_max_ps_256,    INTR_TYPE_2OP, X86ISD::FMAX, 0),
   X86_INTRINSIC_DATA(avx_min_pd_256,    INTR_TYPE_2OP, X86ISD::FMIN, 0),
   X86_INTRINSIC_DATA(avx_min_ps_256,    INTR_TYPE_2OP, X86ISD::FMIN, 0),
+  X86_INTRINSIC_DATA(avx_rcp_ps_256,    INTR_TYPE_1OP, X86ISD::FRCP, 0),
+  X86_INTRINSIC_DATA(avx_rsqrt_ps_256,  INTR_TYPE_1OP, X86ISD::FRSQRT, 0),
   X86_INTRINSIC_DATA(avx_sqrt_pd_256,   INTR_TYPE_1OP, ISD::FSQRT, 0),
   X86_INTRINSIC_DATA(avx_sqrt_ps_256,   INTR_TYPE_1OP, ISD::FSQRT, 0),
   X86_INTRINSIC_DATA(avx_vperm2f128_pd_256, INTR_TYPE_3OP, X86ISD::VPERM2X128, 0),
   X86_INTRINSIC_DATA(avx_vperm2f128_ps_256, INTR_TYPE_3OP, X86ISD::VPERM2X128, 0),
   X86_INTRINSIC_DATA(avx_vperm2f128_si_256, INTR_TYPE_3OP, X86ISD::VPERM2X128, 0),
-  X86_INTRINSIC_DATA(fma_mask_vfmadd_pd_128,    FMA_OP_MASK, X86ISD::FMADD, 0),
-  X86_INTRINSIC_DATA(fma_mask_vfmadd_pd_256,    FMA_OP_MASK, X86ISD::FMADD, 0),
-  X86_INTRINSIC_DATA(fma_mask_vfmadd_pd_512,    FMA_OP_MASK, X86ISD::FMADD,
-                     X86ISD::FMADD_RND),
-  X86_INTRINSIC_DATA(fma_mask_vfmadd_ps_128,    FMA_OP_MASK, X86ISD::FMADD, 0),
-  X86_INTRINSIC_DATA(fma_mask_vfmadd_ps_256,    FMA_OP_MASK, X86ISD::FMADD, 0),
-  X86_INTRINSIC_DATA(fma_mask_vfmadd_ps_512,    FMA_OP_MASK, X86ISD::FMADD,
-                     X86ISD::FMADD_RND),
-  X86_INTRINSIC_DATA(fma_mask_vfmaddsub_pd_128, FMA_OP_MASK, X86ISD::FMADDSUB, 0),
-  X86_INTRINSIC_DATA(fma_mask_vfmaddsub_pd_256, FMA_OP_MASK, X86ISD::FMADDSUB, 0),
-  X86_INTRINSIC_DATA(fma_mask_vfmaddsub_pd_512, FMA_OP_MASK, X86ISD::FMADDSUB,
-                     X86ISD::FMADDSUB_RND),
-  X86_INTRINSIC_DATA(fma_mask_vfmaddsub_ps_128, FMA_OP_MASK, X86ISD::FMADDSUB, 0),
-  X86_INTRINSIC_DATA(fma_mask_vfmaddsub_ps_256, FMA_OP_MASK, X86ISD::FMADDSUB, 0),
-  X86_INTRINSIC_DATA(fma_mask_vfmaddsub_ps_512, FMA_OP_MASK, X86ISD::FMADDSUB,
-                     X86ISD::FMADDSUB_RND),
-  X86_INTRINSIC_DATA(fma_mask_vfmsub_pd_128,    FMA_OP_MASK, X86ISD::FMSUB, 0),
-  X86_INTRINSIC_DATA(fma_mask_vfmsub_pd_256,    FMA_OP_MASK, X86ISD::FMSUB, 0),
-  X86_INTRINSIC_DATA(fma_mask_vfmsub_pd_512,    FMA_OP_MASK, X86ISD::FMSUB,
-                     X86ISD::FMSUB_RND),
-  X86_INTRINSIC_DATA(fma_mask_vfmsub_ps_128,    FMA_OP_MASK, X86ISD::FMSUB, 0),
-  X86_INTRINSIC_DATA(fma_mask_vfmsub_ps_256,    FMA_OP_MASK, X86ISD::FMSUB, 0),
-  X86_INTRINSIC_DATA(fma_mask_vfmsub_ps_512,    FMA_OP_MASK, X86ISD::FMSUB,
-                     X86ISD::FMSUB_RND),
-  X86_INTRINSIC_DATA(fma_mask_vfmsubadd_pd_128, FMA_OP_MASK, X86ISD::FMSUBADD, 0),
-  X86_INTRINSIC_DATA(fma_mask_vfmsubadd_pd_256, FMA_OP_MASK, X86ISD::FMSUBADD, 0),
-  X86_INTRINSIC_DATA(fma_mask_vfmsubadd_pd_512, FMA_OP_MASK, X86ISD::FMSUBADD,
-                     X86ISD::FMSUBADD_RND),
-  X86_INTRINSIC_DATA(fma_mask_vfmsubadd_ps_128, FMA_OP_MASK, X86ISD::FMSUBADD, 0),
-  X86_INTRINSIC_DATA(fma_mask_vfmsubadd_ps_256, FMA_OP_MASK, X86ISD::FMSUBADD, 0),
-  X86_INTRINSIC_DATA(fma_mask_vfmsubadd_ps_512, FMA_OP_MASK, X86ISD::FMSUBADD,
-                     X86ISD::FMSUBADD_RND),
-  X86_INTRINSIC_DATA(fma_mask_vfnmadd_pd_128,   FMA_OP_MASK, X86ISD::FNMADD, 0),
-  X86_INTRINSIC_DATA(fma_mask_vfnmadd_pd_256,   FMA_OP_MASK, X86ISD::FNMADD, 0),
-  X86_INTRINSIC_DATA(fma_mask_vfnmadd_pd_512,   FMA_OP_MASK, X86ISD::FNMADD,
-                     X86ISD::FNMADD_RND),
-  X86_INTRINSIC_DATA(fma_mask_vfnmadd_ps_128,   FMA_OP_MASK, X86ISD::FNMADD, 0),
-  X86_INTRINSIC_DATA(fma_mask_vfnmadd_ps_256,   FMA_OP_MASK, X86ISD::FNMADD, 0),
-  X86_INTRINSIC_DATA(fma_mask_vfnmadd_ps_512,   FMA_OP_MASK, X86ISD::FNMADD,
-                     X86ISD::FNMADD_RND),
-  X86_INTRINSIC_DATA(fma_mask_vfnmsub_pd_128,   FMA_OP_MASK, X86ISD::FNMSUB, 0),
-  X86_INTRINSIC_DATA(fma_mask_vfnmsub_pd_256,   FMA_OP_MASK, X86ISD::FNMSUB, 0),
-  X86_INTRINSIC_DATA(fma_mask_vfnmsub_pd_512,   FMA_OP_MASK, X86ISD::FNMSUB,
-                     X86ISD::FNMSUB_RND),
-  X86_INTRINSIC_DATA(fma_mask_vfnmsub_ps_128,   FMA_OP_MASK, X86ISD::FNMSUB, 0),
-  X86_INTRINSIC_DATA(fma_mask_vfnmsub_ps_256,   FMA_OP_MASK, X86ISD::FNMSUB, 0),
-  X86_INTRINSIC_DATA(fma_mask_vfnmsub_ps_512,   FMA_OP_MASK, X86ISD::FNMSUB,
-                     X86ISD::FNMSUB_RND),
   X86_INTRINSIC_DATA(fma_vfmadd_pd,        INTR_TYPE_3OP, X86ISD::FMADD, 0),
   X86_INTRINSIC_DATA(fma_vfmadd_pd_256,    INTR_TYPE_3OP, X86ISD::FMADD, 0),
   X86_INTRINSIC_DATA(fma_vfmadd_ps,        INTR_TYPE_3OP, X86ISD::FMADD, 0),
@@ -514,10 +1434,10 @@ static const IntrinsicData  IntrinsicsWithoutChain[] = {
   X86_INTRINSIC_DATA(sse2_packssdw_128, INTR_TYPE_2OP, X86ISD::PACKSS, 0),
   X86_INTRINSIC_DATA(sse2_packsswb_128, INTR_TYPE_2OP, X86ISD::PACKSS, 0),
   X86_INTRINSIC_DATA(sse2_packuswb_128, INTR_TYPE_2OP, X86ISD::PACKUS, 0),
-  X86_INTRINSIC_DATA(sse2_pmaxs_w,      INTR_TYPE_2OP, X86ISD::SMAX, 0),
-  X86_INTRINSIC_DATA(sse2_pmaxu_b,      INTR_TYPE_2OP, X86ISD::UMAX, 0),
-  X86_INTRINSIC_DATA(sse2_pmins_w,      INTR_TYPE_2OP, X86ISD::SMIN, 0),
-  X86_INTRINSIC_DATA(sse2_pminu_b,      INTR_TYPE_2OP, X86ISD::UMIN, 0),
+  X86_INTRINSIC_DATA(sse2_pmaxs_w,      INTR_TYPE_2OP, ISD::SMAX, 0),
+  X86_INTRINSIC_DATA(sse2_pmaxu_b,      INTR_TYPE_2OP, ISD::UMAX, 0),
+  X86_INTRINSIC_DATA(sse2_pmins_w,      INTR_TYPE_2OP, ISD::SMIN, 0),
+  X86_INTRINSIC_DATA(sse2_pminu_b,      INTR_TYPE_2OP, ISD::UMIN, 0),
   X86_INTRINSIC_DATA(sse2_pmulh_w,      INTR_TYPE_2OP, ISD::MULHS, 0),
   X86_INTRINSIC_DATA(sse2_pmulhu_w,     INTR_TYPE_2OP, ISD::MULHU, 0),
   X86_INTRINSIC_DATA(sse2_pmulu_dq,     INTR_TYPE_2OP, X86ISD::PMULUDQ, 0),
@@ -555,14 +1475,14 @@ static const IntrinsicData  IntrinsicsWithoutChain[] = {
   X86_INTRINSIC_DATA(sse3_hsub_ps,      INTR_TYPE_2OP, X86ISD::FHSUB, 0),
   X86_INTRINSIC_DATA(sse41_insertps,    INTR_TYPE_3OP, X86ISD::INSERTPS, 0),
   X86_INTRINSIC_DATA(sse41_packusdw,    INTR_TYPE_2OP, X86ISD::PACKUS, 0),
-  X86_INTRINSIC_DATA(sse41_pmaxsb,      INTR_TYPE_2OP, X86ISD::SMAX, 0),
-  X86_INTRINSIC_DATA(sse41_pmaxsd,      INTR_TYPE_2OP, X86ISD::SMAX, 0),
-  X86_INTRINSIC_DATA(sse41_pmaxud,      INTR_TYPE_2OP, X86ISD::UMAX, 0),
-  X86_INTRINSIC_DATA(sse41_pmaxuw,      INTR_TYPE_2OP, X86ISD::UMAX, 0),
-  X86_INTRINSIC_DATA(sse41_pminsb,      INTR_TYPE_2OP, X86ISD::SMIN, 0),
-  X86_INTRINSIC_DATA(sse41_pminsd,      INTR_TYPE_2OP, X86ISD::SMIN, 0),
-  X86_INTRINSIC_DATA(sse41_pminud,      INTR_TYPE_2OP, X86ISD::UMIN, 0),
-  X86_INTRINSIC_DATA(sse41_pminuw,      INTR_TYPE_2OP, X86ISD::UMIN, 0),
+  X86_INTRINSIC_DATA(sse41_pmaxsb,      INTR_TYPE_2OP, ISD::SMAX, 0),
+  X86_INTRINSIC_DATA(sse41_pmaxsd,      INTR_TYPE_2OP, ISD::SMAX, 0),
+  X86_INTRINSIC_DATA(sse41_pmaxud,      INTR_TYPE_2OP, ISD::UMAX, 0),
+  X86_INTRINSIC_DATA(sse41_pmaxuw,      INTR_TYPE_2OP, ISD::UMAX, 0),
+  X86_INTRINSIC_DATA(sse41_pminsb,      INTR_TYPE_2OP, ISD::SMIN, 0),
+  X86_INTRINSIC_DATA(sse41_pminsd,      INTR_TYPE_2OP, ISD::SMIN, 0),
+  X86_INTRINSIC_DATA(sse41_pminud,      INTR_TYPE_2OP, ISD::UMIN, 0),
+  X86_INTRINSIC_DATA(sse41_pminuw,      INTR_TYPE_2OP, ISD::UMIN, 0),
   X86_INTRINSIC_DATA(sse41_pmovsxbd,    INTR_TYPE_1OP, X86ISD::VSEXT, 0),
   X86_INTRINSIC_DATA(sse41_pmovsxbq,    INTR_TYPE_1OP, X86ISD::VSEXT, 0),
   X86_INTRINSIC_DATA(sse41_pmovsxbw,    INTR_TYPE_1OP, X86ISD::VSEXT, 0),
@@ -576,6 +1496,8 @@ static const IntrinsicData  IntrinsicsWithoutChain[] = {
   X86_INTRINSIC_DATA(sse41_pmovzxwd,    INTR_TYPE_1OP, X86ISD::VZEXT, 0),
   X86_INTRINSIC_DATA(sse41_pmovzxwq,    INTR_TYPE_1OP, X86ISD::VZEXT, 0),
   X86_INTRINSIC_DATA(sse41_pmuldq,      INTR_TYPE_2OP, X86ISD::PMULDQ, 0),
+  X86_INTRINSIC_DATA(sse4a_extrqi,      INTR_TYPE_3OP, X86ISD::EXTRQI, 0),
+  X86_INTRINSIC_DATA(sse4a_insertqi,    INTR_TYPE_4OP, X86ISD::INSERTQI, 0),
   X86_INTRINSIC_DATA(sse_comieq_ss,     COMI, X86ISD::COMI, ISD::SETEQ),
   X86_INTRINSIC_DATA(sse_comige_ss,     COMI, X86ISD::COMI, ISD::SETGE),
   X86_INTRINSIC_DATA(sse_comigt_ss,     COMI, X86ISD::COMI, ISD::SETGT),
@@ -584,6 +1506,8 @@ static const IntrinsicData  IntrinsicsWithoutChain[] = {
   X86_INTRINSIC_DATA(sse_comineq_ss,    COMI, X86ISD::COMI, ISD::SETNE),
   X86_INTRINSIC_DATA(sse_max_ps,        INTR_TYPE_2OP, X86ISD::FMAX, 0),
   X86_INTRINSIC_DATA(sse_min_ps,        INTR_TYPE_2OP, X86ISD::FMIN, 0),
+  X86_INTRINSIC_DATA(sse_rcp_ps,        INTR_TYPE_1OP, X86ISD::FRCP, 0),
+  X86_INTRINSIC_DATA(sse_rsqrt_ps,      INTR_TYPE_1OP, X86ISD::FRSQRT, 0),
   X86_INTRINSIC_DATA(sse_sqrt_ps,       INTR_TYPE_1OP, ISD::FSQRT, 0),
   X86_INTRINSIC_DATA(sse_ucomieq_ss,    COMI, X86ISD::UCOMI, ISD::SETEQ),
   X86_INTRINSIC_DATA(sse_ucomige_ss,    COMI, X86ISD::UCOMI, ISD::SETGE),
diff --git a/lib/Target/X86/X86MCInstLower.cpp b/lib/Target/X86/X86MCInstLower.cpp
index 12204b1b671..68977d38e15 100644
--- a/lib/Target/X86/X86MCInstLower.cpp
+++ b/lib/Target/X86/X86MCInstLower.cpp
@@ -17,6 +17,7 @@
 #include "InstPrinter/X86ATTInstPrinter.h"
 #include "MCTargetDesc/X86BaseInfo.h"
 #include "Utils/X86ShuffleDecode.h"
+#include "llvm/ADT/Optional.h"
 #include "llvm/ADT/SmallString.h"
 #include "llvm/CodeGen/MachineFunction.h"
 #include "llvm/CodeGen/MachineConstantPool.h"
@@ -30,6 +31,7 @@
 #include "llvm/MC/MCCodeEmitter.h"
 #include "llvm/MC/MCContext.h"
 #include "llvm/MC/MCExpr.h"
+#include "llvm/MC/MCFixup.h"
 #include "llvm/MC/MCInst.h"
 #include "llvm/MC/MCInstBuilder.h"
 #include "llvm/MC/MCStreamer.h"
@@ -49,6 +51,8 @@ class X86MCInstLower {
 public:
   X86MCInstLower(const MachineFunction &MF, X86AsmPrinter &asmprinter);
 
+  Optional<MCOperand> LowerMachineOperand(const MachineInstr *MI,
+                                          const MachineOperand &MO) const;
   void Lower(const MachineInstr *MI, MCInst &OutMI) const;
 
   MCSymbol *GetSymbolFromOperand(const MachineOperand &MO) const;
@@ -87,7 +91,7 @@ namespace llvm {
       SmallString<256> Code;
       SmallVector<MCFixup, 4> Fixups;
       raw_svector_ostream VecOS(Code);
-      CodeEmitter->EncodeInstruction(Inst, VecOS, Fixups, STI);
+      CodeEmitter->encodeInstruction(Inst, VecOS, Fixups, STI);
       VecOS.flush();
       CurrentShadowSize += Code.size();
       if (CurrentShadowSize >= RequiredShadowSize)
@@ -105,7 +109,7 @@ namespace llvm {
   }
 
   void X86AsmPrinter::EmitAndCountInstruction(MCInst &Inst) {
-    OutStreamer.EmitInstruction(Inst, getSubtargetInfo());
+    OutStreamer->EmitInstruction(Inst, getSubtargetInfo());
     SMShadowTracker.count(Inst, getSubtargetInfo());
   }
 } // end llvm namespace
@@ -124,9 +128,10 @@ MachineModuleInfoMachO &X86MCInstLower::getMachOMMI() const {
 /// operand to an MCSymbol.
 MCSymbol *X86MCInstLower::
 GetSymbolFromOperand(const MachineOperand &MO) const {
-  const DataLayout *DL = TM.getDataLayout();
+  const DataLayout &DL = MF.getDataLayout();
   assert((MO.isGlobal() || MO.isSymbol() || MO.isMBB()) && "Isn't a symbol reference");
 
+  MCSymbol *Sym = nullptr;
   SmallString<128> Name;
   StringRef Suffix;
 
@@ -146,7 +151,7 @@ GetSymbolFromOperand(const MachineOperand &MO) const {
   }
 
   if (!Suffix.empty())
-    Name += DL->getPrivateGlobalPrefix();
+    Name += DL.getPrivateGlobalPrefix();
 
   unsigned PrefixLen = Name.size();
 
@@ -154,14 +159,16 @@ GetSymbolFromOperand(const MachineOperand &MO) const {
     const GlobalValue *GV = MO.getGlobal();
     AsmPrinter.getNameWithPrefix(Name, GV);
   } else if (MO.isSymbol()) {
-    getMang()->getNameWithPrefix(Name, MO.getSymbolName());
+    Mangler::getNameWithPrefix(Name, MO.getSymbolName(), DL);
   } else if (MO.isMBB()) {
-    Name += MO.getMBB()->getSymbol()->getName();
+    assert(Suffix.empty());
+    Sym = MO.getMBB()->getSymbol();
   }
   unsigned OrigLen = Name.size() - PrefixLen;
 
   Name += Suffix;
-  MCSymbol *Sym = Ctx.GetOrCreateSymbol(Name);
+  if (!Sym)
+    Sym = Ctx.getOrCreateSymbol(Name);
 
   StringRef OrigName = StringRef(Name).substr(PrefixLen, OrigLen);
 
@@ -208,7 +215,7 @@ GetSymbolFromOperand(const MachineOperand &MO) const {
     } else {
       StubSym =
         MachineModuleInfoImpl::
-        StubValueTy(Ctx.GetOrCreateSymbol(OrigName), false);
+        StubValueTy(Ctx.getOrCreateSymbol(OrigName), false);
     }
     break;
   }
@@ -235,10 +242,10 @@ MCOperand X86MCInstLower::LowerSymbolOperand(const MachineOperand &MO,
 
   case X86II::MO_TLVP:      RefKind = MCSymbolRefExpr::VK_TLVP; break;
   case X86II::MO_TLVP_PIC_BASE:
-    Expr = MCSymbolRefExpr::Create(Sym, MCSymbolRefExpr::VK_TLVP, Ctx);
+    Expr = MCSymbolRefExpr::create(Sym, MCSymbolRefExpr::VK_TLVP, Ctx);
     // Subtract the pic base.
-    Expr = MCBinaryExpr::CreateSub(Expr,
-                                  MCSymbolRefExpr::Create(MF.getPICBaseSymbol(),
+    Expr = MCBinaryExpr::createSub(Expr,
+                                  MCSymbolRefExpr::create(MF.getPICBaseSymbol(),
                                                            Ctx),
                                    Ctx);
     break;
@@ -259,10 +266,10 @@ MCOperand X86MCInstLower::LowerSymbolOperand(const MachineOperand &MO,
   case X86II::MO_PIC_BASE_OFFSET:
   case X86II::MO_DARWIN_NONLAZY_PIC_BASE:
   case X86II::MO_DARWIN_HIDDEN_NONLAZY_PIC_BASE:
-    Expr = MCSymbolRefExpr::Create(Sym, Ctx);
+    Expr = MCSymbolRefExpr::create(Sym, Ctx);
     // Subtract the pic base.
-    Expr = MCBinaryExpr::CreateSub(Expr,
-                            MCSymbolRefExpr::Create(MF.getPICBaseSymbol(), Ctx),
+    Expr = MCBinaryExpr::createSub(Expr,
+                            MCSymbolRefExpr::create(MF.getPICBaseSymbol(), Ctx),
                                    Ctx);
     if (MO.isJTI()) {
       assert(MAI.doesSetDirectiveSuppressesReloc());
@@ -270,21 +277,21 @@ MCOperand X86MCInstLower::LowerSymbolOperand(const MachineOperand &MO,
       // relocations the assembler will generate for differences between
       // local labels. This is only safe when the symbols are in the same
       // section so we are restricting it to jumptable references.
-      MCSymbol *Label = Ctx.CreateTempSymbol();
-      AsmPrinter.OutStreamer.EmitAssignment(Label, Expr);
-      Expr = MCSymbolRefExpr::Create(Label, Ctx);
+      MCSymbol *Label = Ctx.createTempSymbol();
+      AsmPrinter.OutStreamer->EmitAssignment(Label, Expr);
+      Expr = MCSymbolRefExpr::create(Label, Ctx);
     }
     break;
   }
 
   if (!Expr)
-    Expr = MCSymbolRefExpr::Create(Sym, RefKind, Ctx);
+    Expr = MCSymbolRefExpr::create(Sym, RefKind, Ctx);
 
   if (!MO.isJTI() && !MO.isMBB() && MO.getOffset())
-    Expr = MCBinaryExpr::CreateAdd(Expr,
-                                   MCConstantExpr::Create(MO.getOffset(), Ctx),
+    Expr = MCBinaryExpr::createAdd(Expr,
+                                   MCConstantExpr::create(MO.getOffset(), Ctx),
                                    Ctx);
-  return MCOperand::CreateExpr(Expr);
+  return MCOperand::createExpr(Expr);
 }
 
 
@@ -394,47 +401,45 @@ static unsigned getRetOpcode(const X86Subtarget &Subtarget) {
   return Subtarget.is64Bit() ? X86::RETQ : X86::RETL;
 }
 
+Optional<MCOperand>
+X86MCInstLower::LowerMachineOperand(const MachineInstr *MI,
+                                    const MachineOperand &MO) const {
+  switch (MO.getType()) {
+  default:
+    MI->dump();
+    llvm_unreachable("unknown operand type");
+  case MachineOperand::MO_Register:
+    // Ignore all implicit register operands.
+    if (MO.isImplicit())
+      return None;
+    return MCOperand::createReg(MO.getReg());
+  case MachineOperand::MO_Immediate:
+    return MCOperand::createImm(MO.getImm());
+  case MachineOperand::MO_MachineBasicBlock:
+  case MachineOperand::MO_GlobalAddress:
+  case MachineOperand::MO_ExternalSymbol:
+    return LowerSymbolOperand(MO, GetSymbolFromOperand(MO));
+  case MachineOperand::MO_MCSymbol:
+    return LowerSymbolOperand(MO, MO.getMCSymbol());
+  case MachineOperand::MO_JumpTableIndex:
+    return LowerSymbolOperand(MO, AsmPrinter.GetJTISymbol(MO.getIndex()));
+  case MachineOperand::MO_ConstantPoolIndex:
+    return LowerSymbolOperand(MO, AsmPrinter.GetCPISymbol(MO.getIndex()));
+  case MachineOperand::MO_BlockAddress:
+    return LowerSymbolOperand(
+        MO, AsmPrinter.GetBlockAddressSymbol(MO.getBlockAddress()));
+  case MachineOperand::MO_RegisterMask:
+    // Ignore call clobbers.
+    return None;
+  }
+}
+
 void X86MCInstLower::Lower(const MachineInstr *MI, MCInst &OutMI) const {
   OutMI.setOpcode(MI->getOpcode());
 
-  for (unsigned i = 0, e = MI->getNumOperands(); i != e; ++i) {
-    const MachineOperand &MO = MI->getOperand(i);
-
-    MCOperand MCOp;
-    switch (MO.getType()) {
-    default:
-      MI->dump();
-      llvm_unreachable("unknown operand type");
-    case MachineOperand::MO_Register:
-      // Ignore all implicit register operands.
-      if (MO.isImplicit()) continue;
-      MCOp = MCOperand::CreateReg(MO.getReg());
-      break;
-    case MachineOperand::MO_Immediate:
-      MCOp = MCOperand::CreateImm(MO.getImm());
-      break;
-    case MachineOperand::MO_MachineBasicBlock:
-    case MachineOperand::MO_GlobalAddress:
-    case MachineOperand::MO_ExternalSymbol:
-      MCOp = LowerSymbolOperand(MO, GetSymbolFromOperand(MO));
-      break;
-    case MachineOperand::MO_JumpTableIndex:
-      MCOp = LowerSymbolOperand(MO, AsmPrinter.GetJTISymbol(MO.getIndex()));
-      break;
-    case MachineOperand::MO_ConstantPoolIndex:
-      MCOp = LowerSymbolOperand(MO, AsmPrinter.GetCPISymbol(MO.getIndex()));
-      break;
-    case MachineOperand::MO_BlockAddress:
-      MCOp = LowerSymbolOperand(MO,
-                     AsmPrinter.GetBlockAddressSymbol(MO.getBlockAddress()));
-      break;
-    case MachineOperand::MO_RegisterMask:
-      // Ignore call clobbers.
-      continue;
-    }
-
-    OutMI.addOperand(MCOp);
-  }
+  for (const MachineOperand &MO : MI->operands())
+    if (auto MaybeMCOp = LowerMachineOperand(MI, MO))
+      OutMI.addOperand(MaybeMCOp.getValue());
 
   // Handle a few special cases to eliminate operand modifiers.
 ReSimplify:
@@ -683,20 +688,8 @@ void X86AsmPrinter::LowerTlsAddr(X86MCInstLower &MCInstLowering,
 
   bool needsPadding = MI.getOpcode() == X86::TLS_addr64;
 
-  MCContext &context = OutStreamer.getContext();
-
-  // @LOCALMOD-START
-  // TODO(dschuff): Instead of testing the target OS we should add (upstream) a
-  // method to one of the MC classes to test whether bundle alignment is enabled
-  // (currently only the AsmBackend has this, which can't be used when emitting
-  // asm files). That facility should be used here and in MCAsmStreamer.cpp
-  // (this should be done once we are unblocked from switching to gas by
-  // default, so we know exactly where we need it).
-  Triple TT(getSubtargetInfo().getTargetTriple());
-  const bool needsBundleLock = TT.getOS() == Triple::NaCl;
-  if (needsBundleLock)
-    OutStreamer.EmitBundleLock(false);
-  // @LOCALMOD-END
+  MCContext &context = OutStreamer->getContext();
+
   if (needsPadding)
     EmitAndCountInstruction(MCInstBuilder(X86::DATA16_PREFIX));
 
@@ -717,33 +710,33 @@ void X86AsmPrinter::LowerTlsAddr(X86MCInstLower &MCInstLowering,
   }
 
   MCSymbol *sym = MCInstLowering.GetSymbolFromOperand(MI.getOperand(3));
-  const MCSymbolRefExpr *symRef = MCSymbolRefExpr::Create(sym, SRVK, context);
+  const MCSymbolRefExpr *symRef = MCSymbolRefExpr::create(sym, SRVK, context);
 
   MCInst LEA;
   if (is64Bits) {
     LEA.setOpcode(X86::LEA64r);
-    LEA.addOperand(MCOperand::CreateReg(X86::RDI)); // dest
-    LEA.addOperand(MCOperand::CreateReg(X86::RIP)); // base
-    LEA.addOperand(MCOperand::CreateImm(1));        // scale
-    LEA.addOperand(MCOperand::CreateReg(0));        // index
-    LEA.addOperand(MCOperand::CreateExpr(symRef));  // disp
-    LEA.addOperand(MCOperand::CreateReg(0));        // seg
+    LEA.addOperand(MCOperand::createReg(X86::RDI)); // dest
+    LEA.addOperand(MCOperand::createReg(X86::RIP)); // base
+    LEA.addOperand(MCOperand::createImm(1));        // scale
+    LEA.addOperand(MCOperand::createReg(0));        // index
+    LEA.addOperand(MCOperand::createExpr(symRef));  // disp
+    LEA.addOperand(MCOperand::createReg(0));        // seg
   } else if (SRVK == MCSymbolRefExpr::VK_TLSLDM) {
     LEA.setOpcode(X86::LEA32r);
-    LEA.addOperand(MCOperand::CreateReg(X86::EAX)); // dest
-    LEA.addOperand(MCOperand::CreateReg(X86::EBX)); // base
-    LEA.addOperand(MCOperand::CreateImm(1));        // scale
-    LEA.addOperand(MCOperand::CreateReg(0));        // index
-    LEA.addOperand(MCOperand::CreateExpr(symRef));  // disp
-    LEA.addOperand(MCOperand::CreateReg(0));        // seg
+    LEA.addOperand(MCOperand::createReg(X86::EAX)); // dest
+    LEA.addOperand(MCOperand::createReg(X86::EBX)); // base
+    LEA.addOperand(MCOperand::createImm(1));        // scale
+    LEA.addOperand(MCOperand::createReg(0));        // index
+    LEA.addOperand(MCOperand::createExpr(symRef));  // disp
+    LEA.addOperand(MCOperand::createReg(0));        // seg
   } else {
     LEA.setOpcode(X86::LEA32r);
-    LEA.addOperand(MCOperand::CreateReg(X86::EAX)); // dest
-    LEA.addOperand(MCOperand::CreateReg(0));        // base
-    LEA.addOperand(MCOperand::CreateImm(1));        // scale
-    LEA.addOperand(MCOperand::CreateReg(X86::EBX)); // index
-    LEA.addOperand(MCOperand::CreateExpr(symRef));  // disp
-    LEA.addOperand(MCOperand::CreateReg(0));        // seg
+    LEA.addOperand(MCOperand::createReg(X86::EAX)); // dest
+    LEA.addOperand(MCOperand::createReg(0));        // base
+    LEA.addOperand(MCOperand::createImm(1));        // scale
+    LEA.addOperand(MCOperand::createReg(X86::EBX)); // index
+    LEA.addOperand(MCOperand::createExpr(symRef));  // disp
+    LEA.addOperand(MCOperand::createReg(0));        // seg
   }
   EmitAndCountInstruction(LEA);
 
@@ -754,19 +747,15 @@ void X86AsmPrinter::LowerTlsAddr(X86MCInstLower &MCInstLowering,
   }
 
   StringRef name = is64Bits ? "__tls_get_addr" : "___tls_get_addr";
-  MCSymbol *tlsGetAddr = context.GetOrCreateSymbol(name);
+  MCSymbol *tlsGetAddr = context.getOrCreateSymbol(name);
   const MCSymbolRefExpr *tlsRef =
-    MCSymbolRefExpr::Create(tlsGetAddr,
+    MCSymbolRefExpr::create(tlsGetAddr,
                             MCSymbolRefExpr::VK_PLT,
                             context);
 
   EmitAndCountInstruction(MCInstBuilder(is64Bits ? X86::CALL64pcrel32
                                                  : X86::CALLpcrel32)
                             .addExpr(tlsRef));
-  // @LOCALMOD-START
-  if (needsBundleLock)
-    OutStreamer.EmitBundleUnlock();
-  // @LOCALMOD-END
 }
 
 /// \brief Emit the optimal amount of multi-byte nops on X86.
@@ -820,62 +809,89 @@ static void EmitNops(MCStreamer &OS, unsigned NumBytes, bool Is64Bit, const MCSu
   } // while (NumBytes)
 }
 
-static void LowerSTATEPOINT(MCStreamer &OS, StackMaps &SM,
-                            const MachineInstr &MI, bool Is64Bit,
-                            const TargetMachine& TM,
-                            const MCSubtargetInfo& STI,
-                            X86MCInstLower &MCInstLowering) {
-  assert(Is64Bit && "Statepoint currently only supports X86-64");
-
-  // Lower call target and choose correct opcode
-  const MachineOperand &call_target = StatepointOpers(&MI).getCallTarget();
-  MCOperand call_target_mcop;
-  unsigned call_opcode;
-  switch (call_target.getType()) {
-  case MachineOperand::MO_GlobalAddress:
-  case MachineOperand::MO_ExternalSymbol:
-    call_target_mcop = MCInstLowering.LowerSymbolOperand(
-      call_target,
-      MCInstLowering.GetSymbolFromOperand(call_target));
-    call_opcode = X86::CALL64pcrel32;
-    // Currently, we only support relative addressing with statepoints.
-    // Otherwise, we'll need a scratch register to hold the target
-    // address.  You'll fail asserts during load & relocation if this
-    // symbol is to far away. (TODO: support non-relative addressing)
-    break;
-  case MachineOperand::MO_Immediate:
-    call_target_mcop = MCOperand::CreateImm(call_target.getImm());
-    call_opcode = X86::CALL64pcrel32;
-    // Currently, we only support relative addressing with statepoints.
-    // Otherwise, we'll need a scratch register to hold the target
-    // immediate.  You'll fail asserts during load & relocation if this
-    // address is to far away. (TODO: support non-relative addressing)
-    break;
-  case MachineOperand::MO_Register:
-    call_target_mcop = MCOperand::CreateReg(call_target.getReg());
-    call_opcode = X86::CALL64r;
-    break;
-  default:
-    llvm_unreachable("Unsupported operand type in statepoint call target");
-    break;
-  }
+void X86AsmPrinter::LowerSTATEPOINT(const MachineInstr &MI,
+                                    X86MCInstLower &MCIL) {
+  assert(Subtarget->is64Bit() && "Statepoint currently only supports X86-64");
 
-  // Emit call
-  MCInst call_inst;
-  call_inst.setOpcode(call_opcode);
-  call_inst.addOperand(call_target_mcop);
-  OS.EmitInstruction(call_inst, STI);
+  StatepointOpers SOpers(&MI);
+  if (unsigned PatchBytes = SOpers.getNumPatchBytes()) {
+    EmitNops(*OutStreamer, PatchBytes, Subtarget->is64Bit(),
+             getSubtargetInfo());
+  } else {
+    // Lower call target and choose correct opcode
+    const MachineOperand &CallTarget = SOpers.getCallTarget();
+    MCOperand CallTargetMCOp;
+    unsigned CallOpcode;
+    switch (CallTarget.getType()) {
+    case MachineOperand::MO_GlobalAddress:
+    case MachineOperand::MO_ExternalSymbol:
+      CallTargetMCOp = MCIL.LowerSymbolOperand(
+          CallTarget, MCIL.GetSymbolFromOperand(CallTarget));
+      CallOpcode = X86::CALL64pcrel32;
+      // Currently, we only support relative addressing with statepoints.
+      // Otherwise, we'll need a scratch register to hold the target
+      // address.  You'll fail asserts during load & relocation if this
+      // symbol is to far away. (TODO: support non-relative addressing)
+      break;
+    case MachineOperand::MO_Immediate:
+      CallTargetMCOp = MCOperand::createImm(CallTarget.getImm());
+      CallOpcode = X86::CALL64pcrel32;
+      // Currently, we only support relative addressing with statepoints.
+      // Otherwise, we'll need a scratch register to hold the target
+      // immediate.  You'll fail asserts during load & relocation if this
+      // address is to far away. (TODO: support non-relative addressing)
+      break;
+    case MachineOperand::MO_Register:
+      CallTargetMCOp = MCOperand::createReg(CallTarget.getReg());
+      CallOpcode = X86::CALL64r;
+      break;
+    default:
+      llvm_unreachable("Unsupported operand type in statepoint call target");
+      break;
+    }
+
+    // Emit call
+    MCInst CallInst;
+    CallInst.setOpcode(CallOpcode);
+    CallInst.addOperand(CallTargetMCOp);
+    OutStreamer->EmitInstruction(CallInst, getSubtargetInfo());
+  }
 
   // Record our statepoint node in the same section used by STACKMAP
   // and PATCHPOINT
   SM.recordStatepoint(MI);
 }
 
+void X86AsmPrinter::LowerFAULTING_LOAD_OP(const MachineInstr &MI,
+                                       X86MCInstLower &MCIL) {
+  // FAULTING_LOAD_OP <def>, <handler label>, <load opcode>, <load operands>
+
+  unsigned LoadDefRegister = MI.getOperand(0).getReg();
+  MCSymbol *HandlerLabel = MI.getOperand(1).getMCSymbol();
+  unsigned LoadOpcode = MI.getOperand(2).getImm();
+  unsigned LoadOperandsBeginIdx = 3;
+
+  FM.recordFaultingOp(FaultMaps::FaultingLoad, HandlerLabel);
+
+  MCInst LoadMI;
+  LoadMI.setOpcode(LoadOpcode);
+
+  if (LoadDefRegister != X86::NoRegister)
+    LoadMI.addOperand(MCOperand::createReg(LoadDefRegister));
+
+  for (auto I = MI.operands_begin() + LoadOperandsBeginIdx,
+            E = MI.operands_end();
+       I != E; ++I)
+    if (auto MaybeOperand = MCIL.LowerMachineOperand(&MI, *I))
+      LoadMI.addOperand(MaybeOperand.getValue());
+
+  OutStreamer->EmitInstruction(LoadMI, getSubtargetInfo());
+}
 
 // Lower a stackmap of the form:
 // <id>, <shadowBytes>, ...
 void X86AsmPrinter::LowerSTACKMAP(const MachineInstr &MI) {
-  SMShadowTracker.emitShadowPadding(OutStreamer, getSubtargetInfo());
+  SMShadowTracker.emitShadowPadding(*OutStreamer, getSubtargetInfo());
   SM.recordStackMap(MI);
   unsigned NumShadowBytes = MI.getOperand(1).getImm();
   SMShadowTracker.reset(NumShadowBytes);
@@ -883,18 +899,40 @@ void X86AsmPrinter::LowerSTACKMAP(const MachineInstr &MI) {
 
 // Lower a patchpoint of the form:
 // [<def>], <id>, <numBytes>, <target>, <numArgs>, <cc>, ...
-void X86AsmPrinter::LowerPATCHPOINT(const MachineInstr &MI) {
+void X86AsmPrinter::LowerPATCHPOINT(const MachineInstr &MI,
+                                    X86MCInstLower &MCIL) {
   assert(Subtarget->is64Bit() && "Patchpoint currently only supports X86-64");
 
-  SMShadowTracker.emitShadowPadding(OutStreamer, getSubtargetInfo());
+  SMShadowTracker.emitShadowPadding(*OutStreamer, getSubtargetInfo());
 
   SM.recordPatchPoint(MI);
 
   PatchPointOpers opers(&MI);
   unsigned ScratchIdx = opers.getNextScratchIdx();
   unsigned EncodedBytes = 0;
-  int64_t CallTarget = opers.getMetaOper(PatchPointOpers::TargetPos).getImm();
-  if (CallTarget) {
+  const MachineOperand &CalleeMO =
+    opers.getMetaOper(PatchPointOpers::TargetPos);
+
+  // Check for null target. If target is non-null (i.e. is non-zero or is
+  // symbolic) then emit a call.
+  if (!(CalleeMO.isImm() && !CalleeMO.getImm())) {
+    MCOperand CalleeMCOp;
+    switch (CalleeMO.getType()) {
+    default:
+      /// FIXME: Add a verifier check for bad callee types.
+      llvm_unreachable("Unrecognized callee operand type.");
+    case MachineOperand::MO_Immediate:
+      if (CalleeMO.getImm())
+        CalleeMCOp = MCOperand::createImm(CalleeMO.getImm());
+      break;
+    case MachineOperand::MO_ExternalSymbol:
+    case MachineOperand::MO_GlobalAddress:
+      CalleeMCOp =
+        MCIL.LowerSymbolOperand(CalleeMO,
+                                MCIL.GetSymbolFromOperand(CalleeMO));
+      break;
+    }
+
     // Emit MOV to materialize the target address and the CALL to target.
     // This is encoded with 12-13 bytes, depending on which register is used.
     unsigned ScratchReg = MI.getOperand(ScratchIdx).getReg();
@@ -902,16 +940,18 @@ void X86AsmPrinter::LowerPATCHPOINT(const MachineInstr &MI) {
       EncodedBytes = 13;
     else
       EncodedBytes = 12;
-    EmitAndCountInstruction(MCInstBuilder(X86::MOV64ri).addReg(ScratchReg)
-                                                       .addImm(CallTarget));
+
+    EmitAndCountInstruction(
+        MCInstBuilder(X86::MOV64ri).addReg(ScratchReg).addOperand(CalleeMCOp));
     EmitAndCountInstruction(MCInstBuilder(X86::CALL64r).addReg(ScratchReg));
   }
+
   // Emit padding.
   unsigned NumBytes = opers.getMetaOper(PatchPointOpers::NBytesPos).getImm();
   assert(NumBytes >= EncodedBytes &&
          "Patchpoint can't request size less than the length of a call.");
 
-  EmitNops(OutStreamer, NumBytes - EncodedBytes, Subtarget->is64Bit(),
+  EmitNops(*OutStreamer, NumBytes - EncodedBytes, Subtarget->is64Bit(),
            getSubtargetInfo());
 }
 
@@ -1007,29 +1047,13 @@ void X86AsmPrinter::EmitInstruction(const MachineInstr *MI) {
   X86MCInstLower MCInstLowering(*MF, *this);
   const X86RegisterInfo *RI = MF->getSubtarget<X86Subtarget>().getRegisterInfo();
 
-  // @LOCALMOD-START
-  // MI Bundles are used to represent bundle-locked groups.
-  // MBB iterators skip bundles, so when we reach a bundle head, unpack it here.
-  if (MI->isBundle()) {
-    MachineBasicBlock::const_iterator MBBI(MI);
-    MachineBasicBlock::const_instr_iterator MII (MBBI.getInstrIterator());
-    ++MBBI;
-    OutStreamer.EmitBundleLock(false);
-    while (++MII != MBBI) {
-      EmitInstruction(MII);
-    }
-    OutStreamer.EmitBundleUnlock();
-    return;
-  }
-  // @LOCALMOD-END
-
   switch (MI->getOpcode()) {
   case TargetOpcode::DBG_VALUE:
     llvm_unreachable("Should be handled target independently");
 
   // Emit nothing here but a comment if we can.
   case X86::Int_MemBarrier:
-    OutStreamer.emitRawComment("MEMBARRIER");
+    OutStreamer->emitRawComment("MEMBARRIER");
     return;
 
 
@@ -1037,8 +1061,8 @@ void X86AsmPrinter::EmitInstruction(const MachineInstr *MI) {
   case X86::EH_RETURN64: {
     // Lower these as normal, but add some comments.
     unsigned Reg = MI->getOperand(0).getReg();
-    OutStreamer.AddComment(StringRef("eh_return, addr: %") +
-                           X86ATTInstPrinter::getRegisterName(Reg));
+    OutStreamer->AddComment(StringRef("eh_return, addr: %") +
+                            X86ATTInstPrinter::getRegisterName(Reg));
     break;
   }
   case X86::TAILJMPr:
@@ -1051,7 +1075,7 @@ void X86AsmPrinter::EmitInstruction(const MachineInstr *MI) {
   case X86::TAILJMPm64_REX:
   case X86::TAILJMPd64_REX:
     // Lower these as normal, but add some comments.
-    OutStreamer.AddComment("TAILCALL");
+    OutStreamer->AddComment("TAILCALL");
     break;
 
   case X86::TLS_addr32:
@@ -1072,10 +1096,10 @@ void X86AsmPrinter::EmitInstruction(const MachineInstr *MI) {
     // FIXME: We would like an efficient form for this, so we don't have to do a
     // lot of extra uniquing.
     EmitAndCountInstruction(MCInstBuilder(X86::CALLpcrel32)
-      .addExpr(MCSymbolRefExpr::Create(PICBase, OutContext)));
+      .addExpr(MCSymbolRefExpr::create(PICBase, OutContext)));
 
     // Emit the label.
-    OutStreamer.EmitLabel(PICBase);
+    OutStreamer->EmitLabel(PICBase);
 
     // popl $reg
     EmitAndCountInstruction(MCInstBuilder(X86::POP32r)
@@ -1095,41 +1119,18 @@ void X86AsmPrinter::EmitInstruction(const MachineInstr *MI) {
     //   MYGLOBAL + (. - PICBASE)
     // However, we can't generate a ".", so just emit a new label here and refer
     // to it.
-
-    // @LOCALMOD-START
-    // When using the gas assembler, emitting a new label and then using it in
-    // the difference expression gives the wrong value when the instruction
-    // following the label gets bundle padding added to it. (The label will
-    // point to the nop padding instead of the instruction, which works when
-    // jumping to the label, but not when calculating GOT addresses. See
-    // https://code.google.com/p/nativeclient/issues/detail?id=3982) For this
-    // case, just emit the instruction using the "." directly rather than going
-    // through all the MC layers. The LLVM assembler is smart enough to make the
-    // label point to the actual instruction rather than the nop, so this hack
-    // isn't necessary.
-    if (Subtarget->isTargetNaCl32() &&
-        OutStreamer.hasRawTextSupport()) {
-      std::string OS;
-      raw_string_ostream ROS(OS);
-      ROS << "\taddl\t$_GLOBAL_OFFSET_TABLE_+(.-" << *MF->getPICBaseSymbol()
-          << "), %"
-          << X86ATTInstPrinter::getRegisterName(MI->getOperand(0).getReg());
-      OutStreamer.EmitRawText(ROS.str());
-      return;
-    }
-    // @LOCALMOD-END
-    MCSymbol *DotSym = OutContext.CreateTempSymbol();
-    OutStreamer.EmitLabel(DotSym);
+    MCSymbol *DotSym = OutContext.createTempSymbol();
+    OutStreamer->EmitLabel(DotSym);
 
     // Now that we have emitted the label, lower the complex operand expression.
     MCSymbol *OpSym = MCInstLowering.GetSymbolFromOperand(MI->getOperand(2));
 
-    const MCExpr *DotExpr = MCSymbolRefExpr::Create(DotSym, OutContext);
+    const MCExpr *DotExpr = MCSymbolRefExpr::create(DotSym, OutContext);
     const MCExpr *PICBase =
-      MCSymbolRefExpr::Create(MF->getPICBaseSymbol(), OutContext);
-    DotExpr = MCBinaryExpr::CreateSub(DotExpr, PICBase, OutContext);
+      MCSymbolRefExpr::create(MF->getPICBaseSymbol(), OutContext);
+    DotExpr = MCBinaryExpr::createSub(DotExpr, PICBase, OutContext);
 
-    DotExpr = MCBinaryExpr::CreateAdd(MCSymbolRefExpr::Create(OpSym,OutContext),
+    DotExpr = MCBinaryExpr::createAdd(MCSymbolRefExpr::create(OpSym,OutContext),
                                       DotExpr, OutContext);
 
     EmitAndCountInstruction(MCInstBuilder(X86::ADD32ri)
@@ -1139,14 +1140,16 @@ void X86AsmPrinter::EmitInstruction(const MachineInstr *MI) {
     return;
   }
   case TargetOpcode::STATEPOINT:
-    return LowerSTATEPOINT(OutStreamer, SM, *MI, Subtarget->is64Bit(), TM,
-                           getSubtargetInfo(), MCInstLowering);
+    return LowerSTATEPOINT(*MI, MCInstLowering);
+
+  case TargetOpcode::FAULTING_LOAD_OP:
+    return LowerFAULTING_LOAD_OP(*MI, MCInstLowering);
 
   case TargetOpcode::STACKMAP:
     return LowerSTACKMAP(*MI);
 
   case TargetOpcode::PATCHPOINT:
-    return LowerPATCHPOINT(*MI);
+    return LowerPATCHPOINT(*MI, MCInstLowering);
 
   case X86::MORESTACK_RET:
     EmitAndCountInstruction(MCInstBuilder(getRetOpcode(*Subtarget)));
@@ -1161,34 +1164,34 @@ void X86AsmPrinter::EmitInstruction(const MachineInstr *MI) {
     return;
 
   case X86::SEH_PushReg:
-    OutStreamer.EmitWinCFIPushReg(RI->getSEHRegNum(MI->getOperand(0).getImm()));
+    OutStreamer->EmitWinCFIPushReg(RI->getSEHRegNum(MI->getOperand(0).getImm()));
     return;
 
   case X86::SEH_SaveReg:
-    OutStreamer.EmitWinCFISaveReg(RI->getSEHRegNum(MI->getOperand(0).getImm()),
-                                  MI->getOperand(1).getImm());
+    OutStreamer->EmitWinCFISaveReg(RI->getSEHRegNum(MI->getOperand(0).getImm()),
+                                   MI->getOperand(1).getImm());
     return;
 
   case X86::SEH_SaveXMM:
-    OutStreamer.EmitWinCFISaveXMM(RI->getSEHRegNum(MI->getOperand(0).getImm()),
-                                  MI->getOperand(1).getImm());
+    OutStreamer->EmitWinCFISaveXMM(RI->getSEHRegNum(MI->getOperand(0).getImm()),
+                                   MI->getOperand(1).getImm());
     return;
 
   case X86::SEH_StackAlloc:
-    OutStreamer.EmitWinCFIAllocStack(MI->getOperand(0).getImm());
+    OutStreamer->EmitWinCFIAllocStack(MI->getOperand(0).getImm());
     return;
 
   case X86::SEH_SetFrame:
-    OutStreamer.EmitWinCFISetFrame(RI->getSEHRegNum(MI->getOperand(0).getImm()),
-                                   MI->getOperand(1).getImm());
+    OutStreamer->EmitWinCFISetFrame(RI->getSEHRegNum(MI->getOperand(0).getImm()),
+                                    MI->getOperand(1).getImm());
     return;
 
   case X86::SEH_PushFrame:
-    OutStreamer.EmitWinCFIPushFrame(MI->getOperand(0).getImm());
+    OutStreamer->EmitWinCFIPushFrame(MI->getOperand(0).getImm());
     return;
 
   case X86::SEH_EndPrologue:
-    OutStreamer.EmitWinCFIEndProlog();
+    OutStreamer->EmitWinCFIEndProlog();
     return;
 
   case X86::SEH_Epilogue: {
@@ -1212,7 +1215,7 @@ void X86AsmPrinter::EmitInstruction(const MachineInstr *MI) {
   case X86::PSHUFBrm:
   case X86::VPSHUFBrm:
   case X86::VPSHUFBYrm: {
-    if (!OutStreamer.isVerboseAsm())
+    if (!OutStreamer->isVerboseAsm())
       break;
     assert(MI->getNumOperands() > 5 &&
            "We should always have at least 5 operands!");
@@ -1224,7 +1227,7 @@ void X86AsmPrinter::EmitInstruction(const MachineInstr *MI) {
       SmallVector<int, 16> Mask;
       DecodePSHUFBMask(C, Mask);
       if (!Mask.empty())
-        OutStreamer.AddComment(getShuffleComment(DstOp, SrcOp, Mask));
+        OutStreamer->AddComment(getShuffleComment(DstOp, SrcOp, Mask));
     }
     break;
   }
@@ -1232,7 +1235,7 @@ void X86AsmPrinter::EmitInstruction(const MachineInstr *MI) {
   case X86::VPERMILPDrm:
   case X86::VPERMILPSYrm:
   case X86::VPERMILPDYrm: {
-    if (!OutStreamer.isVerboseAsm())
+    if (!OutStreamer->isVerboseAsm())
       break;
     assert(MI->getNumOperands() > 5 &&
            "We should always have at least 5 operands!");
@@ -1244,7 +1247,7 @@ void X86AsmPrinter::EmitInstruction(const MachineInstr *MI) {
       SmallVector<int, 16> Mask;
       DecodeVPERMILPMask(C, Mask);
       if (!Mask.empty())
-        OutStreamer.AddComment(getShuffleComment(DstOp, SrcOp, Mask));
+        OutStreamer->AddComment(getShuffleComment(DstOp, SrcOp, Mask));
     }
     break;
   }
@@ -1269,7 +1272,7 @@ void X86AsmPrinter::EmitInstruction(const MachineInstr *MI) {
   case X86::MOVDQUrm:
   case X86::VMOVDQUrm:
   case X86::VMOVDQUYrm:
-    if (!OutStreamer.isVerboseAsm())
+    if (!OutStreamer->isVerboseAsm())
       break;
     if (MI->getNumOperands() > 4)
     if (auto *C = getConstantFromPool(*MI, MI->getOperand(4))) {
@@ -1292,7 +1295,7 @@ void X86AsmPrinter::EmitInstruction(const MachineInstr *MI) {
             CS << "?";
         }
         CS << "]";
-        OutStreamer.AddComment(CS.str());
+        OutStreamer->AddComment(CS.str());
       } else if (auto *CV = dyn_cast<ConstantVector>(C)) {
         CS << "<";
         for (int i = 0, NumOperands = CV->getNumOperands(); i < NumOperands; ++i) {
@@ -1312,7 +1315,7 @@ void X86AsmPrinter::EmitInstruction(const MachineInstr *MI) {
           }
         }
         CS << ">";
-        OutStreamer.AddComment(CS.str());
+        OutStreamer->AddComment(CS.str());
       }
     }
     break;
@@ -1330,9 +1333,9 @@ void X86AsmPrinter::EmitInstruction(const MachineInstr *MI) {
     SMShadowTracker.count(TmpInst, getSubtargetInfo());
     // Then flush the shadow so that we fill with nops before the call, not
     // after it.
-    SMShadowTracker.emitShadowPadding(OutStreamer, getSubtargetInfo());
+    SMShadowTracker.emitShadowPadding(*OutStreamer, getSubtargetInfo());
     // Then emit the call
-    OutStreamer.EmitInstruction(TmpInst, getSubtargetInfo());
+    OutStreamer->EmitInstruction(TmpInst, getSubtargetInfo());
     return;
   }
 
diff --git a/lib/Target/X86/X86MachineFunctionInfo.cpp b/lib/Target/X86/X86MachineFunctionInfo.cpp
index ac2cdc8c656..c9e636f1eb0 100644
--- a/lib/Target/X86/X86MachineFunctionInfo.cpp
+++ b/lib/Target/X86/X86MachineFunctionInfo.cpp
@@ -1,4 +1,4 @@
-//===-- X86MachineFuctionInfo.cpp - X86 machine function info -------------===//
+//===-- X86MachineFunctionInfo.cpp - X86 machine function info ------------===//
 //
 //                     The LLVM Compiler Infrastructure
 //
diff --git a/lib/Target/X86/X86MachineFunctionInfo.h b/lib/Target/X86/X86MachineFunctionInfo.h
index d598b55aae3..092f5b10d21 100644
--- a/lib/Target/X86/X86MachineFunctionInfo.h
+++ b/lib/Target/X86/X86MachineFunctionInfo.h
@@ -1,4 +1,4 @@
-//===-- X86MachineFuctionInfo.h - X86 machine function info -----*- C++ -*-===//
+//===-- X86MachineFunctionInfo.h - X86 machine function info ----*- C++ -*-===//
 //
 //                     The LLVM Compiler Infrastructure
 //
@@ -30,59 +30,67 @@ class X86MachineFunctionInfo : public MachineFunctionInfo {
   /// pointer for reasons other than it containing dynamic allocation or
   /// that FP eliminatation is turned off. For example, Cygwin main function
   /// contains stack pointer re-alignment code which requires FP.
-  bool ForceFramePointer;
+  bool ForceFramePointer = false;
 
   /// RestoreBasePointerOffset - Non-zero if the function has base pointer
   /// and makes call to llvm.eh.sjlj.setjmp. When non-zero, the value is a
   /// displacement from the frame pointer to a slot where the base pointer
   /// is stashed.
-  signed char RestoreBasePointerOffset;
+  signed char RestoreBasePointerOffset = 0;
 
   /// CalleeSavedFrameSize - Size of the callee-saved register portion of the
   /// stack frame in bytes.
-  unsigned CalleeSavedFrameSize;
+  unsigned CalleeSavedFrameSize = 0;
 
   /// BytesToPopOnReturn - Number of bytes function pops on return (in addition
   /// to the space used by the return address).
   /// Used on windows platform for stdcall & fastcall name decoration
-  unsigned BytesToPopOnReturn;
+  unsigned BytesToPopOnReturn = 0;
 
   /// ReturnAddrIndex - FrameIndex for return slot.
-  int ReturnAddrIndex;
+  int ReturnAddrIndex = 0;
 
   /// \brief FrameIndex for return slot.
-  int FrameAddrIndex;
+  int FrameAddrIndex = 0;
 
   /// TailCallReturnAddrDelta - The number of bytes by which return address
   /// stack slot is moved as the result of tail call optimization.
-  int TailCallReturnAddrDelta;
+  int TailCallReturnAddrDelta = 0;
 
   /// SRetReturnReg - Some subtargets require that sret lowering includes
   /// returning the value of the returned struct in a register. This field
   /// holds the virtual register into which the sret argument is passed.
-  unsigned SRetReturnReg;
+  unsigned SRetReturnReg = 0;
 
   /// GlobalBaseReg - keeps track of the virtual register initialized for
   /// use as the global base register. This is used for PIC in some PIC
   /// relocation models.
-  unsigned GlobalBaseReg;
+  unsigned GlobalBaseReg = 0;
 
   /// VarArgsFrameIndex - FrameIndex for start of varargs area.
-  int VarArgsFrameIndex;
+  int VarArgsFrameIndex = 0;
   /// RegSaveFrameIndex - X86-64 vararg func register save area.
-  int RegSaveFrameIndex;
+  int RegSaveFrameIndex = 0;
   /// VarArgsGPOffset - X86-64 vararg func int reg offset.
-  unsigned VarArgsGPOffset;
+  unsigned VarArgsGPOffset = 0;
   /// VarArgsFPOffset - X86-64 vararg func fp reg offset.
-  unsigned VarArgsFPOffset;
+  unsigned VarArgsFPOffset = 0;
   /// ArgumentStackSize - The number of bytes on stack consumed by the arguments
   /// being passed on the stack.
-  unsigned ArgumentStackSize;
+  unsigned ArgumentStackSize = 0;
   /// NumLocalDynamics - Number of local-dynamic TLS accesses.
-  unsigned NumLocalDynamics;
+  unsigned NumLocalDynamics = 0;
   /// HasPushSequences - Keeps track of whether this function uses sequences
   /// of pushes to pass function parameters.
-  bool HasPushSequences;
+  bool HasPushSequences = false;
+
+  /// True if the function uses llvm.x86.seh.restoreframe, and it needed a spill
+  /// slot for the frame pointer.
+  bool HasSEHFramePtrSave = false;
+
+  /// The frame index of a stack object containing the original frame pointer
+  /// used to address arguments in a function using a base pointer.
+  int SEHFramePtrSaveIndex = 0;
 
 private:
   /// ForwardedMustTailRegParms - A list of virtual and physical registers
@@ -90,40 +98,9 @@ class X86MachineFunctionInfo : public MachineFunctionInfo {
   SmallVector<ForwardedRegister, 1> ForwardedMustTailRegParms;
 
 public:
-  X86MachineFunctionInfo() : ForceFramePointer(false),
-                             RestoreBasePointerOffset(0),
-                             CalleeSavedFrameSize(0),
-                             BytesToPopOnReturn(0),
-                             ReturnAddrIndex(0),
-                             FrameAddrIndex(0),
-                             TailCallReturnAddrDelta(0),
-                             SRetReturnReg(0),
-                             GlobalBaseReg(0),
-                             VarArgsFrameIndex(0),
-                             RegSaveFrameIndex(0),
-                             VarArgsGPOffset(0),
-                             VarArgsFPOffset(0),
-                             ArgumentStackSize(0),
-                             NumLocalDynamics(0),
-                             HasPushSequences(false) {}
-
-  explicit X86MachineFunctionInfo(MachineFunction &MF)
-    : ForceFramePointer(false),
-      RestoreBasePointerOffset(0),
-      CalleeSavedFrameSize(0),
-      BytesToPopOnReturn(0),
-      ReturnAddrIndex(0),
-      FrameAddrIndex(0),
-      TailCallReturnAddrDelta(0),
-      SRetReturnReg(0),
-      GlobalBaseReg(0),
-      VarArgsFrameIndex(0),
-      RegSaveFrameIndex(0),
-      VarArgsGPOffset(0),
-      VarArgsFPOffset(0),
-      ArgumentStackSize(0),
-      NumLocalDynamics(0),
-      HasPushSequences(false) {}
+  X86MachineFunctionInfo() = default;
+
+  explicit X86MachineFunctionInfo(MachineFunction &MF) {}
 
   bool getForceFramePointer() const { return ForceFramePointer;}
   void setForceFramePointer(bool forceFP) { ForceFramePointer = forceFP; }
@@ -174,6 +151,12 @@ class X86MachineFunctionInfo : public MachineFunctionInfo {
   unsigned getNumLocalDynamicTLSAccesses() const { return NumLocalDynamics; }
   void incNumLocalDynamicTLSAccesses() { ++NumLocalDynamics; }
 
+  bool getHasSEHFramePtrSave() const { return HasSEHFramePtrSave; }
+  void setHasSEHFramePtrSave(bool V) { HasSEHFramePtrSave = V; }
+
+  int getSEHFramePtrSaveIndex() const { return SEHFramePtrSaveIndex; }
+  void setSEHFramePtrSaveIndex(int Index) { SEHFramePtrSaveIndex = Index; }
+
   SmallVectorImpl<ForwardedRegister> &getForwardedMustTailRegParms() {
     return ForwardedMustTailRegParms;
   }
diff --git a/lib/Target/X86/X86NaClDecls.h b/lib/Target/X86/X86NaClDecls.h
deleted file mode 100644
index 4050187c68c..00000000000
--- a/lib/Target/X86/X86NaClDecls.h
+++ /dev/null
@@ -1,28 +0,0 @@
-//===-- X86NaClDecls.h - Common X86 NaCl declarations -----------*- C++ -*-===//
-//
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
-//
-//===----------------------------------------------------------------------===//
-//
-// This file provides various NaCl-related declarations for the X86-32
-// and X86-64 architectures.
-//
-//===----------------------------------------------------------------------===//
-
-#ifndef X86NACLDECLS_H
-#define X86NACLDECLS_H
-
-#include "llvm/Support/CommandLine.h"
-
-using namespace llvm;
-
-extern const int kNaClX86InstructionBundleSize;
-
-extern cl::opt<bool> FlagRestrictR15;
-extern cl::opt<bool> FlagUseZeroBasedSandbox;
-extern cl::opt<bool> FlagHideSandboxBase;
-
-#endif    // X86NACLDECLS_H
diff --git a/lib/Target/X86/X86NaClRewritePass.cpp b/lib/Target/X86/X86NaClRewritePass.cpp
deleted file mode 100644
index 761c697eb9d..00000000000
--- a/lib/Target/X86/X86NaClRewritePass.cpp
+++ /dev/null
@@ -1,768 +0,0 @@
-//=== X86NaClRewritePAss.cpp - Rewrite instructions for NaCl SFI --*- C++ -*-=//
-//
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
-//
-//===----------------------------------------------------------------------===//
-//
-// This file contains a pass that ensures stores and loads and stack/frame
-// pointer addresses are within the NaCl sandbox (for x86-64).
-// It also ensures that indirect control flow follows NaCl requirments.
-//
-// The other major portion of rewriting for NaCl is done in X86InstrNaCl.cpp,
-// which is responsible for expanding the NaCl-specific operations introduced
-// here and also the intrinsic functions to support setjmp, etc.
-//===----------------------------------------------------------------------===//
-#define DEBUG_TYPE "x86-sandboxing"
-
-#include "X86.h"
-#include "X86InstrInfo.h"
-#include "X86NaClDecls.h"
-#include "X86Subtarget.h"
-#include "llvm/CodeGen/MachineFunctionPass.h"
-#include "llvm/CodeGen/MachineInstr.h"
-#include "llvm/CodeGen/MachineInstrBuilder.h"
-#include "llvm/CodeGen/MachineJumpTableInfo.h"
-#include "llvm/Support/CommandLine.h"
-#include "llvm/Support/Debug.h"
-#include "llvm/Support/FormattedStream.h"
-#include "llvm/Support/raw_ostream.h"
-#include "llvm/Support/CommandLine.h"
-
-using namespace llvm;
-
-cl::opt<bool> FlagRestrictR15("sfi-restrict-r15",
-                              cl::desc("Restrict use of %r15.  This flag can"
-                                       " be turned off for the zero-based"
-                                       " sandbox model."),
-                              cl::init(true));
-
-namespace {
-  class X86NaClRewritePass : public MachineFunctionPass {
-  public:
-    static char ID;
-    X86NaClRewritePass() : MachineFunctionPass(ID) {}
-
-    virtual bool runOnMachineFunction(MachineFunction &Fn);
-
-    virtual const char *getPassName() const {
-      return "NaCl Rewrites";
-    }
-
-  private:
-
-    const TargetMachine *TM;
-    const TargetInstrInfo *TII;
-    const TargetRegisterInfo *TRI;
-    const X86Subtarget *Subtarget;
-    bool Is64Bit;
-
-    bool runOnMachineBasicBlock(MachineBasicBlock &MBB);
-
-    void TraceLog(const char *func,
-                  const MachineBasicBlock &MBB,
-                  const MachineBasicBlock::iterator MBBI) const;
-
-    bool ApplyRewrites(MachineBasicBlock &MBB,
-                      MachineBasicBlock::iterator MBBI);
-    bool ApplyStackSFI(MachineBasicBlock &MBB,
-                       MachineBasicBlock::iterator MBBI);
-
-    bool ApplyMemorySFI(MachineBasicBlock &MBB,
-                        MachineBasicBlock::iterator MBBI);
-
-    bool ApplyFrameSFI(MachineBasicBlock &MBB,
-                       MachineBasicBlock::iterator MBBI);
-
-    bool ApplyControlSFI(MachineBasicBlock &MBB,
-                         MachineBasicBlock::iterator MBBI);
-
-    bool AlignJumpTableTargets(MachineFunction &MF);
-  };
-
-  char X86NaClRewritePass::ID = 0;
-
-}
-
-static void DumpInstructionVerbose(const MachineInstr &MI) {
-  DEBUG({
-      dbgs() << MI;
-      dbgs() << MI.getNumOperands() << " operands:" << "\n";
-      for (unsigned i = 0; i < MI.getNumOperands(); ++i) {
-        const MachineOperand& op = MI.getOperand(i);
-        dbgs() << "  " << i << "(" << (unsigned)op.getType() << "):" << op
-               << "\n";
-      }
-      dbgs() << "\n";
-    });
-}
-
-static bool IsPushPop(MachineInstr &MI) {
-  const unsigned Opcode = MI.getOpcode();
-  switch (Opcode) {
-   default:
-    return false;
-   case X86::PUSH64r:
-   case X86::POP64r:
-    return true;
-  }
-}
-
-static bool IsStore(MachineInstr &MI) {
-  return MI.mayStore();
-}
-
-static bool IsLoad(MachineInstr &MI) {
-  return MI.mayLoad();
-}
-
-static bool IsFrameChange(MachineInstr &MI, const TargetRegisterInfo *TRI) {
-  return MI.modifiesRegister(X86::EBP, TRI);
-}
-
-static bool IsStackChange(MachineInstr &MI, const TargetRegisterInfo *TRI) {
-  return MI.modifiesRegister(X86::ESP, TRI);
-}
-
-static bool HasControlFlow(const MachineInstr &MI) {
- return MI.getDesc().isBranch() ||
-        MI.getDesc().isCall() ||
-        MI.getDesc().isReturn() ||
-        MI.getDesc().isTerminator() ||
-        MI.getDesc().isBarrier();
-}
-
-static bool IsDirectBranch(const MachineInstr &MI) {
-  return  MI.getDesc().isBranch() &&
-         !MI.getDesc().isIndirectBranch();
-}
-
-static bool IsRegAbsolute(unsigned Reg) {
-  const bool RestrictR15 = FlagRestrictR15;
-  assert(FlagUseZeroBasedSandbox || RestrictR15);
-  return (Reg == X86::RSP || Reg == X86::RBP ||
-          (Reg == X86::R15 && RestrictR15));
-}
-
-static bool FindMemoryOperand(const MachineInstr &MI,
-                              SmallVectorImpl<unsigned>* indices) {
-  int NumFound = 0;
-  for (unsigned i = 0; i < MI.getNumOperands(); ) {
-    if (isMem(&MI, i)) {
-      NumFound++;
-      indices->push_back(i);
-      i += X86::AddrNumOperands;
-    } else {
-      i++;
-    }
-  }
-
-  // Intrinsics and other functions can have mayLoad and mayStore to reflect
-  // the side effects of those functions.  This function is used to find
-  // explicit memory references in the instruction, of which there are none.
-  if (NumFound == 0)
-    return false;
-
-  return true;
-}
-
-static unsigned PromoteRegTo64(unsigned RegIn) {
-  if (RegIn == 0)
-    return 0;
-  unsigned RegOut = getX86SubSuperRegister(RegIn, MVT::i64, false);
-  assert(RegOut != 0);
-  return RegOut;
-}
-
-static unsigned DemoteRegTo32(unsigned RegIn) {
-  if (RegIn == 0)
-    return 0;
-  unsigned RegOut = getX86SubSuperRegister(RegIn, MVT::i32, false);
-  assert(RegOut != 0);
-  return RegOut;
-}
-
-
-//
-// True if this MI restores RSP from RBP with a slight adjustment offset.
-//
-static bool MatchesSPAdj(const MachineInstr &MI) {
-  assert (MI.getOpcode() == X86::LEA64r && "Call to MatchesSPAdj w/ non LEA");
-  const MachineOperand &DestReg = MI.getOperand(0);
-  const MachineOperand &BaseReg = MI.getOperand(1);
-  const MachineOperand &Scale = MI.getOperand(2);
-  const MachineOperand &IndexReg = MI.getOperand(3);
-  const MachineOperand &Offset = MI.getOperand(4);
-  return (DestReg.isReg() && DestReg.getReg() == X86::RSP &&
-          BaseReg.isReg() && BaseReg.getReg() == X86::RBP &&
-          Scale.getImm() == 1 &&
-          IndexReg.isReg() && IndexReg.getReg() == 0 &&
-          Offset.isImm());
-}
-
-void
-X86NaClRewritePass::TraceLog(const char *func,
-                             const MachineBasicBlock &MBB,
-                             const MachineBasicBlock::iterator MBBI) const {
-  DEBUG(dbgs() << "@" << func
-        << "(" << MBB.getName() << ", " << (*MBBI) << ")\n");
-}
-
-bool X86NaClRewritePass::ApplyStackSFI(MachineBasicBlock &MBB,
-                                       MachineBasicBlock::iterator MBBI) {
-  TraceLog("ApplyStackSFI", MBB, MBBI);
-  assert(Is64Bit);
-  MachineInstr &MI = *MBBI;
-
-  if (!IsStackChange(MI, TRI))
-    return false;
-
-  if (IsPushPop(MI))
-    return false;
-
-  if (MI.getDesc().isCall())
-    return false;
-
-  unsigned Opc = MI.getOpcode();
-  DebugLoc DL = MI.getDebugLoc();
-  unsigned DestReg = MI.getOperand(0).getReg();
-  assert(DestReg == X86::ESP || DestReg == X86::RSP);
-
-  unsigned NewOpc = 0;
-  switch (Opc) {
-  case X86::ADD64ri8 : NewOpc = X86::NACL_ASPi8; break;
-  case X86::ADD64ri32: NewOpc = X86::NACL_ASPi32; break;
-  case X86::SUB64ri8 : NewOpc = X86::NACL_SSPi8; break;
-  case X86::SUB64ri32: NewOpc = X86::NACL_SSPi32; break;
-  case X86::AND64ri8 : NewOpc = X86::NACL_ANDSPi8; break;
-  case X86::AND64ri32: NewOpc = X86::NACL_ANDSPi32; break;
-  }
-  if (NewOpc) {
-    BuildMI(MBB, MBBI, DL, TII->get(NewOpc))
-      .addImm(MI.getOperand(2).getImm())
-      .addReg(FlagUseZeroBasedSandbox ? 0 : X86::R15);
-    MI.eraseFromParent();
-    return true;
-  }
-
-  // Promote "MOV ESP, EBP" to a 64-bit move
-  if (Opc == X86::MOV32rr && MI.getOperand(1).getReg() == X86::EBP) {
-    MI.getOperand(0).setReg(X86::RSP);
-    MI.getOperand(1).setReg(X86::RBP);
-    MI.setDesc(TII->get(X86::MOV64rr));
-    Opc = X86::MOV64rr;
-  }
-
-  // "MOV RBP, RSP" is already safe
-  if (Opc == X86::MOV64rr && MI.getOperand(1).getReg() == X86::RBP) {
-    return true;
-  }
-
-  assert(Opc != X86::LEA32r && "Invalid opcode in 64-bit mode!");
-  if (Opc == X86::LEA64_32r){
-    unsigned BaseReg = MI.getOperand(1).getReg();
-    if (BaseReg == X86::EBP) {
-      // leal N(%ebp), %esp can be promoted to leaq N(%rbp), %rsp, which
-      // converts to SPAJDi32 below.
-      unsigned DestReg = MI.getOperand(0).getReg();
-      unsigned Scale   = MI.getOperand(2).getImm();
-      unsigned IndexReg = MI.getOperand(3).getReg();
-      assert(DestReg == X86::ESP);
-      assert(Scale == 1);
-      assert(BaseReg == X86::EBP);
-      assert(IndexReg == 0);
-      MI.getOperand(0).setReg(X86::RSP);
-      MI.getOperand(1).setReg(X86::RBP);
-      MI.setDesc(TII->get(X86::LEA64r));
-      Opc = X86::LEA64r;
-    } else {
-      // Create a MachineInstr bundle (i.e. a bundle-locked group) and fix up
-      // the stack pointer by adding R15.  TODO(dschuff): generalize this for
-      // other uses if needed, and try to replace some pseudos if
-      // possible. Eventually replace with auto-sandboxing.
-      auto NextMBBI = MBBI;
-      ++NextMBBI;
-      BuildMI(MBB, NextMBBI, MBBI->getDebugLoc(),
-              TII->get(X86::ADD64rr), X86::RSP)
-          .addReg(X86::RSP).addReg(X86::R15);
-      MIBundleBuilder(MBB, MBBI, NextMBBI);
-      finalizeBundle(MBB, MBBI.getInstrIterator());
-      return true;
-    }
-  }
-
-  if (Opc == X86::LEA64r && MatchesSPAdj(MI)) {
-    const MachineOperand &Offset = MI.getOperand(4);
-    BuildMI(MBB, MBBI, DL, TII->get(X86::NACL_SPADJi32))
-      .addImm(Offset.getImm())
-      .addReg(FlagUseZeroBasedSandbox ? 0 : X86::R15);
-    MI.eraseFromParent();
-    return true;
-  }
-
-  if (Opc == X86::MOV32rr || Opc == X86::MOV64rr) {
-    BuildMI(MBB, MBBI, DL, TII->get(X86::NACL_RESTSPr))
-      .addReg(DemoteRegTo32(MI.getOperand(1).getReg()))
-      .addReg(FlagUseZeroBasedSandbox ? 0 : X86::R15);
-    MI.eraseFromParent();
-    return true;
-  }
-
-  if (Opc == X86::MOV32rm) {
-    BuildMI(MBB, MBBI, DL, TII->get(X86::NACL_RESTSPm))
-      .addOperand(MI.getOperand(1)) // Base
-      .addOperand(MI.getOperand(2)) // Scale
-      .addOperand(MI.getOperand(3)) // Index
-      .addOperand(MI.getOperand(4)) // Offset
-      .addOperand(MI.getOperand(5)) // Segment
-      .addReg(FlagUseZeroBasedSandbox ? 0 : X86::R15);
-    MI.eraseFromParent();
-    return true;
-  }
-
-  DEBUG(DumpInstructionVerbose(MI));
-  llvm_unreachable("Unhandled Stack SFI");
-}
-
-bool X86NaClRewritePass::ApplyFrameSFI(MachineBasicBlock &MBB,
-                                       MachineBasicBlock::iterator MBBI) {
-  TraceLog("ApplyFrameSFI", MBB, MBBI);
-  assert(Is64Bit);
-  MachineInstr &MI = *MBBI;
-
-  if (!IsFrameChange(MI, TRI))
-    return false;
-
-  unsigned Opc = MI.getOpcode();
-  DebugLoc DL = MI.getDebugLoc();
-
-  // Handle moves to RBP
-  if (Opc == X86::MOV64rr) {
-    assert(MI.getOperand(0).getReg() == X86::RBP);
-    unsigned SrcReg = MI.getOperand(1).getReg();
-
-    // MOV RBP, RSP is already safe
-    if (SrcReg == X86::RSP)
-      return false;
-
-    // Rewrite: mov %rbp, %rX
-    // To:      naclrestbp %eX, %rZP
-    BuildMI(MBB, MBBI, DL, TII->get(X86::NACL_RESTBPr))
-      .addReg(DemoteRegTo32(SrcReg))
-      .addReg(FlagUseZeroBasedSandbox ? 0 : X86::R15); // rZP
-    MI.eraseFromParent();
-    return true;
-  }
-
-  // Handle memory moves to RBP
-  if (Opc == X86::MOV64rm) {
-    assert(MI.getOperand(0).getReg() == X86::RBP);
-
-    // Zero-based sandbox model uses address clipping
-    if (FlagUseZeroBasedSandbox)
-      return false;
-
-    // Rewrite: mov %rbp, (...)
-    // To:      naclrestbp (...), %rZP
-    BuildMI(MBB, MBBI, DL, TII->get(X86::NACL_RESTBPm))
-      .addOperand(MI.getOperand(1))  // Base
-      .addOperand(MI.getOperand(2))  // Scale
-      .addOperand(MI.getOperand(3))  // Index
-      .addOperand(MI.getOperand(4))  // Offset
-      .addOperand(MI.getOperand(5))  // Segment
-      .addReg(FlagUseZeroBasedSandbox ? 0 : X86::R15); // rZP
-    MI.eraseFromParent();
-    return true;
-  }
-
-  // Popping onto RBP
-  // Rewrite to:
-  //   naclrestbp (%rsp), %rZP
-  //   naclasp $8, %rZP
-  //
-  // TODO(pdox): Consider rewriting to this instead:
-  //   .bundle_lock
-  //   pop %rbp
-  //   mov %ebp,%ebp
-  //   add %rZP, %rbp
-  //   .bundle_unlock
-  if (Opc == X86::POP64r) {
-    assert(MI.getOperand(0).getReg() == X86::RBP);
-
-    BuildMI(MBB, MBBI, DL, TII->get(X86::NACL_RESTBPm))
-      .addReg(X86::RSP)  // Base
-      .addImm(1)  // Scale
-      .addReg(0)  // Index
-      .addImm(0)  // Offset
-      .addReg(0)  // Segment
-      .addReg(FlagUseZeroBasedSandbox ? 0 : X86::R15); // rZP
-
-    BuildMI(MBB, MBBI, DL, TII->get(X86::NACL_ASPi8))
-      .addImm(8)
-      .addReg(FlagUseZeroBasedSandbox ? 0 : X86::R15);
-
-    MI.eraseFromParent();
-    return true;
-  }
-
-  DEBUG(DumpInstructionVerbose(MI));
-  llvm_unreachable("Unhandled Frame SFI");
-}
-
-bool X86NaClRewritePass::ApplyControlSFI(MachineBasicBlock &MBB,
-                                         MachineBasicBlock::iterator MBBI) {
-  const bool HideSandboxBase = (FlagHideSandboxBase &&
-                                Is64Bit && !FlagUseZeroBasedSandbox);
-  TraceLog("ApplyControlSFI", MBB, MBBI);
-  MachineInstr &MI = *MBBI;
-
-  if (!HasControlFlow(MI))
-    return false;
-
-  // Direct branches are OK
-  if (IsDirectBranch(MI))
-    return false;
-
-  DebugLoc DL = MI.getDebugLoc();
-  unsigned Opc = MI.getOpcode();
-
-  // Rewrite indirect jump/call instructions
-  unsigned NewOpc = 0;
-  switch (Opc) {
-  // 32-bit
-  case X86::JMP32r               : NewOpc = X86::NACL_JMP32r; break;
-  case X86::TAILJMPr             : NewOpc = X86::NACL_JMP32r; break;
-  case X86::CALL32r              : NewOpc = X86::NACL_CALL32r; break;
-  // 64-bit
-  case X86::NACL_CG_JMP64r       : NewOpc = X86::NACL_JMP64r; break;
-  case X86::CALL64r              : NewOpc = X86::NACL_CALL64r; break;
-  case X86::TAILJMPr64           : NewOpc = X86::NACL_JMP64r; break;
-  }
-  if (NewOpc) {
-    unsigned TargetReg = MI.getOperand(0).getReg();
-    if (Is64Bit) {
-      // CALL64r, etc. take a 64-bit register as a target. However, NaCl gas
-      // expects naclcall/nacljmp pseudos to have 32-bit regs as targets
-      // so NACL_CALL64r and NACL_JMP64r stick with that as well.
-      // Demote any 64-bit register to 32-bit to match the expectations.
-      TargetReg = DemoteRegTo32(TargetReg);
-    }
-    MachineInstrBuilder NewMI =
-     BuildMI(MBB, MBBI, DL, TII->get(NewOpc))
-       .addReg(TargetReg);
-    if (Is64Bit) {
-      NewMI.addReg(FlagUseZeroBasedSandbox ? 0 : X86::R15);
-    }
-    MI.eraseFromParent();
-    return true;
-  }
-
-  // EH_RETURN has a single argment which is not actually used directly.
-  // The argument gives the location where to reposition the stack pointer
-  // before returning. EmitPrologue takes care of that repositioning.
-  // So EH_RETURN just ultimately emits a plain "ret".
-  // RETI returns and pops some number of bytes from the stack.
-  if (Opc == X86::RETL || Opc == X86::RETQ ||
-      Opc == X86::EH_RETURN || Opc == X86::EH_RETURN64 ||
-      Opc == X86::RETIL || Opc == X86::RETIQ) {
-    // To maintain compatibility with nacl-as, for now we don't emit naclret.
-    // MI.setDesc(TII->get(Is64Bit ? X86::NACL_RET64 : X86::NACL_RET32));
-    //
-    // For NaCl64 returns, follow the convention of using r11 to hold
-    // the target of an indirect jump to avoid potentially leaking the
-    // sandbox base address.
-    unsigned RegTarget;
-    if (Is64Bit) {
-      RegTarget = (HideSandboxBase ? X86::R11 : X86::RCX);
-      BuildMI(MBB, MBBI, DL, TII->get(X86::POP64r), RegTarget);
-      if (Opc == X86::RETIL || Opc == X86::RETIQ) {
-        BuildMI(MBB, MBBI, DL, TII->get(X86::NACL_ASPi32))
-          .addOperand(MI.getOperand(0))
-          .addReg(FlagUseZeroBasedSandbox ? 0 : X86::R15);
-      }
-
-      BuildMI(MBB, MBBI, DL, TII->get(X86::NACL_JMP64r))
-        .addReg(getX86SubSuperRegister(RegTarget, MVT::i32, false))
-        .addReg(FlagUseZeroBasedSandbox ? 0 : X86::R15);
-    } else {
-      RegTarget = X86::ECX;
-      BuildMI(MBB, MBBI, DL, TII->get(X86::POP32r), RegTarget);
-      if (Opc == X86::RETIL || Opc == X86::RETIQ) {
-        BuildMI(MBB, MBBI, DL, TII->get(X86::ADD32ri), X86::ESP)
-          .addReg(X86::ESP)
-          .addOperand(MI.getOperand(0));
-      }
-      BuildMI(MBB, MBBI, DL, TII->get(X86::NACL_JMP32r))
-        .addReg(RegTarget);
-    }
-    MI.eraseFromParent();
-    return true;
-  }
-
-  // Traps are OK (but are considered to have control flow
-  // being a terminator like RET).
-  if (Opc == X86::TRAP)
-    return false;
-
-  DEBUG(DumpInstructionVerbose(MI));
-  llvm_unreachable("Unhandled Control SFI");
-}
-
-//
-// Sandboxes loads and stores (64-bit only)
-//
-bool X86NaClRewritePass::ApplyMemorySFI(MachineBasicBlock &MBB,
-                                        MachineBasicBlock::iterator MBBI) {
-  TraceLog("ApplyMemorySFI", MBB, MBBI);
-  assert(Is64Bit);
-  MachineInstr &MI = *MBBI;
-
-  if (!IsLoad(MI) && !IsStore(MI))
-    return false;
-
-  if (IsPushPop(MI))
-    return false;
-
-  SmallVector<unsigned, 2> MemOps;
-  if (!FindMemoryOperand(MI, &MemOps))
-    return false;
-  bool Modified = false;
-  for (unsigned MemOp : MemOps) {
-    MachineOperand &BaseReg  = MI.getOperand(MemOp + 0);
-    MachineOperand &Scale = MI.getOperand(MemOp + 1);
-    MachineOperand &IndexReg  = MI.getOperand(MemOp + 2);
-    //MachineOperand &Disp = MI.getOperand(MemOp + 3);
-    MachineOperand &SegmentReg = MI.getOperand(MemOp + 4);
-
-    // RIP-relative addressing is safe.
-    if (BaseReg.getReg() == X86::RIP)
-      continue;
-
-    // Make sure the base and index are 64-bit registers.
-    IndexReg.setReg(PromoteRegTo64(IndexReg.getReg()));
-    BaseReg.setReg(PromoteRegTo64(BaseReg.getReg()));
-    assert(IndexReg.getSubReg() == 0);
-    assert(BaseReg.getSubReg() == 0);
-
-    bool AbsoluteBase = IsRegAbsolute(BaseReg.getReg());
-    bool AbsoluteIndex = IsRegAbsolute(IndexReg.getReg());
-    unsigned AddrReg = 0;
-
-    if (AbsoluteBase && AbsoluteIndex) {
-      llvm_unreachable("Unexpected absolute register pair");
-    } else if (AbsoluteBase) {
-      AddrReg = IndexReg.getReg();
-    } else if (AbsoluteIndex) {
-      assert(!BaseReg.getReg() && "Unexpected base register");
-      assert(Scale.getImm() == 1);
-      AddrReg = 0;
-    } else {
-      if (!BaseReg.getReg()) {
-        // No base, fill in relative.
-        BaseReg.setReg(FlagUseZeroBasedSandbox ? 0 : X86::R15);
-        AddrReg = IndexReg.getReg();
-      } else if (!FlagUseZeroBasedSandbox) {
-        // Switch base and index registers if index register is undefined.
-        // That is do conversions like "mov d(%r,0,0) -> mov d(%r15, %r, 1)".
-        assert (!IndexReg.getReg()
-                && "Unexpected index and base register");
-        IndexReg.setReg(BaseReg.getReg());
-        Scale.setImm(1);
-        BaseReg.setReg(X86::R15);
-        AddrReg = IndexReg.getReg();
-      } else {
-        llvm_unreachable(
-            "Unexpected index and base register");
-      }
-    }
-
-    if (AddrReg) {
-      assert(!SegmentReg.getReg() && "Unexpected segment register");
-      SegmentReg.setReg(X86::PSEUDO_NACL_SEG);
-      Modified = true;
-    }
-  }
-
-  return Modified;
-}
-
-bool X86NaClRewritePass::ApplyRewrites(MachineBasicBlock &MBB,
-                                       MachineBasicBlock::iterator MBBI) {
-  MachineInstr &MI = *MBBI;
-  DebugLoc DL = MI.getDebugLoc();
-  unsigned Opc = MI.getOpcode();
-
-  // These direct jumps need their opcode rewritten
-  // and variable operands removed.
-  unsigned NewOpc = 0;
-  switch (Opc) {
-  // 32-bit direct calls are handled unmodified by the assemblers
-  case X86::CALLpcrel32          : return true;
-  case X86::TAILJMPd             : NewOpc = X86::JMP_4; break;
-  case X86::NACL_CG_TAILJMPd64   : NewOpc = X86::JMP_4; break;
-  case X86::NACL_CG_CALL64pcrel32: NewOpc = X86::NACL_CALL64d; break;
-  }
-  if (NewOpc) {
-    BuildMI(MBB, MBBI, DL, TII->get(NewOpc))
-      .addOperand(MI.getOperand(0));
-    MI.eraseFromParent();
-    return true;
-  }
-
-  // General Dynamic NaCl TLS model
-  // http://code.google.com/p/nativeclient/issues/detail?id=1685
-  if (Opc == X86::NACL_CG_GD_TLS_addr64) {
-
-    // Rewrite to:
-    //   leaq $sym@TLSGD(%rip), %rdi
-    //   call __tls_get_addr@PLT
-    BuildMI(MBB, MBBI, DL, TII->get(X86::LEA64r), X86::RDI)
-        .addReg(X86::RIP) // Base
-        .addImm(1) // Scale
-        .addReg(0) // Index
-        .addGlobalAddress(MI.getOperand(3).getGlobal(), 0,
-                          MI.getOperand(3).getTargetFlags())
-        .addReg(0); // Segment
-    BuildMI(MBB, MBBI, DL, TII->get(X86::NACL_CALL64d))
-        .addExternalSymbol("__tls_get_addr", X86II::MO_PLT);
-    MI.eraseFromParent();
-    return true;
-  }
-
-  // Local Exec NaCl TLS Model
-  if (Opc == X86::NACL_CG_LE_TLS_addr64 ||
-      Opc == X86::NACL_CG_LE_TLS_addr32) {
-    unsigned CallOpc, LeaOpc, Reg;
-    // Rewrite to:
-    //   call __nacl_read_tp@PLT
-    //   lea $sym@flag(,%reg), %reg
-    if (Opc == X86::NACL_CG_LE_TLS_addr64) {
-      CallOpc = X86::NACL_CALL64d;
-      LeaOpc = X86::LEA64r;
-      Reg = X86::RAX;
-    } else {
-      CallOpc = X86::CALLpcrel32;
-      LeaOpc = X86::LEA32r;
-      Reg = X86::EAX;
-    }
-    BuildMI(MBB, MBBI, DL, TII->get(CallOpc))
-        .addExternalSymbol("__nacl_read_tp", X86II::MO_PLT);
-    BuildMI(MBB, MBBI, DL, TII->get(LeaOpc), Reg)
-        .addReg(0) // Base
-        .addImm(1) // Scale
-        .addReg(Reg) // Index
-        .addGlobalAddress(MI.getOperand(3).getGlobal(), 0,
-                          MI.getOperand(3).getTargetFlags())
-        .addReg(0); // Segment
-    MI.eraseFromParent();
-    return true;
-  }
-
-  // Initial Exec NaCl TLS Model
-  if (Opc == X86::NACL_CG_IE_TLS_addr64 ||
-      Opc == X86::NACL_CG_IE_TLS_addr32) {
-    unsigned CallOpc, AddOpc, Base, Reg;
-    // Rewrite to:
-    //   call __nacl_read_tp@PLT
-    //   addq sym@flag(%base), %reg
-    if (Opc == X86::NACL_CG_IE_TLS_addr64) {
-      CallOpc = X86::NACL_CALL64d;
-      AddOpc = X86::ADD64rm;
-      Base = X86::RIP;
-      Reg = X86::RAX;
-    } else {
-      CallOpc = X86::CALLpcrel32;
-      AddOpc = X86::ADD32rm;
-      Base = MI.getOperand(3).getTargetFlags() == X86II::MO_INDNTPOFF ?
-          0 : X86::EBX; // EBX for GOTNTPOFF.
-      Reg = X86::EAX;
-    }
-    BuildMI(MBB, MBBI, DL, TII->get(CallOpc))
-        .addExternalSymbol("__nacl_read_tp", X86II::MO_PLT);
-    BuildMI(MBB, MBBI, DL, TII->get(AddOpc), Reg)
-        .addReg(Reg)
-        .addReg(Base)
-        .addImm(1) // Scale
-        .addReg(0) // Index
-        .addGlobalAddress(MI.getOperand(3).getGlobal(), 0,
-                          MI.getOperand(3).getTargetFlags())
-        .addReg(0); // Segment
-    MI.eraseFromParent();
-    return true;
-  }
-
-  return false;
-}
-
-bool X86NaClRewritePass::AlignJumpTableTargets(MachineFunction &MF) {
-  bool Modified = true;
-
-  MF.setAlignment(5); // log2, 32 = 2^5
-
-  MachineJumpTableInfo *JTI = MF.getJumpTableInfo();
-  if (JTI != NULL) {
-    const std::vector<MachineJumpTableEntry> &JT = JTI->getJumpTables();
-    for (unsigned i = 0; i < JT.size(); ++i) {
-      const std::vector<MachineBasicBlock*> &MBBs = JT[i].MBBs;
-      for (unsigned j = 0; j < MBBs.size(); ++j) {
-        MBBs[j]->setAlignment(5);
-        Modified |= true;
-      }
-    }
-  }
-  return Modified;
-}
-
-bool X86NaClRewritePass::runOnMachineFunction(MachineFunction &MF) {
-  bool Modified = false;
-
-  TM = &MF.getTarget();
-  TII = MF.getSubtarget().getInstrInfo();
-  TRI = MF.getSubtarget().getRegisterInfo();
-  Subtarget = &MF.getSubtarget<X86Subtarget>();
-  Is64Bit = Subtarget->is64Bit();
-
-  assert(Subtarget->isTargetNaCl() && "Unexpected target in NaClRewritePass!");
-
-  DEBUG(dbgs() << "*************** NaCl Rewrite Pass ***************\n");
-  for (MachineFunction::iterator MFI = MF.begin(), E = MF.end();
-       MFI != E;
-       ++MFI) {
-    Modified |= runOnMachineBasicBlock(*MFI);
-  }
-  Modified |= AlignJumpTableTargets(MF);
-  DEBUG(dbgs() << "*************** NaCl Rewrite DONE  ***************\n");
-  return Modified;
-}
-
-bool X86NaClRewritePass::runOnMachineBasicBlock(MachineBasicBlock &MBB) {
-  bool Modified = false;
-  if (MBB.hasAddressTaken()) {
-    //FIXME: use a symbolic constant or get this value from some configuration
-    MBB.setAlignment(5);
-    Modified = true;
-  }
-  for (MachineBasicBlock::iterator MBBI = MBB.begin(), NextMBBI = MBBI;
-       MBBI != MBB.end(); MBBI = NextMBBI) {
-    ++NextMBBI;
-    // When one of these methods makes a change,
-    // it returns true, skipping the others.
-    if (ApplyRewrites(MBB, MBBI) ||
-        (Is64Bit && ApplyStackSFI(MBB, MBBI)) ||
-        (Is64Bit && ApplyMemorySFI(MBB, MBBI)) ||
-        (Is64Bit && ApplyFrameSFI(MBB, MBBI)) ||
-        ApplyControlSFI(MBB, MBBI)) {
-      Modified = true;
-    }
-  }
-  return Modified;
-}
-
-/// createX86NaClRewritePassPass - returns an instance of the pass.
-namespace llvm {
-  FunctionPass* createX86NaClRewritePass() {
-    return new X86NaClRewritePass();
-  }
-}
diff --git a/lib/Target/X86/X86RegisterInfo.cpp b/lib/Target/X86/X86RegisterInfo.cpp
index 8baf05281a0..5c4acc83ef9 100644
--- a/lib/Target/X86/X86RegisterInfo.cpp
+++ b/lib/Target/X86/X86RegisterInfo.cpp
@@ -17,7 +17,6 @@
 #include "X86FrameLowering.h"
 #include "X86InstrBuilder.h"
 #include "X86MachineFunctionInfo.h"
-#include "X86NaClDecls.h" // @LOCALMOD
 #include "X86Subtarget.h"
 #include "X86TargetMachine.h"
 #include "llvm/ADT/BitVector.h"
@@ -45,12 +44,6 @@ using namespace llvm;
 #define GET_REGINFO_TARGET_DESC
 #include "X86GenRegisterInfo.inc"
 
-cl::opt<bool>
-ForceStackAlign("force-align-stack",
-                 cl::desc("Force align the stack to the minimum alignment"
-                           " needed for the function."),
-                 cl::init(false), cl::Hidden);
-
 static cl::opt<bool>
 EnableBasePointer("x86-use-base-pointer", cl::Hidden, cl::init(true),
           cl::desc("Enable use of a base pointer for complex stack frames"));
@@ -168,24 +161,20 @@ X86RegisterInfo::getPointerRegClass(const MachineFunction &MF,
   switch (Kind) {
   default: llvm_unreachable("Unexpected Kind in getPointerRegClass!");
   case 0: // Normal GPRs.
-    // @LOCALMOD -- NaCl is ILP32, but 32-bit pointers become 64-bit
-    // after sandboxing (clobbers a full 64-bit reg).
-    if (Subtarget.isTarget64BitLP64() || Subtarget.isTargetNaCl64())
+    if (Subtarget.isTarget64BitLP64())
       return &X86::GR64RegClass;
     return &X86::GR32RegClass;
   case 1: // Normal GPRs except the stack pointer (for encoding reasons).
-    // @LOCALMOD -- NaCl is ILP32, but 32-bit pointers become 64-bit
-    // after sandboxing (clobbers a full 64-bit reg).
-    if (Subtarget.isTarget64BitLP64() || Subtarget.isTargetNaCl64())
+    if (Subtarget.isTarget64BitLP64())
       return &X86::GR64_NOSPRegClass;
     return &X86::GR32_NOSPRegClass;
   case 2: // Available for tailcall (not callee-saved GPRs).
-    if (IsWin64)
+    const Function *F = MF.getFunction();
+    if (IsWin64 || (F && F->getCallingConv() == CallingConv::X86_64_Win64))
       return &X86::GR64_TCW64RegClass;
     else if (Is64Bit)
       return &X86::GR64_TCRegClass;
 
-    const Function *F = MF.getFunction();
     bool hasHipeCC = (F ? F->getCallingConv() == CallingConv::HiPE : false);
     if (hasHipeCC)
       return &X86::GR32RegClass;
@@ -207,7 +196,7 @@ X86RegisterInfo::getCrossCopyRegClass(const TargetRegisterClass *RC) const {
 unsigned
 X86RegisterInfo::getRegPressureLimit(const TargetRegisterClass *RC,
                                      MachineFunction &MF) const {
-  const TargetFrameLowering *TFI = MF.getSubtarget().getFrameLowering();
+  const X86FrameLowering *TFI = getFrameLowering(MF);
 
   unsigned FPDiff = TFI->hasFP(MF) ? 1 : 0;
   switch (RC->getID()) {
@@ -273,17 +262,9 @@ X86RegisterInfo::getCalleeSavedRegs(const MachineFunction *MF) const {
     break;
   }
 
-  bool IsNaCl = Subtarget.isTargetNaCl(); // @LOCALMOD
   if (Is64Bit) {
     if (IsWin64)
       return CSR_Win64_SaveList;
-    // @LOCALMOD-BEGIN
-    if (IsNaCl) {
-      if (CallsEHReturn)
-        return CSR_NaCl64EHRet_SaveList;
-      return CSR_NaCl64_SaveList;
-    }
-    // @LOCALMOD-END
     if (CallsEHReturn)
       return CSR_64EHRet_SaveList;
     return CSR_64_SaveList;
@@ -356,7 +337,7 @@ X86RegisterInfo::getNoPreservedMask() const {
 
 BitVector X86RegisterInfo::getReservedRegs(const MachineFunction &MF) const {
   BitVector Reserved(getNumRegs());
-  const TargetFrameLowering *TFI = MF.getSubtarget().getFrameLowering();
+  const X86FrameLowering *TFI = getFrameLowering(MF);
 
   // Set the stack-pointer register and its aliases as reserved.
   for (MCSubRegIterator I(X86::RSP, this, /*IncludeSelf=*/true); I.isValid();
@@ -429,41 +410,33 @@ BitVector X86RegisterInfo::getReservedRegs(const MachineFunction &MF) const {
     }
   }
 
-  // @LOCALMOD-START
-  const X86Subtarget& Subtarget = MF.getSubtarget<X86Subtarget>();
-  const bool RestrictR15 = FlagRestrictR15;
-  assert(FlagUseZeroBasedSandbox || RestrictR15);
-  if (Subtarget.isTargetNaCl64()) {
-    if (RestrictR15) {
-      Reserved.set(X86::R15);
-      Reserved.set(X86::R15D);
-      Reserved.set(X86::R15W);
-      Reserved.set(X86::R15B);
-    }
-    Reserved.set(X86::RBP);
-    Reserved.set(X86::EBP);
-    Reserved.set(X86::BP);
-    Reserved.set(X86::BPL);
-    const bool RestrictR11 = FlagHideSandboxBase && !FlagUseZeroBasedSandbox;
-    if (RestrictR11) {
-      // Restrict r11 so that it can be used for indirect jump
-      // sequences that don't leak the sandbox base address onto the
-      // stack.
-      Reserved.set(X86::R11);
-      Reserved.set(X86::R11D);
-      Reserved.set(X86::R11W);
-      Reserved.set(X86::R11B);
-    }
-  }
-  // @LOCALMOD-END
-
   return Reserved;
 }
 
+void X86RegisterInfo::adjustStackMapLiveOutMask(uint32_t *Mask) const {
+  // Check if the EFLAGS register is marked as live-out. This shouldn't happen,
+  // because the calling convention defines the EFLAGS register as NOT
+  // preserved.
+  //
+  // Unfortunatelly the EFLAGS show up as live-out after branch folding. Adding
+  // an assert to track this and clear the register afterwards to avoid
+  // unnecessary crashes during release builds.
+  assert(!(Mask[X86::EFLAGS / 32] & (1U << (X86::EFLAGS % 32))) &&
+         "EFLAGS are not live-out from a patchpoint.");
+
+  // Also clean other registers that don't need preserving (IP).
+  for (auto Reg : {X86::EFLAGS, X86::RIP, X86::EIP, X86::IP})
+    Mask[Reg / 32] &= ~(1U << (Reg % 32));
+}
+
 //===----------------------------------------------------------------------===//
 // Stack Frame Processing methods
 //===----------------------------------------------------------------------===//
 
+static bool CantUseSP(const MachineFrameInfo *MFI) {
+  return MFI->hasVarSizedObjects() || MFI->hasOpaqueSPAdjustment();
+}
+
 bool X86RegisterInfo::hasBasePointer(const MachineFunction &MF) const {
    const MachineFrameInfo *MFI = MF.getFrameInfo();
 
@@ -476,13 +449,11 @@ bool X86RegisterInfo::hasBasePointer(const MachineFunction &MF) const {
    // reference locals while also adjusting the stack pointer.  When we can't
    // use both the SP and the FP, we need a separate base pointer register.
    bool CantUseFP = needsStackRealignment(MF);
-   bool CantUseSP =
-       MFI->hasVarSizedObjects() || MFI->hasInlineAsmWithSPAdjust();
-   return CantUseFP && CantUseSP;
+   return CantUseFP && CantUseSP(MFI);
 }
 
 bool X86RegisterInfo::canRealignStack(const MachineFunction &MF) const {
-  if (MF.getFunction()->hasFnAttribute("no-realign-stack"))
+  if (!TargetRegisterInfo::canRealignStack(MF))
     return false;
 
   const MachineFrameInfo *MFI = MF.getFrameInfo();
@@ -495,26 +466,11 @@ bool X86RegisterInfo::canRealignStack(const MachineFunction &MF) const {
 
   // If a base pointer is necessary.  Check that it isn't too late to reserve
   // it.
-  if (MFI->hasVarSizedObjects())
+  if (CantUseSP(MFI))
     return MRI->canReserveReg(BasePtr);
   return true;
 }
 
-bool X86RegisterInfo::needsStackRealignment(const MachineFunction &MF) const {
-  const MachineFrameInfo *MFI = MF.getFrameInfo();
-  const Function *F = MF.getFunction();
-  unsigned StackAlign =
-    MF.getSubtarget().getFrameLowering()->getStackAlignment();
-  bool requiresRealignment = ((MFI->getMaxAlignment() > StackAlign) ||
-                              F->hasFnAttribute(Attribute::StackAlignment));
-
-  // If we've requested that we force align the stack do so now.
-  if (ForceStackAlign)
-    return canRealignStack(MF);
-
-  return requiresRealignment && canRealignStack(MF);
-}
-
 bool X86RegisterInfo::hasReservedSpillSlot(const MachineFunction &MF,
                                            unsigned Reg, int &FrameIdx) const {
   // Since X86 defines assignCalleeSavedSpillSlots which always return true
@@ -528,12 +484,13 @@ X86RegisterInfo::eliminateFrameIndex(MachineBasicBlock::iterator II,
                                      RegScavenger *RS) const {
   MachineInstr &MI = *II;
   MachineFunction &MF = *MI.getParent()->getParent();
-  const TargetFrameLowering *TFI = MF.getSubtarget().getFrameLowering();
+  const X86FrameLowering *TFI = getFrameLowering(MF);
   int FrameIndex = MI.getOperand(FIOperandNum).getIndex();
   unsigned BasePtr;
 
   unsigned Opc = MI.getOpcode();
-  bool AfterFPPop = Opc == X86::TAILJMPm64 || Opc == X86::TAILJMPm;
+  bool AfterFPPop = Opc == X86::TAILJMPm64 || Opc == X86::TAILJMPm ||
+                    Opc == X86::TCRETURNmi || Opc == X86::TCRETURNmi64;
   if (hasBasePointer(MF))
     BasePtr = (FrameIndex < 0 ? FramePtr : getBaseRegister());
   else if (needsStackRealignment(MF))
@@ -543,18 +500,17 @@ X86RegisterInfo::eliminateFrameIndex(MachineBasicBlock::iterator II,
   else
     BasePtr = (TFI->hasFP(MF) ? FramePtr : StackPtr);
 
-  // FRAME_ALLOC uses a single offset, with no register. It only works in the
+  // LOCAL_ESCAPE uses a single offset, with no register. It only works in the
   // simple FP case, and doesn't work with stack realignment. On 32-bit, the
   // offset is from the traditional base pointer location.  On 64-bit, the
   // offset is from the SP at the end of the prologue, not the FP location. This
   // matches the behavior of llvm.frameaddress.
-  if (Opc == TargetOpcode::FRAME_ALLOC) {
+  if (Opc == TargetOpcode::LOCAL_ESCAPE) {
     MachineOperand &FI = MI.getOperand(FIOperandNum);
     bool IsWinEH = MF.getTarget().getMCAsmInfo()->usesWindowsCFI();
     int Offset;
     if (IsWinEH)
-      Offset = static_cast<const X86FrameLowering *>(TFI)
-                     ->getFrameIndexOffsetFromSP(MF, FrameIndex);
+      Offset = TFI->getFrameIndexOffsetFromSP(MF, FrameIndex);
     else
       Offset = TFI->getFrameIndexOffset(MF, FrameIndex);
     FI.ChangeToImmediate(Offset);
@@ -608,7 +564,7 @@ X86RegisterInfo::eliminateFrameIndex(MachineBasicBlock::iterator II,
 }
 
 unsigned X86RegisterInfo::getFrameRegister(const MachineFunction &MF) const {
-  const TargetFrameLowering *TFI = MF.getSubtarget().getFrameLowering();
+  const X86FrameLowering *TFI = getFrameLowering(MF);
   return TFI->hasFP(MF) ? FramePtr : StackPtr;
 }
 
@@ -622,10 +578,10 @@ X86RegisterInfo::getPtrSizedFrameRegister(const MachineFunction &MF) const {
 }
 
 namespace llvm {
-unsigned getX86SubSuperRegister(unsigned Reg, MVT::SimpleValueType VT,
-                                bool High) {
+unsigned getX86SubSuperRegisterOrZero(unsigned Reg, MVT::SimpleValueType VT,
+                                      bool High) {
   switch (VT) {
-  default: llvm_unreachable("Unexpected VT");
+  default: return 0;
   case MVT::i8:
     if (High) {
       switch (Reg) {
@@ -649,7 +605,7 @@ unsigned getX86SubSuperRegister(unsigned Reg, MVT::SimpleValueType VT,
       }
     } else {
       switch (Reg) {
-      default: llvm_unreachable("Unexpected register");
+      default: return 0;
       case X86::AH: case X86::AL: case X86::AX: case X86::EAX: case X86::RAX:
         return X86::AL;
       case X86::DH: case X86::DL: case X86::DX: case X86::EDX: case X86::RDX:
@@ -686,7 +642,7 @@ unsigned getX86SubSuperRegister(unsigned Reg, MVT::SimpleValueType VT,
     }
   case MVT::i16:
     switch (Reg) {
-    default: llvm_unreachable("Unexpected register");
+    default: return 0;
     case X86::AH: case X86::AL: case X86::AX: case X86::EAX: case X86::RAX:
       return X86::AX;
     case X86::DH: case X86::DL: case X86::DX: case X86::EDX: case X86::RDX:
@@ -722,7 +678,7 @@ unsigned getX86SubSuperRegister(unsigned Reg, MVT::SimpleValueType VT,
     }
   case MVT::i32:
     switch (Reg) {
-    default: llvm_unreachable("Unexpected register");
+    default: return 0;
     case X86::AH: case X86::AL: case X86::AX: case X86::EAX: case X86::RAX:
       return X86::EAX;
     case X86::DH: case X86::DL: case X86::DX: case X86::EDX: case X86::RDX:
@@ -755,13 +711,10 @@ unsigned getX86SubSuperRegister(unsigned Reg, MVT::SimpleValueType VT,
       return X86::R14D;
     case X86::R15B: case X86::R15W: case X86::R15D: case X86::R15:
       return X86::R15D;
-    // @LOCALMOD. TODO: possibly revert this after LEA .td fixes
-    case X86::EIP: case X86::RIP:
-      return X86::EIP;
     }
   case MVT::i64:
     switch (Reg) {
-    default: llvm_unreachable("Unexpected register");
+    default: return 0;
     case X86::AH: case X86::AL: case X86::AX: case X86::EAX: case X86::RAX:
       return X86::RAX;
     case X86::DH: case X86::DL: case X86::DX: case X86::EDX: case X86::RDX:
@@ -794,13 +747,18 @@ unsigned getX86SubSuperRegister(unsigned Reg, MVT::SimpleValueType VT,
       return X86::R14;
     case X86::R15B: case X86::R15W: case X86::R15D: case X86::R15:
       return X86::R15;
-    // @LOCALMOD. TODO: possibly revert this after LEA .td fixes
-    case X86::EIP: case X86::RIP:
-      return X86::RIP;
     }
   }
 }
 
+unsigned getX86SubSuperRegister(unsigned Reg, MVT::SimpleValueType VT,
+                                bool High) {
+  unsigned Res = getX86SubSuperRegisterOrZero(Reg, VT, High);
+  if (Res == 0)
+    llvm_unreachable("Unexpected register or VT");
+  return Res;
+}
+
 unsigned get512BitSuperRegister(unsigned Reg) {
   if (Reg >= X86::XMM0 && Reg <= X86::XMM31)
     return X86::ZMM0 + (Reg - X86::XMM0);
diff --git a/lib/Target/X86/X86RegisterInfo.h b/lib/Target/X86/X86RegisterInfo.h
index 74edab9fb8a..f18d4d203bc 100644
--- a/lib/Target/X86/X86RegisterInfo.h
+++ b/lib/Target/X86/X86RegisterInfo.h
@@ -20,6 +20,8 @@
 #include "X86GenRegisterInfo.inc"
 
 namespace llvm {
+  class Triple;
+
 class X86RegisterInfo final : public X86GenRegisterInfo {
 private:
   /// Is64Bit - Is the target 64-bits.
@@ -102,11 +104,11 @@ class X86RegisterInfo final : public X86GenRegisterInfo {
   /// register scavenger to determine what registers are free.
   BitVector getReservedRegs(const MachineFunction &MF) const override;
 
-  bool hasBasePointer(const MachineFunction &MF) const;
+  void adjustStackMapLiveOutMask(uint32_t *Mask) const override;
 
-  bool canRealignStack(const MachineFunction &MF) const;
+  bool hasBasePointer(const MachineFunction &MF) const;
 
-  bool needsStackRealignment(const MachineFunction &MF) const override;
+  bool canRealignStack(const MachineFunction &MF) const override;
 
   bool hasReservedSpillSlot(const MachineFunction &MF, unsigned Reg,
                             int &FrameIdx) const override;
@@ -124,11 +126,16 @@ class X86RegisterInfo final : public X86GenRegisterInfo {
   unsigned getSlotSize() const { return SlotSize; }
 };
 
-// getX86SubSuperRegister - X86 utility function. It returns the sub or super
-// register of a specific X86 register.
-// e.g. getX86SubSuperRegister(X86::EAX, MVT::i16) return X86:AX
+/// Returns the sub or super register of a specific X86 register.
+/// e.g. getX86SubSuperRegister(X86::EAX, MVT::i16) returns X86::AX.
+/// Aborts on error.
 unsigned getX86SubSuperRegister(unsigned, MVT::SimpleValueType, bool High=false);
 
+/// Returns the sub or super register of a specific X86 register.
+/// Like getX86SubSuperRegister() but returns 0 on error.
+unsigned getX86SubSuperRegisterOrZero(unsigned, MVT::SimpleValueType,
+                                      bool High = false);
+
 //get512BitRegister - X86 utility - returns 512-bit super register
 unsigned get512BitSuperRegister(unsigned Reg);
 
diff --git a/lib/Target/X86/X86RegisterInfo.td b/lib/Target/X86/X86RegisterInfo.td
index 757ab8d7317..cdb151c26a0 100644
--- a/lib/Target/X86/X86RegisterInfo.td
+++ b/lib/Target/X86/X86RegisterInfo.td
@@ -262,8 +262,6 @@ def ES : X86Reg<"es", 0>;
 def FS : X86Reg<"fs", 4>;
 def GS : X86Reg<"gs", 5>;
 
-def PSEUDO_NACL_SEG : X86Reg<"nacl", 6>; // @LOCALMOD
-
 // Debug registers
 def DR0  : X86Reg<"dr0",   0>;
 def DR1  : X86Reg<"dr1",   1>;
@@ -304,6 +302,11 @@ def CR15 : X86Reg<"cr15", 15>;
 def EIZ : X86Reg<"eiz", 4>;
 def RIZ : X86Reg<"riz", 4>;
 
+// Bound registers, used in MPX instructions
+def BND0 : X86Reg<"bnd0",   0>;
+def BND1 : X86Reg<"bnd1",   1>;
+def BND2 : X86Reg<"bnd2",   2>;
+def BND3 : X86Reg<"bnd3",   3>;
 
 //===----------------------------------------------------------------------===//
 // Register Class Definitions... now that we have all of the pieces, define the
@@ -349,9 +352,7 @@ def GR64 : RegisterClass<"X86", [i64], 64,
 // Segment registers for use by MOV instructions (and others) that have a
 //   segment register as one operand.  Always contain a 16-bit segment
 //   descriptor.
-def SEGMENT_REG : RegisterClass<"X86", [i16], 16, (add CS, DS, SS, ES, FS, GS,
-                                                   // @LOCALMOD
-                                                   PSEUDO_NACL_SEG)>;
+def SEGMENT_REG : RegisterClass<"X86", [i16], 16, (add CS, DS, SS, ES, FS, GS)>;
 
 // Debug registers.
 def DEBUG_REG : RegisterClass<"X86", [i32], 32, (sequence "DR%u", 0, 7)>;
@@ -488,3 +489,6 @@ def VK8WM   : RegisterClass<"X86", [v8i1],  8,  (sub VK8, K0)> {let Size = 8;}
 def VK16WM  : RegisterClass<"X86", [v16i1], 16, (add VK8WM)>   {let Size = 16;}
 def VK32WM  : RegisterClass<"X86", [v32i1], 32, (add VK16WM)> {let Size = 32;}
 def VK64WM  : RegisterClass<"X86", [v64i1], 64, (add VK32WM)> {let Size = 64;}
+
+// Bound registers
+def BNDR : RegisterClass<"X86", [v2i64], 128, (sequence "BND%u", 0, 3)>;
\ No newline at end of file
diff --git a/lib/Target/X86/X86SelectionDAGInfo.cpp b/lib/Target/X86/X86SelectionDAGInfo.cpp
index d2b8dcacabe..ce79fcf9ad8 100644
--- a/lib/Target/X86/X86SelectionDAGInfo.cpp
+++ b/lib/Target/X86/X86SelectionDAGInfo.cpp
@@ -24,11 +24,6 @@ using namespace llvm;
 
 #define DEBUG_TYPE "x86-selectiondag-info"
 
-X86SelectionDAGInfo::X86SelectionDAGInfo(const DataLayout &DL)
-    : TargetSelectionDAGInfo(&DL) {}
-
-X86SelectionDAGInfo::~X86SelectionDAGInfo() {}
-
 bool X86SelectionDAGInfo::isBaseRegConflictPossible(
     SelectionDAG &DAG, ArrayRef<unsigned> ClobberSet) const {
   // We cannot use TRI->hasBasePointer() until *after* we select all basic
@@ -37,7 +32,7 @@ bool X86SelectionDAGInfo::isBaseRegConflictPossible(
   // dynamic stack adjustments (hopefully rare) and the base pointer would
   // conflict if we had to use it.
   MachineFrameInfo *MFI = DAG.getMachineFunction().getFrameInfo();
-  if (!MFI->hasVarSizedObjects() && !MFI->hasInlineAsmWithSPAdjust())
+  if (!MFI->hasVarSizedObjects() && !MFI->hasOpaqueSPAdjustment())
     return false;
 
   const X86RegisterInfo *TRI = static_cast<const X86RegisterInfo *>(
@@ -60,13 +55,6 @@ X86SelectionDAGInfo::EmitTargetCodeForMemset(SelectionDAG &DAG, SDLoc dl,
   const X86Subtarget &Subtarget =
       DAG.getMachineFunction().getSubtarget<X86Subtarget>();
 
-  // @LOCALMOD-BEGIN
-  if (Subtarget.isTargetNaCl()) {
-    // TODO: Can we allow this optimization for Native Client?
-    // At the very least, pointer size needs to be fixed below.
-    return SDValue();
-  }
-  // @LOCALMOD-END
 #ifndef NDEBUG
   // If the base register might conflict with our physical registers, bail out.
   const unsigned ClobberSet[] = {X86::RCX, X86::RAX, X86::RDI,
@@ -88,8 +76,9 @@ X86SelectionDAGInfo::EmitTargetCodeForMemset(SelectionDAG &DAG, SDLoc dl,
 
     if (const char *bzeroEntry =  V &&
         V->isNullValue() ? Subtarget.getBZeroEntry() : nullptr) {
-      EVT IntPtr = DAG.getTargetLoweringInfo().getPointerTy();
-      Type *IntPtrTy = getDataLayout()->getIntPtrType(*DAG.getContext());
+      EVT IntPtr =
+          DAG.getTargetLoweringInfo().getPointerTy(DAG.getDataLayout());
+      Type *IntPtrTy = DAG.getDataLayout().getIntPtrType(*DAG.getContext());
       TargetLowering::ArgListTy Args;
       TargetLowering::ArgListEntry Entry;
       Entry.Node = Dst;
@@ -145,22 +134,22 @@ X86SelectionDAGInfo::EmitTargetCodeForMemset(SelectionDAG &DAG, SDLoc dl,
     default:  // Byte aligned
       AVT = MVT::i8;
       ValReg = X86::AL;
-      Count = DAG.getIntPtrConstant(SizeVal);
+      Count = DAG.getIntPtrConstant(SizeVal, dl);
       break;
     }
 
     if (AVT.bitsGT(MVT::i8)) {
       unsigned UBytes = AVT.getSizeInBits() / 8;
-      Count = DAG.getIntPtrConstant(SizeVal / UBytes);
+      Count = DAG.getIntPtrConstant(SizeVal / UBytes, dl);
       BytesLeft = SizeVal % UBytes;
     }
 
-    Chain  = DAG.getCopyToReg(Chain, dl, ValReg, DAG.getConstant(Val, AVT),
+    Chain  = DAG.getCopyToReg(Chain, dl, ValReg, DAG.getConstant(Val, dl, AVT),
                               InFlag);
     InFlag = Chain.getValue(1);
   } else {
     AVT = MVT::i8;
-    Count  = DAG.getIntPtrConstant(SizeVal);
+    Count  = DAG.getIntPtrConstant(SizeVal, dl);
     Chain  = DAG.getCopyToReg(Chain, dl, X86::AL, Src, InFlag);
     InFlag = Chain.getValue(1);
   }
@@ -181,7 +170,8 @@ X86SelectionDAGInfo::EmitTargetCodeForMemset(SelectionDAG &DAG, SDLoc dl,
     Count  = Size;
     EVT CVT = Count.getValueType();
     SDValue Left = DAG.getNode(ISD::AND, dl, CVT, Count,
-                               DAG.getConstant((AVT == MVT::i64) ? 7 : 3, CVT));
+                               DAG.getConstant((AVT == MVT::i64) ? 7 : 3, dl,
+                                               CVT));
     Chain  = DAG.getCopyToReg(Chain, dl, (CVT == MVT::i64) ? X86::RCX :
                                                              X86::ECX,
                               Left, InFlag);
@@ -197,9 +187,9 @@ X86SelectionDAGInfo::EmitTargetCodeForMemset(SelectionDAG &DAG, SDLoc dl,
 
     Chain = DAG.getMemset(Chain, dl,
                           DAG.getNode(ISD::ADD, dl, AddrVT, Dst,
-                                      DAG.getConstant(Offset, AddrVT)),
+                                      DAG.getConstant(Offset, dl, AddrVT)),
                           Src,
-                          DAG.getConstant(BytesLeft, SizeVT),
+                          DAG.getConstant(BytesLeft, dl, SizeVT),
                           Align, isVolatile, false,
                           DstPtrInfo.getWithOffset(Offset));
   }
@@ -223,13 +213,6 @@ SDValue X86SelectionDAGInfo::EmitTargetCodeForMemcpy(
   if (!AlwaysInline && SizeVal > Subtarget.getMaxInlineSizeThreshold())
     return SDValue();
 
-  // @LOCALMOD-BEGIN
-  if (Subtarget.isTargetNaCl()) {
-    // TODO(pdox): Allow use of the NaCl pseudo-instruction for REP MOV
-    return SDValue();
-  }
-  // @LOCALMOD-END
-
   /// If not DWORD aligned, it is more efficient to call the library.  However
   /// if calling the library is not allowed (AlwaysInline), then soldier on as
   /// the code generated here is better than the long load-store sequence we
@@ -262,7 +245,7 @@ SDValue X86SelectionDAGInfo::EmitTargetCodeForMemcpy(
 
   unsigned UBytes = AVT.getSizeInBits() / 8;
   unsigned CountVal = SizeVal / UBytes;
-  SDValue Count = DAG.getIntPtrConstant(CountVal);
+  SDValue Count = DAG.getIntPtrConstant(CountVal, dl);
   unsigned BytesLeft = SizeVal % UBytes;
 
   SDValue InFlag;
@@ -293,10 +276,12 @@ SDValue X86SelectionDAGInfo::EmitTargetCodeForMemcpy(
     EVT SizeVT = Size.getValueType();
     Results.push_back(DAG.getMemcpy(Chain, dl,
                                     DAG.getNode(ISD::ADD, dl, DstVT, Dst,
-                                                DAG.getConstant(Offset, DstVT)),
+                                                DAG.getConstant(Offset, dl,
+                                                                DstVT)),
                                     DAG.getNode(ISD::ADD, dl, SrcVT, Src,
-                                                DAG.getConstant(Offset, SrcVT)),
-                                    DAG.getConstant(BytesLeft, SizeVT),
+                                                DAG.getConstant(Offset, dl,
+                                                                SrcVT)),
+                                    DAG.getConstant(BytesLeft, dl, SizeVT),
                                     Align, isVolatile, AlwaysInline, false,
                                     DstPtrInfo.getWithOffset(Offset),
                                     SrcPtrInfo.getWithOffset(Offset)));
diff --git a/lib/Target/X86/X86SelectionDAGInfo.h b/lib/Target/X86/X86SelectionDAGInfo.h
index eb7e0ed9de6..961bd8c8d5e 100644
--- a/lib/Target/X86/X86SelectionDAGInfo.h
+++ b/lib/Target/X86/X86SelectionDAGInfo.h
@@ -29,8 +29,7 @@ class X86SelectionDAGInfo : public TargetSelectionDAGInfo {
                                  ArrayRef<unsigned> ClobberSet) const;
 
 public:
-  explicit X86SelectionDAGInfo(const DataLayout &DL);
-  ~X86SelectionDAGInfo();
+  explicit X86SelectionDAGInfo() = default;
 
   SDValue EmitTargetCodeForMemset(SelectionDAG &DAG, SDLoc dl,
                                   SDValue Chain,
diff --git a/lib/Target/X86/X86Subtarget.cpp b/lib/Target/X86/X86Subtarget.cpp
index 0ec6b51cf63..dff3624b7ef 100644
--- a/lib/Target/X86/X86Subtarget.cpp
+++ b/lib/Target/X86/X86Subtarget.cpp
@@ -68,7 +68,7 @@ ClassifyGlobalReference(const GlobalValue *GV, const TargetMachine &TM) const {
   if (GV->hasDLLImportStorageClass())
     return X86II::MO_DLLIMPORT;
 
-  bool isDecl = GV->isDeclarationForLinker();
+  bool isDef = GV->isStrongDefinitionForLinker();
 
   // X86-64 in PIC mode.
   if (isPICStyleRIPRel()) {
@@ -80,8 +80,7 @@ ClassifyGlobalReference(const GlobalValue *GV, const TargetMachine &TM) const {
       // If symbol visibility is hidden, the extra load is not needed if
       // target is x86-64 or the symbol is definitely defined in the current
       // translation unit.
-      if (GV->hasDefaultVisibility() &&
-          (isDecl || GV->isWeakForLinker()))
+      if (GV->hasDefaultVisibility() && !isDef)
         return X86II::MO_GOTPCREL;
     } else if (!isTargetWin64()) {
       assert(isTargetELF() && "Unknown rip-relative target");
@@ -107,7 +106,7 @@ ClassifyGlobalReference(const GlobalValue *GV, const TargetMachine &TM) const {
 
     // If this is a strong reference to a definition, it is definitely not
     // through a stub.
-    if (!isDecl && !GV->isWeakForLinker())
+    if (isDef)
       return X86II::MO_PIC_BASE_OFFSET;
 
     // Unless we have a symbol with hidden visibility, we have to go through a
@@ -117,7 +116,7 @@ ClassifyGlobalReference(const GlobalValue *GV, const TargetMachine &TM) const {
 
     // If symbol visibility is hidden, we have a stub for common symbol
     // references and external declarations.
-    if (isDecl || GV->hasCommonLinkage()) {
+    if (GV->isDeclarationForLinker() || GV->hasCommonLinkage()) {
       // Hidden $non_lazy_ptr reference.
       return X86II::MO_DARWIN_HIDDEN_NONLAZY_PIC_BASE;
     }
@@ -131,7 +130,7 @@ ClassifyGlobalReference(const GlobalValue *GV, const TargetMachine &TM) const {
 
     // If this is a strong reference to a definition, it is definitely not
     // through a stub.
-    if (!isDecl && !GV->isWeakForLinker())
+    if (isDef)
       return X86II::MO_NO_FLAG;
 
     // Unless we have a symbol with hidden visibility, we have to go through a
@@ -175,15 +174,7 @@ bool X86Subtarget::IsLegalToCallImmediateAddr(const TargetMachine &TM) const {
   // the following check for Win32 should be removed.
   if (In64BitMode || isTargetWin32())
     return false;
-  // @LOCALMOD-BEGIN
-  // BUG= http://code.google.com/p/nativeclient/issues/detail?id=2367
-  // For NaCl dynamic linking we do not want to generate a text relocation to
-  // an absolute address in PIC mode.  Such a situation arises from
-  // test/CodeGen/X86/call-imm.ll with the default implementation.
-  // For other platforms we retain the default behavior.
-  return (isTargetELF() && !isTargetNaCl()) ||
-         TM.getRelocationModel() == Reloc::Static;
-  // @LOCALMOD-END
+  return isTargetELF() || TM.getRelocationModel() == Reloc::Static;
 }
 
 void X86Subtarget::initSubtargetFeatures(StringRef CPU, StringRef FS) {
@@ -201,12 +192,9 @@ void X86Subtarget::initSubtargetFeatures(StringRef CPU, StringRef FS) {
       FullFS = "+64bit,+sse2";
   }
 
-  // If feature string is not empty, parse features string.
+  // Parse features string and set the CPU.
   ParseSubtargetFeatures(CPUName, FullFS);
 
-  // Make sure the right MCSchedModel is used.
-  InitCPUSchedModel(CPUName);
-
   InstrItins = getInstrItineraryForCPU(CPUName);
 
   // It's important to keep the MCSubtargetInfo feature bits in sync with
@@ -227,11 +215,10 @@ void X86Subtarget::initSubtargetFeatures(StringRef CPU, StringRef FS) {
          "64-bit code requested on a subtarget that doesn't support it!");
 
   // Stack alignment is 16 bytes on Darwin, Linux and Solaris (both
-  // 32 and 64 bit), NaCl and for all 64-bit targets.
+  // 32 and 64 bit) and for all 64-bit targets.
   if (StackAlignOverride)
     stackAlignment = StackAlignOverride;
   else if (isTargetDarwin() || isTargetLinux() || isTargetSolaris() ||
-           isTargetNaCl() || // @LOCALMOD
            In64BitMode)
     stackAlignment = 16;
 }
@@ -268,6 +255,7 @@ void X86Subtarget::initializeEnvironment() {
   HasSHA = false;
   HasPRFCHW = false;
   HasRDSEED = false;
+  HasMPX = false;
   IsBTMemSlow = false;
   IsSHLDSlow = false;
   IsUAMemFast = false;
@@ -282,11 +270,10 @@ void X86Subtarget::initializeEnvironment() {
   LEAUsesAG = false;
   SlowLEA = false;
   SlowIncDec = false;
-  UseSqrtEst = false;
-  UseReciprocalEst = false;
   stackAlignment = 4;
   // FIXME: this is a known good value for Yonah. How about others?
   MaxInlineSizeThreshold = 128;
+  UseSoftFloat = false;
 }
 
 X86Subtarget &X86Subtarget::initializeSubtargetDependencies(StringRef CPU,
@@ -296,7 +283,7 @@ X86Subtarget &X86Subtarget::initializeSubtargetDependencies(StringRef CPU,
   return *this;
 }
 
-X86Subtarget::X86Subtarget(const std::string &TT, const std::string &CPU,
+X86Subtarget::X86Subtarget(const Triple &TT, const std::string &CPU,
                            const std::string &FS, const X86TargetMachine &TM,
                            unsigned StackAlignOverride)
     : X86GenSubtargetInfo(TT, CPU, FS), X86ProcFamily(Others),
@@ -307,10 +294,8 @@ X86Subtarget::X86Subtarget(const std::string &TT, const std::string &CPU,
                   TargetTriple.getEnvironment() != Triple::CODE16),
       In16BitMode(TargetTriple.getArch() == Triple::x86 &&
                   TargetTriple.getEnvironment() == Triple::CODE16),
-      TSInfo(*TM.getDataLayout()),
-      InstrInfo(initializeSubtargetDependencies(CPU, FS)), TLInfo(TM, *this),
-      FrameLowering(TargetFrameLowering::StackGrowsDown, getStackAlignment(),
-                    is64Bit() ? -8 : -4) {
+      TSInfo(), InstrInfo(initializeSubtargetDependencies(CPU, FS)),
+      TLInfo(TM, *this), FrameLowering(*this, getStackAlignment()) {
   // Determine the PICStyle based on the target selected.
   if (TM.getRelocationModel() == Reloc::Static) {
     // Unless we're in PIC or DynamicNoPIC mode, set the PIC style to None.
@@ -335,3 +320,4 @@ X86Subtarget::X86Subtarget(const std::string &TT, const std::string &CPU,
 bool X86Subtarget::enableEarlyIfConversion() const {
   return hasCMov() && X86EarlyIfConv;
 }
+
diff --git a/lib/Target/X86/X86Subtarget.h b/lib/Target/X86/X86Subtarget.h
index a5dc1890af7..f026d4295f7 100644
--- a/lib/Target/X86/X86Subtarget.h
+++ b/lib/Target/X86/X86Subtarget.h
@@ -190,16 +190,6 @@ class X86Subtarget final : public X86GenSubtargetInfo {
   /// True if INC and DEC instructions are slow when writing to flags
   bool SlowIncDec;
 
-  /// Use the RSQRT* instructions to optimize square root calculations.
-  /// For this to be profitable, the cost of FSQRT and FDIV must be
-  /// substantially higher than normal FP ops like FADD and FMUL.
-  bool UseSqrtEst;
-
-  /// Use the RCP* instructions to optimize FP division calculations.
-  /// For this to be profitable, the cost of FDIV must be
-  /// substantially higher than normal FP ops like FADD and FMUL.
-  bool UseReciprocalEst;
-
   /// Processor has AVX-512 PreFetch Instructions
   bool HasPFI;
 
@@ -218,6 +208,12 @@ class X86Subtarget final : public X86GenSubtargetInfo {
   /// Processor has AVX-512 Vector Length eXtenstions
   bool HasVLX;
 
+  /// Processot supports MPX - Memory Protection Extensions
+  bool HasMPX;
+
+  /// Use software floating point for code generation.
+  bool UseSoftFloat;
+
   /// The minimum alignment known to hold of the stack frame on
   /// entry to the function and which must be maintained by every function.
   unsigned stackAlignment;
@@ -257,9 +253,8 @@ class X86Subtarget final : public X86GenSubtargetInfo {
   /// This constructor initializes the data members to match that
   /// of the specified triple.
   ///
-  X86Subtarget(const std::string &TT, const std::string &CPU,
-               const std::string &FS, const X86TargetMachine &TM,
-               unsigned StackAlignOverride);
+  X86Subtarget(const Triple &TT, const std::string &CPU, const std::string &FS,
+               const X86TargetMachine &TM, unsigned StackAlignOverride);
 
   const X86TargetLowering *getTargetLowering() const override {
     return &TLInfo;
@@ -320,9 +315,6 @@ class X86Subtarget final : public X86GenSubtargetInfo {
                            !TargetTriple.isOSNaCl());
   }
 
-  // @LOCALMOD -- TODO(jvoung): try to use isTarget64BitLP64() instead.
-  bool has64BitPointers() const { return is64Bit() && !isTargetNaCl(); }
-
   PICStyles::Style getPICStyle() const { return PICStyle; }
   void setPICStyle(PICStyles::Style Style)  { PICStyle = Style; }
 
@@ -377,17 +369,17 @@ class X86Subtarget final : public X86GenSubtargetInfo {
   bool LEAusesAG() const { return LEAUsesAG; }
   bool slowLEA() const { return SlowLEA; }
   bool slowIncDec() const { return SlowIncDec; }
-  bool useSqrtEst() const { return UseSqrtEst; }
-  bool useReciprocalEst() const { return UseReciprocalEst; }
   bool hasCDI() const { return HasCDI; }
   bool hasPFI() const { return HasPFI; }
   bool hasERI() const { return HasERI; }
   bool hasDQI() const { return HasDQI; }
   bool hasBWI() const { return HasBWI; }
   bool hasVLX() const { return HasVLX; }
+  bool hasMPX() const { return HasMPX; }
 
   bool isAtom() const { return X86ProcFamily == IntelAtom; }
   bool isSLM() const { return X86ProcFamily == IntelSLM; }
+  bool useSoftFloat() const { return UseSoftFloat; }
 
   const Triple &getTargetTriple() const { return TargetTriple; }
 
@@ -455,8 +447,26 @@ class X86Subtarget final : public X86GenSubtargetInfo {
   }
 
   bool isCallingConvWin64(CallingConv::ID CC) const {
-    return (isTargetWin64() && CC != CallingConv::X86_64_SysV) ||
-           CC == CallingConv::X86_64_Win64;
+    switch (CC) {
+    // On Win64, all these conventions just use the default convention.
+    case CallingConv::C:
+    case CallingConv::Fast:
+    case CallingConv::X86_FastCall:
+    case CallingConv::X86_StdCall:
+    case CallingConv::X86_ThisCall:
+    case CallingConv::X86_VectorCall:
+    case CallingConv::Intel_OCL_BI:
+      return isTargetWin64();
+    // This convention allows using the Win64 convention on other targets.
+    case CallingConv::X86_64_Win64:
+      return true;
+    // This convention allows using the SysV convention on Windows targets.
+    case CallingConv::X86_64_SysV:
+      return false;
+    // Otherwise, who knows what this is.
+    default:
+      return false;
+    }
   }
 
   /// ClassifyGlobalReference - Classify a global variable reference for the
diff --git a/lib/Target/X86/X86TargetMachine.cpp b/lib/Target/X86/X86TargetMachine.cpp
index 64b33da4c27..fb9cb4ba4c8 100644
--- a/lib/Target/X86/X86TargetMachine.cpp
+++ b/lib/Target/X86/X86TargetMachine.cpp
@@ -24,12 +24,9 @@
 #include "llvm/Target/TargetOptions.h"
 using namespace llvm;
 
-// @LOCALMOD-START
-static cl::opt<bool>
-MalignDouble("malign-double", cl::Hidden,
-             cl::desc("Align i64 and f64 types to 8 bytes"));
-// @LOCALMOD-END
-
+static cl::opt<bool> EnableMachineCombinerPass("x86-machine-combiner",
+                               cl::desc("Enable the machine combiner pass"),
+                               cl::init(true), cl::Hidden);
 
 extern "C" void LLVMInitializeX86Target() {
   // Register the target.
@@ -67,8 +64,7 @@ static std::string computeDataLayout(const Triple &TT) {
     Ret += "-p:32:32";
 
   // Some ABIs align 64 bit integers and doubles to 64 bits, others to 32.
-  if (TT.isArch64Bit() || TT.isOSWindows() || TT.isOSNaCl() ||
-      MalignDouble) //@LOCALMOD
+  if (TT.isArch64Bit() || TT.isOSWindows() || TT.isOSNaCl())
     Ret += "-i64:64";
   else
     Ret += "-f64:32:64";
@@ -89,7 +85,7 @@ static std::string computeDataLayout(const Triple &TT) {
 
   // The stack is aligned to 32 bits on some ABIs and 128 bits on others.
   if (!TT.isArch64Bit() && TT.isOSWindows())
-    Ret += "-S32";
+    Ret += "-a:0:32-S32";
   else
     Ret += "-S128";
 
@@ -98,18 +94,15 @@ static std::string computeDataLayout(const Triple &TT) {
 
 /// X86TargetMachine ctor - Create an X86 target.
 ///
-X86TargetMachine::X86TargetMachine(const Target &T, StringRef TT, StringRef CPU,
-                                   StringRef FS, const TargetOptions &Options,
+X86TargetMachine::X86TargetMachine(const Target &T, const Triple &TT,
+                                   StringRef CPU, StringRef FS,
+                                   const TargetOptions &Options,
                                    Reloc::Model RM, CodeModel::Model CM,
                                    CodeGenOpt::Level OL)
-    : LLVMTargetMachine(T, computeDataLayout(Triple(TT)), TT, CPU, FS, Options,
-                        RM, CM, OL),
-      TLOF(createTLOF(Triple(getTargetTriple()))),
+    : LLVMTargetMachine(T, computeDataLayout(TT), TT, CPU, FS, Options, RM, CM,
+                        OL),
+      TLOF(createTLOF(getTargetTriple())),
       Subtarget(TT, CPU, FS, *this, Options.StackAlignmentOverride) {
-  // default to hard float ABI
-  if (Options.FloatABIType == FloatABI::Default)
-    this->Options.FloatABIType = FloatABI::Hard;
-
   // Windows stack unwinder gets confused when execution flow "falls through"
   // after a call to 'noreturn' function.
   // To prevent that, we emit a trap for 'unreachable' IR instructions.
@@ -117,6 +110,16 @@ X86TargetMachine::X86TargetMachine(const Target &T, StringRef TT, StringRef CPU,
   if (Subtarget.isTargetWin64())
     this->Options.TrapUnreachable = true;
 
+  // By default (and when -ffast-math is on), enable estimate codegen for
+  // everything except scalar division. By default, use 1 refinement step for
+  // all operations. Defaults may be overridden by using command-line options.
+  // Scalar division estimates are disabled because they break too much
+  // real-world code. These defaults match GCC behavior.
+  this->Options.Reciprocals.setDefaults("sqrtf", true, 1);
+  this->Options.Reciprocals.setDefaults("divf", false, 1);
+  this->Options.Reciprocals.setDefaults("vec-sqrtf", true, 1);
+  this->Options.Reciprocals.setDefaults("vec-divf", true, 1);
+
   initAsmInfo();
 }
 
@@ -139,13 +142,15 @@ X86TargetMachine::getSubtargetImpl(const Function &F) const {
   // function before we can generate a subtarget. We also need to use
   // it as a key for the subtarget since that can be the only difference
   // between two functions.
-  Attribute SFAttr = F.getFnAttribute("use-soft-float");
-  bool SoftFloat = !SFAttr.hasAttribute(Attribute::None)
-                       ? SFAttr.getValueAsString() == "true"
-                       : Options.UseSoftFloat;
-
-  auto &I = SubtargetMap[CPU + FS + (SoftFloat ? "use-soft-float=true"
-                                               : "use-soft-float=false")];
+  bool SoftFloat =
+      F.hasFnAttribute("use-soft-float") &&
+      F.getFnAttribute("use-soft-float").getValueAsString() == "true";
+  // If the soft float attribute is set on the function turn on the soft float
+  // subtarget feature.
+  if (SoftFloat)
+    FS += FS.empty() ? "+soft-float" : ",+soft-float";
+
+  auto &I = SubtargetMap[CPU + FS];
   if (!I) {
     // This needs to be done before we create a new subtarget since any
     // creation will depend on the TM and the code generation flags on the
@@ -193,9 +198,11 @@ class X86PassConfig : public TargetPassConfig {
   void addIRPasses() override;
   bool addInstSelector() override;
   bool addILPOpts() override;
+  bool addPreISel() override;
   void addPreRegAlloc() override;
   void addPostRegAlloc() override;
   void addPreEmitPass() override;
+  void addPreSched2() override;
 };
 } // namespace
 
@@ -214,7 +221,7 @@ bool X86PassConfig::addInstSelector() {
   addPass(createX86ISelDag(getX86TargetMachine(), getOptLevel()));
 
   // For ELF, cleanup any local-dynamic TLS accesses.
-  if (Triple(TM->getTargetTriple()).isOSBinFormatELF() &&
+  if (TM->getTargetTriple().isOSBinFormatELF() &&
       getOptLevel() != CodeGenOpt::None)
     addPass(createCleanupLocalDynamicTLSPass());
 
@@ -225,6 +232,16 @@ bool X86PassConfig::addInstSelector() {
 
 bool X86PassConfig::addILPOpts() {
   addPass(&EarlyIfConverterID);
+  if (EnableMachineCombinerPass)
+    addPass(&MachineCombinerID);
+  return true;
+}
+
+bool X86PassConfig::addPreISel() {
+  // Only add this pass for 32-bit x86 Windows.
+  const Triple &TT = TM->getTargetTriple();
+  if (TT.isOSWindows() && TT.getArch() == Triple::x86)
+    addPass(createX86WinEHStatePass());
   return true;
 }
 
@@ -236,6 +253,8 @@ void X86PassConfig::addPostRegAlloc() {
   addPass(createX86FloatingPointStackifierPass());
 }
 
+void X86PassConfig::addPreSched2() { addPass(createX86ExpandPseudoPass()); }
+
 void X86PassConfig::addPreEmitPass() {
   if (getOptLevel() != CodeGenOpt::None)
     addPass(createExecutionDependencyFixPass(&X86::VR128RegClass));
@@ -247,9 +266,4 @@ void X86PassConfig::addPreEmitPass() {
     addPass(createX86PadShortFunctions());
     addPass(createX86FixupLEAs());
   }
-  // @LOCALMOD-START
-  if (Triple(TM->getTargetTriple()).isOSNaCl()) {
-    addPass(createX86NaClRewritePass());
-  }
-  // @LOCALMOD-END
 }
diff --git a/lib/Target/X86/X86TargetMachine.h b/lib/Target/X86/X86TargetMachine.h
index c9833ed39e2..262955698e4 100644
--- a/lib/Target/X86/X86TargetMachine.h
+++ b/lib/Target/X86/X86TargetMachine.h
@@ -29,8 +29,8 @@ class X86TargetMachine final : public LLVMTargetMachine {
   mutable StringMap<std::unique_ptr<X86Subtarget>> SubtargetMap;
 
 public:
-  X86TargetMachine(const Target &T, StringRef TT, StringRef CPU, StringRef FS,
-                   const TargetOptions &Options, Reloc::Model RM,
+  X86TargetMachine(const Target &T, const Triple &TT, StringRef CPU,
+                   StringRef FS, const TargetOptions &Options, Reloc::Model RM,
                    CodeModel::Model CM, CodeGenOpt::Level OL);
   ~X86TargetMachine() override;
   const X86Subtarget *getSubtargetImpl(const Function &F) const override;
diff --git a/lib/Target/X86/X86TargetObjectFile.cpp b/lib/Target/X86/X86TargetObjectFile.cpp
index e0b6db8fed6..b4dc6ed149e 100644
--- a/lib/Target/X86/X86TargetObjectFile.cpp
+++ b/lib/Target/X86/X86TargetObjectFile.cpp
@@ -8,7 +8,6 @@
 //===----------------------------------------------------------------------===//
 
 #include "X86TargetObjectFile.h"
-#include "X86Subtarget.h"  // @LOCALMOD
 #include "llvm/ADT/StringExtras.h"
 #include "llvm/IR/Mangler.h"
 #include "llvm/IR/Operator.h"
@@ -33,9 +32,9 @@ const MCExpr *X86_64MachoTargetObjectFile::getTTypeGlobalReference(
   if ((Encoding & DW_EH_PE_indirect) && (Encoding & DW_EH_PE_pcrel)) {
     const MCSymbol *Sym = TM.getSymbol(GV, Mang);
     const MCExpr *Res =
-      MCSymbolRefExpr::Create(Sym, MCSymbolRefExpr::VK_GOTPCREL, getContext());
-    const MCExpr *Four = MCConstantExpr::Create(4, getContext());
-    return MCBinaryExpr::CreateAdd(Res, Four, getContext());
+      MCSymbolRefExpr::create(Sym, MCSymbolRefExpr::VK_GOTPCREL, getContext());
+    const MCExpr *Four = MCConstantExpr::create(4, getContext());
+    return MCBinaryExpr::createAdd(Res, Four, getContext());
   }
 
   return TargetLoweringObjectFileMachO::getTTypeGlobalReference(
@@ -56,14 +55,14 @@ const MCExpr *X86_64MachoTargetObjectFile::getIndirectSymViaGOTPCRel(
   // foo@GOTPCREL+4+<offset>.
   unsigned FinalOff = Offset+MV.getConstant()+4;
   const MCExpr *Res =
-    MCSymbolRefExpr::Create(Sym, MCSymbolRefExpr::VK_GOTPCREL, getContext());
-  const MCExpr *Off = MCConstantExpr::Create(FinalOff, getContext());
-  return MCBinaryExpr::CreateAdd(Res, Off, getContext());
+    MCSymbolRefExpr::create(Sym, MCSymbolRefExpr::VK_GOTPCREL, getContext());
+  const MCExpr *Off = MCConstantExpr::create(FinalOff, getContext());
+  return MCBinaryExpr::createAdd(Res, Off, getContext());
 }
 
 const MCExpr *X86ELFTargetObjectFile::getDebugThreadLocalSymbol(
     const MCSymbol *Sym) const {
-  return MCSymbolRefExpr::Create(Sym, MCSymbolRefExpr::VK_DTPOFF, getContext());
+  return MCSymbolRefExpr::create(Sym, MCSymbolRefExpr::VK_DTPOFF, getContext());
 }
 
 void
@@ -117,7 +116,7 @@ const MCExpr *X86WindowsTargetObjectFile::getExecutableRelativeSymbol(
   if (GOLHS->isThreadLocal())
     return nullptr;
 
-  return MCSymbolRefExpr::Create(TM.getSymbol(GOLHS, Mang),
+  return MCSymbolRefExpr::create(TM.getSymbol(GOLHS, Mang),
                                  MCSymbolRefExpr::VK_COFF_IMGREL32,
                                  getContext());
 }
@@ -132,53 +131,44 @@ static std::string APIntToHexString(const APInt &AI) {
   return HexString;
 }
 
-
 static std::string scalarConstantToHexString(const Constant *C) {
   Type *Ty = C->getType();
-  APInt AI;
   if (isa<UndefValue>(C)) {
-    AI = APInt(Ty->getPrimitiveSizeInBits(), /*val=*/0);
-  } else if (Ty->isFloatTy() || Ty->isDoubleTy()) {
-    const auto *CFP = cast<ConstantFP>(C);
-    AI = CFP->getValueAPF().bitcastToAPInt();
-  } else if (Ty->isIntegerTy()) {
-    const auto *CI = cast<ConstantInt>(C);
-    AI = CI->getValue();
+    return APIntToHexString(APInt::getNullValue(Ty->getPrimitiveSizeInBits()));
+  } else if (const auto *CFP = dyn_cast<ConstantFP>(C)) {
+    return APIntToHexString(CFP->getValueAPF().bitcastToAPInt());
+  } else if (const auto *CI = dyn_cast<ConstantInt>(C)) {
+    return APIntToHexString(CI->getValue());
   } else {
-    llvm_unreachable("unexpected constant pool element type!");
+    unsigned NumElements;
+    if (isa<VectorType>(Ty))
+      NumElements = Ty->getVectorNumElements();
+    else
+      NumElements = Ty->getArrayNumElements();
+    std::string HexString;
+    for (int I = NumElements - 1, E = -1; I != E; --I)
+      HexString += scalarConstantToHexString(C->getAggregateElement(I));
+    return HexString;
   }
-  return APIntToHexString(AI);
 }
 
-const MCSection *
-X86WindowsTargetObjectFile::getSectionForConstant(SectionKind Kind,
-                                                  const Constant *C) const {
-  if (Kind.isReadOnly()) {
-    if (C) {
-      Type *Ty = C->getType();
-      SmallString<32> COMDATSymName;
-      if (Ty->isFloatTy() || Ty->isDoubleTy()) {
-        COMDATSymName = "__real@";
-        COMDATSymName += scalarConstantToHexString(C);
-      } else if (const auto *VTy = dyn_cast<VectorType>(Ty)) {
-        uint64_t NumBits = VTy->getBitWidth();
-        if (NumBits == 128 || NumBits == 256) {
-          COMDATSymName = NumBits == 128 ? "__xmm@" : "__ymm@";
-          for (int I = VTy->getNumElements() - 1, E = -1; I != E; --I)
-            COMDATSymName +=
-                scalarConstantToHexString(C->getAggregateElement(I));
-        }
-      }
-      if (!COMDATSymName.empty()) {
-        unsigned Characteristics = COFF::IMAGE_SCN_CNT_INITIALIZED_DATA |
-                                   COFF::IMAGE_SCN_MEM_READ |
-                                   COFF::IMAGE_SCN_LNK_COMDAT;
-        return getContext().getCOFFSection(".rdata", Characteristics, Kind,
-                                           COMDATSymName,
-                                           COFF::IMAGE_COMDAT_SELECT_ANY);
-      }
-    }
+MCSection *X86WindowsTargetObjectFile::getSectionForConstant(
+    const DataLayout &DL, SectionKind Kind, const Constant *C) const {
+  if (Kind.isMergeableConst() && C) {
+    const unsigned Characteristics = COFF::IMAGE_SCN_CNT_INITIALIZED_DATA |
+                                     COFF::IMAGE_SCN_MEM_READ |
+                                     COFF::IMAGE_SCN_LNK_COMDAT;
+    std::string COMDATSymName;
+    if (Kind.isMergeableConst4() || Kind.isMergeableConst8())
+      COMDATSymName = "__real@" + scalarConstantToHexString(C);
+    else if (Kind.isMergeableConst16())
+      COMDATSymName = "__xmm@" + scalarConstantToHexString(C);
+
+    if (!COMDATSymName.empty())
+      return getContext().getCOFFSection(".rdata", Characteristics, Kind,
+                                         COMDATSymName,
+                                         COFF::IMAGE_COMDAT_SELECT_ANY);
   }
 
-  return TargetLoweringObjectFile::getSectionForConstant(Kind, C);
+  return TargetLoweringObjectFile::getSectionForConstant(DL, Kind, C);
 }
diff --git a/lib/Target/X86/X86TargetObjectFile.h b/lib/Target/X86/X86TargetObjectFile.h
index 2e25fb2b849..6b2448cc9de 100644
--- a/lib/Target/X86/X86TargetObjectFile.h
+++ b/lib/Target/X86/X86TargetObjectFile.h
@@ -58,8 +58,8 @@ namespace llvm {
 
     /// \brief Given a mergeable constant with the specified size and relocation
     /// information, return a section that it should be placed in.
-    const MCSection *getSectionForConstant(SectionKind Kind,
-                                           const Constant *C) const override;
+    MCSection *getSectionForConstant(const DataLayout &DL, SectionKind Kind,
+                                     const Constant *C) const override;
   };
 
 } // end namespace llvm
diff --git a/lib/Target/X86/X86TargetTransformInfo.cpp b/lib/Target/X86/X86TargetTransformInfo.cpp
index 5136619235b..673aefd046f 100644
--- a/lib/Target/X86/X86TargetTransformInfo.cpp
+++ b/lib/Target/X86/X86TargetTransformInfo.cpp
@@ -66,7 +66,13 @@ unsigned X86TTIImpl::getRegisterBitWidth(bool Vector) {
 
 }
 
-unsigned X86TTIImpl::getMaxInterleaveFactor() {
+unsigned X86TTIImpl::getMaxInterleaveFactor(unsigned VF) {
+  // If the loop will not be vectorized, don't interleave the loop.
+  // Let regular unroll to unroll the loop, which saves the overflow
+  // check and memory check cost.
+  if (VF == 1)
+    return 1;
+
   if (ST->isAtom())
     return 1;
 
@@ -83,7 +89,7 @@ unsigned X86TTIImpl::getArithmeticInstrCost(
     TTI::OperandValueKind Op2Info, TTI::OperandValueProperties Opd1PropInfo,
     TTI::OperandValueProperties Opd2PropInfo) {
   // Legalize the type.
-  std::pair<unsigned, MVT> LT = TLI->getTypeLegalizationCost(Ty);
+  std::pair<unsigned, MVT> LT = TLI->getTypeLegalizationCost(DL, Ty);
 
   int ISD = TLI->InstructionOpcodeToISD(Opcode);
   assert(ISD && "Invalid opcode");
@@ -111,6 +117,8 @@ unsigned X86TTIImpl::getArithmeticInstrCost(
 
   static const CostTblEntry<MVT::SimpleValueType>
   AVX2UniformConstCostTable[] = {
+    { ISD::SRA,  MVT::v4i64,   4 }, // 2 x psrad + shuffle.
+
     { ISD::SDIV, MVT::v16i16,  6 }, // vpmulhw sequence
     { ISD::UDIV, MVT::v16i16,  6 }, // vpmulhuw sequence
     { ISD::SDIV, MVT::v8i32,  15 }, // vpmuldq sequence
@@ -147,15 +155,16 @@ unsigned X86TTIImpl::getArithmeticInstrCost(
     { ISD::SHL,     MVT::v4i64,    1 },
     { ISD::SRL,     MVT::v4i64,    1 },
 
-    { ISD::SHL,  MVT::v32i8,  42 }, // cmpeqb sequence.
-    { ISD::SHL,  MVT::v16i16,  16*10 }, // Scalarized.
+    { ISD::SHL,  MVT::v32i8,      11 }, // vpblendvb sequence.
+    { ISD::SHL,  MVT::v16i16,     10 }, // extend/vpsrlvd/pack sequence.
 
-    { ISD::SRL,  MVT::v32i8,  32*10 }, // Scalarized.
-    { ISD::SRL,  MVT::v16i16,  8*10 }, // Scalarized.
+    { ISD::SRL,  MVT::v32i8,      11 }, // vpblendvb sequence.
+    { ISD::SRL,  MVT::v16i16,     10 }, // extend/vpsrlvd/pack sequence.
 
-    { ISD::SRA,  MVT::v32i8,  32*10 }, // Scalarized.
-    { ISD::SRA,  MVT::v16i16,  16*10 }, // Scalarized.
-    { ISD::SRA,  MVT::v4i64,  4*10 }, // Scalarized.
+    { ISD::SRA,  MVT::v32i8,      24 }, // vpblendvb sequence.
+    { ISD::SRA,  MVT::v16i16,     10 }, // extend/vpsravd/pack sequence.
+    { ISD::SRA,  MVT::v2i64,       4 }, // srl/xor/sub sequence.
+    { ISD::SRA,  MVT::v4i64,       4 }, // srl/xor/sub sequence.
 
     // Vectorizing division is a bad idea. See the SSE2 table for more comments.
     { ISD::SDIV,  MVT::v32i8,  32*20 },
@@ -205,6 +214,7 @@ unsigned X86TTIImpl::getArithmeticInstrCost(
     { ISD::SRA,  MVT::v16i8,  4 }, // psrlw, pand, pxor, psubb.
     { ISD::SRA,  MVT::v8i16,  1 }, // psraw.
     { ISD::SRA,  MVT::v4i32,  1 }, // psrad.
+    { ISD::SRA,  MVT::v2i64,  4 }, // 2 x psrad + shuffle.
 
     { ISD::SDIV, MVT::v8i16,  6 }, // pmulhw sequence
     { ISD::UDIV, MVT::v8i16,  6 }, // pmulhuw sequence
@@ -247,21 +257,21 @@ unsigned X86TTIImpl::getArithmeticInstrCost(
     // to ISel. The cost model must return worst case assumptions because it is
     // used for vectorization and we don't want to make vectorized code worse
     // than scalar code.
-    { ISD::SHL,  MVT::v16i8,  30 }, // cmpeqb sequence.
-    { ISD::SHL,  MVT::v8i16,  8*10 }, // Scalarized.
-    { ISD::SHL,  MVT::v4i32,  2*5 }, // We optimized this using mul.
-    { ISD::SHL,  MVT::v2i64,  2*10 }, // Scalarized.
-    { ISD::SHL,  MVT::v4i64,  4*10 }, // Scalarized.
-
-    { ISD::SRL,  MVT::v16i8,  16*10 }, // Scalarized.
-    { ISD::SRL,  MVT::v8i16,  8*10 }, // Scalarized.
-    { ISD::SRL,  MVT::v4i32,  4*10 }, // Scalarized.
-    { ISD::SRL,  MVT::v2i64,  2*10 }, // Scalarized.
-
-    { ISD::SRA,  MVT::v16i8,  16*10 }, // Scalarized.
-    { ISD::SRA,  MVT::v8i16,  8*10 }, // Scalarized.
-    { ISD::SRA,  MVT::v4i32,  4*10 }, // Scalarized.
-    { ISD::SRA,  MVT::v2i64,  2*10 }, // Scalarized.
+    { ISD::SHL,  MVT::v16i8,    26 }, // cmpgtb sequence.
+    { ISD::SHL,  MVT::v8i16,    32 }, // cmpgtb sequence.
+    { ISD::SHL,  MVT::v4i32,   2*5 }, // We optimized this using mul.
+    { ISD::SHL,  MVT::v2i64,     4 }, // splat+shuffle sequence.
+    { ISD::SHL,  MVT::v4i64,     8 }, // splat+shuffle sequence.
+
+    { ISD::SRL,  MVT::v16i8,    26 }, // cmpgtb sequence.
+    { ISD::SRL,  MVT::v8i16,    32 }, // cmpgtb sequence.
+    { ISD::SRL,  MVT::v4i32,    16 }, // Shift each lane + blend.
+    { ISD::SRL,  MVT::v2i64,     4 }, // splat+shuffle sequence.
+
+    { ISD::SRA,  MVT::v16i8,    54 }, // unpacked cmpgtb sequence.
+    { ISD::SRA,  MVT::v8i16,    32 }, // cmpgtb sequence.
+    { ISD::SRA,  MVT::v4i32,    16 }, // Shift each lane + blend.
+    { ISD::SRA,  MVT::v2i64,    12 }, // srl/xor/sub sequence.
 
     // It is not a good idea to vectorize division. We have to scalarize it and
     // in the process we will often end up having to spilling regular
@@ -346,7 +356,7 @@ unsigned X86TTIImpl::getShuffleCost(TTI::ShuffleKind Kind, Type *Tp, int Index,
     return BaseT::getShuffleCost(Kind, Tp, Index, SubTp);
 
   if (Kind == TTI::SK_Reverse) {
-    std::pair<unsigned, MVT> LT = TLI->getTypeLegalizationCost(Tp);
+    std::pair<unsigned, MVT> LT = TLI->getTypeLegalizationCost(DL, Tp);
     unsigned Cost = 1;
     if (LT.second.getSizeInBits() > 128)
       Cost = 3; // Extract + insert + copy.
@@ -358,7 +368,7 @@ unsigned X86TTIImpl::getShuffleCost(TTI::ShuffleKind Kind, Type *Tp, int Index,
   if (Kind == TTI::SK_Alternate) {
     // 64-bit packed float vectors (v2f32) are widened to type v4f32.
     // 64-bit packed integer vectors (v2i32) are promoted to type v2i64.
-    std::pair<unsigned, MVT> LT = TLI->getTypeLegalizationCost(Tp);
+    std::pair<unsigned, MVT> LT = TLI->getTypeLegalizationCost(DL, Tp);
 
     // The backend knows how to generate a single VEX.256 version of
     // instruction VPBLENDW if the target supports AVX2.
@@ -458,40 +468,6 @@ unsigned X86TTIImpl::getCastInstrCost(unsigned Opcode, Type *Dst, Type *Src) {
   int ISD = TLI->InstructionOpcodeToISD(Opcode);
   assert(ISD && "Invalid opcode");
 
-  std::pair<unsigned, MVT> LTSrc = TLI->getTypeLegalizationCost(Src);
-  std::pair<unsigned, MVT> LTDest = TLI->getTypeLegalizationCost(Dst);
-
-  static const TypeConversionCostTblEntry<MVT::SimpleValueType>
-  SSE2ConvTbl[] = {
-    // These are somewhat magic numbers justified by looking at the output of
-    // Intel's IACA, running some kernels and making sure when we take
-    // legalization into account the throughput will be overestimated.
-    { ISD::UINT_TO_FP, MVT::v2f64, MVT::v2i64, 2*10 },
-    { ISD::UINT_TO_FP, MVT::v2f64, MVT::v4i32, 4*10 },
-    { ISD::UINT_TO_FP, MVT::v2f64, MVT::v8i16, 8*10 },
-    { ISD::UINT_TO_FP, MVT::v2f64, MVT::v16i8, 16*10 },
-    { ISD::SINT_TO_FP, MVT::v2f64, MVT::v2i64, 2*10 },
-    { ISD::SINT_TO_FP, MVT::v2f64, MVT::v4i32, 4*10 },
-    { ISD::SINT_TO_FP, MVT::v2f64, MVT::v8i16, 8*10 },
-    { ISD::SINT_TO_FP, MVT::v2f64, MVT::v16i8, 16*10 },
-    // There are faster sequences for float conversions.
-    { ISD::UINT_TO_FP, MVT::v4f32, MVT::v2i64, 15 },
-    { ISD::UINT_TO_FP, MVT::v4f32, MVT::v4i32, 8 },
-    { ISD::UINT_TO_FP, MVT::v4f32, MVT::v8i16, 15 },
-    { ISD::UINT_TO_FP, MVT::v4f32, MVT::v16i8, 8 },
-    { ISD::SINT_TO_FP, MVT::v4f32, MVT::v2i64, 15 },
-    { ISD::SINT_TO_FP, MVT::v4f32, MVT::v4i32, 15 },
-    { ISD::SINT_TO_FP, MVT::v4f32, MVT::v8i16, 15 },
-    { ISD::SINT_TO_FP, MVT::v4f32, MVT::v16i8, 8 },
-  };
-
-  if (ST->hasSSE2() && !ST->hasAVX()) {
-    int Idx =
-        ConvertCostTableLookup(SSE2ConvTbl, ISD, LTDest.second, LTSrc.second);
-    if (Idx != -1)
-      return LTSrc.first * SSE2ConvTbl[Idx].Cost;
-  }
-
   static const TypeConversionCostTblEntry<MVT::SimpleValueType>
   AVX512ConversionTbl[] = {
     { ISD::FP_EXTEND, MVT::v8f64,   MVT::v8f32,  1 },
@@ -525,19 +501,6 @@ unsigned X86TTIImpl::getCastInstrCost(unsigned Opcode, Type *Dst, Type *Src) {
     { ISD::SINT_TO_FP,  MVT::v8f64,  MVT::v8i32,  1 },
   };
 
-  if (ST->hasAVX512()) {
-    int Idx = ConvertCostTableLookup(AVX512ConversionTbl, ISD, LTDest.second,
-                                     LTSrc.second);
-    if (Idx != -1)
-      return AVX512ConversionTbl[Idx].Cost;
-  }
-  EVT SrcTy = TLI->getValueType(Src);
-  EVT DstTy = TLI->getValueType(Dst);
-
-  // The function getSimpleVT only handles simple value types.
-  if (!SrcTy.isSimple() || !DstTy.isSimple())
-    return BaseT::getCastInstrCost(Opcode, Dst, Src);
-
   static const TypeConversionCostTblEntry<MVT::SimpleValueType>
   AVX2ConversionTbl[] = {
     { ISD::SIGN_EXTEND, MVT::v16i16, MVT::v16i8,  1 },
@@ -641,6 +604,54 @@ unsigned X86TTIImpl::getCastInstrCost(unsigned Opcode, Type *Dst, Type *Src) {
     { ISD::FP_TO_UINT,  MVT::v4i32, MVT::v4f64, 4*4 },
   };
 
+  static const TypeConversionCostTblEntry<MVT::SimpleValueType>
+  SSE2ConvTbl[] = {
+    // These are somewhat magic numbers justified by looking at the output of
+    // Intel's IACA, running some kernels and making sure when we take
+    // legalization into account the throughput will be overestimated.
+    { ISD::UINT_TO_FP, MVT::v2f64, MVT::v2i64, 2*10 },
+    { ISD::UINT_TO_FP, MVT::v2f64, MVT::v4i32, 4*10 },
+    { ISD::UINT_TO_FP, MVT::v2f64, MVT::v8i16, 8*10 },
+    { ISD::UINT_TO_FP, MVT::v2f64, MVT::v16i8, 16*10 },
+    { ISD::SINT_TO_FP, MVT::v2f64, MVT::v2i64, 2*10 },
+    { ISD::SINT_TO_FP, MVT::v2f64, MVT::v4i32, 4*10 },
+    { ISD::SINT_TO_FP, MVT::v2f64, MVT::v8i16, 8*10 },
+    { ISD::SINT_TO_FP, MVT::v2f64, MVT::v16i8, 16*10 },
+    // There are faster sequences for float conversions.
+    { ISD::UINT_TO_FP, MVT::v4f32, MVT::v2i64, 15 },
+    { ISD::UINT_TO_FP, MVT::v4f32, MVT::v4i32, 8 },
+    { ISD::UINT_TO_FP, MVT::v4f32, MVT::v8i16, 15 },
+    { ISD::UINT_TO_FP, MVT::v4f32, MVT::v16i8, 8 },
+    { ISD::SINT_TO_FP, MVT::v4f32, MVT::v2i64, 15 },
+    { ISD::SINT_TO_FP, MVT::v4f32, MVT::v4i32, 15 },
+    { ISD::SINT_TO_FP, MVT::v4f32, MVT::v8i16, 15 },
+    { ISD::SINT_TO_FP, MVT::v4f32, MVT::v16i8, 8 },
+  };
+
+  std::pair<unsigned, MVT> LTSrc = TLI->getTypeLegalizationCost(DL, Src);
+  std::pair<unsigned, MVT> LTDest = TLI->getTypeLegalizationCost(DL, Dst);
+
+  if (ST->hasSSE2() && !ST->hasAVX()) {
+    int Idx =
+        ConvertCostTableLookup(SSE2ConvTbl, ISD, LTDest.second, LTSrc.second);
+    if (Idx != -1)
+      return LTSrc.first * SSE2ConvTbl[Idx].Cost;
+  }
+
+  if (ST->hasAVX512()) {
+    int Idx = ConvertCostTableLookup(AVX512ConversionTbl, ISD, LTDest.second,
+                                     LTSrc.second);
+    if (Idx != -1)
+      return AVX512ConversionTbl[Idx].Cost;
+  }
+
+  EVT SrcTy = TLI->getValueType(DL, Src);
+  EVT DstTy = TLI->getValueType(DL, Dst);
+
+  // The function getSimpleVT only handles simple value types.
+  if (!SrcTy.isSimple() || !DstTy.isSimple())
+    return BaseT::getCastInstrCost(Opcode, Dst, Src);
+
   if (ST->hasAVX2()) {
     int Idx = ConvertCostTableLookup(AVX2ConversionTbl, ISD,
                                      DstTy.getSimpleVT(), SrcTy.getSimpleVT());
@@ -661,7 +672,7 @@ unsigned X86TTIImpl::getCastInstrCost(unsigned Opcode, Type *Dst, Type *Src) {
 unsigned X86TTIImpl::getCmpSelInstrCost(unsigned Opcode, Type *ValTy,
                                         Type *CondTy) {
   // Legalize the type.
-  std::pair<unsigned, MVT> LT = TLI->getTypeLegalizationCost(ValTy);
+  std::pair<unsigned, MVT> LT = TLI->getTypeLegalizationCost(DL, ValTy);
 
   MVT MTy = LT.second;
 
@@ -734,7 +745,7 @@ unsigned X86TTIImpl::getVectorInstrCost(unsigned Opcode, Type *Val,
 
   if (Index != -1U) {
     // Legalize the type.
-    std::pair<unsigned, MVT> LT = TLI->getTypeLegalizationCost(Val);
+    std::pair<unsigned, MVT> LT = TLI->getTypeLegalizationCost(DL, Val);
 
     // This type is legalized to a scalar type.
     if (!LT.second.isVector())
@@ -797,7 +808,7 @@ unsigned X86TTIImpl::getMemoryOpCost(unsigned Opcode, Type *Src,
   }
 
   // Legalize the type.
-  std::pair<unsigned, MVT> LT = TLI->getTypeLegalizationCost(Src);
+  std::pair<unsigned, MVT> LT = TLI->getTypeLegalizationCost(DL, Src);
   assert((Opcode == Instruction::Load || Opcode == Instruction::Store) &&
          "Invalid Opcode");
 
@@ -844,9 +855,9 @@ unsigned X86TTIImpl::getMaskedMemoryOpCost(unsigned Opcode, Type *SrcTy,
   }
 
   // Legalize the type.
-  std::pair<unsigned, MVT> LT = TLI->getTypeLegalizationCost(SrcVTy);
+  std::pair<unsigned, MVT> LT = TLI->getTypeLegalizationCost(DL, SrcVTy);
   unsigned Cost = 0;
-  if (LT.second != TLI->getValueType(SrcVTy).getSimpleVT() &&
+  if (LT.second != TLI->getValueType(DL, SrcVTy).getSimpleVT() &&
       LT.second.getVectorNumElements() == NumElem)
     // Promotion requires expand/truncate for data and a shuffle for mask.
     Cost += getShuffleCost(TTI::SK_Alternate, SrcVTy, 0, 0) +
@@ -881,7 +892,7 @@ unsigned X86TTIImpl::getAddressComputationCost(Type *Ty, bool IsComplex) {
 unsigned X86TTIImpl::getReductionCost(unsigned Opcode, Type *ValTy,
                                       bool IsPairwise) {
 
-  std::pair<unsigned, MVT> LT = TLI->getTypeLegalizationCost(ValTy);
+  std::pair<unsigned, MVT> LT = TLI->getTypeLegalizationCost(DL, ValTy);
 
   MVT MTy = LT.second;
 
@@ -1111,11 +1122,11 @@ unsigned X86TTIImpl::getIntImmCost(Intrinsic::ID IID, unsigned Idx,
 
 bool X86TTIImpl::isLegalMaskedLoad(Type *DataTy, int Consecutive) {
   int DataWidth = DataTy->getPrimitiveSizeInBits();
-  
+
   // Todo: AVX512 allows gather/scatter, works with strided and random as well
   if ((DataWidth < 32) || (Consecutive == 0))
     return false;
-  if (ST->hasAVX512() || ST->hasAVX2()) 
+  if (ST->hasAVX512() || ST->hasAVX2())
     return true;
   return false;
 }
@@ -1124,3 +1135,18 @@ bool X86TTIImpl::isLegalMaskedStore(Type *DataType, int Consecutive) {
   return isLegalMaskedLoad(DataType, Consecutive);
 }
 
+bool X86TTIImpl::areInlineCompatible(const Function *Caller,
+                                     const Function *Callee) const {
+  const TargetMachine &TM = getTLI()->getTargetMachine();
+
+  // Work this as a subsetting of subtarget features.
+  const FeatureBitset &CallerBits =
+      TM.getSubtargetImpl(*Caller)->getFeatureBits();
+  const FeatureBitset &CalleeBits =
+      TM.getSubtargetImpl(*Callee)->getFeatureBits();
+
+  // FIXME: This is likely too limiting as it will include subtarget features
+  // that we might not care about for inlining, but it is conservatively
+  // correct.
+  return (CallerBits & CalleeBits) == CalleeBits;
+}
diff --git a/lib/Target/X86/X86TargetTransformInfo.h b/lib/Target/X86/X86TargetTransformInfo.h
index 9f0adcfef62..23efcade51d 100644
--- a/lib/Target/X86/X86TargetTransformInfo.h
+++ b/lib/Target/X86/X86TargetTransformInfo.h
@@ -40,7 +40,8 @@ class X86TTIImpl : public BasicTTIImplBase<X86TTIImpl> {
 
 public:
   explicit X86TTIImpl(const X86TargetMachine *TM, Function &F)
-      : BaseT(TM), ST(TM->getSubtargetImpl(F)), TLI(ST->getTargetLowering()) {}
+      : BaseT(TM, F.getParent()->getDataLayout()), ST(TM->getSubtargetImpl(F)),
+        TLI(ST->getTargetLowering()) {}
 
   // Provide value semantics. MSVC requires that we spell all of these out.
   X86TTIImpl(const X86TTIImpl &Arg)
@@ -48,18 +49,6 @@ class X86TTIImpl : public BasicTTIImplBase<X86TTIImpl> {
   X86TTIImpl(X86TTIImpl &&Arg)
       : BaseT(std::move(static_cast<BaseT &>(Arg))), ST(std::move(Arg.ST)),
         TLI(std::move(Arg.TLI)) {}
-  X86TTIImpl &operator=(const X86TTIImpl &RHS) {
-    BaseT::operator=(static_cast<const BaseT &>(RHS));
-    ST = RHS.ST;
-    TLI = RHS.TLI;
-    return *this;
-  }
-  X86TTIImpl &operator=(X86TTIImpl &&RHS) {
-    BaseT::operator=(std::move(static_cast<BaseT &>(RHS)));
-    ST = std::move(RHS.ST);
-    TLI = std::move(RHS.TLI);
-    return *this;
-  }
 
   /// \name Scalar TTI Implementations
   /// @{
@@ -72,7 +61,7 @@ class X86TTIImpl : public BasicTTIImplBase<X86TTIImpl> {
 
   unsigned getNumberOfRegisters(bool Vector);
   unsigned getRegisterBitWidth(bool Vector);
-  unsigned getMaxInterleaveFactor();
+  unsigned getMaxInterleaveFactor(unsigned VF);
   unsigned getArithmeticInstrCost(
       unsigned Opcode, Type *Ty,
       TTI::OperandValueKind Opd1Info = TTI::OK_AnyValue,
@@ -103,6 +92,8 @@ class X86TTIImpl : public BasicTTIImplBase<X86TTIImpl> {
                          Type *Ty);
   bool isLegalMaskedLoad(Type *DataType, int Consecutive);
   bool isLegalMaskedStore(Type *DataType, int Consecutive);
+  bool areInlineCompatible(const Function *Caller,
+                           const Function *Callee) const;
 
   /// @}
 };
diff --git a/lib/Target/X86/X86VZeroUpper.cpp b/lib/Target/X86/X86VZeroUpper.cpp
index 99ba4c02a46..6925b272b4a 100644
--- a/lib/Target/X86/X86VZeroUpper.cpp
+++ b/lib/Target/X86/X86VZeroUpper.cpp
@@ -281,8 +281,8 @@ bool VZeroUpperInserter::runOnMachineFunction(MachineFunction &MF) {
   // Process all blocks. This will compute block exit states, record the first
   // unguarded call in each block, and add successors of dirty blocks to the
   // DirtySuccessors list.
-  for (MachineFunction::iterator I = MF.begin(), E = MF.end(); I != E; ++I)
-    processBasicBlock(*I);
+  for (MachineBasicBlock &MBB : MF)
+    processBasicBlock(MBB);
 
   // If any YMM regs are live in to this function, add the entry block to the
   // DirtySuccessors list
diff --git a/lib/Target/X86/X86WinEHState.cpp b/lib/Target/X86/X86WinEHState.cpp
new file mode 100644
index 00000000000..9190d0be9e4
--- /dev/null
+++ b/lib/Target/X86/X86WinEHState.cpp
@@ -0,0 +1,556 @@
+//===-- X86WinEHState - Insert EH state updates for win32 exceptions ------===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// All functions using an MSVC EH personality use an explicitly updated state
+// number stored in an exception registration stack object. The registration
+// object is linked into a thread-local chain of registrations stored at fs:00.
+// This pass adds the registration object and EH state updates.
+//
+//===----------------------------------------------------------------------===//
+
+#include "X86.h"
+#include "llvm/Analysis/LibCallSemantics.h"
+#include "llvm/CodeGen/MachineModuleInfo.h"
+#include "llvm/CodeGen/Passes.h"
+#include "llvm/CodeGen/WinEHFuncInfo.h"
+#include "llvm/IR/Dominators.h"
+#include "llvm/IR/Function.h"
+#include "llvm/IR/IRBuilder.h"
+#include "llvm/IR/Instructions.h"
+#include "llvm/IR/IntrinsicInst.h"
+#include "llvm/IR/Module.h"
+#include "llvm/IR/PatternMatch.h"
+#include "llvm/Pass.h"
+#include "llvm/Support/Debug.h"
+#include "llvm/Support/raw_ostream.h"
+#include "llvm/Transforms/Utils/BasicBlockUtils.h"
+#include "llvm/Transforms/Utils/Cloning.h"
+#include "llvm/Transforms/Utils/Local.h"
+
+using namespace llvm;
+using namespace llvm::PatternMatch;
+
+#define DEBUG_TYPE "winehstate"
+
+namespace {
+class WinEHStatePass : public FunctionPass {
+public:
+  static char ID; // Pass identification, replacement for typeid.
+
+  WinEHStatePass() : FunctionPass(ID) {}
+
+  bool runOnFunction(Function &Fn) override;
+
+  bool doInitialization(Module &M) override;
+
+  bool doFinalization(Module &M) override;
+
+  void getAnalysisUsage(AnalysisUsage &AU) const override;
+
+  const char *getPassName() const override {
+    return "Windows 32-bit x86 EH state insertion";
+  }
+
+private:
+  void emitExceptionRegistrationRecord(Function *F);
+
+  void linkExceptionRegistration(IRBuilder<> &Builder, Function *Handler);
+  void unlinkExceptionRegistration(IRBuilder<> &Builder);
+  void addCXXStateStores(Function &F, MachineModuleInfo &MMI);
+  void addSEHStateStores(Function &F, MachineModuleInfo &MMI);
+  void addCXXStateStoresToFunclet(Value *ParentRegNode, WinEHFuncInfo &FuncInfo,
+                                  Function &F, int BaseState);
+  void insertStateNumberStore(Value *ParentRegNode, Instruction *IP, int State);
+
+  Value *emitEHLSDA(IRBuilder<> &Builder, Function *F);
+
+  Function *generateLSDAInEAXThunk(Function *ParentFunc);
+
+  int escapeRegNode(Function &F);
+
+  // Module-level type getters.
+  Type *getEHLinkRegistrationType();
+  Type *getSEHRegistrationType();
+  Type *getCXXEHRegistrationType();
+
+  // Per-module data.
+  Module *TheModule = nullptr;
+  StructType *EHLinkRegistrationTy = nullptr;
+  StructType *CXXEHRegistrationTy = nullptr;
+  StructType *SEHRegistrationTy = nullptr;
+  Function *FrameRecover = nullptr;
+  Function *FrameAddress = nullptr;
+  Function *FrameEscape = nullptr;
+
+  // Per-function state
+  EHPersonality Personality = EHPersonality::Unknown;
+  Function *PersonalityFn = nullptr;
+
+  /// The stack allocation containing all EH data, including the link in the
+  /// fs:00 chain and the current state.
+  AllocaInst *RegNode = nullptr;
+
+  /// Struct type of RegNode. Used for GEPing.
+  Type *RegNodeTy = nullptr;
+
+  /// The index of the state field of RegNode.
+  int StateFieldIndex = ~0U;
+
+  /// The linked list node subobject inside of RegNode.
+  Value *Link = nullptr;
+};
+}
+
+FunctionPass *llvm::createX86WinEHStatePass() { return new WinEHStatePass(); }
+
+char WinEHStatePass::ID = 0;
+
+bool WinEHStatePass::doInitialization(Module &M) {
+  TheModule = &M;
+  FrameEscape = Intrinsic::getDeclaration(TheModule, Intrinsic::localescape);
+  FrameRecover = Intrinsic::getDeclaration(TheModule, Intrinsic::localrecover);
+  FrameAddress = Intrinsic::getDeclaration(TheModule, Intrinsic::frameaddress);
+  return false;
+}
+
+bool WinEHStatePass::doFinalization(Module &M) {
+  assert(TheModule == &M);
+  TheModule = nullptr;
+  EHLinkRegistrationTy = nullptr;
+  CXXEHRegistrationTy = nullptr;
+  SEHRegistrationTy = nullptr;
+  FrameEscape = nullptr;
+  FrameRecover = nullptr;
+  FrameAddress = nullptr;
+  return false;
+}
+
+void WinEHStatePass::getAnalysisUsage(AnalysisUsage &AU) const {
+  // This pass should only insert a stack allocation, memory accesses, and
+  // localrecovers.
+  AU.setPreservesCFG();
+}
+
+bool WinEHStatePass::runOnFunction(Function &F) {
+  // If this is an outlined handler, don't do anything. We'll do state insertion
+  // for it in the parent.
+  StringRef WinEHParentName =
+      F.getFnAttribute("wineh-parent").getValueAsString();
+  if (WinEHParentName != F.getName() && !WinEHParentName.empty())
+    return false;
+
+  // Check the personality. Do nothing if this is not an MSVC personality.
+  if (!F.hasPersonalityFn())
+    return false;
+  PersonalityFn =
+      dyn_cast<Function>(F.getPersonalityFn()->stripPointerCasts());
+  if (!PersonalityFn)
+    return false;
+  Personality = classifyEHPersonality(PersonalityFn);
+  if (!isMSVCEHPersonality(Personality))
+    return false;
+
+  // Disable frame pointer elimination in this function.
+  // FIXME: Do the nested handlers need to keep the parent ebp in ebp, or can we
+  // use an arbitrary register?
+  F.addFnAttr("no-frame-pointer-elim", "true");
+
+  emitExceptionRegistrationRecord(&F);
+
+  auto *MMIPtr = getAnalysisIfAvailable<MachineModuleInfo>();
+  assert(MMIPtr && "MachineModuleInfo should always be available");
+  MachineModuleInfo &MMI = *MMIPtr;
+  switch (Personality) {
+  default: llvm_unreachable("unexpected personality function");
+  case EHPersonality::MSVC_CXX:    addCXXStateStores(F, MMI); break;
+  case EHPersonality::MSVC_X86SEH: addSEHStateStores(F, MMI); break;
+  }
+
+  // Reset per-function state.
+  PersonalityFn = nullptr;
+  Personality = EHPersonality::Unknown;
+  return true;
+}
+
+/// Get the common EH registration subobject:
+///   typedef _EXCEPTION_DISPOSITION (*PEXCEPTION_ROUTINE)(
+///       _EXCEPTION_RECORD *, void *, _CONTEXT *, void *);
+///   struct EHRegistrationNode {
+///     EHRegistrationNode *Next;
+///     PEXCEPTION_ROUTINE Handler;
+///   };
+Type *WinEHStatePass::getEHLinkRegistrationType() {
+  if (EHLinkRegistrationTy)
+    return EHLinkRegistrationTy;
+  LLVMContext &Context = TheModule->getContext();
+  EHLinkRegistrationTy = StructType::create(Context, "EHRegistrationNode");
+  Type *FieldTys[] = {
+      EHLinkRegistrationTy->getPointerTo(0), // EHRegistrationNode *Next
+      Type::getInt8PtrTy(Context) // EXCEPTION_DISPOSITION (*Handler)(...)
+  };
+  EHLinkRegistrationTy->setBody(FieldTys, false);
+  return EHLinkRegistrationTy;
+}
+
+/// The __CxxFrameHandler3 registration node:
+///   struct CXXExceptionRegistration {
+///     void *SavedESP;
+///     EHRegistrationNode SubRecord;
+///     int32_t TryLevel;
+///   };
+Type *WinEHStatePass::getCXXEHRegistrationType() {
+  if (CXXEHRegistrationTy)
+    return CXXEHRegistrationTy;
+  LLVMContext &Context = TheModule->getContext();
+  Type *FieldTys[] = {
+      Type::getInt8PtrTy(Context), // void *SavedESP
+      getEHLinkRegistrationType(), // EHRegistrationNode SubRecord
+      Type::getInt32Ty(Context)    // int32_t TryLevel
+  };
+  CXXEHRegistrationTy =
+      StructType::create(FieldTys, "CXXExceptionRegistration");
+  return CXXEHRegistrationTy;
+}
+
+/// The _except_handler3/4 registration node:
+///   struct EH4ExceptionRegistration {
+///     void *SavedESP;
+///     _EXCEPTION_POINTERS *ExceptionPointers;
+///     EHRegistrationNode SubRecord;
+///     int32_t EncodedScopeTable;
+///     int32_t TryLevel;
+///   };
+Type *WinEHStatePass::getSEHRegistrationType() {
+  if (SEHRegistrationTy)
+    return SEHRegistrationTy;
+  LLVMContext &Context = TheModule->getContext();
+  Type *FieldTys[] = {
+      Type::getInt8PtrTy(Context), // void *SavedESP
+      Type::getInt8PtrTy(Context), // void *ExceptionPointers
+      getEHLinkRegistrationType(), // EHRegistrationNode SubRecord
+      Type::getInt32Ty(Context),   // int32_t EncodedScopeTable
+      Type::getInt32Ty(Context)    // int32_t TryLevel
+  };
+  SEHRegistrationTy = StructType::create(FieldTys, "SEHExceptionRegistration");
+  return SEHRegistrationTy;
+}
+
+// Emit an exception registration record. These are stack allocations with the
+// common subobject of two pointers: the previous registration record (the old
+// fs:00) and the personality function for the current frame. The data before
+// and after that is personality function specific.
+void WinEHStatePass::emitExceptionRegistrationRecord(Function *F) {
+  assert(Personality == EHPersonality::MSVC_CXX ||
+         Personality == EHPersonality::MSVC_X86SEH);
+
+  StringRef PersonalityName = PersonalityFn->getName();
+  IRBuilder<> Builder(&F->getEntryBlock(), F->getEntryBlock().begin());
+  Type *Int8PtrType = Builder.getInt8PtrTy();
+  if (Personality == EHPersonality::MSVC_CXX) {
+    RegNodeTy = getCXXEHRegistrationType();
+    RegNode = Builder.CreateAlloca(RegNodeTy);
+    // SavedESP = llvm.stacksave()
+    Value *SP = Builder.CreateCall(
+        Intrinsic::getDeclaration(TheModule, Intrinsic::stacksave), {});
+    Builder.CreateStore(SP, Builder.CreateStructGEP(RegNodeTy, RegNode, 0));
+    // TryLevel = -1
+    StateFieldIndex = 2;
+    insertStateNumberStore(RegNode, Builder.GetInsertPoint(), -1);
+    // Handler = __ehhandler$F
+    Function *Trampoline = generateLSDAInEAXThunk(F);
+    Link = Builder.CreateStructGEP(RegNodeTy, RegNode, 1);
+    linkExceptionRegistration(Builder, Trampoline);
+  } else if (Personality == EHPersonality::MSVC_X86SEH) {
+    // If _except_handler4 is in use, some additional guard checks and prologue
+    // stuff is required.
+    bool UseStackGuard = (PersonalityName == "_except_handler4");
+    RegNodeTy = getSEHRegistrationType();
+    RegNode = Builder.CreateAlloca(RegNodeTy);
+    // SavedESP = llvm.stacksave()
+    Value *SP = Builder.CreateCall(
+        Intrinsic::getDeclaration(TheModule, Intrinsic::stacksave), {});
+    Builder.CreateStore(SP, Builder.CreateStructGEP(RegNodeTy, RegNode, 0));
+    // TryLevel = -2 / -1
+    StateFieldIndex = 4;
+    insertStateNumberStore(RegNode, Builder.GetInsertPoint(),
+                           UseStackGuard ? -2 : -1);
+    // ScopeTable = llvm.x86.seh.lsda(F)
+    Value *FI8 = Builder.CreateBitCast(F, Int8PtrType);
+    Value *LSDA = Builder.CreateCall(
+        Intrinsic::getDeclaration(TheModule, Intrinsic::x86_seh_lsda), FI8);
+    Type *Int32Ty = Type::getInt32Ty(TheModule->getContext());
+    LSDA = Builder.CreatePtrToInt(LSDA, Int32Ty);
+    // If using _except_handler4, xor the address of the table with
+    // __security_cookie.
+    if (UseStackGuard) {
+      Value *Cookie =
+          TheModule->getOrInsertGlobal("__security_cookie", Int32Ty);
+      Value *Val = Builder.CreateLoad(Int32Ty, Cookie);
+      LSDA = Builder.CreateXor(LSDA, Val);
+    }
+    Builder.CreateStore(LSDA, Builder.CreateStructGEP(RegNodeTy, RegNode, 3));
+    Link = Builder.CreateStructGEP(RegNodeTy, RegNode, 2);
+    linkExceptionRegistration(Builder, PersonalityFn);
+  } else {
+    llvm_unreachable("unexpected personality function");
+  }
+
+  // Insert an unlink before all returns.
+  for (BasicBlock &BB : *F) {
+    TerminatorInst *T = BB.getTerminator();
+    if (!isa<ReturnInst>(T))
+      continue;
+    Builder.SetInsertPoint(T);
+    unlinkExceptionRegistration(Builder);
+  }
+}
+
+Value *WinEHStatePass::emitEHLSDA(IRBuilder<> &Builder, Function *F) {
+  Value *FI8 = Builder.CreateBitCast(F, Type::getInt8PtrTy(F->getContext()));
+  return Builder.CreateCall(
+      Intrinsic::getDeclaration(TheModule, Intrinsic::x86_seh_lsda), FI8);
+}
+
+/// Generate a thunk that puts the LSDA of ParentFunc in EAX and then calls
+/// PersonalityFn, forwarding the parameters passed to PEXCEPTION_ROUTINE:
+///   typedef _EXCEPTION_DISPOSITION (*PEXCEPTION_ROUTINE)(
+///       _EXCEPTION_RECORD *, void *, _CONTEXT *, void *);
+/// We essentially want this code:
+///   movl $lsda, %eax
+///   jmpl ___CxxFrameHandler3
+Function *WinEHStatePass::generateLSDAInEAXThunk(Function *ParentFunc) {
+  LLVMContext &Context = ParentFunc->getContext();
+  Type *Int32Ty = Type::getInt32Ty(Context);
+  Type *Int8PtrType = Type::getInt8PtrTy(Context);
+  Type *ArgTys[5] = {Int8PtrType, Int8PtrType, Int8PtrType, Int8PtrType,
+                     Int8PtrType};
+  FunctionType *TrampolineTy =
+      FunctionType::get(Int32Ty, makeArrayRef(&ArgTys[0], 4),
+                        /*isVarArg=*/false);
+  FunctionType *TargetFuncTy =
+      FunctionType::get(Int32Ty, makeArrayRef(&ArgTys[0], 5),
+                        /*isVarArg=*/false);
+  Function *Trampoline =
+      Function::Create(TrampolineTy, GlobalValue::InternalLinkage,
+                       Twine("__ehhandler$") + GlobalValue::getRealLinkageName(
+                                                   ParentFunc->getName()),
+                       TheModule);
+  BasicBlock *EntryBB = BasicBlock::Create(Context, "entry", Trampoline);
+  IRBuilder<> Builder(EntryBB);
+  Value *LSDA = emitEHLSDA(Builder, ParentFunc);
+  Value *CastPersonality =
+      Builder.CreateBitCast(PersonalityFn, TargetFuncTy->getPointerTo());
+  auto AI = Trampoline->arg_begin();
+  Value *Args[5] = {LSDA, AI++, AI++, AI++, AI++};
+  CallInst *Call = Builder.CreateCall(CastPersonality, Args);
+  // Can't use musttail due to prototype mismatch, but we can use tail.
+  Call->setTailCall(true);
+  // Set inreg so we pass it in EAX.
+  Call->addAttribute(1, Attribute::InReg);
+  Builder.CreateRet(Call);
+  return Trampoline;
+}
+
+void WinEHStatePass::linkExceptionRegistration(IRBuilder<> &Builder,
+                                               Function *Handler) {
+  // Emit the .safeseh directive for this function.
+  Handler->addFnAttr("safeseh");
+
+  Type *LinkTy = getEHLinkRegistrationType();
+  // Handler = Handler
+  Value *HandlerI8 = Builder.CreateBitCast(Handler, Builder.getInt8PtrTy());
+  Builder.CreateStore(HandlerI8, Builder.CreateStructGEP(LinkTy, Link, 1));
+  // Next = [fs:00]
+  Constant *FSZero =
+      Constant::getNullValue(LinkTy->getPointerTo()->getPointerTo(257));
+  Value *Next = Builder.CreateLoad(FSZero);
+  Builder.CreateStore(Next, Builder.CreateStructGEP(LinkTy, Link, 0));
+  // [fs:00] = Link
+  Builder.CreateStore(Link, FSZero);
+}
+
+void WinEHStatePass::unlinkExceptionRegistration(IRBuilder<> &Builder) {
+  // Clone Link into the current BB for better address mode folding.
+  if (auto *GEP = dyn_cast<GetElementPtrInst>(Link)) {
+    GEP = cast<GetElementPtrInst>(GEP->clone());
+    Builder.Insert(GEP);
+    Link = GEP;
+  }
+  Type *LinkTy = getEHLinkRegistrationType();
+  // [fs:00] = Link->Next
+  Value *Next =
+      Builder.CreateLoad(Builder.CreateStructGEP(LinkTy, Link, 0));
+  Constant *FSZero =
+      Constant::getNullValue(LinkTy->getPointerTo()->getPointerTo(257));
+  Builder.CreateStore(Next, FSZero);
+}
+
+void WinEHStatePass::addCXXStateStores(Function &F, MachineModuleInfo &MMI) {
+  WinEHFuncInfo &FuncInfo = MMI.getWinEHFuncInfo(&F);
+  calculateWinCXXEHStateNumbers(&F, FuncInfo);
+
+  // The base state for the parent is -1.
+  addCXXStateStoresToFunclet(RegNode, FuncInfo, F, -1);
+
+  // Set up RegNodeEscapeIndex
+  int RegNodeEscapeIndex = escapeRegNode(F);
+  FuncInfo.EHRegNodeEscapeIndex = RegNodeEscapeIndex;
+
+  // Only insert stores in catch handlers.
+  Constant *FI8 =
+      ConstantExpr::getBitCast(&F, Type::getInt8PtrTy(TheModule->getContext()));
+  for (auto P : FuncInfo.HandlerBaseState) {
+    Function *Handler = const_cast<Function *>(P.first);
+    int BaseState = P.second;
+    IRBuilder<> Builder(&Handler->getEntryBlock(),
+                        Handler->getEntryBlock().begin());
+    // FIXME: Find and reuse such a call if present.
+    Value *ParentFP = Builder.CreateCall(FrameAddress, {Builder.getInt32(1)});
+    Value *RecoveredRegNode = Builder.CreateCall(
+        FrameRecover, {FI8, ParentFP, Builder.getInt32(RegNodeEscapeIndex)});
+    RecoveredRegNode =
+        Builder.CreateBitCast(RecoveredRegNode, RegNodeTy->getPointerTo(0));
+    addCXXStateStoresToFunclet(RecoveredRegNode, FuncInfo, *Handler, BaseState);
+  }
+}
+
+/// Escape RegNode so that we can access it from child handlers. Find the call
+/// to localescape, if any, in the entry block and append RegNode to the list
+/// of arguments.
+int WinEHStatePass::escapeRegNode(Function &F) {
+  // Find the call to localescape and extract its arguments.
+  IntrinsicInst *EscapeCall = nullptr;
+  for (Instruction &I : F.getEntryBlock()) {
+    IntrinsicInst *II = dyn_cast<IntrinsicInst>(&I);
+    if (II && II->getIntrinsicID() == Intrinsic::localescape) {
+      EscapeCall = II;
+      break;
+    }
+  }
+  SmallVector<Value *, 8> Args;
+  if (EscapeCall) {
+    auto Ops = EscapeCall->arg_operands();
+    Args.append(Ops.begin(), Ops.end());
+  }
+  Args.push_back(RegNode);
+
+  // Replace the call (if it exists) with new one. Otherwise, insert at the end
+  // of the entry block.
+  Instruction *InsertPt = EscapeCall;
+  if (!EscapeCall)
+    InsertPt = F.getEntryBlock().getTerminator();
+  IRBuilder<> Builder(&F.getEntryBlock(), InsertPt);
+  Builder.CreateCall(FrameEscape, Args);
+  if (EscapeCall)
+    EscapeCall->eraseFromParent();
+  return Args.size() - 1;
+}
+
+void WinEHStatePass::addCXXStateStoresToFunclet(Value *ParentRegNode,
+                                                WinEHFuncInfo &FuncInfo,
+                                                Function &F, int BaseState) {
+  // Iterate all the instructions and emit state number stores.
+  for (BasicBlock &BB : F) {
+    for (Instruction &I : BB) {
+      if (auto *CI = dyn_cast<CallInst>(&I)) {
+        // Possibly throwing call instructions have no actions to take after
+        // an unwind. Ensure they are in the -1 state.
+        if (CI->doesNotThrow())
+          continue;
+        insertStateNumberStore(ParentRegNode, CI, BaseState);
+      } else if (auto *II = dyn_cast<InvokeInst>(&I)) {
+        // Look up the state number of the landingpad this unwinds to.
+        LandingPadInst *LPI = II->getUnwindDest()->getLandingPadInst();
+        // FIXME: Why does this assertion fail?
+        //assert(FuncInfo.LandingPadStateMap.count(LPI) && "LP has no state!");
+        int State = FuncInfo.LandingPadStateMap[LPI];
+        insertStateNumberStore(ParentRegNode, II, State);
+      }
+    }
+  }
+}
+
+/// Assign every distinct landingpad a unique state number for SEH. Unlike C++
+/// EH, we can use this very simple algorithm while C++ EH cannot because catch
+/// handlers aren't outlined and the runtime doesn't have to figure out which
+/// catch handler frame to unwind to.
+/// FIXME: __finally blocks are outlined, so this approach may break down there.
+void WinEHStatePass::addSEHStateStores(Function &F, MachineModuleInfo &MMI) {
+  WinEHFuncInfo &FuncInfo = MMI.getWinEHFuncInfo(&F);
+
+  // Remember and return the index that we used. We save it in WinEHFuncInfo so
+  // that we can lower llvm.x86.seh.recoverfp later in filter functions without
+  // too much trouble.
+  int RegNodeEscapeIndex = escapeRegNode(F);
+  FuncInfo.EHRegNodeEscapeIndex = RegNodeEscapeIndex;
+
+  // Iterate all the instructions and emit state number stores.
+  int CurState = 0;
+  SmallPtrSet<BasicBlock *, 4> ExceptBlocks;
+  for (BasicBlock &BB : F) {
+    for (auto I = BB.begin(), E = BB.end(); I != E; ++I) {
+      if (auto *CI = dyn_cast<CallInst>(I)) {
+        auto *Intrin = dyn_cast<IntrinsicInst>(CI);
+        if (Intrin) {
+          // Calls that "don't throw" are considered to be able to throw asynch
+          // exceptions, but intrinsics cannot.
+          continue;
+        }
+        insertStateNumberStore(RegNode, CI, -1);
+      } else if (auto *II = dyn_cast<InvokeInst>(I)) {
+        // Look up the state number of the landingpad this unwinds to.
+        LandingPadInst *LPI = II->getUnwindDest()->getLandingPadInst();
+        auto InsertionPair =
+            FuncInfo.LandingPadStateMap.insert(std::make_pair(LPI, CurState));
+        auto Iter = InsertionPair.first;
+        int &State = Iter->second;
+        bool Inserted = InsertionPair.second;
+        if (Inserted) {
+          // Each action consumes a state number.
+          auto *EHActions = cast<IntrinsicInst>(LPI->getNextNode());
+          SmallVector<std::unique_ptr<ActionHandler>, 4> ActionList;
+          parseEHActions(EHActions, ActionList);
+          assert(!ActionList.empty());
+          CurState += ActionList.size();
+          State += ActionList.size() - 1;
+
+          // Remember all the __except block targets.
+          for (auto &Handler : ActionList) {
+            if (auto *CH = dyn_cast<CatchHandler>(Handler.get())) {
+              auto *BA = cast<BlockAddress>(CH->getHandlerBlockOrFunc());
+#ifndef NDEBUG
+              for (BasicBlock *Pred : predecessors(BA->getBasicBlock()))
+                assert(Pred->isLandingPad() &&
+                       "WinEHPrepare failed to split block");
+#endif
+              ExceptBlocks.insert(BA->getBasicBlock());
+            }
+          }
+        }
+        insertStateNumberStore(RegNode, II, State);
+      }
+    }
+  }
+
+  // Insert llvm.x86.seh.restoreframe() into each __except block.
+  Function *RestoreFrame =
+      Intrinsic::getDeclaration(TheModule, Intrinsic::x86_seh_restoreframe);
+  for (BasicBlock *ExceptBB : ExceptBlocks) {
+    IRBuilder<> Builder(ExceptBB->begin());
+    Builder.CreateCall(RestoreFrame, {});
+  }
+}
+
+void WinEHStatePass::insertStateNumberStore(Value *ParentRegNode,
+                                            Instruction *IP, int State) {
+  IRBuilder<> Builder(IP);
+  Value *StateField =
+      Builder.CreateStructGEP(RegNodeTy, ParentRegNode, StateFieldIndex);
+  Builder.CreateStore(Builder.getInt32(State), StateField);
+}
diff --git a/lib/Target/XCore/Disassembler/XCoreDisassembler.cpp b/lib/Target/XCore/Disassembler/XCoreDisassembler.cpp
index 640e6b09d64..2e44ac949b2 100644
--- a/lib/Target/XCore/Disassembler/XCoreDisassembler.cpp
+++ b/lib/Target/XCore/Disassembler/XCoreDisassembler.cpp
@@ -204,7 +204,7 @@ static DecodeStatus DecodeGRRegsRegisterClass(MCInst &Inst,
   if (RegNo > 11)
     return MCDisassembler::Fail;
   unsigned Reg = getReg(Decoder, XCore::GRRegsRegClassID, RegNo);
-  Inst.addOperand(MCOperand::CreateReg(Reg));
+  Inst.addOperand(MCOperand::createReg(Reg));
   return MCDisassembler::Success;
 }
 
@@ -216,7 +216,7 @@ static DecodeStatus DecodeRRegsRegisterClass(MCInst &Inst,
   if (RegNo > 15)
     return MCDisassembler::Fail;
   unsigned Reg = getReg(Decoder, XCore::RRegsRegClassID, RegNo);
-  Inst.addOperand(MCOperand::CreateReg(Reg));
+  Inst.addOperand(MCOperand::createReg(Reg));
   return MCDisassembler::Success;
 }
 
@@ -227,13 +227,13 @@ static DecodeStatus DecodeBitpOperand(MCInst &Inst, unsigned Val,
   static unsigned Values[] = {
     32 /*bpw*/, 1, 2, 3, 4, 5, 6, 7, 8, 16, 24, 32
   };
-  Inst.addOperand(MCOperand::CreateImm(Values[Val]));
+  Inst.addOperand(MCOperand::createImm(Values[Val]));
   return MCDisassembler::Success;
 }
 
 static DecodeStatus DecodeNegImmOperand(MCInst &Inst, unsigned Val,
                                         uint64_t Address, const void *Decoder) {
-  Inst.addOperand(MCOperand::CreateImm(-(int64_t)Val));
+  Inst.addOperand(MCOperand::createImm(-(int64_t)Val));
   return MCDisassembler::Success;
 }
 
@@ -362,7 +362,7 @@ Decode2RImmInstruction(MCInst &Inst, unsigned Insn, uint64_t Address,
   if (S != MCDisassembler::Success)
     return Decode2OpInstructionFail(Inst, Insn, Address, Decoder);
 
-  Inst.addOperand(MCOperand::CreateImm(Op1));
+  Inst.addOperand(MCOperand::createImm(Op1));
   DecodeGRRegsRegisterClass(Inst, Op2, Address, Decoder);
   return S;
 }
@@ -403,7 +403,7 @@ DecodeRUSInstruction(MCInst &Inst, unsigned Insn, uint64_t Address,
     return Decode2OpInstructionFail(Inst, Insn, Address, Decoder);
 
   DecodeGRRegsRegisterClass(Inst, Op1, Address, Decoder);
-  Inst.addOperand(MCOperand::CreateImm(Op2));
+  Inst.addOperand(MCOperand::createImm(Op2));
   return S;
 }
 
@@ -552,7 +552,7 @@ Decode3RImmInstruction(MCInst &Inst, unsigned Insn, uint64_t Address,
   unsigned Op1, Op2, Op3;
   DecodeStatus S = Decode3OpInstruction(Insn, Op1, Op2, Op3);
   if (S == MCDisassembler::Success) {
-    Inst.addOperand(MCOperand::CreateImm(Op1));
+    Inst.addOperand(MCOperand::createImm(Op1));
     DecodeGRRegsRegisterClass(Inst, Op2, Address, Decoder);
     DecodeGRRegsRegisterClass(Inst, Op3, Address, Decoder);
   }
@@ -567,7 +567,7 @@ Decode2RUSInstruction(MCInst &Inst, unsigned Insn, uint64_t Address,
   if (S == MCDisassembler::Success) {
     DecodeGRRegsRegisterClass(Inst, Op1, Address, Decoder);
     DecodeGRRegsRegisterClass(Inst, Op2, Address, Decoder);
-    Inst.addOperand(MCOperand::CreateImm(Op3));
+    Inst.addOperand(MCOperand::createImm(Op3));
   }
   return S;
 }
@@ -623,7 +623,7 @@ DecodeL2RUSInstruction(MCInst &Inst, unsigned Insn, uint64_t Address,
   if (S == MCDisassembler::Success) {
     DecodeGRRegsRegisterClass(Inst, Op1, Address, Decoder);
     DecodeGRRegsRegisterClass(Inst, Op2, Address, Decoder);
-    Inst.addOperand(MCOperand::CreateImm(Op3));
+    Inst.addOperand(MCOperand::createImm(Op3));
   }
   return S;
 }
diff --git a/lib/Target/XCore/InstPrinter/XCoreInstPrinter.cpp b/lib/Target/XCore/InstPrinter/XCoreInstPrinter.cpp
index 36b3b02a707..500c84d2a41 100644
--- a/lib/Target/XCore/InstPrinter/XCoreInstPrinter.cpp
+++ b/lib/Target/XCore/InstPrinter/XCoreInstPrinter.cpp
@@ -45,7 +45,8 @@ printInlineJT32(const MCInst *MI, int opNum, raw_ostream &O) {
   report_fatal_error("can't handle InlineJT32");
 }
 
-static void printExpr(const MCExpr *Expr, raw_ostream &OS) {
+static void printExpr(const MCExpr *Expr, const MCAsmInfo *MAI,
+                      raw_ostream &OS) {
   int Offset = 0;
   const MCSymbolRefExpr *SRE;
 
@@ -60,7 +61,7 @@ static void printExpr(const MCExpr *Expr, raw_ostream &OS) {
   }
   assert(SRE->getKind() == MCSymbolRefExpr::VK_None);
 
-  OS << SRE->getSymbol();
+  SRE->getSymbol().print(OS, MAI);
 
   if (Offset) {
     if (Offset > 0)
@@ -83,5 +84,5 @@ printOperand(const MCInst *MI, unsigned OpNo, raw_ostream &O) {
   }
 
   assert(Op.isExpr() && "unknown operand kind in printOperand");
-  printExpr(Op.getExpr(), O);
+  printExpr(Op.getExpr(), &MAI, O);
 }
diff --git a/lib/Target/XCore/LLVMBuild.txt b/lib/Target/XCore/LLVMBuild.txt
index 0504e8ab8f0..401e0526f58 100644
--- a/lib/Target/XCore/LLVMBuild.txt
+++ b/lib/Target/XCore/LLVMBuild.txt
@@ -29,5 +29,17 @@ has_disassembler = 1
 type = Library
 name = XCoreCodeGen
 parent = XCore
-required_libraries = Analysis AsmPrinter CodeGen Core MC SelectionDAG Support Target TransformUtils XCoreAsmPrinter XCoreDesc XCoreInfo
+required_libraries =
+ Analysis
+ AsmPrinter
+ CodeGen
+ Core
+ MC
+ SelectionDAG
+ Support
+ Target
+ TransformUtils
+ XCoreAsmPrinter
+ XCoreDesc
+ XCoreInfo
 add_to_library_groups = XCore
diff --git a/lib/Target/XCore/MCTargetDesc/XCoreMCAsmInfo.cpp b/lib/Target/XCore/MCTargetDesc/XCoreMCAsmInfo.cpp
index f2d2b37d6f2..3178a4edbb3 100644
--- a/lib/Target/XCore/MCTargetDesc/XCoreMCAsmInfo.cpp
+++ b/lib/Target/XCore/MCTargetDesc/XCoreMCAsmInfo.cpp
@@ -8,12 +8,11 @@
 //===----------------------------------------------------------------------===//
 
 #include "XCoreMCAsmInfo.h"
-#include "llvm/ADT/StringRef.h"
 using namespace llvm;
 
 void XCoreMCAsmInfo::anchor() { }
 
-XCoreMCAsmInfo::XCoreMCAsmInfo(StringRef TT) {
+XCoreMCAsmInfo::XCoreMCAsmInfo(const Triple &TT) {
   SupportsDebugInformation = true;
   Data16bitsDirective = "\t.short\t";
   Data32bitsDirective = "\t.long\t";
diff --git a/lib/Target/XCore/MCTargetDesc/XCoreMCAsmInfo.h b/lib/Target/XCore/MCTargetDesc/XCoreMCAsmInfo.h
index 26df211eece..39581e424e8 100644
--- a/lib/Target/XCore/MCTargetDesc/XCoreMCAsmInfo.h
+++ b/lib/Target/XCore/MCTargetDesc/XCoreMCAsmInfo.h
@@ -17,14 +17,14 @@
 #include "llvm/MC/MCAsmInfoELF.h"
 
 namespace llvm {
-  class StringRef;
-  class Target;
+class Triple;
 
-  class XCoreMCAsmInfo : public MCAsmInfoELF {
-    void anchor() override;
-  public:
-    explicit XCoreMCAsmInfo(StringRef TT);
-  };
+class XCoreMCAsmInfo : public MCAsmInfoELF {
+  void anchor() override;
+
+public:
+  explicit XCoreMCAsmInfo(const Triple &TT);
+};
 
 } // namespace llvm
 
diff --git a/lib/Target/XCore/MCTargetDesc/XCoreMCTargetDesc.cpp b/lib/Target/XCore/MCTargetDesc/XCoreMCTargetDesc.cpp
index 4a790c8a60b..b4085835f28 100644
--- a/lib/Target/XCore/MCTargetDesc/XCoreMCTargetDesc.cpp
+++ b/lib/Target/XCore/MCTargetDesc/XCoreMCTargetDesc.cpp
@@ -40,21 +40,19 @@ static MCInstrInfo *createXCoreMCInstrInfo() {
   return X;
 }
 
-static MCRegisterInfo *createXCoreMCRegisterInfo(StringRef TT) {
+static MCRegisterInfo *createXCoreMCRegisterInfo(const Triple &TT) {
   MCRegisterInfo *X = new MCRegisterInfo();
   InitXCoreMCRegisterInfo(X, XCore::LR);
   return X;
 }
 
-static MCSubtargetInfo *createXCoreMCSubtargetInfo(StringRef TT, StringRef CPU,
-                                                   StringRef FS) {
-  MCSubtargetInfo *X = new MCSubtargetInfo();
-  InitXCoreMCSubtargetInfo(X, TT, CPU, FS);
-  return X;
+static MCSubtargetInfo *
+createXCoreMCSubtargetInfo(const Triple &TT, StringRef CPU, StringRef FS) {
+  return createXCoreMCSubtargetInfoImpl(TT, CPU, FS);
 }
 
 static MCAsmInfo *createXCoreMCAsmInfo(const MCRegisterInfo &MRI,
-                                       StringRef TT) {
+                                       const Triple &TT) {
   MCAsmInfo *MAI = new XCoreMCAsmInfo(TT);
 
   // Initial state of the frame pointer is SP.
@@ -64,7 +62,8 @@ static MCAsmInfo *createXCoreMCAsmInfo(const MCRegisterInfo &MRI,
   return MAI;
 }
 
-static MCCodeGenInfo *createXCoreMCCodeGenInfo(StringRef TT, Reloc::Model RM,
+static MCCodeGenInfo *createXCoreMCCodeGenInfo(const Triple &TT,
+                                               Reloc::Model RM,
                                                CodeModel::Model CM,
                                                CodeGenOpt::Level OL) {
   MCCodeGenInfo *X = new MCCodeGenInfo();
@@ -77,7 +76,7 @@ static MCCodeGenInfo *createXCoreMCCodeGenInfo(StringRef TT, Reloc::Model RM,
   if (CM != CodeModel::Small && CM != CodeModel::Large)
     report_fatal_error("Target only supports CodeModel Small or Large");
 
-  X->InitMCCodeGenInfo(RM, CM, OL);
+  X->initMCCodeGenInfo(RM, CM, OL);
   return X;
 }
 
diff --git a/lib/Target/XCore/TargetInfo/XCoreTargetInfo.cpp b/lib/Target/XCore/TargetInfo/XCoreTargetInfo.cpp
index 00e34e04fbe..c78cde9ac8a 100644
--- a/lib/Target/XCore/TargetInfo/XCoreTargetInfo.cpp
+++ b/lib/Target/XCore/TargetInfo/XCoreTargetInfo.cpp
@@ -14,6 +14,6 @@ using namespace llvm;
 
 Target llvm::TheXCoreTarget;
 
-extern "C" void LLVMInitializeXCoreTargetInfo() { 
+extern "C" void LLVMInitializeXCoreTargetInfo() {
   RegisterTarget<Triple::xcore> X(TheXCoreTarget, "xcore", "XCore");
 }
diff --git a/lib/Target/XCore/XCoreAsmPrinter.cpp b/lib/Target/XCore/XCoreAsmPrinter.cpp
index 4f7a7e95640..b00cdd5040e 100644
--- a/lib/Target/XCore/XCoreAsmPrinter.cpp
+++ b/lib/Target/XCore/XCoreAsmPrinter.cpp
@@ -37,7 +37,7 @@
 #include "llvm/MC/MCExpr.h"
 #include "llvm/MC/MCInst.h"
 #include "llvm/MC/MCStreamer.h"
-#include "llvm/MC/MCSymbol.h"
+#include "llvm/MC/MCSymbolELF.h"
 #include "llvm/Support/ErrorHandling.h"
 #include "llvm/Support/TargetRegistry.h"
 #include "llvm/Support/raw_ostream.h"
@@ -86,7 +86,7 @@ namespace {
 } // end of anonymous namespace
 
 XCoreTargetStreamer &XCoreAsmPrinter::getTargetStreamer() {
-  return static_cast<XCoreTargetStreamer&>(*OutStreamer.getTargetStreamer());
+  return static_cast<XCoreTargetStreamer&>(*OutStreamer->getTargetStreamer());
 }
 
 void XCoreAsmPrinter::emitArrayBound(MCSymbol *Sym, const GlobalVariable *GV) {
@@ -96,15 +96,15 @@ void XCoreAsmPrinter::emitArrayBound(MCSymbol *Sym, const GlobalVariable *GV) {
   if (ArrayType *ATy = dyn_cast<ArrayType>(
                         cast<PointerType>(GV->getType())->getElementType())) {
 
-    MCSymbol *SymGlob = OutContext.GetOrCreateSymbol(
+    MCSymbol *SymGlob = OutContext.getOrCreateSymbol(
                           Twine(Sym->getName() + StringRef(".globound")));
-    OutStreamer.EmitSymbolAttribute(SymGlob, MCSA_Global);
-    OutStreamer.EmitAssignment(SymGlob,
-                               MCConstantExpr::Create(ATy->getNumElements(),
-                                                      OutContext));
+    OutStreamer->EmitSymbolAttribute(SymGlob, MCSA_Global);
+    OutStreamer->EmitAssignment(SymGlob,
+                                MCConstantExpr::create(ATy->getNumElements(),
+                                                       OutContext));
     if (GV->hasWeakLinkage() || GV->hasLinkOnceLinkage() ||
         GV->hasCommonLinkage()) {
-      OutStreamer.EmitSymbolAttribute(SymGlob, MCSA_Weak);
+      OutStreamer->EmitSymbolAttribute(SymGlob, MCSA_Weak);
     }
   }
 }
@@ -115,14 +115,14 @@ void XCoreAsmPrinter::EmitGlobalVariable(const GlobalVariable *GV) {
       EmitSpecialLLVMGlobal(GV))
     return;
 
-  const DataLayout *TD = TM.getDataLayout();
-  OutStreamer.SwitchSection(
+  const DataLayout &DL = getDataLayout();
+  OutStreamer->SwitchSection(
       getObjFileLowering().SectionForGlobal(GV, *Mang, TM));
 
   MCSymbol *GVSym = getSymbol(GV);
   const Constant *C = GV->getInitializer();
-  unsigned Align = (unsigned)TD->getPreferredTypeAlignmentShift(C->getType());
-  
+  unsigned Align = (unsigned)DL.getPreferredTypeAlignmentShift(C->getType());
+
   // Mark the start of the global
   getTargetStreamer().emitCCTopData(GVSym->getName());
 
@@ -136,11 +136,11 @@ void XCoreAsmPrinter::EmitGlobalVariable(const GlobalVariable *GV) {
   case GlobalValue::ExternalLinkage:
   case GlobalValue::CommonLinkage:
     emitArrayBound(GVSym, GV);
-    OutStreamer.EmitSymbolAttribute(GVSym, MCSA_Global);
+    OutStreamer->EmitSymbolAttribute(GVSym, MCSA_Global);
 
     if (GV->hasWeakLinkage() || GV->hasLinkOnceLinkage() ||
         GV->hasCommonLinkage())
-      OutStreamer.EmitSymbolAttribute(GVSym, MCSA_Weak);
+      OutStreamer->EmitSymbolAttribute(GVSym, MCSA_Weak);
     // FALL THROUGH
   case GlobalValue::InternalLinkage:
   case GlobalValue::PrivateLinkage:
@@ -154,18 +154,19 @@ void XCoreAsmPrinter::EmitGlobalVariable(const GlobalVariable *GV) {
   if (GV->isThreadLocal()) {
     report_fatal_error("TLS is not supported by this target!");
   }
-  unsigned Size = TD->getTypeAllocSize(C->getType());
+  unsigned Size = DL.getTypeAllocSize(C->getType());
   if (MAI->hasDotTypeDotSizeDirective()) {
-    OutStreamer.EmitSymbolAttribute(GVSym, MCSA_ELF_TypeObject);
-    OutStreamer.EmitELFSize(GVSym, MCConstantExpr::Create(Size, OutContext));
+    OutStreamer->EmitSymbolAttribute(GVSym, MCSA_ELF_TypeObject);
+    OutStreamer->emitELFSize(cast<MCSymbolELF>(GVSym),
+                             MCConstantExpr::create(Size, OutContext));
   }
-  OutStreamer.EmitLabel(GVSym);
-  
-  EmitGlobalConstant(C);
+  OutStreamer->EmitLabel(GVSym);
+
+  EmitGlobalConstant(DL, C);
   // The ABI requires that unsigned scalar types smaller than 32 bits
   // are padded to 32 bits.
   if (Size < 4)
-    OutStreamer.EmitZeros(4 - Size);
+    OutStreamer->EmitZeros(4 - Size);
   
   // Mark the end of the global
   getTargetStreamer().emitCCBottomData(GVSym->getName());
@@ -185,7 +186,7 @@ void XCoreAsmPrinter::EmitFunctionBodyEnd() {
 void XCoreAsmPrinter::EmitFunctionEntryLabel() {
   // Mark the start of the function
   getTargetStreamer().emitCCTopFunction(CurrentFnSym->getName());
-  OutStreamer.EmitLabel(CurrentFnSym);
+  OutStreamer->EmitLabel(CurrentFnSym);
 }
 
 void XCoreAsmPrinter::
@@ -201,13 +202,13 @@ printInlineJT(const MachineInstr *MI, int opNum, raw_ostream &O,
     MachineBasicBlock *MBB = JTBBs[i];
     if (i > 0)
       O << ",";
-    O << *MBB->getSymbol();
+    MBB->getSymbol()->print(O, MAI);
   }
 }
 
 void XCoreAsmPrinter::printOperand(const MachineInstr *MI, int opNum,
                                    raw_ostream &O) {
-  const DataLayout *DL = TM.getDataLayout();
+  const DataLayout &DL = getDataLayout();
   const MachineOperand &MO = MI->getOperand(opNum);
   switch (MO.getType()) {
   case MachineOperand::MO_Register:
@@ -217,17 +218,17 @@ void XCoreAsmPrinter::printOperand(const MachineInstr *MI, int opNum,
     O << MO.getImm();
     break;
   case MachineOperand::MO_MachineBasicBlock:
-    O << *MO.getMBB()->getSymbol();
+    MO.getMBB()->getSymbol()->print(O, MAI);
     break;
   case MachineOperand::MO_GlobalAddress:
-    O << *getSymbol(MO.getGlobal());
+    getSymbol(MO.getGlobal())->print(O, MAI);
     break;
   case MachineOperand::MO_ConstantPoolIndex:
-    O << DL->getPrivateGlobalPrefix() << "CPI" << getFunctionNumber()
-      << '_' << MO.getIndex();
+    O << DL.getPrivateGlobalPrefix() << "CPI" << getFunctionNumber() << '_'
+      << MO.getIndex();
     break;
   case MachineOperand::MO_BlockAddress:
-    O << *GetBlockAddressSymbol(MO.getBlockAddress());
+    GetBlockAddressSymbol(MO.getBlockAddress())->print(O, MAI);
     break;
   default:
     llvm_unreachable("not implemented");
@@ -275,7 +276,7 @@ void XCoreAsmPrinter::EmitInstruction(const MachineInstr *MI) {
       O << "\tmov "
         << XCoreInstPrinter::getRegisterName(MI->getOperand(0).getReg()) << ", "
         << XCoreInstPrinter::getRegisterName(MI->getOperand(1).getReg());
-      OutStreamer.EmitRawText(O.str());
+      OutStreamer->EmitRawText(O.str());
       return;
     }
     break;
@@ -288,14 +289,14 @@ void XCoreAsmPrinter::EmitInstruction(const MachineInstr *MI) {
     else
       printInlineJT32(MI, 0, O);
     O << '\n';
-    OutStreamer.EmitRawText(O.str());
+    OutStreamer->EmitRawText(O.str());
     return;
   }
 
   MCInst TmpInst;
   MCInstLowering.Lower(MI, TmpInst);
 
-  EmitToStreamer(OutStreamer, TmpInst);
+  EmitToStreamer(*OutStreamer, TmpInst);
 }
 
 // Force static initialization.
diff --git a/lib/Target/XCore/XCoreFrameLowering.cpp b/lib/Target/XCore/XCoreFrameLowering.cpp
index e0ac0e56781..76c3d8130e7 100644
--- a/lib/Target/XCore/XCoreFrameLowering.cpp
+++ b/lib/Target/XCore/XCoreFrameLowering.cpp
@@ -220,8 +220,9 @@ bool XCoreFrameLowering::hasFP(const MachineFunction &MF) const {
          MF.getFrameInfo()->hasVarSizedObjects();
 }
 
-void XCoreFrameLowering::emitPrologue(MachineFunction &MF) const {
-  MachineBasicBlock &MBB = MF.front();   // Prolog goes in entry BB
+void XCoreFrameLowering::emitPrologue(MachineFunction &MF,
+                                      MachineBasicBlock &MBB) const {
+  assert(&MF.front() == &MBB && "Shrink-wrapping not yet supported");
   MachineBasicBlock::iterator MBBI = MBB.begin();
   MachineFrameInfo *MFI = MF.getFrameInfo();
   MachineModuleInfo *MMI = &MF.getMMI();
@@ -524,12 +525,15 @@ eliminateCallFramePseudoInstr(MachineFunction &MF, MachineBasicBlock &MBB,
   MBB.erase(I);
 }
 
-void XCoreFrameLowering::
-processFunctionBeforeCalleeSavedScan(MachineFunction &MF,
-                                     RegScavenger *RS) const {
+void XCoreFrameLowering::determineCalleeSaves(MachineFunction &MF,
+                                              BitVector &SavedRegs,
+                                              RegScavenger *RS) const {
+  TargetFrameLowering::determineCalleeSaves(MF, SavedRegs, RS);
+
   XCoreFunctionInfo *XFI = MF.getInfo<XCoreFunctionInfo>();
 
-  bool LRUsed = MF.getRegInfo().isPhysRegUsed(XCore::LR);
+  const MachineRegisterInfo &MRI = MF.getRegInfo();
+  bool LRUsed = MRI.isPhysRegModified(XCore::LR);
 
   if (!LRUsed && !MF.getFunction()->isVarArg() &&
       MF.getFrameInfo()->estimateStackSize(MF))
@@ -549,7 +553,7 @@ processFunctionBeforeCalleeSavedScan(MachineFunction &MF,
   if (LRUsed) {
     // We will handle the LR in the prologue/epilogue
     // and allocate space on the stack ourselves.
-    MF.getRegInfo().setPhysRegUnused(XCore::LR);
+    SavedRegs.reset(XCore::LR);
     XFI->createLRSpillSlot(MF);
   }
 
diff --git a/lib/Target/XCore/XCoreFrameLowering.h b/lib/Target/XCore/XCoreFrameLowering.h
index 7b169c2b8cd..69c71adc8d3 100644
--- a/lib/Target/XCore/XCoreFrameLowering.h
+++ b/lib/Target/XCore/XCoreFrameLowering.h
@@ -27,7 +27,8 @@ namespace llvm {
 
     /// emitProlog/emitEpilog - These methods insert prolog and epilog code into
     /// the function.
-    void emitPrologue(MachineFunction &MF) const override;
+    void emitPrologue(MachineFunction &MF,
+                      MachineBasicBlock &MBB) const override;
     void emitEpilogue(MachineFunction &MF,
                       MachineBasicBlock &MBB) const override;
 
@@ -46,8 +47,8 @@ namespace llvm {
 
     bool hasFP(const MachineFunction &MF) const override;
 
-    void processFunctionBeforeCalleeSavedScan(MachineFunction &MF,
-                                     RegScavenger *RS = nullptr) const override;
+    void determineCalleeSaves(MachineFunction &MF, BitVector &SavedRegs,
+                              RegScavenger *RS = nullptr) const override;
 
     void processFunctionBeforeFrameFinalized(MachineFunction &MF,
                                      RegScavenger *RS = nullptr) const override;
diff --git a/lib/Target/XCore/XCoreISelDAGToDAG.cpp b/lib/Target/XCore/XCoreISelDAGToDAG.cpp
index 5c7ea5e3640..9d4a966dfba 100644
--- a/lib/Target/XCore/XCoreISelDAGToDAG.cpp
+++ b/lib/Target/XCore/XCoreISelDAGToDAG.cpp
@@ -47,8 +47,8 @@ namespace {
 
     /// getI32Imm - Return a target constant with the specified value, of type
     /// i32.
-    inline SDValue getI32Imm(unsigned Imm) {
-      return CurDAG->getTargetConstant(Imm, MVT::i32);
+    inline SDValue getI32Imm(unsigned Imm, SDLoc dl) {
+      return CurDAG->getTargetConstant(Imm, dl, MVT::i32);
     }
 
     inline bool immMskBitp(SDNode *inN) const {
@@ -90,7 +90,7 @@ bool XCoreDAGToDAGISel::SelectADDRspii(SDValue Addr, SDValue &Base,
   FrameIndexSDNode *FIN = nullptr;
   if ((FIN = dyn_cast<FrameIndexSDNode>(Addr))) {
     Base = CurDAG->getTargetFrameIndex(FIN->getIndex(), MVT::i32);
-    Offset = CurDAG->getTargetConstant(0, MVT::i32);
+    Offset = CurDAG->getTargetConstant(0, SDLoc(Addr), MVT::i32);
     return true;
   }
   if (Addr.getOpcode() == ISD::ADD) {
@@ -100,7 +100,8 @@ bool XCoreDAGToDAGISel::SelectADDRspii(SDValue Addr, SDValue &Base,
       && (CN->getSExtValue() % 4 == 0 && CN->getSExtValue() >= 0)) {
       // Constant positive word offset from frame index
       Base = CurDAG->getTargetFrameIndex(FIN->getIndex(), MVT::i32);
-      Offset = CurDAG->getTargetConstant(CN->getSExtValue(), MVT::i32);
+      Offset = CurDAG->getTargetConstant(CN->getSExtValue(), SDLoc(Addr),
+                                         MVT::i32);
       return true;
     }
   }
@@ -138,15 +139,14 @@ SDNode *XCoreDAGToDAGISel::Select(SDNode *N) {
     if (immMskBitp(N)) {
       // Transformation function: get the size of a mask
       // Look for the first non-zero bit
-      SDValue MskSize = getI32Imm(32 - countLeadingZeros((uint32_t)Val));
+      SDValue MskSize = getI32Imm(32 - countLeadingZeros((uint32_t)Val), dl);
       return CurDAG->getMachineNode(XCore::MKMSK_rus, dl,
                                     MVT::i32, MskSize);
     }
     else if (!isUInt<16>(Val)) {
-      SDValue CPIdx =
-        CurDAG->getTargetConstantPool(ConstantInt::get(
-                              Type::getInt32Ty(*CurDAG->getContext()), Val),
-                                      getTargetLowering()->getPointerTy());
+      SDValue CPIdx = CurDAG->getTargetConstantPool(
+          ConstantInt::get(Type::getInt32Ty(*CurDAG->getContext()), Val),
+          getTargetLowering()->getPointerTy(CurDAG->getDataLayout()));
       SDNode *node = CurDAG->getMachineNode(XCore::LDWCP_lru6, dl, MVT::i32,
                                             MVT::Other, CPIdx,
                                             CurDAG->getEntryNode());
@@ -256,7 +256,7 @@ SDNode *XCoreDAGToDAGISel::SelectBRIND(SDNode *N) {
   // after with clrsr 1. If any resources owned by the thread are ready an event
   // will be taken. If no resource is ready we branch to the address which was
   // the operand to the checkevent intrinsic.
-  SDValue constOne = getI32Imm(1);
+  SDValue constOne = getI32Imm(1, dl);
   SDValue Glue =
     SDValue(CurDAG->getMachineNode(XCore::SETSR_branch_u6, dl, MVT::Glue,
                                    constOne, Chain), 0);
diff --git a/lib/Target/XCore/XCoreISelLowering.cpp b/lib/Target/XCore/XCoreISelLowering.cpp
index c4e3bb8da6d..032df42cd9e 100644
--- a/lib/Target/XCore/XCoreISelLowering.cpp
+++ b/lib/Target/XCore/XCoreISelLowering.cpp
@@ -44,8 +44,9 @@ using namespace llvm;
 const char *XCoreTargetLowering::
 getTargetNodeName(unsigned Opcode) const
 {
-  switch (Opcode)
+  switch ((XCoreISD::NodeType)Opcode)
   {
+    case XCoreISD::FIRST_NUMBER      : break;
     case XCoreISD::BL                : return "XCoreISD::BL";
     case XCoreISD::PCRelativeWrapper : return "XCoreISD::PCRelativeWrapper";
     case XCoreISD::DPRelativeWrapper : return "XCoreISD::DPRelativeWrapper";
@@ -64,8 +65,8 @@ getTargetNodeName(unsigned Opcode) const
     case XCoreISD::FRAME_TO_ARGS_OFFSET : return "XCoreISD::FRAME_TO_ARGS_OFFSET";
     case XCoreISD::EH_RETURN         : return "XCoreISD::EH_RETURN";
     case XCoreISD::MEMBARRIER        : return "XCoreISD::MEMBARRIER";
-    default                          : return nullptr;
   }
+  return nullptr;
 }
 
 XCoreTargetLowering::XCoreTargetLowering(const TargetMachine &TM,
@@ -280,7 +281,8 @@ static bool IsSmallObject(const GlobalValue *GV, const XCoreTargetLowering &XTL)
   if (!ObjType->isSized())
     return false;
 
-  unsigned ObjSize = XTL.getDataLayout()->getTypeAllocSize(ObjType);
+  auto &DL = GV->getParent()->getDataLayout();
+  unsigned ObjSize = DL.getTypeAllocSize(ObjType);
   return ObjSize < CodeModelLargeSize && ObjSize != 0;
 }
 
@@ -298,7 +300,7 @@ LowerGlobalAddress(SDValue Op, SelectionDAG &DAG) const
     GA = getGlobalAddressWrapper(GA, GV, DAG);
     // Handle the rest of the offset.
     if (Offset != FoldedOffset) {
-      SDValue Remaining = DAG.getConstant(Offset - FoldedOffset, MVT::i32);
+      SDValue Remaining = DAG.getConstant(Offset - FoldedOffset, DL, MVT::i32);
       GA = DAG.getNode(ISD::ADD, DL, MVT::i32, GA, Remaining);
     }
     return GA;
@@ -311,8 +313,9 @@ LowerGlobalAddress(SDValue Op, SelectionDAG &DAG) const
     Constant *GAI = ConstantExpr::getGetElementPtr(
         Type::getInt8Ty(*DAG.getContext()), GA, Idx);
     SDValue CP = DAG.getConstantPool(GAI, MVT::i32);
-    return DAG.getLoad(getPointerTy(), DL, DAG.getEntryNode(), CP,
-                       MachinePointerInfo(), false, false, false, 0);
+    return DAG.getLoad(getPointerTy(DAG.getDataLayout()), DL,
+                       DAG.getEntryNode(), CP, MachinePointerInfo(), false,
+                       false, false, 0);
   }
 }
 
@@ -320,11 +323,11 @@ SDValue XCoreTargetLowering::
 LowerBlockAddress(SDValue Op, SelectionDAG &DAG) const
 {
   SDLoc DL(Op);
-
+  auto PtrVT = getPointerTy(DAG.getDataLayout());
   const BlockAddress *BA = cast<BlockAddressSDNode>(Op)->getBlockAddress();
-  SDValue Result = DAG.getTargetBlockAddress(BA, getPointerTy());
+  SDValue Result = DAG.getTargetBlockAddress(BA, PtrVT);
 
-  return DAG.getNode(XCoreISD::PCRelativeWrapper, DL, getPointerTy(), Result);
+  return DAG.getNode(XCoreISD::PCRelativeWrapper, DL, PtrVT, Result);
 }
 
 SDValue XCoreTargetLowering::
@@ -368,7 +371,7 @@ LowerBR_JT(SDValue Op, SelectionDAG &DAG) const
   }
   assert((NumEntries >> 31) == 0);
   SDValue ScaledIndex = DAG.getNode(ISD::SHL, dl, MVT::i32, Index,
-                                    DAG.getConstant(1, MVT::i32));
+                                    DAG.getConstant(1, dl, MVT::i32));
   return DAG.getNode(XCoreISD::BR_JT32, dl, MVT::Other, Chain, TargetJT,
                      ScaledIndex);
 }
@@ -377,9 +380,10 @@ SDValue XCoreTargetLowering::
 lowerLoadWordFromAlignedBasePlusOffset(SDLoc DL, SDValue Chain, SDValue Base,
                                        int64_t Offset, SelectionDAG &DAG) const
 {
+  auto PtrVT = getPointerTy(DAG.getDataLayout());
   if ((Offset & 0x3) == 0) {
-    return DAG.getLoad(getPointerTy(), DL, Chain, Base, MachinePointerInfo(),
-                       false, false, false, 0);
+    return DAG.getLoad(PtrVT, DL, Chain, Base, MachinePointerInfo(), false,
+                       false, false, 0);
   }
   // Lower to pair of consecutive word aligned loads plus some bit shifting.
   int32_t HighOffset = RoundUpToAlignment(Offset, 4);
@@ -393,18 +397,16 @@ lowerLoadWordFromAlignedBasePlusOffset(SDLoc DL, SDValue Chain, SDValue Base,
                                     HighOffset);
   } else {
     LowAddr = DAG.getNode(ISD::ADD, DL, MVT::i32, Base,
-                          DAG.getConstant(LowOffset, MVT::i32));
+                          DAG.getConstant(LowOffset, DL, MVT::i32));
     HighAddr = DAG.getNode(ISD::ADD, DL, MVT::i32, Base,
-                           DAG.getConstant(HighOffset, MVT::i32));
+                           DAG.getConstant(HighOffset, DL, MVT::i32));
   }
-  SDValue LowShift = DAG.getConstant((Offset - LowOffset) * 8, MVT::i32);
-  SDValue HighShift = DAG.getConstant((HighOffset - Offset) * 8, MVT::i32);
+  SDValue LowShift = DAG.getConstant((Offset - LowOffset) * 8, DL, MVT::i32);
+  SDValue HighShift = DAG.getConstant((HighOffset - Offset) * 8, DL, MVT::i32);
 
-  SDValue Low = DAG.getLoad(getPointerTy(), DL, Chain,
-                            LowAddr, MachinePointerInfo(),
+  SDValue Low = DAG.getLoad(PtrVT, DL, Chain, LowAddr, MachinePointerInfo(),
                             false, false, false, 0);
-  SDValue High = DAG.getLoad(getPointerTy(), DL, Chain,
-                             HighAddr, MachinePointerInfo(),
+  SDValue High = DAG.getLoad(PtrVT, DL, Chain, HighAddr, MachinePointerInfo(),
                              false, false, false, 0);
   SDValue LowShifted = DAG.getNode(ISD::SRL, DL, MVT::i32, Low, LowShift);
   SDValue HighShifted = DAG.getNode(ISD::SHL, DL, MVT::i32, High, HighShift);
@@ -434,8 +436,9 @@ LowerLOAD(SDValue Op, SelectionDAG &DAG) const {
                                      LD->getAlignment()))
     return SDValue();
 
-  unsigned ABIAlignment = getDataLayout()->
-    getABITypeAlignment(LD->getMemoryVT().getTypeForEVT(*DAG.getContext()));
+  auto &TD = DAG.getDataLayout();
+  unsigned ABIAlignment = TD.getABITypeAlignment(
+      LD->getMemoryVT().getTypeForEVT(*DAG.getContext()));
   // Leave aligned load alone.
   if (LD->getAlignment() >= ABIAlignment)
     return SDValue();
@@ -469,14 +472,14 @@ LowerLOAD(SDValue Op, SelectionDAG &DAG) const {
                                  LD->isVolatile(), LD->isNonTemporal(),
                                  LD->isInvariant(), 2);
     SDValue HighAddr = DAG.getNode(ISD::ADD, DL, MVT::i32, BasePtr,
-                                   DAG.getConstant(2, MVT::i32));
+                                   DAG.getConstant(2, DL, MVT::i32));
     SDValue High = DAG.getExtLoad(ISD::EXTLOAD, DL, MVT::i32, Chain,
                                   HighAddr,
                                   LD->getPointerInfo().getWithOffset(2),
                                   MVT::i16, LD->isVolatile(),
                                   LD->isNonTemporal(), LD->isInvariant(), 2);
     SDValue HighShifted = DAG.getNode(ISD::SHL, DL, MVT::i32, High,
-                                      DAG.getConstant(16, MVT::i32));
+                                      DAG.getConstant(16, DL, MVT::i32));
     SDValue Result = DAG.getNode(ISD::OR, DL, MVT::i32, Low, HighShifted);
     Chain = DAG.getNode(ISD::TokenFactor, DL, MVT::Other, Low.getValue(1),
                              High.getValue(1));
@@ -485,7 +488,7 @@ LowerLOAD(SDValue Op, SelectionDAG &DAG) const {
   }
 
   // Lower to a call to __misaligned_load(BasePtr).
-  Type *IntPtrTy = getDataLayout()->getIntPtrType(*DAG.getContext());
+  Type *IntPtrTy = TD.getIntPtrType(*DAG.getContext());
   TargetLowering::ArgListTy Args;
   TargetLowering::ArgListEntry Entry;
 
@@ -494,10 +497,11 @@ LowerLOAD(SDValue Op, SelectionDAG &DAG) const {
   Args.push_back(Entry);
 
   TargetLowering::CallLoweringInfo CLI(DAG);
-  CLI.setDebugLoc(DL).setChain(Chain)
-    .setCallee(CallingConv::C, IntPtrTy,
-               DAG.getExternalSymbol("__misaligned_load", getPointerTy()),
-               std::move(Args), 0);
+  CLI.setDebugLoc(DL).setChain(Chain).setCallee(
+      CallingConv::C, IntPtrTy,
+      DAG.getExternalSymbol("__misaligned_load",
+                            getPointerTy(DAG.getDataLayout())),
+      std::move(Args), 0);
 
   std::pair<SDValue, SDValue> CallResult = LowerCallTo(CLI);
   SDValue Ops[] = { CallResult.first, CallResult.second };
@@ -515,8 +519,8 @@ LowerSTORE(SDValue Op, SelectionDAG &DAG) const
                                      ST->getAlignment())) {
     return SDValue();
   }
-  unsigned ABIAlignment = getDataLayout()->
-    getABITypeAlignment(ST->getMemoryVT().getTypeForEVT(*DAG.getContext()));
+  unsigned ABIAlignment = DAG.getDataLayout().getABITypeAlignment(
+      ST->getMemoryVT().getTypeForEVT(*DAG.getContext()));
   // Leave aligned store alone.
   if (ST->getAlignment() >= ABIAlignment) {
     return SDValue();
@@ -529,13 +533,13 @@ LowerSTORE(SDValue Op, SelectionDAG &DAG) const
   if (ST->getAlignment() == 2) {
     SDValue Low = Value;
     SDValue High = DAG.getNode(ISD::SRL, dl, MVT::i32, Value,
-                                      DAG.getConstant(16, MVT::i32));
+                                      DAG.getConstant(16, dl, MVT::i32));
     SDValue StoreLow = DAG.getTruncStore(Chain, dl, Low, BasePtr,
                                          ST->getPointerInfo(), MVT::i16,
                                          ST->isVolatile(), ST->isNonTemporal(),
                                          2);
     SDValue HighAddr = DAG.getNode(ISD::ADD, dl, MVT::i32, BasePtr,
-                                   DAG.getConstant(2, MVT::i32));
+                                   DAG.getConstant(2, dl, MVT::i32));
     SDValue StoreHigh = DAG.getTruncStore(Chain, dl, High, HighAddr,
                                           ST->getPointerInfo().getWithOffset(2),
                                           MVT::i16, ST->isVolatile(),
@@ -544,7 +548,7 @@ LowerSTORE(SDValue Op, SelectionDAG &DAG) const
   }
 
   // Lower to a call to __misaligned_store(BasePtr, Value).
-  Type *IntPtrTy = getDataLayout()->getIntPtrType(*DAG.getContext());
+  Type *IntPtrTy = DAG.getDataLayout().getIntPtrType(*DAG.getContext());
   TargetLowering::ArgListTy Args;
   TargetLowering::ArgListEntry Entry;
 
@@ -556,10 +560,11 @@ LowerSTORE(SDValue Op, SelectionDAG &DAG) const
   Args.push_back(Entry);
 
   TargetLowering::CallLoweringInfo CLI(DAG);
-  CLI.setDebugLoc(dl).setChain(Chain)
-    .setCallee(CallingConv::C, Type::getVoidTy(*DAG.getContext()),
-               DAG.getExternalSymbol("__misaligned_store", getPointerTy()),
-               std::move(Args), 0);
+  CLI.setDebugLoc(dl).setChain(Chain).setCallee(
+      CallingConv::C, Type::getVoidTy(*DAG.getContext()),
+      DAG.getExternalSymbol("__misaligned_store",
+                            getPointerTy(DAG.getDataLayout())),
+      std::move(Args), 0);
 
   std::pair<SDValue, SDValue> CallResult = LowerCallTo(CLI);
   return CallResult.second;
@@ -573,7 +578,7 @@ LowerSMUL_LOHI(SDValue Op, SelectionDAG &DAG) const
   SDLoc dl(Op);
   SDValue LHS = Op.getOperand(0);
   SDValue RHS = Op.getOperand(1);
-  SDValue Zero = DAG.getConstant(0, MVT::i32);
+  SDValue Zero = DAG.getConstant(0, dl, MVT::i32);
   SDValue Hi = DAG.getNode(XCoreISD::MACCS, dl,
                            DAG.getVTList(MVT::i32, MVT::i32), Zero, Zero,
                            LHS, RHS);
@@ -590,7 +595,7 @@ LowerUMUL_LOHI(SDValue Op, SelectionDAG &DAG) const
   SDLoc dl(Op);
   SDValue LHS = Op.getOperand(0);
   SDValue RHS = Op.getOperand(1);
-  SDValue Zero = DAG.getConstant(0, MVT::i32);
+  SDValue Zero = DAG.getConstant(0, dl, MVT::i32);
   SDValue Hi = DAG.getNode(XCoreISD::LMUL, dl,
                            DAG.getVTList(MVT::i32, MVT::i32), LHS, RHS,
                            Zero, Zero);
@@ -675,13 +680,13 @@ TryExpandADDWithMul(SDNode *N, SelectionDAG &DAG) const
   SDLoc dl(N);
   SDValue LL, RL, AddendL, AddendH;
   LL = DAG.getNode(ISD::EXTRACT_ELEMENT, dl, MVT::i32,
-                   Mul.getOperand(0),  DAG.getConstant(0, MVT::i32));
+                   Mul.getOperand(0), DAG.getConstant(0, dl, MVT::i32));
   RL = DAG.getNode(ISD::EXTRACT_ELEMENT, dl, MVT::i32,
-                   Mul.getOperand(1),  DAG.getConstant(0, MVT::i32));
+                   Mul.getOperand(1), DAG.getConstant(0, dl, MVT::i32));
   AddendL = DAG.getNode(ISD::EXTRACT_ELEMENT, dl, MVT::i32,
-                        Other,  DAG.getConstant(0, MVT::i32));
+                        Other, DAG.getConstant(0, dl, MVT::i32));
   AddendH = DAG.getNode(ISD::EXTRACT_ELEMENT, dl, MVT::i32,
-                        Other,  DAG.getConstant(1, MVT::i32));
+                        Other, DAG.getConstant(1, dl, MVT::i32));
   APInt HighMask = APInt::getHighBitsSet(64, 32);
   unsigned LHSSB = DAG.ComputeNumSignBits(Mul.getOperand(0));
   unsigned RHSSB = DAG.ComputeNumSignBits(Mul.getOperand(1));
@@ -704,9 +709,9 @@ TryExpandADDWithMul(SDNode *N, SelectionDAG &DAG) const
   }
   SDValue LH, RH;
   LH = DAG.getNode(ISD::EXTRACT_ELEMENT, dl, MVT::i32,
-                   Mul.getOperand(0),  DAG.getConstant(1, MVT::i32));
+                   Mul.getOperand(0), DAG.getConstant(1, dl, MVT::i32));
   RH = DAG.getNode(ISD::EXTRACT_ELEMENT, dl, MVT::i32,
-                   Mul.getOperand(1),  DAG.getConstant(1, MVT::i32));
+                   Mul.getOperand(1), DAG.getConstant(1, dl, MVT::i32));
   SDValue Hi = DAG.getNode(XCoreISD::MACCU, dl,
                            DAG.getVTList(MVT::i32, MVT::i32), AddendH,
                            AddendL, LL, RL);
@@ -735,18 +740,22 @@ ExpandADDSUB(SDNode *N, SelectionDAG &DAG) const
 
   // Extract components
   SDValue LHSL = DAG.getNode(ISD::EXTRACT_ELEMENT, dl, MVT::i32,
-                            N->getOperand(0),  DAG.getConstant(0, MVT::i32));
+                             N->getOperand(0),
+                             DAG.getConstant(0, dl, MVT::i32));
   SDValue LHSH = DAG.getNode(ISD::EXTRACT_ELEMENT, dl, MVT::i32,
-                            N->getOperand(0),  DAG.getConstant(1, MVT::i32));
+                             N->getOperand(0),
+                             DAG.getConstant(1, dl, MVT::i32));
   SDValue RHSL = DAG.getNode(ISD::EXTRACT_ELEMENT, dl, MVT::i32,
-                             N->getOperand(1), DAG.getConstant(0, MVT::i32));
+                             N->getOperand(1),
+                             DAG.getConstant(0, dl, MVT::i32));
   SDValue RHSH = DAG.getNode(ISD::EXTRACT_ELEMENT, dl, MVT::i32,
-                             N->getOperand(1), DAG.getConstant(1, MVT::i32));
+                             N->getOperand(1),
+                             DAG.getConstant(1, dl, MVT::i32));
 
   // Expand
   unsigned Opcode = (N->getOpcode() == ISD::ADD) ? XCoreISD::LADD :
                                                    XCoreISD::LSUB;
-  SDValue Zero = DAG.getConstant(0, MVT::i32);
+  SDValue Zero = DAG.getConstant(0, dl, MVT::i32);
   SDValue Lo = DAG.getNode(Opcode, dl, DAG.getVTList(MVT::i32, MVT::i32),
                            LHSL, RHSL, Zero);
   SDValue Carry(Lo.getNode(), 1);
@@ -775,7 +784,8 @@ LowerVAARG(SDValue Op, SelectionDAG &DAG) const
                                false, false, false, 0);
   // Increment the pointer, VAList, to the next vararg
   SDValue nextPtr = DAG.getNode(ISD::ADD, dl, PtrVT, VAList,
-                                DAG.getIntPtrConstant(VT.getSizeInBits() / 8));
+                                DAG.getIntPtrConstant(VT.getSizeInBits() / 8,
+                                                      dl));
   // Store the incremented VAList to the legalized pointer
   InChain = DAG.getStore(VAList.getValue(1), dl, nextPtr, VAListPtr,
                          MachinePointerInfo(SV), false, false, 0);
@@ -827,9 +837,9 @@ LowerRETURNADDR(SDValue Op, SelectionDAG &DAG) const {
   XCoreFunctionInfo *XFI = MF.getInfo<XCoreFunctionInfo>();
   int FI = XFI->createLRSpillSlot(MF);
   SDValue FIN = DAG.getFrameIndex(FI, MVT::i32);
-  return DAG.getLoad(getPointerTy(), SDLoc(Op), DAG.getEntryNode(), FIN,
-                     MachinePointerInfo::getFixedStack(FI), false, false,
-                     false, 0);
+  return DAG.getLoad(
+      getPointerTy(DAG.getDataLayout()), SDLoc(Op), DAG.getEntryNode(), FIN,
+      MachinePointerInfo::getFixedStack(FI), false, false, false, 0);
 }
 
 SDValue XCoreTargetLowering::
@@ -910,30 +920,30 @@ LowerINIT_TRAMPOLINE(SDValue Op, SelectionDAG &DAG) const {
   SDValue Addr = Trmp;
 
   SDLoc dl(Op);
-  OutChains[0] = DAG.getStore(Chain, dl, DAG.getConstant(0x0a3cd805, MVT::i32),
-                              Addr, MachinePointerInfo(TrmpAddr), false, false,
-                              0);
+  OutChains[0] = DAG.getStore(Chain, dl,
+                              DAG.getConstant(0x0a3cd805, dl, MVT::i32), Addr,
+                              MachinePointerInfo(TrmpAddr), false, false, 0);
 
   Addr = DAG.getNode(ISD::ADD, dl, MVT::i32, Trmp,
-                     DAG.getConstant(4, MVT::i32));
-  OutChains[1] = DAG.getStore(Chain, dl, DAG.getConstant(0xd80456c0, MVT::i32),
-                              Addr, MachinePointerInfo(TrmpAddr, 4), false,
-                              false, 0);
+                     DAG.getConstant(4, dl, MVT::i32));
+  OutChains[1] = DAG.getStore(Chain, dl,
+                              DAG.getConstant(0xd80456c0, dl, MVT::i32), Addr,
+                              MachinePointerInfo(TrmpAddr, 4), false, false, 0);
 
   Addr = DAG.getNode(ISD::ADD, dl, MVT::i32, Trmp,
-                     DAG.getConstant(8, MVT::i32));
-  OutChains[2] = DAG.getStore(Chain, dl, DAG.getConstant(0x27fb0a3c, MVT::i32),
-                              Addr, MachinePointerInfo(TrmpAddr, 8), false,
-                              false, 0);
+                     DAG.getConstant(8, dl, MVT::i32));
+  OutChains[2] = DAG.getStore(Chain, dl,
+                              DAG.getConstant(0x27fb0a3c, dl, MVT::i32), Addr,
+                              MachinePointerInfo(TrmpAddr, 8), false, false, 0);
 
   Addr = DAG.getNode(ISD::ADD, dl, MVT::i32, Trmp,
-                     DAG.getConstant(12, MVT::i32));
+                     DAG.getConstant(12, dl, MVT::i32));
   OutChains[3] = DAG.getStore(Chain, dl, Nest, Addr,
                               MachinePointerInfo(TrmpAddr, 12), false, false,
                               0);
 
   Addr = DAG.getNode(ISD::ADD, dl, MVT::i32, Trmp,
-                     DAG.getConstant(16, MVT::i32));
+                     DAG.getConstant(16, dl, MVT::i32));
   OutChains[4] = DAG.getStore(Chain, dl, FPtr, Addr,
                               MachinePointerInfo(TrmpAddr, 16), false, false,
                               0);
@@ -973,11 +983,10 @@ LowerATOMIC_LOAD(SDValue Op, SelectionDAG &DAG) const {
   if (N->getMemoryVT() == MVT::i32) {
     if (N->getAlignment() < 4)
       report_fatal_error("atomic load must be aligned");
-    return DAG.getLoad(getPointerTy(), SDLoc(Op), N->getChain(),
-                       N->getBasePtr(), N->getPointerInfo(),
-                       N->isVolatile(), N->isNonTemporal(),
-                       N->isInvariant(), N->getAlignment(),
-                       N->getAAInfo(), N->getRanges());
+    return DAG.getLoad(getPointerTy(DAG.getDataLayout()), SDLoc(Op),
+                       N->getChain(), N->getBasePtr(), N->getPointerInfo(),
+                       N->isVolatile(), N->isNonTemporal(), N->isInvariant(),
+                       N->getAlignment(), N->getAAInfo(), N->getRanges());
   }
   if (N->getMemoryVT() == MVT::i16) {
     if (N->getAlignment() < 2)
@@ -1096,7 +1105,7 @@ LowerCallResult(SDValue Chain, SDValue InFlag,
     int offset = ResultMemLocs[i].first;
     unsigned index = ResultMemLocs[i].second;
     SDVTList VTs = DAG.getVTList(MVT::i32, MVT::Other);
-    SDValue Ops[] = { Chain, DAG.getConstant(offset / 4, MVT::i32) };
+    SDValue Ops[] = { Chain, DAG.getConstant(offset / 4, dl, MVT::i32) };
     SDValue load = DAG.getNode(XCoreISD::LDWSP, dl, VTs, Ops);
     InVals[index] = load;
     MemOpChains.push_back(load.getValue(1));
@@ -1144,9 +1153,10 @@ XCoreTargetLowering::LowerCCCCallTo(SDValue Chain, SDValue Callee,
 
   // Get a count of how many bytes are to be pushed on the stack.
   unsigned NumBytes = RetCCInfo.getNextStackOffset();
+  auto PtrVT = getPointerTy(DAG.getDataLayout());
 
-  Chain = DAG.getCALLSEQ_START(Chain,DAG.getConstant(NumBytes,
-                                 getPointerTy(), true), dl);
+  Chain = DAG.getCALLSEQ_START(Chain,
+                               DAG.getConstant(NumBytes, dl, PtrVT, true), dl);
 
   SmallVector<std::pair<unsigned, SDValue>, 4> RegsToPass;
   SmallVector<SDValue, 12> MemOpChains;
@@ -1182,7 +1192,8 @@ XCoreTargetLowering::LowerCCCCallTo(SDValue Chain, SDValue Callee,
 
       MemOpChains.push_back(DAG.getNode(XCoreISD::STWSP, dl, MVT::Other,
                                         Chain, Arg,
-                                        DAG.getConstant(Offset/4, MVT::i32)));
+                                        DAG.getConstant(Offset/4, dl,
+                                                        MVT::i32)));
     }
   }
 
@@ -1232,10 +1243,8 @@ XCoreTargetLowering::LowerCCCCallTo(SDValue Chain, SDValue Callee,
   InFlag = Chain.getValue(1);
 
   // Create the CALLSEQ_END node.
-  Chain = DAG.getCALLSEQ_END(Chain,
-                             DAG.getConstant(NumBytes, getPointerTy(), true),
-                             DAG.getConstant(0, getPointerTy(), true),
-                             InFlag, dl);
+  Chain = DAG.getCALLSEQ_END(Chain, DAG.getConstant(NumBytes, dl, PtrVT, true),
+                             DAG.getConstant(0, dl, PtrVT, true), InFlag, dl);
   InFlag = Chain.getValue(1);
 
   // Handle result values, copying them out of physregs into vregs that we
@@ -1422,7 +1431,7 @@ XCoreTargetLowering::LowerCCCArguments(SDValue Chain,
       SDValue FIN = DAG.getFrameIndex(FI, MVT::i32);
       InVals.push_back(FIN);
       MemOps.push_back(DAG.getMemcpy(Chain, dl, FIN, ArgDI->SDV,
-                                     DAG.getConstant(Size, MVT::i32),
+                                     DAG.getConstant(Size, dl, MVT::i32),
                                      Align, false, false, false,
                                      MachinePointerInfo(),
                                      MachinePointerInfo()));
@@ -1487,7 +1496,7 @@ XCoreTargetLowering::LowerReturn(SDValue Chain,
   SmallVector<SDValue, 4> RetOps(1, Chain);
 
   // Return on XCore is always a "retsp 0"
-  RetOps.push_back(DAG.getConstant(0, MVT::i32));
+  RetOps.push_back(DAG.getConstant(0, dl, MVT::i32));
 
   SmallVector<SDValue, 4> MemOpChains;
   // Handle return values that must be copied to memory.
@@ -1671,9 +1680,9 @@ SDValue XCoreTargetLowering::PerformDAGCombine(SDNode *N,
 
     // fold (ladd 0, 0, x) -> 0, x & 1
     if (N0C && N0C->isNullValue() && N1C && N1C->isNullValue()) {
-      SDValue Carry = DAG.getConstant(0, VT);
+      SDValue Carry = DAG.getConstant(0, dl, VT);
       SDValue Result = DAG.getNode(ISD::AND, dl, VT, N2,
-                                   DAG.getConstant(1, VT));
+                                   DAG.getConstant(1, dl, VT));
       SDValue Ops[] = { Result, Carry };
       return DAG.getMergeValues(Ops, dl);
     }
@@ -1686,7 +1695,7 @@ SDValue XCoreTargetLowering::PerformDAGCombine(SDNode *N,
                                          VT.getSizeInBits() - 1);
       DAG.computeKnownBits(N2, KnownZero, KnownOne);
       if ((KnownZero & Mask) == Mask) {
-        SDValue Carry = DAG.getConstant(0, VT);
+        SDValue Carry = DAG.getConstant(0, dl, VT);
         SDValue Result = DAG.getNode(ISD::ADD, dl, VT, N0, N2);
         SDValue Ops[] = { Result, Carry };
         return DAG.getMergeValues(Ops, dl);
@@ -1711,7 +1720,7 @@ SDValue XCoreTargetLowering::PerformDAGCombine(SDNode *N,
       if ((KnownZero & Mask) == Mask) {
         SDValue Borrow = N2;
         SDValue Result = DAG.getNode(ISD::SUB, dl, VT,
-                                     DAG.getConstant(0, VT), N2);
+                                     DAG.getConstant(0, dl, VT), N2);
         SDValue Ops[] = { Result, Borrow };
         return DAG.getMergeValues(Ops, dl);
       }
@@ -1725,7 +1734,7 @@ SDValue XCoreTargetLowering::PerformDAGCombine(SDNode *N,
                                          VT.getSizeInBits() - 1);
       DAG.computeKnownBits(N2, KnownZero, KnownOne);
       if ((KnownZero & Mask) == Mask) {
-        SDValue Borrow = DAG.getConstant(0, VT);
+        SDValue Borrow = DAG.getConstant(0, dl, VT);
         SDValue Result = DAG.getNode(ISD::SUB, dl, VT, N0, N2);
         SDValue Ops[] = { Result, Borrow };
         return DAG.getMergeValues(Ops, dl);
@@ -1791,13 +1800,13 @@ SDValue XCoreTargetLowering::PerformDAGCombine(SDNode *N,
         DAG.MaskedValueIsZero(Addend0, HighMask) &&
         DAG.MaskedValueIsZero(Addend1, HighMask)) {
       SDValue Mul0L = DAG.getNode(ISD::EXTRACT_ELEMENT, dl, MVT::i32,
-                                  Mul0, DAG.getConstant(0, MVT::i32));
+                                  Mul0, DAG.getConstant(0, dl, MVT::i32));
       SDValue Mul1L = DAG.getNode(ISD::EXTRACT_ELEMENT, dl, MVT::i32,
-                                  Mul1, DAG.getConstant(0, MVT::i32));
+                                  Mul1, DAG.getConstant(0, dl, MVT::i32));
       SDValue Addend0L = DAG.getNode(ISD::EXTRACT_ELEMENT, dl, MVT::i32,
-                                     Addend0, DAG.getConstant(0, MVT::i32));
+                                     Addend0, DAG.getConstant(0, dl, MVT::i32));
       SDValue Addend1L = DAG.getNode(ISD::EXTRACT_ELEMENT, dl, MVT::i32,
-                                     Addend1, DAG.getConstant(0, MVT::i32));
+                                     Addend1, DAG.getConstant(0, dl, MVT::i32));
       SDValue Hi = DAG.getNode(XCoreISD::LMUL, dl,
                                DAG.getVTList(MVT::i32, MVT::i32), Mul0L, Mul1L,
                                Addend0L, Addend1L);
@@ -1819,10 +1828,9 @@ SDValue XCoreTargetLowering::PerformDAGCombine(SDNode *N,
     SDValue Chain = ST->getChain();
 
     unsigned StoreBits = ST->getMemoryVT().getStoreSizeInBits();
-    if (StoreBits % 8) {
-      break;
-    }
-    unsigned ABIAlignment = getDataLayout()->getABITypeAlignment(
+    assert((StoreBits % 8) == 0 &&
+           "Store size in bits must be a multiple of 8");
+    unsigned ABIAlignment = DAG.getDataLayout().getABITypeAlignment(
         ST->getMemoryVT().getTypeForEVT(*DCI.DAG.getContext()));
     unsigned Alignment = ST->getAlignment();
     if (Alignment >= ABIAlignment) {
@@ -1837,7 +1845,7 @@ SDValue XCoreTargetLowering::PerformDAGCombine(SDNode *N,
         bool isTail = isInTailCallPosition(DAG, ST, Chain);
         return DAG.getMemmove(Chain, dl, ST->getBasePtr(),
                               LD->getBasePtr(),
-                              DAG.getConstant(StoreBits/8, MVT::i32),
+                              DAG.getConstant(StoreBits/8, dl, MVT::i32),
                               Alignment, false, isTail, ST->getPointerInfo(),
                               LD->getPointerInfo());
       }
@@ -1916,14 +1924,13 @@ static inline bool isImmUs4(int64_t val)
 
 /// isLegalAddressingMode - Return true if the addressing mode represented
 /// by AM is legal for this target, for a load/store of the specified type.
-bool
-XCoreTargetLowering::isLegalAddressingMode(const AddrMode &AM,
-                                              Type *Ty) const {
+bool XCoreTargetLowering::isLegalAddressingMode(const DataLayout &DL,
+                                                const AddrMode &AM, Type *Ty,
+                                                unsigned AS) const {
   if (Ty->getTypeID() == Type::VoidTyID)
     return AM.Scale == 0 && isImmUs(AM.BaseOffs) && isImmUs4(AM.BaseOffs);
 
-  const DataLayout *TD = TM.getDataLayout();
-  unsigned Size = TD->getTypeAllocSize(Ty);
+  unsigned Size = DL.getTypeAllocSize(Ty);
   if (AM.BaseGV) {
     return Size >= 4 && !AM.HasBaseReg && AM.Scale == 0 &&
                  AM.BaseOffs%4 == 0;
@@ -1961,7 +1968,7 @@ XCoreTargetLowering::isLegalAddressingMode(const AddrMode &AM,
 
 std::pair<unsigned, const TargetRegisterClass *>
 XCoreTargetLowering::getRegForInlineAsmConstraint(const TargetRegisterInfo *TRI,
-                                                  const std::string &Constraint,
+                                                  StringRef Constraint,
                                                   MVT VT) const {
   if (Constraint.size() == 1) {
     switch (Constraint[0]) {
diff --git a/lib/Target/XCore/XCoreISelLowering.h b/lib/Target/XCore/XCoreISelLowering.h
index b20fc014ccd..ddd675c5164 100644
--- a/lib/Target/XCore/XCoreISelLowering.h
+++ b/lib/Target/XCore/XCoreISelLowering.h
@@ -26,7 +26,7 @@ namespace llvm {
   class XCoreTargetMachine;
 
   namespace XCoreISD {
-    enum NodeType {
+    enum NodeType : unsigned {
       // Start the numbering where the builtin ops and target ops leave off.
       FIRST_NUMBER = ISD::BUILTIN_OP_END,
 
@@ -101,7 +101,9 @@ namespace llvm {
 
 
     unsigned getJumpTableEncoding() const override;
-    MVT getScalarShiftAmountTy(EVT LHSTy) const override { return MVT::i32; }
+    MVT getScalarShiftAmountTy(const DataLayout &DL, EVT) const override {
+      return MVT::i32;
+    }
 
     /// LowerOperation - Provide custom lowering hooks for some operations.
     SDValue LowerOperation(SDValue Op, SelectionDAG &DAG) const override;
@@ -120,7 +122,8 @@ namespace llvm {
       EmitInstrWithCustomInserter(MachineInstr *MI,
                                   MachineBasicBlock *MBB) const override;
 
-    bool isLegalAddressingMode(const AddrMode &AM, Type *Ty) const override;
+    bool isLegalAddressingMode(const DataLayout &DL, const AddrMode &AM,
+                               Type *Ty, unsigned AS) const override;
 
   private:
     const TargetMachine &TM;
@@ -174,14 +177,7 @@ namespace llvm {
     // Inline asm support
     std::pair<unsigned, const TargetRegisterClass *>
     getRegForInlineAsmConstraint(const TargetRegisterInfo *TRI,
-                                 const std::string &Constraint,
-                                 MVT VT) const override;
-
-    unsigned getInlineAsmMemConstraint(
-        const std::string &ConstraintCode) const override {
-      // FIXME: Map different constraints differently.
-      return InlineAsm::Constraint_m;
-    }
+                                 StringRef Constraint, MVT VT) const override;
 
     // Expand specifics
     SDValue TryExpandADDWithMul(SDNode *Op, SelectionDAG &DAG) const;
diff --git a/lib/Target/XCore/XCoreInstrInfo.cpp b/lib/Target/XCore/XCoreInstrInfo.cpp
index c310aa3a179..ee30344dcc2 100644
--- a/lib/Target/XCore/XCoreInstrInfo.cpp
+++ b/lib/Target/XCore/XCoreInstrInfo.cpp
@@ -196,15 +196,10 @@ XCoreInstrInfo::AnalyzeBranch(MachineBasicBlock &MBB, MachineBasicBlock *&TBB,
                               SmallVectorImpl<MachineOperand> &Cond,
                               bool AllowModify) const {
   // If the block has no terminators, it just falls into the block after it.
-  MachineBasicBlock::iterator I = MBB.end();
-  if (I == MBB.begin())
+  MachineBasicBlock::iterator I = MBB.getLastNonDebugInstr();
+  if (I == MBB.end())
     return false;
-  --I;
-  while (I->isDebugValue()) {
-    if (I == MBB.begin())
-      return false;
-    --I;
-  }
+
   if (!isUnpredicatedTerminator(I))
     return false;
 
@@ -281,7 +276,7 @@ XCoreInstrInfo::AnalyzeBranch(MachineBasicBlock &MBB, MachineBasicBlock *&TBB,
 unsigned
 XCoreInstrInfo::InsertBranch(MachineBasicBlock &MBB,MachineBasicBlock *TBB,
                              MachineBasicBlock *FBB,
-                             const SmallVectorImpl<MachineOperand> &Cond,
+                             ArrayRef<MachineOperand> Cond,
                              DebugLoc DL)const{
   // Shouldn't be a fall through.
   assert(TBB && "InsertBranch must not be told to insert a fallthrough");
@@ -312,14 +307,10 @@ XCoreInstrInfo::InsertBranch(MachineBasicBlock &MBB,MachineBasicBlock *TBB,
 
 unsigned
 XCoreInstrInfo::RemoveBranch(MachineBasicBlock &MBB) const {
-  MachineBasicBlock::iterator I = MBB.end();
-  if (I == MBB.begin()) return 0;
-  --I;
-  while (I->isDebugValue()) {
-    if (I == MBB.begin())
-      return 0;
-    --I;
-  }
+  MachineBasicBlock::iterator I = MBB.getLastNonDebugInstr();
+  if (I == MBB.end())
+    return 0;
+
   if (!IsBRU(I->getOpcode()) && !IsCondBranch(I->getOpcode()))
     return 0;
   
diff --git a/lib/Target/XCore/XCoreInstrInfo.h b/lib/Target/XCore/XCoreInstrInfo.h
index 60bb3f8c39a..b958c361f5a 100644
--- a/lib/Target/XCore/XCoreInstrInfo.h
+++ b/lib/Target/XCore/XCoreInstrInfo.h
@@ -56,8 +56,7 @@ class XCoreInstrInfo : public XCoreGenInstrInfo {
                      bool AllowModify) const override;
 
   unsigned InsertBranch(MachineBasicBlock &MBB, MachineBasicBlock *TBB,
-                        MachineBasicBlock *FBB,
-                        const SmallVectorImpl<MachineOperand> &Cond,
+                        MachineBasicBlock *FBB, ArrayRef<MachineOperand> Cond,
                         DebugLoc DL) const override;
 
   unsigned RemoveBranch(MachineBasicBlock &MBB) const override;
diff --git a/lib/Target/XCore/XCoreInstrInfo.td b/lib/Target/XCore/XCoreInstrInfo.td
index 8e9bb452560..8110b918c22 100644
--- a/lib/Target/XCore/XCoreInstrInfo.td
+++ b/lib/Target/XCore/XCoreInstrInfo.td
@@ -94,33 +94,34 @@ def XCoreMemBarrier : SDNode<"XCoreISD::MEMBARRIER", SDT_XCoreMEMBARRIER,
 def div4_xform : SDNodeXForm<imm, [{
   // Transformation function: imm/4
   assert(N->getZExtValue() % 4 == 0);
-  return getI32Imm(N->getZExtValue()/4);
+  return getI32Imm(N->getZExtValue()/4, SDLoc(N));
 }]>;
 
 def msksize_xform : SDNodeXForm<imm, [{
   // Transformation function: get the size of a mask
   assert(isMask_32(N->getZExtValue()));
   // look for the first non-zero bit
-  return getI32Imm(32 - countLeadingZeros((uint32_t)N->getZExtValue()));
+  return getI32Imm(32 - countLeadingZeros((uint32_t)N->getZExtValue()),
+                   SDLoc(N));
 }]>;
 
 def neg_xform : SDNodeXForm<imm, [{
   // Transformation function: -imm
   uint32_t value = N->getZExtValue();
-  return getI32Imm(-value);
+  return getI32Imm(-value, SDLoc(N));
 }]>;
 
 def bpwsub_xform : SDNodeXForm<imm, [{
   // Transformation function: 32-imm
   uint32_t value = N->getZExtValue();
-  return getI32Imm(32-value);
+  return getI32Imm(32 - value, SDLoc(N));
 }]>;
 
 def div4neg_xform : SDNodeXForm<imm, [{
   // Transformation function: -imm/4
   uint32_t value = N->getZExtValue();
   assert(-value % 4 == 0);
-  return getI32Imm(-value/4);
+  return getI32Imm(-value/4, SDLoc(N));
 }]>;
 
 def immUs4Neg : PatLeaf<(imm), [{
diff --git a/lib/Target/XCore/XCoreLowerThreadLocal.cpp b/lib/Target/XCore/XCoreLowerThreadLocal.cpp
index 9fb63e990a1..996c6f59346 100644
--- a/lib/Target/XCore/XCoreLowerThreadLocal.cpp
+++ b/lib/Target/XCore/XCoreLowerThreadLocal.cpp
@@ -209,7 +209,7 @@ bool XCoreLowerThreadLocal::lowerGlobal(GlobalVariable *GV) {
     IRBuilder<> Builder(Inst);
     Function *GetID = Intrinsic::getDeclaration(GV->getParent(),
                                                 Intrinsic::xcore_getid);
-    Value *ThreadID = Builder.CreateCall(GetID);
+    Value *ThreadID = Builder.CreateCall(GetID, {});
     SmallVector<Value *, 2> Indices;
     Indices.push_back(Constant::getNullValue(Type::getInt64Ty(Ctx)));
     Indices.push_back(ThreadID);
diff --git a/lib/Target/XCore/XCoreMCInstLower.cpp b/lib/Target/XCore/XCoreMCInstLower.cpp
index dfdadcf3fe5..03c5fa2e9c4 100644
--- a/lib/Target/XCore/XCoreMCInstLower.cpp
+++ b/lib/Target/XCore/XCoreMCInstLower.cpp
@@ -65,17 +65,17 @@ MCOperand XCoreMCInstLower::LowerSymbolOperand(const MachineOperand &MO,
       llvm_unreachable("<unknown operand type>");
   }
 
-  const MCSymbolRefExpr *MCSym = MCSymbolRefExpr::Create(Symbol, Kind, *Ctx);
+  const MCSymbolRefExpr *MCSym = MCSymbolRefExpr::create(Symbol, Kind, *Ctx);
 
   if (!Offset)
-    return MCOperand::CreateExpr(MCSym);
+    return MCOperand::createExpr(MCSym);
 
   // Assume offset is never negative.
   assert(Offset > 0);
 
-  const MCConstantExpr *OffsetExpr =  MCConstantExpr::Create(Offset, *Ctx);
-  const MCBinaryExpr *Add = MCBinaryExpr::CreateAdd(MCSym, OffsetExpr, *Ctx);
-  return MCOperand::CreateExpr(Add);
+  const MCConstantExpr *OffsetExpr =  MCConstantExpr::create(Offset, *Ctx);
+  const MCBinaryExpr *Add = MCBinaryExpr::createAdd(MCSym, OffsetExpr, *Ctx);
+  return MCOperand::createExpr(Add);
 }
 
 MCOperand XCoreMCInstLower::LowerOperand(const MachineOperand &MO,
@@ -87,9 +87,9 @@ MCOperand XCoreMCInstLower::LowerOperand(const MachineOperand &MO,
     case MachineOperand::MO_Register:
       // Ignore all implicit register operands.
       if (MO.isImplicit()) break;
-      return MCOperand::CreateReg(MO.getReg());
+      return MCOperand::createReg(MO.getReg());
     case MachineOperand::MO_Immediate:
-      return MCOperand::CreateImm(MO.getImm() + offset);
+      return MCOperand::createImm(MO.getImm() + offset);
     case MachineOperand::MO_MachineBasicBlock:
     case MachineOperand::MO_GlobalAddress:
     case MachineOperand::MO_ExternalSymbol:
diff --git a/lib/Target/XCore/XCoreMachineFunctionInfo.cpp b/lib/Target/XCore/XCoreMachineFunctionInfo.cpp
index 9ef9752d0a5..6c770969e32 100644
--- a/lib/Target/XCore/XCoreMachineFunctionInfo.cpp
+++ b/lib/Target/XCore/XCoreMachineFunctionInfo.cpp
@@ -1,4 +1,4 @@
-//===-- XCoreMachineFuctionInfo.cpp - XCore machine function info ---------===//
+//===-- XCoreMachineFunctionInfo.cpp - XCore machine function info --------===//
 //
 //                     The LLVM Compiler Infrastructure
 //
diff --git a/lib/Target/XCore/XCoreMachineFunctionInfo.h b/lib/Target/XCore/XCoreMachineFunctionInfo.h
index 078ffde18fb..cdcc52fdc32 100644
--- a/lib/Target/XCore/XCoreMachineFunctionInfo.h
+++ b/lib/Target/XCore/XCoreMachineFunctionInfo.h
@@ -1,4 +1,4 @@
-//===-- XCoreMachineFuctionInfo.h - XCore machine function info -*- C++ -*-===//
+//===- XCoreMachineFunctionInfo.h - XCore machine function info -*- C++ -*-===//
 //
 //                     The LLVM Compiler Infrastructure
 //
diff --git a/lib/Target/XCore/XCoreRegisterInfo.cpp b/lib/Target/XCore/XCoreRegisterInfo.cpp
index 1d569e8936d..1cfb57dc3af 100644
--- a/lib/Target/XCore/XCoreRegisterInfo.cpp
+++ b/lib/Target/XCore/XCoreRegisterInfo.cpp
@@ -222,7 +222,7 @@ XCoreRegisterInfo::getCalleeSavedRegs(const MachineFunction *MF) const {
     XCore::R8, XCore::R9,
     0
   };
-  const TargetFrameLowering *TFI = MF->getSubtarget().getFrameLowering();
+  const XCoreFrameLowering *TFI = getFrameLowering(*MF);
   if (TFI->hasFP(*MF))
     return CalleeSavedRegsFP;
   return CalleeSavedRegs;
@@ -230,7 +230,7 @@ XCoreRegisterInfo::getCalleeSavedRegs(const MachineFunction *MF) const {
 
 BitVector XCoreRegisterInfo::getReservedRegs(const MachineFunction &MF) const {
   BitVector Reserved(getNumRegs());
-  const TargetFrameLowering *TFI = MF.getSubtarget().getFrameLowering();
+  const XCoreFrameLowering *TFI = getFrameLowering(MF);
 
   Reserved.set(XCore::CP);
   Reserved.set(XCore::DP);
@@ -270,7 +270,7 @@ XCoreRegisterInfo::eliminateFrameIndex(MachineBasicBlock::iterator II,
   const XCoreInstrInfo &TII =
       *static_cast<const XCoreInstrInfo *>(MF.getSubtarget().getInstrInfo());
 
-  const TargetFrameLowering *TFI = MF.getSubtarget().getFrameLowering();
+  const XCoreFrameLowering *TFI = getFrameLowering(MF);
   int Offset = MF.getFrameInfo()->getObjectOffset(FrameIndex);
   int StackSize = MF.getFrameInfo()->getStackSize();
 
@@ -324,7 +324,7 @@ XCoreRegisterInfo::eliminateFrameIndex(MachineBasicBlock::iterator II,
 
 
 unsigned XCoreRegisterInfo::getFrameRegister(const MachineFunction &MF) const {
-  const TargetFrameLowering *TFI = MF.getSubtarget().getFrameLowering();
+  const XCoreFrameLowering *TFI = getFrameLowering(MF);
 
   return TFI->hasFP(MF) ? XCore::R10 : XCore::SP;
 }
diff --git a/lib/Target/XCore/XCoreSelectionDAGInfo.cpp b/lib/Target/XCore/XCoreSelectionDAGInfo.cpp
index a34884480ce..40568d124de 100644
--- a/lib/Target/XCore/XCoreSelectionDAGInfo.cpp
+++ b/lib/Target/XCore/XCoreSelectionDAGInfo.cpp
@@ -16,12 +16,6 @@ using namespace llvm;
 
 #define DEBUG_TYPE "xcore-selectiondag-info"
 
-XCoreSelectionDAGInfo::XCoreSelectionDAGInfo(const DataLayout &DL)
-    : TargetSelectionDAGInfo(&DL) {}
-
-XCoreSelectionDAGInfo::~XCoreSelectionDAGInfo() {
-}
-
 SDValue XCoreSelectionDAGInfo::
 EmitTargetCodeForMemcpy(SelectionDAG &DAG, SDLoc dl, SDValue Chain,
                         SDValue Dst, SDValue Src, SDValue Size, unsigned Align,
@@ -36,18 +30,20 @@ EmitTargetCodeForMemcpy(SelectionDAG &DAG, SDLoc dl, SDValue Chain,
     const TargetLowering &TLI = *DAG.getSubtarget().getTargetLowering();
     TargetLowering::ArgListTy Args;
     TargetLowering::ArgListEntry Entry;
-    Entry.Ty = TLI.getDataLayout()->getIntPtrType(*DAG.getContext());
+    Entry.Ty = DAG.getDataLayout().getIntPtrType(*DAG.getContext());
     Entry.Node = Dst; Args.push_back(Entry);
     Entry.Node = Src; Args.push_back(Entry);
     Entry.Node = Size; Args.push_back(Entry);
 
     TargetLowering::CallLoweringInfo CLI(DAG);
-    CLI.setDebugLoc(dl).setChain(Chain)
-      .setCallee(TLI.getLibcallCallingConv(RTLIB::MEMCPY),
-                 Type::getVoidTy(*DAG.getContext()),
-                 DAG.getExternalSymbol("__memcpy_4", TLI.getPointerTy()),
-                 std::move(Args), 0)
-      .setDiscardResult();
+    CLI.setDebugLoc(dl)
+        .setChain(Chain)
+        .setCallee(TLI.getLibcallCallingConv(RTLIB::MEMCPY),
+                   Type::getVoidTy(*DAG.getContext()),
+                   DAG.getExternalSymbol("__memcpy_4",
+                                         TLI.getPointerTy(DAG.getDataLayout())),
+                   std::move(Args), 0)
+        .setDiscardResult();
 
     std::pair<SDValue,SDValue> CallResult = TLI.LowerCallTo(CLI);
     return CallResult.second;
diff --git a/lib/Target/XCore/XCoreSelectionDAGInfo.h b/lib/Target/XCore/XCoreSelectionDAGInfo.h
index cfd80b3f317..77b3527d77e 100644
--- a/lib/Target/XCore/XCoreSelectionDAGInfo.h
+++ b/lib/Target/XCore/XCoreSelectionDAGInfo.h
@@ -22,8 +22,6 @@ class XCoreTargetMachine;
 
 class XCoreSelectionDAGInfo : public TargetSelectionDAGInfo {
 public:
-  explicit XCoreSelectionDAGInfo(const DataLayout &DL);
-  ~XCoreSelectionDAGInfo();
 
   SDValue
   EmitTargetCodeForMemcpy(SelectionDAG &DAG, SDLoc dl,
diff --git a/lib/Target/XCore/XCoreSubtarget.cpp b/lib/Target/XCore/XCoreSubtarget.cpp
index 79960207a45..99ad2c88504 100644
--- a/lib/Target/XCore/XCoreSubtarget.cpp
+++ b/lib/Target/XCore/XCoreSubtarget.cpp
@@ -25,7 +25,7 @@ using namespace llvm;
 
 void XCoreSubtarget::anchor() { }
 
-XCoreSubtarget::XCoreSubtarget(const std::string &TT, const std::string &CPU,
+XCoreSubtarget::XCoreSubtarget(const Triple &TT, const std::string &CPU,
                                const std::string &FS, const TargetMachine &TM)
     : XCoreGenSubtargetInfo(TT, CPU, FS), InstrInfo(), FrameLowering(*this),
-      TLInfo(TM, *this), TSInfo(*TM.getDataLayout()) {}
+      TLInfo(TM, *this), TSInfo() {}
diff --git a/lib/Target/XCore/XCoreSubtarget.h b/lib/Target/XCore/XCoreSubtarget.h
index da51ef1c7a8..f01fb6714d8 100644
--- a/lib/Target/XCore/XCoreSubtarget.h
+++ b/lib/Target/XCore/XCoreSubtarget.h
@@ -40,9 +40,9 @@ class XCoreSubtarget : public XCoreGenSubtargetInfo {
   /// This constructor initializes the data members to match that
   /// of the specified triple.
   ///
-  XCoreSubtarget(const std::string &TT, const std::string &CPU,
+  XCoreSubtarget(const Triple &TT, const std::string &CPU,
                  const std::string &FS, const TargetMachine &TM);
-  
+
   /// ParseSubtargetFeatures - Parses features string setting specified 
   /// subtarget options.  Definition of function is auto generated by tblgen.
   void ParseSubtargetFeatures(StringRef CPU, StringRef FS);
diff --git a/lib/Target/XCore/XCoreTargetMachine.cpp b/lib/Target/XCore/XCoreTargetMachine.cpp
index 228dc1c9db5..f420081868f 100644
--- a/lib/Target/XCore/XCoreTargetMachine.cpp
+++ b/lib/Target/XCore/XCoreTargetMachine.cpp
@@ -22,7 +22,7 @@ using namespace llvm;
 
 /// XCoreTargetMachine ctor - Create an ILP32 architecture model
 ///
-XCoreTargetMachine::XCoreTargetMachine(const Target &T, StringRef TT,
+XCoreTargetMachine::XCoreTargetMachine(const Target &T, const Triple &TT,
                                        StringRef CPU, StringRef FS,
                                        const TargetOptions &Options,
                                        Reloc::Model RM, CodeModel::Model CM,
@@ -85,6 +85,7 @@ extern "C" void LLVMInitializeXCoreTarget() {
 }
 
 TargetIRAnalysis XCoreTargetMachine::getTargetIRAnalysis() {
-  return TargetIRAnalysis(
-      [this](Function &) { return TargetTransformInfo(XCoreTTIImpl(this)); });
+  return TargetIRAnalysis([this](Function &F) {
+    return TargetTransformInfo(XCoreTTIImpl(this, F));
+  });
 }
diff --git a/lib/Target/XCore/XCoreTargetMachine.h b/lib/Target/XCore/XCoreTargetMachine.h
index 0d324ab1e72..a8addfc3e42 100644
--- a/lib/Target/XCore/XCoreTargetMachine.h
+++ b/lib/Target/XCore/XCoreTargetMachine.h
@@ -23,8 +23,8 @@ class XCoreTargetMachine : public LLVMTargetMachine {
   std::unique_ptr<TargetLoweringObjectFile> TLOF;
   XCoreSubtarget Subtarget;
 public:
-  XCoreTargetMachine(const Target &T, StringRef TT,
-                     StringRef CPU, StringRef FS, const TargetOptions &Options,
+  XCoreTargetMachine(const Target &T, const Triple &TT, StringRef CPU,
+                     StringRef FS, const TargetOptions &Options,
                      Reloc::Model RM, CodeModel::Model CM,
                      CodeGenOpt::Level OL);
   ~XCoreTargetMachine() override;
diff --git a/lib/Target/XCore/XCoreTargetObjectFile.cpp b/lib/Target/XCore/XCoreTargetObjectFile.cpp
index c435b36fae1..eb3072195dc 100644
--- a/lib/Target/XCore/XCoreTargetObjectFile.cpp
+++ b/lib/Target/XCore/XCoreTargetObjectFile.cpp
@@ -95,7 +95,7 @@ static unsigned getXCoreSectionFlags(SectionKind K, bool IsCPRel) {
   return Flags;
 }
 
-const MCSection *
+MCSection *
 XCoreTargetObjectFile::getExplicitSectionGlobal(const GlobalValue *GV,
                                                 SectionKind Kind, Mangler &Mang,
                                                 const TargetMachine &TM) const {
@@ -108,9 +108,10 @@ XCoreTargetObjectFile::getExplicitSectionGlobal(const GlobalValue *GV,
                                     getXCoreSectionFlags(Kind, IsCPRel));
 }
 
-const MCSection *XCoreTargetObjectFile::
-SelectSectionForGlobal(const GlobalValue *GV, SectionKind Kind, Mangler &Mang,
-                       const TargetMachine &TM) const{
+MCSection *
+XCoreTargetObjectFile::SelectSectionForGlobal(const GlobalValue *GV,
+                                              SectionKind Kind, Mangler &Mang,
+                                              const TargetMachine &TM) const {
 
   bool UseCPRel = GV->isLocalLinkage(GV->getLinkage());
 
@@ -122,8 +123,9 @@ SelectSectionForGlobal(const GlobalValue *GV, SectionKind Kind, Mangler &Mang,
     if (Kind.isMergeableConst16())      return MergeableConst16Section;
   }
   Type *ObjType = GV->getType()->getPointerElementType();
+  auto &DL = GV->getParent()->getDataLayout();
   if (TM.getCodeModel() == CodeModel::Small || !ObjType->isSized() ||
-      TM.getDataLayout()->getTypeAllocSize(ObjType) < CodeModelLargeSize) {
+      DL.getTypeAllocSize(ObjType) < CodeModelLargeSize) {
     if (Kind.isReadOnly())              return UseCPRel? ReadOnlySection
                                                        : DataRelROSection;
     if (Kind.isBSS() || Kind.isCommon())return BSSSection;
@@ -141,9 +143,8 @@ SelectSectionForGlobal(const GlobalValue *GV, SectionKind Kind, Mangler &Mang,
   report_fatal_error("Target does not support TLS or Common sections");
 }
 
-const MCSection *
-XCoreTargetObjectFile::getSectionForConstant(SectionKind Kind,
-                                             const Constant *C) const {
+MCSection *XCoreTargetObjectFile::getSectionForConstant(
+    const DataLayout &DL, SectionKind Kind, const Constant *C) const {
   if (Kind.isMergeableConst4())           return MergeableConst4Section;
   if (Kind.isMergeableConst8())           return MergeableConst8Section;
   if (Kind.isMergeableConst16())          return MergeableConst16Section;
diff --git a/lib/Target/XCore/XCoreTargetObjectFile.h b/lib/Target/XCore/XCoreTargetObjectFile.h
index 7d3f49d222d..6701c661a73 100644
--- a/lib/Target/XCore/XCoreTargetObjectFile.h
+++ b/lib/Target/XCore/XCoreTargetObjectFile.h
@@ -17,25 +17,24 @@ namespace llvm {
 static const unsigned CodeModelLargeSize = 256;
 
   class XCoreTargetObjectFile : public TargetLoweringObjectFileELF {
-   const MCSection *BSSSectionLarge;
-   const MCSection *DataSectionLarge;
-   const MCSection *ReadOnlySectionLarge;
-   const MCSection *DataRelROSectionLarge;
+    MCSection *BSSSectionLarge;
+    MCSection *DataSectionLarge;
+    MCSection *ReadOnlySectionLarge;
+    MCSection *DataRelROSectionLarge;
+
   public:
     void Initialize(MCContext &Ctx, const TargetMachine &TM) override;
 
-    const MCSection *
-      getExplicitSectionGlobal(const GlobalValue *GV,
-                               SectionKind Kind, Mangler &Mang,
-                               const TargetMachine &TM) const override;
+    MCSection *getExplicitSectionGlobal(const GlobalValue *GV, SectionKind Kind,
+                                        Mangler &Mang,
+                                        const TargetMachine &TM) const override;
 
-    const MCSection *
-      SelectSectionForGlobal(const GlobalValue *GV, SectionKind Kind,
-                             Mangler &Mang,
-                             const TargetMachine &TM) const override;
+    MCSection *SelectSectionForGlobal(const GlobalValue *GV, SectionKind Kind,
+                                      Mangler &Mang,
+                                      const TargetMachine &TM) const override;
 
-    const MCSection *getSectionForConstant(SectionKind Kind,
-                                           const Constant *C) const override;
+    MCSection *getSectionForConstant(const DataLayout &DL, SectionKind Kind,
+                                     const Constant *C) const override;
   };
 } // end namespace llvm
 
diff --git a/lib/Target/XCore/XCoreTargetTransformInfo.h b/lib/Target/XCore/XCoreTargetTransformInfo.h
index 70b47dfa115..e23aef3e3b4 100644
--- a/lib/Target/XCore/XCoreTargetTransformInfo.h
+++ b/lib/Target/XCore/XCoreTargetTransformInfo.h
@@ -37,8 +37,9 @@ class XCoreTTIImpl : public BasicTTIImplBase<XCoreTTIImpl> {
   const XCoreTargetLowering *getTLI() const { return TLI; }
 
 public:
-  explicit XCoreTTIImpl(const XCoreTargetMachine *TM)
-      : BaseT(TM), ST(TM->getSubtargetImpl()), TLI(ST->getTargetLowering()) {}
+  explicit XCoreTTIImpl(const XCoreTargetMachine *TM, Function &F)
+      : BaseT(TM, F.getParent()->getDataLayout()), ST(TM->getSubtargetImpl()),
+        TLI(ST->getTargetLowering()) {}
 
   // Provide value semantics. MSVC requires that we spell all of these out.
   XCoreTTIImpl(const XCoreTTIImpl &Arg)
@@ -46,18 +47,6 @@ class XCoreTTIImpl : public BasicTTIImplBase<XCoreTTIImpl> {
   XCoreTTIImpl(XCoreTTIImpl &&Arg)
       : BaseT(std::move(static_cast<BaseT &>(Arg))), ST(std::move(Arg.ST)),
         TLI(std::move(Arg.TLI)) {}
-  XCoreTTIImpl &operator=(const XCoreTTIImpl &RHS) {
-    BaseT::operator=(static_cast<const BaseT &>(RHS));
-    ST = RHS.ST;
-    TLI = RHS.TLI;
-    return *this;
-  }
-  XCoreTTIImpl &operator=(XCoreTTIImpl &&RHS) {
-    BaseT::operator=(std::move(static_cast<BaseT &>(RHS)));
-    ST = std::move(RHS.ST);
-    TLI = std::move(RHS.TLI);
-    return *this;
-  }
 
   unsigned getNumberOfRegisters(bool Vector) {
     if (Vector) {
diff --git a/lib/Transforms/CMakeLists.txt b/lib/Transforms/CMakeLists.txt
index b633fbaa064..328bc13cdd0 100644
--- a/lib/Transforms/CMakeLists.txt
+++ b/lib/Transforms/CMakeLists.txt
@@ -7,4 +7,3 @@ add_subdirectory(Vectorize)
 add_subdirectory(Hello)
 add_subdirectory(ObjCARC)
 add_subdirectory(NaCl)
-add_subdirectory(MinSFI)
diff --git a/lib/Transforms/Hello/CMakeLists.txt b/lib/Transforms/Hello/CMakeLists.txt
index 3851b35871f..e0b81907c7f 100644
--- a/lib/Transforms/Hello/CMakeLists.txt
+++ b/lib/Transforms/Hello/CMakeLists.txt
@@ -12,4 +12,7 @@ endif()
 
 add_llvm_loadable_module( LLVMHello
   Hello.cpp
+
+  DEPENDS
+  intrinsics_gen
   )
diff --git a/lib/Transforms/IPO/ArgumentPromotion.cpp b/lib/Transforms/IPO/ArgumentPromotion.cpp
index 56975eabaee..f330f2e3b4e 100644
--- a/lib/Transforms/IPO/ArgumentPromotion.cpp
+++ b/lib/Transforms/IPO/ArgumentPromotion.cpp
@@ -36,6 +36,7 @@
 #include "llvm/Analysis/AliasAnalysis.h"
 #include "llvm/Analysis/CallGraph.h"
 #include "llvm/Analysis/CallGraphSCCPass.h"
+#include "llvm/Analysis/ValueTracking.h"
 #include "llvm/IR/CFG.h"
 #include "llvm/IR/CallSite.h"
 #include "llvm/IR/Constants.h"
@@ -89,7 +90,7 @@ namespace {
     bool doInitialization(CallGraph &CG) override;
     /// The maximum number of elements to expand, or 0 for unlimited.
     unsigned maxElements;
-    DenseMap<const Function *, DISubprogram> FunctionDIs;
+    DenseMap<const Function *, DISubprogram *> FunctionDIs;
   };
 }
 
@@ -244,6 +245,24 @@ CallGraphNode *ArgPromotion::PromoteArguments(CallGraphNode *CGN) {
     Argument *PtrArg = PointerArgs[i];
     Type *AgTy = cast<PointerType>(PtrArg->getType())->getElementType();
 
+    // Replace sret attribute with noalias. This reduces register pressure by
+    // avoiding a register copy.
+    if (PtrArg->hasStructRetAttr()) {
+      unsigned ArgNo = PtrArg->getArgNo();
+      F->setAttributes(
+          F->getAttributes()
+              .removeAttribute(F->getContext(), ArgNo + 1, Attribute::StructRet)
+              .addAttribute(F->getContext(), ArgNo + 1, Attribute::NoAlias));
+      for (Use &U : F->uses()) {
+        CallSite CS(U.getUser());
+        CS.setAttributes(
+            CS.getAttributes()
+                .removeAttribute(F->getContext(), ArgNo + 1,
+                                 Attribute::StructRet)
+                .addAttribute(F->getContext(), ArgNo + 1, Attribute::NoAlias));
+      }
+    }
+
     // If this is a byval argument, and if the aggregate type is small, just
     // pass the elements, which is always safe, if the passed value is densely
     // packed or if we can prove the padding bytes are never accessed. This does
@@ -262,8 +281,8 @@ CallGraphNode *ArgPromotion::PromoteArguments(CallGraphNode *CGN) {
         
         // If all the elements are single-value types, we can promote it.
         bool AllSimple = true;
-        for (unsigned i = 0, e = STy->getNumElements(); i != e; ++i) {
-          if (!STy->getElementType(i)->isSingleValueType()) {
+        for (const auto *EltTy : STy->elements()) {
+          if (!EltTy->isSingleValueType()) {
             AllSimple = false;
             break;
           }
@@ -284,8 +303,8 @@ CallGraphNode *ArgPromotion::PromoteArguments(CallGraphNode *CGN) {
     if (isSelfRecursive) {
       if (StructType *STy = dyn_cast<StructType>(AgTy)) {
         bool RecursiveType = false;
-        for (unsigned i = 0, e = STy->getNumElements(); i != e; ++i) {
-          if (STy->getElementType(i) == PtrArg->getType()) {
+        for (const auto *EltTy : STy->elements()) {
+          if (EltTy == PtrArg->getType()) {
             RecursiveType = true;
             break;
           }
@@ -321,7 +340,7 @@ static bool AllCallersPassInValidPointerForArgument(Argument *Arg) {
     CallSite CS(U);
     assert(CS && "Should only have direct calls!");
 
-    if (!CS.getArgument(ArgNo)->isDereferenceablePointer(DL))
+    if (!isDereferenceablePointer(CS.getArgument(ArgNo), DL))
       return false;
   }
   return true;
@@ -482,7 +501,6 @@ bool ArgPromotion::isSafeToPromoteArgument(Argument *Arg,
       if (GEP->use_empty()) {
         // Dead GEP's cause trouble later.  Just remove them if we run into
         // them.
-        getAnalysis<AliasAnalysis>().deleteValue(GEP);
         GEP->eraseFromParent();
         // TODO: This runs the above loop over and over again for dead GEPs
         // Couldn't we just do increment the UI iterator earlier and erase the
@@ -552,9 +570,8 @@ bool ArgPromotion::isSafeToPromoteArgument(Argument *Arg,
     LoadInst *Load = Loads[i];
     BasicBlock *BB = Load->getParent();
 
-    AliasAnalysis::Location Loc = AA.getLocation(Load);
-    if (AA.canInstructionRangeModRef(BB->front(), *Load, Loc,
-        AliasAnalysis::Mod))
+    MemoryLocation Loc = MemoryLocation::get(Load);
+    if (AA.canInstructionRangeModRef(BB->front(), *Load, Loc, MRI_Mod))
       return false;  // Pointer is invalidated!
 
     // Now check every path from the entry block to the load for transparency.
@@ -705,7 +722,7 @@ CallGraphNode *ArgPromotion::DoPromotion(Function *F,
   // Patch the pointer to LLVM function in debug info descriptor.
   auto DI = FunctionDIs.find(F);
   if (DI != FunctionDIs.end()) {
-    DISubprogram SP = DI->second;
+    DISubprogram *SP = DI->second;
     SP->replaceFunction(NF);
     // Ensure the map is updated so it can be reused on subsequent argument
     // promotions of the same function.
@@ -724,10 +741,6 @@ CallGraphNode *ArgPromotion::DoPromotion(Function *F,
   F->getParent()->getFunctionList().insert(F, NF);
   NF->takeName(F);
 
-  // Get the alias analysis information that we need to update to reflect our
-  // changes.
-  AliasAnalysis &AA = getAnalysis<AliasAnalysis>();
-
   // Get the callgraph information that we need to update to reflect our
   // changes.
   CallGraph &CG = getAnalysis<CallGraphWrapperPass>().getCallGraph();
@@ -773,7 +786,7 @@ CallGraphNode *ArgPromotion::DoPromotion(Function *F,
         for (unsigned i = 0, e = STy->getNumElements(); i != e; ++i) {
           Idxs[1] = ConstantInt::get(Type::getInt32Ty(F->getContext()), i);
           Value *Idx = GetElementPtrInst::Create(
-              STy, *AI, Idxs, (*AI)->getName() + "." + utostr(i), Call);
+              STy, *AI, Idxs, (*AI)->getName() + "." + Twine(i), Call);
           // TODO: Tell AA about the new values?
           Args.push_back(new LoadInst(Idx, Idx->getName()+".val", Call));
         }
@@ -806,7 +819,6 @@ CallGraphNode *ArgPromotion::DoPromotion(Function *F,
             V = GetElementPtrInst::Create(SI->first, V, Ops,
                                           V->getName() + ".idx", Call);
             Ops.clear();
-            AA.copyValue(OrigLoad->getOperand(0), V);
           }
           // Since we're replacing a load make sure we take the alignment
           // of the previous load.
@@ -818,7 +830,6 @@ CallGraphNode *ArgPromotion::DoPromotion(Function *F,
           newLoad->setAAMetadata(AAInfo);
 
           Args.push_back(newLoad);
-          AA.copyValue(OrigLoad, Args.back());
         }
       }
 
@@ -856,10 +867,6 @@ CallGraphNode *ArgPromotion::DoPromotion(Function *F,
     Args.clear();
     AttributesVec.clear();
 
-    // Update the alias analysis implementation to know that we are replacing
-    // the old call with a new one.
-    AA.replaceWithNewValue(Call, New);
-
     // Update the callgraph to know that the callsite has been transformed.
     CallGraphNode *CalleeNode = CG[Call->getParent()->getParent()];
     CalleeNode->replaceCallEdge(CS, CallSite(New), NF_CGN);
@@ -889,7 +896,6 @@ CallGraphNode *ArgPromotion::DoPromotion(Function *F,
       // new version.
       I->replaceAllUsesWith(I2);
       I2->takeName(I);
-      AA.replaceWithNewValue(I, I2);
       ++I2;
       continue;
     }
@@ -918,7 +924,6 @@ CallGraphNode *ArgPromotion::DoPromotion(Function *F,
       // Anything that used the arg should now use the alloca.
       I->replaceAllUsesWith(TheAlloca);
       TheAlloca->takeName(I);
-      AA.replaceWithNewValue(I, TheAlloca);
 
       // If the alloca is used in a call, we must clear the tail flag since
       // the callee now uses an alloca from the caller.
@@ -931,10 +936,8 @@ CallGraphNode *ArgPromotion::DoPromotion(Function *F,
       continue;
     }
 
-    if (I->use_empty()) {
-      AA.deleteValue(I);
+    if (I->use_empty())
       continue;
-    }
 
     // Otherwise, if we promoted this argument, then all users are load
     // instructions (or GEPs with only load users), and all loads should be
@@ -947,7 +950,6 @@ CallGraphNode *ArgPromotion::DoPromotion(Function *F,
                "Load element should sort to front!");
         I2->setName(I->getName()+".val");
         LI->replaceAllUsesWith(I2);
-        AA.replaceWithNewValue(LI, I2);
         LI->eraseFromParent();
         DEBUG(dbgs() << "*** Promoted load of argument '" << I->getName()
               << "' in function '" << F->getName() << "'\n");
@@ -984,10 +986,8 @@ CallGraphNode *ArgPromotion::DoPromotion(Function *F,
         while (!GEP->use_empty()) {
           LoadInst *L = cast<LoadInst>(GEP->user_back());
           L->replaceAllUsesWith(TheArg);
-          AA.replaceWithNewValue(L, TheArg);
           L->eraseFromParent();
         }
-        AA.deleteValue(GEP);
         GEP->eraseFromParent();
       }
     }
@@ -996,10 +996,6 @@ CallGraphNode *ArgPromotion::DoPromotion(Function *F,
     std::advance(I2, ArgIndices.size());
   }
 
-  // Tell the alias analysis that the old function is about to disappear.
-  AA.replaceWithNewValue(F, NF);
-
-  
   NF_CGN->stealCalledFunctionsFrom(CG[F]);
   
   // Now that the old function is dead, delete it.  If there is a dangling
diff --git a/lib/Transforms/IPO/CMakeLists.txt b/lib/Transforms/IPO/CMakeLists.txt
index 3df17b920a9..336dac45e13 100644
--- a/lib/Transforms/IPO/CMakeLists.txt
+++ b/lib/Transforms/IPO/CMakeLists.txt
@@ -3,6 +3,7 @@ add_llvm_library(LLVMipo
   BarrierNoopPass.cpp
   ConstantMerge.cpp
   DeadArgumentElimination.cpp
+  ElimAvailExtern.cpp
   ExtractGV.cpp
   FunctionAttrs.cpp
   GlobalDCE.cpp
diff --git a/lib/Transforms/IPO/DeadArgumentElimination.cpp b/lib/Transforms/IPO/DeadArgumentElimination.cpp
index 3be23d55f5a..d0447640259 100644
--- a/lib/Transforms/IPO/DeadArgumentElimination.cpp
+++ b/lib/Transforms/IPO/DeadArgumentElimination.cpp
@@ -127,7 +127,7 @@ namespace {
     // As the code generation for module is finished (and DIBuilder is
     // finalized) we assume that subprogram descriptors won't be changed, and
     // they are stored in map for short duration anyway.
-    DenseMap<const Function *, DISubprogram> FunctionDIs;
+    DenseMap<const Function *, DISubprogram *> FunctionDIs;
 
   protected:
     // DAH uses this to specify a different ID.
@@ -303,7 +303,7 @@ bool DAE::DeleteDeadVarargs(Function &Fn) {
   // Patch the pointer to LLVM function in debug info descriptor.
   auto DI = FunctionDIs.find(&Fn);
   if (DI != FunctionDIs.end()) {
-    DISubprogram SP = DI->second;
+    DISubprogram *SP = DI->second;
     SP->replaceFunction(NF);
     // Ensure the map is updated so it can be reused on non-varargs argument
     // eliminations of the same function.
@@ -326,7 +326,18 @@ bool DAE::DeleteDeadVarargs(Function &Fn) {
 /// instead.
 bool DAE::RemoveDeadArgumentsFromCallers(Function &Fn)
 {
-  if (Fn.isDeclaration() || Fn.mayBeOverridden())
+  // We cannot change the arguments if this TU does not define the function or
+  // if the linker may choose a function body from another TU, even if the
+  // nominal linkage indicates that other copies of the function have the same
+  // semantics. In the below example, the dead load from %p may not have been
+  // eliminated from the linker-chosen copy of f, so replacing %p with undef
+  // in callers may introduce undefined behavior.
+  //
+  // define linkonce_odr void @f(i32* %p) {
+  //   %v = load i32 %p
+  //   ret void
+  // }
+  if (!Fn.isStrongDefinitionForLinker())
     return false;
 
   // Functions with local linkage should already have been handled, except the
@@ -334,19 +345,6 @@ bool DAE::RemoveDeadArgumentsFromCallers(Function &Fn)
   if (Fn.hasLocalLinkage() && !Fn.getFunctionType()->isVarArg())
     return false;
 
-  // If a function seen at compile time is not necessarily the one linked to
-  // the binary being built, it is illegal to change the actual arguments
-  // passed to it. These functions can be captured by isWeakForLinker().
-  // *NOTE* that mayBeOverridden() is insufficient for this purpose as it
-  // doesn't include linkage types like AvailableExternallyLinkage and
-  // LinkOnceODRLinkage. Take link_odr* as an example, it indicates a set of
-  // *EQUIVALENT* globals that can be merged at link-time. However, the
-  // semantic of *EQUIVALENT*-functions includes parameters. Changing
-  // parameters breaks this assumption.
-  //
-  if (Fn.isWeakForLinker())
-    return false;
-
   if (Fn.use_empty())
     return false;
 
@@ -849,17 +847,12 @@ bool DAE::RemoveDeadStuffFromFunction(Function *F) {
   // here. Currently, this should not be possible, but special handling might be
   // required when new return value attributes are added.
   if (NRetTy->isVoidTy())
-    RAttrs =
-      AttributeSet::get(NRetTy->getContext(), AttributeSet::ReturnIndex,
-                        AttrBuilder(RAttrs, AttributeSet::ReturnIndex).
-         removeAttributes(AttributeFuncs::
-                          typeIncompatible(NRetTy, AttributeSet::ReturnIndex),
-                          AttributeSet::ReturnIndex));
+    RAttrs = RAttrs.removeAttributes(NRetTy->getContext(),
+                                     AttributeSet::ReturnIndex,
+                                     AttributeFuncs::typeIncompatible(NRetTy));
   else
     assert(!AttrBuilder(RAttrs, AttributeSet::ReturnIndex).
-             hasAttributes(AttributeFuncs::
-                           typeIncompatible(NRetTy, AttributeSet::ReturnIndex),
-                           AttributeSet::ReturnIndex) &&
+             overlaps(AttributeFuncs::typeIncompatible(NRetTy)) &&
            "Return attributes no longer compatible?");
 
   if (RAttrs.hasAttributes(AttributeSet::ReturnIndex))
@@ -903,13 +896,9 @@ bool DAE::RemoveDeadStuffFromFunction(Function *F) {
     AttributeSet RAttrs = CallPAL.getRetAttributes();
 
     // Adjust in case the function was changed to return void.
-    RAttrs =
-      AttributeSet::get(NF->getContext(), AttributeSet::ReturnIndex,
-                        AttrBuilder(RAttrs, AttributeSet::ReturnIndex).
-        removeAttributes(AttributeFuncs::
-                         typeIncompatible(NF->getReturnType(),
-                                          AttributeSet::ReturnIndex),
-                         AttributeSet::ReturnIndex));
+    RAttrs = RAttrs.removeAttributes(NRetTy->getContext(),
+                                     AttributeSet::ReturnIndex,
+                        AttributeFuncs::typeIncompatible(NF->getReturnType()));
     if (RAttrs.hasAttributes(AttributeSet::ReturnIndex))
       AttributesVec.push_back(AttributeSet::get(NF->getContext(), RAttrs));
 
diff --git a/lib/Transforms/IPO/ElimAvailExtern.cpp b/lib/Transforms/IPO/ElimAvailExtern.cpp
new file mode 100644
index 00000000000..67ba72d6a36
--- /dev/null
+++ b/lib/Transforms/IPO/ElimAvailExtern.cpp
@@ -0,0 +1,84 @@
+//===-- ElimAvailExtern.cpp - DCE unreachable internal functions ----------------===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This transform is designed to eliminate available external global
+// definitions from the program, turning them into declarations.
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/Transforms/IPO.h"
+#include "llvm/ADT/Statistic.h"
+#include "llvm/IR/Constants.h"
+#include "llvm/IR/Instructions.h"
+#include "llvm/IR/Module.h"
+#include "llvm/Transforms/Utils/CtorUtils.h"
+#include "llvm/Transforms/Utils/GlobalStatus.h"
+#include "llvm/Pass.h"
+using namespace llvm;
+
+#define DEBUG_TYPE "elim-avail-extern"
+
+STATISTIC(NumFunctions, "Number of functions removed");
+STATISTIC(NumVariables, "Number of global variables removed");
+
+namespace {
+  struct EliminateAvailableExternally : public ModulePass {
+    static char ID; // Pass identification, replacement for typeid
+    EliminateAvailableExternally() : ModulePass(ID) {
+      initializeEliminateAvailableExternallyPass(
+          *PassRegistry::getPassRegistry());
+    }
+
+    // run - Do the EliminateAvailableExternally pass on the specified module,
+    // optionally updating the specified callgraph to reflect the changes.
+    //
+    bool runOnModule(Module &M) override;
+  };
+}
+
+char EliminateAvailableExternally::ID = 0;
+INITIALIZE_PASS(EliminateAvailableExternally, "elim-avail-extern",
+                "Eliminate Available Externally Globals", false, false)
+
+ModulePass *llvm::createEliminateAvailableExternallyPass() {
+  return new EliminateAvailableExternally();
+}
+
+bool EliminateAvailableExternally::runOnModule(Module &M) {
+  bool Changed = false;
+
+  // Drop initializers of available externally global variables.
+  for (Module::global_iterator I = M.global_begin(), E = M.global_end();
+       I != E; ++I) {
+    if (!I->hasAvailableExternallyLinkage())
+      continue;
+    if (I->hasInitializer()) {
+      Constant *Init = I->getInitializer();
+      I->setInitializer(nullptr);
+      if (isSafeToDestroyConstant(Init))
+        Init->destroyConstant();
+    }
+    I->removeDeadConstantUsers();
+    I->setLinkage(GlobalValue::ExternalLinkage);
+    NumVariables++;
+  }
+
+  // Drop the bodies of available externally functions.
+  for (Module::iterator I = M.begin(), E = M.end(); I != E; ++I) {
+    if (!I->hasAvailableExternallyLinkage())
+      continue;
+    if (!I->isDeclaration())
+      // This will set the linkage to external
+      I->deleteBody();
+    I->removeDeadConstantUsers();
+    NumFunctions++;
+  }
+
+  return Changed;
+}
diff --git a/lib/Transforms/IPO/ExtractGV.cpp b/lib/Transforms/IPO/ExtractGV.cpp
index 2f8c7d9349b..b9462f2ffc7 100644
--- a/lib/Transforms/IPO/ExtractGV.cpp
+++ b/lib/Transforms/IPO/ExtractGV.cpp
@@ -93,8 +93,11 @@ namespace {
 
         makeVisible(*I, Delete);
 
-        if (Delete)
+        if (Delete) {
+          // Make this a declaration and drop it's comdat.
           I->setInitializer(nullptr);
+          I->setComdat(nullptr);
+        }
       }
 
       // Visit the Functions.
@@ -108,8 +111,11 @@ namespace {
 
         makeVisible(*I, Delete);
 
-        if (Delete)
+        if (Delete) {
+          // Make this a declaration and drop it's comdat.
           I->deleteBody();
+          I->setComdat(nullptr);
+        }
       }
 
       // Visit the Aliases.
diff --git a/lib/Transforms/IPO/FunctionAttrs.cpp b/lib/Transforms/IPO/FunctionAttrs.cpp
index c7e9afa7323..206b630c792 100644
--- a/lib/Transforms/IPO/FunctionAttrs.cpp
+++ b/lib/Transforms/IPO/FunctionAttrs.cpp
@@ -166,8 +166,8 @@ bool FunctionAttrs::AddReadAttrs(const CallGraphSCC &SCC) {
       // memory and give up.
       return false;
 
-    AliasAnalysis::ModRefBehavior MRB = AA->getModRefBehavior(F);
-    if (MRB == AliasAnalysis::DoesNotAccessMemory)
+    FunctionModRefBehavior MRB = AA->getModRefBehavior(F);
+    if (MRB == FMRB_DoesNotAccessMemory)
       // Already perfect!
       continue;
 
@@ -193,7 +193,7 @@ bool FunctionAttrs::AddReadAttrs(const CallGraphSCC &SCC) {
         // Ignore calls to functions in the same SCC.
         if (CS.getCalledFunction() && SCCNodes.count(CS.getCalledFunction()))
           continue;
-        AliasAnalysis::ModRefBehavior MRB = AA->getModRefBehavior(CS);
+        FunctionModRefBehavior MRB = AA->getModRefBehavior(CS);
         // If the call doesn't access arbitrary memory, we may be able to
         // figure out something.
         if (AliasAnalysis::onlyAccessesArgPointees(MRB)) {
@@ -208,13 +208,12 @@ bool FunctionAttrs::AddReadAttrs(const CallGraphSCC &SCC) {
                 AAMDNodes AAInfo;
                 I->getAAMetadata(AAInfo);
 
-                AliasAnalysis::Location Loc(Arg,
-                                            AliasAnalysis::UnknownSize, AAInfo);
+                MemoryLocation Loc(Arg, MemoryLocation::UnknownSize, AAInfo);
                 if (!AA->pointsToConstantMemory(Loc, /*OrLocal=*/true)) {
-                  if (MRB & AliasAnalysis::Mod)
+                  if (MRB & MRI_Mod)
                     // Writes non-local memory.  Give up.
                     return false;
-                  if (MRB & AliasAnalysis::Ref)
+                  if (MRB & MRI_Ref)
                     // Ok, it reads non-local memory.
                     ReadsMemory = true;
                 }
@@ -223,29 +222,29 @@ bool FunctionAttrs::AddReadAttrs(const CallGraphSCC &SCC) {
           continue;
         }
         // The call could access any memory. If that includes writes, give up.
-        if (MRB & AliasAnalysis::Mod)
+        if (MRB & MRI_Mod)
           return false;
         // If it reads, note it.
-        if (MRB & AliasAnalysis::Ref)
+        if (MRB & MRI_Ref)
           ReadsMemory = true;
         continue;
       } else if (LoadInst *LI = dyn_cast<LoadInst>(I)) {
         // Ignore non-volatile loads from local memory. (Atomic is okay here.)
         if (!LI->isVolatile()) {
-          AliasAnalysis::Location Loc = AA->getLocation(LI);
+          MemoryLocation Loc = MemoryLocation::get(LI);
           if (AA->pointsToConstantMemory(Loc, /*OrLocal=*/true))
             continue;
         }
       } else if (StoreInst *SI = dyn_cast<StoreInst>(I)) {
         // Ignore non-volatile stores to local memory. (Atomic is okay here.)
         if (!SI->isVolatile()) {
-          AliasAnalysis::Location Loc = AA->getLocation(SI);
+          MemoryLocation Loc = MemoryLocation::get(SI);
           if (AA->pointsToConstantMemory(Loc, /*OrLocal=*/true))
             continue;
         }
       } else if (VAArgInst *VI = dyn_cast<VAArgInst>(I)) {
         // Ignore vaargs on local memory.
-        AliasAnalysis::Location Loc = AA->getLocation(VI);
+        MemoryLocation Loc = MemoryLocation::get(VI);
         if (AA->pointsToConstantMemory(Loc, /*OrLocal=*/true))
           continue;
       }
@@ -416,7 +415,6 @@ determinePointerReadAttrs(Argument *A,
                                                        
   SmallVector<Use*, 32> Worklist;
   SmallSet<Use*, 32> Visited;
-  int Count = 0;
 
   // inalloca arguments are always clobbered by the call.
   if (A->hasInAllocaAttr())
@@ -426,9 +424,6 @@ determinePointerReadAttrs(Argument *A,
   // We don't need to track IsWritten. If A is written to, return immediately.
 
   for (Use &U : A->uses()) {
-    if (Count++ >= 20)
-      return Attribute::None;
-
     Visited.insert(&U);
     Worklist.push_back(&U);
   }
@@ -703,10 +698,14 @@ bool FunctionAttrs::AddArgumentAttrs(const CallGraphSCC &SCC) {
     }
 
     if (ReadAttr != Attribute::None) {
-      AttrBuilder B;
+      AttrBuilder B, R;
       B.addAttribute(ReadAttr);
+      R.addAttribute(Attribute::ReadOnly)
+        .addAttribute(Attribute::ReadNone);
       for (unsigned i = 0, e = ArgumentSCC.size(); i != e; ++i) {
         Argument *A = ArgumentSCC[i]->Definition;
+        // Clear out existing readonly/readnone attributes
+        A->removeAttr(AttributeSet::get(A->getContext(), A->getArgNo() + 1, R));
         A->addAttr(AttributeSet::get(A->getContext(), A->getArgNo() + 1, B));
         ReadAttr == Attribute::ReadOnly ? ++NumReadOnlyArg : ++NumReadNoneArg;
         Changed = true;
@@ -755,8 +754,8 @@ bool FunctionAttrs::IsFunctionMallocLike(Function *F,
         }
         case Instruction::PHI: {
           PHINode *PN = cast<PHINode>(RVI);
-          for (int i = 0, e = PN->getNumIncomingValues(); i != e; ++i)
-            FlowsToReturn.insert(PN->getIncomingValue(i));
+          for (Value *IncValue : PN->incoming_values())
+            FlowsToReturn.insert(IncValue);
           continue;
         }
 
@@ -1233,11 +1232,8 @@ bool FunctionAttrs::inferPrototypeAttributes(Function &F) {
   case LibFunc::fseek:
   case LibFunc::ftell:
   case LibFunc::fgetc:
-  // @LOCALMOD-BEGIN:
-  // https://code.google.com/p/nativeclient/issues/detail?id=4089
-  case LibFunc::Fseeko:
-  case LibFunc::Ftello:
-  // @LOCALMOD-END
+  case LibFunc::fseeko:
+  case LibFunc::ftello:
   case LibFunc::fileno:
   case LibFunc::fflush:
   case LibFunc::fclose:
diff --git a/lib/Transforms/IPO/GlobalDCE.cpp b/lib/Transforms/IPO/GlobalDCE.cpp
index ba04c80508c..1cc657458d6 100644
--- a/lib/Transforms/IPO/GlobalDCE.cpp
+++ b/lib/Transforms/IPO/GlobalDCE.cpp
@@ -92,33 +92,28 @@ bool GlobalDCE::runOnModule(Module &M) {
       ComdatMembers.insert(std::make_pair(C, &GA));
 
   // Loop over the module, adding globals which are obviously necessary.
-  for (Module::iterator I = M.begin(), E = M.end(); I != E; ++I) {
-    Changed |= RemoveUnusedGlobalValue(*I);
+  for (Function &F : M) {
+    Changed |= RemoveUnusedGlobalValue(F);
     // Functions with external linkage are needed if they have a body
-    if (!I->isDeclaration() && !I->hasAvailableExternallyLinkage()) {
-      if (!I->isDiscardableIfUnused())
-        GlobalIsNeeded(I);
-    }
+    if (!F.isDeclaration() && !F.hasAvailableExternallyLinkage())
+      if (!F.isDiscardableIfUnused())
+        GlobalIsNeeded(&F);
   }
 
-  for (Module::global_iterator I = M.global_begin(), E = M.global_end();
-       I != E; ++I) {
-    Changed |= RemoveUnusedGlobalValue(*I);
+  for (GlobalVariable &GV : M.globals()) {
+    Changed |= RemoveUnusedGlobalValue(GV);
     // Externally visible & appending globals are needed, if they have an
     // initializer.
-    if (!I->isDeclaration() && !I->hasAvailableExternallyLinkage()) {
-      if (!I->isDiscardableIfUnused())
-        GlobalIsNeeded(I);
-    }
+    if (!GV.isDeclaration() && !GV.hasAvailableExternallyLinkage())
+      if (!GV.isDiscardableIfUnused())
+        GlobalIsNeeded(&GV);
   }
 
-  for (Module::alias_iterator I = M.alias_begin(), E = M.alias_end();
-       I != E; ++I) {
-    Changed |= RemoveUnusedGlobalValue(*I);
+  for (GlobalAlias &GA : M.aliases()) {
+    Changed |= RemoveUnusedGlobalValue(GA);
     // Externally visible aliases are needed.
-    if (!I->isDiscardableIfUnused()) {
-      GlobalIsNeeded(I);
-    }
+    if (!GA.isDiscardableIfUnused())
+      GlobalIsNeeded(&GA);
   }
 
   // Now that all globals which are needed are in the AliveGlobals set, we loop
@@ -126,52 +121,50 @@ bool GlobalDCE::runOnModule(Module &M) {
   //
 
   // The first pass is to drop initializers of global variables which are dead.
-  std::vector<GlobalVariable*> DeadGlobalVars;   // Keep track of dead globals
-  for (Module::global_iterator I = M.global_begin(), E = M.global_end();
-       I != E; ++I)
-    if (!AliveGlobals.count(I)) {
-      DeadGlobalVars.push_back(I);         // Keep track of dead globals
-      if (I->hasInitializer()) {
-        Constant *Init = I->getInitializer();
-        I->setInitializer(nullptr);
+  std::vector<GlobalVariable *> DeadGlobalVars; // Keep track of dead globals
+  for (GlobalVariable &GV : M.globals())
+    if (!AliveGlobals.count(&GV)) {
+      DeadGlobalVars.push_back(&GV);         // Keep track of dead globals
+      if (GV.hasInitializer()) {
+        Constant *Init = GV.getInitializer();
+        GV.setInitializer(nullptr);
         if (isSafeToDestroyConstant(Init))
           Init->destroyConstant();
       }
     }
 
   // The second pass drops the bodies of functions which are dead...
-  std::vector<Function*> DeadFunctions;
-  for (Module::iterator I = M.begin(), E = M.end(); I != E; ++I)
-    if (!AliveGlobals.count(I)) {
-      DeadFunctions.push_back(I);         // Keep track of dead globals
-      if (!I->isDeclaration())
-        I->deleteBody();
+  std::vector<Function *> DeadFunctions;
+  for (Function &F : M)
+    if (!AliveGlobals.count(&F)) {
+      DeadFunctions.push_back(&F);         // Keep track of dead globals
+      if (!F.isDeclaration())
+        F.deleteBody();
     }
 
   // The third pass drops targets of aliases which are dead...
   std::vector<GlobalAlias*> DeadAliases;
-  for (Module::alias_iterator I = M.alias_begin(), E = M.alias_end(); I != E;
-       ++I)
-    if (!AliveGlobals.count(I)) {
-      DeadAliases.push_back(I);
-      I->setAliasee(nullptr);
+  for (GlobalAlias &GA : M.aliases())
+    if (!AliveGlobals.count(&GA)) {
+      DeadAliases.push_back(&GA);
+      GA.setAliasee(nullptr);
     }
 
   if (!DeadFunctions.empty()) {
     // Now that all interferences have been dropped, delete the actual objects
     // themselves.
-    for (unsigned i = 0, e = DeadFunctions.size(); i != e; ++i) {
-      RemoveUnusedGlobalValue(*DeadFunctions[i]);
-      M.getFunctionList().erase(DeadFunctions[i]);
+    for (Function *F : DeadFunctions) {
+      RemoveUnusedGlobalValue(*F);
+      M.getFunctionList().erase(F);
     }
     NumFunctions += DeadFunctions.size();
     Changed = true;
   }
 
   if (!DeadGlobalVars.empty()) {
-    for (unsigned i = 0, e = DeadGlobalVars.size(); i != e; ++i) {
-      RemoveUnusedGlobalValue(*DeadGlobalVars[i]);
-      M.getGlobalList().erase(DeadGlobalVars[i]);
+    for (GlobalVariable *GV : DeadGlobalVars) {
+      RemoveUnusedGlobalValue(*GV);
+      M.getGlobalList().erase(GV);
     }
     NumVariables += DeadGlobalVars.size();
     Changed = true;
@@ -179,9 +172,9 @@ bool GlobalDCE::runOnModule(Module &M) {
 
   // Now delete any dead aliases.
   if (!DeadAliases.empty()) {
-    for (unsigned i = 0, e = DeadAliases.size(); i != e; ++i) {
-      RemoveUnusedGlobalValue(*DeadAliases[i]);
-      M.getAliasList().erase(DeadAliases[i]);
+    for (GlobalAlias *GA : DeadAliases) {
+      RemoveUnusedGlobalValue(*GA);
+      M.getAliasList().erase(GA);
     }
     NumAliases += DeadAliases.size();
     Changed = true;
@@ -228,12 +221,15 @@ void GlobalDCE::GlobalIsNeeded(GlobalValue *G) {
     if (F->hasPrologueData())
       MarkUsedGlobalsAsNeeded(F->getPrologueData());
 
-    for (Function::iterator BB = F->begin(), E = F->end(); BB != E; ++BB)
-      for (BasicBlock::iterator I = BB->begin(), E = BB->end(); I != E; ++I)
-        for (User::op_iterator U = I->op_begin(), E = I->op_end(); U != E; ++U)
-          if (GlobalValue *GV = dyn_cast<GlobalValue>(*U))
+    if (F->hasPersonalityFn())
+      MarkUsedGlobalsAsNeeded(F->getPersonalityFn());
+
+    for (BasicBlock &BB : *F)
+      for (Instruction &I : BB)
+        for (Use &U : I.operands())
+          if (GlobalValue *GV = dyn_cast<GlobalValue>(U))
             GlobalIsNeeded(GV);
-          else if (Constant *C = dyn_cast<Constant>(*U))
+          else if (Constant *C = dyn_cast<Constant>(U))
             MarkUsedGlobalsAsNeeded(C);
   }
 }
@@ -244,9 +240,9 @@ void GlobalDCE::MarkUsedGlobalsAsNeeded(Constant *C) {
 
   // Loop over all of the operands of the constant, adding any globals they
   // use to the list of needed globals.
-  for (User::op_iterator I = C->op_begin(), E = C->op_end(); I != E; ++I) {
+  for (Use &U : C->operands()) {
     // If we've already processed this constant there's no need to do it again.
-    Constant *Op = dyn_cast<Constant>(*I);
+    Constant *Op = dyn_cast<Constant>(U);
     if (Op && SeenConstants.insert(Op).second)
       MarkUsedGlobalsAsNeeded(Op);
   }
@@ -259,7 +255,8 @@ void GlobalDCE::MarkUsedGlobalsAsNeeded(Constant *C) {
 // might make it deader.
 //
 bool GlobalDCE::RemoveUnusedGlobalValue(GlobalValue &GV) {
-  if (GV.use_empty()) return false;
+  if (GV.use_empty())
+    return false;
   GV.removeDeadConstantUsers();
   return GV.use_empty();
 }
diff --git a/lib/Transforms/IPO/GlobalOpt.cpp b/lib/Transforms/IPO/GlobalOpt.cpp
index b8e397534f6..3602f359136 100644
--- a/lib/Transforms/IPO/GlobalOpt.cpp
+++ b/lib/Transforms/IPO/GlobalOpt.cpp
@@ -564,7 +564,7 @@ static GlobalVariable *SRAGlobal(GlobalVariable *GV, const DataLayout &DL) {
     if (Val >= NewGlobals.size()) Val = 0; // Out of bound array access.
 
     Value *NewPtr = NewGlobals[Val];
-    Type *NewTy = NewGlobals[Val]->getType();
+    Type *NewTy = NewGlobals[Val]->getValueType();
 
     // Form a shorter GEP if needed.
     if (GEP->getNumOperands() > 3) {
@@ -575,7 +575,6 @@ static GlobalVariable *SRAGlobal(GlobalVariable *GV, const DataLayout &DL) {
           Idxs.push_back(CE->getOperand(i));
         NewPtr =
             ConstantExpr::getGetElementPtr(NewTy, cast<Constant>(NewPtr), Idxs);
-        NewTy = GetElementPtrInst::getIndexedType(NewTy, Idxs);
       } else {
         GetElementPtrInst *GEPI = cast<GetElementPtrInst>(GEP);
         SmallVector<Value*, 8> Idxs;
@@ -583,8 +582,7 @@ static GlobalVariable *SRAGlobal(GlobalVariable *GV, const DataLayout &DL) {
         for (unsigned i = 3, e = GEPI->getNumOperands(); i != e; ++i)
           Idxs.push_back(GEPI->getOperand(i));
         NewPtr = GetElementPtrInst::Create(
-            NewPtr->getType()->getPointerElementType(), NewPtr, Idxs,
-            GEPI->getName() + "." + Twine(Val), GEPI);
+            NewTy, NewPtr, Idxs, GEPI->getName() + "." + Twine(Val), GEPI);
       }
     }
     GEP->replaceAllUsesWith(NewPtr);
@@ -2050,11 +2048,9 @@ isSimpleEnoughValueToCommitHelper(Constant *C,
   // Aggregate values are safe if all their elements are.
   if (isa<ConstantArray>(C) || isa<ConstantStruct>(C) ||
       isa<ConstantVector>(C)) {
-    for (unsigned i = 0, e = C->getNumOperands(); i != e; ++i) {
-      Constant *Op = cast<Constant>(C->getOperand(i));
-      if (!isSimpleEnoughValueToCommit(Op, SimpleConstants, DL))
+    for (Value *Op : C->operands())
+      if (!isSimpleEnoughValueToCommit(cast<Constant>(Op), SimpleConstants, DL))
         return false;
-    }
     return true;
   }
 
@@ -3141,3 +3137,4 @@ bool GlobalOpt::runOnModule(Module &M) {
 
   return Changed;
 }
+
diff --git a/lib/Transforms/IPO/IPO.cpp b/lib/Transforms/IPO/IPO.cpp
index fcacec3286f..50f56b0f2af 100644
--- a/lib/Transforms/IPO/IPO.cpp
+++ b/lib/Transforms/IPO/IPO.cpp
@@ -46,6 +46,7 @@ void llvm::initializeIPO(PassRegistry &Registry) {
   initializeStripDeadDebugInfoPass(Registry);
   initializeStripNonDebugSymbolsPass(Registry);
   initializeBarrierNoopPass(Registry);
+  initializeEliminateAvailableExternallyPass(Registry);
 }
 
 void LLVMInitializeIPO(LLVMPassRegistryRef R) {
diff --git a/lib/Transforms/IPO/Inliner.cpp b/lib/Transforms/IPO/Inliner.cpp
index 3aa4ee50427..09fd11d652e 100644
--- a/lib/Transforms/IPO/Inliner.cpp
+++ b/lib/Transforms/IPO/Inliner.cpp
@@ -93,19 +93,26 @@ static void AdjustCallerSSPLevel(Function *Caller, Function *Callee) {
   // clutter to the IR.
   AttrBuilder B;
   B.addAttribute(Attribute::StackProtect)
-    .addAttribute(Attribute::StackProtectStrong);
+    .addAttribute(Attribute::StackProtectStrong)
+    .addAttribute(Attribute::StackProtectReq);
   AttributeSet OldSSPAttr = AttributeSet::get(Caller->getContext(),
                                               AttributeSet::FunctionIndex,
                                               B);
 
-  if (Callee->hasFnAttribute(Attribute::StackProtectReq)) {
+  if (Callee->hasFnAttribute(Attribute::SafeStack)) {
+    Caller->removeAttributes(AttributeSet::FunctionIndex, OldSSPAttr);
+    Caller->addFnAttr(Attribute::SafeStack);
+  } else if (Callee->hasFnAttribute(Attribute::StackProtectReq) &&
+             !Caller->hasFnAttribute(Attribute::SafeStack)) {
     Caller->removeAttributes(AttributeSet::FunctionIndex, OldSSPAttr);
     Caller->addFnAttr(Attribute::StackProtectReq);
   } else if (Callee->hasFnAttribute(Attribute::StackProtectStrong) &&
+             !Caller->hasFnAttribute(Attribute::SafeStack) &&
              !Caller->hasFnAttribute(Attribute::StackProtectReq)) {
     Caller->removeAttributes(AttributeSet::FunctionIndex, OldSSPAttr);
     Caller->addFnAttr(Attribute::StackProtectStrong);
   } else if (Callee->hasFnAttribute(Attribute::StackProtect) &&
+             !Caller->hasFnAttribute(Attribute::SafeStack) &&
              !Caller->hasFnAttribute(Attribute::StackProtectReq) &&
              !Caller->hasFnAttribute(Attribute::StackProtectStrong))
     Caller->addFnAttr(Attribute::StackProtect);
@@ -192,8 +199,7 @@ static bool InlineCallIfPossible(CallSite CS, InlineFunctionInfo &IFI,
     // set to keep track of which "available" allocas are being used by this
     // function.  Also, AllocasForType can be empty of course!
     bool MergedAwayAlloca = false;
-    for (unsigned i = 0, e = AllocasForType.size(); i != e; ++i) {
-      AllocaInst *AvailableAlloca = AllocasForType[i];
+    for (AllocaInst *AvailableAlloca : AllocasForType) {
 
       unsigned Align1 = AI->getAlignment(),
                Align2 = AvailableAlloca->getAlignment();
@@ -431,8 +437,8 @@ bool Inliner::runOnSCC(CallGraphSCC &SCC) {
 
   SmallPtrSet<Function*, 8> SCCFunctions;
   DEBUG(dbgs() << "Inliner visiting SCC:");
-  for (CallGraphSCC::iterator I = SCC.begin(), E = SCC.end(); I != E; ++I) {
-    Function *F = (*I)->getFunction();
+  for (CallGraphNode *Node : SCC) {
+    Function *F = Node->getFunction();
     if (F) SCCFunctions.insert(F);
     DEBUG(dbgs() << " " << (F ? F->getName() : "INDIRECTNODE"));
   }
@@ -448,13 +454,13 @@ bool Inliner::runOnSCC(CallGraphSCC &SCC) {
   // index into the InlineHistory vector.
   SmallVector<std::pair<Function*, int>, 8> InlineHistory;
 
-  for (CallGraphSCC::iterator I = SCC.begin(), E = SCC.end(); I != E; ++I) {
-    Function *F = (*I)->getFunction();
+  for (CallGraphNode *Node : SCC) {
+    Function *F = Node->getFunction();
     if (!F) continue;
     
-    for (Function::iterator BB = F->begin(), E = F->end(); BB != E; ++BB)
-      for (BasicBlock::iterator I = BB->begin(), E = BB->end(); I != E; ++I) {
-        CallSite CS(cast<Value>(I));
+    for (BasicBlock &BB : *F)
+      for (Instruction &I : BB) {
+        CallSite CS(cast<Value>(&I));
         // If this isn't a call, or it is a call to an intrinsic, it can
         // never be inlined.
         if (!CS || isa<IntrinsicInst>(I))
@@ -463,8 +469,9 @@ bool Inliner::runOnSCC(CallGraphSCC &SCC) {
         // If this is a direct call to an external function, we can never inline
         // it.  If it is an indirect call, inlining may resolve it to be a
         // direct call, so we keep it.
-        if (CS.getCalledFunction() && CS.getCalledFunction()->isDeclaration())
-          continue;
+        if (Function *Callee = CS.getCalledFunction())
+          if (Callee->isDeclaration())
+            continue;
         
         CallSites.push_back(std::make_pair(CS, -1));
       }
@@ -475,7 +482,7 @@ bool Inliner::runOnSCC(CallGraphSCC &SCC) {
   // If there are no calls in this function, exit early.
   if (CallSites.empty())
     return false;
-  
+
   // Now that we have all of the call sites, move the ones to functions in the
   // current SCC to the end of the list.
   unsigned FirstCallInSCC = CallSites.size();
@@ -496,6 +503,7 @@ bool Inliner::runOnSCC(CallGraphSCC &SCC) {
     LocalChange = false;
     // Iterate over the outer loop because inlining functions can cause indirect
     // calls to become direct calls.
+    // CallSites may be modified inside so ranged for loop can not be used.
     for (unsigned CSi = 0; CSi != CallSites.size(); ++CSi) {
       CallSite CS = CallSites[CSi].first;
       
@@ -566,11 +574,8 @@ bool Inliner::runOnSCC(CallGraphSCC &SCC) {
           int NewHistoryID = InlineHistory.size();
           InlineHistory.push_back(std::make_pair(Callee, InlineHistoryID));
 
-          for (unsigned i = 0, e = InlineInfo.InlinedCalls.size();
-               i != e; ++i) {
-            Value *Ptr = InlineInfo.InlinedCalls[i];
+          for (Value *Ptr : InlineInfo.InlinedCalls)
             CallSites.push_back(std::make_pair(CallSite(Ptr), NewHistoryID));
-          }
         }
       }
       
@@ -587,7 +592,7 @@ bool Inliner::runOnSCC(CallGraphSCC &SCC) {
         DEBUG(dbgs() << "    -> Deleting dead function: "
               << Callee->getName() << "\n");
         CallGraphNode *CalleeNode = CG[Callee];
-        
+
         // Remove any call graph edges from the callee to its callees.
         CalleeNode->removeAllCalledFunctions();
         
@@ -625,11 +630,26 @@ bool Inliner::doFinalization(CallGraph &CG) {
 /// Remove dead functions that are not included in DNR (Do Not Remove) list.
 bool Inliner::removeDeadFunctions(CallGraph &CG, bool AlwaysInlineOnly) {
   SmallVector<CallGraphNode*, 16> FunctionsToRemove;
+  SmallVector<CallGraphNode *, 16> DeadFunctionsInComdats;
+  SmallDenseMap<const Comdat *, int, 16> ComdatEntriesAlive;
+
+  auto RemoveCGN = [&](CallGraphNode *CGN) {
+    // Remove any call graph edges from the function to its callees.
+    CGN->removeAllCalledFunctions();
+
+    // Remove any edges from the external node to the function's call graph
+    // node.  These edges might have been made irrelegant due to
+    // optimization of the program.
+    CG.getExternalCallingNode()->removeAnyCallEdgeTo(CGN);
+
+    // Removing the node for callee from the call graph and delete it.
+    FunctionsToRemove.push_back(CGN);
+  };
 
   // Scan for all of the functions, looking for ones that should now be removed
   // from the program.  Insert the dead ones in the FunctionsToRemove set.
-  for (CallGraph::iterator I = CG.begin(), E = CG.end(); I != E; ++I) {
-    CallGraphNode *CGN = I->second;
+  for (auto I : CG) {
+    CallGraphNode *CGN = I.second;
     Function *F = CGN->getFunction();
     if (!F || F->isDeclaration())
       continue;
@@ -651,20 +671,45 @@ bool Inliner::removeDeadFunctions(CallGraph &CG, bool AlwaysInlineOnly) {
     // without also dropping the other members of the COMDAT.
     // The inliner doesn't visit non-function entities which are in COMDAT
     // groups so it is unsafe to do so *unless* the linkage is local.
-    if (!F->hasLocalLinkage() && F->hasComdat())
-      continue;
-    
-    // Remove any call graph edges from the function to its callees.
-    CGN->removeAllCalledFunctions();
+    if (!F->hasLocalLinkage()) {
+      if (const Comdat *C = F->getComdat()) {
+        --ComdatEntriesAlive[C];
+        DeadFunctionsInComdats.push_back(CGN);
+        continue;
+      }
+    }
 
-    // Remove any edges from the external node to the function's call graph
-    // node.  These edges might have been made irrelegant due to
-    // optimization of the program.
-    CG.getExternalCallingNode()->removeAnyCallEdgeTo(CGN);
+    RemoveCGN(CGN);
+  }
+  if (!DeadFunctionsInComdats.empty()) {
+    // Count up all the entities in COMDAT groups
+    auto ComdatGroupReferenced = [&](const Comdat *C) {
+      auto I = ComdatEntriesAlive.find(C);
+      if (I != ComdatEntriesAlive.end())
+        ++(I->getSecond());
+    };
+    for (const Function &F : CG.getModule())
+      if (const Comdat *C = F.getComdat())
+        ComdatGroupReferenced(C);
+    for (const GlobalVariable &GV : CG.getModule().globals())
+      if (const Comdat *C = GV.getComdat())
+        ComdatGroupReferenced(C);
+    for (const GlobalAlias &GA : CG.getModule().aliases())
+      if (const Comdat *C = GA.getComdat())
+        ComdatGroupReferenced(C);
+    for (CallGraphNode *CGN : DeadFunctionsInComdats) {
+      Function *F = CGN->getFunction();
+      const Comdat *C = F->getComdat();
+      int NumAlive = ComdatEntriesAlive[C];
+      // We can remove functions in a COMDAT group if the entire group is dead.
+      assert(NumAlive >= 0);
+      if (NumAlive > 0)
+        continue;
 
-    // Removing the node for callee from the call graph and delete it.
-    FunctionsToRemove.push_back(CGN);
+      RemoveCGN(CGN);
+    }
   }
+
   if (FunctionsToRemove.empty())
     return false;
 
@@ -679,10 +724,8 @@ bool Inliner::removeDeadFunctions(CallGraph &CG, bool AlwaysInlineOnly) {
   FunctionsToRemove.erase(std::unique(FunctionsToRemove.begin(),
                                       FunctionsToRemove.end()),
                           FunctionsToRemove.end());
-  for (SmallVectorImpl<CallGraphNode *>::iterator I = FunctionsToRemove.begin(),
-                                                  E = FunctionsToRemove.end();
-       I != E; ++I) {
-    delete CG.removeFunctionFromModule(*I);
+  for (CallGraphNode *CGN : FunctionsToRemove) {
+    delete CG.removeFunctionFromModule(CGN);
     ++NumDeleted;
   }
   return true;
diff --git a/lib/Transforms/IPO/Internalize.cpp b/lib/Transforms/IPO/Internalize.cpp
index 7950163f757..ede5fb18d76 100644
--- a/lib/Transforms/IPO/Internalize.cpp
+++ b/lib/Transforms/IPO/Internalize.cpp
@@ -60,6 +60,10 @@ namespace {
     explicit InternalizePass();
     explicit InternalizePass(ArrayRef<const char *> ExportList);
     void LoadFile(const char *Filename);
+    bool maybeInternalize(GlobalValue &GV,
+                          const std::set<const Comdat *> &ExternalComdats);
+    void checkComdatVisibility(GlobalValue &GV,
+                               std::set<const Comdat *> &ExternalComdats);
     bool runOnModule(Module &M) override;
 
     void getAnalysisUsage(AnalysisUsage &AU) const override {
@@ -105,40 +109,85 @@ void InternalizePass::LoadFile(const char *Filename) {
   }
 }
 
-static bool shouldInternalize(const GlobalValue &GV,
-                              const std::set<std::string> &ExternalNames) {
+static bool isExternallyVisible(const GlobalValue &GV,
+                                const std::set<std::string> &ExternalNames) {
   // Function must be defined here
   if (GV.isDeclaration())
-    return false;
+    return true;
 
   // Available externally is really just a "declaration with a body".
   if (GV.hasAvailableExternallyLinkage())
-    return false;
+    return true;
 
   // Assume that dllexported symbols are referenced elsewhere
   if (GV.hasDLLExportStorageClass())
-    return false;
-
-  // Already has internal linkage
-  if (GV.hasLocalLinkage())
-    return false;
+    return true;
 
   // Marked to keep external?
-  if (ExternalNames.count(GV.getName()))
-    return false;
+  if (!GV.hasLocalLinkage() && ExternalNames.count(GV.getName()))
+    return true;
+
+  return false;
+}
 
+// Internalize GV if it is possible to do so, i.e. it is not externally visible
+// and is not a member of an externally visible comdat.
+bool InternalizePass::maybeInternalize(
+    GlobalValue &GV, const std::set<const Comdat *> &ExternalComdats) {
+  if (Comdat *C = GV.getComdat()) {
+    if (ExternalComdats.count(C))
+      return false;
+
+    // If a comdat is not externally visible we can drop it.
+    if (auto GO = dyn_cast<GlobalObject>(&GV))
+      GO->setComdat(nullptr);
+
+    if (GV.hasLocalLinkage())
+      return false;
+  } else {
+    if (GV.hasLocalLinkage())
+      return false;
+
+    if (isExternallyVisible(GV, ExternalNames))
+      return false;
+  }
+
+  GV.setVisibility(GlobalValue::DefaultVisibility);
+  GV.setLinkage(GlobalValue::InternalLinkage);
   return true;
 }
 
+// If GV is part of a comdat and is externally visible, keep track of its
+// comdat so that we don't internalize any of its members.
+void InternalizePass::checkComdatVisibility(
+    GlobalValue &GV, std::set<const Comdat *> &ExternalComdats) {
+  Comdat *C = GV.getComdat();
+  if (!C)
+    return;
+
+  if (isExternallyVisible(GV, ExternalNames))
+    ExternalComdats.insert(C);
+}
+
 bool InternalizePass::runOnModule(Module &M) {
   CallGraphWrapperPass *CGPass = getAnalysisIfAvailable<CallGraphWrapperPass>();
   CallGraph *CG = CGPass ? &CGPass->getCallGraph() : nullptr;
   CallGraphNode *ExternalNode = CG ? CG->getExternalCallingNode() : nullptr;
-  bool Changed = false;
 
   SmallPtrSet<GlobalValue *, 8> Used;
   collectUsedGlobalVariables(M, Used, false);
 
+  // Collect comdat visiblity information for the module.
+  std::set<const Comdat *> ExternalComdats;
+  if (!M.getComdatSymbolTable().empty()) {
+    for (Function &F : M)
+      checkComdatVisibility(F, ExternalComdats);
+    for (GlobalVariable &GV : M.globals())
+      checkComdatVisibility(GV, ExternalComdats);
+    for (GlobalAlias &GA : M.aliases())
+      checkComdatVisibility(GA, ExternalComdats);
+  }
+
   // We must assume that globals in llvm.used have a reference that not even
   // the linker can see, so we don't internalize them.
   // For llvm.compiler.used the situation is a bit fuzzy. The assembler and
@@ -154,17 +203,13 @@ bool InternalizePass::runOnModule(Module &M) {
 
   // Mark all functions not in the api as internal.
   for (Module::iterator I = M.begin(), E = M.end(); I != E; ++I) {
-    if (!shouldInternalize(*I, ExternalNames))
+    if (!maybeInternalize(*I, ExternalComdats))
       continue;
 
-    I->setVisibility(GlobalValue::DefaultVisibility);
-    I->setLinkage(GlobalValue::InternalLinkage);
-
     if (ExternalNode)
       // Remove a callgraph edge from the external node to this function.
       ExternalNode->removeOneAbstractEdgeTo((*CG)[I]);
 
-    Changed = true;
     ++NumFunctions;
     DEBUG(dbgs() << "Internalizing func " << I->getName() << "\n");
   }
@@ -191,12 +236,9 @@ bool InternalizePass::runOnModule(Module &M) {
   // internal as well.
   for (Module::global_iterator I = M.global_begin(), E = M.global_end();
        I != E; ++I) {
-    if (!shouldInternalize(*I, ExternalNames))
+    if (!maybeInternalize(*I, ExternalComdats))
       continue;
 
-    I->setVisibility(GlobalValue::DefaultVisibility);
-    I->setLinkage(GlobalValue::InternalLinkage);
-    Changed = true;
     ++NumGlobals;
     DEBUG(dbgs() << "Internalized gvar " << I->getName() << "\n");
   }
@@ -204,17 +246,20 @@ bool InternalizePass::runOnModule(Module &M) {
   // Mark all aliases that are not in the api as internal as well.
   for (Module::alias_iterator I = M.alias_begin(), E = M.alias_end();
        I != E; ++I) {
-    if (!shouldInternalize(*I, ExternalNames))
+    if (!maybeInternalize(*I, ExternalComdats))
       continue;
 
-    I->setVisibility(GlobalValue::DefaultVisibility);
-    I->setLinkage(GlobalValue::InternalLinkage);
-    Changed = true;
     ++NumAliases;
     DEBUG(dbgs() << "Internalized alias " << I->getName() << "\n");
   }
 
-  return Changed;
+  // We do not keep track of whether this pass changed the module because
+  // it adds unnecessary complexity:
+  // 1) This pass will generally be near the start of the pass pipeline, so
+  //    there will be no analyses to invalidate.
+  // 2) This pass will most likely end up changing the module and it isn't worth
+  //    worrying about optimizing the case where the module is unchanged.
+  return true;
 }
 
 ModulePass *llvm::createInternalizePass() { return new InternalizePass(); }
diff --git a/lib/Transforms/IPO/LowerBitSets.cpp b/lib/Transforms/IPO/LowerBitSets.cpp
index f3f8529911c..bf386a6c618 100644
--- a/lib/Transforms/IPO/LowerBitSets.cpp
+++ b/lib/Transforms/IPO/LowerBitSets.cpp
@@ -26,6 +26,8 @@
 #include "llvm/IR/Module.h"
 #include "llvm/IR/Operator.h"
 #include "llvm/Pass.h"
+#include "llvm/Support/Debug.h"
+#include "llvm/Support/raw_ostream.h"
 #include "llvm/Transforms/Utils/BasicBlockUtils.h"
 
 using namespace llvm;
@@ -90,6 +92,22 @@ bool BitSetInfo::containsValue(
   return false;
 }
 
+void BitSetInfo::print(raw_ostream &OS) const {
+  OS << "offset " << ByteOffset << " size " << BitSize << " align "
+     << (1 << AlignLog2);
+
+  if (isAllOnes()) {
+    OS << " all-ones\n";
+    return;
+  }
+
+  OS << " { ";
+  for (uint64_t B : Bits)
+    OS << B << ' ';
+  OS << "}\n";
+  return;
+}
+
 BitSetInfo BitSetBuilder::build() {
   if (Min > Max)
     Min = 0;
@@ -271,8 +289,10 @@ BitSetInfo LowerBitSets::buildBitSet(
     for (MDNode *Op : BitSetNM->operands()) {
       if (Op->getOperand(0) != BitSet || !Op->getOperand(1))
         continue;
-      auto OpGlobal = cast<GlobalVariable>(
+      auto OpGlobal = dyn_cast<GlobalVariable>(
           cast<ConstantAsMetadata>(Op->getOperand(1))->getValue());
+      if (!OpGlobal)
+        continue;
       uint64_t Offset =
           cast<ConstantInt>(cast<ConstantAsMetadata>(Op->getOperand(2))
                                 ->getValue())->getZExtValue();
@@ -358,8 +378,9 @@ void LowerBitSets::allocateByteArrays() {
     if (LinkerSubsectionsViaSymbols) {
       BAI->ByteArray->replaceAllUsesWith(GEP);
     } else {
-      GlobalAlias *Alias = GlobalAlias::create(
-          Int8Ty, 0, GlobalValue::PrivateLinkage, "bits", GEP, M);
+      GlobalAlias *Alias =
+          GlobalAlias::create(PointerType::getUnqual(Int8Ty),
+                              GlobalValue::PrivateLinkage, "bits", GEP, M);
       BAI->ByteArray->replaceAllUsesWith(Alias);
     }
     BAI->ByteArray->eraseFromParent();
@@ -401,7 +422,7 @@ Value *LowerBitSets::createBitSetTest(IRBuilder<> &B, BitSetInfo &BSI,
       // Each use of the byte array uses a different alias. This makes the
       // backend less likely to reuse previously computed byte array addresses,
       // improving the security of the CFI mechanism based on this pass.
-      ByteArray = GlobalAlias::create(BAI->ByteArray->getValueType(), 0,
+      ByteArray = GlobalAlias::create(BAI->ByteArray->getType(),
                                       GlobalValue::PrivateLinkage, "bits_use",
                                       ByteArray, M);
     }
@@ -529,6 +550,10 @@ void LowerBitSets::buildBitSetsFromGlobals(
   for (MDString *BS : BitSets) {
     // Build the bitset.
     BitSetInfo BSI = buildBitSet(BS, GlobalLayout);
+    DEBUG({
+      dbgs() << BS->getString() << ": ";
+      BSI.print(dbgs());
+    });
 
     ByteArrayInfo *BAI = 0;
 
@@ -553,10 +578,9 @@ void LowerBitSets::buildBitSetsFromGlobals(
     if (LinkerSubsectionsViaSymbols) {
       Globals[I]->replaceAllUsesWith(CombinedGlobalElemPtr);
     } else {
-      GlobalAlias *GAlias = GlobalAlias::create(
-          Globals[I]->getType()->getElementType(),
-          Globals[I]->getType()->getAddressSpace(), Globals[I]->getLinkage(),
-          "", CombinedGlobalElemPtr, M);
+      GlobalAlias *GAlias =
+          GlobalAlias::create(Globals[I]->getType(), Globals[I]->getLinkage(),
+                              "", CombinedGlobalElemPtr, M);
       GAlias->takeName(Globals[I]);
       Globals[I]->replaceAllUsesWith(GAlias);
     }
@@ -621,7 +645,7 @@ bool LowerBitSets::buildBitSets() {
         report_fatal_error("Bit set element must be a constant");
       auto OpGlobal = dyn_cast<GlobalVariable>(OpConstMD->getValue());
       if (!OpGlobal)
-        report_fatal_error("Bit set element must refer to global");
+        continue;
 
       auto OffsetConstMD = dyn_cast<ConstantAsMetadata>(Op->getOperand(2));
       if (!OffsetConstMD)
@@ -675,8 +699,10 @@ bool LowerBitSets::buildBitSets() {
         if (I == BitSetIndices.end())
           continue;
 
-        auto OpGlobal = cast<GlobalVariable>(
+        auto OpGlobal = dyn_cast<GlobalVariable>(
             cast<ConstantAsMetadata>(Op->getOperand(1))->getValue());
+        if (!OpGlobal)
+          continue;
         BitSetMembers[I->second].insert(GlobalIndices[OpGlobal]);
       }
     }
diff --git a/lib/Transforms/IPO/MergeFunctions.cpp b/lib/Transforms/IPO/MergeFunctions.cpp
index 596674d6816..7ece66302f9 100644
--- a/lib/Transforms/IPO/MergeFunctions.cpp
+++ b/lib/Transforms/IPO/MergeFunctions.cpp
@@ -389,11 +389,21 @@ class FunctionComparator {
 };
 
 class FunctionNode {
-  AssertingVH<Function> F;
+  mutable AssertingVH<Function> F;
 
 public:
   FunctionNode(Function *F) : F(F) {}
   Function *getFunc() const { return F; }
+
+  /// Replace the reference to the function F by the function G, assuming their
+  /// implementations are equal.
+  void replaceBy(Function *G) const {
+    assert(!(*this < FunctionNode(G)) && !(FunctionNode(G) < *this) &&
+           "The two functions must be equal");
+
+    F = G;
+  }
+
   void release() { F = 0; }
   bool operator<(const FunctionNode &RHS) const {
     return (FunctionComparator(F, RHS.getFunc()).compare()) == -1;
@@ -719,6 +729,15 @@ int FunctionComparator::cmpOperations(const Instruction *L,
                            R->getRawSubclassOptionalData()))
     return Res;
 
+  if (const AllocaInst *AI = dyn_cast<AllocaInst>(L)) {
+    if (int Res = cmpTypes(AI->getAllocatedType(),
+                           cast<AllocaInst>(R)->getAllocatedType()))
+      return Res;
+    if (int Res =
+            cmpNumbers(AI->getAlignment(), cast<AllocaInst>(R)->getAlignment()))
+      return Res;
+  }
+
   // We have two instructions of identical opcode and #operands.  Check to see
   // if all operands are the same type
   for (unsigned i = 0, e = L->getNumOperands(); i != e; ++i) {
@@ -1113,6 +1132,9 @@ class MergeFunctions : public ModulePass {
   /// Replace G with an alias to F. Deletes G.
   void writeAlias(Function *F, Function *G);
 
+  /// Replace function F with function G in the function tree.
+  void replaceFunctionInTree(FnTreeType::iterator &IterToF, Function *G);
+
   /// The set of all distinct functions. Use the insert() and remove() methods
   /// to modify it.
   FnTreeType FnTree;
@@ -1260,6 +1282,25 @@ void MergeFunctions::replaceDirectCallers(Function *Old, Function *New) {
     ++UI;
     CallSite CS(U->getUser());
     if (CS && CS.isCallee(U)) {
+      // Transfer the called function's attributes to the call site. Due to the
+      // bitcast we will 'loose' ABI changing attributes because the 'called
+      // function' is no longer a Function* but the bitcast. Code that looks up
+      // the attributes from the called function will fail.
+      auto &Context = New->getContext();
+      auto NewFuncAttrs = New->getAttributes();
+      auto CallSiteAttrs = CS.getAttributes();
+
+      CallSiteAttrs = CallSiteAttrs.addAttributes(
+          Context, AttributeSet::ReturnIndex, NewFuncAttrs.getRetAttributes());
+
+      for (unsigned argIdx = 0; argIdx < CS.arg_size(); argIdx++) {
+        AttributeSet Attrs = NewFuncAttrs.getParamAttributes(argIdx);
+        if (Attrs.getNumSlots())
+          CallSiteAttrs = CallSiteAttrs.addAttributes(Context, argIdx, Attrs);
+      }
+
+      CS.setAttributes(CallSiteAttrs);
+
       remove(CS.getInstruction()->getParent()->getParent());
       U->set(BitcastNew);
     }
@@ -1358,8 +1399,7 @@ void MergeFunctions::writeThunk(Function *F, Function *G) {
 // Replace G with an alias to F and delete G.
 void MergeFunctions::writeAlias(Function *F, Function *G) {
   PointerType *PTy = G->getType();
-  auto *GA = GlobalAlias::create(PTy->getElementType(), PTy->getAddressSpace(),
-                                 G->getLinkage(), "", F);
+  auto *GA = GlobalAlias::create(PTy, G->getLinkage(), "", F);
   F->setAlignment(std::max(F->getAlignment(), G->getAlignment()));
   GA->takeName(G);
   GA->setVisibility(G->getVisibility());
@@ -1376,28 +1416,26 @@ void MergeFunctions::mergeTwoFunctions(Function *F, Function *G) {
   if (F->mayBeOverridden()) {
     assert(G->mayBeOverridden());
 
-    if (HasGlobalAliases) {
-      // Make them both thunks to the same internal function.
-      Function *H = Function::Create(F->getFunctionType(), F->getLinkage(), "",
-                                     F->getParent());
-      H->copyAttributesFrom(F);
-      H->takeName(F);
-      removeUsers(F);
-      F->replaceAllUsesWith(H);
+    // Make them both thunks to the same internal function.
+    Function *H = Function::Create(F->getFunctionType(), F->getLinkage(), "",
+                                   F->getParent());
+    H->copyAttributesFrom(F);
+    H->takeName(F);
+    removeUsers(F);
+    F->replaceAllUsesWith(H);
 
-      unsigned MaxAlignment = std::max(G->getAlignment(), H->getAlignment());
+    unsigned MaxAlignment = std::max(G->getAlignment(), H->getAlignment());
 
+    if (HasGlobalAliases) {
       writeAlias(F, G);
       writeAlias(F, H);
-
-      F->setAlignment(MaxAlignment);
-      F->setLinkage(GlobalValue::PrivateLinkage);
     } else {
-      // We can't merge them. Instead, pick one and update all direct callers
-      // to call it and hope that we improve the instruction cache hit rate.
-      replaceDirectCallers(G, F);
+      writeThunk(F, G);
+      writeThunk(F, H);
     }
 
+    F->setAlignment(MaxAlignment);
+    F->setLinkage(GlobalValue::PrivateLinkage);
     ++NumDoubleWeak;
   } else {
     writeThunkOrAlias(F, G);
@@ -1406,6 +1444,21 @@ void MergeFunctions::mergeTwoFunctions(Function *F, Function *G) {
   ++NumFunctionsMerged;
 }
 
+/// Replace function F for function G in the map.
+void MergeFunctions::replaceFunctionInTree(FnTreeType::iterator &IterToF,
+                                           Function *G) {
+  Function *F = IterToF->getFunc();
+
+  // A total order is already guaranteed otherwise because we process strong
+  // functions before weak functions.
+  assert(((F->mayBeOverridden() && G->mayBeOverridden()) ||
+          (!F->mayBeOverridden() && !G->mayBeOverridden())) &&
+         "Only change functions if both are strong or both are weak");
+  (void)F;
+
+  IterToF->replaceBy(G);
+}
+
 // Insert a ComparableFunction into the FnTree, or merge it away if equal to one
 // that was already inserted.
 bool MergeFunctions::insert(Function *NewFunction) {
@@ -1431,6 +1484,22 @@ bool MergeFunctions::insert(Function *NewFunction) {
     }
   }
 
+  // Impose a total order (by name) on the replacement of functions. This is
+  // important when operating on more than one module independently to prevent
+  // cycles of thunks calling each other when the modules are linked together.
+  //
+  // When one function is weak and the other is strong there is an order imposed
+  // already. We process strong functions before weak functions.
+  if ((OldF.getFunc()->mayBeOverridden() && NewFunction->mayBeOverridden()) ||
+      (!OldF.getFunc()->mayBeOverridden() && !NewFunction->mayBeOverridden()))
+    if (OldF.getFunc()->getName() > NewFunction->getName()) {
+      // Swap the two functions.
+      Function *F = OldF.getFunc();
+      replaceFunctionInTree(Result.first, NewFunction);
+      NewFunction = F;
+      assert(OldF.getFunc() != F && "Must have swapped the functions.");
+    }
+
   // Never thunk a strong function to a weak function.
   assert(!OldF.getFunc()->mayBeOverridden() || NewFunction->mayBeOverridden());
 
@@ -1457,7 +1526,7 @@ void MergeFunctions::remove(Function *F) {
   if (Erased) {
     DEBUG(dbgs() << "Removed " << F->getName()
                  << " from set and deferred it.\n");
-    Deferred.push_back(F);
+    Deferred.emplace_back(F);
   }
 }
 
@@ -1466,6 +1535,8 @@ void MergeFunctions::remove(Function *F) {
 void MergeFunctions::removeUsers(Value *V) {
   std::vector<Value *> Worklist;
   Worklist.push_back(V);
+  SmallSet<Value*, 8> Visited;
+  Visited.insert(V);
   while (!Worklist.empty()) {
     Value *V = Worklist.back();
     Worklist.pop_back();
@@ -1476,8 +1547,10 @@ void MergeFunctions::removeUsers(Value *V) {
       } else if (isa<GlobalValue>(U)) {
         // do nothing
       } else if (Constant *C = dyn_cast<Constant>(U)) {
-        for (User *UU : C->users())
-          Worklist.push_back(UU);
+        for (User *UU : C->users()) {
+          if (!Visited.insert(UU).second)
+            Worklist.push_back(UU);
+        }
       }
     }
   }
diff --git a/lib/Transforms/IPO/PassManagerBuilder.cpp b/lib/Transforms/IPO/PassManagerBuilder.cpp
index 502451bf432..36ae6b2d717 100644
--- a/lib/Transforms/IPO/PassManagerBuilder.cpp
+++ b/lib/Transforms/IPO/PassManagerBuilder.cpp
@@ -85,12 +85,20 @@ static cl::opt<bool> EnableLoopInterchange(
     "enable-loopinterchange", cl::init(false), cl::Hidden,
     cl::desc("Enable the new, experimental LoopInterchange Pass"));
 
+static cl::opt<bool> EnableLoopDistribute(
+    "enable-loop-distribute", cl::init(false), cl::Hidden,
+    cl::desc("Enable the new, experimental LoopDistribution Pass"));
+
+static cl::opt<bool> EnableNonLTOGlobalsModRef(
+    "enable-non-lto-gmr", cl::init(false), cl::Hidden,
+    cl::desc(
+        "Enable the GlobalsModRef AliasAnalysis outside of the LTO pipeline."));
+
 PassManagerBuilder::PassManagerBuilder() {
     OptLevel = 2;
     SizeLevel = 0;
     LibraryInfo = nullptr;
     Inliner = nullptr;
-    DisableTailCalls = false;
     DisableUnitAtATime = false;
     DisableUnrollLoops = false;
     BBVectorize = RunBBVectorization;
@@ -102,6 +110,7 @@ PassManagerBuilder::PassManagerBuilder() {
     VerifyInput = false;
     VerifyOutput = false;
     MergeFunctions = false;
+    PrepareForLTO = false;
 }
 
 PassManagerBuilder::~PassManagerBuilder() {
@@ -209,6 +218,12 @@ void PassManagerBuilder::populateModulePassManager(
     MPM.add(createCFGSimplificationPass());   // Clean up after IPCP & DAE
   }
 
+  if (EnableNonLTOGlobalsModRef)
+    // We add a module alias analysis pass here. In part due to bugs in the
+    // analysis infrastructure this "works" in that the analysis stays alive
+    // for the entire SCC pass run below.
+    MPM.add(createGlobalsModRefPass());
+
   // Start of CallGraph SCC passes.
   if (!DisableUnitAtATime)
     MPM.add(createPruneEHPass());             // Remove dead EH info
@@ -234,8 +249,7 @@ void PassManagerBuilder::populateModulePassManager(
   MPM.add(createInstructionCombiningPass());  // Combine silly seq's
   addExtensionsToPM(EP_Peephole, MPM);
 
-  if (!DisableTailCalls)
-    MPM.add(createTailCallEliminationPass()); // Eliminate tail calls
+  MPM.add(createTailCallEliminationPass()); // Eliminate tail calls
   MPM.add(createCFGSimplificationPass());     // Merge & remove BBs
   MPM.add(createReassociatePass());           // Reassociate expressions
   // Rotate Loop - disable header duplication at -Oz
@@ -246,9 +260,10 @@ void PassManagerBuilder::populateModulePassManager(
   MPM.add(createIndVarSimplifyPass());        // Canonicalize indvars
   MPM.add(createLoopIdiomPass());             // Recognize idioms like memset.
   MPM.add(createLoopDeletionPass());          // Delete dead loops
-  if (EnableLoopInterchange)
+  if (EnableLoopInterchange) {
     MPM.add(createLoopInterchangePass()); // Interchange loops
-
+    MPM.add(createCFGSimplificationPass());
+  }
   if (!DisableUnrollLoops)
     MPM.add(createSimpleLoopUnrollPass());    // Unroll small loops
   addExtensionsToPM(EP_LoopOptimizerEnd, MPM);
@@ -311,13 +326,38 @@ void PassManagerBuilder::populateModulePassManager(
   // we must insert a no-op module pass to reset the pass manager.
   MPM.add(createBarrierNoopPass());
 
+  if (EnableNonLTOGlobalsModRef)
+    // We add a fresh GlobalsModRef run at this point. This is particularly
+    // useful as the above will have inlined, DCE'ed, and function-attr
+    // propagated everything. We should at this point have a reasonably minimal
+    // and richly annotated call graph. By computing aliasing and mod/ref
+    // information for all local globals here, the late loop passes and notably
+    // the vectorizer will be able to use them to help recognize vectorizable
+    // memory operations.
+    //
+    // Note that this relies on a bug in the pass manager which preserves
+    // a module analysis into a function pass pipeline (and throughout it) so
+    // long as the first function pass doesn't invalidate the module analysis.
+    // Thus both Float2Int and LoopRotate have to preserve AliasAnalysis for
+    // this to work. Fortunately, it is trivial to preserve AliasAnalysis
+    // (doing nothing preserves it as it is required to be conservatively
+    // correct in the face of IR changes).
+    MPM.add(createGlobalsModRefPass());
+
   if (RunFloat2Int)
     MPM.add(createFloat2IntPass());
 
+  addExtensionsToPM(EP_VectorizerStart, MPM);
+
   // Re-rotate loops in all our loop nests. These may have fallout out of
   // rotated form due to GVN or other transformations, and the vectorizer relies
-  // on the rotated form.
-  MPM.add(createLoopRotatePass());
+  // on the rotated form. Disable header duplication at -Oz.
+  MPM.add(createLoopRotatePass(SizeLevel == 2 ? 0 : -1));
+
+  // Distribute loops to allow partial vectorization.  I.e. isolate dependences
+  // into separate loop that would otherwise inhibit vectorization.
+  if (EnableLoopDistribute)
+    MPM.add(createLoopDistributePass());
 
   MPM.add(createLoopVectorizePass(DisableUnrollLoops, LoopVectorize));
   // FIXME: Because of #pragma vectorize enable, the passes below are always
@@ -372,9 +412,8 @@ void PassManagerBuilder::populateModulePassManager(
   if (!DisableUnrollLoops) {
     MPM.add(createLoopUnrollPass());    // Unroll small loops
 
-    // This is a barrier pass to avoid combine LICM pass and loop unroll pass
-    // within same loop pass manager.
-    MPM.add(createInstructionSimplifierPass());
+    // LoopUnroll may generate some redundency to cleanup.
+    MPM.add(createInstructionCombiningPass());
 
     // Runtime unrolling will introduce runtime check in loop prologue. If the
     // unrolled loop is a inner loop, then the prologue will be inside the
@@ -394,6 +433,17 @@ void PassManagerBuilder::populateModulePassManager(
     // GlobalOpt already deletes dead functions and globals, at -O2 try a
     // late pass of GlobalDCE.  It is capable of deleting dead cycles.
     if (OptLevel > 1) {
+      if (!PrepareForLTO) {
+        // Remove avail extern fns and globals definitions if we aren't
+        // compiling an object file for later LTO. For LTO we want to preserve
+        // these so they are eligible for inlining at link-time. Note if they
+        // are unreferenced they will be removed by GlobalDCE below, so
+        // this only impacts referenced available externally globals.
+        // Eventually they will be suppressed during codegen, but eliminating
+        // here enables more opportunity for GlobalDCE as it may make
+        // globals referenced by available external functions dead.
+        MPM.add(createEliminateAvailableExternallyPass());
+      }
       MPM.add(createGlobalDCEPass());         // Remove dead fns and globals.
       MPM.add(createConstantMergePass());     // Merge dup global constants
     }
diff --git a/lib/Transforms/IPO/PruneEH.cpp b/lib/Transforms/IPO/PruneEH.cpp
index 1943b930cbf..b2f1010c9a0 100644
--- a/lib/Transforms/IPO/PruneEH.cpp
+++ b/lib/Transforms/IPO/PruneEH.cpp
@@ -25,6 +25,7 @@
 #include "llvm/IR/CFG.h"
 #include "llvm/IR/Constants.h"
 #include "llvm/IR/Function.h"
+#include "llvm/IR/InlineAsm.h"
 #include "llvm/IR/Instructions.h"
 #include "llvm/IR/IntrinsicInst.h"
 #include "llvm/IR/LLVMContext.h"
@@ -97,42 +98,54 @@ bool PruneEH::runOnSCC(CallGraphSCC &SCC) {
     } else {
       bool CheckUnwind = !SCCMightUnwind && !F->doesNotThrow();
       bool CheckReturn = !SCCMightReturn && !F->doesNotReturn();
+      // Determine if we should scan for InlineAsm in a naked function as it
+      // is the only way to return without a ReturnInst.  Only do this for
+      // no-inline functions as functions which may be inlined cannot
+      // meaningfully return via assembly.
+      bool CheckReturnViaAsm = CheckReturn &&
+                               F->hasFnAttribute(Attribute::Naked) &&
+                               F->hasFnAttribute(Attribute::NoInline);
 
       if (!CheckUnwind && !CheckReturn)
         continue;
 
-      // Check to see if this function performs an unwind or calls an
-      // unwinding function.
-      for (Function::iterator BB = F->begin(), E = F->end(); BB != E; ++BB) {
-        if (CheckUnwind && isa<ResumeInst>(BB->getTerminator())) {
-          // Uses unwind / resume!
+      for (const BasicBlock &BB : *F) {
+        const TerminatorInst *TI = BB.getTerminator();
+        if (CheckUnwind && TI->mayThrow()) {
           SCCMightUnwind = true;
-        } else if (CheckReturn && isa<ReturnInst>(BB->getTerminator())) {
+        } else if (CheckReturn && isa<ReturnInst>(TI)) {
           SCCMightReturn = true;
         }
 
-        // Invoke instructions don't allow unwinding to continue, so we are
-        // only interested in call instructions.
-        if (CheckUnwind && !SCCMightUnwind)
-          for (BasicBlock::iterator I = BB->begin(), E = BB->end(); I != E; ++I)
-            if (CallInst *CI = dyn_cast<CallInst>(I)) {
-              if (CI->doesNotThrow()) {
-                // This call cannot throw.
-              } else if (Function *Callee = CI->getCalledFunction()) {
+        for (const Instruction &I : BB) {
+          if ((!CheckUnwind || SCCMightUnwind) &&
+              (!CheckReturnViaAsm || SCCMightReturn))
+            break;
+
+          // Check to see if this function performs an unwind or calls an
+          // unwinding function.
+          if (CheckUnwind && !SCCMightUnwind && I.mayThrow()) {
+            bool InstMightUnwind = true;
+            if (const auto *CI = dyn_cast<CallInst>(&I)) {
+              if (Function *Callee = CI->getCalledFunction()) {
                 CallGraphNode *CalleeNode = CG[Callee];
-                // If the callee is outside our current SCC then we may
-                // throw because it might.
-                if (!SCCNodes.count(CalleeNode)) {
-                  SCCMightUnwind = true;
-                  break;
-                }
-              } else {
-                // Indirect call, it might throw.
-                SCCMightUnwind = true;
-                break;
+                // If the callee is outside our current SCC then we may throw
+                // because it might.  If it is inside, do nothing.
+                if (SCCNodes.count(CalleeNode) > 0)
+                  InstMightUnwind = false;
               }
             }
-        if (SCCMightUnwind && SCCMightReturn) break;
+            SCCMightUnwind |= InstMightUnwind;
+          }
+          if (CheckReturnViaAsm && !SCCMightReturn)
+            if (auto ICS = ImmutableCallSite(&I))
+              if (const auto *IA = dyn_cast<InlineAsm>(ICS.getCalledValue()))
+                if (IA->hasSideEffects())
+                  SCCMightReturn = true;
+        }
+
+        if (SCCMightUnwind && SCCMightReturn)
+          break;
       }
     }
   }
@@ -177,7 +190,7 @@ bool PruneEH::SimplifyFunction(Function *F) {
   bool MadeChange = false;
   for (Function::iterator BB = F->begin(), E = F->end(); BB != E; ++BB) {
     if (InvokeInst *II = dyn_cast<InvokeInst>(BB->getTerminator()))
-      if (II->doesNotThrow() && canSimplifyInvokeNoUnwind(II)) {
+      if (II->doesNotThrow() && canSimplifyInvokeNoUnwind(F)) {
         SmallVector<Value*, 8> Args(II->op_begin(), II->op_end() - 3);
         // Insert a call instruction before the invoke.
         CallInst *Call = CallInst::Create(II->getCalledValue(), Args, "", II);
diff --git a/lib/Transforms/IPO/StripSymbols.cpp b/lib/Transforms/IPO/StripSymbols.cpp
index ad7c5a08765..a4f30c58f93 100644
--- a/lib/Transforms/IPO/StripSymbols.cpp
+++ b/lib/Transforms/IPO/StripSymbols.cpp
@@ -142,9 +142,9 @@ static bool OnlyUsedBy(Value *V, Value *Usr) {
 static void RemoveDeadConstant(Constant *C) {
   assert(C->use_empty() && "Constant is not dead!");
   SmallPtrSet<Constant*, 4> Operands;
-  for (unsigned i = 0, e = C->getNumOperands(); i != e; ++i)
-    if (OnlyUsedBy(C->getOperand(i), C))
-      Operands.insert(cast<Constant>(C->getOperand(i)));
+  for (Value *Op : C->operands())
+    if (OnlyUsedBy(Op, C))
+      Operands.insert(cast<Constant>(Op));
   if (GlobalVariable *GV = dyn_cast<GlobalVariable>(C)) {
     if (!GV->hasLocalLinkage()) return;   // Don't delete non-static globals.
     GV->eraseFromParent();
@@ -305,13 +305,10 @@ bool StripDeadDebugInfo::runOnModule(Module &M) {
   SmallVector<Metadata *, 64> LiveSubprograms;
   DenseSet<const MDNode *> VisitedSet;
 
-  for (MDCompileUnit *DIC : F.compile_units()) {
+  for (DICompileUnit *DIC : F.compile_units()) {
     // Create our live subprogram list.
-    MDSubprogramArray SPs = DIC->getSubprograms();
     bool SubprogramChange = false;
-    for (unsigned i = 0, e = SPs.size(); i != e; ++i) {
-      DISubprogram DISP = SPs[i];
-
+    for (DISubprogram *DISP : DIC->getSubprograms()) {
       // Make sure we visit each subprogram only once.
       if (!VisitedSet.insert(DISP).second)
         continue;
@@ -324,11 +321,8 @@ bool StripDeadDebugInfo::runOnModule(Module &M) {
     }
 
     // Create our live global variable list.
-    MDGlobalVariableArray GVs = DIC->getGlobalVariables();
     bool GlobalVariableChange = false;
-    for (unsigned i = 0, e = GVs.size(); i != e; ++i) {
-      DIGlobalVariable DIG = GVs[i];
-
+    for (DIGlobalVariable *DIG : DIC->getGlobalVariables()) {
       // Make sure we only visit each global variable only once.
       if (!VisitedSet.insert(DIG).second)
         continue;
diff --git a/lib/Transforms/InstCombine/InstCombineAddSub.cpp b/lib/Transforms/InstCombine/InstCombineAddSub.cpp
index c608f84bc7b..a8d01725517 100644
--- a/lib/Transforms/InstCombine/InstCombineAddSub.cpp
+++ b/lib/Transforms/InstCombine/InstCombineAddSub.cpp
@@ -1160,20 +1160,8 @@ Instruction *InstCombiner::visitAdd(BinaryOperator &I) {
     return ReplaceInstUsesWith(I, V);
 
   // A+B --> A|B iff A and B have no bits set in common.
-  if (IntegerType *IT = dyn_cast<IntegerType>(I.getType())) {
-    APInt LHSKnownOne(IT->getBitWidth(), 0);
-    APInt LHSKnownZero(IT->getBitWidth(), 0);
-    computeKnownBits(LHS, LHSKnownZero, LHSKnownOne, 0, &I);
-    if (LHSKnownZero != 0) {
-      APInt RHSKnownOne(IT->getBitWidth(), 0);
-      APInt RHSKnownZero(IT->getBitWidth(), 0);
-      computeKnownBits(RHS, RHSKnownZero, RHSKnownOne, 0, &I);
-
-      // No bits in common -> bitwise or.
-      if ((LHSKnownZero|RHSKnownZero).isAllOnesValue())
-        return BinaryOperator::CreateOr(LHS, RHS);
-    }
-  }
+  if (haveNoCommonBitsSet(LHS, RHS, DL, AC, &I, DT))
+    return BinaryOperator::CreateOr(LHS, RHS);
 
   if (Constant *CRHS = dyn_cast<Constant>(RHS)) {
     Value *X;
@@ -1586,6 +1574,19 @@ Instruction *InstCombiner::visitSub(BinaryOperator &I) {
           CI->getValue() == I.getType()->getPrimitiveSizeInBits() - 1)
         return BinaryOperator::CreateLShr(X, CI);
     }
+
+    // Turn this into a xor if LHS is 2^n-1 and the remaining bits are known
+    // zero.
+    APInt IntVal = C->getValue();
+    if ((IntVal + 1).isPowerOf2()) {
+      unsigned BitWidth = I.getType()->getScalarSizeInBits();
+      APInt KnownZero(BitWidth, 0);
+      APInt KnownOne(BitWidth, 0);
+      computeKnownBits(&I, KnownZero, KnownOne, 0, &I);
+      if ((IntVal | KnownZero).isAllOnesValue()) {
+        return BinaryOperator::CreateXor(Op1, C);
+      }
+    }
   }
 
 
diff --git a/lib/Transforms/InstCombine/InstCombineCalls.cpp b/lib/Transforms/InstCombine/InstCombineCalls.cpp
index 56b6cd30178..8083e6a72e7 100644
--- a/lib/Transforms/InstCombine/InstCombineCalls.cpp
+++ b/lib/Transforms/InstCombine/InstCombineCalls.cpp
@@ -13,6 +13,7 @@
 
 #include "InstCombineInternal.h"
 #include "llvm/ADT/Statistic.h"
+#include "llvm/Analysis/InstructionSimplify.h"
 #include "llvm/Analysis/MemoryBuiltins.h"
 #include "llvm/IR/CallSite.h"
 #include "llvm/IR/Dominators.h"
@@ -66,8 +67,7 @@ Instruction *InstCombiner::SimplifyMemTransfer(MemIntrinsic *MI) {
   unsigned CopyAlign = MI->getAlignment();
 
   if (CopyAlign < MinAlign) {
-    MI->setAlignment(ConstantInt::get(MI->getAlignmentType(),
-                                             MinAlign, false));
+    MI->setAlignment(ConstantInt::get(MI->getAlignmentType(), MinAlign, false));
     return MI;
   }
 
@@ -197,12 +197,30 @@ Instruction *InstCombiner::SimplifyMemSet(MemSetInst *MI) {
   return nullptr;
 }
 
+static Value *SimplifyX86extend(const IntrinsicInst &II,
+                                InstCombiner::BuilderTy &Builder,
+                                bool SignExtend) {
+  VectorType *SrcTy = cast<VectorType>(II.getArgOperand(0)->getType());
+  VectorType *DstTy = cast<VectorType>(II.getType());
+  unsigned NumDstElts = DstTy->getNumElements();
+
+  // Extract a subvector of the first NumDstElts lanes and sign/zero extend.
+  SmallVector<int, 8> ShuffleMask;
+  for (int i = 0; i != (int)NumDstElts; ++i)
+    ShuffleMask.push_back(i);
+
+  Value *SV = Builder.CreateShuffleVector(II.getArgOperand(0),
+                                          UndefValue::get(SrcTy), ShuffleMask);
+  return SignExtend ? Builder.CreateSExt(SV, DstTy)
+                    : Builder.CreateZExt(SV, DstTy);
+}
+
 static Value *SimplifyX86insertps(const IntrinsicInst &II,
                                   InstCombiner::BuilderTy &Builder) {
   if (auto *CInt = dyn_cast<ConstantInt>(II.getArgOperand(2))) {
     VectorType *VecTy = cast<VectorType>(II.getType());
-    ConstantAggregateZero *ZeroVector = ConstantAggregateZero::get(VecTy);
-    
+    assert(VecTy->getNumElements() == 4 && "insertps with wrong vector type");
+
     // The immediate permute control byte looks like this:
     //    [3:0] - zero mask for each 32-bit lane
     //    [5:4] - select one 32-bit destination lane
@@ -213,25 +231,42 @@ static Value *SimplifyX86insertps(const IntrinsicInst &II,
     uint8_t DestLane = (Imm >> 4) & 0x3;
     uint8_t SourceLane = (Imm >> 6) & 0x3;
 
+    ConstantAggregateZero *ZeroVector = ConstantAggregateZero::get(VecTy);
+
     // If all zero mask bits are set, this was just a weird way to
     // generate a zero vector.
     if (ZMask == 0xf)
       return ZeroVector;
-    
-    // TODO: Model this case as two shuffles or a 'logical and' plus shuffle?
-    if (ZMask)
-      return nullptr;
 
-    assert(VecTy->getNumElements() == 4 && "insertps with wrong vector type");
-
-    // If we're not zeroing anything, this is a single shuffle.
-    // Replace the selected destination lane with the selected source lane.
-    // For all other lanes, pass the first source bits through.
+    // Initialize by passing all of the first source bits through.
     int ShuffleMask[4] = { 0, 1, 2, 3 };
-    ShuffleMask[DestLane] = SourceLane + 4;
-    
-    return Builder.CreateShuffleVector(II.getArgOperand(0), II.getArgOperand(1),
-                                       ShuffleMask);
+
+    // We may replace the second operand with the zero vector.
+    Value *V1 = II.getArgOperand(1);
+
+    if (ZMask) {
+      // If the zero mask is being used with a single input or the zero mask
+      // overrides the destination lane, this is a shuffle with the zero vector.
+      if ((II.getArgOperand(0) == II.getArgOperand(1)) ||
+          (ZMask & (1 << DestLane))) {
+        V1 = ZeroVector;
+        // We may still move 32-bits of the first source vector from one lane
+        // to another.
+        ShuffleMask[DestLane] = SourceLane;
+        // The zero mask may override the previous insert operation.
+        for (unsigned i = 0; i < 4; ++i)
+          if ((ZMask >> i) & 0x1)
+            ShuffleMask[i] = i + 4;
+      } else {
+        // TODO: Model this case as 2 shuffles or a 'logical and' plus shuffle?
+        return nullptr;
+      }
+    } else {
+      // Replace the selected destination lane with the selected source lane.
+      ShuffleMask[DestLane] = SourceLane + 4;
+    }
+
+    return Builder.CreateShuffleVector(II.getArgOperand(0), V1, ShuffleMask);
   }
   return nullptr;
 }
@@ -271,7 +306,7 @@ static Value *SimplifyX86vperm2(const IntrinsicInst &II,
     // The high bit of the selection field chooses the 1st or 2nd operand.
     bool LowInputSelect = Imm & 0x02;
     bool HighInputSelect = Imm & 0x20;
-    
+
     // The low bit of the selection field chooses the low or high half
     // of the selected operand.
     bool LowHalfSelect = Imm & 0x01;
@@ -280,11 +315,11 @@ static Value *SimplifyX86vperm2(const IntrinsicInst &II,
     // Determine which operand(s) are actually in use for this instruction.
     Value *V0 = LowInputSelect ? II.getArgOperand(1) : II.getArgOperand(0);
     Value *V1 = HighInputSelect ? II.getArgOperand(1) : II.getArgOperand(0);
-    
+
     // If needed, replace operands based on zero mask.
     V0 = LowHalfZero ? ZeroVector : V0;
     V1 = HighHalfZero ? ZeroVector : V1;
-    
+
     // Permute low half of result.
     unsigned StartIndex = LowHalfSelect ? HalfSize : 0;
     for (unsigned i = 0; i < HalfSize; ++i)
@@ -306,6 +341,11 @@ static Value *SimplifyX86vperm2(const IntrinsicInst &II,
 /// the heavy lifting.
 ///
 Instruction *InstCombiner::visitCallInst(CallInst &CI) {
+  auto Args = CI.arg_operands();
+  if (Value *V = SimplifyCall(CI.getCalledValue(), Args.begin(), Args.end(), DL,
+                              TLI, DT, AC))
+    return ReplaceInstUsesWith(CI, V);
+
   if (isFreeCall(&CI, TLI))
     return visitFree(CI);
 
@@ -607,9 +647,12 @@ Instruction *InstCombiner::visitCallInst(CallInst &CI) {
     // Turn PPC QPX qvlfs -> load if the pointer is known aligned.
     if (getOrEnforceKnownAlignment(II->getArgOperand(0), 16, DL, II, AC, DT) >=
         16) {
+      Type *VTy = VectorType::get(Builder->getFloatTy(),
+                                  II->getType()->getVectorNumElements());
       Value *Ptr = Builder->CreateBitCast(II->getArgOperand(0),
-                                         PointerType::getUnqual(II->getType()));
-      return new LoadInst(Ptr);
+                                         PointerType::getUnqual(VTy));
+      Value *Load = Builder->CreateLoad(Ptr);
+      return new FPExtInst(Load, II->getType());
     }
     break;
   case Intrinsic::ppc_qpx_qvlfd:
@@ -625,10 +668,12 @@ Instruction *InstCombiner::visitCallInst(CallInst &CI) {
     // Turn PPC QPX qvstfs -> store if the pointer is known aligned.
     if (getOrEnforceKnownAlignment(II->getArgOperand(1), 16, DL, II, AC, DT) >=
         16) {
-      Type *OpPtrTy =
-        PointerType::getUnqual(II->getArgOperand(0)->getType());
+      Type *VTy = VectorType::get(Builder->getFloatTy(),
+          II->getArgOperand(0)->getType()->getVectorNumElements());
+      Value *TOp = Builder->CreateFPTrunc(II->getArgOperand(0), VTy);
+      Type *OpPtrTy = PointerType::getUnqual(VTy);
       Value *Ptr = Builder->CreateBitCast(II->getArgOperand(1), OpPtrTy);
-      return new StoreInst(II->getArgOperand(0), Ptr);
+      return new StoreInst(TOp, Ptr);
     }
     break;
   case Intrinsic::ppc_qpx_qvstfd:
@@ -750,49 +795,63 @@ Instruction *InstCombiner::visitCallInst(CallInst &CI) {
         Builder->CreateVectorSplat(VWidth, VTCI));
   }
 
+  case Intrinsic::x86_sse41_pmovsxbd:
+  case Intrinsic::x86_sse41_pmovsxbq:
   case Intrinsic::x86_sse41_pmovsxbw:
-  case Intrinsic::x86_sse41_pmovsxwd:
   case Intrinsic::x86_sse41_pmovsxdq:
+  case Intrinsic::x86_sse41_pmovsxwd:
+  case Intrinsic::x86_sse41_pmovsxwq:
+  case Intrinsic::x86_avx2_pmovsxbd:
+  case Intrinsic::x86_avx2_pmovsxbq:
+  case Intrinsic::x86_avx2_pmovsxbw:
+  case Intrinsic::x86_avx2_pmovsxdq:
+  case Intrinsic::x86_avx2_pmovsxwd:
+  case Intrinsic::x86_avx2_pmovsxwq:
+    if (Value *V = SimplifyX86extend(*II, *Builder, true))
+      return ReplaceInstUsesWith(*II, V);
+    break;
+
+  case Intrinsic::x86_sse41_pmovzxbd:
+  case Intrinsic::x86_sse41_pmovzxbq:
   case Intrinsic::x86_sse41_pmovzxbw:
+  case Intrinsic::x86_sse41_pmovzxdq:
   case Intrinsic::x86_sse41_pmovzxwd:
-  case Intrinsic::x86_sse41_pmovzxdq: {
-    // pmov{s|z}x ignores the upper half of their input vectors.
-    unsigned VWidth =
-      cast<VectorType>(II->getArgOperand(0)->getType())->getNumElements();
-    unsigned LowHalfElts = VWidth / 2;
-    APInt InputDemandedElts(APInt::getBitsSet(VWidth, 0, LowHalfElts));
-    APInt UndefElts(VWidth, 0);
-    if (Value *TmpV = SimplifyDemandedVectorElts(
-            II->getArgOperand(0), InputDemandedElts, UndefElts)) {
-      II->setArgOperand(0, TmpV);
-      return II;
-    }
+  case Intrinsic::x86_sse41_pmovzxwq:
+  case Intrinsic::x86_avx2_pmovzxbd:
+  case Intrinsic::x86_avx2_pmovzxbq:
+  case Intrinsic::x86_avx2_pmovzxbw:
+  case Intrinsic::x86_avx2_pmovzxdq:
+  case Intrinsic::x86_avx2_pmovzxwd:
+  case Intrinsic::x86_avx2_pmovzxwq:
+    if (Value *V = SimplifyX86extend(*II, *Builder, false))
+      return ReplaceInstUsesWith(*II, V);
     break;
-  }
+
   case Intrinsic::x86_sse41_insertps:
     if (Value *V = SimplifyX86insertps(*II, *Builder))
       return ReplaceInstUsesWith(*II, V);
     break;
-    
+
   case Intrinsic::x86_sse4a_insertqi: {
     // insertqi x, y, 64, 0 can just copy y's lower bits and leave the top
     // ones undef
     // TODO: eventually we should lower this intrinsic to IR
-    if (auto CIWidth = dyn_cast<ConstantInt>(II->getArgOperand(2))) {
-      if (auto CIStart = dyn_cast<ConstantInt>(II->getArgOperand(3))) {
-        unsigned Index = CIStart->getZExtValue();
+    if (auto CILength = dyn_cast<ConstantInt>(II->getArgOperand(2))) {
+      if (auto CIIndex = dyn_cast<ConstantInt>(II->getArgOperand(3))) {
+        unsigned Index = CIIndex->getZExtValue();
         // From AMD documentation: "a value of zero in the field length is
         // defined as length of 64".
-        unsigned Length = CIWidth->equalsInt(0) ? 64 : CIWidth->getZExtValue();
+        unsigned Length = CILength->equalsInt(0) ? 64 : CILength->getZExtValue();
 
         // From AMD documentation: "If the sum of the bit index + length field
         // is greater than 64, the results are undefined".
+        unsigned End = Index + Length;
 
         // Note that both field index and field length are 8-bit quantities.
         // Since variables 'Index' and 'Length' are unsigned values
         // obtained from zero-extending field index and field length
         // respectively, their sum should never wrap around.
-        if ((Index + Length) > 64)
+        if (End > 64)
           return ReplaceInstUsesWith(CI, UndefValue::get(II->getType()));
 
         if (Length == 64 && Index == 0) {
@@ -804,7 +863,6 @@ Instruction *InstCombiner::visitCallInst(CallInst &CI) {
               Builder->CreateShuffleVector(
                   Vec, Undef, ConstantDataVector::get(
                                   II->getContext(), makeArrayRef(Mask))));
-
         } else if (auto Source =
                        dyn_cast<IntrinsicInst>(II->getArgOperand(0))) {
           if (Source->hasOneUse() &&
@@ -812,37 +870,34 @@ Instruction *InstCombiner::visitCallInst(CallInst &CI) {
             // If the source of the insert has only one use and it's another
             // insert (and they're both inserting from the same vector), try to
             // bundle both together.
-            auto CISourceWidth =
+            auto CISourceLength =
                 dyn_cast<ConstantInt>(Source->getArgOperand(2));
-            auto CISourceStart =
+            auto CISourceIndex =
                 dyn_cast<ConstantInt>(Source->getArgOperand(3));
-            if (CISourceStart && CISourceWidth) {
-              unsigned Start = CIStart->getZExtValue();
-              unsigned Width = CIWidth->getZExtValue();
-              unsigned End = Start + Width;
-              unsigned SourceStart = CISourceStart->getZExtValue();
-              unsigned SourceWidth = CISourceWidth->getZExtValue();
-              unsigned SourceEnd = SourceStart + SourceWidth;
-              unsigned NewStart, NewWidth;
+            if (CISourceIndex && CISourceLength) {
+              unsigned SourceIndex = CISourceIndex->getZExtValue();
+              unsigned SourceLength = CISourceLength->getZExtValue();
+              unsigned SourceEnd = SourceIndex + SourceLength;
+              unsigned NewIndex, NewLength;
               bool ShouldReplace = false;
-              if (Start <= SourceStart && SourceStart <= End) {
-                NewStart = Start;
-                NewWidth = std::max(End, SourceEnd) - NewStart;
+              if (Index <= SourceIndex && SourceIndex <= End) {
+                NewIndex = Index;
+                NewLength = std::max(End, SourceEnd) - NewIndex;
                 ShouldReplace = true;
-              } else if (SourceStart <= Start && Start <= SourceEnd) {
-                NewStart = SourceStart;
-                NewWidth = std::max(SourceEnd, End) - NewStart;
+              } else if (SourceIndex <= Index && Index <= SourceEnd) {
+                NewIndex = SourceIndex;
+                NewLength = std::max(SourceEnd, End) - NewIndex;
                 ShouldReplace = true;
               }
 
               if (ShouldReplace) {
-                Constant *ConstantWidth = ConstantInt::get(
-                    II->getArgOperand(2)->getType(), NewWidth, false);
-                Constant *ConstantStart = ConstantInt::get(
-                    II->getArgOperand(3)->getType(), NewStart, false);
+                Constant *ConstantLength = ConstantInt::get(
+                    II->getArgOperand(2)->getType(), NewLength, false);
+                Constant *ConstantIndex = ConstantInt::get(
+                    II->getArgOperand(3)->getType(), NewIndex, false);
                 Value *Args[4] = { Source->getArgOperand(0),
-                                   II->getArgOperand(1), ConstantWidth,
-                                   ConstantStart };
+                                   II->getArgOperand(1), ConstantLength,
+                                   ConstantIndex };
                 Module *M = CI.getParent()->getParent()->getParent();
                 Value *F =
                     Intrinsic::getDeclaration(M, Intrinsic::x86_sse4a_insertqi);
@@ -1179,7 +1234,8 @@ Instruction *InstCombiner::visitCallInst(CallInst &CI) {
     // facts about the relocate value, while being careful to
     // preserve relocation semantics.
     GCRelocateOperands Operands(II);
-    Value *DerivedPtr = Operands.derivedPtr();
+    Value *DerivedPtr = Operands.getDerivedPtr();
+    auto *GCRelocateType = cast<PointerType>(II->getType());
 
     // Remove the relocation if unused, note that this check is required
     // to prevent the cases below from looping forever.
@@ -1190,21 +1246,25 @@ Instruction *InstCombiner::visitCallInst(CallInst &CI) {
     // TODO: provide a hook for this in GCStrategy.  This is clearly legal for
     // most practical collectors, but there was discussion in the review thread
     // about whether it was legal for all possible collectors.
-    if (isa<UndefValue>(DerivedPtr))
-      return ReplaceInstUsesWith(*II, DerivedPtr);
+    if (isa<UndefValue>(DerivedPtr)) {
+      // gc_relocate is uncasted. Use undef of gc_relocate's type to replace it.
+      return ReplaceInstUsesWith(*II, UndefValue::get(GCRelocateType));
+    }
 
     // The relocation of null will be null for most any collector.
     // TODO: provide a hook for this in GCStrategy.  There might be some weird
     // collector this property does not hold for.
-    if (isa<ConstantPointerNull>(DerivedPtr))
-      return ReplaceInstUsesWith(*II, DerivedPtr);
+    if (isa<ConstantPointerNull>(DerivedPtr)) {
+      // gc_relocate is uncasted. Use null-pointer of gc_relocate's type to replace it.
+      return ReplaceInstUsesWith(*II, ConstantPointerNull::get(GCRelocateType));
+    }
 
     // isKnownNonNull -> nonnull attribute
     if (isKnownNonNull(DerivedPtr))
       II->addAttribute(AttributeSet::ReturnIndex, Attribute::NonNull);
 
     // isDereferenceablePointer -> deref attribute
-    if (DerivedPtr->isDereferenceablePointer(DL)) {
+    if (isDereferenceablePointer(DerivedPtr, DL)) {
       if (Argument *A = dyn_cast<Argument>(DerivedPtr)) {
         uint64_t Bytes = A->getDereferenceableBytes();
         II->addDereferenceableAttr(AttributeSet::ReturnIndex, Bytes);
@@ -1358,11 +1418,29 @@ static IntrinsicInst *FindInitTrampoline(Value *Callee) {
 // visitCallSite - Improvements for call and invoke instructions.
 //
 Instruction *InstCombiner::visitCallSite(CallSite CS) {
+
   if (isAllocLikeFn(CS.getInstruction(), TLI))
     return visitAllocSite(*CS.getInstruction());
 
   bool Changed = false;
 
+  // Mark any parameters that are known to be non-null with the nonnull
+  // attribute.  This is helpful for inlining calls to functions with null
+  // checks on their arguments.
+  unsigned ArgNo = 0;
+  for (Value *V : CS.args()) {
+    if (!CS.paramHasAttr(ArgNo+1, Attribute::NonNull) &&
+        isKnownNonNull(V)) {
+      AttributeSet AS = CS.getAttributes();
+      AS = AS.addAttribute(CS.getInstruction()->getContext(), ArgNo+1,
+                           Attribute::NonNull);
+      CS.setAttributes(AS);
+      Changed = true;
+    }
+    ArgNo++;
+  }
+  assert(ArgNo == CS.arg_size() && "sanity check");
+
   // If the callee is a pointer to a function, attempt to move any casts to the
   // arguments of the call/invoke.
   Value *Callee = CS.getCalledValue();
@@ -1496,10 +1574,7 @@ bool InstCombiner::transformConstExprCastCall(CallSite CS) {
 
     if (!CallerPAL.isEmpty() && !Caller->use_empty()) {
       AttrBuilder RAttrs(CallerPAL, AttributeSet::ReturnIndex);
-      if (RAttrs.
-          hasAttributes(AttributeFuncs::
-                        typeIncompatible(NewRetTy, AttributeSet::ReturnIndex),
-                        AttributeSet::ReturnIndex))
+      if (RAttrs.overlaps(AttributeFuncs::typeIncompatible(NewRetTy)))
         return false;   // Attribute not compatible with transformed value.
     }
 
@@ -1540,8 +1615,7 @@ bool InstCombiner::transformConstExprCastCall(CallSite CS) {
       return false;   // Cannot transform this parameter value.
 
     if (AttrBuilder(CallerPAL.getParamAttributes(i + 1), i + 1).
-          hasAttributes(AttributeFuncs::
-                        typeIncompatible(ParamTy, i + 1), i + 1))
+          overlaps(AttributeFuncs::typeIncompatible(ParamTy)))
       return false;   // Attribute not compatible with transformed value.
 
     if (CS.isInAllocaArgument(i))
@@ -1614,10 +1688,7 @@ bool InstCombiner::transformConstExprCastCall(CallSite CS) {
 
   // If the return value is not being used, the type may not be compatible
   // with the existing attributes.  Wipe out any problematic attributes.
-  RAttrs.
-    removeAttributes(AttributeFuncs::
-                     typeIncompatible(NewRetTy, AttributeSet::ReturnIndex),
-                     AttributeSet::ReturnIndex);
+  RAttrs.remove(AttributeFuncs::typeIncompatible(NewRetTy));
 
   // Add the new return attributes.
   if (RAttrs.hasAttributes())
diff --git a/lib/Transforms/InstCombine/InstCombineCasts.cpp b/lib/Transforms/InstCombine/InstCombineCasts.cpp
index bd79a2697c1..48ab0eb2c1b 100644
--- a/lib/Transforms/InstCombine/InstCombineCasts.cpp
+++ b/lib/Transforms/InstCombine/InstCombineCasts.cpp
@@ -418,8 +418,8 @@ static bool CanEvaluateTruncated(Value *V, Type *Ty, InstCombiner &IC,
     // get into trouble with cyclic PHIs here because we only consider
     // instructions with a single use.
     PHINode *PN = cast<PHINode>(I);
-    for (unsigned i = 0, e = PN->getNumIncomingValues(); i != e; ++i)
-      if (!CanEvaluateTruncated(PN->getIncomingValue(i), Ty, IC, CxtI))
+    for (Value *IncValue : PN->incoming_values())
+      if (!CanEvaluateTruncated(IncValue, Ty, IC, CxtI))
         return false;
     return true;
   }
@@ -435,6 +435,15 @@ Instruction *InstCombiner::visitTrunc(TruncInst &CI) {
   if (Instruction *Result = commonCastTransforms(CI))
     return Result;
 
+  // Test if the trunc is the user of a select which is part of a
+  // minimum or maximum operation. If so, don't do any more simplification.
+  // Even simplifying demanded bits can break the canonical form of a 
+  // min/max.
+  Value *LHS, *RHS;
+  if (SelectInst *SI = dyn_cast<SelectInst>(CI.getOperand(0)))
+    if (matchSelectPattern(SI, LHS, RHS) != SPF_UNKNOWN)
+      return nullptr;
+  
   // See if we can simplify any instructions used by the input whose sole
   // purpose is to compute bits we don't care about.
   if (SimplifyDemandedInstructionBits(CI))
@@ -1029,8 +1038,8 @@ static bool CanEvaluateSExtd(Value *V, Type *Ty) {
     // get into trouble with cyclic PHIs here because we only consider
     // instructions with a single use.
     PHINode *PN = cast<PHINode>(I);
-    for (unsigned i = 0, e = PN->getNumIncomingValues(); i != e; ++i)
-      if (!CanEvaluateSExtd(PN->getIncomingValue(i), Ty)) return false;
+    for (Value *IncValue : PN->incoming_values())
+      if (!CanEvaluateSExtd(IncValue, Ty)) return false;
     return true;
   }
   default:
@@ -1770,19 +1779,17 @@ Instruction *InstCombiner::visitBitCast(BitCastInst &CI) {
     // If the source and destination are pointers, and this cast is equivalent
     // to a getelementptr X, 0, 0, 0...  turn it into the appropriate gep.
     // This can enhance SROA and other transforms that want type-safe pointers.
-    Constant *ZeroUInt =
-      Constant::getNullValue(Type::getInt32Ty(CI.getContext()));
     unsigned NumZeros = 0;
     while (SrcElTy != DstElTy &&
            isa<CompositeType>(SrcElTy) && !SrcElTy->isPointerTy() &&
            SrcElTy->getNumContainedTypes() /* not "{}" */) {
-      SrcElTy = cast<CompositeType>(SrcElTy)->getTypeAtIndex(ZeroUInt);
+      SrcElTy = cast<CompositeType>(SrcElTy)->getTypeAtIndex(0U);
       ++NumZeros;
     }
 
     // If we found a path from the src to dest, create the getelementptr now.
     if (SrcElTy == DstElTy) {
-      SmallVector<Value*, 8> Idxs(NumZeros+1, ZeroUInt);
+      SmallVector<Value *, 8> Idxs(NumZeros + 1, Builder->getInt32(0));
       return GetElementPtrInst::CreateInBounds(Src, Idxs);
     }
   }
diff --git a/lib/Transforms/InstCombine/InstCombineCompares.cpp b/lib/Transforms/InstCombine/InstCombineCompares.cpp
index ce5c44b168f..72a7517b87f 100644
--- a/lib/Transforms/InstCombine/InstCombineCompares.cpp
+++ b/lib/Transforms/InstCombine/InstCombineCompares.cpp
@@ -2102,7 +2102,7 @@ static Instruction *ProcessUGT_ADDCST_ADD(ICmpInst &I, Value *A, Value *B,
 
   Value *TruncA = Builder->CreateTrunc(A, NewType, A->getName()+".trunc");
   Value *TruncB = Builder->CreateTrunc(B, NewType, B->getName()+".trunc");
-  CallInst *Call = Builder->CreateCall2(F, TruncA, TruncB, "sadd");
+  CallInst *Call = Builder->CreateCall(F, {TruncA, TruncB}, "sadd");
   Value *Add = Builder->CreateExtractValue(Call, 0, "sadd.result");
   Value *ZExt = Builder->CreateZExt(Add, OrigAdd->getType());
 
@@ -2144,15 +2144,9 @@ bool InstCombiner::OptimizeOverflowCheck(OverflowCheckFlavor OCF, Value *LHS,
   }
   // FALL THROUGH uadd into sadd
   case OCF_SIGNED_ADD: {
-    // X + undef -> undef
-    if (isa<UndefValue>(RHS))
-      return SetResult(UndefValue::get(RHS->getType()),
-                       UndefValue::get(Builder->getInt1Ty()), false);
-
-    if (ConstantInt *ConstRHS = dyn_cast<ConstantInt>(RHS))
-      // X + 0 -> {X, false}
-      if (ConstRHS->isZero())
-        return SetResult(LHS, Builder->getFalse(), false);
+    // X + 0 -> {X, false}
+    if (match(RHS, m_Zero()))
+      return SetResult(LHS, Builder->getFalse(), false);
 
     // We can strength reduce this signed add into a regular add if we can prove
     // that it will never overflow.
@@ -2160,21 +2154,14 @@ bool InstCombiner::OptimizeOverflowCheck(OverflowCheckFlavor OCF, Value *LHS,
       if (WillNotOverflowSignedAdd(LHS, RHS, OrigI))
         return SetResult(Builder->CreateNSWAdd(LHS, RHS), Builder->getFalse(),
                          true);
+    break;
   }
 
   case OCF_UNSIGNED_SUB:
   case OCF_SIGNED_SUB: {
-    // undef - X -> undef
-    // X - undef -> undef
-    if (isa<UndefValue>(LHS) || isa<UndefValue>(RHS))
-      return SetResult(UndefValue::get(LHS->getType()),
-                       UndefValue::get(Builder->getInt1Ty()), false);
-
-    if (ConstantInt *ConstRHS = dyn_cast<ConstantInt>(RHS))
-      // X - 0 -> {X, false}
-      if (ConstRHS->isZero())
-        return SetResult(UndefValue::get(LHS->getType()), Builder->getFalse(),
-                         false);
+    // X - 0 -> {X, false}
+    if (match(RHS, m_Zero()))
+      return SetResult(LHS, Builder->getFalse(), false);
 
     if (OCF == OCF_SIGNED_SUB) {
       if (WillNotOverflowSignedSub(LHS, RHS, OrigI))
@@ -2199,24 +2186,21 @@ bool InstCombiner::OptimizeOverflowCheck(OverflowCheckFlavor OCF, Value *LHS,
   case OCF_SIGNED_MUL:
     // X * undef -> undef
     if (isa<UndefValue>(RHS))
-      return SetResult(UndefValue::get(LHS->getType()),
-                       UndefValue::get(Builder->getInt1Ty()), false);
-
-    if (ConstantInt *RHSI = dyn_cast<ConstantInt>(RHS)) {
-      // X * 0 -> {0, false}
-      if (RHSI->isZero())
-        return SetResult(Constant::getNullValue(RHS->getType()),
-                         Builder->getFalse(), false);
-
-      // X * 1 -> {X, false}
-      if (RHSI->equalsInt(1))
-        return SetResult(LHS, Builder->getFalse(), false);
-    }
+      return SetResult(RHS, UndefValue::get(Builder->getInt1Ty()), false);
+
+    // X * 0 -> {0, false}
+    if (match(RHS, m_Zero()))
+      return SetResult(RHS, Builder->getFalse(), false);
+
+    // X * 1 -> {X, false}
+    if (match(RHS, m_One()))
+      return SetResult(LHS, Builder->getFalse(), false);
 
     if (OCF == OCF_SIGNED_MUL)
       if (WillNotOverflowSignedMul(LHS, RHS, OrigI))
         return SetResult(Builder->CreateNSWMul(LHS, RHS), Builder->getFalse(),
                          true);
+    break;
   }
 
   return false;
@@ -2389,7 +2373,7 @@ static Instruction *ProcessUMulZExtIdiom(ICmpInst &I, Value *MulVal,
     MulB = Builder->CreateZExt(B, MulType);
   Value *F =
       Intrinsic::getDeclaration(M, Intrinsic::umul_with_overflow, MulType);
-  CallInst *Call = Builder->CreateCall2(F, MulA, MulB, "umul");
+  CallInst *Call = Builder->CreateCall(F, {MulA, MulB}, "umul");
   IC.Worklist.Add(MulInstr);
 
   // If there are uses of mul result other than the comparison, we know that
@@ -2667,7 +2651,8 @@ Instruction *InstCombiner::visitICmpInst(ICmpInst &I) {
     Changed = true;
   }
 
-  if (Value *V = SimplifyICmpInst(I.getPredicate(), Op0, Op1, DL, TLI, DT, AC))
+  if (Value *V =
+          SimplifyICmpInst(I.getPredicate(), Op0, Op1, DL, TLI, DT, AC, &I))
     return ReplaceInstUsesWith(I, V);
 
   // comparing -val or val with non-zero is the same as just comparing val
@@ -3948,7 +3933,8 @@ Instruction *InstCombiner::visitFCmpInst(FCmpInst &I) {
 
   Value *Op0 = I.getOperand(0), *Op1 = I.getOperand(1);
 
-  if (Value *V = SimplifyFCmpInst(I.getPredicate(), Op0, Op1, DL, TLI, DT, AC))
+  if (Value *V = SimplifyFCmpInst(I.getPredicate(), Op0, Op1,
+                                  I.getFastMathFlags(), DL, TLI, DT, AC, &I))
     return ReplaceInstUsesWith(I, V);
 
   // Simplify 'fcmp pred X, X'
@@ -3975,6 +3961,19 @@ Instruction *InstCombiner::visitFCmpInst(FCmpInst &I) {
     }
   }
 
+  // Test if the FCmpInst instruction is used exclusively by a select as
+  // part of a minimum or maximum operation. If so, refrain from doing
+  // any other folding. This helps out other analyses which understand
+  // non-obfuscated minimum and maximum idioms, such as ScalarEvolution
+  // and CodeGen. And in this case, at least one of the comparison
+  // operands has at least one user besides the compare (the select),
+  // which would often largely negate the benefit of folding anyway.
+  if (I.hasOneUse())
+    if (SelectInst *SI = dyn_cast<SelectInst>(*I.user_begin()))
+      if ((SI->getOperand(1) == Op0 && SI->getOperand(2) == Op1) ||
+          (SI->getOperand(2) == Op0 && SI->getOperand(1) == Op1))
+        return nullptr;
+
   // Handle fcmp with constant RHS
   if (Constant *RHSC = dyn_cast<Constant>(Op1)) {
     if (Instruction *LHSI = dyn_cast<Instruction>(Op0))
diff --git a/lib/Transforms/InstCombine/InstCombineInternal.h b/lib/Transforms/InstCombine/InstCombineInternal.h
index d0caf344d64..ac934f1bd85 100644
--- a/lib/Transforms/InstCombine/InstCombineInternal.h
+++ b/lib/Transforms/InstCombine/InstCombineInternal.h
@@ -15,6 +15,7 @@
 #ifndef LLVM_LIB_TRANSFORMS_INSTCOMBINE_INSTCOMBINEINTERNAL_H
 #define LLVM_LIB_TRANSFORMS_INSTCOMBINE_INSTCOMBINEINTERNAL_H
 
+#include "llvm/Analysis/AliasAnalysis.h"
 #include "llvm/Analysis/AssumptionCache.h"
 #include "llvm/Analysis/LoopInfo.h"
 #include "llvm/Analysis/TargetFolder.h"
@@ -39,17 +40,6 @@ class DbgDeclareInst;
 class MemIntrinsic;
 class MemSetInst;
 
-/// \brief Specific patterns of select instructions we can match.
-enum SelectPatternFlavor {
-  SPF_UNKNOWN = 0,
-  SPF_SMIN,
-  SPF_UMIN,
-  SPF_SMAX,
-  SPF_UMAX,
-  SPF_ABS,
-  SPF_NABS
-};
-
 /// \brief Assign a complexity or rank value to LLVM Values.
 ///
 /// This routine maps IR values to various complexity ranks:
@@ -188,6 +178,8 @@ class LLVM_LIBRARY_VISIBILITY InstCombiner
   // Mode in which we are running the combiner.
   const bool MinimizeSize;
 
+  AliasAnalysis *AA;
+
   // Required analyses.
   // FIXME: These can never be null and should be references.
   AssumptionCache *AC;
@@ -203,10 +195,11 @@ class LLVM_LIBRARY_VISIBILITY InstCombiner
 
 public:
   InstCombiner(InstCombineWorklist &Worklist, BuilderTy *Builder,
-               bool MinimizeSize, AssumptionCache *AC, TargetLibraryInfo *TLI,
+               bool MinimizeSize, AliasAnalysis *AA,
+               AssumptionCache *AC, TargetLibraryInfo *TLI,
                DominatorTree *DT, const DataLayout &DL, LoopInfo *LI)
       : Worklist(Worklist), Builder(Builder), MinimizeSize(MinimizeSize),
-        AC(AC), TLI(TLI), DT(DT), DL(DL), LI(LI), MadeIRChange(false) {}
+        AA(AA), AC(AC), TLI(TLI), DT(DT), DL(DL), LI(LI), MadeIRChange(false) {}
 
   /// \brief Run the combiner over the entire worklist until it is empty.
   ///
diff --git a/lib/Transforms/InstCombine/InstCombineLoadStoreAlloca.cpp b/lib/Transforms/InstCombine/InstCombineLoadStoreAlloca.cpp
index d8a559c68a9..e3179dbeece 100644
--- a/lib/Transforms/InstCombine/InstCombineLoadStoreAlloca.cpp
+++ b/lib/Transforms/InstCombine/InstCombineLoadStoreAlloca.cpp
@@ -314,7 +314,8 @@ Instruction *InstCombiner::visitAllocaInst(AllocaInst &AI) {
 ///
 /// Note that this will create all of the instructions with whatever insert
 /// point the \c InstCombiner currently is using.
-static LoadInst *combineLoadToNewType(InstCombiner &IC, LoadInst &LI, Type *NewTy) {
+static LoadInst *combineLoadToNewType(InstCombiner &IC, LoadInst &LI, Type *NewTy,
+                                      const Twine &Suffix = "") {
   Value *Ptr = LI.getPointerOperand();
   unsigned AS = LI.getPointerAddressSpace();
   SmallVector<std::pair<unsigned, MDNode *>, 8> MD;
@@ -322,7 +323,7 @@ static LoadInst *combineLoadToNewType(InstCombiner &IC, LoadInst &LI, Type *NewT
 
   LoadInst *NewLoad = IC.Builder->CreateAlignedLoad(
       IC.Builder->CreateBitCast(Ptr, NewTy->getPointerTo(AS)),
-      LI.getAlignment(), LI.getName());
+      LI.getAlignment(), LI.getName() + Suffix);
   MDBuilder MDB(NewLoad->getContext());
   for (const auto &MDPair : MD) {
     unsigned ID = MDPair.first;
@@ -482,12 +483,17 @@ static Instruction *combineLoadToOperationType(InstCombiner &IC, LoadInst &LI) {
   }
 
   // Fold away bit casts of the loaded value by loading the desired type.
+  // We can do this for BitCastInsts as well as casts from and to pointer types,
+  // as long as those are noops (i.e., the source or dest type have the same
+  // bitwidth as the target's pointers).
   if (LI.hasOneUse())
-    if (auto *BC = dyn_cast<BitCastInst>(LI.user_back())) {
-      LoadInst *NewLoad = combineLoadToNewType(IC, LI, BC->getDestTy());
-      BC->replaceAllUsesWith(NewLoad);
-      IC.EraseInstFromFunction(*BC);
-      return &LI;
+    if (auto* CI = dyn_cast<CastInst>(LI.user_back())) {
+      if (CI->isNoopCast(DL)) {
+        LoadInst *NewLoad = combineLoadToNewType(IC, LI, CI->getDestTy());
+        CI->replaceAllUsesWith(NewLoad);
+        IC.EraseInstFromFunction(*CI);
+        return &LI;
+      }
     }
 
   // FIXME: We should also canonicalize loads of vectors when their elements are
@@ -495,6 +501,41 @@ static Instruction *combineLoadToOperationType(InstCombiner &IC, LoadInst &LI) {
   return nullptr;
 }
 
+static Instruction *unpackLoadToAggregate(InstCombiner &IC, LoadInst &LI) {
+  // FIXME: We could probably with some care handle both volatile and atomic
+  // stores here but it isn't clear that this is important.
+  if (!LI.isSimple())
+    return nullptr;
+
+  Type *T = LI.getType();
+  if (!T->isAggregateType())
+    return nullptr;
+
+  assert(LI.getAlignment() && "Alignement must be set at this point");
+
+  if (auto *ST = dyn_cast<StructType>(T)) {
+    // If the struct only have one element, we unpack.
+    if (ST->getNumElements() == 1) {
+      LoadInst *NewLoad = combineLoadToNewType(IC, LI, ST->getTypeAtIndex(0U),
+                                               ".unpack");
+      return IC.ReplaceInstUsesWith(LI, IC.Builder->CreateInsertValue(
+        UndefValue::get(T), NewLoad, 0, LI.getName()));
+    }
+  }
+
+  if (auto *AT = dyn_cast<ArrayType>(T)) {
+    // If the array only have one element, we unpack.
+    if (AT->getNumElements() == 1) {
+      LoadInst *NewLoad = combineLoadToNewType(IC, LI, AT->getElementType(),
+                                               ".unpack");
+      return IC.ReplaceInstUsesWith(LI, IC.Builder->CreateInsertValue(
+        UndefValue::get(T), NewLoad, 0, LI.getName()));
+    }
+  }
+
+  return nullptr;
+}
+
 // If we can determine that all possible objects pointed to by the provided
 // pointer value are, not only dereferenceable, but also definitively less than
 // or equal to the provided maximum size, then return true. Otherwise, return
@@ -520,8 +561,8 @@ static bool isObjectSizeLessThanOrEq(Value *V, uint64_t MaxSize,
     }
 
     if (PHINode *PN = dyn_cast<PHINode>(P)) {
-      for (unsigned i = 0, e = PN->getNumIncomingValues(); i != e; ++i)
-        Worklist.push_back(PN->getIncomingValue(i));
+      for (Value *IncValue : PN->incoming_values())
+        Worklist.push_back(IncValue);
       continue;
     }
 
@@ -701,14 +742,32 @@ Instruction *InstCombiner::visitLoadInst(LoadInst &LI) {
   // FIXME: Some of it is okay for atomic loads; needs refactoring.
   if (!LI.isSimple()) return nullptr;
 
+  if (Instruction *Res = unpackLoadToAggregate(*this, LI))
+    return Res;
+
   // Do really simple store-to-load forwarding and load CSE, to catch cases
   // where there are several consecutive memory accesses to the same location,
   // separated by a few arithmetic operations.
   BasicBlock::iterator BBI = &LI;
-  if (Value *AvailableVal = FindAvailableLoadedValue(Op, LI.getParent(), BBI,6))
+  AAMDNodes AATags;
+  if (Value *AvailableVal = FindAvailableLoadedValue(Op, LI.getParent(), BBI,
+                                                     6, AA, &AATags)) {
+    if (LoadInst *NLI = dyn_cast<LoadInst>(AvailableVal)) {
+      unsigned KnownIDs[] = {
+        LLVMContext::MD_tbaa,
+        LLVMContext::MD_alias_scope,
+        LLVMContext::MD_noalias,
+        LLVMContext::MD_range,
+        LLVMContext::MD_invariant_load,
+        LLVMContext::MD_nonnull,
+      };
+      combineMetadata(NLI, &LI, KnownIDs);
+    };
+
     return ReplaceInstUsesWith(
         LI, Builder->CreateBitOrPointerCast(AvailableVal, LI.getType(),
                                             LI.getName() + ".cast"));
+  }
 
   // load(gep null, ...) -> unreachable
   if (GetElementPtrInst *GEPI = dyn_cast<GetElementPtrInst>(Op)) {
@@ -832,7 +891,7 @@ static bool unpackStoreToAggregate(InstCombiner &IC, StoreInst &SI) {
   if (!T->isAggregateType())
     return false;
 
-  if (StructType *ST = dyn_cast<StructType>(T)) {
+  if (auto *ST = dyn_cast<StructType>(T)) {
     // If the struct only have one element, we unpack.
     if (ST->getNumElements() == 1) {
       V = IC.Builder->CreateExtractValue(V, 0);
@@ -841,6 +900,15 @@ static bool unpackStoreToAggregate(InstCombiner &IC, StoreInst &SI) {
     }
   }
 
+  if (auto *AT = dyn_cast<ArrayType>(T)) {
+    // If the array only have one element, we unpack.
+    if (AT->getNumElements() == 1) {
+      V = IC.Builder->CreateExtractValue(V, 0);
+      combineStoreToNewValue(IC, SI, V);
+      return true;
+    }
+  }
+
   return false;
 }
 
diff --git a/lib/Transforms/InstCombine/InstCombineMulDivRem.cpp b/lib/Transforms/InstCombine/InstCombineMulDivRem.cpp
index 35513f1ed31..a554e9f628e 100644
--- a/lib/Transforms/InstCombine/InstCombineMulDivRem.cpp
+++ b/lib/Transforms/InstCombine/InstCombineMulDivRem.cpp
@@ -217,12 +217,16 @@ Instruction *InstCombiner::visitMul(BinaryOperator &I) {
         NewCst = getLogBase2Vector(CV);
 
       if (NewCst) {
+        unsigned Width = NewCst->getType()->getPrimitiveSizeInBits();
         BinaryOperator *Shl = BinaryOperator::CreateShl(NewOp, NewCst);
 
         if (I.hasNoUnsignedWrap())
           Shl->setHasNoUnsignedWrap();
-        if (I.hasNoSignedWrap() && NewCst->isNotMinSignedValue())
-          Shl->setHasNoSignedWrap();
+        if (I.hasNoSignedWrap()) {
+          uint64_t V;
+          if (match(NewCst, m_ConstantInt(V)) && V != Width - 1)
+            Shl->setHasNoSignedWrap();
+        }
 
         return Shl;
       }
diff --git a/lib/Transforms/InstCombine/InstCombinePHI.cpp b/lib/Transforms/InstCombine/InstCombinePHI.cpp
index ca2caed458b..460f6eb6a82 100644
--- a/lib/Transforms/InstCombine/InstCombinePHI.cpp
+++ b/lib/Transforms/InstCombine/InstCombinePHI.cpp
@@ -19,9 +19,8 @@ using namespace llvm;
 
 #define DEBUG_TYPE "instcombine"
 
-/// FoldPHIArgBinOpIntoPHI - If we have something like phi [add (a,b), add(a,c)]
-/// and if a/b/c and the add's all have a single use, turn this into a phi
-/// and a single binop.
+/// If we have something like phi [add (a,b), add(a,c)] and if a/b/c and the
+/// adds all have a single use, turn this into a phi and a single binop.
 Instruction *InstCombiner::FoldPHIArgBinOpIntoPHI(PHINode &PN) {
   Instruction *FirstInst = cast<Instruction>(PN.getIncomingValue(0));
   assert(isa<BinaryOperator>(FirstInst) || isa<CmpInst>(FirstInst));
@@ -238,10 +237,9 @@ Instruction *InstCombiner::FoldPHIArgGEPIntoPHI(PHINode &PN) {
 }
 
 
-/// isSafeAndProfitableToSinkLoad - Return true if we know that it is safe to
-/// sink the load out of the block that defines it.  This means that it must be
-/// obvious the value of the load is not changed from the point of the load to
-/// the end of the block it is in.
+/// Return true if we know that it is safe to sink the load out of the block
+/// that defines it. This means that it must be obvious the value of the load is
+/// not changed from the point of the load to the end of the block it is in.
 ///
 /// Finally, it is safe, but not profitable, to sink a load targeting a
 /// non-address-taken alloca.  Doing so will cause us to not promote the alloca
@@ -375,8 +373,8 @@ Instruction *InstCombiner::FoldPHIArgLoadIntoPHI(PHINode &PN) {
   // and mark all the input loads as non-volatile.  If we don't do this, we will
   // insert a new volatile load and the old ones will not be deletable.
   if (isVolatile)
-    for (unsigned i = 0, e = PN.getNumIncomingValues(); i != e; ++i)
-      cast<LoadInst>(PN.getIncomingValue(i))->setVolatile(false);
+    for (Value *IncValue : PN.incoming_values())
+      cast<LoadInst>(IncValue)->setVolatile(false);
 
   LoadInst *NewLI = new LoadInst(PhiVal, "", isVolatile, LoadAlignment);
   NewLI->setDebugLoc(FirstLI->getDebugLoc());
@@ -385,9 +383,9 @@ Instruction *InstCombiner::FoldPHIArgLoadIntoPHI(PHINode &PN) {
 
 
 
-/// FoldPHIArgOpIntoPHI - If all operands to a PHI node are the same "unary"
-/// operator and they all are only used by the PHI, PHI together their
-/// inputs, and do the operation once, to the result of the PHI.
+/// If all operands to a PHI node are the same "unary" operator and they all are
+/// only used by the PHI, PHI together their inputs, and do the operation once,
+/// to the result of the PHI.
 Instruction *InstCombiner::FoldPHIArgOpIntoPHI(PHINode &PN) {
   Instruction *FirstInst = cast<Instruction>(PN.getIncomingValue(0));
 
@@ -503,8 +501,7 @@ Instruction *InstCombiner::FoldPHIArgOpIntoPHI(PHINode &PN) {
   return NewCI;
 }
 
-/// DeadPHICycle - Return true if this PHI node is only used by a PHI node cycle
-/// that is dead.
+/// Return true if this PHI node is only used by a PHI node cycle that is dead.
 static bool DeadPHICycle(PHINode *PN,
                          SmallPtrSetImpl<PHINode*> &PotentiallyDeadPHIs) {
   if (PN->use_empty()) return true;
@@ -524,8 +521,8 @@ static bool DeadPHICycle(PHINode *PN,
   return false;
 }
 
-/// PHIsEqualValue - Return true if this phi node is always equal to
-/// NonPhiInVal.  This happens with mutually cyclic phi nodes like:
+/// Return true if this phi node is always equal to NonPhiInVal.
+/// This happens with mutually cyclic phi nodes like:
 ///   z = some value; x = phi (y, z); y = phi (x, z)
 static bool PHIsEqualValue(PHINode *PN, Value *NonPhiInVal,
                            SmallPtrSetImpl<PHINode*> &ValueEqualPHIs) {
@@ -539,8 +536,7 @@ static bool PHIsEqualValue(PHINode *PN, Value *NonPhiInVal,
 
   // Scan the operands to see if they are either phi nodes or are equal to
   // the value.
-  for (unsigned i = 0, e = PN->getNumIncomingValues(); i != e; ++i) {
-    Value *Op = PN->getIncomingValue(i);
+  for (Value *Op : PN->incoming_values()) {
     if (PHINode *OpPN = dyn_cast<PHINode>(Op)) {
       if (!PHIsEqualValue(OpPN, NonPhiInVal, ValueEqualPHIs))
         return false;
@@ -607,10 +603,10 @@ namespace llvm {
 }
 
 
-/// SliceUpIllegalIntegerPHI - This is an integer PHI and we know that it has an
-/// illegal type: see if it is only used by trunc or trunc(lshr) operations.  If
-/// so, we split the PHI into the various pieces being extracted.  This sort of
-/// thing is introduced when SROA promotes an aggregate to large integer values.
+/// This is an integer PHI and we know that it has an illegal type: see if it is
+/// only used by trunc or trunc(lshr) operations. If so, we split the PHI into
+/// the various pieces being extracted. This sort of thing is introduced when
+/// SROA promotes an aggregate to large integer values.
 ///
 /// TODO: The user of the trunc may be an bitcast to float/double/vector or an
 /// inttoptr.  We should produce new PHIs in the right type.
diff --git a/lib/Transforms/InstCombine/InstCombineSelect.cpp b/lib/Transforms/InstCombine/InstCombineSelect.cpp
index b28611f3604..f51442a9f36 100644
--- a/lib/Transforms/InstCombine/InstCombineSelect.cpp
+++ b/lib/Transforms/InstCombine/InstCombineSelect.cpp
@@ -14,85 +14,52 @@
 #include "InstCombineInternal.h"
 #include "llvm/Analysis/ConstantFolding.h"
 #include "llvm/Analysis/InstructionSimplify.h"
+#include "llvm/Analysis/ValueTracking.h"
 #include "llvm/IR/PatternMatch.h"
 using namespace llvm;
 using namespace PatternMatch;
 
 #define DEBUG_TYPE "instcombine"
 
-/// MatchSelectPattern - Pattern match integer [SU]MIN, [SU]MAX, and ABS idioms,
-/// returning the kind and providing the out parameter results if we
-/// successfully match.
 static SelectPatternFlavor
-MatchSelectPattern(Value *V, Value *&LHS, Value *&RHS) {
-  SelectInst *SI = dyn_cast<SelectInst>(V);
-  if (!SI) return SPF_UNKNOWN;
-
-  ICmpInst *ICI = dyn_cast<ICmpInst>(SI->getCondition());
-  if (!ICI) return SPF_UNKNOWN;
-
-  ICmpInst::Predicate Pred = ICI->getPredicate();
-  Value *CmpLHS = ICI->getOperand(0);
-  Value *CmpRHS = ICI->getOperand(1);
-  Value *TrueVal = SI->getTrueValue();
-  Value *FalseVal = SI->getFalseValue();
-
-  LHS = CmpLHS;
-  RHS = CmpRHS;
-
-  // (icmp X, Y) ? X : Y
-  if (TrueVal == CmpLHS && FalseVal == CmpRHS) {
-    switch (Pred) {
-    default: return SPF_UNKNOWN; // Equality.
-    case ICmpInst::ICMP_UGT:
-    case ICmpInst::ICMP_UGE: return SPF_UMAX;
-    case ICmpInst::ICMP_SGT:
-    case ICmpInst::ICMP_SGE: return SPF_SMAX;
-    case ICmpInst::ICMP_ULT:
-    case ICmpInst::ICMP_ULE: return SPF_UMIN;
-    case ICmpInst::ICMP_SLT:
-    case ICmpInst::ICMP_SLE: return SPF_SMIN;
-    }
-  }
-
-  // (icmp X, Y) ? Y : X
-  if (TrueVal == CmpRHS && FalseVal == CmpLHS) {
-    switch (Pred) {
-    default: return SPF_UNKNOWN; // Equality.
-    case ICmpInst::ICMP_UGT:
-    case ICmpInst::ICMP_UGE: return SPF_UMIN;
-    case ICmpInst::ICMP_SGT:
-    case ICmpInst::ICMP_SGE: return SPF_SMIN;
-    case ICmpInst::ICMP_ULT:
-    case ICmpInst::ICMP_ULE: return SPF_UMAX;
-    case ICmpInst::ICMP_SLT:
-    case ICmpInst::ICMP_SLE: return SPF_SMAX;
-    }
+getInverseMinMaxSelectPattern(SelectPatternFlavor SPF) {
+  switch (SPF) {
+  default:
+    llvm_unreachable("unhandled!");
+
+  case SPF_SMIN:
+    return SPF_SMAX;
+  case SPF_UMIN:
+    return SPF_UMAX;
+  case SPF_SMAX:
+    return SPF_SMIN;
+  case SPF_UMAX:
+    return SPF_UMIN;
   }
+}
 
-  if (ConstantInt *C1 = dyn_cast<ConstantInt>(CmpRHS)) {
-    if ((CmpLHS == TrueVal && match(FalseVal, m_Neg(m_Specific(CmpLHS)))) ||
-        (CmpLHS == FalseVal && match(TrueVal, m_Neg(m_Specific(CmpLHS))))) {
-
-      // ABS(X) ==> (X >s 0) ? X : -X and (X >s -1) ? X : -X
-      // NABS(X) ==> (X >s 0) ? -X : X and (X >s -1) ? -X : X
-      if (Pred == ICmpInst::ICMP_SGT && (C1->isZero() || C1->isMinusOne())) {
-        return (CmpLHS == TrueVal) ? SPF_ABS : SPF_NABS;
-      }
-
-      // ABS(X) ==> (X <s 0) ? -X : X and (X <s 1) ? -X : X
-      // NABS(X) ==> (X <s 0) ? X : -X and (X <s 1) ? X : -X
-      if (Pred == ICmpInst::ICMP_SLT && (C1->isZero() || C1->isOne())) {
-        return (CmpLHS == FalseVal) ? SPF_ABS : SPF_NABS;
-      }
-    }
+static CmpInst::Predicate getICmpPredicateForMinMax(SelectPatternFlavor SPF) {
+  switch (SPF) {
+  default:
+    llvm_unreachable("unhandled!");
+
+  case SPF_SMIN:
+    return ICmpInst::ICMP_SLT;
+  case SPF_UMIN:
+    return ICmpInst::ICMP_ULT;
+  case SPF_SMAX:
+    return ICmpInst::ICMP_SGT;
+  case SPF_UMAX:
+    return ICmpInst::ICMP_UGT;
   }
-
-  // TODO: (X > 4) ? X : 5   -->  (X >= 5) ? X : 5  -->  MAX(X, 5)
-
-  return SPF_UNKNOWN;
 }
 
+static Value *generateMinMaxSelectPattern(InstCombiner::BuilderTy *Builder,
+                                          SelectPatternFlavor SPF, Value *A,
+                                          Value *B) {
+  CmpInst::Predicate Pred = getICmpPredicateForMinMax(SPF);
+  return Builder->CreateSelect(Builder->CreateICmp(Pred, A, B), A, B);
+}
 
 /// GetSelectFoldableOperands - We want to turn code that looks like this:
 ///   %C = or %A, %B
@@ -309,72 +276,6 @@ Instruction *InstCombiner::FoldSelectIntoOp(SelectInst &SI, Value *TrueVal,
   return nullptr;
 }
 
-/// SimplifyWithOpReplaced - See if V simplifies when its operand Op is
-/// replaced with RepOp.
-static Value *SimplifyWithOpReplaced(Value *V, Value *Op, Value *RepOp,
-                                     const TargetLibraryInfo *TLI,
-                                     const DataLayout &DL, DominatorTree *DT,
-                                     AssumptionCache *AC) {
-  // Trivial replacement.
-  if (V == Op)
-    return RepOp;
-
-  Instruction *I = dyn_cast<Instruction>(V);
-  if (!I)
-    return nullptr;
-
-  // If this is a binary operator, try to simplify it with the replaced op.
-  if (BinaryOperator *B = dyn_cast<BinaryOperator>(I)) {
-    if (B->getOperand(0) == Op)
-      return SimplifyBinOp(B->getOpcode(), RepOp, B->getOperand(1), DL, TLI);
-    if (B->getOperand(1) == Op)
-      return SimplifyBinOp(B->getOpcode(), B->getOperand(0), RepOp, DL, TLI);
-  }
-
-  // Same for CmpInsts.
-  if (CmpInst *C = dyn_cast<CmpInst>(I)) {
-    if (C->getOperand(0) == Op)
-      return SimplifyCmpInst(C->getPredicate(), RepOp, C->getOperand(1), DL,
-                             TLI, DT, AC);
-    if (C->getOperand(1) == Op)
-      return SimplifyCmpInst(C->getPredicate(), C->getOperand(0), RepOp, DL,
-                             TLI, DT, AC);
-  }
-
-  // TODO: We could hand off more cases to instsimplify here.
-
-  // If all operands are constant after substituting Op for RepOp then we can
-  // constant fold the instruction.
-  if (Constant *CRepOp = dyn_cast<Constant>(RepOp)) {
-    // Build a list of all constant operands.
-    SmallVector<Constant*, 8> ConstOps;
-    for (unsigned i = 0, e = I->getNumOperands(); i != e; ++i) {
-      if (I->getOperand(i) == Op)
-        ConstOps.push_back(CRepOp);
-      else if (Constant *COp = dyn_cast<Constant>(I->getOperand(i)))
-        ConstOps.push_back(COp);
-      else
-        break;
-    }
-
-    // All operands were constants, fold it.
-    if (ConstOps.size() == I->getNumOperands()) {
-      if (CmpInst *C = dyn_cast<CmpInst>(I))
-        return ConstantFoldCompareInstOperands(C->getPredicate(), ConstOps[0],
-                                               ConstOps[1], DL, TLI);
-
-      if (LoadInst *LI = dyn_cast<LoadInst>(I))
-        if (!LI->isVolatile())
-          return ConstantFoldLoadFromConstPtr(ConstOps[0], DL);
-
-      return ConstantFoldInstOperands(I->getOpcode(), I->getType(), ConstOps,
-                                      DL, TLI);
-    }
-  }
-
-  return nullptr;
-}
-
 /// foldSelectICmpAndOr - We want to turn:
 ///   (select (icmp eq (and X, C1), 0), Y, (or Y, C2))
 /// into:
@@ -510,14 +411,6 @@ Instruction *InstCombiner::visitSelectInstWithICmp(SelectInst &SI,
   // here, so make sure the select is the only user.
   if (ICI->hasOneUse())
     if (ConstantInt *CI = dyn_cast<ConstantInt>(CmpRHS)) {
-      // X < MIN ? T : F  -->  F
-      if ((Pred == ICmpInst::ICMP_SLT || Pred == ICmpInst::ICMP_ULT)
-          && CI->isMinValue(Pred == ICmpInst::ICMP_SLT))
-        return ReplaceInstUsesWith(SI, FalseVal);
-      // X > MAX ? T : F  -->  F
-      else if ((Pred == ICmpInst::ICMP_SGT || Pred == ICmpInst::ICMP_UGT)
-               && CI->isMaxValue(Pred == ICmpInst::ICMP_SGT))
-        return ReplaceInstUsesWith(SI, FalseVal);
       switch (Pred) {
       default: break;
       case ICmpInst::ICMP_ULT:
@@ -631,33 +524,6 @@ Instruction *InstCombiner::visitSelectInstWithICmp(SelectInst &SI,
     }
   }
 
-  // If we have an equality comparison then we know the value in one of the
-  // arms of the select. See if substituting this value into the arm and
-  // simplifying the result yields the same value as the other arm.
-  if (Pred == ICmpInst::ICMP_EQ) {
-    if (SimplifyWithOpReplaced(FalseVal, CmpLHS, CmpRHS, TLI, DL, DT, AC) ==
-            TrueVal ||
-        SimplifyWithOpReplaced(FalseVal, CmpRHS, CmpLHS, TLI, DL, DT, AC) ==
-            TrueVal)
-      return ReplaceInstUsesWith(SI, FalseVal);
-    if (SimplifyWithOpReplaced(TrueVal, CmpLHS, CmpRHS, TLI, DL, DT, AC) ==
-            FalseVal ||
-        SimplifyWithOpReplaced(TrueVal, CmpRHS, CmpLHS, TLI, DL, DT, AC) ==
-            FalseVal)
-      return ReplaceInstUsesWith(SI, FalseVal);
-  } else if (Pred == ICmpInst::ICMP_NE) {
-    if (SimplifyWithOpReplaced(TrueVal, CmpLHS, CmpRHS, TLI, DL, DT, AC) ==
-            FalseVal ||
-        SimplifyWithOpReplaced(TrueVal, CmpRHS, CmpLHS, TLI, DL, DT, AC) ==
-            FalseVal)
-      return ReplaceInstUsesWith(SI, TrueVal);
-    if (SimplifyWithOpReplaced(FalseVal, CmpLHS, CmpRHS, TLI, DL, DT, AC) ==
-            TrueVal ||
-        SimplifyWithOpReplaced(FalseVal, CmpRHS, CmpLHS, TLI, DL, DT, AC) ==
-            TrueVal)
-      return ReplaceInstUsesWith(SI, TrueVal);
-  }
-
   // NOTE: if we wanted to, this is where to detect integer MIN/MAX
 
   if (CmpRHS != CmpLHS && isa<Constant>(CmpRHS)) {
@@ -672,7 +538,8 @@ Instruction *InstCombiner::visitSelectInstWithICmp(SelectInst &SI,
     }
   }
 
-  if (unsigned BitWidth = TrueVal->getType()->getScalarSizeInBits()) {
+  {
+    unsigned BitWidth = DL.getTypeSizeInBits(TrueVal->getType());
     APInt MinSignedValue = APInt::getSignBit(BitWidth);
     Value *X;
     const APInt *Y, *C;
@@ -829,6 +696,52 @@ Instruction *InstCombiner::FoldSPFofSPF(Instruction *Inner,
         SI->getCondition(), SI->getFalseValue(), SI->getTrueValue());
     return ReplaceInstUsesWith(Outer, NewSI);
   }
+
+  auto IsFreeOrProfitableToInvert =
+      [&](Value *V, Value *&NotV, bool &ElidesXor) {
+    if (match(V, m_Not(m_Value(NotV)))) {
+      // If V has at most 2 uses then we can get rid of the xor operation
+      // entirely.
+      ElidesXor |= !V->hasNUsesOrMore(3);
+      return true;
+    }
+
+    if (IsFreeToInvert(V, !V->hasNUsesOrMore(3))) {
+      NotV = nullptr;
+      return true;
+    }
+
+    return false;
+  };
+
+  Value *NotA, *NotB, *NotC;
+  bool ElidesXor = false;
+
+  // MIN(MIN(~A, ~B), ~C) == ~MAX(MAX(A, B), C)
+  // MIN(MAX(~A, ~B), ~C) == ~MAX(MIN(A, B), C)
+  // MAX(MIN(~A, ~B), ~C) == ~MIN(MAX(A, B), C)
+  // MAX(MAX(~A, ~B), ~C) == ~MIN(MIN(A, B), C)
+  //
+  // This transform is performance neutral if we can elide at least one xor from
+  // the set of three operands, since we'll be tacking on an xor at the very
+  // end.
+  if (IsFreeOrProfitableToInvert(A, NotA, ElidesXor) &&
+      IsFreeOrProfitableToInvert(B, NotB, ElidesXor) &&
+      IsFreeOrProfitableToInvert(C, NotC, ElidesXor) && ElidesXor) {
+    if (!NotA)
+      NotA = Builder->CreateNot(A);
+    if (!NotB)
+      NotB = Builder->CreateNot(B);
+    if (!NotC)
+      NotC = Builder->CreateNot(C);
+
+    Value *NewInner = generateMinMaxSelectPattern(
+        Builder, getInverseMinMaxSelectPattern(SPF1), NotA, NotB);
+    Value *NewOuter = Builder->CreateNot(generateMinMaxSelectPattern(
+        Builder, getInverseMinMaxSelectPattern(SPF2), NewInner, NotC));
+    return ReplaceInstUsesWith(Outer, NewOuter);
+  }
+
   return nullptr;
 }
 
@@ -1141,23 +1054,35 @@ Instruction *InstCombiner::visitSelectInst(SelectInst &SI) {
       }
 
   // See if we can fold the select into one of our operands.
-  if (SI.getType()->isIntegerTy()) {
+  if (SI.getType()->isIntOrIntVectorTy()) {
     if (Instruction *FoldI = FoldSelectIntoOp(SI, TrueVal, FalseVal))
       return FoldI;
 
     Value *LHS, *RHS, *LHS2, *RHS2;
-    SelectPatternFlavor SPF = MatchSelectPattern(&SI, LHS, RHS);
+    Instruction::CastOps CastOp;
+    SelectPatternFlavor SPF = matchSelectPattern(&SI, LHS, RHS, &CastOp);
 
-    // MAX(MAX(a, b), a) -> MAX(a, b)
-    // MIN(MIN(a, b), a) -> MIN(a, b)
-    // MAX(MIN(a, b), a) -> a
-    // MIN(MAX(a, b), a) -> a
     if (SPF) {
-      if (SelectPatternFlavor SPF2 = MatchSelectPattern(LHS, LHS2, RHS2))
+      // Canonicalize so that type casts are outside select patterns.
+      if (LHS->getType()->getPrimitiveSizeInBits() !=
+          SI.getType()->getPrimitiveSizeInBits()) {
+        CmpInst::Predicate Pred = getICmpPredicateForMinMax(SPF);
+        Value *Cmp = Builder->CreateICmp(Pred, LHS, RHS);
+        Value *NewSI = Builder->CreateCast(CastOp,
+                                           Builder->CreateSelect(Cmp, LHS, RHS),
+                                           SI.getType());
+        return ReplaceInstUsesWith(SI, NewSI);
+      }
+
+      // MAX(MAX(a, b), a) -> MAX(a, b)
+      // MIN(MIN(a, b), a) -> MIN(a, b)
+      // MAX(MIN(a, b), a) -> a
+      // MIN(MAX(a, b), a) -> a
+      if (SelectPatternFlavor SPF2 = matchSelectPattern(LHS, LHS2, RHS2))
         if (Instruction *R = FoldSPFofSPF(cast<Instruction>(LHS),SPF2,LHS2,RHS2,
                                           SI, SPF, RHS))
           return R;
-      if (SelectPatternFlavor SPF2 = MatchSelectPattern(RHS, LHS2, RHS2))
+      if (SelectPatternFlavor SPF2 = matchSelectPattern(RHS, LHS2, RHS2))
         if (Instruction *R = FoldSPFofSPF(cast<Instruction>(RHS),SPF2,LHS2,RHS2,
                                           SI, SPF, LHS))
           return R;
diff --git a/lib/Transforms/InstCombine/InstCombineShifts.cpp b/lib/Transforms/InstCombine/InstCombineShifts.cpp
index a414ec600ab..d04ed58b014 100644
--- a/lib/Transforms/InstCombine/InstCombineShifts.cpp
+++ b/lib/Transforms/InstCombine/InstCombineShifts.cpp
@@ -175,8 +175,8 @@ static bool CanEvaluateShifted(Value *V, unsigned NumBits, bool isLeftShift,
     // get into trouble with cyclic PHIs here because we only consider
     // instructions with a single use.
     PHINode *PN = cast<PHINode>(I);
-    for (unsigned i = 0, e = PN->getNumIncomingValues(); i != e; ++i)
-      if (!CanEvaluateShifted(PN->getIncomingValue(i), NumBits, isLeftShift,
+    for (Value *IncValue : PN->incoming_values())
+      if (!CanEvaluateShifted(IncValue, NumBits, isLeftShift,
                               IC, PN))
         return false;
     return true;
diff --git a/lib/Transforms/InstCombine/InstCombineSimplifyDemanded.cpp b/lib/Transforms/InstCombine/InstCombineSimplifyDemanded.cpp
index cd391d0385e..80628b23f11 100644
--- a/lib/Transforms/InstCombine/InstCombineSimplifyDemanded.cpp
+++ b/lib/Transforms/InstCombine/InstCombineSimplifyDemanded.cpp
@@ -13,6 +13,7 @@
 //===----------------------------------------------------------------------===//
 
 #include "InstCombineInternal.h"
+#include "llvm/Analysis/ValueTracking.h"
 #include "llvm/IR/IntrinsicInst.h"
 #include "llvm/IR/PatternMatch.h"
 
@@ -43,19 +44,6 @@ static bool ShrinkDemandedConstant(Instruction *I, unsigned OpNo,
   Demanded &= OpC->getValue();
   I->setOperand(OpNo, ConstantInt::get(OpC->getType(), Demanded));
 
-  // If either 'nsw' or 'nuw' is set and the constant is negative,
-  // removing *any* bits from the constant could make overflow occur.
-  // Remove 'nsw' and 'nuw' from the instruction in this case.
-  if (auto *OBO = dyn_cast<OverflowingBinaryOperator>(I)) {
-    assert(OBO->getOpcode() == Instruction::Add);
-    if (OBO->hasNoSignedWrap() || OBO->hasNoUnsignedWrap()) {
-      if (OpC->getValue().isNegative()) {
-        cast<BinaryOperator>(OBO)->setHasNoSignedWrap(false);
-        cast<BinaryOperator>(OBO)->setHasNoUnsignedWrap(false);
-      }
-    }
-  }
-
   return true;
 }
 
@@ -83,9 +71,9 @@ bool InstCombiner::SimplifyDemandedInstructionBits(Instruction &Inst) {
 bool InstCombiner::SimplifyDemandedBits(Use &U, APInt DemandedMask,
                                         APInt &KnownZero, APInt &KnownOne,
                                         unsigned Depth) {
-  Value *NewVal =
-      SimplifyDemandedUseBits(U.get(), DemandedMask, KnownZero, KnownOne, Depth,
-                              dyn_cast<Instruction>(U.getUser()));
+  auto *UserI = dyn_cast<Instruction>(U.getUser());
+  Value *NewVal = SimplifyDemandedUseBits(U.get(), DemandedMask, KnownZero,
+                                          KnownOne, Depth, UserI);
   if (!NewVal) return false;
   U = NewVal;
   return true;
@@ -419,6 +407,12 @@ Value *InstCombiner::SimplifyDemandedUseBits(Value *V, APInt DemandedMask,
     break;
   }
   case Instruction::Select:
+    // If this is a select as part of a min/max pattern, don't simplify any
+    // further in case we break the structure.
+    Value *LHS, *RHS;
+    if (matchSelectPattern(I, LHS, RHS) != SPF_UNKNOWN)
+      return nullptr;
+      
     if (SimplifyDemandedBits(I->getOperandUse(2), DemandedMask, RHSKnownZero,
                              RHSKnownOne, Depth + 1) ||
         SimplifyDemandedBits(I->getOperandUse(1), DemandedMask, LHSKnownZero,
@@ -528,113 +522,35 @@ Value *InstCombiner::SimplifyDemandedUseBits(Value *V, APInt DemandedMask,
     }
     break;
   }
-  case Instruction::Add: {
-    // Figure out what the input bits are.  If the top bits of the and result
-    // are not demanded, then the add doesn't demand them from its input
-    // either.
+  case Instruction::Add:
+  case Instruction::Sub: {
+    /// If the high-bits of an ADD/SUB are not demanded, then we do not care
+    /// about the high bits of the operands.
     unsigned NLZ = DemandedMask.countLeadingZeros();
-
-    // If there is a constant on the RHS, there are a variety of xformations
-    // we can do.
-    if (ConstantInt *RHS = dyn_cast<ConstantInt>(I->getOperand(1))) {
-      // If null, this should be simplified elsewhere.  Some of the xforms here
-      // won't work if the RHS is zero.
-      if (RHS->isZero())
-        break;
-
-      // If the top bit of the output is demanded, demand everything from the
-      // input.  Otherwise, we demand all the input bits except NLZ top bits.
-      APInt InDemandedBits(APInt::getLowBitsSet(BitWidth, BitWidth - NLZ));
-
-      // Find information about known zero/one bits in the input.
-      if (SimplifyDemandedBits(I->getOperandUse(0), InDemandedBits,
-                               LHSKnownZero, LHSKnownOne, Depth + 1))
-        return I;
-
-      // If the RHS of the add has bits set that can't affect the input, reduce
-      // the constant.
-      if (ShrinkDemandedConstant(I, 1, InDemandedBits))
-        return I;
-
-      // Avoid excess work.
-      if (LHSKnownZero == 0 && LHSKnownOne == 0)
-        break;
-
-      // Turn it into OR if input bits are zero.
-      if ((LHSKnownZero & RHS->getValue()) == RHS->getValue()) {
-        Instruction *Or =
-          BinaryOperator::CreateOr(I->getOperand(0), I->getOperand(1),
-                                   I->getName());
-        return InsertNewInstWith(Or, *I);
-      }
-
-      // We can say something about the output known-zero and known-one bits,
-      // depending on potential carries from the input constant and the
-      // unknowns.  For example if the LHS is known to have at most the 0x0F0F0
-      // bits set and the RHS constant is 0x01001, then we know we have a known
-      // one mask of 0x00001 and a known zero mask of 0xE0F0E.
-
-      // To compute this, we first compute the potential carry bits.  These are
-      // the bits which may be modified.  I'm not aware of a better way to do
-      // this scan.
-      const APInt &RHSVal = RHS->getValue();
-      APInt CarryBits((~LHSKnownZero + RHSVal) ^ (~LHSKnownZero ^ RHSVal));
-
-      // Now that we know which bits have carries, compute the known-1/0 sets.
-
-      // Bits are known one if they are known zero in one operand and one in the
-      // other, and there is no input carry.
-      KnownOne = ((LHSKnownZero & RHSVal) |
-                  (LHSKnownOne & ~RHSVal)) & ~CarryBits;
-
-      // Bits are known zero if they are known zero in both operands and there
-      // is no input carry.
-      KnownZero = LHSKnownZero & ~RHSVal & ~CarryBits;
-    } else {
-      // If the high-bits of this ADD are not demanded, then it does not demand
-      // the high bits of its LHS or RHS.
-      if (DemandedMask[BitWidth-1] == 0) {
-        // Right fill the mask of bits for this ADD to demand the most
-        // significant bit and all those below it.
-        APInt DemandedFromOps(APInt::getLowBitsSet(BitWidth, BitWidth-NLZ));
-        if (SimplifyDemandedBits(I->getOperandUse(0), DemandedFromOps,
-                                 LHSKnownZero, LHSKnownOne, Depth + 1) ||
-            SimplifyDemandedBits(I->getOperandUse(1), DemandedFromOps,
-                                 LHSKnownZero, LHSKnownOne, Depth + 1))
-          return I;
-      }
-    }
-    break;
-  }
-  case Instruction::Sub:
-    // If the high-bits of this SUB are not demanded, then it does not demand
-    // the high bits of its LHS or RHS.
-    if (DemandedMask[BitWidth-1] == 0) {
-      // Right fill the mask of bits for this SUB to demand the most
+    if (NLZ > 0) {
+      // Right fill the mask of bits for this ADD/SUB to demand the most
       // significant bit and all those below it.
-      uint32_t NLZ = DemandedMask.countLeadingZeros();
       APInt DemandedFromOps(APInt::getLowBitsSet(BitWidth, BitWidth-NLZ));
       if (SimplifyDemandedBits(I->getOperandUse(0), DemandedFromOps,
                                LHSKnownZero, LHSKnownOne, Depth + 1) ||
+          ShrinkDemandedConstant(I, 1, DemandedFromOps) ||
           SimplifyDemandedBits(I->getOperandUse(1), DemandedFromOps,
-                               LHSKnownZero, LHSKnownOne, Depth + 1))
+                               LHSKnownZero, LHSKnownOne, Depth + 1)) {
+        // Disable the nsw and nuw flags here: We can no longer guarantee that
+        // we won't wrap after simplification. Removing the nsw/nuw flags is
+        // legal here because the top bit is not demanded.
+        BinaryOperator &BinOP = *cast<BinaryOperator>(I);
+        BinOP.setHasNoSignedWrap(false);
+        BinOP.setHasNoUnsignedWrap(false);
         return I;
+      }
     }
 
-    // Otherwise just hand the sub off to computeKnownBits to fill in
+    // Otherwise just hand the add/sub off to computeKnownBits to fill in
     // the known zeros and ones.
     computeKnownBits(V, KnownZero, KnownOne, Depth, CxtI);
-
-    // Turn this into a xor if LHS is 2^n-1 and the remaining bits are known
-    // zero.
-    if (ConstantInt *C0 = dyn_cast<ConstantInt>(I->getOperand(0))) {
-      APInt I0 = C0->getValue();
-      if ((I0 + 1).isPowerOf2() && (I0 | KnownZero).isAllOnesValue()) {
-        Instruction *Xor = BinaryOperator::CreateXor(I->getOperand(1), C0);
-        return InsertNewInstWith(Xor, *I);
-      }
-    }
     break;
+  }
   case Instruction::Shl:
     if (ConstantInt *SA = dyn_cast<ConstantInt>(I->getOperand(1))) {
       {
diff --git a/lib/Transforms/InstCombine/InstCombineVectorOps.cpp b/lib/Transforms/InstCombine/InstCombineVectorOps.cpp
index 7f86e7d24ff..79583155d00 100644
--- a/lib/Transforms/InstCombine/InstCombineVectorOps.cpp
+++ b/lib/Transforms/InstCombine/InstCombineVectorOps.cpp
@@ -14,6 +14,8 @@
 
 #include "InstCombineInternal.h"
 #include "llvm/ADT/DenseMap.h"
+#include "llvm/Analysis/InstructionSimplify.h"
+#include "llvm/Analysis/VectorUtils.h"
 #include "llvm/IR/PatternMatch.h"
 using namespace llvm;
 using namespace PatternMatch;
@@ -60,56 +62,6 @@ static bool CheapToScalarize(Value *V, bool isConstant) {
   return false;
 }
 
-/// FindScalarElement - Given a vector and an element number, see if the scalar
-/// value is already around as a register, for example if it were inserted then
-/// extracted from the vector.
-static Value *FindScalarElement(Value *V, unsigned EltNo) {
-  assert(V->getType()->isVectorTy() && "Not looking at a vector?");
-  VectorType *VTy = cast<VectorType>(V->getType());
-  unsigned Width = VTy->getNumElements();
-  if (EltNo >= Width)  // Out of range access.
-    return UndefValue::get(VTy->getElementType());
-
-  if (Constant *C = dyn_cast<Constant>(V))
-    return C->getAggregateElement(EltNo);
-
-  if (InsertElementInst *III = dyn_cast<InsertElementInst>(V)) {
-    // If this is an insert to a variable element, we don't know what it is.
-    if (!isa<ConstantInt>(III->getOperand(2)))
-      return nullptr;
-    unsigned IIElt = cast<ConstantInt>(III->getOperand(2))->getZExtValue();
-
-    // If this is an insert to the element we are looking for, return the
-    // inserted value.
-    if (EltNo == IIElt)
-      return III->getOperand(1);
-
-    // Otherwise, the insertelement doesn't modify the value, recurse on its
-    // vector input.
-    return FindScalarElement(III->getOperand(0), EltNo);
-  }
-
-  if (ShuffleVectorInst *SVI = dyn_cast<ShuffleVectorInst>(V)) {
-    unsigned LHSWidth = SVI->getOperand(0)->getType()->getVectorNumElements();
-    int InEl = SVI->getMaskValue(EltNo);
-    if (InEl < 0)
-      return UndefValue::get(VTy->getElementType());
-    if (InEl < (int)LHSWidth)
-      return FindScalarElement(SVI->getOperand(0), InEl);
-    return FindScalarElement(SVI->getOperand(1), InEl - LHSWidth);
-  }
-
-  // Extract a value from a vector add operation with a constant zero.
-  Value *Val = nullptr; Constant *Con = nullptr;
-  if (match(V, m_Add(m_Value(Val), m_Constant(Con)))) {
-    if (Con->getAggregateElement(EltNo)->isNullValue())
-      return FindScalarElement(Val, EltNo);
-  }
-
-  // Otherwise, we don't know.
-  return nullptr;
-}
-
 // If we have a PHI node with a vector type that has only 2 uses: feed
 // itself and be an operand of extractelement at a constant location,
 // try to replace the PHI of the vector type with a PHI of a scalar type.
@@ -178,6 +130,10 @@ Instruction *InstCombiner::scalarizePHI(ExtractElementInst &EI, PHINode *PN) {
 }
 
 Instruction *InstCombiner::visitExtractElementInst(ExtractElementInst &EI) {
+  if (Value *V = SimplifyExtractElementInst(
+          EI.getVectorOperand(), EI.getIndexOperand(), DL, TLI, DT, AC))
+    return ReplaceInstUsesWith(EI, V);
+
   // If vector val is constant with all elements the same, replace EI with
   // that element.  We handle a known element # below.
   if (Constant *C = dyn_cast<Constant>(EI.getOperand(0)))
@@ -190,10 +146,8 @@ Instruction *InstCombiner::visitExtractElementInst(ExtractElementInst &EI) {
     unsigned IndexVal = IdxC->getZExtValue();
     unsigned VectorWidth = EI.getVectorOperandType()->getNumElements();
 
-    // If this is extracting an invalid index, turn this into undef, to avoid
-    // crashing the code below.
-    if (IndexVal >= VectorWidth)
-      return ReplaceInstUsesWith(EI, UndefValue::get(EI.getType()));
+    // InstSimplify handles cases where the index is invalid.
+    assert(IndexVal < VectorWidth);
 
     // This instruction only demands the single element from the input vector.
     // If the input vector has a single use, simplify it based on this use
@@ -209,16 +163,13 @@ Instruction *InstCombiner::visitExtractElementInst(ExtractElementInst &EI) {
       }
     }
 
-    if (Value *Elt = FindScalarElement(EI.getOperand(0), IndexVal))
-      return ReplaceInstUsesWith(EI, Elt);
-
     // If the this extractelement is directly using a bitcast from a vector of
     // the same number of elements, see if we can find the source element from
     // it.  In this case, we will end up needing to bitcast the scalars.
     if (BitCastInst *BCI = dyn_cast<BitCastInst>(EI.getOperand(0))) {
       if (VectorType *VT = dyn_cast<VectorType>(BCI->getOperand(0)->getType()))
         if (VT->getNumElements() == VectorWidth)
-          if (Value *Elt = FindScalarElement(BCI->getOperand(0), IndexVal))
+          if (Value *Elt = findScalarElement(BCI->getOperand(0), IndexVal))
             return new BitCastInst(Elt, EI.getType());
     }
 
diff --git a/lib/Transforms/InstCombine/InstructionCombining.cpp b/lib/Transforms/InstCombine/InstructionCombining.cpp
index 3b46156c1ab..d04ae0702a6 100644
--- a/lib/Transforms/InstCombine/InstructionCombining.cpp
+++ b/lib/Transforms/InstCombine/InstructionCombining.cpp
@@ -452,6 +452,7 @@ static Value *tryFactorization(InstCombiner::BuilderTy *Builder,
   if (!A || !C || !B || !D)
     return nullptr;
 
+  Value *V = nullptr;
   Value *SimplifiedInst = nullptr;
   Value *LHS = I.getOperand(0), *RHS = I.getOperand(1);
   Instruction::BinaryOps TopLevelOpcode = I.getOpcode();
@@ -468,7 +469,7 @@ static Value *tryFactorization(InstCombiner::BuilderTy *Builder,
         std::swap(C, D);
       // Consider forming "A op' (B op D)".
       // If "B op D" simplifies then it can be formed with no cost.
-      Value *V = SimplifyBinOp(TopLevelOpcode, B, D, DL);
+      V = SimplifyBinOp(TopLevelOpcode, B, D, DL);
       // If "B op D" doesn't simplify then only go on if both of the existing
       // operations "A op' B" and "C op' D" will be zapped as no longer used.
       if (!V && LHS->hasOneUse() && RHS->hasOneUse())
@@ -487,7 +488,7 @@ static Value *tryFactorization(InstCombiner::BuilderTy *Builder,
         std::swap(C, D);
       // Consider forming "(A op C) op' B".
       // If "A op C" simplifies then it can be formed with no cost.
-      Value *V = SimplifyBinOp(TopLevelOpcode, A, C, DL);
+      V = SimplifyBinOp(TopLevelOpcode, A, C, DL);
 
       // If "A op C" doesn't simplify then only go on if both of the existing
       // operations "A op' B" and "C op' D" will be zapped as no longer used.
@@ -517,7 +518,19 @@ static Value *tryFactorization(InstCombiner::BuilderTy *Builder,
         if (BinaryOperator *Op1 = dyn_cast<BinaryOperator>(RHS))
           if (isa<OverflowingBinaryOperator>(Op1))
             HasNSW &= Op1->hasNoSignedWrap();
-        BO->setHasNoSignedWrap(HasNSW);
+
+        // We can propogate 'nsw' if we know that
+        //  %Y = mul nsw i16 %X, C
+        //  %Z = add nsw i16 %Y, %X
+        // =>
+        //  %Z = mul nsw i16 %X, C+1
+        //
+        // iff C+1 isn't INT_MIN
+        const APInt *CInt;
+        if (TopLevelOpcode == Instruction::Add &&
+            InnerOpcode == Instruction::Mul)
+          if (match(V, m_APInt(CInt)) && !CInt->isMinSignedValue())
+            BO->setHasNoSignedWrap(HasNSW);
       }
     }
   }
@@ -610,6 +623,33 @@ Value *InstCombiner::SimplifyUsingDistributiveLaws(BinaryOperator &I) {
       }
   }
 
+  // (op (select (a, c, b)), (select (a, d, b))) -> (select (a, (op c, d), 0))
+  // (op (select (a, b, c)), (select (a, b, d))) -> (select (a, 0, (op c, d)))
+  if (auto *SI0 = dyn_cast<SelectInst>(LHS)) {
+    if (auto *SI1 = dyn_cast<SelectInst>(RHS)) {
+      if (SI0->getCondition() == SI1->getCondition()) {
+        Value *SI = nullptr;
+        if (Value *V = SimplifyBinOp(TopLevelOpcode, SI0->getFalseValue(),
+                                     SI1->getFalseValue(), DL, TLI, DT, AC))
+          SI = Builder->CreateSelect(SI0->getCondition(),
+                                     Builder->CreateBinOp(TopLevelOpcode,
+                                                          SI0->getTrueValue(),
+                                                          SI1->getTrueValue()),
+                                     V);
+        if (Value *V = SimplifyBinOp(TopLevelOpcode, SI0->getTrueValue(),
+                                     SI1->getTrueValue(), DL, TLI, DT, AC))
+          SI = Builder->CreateSelect(
+              SI0->getCondition(), V,
+              Builder->CreateBinOp(TopLevelOpcode, SI0->getFalseValue(),
+                                   SI1->getFalseValue()));
+        if (SI) {
+          SI->takeName(&I);
+          return SI;
+        }
+      }
+    }
+  }
+
   return nullptr;
 }
 
@@ -714,6 +754,22 @@ Instruction *InstCombiner::FoldOpIntoSelect(Instruction &Op, SelectInst *SI) {
         return nullptr;
     }
 
+    // Test if a CmpInst instruction is used exclusively by a select as
+    // part of a minimum or maximum operation. If so, refrain from doing
+    // any other folding. This helps out other analyses which understand
+    // non-obfuscated minimum and maximum idioms, such as ScalarEvolution
+    // and CodeGen. And in this case, at least one of the comparison
+    // operands has at least one user besides the compare (the select),
+    // which would often largely negate the benefit of folding anyway.
+    if (auto *CI = dyn_cast<CmpInst>(SI->getCondition())) {
+      if (CI->hasOneUse()) {
+        Value *Op0 = CI->getOperand(0), *Op1 = CI->getOperand(1);
+        if ((SI->getOperand(1) == Op0 && SI->getOperand(2) == Op1) ||
+            (SI->getOperand(2) == Op0 && SI->getOperand(1) == Op1))
+          return nullptr;
+      }
+    }
+
     Value *SelectTrueVal = FoldOperationIntoSelectOperand(Op, TV, this);
     Value *SelectFalseVal = FoldOperationIntoSelectOperand(Op, FV, this);
 
@@ -723,7 +779,6 @@ Instruction *InstCombiner::FoldOpIntoSelect(Instruction &Op, SelectInst *SI) {
   return nullptr;
 }
 
-
 /// FoldOpIntoPhi - Given a binary operator, cast instruction, or select which
 /// has a PHI node as operand #0, see if we can fold the instruction into the
 /// PHI (which is only possible if all operands to the PHI are constants).
@@ -1467,6 +1522,11 @@ Instruction *InstCombiner::visitGetElementPtrInst(GetElementPtrInst &GEP) {
         // normalized.
         if (SO1->getType() != GO1->getType())
           return nullptr;
+        // Only do the combine when GO1 and SO1 are both constants. Only in
+        // this case, we are sure the cost after the merge is never more than
+        // that before the merge.
+        if (!isa<Constant>(GO1) || !isa<Constant>(SO1))
+          return nullptr;
         Sum = Builder->CreateAdd(SO1, GO1, PtrOp->getName()+".sum");
       }
 
@@ -1597,6 +1657,7 @@ Instruction *InstCombiner::visitGetElementPtrInst(GetElementPtrInst &GEP) {
             // is a leading zero) we can fold the cast into this GEP.
             if (StrippedPtrTy->getAddressSpace() == GEP.getAddressSpace()) {
               GEP.setOperand(0, StrippedPtr);
+              GEP.setSourceElementType(XATy);
               return &GEP;
             }
             // Cannot replace the base pointer directly because StrippedPtr's
@@ -1809,7 +1870,7 @@ isAllocSiteRemovable(Instruction *AI, SmallVectorImpl<WeakVH> &Users,
 
       case Instruction::BitCast:
       case Instruction::GetElementPtr:
-        Users.push_back(I);
+        Users.emplace_back(I);
         Worklist.push_back(I);
         continue;
 
@@ -1818,7 +1879,7 @@ isAllocSiteRemovable(Instruction *AI, SmallVectorImpl<WeakVH> &Users,
         // We can fold eq/ne comparisons with null to false/true, respectively.
         if (!ICI->isEquality() || !isa<ConstantPointerNull>(ICI->getOperand(1)))
           return false;
-        Users.push_back(I);
+        Users.emplace_back(I);
         continue;
       }
 
@@ -1844,13 +1905,13 @@ isAllocSiteRemovable(Instruction *AI, SmallVectorImpl<WeakVH> &Users,
           case Intrinsic::lifetime_start:
           case Intrinsic::lifetime_end:
           case Intrinsic::objectsize:
-            Users.push_back(I);
+            Users.emplace_back(I);
             continue;
           }
         }
 
         if (isFreeCall(I, TLI)) {
-          Users.push_back(I);
+          Users.emplace_back(I);
           continue;
         }
         return false;
@@ -1859,7 +1920,7 @@ isAllocSiteRemovable(Instruction *AI, SmallVectorImpl<WeakVH> &Users,
         StoreInst *SI = cast<StoreInst>(I);
         if (SI->isVolatile() || SI->getPointerOperand() != PI)
           return false;
-        Users.push_back(I);
+        Users.emplace_back(I);
         continue;
       }
       }
@@ -2091,7 +2152,7 @@ Instruction *InstCombiner::visitSwitchInst(SwitchInst &SI) {
 
   // Truncate the condition operand if the new type is equal to or larger than
   // the largest legal integer type. We need to be conservative here since
-  // x86 generates redundant zero-extenstion instructions if the operand is
+  // x86 generates redundant zero-extension instructions if the operand is
   // truncated to i8 or i16.
   bool TruncCond = false;
   if (NewWidth > 0 && BitWidth > NewWidth &&
@@ -2140,16 +2201,9 @@ Instruction *InstCombiner::visitExtractValueInst(ExtractValueInst &EV) {
   if (!EV.hasIndices())
     return ReplaceInstUsesWith(EV, Agg);
 
-  if (Constant *C = dyn_cast<Constant>(Agg)) {
-    if (Constant *C2 = C->getAggregateElement(*EV.idx_begin())) {
-      if (EV.getNumIndices() == 0)
-        return ReplaceInstUsesWith(EV, C2);
-      // Extract the remaining indices out of the constant indexed by the
-      // first index
-      return ExtractValueInst::Create(C2, EV.getIndices().slice(1));
-    }
-    return nullptr; // Can't handle other constants
-  }
+  if (Value *V =
+          SimplifyExtractValueInst(Agg, EV.getIndices(), DL, TLI, DT, AC))
+    return ReplaceInstUsesWith(EV, V);
 
   if (InsertValueInst *IV = dyn_cast<InsertValueInst>(Agg)) {
     // We're extracting from an insertvalue instruction, compare the indices
@@ -2319,7 +2373,8 @@ Instruction *InstCombiner::visitLandingPadInst(LandingPadInst &LI) {
   // The logic here should be correct for any real-world personality function.
   // However if that turns out not to be true, the offending logic can always
   // be conditioned on the personality function, like the catch-all logic is.
-  EHPersonality Personality = classifyEHPersonality(LI.getPersonalityFn());
+  EHPersonality Personality =
+      classifyEHPersonality(LI.getParent()->getParent()->getPersonalityFn());
 
   // Simplify the list of clauses, eg by removing repeated catch clauses
   // (these are often created by inlining).
@@ -2586,7 +2641,6 @@ Instruction *InstCombiner::visitLandingPadInst(LandingPadInst &LI) {
   // with a new one.
   if (MakeNewInstruction) {
     LandingPadInst *NLI = LandingPadInst::Create(LI.getType(),
-                                                 LI.getPersonalityFn(),
                                                  NewClauses.size());
     for (unsigned i = 0, e = NewClauses.size(); i != e; ++i)
       NLI->addClause(NewClauses[i]);
@@ -2657,7 +2711,8 @@ bool InstCombiner::run() {
     }
 
     // Instruction isn't dead, see if we can constant propagate it.
-    if (!I->use_empty() && isa<Constant>(I->getOperand(0))) {
+    if (!I->use_empty() &&
+        (I->getNumOperands() == 0 || isa<Constant>(I->getOperand(0)))) {
       if (Constant *C = ConstantFoldInstruction(I, DL, TLI)) {
         DEBUG(dbgs() << "IC: ConstFold to: " << *C << " from: " << *I << '\n');
 
@@ -2812,7 +2867,8 @@ static bool AddReachableCodeToWorklist(BasicBlock *BB, const DataLayout &DL,
       }
 
       // ConstantProp instruction if trivially constant.
-      if (!Inst->use_empty() && isa<Constant>(Inst->getOperand(0)))
+      if (!Inst->use_empty() &&
+          (Inst->getNumOperands() == 0 || isa<Constant>(Inst->getOperand(0))))
         if (Constant *C = ConstantFoldInstruction(Inst, DL, TLI)) {
           DEBUG(dbgs() << "IC: ConstFold to: " << *C << " from: "
                        << *Inst << '\n');
@@ -2936,8 +2992,9 @@ static bool prepareICWorklistFromFunction(Function &F, const DataLayout &DL,
 
 static bool
 combineInstructionsOverFunction(Function &F, InstCombineWorklist &Worklist,
-                                AssumptionCache &AC, TargetLibraryInfo &TLI,
-                                DominatorTree &DT, LoopInfo *LI = nullptr) {
+                                AliasAnalysis *AA, AssumptionCache &AC,
+                                TargetLibraryInfo &TLI, DominatorTree &DT,
+                                LoopInfo *LI = nullptr) {
   // Minimizing size?
   bool MinimizeSize = F.hasFnAttribute(Attribute::MinSize);
   auto &DL = F.getParent()->getDataLayout();
@@ -2962,7 +3019,8 @@ combineInstructionsOverFunction(Function &F, InstCombineWorklist &Worklist,
     if (prepareICWorklistFromFunction(F, DL, &TLI, Worklist))
       Changed = true;
 
-    InstCombiner IC(Worklist, &Builder, MinimizeSize, &AC, &TLI, &DT, DL, LI);
+    InstCombiner IC(Worklist, &Builder, MinimizeSize,
+                    AA, &AC, &TLI, &DT, DL, LI);
     if (IC.run())
       Changed = true;
 
@@ -2981,7 +3039,8 @@ PreservedAnalyses InstCombinePass::run(Function &F,
 
   auto *LI = AM->getCachedResult<LoopAnalysis>(F);
 
-  if (!combineInstructionsOverFunction(F, Worklist, AC, TLI, DT, LI))
+  // FIXME: The AliasAnalysis is not yet supported in the new pass manager
+  if (!combineInstructionsOverFunction(F, Worklist, nullptr, AC, TLI, DT, LI))
     // No changes, all analyses are preserved.
     return PreservedAnalyses::all();
 
@@ -3014,6 +3073,7 @@ class InstructionCombiningPass : public FunctionPass {
 
 void InstructionCombiningPass::getAnalysisUsage(AnalysisUsage &AU) const {
   AU.setPreservesCFG();
+  AU.addRequired<AliasAnalysis>();
   AU.addRequired<AssumptionCacheTracker>();
   AU.addRequired<TargetLibraryInfoWrapperPass>();
   AU.addRequired<DominatorTreeWrapperPass>();
@@ -3025,6 +3085,7 @@ bool InstructionCombiningPass::runOnFunction(Function &F) {
     return false;
 
   // Required analyses.
+  auto AA = &getAnalysis<AliasAnalysis>();
   auto &AC = getAnalysis<AssumptionCacheTracker>().getAssumptionCache(F);
   auto &TLI = getAnalysis<TargetLibraryInfoWrapperPass>().getTLI();
   auto &DT = getAnalysis<DominatorTreeWrapperPass>().getDomTree();
@@ -3033,7 +3094,7 @@ bool InstructionCombiningPass::runOnFunction(Function &F) {
   auto *LIWP = getAnalysisIfAvailable<LoopInfoWrapperPass>();
   auto *LI = LIWP ? &LIWP->getLoopInfo() : nullptr;
 
-  return combineInstructionsOverFunction(F, Worklist, AC, TLI, DT, LI);
+  return combineInstructionsOverFunction(F, Worklist, AA, AC, TLI, DT, LI);
 }
 
 char InstructionCombiningPass::ID = 0;
@@ -3042,6 +3103,7 @@ INITIALIZE_PASS_BEGIN(InstructionCombiningPass, "instcombine",
 INITIALIZE_PASS_DEPENDENCY(AssumptionCacheTracker)
 INITIALIZE_PASS_DEPENDENCY(TargetLibraryInfoWrapperPass)
 INITIALIZE_PASS_DEPENDENCY(DominatorTreeWrapperPass)
+INITIALIZE_AG_DEPENDENCY(AliasAnalysis)
 INITIALIZE_PASS_END(InstructionCombiningPass, "instcombine",
                     "Combine redundant instructions", false, false)
 
diff --git a/lib/Transforms/Instrumentation/AddressSanitizer.cpp b/lib/Transforms/Instrumentation/AddressSanitizer.cpp
index 8d6d3ce3e61..071cd796b66 100644
--- a/lib/Transforms/Instrumentation/AddressSanitizer.cpp
+++ b/lib/Transforms/Instrumentation/AddressSanitizer.cpp
@@ -18,6 +18,7 @@
 #include "llvm/ADT/DenseMap.h"
 #include "llvm/ADT/DenseSet.h"
 #include "llvm/ADT/DepthFirstIterator.h"
+#include "llvm/ADT/SetVector.h"
 #include "llvm/ADT/SmallSet.h"
 #include "llvm/ADT/SmallString.h"
 #include "llvm/ADT/SmallVector.h"
@@ -67,6 +68,7 @@ static const uint64_t kDefaultShadowOffset32 = 1ULL << 29;
 static const uint64_t kIOSShadowOffset32 = 1ULL << 30;
 static const uint64_t kDefaultShadowOffset64 = 1ULL << 44;
 static const uint64_t kSmallX86_64ShadowOffset = 0x7FFF8000;  // < 2G.
+static const uint64_t kLinuxKasan_ShadowOffset64 = 0xdffffc0000000000;
 static const uint64_t kPPC64_ShadowOffset64 = 1ULL << 41;
 static const uint64_t kMIPS32_ShadowOffset32 = 0x0aaa0000;
 static const uint64_t kMIPS64_ShadowOffset64 = 1ULL << 37;
@@ -89,7 +91,9 @@ static const char *const kAsanUnregisterGlobalsName =
     "__asan_unregister_globals";
 static const char *const kAsanPoisonGlobalsName = "__asan_before_dynamic_init";
 static const char *const kAsanUnpoisonGlobalsName = "__asan_after_dynamic_init";
-static const char *const kAsanInitName = "__asan_init_v5";
+static const char *const kAsanInitName = "__asan_init";
+static const char *const kAsanVersionCheckName =
+    "__asan_version_mismatch_check_v6";
 static const char *const kAsanPtrCmp = "__sanitizer_ptr_cmp";
 static const char *const kAsanPtrSub = "__sanitizer_ptr_sub";
 static const char *const kAsanHandleNoReturnName = "__asan_handle_no_return";
@@ -106,16 +110,18 @@ static const char *const kAsanUnpoisonStackMemoryName =
 static const char *const kAsanOptionDetectUAR =
     "__asan_option_detect_stack_use_after_return";
 
+static const char *const kAsanAllocaPoison = "__asan_alloca_poison";
+static const char *const kAsanAllocasUnpoison = "__asan_allocas_unpoison";
+
 // Accesses sizes are powers of two: 1, 2, 4, 8, 16.
 static const size_t kNumberOfAccessSizes = 5;
 
 static const unsigned kAllocaRzSize = 32;
-static const unsigned kAsanAllocaLeftMagic = 0xcacacacaU;
-static const unsigned kAsanAllocaRightMagic = 0xcbcbcbcbU;
-static const unsigned kAsanAllocaPartialVal1 = 0xcbcbcb00U;
-static const unsigned kAsanAllocaPartialVal2 = 0x000000cbU;
 
 // Command-line flags.
+static cl::opt<bool> ClEnableKasan(
+    "asan-kernel", cl::desc("Enable KernelAddressSanitizer instrumentation"),
+    cl::Hidden, cl::init(false));
 
 // This flag may need to be replaced with -f[no-]asan-reads.
 static cl::opt<bool> ClInstrumentReads("asan-instrument-reads",
@@ -230,8 +236,6 @@ static cl::opt<int> ClDebugMax("asan-debug-max", cl::desc("Debug man inst"),
 
 STATISTIC(NumInstrumentedReads, "Number of instrumented reads");
 STATISTIC(NumInstrumentedWrites, "Number of instrumented writes");
-STATISTIC(NumInstrumentedDynamicAllocas,
-          "Number of instrumented dynamic allocas");
 STATISTIC(NumOptimizedAccessesToGlobalVar,
           "Number of optimized accesses to global vars");
 STATISTIC(NumOptimizedAccessesToStackVar,
@@ -250,8 +254,8 @@ struct LocationMetadata {
 
   void parse(MDNode *MDN) {
     assert(MDN->getNumOperands() == 3);
-    MDString *MDFilename = cast<MDString>(MDN->getOperand(0));
-    Filename = MDFilename->getString();
+    MDString *DIFilename = cast<MDString>(MDN->getOperand(0));
+    Filename = DIFilename->getString();
     LineNo =
         mdconst::extract<ConstantInt>(MDN->getOperand(1))->getLimitedValue();
     ColumnNo =
@@ -318,7 +322,8 @@ struct ShadowMapping {
   bool OrShadowOffset;
 };
 
-static ShadowMapping getShadowMapping(Triple &TargetTriple, int LongSize) {
+static ShadowMapping getShadowMapping(Triple &TargetTriple, int LongSize,
+                                      bool IsKasan) {
   bool IsAndroid = TargetTriple.getEnvironment() == llvm::Triple::Android;
   bool IsIOS = TargetTriple.isiOS();
   bool IsFreeBSD = TargetTriple.isOSFreeBSD();
@@ -336,6 +341,8 @@ static ShadowMapping getShadowMapping(Triple &TargetTriple, int LongSize) {
   ShadowMapping Mapping;
 
   if (LongSize == 32) {
+    // Android is always PIE, which means that the beginning of the address
+    // space is always available.
     if (IsAndroid)
       Mapping.Offset = 0;
     else if (IsMIPS32)
@@ -353,9 +360,12 @@ static ShadowMapping getShadowMapping(Triple &TargetTriple, int LongSize) {
       Mapping.Offset = kPPC64_ShadowOffset64;
     else if (IsFreeBSD)
       Mapping.Offset = kFreeBSD_ShadowOffset64;
-    else if (IsLinux && IsX86_64)
-      Mapping.Offset = kSmallX86_64ShadowOffset;
-    else if (IsMIPS64)
+    else if (IsLinux && IsX86_64) {
+      if (IsKasan)
+        Mapping.Offset = kLinuxKasan_ShadowOffset64;
+      else
+        Mapping.Offset = kSmallX86_64ShadowOffset;
+    } else if (IsMIPS64)
       Mapping.Offset = kMIPS64_ShadowOffset64;
     else if (IsAArch64)
       Mapping.Offset = kAArch64_ShadowOffset64;
@@ -384,7 +394,8 @@ static size_t RedzoneSizeForScale(int MappingScale) {
 
 /// AddressSanitizer: instrument the code in module to find memory bugs.
 struct AddressSanitizer : public FunctionPass {
-  AddressSanitizer() : FunctionPass(ID) {
+  explicit AddressSanitizer(bool CompileKernel = false)
+      : FunctionPass(ID), CompileKernel(CompileKernel || ClEnableKasan) {
     initializeAddressSanitizerPass(*PassRegistry::getPassRegistry());
   }
   const char *getPassName() const override {
@@ -402,11 +413,16 @@ struct AddressSanitizer : public FunctionPass {
   }
   /// Check if we want (and can) handle this alloca.
   bool isInterestingAlloca(AllocaInst &AI);
+
+  // Check if we have dynamic alloca.
+  bool isDynamicAlloca(AllocaInst &AI) const {
+    return AI.isArrayAllocation() || !AI.isStaticAlloca();
+  }
+
   /// If it is an interesting memory access, return the PointerOperand
   /// and set IsWrite/Alignment. Otherwise return nullptr.
   Value *isInterestingMemoryAccess(Instruction *I, bool *IsWrite,
-                                   uint64_t *TypeSize,
-                                   unsigned *Alignment);
+                                   uint64_t *TypeSize, unsigned *Alignment);
   void instrumentMop(ObjectSizeOffsetVisitor &ObjSizeVis, Instruction *I,
                      bool UseCalls, const DataLayout &DL);
   void instrumentPointerComparisonOrSubtraction(Instruction *I);
@@ -426,6 +442,7 @@ struct AddressSanitizer : public FunctionPass {
   Value *memToShadow(Value *Shadow, IRBuilder<> &IRB);
   bool runOnFunction(Function &F) override;
   bool maybeInsertAsanInitAtFunctionEntry(Function &F);
+  void markEscapedLocalAllocas(Function &F);
   bool doInitialization(Module &M) override;
   static char ID;  // Pass identification, replacement for typeid
 
@@ -439,14 +456,25 @@ struct AddressSanitizer : public FunctionPass {
   bool isSafeAccess(ObjectSizeOffsetVisitor &ObjSizeVis, Value *Addr,
                     uint64_t TypeSize) const;
 
+  /// Helper to cleanup per-function state.
+  struct FunctionStateRAII {
+    AddressSanitizer *Pass;
+    FunctionStateRAII(AddressSanitizer *Pass) : Pass(Pass) {
+      assert(Pass->ProcessedAllocas.empty() &&
+             "last pass forgot to clear cache");
+    }
+    ~FunctionStateRAII() { Pass->ProcessedAllocas.clear(); }
+  };
+
   LLVMContext *C;
   Triple TargetTriple;
   int LongSize;
+  bool CompileKernel;
   Type *IntptrTy;
   ShadowMapping Mapping;
   DominatorTree *DT;
-  Function *AsanCtorFunction;
-  Function *AsanInitFunction;
+  Function *AsanCtorFunction = nullptr;
+  Function *AsanInitFunction = nullptr;
   Function *AsanHandleNoReturnFunc;
   Function *AsanPtrCmpFunction, *AsanPtrSubFunction;
   // This array is indexed by AccessIsWrite, Experiment and log2(AccessSize).
@@ -465,7 +493,8 @@ struct AddressSanitizer : public FunctionPass {
 
 class AddressSanitizerModule : public ModulePass {
  public:
-  AddressSanitizerModule() : ModulePass(ID) {}
+  explicit AddressSanitizerModule(bool CompileKernel = false)
+      : ModulePass(ID), CompileKernel(CompileKernel || ClEnableKasan) {}
   bool runOnModule(Module &M) override;
   static char ID;  // Pass identification, replacement for typeid
   const char *getPassName() const override { return "AddressSanitizerModule"; }
@@ -482,6 +511,7 @@ class AddressSanitizerModule : public ModulePass {
   }
 
   GlobalsMetadata GlobalsMD;
+  bool CompileKernel;
   Type *IntptrTy;
   LLVMContext *C;
   Triple TargetTriple;
@@ -511,12 +541,14 @@ struct FunctionStackPoisoner : public InstVisitor<FunctionStackPoisoner> {
   ShadowMapping Mapping;
 
   SmallVector<AllocaInst *, 16> AllocaVec;
+  SmallSetVector<AllocaInst *, 16> NonInstrumentedStaticAllocaVec;
   SmallVector<Instruction *, 8> RetVec;
   unsigned StackAlignment;
 
   Function *AsanStackMallocFunc[kMaxAsanStackMallocSizeClass + 1],
       *AsanStackFreeFunc[kMaxAsanStackMallocSizeClass + 1];
   Function *AsanPoisonStackMemoryFunc, *AsanUnpoisonStackMemoryFunc;
+  Function *AsanAllocaPoisonFunc, *AsanAllocasUnpoisonFunc;
 
   // Stores a place and arguments of poisoning/unpoisoning call for alloca.
   struct AllocaPoisonCall {
@@ -527,29 +559,17 @@ struct FunctionStackPoisoner : public InstVisitor<FunctionStackPoisoner> {
   };
   SmallVector<AllocaPoisonCall, 8> AllocaPoisonCallVec;
 
-  // Stores left and right redzone shadow addresses for dynamic alloca
-  // and pointer to alloca instruction itself.
-  // LeftRzAddr is a shadow address for alloca left redzone.
-  // RightRzAddr is a shadow address for alloca right redzone.
-  struct DynamicAllocaCall {
-    AllocaInst *AI;
-    Value *LeftRzAddr;
-    Value *RightRzAddr;
-    bool Poison;
-    explicit DynamicAllocaCall(AllocaInst *AI, Value *LeftRzAddr = nullptr,
-                               Value *RightRzAddr = nullptr)
-        : AI(AI),
-          LeftRzAddr(LeftRzAddr),
-          RightRzAddr(RightRzAddr),
-          Poison(true) {}
-  };
-  SmallVector<DynamicAllocaCall, 1> DynamicAllocaVec;
+  SmallVector<AllocaInst *, 1> DynamicAllocaVec;
+  SmallVector<IntrinsicInst *, 1> StackRestoreVec;
+  AllocaInst *DynamicAllocaLayout = nullptr;
+  IntrinsicInst *LocalEscapeCall = nullptr;
 
   // Maps Value to an AllocaInst from which the Value is originated.
   typedef DenseMap<Value *, AllocaInst *> AllocaForValueMapTy;
   AllocaForValueMapTy AllocaForValue;
 
-  bool HasNonEmptyInlineAsm;
+  bool HasNonEmptyInlineAsm = false;
+  bool HasReturnsTwiceCall = false;
   std::unique_ptr<CallInst> EmptyInlineAsm;
 
   FunctionStackPoisoner(Function &F, AddressSanitizer &ASan)
@@ -561,7 +581,6 @@ struct FunctionStackPoisoner : public InstVisitor<FunctionStackPoisoner> {
         IntptrPtrTy(PointerType::get(IntptrTy, 0)),
         Mapping(ASan.Mapping),
         StackAlignment(1 << Mapping.Scale),
-        HasNonEmptyInlineAsm(false),
         EmptyInlineAsm(CallInst::Create(ASan.EmptyAsm)) {}
 
   bool runOnFunction() {
@@ -586,41 +605,29 @@ struct FunctionStackPoisoner : public InstVisitor<FunctionStackPoisoner> {
   // Then unpoison everything back before the function returns.
   void poisonStack();
 
+  void createDynamicAllocasInitStorage();
+
   // ----------------------- Visitors.
   /// \brief Collect all Ret instructions.
   void visitReturnInst(ReturnInst &RI) { RetVec.push_back(&RI); }
 
-  // Unpoison dynamic allocas redzones.
-  void unpoisonDynamicAlloca(DynamicAllocaCall &AllocaCall) {
-    if (!AllocaCall.Poison) return;
-    for (auto Ret : RetVec) {
-      IRBuilder<> IRBRet(Ret);
-      PointerType *Int32PtrTy = PointerType::getUnqual(IRBRet.getInt32Ty());
-      Value *Zero = Constant::getNullValue(IRBRet.getInt32Ty());
-      Value *PartialRzAddr = IRBRet.CreateSub(AllocaCall.RightRzAddr,
-                                              ConstantInt::get(IntptrTy, 4));
-      IRBRet.CreateStore(
-          Zero, IRBRet.CreateIntToPtr(AllocaCall.LeftRzAddr, Int32PtrTy));
-      IRBRet.CreateStore(Zero,
-                         IRBRet.CreateIntToPtr(PartialRzAddr, Int32PtrTy));
-      IRBRet.CreateStore(
-          Zero, IRBRet.CreateIntToPtr(AllocaCall.RightRzAddr, Int32PtrTy));
-    }
+  void unpoisonDynamicAllocasBeforeInst(Instruction *InstBefore,
+                                        Value *SavedStack) {
+    IRBuilder<> IRB(InstBefore);
+    IRB.CreateCall(AsanAllocasUnpoisonFunc,
+                   {IRB.CreateLoad(DynamicAllocaLayout),
+                    IRB.CreatePtrToInt(SavedStack, IntptrTy)});
   }
 
-  // Right shift for BigEndian and left shift for LittleEndian.
-  Value *shiftAllocaMagic(Value *Val, IRBuilder<> &IRB, Value *Shift) {
-    auto &DL = F.getParent()->getDataLayout();
-    return DL.isLittleEndian() ? IRB.CreateShl(Val, Shift)
-                               : IRB.CreateLShr(Val, Shift);
-  }
+  // Unpoison dynamic allocas redzones.
+  void unpoisonDynamicAllocas() {
+    for (auto &Ret : RetVec)
+      unpoisonDynamicAllocasBeforeInst(Ret, DynamicAllocaLayout);
 
-  // Compute PartialRzMagic for dynamic alloca call. Since we don't know the
-  // size of requested memory until runtime, we should compute it dynamically.
-  // If PartialSize is 0, PartialRzMagic would contain kAsanAllocaRightMagic,
-  // otherwise it would contain the value that we will use to poison the
-  // partial redzone for alloca call.
-  Value *computePartialRzMagic(Value *PartialSize, IRBuilder<> &IRB);
+    for (auto &StackRestoreInst : StackRestoreVec)
+      unpoisonDynamicAllocasBeforeInst(StackRestoreInst,
+                                       StackRestoreInst->getOperand(0));
+  }
 
   // Deploy and poison redzones around dynamic alloca call. To do this, we
   // should replace this call with another one with changed parameters and
@@ -632,20 +639,18 @@ struct FunctionStackPoisoner : public InstVisitor<FunctionStackPoisoner> {
   //   addr = tmp + 32 (first 32 bytes are for the left redzone).
   // Additional_size is added to make new memory allocation contain not only
   // requested memory, but also left, partial and right redzones.
-  // After that, we should poison redzones:
-  // (1) Left redzone with kAsanAllocaLeftMagic.
-  // (2) Partial redzone with the value, computed in runtime by
-  //     computePartialRzMagic function.
-  // (3) Right redzone with kAsanAllocaRightMagic.
-  void handleDynamicAllocaCall(DynamicAllocaCall &AllocaCall);
+  void handleDynamicAllocaCall(AllocaInst *AI);
 
   /// \brief Collect Alloca instructions we want (and can) handle.
   void visitAllocaInst(AllocaInst &AI) {
-    if (!ASan.isInterestingAlloca(AI)) return;
+    if (!ASan.isInterestingAlloca(AI)) {
+      if (AI.isStaticAlloca()) NonInstrumentedStaticAllocaVec.insert(&AI);
+      return;
+    }
 
     StackAlignment = std::max(StackAlignment, AI.getAlignment());
-    if (isDynamicAlloca(AI))
-      DynamicAllocaVec.push_back(DynamicAllocaCall(&AI));
+    if (ASan.isDynamicAlloca(AI))
+      DynamicAllocaVec.push_back(&AI);
     else
       AllocaVec.push_back(&AI);
   }
@@ -653,8 +658,10 @@ struct FunctionStackPoisoner : public InstVisitor<FunctionStackPoisoner> {
   /// \brief Collect lifetime intrinsic calls to check for use-after-scope
   /// errors.
   void visitIntrinsicInst(IntrinsicInst &II) {
-    if (!ClCheckLifetime) return;
     Intrinsic::ID ID = II.getIntrinsicID();
+    if (ID == Intrinsic::stackrestore) StackRestoreVec.push_back(&II);
+    if (ID == Intrinsic::localescape) LocalEscapeCall = &II;
+    if (!ClCheckLifetime) return;
     if (ID != Intrinsic::lifetime_start && ID != Intrinsic::lifetime_end)
       return;
     // Found lifetime intrinsic, add ASan instrumentation if necessary.
@@ -675,9 +682,13 @@ struct FunctionStackPoisoner : public InstVisitor<FunctionStackPoisoner> {
     AllocaPoisonCallVec.push_back(APC);
   }
 
-  void visitCallInst(CallInst &CI) {
-    HasNonEmptyInlineAsm |=
-        CI.isInlineAsm() && !CI.isIdenticalTo(EmptyInlineAsm.get());
+  void visitCallSite(CallSite CS) {
+    Instruction *I = CS.getInstruction();
+    if (CallInst *CI = dyn_cast<CallInst>(I)) {
+      HasNonEmptyInlineAsm |=
+          CI->isInlineAsm() && !CI->isIdenticalTo(EmptyInlineAsm.get());
+      HasReturnsTwiceCall |= CI->canReturnTwice();
+    }
   }
 
   // ---------------------- Helpers.
@@ -690,9 +701,6 @@ struct FunctionStackPoisoner : public InstVisitor<FunctionStackPoisoner> {
     return true;
   }
 
-  bool isDynamicAlloca(AllocaInst &AI) const {
-    return AI.isArrayAllocation() || !AI.isStaticAlloca();
-  }
   /// Finds alloca where the value comes from.
   AllocaInst *findAllocaForValue(Value *V);
   void poisonRedZones(ArrayRef<uint8_t> ShadowBytes, IRBuilder<> &IRB,
@@ -719,8 +727,8 @@ INITIALIZE_PASS_END(
     AddressSanitizer, "asan",
     "AddressSanitizer: detects use-after-free and out-of-bounds bugs.", false,
     false)
-FunctionPass *llvm::createAddressSanitizerFunctionPass() {
-  return new AddressSanitizer();
+FunctionPass *llvm::createAddressSanitizerFunctionPass(bool CompileKernel) {
+  return new AddressSanitizer(CompileKernel);
 }
 
 char AddressSanitizerModule::ID = 0;
@@ -729,8 +737,8 @@ INITIALIZE_PASS(
     "AddressSanitizer: detects use-after-free and out-of-bounds bugs."
     "ModulePass",
     false, false)
-ModulePass *llvm::createAddressSanitizerModulePass() {
-  return new AddressSanitizerModule();
+ModulePass *llvm::createAddressSanitizerModulePass(bool CompileKernel) {
+  return new AddressSanitizerModule(CompileKernel);
 }
 
 static size_t TypeSizeToSizeIndex(uint32_t TypeSize) {
@@ -789,17 +797,17 @@ Value *AddressSanitizer::memToShadow(Value *Shadow, IRBuilder<> &IRB) {
 void AddressSanitizer::instrumentMemIntrinsic(MemIntrinsic *MI) {
   IRBuilder<> IRB(MI);
   if (isa<MemTransferInst>(MI)) {
-    IRB.CreateCall3(
+    IRB.CreateCall(
         isa<MemMoveInst>(MI) ? AsanMemmove : AsanMemcpy,
-        IRB.CreatePointerCast(MI->getOperand(0), IRB.getInt8PtrTy()),
-        IRB.CreatePointerCast(MI->getOperand(1), IRB.getInt8PtrTy()),
-        IRB.CreateIntCast(MI->getOperand(2), IntptrTy, false));
+        {IRB.CreatePointerCast(MI->getOperand(0), IRB.getInt8PtrTy()),
+         IRB.CreatePointerCast(MI->getOperand(1), IRB.getInt8PtrTy()),
+         IRB.CreateIntCast(MI->getOperand(2), IntptrTy, false)});
   } else if (isa<MemSetInst>(MI)) {
-    IRB.CreateCall3(
+    IRB.CreateCall(
         AsanMemset,
-        IRB.CreatePointerCast(MI->getOperand(0), IRB.getInt8PtrTy()),
-        IRB.CreateIntCast(MI->getOperand(1), IRB.getInt32Ty(), false),
-        IRB.CreateIntCast(MI->getOperand(2), IntptrTy, false));
+        {IRB.CreatePointerCast(MI->getOperand(0), IRB.getInt8PtrTy()),
+         IRB.CreateIntCast(MI->getOperand(1), IRB.getInt32Ty(), false),
+         IRB.CreateIntCast(MI->getOperand(2), IntptrTy, false)});
   }
   MI->eraseFromParent();
 }
@@ -811,12 +819,14 @@ bool AddressSanitizer::isInterestingAlloca(AllocaInst &AI) {
   if (PreviouslySeenAllocaInfo != ProcessedAllocas.end())
     return PreviouslySeenAllocaInfo->getSecond();
 
-  bool IsInteresting = (AI.getAllocatedType()->isSized() &&
-    // alloca() may be called with 0 size, ignore it.
-    getAllocaSizeInBytes(&AI) > 0 &&
-    // We are only interested in allocas not promotable to registers.
-    // Promotable allocas are common under -O0.
-    (!ClSkipPromotableAllocas || !isAllocaPromotable(&AI)));
+  bool IsInteresting =
+      (AI.getAllocatedType()->isSized() &&
+       // alloca() may be called with 0 size, ignore it.
+       getAllocaSizeInBytes(&AI) > 0 &&
+       // We are only interested in allocas not promotable to registers.
+       // Promotable allocas are common under -O0.
+       (!ClSkipPromotableAllocas || !isAllocaPromotable(&AI) ||
+        isDynamicAlloca(AI)));
 
   ProcessedAllocas[&AI] = IsInteresting;
   return IsInteresting;
@@ -906,7 +916,7 @@ void AddressSanitizer::instrumentPointerComparisonOrSubtraction(
     if (Param[i]->getType()->isPointerTy())
       Param[i] = IRB.CreatePointerCast(Param[i], IntptrTy);
   }
-  IRB.CreateCall2(F, Param[0], Param[1]);
+  IRB.CreateCall(F, Param);
 }
 
 void AddressSanitizer::instrumentMop(ObjectSizeOffsetVisitor &ObjSizeVis,
@@ -978,24 +988,24 @@ Instruction *AddressSanitizer::generateCrashCode(Instruction *InsertBefore,
   CallInst *Call = nullptr;
   if (SizeArgument) {
     if (Exp == 0)
-      Call = IRB.CreateCall2(AsanErrorCallbackSized[IsWrite][0], Addr,
-                             SizeArgument);
+      Call = IRB.CreateCall(AsanErrorCallbackSized[IsWrite][0],
+                            {Addr, SizeArgument});
     else
-      Call = IRB.CreateCall3(AsanErrorCallbackSized[IsWrite][1], Addr,
-                             SizeArgument, ExpVal);
+      Call = IRB.CreateCall(AsanErrorCallbackSized[IsWrite][1],
+                            {Addr, SizeArgument, ExpVal});
   } else {
     if (Exp == 0)
       Call =
           IRB.CreateCall(AsanErrorCallback[IsWrite][0][AccessSizeIndex], Addr);
     else
-      Call = IRB.CreateCall2(AsanErrorCallback[IsWrite][1][AccessSizeIndex],
-                             Addr, ExpVal);
+      Call = IRB.CreateCall(AsanErrorCallback[IsWrite][1][AccessSizeIndex],
+                            {Addr, ExpVal});
   }
 
   // We don't do Call->setDoesNotReturn() because the BB already has
   // UnreachableInst at the end.
   // This EmptyAsm is required to avoid callback merge.
-  IRB.CreateCall(EmptyAsm);
+  IRB.CreateCall(EmptyAsm, {});
   return Call;
 }
 
@@ -1031,8 +1041,8 @@ void AddressSanitizer::instrumentAddress(Instruction *OrigIns,
       IRB.CreateCall(AsanMemoryAccessCallback[IsWrite][0][AccessSizeIndex],
                      AddrLong);
     else
-      IRB.CreateCall2(AsanMemoryAccessCallback[IsWrite][1][AccessSizeIndex],
-                      AddrLong, ConstantInt::get(IRB.getInt32Ty(), Exp));
+      IRB.CreateCall(AsanMemoryAccessCallback[IsWrite][1][AccessSizeIndex],
+                     {AddrLong, ConstantInt::get(IRB.getInt32Ty(), Exp)});
     return;
   }
 
@@ -1083,11 +1093,11 @@ void AddressSanitizer::instrumentUnusualSizeOrAlignment(
   Value *AddrLong = IRB.CreatePointerCast(Addr, IntptrTy);
   if (UseCalls) {
     if (Exp == 0)
-      IRB.CreateCall2(AsanMemoryAccessCallbackSized[IsWrite][0], AddrLong,
-                      Size);
+      IRB.CreateCall(AsanMemoryAccessCallbackSized[IsWrite][0],
+                     {AddrLong, Size});
     else
-      IRB.CreateCall3(AsanMemoryAccessCallbackSized[IsWrite][1], AddrLong, Size,
-                      ConstantInt::get(IRB.getInt32Ty(), Exp));
+      IRB.CreateCall(AsanMemoryAccessCallbackSized[IsWrite][1],
+                     {AddrLong, Size, ConstantInt::get(IRB.getInt32Ty(), Exp)});
   } else {
     Value *LastByte = IRB.CreateIntToPtr(
         IRB.CreateAdd(AddrLong, ConstantInt::get(IntptrTy, TypeSize / 8 - 1)),
@@ -1158,6 +1168,20 @@ bool AddressSanitizerModule::ShouldInstrumentGlobal(GlobalVariable *G) {
   if (G->hasSection()) {
     StringRef Section(G->getSection());
 
+    // Globals from llvm.metadata aren't emitted, do not instrument them.
+    if (Section == "llvm.metadata") return false;
+    // Do not instrument globals from special LLVM sections.
+    if (Section.find("__llvm") != StringRef::npos) return false;
+
+    // Callbacks put into the CRT initializer/terminator sections
+    // should not be instrumented.
+    // See https://code.google.com/p/address-sanitizer/issues/detail?id=305
+    // and http://msdn.microsoft.com/en-US/en-en/library/bb918180(v=vs.120).aspx
+    if (Section.startswith(".CRT")) {
+      DEBUG(dbgs() << "Ignoring a global initializer callback: " << *G << "\n");
+      return false;
+    }
+
     if (TargetTriple.isOSBinFormatMachO()) {
       StringRef ParsedSegment, ParsedSection;
       unsigned TAA = 0, StubSize = 0;
@@ -1165,8 +1189,8 @@ bool AddressSanitizerModule::ShouldInstrumentGlobal(GlobalVariable *G) {
       std::string ErrorCode = MCSectionMachO::ParseSectionSpecifier(
           Section, ParsedSegment, ParsedSection, TAA, TAAParsed, StubSize);
       if (!ErrorCode.empty()) {
-        report_fatal_error("Invalid section specifier '" + ParsedSection +
-                           "': " + ErrorCode + ".");
+        assert(false && "Invalid section specifier.");
+        return false;
       }
 
       // Ignore the globals from the __OBJC section. The ObjC runtime assumes
@@ -1196,18 +1220,6 @@ bool AddressSanitizerModule::ShouldInstrumentGlobal(GlobalVariable *G) {
         return false;
       }
     }
-
-    // Callbacks put into the CRT initializer/terminator sections
-    // should not be instrumented.
-    // See https://code.google.com/p/address-sanitizer/issues/detail?id=305
-    // and http://msdn.microsoft.com/en-US/en-en/library/bb918180(v=vs.120).aspx
-    if (Section.startswith(".CRT")) {
-      DEBUG(dbgs() << "Ignoring a global initializer callback: " << *G << "\n");
-      return false;
-    }
-
-    // Globals from llvm.metadata aren't emitted, do not instrument them.
-    if (Section == "llvm.metadata") return false;
   }
 
   return true;
@@ -1347,9 +1359,9 @@ bool AddressSanitizerModule::InstrumentGlobals(IRBuilder<> &IRB, Module &M) {
   // Create calls for poisoning before initializers run and unpoisoning after.
   if (HasDynamicallyInitializedGlobals)
     createInitializerPoisonCalls(M, ModuleName);
-  IRB.CreateCall2(AsanRegisterGlobals,
-                  IRB.CreatePointerCast(AllGlobals, IntptrTy),
-                  ConstantInt::get(IntptrTy, n));
+  IRB.CreateCall(AsanRegisterGlobals,
+                 {IRB.CreatePointerCast(AllGlobals, IntptrTy),
+                  ConstantInt::get(IntptrTy, n)});
 
   // We also need to unregister globals at the end, e.g. when a shared library
   // gets closed.
@@ -1358,9 +1370,9 @@ bool AddressSanitizerModule::InstrumentGlobals(IRBuilder<> &IRB, Module &M) {
                        GlobalValue::InternalLinkage, kAsanModuleDtorName, &M);
   BasicBlock *AsanDtorBB = BasicBlock::Create(*C, "", AsanDtorFunction);
   IRBuilder<> IRB_Dtor(ReturnInst::Create(*C, AsanDtorBB));
-  IRB_Dtor.CreateCall2(AsanUnregisterGlobals,
-                       IRB.CreatePointerCast(AllGlobals, IntptrTy),
-                       ConstantInt::get(IntptrTy, n));
+  IRB_Dtor.CreateCall(AsanUnregisterGlobals,
+                      {IRB.CreatePointerCast(AllGlobals, IntptrTy),
+                       ConstantInt::get(IntptrTy, n)});
   appendToGlobalDtors(M, AsanDtorFunction, kAsanCtorAndDtorPriority);
 
   DEBUG(dbgs() << M);
@@ -1372,16 +1384,18 @@ bool AddressSanitizerModule::runOnModule(Module &M) {
   int LongSize = M.getDataLayout().getPointerSizeInBits();
   IntptrTy = Type::getIntNTy(*C, LongSize);
   TargetTriple = Triple(M.getTargetTriple());
-  Mapping = getShadowMapping(TargetTriple, LongSize);
+  Mapping = getShadowMapping(TargetTriple, LongSize, CompileKernel);
   initializeCallbacks(M);
 
   bool Changed = false;
 
-  Function *CtorFunc = M.getFunction(kAsanModuleCtorName);
-  assert(CtorFunc);
-  IRBuilder<> IRB(CtorFunc->getEntryBlock().getTerminator());
-
-  if (ClGlobals) Changed |= InstrumentGlobals(IRB, M);
+  // TODO(glider): temporarily disabled globals instrumentation for KASan.
+  if (ClGlobals && !CompileKernel) {
+    Function *CtorFunc = M.getFunction(kAsanModuleCtorName);
+    assert(CtorFunc);
+    IRBuilder<> IRB(CtorFunc->getEntryBlock().getTerminator());
+    Changed |= InstrumentGlobals(IRB, M);
+  }
 
   return Changed;
 }
@@ -1394,38 +1408,44 @@ void AddressSanitizer::initializeCallbacks(Module &M) {
     for (size_t AccessIsWrite = 0; AccessIsWrite <= 1; AccessIsWrite++) {
       const std::string TypeStr = AccessIsWrite ? "store" : "load";
       const std::string ExpStr = Exp ? "exp_" : "";
+      const std::string SuffixStr = CompileKernel ? "N" : "_n";
+      const std::string EndingStr = CompileKernel ? "_noabort" : "";
       const Type *ExpType = Exp ? Type::getInt32Ty(*C) : nullptr;
+      // TODO(glider): for KASan builds add _noabort to error reporting
+      // functions and make them actually noabort (remove the UnreachableInst).
       AsanErrorCallbackSized[AccessIsWrite][Exp] =
           checkSanitizerInterfaceFunction(M.getOrInsertFunction(
-              kAsanReportErrorTemplate + ExpStr + TypeStr + "_n",
+              kAsanReportErrorTemplate + ExpStr + TypeStr + SuffixStr,
               IRB.getVoidTy(), IntptrTy, IntptrTy, ExpType, nullptr));
       AsanMemoryAccessCallbackSized[AccessIsWrite][Exp] =
           checkSanitizerInterfaceFunction(M.getOrInsertFunction(
-              ClMemoryAccessCallbackPrefix + ExpStr + TypeStr + "N",
+              ClMemoryAccessCallbackPrefix + ExpStr + TypeStr + "N" + EndingStr,
               IRB.getVoidTy(), IntptrTy, IntptrTy, ExpType, nullptr));
       for (size_t AccessSizeIndex = 0; AccessSizeIndex < kNumberOfAccessSizes;
            AccessSizeIndex++) {
         const std::string Suffix = TypeStr + itostr(1 << AccessSizeIndex);
         AsanErrorCallback[AccessIsWrite][Exp][AccessSizeIndex] =
             checkSanitizerInterfaceFunction(M.getOrInsertFunction(
-                kAsanReportErrorTemplate + ExpStr + Suffix, IRB.getVoidTy(),
-                IntptrTy, ExpType, nullptr));
+                kAsanReportErrorTemplate + ExpStr + Suffix,
+                IRB.getVoidTy(), IntptrTy, ExpType, nullptr));
         AsanMemoryAccessCallback[AccessIsWrite][Exp][AccessSizeIndex] =
             checkSanitizerInterfaceFunction(M.getOrInsertFunction(
-                ClMemoryAccessCallbackPrefix + ExpStr + Suffix, IRB.getVoidTy(),
-                IntptrTy, ExpType, nullptr));
+                ClMemoryAccessCallbackPrefix + ExpStr + Suffix + EndingStr,
+                IRB.getVoidTy(), IntptrTy, ExpType, nullptr));
       }
     }
   }
 
+  const std::string MemIntrinCallbackPrefix =
+      CompileKernel ? std::string("") : ClMemoryAccessCallbackPrefix;
   AsanMemmove = checkSanitizerInterfaceFunction(M.getOrInsertFunction(
-      ClMemoryAccessCallbackPrefix + "memmove", IRB.getInt8PtrTy(),
+      MemIntrinCallbackPrefix + "memmove", IRB.getInt8PtrTy(),
       IRB.getInt8PtrTy(), IRB.getInt8PtrTy(), IntptrTy, nullptr));
   AsanMemcpy = checkSanitizerInterfaceFunction(M.getOrInsertFunction(
-      ClMemoryAccessCallbackPrefix + "memcpy", IRB.getInt8PtrTy(),
+      MemIntrinCallbackPrefix + "memcpy", IRB.getInt8PtrTy(),
       IRB.getInt8PtrTy(), IRB.getInt8PtrTy(), IntptrTy, nullptr));
   AsanMemset = checkSanitizerInterfaceFunction(M.getOrInsertFunction(
-      ClMemoryAccessCallbackPrefix + "memset", IRB.getInt8PtrTy(),
+      MemIntrinCallbackPrefix + "memset", IRB.getInt8PtrTy(),
       IRB.getInt8PtrTy(), IRB.getInt32Ty(), IntptrTy, nullptr));
 
   AsanHandleNoReturnFunc = checkSanitizerInterfaceFunction(
@@ -1452,20 +1472,14 @@ bool AddressSanitizer::doInitialization(Module &M) {
   IntptrTy = Type::getIntNTy(*C, LongSize);
   TargetTriple = Triple(M.getTargetTriple());
 
-  AsanCtorFunction =
-      Function::Create(FunctionType::get(Type::getVoidTy(*C), false),
-                       GlobalValue::InternalLinkage, kAsanModuleCtorName, &M);
-  BasicBlock *AsanCtorBB = BasicBlock::Create(*C, "", AsanCtorFunction);
-  // call __asan_init in the module ctor.
-  IRBuilder<> IRB(ReturnInst::Create(*C, AsanCtorBB));
-  AsanInitFunction = checkSanitizerInterfaceFunction(
-      M.getOrInsertFunction(kAsanInitName, IRB.getVoidTy(), nullptr));
-  AsanInitFunction->setLinkage(Function::ExternalLinkage);
-  IRB.CreateCall(AsanInitFunction);
-
-  Mapping = getShadowMapping(TargetTriple, LongSize);
-
-  appendToGlobalCtors(M, AsanCtorFunction, kAsanCtorAndDtorPriority);
+  if (!CompileKernel) {
+    std::tie(AsanCtorFunction, AsanInitFunction) =
+        createSanitizerCtorAndInitFunctions(
+            M, kAsanModuleCtorName, kAsanInitName,
+            /*InitArgTypes=*/{}, /*InitArgs=*/{}, kAsanVersionCheckName);
+    appendToGlobalCtors(M, AsanCtorFunction, kAsanCtorAndDtorPriority);
+  }
+  Mapping = getShadowMapping(TargetTriple, LongSize, CompileKernel);
   return true;
 }
 
@@ -1479,12 +1493,40 @@ bool AddressSanitizer::maybeInsertAsanInitAtFunctionEntry(Function &F) {
   // instrumented functions.
   if (F.getName().find(" load]") != std::string::npos) {
     IRBuilder<> IRB(F.begin()->begin());
-    IRB.CreateCall(AsanInitFunction);
+    IRB.CreateCall(AsanInitFunction, {});
     return true;
   }
   return false;
 }
 
+void AddressSanitizer::markEscapedLocalAllocas(Function &F) {
+  // Find the one possible call to llvm.localescape and pre-mark allocas passed
+  // to it as uninteresting. This assumes we haven't started processing allocas
+  // yet. This check is done up front because iterating the use list in
+  // isInterestingAlloca would be algorithmically slower.
+  assert(ProcessedAllocas.empty() && "must process localescape before allocas");
+
+  // Try to get the declaration of llvm.localescape. If it's not in the module,
+  // we can exit early.
+  if (!F.getParent()->getFunction("llvm.localescape")) return;
+
+  // Look for a call to llvm.localescape call in the entry block. It can't be in
+  // any other block.
+  for (Instruction &I : F.getEntryBlock()) {
+    IntrinsicInst *II = dyn_cast<IntrinsicInst>(&I);
+    if (II && II->getIntrinsicID() == Intrinsic::localescape) {
+      // We found a call. Mark all the allocas passed in as uninteresting.
+      for (Value *Arg : II->arg_operands()) {
+        AllocaInst *AI = dyn_cast<AllocaInst>(Arg->stripPointerCasts());
+        assert(AI && AI->isStaticAlloca() &&
+               "non-static alloca arg to localescape");
+        ProcessedAllocas[AI] = false;
+      }
+      break;
+    }
+  }
+}
+
 bool AddressSanitizer::runOnFunction(Function &F) {
   if (&F == AsanCtorFunction) return false;
   if (F.getLinkage() == GlobalValue::AvailableExternallyLinkage) return false;
@@ -1500,6 +1542,12 @@ bool AddressSanitizer::runOnFunction(Function &F) {
 
   if (!ClDebugFunc.empty() && ClDebugFunc != F.getName()) return false;
 
+  FunctionStateRAII CleanupObj(this);
+
+  // We can't instrument allocas used with llvm.localescape. Only static allocas
+  // can be passed to that intrinsic.
+  markEscapedLocalAllocas(F);
+
   // We want to instrument every address only once per basic block (unless there
   // are calls between uses).
   SmallSet<Value *, 16> TempsToInstrument;
@@ -1547,11 +1595,10 @@ bool AddressSanitizer::runOnFunction(Function &F) {
     }
   }
 
-  bool UseCalls = false;
-  if (ClInstrumentationWithCallsThreshold >= 0 &&
-      ToInstrument.size() > (unsigned)ClInstrumentationWithCallsThreshold)
-    UseCalls = true;
-
+  bool UseCalls =
+      CompileKernel ||
+      (ClInstrumentationWithCallsThreshold >= 0 &&
+       ToInstrument.size() > (unsigned)ClInstrumentationWithCallsThreshold);
   const TargetLibraryInfo *TLI =
       &getAnalysis<TargetLibraryInfoWrapperPass>().getTLI();
   const DataLayout &DL = F.getParent()->getDataLayout();
@@ -1579,7 +1626,7 @@ bool AddressSanitizer::runOnFunction(Function &F) {
   // See e.g. http://code.google.com/p/address-sanitizer/issues/detail?id=37
   for (auto CI : NoReturnCalls) {
     IRBuilder<> IRB(CI);
-    IRB.CreateCall(AsanHandleNoReturnFunc);
+    IRB.CreateCall(AsanHandleNoReturnFunc, {});
   }
 
   for (auto Inst : PointerComparisonsOrSubtracts) {
@@ -1623,6 +1670,11 @@ void FunctionStackPoisoner::initializeCallbacks(Module &M) {
   AsanUnpoisonStackMemoryFunc = checkSanitizerInterfaceFunction(
       M.getOrInsertFunction(kAsanUnpoisonStackMemoryName, IRB.getVoidTy(),
                             IntptrTy, IntptrTy, nullptr));
+  AsanAllocaPoisonFunc = checkSanitizerInterfaceFunction(M.getOrInsertFunction(
+      kAsanAllocaPoison, IRB.getVoidTy(), IntptrTy, IntptrTy, nullptr));
+  AsanAllocasUnpoisonFunc =
+      checkSanitizerInterfaceFunction(M.getOrInsertFunction(
+          kAsanAllocasUnpoison, IRB.getVoidTy(), IntptrTy, IntptrTy, nullptr));
 }
 
 void FunctionStackPoisoner::poisonRedZones(ArrayRef<uint8_t> ShadowBytes,
@@ -1682,12 +1734,6 @@ void FunctionStackPoisoner::SetShadowToStackAfterReturnInlined(
   }
 }
 
-static DebugLoc getFunctionEntryDebugLocation(Function &F) {
-  for (const auto &Inst : F.getEntryBlock())
-    if (!isa<AllocaInst>(Inst)) return Inst.getDebugLoc();
-  return DebugLoc();
-}
-
 PHINode *FunctionStackPoisoner::createPHI(IRBuilder<> &IRB, Value *Cond,
                                           Value *ValueIfTrue,
                                           Instruction *ThenTerm,
@@ -1718,26 +1764,49 @@ Value *FunctionStackPoisoner::createAllocaForLayout(
   return IRB.CreatePointerCast(Alloca, IntptrTy);
 }
 
+void FunctionStackPoisoner::createDynamicAllocasInitStorage() {
+  BasicBlock &FirstBB = *F.begin();
+  IRBuilder<> IRB(dyn_cast<Instruction>(FirstBB.begin()));
+  DynamicAllocaLayout = IRB.CreateAlloca(IntptrTy, nullptr);
+  IRB.CreateStore(Constant::getNullValue(IntptrTy), DynamicAllocaLayout);
+  DynamicAllocaLayout->setAlignment(32);
+}
+
 void FunctionStackPoisoner::poisonStack() {
   assert(AllocaVec.size() > 0 || DynamicAllocaVec.size() > 0);
 
-  if (ClInstrumentAllocas) {
+  if (ClInstrumentAllocas && DynamicAllocaVec.size() > 0) {
     // Handle dynamic allocas.
-    for (auto &AllocaCall : DynamicAllocaVec) {
-      handleDynamicAllocaCall(AllocaCall);
-      unpoisonDynamicAlloca(AllocaCall);
-    }
+    createDynamicAllocasInitStorage();
+    for (auto &AI : DynamicAllocaVec) handleDynamicAllocaCall(AI);
+
+    unpoisonDynamicAllocas();
   }
 
   if (AllocaVec.size() == 0) return;
 
   int StackMallocIdx = -1;
-  DebugLoc EntryDebugLocation = getFunctionEntryDebugLocation(F);
+  DebugLoc EntryDebugLocation;
+  if (auto SP = getDISubprogram(&F))
+    EntryDebugLocation = DebugLoc::get(SP->getScopeLine(), 0, SP);
 
   Instruction *InsBefore = AllocaVec[0];
   IRBuilder<> IRB(InsBefore);
   IRB.SetCurrentDebugLocation(EntryDebugLocation);
 
+  // Make sure non-instrumented allocas stay in the entry block. Otherwise,
+  // debug info is broken, because only entry-block allocas are treated as
+  // regular stack slots.
+  auto InsBeforeB = InsBefore->getParent();
+  assert(InsBeforeB == &F.getEntryBlock());
+  for (BasicBlock::iterator I = InsBefore; I != InsBeforeB->end(); ++I)
+    if (auto *AI = dyn_cast_or_null<AllocaInst>(I))
+      if (NonInstrumentedStaticAllocaVec.count(AI) > 0)
+        AI->moveBefore(InsBefore);
+
+  // If we have a call to llvm.localescape, keep it in the entry block.
+  if (LocalEscapeCall) LocalEscapeCall->moveBefore(InsBefore);
+
   SmallVector<ASanStackVariableDescription, 16> SVD;
   SVD.reserve(AllocaVec.size());
   for (AllocaInst *AI : AllocaVec) {
@@ -1753,13 +1822,17 @@ void FunctionStackPoisoner::poisonStack() {
   ComputeASanStackFrameLayout(SVD, 1UL << Mapping.Scale, MinHeaderSize, &L);
   DEBUG(dbgs() << L.DescriptionString << " --- " << L.FrameSize << "\n");
   uint64_t LocalStackSize = L.FrameSize;
-  bool DoStackMalloc =
-      ClUseAfterReturn && LocalStackSize <= kMaxStackMallocSize;
-  // Don't do dynamic alloca in presence of inline asm: too often it makes
-  // assumptions on which registers are available. Don't do stack malloc in the
-  // presence of inline asm on 32-bit platforms for the same reason.
-  bool DoDynamicAlloca = ClDynamicAllocaStack && !HasNonEmptyInlineAsm;
-  DoStackMalloc &= !HasNonEmptyInlineAsm || ASan.LongSize != 32;
+  bool DoStackMalloc = ClUseAfterReturn && !ASan.CompileKernel &&
+                       LocalStackSize <= kMaxStackMallocSize;
+  bool DoDynamicAlloca = ClDynamicAllocaStack;
+  // Don't do dynamic alloca or stack malloc if:
+  // 1) There is inline asm: too often it makes assumptions on which registers
+  //    are available.
+  // 2) There is a returns_twice call (typically setjmp), which is
+  //    optimization-hostile, and doesn't play well with introduced indirect
+  //    register-relative calculation of local variable addresses.
+  DoDynamicAlloca &= !HasNonEmptyInlineAsm && !HasReturnsTwiceCall;
+  DoStackMalloc &= !HasNonEmptyInlineAsm && !HasReturnsTwiceCall;
 
   Value *StaticAlloca =
       DoDynamicAlloca ? nullptr : createAllocaForLayout(IRB, L, false);
@@ -1893,8 +1966,9 @@ void FunctionStackPoisoner::poisonStack() {
             IRBPoison.CreateIntToPtr(SavedFlagPtr, IRBPoison.getInt8PtrTy()));
       } else {
         // For larger frames call __asan_stack_free_*.
-        IRBPoison.CreateCall2(AsanStackFreeFunc[StackMallocIdx], FakeStack,
-                              ConstantInt::get(IntptrTy, LocalStackSize));
+        IRBPoison.CreateCall(
+            AsanStackFreeFunc[StackMallocIdx],
+            {FakeStack, ConstantInt::get(IntptrTy, LocalStackSize)});
       }
 
       IRBuilder<> IRBElse(ElseTerm);
@@ -1917,9 +1991,9 @@ void FunctionStackPoisoner::poisonAlloca(Value *V, uint64_t Size,
   // For now just insert the call to ASan runtime.
   Value *AddrArg = IRB.CreatePointerCast(V, IntptrTy);
   Value *SizeArg = ConstantInt::get(IntptrTy, Size);
-  IRB.CreateCall2(
+  IRB.CreateCall(
       DoPoison ? AsanPoisonStackMemoryFunc : AsanUnpoisonStackMemoryFunc,
-      AddrArg, SizeArg);
+      {AddrArg, SizeArg});
 }
 
 // Handling llvm.lifetime intrinsics for a given %alloca:
@@ -1946,8 +2020,7 @@ AllocaInst *FunctionStackPoisoner::findAllocaForValue(Value *V) {
   if (CastInst *CI = dyn_cast<CastInst>(V))
     Res = findAllocaForValue(CI->getOperand(0));
   else if (PHINode *PN = dyn_cast<PHINode>(V)) {
-    for (unsigned i = 0, e = PN->getNumIncomingValues(); i != e; ++i) {
-      Value *IncValue = PN->getIncomingValue(i);
+    for (Value *IncValue : PN->incoming_values()) {
       // Allow self-referencing phi-nodes.
       if (IncValue == PN) continue;
       AllocaInst *IncValueAI = findAllocaForValue(IncValue);
@@ -1961,78 +2034,25 @@ AllocaInst *FunctionStackPoisoner::findAllocaForValue(Value *V) {
   return Res;
 }
 
-// Compute PartialRzMagic for dynamic alloca call. PartialRzMagic is
-// constructed from two separate 32-bit numbers: PartialRzMagic = Val1 | Val2.
-// (1) Val1 is resposible for forming base value for PartialRzMagic, containing
-//     only 00 for fully addressable and 0xcb for fully poisoned bytes for each
-//     8-byte chunk of user memory respectively.
-// (2) Val2 forms the value for marking first poisoned byte in shadow memory
-//     with appropriate value (0x01 - 0x07 or 0xcb if Padding % 8 == 0).
-
-// Shift = Padding & ~7; // the number of bits we need to shift to access first
-//                          chunk in shadow memory, containing nonzero bytes.
-// Example:
-// Padding = 21                       Padding = 16
-// Shadow:  |00|00|05|cb|          Shadow:  |00|00|cb|cb|
-//                ^                               ^
-//                |                               |
-// Shift = 21 & ~7 = 16            Shift = 16 & ~7 = 16
-//
-// Val1 = 0xcbcbcbcb << Shift;
-// PartialBits = Padding ? Padding & 7 : 0xcb;
-// Val2 = PartialBits << Shift;
-// Result = Val1 | Val2;
-Value *FunctionStackPoisoner::computePartialRzMagic(Value *PartialSize,
-                                                    IRBuilder<> &IRB) {
-  PartialSize = IRB.CreateIntCast(PartialSize, IRB.getInt32Ty(), false);
-  Value *Shift = IRB.CreateAnd(PartialSize, IRB.getInt32(~7));
-  unsigned Val1Int = kAsanAllocaPartialVal1;
-  unsigned Val2Int = kAsanAllocaPartialVal2;
-  if (!F.getParent()->getDataLayout().isLittleEndian()) {
-    Val1Int = sys::getSwappedBytes(Val1Int);
-    Val2Int = sys::getSwappedBytes(Val2Int);
-  }
-  Value *Val1 = shiftAllocaMagic(IRB.getInt32(Val1Int), IRB, Shift);
-  Value *PartialBits = IRB.CreateAnd(PartialSize, IRB.getInt32(7));
-  // For BigEndian get 0x000000YZ -> 0xYZ000000.
-  if (F.getParent()->getDataLayout().isBigEndian())
-    PartialBits = IRB.CreateShl(PartialBits, IRB.getInt32(24));
-  Value *Val2 = IRB.getInt32(Val2Int);
-  Value *Cond =
-      IRB.CreateICmpNE(PartialBits, Constant::getNullValue(IRB.getInt32Ty()));
-  Val2 = IRB.CreateSelect(Cond, shiftAllocaMagic(PartialBits, IRB, Shift),
-                          shiftAllocaMagic(Val2, IRB, Shift));
-  return IRB.CreateOr(Val1, Val2);
-}
-
-void FunctionStackPoisoner::handleDynamicAllocaCall(
-    DynamicAllocaCall &AllocaCall) {
-  AllocaInst *AI = AllocaCall.AI;
-  if (!doesDominateAllExits(AI)) {
-    // We do not yet handle complex allocas
-    AllocaCall.Poison = false;
-    return;
-  }
-
+void FunctionStackPoisoner::handleDynamicAllocaCall(AllocaInst *AI) {
   IRBuilder<> IRB(AI);
 
-  PointerType *Int32PtrTy = PointerType::getUnqual(IRB.getInt32Ty());
   const unsigned Align = std::max(kAllocaRzSize, AI->getAlignment());
   const uint64_t AllocaRedzoneMask = kAllocaRzSize - 1;
 
   Value *Zero = Constant::getNullValue(IntptrTy);
   Value *AllocaRzSize = ConstantInt::get(IntptrTy, kAllocaRzSize);
   Value *AllocaRzMask = ConstantInt::get(IntptrTy, AllocaRedzoneMask);
-  Value *NotAllocaRzMask = ConstantInt::get(IntptrTy, ~AllocaRedzoneMask);
 
   // Since we need to extend alloca with additional memory to locate
   // redzones, and OldSize is number of allocated blocks with
   // ElementSize size, get allocated memory size in bytes by
   // OldSize * ElementSize.
-  unsigned ElementSize =
+  const unsigned ElementSize =
       F.getParent()->getDataLayout().getTypeAllocSize(AI->getAllocatedType());
-  Value *OldSize = IRB.CreateMul(AI->getArraySize(),
-                                 ConstantInt::get(IntptrTy, ElementSize));
+  Value *OldSize =
+      IRB.CreateMul(IRB.CreateIntCast(AI->getArraySize(), IntptrTy, false),
+                    ConstantInt::get(IntptrTy, ElementSize));
 
   // PartialSize = OldSize % 32
   Value *PartialSize = IRB.CreateAnd(OldSize, AllocaRzMask);
@@ -2060,43 +2080,20 @@ void FunctionStackPoisoner::handleDynamicAllocaCall(
   Value *NewAddress = IRB.CreateAdd(IRB.CreatePtrToInt(NewAlloca, IntptrTy),
                                     ConstantInt::get(IntptrTy, Align));
 
-  Value *NewAddressPtr = IRB.CreateIntToPtr(NewAddress, AI->getType());
-
-  // LeftRzAddress = NewAddress - kAllocaRzSize
-  Value *LeftRzAddress = IRB.CreateSub(NewAddress, AllocaRzSize);
-
-  // Poisoning left redzone.
-  AllocaCall.LeftRzAddr = ASan.memToShadow(LeftRzAddress, IRB);
-  IRB.CreateStore(ConstantInt::get(IRB.getInt32Ty(), kAsanAllocaLeftMagic),
-                  IRB.CreateIntToPtr(AllocaCall.LeftRzAddr, Int32PtrTy));
-
-  // PartialRzAligned = PartialRzAddr & ~AllocaRzMask
-  Value *PartialRzAddr = IRB.CreateAdd(NewAddress, OldSize);
-  Value *PartialRzAligned = IRB.CreateAnd(PartialRzAddr, NotAllocaRzMask);
+  // Insert __asan_alloca_poison call for new created alloca.
+  IRB.CreateCall(AsanAllocaPoisonFunc, {NewAddress, OldSize});
 
-  // Poisoning partial redzone.
-  Value *PartialRzMagic = computePartialRzMagic(PartialSize, IRB);
-  Value *PartialRzShadowAddr = ASan.memToShadow(PartialRzAligned, IRB);
-  IRB.CreateStore(PartialRzMagic,
-                  IRB.CreateIntToPtr(PartialRzShadowAddr, Int32PtrTy));
+  // Store the last alloca's address to DynamicAllocaLayout. We'll need this
+  // for unpoisoning stuff.
+  IRB.CreateStore(IRB.CreatePtrToInt(NewAlloca, IntptrTy), DynamicAllocaLayout);
 
-  // RightRzAddress
-  //   =  (PartialRzAddr + AllocaRzMask) & ~AllocaRzMask
-  Value *RightRzAddress = IRB.CreateAnd(
-      IRB.CreateAdd(PartialRzAddr, AllocaRzMask), NotAllocaRzMask);
-
-  // Poisoning right redzone.
-  AllocaCall.RightRzAddr = ASan.memToShadow(RightRzAddress, IRB);
-  IRB.CreateStore(ConstantInt::get(IRB.getInt32Ty(), kAsanAllocaRightMagic),
-                  IRB.CreateIntToPtr(AllocaCall.RightRzAddr, Int32PtrTy));
+  Value *NewAddressPtr = IRB.CreateIntToPtr(NewAddress, AI->getType());
 
-  // Replace all uses of AddessReturnedByAlloca with NewAddress.
+  // Replace all uses of AddessReturnedByAlloca with NewAddressPtr.
   AI->replaceAllUsesWith(NewAddressPtr);
 
-  // We are done. Erase old alloca and store left, partial and right redzones
-  // shadow addresses for future unpoisoning.
+  // We are done. Erase old alloca from parent.
   AI->eraseFromParent();
-  NumInstrumentedDynamicAllocas++;
 }
 
 // isSafeAccess returns true if Addr is always inbounds with respect to its
diff --git a/lib/Transforms/Instrumentation/BoundsChecking.cpp b/lib/Transforms/Instrumentation/BoundsChecking.cpp
index 8113834ec0e..f6858034d79 100644
--- a/lib/Transforms/Instrumentation/BoundsChecking.cpp
+++ b/lib/Transforms/Instrumentation/BoundsChecking.cpp
@@ -82,7 +82,7 @@ BasicBlock *BoundsChecking::getTrapBB() {
   Builder->SetInsertPoint(TrapBB);
 
   llvm::Value *F = Intrinsic::getDeclaration(Fn->getParent(), Intrinsic::trap);
-  CallInst *TrapCall = Builder->CreateCall(F);
+  CallInst *TrapCall = Builder->CreateCall(F, {});
   TrapCall->setDoesNotReturn();
   TrapCall->setDoesNotThrow();
   TrapCall->setDebugLoc(Inst->getDebugLoc());
diff --git a/lib/Transforms/Instrumentation/CMakeLists.txt b/lib/Transforms/Instrumentation/CMakeLists.txt
index b2ff03343eb..9b81f4bb161 100644
--- a/lib/Transforms/Instrumentation/CMakeLists.txt
+++ b/lib/Transforms/Instrumentation/CMakeLists.txt
@@ -6,6 +6,7 @@ add_llvm_library(LLVMInstrumentation
   MemorySanitizer.cpp
   Instrumentation.cpp
   InstrProfiling.cpp
+  SafeStack.cpp
   SanitizerCoverage.cpp
   ThreadSanitizer.cpp
 
diff --git a/lib/Transforms/Instrumentation/DataFlowSanitizer.cpp b/lib/Transforms/Instrumentation/DataFlowSanitizer.cpp
index 06d5aed0c5f..8344e087a95 100644
--- a/lib/Transforms/Instrumentation/DataFlowSanitizer.cpp
+++ b/lib/Transforms/Instrumentation/DataFlowSanitizer.cpp
@@ -248,7 +248,7 @@ class DataFlowSanitizer : public ModulePass {
   DFSanABIList ABIList;
   DenseMap<Value *, Function *> UnwrappedFnMap;
   AttributeSet ReadOnlyNoneAttrs;
-  DenseMap<const Function *, DISubprogram> FunctionDIs;
+  DenseMap<const Function *, DISubprogram *> FunctionDIs;
 
   Value *getShadowAddress(Value *Addr, Instruction *Pos);
   bool isInstrumented(const Function *F);
@@ -420,6 +420,8 @@ bool DataFlowSanitizer::doInitialization(Module &M) {
   bool IsX86_64 = TargetTriple.getArch() == llvm::Triple::x86_64;
   bool IsMIPS64 = TargetTriple.getArch() == llvm::Triple::mips64 ||
                   TargetTriple.getArch() == llvm::Triple::mips64el;
+  bool IsAArch64 = TargetTriple.getArch() == llvm::Triple::aarch64 ||
+                   TargetTriple.getArch() == llvm::Triple::aarch64_be;
 
   const DataLayout &DL = M.getDataLayout();
 
@@ -434,6 +436,8 @@ bool DataFlowSanitizer::doInitialization(Module &M) {
     ShadowPtrMask = ConstantInt::getSigned(IntptrTy, ~0x700000000000LL);
   else if (IsMIPS64)
     ShadowPtrMask = ConstantInt::getSigned(IntptrTy, ~0xF000000000LL);
+  else if (IsAArch64)
+    ShadowPtrMask = ConstantInt::getSigned(IntptrTy, ~0x7800000000LL);
   else
     report_fatal_error("unsupported triple");
 
@@ -526,9 +530,9 @@ DataFlowSanitizer::buildWrapperFunction(Function *F, StringRef NewFName,
                                     F->getParent());
   NewF->copyAttributesFrom(F);
   NewF->removeAttributes(
-      AttributeSet::ReturnIndex,
-      AttributeFuncs::typeIncompatible(NewFT->getReturnType(),
-                                       AttributeSet::ReturnIndex));
+    AttributeSet::ReturnIndex,
+    AttributeSet::get(F->getContext(), AttributeSet::ReturnIndex,
+                    AttributeFuncs::typeIncompatible(NewFT->getReturnType())));
 
   BasicBlock *BB = BasicBlock::Create(*Ctx, "entry", NewF);
   if (F->isVarArg()) {
@@ -703,9 +707,9 @@ bool DataFlowSanitizer::runOnModule(Module &M) {
         Function *NewF = Function::Create(NewFT, F.getLinkage(), "", &M);
         NewF->copyAttributesFrom(&F);
         NewF->removeAttributes(
-            AttributeSet::ReturnIndex,
-            AttributeFuncs::typeIncompatible(NewFT->getReturnType(),
-                                             AttributeSet::ReturnIndex));
+          AttributeSet::ReturnIndex,
+          AttributeSet::get(NewF->getContext(), AttributeSet::ReturnIndex,
+                    AttributeFuncs::typeIncompatible(NewFT->getReturnType())));
         for (Function::arg_iterator FArg = F.arg_begin(),
                                     NewFArg = NewF->arg_begin(),
                                     FArgEnd = F.arg_end();
@@ -850,7 +854,7 @@ bool DataFlowSanitizer::runOnModule(Module &M) {
         BranchInst *BI = cast<BranchInst>(SplitBlockAndInsertIfThen(
             Ne, Pos, /*Unreachable=*/false, ColdCallWeights));
         IRBuilder<> ThenIRB(BI);
-        ThenIRB.CreateCall(DFSF.DFS.DFSanNonzeroLabelFn);
+        ThenIRB.CreateCall(DFSF.DFS.DFSanNonzeroLabelFn, {});
       }
     }
   }
@@ -865,7 +869,7 @@ Value *DFSanFunction::getArgTLSPtr() {
     return ArgTLSPtr = DFS.ArgTLS;
 
   IRBuilder<> IRB(F->getEntryBlock().begin());
-  return ArgTLSPtr = IRB.CreateCall(DFS.GetArgTLS);
+  return ArgTLSPtr = IRB.CreateCall(DFS.GetArgTLS, {});
 }
 
 Value *DFSanFunction::getRetvalTLS() {
@@ -875,7 +879,7 @@ Value *DFSanFunction::getRetvalTLS() {
     return RetvalTLSPtr = DFS.RetvalTLS;
 
   IRBuilder<> IRB(F->getEntryBlock().begin());
-  return RetvalTLSPtr = IRB.CreateCall(DFS.GetRetvalTLS);
+  return RetvalTLSPtr = IRB.CreateCall(DFS.GetRetvalTLS, {});
 }
 
 Value *DFSanFunction::getArgTLS(unsigned Idx, Instruction *Pos) {
@@ -972,7 +976,7 @@ Value *DFSanFunction::combineShadows(Value *V1, Value *V2, Instruction *Pos) {
 
   IRBuilder<> IRB(Pos);
   if (AvoidNewBlocks) {
-    CallInst *Call = IRB.CreateCall2(DFS.DFSanCheckedUnionFn, V1, V2);
+    CallInst *Call = IRB.CreateCall(DFS.DFSanCheckedUnionFn, {V1, V2});
     Call->addAttribute(AttributeSet::ReturnIndex, Attribute::ZExt);
     Call->addAttribute(1, Attribute::ZExt);
     Call->addAttribute(2, Attribute::ZExt);
@@ -985,7 +989,7 @@ Value *DFSanFunction::combineShadows(Value *V1, Value *V2, Instruction *Pos) {
     BranchInst *BI = cast<BranchInst>(SplitBlockAndInsertIfThen(
         Ne, Pos, /*Unreachable=*/false, DFS.ColdCallWeights, &DT));
     IRBuilder<> ThenIRB(BI);
-    CallInst *Call = ThenIRB.CreateCall2(DFS.DFSanUnionFn, V1, V2);
+    CallInst *Call = ThenIRB.CreateCall(DFS.DFSanUnionFn, {V1, V2});
     Call->addAttribute(AttributeSet::ReturnIndex, Attribute::ZExt);
     Call->addAttribute(1, Attribute::ZExt);
     Call->addAttribute(2, Attribute::ZExt);
@@ -1087,8 +1091,9 @@ Value *DFSanFunction::loadShadow(Value *Addr, uint64_t Size, uint64_t Align,
     // shadow is non-equal.
     BasicBlock *FallbackBB = BasicBlock::Create(*DFS.Ctx, "", F);
     IRBuilder<> FallbackIRB(FallbackBB);
-    CallInst *FallbackCall = FallbackIRB.CreateCall2(
-        DFS.DFSanUnionLoadFn, ShadowAddr, ConstantInt::get(DFS.IntptrTy, Size));
+    CallInst *FallbackCall = FallbackIRB.CreateCall(
+        DFS.DFSanUnionLoadFn,
+        {ShadowAddr, ConstantInt::get(DFS.IntptrTy, Size)});
     FallbackCall->addAttribute(AttributeSet::ReturnIndex, Attribute::ZExt);
 
     // Compare each of the shadows stored in the loaded 64 bits to each other,
@@ -1144,8 +1149,8 @@ Value *DFSanFunction::loadShadow(Value *Addr, uint64_t Size, uint64_t Align,
   }
 
   IRBuilder<> IRB(Pos);
-  CallInst *FallbackCall = IRB.CreateCall2(
-      DFS.DFSanUnionLoadFn, ShadowAddr, ConstantInt::get(DFS.IntptrTy, Size));
+  CallInst *FallbackCall = IRB.CreateCall(
+      DFS.DFSanUnionLoadFn, {ShadowAddr, ConstantInt::get(DFS.IntptrTy, Size)});
   FallbackCall->addAttribute(AttributeSet::ReturnIndex, Attribute::ZExt);
   return FallbackCall;
 }
@@ -1332,10 +1337,10 @@ void DFSanVisitor::visitSelectInst(SelectInst &I) {
 void DFSanVisitor::visitMemSetInst(MemSetInst &I) {
   IRBuilder<> IRB(&I);
   Value *ValShadow = DFSF.getShadow(I.getValue());
-  IRB.CreateCall3(
-      DFSF.DFS.DFSanSetLabelFn, ValShadow,
-      IRB.CreateBitCast(I.getDest(), Type::getInt8PtrTy(*DFSF.DFS.Ctx)),
-      IRB.CreateZExtOrTrunc(I.getLength(), DFSF.DFS.IntptrTy));
+  IRB.CreateCall(DFSF.DFS.DFSanSetLabelFn,
+                 {ValShadow, IRB.CreateBitCast(I.getDest(), Type::getInt8PtrTy(
+                                                                *DFSF.DFS.Ctx)),
+                  IRB.CreateZExtOrTrunc(I.getLength(), DFSF.DFS.IntptrTy)});
 }
 
 void DFSanVisitor::visitMemTransferInst(MemTransferInst &I) {
@@ -1357,8 +1362,8 @@ void DFSanVisitor::visitMemTransferInst(MemTransferInst &I) {
   Type *Int8Ptr = Type::getInt8PtrTy(*DFSF.DFS.Ctx);
   DestShadow = IRB.CreateBitCast(DestShadow, Int8Ptr);
   SrcShadow = IRB.CreateBitCast(SrcShadow, Int8Ptr);
-  IRB.CreateCall5(I.getCalledValue(), DestShadow, SrcShadow, LenShadow,
-                  AlignShadow, I.getVolatileCst());
+  IRB.CreateCall(I.getCalledValue(), {DestShadow, SrcShadow, LenShadow,
+                                      AlignShadow, I.getVolatileCst()});
 }
 
 void DFSanVisitor::visitReturnInst(ReturnInst &RI) {
@@ -1587,8 +1592,7 @@ void DFSanVisitor::visitCallSite(CallSite CS) {
     NewCS.setCallingConv(CS.getCallingConv());
     NewCS.setAttributes(CS.getAttributes().removeAttributes(
         *DFSF.DFS.Ctx, AttributeSet::ReturnIndex,
-        AttributeFuncs::typeIncompatible(NewCS.getInstruction()->getType(),
-                                         AttributeSet::ReturnIndex)));
+        AttributeFuncs::typeIncompatible(NewCS.getInstruction()->getType())));
 
     if (Next) {
       ExtractValueInst *ExVal =
diff --git a/lib/Transforms/Instrumentation/GCOVProfiling.cpp b/lib/Transforms/Instrumentation/GCOVProfiling.cpp
index 368a81d7aba..9a3ed5c04ef 100644
--- a/lib/Transforms/Instrumentation/GCOVProfiling.cpp
+++ b/lib/Transforms/Instrumentation/GCOVProfiling.cpp
@@ -126,7 +126,7 @@ namespace {
     Function *insertFlush(ArrayRef<std::pair<GlobalVariable*, MDNode*> >);
     void insertIndirectCounterIncrement();
 
-    std::string mangleName(DICompileUnit CU, const char *NewStem);
+    std::string mangleName(const DICompileUnit *CU, const char *NewStem);
 
     GCOVOptions Options;
 
@@ -149,7 +149,7 @@ ModulePass *llvm::createGCOVProfilerPass(const GCOVOptions &Options) {
   return new GCOVProfiler(Options);
 }
 
-static StringRef getFunctionName(MDSubprogram *SP) {
+static StringRef getFunctionName(const DISubprogram *SP) {
   if (!SP->getLinkageName().empty())
     return SP->getLinkageName();
   return SP->getName();
@@ -309,7 +309,7 @@ namespace {
   // object users can construct, the blocks and lines will be rooted here.
   class GCOVFunction : public GCOVRecord {
    public:
-     GCOVFunction(DISubprogram SP, raw_ostream *os, uint32_t Ident,
+     GCOVFunction(const DISubprogram *SP, raw_ostream *os, uint32_t Ident,
                   bool UseCfgChecksum, bool ExitBlockBeforeBody)
          : SP(SP), Ident(Ident), UseCfgChecksum(UseCfgChecksum), CfgChecksum(0),
            ReturnBlock(1, os) {
@@ -411,7 +411,7 @@ namespace {
     }
 
    private:
-    DISubprogram SP;
+     const DISubprogram *SP;
     uint32_t Ident;
     uint32_t FuncChecksum;
     bool UseCfgChecksum;
@@ -421,7 +421,8 @@ namespace {
   };
 }
 
-std::string GCOVProfiler::mangleName(DICompileUnit CU, const char *NewStem) {
+std::string GCOVProfiler::mangleName(const DICompileUnit *CU,
+                                     const char *NewStem) {
   if (NamedMDNode *GCov = M->getNamedMetadata("llvm.gcov")) {
     for (int i = 0, e = GCov->getNumOperands(); i != e; ++i) {
       MDNode *N = GCov->getOperand(i);
@@ -487,7 +488,7 @@ void GCOVProfiler::emitProfileNotes() {
     // this pass over the original .o's as they're produced, or run it after
     // LTO, we'll generate the same .gcno files.
 
-    DICompileUnit CU = cast<MDCompileUnit>(CU_Nodes->getOperand(i));
+    auto *CU = cast<DICompileUnit>(CU_Nodes->getOperand(i));
     std::error_code EC;
     raw_fd_ostream out(mangleName(CU, "gcno"), EC, sys::fs::F_None);
     std::string EdgeDestinations;
@@ -570,7 +571,7 @@ bool GCOVProfiler::emitProfileArcs() {
   bool Result = false;
   bool InsertIndCounterIncrCode = false;
   for (unsigned i = 0, e = CU_Nodes->getNumOperands(); i != e; ++i) {
-    DICompileUnit CU = cast<MDCompileUnit>(CU_Nodes->getOperand(i));
+    auto *CU = cast<DICompileUnit>(CU_Nodes->getOperand(i));
     SmallVector<std::pair<GlobalVariable *, MDNode *>, 8> CountersBySP;
     for (auto *SP : CU->getSubprograms()) {
       Function *F = SP->getFunction();
@@ -653,8 +654,8 @@ bool GCOVProfiler::emitProfileArcs() {
 
           // Build code to increment the counter.
           InsertIndCounterIncrCode = true;
-          Builder.CreateCall2(getIncrementIndirectCounterFunc(),
-                              EdgeState, CounterPtrArray);
+          Builder.CreateCall(getIncrementIndirectCounterFunc(),
+                             {EdgeState, CounterPtrArray});
         }
       }
     }
@@ -687,7 +688,7 @@ bool GCOVProfiler::emitProfileArcs() {
     // Initialize the environment and register the local writeout and flush
     // functions.
     Constant *GCOVInit = M->getOrInsertFunction("llvm_gcov_init", FTy);
-    Builder.CreateCall2(GCOVInit, WriteoutF, FlushF);
+    Builder.CreateCall(GCOVInit, {WriteoutF, FlushF});
     Builder.CreateRetVoid();
 
     appendToGlobalCtors(*M, F, 0);
@@ -846,34 +847,34 @@ Function *GCOVProfiler::insertCounterWriteout(
   NamedMDNode *CU_Nodes = M->getNamedMetadata("llvm.dbg.cu");
   if (CU_Nodes) {
     for (unsigned i = 0, e = CU_Nodes->getNumOperands(); i != e; ++i) {
-      DICompileUnit CU = cast<MDCompileUnit>(CU_Nodes->getOperand(i));
+      auto *CU = cast<DICompileUnit>(CU_Nodes->getOperand(i));
       std::string FilenameGcda = mangleName(CU, "gcda");
       uint32_t CfgChecksum = FileChecksums.empty() ? 0 : FileChecksums[i];
-      Builder.CreateCall3(StartFile,
-                          Builder.CreateGlobalStringPtr(FilenameGcda),
+      Builder.CreateCall(StartFile,
+                         {Builder.CreateGlobalStringPtr(FilenameGcda),
                           Builder.CreateGlobalStringPtr(ReversedVersion),
-                          Builder.getInt32(CfgChecksum));
+                          Builder.getInt32(CfgChecksum)});
       for (unsigned j = 0, e = CountersBySP.size(); j != e; ++j) {
-        auto *SP = cast_or_null<MDSubprogram>(CountersBySP[j].second);
+        auto *SP = cast_or_null<DISubprogram>(CountersBySP[j].second);
         uint32_t FuncChecksum = Funcs.empty() ? 0 : Funcs[j]->getFuncChecksum();
-        Builder.CreateCall5(
-            EmitFunction, Builder.getInt32(j),
-            Options.FunctionNamesInData ?
-              Builder.CreateGlobalStringPtr(getFunctionName(SP)) :
-              Constant::getNullValue(Builder.getInt8PtrTy()),
-            Builder.getInt32(FuncChecksum),
-            Builder.getInt8(Options.UseCfgChecksum),
-            Builder.getInt32(CfgChecksum));
+        Builder.CreateCall(
+            EmitFunction,
+            {Builder.getInt32(j),
+             Options.FunctionNamesInData
+                 ? Builder.CreateGlobalStringPtr(getFunctionName(SP))
+                 : Constant::getNullValue(Builder.getInt8PtrTy()),
+             Builder.getInt32(FuncChecksum),
+             Builder.getInt8(Options.UseCfgChecksum),
+             Builder.getInt32(CfgChecksum)});
 
         GlobalVariable *GV = CountersBySP[j].first;
         unsigned Arcs =
           cast<ArrayType>(GV->getType()->getElementType())->getNumElements();
-        Builder.CreateCall2(EmitArcs,
-                            Builder.getInt32(Arcs),
-                            Builder.CreateConstGEP2_64(GV, 0, 0));
+        Builder.CreateCall(EmitArcs, {Builder.getInt32(Arcs),
+                                      Builder.CreateConstGEP2_64(GV, 0, 0)});
       }
-      Builder.CreateCall(SummaryInfo);
-      Builder.CreateCall(EndFile);
+      Builder.CreateCall(SummaryInfo, {});
+      Builder.CreateCall(EndFile, {});
     }
   }
 
@@ -953,7 +954,7 @@ insertFlush(ArrayRef<std::pair<GlobalVariable*, MDNode*> > CountersBySP) {
   assert(WriteoutF && "Need to create the writeout function first!");
 
   IRBuilder<> Builder(Entry);
-  Builder.CreateCall(WriteoutF);
+  Builder.CreateCall(WriteoutF, {});
 
   // Zero out the counters.
   for (ArrayRef<std::pair<GlobalVariable *, MDNode *> >::iterator
diff --git a/lib/Transforms/Instrumentation/InstrProfiling.cpp b/lib/Transforms/Instrumentation/InstrProfiling.cpp
index b5a491f3a40..712bf8edc7e 100644
--- a/lib/Transforms/Instrumentation/InstrProfiling.cpp
+++ b/lib/Transforms/Instrumentation/InstrProfiling.cpp
@@ -97,7 +97,8 @@ class InstrProfiling : public ModulePass {
   /// Add uses of our data variables and runtime hook.
   void emitUses();
 
-  /// Create a static initializer for our data, on platforms that need it.
+  /// Create a static initializer for our data, on platforms that need it,
+  /// and for any profile output file that was specified.
   void emitInitialization();
 };
 
@@ -145,8 +146,8 @@ void InstrProfiling::lowerIncrement(InstrProfIncrementInst *Inc) {
 
   IRBuilder<> Builder(Inc->getParent(), *Inc);
   uint64_t Index = Inc->getIndex()->getZExtValue();
-  llvm::Value *Addr = Builder.CreateConstInBoundsGEP2_64(Counters, 0, Index);
-  llvm::Value *Count = Builder.CreateLoad(Addr, "pgocount");
+  Value *Addr = Builder.CreateConstInBoundsGEP2_64(Counters, 0, Index);
+  Value *Count = Builder.CreateLoad(Addr, "pgocount");
   Count = Builder.CreateAdd(Count, Builder.getInt64(1));
   Inc->replaceAllUsesWith(Builder.CreateStore(Count, Addr));
   Inc->eraseFromParent();
@@ -195,9 +196,13 @@ InstrProfiling::getOrCreateRegionCounters(InstrProfIncrementInst *Inc) {
   if (It != RegionCounters.end())
     return It->second;
 
-  // Move the name variable to the right section.
+  // Move the name variable to the right section. Make sure it is placed in the
+  // same comdat as its associated function. Otherwise, we may get multiple
+  // counters for the same function in certain cases.
+  Function *Fn = Inc->getParent()->getParent();
   Name->setSection(getNameSection());
   Name->setAlignment(1);
+  Name->setComdat(Fn->getComdat());
 
   uint64_t NumCounters = Inc->getNumCounters()->getZExtValue();
   LLVMContext &Ctx = M->getContext();
@@ -210,6 +215,7 @@ InstrProfiling::getOrCreateRegionCounters(InstrProfIncrementInst *Inc) {
   Counters->setVisibility(Name->getVisibility());
   Counters->setSection(getCountersSection());
   Counters->setAlignment(8);
+  Counters->setComdat(Fn->getComdat());
 
   RegionCounters[Inc->getName()] = Counters;
 
@@ -234,6 +240,7 @@ InstrProfiling::getOrCreateRegionCounters(InstrProfIncrementInst *Inc) {
   Data->setVisibility(Name->getVisibility());
   Data->setSection(getDataSection());
   Data->setAlignment(8);
+  Data->setComdat(Fn->getComdat());
 
   // Mark the data variable as used so that it isn't stripped out.
   UsedVars.push_back(Data);
@@ -256,7 +263,7 @@ void InstrProfiling::emitRegistration() {
   if (Options.NoRedZone)
     RegisterF->addFnAttr(Attribute::NoRedZone);
 
-  auto *RuntimeRegisterTy = llvm::FunctionType::get(VoidTy, VoidPtrTy, false);
+  auto *RuntimeRegisterTy = FunctionType::get(VoidTy, VoidPtrTy, false);
   auto *RuntimeRegisterF =
       Function::Create(RuntimeRegisterTy, GlobalVariable::ExternalLinkage,
                        "__llvm_profile_register_function", M);
@@ -303,7 +310,7 @@ void InstrProfiling::emitUses() {
     return;
 
   GlobalVariable *LLVMUsed = M->getGlobalVariable("llvm.used");
-  std::vector<Constant*> MergedVars;
+  std::vector<Constant *> MergedVars;
   if (LLVMUsed) {
     // Collect the existing members of llvm.used.
     ConstantArray *Inits = cast<ConstantArray>(LLVMUsed->getInitializer());
@@ -316,20 +323,22 @@ void InstrProfiling::emitUses() {
   // Add uses for our data.
   for (auto *Value : UsedVars)
     MergedVars.push_back(
-        ConstantExpr::getBitCast(cast<llvm::Constant>(Value), i8PTy));
+        ConstantExpr::getBitCast(cast<Constant>(Value), i8PTy));
 
   // Recreate llvm.used.
   ArrayType *ATy = ArrayType::get(i8PTy, MergedVars.size());
-  LLVMUsed = new llvm::GlobalVariable(
-      *M, ATy, false, llvm::GlobalValue::AppendingLinkage,
-      llvm::ConstantArray::get(ATy, MergedVars), "llvm.used");
+  LLVMUsed =
+      new GlobalVariable(*M, ATy, false, GlobalValue::AppendingLinkage,
+                         ConstantArray::get(ATy, MergedVars), "llvm.used");
 
   LLVMUsed->setSection("llvm.metadata");
 }
 
 void InstrProfiling::emitInitialization() {
+  std::string InstrProfileOutput = Options.InstrProfileOutput;
+
   Constant *RegisterF = M->getFunction("__llvm_profile_register_functions");
-  if (!RegisterF)
+  if (!RegisterF && InstrProfileOutput.empty())
     return;
 
   // Create the initialization function.
@@ -344,7 +353,24 @@ void InstrProfiling::emitInitialization() {
 
   // Add the basic block and the necessary calls.
   IRBuilder<> IRB(BasicBlock::Create(M->getContext(), "", F));
-  IRB.CreateCall(RegisterF);
+  if (RegisterF)
+    IRB.CreateCall(RegisterF, {});
+  if (!InstrProfileOutput.empty()) {
+    auto *Int8PtrTy = Type::getInt8PtrTy(M->getContext());
+    auto *SetNameTy = FunctionType::get(VoidTy, Int8PtrTy, false);
+    auto *SetNameF =
+        Function::Create(SetNameTy, GlobalValue::ExternalLinkage,
+                         "__llvm_profile_override_default_filename", M);
+
+    // Create variable for profile name.
+    Constant *ProfileNameConst =
+        ConstantDataArray::getString(M->getContext(), InstrProfileOutput, true);
+    GlobalVariable *ProfileName =
+        new GlobalVariable(*M, ProfileNameConst->getType(), true,
+                           GlobalValue::PrivateLinkage, ProfileNameConst);
+
+    IRB.CreateCall(SetNameF, IRB.CreatePointerCast(ProfileName, Int8PtrTy));
+  }
   IRB.CreateRetVoid();
 
   appendToGlobalCtors(*M, F, 0);
diff --git a/lib/Transforms/Instrumentation/Instrumentation.cpp b/lib/Transforms/Instrumentation/Instrumentation.cpp
index a91fc0ec2a4..27505859100 100644
--- a/lib/Transforms/Instrumentation/Instrumentation.cpp
+++ b/lib/Transforms/Instrumentation/Instrumentation.cpp
@@ -30,6 +30,7 @@ void llvm::initializeInstrumentation(PassRegistry &Registry) {
   initializeThreadSanitizerPass(Registry);
   initializeSanitizerCoverageModulePass(Registry);
   initializeDataFlowSanitizerPass(Registry);
+  initializeSafeStackPass(Registry);
 }
 
 /// LLVMInitializeInstrumentation - C binding for
diff --git a/lib/Transforms/Instrumentation/MemorySanitizer.cpp b/lib/Transforms/Instrumentation/MemorySanitizer.cpp
index 2b350667b82..226876d407c 100644
--- a/lib/Transforms/Instrumentation/MemorySanitizer.cpp
+++ b/lib/Transforms/Instrumentation/MemorySanitizer.cpp
@@ -191,6 +191,9 @@ static cl::opt<bool> ClCheckConstantShadow("msan-check-constant-shadow",
        cl::desc("Insert checks for constant shadow values"),
        cl::Hidden, cl::init(false));
 
+static const char *const kMsanModuleCtorName = "msan.module_ctor";
+static const char *const kMsanInitName = "__msan_init";
+
 namespace {
 
 // Memory map parameters used in application-to-shadow address calculation.
@@ -233,6 +236,14 @@ static const MemoryMapParams Linux_MIPS64_MemoryMapParams = {
   0x002000000000,  // OriginBase
 };
 
+// ppc64 Linux
+static const MemoryMapParams Linux_PowerPC64_MemoryMapParams = {
+  0x200000000000,  // AndMask
+  0x100000000000,  // XorMask
+  0x080000000000,  // ShadowBase
+  0x1C0000000000,  // OriginBase
+};
+
 // i386 FreeBSD
 static const MemoryMapParams FreeBSD_I386_MemoryMapParams = {
   0x000180000000,  // AndMask
@@ -259,6 +270,11 @@ static const PlatformMemoryMapParams Linux_MIPS_MemoryMapParams = {
   &Linux_MIPS64_MemoryMapParams,
 };
 
+static const PlatformMemoryMapParams Linux_PowerPC_MemoryMapParams = {
+  NULL,
+  &Linux_PowerPC64_MemoryMapParams,
+};
+
 static const PlatformMemoryMapParams FreeBSD_X86_MemoryMapParams = {
   &FreeBSD_I386_MemoryMapParams,
   &FreeBSD_X86_64_MemoryMapParams,
@@ -332,6 +348,7 @@ class MemorySanitizer : public FunctionPass {
   MDNode *OriginStoreWeights;
   /// \brief An empty volatile inline asm that prevents callback merge.
   InlineAsm *EmptyAsm;
+  Function *MsanCtorFunction;
 
   friend struct MemorySanitizerVisitor;
   friend struct VarArgAMD64Helper;
@@ -475,6 +492,10 @@ bool MemorySanitizer::doInitialization(Module &M) {
         case Triple::mips64el:
           MapParams = Linux_MIPS_MemoryMapParams.bits64;
           break;
+        case Triple::ppc64:
+        case Triple::ppc64le:
+          MapParams = Linux_PowerPC_MemoryMapParams.bits64;
+          break;
         default:
           report_fatal_error("unsupported architecture");
       }
@@ -491,9 +512,12 @@ bool MemorySanitizer::doInitialization(Module &M) {
   ColdCallWeights = MDBuilder(*C).createBranchWeights(1, 1000);
   OriginStoreWeights = MDBuilder(*C).createBranchWeights(1, 1000);
 
-  // Insert a call to __msan_init/__msan_track_origins into the module's CTORs.
-  appendToGlobalCtors(M, cast<Function>(M.getOrInsertFunction(
-                      "__msan_init", IRB.getVoidTy(), nullptr)), 0);
+  std::tie(MsanCtorFunction, std::ignore) =
+      createSanitizerCtorAndInitFunctions(M, kMsanModuleCtorName, kMsanInitName,
+                                          /*InitArgTypes=*/{},
+                                          /*InitArgs=*/{});
+
+  appendToGlobalCtors(M, MsanCtorFunction, 0);
 
   if (TrackOrigins)
     new GlobalVariable(M, IRB.getInt32Ty(), true, GlobalValue::WeakODRLinkage,
@@ -666,9 +690,9 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> {
         Value *Fn = MS.MaybeStoreOriginFn[SizeIndex];
         Value *ConvertedShadow2 = IRB.CreateZExt(
             ConvertedShadow, IRB.getIntNTy(8 * (1 << SizeIndex)));
-        IRB.CreateCall3(Fn, ConvertedShadow2,
-                        IRB.CreatePointerCast(Addr, IRB.getInt8PtrTy()),
-                        Origin);
+        IRB.CreateCall(Fn, {ConvertedShadow2,
+                            IRB.CreatePointerCast(Addr, IRB.getInt8PtrTy()),
+                            Origin});
       } else {
         Value *Cmp = IRB.CreateICmpNE(
             ConvertedShadow, getCleanShadow(ConvertedShadow), "_mscmp");
@@ -721,8 +745,8 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> {
           IRB.CreateStore(Origin ? (Value *)Origin : (Value *)IRB.getInt32(0),
                           MS.OriginTLS);
         }
-        IRB.CreateCall(MS.WarningFn);
-        IRB.CreateCall(MS.EmptyAsm);
+        IRB.CreateCall(MS.WarningFn, {});
+        IRB.CreateCall(MS.EmptyAsm, {});
         // FIXME: Insert UnreachableInst if !ClKeepGoing?
         // This may invalidate some of the following checks and needs to be done
         // at the very end.
@@ -738,9 +762,9 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> {
       Value *Fn = MS.MaybeWarningFn[SizeIndex];
       Value *ConvertedShadow2 =
           IRB.CreateZExt(ConvertedShadow, IRB.getIntNTy(8 * (1 << SizeIndex)));
-      IRB.CreateCall2(Fn, ConvertedShadow2, MS.TrackOrigins && Origin
+      IRB.CreateCall(Fn, {ConvertedShadow2, MS.TrackOrigins && Origin
                                                 ? Origin
-                                                : (Value *)IRB.getInt32(0));
+                                                : (Value *)IRB.getInt32(0)});
     } else {
       Value *Cmp = IRB.CreateICmpNE(ConvertedShadow,
                                     getCleanShadow(ConvertedShadow), "_mscmp");
@@ -753,8 +777,8 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> {
         IRB.CreateStore(Origin ? (Value *)Origin : (Value *)IRB.getInt32(0),
                         MS.OriginTLS);
       }
-      IRB.CreateCall(MS.WarningFn);
-      IRB.CreateCall(MS.EmptyAsm);
+      IRB.CreateCall(MS.WarningFn, {});
+      IRB.CreateCall(MS.EmptyAsm, {});
       DEBUG(dbgs() << "  CHECK: " << *Cmp << "\n");
     }
   }
@@ -1795,11 +1819,11 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> {
   /// Similar situation exists for memcpy and memset.
   void visitMemMoveInst(MemMoveInst &I) {
     IRBuilder<> IRB(&I);
-    IRB.CreateCall3(
-      MS.MemmoveFn,
-      IRB.CreatePointerCast(I.getArgOperand(0), IRB.getInt8PtrTy()),
-      IRB.CreatePointerCast(I.getArgOperand(1), IRB.getInt8PtrTy()),
-      IRB.CreateIntCast(I.getArgOperand(2), MS.IntptrTy, false));
+    IRB.CreateCall(
+        MS.MemmoveFn,
+        {IRB.CreatePointerCast(I.getArgOperand(0), IRB.getInt8PtrTy()),
+         IRB.CreatePointerCast(I.getArgOperand(1), IRB.getInt8PtrTy()),
+         IRB.CreateIntCast(I.getArgOperand(2), MS.IntptrTy, false)});
     I.eraseFromParent();
   }
 
@@ -1809,22 +1833,22 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> {
   // alignment.
   void visitMemCpyInst(MemCpyInst &I) {
     IRBuilder<> IRB(&I);
-    IRB.CreateCall3(
-      MS.MemcpyFn,
-      IRB.CreatePointerCast(I.getArgOperand(0), IRB.getInt8PtrTy()),
-      IRB.CreatePointerCast(I.getArgOperand(1), IRB.getInt8PtrTy()),
-      IRB.CreateIntCast(I.getArgOperand(2), MS.IntptrTy, false));
+    IRB.CreateCall(
+        MS.MemcpyFn,
+        {IRB.CreatePointerCast(I.getArgOperand(0), IRB.getInt8PtrTy()),
+         IRB.CreatePointerCast(I.getArgOperand(1), IRB.getInt8PtrTy()),
+         IRB.CreateIntCast(I.getArgOperand(2), MS.IntptrTy, false)});
     I.eraseFromParent();
   }
 
   // Same as memcpy.
   void visitMemSetInst(MemSetInst &I) {
     IRBuilder<> IRB(&I);
-    IRB.CreateCall3(
-      MS.MemsetFn,
-      IRB.CreatePointerCast(I.getArgOperand(0), IRB.getInt8PtrTy()),
-      IRB.CreateIntCast(I.getArgOperand(1), IRB.getInt32Ty(), false),
-      IRB.CreateIntCast(I.getArgOperand(2), MS.IntptrTy, false));
+    IRB.CreateCall(
+        MS.MemsetFn,
+        {IRB.CreatePointerCast(I.getArgOperand(0), IRB.getInt8PtrTy()),
+         IRB.CreateIntCast(I.getArgOperand(1), IRB.getInt32Ty(), false),
+         IRB.CreateIntCast(I.getArgOperand(2), MS.IntptrTy, false)});
     I.eraseFromParent();
   }
 
@@ -1843,15 +1867,15 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> {
   };
 
   static IntrinsicKind getIntrinsicKind(Intrinsic::ID iid) {
-    const int DoesNotAccessMemory = IK_DoesNotAccessMemory;
-    const int OnlyReadsArgumentPointees = IK_OnlyReadsMemory;
-    const int OnlyReadsMemory = IK_OnlyReadsMemory;
-    const int OnlyAccessesArgumentPointees = IK_WritesMemory;
-    const int UnknownModRefBehavior = IK_WritesMemory;
+    const int FMRB_DoesNotAccessMemory = IK_DoesNotAccessMemory;
+    const int FMRB_OnlyReadsArgumentPointees = IK_OnlyReadsMemory;
+    const int FMRB_OnlyReadsMemory = IK_OnlyReadsMemory;
+    const int FMRB_OnlyAccessesArgumentPointees = IK_WritesMemory;
+    const int FMRB_UnknownModRefBehavior = IK_WritesMemory;
 #define GET_INTRINSIC_MODREF_BEHAVIOR
-#define ModRefBehavior IntrinsicKind
+#define FunctionModRefBehavior IntrinsicKind
 #include "llvm/IR/Intrinsics.gen"
-#undef ModRefBehavior
+#undef FunctionModRefBehavior
 #undef GET_INTRINSIC_MODREF_BEHAVIOR
   }
 
@@ -2015,6 +2039,8 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> {
     Value *CopyOp, *ConvertOp;
 
     switch (I.getNumArgOperands()) {
+    case 3:
+      assert(isa<ConstantInt>(I.getArgOperand(2)) && "Invalid rounding mode");
     case 2:
       CopyOp = I.getArgOperand(0);
       ConvertOp = I.getArgOperand(1);
@@ -2105,8 +2131,8 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> {
                              : Lower64ShadowExtend(IRB, S2, getShadowTy(&I));
     Value *V1 = I.getOperand(0);
     Value *V2 = I.getOperand(1);
-    Value *Shift = IRB.CreateCall2(I.getCalledValue(),
-                                   IRB.CreateBitCast(S1, V1->getType()), V2);
+    Value *Shift = IRB.CreateCall(I.getCalledValue(),
+                                  {IRB.CreateBitCast(S1, V1->getType()), V2});
     Shift = IRB.CreateBitCast(Shift, getShadowTy(&I));
     setShadow(&I, IRB.CreateOr(Shift, S2Conv));
     setOriginForNaryOp(I);
@@ -2186,7 +2212,8 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> {
     Function *ShadowFn = Intrinsic::getDeclaration(
         F.getParent(), getSignedPackIntrinsic(I.getIntrinsicID()));
 
-    Value *S = IRB.CreateCall2(ShadowFn, S1_ext, S2_ext, "_msprop_vector_pack");
+    Value *S =
+        IRB.CreateCall(ShadowFn, {S1_ext, S2_ext}, "_msprop_vector_pack");
     if (isX86_MMX) S = IRB.CreateBitCast(S, getShadowTy(&I));
     setShadow(&I, S);
     setOriginForNaryOp(I);
@@ -2537,9 +2564,9 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> {
     const DataLayout &DL = F.getParent()->getDataLayout();
     uint64_t Size = DL.getTypeAllocSize(I.getAllocatedType());
     if (PoisonStack && ClPoisonStackWithCall) {
-      IRB.CreateCall2(MS.MsanPoisonStackFn,
-                      IRB.CreatePointerCast(&I, IRB.getInt8PtrTy()),
-                      ConstantInt::get(MS.IntptrTy, Size));
+      IRB.CreateCall(MS.MsanPoisonStackFn,
+                     {IRB.CreatePointerCast(&I, IRB.getInt8PtrTy()),
+                      ConstantInt::get(MS.IntptrTy, Size)});
     } else {
       Value *ShadowBase = getShadowPtr(&I, Type::getInt8PtrTy(*MS.C), IRB);
       Value *PoisonValue = IRB.getInt8(PoisonStack ? ClPoisonStackPattern : 0);
@@ -2559,11 +2586,11 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> {
           createPrivateNonConstGlobalForString(*F.getParent(),
                                                StackDescription.str());
 
-      IRB.CreateCall4(MS.MsanSetAllocaOrigin4Fn,
-                      IRB.CreatePointerCast(&I, IRB.getInt8PtrTy()),
+      IRB.CreateCall(MS.MsanSetAllocaOrigin4Fn,
+                     {IRB.CreatePointerCast(&I, IRB.getInt8PtrTy()),
                       ConstantInt::get(MS.IntptrTy, Size),
                       IRB.CreatePointerCast(Descr, IRB.getInt8PtrTy()),
-                      IRB.CreatePointerCast(&F, MS.IntptrTy));
+                      IRB.CreatePointerCast(&F, MS.IntptrTy)});
     }
   }
 
@@ -2626,6 +2653,30 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> {
     setOrigin(&I, getCleanOrigin());
   }
 
+  void visitCleanupPadInst(CleanupPadInst &I) {
+    if (!I.getType()->isVoidTy()) {
+      setShadow(&I, getCleanShadow(&I));
+      setOrigin(&I, getCleanOrigin());
+    }
+  }
+
+  void visitCatchPad(CatchPadInst &I) {
+    if (!I.getType()->isVoidTy()) {
+      setShadow(&I, getCleanShadow(&I));
+      setOrigin(&I, getCleanOrigin());
+    }
+  }
+
+  void visitTerminatePad(TerminatePadInst &I) {
+    DEBUG(dbgs() << "TerminatePad: " << I << "\n");
+    // Nothing to do here.
+  }
+
+  void visitCatchEndPadInst(CatchEndPadInst &I) {
+    DEBUG(dbgs() << "CatchEndPad: " << I << "\n");
+    // Nothing to do here.
+  }
+
   void visitGetElementPtrInst(GetElementPtrInst &I) {
     handleShadowOr(I);
   }
@@ -2669,6 +2720,16 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> {
     // Nothing to do here.
   }
 
+  void visitCleanupReturnInst(CleanupReturnInst &CRI) {
+    DEBUG(dbgs() << "CleanupReturn: " << CRI << "\n");
+    // Nothing to do here.
+  }
+
+  void visitCatchReturnInst(CatchReturnInst &CRI) {
+    DEBUG(dbgs() << "CatchReturn: " << CRI << "\n");
+    // Nothing to do here.
+  }
+
   void visitInstruction(Instruction &I) {
     // Everything else: stop propagating and check for poisoned shadow.
     if (ClDumpStrictInstructions)
@@ -2983,6 +3044,8 @@ VarArgHelper *CreateVarArgHelper(Function &Func, MemorySanitizer &Msan,
 }  // namespace
 
 bool MemorySanitizer::runOnFunction(Function &F) {
+  if (&F == MsanCtorFunction)
+    return false;
   MemorySanitizerVisitor Visitor(F, *this);
 
   // Clear out readonly/readnone attributes.
diff --git a/lib/Transforms/Instrumentation/SafeStack.cpp b/lib/Transforms/Instrumentation/SafeStack.cpp
new file mode 100644
index 00000000000..6b185a2b127
--- /dev/null
+++ b/lib/Transforms/Instrumentation/SafeStack.cpp
@@ -0,0 +1,609 @@
+//===-- SafeStack.cpp - Safe Stack Insertion ------------------------------===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This pass splits the stack into the safe stack (kept as-is for LLVM backend)
+// and the unsafe stack (explicitly allocated and managed through the runtime
+// support library).
+//
+// http://clang.llvm.org/docs/SafeStack.html
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/Transforms/Instrumentation.h"
+#include "llvm/ADT/Statistic.h"
+#include "llvm/ADT/Triple.h"
+#include "llvm/Analysis/AliasAnalysis.h"
+#include "llvm/Analysis/TargetTransformInfo.h"
+#include "llvm/IR/Constants.h"
+#include "llvm/IR/DataLayout.h"
+#include "llvm/IR/DerivedTypes.h"
+#include "llvm/IR/DIBuilder.h"
+#include "llvm/IR/Function.h"
+#include "llvm/IR/InstIterator.h"
+#include "llvm/IR/Instructions.h"
+#include "llvm/IR/IntrinsicInst.h"
+#include "llvm/IR/Intrinsics.h"
+#include "llvm/IR/IRBuilder.h"
+#include "llvm/IR/Module.h"
+#include "llvm/Pass.h"
+#include "llvm/Support/CommandLine.h"
+#include "llvm/Support/Debug.h"
+#include "llvm/Support/Format.h"
+#include "llvm/Support/MathExtras.h"
+#include "llvm/Support/raw_os_ostream.h"
+#include "llvm/Transforms/Utils/Local.h"
+#include "llvm/Transforms/Utils/ModuleUtils.h"
+
+using namespace llvm;
+
+#define DEBUG_TYPE "safestack"
+
+namespace llvm {
+
+STATISTIC(NumFunctions, "Total number of functions");
+STATISTIC(NumUnsafeStackFunctions, "Number of functions with unsafe stack");
+STATISTIC(NumUnsafeStackRestorePointsFunctions,
+          "Number of functions that use setjmp or exceptions");
+
+STATISTIC(NumAllocas, "Total number of allocas");
+STATISTIC(NumUnsafeStaticAllocas, "Number of unsafe static allocas");
+STATISTIC(NumUnsafeDynamicAllocas, "Number of unsafe dynamic allocas");
+STATISTIC(NumUnsafeStackRestorePoints, "Number of setjmps and landingpads");
+
+} // namespace llvm
+
+namespace {
+
+/// Check whether a given alloca instruction (AI) should be put on the safe
+/// stack or not. The function analyzes all uses of AI and checks whether it is
+/// only accessed in a memory safe way (as decided statically).
+bool IsSafeStackAlloca(const AllocaInst *AI) {
+  // Go through all uses of this alloca and check whether all accesses to the
+  // allocated object are statically known to be memory safe and, hence, the
+  // object can be placed on the safe stack.
+
+  SmallPtrSet<const Value *, 16> Visited;
+  SmallVector<const Instruction *, 8> WorkList;
+  WorkList.push_back(AI);
+
+  // A DFS search through all uses of the alloca in bitcasts/PHI/GEPs/etc.
+  while (!WorkList.empty()) {
+    const Instruction *V = WorkList.pop_back_val();
+    for (const Use &UI : V->uses()) {
+      auto I = cast<const Instruction>(UI.getUser());
+      assert(V == UI.get());
+
+      switch (I->getOpcode()) {
+      case Instruction::Load:
+        // Loading from a pointer is safe.
+        break;
+      case Instruction::VAArg:
+        // "va-arg" from a pointer is safe.
+        break;
+      case Instruction::Store:
+        if (V == I->getOperand(0))
+          // Stored the pointer - conservatively assume it may be unsafe.
+          return false;
+        // Storing to the pointee is safe.
+        break;
+
+      case Instruction::GetElementPtr:
+        if (!cast<const GetElementPtrInst>(I)->hasAllConstantIndices())
+          // GEP with non-constant indices can lead to memory errors.
+          // This also applies to inbounds GEPs, as the inbounds attribute
+          // represents an assumption that the address is in bounds, rather than
+          // an assertion that it is.
+          return false;
+
+        // We assume that GEP on static alloca with constant indices is safe,
+        // otherwise a compiler would detect it and warn during compilation.
+
+        if (!isa<const ConstantInt>(AI->getArraySize()))
+          // However, if the array size itself is not constant, the access
+          // might still be unsafe at runtime.
+          return false;
+
+      /* fallthrough */
+
+      case Instruction::BitCast:
+      case Instruction::IntToPtr:
+      case Instruction::PHI:
+      case Instruction::PtrToInt:
+      case Instruction::Select:
+        // The object can be safe or not, depending on how the result of the
+        // instruction is used.
+        if (Visited.insert(I).second)
+          WorkList.push_back(cast<const Instruction>(I));
+        break;
+
+      case Instruction::Call:
+      case Instruction::Invoke: {
+        // FIXME: add support for memset and memcpy intrinsics.
+        ImmutableCallSite CS(I);
+
+        // LLVM 'nocapture' attribute is only set for arguments whose address
+        // is not stored, passed around, or used in any other non-trivial way.
+        // We assume that passing a pointer to an object as a 'nocapture'
+        // argument is safe.
+        // FIXME: a more precise solution would require an interprocedural
+        // analysis here, which would look at all uses of an argument inside
+        // the function being called.
+        ImmutableCallSite::arg_iterator B = CS.arg_begin(), E = CS.arg_end();
+        for (ImmutableCallSite::arg_iterator A = B; A != E; ++A)
+          if (A->get() == V && !CS.doesNotCapture(A - B))
+            // The parameter is not marked 'nocapture' - unsafe.
+            return false;
+        continue;
+      }
+
+      default:
+        // The object is unsafe if it is used in any other way.
+        return false;
+      }
+    }
+  }
+
+  // All uses of the alloca are safe, we can place it on the safe stack.
+  return true;
+}
+
+/// The SafeStack pass splits the stack of each function into the
+/// safe stack, which is only accessed through memory safe dereferences
+/// (as determined statically), and the unsafe stack, which contains all
+/// local variables that are accessed in unsafe ways.
+class SafeStack : public FunctionPass {
+  const DataLayout *DL;
+
+  Type *StackPtrTy;
+  Type *IntPtrTy;
+  Type *Int32Ty;
+  Type *Int8Ty;
+
+  Constant *UnsafeStackPtr = nullptr;
+
+  /// Unsafe stack alignment. Each stack frame must ensure that the stack is
+  /// aligned to this value. We need to re-align the unsafe stack if the
+  /// alignment of any object on the stack exceeds this value.
+  ///
+  /// 16 seems like a reasonable upper bound on the alignment of objects that we
+  /// might expect to appear on the stack on most common targets.
+  enum { StackAlignment = 16 };
+
+  /// \brief Build a constant representing a pointer to the unsafe stack
+  /// pointer.
+  Constant *getOrCreateUnsafeStackPtr(Module &M);
+
+  /// \brief Find all static allocas, dynamic allocas, return instructions and
+  /// stack restore points (exception unwind blocks and setjmp calls) in the
+  /// given function and append them to the respective vectors.
+  void findInsts(Function &F, SmallVectorImpl<AllocaInst *> &StaticAllocas,
+                 SmallVectorImpl<AllocaInst *> &DynamicAllocas,
+                 SmallVectorImpl<ReturnInst *> &Returns,
+                 SmallVectorImpl<Instruction *> &StackRestorePoints);
+
+  /// \brief Allocate space for all static allocas in \p StaticAllocas,
+  /// replace allocas with pointers into the unsafe stack and generate code to
+  /// restore the stack pointer before all return instructions in \p Returns.
+  ///
+  /// \returns A pointer to the top of the unsafe stack after all unsafe static
+  /// allocas are allocated.
+  Value *moveStaticAllocasToUnsafeStack(Function &F,
+                                        ArrayRef<AllocaInst *> StaticAllocas,
+                                        ArrayRef<ReturnInst *> Returns);
+
+  /// \brief Generate code to restore the stack after all stack restore points
+  /// in \p StackRestorePoints.
+  ///
+  /// \returns A local variable in which to maintain the dynamic top of the
+  /// unsafe stack if needed.
+  AllocaInst *
+  createStackRestorePoints(Function &F,
+                           ArrayRef<Instruction *> StackRestorePoints,
+                           Value *StaticTop, bool NeedDynamicTop);
+
+  /// \brief Replace all allocas in \p DynamicAllocas with code to allocate
+  /// space dynamically on the unsafe stack and store the dynamic unsafe stack
+  /// top to \p DynamicTop if non-null.
+  void moveDynamicAllocasToUnsafeStack(Function &F, Value *UnsafeStackPtr,
+                                       AllocaInst *DynamicTop,
+                                       ArrayRef<AllocaInst *> DynamicAllocas);
+
+public:
+  static char ID; // Pass identification, replacement for typeid.
+  SafeStack() : FunctionPass(ID), DL(nullptr) {
+    initializeSafeStackPass(*PassRegistry::getPassRegistry());
+  }
+
+  virtual void getAnalysisUsage(AnalysisUsage &AU) const {
+    AU.addRequired<AliasAnalysis>();
+  }
+
+  virtual bool doInitialization(Module &M) {
+    DL = &M.getDataLayout();
+
+    StackPtrTy = Type::getInt8PtrTy(M.getContext());
+    IntPtrTy = DL->getIntPtrType(M.getContext());
+    Int32Ty = Type::getInt32Ty(M.getContext());
+    Int8Ty = Type::getInt8Ty(M.getContext());
+
+    return false;
+  }
+
+  bool runOnFunction(Function &F);
+
+}; // class SafeStack
+
+Constant *SafeStack::getOrCreateUnsafeStackPtr(Module &M) {
+  // The unsafe stack pointer is stored in a global variable with a magic name.
+  const char *kUnsafeStackPtrVar = "__safestack_unsafe_stack_ptr";
+
+  auto UnsafeStackPtr =
+      dyn_cast_or_null<GlobalVariable>(M.getNamedValue(kUnsafeStackPtrVar));
+
+  if (!UnsafeStackPtr) {
+    // The global variable is not defined yet, define it ourselves.
+    // We use the initial-exec TLS model because we do not support the variable
+    // living anywhere other than in the main executable.
+    UnsafeStackPtr = new GlobalVariable(
+        /*Module=*/M, /*Type=*/StackPtrTy,
+        /*isConstant=*/false, /*Linkage=*/GlobalValue::ExternalLinkage,
+        /*Initializer=*/0, /*Name=*/kUnsafeStackPtrVar,
+        /*InsertBefore=*/nullptr,
+        /*ThreadLocalMode=*/GlobalValue::InitialExecTLSModel);
+  } else {
+    // The variable exists, check its type and attributes.
+    if (UnsafeStackPtr->getValueType() != StackPtrTy) {
+      report_fatal_error(Twine(kUnsafeStackPtrVar) + " must have void* type");
+    }
+
+    if (!UnsafeStackPtr->isThreadLocal()) {
+      report_fatal_error(Twine(kUnsafeStackPtrVar) + " must be thread-local");
+    }
+  }
+
+  return UnsafeStackPtr;
+}
+
+void SafeStack::findInsts(Function &F,
+                          SmallVectorImpl<AllocaInst *> &StaticAllocas,
+                          SmallVectorImpl<AllocaInst *> &DynamicAllocas,
+                          SmallVectorImpl<ReturnInst *> &Returns,
+                          SmallVectorImpl<Instruction *> &StackRestorePoints) {
+  for (Instruction &I : inst_range(&F)) {
+    if (auto AI = dyn_cast<AllocaInst>(&I)) {
+      ++NumAllocas;
+
+      if (IsSafeStackAlloca(AI))
+        continue;
+
+      if (AI->isStaticAlloca()) {
+        ++NumUnsafeStaticAllocas;
+        StaticAllocas.push_back(AI);
+      } else {
+        ++NumUnsafeDynamicAllocas;
+        DynamicAllocas.push_back(AI);
+      }
+    } else if (auto RI = dyn_cast<ReturnInst>(&I)) {
+      Returns.push_back(RI);
+    } else if (auto CI = dyn_cast<CallInst>(&I)) {
+      // setjmps require stack restore.
+      if (CI->getCalledFunction() && CI->canReturnTwice())
+        StackRestorePoints.push_back(CI);
+    } else if (auto LP = dyn_cast<LandingPadInst>(&I)) {
+      // Exception landing pads require stack restore.
+      StackRestorePoints.push_back(LP);
+    } else if (auto II = dyn_cast<IntrinsicInst>(&I)) {
+      if (II->getIntrinsicID() == Intrinsic::gcroot)
+        llvm::report_fatal_error(
+            "gcroot intrinsic not compatible with safestack attribute");
+    }
+  }
+}
+
+AllocaInst *
+SafeStack::createStackRestorePoints(Function &F,
+                                    ArrayRef<Instruction *> StackRestorePoints,
+                                    Value *StaticTop, bool NeedDynamicTop) {
+  if (StackRestorePoints.empty())
+    return nullptr;
+
+  IRBuilder<> IRB(StaticTop
+                      ? cast<Instruction>(StaticTop)->getNextNode()
+                      : (Instruction *)F.getEntryBlock().getFirstInsertionPt());
+
+  // We need the current value of the shadow stack pointer to restore
+  // after longjmp or exception catching.
+
+  // FIXME: On some platforms this could be handled by the longjmp/exception
+  // runtime itself.
+
+  AllocaInst *DynamicTop = nullptr;
+  if (NeedDynamicTop)
+    // If we also have dynamic alloca's, the stack pointer value changes
+    // throughout the function. For now we store it in an alloca.
+    DynamicTop = IRB.CreateAlloca(StackPtrTy, /*ArraySize=*/nullptr,
+                                  "unsafe_stack_dynamic_ptr");
+
+  if (!StaticTop)
+    // We need the original unsafe stack pointer value, even if there are
+    // no unsafe static allocas.
+    StaticTop = IRB.CreateLoad(UnsafeStackPtr, false, "unsafe_stack_ptr");
+
+  if (NeedDynamicTop)
+    IRB.CreateStore(StaticTop, DynamicTop);
+
+  // Restore current stack pointer after longjmp/exception catch.
+  for (Instruction *I : StackRestorePoints) {
+    ++NumUnsafeStackRestorePoints;
+
+    IRB.SetInsertPoint(cast<Instruction>(I->getNextNode()));
+    Value *CurrentTop = DynamicTop ? IRB.CreateLoad(DynamicTop) : StaticTop;
+    IRB.CreateStore(CurrentTop, UnsafeStackPtr);
+  }
+
+  return DynamicTop;
+}
+
+Value *
+SafeStack::moveStaticAllocasToUnsafeStack(Function &F,
+                                          ArrayRef<AllocaInst *> StaticAllocas,
+                                          ArrayRef<ReturnInst *> Returns) {
+  if (StaticAllocas.empty())
+    return nullptr;
+
+  IRBuilder<> IRB(F.getEntryBlock().getFirstInsertionPt());
+  DIBuilder DIB(*F.getParent());
+
+  // We explicitly compute and set the unsafe stack layout for all unsafe
+  // static alloca instructions. We save the unsafe "base pointer" in the
+  // prologue into a local variable and restore it in the epilogue.
+
+  // Load the current stack pointer (we'll also use it as a base pointer).
+  // FIXME: use a dedicated register for it ?
+  Instruction *BasePointer =
+      IRB.CreateLoad(UnsafeStackPtr, false, "unsafe_stack_ptr");
+  assert(BasePointer->getType() == StackPtrTy);
+
+  for (ReturnInst *RI : Returns) {
+    IRB.SetInsertPoint(RI);
+    IRB.CreateStore(BasePointer, UnsafeStackPtr);
+  }
+
+  // Compute maximum alignment among static objects on the unsafe stack.
+  unsigned MaxAlignment = 0;
+  for (AllocaInst *AI : StaticAllocas) {
+    Type *Ty = AI->getAllocatedType();
+    unsigned Align =
+        std::max((unsigned)DL->getPrefTypeAlignment(Ty), AI->getAlignment());
+    if (Align > MaxAlignment)
+      MaxAlignment = Align;
+  }
+
+  if (MaxAlignment > StackAlignment) {
+    // Re-align the base pointer according to the max requested alignment.
+    assert(isPowerOf2_32(MaxAlignment));
+    IRB.SetInsertPoint(cast<Instruction>(BasePointer->getNextNode()));
+    BasePointer = cast<Instruction>(IRB.CreateIntToPtr(
+        IRB.CreateAnd(IRB.CreatePtrToInt(BasePointer, IntPtrTy),
+                      ConstantInt::get(IntPtrTy, ~uint64_t(MaxAlignment - 1))),
+        StackPtrTy));
+  }
+
+  // Allocate space for every unsafe static AllocaInst on the unsafe stack.
+  int64_t StaticOffset = 0; // Current stack top.
+  for (AllocaInst *AI : StaticAllocas) {
+    IRB.SetInsertPoint(AI);
+
+    auto CArraySize = cast<ConstantInt>(AI->getArraySize());
+    Type *Ty = AI->getAllocatedType();
+
+    uint64_t Size = DL->getTypeAllocSize(Ty) * CArraySize->getZExtValue();
+    if (Size == 0)
+      Size = 1; // Don't create zero-sized stack objects.
+
+    // Ensure the object is properly aligned.
+    unsigned Align =
+        std::max((unsigned)DL->getPrefTypeAlignment(Ty), AI->getAlignment());
+
+    // Add alignment.
+    // NOTE: we ensure that BasePointer itself is aligned to >= Align.
+    StaticOffset += Size;
+    StaticOffset = RoundUpToAlignment(StaticOffset, Align);
+
+    Value *Off = IRB.CreateGEP(BasePointer, // BasePointer is i8*
+                               ConstantInt::get(Int32Ty, -StaticOffset));
+    Value *NewAI = IRB.CreateBitCast(Off, AI->getType(), AI->getName());
+    if (AI->hasName() && isa<Instruction>(NewAI))
+      cast<Instruction>(NewAI)->takeName(AI);
+
+    // Replace alloc with the new location.
+    replaceDbgDeclareForAlloca(AI, NewAI, DIB, /*Deref=*/true);
+    AI->replaceAllUsesWith(NewAI);
+    AI->eraseFromParent();
+  }
+
+  // Re-align BasePointer so that our callees would see it aligned as
+  // expected.
+  // FIXME: no need to update BasePointer in leaf functions.
+  StaticOffset = RoundUpToAlignment(StaticOffset, StackAlignment);
+
+  // Update shadow stack pointer in the function epilogue.
+  IRB.SetInsertPoint(cast<Instruction>(BasePointer->getNextNode()));
+
+  Value *StaticTop =
+      IRB.CreateGEP(BasePointer, ConstantInt::get(Int32Ty, -StaticOffset),
+                    "unsafe_stack_static_top");
+  IRB.CreateStore(StaticTop, UnsafeStackPtr);
+  return StaticTop;
+}
+
+void SafeStack::moveDynamicAllocasToUnsafeStack(
+    Function &F, Value *UnsafeStackPtr, AllocaInst *DynamicTop,
+    ArrayRef<AllocaInst *> DynamicAllocas) {
+  DIBuilder DIB(*F.getParent());
+
+  for (AllocaInst *AI : DynamicAllocas) {
+    IRBuilder<> IRB(AI);
+
+    // Compute the new SP value (after AI).
+    Value *ArraySize = AI->getArraySize();
+    if (ArraySize->getType() != IntPtrTy)
+      ArraySize = IRB.CreateIntCast(ArraySize, IntPtrTy, false);
+
+    Type *Ty = AI->getAllocatedType();
+    uint64_t TySize = DL->getTypeAllocSize(Ty);
+    Value *Size = IRB.CreateMul(ArraySize, ConstantInt::get(IntPtrTy, TySize));
+
+    Value *SP = IRB.CreatePtrToInt(IRB.CreateLoad(UnsafeStackPtr), IntPtrTy);
+    SP = IRB.CreateSub(SP, Size);
+
+    // Align the SP value to satisfy the AllocaInst, type and stack alignments.
+    unsigned Align = std::max(
+        std::max((unsigned)DL->getPrefTypeAlignment(Ty), AI->getAlignment()),
+        (unsigned)StackAlignment);
+
+    assert(isPowerOf2_32(Align));
+    Value *NewTop = IRB.CreateIntToPtr(
+        IRB.CreateAnd(SP, ConstantInt::get(IntPtrTy, ~uint64_t(Align - 1))),
+        StackPtrTy);
+
+    // Save the stack pointer.
+    IRB.CreateStore(NewTop, UnsafeStackPtr);
+    if (DynamicTop)
+      IRB.CreateStore(NewTop, DynamicTop);
+
+    Value *NewAI = IRB.CreateIntToPtr(SP, AI->getType());
+    if (AI->hasName() && isa<Instruction>(NewAI))
+      NewAI->takeName(AI);
+
+    replaceDbgDeclareForAlloca(AI, NewAI, DIB, /*Deref=*/true);
+    AI->replaceAllUsesWith(NewAI);
+    AI->eraseFromParent();
+  }
+
+  if (!DynamicAllocas.empty()) {
+    // Now go through the instructions again, replacing stacksave/stackrestore.
+    for (inst_iterator It = inst_begin(&F), Ie = inst_end(&F); It != Ie;) {
+      Instruction *I = &*(It++);
+      auto II = dyn_cast<IntrinsicInst>(I);
+      if (!II)
+        continue;
+
+      if (II->getIntrinsicID() == Intrinsic::stacksave) {
+        IRBuilder<> IRB(II);
+        Instruction *LI = IRB.CreateLoad(UnsafeStackPtr);
+        LI->takeName(II);
+        II->replaceAllUsesWith(LI);
+        II->eraseFromParent();
+      } else if (II->getIntrinsicID() == Intrinsic::stackrestore) {
+        IRBuilder<> IRB(II);
+        Instruction *SI = IRB.CreateStore(II->getArgOperand(0), UnsafeStackPtr);
+        SI->takeName(II);
+        assert(II->use_empty());
+        II->eraseFromParent();
+      }
+    }
+  }
+}
+
+bool SafeStack::runOnFunction(Function &F) {
+  auto AA = &getAnalysis<AliasAnalysis>();
+
+  DEBUG(dbgs() << "[SafeStack] Function: " << F.getName() << "\n");
+
+  if (!F.hasFnAttribute(Attribute::SafeStack)) {
+    DEBUG(dbgs() << "[SafeStack]     safestack is not requested"
+                    " for this function\n");
+    return false;
+  }
+
+  if (F.isDeclaration()) {
+    DEBUG(dbgs() << "[SafeStack]     function definition"
+                    " is not available\n");
+    return false;
+  }
+
+  {
+    // Make sure the regular stack protector won't run on this function
+    // (safestack attribute takes precedence).
+    AttrBuilder B;
+    B.addAttribute(Attribute::StackProtect)
+        .addAttribute(Attribute::StackProtectReq)
+        .addAttribute(Attribute::StackProtectStrong);
+    F.removeAttributes(
+        AttributeSet::FunctionIndex,
+        AttributeSet::get(F.getContext(), AttributeSet::FunctionIndex, B));
+  }
+
+  if (AA->onlyReadsMemory(&F)) {
+    // XXX: we don't protect against information leak attacks for now.
+    DEBUG(dbgs() << "[SafeStack]     function only reads memory\n");
+    return false;
+  }
+
+  ++NumFunctions;
+
+  SmallVector<AllocaInst *, 16> StaticAllocas;
+  SmallVector<AllocaInst *, 4> DynamicAllocas;
+  SmallVector<ReturnInst *, 4> Returns;
+
+  // Collect all points where stack gets unwound and needs to be restored
+  // This is only necessary because the runtime (setjmp and unwind code) is
+  // not aware of the unsafe stack and won't unwind/restore it prorerly.
+  // To work around this problem without changing the runtime, we insert
+  // instrumentation to restore the unsafe stack pointer when necessary.
+  SmallVector<Instruction *, 4> StackRestorePoints;
+
+  // Find all static and dynamic alloca instructions that must be moved to the
+  // unsafe stack, all return instructions and stack restore points.
+  findInsts(F, StaticAllocas, DynamicAllocas, Returns, StackRestorePoints);
+
+  if (StaticAllocas.empty() && DynamicAllocas.empty() &&
+      StackRestorePoints.empty())
+    return false; // Nothing to do in this function.
+
+  if (!StaticAllocas.empty() || !DynamicAllocas.empty())
+    ++NumUnsafeStackFunctions; // This function has the unsafe stack.
+
+  if (!StackRestorePoints.empty())
+    ++NumUnsafeStackRestorePointsFunctions;
+
+  if (!UnsafeStackPtr)
+    UnsafeStackPtr = getOrCreateUnsafeStackPtr(*F.getParent());
+
+  // The top of the unsafe stack after all unsafe static allocas are allocated.
+  Value *StaticTop = moveStaticAllocasToUnsafeStack(F, StaticAllocas, Returns);
+
+  // Safe stack object that stores the current unsafe stack top. It is updated
+  // as unsafe dynamic (non-constant-sized) allocas are allocated and freed.
+  // This is only needed if we need to restore stack pointer after longjmp
+  // or exceptions, and we have dynamic allocations.
+  // FIXME: a better alternative might be to store the unsafe stack pointer
+  // before setjmp / invoke instructions.
+  AllocaInst *DynamicTop = createStackRestorePoints(
+      F, StackRestorePoints, StaticTop, !DynamicAllocas.empty());
+
+  // Handle dynamic allocas.
+  moveDynamicAllocasToUnsafeStack(F, UnsafeStackPtr, DynamicTop,
+                                  DynamicAllocas);
+
+  DEBUG(dbgs() << "[SafeStack]     safestack applied\n");
+  return true;
+}
+
+} // end anonymous namespace
+
+char SafeStack::ID = 0;
+INITIALIZE_PASS_BEGIN(SafeStack, "safe-stack",
+                      "Safe Stack instrumentation pass", false, false)
+INITIALIZE_PASS_DEPENDENCY(TargetTransformInfoWrapperPass)
+INITIALIZE_PASS_END(SafeStack, "safe-stack", "Safe Stack instrumentation pass",
+                    false, false)
+
+FunctionPass *llvm::createSafeStackPass() { return new SafeStack(); }
diff --git a/lib/Transforms/Instrumentation/SanitizerCoverage.cpp b/lib/Transforms/Instrumentation/SanitizerCoverage.cpp
index 662513d508d..c91b89df830 100644
--- a/lib/Transforms/Instrumentation/SanitizerCoverage.cpp
+++ b/lib/Transforms/Instrumentation/SanitizerCoverage.cpp
@@ -11,19 +11,17 @@
 // and potentially with other Sanitizers.
 //
 // We create a Guard variable with the same linkage
-// as the function and inject this code into the entry block (CoverageLevel=1)
-// or all blocks (CoverageLevel>=2):
+// as the function and inject this code into the entry block (SCK_Function)
+// or all blocks (SCK_BB):
 // if (Guard < 0) {
 //    __sanitizer_cov(&Guard);
 // }
 // The accesses to Guard are atomic. The rest of the logic is
 // in __sanitizer_cov (it's fine to call it more than once).
 //
-// With CoverageLevel>=3 we also split critical edges this effectively
+// With SCK_Edge we also split critical edges this effectively
 // instrumenting all edges.
 //
-// CoverageLevel>=4 add indirect call profiling implented as a function call.
-//
 // This coverage implementation provides very limited data:
 // it only tells if a given function (block) was ever executed. No counters.
 // But for many use cases this is what we need and the added slowdown small.
@@ -35,6 +33,7 @@
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/IR/CallSite.h"
 #include "llvm/IR/DataLayout.h"
+#include "llvm/IR/DebugInfo.h"
 #include "llvm/IR/Function.h"
 #include "llvm/IR/IRBuilder.h"
 #include "llvm/IR/InlineAsm.h"
@@ -60,6 +59,7 @@ static const char *const kSanCovIndirCallName = "__sanitizer_cov_indir_call16";
 static const char *const kSanCovTraceEnter = "__sanitizer_cov_trace_func_enter";
 static const char *const kSanCovTraceBB = "__sanitizer_cov_trace_basic_block";
 static const char *const kSanCovTraceCmp = "__sanitizer_cov_trace_cmp";
+static const char *const kSanCovTraceSwitch = "__sanitizer_cov_trace_switch";
 static const char *const kSanCovModuleCtorName = "sancov.module_ctor";
 static const uint64_t    kSanCtorAndDtorPriority = 2;
 
@@ -99,11 +99,45 @@ static cl::opt<bool> ClUse8bitCounters("sanitizer-coverage-8bit-counters",
 
 namespace {
 
+SanitizerCoverageOptions getOptions(int LegacyCoverageLevel) {
+  SanitizerCoverageOptions Res;
+  switch (LegacyCoverageLevel) {
+  case 0:
+    Res.CoverageType = SanitizerCoverageOptions::SCK_None;
+    break;
+  case 1:
+    Res.CoverageType = SanitizerCoverageOptions::SCK_Function;
+    break;
+  case 2:
+    Res.CoverageType = SanitizerCoverageOptions::SCK_BB;
+    break;
+  case 3:
+    Res.CoverageType = SanitizerCoverageOptions::SCK_Edge;
+    break;
+  case 4:
+    Res.CoverageType = SanitizerCoverageOptions::SCK_Edge;
+    Res.IndirectCalls = true;
+    break;
+  }
+  return Res;
+}
+
+SanitizerCoverageOptions OverrideFromCL(SanitizerCoverageOptions Options) {
+  // Sets CoverageType and IndirectCalls.
+  SanitizerCoverageOptions CLOpts = getOptions(ClCoverageLevel);
+  Options.CoverageType = std::max(Options.CoverageType, CLOpts.CoverageType);
+  Options.IndirectCalls |= CLOpts.IndirectCalls;
+  Options.TraceBB |= ClExperimentalTracing;
+  Options.TraceCmp |= ClExperimentalCMPTracing;
+  Options.Use8bitCounters |= ClUse8bitCounters;
+  return Options;
+}
+
 class SanitizerCoverageModule : public ModulePass {
  public:
-   SanitizerCoverageModule(int CoverageLevel = 0)
-       : ModulePass(ID),
-         CoverageLevel(std::max(CoverageLevel, (int)ClCoverageLevel)) {}
+  SanitizerCoverageModule(
+      const SanitizerCoverageOptions &Options = SanitizerCoverageOptions())
+      : ModulePass(ID), Options(OverrideFromCL(Options)) {}
   bool runOnModule(Module &M) override;
   bool runOnFunction(Function &F);
   static char ID;  // Pass identification, replacement for typeid
@@ -115,8 +149,10 @@ class SanitizerCoverageModule : public ModulePass {
   void InjectCoverageForIndirectCalls(Function &F,
                                       ArrayRef<Instruction *> IndirCalls);
   void InjectTraceForCmp(Function &F, ArrayRef<Instruction *> CmpTraceTargets);
+  void InjectTraceForSwitch(Function &F,
+                            ArrayRef<Instruction *> SwitchTraceTargets);
   bool InjectCoverage(Function &F, ArrayRef<BasicBlock *> AllBlocks);
-  void SetNoSanitizeMetada(Instruction *I);
+  void SetNoSanitizeMetadata(Instruction *I);
   void InjectCoverageAtBlock(Function &F, BasicBlock &BB, bool UseCalls);
   unsigned NumberOfInstrumentedBlocks() {
     return SanCovFunction->getNumUses() + SanCovWithCheckFunction->getNumUses();
@@ -124,39 +160,37 @@ class SanitizerCoverageModule : public ModulePass {
   Function *SanCovFunction;
   Function *SanCovWithCheckFunction;
   Function *SanCovIndirCallFunction;
-  Function *SanCovModuleInit;
   Function *SanCovTraceEnter, *SanCovTraceBB;
   Function *SanCovTraceCmpFunction;
+  Function *SanCovTraceSwitchFunction;
   InlineAsm *EmptyAsm;
-  Type *IntptrTy, *Int64Ty;
+  Type *IntptrTy, *Int64Ty, *Int64PtrTy;
+  Module *CurModule;
   LLVMContext *C;
   const DataLayout *DL;
 
   GlobalVariable *GuardArray;
   GlobalVariable *EightBitCounterArray;
 
-  int CoverageLevel;
+  SanitizerCoverageOptions Options;
 };
 
 }  // namespace
 
 bool SanitizerCoverageModule::runOnModule(Module &M) {
-  if (!CoverageLevel) return false;
+  if (Options.CoverageType == SanitizerCoverageOptions::SCK_None)
+    return false;
   C = &(M.getContext());
   DL = &M.getDataLayout();
+  CurModule = &M;
   IntptrTy = Type::getIntNTy(*C, DL->getPointerSizeInBits());
   Type *VoidTy = Type::getVoidTy(*C);
   IRBuilder<> IRB(*C);
   Type *Int8PtrTy = PointerType::getUnqual(IRB.getInt8Ty());
   Type *Int32PtrTy = PointerType::getUnqual(IRB.getInt32Ty());
+  Int64PtrTy = PointerType::getUnqual(IRB.getInt64Ty());
   Int64Ty = IRB.getInt64Ty();
 
-  Function *CtorFunc =
-      Function::Create(FunctionType::get(VoidTy, false),
-                       GlobalValue::InternalLinkage, kSanCovModuleCtorName, &M);
-  ReturnInst::Create(*C, BasicBlock::Create(*C, "", CtorFunc));
-  appendToGlobalCtors(M, CtorFunc, kSanCtorAndDtorPriority);
-
   SanCovFunction = checkSanitizerInterfaceFunction(
       M.getOrInsertFunction(kSanCovName, VoidTy, Int32PtrTy, nullptr));
   SanCovWithCheckFunction = checkSanitizerInterfaceFunction(
@@ -167,17 +201,16 @@ bool SanitizerCoverageModule::runOnModule(Module &M) {
   SanCovTraceCmpFunction =
       checkSanitizerInterfaceFunction(M.getOrInsertFunction(
           kSanCovTraceCmp, VoidTy, Int64Ty, Int64Ty, Int64Ty, nullptr));
+  SanCovTraceSwitchFunction =
+      checkSanitizerInterfaceFunction(M.getOrInsertFunction(
+          kSanCovTraceSwitch, VoidTy, Int64Ty, Int64PtrTy, nullptr));
 
-  SanCovModuleInit = checkSanitizerInterfaceFunction(M.getOrInsertFunction(
-      kSanCovModuleInitName, Type::getVoidTy(*C), Int32PtrTy, IntptrTy,
-      Int8PtrTy, Int8PtrTy, nullptr));
-  SanCovModuleInit->setLinkage(Function::ExternalLinkage);
   // We insert an empty inline asm after cov callbacks to avoid callback merge.
   EmptyAsm = InlineAsm::get(FunctionType::get(IRB.getVoidTy(), false),
                             StringRef(""), StringRef(""),
                             /*hasSideEffects=*/true);
 
-  if (ClExperimentalTracing) {
+  if (Options.TraceBB) {
     SanCovTraceEnter = checkSanitizerInterfaceFunction(
         M.getOrInsertFunction(kSanCovTraceEnter, VoidTy, Int32PtrTy, nullptr));
     SanCovTraceBB = checkSanitizerInterfaceFunction(
@@ -192,7 +225,7 @@ bool SanitizerCoverageModule::runOnModule(Module &M) {
   GuardArray =
       new GlobalVariable(M, Int32Ty, false, GlobalValue::ExternalLinkage,
                          nullptr, "__sancov_gen_cov_tmp");
-  if (ClUse8bitCounters)
+  if (Options.Use8bitCounters)
     EightBitCounterArray =
         new GlobalVariable(M, Int8Ty, false, GlobalVariable::ExternalLinkage,
                            nullptr, "__sancov_gen_cov_tmp");
@@ -216,7 +249,7 @@ bool SanitizerCoverageModule::runOnModule(Module &M) {
   GuardArray->eraseFromParent();
 
   GlobalVariable *RealEightBitCounterArray;
-  if (ClUse8bitCounters) {
+  if (Options.Use8bitCounters) {
     // Make sure the array is 16-aligned.
     static const int kCounterAlignment = 16;
     Type *Int8ArrayNTy =
@@ -237,15 +270,19 @@ bool SanitizerCoverageModule::runOnModule(Module &M) {
       new GlobalVariable(M, ModNameStrConst->getType(), true,
                          GlobalValue::PrivateLinkage, ModNameStrConst);
 
-  // Call __sanitizer_cov_module_init
-  IRB.SetInsertPoint(CtorFunc->getEntryBlock().getTerminator());
-  IRB.CreateCall4(
-      SanCovModuleInit, IRB.CreatePointerCast(RealGuardArray, Int32PtrTy),
-      ConstantInt::get(IntptrTy, N),
-      ClUse8bitCounters
-          ? IRB.CreatePointerCast(RealEightBitCounterArray, Int8PtrTy)
-          : Constant::getNullValue(Int8PtrTy),
-      IRB.CreatePointerCast(ModuleName, Int8PtrTy));
+  Function *CtorFunc;
+  std::tie(CtorFunc, std::ignore) = createSanitizerCtorAndInitFunctions(
+      M, kSanCovModuleCtorName, kSanCovModuleInitName,
+      {Int32PtrTy, IntptrTy, Int8PtrTy, Int8PtrTy},
+      {IRB.CreatePointerCast(RealGuardArray, Int32PtrTy),
+       ConstantInt::get(IntptrTy, N),
+       Options.Use8bitCounters
+           ? IRB.CreatePointerCast(RealEightBitCounterArray, Int8PtrTy)
+           : Constant::getNullValue(Int8PtrTy),
+       IRB.CreatePointerCast(ModuleName, Int8PtrTy)});
+
+  appendToGlobalCtors(M, CtorFunc, kSanCtorAndDtorPriority);
+
   return true;
 }
 
@@ -253,42 +290,50 @@ bool SanitizerCoverageModule::runOnFunction(Function &F) {
   if (F.empty()) return false;
   if (F.getName().find(".module_ctor") != std::string::npos)
     return false;  // Should not instrument sanitizer init functions.
-  if (CoverageLevel >= 3)
+  if (Options.CoverageType >= SanitizerCoverageOptions::SCK_Edge)
     SplitAllCriticalEdges(F);
   SmallVector<Instruction*, 8> IndirCalls;
   SmallVector<BasicBlock*, 16> AllBlocks;
   SmallVector<Instruction*, 8> CmpTraceTargets;
+  SmallVector<Instruction*, 8> SwitchTraceTargets;
   for (auto &BB : F) {
     AllBlocks.push_back(&BB);
     for (auto &Inst : BB) {
-      if (CoverageLevel >= 4) {
+      if (Options.IndirectCalls) {
         CallSite CS(&Inst);
         if (CS && !CS.getCalledFunction())
           IndirCalls.push_back(&Inst);
       }
-      if (ClExperimentalCMPTracing)
+      if (Options.TraceCmp) {
         if (isa<ICmpInst>(&Inst))
           CmpTraceTargets.push_back(&Inst);
+        if (isa<SwitchInst>(&Inst))
+          SwitchTraceTargets.push_back(&Inst);
+      }
     }
   }
   InjectCoverage(F, AllBlocks);
   InjectCoverageForIndirectCalls(F, IndirCalls);
   InjectTraceForCmp(F, CmpTraceTargets);
+  InjectTraceForSwitch(F, SwitchTraceTargets);
   return true;
 }
 
 bool SanitizerCoverageModule::InjectCoverage(Function &F,
                                              ArrayRef<BasicBlock *> AllBlocks) {
-  if (!CoverageLevel) return false;
-
-  if (CoverageLevel == 1) {
+  switch (Options.CoverageType) {
+  case SanitizerCoverageOptions::SCK_None:
+    return false;
+  case SanitizerCoverageOptions::SCK_Function:
     InjectCoverageAtBlock(F, F.getEntryBlock(), false);
-  } else {
+    return true;
+  default: {
+    bool UseCalls = ClCoverageBlockThreshold < AllBlocks.size();
     for (auto BB : AllBlocks)
-      InjectCoverageAtBlock(F, *BB,
-                            ClCoverageBlockThreshold < AllBlocks.size());
+      InjectCoverageAtBlock(F, *BB, UseCalls);
+    return true;
+  }
   }
-  return true;
 }
 
 // On every indirect call we call a run-time function
@@ -313,15 +358,50 @@ void SanitizerCoverageModule::InjectCoverageForIndirectCalls(
         *F.getParent(), Ty, false, GlobalValue::PrivateLinkage,
         Constant::getNullValue(Ty), "__sancov_gen_callee_cache");
     CalleeCache->setAlignment(kCacheAlignment);
-    IRB.CreateCall2(SanCovIndirCallFunction,
-                    IRB.CreatePointerCast(Callee, IntptrTy),
-                    IRB.CreatePointerCast(CalleeCache, IntptrTy));
+    IRB.CreateCall(SanCovIndirCallFunction,
+                   {IRB.CreatePointerCast(Callee, IntptrTy),
+                    IRB.CreatePointerCast(CalleeCache, IntptrTy)});
   }
 }
 
+// For every switch statement we insert a call:
+// __sanitizer_cov_trace_switch(CondValue,
+//      {NumCases, ValueSizeInBits, Case0Value, Case1Value, Case2Value, ... })
+
+void SanitizerCoverageModule::InjectTraceForSwitch(
+    Function &F, ArrayRef<Instruction *> SwitchTraceTargets) {
+  for (auto I : SwitchTraceTargets) {
+    if (SwitchInst *SI = dyn_cast<SwitchInst>(I)) {
+      IRBuilder<> IRB(I);
+      SmallVector<Constant *, 16> Initializers;
+      Value *Cond = SI->getCondition();
+      Initializers.push_back(ConstantInt::get(Int64Ty, SI->getNumCases()));
+      Initializers.push_back(
+          ConstantInt::get(Int64Ty, Cond->getType()->getScalarSizeInBits()));
+      if (Cond->getType()->getScalarSizeInBits() <
+          Int64Ty->getScalarSizeInBits())
+        Cond = IRB.CreateIntCast(Cond, Int64Ty, false);
+      for (auto It: SI->cases()) {
+        Constant *C = It.getCaseValue();
+        if (C->getType()->getScalarSizeInBits() <
+            Int64Ty->getScalarSizeInBits())
+          C = ConstantExpr::getCast(CastInst::ZExt, It.getCaseValue(), Int64Ty);
+        Initializers.push_back(C);
+      }
+      ArrayType *ArrayOfInt64Ty = ArrayType::get(Int64Ty, Initializers.size());
+      GlobalVariable *GV = new GlobalVariable(
+          *CurModule, ArrayOfInt64Ty, false, GlobalVariable::InternalLinkage,
+          ConstantArray::get(ArrayOfInt64Ty, Initializers),
+          "__sancov_gen_cov_switch_values");
+      IRB.CreateCall(SanCovTraceSwitchFunction,
+                     {Cond, IRB.CreatePointerCast(GV, Int64PtrTy)});
+    }
+  }
+}
+
+
 void SanitizerCoverageModule::InjectTraceForCmp(
     Function &F, ArrayRef<Instruction *> CmpTraceTargets) {
-  if (!ClExperimentalCMPTracing) return;
   for (auto I : CmpTraceTargets) {
     if (ICmpInst *ICMP = dyn_cast<ICmpInst>(I)) {
       IRBuilder<> IRB(ICMP);
@@ -329,17 +409,17 @@ void SanitizerCoverageModule::InjectTraceForCmp(
       Value *A1 = ICMP->getOperand(1);
       if (!A0->getType()->isIntegerTy()) continue;
       uint64_t TypeSize = DL->getTypeStoreSizeInBits(A0->getType());
-      // __sanitizer_cov_indir_call((type_size << 32) | predicate, A0, A1);
-      IRB.CreateCall3(
+      // __sanitizer_cov_trace_cmp((type_size << 32) | predicate, A0, A1);
+      IRB.CreateCall(
           SanCovTraceCmpFunction,
-          ConstantInt::get(Int64Ty, (TypeSize << 32) | ICMP->getPredicate()),
-          IRB.CreateIntCast(A0, Int64Ty, true),
-          IRB.CreateIntCast(A1, Int64Ty, true));
+          {ConstantInt::get(Int64Ty, (TypeSize << 32) | ICMP->getPredicate()),
+           IRB.CreateIntCast(A0, Int64Ty, true),
+           IRB.CreateIntCast(A1, Int64Ty, true)});
     }
   }
 }
 
-void SanitizerCoverageModule::SetNoSanitizeMetada(Instruction *I) {
+void SanitizerCoverageModule::SetNoSanitizeMetadata(Instruction *I) {
   I->setMetadata(
       I->getParent()->getParent()->getParent()->getMDKindID("nosanitize"),
       MDNode::get(*C, None));
@@ -347,6 +427,13 @@ void SanitizerCoverageModule::SetNoSanitizeMetada(Instruction *I) {
 
 void SanitizerCoverageModule::InjectCoverageAtBlock(Function &F, BasicBlock &BB,
                                                     bool UseCalls) {
+  // Don't insert coverage for unreachable blocks: we will never call
+  // __sanitizer_cov() for them, so counting them in
+  // NumberOfInstrumentedBlocks() might complicate calculation of code coverage
+  // percentage. Also, unreachable instructions frequently have no debug
+  // locations.
+  if (isa<UnreachableInst>(BB.getTerminator()))
+    return;
   BasicBlock::iterator IP = BB.getFirstInsertionPt(), BE = BB.end();
   // Skip static allocas at the top of the entry block so they don't become
   // dynamic when we split the block.  If we used our optimized stack layout,
@@ -358,12 +445,16 @@ void SanitizerCoverageModule::InjectCoverageAtBlock(Function &F, BasicBlock &BB,
   }
 
   bool IsEntryBB = &BB == &F.getEntryBlock();
-  DebugLoc EntryLoc = IsEntryBB && IP->getDebugLoc()
-                          ? IP->getDebugLoc().getFnDebugLoc()
-                          : IP->getDebugLoc();
+  DebugLoc EntryLoc;
+  if (IsEntryBB) {
+    if (auto SP = getDISubprogram(&F))
+      EntryLoc = DebugLoc::get(SP->getScopeLine(), 0, SP);
+  } else {
+    EntryLoc = IP->getDebugLoc();
+  }
+
   IRBuilder<> IRB(IP);
   IRB.SetCurrentDebugLocation(EntryLoc);
-  SmallVector<Value *, 1> Indices;
   Value *GuardP = IRB.CreateAdd(
       IRB.CreatePointerCast(GuardArray, IntptrTy),
       ConstantInt::get(IntptrTy, (1 + NumberOfInstrumentedBlocks()) * 4));
@@ -375,7 +466,7 @@ void SanitizerCoverageModule::InjectCoverageAtBlock(Function &F, BasicBlock &BB,
     LoadInst *Load = IRB.CreateLoad(GuardP);
     Load->setAtomic(Monotonic);
     Load->setAlignment(4);
-    SetNoSanitizeMetada(Load);
+    SetNoSanitizeMetadata(Load);
     Value *Cmp = IRB.CreateICmpSGE(Constant::getNullValue(Load->getType()), Load);
     Instruction *Ins = SplitBlockAndInsertIfThen(
         Cmp, IP, false, MDBuilder(*C).createBranchWeights(1, 100000));
@@ -383,10 +474,10 @@ void SanitizerCoverageModule::InjectCoverageAtBlock(Function &F, BasicBlock &BB,
     IRB.SetCurrentDebugLocation(EntryLoc);
     // __sanitizer_cov gets the PC of the instruction using GET_CALLER_PC.
     IRB.CreateCall(SanCovFunction, GuardP);
-    IRB.CreateCall(EmptyAsm);  // Avoids callback merge.
+    IRB.CreateCall(EmptyAsm, {}); // Avoids callback merge.
   }
 
-  if(ClUse8bitCounters) {
+  if (Options.Use8bitCounters) {
     IRB.SetInsertPoint(IP);
     Value *P = IRB.CreateAdd(
         IRB.CreatePointerCast(EightBitCounterArray, IntptrTy),
@@ -395,11 +486,11 @@ void SanitizerCoverageModule::InjectCoverageAtBlock(Function &F, BasicBlock &BB,
     LoadInst *LI = IRB.CreateLoad(P);
     Value *Inc = IRB.CreateAdd(LI, ConstantInt::get(IRB.getInt8Ty(), 1));
     StoreInst *SI = IRB.CreateStore(Inc, P);
-    SetNoSanitizeMetada(LI);
-    SetNoSanitizeMetada(SI);
+    SetNoSanitizeMetadata(LI);
+    SetNoSanitizeMetadata(SI);
   }
 
-  if (ClExperimentalTracing) {
+  if (Options.TraceBB) {
     // Experimental support for tracing.
     // Insert a callback with the same guard variable as used for coverage.
     IRB.SetInsertPoint(IP);
@@ -411,6 +502,7 @@ char SanitizerCoverageModule::ID = 0;
 INITIALIZE_PASS(SanitizerCoverageModule, "sancov",
     "SanitizerCoverage: TODO."
     "ModulePass", false, false)
-ModulePass *llvm::createSanitizerCoverageModulePass(int CoverageLevel) {
-  return new SanitizerCoverageModule(CoverageLevel);
+ModulePass *llvm::createSanitizerCoverageModulePass(
+    const SanitizerCoverageOptions &Options) {
+  return new SanitizerCoverageModule(Options);
 }
diff --git a/lib/Transforms/Instrumentation/ThreadSanitizer.cpp b/lib/Transforms/Instrumentation/ThreadSanitizer.cpp
index aa8ee5aab7d..1a46bbb8612 100644
--- a/lib/Transforms/Instrumentation/ThreadSanitizer.cpp
+++ b/lib/Transforms/Instrumentation/ThreadSanitizer.cpp
@@ -72,6 +72,9 @@ STATISTIC(NumOmittedReadsFromConstantGlobals,
 STATISTIC(NumOmittedReadsFromVtable, "Number of vtable reads");
 STATISTIC(NumOmittedNonCaptured, "Number of accesses ignored due to capturing");
 
+static const char *const kTsanModuleCtorName = "tsan.module_ctor";
+static const char *const kTsanInitName = "__tsan_init";
+
 namespace {
 
 /// ThreadSanitizer: instrument the code in module to find races.
@@ -113,6 +116,7 @@ struct ThreadSanitizer : public FunctionPass {
   Function *TsanVptrUpdate;
   Function *TsanVptrLoad;
   Function *MemmoveFn, *MemcpyFn, *MemsetFn;
+  Function *TsanCtorFunction;
 };
 }  // namespace
 
@@ -225,13 +229,12 @@ void ThreadSanitizer::initializeCallbacks(Module &M) {
 
 bool ThreadSanitizer::doInitialization(Module &M) {
   const DataLayout &DL = M.getDataLayout();
+  IntptrTy = DL.getIntPtrType(M.getContext());
+  std::tie(TsanCtorFunction, std::ignore) = createSanitizerCtorAndInitFunctions(
+      M, kTsanModuleCtorName, kTsanInitName, /*InitArgTypes=*/{},
+      /*InitArgs=*/{});
 
-  // Always insert a call to __tsan_init into the module's CTORs.
-  IRBuilder<> IRB(M.getContext());
-  IntptrTy = IRB.getIntPtrTy(DL);
-  Value *TsanInit = M.getOrInsertFunction("__tsan_init",
-                                          IRB.getVoidTy(), nullptr);
-  appendToGlobalCtors(M, cast<Function>(TsanInit), 0);
+  appendToGlobalCtors(M, TsanCtorFunction, 0);
 
   return true;
 }
@@ -329,6 +332,10 @@ static bool isAtomic(Instruction *I) {
 }
 
 bool ThreadSanitizer::runOnFunction(Function &F) {
+  // This is required to prevent instrumenting call to __tsan_init from within
+  // the module constructor.
+  if (&F == TsanCtorFunction)
+    return false;
   initializeCallbacks(*F.getParent());
   SmallVector<Instruction*, 8> RetVec;
   SmallVector<Instruction*, 8> AllLoadsAndStores;
@@ -391,7 +398,7 @@ bool ThreadSanitizer::runOnFunction(Function &F) {
     IRB.CreateCall(TsanFuncEntry, ReturnAddress);
     for (auto RetInst : RetVec) {
       IRBuilder<> IRBRet(RetInst);
-      IRBRet.CreateCall(TsanFuncExit);
+      IRBRet.CreateCall(TsanFuncExit, {});
     }
     Res = true;
   }
@@ -420,9 +427,9 @@ bool ThreadSanitizer::instrumentLoadOrStore(Instruction *I,
     if (StoredValue->getType()->isIntegerTy())
       StoredValue = IRB.CreateIntToPtr(StoredValue, IRB.getInt8PtrTy());
     // Call TsanVptrUpdate.
-    IRB.CreateCall2(TsanVptrUpdate,
-                    IRB.CreatePointerCast(Addr, IRB.getInt8PtrTy()),
-                    IRB.CreatePointerCast(StoredValue, IRB.getInt8PtrTy()));
+    IRB.CreateCall(TsanVptrUpdate,
+                   {IRB.CreatePointerCast(Addr, IRB.getInt8PtrTy()),
+                    IRB.CreatePointerCast(StoredValue, IRB.getInt8PtrTy())});
     NumInstrumentedVtableWrites++;
     return true;
   }
@@ -474,16 +481,18 @@ static ConstantInt *createOrdering(IRBuilder<> *IRB, AtomicOrdering ord) {
 bool ThreadSanitizer::instrumentMemIntrinsic(Instruction *I) {
   IRBuilder<> IRB(I);
   if (MemSetInst *M = dyn_cast<MemSetInst>(I)) {
-    IRB.CreateCall3(MemsetFn,
-      IRB.CreatePointerCast(M->getArgOperand(0), IRB.getInt8PtrTy()),
-      IRB.CreateIntCast(M->getArgOperand(1), IRB.getInt32Ty(), false),
-      IRB.CreateIntCast(M->getArgOperand(2), IntptrTy, false));
+    IRB.CreateCall(
+        MemsetFn,
+        {IRB.CreatePointerCast(M->getArgOperand(0), IRB.getInt8PtrTy()),
+         IRB.CreateIntCast(M->getArgOperand(1), IRB.getInt32Ty(), false),
+         IRB.CreateIntCast(M->getArgOperand(2), IntptrTy, false)});
     I->eraseFromParent();
   } else if (MemTransferInst *M = dyn_cast<MemTransferInst>(I)) {
-    IRB.CreateCall3(isa<MemCpyInst>(M) ? MemcpyFn : MemmoveFn,
-      IRB.CreatePointerCast(M->getArgOperand(0), IRB.getInt8PtrTy()),
-      IRB.CreatePointerCast(M->getArgOperand(1), IRB.getInt8PtrTy()),
-      IRB.CreateIntCast(M->getArgOperand(2), IntptrTy, false));
+    IRB.CreateCall(
+        isa<MemCpyInst>(M) ? MemcpyFn : MemmoveFn,
+        {IRB.CreatePointerCast(M->getArgOperand(0), IRB.getInt8PtrTy()),
+         IRB.CreatePointerCast(M->getArgOperand(1), IRB.getInt8PtrTy()),
+         IRB.CreateIntCast(M->getArgOperand(2), IntptrTy, false)});
     I->eraseFromParent();
   }
   return false;
diff --git a/lib/Transforms/LLVMBuild.txt b/lib/Transforms/LLVMBuild.txt
index d60f6c61f1b..3594de54a25 100644
--- a/lib/Transforms/LLVMBuild.txt
+++ b/lib/Transforms/LLVMBuild.txt
@@ -16,7 +16,7 @@
 ;===------------------------------------------------------------------------===;
 
 [common]
-subdirectories = IPO InstCombine Instrumentation Scalar Utils Vectorize ObjCARC NaCl MinSFI
+subdirectories = IPO InstCombine Instrumentation Scalar Utils Vectorize ObjCARC NaCl
 
 [component_0]
 type = Group
diff --git a/lib/Transforms/Makefile b/lib/Transforms/Makefile
index b5ec33cd516..b4bb7cba723 100644
--- a/lib/Transforms/Makefile
+++ b/lib/Transforms/Makefile
@@ -8,14 +8,7 @@
 ##===----------------------------------------------------------------------===##
 
 LEVEL = ../..
-PARALLEL_DIRS = Utils Instrumentation Scalar InstCombine IPO Vectorize Hello ObjCARC NaCl MinSFI
-
-
-# @LOCALMOD-BEGIN
-ifeq ($(PNACL_BROWSER_TRANSLATOR),1)
-  PARALLEL_DIRS := $(filter-out Hello, $(PARALLEL_DIRS))
-endif
-# @LOCALMOD-END
+PARALLEL_DIRS = Utils Instrumentation Scalar InstCombine IPO Vectorize Hello ObjCARC NaCl
 
 include $(LEVEL)/Makefile.config
 
diff --git a/lib/Transforms/MinSFI/AllocateDataSegment.cpp b/lib/Transforms/MinSFI/AllocateDataSegment.cpp
deleted file mode 100644
index 8212df0c390..00000000000
--- a/lib/Transforms/MinSFI/AllocateDataSegment.cpp
+++ /dev/null
@@ -1,134 +0,0 @@
-//===- AllocateDataSegment.cpp - Create a template for the data segment ---===//
-//
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
-//
-//===----------------------------------------------------------------------===//
-//
-// Code sandboxed with MinSFI cannot access the memory containing its data
-// segment directly because it is located outside its address subspace. To
-// this end, this pass collates all of the global variables in the module
-// into an exported global struct named "__sfi_data_segment" and a corresponding
-// global integer holding the overall size. The runtime is expected to link
-// against these variables and to initialize the memory region of the sandbox
-// by copying the data segment template into a fixed address inside the region.
-//
-// This pass assumes that the base of the memory region of the sandbox is
-// aligned to at least 2^29 bytes (=512MB), which is the maximum global variable
-// alignment supported by LLVM.
-//
-//===----------------------------------------------------------------------===//
-
-#include "llvm/Pass.h"
-#include "llvm/IR/Constants.h"
-#include "llvm/IR/DataLayout.h"
-#include "llvm/IR/Module.h"
-#include "llvm/Transforms/MinSFI.h"
-
-using namespace llvm;
-
-static const char ExternalSymName_DataSegment[] = "__sfi_data_segment";
-static const char ExternalSymName_DataSegmentSize[] = "__sfi_data_segment_size";
-
-static const uint32_t DataSegmentBaseAddress = 0x10000;
-
-namespace {
-class AllocateDataSegment : public ModulePass {
- public:
-  static char ID;
-  AllocateDataSegment() : ModulePass(ID) {
-    initializeAllocateDataSegmentPass(*PassRegistry::getPassRegistry());
-  }
-
-  virtual bool runOnModule(Module &M);
-};
-}  // namespace
-
-static inline uint32_t getPadding(uint32_t Offset, const GlobalVariable *GV,
-                                  const DataLayout &DL) {
-  uint32_t Alignment = DL.getPreferredAlignment(GV);
-  if (Alignment <= 1)
-    return 0;
-  else
-    return OffsetToAlignment(Offset, Alignment);
-}
-
-bool AllocateDataSegment::runOnModule(Module &M) {
-  DataLayout DL(&M);
-  Type *I8 = Type::getInt8Ty(M.getContext());
-  Type *I32 = Type::getInt32Ty(M.getContext());
-  Type *IntPtrType = DL.getIntPtrType(M.getContext());
-
-  // First, we do a pass over the global variables, in which we compute
-  // the amount of required padding between them and consequently their
-  // addresses relative to the memory base of the sandbox. References to each
-  // global are then replaced with direct memory pointers.
-  uint32_t VarOffset = 0;
-  DenseMap<GlobalVariable*, uint32_t> VarPadding;
-  for (Module::global_iterator GV = M.global_begin(), E = M.global_end();
-       GV != E; ++GV) {
-    assert(GV->hasInitializer());
-
-    uint32_t Padding = getPadding(VarOffset, GV, DL);
-    VarPadding[GV] = Padding;
-    VarOffset += Padding;
-
-    GV->replaceAllUsesWith(
-        ConstantExpr::getIntToPtr(
-            ConstantInt::get(IntPtrType,
-                             DataSegmentBaseAddress + VarOffset),
-            GV->getType()));
-
-    VarOffset += DL.getTypeStoreSize(GV->getType()->getPointerElementType());
-  }
-
-  // Using the offsets computed above, we prepare the layout and the contents
-  // of the desired data structure. After the type and initializer of each
-  // global is copied, the global is not needed any more and it is erased.
-  SmallVector<Type*, 10> TemplateLayout;
-  SmallVector<Constant*, 10> TemplateData;
-  for (Module::global_iterator It = M.global_begin(), E = M.global_end();
-       It != E; ) {
-    GlobalVariable *GV = It++;
-
-    uint32_t Padding = VarPadding[GV];
-    if (Padding > 0) {
-      Type *PaddingType = ArrayType::get(I8, Padding);
-      TemplateLayout.push_back(PaddingType);
-      TemplateData.push_back(ConstantAggregateZero::get(PaddingType));
-    }
-
-    TemplateLayout.push_back(GV->getType()->getPointerElementType());
-    TemplateData.push_back(GV->getInitializer());
-
-    GV->eraseFromParent();
-  }
-
-  // Finally, we create the struct and size global variables.
-  StructType *TemplateType =
-      StructType::create(M.getContext(), ExternalSymName_DataSegment);
-  TemplateType->setBody(TemplateLayout, /*isPacked=*/true);
-
-  Constant *Template = ConstantStruct::get(TemplateType, TemplateData);
-  new GlobalVariable(M, Template->getType(), /*isConstant=*/true,
-                     GlobalVariable::ExternalLinkage, Template,
-                     ExternalSymName_DataSegment);
-
-  Constant *TemplateSize =
-      ConstantInt::get(I32, DL.getTypeAllocSize(TemplateType));
-  new GlobalVariable(M, TemplateSize->getType(), /*isConstant=*/true,
-                     GlobalVariable::ExternalLinkage, TemplateSize,
-                     ExternalSymName_DataSegmentSize);
-
-  return true;
-}
-
-char AllocateDataSegment::ID = 0;
-INITIALIZE_PASS(AllocateDataSegment, "minsfi-allocate-data-segment",
-                "Create a template for the data segment", false, false)
-
-ModulePass *llvm::createAllocateDataSegmentPass() {
-  return new AllocateDataSegment();
-}
diff --git a/lib/Transforms/MinSFI/CMakeLists.txt b/lib/Transforms/MinSFI/CMakeLists.txt
deleted file mode 100644
index 5e78776c5be..00000000000
--- a/lib/Transforms/MinSFI/CMakeLists.txt
+++ /dev/null
@@ -1,13 +0,0 @@
-add_llvm_library(LLVMMinSFITransforms
-  AllocateDataSegment.cpp
-  ExpandAllocas.cpp
-  MinSFI.cpp
-  RenameEntryPoint.cpp
-  SandboxIndirectCalls.cpp
-  SandboxMemoryAccesses.cpp
-  StripTls.cpp
-  SubstituteUndefs.cpp
-  Utils.cpp
-  )
-
-add_dependencies(LLVMMinSFITransforms LLVMNaClTransforms)
diff --git a/lib/Transforms/MinSFI/ExpandAllocas.cpp b/lib/Transforms/MinSFI/ExpandAllocas.cpp
deleted file mode 100644
index 817e0c29cf1..00000000000
--- a/lib/Transforms/MinSFI/ExpandAllocas.cpp
+++ /dev/null
@@ -1,254 +0,0 @@
-//===----- ExpandAllocas.cpp - Allocate memory on the untrusted stack -----===//
-//
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
-//
-//===----------------------------------------------------------------------===//
-//
-// Code sandboxed with MinSFI cannot access the execution stack directly
-// because the stack lies outside of its address subspace, which prevents it
-// from using memory allocated with the alloca instruction. This pass therefore
-// replaces allocas with memory allocation on a separate stack at a fixed
-// location inside the designated memory region.
-//
-// The new stack does not have to be trusted as it is only used for memory
-// allocation inside the sandbox. The call and ret instructions still operate
-// on the native stack, preventing manipulation with the return address or
-// callee-saved registers.
-//
-// This pass also replaces the @llvm.stacksave and @llvm.stackrestore
-// intrinsics which would otherwise allow access to the native stack pointer.
-// Instead, they are expanded out and save/restore the current untrusted stack
-// pointer.
-//
-// When a function is invoked, the current untrusted stack pointer is obtained
-// from the "__sfi_stack_ptr" global variable (internal to the module). The
-// function then keeps track of the current value of the stack pointer, but
-// must update the global variable prior to any function calls and restore the
-// initial value before it returns.
-//
-// The stack pointer is initialized in the entry function of the module, the
-// _start_minsfi function. The runtime is expected to copy the arguments
-// (a NULL-terminated integer array) at the end of the allocated memory region,
-// i.e. at the bottom of the untrusted stack, and pass the pointer to the array
-// to the entry function. The sandboxed code is then expected to use the
-// pointer not only to access its arguments but also as the initial value of
-// its stack pointer and to grow the stack backwards.
-//
-// If an alloca requests alignment greater than 1, the untrusted stack pointer
-// is aligned accordingly. However, the alignment is applied before the address
-// is sandboxed and therefore the runtime must guarantee that the base address
-// of the sandbox is aligned to at least 2^29 bytes (=512MB), which is the
-// maximum alignment supported by LLVM.
-//
-// Possible optimizations:
-//  - accumulate constant-sized allocas to reduce the number of stores
-//    into the global stack pointer variable
-//  - remove stores into the global pointer if the respective values never
-//    reach a function call
-//  - align frame to 16 bytes
-//
-//===----------------------------------------------------------------------===//
-
-#include "llvm/Pass.h"
-#include "llvm/IR/Constants.h"
-#include "llvm/IR/DataLayout.h"
-#include "llvm/IR/Function.h"
-#include "llvm/IR/Instructions.h"
-#include "llvm/IR/IntrinsicInst.h"
-#include "llvm/IR/Module.h"
-#include "llvm/Support/MathExtras.h"
-#include "llvm/Transforms/MinSFI.h"
-#include "llvm/Transforms/NaCl.h"
-
-using namespace llvm;
-
-static const char InternalSymName_StackPointer[] = "__sfi_stack_ptr";
-
-namespace {
-// ExpandAllocas needs to be a ModulePass because it adds a GlobalVariable.
-class ExpandAllocas : public ModulePass {
-  GlobalVariable *StackPtrVar;
-  Type *IntPtrType, *I8Ptr;
-
-  void runOnFunction(Function &Func);
-  void insertStackPtrInit(Module &M);
-
-public:
-  static char ID;
-  ExpandAllocas() : ModulePass(ID), StackPtrVar(NULL), IntPtrType(NULL),
-                    I8Ptr(NULL) {
-    initializeExpandAllocasPass(*PassRegistry::getPassRegistry());
-  }
-
-  virtual bool runOnModule(Module &M);
-};
-}  // namespace
-
-bool ExpandAllocas::runOnModule(Module &M) {
-  DataLayout DL(&M);
-  IntPtrType = DL.getIntPtrType(M.getContext());
-  I8Ptr = Type::getInt8PtrTy(M.getContext());
-
-  // Create the stack pointer global variable. We are forced to give it some
-  // initial value, but it will be initialized at runtime.
-  StackPtrVar = new GlobalVariable(M, IntPtrType, /*isConstant=*/false,
-                                   GlobalVariable::InternalLinkage,
-                                   ConstantInt::get(IntPtrType, 0),
-                                   InternalSymName_StackPointer);
-
-  for (Module::iterator Func = M.begin(), E = M.end(); Func != E; ++Func)
-    runOnFunction(*Func);
-
-  insertStackPtrInit(M);
-
-  return true;
-}
-
-static inline void replaceWithPointer(Instruction *OrigInst, Value *IntPtr,
-                                      SmallVectorImpl<Instruction*> &Dead) {
-  Instruction *NewInst =
-      new IntToPtrInst(IntPtr, OrigInst->getType(), "", OrigInst);
-  NewInst->takeName(OrigInst);
-  OrigInst->replaceAllUsesWith(NewInst);
-  CopyDebug(NewInst, OrigInst);
-  Dead.push_back(OrigInst);
-}
-
-static inline Instruction *getBBStackPtr(BasicBlock *BB) {
-  return BB->getInstList().begin();
-}
-
-void ExpandAllocas::runOnFunction(Function &Func) {
-  // Do an initial scan of the entire function body. Check whether it contains
-  // instructions which we want to operate on the untrusted stack and return
-  // if there aren't any. Also check whether it contains any function calls.
-  // If not, we will not have to update the global stack pointer variable.
-  bool NoUntrustedStackOps = true;
-  bool MustUpdateStackPtrGlobal = false;
-  for (Function::iterator BB = Func.begin(), E = Func.end(); BB != E; ++BB) {
-    for (BasicBlock::iterator Inst = BB->begin(), E = BB->end(); Inst != E;
-         ++Inst) {
-      NoUntrustedStackOps &= !isa<AllocaInst>(Inst);
-      if (CallInst *Call = dyn_cast<CallInst>(Inst)) {
-        if (isa<IntrinsicInst>(Call)) {
-          unsigned IntrinsicID = Call->getCalledFunction()->getIntrinsicID();
-          bool IsNotStackIntr = IntrinsicID != Intrinsic::stacksave &&
-                                IntrinsicID != Intrinsic::stackrestore;
-          NoUntrustedStackOps &= IsNotStackIntr;
-          MustUpdateStackPtrGlobal |= IsNotStackIntr;
-        } else {
-          MustUpdateStackPtrGlobal = true;
-        }
-      }
-    }
-  }
-
-  if (NoUntrustedStackOps)
-    return;
-
-  SmallVector<Instruction *, 10> DeadInsts;
-  Instruction *InitialStackPtr = new LoadInst(StackPtrVar, "frame_top");
-
-  // First, we insert a new instruction at the beginning of each basic block,
-  // which will represent the value of the stack pointer at that point. For
-  // the entry block, this is the value of the global stack pointer variable.
-  // Other blocks are initialized with empty phi nodes which we will later
-  // fill with the values carried over from the respective predecessors.
-  BasicBlock *EntryBB = &Func.getEntryBlock();
-  for (Function::iterator BB = Func.begin(), E = Func.end(); BB != E; ++BB)
-    BB->getInstList().push_front(
-        ((BasicBlock*)BB == EntryBB) ? InitialStackPtr
-                                     : PHINode::Create(IntPtrType, 2, ""));
-
-  // Now iterate over the instructions and expand out the untrusted stack
-  // operations. Allocas are replaced with pointer arithmetic that pushes
-  // the untrusted stack pointer and updates the global stack pointer variable
-  // if the initial scan identified function calls in the code.
-  // The @llvm.stacksave intrinsic returns the latest value of the stack
-  // pointer, and the @llvm.stackrestore overwrites it and potentially updates
-  // the global variable. If needed, return instructions are prepended with
-  // a store which restores the initial value of the global variable.
-  // At the end of each basic block, the last value of the untrusted stack
-  // pointer is inserted into the phi node at the beginning of each successor
-  // block.
-  for (Function::iterator BB = Func.begin(), EBB = Func.end(); BB != EBB;
-       ++BB) {
-    Instruction *LastTop = getBBStackPtr(BB);
-    for (BasicBlock::iterator Inst = BB->begin(), EInst = BB->end(); 
-         Inst != EInst; ++Inst) {
-      if (AllocaInst *Alloca = dyn_cast<AllocaInst>(Inst)) {
-        Value *SizeOp = Alloca->getArraySize();
-        unsigned Alignment = Alloca->getAlignment();
-        assert(Alloca->getType() == I8Ptr);
-        assert(SizeOp->getType()->isIntegerTy(32));
-        assert(Alignment <= (1 << 29));  // 512MB
-
-        LastTop = BinaryOperator::CreateSub(LastTop, SizeOp, "", Alloca);
-        if (Alignment > 1)
-          LastTop = BinaryOperator::CreateAnd(LastTop,
-                                              ConstantInt::get(IntPtrType,
-                                                               -Alignment),
-                                              "", Alloca);
-        if (MustUpdateStackPtrGlobal)
-          new StoreInst(LastTop, StackPtrVar, Alloca);
-        replaceWithPointer(Alloca, LastTop, DeadInsts);
-      } else if (IntrinsicInst *Intr = dyn_cast<IntrinsicInst>(Inst)) {
-        if (Intr->getIntrinsicID() == Intrinsic::stacksave) {
-          replaceWithPointer(Intr, LastTop, DeadInsts);
-        } else if (Intr->getIntrinsicID() == Intrinsic::stackrestore) {
-          Value *NewStackPtr = Intr->getArgOperand(0);
-          LastTop = new PtrToIntInst(NewStackPtr, IntPtrType, "", Intr);
-          if (MustUpdateStackPtrGlobal)
-            new StoreInst(LastTop, StackPtrVar, Intr);
-          CopyDebug(LastTop, Intr);
-          DeadInsts.push_back(Intr);
-        }
-      } else if (ReturnInst *Return = dyn_cast<ReturnInst>(Inst)) {
-        if (MustUpdateStackPtrGlobal)
-          new StoreInst(InitialStackPtr, StackPtrVar, Return);
-      }
-    }
-
-    // Insert the final frame top value into all successor phi nodes.
-    TerminatorInst *Term = BB->getTerminator();
-    for (unsigned int I = 0; I < Term->getNumSuccessors(); ++I) {
-      PHINode *Succ = cast<PHINode>(getBBStackPtr(Term->getSuccessor(I)));
-      Succ->addIncoming(LastTop, BB);
-    }
-  }
-
-  // Delete the replaced instructions.
-  for (SmallVectorImpl<Instruction *>::const_iterator Inst = DeadInsts.begin(),
-       E = DeadInsts.end(); Inst != E; ++Inst)
-    (*Inst)->eraseFromParent();
-}
-
-void ExpandAllocas::insertStackPtrInit(Module &M) {
-  Function *EntryFunction = M.getFunction(minsfi::EntryFunctionName);
-  if (!EntryFunction)
-    report_fatal_error("ExpandAllocas: Module does not have an entry function");
-
-  // Check the signature of the entry function.
-  Function::ArgumentListType &Args = EntryFunction->getArgumentList();
-  if (Args.size() != 1 || Args.front().getType() != IntPtrType) {
-    report_fatal_error(std::string("ExpandAllocas: Invalid signature of ") +
-                       minsfi::EntryFunctionName);
-  }
-
-  // Insert a store instruction which saves the value of the first argument
-  // to the stack pointer global variable.
-  new StoreInst(&Args.front(), StackPtrVar,
-                EntryFunction->getEntryBlock().getFirstInsertionPt());
-}
-
-char ExpandAllocas::ID = 0;
-INITIALIZE_PASS(ExpandAllocas, "minsfi-expand-allocas",
-                "Expand allocas to allocate memory on an untrusted stack",
-                false, false)
-
-ModulePass *llvm::createExpandAllocasPass() {
-  return new ExpandAllocas();
-}
diff --git a/lib/Transforms/MinSFI/LLVMBuild.txt b/lib/Transforms/MinSFI/LLVMBuild.txt
deleted file mode 100644
index ad15eb1c518..00000000000
--- a/lib/Transforms/MinSFI/LLVMBuild.txt
+++ /dev/null
@@ -1,23 +0,0 @@
-;===- ./lib/Transforms/MinSFI/LLVMBuild.txt --------------------*- Conf -*--===;
-;
-;                     The LLVM Compiler Infrastructure
-;
-; This file is distributed under the University of Illinois Open Source
-; License. See LICENSE.TXT for details.
-;
-;===------------------------------------------------------------------------===;
-;
-; This is an LLVMBuild description file for the components in this subdirectory.
-;
-; For more information on the LLVMBuild system, please see:
-;
-;   http://llvm.org/docs/LLVMBuild.html
-;
-;===------------------------------------------------------------------------===;
-
-[component_0]
-type = Library
-name = MinSFITransforms
-parent = Transforms
-library_name = MinSFITransforms
-required_libraries = Core Support IPO NaClAnalysis NaClTransforms
diff --git a/lib/Transforms/MinSFI/Makefile b/lib/Transforms/MinSFI/Makefile
deleted file mode 100644
index 6edbbb135d9..00000000000
--- a/lib/Transforms/MinSFI/Makefile
+++ /dev/null
@@ -1,15 +0,0 @@
-##===- lib/Transforms/MinSFI/Makefile ----------------------*- Makefile -*-===##
-#
-#                     The LLVM Compiler Infrastructure
-#
-# This file is distributed under the University of Illinois Open Source
-# License. See LICENSE.TXT for details.
-#
-##===----------------------------------------------------------------------===##
-
-LEVEL = ../../..
-LIBRARYNAME = LLVMMinSFITransforms
-BUILD_ARCHIVE = 1
-
-include $(LEVEL)/Makefile.common
-
diff --git a/lib/Transforms/MinSFI/MinSFI.cpp b/lib/Transforms/MinSFI/MinSFI.cpp
deleted file mode 100644
index 66e70c22197..00000000000
--- a/lib/Transforms/MinSFI/MinSFI.cpp
+++ /dev/null
@@ -1,68 +0,0 @@
-//===-- MinSFI.cpp - Lists MinSFI sandboxing passes -----------------------===//
-//
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
-//
-//===----------------------------------------------------------------------===//
-//
-// This file implements the meta-pass "-minsfi". It lists its constituent
-// passes and explains the reasons for their ordering.
-//
-//===----------------------------------------------------------------------===//
-
-#include "llvm/IR/LegacyPassManager.h"
-#include "llvm/Analysis/NaCl.h"
-#include "llvm/Transforms/MinSFI.h"
-
-using namespace llvm;
-
-void llvm::MinSFIPasses(PassManagerBase &PM) {
-  // Nondeterminism is generally undesirable in sandboxed code but more
-  // importantly, use of undefined values can leak protected data. This pass
-  // replaces all undefs with predefined constants. It only modifies operands
-  // of instructions and therefore is not dependent on any other MinSFI or
-  // PNaCl passes.
-  PM.add(createSubstituteUndefsPass());
-
-  // Most MinSFI passes rely on the safety properties guaranteed by the PNaCl
-  // bitcode format. We run the PNaCl ABI verifier to make sure these hold.
-  PNaClABIErrorReporter *ErrorReporter = new PNaClABIErrorReporter();
-  PM.add(createPNaClABIVerifyModulePass(ErrorReporter, false));
-  PM.add(createPNaClABIVerifyFunctionsPass(ErrorReporter));
-
-  // The naming of NaCl's entry point causes a conflict when linking into
-  // native executables. This pass renames the entry function to resolve it.
-  // The pass must be invoked after the PNaCl ABI verifier but otherwise could
-  // be invoked at any point. To avoid confusion, we rename the function
-  // immediately after the verifier and have all the subsequent passes refer to
-  // the new name.
-  PM.add(createRenameEntryPointPass());
-
-  // Sandboxed code cannot access memory allocated on the native stack. This
-  // pass creates an untrusted stack inside the sandbox's memory region, with
-  // the stack pointer stored in a global variable. With some modifications,
-  // the pass could be invoked after SFI, allowing unsandboxed updates of the
-  // stack pointer, but that would increase the size of the compiler-side TCB.
-  PM.add(createExpandAllocasPass());
-
-  // The data segment of the sandbox lies outside its memory region. This pass
-  // generates a template, which the MinSFI runtime copies into the sandbox
-  // during initialization. All globals defined before this pass therefore
-  // remain addressable by the sandboxed code.
-  PM.add(createAllocateDataSegmentPass());
-
-  // Next, we apply SFI sandboxing to pointer-type operands of all memory
-  // access instructions. The pass guarantees that the sandboxed code cannot
-  // read or write beyond the scope of the memory region allocated to it.
-  // All passes running before this one do not have to be trusted in this
-  // respect. Passes running later must not break the guarantee.
-  PM.add(createSandboxMemoryAccessesPass());
-
-  // Lastly, we apply CFI sandboxing on indirect calls. The pass creates
-  // tables of address-taken functions and replaces function pointers with
-  // indices into the tables. This pass is invoked after SFI because it is
-  // crucial that the tables cannot be modified by the sandboxed code.
-  PM.add(createSandboxIndirectCallsPass());
-}
diff --git a/lib/Transforms/MinSFI/RenameEntryPoint.cpp b/lib/Transforms/MinSFI/RenameEntryPoint.cpp
deleted file mode 100644
index fc78b1f3d73..00000000000
--- a/lib/Transforms/MinSFI/RenameEntryPoint.cpp
+++ /dev/null
@@ -1,64 +0,0 @@
-//===- RenameEntryPoint.cpp - Rename _start to avoid linking collisions ---===//
-//
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
-//
-//===----------------------------------------------------------------------===//
-//
-// MinSFI compiles PNaCl bitcode into a native object file and links it into
-// a standard C program. However, both C and PNaCl name their entry points
-// '_start' which causes a linking collision. This pass therefore renames the
-// entry function of the MinSFI module to '_start_minsfi'. By changing the name
-// in the bitcode, we also avoid relying on objcopy.
-//
-//===----------------------------------------------------------------------===//
-
-#include "llvm/Pass.h"
-#include "llvm/IR/Function.h"
-#include "llvm/IR/Module.h"
-#include "llvm/Transforms/MinSFI.h"
-
-using namespace llvm;
-
-static const char PNaClEntryPointName[] = "_start";
-const char minsfi::EntryFunctionName[] = "_start_minsfi";
-
-namespace {
-class RenameEntryPoint : public ModulePass {
- public:
-  static char ID;
-  RenameEntryPoint() : ModulePass(ID) {
-    initializeRenameEntryPointPass(*PassRegistry::getPassRegistry());
-  }
-
-  virtual bool runOnModule(Module &M);
-};
-}  // namespace
-
-bool RenameEntryPoint::runOnModule(Module &M) {
-  if (M.getNamedValue(minsfi::EntryFunctionName)) {
-    report_fatal_error(std::string("RenameEntryPoint: The module already "
-                       "contains a value named '") +
-                       minsfi::EntryFunctionName + "'");
-  }
-
-  Function *EntryFunc = M.getFunction(PNaClEntryPointName);
-  if (!EntryFunc) {
-    report_fatal_error(std::string("RenameEntryPoint: The module does not "
-                       "contain a function named '") +
-                       PNaClEntryPointName + "'");
-  }
-
-  EntryFunc->setName(minsfi::EntryFunctionName);
-  return true;
-}
-
-char RenameEntryPoint::ID = 0;
-INITIALIZE_PASS(RenameEntryPoint, "minsfi-rename-entry-point",
-                "Rename _start to avoid linking collisions", false, false)
-
-ModulePass *llvm::createRenameEntryPointPass() {
-  return new RenameEntryPoint();
-}
diff --git a/lib/Transforms/MinSFI/SandboxIndirectCalls.cpp b/lib/Transforms/MinSFI/SandboxIndirectCalls.cpp
deleted file mode 100644
index 319129a1f77..00000000000
--- a/lib/Transforms/MinSFI/SandboxIndirectCalls.cpp
+++ /dev/null
@@ -1,209 +0,0 @@
-//===- SandboxIndirectCalls.cpp - Apply CFI to indirect function calls ----===//
-//
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
-//
-//===----------------------------------------------------------------------===//
-//
-// This is a pass which applies basic control-flow integrity enforcement to
-// indirect function calls as a mitigation technique against attempts to
-// subvert code execution.
-//
-// Pointers to address-taken functions are placed into global function tables
-// (one function table is created per signature) and pointers to functions are
-// replaced with the respective table indices. Indirect function calls are
-// rewritten to treat the target pointer as an index and to load the actual
-// pointer from the corresponding table.
-//
-// The zero-index entry of each table is set to null to provide consistent
-// behaviour for null pointers. Tables are also padded with null entries to
-// round their size to the nearest power of two and indices passed to calls
-// are bit-masked accordingly in order to prevent buffer overflow during the
-// load from the table.
-//
-// Even if placed into different tables, two functions are never assigned the
-// same index. Interpreting a function pointer as a function of an incorrect
-// signature will therefore result in jumping to null.
-//
-// Pointer arithmetic is not allowed on function pointers and will result in
-// undefined behaviour.
-//
-//===----------------------------------------------------------------------===//
-
-#include "llvm/Pass.h"
-#include "llvm/IR/Constants.h"
-#include "llvm/IR/DataLayout.h"
-#include "llvm/IR/Function.h"
-#include "llvm/IR/Module.h"
-#include "llvm/IR/Instructions.h"
-#include "llvm/IR/Intrinsics.h"
-#include "llvm/Support/MathExtras.h"
-#include "llvm/Transforms/MinSFI.h"
-#include "llvm/Transforms/NaCl.h"
-
-using namespace llvm;
-
-static const char InternalSymName_FunctionTable[] = "__sfi_function_table";
-
-namespace {
-// This pass needs to be a ModulePass because it adds a GlobalVariable.
-class SandboxIndirectCalls : public ModulePass {
- public:
-  static char ID;
-  SandboxIndirectCalls() : ModulePass(ID) {
-    initializeSandboxIndirectCallsPass(*PassRegistry::getPassRegistry());
-  }
-
-  virtual bool runOnModule(Module &M);
-};
-}  // namespace
-
-static inline size_t RoundToPowerOf2(size_t n) {
-  if (isPowerOf2_64(n))
-    return n;
-  else
-    return NextPowerOf2(n);
-}
-
-static inline bool IsPtrToIntUse(const Use &FuncUse) {
-  if (isa<PtrToIntInst>(FuncUse.getUser()))
-    return true;
-  else if (auto *Expr = dyn_cast<ConstantExpr>(FuncUse.getUser()))
-    return Expr->getOpcode() == Instruction::PtrToInt;
-  else
-    return false;
-}
-
-// Function use is a direct call if the user is a call instruction and
-// the function is its last operand.
-static inline bool IsDirectCallUse(const Use &FuncUse) {
-  if (auto *Call = dyn_cast<CallInst>(FuncUse.getUser())) {
-    return FuncUse.getOperandNo() == Call->getNumOperands() - 1;
-  }
-  return false;
-}
-
-bool SandboxIndirectCalls::runOnModule(Module &M) {
-  typedef SmallVector<Constant*, 16> FunctionVector;
-  DataLayout DL(&M);
-  Type *I32 = Type::getInt32Ty(M.getContext());
-  Type *IntPtrType = DL.getIntPtrType(M.getContext());
-
-  // First, we find all address-taken functions and assign each an index.
-  // Pointers in code are then immediately replaced with these indices, even
-  // though the tables have not been created yet.
-  FunctionVector AddrTakenFuncs;
-  for (Module::iterator Func = M.begin(), E = M.end(); Func != E; ++Func) {
-    bool HasIndirectUse = false;
-    Constant *Index = ConstantInt::get(IntPtrType, AddrTakenFuncs.size() + 1);
-    for (auto &Use : Func->uses()) {
-      if (IsPtrToIntUse(Use)) {
-        HasIndirectUse = true;
-        Use.getUser()->replaceAllUsesWith(Index);
-        if (auto *UserInst = dyn_cast<Instruction>(Use.getUser()))
-          UserInst->eraseFromParent();
-      } else if (!IsDirectCallUse(Use)) {
-        report_fatal_error("SandboxIndirectCalls: Invalid reference to "
-                           "function @" + Func->getName());
-      }
-    }
-
-    if (HasIndirectUse)
-      AddrTakenFuncs.push_back(Func);
-  }
-
-  // Return if no address-taken functions have been found.
-  if (AddrTakenFuncs.empty())
-    return false;
-
-  // Generate and fill out the function tables. Their size is rounded up to the
-  // nearest power of two, index zero is reserved for null and functions are
-  // stored under the indices that were assigned to them earlier.
-  size_t FuncIndex = 1;
-  size_t TableSize = RoundToPowerOf2(AddrTakenFuncs.size() + 1);
-  DenseMap<PointerType*, FunctionVector> TableEntries;
-  for (FunctionVector::iterator It = AddrTakenFuncs.begin(),
-       E = AddrTakenFuncs.end(); It != E; ++It) {
-    Constant *Func = (*It);
-    PointerType *FuncType = cast<PointerType>(Func->getType());
-    FunctionVector &Table = TableEntries[FuncType];
-
-    // If this table has not been initialized yet, fill it with nulls.
-    if (Table.empty())
-      Table.assign(TableSize, ConstantPointerNull::get(FuncType));
-
-    Table[FuncIndex++] = Func;
-  }
-
-  // Create a global variable for each of the function tables.
-  DenseMap<PointerType*, GlobalVariable*> TableGlobals;
-  for (DenseMap<PointerType*, FunctionVector>::iterator
-       Iter = TableEntries.begin(), E = TableEntries.end(); Iter != E; ++Iter) {
-    PointerType *FuncType = Iter->first;
-    FunctionVector &Table = Iter->second;
-
-    Constant *TableArray =
-        ConstantArray::get(ArrayType::get(FuncType, TableSize), Table);
-    TableGlobals[FuncType] =
-        new GlobalVariable(M, TableArray->getType(), /*isConstant=*/true,
-                           GlobalVariable::InternalLinkage, TableArray,
-                           InternalSymName_FunctionTable);
-  }
-
-  // Iterate over all call instructions and replace integers casted to function
-  // pointers with a load from the corresponding function table (because now
-  // the integers are not pointers but indices).
-  Constant *IndexMask = ConstantInt::get(IntPtrType, TableSize - 1);
-  for (Module::iterator Func = M.begin(), EFunc = M.end();
-       Func != EFunc; ++Func) {
-    for (Function::iterator BB = Func->begin(), EBB = Func->end();
-         BB != EBB; ++BB) {
-      for (BasicBlock::iterator Inst = BB->begin(), EInst = BB->end();
-           Inst != EInst; ++Inst) {
-        if (CallInst *Call = dyn_cast<CallInst>(Inst)) {
-          Value *Callee = Call->getCalledValue();
-          if (IntToPtrInst *Cast = dyn_cast<IntToPtrInst>(Callee)) {
-            Value *FuncIndex = Cast->getOperand(0);
-            PointerType *FuncType = cast<PointerType>(Cast->getType());
-            Constant *GlobalVar = TableGlobals[FuncType];
-
-            Value *FuncPtr;
-            if (GlobalVar) {
-              Instruction *MaskedIndex =
-                  BinaryOperator::CreateAnd(FuncIndex, IndexMask, "", Call);
-              Value *Indexes[] = { ConstantInt::get(I32, 0), MaskedIndex };
-              Instruction *TableElemPtr = GetElementPtrInst::Create(
-                  cast<PointerType>(GlobalVar->getType())->getElementType(),
-                  GlobalVar, Indexes, "", Call);
-              FuncPtr = CopyDebug(new LoadInst(TableElemPtr, "", Call), Cast);
-            } else {
-              // There is no function table for this signature, i.e. the module
-              // does not contain a function which could be called at this site.
-              // We replace the pointer with a null and put a trap in front of
-              // the call because it should never be called.
-              CallInst::Create(Intrinsic::getDeclaration(&M, Intrinsic::trap),
-                               "", Call);
-              FuncPtr = ConstantPointerNull::get(FuncType);
-            }
-
-            Call->setCalledFunction(FuncPtr);
-            if (Cast->use_empty())
-              Cast->eraseFromParent();
-          }
-        }
-      }
-    }
-  }
-
-  return true;
-}
-
-char SandboxIndirectCalls::ID = 0;
-INITIALIZE_PASS(SandboxIndirectCalls, "minsfi-sandbox-indirect-calls",
-                "Add CFI to indirect calls", false, false)
-
-ModulePass *llvm::createSandboxIndirectCallsPass() {
-  return new SandboxIndirectCalls();
-}
diff --git a/lib/Transforms/MinSFI/SandboxMemoryAccesses.cpp b/lib/Transforms/MinSFI/SandboxMemoryAccesses.cpp
deleted file mode 100644
index 3da07c7da53..00000000000
--- a/lib/Transforms/MinSFI/SandboxMemoryAccesses.cpp
+++ /dev/null
@@ -1,300 +0,0 @@
-//===- SandboxMemoryAccesses.cpp - Apply SFI sandboxing to used pointers --===//
-//
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
-//
-//===----------------------------------------------------------------------===//
-//
-// This pass applies SFI sandboxing to all memory access instructions in the IR.
-// Pointers are truncated to a given number of bits and shifted into a memory
-// region allocated by the runtime. The runtime reads the pointer bit size
-// from the "__sfi_pointer_size" exported constant and stores the base of the
-// correspondingly-sized memory region into the "__sfi_memory_base" global
-// variable.
-//
-// This is meant to be the next to last pass of MinSFI, followed only by a CFI
-// pass. Because there is no runtime verifier, it must be trusted to correctly
-// sandbox all dereferenced pointers.
-//
-// Sandboxed instructions:
-//  - load, store
-//  - memcpy, memmove, memset
-//  - @llvm.nacl.atomic.load.*
-//  - @llvm.nacl.atomic.store.*
-//  - @llvm.nacl.atomic.rmw.*
-//  - @llvm.nacl.atomic.cmpxchg.*
-//
-// Whitelisted instructions:
-//  - ptrtoint
-//  - bitcast
-//
-// This pass fails if code contains instructions with pointer-type operands
-// not listed above. PtrToInt and BitCast instructions are whitelisted because
-// they do not access memory and therefore do not need to be sandboxed.
-//
-// The pass recognizes the pointer arithmetic produced by ExpandGetElementPtr
-// and reuses its final integer value to save target instructions. This
-// optimization, as well as the memcpy, memmove and memset intrinsics, is safe
-// only if the runtime creates a guard region after the dedicated memory region.
-// The guard region must be the same size as the memory region.
-//
-// Both 32-bit and 64-bit architectures are supported. The necessary pointer
-// arithmetic generated by the pass always uses 64-bit integers. However, when
-// compiling for 32-bit targets, the backend is expected to optimize the code
-// by deducing that the top bits are always truncated during the final cast to
-// a pointer.
-//
-// The size of the runtime address subspace can be changed with the
-// "-minsfi-ptrsize" command-line option. Depending on the target architecture,
-// the value of this constant can have an effect on the efficiency of the
-// generated code. On x86-64 and AArch64, 32-bit subspace is the most efficient
-// because pointers can be sandboxed without bit masking. On AArch32, subspaces
-// of 24-31 bits will be more efficient because the bit mask fits into a single
-// BIC instruction immediate. Code for x86 and MIPS is the same for all values.
-//
-//===----------------------------------------------------------------------===//
-
-#include "llvm/Pass.h"
-#include "llvm/IR/DataLayout.h"
-#include "llvm/IR/Function.h"
-#include "llvm/IR/Instructions.h"
-#include "llvm/IR/IntrinsicInst.h"
-#include "llvm/IR/Module.h"
-#include "llvm/IR/NaClAtomicIntrinsics.h"
-#include "llvm/Transforms/MinSFI.h"
-#include "llvm/Transforms/NaCl.h"
-
-using namespace llvm;
-
-static const char ExternalSymName_MemoryBase[] = "__sfi_memory_base";
-static const char ExternalSymName_PointerSize[] = "__sfi_pointer_size";
-
-namespace {
-// This pass needs to be a ModulePass because it adds a GlobalVariable.
-class SandboxMemoryAccesses : public ModulePass {
-  Value *MemBaseVar;
-  Value *PtrMask;
-  DataLayout *DL;
-  Type *I32;
-  Type *I64;
-
-  void sandboxPtrOperand(Instruction *Inst, unsigned int OpNum,
-                         bool IsFirstClassValueAccess, Function &Func,
-                         Value **MemBase);
-  void sandboxLenOperand(Instruction *Inst, unsigned int OpNum);
-  void checkDoesNotHavePointerOperands(Instruction *Inst);
-  void runOnFunction(Function &Func);
-
- public:
-  static char ID;
-  SandboxMemoryAccesses() : ModulePass(ID), MemBaseVar(NULL), PtrMask(NULL),
-                            DL(NULL), I32(NULL), I64(NULL) {
-    initializeSandboxMemoryAccessesPass(*PassRegistry::getPassRegistry());
-  }
-
-  virtual bool runOnModule(Module &M);
-};
-}  // namespace
-
-bool SandboxMemoryAccesses::runOnModule(Module &M) {
-  DataLayout Layout(&M);
-  DL = &Layout;
-  I32 = Type::getInt32Ty(M.getContext());
-  I64 = Type::getInt64Ty(M.getContext());
-
-  // Create a global variable with external linkage that will hold the base
-  // address of the sandbox. This variable is defined and initialized by
-  // the runtime. We assume that all original global variables have been 
-  // removed during the AllocateDataSegment pass.
-  MemBaseVar = M.getOrInsertGlobal(ExternalSymName_MemoryBase, I64);
-
-  // Create an exported global constant holding the size of the sandboxed
-  // pointers. If it is smaller than 32 bits, prepare the corresponding bit mask
-  // will later be applied on pointer and length arguments of instructions.
-  unsigned int PointerSize = minsfi::GetPointerSizeInBits();
-  new GlobalVariable(M, I32, /*isConstant=*/true,
-                     GlobalVariable::ExternalLinkage,
-                     ConstantInt::get(I32, PointerSize),
-                     ExternalSymName_PointerSize);
-  if (PointerSize < 32)
-    PtrMask = ConstantInt::get(I32, (1U << PointerSize) - 1);
-
-  for (Module::iterator Func = M.begin(), E = M.end(); Func != E; ++Func)
-    runOnFunction(*Func);
-
-  return true;
-}
-
-void SandboxMemoryAccesses::sandboxPtrOperand(Instruction *Inst,
-                                              unsigned int OpNum,
-                                              bool IsFirstClassValueAccess,
-                                              Function &Func, Value **MemBase) {
-  // Function must first acquire the sandbox memory region base from
-  // the global variable. If this is the first sandboxed pointer, insert
-  // the corresponding load instruction at the beginning of the function.
-  if (!*MemBase) {
-    Instruction *MemBaseInst = new LoadInst(MemBaseVar, "mem_base");
-    Func.getEntryBlock().getInstList().push_front(MemBaseInst);
-    *MemBase = MemBaseInst;
-  }
-
-  Value *Ptr = Inst->getOperand(OpNum);
-  Value *Truncated = NULL, *OffsetConst = NULL;
-
-  // The ExpandGetElementPtr pass replaces the getelementptr instruction
-  // with pointer arithmetic. If we recognize that pointer arithmetic pattern
-  // here, we can sandbox the pointer more efficiently than in the general
-  // case below.
-  //
-  // The recognized pattern is:
-  //   %0 = add i32 %x, <const>               ; treated as signed, must be >= 0
-  //   %ptr = inttoptr i32 %0 to <type>*
-  // and can be replaced with:
-  //   %0 = zext i32 %x to i64
-  //   %1 = add i64 %0, %mem_base
-  //   %2 = add i64 %1, <const>               ; extended to i64
-  //   %ptr = inttoptr i64 %2 to <type>*
-  //
-  // Since this enables the code to access memory outside the dedicated region,
-  // this is safe only if the memory region is followed by an equally sized
-  // guard region.
-
-  bool OptimizeGEP = false;
-  Instruction *RedundantCast = NULL, *RedundantAdd = NULL;
-  if (IsFirstClassValueAccess) {
-    if (IntToPtrInst *Cast = dyn_cast<IntToPtrInst>(Ptr)) {
-      if (BinaryOperator *Op = dyn_cast<BinaryOperator>(Cast->getOperand(0))) {
-        if (Op->getOpcode() == Instruction::Add) {
-          if (Op->getType()->isIntegerTy(32)) {
-            if (ConstantInt *CI = dyn_cast<ConstantInt>(Op->getOperand(1))) {
-              Type *ValType = Ptr->getType()->getPointerElementType();
-              int64_t MaxOffset = minsfi::GetAddressSubspaceSize() -
-                                  DL->getTypeStoreSize(ValType);
-              int64_t Offset = CI->getSExtValue();
-              if ((Offset >= 0) && (Offset <= MaxOffset)) {
-                Truncated = Op->getOperand(0);
-                OffsetConst = ConstantInt::get(I64, Offset);
-                RedundantCast = Cast;
-                RedundantAdd = Op;
-                OptimizeGEP = true;
-              }
-            }
-          }
-        }
-      }
-    }
-  }
-
-  // If the pattern above has not been recognized, start by truncating
-  // the pointer to i32.
-  if (!OptimizeGEP)
-    Truncated = new PtrToIntInst(Ptr, I32, "", Inst);
-
-  // If the address subspace is smaller than 32 bits, truncate the pointer
-  // further with a bit mask.
-  if (PtrMask)
-    Truncated = BinaryOperator::CreateAnd(Truncated, PtrMask, "", Inst);
-
-  // Sandbox the pointer by zero-extending it back to 64 bits, and adding
-  // the memory region base.
-  Instruction *Extend = new ZExtInst(Truncated, I64, "", Inst);
-  Instruction *AddBase = BinaryOperator::CreateAdd(*MemBase, Extend, "", Inst);
-  Instruction *AddOffset =
-      OptimizeGEP ? BinaryOperator::CreateAdd(AddBase, OffsetConst, "", Inst)
-                  : AddBase;
-  Instruction *SandboxedPtr =
-      new IntToPtrInst(AddOffset, Ptr->getType(), "", Inst);
-
-  // Replace the pointer in the sandboxed operand
-  Inst->setOperand(OpNum, SandboxedPtr);
-
-  if (OptimizeGEP) {
-    // Copy debug information
-    CopyDebug(AddOffset, RedundantAdd);
-    CopyDebug(SandboxedPtr, RedundantCast);
-
-    // Remove instructions if now dead (order matters)
-    if (RedundantCast->use_empty())
-      RedundantCast->eraseFromParent();
-    if (RedundantAdd->use_empty())
-      RedundantAdd->eraseFromParent();
-  }
-}
-
-void SandboxMemoryAccesses::sandboxLenOperand(Instruction *Inst,
-                                              unsigned int OpNum) {
-  // Length is assumed to be an i32 value. If the address subspace is smaller,
-  // truncate the value with a bit mask.
-  if (PtrMask) {
-    Value *Len = Inst->getOperand(OpNum);
-    Instruction *MaskedLen = BinaryOperator::CreateAnd(Len, PtrMask, "", Inst);
-    Inst->setOperand(OpNum, MaskedLen);
-  }
-}
-
-void SandboxMemoryAccesses::checkDoesNotHavePointerOperands(Instruction *Inst) {
-  bool hasPointerOperand = false;
-
-  // Handle Call instructions separately because they always contain
-  // a pointer to the target function. Integrity of calls is guaranteed by CFI.
-  // This pass therefore only checks the function's arguments.
-  if (CallInst *Call = dyn_cast<CallInst>(Inst)) {
-    for (unsigned int I = 0, E = Call->getNumArgOperands(); I < E; ++I)
-      hasPointerOperand |= Call->getArgOperand(I)->getType()->isPointerTy();
-  } else {
-    for (unsigned int I = 0, E = Inst->getNumOperands(); I < E; ++I)
-      hasPointerOperand |= Inst->getOperand(I)->getType()->isPointerTy();
-  }
-
-  if (hasPointerOperand)
-    report_fatal_error("SandboxMemoryAccesses: unexpected instruction with "
-                       "pointer-type operands");
-}
-
-void SandboxMemoryAccesses::runOnFunction(Function &Func) {
-  Value *MemBase = NULL;
-
-  for (Function::iterator BB = Func.begin(), E = Func.end(); BB != E; ++BB) {
-    for (BasicBlock::iterator Inst = BB->begin(), E = BB->end(); Inst != E; 
-         ++Inst) {
-      if (isa<LoadInst>(Inst)) {
-        sandboxPtrOperand(Inst, 0, true, Func, &MemBase);
-      } else if (isa<StoreInst>(Inst)) {
-        sandboxPtrOperand(Inst, 1, true, Func, &MemBase);
-      } else if (isa<MemCpyInst>(Inst) || isa<MemMoveInst>(Inst)) {
-        sandboxPtrOperand(Inst, 0, false, Func, &MemBase);
-        sandboxPtrOperand(Inst, 1, false, Func, &MemBase);
-        sandboxLenOperand(Inst, 2);
-      } else if (isa<MemSetInst>(Inst)) {
-        sandboxPtrOperand(Inst, 0, false, Func, &MemBase);
-        sandboxLenOperand(Inst, 2);
-      } else if (IntrinsicInst *IntrCall = dyn_cast<IntrinsicInst>(Inst)) {
-        switch (IntrCall->getIntrinsicID()) {
-        case Intrinsic::nacl_atomic_load:
-        case Intrinsic::nacl_atomic_cmpxchg:
-          sandboxPtrOperand(IntrCall, 0, true, Func, &MemBase);
-          break;
-        case Intrinsic::nacl_atomic_store:
-        case Intrinsic::nacl_atomic_rmw:
-        case Intrinsic::nacl_atomic_is_lock_free:
-          sandboxPtrOperand(IntrCall, 1, true, Func, &MemBase);
-          break;
-        default:
-          checkDoesNotHavePointerOperands(IntrCall);
-        }
-      } else if (!isa<PtrToIntInst>(Inst) && !isa<BitCastInst>(Inst)) {
-        checkDoesNotHavePointerOperands(Inst);
-      }
-    }
-  }
-}
-
-char SandboxMemoryAccesses::ID = 0;
-INITIALIZE_PASS(SandboxMemoryAccesses, "minsfi-sandbox-memory-accesses",
-                "Add SFI sandboxing to memory accesses", false, false)
-
-ModulePass *llvm::createSandboxMemoryAccessesPass() {
-  return new SandboxMemoryAccesses();
-}
diff --git a/lib/Transforms/MinSFI/StripTls.cpp b/lib/Transforms/MinSFI/StripTls.cpp
deleted file mode 100644
index f15ae0ab30a..00000000000
--- a/lib/Transforms/MinSFI/StripTls.cpp
+++ /dev/null
@@ -1,51 +0,0 @@
-//===- StripTls.cpp - Remove the thread_local attribute from variables ----===//
-//
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
-//
-//===----------------------------------------------------------------------===//
-//
-// Runtime support for thread-local storage depends on pthreads which are
-// currently not supported by MinSFI. This pass removes the thread_local
-// attribute from all global variables until thread support is in place.
-//
-// The pass should be invoked before the pnacl-abi-simplify passes.
-//
-//===----------------------------------------------------------------------===//
-
-#include "llvm/Pass.h"
-#include "llvm/IR/GlobalVariable.h"
-#include "llvm/IR/Module.h"
-
-using namespace llvm;
-
-namespace {
-class StripTls : public ModulePass {
- public:
-  static char ID;
-  StripTls() : ModulePass(ID) {
-    initializeStripTlsPass(*PassRegistry::getPassRegistry());
-  }
-
-  virtual bool runOnModule(Module &M);
-};
-}  // namespace
-
-bool StripTls::runOnModule(Module &M) {
-  bool Changed = false;
-  for (Module::global_iterator GV = M.global_begin(), E = M.global_end();
-       GV != E; ++GV) {
-    if (GV->isThreadLocal()) {
-      GV->setThreadLocal(false);
-      Changed = true;
-    }
-  }
-  return Changed;
-}
-
-char StripTls::ID = 0;
-INITIALIZE_PASS(StripTls, "minsfi-strip-tls",
-                "Remove the thread_local attribute from variables",
-                false, false)
diff --git a/lib/Transforms/MinSFI/SubstituteUndefs.cpp b/lib/Transforms/MinSFI/SubstituteUndefs.cpp
deleted file mode 100644
index 352193bc09b..00000000000
--- a/lib/Transforms/MinSFI/SubstituteUndefs.cpp
+++ /dev/null
@@ -1,96 +0,0 @@
-//===- SubstituteUndefs.cpp - Replace undefs with deterministic constants -===//
-//
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
-//
-//===----------------------------------------------------------------------===//
-//
-// PNaCl bitcode may contain undefined values inside function bodies, i.e. as
-// a placeholder for numerical constants and constant vectors. Their actual
-// value at runtime will most likely be the current value from one of the
-// registers or from the native stack.
-//
-// Using undefined values, the sandboxed code could obtain protected values,
-// such as the base address of the address subspace or a value from another
-// protection domain left in the register file. Additionally, undefined values
-// may introduce undesirable non-determinism.
-//
-// This pass therefore substitutes all undefined expressions with predefined
-// constants.
-//
-//===----------------------------------------------------------------------===//
-
-#include "llvm/Pass.h"
-#include "llvm/IR/Constants.h"
-#include "llvm/IR/Function.h"
-#include "llvm/Transforms/MinSFI.h"
-
-using namespace llvm;
-
-static const uint64_t SubstInt = 0xBAADF00DCAFEBABE;
-static const double SubstFloat = 3.14159265359;
-
-namespace {
-class SubstituteUndefs : public FunctionPass {
- public:
-  static char ID;
-  SubstituteUndefs() : FunctionPass(ID) {
-    initializeSubstituteUndefsPass(*PassRegistry::getPassRegistry());
-  }
-
-  virtual bool runOnFunction(Function &Func);
-};
-}  // namespace
-
-static inline bool isScalarOrVectorInteger(Type *T) {
-  if (T->isIntegerTy())
-    return true;
-  else if (T->isVectorTy() && T->getVectorElementType()->isIntegerTy())
-    return true;
-  else
-    return false;
-}
-
-static inline bool isScalarOrVectorFloatingPoint(Type *T) {
-  if (T->isFloatingPointTy())
-    return true;
-  else if (T->isVectorTy() && T->getVectorElementType()->isFloatingPointTy())
-    return true;
-  else
-    return false;
-}
-
-bool SubstituteUndefs::runOnFunction(Function &Func) {
-  bool HadUndefs = false;
-  for (Function::iterator BB = Func.begin(), E = Func.end(); BB != E; ++BB) {
-    for (BasicBlock::iterator Inst = BB->begin(), E = BB->end(); Inst != E;
-         ++Inst) {
-      for (int Index = 0, NumOps = Inst->getNumOperands(); Index < NumOps;
-           ++Index) {
-        Value *Operand = Inst->getOperand(Index);
-        if (isa<UndefValue>(Operand)) {
-          HadUndefs = true;
-          Type *OpType = Operand->getType();
-          if (isScalarOrVectorInteger(OpType))
-            Inst->setOperand(Index, ConstantInt::get(OpType, SubstInt));
-          else if (isScalarOrVectorFloatingPoint(OpType))
-            Inst->setOperand(Index, ConstantFP::get(OpType, SubstFloat));
-          else
-            assert(false && "Type of undef not permitted by the PNaCl ABI");
-        }
-      }
-    }
-  }
-  return HadUndefs;
-}
-
-char SubstituteUndefs::ID = 0;
-INITIALIZE_PASS(SubstituteUndefs, "minsfi-substitute-undefs",
-                "Replace undef values with deterministic constants",
-                false, false)
-
-FunctionPass *llvm::createSubstituteUndefsPass() {
-  return new SubstituteUndefs();
-}
diff --git a/lib/Transforms/MinSFI/Utils.cpp b/lib/Transforms/MinSFI/Utils.cpp
deleted file mode 100644
index a1f02e2c05f..00000000000
--- a/lib/Transforms/MinSFI/Utils.cpp
+++ /dev/null
@@ -1,28 +0,0 @@
-//===-- Utils.cpp - Helper functions for MinSFI passes --------------------===//
-//
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
-//
-//===----------------------------------------------------------------------===//
-
-#include "llvm/Support/CommandLine.h"
-#include "llvm/Transforms/MinSFI.h"
-
-using namespace llvm;
-
-static cl::opt<uint32_t>
-PointerSizeInBits("minsfi-ptrsize", cl::init(32),
-                  cl::desc("Size of the address subspace in bits"));
-
-uint32_t minsfi::GetPointerSizeInBits() {
-  if (PointerSizeInBits < 20 || PointerSizeInBits > 32)
-    report_fatal_error("MinSFI: Size of the sandboxed pointers is out of "
-                       "bounds (20-32)");
-  return PointerSizeInBits;
-}
-
-uint64_t minsfi::GetAddressSubspaceSize() {
-  return 1LL << GetPointerSizeInBits();
-}
diff --git a/lib/Transforms/NaCl/ExpandVarArgs.cpp b/lib/Transforms/NaCl/ExpandVarArgs.cpp
index 6e2824684d8..02f84f9b867 100644
--- a/lib/Transforms/NaCl/ExpandVarArgs.cpp
+++ b/lib/Transforms/NaCl/ExpandVarArgs.cpp
@@ -228,7 +228,7 @@ static bool ExpandVarArgCall(Module *M, InstType *Call, DataLayout *DL) {
   auto *BufPtr = IRB.CreateBitCast(Buf, I8Ptr, "vararg_lifetime_bitcast");
   auto *BufSize =
       ConstantInt::get(Ctx, APInt(64, DL->getTypeAllocSize(VarArgsTy)));
-  IRB.CreateCall2(LifetimeStart, BufSize, BufPtr);
+  IRB.CreateCall(LifetimeStart, {BufSize, BufPtr});
 
   // Copy variable arguments into buffer.
   int Index = 0;
@@ -264,15 +264,15 @@ static bool ExpandVarArgCall(Module *M, InstType *Call, DataLayout *DL) {
     auto *N = IRB.CreateCall(CastFunc, FixedArgs);
     N->setAttributes(AttributeSet::get(Ctx, Attrs));
     NewCall = N;
-    IRB.CreateCall2(LifetimeEnd, BufSize, BufPtr);
+    IRB.CreateCall(LifetimeEnd, {BufSize, BufPtr});
   } else if (auto *C = dyn_cast<InvokeInst>(Call)) {
     auto *N = IRB.CreateInvoke(CastFunc, C->getNormalDest(), C->getUnwindDest(),
                                FixedArgs, C->getName());
     N->setAttributes(AttributeSet::get(Ctx, Attrs));
     (IRBuilder<>(C->getNormalDest()->getFirstInsertionPt()))
-        .CreateCall2(LifetimeEnd, BufSize, BufPtr);
+        .CreateCall(LifetimeEnd, {BufSize, BufPtr});
     (IRBuilder<>(C->getUnwindDest()->getFirstInsertionPt()))
-        .CreateCall2(LifetimeEnd, BufSize, BufPtr);
+        .CreateCall(LifetimeEnd, {BufSize, BufPtr});
     NewCall = N;
   } else {
     llvm_unreachable("not a call/invoke");
diff --git a/lib/Transforms/NaCl/LowerEmExceptionsPass.cpp b/lib/Transforms/NaCl/LowerEmExceptionsPass.cpp
index 676a0a68050..46e418f63c4 100644
--- a/lib/Transforms/NaCl/LowerEmExceptionsPass.cpp
+++ b/lib/Transforms/NaCl/LowerEmExceptionsPass.cpp
@@ -230,7 +230,7 @@ bool LowerEmExceptions::runOnModule(Module &M) {
       LandingPadInst *LP = *I;
       unsigned Num = LP->getNumClauses();
       SmallVector<Value*,16> NewLPArgs;
-      NewLPArgs.push_back(LP->getPersonalityFn());
+      NewLPArgs.push_back(F->getPersonalityFn());
       for (unsigned i = 0; i < Num; i++) {
         Value *Arg = LP->getClause(i);
         // As a temporary workaround for the lack of aggregate varargs support
diff --git a/lib/Transforms/NaCl/PromoteIntegers.cpp b/lib/Transforms/NaCl/PromoteIntegers.cpp
index 956d82e7f3b..f6232cc61b5 100644
--- a/lib/Transforms/NaCl/PromoteIntegers.cpp
+++ b/lib/Transforms/NaCl/PromoteIntegers.cpp
@@ -87,7 +87,7 @@ class PromoteIntegers : public ModulePass {
   bool runOnModule(Module &M) override;
 
 private:
-  typedef DenseMap<const llvm::Function *, DISubprogram> DebugMap;
+  typedef DenseMap<const llvm::Function *, DISubprogram *> DebugMap;
   TypeMap TypeMapper;
 
   bool ensureCompliantSignature(LLVMContext &Ctx, Function *OldFct, Module &M,
@@ -662,7 +662,7 @@ static bool processFunction(Function &F, DataLayout &DL) {
 
 bool PromoteIntegers::ensureCompliantSignature(
     LLVMContext &Ctx, Function *OldFct, Module &M,
-    DenseMap<const llvm::Function *, DISubprogram> &DISubprogramMap) {
+    DenseMap<const llvm::Function *, DISubprogram *> &DISubprogramMap) {
 
   auto *NewFctType = cast<FunctionType>(
       TypeMapper.getSimpleType(Ctx, OldFct->getFunctionType()));
diff --git a/lib/Transforms/NaCl/RemoveAsmMemory.cpp b/lib/Transforms/NaCl/RemoveAsmMemory.cpp
index eb7a32c9d3f..f06933b6dd2 100644
--- a/lib/Transforms/NaCl/RemoveAsmMemory.cpp
+++ b/lib/Transforms/NaCl/RemoveAsmMemory.cpp
@@ -62,16 +62,7 @@ bool RemoveAsmMemory::runOnFunction(Function &F) {
 }
 
 void AsmDirectivesVisitor::visitCallInst(CallInst &CI) {
-  if (!CI.isInlineAsm() ||
-      !cast<InlineAsm>(CI.getCalledValue())->isAsmMemory())
-    return;
-
-  // In NaCl ``asm("":::"memory")`` always comes in pairs, straddling a
-  // sequentially consistent fence. Other passes rewrite this fence to
-  // an equivalent stable NaCl intrinsic, meaning that this assembly can
-  // be removed.
-  CI.eraseFromParent();
-  ModifiedFunction = true;
+  llvm_unreachable("no longer maintained");
 }
 
 namespace llvm {
diff --git a/lib/Transforms/NaCl/ReplacePtrsWithInts.cpp b/lib/Transforms/NaCl/ReplacePtrsWithInts.cpp
index d1fab1fe9f2..ac56a86a514 100644
--- a/lib/Transforms/NaCl/ReplacePtrsWithInts.cpp
+++ b/lib/Transforms/NaCl/ReplacePtrsWithInts.cpp
@@ -530,7 +530,7 @@ INITIALIZE_PASS(ReplacePtrsWithInts, "replace-ptrs-with-ints",
 
 bool ReplacePtrsWithInts::runOnModule(Module &M) {
   DataLayout DL(&M);
-  DenseMap<const Function *, DISubprogram> FunctionDIs = makeSubprogramMap(M);
+  DenseMap<const Function *, DISubprogram *> FunctionDIs = makeSubprogramMap(M);
   Type *IntPtrType = DL.getIntPtrType(M.getContext());
 
   for (Module::iterator Iter = M.begin(), E = M.end(); Iter != E; ) {
diff --git a/lib/Transforms/NaCl/RewriteAtomics.cpp b/lib/Transforms/NaCl/RewriteAtomics.cpp
index a1b99e31b68..aa8be52660f 100644
--- a/lib/Transforms/NaCl/RewriteAtomics.cpp
+++ b/lib/Transforms/NaCl/RewriteAtomics.cpp
@@ -406,28 +406,6 @@ void AtomicVisitor::visitAtomicCmpXchgInst(AtomicCmpXchgInst &I) {
 /// Note that the assembly gets eliminated by the -remove-asm-memory pass.
 void AtomicVisitor::visitFenceInst(FenceInst &I) {
   return; // XXX EMSCRIPTEN
-  Type *T = Type::getInt32Ty(C); // Fences aren't overloaded on type.
-  BasicBlock::InstListType &IL(I.getParent()->getInstList());
-  bool isFirst = IL.empty() || &*I.getParent()->getInstList().begin() == &I;
-  bool isLast = IL.empty() || &*I.getParent()->getInstList().rbegin() == &I;
-  CallInst *PrevC = isFirst ? 0 : dyn_cast<CallInst>(I.getPrevNode());
-  CallInst *NextC = isLast ? 0 : dyn_cast<CallInst>(I.getNextNode());
-
-  if ((PrevC && PrevC->isInlineAsm() &&
-       cast<InlineAsm>(PrevC->getCalledValue())->isAsmMemory()) &&
-      (NextC && NextC->isInlineAsm() &&
-       cast<InlineAsm>(NextC->getCalledValue())->isAsmMemory()) &&
-      I.getOrdering() == SequentiallyConsistent) {
-    const NaCl::AtomicIntrinsics::AtomicIntrinsic *Intrinsic =
-        findAtomicIntrinsic(I, Intrinsic::nacl_atomic_fence_all, T);
-    replaceInstructionWithIntrinsicCall(I, Intrinsic, T, T,
-                                        ArrayRef<Value *>());
-  } else {
-    const NaCl::AtomicIntrinsics::AtomicIntrinsic *Intrinsic =
-        findAtomicIntrinsic(I, Intrinsic::nacl_atomic_fence, T);
-    Value *Args[] = {freezeMemoryOrder(I, I.getOrdering())};
-    replaceInstructionWithIntrinsicCall(I, Intrinsic, T, T, Args);
-  }
 }
 
 ModulePass *llvm::createRewriteAtomicsPass() { return new RewriteAtomics(); }
diff --git a/lib/Transforms/NaCl/SimplifyStructRegSignatures.cpp b/lib/Transforms/NaCl/SimplifyStructRegSignatures.cpp
index 22bf1654ad4..013d71a39c5 100644
--- a/lib/Transforms/NaCl/SimplifyStructRegSignatures.cpp
+++ b/lib/Transforms/NaCl/SimplifyStructRegSignatures.cpp
@@ -130,7 +130,7 @@ class SimplifyStructRegSignatures : public ModulePass {
 
   bool
   simplifyFunction(LLVMContext &Ctx, Function *OldFunc,
-                   DenseMap<const Function *, DISubprogram> &DISubprogramMap);
+                   DenseMap<const Function *, DISubprogram *> &DISubprogramMap);
 
   void scheduleInstructionsForCleanup(Function *NewFunc);
 
@@ -461,7 +461,7 @@ void SimplifyStructRegSignatures::fixFunctionBody(LLVMContext &Ctx,
 // had to be changed.
 bool SimplifyStructRegSignatures::simplifyFunction(
     LLVMContext &Ctx, Function *OldFunc,
-    DenseMap<const Function *, DISubprogram> &DISubprogramMap) {
+    DenseMap<const Function *, DISubprogram *> &DISubprogramMap) {
   auto *OldFT = OldFunc->getFunctionType();
   auto *NewFT = cast<FunctionType>(Mapper.getSimpleType(Ctx, OldFT));
 
@@ -532,7 +532,7 @@ void SimplifyStructRegSignatures::checkNoUnsupportedInstructions(
   for (auto &BB : Fct->getBasicBlockList())
     for (auto &Inst : BB.getInstList())
       if (auto *Landing = dyn_cast<LandingPadInst>(&Inst)) {
-        auto *LType = Landing->getPersonalityFn()->getType();
+        auto *LType = Fct->getPersonalityFn()->getType();
         if (LType != Mapper.getSimpleType(Ctx, LType)) {
           errs() << *Landing << '\n';
           report_fatal_error("Landing pads with aggregate register "
diff --git a/lib/Transforms/NaCl/StripMetadata.cpp b/lib/Transforms/NaCl/StripMetadata.cpp
index dd5bcdd0576..92c77bb08a9 100644
--- a/lib/Transforms/NaCl/StripMetadata.cpp
+++ b/lib/Transforms/NaCl/StripMetadata.cpp
@@ -161,7 +161,7 @@ bool StripDanglingDISubprograms::runOnModule(Module &M) {
 
   bool Changed = false;
   for (MDNode *N : CU_Nodes->operands()) {
-    auto *CUNode = cast<MDCompileUnit>(N);
+    auto *CUNode = cast<DICompileUnit>(N);
     for (auto *SP : CUNode->getSubprograms()) {
       // For each subprogram in the debug info, check its function for dbg
       // attachments. The allocas and some other stuff in the entry block
diff --git a/lib/Transforms/ObjCARC/ARCInstKind.cpp b/lib/Transforms/ObjCARC/ARCInstKind.cpp
index 72df9ab1c81..afb873a355a 100644
--- a/lib/Transforms/ObjCARC/ARCInstKind.cpp
+++ b/lib/Transforms/ObjCARC/ARCInstKind.cpp
@@ -239,7 +239,7 @@ ARCInstKind llvm::objcarc::GetARCInstKind(const Value *V) {
         ARCInstKind Class = GetFunctionClass(F);
         if (Class != ARCInstKind::CallOrUser)
           return Class;
-        unsigned ID = F->getIntrinsicID();
+        Intrinsic::ID ID = F->getIntrinsicID();
         if (isInertIntrinsic(ID))
           return ARCInstKind::None;
         if (isUseOnlyIntrinsic(ID))
diff --git a/lib/Transforms/ObjCARC/DependencyAnalysis.cpp b/lib/Transforms/ObjCARC/DependencyAnalysis.cpp
index 4edd02904b2..0c6b16d5d01 100644
--- a/lib/Transforms/ObjCARC/DependencyAnalysis.cpp
+++ b/lib/Transforms/ObjCARC/DependencyAnalysis.cpp
@@ -49,7 +49,7 @@ bool llvm::objcarc::CanAlterRefCount(const Instruction *Inst, const Value *Ptr,
   assert(CS && "Only calls can alter reference counts!");
 
   // See if AliasAnalysis can help us with the call.
-  AliasAnalysis::ModRefBehavior MRB = PA.getAA()->getModRefBehavior(CS);
+  FunctionModRefBehavior MRB = PA.getAA()->getModRefBehavior(CS);
   if (AliasAnalysis::onlyReadsMemory(MRB))
     return false;
   if (AliasAnalysis::onlyAccessesArgPointees(MRB)) {
diff --git a/lib/Transforms/ObjCARC/ObjCARCAliasAnalysis.cpp b/lib/Transforms/ObjCARC/ObjCARCAliasAnalysis.cpp
index b1515e38620..e7d05404e9c 100644
--- a/lib/Transforms/ObjCARC/ObjCARCAliasAnalysis.cpp
+++ b/lib/Transforms/ObjCARC/ObjCARCAliasAnalysis.cpp
@@ -57,8 +57,8 @@ ObjCARCAliasAnalysis::getAnalysisUsage(AnalysisUsage &AU) const {
   AliasAnalysis::getAnalysisUsage(AU);
 }
 
-AliasAnalysis::AliasResult
-ObjCARCAliasAnalysis::alias(const Location &LocA, const Location &LocB) {
+AliasResult ObjCARCAliasAnalysis::alias(const MemoryLocation &LocA,
+                                        const MemoryLocation &LocB) {
   if (!EnableARCOpts)
     return AliasAnalysis::alias(LocA, LocB);
 
@@ -67,8 +67,8 @@ ObjCARCAliasAnalysis::alias(const Location &LocA, const Location &LocB) {
   const Value *SA = GetRCIdentityRoot(LocA.Ptr);
   const Value *SB = GetRCIdentityRoot(LocB.Ptr);
   AliasResult Result =
-    AliasAnalysis::alias(Location(SA, LocA.Size, LocA.AATags),
-                         Location(SB, LocB.Size, LocB.AATags));
+      AliasAnalysis::alias(MemoryLocation(SA, LocA.Size, LocA.AATags),
+                           MemoryLocation(SB, LocB.Size, LocB.AATags));
   if (Result != MayAlias)
     return Result;
 
@@ -77,7 +77,7 @@ ObjCARCAliasAnalysis::alias(const Location &LocA, const Location &LocB) {
   const Value *UA = GetUnderlyingObjCPtr(SA, *DL);
   const Value *UB = GetUnderlyingObjCPtr(SB, *DL);
   if (UA != SA || UB != SB) {
-    Result = AliasAnalysis::alias(Location(UA), Location(UB));
+    Result = AliasAnalysis::alias(MemoryLocation(UA), MemoryLocation(UB));
     // We can't use MustAlias or PartialAlias results here because
     // GetUnderlyingObjCPtr may return an offsetted pointer value.
     if (Result == NoAlias)
@@ -89,44 +89,43 @@ ObjCARCAliasAnalysis::alias(const Location &LocA, const Location &LocB) {
   return MayAlias;
 }
 
-bool
-ObjCARCAliasAnalysis::pointsToConstantMemory(const Location &Loc,
-                                             bool OrLocal) {
+bool ObjCARCAliasAnalysis::pointsToConstantMemory(const MemoryLocation &Loc,
+                                                  bool OrLocal) {
   if (!EnableARCOpts)
     return AliasAnalysis::pointsToConstantMemory(Loc, OrLocal);
 
   // First, strip off no-ops, including ObjC-specific no-ops, and try making
   // a precise alias query.
   const Value *S = GetRCIdentityRoot(Loc.Ptr);
-  if (AliasAnalysis::pointsToConstantMemory(Location(S, Loc.Size, Loc.AATags),
-                                            OrLocal))
+  if (AliasAnalysis::pointsToConstantMemory(
+          MemoryLocation(S, Loc.Size, Loc.AATags), OrLocal))
     return true;
 
   // If that failed, climb to the underlying object, including climbing through
   // ObjC-specific no-ops, and try making an imprecise alias query.
   const Value *U = GetUnderlyingObjCPtr(S, *DL);
   if (U != S)
-    return AliasAnalysis::pointsToConstantMemory(Location(U), OrLocal);
+    return AliasAnalysis::pointsToConstantMemory(MemoryLocation(U), OrLocal);
 
   // If that failed, fail. We don't need to chain here, since that's covered
   // by the earlier precise query.
   return false;
 }
 
-AliasAnalysis::ModRefBehavior
+FunctionModRefBehavior
 ObjCARCAliasAnalysis::getModRefBehavior(ImmutableCallSite CS) {
   // We have nothing to do. Just chain to the next AliasAnalysis.
   return AliasAnalysis::getModRefBehavior(CS);
 }
 
-AliasAnalysis::ModRefBehavior
+FunctionModRefBehavior
 ObjCARCAliasAnalysis::getModRefBehavior(const Function *F) {
   if (!EnableARCOpts)
     return AliasAnalysis::getModRefBehavior(F);
 
   switch (GetFunctionClass(F)) {
   case ARCInstKind::NoopCast:
-    return DoesNotAccessMemory;
+    return FMRB_DoesNotAccessMemory;
   default:
     break;
   }
@@ -134,8 +133,8 @@ ObjCARCAliasAnalysis::getModRefBehavior(const Function *F) {
   return AliasAnalysis::getModRefBehavior(F);
 }
 
-AliasAnalysis::ModRefResult
-ObjCARCAliasAnalysis::getModRefInfo(ImmutableCallSite CS, const Location &Loc) {
+ModRefInfo ObjCARCAliasAnalysis::getModRefInfo(ImmutableCallSite CS,
+                                               const MemoryLocation &Loc) {
   if (!EnableARCOpts)
     return AliasAnalysis::getModRefInfo(CS, Loc);
 
@@ -151,7 +150,7 @@ ObjCARCAliasAnalysis::getModRefInfo(ImmutableCallSite CS, const Location &Loc) {
     // These functions don't access any memory visible to the compiler.
     // Note that this doesn't include objc_retainBlock, because it updates
     // pointers when it copies block data.
-    return NoModRef;
+    return MRI_NoModRef;
   default:
     break;
   }
@@ -159,10 +158,9 @@ ObjCARCAliasAnalysis::getModRefInfo(ImmutableCallSite CS, const Location &Loc) {
   return AliasAnalysis::getModRefInfo(CS, Loc);
 }
 
-AliasAnalysis::ModRefResult
-ObjCARCAliasAnalysis::getModRefInfo(ImmutableCallSite CS1,
-                                    ImmutableCallSite CS2) {
+ModRefInfo ObjCARCAliasAnalysis::getModRefInfo(ImmutableCallSite CS1,
+                                               ImmutableCallSite CS2) {
   // TODO: Theoretically we could check for dependencies between objc_* calls
-  // and OnlyAccessesArgumentPointees calls or other well-behaved calls.
+  // and FMRB_OnlyAccessesArgumentPointees calls or other well-behaved calls.
   return AliasAnalysis::getModRefInfo(CS1, CS2);
 }
diff --git a/lib/Transforms/ObjCARC/ObjCARCAliasAnalysis.h b/lib/Transforms/ObjCARC/ObjCARCAliasAnalysis.h
index 3c5a021de26..a002e2862f9 100644
--- a/lib/Transforms/ObjCARC/ObjCARCAliasAnalysis.h
+++ b/lib/Transforms/ObjCARC/ObjCARCAliasAnalysis.h
@@ -56,14 +56,16 @@ namespace objcarc {
     }
 
     void getAnalysisUsage(AnalysisUsage &AU) const override;
-    AliasResult alias(const Location &LocA, const Location &LocB) override;
-    bool pointsToConstantMemory(const Location &Loc, bool OrLocal) override;
-    ModRefBehavior getModRefBehavior(ImmutableCallSite CS) override;
-    ModRefBehavior getModRefBehavior(const Function *F) override;
-    ModRefResult getModRefInfo(ImmutableCallSite CS,
-                               const Location &Loc) override;
-    ModRefResult getModRefInfo(ImmutableCallSite CS1,
-                               ImmutableCallSite CS2) override;
+    AliasResult alias(const MemoryLocation &LocA,
+                      const MemoryLocation &LocB) override;
+    bool pointsToConstantMemory(const MemoryLocation &Loc,
+                                bool OrLocal) override;
+    FunctionModRefBehavior getModRefBehavior(ImmutableCallSite CS) override;
+    FunctionModRefBehavior getModRefBehavior(const Function *F) override;
+    ModRefInfo getModRefInfo(ImmutableCallSite CS,
+                             const MemoryLocation &Loc) override;
+    ModRefInfo getModRefInfo(ImmutableCallSite CS1,
+                             ImmutableCallSite CS2) override;
   };
 
 } // namespace objcarc
diff --git a/lib/Transforms/ObjCARC/ObjCARCContract.cpp b/lib/Transforms/ObjCARC/ObjCARCContract.cpp
index 2a3139f8705..ded930658ab 100644
--- a/lib/Transforms/ObjCARC/ObjCARCContract.cpp
+++ b/lib/Transforms/ObjCARC/ObjCARCContract.cpp
@@ -200,7 +200,7 @@ static StoreInst *findSafeStoreForStoreStrongContraction(LoadInst *Load,
   bool SawRelease = false;
 
   // Get the location associated with Load.
-  AliasAnalysis::Location Loc = AA->getLocation(Load);
+  MemoryLocation Loc = MemoryLocation::get(Load);
 
   // Walk down to find the store and the release, which may be in either order.
   for (auto I = std::next(BasicBlock::iterator(Load)),
@@ -212,7 +212,7 @@ static StoreInst *findSafeStoreForStoreStrongContraction(LoadInst *Load,
       break;
 
     // Now we know that we have not seen either the store or the release. If I
-    // is the the release, mark that we saw the release and continue.
+    // is the release, mark that we saw the release and continue.
     Instruction *Inst = &*I;
     if (Inst == Release) {
       SawRelease = true;
@@ -247,7 +247,7 @@ static StoreInst *findSafeStoreForStoreStrongContraction(LoadInst *Load,
 
     // Ok, now we know we have not seen a store yet. See if Inst can write to
     // our load location, if it can not, just ignore the instruction.
-    if (!(AA->getModRefInfo(Inst, Loc) & AliasAnalysis::Mod))
+    if (!(AA->getModRefInfo(Inst, Loc) & MRI_Mod))
       continue;
 
     Store = dyn_cast<StoreInst>(Inst);
diff --git a/lib/Transforms/ObjCARC/ObjCARCOpts.cpp b/lib/Transforms/ObjCARC/ObjCARCOpts.cpp
index 4d7565879af..9edbb17e8d1 100644
--- a/lib/Transforms/ObjCARC/ObjCARCOpts.cpp
+++ b/lib/Transforms/ObjCARC/ObjCARCOpts.cpp
@@ -105,8 +105,8 @@ static inline bool AreAnyUnderlyingObjectsAnAlloca(const Value *V,
     }
 
     if (const PHINode *PN = dyn_cast<const PHINode>(P)) {
-      for (unsigned i = 0, e = PN->getNumIncomingValues(); i != e; ++i)
-        Worklist.push_back(PN->getIncomingValue(i));
+      for (Value *IncValue : PN->incoming_values())
+        Worklist.push_back(IncValue);
       continue;
     }
   } while (!Worklist.empty());
@@ -1846,7 +1846,7 @@ void ObjCARCOpt::OptimizeWeakCalls(Function &F) {
         Value *Arg = Call->getArgOperand(0);
         Value *EarlierArg = EarlierCall->getArgOperand(0);
         switch (PA.getAA()->alias(Arg, EarlierArg)) {
-        case AliasAnalysis::MustAlias:
+        case MustAlias:
           Changed = true;
           // If the load has a builtin retain, insert a plain retain for it.
           if (Class == ARCInstKind::LoadWeakRetained) {
@@ -1858,10 +1858,10 @@ void ObjCARCOpt::OptimizeWeakCalls(Function &F) {
           Call->replaceAllUsesWith(EarlierCall);
           Call->eraseFromParent();
           goto clobbered;
-        case AliasAnalysis::MayAlias:
-        case AliasAnalysis::PartialAlias:
+        case MayAlias:
+        case PartialAlias:
           goto clobbered;
-        case AliasAnalysis::NoAlias:
+        case NoAlias:
           break;
         }
         break;
@@ -1875,7 +1875,7 @@ void ObjCARCOpt::OptimizeWeakCalls(Function &F) {
         Value *Arg = Call->getArgOperand(0);
         Value *EarlierArg = EarlierCall->getArgOperand(0);
         switch (PA.getAA()->alias(Arg, EarlierArg)) {
-        case AliasAnalysis::MustAlias:
+        case MustAlias:
           Changed = true;
           // If the load has a builtin retain, insert a plain retain for it.
           if (Class == ARCInstKind::LoadWeakRetained) {
@@ -1887,10 +1887,10 @@ void ObjCARCOpt::OptimizeWeakCalls(Function &F) {
           Call->replaceAllUsesWith(EarlierCall->getArgOperand(1));
           Call->eraseFromParent();
           goto clobbered;
-        case AliasAnalysis::MayAlias:
-        case AliasAnalysis::PartialAlias:
+        case MayAlias:
+        case PartialAlias:
           goto clobbered;
-        case AliasAnalysis::NoAlias:
+        case NoAlias:
           break;
         }
         break;
diff --git a/lib/Transforms/ObjCARC/ProvenanceAnalysis.cpp b/lib/Transforms/ObjCARC/ProvenanceAnalysis.cpp
index 15ad8dc522f..9ffdfb4f7f9 100644
--- a/lib/Transforms/ObjCARC/ProvenanceAnalysis.cpp
+++ b/lib/Transforms/ObjCARC/ProvenanceAnalysis.cpp
@@ -62,8 +62,7 @@ bool ProvenanceAnalysis::relatedPHI(const PHINode *A,
 
   // Check each unique source of the PHI node against B.
   SmallPtrSet<const Value *, 4> UniqueSrc;
-  for (unsigned i = 0, e = A->getNumIncomingValues(); i != e; ++i) {
-    const Value *PV1 = A->getIncomingValue(i);
+  for (Value *PV1 : A->incoming_values()) {
     if (UniqueSrc.insert(PV1).second && related(PV1, B, DL))
       return true;
   }
@@ -117,12 +116,12 @@ bool ProvenanceAnalysis::relatedCheck(const Value *A, const Value *B,
 
   // Ask regular AliasAnalysis, for a first approximation.
   switch (AA->alias(A, B)) {
-  case AliasAnalysis::NoAlias:
+  case NoAlias:
     return false;
-  case AliasAnalysis::MustAlias:
-  case AliasAnalysis::PartialAlias:
+  case MustAlias:
+  case PartialAlias:
     return true;
-  case AliasAnalysis::MayAlias:
+  case MayAlias:
     break;
   }
 
diff --git a/lib/Transforms/Scalar/ADCE.cpp b/lib/Transforms/Scalar/ADCE.cpp
index d6fc9164158..824342ad7e3 100644
--- a/lib/Transforms/Scalar/ADCE.cpp
+++ b/lib/Transforms/Scalar/ADCE.cpp
@@ -58,8 +58,8 @@ bool ADCE::runOnFunction(Function& F) {
 
   // Collect the set of "root" instructions that are known live.
   for (Instruction &I : inst_range(F)) {
-    if (isa<TerminatorInst>(I) || isa<DbgInfoIntrinsic>(I) ||
-        isa<LandingPadInst>(I) || I.mayHaveSideEffects()) {
+    if (isa<TerminatorInst>(I) || isa<DbgInfoIntrinsic>(I) || I.isEHPad() ||
+        I.mayHaveSideEffects()) {
       Alive.insert(&I);
       Worklist.push_back(&I);
     }
diff --git a/lib/Transforms/Scalar/BDCE.cpp b/lib/Transforms/Scalar/BDCE.cpp
index 09c605e7673..7252db0a6cd 100644
--- a/lib/Transforms/Scalar/BDCE.cpp
+++ b/lib/Transforms/Scalar/BDCE.cpp
@@ -77,8 +77,8 @@ INITIALIZE_PASS_END(BDCE, "bdce", "Bit-Tracking Dead Code Elimination",
                     false, false)
 
 static bool isAlwaysLive(Instruction *I) {
-  return isa<TerminatorInst>(I) || isa<DbgInfoIntrinsic>(I) ||
-         isa<LandingPadInst>(I) || I->mayHaveSideEffects();
+  return isa<TerminatorInst>(I) || isa<DbgInfoIntrinsic>(I) || I->isEHPad() ||
+         I->mayHaveSideEffects();
 }
 
 void BDCE::determineLiveOperandBits(const Instruction *UserI,
diff --git a/lib/Transforms/Scalar/CMakeLists.txt b/lib/Transforms/Scalar/CMakeLists.txt
index c29fcc3c7dd..7ee279a5660 100644
--- a/lib/Transforms/Scalar/CMakeLists.txt
+++ b/lib/Transforms/Scalar/CMakeLists.txt
@@ -17,6 +17,7 @@ add_llvm_library(LLVMScalarOpts
   LICM.cpp
   LoadCombine.cpp
   LoopDeletion.cpp
+  LoopDistribute.cpp
   LoopIdiomRecognize.cpp
   LoopInstSimplify.cpp
   LoopInterchange.cpp
@@ -44,6 +45,7 @@ add_llvm_library(LLVMScalarOpts
   SeparateConstOffsetFromGEP.cpp
   SimplifyCFGPass.cpp
   Sink.cpp
+  SpeculativeExecution.cpp
   StraightLineStrengthReduce.cpp
   StructurizeCFG.cpp
   TailRecursionElimination.cpp
diff --git a/lib/Transforms/Scalar/CorrelatedValuePropagation.cpp b/lib/Transforms/Scalar/CorrelatedValuePropagation.cpp
index 912d527402a..79624b2e4c4 100644
--- a/lib/Transforms/Scalar/CorrelatedValuePropagation.cpp
+++ b/lib/Transforms/Scalar/CorrelatedValuePropagation.cpp
@@ -103,24 +103,44 @@ bool CorrelatedValuePropagation::processPHI(PHINode *P) {
 
     Value *V = LVI->getConstantOnEdge(Incoming, P->getIncomingBlock(i), BB, P);
 
-    // Look if the incoming value is a select with a constant but LVI tells us
-    // that the incoming value can never be that constant. In that case replace
-    // the incoming value with the other value of the select. This often allows
-    // us to remove the select later.
+    // Look if the incoming value is a select with a scalar condition for which
+    // LVI can tells us the value. In that case replace the incoming value with
+    // the appropriate value of the select. This often allows us to remove the
+    // select later.
     if (!V) {
       SelectInst *SI = dyn_cast<SelectInst>(Incoming);
       if (!SI) continue;
 
-      Constant *C = dyn_cast<Constant>(SI->getFalseValue());
-      if (!C) continue;
+      Value *Condition = SI->getCondition();
+      if (!Condition->getType()->isVectorTy()) {
+        if (Constant *C = LVI->getConstantOnEdge(
+                Condition, P->getIncomingBlock(i), BB, P)) {
+          if (C->isOneValue()) {
+            V = SI->getTrueValue();
+          } else if (C->isZeroValue()) {
+            V = SI->getFalseValue();
+          }
+          // Once LVI learns to handle vector types, we could also add support
+          // for vector type constants that are not all zeroes or all ones.
+        }
+      }
 
-      if (LVI->getPredicateOnEdge(ICmpInst::ICMP_EQ, SI, C,
-                                  P->getIncomingBlock(i), BB, P) !=
-          LazyValueInfo::False)
-        continue;
+      // Look if the select has a constant but LVI tells us that the incoming
+      // value can never be that constant. In that case replace the incoming
+      // value with the other value of the select. This often allows us to
+      // remove the select later.
+      if (!V) {
+        Constant *C = dyn_cast<Constant>(SI->getFalseValue());
+        if (!C) continue;
+
+        if (LVI->getPredicateOnEdge(ICmpInst::ICMP_EQ, SI, C,
+              P->getIncomingBlock(i), BB, P) !=
+            LazyValueInfo::False)
+          continue;
+        V = SI->getTrueValue();
+      }
 
       DEBUG(dbgs() << "CVP: Threading PHI over " << *SI << '\n');
-      V = SI->getTrueValue();
     }
 
     P->setIncomingValue(i, V);
diff --git a/lib/Transforms/Scalar/DeadStoreElimination.cpp b/lib/Transforms/Scalar/DeadStoreElimination.cpp
index 01952cf6e8b..66f23b9a429 100644
--- a/lib/Transforms/Scalar/DeadStoreElimination.cpp
+++ b/lib/Transforms/Scalar/DeadStoreElimination.cpp
@@ -78,7 +78,7 @@ namespace {
     bool runOnBasicBlock(BasicBlock &BB);
     bool HandleFree(CallInst *F);
     bool handleEndBlock(BasicBlock &BB);
-    void RemoveAccessedObjects(const AliasAnalysis::Location &LoadedLoc,
+    void RemoveAccessedObjects(const MemoryLocation &LoadedLoc,
                                SmallSetVector<Value *, 16> &DeadStackObjects,
                                const DataLayout &DL);
 
@@ -194,45 +194,45 @@ static bool hasMemoryWrite(Instruction *I, const TargetLibraryInfo *TLI) {
 /// getLocForWrite - Return a Location stored to by the specified instruction.
 /// If isRemovable returns true, this function and getLocForRead completely
 /// describe the memory operations for this instruction.
-static AliasAnalysis::Location
-getLocForWrite(Instruction *Inst, AliasAnalysis &AA) {
+static MemoryLocation getLocForWrite(Instruction *Inst, AliasAnalysis &AA) {
   if (StoreInst *SI = dyn_cast<StoreInst>(Inst))
-    return AA.getLocation(SI);
+    return MemoryLocation::get(SI);
 
   if (MemIntrinsic *MI = dyn_cast<MemIntrinsic>(Inst)) {
     // memcpy/memmove/memset.
-    AliasAnalysis::Location Loc = AA.getLocationForDest(MI);
+    MemoryLocation Loc = MemoryLocation::getForDest(MI);
     return Loc;
   }
 
   IntrinsicInst *II = dyn_cast<IntrinsicInst>(Inst);
-  if (!II) return AliasAnalysis::Location();
+  if (!II)
+    return MemoryLocation();
 
   switch (II->getIntrinsicID()) {
-  default: return AliasAnalysis::Location(); // Unhandled intrinsic.
+  default:
+    return MemoryLocation(); // Unhandled intrinsic.
   case Intrinsic::init_trampoline:
     // FIXME: We don't know the size of the trampoline, so we can't really
     // handle it here.
-    return AliasAnalysis::Location(II->getArgOperand(0));
+    return MemoryLocation(II->getArgOperand(0));
   case Intrinsic::lifetime_end: {
     uint64_t Len = cast<ConstantInt>(II->getArgOperand(0))->getZExtValue();
-    return AliasAnalysis::Location(II->getArgOperand(1), Len);
+    return MemoryLocation(II->getArgOperand(1), Len);
   }
   }
 }
 
 /// getLocForRead - Return the location read by the specified "hasMemoryWrite"
 /// instruction if any.
-static AliasAnalysis::Location
-getLocForRead(Instruction *Inst, AliasAnalysis &AA) {
+static MemoryLocation getLocForRead(Instruction *Inst, AliasAnalysis &AA) {
   assert(hasMemoryWrite(Inst, AA.getTargetLibraryInfo()) &&
          "Unknown instruction case");
 
   // The only instructions that both read and write are the mem transfer
   // instructions (memcpy/memmove).
   if (MemTransferInst *MTI = dyn_cast<MemTransferInst>(Inst))
-    return AA.getLocationForSource(MTI);
-  return AliasAnalysis::Location();
+    return MemoryLocation::getForSource(MTI);
+  return MemoryLocation();
 }
 
 
@@ -317,7 +317,7 @@ static uint64_t getPointerSize(const Value *V, const DataLayout &DL,
   uint64_t Size;
   if (getObjectSize(V, Size, DL, TLI))
     return Size;
-  return AliasAnalysis::UnknownSize;
+  return MemoryLocation::UnknownSize;
 }
 
 namespace {
@@ -333,8 +333,8 @@ namespace {
 /// completely overwrites a store to the 'Earlier' location.
 /// 'OverwriteEnd' if the end of the 'Earlier' location is completely
 /// overwritten by 'Later', or 'OverwriteUnknown' if nothing can be determined
-static OverwriteResult isOverwrite(const AliasAnalysis::Location &Later,
-                                   const AliasAnalysis::Location &Earlier,
+static OverwriteResult isOverwrite(const MemoryLocation &Later,
+                                   const MemoryLocation &Earlier,
                                    const DataLayout &DL,
                                    const TargetLibraryInfo *TLI,
                                    int64_t &EarlierOff, int64_t &LaterOff) {
@@ -346,8 +346,8 @@ static OverwriteResult isOverwrite(const AliasAnalysis::Location &Later,
   if (P1 == P2) {
     // If we don't know the sizes of either access, then we can't do a
     // comparison.
-    if (Later.Size == AliasAnalysis::UnknownSize ||
-        Earlier.Size == AliasAnalysis::UnknownSize)
+    if (Later.Size == MemoryLocation::UnknownSize ||
+        Earlier.Size == MemoryLocation::UnknownSize)
       return OverwriteUnknown;
 
     // Make sure that the Later size is >= the Earlier size.
@@ -357,8 +357,8 @@ static OverwriteResult isOverwrite(const AliasAnalysis::Location &Later,
 
   // Otherwise, we have to have size information, and the later store has to be
   // larger than the earlier one.
-  if (Later.Size == AliasAnalysis::UnknownSize ||
-      Earlier.Size == AliasAnalysis::UnknownSize)
+  if (Later.Size == MemoryLocation::UnknownSize ||
+      Earlier.Size == MemoryLocation::UnknownSize)
     return OverwriteUnknown;
 
   // Check to see if the later store is to the entire object (either a global,
@@ -374,7 +374,7 @@ static OverwriteResult isOverwrite(const AliasAnalysis::Location &Later,
 
   // If the "Later" store is to a recognizable object, get its size.
   uint64_t ObjectSize = getPointerSize(UO2, DL, TLI);
-  if (ObjectSize != AliasAnalysis::UnknownSize)
+  if (ObjectSize != MemoryLocation::UnknownSize)
     if (ObjectSize == Later.Size && ObjectSize >= Earlier.Size)
       return OverwriteComplete;
 
@@ -441,11 +441,11 @@ static OverwriteResult isOverwrite(const AliasAnalysis::Location &Later,
 /// This function detects when it is unsafe to remove a dependent instruction
 /// because the DSE inducing instruction may be a self-read.
 static bool isPossibleSelfRead(Instruction *Inst,
-                               const AliasAnalysis::Location &InstStoreLoc,
+                               const MemoryLocation &InstStoreLoc,
                                Instruction *DepWrite, AliasAnalysis &AA) {
   // Self reads can only happen for instructions that read memory.  Get the
   // location read.
-  AliasAnalysis::Location InstReadLoc = getLocForRead(Inst, AA);
+  MemoryLocation InstReadLoc = getLocForRead(Inst, AA);
   if (!InstReadLoc.Ptr) return false;  // Not a reading instruction.
 
   // If the read and written loc obviously don't alias, it isn't a read.
@@ -459,7 +459,7 @@ static bool isPossibleSelfRead(Instruction *Inst,
   // Here we don't know if A/B may alias, but we do know that B/B are must
   // aliases, so removing the first memcpy is safe (assuming it writes <= #
   // bytes as the second one.
-  AliasAnalysis::Location DepReadLoc = getLocForRead(DepWrite, AA);
+  MemoryLocation DepReadLoc = getLocForRead(DepWrite, AA);
 
   if (DepReadLoc.Ptr && AA.isMustAlias(InstReadLoc.Ptr, DepReadLoc.Ptr))
     return false;
@@ -525,7 +525,7 @@ bool DSE::runOnBasicBlock(BasicBlock &BB) {
     }
 
     // Figure out what location is being stored to.
-    AliasAnalysis::Location Loc = getLocForWrite(Inst, *AA);
+    MemoryLocation Loc = getLocForWrite(Inst, *AA);
 
     // If we didn't get a useful location, fail.
     if (!Loc.Ptr)
@@ -540,7 +540,7 @@ bool DSE::runOnBasicBlock(BasicBlock &BB) {
       //
       // Find out what memory location the dependent instruction stores.
       Instruction *DepWrite = InstDep.getInst();
-      AliasAnalysis::Location DepLoc = getLocForWrite(DepWrite, *AA);
+      MemoryLocation DepLoc = getLocForWrite(DepWrite, *AA);
       // If we didn't get a useful location, or if it isn't a size, bail out.
       if (!DepLoc.Ptr)
         break;
@@ -609,7 +609,7 @@ bool DSE::runOnBasicBlock(BasicBlock &BB) {
       if (DepWrite == &BB.front()) break;
 
       // Can't look past this instruction if it might read 'Loc'.
-      if (AA->getModRefInfo(DepWrite, Loc) & AliasAnalysis::Ref)
+      if (AA->getModRefInfo(DepWrite, Loc) & MRI_Ref)
         break;
 
       InstDep = MD->getPointerDependencyFrom(Loc, false, DepWrite, &BB);
@@ -645,7 +645,7 @@ static void FindUnconditionalPreds(SmallVectorImpl<BasicBlock *> &Blocks,
 bool DSE::HandleFree(CallInst *F) {
   bool MadeChange = false;
 
-  AliasAnalysis::Location Loc = AliasAnalysis::Location(F->getOperand(0));
+  MemoryLocation Loc = MemoryLocation(F->getOperand(0));
   SmallVector<BasicBlock *, 16> Blocks;
   Blocks.push_back(F->getParent());
   const DataLayout &DL = F->getModule()->getDataLayout();
@@ -795,10 +795,10 @@ bool DSE::handleEndBlock(BasicBlock &BB) {
       // the call is live.
       DeadStackObjects.remove_if([&](Value *I) {
         // See if the call site touches the value.
-        AliasAnalysis::ModRefResult A = AA->getModRefInfo(
+        ModRefInfo A = AA->getModRefInfo(
             CS, I, getPointerSize(I, DL, AA->getTargetLibraryInfo()));
 
-        return A == AliasAnalysis::ModRef || A == AliasAnalysis::Ref;
+        return A == MRI_ModRef || A == MRI_Ref;
       });
 
       // If all of the allocas were clobbered by the call then we're not going
@@ -809,17 +809,17 @@ bool DSE::handleEndBlock(BasicBlock &BB) {
       continue;
     }
 
-    AliasAnalysis::Location LoadedLoc;
+    MemoryLocation LoadedLoc;
 
     // If we encounter a use of the pointer, it is no longer considered dead
     if (LoadInst *L = dyn_cast<LoadInst>(BBI)) {
       if (!L->isUnordered()) // Be conservative with atomic/volatile load
         break;
-      LoadedLoc = AA->getLocation(L);
+      LoadedLoc = MemoryLocation::get(L);
     } else if (VAArgInst *V = dyn_cast<VAArgInst>(BBI)) {
-      LoadedLoc = AA->getLocation(V);
+      LoadedLoc = MemoryLocation::get(V);
     } else if (MemTransferInst *MTI = dyn_cast<MemTransferInst>(BBI)) {
-      LoadedLoc = AA->getLocationForSource(MTI);
+      LoadedLoc = MemoryLocation::getForSource(MTI);
     } else if (!BBI->mayReadFromMemory()) {
       // Instruction doesn't read memory.  Note that stores that weren't removed
       // above will hit this case.
@@ -845,7 +845,7 @@ bool DSE::handleEndBlock(BasicBlock &BB) {
 /// RemoveAccessedObjects - Check to see if the specified location may alias any
 /// of the stack objects in the DeadStackObjects set.  If so, they become live
 /// because the location is being loaded.
-void DSE::RemoveAccessedObjects(const AliasAnalysis::Location &LoadedLoc,
+void DSE::RemoveAccessedObjects(const MemoryLocation &LoadedLoc,
                                 SmallSetVector<Value *, 16> &DeadStackObjects,
                                 const DataLayout &DL) {
   const Value *UnderlyingPointer = GetUnderlyingObject(LoadedLoc.Ptr, DL);
@@ -864,8 +864,8 @@ void DSE::RemoveAccessedObjects(const AliasAnalysis::Location &LoadedLoc,
   // Remove objects that could alias LoadedLoc.
   DeadStackObjects.remove_if([&](Value *I) {
     // See if the loaded location could alias the stack location.
-    AliasAnalysis::Location StackLoc(
-        I, getPointerSize(I, DL, AA->getTargetLibraryInfo()));
+    MemoryLocation StackLoc(I,
+                            getPointerSize(I, DL, AA->getTargetLibraryInfo()));
     return !AA->isNoAlias(StackLoc, LoadedLoc);
   });
 }
diff --git a/lib/Transforms/Scalar/EarlyCSE.cpp b/lib/Transforms/Scalar/EarlyCSE.cpp
index d5b9e03677f..d536a937dce 100644
--- a/lib/Transforms/Scalar/EarlyCSE.cpp
+++ b/lib/Transforms/Scalar/EarlyCSE.cpp
@@ -461,6 +461,30 @@ bool EarlyCSE::processNode(DomTreeNode *Node) {
   if (!BB->getSinglePredecessor())
     ++CurrentGeneration;
 
+  // If this node has a single predecessor which ends in a conditional branch,
+  // we can infer the value of the branch condition given that we took this
+  // path.  We need the single predeccesor to ensure there's not another path
+  // which reaches this block where the condition might hold a different
+  // value.  Since we're adding this to the scoped hash table (like any other
+  // def), it will have been popped if we encounter a future merge block.
+  if (BasicBlock *Pred = BB->getSinglePredecessor())
+    if (auto *BI = dyn_cast<BranchInst>(Pred->getTerminator()))
+      if (BI->isConditional())
+        if (auto *CondInst = dyn_cast<Instruction>(BI->getCondition()))
+          if (SimpleValue::canHandle(CondInst)) {
+            assert(BI->getSuccessor(0) == BB || BI->getSuccessor(1) == BB);
+            auto *ConditionalConstant = (BI->getSuccessor(0) == BB) ?
+              ConstantInt::getTrue(BB->getContext()) :
+              ConstantInt::getFalse(BB->getContext());
+            AvailableValues.insert(CondInst, ConditionalConstant);
+            DEBUG(dbgs() << "EarlyCSE CVP: Add conditional value for '"
+                  << CondInst->getName() << "' as " << *ConditionalConstant
+                  << " in " << BB->getName() << "\n");
+            // Replace all dominated uses with the known value
+            replaceDominatedUsesWith(CondInst, ConditionalConstant, DT,
+                                     BasicBlockEdge(Pred, BB));
+          }
+
   /// LastStore - Keep track of the last non-volatile store that we saw... for
   /// as long as there in no instruction that reads memory.  If we see a store
   /// to the same location, we delete the dead store.  This zaps trivial dead
diff --git a/lib/Transforms/Scalar/Float2Int.cpp b/lib/Transforms/Scalar/Float2Int.cpp
index c9314229c38..8c1dce1c9fb 100644
--- a/lib/Transforms/Scalar/Float2Int.cpp
+++ b/lib/Transforms/Scalar/Float2Int.cpp
@@ -19,6 +19,7 @@
 #include "llvm/ADT/EquivalenceClasses.h"
 #include "llvm/ADT/MapVector.h"
 #include "llvm/ADT/SmallVector.h"
+#include "llvm/Analysis/AliasAnalysis.h"
 #include "llvm/IR/ConstantRange.h"
 #include "llvm/IR/Constants.h"
 #include "llvm/IR/IRBuilder.h"
@@ -60,6 +61,7 @@ namespace {
     bool runOnFunction(Function &F) override;
     void getAnalysisUsage(AnalysisUsage &AU) const override {
       AU.setPreservesCFG();
+      AU.addPreserved<AliasAnalysis>();
     }
 
     void findRoots(Function &F, SmallPtrSet<Instruction*,8> &Roots);
@@ -240,11 +242,11 @@ void Float2Int::walkBackwards(const SmallPtrSetImpl<Instruction*> &Roots) {
 // Walk forwards down the list of seen instructions, so we visit defs before
 // uses.
 void Float2Int::walkForwards() {
-  for (auto It = SeenInsts.rbegin(), E = SeenInsts.rend(); It != E; ++It) {
-    if (It->second != unknownRange())
+  for (auto &It : make_range(SeenInsts.rbegin(), SeenInsts.rend())) {
+    if (It.second != unknownRange())
       continue;
 
-    Instruction *I = It->first;
+    Instruction *I = It.first;
     std::function<ConstantRange(ArrayRef<ConstantRange>)> Op;
     switch (I->getOpcode()) {
       // FIXME: Handle select and phi nodes.
@@ -505,9 +507,8 @@ Value *Float2Int::convert(Instruction *I, Type *ToTy) {
 
 // Perform dead code elimination on the instructions we just modified.
 void Float2Int::cleanup() {
-  for (auto I = ConvertedInsts.rbegin(), E = ConvertedInsts.rend();
-       I != E; ++I)
-    I->first->eraseFromParent();
+  for (auto &I : make_range(ConvertedInsts.rbegin(), ConvertedInsts.rend()))
+    I.first->eraseFromParent();
 }
 
 bool Float2Int::runOnFunction(Function &F) {
diff --git a/lib/Transforms/Scalar/GVN.cpp b/lib/Transforms/Scalar/GVN.cpp
index 97d5b6d59fa..351debb59dd 100644
--- a/lib/Transforms/Scalar/GVN.cpp
+++ b/lib/Transforms/Scalar/GVN.cpp
@@ -656,11 +656,14 @@ namespace {
       LeaderTableEntry* Prev = nullptr;
       LeaderTableEntry* Curr = &LeaderTable[N];
 
-      while (Curr->Val != I || Curr->BB != BB) {
+      while (Curr && (Curr->Val != I || Curr->BB != BB)) {
         Prev = Curr;
         Curr = Curr->Next;
       }
 
+      if (!Curr)
+        return;
+
       if (Prev) {
         Prev->Next = Curr->Next;
       } else {
@@ -693,7 +696,7 @@ namespace {
     }
 
 
-    // Helper fuctions of redundant load elimination 
+    // Helper functions of redundant load elimination 
     bool processLoad(LoadInst *L);
     bool processNonLocalLoad(LoadInst *L);
     void AnalyzeLoadAvailability(LoadInst *LI, LoadDepVect &Deps, 
@@ -716,8 +719,6 @@ namespace {
     void verifyRemoved(const Instruction *I) const;
     bool splitCriticalEdges();
     BasicBlock *splitCriticalEdges(BasicBlock *Pred, BasicBlock *Succ);
-    unsigned replaceAllDominatedUsesWith(Value *From, Value *To,
-                                         const BasicBlockEdge &Root);
     bool propagateEquality(Value *LHS, Value *RHS, const BasicBlockEdge &Root);
     bool processFoldableCondBr(BranchInst *BI);
     void addDeadBlock(BasicBlock *BB);
@@ -854,13 +855,12 @@ static bool CanCoerceMustAliasedValueToLoad(Value *StoredVal,
 
 /// If we saw a store of a value to memory, and
 /// then a load from a must-aliased pointer of a different type, try to coerce
-/// the stored value.  LoadedTy is the type of the load we want to replace and
-/// InsertPt is the place to insert new instructions.
+/// the stored value.  LoadedTy is the type of the load we want to replace.
+/// IRB is IRBuilder used to insert new instructions.
 ///
 /// If we can't do it, return null.
-static Value *CoerceAvailableValueToLoadType(Value *StoredVal,
-                                             Type *LoadedTy,
-                                             Instruction *InsertPt,
+static Value *CoerceAvailableValueToLoadType(Value *StoredVal, Type *LoadedTy,
+                                             IRBuilder<> &IRB,
                                              const DataLayout &DL) {
   if (!CanCoerceMustAliasedValueToLoad(StoredVal, LoadedTy, DL))
     return nullptr;
@@ -876,12 +876,12 @@ static Value *CoerceAvailableValueToLoadType(Value *StoredVal,
     // Pointer to Pointer -> use bitcast.
     if (StoredValTy->getScalarType()->isPointerTy() &&
         LoadedTy->getScalarType()->isPointerTy())
-      return new BitCastInst(StoredVal, LoadedTy, "", InsertPt);
+      return IRB.CreateBitCast(StoredVal, LoadedTy);
 
     // Convert source pointers to integers, which can be bitcast.
     if (StoredValTy->getScalarType()->isPointerTy()) {
       StoredValTy = DL.getIntPtrType(StoredValTy);
-      StoredVal = new PtrToIntInst(StoredVal, StoredValTy, "", InsertPt);
+      StoredVal = IRB.CreatePtrToInt(StoredVal, StoredValTy);
     }
 
     Type *TypeToCastTo = LoadedTy;
@@ -889,11 +889,11 @@ static Value *CoerceAvailableValueToLoadType(Value *StoredVal,
       TypeToCastTo = DL.getIntPtrType(TypeToCastTo);
 
     if (StoredValTy != TypeToCastTo)
-      StoredVal = new BitCastInst(StoredVal, TypeToCastTo, "", InsertPt);
+      StoredVal = IRB.CreateBitCast(StoredVal, TypeToCastTo);
 
     // Cast to pointer if the load needs a pointer type.
     if (LoadedTy->getScalarType()->isPointerTy())
-      StoredVal = new IntToPtrInst(StoredVal, LoadedTy, "", InsertPt);
+      StoredVal = IRB.CreateIntToPtr(StoredVal, LoadedTy);
 
     return StoredVal;
   }
@@ -906,35 +906,34 @@ static Value *CoerceAvailableValueToLoadType(Value *StoredVal,
   // Convert source pointers to integers, which can be manipulated.
   if (StoredValTy->getScalarType()->isPointerTy()) {
     StoredValTy = DL.getIntPtrType(StoredValTy);
-    StoredVal = new PtrToIntInst(StoredVal, StoredValTy, "", InsertPt);
+    StoredVal = IRB.CreatePtrToInt(StoredVal, StoredValTy);
   }
 
   // Convert vectors and fp to integer, which can be manipulated.
   if (!StoredValTy->isIntegerTy()) {
     StoredValTy = IntegerType::get(StoredValTy->getContext(), StoreSize);
-    StoredVal = new BitCastInst(StoredVal, StoredValTy, "", InsertPt);
+    StoredVal = IRB.CreateBitCast(StoredVal, StoredValTy);
   }
 
   // If this is a big-endian system, we need to shift the value down to the low
   // bits so that a truncate will work.
   if (DL.isBigEndian()) {
-    Constant *Val = ConstantInt::get(StoredVal->getType(), StoreSize-LoadSize);
-    StoredVal = BinaryOperator::CreateLShr(StoredVal, Val, "tmp", InsertPt);
+    StoredVal = IRB.CreateLShr(StoredVal, StoreSize - LoadSize, "tmp");
   }
 
   // Truncate the integer to the right size now.
   Type *NewIntTy = IntegerType::get(StoredValTy->getContext(), LoadSize);
-  StoredVal = new TruncInst(StoredVal, NewIntTy, "trunc", InsertPt);
+  StoredVal  = IRB.CreateTrunc(StoredVal, NewIntTy, "trunc");
 
   if (LoadedTy == NewIntTy)
     return StoredVal;
 
   // If the result is a pointer, inttoptr.
   if (LoadedTy->getScalarType()->isPointerTy())
-    return new IntToPtrInst(StoredVal, LoadedTy, "inttoptr", InsertPt);
+    return IRB.CreateIntToPtr(StoredVal, LoadedTy, "inttoptr");
 
   // Otherwise, bitcast.
-  return new BitCastInst(StoredVal, LoadedTy, "bitcast", InsertPt);
+  return IRB.CreateBitCast(StoredVal, LoadedTy, "bitcast");
 }
 
 /// This function is called when we have a
@@ -1124,7 +1123,7 @@ static Value *GetStoreValueForLoad(Value *SrcVal, unsigned Offset,
   uint64_t StoreSize = (DL.getTypeSizeInBits(SrcVal->getType()) + 7) / 8;
   uint64_t LoadSize = (DL.getTypeSizeInBits(LoadTy) + 7) / 8;
 
-  IRBuilder<> Builder(InsertPt->getParent(), InsertPt);
+  IRBuilder<> Builder(InsertPt);
 
   // Compute which bits of the stored value are being used by the load.  Convert
   // to an integer type to start with.
@@ -1147,7 +1146,7 @@ static Value *GetStoreValueForLoad(Value *SrcVal, unsigned Offset,
   if (LoadSize != StoreSize)
     SrcVal = Builder.CreateTrunc(SrcVal, IntegerType::get(Ctx, LoadSize*8));
 
-  return CoerceAvailableValueToLoadType(SrcVal, LoadTy, InsertPt, DL);
+  return CoerceAvailableValueToLoadType(SrcVal, LoadTy, Builder, DL);
 }
 
 /// This function is called when we have a
@@ -1221,7 +1220,7 @@ static Value *GetMemInstValueForLoad(MemIntrinsic *SrcInst, unsigned Offset,
   LLVMContext &Ctx = LoadTy->getContext();
   uint64_t LoadSize = DL.getTypeSizeInBits(LoadTy)/8;
 
-  IRBuilder<> Builder(InsertPt->getParent(), InsertPt);
+  IRBuilder<> Builder(InsertPt);
 
   // We know that this method is only called when the mem transfer fully
   // provides the bits for the load.
@@ -1250,7 +1249,7 @@ static Value *GetMemInstValueForLoad(MemIntrinsic *SrcInst, unsigned Offset,
       ++NumBytesSet;
     }
 
-    return CoerceAvailableValueToLoadType(Val, LoadTy, InsertPt, DL);
+    return CoerceAvailableValueToLoadType(Val, LoadTy, Builder, DL);
   }
 
   // Otherwise, this is a memcpy/memmove from a constant global.
@@ -1302,28 +1301,7 @@ static Value *ConstructSSAForLoadSet(LoadInst *LI,
   }
 
   // Perform PHI construction.
-  Value *V = SSAUpdate.GetValueInMiddleOfBlock(LI->getParent());
-
-  // If new PHI nodes were created, notify alias analysis.
-  if (V->getType()->getScalarType()->isPointerTy()) {
-    AliasAnalysis *AA = gvn.getAliasAnalysis();
-
-    for (unsigned i = 0, e = NewPHIs.size(); i != e; ++i)
-      AA->copyValue(LI, NewPHIs[i]);
-
-    // Now that we've copied information to the new PHIs, scan through
-    // them again and inform alias analysis that we've added potentially
-    // escaping uses to any values that are operands to these PHIs.
-    for (unsigned i = 0, e = NewPHIs.size(); i != e; ++i) {
-      PHINode *P = NewPHIs[i];
-      for (unsigned ii = 0, ee = P->getNumIncomingValues(); ii != ee; ++ii) {
-        unsigned jj = PHINode::getOperandNumForIncomingValue(ii);
-        AA->addEscapingUse(P->getOperandUse(jj));
-      }
-    }
-  }
-
-  return V;
+  return SSAUpdate.GetValueInMiddleOfBlock(LI->getParent());
 }
 
 Value *AvailableValueInBlock::MaterializeAdjustedValue(LoadInst *LI,
@@ -1697,6 +1675,8 @@ bool GVN::PerformLoadPRE(LoadInst *LI, AvailValInBlkVect &ValuesPerBlock,
   LI->replaceAllUsesWith(V);
   if (isa<PHINode>(V))
     V->takeName(LI);
+  if (Instruction *I = dyn_cast<Instruction>(V))
+    I->setDebugLoc(LI->getDebugLoc());
   if (V->getType()->getScalarType()->isPointerTy())
     MD->invalidateCachedPointerInfo(V);
   markInstructionForDeletion(LI);
@@ -1763,6 +1743,8 @@ bool GVN::processNonLocalLoad(LoadInst *LI) {
 
     if (isa<PHINode>(V))
       V->takeName(LI);
+    if (Instruction *I = dyn_cast<Instruction>(V))
+      I->setDebugLoc(LI->getDebugLoc());
     if (V->getType()->getScalarType()->isPointerTy())
       MD->invalidateCachedPointerInfo(V);
     markInstructionForDeletion(LI);
@@ -1783,13 +1765,9 @@ static void patchReplacementInstruction(Instruction *I, Value *Repl) {
   // being replaced.
   BinaryOperator *Op = dyn_cast<BinaryOperator>(I);
   BinaryOperator *ReplOp = dyn_cast<BinaryOperator>(Repl);
-  if (Op && ReplOp && isa<OverflowingBinaryOperator>(Op) &&
-      isa<OverflowingBinaryOperator>(ReplOp)) {
-    if (ReplOp->hasNoSignedWrap() && !Op->hasNoSignedWrap())
-      ReplOp->setHasNoSignedWrap(false);
-    if (ReplOp->hasNoUnsignedWrap() && !Op->hasNoUnsignedWrap())
-      ReplOp->setHasNoUnsignedWrap(false);
-  }
+  if (Op && ReplOp)
+    ReplOp->andIRFlags(Op);
+
   if (Instruction *ReplInst = dyn_cast<Instruction>(Repl)) {
     // FIXME: If both the original and replacement value are part of the
     // same control-flow region (meaning that the execution of one
@@ -1800,7 +1778,7 @@ static void patchReplacementInstruction(Instruction *I, Value *Repl) {
     // In general, GVN unifies expressions over different control-flow
     // regions, and so we need a conservative combination of the noalias
     // scopes.
-    unsigned KnownIDs[] = {
+    static const unsigned KnownIDs[] = {
       LLVMContext::MD_tbaa,
       LLVMContext::MD_alias_scope,
       LLVMContext::MD_noalias,
@@ -1930,8 +1908,9 @@ bool GVN::processLoad(LoadInst *L) {
     // actually have the same type.  See if we know how to reuse the stored
     // value (depending on its type).
     if (StoredVal->getType() != L->getType()) {
+      IRBuilder<> Builder(L);
       StoredVal =
-          CoerceAvailableValueToLoadType(StoredVal, L->getType(), L, DL);
+          CoerceAvailableValueToLoadType(StoredVal, L->getType(), Builder, DL);
       if (!StoredVal)
         return false;
 
@@ -1955,7 +1934,9 @@ bool GVN::processLoad(LoadInst *L) {
     // the same type.  See if we know how to reuse the previously loaded value
     // (depending on its type).
     if (DepLI->getType() != L->getType()) {
-      AvailableVal = CoerceAvailableValueToLoadType(DepLI, L->getType(), L, DL);
+      IRBuilder<> Builder(L);
+      AvailableVal =
+          CoerceAvailableValueToLoadType(DepLI, L->getType(), Builder, DL);
       if (!AvailableVal)
         return false;
 
@@ -2033,23 +2014,6 @@ Value *GVN::findLeader(const BasicBlock *BB, uint32_t num) {
   return Val;
 }
 
-/// Replace all uses of 'From' with 'To' if the use is dominated by the given
-/// basic block.  Returns the number of uses that were replaced.
-unsigned GVN::replaceAllDominatedUsesWith(Value *From, Value *To,
-                                          const BasicBlockEdge &Root) {
-  unsigned Count = 0;
-  for (Value::use_iterator UI = From->use_begin(), UE = From->use_end();
-       UI != UE; ) {
-    Use &U = *UI++;
-
-    if (DT->dominates(Root, U)) {
-      U.set(To);
-      ++Count;
-    }
-  }
-  return Count;
-}
-
 /// There is an edge from 'Src' to 'Dst'.  Return
 /// true if every path from the entry block to 'Dst' passes via this edge.  In
 /// particular 'Dst' must not be reachable via another edge from 'Src'.
@@ -2126,7 +2090,7 @@ bool GVN::propagateEquality(Value *LHS, Value *RHS,
     // LHS always has at least one use that is not dominated by Root, this will
     // never do anything if LHS has only one use.
     if (!LHS->hasOneUse()) {
-      unsigned NumReplacements = replaceAllDominatedUsesWith(LHS, RHS, Root);
+      unsigned NumReplacements = replaceDominatedUsesWith(LHS, RHS, *DT, Root);
       Changed |= NumReplacements > 0;
       NumGVNEqProp += NumReplacements;
     }
@@ -2198,7 +2162,7 @@ bool GVN::propagateEquality(Value *LHS, Value *RHS,
         Value *NotCmp = findLeader(Root.getEnd(), Num);
         if (NotCmp && isa<Instruction>(NotCmp)) {
           unsigned NumReplacements =
-            replaceAllDominatedUsesWith(NotCmp, NotVal, Root);
+            replaceDominatedUsesWith(NotCmp, NotVal, *DT, Root);
           Changed |= NumReplacements > 0;
           NumGVNEqProp += NumReplacements;
         }
@@ -2361,8 +2325,8 @@ bool GVN::runOnFunction(Function& F) {
   for (Function::iterator FI = F.begin(), FE = F.end(); FI != FE; ) {
     BasicBlock *BB = FI++;
 
-    bool removedBlock = MergeBlockIntoPredecessor(
-        BB, DT, /* LoopInfo */ nullptr, VN.getAliasAnalysis(), MD);
+    bool removedBlock =
+        MergeBlockIntoPredecessor(BB, DT, /* LoopInfo */ nullptr, MD);
     if (removedBlock) ++NumGVNBlocks;
 
     Changed |= removedBlock;
@@ -2600,18 +2564,8 @@ bool GVN::performScalarPRE(Instruction *CurInst) {
   addToLeaderTable(ValNo, Phi, CurrentBlock);
   Phi->setDebugLoc(CurInst->getDebugLoc());
   CurInst->replaceAllUsesWith(Phi);
-  if (Phi->getType()->getScalarType()->isPointerTy()) {
-    // Because we have added a PHI-use of the pointer value, it has now
-    // "escaped" from alias analysis' perspective.  We need to inform
-    // AA of this.
-    for (unsigned ii = 0, ee = Phi->getNumIncomingValues(); ii != ee; ++ii) {
-      unsigned jj = PHINode::getOperandNumForIncomingValue(ii);
-      VN.getAliasAnalysis()->addEscapingUse(Phi->getOperandUse(jj));
-    }
-
-    if (MD)
-      MD->invalidateCachedPointerInfo(Phi);
-  }
+  if (MD && Phi->getType()->getScalarType()->isPointerTy())
+    MD->invalidateCachedPointerInfo(Phi);
   VN.erase(CurInst);
   removeFromLeaderTable(ValNo, CurInst, CurrentBlock);
 
@@ -2655,8 +2609,8 @@ bool GVN::performPRE(Function &F) {
 /// Split the critical edge connecting the given two blocks, and return
 /// the block inserted to the critical edge.
 BasicBlock *GVN::splitCriticalEdges(BasicBlock *Pred, BasicBlock *Succ) {
-  BasicBlock *BB = SplitCriticalEdge(
-      Pred, Succ, CriticalEdgeSplittingOptions(getAliasAnalysis(), DT));
+  BasicBlock *BB =
+      SplitCriticalEdge(Pred, Succ, CriticalEdgeSplittingOptions(DT));
   if (MD)
     MD->invalidateCachedPredecessors();
   return BB;
@@ -2670,7 +2624,7 @@ bool GVN::splitCriticalEdges() {
   do {
     std::pair<TerminatorInst*, unsigned> Edge = toSplit.pop_back_val();
     SplitCriticalEdge(Edge.first, Edge.second,
-                      CriticalEdgeSplittingOptions(getAliasAnalysis(), DT));
+                      CriticalEdgeSplittingOptions(DT));
   } while (!toSplit.empty());
   if (MD) MD->invalidateCachedPredecessors();
   return true;
@@ -2822,6 +2776,10 @@ bool GVN::processFoldableCondBr(BranchInst *BI) {
   if (!BI || BI->isUnconditional())
     return false;
 
+  // If a branch has two identical successors, we cannot declare either dead.
+  if (BI->getSuccessor(0) == BI->getSuccessor(1))
+    return false;
+
   ConstantInt *Cond = dyn_cast<ConstantInt>(BI->getCondition());
   if (!Cond)
     return false;
diff --git a/lib/Transforms/Scalar/IndVarSimplify.cpp b/lib/Transforms/Scalar/IndVarSimplify.cpp
index ab8e5b8cc02..2a954d9961f 100644
--- a/lib/Transforms/Scalar/IndVarSimplify.cpp
+++ b/lib/Transforms/Scalar/IndVarSimplify.cpp
@@ -41,6 +41,7 @@
 #include "llvm/IR/Instructions.h"
 #include "llvm/IR/IntrinsicInst.h"
 #include "llvm/IR/LLVMContext.h"
+#include "llvm/IR/PatternMatch.h"
 #include "llvm/IR/Type.h"
 #include "llvm/Support/CommandLine.h"
 #include "llvm/Support/Debug.h"
@@ -68,6 +69,22 @@ static cl::opt<bool> VerifyIndvars(
 static cl::opt<bool> ReduceLiveIVs("liv-reduce", cl::Hidden,
   cl::desc("Reduce live induction variables."));
 
+enum ReplaceExitVal { NeverRepl, OnlyCheapRepl, AlwaysRepl };
+
+static cl::opt<ReplaceExitVal> ReplaceExitValue(
+    "replexitval", cl::Hidden, cl::init(OnlyCheapRepl),
+    cl::desc("Choose the strategy to replace exit value in IndVarSimplify"),
+    cl::values(clEnumValN(NeverRepl, "never", "never replace exit value"),
+               clEnumValN(OnlyCheapRepl, "cheap",
+                          "only replace exit value when the cost is cheap"),
+               clEnumValN(AlwaysRepl, "always",
+                          "always replace exit value whenever possible"),
+               clEnumValEnd));
+
+namespace {
+struct RewritePhi;
+}
+
 namespace {
   class IndVarSimplify : public LoopPass {
     LoopInfo                  *LI;
@@ -112,12 +129,17 @@ namespace {
 
     void SimplifyAndExtend(Loop *L, SCEVExpander &Rewriter, LPPassManager &LPM);
 
+    bool CanLoopBeDeleted(Loop *L, SmallVector<RewritePhi, 8> &RewritePhiSet);
     void RewriteLoopExitValues(Loop *L, SCEVExpander &Rewriter);
 
     Value *LinearFunctionTestReplace(Loop *L, const SCEV *BackedgeTakenCount,
                                      PHINode *IndVar, SCEVExpander &Rewriter);
 
     void SinkUnusedInvariants(Loop *L);
+
+    Value *ExpandSCEVIfNeeded(SCEVExpander &Rewriter, const SCEV *S, Loop *L,
+                              Instruction *InsertPt, Type *Ty,
+                              bool &IsHighCostExpansion);
   };
 }
 
@@ -464,6 +486,67 @@ void IndVarSimplify::RewriteNonIntegerIVs(Loop *L) {
     SE->forgetLoop(L);
 }
 
+namespace {
+// Collect information about PHI nodes which can be transformed in
+// RewriteLoopExitValues.
+struct RewritePhi {
+  PHINode *PN;
+  unsigned Ith;  // Ith incoming value.
+  Value *Val;    // Exit value after expansion.
+  bool HighCost; // High Cost when expansion.
+  bool SafePhi;  // LCSSASafePhiForRAUW.
+
+  RewritePhi(PHINode *P, unsigned I, Value *V, bool H, bool S)
+      : PN(P), Ith(I), Val(V), HighCost(H), SafePhi(S) {}
+};
+}
+
+Value *IndVarSimplify::ExpandSCEVIfNeeded(SCEVExpander &Rewriter, const SCEV *S,
+                                          Loop *L, Instruction *InsertPt,
+                                          Type *ResultTy,
+                                          bool &IsHighCostExpansion) {
+  using namespace llvm::PatternMatch;
+
+  if (!Rewriter.isHighCostExpansion(S, L)) {
+    IsHighCostExpansion = false;
+    return Rewriter.expandCodeFor(S, ResultTy, InsertPt);
+  }
+
+  // Before expanding S into an expensive LLVM expression, see if we can use an
+  // already existing value as the expansion for S.  There is potential to make
+  // this significantly smarter, but this simple heuristic already gets some
+  // interesting cases.
+
+  SmallVector<BasicBlock *, 4> Latches;
+  L->getLoopLatches(Latches);
+
+  for (BasicBlock *BB : Latches) {
+    ICmpInst::Predicate Pred;
+    Instruction *LHS, *RHS;
+    BasicBlock *TrueBB, *FalseBB;
+
+    if (!match(BB->getTerminator(),
+               m_Br(m_ICmp(Pred, m_Instruction(LHS), m_Instruction(RHS)),
+                    TrueBB, FalseBB)))
+      continue;
+
+    if (SE->getSCEV(LHS) == S && DT->dominates(LHS, InsertPt)) {
+      IsHighCostExpansion = false;
+      return LHS;
+    }
+
+    if (SE->getSCEV(RHS) == S && DT->dominates(RHS, InsertPt)) {
+      IsHighCostExpansion = false;
+      return RHS;
+    }
+  }
+
+  // We didn't find anything, fall back to using SCEVExpander.
+  assert(Rewriter.isHighCostExpansion(S, L) && "this should not have changed!");
+  IsHighCostExpansion = true;
+  return Rewriter.expandCodeFor(S, ResultTy, InsertPt);
+}
+
 //===----------------------------------------------------------------------===//
 // RewriteLoopExitValues - Optimize IV users outside the loop.
 // As a side effect, reduces the amount of IV processing within the loop.
@@ -486,6 +569,7 @@ void IndVarSimplify::RewriteLoopExitValues(Loop *L, SCEVExpander &Rewriter) {
   SmallVector<BasicBlock*, 8> ExitBlocks;
   L->getUniqueExitBlocks(ExitBlocks);
 
+  SmallVector<RewritePhi, 8> RewritePhiSet;
   // Find all values that are computed inside the loop, but used outside of it.
   // Because of LCSSA, these values will only occur in LCSSA PHI Nodes.  Scan
   // the exit blocks of the loop to find them.
@@ -595,7 +679,9 @@ void IndVarSimplify::RewriteLoopExitValues(Loop *L, SCEVExpander &Rewriter) {
             continue;
         }
 
-        Value *ExitVal = Rewriter.expandCodeFor(ExitValue, PN->getType(), Inst);
+        bool HighCost = false;
+        Value *ExitVal = ExpandSCEVIfNeeded(Rewriter, ExitValue, L, Inst,
+                                            PN->getType(), HighCost);
 
         DEBUG(dbgs() << "INDVARS: RLEV: AfterLoopVal = " << *ExitVal << '\n'
                      << "  LoopVal = " << *Inst << "\n");
@@ -604,34 +690,43 @@ void IndVarSimplify::RewriteLoopExitValues(Loop *L, SCEVExpander &Rewriter) {
           DeadInsts.push_back(ExitVal);
           continue;
         }
-        Changed = true;
-        ++NumReplaced;
 
-        PN->setIncomingValue(i, ExitVal);
+        // Collect all the candidate PHINodes to be rewritten.
+        RewritePhiSet.push_back(
+            RewritePhi(PN, i, ExitVal, HighCost, LCSSASafePhiForRAUW));
+      }
+    }
+  }
 
-        // If this instruction is dead now, delete it. Don't do it now to avoid
-        // invalidating iterators.
-        if (isInstructionTriviallyDead(Inst, TLI))
-          DeadInsts.push_back(Inst);
+  bool LoopCanBeDel = CanLoopBeDeleted(L, RewritePhiSet);
 
-        // If we determined that this PHI is safe to replace even if an LCSSA
-        // PHI, do so.
-        if (LCSSASafePhiForRAUW) {
-          PN->replaceAllUsesWith(ExitVal);
-          PN->eraseFromParent();
-        }
-      }
+  // Transformation.
+  for (const RewritePhi &Phi : RewritePhiSet) {
+    PHINode *PN = Phi.PN;
+    Value *ExitVal = Phi.Val;
 
-      // If we were unable to completely replace the PHI node, clone the PHI
-      // and delete the original one. This lets IVUsers and any other maps
-      // purge the original user from their records.
-      if (!LCSSASafePhiForRAUW) {
-        PHINode *NewPN = cast<PHINode>(PN->clone());
-        NewPN->takeName(PN);
-        NewPN->insertBefore(PN);
-        PN->replaceAllUsesWith(NewPN);
-        PN->eraseFromParent();
-      }
+    // Only do the rewrite when the ExitValue can be expanded cheaply.
+    // If LoopCanBeDel is true, rewrite exit value aggressively.
+    if (ReplaceExitValue == OnlyCheapRepl && !LoopCanBeDel && Phi.HighCost) {
+      DeadInsts.push_back(ExitVal);
+      continue;
+    }
+
+    Changed = true;
+    ++NumReplaced;
+    Instruction *Inst = cast<Instruction>(PN->getIncomingValue(Phi.Ith));
+    PN->setIncomingValue(Phi.Ith, ExitVal);
+
+    // If this instruction is dead now, delete it. Don't do it now to avoid
+    // invalidating iterators.
+    if (isInstructionTriviallyDead(Inst, TLI))
+      DeadInsts.push_back(Inst);
+
+    // If we determined that this PHI is safe to replace even if an LCSSA
+    // PHI, do so.
+    if (Phi.SafePhi) {
+      PN->replaceAllUsesWith(ExitVal);
+      PN->eraseFromParent();
     }
   }
 
@@ -640,6 +735,65 @@ void IndVarSimplify::RewriteLoopExitValues(Loop *L, SCEVExpander &Rewriter) {
   Rewriter.clearInsertPoint();
 }
 
+/// CanLoopBeDeleted - Check whether it is possible to delete the loop after
+/// rewriting exit value. If it is possible, ignore ReplaceExitValue and
+/// do rewriting aggressively.
+bool IndVarSimplify::CanLoopBeDeleted(
+    Loop *L, SmallVector<RewritePhi, 8> &RewritePhiSet) {
+
+  BasicBlock *Preheader = L->getLoopPreheader();
+  // If there is no preheader, the loop will not be deleted.
+  if (!Preheader)
+    return false;
+
+  // In LoopDeletion pass Loop can be deleted when ExitingBlocks.size() > 1.
+  // We obviate multiple ExitingBlocks case for simplicity.
+  // TODO: If we see testcase with multiple ExitingBlocks can be deleted
+  // after exit value rewriting, we can enhance the logic here.
+  SmallVector<BasicBlock *, 4> ExitingBlocks;
+  L->getExitingBlocks(ExitingBlocks);
+  SmallVector<BasicBlock *, 8> ExitBlocks;
+  L->getUniqueExitBlocks(ExitBlocks);
+  if (ExitBlocks.size() > 1 || ExitingBlocks.size() > 1)
+    return false;
+
+  BasicBlock *ExitBlock = ExitBlocks[0];
+  BasicBlock::iterator BI = ExitBlock->begin();
+  while (PHINode *P = dyn_cast<PHINode>(BI)) {
+    Value *Incoming = P->getIncomingValueForBlock(ExitingBlocks[0]);
+
+    // If the Incoming value of P is found in RewritePhiSet, we know it
+    // could be rewritten to use a loop invariant value in transformation
+    // phase later. Skip it in the loop invariant check below.
+    bool found = false;
+    for (const RewritePhi &Phi : RewritePhiSet) {
+      unsigned i = Phi.Ith;
+      if (Phi.PN == P && (Phi.PN)->getIncomingValue(i) == Incoming) {
+        found = true;
+        break;
+      }
+    }
+
+    Instruction *I;
+    if (!found && (I = dyn_cast<Instruction>(Incoming)))
+      if (!L->hasLoopInvariantOperands(I))
+        return false;
+
+    ++BI;
+  }
+
+  for (Loop::block_iterator LI = L->block_begin(), LE = L->block_end();
+       LI != LE; ++LI) {
+    for (BasicBlock::iterator BI = (*LI)->begin(), BE = (*LI)->end(); BI != BE;
+         ++BI) {
+      if (BI->mayHaveSideEffects())
+        return false;
+    }
+  }
+
+  return true;
+}
+
 //===----------------------------------------------------------------------===//
 //  IV Widening - Extend the width of an IV to cover its widest uses.
 //===----------------------------------------------------------------------===//
@@ -915,8 +1069,8 @@ const SCEVAddRecExpr* WidenIV::GetExtendedOperandRecurrence(NarrowIVDefUse DU) {
   return AddRec;
 }
 
-/// GetWideRecurrence - Is this instruction potentially interesting from
-/// IVUsers' perspective after widening it's type? In other words, can the
+/// GetWideRecurrence - Is this instruction potentially interesting for further
+/// simplification after widening it's type? In other words, can the
 /// extend be safely hoisted out of the loop with SCEV reducing the value to a
 /// recurrence on the same loop. If so, return the sign or zero extended
 /// recurrence. Otherwise return NULL.
@@ -1000,7 +1154,7 @@ Instruction *WidenIV::WidenIVUse(NarrowIVDefUse DU, SCEVExpander &Rewriter) {
         IRBuilder<> Builder(WidePhi->getParent()->getFirstInsertionPt());
         Value *Trunc = Builder.CreateTrunc(WidePhi, DU.NarrowDef->getType());
         UsePhi->replaceAllUsesWith(Trunc);
-        DeadInsts.push_back(UsePhi);
+        DeadInsts.emplace_back(UsePhi);
         DEBUG(dbgs() << "INDVARS: Widen lcssa phi " << *UsePhi
               << " to " << *WidePhi << "\n");
       }
@@ -1033,7 +1187,7 @@ Instruction *WidenIV::WidenIVUse(NarrowIVDefUse DU, SCEVExpander &Rewriter) {
             << " replaced by " << *DU.WideDef << "\n");
       ++NumElimExt;
       DU.NarrowUse->replaceAllUsesWith(NewDef);
-      DeadInsts.push_back(DU.NarrowUse);
+      DeadInsts.emplace_back(DU.NarrowUse);
     }
     // Now that the extend is gone, we want to expose it's uses for potential
     // further simplification. We don't need to directly inform SimplifyIVUsers
@@ -1086,7 +1240,7 @@ Instruction *WidenIV::WidenIVUse(NarrowIVDefUse DU, SCEVExpander &Rewriter) {
   if (WideAddRec != SE->getSCEV(WideUse)) {
     DEBUG(dbgs() << "Wide use expression mismatch: " << *WideUse
           << ": " << *SE->getSCEV(WideUse) << " != " << *WideAddRec << "\n");
-    DeadInsts.push_back(WideUse);
+    DeadInsts.emplace_back(WideUse);
     return nullptr;
   }
 
@@ -1183,7 +1337,7 @@ PHINode *WidenIV::CreateWideIV(SCEVExpander &Rewriter) {
 
     // WidenIVUse may have removed the def-use edge.
     if (DU.NarrowDef->use_empty())
-      DeadInsts.push_back(DU.NarrowDef);
+      DeadInsts.emplace_back(DU.NarrowDef);
   }
   return WidePhi;
 }
@@ -1878,7 +2032,8 @@ bool IndVarSimplify::runOnLoop(Loop *L, LPPassManager &LPM) {
   // loop into any instructions outside of the loop that use the final values of
   // the current expressions.
   //
-  if (!isa<SCEVCouldNotCompute>(BackedgeTakenCount))
+  if (ReplaceExitValue != NeverRepl &&
+      !isa<SCEVCouldNotCompute>(BackedgeTakenCount))
     RewriteLoopExitValues(L, Rewriter);
 
   // Eliminate redundant IV cycles.
@@ -1912,7 +2067,7 @@ bool IndVarSimplify::runOnLoop(Loop *L, LPPassManager &LPM) {
   // which are now dead.
   while (!DeadInsts.empty())
     if (Instruction *Inst =
-          dyn_cast_or_null<Instruction>(&*DeadInsts.pop_back_val()))
+            dyn_cast_or_null<Instruction>(DeadInsts.pop_back_val()))
       RecursivelyDeleteTriviallyDeadInstructions(Inst, TLI);
 
   // The Rewriter may not be used from this point on.
diff --git a/lib/Transforms/Scalar/InductiveRangeCheckElimination.cpp b/lib/Transforms/Scalar/InductiveRangeCheckElimination.cpp
index cbdacad8f28..08fdcc38c04 100644
--- a/lib/Transforms/Scalar/InductiveRangeCheckElimination.cpp
+++ b/lib/Transforms/Scalar/InductiveRangeCheckElimination.cpp
@@ -215,7 +215,7 @@ class InductiveRangeCheckElimination : public LoopPass {
     AU.addRequiredID(LoopSimplifyID);
     AU.addRequiredID(LCSSAID);
     AU.addRequired<ScalarEvolution>();
-    AU.addRequired<BranchProbabilityInfo>();
+    AU.addRequired<BranchProbabilityInfoWrapperPass>();
   }
 
   bool runOnLoop(Loop *L, LPPassManager &LPM) override;
@@ -1400,7 +1400,8 @@ bool InductiveRangeCheckElimination::runOnLoop(Loop *L, LPPassManager &LPM) {
   InductiveRangeCheck::AllocatorTy IRCAlloc;
   SmallVector<InductiveRangeCheck *, 16> RangeChecks;
   ScalarEvolution &SE = getAnalysis<ScalarEvolution>();
-  BranchProbabilityInfo &BPI = getAnalysis<BranchProbabilityInfo>();
+  BranchProbabilityInfo &BPI =
+      getAnalysis<BranchProbabilityInfoWrapperPass>().getBPI();
 
   for (auto BBI : L->getBlocks())
     if (BranchInst *TBI = dyn_cast<BranchInst>(BBI->getTerminator()))
diff --git a/lib/Transforms/Scalar/JumpThreading.cpp b/lib/Transforms/Scalar/JumpThreading.cpp
index 83ac915e606..e845a7f586b 100644
--- a/lib/Transforms/Scalar/JumpThreading.cpp
+++ b/lib/Transforms/Scalar/JumpThreading.cpp
@@ -669,7 +669,8 @@ bool JumpThreading::ProcessBlock(BasicBlock *BB) {
   // because now the condition in this block can be threaded through
   // predecessors of our predecessor block.
   if (BasicBlock *SinglePred = BB->getSinglePredecessor()) {
-    if (SinglePred->getTerminator()->getNumSuccessors() == 1 &&
+    const TerminatorInst *TI = SinglePred->getTerminator();
+    if (!TI->isExceptional() && TI->getNumSuccessors() == 1 &&
         SinglePred != BB && !hasAddressTakenAndUsed(BB)) {
       // If SinglePred was a loop header, BB becomes one.
       if (LoopHeaders.erase(SinglePred))
@@ -758,56 +759,33 @@ bool JumpThreading::ProcessBlock(BasicBlock *BB) {
 
 
   if (CmpInst *CondCmp = dyn_cast<CmpInst>(CondInst)) {
-    // For a comparison where the LHS is outside this block, it's possible
-    // that we've branched on it before.  Used LVI to see if we can simplify
-    // the branch based on that.
+    // If we're branching on a conditional, LVI might be able to determine
+    // it's value at the branch instruction.  We only handle comparisons
+    // against a constant at this time.
+    // TODO: This should be extended to handle switches as well.  
     BranchInst *CondBr = dyn_cast<BranchInst>(BB->getTerminator());
     Constant *CondConst = dyn_cast<Constant>(CondCmp->getOperand(1));
-    pred_iterator PI = pred_begin(BB), PE = pred_end(BB);
-    if (CondBr && CondConst && CondBr->isConditional() && PI != PE &&
-        (!isa<Instruction>(CondCmp->getOperand(0)) ||
-         cast<Instruction>(CondCmp->getOperand(0))->getParent() != BB)) {
-      // For predecessor edge, determine if the comparison is true or false
-      // on that edge.  If they're all true or all false, we can simplify the
-      // branch.
-      // FIXME: We could handle mixed true/false by duplicating code.
-      LazyValueInfo::Tristate Baseline =
-        LVI->getPredicateOnEdge(CondCmp->getPredicate(), CondCmp->getOperand(0),
-                                CondConst, *PI, BB, CondCmp);
-      if (Baseline != LazyValueInfo::Unknown) {
-        // Check that all remaining incoming values match the first one.
-        while (++PI != PE) {
-          LazyValueInfo::Tristate Ret =
-            LVI->getPredicateOnEdge(CondCmp->getPredicate(),
-                                    CondCmp->getOperand(0), CondConst, *PI, BB,
-                                    CondCmp);
-          if (Ret != Baseline) break;
-        }
-
-        // If we terminated early, then one of the values didn't match.
-        if (PI == PE) {
-          unsigned ToRemove = Baseline == LazyValueInfo::True ? 1 : 0;
-          unsigned ToKeep = Baseline == LazyValueInfo::True ? 0 : 1;
-          CondBr->getSuccessor(ToRemove)->removePredecessor(BB, true);
-          BranchInst::Create(CondBr->getSuccessor(ToKeep), CondBr);
-          CondBr->eraseFromParent();
-          return true;
-        }
-      }
-
-    } else if (CondBr && CondConst && CondBr->isConditional()) {
-      // There might be an invariant in the same block with the conditional
-      // that can determine the predicate.
-
+    if (CondBr && CondConst && CondBr->isConditional()) {
       LazyValueInfo::Tristate Ret =
         LVI->getPredicateAt(CondCmp->getPredicate(), CondCmp->getOperand(0),
-                            CondConst, CondCmp);
+                            CondConst, CondBr);
       if (Ret != LazyValueInfo::Unknown) {
         unsigned ToRemove = Ret == LazyValueInfo::True ? 1 : 0;
         unsigned ToKeep = Ret == LazyValueInfo::True ? 0 : 1;
         CondBr->getSuccessor(ToRemove)->removePredecessor(BB, true);
         BranchInst::Create(CondBr->getSuccessor(ToKeep), CondBr);
         CondBr->eraseFromParent();
+        if (CondCmp->use_empty())
+          CondCmp->eraseFromParent();
+        else if (CondCmp->getParent() == BB) {
+          // If the fact we just learned is true for all uses of the
+          // condition, replace it with a constant value
+          auto *CI = Ret == LazyValueInfo::True ?
+            ConstantInt::getTrue(CondCmp->getType()) :
+            ConstantInt::getFalse(CondCmp->getType());
+          CondCmp->replaceAllUsesWith(CI);
+          CondCmp->eraseFromParent();
+        }
         return true;
       }
     }
diff --git a/lib/Transforms/Scalar/LICM.cpp b/lib/Transforms/Scalar/LICM.cpp
index 1333b024f7b..8f3839f40bc 100644
--- a/lib/Transforms/Scalar/LICM.cpp
+++ b/lib/Transforms/Scalar/LICM.cpp
@@ -72,24 +72,30 @@ DisablePromotion("disable-licm-promotion", cl::Hidden,
                  cl::desc("Disable memory promotion in LICM pass"));
 
 static bool inSubLoop(BasicBlock *BB, Loop *CurLoop, LoopInfo *LI);
-static bool isNotUsedInLoop(Instruction &I, Loop *CurLoop);
+static bool isNotUsedInLoop(const Instruction &I, const Loop *CurLoop);
 static bool hoist(Instruction &I, BasicBlock *Preheader);
-static bool sink(Instruction &I, LoopInfo *LI, DominatorTree *DT, 
-                 Loop *CurLoop, AliasSetTracker *CurAST );
-static bool isGuaranteedToExecute(Instruction &Inst, DominatorTree *DT,
-                                  Loop *CurLoop, LICMSafetyInfo *SafetyInfo);
-static bool isSafeToExecuteUnconditionally(Instruction &Inst, DominatorTree *DT,
-                                           Loop *CurLoop,
-                                           LICMSafetyInfo *SafetyInfo);
+static bool sink(Instruction &I, const LoopInfo *LI, const DominatorTree *DT,
+                 const Loop *CurLoop, AliasSetTracker *CurAST );
+static bool isGuaranteedToExecute(const Instruction &Inst,
+                                  const DominatorTree *DT,
+                                  const Loop *CurLoop,
+                                  const LICMSafetyInfo *SafetyInfo);
+static bool isSafeToExecuteUnconditionally(const Instruction &Inst,
+                                           const DominatorTree *DT,
+                                           const TargetLibraryInfo *TLI,
+                                           const Loop *CurLoop,
+                                           const LICMSafetyInfo *SafetyInfo,
+                                           const Instruction *CtxI = nullptr);
 static bool pointerInvalidatedByLoop(Value *V, uint64_t Size,
                                      const AAMDNodes &AAInfo, 
                                      AliasSetTracker *CurAST);
-static Instruction *CloneInstructionInExitBlock(Instruction &I,
+static Instruction *CloneInstructionInExitBlock(const Instruction &I,
                                                 BasicBlock &ExitBlock,
-                                                PHINode &PN, LoopInfo *LI);
+                                                PHINode &PN,
+                                                const LoopInfo *LI);
 static bool canSinkOrHoistInst(Instruction &I, AliasAnalysis *AA,
-                               DominatorTree *DT, Loop *CurLoop,
-                               AliasSetTracker *CurAST,
+                               DominatorTree *DT, TargetLibraryInfo *TLI,
+                               Loop *CurLoop, AliasSetTracker *CurAST,
                                LICMSafetyInfo *SafetyInfo);
 
 namespace {
@@ -333,7 +339,7 @@ bool llvm::sinkRegion(DomTreeNode *N, AliasAnalysis *AA, LoopInfo *LI,
     // operands of the instruction are loop invariant.
     //
     if (isNotUsedInLoop(I, CurLoop) &&
-        canSinkOrHoistInst(I, AA, DT, CurLoop, CurAST, SafetyInfo)) {
+        canSinkOrHoistInst(I, AA, DT, TLI, CurLoop, CurAST, SafetyInfo)) {
       ++II;
       Changed |= sink(I, LI, DT, CurLoop, CurAST);
     }
@@ -382,8 +388,9 @@ bool llvm::hoistRegion(DomTreeNode *N, AliasAnalysis *AA, LoopInfo *LI,
       // is safe to hoist the instruction.
       //
       if (CurLoop->hasLoopInvariantOperands(&I) &&
-          canSinkOrHoistInst(I, AA, DT, CurLoop, CurAST, SafetyInfo) &&
-          isSafeToExecuteUnconditionally(I, DT, CurLoop, SafetyInfo))
+          canSinkOrHoistInst(I, AA, DT, TLI, CurLoop, CurAST, SafetyInfo) &&
+          isSafeToExecuteUnconditionally(I, DT, TLI, CurLoop, SafetyInfo,
+                                 CurLoop->getLoopPreheader()->getTerminator()))
         Changed |= hoist(I, CurLoop->getLoopPreheader());
     }
 
@@ -421,8 +428,8 @@ void llvm::computeLICMSafetyInfo(LICMSafetyInfo * SafetyInfo, Loop * CurLoop) {
 /// instruction.
 ///
 bool canSinkOrHoistInst(Instruction &I, AliasAnalysis *AA, DominatorTree *DT,
-                        Loop *CurLoop, AliasSetTracker *CurAST,
-                        LICMSafetyInfo *SafetyInfo) {
+                        TargetLibraryInfo *TLI, Loop *CurLoop,
+                        AliasSetTracker *CurAST, LICMSafetyInfo *SafetyInfo) {
   // Loads have extra constraints we have to verify before we can hoist them.
   if (LoadInst *LI = dyn_cast<LoadInst>(&I)) {
     if (!LI->isUnordered())
@@ -450,8 +457,8 @@ bool canSinkOrHoistInst(Instruction &I, AliasAnalysis *AA, DominatorTree *DT,
       return false;
 
     // Handle simple cases by querying alias analysis.
-    AliasAnalysis::ModRefBehavior Behavior = AA->getModRefBehavior(CI);
-    if (Behavior == AliasAnalysis::DoesNotAccessMemory)
+    FunctionModRefBehavior Behavior = AA->getModRefBehavior(CI);
+    if (Behavior == FMRB_DoesNotAccessMemory)
       return true;
     if (AliasAnalysis::onlyReadsMemory(Behavior)) {
       // If this call only reads from memory and there are no writes to memory
@@ -482,7 +489,11 @@ bool canSinkOrHoistInst(Instruction &I, AliasAnalysis *AA, DominatorTree *DT,
       !isa<InsertValueInst>(I))
     return false;
 
-  return isSafeToExecuteUnconditionally(I, DT, CurLoop, SafetyInfo);
+  // TODO: Plumb the context instruction through to make hoisting and sinking
+  // more powerful. Hoisting of loads already works due to the special casing
+  // above. 
+  return isSafeToExecuteUnconditionally(I, DT, TLI, CurLoop, SafetyInfo,
+                                        nullptr);
 }
 
 /// Returns true if a PHINode is a trivially replaceable with an
@@ -490,9 +501,9 @@ bool canSinkOrHoistInst(Instruction &I, AliasAnalysis *AA, DominatorTree *DT,
 /// This is true when all incoming values are that instruction.
 /// This pattern occurs most often with LCSSA PHI nodes.
 ///
-static bool isTriviallyReplacablePHI(PHINode &PN, Instruction &I) {
-  for (unsigned i = 0, e = PN.getNumIncomingValues(); i != e; ++i)
-    if (PN.getIncomingValue(i) != &I)
+static bool isTriviallyReplacablePHI(const PHINode &PN, const Instruction &I) {
+  for (const Value *IncValue : PN.incoming_values())
+    if (IncValue != &I)
       return false;
 
   return true;
@@ -502,10 +513,10 @@ static bool isTriviallyReplacablePHI(PHINode &PN, Instruction &I) {
 /// the loop. If this is true, we can sink the instruction to the exit
 /// blocks of the loop.
 ///
-static bool isNotUsedInLoop(Instruction &I, Loop *CurLoop) {
-  for (User *U : I.users()) {
-    Instruction *UI = cast<Instruction>(U);
-    if (PHINode *PN = dyn_cast<PHINode>(UI)) {
+static bool isNotUsedInLoop(const Instruction &I, const Loop *CurLoop) {
+  for (const User *U : I.users()) {
+    const Instruction *UI = cast<Instruction>(U);
+    if (const PHINode *PN = dyn_cast<PHINode>(UI)) {
       // A PHI node where all of the incoming values are this instruction are
       // special -- they can just be RAUW'ed with the instruction and thus
       // don't require a use in the predecessor. This is a particular important
@@ -533,9 +544,10 @@ static bool isNotUsedInLoop(Instruction &I, Loop *CurLoop) {
   return true;
 }
 
-static Instruction *CloneInstructionInExitBlock(Instruction &I,
+static Instruction *CloneInstructionInExitBlock(const Instruction &I,
                                                 BasicBlock &ExitBlock,
-                                                PHINode &PN, LoopInfo *LI) {
+                                                PHINode &PN,
+                                                const LoopInfo *LI) {
   Instruction *New = I.clone();
   ExitBlock.getInstList().insert(ExitBlock.getFirstInsertionPt(), New);
   if (!I.getName().empty()) New->setName(I.getName() + ".le");
@@ -567,8 +579,8 @@ static Instruction *CloneInstructionInExitBlock(Instruction &I,
 /// This method is guaranteed to remove the original instruction from its
 /// position, and may either delete it or move it to outside of the loop.
 ///
-static bool sink(Instruction &I, LoopInfo *LI, DominatorTree *DT, 
-                 Loop *CurLoop, AliasSetTracker *CurAST ) {
+static bool sink(Instruction &I, const LoopInfo *LI, const DominatorTree *DT,
+                 const Loop *CurLoop, AliasSetTracker *CurAST ) {
   DEBUG(dbgs() << "LICM sinking instruction: " << I << "\n");
   bool Changed = false;
   if (isa<LoadInst>(I)) ++NumMovedLoads;
@@ -590,7 +602,8 @@ static bool sink(Instruction &I, LoopInfo *LI, DominatorTree *DT,
   // PHI nodes in exit blocks due to LCSSA form. Just RAUW them with clones of
   // the instruction.
   while (!I.use_empty()) {
-    Instruction *User = I.user_back();
+    Value::user_iterator UI = I.user_begin();
+    auto *User = cast<Instruction>(*UI);
     if (!DT->isReachableFromEntry(User->getParent())) {
       User->replaceUsesOfWith(&I, UndefValue::get(I.getType()));
       continue;
@@ -598,6 +611,16 @@ static bool sink(Instruction &I, LoopInfo *LI, DominatorTree *DT,
     // The user must be a PHI node.
     PHINode *PN = cast<PHINode>(User);
 
+    // Surprisingly, instructions can be used outside of loops without any
+    // exits.  This can only happen in PHI nodes if the incoming block is
+    // unreachable.
+    Use &U = UI.getUse();
+    BasicBlock *BB = PN->getIncomingBlock(U);
+    if (!DT->isReachableFromEntry(BB)) {
+      U = UndefValue::get(I.getType());
+      continue;
+    }
+
     BasicBlock *ExitBlock = PN->getParent();
     assert(ExitBlockSet.count(ExitBlock) &&
            "The LCSSA PHI is not in an exit block!");
@@ -634,21 +657,25 @@ static bool hoist(Instruction &I, BasicBlock *Preheader) {
   return true;
 }
 
-/// Only sink or hoist an instruction if it is not a trapping instruction
+/// Only sink or hoist an instruction if it is not a trapping instruction,
+/// or if the instruction is known not to trap when moved to the preheader.
 /// or if it is a trapping instruction and is guaranteed to execute.
-///
-static bool isSafeToExecuteUnconditionally(Instruction &Inst, DominatorTree *DT,
-                                           Loop *CurLoop,
-                                           LICMSafetyInfo *SafetyInfo) {
-  // If it is not a trapping instruction, it is always safe to hoist.
-  if (isSafeToSpeculativelyExecute(&Inst))
+static bool isSafeToExecuteUnconditionally(const Instruction &Inst, 
+                                           const DominatorTree *DT,
+                                           const TargetLibraryInfo *TLI,
+                                           const Loop *CurLoop,
+                                           const LICMSafetyInfo *SafetyInfo,
+                                           const Instruction *CtxI) {
+  if (isSafeToSpeculativelyExecute(&Inst, CtxI, DT, TLI))
     return true;
 
   return isGuaranteedToExecute(Inst, DT, CurLoop, SafetyInfo);
 }
 
-static bool isGuaranteedToExecute(Instruction &Inst, DominatorTree *DT, 
-                                  Loop *CurLoop, LICMSafetyInfo * SafetyInfo) {
+static bool isGuaranteedToExecute(const Instruction &Inst,
+                                  const DominatorTree *DT,
+                                  const Loop *CurLoop,
+                                  const LICMSafetyInfo * SafetyInfo) {
 
   // We have to check to make sure that the instruction dominates all
   // of the exit blocks.  If it doesn't, then there is a path out of the loop
@@ -704,17 +731,18 @@ namespace {
             // We need to create an LCSSA PHI node for the incoming value and
             // store that.
             PHINode *PN = PHINode::Create(
-                I->getType(), PredCache.GetNumPreds(BB),
+                I->getType(), PredCache.size(BB),
                 I->getName() + ".lcssa", BB->begin());
-            for (BasicBlock **PI = PredCache.GetPreds(BB); *PI; ++PI)
-              PN->addIncoming(I, *PI);
+            for (BasicBlock *Pred : PredCache.get(BB))
+              PN->addIncoming(I, Pred);
             return PN;
           }
       return V;
     }
 
   public:
-    LoopPromoter(Value *SP, const SmallVectorImpl<Instruction *> &Insts,
+    LoopPromoter(Value *SP,
+                 ArrayRef<const Instruction *> Insts,
                  SSAUpdater &S, SmallPtrSetImpl<Value *> &PMA,
                  SmallVectorImpl<BasicBlock *> &LEB,
                  SmallVectorImpl<Instruction *> &LIP, PredIteratorCache &PIC,
@@ -839,11 +867,11 @@ bool llvm::promoteLoopAccessesToScalars(AliasSet &AS,
 
       // If there is an non-load/store instruction in the loop, we can't promote
       // it.
-      if (LoadInst *load = dyn_cast<LoadInst>(UI)) {
+      if (const LoadInst *load = dyn_cast<LoadInst>(UI)) {
         assert(!load->isVolatile() && "AST broken");
         if (!load->isSimple())
           return Changed;
-      } else if (StoreInst *store = dyn_cast<StoreInst>(UI)) {
+      } else if (const StoreInst *store = dyn_cast<StoreInst>(UI)) {
         // Stores *of* the pointer are not interesting, only stores *to* the
         // pointer.
         if (UI->getOperand(1) != ASIV)
@@ -920,7 +948,8 @@ bool llvm::promoteLoopAccessesToScalars(AliasSet &AS,
   // We use the SSAUpdater interface to insert phi nodes as required.
   SmallVector<PHINode*, 16> NewPHIs;
   SSAUpdater SSA(&NewPHIs);
-  LoopPromoter Promoter(SomePtr, LoopUses, SSA, PointerMustAliases, ExitBlocks,
+  LoopPromoter Promoter(SomePtr, LoopUses, SSA,
+                        PointerMustAliases, ExitBlocks,
                         InsertPts, PIC, *CurAST, *LI, DL, Alignment, AATags);
 
   // Set up the preheader to have a definition of the value.  It is the live-out
diff --git a/lib/Transforms/Scalar/LoopDistribute.cpp b/lib/Transforms/Scalar/LoopDistribute.cpp
new file mode 100644
index 00000000000..b2a3e28c3f0
--- /dev/null
+++ b/lib/Transforms/Scalar/LoopDistribute.cpp
@@ -0,0 +1,838 @@
+//===- LoopDistribute.cpp - Loop Distribution Pass ------------------------===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This file implements the Loop Distribution Pass.  Its main focus is to
+// distribute loops that cannot be vectorized due to dependence cycles.  It
+// tries to isolate the offending dependences into a new loop allowing
+// vectorization of the remaining parts.
+//
+// For dependence analysis, the pass uses the LoopVectorizer's
+// LoopAccessAnalysis.  Because this analysis presumes no change in the order of
+// memory operations, special care is taken to preserve the lexical order of
+// these operations.
+//
+// Similarly to the Vectorizer, the pass also supports loop versioning to
+// run-time disambiguate potentially overlapping arrays.
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/ADT/DepthFirstIterator.h"
+#include "llvm/ADT/EquivalenceClasses.h"
+#include "llvm/ADT/STLExtras.h"
+#include "llvm/ADT/Statistic.h"
+#include "llvm/Analysis/LoopAccessAnalysis.h"
+#include "llvm/Analysis/LoopInfo.h"
+#include "llvm/IR/Dominators.h"
+#include "llvm/Pass.h"
+#include "llvm/Support/CommandLine.h"
+#include "llvm/Support/Debug.h"
+#include "llvm/Transforms/Utils/BasicBlockUtils.h"
+#include "llvm/Transforms/Utils/Cloning.h"
+#include "llvm/Transforms/Utils/LoopVersioning.h"
+#include <list>
+
+#define LDIST_NAME "loop-distribute"
+#define DEBUG_TYPE LDIST_NAME
+
+using namespace llvm;
+
+static cl::opt<bool>
+    LDistVerify("loop-distribute-verify", cl::Hidden,
+                cl::desc("Turn on DominatorTree and LoopInfo verification "
+                         "after Loop Distribution"),
+                cl::init(false));
+
+static cl::opt<bool> DistributeNonIfConvertible(
+    "loop-distribute-non-if-convertible", cl::Hidden,
+    cl::desc("Whether to distribute into a loop that may not be "
+             "if-convertible by the loop vectorizer"),
+    cl::init(false));
+
+STATISTIC(NumLoopsDistributed, "Number of loops distributed");
+
+namespace {
+/// \brief Maintains the set of instructions of the loop for a partition before
+/// cloning.  After cloning, it hosts the new loop.
+class InstPartition {
+  typedef SmallPtrSet<Instruction *, 8> InstructionSet;
+
+public:
+  InstPartition(Instruction *I, Loop *L, bool DepCycle = false)
+      : DepCycle(DepCycle), OrigLoop(L), ClonedLoop(nullptr) {
+    Set.insert(I);
+  }
+
+  /// \brief Returns whether this partition contains a dependence cycle.
+  bool hasDepCycle() const { return DepCycle; }
+
+  /// \brief Adds an instruction to this partition.
+  void add(Instruction *I) { Set.insert(I); }
+
+  /// \brief Collection accessors.
+  InstructionSet::iterator begin() { return Set.begin(); }
+  InstructionSet::iterator end() { return Set.end(); }
+  InstructionSet::const_iterator begin() const { return Set.begin(); }
+  InstructionSet::const_iterator end() const { return Set.end(); }
+  bool empty() const { return Set.empty(); }
+
+  /// \brief Moves this partition into \p Other.  This partition becomes empty
+  /// after this.
+  void moveTo(InstPartition &Other) {
+    Other.Set.insert(Set.begin(), Set.end());
+    Set.clear();
+    Other.DepCycle |= DepCycle;
+  }
+
+  /// \brief Populates the partition with a transitive closure of all the
+  /// instructions that the seeded instructions dependent on.
+  void populateUsedSet() {
+    // FIXME: We currently don't use control-dependence but simply include all
+    // blocks (possibly empty at the end) and let simplifycfg mostly clean this
+    // up.
+    for (auto *B : OrigLoop->getBlocks())
+      Set.insert(B->getTerminator());
+
+    // Follow the use-def chains to form a transitive closure of all the
+    // instructions that the originally seeded instructions depend on.
+    SmallVector<Instruction *, 8> Worklist(Set.begin(), Set.end());
+    while (!Worklist.empty()) {
+      Instruction *I = Worklist.pop_back_val();
+      // Insert instructions from the loop that we depend on.
+      for (Value *V : I->operand_values()) {
+        auto *I = dyn_cast<Instruction>(V);
+        if (I && OrigLoop->contains(I->getParent()) && Set.insert(I).second)
+          Worklist.push_back(I);
+      }
+    }
+  }
+
+  /// \brief Clones the original loop.
+  ///
+  /// Updates LoopInfo and DominatorTree using the information that block \p
+  /// LoopDomBB dominates the loop.
+  Loop *cloneLoopWithPreheader(BasicBlock *InsertBefore, BasicBlock *LoopDomBB,
+                               unsigned Index, LoopInfo *LI,
+                               DominatorTree *DT) {
+    ClonedLoop = ::cloneLoopWithPreheader(InsertBefore, LoopDomBB, OrigLoop,
+                                          VMap, Twine(".ldist") + Twine(Index),
+                                          LI, DT, ClonedLoopBlocks);
+    return ClonedLoop;
+  }
+
+  /// \brief The cloned loop.  If this partition is mapped to the original loop,
+  /// this is null.
+  const Loop *getClonedLoop() const { return ClonedLoop; }
+
+  /// \brief Returns the loop where this partition ends up after distribution.
+  /// If this partition is mapped to the original loop then use the block from
+  /// the loop.
+  const Loop *getDistributedLoop() const {
+    return ClonedLoop ? ClonedLoop : OrigLoop;
+  }
+
+  /// \brief The VMap that is populated by cloning and then used in
+  /// remapinstruction to remap the cloned instructions.
+  ValueToValueMapTy &getVMap() { return VMap; }
+
+  /// \brief Remaps the cloned instructions using VMap.
+  void remapInstructions() {
+    remapInstructionsInBlocks(ClonedLoopBlocks, VMap);
+  }
+
+  /// \brief Based on the set of instructions selected for this partition,
+  /// removes the unnecessary ones.
+  void removeUnusedInsts() {
+    SmallVector<Instruction *, 8> Unused;
+
+    for (auto *Block : OrigLoop->getBlocks())
+      for (auto &Inst : *Block)
+        if (!Set.count(&Inst)) {
+          Instruction *NewInst = &Inst;
+          if (!VMap.empty())
+            NewInst = cast<Instruction>(VMap[NewInst]);
+
+          assert(!isa<BranchInst>(NewInst) &&
+                 "Branches are marked used early on");
+          Unused.push_back(NewInst);
+        }
+
+    // Delete the instructions backwards, as it has a reduced likelihood of
+    // having to update as many def-use and use-def chains.
+    for (auto *Inst : make_range(Unused.rbegin(), Unused.rend())) {
+      if (!Inst->use_empty())
+        Inst->replaceAllUsesWith(UndefValue::get(Inst->getType()));
+      Inst->eraseFromParent();
+    }
+  }
+
+  void print() const {
+    if (DepCycle)
+      dbgs() << "  (cycle)\n";
+    for (auto *I : Set)
+      // Prefix with the block name.
+      dbgs() << "  " << I->getParent()->getName() << ":" << *I << "\n";
+  }
+
+  void printBlocks() const {
+    for (auto *BB : getDistributedLoop()->getBlocks())
+      dbgs() << *BB;
+  }
+
+private:
+  /// \brief Instructions from OrigLoop selected for this partition.
+  InstructionSet Set;
+
+  /// \brief Whether this partition contains a dependence cycle.
+  bool DepCycle;
+
+  /// \brief The original loop.
+  Loop *OrigLoop;
+
+  /// \brief The cloned loop.  If this partition is mapped to the original loop,
+  /// this is null.
+  Loop *ClonedLoop;
+
+  /// \brief The blocks of ClonedLoop including the preheader.  If this
+  /// partition is mapped to the original loop, this is empty.
+  SmallVector<BasicBlock *, 8> ClonedLoopBlocks;
+
+  /// \brief These gets populated once the set of instructions have been
+  /// finalized. If this partition is mapped to the original loop, these are not
+  /// set.
+  ValueToValueMapTy VMap;
+};
+
+/// \brief Holds the set of Partitions.  It populates them, merges them and then
+/// clones the loops.
+class InstPartitionContainer {
+  typedef DenseMap<Instruction *, int> InstToPartitionIdT;
+
+public:
+  InstPartitionContainer(Loop *L, LoopInfo *LI, DominatorTree *DT)
+      : L(L), LI(LI), DT(DT) {}
+
+  /// \brief Returns the number of partitions.
+  unsigned getSize() const { return PartitionContainer.size(); }
+
+  /// \brief Adds \p Inst into the current partition if that is marked to
+  /// contain cycles.  Otherwise start a new partition for it.
+  void addToCyclicPartition(Instruction *Inst) {
+    // If the current partition is non-cyclic.  Start a new one.
+    if (PartitionContainer.empty() || !PartitionContainer.back().hasDepCycle())
+      PartitionContainer.emplace_back(Inst, L, /*DepCycle=*/true);
+    else
+      PartitionContainer.back().add(Inst);
+  }
+
+  /// \brief Adds \p Inst into a partition that is not marked to contain
+  /// dependence cycles.
+  ///
+  //  Initially we isolate memory instructions into as many partitions as
+  //  possible, then later we may merge them back together.
+  void addToNewNonCyclicPartition(Instruction *Inst) {
+    PartitionContainer.emplace_back(Inst, L);
+  }
+
+  /// \brief Merges adjacent non-cyclic partitions.
+  ///
+  /// The idea is that we currently only want to isolate the non-vectorizable
+  /// partition.  We could later allow more distribution among these partition
+  /// too.
+  void mergeAdjacentNonCyclic() {
+    mergeAdjacentPartitionsIf(
+        [](const InstPartition *P) { return !P->hasDepCycle(); });
+  }
+
+  /// \brief If a partition contains only conditional stores, we won't vectorize
+  /// it.  Try to merge it with a previous cyclic partition.
+  void mergeNonIfConvertible() {
+    mergeAdjacentPartitionsIf([&](const InstPartition *Partition) {
+      if (Partition->hasDepCycle())
+        return true;
+
+      // Now, check if all stores are conditional in this partition.
+      bool seenStore = false;
+
+      for (auto *Inst : *Partition)
+        if (isa<StoreInst>(Inst)) {
+          seenStore = true;
+          if (!LoopAccessInfo::blockNeedsPredication(Inst->getParent(), L, DT))
+            return false;
+        }
+      return seenStore;
+    });
+  }
+
+  /// \brief Merges the partitions according to various heuristics.
+  void mergeBeforePopulating() {
+    mergeAdjacentNonCyclic();
+    if (!DistributeNonIfConvertible)
+      mergeNonIfConvertible();
+  }
+
+  /// \brief Merges partitions in order to ensure that no loads are duplicated.
+  ///
+  /// We can't duplicate loads because that could potentially reorder them.
+  /// LoopAccessAnalysis provides dependency information with the context that
+  /// the order of memory operation is preserved.
+  ///
+  /// Return if any partitions were merged.
+  bool mergeToAvoidDuplicatedLoads() {
+    typedef DenseMap<Instruction *, InstPartition *> LoadToPartitionT;
+    typedef EquivalenceClasses<InstPartition *> ToBeMergedT;
+
+    LoadToPartitionT LoadToPartition;
+    ToBeMergedT ToBeMerged;
+
+    // Step through the partitions and create equivalence between partitions
+    // that contain the same load.  Also put partitions in between them in the
+    // same equivalence class to avoid reordering of memory operations.
+    for (PartitionContainerT::iterator I = PartitionContainer.begin(),
+                                       E = PartitionContainer.end();
+         I != E; ++I) {
+      auto *PartI = &*I;
+
+      // If a load occurs in two partitions PartI and PartJ, merge all
+      // partitions (PartI, PartJ] into PartI.
+      for (Instruction *Inst : *PartI)
+        if (isa<LoadInst>(Inst)) {
+          bool NewElt;
+          LoadToPartitionT::iterator LoadToPart;
+
+          std::tie(LoadToPart, NewElt) =
+              LoadToPartition.insert(std::make_pair(Inst, PartI));
+          if (!NewElt) {
+            DEBUG(dbgs() << "Merging partitions due to this load in multiple "
+                         << "partitions: " << PartI << ", "
+                         << LoadToPart->second << "\n" << *Inst << "\n");
+
+            auto PartJ = I;
+            do {
+              --PartJ;
+              ToBeMerged.unionSets(PartI, &*PartJ);
+            } while (&*PartJ != LoadToPart->second);
+          }
+        }
+    }
+    if (ToBeMerged.empty())
+      return false;
+
+    // Merge the member of an equivalence class into its class leader.  This
+    // makes the members empty.
+    for (ToBeMergedT::iterator I = ToBeMerged.begin(), E = ToBeMerged.end();
+         I != E; ++I) {
+      if (!I->isLeader())
+        continue;
+
+      auto PartI = I->getData();
+      for (auto PartJ : make_range(std::next(ToBeMerged.member_begin(I)),
+                                   ToBeMerged.member_end())) {
+        PartJ->moveTo(*PartI);
+      }
+    }
+
+    // Remove the empty partitions.
+    PartitionContainer.remove_if(
+        [](const InstPartition &P) { return P.empty(); });
+
+    return true;
+  }
+
+  /// \brief Sets up the mapping between instructions to partitions.  If the
+  /// instruction is duplicated across multiple partitions, set the entry to -1.
+  void setupPartitionIdOnInstructions() {
+    int PartitionID = 0;
+    for (const auto &Partition : PartitionContainer) {
+      for (Instruction *Inst : Partition) {
+        bool NewElt;
+        InstToPartitionIdT::iterator Iter;
+
+        std::tie(Iter, NewElt) =
+            InstToPartitionId.insert(std::make_pair(Inst, PartitionID));
+        if (!NewElt)
+          Iter->second = -1;
+      }
+      ++PartitionID;
+    }
+  }
+
+  /// \brief Populates the partition with everything that the seeding
+  /// instructions require.
+  void populateUsedSet() {
+    for (auto &P : PartitionContainer)
+      P.populateUsedSet();
+  }
+
+  /// \brief This performs the main chunk of the work of cloning the loops for
+  /// the partitions.
+  void cloneLoops(Pass *P) {
+    BasicBlock *OrigPH = L->getLoopPreheader();
+    // At this point the predecessor of the preheader is either the memcheck
+    // block or the top part of the original preheader.
+    BasicBlock *Pred = OrigPH->getSinglePredecessor();
+    assert(Pred && "Preheader does not have a single predecessor");
+    BasicBlock *ExitBlock = L->getExitBlock();
+    assert(ExitBlock && "No single exit block");
+    Loop *NewLoop;
+
+    assert(!PartitionContainer.empty() && "at least two partitions expected");
+    // We're cloning the preheader along with the loop so we already made sure
+    // it was empty.
+    assert(&*OrigPH->begin() == OrigPH->getTerminator() &&
+           "preheader not empty");
+
+    // Create a loop for each partition except the last.  Clone the original
+    // loop before PH along with adding a preheader for the cloned loop.  Then
+    // update PH to point to the newly added preheader.
+    BasicBlock *TopPH = OrigPH;
+    unsigned Index = getSize() - 1;
+    for (auto I = std::next(PartitionContainer.rbegin()),
+              E = PartitionContainer.rend();
+         I != E; ++I, --Index, TopPH = NewLoop->getLoopPreheader()) {
+      auto *Part = &*I;
+
+      NewLoop = Part->cloneLoopWithPreheader(TopPH, Pred, Index, LI, DT);
+
+      Part->getVMap()[ExitBlock] = TopPH;
+      Part->remapInstructions();
+    }
+    Pred->getTerminator()->replaceUsesOfWith(OrigPH, TopPH);
+
+    // Now go in forward order and update the immediate dominator for the
+    // preheaders with the exiting block of the previous loop.  Dominance
+    // within the loop is updated in cloneLoopWithPreheader.
+    for (auto Curr = PartitionContainer.cbegin(),
+              Next = std::next(PartitionContainer.cbegin()),
+              E = PartitionContainer.cend();
+         Next != E; ++Curr, ++Next)
+      DT->changeImmediateDominator(
+          Next->getDistributedLoop()->getLoopPreheader(),
+          Curr->getDistributedLoop()->getExitingBlock());
+  }
+
+  /// \brief Removes the dead instructions from the cloned loops.
+  void removeUnusedInsts() {
+    for (auto &Partition : PartitionContainer)
+      Partition.removeUnusedInsts();
+  }
+
+  /// \brief For each memory pointer, it computes the partitionId the pointer is
+  /// used in.
+  ///
+  /// This returns an array of int where the I-th entry corresponds to I-th
+  /// entry in LAI.getRuntimePointerCheck().  If the pointer is used in multiple
+  /// partitions its entry is set to -1.
+  SmallVector<int, 8>
+  computePartitionSetForPointers(const LoopAccessInfo &LAI) {
+    const RuntimePointerChecking *RtPtrCheck = LAI.getRuntimePointerChecking();
+
+    unsigned N = RtPtrCheck->Pointers.size();
+    SmallVector<int, 8> PtrToPartitions(N);
+    for (unsigned I = 0; I < N; ++I) {
+      Value *Ptr = RtPtrCheck->Pointers[I].PointerValue;
+      auto Instructions =
+          LAI.getInstructionsForAccess(Ptr, RtPtrCheck->Pointers[I].IsWritePtr);
+
+      int &Partition = PtrToPartitions[I];
+      // First set it to uninitialized.
+      Partition = -2;
+      for (Instruction *Inst : Instructions) {
+        // Note that this could be -1 if Inst is duplicated across multiple
+        // partitions.
+        int ThisPartition = this->InstToPartitionId[Inst];
+        if (Partition == -2)
+          Partition = ThisPartition;
+        // -1 means belonging to multiple partitions.
+        else if (Partition == -1)
+          break;
+        else if (Partition != (int)ThisPartition)
+          Partition = -1;
+      }
+      assert(Partition != -2 && "Pointer not belonging to any partition");
+    }
+
+    return PtrToPartitions;
+  }
+
+  void print(raw_ostream &OS) const {
+    unsigned Index = 0;
+    for (const auto &P : PartitionContainer) {
+      OS << "Partition " << Index++ << " (" << &P << "):\n";
+      P.print();
+    }
+  }
+
+  void dump() const { print(dbgs()); }
+
+#ifndef NDEBUG
+  friend raw_ostream &operator<<(raw_ostream &OS,
+                                 const InstPartitionContainer &Partitions) {
+    Partitions.print(OS);
+    return OS;
+  }
+#endif
+
+  void printBlocks() const {
+    unsigned Index = 0;
+    for (const auto &P : PartitionContainer) {
+      dbgs() << "\nPartition " << Index++ << " (" << &P << "):\n";
+      P.printBlocks();
+    }
+  }
+
+private:
+  typedef std::list<InstPartition> PartitionContainerT;
+
+  /// \brief List of partitions.
+  PartitionContainerT PartitionContainer;
+
+  /// \brief Mapping from Instruction to partition Id.  If the instruction
+  /// belongs to multiple partitions the entry contains -1.
+  InstToPartitionIdT InstToPartitionId;
+
+  Loop *L;
+  LoopInfo *LI;
+  DominatorTree *DT;
+
+  /// \brief The control structure to merge adjacent partitions if both satisfy
+  /// the \p Predicate.
+  template <class UnaryPredicate>
+  void mergeAdjacentPartitionsIf(UnaryPredicate Predicate) {
+    InstPartition *PrevMatch = nullptr;
+    for (auto I = PartitionContainer.begin(); I != PartitionContainer.end();) {
+      auto DoesMatch = Predicate(&*I);
+      if (PrevMatch == nullptr && DoesMatch) {
+        PrevMatch = &*I;
+        ++I;
+      } else if (PrevMatch != nullptr && DoesMatch) {
+        I->moveTo(*PrevMatch);
+        I = PartitionContainer.erase(I);
+      } else {
+        PrevMatch = nullptr;
+        ++I;
+      }
+    }
+  }
+};
+
+/// \brief For each memory instruction, this class maintains difference of the
+/// number of unsafe dependences that start out from this instruction minus
+/// those that end here.
+///
+/// By traversing the memory instructions in program order and accumulating this
+/// number, we know whether any unsafe dependence crosses over a program point.
+class MemoryInstructionDependences {
+  typedef MemoryDepChecker::Dependence Dependence;
+
+public:
+  struct Entry {
+    Instruction *Inst;
+    unsigned NumUnsafeDependencesStartOrEnd;
+
+    Entry(Instruction *Inst) : Inst(Inst), NumUnsafeDependencesStartOrEnd(0) {}
+  };
+
+  typedef SmallVector<Entry, 8> AccessesType;
+
+  AccessesType::const_iterator begin() const { return Accesses.begin(); }
+  AccessesType::const_iterator end() const { return Accesses.end(); }
+
+  MemoryInstructionDependences(
+      const SmallVectorImpl<Instruction *> &Instructions,
+      const SmallVectorImpl<Dependence> &InterestingDependences) {
+    Accesses.append(Instructions.begin(), Instructions.end());
+
+    DEBUG(dbgs() << "Backward dependences:\n");
+    for (auto &Dep : InterestingDependences)
+      if (Dep.isPossiblyBackward()) {
+        // Note that the designations source and destination follow the program
+        // order, i.e. source is always first.  (The direction is given by the
+        // DepType.)
+        ++Accesses[Dep.Source].NumUnsafeDependencesStartOrEnd;
+        --Accesses[Dep.Destination].NumUnsafeDependencesStartOrEnd;
+
+        DEBUG(Dep.print(dbgs(), 2, Instructions));
+      }
+  }
+
+private:
+  AccessesType Accesses;
+};
+
+/// \brief Returns the instructions that use values defined in the loop.
+static SmallVector<Instruction *, 8> findDefsUsedOutsideOfLoop(Loop *L) {
+  SmallVector<Instruction *, 8> UsedOutside;
+
+  for (auto *Block : L->getBlocks())
+    // FIXME: I believe that this could use copy_if if the Inst reference could
+    // be adapted into a pointer.
+    for (auto &Inst : *Block) {
+      auto Users = Inst.users();
+      if (std::any_of(Users.begin(), Users.end(), [&](User *U) {
+            auto *Use = cast<Instruction>(U);
+            return !L->contains(Use->getParent());
+          }))
+        UsedOutside.push_back(&Inst);
+    }
+
+  return UsedOutside;
+}
+
+/// \brief The pass class.
+class LoopDistribute : public FunctionPass {
+public:
+  LoopDistribute() : FunctionPass(ID) {
+    initializeLoopDistributePass(*PassRegistry::getPassRegistry());
+  }
+
+  bool runOnFunction(Function &F) override {
+    LI = &getAnalysis<LoopInfoWrapperPass>().getLoopInfo();
+    LAA = &getAnalysis<LoopAccessAnalysis>();
+    DT = &getAnalysis<DominatorTreeWrapperPass>().getDomTree();
+
+    // Build up a worklist of inner-loops to vectorize. This is necessary as the
+    // act of distributing a loop creates new loops and can invalidate iterators
+    // across the loops.
+    SmallVector<Loop *, 8> Worklist;
+
+    for (Loop *TopLevelLoop : *LI)
+      for (Loop *L : depth_first(TopLevelLoop))
+        // We only handle inner-most loops.
+        if (L->empty())
+          Worklist.push_back(L);
+
+    // Now walk the identified inner loops.
+    bool Changed = false;
+    for (Loop *L : Worklist)
+      Changed |= processLoop(L);
+
+    // Process each loop nest in the function.
+    return Changed;
+  }
+
+  void getAnalysisUsage(AnalysisUsage &AU) const override {
+    AU.addRequired<LoopInfoWrapperPass>();
+    AU.addPreserved<LoopInfoWrapperPass>();
+    AU.addRequired<LoopAccessAnalysis>();
+    AU.addRequired<DominatorTreeWrapperPass>();
+    AU.addPreserved<DominatorTreeWrapperPass>();
+  }
+
+  static char ID;
+
+private:
+  /// \brief Filter out checks between pointers from the same partition.
+  ///
+  /// \p PtrToPartition contains the partition number for pointers.  Partition
+  /// number -1 means that the pointer is used in multiple partitions.  In this
+  /// case we can't safely omit the check.
+  SmallVector<RuntimePointerChecking::PointerCheck, 4>
+  includeOnlyCrossPartitionChecks(
+      const SmallVectorImpl<RuntimePointerChecking::PointerCheck> &AllChecks,
+      const SmallVectorImpl<int> &PtrToPartition,
+      const RuntimePointerChecking *RtPtrChecking) {
+    SmallVector<RuntimePointerChecking::PointerCheck, 4> Checks;
+
+    std::copy_if(AllChecks.begin(), AllChecks.end(), std::back_inserter(Checks),
+                 [&](const RuntimePointerChecking::PointerCheck &Check) {
+                   for (unsigned PtrIdx1 : Check.first->Members)
+                     for (unsigned PtrIdx2 : Check.second->Members)
+                       // Only include this check if there is a pair of pointers
+                       // that require checking and the pointers fall into
+                       // separate partitions.
+                       //
+                       // (Note that we already know at this point that the two
+                       // pointer groups need checking but it doesn't follow
+                       // that each pair of pointers within the two groups need
+                       // checking as well.
+                       //
+                       // In other words we don't want to include a check just
+                       // because there is a pair of pointers between the two
+                       // pointer groups that require checks and a different
+                       // pair whose pointers fall into different partitions.)
+                       if (RtPtrChecking->needsChecking(PtrIdx1, PtrIdx2) &&
+                           !RuntimePointerChecking::arePointersInSamePartition(
+                               PtrToPartition, PtrIdx1, PtrIdx2))
+                         return true;
+                   return false;
+                 });
+
+    return Checks;
+  }
+
+  /// \brief Try to distribute an inner-most loop.
+  bool processLoop(Loop *L) {
+    assert(L->empty() && "Only process inner loops.");
+
+    DEBUG(dbgs() << "\nLDist: In \"" << L->getHeader()->getParent()->getName()
+                 << "\" checking " << *L << "\n");
+
+    BasicBlock *PH = L->getLoopPreheader();
+    if (!PH) {
+      DEBUG(dbgs() << "Skipping; no preheader");
+      return false;
+    }
+    if (!L->getExitBlock()) {
+      DEBUG(dbgs() << "Skipping; multiple exit blocks");
+      return false;
+    }
+    // LAA will check that we only have a single exiting block.
+
+    const LoopAccessInfo &LAI = LAA->getInfo(L, ValueToValueMap());
+
+    // Currently, we only distribute to isolate the part of the loop with
+    // dependence cycles to enable partial vectorization.
+    if (LAI.canVectorizeMemory()) {
+      DEBUG(dbgs() << "Skipping; memory operations are safe for vectorization");
+      return false;
+    }
+    auto *InterestingDependences =
+        LAI.getDepChecker().getInterestingDependences();
+    if (!InterestingDependences || InterestingDependences->empty()) {
+      DEBUG(dbgs() << "Skipping; No unsafe dependences to isolate");
+      return false;
+    }
+
+    InstPartitionContainer Partitions(L, LI, DT);
+
+    // First, go through each memory operation and assign them to consecutive
+    // partitions (the order of partitions follows program order).  Put those
+    // with unsafe dependences into "cyclic" partition otherwise put each store
+    // in its own "non-cyclic" partition (we'll merge these later).
+    //
+    // Note that a memory operation (e.g. Load2 below) at a program point that
+    // has an unsafe dependence (Store3->Load1) spanning over it must be
+    // included in the same cyclic partition as the dependent operations.  This
+    // is to preserve the original program order after distribution.  E.g.:
+    //
+    //                NumUnsafeDependencesStartOrEnd  NumUnsafeDependencesActive
+    //  Load1   -.                     1                       0->1
+    //  Load2    | /Unsafe/            0                       1
+    //  Store3  -'                    -1                       1->0
+    //  Load4                          0                       0
+    //
+    // NumUnsafeDependencesActive > 0 indicates this situation and in this case
+    // we just keep assigning to the same cyclic partition until
+    // NumUnsafeDependencesActive reaches 0.
+    const MemoryDepChecker &DepChecker = LAI.getDepChecker();
+    MemoryInstructionDependences MID(DepChecker.getMemoryInstructions(),
+                                     *InterestingDependences);
+
+    int NumUnsafeDependencesActive = 0;
+    for (auto &InstDep : MID) {
+      Instruction *I = InstDep.Inst;
+      // We update NumUnsafeDependencesActive post-instruction, catch the
+      // start of a dependence directly via NumUnsafeDependencesStartOrEnd.
+      if (NumUnsafeDependencesActive ||
+          InstDep.NumUnsafeDependencesStartOrEnd > 0)
+        Partitions.addToCyclicPartition(I);
+      else
+        Partitions.addToNewNonCyclicPartition(I);
+      NumUnsafeDependencesActive += InstDep.NumUnsafeDependencesStartOrEnd;
+      assert(NumUnsafeDependencesActive >= 0 &&
+             "Negative number of dependences active");
+    }
+
+    // Add partitions for values used outside.  These partitions can be out of
+    // order from the original program order.  This is OK because if the
+    // partition uses a load we will merge this partition with the original
+    // partition of the load that we set up in the previous loop (see
+    // mergeToAvoidDuplicatedLoads).
+    auto DefsUsedOutside = findDefsUsedOutsideOfLoop(L);
+    for (auto *Inst : DefsUsedOutside)
+      Partitions.addToNewNonCyclicPartition(Inst);
+
+    DEBUG(dbgs() << "Seeded partitions:\n" << Partitions);
+    if (Partitions.getSize() < 2)
+      return false;
+
+    // Run the merge heuristics: Merge non-cyclic adjacent partitions since we
+    // should be able to vectorize these together.
+    Partitions.mergeBeforePopulating();
+    DEBUG(dbgs() << "\nMerged partitions:\n" << Partitions);
+    if (Partitions.getSize() < 2)
+      return false;
+
+    // Now, populate the partitions with non-memory operations.
+    Partitions.populateUsedSet();
+    DEBUG(dbgs() << "\nPopulated partitions:\n" << Partitions);
+
+    // In order to preserve original lexical order for loads, keep them in the
+    // partition that we set up in the MemoryInstructionDependences loop.
+    if (Partitions.mergeToAvoidDuplicatedLoads()) {
+      DEBUG(dbgs() << "\nPartitions merged to ensure unique loads:\n"
+                   << Partitions);
+      if (Partitions.getSize() < 2)
+        return false;
+    }
+
+    DEBUG(dbgs() << "\nDistributing loop: " << *L << "\n");
+    // We're done forming the partitions set up the reverse mapping from
+    // instructions to partitions.
+    Partitions.setupPartitionIdOnInstructions();
+
+    // To keep things simple have an empty preheader before we version or clone
+    // the loop.  (Also split if this has no predecessor, i.e. entry, because we
+    // rely on PH having a predecessor.)
+    if (!PH->getSinglePredecessor() || &*PH->begin() != PH->getTerminator())
+      SplitBlock(PH, PH->getTerminator(), DT, LI);
+
+    // If we need run-time checks to disambiguate pointers are run-time, version
+    // the loop now.
+    auto PtrToPartition = Partitions.computePartitionSetForPointers(LAI);
+    const auto *RtPtrChecking = LAI.getRuntimePointerChecking();
+    auto AllChecks = RtPtrChecking->generateChecks();
+    auto Checks = includeOnlyCrossPartitionChecks(AllChecks, PtrToPartition,
+                                                  RtPtrChecking);
+    if (!Checks.empty()) {
+      DEBUG(dbgs() << "\nPointers:\n");
+      DEBUG(LAI.getRuntimePointerChecking()->printChecks(dbgs(), Checks));
+      LoopVersioning LVer(std::move(Checks), LAI, L, LI, DT);
+      LVer.versionLoop(this);
+      LVer.addPHINodes(DefsUsedOutside);
+    }
+
+    // Create identical copies of the original loop for each partition and hook
+    // them up sequentially.
+    Partitions.cloneLoops(this);
+
+    // Now, we remove the instruction from each loop that don't belong to that
+    // partition.
+    Partitions.removeUnusedInsts();
+    DEBUG(dbgs() << "\nAfter removing unused Instrs:\n");
+    DEBUG(Partitions.printBlocks());
+
+    if (LDistVerify) {
+      LI->verify();
+      DT->verifyDomTree();
+    }
+
+    ++NumLoopsDistributed;
+    return true;
+  }
+
+  // Analyses used.
+  LoopInfo *LI;
+  LoopAccessAnalysis *LAA;
+  DominatorTree *DT;
+};
+} // anonymous namespace
+
+char LoopDistribute::ID;
+static const char ldist_name[] = "Loop Distribition";
+
+INITIALIZE_PASS_BEGIN(LoopDistribute, LDIST_NAME, ldist_name, false, false)
+INITIALIZE_PASS_DEPENDENCY(LoopInfoWrapperPass)
+INITIALIZE_PASS_DEPENDENCY(LoopAccessAnalysis)
+INITIALIZE_PASS_DEPENDENCY(DominatorTreeWrapperPass)
+INITIALIZE_PASS_END(LoopDistribute, LDIST_NAME, ldist_name, false, false)
+
+namespace llvm {
+FunctionPass *createLoopDistributePass() { return new LoopDistribute(); }
+}
diff --git a/lib/Transforms/Scalar/LoopIdiomRecognize.cpp b/lib/Transforms/Scalar/LoopIdiomRecognize.cpp
index 7bc291784ab..d1ee2268421 100644
--- a/lib/Transforms/Scalar/LoopIdiomRecognize.cpp
+++ b/lib/Transforms/Scalar/LoopIdiomRecognize.cpp
@@ -508,7 +508,7 @@ void NclPopcountRecognize::transform(Instruction *CntInst,
 
     ICmpInst *NewPreCond =
       cast<ICmpInst>(Builder.CreateICmp(PreCond->getPredicate(), Opnd0, Opnd1));
-    PreCond->replaceAllUsesWith(NewPreCond);
+    PreCondBr->setCondition(NewPreCond);
 
     RecursivelyDeleteTriviallyDeadInstructions(PreCond, TLI);
   }
@@ -611,7 +611,9 @@ bool NclPopcountRecognize::recognize() {
 
 bool LoopIdiomRecognize::runOnCountableLoop() {
   const SCEV *BECount = SE->getBackedgeTakenCount(CurLoop);
-  if (isa<SCEVCouldNotCompute>(BECount)) return false;
+  assert(!isa<SCEVCouldNotCompute>(BECount) &&
+    "runOnCountableLoop() called on a loop without a predictable"
+    "backedge-taken count");
 
   // If this loop executes exactly one time, then it should be peeled, not
   // optimized by this pass.
@@ -637,13 +639,12 @@ bool LoopIdiomRecognize::runOnCountableLoop() {
 
   bool MadeChange = false;
   // Scan all the blocks in the loop that are not in subloops.
-  for (Loop::block_iterator BI = CurLoop->block_begin(),
-         E = CurLoop->block_end(); BI != E; ++BI) {
+  for (auto *BB : CurLoop->getBlocks()) {
     // Ignore blocks in subloops.
-    if (LI.getLoopFor(*BI) != CurLoop)
+    if (LI.getLoopFor(BB) != CurLoop)
       continue;
 
-    MadeChange |= runOnLoopBlock(*BI, BECount, ExitBlocks);
+    MadeChange |= runOnLoopBlock(BB, BECount, ExitBlocks);
   }
   return MadeChange;
 }
@@ -825,14 +826,14 @@ processLoopMemSet(MemSetInst *MSI, const SCEV *BECount) {
 /// mayLoopAccessLocation - Return true if the specified loop might access the
 /// specified pointer location, which is a loop-strided access.  The 'Access'
 /// argument specifies what the verboten forms of access are (read or write).
-static bool mayLoopAccessLocation(Value *Ptr,AliasAnalysis::ModRefResult Access,
-                                  Loop *L, const SCEV *BECount,
-                                  unsigned StoreSize, AliasAnalysis &AA,
+static bool mayLoopAccessLocation(Value *Ptr, ModRefInfo Access, Loop *L,
+                                  const SCEV *BECount, unsigned StoreSize,
+                                  AliasAnalysis &AA,
                                   Instruction *IgnoredStore) {
   // Get the location that may be stored across the loop.  Since the access is
   // strided positively through memory, we say that the modified location starts
   // at the pointer and has infinite size.
-  uint64_t AccessSize = AliasAnalysis::UnknownSize;
+  uint64_t AccessSize = MemoryLocation::UnknownSize;
 
   // If the loop iterates a fixed number of times, we can refine the access size
   // to be exactly the size of the memset, which is (BECount+1)*StoreSize
@@ -843,7 +844,7 @@ static bool mayLoopAccessLocation(Value *Ptr,AliasAnalysis::ModRefResult Access,
   // operand in the store.  Store to &A[i] of 100 will always return may alias
   // with store of &A[100], we need to StoreLoc to be "A" with size of 100,
   // which will then no-alias a store to &A[100].
-  AliasAnalysis::Location StoreLoc(Ptr, AccessSize);
+  MemoryLocation StoreLoc(Ptr, AccessSize);
 
   for (Loop::block_iterator BI = L->block_begin(), E = L->block_end(); BI != E;
        ++BI)
@@ -948,9 +949,8 @@ processLoopStridedStore(Value *DestPtr, unsigned StoreSize,
     Expander.expandCodeFor(Ev->getStart(), DestInt8PtrTy,
                            Preheader->getTerminator());
 
-  if (mayLoopAccessLocation(BasePtr, AliasAnalysis::ModRef,
-                            CurLoop, BECount,
-                            StoreSize, getAnalysis<AliasAnalysis>(), TheStore)) {
+  if (mayLoopAccessLocation(BasePtr, MRI_ModRef, CurLoop, BECount, StoreSize,
+                            getAnalysis<AliasAnalysis>(), TheStore)) {
     Expander.clear();
     // If we generated new code for the base pointer, clean up.
     RecursivelyDeleteTriviallyDeadInstructions(BasePtr, TLI);
@@ -1000,7 +1000,7 @@ processLoopStridedStore(Value *DestPtr, unsigned StoreSize,
     GV->setUnnamedAddr(true); // Ok to merge these.
     GV->setAlignment(16);
     Value *PatternPtr = ConstantExpr::getBitCast(GV, Int8PtrTy);
-    NewCall = Builder.CreateCall3(MSP, BasePtr, PatternPtr, NumBytes);
+    NewCall = Builder.CreateCall(MSP, {BasePtr, PatternPtr, NumBytes});
   }
 
   DEBUG(dbgs() << "  Formed memset: " << *NewCall << "\n"
@@ -1046,9 +1046,8 @@ processLoopStoreOfLoopLoad(StoreInst *SI, unsigned StoreSize,
                            Builder.getInt8PtrTy(SI->getPointerAddressSpace()),
                            Preheader->getTerminator());
 
-  if (mayLoopAccessLocation(StoreBasePtr, AliasAnalysis::ModRef,
-                            CurLoop, BECount, StoreSize,
-                            getAnalysis<AliasAnalysis>(), SI)) {
+  if (mayLoopAccessLocation(StoreBasePtr, MRI_ModRef, CurLoop, BECount,
+                            StoreSize, getAnalysis<AliasAnalysis>(), SI)) {
     Expander.clear();
     // If we generated new code for the base pointer, clean up.
     RecursivelyDeleteTriviallyDeadInstructions(StoreBasePtr, TLI);
@@ -1062,8 +1061,8 @@ processLoopStoreOfLoopLoad(StoreInst *SI, unsigned StoreSize,
                            Builder.getInt8PtrTy(LI->getPointerAddressSpace()),
                            Preheader->getTerminator());
 
-  if (mayLoopAccessLocation(LoadBasePtr, AliasAnalysis::Mod, CurLoop, BECount,
-                            StoreSize, getAnalysis<AliasAnalysis>(), SI)) {
+  if (mayLoopAccessLocation(LoadBasePtr, MRI_Mod, CurLoop, BECount, StoreSize,
+                            getAnalysis<AliasAnalysis>(), SI)) {
     Expander.clear();
     // If we generated new code for the base pointer, clean up.
     RecursivelyDeleteTriviallyDeadInstructions(LoadBasePtr, TLI);
diff --git a/lib/Transforms/Scalar/LoopInterchange.cpp b/lib/Transforms/Scalar/LoopInterchange.cpp
index f7626c527c1..9d7e57ffeba 100644
--- a/lib/Transforms/Scalar/LoopInterchange.cpp
+++ b/lib/Transforms/Scalar/LoopInterchange.cpp
@@ -33,6 +33,7 @@
 #include "llvm/IR/IRBuilder.h"
 #include "llvm/IR/InstIterator.h"
 #include "llvm/IR/IntrinsicInst.h"
+#include "llvm/IR/Module.h"
 #include "llvm/Pass.h"
 #include "llvm/Support/Debug.h"
 #include "llvm/Support/raw_ostream.h"
@@ -70,8 +71,8 @@ void printDepMatrix(CharMatrix &DepMatrix) {
 }
 #endif
 
-bool populateDependencyMatrix(CharMatrix &DepMatrix, unsigned Level, Loop *L,
-                              DependenceAnalysis *DA) {
+static bool populateDependencyMatrix(CharMatrix &DepMatrix, unsigned Level,
+                                     Loop *L, DependenceAnalysis *DA) {
   typedef SmallVector<Value *, 16> ValueVector;
   ValueVector MemInstr;
 
@@ -183,8 +184,8 @@ bool populateDependencyMatrix(CharMatrix &DepMatrix, unsigned Level, Loop *L,
 
 // A loop is moved from index 'from' to an index 'to'. Update the Dependence
 // matrix by exchanging the two columns.
-void interChangeDepedencies(CharMatrix &DepMatrix, unsigned FromIndx,
-                            unsigned ToIndx) {
+static void interChangeDepedencies(CharMatrix &DepMatrix, unsigned FromIndx,
+                                   unsigned ToIndx) {
   unsigned numRows = DepMatrix.size();
   for (unsigned i = 0; i < numRows; ++i) {
     char TmpVal = DepMatrix[i][ToIndx];
@@ -195,8 +196,8 @@ void interChangeDepedencies(CharMatrix &DepMatrix, unsigned FromIndx,
 
 // Checks if outermost non '=','S'or'I' dependence in the dependence matrix is
 // '>'
-bool isOuterMostDepPositive(CharMatrix &DepMatrix, unsigned Row,
-                            unsigned Column) {
+static bool isOuterMostDepPositive(CharMatrix &DepMatrix, unsigned Row,
+                                   unsigned Column) {
   for (unsigned i = 0; i <= Column; ++i) {
     if (DepMatrix[Row][i] == '<')
       return false;
@@ -208,8 +209,8 @@ bool isOuterMostDepPositive(CharMatrix &DepMatrix, unsigned Row,
 }
 
 // Checks if no dependence exist in the dependency matrix in Row before Column.
-bool containsNoDependence(CharMatrix &DepMatrix, unsigned Row,
-                          unsigned Column) {
+static bool containsNoDependence(CharMatrix &DepMatrix, unsigned Row,
+                                 unsigned Column) {
   for (unsigned i = 0; i < Column; ++i) {
     if (DepMatrix[Row][i] != '=' || DepMatrix[Row][i] != 'S' ||
         DepMatrix[Row][i] != 'I')
@@ -218,8 +219,9 @@ bool containsNoDependence(CharMatrix &DepMatrix, unsigned Row,
   return true;
 }
 
-bool validDepInterchange(CharMatrix &DepMatrix, unsigned Row,
-                         unsigned OuterLoopId, char InnerDep, char OuterDep) {
+static bool validDepInterchange(CharMatrix &DepMatrix, unsigned Row,
+                                unsigned OuterLoopId, char InnerDep,
+                                char OuterDep) {
 
   if (isOuterMostDepPositive(DepMatrix, Row, OuterLoopId))
     return false;
@@ -253,11 +255,13 @@ bool validDepInterchange(CharMatrix &DepMatrix, unsigned Row,
 }
 
 // Checks if it is legal to interchange 2 loops.
-// [Theorm] A permutation of the loops in a perfect nest is legal if and only if
+// [Theorem] A permutation of the loops in a perfect nest is legal if and only
+// if
 // the direction matrix, after the same permutation is applied to its columns,
 // has no ">" direction as the leftmost non-"=" direction in any row.
-bool isLegalToInterChangeLoops(CharMatrix &DepMatrix, unsigned InnerLoopId,
-                               unsigned OuterLoopId) {
+static bool isLegalToInterChangeLoops(CharMatrix &DepMatrix,
+                                      unsigned InnerLoopId,
+                                      unsigned OuterLoopId) {
 
   unsigned NumRows = DepMatrix.size();
   // For each row check if it is valid to interchange.
@@ -278,21 +282,21 @@ static void populateWorklist(Loop &L, SmallVector<LoopVector, 8> &V) {
   DEBUG(dbgs() << "Calling populateWorklist called\n");
   LoopVector LoopList;
   Loop *CurrentLoop = &L;
-  std::vector<Loop *> vec = CurrentLoop->getSubLoopsVector();
-  while (vec.size() != 0) {
+  const std::vector<Loop *> *Vec = &CurrentLoop->getSubLoops();
+  while (!Vec->empty()) {
     // The current loop has multiple subloops in it hence it is not tightly
     // nested.
     // Discard all loops above it added into Worklist.
-    if (vec.size() != 1) {
+    if (Vec->size() != 1) {
       LoopList.clear();
       return;
     }
     LoopList.push_back(CurrentLoop);
-    CurrentLoop = *(vec.begin());
-    vec = CurrentLoop->getSubLoopsVector();
+    CurrentLoop = Vec->front();
+    Vec = &CurrentLoop->getSubLoops();
   }
   LoopList.push_back(CurrentLoop);
-  V.push_back(LoopList);
+  V.push_back(std::move(LoopList));
 }
 
 static PHINode *getInductionVariable(Loop *L, ScalarEvolution *SE) {
@@ -328,7 +332,8 @@ class LoopInterchangeLegality {
 public:
   LoopInterchangeLegality(Loop *Outer, Loop *Inner, ScalarEvolution *SE,
                           LoopInterchange *Pass)
-      : OuterLoop(Outer), InnerLoop(Inner), SE(SE), CurrentPass(Pass) {}
+      : OuterLoop(Outer), InnerLoop(Inner), SE(SE), CurrentPass(Pass),
+        InnerLoopHasReduction(false) {}
 
   /// Check if the loops can be interchanged.
   bool canInterchangeLoops(unsigned InnerLoopId, unsigned OuterLoopId,
@@ -339,15 +344,24 @@ class LoopInterchangeLegality {
 
   bool currentLimitations();
 
+  bool hasInnerLoopReduction() { return InnerLoopHasReduction; }
+
 private:
   bool tightlyNested(Loop *Outer, Loop *Inner);
-
+  bool containsUnsafeInstructionsInHeader(BasicBlock *BB);
+  bool areAllUsesReductions(Instruction *Ins, Loop *L);
+  bool containsUnsafeInstructionsInLatch(BasicBlock *BB);
+  bool findInductionAndReductions(Loop *L,
+                                  SmallVector<PHINode *, 8> &Inductions,
+                                  SmallVector<PHINode *, 8> &Reductions);
   Loop *OuterLoop;
   Loop *InnerLoop;
 
   /// Scev analysis.
   ScalarEvolution *SE;
   LoopInterchange *CurrentPass;
+
+  bool InnerLoopHasReduction;
 };
 
 /// LoopInterchangeProfitability checks if it is profitable to interchange the
@@ -376,9 +390,11 @@ class LoopInterchangeTransform {
 public:
   LoopInterchangeTransform(Loop *Outer, Loop *Inner, ScalarEvolution *SE,
                            LoopInfo *LI, DominatorTree *DT,
-                           LoopInterchange *Pass, BasicBlock *LoopNestExit)
+                           LoopInterchange *Pass, BasicBlock *LoopNestExit,
+                           bool InnerLoopContainsReductions)
       : OuterLoop(Outer), InnerLoop(Inner), SE(SE), LI(LI), DT(DT),
-        LoopExit(LoopNestExit) {}
+        LoopExit(LoopNestExit),
+        InnerLoopHasReduction(InnerLoopContainsReductions) {}
 
   /// Interchange OuterLoop and InnerLoop.
   bool transform();
@@ -394,6 +410,8 @@ class LoopInterchangeTransform {
   void adjustOuterLoopPreheader();
   void adjustInnerLoopPreheader();
   bool adjustLoopBranches();
+  void updateIncomingBlock(BasicBlock *CurrBlock, BasicBlock *OldPred,
+                           BasicBlock *NewPred);
 
   Loop *OuterLoop;
   Loop *InnerLoop;
@@ -403,6 +421,7 @@ class LoopInterchangeTransform {
   LoopInfo *LI;
   DominatorTree *DT;
   BasicBlock *LoopExit;
+  bool InnerLoopHasReduction;
 };
 
 // Main LoopInterchange Pass
@@ -443,7 +462,7 @@ struct LoopInterchange : public FunctionPass {
     bool Changed = true;
     while (!Worklist.empty()) {
       LoopVector LoopList = Worklist.pop_back_val();
-      Changed = processLoopList(LoopList);
+      Changed = processLoopList(LoopList, F);
     }
     return Changed;
   }
@@ -474,9 +493,9 @@ struct LoopInterchange : public FunctionPass {
     return LoopList.size() - 1;
   }
 
-  bool processLoopList(LoopVector LoopList) {
+  bool processLoopList(LoopVector LoopList, Function &F) {
+
     bool Changed = false;
-    bool containsLCSSAPHI = false;
     CharMatrix DependencyMatrix;
     if (LoopList.size() < 2) {
       DEBUG(dbgs() << "Loop doesn't contain minimum nesting level.\n");
@@ -518,21 +537,11 @@ struct LoopInterchange : public FunctionPass {
     else
       LoopNestExit = OuterMostLoopLatchBI->getSuccessor(0);
 
-    for (auto I = LoopList.begin(), E = LoopList.end(); I != E; ++I) {
-      Loop *L = *I;
-      BasicBlock *Latch = L->getLoopLatch();
-      BasicBlock *Header = L->getHeader();
-      if (Latch && Latch != Header && isa<PHINode>(Latch->begin())) {
-        containsLCSSAPHI = true;
-        break;
-      }
-    }
-
-    // TODO: Handle lcssa PHI's. Currently LCSSA PHI's are not handled. Handle
-    // the same by splitting the loop latch and adjusting loop links
-    // accordingly.
-    if (containsLCSSAPHI)
+    if (isa<PHINode>(LoopNestExit->begin())) {
+      DEBUG(dbgs() << "PHI Nodes in loop nest exit is not handled for now "
+                      "since on failure all loops branch to loop nest exit.\n");
       return false;
+    }
 
     unsigned SelecLoopId = selectLoopForInterchange(LoopList);
     // Move the selected loop outwards to the best posible position.
@@ -546,7 +555,7 @@ struct LoopInterchange : public FunctionPass {
 
       // Update the DependencyMatrix
       interChangeDepedencies(DependencyMatrix, i, i - 1);
-
+      DT->recalculate(F);
 #ifdef DUMP_DEP_MATRICIES
       DEBUG(dbgs() << "Dependence after inter change \n");
       printDepMatrix(DependencyMatrix);
@@ -578,7 +587,7 @@ struct LoopInterchange : public FunctionPass {
     }
 
     LoopInterchangeTransform LIT(OuterLoop, InnerLoop, SE, LI, DT, this,
-                                 LoopNestExit);
+                                 LoopNestExit, LIL.hasInnerLoopReduction());
     LIT.transform();
     DEBUG(dbgs() << "Loops interchanged\n");
     return true;
@@ -586,10 +595,38 @@ struct LoopInterchange : public FunctionPass {
 };
 
 } // end of namespace
+bool LoopInterchangeLegality::areAllUsesReductions(Instruction *Ins, Loop *L) {
+  return !std::any_of(Ins->user_begin(), Ins->user_end(), [=](User *U) -> bool {
+    PHINode *UserIns = dyn_cast<PHINode>(U);
+    RecurrenceDescriptor RD;
+    return !UserIns || !RecurrenceDescriptor::isReductionPHI(UserIns, L, RD);
+  });
+}
+
+bool LoopInterchangeLegality::containsUnsafeInstructionsInHeader(
+    BasicBlock *BB) {
+  for (auto I = BB->begin(), E = BB->end(); I != E; ++I) {
+    // Load corresponding to reduction PHI's are safe while concluding if
+    // tightly nested.
+    if (LoadInst *L = dyn_cast<LoadInst>(I)) {
+      if (!areAllUsesReductions(L, InnerLoop))
+        return true;
+    } else if (I->mayHaveSideEffects() || I->mayReadFromMemory())
+      return true;
+  }
+  return false;
+}
 
-static bool containsUnsafeInstructions(BasicBlock *BB) {
+bool LoopInterchangeLegality::containsUnsafeInstructionsInLatch(
+    BasicBlock *BB) {
   for (auto I = BB->begin(), E = BB->end(); I != E; ++I) {
-    if (I->mayHaveSideEffects() || I->mayReadFromMemory())
+    // Stores corresponding to reductions are safe while concluding if tightly
+    // nested.
+    if (StoreInst *L = dyn_cast<StoreInst>(I)) {
+      PHINode *PHI = dyn_cast<PHINode>(L->getOperand(0));
+      if (!PHI)
+        return true;
+    } else if (I->mayHaveSideEffects() || I->mayReadFromMemory())
       return true;
   }
   return false;
@@ -619,8 +656,8 @@ bool LoopInterchangeLegality::tightlyNested(Loop *OuterLoop, Loop *InnerLoop) {
   DEBUG(dbgs() << "Checking instructions in Loop header and Loop latch \n");
   // We do not have any basic block in between now make sure the outer header
   // and outer loop latch doesnt contain any unsafe instructions.
-  if (containsUnsafeInstructions(OuterLoopHeader) ||
-      containsUnsafeInstructions(OuterLoopLatch))
+  if (containsUnsafeInstructionsInHeader(OuterLoopHeader) ||
+      containsUnsafeInstructionsInLatch(OuterLoopLatch))
     return false;
 
   DEBUG(dbgs() << "Loops are perfectly nested \n");
@@ -628,12 +665,6 @@ bool LoopInterchangeLegality::tightlyNested(Loop *OuterLoop, Loop *InnerLoop) {
   return true;
 }
 
-static unsigned getPHICount(BasicBlock *BB) {
-  unsigned PhiCount = 0;
-  for (auto I = BB->begin(); isa<PHINode>(I); ++I)
-    PhiCount++;
-  return PhiCount;
-}
 
 bool LoopInterchangeLegality::isLoopStructureUnderstood(
     PHINode *InnerInduction) {
@@ -660,34 +691,96 @@ bool LoopInterchangeLegality::isLoopStructureUnderstood(
   return true;
 }
 
+bool LoopInterchangeLegality::findInductionAndReductions(
+    Loop *L, SmallVector<PHINode *, 8> &Inductions,
+    SmallVector<PHINode *, 8> &Reductions) {
+  if (!L->getLoopLatch() || !L->getLoopPredecessor())
+    return false;
+  for (BasicBlock::iterator I = L->getHeader()->begin(); isa<PHINode>(I); ++I) {
+    RecurrenceDescriptor RD;
+    PHINode *PHI = cast<PHINode>(I);
+    ConstantInt *StepValue = nullptr;
+    if (isInductionPHI(PHI, SE, StepValue))
+      Inductions.push_back(PHI);
+    else if (RecurrenceDescriptor::isReductionPHI(PHI, L, RD))
+      Reductions.push_back(PHI);
+    else {
+      DEBUG(
+          dbgs() << "Failed to recognize PHI as an induction or reduction.\n");
+      return false;
+    }
+  }
+  return true;
+}
+
+static bool containsSafePHI(BasicBlock *Block, bool isOuterLoopExitBlock) {
+  for (auto I = Block->begin(); isa<PHINode>(I); ++I) {
+    PHINode *PHI = cast<PHINode>(I);
+    // Reduction lcssa phi will have only 1 incoming block that from loop latch.
+    if (PHI->getNumIncomingValues() > 1)
+      return false;
+    Instruction *Ins = dyn_cast<Instruction>(PHI->getIncomingValue(0));
+    if (!Ins)
+      return false;
+    // Incoming value for lcssa phi's in outer loop exit can only be inner loop
+    // exits lcssa phi else it would not be tightly nested.
+    if (!isa<PHINode>(Ins) && isOuterLoopExitBlock)
+      return false;
+  }
+  return true;
+}
+
+static BasicBlock *getLoopLatchExitBlock(BasicBlock *LatchBlock,
+                                         BasicBlock *LoopHeader) {
+  if (BranchInst *BI = dyn_cast<BranchInst>(LatchBlock->getTerminator())) {
+    unsigned Num = BI->getNumSuccessors();
+    assert(Num == 2);
+    for (unsigned i = 0; i < Num; ++i) {
+      if (BI->getSuccessor(i) == LoopHeader)
+        continue;
+      return BI->getSuccessor(i);
+    }
+  }
+  return nullptr;
+}
+
 // This function indicates the current limitations in the transform as a result
 // of which we do not proceed.
 bool LoopInterchangeLegality::currentLimitations() {
 
   BasicBlock *InnerLoopPreHeader = InnerLoop->getLoopPreheader();
   BasicBlock *InnerLoopHeader = InnerLoop->getHeader();
-  BasicBlock *OuterLoopHeader = OuterLoop->getHeader();
   BasicBlock *InnerLoopLatch = InnerLoop->getLoopLatch();
   BasicBlock *OuterLoopLatch = OuterLoop->getLoopLatch();
+  BasicBlock *OuterLoopHeader = OuterLoop->getHeader();
 
   PHINode *InnerInductionVar;
-  PHINode *OuterInductionVar;
-
-  // We currently handle only 1 induction variable inside the loop. We also do
-  // not handle reductions as of now.
-  if (getPHICount(InnerLoopHeader) > 1)
+  SmallVector<PHINode *, 8> Inductions;
+  SmallVector<PHINode *, 8> Reductions;
+  if (!findInductionAndReductions(InnerLoop, Inductions, Reductions))
     return true;
 
-  if (getPHICount(OuterLoopHeader) > 1)
+  // TODO: Currently we handle only loops with 1 induction variable.
+  if (Inductions.size() != 1) {
+    DEBUG(dbgs() << "We currently only support loops with 1 induction variable."
+                 << "Failed to interchange due to current limitation\n");
     return true;
+  }
+  if (Reductions.size() > 0)
+    InnerLoopHasReduction = true;
 
-  InnerInductionVar = getInductionVariable(InnerLoop, SE);
-  OuterInductionVar = getInductionVariable(OuterLoop, SE);
+  InnerInductionVar = Inductions.pop_back_val();
+  Reductions.clear();
+  if (!findInductionAndReductions(OuterLoop, Inductions, Reductions))
+    return true;
 
-  if (!OuterInductionVar || !InnerInductionVar) {
-    DEBUG(dbgs() << "Induction variable not found\n");
+  // Outer loop cannot have reduction because then loops will not be tightly
+  // nested.
+  if (!Reductions.empty())
+    return true;
+  // TODO: Currently we handle only loops with 1 induction variable.
+  if (Inductions.size() != 1)
     return true;
-  }
 
   // TODO: Triangular loops are not handled for now.
   if (!isLoopStructureUnderstood(InnerInductionVar)) {
@@ -695,16 +788,15 @@ bool LoopInterchangeLegality::currentLimitations() {
     return true;
   }
 
-  // TODO: Loops with LCSSA PHI's are currently not handled.
-  if (isa<PHINode>(OuterLoopLatch->begin())) {
-    DEBUG(dbgs() << "Found and LCSSA PHI in outer loop latch\n");
+  // TODO: We only handle LCSSA PHI's corresponding to reduction for now.
+  BasicBlock *LoopExitBlock =
+      getLoopLatchExitBlock(OuterLoopLatch, OuterLoopHeader);
+  if (!LoopExitBlock || !containsSafePHI(LoopExitBlock, true))
     return true;
-  }
-  if (InnerLoopLatch != InnerLoopHeader &&
-      isa<PHINode>(InnerLoopLatch->begin())) {
-    DEBUG(dbgs() << "Found and LCSSA PHI in inner loop latch\n");
+
+  LoopExitBlock = getLoopLatchExitBlock(InnerLoopLatch, InnerLoopHeader);
+  if (!LoopExitBlock || !containsSafePHI(LoopExitBlock, false))
     return true;
-  }
 
   // TODO: Current limitation: Since we split the inner loop latch at the point
   // were induction variable is incremented (induction.next); We cannot have
@@ -783,12 +875,6 @@ bool LoopInterchangeLegality::canInterchangeLoops(unsigned InnerLoopId,
     InnerLoopPreHeader = InsertPreheaderForLoop(InnerLoop, CurrentPass);
   }
 
-  // Check if the loops are tightly nested.
-  if (!tightlyNested(OuterLoop, InnerLoop)) {
-    DEBUG(dbgs() << "Loops not tightly nested\n");
-    return false;
-  }
-
   // TODO: The loops could not be interchanged due to current limitations in the
   // transform module.
   if (currentLimitations()) {
@@ -796,6 +882,12 @@ bool LoopInterchangeLegality::canInterchangeLoops(unsigned InnerLoopId,
     return false;
   }
 
+  // Check if the loops are tightly nested.
+  if (!tightlyNested(OuterLoop, InnerLoop)) {
+    DEBUG(dbgs() << "Loops not tightly nested\n");
+    return false;
+  }
+
   return true;
 }
 
@@ -920,8 +1012,8 @@ void LoopInterchangeTransform::restructureLoops(Loop *InnerLoop,
     LI->changeTopLevelLoop(OuterLoop, InnerLoop);
   }
 
-  for (Loop::iterator I = InnerLoop->begin(), E = InnerLoop->end(); I != E; ++I)
-    OuterLoop->addChildLoop(InnerLoop->removeChildLoop(I));
+  while (!InnerLoop->empty())
+    OuterLoop->addChildLoop(InnerLoop->removeChildLoop(InnerLoop->begin()));
 
   InnerLoop->addChildLoop(OuterLoop);
 }
@@ -983,9 +1075,33 @@ void LoopInterchangeTransform::splitOuterLoopLatch() {
 
 void LoopInterchangeTransform::splitInnerLoopHeader() {
 
-  // Split the inner loop header out.
+  // Split the inner loop header out. Here make sure that the reduction PHI's
+  // stay in the innerloop body.
   BasicBlock *InnerLoopHeader = InnerLoop->getHeader();
-  SplitBlock(InnerLoopHeader, InnerLoopHeader->getFirstNonPHI(), DT, LI);
+  BasicBlock *InnerLoopPreHeader = InnerLoop->getLoopPreheader();
+  if (InnerLoopHasReduction) {
+    // FIXME: Check if the induction PHI will always be the first PHI.
+    BasicBlock *New = InnerLoopHeader->splitBasicBlock(
+        ++(InnerLoopHeader->begin()), InnerLoopHeader->getName() + ".split");
+    if (LI)
+      if (Loop *L = LI->getLoopFor(InnerLoopHeader))
+        L->addBasicBlockToLoop(New, *LI);
+
+    // Adjust Reduction PHI's in the block.
+    SmallVector<PHINode *, 8> PHIVec;
+    for (auto I = New->begin(); isa<PHINode>(I); ++I) {
+      PHINode *PHI = dyn_cast<PHINode>(I);
+      Value *V = PHI->getIncomingValueForBlock(InnerLoopPreHeader);
+      PHI->replaceAllUsesWith(V);
+      PHIVec.push_back((PHI));
+    }
+    for (auto I = PHIVec.begin(), E = PHIVec.end(); I != E; ++I) {
+      PHINode *P = *I;
+      P->eraseFromParent();
+    }
+  } else {
+    SplitBlock(InnerLoopHeader, InnerLoopHeader->getFirstNonPHI(), DT, LI);
+  }
 
   DEBUG(dbgs() << "Output of splitInnerLoopHeader InnerLoopHeaderSucc & "
                   "InnerLoopHeader \n");
@@ -1015,6 +1131,19 @@ void LoopInterchangeTransform::adjustInnerLoopPreheader() {
   moveBBContents(InnerLoopPreHeader, OuterHeader->getTerminator());
 }
 
+void LoopInterchangeTransform::updateIncomingBlock(BasicBlock *CurrBlock,
+                                                   BasicBlock *OldPred,
+                                                   BasicBlock *NewPred) {
+  for (auto I = CurrBlock->begin(); isa<PHINode>(I); ++I) {
+    PHINode *PHI = cast<PHINode>(I);
+    unsigned Num = PHI->getNumIncomingValues();
+    for (unsigned i = 0; i < Num; ++i) {
+      if (PHI->getIncomingBlock(i) == OldPred)
+        PHI->setIncomingBlock(i, NewPred);
+    }
+  }
+}
+
 bool LoopInterchangeTransform::adjustLoopBranches() {
 
   DEBUG(dbgs() << "adjustLoopBranches called\n");
@@ -1072,6 +1201,10 @@ bool LoopInterchangeTransform::adjustLoopBranches() {
       OuterLoopHeaderBI->setSuccessor(i, InnerLoopHeaderSucessor);
   }
 
+  // Adjust reduction PHI's now that the incoming block has changed.
+  updateIncomingBlock(InnerLoopHeaderSucessor, InnerLoopHeader,
+                      OuterLoopHeader);
+
   BranchInst::Create(OuterLoopPreHeader, InnerLoopHeaderBI);
   InnerLoopHeaderBI->eraseFromParent();
 
@@ -1087,6 +1220,20 @@ bool LoopInterchangeTransform::adjustLoopBranches() {
       InnerLoopLatchPredecessorBI->setSuccessor(i, InnerLoopLatchSuccessor);
   }
 
+  // Adjust PHI nodes in InnerLoopLatchSuccessor. Update all uses of PHI with
+  // the value and remove this PHI node from inner loop.
+  SmallVector<PHINode *, 8> LcssaVec;
+  for (auto I = InnerLoopLatchSuccessor->begin(); isa<PHINode>(I); ++I) {
+    PHINode *LcssaPhi = cast<PHINode>(I);
+    LcssaVec.push_back(LcssaPhi);
+  }
+  for (auto I = LcssaVec.begin(), E = LcssaVec.end(); I != E; ++I) {
+    PHINode *P = *I;
+    Value *Incoming = P->getIncomingValueForBlock(InnerLoopLatch);
+    P->replaceAllUsesWith(Incoming);
+    P->eraseFromParent();
+  }
+
   if (OuterLoopLatchBI->getSuccessor(0) == OuterLoopHeader)
     OuterLoopLatchSuccessor = OuterLoopLatchBI->getSuccessor(1);
   else
@@ -1097,6 +1244,8 @@ bool LoopInterchangeTransform::adjustLoopBranches() {
   else
     InnerLoopLatchBI->setSuccessor(0, OuterLoopLatchSuccessor);
 
+  updateIncomingBlock(OuterLoopLatchSuccessor, OuterLoopLatch, InnerLoopLatch);
+
   if (OuterLoopLatchBI->getSuccessor(0) == OuterLoopLatchSuccessor) {
     OuterLoopLatchBI->setSuccessor(0, InnerLoopLatch);
   } else {
@@ -1117,12 +1266,9 @@ void LoopInterchangeTransform::adjustLoopPreheaders() {
   BranchInst *InnerTermBI =
       cast<BranchInst>(InnerLoopPreHeader->getTerminator());
 
-  BasicBlock *HeaderSplit =
-      SplitBlock(OuterLoopHeader, OuterLoopHeader->getTerminator(), DT, LI);
-  Instruction *InsPoint = HeaderSplit->getFirstNonPHI();
   // These instructions should now be executed inside the loop.
   // Move instruction into a new block after outer header.
-  moveBBContents(InnerLoopPreHeader, InsPoint);
+  moveBBContents(InnerLoopPreHeader, OuterLoopHeader->getTerminator());
   // These instructions were not executed previously in the loop so move them to
   // the older inner loop preheader.
   moveBBContents(OuterLoopPreHeader, InnerTermBI);
diff --git a/lib/Transforms/Scalar/LoopRerollPass.cpp b/lib/Transforms/Scalar/LoopRerollPass.cpp
index ed103e6b8ed..05d8d20b1d2 100644
--- a/lib/Transforms/Scalar/LoopRerollPass.cpp
+++ b/lib/Transforms/Scalar/LoopRerollPass.cpp
@@ -166,7 +166,10 @@ namespace {
     typedef SmallVector<Instruction *, 16> SmallInstructionVector;
     typedef SmallSet<Instruction *, 16>   SmallInstructionSet;
 
-    // A chain of isomorphic instructions, indentified by a single-use PHI,
+    // Map between induction variable and its increment
+    DenseMap<Instruction *, int64_t> IVToIncMap;
+
+    // A chain of isomorphic instructions, identified by a single-use PHI
     // representing a reduction. Only the last value may be used outside the
     // loop.
     struct SimpleLoopReduction {
@@ -335,7 +338,7 @@ namespace {
     //   x[i*3+1] = y2
     //   x[i*3+2] = y3
     //
-    //   Base instruction -> i*3               
+    //   Base instruction -> i*3
     //                    +---+----+
     //                   /    |     \
     //               ST[y1]  +1     +2  <-- Roots
@@ -366,8 +369,10 @@ namespace {
     struct DAGRootTracker {
       DAGRootTracker(LoopReroll *Parent, Loop *L, Instruction *IV,
                      ScalarEvolution *SE, AliasAnalysis *AA,
-                     TargetLibraryInfo *TLI)
-          : Parent(Parent), L(L), SE(SE), AA(AA), TLI(TLI), IV(IV) {}
+                     TargetLibraryInfo *TLI,
+                     DenseMap<Instruction *, int64_t> &IncrMap)
+          : Parent(Parent), L(L), SE(SE), AA(AA), TLI(TLI), IV(IV),
+            IVToIncMap(IncrMap) {}
 
       /// Stage 1: Find all the DAG roots for the induction variable.
       bool findRoots();
@@ -417,7 +422,7 @@ namespace {
       // The loop induction variable.
       Instruction *IV;
       // Loop step amount.
-      uint64_t Inc;
+      int64_t Inc;
       // Loop reroll count; if Inc == 1, this records the scaling applied
       // to the indvar: a[i*2+0] = ...; a[i*2+1] = ... ;
       // If Inc is not 1, Scale = Inc.
@@ -430,6 +435,8 @@ namespace {
       // they are used in (or specially, IL_All for instructions
       // used in the loop increment mechanism).
       UsesTy Uses;
+      // Map between induction variable and its increment
+      DenseMap<Instruction *, int64_t> &IVToIncMap;
     };
 
     void collectPossibleIVs(Loop *L, SmallInstructionVector &PossibleIVs);
@@ -484,13 +491,12 @@ void LoopReroll::collectPossibleIVs(Loop *L,
         continue;
       if (const SCEVConstant *IncSCEV =
           dyn_cast<SCEVConstant>(PHISCEV->getStepRecurrence(*SE))) {
-        if (!IncSCEV->getValue()->getValue().isStrictlyPositive())
-          continue;
-        if (IncSCEV->getValue()->uge(MaxInc))
+        const APInt &AInt = IncSCEV->getValue()->getValue().abs();
+        if (IncSCEV->getValue()->isZero() || AInt.uge(MaxInc))
           continue;
-
-        DEBUG(dbgs() << "LRR: Possible IV: " << *I << " = " <<
-              *PHISCEV << "\n");
+        IVToIncMap[I] = IncSCEV->getValue()->getSExtValue();
+        DEBUG(dbgs() << "LRR: Possible IV: " << *I << " = " << *PHISCEV
+                     << "\n");
         PossibleIVs.push_back(I);
       }
     }
@@ -699,17 +705,11 @@ collectPossibleRoots(Instruction *Base, std::map<int64_t,Instruction*> &Roots) {
       }
     }
 
-    int64_t V = CI->getValue().getSExtValue();
+    int64_t V = std::abs(CI->getValue().getSExtValue());
     if (Roots.find(V) != Roots.end())
       // No duplicates, please.
       return false;
 
-    // FIXME: Add support for negative values.
-    if (V < 0) {
-      DEBUG(dbgs() << "LRR: Aborting due to negative value: " << V << "\n");
-      return false;
-    }
-
     Roots[V] = cast<Instruction>(I);
   }
 
@@ -731,7 +731,7 @@ collectPossibleRoots(Instruction *Base, std::map<int64_t,Instruction*> &Roots) {
   unsigned NumBaseUses = BaseUsers.size();
   if (NumBaseUses == 0)
     NumBaseUses = Roots.begin()->second->getNumUses();
-  
+
   // Check that every node has the same number of users.
   for (auto &KV : Roots) {
     if (KV.first == 0)
@@ -744,7 +744,7 @@ collectPossibleRoots(Instruction *Base, std::map<int64_t,Instruction*> &Roots) {
     }
   }
 
-  return true; 
+  return true;
 }
 
 bool LoopReroll::DAGRootTracker::
@@ -787,7 +787,7 @@ findRootsBase(Instruction *IVU, SmallInstructionSet SubsumedInsts) {
   if (!collectPossibleRoots(IVU, V))
     return false;
 
-  // If we didn't get a root for index zero, then IVU must be 
+  // If we didn't get a root for index zero, then IVU must be
   // subsumed.
   if (V.find(0) == V.end())
     SubsumedInsts.insert(IVU);
@@ -818,13 +818,10 @@ findRootsBase(Instruction *IVU, SmallInstructionSet SubsumedInsts) {
 }
 
 bool LoopReroll::DAGRootTracker::findRoots() {
-
-  const SCEVAddRecExpr *RealIVSCEV = cast<SCEVAddRecExpr>(SE->getSCEV(IV));
-  Inc = cast<SCEVConstant>(RealIVSCEV->getOperand(1))->
-    getValue()->getZExtValue();
+  Inc = IVToIncMap[IV];
 
   assert(RootSets.empty() && "Unclean state!");
-  if (Inc == 1) {
+  if (std::abs(Inc) == 1) {
     for (auto *IVU : IV->users()) {
       if (isLoopIncrement(IVU, IV))
         LoopIncs.push_back(cast<Instruction>(IVU));
@@ -1103,15 +1100,15 @@ bool LoopReroll::DAGRootTracker::validate(ReductionTracker &Reductions) {
                 " vs. " << *RootInst << "\n");
           return false;
         }
-        
+
         RootIt = TryIt;
         RootInst = TryIt->first;
       }
 
       // All instructions between the last root and this root
-      // may belong to some other iteration. If they belong to a 
+      // may belong to some other iteration. If they belong to a
       // future iteration, then they're dangerous to alias with.
-      // 
+      //
       // Note that because we allow a limited amount of flexibility in the order
       // that we visit nodes, LastRootIt might be *before* RootIt, in which
       // case we've already checked this set of instructions so we shouldn't
@@ -1267,6 +1264,7 @@ void LoopReroll::DAGRootTracker::replace(const SCEV *IterCount) {
 
     ++J;
   }
+  bool Negative = IVToIncMap[IV] < 0;
   const DataLayout &DL = Header->getModule()->getDataLayout();
 
   // We need to create a new induction variable for each different BaseInst.
@@ -1275,10 +1273,9 @@ void LoopReroll::DAGRootTracker::replace(const SCEV *IterCount) {
     const SCEVAddRecExpr *RealIVSCEV =
       cast<SCEVAddRecExpr>(SE->getSCEV(DRS.BaseInst));
     const SCEV *Start = RealIVSCEV->getStart();
-    const SCEVAddRecExpr *H = cast<SCEVAddRecExpr>
-      (SE->getAddRecExpr(Start,
-                         SE->getConstant(RealIVSCEV->getType(), 1),
-                         L, SCEV::FlagAnyWrap));
+    const SCEVAddRecExpr *H = cast<SCEVAddRecExpr>(SE->getAddRecExpr(
+        Start, SE->getConstant(RealIVSCEV->getType(), Negative ? -1 : 1), L,
+        SCEV::FlagAnyWrap));
     { // Limit the lifetime of SCEVExpander.
       SCEVExpander Expander(*SE, DL, "reroll");
       Value *NewIV = Expander.expandCodeFor(H, IV->getType(), Header->begin());
@@ -1294,8 +1291,8 @@ void LoopReroll::DAGRootTracker::replace(const SCEV *IterCount) {
           const SCEV *ICSCEV = RealIVSCEV->evaluateAtIteration(IterCount, *SE);
 
           // Iteration count SCEV minus 1
-          const SCEV *ICMinus1SCEV =
-            SE->getMinusSCEV(ICSCEV, SE->getConstant(ICSCEV->getType(), 1));
+          const SCEV *ICMinus1SCEV = SE->getMinusSCEV(
+              ICSCEV, SE->getConstant(ICSCEV->getType(), Negative ? -1 : 1));
 
           Value *ICMinus1; // Iteration count minus 1
           if (isa<SCEVConstant>(ICMinus1SCEV)) {
@@ -1444,13 +1441,13 @@ void LoopReroll::ReductionTracker::replaceSelected() {
 bool LoopReroll::reroll(Instruction *IV, Loop *L, BasicBlock *Header,
                         const SCEV *IterCount,
                         ReductionTracker &Reductions) {
-  DAGRootTracker DAGRoots(this, L, IV, SE, AA, TLI);
+  DAGRootTracker DAGRoots(this, L, IV, SE, AA, TLI, IVToIncMap);
 
   if (!DAGRoots.findRoots())
     return false;
   DEBUG(dbgs() << "LRR: Found all root induction increments for: " <<
                   *IV << "\n");
-  
+
   if (!DAGRoots.validate(Reductions))
     return false;
   if (!Reductions.validateSelected())
@@ -1497,6 +1494,7 @@ bool LoopReroll::runOnLoop(Loop *L, LPPassManager &LPM) {
   // First, we need to find the induction variable with respect to which we can
   // reroll (there may be several possible options).
   SmallInstructionVector PossibleIVs;
+  IVToIncMap.clear();
   collectPossibleIVs(L, PossibleIVs);
 
   if (PossibleIVs.empty()) {
diff --git a/lib/Transforms/Scalar/LoopRotation.cpp b/lib/Transforms/Scalar/LoopRotation.cpp
index a675e1289ba..bf034bc8d19 100644
--- a/lib/Transforms/Scalar/LoopRotation.cpp
+++ b/lib/Transforms/Scalar/LoopRotation.cpp
@@ -13,6 +13,7 @@
 
 #include "llvm/Transforms/Scalar.h"
 #include "llvm/ADT/Statistic.h"
+#include "llvm/Analysis/AliasAnalysis.h"
 #include "llvm/Analysis/AssumptionCache.h"
 #include "llvm/Analysis/CodeMetrics.h"
 #include "llvm/Analysis/InstructionSimplify.h"
@@ -56,6 +57,7 @@ namespace {
 
     // LCSSA form makes instruction renaming easier.
     void getAnalysisUsage(AnalysisUsage &AU) const override {
+      AU.addPreserved<AliasAnalysis>();
       AU.addRequired<AssumptionCacheTracker>();
       AU.addPreserved<DominatorTreeWrapperPass>();
       AU.addRequired<LoopInfoWrapperPass>();
diff --git a/lib/Transforms/Scalar/LoopStrengthReduce.cpp b/lib/Transforms/Scalar/LoopStrengthReduce.cpp
index 099f227af55..059b10ef73f 100644
--- a/lib/Transforms/Scalar/LoopStrengthReduce.cpp
+++ b/lib/Transforms/Scalar/LoopStrengthReduce.cpp
@@ -185,9 +185,8 @@ RegUseTracker::SwapAndDropUse(size_t LUIdx, size_t LastLUIdx) {
 
   // Update RegUses. The data structure is not optimized for this purpose;
   // we must iterate through it and update each of the bit vectors.
-  for (RegUsesTy::iterator I = RegUsesMap.begin(), E = RegUsesMap.end();
-       I != E; ++I) {
-    SmallBitVector &UsedByIndices = I->second.UsedByIndices;
+  for (auto &Pair : RegUsesMap) {
+    SmallBitVector &UsedByIndices = Pair.second.UsedByIndices;
     if (LUIdx < UsedByIndices.size())
       UsedByIndices[LUIdx] =
         LastLUIdx < UsedByIndices.size() ? UsedByIndices[LastLUIdx] : 0;
@@ -257,9 +256,22 @@ struct Formula {
   /// live in an add immediate field rather than a register.
   int64_t UnfoldedOffset;
 
+  /// ZeroExtendScaledReg - This formula zero extends the scale register to
+  /// ZeroExtendType before its use.
+  bool ZeroExtendScaledReg;
+
+  /// ZeroExtendBaseReg - This formula zero extends all the base registers to
+  /// ZeroExtendType before their use.
+  bool ZeroExtendBaseReg;
+
+  /// ZeroExtendType - The destination type of the zero extension implied by
+  /// the above two booleans.
+  Type *ZeroExtendType;
+
   Formula()
       : BaseGV(nullptr), BaseOffset(0), HasBaseReg(false), Scale(0),
-        ScaledReg(nullptr), UnfoldedOffset(0) {}
+        ScaledReg(nullptr), UnfoldedOffset(0), ZeroExtendScaledReg(false),
+        ZeroExtendBaseReg(false), ZeroExtendType(nullptr) {}
 
   void InitialMatch(const SCEV *S, Loop *L, ScalarEvolution &SE);
 
@@ -297,9 +309,8 @@ static void DoInitialMatch(const SCEV *S, Loop *L,
 
   // Look at add operands.
   if (const SCEVAddExpr *Add = dyn_cast<SCEVAddExpr>(S)) {
-    for (SCEVAddExpr::op_iterator I = Add->op_begin(), E = Add->op_end();
-         I != E; ++I)
-      DoInitialMatch(*I, L, Good, Bad, SE);
+    for (const SCEV *S : Add->operands())
+      DoInitialMatch(S, L, Good, Bad, SE);
     return;
   }
 
@@ -326,12 +337,10 @@ static void DoInitialMatch(const SCEV *S, Loop *L,
       DoInitialMatch(NewMul, L, MyGood, MyBad, SE);
       const SCEV *NegOne = SE.getSCEV(ConstantInt::getAllOnesValue(
         SE.getEffectiveSCEVType(NewMul->getType())));
-      for (SmallVectorImpl<const SCEV *>::const_iterator I = MyGood.begin(),
-           E = MyGood.end(); I != E; ++I)
-        Good.push_back(SE.getMulExpr(NegOne, *I));
-      for (SmallVectorImpl<const SCEV *>::const_iterator I = MyBad.begin(),
-           E = MyBad.end(); I != E; ++I)
-        Bad.push_back(SE.getMulExpr(NegOne, *I));
+      for (const SCEV *S : MyGood)
+        Good.push_back(SE.getMulExpr(NegOne, S));
+      for (const SCEV *S : MyBad)
+        Bad.push_back(SE.getMulExpr(NegOne, S));
       return;
     }
 
@@ -417,10 +426,12 @@ size_t Formula::getNumRegs() const {
 /// getType - Return the type of this formula, if it has one, or null
 /// otherwise. This type is meaningless except for the bit size.
 Type *Formula::getType() const {
-  return !BaseRegs.empty() ? BaseRegs.front()->getType() :
-         ScaledReg ? ScaledReg->getType() :
-         BaseGV ? BaseGV->getType() :
-         nullptr;
+  return ZeroExtendType
+             ? ZeroExtendType
+             : !BaseRegs.empty()
+                   ? BaseRegs.front()->getType()
+                   : ScaledReg ? ScaledReg->getType()
+                               : BaseGV ? BaseGV->getType() : nullptr;
 }
 
 /// DeleteBaseReg - Delete the given base reg from the BaseRegs list.
@@ -443,9 +454,8 @@ bool Formula::hasRegsUsedByUsesOtherThan(size_t LUIdx,
   if (ScaledReg)
     if (RegUses.isRegUsedByUsesOtherThan(ScaledReg, LUIdx))
       return true;
-  for (SmallVectorImpl<const SCEV *>::const_iterator I = BaseRegs.begin(),
-       E = BaseRegs.end(); I != E; ++I)
-    if (RegUses.isRegUsedByUsesOtherThan(*I, LUIdx))
+  for (const SCEV *BaseReg : BaseRegs)
+    if (RegUses.isRegUsedByUsesOtherThan(BaseReg, LUIdx))
       return true;
   return false;
 }
@@ -460,10 +470,12 @@ void Formula::print(raw_ostream &OS) const {
     if (!First) OS << " + "; else First = false;
     OS << BaseOffset;
   }
-  for (SmallVectorImpl<const SCEV *>::const_iterator I = BaseRegs.begin(),
-       E = BaseRegs.end(); I != E; ++I) {
+  for (const SCEV *BaseReg : BaseRegs) {
     if (!First) OS << " + "; else First = false;
-    OS << "reg(" << **I << ')';
+    if (ZeroExtendBaseReg)
+      OS << "reg(zext " << *BaseReg << " to " << *ZeroExtendType << ')';
+    else
+      OS << "reg(" << *BaseReg << ')';
   }
   if (HasBaseReg && BaseRegs.empty()) {
     if (!First) OS << " + "; else First = false;
@@ -475,9 +487,12 @@ void Formula::print(raw_ostream &OS) const {
   if (Scale != 0) {
     if (!First) OS << " + "; else First = false;
     OS << Scale << "*reg(";
-    if (ScaledReg)
-      OS << *ScaledReg;
-    else
+    if (ScaledReg) {
+      if (ZeroExtendScaledReg)
+        OS << "(zext " << *ScaledReg << " to " << *ZeroExtendType << ')';
+      else
+        OS << *ScaledReg;
+    } else
       OS << "<unknown>";
     OS << ')';
   }
@@ -576,10 +591,8 @@ static const SCEV *getExactSDiv(const SCEV *LHS, const SCEV *RHS,
   if (const SCEVAddExpr *Add = dyn_cast<SCEVAddExpr>(LHS)) {
     if (IgnoreSignificantBits || isAddSExtable(Add, SE)) {
       SmallVector<const SCEV *, 8> Ops;
-      for (SCEVAddExpr::op_iterator I = Add->op_begin(), E = Add->op_end();
-           I != E; ++I) {
-        const SCEV *Op = getExactSDiv(*I, RHS, SE,
-                                      IgnoreSignificantBits);
+      for (const SCEV *S : Add->operands()) {
+        const SCEV *Op = getExactSDiv(S, RHS, SE, IgnoreSignificantBits);
         if (!Op) return nullptr;
         Ops.push_back(Op);
       }
@@ -593,9 +606,7 @@ static const SCEV *getExactSDiv(const SCEV *LHS, const SCEV *RHS,
     if (IgnoreSignificantBits || isMulSExtable(Mul, SE)) {
       SmallVector<const SCEV *, 4> Ops;
       bool Found = false;
-      for (SCEVMulExpr::op_iterator I = Mul->op_begin(), E = Mul->op_end();
-           I != E; ++I) {
-        const SCEV *S = *I;
+      for (const SCEV *S : Mul->operands()) {
         if (!Found)
           if (const SCEV *Q = getExactSDiv(S, RHS, SE,
                                            IgnoreSignificantBits)) {
@@ -765,9 +776,8 @@ static bool isHighCostExpansion(const SCEV *S,
     return false;
 
   if (const SCEVAddExpr *Add = dyn_cast<SCEVAddExpr>(S)) {
-    for (SCEVAddExpr::op_iterator I = Add->op_begin(), E = Add->op_end();
-         I != E; ++I) {
-      if (isHighCostExpansion(*I, Processed, SE))
+    for (const SCEV *S : Add->operands()) {
+      if (isHighCostExpansion(S, Processed, SE))
         return true;
     }
     return false;
@@ -818,11 +828,11 @@ DeleteTriviallyDeadInstructions(SmallVectorImpl<WeakVH> &DeadInsts) {
     if (!I || !isInstructionTriviallyDead(I))
       continue;
 
-    for (User::op_iterator OI = I->op_begin(), E = I->op_end(); OI != E; ++OI)
-      if (Instruction *U = dyn_cast<Instruction>(*OI)) {
-        *OI = nullptr;
+    for (Use &O : I->operands())
+      if (Instruction *U = dyn_cast<Instruction>(O)) {
+        O = nullptr;
         if (U->use_empty())
-          DeadInsts.push_back(U);
+          DeadInsts.emplace_back(U);
       }
 
     I->eraseFromParent();
@@ -1001,9 +1011,7 @@ void Cost::RateFormula(const TargetTransformInfo &TTI,
     if (isLoser())
       return;
   }
-  for (SmallVectorImpl<const SCEV *>::const_iterator I = F.BaseRegs.begin(),
-       E = F.BaseRegs.end(); I != E; ++I) {
-    const SCEV *BaseReg = *I;
+  for (const SCEV *BaseReg : F.BaseRegs) {
     if (VisitedRegs.count(BaseReg)) {
       Lose();
       return;
@@ -1026,9 +1034,8 @@ void Cost::RateFormula(const TargetTransformInfo &TTI,
   ScaleCost += getScalingFactorCost(TTI, LU, F);
 
   // Tally up the non-zero immediates.
-  for (SmallVectorImpl<int64_t>::const_iterator I = Offsets.begin(),
-       E = Offsets.end(); I != E; ++I) {
-    int64_t Offset = (uint64_t)*I + F.BaseOffset;
+  for (int64_t O : Offsets) {
+    int64_t Offset = (uint64_t)O + F.BaseOffset;
     if (F.BaseGV)
       ImmCost += 64; // Handle symbolic values conservatively.
                      // TODO: This should probably be the pointer size.
@@ -1151,10 +1158,9 @@ void LSRFixup::print(raw_ostream &OS) const {
   OS << ", OperandValToReplace=";
   OperandValToReplace->printAsOperand(OS, /*PrintType=*/false);
 
-  for (PostIncLoopSet::const_iterator I = PostIncLoops.begin(),
-       E = PostIncLoops.end(); I != E; ++I) {
+  for (const Loop *PIL : PostIncLoops) {
     OS << ", PostIncLoop=";
-    (*I)->getHeader()->printAsOperand(OS, /*PrintType=*/false);
+    PIL->getHeader()->printAsOperand(OS, /*PrintType=*/false);
   }
 
   if (LUIdx != ~size_t(0))
@@ -1300,9 +1306,8 @@ bool LSRUse::InsertFormula(const Formula &F) {
   assert((!F.ScaledReg || !F.ScaledReg->isZero()) &&
          "Zero allocated in a scaled register!");
 #ifndef NDEBUG
-  for (SmallVectorImpl<const SCEV *>::const_iterator I =
-       F.BaseRegs.begin(), E = F.BaseRegs.end(); I != E; ++I)
-    assert(!(*I)->isZero() && "Zero allocated in a base register!");
+  for (const SCEV *BaseReg : F.BaseRegs)
+    assert(!BaseReg->isZero() && "Zero allocated in a base register!");
 #endif
 
   // Add the formula to the list.
@@ -1354,11 +1359,11 @@ void LSRUse::print(raw_ostream &OS) const {
   }
 
   OS << ", Offsets={";
-  for (SmallVectorImpl<int64_t>::const_iterator I = Offsets.begin(),
-       E = Offsets.end(); I != E; ++I) {
-    OS << *I;
-    if (std::next(I) != E)
-      OS << ',';
+  bool NeedComma = false;
+  for (int64_t O : Offsets) {
+    if (NeedComma) OS << ',';
+    OS << O;
+    NeedComma = true;
   }
   OS << '}';
 
@@ -1383,9 +1388,6 @@ static bool isAMCompletelyFolded(const TargetTransformInfo &TTI,
   case LSRUse::Address:
     return TTI.isLegalAddressingMode(AccessTy, BaseGV, BaseOffset, HasBaseReg, Scale);
 
-    // Otherwise, just guess that reg+reg addressing is legal.
-    //return ;
-
   case LSRUse::ICmpZero:
     // There's not even a target hook for querying whether it would be legal to
     // fold a GV into an ICmp.
@@ -1751,6 +1753,7 @@ class LSRInstance {
   void GenerateICmpZeroScales(LSRUse &LU, unsigned LUIdx, Formula Base);
   void GenerateScales(LSRUse &LU, unsigned LUIdx, Formula Base);
   void GenerateTruncates(LSRUse &LU, unsigned LUIdx, Formula Base);
+  void GenerateZExts(LSRUse &LU, unsigned LUIdx, Formula Base);
   void GenerateCrossUseConstantOffsets();
   void GenerateAllReuseFormulae();
 
@@ -1925,12 +1928,12 @@ void LSRInstance::OptimizeShadowIV() {
 /// set the IV user and stride information and return true, otherwise return
 /// false.
 bool LSRInstance::FindIVUserForCond(ICmpInst *Cond, IVStrideUse *&CondUse) {
-  for (IVUsers::iterator UI = IU.begin(), E = IU.end(); UI != E; ++UI)
-    if (UI->getUser() == Cond) {
+  for (IVStrideUse &U : IU)
+    if (U.getUser() == Cond) {
       // NOTE: we could handle setcc instructions with multiple uses here, but
       // InstCombine does it as well for simple uses, it's not clear that it
       // occurs enough in real life to handle.
-      CondUse = UI;
+      CondUse = &U;
       return true;
     }
   return false;
@@ -2105,8 +2108,7 @@ LSRInstance::OptimizeLoopTermCond() {
   SmallVector<BasicBlock*, 8> ExitingBlocks;
   L->getExitingBlocks(ExitingBlocks);
 
-  for (unsigned i = 0, e = ExitingBlocks.size(); i != e; ++i) {
-    BasicBlock *ExitingBlock = ExitingBlocks[i];
+  for (BasicBlock *ExitingBlock : ExitingBlocks) {
 
     // Get the terminating condition for the loop if possible.  If we
     // can, we want to change it to use a post-incremented version of its
@@ -2349,9 +2351,7 @@ LSRInstance::FindUseWithSimilarFormula(const Formula &OrigF,
         LU.WidestFixupType == OrigLU.WidestFixupType &&
         LU.HasFormulaWithSameRegs(OrigF)) {
       // Scan through this use's formulae.
-      for (SmallVectorImpl<Formula>::const_iterator I = LU.Formulae.begin(),
-           E = LU.Formulae.end(); I != E; ++I) {
-        const Formula &F = *I;
+      for (const Formula &F : LU.Formulae) {
         // Check to see if this formula has the same registers and symbols
         // as OrigF.
         if (F.BaseRegs == OrigF.BaseRegs &&
@@ -2379,8 +2379,8 @@ void LSRInstance::CollectInterestingTypesAndFactors() {
 
   // Collect interesting types and strides.
   SmallVector<const SCEV *, 4> Worklist;
-  for (IVUsers::const_iterator UI = IU.begin(), E = IU.end(); UI != E; ++UI) {
-    const SCEV *Expr = IU.getExpr(*UI);
+  for (const IVStrideUse &U : IU) {
+    const SCEV *Expr = IU.getExpr(U);
 
     // Collect interesting types.
     Types.insert(SE.getEffectiveSCEVType(Expr->getType()));
@@ -2583,25 +2583,23 @@ isProfitableChain(IVChain &Chain, SmallPtrSetImpl<Instruction*> &Users,
   unsigned NumConstIncrements = 0;
   unsigned NumVarIncrements = 0;
   unsigned NumReusedIncrements = 0;
-  for (IVChain::const_iterator I = Chain.begin(), E = Chain.end();
-       I != E; ++I) {
-
-    if (I->IncExpr->isZero())
+  for (const IVInc &Inc : Chain) {
+    if (Inc.IncExpr->isZero())
       continue;
 
     // Incrementing by zero or some constant is neutral. We assume constants can
     // be folded into an addressing mode or an add's immediate operand.
-    if (isa<SCEVConstant>(I->IncExpr)) {
+    if (isa<SCEVConstant>(Inc.IncExpr)) {
       ++NumConstIncrements;
       continue;
     }
 
-    if (I->IncExpr == LastIncExpr)
+    if (Inc.IncExpr == LastIncExpr)
       ++NumReusedIncrements;
     else
       ++NumVarIncrements;
 
-    LastIncExpr = I->IncExpr;
+    LastIncExpr = Inc.IncExpr;
   }
   // An IV chain with a single increment is handled by LSR's postinc
   // uses. However, a chain with multiple increments requires keeping the IV's
@@ -2836,12 +2834,11 @@ void LSRInstance::FinalizeChain(IVChain &Chain) {
   assert(!Chain.Incs.empty() && "empty IV chains are not allowed");
   DEBUG(dbgs() << "Final Chain: " << *Chain.Incs[0].UserInst << "\n");
 
-  for (IVChain::const_iterator I = Chain.begin(), E = Chain.end();
-       I != E; ++I) {
-    DEBUG(dbgs() << "        Inc: " << *I->UserInst << "\n");
-    User::op_iterator UseI =
-      std::find(I->UserInst->op_begin(), I->UserInst->op_end(), I->IVOperand);
-    assert(UseI != I->UserInst->op_end() && "cannot find IV operand");
+  for (const IVInc &Inc : Chain) {
+    DEBUG(dbgs() << "        Inc: " << Inc.UserInst << "\n");
+    auto UseI = std::find(Inc.UserInst->op_begin(), Inc.UserInst->op_end(),
+                          Inc.IVOperand);
+    assert(UseI != Inc.UserInst->op_end() && "cannot find IV operand");
     IVIncSet.insert(UseI);
   }
 }
@@ -2904,20 +2901,18 @@ void LSRInstance::GenerateIVChain(const IVChain &Chain, SCEVExpander &Rewriter,
   Type *IVTy = IVSrc->getType();
   Type *IntTy = SE.getEffectiveSCEVType(IVTy);
   const SCEV *LeftOverExpr = nullptr;
-  for (IVChain::const_iterator IncI = Chain.begin(),
-         IncE = Chain.end(); IncI != IncE; ++IncI) {
-
-    Instruction *InsertPt = IncI->UserInst;
+  for (const IVInc &Inc : Chain) {
+    Instruction *InsertPt = Inc.UserInst;
     if (isa<PHINode>(InsertPt))
       InsertPt = L->getLoopLatch()->getTerminator();
 
     // IVOper will replace the current IV User's operand. IVSrc is the IV
     // value currently held in a register.
     Value *IVOper = IVSrc;
-    if (!IncI->IncExpr->isZero()) {
+    if (!Inc.IncExpr->isZero()) {
       // IncExpr was the result of subtraction of two narrow values, so must
       // be signed.
-      const SCEV *IncExpr = SE.getNoopOrSignExtend(IncI->IncExpr, IntTy);
+      const SCEV *IncExpr = SE.getNoopOrSignExtend(Inc.IncExpr, IntTy);
       LeftOverExpr = LeftOverExpr ?
         SE.getAddExpr(LeftOverExpr, IncExpr) : IncExpr;
     }
@@ -2930,22 +2925,21 @@ void LSRInstance::GenerateIVChain(const IVChain &Chain, SCEVExpander &Rewriter,
       IVOper = Rewriter.expandCodeFor(IVOperExpr, IVTy, InsertPt);
 
       // If an IV increment can't be folded, use it as the next IV value.
-      if (!canFoldIVIncExpr(LeftOverExpr, IncI->UserInst, IncI->IVOperand,
-                            TTI)) {
+      if (!canFoldIVIncExpr(LeftOverExpr, Inc.UserInst, Inc.IVOperand, TTI)) {
         assert(IVTy == IVOper->getType() && "inconsistent IV increment type");
         IVSrc = IVOper;
         LeftOverExpr = nullptr;
       }
     }
-    Type *OperTy = IncI->IVOperand->getType();
+    Type *OperTy = Inc.IVOperand->getType();
     if (IVTy != OperTy) {
       assert(SE.getTypeSizeInBits(IVTy) >= SE.getTypeSizeInBits(OperTy) &&
              "cannot extend a chained IV");
       IRBuilder<> Builder(InsertPt);
       IVOper = Builder.CreateTruncOrBitCast(IVOper, OperTy, "lsr.chain");
     }
-    IncI->UserInst->replaceUsesOfWith(IncI->IVOperand, IVOper);
-    DeadInsts.push_back(IncI->IVOperand);
+    Inc.UserInst->replaceUsesOfWith(Inc.IVOperand, IVOper);
+    DeadInsts.emplace_back(Inc.IVOperand);
   }
   // If LSR created a new, wider phi, we may also replace its postinc. We only
   // do this if we also found a wide value for the head of the chain.
@@ -2967,17 +2961,17 @@ void LSRInstance::GenerateIVChain(const IVChain &Chain, SCEVExpander &Rewriter,
         IVOper = Builder.CreatePointerCast(IVSrc, PostIncTy, "lsr.chain");
       }
       Phi->replaceUsesOfWith(PostIncV, IVOper);
-      DeadInsts.push_back(PostIncV);
+      DeadInsts.emplace_back(PostIncV);
     }
   }
 }
 
 void LSRInstance::CollectFixupsAndInitialFormulae() {
-  for (IVUsers::const_iterator UI = IU.begin(), E = IU.end(); UI != E; ++UI) {
-    Instruction *UserInst = UI->getUser();
+  for (const IVStrideUse &U : IU) {
+    Instruction *UserInst = U.getUser();
     // Skip IV users that are part of profitable IV Chains.
     User::op_iterator UseI = std::find(UserInst->op_begin(), UserInst->op_end(),
-                                       UI->getOperandValToReplace());
+                                       U.getOperandValToReplace());
     assert(UseI != UserInst->op_end() && "cannot find IV operand");
     if (IVIncSet.count(UseI))
       continue;
@@ -2985,8 +2979,8 @@ void LSRInstance::CollectFixupsAndInitialFormulae() {
     // Record the uses.
     LSRFixup &LF = getNewFixup();
     LF.UserInst = UserInst;
-    LF.OperandValToReplace = UI->getOperandValToReplace();
-    LF.PostIncLoops = UI->getPostIncLoops();
+    LF.OperandValToReplace = U.getOperandValToReplace();
+    LF.PostIncLoops = U.getPostIncLoops();
 
     LSRUse::KindType Kind = LSRUse::Basic;
     Type *AccessTy = nullptr;
@@ -2995,7 +2989,7 @@ void LSRInstance::CollectFixupsAndInitialFormulae() {
       AccessTy = getAccessType(LF.UserInst);
     }
 
-    const SCEV *S = IU.getExpr(*UI);
+    const SCEV *S = IU.getExpr(U);
 
     // Equality (== and !=) ICmps are special. We can rewrite (i == N) as
     // (N - i == 0), and this allows (N - i) to be the expression that we work
@@ -3087,9 +3081,8 @@ LSRInstance::InsertSupplementalFormula(const SCEV *S,
 void LSRInstance::CountRegisters(const Formula &F, size_t LUIdx) {
   if (F.ScaledReg)
     RegUses.CountRegister(F.ScaledReg, LUIdx);
-  for (SmallVectorImpl<const SCEV *>::const_iterator I = F.BaseRegs.begin(),
-       E = F.BaseRegs.end(); I != E; ++I)
-    RegUses.CountRegister(*I, LUIdx);
+  for (const SCEV *BaseReg : F.BaseRegs)
+    RegUses.CountRegister(BaseReg, LUIdx);
 }
 
 /// InsertFormula - If the given formula has not yet been inserted, add it to
@@ -3210,9 +3203,8 @@ static const SCEV *CollectSubexprs(const SCEV *S, const SCEVConstant *C,
 
   if (const SCEVAddExpr *Add = dyn_cast<SCEVAddExpr>(S)) {
     // Break out add operands.
-    for (SCEVAddExpr::op_iterator I = Add->op_begin(), E = Add->op_end();
-         I != E; ++I) {
-      const SCEV *Remainder = CollectSubexprs(*I, C, Ops, L, SE, Depth+1);
+    for (const SCEV *S : Add->operands()) {
+      const SCEV *Remainder = CollectSubexprs(S, C, Ops, L, SE, Depth+1);
       if (Remainder)
         Ops.push_back(C ? SE.getMulExpr(C, Remainder) : Remainder);
     }
@@ -3370,9 +3362,7 @@ void LSRInstance::GenerateCombinations(LSRUse &LU, unsigned LUIdx,
   Formula F = Base;
   F.BaseRegs.clear();
   SmallVector<const SCEV *, 4> Ops;
-  for (SmallVectorImpl<const SCEV *>::const_iterator
-       I = Base.BaseRegs.begin(), E = Base.BaseRegs.end(); I != E; ++I) {
-    const SCEV *BaseReg = *I;
+  for (const SCEV *BaseReg : Base.BaseRegs) {
     if (SE.properlyDominates(BaseReg, L->getHeader()) &&
         !SE.hasComputableLoopEvolution(BaseReg, L))
       Ops.push_back(BaseReg);
@@ -3429,15 +3419,13 @@ void LSRInstance::GenerateConstantOffsetsImpl(
     LSRUse &LU, unsigned LUIdx, const Formula &Base,
     const SmallVectorImpl<int64_t> &Worklist, size_t Idx, bool IsScaledReg) {
   const SCEV *G = IsScaledReg ? Base.ScaledReg : Base.BaseRegs[Idx];
-  for (SmallVectorImpl<int64_t>::const_iterator I = Worklist.begin(),
-                                                E = Worklist.end();
-       I != E; ++I) {
+  for (int64_t Offset : Worklist) {
     Formula F = Base;
-    F.BaseOffset = (uint64_t)Base.BaseOffset - *I;
-    if (isLegalUse(TTI, LU.MinOffset - *I, LU.MaxOffset - *I, LU.Kind,
+    F.BaseOffset = (uint64_t)Base.BaseOffset - Offset;
+    if (isLegalUse(TTI, LU.MinOffset - Offset, LU.MaxOffset - Offset, LU.Kind,
                    LU.AccessTy, F)) {
       // Add the offset to the base register.
-      const SCEV *NewG = SE.getAddExpr(SE.getConstant(G->getType(), *I), G);
+      const SCEV *NewG = SE.getAddExpr(SE.getConstant(G->getType(), Offset), G);
       // If it cancelled out, drop the base register, otherwise update it.
       if (NewG->isZero()) {
         if (IsScaledReg) {
@@ -3503,10 +3491,7 @@ void LSRInstance::GenerateICmpZeroScales(LSRUse &LU, unsigned LUIdx,
   assert(!Base.BaseGV && "ICmpZero use is not legal!");
 
   // Check each interesting stride.
-  for (SmallSetVector<int64_t, 8>::const_iterator
-       I = Factors.begin(), E = Factors.end(); I != E; ++I) {
-    int64_t Factor = *I;
-
+  for (int64_t Factor : Factors) {
     // Check that the multiplication doesn't overflow.
     if (Base.BaseOffset == INT64_MIN && Factor == -1)
       continue;
@@ -3590,10 +3575,7 @@ void LSRInstance::GenerateScales(LSRUse &LU, unsigned LUIdx, Formula Base) {
   assert(Base.Scale == 0 && "Unscale did not did its job!");
 
   // Check each interesting stride.
-  for (SmallSetVector<int64_t, 8>::const_iterator
-       I = Factors.begin(), E = Factors.end(); I != E; ++I) {
-    int64_t Factor = *I;
-
+  for (int64_t Factor : Factors) {
     Base.Scale = Factor;
     Base.HasBaseReg = Base.BaseRegs.size() > 1;
     // Check whether this scale is going to be legal.
@@ -3649,16 +3631,13 @@ void LSRInstance::GenerateTruncates(LSRUse &LU, unsigned LUIdx, Formula Base) {
   if (!DstTy) return;
   DstTy = SE.getEffectiveSCEVType(DstTy);
 
-  for (SmallSetVector<Type *, 4>::const_iterator
-       I = Types.begin(), E = Types.end(); I != E; ++I) {
-    Type *SrcTy = *I;
+  for (Type *SrcTy : Types) {
     if (SrcTy != DstTy && TTI.isTruncateFree(SrcTy, DstTy)) {
       Formula F = Base;
 
-      if (F.ScaledReg) F.ScaledReg = SE.getAnyExtendExpr(F.ScaledReg, *I);
-      for (SmallVectorImpl<const SCEV *>::iterator J = F.BaseRegs.begin(),
-           JE = F.BaseRegs.end(); J != JE; ++J)
-        *J = SE.getAnyExtendExpr(*J, SrcTy);
+      if (F.ScaledReg) F.ScaledReg = SE.getAnyExtendExpr(F.ScaledReg, SrcTy);
+      for (const SCEV *&BaseReg : F.BaseRegs)
+        BaseReg = SE.getAnyExtendExpr(BaseReg, SrcTy);
 
       // TODO: This assumes we've done basic processing on all uses and
       // have an idea what the register usage is.
@@ -3670,6 +3649,64 @@ void LSRInstance::GenerateTruncates(LSRUse &LU, unsigned LUIdx, Formula Base) {
   }
 }
 
+/// GenerateZExts - If a scale or a base register can be rewritten as
+/// "Zext({A,+,1})" then consider a formula of that form.
+void LSRInstance::GenerateZExts(LSRUse &LU, unsigned LUIdx, Formula Base) {
+  // Don't bother with symbolic values.
+  if (Base.BaseGV)
+    return;
+
+  auto CanBeNarrowed = [&](const SCEV *Reg) -> const SCEV * {
+    // Check if the register is an increment can be rewritten as zext(R) where
+    // the zext is free.
+
+    const auto *RegAR = dyn_cast_or_null<SCEVAddRecExpr>(Reg);
+    if (!RegAR)
+      return nullptr;
+
+    const auto *ZExtStart = dyn_cast<SCEVZeroExtendExpr>(RegAR->getStart());
+    const auto *ConstStep =
+        dyn_cast<SCEVConstant>(RegAR->getStepRecurrence(SE));
+    if (!ZExtStart || !ConstStep || ConstStep->getValue()->getValue() != 1)
+      return nullptr;
+
+    const SCEV *NarrowStart = ZExtStart->getOperand();
+    if (!TTI.isZExtFree(NarrowStart->getType(), ZExtStart->getType()))
+      return nullptr;
+
+    const auto *NarrowAR = dyn_cast<SCEVAddRecExpr>(
+        SE.getAddRecExpr(NarrowStart, SE.getConstant(NarrowStart->getType(), 1),
+                         RegAR->getLoop(), RegAR->getNoWrapFlags()));
+
+    if (!NarrowAR || !NarrowAR->getNoWrapFlags(SCEV::FlagNUW))
+      return nullptr;
+
+    return NarrowAR;
+  };
+
+  if (Base.ScaledReg && !Base.ZeroExtendType)
+    if (const SCEV *S = CanBeNarrowed(Base.ScaledReg)) {
+      Formula F = Base;
+      F.ZeroExtendType = Base.ScaledReg->getType();
+      F.ZeroExtendScaledReg = true;
+      F.ScaledReg = S;
+
+      if (isLegalUse(TTI, LU.MinOffset, LU.MaxOffset, LU.Kind, LU.AccessTy, F))
+        InsertFormula(LU, LUIdx, F);
+    }
+
+  if (Base.BaseRegs.size() == 1 && !Base.ZeroExtendType)
+    if (const SCEV *S = CanBeNarrowed(Base.BaseRegs[0])) {
+      Formula F = Base;
+      F.ZeroExtendType = Base.BaseRegs[0]->getType();
+      F.ZeroExtendBaseReg = true;
+      F.BaseRegs[0] = S;
+
+      if (isLegalUse(TTI, LU.MinOffset, LU.MaxOffset, LU.Kind, LU.AccessTy, F))
+        InsertFormula(LU, LUIdx, F);
+    }
+}
+
 namespace {
 
 /// WorkItem - Helper class for GenerateCrossUseConstantOffsets. It's used to
@@ -3705,20 +3742,17 @@ void WorkItem::dump() const {
 void LSRInstance::GenerateCrossUseConstantOffsets() {
   // Group the registers by their value without any added constant offset.
   typedef std::map<int64_t, const SCEV *> ImmMapTy;
-  typedef DenseMap<const SCEV *, ImmMapTy> RegMapTy;
-  RegMapTy Map;
+  DenseMap<const SCEV *, ImmMapTy> Map;
   DenseMap<const SCEV *, SmallBitVector> UsedByIndicesMap;
   SmallVector<const SCEV *, 8> Sequence;
-  for (RegUseTracker::const_iterator I = RegUses.begin(), E = RegUses.end();
-       I != E; ++I) {
-    const SCEV *Reg = *I;
+  for (const SCEV *Use : RegUses) {
+    const SCEV *Reg = Use; // Make a copy for ExtractImmediate to modify.
     int64_t Imm = ExtractImmediate(Reg, SE);
-    std::pair<RegMapTy::iterator, bool> Pair =
-      Map.insert(std::make_pair(Reg, ImmMapTy()));
+    auto Pair = Map.insert(std::make_pair(Reg, ImmMapTy()));
     if (Pair.second)
       Sequence.push_back(Reg);
-    Pair.first->second.insert(std::make_pair(Imm, *I));
-    UsedByIndicesMap[Reg] |= RegUses.getUsedByIndices(*I);
+    Pair.first->second.insert(std::make_pair(Imm, Use));
+    UsedByIndicesMap[Reg] |= RegUses.getUsedByIndices(Use);
   }
 
   // Now examine each set of registers with the same base value. Build up
@@ -3726,9 +3760,7 @@ void LSRInstance::GenerateCrossUseConstantOffsets() {
   // not adding formulae and register counts while we're searching.
   SmallVector<WorkItem, 32> WorkItems;
   SmallSet<std::pair<size_t, int64_t>, 32> UniqueItems;
-  for (SmallVectorImpl<const SCEV *>::const_iterator I = Sequence.begin(),
-       E = Sequence.end(); I != E; ++I) {
-    const SCEV *Reg = *I;
+  for (const SCEV *Reg : Sequence) {
     const ImmMapTy &Imms = Map.find(Reg)->second;
 
     // It's not worthwhile looking for reuse if there's only one offset.
@@ -3736,9 +3768,8 @@ void LSRInstance::GenerateCrossUseConstantOffsets() {
       continue;
 
     DEBUG(dbgs() << "Generating cross-use offsets for " << *Reg << ':';
-          for (ImmMapTy::const_iterator J = Imms.begin(), JE = Imms.end();
-               J != JE; ++J)
-            dbgs() << ' ' << J->first;
+          for (const auto &Entry : Imms)
+            dbgs() << ' ' << Entry.first;
           dbgs() << '\n');
 
     // Examine each offset.
@@ -3783,9 +3814,7 @@ void LSRInstance::GenerateCrossUseConstantOffsets() {
   UniqueItems.clear();
 
   // Now iterate through the worklist and add new formulae.
-  for (SmallVectorImpl<WorkItem>::const_iterator I = WorkItems.begin(),
-       E = WorkItems.end(); I != E; ++I) {
-    const WorkItem &WI = *I;
+  for (const WorkItem &WI : WorkItems) {
     size_t LUIdx = WI.LUIdx;
     LSRUse &LU = Uses[LUIdx];
     int64_t Imm = WI.Imm;
@@ -3850,10 +3879,8 @@ void LSRInstance::GenerateCrossUseConstantOffsets() {
           // If the new formula has a constant in a register, and adding the
           // constant value to the immediate would produce a value closer to
           // zero than the immediate itself, then the formula isn't worthwhile.
-          for (SmallVectorImpl<const SCEV *>::const_iterator
-               J = NewF.BaseRegs.begin(), JE = NewF.BaseRegs.end();
-               J != JE; ++J)
-            if (const SCEVConstant *C = dyn_cast<SCEVConstant>(*J))
+          for (const SCEV *NewReg : NewF.BaseRegs)
+            if (const SCEVConstant *C = dyn_cast<SCEVConstant>(NewReg))
               if ((C->getValue()->getValue() + NewF.BaseOffset).abs().slt(
                    std::abs(NewF.BaseOffset)) &&
                   (C->getValue()->getValue() +
@@ -3899,6 +3926,8 @@ LSRInstance::GenerateAllReuseFormulae() {
     LSRUse &LU = Uses[LUIdx];
     for (size_t i = 0, f = LU.Formulae.size(); i != f; ++i)
       GenerateTruncates(LU, LUIdx, LU.Formulae[i]);
+    for (size_t i = 0, f = LU.Formulae.size(); i != f; ++i)
+      GenerateZExts(LU, LUIdx, LU.Formulae[i]);
   }
 
   GenerateCrossUseConstantOffsets();
@@ -3956,9 +3985,7 @@ void LSRInstance::FilterOutUndesirableDedicatedRegisters() {
       }
       else {
         SmallVector<const SCEV *, 4> Key;
-        for (SmallVectorImpl<const SCEV *>::const_iterator J = F.BaseRegs.begin(),
-               JE = F.BaseRegs.end(); J != JE; ++J) {
-          const SCEV *Reg = *J;
+        for (const SCEV *Reg : F.BaseRegs) {
           if (RegUses.isRegUsedByUsesOtherThan(Reg, LUIdx))
             Key.push_back(Reg);
         }
@@ -4020,9 +4047,8 @@ static const size_t ComplexityLimit = UINT16_MAX;
 /// isn't always sufficient.
 size_t LSRInstance::EstimateSearchSpaceComplexity() const {
   size_t Power = 1;
-  for (SmallVectorImpl<LSRUse>::const_iterator I = Uses.begin(),
-       E = Uses.end(); I != E; ++I) {
-    size_t FSize = I->Formulae.size();
+  for (const LSRUse &LU : Uses) {
+    size_t FSize = LU.Formulae.size();
     if (FSize >= ComplexityLimit) {
       Power = ComplexityLimit;
       break;
@@ -4113,9 +4139,7 @@ void LSRInstance::NarrowSearchSpaceByCollapsingUnrolledCode() {
 
   for (size_t LUIdx = 0, NumUses = Uses.size(); LUIdx != NumUses; ++LUIdx) {
     LSRUse &LU = Uses[LUIdx];
-    for (SmallVectorImpl<Formula>::const_iterator I = LU.Formulae.begin(),
-         E = LU.Formulae.end(); I != E; ++I) {
-      const Formula &F = *I;
+    for (const Formula &F : LU.Formulae) {
       if (F.BaseOffset == 0 || (F.Scale != 0 && F.Scale != 1))
         continue;
 
@@ -4132,9 +4156,7 @@ void LSRInstance::NarrowSearchSpaceByCollapsingUnrolledCode() {
       LUThatHas->AllFixupsOutsideLoop &= LU.AllFixupsOutsideLoop;
 
       // Update the relocs to reference the new use.
-      for (SmallVectorImpl<LSRFixup>::iterator I = Fixups.begin(),
-           E = Fixups.end(); I != E; ++I) {
-        LSRFixup &Fixup = *I;
+      for (LSRFixup &Fixup : Fixups) {
         if (Fixup.LUIdx == LUIdx) {
           Fixup.LUIdx = LUThatHas - &Uses.front();
           Fixup.Offset += F.BaseOffset;
@@ -4215,9 +4237,7 @@ void LSRInstance::NarrowSearchSpaceByPickingWinnerRegs() {
     // to be a good reuse register candidate.
     const SCEV *Best = nullptr;
     unsigned BestNum = 0;
-    for (RegUseTracker::const_iterator I = RegUses.begin(), E = RegUses.end();
-         I != E; ++I) {
-      const SCEV *Reg = *I;
+    for (const SCEV *Reg : RegUses) {
       if (Taken.count(Reg))
         continue;
       if (!Best)
@@ -4305,17 +4325,12 @@ void LSRInstance::SolveRecurse(SmallVectorImpl<const Formula *> &Solution,
 
   SmallPtrSet<const SCEV *, 16> NewRegs;
   Cost NewCost;
-  for (SmallVectorImpl<Formula>::const_iterator I = LU.Formulae.begin(),
-       E = LU.Formulae.end(); I != E; ++I) {
-    const Formula &F = *I;
-
+  for (const Formula &F : LU.Formulae) {
     // Ignore formulae which may not be ideal in terms of register reuse of
     // ReqRegs.  The formula should use all required registers before
     // introducing new ones.
     int NumReqRegsToFind = std::min(F.getNumRegs(), ReqRegs.size());
-    for (SmallSetVector<const SCEV *, 4>::const_iterator J = ReqRegs.begin(),
-         JE = ReqRegs.end(); J != JE; ++J) {
-      const SCEV *Reg = *J;
+    for (const SCEV *Reg : ReqRegs) {
       if ((F.ScaledReg && F.ScaledReg == Reg) ||
           std::find(F.BaseRegs.begin(), F.BaseRegs.end(), Reg) !=
           F.BaseRegs.end()) {
@@ -4423,9 +4438,7 @@ LSRInstance::HoistInsertPosition(BasicBlock::iterator IP,
     bool AllDominate = true;
     Instruction *BetterPos = nullptr;
     Instruction *Tentative = IDom->getTerminator();
-    for (SmallVectorImpl<Instruction *>::const_iterator I = Inputs.begin(),
-         E = Inputs.end(); I != E; ++I) {
-      Instruction *Inst = *I;
+    for (Instruction *Inst : Inputs) {
       if (Inst == Tentative || !DT.dominates(Inst, Tentative)) {
         AllDominate = false;
         break;
@@ -4472,9 +4485,7 @@ LSRInstance::AdjustInsertPositionForExpand(BasicBlock::iterator LowestIP,
   }
   // The expansion must also be dominated by the increment positions of any
   // loops it for which it is using post-inc mode.
-  for (PostIncLoopSet::const_iterator I = LF.PostIncLoops.begin(),
-       E = LF.PostIncLoops.end(); I != E; ++I) {
-    const Loop *PIL = *I;
+  for (const Loop *PIL : LF.PostIncLoops) {
     if (PIL == L) continue;
 
     // Be dominated by the loop exit.
@@ -4549,20 +4560,33 @@ Value *LSRInstance::Expand(const LSRFixup &LF,
   SmallVector<const SCEV *, 8> Ops;
 
   // Expand the BaseRegs portion.
-  for (SmallVectorImpl<const SCEV *>::const_iterator I = F.BaseRegs.begin(),
-       E = F.BaseRegs.end(); I != E; ++I) {
-    const SCEV *Reg = *I;
+  for (const SCEV *Reg : F.BaseRegs) {
     assert(!Reg->isZero() && "Zero allocated in a base register!");
 
     // If we're expanding for a post-inc user, make the post-inc adjustment.
     PostIncLoopSet &Loops = const_cast<PostIncLoopSet &>(LF.PostIncLoops);
-    Reg = TransformForPostIncUse(Denormalize, Reg,
-                                 LF.UserInst, LF.OperandValToReplace,
-                                 Loops, SE, DT);
-
-    Ops.push_back(SE.getUnknown(Rewriter.expandCodeFor(Reg, nullptr, IP)));
+    const SCEV *ExtendedReg =
+        F.ZeroExtendBaseReg ? SE.getZeroExtendExpr(Reg, F.ZeroExtendType) : Reg;
+
+    const SCEV *PostIncReg =
+        TransformForPostIncUse(Denormalize, ExtendedReg, LF.UserInst,
+                               LF.OperandValToReplace, Loops, SE, DT);
+    if (PostIncReg == ExtendedReg) {
+      Value *Expanded = Rewriter.expandCodeFor(Reg, nullptr, IP);
+      if (F.ZeroExtendBaseReg)
+        Expanded = new ZExtInst(Expanded, F.ZeroExtendType, "", IP);
+      Ops.push_back(SE.getUnknown(Expanded));
+    } else {
+      Ops.push_back(
+          SE.getUnknown(Rewriter.expandCodeFor(PostIncReg, nullptr, IP)));
+    }
   }
 
+  // Note on post-inc uses and zero extends -- since the no-wrap behavior for
+  // the post-inc SCEV can be different from the no-wrap behavior of the pre-inc
+  // SCEV, if a post-inc transform is required we do the zero extension on the
+  // pre-inc expression before doing the post-inc transform.
+
   // Expand the ScaledReg portion.
   Value *ICmpScaledV = nullptr;
   if (F.Scale != 0) {
@@ -4570,22 +4594,33 @@ Value *LSRInstance::Expand(const LSRFixup &LF,
 
     // If we're expanding for a post-inc user, make the post-inc adjustment.
     PostIncLoopSet &Loops = const_cast<PostIncLoopSet &>(LF.PostIncLoops);
-    ScaledS = TransformForPostIncUse(Denormalize, ScaledS,
-                                     LF.UserInst, LF.OperandValToReplace,
-                                     Loops, SE, DT);
+    const SCEV *ExtendedScaleS =
+        F.ZeroExtendScaledReg ? SE.getZeroExtendExpr(ScaledS, F.ZeroExtendType)
+                              : ScaledS;
+    const SCEV *PostIncScaleS =
+        TransformForPostIncUse(Denormalize, ExtendedScaleS, LF.UserInst,
+                               LF.OperandValToReplace, Loops, SE, DT);
 
     if (LU.Kind == LSRUse::ICmpZero) {
       // Expand ScaleReg as if it was part of the base regs.
+      Value *Expanded = nullptr;
+      if (PostIncScaleS == ExtendedScaleS) {
+        Expanded = Rewriter.expandCodeFor(ScaledS, nullptr, IP);
+        if (F.ZeroExtendScaledReg)
+          Expanded = new ZExtInst(Expanded, F.ZeroExtendType, "", IP);
+      } else {
+        Expanded = Rewriter.expandCodeFor(PostIncScaleS, nullptr, IP);
+      }
+
       if (F.Scale == 1)
-        Ops.push_back(
-            SE.getUnknown(Rewriter.expandCodeFor(ScaledS, nullptr, IP)));
+        Ops.push_back(SE.getUnknown(Expanded));
       else {
         // An interesting way of "folding" with an icmp is to use a negated
         // scale, which we'll implement by inserting it into the other operand
         // of the icmp.
         assert(F.Scale == -1 &&
                "The only scale supported by ICmpZero uses is -1!");
-        ICmpScaledV = Rewriter.expandCodeFor(ScaledS, nullptr, IP);
+        ICmpScaledV = Expanded;
       }
     } else {
       // Otherwise just expand the scaled register and an explicit scale,
@@ -4599,7 +4634,17 @@ Value *LSRInstance::Expand(const LSRFixup &LF,
         Ops.clear();
         Ops.push_back(SE.getUnknown(FullV));
       }
-      ScaledS = SE.getUnknown(Rewriter.expandCodeFor(ScaledS, nullptr, IP));
+
+      Value *Expanded = nullptr;
+      if (PostIncScaleS == ExtendedScaleS) {
+        Expanded = Rewriter.expandCodeFor(ScaledS, nullptr, IP);
+        if (F.ZeroExtendScaledReg)
+          Expanded = new ZExtInst(Expanded, F.ZeroExtendType, "", IP);
+      } else {
+        Expanded = Rewriter.expandCodeFor(PostIncScaleS, nullptr, IP);
+      }
+
+      ScaledS = SE.getUnknown(Expanded);
       if (F.Scale != 1)
         ScaledS =
             SE.getMulExpr(ScaledS, SE.getConstant(ScaledS->getType(), F.Scale));
@@ -4667,7 +4712,7 @@ Value *LSRInstance::Expand(const LSRFixup &LF,
   // form, update the ICmp's other operand.
   if (LU.Kind == LSRUse::ICmpZero) {
     ICmpInst *CI = cast<ICmpInst>(LF.UserInst);
-    DeadInsts.push_back(CI->getOperand(1));
+    DeadInsts.emplace_back(CI->getOperand(1));
     assert(!F.BaseGV && "ICmp does not support folding a global value and "
                            "a scale at the same time!");
     if (F.Scale == -1) {
@@ -4731,8 +4776,7 @@ void LSRInstance::RewriteForPHI(PHINode *PN,
                                           .setDontDeleteUselessPHIs());
           } else {
             SmallVector<BasicBlock*, 2> NewBBs;
-            SplitLandingPadPredecessors(Parent, BB, "", "", NewBBs,
-                                        /*AliasAnalysis*/ nullptr, &DT, &LI);
+            SplitLandingPadPredecessors(Parent, BB, "", "", NewBBs, &DT, &LI);
             NewBB = NewBBs[0];
           }
           // If NewBB==NULL, then SplitCriticalEdge refused to split because all
@@ -4810,7 +4854,7 @@ void LSRInstance::Rewrite(const LSRFixup &LF,
       LF.UserInst->replaceUsesOfWith(LF.OperandValToReplace, FullV);
   }
 
-  DeadInsts.push_back(LF.OperandValToReplace);
+  DeadInsts.emplace_back(LF.OperandValToReplace);
 }
 
 /// ImplementSolution - Rewrite all the fixup locations with new values,
@@ -4832,25 +4876,20 @@ LSRInstance::ImplementSolution(const SmallVectorImpl<const Formula *> &Solution,
   Rewriter.setIVIncInsertPos(L, IVIncInsertPos);
 
   // Mark phi nodes that terminate chains so the expander tries to reuse them.
-  for (SmallVectorImpl<IVChain>::const_iterator ChainI = IVChainVec.begin(),
-         ChainE = IVChainVec.end(); ChainI != ChainE; ++ChainI) {
-    if (PHINode *PN = dyn_cast<PHINode>(ChainI->tailUserInst()))
+  for (const IVChain &Chain : IVChainVec) {
+    if (PHINode *PN = dyn_cast<PHINode>(Chain.tailUserInst()))
       Rewriter.setChainedPhi(PN);
   }
 
   // Expand the new value definitions and update the users.
-  for (SmallVectorImpl<LSRFixup>::const_iterator I = Fixups.begin(),
-       E = Fixups.end(); I != E; ++I) {
-    const LSRFixup &Fixup = *I;
-
+  for (const LSRFixup &Fixup : Fixups) {
     Rewrite(Fixup, *Solution[Fixup.LUIdx], Rewriter, DeadInsts, P);
 
     Changed = true;
   }
 
-  for (SmallVectorImpl<IVChain>::const_iterator ChainI = IVChainVec.begin(),
-         ChainE = IVChainVec.end(); ChainI != ChainE; ++ChainI) {
-    GenerateIVChain(*ChainI, Rewriter, DeadInsts);
+  for (const IVChain &Chain : IVChainVec) {
+    GenerateIVChain(Chain, Rewriter, DeadInsts);
     Changed = true;
   }
   // Clean up after ourselves. This must be done before deleting any
@@ -4877,10 +4916,10 @@ LSRInstance::LSRInstance(Loop *L, Pass *P)
   // If there's too much analysis to be done, bail early. We won't be able to
   // model the problem anyway.
   unsigned NumUsers = 0;
-  for (IVUsers::const_iterator UI = IU.begin(), E = IU.end(); UI != E; ++UI) {
+  for (const IVStrideUse &U : IU) {
     if (++NumUsers > MaxIVUsers) {
-      DEBUG(dbgs() << "LSR skipping loop, too many IV Users in " << *L
-            << "\n");
+      (void)U;
+      DEBUG(dbgs() << "LSR skipping loop, too many IV Users in " << U << "\n");
       return;
     }
   }
@@ -4949,14 +4988,10 @@ LSRInstance::LSRInstance(Loop *L, Pass *P)
 
 #ifndef NDEBUG
   // Formulae should be legal.
-  for (SmallVectorImpl<LSRUse>::const_iterator I = Uses.begin(), E = Uses.end();
-       I != E; ++I) {
-    const LSRUse &LU = *I;
-    for (SmallVectorImpl<Formula>::const_iterator J = LU.Formulae.begin(),
-                                                  JE = LU.Formulae.end();
-         J != JE; ++J)
+  for (const LSRUse &LU : Uses) {
+    for (const Formula &F : LU.Formulae)
       assert(isLegalUse(TTI, LU.MinOffset, LU.MaxOffset, LU.Kind, LU.AccessTy,
-                        *J) && "Illegal formula generated!");
+                        F) && "Illegal formula generated!");
   };
 #endif
 
@@ -4970,44 +5005,38 @@ void LSRInstance::print_factors_and_types(raw_ostream &OS) const {
   OS << "LSR has identified the following interesting factors and types: ";
   bool First = true;
 
-  for (SmallSetVector<int64_t, 8>::const_iterator
-       I = Factors.begin(), E = Factors.end(); I != E; ++I) {
+  for (int64_t Factor : Factors) {
     if (!First) OS << ", ";
     First = false;
-    OS << '*' << *I;
+    OS << '*' << Factor;
   }
 
-  for (SmallSetVector<Type *, 4>::const_iterator
-       I = Types.begin(), E = Types.end(); I != E; ++I) {
+  for (Type *Ty : Types) {
     if (!First) OS << ", ";
     First = false;
-    OS << '(' << **I << ')';
+    OS << '(' << *Ty << ')';
   }
   OS << '\n';
 }
 
 void LSRInstance::print_fixups(raw_ostream &OS) const {
   OS << "LSR is examining the following fixup sites:\n";
-  for (SmallVectorImpl<LSRFixup>::const_iterator I = Fixups.begin(),
-       E = Fixups.end(); I != E; ++I) {
+  for (const LSRFixup &LF : Fixups) {
     dbgs() << "  ";
-    I->print(OS);
+    LF.print(OS);
     OS << '\n';
   }
 }
 
 void LSRInstance::print_uses(raw_ostream &OS) const {
   OS << "LSR is examining the following uses:\n";
-  for (SmallVectorImpl<LSRUse>::const_iterator I = Uses.begin(),
-       E = Uses.end(); I != E; ++I) {
-    const LSRUse &LU = *I;
+  for (const LSRUse &LU : Uses) {
     dbgs() << "  ";
     LU.print(OS);
     OS << '\n';
-    for (SmallVectorImpl<Formula>::const_iterator J = LU.Formulae.begin(),
-         JE = LU.Formulae.end(); J != JE; ++J) {
+    for (const Formula &F : LU.Formulae) {
       OS << "    ";
-      J->print(OS);
+      F.print(OS);
       OS << '\n';
     }
   }
diff --git a/lib/Transforms/Scalar/LoopUnrollPass.cpp b/lib/Transforms/Scalar/LoopUnrollPass.cpp
index 3de3b3d980b..6b64d4e6ecf 100644
--- a/lib/Transforms/Scalar/LoopUnrollPass.cpp
+++ b/lib/Transforms/Scalar/LoopUnrollPass.cpp
@@ -38,25 +38,25 @@ using namespace llvm;
 #define DEBUG_TYPE "loop-unroll"
 
 static cl::opt<unsigned>
-UnrollThreshold("unroll-threshold", cl::init(150), cl::Hidden,
-  cl::desc("The cut-off point for automatic loop unrolling"));
+    UnrollThreshold("unroll-threshold", cl::init(150), cl::Hidden,
+                    cl::desc("The baseline cost threshold for loop unrolling"));
+
+static cl::opt<unsigned> UnrollPercentDynamicCostSavedThreshold(
+    "unroll-percent-dynamic-cost-saved-threshold", cl::init(20), cl::Hidden,
+    cl::desc("The percentage of estimated dynamic cost which must be saved by "
+             "unrolling to allow unrolling up to the max threshold."));
+
+static cl::opt<unsigned> UnrollDynamicCostSavingsDiscount(
+    "unroll-dynamic-cost-savings-discount", cl::init(2000), cl::Hidden,
+    cl::desc("This is the amount discounted from the total unroll cost when "
+             "the unrolled form has a high dynamic cost savings (triggered by "
+             "the '-unroll-perecent-dynamic-cost-saved-threshold' flag)."));
 
 static cl::opt<unsigned> UnrollMaxIterationsCountToAnalyze(
     "unroll-max-iteration-count-to-analyze", cl::init(0), cl::Hidden,
     cl::desc("Don't allow loop unrolling to simulate more than this number of"
              "iterations when checking full unroll profitability"));
 
-static cl::opt<unsigned> UnrollMinPercentOfOptimized(
-    "unroll-percent-of-optimized-for-complete-unroll", cl::init(20), cl::Hidden,
-    cl::desc("If complete unrolling could trigger further optimizations, and, "
-             "by that, remove the given percent of instructions, perform the "
-             "complete unroll even if it's beyond the threshold"));
-
-static cl::opt<unsigned> UnrollAbsoluteThreshold(
-    "unroll-absolute-threshold", cl::init(2000), cl::Hidden,
-    cl::desc("Don't unroll if the unrolled size is bigger than this threshold,"
-             " even if we can remove big portion of instructions later."));
-
 static cl::opt<unsigned>
 UnrollCount("unroll-count", cl::init(0), cl::Hidden,
   cl::desc("Use this unroll count for all loops including those with "
@@ -82,16 +82,18 @@ namespace {
     static char ID; // Pass ID, replacement for typeid
     LoopUnroll(int T = -1, int C = -1, int P = -1, int R = -1) : LoopPass(ID) {
       CurrentThreshold = (T == -1) ? UnrollThreshold : unsigned(T);
-      CurrentAbsoluteThreshold = UnrollAbsoluteThreshold;
-      CurrentMinPercentOfOptimized = UnrollMinPercentOfOptimized;
+      CurrentPercentDynamicCostSavedThreshold =
+          UnrollPercentDynamicCostSavedThreshold;
+      CurrentDynamicCostSavingsDiscount = UnrollDynamicCostSavingsDiscount;
       CurrentCount = (C == -1) ? UnrollCount : unsigned(C);
       CurrentAllowPartial = (P == -1) ? UnrollAllowPartial : (bool)P;
       CurrentRuntime = (R == -1) ? UnrollRuntime : (bool)R;
 
       UserThreshold = (T != -1) || (UnrollThreshold.getNumOccurrences() > 0);
-      UserAbsoluteThreshold = (UnrollAbsoluteThreshold.getNumOccurrences() > 0);
-      UserPercentOfOptimized =
-          (UnrollMinPercentOfOptimized.getNumOccurrences() > 0);
+      UserPercentDynamicCostSavedThreshold =
+          (UnrollPercentDynamicCostSavedThreshold.getNumOccurrences() > 0);
+      UserDynamicCostSavingsDiscount =
+          (UnrollDynamicCostSavingsDiscount.getNumOccurrences() > 0);
       UserAllowPartial = (P != -1) ||
                          (UnrollAllowPartial.getNumOccurrences() > 0);
       UserRuntime = (R != -1) || (UnrollRuntime.getNumOccurrences() > 0);
@@ -115,18 +117,18 @@ namespace {
 
     unsigned CurrentCount;
     unsigned CurrentThreshold;
-    unsigned CurrentAbsoluteThreshold;
-    unsigned CurrentMinPercentOfOptimized;
-    bool     CurrentAllowPartial;
-    bool     CurrentRuntime;
-    bool     UserCount;            // CurrentCount is user-specified.
-    bool     UserThreshold;        // CurrentThreshold is user-specified.
-    bool UserAbsoluteThreshold;    // CurrentAbsoluteThreshold is
-                                   // user-specified.
-    bool UserPercentOfOptimized;   // CurrentMinPercentOfOptimized is
-                                   // user-specified.
-    bool     UserAllowPartial;     // CurrentAllowPartial is user-specified.
-    bool     UserRuntime;          // CurrentRuntime is user-specified.
+    unsigned CurrentPercentDynamicCostSavedThreshold;
+    unsigned CurrentDynamicCostSavingsDiscount;
+    bool CurrentAllowPartial;
+    bool CurrentRuntime;
+
+    // Flags for whether the 'current' settings are user-specified.
+    bool UserCount;
+    bool UserThreshold;
+    bool UserPercentDynamicCostSavedThreshold;
+    bool UserDynamicCostSavingsDiscount;
+    bool UserAllowPartial;
+    bool UserRuntime;
 
     bool runOnLoop(Loop *L, LPPassManager &LPM) override;
 
@@ -156,8 +158,9 @@ namespace {
     void getUnrollingPreferences(Loop *L, const TargetTransformInfo &TTI,
                                  TargetTransformInfo::UnrollingPreferences &UP) {
       UP.Threshold = CurrentThreshold;
-      UP.AbsoluteThreshold = CurrentAbsoluteThreshold;
-      UP.MinPercentOfOptimized = CurrentMinPercentOfOptimized;
+      UP.PercentDynamicCostSavedThreshold =
+          CurrentPercentDynamicCostSavedThreshold;
+      UP.DynamicCostSavingsDiscount = CurrentDynamicCostSavingsDiscount;
       UP.OptSizeThreshold = OptSizeUnrollThreshold;
       UP.PartialThreshold = CurrentThreshold;
       UP.PartialOptSizeThreshold = OptSizeUnrollThreshold;
@@ -186,33 +189,23 @@ namespace {
     void selectThresholds(const Loop *L, bool HasPragma,
                           const TargetTransformInfo::UnrollingPreferences &UP,
                           unsigned &Threshold, unsigned &PartialThreshold,
-                          unsigned NumberOfOptimizedInstructions) {
+                          unsigned &PercentDynamicCostSavedThreshold,
+                          unsigned &DynamicCostSavingsDiscount) {
       // Determine the current unrolling threshold.  While this is
       // normally set from UnrollThreshold, it is overridden to a
       // smaller value if the current function is marked as
       // optimize-for-size, and the unroll threshold was not user
       // specified.
       Threshold = UserThreshold ? CurrentThreshold : UP.Threshold;
-
-      // If we are allowed to completely unroll if we can remove M% of
-      // instructions, and we know that with complete unrolling we'll be able
-      // to kill N instructions, then we can afford to completely unroll loops
-      // with unrolled size up to N*100/M.
-      // Adjust the threshold according to that:
-      unsigned PercentOfOptimizedForCompleteUnroll =
-          UserPercentOfOptimized ? CurrentMinPercentOfOptimized
-                                 : UP.MinPercentOfOptimized;
-      unsigned AbsoluteThreshold = UserAbsoluteThreshold
-                                       ? CurrentAbsoluteThreshold
-                                       : UP.AbsoluteThreshold;
-      if (PercentOfOptimizedForCompleteUnroll)
-        Threshold = std::max<unsigned>(Threshold,
-                                       NumberOfOptimizedInstructions * 100 /
-                                           PercentOfOptimizedForCompleteUnroll);
-      // But don't allow unrolling loops bigger than absolute threshold.
-      Threshold = std::min<unsigned>(Threshold, AbsoluteThreshold);
-
       PartialThreshold = UserThreshold ? CurrentThreshold : UP.PartialThreshold;
+      PercentDynamicCostSavedThreshold =
+          UserPercentDynamicCostSavedThreshold
+              ? CurrentPercentDynamicCostSavedThreshold
+              : UP.PercentDynamicCostSavedThreshold;
+      DynamicCostSavingsDiscount = UserDynamicCostSavingsDiscount
+                                       ? CurrentDynamicCostSavingsDiscount
+                                       : UP.DynamicCostSavingsDiscount;
+
       if (!UserThreshold &&
           L->getHeader()->getParent()->hasFnAttribute(
               Attribute::OptimizeForSize)) {
@@ -231,6 +224,10 @@ namespace {
               std::max<unsigned>(PartialThreshold, PragmaUnrollThreshold);
       }
     }
+    bool canUnrollCompletely(Loop *L, unsigned Threshold,
+                             unsigned PercentDynamicCostSavedThreshold,
+                             unsigned DynamicCostSavingsDiscount,
+                             uint64_t UnrolledCost, uint64_t RolledDynamicCost);
   };
 }
 
@@ -253,61 +250,7 @@ Pass *llvm::createSimpleLoopUnrollPass() {
   return llvm::createLoopUnrollPass(-1, -1, 0, 0);
 }
 
-static bool isLoadFromConstantInitializer(Value *V) {
-  if (GlobalVariable *GV = dyn_cast<GlobalVariable>(V))
-    if (GV->isConstant() && GV->hasDefinitiveInitializer())
-      return GV->getInitializer();
-  return false;
-}
-
 namespace {
-struct FindConstantPointers {
-  bool LoadCanBeConstantFolded;
-  bool IndexIsConstant;
-  APInt Step;
-  APInt StartValue;
-  Value *BaseAddress;
-  const Loop *L;
-  ScalarEvolution &SE;
-  FindConstantPointers(const Loop *loop, ScalarEvolution &SE)
-      : LoadCanBeConstantFolded(true), IndexIsConstant(true), L(loop), SE(SE) {}
-
-  bool follow(const SCEV *S) {
-    if (const SCEVUnknown *SC = dyn_cast<SCEVUnknown>(S)) {
-      // We've reached the leaf node of SCEV, it's most probably just a
-      // variable. Now it's time to see if it corresponds to a global constant
-      // global (in which case we can eliminate the load), or not.
-      BaseAddress = SC->getValue();
-      LoadCanBeConstantFolded =
-          IndexIsConstant && isLoadFromConstantInitializer(BaseAddress);
-      return false;
-    }
-    if (isa<SCEVConstant>(S))
-      return true;
-    if (const SCEVAddRecExpr *AR = dyn_cast<SCEVAddRecExpr>(S)) {
-      // If the current SCEV expression is AddRec, and its loop isn't the loop
-      // we are about to unroll, then we won't get a constant address after
-      // unrolling, and thus, won't be able to eliminate the load.
-      if (AR->getLoop() != L)
-        return IndexIsConstant = false;
-      // If the step isn't constant, we won't get constant addresses in unrolled
-      // version. Bail out.
-      if (const SCEVConstant *StepSE =
-              dyn_cast<SCEVConstant>(AR->getStepRecurrence(SE)))
-        Step = StepSE->getValue()->getValue();
-      else
-        return IndexIsConstant = false;
-
-      return IndexIsConstant;
-    }
-    // If Result is true, continue traversal.
-    // Otherwise, we have found something that prevents us from (possible) load
-    // elimination.
-    return IndexIsConstant;
-  }
-  bool isDone() const { return !IndexIsConstant; }
-};
-
 // This class is used to get an estimate of the optimization effects that we
 // could get from complete loop unrolling. It comes from the fact that some
 // loads might be replaced with concrete constant values and that could trigger
@@ -324,31 +267,112 @@ struct FindConstantPointers {
 //   v = b[0]* 0 + b[1]* 1 + b[2]* 0
 // And finally:
 //   v = b[1]
-class UnrollAnalyzer : public InstVisitor<UnrollAnalyzer, bool> {
-  typedef InstVisitor<UnrollAnalyzer, bool> Base;
-  friend class InstVisitor<UnrollAnalyzer, bool>;
+class UnrolledInstAnalyzer : private InstVisitor<UnrolledInstAnalyzer, bool> {
+  typedef InstVisitor<UnrolledInstAnalyzer, bool> Base;
+  friend class InstVisitor<UnrolledInstAnalyzer, bool>;
+  struct SimplifiedAddress {
+    Value *Base = nullptr;
+    ConstantInt *Offset = nullptr;
+  };
+
+public:
+  UnrolledInstAnalyzer(unsigned Iteration,
+                       DenseMap<Value *, Constant *> &SimplifiedValues,
+                       const Loop *L, ScalarEvolution &SE)
+      : Iteration(Iteration), SimplifiedValues(SimplifiedValues), L(L), SE(SE) {
+      IterationNumber = SE.getConstant(APInt(64, Iteration));
+  }
+
+  // Allow access to the initial visit method.
+  using Base::visit;
+
+private:
+  /// \brief A cache of pointer bases and constant-folded offsets corresponding
+  /// to GEP (or derived from GEP) instructions.
+  ///
+  /// In order to find the base pointer one needs to perform non-trivial
+  /// traversal of the corresponding SCEV expression, so it's good to have the
+  /// results saved.
+  DenseMap<Value *, SimplifiedAddress> SimplifiedAddresses;
+
+  /// \brief Number of currently simulated iteration.
+  ///
+  /// If an expression is ConstAddress+Constant, then the Constant is
+  /// Start + Iteration*Step, where Start and Step could be obtained from
+  /// SCEVGEPCache.
+  unsigned Iteration;
+
+  /// \brief SCEV expression corresponding to number of currently simulated
+  /// iteration.
+  const SCEV *IterationNumber;
+
+  /// \brief A Value->Constant map for keeping values that we managed to
+  /// constant-fold on the given iteration.
+  ///
+  /// While we walk the loop instructions, we build up and maintain a mapping
+  /// of simplified values specific to this iteration.  The idea is to propagate
+  /// any special information we have about loads that can be replaced with
+  /// constants after complete unrolling, and account for likely simplifications
+  /// post-unrolling.
+  DenseMap<Value *, Constant *> &SimplifiedValues;
 
   const Loop *L;
-  unsigned TripCount;
   ScalarEvolution &SE;
-  const TargetTransformInfo &TTI;
 
-  DenseMap<Value *, Constant *> SimplifiedValues;
-  DenseMap<LoadInst *, Value *> LoadBaseAddresses;
-  SmallPtrSet<Instruction *, 32> CountedInstructions;
-
-  /// \brief Count the number of optimized instructions.
-  unsigned NumberOfOptimizedInstructions;
-
-  // Provide base case for our instruction visit.
-  bool visitInstruction(Instruction &I) { return false; };
-  // TODO: We should also visit ICmp, FCmp, GetElementPtr, Trunc, ZExt, SExt,
-  // FPTrunc, FPExt, FPToUI, FPToSI, UIToFP, SIToFP, BitCast, Select,
-  // ExtractElement, InsertElement, ShuffleVector, ExtractValue, InsertValue.
-  //
-  // Probaly it's worth to hoist the code for estimating the simplifications
-  // effects to a separate class, since we have a very similar code in
-  // InlineCost already.
+  /// \brief Try to simplify instruction \param I using its SCEV expression.
+  ///
+  /// The idea is that some AddRec expressions become constants, which then
+  /// could trigger folding of other instructions. However, that only happens
+  /// for expressions whose start value is also constant, which isn't always the
+  /// case. In another common and important case the start value is just some
+  /// address (i.e. SCEVUnknown) - in this case we compute the offset and save
+  /// it along with the base address instead.
+  bool simplifyInstWithSCEV(Instruction *I) {
+    if (!SE.isSCEVable(I->getType()))
+      return false;
+
+    const SCEV *S = SE.getSCEV(I);
+    if (auto *SC = dyn_cast<SCEVConstant>(S)) {
+      SimplifiedValues[I] = SC->getValue();
+      return true;
+    }
+
+    auto *AR = dyn_cast<SCEVAddRecExpr>(S);
+    if (!AR)
+      return false;
+
+    const SCEV *ValueAtIteration = AR->evaluateAtIteration(IterationNumber, SE);
+    // Check if the AddRec expression becomes a constant.
+    if (auto *SC = dyn_cast<SCEVConstant>(ValueAtIteration)) {
+      SimplifiedValues[I] = SC->getValue();
+      return true;
+    }
+
+    // Check if the offset from the base address becomes a constant.
+    auto *Base = dyn_cast<SCEVUnknown>(SE.getPointerBase(S));
+    if (!Base)
+      return false;
+    auto *Offset =
+        dyn_cast<SCEVConstant>(SE.getMinusSCEV(ValueAtIteration, Base));
+    if (!Offset)
+      return false;
+    SimplifiedAddress Address;
+    Address.Base = Base->getValue();
+    Address.Offset = Offset->getValue();
+    SimplifiedAddresses[I] = Address;
+    return true;
+  }
+
+  /// Base case for the instruction visitor.
+  bool visitInstruction(Instruction &I) {
+    return simplifyInstWithSCEV(&I);
+  }
+
+  /// Try to simplify binary operator I.
+  ///
+  /// TODO: Probaly it's worth to hoist the code for estimating the
+  /// simplifications effects to a separate class, since we have a very similar
+  /// code in InlineCost already.
   bool visitBinaryOperator(BinaryOperator &I) {
     Value *LHS = I.getOperand(0), *RHS = I.getOperand(1);
     if (!isa<Constant>(LHS))
@@ -357,6 +381,7 @@ class UnrollAnalyzer : public InstVisitor<UnrollAnalyzer, bool> {
     if (!isa<Constant>(RHS))
       if (Constant *SimpleRHS = SimplifiedValues.lookup(RHS))
         RHS = SimpleRHS;
+
     Value *SimpleV = nullptr;
     const DataLayout &DL = I.getModule()->getDataLayout();
     if (auto FI = dyn_cast<FPMathOperator>(&I))
@@ -365,215 +390,261 @@ class UnrollAnalyzer : public InstVisitor<UnrollAnalyzer, bool> {
     else
       SimpleV = SimplifyBinOp(I.getOpcode(), LHS, RHS, DL);
 
-    if (SimpleV && CountedInstructions.insert(&I).second)
-      NumberOfOptimizedInstructions += TTI.getUserCost(&I);
-
-    if (Constant *C = dyn_cast_or_null<Constant>(SimpleV)) {
+    if (Constant *C = dyn_cast_or_null<Constant>(SimpleV))
       SimplifiedValues[&I] = C;
+
+    if (SimpleV)
       return true;
-    }
-    return false;
+    return Base::visitBinaryOperator(I);
   }
 
-  Constant *computeLoadValue(LoadInst *LI, unsigned Iteration) {
-    if (!LI)
-      return nullptr;
-    Value *BaseAddr = LoadBaseAddresses[LI];
-    if (!BaseAddr)
-      return nullptr;
+  /// Try to fold load I.
+  bool visitLoad(LoadInst &I) {
+    Value *AddrOp = I.getPointerOperand();
 
-    auto GV = dyn_cast<GlobalVariable>(BaseAddr);
-    if (!GV)
-      return nullptr;
+    auto AddressIt = SimplifiedAddresses.find(AddrOp);
+    if (AddressIt == SimplifiedAddresses.end())
+      return false;
+    ConstantInt *SimplifiedAddrOp = AddressIt->second.Offset;
+
+    auto *GV = dyn_cast<GlobalVariable>(AddressIt->second.Base);
+    // We're only interested in loads that can be completely folded to a
+    // constant.
+    if (!GV || !GV->hasInitializer())
+      return false;
 
     ConstantDataSequential *CDS =
         dyn_cast<ConstantDataSequential>(GV->getInitializer());
     if (!CDS)
-      return nullptr;
-
-    const SCEV *BaseAddrSE = SE.getSCEV(BaseAddr);
-    const SCEV *S = SE.getSCEV(LI->getPointerOperand());
-    const SCEV *OffSE = SE.getMinusSCEV(S, BaseAddrSE);
+      return false;
 
-    APInt StepC, StartC;
-    const SCEVAddRecExpr *AR = dyn_cast<SCEVAddRecExpr>(OffSE);
-    if (!AR)
-      return nullptr;
+    int ElemSize = CDS->getElementType()->getPrimitiveSizeInBits() / 8U;
+    assert(SimplifiedAddrOp->getValue().getActiveBits() < 64 &&
+           "Unexpectedly large index value.");
+    int64_t Index = SimplifiedAddrOp->getSExtValue() / ElemSize;
+    if (Index >= CDS->getNumElements()) {
+      // FIXME: For now we conservatively ignore out of bound accesses, but
+      // we're allowed to perform the optimization in this case.
+      return false;
+    }
 
-    if (const SCEVConstant *StepSE =
-            dyn_cast<SCEVConstant>(AR->getStepRecurrence(SE)))
-      StepC = StepSE->getValue()->getValue();
-    else
-      return nullptr;
+    Constant *CV = CDS->getElementAsConstant(Index);
+    assert(CV && "Constant expected.");
+    SimplifiedValues[&I] = CV;
 
-    if (const SCEVConstant *StartSE = dyn_cast<SCEVConstant>(AR->getStart()))
-      StartC = StartSE->getValue()->getValue();
-    else
-      return nullptr;
+    return true;
+  }
 
-    unsigned ElemSize = CDS->getElementType()->getPrimitiveSizeInBits() / 8U;
-    unsigned Start = StartC.getLimitedValue();
-    unsigned Step = StepC.getLimitedValue();
+  bool visitCastInst(CastInst &I) {
+    // Propagate constants through casts.
+    Constant *COp = dyn_cast<Constant>(I.getOperand(0));
+    if (!COp)
+      COp = SimplifiedValues.lookup(I.getOperand(0));
+    if (COp)
+      if (Constant *C =
+              ConstantExpr::getCast(I.getOpcode(), COp, I.getType())) {
+        SimplifiedValues[&I] = C;
+        return true;
+      }
 
-    unsigned Index = (Start + Step * Iteration) / ElemSize;
-    if (Index >= CDS->getNumElements())
-      return nullptr;
+    return Base::visitCastInst(I);
+  }
 
-    Constant *CV = CDS->getElementAsConstant(Index);
+  bool visitCmpInst(CmpInst &I) {
+    Value *LHS = I.getOperand(0), *RHS = I.getOperand(1);
 
-    return CV;
-  }
+    // First try to handle simplified comparisons.
+    if (!isa<Constant>(LHS))
+      if (Constant *SimpleLHS = SimplifiedValues.lookup(LHS))
+        LHS = SimpleLHS;
+    if (!isa<Constant>(RHS))
+      if (Constant *SimpleRHS = SimplifiedValues.lookup(RHS))
+        RHS = SimpleRHS;
 
-public:
-  UnrollAnalyzer(const Loop *L, unsigned TripCount, ScalarEvolution &SE,
-                 const TargetTransformInfo &TTI)
-      : L(L), TripCount(TripCount), SE(SE), TTI(TTI),
-        NumberOfOptimizedInstructions(0) {}
-
-  // Visit all loads the loop L, and for those that, after complete loop
-  // unrolling, would have a constant address and it will point to a known
-  // constant initializer, record its base address for future use.  It is used
-  // when we estimate number of potentially simplified instructions.
-  void findConstFoldableLoads() {
-    for (auto BB : L->getBlocks()) {
-      for (BasicBlock::iterator I = BB->begin(), E = BB->end(); I != E; ++I) {
-        if (LoadInst *LI = dyn_cast<LoadInst>(I)) {
-          if (!LI->isSimple())
-            continue;
-          Value *AddrOp = LI->getPointerOperand();
-          const SCEV *S = SE.getSCEV(AddrOp);
-          FindConstantPointers Visitor(L, SE);
-          SCEVTraversal<FindConstantPointers> T(Visitor);
-          T.visitAll(S);
-          if (Visitor.IndexIsConstant && Visitor.LoadCanBeConstantFolded) {
-            LoadBaseAddresses[LI] = Visitor.BaseAddress;
+    if (!isa<Constant>(LHS) && !isa<Constant>(RHS)) {
+      auto SimplifiedLHS = SimplifiedAddresses.find(LHS);
+      if (SimplifiedLHS != SimplifiedAddresses.end()) {
+        auto SimplifiedRHS = SimplifiedAddresses.find(RHS);
+        if (SimplifiedRHS != SimplifiedAddresses.end()) {
+          SimplifiedAddress &LHSAddr = SimplifiedLHS->second;
+          SimplifiedAddress &RHSAddr = SimplifiedRHS->second;
+          if (LHSAddr.Base == RHSAddr.Base) {
+            LHS = LHSAddr.Offset;
+            RHS = RHSAddr.Offset;
           }
         }
       }
     }
-  }
 
-  // Given a list of loads that could be constant-folded (LoadBaseAddresses),
-  // estimate number of optimized instructions after substituting the concrete
-  // values for the given Iteration. Also track how many instructions become
-  // dead through this process.
-  unsigned estimateNumberOfOptimizedInstructions(unsigned Iteration) {
-    // We keep a set vector for the worklist so that we don't wast space in the
-    // worklist queuing up the same instruction repeatedly. This can happen due
-    // to multiple operands being the same instruction or due to the same
-    // instruction being an operand of lots of things that end up dead or
-    // simplified.
-    SmallSetVector<Instruction *, 8> Worklist;
-
-    // Clear the simplified values and counts for this iteration.
-    SimplifiedValues.clear();
-    CountedInstructions.clear();
-    NumberOfOptimizedInstructions = 0;
-
-    // We start by adding all loads to the worklist.
-    for (auto &LoadDescr : LoadBaseAddresses) {
-      LoadInst *LI = LoadDescr.first;
-      SimplifiedValues[LI] = computeLoadValue(LI, Iteration);
-      if (CountedInstructions.insert(LI).second)
-        NumberOfOptimizedInstructions += TTI.getUserCost(LI);
-
-      for (User *U : LI->users())
-        Worklist.insert(cast<Instruction>(U));
+    if (Constant *CLHS = dyn_cast<Constant>(LHS)) {
+      if (Constant *CRHS = dyn_cast<Constant>(RHS)) {
+        if (Constant *C = ConstantExpr::getCompare(I.getPredicate(), CLHS, CRHS)) {
+          SimplifiedValues[&I] = C;
+          return true;
+        }
+      }
     }
 
-    // And then we try to simplify every user of every instruction from the
-    // worklist. If we do simplify a user, add it to the worklist to process
-    // its users as well.
-    while (!Worklist.empty()) {
-      Instruction *I = Worklist.pop_back_val();
-      if (!L->contains(I))
-        continue;
-      if (!visit(I))
-        continue;
-      for (User *U : I->users())
-        Worklist.insert(cast<Instruction>(U));
-    }
+    return Base::visitCmpInst(I);
+  }
+};
+} // namespace
+
+
+namespace {
+struct EstimatedUnrollCost {
+  /// \brief The estimated cost after unrolling.
+  unsigned UnrolledCost;
+
+  /// \brief The estimated dynamic cost of executing the instructions in the
+  /// rolled form.
+  unsigned RolledDynamicCost;
+};
+}
+
+/// \brief Figure out if the loop is worth full unrolling.
+///
+/// Complete loop unrolling can make some loads constant, and we need to know
+/// if that would expose any further optimization opportunities.  This routine
+/// estimates this optimization.  It computes cost of unrolled loop
+/// (UnrolledCost) and dynamic cost of the original loop (RolledDynamicCost). By
+/// dynamic cost we mean that we won't count costs of blocks that are known not
+/// to be executed (i.e. if we have a branch in the loop and we know that at the
+/// given iteration its condition would be resolved to true, we won't add up the
+/// cost of the 'false'-block).
+/// \returns Optional value, holding the RolledDynamicCost and UnrolledCost. If
+/// the analysis failed (no benefits expected from the unrolling, or the loop is
+/// too big to analyze), the returned value is None.
+Optional<EstimatedUnrollCost>
+analyzeLoopUnrollCost(const Loop *L, unsigned TripCount, ScalarEvolution &SE,
+                      const TargetTransformInfo &TTI,
+                      unsigned MaxUnrolledLoopSize) {
+  // We want to be able to scale offsets by the trip count and add more offsets
+  // to them without checking for overflows, and we already don't want to
+  // analyze *massive* trip counts, so we force the max to be reasonably small.
+  assert(UnrollMaxIterationsCountToAnalyze < (INT_MAX / 2) &&
+         "The unroll iterations max is too large!");
+
+  // Don't simulate loops with a big or unknown tripcount
+  if (!UnrollMaxIterationsCountToAnalyze || !TripCount ||
+      TripCount > UnrollMaxIterationsCountToAnalyze)
+    return None;
+
+  SmallSetVector<BasicBlock *, 16> BBWorklist;
+  DenseMap<Value *, Constant *> SimplifiedValues;
 
-    // Now that we know the potentially simplifed instructions, estimate number
-    // of instructions that would become dead if we do perform the
-    // simplification.
-
-    // The dead instructions are held in a separate set. This is used to
-    // prevent us from re-examining instructions and make sure we only count
-    // the benifit once. The worklist's internal set handles insertion
-    // deduplication.
-    SmallPtrSet<Instruction *, 16> DeadInstructions;
-
-    // Lambda to enque operands onto the worklist.
-    auto EnqueueOperands = [&](Instruction &I) {
-      for (auto *Op : I.operand_values())
-        if (auto *OpI = dyn_cast<Instruction>(Op))
-          if (!OpI->use_empty())
-            Worklist.insert(OpI);
-    };
-
-    // Start by initializing worklist with simplified instructions.
-    for (auto &FoldedKeyValue : SimplifiedValues)
-      if (auto *FoldedInst = dyn_cast<Instruction>(FoldedKeyValue.first)) {
-        DeadInstructions.insert(FoldedInst);
-
-        // Add each instruction operand of this dead instruction to the
-        // worklist.
-        EnqueueOperands(*FoldedInst);
+  // The estimated cost of the unrolled form of the loop. We try to estimate
+  // this by simplifying as much as we can while computing the estimate.
+  unsigned UnrolledCost = 0;
+  // We also track the estimated dynamic (that is, actually executed) cost in
+  // the rolled form. This helps identify cases when the savings from unrolling
+  // aren't just exposing dead control flows, but actual reduced dynamic
+  // instructions due to the simplifications which we expect to occur after
+  // unrolling.
+  unsigned RolledDynamicCost = 0;
+
+  DEBUG(dbgs() << "Starting LoopUnroll profitability analysis...\n");
+
+  // Simulate execution of each iteration of the loop counting instructions,
+  // which would be simplified.
+  // Since the same load will take different values on different iterations,
+  // we literally have to go through all loop's iterations.
+  for (unsigned Iteration = 0; Iteration < TripCount; ++Iteration) {
+    DEBUG(dbgs() << " Analyzing iteration " << Iteration << "\n");
+    SimplifiedValues.clear();
+    UnrolledInstAnalyzer Analyzer(Iteration, SimplifiedValues, L, SE);
+
+    BBWorklist.clear();
+    BBWorklist.insert(L->getHeader());
+    // Note that we *must not* cache the size, this loop grows the worklist.
+    for (unsigned Idx = 0; Idx != BBWorklist.size(); ++Idx) {
+      BasicBlock *BB = BBWorklist[Idx];
+
+      // Visit all instructions in the given basic block and try to simplify
+      // it.  We don't change the actual IR, just count optimization
+      // opportunities.
+      for (Instruction &I : *BB) {
+        unsigned InstCost = TTI.getUserCost(&I);
+
+        // Visit the instruction to analyze its loop cost after unrolling,
+        // and if the visitor returns false, include this instruction in the
+        // unrolled cost.
+        if (!Analyzer.visit(I))
+          UnrolledCost += InstCost;
+        else {
+          DEBUG(dbgs() << "  " << I
+                       << " would be simplified if loop is unrolled.\n");
+          (void)0;
+        }
+
+        // Also track this instructions expected cost when executing the rolled
+        // loop form.
+        RolledDynamicCost += InstCost;
+
+        // If unrolled body turns out to be too big, bail out.
+        if (UnrolledCost > MaxUnrolledLoopSize) {
+          DEBUG(dbgs() << "  Exceeded threshold.. exiting.\n"
+                       << "  UnrolledCost: " << UnrolledCost
+                       << ", MaxUnrolledLoopSize: " << MaxUnrolledLoopSize
+                       << "\n");
+          return None;
+        }
       }
 
-    // If a definition of an insn is only used by simplified or dead
-    // instructions, it's also dead. Check defs of all instructions from the
-    // worklist.
-    while (!Worklist.empty()) {
-      Instruction *I = Worklist.pop_back_val();
-      if (!L->contains(I))
-        continue;
-      if (DeadInstructions.count(I))
-        continue;
-
-      if (std::all_of(I->user_begin(), I->user_end(), [&](User *U) {
-            return DeadInstructions.count(cast<Instruction>(U));
-          })) {
-        NumberOfOptimizedInstructions += TTI.getUserCost(I);
-        DeadInstructions.insert(I);
-        EnqueueOperands(*I);
+      TerminatorInst *TI = BB->getTerminator();
+
+      // Add in the live successors by first checking whether we have terminator
+      // that may be simplified based on the values simplified by this call.
+      if (BranchInst *BI = dyn_cast<BranchInst>(TI)) {
+        if (BI->isConditional()) {
+          if (Constant *SimpleCond =
+                  SimplifiedValues.lookup(BI->getCondition())) {
+            BasicBlock *Succ = nullptr;
+            // Just take the first successor if condition is undef
+            if (isa<UndefValue>(SimpleCond))
+              Succ = BI->getSuccessor(0);
+            else
+              Succ = BI->getSuccessor(
+                  cast<ConstantInt>(SimpleCond)->isZero() ? 1 : 0);
+            if (L->contains(Succ))
+              BBWorklist.insert(Succ);
+            continue;
+          }
+        }
+      } else if (SwitchInst *SI = dyn_cast<SwitchInst>(TI)) {
+        if (Constant *SimpleCond =
+                SimplifiedValues.lookup(SI->getCondition())) {
+          BasicBlock *Succ = nullptr;
+          // Just take the first successor if condition is undef
+          if (isa<UndefValue>(SimpleCond))
+            Succ = SI->getSuccessor(0);
+          else
+            Succ = SI->findCaseValue(cast<ConstantInt>(SimpleCond))
+                       .getCaseSuccessor();
+          if (L->contains(Succ))
+            BBWorklist.insert(Succ);
+          continue;
+        }
       }
+
+      // Add BB's successors to the worklist.
+      for (BasicBlock *Succ : successors(BB))
+        if (L->contains(Succ))
+          BBWorklist.insert(Succ);
     }
-    return NumberOfOptimizedInstructions;
-  }
-};
-} // namespace
 
-// Complete loop unrolling can make some loads constant, and we need to know if
-// that would expose any further optimization opportunities.
-// This routine estimates this optimization effect and returns the number of
-// instructions, that potentially might be optimized away.
-static unsigned
-approximateNumberOfOptimizedInstructions(const Loop *L, ScalarEvolution &SE,
-                                         unsigned TripCount,
-                                         const TargetTransformInfo &TTI) {
-  if (!TripCount || !UnrollMaxIterationsCountToAnalyze)
-    return 0;
-
-  UnrollAnalyzer UA(L, TripCount, SE, TTI);
-  UA.findConstFoldableLoads();
-
-  // Estimate number of instructions, that could be simplified if we replace a
-  // load with the corresponding constant. Since the same load will take
-  // different values on different iterations, we have to go through all loop's
-  // iterations here. To limit ourselves here, we check only first N
-  // iterations, and then scale the found number, if necessary.
-  unsigned IterationsNumberForEstimate =
-      std::min<unsigned>(UnrollMaxIterationsCountToAnalyze, TripCount);
-  unsigned NumberOfOptimizedInstructions = 0;
-  for (unsigned i = 0; i < IterationsNumberForEstimate; ++i)
-    NumberOfOptimizedInstructions +=
-        UA.estimateNumberOfOptimizedInstructions(i);
-
-  NumberOfOptimizedInstructions *= TripCount / IterationsNumberForEstimate;
-
-  return NumberOfOptimizedInstructions;
+    // If we found no optimization opportunities on the first iteration, we
+    // won't find them on later ones too.
+    if (UnrolledCost == RolledDynamicCost) {
+      DEBUG(dbgs() << "  No opportunities found.. exiting.\n"
+                   << "  UnrolledCost: " << UnrolledCost << "\n");
+      return None;
+    }
+  }
+  DEBUG(dbgs() << "Analysis finished:\n"
+               << "UnrolledCost: " << UnrolledCost << ", "
+               << "RolledDynamicCost: " << RolledDynamicCost << "\n");
+  return {{UnrolledCost, RolledDynamicCost}};
 }
 
 /// ApproximateLoopSize - Approximate the size of the loop.
@@ -679,6 +750,59 @@ static void SetLoopAlreadyUnrolled(Loop *L) {
   L->setLoopID(NewLoopID);
 }
 
+bool LoopUnroll::canUnrollCompletely(Loop *L, unsigned Threshold,
+                                     unsigned PercentDynamicCostSavedThreshold,
+                                     unsigned DynamicCostSavingsDiscount,
+                                     uint64_t UnrolledCost,
+                                     uint64_t RolledDynamicCost) {
+
+  if (Threshold == NoThreshold) {
+    DEBUG(dbgs() << "  Can fully unroll, because no threshold is set.\n");
+    return true;
+  }
+
+  if (UnrolledCost <= Threshold) {
+    DEBUG(dbgs() << "  Can fully unroll, because unrolled cost: "
+                 << UnrolledCost << "<" << Threshold << "\n");
+    return true;
+  }
+
+  assert(UnrolledCost && "UnrolledCost can't be 0 at this point.");
+  assert(RolledDynamicCost >= UnrolledCost &&
+         "Cannot have a higher unrolled cost than a rolled cost!");
+
+  // Compute the percentage of the dynamic cost in the rolled form that is
+  // saved when unrolled. If unrolling dramatically reduces the estimated
+  // dynamic cost of the loop, we use a higher threshold to allow more
+  // unrolling.
+  unsigned PercentDynamicCostSaved =
+      (uint64_t)(RolledDynamicCost - UnrolledCost) * 100ull / RolledDynamicCost;
+
+  if (PercentDynamicCostSaved >= PercentDynamicCostSavedThreshold &&
+      (int64_t)UnrolledCost - (int64_t)DynamicCostSavingsDiscount <=
+          (int64_t)Threshold) {
+    DEBUG(dbgs() << "  Can fully unroll, because unrolling will reduce the "
+                    "expected dynamic cost by " << PercentDynamicCostSaved
+                 << "% (threshold: " << PercentDynamicCostSavedThreshold
+                 << "%)\n"
+                 << "  and the unrolled cost (" << UnrolledCost
+                 << ") is less than the max threshold ("
+                 << DynamicCostSavingsDiscount << ").\n");
+    return true;
+  }
+
+  DEBUG(dbgs() << "  Too large to fully unroll:\n");
+  DEBUG(dbgs() << "    Threshold: " << Threshold << "\n");
+  DEBUG(dbgs() << "    Max threshold: " << DynamicCostSavingsDiscount << "\n");
+  DEBUG(dbgs() << "    Percent cost saved threshold: "
+               << PercentDynamicCostSavedThreshold << "%\n");
+  DEBUG(dbgs() << "    Unrolled cost: " << UnrolledCost << "\n");
+  DEBUG(dbgs() << "    Rolled dynamic cost: " << RolledDynamicCost << "\n");
+  DEBUG(dbgs() << "    Percent cost saved: " << PercentDynamicCostSaved
+               << "\n");
+  return false;
+}
+
 unsigned LoopUnroll::selectUnrollCount(
     const Loop *L, unsigned TripCount, bool PragmaFullUnroll,
     unsigned PragmaCount, const TargetTransformInfo::UnrollingPreferences &UP,
@@ -785,27 +909,34 @@ bool LoopUnroll::runOnLoop(Loop *L, LPPassManager &LPM) {
     return false;
   }
 
-  unsigned NumberOfOptimizedInstructions =
-      approximateNumberOfOptimizedInstructions(L, *SE, TripCount, TTI);
-  DEBUG(dbgs() << "  Complete unrolling could save: "
-               << NumberOfOptimizedInstructions << "\n");
-
   unsigned Threshold, PartialThreshold;
+  unsigned PercentDynamicCostSavedThreshold;
+  unsigned DynamicCostSavingsDiscount;
   selectThresholds(L, HasPragma, UP, Threshold, PartialThreshold,
-                   NumberOfOptimizedInstructions);
+                   PercentDynamicCostSavedThreshold,
+                   DynamicCostSavingsDiscount);
 
   // Given Count, TripCount and thresholds determine the type of
   // unrolling which is to be performed.
   enum { Full = 0, Partial = 1, Runtime = 2 };
   int Unrolling;
   if (TripCount && Count == TripCount) {
-    if (Threshold != NoThreshold && UnrolledSize > Threshold) {
-      DEBUG(dbgs() << "  Too large to fully unroll with count: " << Count
-                   << " because size: " << UnrolledSize << ">" << Threshold
-                   << "\n");
-      Unrolling = Partial;
-    } else {
+    Unrolling = Partial;
+    // If the loop is really small, we don't need to run an expensive analysis.
+    if (canUnrollCompletely(L, Threshold, 100, DynamicCostSavingsDiscount,
+                            UnrolledSize, UnrolledSize)) {
       Unrolling = Full;
+    } else {
+      // The loop isn't that small, but we still can fully unroll it if that
+      // helps to remove a significant number of instructions.
+      // To check that, run additional analysis on the loop.
+      if (Optional<EstimatedUnrollCost> Cost = analyzeLoopUnrollCost(
+              L, TripCount, *SE, TTI, Threshold + DynamicCostSavingsDiscount))
+        if (canUnrollCompletely(L, Threshold, PercentDynamicCostSavedThreshold,
+                                DynamicCostSavingsDiscount, Cost->UnrolledCost,
+                                Cost->RolledDynamicCost)) {
+          Unrolling = Full;
+        }
     }
   } else if (TripCount && Count < TripCount) {
     Unrolling = Partial;
@@ -815,8 +946,10 @@ bool LoopUnroll::runOnLoop(Loop *L, LPPassManager &LPM) {
 
   // Reduce count based on the type of unrolling and the threshold values.
   unsigned OriginalCount = Count;
-  bool AllowRuntime = UserRuntime ? CurrentRuntime : UP.Runtime;
-  if (HasRuntimeUnrollDisablePragma(L)) {
+  bool AllowRuntime =
+      (PragmaCount > 0) || (UserRuntime ? CurrentRuntime : UP.Runtime);
+  // Don't unroll a runtime trip count loop with unroll full pragma.
+  if (HasRuntimeUnrollDisablePragma(L) || PragmaFullUnroll) {
     AllowRuntime = false;
   }
   if (Unrolling == Partial) {
diff --git a/lib/Transforms/Scalar/LoopUnswitch.cpp b/lib/Transforms/Scalar/LoopUnswitch.cpp
index 988d2af3ea9..e457db0e979 100644
--- a/lib/Transforms/Scalar/LoopUnswitch.cpp
+++ b/lib/Transforms/Scalar/LoopUnswitch.cpp
@@ -43,6 +43,7 @@
 #include "llvm/IR/Function.h"
 #include "llvm/IR/Instructions.h"
 #include "llvm/IR/Module.h"
+#include "llvm/IR/MDBuilder.h"
 #include "llvm/Support/CommandLine.h"
 #include "llvm/Support/Debug.h"
 #include "llvm/Support/raw_ostream.h"
@@ -80,6 +81,7 @@ namespace {
 
     struct LoopProperties {
       unsigned CanBeUnswitchedCount;
+      unsigned WasUnswitchedCount;
       unsigned SizeEstimation;
       UnswitchedValsMap UnswitchedVals;
     };
@@ -93,37 +95,52 @@ namespace {
     UnswitchedValsMap *CurLoopInstructions;
     LoopProperties *CurrentLoopProperties;
 
-    // Max size of code we can produce on remained iterations.
+    // A loop unswitching with an estimated cost above this threshold
+    // is not performed. MaxSize is turned into unswitching quota for
+    // the current loop, and reduced correspondingly, though note that
+    // the quota is returned by releaseMemory() when the loop has been
+    // processed, so that MaxSize will return to its previous
+    // value. So in most cases MaxSize will equal the Threshold flag
+    // when a new loop is processed. An exception to that is that
+    // MaxSize will have a smaller value while processing nested loops
+    // that were introduced due to loop unswitching of an outer loop.
+    //
+    // FIXME: The way that MaxSize works is subtle and depends on the
+    // pass manager processing loops and calling releaseMemory() in a
+    // specific order. It would be good to find a more straightforward
+    // way of doing what MaxSize does.
     unsigned MaxSize;
 
-    public:
-
-      LUAnalysisCache() :
-        CurLoopInstructions(nullptr), CurrentLoopProperties(nullptr),
-        MaxSize(Threshold)
-      {}
-
-      // Analyze loop. Check its size, calculate is it possible to unswitch
-      // it. Returns true if we can unswitch this loop.
-      bool countLoop(const Loop *L, const TargetTransformInfo &TTI,
-                     AssumptionCache *AC);
-
-      // Clean all data related to given loop.
-      void forgetLoop(const Loop *L);
-
-      // Mark case value as unswitched.
-      // Since SI instruction can be partly unswitched, in order to avoid
-      // extra unswitching in cloned loops keep track all unswitched values.
-      void setUnswitched(const SwitchInst *SI, const Value *V);
-
-      // Check was this case value unswitched before or not.
-      bool isUnswitched(const SwitchInst *SI, const Value *V);
-
-      // Clone all loop-unswitch related loop properties.
-      // Redistribute unswitching quotas.
-      // Note, that new loop data is stored inside the VMap.
-      void cloneData(const Loop *NewLoop, const Loop *OldLoop,
-                     const ValueToValueMapTy &VMap);
+  public:
+    LUAnalysisCache()
+        : CurLoopInstructions(nullptr), CurrentLoopProperties(nullptr),
+          MaxSize(Threshold) {}
+
+    // Analyze loop. Check its size, calculate is it possible to unswitch
+    // it. Returns true if we can unswitch this loop.
+    bool countLoop(const Loop *L, const TargetTransformInfo &TTI,
+                   AssumptionCache *AC);
+
+    // Clean all data related to given loop.
+    void forgetLoop(const Loop *L);
+
+    // Mark case value as unswitched.
+    // Since SI instruction can be partly unswitched, in order to avoid
+    // extra unswitching in cloned loops keep track all unswitched values.
+    void setUnswitched(const SwitchInst *SI, const Value *V);
+
+    // Check was this case value unswitched before or not.
+    bool isUnswitched(const SwitchInst *SI, const Value *V);
+
+    // Returns true if another unswitching could be done within the cost
+    // threshold.
+    bool CostAllowsUnswitching();
+
+    // Clone all loop-unswitch related loop properties.
+    // Redistribute unswitching quotas.
+    // Note, that new loop data is stored inside the VMap.
+    void cloneData(const Loop *NewLoop, const Loop *OldLoop,
+                   const ValueToValueMapTy &VMap);
   };
 
   class LoopUnswitch : public LoopPass {
@@ -195,10 +212,14 @@ namespace {
     /// Update the appropriate Phi nodes as we do so.
     void SplitExitEdges(Loop *L, const SmallVectorImpl<BasicBlock *> &ExitBlocks);
 
-    bool UnswitchIfProfitable(Value *LoopCond, Constant *Val);
+    bool TryTrivialLoopUnswitch(bool &Changed);
+
+    bool UnswitchIfProfitable(Value *LoopCond, Constant *Val,
+                              TerminatorInst *TI = nullptr);
     void UnswitchTrivialCondition(Loop *L, Value *Cond, Constant *Val,
-                                  BasicBlock *ExitBlock);
-    void UnswitchNontrivialCondition(Value *LIC, Constant *OnVal, Loop *L);
+                                  BasicBlock *ExitBlock, TerminatorInst *TI);
+    void UnswitchNontrivialCondition(Value *LIC, Constant *OnVal, Loop *L,
+                                     TerminatorInst *TI);
 
     void RewriteLoopBodyWithConditionConstant(Loop *L, Value *LIC,
                                               Constant *Val, bool isEqual);
@@ -206,12 +227,10 @@ namespace {
     void EmitPreheaderBranchOnCondition(Value *LIC, Constant *Val,
                                         BasicBlock *TrueDest,
                                         BasicBlock *FalseDest,
-                                        Instruction *InsertPt);
+                                        Instruction *InsertPt,
+                                        TerminatorInst *TI);
 
     void SimplifyCode(std::vector<Instruction*> &Worklist, Loop *L);
-    bool IsTrivialUnswitchCondition(Value *Cond, Constant **Val = nullptr,
-                                    BasicBlock **LoopExit = nullptr);
-
   };
 }
 
@@ -242,12 +261,13 @@ bool LUAnalysisCache::countLoop(const Loop *L, const TargetTransformInfo &TTI,
     // consideration code simplification opportunities and code that can
     // be shared by the resultant unswitched loops.
     CodeMetrics Metrics;
-    for (Loop::block_iterator I = L->block_begin(), E = L->block_end();
-         I != E; ++I)
+    for (Loop::block_iterator I = L->block_begin(), E = L->block_end(); I != E;
+         ++I)
       Metrics.analyzeBasicBlock(*I, TTI, EphValues);
 
-    Props.SizeEstimation = std::min(Metrics.NumInsts, Metrics.NumBlocks * 5);
+    Props.SizeEstimation = Metrics.NumInsts;
     Props.CanBeUnswitchedCount = MaxSize / (Props.SizeEstimation);
+    Props.WasUnswitchedCount = 0;
     MaxSize -= Props.SizeEstimation * Props.CanBeUnswitchedCount;
 
     if (Metrics.notDuplicatable) {
@@ -258,13 +278,6 @@ bool LUAnalysisCache::countLoop(const Loop *L, const TargetTransformInfo &TTI,
     }
   }
 
-  if (!Props.CanBeUnswitchedCount) {
-    DEBUG(dbgs() << "NOT unswitching loop %"
-                 << L->getHeader()->getName() << ", cost too high: "
-                 << L->getBlocks().size() << "\n");
-    return false;
-  }
-
   // Be careful. This links are good only before new loop addition.
   CurrentLoopProperties = &Props;
   CurLoopInstructions = &Props.UnswitchedVals;
@@ -279,7 +292,8 @@ void LUAnalysisCache::forgetLoop(const Loop *L) {
 
   if (LIt != LoopsProperties.end()) {
     LoopProperties &Props = LIt->second;
-    MaxSize += Props.CanBeUnswitchedCount * Props.SizeEstimation;
+    MaxSize += (Props.CanBeUnswitchedCount + Props.WasUnswitchedCount) *
+               Props.SizeEstimation;
     LoopsProperties.erase(LIt);
   }
 
@@ -299,6 +313,10 @@ bool LUAnalysisCache::isUnswitched(const SwitchInst *SI, const Value *V) {
   return (*CurLoopInstructions)[SI].count(V);
 }
 
+bool LUAnalysisCache::CostAllowsUnswitching() {
+  return CurrentLoopProperties->CanBeUnswitchedCount > 0;
+}
+
 // Clone all loop-unswitch related loop properties.
 // Redistribute unswitching quotas.
 // Note, that new loop data is stored inside the VMap.
@@ -312,6 +330,8 @@ void LUAnalysisCache::cloneData(const Loop *NewLoop, const Loop *OldLoop,
   // Reallocate "can-be-unswitched quota"
 
   --OldLoopProps.CanBeUnswitchedCount;
+  ++OldLoopProps.WasUnswitchedCount;
+  NewLoopProps.WasUnswitchedCount = 0;
   unsigned Quota = OldLoopProps.CanBeUnswitchedCount;
   NewLoopProps.CanBeUnswitchedCount = Quota / 2;
   OldLoopProps.CanBeUnswitchedCount = Quota - Quota / 2;
@@ -439,6 +459,11 @@ bool LoopUnswitch::processCurrentLoop() {
           AC))
     return false;
 
+  // Try trivial unswitch first before loop over other basic blocks in the loop.
+  if (TryTrivialLoopUnswitch(Changed)) {
+    return true;
+  }
+
   // Loop over all of the basic blocks in the loop.  If we find an interior
   // block that is branching on a loop-invariant condition, we can unswitch this
   // loop.
@@ -453,8 +478,8 @@ bool LoopUnswitch::processCurrentLoop() {
         // unswitch on it if we desire.
         Value *LoopCond = FindLIVLoopCondition(BI->getCondition(),
                                                currentLoop, Changed);
-        if (LoopCond && UnswitchIfProfitable(LoopCond,
-                                             ConstantInt::getTrue(Context))) {
+        if (LoopCond &&
+            UnswitchIfProfitable(LoopCond, ConstantInt::getTrue(Context), TI)) {
           ++NumBranches;
           return true;
         }
@@ -557,111 +582,28 @@ static BasicBlock *isTrivialLoopExitBlock(Loop *L, BasicBlock *BB) {
   return nullptr;
 }
 
-/// IsTrivialUnswitchCondition - Check to see if this unswitch condition is
-/// trivial: that is, that the condition controls whether or not the loop does
-/// anything at all.  If this is a trivial condition, unswitching produces no
-/// code duplications (equivalently, it produces a simpler loop and a new empty
-/// loop, which gets deleted).
-///
-/// If this is a trivial condition, return true, otherwise return false.  When
-/// returning true, this sets Cond and Val to the condition that controls the
-/// trivial condition: when Cond dynamically equals Val, the loop is known to
-/// exit.  Finally, this sets LoopExit to the BB that the loop exits to when
-/// Cond == Val.
-///
-bool LoopUnswitch::IsTrivialUnswitchCondition(Value *Cond, Constant **Val,
-                                       BasicBlock **LoopExit) {
-  BasicBlock *Header = currentLoop->getHeader();
-  TerminatorInst *HeaderTerm = Header->getTerminator();
-  LLVMContext &Context = Header->getContext();
-
-  BasicBlock *LoopExitBB = nullptr;
-  if (BranchInst *BI = dyn_cast<BranchInst>(HeaderTerm)) {
-    // If the header block doesn't end with a conditional branch on Cond, we
-    // can't handle it.
-    if (!BI->isConditional() || BI->getCondition() != Cond)
-      return false;
-
-    // Check to see if a successor of the branch is guaranteed to
-    // exit through a unique exit block without having any
-    // side-effects.  If so, determine the value of Cond that causes it to do
-    // this.
-    if ((LoopExitBB = isTrivialLoopExitBlock(currentLoop,
-                                             BI->getSuccessor(0)))) {
-      if (Val) *Val = ConstantInt::getTrue(Context);
-    } else if ((LoopExitBB = isTrivialLoopExitBlock(currentLoop,
-                                                    BI->getSuccessor(1)))) {
-      if (Val) *Val = ConstantInt::getFalse(Context);
-    }
-  } else if (SwitchInst *SI = dyn_cast<SwitchInst>(HeaderTerm)) {
-    // If this isn't a switch on Cond, we can't handle it.
-    if (SI->getCondition() != Cond) return false;
-
-    // Check to see if a successor of the switch is guaranteed to go to the
-    // latch block or exit through a one exit block without having any
-    // side-effects.  If so, determine the value of Cond that causes it to do
-    // this.
-    // Note that we can't trivially unswitch on the default case or
-    // on already unswitched cases.
-    for (SwitchInst::CaseIt i = SI->case_begin(), e = SI->case_end();
-         i != e; ++i) {
-      BasicBlock *LoopExitCandidate;
-      if ((LoopExitCandidate = isTrivialLoopExitBlock(currentLoop,
-                                               i.getCaseSuccessor()))) {
-        // Okay, we found a trivial case, remember the value that is trivial.
-        ConstantInt *CaseVal = i.getCaseValue();
-
-        // Check that it was not unswitched before, since already unswitched
-        // trivial vals are looks trivial too.
-        if (BranchesInfo.isUnswitched(SI, CaseVal))
-          continue;
-        LoopExitBB = LoopExitCandidate;
-        if (Val) *Val = CaseVal;
-        break;
-      }
-    }
-  }
-
-  // If we didn't find a single unique LoopExit block, or if the loop exit block
-  // contains phi nodes, this isn't trivial.
-  if (!LoopExitBB || isa<PHINode>(LoopExitBB->begin()))
-    return false;   // Can't handle this.
-
-  if (LoopExit) *LoopExit = LoopExitBB;
-
-  // We already know that nothing uses any scalar values defined inside of this
-  // loop.  As such, we just have to check to see if this loop will execute any
-  // side-effecting instructions (e.g. stores, calls, volatile loads) in the
-  // part of the loop that the code *would* execute.  We already checked the
-  // tail, check the header now.
-  for (BasicBlock::iterator I = Header->begin(), E = Header->end(); I != E; ++I)
-    if (I->mayHaveSideEffects())
-      return false;
-  return true;
-}
-
 /// UnswitchIfProfitable - We have found that we can unswitch currentLoop when
 /// LoopCond == Val to simplify the loop.  If we decide that this is profitable,
 /// unswitch the loop, reprocess the pieces, then return true.
-bool LoopUnswitch::UnswitchIfProfitable(Value *LoopCond, Constant *Val) {
+bool LoopUnswitch::UnswitchIfProfitable(Value *LoopCond, Constant *Val,
+                                        TerminatorInst *TI) {
   Function *F = loopHeader->getParent();
-  Constant *CondVal = nullptr;
-  BasicBlock *ExitBlock = nullptr;
-
-  if (IsTrivialUnswitchCondition(LoopCond, &CondVal, &ExitBlock)) {
-    // If the condition is trivial, always unswitch. There is no code growth
-    // for this case.
-    UnswitchTrivialCondition(currentLoop, LoopCond, CondVal, ExitBlock);
-    return true;
-  }
 
   // Check to see if it would be profitable to unswitch current loop.
+  if (!BranchesInfo.CostAllowsUnswitching()) {
+    DEBUG(dbgs() << "NOT unswitching loop %"
+                 << currentLoop->getHeader()->getName()
+                 << " at non-trivial condition '" << *Val
+                 << "' == " << *LoopCond << "\n"
+                 << ". Cost too high.\n");
+    return false;
+  }
 
   // Do not do non-trivial unswitch while optimizing for size.
   if (OptimizeForSize || F->hasFnAttribute(Attribute::OptimizeForSize))
     return false;
 
-  UnswitchNontrivialCondition(LoopCond, Val, currentLoop);
+  UnswitchNontrivialCondition(LoopCond, Val, currentLoop, TI);
   return true;
 }
 
@@ -685,25 +627,65 @@ static Loop *CloneLoop(Loop *L, Loop *PL, ValueToValueMapTy &VM,
   return New;
 }
 
+static void copyMetadata(Instruction *DstInst, const Instruction *SrcInst,
+                         bool Swapped) {
+  if (!SrcInst || !SrcInst->hasMetadata())
+    return;
+
+  SmallVector<std::pair<unsigned, MDNode *>, 4> MDs;
+  SrcInst->getAllMetadata(MDs);
+  for (auto &MD : MDs) {
+    switch (MD.first) {
+    default:
+      break;
+    case LLVMContext::MD_prof:
+      if (Swapped && MD.second->getNumOperands() == 3 &&
+          isa<MDString>(MD.second->getOperand(0))) {
+        MDString *MDName = cast<MDString>(MD.second->getOperand(0));
+        if (MDName->getString() == "branch_weights") {
+          auto *ValT = cast_or_null<ConstantAsMetadata>(
+                           MD.second->getOperand(1))->getValue();
+          auto *ValF = cast_or_null<ConstantAsMetadata>(
+                           MD.second->getOperand(2))->getValue();
+          assert(ValT && ValF && "Invalid Operands of branch_weights");
+          auto NewMD =
+              MDBuilder(DstInst->getParent()->getContext())
+                  .createBranchWeights(cast<ConstantInt>(ValF)->getZExtValue(),
+                                       cast<ConstantInt>(ValT)->getZExtValue());
+          MD.second = NewMD;
+        }
+      }
+      // fallthrough.
+    case LLVMContext::MD_dbg:
+      DstInst->setMetadata(MD.first, MD.second);
+    }
+  }
+}
+
 /// EmitPreheaderBranchOnCondition - Emit a conditional branch on two values
 /// if LIC == Val, branch to TrueDst, otherwise branch to FalseDest.  Insert the
 /// code immediately before InsertPt.
 void LoopUnswitch::EmitPreheaderBranchOnCondition(Value *LIC, Constant *Val,
                                                   BasicBlock *TrueDest,
                                                   BasicBlock *FalseDest,
-                                                  Instruction *InsertPt) {
+                                                  Instruction *InsertPt,
+                                                  TerminatorInst *TI) {
   // Insert a conditional branch on LIC to the two preheaders.  The original
   // code is the true version and the new code is the false version.
   Value *BranchVal = LIC;
+  bool Swapped = false;
   if (!isa<ConstantInt>(Val) ||
       Val->getType() != Type::getInt1Ty(LIC->getContext()))
     BranchVal = new ICmpInst(InsertPt, ICmpInst::ICMP_EQ, LIC, Val);
-  else if (Val != ConstantInt::getTrue(Val->getContext()))
+  else if (Val != ConstantInt::getTrue(Val->getContext())) {
     // We want to enter the new loop when the condition is true.
     std::swap(TrueDest, FalseDest);
+    Swapped = true;
+  }
 
   // Insert the new branch.
   BranchInst *BI = BranchInst::Create(TrueDest, FalseDest, BranchVal, InsertPt);
+  copyMetadata(BI, TI, Swapped);
 
   // If either edge is critical, split it. This helps preserve LoopSimplify
   // form for enclosing loops.
@@ -717,13 +699,14 @@ void LoopUnswitch::EmitPreheaderBranchOnCondition(Value *LIC, Constant *Val,
 /// where the path through the loop that doesn't execute its body has no
 /// side-effects), unswitch it.  This doesn't involve any code duplication, just
 /// moving the conditional branch outside of the loop and updating loop info.
-void LoopUnswitch::UnswitchTrivialCondition(Loop *L, Value *Cond,
-                                            Constant *Val,
-                                            BasicBlock *ExitBlock) {
+void LoopUnswitch::UnswitchTrivialCondition(Loop *L, Value *Cond, Constant *Val,
+                                            BasicBlock *ExitBlock,
+                                            TerminatorInst *TI) {
   DEBUG(dbgs() << "loop-unswitch: Trivial-Unswitch loop %"
-        << loopHeader->getName() << " [" << L->getBlocks().size()
-        << " blocks] in Function " << L->getHeader()->getParent()->getName()
-        << " on cond: " << *Val << " == " << *Cond << "\n");
+               << loopHeader->getName() << " [" << L->getBlocks().size()
+               << " blocks] in Function "
+               << L->getHeader()->getParent()->getName() << " on cond: " << *Val
+               << " == " << *Cond << "\n");
 
   // First step, split the preheader, so that we know that there is a safe place
   // to insert the conditional branch.  We will change loopPreheader to have a
@@ -744,7 +727,7 @@ void LoopUnswitch::UnswitchTrivialCondition(Loop *L, Value *Cond,
   // Okay, now we have a position to branch from and a position to branch to,
   // insert the new conditional branch.
   EmitPreheaderBranchOnCondition(Cond, Val, NewExit, NewPH,
-                                 loopPreheader->getTerminator());
+                                 loopPreheader->getTerminator(), TI);
   LPM->deleteSimpleAnalysisValue(loopPreheader->getTerminator(), L);
   loopPreheader->getTerminator()->eraseFromParent();
 
@@ -758,6 +741,151 @@ void LoopUnswitch::UnswitchTrivialCondition(Loop *L, Value *Cond,
   ++NumTrivial;
 }
 
+/// TryTrivialLoopUnswitch - Check if the first non-constant condition starting from the
+/// loop header is a trivial unswitch condition: that is, a condition controls whether
+/// or not the loop does anything at all. If it is a trivial condition, unswitching
+/// produces no code duplications (equivalently, it produces a simpler loop and a new
+/// empty loop, which gets deleted). Therefore always unswitch trivial condition.
+///
+bool LoopUnswitch::TryTrivialLoopUnswitch(bool &Changed) {
+  BasicBlock *CurrentBB = currentLoop->getHeader();
+  TerminatorInst *CurrentTerm = CurrentBB->getTerminator();
+  LLVMContext &Context = CurrentBB->getContext();
+
+  // If loop header has only one reachable successor (currently via an
+  // unconditional branch or constant foldable conditional branch, but
+  // should also consider adding constant foldable switch instruction in
+  // future), we should keep looking for trivial condition candidates in
+  // the successor as well. An alternative is to constant fold conditions
+  // and merge successors into loop header (then we only need to check header's
+  // terminator). The reason for not doing this in LoopUnswitch pass is that
+  // it could potentially break LoopPassManager's invariants. Folding dead
+  // branches could either eliminate the current loop or make other loops
+  // unreachable. LCSSA form might also not be preserved after deleting branches.
+  // The following code keeps traversing loop header's successors until it finds
+  // the trivial condition candidate (condition that is not a constant).
+  // Since unswitching generates branches with constant conditions, this
+  // scenario could be very common in practice.
+  SmallSet<BasicBlock*, 8> Visited;
+
+  while (true) {
+    // If we exit loop or reach a previous visited block, then
+    // we can not reach any trivial condition candidates (unfoldable
+    // branch instructions or switch instructions) and no unswitch
+    // can happen. Exit and return false.
+    if (!currentLoop->contains(CurrentBB) || !Visited.insert(CurrentBB).second)
+      return false;
+
+    // Check if this loop will execute any side-effecting instructions (e.g.
+    // stores, calls, volatile loads) in the part of the loop that the code
+    // *would* execute. Check the header first.
+    for (BasicBlock::iterator I : *CurrentBB)
+      if (I->mayHaveSideEffects())
+        return false;
+
+    // FIXME: add check for constant foldable switch instructions.
+    if (BranchInst *BI = dyn_cast<BranchInst>(CurrentTerm)) {
+      if (BI->isUnconditional()) {
+        CurrentBB = BI->getSuccessor(0);
+      } else if (BI->getCondition() == ConstantInt::getTrue(Context)) {
+        CurrentBB = BI->getSuccessor(0);
+      } else if (BI->getCondition() == ConstantInt::getFalse(Context)) {
+        CurrentBB = BI->getSuccessor(1);
+      } else {
+        // Found a trivial condition candidate (non-foldable conditional branch).
+        break;
+      }
+    } else {
+      break;
+    }
+
+    CurrentTerm = CurrentBB->getTerminator();
+  }
+
+  // CondVal is the condition that controls the trivial condition.
+  // LoopExitBB is the BasicBlock that loop exits when meets trivial condition.
+  Constant *CondVal = nullptr;
+  BasicBlock *LoopExitBB = nullptr;
+
+  if (BranchInst *BI = dyn_cast<BranchInst>(CurrentTerm)) {
+    // If this isn't branching on an invariant condition, we can't unswitch it.
+    if (!BI->isConditional())
+      return false;
+
+    Value *LoopCond = FindLIVLoopCondition(BI->getCondition(),
+                                           currentLoop, Changed);
+
+    // Unswitch only if the trivial condition itself is an LIV (not
+    // partial LIV which could occur in and/or)
+    if (!LoopCond || LoopCond != BI->getCondition())
+      return false;
+
+    // Check to see if a successor of the branch is guaranteed to
+    // exit through a unique exit block without having any
+    // side-effects.  If so, determine the value of Cond that causes
+    // it to do this.
+    if ((LoopExitBB = isTrivialLoopExitBlock(currentLoop,
+                                             BI->getSuccessor(0)))) {
+      CondVal = ConstantInt::getTrue(Context);
+    } else if ((LoopExitBB = isTrivialLoopExitBlock(currentLoop,
+                                                    BI->getSuccessor(1)))) {
+      CondVal = ConstantInt::getFalse(Context);
+    }
+
+    // If we didn't find a single unique LoopExit block, or if the loop exit block
+    // contains phi nodes, this isn't trivial.
+    if (!LoopExitBB || isa<PHINode>(LoopExitBB->begin()))
+      return false;   // Can't handle this.
+
+    UnswitchTrivialCondition(currentLoop, LoopCond, CondVal, LoopExitBB, CurrentTerm);
+    ++NumBranches;
+    return true;
+  } else if (SwitchInst *SI = dyn_cast<SwitchInst>(CurrentTerm)) {
+    // If this isn't switching on an invariant condition, we can't unswitch it.
+    Value *LoopCond = FindLIVLoopCondition(SI->getCondition(),
+                                           currentLoop, Changed);
+
+    // Unswitch only if the trivial condition itself is an LIV (not
+    // partial LIV which could occur in and/or)
+    if (!LoopCond || LoopCond != SI->getCondition())
+      return false;
+
+    // Check to see if a successor of the switch is guaranteed to go to the
+    // latch block or exit through a one exit block without having any
+    // side-effects.  If so, determine the value of Cond that causes it to do
+    // this.
+    // Note that we can't trivially unswitch on the default case or
+    // on already unswitched cases.
+    for (SwitchInst::CaseIt i = SI->case_begin(), e = SI->case_end();
+         i != e; ++i) {
+      BasicBlock *LoopExitCandidate;
+      if ((LoopExitCandidate = isTrivialLoopExitBlock(currentLoop,
+                                               i.getCaseSuccessor()))) {
+        // Okay, we found a trivial case, remember the value that is trivial.
+        ConstantInt *CaseVal = i.getCaseValue();
+
+        // Check that it was not unswitched before, since already unswitched
+        // trivial vals are looks trivial too.
+        if (BranchesInfo.isUnswitched(SI, CaseVal))
+          continue;
+        LoopExitBB = LoopExitCandidate;
+        CondVal = CaseVal;
+        break;
+      }
+    }
+
+    // If we didn't find a single unique LoopExit block, or if the loop exit block
+    // contains phi nodes, this isn't trivial.
+    if (!LoopExitBB || isa<PHINode>(LoopExitBB->begin()))
+      return false;   // Can't handle this.
+
+    UnswitchTrivialCondition(currentLoop, LoopCond, CondVal, LoopExitBB, nullptr);
+    ++NumSwitches;
+    return true;
+  }
+  return false;
+}
+
 /// SplitExitEdges - Split all of the edges from inside the loop to their exit
 /// blocks.  Update the appropriate Phi nodes as we do so.
 void LoopUnswitch::SplitExitEdges(Loop *L,
@@ -770,8 +898,7 @@ void LoopUnswitch::SplitExitEdges(Loop *L,
 
     // Although SplitBlockPredecessors doesn't preserve loop-simplify in
     // general, if we call it on all predecessors of all exits then it does.
-    SplitBlockPredecessors(ExitBlock, Preds, ".us-lcssa",
-                           /*AliasAnalysis*/ nullptr, DT, LI,
+    SplitBlockPredecessors(ExitBlock, Preds, ".us-lcssa", DT, LI,
                            /*PreserveLCSSA*/ true);
   }
 }
@@ -780,7 +907,7 @@ void LoopUnswitch::SplitExitEdges(Loop *L,
 /// to unswitch when LIC equal Val.  Split it into loop versions and test the
 /// condition outside of either loop.  Return the loops created as Out1/Out2.
 void LoopUnswitch::UnswitchNontrivialCondition(Value *LIC, Constant *Val,
-                                               Loop *L) {
+                                               Loop *L, TerminatorInst *TI) {
   Function *F = loopHeader->getParent();
   DEBUG(dbgs() << "loop-unswitch: Unswitching loop %"
         << loopHeader->getName() << " [" << L->getBlocks().size()
@@ -897,7 +1024,8 @@ void LoopUnswitch::UnswitchNontrivialCondition(Value *LIC, Constant *Val,
          "Preheader splitting did not work correctly!");
 
   // Emit the new branch that selects between the two versions of this loop.
-  EmitPreheaderBranchOnCondition(LIC, Val, NewBlocks[0], LoopBlocks[0], OldBR);
+  EmitPreheaderBranchOnCondition(LIC, Val, NewBlocks[0], LoopBlocks[0], OldBR,
+                                 TI);
   LPM->deleteSimpleAnalysisValue(OldBR, L);
   OldBR->eraseFromParent();
 
diff --git a/lib/Transforms/Scalar/MemCpyOptimizer.cpp b/lib/Transforms/Scalar/MemCpyOptimizer.cpp
index d6514735c87..46105dab1d8 100644
--- a/lib/Transforms/Scalar/MemCpyOptimizer.cpp
+++ b/lib/Transforms/Scalar/MemCpyOptimizer.cpp
@@ -30,7 +30,7 @@
 #include "llvm/Support/Debug.h"
 #include "llvm/Support/raw_ostream.h"
 #include "llvm/Transforms/Utils/Local.h"
-#include <list>
+#include <algorithm>
 using namespace llvm;
 
 #define DEBUG_TYPE "memcpyopt"
@@ -200,15 +200,14 @@ bool MemsetRange::isProfitableToUseMemset(const DataLayout &DL) const {
 
 namespace {
 class MemsetRanges {
-  /// Ranges - A sorted list of the memset ranges.  We use std::list here
-  /// because each element is relatively large and expensive to copy.
-  std::list<MemsetRange> Ranges;
-  typedef std::list<MemsetRange>::iterator range_iterator;
+  /// Ranges - A sorted list of the memset ranges.
+  SmallVector<MemsetRange, 8> Ranges;
+  typedef SmallVectorImpl<MemsetRange>::iterator range_iterator;
   const DataLayout &DL;
 public:
   MemsetRanges(const DataLayout &DL) : DL(DL) {}
 
-  typedef std::list<MemsetRange>::const_iterator const_iterator;
+  typedef SmallVectorImpl<MemsetRange>::const_iterator const_iterator;
   const_iterator begin() const { return Ranges.begin(); }
   const_iterator end() const { return Ranges.end(); }
   bool empty() const { return Ranges.empty(); }
@@ -243,23 +242,17 @@ class MemsetRanges {
 /// addRange - Add a new store to the MemsetRanges data structure.  This adds a
 /// new range for the specified store at the specified offset, merging into
 /// existing ranges as appropriate.
-///
-/// Do a linear search of the ranges to see if this can be joined and/or to
-/// find the insertion point in the list.  We keep the ranges sorted for
-/// simplicity here.  This is a linear search of a linked list, which is ugly,
-/// however the number of ranges is limited, so this won't get crazy slow.
 void MemsetRanges::addRange(int64_t Start, int64_t Size, Value *Ptr,
                             unsigned Alignment, Instruction *Inst) {
   int64_t End = Start+Size;
-  range_iterator I = Ranges.begin(), E = Ranges.end();
 
-  while (I != E && Start > I->End)
-    ++I;
+  range_iterator I = std::lower_bound(Ranges.begin(), Ranges.end(), Start,
+    [](const MemsetRange &LHS, int64_t RHS) { return LHS.End < RHS; });
 
   // We now know that I == E, in which case we didn't find anything to merge
   // with, or that Start <= I->End.  If End < I->Start or I == E, then we need
   // to insert a new range.  Handle this now.
-  if (I == E || End < I->Start) {
+  if (I == Ranges.end() || End < I->Start) {
     MemsetRange &R = *Ranges.insert(I, MemsetRange());
     R.Start        = Start;
     R.End          = End;
@@ -295,7 +288,7 @@ void MemsetRanges::addRange(int64_t Start, int64_t Size, Value *Ptr,
   if (End > I->End) {
     I->End = End;
     range_iterator NextI = I;
-    while (++NextI != E && End >= NextI->Start) {
+    while (++NextI != Ranges.end() && End >= NextI->Start) {
       // Merge the range in.
       I->TheStores.append(NextI->TheStores.begin(), NextI->TheStores.end());
       if (NextI->End > I->End)
@@ -337,15 +330,16 @@ namespace {
       AU.addPreserved<MemoryDependenceAnalysis>();
     }
 
-    // Helper fuctions
+    // Helper functions
     bool processStore(StoreInst *SI, BasicBlock::iterator &BBI);
     bool processMemSet(MemSetInst *SI, BasicBlock::iterator &BBI);
     bool processMemCpy(MemCpyInst *M);
     bool processMemMove(MemMoveInst *M);
     bool performCallSlotOptzn(Instruction *cpy, Value *cpyDst, Value *cpySrc,
                               uint64_t cpyLen, unsigned cpyAlign, CallInst *C);
-    bool processMemCpyMemCpyDependence(MemCpyInst *M, MemCpyInst *MDep,
-                                       uint64_t MSize);
+    bool processMemCpyMemCpyDependence(MemCpyInst *M, MemCpyInst *MDep);
+    bool processMemSetMemCpyDependence(MemCpyInst *M, MemSetInst *MDep);
+    bool performMemCpyToMemSetOptzn(MemCpyInst *M, MemSetInst *MDep);
     bool processByValArgument(CallSite CS, unsigned ArgNo);
     Instruction *tryMergingIntoMemset(Instruction *I, Value *StartPtr,
                                       Value *ByteVal);
@@ -509,10 +503,10 @@ bool MemCpyOpt::processStore(StoreInst *SI, BasicBlock::iterator &BBI) {
         // Check that nothing touches the dest of the "copy" between
         // the call and the store.
         AliasAnalysis &AA = getAnalysis<AliasAnalysis>();
-        AliasAnalysis::Location StoreLoc = AA.getLocation(SI);
+        MemoryLocation StoreLoc = MemoryLocation::get(SI);
         for (BasicBlock::iterator I = --BasicBlock::iterator(SI),
                                   E = C; I != E; --I) {
-          if (AA.getModRefInfo(&*I, StoreLoc) != AliasAnalysis::NoModRef) {
+          if (AA.getModRefInfo(&*I, StoreLoc) != MRI_NoModRef) {
             C = nullptr;
             break;
           }
@@ -710,11 +704,11 @@ bool MemCpyOpt::performCallSlotOptzn(Instruction *cpy,
   // the use analysis, we also need to know that it does not sneakily
   // access dest.  We rely on AA to figure this out for us.
   AliasAnalysis &AA = getAnalysis<AliasAnalysis>();
-  AliasAnalysis::ModRefResult MR = AA.getModRefInfo(C, cpyDest, srcSize);
+  ModRefInfo MR = AA.getModRefInfo(C, cpyDest, srcSize);
   // If necessary, perform additional analysis.
-  if (MR != AliasAnalysis::NoModRef)
+  if (MR != MRI_NoModRef)
     MR = AA.callCapturesBefore(C, cpyDest, srcSize, &DT);
-  if (MR != AliasAnalysis::NoModRef)
+  if (MR != MRI_NoModRef)
     return false;
 
   // All the checks have passed, so do the transformation.
@@ -764,10 +758,9 @@ bool MemCpyOpt::performCallSlotOptzn(Instruction *cpy,
 
 /// processMemCpyMemCpyDependence - We've found that the (upward scanning)
 /// memory dependence of memcpy 'M' is the memcpy 'MDep'.  Try to simplify M to
-/// copy from MDep's input if we can.  MSize is the size of M's copy.
+/// copy from MDep's input if we can.
 ///
-bool MemCpyOpt::processMemCpyMemCpyDependence(MemCpyInst *M, MemCpyInst *MDep,
-                                              uint64_t MSize) {
+bool MemCpyOpt::processMemCpyMemCpyDependence(MemCpyInst *M, MemCpyInst *MDep) {
   // We can only transforms memcpy's where the dest of one is the source of the
   // other.
   if (M->getSource() != MDep->getDest() || MDep->isVolatile())
@@ -802,9 +795,8 @@ bool MemCpyOpt::processMemCpyMemCpyDependence(MemCpyInst *M, MemCpyInst *MDep,
   //
   // NOTE: This is conservative, it will stop on any read from the source loc,
   // not just the defining memcpy.
-  MemDepResult SourceDep =
-    MD->getPointerDependencyFrom(AA.getLocationForSource(MDep),
-                                 false, M, M->getParent());
+  MemDepResult SourceDep = MD->getPointerDependencyFrom(
+      MemoryLocation::getForSource(MDep), false, M, M->getParent());
   if (!SourceDep.isClobber() || SourceDep.getInst() != MDep)
     return false;
 
@@ -812,7 +804,8 @@ bool MemCpyOpt::processMemCpyMemCpyDependence(MemCpyInst *M, MemCpyInst *MDep,
   // source and dest might overlap.  We still want to eliminate the intermediate
   // value, but we have to generate a memmove instead of memcpy.
   bool UseMemMove = false;
-  if (!AA.isNoAlias(AA.getLocationForDest(M), AA.getLocationForSource(MDep)))
+  if (!AA.isNoAlias(MemoryLocation::getForDest(M),
+                    MemoryLocation::getForSource(MDep)))
     UseMemMove = true;
 
   // If all checks passed, then we can transform M.
@@ -839,6 +832,102 @@ bool MemCpyOpt::processMemCpyMemCpyDependence(MemCpyInst *M, MemCpyInst *MDep,
   return true;
 }
 
+/// We've found that the (upward scanning) memory dependence of \p MemCpy is
+/// \p MemSet.  Try to simplify \p MemSet to only set the trailing bytes that
+/// weren't copied over by \p MemCpy.
+///
+/// In other words, transform:
+/// \code
+///   memset(dst, c, dst_size);
+///   memcpy(dst, src, src_size);
+/// \endcode
+/// into:
+/// \code
+///   memcpy(dst, src, src_size);
+///   memset(dst + src_size, c, dst_size <= src_size ? 0 : dst_size - src_size);
+/// \endcode
+bool MemCpyOpt::processMemSetMemCpyDependence(MemCpyInst *MemCpy,
+                                              MemSetInst *MemSet) {
+  // We can only transform memset/memcpy with the same destination.
+  if (MemSet->getDest() != MemCpy->getDest())
+    return false;
+
+  // Check that there are no other dependencies on the memset destination.
+  MemDepResult DstDepInfo = MD->getPointerDependencyFrom(
+      MemoryLocation::getForDest(MemSet), false, MemCpy, MemCpy->getParent());
+  if (DstDepInfo.getInst() != MemSet)
+    return false;
+
+  // Use the same i8* dest as the memcpy, killing the memset dest if different.
+  Value *Dest = MemCpy->getRawDest();
+  Value *DestSize = MemSet->getLength();
+  Value *SrcSize = MemCpy->getLength();
+
+  // By default, create an unaligned memset.
+  unsigned Align = 1;
+  // If Dest is aligned, and SrcSize is constant, use the minimum alignment
+  // of the sum.
+  const unsigned DestAlign =
+      std::max(MemSet->getAlignment(), MemCpy->getAlignment());
+  if (DestAlign > 1)
+    if (ConstantInt *SrcSizeC = dyn_cast<ConstantInt>(SrcSize))
+      Align = MinAlign(SrcSizeC->getZExtValue(), DestAlign);
+
+  IRBuilder<> Builder(MemCpy);
+
+  // If the sizes have different types, zext the smaller one.
+  if (DestSize->getType() != SrcSize->getType()) {
+    if (DestSize->getType()->getIntegerBitWidth() >
+        SrcSize->getType()->getIntegerBitWidth())
+      SrcSize = Builder.CreateZExt(SrcSize, DestSize->getType());
+    else
+      DestSize = Builder.CreateZExt(DestSize, SrcSize->getType());
+  }
+
+  Value *MemsetLen =
+      Builder.CreateSelect(Builder.CreateICmpULE(DestSize, SrcSize),
+                           ConstantInt::getNullValue(DestSize->getType()),
+                           Builder.CreateSub(DestSize, SrcSize));
+  Builder.CreateMemSet(Builder.CreateGEP(Dest, SrcSize), MemSet->getOperand(1),
+                       MemsetLen, Align);
+
+  MD->removeInstruction(MemSet);
+  MemSet->eraseFromParent();
+  return true;
+}
+
+/// Transform memcpy to memset when its source was just memset.
+/// In other words, turn:
+/// \code
+///   memset(dst1, c, dst1_size);
+///   memcpy(dst2, dst1, dst2_size);
+/// \endcode
+/// into:
+/// \code
+///   memset(dst1, c, dst1_size);
+///   memset(dst2, c, dst2_size);
+/// \endcode
+/// When dst2_size <= dst1_size.
+///
+/// The \p MemCpy must have a Constant length.
+bool MemCpyOpt::performMemCpyToMemSetOptzn(MemCpyInst *MemCpy,
+                                           MemSetInst *MemSet) {
+  // This only makes sense on memcpy(..., memset(...), ...).
+  if (MemSet->getRawDest() != MemCpy->getRawSource())
+    return false;
+
+  ConstantInt *CopySize = cast<ConstantInt>(MemCpy->getLength());
+  ConstantInt *MemSetSize = dyn_cast<ConstantInt>(MemSet->getLength());
+  // Make sure the memcpy doesn't read any more than what the memset wrote.
+  // Don't worry about sizes larger than i64.
+  if (!MemSetSize || CopySize->getZExtValue() > MemSetSize->getZExtValue())
+    return false;
+
+  IRBuilder<> Builder(MemCpy);
+  Builder.CreateMemSet(MemCpy->getRawDest(), MemSet->getOperand(1),
+                       CopySize, MemCpy->getAlignment());
+  return true;
+}
 
 /// processMemCpy - perform simplification of memcpy's.  If we have memcpy A
 /// which copies X to Y, and memcpy B which copies Y to Z, then we can rewrite
@@ -869,17 +958,26 @@ bool MemCpyOpt::processMemCpy(MemCpyInst *M) {
         return true;
       }
 
+  MemDepResult DepInfo = MD->getDependency(M);
+
+  // Try to turn a partially redundant memset + memcpy into
+  // memcpy + smaller memset.  We don't need the memcpy size for this.
+  if (DepInfo.isClobber())
+    if (MemSetInst *MDep = dyn_cast<MemSetInst>(DepInfo.getInst()))
+      if (processMemSetMemCpyDependence(M, MDep))
+        return true;
+
   // The optimizations after this point require the memcpy size.
   ConstantInt *CopySize = dyn_cast<ConstantInt>(M->getLength());
   if (!CopySize) return false;
 
-  // The are three possible optimizations we can do for memcpy:
+  // There are four possible optimizations we can do for memcpy:
   //   a) memcpy-memcpy xform which exposes redundance for DSE.
   //   b) call-memcpy xform for return slot optimization.
   //   c) memcpy from freshly alloca'd space or space that has just started its
   //      lifetime copies undefined data, and we can therefore eliminate the
   //      memcpy in favor of the data that was already at the destination.
-  MemDepResult DepInfo = MD->getDependency(M);
+  //   d) memcpy from a just-memset'd source can be turned into memset.
   if (DepInfo.isClobber()) {
     if (CallInst *C = dyn_cast<CallInst>(DepInfo.getInst())) {
       if (performCallSlotOptzn(M, M->getDest(), M->getSource(),
@@ -892,12 +990,13 @@ bool MemCpyOpt::processMemCpy(MemCpyInst *M) {
     }
   }
 
-  AliasAnalysis::Location SrcLoc = AliasAnalysis::getLocationForSource(M);
+  MemoryLocation SrcLoc = MemoryLocation::getForSource(M);
   MemDepResult SrcDepInfo = MD->getPointerDependencyFrom(SrcLoc, true,
                                                          M, M->getParent());
+
   if (SrcDepInfo.isClobber()) {
     if (MemCpyInst *MDep = dyn_cast<MemCpyInst>(SrcDepInfo.getInst()))
-      return processMemCpyMemCpyDependence(M, MDep, CopySize->getZExtValue());
+      return processMemCpyMemCpyDependence(M, MDep);
   } else if (SrcDepInfo.isDef()) {
     Instruction *I = SrcDepInfo.getInst();
     bool hasUndefContents = false;
@@ -919,6 +1018,15 @@ bool MemCpyOpt::processMemCpy(MemCpyInst *M) {
     }
   }
 
+  if (SrcDepInfo.isClobber())
+    if (MemSetInst *MDep = dyn_cast<MemSetInst>(SrcDepInfo.getInst()))
+      if (performMemCpyToMemSetOptzn(M, MDep)) {
+        MD->removeInstruction(M);
+        M->eraseFromParent();
+        ++NumCpyToSet;
+        return true;
+      }
+
   return false;
 }
 
@@ -931,7 +1039,8 @@ bool MemCpyOpt::processMemMove(MemMoveInst *M) {
     return false;
 
   // See if the pointers alias.
-  if (!AA.isNoAlias(AA.getLocationForDest(M), AA.getLocationForSource(M)))
+  if (!AA.isNoAlias(MemoryLocation::getForDest(M),
+                    MemoryLocation::getForSource(M)))
     return false;
 
   DEBUG(dbgs() << "MemCpyOpt: Optimizing memmove -> memcpy: " << *M << "\n");
@@ -959,10 +1068,9 @@ bool MemCpyOpt::processByValArgument(CallSite CS, unsigned ArgNo) {
   Value *ByValArg = CS.getArgument(ArgNo);
   Type *ByValTy = cast<PointerType>(ByValArg->getType())->getElementType();
   uint64_t ByValSize = DL.getTypeAllocSize(ByValTy);
-  MemDepResult DepInfo =
-    MD->getPointerDependencyFrom(AliasAnalysis::Location(ByValArg, ByValSize),
-                                 true, CS.getInstruction(),
-                                 CS.getInstruction()->getParent());
+  MemDepResult DepInfo = MD->getPointerDependencyFrom(
+      MemoryLocation(ByValArg, ByValSize), true, CS.getInstruction(),
+      CS.getInstruction()->getParent());
   if (!DepInfo.isClobber())
     return false;
 
@@ -1005,8 +1113,8 @@ bool MemCpyOpt::processByValArgument(CallSite CS, unsigned ArgNo) {
   // NOTE: This is conservative, it will stop on any read from the source loc,
   // not just the defining memcpy.
   MemDepResult SourceDep =
-    MD->getPointerDependencyFrom(AliasAnalysis::getLocationForSource(MDep),
-                                 false, CS.getInstruction(), MDep->getParent());
+      MD->getPointerDependencyFrom(MemoryLocation::getForSource(MDep), false,
+                                   CS.getInstruction(), MDep->getParent());
   if (!SourceDep.isClobber() || SourceDep.getInst() != MDep)
     return false;
 
diff --git a/lib/Transforms/Scalar/MergedLoadStoreMotion.cpp b/lib/Transforms/Scalar/MergedLoadStoreMotion.cpp
index 73f42961cb1..6a094b2a7a1 100644
--- a/lib/Transforms/Scalar/MergedLoadStoreMotion.cpp
+++ b/lib/Transforms/Scalar/MergedLoadStoreMotion.cpp
@@ -117,8 +117,8 @@ class MergedLoadStoreMotion : public FunctionPass {
   // This transformation requires dominator postdominator info
   void getAnalysisUsage(AnalysisUsage &AU) const override {
     AU.addRequired<TargetLibraryInfoWrapperPass>();
-    AU.addRequired<MemoryDependenceAnalysis>();
     AU.addRequired<AliasAnalysis>();
+    AU.addPreserved<MemoryDependenceAnalysis>();
     AU.addPreserved<AliasAnalysis>();
   }
 
@@ -144,9 +144,8 @@ class MergedLoadStoreMotion : public FunctionPass {
   // Routines for sinking stores
   StoreInst *canSinkFromBlock(BasicBlock *BB, StoreInst *SI);
   PHINode *getPHIOperand(BasicBlock *BB, StoreInst *S0, StoreInst *S1);
-  bool isStoreSinkBarrierInRange(const Instruction& Start,
-                                 const Instruction& End,
-                                 AliasAnalysis::Location Loc);
+  bool isStoreSinkBarrierInRange(const Instruction &Start,
+                                 const Instruction &End, MemoryLocation Loc);
   bool sinkStore(BasicBlock *BB, StoreInst *SinkCand, StoreInst *ElseInst);
   bool mergeStores(BasicBlock *BB);
   // The mergeLoad/Store algorithms could have Size0 * Size1 complexity,
@@ -241,8 +240,8 @@ bool MergedLoadStoreMotion::isDiamondHead(BasicBlock *BB) {
 bool MergedLoadStoreMotion::isLoadHoistBarrierInRange(const Instruction& Start, 
                                                       const Instruction& End,
                                                       LoadInst* LI) {
-  AliasAnalysis::Location Loc = AA->getLocation(LI);
-  return AA->canInstructionRangeModRef(Start, End, Loc, AliasAnalysis::Mod);
+  MemoryLocation Loc = MemoryLocation::get(LI);
+  return AA->canInstructionRangeModRef(Start, End, Loc, MRI_Mod);
 }
 
 ///
@@ -266,8 +265,8 @@ LoadInst *MergedLoadStoreMotion::canHoistFromBlock(BasicBlock *BB1,
     LoadInst *Load1 = dyn_cast<LoadInst>(Inst);
     BasicBlock *BB0 = Load0->getParent();
 
-    AliasAnalysis::Location Loc0 = AA->getLocation(Load0);
-    AliasAnalysis::Location Loc1 = AA->getLocation(Load1);
+    MemoryLocation Loc0 = MemoryLocation::get(Load0);
+    MemoryLocation Loc1 = MemoryLocation::get(Load1);
     if (AA->isMustAlias(Loc0, Loc1) && Load0->isSameOperationAs(Load1) &&
         !isLoadHoistBarrierInRange(BB1->front(), *Load1, Load1) &&
         !isLoadHoistBarrierInRange(BB0->front(), *Load0, Load0)) {
@@ -302,10 +301,6 @@ void MergedLoadStoreMotion::hoistInstruction(BasicBlock *BB,
   // Merged instruction
   Instruction *HoistedInst = HoistCand->clone();
 
-  // Notify AA of the new value.
-  if (isa<LoadInst>(HoistCand))
-    AA->copyValue(HoistCand, HoistedInst);
-
   // Hoist instruction.
   HoistedInst->insertBefore(HoistPt);
 
@@ -400,11 +395,10 @@ bool MergedLoadStoreMotion::mergeLoads(BasicBlock *BB) {
 /// happening it is considered a sink barrier.
 ///
 
-bool MergedLoadStoreMotion::isStoreSinkBarrierInRange(const Instruction& Start,
-                                                      const Instruction& End,
-                                                      AliasAnalysis::Location
-                                                      Loc) {
-  return AA->canInstructionRangeModRef(Start, End, Loc, AliasAnalysis::ModRef);
+bool MergedLoadStoreMotion::isStoreSinkBarrierInRange(const Instruction &Start,
+                                                      const Instruction &End,
+                                                      MemoryLocation Loc) {
+  return AA->canInstructionRangeModRef(Start, End, Loc, MRI_ModRef);
 }
 
 ///
@@ -425,8 +419,8 @@ StoreInst *MergedLoadStoreMotion::canSinkFromBlock(BasicBlock *BB1,
 
     StoreInst *Store1 = cast<StoreInst>(Inst);
 
-    AliasAnalysis::Location Loc0 = AA->getLocation(Store0);
-    AliasAnalysis::Location Loc1 = AA->getLocation(Store1);
+    MemoryLocation Loc0 = MemoryLocation::get(Store0);
+    MemoryLocation Loc1 = MemoryLocation::get(Store1);
     if (AA->isMustAlias(Loc0, Loc1) && Store0->isSameOperationAs(Store1) &&
       !isStoreSinkBarrierInRange(*(std::next(BasicBlock::iterator(Store1))),
                                  BB1->back(), Loc1) &&
@@ -452,18 +446,8 @@ PHINode *MergedLoadStoreMotion::getPHIOperand(BasicBlock *BB, StoreInst *S0,
                             BB->begin());
     NewPN->addIncoming(Opd1, S0->getParent());
     NewPN->addIncoming(Opd2, S1->getParent());
-    if (NewPN->getType()->getScalarType()->isPointerTy()) {
-      // Notify AA of the new value.
-      AA->copyValue(Opd1, NewPN);
-      AA->copyValue(Opd2, NewPN);
-      // AA needs to be informed when a PHI-use of the pointer value is added
-      for (unsigned I = 0, E = NewPN->getNumIncomingValues(); I != E; ++I) {
-        unsigned J = PHINode::getOperandNumForIncomingValue(I);
-        AA->addEscapingUse(NewPN->getOperandUse(J));
-      }
-      if (MD)
-        MD->invalidateCachedPointerInfo(NewPN);
-    }
+    if (MD && NewPN->getType()->getScalarType()->isPointerTy())
+      MD->invalidateCachedPointerInfo(NewPN);
   }
   return NewPN;
 }
@@ -493,7 +477,6 @@ bool MergedLoadStoreMotion::sinkStore(BasicBlock *BB, StoreInst *S0,
     // Create the new store to be inserted at the join point.
     StoreInst *SNew = (StoreInst *)(S0->clone());
     Instruction *ANew = A0->clone();
-    AA->copyValue(S0, SNew);
     SNew->insertBefore(InsertPt);
     ANew->insertBefore(SNew);
 
@@ -580,7 +563,7 @@ bool MergedLoadStoreMotion::mergeStores(BasicBlock *T) {
 /// \brief Run the transformation for each function
 ///
 bool MergedLoadStoreMotion::runOnFunction(Function &F) {
-  MD = &getAnalysis<MemoryDependenceAnalysis>();
+  MD = getAnalysisIfAvailable<MemoryDependenceAnalysis>();
   AA = &getAnalysis<AliasAnalysis>();
 
   bool Changed = false;
diff --git a/lib/Transforms/Scalar/NaryReassociate.cpp b/lib/Transforms/Scalar/NaryReassociate.cpp
index fea7641126f..58b9c9d092d 100644
--- a/lib/Transforms/Scalar/NaryReassociate.cpp
+++ b/lib/Transforms/Scalar/NaryReassociate.cpp
@@ -74,20 +74,18 @@
 // 1) We only considers n-ary adds for now. This should be extended and
 // generalized.
 //
-// 2) Besides arithmetic operations, similar reassociation can be applied to
-// GEPs. For example, if
-//   X = &arr[a]
-// dominates
-//   Y = &arr[a + b]
-// we may rewrite Y into X + b.
-//
 //===----------------------------------------------------------------------===//
 
+#include "llvm/Analysis/AssumptionCache.h"
 #include "llvm/Analysis/ScalarEvolution.h"
 #include "llvm/Analysis/TargetLibraryInfo.h"
+#include "llvm/Analysis/TargetTransformInfo.h"
+#include "llvm/Analysis/ValueTracking.h"
 #include "llvm/IR/Dominators.h"
 #include "llvm/IR/Module.h"
 #include "llvm/IR/PatternMatch.h"
+#include "llvm/Support/Debug.h"
+#include "llvm/Support/raw_ostream.h"
 #include "llvm/Transforms/Scalar.h"
 #include "llvm/Transforms/Utils/Local.h"
 using namespace llvm;
@@ -104,15 +102,21 @@ class NaryReassociate : public FunctionPass {
     initializeNaryReassociatePass(*PassRegistry::getPassRegistry());
   }
 
+  bool doInitialization(Module &M) override {
+    DL = &M.getDataLayout();
+    return false;
+  }
   bool runOnFunction(Function &F) override;
 
   void getAnalysisUsage(AnalysisUsage &AU) const override {
     AU.addPreserved<DominatorTreeWrapperPass>();
     AU.addPreserved<ScalarEvolution>();
     AU.addPreserved<TargetLibraryInfoWrapperPass>();
+    AU.addRequired<AssumptionCacheTracker>();
     AU.addRequired<DominatorTreeWrapperPass>();
     AU.addRequired<ScalarEvolution>();
     AU.addRequired<TargetLibraryInfoWrapperPass>();
+    AU.addRequired<TargetTransformInfoWrapperPass>();
     AU.setPreservesCFG();
   }
 
@@ -120,16 +124,55 @@ class NaryReassociate : public FunctionPass {
   // Runs only one iteration of the dominator-based algorithm. See the header
   // comments for why we need multiple iterations.
   bool doOneIteration(Function &F);
-  // Reasssociates I to a better form.
-  Instruction *tryReassociateAdd(Instruction *I);
+
+  // Reassociates I for better CSE.
+  Instruction *tryReassociate(Instruction *I);
+
+  // Reassociate GEP for better CSE.
+  Instruction *tryReassociateGEP(GetElementPtrInst *GEP);
+  // Try splitting GEP at the I-th index and see whether either part can be
+  // CSE'ed. This is a helper function for tryReassociateGEP.
+  //
+  // \p IndexedType The element type indexed by GEP's I-th index. This is
+  //                equivalent to
+  //                  GEP->getIndexedType(GEP->getPointerOperand(), 0-th index,
+  //                                      ..., i-th index).
+  GetElementPtrInst *tryReassociateGEPAtIndex(GetElementPtrInst *GEP,
+                                              unsigned I, Type *IndexedType);
+  // Given GEP's I-th index = LHS + RHS, see whether &Base[..][LHS][..] or
+  // &Base[..][RHS][..] can be CSE'ed and rewrite GEP accordingly.
+  GetElementPtrInst *tryReassociateGEPAtIndex(GetElementPtrInst *GEP,
+                                              unsigned I, Value *LHS,
+                                              Value *RHS, Type *IndexedType);
+
+  // Reassociate Add for better CSE.
+  Instruction *tryReassociateAdd(BinaryOperator *I);
   // A helper function for tryReassociateAdd. LHS and RHS are explicitly passed.
   Instruction *tryReassociateAdd(Value *LHS, Value *RHS, Instruction *I);
   // Rewrites I to LHS + RHS if LHS is computed already.
   Instruction *tryReassociatedAdd(const SCEV *LHS, Value *RHS, Instruction *I);
 
+  // Returns the closest dominator of \c Dominatee that computes
+  // \c CandidateExpr. Returns null if not found.
+  Instruction *findClosestMatchingDominator(const SCEV *CandidateExpr,
+                                            Instruction *Dominatee);
+  // GetElementPtrInst implicitly sign-extends an index if the index is shorter
+  // than the pointer size. This function returns whether Index is shorter than
+  // GEP's pointer size, i.e., whether Index needs to be sign-extended in order
+  // to be an index of GEP.
+  bool requiresSignExtension(Value *Index, GetElementPtrInst *GEP);
+  // Returns whether V is known to be non-negative at context \c Ctxt.
+  bool isKnownNonNegative(Value *V, Instruction *Ctxt);
+  // Returns whether AO may sign overflow at context \c Ctxt. It computes a
+  // conservative result -- it answers true when not sure.
+  bool maySignOverflow(AddOperator *AO, Instruction *Ctxt);
+
+  AssumptionCache *AC;
+  const DataLayout *DL;
   DominatorTree *DT;
   ScalarEvolution *SE;
   TargetLibraryInfo *TLI;
+  TargetTransformInfo *TTI;
   // A lookup table quickly telling which instructions compute the given SCEV.
   // Note that there can be multiple instructions at different locations
   // computing to the same SCEV, so we map a SCEV to an instruction list.  For
@@ -146,9 +189,11 @@ class NaryReassociate : public FunctionPass {
 char NaryReassociate::ID = 0;
 INITIALIZE_PASS_BEGIN(NaryReassociate, "nary-reassociate", "Nary reassociation",
                       false, false)
+INITIALIZE_PASS_DEPENDENCY(AssumptionCacheTracker)
 INITIALIZE_PASS_DEPENDENCY(DominatorTreeWrapperPass)
 INITIALIZE_PASS_DEPENDENCY(ScalarEvolution)
 INITIALIZE_PASS_DEPENDENCY(TargetLibraryInfoWrapperPass)
+INITIALIZE_PASS_DEPENDENCY(TargetTransformInfoWrapperPass)
 INITIALIZE_PASS_END(NaryReassociate, "nary-reassociate", "Nary reassociation",
                     false, false)
 
@@ -160,9 +205,11 @@ bool NaryReassociate::runOnFunction(Function &F) {
   if (skipOptnoneFunction(F))
     return false;
 
+  AC = &getAnalysis<AssumptionCacheTracker>().getAssumptionCache(F);
   DT = &getAnalysis<DominatorTreeWrapperPass>().getDomTree();
   SE = &getAnalysis<ScalarEvolution>();
   TLI = &getAnalysis<TargetLibraryInfoWrapperPass>().getTLI();
+  TTI = &getAnalysis<TargetTransformInfoWrapperPass>().getTTI(F);
 
   bool Changed = false, ChangedInThisIteration;
   do {
@@ -172,33 +219,264 @@ bool NaryReassociate::runOnFunction(Function &F) {
   return Changed;
 }
 
+// Whitelist the instruction types NaryReassociate handles for now.
+static bool isPotentiallyNaryReassociable(Instruction *I) {
+  switch (I->getOpcode()) {
+  case Instruction::Add:
+  case Instruction::GetElementPtr:
+    return true;
+  default:
+    return false;
+  }
+}
+
 bool NaryReassociate::doOneIteration(Function &F) {
   bool Changed = false;
   SeenExprs.clear();
-  // Traverse the dominator tree in the depth-first order. This order makes sure
-  // all bases of a candidate are in Candidates when we process it.
+  // Process the basic blocks in pre-order of the dominator tree. This order
+  // ensures that all bases of a candidate are in Candidates when we process it.
   for (auto Node = GraphTraits<DominatorTree *>::nodes_begin(DT);
        Node != GraphTraits<DominatorTree *>::nodes_end(DT); ++Node) {
     BasicBlock *BB = Node->getBlock();
     for (auto I = BB->begin(); I != BB->end(); ++I) {
-      if (I->getOpcode() == Instruction::Add) {
-        if (Instruction *NewI = tryReassociateAdd(I)) {
+      if (SE->isSCEVable(I->getType()) && isPotentiallyNaryReassociable(I)) {
+        const SCEV *OldSCEV = SE->getSCEV(I);
+        if (Instruction *NewI = tryReassociate(I)) {
           Changed = true;
           SE->forgetValue(I);
           I->replaceAllUsesWith(NewI);
           RecursivelyDeleteTriviallyDeadInstructions(I, TLI);
           I = NewI;
         }
-        // We should add the rewritten instruction because tryReassociateAdd may
-        // have invalidated the original one.
-        SeenExprs[SE->getSCEV(I)].push_back(I);
+        // Add the rewritten instruction to SeenExprs; the original instruction
+        // is deleted.
+        const SCEV *NewSCEV = SE->getSCEV(I);
+        SeenExprs[NewSCEV].push_back(I);
+        // Ideally, NewSCEV should equal OldSCEV because tryReassociate(I)
+        // is equivalent to I. However, ScalarEvolution::getSCEV may
+        // weaken nsw causing NewSCEV not to equal OldSCEV. For example, suppose
+        // we reassociate
+        //   I = &a[sext(i +nsw j)] // assuming sizeof(a[0]) = 4
+        // to
+        //   NewI = &a[sext(i)] + sext(j).
+        //
+        // ScalarEvolution computes
+        //   getSCEV(I)    = a + 4 * sext(i + j)
+        //   getSCEV(newI) = a + 4 * sext(i) + 4 * sext(j)
+        // which are different SCEVs.
+        //
+        // To alleviate this issue of ScalarEvolution not always capturing
+        // equivalence, we add I to SeenExprs[OldSCEV] as well so that we can
+        // map both SCEV before and after tryReassociate(I) to I.
+        //
+        // This improvement is exercised in @reassociate_gep_nsw in nary-gep.ll.
+        if (NewSCEV != OldSCEV)
+          SeenExprs[OldSCEV].push_back(I);
       }
     }
   }
   return Changed;
 }
 
-Instruction *NaryReassociate::tryReassociateAdd(Instruction *I) {
+Instruction *NaryReassociate::tryReassociate(Instruction *I) {
+  switch (I->getOpcode()) {
+  case Instruction::Add:
+    return tryReassociateAdd(cast<BinaryOperator>(I));
+  case Instruction::GetElementPtr:
+    return tryReassociateGEP(cast<GetElementPtrInst>(I));
+  default:
+    llvm_unreachable("should be filtered out by isPotentiallyNaryReassociable");
+  }
+}
+
+// FIXME: extract this method into TTI->getGEPCost.
+static bool isGEPFoldable(GetElementPtrInst *GEP,
+                          const TargetTransformInfo *TTI,
+                          const DataLayout *DL) {
+  GlobalVariable *BaseGV = nullptr;
+  int64_t BaseOffset = 0;
+  bool HasBaseReg = false;
+  int64_t Scale = 0;
+
+  if (GlobalVariable *GV = dyn_cast<GlobalVariable>(GEP->getPointerOperand()))
+    BaseGV = GV;
+  else
+    HasBaseReg = true;
+
+  gep_type_iterator GTI = gep_type_begin(GEP);
+  for (auto I = GEP->idx_begin(); I != GEP->idx_end(); ++I, ++GTI) {
+    if (isa<SequentialType>(*GTI)) {
+      int64_t ElementSize = DL->getTypeAllocSize(GTI.getIndexedType());
+      if (ConstantInt *ConstIdx = dyn_cast<ConstantInt>(*I)) {
+        BaseOffset += ConstIdx->getSExtValue() * ElementSize;
+      } else {
+        // Needs scale register.
+        if (Scale != 0) {
+          // No addressing mode takes two scale registers.
+          return false;
+        }
+        Scale = ElementSize;
+      }
+    } else {
+      StructType *STy = cast<StructType>(*GTI);
+      uint64_t Field = cast<ConstantInt>(*I)->getZExtValue();
+      BaseOffset += DL->getStructLayout(STy)->getElementOffset(Field);
+    }
+  }
+
+  unsigned AddrSpace = GEP->getPointerAddressSpace();
+  return TTI->isLegalAddressingMode(GEP->getType()->getElementType(), BaseGV,
+                                    BaseOffset, HasBaseReg, Scale, AddrSpace);
+}
+
+Instruction *NaryReassociate::tryReassociateGEP(GetElementPtrInst *GEP) {
+  // Not worth reassociating GEP if it is foldable.
+  if (isGEPFoldable(GEP, TTI, DL))
+    return nullptr;
+
+  gep_type_iterator GTI = gep_type_begin(*GEP);
+  for (unsigned I = 1, E = GEP->getNumOperands(); I != E; ++I) {
+    if (isa<SequentialType>(*GTI++)) {
+      if (auto *NewGEP = tryReassociateGEPAtIndex(GEP, I - 1, *GTI)) {
+        return NewGEP;
+      }
+    }
+  }
+  return nullptr;
+}
+
+bool NaryReassociate::requiresSignExtension(Value *Index,
+                                            GetElementPtrInst *GEP) {
+  unsigned PointerSizeInBits =
+      DL->getPointerSizeInBits(GEP->getType()->getPointerAddressSpace());
+  return cast<IntegerType>(Index->getType())->getBitWidth() < PointerSizeInBits;
+}
+
+bool NaryReassociate::isKnownNonNegative(Value *V, Instruction *Ctxt) {
+  bool NonNegative, Negative;
+  // TODO: ComputeSignBits is expensive. Consider caching the results.
+  ComputeSignBit(V, NonNegative, Negative, *DL, 0, AC, Ctxt, DT);
+  return NonNegative;
+}
+
+bool NaryReassociate::maySignOverflow(AddOperator *AO, Instruction *Ctxt) {
+  if (AO->hasNoSignedWrap())
+    return false;
+
+  Value *LHS = AO->getOperand(0), *RHS = AO->getOperand(1);
+  // If LHS or RHS has the same sign as the sum, AO doesn't sign overflow.
+  // TODO: handle the negative case as well.
+  if (isKnownNonNegative(AO, Ctxt) &&
+      (isKnownNonNegative(LHS, Ctxt) || isKnownNonNegative(RHS, Ctxt)))
+    return false;
+
+  return true;
+}
+
+GetElementPtrInst *
+NaryReassociate::tryReassociateGEPAtIndex(GetElementPtrInst *GEP, unsigned I,
+                                          Type *IndexedType) {
+  Value *IndexToSplit = GEP->getOperand(I + 1);
+  if (SExtInst *SExt = dyn_cast<SExtInst>(IndexToSplit)) {
+    IndexToSplit = SExt->getOperand(0);
+  } else if (ZExtInst *ZExt = dyn_cast<ZExtInst>(IndexToSplit)) {
+    // zext can be treated as sext if the source is non-negative.
+    if (isKnownNonNegative(ZExt->getOperand(0), GEP))
+      IndexToSplit = ZExt->getOperand(0);
+  }
+
+  if (AddOperator *AO = dyn_cast<AddOperator>(IndexToSplit)) {
+    // If the I-th index needs sext and the underlying add is not equipped with
+    // nsw, we cannot split the add because
+    //   sext(LHS + RHS) != sext(LHS) + sext(RHS).
+    if (requiresSignExtension(IndexToSplit, GEP) && maySignOverflow(AO, GEP))
+      return nullptr;
+    Value *LHS = AO->getOperand(0), *RHS = AO->getOperand(1);
+    // IndexToSplit = LHS + RHS.
+    if (auto *NewGEP = tryReassociateGEPAtIndex(GEP, I, LHS, RHS, IndexedType))
+      return NewGEP;
+    // Symmetrically, try IndexToSplit = RHS + LHS.
+    if (LHS != RHS) {
+      if (auto *NewGEP =
+              tryReassociateGEPAtIndex(GEP, I, RHS, LHS, IndexedType))
+        return NewGEP;
+    }
+  }
+  return nullptr;
+}
+
+GetElementPtrInst *NaryReassociate::tryReassociateGEPAtIndex(
+    GetElementPtrInst *GEP, unsigned I, Value *LHS, Value *RHS,
+    Type *IndexedType) {
+  // Look for GEP's closest dominator that has the same SCEV as GEP except that
+  // the I-th index is replaced with LHS.
+  SmallVector<const SCEV *, 4> IndexExprs;
+  for (auto Index = GEP->idx_begin(); Index != GEP->idx_end(); ++Index)
+    IndexExprs.push_back(SE->getSCEV(*Index));
+  // Replace the I-th index with LHS.
+  IndexExprs[I] = SE->getSCEV(LHS);
+  if (isKnownNonNegative(LHS, GEP) &&
+      DL->getTypeSizeInBits(LHS->getType()) <
+          DL->getTypeSizeInBits(GEP->getOperand(I)->getType())) {
+    // Zero-extend LHS if it is non-negative. InstCombine canonicalizes sext to
+    // zext if the source operand is proved non-negative. We should do that
+    // consistently so that CandidateExpr more likely appears before. See
+    // @reassociate_gep_assume for an example of this canonicalization.
+    IndexExprs[I] =
+        SE->getZeroExtendExpr(IndexExprs[I], GEP->getOperand(I)->getType());
+  }
+  const SCEV *CandidateExpr = SE->getGEPExpr(
+      GEP->getSourceElementType(), SE->getSCEV(GEP->getPointerOperand()),
+      IndexExprs, GEP->isInBounds());
+
+  auto *Candidate = findClosestMatchingDominator(CandidateExpr, GEP);
+  if (Candidate == nullptr)
+    return nullptr;
+
+  PointerType *TypeOfCandidate = dyn_cast<PointerType>(Candidate->getType());
+  // Pretty rare but theoretically possible when a numeric value happens to
+  // share CandidateExpr.
+  if (TypeOfCandidate == nullptr)
+    return nullptr;
+
+  // NewGEP = (char *)Candidate + RHS * sizeof(IndexedType)
+  uint64_t IndexedSize = DL->getTypeAllocSize(IndexedType);
+  Type *ElementType = TypeOfCandidate->getElementType();
+  uint64_t ElementSize = DL->getTypeAllocSize(ElementType);
+  // Another less rare case: because I is not necessarily the last index of the
+  // GEP, the size of the type at the I-th index (IndexedSize) is not
+  // necessarily divisible by ElementSize. For example,
+  //
+  // #pragma pack(1)
+  // struct S {
+  //   int a[3];
+  //   int64 b[8];
+  // };
+  // #pragma pack()
+  //
+  // sizeof(S) = 100 is indivisible by sizeof(int64) = 8.
+  //
+  // TODO: bail out on this case for now. We could emit uglygep.
+  if (IndexedSize % ElementSize != 0)
+    return nullptr;
+
+  // NewGEP = &Candidate[RHS * (sizeof(IndexedType) / sizeof(Candidate[0])));
+  IRBuilder<> Builder(GEP);
+  Type *IntPtrTy = DL->getIntPtrType(TypeOfCandidate);
+  if (RHS->getType() != IntPtrTy)
+    RHS = Builder.CreateSExtOrTrunc(RHS, IntPtrTy);
+  if (IndexedSize != ElementSize) {
+    RHS = Builder.CreateMul(
+        RHS, ConstantInt::get(IntPtrTy, IndexedSize / ElementSize));
+  }
+  GetElementPtrInst *NewGEP =
+      cast<GetElementPtrInst>(Builder.CreateGEP(Candidate, RHS));
+  NewGEP->setIsInBounds(GEP->isInBounds());
+  NewGEP->takeName(GEP);
+  return NewGEP;
+}
+
+Instruction *NaryReassociate::tryReassociateAdd(BinaryOperator *I) {
   Value *LHS = I->getOperand(0), *RHS = I->getOperand(1);
   if (auto *NewI = tryReassociateAdd(LHS, RHS, I))
     return NewI;
@@ -216,37 +494,48 @@ Instruction *NaryReassociate::tryReassociateAdd(Value *LHS, Value *RHS,
     //   = (A + RHS) + B or (B + RHS) + A
     const SCEV *AExpr = SE->getSCEV(A), *BExpr = SE->getSCEV(B);
     const SCEV *RHSExpr = SE->getSCEV(RHS);
-    if (auto *NewI = tryReassociatedAdd(SE->getAddExpr(AExpr, RHSExpr), B, I))
-      return NewI;
-    if (auto *NewI = tryReassociatedAdd(SE->getAddExpr(BExpr, RHSExpr), A, I))
-      return NewI;
+    if (BExpr != RHSExpr) {
+      if (auto *NewI = tryReassociatedAdd(SE->getAddExpr(AExpr, RHSExpr), B, I))
+        return NewI;
+    }
+    if (AExpr != RHSExpr) {
+      if (auto *NewI = tryReassociatedAdd(SE->getAddExpr(BExpr, RHSExpr), A, I))
+        return NewI;
+    }
   }
   return nullptr;
 }
 
 Instruction *NaryReassociate::tryReassociatedAdd(const SCEV *LHSExpr,
                                                  Value *RHS, Instruction *I) {
-  auto Pos = SeenExprs.find(LHSExpr);
-  // Bail out if LHSExpr is not previously seen.
+  // Look for the closest dominator LHS of I that computes LHSExpr, and replace
+  // I with LHS + RHS.
+  auto *LHS = findClosestMatchingDominator(LHSExpr, I);
+  if (LHS == nullptr)
+    return nullptr;
+
+  Instruction *NewI = BinaryOperator::CreateAdd(LHS, RHS, "", I);
+  NewI->takeName(I);
+  return NewI;
+}
+
+Instruction *
+NaryReassociate::findClosestMatchingDominator(const SCEV *CandidateExpr,
+                                              Instruction *Dominatee) {
+  auto Pos = SeenExprs.find(CandidateExpr);
   if (Pos == SeenExprs.end())
     return nullptr;
 
-  auto &LHSCandidates = Pos->second;
-  // Look for the closest dominator LHS of I that computes LHSExpr, and replace
-  // I with LHS + RHS.
-  //
-  // Because we traverse the dominator tree in the pre-order, a
+  auto &Candidates = Pos->second;
+  // Because we process the basic blocks in pre-order of the dominator tree, a
   // candidate that doesn't dominate the current instruction won't dominate any
   // future instruction either. Therefore, we pop it out of the stack. This
   // optimization makes the algorithm O(n).
-  while (!LHSCandidates.empty()) {
-    Instruction *LHS = LHSCandidates.back();
-    if (DT->dominates(LHS, I)) {
-      Instruction *NewI = BinaryOperator::CreateAdd(LHS, RHS, "", I);
-      NewI->takeName(I);
-      return NewI;
-    }
-    LHSCandidates.pop_back();
+  while (!Candidates.empty()) {
+    Instruction *Candidate = Candidates.back();
+    if (DT->dominates(Candidate, Dominatee))
+      return Candidate;
+    Candidates.pop_back();
   }
   return nullptr;
 }
diff --git a/lib/Transforms/Scalar/PlaceSafepoints.cpp b/lib/Transforms/Scalar/PlaceSafepoints.cpp
index 536f2a673bb..098e6eff15f 100644
--- a/lib/Transforms/Scalar/PlaceSafepoints.cpp
+++ b/lib/Transforms/Scalar/PlaceSafepoints.cpp
@@ -16,7 +16,7 @@
 // return PC of the call.  A runtime can determine where values listed in the
 // deopt arguments and (after RewriteStatepointsForGC) gc arguments are located
 // on the stack when the code is suspended inside such a call.  Every parse
-// point is represented by a call wrapped in an gc.statepoint intrinsic.  
+// point is represented by a call wrapped in an gc.statepoint intrinsic.
 // - A "poll" is an explicit check in the generated code to determine if the
 // runtime needs the generated code to cooperate by calling a helper routine
 // and thus suspending its execution at a known state. The call to the helper
@@ -51,7 +51,9 @@
 #include "llvm/Pass.h"
 #include "llvm/IR/LegacyPassManager.h"
 #include "llvm/ADT/SetOperations.h"
+#include "llvm/ADT/SetVector.h"
 #include "llvm/ADT/Statistic.h"
+#include "llvm/ADT/StringRef.h"
 #include "llvm/Analysis/LoopPass.h"
 #include "llvm/Analysis/LoopInfo.h"
 #include "llvm/Analysis/ScalarEvolution.h"
@@ -109,9 +111,9 @@ static cl::opt<bool> TraceLSP("spp-trace", cl::Hidden, cl::init(false));
 
 namespace {
 
-/** An analysis pass whose purpose is to identify each of the backedges in
-    the function which require a safepoint poll to be inserted. */
-struct PlaceBackedgeSafepointsImpl : public LoopPass {
+/// An analysis pass whose purpose is to identify each of the backedges in
+/// the function which require a safepoint poll to be inserted.
+struct PlaceBackedgeSafepointsImpl : public FunctionPass {
   static char ID;
 
   /// The output of the pass - gives a list of each backedge (described by
@@ -121,20 +123,38 @@ struct PlaceBackedgeSafepointsImpl : public LoopPass {
   /// True unless we're running spp-no-calls in which case we need to disable
   /// the call dependend placement opts.
   bool CallSafepointsEnabled;
+
+  ScalarEvolution *SE = nullptr;
+  DominatorTree *DT = nullptr;
+  LoopInfo *LI = nullptr;
+
   PlaceBackedgeSafepointsImpl(bool CallSafepoints = false)
-      : LoopPass(ID), CallSafepointsEnabled(CallSafepoints) {
+      : FunctionPass(ID), CallSafepointsEnabled(CallSafepoints) {
     initializePlaceBackedgeSafepointsImplPass(*PassRegistry::getPassRegistry());
   }
 
-  bool runOnLoop(Loop *, LPPassManager &LPM) override;
+  bool runOnLoop(Loop *);
+  void runOnLoopAndSubLoops(Loop *L) {
+    // Visit all the subloops
+    for (auto I = L->begin(), E = L->end(); I != E; I++)
+      runOnLoopAndSubLoops(*I);
+    runOnLoop(L);
+  }
+
+  bool runOnFunction(Function &F) override {
+    SE = &getAnalysis<ScalarEvolution>();
+    DT = &getAnalysis<DominatorTreeWrapperPass>().getDomTree();
+    LI = &getAnalysis<LoopInfoWrapperPass>().getLoopInfo();
+    for (auto I = LI->begin(), E = LI->end(); I != E; I++) {
+      runOnLoopAndSubLoops(*I);
+    }
+    return false;
+  }
 
   void getAnalysisUsage(AnalysisUsage &AU) const override {
-    // needed for determining if the loop is finite
+    AU.addRequired<DominatorTreeWrapperPass>();
     AU.addRequired<ScalarEvolution>();
-    // to ensure each edge has a single backedge
-    // TODO: is this still required?
-    AU.addRequiredID(LoopSimplifyID);
-
+    AU.addRequired<LoopInfoWrapperPass>();
     // We no longer modify the IR at all in this pass.  Thus all
     // analysis are preserved.
     AU.setPreservesAll();
@@ -147,20 +167,13 @@ static cl::opt<bool> NoCall("spp-no-call", cl::Hidden, cl::init(false));
 static cl::opt<bool> NoBackedge("spp-no-backedge", cl::Hidden, cl::init(false));
 
 namespace {
-struct PlaceSafepoints : public ModulePass {
+struct PlaceSafepoints : public FunctionPass {
   static char ID; // Pass identification, replacement for typeid
 
-  PlaceSafepoints() : ModulePass(ID) {
+  PlaceSafepoints() : FunctionPass(ID) {
     initializePlaceSafepointsPass(*PassRegistry::getPassRegistry());
   }
-  bool runOnModule(Module &M) override {
-    bool modified = false;
-    for (Function &F : M) {
-      modified |= runOnFunction(F);
-    }
-    return modified;
-  }
-  bool runOnFunction(Function &F);
+  bool runOnFunction(Function &F) override;
 
   void getAnalysisUsage(AnalysisUsage &AU) const override {
     // We modify the graph wholesale (inlining, block insertion, etc).  We
@@ -174,7 +187,7 @@ struct PlaceSafepoints : public ModulePass {
 // not handle the parsability of state at the runtime call, that's the
 // callers job.
 static void
-InsertSafepointPoll(DominatorTree &DT, Instruction *after,
+InsertSafepointPoll(Instruction *InsertBefore,
                     std::vector<CallSite> &ParsePointsNeeded /*rval*/);
 
 static bool isGCLeafFunction(const CallSite &CS);
@@ -314,26 +327,17 @@ static void scanInlinedCode(Instruction *start, Instruction *end,
   }
 }
 
-bool PlaceBackedgeSafepointsImpl::runOnLoop(Loop *L, LPPassManager &LPM) {
-  ScalarEvolution *SE = &getAnalysis<ScalarEvolution>();
-
-  // Loop through all predecessors of the loop header and identify all
-  // backedges.  We need to place a safepoint on every backedge (potentially).
-  // Note: Due to LoopSimplify there should only be one.  Assert?  Or can we
-  // relax this?
+bool PlaceBackedgeSafepointsImpl::runOnLoop(Loop *L) {
+  // Loop through all loop latches (branches controlling backedges).  We need
+  // to place a safepoint on every backedge (potentially).
+  // Note: In common usage, there will be only one edge due to LoopSimplify
+  // having run sometime earlier in the pipeline, but this code must be correct
+  // w.r.t. loops with multiple backedges.
   BasicBlock *header = L->getHeader();
-
-  // TODO: Use the analysis pass infrastructure for this.  There is no reason
-  // to recalculate this here.
-  DominatorTree DT;
-  DT.recalculate(*header->getParent());
-
-  bool modified = false;
-  for (BasicBlock *pred : predecessors(header)) {
-    if (!L->contains(pred)) {
-      // This is not a backedge, it's coming from outside the loop
-      continue;
-    }
+  SmallVector<BasicBlock*, 16> LoopLatches;
+  L->getLoopLatches(LoopLatches);
+  for (BasicBlock *pred : LoopLatches) {
+    assert(L->contains(pred));
 
     // Make a policy decision about whether this loop needs a safepoint or
     // not.  Note that this is about unburdening the optimizer in loops, not
@@ -346,7 +350,7 @@ bool PlaceBackedgeSafepointsImpl::runOnLoop(Loop *L, LPPassManager &LPM) {
         continue;
       }
       if (CallSafepointsEnabled &&
-          containsUnconditionalCallSafepoint(L, header, pred, DT)) {
+          containsUnconditionalCallSafepoint(L, header, pred, *DT)) {
         // Note: This is only semantically legal since we won't do any further
         // IPO or inlining before the actual call insertion..  If we hadn't, we
         // might latter loose this call safepoint.
@@ -362,9 +366,6 @@ bool PlaceBackedgeSafepointsImpl::runOnLoop(Loop *L, LPPassManager &LPM) {
     // not help runtime performance that much, but it might help our ability to
     // optimize the inner loop.
 
-    // We're unconditionally going to modify this loop.
-    modified = true;
-
     // Safepoint insertion would involve creating a new basic block (as the
     // target of the current backedge) which does the safepoint (of all live
     // variables) and branches to the true header
@@ -378,7 +379,33 @@ bool PlaceBackedgeSafepointsImpl::runOnLoop(Loop *L, LPPassManager &LPM) {
     PollLocations.push_back(term);
   }
 
-  return modified;
+  return false;
+}
+
+/// Returns true if an entry safepoint is not required before this callsite in
+/// the caller function.
+static bool doesNotRequireEntrySafepointBefore(const CallSite &CS) {
+  Instruction *Inst = CS.getInstruction();
+  if (IntrinsicInst *II = dyn_cast<IntrinsicInst>(Inst)) {
+    switch (II->getIntrinsicID()) {
+    case Intrinsic::experimental_gc_statepoint:
+    case Intrinsic::experimental_patchpoint_void:
+    case Intrinsic::experimental_patchpoint_i64:
+      // The can wrap an actual call which may grow the stack by an unbounded
+      // amount or run forever.
+      return false;
+    default:
+      // Most LLVM intrinsics are things which do not expand to actual calls, or
+      // at least if they do, are leaf functions that cause only finite stack
+      // growth.  In particular, the optimizer likes to form things like memsets
+      // out of stores in the original IR.  Another important example is
+      // llvm.localescape which must occur in the entry block.  Inserting a
+      // safepoint before it is not legal since it could push the localescape
+      // out of the entry block.
+      return true;
+    }
+  }
+  return false;
 }
 
 static Instruction *findLocationForEntrySafepoint(Function &F,
@@ -390,12 +417,6 @@ static Instruction *findLocationForEntrySafepoint(Function &F,
   // that can grow the stack.  This, combined with backedge polls,
   // give us all the progress guarantees we need.
 
-  // Due to the way the frontend generates IR, we may have a couple of initial
-  // basic blocks before the first bytecode.  These will be single-entry
-  // single-exit blocks which conceptually are just part of the first 'real
-  // basic block'.  Since we don't have deopt state until the first bytecode,
-  // walk forward until we've found the first unconditional branch or merge.
-
   // hasNextInstruction and nextInstruction are used to iterate
   // through a "straight line" execution sequence.
 
@@ -421,16 +442,16 @@ static Instruction *findLocationForEntrySafepoint(Function &F,
   for (cursor = F.getEntryBlock().begin(); hasNextInstruction(cursor);
        cursor = nextInstruction(cursor)) {
 
-    // We need to stop going forward as soon as we see a call that can
-    // grow the stack (i.e. the call target has a non-zero frame
-    // size).
-    if (CallSite(cursor)) {
-      if (IntrinsicInst *II = dyn_cast<IntrinsicInst>(cursor)) {
-        // llvm.assume(...) are not really calls.
-        if (II->getIntrinsicID() == Intrinsic::assume) {
-          continue;
-        }
-      }
+    // We need to ensure a safepoint poll occurs before any 'real' call.  The
+    // easiest way to ensure finite execution between safepoints in the face of
+    // recursive and mutually recursive functions is to enforce that each take
+    // a safepoint.  Additionally, we need to ensure a poll before any call
+    // which can grow the stack by an unbounded amount.  This isn't required
+    // for GC semantics per se, but is a common requirement for languages
+    // which detect stack overflow via guard pages and then throw exceptions.
+    if (auto CS = CallSite(cursor)) {
+      if (doesNotRequireEntrySafepointBefore(CS))
+        continue;
       break;
     }
   }
@@ -438,22 +459,7 @@ static Instruction *findLocationForEntrySafepoint(Function &F,
   assert((hasNextInstruction(cursor) || cursor->isTerminator()) &&
          "either we stopped because of a call, or because of terminator");
 
-  if (cursor->isTerminator()) {
-    return cursor;
-  }
-
-  BasicBlock *BB = cursor->getParent();
-  SplitBlock(BB, cursor, nullptr);
-
-  // Note: SplitBlock modifies the DT.  Simply passing a Pass (which is a
-  // module pass) is not enough.
-  DT.recalculate(F);
-#ifndef NDEBUG
-  // SplitBlock updates the DT
-  DT.verifyDomTree();
-#endif
-
-  return BB->getTerminator();
+  return cursor;
 }
 
 /// Identify the list of call sites which need to be have parseable state
@@ -490,7 +496,7 @@ template <typename T> static void unique_unsorted(std::vector<T> &vec) {
   }
 }
 
-static std::string GCSafepointPollName("gc.safepoint_poll");
+static const char *const GCSafepointPollName = "gc.safepoint_poll";
 
 static bool isGCSafepointPoll(Function &F) {
   return F.getName().equals(GCSafepointPollName);
@@ -498,12 +504,15 @@ static bool isGCSafepointPoll(Function &F) {
 
 /// Returns true if this function should be rewritten to include safepoint
 /// polls and parseable call sites.  The main point of this function is to be
-/// an extension point for custom logic. 
+/// an extension point for custom logic.
 static bool shouldRewriteFunction(Function &F) {
   // TODO: This should check the GCStrategy
   if (F.hasGC()) {
-    const std::string StatepointExampleName("statepoint-example");
-    return StatepointExampleName == F.getGC();
+    const char *FunctionGCName = F.getGC();
+    const StringRef StatepointExampleName("statepoint-example");
+    const StringRef CoreCLRName("coreclr");
+    return (StatepointExampleName == FunctionGCName) ||
+           (CoreCLRName == FunctionGCName);
   } else
     return false;
 }
@@ -514,6 +523,23 @@ static bool enableEntrySafepoints(Function &F) { return !NoEntry; }
 static bool enableBackedgeSafepoints(Function &F) { return !NoBackedge; }
 static bool enableCallSafepoints(Function &F) { return !NoCall; }
 
+// Normalize basic block to make it ready to be target of invoke statepoint.
+// Ensure that 'BB' does not have phi nodes. It may require spliting it.
+static BasicBlock *normalizeForInvokeSafepoint(BasicBlock *BB,
+                                               BasicBlock *InvokeParent) {
+  BasicBlock *ret = BB;
+
+  if (!BB->getUniquePredecessor()) {
+    ret = SplitBlockPredecessors(BB, InvokeParent, "");
+  }
+
+  // Now that 'ret' has unique predecessor we can safely remove all phi nodes
+  // from it
+  FoldSingleEntryPHINodes(ret);
+  assert(!isa<PHINode>(ret->begin()));
+
+  return ret;
+}
 
 bool PlaceSafepoints::runOnFunction(Function &F) {
   if (F.isDeclaration() || F.empty()) {
@@ -525,7 +551,7 @@ bool PlaceSafepoints::runOnFunction(Function &F) {
   if (isGCSafepointPoll(F)) {
     // Given we're inlining this inside of safepoint poll insertion, this
     // doesn't make any sense.  Note that we do make any contained calls
-    // parseable after we inline a poll.  
+    // parseable after we inline a poll.
     return false;
   }
 
@@ -544,10 +570,10 @@ bool PlaceSafepoints::runOnFunction(Function &F) {
   // actually insert parse points yet.  That will be done for all polls and
   // calls in a single pass.
 
-  // Note: With the migration, we need to recompute this for each 'pass'.  Once
-  // we merge these, we'll do it once before the analysis
   DominatorTree DT;
+  DT.recalculate(F);
 
+  SmallVector<Instruction *, 16> PollsNeeded;
   std::vector<CallSite> ParsePointNeeded;
 
   if (enableBackedgeSafepoints(F)) {
@@ -560,23 +586,34 @@ bool PlaceSafepoints::runOnFunction(Function &F) {
     PlaceBackedgeSafepointsImpl *PBS =
       new PlaceBackedgeSafepointsImpl(CanAssumeCallSafepoints);
     FPM.add(PBS);
-    // Note: While the analysis pass itself won't modify the IR, LoopSimplify
-    // (which it depends on) may.  i.e. analysis must be recalculated after run
     FPM.run(F);
 
     // We preserve dominance information when inserting the poll, otherwise
     // we'd have to recalculate this on every insert
     DT.recalculate(F);
 
+    auto &PollLocations = PBS->PollLocations;
+
+    auto OrderByBBName = [](Instruction *a, Instruction *b) {
+      return a->getParent()->getName() < b->getParent()->getName();
+    };
+    // We need the order of list to be stable so that naming ends up stable
+    // when we split edges.  This makes test cases much easier to write.
+    std::sort(PollLocations.begin(), PollLocations.end(), OrderByBBName);
+
+    // We can sometimes end up with duplicate poll locations.  This happens if
+    // a single loop is visited more than once.   The fact this happens seems
+    // wrong, but it does happen for the split-backedge.ll test case.
+    PollLocations.erase(std::unique(PollLocations.begin(),
+                                    PollLocations.end()),
+                        PollLocations.end());
+
     // Insert a poll at each point the analysis pass identified
-    for (size_t i = 0; i < PBS->PollLocations.size(); i++) {
+    // The poll location must be the terminator of a loop latch block.
+    for (TerminatorInst *Term : PollLocations) {
       // We are inserting a poll, the function is modified
       modified = true;
 
-      // The poll location must be the terminator of a loop latch block.
-      TerminatorInst *Term = PBS->PollLocations[i];
-
-      std::vector<CallSite> ParsePoints;
       if (SplitBackedge) {
         // Split the backedge of the loop and insert the poll within that new
         // basic block.  This creates a loop with two latches per original
@@ -587,8 +624,8 @@ bool PlaceSafepoints::runOnFunction(Function &F) {
         // Since this is a latch, at least one of the successors must dominate
         // it. Its possible that we have a) duplicate edges to the same header
         // and b) edges to distinct loop headers.  We need to insert pools on
-        // each. (Note: This still relies on LoopSimplify.)
-        DenseSet<BasicBlock *> Headers;
+        // each.
+        SetVector<BasicBlock *> Headers;
         for (unsigned i = 0; i < Term->getNumSuccessors(); i++) {
           BasicBlock *Succ = Term->getSuccessor(i);
           if (DT.dominates(Succ, Term->getParent())) {
@@ -600,46 +637,45 @@ bool PlaceSafepoints::runOnFunction(Function &F) {
         // The split loop structure here is so that we only need to recalculate
         // the dominator tree once.  Alternatively, we could just keep it up to
         // date and use a more natural merged loop.
-        DenseSet<BasicBlock *> SplitBackedges;
+        SetVector<BasicBlock *> SplitBackedges;
         for (BasicBlock *Header : Headers) {
-          BasicBlock *NewBB = SplitEdge(Term->getParent(), Header, nullptr);
-          SplitBackedges.insert(NewBB);
-        }
-        DT.recalculate(F);
-        for (BasicBlock *NewBB : SplitBackedges) {
-          InsertSafepointPoll(DT, NewBB->getTerminator(), ParsePoints);
+          BasicBlock *NewBB = SplitEdge(Term->getParent(), Header, &DT);
+          PollsNeeded.push_back(NewBB->getTerminator());
           NumBackedgeSafepoints++;
         }
-
       } else {
         // Split the latch block itself, right before the terminator.
-        InsertSafepointPoll(DT, Term, ParsePoints);
+        PollsNeeded.push_back(Term);
         NumBackedgeSafepoints++;
       }
-
-      // Record the parse points for later use
-      ParsePointNeeded.insert(ParsePointNeeded.end(), ParsePoints.begin(),
-                              ParsePoints.end());
     }
   }
 
   if (enableEntrySafepoints(F)) {
-    DT.recalculate(F);
-    Instruction *term = findLocationForEntrySafepoint(F, DT);
-    if (!term) {
+    Instruction *Location = findLocationForEntrySafepoint(F, DT);
+    if (!Location) {
       // policy choice not to insert?
     } else {
-      std::vector<CallSite> RuntimeCalls;
-      InsertSafepointPoll(DT, term, RuntimeCalls);
+      PollsNeeded.push_back(Location);
       modified = true;
       NumEntrySafepoints++;
-      ParsePointNeeded.insert(ParsePointNeeded.end(), RuntimeCalls.begin(),
-                              RuntimeCalls.end());
     }
   }
 
+  // Now that we've identified all the needed safepoint poll locations, insert
+  // safepoint polls themselves.
+  for (Instruction *PollLocation : PollsNeeded) {
+    std::vector<CallSite> RuntimeCalls;
+    InsertSafepointPoll(PollLocation, RuntimeCalls);
+    ParsePointNeeded.insert(ParsePointNeeded.end(), RuntimeCalls.begin(),
+                            RuntimeCalls.end());
+  }
+  PollsNeeded.clear(); // make sure we don't accidentally use
+  // The dominator tree has been invalidated by the inlining performed in the
+  // above loop.  TODO: Teach the inliner how to update the dom tree?
+  DT.recalculate(F);
+
   if (enableCallSafepoints(F)) {
-    DT.recalculate(F);
     std::vector<CallSite> Calls;
     findCallSafepoints(F, Calls);
     NumCallSafepoints += Calls.size();
@@ -654,7 +690,6 @@ bool PlaceSafepoints::runOnFunction(Function &F) {
   unique_unsorted(ParsePointNeeded);
 
   // Any parse point (no matter what source) will be handled here
-  DT.recalculate(F); // Needed?
 
   // We're about to start modifying the function
   if (!ParsePointNeeded.empty())
@@ -667,6 +702,17 @@ bool PlaceSafepoints::runOnFunction(Function &F) {
   Results.reserve(ParsePointNeeded.size());
   for (size_t i = 0; i < ParsePointNeeded.size(); i++) {
     CallSite &CS = ParsePointNeeded[i];
+
+    // For invoke statepoints we need to remove all phi nodes at the normal
+    // destination block.
+    // Reason for this is that we can place gc_result only after last phi node
+    // in basic block. We will get malformed code after RAUW for the
+    // gc_result if one of this phi nodes uses result from the invoke.
+    if (InvokeInst *Invoke = dyn_cast<InvokeInst>(CS.getInstruction())) {
+      normalizeForInvokeSafepoint(Invoke->getNormalDest(),
+                                  Invoke->getParent());
+    }
+
     Value *GCResult = ReplaceWithStatepoint(CS, nullptr);
     Results.push_back(GCResult);
   }
@@ -677,20 +723,8 @@ bool PlaceSafepoints::runOnFunction(Function &F) {
     CallSite &CS = ParsePointNeeded[i];
     Value *GCResult = Results[i];
     if (GCResult) {
-      // In case if we inserted result in a different basic block than the
-      // original safepoint (this can happen for invokes). We need to be sure
-      // that
-      // original result value was not used in any of the phi nodes at the
-      // beginning of basic block with gc result. Because we know that all such
-      // blocks will have single predecessor we can safely assume that all phi
-      // nodes have single entry (because of normalizeBBForInvokeSafepoint).
-      // Just remove them all here.
-      if (CS.isInvoke()) {
-        FoldSingleEntryPHINodes(cast<Instruction>(GCResult)->getParent(),
-                                nullptr);
-        assert(
-            !isa<PHINode>(cast<Instruction>(GCResult)->getParent()->begin()));
-      }
+      // Can not RAUW for the invoke gc result in case of phi nodes preset.
+      assert(CS.isCall() || !isa<PHINode>(cast<Instruction>(GCResult)->getParent()->begin()));
 
       // Replace all uses with the new call
       CS.getInstruction()->replaceAllUsesWith(GCResult);
@@ -706,13 +740,16 @@ bool PlaceSafepoints::runOnFunction(Function &F) {
 char PlaceBackedgeSafepointsImpl::ID = 0;
 char PlaceSafepoints::ID = 0;
 
-ModulePass *llvm::createPlaceSafepointsPass() { return new PlaceSafepoints(); }
+FunctionPass *llvm::createPlaceSafepointsPass() {
+  return new PlaceSafepoints();
+}
 
 INITIALIZE_PASS_BEGIN(PlaceBackedgeSafepointsImpl,
                       "place-backedge-safepoints-impl",
                       "Place Backedge Safepoints", false, false)
 INITIALIZE_PASS_DEPENDENCY(ScalarEvolution)
-INITIALIZE_PASS_DEPENDENCY(LoopSimplify)
+INITIALIZE_PASS_DEPENDENCY(DominatorTreeWrapperPass)
+INITIALIZE_PASS_DEPENDENCY(LoopInfoWrapperPass)
 INITIALIZE_PASS_END(PlaceBackedgeSafepointsImpl,
                     "place-backedge-safepoints-impl",
                     "Place Backedge Safepoints", false, false)
@@ -748,43 +785,39 @@ static bool isGCLeafFunction(const CallSite &CS) {
 }
 
 static void
-InsertSafepointPoll(DominatorTree &DT, Instruction *term,
+InsertSafepointPoll(Instruction *InsertBefore,
                     std::vector<CallSite> &ParsePointsNeeded /*rval*/) {
-  Module *M = term->getParent()->getParent()->getParent();
-  assert(M);
+  BasicBlock *OrigBB = InsertBefore->getParent();
+  Module *M = InsertBefore->getModule();
+  assert(M && "must be part of a module");
 
   // Inline the safepoint poll implementation - this will get all the branch,
   // control flow, etc..  Most importantly, it will introduce the actual slow
   // path call - where we need to insert a safepoint (parsepoint).
-  FunctionType *ftype =
-      FunctionType::get(Type::getVoidTy(M->getContext()), false);
-  assert(ftype && "null?");
-  // Note: This cast can fail if there's a function of the same name with a
-  // different type inserted previously
-  Function *F =
-      dyn_cast<Function>(M->getOrInsertFunction("gc.safepoint_poll", ftype));
-  assert(F && "void @gc.safepoint_poll() must be defined");
+
+  auto *F = M->getFunction(GCSafepointPollName);
+  assert(F->getType()->getElementType() ==
+         FunctionType::get(Type::getVoidTy(M->getContext()), false) &&
+         "gc.safepoint_poll declared with wrong type");
   assert(!F->empty() && "gc.safepoint_poll must be a non-empty function");
-  CallInst *poll = CallInst::Create(F, "", term);
+  CallInst *PollCall = CallInst::Create(F, "", InsertBefore);
 
   // Record some information about the call site we're replacing
-  BasicBlock *OrigBB = term->getParent();
-  BasicBlock::iterator before(poll), after(poll);
+  BasicBlock::iterator before(PollCall), after(PollCall);
   bool isBegin(false);
-  if (before == term->getParent()->begin()) {
+  if (before == OrigBB->begin()) {
     isBegin = true;
   } else {
     before--;
   }
   after++;
-  assert(after != poll->getParent()->end() && "must have successor");
-  assert(DT.dominates(before, after) && "trivially true");
+  assert(after != OrigBB->end() && "must have successor");
 
   // do the actual inlining
   InlineFunctionInfo IFI;
-  bool inlineStatus = InlineFunction(poll, IFI);
-  assert(inlineStatus && "inline must succeed");
-  (void)inlineStatus; // suppress warning in release-asserts
+  bool InlineStatus = InlineFunction(PollCall, IFI);
+  assert(InlineStatus && "inline must succeed");
+  (void)InlineStatus; // suppress warning in release-asserts
 
   // Check post conditions
   assert(IFI.StaticAllocas.empty() && "can't have allocs");
@@ -807,13 +840,6 @@ InsertSafepointPoll(DominatorTree &DT, Instruction *term,
          "malformed poll function");
 
   scanInlinedCode(&*(start), &*(after), calls, BBs);
-
-  // Recompute since we've invalidated cached data.  Conceptually we
-  // shouldn't need to do this, but implementation wise we appear to.  Needed
-  // so we can insert safepoints correctly.
-  // TODO: update more cheaply
-  DT.recalculate(*after->getParent()->getParent());
-
   assert(!calls.empty() && "slow path not found for safepoint poll");
 
   // Record the fact we need a parsable state at the runtime call contained in
@@ -835,37 +861,13 @@ InsertSafepointPoll(DominatorTree &DT, Instruction *term,
   assert(ParsePointsNeeded.size() <= calls.size());
 }
 
-// Normalize basic block to make it ready to be target of invoke statepoint.
-// It means spliting it to have single predecessor. Return newly created BB
-// ready to be successor of invoke statepoint.
-static BasicBlock *normalizeBBForInvokeSafepoint(BasicBlock *BB,
-                                                 BasicBlock *InvokeParent) {
-  BasicBlock *ret = BB;
-
-  if (!BB->getUniquePredecessor()) {
-    ret = SplitBlockPredecessors(BB, InvokeParent, "");
-  }
-
-  // Another requirement for such basic blocks is to not have any phi nodes.
-  // Since we just ensured that new BB will have single predecessor,
-  // all phi nodes in it will have one value. Here it would be naturall place
-  // to
-  // remove them all. But we can not do this because we are risking to remove
-  // one of the values stored in liveset of another statepoint. We will do it
-  // later after placing all safepoints.
-
-  return ret;
-}
-
 /// Replaces the given call site (Call or Invoke) with a gc.statepoint
 /// intrinsic with an empty deoptimization arguments list.  This does
 /// NOT do explicit relocation for GC support.
 static Value *ReplaceWithStatepoint(const CallSite &CS, /* to replace */
                                     Pass *P) {
-  BasicBlock *BB = CS.getInstruction()->getParent();
-  Function *F = BB->getParent();
-  Module *M = F->getParent();
-  assert(M && "must be set");
+  assert(CS.getInstruction()->getParent()->getParent()->getParent() &&
+         "must be set");
 
   // TODO: technically, a pass is not allowed to get functions from within a
   // function pass since it might trigger a new function addition.  Refactor
@@ -876,111 +878,109 @@ static Value *ReplaceWithStatepoint(const CallSite &CS, /* to replace */
   // immediately before the previous instruction under the assumption that all
   // arguments will be available here.  We can't insert afterwards since we may
   // be replacing a terminator.
-  Instruction *insertBefore = CS.getInstruction();
-  IRBuilder<> Builder(insertBefore);
+  IRBuilder<> Builder(CS.getInstruction());
 
   // Note: The gc args are not filled in at this time, that's handled by
   // RewriteStatepointsForGC (which is currently under review).
 
   // Create the statepoint given all the arguments
-  Instruction *token = nullptr;
-  AttributeSet return_attributes;
+  Instruction *Token = nullptr;
+
+  uint64_t ID;
+  uint32_t NumPatchBytes;
+
+  AttributeSet OriginalAttrs = CS.getAttributes();
+  Attribute AttrID =
+      OriginalAttrs.getAttribute(AttributeSet::FunctionIndex, "statepoint-id");
+  Attribute AttrNumPatchBytes = OriginalAttrs.getAttribute(
+      AttributeSet::FunctionIndex, "statepoint-num-patch-bytes");
+
+  AttrBuilder AttrsToRemove;
+  bool HasID = AttrID.isStringAttribute() &&
+               !AttrID.getValueAsString().getAsInteger(10, ID);
+
+  if (HasID)
+    AttrsToRemove.addAttribute("statepoint-id");
+  else
+    ID = 0xABCDEF00;
+
+  bool HasNumPatchBytes =
+      AttrNumPatchBytes.isStringAttribute() &&
+      !AttrNumPatchBytes.getValueAsString().getAsInteger(10, NumPatchBytes);
+
+  if (HasNumPatchBytes)
+    AttrsToRemove.addAttribute("statepoint-num-patch-bytes");
+  else
+    NumPatchBytes = 0;
+
+  OriginalAttrs = OriginalAttrs.removeAttributes(
+      CS.getInstruction()->getContext(), AttributeSet::FunctionIndex,
+      AttrsToRemove);
+
   if (CS.isCall()) {
-    CallInst *toReplace = cast<CallInst>(CS.getInstruction());
-    CallInst *Call = Builder.CreateGCStatepoint(
-        CS.getCalledValue(), makeArrayRef(CS.arg_begin(), CS.arg_end()), None,
-        None, "safepoint_token");
-    Call->setTailCall(toReplace->isTailCall());
-    Call->setCallingConv(toReplace->getCallingConv());
-
-    // Before we have to worry about GC semantics, all attributes are legal
-    AttributeSet new_attrs = toReplace->getAttributes();
-    // In case if we can handle this set of sttributes - set up function attrs
-    // directly on statepoint and return attrs later for gc_result intrinsic.
-    Call->setAttributes(new_attrs.getFnAttributes());
-    return_attributes = new_attrs.getRetAttributes();
-    // TODO: handle param attributes
-
-    token = Call;
+    CallInst *ToReplace = cast<CallInst>(CS.getInstruction());
+    CallInst *Call = Builder.CreateGCStatepointCall(
+        ID, NumPatchBytes, CS.getCalledValue(),
+        makeArrayRef(CS.arg_begin(), CS.arg_end()), None, None,
+        "safepoint_token");
+    Call->setTailCall(ToReplace->isTailCall());
+    Call->setCallingConv(ToReplace->getCallingConv());
 
-    // Put the following gc_result and gc_relocate calls immediately after the
-    // the old call (which we're about to delete)
-    BasicBlock::iterator next(toReplace);
-    assert(BB->end() != next && "not a terminator, must have next");
-    next++;
-    Instruction *IP = &*(next);
-    Builder.SetInsertPoint(IP);
-    Builder.SetCurrentDebugLocation(IP->getDebugLoc());
+    // In case if we can handle this set of attributes - set up function
+    // attributes directly on statepoint and return attributes later for
+    // gc_result intrinsic.
+    Call->setAttributes(OriginalAttrs.getFnAttributes());
+
+    Token = Call;
 
+    // Put the following gc_result and gc_relocate calls immediately after the
+    // the old call (which we're about to delete).
+    assert(ToReplace->getNextNode() && "not a terminator, must have next");
+    Builder.SetInsertPoint(ToReplace->getNextNode());
+    Builder.SetCurrentDebugLocation(ToReplace->getNextNode()->getDebugLoc());
   } else if (CS.isInvoke()) {
-    // TODO: make CreateGCStatepoint return an Instruction that we can cast to a
-    // Call or Invoke, instead of doing this junk here.
-
-    // Fill in the one generic type'd argument (the function is also
-    // vararg)
-    std::vector<Type *> argTypes;
-    argTypes.push_back(CS.getCalledValue()->getType());
-
-    Function *gc_statepoint_decl = Intrinsic::getDeclaration(
-        M, Intrinsic::experimental_gc_statepoint, argTypes);
-
-    // First, create the statepoint (with all live ptrs as arguments).
-    std::vector<llvm::Value *> args;
-    // target, #call args, unused, ... call parameters, #deopt args, ... deopt
-    // parameters, ... gc parameters
-    Value *Target = CS.getCalledValue();
-    args.push_back(Target);
-    int callArgSize = CS.arg_size();
-    // #call args
-    args.push_back(Builder.getInt32(callArgSize));
-    // unused
-    args.push_back(Builder.getInt32(0));
-    // call parameters
-    args.insert(args.end(), CS.arg_begin(), CS.arg_end());
-    // #deopt args: 0
-    args.push_back(Builder.getInt32(0));
-
-    InvokeInst *toReplace = cast<InvokeInst>(CS.getInstruction());
+    InvokeInst *ToReplace = cast<InvokeInst>(CS.getInstruction());
 
     // Insert the new invoke into the old block.  We'll remove the old one in a
     // moment at which point this will become the new terminator for the
     // original block.
-    InvokeInst *invoke = InvokeInst::Create(
-        gc_statepoint_decl, toReplace->getNormalDest(),
-        toReplace->getUnwindDest(), args, "", toReplace->getParent());
-    invoke->setCallingConv(toReplace->getCallingConv());
+    Builder.SetInsertPoint(ToReplace->getParent());
+    InvokeInst *Invoke = Builder.CreateGCStatepointInvoke(
+        ID, NumPatchBytes, CS.getCalledValue(), ToReplace->getNormalDest(),
+        ToReplace->getUnwindDest(), makeArrayRef(CS.arg_begin(), CS.arg_end()),
+        None, None, "safepoint_token");
+
+    Invoke->setCallingConv(ToReplace->getCallingConv());
 
-    // Currently we will fail on parameter attributes and on certain
-    // function attributes.
-    AttributeSet new_attrs = toReplace->getAttributes();
-    // In case if we can handle this set of sttributes - set up function attrs
-    // directly on statepoint and return attrs later for gc_result intrinsic.
-    invoke->setAttributes(new_attrs.getFnAttributes());
-    return_attributes = new_attrs.getRetAttributes();
+    // In case if we can handle this set of attributes - set up function
+    // attributes directly on statepoint and return attributes later for
+    // gc_result intrinsic.
+    Invoke->setAttributes(OriginalAttrs.getFnAttributes());
 
-    token = invoke;
+    Token = Invoke;
 
     // We'll insert the gc.result into the normal block
-    BasicBlock *normalDest = normalizeBBForInvokeSafepoint(
-        toReplace->getNormalDest(), invoke->getParent());
-    Instruction *IP = &*(normalDest->getFirstInsertionPt());
+    BasicBlock *NormalDest = ToReplace->getNormalDest();
+    // Can not insert gc.result in case of phi nodes preset.
+    // Should have removed this cases prior to runnning this function
+    assert(!isa<PHINode>(NormalDest->begin()));
+    Instruction *IP = &*(NormalDest->getFirstInsertionPt());
     Builder.SetInsertPoint(IP);
   } else {
     llvm_unreachable("unexpect type of CallSite");
   }
-  assert(token);
+  assert(Token);
 
   // Handle the return value of the original call - update all uses to use a
   // gc_result hanging off the statepoint node we just inserted
 
   // Only add the gc_result iff there is actually a used result
   if (!CS.getType()->isVoidTy() && !CS.getInstruction()->use_empty()) {
-    std::string takenName =
-      CS.getInstruction()->hasName() ? CS.getInstruction()->getName() : "";
-    CallInst *gc_result =
-        Builder.CreateGCResult(token, CS.getType(), takenName);
-    gc_result->setAttributes(return_attributes);
-    return gc_result;
+    std::string TakenName =
+        CS.getInstruction()->hasName() ? CS.getInstruction()->getName() : "";
+    CallInst *GCResult = Builder.CreateGCResult(Token, CS.getType(), TakenName);
+    GCResult->setAttributes(OriginalAttrs.getRetAttributes());
+    return GCResult;
   } else {
     // No return value for the call.
     return nullptr;
diff --git a/lib/Transforms/Scalar/Reassociate.cpp b/lib/Transforms/Scalar/Reassociate.cpp
index 307cc73d991..d1acf785d07 100644
--- a/lib/Transforms/Scalar/Reassociate.cpp
+++ b/lib/Transforms/Scalar/Reassociate.cpp
@@ -59,7 +59,7 @@ namespace {
 }
 
 #ifndef NDEBUG
-/// PrintOps - Print out the expression identified in the Ops list.
+/// Print out the expression identified in the Ops list.
 ///
 static void PrintOps(Instruction *I, const SmallVectorImpl<ValueEntry> &Ops) {
   Module *M = I->getParent()->getParent()->getParent();
@@ -233,8 +233,8 @@ INITIALIZE_PASS(Reassociate, "reassociate",
 // Public interface to the Reassociate pass
 FunctionPass *llvm::createReassociatePass() { return new Reassociate(); }
 
-/// isReassociableOp - Return true if V is an instruction of the specified
-/// opcode and if it only has one use.
+/// Return true if V is an instruction of the specified opcode and if it
+/// only has one use.
 static BinaryOperator *isReassociableOp(Value *V, unsigned Opcode) {
   if (V->hasOneUse() && isa<Instruction>(V) &&
       cast<Instruction>(V)->getOpcode() == Opcode &&
@@ -382,8 +382,7 @@ static BinaryOperator *CreateNeg(Value *S1, const Twine &Name,
   }
 }
 
-/// LowerNegateToMultiply - Replace 0-X with X*-1.
-///
+/// Replace 0-X with X*-1.
 static BinaryOperator *LowerNegateToMultiply(Instruction *Neg) {
   Type *Ty = Neg->getType();
   Constant *NegOne = Ty->isIntOrIntVectorTy() ?
@@ -397,8 +396,8 @@ static BinaryOperator *LowerNegateToMultiply(Instruction *Neg) {
   return Res;
 }
 
-/// CarmichaelShift - Returns k such that lambda(2^Bitwidth) = 2^k, where lambda
-/// is the Carmichael function. This means that x^(2^k) === 1 mod 2^Bitwidth for
+/// Returns k such that lambda(2^Bitwidth) = 2^k, where lambda is the Carmichael
+/// function. This means that x^(2^k) === 1 mod 2^Bitwidth for
 /// every odd x, i.e. x^(2^k) = 1 for every odd x in Bitwidth-bit arithmetic.
 /// Note that 0 <= k < Bitwidth, and if Bitwidth > 3 then x^(2^k) = 0 for every
 /// even x in Bitwidth-bit arithmetic.
@@ -408,7 +407,7 @@ static unsigned CarmichaelShift(unsigned Bitwidth) {
   return Bitwidth - 2;
 }
 
-/// IncorporateWeight - Add the extra weight 'RHS' to the existing weight 'LHS',
+/// Add the extra weight 'RHS' to the existing weight 'LHS',
 /// reducing the combined weight using any special properties of the operation.
 /// The existing weight LHS represents the computation X op X op ... op X where
 /// X occurs LHS times.  The combined weight represents  X op X op ... op X with
@@ -490,7 +489,7 @@ static void IncorporateWeight(APInt &LHS, const APInt &RHS, unsigned Opcode) {
 
 typedef std::pair<Value*, APInt> RepeatedValue;
 
-/// LinearizeExprTree - Given an associative binary expression, return the leaf
+/// Given an associative binary expression, return the leaf
 /// nodes in Ops along with their weights (how many times the leaf occurs).  The
 /// original expression is the same as
 ///   (Ops[0].first op Ops[0].first op ... Ops[0].first)  <- Ops[0].second times
@@ -734,14 +733,14 @@ static bool LinearizeExprTree(BinaryOperator *I,
   if (Ops.empty()) {
     Constant *Identity = ConstantExpr::getBinOpIdentity(Opcode, I->getType());
     assert(Identity && "Associative operation without identity!");
-    Ops.push_back(std::make_pair(Identity, APInt(Bitwidth, 1)));
+    Ops.emplace_back(Identity, APInt(Bitwidth, 1));
   }
 
   return Changed;
 }
 
-// RewriteExprTree - Now that the operands for this expression tree are
-// linearized and optimized, emit them in-order.
+/// Now that the operands for this expression tree are
+/// linearized and optimized, emit them in-order.
 void Reassociate::RewriteExprTree(BinaryOperator *I,
                                   SmallVectorImpl<ValueEntry> &Ops) {
   assert(Ops.size() > 1 && "Single values should be used directly!");
@@ -910,7 +909,7 @@ void Reassociate::RewriteExprTree(BinaryOperator *I,
     RedoInsts.insert(NodesToRewrite[i]);
 }
 
-/// NegateValue - Insert instructions before the instruction pointed to by BI,
+/// Insert instructions before the instruction pointed to by BI,
 /// that computes the negative version of the value specified.  The negative
 /// version of the value is returned, and BI is left pointing at the instruction
 /// that should be processed next by the reassociation pass.
@@ -937,6 +936,10 @@ static Value *NegateValue(Value *V, Instruction *BI) {
     // Push the negates through the add.
     I->setOperand(0, NegateValue(I->getOperand(0), BI));
     I->setOperand(1, NegateValue(I->getOperand(1), BI));
+    if (I->getOpcode() == Instruction::Add) {
+      I->setHasNoUnsignedWrap(false);
+      I->setHasNoSignedWrap(false);
+    }
 
     // We must move the add instruction here, because the neg instructions do
     // not dominate the old add instruction in general.  By moving it, we are
@@ -977,6 +980,12 @@ static Value *NegateValue(Value *V, Instruction *BI) {
       InsertPt = TheNeg->getParent()->getParent()->getEntryBlock().begin();
     }
     TheNeg->moveBefore(InsertPt);
+    if (TheNeg->getOpcode() == Instruction::Sub) {
+      TheNeg->setHasNoUnsignedWrap(false);
+      TheNeg->setHasNoSignedWrap(false);
+    } else {
+      TheNeg->andIRFlags(BI);
+    }
     return TheNeg;
   }
 
@@ -985,8 +994,7 @@ static Value *NegateValue(Value *V, Instruction *BI) {
   return CreateNeg(V, V->getName() + ".neg", BI, BI);
 }
 
-/// ShouldBreakUpSubtract - Return true if we should break up this subtract of
-/// X-Y into (X + -Y).
+/// Return true if we should break up this subtract of X-Y into (X + -Y).
 static bool ShouldBreakUpSubtract(Instruction *Sub) {
   // If this is a negation, we can't split it up!
   if (BinaryOperator::isNeg(Sub) || BinaryOperator::isFNeg(Sub))
@@ -1015,9 +1023,8 @@ static bool ShouldBreakUpSubtract(Instruction *Sub) {
   return false;
 }
 
-/// BreakUpSubtract - If we have (X-Y), and if either X is an add, or if this is
-/// only used by an add, transform this into (X+(0-Y)) to promote better
-/// reassociation.
+/// If we have (X-Y), and if either X is an add, or if this is only used by an
+/// add, transform this into (X+(0-Y)) to promote better reassociation.
 static BinaryOperator *BreakUpSubtract(Instruction *Sub) {
   // Convert a subtract into an add and a neg instruction. This allows sub
   // instructions to be commuted with other add instructions.
@@ -1039,9 +1046,8 @@ static BinaryOperator *BreakUpSubtract(Instruction *Sub) {
   return New;
 }
 
-/// ConvertShiftToMul - If this is a shift of a reassociable multiply or is used
-/// by one, change this into a multiply by a constant to assist with further
-/// reassociation.
+/// If this is a shift of a reassociable multiply or is used by one, change
+/// this into a multiply by a constant to assist with further reassociation.
 static BinaryOperator *ConvertShiftToMul(Instruction *Shl) {
   Constant *MulCst = ConstantInt::get(Shl->getType(), 1);
   MulCst = ConstantExpr::getShl(MulCst, cast<Constant>(Shl->getOperand(1)));
@@ -1066,10 +1072,9 @@ static BinaryOperator *ConvertShiftToMul(Instruction *Shl) {
   return Mul;
 }
 
-/// FindInOperandList - Scan backwards and forwards among values with the same
-/// rank as element i to see if X exists.  If X does not exist, return i.  This
-/// is useful when scanning for 'x' when we see '-x' because they both get the
-/// same rank.
+/// Scan backwards and forwards among values with the same rank as element i
+/// to see if X exists.  If X does not exist, return i.  This is useful when
+/// scanning for 'x' when we see '-x' because they both get the same rank.
 static unsigned FindInOperandList(SmallVectorImpl<ValueEntry> &Ops, unsigned i,
                                   Value *X) {
   unsigned XRank = Ops[i].Rank;
@@ -1094,7 +1099,7 @@ static unsigned FindInOperandList(SmallVectorImpl<ValueEntry> &Ops, unsigned i,
   return i;
 }
 
-/// EmitAddTreeOfValues - Emit a tree of add instructions, summing Ops together
+/// Emit a tree of add instructions, summing Ops together
 /// and returning the result.  Insert the tree before I.
 static Value *EmitAddTreeOfValues(Instruction *I,
                                   SmallVectorImpl<WeakVH> &Ops){
@@ -1106,8 +1111,8 @@ static Value *EmitAddTreeOfValues(Instruction *I,
   return CreateAdd(V2, V1, "tmp", I, I);
 }
 
-/// RemoveFactorFromExpression - If V is an expression tree that is a
-/// multiplication sequence, and if this sequence contains a multiply by Factor,
+/// If V is an expression tree that is a multiplication sequence,
+/// and if this sequence contains a multiply by Factor,
 /// remove Factor from the tree and return the new tree.
 Value *Reassociate::RemoveFactorFromExpression(Value *V, Value *Factor) {
   BinaryOperator *BO = isReassociableOp(V, Instruction::Mul, Instruction::FMul);
@@ -1179,8 +1184,8 @@ Value *Reassociate::RemoveFactorFromExpression(Value *V, Value *Factor) {
   return V;
 }
 
-/// FindSingleUseMultiplyFactors - If V is a single-use multiply, recursively
-/// add its operands as factors, otherwise add V to the list of factors.
+/// If V is a single-use multiply, recursively add its operands as factors,
+/// otherwise add V to the list of factors.
 ///
 /// Ops is the top-level list of add operands we're trying to factor.
 static void FindSingleUseMultiplyFactors(Value *V,
@@ -1197,10 +1202,9 @@ static void FindSingleUseMultiplyFactors(Value *V,
   FindSingleUseMultiplyFactors(BO->getOperand(0), Factors, Ops);
 }
 
-/// OptimizeAndOrXor - Optimize a series of operands to an 'and', 'or', or 'xor'
-/// instruction.  This optimizes based on identities.  If it can be reduced to
-/// a single Value, it is returned, otherwise the Ops list is mutated as
-/// necessary.
+/// Optimize a series of operands to an 'and', 'or', or 'xor' instruction.
+/// This optimizes based on identities.  If it can be reduced to a single Value,
+/// it is returned, otherwise the Ops list is mutated as necessary.
 static Value *OptimizeAndOrXor(unsigned Opcode,
                                SmallVectorImpl<ValueEntry> &Ops) {
   // Scan the operand lists looking for X and ~X pairs, along with X,X pairs.
@@ -1490,7 +1494,7 @@ Value *Reassociate::OptimizeXor(Instruction *I,
   return nullptr;
 }
 
-/// OptimizeAdd - Optimize a series of operands to an 'add' instruction.  This
+/// Optimize a series of operands to an 'add' instruction.  This
 /// optimizes based on identities.  If it can be reduced to a single Value, it
 /// is returned, otherwise the Ops list is mutated as necessary.
 Value *Reassociate::OptimizeAdd(Instruction *I,
@@ -1943,8 +1947,7 @@ Value *Reassociate::OptimizeExpression(BinaryOperator *I,
   return nullptr;
 }
 
-/// EraseInst - Zap the given instruction, adding interesting operands to the
-/// work list.
+/// Zap the given instruction, adding interesting operands to the work list.
 void Reassociate::EraseInst(Instruction *I) {
   assert(isInstructionTriviallyDead(I) && "Trivially dead instructions only!");
   SmallVector<Value*, 8> Ops(I->op_begin(), I->op_end());
@@ -1973,38 +1976,35 @@ Instruction *Reassociate::canonicalizeNegConstExpr(Instruction *I) {
   if (!I->hasOneUse() || I->getType()->isVectorTy())
     return nullptr;
 
-  // Must be a mul, fmul, or fdiv instruction.
+  // Must be a fmul or fdiv instruction.
   unsigned Opcode = I->getOpcode();
-  if (Opcode != Instruction::Mul && Opcode != Instruction::FMul &&
-      Opcode != Instruction::FDiv)
+  if (Opcode != Instruction::FMul && Opcode != Instruction::FDiv)
+    return nullptr;
+
+  auto *C0 = dyn_cast<ConstantFP>(I->getOperand(0));
+  auto *C1 = dyn_cast<ConstantFP>(I->getOperand(1));
+
+  // Both operands are constant, let it get constant folded away.
+  if (C0 && C1)
     return nullptr;
 
-  // Must have at least one constant operand.
-  Constant *C0 = dyn_cast<Constant>(I->getOperand(0));
-  Constant *C1 = dyn_cast<Constant>(I->getOperand(1));
-  if (!C0 && !C1)
+  ConstantFP *CF = C0 ? C0 : C1;
+
+  // Must have one constant operand.
+  if (!CF)
     return nullptr;
 
-  // Must be a negative ConstantInt or ConstantFP.
-  Constant *C = C0 ? C0 : C1;
-  unsigned ConstIdx = C0 ? 0 : 1;
-  if (auto *CI = dyn_cast<ConstantInt>(C)) {
-    if (!CI->isNegative() || CI->isMinValue(true))
-      return nullptr;
-  } else if (auto *CF = dyn_cast<ConstantFP>(C)) {
-    if (!CF->isNegative())
-      return nullptr;
-  } else
+  // Must be a negative ConstantFP.
+  if (!CF->isNegative())
     return nullptr;
 
   // User must be a binary operator with one or more uses.
   Instruction *User = I->user_back();
-  if (!isa<BinaryOperator>(User) || !User->getNumUses())
+  if (!isa<BinaryOperator>(User) || !User->hasNUsesOrMore(1))
     return nullptr;
 
   unsigned UserOpcode = User->getOpcode();
-  if (UserOpcode != Instruction::Add && UserOpcode != Instruction::FAdd &&
-      UserOpcode != Instruction::Sub && UserOpcode != Instruction::FSub)
+  if (UserOpcode != Instruction::FAdd && UserOpcode != Instruction::FSub)
     return nullptr;
 
   // Subtraction is not commutative. Explicitly, the following transform is
@@ -2013,14 +2013,9 @@ Instruction *Reassociate::canonicalizeNegConstExpr(Instruction *I) {
     return nullptr;
 
   // Change the sign of the constant.
-  if (ConstantInt *CI = dyn_cast<ConstantInt>(C))
-    I->setOperand(ConstIdx, ConstantInt::get(CI->getContext(), -CI->getValue()));
-  else {
-    ConstantFP *CF = cast<ConstantFP>(C);
-    APFloat Val = CF->getValueAPF();
-    Val.changeSign();
-    I->setOperand(ConstIdx, ConstantFP::get(CF->getContext(), Val));
-  }
+  APFloat Val = CF->getValueAPF();
+  Val.changeSign();
+  I->setOperand(C0 ? 0 : 1, ConstantFP::get(CF->getContext(), Val));
 
   // Canonicalize I to RHS to simplify the next bit of logic. E.g.,
   // ((-Const*y) + x) -> (x + (-Const*y)).
@@ -2030,15 +2025,9 @@ Instruction *Reassociate::canonicalizeNegConstExpr(Instruction *I) {
   Value *Op0 = User->getOperand(0);
   Value *Op1 = User->getOperand(1);
   BinaryOperator *NI;
-  switch(UserOpcode) {
+  switch (UserOpcode) {
   default:
     llvm_unreachable("Unexpected Opcode!");
-  case Instruction::Add:
-    NI = BinaryOperator::CreateSub(Op0, Op1);
-    break;
-  case Instruction::Sub:
-    NI = BinaryOperator::CreateAdd(Op0, Op1);
-    break;
   case Instruction::FAdd:
     NI = BinaryOperator::CreateFSub(Op0, Op1);
     NI->setFastMathFlags(cast<FPMathOperator>(User)->getFastMathFlags());
@@ -2058,7 +2047,7 @@ Instruction *Reassociate::canonicalizeNegConstExpr(Instruction *I) {
   return NI;
 }
 
-/// OptimizeInst - Inspect and optimize the given instruction. Note that erasing
+/// Inspect and optimize the given instruction. Note that erasing
 /// instructions is not allowed.
 void Reassociate::OptimizeInst(Instruction *I) {
   // Only consider operations that we understand.
@@ -2191,7 +2180,7 @@ void Reassociate::ReassociateExpression(BinaryOperator *I) {
   // the vector.
   std::stable_sort(Ops.begin(), Ops.end());
 
-  // OptimizeExpression - Now that we have the expression tree in a convenient
+  // Now that we have the expression tree in a convenient
   // sorted form, optimize it globally if possible.
   if (Value *V = OptimizeExpression(I, Ops)) {
     if (V == I)
diff --git a/lib/Transforms/Scalar/RewriteStatepointsForGC.cpp b/lib/Transforms/Scalar/RewriteStatepointsForGC.cpp
index ba48e0a74b0..aa30cfb8ff9 100644
--- a/lib/Transforms/Scalar/RewriteStatepointsForGC.cpp
+++ b/lib/Transforms/Scalar/RewriteStatepointsForGC.cpp
@@ -14,10 +14,12 @@
 
 #include "llvm/Pass.h"
 #include "llvm/Analysis/CFG.h"
+#include "llvm/Analysis/TargetTransformInfo.h"
 #include "llvm/ADT/SetOperations.h"
 #include "llvm/ADT/Statistic.h"
 #include "llvm/ADT/DenseSet.h"
 #include "llvm/ADT/SetVector.h"
+#include "llvm/ADT/StringRef.h"
 #include "llvm/IR/BasicBlock.h"
 #include "llvm/IR/CallSite.h"
 #include "llvm/IR/Dominators.h"
@@ -28,6 +30,7 @@
 #include "llvm/IR/Intrinsics.h"
 #include "llvm/IR/IntrinsicInst.h"
 #include "llvm/IR/Module.h"
+#include "llvm/IR/MDBuilder.h"
 #include "llvm/IR/Statepoint.h"
 #include "llvm/IR/Value.h"
 #include "llvm/IR/Verifier.h"
@@ -56,6 +59,12 @@ static cl::opt<bool> PrintLiveSetSize("spp-print-liveset-size", cl::Hidden,
 static cl::opt<bool> PrintBasePointers("spp-print-base-pointers", cl::Hidden,
                                        cl::init(false));
 
+// Cost threshold measuring when it is profitable to rematerialize value instead
+// of relocating it
+static cl::opt<unsigned>
+RematerializationThreshold("spp-rematerialization-threshold", cl::Hidden,
+                           cl::init(6));
+
 #ifdef XDEBUG
 static bool ClobberNonLive = true;
 #else
@@ -66,25 +75,54 @@ static cl::opt<bool, true> ClobberNonLiveOverride("rs4gc-clobber-non-live",
                                                   cl::Hidden);
 
 namespace {
-struct RewriteStatepointsForGC : public FunctionPass {
+struct RewriteStatepointsForGC : public ModulePass {
   static char ID; // Pass identification, replacement for typeid
 
-  RewriteStatepointsForGC() : FunctionPass(ID) {
+  RewriteStatepointsForGC() : ModulePass(ID) {
     initializeRewriteStatepointsForGCPass(*PassRegistry::getPassRegistry());
   }
-  bool runOnFunction(Function &F) override;
+  bool runOnFunction(Function &F);
+  bool runOnModule(Module &M) override {
+    bool Changed = false;
+    for (Function &F : M)
+      Changed |= runOnFunction(F);
+
+    if (Changed) {
+      // stripDereferenceabilityInfo asserts that shouldRewriteStatepointsIn
+      // returns true for at least one function in the module.  Since at least
+      // one function changed, we know that the precondition is satisfied.
+      stripDereferenceabilityInfo(M);
+    }
+
+    return Changed;
+  }
 
   void getAnalysisUsage(AnalysisUsage &AU) const override {
     // We add and rewrite a bunch of instructions, but don't really do much
     // else.  We could in theory preserve a lot more analyses here.
     AU.addRequired<DominatorTreeWrapperPass>();
-  }
+    AU.addRequired<TargetTransformInfoWrapperPass>();
+  }
+
+  /// The IR fed into RewriteStatepointsForGC may have had attributes implying
+  /// dereferenceability that are no longer valid/correct after
+  /// RewriteStatepointsForGC has run.  This is because semantically, after
+  /// RewriteStatepointsForGC runs, all calls to gc.statepoint "free" the entire
+  /// heap.  stripDereferenceabilityInfo (conservatively) restores correctness
+  /// by erasing all attributes in the module that externally imply
+  /// dereferenceability.
+  ///
+  void stripDereferenceabilityInfo(Module &M);
+
+  // Helpers for stripDereferenceabilityInfo
+  void stripDereferenceabilityInfoFromBody(Function &F);
+  void stripDereferenceabilityInfoFromPrototype(Function &F);
 };
 } // namespace
 
 char RewriteStatepointsForGC::ID = 0;
 
-FunctionPass *llvm::createRewriteStatepointsForGCPass() {
+ModulePass *llvm::createRewriteStatepointsForGCPass() {
   return new RewriteStatepointsForGC();
 }
 
@@ -123,6 +161,7 @@ struct GCPtrLivenessData {
 // types, then update all the second type to the first type
 typedef DenseMap<Value *, Value *> DefiningValueMapTy;
 typedef DenseSet<llvm::Value *> StatepointLiveSetTy;
+typedef DenseMap<Instruction *, Value *> RematerializedValueMapTy;
 
 struct PartiallyConstructedSafepointRecord {
   /// The set of values known to be live accross this safepoint
@@ -138,6 +177,11 @@ struct PartiallyConstructedSafepointRecord {
   /// Instruction to which exceptional gc relocates are attached
   /// Makes it easier to iterate through them during relocationViaAlloca.
   Instruction *UnwindToken;
+
+  /// Record live values we are rematerialized instead of relocating.
+  /// They are not included into 'liveset' field.
+  /// Maps rematerialized copy to it's original value.
+  RematerializedValueMapTy RematerializedValues;
 };
 }
 
@@ -248,9 +292,19 @@ static void analyzeParsePointLiveness(
   result.liveset = liveset;
 }
 
-/// If we can trivially determine that this vector contains only base pointers,
-/// return the base instruction.
-static Value *findBaseOfVector(Value *I) {
+static Value *findBaseDefiningValue(Value *I);
+
+/// Return a base defining value for the 'Index' element of the given vector
+/// instruction 'I'.  If Index is null, returns a BDV for the entire vector
+/// 'I'.  As an optimization, this method will try to determine when the 
+/// element is known to already be a base pointer.  If this can be established,
+/// the second value in the returned pair will be true.  Note that either a
+/// vector or a pointer typed value can be returned.  For the former, the
+/// vector returned is a BDV (and possibly a base) of the entire vector 'I'.
+/// If the later, the return pointer is a BDV (or possibly a base) for the
+/// particular element in 'I'.  
+static std::pair<Value *, bool>
+findBaseDefiningValueOfVector(Value *I, Value *Index = nullptr) {
   assert(I->getType()->isVectorTy() &&
          cast<VectorType>(I->getType())->getElementType()->isPointerTy() &&
          "Illegal to ask for the base pointer of a non-pointer type");
@@ -260,7 +314,7 @@ static Value *findBaseOfVector(Value *I) {
 
   if (isa<Argument>(I))
     // An incoming argument to the function is a base pointer
-    return I;
+    return std::make_pair(I, true);
 
   // We shouldn't see the address of a global as a vector value?
   assert(!isa<GlobalVariable>(I) &&
@@ -271,7 +325,7 @@ static Value *findBaseOfVector(Value *I) {
   if (isa<UndefValue>(I))
     // utterly meaningless, but useful for dealing with partially optimized
     // code.
-    return I;
+    return std::make_pair(I, true);
 
   // Due to inheritance, this must be _after_ the global variable and undef
   // checks
@@ -279,36 +333,99 @@ static Value *findBaseOfVector(Value *I) {
     assert(!isa<GlobalVariable>(I) && !isa<UndefValue>(I) &&
            "order of checks wrong!");
     assert(Con->isNullValue() && "null is the only case which makes sense");
-    return Con;
+    return std::make_pair(Con, true);
   }
-
+  
   if (isa<LoadInst>(I))
-    return I;
-
-  // Note: This code is currently rather incomplete.  We are essentially only
-  // handling cases where the vector element is trivially a base pointer.  We
-  // need to update the entire base pointer construction algorithm to know how
-  // to track vector elements and potentially scalarize, but the case which
-  // would motivate the work hasn't shown up in real workloads yet.
-  llvm_unreachable("no base found for vector element");
+    return std::make_pair(I, true);
+  
+  // For an insert element, we might be able to look through it if we know
+  // something about the indexes.
+  if (InsertElementInst *IEI = dyn_cast<InsertElementInst>(I)) {
+    if (Index) {
+      Value *InsertIndex = IEI->getOperand(2);
+      // This index is inserting the value, look for its BDV
+      if (InsertIndex == Index)
+        return std::make_pair(findBaseDefiningValue(IEI->getOperand(1)), false);
+      // Both constant, and can't be equal per above. This insert is definitely
+      // not relevant, look back at the rest of the vector and keep trying.
+      if (isa<ConstantInt>(Index) && isa<ConstantInt>(InsertIndex))
+        return findBaseDefiningValueOfVector(IEI->getOperand(0), Index);
+    }
+    
+    // We don't know whether this vector contains entirely base pointers or
+    // not.  To be conservatively correct, we treat it as a BDV and will
+    // duplicate code as needed to construct a parallel vector of bases.
+    return std::make_pair(IEI, false);
+  }
+
+  if (isa<ShuffleVectorInst>(I))
+    // We don't know whether this vector contains entirely base pointers or
+    // not.  To be conservatively correct, we treat it as a BDV and will
+    // duplicate code as needed to construct a parallel vector of bases.
+    // TODO: There a number of local optimizations which could be applied here
+    // for particular sufflevector patterns.
+    return std::make_pair(I, false);
+
+  // A PHI or Select is a base defining value.  The outer findBasePointer
+  // algorithm is responsible for constructing a base value for this BDV.
+  assert((isa<SelectInst>(I) || isa<PHINode>(I)) &&
+         "unknown vector instruction - no base found for vector element");
+  return std::make_pair(I, false);
 }
 
+static bool isKnownBaseResult(Value *V);
+
 /// Helper function for findBasePointer - Will return a value which either a)
 /// defines the base pointer for the input or b) blocks the simple search
 /// (i.e. a PHI or Select of two derived pointers)
 static Value *findBaseDefiningValue(Value *I) {
+  if (I->getType()->isVectorTy())
+    return findBaseDefiningValueOfVector(I).first;
+  
   assert(I->getType()->isPointerTy() &&
          "Illegal to ask for the base pointer of a non-pointer type");
 
   // This case is a bit of a hack - it only handles extracts from vectors which
-  // trivially contain only base pointers.  See note inside the function for
-  // how to improve this.
+  // trivially contain only base pointers or cases where we can directly match
+  // the index of the original extract element to an insertion into the vector.
+  // See note inside the function for how to improve this.
   if (auto *EEI = dyn_cast<ExtractElementInst>(I)) {
     Value *VectorOperand = EEI->getVectorOperand();
-    Value *VectorBase = findBaseOfVector(VectorOperand);
-    (void)VectorBase;
-    assert(VectorBase && "extract element not known to be a trivial base");
-    return EEI;
+    Value *Index = EEI->getIndexOperand();
+    std::pair<Value *, bool> pair =
+      findBaseDefiningValueOfVector(VectorOperand, Index);
+    Value *VectorBase = pair.first;
+    if (VectorBase->getType()->isPointerTy())
+      // We found a BDV for this specific element with the vector.  This is an
+      // optimization, but in practice it covers most of the useful cases
+      // created via scalarization.
+      return VectorBase;
+    else {
+      assert(VectorBase->getType()->isVectorTy());
+      if (pair.second)
+        // If the entire vector returned is known to be entirely base pointers,
+        // then the extractelement is valid base for this value.
+        return EEI;
+      else {
+        // Otherwise, we have an instruction which potentially produces a
+        // derived pointer and we need findBasePointers to clone code for us
+        // such that we can create an instruction which produces the
+        // accompanying base pointer.
+        // Note: This code is currently rather incomplete.  We don't currently
+        // support the general form of shufflevector of insertelement.
+        // Conceptually, these are just 'base defining values' of the same
+        // variety as phi or select instructions.  We need to update the
+        // findBasePointers algorithm to insert new 'base-only' versions of the
+        // original instructions. This is relative straight forward to do, but
+        // the case which would motivate the work hasn't shown up in real
+        // workloads yet.  
+        assert((isa<PHINode>(VectorBase) || isa<SelectInst>(VectorBase)) &&
+               "need to extend findBasePointers for generic vector"
+               "instruction cases");
+        return VectorBase;
+      }
+    }
   }
 
   if (isa<Argument>(I))
@@ -429,13 +546,10 @@ static Value *findBaseDefiningValueCached(Value *I, DefiningValueMapTy &Cache) {
   Value *&Cached = Cache[I];
   if (!Cached) {
     Cached = findBaseDefiningValue(I);
+    DEBUG(dbgs() << "fBDV-cached: " << I->getName() << " -> "
+                 << Cached->getName() << "\n");
   }
   assert(Cache[I] != nullptr);
-
-  if (TraceLSP) {
-    dbgs() << "fBDV-cached: " << I->getName() << " -> " << Cached->getName()
-           << "\n";
-  }
   return Cached;
 }
 
@@ -470,17 +584,19 @@ static bool isKnownBaseResult(Value *V) {
   return false;
 }
 
-// TODO: find a better name for this
 namespace {
-class PhiState {
+/// Models the state of a single base defining value in the findBasePointer
+/// algorithm for determining where a new instruction is needed to propagate
+/// the base of this BDV.
+class BDVState {
 public:
   enum Status { Unknown, Base, Conflict };
 
-  PhiState(Status s, Value *b = nullptr) : status(s), base(b) {
+  BDVState(Status s, Value *b = nullptr) : status(s), base(b) {
     assert(status != Base || b);
   }
-  PhiState(Value *b) : status(Base), base(b) {}
-  PhiState() : status(Unknown), base(nullptr) {}
+  explicit BDVState(Value *b) : status(Base), base(b) {}
+  BDVState() : status(Unknown), base(nullptr) {}
 
   Status getStatus() const { return status; }
   Value *getBase() const { return base; }
@@ -489,15 +605,18 @@ class PhiState {
   bool isUnknown() const { return getStatus() == Unknown; }
   bool isConflict() const { return getStatus() == Conflict; }
 
-  bool operator==(const PhiState &other) const {
+  bool operator==(const BDVState &other) const {
     return base == other.base && status == other.status;
   }
 
-  bool operator!=(const PhiState &other) const { return !(*this == other); }
+  bool operator!=(const BDVState &other) const { return !(*this == other); }
 
-  void dump() {
-    errs() << status << " (" << base << " - "
-           << (base ? base->getName() : "nullptr") << "): ";
+  LLVM_DUMP_METHOD
+  void dump() const { print(dbgs()); dbgs() << '\n'; }
+  
+  void print(raw_ostream &OS) const {
+    OS << status << " (" << base << " - "
+       << (base ? base->getName() : "nullptr") << "): ";
   }
 
 private:
@@ -505,56 +624,48 @@ class PhiState {
   Value *base; // non null only if status == base
 };
 
-typedef DenseMap<Value *, PhiState> ConflictStateMapTy;
-// Values of type PhiState form a lattice, and this is a helper
+inline raw_ostream &operator<<(raw_ostream &OS, const BDVState &State) {
+  State.print(OS);
+  return OS;
+}
+
+
+typedef DenseMap<Value *, BDVState> ConflictStateMapTy;
+// Values of type BDVState form a lattice, and this is a helper
 // class that implementes the meet operation.  The meat of the meet
-// operation is implemented in MeetPhiStates::pureMeet
-class MeetPhiStates {
+// operation is implemented in MeetBDVStates::pureMeet
+class MeetBDVStates {
 public:
-  // phiStates is a mapping from PHINodes and SelectInst's to PhiStates.
-  explicit MeetPhiStates(const ConflictStateMapTy &phiStates)
-      : phiStates(phiStates) {}
-
-  // Destructively meet the current result with the base V.  V can
-  // either be a merge instruction (SelectInst / PHINode), in which
-  // case its status is looked up in the phiStates map; or a regular
-  // SSA value, in which case it is assumed to be a base.
-  void meetWith(Value *V) {
-    PhiState otherState = getStateForBDV(V);
-    assert((MeetPhiStates::pureMeet(otherState, currentResult) ==
-            MeetPhiStates::pureMeet(currentResult, otherState)) &&
-           "math is wrong: meet does not commute!");
-    currentResult = MeetPhiStates::pureMeet(otherState, currentResult);
+  /// Initializes the currentResult to the TOP state so that if can be met with
+  /// any other state to produce that state.
+  MeetBDVStates() {}
+
+  // Destructively meet the current result with the given BDVState
+  void meetWith(BDVState otherState) {
+    currentResult = meet(otherState, currentResult);
   }
 
-  PhiState getResult() const { return currentResult; }
+  BDVState getResult() const { return currentResult; }
 
 private:
-  const ConflictStateMapTy &phiStates;
-  PhiState currentResult;
-
-  /// Return a phi state for a base defining value.  We'll generate a new
-  /// base state for known bases and expect to find a cached state otherwise
-  PhiState getStateForBDV(Value *baseValue) {
-    if (isKnownBaseResult(baseValue)) {
-      return PhiState(baseValue);
-    } else {
-      return lookupFromMap(baseValue);
-    }
-  }
+  BDVState currentResult;
 
-  PhiState lookupFromMap(Value *V) {
-    auto I = phiStates.find(V);
-    assert(I != phiStates.end() && "lookup failed!");
-    return I->second;
+  /// Perform a meet operation on two elements of the BDVState lattice.
+  static BDVState meet(BDVState LHS, BDVState RHS) {
+    assert((pureMeet(LHS, RHS) == pureMeet(RHS, LHS)) &&
+           "math is wrong: meet does not commute!");
+    BDVState Result = pureMeet(LHS, RHS);
+    DEBUG(dbgs() << "meet of " << LHS << " with " << RHS
+                 << " produced " << Result << "\n");
+    return Result;
   }
 
-  static PhiState pureMeet(const PhiState &stateA, const PhiState &stateB) {
+  static BDVState pureMeet(const BDVState &stateA, const BDVState &stateB) {
     switch (stateA.getStatus()) {
-    case PhiState::Unknown:
+    case BDVState::Unknown:
       return stateB;
 
-    case PhiState::Base:
+    case BDVState::Base:
       assert(stateA.getBase() && "can't be null");
       if (stateB.isUnknown())
         return stateA;
@@ -564,12 +675,12 @@ class MeetPhiStates {
           assert(stateA == stateB && "equality broken!");
           return stateA;
         }
-        return PhiState(PhiState::Conflict);
+        return BDVState(BDVState::Conflict);
       }
       assert(stateB.isConflict() && "only three states!");
-      return PhiState(PhiState::Conflict);
+      return BDVState(BDVState::Conflict);
 
-    case PhiState::Conflict:
+    case BDVState::Conflict:
       return stateA;
     }
     llvm_unreachable("only three states!");
@@ -609,62 +720,73 @@ static Value *findBasePointer(Value *I, DefiningValueMapTy &cache) {
   // analougous to pessimistic data flow and would likely lead to an
   // overall worse solution.
 
+#ifndef NDEBUG
+  auto isExpectedBDVType = [](Value *BDV) {
+    return isa<PHINode>(BDV) || isa<SelectInst>(BDV);
+  };
+#endif
+
+  // Once populated, will contain a mapping from each potentially non-base BDV
+  // to a lattice value (described above) which corresponds to that BDV.
   ConflictStateMapTy states;
-  states[def] = PhiState();
   // Recursively fill in all phis & selects reachable from the initial one
   // for which we don't already know a definite base value for
-  // TODO: This should be rewritten with a worklist
-  bool done = false;
-  while (!done) {
-    done = true;
-    // Since we're adding elements to 'states' as we run, we can't keep
-    // iterators into the set.
-    SmallVector<Value *, 16> Keys;
-    Keys.reserve(states.size());
-    for (auto Pair : states) {
-      Value *V = Pair.first;
-      Keys.push_back(V);
-    }
-    for (Value *v : Keys) {
-      assert(!isKnownBaseResult(v) && "why did it get added?");
-      if (PHINode *phi = dyn_cast<PHINode>(v)) {
-        assert(phi->getNumIncomingValues() > 0 &&
-               "zero input phis are illegal");
-        for (Value *InVal : phi->incoming_values()) {
-          Value *local = findBaseOrBDV(InVal, cache);
-          if (!isKnownBaseResult(local) && states.find(local) == states.end()) {
-            states[local] = PhiState();
-            done = false;
-          }
-        }
-      } else if (SelectInst *sel = dyn_cast<SelectInst>(v)) {
-        Value *local = findBaseOrBDV(sel->getTrueValue(), cache);
-        if (!isKnownBaseResult(local) && states.find(local) == states.end()) {
-          states[local] = PhiState();
-          done = false;
-        }
-        local = findBaseOrBDV(sel->getFalseValue(), cache);
-        if (!isKnownBaseResult(local) && states.find(local) == states.end()) {
-          states[local] = PhiState();
-          done = false;
-        }
+  /* scope */ {
+    DenseSet<Value *> Visited;
+    SmallVector<Value*, 16> Worklist;
+    Worklist.push_back(def);
+    Visited.insert(def);
+    while (!Worklist.empty()) {
+      Value *Current = Worklist.pop_back_val();
+      assert(!isKnownBaseResult(Current) && "why did it get added?");
+
+      auto visitIncomingValue = [&](Value *InVal) {
+        Value *Base = findBaseOrBDV(InVal, cache);
+        if (isKnownBaseResult(Base))
+          // Known bases won't need new instructions introduced and can be
+          // ignored safely
+          return;
+        assert(isExpectedBDVType(Base) && "the only non-base values "
+               "we see should be base defining values");
+        if (Visited.insert(Base).second)
+          Worklist.push_back(Base);
+      };
+      if (PHINode *Phi = dyn_cast<PHINode>(Current)) {
+        for (Value *InVal : Phi->incoming_values())
+          visitIncomingValue(InVal);
+      } else {
+        SelectInst *Sel = cast<SelectInst>(Current);
+        visitIncomingValue(Sel->getTrueValue());
+        visitIncomingValue(Sel->getFalseValue());
       }
     }
+    // The frontier of visited instructions are the ones we might need to
+    // duplicate, so fill in the starting state for the optimistic algorithm
+    // that follows.
+    for (Value *BDV : Visited) {
+      states[BDV] = BDVState();
+    }
   }
 
   if (TraceLSP) {
     errs() << "States after initialization:\n";
-    for (auto Pair : states) {
-      Instruction *v = cast<Instruction>(Pair.first);
-      PhiState state = Pair.second;
-      state.dump();
-      v->dump();
-    }
+    for (auto Pair : states)
+      dbgs() << " " << Pair.second << " for " << Pair.first << "\n";
   }
 
   // TODO: come back and revisit the state transitions around inputs which
   // have reached conflict state.  The current version seems too conservative.
 
+  // Return a phi state for a base defining value.  We'll generate a new
+  // base state for known bases and expect to find a cached state otherwise.
+  auto getStateForBDV = [&](Value *baseValue) {
+    if (isKnownBaseResult(baseValue))
+      return BDVState(baseValue);
+    auto I = states.find(baseValue);
+    assert(I != states.end() && "lookup failed!");
+    return I->second;
+  };
+
   bool progress = true;
   while (progress) {
 #ifndef NDEBUG
@@ -673,18 +795,26 @@ static Value *findBasePointer(Value *I, DefiningValueMapTy &cache) {
     progress = false;
     // We're only changing keys in this loop, thus safe to keep iterators
     for (auto Pair : states) {
-      MeetPhiStates calculateMeet(states);
       Value *v = Pair.first;
       assert(!isKnownBaseResult(v) && "why did it get added?");
+
+      // Given an input value for the current instruction, return a BDVState
+      // instance which represents the BDV of that value.
+      auto getStateForInput = [&](Value *V) mutable {
+        Value *BDV = findBaseOrBDV(V, cache);
+        return getStateForBDV(BDV);
+      };
+
+      MeetBDVStates calculateMeet;
       if (SelectInst *select = dyn_cast<SelectInst>(v)) {
-        calculateMeet.meetWith(findBaseOrBDV(select->getTrueValue(), cache));
-        calculateMeet.meetWith(findBaseOrBDV(select->getFalseValue(), cache));
+        calculateMeet.meetWith(getStateForInput(select->getTrueValue()));
+        calculateMeet.meetWith(getStateForInput(select->getFalseValue()));
       } else
         for (Value *Val : cast<PHINode>(v)->incoming_values())
-          calculateMeet.meetWith(findBaseOrBDV(Val, cache));
+          calculateMeet.meetWith(getStateForInput(Val));
 
-      PhiState oldState = states[v];
-      PhiState newState = calculateMeet.getResult();
+      BDVState oldState = states[v];
+      BDVState newState = calculateMeet.getResult();
       if (oldState != newState) {
         progress = true;
         states[v] = newState;
@@ -697,12 +827,8 @@ static Value *findBasePointer(Value *I, DefiningValueMapTy &cache) {
 
   if (TraceLSP) {
     errs() << "States after meet iteration:\n";
-    for (auto Pair : states) {
-      Instruction *v = cast<Instruction>(Pair.first);
-      PhiState state = Pair.second;
-      state.dump();
-      v->dump();
-    }
+    for (auto Pair : states)
+      dbgs() << " " << Pair.second << " for " << Pair.first << "\n";
   }
 
   // Insert Phis for all conflicts
@@ -718,51 +844,42 @@ static Value *findBasePointer(Value *I, DefiningValueMapTy &cache) {
   std::sort(Keys.begin(), Keys.end(), order_by_name);
   // TODO: adjust naming patterns to avoid this order of iteration dependency
   for (Value *V : Keys) {
-    Instruction *v = cast<Instruction>(V);
-    PhiState state = states[V];
-    assert(!isKnownBaseResult(v) && "why did it get added?");
-    assert(!state.isUnknown() && "Optimistic algorithm didn't complete!");
-    if (!state.isConflict())
+    Instruction *I = cast<Instruction>(V);
+    BDVState State = states[I];
+    assert(!isKnownBaseResult(I) && "why did it get added?");
+    assert(!State.isUnknown() && "Optimistic algorithm didn't complete!");
+    if (!State.isConflict())
       continue;
 
-    if (isa<PHINode>(v)) {
-      int num_preds =
-          std::distance(pred_begin(v->getParent()), pred_end(v->getParent()));
-      assert(num_preds > 0 && "how did we reach here");
-      PHINode *phi = PHINode::Create(v->getType(), num_preds, "base_phi", v);
-      // Add metadata marking this as a base value
-      auto *const_1 = ConstantInt::get(
-          Type::getInt32Ty(
-              v->getParent()->getParent()->getParent()->getContext()),
-          1);
-      auto MDConst = ConstantAsMetadata::get(const_1);
-      MDNode *md = MDNode::get(
-          v->getParent()->getParent()->getParent()->getContext(), MDConst);
-      phi->setMetadata("is_base_value", md);
-      states[v] = PhiState(PhiState::Conflict, phi);
-    } else {
-      SelectInst *sel = cast<SelectInst>(v);
+    /// Create and insert a new instruction which will represent the base of
+    /// the given instruction 'I'.
+    auto MakeBaseInstPlaceholder = [](Instruction *I) -> Instruction* {
+      if (isa<PHINode>(I)) {
+        BasicBlock *BB = I->getParent();
+        int NumPreds = std::distance(pred_begin(BB), pred_end(BB));
+        assert(NumPreds > 0 && "how did we reach here");
+        std::string Name = I->hasName() ?
+           (I->getName() + ".base").str() : "base_phi";
+        return PHINode::Create(I->getType(), NumPreds, Name, I);
+      }
+      SelectInst *Sel = cast<SelectInst>(I);
       // The undef will be replaced later
-      UndefValue *undef = UndefValue::get(sel->getType());
-      SelectInst *basesel = SelectInst::Create(sel->getCondition(), undef,
-                                               undef, "base_select", sel);
-      // Add metadata marking this as a base value
-      auto *const_1 = ConstantInt::get(
-          Type::getInt32Ty(
-              v->getParent()->getParent()->getParent()->getContext()),
-          1);
-      auto MDConst = ConstantAsMetadata::get(const_1);
-      MDNode *md = MDNode::get(
-          v->getParent()->getParent()->getParent()->getContext(), MDConst);
-      basesel->setMetadata("is_base_value", md);
-      states[v] = PhiState(PhiState::Conflict, basesel);
-    }
+      UndefValue *Undef = UndefValue::get(Sel->getType());
+      std::string Name = I->hasName() ?
+         (I->getName() + ".base").str() : "base_select";
+      return SelectInst::Create(Sel->getCondition(), Undef,
+                                Undef, Name, Sel);
+    };
+    Instruction *BaseInst = MakeBaseInstPlaceholder(I);
+    // Add metadata marking this as a base value
+    BaseInst->setMetadata("is_base_value", MDNode::get(I->getContext(), {}));
+    states[I] = BDVState(BDVState::Conflict, BaseInst);
   }
 
   // Fixup all the inputs of the new PHIs
   for (auto Pair : states) {
     Instruction *v = cast<Instruction>(Pair.first);
-    PhiState state = Pair.second;
+    BDVState state = Pair.second;
 
     assert(!isKnownBaseResult(v) && "why did it get added?");
     assert(!state.isUnknown() && "Optimistic algorithm didn't complete!");
@@ -795,7 +912,7 @@ static Value *findBasePointer(Value *I, DefiningValueMapTy &cache) {
             // Either conflict or base.
             assert(states.count(base));
             base = states[base].getBase();
-            assert(base != nullptr && "unknown PhiState!");
+            assert(base != nullptr && "unknown BDVState!");
           }
 
           // In essense this assert states: the only way two
@@ -818,7 +935,7 @@ static Value *findBasePointer(Value *I, DefiningValueMapTy &cache) {
           // Either conflict or base.
           assert(states.count(base));
           base = states[base].getBase();
-          assert(base != nullptr && "unknown PhiState!");
+          assert(base != nullptr && "unknown BDVState!");
         }
         assert(base && "can't be null");
         // Must use original input BB since base may not be Instruction
@@ -844,7 +961,7 @@ static Value *findBasePointer(Value *I, DefiningValueMapTy &cache) {
           // Either conflict or base.
           assert(states.count(base));
           base = states[base].getBase();
-          assert(base != nullptr && "unknown PhiState!");
+          assert(base != nullptr && "unknown BDVState!");
         }
         assert(base && "can't be null");
         // Must use original input BB since base may not be Instruction
@@ -989,14 +1106,11 @@ static void recomputeLiveInValues(
 // goes through the statepoint.  We might need to split an edge to make this
 // possible.
 static BasicBlock *
-normalizeForInvokeSafepoint(BasicBlock *BB, BasicBlock *InvokeParent, Pass *P) {
-  DominatorTree *DT = nullptr;
-  if (auto *DTP = P->getAnalysisIfAvailable<DominatorTreeWrapperPass>())
-    DT = &DTP->getDomTree();
-
+normalizeForInvokeSafepoint(BasicBlock *BB, BasicBlock *InvokeParent,
+                            DominatorTree &DT) {
   BasicBlock *Ret = BB;
   if (!BB->getUniquePredecessor()) {
-    Ret = SplitBlockPredecessors(BB, InvokeParent, "", nullptr, DT);
+    Ret = SplitBlockPredecessors(BB, InvokeParent, "", &DT);
   }
 
   // Now that 'ret' has unique predecessor we can safely remove all phi nodes
@@ -1059,47 +1173,42 @@ static AttributeSet legalizeCallAttributes(AttributeSet AS) {
 ///   statepointToken - statepoint instruction to which relocates should be
 ///   bound.
 ///   Builder - Llvm IR builder to be used to construct new calls.
-static void CreateGCRelocates(ArrayRef<llvm::Value *> liveVariables,
-                              const int liveStart,
-                              ArrayRef<llvm::Value *> basePtrs,
-                              Instruction *statepointToken,
+static void CreateGCRelocates(ArrayRef<llvm::Value *> LiveVariables,
+                              const int LiveStart,
+                              ArrayRef<llvm::Value *> BasePtrs,
+                              Instruction *StatepointToken,
                               IRBuilder<> Builder) {
-  SmallVector<Instruction *, 64> NewDefs;
-  NewDefs.reserve(liveVariables.size());
-
-  Module *M = statepointToken->getParent()->getParent()->getParent();
-
-  for (unsigned i = 0; i < liveVariables.size(); i++) {
-    // We generate a (potentially) unique declaration for every pointer type
-    // combination.  This results is some blow up the function declarations in
-    // the IR, but removes the need for argument bitcasts which shrinks the IR
-    // greatly and makes it much more readable.
-    SmallVector<Type *, 1> types;                 // one per 'any' type
-    types.push_back(liveVariables[i]->getType()); // result type
-    Value *gc_relocate_decl = Intrinsic::getDeclaration(
-        M, Intrinsic::experimental_gc_relocate, types);
-
+  if (LiveVariables.empty())
+    return;
+  
+  // All gc_relocate are set to i8 addrspace(1)* type. We originally generated
+  // unique declarations for each pointer type, but this proved problematic
+  // because the intrinsic mangling code is incomplete and fragile.  Since
+  // we're moving towards a single unified pointer type anyways, we can just
+  // cast everything to an i8* of the right address space.  A bitcast is added
+  // later to convert gc_relocate to the actual value's type. 
+  Module *M = StatepointToken->getModule();
+  auto AS = cast<PointerType>(LiveVariables[0]->getType())->getAddressSpace();
+  Type *Types[] = {Type::getInt8PtrTy(M->getContext(), AS)};
+  Value *GCRelocateDecl =
+    Intrinsic::getDeclaration(M, Intrinsic::experimental_gc_relocate, Types);
+
+  for (unsigned i = 0; i < LiveVariables.size(); i++) {
     // Generate the gc.relocate call and save the result
-    Value *baseIdx =
-        ConstantInt::get(Type::getInt32Ty(M->getContext()),
-                         liveStart + find_index(liveVariables, basePtrs[i]));
-    Value *liveIdx = ConstantInt::get(
-        Type::getInt32Ty(M->getContext()),
-        liveStart + find_index(liveVariables, liveVariables[i]));
+    Value *BaseIdx =
+      Builder.getInt32(LiveStart + find_index(LiveVariables, BasePtrs[i]));
+    Value *LiveIdx =
+      Builder.getInt32(LiveStart + find_index(LiveVariables, LiveVariables[i]));
 
     // only specify a debug name if we can give a useful one
-    Value *reloc = Builder.CreateCall3(
-        gc_relocate_decl, statepointToken, baseIdx, liveIdx,
-        liveVariables[i]->hasName() ? liveVariables[i]->getName() + ".relocated"
+    CallInst *Reloc = Builder.CreateCall(
+        GCRelocateDecl, {StatepointToken, BaseIdx, LiveIdx},
+        LiveVariables[i]->hasName() ? LiveVariables[i]->getName() + ".relocated"
                                     : "");
     // Trick CodeGen into thinking there are lots of free registers at this
     // fake call.
-    cast<CallInst>(reloc)->setCallingConv(CallingConv::Cold);
-
-    NewDefs.push_back(cast<Instruction>(reloc));
+    Reloc->setCallingConv(CallingConv::Cold);
   }
-  assert(NewDefs.size() == liveVariables.size() &&
-         "missing or extra redefinition at safepoint");
 }
 
 static void
@@ -1178,7 +1287,7 @@ makeStatepointExplicitImpl(const CallSite &CS, /* to replace */
     // original block.
     InvokeInst *invoke = InvokeInst::Create(
         gc_statepoint_decl, toReplace->getNormalDest(),
-        toReplace->getUnwindDest(), args, "", toReplace->getParent());
+        toReplace->getUnwindDest(), args, "statepoint_token", toReplace->getParent());
     invoke->setCallingConv(toReplace->getCallingConv());
 
     // Currently we will fail on parameter attributes and on certain
@@ -1209,10 +1318,8 @@ makeStatepointExplicitImpl(const CallSite &CS, /* to replace */
             unwindBlock->getLandingPadInst(), idx, "relocate_token"));
     result.UnwindToken = exceptional_token;
 
-    // Just throw away return value. We will use the one we got for normal
-    // block.
-    (void)CreateGCRelocates(liveVariables, live_start, basePtrs,
-                            exceptional_token, Builder);
+    CreateGCRelocates(liveVariables, live_start, basePtrs,
+                      exceptional_token, Builder);
 
     // Generate gc relocates and returns for normal block
     BasicBlock *normalDest = toReplace->getNormalDest();
@@ -1295,8 +1402,7 @@ makeStatepointExplicit(DominatorTree &DT, const CallSite &CS, Pass *P,
   basevec.reserve(liveset.size());
   for (Value *L : liveset) {
     livevec.push_back(L);
-
-    assert(PointerToBase.find(L) != PointerToBase.end());
+    assert(PointerToBase.count(L));
     Value *base = PointerToBase[L];
     basevec.push_back(base);
   }
@@ -1319,41 +1425,75 @@ makeStatepointExplicit(DominatorTree &DT, const CallSite &CS, Pass *P,
 // Add visited values into the visitedLiveValues set we will later use them
 // for sanity check.
 static void
-insertRelocationStores(iterator_range<Value::user_iterator> gcRelocs,
-                       DenseMap<Value *, Value *> &allocaMap,
-                       DenseSet<Value *> &visitedLiveValues) {
+insertRelocationStores(iterator_range<Value::user_iterator> GCRelocs,
+                       DenseMap<Value *, Value *> &AllocaMap,
+                       DenseSet<Value *> &VisitedLiveValues) {
 
-  for (User *U : gcRelocs) {
+  for (User *U : GCRelocs) {
     if (!isa<IntrinsicInst>(U))
       continue;
 
-    IntrinsicInst *relocatedValue = cast<IntrinsicInst>(U);
+    IntrinsicInst *RelocatedValue = cast<IntrinsicInst>(U);
 
     // We only care about relocates
-    if (relocatedValue->getIntrinsicID() !=
+    if (RelocatedValue->getIntrinsicID() !=
         Intrinsic::experimental_gc_relocate) {
       continue;
     }
 
-    GCRelocateOperands relocateOperands(relocatedValue);
-    Value *originalValue = const_cast<Value *>(relocateOperands.derivedPtr());
-    assert(allocaMap.count(originalValue));
-    Value *alloca = allocaMap[originalValue];
+    GCRelocateOperands RelocateOperands(RelocatedValue);
+    Value *OriginalValue =
+        const_cast<Value *>(RelocateOperands.getDerivedPtr());
+    assert(AllocaMap.count(OriginalValue));
+    Value *Alloca = AllocaMap[OriginalValue];
 
     // Emit store into the related alloca
-    StoreInst *store = new StoreInst(relocatedValue, alloca);
-    store->insertAfter(relocatedValue);
+    // All gc_relocate are i8 addrspace(1)* typed, and it must be bitcasted to
+    // the correct type according to alloca.
+    assert(RelocatedValue->getNextNode() && "Should always have one since it's not a terminator");
+    IRBuilder<> Builder(RelocatedValue->getNextNode());
+    Value *CastedRelocatedValue =
+        Builder.CreateBitCast(RelocatedValue, cast<AllocaInst>(Alloca)->getAllocatedType(),
+        RelocatedValue->hasName() ? RelocatedValue->getName() + ".casted" : "");
+
+    StoreInst *Store = new StoreInst(CastedRelocatedValue, Alloca);
+    Store->insertAfter(cast<Instruction>(CastedRelocatedValue));
+
+#ifndef NDEBUG
+    VisitedLiveValues.insert(OriginalValue);
+#endif
+  }
+}
+
+// Helper function for the "relocationViaAlloca". Similar to the
+// "insertRelocationStores" but works for rematerialized values.
+static void
+insertRematerializationStores(
+  RematerializedValueMapTy RematerializedValues,
+  DenseMap<Value *, Value *> &AllocaMap,
+  DenseSet<Value *> &VisitedLiveValues) {
+
+  for (auto RematerializedValuePair: RematerializedValues) {
+    Instruction *RematerializedValue = RematerializedValuePair.first;
+    Value *OriginalValue = RematerializedValuePair.second;
+
+    assert(AllocaMap.count(OriginalValue) &&
+           "Can not find alloca for rematerialized value");
+    Value *Alloca = AllocaMap[OriginalValue];
+
+    StoreInst *Store = new StoreInst(RematerializedValue, Alloca);
+    Store->insertAfter(RematerializedValue);
 
 #ifndef NDEBUG
-    visitedLiveValues.insert(originalValue);
+    VisitedLiveValues.insert(OriginalValue);
 #endif
   }
 }
 
 /// do all the relocation update via allocas and mem2reg
 static void relocationViaAlloca(
-    Function &F, DominatorTree &DT, ArrayRef<Value *> live,
-    ArrayRef<struct PartiallyConstructedSafepointRecord> records) {
+    Function &F, DominatorTree &DT, ArrayRef<Value *> Live,
+    ArrayRef<struct PartiallyConstructedSafepointRecord> Records) {
 #ifndef NDEBUG
   // record initial number of (static) allocas; we'll check we have the same
   // number when we get done.
@@ -1365,17 +1505,38 @@ static void relocationViaAlloca(
 #endif
 
   // TODO-PERF: change data structures, reserve
-  DenseMap<Value *, Value *> allocaMap;
+  DenseMap<Value *, Value *> AllocaMap;
   SmallVector<AllocaInst *, 200> PromotableAllocas;
-  PromotableAllocas.reserve(live.size());
+  // Used later to chack that we have enough allocas to store all values
+  std::size_t NumRematerializedValues = 0;
+  PromotableAllocas.reserve(Live.size());
+
+  // Emit alloca for "LiveValue" and record it in "allocaMap" and
+  // "PromotableAllocas"
+  auto emitAllocaFor = [&](Value *LiveValue) {
+    AllocaInst *Alloca = new AllocaInst(LiveValue->getType(), "",
+                                        F.getEntryBlock().getFirstNonPHI());
+    AllocaMap[LiveValue] = Alloca;
+    PromotableAllocas.push_back(Alloca);
+  };
 
   // emit alloca for each live gc pointer
-  for (unsigned i = 0; i < live.size(); i++) {
-    Value *liveValue = live[i];
-    AllocaInst *alloca = new AllocaInst(liveValue->getType(), "",
-                                        F.getEntryBlock().getFirstNonPHI());
-    allocaMap[liveValue] = alloca;
-    PromotableAllocas.push_back(alloca);
+  for (unsigned i = 0; i < Live.size(); i++) {
+    emitAllocaFor(Live[i]);
+  }
+
+  // emit allocas for rematerialized values
+  for (size_t i = 0; i < Records.size(); i++) {
+    const struct PartiallyConstructedSafepointRecord &Info = Records[i];
+
+    for (auto RematerializedValuePair : Info.RematerializedValues) {
+      Value *OriginalValue = RematerializedValuePair.second;
+      if (AllocaMap.count(OriginalValue) != 0)
+        continue;
+
+      emitAllocaFor(OriginalValue);
+      ++NumRematerializedValues;
+    }
   }
 
   // The next two loops are part of the same conceptual operation.  We need to
@@ -1388,23 +1549,27 @@ static void relocationViaAlloca(
   // this gc pointer and it is not a gc_result)
   // this must happen before we update the statepoint with load of alloca
   // otherwise we lose the link between statepoint and old def
-  for (size_t i = 0; i < records.size(); i++) {
-    const struct PartiallyConstructedSafepointRecord &info = records[i];
-    Value *Statepoint = info.StatepointToken;
+  for (size_t i = 0; i < Records.size(); i++) {
+    const struct PartiallyConstructedSafepointRecord &Info = Records[i];
+    Value *Statepoint = Info.StatepointToken;
 
     // This will be used for consistency check
-    DenseSet<Value *> visitedLiveValues;
+    DenseSet<Value *> VisitedLiveValues;
 
     // Insert stores for normal statepoint gc relocates
-    insertRelocationStores(Statepoint->users(), allocaMap, visitedLiveValues);
+    insertRelocationStores(Statepoint->users(), AllocaMap, VisitedLiveValues);
 
     // In case if it was invoke statepoint
     // we will insert stores for exceptional path gc relocates.
     if (isa<InvokeInst>(Statepoint)) {
-      insertRelocationStores(info.UnwindToken->users(), allocaMap,
-                             visitedLiveValues);
+      insertRelocationStores(Info.UnwindToken->users(), AllocaMap,
+                             VisitedLiveValues);
     }
 
+    // Do similar thing with rematerialized values
+    insertRematerializationStores(Info.RematerializedValues, AllocaMap,
+                                  VisitedLiveValues);
+
     if (ClobberNonLive) {
       // As a debuging aid, pretend that an unrelocated pointer becomes null at
       // the gc.statepoint.  This will turn some subtle GC problems into
@@ -1412,12 +1577,12 @@ static void relocationViaAlloca(
       // lots of gc.statepoints this is extremely costly both memory and time
       // wise.
       SmallVector<AllocaInst *, 64> ToClobber;
-      for (auto Pair : allocaMap) {
+      for (auto Pair : AllocaMap) {
         Value *Def = Pair.first;
         AllocaInst *Alloca = cast<AllocaInst>(Pair.second);
 
         // This value was relocated
-        if (visitedLiveValues.count(Def)) {
+        if (VisitedLiveValues.count(Def)) {
           continue;
         }
         ToClobber.push_back(Alloca);
@@ -1428,8 +1593,8 @@ static void relocationViaAlloca(
           auto AIType = cast<PointerType>(AI->getType());
           auto PT = cast<PointerType>(AIType->getElementType());
           Constant *CPN = ConstantPointerNull::get(PT);
-          StoreInst *store = new StoreInst(CPN, AI);
-          store->insertBefore(IP);
+          StoreInst *Store = new StoreInst(CPN, AI);
+          Store->insertBefore(IP);
         }
       };
 
@@ -1446,72 +1611,70 @@ static void relocationViaAlloca(
     }
   }
   // update use with load allocas and add store for gc_relocated
-  for (auto Pair : allocaMap) {
-    Value *def = Pair.first;
-    Value *alloca = Pair.second;
+  for (auto Pair : AllocaMap) {
+    Value *Def = Pair.first;
+    Value *Alloca = Pair.second;
 
     // we pre-record the uses of allocas so that we dont have to worry about
     // later update
     // that change the user information.
-    SmallVector<Instruction *, 20> uses;
+    SmallVector<Instruction *, 20> Uses;
     // PERF: trade a linear scan for repeated reallocation
-    uses.reserve(std::distance(def->user_begin(), def->user_end()));
-    for (User *U : def->users()) {
+    Uses.reserve(std::distance(Def->user_begin(), Def->user_end()));
+    for (User *U : Def->users()) {
       if (!isa<ConstantExpr>(U)) {
         // If the def has a ConstantExpr use, then the def is either a
         // ConstantExpr use itself or null.  In either case
         // (recursively in the first, directly in the second), the oop
         // it is ultimately dependent on is null and this particular
         // use does not need to be fixed up.
-        uses.push_back(cast<Instruction>(U));
+        Uses.push_back(cast<Instruction>(U));
       }
     }
 
-    std::sort(uses.begin(), uses.end());
-    auto last = std::unique(uses.begin(), uses.end());
-    uses.erase(last, uses.end());
-
-    for (Instruction *use : uses) {
-      if (isa<PHINode>(use)) {
-        PHINode *phi = cast<PHINode>(use);
-        for (unsigned i = 0; i < phi->getNumIncomingValues(); i++) {
-          if (def == phi->getIncomingValue(i)) {
-            LoadInst *load = new LoadInst(
-                alloca, "", phi->getIncomingBlock(i)->getTerminator());
-            phi->setIncomingValue(i, load);
+    std::sort(Uses.begin(), Uses.end());
+    auto Last = std::unique(Uses.begin(), Uses.end());
+    Uses.erase(Last, Uses.end());
+
+    for (Instruction *Use : Uses) {
+      if (isa<PHINode>(Use)) {
+        PHINode *Phi = cast<PHINode>(Use);
+        for (unsigned i = 0; i < Phi->getNumIncomingValues(); i++) {
+          if (Def == Phi->getIncomingValue(i)) {
+            LoadInst *Load = new LoadInst(
+                Alloca, "", Phi->getIncomingBlock(i)->getTerminator());
+            Phi->setIncomingValue(i, Load);
           }
         }
       } else {
-        LoadInst *load = new LoadInst(alloca, "", use);
-        use->replaceUsesOfWith(def, load);
+        LoadInst *Load = new LoadInst(Alloca, "", Use);
+        Use->replaceUsesOfWith(Def, Load);
       }
     }
 
     // emit store for the initial gc value
     // store must be inserted after load, otherwise store will be in alloca's
     // use list and an extra load will be inserted before it
-    StoreInst *store = new StoreInst(def, alloca);
-    if (Instruction *inst = dyn_cast<Instruction>(def)) {
-      if (InvokeInst *invoke = dyn_cast<InvokeInst>(inst)) {
+    StoreInst *Store = new StoreInst(Def, Alloca);
+    if (Instruction *Inst = dyn_cast<Instruction>(Def)) {
+      if (InvokeInst *Invoke = dyn_cast<InvokeInst>(Inst)) {
         // InvokeInst is a TerminatorInst so the store need to be inserted
         // into its normal destination block.
-        BasicBlock *normalDest = invoke->getNormalDest();
-        store->insertBefore(normalDest->getFirstNonPHI());
+        BasicBlock *NormalDest = Invoke->getNormalDest();
+        Store->insertBefore(NormalDest->getFirstNonPHI());
       } else {
-        assert(!inst->isTerminator() &&
+        assert(!Inst->isTerminator() &&
                "The only TerminatorInst that can produce a value is "
                "InvokeInst which is handled above.");
-        store->insertAfter(inst);
+        Store->insertAfter(Inst);
       }
     } else {
-      assert((isa<Argument>(def) || isa<GlobalVariable>(def) ||
-              isa<ConstantPointerNull>(def)) &&
-             "Must be argument or global");
-      store->insertAfter(cast<Instruction>(alloca));
+      assert(isa<Argument>(Def));
+      Store->insertAfter(cast<Instruction>(Alloca));
     }
   }
 
-  assert(PromotableAllocas.size() == live.size() &&
+  assert(PromotableAllocas.size() == Live.size() + NumRematerializedValues &&
          "we must have the same allocas with lives");
   if (!PromotableAllocas.empty()) {
     // apply mem2reg to promote alloca to SSA
@@ -1531,17 +1694,10 @@ static void relocationViaAlloca(
 /// vector.  Doing so has the effect of changing the output of a couple of
 /// tests in ways which make them less useful in testing fused safepoints.
 template <typename T> static void unique_unsorted(SmallVectorImpl<T> &Vec) {
-  DenseSet<T> Seen;
-  SmallVector<T, 128> TempVec;
-  TempVec.reserve(Vec.size());
-  for (auto Element : Vec)
-    TempVec.push_back(Element);
-  Vec.clear();
-  for (auto V : TempVec) {
-    if (Seen.insert(V).second) {
-      Vec.push_back(V);
-    }
-  }
+  SmallSet<T, 8> Seen;
+  Vec.erase(std::remove_if(Vec.begin(), Vec.end(), [&](const T &V) {
+              return !Seen.insert(V).second;
+            }), Vec.end());
 }
 
 /// Insert holders so that each Value is obviously live through the entire
@@ -1591,7 +1747,9 @@ static void findLiveReferences(
 /// slightly non-trivial since it requires a format change.  Given how rare
 /// such cases are (for the moment?) scalarizing is an acceptable comprimise.
 static void splitVectorValues(Instruction *StatepointInst,
-                              StatepointLiveSetTy &LiveSet, DominatorTree &DT) {
+                              StatepointLiveSetTy &LiveSet,
+                              DenseMap<Value *, Value *>& PointerToBase,
+                              DominatorTree &DT) {
   SmallVector<Value *, 16> ToSplit;
   for (Value *V : LiveSet)
     if (isa<VectorType>(V->getType()))
@@ -1600,14 +1758,14 @@ static void splitVectorValues(Instruction *StatepointInst,
   if (ToSplit.empty())
     return;
 
+  DenseMap<Value *, SmallVector<Value *, 16>> ElementMapping;
+
   Function &F = *(StatepointInst->getParent()->getParent());
 
   DenseMap<Value *, AllocaInst *> AllocaMap;
   // First is normal return, second is exceptional return (invoke only)
   DenseMap<Value *, std::pair<Value *, Value *>> Replacements;
   for (Value *V : ToSplit) {
-    LiveSet.erase(V);
-
     AllocaInst *Alloca =
         new AllocaInst(V->getType(), "", F.getEntryBlock().getFirstNonPHI());
     AllocaMap[V] = Alloca;
@@ -1617,7 +1775,7 @@ static void splitVectorValues(Instruction *StatepointInst,
     SmallVector<Value *, 16> Elements;
     for (unsigned i = 0; i < VT->getNumElements(); i++)
       Elements.push_back(Builder.CreateExtractElement(V, Builder.getInt32(i)));
-    LiveSet.insert(Elements.begin(), Elements.end());
+    ElementMapping[V] = Elements;
 
     auto InsertVectorReform = [&](Instruction *IP) {
       Builder.SetInsertPoint(IP);
@@ -1650,6 +1808,7 @@ static void splitVectorValues(Instruction *StatepointInst,
       Replacements[V].second = InsertVectorReform(IP);
     }
   }
+
   for (Value *V : ToSplit) {
     AllocaInst *Alloca = AllocaMap[V];
 
@@ -1693,6 +1852,220 @@ static void splitVectorValues(Instruction *StatepointInst,
   for (Value *V : ToSplit)
     Allocas.push_back(AllocaMap[V]);
   PromoteMemToReg(Allocas, DT);
+
+  // Update our tracking of live pointers and base mappings to account for the
+  // changes we just made.
+  for (Value *V : ToSplit) {
+    auto &Elements = ElementMapping[V];
+
+    LiveSet.erase(V);
+    LiveSet.insert(Elements.begin(), Elements.end());
+    // We need to update the base mapping as well.
+    assert(PointerToBase.count(V));
+    Value *OldBase = PointerToBase[V];
+    auto &BaseElements = ElementMapping[OldBase];
+    PointerToBase.erase(V);
+    assert(Elements.size() == BaseElements.size());
+    for (unsigned i = 0; i < Elements.size(); i++) {
+      Value *Elem = Elements[i];
+      PointerToBase[Elem] = BaseElements[i];
+    }
+  }
+}
+
+// Helper function for the "rematerializeLiveValues". It walks use chain
+// starting from the "CurrentValue" until it meets "BaseValue". Only "simple"
+// values are visited (currently it is GEP's and casts). Returns true if it
+// sucessfully reached "BaseValue" and false otherwise.
+// Fills "ChainToBase" array with all visited values. "BaseValue" is not
+// recorded.
+static bool findRematerializableChainToBasePointer(
+  SmallVectorImpl<Instruction*> &ChainToBase,
+  Value *CurrentValue, Value *BaseValue) {
+
+  // We have found a base value
+  if (CurrentValue == BaseValue) {
+    return true;
+  }
+
+  if (GetElementPtrInst *GEP = dyn_cast<GetElementPtrInst>(CurrentValue)) {
+    ChainToBase.push_back(GEP);
+    return findRematerializableChainToBasePointer(ChainToBase,
+                                                  GEP->getPointerOperand(),
+                                                  BaseValue);
+  }
+
+  if (CastInst *CI = dyn_cast<CastInst>(CurrentValue)) {
+    Value *Def = CI->stripPointerCasts();
+
+    // This two checks are basically similar. First one is here for the
+    // consistency with findBasePointers logic.
+    assert(!isa<CastInst>(Def) && "not a pointer cast found");
+    if (!CI->isNoopCast(CI->getModule()->getDataLayout()))
+      return false;
+
+    ChainToBase.push_back(CI);
+    return findRematerializableChainToBasePointer(ChainToBase, Def, BaseValue);
+  }
+
+  // Not supported instruction in the chain
+  return false;
+}
+
+// Helper function for the "rematerializeLiveValues". Compute cost of the use
+// chain we are going to rematerialize.
+static unsigned
+chainToBasePointerCost(SmallVectorImpl<Instruction*> &Chain,
+                       TargetTransformInfo &TTI) {
+  unsigned Cost = 0;
+
+  for (Instruction *Instr : Chain) {
+    if (CastInst *CI = dyn_cast<CastInst>(Instr)) {
+      assert(CI->isNoopCast(CI->getModule()->getDataLayout()) &&
+             "non noop cast is found during rematerialization");
+
+      Type *SrcTy = CI->getOperand(0)->getType();
+      Cost += TTI.getCastInstrCost(CI->getOpcode(), CI->getType(), SrcTy);
+
+    } else if (GetElementPtrInst *GEP = dyn_cast<GetElementPtrInst>(Instr)) {
+      // Cost of the address calculation
+      Type *ValTy = GEP->getPointerOperandType()->getPointerElementType();
+      Cost += TTI.getAddressComputationCost(ValTy);
+
+      // And cost of the GEP itself
+      // TODO: Use TTI->getGEPCost here (it exists, but appears to be not
+      //       allowed for the external usage)
+      if (!GEP->hasAllConstantIndices())
+        Cost += 2;
+
+    } else {
+      llvm_unreachable("unsupported instruciton type during rematerialization");
+    }
+  }
+
+  return Cost;
+}
+
+// From the statepoint liveset pick values that are cheaper to recompute then to
+// relocate. Remove this values from the liveset, rematerialize them after
+// statepoint and record them in "Info" structure. Note that similar to
+// relocated values we don't do any user adjustments here.
+static void rematerializeLiveValues(CallSite CS,
+                                    PartiallyConstructedSafepointRecord &Info,
+                                    TargetTransformInfo &TTI) {
+  const unsigned int ChainLengthThreshold = 10;
+
+  // Record values we are going to delete from this statepoint live set.
+  // We can not di this in following loop due to iterator invalidation.
+  SmallVector<Value *, 32> LiveValuesToBeDeleted;
+
+  for (Value *LiveValue: Info.liveset) {
+    // For each live pointer find it's defining chain
+    SmallVector<Instruction *, 3> ChainToBase;
+    assert(Info.PointerToBase.count(LiveValue));
+    bool FoundChain =
+      findRematerializableChainToBasePointer(ChainToBase,
+                                             LiveValue,
+                                             Info.PointerToBase[LiveValue]);
+    // Nothing to do, or chain is too long
+    if (!FoundChain ||
+        ChainToBase.size() == 0 ||
+        ChainToBase.size() > ChainLengthThreshold)
+      continue;
+
+    // Compute cost of this chain
+    unsigned Cost = chainToBasePointerCost(ChainToBase, TTI);
+    // TODO: We can also account for cases when we will be able to remove some
+    //       of the rematerialized values by later optimization passes. I.e if
+    //       we rematerialized several intersecting chains. Or if original values
+    //       don't have any uses besides this statepoint.
+
+    // For invokes we need to rematerialize each chain twice - for normal and
+    // for unwind basic blocks. Model this by multiplying cost by two.
+    if (CS.isInvoke()) {
+      Cost *= 2;
+    }
+    // If it's too expensive - skip it
+    if (Cost >= RematerializationThreshold)
+      continue;
+
+    // Remove value from the live set
+    LiveValuesToBeDeleted.push_back(LiveValue);
+
+    // Clone instructions and record them inside "Info" structure
+
+    // Walk backwards to visit top-most instructions first
+    std::reverse(ChainToBase.begin(), ChainToBase.end());
+
+    // Utility function which clones all instructions from "ChainToBase"
+    // and inserts them before "InsertBefore". Returns rematerialized value
+    // which should be used after statepoint.
+    auto rematerializeChain = [&ChainToBase](Instruction *InsertBefore) {
+      Instruction *LastClonedValue = nullptr;
+      Instruction *LastValue = nullptr;
+      for (Instruction *Instr: ChainToBase) {
+        // Only GEP's and casts are suported as we need to be careful to not
+        // introduce any new uses of pointers not in the liveset.
+        // Note that it's fine to introduce new uses of pointers which were
+        // otherwise not used after this statepoint.
+        assert(isa<GetElementPtrInst>(Instr) || isa<CastInst>(Instr));
+
+        Instruction *ClonedValue = Instr->clone();
+        ClonedValue->insertBefore(InsertBefore);
+        ClonedValue->setName(Instr->getName() + ".remat");
+
+        // If it is not first instruction in the chain then it uses previously
+        // cloned value. We should update it to use cloned value.
+        if (LastClonedValue) {
+          assert(LastValue);
+          ClonedValue->replaceUsesOfWith(LastValue, LastClonedValue);
+#ifndef NDEBUG
+          // Assert that cloned instruction does not use any instructions from
+          // this chain other than LastClonedValue
+          for (auto OpValue : ClonedValue->operand_values()) {
+            assert(std::find(ChainToBase.begin(), ChainToBase.end(), OpValue) ==
+                       ChainToBase.end() &&
+                   "incorrect use in rematerialization chain");
+          }
+#endif
+        }
+
+        LastClonedValue = ClonedValue;
+        LastValue = Instr;
+      }
+      assert(LastClonedValue);
+      return LastClonedValue;
+    };
+
+    // Different cases for calls and invokes. For invokes we need to clone
+    // instructions both on normal and unwind path.
+    if (CS.isCall()) {
+      Instruction *InsertBefore = CS.getInstruction()->getNextNode();
+      assert(InsertBefore);
+      Instruction *RematerializedValue = rematerializeChain(InsertBefore);
+      Info.RematerializedValues[RematerializedValue] = LiveValue;
+    } else {
+      InvokeInst *Invoke = cast<InvokeInst>(CS.getInstruction());
+
+      Instruction *NormalInsertBefore =
+          Invoke->getNormalDest()->getFirstInsertionPt();
+      Instruction *UnwindInsertBefore =
+          Invoke->getUnwindDest()->getFirstInsertionPt();
+
+      Instruction *NormalRematerializedValue =
+          rematerializeChain(NormalInsertBefore);
+      Instruction *UnwindRematerializedValue =
+          rematerializeChain(UnwindInsertBefore);
+
+      Info.RematerializedValues[NormalRematerializedValue] = LiveValue;
+      Info.RematerializedValues[UnwindRematerializedValue] = LiveValue;
+    }
+  }
+
+  // Remove rematerializaed values from the live set
+  for (auto LiveValue: LiveValuesToBeDeleted) {
+    Info.liveset.erase(LiveValue);
+  }
 }
 
 static bool insertParsePoints(Function &F, DominatorTree &DT, Pass *P,
@@ -1719,9 +2092,9 @@ static bool insertParsePoints(Function &F, DominatorTree &DT, Pass *P,
       continue;
     InvokeInst *invoke = cast<InvokeInst>(CS.getInstruction());
     normalizeForInvokeSafepoint(invoke->getNormalDest(), invoke->getParent(),
-                                P);
+                                DT);
     normalizeForInvokeSafepoint(invoke->getUnwindDest(), invoke->getParent(),
-                                P);
+                                DT);
   }
 
   // A list of dummy calls added to the IR to keep various values obviously
@@ -1759,17 +2132,6 @@ static bool insertParsePoints(Function &F, DominatorTree &DT, Pass *P,
   // site.
   findLiveReferences(F, DT, P, toUpdate, records);
 
-  // Do a limited scalarization of any live at safepoint vector values which
-  // contain pointers.  This enables this pass to run after vectorization at
-  // the cost of some possible performance loss.  TODO: it would be nice to
-  // natively support vectors all the way through the backend so we don't need
-  // to scalarize here.
-  for (size_t i = 0; i < records.size(); i++) {
-    struct PartiallyConstructedSafepointRecord &info = records[i];
-    Instruction *statepoint = toUpdate[i].getInstruction();
-    splitVectorValues(cast<Instruction>(statepoint), info.liveset, DT);
-  }
-
   // B) Find the base pointers for each live pointer
   /* scope for caching */ {
     // Cache the 'defining value' relation used in the computation and
@@ -1830,6 +2192,31 @@ static bool insertParsePoints(Function &F, DominatorTree &DT, Pass *P,
   }
   holders.clear();
 
+  // Do a limited scalarization of any live at safepoint vector values which
+  // contain pointers.  This enables this pass to run after vectorization at
+  // the cost of some possible performance loss.  TODO: it would be nice to
+  // natively support vectors all the way through the backend so we don't need
+  // to scalarize here.
+  for (size_t i = 0; i < records.size(); i++) {
+    struct PartiallyConstructedSafepointRecord &info = records[i];
+    Instruction *statepoint = toUpdate[i].getInstruction();
+    splitVectorValues(cast<Instruction>(statepoint), info.liveset,
+                      info.PointerToBase, DT);
+  }
+
+  // In order to reduce live set of statepoint we might choose to rematerialize
+  // some values instead of relocating them. This is purelly an optimization and
+  // does not influence correctness.
+  TargetTransformInfo &TTI =
+    P->getAnalysis<TargetTransformInfoWrapperPass>().getTTI(F);
+
+  for (size_t i = 0; i < records.size(); i++) {
+    struct PartiallyConstructedSafepointRecord &info = records[i];
+    CallSite &CS = toUpdate[i];
+
+    rematerializeLiveValues(CS, info, TTI);
+  }
+
   // Now run through and replace the existing statepoints with new ones with
   // the live variables listed.  We do not yet update uses of the values being
   // relocated. We have references to live variables that need to
@@ -1887,17 +2274,99 @@ static bool insertParsePoints(Function &F, DominatorTree &DT, Pass *P,
   return !records.empty();
 }
 
+// Handles both return values and arguments for Functions and CallSites.
+template <typename AttrHolder>
+static void RemoveDerefAttrAtIndex(LLVMContext &Ctx, AttrHolder &AH,
+                                   unsigned Index) {
+  AttrBuilder R;
+  if (AH.getDereferenceableBytes(Index))
+    R.addAttribute(Attribute::get(Ctx, Attribute::Dereferenceable,
+                                  AH.getDereferenceableBytes(Index)));
+  if (AH.getDereferenceableOrNullBytes(Index))
+    R.addAttribute(Attribute::get(Ctx, Attribute::DereferenceableOrNull,
+                                  AH.getDereferenceableOrNullBytes(Index)));
+
+  if (!R.empty())
+    AH.setAttributes(AH.getAttributes().removeAttributes(
+        Ctx, Index, AttributeSet::get(Ctx, Index, R)));
+}
+
+void
+RewriteStatepointsForGC::stripDereferenceabilityInfoFromPrototype(Function &F) {
+  LLVMContext &Ctx = F.getContext();
+
+  for (Argument &A : F.args())
+    if (isa<PointerType>(A.getType()))
+      RemoveDerefAttrAtIndex(Ctx, F, A.getArgNo() + 1);
+
+  if (isa<PointerType>(F.getReturnType()))
+    RemoveDerefAttrAtIndex(Ctx, F, AttributeSet::ReturnIndex);
+}
+
+void RewriteStatepointsForGC::stripDereferenceabilityInfoFromBody(Function &F) {
+  if (F.empty())
+    return;
+
+  LLVMContext &Ctx = F.getContext();
+  MDBuilder Builder(Ctx);
+
+  for (Instruction &I : inst_range(F)) {
+    if (const MDNode *MD = I.getMetadata(LLVMContext::MD_tbaa)) {
+      assert(MD->getNumOperands() < 5 && "unrecognized metadata shape!");
+      bool IsImmutableTBAA =
+          MD->getNumOperands() == 4 &&
+          mdconst::extract<ConstantInt>(MD->getOperand(3))->getValue() == 1;
+
+      if (!IsImmutableTBAA)
+        continue; // no work to do, MD_tbaa is already marked mutable
+
+      MDNode *Base = cast<MDNode>(MD->getOperand(0));
+      MDNode *Access = cast<MDNode>(MD->getOperand(1));
+      uint64_t Offset =
+          mdconst::extract<ConstantInt>(MD->getOperand(2))->getZExtValue();
+
+      MDNode *MutableTBAA =
+          Builder.createTBAAStructTagNode(Base, Access, Offset);
+      I.setMetadata(LLVMContext::MD_tbaa, MutableTBAA);
+    }
+
+    if (CallSite CS = CallSite(&I)) {
+      for (int i = 0, e = CS.arg_size(); i != e; i++)
+        if (isa<PointerType>(CS.getArgument(i)->getType()))
+          RemoveDerefAttrAtIndex(Ctx, CS, i + 1);
+      if (isa<PointerType>(CS.getType()))
+        RemoveDerefAttrAtIndex(Ctx, CS, AttributeSet::ReturnIndex);
+    }
+  }
+}
+
 /// Returns true if this function should be rewritten by this pass.  The main
 /// point of this function is as an extension point for custom logic.
 static bool shouldRewriteStatepointsIn(Function &F) {
   // TODO: This should check the GCStrategy
   if (F.hasGC()) {
-    const std::string StatepointExampleName("statepoint-example");
-    return StatepointExampleName == F.getGC();
+    const char *FunctionGCName = F.getGC();
+    const StringRef StatepointExampleName("statepoint-example");
+    const StringRef CoreCLRName("coreclr");
+    return (StatepointExampleName == FunctionGCName) ||
+           (CoreCLRName == FunctionGCName);
   } else
     return false;
 }
 
+void RewriteStatepointsForGC::stripDereferenceabilityInfo(Module &M) {
+#ifndef NDEBUG
+  assert(std::any_of(M.begin(), M.end(), shouldRewriteStatepointsIn) &&
+         "precondition!");
+#endif
+
+  for (Function &F : M)
+    stripDereferenceabilityInfoFromPrototype(F);
+
+  for (Function &F : M)
+    stripDereferenceabilityInfoFromBody(F);
+}
+
 bool RewriteStatepointsForGC::runOnFunction(Function &F) {
   // Nothing to do for declarations.
   if (F.isDeclaration() || F.empty())
@@ -1908,7 +2377,7 @@ bool RewriteStatepointsForGC::runOnFunction(Function &F) {
   if (!shouldRewriteStatepointsIn(F))
     return false;
 
-  DominatorTree &DT = getAnalysis<DominatorTreeWrapperPass>().getDomTree();
+  DominatorTree &DT = getAnalysis<DominatorTreeWrapperPass>(F).getDomTree();
 
   // Gather all the statepoints which need rewritten.  Be careful to only
   // consider those in reachable code since we need to ask dominance queries
@@ -1958,11 +2427,6 @@ bool RewriteStatepointsForGC::runOnFunction(Function &F) {
 // TODO: Consider using bitvectors for liveness, the set of potentially
 // interesting values should be small and easy to pre-compute.
 
-/// Is this value a constant consisting of entirely null values?
-static bool isConstantNull(Value *V) {
-  return isa<Constant>(V) && cast<Constant>(V)->isNullValue();
-}
-
 /// Compute the live-in set for the location rbegin starting from
 /// the live-out set of the basic block
 static void computeLiveInValues(BasicBlock::reverse_iterator rbegin,
@@ -1984,9 +2448,17 @@ static void computeLiveInValues(BasicBlock::reverse_iterator rbegin,
     for (Value *V : I->operands()) {
       assert(!isUnhandledGCPointerType(V->getType()) &&
              "support for FCA unimplemented");
-      if (isHandledGCPointerType(V->getType()) && !isConstantNull(V) &&
-          !isa<UndefValue>(V)) {
-        // The choice to exclude null and undef is arbitrary here.  Reconsider?
+      if (isHandledGCPointerType(V->getType()) && !isa<Constant>(V)) {
+        // The choice to exclude all things constant here is slightly subtle.
+        // There are two idependent reasons:
+        // - We assume that things which are constant (from LLVM's definition)
+        // do not move at runtime.  For example, the address of a global
+        // variable is fixed, even though it's contents may not be.
+        // - Second, we can't disallow arbitrary inttoptr constants even
+        // if the language frontend does.  Optimization passes are free to
+        // locally exploit facts without respect to global reachability.  This
+        // can create sections of code which are dynamically unreachable and
+        // contain just about anything.  (see constants.ll in tests)
         LiveTmp.insert(V);
       }
     }
@@ -2002,9 +2474,7 @@ static void computeLiveOutSeed(BasicBlock *BB, DenseSet<Value *> &LiveTmp) {
       Value *V = Phi->getIncomingValueForBlock(BB);
       assert(!isUnhandledGCPointerType(V->getType()) &&
              "support for FCA unimplemented");
-      if (isHandledGCPointerType(V->getType()) && !isConstantNull(V) &&
-          !isa<UndefValue>(V)) {
-        // The choice to exclude null and undef is arbitrary here.  Reconsider?
+      if (isHandledGCPointerType(V->getType()) && !isa<Constant>(V)) {
         LiveTmp.insert(V);
       }
     }
diff --git a/lib/Transforms/Scalar/SCCP.cpp b/lib/Transforms/Scalar/SCCP.cpp
index bc068f78c57..c625b0f3362 100644
--- a/lib/Transforms/Scalar/SCCP.cpp
+++ b/lib/Transforms/Scalar/SCCP.cpp
@@ -539,9 +539,9 @@ void SCCPSolver::getFeasibleSuccessors(TerminatorInst &TI,
     return;
   }
 
-  if (isa<InvokeInst>(TI)) {
-    // Invoke instructions successors are always executable.
-    Succs[0] = Succs[1] = true;
+  // Unwinding instructions successors are always executable.
+  if (TI.isExceptional()) {
+    Succs.assign(TI.getNumSuccessors(), true);
     return;
   }
 
@@ -605,8 +605,8 @@ bool SCCPSolver::isEdgeFeasible(BasicBlock *From, BasicBlock *To) {
     return BI->getSuccessor(CI->isZero()) == To;
   }
 
-  // Invoke instructions successors are always executable.
-  if (isa<InvokeInst>(TI))
+  // Unwinding instructions successors are always executable.
+  if (TI->isExceptional())
     return true;
 
   if (SwitchInst *SI = dyn_cast<SwitchInst>(TI)) {
@@ -1055,7 +1055,7 @@ void SCCPSolver::visitLoadInst(LoadInst &I) {
 
   // load null -> null
   if (isa<ConstantPointerNull>(Ptr) && I.getPointerAddressSpace() == 0)
-    return markConstant(IV, &I, Constant::getNullValue(I.getType()));
+    return markConstant(IV, &I, UndefValue::get(I.getType()));
 
   // Transform load (constant global) into the value loaded.
   if (GlobalVariable *GV = dyn_cast<GlobalVariable>(Ptr)) {
@@ -1799,11 +1799,10 @@ bool IPSCCP::runOnModule(Module &M) {
         if (!TI->use_empty())
           TI->replaceAllUsesWith(UndefValue::get(TI->getType()));
         TI->eraseFromParent();
+        new UnreachableInst(M.getContext(), BB);
 
         if (&*BB != &F->front())
           BlocksToErase.push_back(BB);
-        else
-          new UnreachableInst(M.getContext(), BB);
         continue;
       }
 
diff --git a/lib/Transforms/Scalar/SROA.cpp b/lib/Transforms/Scalar/SROA.cpp
index 59dc52811c9..947513a3657 100644
--- a/lib/Transforms/Scalar/SROA.cpp
+++ b/lib/Transforms/Scalar/SROA.cpp
@@ -1088,7 +1088,8 @@ class AllocaPromoter : public LoadAndStorePromoter {
   SmallVector<DbgValueInst *, 4> DVIs;
 
 public:
-  AllocaPromoter(const SmallVectorImpl<Instruction *> &Insts, SSAUpdater &S,
+  AllocaPromoter(ArrayRef<const Instruction *> Insts,
+                 SSAUpdater &S,
                  AllocaInst &AI, DIBuilder &DIB)
       : LoadAndStorePromoter(Insts, S), AI(AI), DIB(DIB) {}
 
@@ -1096,8 +1097,8 @@ class AllocaPromoter : public LoadAndStorePromoter {
     // Retain the debug information attached to the alloca for use when
     // rewriting loads and stores.
     if (auto *L = LocalAsMetadata::getIfExists(&AI)) {
-      if (auto *DebugNode = MetadataAsValue::getIfExists(AI.getContext(), L)) {
-        for (User *U : DebugNode->users())
+      if (auto *DINode = MetadataAsValue::getIfExists(AI.getContext(), L)) {
+        for (User *U : DINode->users())
           if (DbgDeclareInst *DDI = dyn_cast<DbgDeclareInst>(U))
             DDIs.push_back(DDI);
           else if (DbgValueInst *DVI = dyn_cast<DbgValueInst>(U))
@@ -1166,9 +1167,9 @@ class AllocaPromoter : public LoadAndStorePromoter {
       } else {
         continue;
       }
-      DIB.insertDbgValueIntrinsic(Arg, 0, DIVariable(DVI->getVariable()),
-                                  DIExpression(DVI->getExpression()),
-                                  DVI->getDebugLoc(), Inst);
+      DIB.insertDbgValueIntrinsic(Arg, 0, DVI->getVariable(),
+                                  DVI->getExpression(), DVI->getDebugLoc(),
+                                  Inst);
     }
   }
 };
@@ -1406,7 +1407,7 @@ static bool isSafePHIToSpeculate(PHINode &PN) {
     // If this pointer is always safe to load, or if we can prove that there
     // is already a load in the block, then we can move the load to the pred
     // block.
-    if (InVal->isDereferenceablePointer(DL) ||
+    if (isDereferenceablePointer(InVal, DL) ||
         isSafeToLoadUnconditionally(InVal, TI, MaxAlign))
       continue;
 
@@ -1476,8 +1477,8 @@ static bool isSafeSelectToSpeculate(SelectInst &SI) {
   Value *TValue = SI.getTrueValue();
   Value *FValue = SI.getFalseValue();
   const DataLayout &DL = SI.getModule()->getDataLayout();
-  bool TDerefable = TValue->isDereferenceablePointer(DL);
-  bool FDerefable = FValue->isDereferenceablePointer(DL);
+  bool TDerefable = isDereferenceablePointer(TValue, DL);
+  bool FDerefable = isDereferenceablePointer(FValue, DL);
 
   for (User *U : SI.users()) {
     LoadInst *LI = dyn_cast<LoadInst>(U);
@@ -1846,10 +1847,17 @@ static unsigned getAdjustedAlignment(Instruction *I, uint64_t Offset,
 static bool canConvertValue(const DataLayout &DL, Type *OldTy, Type *NewTy) {
   if (OldTy == NewTy)
     return true;
-  if (IntegerType *OldITy = dyn_cast<IntegerType>(OldTy))
-    if (IntegerType *NewITy = dyn_cast<IntegerType>(NewTy))
-      if (NewITy->getBitWidth() >= OldITy->getBitWidth())
-        return true;
+
+  // For integer types, we can't handle any bit-width differences. This would
+  // break both vector conversions with extension and introduce endianness
+  // issues when in conjunction with loads and stores.
+  if (isa<IntegerType>(OldTy) && isa<IntegerType>(NewTy)) {
+    assert(cast<IntegerType>(OldTy)->getBitWidth() !=
+               cast<IntegerType>(NewTy)->getBitWidth() &&
+           "We can't have the same bitwidth for different int types");
+    return false;
+  }
+
   if (DL.getTypeSizeInBits(NewTy) != DL.getTypeSizeInBits(OldTy))
     return false;
   if (!NewTy->isSingleValueType() || !OldTy->isSingleValueType())
@@ -1884,10 +1892,8 @@ static Value *convertValue(const DataLayout &DL, IRBuilderTy &IRB, Value *V,
   if (OldTy == NewTy)
     return V;
 
-  if (IntegerType *OldITy = dyn_cast<IntegerType>(OldTy))
-    if (IntegerType *NewITy = dyn_cast<IntegerType>(NewTy))
-      if (NewITy->getBitWidth() > OldITy->getBitWidth())
-        return IRB.CreateZExt(V, NewITy);
+  assert(!(isa<IntegerType>(OldTy) && isa<IntegerType>(NewTy)) &&
+         "Integer types must be the exact same to convert.");
 
   // See if we need inttoptr for this type pair. A cast involving both scalars
   // and vectors requires and additional bitcast.
@@ -2133,6 +2139,9 @@ static bool isIntegerWideningViableForSlice(const Slice &S,
   if (LoadInst *LI = dyn_cast<LoadInst>(U->getUser())) {
     if (LI->isVolatile())
       return false;
+    // We can't handle loads that extend past the allocated memory.
+    if (DL.getTypeStoreSize(LI->getType()) > Size)
+      return false;
     // Note that we don't count vector loads or stores as whole-alloca
     // operations which enable integer widening because we would prefer to use
     // vector widening instead.
@@ -2151,6 +2160,9 @@ static bool isIntegerWideningViableForSlice(const Slice &S,
     Type *ValueTy = SI->getValueOperand()->getType();
     if (SI->isVolatile())
       return false;
+    // We can't handle stores that extend past the allocated memory.
+    if (DL.getTypeStoreSize(ValueTy) > Size)
+      return false;
     // Note that we don't count vector loads or stores as whole-alloca
     // operations which enable integer widening because we would prefer to use
     // vector widening instead.
@@ -2584,6 +2596,7 @@ class AllocaSliceRewriter : public InstVisitor<AllocaSliceRewriter, bool> {
 
     Type *TargetTy = IsSplit ? Type::getIntNTy(LI.getContext(), SliceSize * 8)
                              : LI.getType();
+    const bool IsLoadPastEnd = DL.getTypeStoreSize(TargetTy) > SliceSize;
     bool IsPtrAdjusted = false;
     Value *V;
     if (VecTy) {
@@ -2591,14 +2604,36 @@ class AllocaSliceRewriter : public InstVisitor<AllocaSliceRewriter, bool> {
     } else if (IntTy && LI.getType()->isIntegerTy()) {
       V = rewriteIntegerLoad(LI);
     } else if (NewBeginOffset == NewAllocaBeginOffset &&
-               canConvertValue(DL, NewAllocaTy, LI.getType())) {
-      V = IRB.CreateAlignedLoad(&NewAI, NewAI.getAlignment(), LI.isVolatile(),
-                                LI.getName());
+               NewEndOffset == NewAllocaEndOffset &&
+               (canConvertValue(DL, NewAllocaTy, TargetTy) ||
+                (IsLoadPastEnd && NewAllocaTy->isIntegerTy() &&
+                 TargetTy->isIntegerTy()))) {
+      LoadInst *NewLI = IRB.CreateAlignedLoad(&NewAI, NewAI.getAlignment(),
+                                              LI.isVolatile(), LI.getName());
+      if (LI.isVolatile())
+        NewLI->setAtomic(LI.getOrdering(), LI.getSynchScope());
+      V = NewLI;
+
+      // If this is an integer load past the end of the slice (which means the
+      // bytes outside the slice are undef or this load is dead) just forcibly
+      // fix the integer size with correct handling of endianness.
+      if (auto *AITy = dyn_cast<IntegerType>(NewAllocaTy))
+        if (auto *TITy = dyn_cast<IntegerType>(TargetTy))
+          if (AITy->getBitWidth() < TITy->getBitWidth()) {
+            V = IRB.CreateZExt(V, TITy, "load.ext");
+            if (DL.isBigEndian())
+              V = IRB.CreateShl(V, TITy->getBitWidth() - AITy->getBitWidth(),
+                                "endian_shift");
+          }
     } else {
       Type *LTy = TargetTy->getPointerTo();
-      V = IRB.CreateAlignedLoad(getNewAllocaSlicePtr(IRB, LTy),
-                                getSliceAlign(TargetTy), LI.isVolatile(),
-                                LI.getName());
+      LoadInst *NewLI = IRB.CreateAlignedLoad(getNewAllocaSlicePtr(IRB, LTy),
+                                              getSliceAlign(TargetTy),
+                                              LI.isVolatile(), LI.getName());
+      if (LI.isVolatile())
+        NewLI->setAtomic(LI.getOrdering(), LI.getSynchScope());
+
+      V = NewLI;
       IsPtrAdjusted = true;
     }
     V = convertValue(DL, IRB, V, TargetTy);
@@ -2709,10 +2744,25 @@ class AllocaSliceRewriter : public InstVisitor<AllocaSliceRewriter, bool> {
     if (IntTy && V->getType()->isIntegerTy())
       return rewriteIntegerStore(V, SI);
 
+    const bool IsStorePastEnd = DL.getTypeStoreSize(V->getType()) > SliceSize;
     StoreInst *NewSI;
     if (NewBeginOffset == NewAllocaBeginOffset &&
         NewEndOffset == NewAllocaEndOffset &&
-        canConvertValue(DL, V->getType(), NewAllocaTy)) {
+        (canConvertValue(DL, V->getType(), NewAllocaTy) ||
+         (IsStorePastEnd && NewAllocaTy->isIntegerTy() &&
+          V->getType()->isIntegerTy()))) {
+      // If this is an integer store past the end of slice (and thus the bytes
+      // past that point are irrelevant or this is unreachable), truncate the
+      // value prior to storing.
+      if (auto *VITy = dyn_cast<IntegerType>(V->getType()))
+        if (auto *AITy = dyn_cast<IntegerType>(NewAllocaTy))
+          if (VITy->getBitWidth() > AITy->getBitWidth()) {
+            if (DL.isBigEndian())
+              V = IRB.CreateLShr(V, VITy->getBitWidth() - AITy->getBitWidth(),
+                                 "endian_shift");
+            V = IRB.CreateTrunc(V, AITy, "load.trunc");
+          }
+
       V = convertValue(DL, IRB, V, NewAllocaTy);
       NewSI = IRB.CreateAlignedStore(V, &NewAI, NewAI.getAlignment(),
                                      SI.isVolatile());
@@ -2721,7 +2771,8 @@ class AllocaSliceRewriter : public InstVisitor<AllocaSliceRewriter, bool> {
       NewSI = IRB.CreateAlignedStore(V, NewPtr, getSliceAlign(V->getType()),
                                      SI.isVolatile());
     }
-    (void)NewSI;
+    if (SI.isVolatile())
+      NewSI->setAtomic(SI.getOrdering(), SI.getSynchScope());
     Pass.DeadInsts.insert(&SI);
     deleteIfTriviallyDead(OldOp);
 
@@ -4181,15 +4232,15 @@ bool SROA::splitAlloca(AllocaInst &AI, AllocaSlices &AS) {
   // Migrate debug information from the old alloca to the new alloca(s)
   // and the individial partitions.
   if (DbgDeclareInst *DbgDecl = FindAllocaDbgDeclare(&AI)) {
-    DIVariable Var(DbgDecl->getVariable());
-    DIExpression Expr(DbgDecl->getExpression());
+    auto *Var = DbgDecl->getVariable();
+    auto *Expr = DbgDecl->getExpression();
     DIBuilder DIB(*AI.getParent()->getParent()->getParent(),
                   /*AllowUnresolved*/ false);
     bool IsSplit = Pieces.size() > 1;
     for (auto Piece : Pieces) {
       // Create a piece expression describing the new partition or reuse AI's
       // expression if there is only one partition.
-      DIExpression PieceExpr = Expr;
+      auto *PieceExpr = Expr;
       if (IsSplit || Expr->isBitPiece()) {
         // If this alloca is already a scalar replacement of a larger aggregate,
         // Piece.Offset describes the offset inside the scalar.
diff --git a/lib/Transforms/Scalar/SampleProfile.cpp b/lib/Transforms/Scalar/SampleProfile.cpp
index f99fe3f5512..c8dfa54a4aa 100644
--- a/lib/Transforms/Scalar/SampleProfile.cpp
+++ b/lib/Transforms/Scalar/SampleProfile.cpp
@@ -224,7 +224,7 @@ unsigned SampleProfileLoader::getInstWeight(Instruction &Inst) {
   if (Lineno < HeaderLineno)
     return 0;
 
-  DILocation DIL = DLoc.get();
+  const DILocation *DIL = DLoc;
   int LOffset = Lineno - HeaderLineno;
   unsigned Discriminator = DIL->getDiscriminator();
   unsigned Weight = Samples->samplesAt(LOffset, Discriminator);
@@ -282,7 +282,7 @@ bool SampleProfileLoader::computeBlockWeights(Function &F) {
 /// \brief Find equivalence classes for the given block.
 ///
 /// This finds all the blocks that are guaranteed to execute the same
-/// number of times as \p BB1. To do this, it traverses all the the
+/// number of times as \p BB1. To do this, it traverses all the
 /// descendants of \p BB1 in the dominator or post-dominator tree.
 ///
 /// A block BB2 will be in the same equivalence class as \p BB1 if
@@ -580,6 +580,10 @@ void SampleProfileLoader::propagateWeights(Function &F) {
   bool Changed = true;
   unsigned i = 0;
 
+  // Add an entry count to the function using the samples gathered
+  // at the function entry.
+  F.setEntryCount(Samples->getHeadSamples());
+
   // Before propagation starts, build, for each block, a list of
   // unique predecessors and successors. This is necessary to handle
   // identical edges in multiway branches. Since we visit all blocks and all
@@ -642,7 +646,7 @@ void SampleProfileLoader::propagateWeights(Function &F) {
 /// \returns the line number where \p F is defined. If it returns 0,
 ///          it means that there is no debug information available for \p F.
 unsigned SampleProfileLoader::getFunctionLoc(Function &F) {
-  if (MDSubprogram *S = getDISubprogram(&F))
+  if (DISubprogram *S = getDISubprogram(&F))
     return S->getLine();
 
   // If could not find the start of \p F, emit a diagnostic to inform the user
diff --git a/lib/Transforms/Scalar/Scalar.cpp b/lib/Transforms/Scalar/Scalar.cpp
index 42095ae261e..d5d360571f8 100644
--- a/lib/Transforms/Scalar/Scalar.cpp
+++ b/lib/Transforms/Scalar/Scalar.cpp
@@ -74,11 +74,13 @@ void llvm::initializeScalarOpts(PassRegistry &Registry) {
   initializeSinkingPass(Registry);
   initializeTailCallElimPass(Registry);
   initializeSeparateConstOffsetFromGEPPass(Registry);
+  initializeSpeculativeExecutionPass(Registry);
   initializeStraightLineStrengthReducePass(Registry);
   initializeLoadCombinePass(Registry);
   initializePlaceBackedgeSafepointsImplPass(Registry);
   initializePlaceSafepointsPass(Registry);
   initializeFloat2IntPass(Registry);
+  initializeLoopDistributePass(Registry);
 }
 
 void LLVMInitializeScalarOpts(LLVMPassRegistryRef R) {
diff --git a/lib/Transforms/Scalar/ScalarReplAggregates.cpp b/lib/Transforms/Scalar/ScalarReplAggregates.cpp
index 693c5ae9708..d955da7ce75 100644
--- a/lib/Transforms/Scalar/ScalarReplAggregates.cpp
+++ b/lib/Transforms/Scalar/ScalarReplAggregates.cpp
@@ -1052,7 +1052,7 @@ class AllocaPromoter : public LoadAndStorePromoter {
   SmallVector<DbgDeclareInst *, 4> DDIs;
   SmallVector<DbgValueInst *, 4> DVIs;
 public:
-  AllocaPromoter(const SmallVectorImpl<Instruction*> &Insts, SSAUpdater &S,
+  AllocaPromoter(ArrayRef<Instruction*> Insts, SSAUpdater &S,
                  DIBuilder *DB)
     : LoadAndStorePromoter(Insts, S), AI(nullptr), DIB(DB) {}
 
@@ -1060,8 +1060,8 @@ class AllocaPromoter : public LoadAndStorePromoter {
     // Remember which alloca we're promoting (for isInstInList).
     this->AI = AI;
     if (auto *L = LocalAsMetadata::getIfExists(AI)) {
-      if (auto *DebugNode = MetadataAsValue::getIfExists(AI->getContext(), L)) {
-        for (User *U : DebugNode->users())
+      if (auto *DINode = MetadataAsValue::getIfExists(AI->getContext(), L)) {
+        for (User *U : DINode->users())
           if (DbgDeclareInst *DDI = dyn_cast<DbgDeclareInst>(U))
             DDIs.push_back(DDI);
           else if (DbgValueInst *DVI = dyn_cast<DbgValueInst>(U))
@@ -1117,9 +1117,9 @@ class AllocaPromoter : public LoadAndStorePromoter {
       } else {
         continue;
       }
-      DIB->insertDbgValueIntrinsic(Arg, 0, DIVariable(DVI->getVariable()),
-                                   DIExpression(DVI->getExpression()),
-                                   DVI->getDebugLoc(), Inst);
+      DIB->insertDbgValueIntrinsic(Arg, 0, DVI->getVariable(),
+                                   DVI->getExpression(), DVI->getDebugLoc(),
+                                   Inst);
     }
   }
 };
@@ -1140,8 +1140,8 @@ class AllocaPromoter : public LoadAndStorePromoter {
 /// the select can be loaded unconditionally.
 static bool isSafeSelectToSpeculate(SelectInst *SI) {
   const DataLayout &DL = SI->getModule()->getDataLayout();
-  bool TDerefable = SI->getTrueValue()->isDereferenceablePointer(DL);
-  bool FDerefable = SI->getFalseValue()->isDereferenceablePointer(DL);
+  bool TDerefable = isDereferenceablePointer(SI->getTrueValue(), DL);
+  bool FDerefable = isDereferenceablePointer(SI->getFalseValue(), DL);
 
   for (User *U : SI->users()) {
     LoadInst *LI = dyn_cast<LoadInst>(U);
@@ -1228,7 +1228,7 @@ static bool isSafePHIToSpeculate(PHINode *PN) {
 
     // If this pointer is always safe to load, or if we can prove that there is
     // already a load in the block, then we can move the load to the pred block.
-    if (InVal->isDereferenceablePointer(DL) ||
+    if (isDereferenceablePointer(InVal, DL) ||
         isSafeToLoadUnconditionally(InVal, Pred->getTerminator(), MaxAlign))
       continue;
 
diff --git a/lib/Transforms/Scalar/Scalarizer.cpp b/lib/Transforms/Scalar/Scalarizer.cpp
index d55dc6a20a0..3b8307c4316 100644
--- a/lib/Transforms/Scalar/Scalarizer.cpp
+++ b/lib/Transforms/Scalar/Scalarizer.cpp
@@ -247,6 +247,7 @@ bool Scalarizer::doInitialization(Module &M) {
 }
 
 bool Scalarizer::runOnFunction(Function &F) {
+  assert(Gathered.empty() && Scattered.empty());
   for (Function::iterator BBI = F.begin(), BBE = F.end(); BBI != BBE; ++BBI) {
     BasicBlock *BB = BBI;
     for (BasicBlock::iterator II = BB->begin(), IE = BB->end(); II != IE;) {
@@ -636,7 +637,9 @@ bool Scalarizer::visitStoreInst(StoreInst &SI) {
 // Delete the instructions that we scalarized.  If a full vector result
 // is still needed, recreate it using InsertElements.
 bool Scalarizer::finish() {
-  if (Gathered.empty())
+  // The presence of data in Gathered or Scattered indicates changes
+  // made to the Function.
+  if (Gathered.empty() && Scattered.empty())
     return false;
   for (GatherList::iterator GMI = Gathered.begin(), GME = Gathered.end();
        GMI != GME; ++GMI) {
diff --git a/lib/Transforms/Scalar/SeparateConstOffsetFromGEP.cpp b/lib/Transforms/Scalar/SeparateConstOffsetFromGEP.cpp
index 8af47530dbc..4a875311881 100644
--- a/lib/Transforms/Scalar/SeparateConstOffsetFromGEP.cpp
+++ b/lib/Transforms/Scalar/SeparateConstOffsetFromGEP.cpp
@@ -160,6 +160,7 @@
 #include "llvm/Analysis/ValueTracking.h"
 #include "llvm/IR/Constants.h"
 #include "llvm/IR/DataLayout.h"
+#include "llvm/IR/Dominators.h"
 #include "llvm/IR/Instructions.h"
 #include "llvm/IR/LLVMContext.h"
 #include "llvm/IR/Module.h"
@@ -167,6 +168,7 @@
 #include "llvm/Support/CommandLine.h"
 #include "llvm/Support/raw_ostream.h"
 #include "llvm/Transforms/Scalar.h"
+#include "llvm/Transforms/Utils/Local.h"
 #include "llvm/Target/TargetMachine.h"
 #include "llvm/Target/TargetSubtargetInfo.h"
 #include "llvm/IR/IRBuilder.h"
@@ -177,6 +179,13 @@ static cl::opt<bool> DisableSeparateConstOffsetFromGEP(
     "disable-separate-const-offset-from-gep", cl::init(false),
     cl::desc("Do not separate the constant offset from a GEP instruction"),
     cl::Hidden);
+// Setting this flag may emit false positives when the input module already
+// contains dead instructions. Therefore, we set it only in unit tests that are
+// free of dead code.
+static cl::opt<bool>
+    VerifyNoDeadCode("reassociate-geps-verify-no-dead-code", cl::init(false),
+                     cl::desc("Verify this pass produces no dead code"),
+                     cl::Hidden);
 
 namespace {
 
@@ -194,20 +203,26 @@ namespace {
 /// 5); nor can we transform (3 * (a + 5)) to (3 * a + 5), however in this case,
 /// -instcombine probably already optimized (3 * (a + 5)) to (3 * a + 15).
 class ConstantOffsetExtractor {
- public:
+public:
   /// Extracts a constant offset from the given GEP index. It returns the
   /// new index representing the remainder (equal to the original index minus
   /// the constant offset), or nullptr if we cannot extract a constant offset.
-  /// \p Idx    The given GEP index
-  /// \p GEP    The given GEP
-   static Value *Extract(Value *Idx, GetElementPtrInst *GEP);
+  /// \p Idx The given GEP index
+  /// \p GEP The given GEP
+  /// \p UserChainTail Outputs the tail of UserChain so that we can
+  ///                  garbage-collect unused instructions in UserChain.
+  static Value *Extract(Value *Idx, GetElementPtrInst *GEP,
+                        User *&UserChainTail, const DominatorTree *DT);
   /// Looks for a constant offset from the given GEP index without extracting
   /// it. It returns the numeric value of the extracted constant offset (0 if
   /// failed). The meaning of the arguments are the same as Extract.
-   static int64_t Find(Value *Idx, GetElementPtrInst *GEP);
+  static int64_t Find(Value *Idx, GetElementPtrInst *GEP,
+                      const DominatorTree *DT);
 
- private:
-   ConstantOffsetExtractor(Instruction *InsertionPt) : IP(InsertionPt) {}
+private:
+  ConstantOffsetExtractor(Instruction *InsertionPt, const DominatorTree *DT)
+      : IP(InsertionPt), DL(InsertionPt->getModule()->getDataLayout()), DT(DT) {
+  }
   /// Searches the expression that computes V for a non-zero constant C s.t.
   /// V can be reassociated into the form V' + C. If the searching is
   /// successful, returns C and update UserChain as a def-use chain from C to V;
@@ -265,12 +280,6 @@ class ConstantOffsetExtractor {
   /// returns "sext i32 (zext i16 V to i32) to i64".
   Value *applyExts(Value *V);
 
-  /// Returns true if LHS and RHS have no bits in common, i.e., LHS | RHS == 0.
-  bool NoCommonBits(Value *LHS, Value *RHS) const;
-  /// Computes which bits are known to be one or zero.
-  /// \p KnownOne Mask of all bits that are known to be one.
-  /// \p KnownZero Mask of all bits that are known to be zero.
-  void ComputeKnownBits(Value *V, APInt &KnownOne, APInt &KnownZero) const;
   /// A helper function that returns whether we can trace into the operands
   /// of binary operator BO for a constant offset.
   ///
@@ -292,28 +301,35 @@ class ConstantOffsetExtractor {
   /// sext/zext instructions along UserChain.
   SmallVector<CastInst *, 16> ExtInsts;
   Instruction *IP;  /// Insertion position of cloned instructions.
+  const DataLayout &DL;
+  const DominatorTree *DT;
 };
 
 /// \brief A pass that tries to split every GEP in the function into a variadic
 /// base and a constant offset. It is a FunctionPass because searching for the
 /// constant offset may inspect other basic blocks.
 class SeparateConstOffsetFromGEP : public FunctionPass {
- public:
+public:
   static char ID;
   SeparateConstOffsetFromGEP(const TargetMachine *TM = nullptr,
                              bool LowerGEP = false)
-      : FunctionPass(ID), TM(TM), LowerGEP(LowerGEP) {
+      : FunctionPass(ID), DL(nullptr), DT(nullptr), TM(TM), LowerGEP(LowerGEP) {
     initializeSeparateConstOffsetFromGEPPass(*PassRegistry::getPassRegistry());
   }
 
   void getAnalysisUsage(AnalysisUsage &AU) const override {
+    AU.addRequired<DominatorTreeWrapperPass>();
     AU.addRequired<TargetTransformInfoWrapperPass>();
     AU.setPreservesCFG();
   }
 
+  bool doInitialization(Module &M) override {
+    DL = &M.getDataLayout();
+    return false;
+  }
   bool runOnFunction(Function &F) override;
 
- private:
+private:
   /// Tries to split the given GEP into a variadic base and a constant offset,
   /// and returns true if the splitting succeeds.
   bool splitGEP(GetElementPtrInst *GEP);
@@ -357,7 +373,11 @@ class SeparateConstOffsetFromGEP : public FunctionPass {
   ///
   /// Verified in @i32_add in split-gep.ll
   bool canonicalizeArrayIndicesToPointerSize(GetElementPtrInst *GEP);
+  /// Verify F is free of dead code.
+  void verifyNoDeadCode(Function &F);
 
+  const DataLayout *DL;
+  const DominatorTree *DT;
   const TargetMachine *TM;
   /// Whether to lower a GEP with multiple indices into arithmetic operations or
   /// multiple GEPs with a single index.
@@ -370,6 +390,7 @@ INITIALIZE_PASS_BEGIN(
     SeparateConstOffsetFromGEP, "separate-const-offset-from-gep",
     "Split GEPs to a variadic base and a constant offset for better CSE", false,
     false)
+INITIALIZE_PASS_DEPENDENCY(DominatorTreeWrapperPass)
 INITIALIZE_PASS_DEPENDENCY(TargetTransformInfoWrapperPass)
 INITIALIZE_PASS_END(
     SeparateConstOffsetFromGEP, "separate-const-offset-from-gep",
@@ -398,7 +419,8 @@ bool ConstantOffsetExtractor::CanTraceInto(bool SignExtended,
   Value *LHS = BO->getOperand(0), *RHS = BO->getOperand(1);
   // Do not trace into "or" unless it is equivalent to "add". If LHS and RHS
   // don't have common bits, (LHS | RHS) is equivalent to (LHS + RHS).
-  if (BO->getOpcode() == Instruction::Or && !NoCommonBits(LHS, RHS))
+  if (BO->getOpcode() == Instruction::Or &&
+      !haveNoCommonBitsSet(LHS, RHS, DL, nullptr, BO, DT))
     return false;
 
   // In addition, tracing into BO requires that its surrounding s/zext (if
@@ -483,9 +505,8 @@ APInt ConstantOffsetExtractor::find(Value *V, bool SignExtended,
     ConstantOffset = CI->getValue();
   } else if (BinaryOperator *BO = dyn_cast<BinaryOperator>(V)) {
     // Trace into subexpressions for more hoisting opportunities.
-    if (CanTraceInto(SignExtended, ZeroExtended, BO, NonNegative)) {
+    if (CanTraceInto(SignExtended, ZeroExtended, BO, NonNegative))
       ConstantOffset = findInEitherOperand(BO, SignExtended, ZeroExtended);
-    }
   } else if (isa<SExtInst>(V)) {
     ConstantOffset = find(U->getOperand(0), /* SignExtended */ true,
                           ZeroExtended, NonNegative).sext(BitWidth);
@@ -582,6 +603,11 @@ Value *ConstantOffsetExtractor::removeConstOffset(unsigned ChainIndex) {
   }
 
   BinaryOperator *BO = cast<BinaryOperator>(UserChain[ChainIndex]);
+  assert(BO->getNumUses() <= 1 &&
+         "distributeExtsAndCloneChain clones each BinaryOperator in "
+         "UserChain, so no one should be used more than "
+         "once");
+
   unsigned OpNo = (BO->getOperand(0) == UserChain[ChainIndex - 1] ? 0 : 1);
   assert(BO->getOperand(OpNo) == UserChain[ChainIndex - 1]);
   Value *NextInChain = removeConstOffset(ChainIndex - 1);
@@ -594,6 +620,7 @@ Value *ConstantOffsetExtractor::removeConstOffset(unsigned ChainIndex) {
       return TheOther;
   }
 
+  BinaryOperator::BinaryOps NewOp = BO->getOpcode();
   if (BO->getOpcode() == Instruction::Or) {
     // Rebuild "or" as "add", because "or" may be invalid for the new
     // epxression.
@@ -608,72 +635,50 @@ Value *ConstantOffsetExtractor::removeConstOffset(unsigned ChainIndex) {
     //
     // Replacing the "or" with "add" is fine, because
     //   a | (b + 5) = a + (b + 5) = (a + b) + 5
-    if (OpNo == 0) {
-      return BinaryOperator::CreateAdd(NextInChain, TheOther, BO->getName(),
-                                       IP);
-    } else {
-      return BinaryOperator::CreateAdd(TheOther, NextInChain, BO->getName(),
-                                       IP);
-    }
+    NewOp = Instruction::Add;
   }
 
-  // We can reuse BO in this case, because the new expression shares the same
-  // instruction type and BO is used at most once.
-  assert(BO->getNumUses() <= 1 &&
-         "distributeExtsAndCloneChain clones each BinaryOperator in "
-         "UserChain, so no one should be used more than "
-         "once");
-  BO->setOperand(OpNo, NextInChain);
-  BO->setHasNoSignedWrap(false);
-  BO->setHasNoUnsignedWrap(false);
-  // Make sure it appears after all instructions we've inserted so far.
-  BO->moveBefore(IP);
-  return BO;
+  BinaryOperator *NewBO;
+  if (OpNo == 0) {
+    NewBO = BinaryOperator::Create(NewOp, NextInChain, TheOther, "", IP);
+  } else {
+    NewBO = BinaryOperator::Create(NewOp, TheOther, NextInChain, "", IP);
+  }
+  NewBO->takeName(BO);
+  return NewBO;
 }
 
-Value *ConstantOffsetExtractor::Extract(Value *Idx, GetElementPtrInst *GEP) {
-  ConstantOffsetExtractor Extractor(GEP);
+Value *ConstantOffsetExtractor::Extract(Value *Idx, GetElementPtrInst *GEP,
+                                        User *&UserChainTail,
+                                        const DominatorTree *DT) {
+  ConstantOffsetExtractor Extractor(GEP, DT);
   // Find a non-zero constant offset first.
   APInt ConstantOffset =
       Extractor.find(Idx, /* SignExtended */ false, /* ZeroExtended */ false,
                      GEP->isInBounds());
-  if (ConstantOffset == 0)
+  if (ConstantOffset == 0) {
+    UserChainTail = nullptr;
     return nullptr;
+  }
   // Separates the constant offset from the GEP index.
-  return Extractor.rebuildWithoutConstOffset();
+  Value *IdxWithoutConstOffset = Extractor.rebuildWithoutConstOffset();
+  UserChainTail = Extractor.UserChain.back();
+  return IdxWithoutConstOffset;
 }
 
-int64_t ConstantOffsetExtractor::Find(Value *Idx, GetElementPtrInst *GEP) {
+int64_t ConstantOffsetExtractor::Find(Value *Idx, GetElementPtrInst *GEP,
+                                      const DominatorTree *DT) {
   // If Idx is an index of an inbound GEP, Idx is guaranteed to be non-negative.
-  return ConstantOffsetExtractor(GEP)
+  return ConstantOffsetExtractor(GEP, DT)
       .find(Idx, /* SignExtended */ false, /* ZeroExtended */ false,
             GEP->isInBounds())
       .getSExtValue();
 }
 
-void ConstantOffsetExtractor::ComputeKnownBits(Value *V, APInt &KnownOne,
-                                               APInt &KnownZero) const {
-  IntegerType *IT = cast<IntegerType>(V->getType());
-  KnownOne = APInt(IT->getBitWidth(), 0);
-  KnownZero = APInt(IT->getBitWidth(), 0);
-  const DataLayout &DL = IP->getModule()->getDataLayout();
-  llvm::computeKnownBits(V, KnownZero, KnownOne, DL, 0);
-}
-
-bool ConstantOffsetExtractor::NoCommonBits(Value *LHS, Value *RHS) const {
-  assert(LHS->getType() == RHS->getType() &&
-         "LHS and RHS should have the same type");
-  APInt LHSKnownOne, LHSKnownZero, RHSKnownOne, RHSKnownZero;
-  ComputeKnownBits(LHS, LHSKnownOne, LHSKnownZero);
-  ComputeKnownBits(RHS, RHSKnownOne, RHSKnownZero);
-  return (LHSKnownZero | RHSKnownZero).isAllOnesValue();
-}
-
 bool SeparateConstOffsetFromGEP::canonicalizeArrayIndicesToPointerSize(
     GetElementPtrInst *GEP) {
   bool Changed = false;
-  const DataLayout &DL = GEP->getModule()->getDataLayout();
-  Type *IntPtrTy = DL.getIntPtrType(GEP->getType());
+  Type *IntPtrTy = DL->getIntPtrType(GEP->getType());
   gep_type_iterator GTI = gep_type_begin(*GEP);
   for (User::op_iterator I = GEP->op_begin() + 1, E = GEP->op_end();
        I != E; ++I, ++GTI) {
@@ -694,19 +699,18 @@ SeparateConstOffsetFromGEP::accumulateByteOffset(GetElementPtrInst *GEP,
   NeedsExtraction = false;
   int64_t AccumulativeByteOffset = 0;
   gep_type_iterator GTI = gep_type_begin(*GEP);
-  const DataLayout &DL = GEP->getModule()->getDataLayout();
   for (unsigned I = 1, E = GEP->getNumOperands(); I != E; ++I, ++GTI) {
     if (isa<SequentialType>(*GTI)) {
       // Tries to extract a constant offset from this GEP index.
       int64_t ConstantOffset =
-          ConstantOffsetExtractor::Find(GEP->getOperand(I), GEP);
+          ConstantOffsetExtractor::Find(GEP->getOperand(I), GEP, DT);
       if (ConstantOffset != 0) {
         NeedsExtraction = true;
         // A GEP may have multiple indices.  We accumulate the extracted
         // constant offset to a byte offset, and later offset the remainder of
         // the original GEP with this byte offset.
         AccumulativeByteOffset +=
-            ConstantOffset * DL.getTypeAllocSize(GTI.getIndexedType());
+            ConstantOffset * DL->getTypeAllocSize(GTI.getIndexedType());
       }
     } else if (LowerGEP) {
       StructType *StTy = cast<StructType>(*GTI);
@@ -715,7 +719,7 @@ SeparateConstOffsetFromGEP::accumulateByteOffset(GetElementPtrInst *GEP,
       if (Field != 0) {
         NeedsExtraction = true;
         AccumulativeByteOffset +=
-            DL.getStructLayout(StTy)->getElementOffset(Field);
+            DL->getStructLayout(StTy)->getElementOffset(Field);
       }
     }
   }
@@ -725,8 +729,7 @@ SeparateConstOffsetFromGEP::accumulateByteOffset(GetElementPtrInst *GEP,
 void SeparateConstOffsetFromGEP::lowerToSingleIndexGEPs(
     GetElementPtrInst *Variadic, int64_t AccumulativeByteOffset) {
   IRBuilder<> Builder(Variadic);
-  const DataLayout &DL = Variadic->getModule()->getDataLayout();
-  Type *IntPtrTy = DL.getIntPtrType(Variadic->getType());
+  Type *IntPtrTy = DL->getIntPtrType(Variadic->getType());
 
   Type *I8PtrTy =
       Builder.getInt8PtrTy(Variadic->getType()->getPointerAddressSpace());
@@ -746,7 +749,7 @@ void SeparateConstOffsetFromGEP::lowerToSingleIndexGEPs(
           continue;
 
       APInt ElementSize = APInt(IntPtrTy->getIntegerBitWidth(),
-                                DL.getTypeAllocSize(GTI.getIndexedType()));
+                                DL->getTypeAllocSize(GTI.getIndexedType()));
       // Scale the index by element size.
       if (ElementSize != 1) {
         if (ElementSize.isPowerOf2()) {
@@ -779,8 +782,7 @@ void
 SeparateConstOffsetFromGEP::lowerToArithmetics(GetElementPtrInst *Variadic,
                                                int64_t AccumulativeByteOffset) {
   IRBuilder<> Builder(Variadic);
-  const DataLayout &DL = Variadic->getModule()->getDataLayout();
-  Type *IntPtrTy = DL.getIntPtrType(Variadic->getType());
+  Type *IntPtrTy = DL->getIntPtrType(Variadic->getType());
 
   Value *ResultPtr = Builder.CreatePtrToInt(Variadic->getOperand(0), IntPtrTy);
   gep_type_iterator GTI = gep_type_begin(*Variadic);
@@ -796,7 +798,7 @@ SeparateConstOffsetFromGEP::lowerToArithmetics(GetElementPtrInst *Variadic,
           continue;
 
       APInt ElementSize = APInt(IntPtrTy->getIntegerBitWidth(),
-                                DL.getTypeAllocSize(GTI.getIndexedType()));
+                                DL->getTypeAllocSize(GTI.getIndexedType()));
       // Scale the index by element size.
       if (ElementSize != 1) {
         if (ElementSize.isPowerOf2()) {
@@ -850,9 +852,11 @@ bool SeparateConstOffsetFromGEP::splitGEP(GetElementPtrInst *GEP) {
     TargetTransformInfo &TTI =
         getAnalysis<TargetTransformInfoWrapperPass>().getTTI(
             *GEP->getParent()->getParent());
+    unsigned AddrSpace = GEP->getPointerAddressSpace();
     if (!TTI.isLegalAddressingMode(GEP->getType()->getElementType(),
                                    /*BaseGV=*/nullptr, AccumulativeByteOffset,
-                                   /*HasBaseReg=*/true, /*Scale=*/0)) {
+                                   /*HasBaseReg=*/true, /*Scale=*/0,
+                                   AddrSpace)) {
       return Changed;
     }
   }
@@ -869,9 +873,17 @@ bool SeparateConstOffsetFromGEP::splitGEP(GetElementPtrInst *GEP) {
     if (isa<SequentialType>(*GTI)) {
       // Splits this GEP index into a variadic part and a constant offset, and
       // uses the variadic part as the new index.
-      Value *NewIdx = ConstantOffsetExtractor::Extract(GEP->getOperand(I), GEP);
+      Value *OldIdx = GEP->getOperand(I);
+      User *UserChainTail;
+      Value *NewIdx =
+          ConstantOffsetExtractor::Extract(OldIdx, GEP, UserChainTail, DT);
       if (NewIdx != nullptr) {
+        // Switches to the index with the constant offset removed.
         GEP->setOperand(I, NewIdx);
+        // After switching to the new index, we can garbage-collect UserChain
+        // and the old index if they are not used.
+        RecursivelyDeleteTriviallyDeadInstructions(UserChainTail);
+        RecursivelyDeleteTriviallyDeadInstructions(OldIdx);
       }
     }
   }
@@ -946,10 +958,9 @@ bool SeparateConstOffsetFromGEP::splitGEP(GetElementPtrInst *GEP) {
   // Per ANSI C standard, signed / unsigned = unsigned and signed % unsigned =
   // unsigned.. Therefore, we cast ElementTypeSizeOfGEP to signed because it is
   // used with unsigned integers later.
-  const DataLayout &DL = GEP->getModule()->getDataLayout();
   int64_t ElementTypeSizeOfGEP = static_cast<int64_t>(
-      DL.getTypeAllocSize(GEP->getType()->getElementType()));
-  Type *IntPtrTy = DL.getIntPtrType(GEP->getType());
+      DL->getTypeAllocSize(GEP->getType()->getElementType()));
+  Type *IntPtrTy = DL->getIntPtrType(GEP->getType());
   if (AccumulativeByteOffset % ElementTypeSizeOfGEP == 0) {
     // Very likely. As long as %gep is natually aligned, the byte offset we
     // extracted should be a multiple of sizeof(*%gep).
@@ -996,6 +1007,8 @@ bool SeparateConstOffsetFromGEP::runOnFunction(Function &F) {
   if (DisableSeparateConstOffsetFromGEP)
     return false;
 
+  DT = &getAnalysis<DominatorTreeWrapperPass>().getDomTree();
+
   bool Changed = false;
   for (Function::iterator B = F.begin(), BE = F.end(); B != BE; ++B) {
     for (BasicBlock::iterator I = B->begin(), IE = B->end(); I != IE; ) {
@@ -1006,5 +1019,22 @@ bool SeparateConstOffsetFromGEP::runOnFunction(Function &F) {
       // already.
     }
   }
+
+  if (VerifyNoDeadCode)
+    verifyNoDeadCode(F);
+
   return Changed;
 }
+
+void SeparateConstOffsetFromGEP::verifyNoDeadCode(Function &F) {
+  for (auto &B : F) {
+    for (auto &I : B) {
+      if (isInstructionTriviallyDead(&I)) {
+        std::string ErrMessage;
+        raw_string_ostream RSO(ErrMessage);
+        RSO << "Dead instruction detected!\n" << I << "\n";
+        llvm_unreachable(RSO.str().c_str());
+      }
+    }
+  }
+}
diff --git a/lib/Transforms/Scalar/SimplifyCFGPass.cpp b/lib/Transforms/Scalar/SimplifyCFGPass.cpp
index 8566cd9736d..231411a16c0 100644
--- a/lib/Transforms/Scalar/SimplifyCFGPass.cpp
+++ b/lib/Transforms/Scalar/SimplifyCFGPass.cpp
@@ -48,8 +48,8 @@ UserBonusInstThreshold("bonus-inst-threshold", cl::Hidden, cl::init(1),
 
 STATISTIC(NumSimpl, "Number of blocks simplified");
 
-/// mergeEmptyReturnBlocks - If we have more than one empty (other than phi
-/// node) return blocks, merge them together to promote recursive block merging.
+/// If we have more than one empty (other than phi node) return blocks,
+/// merge them together to promote recursive block merging.
 static bool mergeEmptyReturnBlocks(Function &F) {
   bool Changed = false;
 
@@ -124,7 +124,7 @@ static bool mergeEmptyReturnBlocks(Function &F) {
   return Changed;
 }
 
-/// iterativelySimplifyCFG - Call SimplifyCFG on all the blocks in the function,
+/// Call SimplifyCFG on all the blocks in the function,
 /// iterating until no more changes are made.
 static bool iterativelySimplifyCFG(Function &F, const TargetTransformInfo &TTI,
                                    AssumptionCache *AC,
@@ -134,8 +134,7 @@ static bool iterativelySimplifyCFG(Function &F, const TargetTransformInfo &TTI,
   while (LocalChange) {
     LocalChange = false;
 
-    // Loop over all of the basic blocks and remove them if they are unneeded...
-    //
+    // Loop over all of the basic blocks and remove them if they are unneeded.
     for (Function::iterator BBIt = F.begin(); BBIt != F.end(); ) {
       if (SimplifyCFG(BBIt++, TTI, BonusInstThreshold, AC)) {
         LocalChange = true;
@@ -159,7 +158,7 @@ static bool simplifyFunctionCFG(Function &F, const TargetTransformInfo &TTI,
   // iterativelySimplifyCFG can (rarely) make some loops dead.  If this happens,
   // removeUnreachableBlocks is needed to nuke them, which means we should
   // iterate between the two optimizations.  We structure the code like this to
-  // avoid reruning iterativelySimplifyCFG if the second pass of
+  // avoid rerunning iterativelySimplifyCFG if the second pass of
   // removeUnreachableBlocks doesn't do anything.
   if (!removeUnreachableBlocks(F))
     return true;
@@ -193,11 +192,18 @@ namespace {
 struct CFGSimplifyPass : public FunctionPass {
   static char ID; // Pass identification, replacement for typeid
   unsigned BonusInstThreshold;
-  CFGSimplifyPass(int T = -1) : FunctionPass(ID) {
+  std::function<bool(const Function &)> PredicateFtor;
+
+  CFGSimplifyPass(int T = -1,
+                  std::function<bool(const Function &)> Ftor = nullptr)
+      : FunctionPass(ID), PredicateFtor(Ftor) {
     BonusInstThreshold = (T == -1) ? UserBonusInstThreshold : unsigned(T);
     initializeCFGSimplifyPassPass(*PassRegistry::getPassRegistry());
   }
   bool runOnFunction(Function &F) override {
+    if (PredicateFtor && !PredicateFtor(F))
+      return false;
+
     if (skipOptnoneFunction(F))
       return false;
 
@@ -224,7 +230,9 @@ INITIALIZE_PASS_END(CFGSimplifyPass, "simplifycfg", "Simplify the CFG", false,
                     false)
 
 // Public interface to the CFGSimplification pass
-FunctionPass *llvm::createCFGSimplificationPass(int Threshold) {
-  return new CFGSimplifyPass(Threshold);
+FunctionPass *
+llvm::createCFGSimplificationPass(int Threshold,
+                                  std::function<bool(const Function &)> Ftor) {
+  return new CFGSimplifyPass(Threshold, Ftor);
 }
 
diff --git a/lib/Transforms/Scalar/Sink.cpp b/lib/Transforms/Scalar/Sink.cpp
index b169d5612f0..17684c04dc4 100644
--- a/lib/Transforms/Scalar/Sink.cpp
+++ b/lib/Transforms/Scalar/Sink.cpp
@@ -163,15 +163,21 @@ static bool isSafeToMove(Instruction *Inst, AliasAnalysis *AA,
   }
 
   if (LoadInst *L = dyn_cast<LoadInst>(Inst)) {
-    AliasAnalysis::Location Loc = AA->getLocation(L);
+    MemoryLocation Loc = MemoryLocation::get(L);
     for (Instruction *S : Stores)
-      if (AA->getModRefInfo(S, Loc) & AliasAnalysis::Mod)
+      if (AA->getModRefInfo(S, Loc) & MRI_Mod)
         return false;
   }
 
   if (isa<TerminatorInst>(Inst) || isa<PHINode>(Inst))
     return false;
 
+  // Convergent operations can only be moved to control equivalent blocks.
+  if (auto CS = CallSite(Inst)) {
+    if (CS.hasFnAttr(Attribute::Convergent))
+      return false;
+  }
+
   return true;
 }
 
diff --git a/lib/Transforms/Scalar/SpeculativeExecution.cpp b/lib/Transforms/Scalar/SpeculativeExecution.cpp
new file mode 100644
index 00000000000..ff3f00a2e2f
--- /dev/null
+++ b/lib/Transforms/Scalar/SpeculativeExecution.cpp
@@ -0,0 +1,243 @@
+//===- SpeculativeExecution.cpp ---------------------------------*- C++ -*-===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This pass hoists instructions to enable speculative execution on
+// targets where branches are expensive. This is aimed at GPUs. It
+// currently works on simple if-then and if-then-else
+// patterns.
+//
+// Removing branches is not the only motivation for this
+// pass. E.g. consider this code and assume that there is no
+// addressing mode for multiplying by sizeof(*a):
+//
+//   if (b > 0)
+//     c = a[i + 1]
+//   if (d > 0)
+//     e = a[i + 2]
+//
+// turns into
+//
+//   p = &a[i + 1];
+//   if (b > 0)
+//     c = *p;
+//   q = &a[i + 2];
+//   if (d > 0)
+//     e = *q;
+//
+// which could later be optimized to
+//
+//   r = &a[i];
+//   if (b > 0)
+//     c = r[1];
+//   if (d > 0)
+//     e = r[2];
+//
+// Later passes sink back much of the speculated code that did not enable
+// further optimization.
+//
+// This pass is more aggressive than the function SpeculativeyExecuteBB in
+// SimplifyCFG. SimplifyCFG will not speculate if no selects are introduced and
+// it will speculate at most one instruction. It also will not speculate if
+// there is a value defined in the if-block that is only used in the then-block.
+// These restrictions make sense since the speculation in SimplifyCFG seems
+// aimed at introducing cheap selects, while this pass is intended to do more
+// aggressive speculation while counting on later passes to either capitalize on
+// that or clean it up.
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/ADT/SmallSet.h"
+#include "llvm/Analysis/TargetTransformInfo.h"
+#include "llvm/Analysis/ValueTracking.h"
+#include "llvm/IR/Instructions.h"
+#include "llvm/IR/Module.h"
+#include "llvm/IR/Operator.h"
+#include "llvm/Support/CommandLine.h"
+#include "llvm/Support/Debug.h"
+
+using namespace llvm;
+
+#define DEBUG_TYPE "speculative-execution"
+
+// The risk that speculation will not pay off increases with the
+// number of instructions speculated, so we put a limit on that.
+static cl::opt<unsigned> SpecExecMaxSpeculationCost(
+    "spec-exec-max-speculation-cost", cl::init(7), cl::Hidden,
+    cl::desc("Speculative execution is not applied to basic blocks where "
+             "the cost of the instructions to speculatively execute "
+             "exceeds this limit."));
+
+// Speculating just a few instructions from a larger block tends not
+// to be profitable and this limit prevents that. A reason for that is
+// that small basic blocks are more likely to be candidates for
+// further optimization.
+static cl::opt<unsigned> SpecExecMaxNotHoisted(
+    "spec-exec-max-not-hoisted", cl::init(5), cl::Hidden,
+    cl::desc("Speculative execution is not applied to basic blocks where the "
+             "number of instructions that would not be speculatively executed "
+             "exceeds this limit."));
+
+namespace {
+class SpeculativeExecution : public FunctionPass {
+ public:
+  static char ID;
+  SpeculativeExecution(): FunctionPass(ID) {}
+
+  void getAnalysisUsage(AnalysisUsage &AU) const override;
+  bool runOnFunction(Function &F) override;
+
+ private:
+  bool runOnBasicBlock(BasicBlock &B);
+  bool considerHoistingFromTo(BasicBlock &FromBlock, BasicBlock &ToBlock);
+
+  const TargetTransformInfo *TTI = nullptr;
+};
+} // namespace
+
+char SpeculativeExecution::ID = 0;
+INITIALIZE_PASS_BEGIN(SpeculativeExecution, "speculative-execution",
+                      "Speculatively execute instructions", false, false)
+INITIALIZE_PASS_DEPENDENCY(TargetTransformInfoWrapperPass)
+INITIALIZE_PASS_END(SpeculativeExecution, "speculative-execution",
+                      "Speculatively execute instructions", false, false)
+
+void SpeculativeExecution::getAnalysisUsage(AnalysisUsage &AU) const {
+  AU.addRequired<TargetTransformInfoWrapperPass>();
+}
+
+bool SpeculativeExecution::runOnFunction(Function &F) {
+  if (skipOptnoneFunction(F))
+    return false;
+
+  TTI = &getAnalysis<TargetTransformInfoWrapperPass>().getTTI(F);
+
+  bool Changed = false;
+  for (auto& B : F) {
+    Changed |= runOnBasicBlock(B);
+  }
+  return Changed;
+}
+
+bool SpeculativeExecution::runOnBasicBlock(BasicBlock &B) {
+  BranchInst *BI = dyn_cast<BranchInst>(B.getTerminator());
+  if (BI == nullptr)
+    return false;
+
+  if (BI->getNumSuccessors() != 2)
+    return false;
+  BasicBlock &Succ0 = *BI->getSuccessor(0);
+  BasicBlock &Succ1 = *BI->getSuccessor(1);
+
+  if (&B == &Succ0 || &B == &Succ1 || &Succ0 == &Succ1) {
+    return false;
+  }
+
+  // Hoist from if-then (triangle).
+  if (Succ0.getSinglePredecessor() != nullptr &&
+      Succ0.getSingleSuccessor() == &Succ1) {
+    return considerHoistingFromTo(Succ0, B);
+  }
+
+  // Hoist from if-else (triangle).
+  if (Succ1.getSinglePredecessor() != nullptr &&
+      Succ1.getSingleSuccessor() == &Succ0) {
+    return considerHoistingFromTo(Succ1, B);
+  }
+
+  // Hoist from if-then-else (diamond), but only if it is equivalent to
+  // an if-else or if-then due to one of the branches doing nothing.
+  if (Succ0.getSinglePredecessor() != nullptr &&
+      Succ1.getSinglePredecessor() != nullptr &&
+      Succ1.getSingleSuccessor() != nullptr &&
+      Succ1.getSingleSuccessor() != &B &&
+      Succ1.getSingleSuccessor() == Succ0.getSingleSuccessor()) {
+    // If a block has only one instruction, then that is a terminator
+    // instruction so that the block does nothing. This does happen.
+    if (Succ1.size() == 1) // equivalent to if-then
+      return considerHoistingFromTo(Succ0, B);
+    if (Succ0.size() == 1) // equivalent to if-else
+      return considerHoistingFromTo(Succ1, B);
+  }
+
+  return false;
+}
+
+static unsigned ComputeSpeculationCost(const Instruction *I,
+                                       const TargetTransformInfo &TTI) {
+  switch (Operator::getOpcode(I)) {
+    case Instruction::GetElementPtr:
+    case Instruction::Add:
+    case Instruction::Mul:
+    case Instruction::And:
+    case Instruction::Or:
+    case Instruction::Select:
+    case Instruction::Shl:
+    case Instruction::Sub:
+    case Instruction::LShr:
+    case Instruction::AShr:
+    case Instruction::Xor:
+    case Instruction::ZExt:
+    case Instruction::SExt:
+      return TTI.getUserCost(I);
+
+    default:
+      return UINT_MAX; // Disallow anything not whitelisted.
+  }
+}
+
+bool SpeculativeExecution::considerHoistingFromTo(BasicBlock &FromBlock,
+                                                  BasicBlock &ToBlock) {
+  SmallSet<const Instruction *, 8> NotHoisted;
+  const auto AllPrecedingUsesFromBlockHoisted = [&NotHoisted](User *U) {
+    for (Value* V : U->operand_values()) {
+      if (Instruction *I = dyn_cast<Instruction>(V)) {
+        if (NotHoisted.count(I) > 0)
+          return false;
+      }
+    }
+    return true;
+  };
+
+  unsigned TotalSpeculationCost = 0;
+  for (auto& I : FromBlock) {
+    const unsigned Cost = ComputeSpeculationCost(&I, *TTI);
+    if (Cost != UINT_MAX && isSafeToSpeculativelyExecute(&I) &&
+        AllPrecedingUsesFromBlockHoisted(&I)) {
+      TotalSpeculationCost += Cost;
+      if (TotalSpeculationCost > SpecExecMaxSpeculationCost)
+        return false;  // too much to hoist
+    } else {
+      NotHoisted.insert(&I);
+      if (NotHoisted.size() > SpecExecMaxNotHoisted)
+        return false; // too much left behind
+    }
+  }
+
+  if (TotalSpeculationCost == 0)
+    return false; // nothing to hoist
+
+  for (auto I = FromBlock.begin(); I != FromBlock.end();) {
+    // We have to increment I before moving Current as moving Current
+    // changes the list that I is iterating through.
+    auto Current = I;
+    ++I;
+    if (!NotHoisted.count(Current)) {
+      Current->moveBefore(ToBlock.getTerminator());
+    }
+  }
+  return true;
+}
+
+namespace llvm {
+
+FunctionPass *createSpeculativeExecutionPass() {
+  return new SpeculativeExecution();
+}
+
+}  // namespace llvm
diff --git a/lib/Transforms/Scalar/StraightLineStrengthReduce.cpp b/lib/Transforms/Scalar/StraightLineStrengthReduce.cpp
index 2fc93681e98..b2b886c4f11 100644
--- a/lib/Transforms/Scalar/StraightLineStrengthReduce.cpp
+++ b/lib/Transforms/Scalar/StraightLineStrengthReduce.cpp
@@ -61,6 +61,7 @@
 #include "llvm/ADT/FoldingSet.h"
 #include "llvm/Analysis/ScalarEvolution.h"
 #include "llvm/Analysis/TargetTransformInfo.h"
+#include "llvm/Analysis/ValueTracking.h"
 #include "llvm/IR/DataLayout.h"
 #include "llvm/IR/Dominators.h"
 #include "llvm/IR/IRBuilder.h"
@@ -68,6 +69,7 @@
 #include "llvm/IR/PatternMatch.h"
 #include "llvm/Support/raw_ostream.h"
 #include "llvm/Transforms/Scalar.h"
+#include "llvm/Transforms/Utils/Local.h"
 
 using namespace llvm;
 using namespace PatternMatch;
@@ -222,14 +224,17 @@ FunctionPass *llvm::createStraightLineStrengthReducePass() {
 bool StraightLineStrengthReduce::isBasisFor(const Candidate &Basis,
                                             const Candidate &C) {
   return (Basis.Ins != C.Ins && // skip the same instruction
+          // They must have the same type too. Basis.Base == C.Base doesn't
+          // guarantee their types are the same (PR23975).
+          Basis.Ins->getType() == C.Ins->getType() &&
           // Basis must dominate C in order to rewrite C with respect to Basis.
           DT->dominates(Basis.Ins->getParent(), C.Ins->getParent()) &&
           // They share the same base, stride, and candidate kind.
-          Basis.Base == C.Base &&
-          Basis.Stride == C.Stride &&
+          Basis.Base == C.Base && Basis.Stride == C.Stride &&
           Basis.CandidateKind == C.CandidateKind);
 }
 
+// TODO: use TTI->getGEPCost.
 static bool isGEPFoldable(GetElementPtrInst *GEP,
                           const TargetTransformInfo *TTI,
                           const DataLayout *DL) {
@@ -263,8 +268,10 @@ static bool isGEPFoldable(GetElementPtrInst *GEP,
       BaseOffset += DL->getStructLayout(STy)->getElementOffset(Field);
     }
   }
+
+  unsigned AddrSpace = GEP->getPointerAddressSpace();
   return TTI->isLegalAddressingMode(GEP->getType()->getElementType(), BaseGV,
-                                    BaseOffset, HasBaseReg, Scale);
+                                    BaseOffset, HasBaseReg, Scale, AddrSpace);
 }
 
 // Returns whether (Base + Index * Stride) can be folded to an addressing mode.
@@ -403,20 +410,37 @@ void StraightLineStrengthReduce::allocateCandidatesAndFindBasisForAdd(
   }
 }
 
+// Returns true if A matches B + C where C is constant.
+static bool matchesAdd(Value *A, Value *&B, ConstantInt *&C) {
+  return (match(A, m_Add(m_Value(B), m_ConstantInt(C))) ||
+          match(A, m_Add(m_ConstantInt(C), m_Value(B))));
+}
+
+// Returns true if A matches B | C where C is constant.
+static bool matchesOr(Value *A, Value *&B, ConstantInt *&C) {
+  return (match(A, m_Or(m_Value(B), m_ConstantInt(C))) ||
+          match(A, m_Or(m_ConstantInt(C), m_Value(B))));
+}
+
 void StraightLineStrengthReduce::allocateCandidatesAndFindBasisForMul(
     Value *LHS, Value *RHS, Instruction *I) {
   Value *B = nullptr;
   ConstantInt *Idx = nullptr;
-  // Only handle the canonical operand ordering.
-  if (match(LHS, m_Add(m_Value(B), m_ConstantInt(Idx)))) {
+  if (matchesAdd(LHS, B, Idx)) {
     // If LHS is in the form of "Base + Index", then I is in the form of
     // "(Base + Index) * RHS".
     allocateCandidatesAndFindBasis(Candidate::Mul, SE->getSCEV(B), Idx, RHS, I);
+  } else if (matchesOr(LHS, B, Idx) && haveNoCommonBitsSet(B, Idx, *DL)) {
+    // If LHS is in the form of "Base | Index" and Base and Index have no common
+    // bits set, then
+    //   Base | Index = Base + Index
+    // and I is thus in the form of "(Base + Index) * RHS".
+    allocateCandidatesAndFindBasis(Candidate::Mul, SE->getSCEV(B), Idx, RHS, I);
   } else {
     // Otherwise, at least try the form (LHS + 0) * RHS.
     ConstantInt *Zero = ConstantInt::get(cast<IntegerType>(I->getType()), 0);
     allocateCandidatesAndFindBasis(Candidate::Mul, SE->getSCEV(LHS), Zero, RHS,
-                                  I);
+                                   I);
   }
 }
 
@@ -490,31 +514,34 @@ void StraightLineStrengthReduce::allocateCandidatesAndFindBasisForGEP(
   if (GEP->getType()->isVectorTy())
     return;
 
-  const SCEV *GEPExpr = SE->getSCEV(GEP);
-  Type *IntPtrTy = DL->getIntPtrType(GEP->getType());
+  SmallVector<const SCEV *, 4> IndexExprs;
+  for (auto I = GEP->idx_begin(); I != GEP->idx_end(); ++I)
+    IndexExprs.push_back(SE->getSCEV(*I));
 
   gep_type_iterator GTI = gep_type_begin(GEP);
-  for (auto I = GEP->idx_begin(); I != GEP->idx_end(); ++I) {
+  for (unsigned I = 1, E = GEP->getNumOperands(); I != E; ++I) {
     if (!isa<SequentialType>(*GTI++))
       continue;
-    Value *ArrayIdx = *I;
-    // Compute the byte offset of this index.
+
+    const SCEV *OrigIndexExpr = IndexExprs[I - 1];
+    IndexExprs[I - 1] = SE->getConstant(OrigIndexExpr->getType(), 0);
+
+    // The base of this candidate is GEP's base plus the offsets of all
+    // indices except this current one.
+    const SCEV *BaseExpr = SE->getGEPExpr(GEP->getSourceElementType(),
+                                          SE->getSCEV(GEP->getPointerOperand()),
+                                          IndexExprs, GEP->isInBounds());
+    Value *ArrayIdx = GEP->getOperand(I);
     uint64_t ElementSize = DL->getTypeAllocSize(*GTI);
-    const SCEV *ElementSizeExpr = SE->getSizeOfExpr(IntPtrTy, *GTI);
-    const SCEV *ArrayIdxExpr = SE->getSCEV(ArrayIdx);
-    ArrayIdxExpr = SE->getTruncateOrSignExtend(ArrayIdxExpr, IntPtrTy);
-    const SCEV *LocalOffset =
-        SE->getMulExpr(ArrayIdxExpr, ElementSizeExpr, SCEV::FlagNSW);
-    // The base of this candidate equals GEPExpr less the byte offset of this
-    // index.
-    const SCEV *Base = SE->getMinusSCEV(GEPExpr, LocalOffset);
-    factorArrayIndex(ArrayIdx, Base, ElementSize, GEP);
+    factorArrayIndex(ArrayIdx, BaseExpr, ElementSize, GEP);
     // When ArrayIdx is the sext of a value, we try to factor that value as
     // well.  Handling this case is important because array indices are
     // typically sign-extended to the pointer size.
     Value *TruncatedArrayIdx = nullptr;
     if (match(ArrayIdx, m_SExt(m_Value(TruncatedArrayIdx))))
-      factorArrayIndex(TruncatedArrayIdx, Base, ElementSize, GEP);
+      factorArrayIndex(TruncatedArrayIdx, BaseExpr, ElementSize, GEP);
+
+    IndexExprs[I - 1] = OrigIndexExpr;
   }
 }
 
@@ -599,10 +626,24 @@ void StraightLineStrengthReduce::rewriteCandidateWithBasis(
   switch (C.CandidateKind) {
   case Candidate::Add:
   case Candidate::Mul:
+    // C = Basis + Bump
     if (BinaryOperator::isNeg(Bump)) {
+      // If Bump is a neg instruction, emit C = Basis - (-Bump).
       Reduced =
           Builder.CreateSub(Basis.Ins, BinaryOperator::getNegArgument(Bump));
+      // We only use the negative argument of Bump, and Bump itself may be
+      // trivially dead.
+      RecursivelyDeleteTriviallyDeadInstructions(Bump);
     } else {
+      // It's tempting to preserve nsw on Bump and/or Reduced. However, it's
+      // usually unsound, e.g.,
+      //
+      // X = (-2 +nsw 1) *nsw INT_MAX
+      // Y = (-2 +nsw 3) *nsw INT_MAX
+      //   =>
+      // Y = X + 2 * INT_MAX
+      //
+      // Neither + and * in the resultant expression are nsw.
       Reduced = Builder.CreateAdd(Basis.Ins, Bump);
     }
     break;
@@ -637,7 +678,6 @@ void StraightLineStrengthReduce::rewriteCandidateWithBasis(
   };
   Reduced->takeName(C.Ins);
   C.Ins->replaceAllUsesWith(Reduced);
-  C.Ins->dropAllReferences();
   // Unlink C.Ins so that we can skip other candidates also corresponding to
   // C.Ins. The actual deletion is postponed to the end of runOnFunction.
   C.Ins->removeFromParent();
@@ -670,8 +710,13 @@ bool StraightLineStrengthReduce::runOnFunction(Function &F) {
   }
 
   // Delete all unlink instructions.
-  for (auto I : UnlinkedInstructions) {
-    delete I;
+  for (auto *UnlinkedInst : UnlinkedInstructions) {
+    for (unsigned I = 0, E = UnlinkedInst->getNumOperands(); I != E; ++I) {
+      Value *Op = UnlinkedInst->getOperand(I);
+      UnlinkedInst->setOperand(I, nullptr);
+      RecursivelyDeleteTriviallyDeadInstructions(Op);
+    }
+    delete UnlinkedInst;
   }
   bool Ret = !UnlinkedInstructions.empty();
   UnlinkedInstructions.clear();
diff --git a/lib/Transforms/Scalar/TailRecursionElimination.cpp b/lib/Transforms/Scalar/TailRecursionElimination.cpp
index 9eef1327c3f..c7de2e2965c 100644
--- a/lib/Transforms/Scalar/TailRecursionElimination.cpp
+++ b/lib/Transforms/Scalar/TailRecursionElimination.cpp
@@ -158,6 +158,9 @@ bool TailCallElim::runOnFunction(Function &F) {
   if (skipOptnoneFunction(F))
     return false;
 
+  if (F.getFnAttribute("disable-tail-calls").getValueAsString() == "true")
+    return false;
+
   bool AllCallsAreTailCalls = false;
   bool Modified = markTails(F, AllCallsAreTailCalls);
   if (AllCallsAreTailCalls)
diff --git a/lib/Transforms/Utils/AddDiscriminators.cpp b/lib/Transforms/Utils/AddDiscriminators.cpp
index c1cd39abea7..e9f62391a44 100644
--- a/lib/Transforms/Utils/AddDiscriminators.cpp
+++ b/lib/Transforms/Utils/AddDiscriminators.cpp
@@ -174,14 +174,14 @@ bool AddDiscriminators::runOnFunction(Function &F) {
   for (Function::iterator I = F.begin(), E = F.end(); I != E; ++I) {
     BasicBlock *B = I;
     TerminatorInst *Last = B->getTerminator();
-    DILocation LastDIL = Last->getDebugLoc().get();
+    const DILocation *LastDIL = Last->getDebugLoc();
     if (!LastDIL)
       continue;
 
     for (unsigned I = 0; I < Last->getNumSuccessors(); ++I) {
       BasicBlock *Succ = Last->getSuccessor(I);
       Instruction *First = Succ->getFirstNonPHIOrDbgOrLifetime();
-      DILocation FirstDIL = First->getDebugLoc().get();
+      const DILocation *FirstDIL = First->getDebugLoc();
       if (!FirstDIL)
         continue;
 
@@ -194,18 +194,18 @@ bool AddDiscriminators::runOnFunction(Function &F) {
         // number for it.
         StringRef Filename = FirstDIL->getFilename();
         auto *Scope = FirstDIL->getScope();
-        DIFile File = Builder.createFile(Filename, Scope->getDirectory());
+        auto *File = Builder.createFile(Filename, Scope->getDirectory());
 
         // FIXME: Calculate the discriminator here, based on local information,
-        // and delete MDLocation::computeNewDiscriminator().  The current
+        // and delete DILocation::computeNewDiscriminator().  The current
         // solution gives different results depending on other modules in the
         // same context.  All we really need is to discriminate between
         // FirstDIL and LastDIL -- a local map would suffice.
         unsigned Discriminator = FirstDIL->computeNewDiscriminator();
-        DILexicalBlockFile NewScope =
+        auto *NewScope =
             Builder.createLexicalBlockFile(Scope, File, Discriminator);
         auto *NewDIL =
-            MDLocation::get(Ctx, FirstDIL->getLine(), FirstDIL->getColumn(),
+            DILocation::get(Ctx, FirstDIL->getLine(), FirstDIL->getColumn(),
                             NewScope, FirstDIL->getInlinedAt());
         DebugLoc newDebugLoc = NewDIL;
 
diff --git a/lib/Transforms/Utils/BasicBlockUtils.cpp b/lib/Transforms/Utils/BasicBlockUtils.cpp
index b4552576e46..1bf92e35abc 100644
--- a/lib/Transforms/Utils/BasicBlockUtils.cpp
+++ b/lib/Transforms/Utils/BasicBlockUtils.cpp
@@ -65,7 +65,7 @@ void llvm::DeleteDeadBlock(BasicBlock *BB) {
 /// any single-entry PHI nodes in it, fold them away.  This handles the case
 /// when all entries to the PHI nodes in a block are guaranteed equal, such as
 /// when the block has exactly one predecessor.
-void llvm::FoldSingleEntryPHINodes(BasicBlock *BB, AliasAnalysis *AA,
+void llvm::FoldSingleEntryPHINodes(BasicBlock *BB,
                                    MemoryDependenceAnalysis *MemDep) {
   if (!isa<PHINode>(BB->begin())) return;
 
@@ -77,8 +77,6 @@ void llvm::FoldSingleEntryPHINodes(BasicBlock *BB, AliasAnalysis *AA,
 
     if (MemDep)
       MemDep->removeInstruction(PN);  // Memdep updates AA itself.
-    else if (AA && isa<PointerType>(PN->getType()))
-      AA->deleteValue(PN);
 
     PN->eraseFromParent();
   }
@@ -108,7 +106,7 @@ bool llvm::DeleteDeadPHIs(BasicBlock *BB, const TargetLibraryInfo *TLI) {
 /// MergeBlockIntoPredecessor - Attempts to merge a block into its predecessor,
 /// if possible.  The return value indicates success or failure.
 bool llvm::MergeBlockIntoPredecessor(BasicBlock *BB, DominatorTree *DT,
-                                     LoopInfo *LI, AliasAnalysis *AA,
+                                     LoopInfo *LI,
                                      MemoryDependenceAnalysis *MemDep) {
   // Don't merge away blocks who have their address taken.
   if (BB->hasAddressTaken()) return false;
@@ -119,8 +117,9 @@ bool llvm::MergeBlockIntoPredecessor(BasicBlock *BB, DominatorTree *DT,
 
   // Don't break self-loops.
   if (PredBB == BB) return false;
-  // Don't break invokes.
-  if (isa<InvokeInst>(PredBB->getTerminator())) return false;
+  // Don't break unwinding instructions.
+  if (PredBB->getTerminator()->isExceptional())
+    return false;
 
   succ_iterator SI(succ_begin(PredBB)), SE(succ_end(PredBB));
   BasicBlock *OnlySucc = BB;
@@ -136,8 +135,8 @@ bool llvm::MergeBlockIntoPredecessor(BasicBlock *BB, DominatorTree *DT,
   // Can't merge if there is PHI loop.
   for (BasicBlock::iterator BI = BB->begin(), BE = BB->end(); BI != BE; ++BI) {
     if (PHINode *PN = dyn_cast<PHINode>(BI)) {
-      for (unsigned i = 0, e = PN->getNumIncomingValues(); i != e; ++i)
-        if (PN->getIncomingValue(i) == PN)
+      for (Value *IncValue : PN->incoming_values())
+        if (IncValue == PN)
           return false;
     } else
       break;
@@ -145,7 +144,7 @@ bool llvm::MergeBlockIntoPredecessor(BasicBlock *BB, DominatorTree *DT,
 
   // Begin by getting rid of unneeded PHIs.
   if (isa<PHINode>(BB->front()))
-    FoldSingleEntryPHINodes(BB, AA, MemDep);
+    FoldSingleEntryPHINodes(BB, MemDep);
 
   // Delete the unconditional branch from the predecessor...
   PredBB->getInstList().pop_back();
@@ -211,6 +210,11 @@ void llvm::ReplaceInstWithInst(BasicBlock::InstListType &BIL,
   assert(I->getParent() == nullptr &&
          "ReplaceInstWithInst: Instruction already inserted into basic block!");
 
+  // Copy debug location to newly added instruction, if it wasn't already set
+  // by the caller.
+  if (!I->getDebugLoc())
+    I->setDebugLoc(BI->getDebugLoc());
+
   // Insert the new instruction into the basic block...
   BasicBlock::iterator New = BIL.insert(BI, I);
 
@@ -280,7 +284,7 @@ llvm::SplitAllCriticalEdges(Function &F,
 BasicBlock *llvm::SplitBlock(BasicBlock *Old, Instruction *SplitPt,
                              DominatorTree *DT, LoopInfo *LI) {
   BasicBlock::iterator SplitIt = SplitPt;
-  while (isa<PHINode>(SplitIt) || isa<LandingPadInst>(SplitIt))
+  while (isa<PHINode>(SplitIt) || SplitIt->isEHPad())
     ++SplitIt;
   BasicBlock *New = Old->splitBasicBlock(SplitIt, Old->getName()+".split");
 
@@ -388,7 +392,7 @@ static void UpdateAnalysisInformation(BasicBlock *OldBB, BasicBlock *NewBB,
 /// from NewBB. This also updates AliasAnalysis, if available.
 static void UpdatePHINodes(BasicBlock *OrigBB, BasicBlock *NewBB,
                            ArrayRef<BasicBlock *> Preds, BranchInst *BI,
-                           AliasAnalysis *AA, bool HasLoopExit) {
+                           bool HasLoopExit) {
   // Otherwise, create a new PHI node in NewBB for each PHI node in OrigBB.
   SmallPtrSet<BasicBlock *, 16> PredSet(Preds.begin(), Preds.end());
   for (BasicBlock::iterator I = OrigBB->begin(); isa<PHINode>(I); ) {
@@ -435,8 +439,6 @@ static void UpdatePHINodes(BasicBlock *OrigBB, BasicBlock *NewBB,
     // Create the new PHI node, insert it into NewBB at the end of the block
     PHINode *NewPHI =
         PHINode::Create(PN->getType(), Preds.size(), PN->getName() + ".ph", BI);
-    if (AA)
-      AA->copyValue(PN, NewPHI);
 
     // NOTE! This loop walks backwards for a reason! First off, this minimizes
     // the cost of removal if we end up removing a large number of values, and
@@ -471,26 +473,30 @@ static void UpdatePHINodes(BasicBlock *OrigBB, BasicBlock *NewBB,
 ///
 BasicBlock *llvm::SplitBlockPredecessors(BasicBlock *BB,
                                          ArrayRef<BasicBlock *> Preds,
-                                         const char *Suffix, AliasAnalysis *AA,
-                                         DominatorTree *DT, LoopInfo *LI,
-                                         bool PreserveLCSSA) {
+                                         const char *Suffix, DominatorTree *DT,
+                                         LoopInfo *LI, bool PreserveLCSSA) {
+  // Do not attempt to split that which cannot be split.
+  if (!BB->canSplitPredecessors())
+    return nullptr;
+
   // For the landingpads we need to act a bit differently.
   // Delegate this work to the SplitLandingPadPredecessors.
   if (BB->isLandingPad()) {
     SmallVector<BasicBlock*, 2> NewBBs;
     std::string NewName = std::string(Suffix) + ".split-lp";
 
-    SplitLandingPadPredecessors(BB, Preds, Suffix, NewName.c_str(),
-                                NewBBs, AA, DT, LI, PreserveLCSSA);
+    SplitLandingPadPredecessors(BB, Preds, Suffix, NewName.c_str(), NewBBs, DT,
+                                LI, PreserveLCSSA);
     return NewBBs[0];
   }
 
   // Create new basic block, insert right before the original block.
-  BasicBlock *NewBB = BasicBlock::Create(BB->getContext(), BB->getName()+Suffix,
-                                         BB->getParent(), BB);
+  BasicBlock *NewBB = BasicBlock::Create(
+      BB->getContext(), BB->getName() + Suffix, BB->getParent(), BB);
 
   // The new block unconditionally branches to the old block.
   BranchInst *BI = BranchInst::Create(BB, NewBB);
+  BI->setDebugLoc(BB->getFirstNonPHI()->getDebugLoc());
 
   // Move the edges from Preds to point to NewBB instead of BB.
   for (unsigned i = 0, e = Preds.size(); i != e; ++i) {
@@ -519,7 +525,7 @@ BasicBlock *llvm::SplitBlockPredecessors(BasicBlock *BB,
                             HasLoopExit);
 
   // Update the PHI nodes in BB with the values coming from NewBB.
-  UpdatePHINodes(BB, NewBB, Preds, BI, AA, HasLoopExit);
+  UpdatePHINodes(BB, NewBB, Preds, BI, HasLoopExit);
   return NewBB;
 }
 
@@ -540,8 +546,8 @@ void llvm::SplitLandingPadPredecessors(BasicBlock *OrigBB,
                                        ArrayRef<BasicBlock *> Preds,
                                        const char *Suffix1, const char *Suffix2,
                                        SmallVectorImpl<BasicBlock *> &NewBBs,
-                                       AliasAnalysis *AA, DominatorTree *DT,
-                                       LoopInfo *LI, bool PreserveLCSSA) {
+                                       DominatorTree *DT, LoopInfo *LI,
+                                       bool PreserveLCSSA) {
   assert(OrigBB->isLandingPad() && "Trying to split a non-landing pad!");
 
   // Create a new basic block for OrigBB's predecessors listed in Preds. Insert
@@ -553,6 +559,7 @@ void llvm::SplitLandingPadPredecessors(BasicBlock *OrigBB,
 
   // The new block unconditionally branches to the old block.
   BranchInst *BI1 = BranchInst::Create(OrigBB, NewBB1);
+  BI1->setDebugLoc(OrigBB->getFirstNonPHI()->getDebugLoc());
 
   // Move the edges from Preds to point to NewBB1 instead of OrigBB.
   for (unsigned i = 0, e = Preds.size(); i != e; ++i) {
@@ -569,7 +576,7 @@ void llvm::SplitLandingPadPredecessors(BasicBlock *OrigBB,
                             HasLoopExit);
 
   // Update the PHI nodes in OrigBB with the values coming from NewBB1.
-  UpdatePHINodes(OrigBB, NewBB1, Preds, BI1, AA, HasLoopExit);
+  UpdatePHINodes(OrigBB, NewBB1, Preds, BI1, HasLoopExit);
 
   // Move the remaining edges from OrigBB to point to NewBB2.
   SmallVector<BasicBlock*, 8> NewBB2Preds;
@@ -593,6 +600,7 @@ void llvm::SplitLandingPadPredecessors(BasicBlock *OrigBB,
 
     // The new block unconditionally branches to the old block.
     BranchInst *BI2 = BranchInst::Create(OrigBB, NewBB2);
+    BI2->setDebugLoc(OrigBB->getFirstNonPHI()->getDebugLoc());
 
     // Move the remaining edges from OrigBB to point to NewBB2.
     for (SmallVectorImpl<BasicBlock*>::iterator
@@ -605,7 +613,7 @@ void llvm::SplitLandingPadPredecessors(BasicBlock *OrigBB,
                               PreserveLCSSA, HasLoopExit);
 
     // Update the PHI nodes in OrigBB with the values coming from NewBB2.
-    UpdatePHINodes(OrigBB, NewBB2, NewBB2Preds, BI2, AA, HasLoopExit);
+    UpdatePHINodes(OrigBB, NewBB2, NewBB2Preds, BI2, HasLoopExit);
   }
 
   LandingPadInst *LPad = OrigBB->getLandingPadInst();
@@ -713,7 +721,6 @@ TerminatorInst *llvm::SplitBlockAndInsertIfThen(Value *Cond,
   CheckTerm->setDebugLoc(SplitBefore->getDebugLoc());
   BranchInst *HeadNewTerm =
     BranchInst::Create(/*ifTrue*/ThenBlock, /*ifFalse*/Tail, Cond);
-  HeadNewTerm->setDebugLoc(SplitBefore->getDebugLoc());
   HeadNewTerm->setMetadata(LLVMContext::MD_prof, BranchWeights);
   ReplaceInstWithInst(HeadOldTerm, HeadNewTerm);
 
@@ -763,7 +770,6 @@ void llvm::SplitBlockAndInsertIfThenElse(Value *Cond, Instruction *SplitBefore,
   (*ElseTerm)->setDebugLoc(SplitBefore->getDebugLoc());
   BranchInst *HeadNewTerm =
     BranchInst::Create(/*ifTrue*/ThenBlock, /*ifFalse*/ElseBlock, Cond);
-  HeadNewTerm->setDebugLoc(SplitBefore->getDebugLoc());
   HeadNewTerm->setMetadata(LLVMContext::MD_prof, BranchWeights);
   ReplaceInstWithInst(HeadOldTerm, HeadNewTerm);
 }
diff --git a/lib/Transforms/Utils/BreakCriticalEdges.cpp b/lib/Transforms/Utils/BreakCriticalEdges.cpp
index 7e83c9eeceb..565374c2e2f 100644
--- a/lib/Transforms/Utils/BreakCriticalEdges.cpp
+++ b/lib/Transforms/Utils/BreakCriticalEdges.cpp
@@ -197,7 +197,6 @@ BasicBlock *llvm::SplitCriticalEdge(TerminatorInst *TI, unsigned SuccNum,
   }
 
   // If we have nothing to update, just return.
-  auto *AA = Options.AA;
   auto *DT = Options.DT;
   auto *LI = Options.LI;
   if (!DT && !LI)
@@ -322,7 +321,7 @@ BasicBlock *llvm::SplitCriticalEdge(TerminatorInst *TI, unsigned SuccNum,
           assert(!DestBB->isLandingPad() &&
                  "We don't split edges to landing pads!");
           BasicBlock *NewExitBB = SplitBlockPredecessors(
-              DestBB, LoopPreds, "split", AA, DT, LI, Options.PreserveLCSSA);
+              DestBB, LoopPreds, "split", DT, LI, Options.PreserveLCSSA);
           if (Options.PreserveLCSSA)
             createPHIsForSplitLoopExit(LoopPreds, NewExitBB, DestBB);
         }
diff --git a/lib/Transforms/Utils/BuildLibCalls.cpp b/lib/Transforms/Utils/BuildLibCalls.cpp
index 671cbfec0a2..8aa7b2a65ba 100644
--- a/lib/Transforms/Utils/BuildLibCalls.cpp
+++ b/lib/Transforms/Utils/BuildLibCalls.cpp
@@ -74,7 +74,7 @@ Value *llvm::EmitStrNLen(Value *Ptr, Value *MaxLen, IRBuilder<> &B,
       M->getOrInsertFunction("strnlen", AttributeSet::get(M->getContext(), AS),
                              DL.getIntPtrType(Context), B.getInt8PtrTy(),
                              DL.getIntPtrType(Context), nullptr);
-  CallInst *CI = B.CreateCall2(StrNLen, CastToCStr(Ptr, B), MaxLen, "strnlen");
+  CallInst *CI = B.CreateCall(StrNLen, {CastToCStr(Ptr, B), MaxLen}, "strnlen");
   if (const Function *F = dyn_cast<Function>(StrNLen->stripPointerCasts()))
     CI->setCallingConv(F->getCallingConv());
 
@@ -100,8 +100,8 @@ Value *llvm::EmitStrChr(Value *Ptr, char C, IRBuilder<> &B,
                                             AttributeSet::get(M->getContext(),
                                                              AS),
                                             I8Ptr, I8Ptr, I32Ty, nullptr);
-  CallInst *CI = B.CreateCall2(StrChr, CastToCStr(Ptr, B),
-                               ConstantInt::get(I32Ty, C), "strchr");
+  CallInst *CI = B.CreateCall(
+      StrChr, {CastToCStr(Ptr, B), ConstantInt::get(I32Ty, C)}, "strchr");
   if (const Function *F = dyn_cast<Function>(StrChr->stripPointerCasts()))
     CI->setCallingConv(F->getCallingConv());
   return CI;
@@ -124,8 +124,8 @@ Value *llvm::EmitStrNCmp(Value *Ptr1, Value *Ptr2, Value *Len, IRBuilder<> &B,
   Value *StrNCmp = M->getOrInsertFunction(
       "strncmp", AttributeSet::get(M->getContext(), AS), B.getInt32Ty(),
       B.getInt8PtrTy(), B.getInt8PtrTy(), DL.getIntPtrType(Context), nullptr);
-  CallInst *CI = B.CreateCall3(StrNCmp, CastToCStr(Ptr1, B),
-                               CastToCStr(Ptr2, B), Len, "strncmp");
+  CallInst *CI = B.CreateCall(
+      StrNCmp, {CastToCStr(Ptr1, B), CastToCStr(Ptr2, B), Len}, "strncmp");
 
   if (const Function *F = dyn_cast<Function>(StrNCmp->stripPointerCasts()))
     CI->setCallingConv(F->getCallingConv());
@@ -149,8 +149,8 @@ Value *llvm::EmitStrCpy(Value *Dst, Value *Src, IRBuilder<> &B,
   Value *StrCpy = M->getOrInsertFunction(Name,
                                          AttributeSet::get(M->getContext(), AS),
                                          I8Ptr, I8Ptr, I8Ptr, nullptr);
-  CallInst *CI = B.CreateCall2(StrCpy, CastToCStr(Dst, B), CastToCStr(Src, B),
-                               Name);
+  CallInst *CI =
+      B.CreateCall(StrCpy, {CastToCStr(Dst, B), CastToCStr(Src, B)}, Name);
   if (const Function *F = dyn_cast<Function>(StrCpy->stripPointerCasts()))
     CI->setCallingConv(F->getCallingConv());
   return CI;
@@ -174,8 +174,8 @@ Value *llvm::EmitStrNCpy(Value *Dst, Value *Src, Value *Len, IRBuilder<> &B,
                                                             AS),
                                           I8Ptr, I8Ptr, I8Ptr,
                                           Len->getType(), nullptr);
-  CallInst *CI = B.CreateCall3(StrNCpy, CastToCStr(Dst, B), CastToCStr(Src, B),
-                               Len, "strncpy");
+  CallInst *CI = B.CreateCall(
+      StrNCpy, {CastToCStr(Dst, B), CastToCStr(Src, B), Len}, "strncpy");
   if (const Function *F = dyn_cast<Function>(StrNCpy->stripPointerCasts()))
     CI->setCallingConv(F->getCallingConv());
   return CI;
@@ -201,7 +201,7 @@ Value *llvm::EmitMemCpyChk(Value *Dst, Value *Src, Value *Len, Value *ObjSize,
       DL.getIntPtrType(Context), nullptr);
   Dst = CastToCStr(Dst, B);
   Src = CastToCStr(Src, B);
-  CallInst *CI = B.CreateCall4(MemCpy, Dst, Src, Len, ObjSize);
+  CallInst *CI = B.CreateCall(MemCpy, {Dst, Src, Len, ObjSize});
   if (const Function *F = dyn_cast<Function>(MemCpy->stripPointerCasts()))
     CI->setCallingConv(F->getCallingConv());
   return CI;
@@ -222,7 +222,7 @@ Value *llvm::EmitMemChr(Value *Ptr, Value *Val, Value *Len, IRBuilder<> &B,
   Value *MemChr = M->getOrInsertFunction(
       "memchr", AttributeSet::get(M->getContext(), AS), B.getInt8PtrTy(),
       B.getInt8PtrTy(), B.getInt32Ty(), DL.getIntPtrType(Context), nullptr);
-  CallInst *CI = B.CreateCall3(MemChr, CastToCStr(Ptr, B), Val, Len, "memchr");
+  CallInst *CI = B.CreateCall(MemChr, {CastToCStr(Ptr, B), Val, Len}, "memchr");
 
   if (const Function *F = dyn_cast<Function>(MemChr->stripPointerCasts()))
     CI->setCallingConv(F->getCallingConv());
@@ -247,8 +247,8 @@ Value *llvm::EmitMemCmp(Value *Ptr1, Value *Ptr2, Value *Len, IRBuilder<> &B,
   Value *MemCmp = M->getOrInsertFunction(
       "memcmp", AttributeSet::get(M->getContext(), AS), B.getInt32Ty(),
       B.getInt8PtrTy(), B.getInt8PtrTy(), DL.getIntPtrType(Context), nullptr);
-  CallInst *CI = B.CreateCall3(MemCmp, CastToCStr(Ptr1, B), CastToCStr(Ptr2, B),
-                               Len, "memcmp");
+  CallInst *CI = B.CreateCall(
+      MemCmp, {CastToCStr(Ptr1, B), CastToCStr(Ptr2, B), Len}, "memcmp");
 
   if (const Function *F = dyn_cast<Function>(MemCmp->stripPointerCasts()))
     CI->setCallingConv(F->getCallingConv());
@@ -304,7 +304,7 @@ Value *llvm::EmitBinaryFloatFnCall(Value *Op1, Value *Op2, StringRef Name,
   Module *M = B.GetInsertBlock()->getParent()->getParent();
   Value *Callee = M->getOrInsertFunction(Name, Op1->getType(),
                                          Op1->getType(), Op2->getType(), nullptr);
-  CallInst *CI = B.CreateCall2(Callee, Op1, Op2, Name);
+  CallInst *CI = B.CreateCall(Callee, {Op1, Op2}, Name);
   CI->setAttributes(Attrs);
   if (const Function *F = dyn_cast<Function>(Callee->stripPointerCasts()))
     CI->setCallingConv(F->getCallingConv());
@@ -384,7 +384,7 @@ Value *llvm::EmitFPutC(Value *Char, Value *File, IRBuilder<> &B,
                                File->getType(), nullptr);
   Char = B.CreateIntCast(Char, B.getInt32Ty(), /*isSigned*/true,
                          "chari");
-  CallInst *CI = B.CreateCall2(F, Char, File, "fputc");
+  CallInst *CI = B.CreateCall(F, {Char, File}, "fputc");
 
   if (const Function *Fn = dyn_cast<Function>(F->stripPointerCasts()))
     CI->setCallingConv(Fn->getCallingConv());
@@ -416,7 +416,7 @@ Value *llvm::EmitFPutS(Value *Str, Value *File, IRBuilder<> &B,
     F = M->getOrInsertFunction(FPutsName, B.getInt32Ty(),
                                B.getInt8PtrTy(),
                                File->getType(), nullptr);
-  CallInst *CI = B.CreateCall2(F, CastToCStr(Str, B), File, "fputs");
+  CallInst *CI = B.CreateCall(F, {CastToCStr(Str, B), File}, "fputs");
 
   if (const Function *Fn = dyn_cast<Function>(F->stripPointerCasts()))
     CI->setCallingConv(Fn->getCallingConv());
@@ -450,8 +450,8 @@ Value *llvm::EmitFWrite(Value *Ptr, Value *Size, Value *File, IRBuilder<> &B,
                                DL.getIntPtrType(Context), File->getType(),
                                nullptr);
   CallInst *CI =
-      B.CreateCall4(F, CastToCStr(Ptr, B), Size,
-                    ConstantInt::get(DL.getIntPtrType(Context), 1), File);
+      B.CreateCall(F, {CastToCStr(Ptr, B), Size,
+                       ConstantInt::get(DL.getIntPtrType(Context), 1), File});
 
   if (const Function *Fn = dyn_cast<Function>(F->stripPointerCasts()))
     CI->setCallingConv(Fn->getCallingConv());
diff --git a/lib/Transforms/Utils/CMakeLists.txt b/lib/Transforms/Utils/CMakeLists.txt
index 470e2d09132..716e655affb 100644
--- a/lib/Transforms/Utils/CMakeLists.txt
+++ b/lib/Transforms/Utils/CMakeLists.txt
@@ -22,6 +22,7 @@ add_llvm_library(LLVMTransformUtils
   LoopUnroll.cpp
   LoopUnrollRuntime.cpp
   LoopUtils.cpp
+  LoopVersioning.cpp
   LowerInvoke.cpp
   LowerSwitch.cpp
   Mem2Reg.cpp
diff --git a/lib/Transforms/Utils/CloneFunction.cpp b/lib/Transforms/Utils/CloneFunction.cpp
index f200b588194..cc4d6c6fb19 100644
--- a/lib/Transforms/Utils/CloneFunction.cpp
+++ b/lib/Transforms/Utils/CloneFunction.cpp
@@ -17,6 +17,7 @@
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/Analysis/ConstantFolding.h"
 #include "llvm/Analysis/InstructionSimplify.h"
+#include "llvm/Analysis/LoopInfo.h"
 #include "llvm/IR/CFG.h"
 #include "llvm/IR/Constants.h"
 #include "llvm/IR/DebugInfo.h"
@@ -154,9 +155,10 @@ void llvm::CloneFunctionInto(Function *NewFunc, const Function *OldFunc,
                        TypeMapper, Materializer);
 }
 
-// Find the MDNode which corresponds to the DISubprogram data that described F.
-static MDNode* FindSubprogram(const Function *F, DebugInfoFinder &Finder) {
-  for (DISubprogram Subprogram : Finder.subprograms()) {
+// Find the MDNode which corresponds to the subprogram data that described F.
+static DISubprogram *FindSubprogram(const Function *F,
+                                    DebugInfoFinder &Finder) {
+  for (DISubprogram *Subprogram : Finder.subprograms()) {
     if (Subprogram->describes(F))
       return Subprogram;
   }
@@ -165,7 +167,8 @@ static MDNode* FindSubprogram(const Function *F, DebugInfoFinder &Finder) {
 
 // Add an operand to an existing MDNode. The new operand will be added at the
 // back of the operand list.
-static void AddOperand(DICompileUnit CU, MDSubprogramArray SPs, Metadata *NewSP) {
+static void AddOperand(DICompileUnit *CU, DISubprogramArray SPs,
+                       Metadata *NewSP) {
   SmallVector<Metadata *, 16> NewSPs;
   NewSPs.reserve(SPs.size() + 1);
   for (auto *SP : SPs)
@@ -181,16 +184,16 @@ static void CloneDebugInfoMetadata(Function *NewFunc, const Function *OldFunc,
   DebugInfoFinder Finder;
   Finder.processModule(*OldFunc->getParent());
 
-  const MDNode *OldSubprogramMDNode = FindSubprogram(OldFunc, Finder);
+  const DISubprogram *OldSubprogramMDNode = FindSubprogram(OldFunc, Finder);
   if (!OldSubprogramMDNode) return;
 
   // Ensure that OldFunc appears in the map.
   // (if it's already there it must point to NewFunc anyway)
   VMap[OldFunc] = NewFunc;
-  DISubprogram NewSubprogram =
-      cast<MDSubprogram>(MapMetadata(OldSubprogramMDNode, VMap));
+  auto *NewSubprogram =
+      cast<DISubprogram>(MapMetadata(OldSubprogramMDNode, VMap));
 
-  for (DICompileUnit CU : Finder.compile_units()) {
+  for (auto *CU : Finder.compile_units()) {
     auto Subprograms = CU->getSubprograms();
     // If the compile unit's function list contains the old function, it should
     // also contain the new one.
@@ -718,3 +721,68 @@ void llvm::CloneAndPruneFunctionInto(Function *NewFunc, const Function *OldFunc,
                             ModuleLevelChanges, Returns, NameSuffix, CodeInfo,
                             nullptr);
 }
+
+/// \brief Remaps instructions in \p Blocks using the mapping in \p VMap.
+void llvm::remapInstructionsInBlocks(
+    const SmallVectorImpl<BasicBlock *> &Blocks, ValueToValueMapTy &VMap) {
+  // Rewrite the code to refer to itself.
+  for (auto *BB : Blocks)
+    for (auto &Inst : *BB)
+      RemapInstruction(&Inst, VMap,
+                       RF_NoModuleLevelChanges | RF_IgnoreMissingEntries);
+}
+
+/// \brief Clones a loop \p OrigLoop.  Returns the loop and the blocks in \p
+/// Blocks.
+///
+/// Updates LoopInfo and DominatorTree assuming the loop is dominated by block
+/// \p LoopDomBB.  Insert the new blocks before block specified in \p Before.
+Loop *llvm::cloneLoopWithPreheader(BasicBlock *Before, BasicBlock *LoopDomBB,
+                                   Loop *OrigLoop, ValueToValueMapTy &VMap,
+                                   const Twine &NameSuffix, LoopInfo *LI,
+                                   DominatorTree *DT,
+                                   SmallVectorImpl<BasicBlock *> &Blocks) {
+  Function *F = OrigLoop->getHeader()->getParent();
+  Loop *ParentLoop = OrigLoop->getParentLoop();
+
+  Loop *NewLoop = new Loop();
+  if (ParentLoop)
+    ParentLoop->addChildLoop(NewLoop);
+  else
+    LI->addTopLevelLoop(NewLoop);
+
+  BasicBlock *OrigPH = OrigLoop->getLoopPreheader();
+  assert(OrigPH && "No preheader");
+  BasicBlock *NewPH = CloneBasicBlock(OrigPH, VMap, NameSuffix, F);
+  // To rename the loop PHIs.
+  VMap[OrigPH] = NewPH;
+  Blocks.push_back(NewPH);
+
+  // Update LoopInfo.
+  if (ParentLoop)
+    ParentLoop->addBasicBlockToLoop(NewPH, *LI);
+
+  // Update DominatorTree.
+  DT->addNewBlock(NewPH, LoopDomBB);
+
+  for (BasicBlock *BB : OrigLoop->getBlocks()) {
+    BasicBlock *NewBB = CloneBasicBlock(BB, VMap, NameSuffix, F);
+    VMap[BB] = NewBB;
+
+    // Update LoopInfo.
+    NewLoop->addBasicBlockToLoop(NewBB, *LI);
+
+    // Update DominatorTree.
+    BasicBlock *IDomBB = DT->getNode(BB)->getIDom()->getBlock();
+    DT->addNewBlock(NewBB, cast<BasicBlock>(VMap[IDomBB]));
+
+    Blocks.push_back(NewBB);
+  }
+
+  // Move them physically from the end of the block list.
+  F->getBasicBlockList().splice(Before, F->getBasicBlockList(), NewPH);
+  F->getBasicBlockList().splice(Before, F->getBasicBlockList(),
+                                NewLoop->getHeader(), F->end());
+
+  return NewLoop;
+}
diff --git a/lib/Transforms/Utils/CloneModule.cpp b/lib/Transforms/Utils/CloneModule.cpp
index fae9ff5bce0..61f1811e7b4 100644
--- a/lib/Transforms/Utils/CloneModule.cpp
+++ b/lib/Transforms/Utils/CloneModule.cpp
@@ -69,9 +69,7 @@ Module *llvm::CloneModule(const Module *M, ValueToValueMapTy &VMap) {
   for (Module::const_alias_iterator I = M->alias_begin(), E = M->alias_end();
        I != E; ++I) {
     auto *PTy = cast<PointerType>(I->getType());
-    auto *GA =
-        GlobalAlias::create(PTy->getElementType(), PTy->getAddressSpace(),
-                            I->getLinkage(), I->getName(), New);
+    auto *GA = GlobalAlias::create(PTy, I->getLinkage(), I->getName(), New);
     GA->copyAttributesFrom(I);
     VMap[I] = GA;
   }
@@ -101,7 +99,11 @@ Module *llvm::CloneModule(const Module *M, ValueToValueMapTy &VMap) {
 
       SmallVector<ReturnInst*, 8> Returns;  // Ignore returns cloned.
       CloneFunctionInto(F, I, VMap, /*ModuleLevelChanges=*/true, Returns);
+
     }
+
+    if (I->hasPersonalityFn())
+      F->setPersonalityFn(MapValue(I->getPersonalityFn(), VMap));
   }
 
   // And aliases
diff --git a/lib/Transforms/Utils/InlineFunction.cpp b/lib/Transforms/Utils/InlineFunction.cpp
index a08ffbeb329..a7515bbf961 100644
--- a/lib/Transforms/Utils/InlineFunction.cpp
+++ b/lib/Transforms/Utils/InlineFunction.cpp
@@ -63,8 +63,8 @@ bool llvm::InlineFunction(InvokeInst *II, InlineFunctionInfo &IFI,
 }
 
 namespace {
-  /// A class for recording information about inlining through an invoke.
-  class InvokeInliningInfo {
+  /// A class for recording information about inlining a landing pad.
+  class LandingPadInliningInfo {
     BasicBlock *OuterResumeDest; ///< Destination of the invoke's unwind.
     BasicBlock *InnerResumeDest; ///< Destination for the callee's resume.
     LandingPadInst *CallerLPad;  ///< LandingPadInst associated with the invoke.
@@ -72,7 +72,7 @@ namespace {
     SmallVector<Value*, 8> UnwindDestPHIValues;
 
   public:
-    InvokeInliningInfo(InvokeInst *II)
+    LandingPadInliningInfo(InvokeInst *II)
       : OuterResumeDest(II->getUnwindDest()), InnerResumeDest(nullptr),
         CallerLPad(nullptr), InnerEHValuesPHI(nullptr) {
       // If there are PHI nodes in the unwind destination block, we need to keep
@@ -124,7 +124,7 @@ namespace {
 }
 
 /// Get or create a target for the branch from ResumeInsts.
-BasicBlock *InvokeInliningInfo::getInnerResumeDest() {
+BasicBlock *LandingPadInliningInfo::getInnerResumeDest() {
   if (InnerResumeDest) return InnerResumeDest;
 
   // Split the landing pad.
@@ -162,8 +162,8 @@ BasicBlock *InvokeInliningInfo::getInnerResumeDest() {
 /// When the landing pad block has only one predecessor, this is a simple
 /// branch. When there is more than one predecessor, we need to split the
 /// landing pad block after the landingpad instruction and jump to there.
-void InvokeInliningInfo::forwardResume(ResumeInst *RI,
-                               SmallPtrSetImpl<LandingPadInst*> &InlinedLPads) {
+void LandingPadInliningInfo::forwardResume(
+    ResumeInst *RI, SmallPtrSetImpl<LandingPadInst *> &InlinedLPads) {
   BasicBlock *Dest = getInnerResumeDest();
   BasicBlock *Src = RI->getParent();
 
@@ -182,8 +182,8 @@ void InvokeInliningInfo::forwardResume(ResumeInst *RI,
 /// This function analyze BB to see if there are any calls, and if so,
 /// it rewrites them to be invokes that jump to InvokeDest and fills in the PHI
 /// nodes in that block with the values specified in InvokeDestPHIValues.
-static void HandleCallsInBlockInlinedThroughInvoke(BasicBlock *BB,
-                                                   InvokeInliningInfo &Invoke) {
+static BasicBlock *
+HandleCallsInBlockInlinedThroughInvoke(BasicBlock *BB, BasicBlock *UnwindEdge) {
   for (BasicBlock::iterator BBI = BB->begin(), E = BB->end(); BBI != E; ) {
     Instruction *I = BBI++;
 
@@ -206,8 +206,7 @@ static void HandleCallsInBlockInlinedThroughInvoke(BasicBlock *BB,
     // Create the new invoke instruction.
     ImmutableCallSite CS(CI);
     SmallVector<Value*, 8> InvokeArgs(CS.arg_begin(), CS.arg_end());
-    InvokeInst *II = InvokeInst::Create(CI->getCalledValue(), Split,
-                                        Invoke.getOuterResumeDest(),
+    InvokeInst *II = InvokeInst::Create(CI->getCalledValue(), Split, UnwindEdge,
                                         InvokeArgs, CI->getName(), BB);
     II->setDebugLoc(CI->getDebugLoc());
     II->setCallingConv(CI->getCallingConv());
@@ -219,12 +218,9 @@ static void HandleCallsInBlockInlinedThroughInvoke(BasicBlock *BB,
 
     // Delete the original call
     Split->getInstList().pop_front();
-
-    // Update any PHI nodes in the exceptional block to indicate that there is
-    // now a new entry in them.
-    Invoke.addIncomingPHIValuesFor(BB);
-    return;
+    return BB;
   }
+  return nullptr;
 }
 
 /// If we inlined an invoke site, we need to convert calls
@@ -233,8 +229,8 @@ static void HandleCallsInBlockInlinedThroughInvoke(BasicBlock *BB,
 /// II is the invoke instruction being inlined.  FirstNewBlock is the first
 /// block of the inlined code (the last block is the end of the function),
 /// and InlineCodeInfo is information about the code that got inlined.
-static void HandleInlinedInvoke(InvokeInst *II, BasicBlock *FirstNewBlock,
-                                ClonedCodeInfo &InlinedCodeInfo) {
+static void HandleInlinedLandingPad(InvokeInst *II, BasicBlock *FirstNewBlock,
+                                    ClonedCodeInfo &InlinedCodeInfo) {
   BasicBlock *InvokeDest = II->getUnwindDest();
 
   Function *Caller = FirstNewBlock->getParent();
@@ -242,7 +238,7 @@ static void HandleInlinedInvoke(InvokeInst *II, BasicBlock *FirstNewBlock,
   // The inlined code is currently at the end of the function, scan from the
   // start of the inlined code to its end, checking for stuff we need to
   // rewrite.
-  InvokeInliningInfo Invoke(II);
+  LandingPadInliningInfo Invoke(II);
 
   // Get all of the inlined landing pad instructions.
   SmallPtrSet<LandingPadInst*, 16> InlinedLPads;
@@ -264,7 +260,11 @@ static void HandleInlinedInvoke(InvokeInst *II, BasicBlock *FirstNewBlock,
 
   for (Function::iterator BB = FirstNewBlock, E = Caller->end(); BB != E; ++BB){
     if (InlinedCodeInfo.ContainsCalls)
-      HandleCallsInBlockInlinedThroughInvoke(BB, Invoke);
+      if (BasicBlock *NewBB = HandleCallsInBlockInlinedThroughInvoke(
+              BB, Invoke.getOuterResumeDest()))
+        // Update any PHI nodes in the exceptional block to indicate that there
+        // is now a new entry in them.
+        Invoke.addIncomingPHIValuesFor(NewBB);
 
     // Forward any resumes that are remaining here.
     if (ResumeInst *RI = dyn_cast<ResumeInst>(BB->getTerminator()))
@@ -278,6 +278,102 @@ static void HandleInlinedInvoke(InvokeInst *II, BasicBlock *FirstNewBlock,
   InvokeDest->removePredecessor(II->getParent());
 }
 
+/// If we inlined an invoke site, we need to convert calls
+/// in the body of the inlined function into invokes.
+///
+/// II is the invoke instruction being inlined.  FirstNewBlock is the first
+/// block of the inlined code (the last block is the end of the function),
+/// and InlineCodeInfo is information about the code that got inlined.
+static void HandleInlinedEHPad(InvokeInst *II, BasicBlock *FirstNewBlock,
+                               ClonedCodeInfo &InlinedCodeInfo) {
+  BasicBlock *UnwindDest = II->getUnwindDest();
+  Function *Caller = FirstNewBlock->getParent();
+
+  assert(UnwindDest->getFirstNonPHI()->isEHPad() && "unexpected BasicBlock!");
+
+  // If there are PHI nodes in the unwind destination block, we need to keep
+  // track of which values came into them from the invoke before removing the
+  // edge from this block.
+  SmallVector<Value *, 8> UnwindDestPHIValues;
+  llvm::BasicBlock *InvokeBB = II->getParent();
+  for (Instruction &I : *UnwindDest) {
+    // Save the value to use for this edge.
+    PHINode *PHI = dyn_cast<PHINode>(&I);
+    if (!PHI)
+      break;
+    UnwindDestPHIValues.push_back(PHI->getIncomingValueForBlock(InvokeBB));
+  }
+
+  // Add incoming-PHI values to the unwind destination block for the given basic
+  // block, using the values for the original invoke's source block.
+  auto UpdatePHINodes = [&](BasicBlock *Src) {
+    BasicBlock::iterator I = UnwindDest->begin();
+    for (Value *V : UnwindDestPHIValues) {
+      PHINode *PHI = cast<PHINode>(I);
+      PHI->addIncoming(V, Src);
+      ++I;
+    }
+  };
+
+  // Forward EH terminator instructions to the caller's invoke destination.
+  // This is as simple as connect all the instructions which 'unwind to caller'
+  // to the invoke destination.
+  for (Function::iterator BB = FirstNewBlock, E = Caller->end(); BB != E;
+       ++BB) {
+    Instruction *I = BB->getFirstNonPHI();
+    if (I->isEHPad()) {
+      if (auto *CEPI = dyn_cast<CatchEndPadInst>(I)) {
+        if (CEPI->unwindsToCaller()) {
+          CatchEndPadInst::Create(CEPI->getContext(), UnwindDest, CEPI);
+          CEPI->eraseFromParent();
+          UpdatePHINodes(BB);
+        }
+      } else if (auto *TPI = dyn_cast<TerminatePadInst>(I)) {
+        if (TPI->unwindsToCaller()) {
+          SmallVector<Value *, 3> TerminatePadArgs;
+          for (Value *Operand : TPI->operands())
+            TerminatePadArgs.push_back(Operand);
+          TerminatePadInst::Create(TPI->getContext(), UnwindDest, TPI);
+          TPI->eraseFromParent();
+          UpdatePHINodes(BB);
+        }
+      } else if (auto *CPI = dyn_cast<CleanupPadInst>(I)) {
+        if (CPI->getNumOperands() == 0) {
+          CleanupPadInst::Create(CPI->getType(), {UnwindDest}, CPI->getName(),
+                                 CPI);
+          CPI->eraseFromParent();
+        }
+      } else {
+        assert(isa<CatchPadInst>(I));
+      }
+    }
+
+    if (auto *CRI = dyn_cast<CleanupReturnInst>(BB->getTerminator())) {
+      if (CRI->unwindsToCaller()) {
+        CleanupReturnInst::Create(CRI->getContext(), CRI->getReturnValue(),
+                                  UnwindDest, CRI);
+        CRI->eraseFromParent();
+        UpdatePHINodes(BB);
+      }
+    }
+  }
+
+  if (InlinedCodeInfo.ContainsCalls)
+    for (Function::iterator BB = FirstNewBlock, E = Caller->end(); BB != E;
+         ++BB)
+      if (BasicBlock *NewBB =
+              HandleCallsInBlockInlinedThroughInvoke(BB, UnwindDest))
+        // Update any PHI nodes in the exceptional block to indicate that there
+        // is now a new entry in them.
+        UpdatePHINodes(NewBB);
+
+  // Now that everything is happy, we have one final detail.  The PHI nodes in
+  // the exception destination block still have entries due to the original
+  // invoke instruction. Eliminate these entries (which might even delete the
+  // PHI node) now.
+  UnwindDest->removePredecessor(InvokeBB);
+}
+
 /// When inlining a function that contains noalias scope metadata,
 /// this metadata needs to be cloned so that the inlined blocks
 /// have different "unqiue scopes" at every call site. Were this not done, then
@@ -481,9 +577,9 @@ static void AddAliasScopeMetadata(CallSite CS, ValueToValueMapTy &VMap,
 
         IsFuncCall = true;
         if (AA) {
-          AliasAnalysis::ModRefBehavior MRB = AA->getModRefBehavior(ICS);
-          if (MRB == AliasAnalysis::OnlyAccessesArgumentPointees ||
-              MRB == AliasAnalysis::OnlyReadsArgumentPointees)
+          FunctionModRefBehavior MRB = AA->getModRefBehavior(ICS);
+          if (MRB == FMRB_OnlyAccessesArgumentPointees ||
+              MRB == FMRB_OnlyReadsArgumentPointees)
             IsArgMemOnlyCall = true;
         }
 
@@ -830,17 +926,16 @@ static bool hasLifetimeMarkers(AllocaInst *AI) {
 /// Rebuild the entire inlined-at chain for this instruction so that the top of
 /// the chain now is inlined-at the new call site.
 static DebugLoc
-updateInlinedAtInfo(DebugLoc DL, MDLocation *InlinedAtNode,
-                    LLVMContext &Ctx,
-                    DenseMap<const MDLocation *, MDLocation *> &IANodes) {
-  SmallVector<MDLocation*, 3> InlinedAtLocations;
-  MDLocation *Last = InlinedAtNode;
-  MDLocation *CurInlinedAt = DL;
+updateInlinedAtInfo(DebugLoc DL, DILocation *InlinedAtNode, LLVMContext &Ctx,
+                    DenseMap<const DILocation *, DILocation *> &IANodes) {
+  SmallVector<DILocation *, 3> InlinedAtLocations;
+  DILocation *Last = InlinedAtNode;
+  DILocation *CurInlinedAt = DL;
 
   // Gather all the inlined-at nodes
-  while (MDLocation *IA = CurInlinedAt->getInlinedAt()) {
+  while (DILocation *IA = CurInlinedAt->getInlinedAt()) {
     // Skip any we've already built nodes for
-    if (MDLocation *Found = IANodes[IA]) {
+    if (DILocation *Found = IANodes[IA]) {
       Last = Found;
       break;
     }
@@ -852,10 +947,9 @@ updateInlinedAtInfo(DebugLoc DL, MDLocation *InlinedAtNode,
   // Starting from the top, rebuild the nodes to point to the new inlined-at
   // location (then rebuilding the rest of the chain behind it) and update the
   // map of already-constructed inlined-at nodes.
-  for (auto I = InlinedAtLocations.rbegin(), E = InlinedAtLocations.rend();
-       I != E; ++I) {
-    const MDLocation *MD = *I;
-    Last = IANodes[MD] = MDLocation::getDistinct(
+  for (const DILocation *MD : make_range(InlinedAtLocations.rbegin(),
+                                         InlinedAtLocations.rend())) {
+    Last = IANodes[MD] = DILocation::getDistinct(
         Ctx, MD->getLine(), MD->getColumn(), MD->getScope(), Last);
   }
 
@@ -873,18 +967,18 @@ static void fixupLineNumbers(Function *Fn, Function::iterator FI,
     return;
 
   auto &Ctx = Fn->getContext();
-  MDLocation *InlinedAtNode = TheCallDL;
+  DILocation *InlinedAtNode = TheCallDL;
 
   // Create a unique call site, not to be confused with any other call from the
   // same location.
-  InlinedAtNode = MDLocation::getDistinct(
+  InlinedAtNode = DILocation::getDistinct(
       Ctx, InlinedAtNode->getLine(), InlinedAtNode->getColumn(),
       InlinedAtNode->getScope(), InlinedAtNode->getInlinedAt());
 
   // Cache the inlined-at nodes as they're built so they are reused, without
   // this every instruction's inlined-at chain would become distinct from each
   // other.
-  DenseMap<const MDLocation *, MDLocation *> IANodes;
+  DenseMap<const DILocation *, DILocation *> IANodes;
 
   for (; FI != Fn->end(); ++FI) {
     for (BasicBlock::iterator BI = FI->begin(), BE = FI->end();
@@ -950,35 +1044,23 @@ bool llvm::InlineFunction(CallSite CS, InlineFunctionInfo &IFI,
   }
 
   // Get the personality function from the callee if it contains a landing pad.
-  Value *CalleePersonality = nullptr;
-  for (Function::const_iterator I = CalledFunc->begin(), E = CalledFunc->end();
-       I != E; ++I)
-    if (const InvokeInst *II = dyn_cast<InvokeInst>(I->getTerminator())) {
-      const BasicBlock *BB = II->getUnwindDest();
-      const LandingPadInst *LP = BB->getLandingPadInst();
-      CalleePersonality = LP->getPersonalityFn();
-      break;
-    }
+  Constant *CalledPersonality =
+      CalledFunc->hasPersonalityFn() ? CalledFunc->getPersonalityFn() : nullptr;
 
   // Find the personality function used by the landing pads of the caller. If it
   // exists, then check to see that it matches the personality function used in
   // the callee.
-  if (CalleePersonality) {
-    for (Function::const_iterator I = Caller->begin(), E = Caller->end();
-         I != E; ++I)
-      if (const InvokeInst *II = dyn_cast<InvokeInst>(I->getTerminator())) {
-        const BasicBlock *BB = II->getUnwindDest();
-        const LandingPadInst *LP = BB->getLandingPadInst();
-
-        // If the personality functions match, then we can perform the
-        // inlining. Otherwise, we can't inline.
-        // TODO: This isn't 100% true. Some personality functions are proper
-        //       supersets of others and can be used in place of the other.
-        if (LP->getPersonalityFn() != CalleePersonality)
-          return false;
-
-        break;
-      }
+  Constant *CallerPersonality =
+      Caller->hasPersonalityFn() ? Caller->getPersonalityFn() : nullptr;
+  if (CalledPersonality) {
+    if (!CallerPersonality)
+      Caller->setPersonalityFn(CalledPersonality);
+    // If the personality functions match, then we can perform the
+    // inlining. Otherwise, we can't inline.
+    // TODO: This isn't 100% true. Some personality functions are proper
+    //       supersets of others and can be used in place of the other.
+    else if (CalledPersonality != CallerPersonality)
+      return false;
   }
 
   // Get an iterator to the last basic block in the function, which will have
@@ -1167,7 +1249,11 @@ bool llvm::InlineFunction(CallSite CS, InlineFunctionInfo &IFI,
         Type *AllocaType = AI->getAllocatedType();
         uint64_t AllocaTypeSize = DL.getTypeAllocSize(AllocaType);
         uint64_t AllocaArraySize = AIArraySize->getLimitedValue();
-        assert(AllocaArraySize > 0 && "array size of AllocaInst is zero");
+
+        // Don't add markers for zero-sized allocas.
+        if (AllocaArraySize == 0)
+          continue;
+
         // Check that array size doesn't saturate uint64_t and doesn't
         // overflow when it's multiplied by type size.
         if (AllocaArraySize != ~0ULL &&
@@ -1199,7 +1285,7 @@ bool llvm::InlineFunction(CallSite CS, InlineFunctionInfo &IFI,
 
     // Insert the llvm.stacksave.
     CallInst *SavedPtr = IRBuilder<>(FirstNewBlock, FirstNewBlock->begin())
-      .CreateCall(StackSave, "savedstack");
+                             .CreateCall(StackSave, {}, "savedstack");
 
     // Insert a call to llvm.stackrestore before any return instructions in the
     // inlined function.
@@ -1214,8 +1300,15 @@ bool llvm::InlineFunction(CallSite CS, InlineFunctionInfo &IFI,
 
   // If we are inlining for an invoke instruction, we must make sure to rewrite
   // any call instructions into invoke instructions.
-  if (InvokeInst *II = dyn_cast<InvokeInst>(TheCall))
-    HandleInlinedInvoke(II, FirstNewBlock, InlinedFunctionInfo);
+  if (auto *II = dyn_cast<InvokeInst>(TheCall)) {
+    BasicBlock *UnwindDest = II->getUnwindDest();
+    Instruction *FirstNonPHI = UnwindDest->getFirstNonPHI();
+    if (isa<LandingPadInst>(FirstNonPHI)) {
+      HandleInlinedLandingPad(II, FirstNewBlock, InlinedFunctionInfo);
+    } else {
+      HandleInlinedEHPad(II, FirstNewBlock, InlinedFunctionInfo);
+    }
+  }
 
   // Handle any inlined musttail call sites.  In order for a new call site to be
   // musttail, the source of the clone and the inlined call site must have been
diff --git a/lib/Transforms/Utils/IntegerDivision.cpp b/lib/Transforms/Utils/IntegerDivision.cpp
index 0ae746cc83d..30edf3b7aae 100644
--- a/lib/Transforms/Utils/IntegerDivision.cpp
+++ b/lib/Transforms/Utils/IntegerDivision.cpp
@@ -252,8 +252,8 @@ static Value *generateUnsignedDivisionCode(Value *Dividend, Value *Divisor,
   Value *Ret0_1      = Builder.CreateICmpEQ(Divisor, Zero);
   Value *Ret0_2      = Builder.CreateICmpEQ(Dividend, Zero);
   Value *Ret0_3      = Builder.CreateOr(Ret0_1, Ret0_2);
-  Value *Tmp0        = Builder.CreateCall2(CTLZ, Divisor, True);
-  Value *Tmp1        = Builder.CreateCall2(CTLZ, Dividend, True);
+  Value *Tmp0 = Builder.CreateCall(CTLZ, {Divisor, True});
+  Value *Tmp1 = Builder.CreateCall(CTLZ, {Dividend, True});
   Value *SR          = Builder.CreateSub(Tmp0, Tmp1);
   Value *Ret0_4      = Builder.CreateICmpUGT(SR, MSB);
   Value *Ret0        = Builder.CreateOr(Ret0_3, Ret0_4);
diff --git a/lib/Transforms/Utils/LCSSA.cpp b/lib/Transforms/Utils/LCSSA.cpp
index 1cba367a3e4..9d40b6989d6 100644
--- a/lib/Transforms/Utils/LCSSA.cpp
+++ b/lib/Transforms/Utils/LCSSA.cpp
@@ -112,17 +112,17 @@ static bool processInstruction(Loop &L, Instruction &Inst, DominatorTree &DT,
     if (SSAUpdate.HasValueForBlock(ExitBB))
       continue;
 
-    PHINode *PN = PHINode::Create(Inst.getType(), PredCache.GetNumPreds(ExitBB),
+    PHINode *PN = PHINode::Create(Inst.getType(), PredCache.size(ExitBB),
                                   Inst.getName() + ".lcssa", ExitBB->begin());
 
     // Add inputs from inside the loop for this PHI.
-    for (BasicBlock **PI = PredCache.GetPreds(ExitBB); *PI; ++PI) {
-      PN->addIncoming(&Inst, *PI);
+    for (BasicBlock *Pred : PredCache.get(ExitBB)) {
+      PN->addIncoming(&Inst, Pred);
 
       // If the exit block has a predecessor not within the loop, arrange for
       // the incoming value use corresponding to that predecessor to be
       // rewritten in terms of a different LCSSA PHI.
-      if (!L.contains(*PI))
+      if (!L.contains(Pred))
         UsesToRewrite.push_back(
             &PN->getOperandUse(PN->getOperandNumForIncomingValue(
                  PN->getNumIncomingValues() - 1)));
@@ -299,9 +299,6 @@ struct LCSSA : public FunctionPass {
     AU.addPreserved<AliasAnalysis>();
     AU.addPreserved<ScalarEvolution>();
   }
-
-private:
-  void verifyAnalysis() const override;
 };
 }
 
@@ -329,18 +326,3 @@ bool LCSSA::runOnFunction(Function &F) {
   return Changed;
 }
 
-static void verifyLoop(Loop &L, DominatorTree &DT) {
-  // Recurse depth-first through inner loops.
-  for (Loop::iterator LI = L.begin(), LE = L.end(); LI != LE; ++LI)
-    verifyLoop(**LI, DT);
-
-  // Check the special guarantees that LCSSA makes.
-  //assert(L.isLCSSAForm(DT) && "LCSSA form not preserved!");
-}
-
-void LCSSA::verifyAnalysis() const {
-  // Verify each loop nest in the function, assuming LI still points at that
-  // function's loop info.
-  for (LoopInfo::iterator I = LI->begin(), E = LI->end(); I != E; ++I)
-    verifyLoop(**I, *DT);
-}
diff --git a/lib/Transforms/Utils/Local.cpp b/lib/Transforms/Utils/Local.cpp
index 1c9760e0ff9..71a55d495d5 100644
--- a/lib/Transforms/Utils/Local.cpp
+++ b/lib/Transforms/Utils/Local.cpp
@@ -14,6 +14,8 @@
 
 #include "llvm/Transforms/Utils/Local.h"
 #include "llvm/ADT/DenseMap.h"
+#include "llvm/ADT/DenseSet.h"
+#include "llvm/ADT/Hashing.h"
 #include "llvm/ADT/STLExtras.h"
 #include "llvm/ADT/SmallPtrSet.h"
 #include "llvm/ADT/Statistic.h"
@@ -281,8 +283,9 @@ bool llvm::isInstructionTriviallyDead(Instruction *I,
                                       const TargetLibraryInfo *TLI) {
   if (!I->use_empty() || isa<TerminatorInst>(I)) return false;
 
-  // We don't want the landingpad instruction removed by anything this general.
-  if (isa<LandingPadInst>(I))
+  // We don't want the landingpad-like instructions removed by anything this
+  // general.
+  if (I->isEHPad())
     return false;
 
   // We don't want debug info removed by anything this general, unless
@@ -828,64 +831,45 @@ bool llvm::TryToSimplifyUncondBranchFromEmptyBlock(BasicBlock *BB) {
 /// orders them so it usually won't matter.
 ///
 bool llvm::EliminateDuplicatePHINodes(BasicBlock *BB) {
-  bool Changed = false;
-
   // This implementation doesn't currently consider undef operands
   // specially. Theoretically, two phis which are identical except for
   // one having an undef where the other doesn't could be collapsed.
 
-  // Map from PHI hash values to PHI nodes. If multiple PHIs have
-  // the same hash value, the element is the first PHI in the
-  // linked list in CollisionMap.
-  DenseMap<uintptr_t, PHINode *> HashMap;
+  struct PHIDenseMapInfo {
+    static PHINode *getEmptyKey() {
+      return DenseMapInfo<PHINode *>::getEmptyKey();
+    }
+    static PHINode *getTombstoneKey() {
+      return DenseMapInfo<PHINode *>::getTombstoneKey();
+    }
+    static unsigned getHashValue(PHINode *PN) {
+      // Compute a hash value on the operands. Instcombine will likely have
+      // sorted them, which helps expose duplicates, but we have to check all
+      // the operands to be safe in case instcombine hasn't run.
+      return static_cast<unsigned>(hash_combine(
+          hash_combine_range(PN->value_op_begin(), PN->value_op_end()),
+          hash_combine_range(PN->block_begin(), PN->block_end())));
+    }
+    static bool isEqual(PHINode *LHS, PHINode *RHS) {
+      if (LHS == getEmptyKey() || LHS == getTombstoneKey() ||
+          RHS == getEmptyKey() || RHS == getTombstoneKey())
+        return LHS == RHS;
+      return LHS->isIdenticalTo(RHS);
+    }
+  };
 
-  // Maintain linked lists of PHI nodes with common hash values.
-  DenseMap<PHINode *, PHINode *> CollisionMap;
+  // Set of unique PHINodes.
+  DenseSet<PHINode *, PHIDenseMapInfo> PHISet;
 
   // Examine each PHI.
-  for (BasicBlock::iterator I = BB->begin();
-       PHINode *PN = dyn_cast<PHINode>(I++); ) {
-    // Compute a hash value on the operands. Instcombine will likely have sorted
-    // them, which helps expose duplicates, but we have to check all the
-    // operands to be safe in case instcombine hasn't run.
-    uintptr_t Hash = 0;
-    // This hash algorithm is quite weak as hash functions go, but it seems
-    // to do a good enough job for this particular purpose, and is very quick.
-    for (User::op_iterator I = PN->op_begin(), E = PN->op_end(); I != E; ++I) {
-      Hash ^= reinterpret_cast<uintptr_t>(static_cast<Value *>(*I));
-      Hash = (Hash << 7) | (Hash >> (sizeof(uintptr_t) * CHAR_BIT - 7));
-    }
-    for (PHINode::block_iterator I = PN->block_begin(), E = PN->block_end();
-         I != E; ++I) {
-      Hash ^= reinterpret_cast<uintptr_t>(static_cast<BasicBlock *>(*I));
-      Hash = (Hash << 7) | (Hash >> (sizeof(uintptr_t) * CHAR_BIT - 7));
-    }
-    // Avoid colliding with the DenseMap sentinels ~0 and ~0-1.
-    Hash >>= 1;
-    // If we've never seen this hash value before, it's a unique PHI.
-    std::pair<DenseMap<uintptr_t, PHINode *>::iterator, bool> Pair =
-      HashMap.insert(std::make_pair(Hash, PN));
-    if (Pair.second) continue;
-    // Otherwise it's either a duplicate or a hash collision.
-    for (PHINode *OtherPN = Pair.first->second; ; ) {
-      if (OtherPN->isIdenticalTo(PN)) {
-        // A duplicate. Replace this PHI with its duplicate.
-        PN->replaceAllUsesWith(OtherPN);
-        PN->eraseFromParent();
-        Changed = true;
-        break;
-      }
-      // A non-duplicate hash collision.
-      DenseMap<PHINode *, PHINode *>::iterator I = CollisionMap.find(OtherPN);
-      if (I == CollisionMap.end()) {
-        // Set this PHI to be the head of the linked list of colliding PHIs.
-        PHINode *Old = Pair.first->second;
-        Pair.first->second = PN;
-        CollisionMap[PN] = Old;
-        break;
-      }
-      // Proceed to the next PHI in the list.
-      OtherPN = I->second;
+  bool Changed = false;
+  for (auto I = BB->begin(); PHINode *PN = dyn_cast<PHINode>(I++);) {
+    auto Inserted = PHISet.insert(PN);
+    if (!Inserted.second) {
+      // A duplicate. Replace this PHI with its duplicate.
+      PN->replaceAllUsesWith(*Inserted.first);
+      PN->eraseFromParent();
+      Changed = true;
     }
   }
 
@@ -917,13 +901,10 @@ static unsigned enforceKnownAlignment(Value *V, unsigned Align,
 
   if (auto *GO = dyn_cast<GlobalObject>(V)) {
     // If there is a large requested alignment and we can, bump up the alignment
-    // of the global.
-    if (GO->isDeclaration())
-      return Align;
-    // If the memory we set aside for the global may not be the memory used by
-    // the final program then it is impossible for us to reliably enforce the
-    // preferred alignment.
-    if (GO->isWeakForLinker())
+    // of the global.  If the memory we set aside for the global may not be the
+    // memory used by the final program then it is impossible for us to reliably
+    // enforce the preferred alignment.
+    if (!GO->isStrongDefinitionForLinker())
       return Align;
 
     if (GO->getAlignment() >= PrefAlign)
@@ -978,7 +959,7 @@ unsigned llvm::getOrEnforceKnownAlignment(Value *V, unsigned PrefAlign,
 ///
 
 /// See if there is a dbg.value intrinsic for DIVar before I.
-static bool LdStHasDebugValue(DIVariable &DIVar, Instruction *I) {
+static bool LdStHasDebugValue(const DILocalVariable *DIVar, Instruction *I) {
   // Since we can't guarantee that the original dbg.declare instrinsic
   // is removed by LowerDbgDeclare(), we need to make sure that we are
   // not inserting the same dbg.value intrinsic over and over.
@@ -998,10 +979,9 @@ static bool LdStHasDebugValue(DIVariable &DIVar, Instruction *I) {
 /// that has an associated llvm.dbg.decl intrinsic.
 bool llvm::ConvertDebugDeclareToDebugValue(DbgDeclareInst *DDI,
                                            StoreInst *SI, DIBuilder &Builder) {
-  DIVariable DIVar = DDI->getVariable();
-  DIExpression DIExpr = DDI->getExpression();
-  if (!DIVar)
-    return false;
+  auto *DIVar = DDI->getVariable();
+  auto *DIExpr = DDI->getExpression();
+  assert(DIVar && "Missing variable");
 
   if (LdStHasDebugValue(DIVar, SI))
     return true;
@@ -1026,10 +1006,9 @@ bool llvm::ConvertDebugDeclareToDebugValue(DbgDeclareInst *DDI,
 /// that has an associated llvm.dbg.decl intrinsic.
 bool llvm::ConvertDebugDeclareToDebugValue(DbgDeclareInst *DDI,
                                            LoadInst *LI, DIBuilder &Builder) {
-  DIVariable DIVar = DDI->getVariable();
-  DIExpression DIExpr = DDI->getExpression();
-  if (!DIVar)
-    return false;
+  auto *DIVar = DDI->getVariable();
+  auto *DIExpr = DDI->getExpression();
+  assert(DIVar && "Missing variable");
 
   if (LdStHasDebugValue(DIVar, LI))
     return true;
@@ -1077,9 +1056,9 @@ bool llvm::LowerDbgDeclare(Function &F) {
           // This is a call by-value or some other instruction that
           // takes a pointer to the variable. Insert a *value*
           // intrinsic that describes the alloca.
-          DIB.insertDbgValueIntrinsic(AI, 0, DIVariable(DDI->getVariable()),
-                                      DIExpression(DDI->getExpression()),
-                                      DDI->getDebugLoc(), CI);
+          DIB.insertDbgValueIntrinsic(AI, 0, DDI->getVariable(),
+                                      DDI->getExpression(), DDI->getDebugLoc(),
+                                      CI);
         }
       DDI->eraseFromParent();
     }
@@ -1105,10 +1084,9 @@ bool llvm::replaceDbgDeclareForAlloca(AllocaInst *AI, Value *NewAllocaAddress,
   if (!DDI)
     return false;
   DebugLoc Loc = DDI->getDebugLoc();
-  DIVariable DIVar = DDI->getVariable();
-  DIExpression DIExpr = DDI->getExpression();
-  if (!DIVar)
-    return false;
+  auto *DIVar = DDI->getVariable();
+  auto *DIExpr = DDI->getExpression();
+  assert(DIVar && "Missing variable");
 
   if (Deref) {
     // Create a copy of the original DIDescriptor for user variable, prepending
@@ -1176,10 +1154,11 @@ static void changeToCall(InvokeInst *II) {
   II->eraseFromParent();
 }
 
-static bool markAliveBlocks(BasicBlock *BB,
+static bool markAliveBlocks(Function &F,
                             SmallPtrSetImpl<BasicBlock*> &Reachable) {
 
   SmallVector<BasicBlock*, 128> Worklist;
+  BasicBlock *BB = F.begin();
   Worklist.push_back(BB);
   Reachable.insert(BB);
   bool Changed = false;
@@ -1250,7 +1229,7 @@ static bool markAliveBlocks(BasicBlock *BB,
       if (isa<ConstantPointerNull>(Callee) || isa<UndefValue>(Callee)) {
         changeToUnreachable(II, true);
         Changed = true;
-      } else if (II->doesNotThrow() && canSimplifyInvokeNoUnwind(II)) {
+      } else if (II->doesNotThrow() && canSimplifyInvokeNoUnwind(&F)) {
         if (II->use_empty() && II->onlyReadsMemory()) {
           // jump to the normal destination branch.
           BranchInst::Create(II->getNormalDest(), II);
@@ -1275,7 +1254,7 @@ static bool markAliveBlocks(BasicBlock *BB,
 /// otherwise.
 bool llvm::removeUnreachableBlocks(Function &F) {
   SmallPtrSet<BasicBlock*, 128> Reachable;
-  bool Changed = markAliveBlocks(F.begin(), Reachable);
+  bool Changed = markAliveBlocks(F, Reachable);
 
   // If there are unreachable blocks in the CFG...
   if (Reachable.size() == F.size())
@@ -1346,3 +1325,23 @@ void llvm::combineMetadata(Instruction *K, const Instruction *J, ArrayRef<unsign
     }
   }
 }
+
+unsigned llvm::replaceDominatedUsesWith(Value *From, Value *To,
+                                        DominatorTree &DT,
+                                        const BasicBlockEdge &Root) {
+  assert(From->getType() == To->getType());
+  
+  unsigned Count = 0;
+  for (Value::use_iterator UI = From->use_begin(), UE = From->use_end();
+       UI != UE; ) {
+    Use &U = *UI++;
+    if (DT.dominates(Root, U)) {
+      U.set(To);
+      DEBUG(dbgs() << "Replace dominated use of '"
+            << From->getName() << "' as "
+            << *To << " in " << *U << "\n");
+      ++Count;
+    }
+  }
+  return Count;
+}
diff --git a/lib/Transforms/Utils/LoopSimplify.cpp b/lib/Transforms/Utils/LoopSimplify.cpp
index 90dfabaeb35..1298011e78a 100644
--- a/lib/Transforms/Utils/LoopSimplify.cpp
+++ b/lib/Transforms/Utils/LoopSimplify.cpp
@@ -116,7 +116,6 @@ BasicBlock *llvm::InsertPreheaderForLoop(Loop *L, Pass *PP) {
   BasicBlock *Header = L->getHeader();
 
   // Get analyses that we try to update.
-  auto *AA = PP->getAnalysisIfAvailable<AliasAnalysis>();
   auto *DTWP = PP->getAnalysisIfAvailable<DominatorTreeWrapperPass>();
   auto *DT = DTWP ? &DTWP->getDomTree() : nullptr;
   auto *LIWP = PP->getAnalysisIfAvailable<LoopInfoWrapperPass>();
@@ -141,11 +140,11 @@ BasicBlock *llvm::InsertPreheaderForLoop(Loop *L, Pass *PP) {
 
   // Split out the loop pre-header.
   BasicBlock *PreheaderBB;
-  PreheaderBB = SplitBlockPredecessors(Header, OutsideBlocks, ".preheader",
-                                       AA, DT, LI, PreserveLCSSA);
+  PreheaderBB = SplitBlockPredecessors(Header, OutsideBlocks, ".preheader", DT,
+                                       LI, PreserveLCSSA);
+  if (!PreheaderBB)
+    return nullptr;
 
-  PreheaderBB->getTerminator()->setDebugLoc(
-                                      Header->getFirstNonPHI()->getDebugLoc());
   DEBUG(dbgs() << "LoopSimplify: Creating pre-header "
                << PreheaderBB->getName() << "\n");
 
@@ -161,8 +160,8 @@ BasicBlock *llvm::InsertPreheaderForLoop(Loop *L, Pass *PP) {
 /// This method is used to split exit blocks that have predecessors outside of
 /// the loop.
 static BasicBlock *rewriteLoopExitBlock(Loop *L, BasicBlock *Exit,
-                                        AliasAnalysis *AA, DominatorTree *DT,
-                                        LoopInfo *LI, Pass *PP) {
+                                        DominatorTree *DT, LoopInfo *LI,
+                                        Pass *PP) {
   SmallVector<BasicBlock*, 8> LoopBlocks;
   for (pred_iterator I = pred_begin(Exit), E = pred_end(Exit); I != E; ++I) {
     BasicBlock *P = *I;
@@ -179,8 +178,10 @@ static BasicBlock *rewriteLoopExitBlock(Loop *L, BasicBlock *Exit,
 
   bool PreserveLCSSA = PP->mustPreserveAnalysisID(LCSSAID);
 
-  NewExitBB = SplitBlockPredecessors(Exit, LoopBlocks, ".loopexit", AA, DT,
-                                     LI, PreserveLCSSA);
+  NewExitBB = SplitBlockPredecessors(Exit, LoopBlocks, ".loopexit", DT, LI,
+                                     PreserveLCSSA);
+  if (!NewExitBB)
+    return nullptr;
 
   DEBUG(dbgs() << "LoopSimplify: Creating dedicated exit block "
                << NewExitBB->getName() << "\n");
@@ -208,8 +209,7 @@ static void addBlockAndPredsToSet(BasicBlock *InputBB, BasicBlock *StopBlock,
 
 /// \brief The first part of loop-nestification is to find a PHI node that tells
 /// us how to partition the loops.
-static PHINode *findPHIToPartitionLoops(Loop *L, AliasAnalysis *AA,
-                                        DominatorTree *DT,
+static PHINode *findPHIToPartitionLoops(Loop *L, DominatorTree *DT,
                                         AssumptionCache *AC) {
   const DataLayout &DL = L->getHeader()->getModule()->getDataLayout();
   for (BasicBlock::iterator I = L->getHeader()->begin(); isa<PHINode>(I); ) {
@@ -218,7 +218,6 @@ static PHINode *findPHIToPartitionLoops(Loop *L, AliasAnalysis *AA,
     if (Value *V = SimplifyInstruction(PN, DL, nullptr, DT, AC)) {
       // This is a degenerate PHI already, don't modify it!
       PN->replaceAllUsesWith(V);
-      if (AA) AA->deleteValue(PN);
       PN->eraseFromParent();
       continue;
     }
@@ -253,18 +252,20 @@ static PHINode *findPHIToPartitionLoops(Loop *L, AliasAnalysis *AA,
 /// created.
 ///
 static Loop *separateNestedLoop(Loop *L, BasicBlock *Preheader,
-                                AliasAnalysis *AA, DominatorTree *DT,
-                                LoopInfo *LI, ScalarEvolution *SE, Pass *PP,
+                                DominatorTree *DT, LoopInfo *LI,
+                                ScalarEvolution *SE, Pass *PP,
                                 AssumptionCache *AC) {
   // Don't try to separate loops without a preheader.
   if (!Preheader)
     return nullptr;
 
   // The header is not a landing pad; preheader insertion should ensure this.
-  assert(!L->getHeader()->isLandingPad() &&
-         "Can't insert backedge to landing pad");
+  BasicBlock *Header = L->getHeader();
+  assert(!Header->isLandingPad() && "Can't insert backedge to landing pad");
+  if (!Header->canSplitPredecessors())
+    return nullptr;
 
-  PHINode *PN = findPHIToPartitionLoops(L, AA, DT, AC);
+  PHINode *PN = findPHIToPartitionLoops(L, DT, AC);
   if (!PN) return nullptr;  // No known way to partition.
 
   // Pull out all predecessors that have varying values in the loop.  This
@@ -290,9 +291,8 @@ static Loop *separateNestedLoop(Loop *L, BasicBlock *Preheader,
 
   bool PreserveLCSSA = PP->mustPreserveAnalysisID(LCSSAID);
 
-  BasicBlock *Header = L->getHeader();
   BasicBlock *NewBB = SplitBlockPredecessors(Header, OuterLoopPreds, ".outer",
-                                             AA, DT, LI, PreserveLCSSA);
+                                             DT, LI, PreserveLCSSA);
 
   // Make sure that NewBB is put someplace intelligent, which doesn't mess up
   // code layout too horribly.
@@ -359,7 +359,6 @@ static Loop *separateNestedLoop(Loop *L, BasicBlock *Preheader,
 /// and have that block branch to the loop header.  This ensures that loops
 /// have exactly one backedge.
 static BasicBlock *insertUniqueBackedgeBlock(Loop *L, BasicBlock *Preheader,
-                                             AliasAnalysis *AA,
                                              DominatorTree *DT, LoopInfo *LI) {
   assert(L->getNumBackEdges() > 1 && "Must have > 1 backedge!");
 
@@ -388,8 +387,9 @@ static BasicBlock *insertUniqueBackedgeBlock(Loop *L, BasicBlock *Preheader,
 
   // Create and insert the new backedge block...
   BasicBlock *BEBlock = BasicBlock::Create(Header->getContext(),
-                                           Header->getName()+".backedge", F);
+                                           Header->getName() + ".backedge", F);
   BranchInst *BETerminator = BranchInst::Create(Header, BEBlock);
+  BETerminator->setDebugLoc(Header->getFirstNonPHI()->getDebugLoc());
 
   DEBUG(dbgs() << "LoopSimplify: Inserting unique backedge block "
                << BEBlock->getName() << "\n");
@@ -404,7 +404,6 @@ static BasicBlock *insertUniqueBackedgeBlock(Loop *L, BasicBlock *Preheader,
     PHINode *PN = cast<PHINode>(I);
     PHINode *NewPN = PHINode::Create(PN->getType(), BackedgeBlocks.size(),
                                      PN->getName()+".be", BETerminator);
-    if (AA) AA->copyValue(PN, NewPN);
 
     // Loop over the PHI node, moving all entries except the one for the
     // preheader over to the new PHI node.
@@ -445,7 +444,6 @@ static BasicBlock *insertUniqueBackedgeBlock(Loop *L, BasicBlock *Preheader,
     // eliminate the PHI Node.
     if (HasUniqueIncomingValue) {
       NewPN->replaceAllUsesWith(UniqueValue);
-      if (AA) AA->deleteValue(NewPN);
       BEBlock->getInstList().erase(NewPN);
     }
   }
@@ -478,7 +476,7 @@ static BasicBlock *insertUniqueBackedgeBlock(Loop *L, BasicBlock *Preheader,
 /// specific analyses. Rather than a pass it would be much cleaner and more
 /// explicit if they accepted the analysis directly and then updated it.
 static bool simplifyOneLoop(Loop *L, SmallVectorImpl<Loop *> &Worklist,
-                            AliasAnalysis *AA, DominatorTree *DT, LoopInfo *LI,
+                            DominatorTree *DT, LoopInfo *LI,
                             ScalarEvolution *SE, Pass *PP,
                             AssumptionCache *AC) {
   bool Changed = false;
@@ -570,7 +568,7 @@ static bool simplifyOneLoop(Loop *L, SmallVectorImpl<Loop *> &Worklist,
       // Must be exactly this loop: no subloops, parent loops, or non-loop preds
       // allowed.
       if (!L->contains(*PI)) {
-        if (rewriteLoopExitBlock(L, ExitBlock, AA, DT, LI, PP)) {
+        if (rewriteLoopExitBlock(L, ExitBlock, DT, LI, PP)) {
           ++NumInserted;
           Changed = true;
         }
@@ -586,8 +584,7 @@ static bool simplifyOneLoop(Loop *L, SmallVectorImpl<Loop *> &Worklist,
     // this for loops with a giant number of backedges, just factor them into a
     // common backedge instead.
     if (L->getNumBackEdges() < 8) {
-      if (Loop *OuterL =
-              separateNestedLoop(L, Preheader, AA, DT, LI, SE, PP, AC)) {
+      if (Loop *OuterL = separateNestedLoop(L, Preheader, DT, LI, SE, PP, AC)) {
         ++NumNested;
         // Enqueue the outer loop as it should be processed next in our
         // depth-first nest walk.
@@ -604,7 +601,7 @@ static bool simplifyOneLoop(Loop *L, SmallVectorImpl<Loop *> &Worklist,
     // If we either couldn't, or didn't want to, identify nesting of the loops,
     // insert a new block that all backedges target, then make it jump to the
     // loop header.
-    LoopLatch = insertUniqueBackedgeBlock(L, Preheader, AA, DT, LI);
+    LoopLatch = insertUniqueBackedgeBlock(L, Preheader, DT, LI);
     if (LoopLatch) {
       ++NumInserted;
       Changed = true;
@@ -620,7 +617,6 @@ static bool simplifyOneLoop(Loop *L, SmallVectorImpl<Loop *> &Worklist,
   for (BasicBlock::iterator I = L->getHeader()->begin();
        (PN = dyn_cast<PHINode>(I++)); )
     if (Value *V = SimplifyInstruction(PN, DL, nullptr, DT, AC)) {
-      if (AA) AA->deleteValue(PN);
       if (SE) SE->forgetValue(PN);
       PN->replaceAllUsesWith(V);
       PN->eraseFromParent();
@@ -719,8 +715,7 @@ static bool simplifyOneLoop(Loop *L, SmallVectorImpl<Loop *> &Worklist,
 }
 
 bool llvm::simplifyLoop(Loop *L, DominatorTree *DT, LoopInfo *LI, Pass *PP,
-                        AliasAnalysis *AA, ScalarEvolution *SE,
-                        AssumptionCache *AC) {
+                        ScalarEvolution *SE, AssumptionCache *AC) {
   bool Changed = false;
 
   // Worklist maintains our depth-first queue of loops in this nest to process.
@@ -736,8 +731,8 @@ bool llvm::simplifyLoop(Loop *L, DominatorTree *DT, LoopInfo *LI, Pass *PP,
   }
 
   while (!Worklist.empty())
-    Changed |= simplifyOneLoop(Worklist.pop_back_val(), Worklist, AA, DT, LI,
-                               SE, PP, AC);
+    Changed |=
+        simplifyOneLoop(Worklist.pop_back_val(), Worklist, DT, LI, SE, PP, AC);
 
   return Changed;
 }
@@ -749,9 +744,6 @@ namespace {
       initializeLoopSimplifyPass(*PassRegistry::getPassRegistry());
     }
 
-    // AA - If we have an alias analysis object to update, this is it, otherwise
-    // this is null.
-    AliasAnalysis *AA;
     DominatorTree *DT;
     LoopInfo *LI;
     ScalarEvolution *SE;
@@ -798,7 +790,6 @@ Pass *llvm::createLoopSimplifyPass() { return new LoopSimplify(); }
 ///
 bool LoopSimplify::runOnFunction(Function &F) {
   bool Changed = false;
-  AA = getAnalysisIfAvailable<AliasAnalysis>();
   LI = &getAnalysis<LoopInfoWrapperPass>().getLoopInfo();
   DT = &getAnalysis<DominatorTreeWrapperPass>().getDomTree();
   SE = getAnalysisIfAvailable<ScalarEvolution>();
@@ -806,7 +797,7 @@ bool LoopSimplify::runOnFunction(Function &F) {
 
   // Simplify each loop nest in the function.
   for (LoopInfo::iterator I = LI->begin(), E = LI->end(); I != E; ++I)
-    Changed |= simplifyLoop(*I, DT, LI, this, AA, SE, AC);
+    Changed |= simplifyLoop(*I, DT, LI, this, SE, AC);
 
   return Changed;
 }
diff --git a/lib/Transforms/Utils/LoopUnroll.cpp b/lib/Transforms/Utils/LoopUnroll.cpp
index 1dbce474683..6e7a571da51 100644
--- a/lib/Transforms/Utils/LoopUnroll.cpp
+++ b/lib/Transforms/Utils/LoopUnroll.cpp
@@ -541,7 +541,7 @@ bool llvm::UnrollLoop(Loop *L, unsigned Count, unsigned TripCount,
     if (!OuterL && !CompletelyUnroll)
       OuterL = L;
     if (OuterL) {
-      simplifyLoop(OuterL, DT, LI, PP, /*AliasAnalysis*/ nullptr, SE, AC);
+      simplifyLoop(OuterL, DT, LI, PP, SE, AC);
 
       // LCSSA must be performed on the outermost affected loop. The unrolled
       // loop's last loop latch is guaranteed to be in the outermost loop after
diff --git a/lib/Transforms/Utils/LoopUnrollRuntime.cpp b/lib/Transforms/Utils/LoopUnrollRuntime.cpp
index d1774dfa10d..5658e9201c3 100644
--- a/lib/Transforms/Utils/LoopUnrollRuntime.cpp
+++ b/lib/Transforms/Utils/LoopUnrollRuntime.cpp
@@ -62,8 +62,8 @@ STATISTIC(NumRuntimeUnrolled,
 static void ConnectProlog(Loop *L, Value *BECount, unsigned Count,
                           BasicBlock *LastPrologBB, BasicBlock *PrologEnd,
                           BasicBlock *OrigPH, BasicBlock *NewPH,
-                          ValueToValueMapTy &VMap, AliasAnalysis *AA,
-                          DominatorTree *DT, LoopInfo *LI, Pass *P) {
+                          ValueToValueMapTy &VMap, DominatorTree *DT,
+                          LoopInfo *LI, Pass *P) {
   BasicBlock *Latch = L->getLoopLatch();
   assert(Latch && "Loop must have a latch");
 
@@ -86,7 +86,7 @@ static void ConnectProlog(Loop *L, Value *BECount, unsigned Count,
       if (L->contains(PN)) {
         NewPN->addIncoming(PN->getIncomingValueForBlock(NewPH), OrigPH);
       } else {
-        NewPN->addIncoming(Constant::getNullValue(PN->getType()), OrigPH);
+        NewPN->addIncoming(UndefValue::get(PN->getType()), OrigPH);
       }
 
       Value *V = PN->getIncomingValueForBlock(Latch);
@@ -113,6 +113,7 @@ static void ConnectProlog(Loop *L, Value *BECount, unsigned Count,
   // Create a branch around the orignal loop, which is taken if there are no
   // iterations remaining to be executed after running the prologue.
   Instruction *InsertPt = PrologEnd->getTerminator();
+  IRBuilder<> B(InsertPt);
 
   assert(Count != 0 && "nonsensical Count!");
 
@@ -120,17 +121,16 @@ static void ConnectProlog(Loop *L, Value *BECount, unsigned Count,
   // (since Count is a power of 2).  This means %xtraiter is (BECount + 1) and
   // and all of the iterations of this loop were executed by the prologue.  Note
   // that if BECount <u (Count - 1) then (BECount + 1) cannot unsigned-overflow.
-  Instruction *BrLoopExit =
-    new ICmpInst(InsertPt, ICmpInst::ICMP_ULT, BECount,
-                 ConstantInt::get(BECount->getType(), Count - 1));
+  Value *BrLoopExit =
+      B.CreateICmpULT(BECount, ConstantInt::get(BECount->getType(), Count - 1));
   BasicBlock *Exit = L->getUniqueExitBlock();
   assert(Exit && "Loop must have a single exit block only");
   // Split the exit to maintain loop canonicalization guarantees
   SmallVector<BasicBlock*, 4> Preds(pred_begin(Exit), pred_end(Exit));
-  SplitBlockPredecessors(Exit, Preds, ".unr-lcssa", AA, DT, LI,
+  SplitBlockPredecessors(Exit, Preds, ".unr-lcssa", DT, LI,
                          P->mustPreserveAnalysisID(LCSSAID));
   // Add the branch to the exit block (around the unrolled loop)
-  BranchInst::Create(Exit, NewPH, BrLoopExit, InsertPt);
+  B.CreateCondBr(BrLoopExit, Exit, NewPH);
   InsertPt->eraseFromParent();
 }
 
@@ -184,23 +184,22 @@ static void CloneLoopBlocks(Loop *L, Value *NewIter, const bool UnrollProlog,
       VMap.erase((*BB)->getTerminator());
       BasicBlock *FirstLoopBB = cast<BasicBlock>(VMap[Header]);
       BranchInst *LatchBR = cast<BranchInst>(NewBB->getTerminator());
+      IRBuilder<> Builder(LatchBR);
       if (UnrollProlog) {
-        LatchBR->eraseFromParent();
-        BranchInst::Create(InsertBot, NewBB);
+        Builder.CreateBr(InsertBot);
       } else {
         PHINode *NewIdx = PHINode::Create(NewIter->getType(), 2, "prol.iter",
                                           FirstLoopBB->getFirstNonPHI());
-        IRBuilder<> Builder(LatchBR);
         Value *IdxSub =
             Builder.CreateSub(NewIdx, ConstantInt::get(NewIdx->getType(), 1),
                               NewIdx->getName() + ".sub");
         Value *IdxCmp =
             Builder.CreateIsNotNull(IdxSub, NewIdx->getName() + ".cmp");
-        BranchInst::Create(FirstLoopBB, InsertBot, IdxCmp, NewBB);
+        Builder.CreateCondBr(IdxCmp, FirstLoopBB, InsertBot);
         NewIdx->addIncoming(NewIter, InsertTop);
         NewIdx->addIncoming(IdxSub, NewBB);
-        LatchBR->eraseFromParent();
       }
+      LatchBR->eraseFromParent();
     }
   }
 
@@ -370,7 +369,7 @@ bool llvm::UnrollRuntimeLoopProlog(Loop *L, unsigned Count,
 
   // Branch to either the extra iterations or the cloned/unrolled loop
   // We will fix up the true branch label when adding loop body copies
-  BranchInst::Create(PEnd, PEnd, BranchVal, PreHeaderBR);
+  B.CreateCondBr(BranchVal, PEnd, PEnd);
   assert(PreHeaderBR->isUnconditional() &&
          PreHeaderBR->getSuccessor(0) == PEnd &&
          "CFG edges in Preheader are not correct");
@@ -415,8 +414,8 @@ bool llvm::UnrollRuntimeLoopProlog(Loop *L, unsigned Count,
   // Connect the prolog code to the original loop and update the
   // PHI functions.
   BasicBlock *LastLoopBB = cast<BasicBlock>(VMap[Latch]);
-  ConnectProlog(L, BECount, Count, LastLoopBB, PEnd, PH, NewPH, VMap,
-                /*AliasAnalysis*/ nullptr, DT, LI, LPM->getAsPass());
+  ConnectProlog(L, BECount, Count, LastLoopBB, PEnd, PH, NewPH, VMap, DT, LI,
+                LPM->getAsPass());
   NumRuntimeUnrolled++;
   return true;
 }
diff --git a/lib/Transforms/Utils/LoopUtils.cpp b/lib/Transforms/Utils/LoopUtils.cpp
index 5f25e6b2cb6..5cbde94a98e 100644
--- a/lib/Transforms/Utils/LoopUtils.cpp
+++ b/lib/Transforms/Utils/LoopUtils.cpp
@@ -26,17 +26,17 @@ using namespace llvm::PatternMatch;
 
 #define DEBUG_TYPE "loop-utils"
 
-bool ReductionDescriptor::areAllUsesIn(Instruction *I,
-                                       SmallPtrSetImpl<Instruction *> &Set) {
+bool RecurrenceDescriptor::areAllUsesIn(Instruction *I,
+                                        SmallPtrSetImpl<Instruction *> &Set) {
   for (User::op_iterator Use = I->op_begin(), E = I->op_end(); Use != E; ++Use)
     if (!Set.count(dyn_cast<Instruction>(*Use)))
       return false;
   return true;
 }
 
-bool ReductionDescriptor::AddReductionVar(PHINode *Phi, ReductionKind Kind,
-                                          Loop *TheLoop, bool HasFunNoNaNAttr,
-                                          ReductionDescriptor &RedDes) {
+bool RecurrenceDescriptor::AddReductionVar(PHINode *Phi, RecurrenceKind Kind,
+                                           Loop *TheLoop, bool HasFunNoNaNAttr,
+                                           RecurrenceDescriptor &RedDes) {
   if (Phi->getNumIncomingValues() != 2)
     return false;
 
@@ -66,7 +66,7 @@ bool ReductionDescriptor::AddReductionVar(PHINode *Phi, ReductionKind Kind,
   // the number of instruction we saw from the recognized min/max pattern,
   //  to make sure we only see exactly the two instructions.
   unsigned NumCmpSelectPatternInst = 0;
-  ReductionInstDesc ReduxDesc(false, nullptr);
+  InstDesc ReduxDesc(false, nullptr);
 
   SmallPtrSet<Instruction *, 8> VisitedInsts;
   SmallVector<Instruction *, 8> Worklist;
@@ -111,8 +111,8 @@ bool ReductionDescriptor::AddReductionVar(PHINode *Phi, ReductionKind Kind,
       return false;
 
     // Any reduction instruction must be of one of the allowed kinds.
-    ReduxDesc = isReductionInstr(Cur, Kind, ReduxDesc, HasFunNoNaNAttr);
-    if (!ReduxDesc.isReduction())
+    ReduxDesc = isRecurrenceInstr(Cur, Kind, ReduxDesc, HasFunNoNaNAttr);
+    if (!ReduxDesc.isRecurrence())
       return false;
 
     // A reduction operation must only have one use of the reduction value.
@@ -164,7 +164,7 @@ bool ReductionDescriptor::AddReductionVar(PHINode *Phi, ReductionKind Kind,
       // Process instructions only once (termination). Each reduction cycle
       // value must only be used once, except by phi nodes and min/max
       // reductions which are represented as a cmp followed by a select.
-      ReductionInstDesc IgnoredVal(false, nullptr);
+      InstDesc IgnoredVal(false, nullptr);
       if (VisitedInsts.insert(UI).second) {
         if (isa<PHINode>(UI))
           PHIs.push_back(UI);
@@ -173,7 +173,7 @@ bool ReductionDescriptor::AddReductionVar(PHINode *Phi, ReductionKind Kind,
       } else if (!isa<PHINode>(UI) &&
                  ((!isa<FCmpInst>(UI) && !isa<ICmpInst>(UI) &&
                    !isa<SelectInst>(UI)) ||
-                  !isMinMaxSelectCmpPattern(UI, IgnoredVal).isReduction()))
+                  !isMinMaxSelectCmpPattern(UI, IgnoredVal).isRecurrence()))
         return false;
 
       // Remember that we completed the cycle.
@@ -197,11 +197,11 @@ bool ReductionDescriptor::AddReductionVar(PHINode *Phi, ReductionKind Kind,
   // only have a single instruction with out-of-loop users.
 
   // The ExitInstruction(Instruction which is allowed to have out-of-loop users)
-  // is saved as part of the ReductionDescriptor.
+  // is saved as part of the RecurrenceDescriptor.
 
   // Save the description of this reduction variable.
-  ReductionDescriptor RD(RdxStart, ExitInstruction, Kind,
-                         ReduxDesc.getMinMaxKind());
+  RecurrenceDescriptor RD(RdxStart, ExitInstruction, Kind,
+                          ReduxDesc.getMinMaxKind());
 
   RedDes = RD;
 
@@ -210,9 +210,8 @@ bool ReductionDescriptor::AddReductionVar(PHINode *Phi, ReductionKind Kind,
 
 /// Returns true if the instruction is a Select(ICmp(X, Y), X, Y) instruction
 /// pattern corresponding to a min(X, Y) or max(X, Y).
-ReductionInstDesc
-ReductionDescriptor::isMinMaxSelectCmpPattern(Instruction *I,
-                                              ReductionInstDesc &Prev) {
+RecurrenceDescriptor::InstDesc
+RecurrenceDescriptor::isMinMaxSelectCmpPattern(Instruction *I, InstDesc &Prev) {
 
   assert((isa<ICmpInst>(I) || isa<FCmpInst>(I) || isa<SelectInst>(I)) &&
          "Expect a select instruction");
@@ -223,84 +222,83 @@ ReductionDescriptor::isMinMaxSelectCmpPattern(Instruction *I,
   // select.
   if ((Cmp = dyn_cast<ICmpInst>(I)) || (Cmp = dyn_cast<FCmpInst>(I))) {
     if (!Cmp->hasOneUse() || !(Select = dyn_cast<SelectInst>(*I->user_begin())))
-      return ReductionInstDesc(false, I);
-    return ReductionInstDesc(Select, Prev.getMinMaxKind());
+      return InstDesc(false, I);
+    return InstDesc(Select, Prev.getMinMaxKind());
   }
 
   // Only handle single use cases for now.
   if (!(Select = dyn_cast<SelectInst>(I)))
-    return ReductionInstDesc(false, I);
+    return InstDesc(false, I);
   if (!(Cmp = dyn_cast<ICmpInst>(I->getOperand(0))) &&
       !(Cmp = dyn_cast<FCmpInst>(I->getOperand(0))))
-    return ReductionInstDesc(false, I);
+    return InstDesc(false, I);
   if (!Cmp->hasOneUse())
-    return ReductionInstDesc(false, I);
+    return InstDesc(false, I);
 
   Value *CmpLeft;
   Value *CmpRight;
 
   // Look for a min/max pattern.
   if (m_UMin(m_Value(CmpLeft), m_Value(CmpRight)).match(Select))
-    return ReductionInstDesc(Select, ReductionInstDesc::MRK_UIntMin);
+    return InstDesc(Select, MRK_UIntMin);
   else if (m_UMax(m_Value(CmpLeft), m_Value(CmpRight)).match(Select))
-    return ReductionInstDesc(Select, ReductionInstDesc::MRK_UIntMax);
+    return InstDesc(Select, MRK_UIntMax);
   else if (m_SMax(m_Value(CmpLeft), m_Value(CmpRight)).match(Select))
-    return ReductionInstDesc(Select, ReductionInstDesc::MRK_SIntMax);
+    return InstDesc(Select, MRK_SIntMax);
   else if (m_SMin(m_Value(CmpLeft), m_Value(CmpRight)).match(Select))
-    return ReductionInstDesc(Select, ReductionInstDesc::MRK_SIntMin);
+    return InstDesc(Select, MRK_SIntMin);
   else if (m_OrdFMin(m_Value(CmpLeft), m_Value(CmpRight)).match(Select))
-    return ReductionInstDesc(Select, ReductionInstDesc::MRK_FloatMin);
+    return InstDesc(Select, MRK_FloatMin);
   else if (m_OrdFMax(m_Value(CmpLeft), m_Value(CmpRight)).match(Select))
-    return ReductionInstDesc(Select, ReductionInstDesc::MRK_FloatMax);
+    return InstDesc(Select, MRK_FloatMax);
   else if (m_UnordFMin(m_Value(CmpLeft), m_Value(CmpRight)).match(Select))
-    return ReductionInstDesc(Select, ReductionInstDesc::MRK_FloatMin);
+    return InstDesc(Select, MRK_FloatMin);
   else if (m_UnordFMax(m_Value(CmpLeft), m_Value(CmpRight)).match(Select))
-    return ReductionInstDesc(Select, ReductionInstDesc::MRK_FloatMax);
+    return InstDesc(Select, MRK_FloatMax);
 
-  return ReductionInstDesc(false, I);
+  return InstDesc(false, I);
 }
 
-ReductionInstDesc ReductionDescriptor::isReductionInstr(Instruction *I,
-                                                        ReductionKind Kind,
-                                                        ReductionInstDesc &Prev,
-                                                        bool HasFunNoNaNAttr) {
+RecurrenceDescriptor::InstDesc
+RecurrenceDescriptor::isRecurrenceInstr(Instruction *I, RecurrenceKind Kind,
+                                        InstDesc &Prev, bool HasFunNoNaNAttr) {
   bool FP = I->getType()->isFloatingPointTy();
   bool FastMath = FP && I->hasUnsafeAlgebra();
   switch (I->getOpcode()) {
   default:
-    return ReductionInstDesc(false, I);
+    return InstDesc(false, I);
   case Instruction::PHI:
     if (FP &&
         (Kind != RK_FloatMult && Kind != RK_FloatAdd && Kind != RK_FloatMinMax))
-      return ReductionInstDesc(false, I);
-    return ReductionInstDesc(I, Prev.getMinMaxKind());
+      return InstDesc(false, I);
+    return InstDesc(I, Prev.getMinMaxKind());
   case Instruction::Sub:
   case Instruction::Add:
-    return ReductionInstDesc(Kind == RK_IntegerAdd, I);
+    return InstDesc(Kind == RK_IntegerAdd, I);
   case Instruction::Mul:
-    return ReductionInstDesc(Kind == RK_IntegerMult, I);
+    return InstDesc(Kind == RK_IntegerMult, I);
   case Instruction::And:
-    return ReductionInstDesc(Kind == RK_IntegerAnd, I);
+    return InstDesc(Kind == RK_IntegerAnd, I);
   case Instruction::Or:
-    return ReductionInstDesc(Kind == RK_IntegerOr, I);
+    return InstDesc(Kind == RK_IntegerOr, I);
   case Instruction::Xor:
-    return ReductionInstDesc(Kind == RK_IntegerXor, I);
+    return InstDesc(Kind == RK_IntegerXor, I);
   case Instruction::FMul:
-    return ReductionInstDesc(Kind == RK_FloatMult && FastMath, I);
+    return InstDesc(Kind == RK_FloatMult && FastMath, I);
   case Instruction::FSub:
   case Instruction::FAdd:
-    return ReductionInstDesc(Kind == RK_FloatAdd && FastMath, I);
+    return InstDesc(Kind == RK_FloatAdd && FastMath, I);
   case Instruction::FCmp:
   case Instruction::ICmp:
   case Instruction::Select:
     if (Kind != RK_IntegerMinMax &&
         (!HasFunNoNaNAttr || Kind != RK_FloatMinMax))
-      return ReductionInstDesc(false, I);
+      return InstDesc(false, I);
     return isMinMaxSelectCmpPattern(I, Prev);
   }
 }
 
-bool ReductionDescriptor::hasMultipleUsesOf(
+bool RecurrenceDescriptor::hasMultipleUsesOf(
     Instruction *I, SmallPtrSetImpl<Instruction *> &Insts) {
   unsigned NumUses = 0;
   for (User::op_iterator Use = I->op_begin(), E = I->op_end(); Use != E;
@@ -313,8 +311,8 @@ bool ReductionDescriptor::hasMultipleUsesOf(
 
   return false;
 }
-bool ReductionDescriptor::isReductionPHI(PHINode *Phi, Loop *TheLoop,
-                                         ReductionDescriptor &RedDes) {
+bool RecurrenceDescriptor::isReductionPHI(PHINode *Phi, Loop *TheLoop,
+                                          RecurrenceDescriptor &RedDes) {
 
   bool HasFunNoNaNAttr = false;
   BasicBlock *Header = TheLoop->getHeader();
@@ -366,7 +364,8 @@ bool ReductionDescriptor::isReductionPHI(PHINode *Phi, Loop *TheLoop,
 
 /// This function returns the identity element (or neutral element) for
 /// the operation K.
-Constant *ReductionDescriptor::getReductionIdentity(ReductionKind K, Type *Tp) {
+Constant *RecurrenceDescriptor::getRecurrenceIdentity(RecurrenceKind K,
+                                                      Type *Tp) {
   switch (K) {
   case RK_IntegerXor:
   case RK_IntegerAdd:
@@ -386,12 +385,12 @@ Constant *ReductionDescriptor::getReductionIdentity(ReductionKind K, Type *Tp) {
     // Adding zero to a number does not change it.
     return ConstantFP::get(Tp, 0.0L);
   default:
-    llvm_unreachable("Unknown reduction kind");
+    llvm_unreachable("Unknown recurrence kind");
   }
 }
 
-/// This function translates the reduction kind to an LLVM binary operator.
-unsigned ReductionDescriptor::getReductionBinOp(ReductionKind Kind) {
+/// This function translates the recurrence kind to an LLVM binary operator.
+unsigned RecurrenceDescriptor::getRecurrenceBinOp(RecurrenceKind Kind) {
   switch (Kind) {
   case RK_IntegerAdd:
     return Instruction::Add;
@@ -412,41 +411,39 @@ unsigned ReductionDescriptor::getReductionBinOp(ReductionKind Kind) {
   case RK_FloatMinMax:
     return Instruction::FCmp;
   default:
-    llvm_unreachable("Unknown reduction operation");
+    llvm_unreachable("Unknown recurrence operation");
   }
 }
 
-Value *
-ReductionDescriptor::createMinMaxOp(IRBuilder<> &Builder,
-                                    ReductionInstDesc::MinMaxReductionKind RK,
-                                    Value *Left, Value *Right) {
+Value *RecurrenceDescriptor::createMinMaxOp(IRBuilder<> &Builder,
+                                            MinMaxRecurrenceKind RK,
+                                            Value *Left, Value *Right) {
   CmpInst::Predicate P = CmpInst::ICMP_NE;
   switch (RK) {
   default:
-    llvm_unreachable("Unknown min/max reduction kind");
-  case ReductionInstDesc::MRK_UIntMin:
+    llvm_unreachable("Unknown min/max recurrence kind");
+  case MRK_UIntMin:
     P = CmpInst::ICMP_ULT;
     break;
-  case ReductionInstDesc::MRK_UIntMax:
+  case MRK_UIntMax:
     P = CmpInst::ICMP_UGT;
     break;
-  case ReductionInstDesc::MRK_SIntMin:
+  case MRK_SIntMin:
     P = CmpInst::ICMP_SLT;
     break;
-  case ReductionInstDesc::MRK_SIntMax:
+  case MRK_SIntMax:
     P = CmpInst::ICMP_SGT;
     break;
-  case ReductionInstDesc::MRK_FloatMin:
+  case MRK_FloatMin:
     P = CmpInst::FCMP_OLT;
     break;
-  case ReductionInstDesc::MRK_FloatMax:
+  case MRK_FloatMax:
     P = CmpInst::FCMP_OGT;
     break;
   }
 
   Value *Cmp;
-  if (RK == ReductionInstDesc::MRK_FloatMin ||
-      RK == ReductionInstDesc::MRK_FloatMax)
+  if (RK == MRK_FloatMin || RK == MRK_FloatMax)
     Cmp = Builder.CreateFCmp(P, Left, Right, "rdx.minmax.cmp");
   else
     Cmp = Builder.CreateICmp(P, Left, Right, "rdx.minmax.cmp");
diff --git a/lib/Transforms/Utils/LoopVersioning.cpp b/lib/Transforms/Utils/LoopVersioning.cpp
new file mode 100644
index 00000000000..e4af2e3cc95
--- /dev/null
+++ b/lib/Transforms/Utils/LoopVersioning.cpp
@@ -0,0 +1,108 @@
+//===- LoopVersioning.cpp - Utility to version a loop ---------------------===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This file defines a utility class to perform loop versioning.  The versioned
+// loop speculates that otherwise may-aliasing memory accesses don't overlap and
+// emits checks to prove this.
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/Analysis/LoopAccessAnalysis.h"
+#include "llvm/Analysis/LoopInfo.h"
+#include "llvm/IR/Dominators.h"
+#include "llvm/Transforms/Utils/BasicBlockUtils.h"
+#include "llvm/Transforms/Utils/Cloning.h"
+#include "llvm/Transforms/Utils/LoopVersioning.h"
+
+using namespace llvm;
+
+LoopVersioning::LoopVersioning(
+    SmallVector<RuntimePointerChecking::PointerCheck, 4> Checks,
+    const LoopAccessInfo &LAI, Loop *L, LoopInfo *LI, DominatorTree *DT,
+    const SmallVector<int, 8> *PtrToPartition)
+    : VersionedLoop(L), NonVersionedLoop(nullptr),
+      PtrToPartition(PtrToPartition), Checks(std::move(Checks)), LAI(LAI),
+      LI(LI), DT(DT) {
+  assert(L->getExitBlock() && "No single exit block");
+  assert(L->getLoopPreheader() && "No preheader");
+}
+
+bool LoopVersioning::needsRuntimeChecks() const {
+  return LAI.getRuntimePointerChecking()->needsAnyChecking(PtrToPartition);
+}
+
+void LoopVersioning::versionLoop(Pass *P) {
+  Instruction *FirstCheckInst;
+  Instruction *MemRuntimeCheck;
+  // Add the memcheck in the original preheader (this is empty initially).
+  BasicBlock *MemCheckBB = VersionedLoop->getLoopPreheader();
+  std::tie(FirstCheckInst, MemRuntimeCheck) =
+      LAI.addRuntimeCheck(MemCheckBB->getTerminator(), Checks);
+  assert(MemRuntimeCheck && "called even though needsAnyChecking = false");
+
+  // Rename the block to make the IR more readable.
+  MemCheckBB->setName(VersionedLoop->getHeader()->getName() + ".lver.memcheck");
+
+  // Create empty preheader for the loop (and after cloning for the
+  // non-versioned loop).
+  BasicBlock *PH = SplitBlock(MemCheckBB, MemCheckBB->getTerminator(), DT, LI);
+  PH->setName(VersionedLoop->getHeader()->getName() + ".ph");
+
+  // Clone the loop including the preheader.
+  //
+  // FIXME: This does not currently preserve SimplifyLoop because the exit
+  // block is a join between the two loops.
+  SmallVector<BasicBlock *, 8> NonVersionedLoopBlocks;
+  NonVersionedLoop =
+      cloneLoopWithPreheader(PH, MemCheckBB, VersionedLoop, VMap, ".lver.orig",
+                             LI, DT, NonVersionedLoopBlocks);
+  remapInstructionsInBlocks(NonVersionedLoopBlocks, VMap);
+
+  // Insert the conditional branch based on the result of the memchecks.
+  Instruction *OrigTerm = MemCheckBB->getTerminator();
+  BranchInst::Create(NonVersionedLoop->getLoopPreheader(),
+                     VersionedLoop->getLoopPreheader(), MemRuntimeCheck,
+                     OrigTerm);
+  OrigTerm->eraseFromParent();
+
+  // The loops merge in the original exit block.  This is now dominated by the
+  // memchecking block.
+  DT->changeImmediateDominator(VersionedLoop->getExitBlock(), MemCheckBB);
+}
+
+void LoopVersioning::addPHINodes(
+    const SmallVectorImpl<Instruction *> &DefsUsedOutside) {
+  BasicBlock *PHIBlock = VersionedLoop->getExitBlock();
+  assert(PHIBlock && "No single successor to loop exit block");
+
+  for (auto *Inst : DefsUsedOutside) {
+    auto *NonVersionedLoopInst = cast<Instruction>(VMap[Inst]);
+    PHINode *PN;
+
+    // First see if we have a single-operand PHI with the value defined by the
+    // original loop.
+    for (auto I = PHIBlock->begin(); (PN = dyn_cast<PHINode>(I)); ++I) {
+      assert(PN->getNumOperands() == 1 &&
+             "Exit block should only have on predecessor");
+      if (PN->getIncomingValue(0) == Inst)
+        break;
+    }
+    // If not create it.
+    if (!PN) {
+      PN = PHINode::Create(Inst->getType(), 2, Inst->getName() + ".lver",
+                           PHIBlock->begin());
+      for (auto *User : Inst->users())
+        if (!VersionedLoop->contains(cast<Instruction>(User)->getParent()))
+          User->replaceUsesOfWith(Inst, PN);
+      PN->addIncoming(Inst, VersionedLoop->getExitingBlock());
+    }
+    // Add the new incoming value from the non-versioned loop.
+    PN->addIncoming(NonVersionedLoopInst, NonVersionedLoop->getExitingBlock());
+  }
+}
diff --git a/lib/Transforms/Utils/LowerSwitch.cpp b/lib/Transforms/Utils/LowerSwitch.cpp
index e0e0e900949..4acd988691d 100644
--- a/lib/Transforms/Utils/LowerSwitch.cpp
+++ b/lib/Transforms/Utils/LowerSwitch.cpp
@@ -364,9 +364,9 @@ unsigned LowerSwitch::Clusterify(CaseVector& Cases, SwitchInst *SI) {
   std::sort(Cases.begin(), Cases.end(), CaseCmp());
 
   // Merge case into clusters
-  if (Cases.size()>=2)
-    for (CaseItr I = Cases.begin(), J = std::next(Cases.begin());
-         J != Cases.end();) {
+  if (Cases.size() >= 2) {
+    CaseItr I = Cases.begin();
+    for (CaseItr J = std::next(I), E = Cases.end(); J != E; ++J) {
       int64_t nextValue = J->Low->getSExtValue();
       int64_t currentValue = I->High->getSExtValue();
       BasicBlock* nextBB = J->BB;
@@ -374,13 +374,16 @@ unsigned LowerSwitch::Clusterify(CaseVector& Cases, SwitchInst *SI) {
 
       // If the two neighboring cases go to the same destination, merge them
       // into a single case.
-      if ((nextValue-currentValue==1) && (currentBB == nextBB)) {
+      assert(nextValue > currentValue && "Cases should be strictly ascending");
+      if ((nextValue == currentValue + 1) && (currentBB == nextBB)) {
         I->High = J->High;
-        J = Cases.erase(J);
-      } else {
-        I = J++;
+        // FIXME: Combine branch weights.
+      } else if (++I != J) {
+        *I = *J;
       }
     }
+    Cases.erase(std::next(I), Cases.end());
+  }
 
   for (CaseItr I=Cases.begin(), E=Cases.end(); I!=E; ++I, ++numCmps) {
     if (I->Low != I->High)
@@ -476,12 +479,10 @@ void LowerSwitch::processSwitchInst(SwitchInst *SI) {
     // cases.
     assert(MaxPop > 0 && PopSucc);
     Default = PopSucc;
-    for (CaseItr I = Cases.begin(); I != Cases.end();) {
-      if (I->BB == PopSucc)
-        I = Cases.erase(I);
-      else
-        ++I;
-    }
+    Cases.erase(std::remove_if(
+                    Cases.begin(), Cases.end(),
+                    [PopSucc](const CaseRange &R) { return R.BB == PopSucc; }),
+                Cases.end());
 
     // If there are no cases left, just branch.
     if (Cases.empty()) {
diff --git a/lib/Transforms/Utils/ModuleUtils.cpp b/lib/Transforms/Utils/ModuleUtils.cpp
index 014574d859d..c5abf76d93f 100644
--- a/lib/Transforms/Utils/ModuleUtils.cpp
+++ b/lib/Transforms/Utils/ModuleUtils.cpp
@@ -104,3 +104,32 @@ Function *llvm::checkSanitizerInterfaceFunction(Constant *FuncOrBitcast) {
   Stream << "Sanitizer interface function redefined: " << *FuncOrBitcast;
   report_fatal_error(Err);
 }
+
+std::pair<Function *, Function *> llvm::createSanitizerCtorAndInitFunctions(
+    Module &M, StringRef CtorName, StringRef InitName,
+    ArrayRef<Type *> InitArgTypes, ArrayRef<Value *> InitArgs,
+    StringRef VersionCheckName) {
+  assert(!InitName.empty() && "Expected init function name");
+  assert(InitArgTypes.size() == InitArgTypes.size() &&
+         "Sanitizer's init function expects different number of arguments");
+  Function *Ctor = Function::Create(
+      FunctionType::get(Type::getVoidTy(M.getContext()), false),
+      GlobalValue::InternalLinkage, CtorName, &M);
+  BasicBlock *CtorBB = BasicBlock::Create(M.getContext(), "", Ctor);
+  IRBuilder<> IRB(ReturnInst::Create(M.getContext(), CtorBB));
+  Function *InitFunction =
+      checkSanitizerInterfaceFunction(M.getOrInsertFunction(
+          InitName, FunctionType::get(IRB.getVoidTy(), InitArgTypes, false),
+          AttributeSet()));
+  InitFunction->setLinkage(Function::ExternalLinkage);
+  IRB.CreateCall(InitFunction, InitArgs);
+  if (!VersionCheckName.empty()) {
+    Function *VersionCheckFunction =
+        checkSanitizerInterfaceFunction(M.getOrInsertFunction(
+            VersionCheckName, FunctionType::get(IRB.getVoidTy(), {}, false),
+            AttributeSet()));
+    IRB.CreateCall(VersionCheckFunction, {});
+  }
+  return std::make_pair(Ctor, InitFunction);
+}
+
diff --git a/lib/Transforms/Utils/PromoteMemoryToRegister.cpp b/lib/Transforms/Utils/PromoteMemoryToRegister.cpp
index 54e1733c3f6..828d6a173bb 100644
--- a/lib/Transforms/Utils/PromoteMemoryToRegister.cpp
+++ b/lib/Transforms/Utils/PromoteMemoryToRegister.cpp
@@ -13,16 +13,6 @@
 // traversing the function in depth-first order to rewrite loads and stores as
 // appropriate.
 //
-// The algorithm used here is based on:
-//
-//   Sreedhar and Gao. A linear time algorithm for placing phi-nodes.
-//   In Proceedings of the 22nd ACM SIGPLAN-SIGACT Symposium on Principles of
-//   Programming Languages
-//   POPL '95. ACM, New York, NY, 62-73.
-//
-// It has been modified to not explicitly use the DJ graph data structure and to
-// directly compute pruned SSA using per-variable liveness information.
-//
 //===----------------------------------------------------------------------===//
 
 #include "llvm/Transforms/Utils/PromoteMemToReg.h"
@@ -34,6 +24,7 @@
 #include "llvm/ADT/Statistic.h"
 #include "llvm/Analysis/AliasSetTracker.h"
 #include "llvm/Analysis/InstructionSimplify.h"
+#include "llvm/Analysis/IteratedDominanceFrontier.h"
 #include "llvm/Analysis/ValueTracking.h"
 #include "llvm/IR/CFG.h"
 #include "llvm/IR/Constants.h"
@@ -48,7 +39,6 @@
 #include "llvm/IR/Module.h"
 #include "llvm/Transforms/Utils/Local.h"
 #include <algorithm>
-#include <queue>
 using namespace llvm;
 
 #define DEBUG_TYPE "mem2reg"
@@ -275,9 +265,6 @@ struct PromoteMem2Reg {
   /// behavior.
   DenseMap<BasicBlock *, unsigned> BBNumbers;
 
-  /// Maps DomTreeNodes to their level in the dominator tree.
-  DenseMap<DomTreeNode *, unsigned> DomLevels;
-
   /// Lazily compute the number of predecessors a block has.
   DenseMap<const BasicBlock *, unsigned> BBNumPreds;
 
@@ -304,8 +291,6 @@ struct PromoteMem2Reg {
     return NP - 1;
   }
 
-  void DetermineInsertionPoint(AllocaInst *AI, unsigned AllocaNum,
-                               AllocaInfo &Info);
   void ComputeLiveInBlocks(AllocaInst *AI, AllocaInfo &Info,
                            const SmallPtrSetImpl<BasicBlock *> &DefBlocks,
                            SmallPtrSetImpl<BasicBlock *> &LiveInBlocks);
@@ -440,14 +425,17 @@ static bool rewriteSingleStoreAlloca(AllocaInst *AI, AllocaInfo &Info,
 /// using the Alloca.
 ///
 /// If we cannot promote this alloca (because it is read before it is written),
-/// return true.  This is necessary in cases where, due to control flow, the
-/// alloca is potentially undefined on some control flow paths.  e.g. code like
-/// this is potentially correct:
-///
-///   for (...) { if (c) { A = undef; undef = B; } }
-///
-/// ... so long as A is not used before undef is set.
-static void promoteSingleBlockAlloca(AllocaInst *AI, const AllocaInfo &Info,
+/// return false.  This is necessary in cases where, due to control flow, the
+/// alloca is undefined only on some control flow paths.  e.g. code like
+/// this is correct in LLVM IR:
+///  // A is an alloca with no stores so far
+///  for (...) {
+///    int t = *A;
+///    if (!first_iteration)
+///      use(t);
+///    *A = 42;
+///  }
+static bool promoteSingleBlockAlloca(AllocaInst *AI, const AllocaInfo &Info,
                                      LargeBlockInfo &LBI,
                                      AliasSetTracker *AST) {
   // The trickiest case to handle is when we have large blocks. Because of this,
@@ -482,10 +470,15 @@ static void promoteSingleBlockAlloca(AllocaInst *AI, const AllocaInfo &Info,
                          std::make_pair(LoadIdx,
                                         static_cast<StoreInst *>(nullptr)),
                          less_first());
-
-    if (I == StoresByIndex.begin())
-      // If there is no store before this load, the load takes the undef value.
-      LI->replaceAllUsesWith(UndefValue::get(LI->getType()));
+    if (I == StoresByIndex.begin()) {
+      if (StoresByIndex.empty())
+        // If there are no stores, the load takes the undef value.
+        LI->replaceAllUsesWith(UndefValue::get(LI->getType()));
+      else
+        // There is no store before this load, bail out (load may be affected
+        // by the following stores - see main comment).
+        return false;
+    }
     else
       // Otherwise, there was a store before this load, the load takes its value.
       LI->replaceAllUsesWith(std::prev(I)->second->getOperand(0));
@@ -521,6 +514,7 @@ static void promoteSingleBlockAlloca(AllocaInst *AI, const AllocaInfo &Info,
   }
 
   ++NumLocalPromoted;
+  return true;
 }
 
 void PromoteMem2Reg::run() {
@@ -532,6 +526,7 @@ void PromoteMem2Reg::run() {
 
   AllocaInfo Info;
   LargeBlockInfo LBI;
+  IDFCalculator IDF(DT);
 
   for (unsigned AllocaNum = 0; AllocaNum != Allocas.size(); ++AllocaNum) {
     AllocaInst *AI = Allocas[AllocaNum];
@@ -571,39 +566,19 @@ void PromoteMem2Reg::run() {
 
     // If the alloca is only read and written in one basic block, just perform a
     // linear sweep over the block to eliminate it.
-    if (Info.OnlyUsedInOneBlock) {
-      promoteSingleBlockAlloca(AI, Info, LBI, AST);
-
+    if (Info.OnlyUsedInOneBlock &&
+        promoteSingleBlockAlloca(AI, Info, LBI, AST)) {
       // The alloca has been processed, move on.
       RemoveFromAllocasList(AllocaNum);
       continue;
     }
 
-    // If we haven't computed dominator tree levels, do so now.
-    if (DomLevels.empty()) {
-      SmallVector<DomTreeNode *, 32> Worklist;
-
-      DomTreeNode *Root = DT.getRootNode();
-      DomLevels[Root] = 0;
-      Worklist.push_back(Root);
-
-      while (!Worklist.empty()) {
-        DomTreeNode *Node = Worklist.pop_back_val();
-        unsigned ChildLevel = DomLevels[Node] + 1;
-        for (DomTreeNode::iterator CI = Node->begin(), CE = Node->end();
-             CI != CE; ++CI) {
-          DomLevels[*CI] = ChildLevel;
-          Worklist.push_back(*CI);
-        }
-      }
-    }
-
     // If we haven't computed a numbering for the BB's in the function, do so
     // now.
     if (BBNumbers.empty()) {
       unsigned ID = 0;
-      for (Function::iterator I = F.begin(), E = F.end(); I != E; ++I)
-        BBNumbers[I] = ID++;
+      for (auto &BB : F)
+        BBNumbers[&BB] = ID++;
     }
 
     // If we have an AST to keep updated, remember some pointer value that is
@@ -622,7 +597,34 @@ void PromoteMem2Reg::run() {
     // the standard SSA construction algorithm.  Determine which blocks need PHI
     // nodes and see if we can optimize out some work by avoiding insertion of
     // dead phi nodes.
-    DetermineInsertionPoint(AI, AllocaNum, Info);
+
+
+    // Unique the set of defining blocks for efficient lookup.
+    SmallPtrSet<BasicBlock *, 32> DefBlocks;
+    DefBlocks.insert(Info.DefiningBlocks.begin(), Info.DefiningBlocks.end());
+
+    // Determine which blocks the value is live in.  These are blocks which lead
+    // to uses.
+    SmallPtrSet<BasicBlock *, 32> LiveInBlocks;
+    ComputeLiveInBlocks(AI, Info, DefBlocks, LiveInBlocks);
+
+    // At this point, we're committed to promoting the alloca using IDF's, and
+    // the standard SSA construction algorithm.  Determine which blocks need phi
+    // nodes and see if we can optimize out some work by avoiding insertion of
+    // dead phi nodes.
+    IDF.setLiveInBlocks(LiveInBlocks);
+    IDF.setDefiningBlocks(DefBlocks);
+    SmallVector<BasicBlock *, 32> PHIBlocks;
+    IDF.calculate(PHIBlocks);
+    if (PHIBlocks.size() > 1)
+      std::sort(PHIBlocks.begin(), PHIBlocks.end(),
+                [this](BasicBlock *A, BasicBlock *B) {
+                  return BBNumbers.lookup(A) < BBNumbers.lookup(B);
+                });
+
+    unsigned CurrentVersion = 0;
+    for (unsigned i = 0, e = PHIBlocks.size(); i != e; ++i)
+      QueuePhiNode(PHIBlocks[i], AllocaNum, CurrentVersion);
   }
 
   if (Allocas.empty())
@@ -642,7 +644,7 @@ void PromoteMem2Reg::run() {
   // and inserting the phi nodes we marked as necessary
   //
   std::vector<RenamePassData> RenamePassWorkList;
-  RenamePassWorkList.push_back(RenamePassData(F.begin(), nullptr, Values));
+  RenamePassWorkList.emplace_back(F.begin(), nullptr, std::move(Values));
   do {
     RenamePassData RPD;
     RPD.swap(RenamePassWorkList.back());
@@ -844,98 +846,6 @@ void PromoteMem2Reg::ComputeLiveInBlocks(
   }
 }
 
-/// At this point, we're committed to promoting the alloca using IDF's, and the
-/// standard SSA construction algorithm.  Determine which blocks need phi nodes
-/// and see if we can optimize out some work by avoiding insertion of dead phi
-/// nodes.
-void PromoteMem2Reg::DetermineInsertionPoint(AllocaInst *AI, unsigned AllocaNum,
-                                             AllocaInfo &Info) {
-  // Unique the set of defining blocks for efficient lookup.
-  SmallPtrSet<BasicBlock *, 32> DefBlocks;
-  DefBlocks.insert(Info.DefiningBlocks.begin(), Info.DefiningBlocks.end());
-
-  // Determine which blocks the value is live in.  These are blocks which lead
-  // to uses.
-  SmallPtrSet<BasicBlock *, 32> LiveInBlocks;
-  ComputeLiveInBlocks(AI, Info, DefBlocks, LiveInBlocks);
-
-  // Use a priority queue keyed on dominator tree level so that inserted nodes
-  // are handled from the bottom of the dominator tree upwards.
-  typedef std::pair<DomTreeNode *, unsigned> DomTreeNodePair;
-  typedef std::priority_queue<DomTreeNodePair, SmallVector<DomTreeNodePair, 32>,
-                              less_second> IDFPriorityQueue;
-  IDFPriorityQueue PQ;
-
-  for (BasicBlock *BB : DefBlocks) {
-    if (DomTreeNode *Node = DT.getNode(BB))
-      PQ.push(std::make_pair(Node, DomLevels[Node]));
-  }
-
-  SmallVector<std::pair<unsigned, BasicBlock *>, 32> DFBlocks;
-  SmallVector<DomTreeNode *, 32> Worklist;
-  SmallPtrSet<DomTreeNode *, 32> VisitedPQ;
-  SmallPtrSet<DomTreeNode *, 32> VisitedWorklist;
-
-  while (!PQ.empty()) {
-    DomTreeNodePair RootPair = PQ.top();
-    PQ.pop();
-    DomTreeNode *Root = RootPair.first;
-    unsigned RootLevel = RootPair.second;
-
-    // Walk all dominator tree children of Root, inspecting their CFG edges with
-    // targets elsewhere on the dominator tree. Only targets whose level is at
-    // most Root's level are added to the iterated dominance frontier of the
-    // definition set.
-
-    Worklist.clear();
-    Worklist.push_back(Root);
-    VisitedWorklist.insert(Root);
-
-    while (!Worklist.empty()) {
-      DomTreeNode *Node = Worklist.pop_back_val();
-      BasicBlock *BB = Node->getBlock();
-
-      for (succ_iterator SI = succ_begin(BB), SE = succ_end(BB); SI != SE;
-           ++SI) {
-        DomTreeNode *SuccNode = DT.getNode(*SI);
-
-        // Quickly skip all CFG edges that are also dominator tree edges instead
-        // of catching them below.
-        if (SuccNode->getIDom() == Node)
-          continue;
-
-        unsigned SuccLevel = DomLevels[SuccNode];
-        if (SuccLevel > RootLevel)
-          continue;
-
-        if (!VisitedPQ.insert(SuccNode).second)
-          continue;
-
-        BasicBlock *SuccBB = SuccNode->getBlock();
-        if (!LiveInBlocks.count(SuccBB))
-          continue;
-
-        DFBlocks.push_back(std::make_pair(BBNumbers[SuccBB], SuccBB));
-        if (!DefBlocks.count(SuccBB))
-          PQ.push(std::make_pair(SuccNode, SuccLevel));
-      }
-
-      for (DomTreeNode::iterator CI = Node->begin(), CE = Node->end(); CI != CE;
-           ++CI) {
-        if (VisitedWorklist.insert(*CI).second)
-          Worklist.push_back(*CI);
-      }
-    }
-  }
-
-  if (DFBlocks.size() > 1)
-    std::sort(DFBlocks.begin(), DFBlocks.end());
-
-  unsigned CurrentVersion = 0;
-  for (unsigned i = 0, e = DFBlocks.size(); i != e; ++i)
-    QueuePhiNode(DFBlocks[i].second, AllocaNum, CurrentVersion);
-}
-
 /// \brief Queue a phi-node to be added to a basic-block for a specific Alloca.
 ///
 /// Returns true if there wasn't already a phi-node for that variable
@@ -1071,7 +981,7 @@ void PromoteMem2Reg::RenamePass(BasicBlock *BB, BasicBlock *Pred,
 
   for (; I != E; ++I)
     if (VisitedSuccs.insert(*I).second)
-      Worklist.push_back(RenamePassData(*I, Pred, IncomingVals));
+      Worklist.emplace_back(*I, Pred, IncomingVals);
 
   goto NextIteration;
 }
diff --git a/lib/Transforms/Utils/SSAUpdater.cpp b/lib/Transforms/Utils/SSAUpdater.cpp
index 955ce30ed3d..88b39dd7f66 100644
--- a/lib/Transforms/Utils/SSAUpdater.cpp
+++ b/lib/Transforms/Utils/SSAUpdater.cpp
@@ -322,12 +322,12 @@ Value *SSAUpdater::GetValueAtEndOfBlockInternal(BasicBlock *BB) {
 //===----------------------------------------------------------------------===//
 
 LoadAndStorePromoter::
-LoadAndStorePromoter(const SmallVectorImpl<Instruction*> &Insts,
+LoadAndStorePromoter(ArrayRef<const Instruction*> Insts,
                      SSAUpdater &S, StringRef BaseName) : SSA(S) {
   if (Insts.empty()) return;
   
-  Value *SomeVal;
-  if (LoadInst *LI = dyn_cast<LoadInst>(Insts[0]))
+  const Value *SomeVal;
+  if (const LoadInst *LI = dyn_cast<LoadInst>(Insts[0]))
     SomeVal = LI;
   else
     SomeVal = cast<StoreInst>(Insts[0])->getOperand(0);
diff --git a/lib/Transforms/Utils/SimplifyCFG.cpp b/lib/Transforms/Utils/SimplifyCFG.cpp
index 7c239cb19b7..36781c1189c 100644
--- a/lib/Transforms/Utils/SimplifyCFG.cpp
+++ b/lib/Transforms/Utils/SimplifyCFG.cpp
@@ -138,9 +138,8 @@ class SimplifyCFGOpt {
 };
 }
 
-/// SafeToMergeTerminators - Return true if it is safe to merge these two
+/// Return true if it is safe to merge these two
 /// terminator instructions together.
-///
 static bool SafeToMergeTerminators(TerminatorInst *SI1, TerminatorInst *SI2) {
   if (SI1 == SI2) return false;  // Can't merge with self!
 
@@ -164,11 +163,9 @@ static bool SafeToMergeTerminators(TerminatorInst *SI1, TerminatorInst *SI2) {
   return true;
 }
 
-/// isProfitableToFoldUnconditional - Return true if it is safe and profitable
-/// to merge these two terminator instructions together, where SI1 is an
-/// unconditional branch. PhiNodes will store all PHI nodes in common
-/// successors.
-///
+/// Return true if it is safe and profitable to merge these two terminator
+/// instructions together, where SI1 is an unconditional branch. PhiNodes will
+/// store all PHI nodes in common successors.
 static bool isProfitableToFoldUnconditional(BranchInst *SI1,
                                           BranchInst *SI2,
                                           Instruction *Cond,
@@ -205,10 +202,10 @@ static bool isProfitableToFoldUnconditional(BranchInst *SI1,
   return true;
 }
 
-/// AddPredecessorToBlock - Update PHI nodes in Succ to indicate that there will
-/// now be entries in it from the 'NewPred' block.  The values that will be
-/// flowing into the PHI nodes will be the same as those coming in from
-/// ExistPred, an existing predecessor of Succ.
+/// Update PHI nodes in Succ to indicate that there will now be entries in it
+/// from the 'NewPred' block. The values that will be flowing into the PHI nodes
+/// will be the same as those coming in from ExistPred, an existing predecessor
+/// of Succ.
 static void AddPredecessorToBlock(BasicBlock *Succ, BasicBlock *NewPred,
                                   BasicBlock *ExistPred) {
   if (!isa<PHINode>(Succ->begin())) return; // Quick exit if nothing to do
@@ -219,9 +216,9 @@ static void AddPredecessorToBlock(BasicBlock *Succ, BasicBlock *NewPred,
     PN->addIncoming(PN->getIncomingValueForBlock(ExistPred), NewPred);
 }
 
-/// ComputeSpeculationCost - Compute an abstract "cost" of speculating the
-/// given instruction, which is assumed to be safe to speculate. TCC_Free means
-/// cheap, TCC_Basic means less cheap, and TCC_Expensive means prohibitively
+/// Compute an abstract "cost" of speculating the given instruction,
+/// which is assumed to be safe to speculate. TCC_Free means cheap,
+/// TCC_Basic means less cheap, and TCC_Expensive means prohibitively
 /// expensive.
 static unsigned ComputeSpeculationCost(const User *I,
                                        const TargetTransformInfo &TTI) {
@@ -229,8 +226,8 @@ static unsigned ComputeSpeculationCost(const User *I,
          "Instruction is not safe to speculatively execute!");
   return TTI.getUserCost(I);
 }
-/// DominatesMergePoint - If we have a merge point of an "if condition" as
-/// accepted above, return true if the specified value dominates the block.  We
+/// If we have a merge point of an "if condition" as accepted above,
+/// return true if the specified value dominates the block.  We
 /// don't handle the true generality of domination here, just a special case
 /// which works well enough for us.
 ///
@@ -302,7 +299,7 @@ static bool DominatesMergePoint(Value *V, BasicBlock *BB,
   return true;
 }
 
-/// GetConstantInt - Extract ConstantInt from value, looking through IntToPtr
+/// Extract ConstantInt from value, looking through IntToPtr
 /// and PointerNullValue. Return NULL if value is not a constant int.
 static ConstantInt *GetConstantInt(Value *V, const DataLayout &DL) {
   // Normal constant int.
@@ -456,7 +453,7 @@ struct ConstantComparesGatherer {
 
   }
 
-  /// gather - Given a potentially 'or'd or 'and'd together collection of icmp
+  /// Given a potentially 'or'd or 'and'd together collection of icmp
   /// eq/ne/lt/gt instructions that compare a value against a constant, extract
   /// the value being compared, and stick the list constants into the Vals
   /// vector.
@@ -519,7 +516,7 @@ static void EraseTerminatorInstAndDCECond(TerminatorInst *TI) {
   if (Cond) RecursivelyDeleteTriviallyDeadInstructions(Cond);
 }
 
-/// isValueEqualityComparison - Return true if the specified terminator checks
+/// Return true if the specified terminator checks
 /// to see if a value is equal to constant integer value.
 Value *SimplifyCFGOpt::isValueEqualityComparison(TerminatorInst *TI) {
   Value *CV = nullptr;
@@ -547,7 +544,7 @@ Value *SimplifyCFGOpt::isValueEqualityComparison(TerminatorInst *TI) {
   return CV;
 }
 
-/// GetValueEqualityComparisonCases - Given a value comparison instruction,
+/// Given a value comparison instruction,
 /// decode all of the 'cases' that it represents and return the 'default' block.
 BasicBlock *SimplifyCFGOpt::
 GetValueEqualityComparisonCases(TerminatorInst *TI,
@@ -571,15 +568,14 @@ GetValueEqualityComparisonCases(TerminatorInst *TI,
 }
 
 
-/// EliminateBlockCases - Given a vector of bb/value pairs, remove any entries
+/// Given a vector of bb/value pairs, remove any entries
 /// in the list that match the specified block.
 static void EliminateBlockCases(BasicBlock *BB,
                               std::vector<ValueEqualityComparisonCase> &Cases) {
   Cases.erase(std::remove(Cases.begin(), Cases.end(), BB), Cases.end());
 }
 
-/// ValuesOverlap - Return true if there are any keys in C1 that exist in C2 as
-/// well.
+/// Return true if there are any keys in C1 that exist in C2 as well.
 static bool
 ValuesOverlap(std::vector<ValueEqualityComparisonCase> &C1,
               std::vector<ValueEqualityComparisonCase > &C2) {
@@ -613,12 +609,11 @@ ValuesOverlap(std::vector<ValueEqualityComparisonCase> &C1,
   return false;
 }
 
-/// SimplifyEqualityComparisonWithOnlyPredecessor - If TI is known to be a
-/// terminator instruction and its block is known to only have a single
-/// predecessor block, check to see if that predecessor is also a value
-/// comparison with the same value, and if that comparison determines the
-/// outcome of this comparison.  If so, simplify TI.  This does a very limited
-/// form of jump threading.
+/// If TI is known to be a terminator instruction and its block is known to
+/// only have a single predecessor block, check to see if that predecessor is
+/// also a value comparison with the same value, and if that comparison
+/// determines the outcome of this comparison. If so, simplify TI. This does a
+/// very limited form of jump threading.
 bool SimplifyCFGOpt::
 SimplifyEqualityComparisonWithOnlyPredecessor(TerminatorInst *TI,
                                               BasicBlock *Pred,
@@ -754,7 +749,7 @@ SimplifyEqualityComparisonWithOnlyPredecessor(TerminatorInst *TI,
 }
 
 namespace {
-  /// ConstantIntOrdering - This class implements a stable ordering of constant
+  /// This class implements a stable ordering of constant
   /// integers that does not depend on their address.  This is important for
   /// applications that sort ConstantInt's to ensure uniqueness.
   struct ConstantIntOrdering {
@@ -817,8 +812,8 @@ static void FitWeights(MutableArrayRef<uint64_t> Weights) {
   }
 }
 
-/// FoldValueComparisonIntoPredecessors - The specified terminator is a value
-/// equality comparison instruction (either a switch or a branch on "X == c").
+/// The specified terminator is a value equality comparison instruction
+/// (either a switch or a branch on "X == c").
 /// See if any of the predecessors of the terminator block are value comparisons
 /// on the same value.  If so, and if safe to do so, fold them together.
 bool SimplifyCFGOpt::FoldValueComparisonIntoPredecessors(TerminatorInst *TI,
@@ -1027,10 +1022,9 @@ bool SimplifyCFGOpt::FoldValueComparisonIntoPredecessors(TerminatorInst *TI,
   return Changed;
 }
 
-// isSafeToHoistInvoke - If we would need to insert a select that uses the
-// value of this invoke (comments in HoistThenElseCodeToIf explain why we
-// would need to do this), we can't hoist the invoke, as there is nowhere
-// to put the select in this case.
+// If we would need to insert a select that uses the value of this invoke
+// (comments in HoistThenElseCodeToIf explain why we would need to do this), we
+// can't hoist the invoke, as there is nowhere to put the select in this case.
 static bool isSafeToHoistInvoke(BasicBlock *BB1, BasicBlock *BB2,
                                 Instruction *I1, Instruction *I2) {
   for (succ_iterator SI = succ_begin(BB1), E = succ_end(BB1); SI != E; ++SI) {
@@ -1049,9 +1043,9 @@ static bool isSafeToHoistInvoke(BasicBlock *BB1, BasicBlock *BB2,
 
 static bool passingValueIsAlwaysUndefined(Value *V, Instruction *I);
 
-/// HoistThenElseCodeToIf - Given a conditional branch that goes to BB1 and
-/// BB2, hoist any common code in the two blocks up into the branch block.  The
-/// caller of this function guarantees that BI's block dominates BB1 and BB2.
+/// Given a conditional branch that goes to BB1 and BB2, hoist any common code
+/// in the two blocks up into the branch block. The caller of this function
+/// guarantees that BI's block dominates BB1 and BB2.
 static bool HoistThenElseCodeToIf(BranchInst *BI,
                                   const TargetTransformInfo &TTI) {
   // This does very trivial matching, with limited scanning, to find identical
@@ -1197,7 +1191,7 @@ static bool HoistThenElseCodeToIf(BranchInst *BI,
   return true;
 }
 
-/// SinkThenElseCodeToEnd - Given an unconditional branch that goes to BBEnd,
+/// Given an unconditional branch that goes to BBEnd,
 /// check whether BBEnd has only two predecessors and the other predecessor
 /// ends with an unconditional branch. If it is true, sink any common code
 /// in the two predecessors to BBEnd.
@@ -1656,8 +1650,7 @@ static bool HasNoDuplicateCall(const BasicBlock *BB) {
   return false;
 }
 
-/// BlockIsSimpleEnoughToThreadThrough - Return true if we can thread a branch
-/// across this block.
+/// Return true if we can thread a branch across this block.
 static bool BlockIsSimpleEnoughToThreadThrough(BasicBlock *BB) {
   BranchInst *BI = cast<BranchInst>(BB->getTerminator());
   unsigned Size = 0;
@@ -1681,10 +1674,9 @@ static bool BlockIsSimpleEnoughToThreadThrough(BasicBlock *BB) {
   return true;
 }
 
-/// FoldCondBranchOnPHI - If we have a conditional branch on a PHI node value
-/// that is defined in the same block as the branch and if any PHI entries are
-/// constants, thread edges corresponding to that entry to be branches to their
-/// ultimate destination.
+/// If we have a conditional branch on a PHI node value that is defined in the
+/// same block as the branch and if any PHI entries are constants, thread edges
+/// corresponding to that entry to be branches to their ultimate destination.
 static bool FoldCondBranchOnPHI(BranchInst *BI, const DataLayout &DL) {
   BasicBlock *BB = BI->getParent();
   PHINode *PN = dyn_cast<PHINode>(BI->getCondition());
@@ -1781,8 +1773,8 @@ static bool FoldCondBranchOnPHI(BranchInst *BI, const DataLayout &DL) {
   return false;
 }
 
-/// FoldTwoEntryPHINode - Given a BB that starts with the specified two-entry
-/// PHI node, see if we can eliminate it.
+/// Given a BB that starts with the specified two-entry PHI node,
+/// see if we can eliminate it.
 static bool FoldTwoEntryPHINode(PHINode *PN, const TargetTransformInfo &TTI,
                                 const DataLayout &DL) {
   // Ok, this is a two entry PHI node.  Check to see if this is a simple "if
@@ -1920,8 +1912,8 @@ static bool FoldTwoEntryPHINode(PHINode *PN, const TargetTransformInfo &TTI,
   return true;
 }
 
-/// SimplifyCondBranchToTwoReturns - If we found a conditional branch that goes
-/// to two returning blocks, try to merge them together into one return,
+/// If we found a conditional branch that goes to two returning blocks,
+/// try to merge them together into one return,
 /// introducing a select if the return values disagree.
 static bool SimplifyCondBranchToTwoReturns(BranchInst *BI,
                                            IRBuilder<> &Builder) {
@@ -2008,10 +2000,9 @@ static bool SimplifyCondBranchToTwoReturns(BranchInst *BI,
   return true;
 }
 
-/// ExtractBranchMetadata - Given a conditional BranchInstruction, retrieve the
-/// probabilities of the branch taking each edge. Fills in the two APInt
-/// parameters and return true, or returns false if no or invalid metadata was
-/// found.
+/// Given a conditional BranchInstruction, retrieve the probabilities of the
+/// branch taking each edge. Fills in the two APInt parameters and returns true,
+/// or returns false if no or invalid metadata was found.
 static bool ExtractBranchMetadata(BranchInst *BI,
                                   uint64_t &ProbTrue, uint64_t &ProbFalse) {
   assert(BI->isConditional() &&
@@ -2028,9 +2019,8 @@ static bool ExtractBranchMetadata(BranchInst *BI,
   return true;
 }
 
-/// checkCSEInPredecessor - Return true if the given instruction is available
+/// Return true if the given instruction is available
 /// in its predecessor block. If yes, the instruction will be removed.
-///
 static bool checkCSEInPredecessor(Instruction *Inst, BasicBlock *PB) {
   if (!isa<BinaryOperator>(Inst) && !isa<CmpInst>(Inst))
     return false;
@@ -2046,9 +2036,9 @@ static bool checkCSEInPredecessor(Instruction *Inst, BasicBlock *PB) {
   return false;
 }
 
-/// FoldBranchToCommonDest - If this basic block is simple enough, and if a
-/// predecessor branches to us and one of our successors, fold the block into
-/// the predecessor and use logical operations to pick the right destination.
+/// If this basic block is simple enough, and if a predecessor branches to us
+/// and one of our successors, fold the block into the predecessor and use
+/// logical operations to pick the right destination.
 bool llvm::FoldBranchToCommonDest(BranchInst *BI, unsigned BonusInstThreshold) {
   BasicBlock *BB = BI->getParent();
 
@@ -2190,11 +2180,11 @@ bool llvm::FoldBranchToCommonDest(BranchInst *BI, unsigned BonusInstThreshold) {
     }
 
     // If we have bonus instructions, clone them into the predecessor block.
-    // Note that there may be mutliple predecessor blocks, so we cannot move
+    // Note that there may be multiple predecessor blocks, so we cannot move
     // bonus instructions to a predecessor block.
     ValueToValueMapTy VMap; // maps original values to cloned values
     // We already make sure Cond is the last instruction before BI. Therefore,
-    // every instructions before Cond other than DbgInfoIntrinsic are bonus
+    // all instructions before Cond other than DbgInfoIntrinsic are bonus
     // instructions.
     for (auto BonusInst = BB->begin(); Cond != BonusInst; ++BonusInst) {
       if (isa<DbgInfoIntrinsic>(BonusInst))
@@ -2342,8 +2332,8 @@ bool llvm::FoldBranchToCommonDest(BranchInst *BI, unsigned BonusInstThreshold) {
   return false;
 }
 
-/// SimplifyCondBranchToCondBranch - If we have a conditional branch as a
-/// predecessor of another block, this function tries to simplify it.  We know
+/// If we have a conditional branch as a predecessor of another block,
+/// this function tries to simplify it.  We know
 /// that PBI and BI are both conditional branches, and BI is in one of the
 /// successor blocks of PBI - PBI branches to BI.
 static bool SimplifyCondBranchToCondBranch(BranchInst *PBI, BranchInst *BI) {
@@ -2558,8 +2548,8 @@ static bool SimplifyCondBranchToCondBranch(BranchInst *PBI, BranchInst *BI) {
   return true;
 }
 
-// SimplifyTerminatorOnSelect - Simplifies a terminator by replacing it with a
-// branch to TrueBB if Cond is true or to FalseBB if Cond is false.
+// Simplifies a terminator by replacing it with a branch to TrueBB if Cond is
+// true or to FalseBB if Cond is false.
 // Takes care of updating the successors and removing the old terminator.
 // Also makes sure not to introduce new successors by assuming that edges to
 // non-successor TrueBBs and FalseBBs aren't reachable.
@@ -2624,7 +2614,7 @@ static bool SimplifyTerminatorOnSelect(TerminatorInst *OldTerm, Value *Cond,
   return true;
 }
 
-// SimplifySwitchOnSelect - Replaces
+// Replaces
 //   (switch (select cond, X, Y)) on constant X, Y
 // with a branch - conditional if X and Y lead to distinct BBs,
 // unconditional otherwise.
@@ -2659,7 +2649,7 @@ static bool SimplifySwitchOnSelect(SwitchInst *SI, SelectInst *Select) {
                                     TrueWeight, FalseWeight);
 }
 
-// SimplifyIndirectBrOnSelect - Replaces
+// Replaces
 //   (indirectbr (select cond, blockaddress(@fn, BlockA),
 //                             blockaddress(@fn, BlockB)))
 // with
@@ -2680,8 +2670,8 @@ static bool SimplifyIndirectBrOnSelect(IndirectBrInst *IBI, SelectInst *SI) {
                                     0, 0);
 }
 
-/// TryToSimplifyUncondBranchWithICmpInIt - This is called when we find an icmp
-/// instruction (a seteq/setne with a constant) as the only instruction in a
+/// This is called when we find an icmp instruction
+/// (a seteq/setne with a constant) as the only instruction in a
 /// block that ends with an uncond branch.  We are looking for a very specific
 /// pattern that occurs when "A == 1 || A == 2 || A == 3" gets simplified.  In
 /// this case, we merge the first two "or's of icmp" into a switch, but then the
@@ -2802,7 +2792,7 @@ static bool TryToSimplifyUncondBranchWithICmpInIt(
   return true;
 }
 
-/// SimplifyBranchOnICmpChain - The specified branch is a conditional branch.
+/// The specified branch is a conditional branch.
 /// Check to see if it is branching on an or/and chain of icmp instructions, and
 /// fold it into a switch instruction if so.
 static bool SimplifyBranchOnICmpChain(BranchInst *BI, IRBuilder<> &Builder,
@@ -3239,7 +3229,7 @@ static bool TurnSwitchRangeIntoICmp(SwitchInst *SI, IRBuilder<> &Builder) {
   return true;
 }
 
-/// EliminateDeadSwitchCases - Compute masked bits for the condition of a switch
+/// Compute masked bits for the condition of a switch
 /// and use it to remove dead cases.
 static bool EliminateDeadSwitchCases(SwitchInst *SI, AssumptionCache *AC,
                                      const DataLayout &DL) {
@@ -3290,8 +3280,8 @@ static bool EliminateDeadSwitchCases(SwitchInst *SI, AssumptionCache *AC,
   return !DeadCases.empty();
 }
 
-/// FindPHIForConditionForwarding - If BB would be eligible for simplification
-/// by TryToSimplifyUncondBranchFromEmptyBlock (i.e. it is empty and terminated
+/// If BB would be eligible for simplification by
+/// TryToSimplifyUncondBranchFromEmptyBlock (i.e. it is empty and terminated
 /// by an unconditional branch), look at the phi node for BB in the successor
 /// block and see if the incoming value is equal to CaseValue. If so, return
 /// the phi node, and set PhiIndex to BB's index in the phi node.
@@ -3324,9 +3314,9 @@ static PHINode *FindPHIForConditionForwarding(ConstantInt *CaseValue,
   return nullptr;
 }
 
-/// ForwardSwitchConditionToPHI - Try to forward the condition of a switch
-/// instruction to a phi node dominated by the switch, if that would mean that
-/// some of the destination blocks of the switch can be folded away.
+/// Try to forward the condition of a switch instruction to a phi node
+/// dominated by the switch, if that would mean that some of the destination
+/// blocks of the switch can be folded away.
 /// Returns true if a change is made.
 static bool ForwardSwitchConditionToPHI(SwitchInst *SI) {
   typedef DenseMap<PHINode*, SmallVector<int,4> > ForwardingNodesMap;
@@ -3361,7 +3351,7 @@ static bool ForwardSwitchConditionToPHI(SwitchInst *SI) {
   return Changed;
 }
 
-/// ValidLookupTableConstant - Return true if the backend will be able to handle
+/// Return true if the backend will be able to handle
 /// initializing an array of constants like C.
 static bool ValidLookupTableConstant(Constant *C) {
   if (C->isThreadDependent())
@@ -3379,7 +3369,7 @@ static bool ValidLookupTableConstant(Constant *C) {
       isa<UndefValue>(C);
 }
 
-/// LookupConstant - If V is a Constant, return it. Otherwise, try to look up
+/// If V is a Constant, return it. Otherwise, try to look up
 /// its constant value in ConstantPool, returning 0 if it's not there.
 static Constant *LookupConstant(Value *V,
                          const SmallDenseMap<Value*, Constant*>& ConstantPool) {
@@ -3388,7 +3378,7 @@ static Constant *LookupConstant(Value *V,
   return ConstantPool.lookup(V);
 }
 
-/// ConstantFold - Try to fold instruction I into a constant. This works for
+/// Try to fold instruction I into a constant. This works for
 /// simple instructions such as binary operations where both operands are
 /// constant or can be replaced by constants from the ConstantPool. Returns the
 /// resulting constant on success, 0 otherwise.
@@ -3422,7 +3412,7 @@ ConstantFold(Instruction *I, const DataLayout &DL,
   return ConstantFoldInstOperands(I->getOpcode(), I->getType(), COps, DL);
 }
 
-/// GetCaseResults - Try to determine the resulting constant values in phi nodes
+/// Try to determine the resulting constant values in phi nodes
 /// at the common destination basic block, *CommonDest, for one of the case
 /// destionations CaseDest corresponding to value CaseVal (0 for the default
 /// case), of a switch instruction SI.
@@ -3501,8 +3491,8 @@ GetCaseResults(SwitchInst *SI, ConstantInt *CaseVal, BasicBlock *CaseDest,
   return Res.size() > 0;
 }
 
-// MapCaseToResult - Helper function used to
-// add CaseVal to the list of cases that generate Result.
+// Helper function used to add CaseVal to the list of cases that generate
+// Result.
 static void MapCaseToResult(ConstantInt *CaseVal,
     SwitchCaseResultVectorTy &UniqueResults,
     Constant *Result) {
@@ -3516,7 +3506,7 @@ static void MapCaseToResult(ConstantInt *CaseVal,
         SmallVector<ConstantInt*, 4>(1, CaseVal)));
 }
 
-// InitializeUniqueCases - Helper function that initializes a map containing
+// Helper function that initializes a map containing
 // results for the PHI node of the common destination block for a switch
 // instruction. Returns false if multiple PHI nodes have been found or if
 // there is not a common destination block for the switch.
@@ -3561,9 +3551,8 @@ static bool InitializeUniqueCases(SwitchInst *SI, PHINode *&PHI,
   return true;
 }
 
-// ConvertTwoCaseSwitch - Helper function that checks if it is possible to
-// transform a switch with only two cases (or two cases + default)
-// that produces a result into a value select.
+// Helper function that checks if it is possible to transform a switch with only
+// two cases (or two cases + default) that produces a result into a select.
 // Example:
 // switch (a) {
 //   case 10:                %0 = icmp eq i32 %a, 10
@@ -3603,9 +3592,8 @@ ConvertTwoCaseSwitch(const SwitchCaseResultVectorTy &ResultVector,
   return nullptr;
 }
 
-// RemoveSwitchAfterSelectConversion - Helper function to cleanup a switch
-// instruction that has been converted into a select, fixing up PHI nodes and
-// basic blocks.
+// Helper function to cleanup a switch instruction that has been converted into
+// a select, fixing up PHI nodes and basic blocks.
 static void RemoveSwitchAfterSelectConversion(SwitchInst *SI, PHINode *PHI,
                                               Value *SelectValue,
                                               IRBuilder<> &Builder) {
@@ -3627,7 +3615,7 @@ static void RemoveSwitchAfterSelectConversion(SwitchInst *SI, PHINode *PHI,
   SI->eraseFromParent();
 }
 
-/// SwitchToSelect - If the switch is only used to initialize one or more
+/// If the switch is only used to initialize one or more
 /// phi nodes in a common successor block with only two different
 /// constant values, replace the switch with select.
 static bool SwitchToSelect(SwitchInst *SI, IRBuilder<> &Builder,
@@ -3659,23 +3647,21 @@ static bool SwitchToSelect(SwitchInst *SI, IRBuilder<> &Builder,
 }
 
 namespace {
-  /// SwitchLookupTable - This class represents a lookup table that can be used
-  /// to replace a switch.
+  /// This class represents a lookup table that can be used to replace a switch.
   class SwitchLookupTable {
   public:
-    /// SwitchLookupTable - Create a lookup table to use as a switch replacement
-    /// with the contents of Values, using DefaultValue to fill any holes in the
-    /// table.
+    /// Create a lookup table to use as a switch replacement with the contents
+    /// of Values, using DefaultValue to fill any holes in the table.
     SwitchLookupTable(
         Module &M, uint64_t TableSize, ConstantInt *Offset,
         const SmallVectorImpl<std::pair<ConstantInt *, Constant *>> &Values,
         Constant *DefaultValue, const DataLayout &DL);
 
-    /// BuildLookup - Build instructions with Builder to retrieve the value at
+    /// Build instructions with Builder to retrieve the value at
     /// the position given by Index in the lookup table.
     Value *BuildLookup(Value *Index, IRBuilder<> &Builder);
 
-    /// WouldFitInRegister - Return true if a table with TableSize elements of
+    /// Return true if a table with TableSize elements of
     /// type ElementType would fit in a target-legal register.
     static bool WouldFitInRegister(const DataLayout &DL, uint64_t TableSize,
                                    const Type *ElementType);
@@ -3907,9 +3893,8 @@ bool SwitchLookupTable::WouldFitInRegister(const DataLayout &DL,
   return DL.fitsInLegalInteger(TableSize * IT->getBitWidth());
 }
 
-/// ShouldBuildLookupTable - Determine whether a lookup table should be built
-/// for this switch, based on the number of cases, size of the table and the
-/// types of the results.
+/// Determine whether a lookup table should be built for this switch, based on
+/// the number of cases, size of the table, and the types of the results.
 static bool
 ShouldBuildLookupTable(SwitchInst *SI, uint64_t TableSize,
                        const TargetTransformInfo &TTI, const DataLayout &DL,
@@ -4033,9 +4018,9 @@ static void reuseTableCompare(User *PhiUser, BasicBlock *PhiBlock,
   }
 }
 
-/// SwitchToLookupTable - If the switch is only used to initialize one or more
-/// phi nodes in a common successor block with different constant values,
-/// replace the switch with lookup tables.
+/// If the switch is only used to initialize one or more phi nodes in a common
+/// successor block with different constant values, replace the switch with
+/// lookup tables.
 static bool SwitchToLookupTable(SwitchInst *SI, IRBuilder<> &Builder,
                                 const DataLayout &DL,
                                 const TargetTransformInfo &TTI) {
@@ -4058,7 +4043,7 @@ static bool SwitchToLookupTable(SwitchInst *SI, IRBuilder<> &Builder,
     return false;
 
   // Figure out the corresponding result for each case value and phi node in the
-  // common destination, as well as the the min and max case values.
+  // common destination, as well as the min and max case values.
   assert(SI->case_begin() != SI->case_end());
   SwitchInst::CaseIt CI = SI->case_begin();
   ConstantInt *MinCaseVal = CI.getCaseValue();
@@ -4159,10 +4144,8 @@ static bool SwitchToLookupTable(SwitchInst *SI, IRBuilder<> &Builder,
 
   if (!DefaultIsReachable || GeneratingCoveredLookupTable) {
     Builder.CreateBr(LookupBB);
-    // We cached PHINodes in PHIs, to avoid accessing deleted PHINodes later,
-    // do not delete PHINodes here.
-    SI->getDefaultDest()->removePredecessor(SI->getParent(),
-                                            /*DontDeleteUselessPHIs=*/true);
+    // Note: We call removeProdecessor later since we need to be able to get the
+    // PHI value for the default case in case we're using a bit mask.
   } else {
     Value *Cmp = Builder.CreateICmpULT(TableIndex, ConstantInt::get(
                                        MinCaseVal->getType(), TableSize));
@@ -4214,6 +4197,13 @@ static bool SwitchToLookupTable(SwitchInst *SI, IRBuilder<> &Builder,
     AddPredecessorToBlock(SI->getDefaultDest(), MaskBB, SI->getParent());
   }
 
+  if (!DefaultIsReachable || GeneratingCoveredLookupTable) {
+    // We cached PHINodes in PHIs, to avoid accessing deleted PHINodes later,
+    // do not delete PHINodes here.
+    SI->getDefaultDest()->removePredecessor(SI->getParent(),
+                                            /*DontDeleteUselessPHIs=*/true);
+  }
+
   bool ReturnedEarly = false;
   for (size_t I = 0, E = PHIs.size(); I != E; ++I) {
     PHINode *PHI = PHIs[I];
@@ -4686,8 +4676,8 @@ bool SimplifyCFGOpt::run(BasicBlock *BB) {
   return Changed;
 }
 
-/// SimplifyCFG - This function is used to do simplification of a CFG.  For
-/// example, it adjusts branches to branches to eliminate the extra hop, it
+/// This function is used to do simplification of a CFG.
+/// For example, it adjusts branches to branches to eliminate the extra hop,
 /// eliminates unreachable basic blocks, and does other "peephole" optimization
 /// of the CFG.  It returns true if a modification was made.
 ///
diff --git a/lib/Transforms/Utils/SimplifyIndVar.cpp b/lib/Transforms/Utils/SimplifyIndVar.cpp
index 8bfc5fbca6f..b3055a492be 100644
--- a/lib/Transforms/Utils/SimplifyIndVar.cpp
+++ b/lib/Transforms/Utils/SimplifyIndVar.cpp
@@ -17,7 +17,6 @@
 #include "llvm/ADT/STLExtras.h"
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/ADT/Statistic.h"
-#include "llvm/Analysis/IVUsers.h"
 #include "llvm/Analysis/LoopInfo.h"
 #include "llvm/Analysis/LoopPass.h"
 #include "llvm/Analysis/ScalarEvolutionExpressions.h"
@@ -55,7 +54,7 @@ namespace {
 
   public:
     SimplifyIndvar(Loop *Loop, ScalarEvolution *SE, LoopInfo *LI,
-                   SmallVectorImpl<WeakVH> &Dead, IVUsers *IVU = nullptr)
+                   SmallVectorImpl<WeakVH> &Dead)
         : L(Loop), LI(LI), SE(SE), DeadInsts(Dead), Changed(false) {
       assert(LI && "IV simplification requires LoopInfo");
     }
@@ -142,7 +141,7 @@ Value *SimplifyIndvar::foldIVUser(Instruction *UseInst, Instruction *IVOperand)
   ++NumElimOperand;
   Changed = true;
   if (IVOperand->use_empty())
-    DeadInsts.push_back(IVOperand);
+    DeadInsts.emplace_back(IVOperand);
   return IVSrc;
 }
 
@@ -167,19 +166,68 @@ void SimplifyIndvar::eliminateIVComparison(ICmpInst *ICmp, Value *IVOperand) {
   S = SE->getSCEVAtScope(S, ICmpLoop);
   X = SE->getSCEVAtScope(X, ICmpLoop);
 
+  ICmpInst::Predicate InvariantPredicate;
+  const SCEV *InvariantLHS, *InvariantRHS;
+
+  const char *Verb = nullptr;
+
   // If the condition is always true or always false, replace it with
   // a constant value.
-  if (SE->isKnownPredicate(Pred, S, X))
+  if (SE->isKnownPredicate(Pred, S, X)) {
     ICmp->replaceAllUsesWith(ConstantInt::getTrue(ICmp->getContext()));
-  else if (SE->isKnownPredicate(ICmpInst::getInversePredicate(Pred), S, X))
+    DeadInsts.emplace_back(ICmp);
+    Verb = "Eliminated";
+  } else if (SE->isKnownPredicate(ICmpInst::getInversePredicate(Pred), S, X)) {
     ICmp->replaceAllUsesWith(ConstantInt::getFalse(ICmp->getContext()));
-  else
+    DeadInsts.emplace_back(ICmp);
+    Verb = "Eliminated";
+  } else if (isa<PHINode>(IVOperand) &&
+             SE->isLoopInvariantPredicate(Pred, S, X, ICmpLoop,
+                                          InvariantPredicate, InvariantLHS,
+                                          InvariantRHS)) {
+
+    // Rewrite the comparision to a loop invariant comparision if it can be done
+    // cheaply, where cheaply means "we don't need to emit any new
+    // instructions".
+
+    Value *NewLHS = nullptr, *NewRHS = nullptr;
+
+    if (S == InvariantLHS || X == InvariantLHS)
+      NewLHS =
+          ICmp->getOperand(S == InvariantLHS ? IVOperIdx : (1 - IVOperIdx));
+
+    if (S == InvariantRHS || X == InvariantRHS)
+      NewRHS =
+          ICmp->getOperand(S == InvariantRHS ? IVOperIdx : (1 - IVOperIdx));
+
+    for (Value *Incoming : cast<PHINode>(IVOperand)->incoming_values()) {
+      if (NewLHS && NewRHS)
+        break;
+
+      const SCEV *IncomingS = SE->getSCEV(Incoming);
+
+      if (!NewLHS && IncomingS == InvariantLHS)
+        NewLHS = Incoming;
+      if (!NewRHS && IncomingS == InvariantRHS)
+        NewRHS = Incoming;
+    }
+
+    if (!NewLHS || !NewRHS)
+      // We could not find an existing value to replace either LHS or RHS.
+      // Generating new instructions has subtler tradeoffs, so avoid doing that
+      // for now.
+      return;
+
+    Verb = "Simplified";
+    ICmp->setPredicate(InvariantPredicate);
+    ICmp->setOperand(0, NewLHS);
+    ICmp->setOperand(1, NewRHS);
+  } else
     return;
 
-  DEBUG(dbgs() << "INDVARS: Eliminated comparison: " << *ICmp << '\n');
+  DEBUG(dbgs() << "INDVARS: " << Verb << " comparison: " << *ICmp << '\n');
   ++NumElimCmp;
   Changed = true;
-  DeadInsts.push_back(ICmp);
 }
 
 /// SimplifyIVUsers helper for eliminating useless
@@ -230,7 +278,7 @@ void SimplifyIndvar::eliminateIVRemainder(BinaryOperator *Rem,
   DEBUG(dbgs() << "INDVARS: Simplified rem: " << *Rem << '\n');
   ++NumElimRem;
   Changed = true;
-  DeadInsts.push_back(Rem);
+  DeadInsts.emplace_back(Rem);
 }
 
 /// Eliminate an operation that consumes a simple IV and has
@@ -261,7 +309,7 @@ bool SimplifyIndvar::eliminateIVUser(Instruction *UseInst,
   UseInst->replaceAllUsesWith(IVOperand);
   ++NumElimIdentity;
   Changed = true;
-  DeadInsts.push_back(UseInst);
+  DeadInsts.emplace_back(UseInst);
   return true;
 }
 
@@ -387,7 +435,7 @@ Instruction *SimplifyIndvar::splitOverflowIntrinsic(Instruction *IVUser,
          "Bad add instruction created from overflow intrinsic.");
 
   AddVal->replaceAllUsesWith(AddInst);
-  DeadInsts.push_back(AddVal);
+  DeadInsts.emplace_back(AddVal);
   return AddInst;
 }
 
diff --git a/lib/Transforms/Utils/SimplifyLibCalls.cpp b/lib/Transforms/Utils/SimplifyLibCalls.cpp
index 42102e7d1e8..6bbf8287e22 100644
--- a/lib/Transforms/Utils/SimplifyLibCalls.cpp
+++ b/lib/Transforms/Utils/SimplifyLibCalls.cpp
@@ -973,7 +973,7 @@ Value *LibCallSimplifier::optimizeUnaryDoubleFP(CallInst *CI, IRBuilder<> &B,
   // floor((double)floatval) -> (double)floorf(floatval)
   if (Callee->isIntrinsic()) {
     Module *M = CI->getParent()->getParent()->getParent();
-    Intrinsic::ID IID = (Intrinsic::ID) Callee->getIntrinsicID();
+    Intrinsic::ID IID = Callee->getIntrinsicID();
     Function *F = Intrinsic::getDeclaration(M, IID, B.getFloatTy());
     V = B.CreateCall(F, V);
   } else {
@@ -1150,7 +1150,7 @@ Value *LibCallSimplifier::optimizeExp2(CallInst *CI, IRBuilder<> &B) {
       Value *Callee =
           M->getOrInsertFunction(TLI->getName(LdExp), Op->getType(),
                                  Op->getType(), B.getInt32Ty(), nullptr);
-      CallInst *CI = B.CreateCall2(Callee, One, LdExpArg);
+      CallInst *CI = B.CreateCall(Callee, {One, LdExpArg});
       if (const Function *F = dyn_cast<Function>(Callee->stripPointerCasts()))
         CI->setCallingConv(F->getCallingConv());
 
@@ -1436,7 +1436,7 @@ Value *LibCallSimplifier::optimizeFFS(CallInst *CI, IRBuilder<> &B) {
   Type *ArgType = Op->getType();
   Value *F =
       Intrinsic::getDeclaration(Callee->getParent(), Intrinsic::cttz, ArgType);
-  Value *V = B.CreateCall2(F, Op, B.getFalse(), "cttz");
+  Value *V = B.CreateCall(F, {Op, B.getFalse()}, "cttz");
   V = B.CreateAdd(V, ConstantInt::get(V->getType(), 1));
   V = B.CreateIntCast(V, B.getInt32Ty(), false);
 
diff --git a/lib/Transforms/Utils/ValueMapper.cpp b/lib/Transforms/Utils/ValueMapper.cpp
index 54c76887234..8c72641da9e 100644
--- a/lib/Transforms/Utils/ValueMapper.cpp
+++ b/lib/Transforms/Utils/ValueMapper.cpp
@@ -13,6 +13,7 @@
 //===----------------------------------------------------------------------===//
 
 #include "llvm/Transforms/Utils/ValueMapper.h"
+#include "llvm/IR/CallSite.h"
 #include "llvm/IR/Constants.h"
 #include "llvm/IR/Function.h"
 #include "llvm/IR/InlineAsm.h"
@@ -383,7 +384,27 @@ void llvm::RemapInstruction(Instruction *I, ValueToValueMapTy &VMap,
       I->setMetadata(MI->first, New);
   }
   
+  if (!TypeMapper)
+    return;
+
   // If the instruction's type is being remapped, do so now.
-  if (TypeMapper)
-    I->mutateType(TypeMapper->remapType(I->getType()));
+  if (auto CS = CallSite(I)) {
+    SmallVector<Type *, 3> Tys;
+    FunctionType *FTy = CS.getFunctionType();
+    Tys.reserve(FTy->getNumParams());
+    for (Type *Ty : FTy->params())
+      Tys.push_back(TypeMapper->remapType(Ty));
+    CS.mutateFunctionType(FunctionType::get(
+        TypeMapper->remapType(I->getType()), Tys, FTy->isVarArg()));
+    return;
+  }
+  if (auto *AI = dyn_cast<AllocaInst>(I))
+    AI->setAllocatedType(TypeMapper->remapType(AI->getAllocatedType()));
+  if (auto *GEP = dyn_cast<GetElementPtrInst>(I)) {
+    GEP->setSourceElementType(
+        TypeMapper->remapType(GEP->getSourceElementType()));
+    GEP->setResultElementType(
+        TypeMapper->remapType(GEP->getResultElementType()));
+  }
+  I->mutateType(TypeMapper->remapType(I->getType()));
 }
diff --git a/lib/Transforms/Vectorize/BBVectorize.cpp b/lib/Transforms/Vectorize/BBVectorize.cpp
index 29fb01f1b2e..ed134141e93 100644
--- a/lib/Transforms/Vectorize/BBVectorize.cpp
+++ b/lib/Transforms/Vectorize/BBVectorize.cpp
@@ -662,7 +662,7 @@ namespace {
       Function *F = I->getCalledFunction();
       if (!F) return false;
 
-      Intrinsic::ID IID = (Intrinsic::ID) F->getIntrinsicID();
+      Intrinsic::ID IID = F->getIntrinsicID();
       if (!IID) return false;
 
       switch(IID) {
@@ -1098,7 +1098,7 @@ namespace {
     CallInst *CI = dyn_cast<CallInst>(I);
     Function *FI;
     if (CI && (FI = CI->getCalledFunction())) {
-      Intrinsic::ID IID = (Intrinsic::ID) FI->getIntrinsicID();
+      Intrinsic::ID IID = FI->getIntrinsicID();
       if (IID == Intrinsic::powi || IID == Intrinsic::ctlz ||
           IID == Intrinsic::cttz) {
         Value *A1I = CI->getArgOperand(1),
@@ -2770,7 +2770,7 @@ namespace {
         continue;
       } else if (isa<CallInst>(I)) {
         Function *F = cast<CallInst>(I)->getCalledFunction();
-        Intrinsic::ID IID = (Intrinsic::ID) F->getIntrinsicID();
+        Intrinsic::ID IID = F->getIntrinsicID();
         if (o == NumOperands-1) {
           BasicBlock &BB = *I->getParent();
 
@@ -2806,55 +2806,51 @@ namespace {
                      Instruction *J, Instruction *K,
                      Instruction *&InsertionPt,
                      Instruction *&K1, Instruction *&K2) {
-    if (isa<StoreInst>(I)) {
-      AA->replaceWithNewValue(I, K);
-      AA->replaceWithNewValue(J, K);
-    } else {
-      Type *IType = I->getType();
-      Type *JType = J->getType();
+    if (isa<StoreInst>(I))
+      return;
 
-      VectorType *VType = getVecTypeForPair(IType, JType);
-      unsigned numElem = VType->getNumElements();
+    Type *IType = I->getType();
+    Type *JType = J->getType();
 
-      unsigned numElemI = getNumScalarElements(IType);
-      unsigned numElemJ = getNumScalarElements(JType);
+    VectorType *VType = getVecTypeForPair(IType, JType);
+    unsigned numElem = VType->getNumElements();
 
-      if (IType->isVectorTy()) {
-        std::vector<Constant*> Mask1(numElemI), Mask2(numElemI);
-        for (unsigned v = 0; v < numElemI; ++v) {
-          Mask1[v] = ConstantInt::get(Type::getInt32Ty(Context), v);
-          Mask2[v] = ConstantInt::get(Type::getInt32Ty(Context), numElemJ+v);
-        }
+    unsigned numElemI = getNumScalarElements(IType);
+    unsigned numElemJ = getNumScalarElements(JType);
 
-        K1 = new ShuffleVectorInst(K, UndefValue::get(VType),
-                                   ConstantVector::get( Mask1),
-                                   getReplacementName(K, false, 1));
-      } else {
-        Value *CV0 = ConstantInt::get(Type::getInt32Ty(Context), 0);
-        K1 = ExtractElementInst::Create(K, CV0,
-                                          getReplacementName(K, false, 1));
+    if (IType->isVectorTy()) {
+      std::vector<Constant *> Mask1(numElemI), Mask2(numElemI);
+      for (unsigned v = 0; v < numElemI; ++v) {
+        Mask1[v] = ConstantInt::get(Type::getInt32Ty(Context), v);
+        Mask2[v] = ConstantInt::get(Type::getInt32Ty(Context), numElemJ + v);
       }
 
-      if (JType->isVectorTy()) {
-        std::vector<Constant*> Mask1(numElemJ), Mask2(numElemJ);
-        for (unsigned v = 0; v < numElemJ; ++v) {
-          Mask1[v] = ConstantInt::get(Type::getInt32Ty(Context), v);
-          Mask2[v] = ConstantInt::get(Type::getInt32Ty(Context), numElemI+v);
-        }
+      K1 = new ShuffleVectorInst(K, UndefValue::get(VType),
+                                 ConstantVector::get(Mask1),
+                                 getReplacementName(K, false, 1));
+    } else {
+      Value *CV0 = ConstantInt::get(Type::getInt32Ty(Context), 0);
+      K1 = ExtractElementInst::Create(K, CV0, getReplacementName(K, false, 1));
+    }
 
-        K2 = new ShuffleVectorInst(K, UndefValue::get(VType),
-                                   ConstantVector::get( Mask2),
-                                   getReplacementName(K, false, 2));
-      } else {
-        Value *CV1 = ConstantInt::get(Type::getInt32Ty(Context), numElem-1);
-        K2 = ExtractElementInst::Create(K, CV1,
-                                          getReplacementName(K, false, 2));
+    if (JType->isVectorTy()) {
+      std::vector<Constant *> Mask1(numElemJ), Mask2(numElemJ);
+      for (unsigned v = 0; v < numElemJ; ++v) {
+        Mask1[v] = ConstantInt::get(Type::getInt32Ty(Context), v);
+        Mask2[v] = ConstantInt::get(Type::getInt32Ty(Context), numElemI + v);
       }
 
-      K1->insertAfter(K);
-      K2->insertAfter(K1);
-      InsertionPt = K2;
+      K2 = new ShuffleVectorInst(K, UndefValue::get(VType),
+                                 ConstantVector::get(Mask2),
+                                 getReplacementName(K, false, 2));
+    } else {
+      Value *CV1 = ConstantInt::get(Type::getInt32Ty(Context), numElem - 1);
+      K2 = ExtractElementInst::Create(K, CV1, getReplacementName(K, false, 2));
     }
+
+    K1->insertAfter(K);
+    K2->insertAfter(K1);
+    InsertionPt = K2;
   }
 
   // Move all uses of the function I (including pairing-induced uses) after J.
@@ -3103,7 +3099,17 @@ namespace {
       else if (H->hasName())
         K->takeName(H);
 
-      if (!isa<StoreInst>(K))
+      if (auto CS = CallSite(K)) {
+        SmallVector<Type *, 3> Tys;
+        FunctionType *Old = CS.getFunctionType();
+        unsigned NumOld = Old->getNumParams();
+        assert(NumOld <= ReplacedOperands.size());
+        for (unsigned i = 0; i != NumOld; ++i)
+          Tys.push_back(ReplacedOperands[i]->getType());
+        CS.mutateFunctionType(
+            FunctionType::get(getVecTypeForPair(L->getType(), H->getType()),
+                              Tys, Old->isVarArg()));
+      } else if (!isa<StoreInst>(K))
         K->mutateType(getVecTypeForPair(L->getType(), H->getType()));
 
       unsigned KnownIDs[] = {
@@ -3135,8 +3141,6 @@ namespace {
       if (!isa<StoreInst>(I)) {
         L->replaceAllUsesWith(K1);
         H->replaceAllUsesWith(K2);
-        AA->replaceWithNewValue(L, K1);
-        AA->replaceWithNewValue(H, K2);
       }
 
       // Instructions that may read from memory may be in the load move set.
diff --git a/lib/Transforms/Vectorize/LoopVectorize.cpp b/lib/Transforms/Vectorize/LoopVectorize.cpp
index 16eb5ee499b..7ad28b85a91 100644
--- a/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -34,6 +34,10 @@
 // Variable uniformity checks are inspired by:
 //  Karrenberg, R. and Hack, S. Whole Function Vectorization.
 //
+// The interleaved access vectorization is based on the paper:
+//  Dorit Nuzman, Ira Rosen and Ayal Zaks.  Auto-Vectorization of Interleaved
+//  Data for SIMD
+//
 // Other ideas/concepts are from:
 //  A. Zaks and D. Nuzman. Autovectorization in GCC-two years later.
 //
@@ -92,7 +96,7 @@
 #include "llvm/Transforms/Scalar.h"
 #include "llvm/Transforms/Utils/BasicBlockUtils.h"
 #include "llvm/Transforms/Utils/Local.h"
-#include "llvm/Transforms/Utils/VectorUtils.h"
+#include "llvm/Analysis/VectorUtils.h"
 #include "llvm/Transforms/Utils/LoopUtils.h"
 #include <algorithm>
 #include <map>
@@ -134,8 +138,19 @@ static cl::opt<bool> EnableMemAccessVersioning(
     "enable-mem-access-versioning", cl::init(true), cl::Hidden,
     cl::desc("Enable symblic stride memory access versioning"));
 
-/// We don't unroll loops with a known constant trip count below this number.
-static const unsigned TinyTripCountUnrollThreshold = 128;
+static cl::opt<bool> EnableInterleavedMemAccesses(
+    "enable-interleaved-mem-accesses", cl::init(false), cl::Hidden,
+    cl::desc("Enable vectorization on interleaved memory accesses in a loop"));
+
+/// Maximum factor for an interleaved memory access.
+static cl::opt<unsigned> MaxInterleaveGroupFactor(
+    "max-interleave-group-factor", cl::Hidden,
+    cl::desc("Maximum factor for an interleaved access group (default = 8)"),
+    cl::init(8));
+
+/// We don't interleave loops with a known constant trip count below this
+/// number.
+static const unsigned TinyTripCountInterleaveThreshold = 128;
 
 static cl::opt<unsigned> ForceTargetNumScalarRegs(
     "force-target-num-scalar-regs", cl::init(0), cl::Hidden,
@@ -166,7 +181,8 @@ static cl::opt<unsigned> ForceTargetInstructionCost(
 
 static cl::opt<unsigned> SmallLoopCost(
     "small-loop-cost", cl::init(20), cl::Hidden,
-    cl::desc("The cost of a loop that is considered 'small' by the unroller."));
+    cl::desc(
+        "The cost of a loop that is considered 'small' by the interleaver."));
 
 static cl::opt<bool> LoopVectorizeWithBlockFrequency(
     "loop-vectorize-with-block-frequency", cl::init(false), cl::Hidden,
@@ -174,10 +190,11 @@ static cl::opt<bool> LoopVectorizeWithBlockFrequency(
              "heuristics minimizing code growth in cold regions and being more "
              "aggressive in hot regions."));
 
-// Runtime unroll loops for load/store throughput.
-static cl::opt<bool> EnableLoadStoreRuntimeUnroll(
-    "enable-loadstore-runtime-unroll", cl::init(true), cl::Hidden,
-    cl::desc("Enable runtime unrolling until load/store ports are saturated"));
+// Runtime interleave loops for load/store throughput.
+static cl::opt<bool> EnableLoadStoreRuntimeInterleave(
+    "enable-loadstore-runtime-interleave", cl::init(true), cl::Hidden,
+    cl::desc(
+        "Enable runtime interleaving until load/store ports are saturated"));
 
 /// The number of stores in a loop that are allowed to need predication.
 static cl::opt<unsigned> NumberOfStoresToPredicate(
@@ -186,15 +203,15 @@ static cl::opt<unsigned> NumberOfStoresToPredicate(
 
 static cl::opt<bool> EnableIndVarRegisterHeur(
     "enable-ind-var-reg-heur", cl::init(true), cl::Hidden,
-    cl::desc("Count the induction variable only once when unrolling"));
+    cl::desc("Count the induction variable only once when interleaving"));
 
 static cl::opt<bool> EnableCondStoresVectorization(
     "enable-cond-stores-vec", cl::init(false), cl::Hidden,
     cl::desc("Enable if predication of stores during vectorization."));
 
-static cl::opt<unsigned> MaxNestedScalarReductionUF(
-    "max-nested-scalar-reduction-unroll", cl::init(2), cl::Hidden,
-    cl::desc("The maximum unroll factor to use when unrolling a scalar "
+static cl::opt<unsigned> MaxNestedScalarReductionIC(
+    "max-nested-scalar-reduction-interleave", cl::init(2), cl::Hidden,
+    cl::desc("The maximum interleave count to use when interleaving a scalar "
              "reduction in a nested loop."));
 
 namespace {
@@ -280,12 +297,12 @@ class InnerLoopVectorizer {
   /// originated from one scalar instruction.
   typedef SmallVector<Value*, 2> VectorParts;
 
-  // When we if-convert we need create edge masks. We have to cache values so
-  // that we don't end up with exponential recursion/IR.
+  // When we if-convert we need to create edge masks. We have to cache values
+  // so that we don't end up with exponential recursion/IR.
   typedef DenseMap<std::pair<BasicBlock*, BasicBlock*>,
                    VectorParts> EdgeMaskCache;
 
-  /// \brief Add checks for strides that where assumed to be 1.
+  /// \brief Add checks for strides that were assumed to be 1.
   ///
   /// Returns the last check instruction and the first check instruction in the
   /// pair as (first, last).
@@ -351,6 +368,9 @@ class InnerLoopVectorizer {
   /// broadcast them into a vector.
   VectorParts &getVectorValue(Value *V);
 
+  /// Try to vectorize the interleaved access group that \p Instr belongs to.
+  void vectorizeInterleaveGroup(Instruction *Instr);
+
   /// Generate a shuffle sequence that will reverse the vector Vec.
   virtual Value *reverseVector(Value *Vec);
 
@@ -545,6 +565,219 @@ static void propagateMetadata(SmallVectorImpl<Value *> &To, const Instruction *F
       propagateMetadata(I, From);
 }
 
+/// \brief The group of interleaved loads/stores sharing the same stride and
+/// close to each other.
+///
+/// Each member in this group has an index starting from 0, and the largest
+/// index should be less than interleaved factor, which is equal to the absolute
+/// value of the access's stride.
+///
+/// E.g. An interleaved load group of factor 4:
+///        for (unsigned i = 0; i < 1024; i+=4) {
+///          a = A[i];                           // Member of index 0
+///          b = A[i+1];                         // Member of index 1
+///          d = A[i+3];                         // Member of index 3
+///          ...
+///        }
+///
+///      An interleaved store group of factor 4:
+///        for (unsigned i = 0; i < 1024; i+=4) {
+///          ...
+///          A[i]   = a;                         // Member of index 0
+///          A[i+1] = b;                         // Member of index 1
+///          A[i+2] = c;                         // Member of index 2
+///          A[i+3] = d;                         // Member of index 3
+///        }
+///
+/// Note: the interleaved load group could have gaps (missing members), but
+/// the interleaved store group doesn't allow gaps.
+class InterleaveGroup {
+public:
+  InterleaveGroup(Instruction *Instr, int Stride, unsigned Align)
+      : Align(Align), SmallestKey(0), LargestKey(0), InsertPos(Instr) {
+    assert(Align && "The alignment should be non-zero");
+
+    Factor = std::abs(Stride);
+    assert(Factor > 1 && "Invalid interleave factor");
+
+    Reverse = Stride < 0;
+    Members[0] = Instr;
+  }
+
+  bool isReverse() const { return Reverse; }
+  unsigned getFactor() const { return Factor; }
+  unsigned getAlignment() const { return Align; }
+  unsigned getNumMembers() const { return Members.size(); }
+
+  /// \brief Try to insert a new member \p Instr with index \p Index and
+  /// alignment \p NewAlign. The index is related to the leader and it could be
+  /// negative if it is the new leader.
+  ///
+  /// \returns false if the instruction doesn't belong to the group.
+  bool insertMember(Instruction *Instr, int Index, unsigned NewAlign) {
+    assert(NewAlign && "The new member's alignment should be non-zero");
+
+    int Key = Index + SmallestKey;
+
+    // Skip if there is already a member with the same index.
+    if (Members.count(Key))
+      return false;
+
+    if (Key > LargestKey) {
+      // The largest index is always less than the interleave factor.
+      if (Index >= static_cast<int>(Factor))
+        return false;
+
+      LargestKey = Key;
+    } else if (Key < SmallestKey) {
+      // The largest index is always less than the interleave factor.
+      if (LargestKey - Key >= static_cast<int>(Factor))
+        return false;
+
+      SmallestKey = Key;
+    }
+
+    // It's always safe to select the minimum alignment.
+    Align = std::min(Align, NewAlign);
+    Members[Key] = Instr;
+    return true;
+  }
+
+  /// \brief Get the member with the given index \p Index
+  ///
+  /// \returns nullptr if contains no such member.
+  Instruction *getMember(unsigned Index) const {
+    int Key = SmallestKey + Index;
+    if (!Members.count(Key))
+      return nullptr;
+
+    return Members.find(Key)->second;
+  }
+
+  /// \brief Get the index for the given member. Unlike the key in the member
+  /// map, the index starts from 0.
+  unsigned getIndex(Instruction *Instr) const {
+    for (auto I : Members)
+      if (I.second == Instr)
+        return I.first - SmallestKey;
+
+    llvm_unreachable("InterleaveGroup contains no such member");
+  }
+
+  Instruction *getInsertPos() const { return InsertPos; }
+  void setInsertPos(Instruction *Inst) { InsertPos = Inst; }
+
+private:
+  unsigned Factor; // Interleave Factor.
+  bool Reverse;
+  unsigned Align;
+  DenseMap<int, Instruction *> Members;
+  int SmallestKey;
+  int LargestKey;
+
+  // To avoid breaking dependences, vectorized instructions of an interleave
+  // group should be inserted at either the first load or the last store in
+  // program order.
+  //
+  // E.g. %even = load i32             // Insert Position
+  //      %add = add i32 %even         // Use of %even
+  //      %odd = load i32
+  //
+  //      store i32 %even
+  //      %odd = add i32               // Def of %odd
+  //      store i32 %odd               // Insert Position
+  Instruction *InsertPos;
+};
+
+/// \brief Drive the analysis of interleaved memory accesses in the loop.
+///
+/// Use this class to analyze interleaved accesses only when we can vectorize
+/// a loop. Otherwise it's meaningless to do analysis as the vectorization
+/// on interleaved accesses is unsafe.
+///
+/// The analysis collects interleave groups and records the relationships
+/// between the member and the group in a map.
+class InterleavedAccessInfo {
+public:
+  InterleavedAccessInfo(ScalarEvolution *SE, Loop *L, DominatorTree *DT)
+      : SE(SE), TheLoop(L), DT(DT) {}
+
+  ~InterleavedAccessInfo() {
+    SmallSet<InterleaveGroup *, 4> DelSet;
+    // Avoid releasing a pointer twice.
+    for (auto &I : InterleaveGroupMap)
+      DelSet.insert(I.second);
+    for (auto *Ptr : DelSet)
+      delete Ptr;
+  }
+
+  /// \brief Analyze the interleaved accesses and collect them in interleave
+  /// groups. Substitute symbolic strides using \p Strides.
+  void analyzeInterleaving(const ValueToValueMap &Strides);
+
+  /// \brief Check if \p Instr belongs to any interleave group.
+  bool isInterleaved(Instruction *Instr) const {
+    return InterleaveGroupMap.count(Instr);
+  }
+
+  /// \brief Get the interleave group that \p Instr belongs to.
+  ///
+  /// \returns nullptr if doesn't have such group.
+  InterleaveGroup *getInterleaveGroup(Instruction *Instr) const {
+    if (InterleaveGroupMap.count(Instr))
+      return InterleaveGroupMap.find(Instr)->second;
+    return nullptr;
+  }
+
+private:
+  ScalarEvolution *SE;
+  Loop *TheLoop;
+  DominatorTree *DT;
+
+  /// Holds the relationships between the members and the interleave group.
+  DenseMap<Instruction *, InterleaveGroup *> InterleaveGroupMap;
+
+  /// \brief The descriptor for a strided memory access.
+  struct StrideDescriptor {
+    StrideDescriptor(int Stride, const SCEV *Scev, unsigned Size,
+                     unsigned Align)
+        : Stride(Stride), Scev(Scev), Size(Size), Align(Align) {}
+
+    StrideDescriptor() : Stride(0), Scev(nullptr), Size(0), Align(0) {}
+
+    int Stride; // The access's stride. It is negative for a reverse access.
+    const SCEV *Scev; // The scalar expression of this access
+    unsigned Size;    // The size of the memory object.
+    unsigned Align;   // The alignment of this access.
+  };
+
+  /// \brief Create a new interleave group with the given instruction \p Instr,
+  /// stride \p Stride and alignment \p Align.
+  ///
+  /// \returns the newly created interleave group.
+  InterleaveGroup *createInterleaveGroup(Instruction *Instr, int Stride,
+                                         unsigned Align) {
+    assert(!InterleaveGroupMap.count(Instr) &&
+           "Already in an interleaved access group");
+    InterleaveGroupMap[Instr] = new InterleaveGroup(Instr, Stride, Align);
+    return InterleaveGroupMap[Instr];
+  }
+
+  /// \brief Release the group and remove all the relationships.
+  void releaseGroup(InterleaveGroup *Group) {
+    for (unsigned i = 0; i < Group->getFactor(); i++)
+      if (Instruction *Member = Group->getMember(i))
+        InterleaveGroupMap.erase(Member);
+
+    delete Group;
+  }
+
+  /// \brief Collect all the accesses with a constant stride in program order.
+  void collectConstStridedAccesses(
+      MapVector<Instruction *, StrideDescriptor> &StrideAccesses,
+      const ValueToValueMap &Strides);
+};
+
 /// LoopVectorizationLegality checks if it is legal to vectorize a loop, and
 /// to what vectorization factor.
 /// This class does not look at the profitability of vectorization, only the
@@ -565,8 +798,8 @@ class LoopVectorizationLegality {
                             Function *F, const TargetTransformInfo *TTI,
                             LoopAccessAnalysis *LAA)
       : NumPredStores(0), TheLoop(L), SE(SE), TLI(TLI), TheFunction(F),
-        TTI(TTI), DT(DT), LAA(LAA), LAI(nullptr), Induction(nullptr),
-        WidestIndTy(nullptr), HasFunNoNaNAttr(false) {}
+        TTI(TTI), DT(DT), LAA(LAA), LAI(nullptr), InterleaveInfo(SE, L, DT),
+        Induction(nullptr), WidestIndTy(nullptr), HasFunNoNaNAttr(false) {}
 
   /// This enum represents the kinds of inductions that we support.
   enum InductionKind {
@@ -644,7 +877,7 @@ class LoopVectorizationLegality {
 
   /// ReductionList contains the reduction descriptors for all
   /// of the reductions that were found in the loop.
-  typedef DenseMap<PHINode*, ReductionDescriptor> ReductionList;
+  typedef DenseMap<PHINode *, RecurrenceDescriptor> ReductionList;
 
   /// InductionList saves induction variables and maps them to the
   /// induction descriptor.
@@ -691,14 +924,24 @@ class LoopVectorizationLegality {
   bool isUniformAfterVectorization(Instruction* I) { return Uniforms.count(I); }
 
   /// Returns the information that we collected about runtime memory check.
-  const LoopAccessInfo::RuntimePointerCheck *getRuntimePointerCheck() const {
-    return LAI->getRuntimePointerCheck();
+  const RuntimePointerChecking *getRuntimePointerChecking() const {
+    return LAI->getRuntimePointerChecking();
   }
 
   const LoopAccessInfo *getLAI() const {
     return LAI;
   }
 
+  /// \brief Check if \p Instr belongs to any interleaved access group.
+  bool isAccessInterleaved(Instruction *Instr) {
+    return InterleaveInfo.isInterleaved(Instr);
+  }
+
+  /// \brief Get the interleaved access group that \p Instr belongs to.
+  const InterleaveGroup *getInterleavedAccessGroup(Instruction *Instr) {
+    return InterleaveInfo.getInterleaveGroup(Instr);
+  }
+
   unsigned getMaxSafeDepDistBytes() { return LAI->getMaxSafeDepDistBytes(); }
 
   bool hasStride(Value *V) { return StrideSet.count(V); }
@@ -794,6 +1037,10 @@ class LoopVectorizationLegality {
   // null until canVectorizeMemory sets it up.
   const LoopAccessInfo *LAI;
 
+  /// The interleave access information contains groups of interleaved accesses
+  /// with the same stride and close to each other.
+  InterleavedAccessInfo InterleaveInfo;
+
   //  ---  vectorization state --- //
 
   /// Holds the integer induction variable. This is the counter of the
@@ -861,12 +1108,19 @@ class LoopVectorizationCostModel {
   /// 64 bit loop indices.
   unsigned getWidestType();
 
+  /// \return The desired interleave count.
+  /// If interleave count has been specified by metadata it will be returned.
+  /// Otherwise, the interleave count is computed and returned. VF and LoopCost
+  /// are the selected vectorization factor and the cost of the selected VF.
+  unsigned selectInterleaveCount(bool OptForSize, unsigned VF,
+                                 unsigned LoopCost);
+
   /// \return The most profitable unroll factor.
-  /// If UserUF is non-zero then this method finds the best unroll-factor
-  /// based on register pressure and other parameters.
-  /// VF and LoopCost are the selected vectorization factor and the cost of the
-  /// selected VF.
-  unsigned selectUnrollFactor(bool OptForSize, unsigned VF, unsigned LoopCost);
+  /// This method finds the best unroll-factor based on register pressure and
+  /// other parameters. VF and LoopCost are the selected vectorization factor
+  /// and the cost of the selected VF.
+  unsigned computeInterleaveCount(bool OptForSize, unsigned VF,
+                                  unsigned LoopCost);
 
   /// \brief A struct that represents some properties of the register usage
   /// of a loop.
@@ -1200,7 +1454,7 @@ struct LoopVectorize : public FunctionPass {
     LI = &getAnalysis<LoopInfoWrapperPass>().getLoopInfo();
     TTI = &getAnalysis<TargetTransformInfoWrapperPass>().getTTI(F);
     DT = &getAnalysis<DominatorTreeWrapperPass>().getDomTree();
-    BFI = &getAnalysis<BlockFrequencyInfo>();
+    BFI = &getAnalysis<BlockFrequencyInfoWrapperPass>().getBFI();
     auto *TLIP = getAnalysisIfAvailable<TargetLibraryInfoWrapperPass>();
     TLI = TLIP ? &TLIP->getTLI() : nullptr;
     AA = &getAnalysis<AliasAnalysis>();
@@ -1212,9 +1466,14 @@ struct LoopVectorize : public FunctionPass {
     const BranchProbability ColdProb(1, 5); // 20%
     ColdEntryFreq = BlockFrequency(BFI->getEntryFreq()) * ColdProb;
 
-    // If the target claims to have no vector registers don't attempt
-    // vectorization.
-    if (!TTI->getNumberOfRegisters(true))
+    // Don't attempt if
+    // 1. the target claims to have no vector registers, and
+    // 2. interleaving won't help ILP.
+    //
+    // The second condition is necessary because, even if the target has no
+    // vector registers, loop vectorization may still enable scalar
+    // interleaving.
+    if (!TTI->getNumberOfRegisters(true) && TTI->getMaxInterleaveFactor(1) < 2)
       return false;
 
     // Build up a worklist of inner-loops to vectorize. This is necessary as
@@ -1389,18 +1648,17 @@ struct LoopVectorize : public FunctionPass {
     const LoopVectorizationCostModel::VectorizationFactor VF =
         CM.selectVectorizationFactor(OptForSize);
 
-    // Select the unroll factor.
-    const unsigned UF =
-        CM.selectUnrollFactor(OptForSize, VF.Width, VF.Cost);
+    // Select the interleave count.
+    unsigned IC = CM.selectInterleaveCount(OptForSize, VF.Width, VF.Cost);
 
     DEBUG(dbgs() << "LV: Found a vectorizable loop (" << VF.Width << ") in "
                  << DebugLocStr << '\n');
-    DEBUG(dbgs() << "LV: Unroll Factor is " << UF << '\n');
+    DEBUG(dbgs() << "LV: Interleave Count is " << IC << '\n');
 
     if (VF.Width == 1) {
       DEBUG(dbgs() << "LV: Vectorization is possible but not beneficial\n");
 
-      if (UF == 1) {
+      if (IC == 1) {
         emitOptimizationRemarkAnalysis(
             F->getContext(), DEBUG_TYPE, *F, L->getStartLoc(),
             "not beneficial to vectorize and user disabled interleaving");
@@ -1410,17 +1668,14 @@ struct LoopVectorize : public FunctionPass {
 
       // Report the unrolling decision.
       emitOptimizationRemark(F->getContext(), DEBUG_TYPE, *F, L->getStartLoc(),
-                             Twine("unrolled with interleaving factor " +
-                                   Twine(UF) +
+                             Twine("interleaved by " + Twine(IC) +
                                    " (vectorization not beneficial)"));
 
-      // We decided not to vectorize, but we may want to unroll.
-
-      InnerLoopUnroller Unroller(L, SE, LI, DT, TLI, TTI, UF);
+      InnerLoopUnroller Unroller(L, SE, LI, DT, TLI, TTI, IC);
       Unroller.vectorize(&LVL);
     } else {
       // If we decided that it is *legal* to vectorize the loop then do it.
-      InnerLoopVectorizer LB(L, SE, LI, DT, TLI, TTI, VF.Width, UF);
+      InnerLoopVectorizer LB(L, SE, LI, DT, TLI, TTI, VF.Width, IC);
       LB.vectorize(&LVL);
       ++LoopsVectorized;
 
@@ -1431,10 +1686,10 @@ struct LoopVectorize : public FunctionPass {
         AddRuntimeUnrollDisableMetaData(L);
 
       // Report the vectorization decision.
-      emitOptimizationRemark(
-          F->getContext(), DEBUG_TYPE, *F, L->getStartLoc(),
-          Twine("vectorized loop (vectorization factor: ") + Twine(VF.Width) +
-              ", unrolling interleave factor: " + Twine(UF) + ")");
+      emitOptimizationRemark(F->getContext(), DEBUG_TYPE, *F, L->getStartLoc(),
+                             Twine("vectorized loop (vectorization width: ") +
+                                 Twine(VF.Width) + ", interleaved count: " +
+                                 Twine(IC) + ")");
     }
 
     // Mark the loop as already vectorized to avoid vectorizing again.
@@ -1448,7 +1703,7 @@ struct LoopVectorize : public FunctionPass {
     AU.addRequired<AssumptionCacheTracker>();
     AU.addRequiredID(LoopSimplifyID);
     AU.addRequiredID(LCSSAID);
-    AU.addRequired<BlockFrequencyInfo>();
+    AU.addRequired<BlockFrequencyInfoWrapperPass>();
     AU.addRequired<DominatorTreeWrapperPass>();
     AU.addRequired<LoopInfoWrapperPass>();
     AU.addRequired<ScalarEvolution>();
@@ -1516,31 +1771,6 @@ Value *InnerLoopVectorizer::getStepVector(Value *Val, int StartIdx,
   return Builder.CreateAdd(Val, Step, "induction");
 }
 
-/// \brief Find the operand of the GEP that should be checked for consecutive
-/// stores. This ignores trailing indices that have no effect on the final
-/// pointer.
-static unsigned getGEPInductionOperand(const GetElementPtrInst *Gep) {
-  const DataLayout &DL = Gep->getModule()->getDataLayout();
-  unsigned LastOperand = Gep->getNumOperands() - 1;
-  unsigned GEPAllocSize = DL.getTypeAllocSize(
-      cast<PointerType>(Gep->getType()->getScalarType())->getElementType());
-
-  // Walk backwards and try to peel off zeros.
-  while (LastOperand > 1 && match(Gep->getOperand(LastOperand), m_Zero())) {
-    // Find the type we're currently indexing into.
-    gep_type_iterator GEPTI = gep_type_begin(Gep);
-    std::advance(GEPTI, LastOperand - 1);
-
-    // If it's a type with the same allocation size as the result of the GEP we
-    // can peel off the zero index.
-    if (DL.getTypeAllocSize(*GEPTI) != GEPAllocSize)
-      break;
-    --LastOperand;
-  }
-
-  return LastOperand;
-}
-
 int LoopVectorizationLegality::isConsecutivePtr(Value *Ptr) {
   assert(Ptr->getType()->isPointerTy() && "Unexpected non-ptr");
   // Make sure that the pointer does not point to structs.
@@ -1659,6 +1889,251 @@ Value *InnerLoopVectorizer::reverseVector(Value *Vec) {
                                      "reverse");
 }
 
+// Get a mask to interleave \p NumVec vectors into a wide vector.
+// I.e.  <0, VF, VF*2, ..., VF*(NumVec-1), 1, VF+1, VF*2+1, ...>
+// E.g. For 2 interleaved vectors, if VF is 4, the mask is:
+//      <0, 4, 1, 5, 2, 6, 3, 7>
+static Constant *getInterleavedMask(IRBuilder<> &Builder, unsigned VF,
+                                    unsigned NumVec) {
+  SmallVector<Constant *, 16> Mask;
+  for (unsigned i = 0; i < VF; i++)
+    for (unsigned j = 0; j < NumVec; j++)
+      Mask.push_back(Builder.getInt32(j * VF + i));
+
+  return ConstantVector::get(Mask);
+}
+
+// Get the strided mask starting from index \p Start.
+// I.e.  <Start, Start + Stride, ..., Start + Stride*(VF-1)>
+static Constant *getStridedMask(IRBuilder<> &Builder, unsigned Start,
+                                unsigned Stride, unsigned VF) {
+  SmallVector<Constant *, 16> Mask;
+  for (unsigned i = 0; i < VF; i++)
+    Mask.push_back(Builder.getInt32(Start + i * Stride));
+
+  return ConstantVector::get(Mask);
+}
+
+// Get a mask of two parts: The first part consists of sequential integers
+// starting from 0, The second part consists of UNDEFs.
+// I.e. <0, 1, 2, ..., NumInt - 1, undef, ..., undef>
+static Constant *getSequentialMask(IRBuilder<> &Builder, unsigned NumInt,
+                                   unsigned NumUndef) {
+  SmallVector<Constant *, 16> Mask;
+  for (unsigned i = 0; i < NumInt; i++)
+    Mask.push_back(Builder.getInt32(i));
+
+  Constant *Undef = UndefValue::get(Builder.getInt32Ty());
+  for (unsigned i = 0; i < NumUndef; i++)
+    Mask.push_back(Undef);
+
+  return ConstantVector::get(Mask);
+}
+
+// Concatenate two vectors with the same element type. The 2nd vector should
+// not have more elements than the 1st vector. If the 2nd vector has less
+// elements, extend it with UNDEFs.
+static Value *ConcatenateTwoVectors(IRBuilder<> &Builder, Value *V1,
+                                    Value *V2) {
+  VectorType *VecTy1 = dyn_cast<VectorType>(V1->getType());
+  VectorType *VecTy2 = dyn_cast<VectorType>(V2->getType());
+  assert(VecTy1 && VecTy2 &&
+         VecTy1->getScalarType() == VecTy2->getScalarType() &&
+         "Expect two vectors with the same element type");
+
+  unsigned NumElts1 = VecTy1->getNumElements();
+  unsigned NumElts2 = VecTy2->getNumElements();
+  assert(NumElts1 >= NumElts2 && "Unexpect the first vector has less elements");
+
+  if (NumElts1 > NumElts2) {
+    // Extend with UNDEFs.
+    Constant *ExtMask =
+        getSequentialMask(Builder, NumElts2, NumElts1 - NumElts2);
+    V2 = Builder.CreateShuffleVector(V2, UndefValue::get(VecTy2), ExtMask);
+  }
+
+  Constant *Mask = getSequentialMask(Builder, NumElts1 + NumElts2, 0);
+  return Builder.CreateShuffleVector(V1, V2, Mask);
+}
+
+// Concatenate vectors in the given list. All vectors have the same type.
+static Value *ConcatenateVectors(IRBuilder<> &Builder,
+                                 ArrayRef<Value *> InputList) {
+  unsigned NumVec = InputList.size();
+  assert(NumVec > 1 && "Should be at least two vectors");
+
+  SmallVector<Value *, 8> ResList;
+  ResList.append(InputList.begin(), InputList.end());
+  do {
+    SmallVector<Value *, 8> TmpList;
+    for (unsigned i = 0; i < NumVec - 1; i += 2) {
+      Value *V0 = ResList[i], *V1 = ResList[i + 1];
+      assert((V0->getType() == V1->getType() || i == NumVec - 2) &&
+             "Only the last vector may have a different type");
+
+      TmpList.push_back(ConcatenateTwoVectors(Builder, V0, V1));
+    }
+
+    // Push the last vector if the total number of vectors is odd.
+    if (NumVec % 2 != 0)
+      TmpList.push_back(ResList[NumVec - 1]);
+
+    ResList = TmpList;
+    NumVec = ResList.size();
+  } while (NumVec > 1);
+
+  return ResList[0];
+}
+
+// Try to vectorize the interleave group that \p Instr belongs to.
+//
+// E.g. Translate following interleaved load group (factor = 3):
+//   for (i = 0; i < N; i+=3) {
+//     R = Pic[i];             // Member of index 0
+//     G = Pic[i+1];           // Member of index 1
+//     B = Pic[i+2];           // Member of index 2
+//     ... // do something to R, G, B
+//   }
+// To:
+//   %wide.vec = load <12 x i32>                       ; Read 4 tuples of R,G,B
+//   %R.vec = shuffle %wide.vec, undef, <0, 3, 6, 9>   ; R elements
+//   %G.vec = shuffle %wide.vec, undef, <1, 4, 7, 10>  ; G elements
+//   %B.vec = shuffle %wide.vec, undef, <2, 5, 8, 11>  ; B elements
+//
+// Or translate following interleaved store group (factor = 3):
+//   for (i = 0; i < N; i+=3) {
+//     ... do something to R, G, B
+//     Pic[i]   = R;           // Member of index 0
+//     Pic[i+1] = G;           // Member of index 1
+//     Pic[i+2] = B;           // Member of index 2
+//   }
+// To:
+//   %R_G.vec = shuffle %R.vec, %G.vec, <0, 1, 2, ..., 7>
+//   %B_U.vec = shuffle %B.vec, undef, <0, 1, 2, 3, u, u, u, u>
+//   %interleaved.vec = shuffle %R_G.vec, %B_U.vec,
+//        <0, 4, 8, 1, 5, 9, 2, 6, 10, 3, 7, 11>    ; Interleave R,G,B elements
+//   store <12 x i32> %interleaved.vec              ; Write 4 tuples of R,G,B
+void InnerLoopVectorizer::vectorizeInterleaveGroup(Instruction *Instr) {
+  const InterleaveGroup *Group = Legal->getInterleavedAccessGroup(Instr);
+  assert(Group && "Fail to get an interleaved access group.");
+
+  // Skip if current instruction is not the insert position.
+  if (Instr != Group->getInsertPos())
+    return;
+
+  LoadInst *LI = dyn_cast<LoadInst>(Instr);
+  StoreInst *SI = dyn_cast<StoreInst>(Instr);
+  Value *Ptr = LI ? LI->getPointerOperand() : SI->getPointerOperand();
+
+  // Prepare for the vector type of the interleaved load/store.
+  Type *ScalarTy = LI ? LI->getType() : SI->getValueOperand()->getType();
+  unsigned InterleaveFactor = Group->getFactor();
+  Type *VecTy = VectorType::get(ScalarTy, InterleaveFactor * VF);
+  Type *PtrTy = VecTy->getPointerTo(Ptr->getType()->getPointerAddressSpace());
+
+  // Prepare for the new pointers.
+  setDebugLocFromInst(Builder, Ptr);
+  VectorParts &PtrParts = getVectorValue(Ptr);
+  SmallVector<Value *, 2> NewPtrs;
+  unsigned Index = Group->getIndex(Instr);
+  for (unsigned Part = 0; Part < UF; Part++) {
+    // Extract the pointer for current instruction from the pointer vector. A
+    // reverse access uses the pointer in the last lane.
+    Value *NewPtr = Builder.CreateExtractElement(
+        PtrParts[Part],
+        Group->isReverse() ? Builder.getInt32(VF - 1) : Builder.getInt32(0));
+
+    // Notice current instruction could be any index. Need to adjust the address
+    // to the member of index 0.
+    //
+    // E.g.  a = A[i+1];     // Member of index 1 (Current instruction)
+    //       b = A[i];       // Member of index 0
+    // Current pointer is pointed to A[i+1], adjust it to A[i].
+    //
+    // E.g.  A[i+1] = a;     // Member of index 1
+    //       A[i]   = b;     // Member of index 0
+    //       A[i+2] = c;     // Member of index 2 (Current instruction)
+    // Current pointer is pointed to A[i+2], adjust it to A[i].
+    NewPtr = Builder.CreateGEP(NewPtr, Builder.getInt32(-Index));
+
+    // Cast to the vector pointer type.
+    NewPtrs.push_back(Builder.CreateBitCast(NewPtr, PtrTy));
+  }
+
+  setDebugLocFromInst(Builder, Instr);
+  Value *UndefVec = UndefValue::get(VecTy);
+
+  // Vectorize the interleaved load group.
+  if (LI) {
+    for (unsigned Part = 0; Part < UF; Part++) {
+      Instruction *NewLoadInstr = Builder.CreateAlignedLoad(
+          NewPtrs[Part], Group->getAlignment(), "wide.vec");
+
+      for (unsigned i = 0; i < InterleaveFactor; i++) {
+        Instruction *Member = Group->getMember(i);
+
+        // Skip the gaps in the group.
+        if (!Member)
+          continue;
+
+        Constant *StrideMask = getStridedMask(Builder, i, InterleaveFactor, VF);
+        Value *StridedVec = Builder.CreateShuffleVector(
+            NewLoadInstr, UndefVec, StrideMask, "strided.vec");
+
+        // If this member has different type, cast the result type.
+        if (Member->getType() != ScalarTy) {
+          VectorType *OtherVTy = VectorType::get(Member->getType(), VF);
+          StridedVec = Builder.CreateBitOrPointerCast(StridedVec, OtherVTy);
+        }
+
+        VectorParts &Entry = WidenMap.get(Member);
+        Entry[Part] =
+            Group->isReverse() ? reverseVector(StridedVec) : StridedVec;
+      }
+
+      propagateMetadata(NewLoadInstr, Instr);
+    }
+    return;
+  }
+
+  // The sub vector type for current instruction.
+  VectorType *SubVT = VectorType::get(ScalarTy, VF);
+
+  // Vectorize the interleaved store group.
+  for (unsigned Part = 0; Part < UF; Part++) {
+    // Collect the stored vector from each member.
+    SmallVector<Value *, 4> StoredVecs;
+    for (unsigned i = 0; i < InterleaveFactor; i++) {
+      // Interleaved store group doesn't allow a gap, so each index has a member
+      Instruction *Member = Group->getMember(i);
+      assert(Member && "Fail to get a member from an interleaved store group");
+
+      Value *StoredVec =
+          getVectorValue(dyn_cast<StoreInst>(Member)->getValueOperand())[Part];
+      if (Group->isReverse())
+        StoredVec = reverseVector(StoredVec);
+
+      // If this member has different type, cast it to an unified type.
+      if (StoredVec->getType() != SubVT)
+        StoredVec = Builder.CreateBitOrPointerCast(StoredVec, SubVT);
+
+      StoredVecs.push_back(StoredVec);
+    }
+
+    // Concatenate all vectors into a wide vector.
+    Value *WideVec = ConcatenateVectors(Builder, StoredVecs);
+
+    // Interleave the elements in the wide vector.
+    Constant *IMask = getInterleavedMask(Builder, VF, InterleaveFactor);
+    Value *IVec = Builder.CreateShuffleVector(WideVec, UndefVec, IMask,
+                                              "interleaved.vec");
+
+    Instruction *NewStoreInstr =
+        Builder.CreateAlignedStore(IVec, NewPtrs[Part], Group->getAlignment());
+    propagateMetadata(NewStoreInstr, Instr);
+  }
+}
+
 void InnerLoopVectorizer::vectorizeMemoryInstruction(Instruction *Instr) {
   // Attempt to issue a wide load.
   LoadInst *LI = dyn_cast<LoadInst>(Instr);
@@ -1666,6 +2141,10 @@ void InnerLoopVectorizer::vectorizeMemoryInstruction(Instruction *Instr) {
 
   assert((LI || SI) && "Invalid Load/Store instruction");
 
+  // Try to vectorize the interleave group if this access is interleaved.
+  if (Legal->isAccessInterleaved(Instr))
+    return vectorizeInterleaveGroup(Instr);
+
   Type *ScalarDataTy = LI ? LI->getType() : SI->getValueOperand()->getType();
   Type *DataTy = VectorType::get(ScalarDataTy, VF);
   Value *Ptr = LI ? LI->getPointerOperand() : SI->getPointerOperand();
@@ -1922,9 +2401,8 @@ void InnerLoopVectorizer::scalarizeInstruction(Instruction *Instr, bool IfPredic
          LoopVectorBody.push_back(NewIfBlock);
          VectorLp->addBasicBlockToLoop(NewIfBlock, *LI);
          Builder.SetInsertPoint(InsertPt);
-         Instruction *OldBr = IfBlock->getTerminator();
-         BranchInst::Create(CondBlock, NewIfBlock, Cmp, OldBr);
-         OldBr->eraseFromParent();
+         ReplaceInstWithInst(IfBlock->getTerminator(),
+                             BranchInst::Create(CondBlock, NewIfBlock, Cmp));
          IfBlock = NewIfBlock;
       }
     }
@@ -2011,9 +2489,9 @@ void InnerLoopVectorizer::createEmptyLoop() {
    */
 
   BasicBlock *OldBasicBlock = OrigLoop->getHeader();
-  BasicBlock *BypassBlock = OrigLoop->getLoopPreheader();
+  BasicBlock *VectorPH = OrigLoop->getLoopPreheader();
   BasicBlock *ExitBlock = OrigLoop->getExitBlock();
-  assert(BypassBlock && "Invalid loop structure");
+  assert(VectorPH && "Invalid loop structure");
   assert(ExitBlock && "Must have an exit block");
 
   // Some loops have a single integer induction variable, while other loops
@@ -2053,44 +2531,35 @@ void InnerLoopVectorizer::createEmptyLoop() {
   // loop.
   Value *BackedgeCount =
       Exp.expandCodeFor(BackedgeTakeCount, BackedgeTakeCount->getType(),
-                        BypassBlock->getTerminator());
+                        VectorPH->getTerminator());
   if (BackedgeCount->getType()->isPointerTy())
     BackedgeCount = CastInst::CreatePointerCast(BackedgeCount, IdxTy,
                                                 "backedge.ptrcnt.to.int",
-                                                BypassBlock->getTerminator());
+                                                VectorPH->getTerminator());
   Instruction *CheckBCOverflow =
       CmpInst::Create(Instruction::ICmp, CmpInst::ICMP_EQ, BackedgeCount,
                       Constant::getAllOnesValue(BackedgeCount->getType()),
-                      "backedge.overflow", BypassBlock->getTerminator());
+                      "backedge.overflow", VectorPH->getTerminator());
 
   // The loop index does not have to start at Zero. Find the original start
   // value from the induction PHI node. If we don't have an induction variable
   // then we know that it starts at zero.
-  Builder.SetInsertPoint(BypassBlock->getTerminator());
-  Value *StartIdx = ExtendedIdx = OldInduction ?
-    Builder.CreateZExt(OldInduction->getIncomingValueForBlock(BypassBlock),
-                       IdxTy):
-    ConstantInt::get(IdxTy, 0);
-
-  // We need an instruction to anchor the overflow check on. StartIdx needs to
-  // be defined before the overflow check branch. Because the scalar preheader
-  // is going to merge the start index and so the overflow branch block needs to
-  // contain a definition of the start index.
-  Instruction *OverflowCheckAnchor = BinaryOperator::CreateAdd(
-      StartIdx, ConstantInt::get(IdxTy, 0), "overflow.check.anchor",
-      BypassBlock->getTerminator());
+  Builder.SetInsertPoint(VectorPH->getTerminator());
+  Value *StartIdx = ExtendedIdx =
+      OldInduction
+          ? Builder.CreateZExt(OldInduction->getIncomingValueForBlock(VectorPH),
+                               IdxTy)
+          : ConstantInt::get(IdxTy, 0);
 
   // Count holds the overall loop count (N).
   Value *Count = Exp.expandCodeFor(ExitCount, ExitCount->getType(),
-                                   BypassBlock->getTerminator());
+                                   VectorPH->getTerminator());
 
-  LoopBypassBlocks.push_back(BypassBlock);
+  LoopBypassBlocks.push_back(VectorPH);
 
   // Split the single block loop into the two loop structure described above.
-  BasicBlock *VectorPH =
-  BypassBlock->splitBasicBlock(BypassBlock->getTerminator(), "vector.ph");
   BasicBlock *VecBody =
-  VectorPH->splitBasicBlock(VectorPH->getTerminator(), "vector.body");
+      VectorPH->splitBasicBlock(VectorPH->getTerminator(), "vector.body");
   BasicBlock *MiddleBlock =
   VecBody->splitBasicBlock(VecBody->getTerminator(), "middle.block");
   BasicBlock *ScalarPH =
@@ -2105,7 +2574,6 @@ void InnerLoopVectorizer::createEmptyLoop() {
   if (ParentLoop) {
     ParentLoop->addChildLoop(Lp);
     ParentLoop->addBasicBlockToLoop(ScalarPH, *LI);
-    ParentLoop->addBasicBlockToLoop(VectorPH, *LI);
     ParentLoop->addBasicBlockToLoop(MiddleBlock, *LI);
   } else {
     LI->addTopLevelLoop(Lp);
@@ -2123,9 +2591,20 @@ void InnerLoopVectorizer::createEmptyLoop() {
   // times the unroll factor (num of SIMD instructions).
   Constant *Step = ConstantInt::get(IdxTy, VF * UF);
 
+  // Generate code to check that the loop's trip count that we computed by
+  // adding one to the backedge-taken count will not overflow.
+  BasicBlock *NewVectorPH =
+      VectorPH->splitBasicBlock(VectorPH->getTerminator(), "overflow.checked");
+  if (ParentLoop)
+    ParentLoop->addBasicBlockToLoop(NewVectorPH, *LI);
+  ReplaceInstWithInst(
+      VectorPH->getTerminator(),
+      BranchInst::Create(ScalarPH, NewVectorPH, CheckBCOverflow));
+  VectorPH = NewVectorPH;
+
   // This is the IR builder that we use to add all of the logic for bypassing
   // the new vector loop.
-  IRBuilder<> BypassBuilder(BypassBlock->getTerminator());
+  IRBuilder<> BypassBuilder(VectorPH->getTerminator());
   setDebugLocFromInst(BypassBuilder,
                       getDebugLocFromInstOrOperands(OldInduction));
 
@@ -2154,24 +2633,14 @@ void InnerLoopVectorizer::createEmptyLoop() {
   // jump to the scalar loop.
   Value *Cmp =
       BypassBuilder.CreateICmpEQ(IdxEndRoundDown, StartIdx, "cmp.zero");
-
-  BasicBlock *LastBypassBlock = BypassBlock;
-
-  // Generate code to check that the loops trip count that we computed by adding
-  // one to the backedge-taken count will not overflow.
-  {
-    auto PastOverflowCheck =
-        std::next(BasicBlock::iterator(OverflowCheckAnchor));
-    BasicBlock *CheckBlock =
-      LastBypassBlock->splitBasicBlock(PastOverflowCheck, "overflow.checked");
-    if (ParentLoop)
-      ParentLoop->addBasicBlockToLoop(CheckBlock, *LI);
-    LoopBypassBlocks.push_back(CheckBlock);
-    Instruction *OldTerm = LastBypassBlock->getTerminator();
-    BranchInst::Create(ScalarPH, CheckBlock, CheckBCOverflow, OldTerm);
-    OldTerm->eraseFromParent();
-    LastBypassBlock = CheckBlock;
-  }
+  NewVectorPH =
+      VectorPH->splitBasicBlock(VectorPH->getTerminator(), "vector.ph");
+  if (ParentLoop)
+    ParentLoop->addBasicBlockToLoop(NewVectorPH, *LI);
+  LoopBypassBlocks.push_back(VectorPH);
+  ReplaceInstWithInst(VectorPH->getTerminator(),
+                      BranchInst::Create(MiddleBlock, NewVectorPH, Cmp));
+  VectorPH = NewVectorPH;
 
   // Generate the code to check that the strides we assumed to be one are really
   // one. We want the new basic block to start at the first instruction in a
@@ -2179,24 +2648,24 @@ void InnerLoopVectorizer::createEmptyLoop() {
   Instruction *StrideCheck;
   Instruction *FirstCheckInst;
   std::tie(FirstCheckInst, StrideCheck) =
-      addStrideCheck(LastBypassBlock->getTerminator());
+      addStrideCheck(VectorPH->getTerminator());
   if (StrideCheck) {
     AddedSafetyChecks = true;
     // Create a new block containing the stride check.
-    BasicBlock *CheckBlock =
-        LastBypassBlock->splitBasicBlock(FirstCheckInst, "vector.stridecheck");
+    VectorPH->setName("vector.stridecheck");
+    NewVectorPH =
+        VectorPH->splitBasicBlock(VectorPH->getTerminator(), "vector.ph");
     if (ParentLoop)
-      ParentLoop->addBasicBlockToLoop(CheckBlock, *LI);
-    LoopBypassBlocks.push_back(CheckBlock);
+      ParentLoop->addBasicBlockToLoop(NewVectorPH, *LI);
+    LoopBypassBlocks.push_back(VectorPH);
 
     // Replace the branch into the memory check block with a conditional branch
     // for the "few elements case".
-    Instruction *OldTerm = LastBypassBlock->getTerminator();
-    BranchInst::Create(MiddleBlock, CheckBlock, Cmp, OldTerm);
-    OldTerm->eraseFromParent();
+    ReplaceInstWithInst(
+        VectorPH->getTerminator(),
+        BranchInst::Create(MiddleBlock, NewVectorPH, StrideCheck));
 
-    Cmp = StrideCheck;
-    LastBypassBlock = CheckBlock;
+    VectorPH = NewVectorPH;
   }
 
   // Generate the code that checks in runtime if arrays overlap. We put the
@@ -2204,30 +2673,26 @@ void InnerLoopVectorizer::createEmptyLoop() {
   // faster.
   Instruction *MemRuntimeCheck;
   std::tie(FirstCheckInst, MemRuntimeCheck) =
-    Legal->getLAI()->addRuntimeCheck(LastBypassBlock->getTerminator());
+      Legal->getLAI()->addRuntimeCheck(VectorPH->getTerminator());
   if (MemRuntimeCheck) {
     AddedSafetyChecks = true;
     // Create a new block containing the memory check.
-    BasicBlock *CheckBlock =
-        LastBypassBlock->splitBasicBlock(FirstCheckInst, "vector.memcheck");
+    VectorPH->setName("vector.memcheck");
+    NewVectorPH =
+        VectorPH->splitBasicBlock(VectorPH->getTerminator(), "vector.ph");
     if (ParentLoop)
-      ParentLoop->addBasicBlockToLoop(CheckBlock, *LI);
-    LoopBypassBlocks.push_back(CheckBlock);
+      ParentLoop->addBasicBlockToLoop(NewVectorPH, *LI);
+    LoopBypassBlocks.push_back(VectorPH);
 
     // Replace the branch into the memory check block with a conditional branch
     // for the "few elements case".
-    Instruction *OldTerm = LastBypassBlock->getTerminator();
-    BranchInst::Create(MiddleBlock, CheckBlock, Cmp, OldTerm);
-    OldTerm->eraseFromParent();
+    ReplaceInstWithInst(
+        VectorPH->getTerminator(),
+        BranchInst::Create(MiddleBlock, NewVectorPH, MemRuntimeCheck));
 
-    Cmp = MemRuntimeCheck;
-    LastBypassBlock = CheckBlock;
+    VectorPH = NewVectorPH;
   }
 
-  LastBypassBlock->getTerminator()->eraseFromParent();
-  BranchInst::Create(MiddleBlock, VectorPH, Cmp,
-                     LastBypassBlock);
-
   // We are going to resume the execution of the scalar loop.
   // Go over all of the induction variables that we found and fix the
   // PHIs that are left in the scalar version of the loop.
@@ -2363,10 +2828,8 @@ void InnerLoopVectorizer::createEmptyLoop() {
   Value *CmpN = CmpInst::Create(Instruction::ICmp, CmpInst::ICMP_EQ, IdxEnd,
                                 ResumeIndex, "cmp.n",
                                 MiddleBlock->getTerminator());
-
-  BranchInst::Create(ExitBlock, ScalarPH, CmpN, MiddleBlock->getTerminator());
-  // Remove the old terminator.
-  MiddleBlock->getTerminator()->eraseFromParent();
+  ReplaceInstWithInst(MiddleBlock->getTerminator(),
+                      BranchInst::Create(ExitBlock, ScalarPH, CmpN));
 
   // Create i+1 and fill the PHINode.
   Value *NextIdx = Builder.CreateAdd(Induction, Step, "index.next");
@@ -2605,13 +3068,13 @@ void InnerLoopVectorizer::vectorizeLoop() {
     // Find the reduction variable descriptor.
     assert(Legal->getReductionVars()->count(RdxPhi) &&
            "Unable to find the reduction variable");
-    ReductionDescriptor RdxDesc = (*Legal->getReductionVars())[RdxPhi];
+    RecurrenceDescriptor RdxDesc = (*Legal->getReductionVars())[RdxPhi];
 
-    ReductionDescriptor::ReductionKind RK = RdxDesc.getReductionKind();
-    TrackingVH<Value> ReductionStartValue = RdxDesc.getReductionStartValue();
+    RecurrenceDescriptor::RecurrenceKind RK = RdxDesc.getRecurrenceKind();
+    TrackingVH<Value> ReductionStartValue = RdxDesc.getRecurrenceStartValue();
     Instruction *LoopExitInst = RdxDesc.getLoopExitInstr();
-    ReductionInstDesc::MinMaxReductionKind MinMaxKind =
-        RdxDesc.getMinMaxReductionKind();
+    RecurrenceDescriptor::MinMaxRecurrenceKind MinMaxKind =
+        RdxDesc.getMinMaxRecurrenceKind();
     setDebugLocFromInst(Builder, ReductionStartValue);
 
     // We need to generate a reduction vector from the incoming scalar.
@@ -2628,8 +3091,8 @@ void InnerLoopVectorizer::vectorizeLoop() {
     // one for multiplication, -1 for And.
     Value *Identity;
     Value *VectorStart;
-    if (RK == ReductionDescriptor::RK_IntegerMinMax ||
-        RK == ReductionDescriptor::RK_FloatMinMax) {
+    if (RK == RecurrenceDescriptor::RK_IntegerMinMax ||
+        RK == RecurrenceDescriptor::RK_FloatMinMax) {
       // MinMax reduction have the start value as their identify.
       if (VF == 1) {
         VectorStart = Identity = ReductionStartValue;
@@ -2639,8 +3102,8 @@ void InnerLoopVectorizer::vectorizeLoop() {
       }
     } else {
       // Handle other reduction kinds:
-      Constant *Iden =
-          ReductionDescriptor::getReductionIdentity(RK, VecTy->getScalarType());
+      Constant *Iden = RecurrenceDescriptor::getRecurrenceIdentity(
+          RK, VecTy->getScalarType());
       if (VF == 1) {
         Identity = Iden;
         // This vector is the Identity vector where the first element is the
@@ -2697,7 +3160,7 @@ void InnerLoopVectorizer::vectorizeLoop() {
 
     // Reduce all of the unrolled parts into a single vector.
     Value *ReducedPartRdx = RdxParts[0];
-    unsigned Op = ReductionDescriptor::getReductionBinOp(RK);
+    unsigned Op = RecurrenceDescriptor::getRecurrenceBinOp(RK);
     setDebugLocFromInst(Builder, ReducedPartRdx);
     for (unsigned part = 1; part < UF; ++part) {
       if (Op != Instruction::ICmp && Op != Instruction::FCmp)
@@ -2706,7 +3169,7 @@ void InnerLoopVectorizer::vectorizeLoop() {
             Builder.CreateBinOp((Instruction::BinaryOps)Op, RdxParts[part],
                                 ReducedPartRdx, "bin.rdx"));
       else
-        ReducedPartRdx = ReductionDescriptor::createMinMaxOp(
+        ReducedPartRdx = RecurrenceDescriptor::createMinMaxOp(
             Builder, MinMaxKind, ReducedPartRdx, RdxParts[part]);
     }
 
@@ -2738,8 +3201,8 @@ void InnerLoopVectorizer::vectorizeLoop() {
           TmpVec = addFastMathFlag(Builder.CreateBinOp(
               (Instruction::BinaryOps)Op, TmpVec, Shuf, "bin.rdx"));
         else
-          TmpVec = ReductionDescriptor::createMinMaxOp(Builder, MinMaxKind,
-                                                       TmpVec, Shuf);
+          TmpVec = RecurrenceDescriptor::createMinMaxOp(Builder, MinMaxKind,
+                                                        TmpVec, Shuf);
       }
 
       // The result is in the first element of the vector.
@@ -3344,7 +3807,7 @@ bool LoopVectorizationLegality::canVectorize() {
   }
 
   // We can only vectorize innermost loops.
-  if (!TheLoop->getSubLoopsVector().empty()) {
+  if (!TheLoop->empty()) {
     emitAnalysis(VectorizationReport() << "loop is not the innermost loop");
     return false;
   }
@@ -3410,10 +3873,15 @@ bool LoopVectorizationLegality::canVectorize() {
   // Collect all of the variables that remain uniform after vectorization.
   collectLoopUniforms();
 
-  DEBUG(dbgs() << "LV: We can vectorize this loop" <<
-        (LAI->getRuntimePointerCheck()->Need ? " (with a runtime bound check)" :
-         "")
-        <<"!\n");
+  DEBUG(dbgs() << "LV: We can vectorize this loop"
+               << (LAI->getRuntimePointerChecking()->Need
+                       ? " (with a runtime bound check)"
+                       : "")
+               << "!\n");
+
+  // Analyze interleaved memory accesses.
+  if (EnableInterleavedMemAccesses)
+    InterleaveInfo.analyzeInterleaving(Strides);
 
   // Okay! We can vectorize. At this point we don't have any other mem analysis
   // which may limit our maximum vectorization factor, so just return true with
@@ -3550,8 +4018,8 @@ bool LoopVectorizationLegality::canVectorizeInstrs() {
           continue;
         }
 
-        if (ReductionDescriptor::isReductionPHI(Phi, TheLoop,
-                                                Reductions[Phi])) {
+        if (RecurrenceDescriptor::isReductionPHI(Phi, TheLoop,
+                                                 Reductions[Phi])) {
           AllowedExit.insert(Reductions[Phi].getLoopExitInstr());
           continue;
         }
@@ -3639,118 +4107,6 @@ bool LoopVectorizationLegality::canVectorizeInstrs() {
   return true;
 }
 
-///\brief Remove GEPs whose indices but the last one are loop invariant and
-/// return the induction operand of the gep pointer.
-static Value *stripGetElementPtr(Value *Ptr, ScalarEvolution *SE, Loop *Lp) {
-  GetElementPtrInst *GEP = dyn_cast<GetElementPtrInst>(Ptr);
-  if (!GEP)
-    return Ptr;
-
-  unsigned InductionOperand = getGEPInductionOperand(GEP);
-
-  // Check that all of the gep indices are uniform except for our induction
-  // operand.
-  for (unsigned i = 0, e = GEP->getNumOperands(); i != e; ++i)
-    if (i != InductionOperand &&
-        !SE->isLoopInvariant(SE->getSCEV(GEP->getOperand(i)), Lp))
-      return Ptr;
-  return GEP->getOperand(InductionOperand);
-}
-
-///\brief Look for a cast use of the passed value.
-static Value *getUniqueCastUse(Value *Ptr, Loop *Lp, Type *Ty) {
-  Value *UniqueCast = nullptr;
-  for (User *U : Ptr->users()) {
-    CastInst *CI = dyn_cast<CastInst>(U);
-    if (CI && CI->getType() == Ty) {
-      if (!UniqueCast)
-        UniqueCast = CI;
-      else
-        return nullptr;
-    }
-  }
-  return UniqueCast;
-}
-
-///\brief Get the stride of a pointer access in a loop.
-/// Looks for symbolic strides "a[i*stride]". Returns the symbolic stride as a
-/// pointer to the Value, or null otherwise.
-static Value *getStrideFromPointer(Value *Ptr, ScalarEvolution *SE, Loop *Lp) {
-  const PointerType *PtrTy = dyn_cast<PointerType>(Ptr->getType());
-  if (!PtrTy || PtrTy->isAggregateType())
-    return nullptr;
-
-  // Try to remove a gep instruction to make the pointer (actually index at this
-  // point) easier analyzable. If OrigPtr is equal to Ptr we are analzying the
-  // pointer, otherwise, we are analyzing the index.
-  Value *OrigPtr = Ptr;
-
-  // The size of the pointer access.
-  int64_t PtrAccessSize = 1;
-
-  Ptr = stripGetElementPtr(Ptr, SE, Lp);
-  const SCEV *V = SE->getSCEV(Ptr);
-
-  if (Ptr != OrigPtr)
-    // Strip off casts.
-    while (const SCEVCastExpr *C = dyn_cast<SCEVCastExpr>(V))
-      V = C->getOperand();
-
-  const SCEVAddRecExpr *S = dyn_cast<SCEVAddRecExpr>(V);
-  if (!S)
-    return nullptr;
-
-  V = S->getStepRecurrence(*SE);
-  if (!V)
-    return nullptr;
-
-  // Strip off the size of access multiplication if we are still analyzing the
-  // pointer.
-  if (OrigPtr == Ptr) {
-    const DataLayout &DL = Lp->getHeader()->getModule()->getDataLayout();
-    DL.getTypeAllocSize(PtrTy->getElementType());
-    if (const SCEVMulExpr *M = dyn_cast<SCEVMulExpr>(V)) {
-      if (M->getOperand(0)->getSCEVType() != scConstant)
-        return nullptr;
-
-      const APInt &APStepVal =
-          cast<SCEVConstant>(M->getOperand(0))->getValue()->getValue();
-
-      // Huge step value - give up.
-      if (APStepVal.getBitWidth() > 64)
-        return nullptr;
-
-      int64_t StepVal = APStepVal.getSExtValue();
-      if (PtrAccessSize != StepVal)
-        return nullptr;
-      V = M->getOperand(1);
-    }
-  }
-
-  // Strip off casts.
-  Type *StripedOffRecurrenceCast = nullptr;
-  if (const SCEVCastExpr *C = dyn_cast<SCEVCastExpr>(V)) {
-    StripedOffRecurrenceCast = C->getType();
-    V = C->getOperand();
-  }
-
-  // Look for the loop invariant symbolic value.
-  const SCEVUnknown *U = dyn_cast<SCEVUnknown>(V);
-  if (!U)
-    return nullptr;
-
-  Value *Stride = U->getValue();
-  if (!Lp->isLoopInvariant(Stride))
-    return nullptr;
-
-  // If we have stripped off the recurrence cast we have to make sure that we
-  // return the value that is used in this loop so that we can replace it later.
-  if (StripedOffRecurrenceCast)
-    Stride = getUniqueCastUse(Stride, Lp, StripedOffRecurrenceCast);
-
-  return Stride;
-}
-
 void LoopVectorizationLegality::collectStridedAccess(Value *MemAccess) {
   Value *Ptr = nullptr;
   if (LoadInst *LI = dyn_cast<LoadInst>(MemAccess))
@@ -3930,11 +4286,171 @@ bool LoopVectorizationLegality::blockCanBePredicated(BasicBlock *BB,
   return true;
 }
 
+void InterleavedAccessInfo::collectConstStridedAccesses(
+    MapVector<Instruction *, StrideDescriptor> &StrideAccesses,
+    const ValueToValueMap &Strides) {
+  // Holds load/store instructions in program order.
+  SmallVector<Instruction *, 16> AccessList;
+
+  for (auto *BB : TheLoop->getBlocks()) {
+    bool IsPred = LoopAccessInfo::blockNeedsPredication(BB, TheLoop, DT);
+
+    for (auto &I : *BB) {
+      if (!isa<LoadInst>(&I) && !isa<StoreInst>(&I))
+        continue;
+      // FIXME: Currently we can't handle mixed accesses and predicated accesses
+      if (IsPred)
+        return;
+
+      AccessList.push_back(&I);
+    }
+  }
+
+  if (AccessList.empty())
+    return;
+
+  auto &DL = TheLoop->getHeader()->getModule()->getDataLayout();
+  for (auto I : AccessList) {
+    LoadInst *LI = dyn_cast<LoadInst>(I);
+    StoreInst *SI = dyn_cast<StoreInst>(I);
+
+    Value *Ptr = LI ? LI->getPointerOperand() : SI->getPointerOperand();
+    int Stride = isStridedPtr(SE, Ptr, TheLoop, Strides);
+
+    // The factor of the corresponding interleave group.
+    unsigned Factor = std::abs(Stride);
+
+    // Ignore the access if the factor is too small or too large.
+    if (Factor < 2 || Factor > MaxInterleaveGroupFactor)
+      continue;
+
+    const SCEV *Scev = replaceSymbolicStrideSCEV(SE, Strides, Ptr);
+    PointerType *PtrTy = dyn_cast<PointerType>(Ptr->getType());
+    unsigned Size = DL.getTypeAllocSize(PtrTy->getElementType());
+
+    // An alignment of 0 means target ABI alignment.
+    unsigned Align = LI ? LI->getAlignment() : SI->getAlignment();
+    if (!Align)
+      Align = DL.getABITypeAlignment(PtrTy->getElementType());
+
+    StrideAccesses[I] = StrideDescriptor(Stride, Scev, Size, Align);
+  }
+}
+
+// Analyze interleaved accesses and collect them into interleave groups.
+//
+// Notice that the vectorization on interleaved groups will change instruction
+// orders and may break dependences. But the memory dependence check guarantees
+// that there is no overlap between two pointers of different strides, element
+// sizes or underlying bases.
+//
+// For pointers sharing the same stride, element size and underlying base, no
+// need to worry about Read-After-Write dependences and Write-After-Read
+// dependences.
+//
+// E.g. The RAW dependence:  A[i] = a;
+//                           b = A[i];
+// This won't exist as it is a store-load forwarding conflict, which has
+// already been checked and forbidden in the dependence check.
+//
+// E.g. The WAR dependence:  a = A[i];  // (1)
+//                           A[i] = b;  // (2)
+// The store group of (2) is always inserted at or below (2), and the load group
+// of (1) is always inserted at or above (1). The dependence is safe.
+void InterleavedAccessInfo::analyzeInterleaving(
+    const ValueToValueMap &Strides) {
+  DEBUG(dbgs() << "LV: Analyzing interleaved accesses...\n");
+
+  // Holds all the stride accesses.
+  MapVector<Instruction *, StrideDescriptor> StrideAccesses;
+  collectConstStridedAccesses(StrideAccesses, Strides);
+
+  if (StrideAccesses.empty())
+    return;
+
+  // Holds all interleaved store groups temporarily.
+  SmallSetVector<InterleaveGroup *, 4> StoreGroups;
+
+  // Search the load-load/write-write pair B-A in bottom-up order and try to
+  // insert B into the interleave group of A according to 3 rules:
+  //   1. A and B have the same stride.
+  //   2. A and B have the same memory object size.
+  //   3. B belongs to the group according to the distance.
+  //
+  // The bottom-up order can avoid breaking the Write-After-Write dependences
+  // between two pointers of the same base.
+  // E.g.  A[i]   = a;   (1)
+  //       A[i]   = b;   (2)
+  //       A[i+1] = c    (3)
+  // We form the group (2)+(3) in front, so (1) has to form groups with accesses
+  // above (1), which guarantees that (1) is always above (2).
+  for (auto I = StrideAccesses.rbegin(), E = StrideAccesses.rend(); I != E;
+       ++I) {
+    Instruction *A = I->first;
+    StrideDescriptor DesA = I->second;
+
+    InterleaveGroup *Group = getInterleaveGroup(A);
+    if (!Group) {
+      DEBUG(dbgs() << "LV: Creating an interleave group with:" << *A << '\n');
+      Group = createInterleaveGroup(A, DesA.Stride, DesA.Align);
+    }
+
+    if (A->mayWriteToMemory())
+      StoreGroups.insert(Group);
+
+    for (auto II = std::next(I); II != E; ++II) {
+      Instruction *B = II->first;
+      StrideDescriptor DesB = II->second;
+
+      // Ignore if B is already in a group or B is a different memory operation.
+      if (isInterleaved(B) || A->mayReadFromMemory() != B->mayReadFromMemory())
+        continue;
+
+      // Check the rule 1 and 2.
+      if (DesB.Stride != DesA.Stride || DesB.Size != DesA.Size)
+        continue;
+
+      // Calculate the distance and prepare for the rule 3.
+      const SCEVConstant *DistToA =
+          dyn_cast<SCEVConstant>(SE->getMinusSCEV(DesB.Scev, DesA.Scev));
+      if (!DistToA)
+        continue;
+
+      int DistanceToA = DistToA->getValue()->getValue().getSExtValue();
+
+      // Skip if the distance is not multiple of size as they are not in the
+      // same group.
+      if (DistanceToA % static_cast<int>(DesA.Size))
+        continue;
+
+      // The index of B is the index of A plus the related index to A.
+      int IndexB =
+          Group->getIndex(A) + DistanceToA / static_cast<int>(DesA.Size);
+
+      // Try to insert B into the group.
+      if (Group->insertMember(B, IndexB, DesB.Align)) {
+        DEBUG(dbgs() << "LV: Inserted:" << *B << '\n'
+                     << "    into the interleave group with" << *A << '\n');
+        InterleaveGroupMap[B] = Group;
+
+        // Set the first load in program order as the insert position.
+        if (B->mayReadFromMemory())
+          Group->setInsertPos(B);
+      }
+    } // Iteration on instruction B
+  }   // Iteration on instruction A
+
+  // Remove interleaved store groups with gaps.
+  for (InterleaveGroup *Group : StoreGroups)
+    if (Group->getNumMembers() != Group->getFactor())
+      releaseGroup(Group);
+}
+
 LoopVectorizationCostModel::VectorizationFactor
 LoopVectorizationCostModel::selectVectorizationFactor(bool OptForSize) {
   // Width 1 means no vectorize
   VectorizationFactor Factor = { 1U, 0U };
-  if (OptForSize && Legal->getRuntimePointerCheck()->Need) {
+  if (OptForSize && Legal->getRuntimePointerChecking()->Need) {
     emitAnalysis(VectorizationReport() <<
                  "runtime pointer checks needed. Enable vectorization of this "
                  "loop with '#pragma clang loop vectorize(enable)' when "
@@ -3992,10 +4508,9 @@ LoopVectorizationCostModel::selectVectorizationFactor(bool OptForSize) {
 
     if (VF == 0)
       VF = MaxVectorSize;
-
-    // If the trip count that we found modulo the vectorization factor is not
-    // zero then we require a tail.
-    if (VF < 2) {
+    else {
+      // If the trip count that we found modulo the vectorization factor is not
+      // zero then we require a tail.
       emitAnalysis(VectorizationReport() <<
                    "cannot optimize for size and vectorize at the "
                    "same time. Enable vectorization of this loop "
@@ -4095,41 +4610,40 @@ unsigned LoopVectorizationCostModel::getWidestType() {
   return MaxWidth;
 }
 
-unsigned
-LoopVectorizationCostModel::selectUnrollFactor(bool OptForSize,
-                                               unsigned VF,
-                                               unsigned LoopCost) {
+unsigned LoopVectorizationCostModel::selectInterleaveCount(bool OptForSize,
+                                                           unsigned VF,
+                                                           unsigned LoopCost) {
 
-  // -- The unroll heuristics --
-  // We unroll the loop in order to expose ILP and reduce the loop overhead.
+  // -- The interleave heuristics --
+  // We interleave the loop in order to expose ILP and reduce the loop overhead.
   // There are many micro-architectural considerations that we can't predict
   // at this level. For example, frontend pressure (on decode or fetch) due to
   // code size, or the number and capabilities of the execution ports.
   //
-  // We use the following heuristics to select the unroll factor:
-  // 1. If the code has reductions, then we unroll in order to break the cross
+  // We use the following heuristics to select the interleave count:
+  // 1. If the code has reductions, then we interleave to break the cross
   // iteration dependency.
-  // 2. If the loop is really small, then we unroll in order to reduce the loop
+  // 2. If the loop is really small, then we interleave to reduce the loop
   // overhead.
-  // 3. We don't unroll if we think that we will spill registers to memory due
-  // to the increased register pressure.
+  // 3. We don't interleave if we think that we will spill registers to memory
+  // due to the increased register pressure.
 
   // Use the user preference, unless 'auto' is selected.
   int UserUF = Hints->getInterleave();
   if (UserUF != 0)
     return UserUF;
 
-  // When we optimize for size, we don't unroll.
+  // When we optimize for size, we don't interleave.
   if (OptForSize)
     return 1;
 
-  // We used the distance for the unroll factor.
+  // We used the distance for the interleave count.
   if (Legal->getMaxSafeDepDistBytes() != -1U)
     return 1;
 
-  // Do not unroll loops with a relatively small trip count.
+  // Do not interleave loops with a relatively small trip count.
   unsigned TC = SE->getSmallConstantTripCount(TheLoop);
-  if (TC > 1 && TC < TinyTripCountUnrollThreshold)
+  if (TC > 1 && TC < TinyTripCountInterleaveThreshold)
     return 1;
 
   unsigned TargetNumRegisters = TTI.getNumberOfRegisters(VF > 1);
@@ -4150,32 +4664,32 @@ LoopVectorizationCostModel::selectUnrollFactor(bool OptForSize,
   R.MaxLocalUsers = std::max(R.MaxLocalUsers, 1U);
   R.NumInstructions = std::max(R.NumInstructions, 1U);
 
-  // We calculate the unroll factor using the following formula.
+  // We calculate the interleave count using the following formula.
   // Subtract the number of loop invariants from the number of available
-  // registers. These registers are used by all of the unrolled instances.
+  // registers. These registers are used by all of the interleaved instances.
   // Next, divide the remaining registers by the number of registers that is
   // required by the loop, in order to estimate how many parallel instances
   // fit without causing spills. All of this is rounded down if necessary to be
-  // a power of two. We want power of two unroll factors to simplify any
+  // a power of two. We want power of two interleave count to simplify any
   // addressing operations or alignment considerations.
-  unsigned UF = PowerOf2Floor((TargetNumRegisters - R.LoopInvariantRegs) /
+  unsigned IC = PowerOf2Floor((TargetNumRegisters - R.LoopInvariantRegs) /
                               R.MaxLocalUsers);
 
-  // Don't count the induction variable as unrolled.
+  // Don't count the induction variable as interleaved.
   if (EnableIndVarRegisterHeur)
-    UF = PowerOf2Floor((TargetNumRegisters - R.LoopInvariantRegs - 1) /
+    IC = PowerOf2Floor((TargetNumRegisters - R.LoopInvariantRegs - 1) /
                        std::max(1U, (R.MaxLocalUsers - 1)));
 
-  // Clamp the unroll factor ranges to reasonable factors.
-  unsigned MaxInterleaveSize = TTI.getMaxInterleaveFactor();
+  // Clamp the interleave ranges to reasonable counts.
+  unsigned MaxInterleaveCount = TTI.getMaxInterleaveFactor(VF);
 
-  // Check if the user has overridden the unroll max.
+  // Check if the user has overridden the max.
   if (VF == 1) {
     if (ForceTargetMaxScalarInterleaveFactor.getNumOccurrences() > 0)
-      MaxInterleaveSize = ForceTargetMaxScalarInterleaveFactor;
+      MaxInterleaveCount = ForceTargetMaxScalarInterleaveFactor;
   } else {
     if (ForceTargetMaxVectorInterleaveFactor.getNumOccurrences() > 0)
-      MaxInterleaveSize = ForceTargetMaxVectorInterleaveFactor;
+      MaxInterleaveCount = ForceTargetMaxVectorInterleaveFactor;
   }
 
   // If we did not calculate the cost for VF (because the user selected the VF)
@@ -4183,72 +4697,74 @@ LoopVectorizationCostModel::selectUnrollFactor(bool OptForSize,
   if (LoopCost == 0)
     LoopCost = expectedCost(VF);
 
-  // Clamp the calculated UF to be between the 1 and the max unroll factor
+  // Clamp the calculated IC to be between the 1 and the max interleave count
   // that the target allows.
-  if (UF > MaxInterleaveSize)
-    UF = MaxInterleaveSize;
-  else if (UF < 1)
-    UF = 1;
+  if (IC > MaxInterleaveCount)
+    IC = MaxInterleaveCount;
+  else if (IC < 1)
+    IC = 1;
 
-  // Unroll if we vectorized this loop and there is a reduction that could
-  // benefit from unrolling.
+  // Interleave if we vectorized this loop and there is a reduction that could
+  // benefit from interleaving.
   if (VF > 1 && Legal->getReductionVars()->size()) {
-    DEBUG(dbgs() << "LV: Unrolling because of reductions.\n");
-    return UF;
+    DEBUG(dbgs() << "LV: Interleaving because of reductions.\n");
+    return IC;
   }
 
   // Note that if we've already vectorized the loop we will have done the
-  // runtime check and so unrolling won't require further checks.
-  bool UnrollingRequiresRuntimePointerCheck =
-      (VF == 1 && Legal->getRuntimePointerCheck()->Need);
+  // runtime check and so interleaving won't require further checks.
+  bool InterleavingRequiresRuntimePointerCheck =
+      (VF == 1 && Legal->getRuntimePointerChecking()->Need);
 
-  // We want to unroll small loops in order to reduce the loop overhead and
+  // We want to interleave small loops in order to reduce the loop overhead and
   // potentially expose ILP opportunities.
   DEBUG(dbgs() << "LV: Loop cost is " << LoopCost << '\n');
-  if (!UnrollingRequiresRuntimePointerCheck &&
-      LoopCost < SmallLoopCost) {
+  if (!InterleavingRequiresRuntimePointerCheck && LoopCost < SmallLoopCost) {
     // We assume that the cost overhead is 1 and we use the cost model
-    // to estimate the cost of the loop and unroll until the cost of the
+    // to estimate the cost of the loop and interleave until the cost of the
     // loop overhead is about 5% of the cost of the loop.
-    unsigned SmallUF = std::min(UF, (unsigned)PowerOf2Floor(SmallLoopCost / LoopCost));
+    unsigned SmallIC =
+        std::min(IC, (unsigned)PowerOf2Floor(SmallLoopCost / LoopCost));
 
-    // Unroll until store/load ports (estimated by max unroll factor) are
+    // Interleave until store/load ports (estimated by max interleave count) are
     // saturated.
     unsigned NumStores = Legal->getNumStores();
     unsigned NumLoads = Legal->getNumLoads();
-    unsigned StoresUF = UF / (NumStores ? NumStores : 1);
-    unsigned LoadsUF = UF /  (NumLoads ? NumLoads : 1);
+    unsigned StoresIC = IC / (NumStores ? NumStores : 1);
+    unsigned LoadsIC = IC / (NumLoads ? NumLoads : 1);
 
     // If we have a scalar reduction (vector reductions are already dealt with
     // by this point), we can increase the critical path length if the loop
-    // we're unrolling is inside another loop. Limit, by default to 2, so the
+    // we're interleaving is inside another loop. Limit, by default to 2, so the
     // critical path only gets increased by one reduction operation.
     if (Legal->getReductionVars()->size() &&
         TheLoop->getLoopDepth() > 1) {
-      unsigned F = static_cast<unsigned>(MaxNestedScalarReductionUF);
-      SmallUF = std::min(SmallUF, F);
-      StoresUF = std::min(StoresUF, F);
-      LoadsUF = std::min(LoadsUF, F);
+      unsigned F = static_cast<unsigned>(MaxNestedScalarReductionIC);
+      SmallIC = std::min(SmallIC, F);
+      StoresIC = std::min(StoresIC, F);
+      LoadsIC = std::min(LoadsIC, F);
     }
 
-    if (EnableLoadStoreRuntimeUnroll && std::max(StoresUF, LoadsUF) > SmallUF) {
-      DEBUG(dbgs() << "LV: Unrolling to saturate store or load ports.\n");
-      return std::max(StoresUF, LoadsUF);
+    if (EnableLoadStoreRuntimeInterleave &&
+        std::max(StoresIC, LoadsIC) > SmallIC) {
+      DEBUG(dbgs() << "LV: Interleaving to saturate store or load ports.\n");
+      return std::max(StoresIC, LoadsIC);
     }
 
-    DEBUG(dbgs() << "LV: Unrolling to reduce branch cost.\n");
-    return SmallUF;
+    DEBUG(dbgs() << "LV: Interleaving to reduce branch cost.\n");
+    return SmallIC;
   }
 
-  // Unroll if this is a large loop (small loops are already dealt with by this
-  // point) that could benefit from interleaved unrolling.
+  // Interleave if this is a large loop (small loops are already dealt with by
+  // this
+  // point) that could benefit from interleaving.
   bool HasReductions = (Legal->getReductionVars()->size() > 0);
   if (TTI.enableAggressiveInterleaving(HasReductions)) {
-    DEBUG(dbgs() << "LV: Unrolling to expose ILP.\n");
-    return UF;
+    DEBUG(dbgs() << "LV: Interleaving to expose ILP.\n");
+    return IC;
   }
 
-  DEBUG(dbgs() << "LV: Not Unrolling.\n");
+  DEBUG(dbgs() << "LV: Not Interleaving.\n");
   return 1;
 }
 
@@ -4582,6 +5098,46 @@ LoopVectorizationCostModel::getInstructionCost(Instruction *I, unsigned VF) {
       return TTI.getAddressComputationCost(VectorTy) +
         TTI.getMemoryOpCost(I->getOpcode(), VectorTy, Alignment, AS);
 
+    // For an interleaved access, calculate the total cost of the whole
+    // interleave group.
+    if (Legal->isAccessInterleaved(I)) {
+      auto Group = Legal->getInterleavedAccessGroup(I);
+      assert(Group && "Fail to get an interleaved access group.");
+
+      // Only calculate the cost once at the insert position.
+      if (Group->getInsertPos() != I)
+        return 0;
+
+      unsigned InterleaveFactor = Group->getFactor();
+      Type *WideVecTy =
+          VectorType::get(VectorTy->getVectorElementType(),
+                          VectorTy->getVectorNumElements() * InterleaveFactor);
+
+      // Holds the indices of existing members in an interleaved load group.
+      // An interleaved store group doesn't need this as it dones't allow gaps.
+      SmallVector<unsigned, 4> Indices;
+      if (LI) {
+        for (unsigned i = 0; i < InterleaveFactor; i++)
+          if (Group->getMember(i))
+            Indices.push_back(i);
+      }
+
+      // Calculate the cost of the whole interleaved group.
+      unsigned Cost = TTI.getInterleavedMemoryOpCost(
+          I->getOpcode(), WideVecTy, Group->getFactor(), Indices,
+          Group->getAlignment(), AS);
+
+      if (Group->isReverse())
+        Cost +=
+            Group->getNumMembers() *
+            TTI.getShuffleCost(TargetTransformInfo::SK_Reverse, VectorTy, 0);
+
+      // FIXME: The interleaved load group with a huge gap could be even more
+      // expensive than scalar operations. Then we could ignore such group and
+      // use scalar operations instead.
+      return Cost;
+    }
+
     // Scalarized loads/stores.
     int ConsecutiveStride = Legal->isConsecutivePtr(Ptr);
     bool Reverse = ConsecutiveStride < 0;
@@ -4686,7 +5242,7 @@ INITIALIZE_PASS_BEGIN(LoopVectorize, LV_NAME, lv_name, false, false)
 INITIALIZE_PASS_DEPENDENCY(TargetTransformInfoWrapperPass)
 INITIALIZE_AG_DEPENDENCY(AliasAnalysis)
 INITIALIZE_PASS_DEPENDENCY(AssumptionCacheTracker)
-INITIALIZE_PASS_DEPENDENCY(BlockFrequencyInfo)
+INITIALIZE_PASS_DEPENDENCY(BlockFrequencyInfoWrapperPass)
 INITIALIZE_PASS_DEPENDENCY(DominatorTreeWrapperPass)
 INITIALIZE_PASS_DEPENDENCY(ScalarEvolution)
 INITIALIZE_PASS_DEPENDENCY(LCSSA)
@@ -4817,9 +5373,8 @@ void InnerLoopUnroller::scalarizeInstruction(Instruction *Instr,
         LoopVectorBody.push_back(NewIfBlock);
         VectorLp->addBasicBlockToLoop(NewIfBlock, *LI);
         Builder.SetInsertPoint(InsertPt);
-        Instruction *OldBr = IfBlock->getTerminator();
-        BranchInst::Create(CondBlock, NewIfBlock, Cmp, OldBr);
-        OldBr->eraseFromParent();
+        ReplaceInstWithInst(IfBlock->getTerminator(),
+                            BranchInst::Create(CondBlock, NewIfBlock, Cmp));
         IfBlock = NewIfBlock;
       }
   }
diff --git a/lib/Transforms/Vectorize/SLPVectorizer.cpp b/lib/Transforms/Vectorize/SLPVectorizer.cpp
index 7267f58d1c9..b215a256af0 100644
--- a/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -43,7 +43,7 @@
 #include "llvm/Support/CommandLine.h"
 #include "llvm/Support/Debug.h"
 #include "llvm/Support/raw_ostream.h"
-#include "llvm/Transforms/Utils/VectorUtils.h"
+#include "llvm/Analysis/VectorUtils.h"
 #include <algorithm>
 #include <map>
 #include <memory>
@@ -69,8 +69,13 @@ static cl::opt<bool> ShouldStartVectorizeHorAtStore(
     cl::desc(
         "Attempt to vectorize horizontal reductions feeding into a store"));
 
+static cl::opt<int>
+MaxVectorRegSizeOption("slp-max-reg-size", cl::init(128), cl::Hidden,
+    cl::desc("Attempt to vectorize for this register size in bits"));
+
 namespace {
 
+// FIXME: Set this via cl::opt to allow overriding.
 static const unsigned MinVecRegSize = 128;
 
 static const unsigned RecursionMaxDepth = 12;
@@ -315,12 +320,12 @@ static bool InTreeUserNeedToExtract(Value *Scalar, Instruction *UserInst,
 }
 
 /// \returns the AA location that is being access by the instruction.
-static AliasAnalysis::Location getLocation(Instruction *I, AliasAnalysis *AA) {
+static MemoryLocation getLocation(Instruction *I, AliasAnalysis *AA) {
   if (StoreInst *SI = dyn_cast<StoreInst>(I))
-    return AA->getLocation(SI);
+    return MemoryLocation::get(SI);
   if (LoadInst *LI = dyn_cast<LoadInst>(I))
-    return AA->getLocation(LI);
-  return AliasAnalysis::Location();
+    return MemoryLocation::get(LI);
+  return MemoryLocation();
 }
 
 /// \returns True if the instruction is not a volatile or atomic load/store.
@@ -472,7 +477,7 @@ class BoUpSLP {
 
   /// Create a new VectorizableTree entry.
   TreeEntry *newTreeEntry(ArrayRef<Value *> VL, bool Vectorized) {
-    VectorizableTree.push_back(TreeEntry());
+    VectorizableTree.emplace_back();
     int idx = VectorizableTree.size() - 1;
     TreeEntry *Last = &VectorizableTree[idx];
     Last->Scalars.insert(Last->Scalars.begin(), VL.begin(), VL.end());
@@ -501,7 +506,7 @@ class BoUpSLP {
   /// This POD struct describes one external user in the vectorized tree.
   struct ExternalUser {
     ExternalUser (Value *S, llvm::User *U, int L) :
-      Scalar(S), User(U), Lane(L){};
+      Scalar(S), User(U), Lane(L){}
     // Which scalar in our function.
     Value *Scalar;
     // Which user that uses the scalar.
@@ -515,7 +520,7 @@ class BoUpSLP {
   ///
   /// \p Loc1 is the location of \p Inst1. It is passed explicitly because it
   /// is invariant in the calling loop.
-  bool isAliased(const AliasAnalysis::Location &Loc1, Instruction *Inst1,
+  bool isAliased(const MemoryLocation &Loc1, Instruction *Inst1,
                  Instruction *Inst2) {
 
     // First check if the result is already in the cache.
@@ -524,7 +529,7 @@ class BoUpSLP {
     if (result.hasValue()) {
       return result.getValue();
     }
-    AliasAnalysis::Location Loc2 = getLocation(Inst2, AA);
+    MemoryLocation Loc2 = getLocation(Inst2, AA);
     bool aliased = true;
     if (Loc1.Ptr && Loc2.Ptr && isSimple(Inst1) && isSimple(Inst2)) {
       // Do the alias check.
@@ -1637,8 +1642,10 @@ bool BoUpSLP::isFullyVectorizableTinyTree() {
   if (VectorizableTree.size() != 2)
     return false;
 
-  // Handle splat stores.
-  if (!VectorizableTree[0].NeedToGather && isSplat(VectorizableTree[1].Scalars))
+  // Handle splat and all-constants stores.
+  if (!VectorizableTree[0].NeedToGather &&
+      (allConstant(VectorizableTree[1].Scalars) ||
+       isSplat(VectorizableTree[1].Scalars)))
     return true;
 
   // Gathering cost would be too much for tiny trees.
@@ -2134,9 +2141,8 @@ Value *BoUpSLP::vectorizeTree(TreeEntry *E) {
         }
 
         // Prepare the operand vector.
-        for (unsigned j = 0; j < E->Scalars.size(); ++j)
-          Operands.push_back(cast<PHINode>(E->Scalars[j])->
-                             getIncomingValueForBlock(IBB));
+        for (Value *V : E->Scalars)
+          Operands.push_back(cast<PHINode>(V)->getIncomingValueForBlock(IBB));
 
         Builder.SetInsertPoint(IBB->getTerminator());
         Builder.SetCurrentDebugLocation(PH->getDebugLoc());
@@ -2170,8 +2176,8 @@ Value *BoUpSLP::vectorizeTree(TreeEntry *E) {
     case Instruction::FPTrunc:
     case Instruction::BitCast: {
       ValueList INVL;
-      for (int i = 0, e = E->Scalars.size(); i < e; ++i)
-        INVL.push_back(cast<Instruction>(E->Scalars[i])->getOperand(0));
+      for (Value *V : E->Scalars)
+        INVL.push_back(cast<Instruction>(V)->getOperand(0));
 
       setInsertPointAfterBundle(E->Scalars);
 
@@ -2189,9 +2195,9 @@ Value *BoUpSLP::vectorizeTree(TreeEntry *E) {
     case Instruction::FCmp:
     case Instruction::ICmp: {
       ValueList LHSV, RHSV;
-      for (int i = 0, e = E->Scalars.size(); i < e; ++i) {
-        LHSV.push_back(cast<Instruction>(E->Scalars[i])->getOperand(0));
-        RHSV.push_back(cast<Instruction>(E->Scalars[i])->getOperand(1));
+      for (Value *V : E->Scalars) {
+        LHSV.push_back(cast<Instruction>(V)->getOperand(0));
+        RHSV.push_back(cast<Instruction>(V)->getOperand(1));
       }
 
       setInsertPointAfterBundle(E->Scalars);
@@ -2215,10 +2221,10 @@ Value *BoUpSLP::vectorizeTree(TreeEntry *E) {
     }
     case Instruction::Select: {
       ValueList TrueVec, FalseVec, CondVec;
-      for (int i = 0, e = E->Scalars.size(); i < e; ++i) {
-        CondVec.push_back(cast<Instruction>(E->Scalars[i])->getOperand(0));
-        TrueVec.push_back(cast<Instruction>(E->Scalars[i])->getOperand(1));
-        FalseVec.push_back(cast<Instruction>(E->Scalars[i])->getOperand(2));
+      for (Value *V : E->Scalars) {
+        CondVec.push_back(cast<Instruction>(V)->getOperand(0));
+        TrueVec.push_back(cast<Instruction>(V)->getOperand(1));
+        FalseVec.push_back(cast<Instruction>(V)->getOperand(2));
       }
 
       setInsertPointAfterBundle(E->Scalars);
@@ -2257,9 +2263,9 @@ Value *BoUpSLP::vectorizeTree(TreeEntry *E) {
       if (isa<BinaryOperator>(VL0) && VL0->isCommutative())
         reorderInputsAccordingToOpcode(E->Scalars, LHSVL, RHSVL);
       else
-        for (int i = 0, e = E->Scalars.size(); i < e; ++i) {
-          LHSVL.push_back(cast<Instruction>(E->Scalars[i])->getOperand(0));
-          RHSVL.push_back(cast<Instruction>(E->Scalars[i])->getOperand(1));
+        for (Value *V : E->Scalars) {
+          LHSVL.push_back(cast<Instruction>(V)->getOperand(0));
+          RHSVL.push_back(cast<Instruction>(V)->getOperand(1));
         }
 
       setInsertPointAfterBundle(E->Scalars);
@@ -2320,8 +2326,8 @@ Value *BoUpSLP::vectorizeTree(TreeEntry *E) {
       unsigned AS = SI->getPointerAddressSpace();
 
       ValueList ValueOp;
-      for (int i = 0, e = E->Scalars.size(); i < e; ++i)
-        ValueOp.push_back(cast<StoreInst>(E->Scalars[i])->getValueOperand());
+      for (Value *V : E->Scalars)
+        ValueOp.push_back(cast<StoreInst>(V)->getValueOperand());
 
       setInsertPointAfterBundle(E->Scalars);
 
@@ -2349,8 +2355,8 @@ Value *BoUpSLP::vectorizeTree(TreeEntry *E) {
       setInsertPointAfterBundle(E->Scalars);
 
       ValueList Op0VL;
-      for (int i = 0, e = E->Scalars.size(); i < e; ++i)
-        Op0VL.push_back(cast<GetElementPtrInst>(E->Scalars[i])->getOperand(0));
+      for (Value *V : E->Scalars)
+        Op0VL.push_back(cast<GetElementPtrInst>(V)->getOperand(0));
 
       Value *Op0 = vectorizeTree(Op0VL);
 
@@ -2358,8 +2364,8 @@ Value *BoUpSLP::vectorizeTree(TreeEntry *E) {
       for (int j = 1, e = cast<GetElementPtrInst>(VL0)->getNumOperands(); j < e;
            ++j) {
         ValueList OpVL;
-        for (int i = 0, e = E->Scalars.size(); i < e; ++i)
-          OpVL.push_back(cast<GetElementPtrInst>(E->Scalars[i])->getOperand(j));
+        for (Value *V : E->Scalars)
+          OpVL.push_back(cast<GetElementPtrInst>(V)->getOperand(j));
 
         Value *OpVec = vectorizeTree(OpVL);
         OpVecs.push_back(OpVec);
@@ -2382,7 +2388,7 @@ Value *BoUpSLP::vectorizeTree(TreeEntry *E) {
       Intrinsic::ID IID  = Intrinsic::not_intrinsic;
       Value *ScalarArg = nullptr;
       if (CI && (FI = CI->getCalledFunction())) {
-        IID = (Intrinsic::ID) FI->getIntrinsicID();
+        IID = FI->getIntrinsicID();
       }
       std::vector<Value *> OpVecs;
       for (int j = 0, e = CI->getNumArgOperands(); j < e; ++j) {
@@ -2395,8 +2401,8 @@ Value *BoUpSLP::vectorizeTree(TreeEntry *E) {
           OpVecs.push_back(CEI->getArgOperand(j));
           continue;
         }
-        for (int i = 0, e = E->Scalars.size(); i < e; ++i) {
-          CallInst *CEI = cast<CallInst>(E->Scalars[i]);
+        for (Value *V : E->Scalars) {
+          CallInst *CEI = cast<CallInst>(V);
           OpVL.push_back(CEI->getArgOperand(j));
         }
 
@@ -2903,7 +2909,7 @@ void BoUpSLP::BlockScheduling::calculateDependencies(ScheduleData *SD,
         ScheduleData *DepDest = BundleMember->NextLoadStore;
         if (DepDest) {
           Instruction *SrcInst = BundleMember->Inst;
-          AliasAnalysis::Location SrcLoc = getLocation(SrcInst, SLP->AA);
+          MemoryLocation SrcLoc = getLocation(SrcInst, SLP->AA);
           bool SrcMayWrite = BundleMember->Inst->mayWriteToMemory();
           unsigned numAliased = 0;
           unsigned DistToSrc = 1;
@@ -3087,6 +3093,17 @@ struct SLPVectorizer : public FunctionPass {
     if (!TTI->getNumberOfRegisters(true))
       return false;
 
+    // Use the vector register size specified by the target unless overridden
+    // by a command-line option.
+    // TODO: It would be better to limit the vectorization factor based on
+    //       data type rather than just register size. For example, x86 AVX has
+    //       256-bit registers, but it does not support integer operations
+    //       at that width (that requires AVX2).
+    if (MaxVectorRegSizeOption.getNumOccurrences())
+      MaxVecRegSize = MaxVectorRegSizeOption;
+    else
+      MaxVecRegSize = TTI->getRegisterBitWidth(true);
+
     // Don't vectorize when the attribute NoImplicitFloat is used.
     if (F.hasFnAttribute(Attribute::NoImplicitFloat))
       return false;
@@ -3164,12 +3181,13 @@ struct SLPVectorizer : public FunctionPass {
   bool vectorizeChainsInBlock(BasicBlock *BB, BoUpSLP &R);
 
   bool vectorizeStoreChain(ArrayRef<Value *> Chain, int CostThreshold,
-                           BoUpSLP &R);
+                           BoUpSLP &R, unsigned VecRegSize);
 
   bool vectorizeStores(ArrayRef<StoreInst *> Stores, int costThreshold,
                        BoUpSLP &R);
 private:
   StoreListMap StoreRefs;
+  unsigned MaxVecRegSize; // This is set by TTI or overridden by cl::opt.
 };
 
 /// \brief Check that the Values in the slice in VL array are still existent in
@@ -3184,14 +3202,15 @@ static bool hasValueBeenRAUWed(ArrayRef<Value *> VL, ArrayRef<WeakVH> VH,
 }
 
 bool SLPVectorizer::vectorizeStoreChain(ArrayRef<Value *> Chain,
-                                          int CostThreshold, BoUpSLP &R) {
+                                        int CostThreshold, BoUpSLP &R,
+                                        unsigned VecRegSize) {
   unsigned ChainLen = Chain.size();
   DEBUG(dbgs() << "SLP: Analyzing a store chain of length " << ChainLen
         << "\n");
   Type *StoreTy = cast<StoreInst>(Chain[0])->getValueOperand()->getType();
   auto &DL = cast<StoreInst>(Chain[0])->getModule()->getDataLayout();
   unsigned Sz = DL.getTypeSizeInBits(StoreTy);
-  unsigned VF = MinVecRegSize / Sz;
+  unsigned VF = VecRegSize / Sz;
 
   if (!isPowerOf2_32(Sz) || VF < 2)
     return false;
@@ -3243,15 +3262,26 @@ bool SLPVectorizer::vectorizeStores(ArrayRef<StoreInst *> Stores,
 
   // Do a quadratic search on all of the given stores and find
   // all of the pairs of stores that follow each other.
+  SmallVector<unsigned, 16> IndexQueue;
   for (unsigned i = 0, e = Stores.size(); i < e; ++i) {
-    for (unsigned j = 0; j < e; ++j) {
-      if (i == j)
-        continue;
-      const DataLayout &DL = Stores[i]->getModule()->getDataLayout();
-      if (R.isConsecutiveAccess(Stores[i], Stores[j], DL)) {
-        Tails.insert(Stores[j]);
+    const DataLayout &DL = Stores[i]->getModule()->getDataLayout();
+    IndexQueue.clear();
+    // If a store has multiple consecutive store candidates, search Stores
+    // array according to the sequence: from i+1 to e, then from i-1 to 0.
+    // This is because usually pairing with immediate succeeding or preceding
+    // candidate create the best chance to find slp vectorization opportunity.
+    unsigned j = 0;
+    for (j = i + 1; j < e; ++j)
+      IndexQueue.push_back(j);
+    for (j = i; j > 0; --j)
+      IndexQueue.push_back(j - 1);
+
+    for (auto &k : IndexQueue) {
+      if (R.isConsecutiveAccess(Stores[i], Stores[k], DL)) {
+        Tails.insert(Stores[k]);
         Heads.insert(Stores[i]);
-        ConsecutiveChain[Stores[i]] = Stores[j];
+        ConsecutiveChain[Stores[i]] = Stores[k];
+        break;
       }
     }
   }
@@ -3275,12 +3305,16 @@ bool SLPVectorizer::vectorizeStores(ArrayRef<StoreInst *> Stores,
       I = ConsecutiveChain[I];
     }
 
-    bool Vectorized = vectorizeStoreChain(Operands, costThreshold, R);
-
-    // Mark the vectorized stores so that we don't vectorize them again.
-    if (Vectorized)
-      VectorizedStores.insert(Operands.begin(), Operands.end());
-    Changed |= Vectorized;
+    // FIXME: Is division-by-2 the correct step? Should we assert that the
+    // register size is a power-of-2?
+    for (unsigned Size = MaxVecRegSize; Size >= MinVecRegSize; Size /= 2) {
+      if (vectorizeStoreChain(Operands, costThreshold, R, Size)) {
+        // Mark the vectorized stores so that we don't vectorize them again.
+        VectorizedStores.insert(Operands.begin(), Operands.end());
+        Changed = true;
+        break;
+      }
+    }
   }
 
   return Changed;
@@ -3291,8 +3325,8 @@ unsigned SLPVectorizer::collectStores(BasicBlock *BB, BoUpSLP &R) {
   unsigned count = 0;
   StoreRefs.clear();
   const DataLayout &DL = BB->getModule()->getDataLayout();
-  for (BasicBlock::iterator it = BB->begin(), e = BB->end(); it != e; ++it) {
-    StoreInst *SI = dyn_cast<StoreInst>(it);
+  for (Instruction &I : *BB) {
+    StoreInst *SI = dyn_cast<StoreInst>(&I);
     if (!SI)
       continue;
 
@@ -3340,13 +3374,15 @@ bool SLPVectorizer::tryToVectorizeList(ArrayRef<Value *> VL, BoUpSLP &R,
 
   Type *Ty0 = I0->getType();
   unsigned Sz = DL.getTypeSizeInBits(Ty0);
+  // FIXME: Register size should be a parameter to this function, so we can
+  // try different vectorization factors.
   unsigned VF = MinVecRegSize / Sz;
 
-  for (int i = 0, e = VL.size(); i < e; ++i) {
-    Type *Ty = VL[i]->getType();
+  for (Value *V : VL) {
+    Type *Ty = V->getType();
     if (!isValidElementType(Ty))
       return false;
-    Instruction *Inst = dyn_cast<Instruction>(VL[i]);
+    Instruction *Inst = dyn_cast<Instruction>(V);
     if (!Inst || Inst->getOpcode() != Opcode0)
       return false;
   }
@@ -3569,6 +3605,8 @@ class HorizontalReduction {
     const DataLayout &DL = B->getModule()->getDataLayout();
     ReductionOpcode = B->getOpcode();
     ReducedValueOpcode = 0;
+    // FIXME: Register size should be a parameter to this function, so we can
+    // try different vectorization factors.
     ReduxWidth = MinVecRegSize / DL.getTypeSizeInBits(Ty);
     ReductionRoot = B;
     ReductionPHI = Phi;
@@ -3995,6 +4033,9 @@ bool SLPVectorizer::vectorizeStoreChains(BoUpSLP &R) {
           << it->second.size() << ".\n");
 
     // Process the stores in chunks of 16.
+    // TODO: The limit of 16 inhibits greater vectorization factors.
+    //       For example, AVX2 supports v32i8. Increasing this limit, however,
+    //       may cause a significant compile-time increase.
     for (unsigned CI = 0, CE = it->second.size(); CI < CE; CI+=16) {
       unsigned Len = std::min<unsigned>(CE - CI, 16);
       Changed |= vectorizeStores(makeArrayRef(&it->second[CI], Len),
diff --git a/projects/CMakeLists.txt b/projects/CMakeLists.txt
index 428ce84efd7..d3b2fc37af7 100644
--- a/projects/CMakeLists.txt
+++ b/projects/CMakeLists.txt
@@ -7,7 +7,8 @@ foreach(entry ${entries})
     if((NOT ${entry} STREQUAL ${CMAKE_CURRENT_SOURCE_DIR}/compiler-rt) AND
        (NOT ${entry} STREQUAL ${CMAKE_CURRENT_SOURCE_DIR}/dragonegg) AND
        (NOT ${entry} STREQUAL ${CMAKE_CURRENT_SOURCE_DIR}/libcxx) AND
-       (NOT ${entry} STREQUAL ${CMAKE_CURRENT_SOURCE_DIR}/libcxxabi))
+       (NOT ${entry} STREQUAL ${CMAKE_CURRENT_SOURCE_DIR}/libcxxabi) AND
+       (NOT ${entry} STREQUAL ${CMAKE_CURRENT_SOURCE_DIR}/libunwind))
       add_subdirectory(${entry})
     endif()
   endif()
@@ -23,6 +24,7 @@ if(${LLVM_BUILD_RUNTIME})
     # first.
     add_llvm_external_project(libcxxabi)
     add_llvm_external_project(libcxx)
+    add_llvm_external_project(libunwind)
   endif()
   if(NOT LLVM_BUILD_EXTERNAL_COMPILER_RT)
     add_llvm_external_project(compiler-rt)
diff --git a/resources/windows_version_resource.rc b/resources/windows_version_resource.rc
new file mode 100644
index 00000000000..6c96a4d215d
--- /dev/null
+++ b/resources/windows_version_resource.rc
@@ -0,0 +1,89 @@
+// Microsoft Visual C++ resource script for embedding version information.
+// The format is described at:
+//   http://msdn.microsoft.com/en-gb/library/windows/desktop/aa380599(v=vs.85).aspx
+// The VERSIONINFO resource is described at:
+//   https://msdn.microsoft.com/en-gb/library/windows/desktop/aa381058(v=vs.85).aspx
+
+
+// Default values for required fields.
+
+#ifndef RC_VERSION_FIELD_1
+#define RC_VERSION_FIELD_1 0
+#endif
+
+#ifndef RC_VERSION_FIELD_2
+#define RC_VERSION_FIELD_2 0
+#endif
+
+#ifndef RC_VERSION_FIELD_3
+#define RC_VERSION_FIELD_3 0
+#endif
+
+#ifndef RC_VERSION_FIELD_4
+#define RC_VERSION_FIELD_4 0
+#endif
+
+#ifndef RC_COMPANY_NAME
+#define RC_COMPANY_NAME ""
+#endif
+
+#ifndef RC_FILE_DESCRIPTION
+#define RC_FILE_DESCRIPTION ""
+#endif
+
+#ifndef RC_FILE_VERSION
+#define RC_FILE_VERSION ""
+#endif
+
+#ifndef RC_INTERNAL_NAME
+#define RC_INTERNAL_NAME ""
+#endif
+
+#ifndef RC_ORIGINAL_FILENAME
+#define RC_ORIGINAL_FILENAME ""
+#endif
+
+#ifndef RC_PRODUCT_NAME
+#define RC_PRODUCT_NAME ""
+#endif
+
+#ifndef RC_PRODUCT_VERSION
+#define RC_PRODUCT_VERSION ""
+#endif
+
+
+1 VERSIONINFO
+FILEVERSION RC_VERSION_FIELD_1,RC_VERSION_FIELD_2,RC_VERSION_FIELD_3,RC_VERSION_FIELD_4
+BEGIN
+  BLOCK "StringFileInfo"
+  BEGIN
+    BLOCK "040904B0"
+    BEGIN
+      // Required strings
+      VALUE "CompanyName", RC_COMPANY_NAME
+      VALUE "FileDescription", RC_FILE_DESCRIPTION
+      VALUE "FileVersion", RC_FILE_VERSION
+      VALUE "InternalName", RC_INTERNAL_NAME
+      VALUE "OriginalFilename", RC_ORIGINAL_FILENAME
+      VALUE "ProductName", RC_PRODUCT_NAME
+      VALUE "ProductVersion", RC_PRODUCT_VERSION
+
+      // Optional strings
+#ifdef RC_COMMENTS
+        VALUE "Comments", RC_COMMENTS
+#endif
+
+#ifdef RC_COPYRIGHT
+        VALUE "LegalCopyright", RC_COPYRIGHT
+#endif
+    END
+  END
+
+  BLOCK "VarFileInfo"
+  BEGIN
+    // The translation must correspond to the above BLOCK inside StringFileInfo
+    // langID     0x0409  U.S. English
+    // charsetID  0x04B0  Unicode
+    VALUE "Translation", 0x0409, 0x04B0
+  END
+END
\ No newline at end of file
diff --git a/test/Analysis/BasicAA/cs-cs.ll b/test/Analysis/BasicAA/cs-cs.ll
index 78670b61ca1..427242580ed 100644
--- a/test/Analysis/BasicAA/cs-cs.ll
+++ b/test/Analysis/BasicAA/cs-cs.ll
@@ -233,4 +233,7 @@ define void @test6(i8* %P) nounwind ssp {
 ; CHECK: Just Ref:   call void @a_readonly_func(i8* %P) <->   call void @llvm.memset.p0i8.i64(i8* %P, i8 -51, i64 32, i32 8, i1 false)
 }
 
-attributes #0 = { nounwind }
+attributes #0 = { nounwind readonly }
+attributes #1 = { nounwind }
+attributes #2 = { noinline nounwind readonly }
+attributes #3 = { nounwind ssp }
diff --git a/test/Analysis/BasicAA/modref.ll b/test/Analysis/BasicAA/modref.ll
index e124d6cbe20..3084f809c37 100644
--- a/test/Analysis/BasicAA/modref.ll
+++ b/test/Analysis/BasicAA/modref.ll
@@ -145,6 +145,51 @@ entry:
 ; CHECK: load i32, i32*
 }
 
+;; Check that aa correctly handles functions marked with argmemonly
+;; attribute.
+declare i32 @func_argmemonly(i32 * %P) argmemonly
+
+;; Can not remove redundant load, function may write to it.
+; CHECK-LABEL: @test8(
+define i32 @test8(i32 *%P) {
+  %V1 = load i32, i32* %P
+  call i32 @func_argmemonly(i32* %P)
+  %V2 = load i32, i32* %P
+  %Diff = sub i32 %V1, %V2
+  ret i32 %Diff
+  ; CHECK: load
+  ; CHECK: load
+  ; CHECK: sub
+  ; CHECK: ret i32 %Diff
+}
+
+;; In this case load can be removed, function clobbers only %P2.
+; CHECK-LABEL: @test9(
+define i32 @test9(i32* %P, i32* noalias %P2) {
+  %V1 = load i32, i32* %P
+  call i32 @func_argmemonly(i32* %P2)
+  %V2 = load i32, i32* %P
+  %Diff = sub i32 %V1, %V2
+  ret i32 %Diff
+  ; CHECK-NOT: load
+  ; CHECK: ret i32 0
+}
+
+;; In this case load can *not* be removed. Function clobers only %P2 but it may
+;; alias with %P.
+; CHECK-LABEL: @test10(
+define i32 @test10(i32* %P, i32* %P2) {
+  %V1 = load i32, i32* %P
+  call i32 @func_argmemonly(i32* %P2)
+  %V2 = load i32, i32* %P
+  %Diff = sub i32 %V1, %V2
+  ret i32 %Diff
+  ; CHECK: load
+  ; CHECK: load
+  ; CHECK: sub
+  ; CHECK: ret i32 %Diff
+}
+
 declare void @llvm.memset.p0i8.i32(i8* nocapture, i8, i32, i32, i1) nounwind
 declare void @llvm.memset.p0i8.i8(i8* nocapture, i8, i8, i32, i1) nounwind
 declare void @llvm.memcpy.p0i8.p0i8.i8(i8* nocapture, i8* nocapture, i8, i32, i1) nounwind
diff --git a/test/Analysis/BasicAA/phi-loop.ll b/test/Analysis/BasicAA/phi-loop.ll
new file mode 100644
index 00000000000..6337bfbc1cf
--- /dev/null
+++ b/test/Analysis/BasicAA/phi-loop.ll
@@ -0,0 +1,75 @@
+; RUN: opt < %s -basicaa -basicaa-recphi=1 -gvn -S | FileCheck %s
+;
+; Check that section->word_ofs doesn't get reloaded in every iteration of the
+; for loop.
+;
+; Code:
+;
+; typedef struct {
+;   unsigned num_words;
+;   unsigned word_ofs;
+;   const unsigned *data;
+; } section_t;
+; 
+; 
+; void test2(const section_t * restrict section, unsigned * restrict dst) {;
+;   while (section->data != NULL) {
+;     const unsigned *src = section->data;
+;     for (unsigned i=0; i < section->num_words; ++i) {
+;       dst[section->word_ofs + i] = src[i];
+;     }   
+; 
+;     ++section;
+;   }
+; }
+; 
+
+; CHECK-LABEL: for.body:
+; CHECK-NOT: load i32, i32* %word_ofs
+
+%struct.section_t = type { i32, i32, i32* }
+
+define void @test2(%struct.section_t* noalias nocapture readonly %section, i32* noalias nocapture %dst) {
+entry:
+  %data13 = getelementptr inbounds %struct.section_t, %struct.section_t* %section, i32 0, i32 2
+  %0 = load i32*, i32** %data13, align 4
+  %cmp14 = icmp eq i32* %0, null
+  br i1 %cmp14, label %while.end, label %for.cond.preheader
+
+for.cond.preheader:                               ; preds = %entry, %for.end
+  %1 = phi i32* [ %6, %for.end ], [ %0, %entry ]
+  %section.addr.015 = phi %struct.section_t* [ %incdec.ptr, %for.end ], [ %section, %entry ]
+  %num_words = getelementptr inbounds %struct.section_t, %struct.section_t* %section.addr.015, i32 0, i32 0
+  %2 = load i32, i32* %num_words, align 4
+  %cmp211 = icmp eq i32 %2, 0
+  br i1 %cmp211, label %for.end, label %for.body.lr.ph
+
+for.body.lr.ph:                                   ; preds = %for.cond.preheader
+  %word_ofs = getelementptr inbounds %struct.section_t, %struct.section_t* %section.addr.015, i32 0, i32 1
+  br label %for.body
+
+for.body:                                         ; preds = %for.body.lr.ph, %for.body
+  %arrayidx.phi = phi i32* [ %1, %for.body.lr.ph ], [ %arrayidx.inc, %for.body ]
+  %i.012 = phi i32 [ 0, %for.body.lr.ph ], [ %inc, %for.body ]
+  %3 = load i32, i32* %arrayidx.phi, align 4
+  %4 = load i32, i32* %word_ofs, align 4
+  %add = add i32 %4, %i.012
+  %arrayidx3 = getelementptr inbounds i32, i32* %dst, i32 %add
+  store i32 %3, i32* %arrayidx3, align 4
+  %inc = add i32 %i.012, 1
+  %5 = load i32, i32* %num_words, align 4
+  %cmp2 = icmp ult i32 %inc, %5
+  %arrayidx.inc = getelementptr i32, i32* %arrayidx.phi, i32 1
+  br i1 %cmp2, label %for.body, label %for.end
+
+for.end:                                          ; preds = %for.body, %for.cond.preheader
+  %incdec.ptr = getelementptr inbounds %struct.section_t, %struct.section_t* %section.addr.015, i32 1
+  %data = getelementptr inbounds %struct.section_t, %struct.section_t* %section.addr.015, i32 1, i32 2
+  %6 = load i32*, i32** %data, align 4
+  %cmp = icmp eq i32* %6, null
+  br i1 %cmp, label %while.end, label %for.cond.preheader
+
+while.end:                                        ; preds = %for.end, %entry
+  ret void
+}
+
diff --git a/test/Analysis/BlockFrequencyInfo/irreducible.ll b/test/Analysis/BlockFrequencyInfo/irreducible.ll
index b275aae6279..c1b1c2a7a23 100644
--- a/test/Analysis/BlockFrequencyInfo/irreducible.ll
+++ b/test/Analysis/BlockFrequencyInfo/irreducible.ll
@@ -130,9 +130,6 @@ exit:
 ; At the first step, c1 and c2 each get 1/3 of the entry.  At each subsequent
 ; step, c1 and c2 each get 1/3 of what's left in c1 and c2 combined.  This
 ; infinite series sums to 1.
-;
-; Since the currently algorithm *always* assumes entry blocks are equal,
-; -block-freq gets the right answers here.
 define void @crossloops(i2 %x) {
 ; CHECK-LABEL: Printing analysis {{.*}} for function 'crossloops':
 ; CHECK-NEXT: block-frequency-info: crossloops
@@ -386,7 +383,7 @@ exit:
 ;
 ; This testcases uses non-trivial branch weights.  The CHECK statements here
 ; will start to fail if we change -block-freq to be more accurate.  Currently,
-; we expect left, right and top to be treated as equal headers.
+; loop headers are affected by the weight of their corresponding back edges.
 define void @nonentry_header(i1 %x, i2 %y) {
 ; CHECK-LABEL: Printing analysis {{.*}} for function 'nonentry_header':
 ; CHECK-NEXT: block-frequency-info: nonentry_header
@@ -395,15 +392,15 @@ entry:
   br i1 %x, label %left, label %right, !prof !21
 
 left:
-; CHECK-NEXT: left: float = 3.0,
+; CHECK-NEXT: left: float = 0.14
   br i1 %x, label %top, label %bottom, !prof !22
 
 right:
-; CHECK-NEXT: right: float = 3.0,
+; CHECK-NEXT: right: float = 0.42
   br i1 %x, label %top, label %bottom, !prof !22
 
 top:
-; CHECK-NEXT: top: float = 3.0,
+; CHECK-NEXT: top: float = 8.43
   switch i2 %y, label %exit [ i2 0, label %left
                               i2 1, label %right
                               i2 2, label %bottom ], !prof !23
diff --git a/test/Analysis/BranchProbabilityInfo/pr22718.ll b/test/Analysis/BranchProbabilityInfo/pr22718.ll
new file mode 100644
index 00000000000..0de4d643c9c
--- /dev/null
+++ b/test/Analysis/BranchProbabilityInfo/pr22718.ll
@@ -0,0 +1,84 @@
+; RUN: opt < %s -analyze -branch-prob | FileCheck %s
+
+; In this test, the else clause is taken about 90% of the time. This was not
+; reflected in the probability computation because the weight is larger than
+; the branch weight cap (about 2 billion).
+;
+; CHECK: edge for.body -> if.then probability is 216661881 / 2166666667 = 9.9
+; CHECK: edge for.body -> if.else probability is 1950004786 / 2166666667 = 90.0
+
+@y = common global i64 0, align 8
+@x = common global i64 0, align 8
+@.str = private unnamed_addr constant [17 x i8] c"x = %lu\0Ay = %lu\0A\00", align 1
+
+; Function Attrs: inlinehint nounwind uwtable
+define i32 @main() #0 {
+entry:
+  %retval = alloca i32, align 4
+  %i = alloca i64, align 8
+  store i32 0, i32* %retval
+  store i64 0, i64* @y, align 8
+  store i64 0, i64* @x, align 8
+  call void @srand(i32 422304) #3
+  store i64 0, i64* %i, align 8
+  br label %for.cond
+
+for.cond:                                         ; preds = %for.inc, %entry
+  %0 = load i64, i64* %i, align 8
+  %cmp = icmp ult i64 %0, 13000000000
+  br i1 %cmp, label %for.body, label %for.end, !prof !1
+
+for.body:                                         ; preds = %for.cond
+  %call = call i32 @rand() #3
+  %conv = sitofp i32 %call to double
+  %mul = fmul double %conv, 1.000000e+02
+  %div = fdiv double %mul, 0x41E0000000000000
+  %cmp1 = fcmp ogt double %div, 9.000000e+01
+  br i1 %cmp1, label %if.then, label %if.else, !prof !2
+
+if.then:                                          ; preds = %for.body
+  %1 = load i64, i64* @x, align 8
+  %inc = add i64 %1, 1
+  store i64 %inc, i64* @x, align 8
+  br label %if.end
+
+if.else:                                          ; preds = %for.body
+  %2 = load i64, i64* @y, align 8
+  %inc3 = add i64 %2, 1
+  store i64 %inc3, i64* @y, align 8
+  br label %if.end
+
+if.end:                                           ; preds = %if.else, %if.then
+  br label %for.inc
+
+for.inc:                                          ; preds = %if.end
+  %3 = load i64, i64* %i, align 8
+  %inc4 = add i64 %3, 1
+  store i64 %inc4, i64* %i, align 8
+  br label %for.cond
+
+for.end:                                          ; preds = %for.cond
+  %4 = load i64, i64* @x, align 8
+  %5 = load i64, i64* @y, align 8
+  %call5 = call i32 (i8*, ...) @printf(i8* getelementptr inbounds ([17 x i8], [17 x i8]* @.str, i32 0, i32 0), i64 %4, i64 %5)
+  ret i32 0
+}
+
+; Function Attrs: nounwind
+declare void @srand(i32) #1
+
+; Function Attrs: nounwind
+declare i32 @rand() #1
+
+declare i32 @printf(i8*, ...) #2
+
+attributes #0 = { inlinehint nounwind uwtable "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+sse,+sse2" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #1 = { nounwind "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+sse,+sse2" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #2 = { "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+sse,+sse2" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #3 = { nounwind }
+
+!llvm.ident = !{!0}
+
+!0 = !{!"clang version 3.7.0 (trunk 236218) (llvm/trunk 236235)"}
+!1 = !{!"branch_weights", i32 -1044967295, i32 1}
+!2 = !{!"branch_weights", i32 433323762, i32 -394957723}
diff --git a/test/Analysis/CallGraph/do-nothing-intrinsic.ll b/test/Analysis/CallGraph/do-nothing-intrinsic.ll
index f28ad10f57c..54623717083 100644
--- a/test/Analysis/CallGraph/do-nothing-intrinsic.ll
+++ b/test/Analysis/CallGraph/do-nothing-intrinsic.ll
@@ -1,11 +1,11 @@
 ; RUN: opt < %s -basiccg
 ; PR13903
 
-define void @main() {
+define void @main() personality i8 0 {
   invoke void @llvm.donothing()
           to label %ret unwind label %unw
 unw:
-  %tmp = landingpad i8 personality i8 0 cleanup
+  %tmp = landingpad i8 cleanup
   br label %ret
 ret:
   ret void
diff --git a/test/Analysis/CallGraph/non-leaf-intrinsics.ll b/test/Analysis/CallGraph/non-leaf-intrinsics.ll
new file mode 100644
index 00000000000..11bed6abce6
--- /dev/null
+++ b/test/Analysis/CallGraph/non-leaf-intrinsics.ll
@@ -0,0 +1,32 @@
+; RUN: opt -S -print-callgraph -disable-output < %s 2>&1 | FileCheck %s
+
+declare void @llvm.experimental.patchpoint.void(i64, i32, i8*, i32, ...)
+declare i32 @llvm.experimental.gc.statepoint.p0f_isVoidf(i64, i32, void ()*, i32, i32, ...)
+
+define private void @f() {
+  ret void
+}
+
+define void @calls_statepoint(i8 addrspace(1)* %arg) gc "statepoint-example" {
+entry:
+  %cast = bitcast i8 addrspace(1)* %arg to i64 addrspace(1)*
+  %safepoint_token = call i32 (i64, i32, void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 0, i32 0, void ()* @f, i32 0, i32 0, i32 0, i32 5, i32 0, i32 0, i32 0, i32 10, i32 0, i8 addrspace(1)* %arg, i64 addrspace(1)* %cast, i8 addrspace(1)* %arg, i8 addrspace(1)* %arg)
+  ret void
+}
+
+define void @calls_patchpoint() {
+entry:
+  %c = bitcast void()* @f to i8*
+  tail call void (i64, i32, i8*, i32, ...) @llvm.experimental.patchpoint.void(i64 1, i32 15, i8* %c, i32 0, i16 65535, i16 -1, i32 65536, i32 2000000000, i32 2147483647, i32 -1, i32 4294967295, i32 4294967296, i64 2147483648, i64 4294967295, i64 4294967296, i64 -1)
+  ret void
+}
+
+
+; CHECK: Call graph node <<null function>>
+; CHECK:  CS<0x0> calls function 'f'
+
+; CHECK: Call graph node for function: 'calls_patchpoint'
+; CHECK-NEXT:  CS<[[addr_1:[^>]+]]> calls external node
+
+; CHECK: Call graph node for function: 'calls_statepoint'
+; CHECK-NEXT:  CS<[[addr_0:[^>]+]]> calls external node
diff --git a/test/Analysis/CostModel/ARM/gep.ll b/test/Analysis/CostModel/ARM/gep.ll
index 624ca113a30..a70d6d42b61 100644
--- a/test/Analysis/CostModel/ARM/gep.ll
+++ b/test/Analysis/CostModel/ARM/gep.ll
@@ -3,41 +3,85 @@
 target datalayout = "e-p:32:32:32-i1:8:32-i8:8:32-i16:16:32-i32:32:32-i64:32:64-f32:32:32-f64:32:64-v64:32:64-v128:32:128-a0:0:32-n32-S32"
 target triple = "thumbv7-apple-ios6.0.0"
 
-define void @test_geps() {
-  ; Cost of scalar integer geps should be one. We can't always expect it to be
-  ; folded into the instruction addressing mode.
-;CHECK: cost of 1 for instruction: {{.*}} getelementptr inbounds i8, i8*
+define void @test_geps(i32 %i) {
+  ; GEPs with index 0 are essentially NOOPs.
+;CHECK: cost of 0 for instruction: {{.*}} getelementptr inbounds i8, i8*
   %a0 = getelementptr inbounds i8, i8* undef, i32 0
-;CHECK: cost of 1 for instruction: {{.*}} getelementptr inbounds i16, i16*
+;CHECK: cost of 0 for instruction: {{.*}} getelementptr inbounds i16, i16*
   %a1 = getelementptr inbounds i16, i16* undef, i32 0
-;CHECK: cost of 1 for instruction: {{.*}} getelementptr inbounds i32, i32*
+;CHECK: cost of 0 for instruction: {{.*}} getelementptr inbounds i32, i32*
   %a2 = getelementptr inbounds i32, i32* undef, i32 0
-
-;CHECK: cost of 1 for instruction: {{.*}} getelementptr inbounds i64, i64*
+;CHECK: cost of 0 for instruction: {{.*}} getelementptr inbounds i64, i64*
   %a3 = getelementptr inbounds i64, i64* undef, i32 0
-
-  ; Cost of scalar floating point geps should be one. We cannot fold the address
-  ; computation.
-;CHECK: cost of 1 for instruction: {{.*}} getelementptr inbounds float, float*
+;CHECK: cost of 0 for instruction: {{.*}} getelementptr inbounds float, float*
   %a4 = getelementptr inbounds float, float* undef, i32 0
-;CHECK: cost of 1 for instruction: {{.*}} getelementptr inbounds double, double*
+;CHECK: cost of 0 for instruction: {{.*}} getelementptr inbounds double, double*
   %a5 = getelementptr inbounds double, double* undef, i32 0
-
-
-  ; Cost of vector geps should be one. We cannot fold the address computation.
-;CHECK: cost of 1 for instruction: {{.*}} getelementptr inbounds <4 x i8>, <4 x i8>*
+;CHECK: cost of 0 for instruction: {{.*}} getelementptr inbounds <4 x i8>, <4 x i8>*
   %a7 = getelementptr inbounds <4 x i8>, <4 x i8>* undef, i32 0
-;CHECK: cost of 1 for instruction: {{.*}} getelementptr inbounds <4 x i16>, <4 x i16>*
+;CHECK: cost of 0 for instruction: {{.*}} getelementptr inbounds <4 x i16>, <4 x i16>*
   %a8 = getelementptr inbounds <4 x i16>, <4 x i16>* undef, i32 0
-;CHECK: cost of 1 for instruction: {{.*}} getelementptr inbounds <4 x i32>, <4 x i32>*
+;CHECK: cost of 0 for instruction: {{.*}} getelementptr inbounds <4 x i32>, <4 x i32>*
   %a9 = getelementptr inbounds <4 x i32>, <4 x i32>* undef, i32 0
-;CHECK: cost of 1 for instruction: {{.*}} getelementptr inbounds <4 x i64>, <4 x i64>*
+;CHECK: cost of 0 for instruction: {{.*}} getelementptr inbounds <4 x i64>, <4 x i64>*
   %a10 = getelementptr inbounds <4 x i64>, <4 x i64>* undef, i32 0
-;CHECK: cost of 1 for instruction: {{.*}} getelementptr inbounds <4 x float>, <4 x float>*
+;CHECK: cost of 0 for instruction: {{.*}} getelementptr inbounds <4 x float>, <4 x float>*
   %a11 = getelementptr inbounds <4 x float>, <4 x float>* undef, i32 0
-;CHECK: cost of 1 for instruction: {{.*}} getelementptr inbounds <4 x double>, <4 x double>*
+;CHECK: cost of 0 for instruction: {{.*}} getelementptr inbounds <4 x double>, <4 x double>*
   %a12 = getelementptr inbounds <4 x double>, <4 x double>* undef, i32 0
 
+  ; Cost of GEPs is one if we cannot fold the address computation.
+;CHECK: cost of 0 for instruction: {{.*}} getelementptr inbounds i8, i8*
+  %b0 = getelementptr inbounds i8, i8* undef, i32 1024
+;CHECK: cost of 0 for instruction: {{.*}} getelementptr inbounds i16, i16*
+  %b1 = getelementptr inbounds i16, i16* undef, i32 1024
+  ; Thumb-2 cannot fold offset >= 2^12 into address computation.
+;CHECK: cost of 1 for instruction: {{.*}} getelementptr inbounds i32, i32*
+  %b2 = getelementptr inbounds i32, i32* undef, i32 1024
+;CHECK: cost of 1 for instruction: {{.*}} getelementptr inbounds i64, i64*
+  %b3 = getelementptr inbounds i64, i64* undef, i32 1024
+;CHECK: cost of 1 for instruction: {{.*}} getelementptr inbounds float, float*
+  %b4 = getelementptr inbounds float, float* undef, i32 1024
+;CHECK: cost of 1 for instruction: {{.*}} getelementptr inbounds double, double*
+  %b5 = getelementptr inbounds double, double* undef, i32 1024
+;CHECK: cost of 0 for instruction: {{.*}} getelementptr inbounds <4 x i8>, <4 x i8>*
+  %b7 = getelementptr inbounds <4 x i8>, <4 x i8>* undef, i32 1
+;CHECK: cost of 0 for instruction: {{.*}} getelementptr inbounds <4 x i16>, <4 x i16>*
+  %b8 = getelementptr inbounds <4 x i16>, <4 x i16>* undef, i32 1
+;CHECK: cost of 0 for instruction: {{.*}} getelementptr inbounds <4 x i32>, <4 x i32>*
+  %b9 = getelementptr inbounds <4 x i32>, <4 x i32>* undef, i32 1
+;CHECK: cost of 0 for instruction: {{.*}} getelementptr inbounds <4 x i64>, <4 x i64>*
+  %b10 = getelementptr inbounds <4 x i64>, <4 x i64>* undef, i32 1
+;CHECK: cost of 0 for instruction: {{.*}} getelementptr inbounds <4 x float>, <4 x float>*
+  %b11 = getelementptr inbounds <4 x float>, <4 x float>* undef, i32 1
+;CHECK: cost of 0 for instruction: {{.*}} getelementptr inbounds <4 x double>, <4 x double>*
+  %b12 = getelementptr inbounds <4 x double>, <4 x double>* undef, i32 1
+
+;CHECK: cost of 0 for instruction: {{.*}} getelementptr inbounds i8, i8*
+  %c0 = getelementptr inbounds i8, i8* undef, i32 %i
+;CHECK: cost of 0 for instruction: {{.*}} getelementptr inbounds i16, i16*
+  %c1 = getelementptr inbounds i16, i16* undef, i32 %i
+;CHECK: cost of 0 for instruction: {{.*}} getelementptr inbounds i32, i32*
+  %c2 = getelementptr inbounds i32, i32* undef, i32 %i
+;CHECK: cost of 0 for instruction: {{.*}} getelementptr inbounds i64, i64*
+  %c3 = getelementptr inbounds i64, i64* undef, i32 %i
+;CHECK: cost of 0 for instruction: {{.*}} getelementptr inbounds float, float*
+  %c4 = getelementptr inbounds float, float* undef, i32 %i
+;CHECK: cost of 0 for instruction: {{.*}} getelementptr inbounds double, double*
+  %c5 = getelementptr inbounds double, double* undef, i32 %i
+;CHECK: cost of 0 for instruction: {{.*}} getelementptr inbounds <4 x i8>, <4 x i8>*
+  %c7 = getelementptr inbounds <4 x i8>, <4 x i8>* undef, i32 %i
+;CHECK: cost of 0 for instruction: {{.*}} getelementptr inbounds <4 x i16>, <4 x i16>*
+  %c8 = getelementptr inbounds <4 x i16>, <4 x i16>* undef, i32 %i
+  ; Thumb-2 cannot fold scales larger than 8 to address computation.
+;CHECK: cost of 1 for instruction: {{.*}} getelementptr inbounds <4 x i32>, <4 x i32>*
+  %c9 = getelementptr inbounds <4 x i32>, <4 x i32>* undef, i32 %i
+;CHECK: cost of 1 for instruction: {{.*}} getelementptr inbounds <4 x i64>, <4 x i64>*
+  %c10 = getelementptr inbounds <4 x i64>, <4 x i64>* undef, i32 %i
+;CHECK: cost of 1 for instruction: {{.*}} getelementptr inbounds <4 x float>, <4 x float>*
+  %c11 = getelementptr inbounds <4 x float>, <4 x float>* undef, i32 %i
+;CHECK: cost of 1 for instruction: {{.*}} getelementptr inbounds <4 x double>, <4 x double>*
+  %c12 = getelementptr inbounds <4 x double>, <4 x double>* undef, i32 %i
 
   ret void
 }
diff --git a/test/Analysis/CostModel/X86/arith.ll b/test/Analysis/CostModel/X86/arith.ll
index 92f5a1ec3a0..0b61d3cd421 100644
--- a/test/Analysis/CostModel/X86/arith.ll
+++ b/test/Analysis/CostModel/X86/arith.ll
@@ -94,7 +94,7 @@ define void @shift() {
   ; AVX2: cost of 1 {{.*}} ashr
   %C0 = ashr <4 x i32> undef, undef
   ; AVX: cost of 6 {{.*}} ashr
-  ; AVX2: cost of 20 {{.*}} ashr
+  ; AVX2: cost of 4 {{.*}} ashr
   %C1 = ashr <2 x i64> undef, undef
 
   ret void
@@ -121,7 +121,7 @@ define void @avx2shift() {
   ; AVX2: cost of 1 {{.*}} ashr
   %C0 = ashr <8 x i32> undef, undef
   ; AVX: cost of 12 {{.*}} ashr
-  ; AVX2: cost of 40 {{.*}} ashr
+  ; AVX2: cost of 4 {{.*}} ashr
   %C1 = ashr <4 x i64> undef, undef
 
   ret void
diff --git a/test/Analysis/CostModel/X86/sitofp.ll b/test/Analysis/CostModel/X86/sitofp.ll
index edc937ecf94..dcd0088d0df 100644
--- a/test/Analysis/CostModel/X86/sitofp.ll
+++ b/test/Analysis/CostModel/X86/sitofp.ll
@@ -1,9 +1,20 @@
-; RUN: opt -mtriple=x86_64-apple-darwin -mcpu=core2 -cost-model -analyze < %s | FileCheck --check-prefix=SSE2 %s
-; RUN: opt -mtriple=x86_64-apple-darwin -mcpu=knl -cost-model -analyze < %s | FileCheck --check-prefix=AVX512F %s
+; RUN: opt -mtriple=x86_64-apple-darwin -mattr=+sse2 -cost-model -analyze < %s | FileCheck --check-prefix=SSE --check-prefix=SSE2 %s
+; RUN: opt -mtriple=x86_64-apple-darwin -mattr=+avx  -cost-model -analyze < %s | FileCheck --check-prefix=AVX --check-prefix=AVX1 %s
+; RUN: opt -mtriple=x86_64-apple-darwin -mattr=+avx2 -cost-model -analyze < %s | FileCheck --check-prefix=AVX --check-prefix=AVX2 %s
+; RUN: opt -mtriple=x86_64-apple-darwin -mattr=+avx512f -cost-model -analyze < %s | FileCheck --check-prefix=AVX512F %s
 
 define <2 x double> @sitofpv2i8v2double(<2 x i8> %a) {
   ; SSE2: sitofpv2i8v2double
   ; SSE2: cost of 20 {{.*}} sitofp
+  ;
+  ; AVX1: sitofpv2i8v2double
+  ; AVX1: cost of 4 {{.*}} sitofp
+  ;
+  ; AVX2: sitofpv2i8v2double
+  ; AVX2: cost of 4 {{.*}} sitofp
+  ;
+  ; AVX512F: sitofpv2i8v2double
+  ; AVX512F: cost of 4 {{.*}} sitofp
   %1 = sitofp <2 x i8> %a to <2 x double>
   ret <2 x double> %1
 }
@@ -11,6 +22,15 @@ define <2 x double> @sitofpv2i8v2double(<2 x i8> %a) {
 define <4 x double> @sitofpv4i8v4double(<4 x i8> %a) {
   ; SSE2: sitofpv4i8v4double
   ; SSE2: cost of 40 {{.*}} sitofp
+  ;
+  ; AVX1: sitofpv4i8v4double
+  ; AVX1: cost of 3 {{.*}} sitofp
+  ;
+  ; AVX2: sitofpv4i8v4double
+  ; AVX2: cost of 3 {{.*}} sitofp
+  ;
+  ; AVX512F: sitofpv4i8v4double
+  ; AVX512F: cost of 3 {{.*}} sitofp
   %1 = sitofp <4 x i8> %a to <4 x double>
   ret <4 x double> %1
 }
@@ -18,13 +38,31 @@ define <4 x double> @sitofpv4i8v4double(<4 x i8> %a) {
 define <8 x double> @sitofpv8i8v8double(<8 x i8> %a) {
   ; SSE2: sitofpv8i8v8double
   ; SSE2: cost of 80 {{.*}} sitofp
-%1 = sitofp <8 x i8> %a to <8 x double>
+  ;
+  ; AVX1: sitofpv8i8v8double
+  ; AVX1: cost of 20 {{.*}} sitofp
+  ;
+  ; AVX2: sitofpv8i8v8double
+  ; AVX2: cost of 20 {{.*}} sitofp
+  ;
+  ; AVX512F: sitofpv8i8v8double
+  ; AVX512F: cost of 2 {{.*}} sitofp
+  %1 = sitofp <8 x i8> %a to <8 x double>
   ret <8 x double> %1
 }
 
 define <16 x double> @sitofpv16i8v16double(<16 x i8> %a) {
   ; SSE2: sitofpv16i8v16double
   ; SSE2: cost of 160 {{.*}} sitofp
+  ;
+  ; AVX1: sitofpv16i8v16double
+  ; AVX1: cost of 40 {{.*}} sitofp
+  ;
+  ; AVX2: sitofpv16i8v16double
+  ; AVX2: cost of 40 {{.*}} sitofp
+  ;
+  ; AVX512F: sitofpv16i8v16double
+  ; AVX512F: cost of 44 {{.*}} sitofp
   %1 = sitofp <16 x i8> %a to <16 x double>
   ret <16 x double> %1
 }
@@ -32,6 +70,15 @@ define <16 x double> @sitofpv16i8v16double(<16 x i8> %a) {
 define <32 x double> @sitofpv32i8v32double(<32 x i8> %a) {
   ; SSE2: sitofpv32i8v32double
   ; SSE2: cost of 320 {{.*}} sitofp
+  ;
+  ; AVX1: sitofpv32i8v32double
+  ; AVX1: cost of 80 {{.*}} sitofp
+  ;
+  ; AVX2: sitofpv32i8v32double
+  ; AVX2: cost of 80 {{.*}} sitofp
+  ;
+  ; AVX512F: sitofpv32i8v32double
+  ; AVX512F: cost of 88 {{.*}} sitofp
   %1 = sitofp <32 x i8> %a to <32 x double>
   ret <32 x double> %1
 }
@@ -39,6 +86,15 @@ define <32 x double> @sitofpv32i8v32double(<32 x i8> %a) {
 define <2 x double> @sitofpv2i16v2double(<2 x i16> %a) {
   ; SSE2: sitofpv2i16v2double
   ; SSE2: cost of 20 {{.*}} sitofp
+  ;
+  ; AVX1: sitofpv2i16v2double
+  ; AVX1: cost of 4 {{.*}} sitofp
+  ;
+  ; AVX2: sitofpv2i16v2double
+  ; AVX2: cost of 4 {{.*}} sitofp
+  ;
+  ; AVX512F: sitofpv2i16v2double
+  ; AVX512F: cost of 4 {{.*}} sitofp
   %1 = sitofp <2 x i16> %a to <2 x double>
   ret <2 x double> %1
 }
@@ -46,6 +102,15 @@ define <2 x double> @sitofpv2i16v2double(<2 x i16> %a) {
 define <4 x double> @sitofpv4i16v4double(<4 x i16> %a) {
   ; SSE2: sitofpv4i16v4double
   ; SSE2: cost of 40 {{.*}} sitofp
+  ;
+  ; AVX1: sitofpv4i16v4double
+  ; AVX1: cost of 3 {{.*}} sitofp
+  ;
+  ; AVX2: sitofpv4i16v4double
+  ; AVX2: cost of 3 {{.*}} sitofp
+  ;
+  ; AVX512F: sitofpv4i16v4double
+  ; AVX512F: cost of 3 {{.*}} sitofp
   %1 = sitofp <4 x i16> %a to <4 x double>
   ret <4 x double> %1
 }
@@ -53,6 +118,15 @@ define <4 x double> @sitofpv4i16v4double(<4 x i16> %a) {
 define <8 x double> @sitofpv8i16v8double(<8 x i16> %a) {
   ; SSE2: sitofpv8i16v8double
   ; SSE2: cost of 80 {{.*}} sitofp
+  ;
+  ; AVX1: sitofpv8i16v8double
+  ; AVX1: cost of 20 {{.*}} sitofp
+  ;
+  ; AVX2: sitofpv8i16v8double
+  ; AVX2: cost of 20 {{.*}} sitofp
+  ;
+  ; AVX512F: sitofpv8i16v8double
+  ; AVX512F: cost of 2 {{.*}} sitofp
   %1 = sitofp <8 x i16> %a to <8 x double>
   ret <8 x double> %1
 }
@@ -60,6 +134,15 @@ define <8 x double> @sitofpv8i16v8double(<8 x i16> %a) {
 define <16 x double> @sitofpv16i16v16double(<16 x i16> %a) {
   ; SSE2: sitofpv16i16v16double
   ; SSE2: cost of 160 {{.*}} sitofp
+  ;
+  ; AVX1: sitofpv16i16v16double
+  ; AVX1: cost of 40 {{.*}} sitofp
+  ;
+  ; AVX2: sitofpv16i16v16double
+  ; AVX2: cost of 40 {{.*}} sitofp
+  ;
+  ; AVX512F: sitofpv16i16v16double
+  ; AVX512F: cost of 44 {{.*}} sitofp
   %1 = sitofp <16 x i16> %a to <16 x double>
   ret <16 x double> %1
 }
@@ -67,6 +150,15 @@ define <16 x double> @sitofpv16i16v16double(<16 x i16> %a) {
 define <32 x double> @sitofpv32i16v32double(<32 x i16> %a) {
   ; SSE2: sitofpv32i16v32double
   ; SSE2: cost of 320 {{.*}} sitofp
+  ;
+  ; AVX1: sitofpv32i16v32double
+  ; AVX1: cost of 80 {{.*}} sitofp
+  ;
+  ; AVX2: sitofpv32i16v32double
+  ; AVX2: cost of 80 {{.*}} sitofp
+  ;
+  ; AVX512F: sitofpv32i16v32double
+  ; AVX512F: cost of 88 {{.*}} sitofp
   %1 = sitofp <32 x i16> %a to <32 x double>
   ret <32 x double> %1
 }
@@ -74,6 +166,15 @@ define <32 x double> @sitofpv32i16v32double(<32 x i16> %a) {
 define <2 x double> @sitofpv2i32v2double(<2 x i32> %a) {
   ; SSE2: sitofpv2i32v2double
   ; SSE2: cost of 20 {{.*}} sitofp
+  ;
+  ; AVX1: sitofpv2i32v2double
+  ; AVX1: cost of 4 {{.*}} sitofp
+  ;
+  ; AVX2: sitofpv2i32v2double
+  ; AVX2: cost of 4 {{.*}} sitofp
+  ;
+  ; AVX512F: sitofpv2i32v2double
+  ; AVX512F: cost of 4 {{.*}} sitofp
   %1 = sitofp <2 x i32> %a to <2 x double>
   ret <2 x double> %1
 }
@@ -81,6 +182,15 @@ define <2 x double> @sitofpv2i32v2double(<2 x i32> %a) {
 define <4 x double> @sitofpv4i32v4double(<4 x i32> %a) {
   ; SSE2: sitofpv4i32v4double
   ; SSE2: cost of 40 {{.*}} sitofp
+  ;
+  ; AVX1: sitofpv4i32v4double
+  ; AVX1: cost of 1 {{.*}} sitofp
+  ;
+  ; AVX2: sitofpv4i32v4double
+  ; AVX2: cost of 1 {{.*}} sitofp
+  ;
+  ; AVX512F: sitofpv4i32v4double
+  ; AVX512F: cost of 1 {{.*}} sitofp
   %1 = sitofp <4 x i32> %a to <4 x double>
   ret <4 x double> %1
 }
@@ -88,6 +198,15 @@ define <4 x double> @sitofpv4i32v4double(<4 x i32> %a) {
 define <8 x double> @sitofpv8i32v8double(<8 x i32> %a) {
   ; SSE2: sitofpv8i32v8double
   ; SSE2: cost of 80 {{.*}} sitofp
+  ;
+  ; AVX1: sitofpv8i32v8double
+  ; AVX1: cost of 20 {{.*}} sitofp
+  ;
+  ; AVX2: sitofpv8i32v8double
+  ; AVX2: cost of 20 {{.*}} sitofp
+  ;
+  ; AVX512F: sitofpv8i32v8double
+  ; AVX512F: cost of 1 {{.*}} sitofp
   %1 = sitofp <8 x i32> %a to <8 x double>
   ret <8 x double> %1
 }
@@ -95,6 +214,15 @@ define <8 x double> @sitofpv8i32v8double(<8 x i32> %a) {
 define <16 x double> @sitofpv16i32v16double(<16 x i32> %a) {
   ; SSE2: sitofpv16i32v16double
   ; SSE2: cost of 160 {{.*}} sitofp
+  ;
+  ; AVX1: sitofpv16i32v16double
+  ; AVX1: cost of 40 {{.*}} sitofp
+  ;
+  ; AVX2: sitofpv16i32v16double
+  ; AVX2: cost of 40 {{.*}} sitofp
+  ;
+  ; AVX512F: sitofpv16i32v16double
+  ; AVX512F: cost of 44 {{.*}} sitofp
   %1 = sitofp <16 x i32> %a to <16 x double>
   ret <16 x double> %1
 }
@@ -102,6 +230,15 @@ define <16 x double> @sitofpv16i32v16double(<16 x i32> %a) {
 define <32 x double> @sitofpv32i32v32double(<32 x i32> %a) {
   ; SSE2: sitofpv32i32v32double
   ; SSE2: cost of 320 {{.*}} sitofp
+  ;
+  ; AVX1: sitofpv32i32v32double
+  ; AVX1: cost of 80 {{.*}} sitofp
+  ;
+  ; AVX2: sitofpv32i32v32double
+  ; AVX2: cost of 80 {{.*}} sitofp
+  ;
+  ; AVX512F: sitofpv32i32v32double
+  ; AVX512F: cost of 88 {{.*}} sitofp
   %1 = sitofp <32 x i32> %a to <32 x double>
   ret <32 x double> %1
 }
@@ -109,6 +246,15 @@ define <32 x double> @sitofpv32i32v32double(<32 x i32> %a) {
 define <2 x double> @sitofpv2i64v2double(<2 x i64> %a) {
   ; SSE2: sitofpv2i64v2double
   ; SSE2: cost of 20 {{.*}} sitofp
+  ;
+  ; AVX1: sitofpv2i64v2double
+  ; AVX1: cost of 4 {{.*}} sitofp
+  ;
+  ; AVX2: sitofpv2i64v2double
+  ; AVX2: cost of 4 {{.*}} sitofp
+  ;
+  ; AVX512F: sitofpv2i64v2double
+  ; AVX512F: cost of 4 {{.*}} sitofp
   %1 = sitofp <2 x i64> %a to <2 x double>
   ret <2 x double> %1
 }
@@ -116,20 +262,47 @@ define <2 x double> @sitofpv2i64v2double(<2 x i64> %a) {
 define <4 x double> @sitofpv4i64v4double(<4 x i64> %a) {
   ; SSE2: sitofpv4i64v4double
   ; SSE2: cost of 40 {{.*}} sitofp
+  ;
+  ; AVX1: sitofpv4i64v4double
+  ; AVX1: cost of 10 {{.*}} sitofp
+  ;
+  ; AVX2: sitofpv4i64v4double
+  ; AVX2: cost of 10 {{.*}} sitofp
+  ;
+  ; AVX512F: sitofpv4i64v4double
+  ; AVX512F: cost of 10 {{.*}} sitofp
   %1 = sitofp <4 x i64> %a to <4 x double>
   ret <4 x double> %1
 }
 
 define <8 x double> @sitofpv8i64v8double(<8 x i64> %a) {
-  %1 = sitofp <8 x i64> %a to <8 x double>
   ; SSE2: sitofpv8i64v8double
   ; SSE2: cost of 80 {{.*}} sitofp
+  ;
+  ; AVX1: sitofpv8i64v8double
+  ; AVX1: cost of 20 {{.*}} sitofp
+  ;
+  ; AVX2: sitofpv8i64v8double
+  ; AVX2: cost of 20 {{.*}} sitofp
+  ;
+  ; AVX512F: sitofpv8i64v8double
+  ; AVX512F: cost of 22 {{.*}} sitofp
+  %1 = sitofp <8 x i64> %a to <8 x double>
   ret <8 x double> %1
 }
 
 define <16 x double> @sitofpv16i64v16double(<16 x i64> %a) {
   ; SSE2: sitofpv16i64v16double
   ; SSE2: cost of 160 {{.*}} sitofp
+  ;
+  ; AVX1: sitofpv16i64v16double
+  ; AVX1: cost of 40 {{.*}} sitofp
+  ;
+  ; AVX2: sitofpv16i64v16double
+  ; AVX2: cost of 40 {{.*}} sitofp
+  ;
+  ; AVX512F: sitofpv16i64v16double
+  ; AVX512F: cost of 44 {{.*}} sitofp
   %1 = sitofp <16 x i64> %a to <16 x double>
   ret <16 x double> %1
 }
@@ -137,6 +310,15 @@ define <16 x double> @sitofpv16i64v16double(<16 x i64> %a) {
 define <32 x double> @sitofpv32i64v32double(<32 x i64> %a) {
   ; SSE2: sitofpv32i64v32double
   ; SSE2: cost of 320 {{.*}} sitofp
+  ;
+  ; AVX1: sitofpv32i64v32double
+  ; AVX1: cost of 80 {{.*}} sitofp
+  ;
+  ; AVX2: sitofpv32i64v32double
+  ; AVX2: cost of 80 {{.*}} sitofp
+  ;
+  ; AVX512F: sitofpv32i64v32double
+  ; AVX512F: cost of 88 {{.*}} sitofp
   %1 = sitofp <32 x i64> %a to <32 x double>
   ret <32 x double> %1
 }
@@ -144,6 +326,15 @@ define <32 x double> @sitofpv32i64v32double(<32 x i64> %a) {
 define <2 x float> @sitofpv2i8v2float(<2 x i8> %a) {
   ; SSE2: sitofpv2i8v2float
   ; SSE2: cost of 15 {{.*}} sitofp
+  ;
+  ; AVX1: sitofpv2i8v2float
+  ; AVX1: cost of 4 {{.*}} sitofp
+  ;
+  ; AVX2: sitofpv2i8v2float
+  ; AVX2: cost of 4 {{.*}} sitofp
+  ;
+  ; AVX512F: sitofpv2i8v2float
+  ; AVX512F: cost of 4 {{.*}} sitofp
   %1 = sitofp <2 x i8> %a to <2 x float>
   ret <2 x float> %1
 }
@@ -151,6 +342,15 @@ define <2 x float> @sitofpv2i8v2float(<2 x i8> %a) {
 define <4 x float> @sitofpv4i8v4float(<4 x i8> %a) {
   ; SSE2: sitofpv4i8v4float
   ; SSE2: cost of 15 {{.*}} sitofp
+  ;
+  ; AVX1: sitofpv4i8v4float
+  ; AVX1: cost of 3 {{.*}} sitofp
+  ;
+  ; AVX2: sitofpv4i8v4float
+  ; AVX2: cost of 3 {{.*}} sitofp
+  ;
+  ; AVX512F: sitofpv4i8v4float
+  ; AVX512F: cost of 3 {{.*}} sitofp
   %1 = sitofp <4 x i8> %a to <4 x float>
   ret <4 x float> %1
 }
@@ -158,6 +358,15 @@ define <4 x float> @sitofpv4i8v4float(<4 x i8> %a) {
 define <8 x float> @sitofpv8i8v8float(<8 x i8> %a) {
   ; SSE2: sitofpv8i8v8float
   ; SSE2: cost of 15 {{.*}} sitofp
+  ;
+  ; AVX1: sitofpv8i8v8float
+  ; AVX1: cost of 8 {{.*}} sitofp
+  ;
+  ; AVX2: sitofpv8i8v8float
+  ; AVX2: cost of 8 {{.*}} sitofp
+  ;
+  ; AVX512F: sitofpv8i8v8float
+  ; AVX512F: cost of 8 {{.*}} sitofp
   %1 = sitofp <8 x i8> %a to <8 x float>
   ret <8 x float> %1
 }
@@ -165,6 +374,15 @@ define <8 x float> @sitofpv8i8v8float(<8 x i8> %a) {
 define <16 x float> @sitofpv16i8v16float(<16 x i8> %a) {
   ; SSE2: sitofpv16i8v16float
   ; SSE2: cost of 8 {{.*}} sitofp
+  ;
+  ; AVX1: sitofpv16i8v16float
+  ; AVX1: cost of 44 {{.*}} sitofp
+  ;
+  ; AVX2: sitofpv16i8v16float
+  ; AVX2: cost of 44 {{.*}} sitofp
+  ;
+  ; AVX512F: sitofpv16i8v16float
+  ; AVX512F: cost of 2 {{.*}} sitofp
   %1 = sitofp <16 x i8> %a to <16 x float>
   ret <16 x float> %1
 }
@@ -172,6 +390,15 @@ define <16 x float> @sitofpv16i8v16float(<16 x i8> %a) {
 define <32 x float> @sitofpv32i8v32float(<32 x i8> %a) {
   ; SSE2: sitofpv32i8v32float
   ; SSE2: cost of 16 {{.*}} sitofp
+  ;
+  ; AVX1: sitofpv32i8v32float
+  ; AVX1: cost of 88 {{.*}} sitofp
+  ;
+  ; AVX2: sitofpv32i8v32float
+  ; AVX2: cost of 88 {{.*}} sitofp
+  ;
+  ; AVX512F: sitofpv32i8v32float
+  ; AVX512F: cost of 92 {{.*}} sitofp
   %1 = sitofp <32 x i8> %a to <32 x float>
   ret <32 x float> %1
 }
@@ -179,6 +406,15 @@ define <32 x float> @sitofpv32i8v32float(<32 x i8> %a) {
 define <2 x float> @sitofpv2i16v2float(<2 x i16> %a) {
   ; SSE2: sitofpv2i16v2float
   ; SSE2: cost of 15 {{.*}} sitofp
+  ;
+  ; AVX1: sitofpv2i16v2float
+  ; AVX1: cost of 4 {{.*}} sitofp
+  ;
+  ; AVX2: sitofpv2i16v2float
+  ; AVX2: cost of 4 {{.*}} sitofp
+  ;
+  ; AVX512F: sitofpv2i16v2float
+  ; AVX512F: cost of 4 {{.*}} sitofp
   %1 = sitofp <2 x i16> %a to <2 x float>
   ret <2 x float> %1
 }
@@ -186,6 +422,15 @@ define <2 x float> @sitofpv2i16v2float(<2 x i16> %a) {
 define <4 x float> @sitofpv4i16v4float(<4 x i16> %a) {
   ; SSE2: sitofpv4i16v4float
   ; SSE2: cost of 15 {{.*}} sitofp
+  ;
+  ; AVX1: sitofpv4i16v4float
+  ; AVX1: cost of 3 {{.*}} sitofp
+  ;
+  ; AVX2: sitofpv4i16v4float
+  ; AVX2: cost of 3 {{.*}} sitofp
+  ;
+  ; AVX512F: sitofpv4i16v4float
+  ; AVX512F: cost of 3 {{.*}} sitofp
   %1 = sitofp <4 x i16> %a to <4 x float>
   ret <4 x float> %1
 }
@@ -193,6 +438,15 @@ define <4 x float> @sitofpv4i16v4float(<4 x i16> %a) {
 define <8 x float> @sitofpv8i16v8float(<8 x i16> %a) {
   ; SSE2: sitofpv8i16v8float
   ; SSE2: cost of 15 {{.*}} sitofp
+  ;
+  ; AVX1: sitofpv8i16v8float
+  ; AVX1: cost of 5 {{.*}} sitofp
+  ;
+  ; AVX2: sitofpv8i16v8float
+  ; AVX2: cost of 5 {{.*}} sitofp
+  ;
+  ; AVX512F: sitofpv8i16v8float
+  ; AVX512F: cost of 5 {{.*}} sitofp
   %1 = sitofp <8 x i16> %a to <8 x float>
   ret <8 x float> %1
 }
@@ -200,6 +454,15 @@ define <8 x float> @sitofpv8i16v8float(<8 x i16> %a) {
 define <16 x float> @sitofpv16i16v16float(<16 x i16> %a) {
   ; SSE2: sitofpv16i16v16float
   ; SSE2: cost of 30 {{.*}} sitofp
+  ;
+  ; AVX1: sitofpv16i16v16float
+  ; AVX1: cost of 44 {{.*}} sitofp
+  ;
+  ; AVX2: sitofpv16i16v16float
+  ; AVX2: cost of 44 {{.*}} sitofp
+  ;
+  ; AVX512F: sitofpv16i16v16float
+  ; AVX512F: cost of 2 {{.*}} sitofp
   %1 = sitofp <16 x i16> %a to <16 x float>
   ret <16 x float> %1
 }
@@ -207,6 +470,15 @@ define <16 x float> @sitofpv16i16v16float(<16 x i16> %a) {
 define <32 x float> @sitofpv32i16v32float(<32 x i16> %a) {
   ; SSE2: sitofpv32i16v32float
   ; SSE2: cost of 60 {{.*}} sitofp
+  ;
+  ; AVX1: sitofpv32i16v32float
+  ; AVX1: cost of 88 {{.*}} sitofp
+  ;
+  ; AVX2: sitofpv32i16v32float
+  ; AVX2: cost of 88 {{.*}} sitofp
+  ;
+  ; AVX512F: sitofpv32i16v32float
+  ; AVX512F: cost of 2 {{.*}} sitofp
   %1 = sitofp <32 x i16> %a to <32 x float>
   ret <32 x float> %1
 }
@@ -214,6 +486,15 @@ define <32 x float> @sitofpv32i16v32float(<32 x i16> %a) {
 define <2 x float> @sitofpv2i32v2float(<2 x i32> %a) {
   ; SSE2: sitofpv2i32v2float
   ; SSE2: cost of 15 {{.*}} sitofp
+  ;
+  ; AVX1: sitofpv2i32v2float
+  ; AVX1: cost of 4 {{.*}} sitofp
+  ;
+  ; AVX2: sitofpv2i32v2float
+  ; AVX2: cost of 4 {{.*}} sitofp
+  ;
+  ; AVX512F: sitofpv2i32v2float
+  ; AVX512F: cost of 4 {{.*}} sitofp
   %1 = sitofp <2 x i32> %a to <2 x float>
   ret <2 x float> %1
 }
@@ -221,6 +502,15 @@ define <2 x float> @sitofpv2i32v2float(<2 x i32> %a) {
 define <4 x float> @sitofpv4i32v4float(<4 x i32> %a) {
   ; SSE2: sitofpv4i32v4float
   ; SSE2: cost of 15 {{.*}} sitofp
+  ;
+  ; AVX1: sitofpv4i32v4float
+  ; AVX1: cost of 1 {{.*}} sitofp
+  ;
+  ; AVX2: sitofpv4i32v4float
+  ; AVX2: cost of 1 {{.*}} sitofp
+  ;
+  ; AVX512F: sitofpv4i32v4float
+  ; AVX512F: cost of 1 {{.*}} sitofp
   %1 = sitofp <4 x i32> %a to <4 x float>
   ret <4 x float> %1
 }
@@ -228,6 +518,15 @@ define <4 x float> @sitofpv4i32v4float(<4 x i32> %a) {
 define <8 x float> @sitofpv8i32v8float(<8 x i32> %a) {
   ; SSE2: sitofpv8i32v8float
   ; SSE2: cost of 30 {{.*}} sitofp
+  ;
+  ; AVX1: sitofpv8i32v8float
+  ; AVX1: cost of 1 {{.*}} sitofp
+  ;
+  ; AVX2: sitofpv8i32v8float
+  ; AVX2: cost of 1 {{.*}} sitofp
+  ;
+  ; AVX512F: sitofpv8i32v8float
+  ; AVX512F: cost of 1 {{.*}} sitofp
   %1 = sitofp <8 x i32> %a to <8 x float>
   ret <8 x float> %1
 }
@@ -235,6 +534,15 @@ define <8 x float> @sitofpv8i32v8float(<8 x i32> %a) {
 define <16 x float> @sitofpv16i32v16float(<16 x i32> %a) {
   ; SSE2: sitofpv16i32v16float
   ; SSE2: cost of 60 {{.*}} sitofp
+  ;
+  ; AVX1: sitofpv16i32v16float
+  ; AVX1: cost of 44 {{.*}} sitofp
+  ;
+  ; AVX2: sitofpv16i32v16float
+  ; AVX2: cost of 44 {{.*}} sitofp
+  ;
+  ; AVX512F: sitofpv16i32v16float
+  ; AVX512F: cost of 1 {{.*}} sitofp
   %1 = sitofp <16 x i32> %a to <16 x float>
   ret <16 x float> %1
 }
@@ -242,6 +550,15 @@ define <16 x float> @sitofpv16i32v16float(<16 x i32> %a) {
 define <32 x float> @sitofpv32i32v32float(<32 x i32> %a) {
   ; SSE2: sitofpv32i32v32float
   ; SSE2: cost of 120 {{.*}} sitofp
+  ;
+  ; AVX1: sitofpv32i32v32float
+  ; AVX1: cost of 88 {{.*}} sitofp
+  ;
+  ; AVX2: sitofpv32i32v32float
+  ; AVX2: cost of 88 {{.*}} sitofp
+  ;
+  ; AVX512F: sitofpv32i32v32float
+  ; AVX512F: cost of 1 {{.*}} sitofp
   %1 = sitofp <32 x i32> %a to <32 x float>
   ret <32 x float> %1
 }
@@ -249,6 +566,15 @@ define <32 x float> @sitofpv32i32v32float(<32 x i32> %a) {
 define <2 x float> @sitofpv2i64v2float(<2 x i64> %a) {
   ; SSE2: sitofpv2i64v2float
   ; SSE2: cost of 15 {{.*}} sitofp
+  ;
+  ; AVX1: sitofpv2i64v2float
+  ; AVX1: cost of 4 {{.*}} sitofp
+  ;
+  ; AVX2: sitofpv2i64v2float
+  ; AVX2: cost of 4 {{.*}} sitofp
+  ;
+  ; AVX512F: sitofpv2i64v2float
+  ; AVX512F: cost of 4 {{.*}} sitofp
   %1 = sitofp <2 x i64> %a to <2 x float>
   ret <2 x float> %1
 }
@@ -256,6 +582,15 @@ define <2 x float> @sitofpv2i64v2float(<2 x i64> %a) {
 define <4 x float> @sitofpv4i64v4float(<4 x i64> %a) {
   ; SSE2: sitofpv4i64v4float
   ; SSE2: cost of 30 {{.*}} sitofp
+  ;
+  ; AVX1: sitofpv4i64v4float
+  ; AVX1: cost of 10 {{.*}} sitofp
+  ;
+  ; AVX2: sitofpv4i64v4float
+  ; AVX2: cost of 10 {{.*}} sitofp
+  ;
+  ; AVX512F: sitofpv4i64v4float
+  ; AVX512F: cost of 10 {{.*}} sitofp
   %1 = sitofp <4 x i64> %a to <4 x float>
   ret <4 x float> %1
 }
@@ -263,6 +598,15 @@ define <4 x float> @sitofpv4i64v4float(<4 x i64> %a) {
 define <8 x float> @sitofpv8i64v8float(<8 x i64> %a) {
   ; SSE2: sitofpv8i64v8float
   ; SSE2: cost of 60 {{.*}} sitofp
+  ;
+  ; AVX1: sitofpv8i64v8float
+  ; AVX1: cost of 22 {{.*}} sitofp
+  ;
+  ; AVX2: sitofpv8i64v8float
+  ; AVX2: cost of 22 {{.*}} sitofp
+  ;
+  ; AVX512F: sitofpv8i64v8float
+  ; AVX512F: cost of 22 {{.*}} sitofp
   %1 = sitofp <8 x i64> %a to <8 x float>
   ret <8 x float> %1
 }
@@ -270,6 +614,15 @@ define <8 x float> @sitofpv8i64v8float(<8 x i64> %a) {
 define <16 x float> @sitofpv16i64v16float(<16 x i64> %a) {
   ; SSE2: sitofpv16i64v16float
   ; SSE2: cost of 120 {{.*}} sitofp
+  ;
+  ; AVX1: sitofpv16i64v16float
+  ; AVX1: cost of 44 {{.*}} sitofp
+  ;
+  ; AVX2: sitofpv16i64v16float
+  ; AVX2: cost of 44 {{.*}} sitofp
+  ;
+  ; AVX512F: sitofpv16i64v16float
+  ; AVX512F: cost of 46 {{.*}} sitofp
   %1 = sitofp <16 x i64> %a to <16 x float>
   ret <16 x float> %1
 }
@@ -277,49 +630,48 @@ define <16 x float> @sitofpv16i64v16float(<16 x i64> %a) {
 define <32 x float> @sitofpv32i64v32float(<32 x i64> %a) {
   ; SSE2: sitofpv32i64v32float
   ; SSE2: cost of 240 {{.*}} sitofp
+  ;
+  ; AVX1: sitofpv32i64v32float
+  ; AVX1: cost of 88 {{.*}} sitofp
+  ;
+  ; AVX2: sitofpv32i64v32float
+  ; AVX2: cost of 88 {{.*}} sitofp
+  ;
+  ; AVX512F: sitofpv32i64v32float
+  ; AVX512F: cost of 92 {{.*}} sitofp
   %1 = sitofp <32 x i64> %a to <32 x float>
   ret <32 x float> %1
 }
 
-; AVX512F-LABEL: sitofp_16i8_float
-; AVX512F: cost of 2 {{.*}} sitofp
-define <16 x float> @sitofp_16i8_float(<16 x i8> %a) {
-  %1 = sitofp <16 x i8> %a to <16 x float>
-  ret <16 x float> %1
-}
-
-define <16 x float> @sitofp_16i16_float(<16 x i16> %a) {
-  ; AVX512F-LABEL: sitofp_16i16_float
-  ; AVX512F: cost of 2 {{.*}} sitofp
-  %1 = sitofp <16 x i16> %a to <16 x float>
-  ret <16 x float> %1
-}
-
-; AVX512F-LABEL: sitofp_8i8_double
-; AVX512F: cost of 2 {{.*}} sitofp
-define <8 x double> @sitofp_8i8_double(<8 x i8> %a) {
-  %1 = sitofp <8 x i8> %a to <8 x double>
-  ret <8 x double> %1
-}
-
-; AVX512F-LABEL: sitofp_8i16_double
-; AVX512F: cost of 2 {{.*}} sitofp
-define <8 x double> @sitofp_8i16_double(<8 x i16> %a) {
-  %1 = sitofp <8 x i16> %a to <8 x double>
-  ret <8 x double> %1
-}
-
-; AVX512F-LABEL: sitofp_8i1_double
-; AVX512F: cost of 4 {{.*}} sitofp
-define <8 x double> @sitofp_8i1_double(<8 x double> %a) {
+define <8 x double> @sitofpv8i1v8double(<8 x double> %a) {
+  ; SSE2: sitofpv8i1v8double
+  ; SSE2: cost of 80 {{.*}} sitofp
+  ;
+  ; AVX1: sitofpv8i1v8double
+  ; AVX1: cost of 20 {{.*}} sitofp
+  ;
+  ; AVX2: sitofpv8i1v8double
+  ; AVX2: cost of 20 {{.*}} sitofp
+  ;
+  ; AVX512F: sitofpv8i1v8double
+  ; AVX512F: cost of 4 {{.*}} sitofp
   %cmpres = fcmp ogt <8 x double> %a, zeroinitializer
   %1 = sitofp <8 x i1> %cmpres to <8 x double>
   ret <8 x double> %1
 }
 
-; AVX512F-LABEL: sitofp_16i1_float
-; AVX512F: cost of 3 {{.*}} sitofp
-define <16 x float> @sitofp_16i1_float(<16 x float> %a) {
+define <16 x float> @sitofpv16i1v16float(<16 x float> %a) {
+  ; SSE2: sitofpv16i1v16float
+  ; SSE2: cost of 8 {{.*}} sitofp
+  ;
+  ; AVX1: sitofpv16i1v16float
+  ; AVX1: cost of 44 {{.*}} sitofp
+  ;
+  ; AVX2: sitofpv16i1v16float
+  ; AVX2: cost of 44 {{.*}} sitofp
+  ;
+  ; AVX512F: sitofpv16i1v16float
+  ; AVX512F: cost of 3 {{.*}} sitofp
   %cmpres = fcmp ogt <16 x float> %a, zeroinitializer
   %1 = sitofp <16 x i1> %cmpres to <16 x float>
   ret <16 x float> %1
diff --git a/test/Analysis/CostModel/X86/testshiftashr.ll b/test/Analysis/CostModel/X86/testshiftashr.ll
index d96a92fe2a8..13f2bd2019d 100644
--- a/test/Analysis/CostModel/X86/testshiftashr.ll
+++ b/test/Analysis/CostModel/X86/testshiftashr.ll
@@ -5,9 +5,9 @@
 define %shifttype @shift2i16(%shifttype %a, %shifttype %b) {
 entry:
   ; SSE2: shift2i16
-  ; SSE2: cost of 20 {{.*}} ashr
+  ; SSE2: cost of 12 {{.*}} ashr
   ; SSE2-CODEGEN: shift2i16
-  ; SSE2-CODEGEN: sarq %cl
+  ; SSE2-CODEGEN: psrlq
 
   %0 = ashr %shifttype %a , %b
   ret %shifttype %0
@@ -17,9 +17,9 @@ entry:
 define %shifttype4i16 @shift4i16(%shifttype4i16 %a, %shifttype4i16 %b) {
 entry:
   ; SSE2: shift4i16
-  ; SSE2: cost of 40 {{.*}} ashr
+  ; SSE2: cost of 16 {{.*}} ashr
   ; SSE2-CODEGEN: shift4i16
-  ; SSE2-CODEGEN: sarl %cl
+  ; SSE2-CODEGEN: psrad
 
   %0 = ashr %shifttype4i16 %a , %b
   ret %shifttype4i16 %0
@@ -29,9 +29,9 @@ entry:
 define %shifttype8i16 @shift8i16(%shifttype8i16 %a, %shifttype8i16 %b) {
 entry:
   ; SSE2: shift8i16
-  ; SSE2: cost of 80 {{.*}} ashr
+  ; SSE2: cost of 32 {{.*}} ashr
   ; SSE2-CODEGEN: shift8i16
-  ; SSE2-CODEGEN: sarw %cl
+  ; SSE2-CODEGEN: psraw
 
   %0 = ashr %shifttype8i16 %a , %b
   ret %shifttype8i16 %0
@@ -41,9 +41,9 @@ entry:
 define %shifttype16i16 @shift16i16(%shifttype16i16 %a, %shifttype16i16 %b) {
 entry:
   ; SSE2: shift16i16
-  ; SSE2: cost of 160 {{.*}} ashr
+  ; SSE2: cost of 64 {{.*}} ashr
   ; SSE2-CODEGEN: shift16i16
-  ; SSE2-CODEGEN: sarw %cl
+  ; SSE2-CODEGEN: psraw
 
   %0 = ashr %shifttype16i16 %a , %b
   ret %shifttype16i16 %0
@@ -53,9 +53,9 @@ entry:
 define %shifttype32i16 @shift32i16(%shifttype32i16 %a, %shifttype32i16 %b) {
 entry:
   ; SSE2: shift32i16
-  ; SSE2: cost of 320 {{.*}} ashr
+  ; SSE2: cost of 128 {{.*}} ashr
   ; SSE2-CODEGEN: shift32i16
-  ; SSE2-CODEGEN: sarw %cl
+  ; SSE2-CODEGEN: psraw
 
   %0 = ashr %shifttype32i16 %a , %b
   ret %shifttype32i16 %0
@@ -65,9 +65,9 @@ entry:
 define %shifttype2i32 @shift2i32(%shifttype2i32 %a, %shifttype2i32 %b) {
 entry:
   ; SSE2: shift2i32
-  ; SSE2: cost of 20 {{.*}} ashr
+  ; SSE2: cost of 12 {{.*}} ashr
   ; SSE2-CODEGEN: shift2i32
-  ; SSE2-CODEGEN: sarq %cl
+  ; SSE2-CODEGEN: psrlq
 
   %0 = ashr %shifttype2i32 %a , %b
   ret %shifttype2i32 %0
@@ -77,9 +77,9 @@ entry:
 define %shifttype4i32 @shift4i32(%shifttype4i32 %a, %shifttype4i32 %b) {
 entry:
   ; SSE2: shift4i32
-  ; SSE2: cost of 40 {{.*}} ashr
+  ; SSE2: cost of 16 {{.*}} ashr
   ; SSE2-CODEGEN: shift4i32
-  ; SSE2-CODEGEN: sarl %cl
+  ; SSE2-CODEGEN: psrad
 
   %0 = ashr %shifttype4i32 %a , %b
   ret %shifttype4i32 %0
@@ -89,9 +89,9 @@ entry:
 define %shifttype8i32 @shift8i32(%shifttype8i32 %a, %shifttype8i32 %b) {
 entry:
   ; SSE2: shift8i32
-  ; SSE2: cost of 80 {{.*}} ashr
+  ; SSE2: cost of 32 {{.*}} ashr
   ; SSE2-CODEGEN: shift8i32
-  ; SSE2-CODEGEN: sarl %cl
+  ; SSE2-CODEGEN: psrad
 
   %0 = ashr %shifttype8i32 %a , %b
   ret %shifttype8i32 %0
@@ -101,9 +101,9 @@ entry:
 define %shifttype16i32 @shift16i32(%shifttype16i32 %a, %shifttype16i32 %b) {
 entry:
   ; SSE2: shift16i32
-  ; SSE2: cost of 160 {{.*}} ashr
+  ; SSE2: cost of 64 {{.*}} ashr
   ; SSE2-CODEGEN: shift16i32
-  ; SSE2-CODEGEN: sarl %cl
+  ; SSE2-CODEGEN: psrad
 
   %0 = ashr %shifttype16i32 %a , %b
   ret %shifttype16i32 %0
@@ -113,9 +113,9 @@ entry:
 define %shifttype32i32 @shift32i32(%shifttype32i32 %a, %shifttype32i32 %b) {
 entry:
   ; SSE2: shift32i32
-  ; SSE2: cost of 320 {{.*}} ashr
+  ; SSE2: cost of 128 {{.*}} ashr
   ; SSE2-CODEGEN: shift32i32
-  ; SSE2-CODEGEN: sarl %cl
+  ; SSE2-CODEGEN: psrad
 
   %0 = ashr %shifttype32i32 %a , %b
   ret %shifttype32i32 %0
@@ -125,9 +125,9 @@ entry:
 define %shifttype2i64 @shift2i64(%shifttype2i64 %a, %shifttype2i64 %b) {
 entry:
   ; SSE2: shift2i64
-  ; SSE2: cost of 20 {{.*}} ashr
+  ; SSE2: cost of 12 {{.*}} ashr
   ; SSE2-CODEGEN: shift2i64
-  ; SSE2-CODEGEN: sarq %cl
+  ; SSE2-CODEGEN: psrlq
 
   %0 = ashr %shifttype2i64 %a , %b
   ret %shifttype2i64 %0
@@ -137,9 +137,9 @@ entry:
 define %shifttype4i64 @shift4i64(%shifttype4i64 %a, %shifttype4i64 %b) {
 entry:
   ; SSE2: shift4i64
-  ; SSE2: cost of 40 {{.*}} ashr
+  ; SSE2: cost of 24 {{.*}} ashr
   ; SSE2-CODEGEN: shift4i64
-  ; SSE2-CODEGEN: sarq %cl
+  ; SSE2-CODEGEN: psrlq
 
   %0 = ashr %shifttype4i64 %a , %b
   ret %shifttype4i64 %0
@@ -149,9 +149,9 @@ entry:
 define %shifttype8i64 @shift8i64(%shifttype8i64 %a, %shifttype8i64 %b) {
 entry:
   ; SSE2: shift8i64
-  ; SSE2: cost of 80 {{.*}} ashr
+  ; SSE2: cost of 48 {{.*}} ashr
   ; SSE2-CODEGEN: shift8i64
-  ; SSE2-CODEGEN: sarq %cl
+  ; SSE2-CODEGEN: psrlq
 
   %0 = ashr %shifttype8i64 %a , %b
   ret %shifttype8i64 %0
@@ -161,9 +161,9 @@ entry:
 define %shifttype16i64 @shift16i64(%shifttype16i64 %a, %shifttype16i64 %b) {
 entry:
   ; SSE2: shift16i64
-  ; SSE2: cost of 160 {{.*}} ashr
+  ; SSE2: cost of 96 {{.*}} ashr
   ; SSE2-CODEGEN: shift16i64
-  ; SSE2-CODEGEN: sarq %cl
+  ; SSE2-CODEGEN: psrlq
 
   %0 = ashr %shifttype16i64 %a , %b
   ret %shifttype16i64 %0
@@ -173,9 +173,9 @@ entry:
 define %shifttype32i64 @shift32i64(%shifttype32i64 %a, %shifttype32i64 %b) {
 entry:
   ; SSE2: shift32i64
-  ; SSE2: cost of 320 {{.*}} ashr
+  ; SSE2: cost of 192 {{.*}} ashr
   ; SSE2-CODEGEN: shift32i64
-  ; SSE2-CODEGEN: sarq %cl
+  ; SSE2-CODEGEN: psrlq
 
   %0 = ashr %shifttype32i64 %a , %b
   ret %shifttype32i64 %0
@@ -185,9 +185,9 @@ entry:
 define %shifttype2i8 @shift2i8(%shifttype2i8 %a, %shifttype2i8 %b) {
 entry:
   ; SSE2: shift2i8
-  ; SSE2: cost of 20 {{.*}} ashr
+  ; SSE2: cost of 12 {{.*}} ashr
   ; SSE2-CODEGEN: shift2i8
-  ; SSE2-CODEGEN: sarq %cl
+  ; SSE2-CODEGEN: psrlq
 
   %0 = ashr %shifttype2i8 %a , %b
   ret %shifttype2i8 %0
@@ -197,9 +197,9 @@ entry:
 define %shifttype4i8 @shift4i8(%shifttype4i8 %a, %shifttype4i8 %b) {
 entry:
   ; SSE2: shift4i8
-  ; SSE2: cost of 40 {{.*}} ashr
+  ; SSE2: cost of 16 {{.*}} ashr
   ; SSE2-CODEGEN: shift4i8
-  ; SSE2-CODEGEN: sarl %cl
+  ; SSE2-CODEGEN: psrad
 
   %0 = ashr %shifttype4i8 %a , %b
   ret %shifttype4i8 %0
@@ -209,9 +209,9 @@ entry:
 define %shifttype8i8 @shift8i8(%shifttype8i8 %a, %shifttype8i8 %b) {
 entry:
   ; SSE2: shift8i8
-  ; SSE2: cost of 80 {{.*}} ashr
+  ; SSE2: cost of 32 {{.*}} ashr
   ; SSE2-CODEGEN: shift8i8
-  ; SSE2-CODEGEN: sarw %cl
+  ; SSE2-CODEGEN: psraw
 
   %0 = ashr %shifttype8i8 %a , %b
   ret %shifttype8i8 %0
@@ -221,9 +221,9 @@ entry:
 define %shifttype16i8 @shift16i8(%shifttype16i8 %a, %shifttype16i8 %b) {
 entry:
   ; SSE2: shift16i8
-  ; SSE2: cost of 160 {{.*}} ashr
+  ; SSE2: cost of 54 {{.*}} ashr
   ; SSE2-CODEGEN: shift16i8
-  ; SSE2-CODEGEN: sarb %cl
+  ; SSE2-CODEGEN: psraw
 
   %0 = ashr %shifttype16i8 %a , %b
   ret %shifttype16i8 %0
@@ -233,9 +233,9 @@ entry:
 define %shifttype32i8 @shift32i8(%shifttype32i8 %a, %shifttype32i8 %b) {
 entry:
   ; SSE2: shift32i8
-  ; SSE2: cost of 320 {{.*}} ashr
+  ; SSE2: cost of 108 {{.*}} ashr
   ; SSE2-CODEGEN: shift32i8
-  ; SSE2-CODEGEN: sarb %cl
+  ; SSE2-CODEGEN: psraw
 
   %0 = ashr %shifttype32i8 %a , %b
   ret %shifttype32i8 %0
@@ -247,9 +247,9 @@ entry:
 define %shifttypec @shift2i16const(%shifttypec %a, %shifttypec %b) {
 entry:
   ; SSE2: shift2i16const
-  ; SSE2: cost of 20 {{.*}} ashr
+  ; SSE2: cost of 4 {{.*}} ashr
   ; SSE2-CODEGEN: shift2i16const
-  ; SSE2-CODEGEN: sarq $
+  ; SSE2-CODEGEN: psrad $3
 
   %0 = ashr %shifttypec %a , <i16 3, i16 3>
   ret %shifttypec %0
@@ -320,9 +320,9 @@ entry:
 define %shifttypec2i32 @shift2i32c(%shifttypec2i32 %a, %shifttypec2i32 %b) {
 entry:
   ; SSE2: shift2i32c
-  ; SSE2: cost of 20 {{.*}} ashr
+  ; SSE2: cost of 4 {{.*}} ashr
   ; SSE2-CODEGEN: shift2i32c
-  ; SSE2-CODEGEN: sarq $3
+  ; SSE2-CODEGEN: psrad $3
 
   %0 = ashr %shifttypec2i32 %a , <i32 3, i32 3>
   ret %shifttypec2i32 %0
@@ -391,9 +391,9 @@ entry:
 define %shifttypec2i64 @shift2i64c(%shifttypec2i64 %a, %shifttypec2i64 %b) {
 entry:
   ; SSE2: shift2i64c
-  ; SSE2: cost of 20 {{.*}} ashr
+  ; SSE2: cost of 4 {{.*}} ashr
   ; SSE2-CODEGEN: shift2i64c
-  ; SSE2-CODEGEN: sarq $3
+  ; SSE2-CODEGEN: psrad $3
 
   %0 = ashr %shifttypec2i64 %a , <i64 3, i64 3>
   ret %shifttypec2i64 %0
@@ -403,9 +403,9 @@ entry:
 define %shifttypec4i64 @shift4i64c(%shifttypec4i64 %a, %shifttypec4i64 %b) {
 entry:
   ; SSE2: shift4i64c
-  ; SSE2: cost of 40 {{.*}} ashr
+  ; SSE2: cost of 8 {{.*}} ashr
   ; SSE2-CODEGEN: shift4i64c
-  ; SSE2-CODEGEN: sarq $3
+  ; SSE2-CODEGEN: psrad $3
 
   %0 = ashr %shifttypec4i64 %a , <i64 3, i64 3, i64 3, i64 3>
   ret %shifttypec4i64 %0
@@ -415,9 +415,9 @@ entry:
 define %shifttypec8i64 @shift8i64c(%shifttypec8i64 %a, %shifttypec8i64 %b) {
 entry:
   ; SSE2: shift8i64c
-  ; SSE2: cost of 80 {{.*}} ashr
+  ; SSE2: cost of 16 {{.*}} ashr
   ; SSE2-CODEGEN: shift8i64c
-  ; SSE2-CODEGEN: sarq $3
+  ; SSE2-CODEGEN: psrad $3
 
  %0 = ashr %shifttypec8i64 %a , <i64 3, i64 3, i64 3, i64 3,
                                  i64 3, i64 3, i64 3, i64 3>
@@ -428,9 +428,9 @@ entry:
 define %shifttypec16i64 @shift16i64c(%shifttypec16i64 %a, %shifttypec16i64 %b) {
 entry:
   ; SSE2: shift16i64c
-  ; SSE2: cost of 160 {{.*}} ashr
+  ; SSE2: cost of 32 {{.*}} ashr
   ; SSE2-CODEGEN: shift16i64c
-  ; SSE2-CODEGEN: sarq $3
+  ; SSE2-CODEGEN: psrad $3
 
   %0 = ashr %shifttypec16i64 %a , <i64 3, i64 3, i64 3, i64 3,
                                    i64 3, i64 3, i64 3, i64 3,
@@ -443,9 +443,9 @@ entry:
 define %shifttypec32i64 @shift32i64c(%shifttypec32i64 %a, %shifttypec32i64 %b) {
 entry:
   ; SSE2: shift32i64c
-  ; SSE2: cost of 320 {{.*}} ashr
+  ; SSE2: cost of 64 {{.*}} ashr
   ; SSE2-CODEGEN: shift32i64c
-  ; SSE2-CODEGEN: sarq $3
+  ; SSE2-CODEGEN: psrad $3
 
   %0 = ashr %shifttypec32i64 %a ,<i64 3, i64 3, i64 3, i64 3,
                                   i64 3, i64 3, i64 3, i64 3,
@@ -462,9 +462,9 @@ entry:
 define %shifttypec2i8 @shift2i8c(%shifttypec2i8 %a, %shifttypec2i8 %b) {
 entry:
   ; SSE2: shift2i8c
-  ; SSE2: cost of 20 {{.*}} ashr
+  ; SSE2: cost of 4 {{.*}} ashr
   ; SSE2-CODEGEN: shift2i8c
-  ; SSE2-CODEGEN: sarq $3
+  ; SSE2-CODEGEN: psrad $3
 
   %0 = ashr %shifttypec2i8 %a , <i8 3, i8 3>
   ret %shifttypec2i8 %0
diff --git a/test/Analysis/CostModel/X86/testshiftlshr.ll b/test/Analysis/CostModel/X86/testshiftlshr.ll
index 78bf0a60830..52f176fe4d6 100644
--- a/test/Analysis/CostModel/X86/testshiftlshr.ll
+++ b/test/Analysis/CostModel/X86/testshiftlshr.ll
@@ -5,7 +5,7 @@
 define %shifttype @shift2i16(%shifttype %a, %shifttype %b) {
 entry:
   ; SSE2: shift2i16
-  ; SSE2: cost of 20 {{.*}} lshr
+  ; SSE2: cost of 4 {{.*}} lshr
   ; SSE2-CODEGEN: shift2i16
   ; SSE2-CODEGEN: psrlq
 
@@ -17,9 +17,9 @@ entry:
 define %shifttype4i16 @shift4i16(%shifttype4i16 %a, %shifttype4i16 %b) {
 entry:
   ; SSE2: shift4i16
-  ; SSE2: cost of 40 {{.*}} lshr
+  ; SSE2: cost of 16 {{.*}} lshr
   ; SSE2-CODEGEN: shift4i16
-  ; SSE2-CODEGEN: shrl %cl
+  ; SSE2-CODEGEN: psrld
 
   %0 = lshr %shifttype4i16 %a , %b
   ret %shifttype4i16 %0
@@ -29,9 +29,9 @@ entry:
 define %shifttype8i16 @shift8i16(%shifttype8i16 %a, %shifttype8i16 %b) {
 entry:
   ; SSE2: shift8i16
-  ; SSE2: cost of 80 {{.*}} lshr
+  ; SSE2: cost of 32 {{.*}} lshr
   ; SSE2-CODEGEN: shift8i16
-  ; SSE2-CODEGEN: shrl %cl
+  ; SSE2-CODEGEN: psrlw
 
   %0 = lshr %shifttype8i16 %a , %b
   ret %shifttype8i16 %0
@@ -41,9 +41,9 @@ entry:
 define %shifttype16i16 @shift16i16(%shifttype16i16 %a, %shifttype16i16 %b) {
 entry:
   ; SSE2: shift16i16
-  ; SSE2: cost of 160 {{.*}} lshr
+  ; SSE2: cost of 64 {{.*}} lshr
   ; SSE2-CODEGEN: shift16i16
-  ; SSE2-CODEGEN: shrl %cl
+  ; SSE2-CODEGEN: psrlw
 
   %0 = lshr %shifttype16i16 %a , %b
   ret %shifttype16i16 %0
@@ -53,9 +53,9 @@ entry:
 define %shifttype32i16 @shift32i16(%shifttype32i16 %a, %shifttype32i16 %b) {
 entry:
   ; SSE2: shift32i16
-  ; SSE2: cost of 320 {{.*}} lshr
+  ; SSE2: cost of 128 {{.*}} lshr
   ; SSE2-CODEGEN: shift32i16
-  ; SSE2-CODEGEN: shrl %cl
+  ; SSE2-CODEGEN: psrlw
 
   %0 = lshr %shifttype32i16 %a , %b
   ret %shifttype32i16 %0
@@ -65,7 +65,7 @@ entry:
 define %shifttype2i32 @shift2i32(%shifttype2i32 %a, %shifttype2i32 %b) {
 entry:
   ; SSE2: shift2i32
-  ; SSE2: cost of 20 {{.*}} lshr
+  ; SSE2: cost of 4 {{.*}} lshr
   ; SSE2-CODEGEN: shift2i32
   ; SSE2-CODEGEN: psrlq
 
@@ -77,9 +77,9 @@ entry:
 define %shifttype4i32 @shift4i32(%shifttype4i32 %a, %shifttype4i32 %b) {
 entry:
   ; SSE2: shift4i32
-  ; SSE2: cost of 40 {{.*}} lshr
+  ; SSE2: cost of 16 {{.*}} lshr
   ; SSE2-CODEGEN: shift4i32
-  ; SSE2-CODEGEN: shrl %cl
+  ; SSE2-CODEGEN: psrld
 
   %0 = lshr %shifttype4i32 %a , %b
   ret %shifttype4i32 %0
@@ -89,9 +89,9 @@ entry:
 define %shifttype8i32 @shift8i32(%shifttype8i32 %a, %shifttype8i32 %b) {
 entry:
   ; SSE2: shift8i32
-  ; SSE2: cost of 80 {{.*}} lshr
+  ; SSE2: cost of 32 {{.*}} lshr
   ; SSE2-CODEGEN: shift8i32
-  ; SSE2-CODEGEN: shrl %cl
+  ; SSE2-CODEGEN: psrld
 
   %0 = lshr %shifttype8i32 %a , %b
   ret %shifttype8i32 %0
@@ -101,9 +101,9 @@ entry:
 define %shifttype16i32 @shift16i32(%shifttype16i32 %a, %shifttype16i32 %b) {
 entry:
   ; SSE2: shift16i32
-  ; SSE2: cost of 160 {{.*}} lshr
+  ; SSE2: cost of 64 {{.*}} lshr
   ; SSE2-CODEGEN: shift16i32
-  ; SSE2-CODEGEN: shrl %cl
+  ; SSE2-CODEGEN: psrld
 
   %0 = lshr %shifttype16i32 %a , %b
   ret %shifttype16i32 %0
@@ -113,9 +113,9 @@ entry:
 define %shifttype32i32 @shift32i32(%shifttype32i32 %a, %shifttype32i32 %b) {
 entry:
   ; SSE2: shift32i32
-  ; SSE2: cost of 320 {{.*}} lshr
+  ; SSE2: cost of 128 {{.*}} lshr
   ; SSE2-CODEGEN: shift32i32
-  ; SSE2-CODEGEN: shrl %cl
+  ; SSE2-CODEGEN: psrld
 
   %0 = lshr %shifttype32i32 %a , %b
   ret %shifttype32i32 %0
@@ -125,7 +125,7 @@ entry:
 define %shifttype2i64 @shift2i64(%shifttype2i64 %a, %shifttype2i64 %b) {
 entry:
   ; SSE2: shift2i64
-  ; SSE2: cost of 20 {{.*}} lshr
+  ; SSE2: cost of 4 {{.*}} lshr
   ; SSE2-CODEGEN: shift2i64
   ; SSE2-CODEGEN: psrlq
 
@@ -137,7 +137,7 @@ entry:
 define %shifttype4i64 @shift4i64(%shifttype4i64 %a, %shifttype4i64 %b) {
 entry:
   ; SSE2: shift4i64
-  ; SSE2: cost of 40 {{.*}} lshr
+  ; SSE2: cost of 8 {{.*}} lshr
   ; SSE2-CODEGEN: shift4i64
   ; SSE2-CODEGEN: psrlq
 
@@ -149,7 +149,7 @@ entry:
 define %shifttype8i64 @shift8i64(%shifttype8i64 %a, %shifttype8i64 %b) {
 entry:
   ; SSE2: shift8i64
-  ; SSE2: cost of 80 {{.*}} lshr
+  ; SSE2: cost of 16 {{.*}} lshr
   ; SSE2-CODEGEN: shift8i64
   ; SSE2-CODEGEN: psrlq
 
@@ -161,7 +161,7 @@ entry:
 define %shifttype16i64 @shift16i64(%shifttype16i64 %a, %shifttype16i64 %b) {
 entry:
   ; SSE2: shift16i64
-  ; SSE2: cost of 160 {{.*}} lshr
+  ; SSE2: cost of 32 {{.*}} lshr
   ; SSE2-CODEGEN: shift16i64
   ; SSE2-CODEGEN: psrlq
 
@@ -173,7 +173,7 @@ entry:
 define %shifttype32i64 @shift32i64(%shifttype32i64 %a, %shifttype32i64 %b) {
 entry:
   ; SSE2: shift32i64
-  ; SSE2: cost of 320 {{.*}} lshr
+  ; SSE2: cost of 64 {{.*}} lshr
   ; SSE2-CODEGEN: shift32i64
   ; SSE2-CODEGEN: psrlq
 
@@ -185,7 +185,7 @@ entry:
 define %shifttype2i8 @shift2i8(%shifttype2i8 %a, %shifttype2i8 %b) {
 entry:
   ; SSE2: shift2i8
-  ; SSE2: cost of 20 {{.*}} lshr
+  ; SSE2: cost of 4 {{.*}} lshr
   ; SSE2-CODEGEN: shift2i8
   ; SSE2-CODEGEN: psrlq
 
@@ -197,9 +197,9 @@ entry:
 define %shifttype4i8 @shift4i8(%shifttype4i8 %a, %shifttype4i8 %b) {
 entry:
   ; SSE2: shift4i8
-  ; SSE2: cost of 40 {{.*}} lshr
+  ; SSE2: cost of 16 {{.*}} lshr
   ; SSE2-CODEGEN: shift4i8
-  ; SSE2-CODEGEN: shrl %cl
+  ; SSE2-CODEGEN: psrld
 
   %0 = lshr %shifttype4i8 %a , %b
   ret %shifttype4i8 %0
@@ -209,9 +209,9 @@ entry:
 define %shifttype8i8 @shift8i8(%shifttype8i8 %a, %shifttype8i8 %b) {
 entry:
   ; SSE2: shift8i8
-  ; SSE2: cost of 80 {{.*}} lshr
+  ; SSE2: cost of 32 {{.*}} lshr
   ; SSE2-CODEGEN: shift8i8
-  ; SSE2-CODEGEN: shrl %cl
+  ; SSE2-CODEGEN: psrlw
 
   %0 = lshr %shifttype8i8 %a , %b
   ret %shifttype8i8 %0
@@ -221,9 +221,9 @@ entry:
 define %shifttype16i8 @shift16i8(%shifttype16i8 %a, %shifttype16i8 %b) {
 entry:
   ; SSE2: shift16i8
-  ; SSE2: cost of 160 {{.*}} lshr
+  ; SSE2: cost of 26 {{.*}} lshr
   ; SSE2-CODEGEN: shift16i8
-  ; SSE2-CODEGEN: shrb %cl
+  ; SSE2-CODEGEN: psrlw
 
   %0 = lshr %shifttype16i8 %a , %b
   ret %shifttype16i8 %0
@@ -233,9 +233,9 @@ entry:
 define %shifttype32i8 @shift32i8(%shifttype32i8 %a, %shifttype32i8 %b) {
 entry:
   ; SSE2: shift32i8
-  ; SSE2: cost of 320 {{.*}} lshr
+  ; SSE2: cost of 52 {{.*}} lshr
   ; SSE2-CODEGEN: shift32i8
-  ; SSE2-CODEGEN: shrb %cl
+  ; SSE2-CODEGEN: psrlw
 
   %0 = lshr %shifttype32i8 %a , %b
   ret %shifttype32i8 %0
diff --git a/test/Analysis/CostModel/X86/testshiftshl.ll b/test/Analysis/CostModel/X86/testshiftshl.ll
index c36e0f5dfdf..e385c5bfeea 100644
--- a/test/Analysis/CostModel/X86/testshiftshl.ll
+++ b/test/Analysis/CostModel/X86/testshiftshl.ll
@@ -5,7 +5,7 @@
 define %shifttype @shift2i16(%shifttype %a, %shifttype %b) {
 entry:
   ; SSE2: shift2i16
-  ; SSE2: cost of 20 {{.*}} shl
+  ; SSE2: cost of 4 {{.*}} shl
   ; SSE2-CODEGEN: shift2i16
   ; SSE2-CODEGEN: psllq
 
@@ -29,9 +29,9 @@ entry:
 define %shifttype8i16 @shift8i16(%shifttype8i16 %a, %shifttype8i16 %b) {
 entry:
   ; SSE2: shift8i16
-  ; SSE2: cost of 80 {{.*}} shl
+  ; SSE2: cost of 32 {{.*}} shl
   ; SSE2-CODEGEN: shift8i16
-  ; SSE2-CODEGEN: shll %cl
+  ; SSE2-CODEGEN: psllw
 
   %0 = shl %shifttype8i16 %a , %b
   ret %shifttype8i16 %0
@@ -41,9 +41,9 @@ entry:
 define %shifttype16i16 @shift16i16(%shifttype16i16 %a, %shifttype16i16 %b) {
 entry:
   ; SSE2: shift16i16
-  ; SSE2: cost of 160 {{.*}} shl
+  ; SSE2: cost of 64 {{.*}} shl
   ; SSE2-CODEGEN: shift16i16
-  ; SSE2-CODEGEN: shll %cl
+  ; SSE2-CODEGEN: psllw
 
   %0 = shl %shifttype16i16 %a , %b
   ret %shifttype16i16 %0
@@ -53,9 +53,9 @@ entry:
 define %shifttype32i16 @shift32i16(%shifttype32i16 %a, %shifttype32i16 %b) {
 entry:
   ; SSE2: shift32i16
-  ; SSE2: cost of 320 {{.*}} shl
+  ; SSE2: cost of 128 {{.*}} shl
   ; SSE2-CODEGEN: shift32i16
-  ; SSE2-CODEGEN: shll %cl
+  ; SSE2-CODEGEN: psllw
 
   %0 = shl %shifttype32i16 %a , %b
   ret %shifttype32i16 %0
@@ -65,7 +65,7 @@ entry:
 define %shifttype2i32 @shift2i32(%shifttype2i32 %a, %shifttype2i32 %b) {
 entry:
   ; SSE2: shift2i32
-  ; SSE2: cost of 20 {{.*}} shl
+  ; SSE2: cost of 4 {{.*}} shl
   ; SSE2-CODEGEN: shift2i32
   ; SSE2-CODEGEN: psllq
 
@@ -125,7 +125,7 @@ entry:
 define %shifttype2i64 @shift2i64(%shifttype2i64 %a, %shifttype2i64 %b) {
 entry:
   ; SSE2: shift2i64
-  ; SSE2: cost of 20 {{.*}} shl
+  ; SSE2: cost of 4 {{.*}} shl
   ; SSE2-CODEGEN: shift2i64
   ; SSE2-CODEGEN: psllq
 
@@ -137,7 +137,7 @@ entry:
 define %shifttype4i64 @shift4i64(%shifttype4i64 %a, %shifttype4i64 %b) {
 entry:
   ; SSE2: shift4i64
-  ; SSE2: cost of 40 {{.*}} shl
+  ; SSE2: cost of 8 {{.*}} shl
   ; SSE2-CODEGEN: shift4i64
   ; SSE2-CODEGEN: psllq
 
@@ -149,7 +149,7 @@ entry:
 define %shifttype8i64 @shift8i64(%shifttype8i64 %a, %shifttype8i64 %b) {
 entry:
   ; SSE2: shift8i64
-  ; SSE2: cost of 80 {{.*}} shl
+  ; SSE2: cost of 16 {{.*}} shl
   ; SSE2-CODEGEN: shift8i64
   ; SSE2-CODEGEN: psllq
 
@@ -161,7 +161,7 @@ entry:
 define %shifttype16i64 @shift16i64(%shifttype16i64 %a, %shifttype16i64 %b) {
 entry:
   ; SSE2: shift16i64
-  ; SSE2: cost of 160 {{.*}} shl
+  ; SSE2: cost of 32 {{.*}} shl
   ; SSE2-CODEGEN: shift16i64
   ; SSE2-CODEGEN: psllq
 
@@ -173,7 +173,7 @@ entry:
 define %shifttype32i64 @shift32i64(%shifttype32i64 %a, %shifttype32i64 %b) {
 entry:
   ; SSE2: shift32i64
-  ; SSE2: cost of 320 {{.*}} shl
+  ; SSE2: cost of 64 {{.*}} shl
   ; SSE2-CODEGEN: shift32i64
   ; SSE2-CODEGEN: psllq
 
@@ -185,7 +185,7 @@ entry:
 define %shifttype2i8 @shift2i8(%shifttype2i8 %a, %shifttype2i8 %b) {
 entry:
   ; SSE2: shift2i8
-  ; SSE2: cost of 20 {{.*}} shl
+  ; SSE2: cost of 4 {{.*}} shl
   ; SSE2-CODEGEN: shift2i8
   ; SSE2-CODEGEN: psllq
 
@@ -209,9 +209,9 @@ entry:
 define %shifttype8i8 @shift8i8(%shifttype8i8 %a, %shifttype8i8 %b) {
 entry:
   ; SSE2: shift8i8
-  ; SSE2: cost of 80 {{.*}} shl
+  ; SSE2: cost of 32 {{.*}} shl
   ; SSE2-CODEGEN: shift8i8
-  ; SSE2-CODEGEN: shll
+  ; SSE2-CODEGEN: psllw
 
   %0 = shl %shifttype8i8 %a , %b
   ret %shifttype8i8 %0
@@ -221,9 +221,9 @@ entry:
 define %shifttype16i8 @shift16i8(%shifttype16i8 %a, %shifttype16i8 %b) {
 entry:
   ; SSE2: shift16i8
-  ; SSE2: cost of 30 {{.*}} shl
+  ; SSE2: cost of 26 {{.*}} shl
   ; SSE2-CODEGEN: shift16i8
-  ; SSE2-CODEGEN: cmpeqb
+  ; SSE2-CODEGEN: psllw
 
   %0 = shl %shifttype16i8 %a , %b
   ret %shifttype16i8 %0
@@ -233,9 +233,9 @@ entry:
 define %shifttype32i8 @shift32i8(%shifttype32i8 %a, %shifttype32i8 %b) {
 entry:
   ; SSE2: shift32i8
-  ; SSE2: cost of 60 {{.*}} shl
+  ; SSE2: cost of 52 {{.*}} shl
   ; SSE2-CODEGEN: shift32i8
-  ; SSE2-CODEGEN: cmpeqb
+  ; SSE2-CODEGEN: psllw
 
   %0 = shl %shifttype32i8 %a , %b
   ret %shifttype32i8 %0
diff --git a/test/Analysis/CostModel/X86/uitofp.ll b/test/Analysis/CostModel/X86/uitofp.ll
index 27ec268b42a..9ffc483e3f5 100644
--- a/test/Analysis/CostModel/X86/uitofp.ll
+++ b/test/Analysis/CostModel/X86/uitofp.ll
@@ -1,18 +1,20 @@
-; RUN: llc -mtriple=x86_64-apple-darwin -mcpu=core2 < %s | FileCheck --check-prefix=SSE2-CODEGEN %s
-; RUN: opt -mtriple=x86_64-apple-darwin -mcpu=core2 -cost-model -analyze < %s | FileCheck --check-prefix=SSE2 %s
-
-; In X86TargetTransformInfo::getCastInstrCost we have code that depends on
-; getSimpleVT on a value type. On AVX2 we execute this code. Make sure we exit
-; early if the type is not a simple value type before we call this function.
-; RUN: opt -mtriple=x86_64-apple-darwin -mcpu=core-avx2 -cost-model -analyze < %s
+; RUN: opt -mtriple=x86_64-apple-darwin -mattr=+sse2 -cost-model -analyze < %s | FileCheck --check-prefix=SSE --check-prefix=SSE2 %s
+; RUN: opt -mtriple=x86_64-apple-darwin -mattr=+avx  -cost-model -analyze < %s | FileCheck --check-prefix=AVX --check-prefix=AVX1 %s
+; RUN: opt -mtriple=x86_64-apple-darwin -mattr=+avx2 -cost-model -analyze < %s | FileCheck --check-prefix=AVX --check-prefix=AVX2 %s
+; RUN: opt -mtriple=x86_64-apple-darwin -mattr=+avx512f -cost-model -analyze < %s | FileCheck --check-prefix=AVX512F %s
 
 define <2 x double> @uitofpv2i8v2double(<2 x i8> %a) {
   ; SSE2: uitofpv2i8v2double
   ; SSE2: cost of 20 {{.*}} uitofp
-  ; SSE2-CODEGEN: uitofpv2i8v2double
-  ; SSE2-CODEGEN: movapd  LCPI
-  ; SSE2-CODEGEN: subpd
-  ; SSE2-CODEGEN: addpd
+  ;
+  ; AVX1: uitofpv2i8v2double
+  ; AVX1: cost of 4 {{.*}} uitofp
+  ;
+  ; AVX2: uitofpv2i8v2double
+  ; AVX2: cost of 4 {{.*}} uitofp
+  ;
+  ; AVX512F: uitofpv2i8v2double
+  ; AVX512F: cost of 4 {{.*}} uitofp
   %1 = uitofp <2 x i8> %a to <2 x double>
   ret <2 x double> %1
 }
@@ -20,10 +22,15 @@ define <2 x double> @uitofpv2i8v2double(<2 x i8> %a) {
 define <4 x double> @uitofpv4i8v4double(<4 x i8> %a) {
   ; SSE2: uitofpv4i8v4double
   ; SSE2: cost of 40 {{.*}} uitofp
-  ; SSE2-CODEGEN: uitofpv4i8v4double
-  ; SSE2-CODEGEN: movapd  LCPI
-  ; SSE2-CODEGEN: subpd
-  ; SSE2-CODEGEN: addpd
+  ;
+  ; AVX1: uitofpv4i8v4double
+  ; AVX1: cost of 2 {{.*}} uitofp
+  ;
+  ; AVX2: uitofpv4i8v4double
+  ; AVX2: cost of 2 {{.*}} uitofp
+  ;
+  ; AVX512F: uitofpv4i8v4double
+  ; AVX512F: cost of 2 {{.*}} uitofp
   %1 = uitofp <4 x i8> %a to <4 x double>
   ret <4 x double> %1
 }
@@ -31,21 +38,31 @@ define <4 x double> @uitofpv4i8v4double(<4 x i8> %a) {
 define <8 x double> @uitofpv8i8v8double(<8 x i8> %a) {
   ; SSE2: uitofpv8i8v8double
   ; SSE2: cost of 80 {{.*}} uitofp
-  ; SSE2-CODEGEN: uitofpv8i8v8double
-  ; SSE2-CODEGEN: movapd  LCPI
-  ; SSE2-CODEGEN: subpd
-  ; SSE2-CODEGEN: addpd
-%1 = uitofp <8 x i8> %a to <8 x double>
+  ;
+  ; AVX1: uitofpv8i8v8double
+  ; AVX1: cost of 20 {{.*}} uitofp
+  ;
+  ; AVX2: uitofpv8i8v8double
+  ; AVX2: cost of 20 {{.*}} uitofp
+  ;
+  ; AVX512F: uitofpv8i8v8double
+  ; AVX512F: cost of 22 {{.*}} uitofp
+  %1 = uitofp <8 x i8> %a to <8 x double>
   ret <8 x double> %1
 }
 
 define <16 x double> @uitofpv16i8v16double(<16 x i8> %a) {
   ; SSE2: uitofpv16i8v16double
   ; SSE2: cost of 160 {{.*}} uitofp
-  ; SSE2-CODEGEN: uitofpv16i8v16double
-  ; SSE2-CODEGEN: movapd  LCPI
-  ; SSE2-CODEGEN: subpd
-  ; SSE2-CODEGEN: addpd
+  ;
+  ; AVX1: uitofpv16i8v16double
+  ; AVX1: cost of 40 {{.*}} uitofp
+  ;
+  ; AVX2: uitofpv16i8v16double
+  ; AVX2: cost of 40 {{.*}} uitofp
+  ;
+  ; AVX512F: uitofpv16i8v16double
+  ; AVX512F: cost of 44 {{.*}} uitofp
   %1 = uitofp <16 x i8> %a to <16 x double>
   ret <16 x double> %1
 }
@@ -53,10 +70,15 @@ define <16 x double> @uitofpv16i8v16double(<16 x i8> %a) {
 define <32 x double> @uitofpv32i8v32double(<32 x i8> %a) {
   ; SSE2: uitofpv32i8v32double
   ; SSE2: cost of 320 {{.*}} uitofp
-  ; SSE2-CODEGEN: uitofpv32i8v32double
-  ; SSE2-CODEGEN: movapd  LCPI
-  ; SSE2-CODEGEN: subpd
-  ; SSE2-CODEGEN: addpd
+  ;
+  ; AVX1: uitofpv32i8v32double
+  ; AVX1: cost of 80 {{.*}} uitofp
+  ;
+  ; AVX2: uitofpv32i8v32double
+  ; AVX2: cost of 80 {{.*}} uitofp
+  ;
+  ; AVX512F: uitofpv32i8v32double
+  ; AVX512F: cost of 88 {{.*}} uitofp
   %1 = uitofp <32 x i8> %a to <32 x double>
   ret <32 x double> %1
 }
@@ -64,10 +86,15 @@ define <32 x double> @uitofpv32i8v32double(<32 x i8> %a) {
 define <2 x double> @uitofpv2i16v2double(<2 x i16> %a) {
   ; SSE2: uitofpv2i16v2double
   ; SSE2: cost of 20 {{.*}} uitofp
-  ; SSE2-CODEGEN: uitofpv2i16v2double
-  ; SSE2-CODEGEN: movapd  LCPI
-  ; SSE2-CODEGEN: subpd
-  ; SSE2-CODEGEN: addpd
+  ;
+  ; AVX1: uitofpv2i16v2double
+  ; AVX1: cost of 4 {{.*}} uitofp
+  ;
+  ; AVX2: uitofpv2i16v2double
+  ; AVX2: cost of 4 {{.*}} uitofp
+  ;
+  ; AVX512F: uitofpv2i16v2double
+  ; AVX512F: cost of 4 {{.*}} uitofp
   %1 = uitofp <2 x i16> %a to <2 x double>
   ret <2 x double> %1
 }
@@ -75,10 +102,15 @@ define <2 x double> @uitofpv2i16v2double(<2 x i16> %a) {
 define <4 x double> @uitofpv4i16v4double(<4 x i16> %a) {
   ; SSE2: uitofpv4i16v4double
   ; SSE2: cost of 40 {{.*}} uitofp
-  ; SSE2-CODEGEN: uitofpv4i16v4double
-  ; SSE2-CODEGEN: movapd  LCPI
-  ; SSE2-CODEGEN: subpd
-  ; SSE2-CODEGEN: addpd
+  ;
+  ; AVX1: uitofpv4i16v4double
+  ; AVX1: cost of 2 {{.*}} uitofp
+  ;
+  ; AVX2: uitofpv4i16v4double
+  ; AVX2: cost of 2 {{.*}} uitofp
+  ;
+  ; AVX512F: uitofpv4i16v4double
+  ; AVX512F: cost of 2 {{.*}} uitofp
   %1 = uitofp <4 x i16> %a to <4 x double>
   ret <4 x double> %1
 }
@@ -86,10 +118,15 @@ define <4 x double> @uitofpv4i16v4double(<4 x i16> %a) {
 define <8 x double> @uitofpv8i16v8double(<8 x i16> %a) {
   ; SSE2: uitofpv8i16v8double
   ; SSE2: cost of 80 {{.*}} uitofp
-  ; SSE2-CODEGEN: uitofpv8i16v8double
-  ; SSE2-CODEGEN: movapd  LCPI
-  ; SSE2-CODEGEN: subpd
-  ; SSE2-CODEGEN: addpd
+  ;
+  ; AVX1: uitofpv8i16v8double
+  ; AVX1: cost of 20 {{.*}} uitofp
+  ;
+  ; AVX2: uitofpv8i16v8double
+  ; AVX2: cost of 20 {{.*}} uitofp
+  ;
+  ; AVX512F: uitofpv8i16v8double
+  ; AVX512F: cost of 22 {{.*}} uitofp
   %1 = uitofp <8 x i16> %a to <8 x double>
   ret <8 x double> %1
 }
@@ -97,10 +134,15 @@ define <8 x double> @uitofpv8i16v8double(<8 x i16> %a) {
 define <16 x double> @uitofpv16i16v16double(<16 x i16> %a) {
   ; SSE2: uitofpv16i16v16double
   ; SSE2: cost of 160 {{.*}} uitofp
-  ; SSE2-CODEGEN: uitofpv16i16v16double
-  ; SSE2-CODEGEN: movapd  LCPI
-  ; SSE2-CODEGEN: subpd
-  ; SSE2-CODEGEN: addpd
+  ;
+  ; AVX1: uitofpv16i16v16double
+  ; AVX1: cost of 40 {{.*}} uitofp
+  ;
+  ; AVX2: uitofpv16i16v16double
+  ; AVX2: cost of 40 {{.*}} uitofp
+  ;
+  ; AVX512F: uitofpv16i16v16double
+  ; AVX512F: cost of 44 {{.*}} uitofp
   %1 = uitofp <16 x i16> %a to <16 x double>
   ret <16 x double> %1
 }
@@ -108,10 +150,15 @@ define <16 x double> @uitofpv16i16v16double(<16 x i16> %a) {
 define <32 x double> @uitofpv32i16v32double(<32 x i16> %a) {
   ; SSE2: uitofpv32i16v32double
   ; SSE2: cost of 320 {{.*}} uitofp
-  ; SSE2-CODEGEN: uitofpv32i16v32double
-  ; SSE2-CODEGEN: movapd  LCPI
-  ; SSE2-CODEGEN: subpd
-  ; SSE2-CODEGEN: addpd
+  ;
+  ; AVX1: uitofpv32i16v32double
+  ; AVX1: cost of 80 {{.*}} uitofp
+  ;
+  ; AVX2: uitofpv32i16v32double
+  ; AVX2: cost of 80 {{.*}} uitofp
+  ;
+  ; AVX512F: uitofpv32i16v32double
+  ; AVX512F: cost of 88 {{.*}} uitofp
   %1 = uitofp <32 x i16> %a to <32 x double>
   ret <32 x double> %1
 }
@@ -119,10 +166,15 @@ define <32 x double> @uitofpv32i16v32double(<32 x i16> %a) {
 define <2 x double> @uitofpv2i32v2double(<2 x i32> %a) {
   ; SSE2: uitofpv2i32v2double
   ; SSE2: cost of 20 {{.*}} uitofp
-  ; SSE2-CODEGEN: uitofpv2i32v2double
-  ; SSE2-CODEGEN: movapd  LCPI
-  ; SSE2-CODEGEN: subpd
-  ; SSE2-CODEGEN: addpd
+  ;
+  ; AVX1: uitofpv2i32v2double
+  ; AVX1: cost of 4 {{.*}} uitofp
+  ;
+  ; AVX2: uitofpv2i32v2double
+  ; AVX2: cost of 4 {{.*}} uitofp
+  ;
+  ; AVX512F: uitofpv2i32v2double
+  ; AVX512F: cost of 4 {{.*}} uitofp
   %1 = uitofp <2 x i32> %a to <2 x double>
   ret <2 x double> %1
 }
@@ -130,10 +182,15 @@ define <2 x double> @uitofpv2i32v2double(<2 x i32> %a) {
 define <4 x double> @uitofpv4i32v4double(<4 x i32> %a) {
   ; SSE2: uitofpv4i32v4double
   ; SSE2: cost of 40 {{.*}} uitofp
-  ; SSE2-CODEGEN: uitofpv4i32v4double
-  ; SSE2-CODEGEN: movapd  LCPI
-  ; SSE2-CODEGEN: subpd
-  ; SSE2-CODEGEN: addpd
+  ;
+  ; AVX1: uitofpv4i32v4double
+  ; AVX1: cost of 6 {{.*}} uitofp
+  ;
+  ; AVX2: uitofpv4i32v4double
+  ; AVX2: cost of 6 {{.*}} uitofp
+  ;
+  ; AVX512F: uitofpv4i32v4double
+  ; AVX512F: cost of 6 {{.*}} uitofp
   %1 = uitofp <4 x i32> %a to <4 x double>
   ret <4 x double> %1
 }
@@ -141,10 +198,15 @@ define <4 x double> @uitofpv4i32v4double(<4 x i32> %a) {
 define <8 x double> @uitofpv8i32v8double(<8 x i32> %a) {
   ; SSE2: uitofpv8i32v8double
   ; SSE2: cost of 80 {{.*}} uitofp
-  ; SSE2-CODEGEN: uitofpv8i32v8double
-  ; SSE2-CODEGEN: movapd  LCPI
-  ; SSE2-CODEGEN: subpd
-  ; SSE2-CODEGEN: addpd
+  ;
+  ; AVX1: uitofpv8i32v8double
+  ; AVX1: cost of 20 {{.*}} uitofp
+  ;
+  ; AVX2: uitofpv8i32v8double
+  ; AVX2: cost of 20 {{.*}} uitofp
+  ;
+  ; AVX512F: uitofpv8i32v8double
+  ; AVX512F: cost of 22 {{.*}} uitofp
   %1 = uitofp <8 x i32> %a to <8 x double>
   ret <8 x double> %1
 }
@@ -152,10 +214,15 @@ define <8 x double> @uitofpv8i32v8double(<8 x i32> %a) {
 define <16 x double> @uitofpv16i32v16double(<16 x i32> %a) {
   ; SSE2: uitofpv16i32v16double
   ; SSE2: cost of 160 {{.*}} uitofp
-  ; SSE2-CODEGEN: uitofpv16i32v16double
-  ; SSE2-CODEGEN: movapd  LCPI
-  ; SSE2-CODEGEN: subpd
-  ; SSE2-CODEGEN: addpd
+  ;
+  ; AVX1: uitofpv16i32v16double
+  ; AVX1: cost of 40 {{.*}} uitofp
+  ;
+  ; AVX2: uitofpv16i32v16double
+  ; AVX2: cost of 40 {{.*}} uitofp
+  ;
+  ; AVX512F: uitofpv16i32v16double
+  ; AVX512F: cost of 44 {{.*}} uitofp
   %1 = uitofp <16 x i32> %a to <16 x double>
   ret <16 x double> %1
 }
@@ -163,10 +230,15 @@ define <16 x double> @uitofpv16i32v16double(<16 x i32> %a) {
 define <32 x double> @uitofpv32i32v32double(<32 x i32> %a) {
   ; SSE2: uitofpv32i32v32double
   ; SSE2: cost of 320 {{.*}} uitofp
-  ; SSE2-CODEGEN: uitofpv32i32v32double
-  ; SSE2-CODEGEN: movapd  LCPI
-  ; SSE2-CODEGEN: subpd
-  ; SSE2-CODEGEN: addpd
+  ;
+  ; AVX1: uitofpv32i32v32double
+  ; AVX1: cost of 80 {{.*}} uitofp
+  ;
+  ; AVX2: uitofpv32i32v32double
+  ; AVX2: cost of 80 {{.*}} uitofp
+  ;
+  ; AVX512F: uitofpv32i32v32double
+  ; AVX512F: cost of 88 {{.*}} uitofp
   %1 = uitofp <32 x i32> %a to <32 x double>
   ret <32 x double> %1
 }
@@ -174,10 +246,15 @@ define <32 x double> @uitofpv32i32v32double(<32 x i32> %a) {
 define <2 x double> @uitofpv2i64v2double(<2 x i64> %a) {
   ; SSE2: uitofpv2i64v2double
   ; SSE2: cost of 20 {{.*}} uitofp
-  ; SSE2-CODEGEN: uitofpv2i64v2double
-  ; SSE2-CODEGEN: movapd  LCPI
-  ; SSE2-CODEGEN: subpd
-  ; SSE2-CODEGEN: addpd
+  ;
+  ; AVX1: uitofpv2i64v2double
+  ; AVX1: cost of 20 {{.*}} uitofp
+  ;
+  ; AVX2: uitofpv2i64v2double
+  ; AVX2: cost of 20 {{.*}} uitofp
+  ;
+  ; AVX512F: uitofpv2i64v2double
+  ; AVX512F: cost of 20 {{.*}} uitofp
   %1 = uitofp <2 x i64> %a to <2 x double>
   ret <2 x double> %1
 }
@@ -185,32 +262,47 @@ define <2 x double> @uitofpv2i64v2double(<2 x i64> %a) {
 define <4 x double> @uitofpv4i64v4double(<4 x i64> %a) {
   ; SSE2: uitofpv4i64v4double
   ; SSE2: cost of 40 {{.*}} uitofp
-  ; SSE2-CODEGEN: uitofpv4i64v4double
-  ; SSE2-CODEGEN: movapd  LCPI
-  ; SSE2-CODEGEN: subpd
-  ; SSE2-CODEGEN: addpd
+  ;
+  ; AVX1: uitofpv4i64v4double
+  ; AVX1: cost of 40 {{.*}} uitofp
+  ;
+  ; AVX2: uitofpv4i64v4double
+  ; AVX2: cost of 40 {{.*}} uitofp
+  ;
+  ; AVX512F: uitofpv4i64v4double
+  ; AVX512F: cost of 40 {{.*}} uitofp
   %1 = uitofp <4 x i64> %a to <4 x double>
   ret <4 x double> %1
 }
 
 define <8 x double> @uitofpv8i64v8double(<8 x i64> %a) {
-  %1 = uitofp <8 x i64> %a to <8 x double>
   ; SSE2: uitofpv8i64v8double
   ; SSE2: cost of 80 {{.*}} uitofp
-  ; SSE2-CODEGEN: uitofpv8i64v8double
-  ; SSE2-CODEGEN: movapd  LCPI
-  ; SSE2-CODEGEN: subpd
-  ; SSE2-CODEGEN: addpd
+  ;
+  ; AVX1: uitofpv8i64v8double
+  ; AVX1: cost of 20 {{.*}} uitofp
+  ;
+  ; AVX2: uitofpv8i64v8double
+  ; AVX2: cost of 20 {{.*}} uitofp
+  ;
+  ; AVX512F: uitofpv8i64v8double
+  ; AVX512F: cost of 22 {{.*}} uitofp
+  %1 = uitofp <8 x i64> %a to <8 x double>
   ret <8 x double> %1
 }
 
 define <16 x double> @uitofpv16i64v16double(<16 x i64> %a) {
   ; SSE2: uitofpv16i64v16double
   ; SSE2: cost of 160 {{.*}} uitofp
-  ; SSE2-CODEGEN: uitofpv16i64v16double
-  ; SSE2-CODEGEN: movapd  LCPI
-  ; SSE2-CODEGEN: subpd
-  ; SSE2-CODEGEN: addpd
+  ;
+  ; AVX1: uitofpv16i64v16double
+  ; AVX1: cost of 40 {{.*}} uitofp
+  ;
+  ; AVX2: uitofpv16i64v16double
+  ; AVX2: cost of 40 {{.*}} uitofp
+  ;
+  ; AVX512F: uitofpv16i64v16double
+  ; AVX512F: cost of 44 {{.*}} uitofp
   %1 = uitofp <16 x i64> %a to <16 x double>
   ret <16 x double> %1
 }
@@ -218,10 +310,15 @@ define <16 x double> @uitofpv16i64v16double(<16 x i64> %a) {
 define <32 x double> @uitofpv32i64v32double(<32 x i64> %a) {
   ; SSE2: uitofpv32i64v32double
   ; SSE2: cost of 320 {{.*}} uitofp
-  ; SSE2-CODEGEN: uitofpv32i64v32double
-  ; SSE2-CODEGEN: movapd  LCPI
-  ; SSE2-CODEGEN: subpd
-  ; SSE2-CODEGEN: addpd
+  ;
+  ; AVX1: uitofpv32i64v32double
+  ; AVX1: cost of 80 {{.*}} uitofp
+  ;
+  ; AVX2: uitofpv32i64v32double
+  ; AVX2: cost of 80 {{.*}} uitofp
+  ;
+  ; AVX512F: uitofpv32i64v32double
+  ; AVX512F: cost of 88 {{.*}} uitofp
   %1 = uitofp <32 x i64> %a to <32 x double>
   ret <32 x double> %1
 }
@@ -229,6 +326,15 @@ define <32 x double> @uitofpv32i64v32double(<32 x i64> %a) {
 define <2 x float> @uitofpv2i8v2float(<2 x i8> %a) {
   ; SSE2: uitofpv2i8v2float
   ; SSE2: cost of 15 {{.*}} uitofp
+  ;
+  ; AVX1: uitofpv2i8v2float
+  ; AVX1: cost of 4 {{.*}} uitofp
+  ;
+  ; AVX2: uitofpv2i8v2float
+  ; AVX2: cost of 4 {{.*}} uitofp
+  ;
+  ; AVX512F: uitofpv2i8v2float
+  ; AVX512F: cost of 4 {{.*}} uitofp
   %1 = uitofp <2 x i8> %a to <2 x float>
   ret <2 x float> %1
 }
@@ -236,6 +342,15 @@ define <2 x float> @uitofpv2i8v2float(<2 x i8> %a) {
 define <4 x float> @uitofpv4i8v4float(<4 x i8> %a) {
   ; SSE2: uitofpv4i8v4float
   ; SSE2: cost of 8 {{.*}} uitofp
+  ;
+  ; AVX1: uitofpv4i8v4float
+  ; AVX1: cost of 2 {{.*}} uitofp
+  ;
+  ; AVX2: uitofpv4i8v4float
+  ; AVX2: cost of 2 {{.*}} uitofp
+  ;
+  ; AVX512F: uitofpv4i8v4float
+  ; AVX512F: cost of 2 {{.*}} uitofp
   %1 = uitofp <4 x i8> %a to <4 x float>
   ret <4 x float> %1
 }
@@ -243,6 +358,15 @@ define <4 x float> @uitofpv4i8v4float(<4 x i8> %a) {
 define <8 x float> @uitofpv8i8v8float(<8 x i8> %a) {
   ; SSE2: uitofpv8i8v8float
   ; SSE2: cost of 15 {{.*}} uitofp
+  ;
+  ; AVX1: uitofpv8i8v8float
+  ; AVX1: cost of 5 {{.*}} uitofp
+  ;
+  ; AVX2: uitofpv8i8v8float
+  ; AVX2: cost of 5 {{.*}} uitofp
+  ;
+  ; AVX512F: uitofpv8i8v8float
+  ; AVX512F: cost of 5 {{.*}} uitofp
   %1 = uitofp <8 x i8> %a to <8 x float>
   ret <8 x float> %1
 }
@@ -250,6 +374,15 @@ define <8 x float> @uitofpv8i8v8float(<8 x i8> %a) {
 define <16 x float> @uitofpv16i8v16float(<16 x i8> %a) {
   ; SSE2: uitofpv16i8v16float
   ; SSE2: cost of 8 {{.*}} uitofp
+  ;
+  ; AVX1: uitofpv16i8v16float
+  ; AVX1: cost of 44 {{.*}} uitofp
+  ;
+  ; AVX2: uitofpv16i8v16float
+  ; AVX2: cost of 44 {{.*}} uitofp
+  ;
+  ; AVX512F: uitofpv16i8v16float
+  ; AVX512F: cost of 46 {{.*}} uitofp
   %1 = uitofp <16 x i8> %a to <16 x float>
   ret <16 x float> %1
 }
@@ -257,6 +390,15 @@ define <16 x float> @uitofpv16i8v16float(<16 x i8> %a) {
 define <32 x float> @uitofpv32i8v32float(<32 x i8> %a) {
   ; SSE2: uitofpv32i8v32float
   ; SSE2: cost of 16 {{.*}} uitofp
+  ;
+  ; AVX1: uitofpv32i8v32float
+  ; AVX1: cost of 88 {{.*}} uitofp
+  ;
+  ; AVX2: uitofpv32i8v32float
+  ; AVX2: cost of 88 {{.*}} uitofp
+  ;
+  ; AVX512F: uitofpv32i8v32float
+  ; AVX512F: cost of 92 {{.*}} uitofp
   %1 = uitofp <32 x i8> %a to <32 x float>
   ret <32 x float> %1
 }
@@ -264,6 +406,15 @@ define <32 x float> @uitofpv32i8v32float(<32 x i8> %a) {
 define <2 x float> @uitofpv2i16v2float(<2 x i16> %a) {
   ; SSE2: uitofpv2i16v2float
   ; SSE2: cost of 15 {{.*}} uitofp
+  ;
+  ; AVX1: uitofpv2i16v2float
+  ; AVX1: cost of 4 {{.*}} uitofp
+  ;
+  ; AVX2: uitofpv2i16v2float
+  ; AVX2: cost of 4 {{.*}} uitofp
+  ;
+  ; AVX512F: uitofpv2i16v2float
+  ; AVX512F: cost of 4 {{.*}} uitofp
   %1 = uitofp <2 x i16> %a to <2 x float>
   ret <2 x float> %1
 }
@@ -271,6 +422,15 @@ define <2 x float> @uitofpv2i16v2float(<2 x i16> %a) {
 define <4 x float> @uitofpv4i16v4float(<4 x i16> %a) {
   ; SSE2: uitofpv4i16v4float
   ; SSE2: cost of 8 {{.*}} uitofp
+  ;
+  ; AVX1: uitofpv4i16v4float
+  ; AVX1: cost of 2 {{.*}} uitofp
+  ;
+  ; AVX2: uitofpv4i16v4float
+  ; AVX2: cost of 2 {{.*}} uitofp
+  ;
+  ; AVX512F: uitofpv4i16v4float
+  ; AVX512F: cost of 2 {{.*}} uitofp
   %1 = uitofp <4 x i16> %a to <4 x float>
   ret <4 x float> %1
 }
@@ -278,6 +438,15 @@ define <4 x float> @uitofpv4i16v4float(<4 x i16> %a) {
 define <8 x float> @uitofpv8i16v8float(<8 x i16> %a) {
   ; SSE2: uitofpv8i16v8float
   ; SSE2: cost of 15 {{.*}} uitofp
+  ;
+  ; AVX1: uitofpv8i16v8float
+  ; AVX1: cost of 5 {{.*}} uitofp
+  ;
+  ; AVX2: uitofpv8i16v8float
+  ; AVX2: cost of 5 {{.*}} uitofp
+  ;
+  ; AVX512F: uitofpv8i16v8float
+  ; AVX512F: cost of 5 {{.*}} uitofp
   %1 = uitofp <8 x i16> %a to <8 x float>
   ret <8 x float> %1
 }
@@ -285,6 +454,15 @@ define <8 x float> @uitofpv8i16v8float(<8 x i16> %a) {
 define <16 x float> @uitofpv16i16v16float(<16 x i16> %a) {
   ; SSE2: uitofpv16i16v16float
   ; SSE2: cost of 30 {{.*}} uitofp
+  ;
+  ; AVX1: uitofpv16i16v16float
+  ; AVX1: cost of 44 {{.*}} uitofp
+  ;
+  ; AVX2: uitofpv16i16v16float
+  ; AVX2: cost of 44 {{.*}} uitofp
+  ;
+  ; AVX512F: uitofpv16i16v16float
+  ; AVX512F: cost of 46 {{.*}} uitofp
   %1 = uitofp <16 x i16> %a to <16 x float>
   ret <16 x float> %1
 }
@@ -292,6 +470,15 @@ define <16 x float> @uitofpv16i16v16float(<16 x i16> %a) {
 define <32 x float> @uitofpv32i16v32float(<32 x i16> %a) {
   ; SSE2: uitofpv32i16v32float
   ; SSE2: cost of 60 {{.*}} uitofp
+  ;
+  ; AVX1: uitofpv32i16v32float
+  ; AVX1: cost of 88 {{.*}} uitofp
+  ;
+  ; AVX2: uitofpv32i16v32float
+  ; AVX2: cost of 88 {{.*}} uitofp
+  ;
+  ; AVX512F: uitofpv32i16v32float
+  ; AVX512F: cost of 92 {{.*}} uitofp
   %1 = uitofp <32 x i16> %a to <32 x float>
   ret <32 x float> %1
 }
@@ -299,6 +486,15 @@ define <32 x float> @uitofpv32i16v32float(<32 x i16> %a) {
 define <2 x float> @uitofpv2i32v2float(<2 x i32> %a) {
   ; SSE2: uitofpv2i32v2float
   ; SSE2: cost of 15 {{.*}} uitofp
+  ;
+  ; AVX1: uitofpv2i32v2float
+  ; AVX1: cost of 4 {{.*}} uitofp
+  ;
+  ; AVX2: uitofpv2i32v2float
+  ; AVX2: cost of 4 {{.*}} uitofp
+  ;
+  ; AVX512F: uitofpv2i32v2float
+  ; AVX512F: cost of 4 {{.*}} uitofp
   %1 = uitofp <2 x i32> %a to <2 x float>
   ret <2 x float> %1
 }
@@ -306,6 +502,15 @@ define <2 x float> @uitofpv2i32v2float(<2 x i32> %a) {
 define <4 x float> @uitofpv4i32v4float(<4 x i32> %a) {
   ; SSE2: uitofpv4i32v4float
   ; SSE2: cost of 8 {{.*}} uitofp
+  ;
+  ; AVX1: uitofpv4i32v4float
+  ; AVX1: cost of 6 {{.*}} uitofp
+  ;
+  ; AVX2: uitofpv4i32v4float
+  ; AVX2: cost of 6 {{.*}} uitofp
+  ;
+  ; AVX512F: uitofpv4i32v4float
+  ; AVX512F: cost of 6 {{.*}} uitofp
   %1 = uitofp <4 x i32> %a to <4 x float>
   ret <4 x float> %1
 }
@@ -313,6 +518,15 @@ define <4 x float> @uitofpv4i32v4float(<4 x i32> %a) {
 define <8 x float> @uitofpv8i32v8float(<8 x i32> %a) {
   ; SSE2: uitofpv8i32v8float
   ; SSE2: cost of 16 {{.*}} uitofp
+  ;
+  ; AVX1: uitofpv8i32v8float
+  ; AVX1: cost of 9 {{.*}} uitofp
+  ;
+  ; AVX2: uitofpv8i32v8float
+  ; AVX2: cost of 8 {{.*}} uitofp
+  ;
+  ; AVX512F: uitofpv8i32v8float
+  ; AVX512F: cost of 8 {{.*}} uitofp
   %1 = uitofp <8 x i32> %a to <8 x float>
   ret <8 x float> %1
 }
@@ -320,6 +534,15 @@ define <8 x float> @uitofpv8i32v8float(<8 x i32> %a) {
 define <16 x float> @uitofpv16i32v16float(<16 x i32> %a) {
   ; SSE2: uitofpv16i32v16float
   ; SSE2: cost of 32 {{.*}} uitofp
+  ;
+  ; AVX1: uitofpv16i32v16float
+  ; AVX1: cost of 44 {{.*}} uitofp
+  ;
+  ; AVX2: uitofpv16i32v16float
+  ; AVX2: cost of 44 {{.*}} uitofp
+  ;
+  ; AVX512F: uitofpv16i32v16float
+  ; AVX512F: cost of 46 {{.*}} uitofp
   %1 = uitofp <16 x i32> %a to <16 x float>
   ret <16 x float> %1
 }
@@ -327,6 +550,15 @@ define <16 x float> @uitofpv16i32v16float(<16 x i32> %a) {
 define <32 x float> @uitofpv32i32v32float(<32 x i32> %a) {
   ; SSE2: uitofpv32i32v32float
   ; SSE2: cost of 64 {{.*}} uitofp
+  ;
+  ; AVX1: uitofpv32i32v32float
+  ; AVX1: cost of 88 {{.*}} uitofp
+  ;
+  ; AVX2: uitofpv32i32v32float
+  ; AVX2: cost of 88 {{.*}} uitofp
+  ;
+  ; AVX512F: uitofpv32i32v32float
+  ; AVX512F: cost of 92 {{.*}} uitofp
   %1 = uitofp <32 x i32> %a to <32 x float>
   ret <32 x float> %1
 }
@@ -334,6 +566,15 @@ define <32 x float> @uitofpv32i32v32float(<32 x i32> %a) {
 define <2 x float> @uitofpv2i64v2float(<2 x i64> %a) {
   ; SSE2: uitofpv2i64v2float
   ; SSE2: cost of 15 {{.*}} uitofp
+  ;
+  ; AVX1: uitofpv2i64v2float
+  ; AVX1: cost of 4 {{.*}} uitofp
+  ;
+  ; AVX2: uitofpv2i64v2float
+  ; AVX2: cost of 4 {{.*}} uitofp
+  ;
+  ; AVX512F: uitofpv2i64v2float
+  ; AVX512F: cost of 4 {{.*}} uitofp
   %1 = uitofp <2 x i64> %a to <2 x float>
   ret <2 x float> %1
 }
@@ -341,6 +582,15 @@ define <2 x float> @uitofpv2i64v2float(<2 x i64> %a) {
 define <4 x float> @uitofpv4i64v4float(<4 x i64> %a) {
   ; SSE2: uitofpv4i64v4float
   ; SSE2: cost of 30 {{.*}} uitofp
+  ;
+  ; AVX1: uitofpv4i64v4float
+  ; AVX1: cost of 10 {{.*}} uitofp
+  ;
+  ; AVX2: uitofpv4i64v4float
+  ; AVX2: cost of 10 {{.*}} uitofp
+  ;
+  ; AVX512F: uitofpv4i64v4float
+  ; AVX512F: cost of 10 {{.*}} uitofp
   %1 = uitofp <4 x i64> %a to <4 x float>
   ret <4 x float> %1
 }
@@ -348,6 +598,15 @@ define <4 x float> @uitofpv4i64v4float(<4 x i64> %a) {
 define <8 x float> @uitofpv8i64v8float(<8 x i64> %a) {
   ; SSE2: uitofpv8i64v8float
   ; SSE2: cost of 60 {{.*}} uitofp
+  ;
+  ; AVX1: uitofpv8i64v8float
+  ; AVX1: cost of 22 {{.*}} uitofp
+  ;
+  ; AVX2: uitofpv8i64v8float
+  ; AVX2: cost of 22 {{.*}} uitofp
+  ;
+  ; AVX512F: uitofpv8i64v8float
+  ; AVX512F: cost of 22 {{.*}} uitofp
   %1 = uitofp <8 x i64> %a to <8 x float>
   ret <8 x float> %1
 }
@@ -355,6 +614,15 @@ define <8 x float> @uitofpv8i64v8float(<8 x i64> %a) {
 define <16 x float> @uitofpv16i64v16float(<16 x i64> %a) {
   ; SSE2: uitofpv16i64v16float
   ; SSE2: cost of 120 {{.*}} uitofp
+  ;
+  ; AVX1: uitofpv16i64v16float
+  ; AVX1: cost of 44 {{.*}} uitofp
+  ;
+  ; AVX2: uitofpv16i64v16float
+  ; AVX2: cost of 44 {{.*}} uitofp
+  ;
+  ; AVX512F: uitofpv16i64v16float
+  ; AVX512F: cost of 46 {{.*}} uitofp
   %1 = uitofp <16 x i64> %a to <16 x float>
   ret <16 x float> %1
 }
@@ -362,6 +630,15 @@ define <16 x float> @uitofpv16i64v16float(<16 x i64> %a) {
 define <32 x float> @uitofpv32i64v32float(<32 x i64> %a) {
   ; SSE2: uitofpv32i64v32float
   ; SSE2: cost of 240 {{.*}} uitofp
+  ;
+  ; AVX1: uitofpv32i64v32float
+  ; AVX1: cost of 88 {{.*}} uitofp
+  ;
+  ; AVX2: uitofpv32i64v32float
+  ; AVX2: cost of 88 {{.*}} uitofp
+  ;
+  ; AVX512F: uitofpv32i64v32float
+  ; AVX512F: cost of 92 {{.*}} uitofp
   %1 = uitofp <32 x i64> %a to <32 x float>
   ret <32 x float> %1
 }
diff --git a/test/Analysis/CostModel/X86/vshift-cost.ll b/test/Analysis/CostModel/X86/vshift-cost.ll
index 84d72463ac0..dd93badc893 100644
--- a/test/Analysis/CostModel/X86/vshift-cost.ll
+++ b/test/Analysis/CostModel/X86/vshift-cost.ll
@@ -62,9 +62,9 @@ define <2 x i64> @test5(<2 x i64> %a) {
   ret <2 x i64> %shl
 }
 ; CHECK: 'Cost Model Analysis' for function 'test5':
-; SSE2: Found an estimated cost of 20 for instruction:   %shl
-; SSE41: Found an estimated cost of 20 for instruction:   %shl
-; AVX: Found an estimated cost of 20 for instruction:   %shl
+; SSE2: Found an estimated cost of 4 for instruction:   %shl
+; SSE41: Found an estimated cost of 4 for instruction:   %shl
+; AVX: Found an estimated cost of 4 for instruction:   %shl
 ; AVX2: Found an estimated cost of 1 for instruction:   %shl
 
 
@@ -117,9 +117,9 @@ define <4 x i64> @test8(<4 x i64> %a) {
   ret <4 x i64> %shl
 }
 ; CHECK: 'Cost Model Analysis' for function 'test8':
-; SSE2: Found an estimated cost of 40 for instruction:   %shl
-; SSE41: Found an estimated cost of 40 for instruction:   %shl
-; AVX: Found an estimated cost of 40 for instruction:   %shl
+; SSE2: Found an estimated cost of 8 for instruction:   %shl
+; SSE41: Found an estimated cost of 8 for instruction:   %shl
+; AVX: Found an estimated cost of 8 for instruction:   %shl
 ; AVX2: Found an estimated cost of 1 for instruction:   %shl
 
 
@@ -159,9 +159,9 @@ define <8 x i64> @test11(<8 x i64> %a) {
   ret <8 x i64> %shl
 }
 ; CHECK: 'Cost Model Analysis' for function 'test11':
-; SSE2: Found an estimated cost of 80 for instruction:   %shl
-; SSE41: Found an estimated cost of 80 for instruction:   %shl
-; AVX: Found an estimated cost of 80 for instruction:   %shl
+; SSE2: Found an estimated cost of 16 for instruction:   %shl
+; SSE41: Found an estimated cost of 16 for instruction:   %shl
+; AVX: Found an estimated cost of 16 for instruction:   %shl
 ; AVX2: Found an estimated cost of 2 for instruction:   %shl
 
 
diff --git a/test/Analysis/CostModel/no_info.ll b/test/Analysis/CostModel/no_info.ll
index 5f3b56ad9cf..931669b6017 100644
--- a/test/Analysis/CostModel/no_info.ll
+++ b/test/Analysis/CostModel/no_info.ll
@@ -5,9 +5,27 @@
 
 ; -- No triple in this module --
 
-;CHECK: cost of 1 {{.*}} add
-;CHECK: cost of 1 {{.*}} ret
+; CHECK-LABEL: function 'no_info'
+; CHECK: cost of 1 {{.*}} add
+; CHECK: cost of 1 {{.*}} ret
 define i32 @no_info(i32 %arg) {
   %e = add i32 %arg, %arg
   ret i32 %e
 }
+
+define i8 @addressing_mode_reg_reg(i8* %a, i32 %b) {
+; CHECK-LABEL: function 'addressing_mode_reg_reg'
+  %p = getelementptr i8, i8* %a, i32 %b ; NoTTI accepts reg+reg addressing.
+; CHECK: cost of 0 {{.*}} getelementptr
+  %v = load i8, i8* %p
+  ret i8 %v
+}
+
+; CHECK-LABEL: function 'addressing_mode_scaled_reg'
+define i32 @addressing_mode_scaled_reg(i32* %a, i32 %b) {
+  ; NoTTI rejects reg+scale*reg addressing.
+  %p = getelementptr i32, i32* %a, i32 %b
+; CHECK: cost of 1 {{.*}} getelementptr
+  %v = load i32, i32* %p
+  ret i32 %v
+}
diff --git a/test/Analysis/Delinearization/divide_by_one.ll b/test/Analysis/Delinearization/divide_by_one.ll
new file mode 100644
index 00000000000..c6e0b134e04
--- /dev/null
+++ b/test/Analysis/Delinearization/divide_by_one.ll
@@ -0,0 +1,70 @@
+; RUN: opt < %s -analyze -delinearize | FileCheck %s
+
+target datalayout = "e-m:e-p:32:32-i1:32-i64:64-a:0-n32"
+
+; Check that division by 1 can be delinearized.
+;
+; void test1(unsigned char *dst, int stride, int bs) {
+;   for (int r = bs; r >= 0; --r)
+;     for (int c = 0; c < bs; ++c)
+;       dst[r * stride + c] = dst[(r + 1) * stride + c - 1];
+; }
+
+; AddRec: {{(-1 + ((1 + %bs) * %stride)),+,(-1 * %stride)}<%for.cond1.preheader>,+,1}<nw><%for.body3>
+; CHECK: Inst:  %0 = load i8, i8* %arrayidx, align 1
+; CHECK: Base offset: %dst
+; CHECK: ArrayDecl[UnknownSize][%stride] with elements of 1 bytes.
+; CHECK: ArrayRef[{(1 + %bs),+,-1}<nw><%for.cond1.preheader>][{-1,+,1}<nw><%for.body3>]
+
+; AddRec: {{(%stride * %bs),+,(-1 * %stride)}<%for.cond1.preheader>,+,1}<nw><%for.body3>
+; CHECK: Inst:  store i8 %0, i8* %arrayidx7, align 1
+; CHECK: Base offset: %dst
+; CHECK: ArrayDecl[UnknownSize][%stride] with elements of 1 bytes.
+; CHECK: ArrayRef[{%bs,+,-1}<nsw><%for.cond1.preheader>][{0,+,1}<nuw><nsw><%for.body3>]
+
+define void @test(i8* nocapture %dst, i32 %stride, i32 %bs) {
+entry:
+  %cmp20 = icmp sgt i32 %bs, -1
+  br i1 %cmp20, label %for.cond1.preheader.lr.ph, label %for.end9
+
+for.cond1.preheader.lr.ph:
+  %cmp218 = icmp slt i32 0, %bs
+  br label %for.cond1.preheader
+
+for.cond1.preheader:
+  %r.021 = phi i32 [ %bs, %for.cond1.preheader.lr.ph ], [ %dec, %for.inc8 ]
+  br i1 %cmp218, label %for.body3.lr.ph, label %for.inc8
+
+for.body3.lr.ph:
+  %add = add nsw i32 %r.021, 1
+  %mul = mul nsw i32 %add, %stride
+  %add4 = add i32 %mul, -1
+  %mul5 = mul nsw i32 %r.021, %stride
+  br label %for.body3
+
+for.body3:
+  %c.019 = phi i32 [ 0, %for.body3.lr.ph ], [ %inc, %for.body3 ]
+  %sub = add i32 %add4, %c.019
+  %arrayidx = getelementptr inbounds i8, i8* %dst, i32 %sub
+  %0 = load i8, i8* %arrayidx, align 1
+  %add6 = add nsw i32 %c.019, %mul5
+  %arrayidx7 = getelementptr inbounds i8, i8* %dst, i32 %add6
+  store i8 %0, i8* %arrayidx7, align 1
+  %inc = add nsw i32 %c.019, 1
+  %cmp2 = icmp slt i32 %inc, %bs
+  br i1 %cmp2, label %for.body3, label %for.cond1.for.inc8_crit_edge
+
+for.cond1.for.inc8_crit_edge:
+  br label %for.inc8
+
+for.inc8:
+  %dec = add nsw i32 %r.021, -1
+  %cmp = icmp sgt i32 %dec, -1
+  br i1 %cmp, label %for.cond1.preheader, label %for.cond.for.end9_crit_edge
+
+for.cond.for.end9_crit_edge:
+  br label %for.end9
+
+for.end9:
+  ret void
+}
diff --git a/test/Analysis/Delinearization/multidim_ivs_and_integer_offsets_3d.ll b/test/Analysis/Delinearization/multidim_ivs_and_integer_offsets_3d.ll
index 317e62c8ef9..4ae976281c4 100644
--- a/test/Analysis/Delinearization/multidim_ivs_and_integer_offsets_3d.ll
+++ b/test/Analysis/Delinearization/multidim_ivs_and_integer_offsets_3d.ll
@@ -11,7 +11,7 @@
 ; AddRec: {{{(56 + (8 * (-4 + (3 * %m)) * %o) + %A),+,(8 * %m * %o)}<%for.i>,+,(8 * %o)}<%for.j>,+,8}<%for.k>
 ; CHECK: Base offset: %A
 ; CHECK: ArrayDecl[UnknownSize][%m][%o] with elements of 8 bytes.
-; CHECK: ArrayRef[{3,+,1}<nw><%for.i>][{-4,+,1}<nw><%for.j>][{7,+,1}<nw><%for.k>]
+; CHECK: ArrayRef[{3,+,1}<nw><%for.i>][{-4,+,1}<nw><%for.j>][{7,+,1}<nuw><nsw><%for.k>]
 
 define void @foo(i64 %n, i64 %m, i64 %o, double* %A) {
 entry:
diff --git a/test/Analysis/Delinearization/multidim_ivs_and_parameteric_offsets_3d.ll b/test/Analysis/Delinearization/multidim_ivs_and_parameteric_offsets_3d.ll
index 9e37b76e59b..893c542c06a 100644
--- a/test/Analysis/Delinearization/multidim_ivs_and_parameteric_offsets_3d.ll
+++ b/test/Analysis/Delinearization/multidim_ivs_and_parameteric_offsets_3d.ll
@@ -11,7 +11,7 @@
 ; AddRec: {{{((8 * ((((%m * %p) + %q) * %o) + %r)) + %A),+,(8 * %m * %o)}<%for.i>,+,(8 * %o)}<%for.j>,+,8}<%for.k>
 ; CHECK: Base offset: %A
 ; CHECK: ArrayDecl[UnknownSize][%m][%o] with elements of 8 bytes.
-; CHECK: ArrayRef[{%p,+,1}<nw><%for.i>][{%q,+,1}<nw><%for.j>][{%r,+,1}<nw><%for.k>]
+; CHECK: ArrayRef[{%p,+,1}<nw><%for.i>][{%q,+,1}<nw><%for.j>][{%r,+,1}<nsw><%for.k>]
 
 define void @foo(i64 %n, i64 %m, i64 %o, double* %A, i64 %p, i64 %q, i64 %r) {
 entry:
diff --git a/test/Analysis/Delinearization/type_mismatch.ll b/test/Analysis/Delinearization/type_mismatch.ll
new file mode 100644
index 00000000000..0aa9a96cb1f
--- /dev/null
+++ b/test/Analysis/Delinearization/type_mismatch.ll
@@ -0,0 +1,29 @@
+; RUN: opt < %s -analyze -delinearize
+; REQUIRES: asserts
+
+; Test that SCEV divide code doesn't crash when attempting to create a SCEV
+; with operands of different types. In this case, the visitAddRecExpr
+; function tries to create an AddRec where the start and step are different
+; types.
+
+target datalayout = "e-m:e-p:32:32-i64:64-a:0-v32:32-n16:32"
+
+define fastcc void @test() {
+entry:
+  %0 = load i16, i16* undef, align 2
+  %conv21 = zext i16 %0 to i32
+  br label %for.cond7.preheader
+
+for.cond7.preheader:
+  %p1.022 = phi i8* [ undef, %entry ], [ %add.ptr, %for.end ]
+  br label %for.body11
+
+for.body11:
+  %arrayidx.phi = phi i8* [ %p1.022, %for.cond7.preheader ], [ undef, %for.body11 ]
+  store i8 undef, i8* %arrayidx.phi, align 1
+  br i1 undef, label %for.body11, label %for.end
+
+for.end:
+  %add.ptr = getelementptr inbounds i8, i8* %p1.022, i32 %conv21
+  br label %for.cond7.preheader
+}
diff --git a/test/Analysis/DependenceAnalysis/NonCanonicalizedSubscript.ll b/test/Analysis/DependenceAnalysis/NonCanonicalizedSubscript.ll
index 1b47341ed4d..6a5e42fbd55 100644
--- a/test/Analysis/DependenceAnalysis/NonCanonicalizedSubscript.ll
+++ b/test/Analysis/DependenceAnalysis/NonCanonicalizedSubscript.ll
@@ -38,3 +38,73 @@ for.body:
 for.end:
   ret void
 }
+
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+;  unsigned i, j;
+;  for (i = 1; i < SIZE; i++) {
+;    for (j = i; j < SIZE; j++) {
+;      a[i][j] = a[i+1][j-1] + 2;
+;    }
+;  }
+;  Extends the previous example to coupled MIV subscripts.
+
+
+@a = global [10004 x [10004 x i32]] zeroinitializer, align 16
+
+; Function Attrs: nounwind uwtable
+define void @coupled_miv_type_mismatch(i32 %n) #0 {
+; CHECK-LABEL: 'Dependence Analysis' for function 'coupled_miv_type_mismatch'
+; DELIN-LABEL: 'Dependence Analysis' for function 'coupled_miv_type_mismatch'
+entry:
+  br label %for.cond
+
+; CHECK: da analyze - input [0 *]!
+; CHECK: da analyze - anti [1 *]!
+; CHECK: da analyze - none!
+; DELIN: da analyze - input [0 *]!
+; DELIN: da analyze - anti [1 *]!
+; DELIN: da analyze - none!
+for.cond:                                         ; preds = %for.inc11, %entry
+  %indvars.iv11 = phi i64 [ %indvars.iv.next12, %for.inc11 ], [ 1, %entry ]
+  %exitcond14 = icmp ne i64 %indvars.iv11, 10000
+  br i1 %exitcond14, label %for.cond1.preheader, label %for.end13
+
+for.cond1.preheader:                              ; preds = %for.cond
+  %0 = trunc i64 %indvars.iv11 to i32
+  br label %for.cond1
+
+for.cond1:                                        ; preds = %for.cond1.preheader, %for.body3
+  %indvars.iv8 = phi i64 [ %indvars.iv11, %for.cond1.preheader ], [ %indvars.iv.next9, %for.body3 ]
+  %j.0 = phi i32 [ %inc, %for.body3 ], [ %0, %for.cond1.preheader ]
+  %lftr.wideiv = trunc i64 %indvars.iv8 to i32
+  %exitcond = icmp ne i32 %lftr.wideiv, 10000
+  br i1 %exitcond, label %for.body3, label %for.inc11
+
+for.body3:                                        ; preds = %for.cond1
+  %sub = add nsw i32 %j.0, -1
+  %idxprom = zext i32 %sub to i64
+  %1 = add nuw nsw i64 %indvars.iv11, 1
+  %arrayidx5 = getelementptr inbounds [10004 x [10004 x i32]], [10004 x [10004 x i32]]* @a, i64 0, i64 %1, i64 %idxprom
+  %2 = load i32, i32* %arrayidx5, align 4
+  %add6 = add nsw i32 %2, 2
+  %arrayidx10 = getelementptr inbounds [10004 x [10004 x i32]], [10004 x [10004 x i32]]* @a, i64 0, i64 %indvars.iv11, i64 %indvars.iv8
+  store i32 %add6, i32* %arrayidx10, align 4
+  %indvars.iv.next9 = add nuw nsw i64 %indvars.iv8, 1
+  %inc = add nuw nsw i32 %j.0, 1
+  br label %for.cond1
+
+for.inc11:                                        ; preds = %for.cond1
+  %indvars.iv.next12 = add nuw nsw i64 %indvars.iv11, 1
+  br label %for.cond
+
+for.end13:                                        ; preds = %for.cond
+  ret void
+}
+
+attributes #0 = { nounwind uwtable "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+sse,+sse2" "unsafe-fp-math"="false" "use-soft-float"="false" }
+
+!llvm.ident = !{!0}
+
+!0 = !{!"clang version 3.7.0 (https://vaivaswatha@bitbucket.org/compilertree/amd_clang.git 93a05fb75ee3411d24e8b2b184fc766a5318403e) (https://vaivaswatha@bitbucket.org/compilertree/amd_llvm.git 166d93d26efc912b517739f64d054a435e8e95cd)"}
diff --git a/test/Analysis/DependenceAnalysis/PR21585.ll b/test/Analysis/DependenceAnalysis/PR21585.ll
new file mode 100644
index 00000000000..bb62c68918f
--- /dev/null
+++ b/test/Analysis/DependenceAnalysis/PR21585.ll
@@ -0,0 +1,105 @@
+; RUN: opt < %s -analyze -basicaa -globalsmodref-aa -da | FileCheck %s
+define void @i32_subscript(i32* %a) {
+entry:
+  br label %for.body
+
+for.body:
+  %i = phi i32 [ 0, %entry ], [ %i.inc, %for.body ]
+  %a.addr = getelementptr i32, i32* %a, i32 %i
+  %a.addr.2 = getelementptr i32, i32* %a, i32 5
+  %0 = load i32, i32* %a.addr, align 4
+  %1 = add i32 %0, 1
+  store i32 %1, i32* %a.addr.2, align 4
+  %i.inc = add nsw i32 %i, 1
+  %i.inc.ext = sext i32 %i to i64
+  %exitcond = icmp ne i64 %i.inc.ext, 100
+  br i1 %exitcond, label %for.body, label %for.end
+
+for.end:
+  ret void
+}
+; CHECK: none
+; CHECK: anti
+; CHECK: output
+
+
+; Test for a bug, which caused an assert in ScalarEvolution because
+; the Dependence Analyzer attempted to zero extend a type to a smaller
+; type.
+
+; void t(unsigned int *a, unsigned int n) {
+;   for (unsigned int i = 0; i != n; i++) {
+;     a[(unsigned short)i] = g;
+;  }}
+
+@g = common global i32 0, align 4
+
+define void @t(i32* noalias %a, i32 %n) nounwind {
+entry:
+  %cmp1 = icmp eq i32 %n, 0
+  br i1 %cmp1, label %for.end, label %for.body
+
+for.body:
+  %i.02 = phi i32 [ %inc, %for.body ], [ 0, %entry ]
+  %0 = load i32, i32* @g, align 4
+  %idxprom = and i32 %i.02, 65535
+  %arrayidx = getelementptr inbounds i32, i32* %a, i32 %idxprom
+  store i32 %0, i32* %arrayidx, align 4
+  %inc = add i32 %i.02, 1
+  %cmp = icmp eq i32 %inc, %n
+  br i1 %cmp, label %for.end, label %for.body
+
+for.end:
+  ret void
+}
+; CHECK: input
+; CHECK: none
+; CHECK: output
+
+define void @i16_wrap(i64* %a) {
+entry:
+  br label %for.body
+for.body:
+  %i = phi i64 [0, %entry], [%i.inc, %for.inc]
+  %i.tr = trunc i64 %i to i16
+  %idx = getelementptr i64, i64* %a, i16 %i.tr
+  %0 = load i64, i64* %idx
+  %1 = add i64 %0, 1
+store i64 %1, i64* %idx
+  br label %for.inc
+
+for.inc:
+  %i.inc = add nuw i64 %i, 1
+  %cmp = icmp ult i64 %i.inc, 17179869184
+  br i1 %cmp, label %for.body, label %for.end
+for.end:
+  ret void
+}
+; CHECK: input
+; CHECK: anti
+; CHECK: output
+
+define void @i8_stride_wrap(i32* noalias %a, i32* noalias %b) {
+entry:
+  br label %for.body
+for.body:
+  %i = phi i32 [1,%entry], [%i.inc, %for.inc]
+  %i.tr = trunc i32 %i to i8
+  %idx = getelementptr i32, i32* %a, i8 %i.tr
+  %idx.2 = getelementptr i32, i32* %b, i32 %i
+  %0 = load i32, i32* %idx, align 4
+  %1 = add i32 %0, 1
+  store i32 %1, i32* %idx.2, align 4
+  br label %for.inc
+
+for.inc:
+  %i.inc = add nsw i32 %i, 256
+  %exitcond = icmp ult i32 %i, 65536
+  br i1 %exitcond, label %for.body, label %for.end
+
+for.end:
+  ret void
+}
+; CHECK: input
+; CHECK: none
+; CHECK: none
diff --git a/test/Analysis/Dominators/invoke.ll b/test/Analysis/Dominators/invoke.ll
index ce5f992d8f4..ab0afd4354a 100644
--- a/test/Analysis/Dominators/invoke.ll
+++ b/test/Analysis/Dominators/invoke.ll
@@ -1,7 +1,7 @@
 ; RUN: opt -verify -disable-output < %s
 ; This tests that we handle unreachable blocks correctly
 
-define void @f() {
+define void @f() personality i8* bitcast (i32 (...)* @__gxx_personality_v0 to i8*) {
   %v1 = invoke i32* @g()
           to label %bb1 unwind label %bb2
   invoke void @__dynamic_cast()
@@ -10,7 +10,7 @@ bb1:
   %Hidden = getelementptr inbounds i32, i32* %v1, i64 1
   ret void
 bb2:
-  %lpad.loopexit80 = landingpad { i8*, i32 } personality i8* bitcast (i32 (...)* @__gxx_personality_v0 to i8*)
+  %lpad.loopexit80 = landingpad { i8*, i32 }
           cleanup
   ret void
 }
diff --git a/test/Analysis/GlobalsModRef/aliastest.ll b/test/Analysis/GlobalsModRef/aliastest.ll
index 3474e130de8..1d7585ca24b 100644
--- a/test/Analysis/GlobalsModRef/aliastest.ll
+++ b/test/Analysis/GlobalsModRef/aliastest.ll
@@ -1,4 +1,7 @@
-; RUN: opt < %s -basicaa -globalsmodref-aa -gvn -S | FileCheck %s
+; RUN: opt < %s -basicaa -globalsmodref-aa -gvn -S -enable-unsafe-globalsmodref-alias-results | FileCheck %s
+;
+; Note that this test relies on an unsafe feature of GlobalsModRef. While this
+; test is correct and safe, GMR's technique for handling this isn't generally.
 
 @X = internal global i32 4		; <i32*> [#uses=1]
 
diff --git a/test/Analysis/GlobalsModRef/atomic-instrs.ll b/test/Analysis/GlobalsModRef/atomic-instrs.ll
new file mode 100644
index 00000000000..9e163ec4d20
--- /dev/null
+++ b/test/Analysis/GlobalsModRef/atomic-instrs.ll
@@ -0,0 +1,37 @@
+; RUN: opt < %s -globalsmodref-aa -gvn -S | FileCheck %s
+
+@X = internal global i32 4
+
+define i32 @test_cmpxchg(i32* %P) {
+; CHECK-LABEL: @test_cmpxchg
+; CHECK-NEXT: store i32 12, i32* @X
+; CHECK-NEXT: call void @modrefX_cmpxchg()
+; CHECK-NEXT: %V = load i32, i32* @X
+; CHECK-NEXT: ret i32 %V
+  store i32 12, i32* @X
+  call void @modrefX_cmpxchg()
+  %V = load i32, i32* @X
+  ret i32 %V
+}
+
+define void @modrefX_cmpxchg() {
+  %1 = cmpxchg i32* @X, i32 0, i32 -1 monotonic monotonic
+  ret void
+}
+
+define i32 @test_atomicrmw(i32* %P) {
+; CHECK-LABEL: @test_atomicrmw
+; CHECK-NEXT: store i32 12, i32* @X
+; CHECK-NEXT: call void @modrefXatomicrmw()
+; CHECK-NEXT: %V = load i32, i32* @X
+; CHECK-NEXT: ret i32 %V
+  store i32 12, i32* @X
+  call void @modrefXatomicrmw()
+  %V = load i32, i32* @X
+  ret i32 %V
+}
+
+define void @modrefXatomicrmw() {
+  %1 = atomicrmw add i32* @X, i32 1 acquire
+  ret void
+}
diff --git a/test/Analysis/GlobalsModRef/indirect-global.ll b/test/Analysis/GlobalsModRef/indirect-global.ll
index 02813232467..992764e5bae 100644
--- a/test/Analysis/GlobalsModRef/indirect-global.ll
+++ b/test/Analysis/GlobalsModRef/indirect-global.ll
@@ -1,4 +1,7 @@
-; RUN: opt < %s -basicaa -globalsmodref-aa -gvn -instcombine -S | FileCheck %s
+; RUN: opt < %s -basicaa -globalsmodref-aa -gvn -instcombine -S -enable-unsafe-globalsmodref-alias-results | FileCheck %s
+;
+; Note that this test relies on an unsafe feature of GlobalsModRef. While this
+; test is correct and safe, GMR's technique for handling this isn't generally.
 
 @G = internal global i32* null		; <i32**> [#uses=3]
 
diff --git a/test/Analysis/GlobalsModRef/nonescaping-noalias.ll b/test/Analysis/GlobalsModRef/nonescaping-noalias.ll
new file mode 100644
index 00000000000..6f54fbbe3e6
--- /dev/null
+++ b/test/Analysis/GlobalsModRef/nonescaping-noalias.ll
@@ -0,0 +1,62 @@
+; RUN: opt < %s -globalsmodref-aa -gvn -S | FileCheck %s
+;
+; This tests the safe no-alias conclusions of GMR -- when there is
+; a non-escaping global as one indentified underlying object and some pointer
+; that would inherently have escaped any other function as the other underlying
+; pointer of an alias query.
+
+@g1 = internal global i32 0
+
+define i32 @test1(i32* %param) {
+; Ensure that we can fold a store to a load of a global across a store to
+; a parameter when the global is non-escaping.
+;
+; CHECK-LABEL: @test1(
+; CHECK: store i32 42, i32* @g1
+; CHECK-NOT: load i32
+; CHECK: ret i32 42
+entry:
+  store i32 42, i32* @g1
+  store i32 7, i32* %param
+  %v = load i32, i32* @g1
+  ret i32 %v
+}
+
+declare i32* @f()
+
+define i32 @test2() {
+; Ensure that we can fold a store to a load of a global across a store to
+; the pointer returned by a function call. Since the global could not escape,
+; this function cannot be returning its address.
+;
+; CHECK-LABEL: @test2(
+; CHECK: store i32 42, i32* @g1
+; CHECK-NOT: load i32
+; CHECK: ret i32 42
+entry:
+  %ptr = call i32* @f() readnone
+  store i32 42, i32* @g1
+  store i32 7, i32* %ptr
+  %v = load i32, i32* @g1
+  ret i32 %v
+}
+
+@g2 = external global i32*
+
+define i32 @test3() {
+; Ensure that we can fold a store to a load of a global across a store to
+; the pointer loaded from that global. Because the global does not escape, it
+; cannot alias a pointer loaded out of a global.
+;
+; CHECK-LABEL: @test3(
+; CHECK: store i32 42, i32* @g1
+; CHECK: store i32 7, i32*
+; CHECK-NOT: load i32
+; CHECK: ret i32 42
+entry:
+  store i32 42, i32* @g1
+  %ptr1 = load i32*, i32** @g2
+  store i32 7, i32* %ptr1
+  %v = load i32, i32* @g1
+  ret i32 %v
+}
diff --git a/test/Analysis/LazyCallGraph/basic.ll b/test/Analysis/LazyCallGraph/basic.ll
index 7c13d2bef39..fce453bc15d 100644
--- a/test/Analysis/LazyCallGraph/basic.ll
+++ b/test/Analysis/LazyCallGraph/basic.ll
@@ -63,7 +63,7 @@ entry:
   ret void
 }
 
-define void ()* @test1(void ()** %x) {
+define void ()* @test1(void ()** %x) personality i32 (...)* @__gxx_personality_v0 {
 ; CHECK-LABEL: Call edges in function: test1
 ; CHECK-NEXT: -> f12
 ; CHECK-NEXT: -> f11
@@ -97,7 +97,7 @@ exit:
   ret void ()* @f11
 
 unwind:
-  %res = landingpad { i8*, i32 } personality i32 (...)* @__gxx_personality_v0
+  %res = landingpad { i8*, i32 }
           cleanup
   resume { i8*, i32 } { i8* bitcast (void ()* @f12 to i8*), i32 42 }
 }
diff --git a/test/Analysis/LazyCallGraph/non-leaf-intrinsics.ll b/test/Analysis/LazyCallGraph/non-leaf-intrinsics.ll
new file mode 100644
index 00000000000..01782e0f2c4
--- /dev/null
+++ b/test/Analysis/LazyCallGraph/non-leaf-intrinsics.ll
@@ -0,0 +1,26 @@
+; RUN: opt -S -disable-output -passes=print-cg < %s 2>&1 | FileCheck %s
+
+declare void @llvm.experimental.patchpoint.void(i64, i32, i8*, i32, ...)
+declare i32 @llvm.experimental.gc.statepoint.p0f_isVoidf(i64, i32, void ()*, i32, i32, ...)
+
+define private void @f() {
+  ret void
+}
+
+define void @calls_statepoint(i8 addrspace(1)* %arg) gc "statepoint-example" {
+; CHECK: Call edges in function: calls_statepoint
+; CHECK-NEXT:  -> f
+entry:
+  %cast = bitcast i8 addrspace(1)* %arg to i64 addrspace(1)*
+  %safepoint_token = call i32 (i64, i32, void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 0, i32 0, void ()* @f, i32 0, i32 0, i32 0, i32 5, i32 0, i32 0, i32 0, i32 10, i32 0, i8 addrspace(1)* %arg, i64 addrspace(1)* %cast, i8 addrspace(1)* %arg, i8 addrspace(1)* %arg)
+  ret void
+}
+
+define void @calls_patchpoint() {
+; CHECK:  Call edges in function: calls_patchpoint
+; CHECK-NEXT:    -> f
+entry:
+  %c = bitcast void()* @f to i8*
+  tail call void (i64, i32, i8*, i32, ...) @llvm.experimental.patchpoint.void(i64 1, i32 15, i8* %c, i32 0, i16 65535, i16 -1, i32 65536, i32 2000000000, i32 2147483647, i32 -1, i32 4294967295, i32 4294967296, i64 2147483648, i64 4294967295, i64 4294967296, i64 -1)
+  ret void
+}
diff --git a/test/Analysis/Lint/cppeh-catch-intrinsics-clean.ll b/test/Analysis/Lint/cppeh-catch-intrinsics-clean.ll
index 8cd44c86a72..743ebace700 100644
--- a/test/Analysis/Lint/cppeh-catch-intrinsics-clean.ll
+++ b/test/Analysis/Lint/cppeh-catch-intrinsics-clean.ll
@@ -12,13 +12,13 @@ declare void @llvm.eh.endcatch()
 @_ZTIi = external constant i8*
 
 ; Function Attrs: uwtable
-define void @test_ref_clean() {
+define void @test_ref_clean() personality i8* bitcast (i32 (...)* @__CxxFrameHandler3 to i8*) {
 entry:
   invoke void @_Z9may_throwv()
           to label %try.cont unwind label %lpad
 
 lpad:                                             ; preds = %entry
-  %0 = landingpad { i8*, i32 } personality i8* bitcast (i32 (...)* @__CxxFrameHandler3 to i8*)
+  %0 = landingpad { i8*, i32 }
           catch i8* bitcast (i8** @_ZTIi to i8*)
   %exn = extractvalue { i8*, i32 } %0, 0
   %sel = extractvalue { i8*, i32 } %0, 1
@@ -43,7 +43,7 @@ eh.resume:                                        ; preds = %catch.dispatch
 }
 
 ; Function Attrs: uwtable
-define void @test_ref_clean_multibranch() {
+define void @test_ref_clean_multibranch() personality i8* bitcast (i32 (...)* @__CxxFrameHandler3 to i8*) {
 entry:
   invoke void @_Z9may_throwv()
           to label %invoke.cont unwind label %lpad
@@ -53,7 +53,7 @@ invoke.cont:
           to label %invoke.cont unwind label %lpad1
 
 lpad:                                             ; preds = %entry
-  %0 = landingpad { i8*, i32 } personality i8* bitcast (i32 (...)* @__CxxFrameHandler3 to i8*)
+  %0 = landingpad { i8*, i32 }
           catch i8* bitcast (i8** @_ZTIi to i8*)
   %exn = extractvalue { i8*, i32 } %0, 0
   %sel = extractvalue { i8*, i32 } %0, 1
@@ -65,7 +65,7 @@ lpad:                                             ; preds = %entry
           to label %try.cont unwind label %lpad
 
 lpad1:                                            ; preds = %entry
-  %l1.0 = landingpad { i8*, i32 } personality i8* bitcast (i32 (...)* @__CxxFrameHandler3 to i8*)
+  %l1.0 = landingpad { i8*, i32 }
 		  cleanup
           catch i8* bitcast (i8** @_ZTIi to i8*)
   %exn1 = extractvalue { i8*, i32 } %l1.0, 0
diff --git a/test/Analysis/Lint/cppeh-catch-intrinsics.ll b/test/Analysis/Lint/cppeh-catch-intrinsics.ll
index 3a0c487c290..19480a2f60f 100644
--- a/test/Analysis/Lint/cppeh-catch-intrinsics.ll
+++ b/test/Analysis/Lint/cppeh-catch-intrinsics.ll
@@ -13,7 +13,7 @@ declare void @llvm.eh.endcatch()
 @_ZTIi = external constant i8*
 
 ; Function Attrs: uwtable
-define void @test_missing_endcatch() {
+define void @test_missing_endcatch() personality i8* bitcast (i32 (...)* @__CxxFrameHandler3 to i8*) {
 ; CHECK: Some paths from llvm.eh.begincatch may not reach llvm.eh.endcatch
 ; CHECK-NEXT: call void @llvm.eh.begincatch(i8* %exn, i8* null)
 entry:
@@ -21,7 +21,7 @@ entry:
           to label %try.cont unwind label %lpad
 
 lpad:                                             ; preds = %entry
-  %0 = landingpad { i8*, i32 } personality i8* bitcast (i32 (...)* @__CxxFrameHandler3 to i8*)
+  %0 = landingpad { i8*, i32 }
           catch i8* bitcast (i8** @_ZTIi to i8*)
   %exn = extractvalue { i8*, i32 } %0, 0
   %sel = extractvalue { i8*, i32 } %0, 1
@@ -45,7 +45,7 @@ eh.resume:                                        ; preds = %catch.dispatch
 }
 
 ; Function Attrs: uwtable
-define void @test_missing_begincatch() {
+define void @test_missing_begincatch() personality i8* bitcast (i32 (...)* @__CxxFrameHandler3 to i8*) {
 ; CHECK: llvm.eh.endcatch may be reachable without passing llvm.eh.begincatch
 ; CHECK-NEXT:  call void @llvm.eh.endcatch()
 entry:
@@ -53,7 +53,7 @@ entry:
           to label %try.cont unwind label %lpad
 
 lpad:                                             ; preds = %entry
-  %0 = landingpad { i8*, i32 } personality i8* bitcast (i32 (...)* @__CxxFrameHandler3 to i8*)
+  %0 = landingpad { i8*, i32 }
           catch i8* bitcast (i8** @_ZTIi to i8*)
   %exn = extractvalue { i8*, i32 } %0, 0
   %sel = extractvalue { i8*, i32 } %0, 1
@@ -77,7 +77,7 @@ eh.resume:                                        ; preds = %catch.dispatch
 }
 
 ; Function Attrs: uwtable
-define void @test_multiple_begin() {
+define void @test_multiple_begin() personality i8* bitcast (i32 (...)* @__CxxFrameHandler3 to i8*) {
 ; CHECK: llvm.eh.begincatch may be called a second time before llvm.eh.endcatch
 ; CHECK-NEXT: call void @llvm.eh.begincatch(i8* %exn, i8* null)
 ; CHECK-NEXT: call void @llvm.eh.begincatch(i8* %exn, i8* null)
@@ -86,7 +86,7 @@ entry:
           to label %try.cont unwind label %lpad
 
 lpad:                                             ; preds = %entry
-  %0 = landingpad { i8*, i32 } personality i8* bitcast (i32 (...)* @__CxxFrameHandler3 to i8*)
+  %0 = landingpad { i8*, i32 }
           catch i8* bitcast (i8** @_ZTIi to i8*)
   %exn = extractvalue { i8*, i32 } %0, 0
   %sel = extractvalue { i8*, i32 } %0, 1
@@ -112,7 +112,7 @@ eh.resume:                                        ; preds = %catch.dispatch
 }
 
 ; Function Attrs: uwtable
-define void @test_multiple_end() {
+define void @test_multiple_end() personality i8* bitcast (i32 (...)* @__CxxFrameHandler3 to i8*) {
 ; CHECK: llvm.eh.endcatch may be called a second time after llvm.eh.begincatch
 ; CHECK-NEXT:  call void @llvm.eh.endcatch()
 ; CHECK-NEXT:  call void @llvm.eh.endcatch()
@@ -121,7 +121,7 @@ entry:
           to label %try.cont unwind label %lpad
 
 lpad:                                             ; preds = %entry
-  %0 = landingpad { i8*, i32 } personality i8* bitcast (i32 (...)* @__CxxFrameHandler3 to i8*)
+  %0 = landingpad { i8*, i32 }
           catch i8* bitcast (i8** @_ZTIi to i8*)
   %exn = extractvalue { i8*, i32 } %0, 0
   %sel = extractvalue { i8*, i32 } %0, 1
@@ -166,7 +166,7 @@ try.cont:                                         ; preds = %invoke.cont2, %entr
 }
 
 ; Function Attrs: uwtable
-define void @test_branch_to_begincatch_with_no_lpad(i32 %fake.sel) {
+define void @test_branch_to_begincatch_with_no_lpad(i32 %fake.sel) personality i8* bitcast (i32 (...)* @__CxxFrameHandler3 to i8*) {
 ; CHECK: llvm.eh.begincatch may be reachable without passing a landingpad
 ; CHECK-NEXT: call void @llvm.eh.begincatch(i8* %exn2, i8* null)
 entry:
@@ -175,7 +175,7 @@ entry:
           to label %catch unwind label %lpad
 
 lpad:                                             ; preds = %entry
-  %0 = landingpad { i8*, i32 } personality i8* bitcast (i32 (...)* @__CxxFrameHandler3 to i8*)
+  %0 = landingpad { i8*, i32 }
           catch i8* bitcast (i8** @_ZTIi to i8*)
   %exn = extractvalue { i8*, i32 } %0, 0
   %sel = extractvalue { i8*, i32 } %0, 1
@@ -211,7 +211,7 @@ eh.resume:                                        ; preds = %catch.dispatch
 }
 
 ; Function Attrs: uwtable
-define void @test_branch_missing_endcatch() {
+define void @test_branch_missing_endcatch() personality i8* bitcast (i32 (...)* @__CxxFrameHandler3 to i8*) {
 ; CHECK: Some paths from llvm.eh.begincatch may not reach llvm.eh.endcatch
 ; CHECK-NEXT: call void @llvm.eh.begincatch(i8* %exn2, i8* null)
 entry:
@@ -223,7 +223,7 @@ invoke.cont:
           to label %invoke.cont unwind label %lpad1
 
 lpad:                                             ; preds = %entry
-  %0 = landingpad { i8*, i32 } personality i8* bitcast (i32 (...)* @__CxxFrameHandler3 to i8*)
+  %0 = landingpad { i8*, i32 }
           catch i8* bitcast (i8** @_ZTIi to i8*)
   %exn = extractvalue { i8*, i32 } %0, 0
   %sel = extractvalue { i8*, i32 } %0, 1
@@ -235,7 +235,7 @@ lpad:                                             ; preds = %entry
           to label %try.cont unwind label %lpad
 
 lpad1:                                            ; preds = %entry
-  %l1.0 = landingpad { i8*, i32 } personality i8* bitcast (i32 (...)* @__CxxFrameHandler3 to i8*)
+  %l1.0 = landingpad { i8*, i32 }
 		  cleanup
           catch i8* bitcast (i8** @_ZTIi to i8*)
   %exn1 = extractvalue { i8*, i32 } %l1.0, 0
diff --git a/test/Analysis/LoopAccessAnalysis/non-wrapping-pointer.ll b/test/Analysis/LoopAccessAnalysis/non-wrapping-pointer.ll
new file mode 100644
index 00000000000..0de1cd1bea6
--- /dev/null
+++ b/test/Analysis/LoopAccessAnalysis/non-wrapping-pointer.ll
@@ -0,0 +1,41 @@
+; RUN: opt -basicaa -loop-accesses -analyze < %s | FileCheck %s
+
+; For this loop:
+;   for (int i = 0; i < n; i++)
+;    A[2 * i] = A[2 * i] + B[i];
+;
+; , SCEV is unable to prove that A[2 * i] does not overflow.  However,
+; analyzing the IR helps us to conclude it and in turn allow dependence
+; analysis.
+
+target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
+
+; CHECK: Memory dependences are safe{{$}}
+
+define void @f(i16* noalias %a,
+               i16* noalias %b, i64 %N) {
+entry:
+  br label %for.body
+
+for.body:                                         ; preds = %for.body, %entry
+  %ind = phi i64 [ 0, %entry ], [ %inc, %for.body ]
+
+  %mul = mul nuw nsw i64 %ind, 2
+
+  %arrayidxA = getelementptr inbounds i16, i16* %a, i64 %mul
+  %loadA = load i16, i16* %arrayidxA, align 2
+
+  %arrayidxB = getelementptr inbounds i16, i16* %b, i64 %ind
+  %loadB = load i16, i16* %arrayidxB, align 2
+
+  %add = mul i16 %loadA, %loadB
+
+  store i16 %add, i16* %arrayidxA, align 2
+
+  %inc = add nuw nsw i64 %ind, 1
+  %exitcond = icmp eq i64 %inc, %N
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:                                          ; preds = %for.body
+  ret void
+}
diff --git a/test/Analysis/LoopAccessAnalysis/number-of-memchecks.ll b/test/Analysis/LoopAccessAnalysis/number-of-memchecks.ll
new file mode 100644
index 00000000000..01b3eda3e12
--- /dev/null
+++ b/test/Analysis/LoopAccessAnalysis/number-of-memchecks.ll
@@ -0,0 +1,287 @@
+; RUN: opt -loop-accesses -analyze < %s | FileCheck %s
+
+target datalayout = "e-m:e-i64:64-i128:128-n32:64-S128"
+target triple = "aarch64--linux-gnueabi"
+
+; 3 reads and 3 writes should need 12 memchecks
+; CHECK: function 'testf':
+; CHECK: Memory dependences are safe with run-time checks
+
+; Memory dependencies have labels starting from 0, so in
+; order to verify that we have n checks, we look for
+; (n-1): and not n:.
+
+; CHECK: Run-time memory checks:
+; CHECK-NEXT: Check 0:
+; CHECK: Check 11:
+; CHECK-NOT: Check 12:
+
+define void @testf(i16* %a,
+               i16* %b,
+               i16* %c,
+               i16* %d,
+               i16* %e,
+               i16* %f) {
+entry:
+  br label %for.body
+
+for.body:                                         ; preds = %for.body, %entry
+  %ind = phi i64 [ 0, %entry ], [ %add, %for.body ]
+
+  %add = add nuw nsw i64 %ind, 1
+
+  %arrayidxA = getelementptr inbounds i16, i16* %a, i64 %ind
+  %loadA = load i16, i16* %arrayidxA, align 2
+
+  %arrayidxB = getelementptr inbounds i16, i16* %b, i64 %ind
+  %loadB = load i16, i16* %arrayidxB, align 2
+
+  %arrayidxC = getelementptr inbounds i16, i16* %c, i64 %ind
+  %loadC = load i16, i16* %arrayidxC, align 2
+
+  %mul = mul i16 %loadB, %loadA
+  %mul1 = mul i16 %mul, %loadC
+
+  %arrayidxD = getelementptr inbounds i16, i16* %d, i64 %ind
+  store i16 %mul1, i16* %arrayidxD, align 2
+
+  %arrayidxE = getelementptr inbounds i16, i16* %e, i64 %ind
+  store i16 %mul, i16* %arrayidxE, align 2
+
+  %arrayidxF = getelementptr inbounds i16, i16* %f, i64 %ind
+  store i16 %mul1, i16* %arrayidxF, align 2
+
+  %exitcond = icmp eq i64 %add, 20
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:                                          ; preds = %for.body
+  ret void
+}
+
+; The following (testg and testh) check that we can group
+; memory checks of accesses which differ by a constant value.
+; Both tests are based on the following C code:
+;
+; void testh(short *a, short *b, short *c) {
+;   unsigned long ind = 0;
+;   for (unsigned long ind = 0; ind < 20; ++ind) {
+;     c[2 * ind] = a[ind] * a[ind + 1];
+;     c[2 * ind + 1] = a[ind] * a[ind + 1] * b[ind];
+;   }
+; }
+;
+; It is sufficient to check the intervals
+; [a, a + 21], [b, b + 20] against [c, c + 41].
+
+; 3 reads and 2 writes - two of the reads can be merged,
+; and the writes can be merged as well. This gives us a
+; total of 2 memory checks.
+
+; CHECK: function 'testg':
+
+; CHECK: Run-time memory checks:
+; CHECK-NEXT:   Check 0:
+; CHECK-NEXT:     Comparing group ([[ZERO:.+]]):
+; CHECK-NEXT:       %arrayidxC1 = getelementptr inbounds i16, i16* %c, i64 %store_ind_inc
+; CHECK-NEXT:       %arrayidxC = getelementptr inbounds i16, i16* %c, i64 %store_ind
+; CHECK-NEXT:     Against group ([[ONE:.+]]):
+; CHECK-NEXT:       %arrayidxA1 = getelementptr inbounds i16, i16* %a, i64 %add
+; CHECK-NEXT:       %arrayidxA = getelementptr inbounds i16, i16* %a, i64 %ind
+; CHECK-NEXT:   Check 1:
+; CHECK-NEXT:     Comparing group ({{.*}}[[ZERO]]):
+; CHECK-NEXT:       %arrayidxC1 = getelementptr inbounds i16, i16* %c, i64 %store_ind_inc
+; CHECK-NEXT:       %arrayidxC = getelementptr inbounds i16, i16* %c, i64 %store_ind
+; CHECK-NEXT:     Against group ([[TWO:.+]]):
+; CHECK-NEXT:       %arrayidxB = getelementptr inbounds i16, i16* %b, i64 %ind
+; CHECK-NEXT:   Grouped accesses:
+; CHECK-NEXT:    Group {{.*}}[[ZERO]]:
+; CHECK-NEXT:       (Low: %c High: (78 + %c))
+; CHECK-NEXT:         Member: {(2 + %c),+,4}
+; CHECK-NEXT:         Member: {%c,+,4}
+; CHECK-NEXT:     Group {{.*}}[[ONE]]:
+; CHECK-NEXT:       (Low: %a High: (40 + %a))
+; CHECK-NEXT:         Member: {(2 + %a),+,2}
+; CHECK-NEXT:         Member: {%a,+,2}
+; CHECK-NEXT:     Group {{.*}}[[TWO]]:
+; CHECK-NEXT:       (Low: %b High: (38 + %b))
+; CHECK-NEXT:         Member: {%b,+,2}
+
+define void @testg(i16* %a,
+               i16* %b,
+               i16* %c) {
+entry:
+  br label %for.body
+
+for.body:                                         ; preds = %for.body, %entry
+  %ind = phi i64 [ 0, %entry ], [ %add, %for.body ]
+  %store_ind = phi i64 [ 0, %entry ], [ %store_ind_next, %for.body ]
+
+  %add = add nuw nsw i64 %ind, 1
+  %store_ind_inc = add nuw nsw i64 %store_ind, 1
+  %store_ind_next = add nuw nsw i64 %store_ind_inc, 1
+
+  %arrayidxA = getelementptr inbounds i16, i16* %a, i64 %ind
+  %loadA = load i16, i16* %arrayidxA, align 2
+
+  %arrayidxA1 = getelementptr inbounds i16, i16* %a, i64 %add
+  %loadA1 = load i16, i16* %arrayidxA1, align 2
+
+  %arrayidxB = getelementptr inbounds i16, i16* %b, i64 %ind
+  %loadB = load i16, i16* %arrayidxB, align 2
+
+  %mul = mul i16 %loadA, %loadA1
+  %mul1 = mul i16 %mul, %loadB
+
+  %arrayidxC = getelementptr inbounds i16, i16* %c, i64 %store_ind
+  store i16 %mul1, i16* %arrayidxC, align 2
+
+  %arrayidxC1 = getelementptr inbounds i16, i16* %c, i64 %store_ind_inc
+  store i16 %mul, i16* %arrayidxC1, align 2
+
+  %exitcond = icmp eq i64 %add, 20
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:                                          ; preds = %for.body
+  ret void
+}
+
+; 3 reads and 2 writes - the writes can be merged into a single
+; group, but the GEPs used for the reads are not marked as inbounds.
+; We can still merge them because we are using a unit stride for
+; accesses, so we cannot overflow the GEPs.
+
+; CHECK: function 'testh':
+; CHECK: Run-time memory checks:
+; CHECK-NEXT:   Check 0:
+; CHECK-NEXT:     Comparing group ([[ZERO:.+]]):
+; CHECK-NEXT:         %arrayidxC1 = getelementptr inbounds i16, i16* %c, i64 %store_ind_inc
+; CHECK-NEXT:         %arrayidxC = getelementptr inbounds i16, i16* %c, i64 %store_ind
+; CHECK-NEXT:     Against group ([[ONE:.+]]):
+; CHECK-NEXT:         %arrayidxA1 = getelementptr i16, i16* %a, i64 %add
+; CHECK-NEXT:         %arrayidxA = getelementptr i16, i16* %a, i64 %ind
+; CHECK-NEXT:   Check 1:
+; CHECK-NEXT:     Comparing group ({{.*}}[[ZERO]]):
+; CHECK-NEXT:         %arrayidxC1 = getelementptr inbounds i16, i16* %c, i64 %store_ind_inc
+; CHECK-NEXT:         %arrayidxC = getelementptr inbounds i16, i16* %c, i64 %store_ind
+; CHECK-NEXT:     Against group ([[TWO:.+]]):
+; CHECK-NEXT:         %arrayidxB = getelementptr i16, i16* %b, i64 %ind
+; CHECK-NEXT:   Grouped accesses:
+; CHECK-NEXT:     Group {{.*}}[[ZERO]]:
+; CHECK-NEXT:       (Low: %c High: (78 + %c))
+; CHECK-NEXT:         Member: {(2 + %c),+,4}
+; CHECK-NEXT:         Member: {%c,+,4}
+; CHECK-NEXT:     Group {{.*}}[[ONE]]:
+; CHECK-NEXT:       (Low: %a High: (40 + %a))
+; CHECK-NEXT:         Member: {(2 + %a),+,2}
+; CHECK-NEXT:         Member: {%a,+,2}
+; CHECK-NEXT:     Group {{.*}}[[TWO]]:
+; CHECK-NEXT:       (Low: %b High: (38 + %b))
+; CHECK-NEXT:         Member: {%b,+,2}
+
+define void @testh(i16* %a,
+               i16* %b,
+               i16* %c) {
+entry:
+  br label %for.body
+
+for.body:                                         ; preds = %for.body, %entry
+  %ind = phi i64 [ 0, %entry ], [ %add, %for.body ]
+  %store_ind = phi i64 [ 0, %entry ], [ %store_ind_next, %for.body ]
+
+  %add = add nuw nsw i64 %ind, 1
+  %store_ind_inc = add nuw nsw i64 %store_ind, 1
+  %store_ind_next = add nuw nsw i64 %store_ind_inc, 1
+
+  %arrayidxA = getelementptr i16, i16* %a, i64 %ind
+  %loadA = load i16, i16* %arrayidxA, align 2
+
+  %arrayidxA1 = getelementptr i16, i16* %a, i64 %add
+  %loadA1 = load i16, i16* %arrayidxA1, align 2
+
+  %arrayidxB = getelementptr i16, i16* %b, i64 %ind
+  %loadB = load i16, i16* %arrayidxB, align 2
+
+  %mul = mul i16 %loadA, %loadA1
+  %mul1 = mul i16 %mul, %loadB
+
+  %arrayidxC = getelementptr inbounds i16, i16* %c, i64 %store_ind
+  store i16 %mul1, i16* %arrayidxC, align 2
+
+  %arrayidxC1 = getelementptr inbounds i16, i16* %c, i64 %store_ind_inc
+  store i16 %mul, i16* %arrayidxC1, align 2
+
+  %exitcond = icmp eq i64 %add, 20
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:                                          ; preds = %for.body
+  ret void
+}
+
+; Don't merge pointers if we need to perform a check against a pointer
+; to the same underlying object (doing so would emit a check that could be
+; falsely invalidated) For example, in the following loop:
+;
+; for (i = 0; i < 5000; ++i)
+;   a[i + offset] = a[i] + a[i + 10000]
+;
+; we should not merge the intervals associated with the reads (0,5000) and
+; (10000, 15000) into (0, 15000) as this will pottentially fail the check
+; against the interval associated with the write.
+;
+; We cannot have this check unless ShouldRetryWithRuntimeCheck is set,
+; and therefore the grouping algorithm would create a separate group for
+; each pointer.
+
+; CHECK: function 'testi':
+; CHECK: Run-time memory checks:
+; CHECK-NEXT:   Check 0:
+; CHECK-NEXT:     Comparing group ([[ZERO:.+]]):
+; CHECK-NEXT:       %storeidx = getelementptr inbounds i16, i16* %a, i64 %store_ind
+; CHECK-NEXT:     Against group ([[ONE:.+]]):
+; CHECK-NEXT:       %arrayidxA1 = getelementptr i16, i16* %a, i64 %ind
+; CHECK-NEXT:   Check 1:
+; CHECK-NEXT:     Comparing group ({{.*}}[[ZERO]]):
+; CHECK-NEXT:       %storeidx = getelementptr inbounds i16, i16* %a, i64 %store_ind
+; CHECK-NEXT:     Against group ([[TWO:.+]]):
+; CHECK-NEXT:       %arrayidxA2 = getelementptr i16, i16* %a, i64 %ind2
+; CHECK-NEXT:   Grouped accesses:
+; CHECK-NEXT:     Group {{.*}}[[ZERO]]:
+; CHECK-NEXT:       (Low: ((2 * %offset) + %a) High: (9998 + (2 * %offset) + %a))
+; CHECK-NEXT:         Member: {((2 * %offset) + %a),+,2}<nsw><%for.body>
+; CHECK-NEXT:     Group {{.*}}[[ONE]]:
+; CHECK-NEXT:       (Low: %a High: (9998 + %a))
+; CHECK-NEXT:         Member: {%a,+,2}<%for.body>
+; CHECK-NEXT:     Group {{.*}}[[TWO]]:
+; CHECK-NEXT:       (Low: (20000 + %a) High: (29998 + %a))
+; CHECK-NEXT:         Member: {(20000 + %a),+,2}<%for.body>
+
+define void @testi(i16* %a,
+                   i64 %offset) {
+entry:
+  br label %for.body
+
+for.body:                                         ; preds = %for.body, %entry
+  %ind = phi i64 [ 0, %entry ], [ %add, %for.body ]
+  %store_ind = phi i64 [ %offset, %entry ], [ %store_ind_inc, %for.body ]
+
+  %add = add nuw nsw i64 %ind, 1
+  %store_ind_inc = add nuw nsw i64 %store_ind, 1
+
+  %arrayidxA1 = getelementptr i16, i16* %a, i64 %ind
+  %ind2 = add nuw nsw i64 %ind, 10000
+  %arrayidxA2 = getelementptr i16, i16* %a, i64 %ind2
+
+  %loadA1 = load i16, i16* %arrayidxA1, align 2
+  %loadA2 = load i16, i16* %arrayidxA2, align 2
+
+  %addres = add i16 %loadA1, %loadA2
+
+  %storeidx = getelementptr inbounds i16, i16* %a, i64 %store_ind
+  store i16 %addres, i16* %storeidx, align 2
+
+  %exitcond = icmp eq i64 %add, 5000
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:                                          ; preds = %for.body
+  ret void
+}
diff --git a/test/Analysis/LoopAccessAnalysis/pointer-with-unknown-bounds.ll b/test/Analysis/LoopAccessAnalysis/pointer-with-unknown-bounds.ll
new file mode 100644
index 00000000000..d05849e2be2
--- /dev/null
+++ b/test/Analysis/LoopAccessAnalysis/pointer-with-unknown-bounds.ll
@@ -0,0 +1,42 @@
+; RUN: opt -loop-accesses -analyze < %s | FileCheck %s
+
+target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
+
+; We shouldn't quit the analysis if we encounter a pointer without known
+; bounds *unless* we actually need to emit a memcheck for it.  (We only
+; compute bounds for SCEVAddRecs so A[i*I] is deemed not having known bounds.)
+;
+; for (i = 0; i < 20; ++i)
+;   A[i*i] *= 2;
+
+; CHECK: for.body:
+; CHECK:     Report: unsafe dependent memory operations in loop
+; CHECK-NOT: Report: cannot identify array bounds
+; CHECK:     Interesting Dependences:
+; CHECK:       Unknown:
+; CHECK:           %loadA = load i16, i16* %arrayidxA, align 2 ->
+; CHECK:           store i16 %mul, i16* %arrayidxA, align 2
+
+define void @f(i16* %a) {
+entry:
+  br label %for.body
+
+for.body:                                         ; preds = %for.body, %entry
+  %ind = phi i64 [ 0, %entry ], [ %add, %for.body ]
+
+  %access_ind = mul i64 %ind, %ind
+
+  %arrayidxA = getelementptr inbounds i16, i16* %a, i64 %access_ind
+  %loadA = load i16, i16* %arrayidxA, align 2
+
+  %mul = mul i16 %loadA, 2
+
+  store i16 %mul, i16* %arrayidxA, align 2
+
+  %add = add nuw nsw i64 %ind, 1
+  %exitcond = icmp eq i64 %add, 20
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:                                          ; preds = %for.body
+  ret void
+}
diff --git a/test/Analysis/LoopAccessAnalysis/resort-to-memchecks-only.ll b/test/Analysis/LoopAccessAnalysis/resort-to-memchecks-only.ll
new file mode 100644
index 00000000000..e7305173dd9
--- /dev/null
+++ b/test/Analysis/LoopAccessAnalysis/resort-to-memchecks-only.ll
@@ -0,0 +1,59 @@
+; RUN: opt -loop-accesses -analyze < %s | FileCheck %s
+
+; We give up analyzing the dependences in this loop due to non-constant
+; distance between A[i+offset] and A[i] and add memchecks to prove
+; independence.  Make sure that no interesting dependences are reported in
+; this case.
+;
+;   for (i = 0; i < n; i++)
+;    A[i + offset] = A[i] * B[i] * C[i];
+
+target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-apple-macosx10.10.0"
+
+; CHECK: Memory dependences are safe with run-time checks
+; CHECK-NEXT: Interesting Dependences:
+; CHECK-NEXT: Run-time memory checks:
+; CHECK-NEXT: 0:
+; CHECK-NEXT: Comparing group
+; CHECK-NEXT:   %arrayidxA2 = getelementptr inbounds i16, i16* %a, i64 %idx
+; CHECK-NEXT: Against group
+; CHECK-NEXT:   %arrayidxA = getelementptr inbounds i16, i16* %a, i64 %indvar
+
+@B = common global i16* null, align 8
+@A = common global i16* null, align 8
+@C = common global i16* null, align 8
+
+define void @f(i64 %offset) {
+entry:
+  %a = load i16*, i16** @A, align 8
+  %b = load i16*, i16** @B, align 8
+  %c = load i16*, i16** @C, align 8
+  br label %for.body
+
+for.body:                                         ; preds = %for.body, %entry
+  %indvar = phi i64 [ 0, %entry ], [ %add, %for.body ]
+
+  %arrayidxA = getelementptr inbounds i16, i16* %a, i64 %indvar
+  %loadA = load i16, i16* %arrayidxA, align 2
+
+  %arrayidxB = getelementptr inbounds i16, i16* %b, i64 %indvar
+  %loadB = load i16, i16* %arrayidxB, align 2
+
+  %arrayidxC = getelementptr inbounds i16, i16* %c, i64 %indvar
+  %loadC = load i16, i16* %arrayidxC, align 2
+
+  %mul = mul i16 %loadB, %loadA
+  %mul1 = mul i16 %mul, %loadC
+
+  %idx = add i64 %indvar, %offset
+  %arrayidxA2 = getelementptr inbounds i16, i16* %a, i64 %idx
+  store i16 %mul1, i16* %arrayidxA2, align 2
+
+  %add = add nuw nsw i64 %indvar, 1
+  %exitcond = icmp eq i64 %add, 20
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:                                          ; preds = %for.body
+  ret void
+}
diff --git a/test/Analysis/LoopAccessAnalysis/reverse-memcheck-bounds.ll b/test/Analysis/LoopAccessAnalysis/reverse-memcheck-bounds.ll
new file mode 100644
index 00000000000..2053e49826b
--- /dev/null
+++ b/test/Analysis/LoopAccessAnalysis/reverse-memcheck-bounds.ll
@@ -0,0 +1,89 @@
+; RUN: opt -loop-accesses -analyze < %s | FileCheck %s
+
+; The runtime memory check code and the access grouping
+; algorithm both assume that the start and end values
+; for an access range are ordered (start <= stop).
+; When generating checks for accesses with negative stride
+; we need to take this into account and swap the interval
+; ends.
+;
+;   for (i = 0; i < 10000; i++) {
+;     B[i] = A[15000 - i] * 3;
+;   }
+
+target datalayout = "e-m:e-i64:64-i128:128-n32:64-S128"
+target triple = "aarch64--linux-gnueabi"
+
+; CHECK: function 'f':
+; CHECK: (Low: (20000 + %a) High: (60000 + %a))
+
+@B = common global i32* null, align 8
+@A = common global i32* null, align 8
+
+define void @f() {
+entry:
+  %a = load i32*, i32** @A, align 8
+  %b = load i32*, i32** @B, align 8
+  br label %for.body
+
+for.body:                                         ; preds = %for.body, %entry
+  %idx = phi i64 [ 0, %entry ], [ %add, %for.body ]
+  %negidx = sub i64 15000, %idx
+
+  %arrayidxA0 = getelementptr inbounds i32, i32* %a, i64 %negidx
+  %loadA0 = load i32, i32* %arrayidxA0, align 2
+
+  %res = mul i32 %loadA0, 3
+
+  %add = add nuw nsw i64 %idx, 1
+
+  %arrayidxB = getelementptr inbounds i32, i32* %b, i64 %idx
+  store i32 %res, i32* %arrayidxB, align 2
+
+  %exitcond = icmp eq i64 %idx, 10000
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:                                          ; preds = %for.body
+  ret void
+}
+
+; CHECK: function 'g':
+; When the stride is not constant, we are forced to do umin/umax to get
+; the interval limits.
+
+;   for (i = 0; i < 10000; i++) {
+;     B[i] = A[15000 - step * i] * 3;
+;   }
+
+; Here it is not obvious what the limits are, since 'step' could be negative.
+
+; CHECK: Low: (-1 + (-1 * ((-60001 + (-1 * %a)) umax (-60001 + (40000 * %step) + (-1 * %a)))))
+; CHECK: High: ((60000 + %a) umax (60000 + (-40000 * %step) + %a))
+
+define void @g(i64 %step) {
+entry:
+  %a = load i32*, i32** @A, align 8
+  %b = load i32*, i32** @B, align 8
+  br label %for.body
+
+for.body:                                         ; preds = %for.body, %entry
+  %idx = phi i64 [ 0, %entry ], [ %add, %for.body ]
+  %idx_mul = mul i64 %idx, %step
+  %negidx = sub i64 15000, %idx_mul
+
+  %arrayidxA0 = getelementptr inbounds i32, i32* %a, i64 %negidx
+  %loadA0 = load i32, i32* %arrayidxA0, align 2
+
+  %res = mul i32 %loadA0, 3
+
+  %add = add nuw nsw i64 %idx, 1
+
+  %arrayidxB = getelementptr inbounds i32, i32* %b, i64 %idx
+  store i32 %res, i32* %arrayidxB, align 2
+
+  %exitcond = icmp eq i64 %idx, 10000
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:                                          ; preds = %for.body
+  ret void
+}
diff --git a/test/Analysis/LoopAccessAnalysis/stride-access-dependence.ll b/test/Analysis/LoopAccessAnalysis/stride-access-dependence.ll
new file mode 100644
index 00000000000..73573566295
--- /dev/null
+++ b/test/Analysis/LoopAccessAnalysis/stride-access-dependence.ll
@@ -0,0 +1,540 @@
+; RUN: opt -loop-accesses -analyze < %s | FileCheck %s
+
+target datalayout = "e-m:e-i64:64-i128:128-n32:64-S128"
+
+; Following cases are no dependence.
+
+; void nodep_Read_Write(int *A) {
+;   int *B = A + 1;
+;   for (unsigned i = 0; i < 1024; i+=3)
+;     B[i] = A[i] + 1;
+; }
+
+; CHECK: function 'nodep_Read_Write':
+; CHECK-NEXT:   for.body:
+; CHECK-NEXT:     Memory dependences are safe
+; CHECK-NEXT:     Interesting Dependences:
+; CHECK-NEXT:     Run-time memory checks:
+
+define void @nodep_Read_Write(i32* nocapture %A) {
+entry:
+  %add.ptr = getelementptr inbounds i32, i32* %A, i64 1
+  br label %for.body
+
+for.cond.cleanup:                                 ; preds = %for.body
+  ret void
+
+for.body:                                         ; preds = %entry, %for.body
+  %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+  %arrayidx = getelementptr inbounds i32, i32* %A, i64 %indvars.iv
+  %0 = load i32, i32* %arrayidx, align 4
+  %add = add nsw i32 %0, 1
+  %arrayidx2 = getelementptr inbounds i32, i32* %add.ptr, i64 %indvars.iv
+  store i32 %add, i32* %arrayidx2, align 4
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 3
+  %cmp = icmp ult i64 %indvars.iv.next, 1024
+  br i1 %cmp, label %for.body, label %for.cond.cleanup
+}
+
+; int nodep_Write_Read(int *A) {
+;   int sum = 0;
+;   for (unsigned i = 0; i < 1024; i+=4) {
+;     A[i] = i;
+;     sum += A[i+3];
+;   }
+;   
+;   return sum;
+; }
+
+; CHECK: function 'nodep_Write_Read':
+; CHECK-NEXT:   for.body:
+; CHECK-NEXT:     Memory dependences are safe
+; CHECK-NEXT:     Interesting Dependences:
+; CHECK-NEXT:     Run-time memory checks:
+
+define i32 @nodep_Write_Read(i32* nocapture %A) {
+entry:
+  br label %for.body
+
+for.cond.cleanup:                                 ; preds = %for.body
+  ret i32 %add3
+
+for.body:                                         ; preds = %entry, %for.body
+  %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+  %sum.013 = phi i32 [ 0, %entry ], [ %add3, %for.body ]
+  %arrayidx = getelementptr inbounds i32, i32* %A, i64 %indvars.iv
+  %0 = trunc i64 %indvars.iv to i32
+  store i32 %0, i32* %arrayidx, align 4
+  %1 = or i64 %indvars.iv, 3
+  %arrayidx2 = getelementptr inbounds i32, i32* %A, i64 %1
+  %2 = load i32, i32* %arrayidx2, align 4
+  %add3 = add nsw i32 %2, %sum.013
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 4
+  %cmp = icmp ult i64 %indvars.iv.next, 1024
+  br i1 %cmp, label %for.body, label %for.cond.cleanup
+}
+
+; void nodep_Write_Write(int *A) {
+;   for (unsigned i = 0; i < 1024; i+=2) {
+;     A[i] = i;
+;     A[i+1] = i+1;
+;   }
+; }
+
+; CHECK: function 'nodep_Write_Write':
+; CHECK-NEXT:   for.body:
+; CHECK-NEXT:     Memory dependences are safe
+; CHECK-NEXT:     Interesting Dependences:
+; CHECK-NEXT:     Run-time memory checks:
+
+define void @nodep_Write_Write(i32* nocapture %A) {
+entry:
+  br label %for.body
+
+for.cond.cleanup:                                 ; preds = %for.body
+  ret void
+
+for.body:                                         ; preds = %entry, %for.body
+  %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+  %arrayidx = getelementptr inbounds i32, i32* %A, i64 %indvars.iv
+  %0 = trunc i64 %indvars.iv to i32
+  store i32 %0, i32* %arrayidx, align 4
+  %1 = or i64 %indvars.iv, 1
+  %arrayidx3 = getelementptr inbounds i32, i32* %A, i64 %1
+  %2 = trunc i64 %1 to i32
+  store i32 %2, i32* %arrayidx3, align 4
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 2
+  %cmp = icmp ult i64 %indvars.iv.next, 1024
+  br i1 %cmp, label %for.body, label %for.cond.cleanup
+}
+
+; Following cases are unsafe depdences and are not vectorizable.
+
+; void unsafe_Read_Write(int *A) {
+;   for (unsigned i = 0; i < 1024; i+=3)
+;     A[i+3] = A[i] + 1;
+; }
+
+; CHECK: function 'unsafe_Read_Write':
+; CHECK-NEXT:   for.body:
+; CHECK-NEXT:     Report: unsafe dependent memory operations in loop
+; CHECK-NEXT:     Interesting Dependences:
+; CHECK-NEXT:      Backward:
+; CHECK-NEXT:           %0 = load i32, i32* %arrayidx, align 4 -> 
+; CHECK-NEXT:           store i32 %add, i32* %arrayidx3, align 4
+
+define void @unsafe_Read_Write(i32* nocapture %A) {
+entry:
+  br label %for.body
+
+for.cond.cleanup:                                 ; preds = %for.body
+  ret void
+
+for.body:                                         ; preds = %entry, %for.body
+  %i.010 = phi i32 [ 0, %entry ], [ %add1, %for.body ]
+  %idxprom = zext i32 %i.010 to i64
+  %arrayidx = getelementptr inbounds i32, i32* %A, i64 %idxprom
+  %0 = load i32, i32* %arrayidx, align 4
+  %add = add nsw i32 %0, 1
+  %add1 = add i32 %i.010, 3
+  %idxprom2 = zext i32 %add1 to i64
+  %arrayidx3 = getelementptr inbounds i32, i32* %A, i64 %idxprom2
+  store i32 %add, i32* %arrayidx3, align 4
+  %cmp = icmp ult i32 %add1, 1024
+  br i1 %cmp, label %for.body, label %for.cond.cleanup
+}
+
+; int unsafe_Write_Read(int *A) {
+;   int sum = 0;
+;   for (unsigned i = 0; i < 1024; i+=4) {
+;     A[i] = i;
+;     sum += A[i+4];
+;   }
+;
+;   return sum;
+; }
+
+; CHECK: function 'unsafe_Write_Read':
+; CHECK-NEXT:   for.body:
+; CHECK-NEXT:     Report: unsafe dependent memory operations in loop
+; CHECK-NEXT:     Interesting Dependences:
+; CHECK-NEXT:      Backward:
+; CHECK-NEXT:           store i32 %0, i32* %arrayidx, align 4 ->
+; CHECK-NEXT:           %1 = load i32, i32* %arrayidx2, align 4
+
+define i32 @unsafe_Write_Read(i32* nocapture %A) {
+entry:
+  br label %for.body
+
+for.cond.cleanup:                                 ; preds = %for.body
+  ret i32 %add3
+
+for.body:                                         ; preds = %entry, %for.body
+  %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+  %sum.013 = phi i32 [ 0, %entry ], [ %add3, %for.body ]
+  %arrayidx = getelementptr inbounds i32, i32* %A, i64 %indvars.iv
+  %0 = trunc i64 %indvars.iv to i32
+  store i32 %0, i32* %arrayidx, align 4
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 4
+  %arrayidx2 = getelementptr inbounds i32, i32* %A, i64 %indvars.iv.next
+  %1 = load i32, i32* %arrayidx2, align 4
+  %add3 = add nsw i32 %1, %sum.013
+  %cmp = icmp ult i64 %indvars.iv.next, 1024
+  br i1 %cmp, label %for.body, label %for.cond.cleanup
+}
+
+; void unsafe_Write_Write(int *A) {
+;   for (unsigned i = 0; i < 1024; i+=2) {
+;     A[i] = i;
+;     A[i+2] = i+1;
+;   }
+; }
+
+; CHECK: function 'unsafe_Write_Write':
+; CHECK-NEXT:   for.body:
+; CHECK-NEXT:     Report: unsafe dependent memory operations in loop
+; CHECK-NEXT:     Interesting Dependences:
+; CHECK-NEXT:      Backward:
+; CHECK-NEXT:           store i32 %0, i32* %arrayidx, align 4 ->
+; CHECK-NEXT:           store i32 %2, i32* %arrayidx3, align 4
+
+define void @unsafe_Write_Write(i32* nocapture %A) {
+entry:
+  br label %for.body
+
+for.cond.cleanup:                                 ; preds = %for.body
+  ret void
+
+for.body:                                         ; preds = %entry, %for.body
+  %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+  %arrayidx = getelementptr inbounds i32, i32* %A, i64 %indvars.iv
+  %0 = trunc i64 %indvars.iv to i32
+  store i32 %0, i32* %arrayidx, align 4
+  %1 = or i64 %indvars.iv, 1
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 2
+  %arrayidx3 = getelementptr inbounds i32, i32* %A, i64 %indvars.iv.next
+  %2 = trunc i64 %1 to i32
+  store i32 %2, i32* %arrayidx3, align 4
+  %cmp = icmp ult i64 %indvars.iv.next, 1024
+  br i1 %cmp, label %for.body, label %for.cond.cleanup
+}
+
+; Following cases check that strided accesses can be vectorized.
+
+; void vectorizable_Read_Write(int *A) {
+;   int *B = A + 4;
+;   for (unsigned i = 0; i < 1024; i+=2)
+;     B[i] = A[i] + 1;
+; }
+
+; CHECK: function 'vectorizable_Read_Write':
+; CHECK-NEXT:   for.body:
+; CHECK-NEXT:     Memory dependences are safe
+; CHECK-NEXT:     Interesting Dependences:
+; CHECK-NEXT:       BackwardVectorizable:
+; CHECK-NEXT:           %0 = load i32, i32* %arrayidx, align 4 ->
+; CHECK-NEXT:           store i32 %add, i32* %arrayidx2, align 4
+
+define void @vectorizable_Read_Write(i32* nocapture %A) {
+entry:
+  %add.ptr = getelementptr inbounds i32, i32* %A, i64 4
+  br label %for.body
+
+for.cond.cleanup:                                 ; preds = %for.body
+  ret void
+
+for.body:                                         ; preds = %entry, %for.body
+  %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+  %arrayidx = getelementptr inbounds i32, i32* %A, i64 %indvars.iv
+  %0 = load i32, i32* %arrayidx, align 4
+  %add = add nsw i32 %0, 1
+  %arrayidx2 = getelementptr inbounds i32, i32* %add.ptr, i64 %indvars.iv
+  store i32 %add, i32* %arrayidx2, align 4
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 2
+  %cmp = icmp ult i64 %indvars.iv.next, 1024
+  br i1 %cmp, label %for.body, label %for.cond.cleanup
+}
+
+; int vectorizable_Write_Read(int *A) {
+;   int *B = A + 4;
+;   int sum = 0;
+;   for (unsigned i = 0; i < 1024; i+=2) {
+;     A[i] = i;
+;     sum += B[i];
+;   }
+;   
+;   return sum;
+; }
+
+; CHECK: function 'vectorizable_Write_Read':
+; CHECK-NEXT:   for.body:
+; CHECK-NEXT:     Memory dependences are safe
+; CHECK-NEXT:     Interesting Dependences:
+; CHECK-NEXT:       BackwardVectorizable:
+; CHECK-NEXT:           store i32 %0, i32* %arrayidx, align 4 ->
+; CHECK-NEXT:           %1 = load i32, i32* %arrayidx2, align 4
+
+define i32 @vectorizable_Write_Read(i32* nocapture %A) {
+entry:
+  %add.ptr = getelementptr inbounds i32, i32* %A, i64 4
+  br label %for.body
+
+for.cond.cleanup:                                 ; preds = %for.body
+  ret i32 %add
+
+for.body:                                         ; preds = %entry, %for.body
+  %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+  %sum.013 = phi i32 [ 0, %entry ], [ %add, %for.body ]
+  %arrayidx = getelementptr inbounds i32, i32* %A, i64 %indvars.iv
+  %0 = trunc i64 %indvars.iv to i32
+  store i32 %0, i32* %arrayidx, align 4
+  %arrayidx2 = getelementptr inbounds i32, i32* %add.ptr, i64 %indvars.iv
+  %1 = load i32, i32* %arrayidx2, align 4
+  %add = add nsw i32 %1, %sum.013
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 2
+  %cmp = icmp ult i64 %indvars.iv.next, 1024
+  br i1 %cmp, label %for.body, label %for.cond.cleanup
+}
+
+; void vectorizable_Write_Write(int *A) {
+;   int *B = A + 4;
+;   for (unsigned i = 0; i < 1024; i+=2) {
+;     A[i] = i;
+;     B[i] = i+1;
+;   }
+; }
+
+; CHECK: function 'vectorizable_Write_Write':
+; CHECK-NEXT:   for.body:
+; CHECK-NEXT:     Memory dependences are safe
+; CHECK-NEXT:     Interesting Dependences:
+; CHECK-NEXT:       BackwardVectorizable:
+; CHECK-NEXT:           store i32 %0, i32* %arrayidx, align 4 -> 
+; CHECK-NEXT:           store i32 %2, i32* %arrayidx2, align 4
+
+define void @vectorizable_Write_Write(i32* nocapture %A) {
+entry:
+  %add.ptr = getelementptr inbounds i32, i32* %A, i64 4
+  br label %for.body
+
+for.cond.cleanup:                                 ; preds = %for.body
+  ret void
+
+for.body:                                         ; preds = %entry, %for.body
+  %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+  %arrayidx = getelementptr inbounds i32, i32* %A, i64 %indvars.iv
+  %0 = trunc i64 %indvars.iv to i32
+  store i32 %0, i32* %arrayidx, align 4
+  %1 = or i64 %indvars.iv, 1
+  %arrayidx2 = getelementptr inbounds i32, i32* %add.ptr, i64 %indvars.iv
+  %2 = trunc i64 %1 to i32
+  store i32 %2, i32* %arrayidx2, align 4
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 2
+  %cmp = icmp ult i64 %indvars.iv.next, 1024
+  br i1 %cmp, label %for.body, label %for.cond.cleanup
+}
+
+; void vectorizable_unscaled_Read_Write(int *A) {
+;   int *B = (int *)((char *)A + 14);
+;   for (unsigned i = 0; i < 1024; i+=2)
+;     B[i] = A[i] + 1;
+; }
+
+; FIXME: This case looks like previous case @vectorizable_Read_Write. It sould
+; be vectorizable.
+
+; CHECK: function 'vectorizable_unscaled_Read_Write':
+; CHECK-NEXT:   for.body:
+; CHECK-NEXT:     Report: unsafe dependent memory operations in loop
+; CHECK-NEXT:     Interesting Dependences:
+; CHECK-NEXT:       BackwardVectorizableButPreventsForwarding:
+; CHECK-NEXT:           %2 = load i32, i32* %arrayidx, align 4 ->
+; CHECK-NEXT:           store i32 %add, i32* %arrayidx2, align 4
+
+define void @vectorizable_unscaled_Read_Write(i32* nocapture %A) {
+entry:
+  %0 = bitcast i32* %A to i8*
+  %add.ptr = getelementptr inbounds i8, i8* %0, i64 14
+  %1 = bitcast i8* %add.ptr to i32*
+  br label %for.body
+
+for.cond.cleanup:                                 ; preds = %for.body
+  ret void
+
+for.body:                                         ; preds = %entry, %for.body
+  %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+  %arrayidx = getelementptr inbounds i32, i32* %A, i64 %indvars.iv
+  %2 = load i32, i32* %arrayidx, align 4
+  %add = add nsw i32 %2, 1
+  %arrayidx2 = getelementptr inbounds i32, i32* %1, i64 %indvars.iv
+  store i32 %add, i32* %arrayidx2, align 4
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 2
+  %cmp = icmp ult i64 %indvars.iv.next, 1024
+  br i1 %cmp, label %for.body, label %for.cond.cleanup
+}
+
+; int vectorizable_unscaled_Write_Read(int *A) {
+;   int *B = (int *)((char *)A + 17);
+;   int sum = 0;
+;   for (unsigned i = 0; i < 1024; i+=2) {
+;     A[i] = i;
+;     sum += B[i];
+;   }
+; 
+;   return sum;
+; }
+
+; CHECK: for function 'vectorizable_unscaled_Write_Read':
+; CHECK-NEXT:   for.body:
+; CHECK-NEXT:     Memory dependences are safe
+; CHECK-NEXT:     Interesting Dependences:
+; CHECK-NEXT:       BackwardVectorizable:
+; CHECK-NEXT:           store i32 %2, i32* %arrayidx, align 4 -> 
+; CHECK-NEXT:           %3 = load i32, i32* %arrayidx2, align 4
+
+define i32 @vectorizable_unscaled_Write_Read(i32* nocapture %A) {
+entry:
+  %0 = bitcast i32* %A to i8*
+  %add.ptr = getelementptr inbounds i8, i8* %0, i64 17
+  %1 = bitcast i8* %add.ptr to i32*
+  br label %for.body
+
+for.cond.cleanup:                                 ; preds = %for.body
+  ret i32 %add
+
+for.body:                                         ; preds = %entry, %for.body
+  %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+  %sum.013 = phi i32 [ 0, %entry ], [ %add, %for.body ]
+  %arrayidx = getelementptr inbounds i32, i32* %A, i64 %indvars.iv
+  %2 = trunc i64 %indvars.iv to i32
+  store i32 %2, i32* %arrayidx, align 4
+  %arrayidx2 = getelementptr inbounds i32, i32* %1, i64 %indvars.iv
+  %3 = load i32, i32* %arrayidx2, align 4
+  %add = add nsw i32 %3, %sum.013
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 2
+  %cmp = icmp ult i64 %indvars.iv.next, 1024
+  br i1 %cmp, label %for.body, label %for.cond.cleanup
+}
+
+; void unsafe_unscaled_Read_Write(int *A) {
+;   int *B = (int *)((char *)A + 11);
+;   for (unsigned i = 0; i < 1024; i+=2)
+;     B[i] = A[i] + 1;
+; }
+
+; CHECK: function 'unsafe_unscaled_Read_Write':
+; CHECK-NEXT:   for.body:
+; CHECK-NEXT:     Report: unsafe dependent memory operations in loop
+; CHECK-NEXT:     Interesting Dependences:
+; CHECK-NEXT:       Backward:
+; CHECK-NEXT:           %2 = load i32, i32* %arrayidx, align 4 -> 
+; CHECK-NEXT:           store i32 %add, i32* %arrayidx2, align 4
+
+define void @unsafe_unscaled_Read_Write(i32* nocapture %A) {
+entry:
+  %0 = bitcast i32* %A to i8*
+  %add.ptr = getelementptr inbounds i8, i8* %0, i64 11
+  %1 = bitcast i8* %add.ptr to i32*
+  br label %for.body
+
+for.cond.cleanup:                                 ; preds = %for.body
+  ret void
+
+for.body:                                         ; preds = %entry, %for.body
+  %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+  %arrayidx = getelementptr inbounds i32, i32* %A, i64 %indvars.iv
+  %2 = load i32, i32* %arrayidx, align 4
+  %add = add nsw i32 %2, 1
+  %arrayidx2 = getelementptr inbounds i32, i32* %1, i64 %indvars.iv
+  store i32 %add, i32* %arrayidx2, align 4
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 2
+  %cmp = icmp ult i64 %indvars.iv.next, 1024
+  br i1 %cmp, label %for.body, label %for.cond.cleanup
+}
+
+; CHECK: function 'unsafe_unscaled_Read_Write2':
+; CHECK-NEXT:   for.body:
+; CHECK-NEXT:     Report: unsafe dependent memory operations in loop
+; CHECK-NEXT:     Interesting Dependences:
+; CHECK-NEXT:       Backward:
+; CHECK-NEXT:           %2 = load i32, i32* %arrayidx, align 4 -> 
+; CHECK-NEXT:           store i32 %add, i32* %arrayidx2, align 4
+
+; void unsafe_unscaled_Read_Write2(int *A) {
+;   int *B = (int *)((char *)A + 1);
+;   for (unsigned i = 0; i < 1024; i+=2)
+;     B[i] = A[i] + 1;
+; }
+
+define void @unsafe_unscaled_Read_Write2(i32* nocapture %A) {
+entry:
+  %0 = bitcast i32* %A to i8*
+  %add.ptr = getelementptr inbounds i8, i8* %0, i64 1
+  %1 = bitcast i8* %add.ptr to i32*
+  br label %for.body
+
+for.cond.cleanup:                                 ; preds = %for.body
+  ret void
+
+for.body:                                         ; preds = %entry, %for.body
+  %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+  %arrayidx = getelementptr inbounds i32, i32* %A, i64 %indvars.iv
+  %2 = load i32, i32* %arrayidx, align 4
+  %add = add nsw i32 %2, 1
+  %arrayidx2 = getelementptr inbounds i32, i32* %1, i64 %indvars.iv
+  store i32 %add, i32* %arrayidx2, align 4
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 2
+  %cmp = icmp ult i64 %indvars.iv.next, 1024
+  br i1 %cmp, label %for.body, label %for.cond.cleanup
+}
+
+; Following case checks that interleaved stores have dependences with another
+; store and can not pass dependence check.
+
+; void interleaved_stores(int *A) {
+;   int *B = (int *) ((char *)A + 1);
+;   for(int i = 0; i < 1024; i+=2) {
+;     B[i]   = i;                // (1)
+;     A[i+1] = i + 1;            // (2)
+;     B[i+1] = i + 1;            // (3)
+;   }
+; }
+;
+; The access (2) has overlaps with (1) and (3).
+
+; CHECK: function 'interleaved_stores':
+; CHECK-NEXT:   for.body:
+; CHECK-NEXT:     Report: unsafe dependent memory operations in loop
+; CHECK-NEXT:     Interesting Dependences:
+; CHECK-NEXT:       Backward:
+; CHECK-NEXT:           store i32 %4, i32* %arrayidx5, align 4 -> 
+; CHECK-NEXT:           store i32 %4, i32* %arrayidx9, align 4
+; CHECK:       Backward:
+; CHECK-NEXT:           store i32 %2, i32* %arrayidx2, align 4 -> 
+; CHECK-NEXT:           store i32 %4, i32* %arrayidx5, align 4
+
+define void @interleaved_stores(i32* nocapture %A) {
+entry:
+  %0 = bitcast i32* %A to i8*
+  %incdec.ptr = getelementptr inbounds i8, i8* %0, i64 1
+  %1 = bitcast i8* %incdec.ptr to i32*
+  br label %for.body
+
+for.cond.cleanup:                                 ; preds = %for.body
+  ret void
+
+for.body:                                         ; preds = %entry, %for.body
+  %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+  %2 = trunc i64 %indvars.iv to i32
+  %arrayidx2 = getelementptr inbounds i32, i32* %1, i64 %indvars.iv
+  store i32 %2, i32* %arrayidx2, align 4
+  %3 = or i64 %indvars.iv, 1
+  %arrayidx5 = getelementptr inbounds i32, i32* %A, i64 %3
+  %4 = trunc i64 %3 to i32
+  store i32 %4, i32* %arrayidx5, align 4
+  %arrayidx9 = getelementptr inbounds i32, i32* %1, i64 %3
+  store i32 %4, i32* %arrayidx9, align 4
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 2
+  %cmp = icmp slt i64 %indvars.iv.next, 1024
+  br i1 %cmp, label %for.body, label %for.cond.cleanup
+}
diff --git a/test/Analysis/LoopAccessAnalysis/underlying-objects-1.ll b/test/Analysis/LoopAccessAnalysis/underlying-objects-1.ll
new file mode 100644
index 00000000000..44820ed3722
--- /dev/null
+++ b/test/Analysis/LoopAccessAnalysis/underlying-objects-1.ll
@@ -0,0 +1,47 @@
+; RUN: opt -basicaa -loop-accesses -analyze < %s | FileCheck %s
+
+; In:
+;
+;   store_ptr = A;
+;   load_ptr = &A[2];
+;   for (i = 0; i < n; i++)
+;    *store_ptr++ = *load_ptr++ *10;  // A[i] = A[i+2] * 10
+;
+; make sure, we look through the PHI to conclude that store_ptr and load_ptr
+; both have A as their underlying object.  The dependence is safe for
+; vectorization requiring no memchecks.
+;
+; Otherwise we would try to prove independence with a memcheck that is going
+; to always fail.
+
+target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-apple-macosx10.10.0"
+
+; CHECK: Memory dependences are safe{{$}}
+
+define void @f(i8* noalias %A, i64 %width) {
+for.body.preheader:
+  %A_ahead = getelementptr inbounds i8, i8* %A, i64 2
+  br label %for.body
+
+for.body:
+  %i = phi i64 [ %i.1, %for.body ], [ 0, %for.body.preheader ]
+  %load_ptr = phi i8* [ %load_ptr.1, %for.body ], [ %A_ahead, %for.body.preheader ]
+  %store_ptr = phi i8* [ %store_ptr.1, %for.body ], [ %A, %for.body.preheader ]
+
+  %loadA = load i8, i8* %load_ptr, align 1
+
+  %mul = mul i8 %loadA, 10
+
+  store i8 %mul, i8* %store_ptr, align 1
+
+  %load_ptr.1 = getelementptr inbounds i8, i8* %load_ptr, i64 1
+  %store_ptr.1 = getelementptr inbounds i8, i8* %store_ptr, i64 1
+  %i.1 = add nuw i64 %i, 1
+
+  %exitcond = icmp eq i64 %i.1, %width
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:
+  ret void
+}
diff --git a/test/Analysis/LoopAccessAnalysis/underlying-objects-2.ll b/test/Analysis/LoopAccessAnalysis/underlying-objects-2.ll
new file mode 100644
index 00000000000..d0bed68188d
--- /dev/null
+++ b/test/Analysis/LoopAccessAnalysis/underlying-objects-2.ll
@@ -0,0 +1,90 @@
+; RUN: opt -basicaa -loop-accesses -analyze < %s | FileCheck %s
+
+; This loop:
+;
+;   int **A;
+;   for (i)
+;     for (j) {
+;        A[i][j] = A[i-1][j] * B[j]
+;        B[j+1] = 2       // backward dep between this and the previous
+;     }
+;
+; is transformed by Load-PRE to stash away A[i] for the next iteration of the
+; outer loop:
+;
+;   Curr = A[0];          // Prev_0
+;   for (i: 1..N) {
+;     Prev = Curr;        // Prev = PHI (Prev_0, Curr)
+;     Curr = A[i];
+;     for (j: 0..N) {
+;        Curr[j] = Prev[j] * B[j]
+;        B[j+1] = 2       // backward dep between this and the previous
+;     }
+;   }
+;
+; Since A[i] and A[i-1] are likely to be independent, getUnderlyingObjects
+; should not assume that Curr and Prev share the same underlying object.
+;
+; If it did we would try to dependence-analyze Curr and Prev and the analysis
+; would fail with non-constant distance.
+;
+; To illustrate one of the negative consequences of this, if the loop has a
+; backward dependence we won't detect this but instead fully fall back on
+; memchecks (that is what LAA does after encountering a case of non-constant
+; distance).
+
+target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-apple-macosx10.10.0"
+
+; CHECK: for_j.body:
+; CHECK-NEXT: Report: unsafe dependent memory operations in loop
+; CHECK-NEXT: Interesting Dependences:
+; CHECK-NEXT: Backward:
+; CHECK-NEXT: %loadB = load i8, i8* %gepB, align 1 ->
+; CHECK-NEXT: store i8 2, i8* %gepB_plus_one, align 1
+
+define void @f(i8** noalias %A, i8* noalias %B, i64 %N) {
+for_i.preheader:
+  %prev_0 = load i8*, i8** %A, align 8
+  br label %for_i.body
+
+for_i.body:
+  %i = phi i64 [1, %for_i.preheader], [%i.1, %for_j.end]
+  %prev = phi i8* [%prev_0, %for_i.preheader], [%curr, %for_j.end]
+  %gep = getelementptr inbounds i8*, i8** %A, i64 %i
+  %curr = load i8*, i8** %gep, align 8
+  br label %for_j.preheader
+
+for_j.preheader:
+  br label %for_j.body
+
+for_j.body:
+  %j = phi i64 [0, %for_j.preheader], [%j.1, %for_j.body]
+
+  %gepPrev = getelementptr inbounds i8, i8* %prev, i64 %j
+  %gepCurr = getelementptr inbounds i8, i8* %curr, i64 %j
+  %gepB = getelementptr inbounds i8, i8* %B, i64 %j
+
+  %loadPrev = load i8, i8* %gepPrev, align 1
+  %loadB = load i8, i8* %gepB, align 1
+
+  %mul = mul i8 %loadPrev, %loadB
+
+  store i8 %mul, i8* %gepCurr, align 1
+
+  %gepB_plus_one = getelementptr inbounds i8, i8* %gepB, i64 1
+  store i8 2, i8* %gepB_plus_one, align 1
+
+  %j.1 = add nuw i64 %j, 1
+  %exitcondj = icmp eq i64 %j.1, %N
+  br i1 %exitcondj, label %for_j.end, label %for_j.body
+
+for_j.end:
+
+  %i.1 = add nuw i64 %i, 1
+  %exitcond = icmp eq i64 %i.1, %N
+  br i1 %exitcond, label %for_i.end, label %for_i.body
+
+for_i.end:
+  ret void
+}
diff --git a/test/Analysis/LoopAccessAnalysis/unsafe-and-rt-checks.ll b/test/Analysis/LoopAccessAnalysis/unsafe-and-rt-checks.ll
index ce8b86ba2c5..237cbc8b987 100644
--- a/test/Analysis/LoopAccessAnalysis/unsafe-and-rt-checks.ll
+++ b/test/Analysis/LoopAccessAnalysis/unsafe-and-rt-checks.ll
@@ -14,10 +14,16 @@ target triple = "x86_64-apple-macosx10.10.0"
 ; CHECK-NEXT:     store i16 %mul1, i16* %arrayidxA_plus_2, align 2
 ; CHECK: Run-time memory checks:
 ; CHECK-NEXT: 0:
+; CHECK-NEXT: Comparing group
+; CHECK-NEXT:   %arrayidxA = getelementptr inbounds i16, i16* %a, i64 %storemerge3
 ; CHECK-NEXT:   %arrayidxA_plus_2 = getelementptr inbounds i16, i16* %a, i64 %add
+; CHECK-NEXT: Against group
 ; CHECK-NEXT:   %arrayidxB = getelementptr inbounds i16, i16* %b, i64 %storemerge3
 ; CHECK-NEXT: 1:
+; CHECK-NEXT: Comparing group
+; CHECK-NEXT:   %arrayidxA = getelementptr inbounds i16, i16* %a, i64 %storemerge3
 ; CHECK-NEXT:   %arrayidxA_plus_2 = getelementptr inbounds i16, i16* %a, i64 %add
+; CHECK-NEXT: Against group
 ; CHECK-NEXT:   %arrayidxC = getelementptr inbounds i16, i16* %c, i64 %storemerge3
 
 @B = common global i16* null, align 8
diff --git a/test/Analysis/MemoryDependenceAnalysis/memdep-block-scan-limit.ll b/test/Analysis/MemoryDependenceAnalysis/memdep-block-scan-limit.ll
new file mode 100644
index 00000000000..20ca31456fd
--- /dev/null
+++ b/test/Analysis/MemoryDependenceAnalysis/memdep-block-scan-limit.ll
@@ -0,0 +1,15 @@
+; RUN: opt -S -memdep -gvn -basicaa < %s | FileCheck %s
+; RUN: opt -S -memdep -memdep-block-scan-limit=1 -gvn -basicaa < %s | FileCheck %s --check-prefix=WITH-LIMIT
+; CHECK-LABEL: @test(
+; CHECK: load
+; CHECK-NOT: load
+; WITH-LIMIT-LABEL: @test(
+; WITH-LIMIT-CHECK: load
+; WITH-LIMIT-CHECK: load
+define i32 @test(i32* %p) {
+ %1 = load i32, i32* %p
+ %2 = add i32 %1, 3
+ %3 = load i32, i32* %p
+ %4 = add i32 %2, %3
+ ret i32 %4
+}
diff --git a/test/Analysis/ScalarEvolution/flags-from-poison.ll b/test/Analysis/ScalarEvolution/flags-from-poison.ll
new file mode 100644
index 00000000000..3fcb0e5bd23
--- /dev/null
+++ b/test/Analysis/ScalarEvolution/flags-from-poison.ll
@@ -0,0 +1,358 @@
+; RUN: opt < %s -S -analyze -scalar-evolution | FileCheck %s
+
+; Positive and negative tests for inferring flags like nsw from
+; reasoning about how a poison value from overflow would trigger
+; undefined behavior.
+
+define void @foo() {
+  ret void
+}
+
+; Example where an add should get the nsw flag, so that a sext can be
+; distributed over the add.
+define void @test-add-nsw(float* %input, i32 %offset, i32 %numIterations) {
+; CHECK-LABEL: @test-add-nsw
+entry:
+  br label %loop
+loop:
+  %i = phi i32 [ %nexti, %loop ], [ 0, %entry ]
+
+; CHECK: %index32 =
+; CHECK: --> {%offset,+,1}<nsw>
+  %index32 = add nsw i32 %i, %offset
+
+; CHECK: %index64 =
+; CHECK: --> {(sext i32 %offset to i64),+,1}<nsw>
+  %index64 = sext i32 %index32 to i64
+
+  %ptr = getelementptr inbounds float, float* %input, i64 %index64
+  %nexti = add nsw i32 %i, 1
+  %f = load float, float* %ptr, align 4
+  call void @foo()
+  %exitcond = icmp eq i32 %nexti, %numIterations
+  br i1 %exitcond, label %exit, label %loop
+exit:
+  ret void
+}
+
+; Example where an add should get the nuw flag.
+define void @test-add-nuw(float* %input, i32 %offset, i32 %numIterations) {
+; CHECK-LABEL: @test-add-nuw
+entry:
+  br label %loop
+loop:
+  %i = phi i32 [ %nexti, %loop ], [ 0, %entry ]
+
+; CHECK: %index32 =
+; CHECK: --> {%offset,+,1}<nuw>
+  %index32 = add nuw i32 %i, %offset
+
+  %ptr = getelementptr inbounds float, float* %input, i32 %index32
+  %nexti = add nuw i32 %i, 1
+  %f = load float, float* %ptr, align 4
+  %exitcond = icmp eq i32 %nexti, %numIterations
+  br i1 %exitcond, label %exit, label %loop
+
+exit:
+  ret void
+}
+
+; With no load to trigger UB from poison, we cannot infer nsw.
+define void @test-add-no-load(float* %input, i32 %offset, i32 %numIterations) {
+; CHECK-LABEL: @test-add-no-load
+entry:
+  br label %loop
+loop:
+  %i = phi i32 [ %nexti, %loop ], [ 0, %entry ]
+
+; CHECK: %index32 =
+; CHECK: --> {%offset,+,1}<nw>
+  %index32 = add nsw i32 %i, %offset
+
+  %ptr = getelementptr inbounds float, float* %input, i32 %index32
+  %nexti = add nuw i32 %i, 1
+  %exitcond = icmp eq i32 %nexti, %numIterations
+  br i1 %exitcond, label %exit, label %loop
+
+exit:
+  ret void
+}
+
+; The current code is only supposed to look at the loop header, so
+; it should not infer nsw in this case, as that would require looking
+; outside the loop header.
+define void @test-add-not-header(float* %input, i32 %offset, i32 %numIterations) {
+; CHECK-LABEL: @test-add-not-header
+entry:
+  br label %loop
+loop:
+  %i = phi i32 [ %nexti, %loop2 ], [ 0, %entry ]
+  br label %loop2
+loop2:
+
+; CHECK: %index32 =
+; CHECK: --> {%offset,+,1}<nw>
+  %index32 = add nsw i32 %i, %offset
+
+  %ptr = getelementptr inbounds float, float* %input, i32 %index32
+  %nexti = add nsw i32 %i, 1
+  %f = load float, float* %ptr, align 4
+  %exitcond = icmp eq i32 %nexti, %numIterations
+  br i1 %exitcond, label %exit, label %loop
+exit:
+  ret void
+}
+
+; Same thing as test-add-not-header, but in this case only the load
+; instruction is outside the loop header.
+define void @test-add-not-header2(float* %input, i32 %offset, i32 %numIterations) {
+; CHECK-LABEL: @test-add-not-header2
+entry:
+  br label %loop
+loop:
+  %i = phi i32 [ %nexti, %loop2 ], [ 0, %entry ]
+
+; CHECK: %index32 =
+; CHECK: --> {%offset,+,1}<nw>
+  %index32 = add nsw i32 %i, %offset
+
+  %ptr = getelementptr inbounds float, float* %input, i32 %index32
+  %nexti = add nsw i32 %i, 1
+  br label %loop2
+loop2:
+  %f = load float, float* %ptr, align 4
+  %exitcond = icmp eq i32 %nexti, %numIterations
+  br i1 %exitcond, label %exit, label %loop
+exit:
+  ret void
+}
+
+; The call instruction makes it not guaranteed that the add will be
+; executed, since it could run forever or throw an exception, so we
+; cannot assume that the UB is realized.
+define void @test-add-call(float* %input, i32 %offset, i32 %numIterations) {
+; CHECK-LABEL: @test-add-call
+entry:
+  br label %loop
+loop:
+  %i = phi i32 [ %nexti, %loop ], [ 0, %entry ]
+
+; CHECK: %index32 =
+; CHECK: --> {%offset,+,1}<nw>
+  call void @foo()
+  %index32 = add nsw i32 %i, %offset
+
+  %ptr = getelementptr inbounds float, float* %input, i32 %index32
+  %nexti = add nsw i32 %i, 1
+  %f = load float, float* %ptr, align 4
+  %exitcond = icmp eq i32 %nexti, %numIterations
+  br i1 %exitcond, label %exit, label %loop
+exit:
+  ret void
+}
+
+; Same issue as test-add-call, but this time the call is between the
+; producer of poison and the load that consumes it.
+define void @test-add-call2(float* %input, i32 %offset, i32 %numIterations) {
+; CHECK-LABEL: @test-add-call2
+entry:
+  br label %loop
+loop:
+  %i = phi i32 [ %nexti, %loop ], [ 0, %entry ]
+
+; CHECK: %index32 =
+; CHECK: --> {%offset,+,1}<nw>
+  %index32 = add nsw i32 %i, %offset
+
+  %ptr = getelementptr inbounds float, float* %input, i32 %index32
+  %nexti = add nsw i32 %i, 1
+  call void @foo()
+  %f = load float, float* %ptr, align 4
+  %exitcond = icmp eq i32 %nexti, %numIterations
+  br i1 %exitcond, label %exit, label %loop
+exit:
+  ret void
+}
+
+; Without inbounds, GEP does not propagate poison in the very
+; conservative approach used here.
+define void @test-add-no-inbounds(float* %input, i32 %offset, i32 %numIterations) {
+; CHECK-LABEL: @test-add-no-inbounds
+entry:
+  br label %loop
+loop:
+  %i = phi i32 [ %nexti, %loop ], [ 0, %entry ]
+
+; CHECK: %index32 =
+; CHECK: --> {%offset,+,1}<nw>
+  %index32 = add nsw i32 %i, %offset
+
+  %ptr = getelementptr float, float* %input, i32 %index32
+  %nexti = add nsw i32 %i, 1
+  %f = load float, float* %ptr, align 4
+  %exitcond = icmp eq i32 %nexti, %numIterations
+  br i1 %exitcond, label %exit, label %loop
+exit:
+  ret void
+}
+
+; Multiplication by a non-zero constant propagates poison if there is
+; a nuw or nsw flag on the multiplication.
+define void @test-add-mul-propagates(float* %input, i32 %offset, i32 %numIterations) {
+; CHECK-LABEL: @test-add-mul-propagates
+entry:
+  br label %loop
+loop:
+  %i = phi i32 [ %nexti, %loop ], [ 0, %entry ]
+
+; CHECK: %index32 =
+; CHECK: --> {%offset,+,1}<nsw>
+  %index32 = add nsw i32 %i, %offset
+
+  %indexmul = mul nuw i32 %index32, 2
+  %ptr = getelementptr inbounds float, float* %input, i32 %indexmul
+  %nexti = add nsw i32 %i, 1
+  %f = load float, float* %ptr, align 4
+  %exitcond = icmp eq i32 %nexti, %numIterations
+  br i1 %exitcond, label %exit, label %loop
+exit:
+  ret void
+}
+
+; Multiplication by a non-constant should not propagate poison in the
+; very conservative approach used here.
+define void @test-add-mul-no-propagation(float* %input, i32 %offset, i32 %numIterations) {
+; CHECK-LABEL: @test-add-mul-no-propagation
+entry:
+  br label %loop
+loop:
+  %i = phi i32 [ %nexti, %loop ], [ 0, %entry ]
+
+; CHECK: %index32 =
+; CHECK: --> {%offset,+,1}<nw>
+  %index32 = add nsw i32 %i, %offset
+
+  %indexmul = mul nsw i32 %index32, %offset
+  %ptr = getelementptr inbounds float, float* %input, i32 %indexmul
+  %nexti = add nsw i32 %i, 1
+  %f = load float, float* %ptr, align 4
+  %exitcond = icmp eq i32 %nexti, %numIterations
+  br i1 %exitcond, label %exit, label %loop
+exit:
+  ret void
+}
+
+; Multiplication by a non-zero constant does not propagate poison
+; without a no-wrap flag.
+define void @test-add-mul-no-propagation2(float* %input, i32 %offset, i32 %numIterations) {
+; CHECK-LABEL: @test-add-mul-no-propagation2
+entry:
+  br label %loop
+loop:
+  %i = phi i32 [ %nexti, %loop ], [ 0, %entry ]
+
+; CHECK: %index32 =
+; CHECK: --> {%offset,+,1}<nw>
+  %index32 = add nsw i32 %i, %offset
+
+  %indexmul = mul i32 %index32, 2
+  %ptr = getelementptr inbounds float, float* %input, i32 %indexmul
+  %nexti = add nsw i32 %i, 1
+  %f = load float, float* %ptr, align 4
+  %exitcond = icmp eq i32 %nexti, %numIterations
+  br i1 %exitcond, label %exit, label %loop
+exit:
+  ret void
+}
+
+; Division by poison triggers UB.
+define void @test-add-div(float* %input, i32 %offset, i32 %numIterations) {
+; CHECK-LABEL: @test-add-div
+entry:
+  br label %loop
+loop:
+  %i = phi i32 [ %nexti, %loop ], [ 0, %entry ]
+
+; CHECK: %j =
+; CHECK: --> {%offset,+,1}<nsw>
+  %j = add nsw i32 %i, %offset
+
+  %q = sdiv i32 %numIterations, %j
+  %nexti = add nsw i32 %i, 1
+  %exitcond = icmp eq i32 %nexti, %numIterations
+  br i1 %exitcond, label %exit, label %loop
+exit:
+  ret void
+}
+
+; Remainder of poison by non-poison divisor does not trigger UB.
+define void @test-add-div2(float* %input, i32 %offset, i32 %numIterations) {
+; CHECK-LABEL: @test-add-div2
+entry:
+  br label %loop
+loop:
+  %i = phi i32 [ %nexti, %loop ], [ 0, %entry ]
+
+; CHECK: %j =
+; CHECK: --> {%offset,+,1}<nw>
+  %j = add nsw i32 %i, %offset
+
+  %q = sdiv i32 %j, %numIterations
+  %nexti = add nsw i32 %i, 1
+  %exitcond = icmp eq i32 %nexti, %numIterations
+  br i1 %exitcond, label %exit, label %loop
+exit:
+  ret void
+}
+
+; Store to poison address triggers UB.
+define void @test-add-store(float* %input, i32 %offset, i32 %numIterations) {
+; CHECK-LABEL: @test-add-store
+entry:
+  br label %loop
+loop:
+  %i = phi i32 [ %nexti, %loop ], [ 0, %entry ]
+
+; CHECK: %index32 =
+; CHECK: --> {%offset,+,1}<nsw>
+  %index32 = add nsw i32 %i, %offset
+
+  %ptr = getelementptr inbounds float, float* %input, i32 %index32
+  %nexti = add nsw i32 %i, 1
+  store float 1.0, float* %ptr, align 4
+  %exitcond = icmp eq i32 %nexti, %numIterations
+  br i1 %exitcond, label %exit, label %loop
+exit:
+  ret void
+}
+
+; Three sequential adds where the middle add should have nsw. There is
+; a special case for sequential adds and this test covers that. We have to
+; put the final add first in the program since otherwise the special case
+; is not triggered, hence the strange basic block ordering.
+define void @test-add-twice(float* %input, i32 %offset, i32 %numIterations) {
+; CHECK-LABEL: @test-add-twice
+entry:
+  br label %loop
+loop2:
+; CHECK: %seq =
+; CHECK: --> {(2 + %offset),+,1}<nw>
+  %seq = add nsw nuw i32 %index32, 1
+  %exitcond = icmp eq i32 %nexti, %numIterations
+  br i1 %exitcond, label %exit, label %loop
+
+loop:
+  %i = phi i32 [ %nexti, %loop2 ], [ 0, %entry ]
+
+  %j = add nsw i32 %i, 1
+; CHECK: %index32 =
+; CHECK: --> {(1 + %offset),+,1}<nsw>
+  %index32 = add nsw i32 %j, %offset
+
+  %ptr = getelementptr inbounds float, float* %input, i32 %index32
+  %nexti = add nsw i32 %i, 1
+  store float 1.0, float* %ptr, align 4
+  br label %loop2
+exit:
+  ret void
+}
diff --git a/test/Analysis/ScopedNoAliasAA/basic-domains.ll b/test/Analysis/ScopedNoAliasAA/basic-domains.ll
index c2b5bbd4e06..2fea2ba068a 100644
--- a/test/Analysis/ScopedNoAliasAA/basic-domains.ll
+++ b/test/Analysis/ScopedNoAliasAA/basic-domains.ll
@@ -5,42 +5,40 @@ target triple = "x86_64-unknown-linux-gnu"
 define void @foo1(float* nocapture %a, float* nocapture readonly %c) #0 {
 entry:
 ; CHECK-LABEL: Function: foo1
-  %0 = load float, float* %c, align 4, !alias.scope !9
+  %0 = load float, float* %c, align 4, !alias.scope !0
   %arrayidx.i = getelementptr inbounds float, float* %a, i64 5
   store float %0, float* %arrayidx.i, align 4, !noalias !6
 
-  %1 = load float, float* %c, align 4, !alias.scope !5
+  %1 = load float, float* %c, align 4, !alias.scope !7
   %arrayidx.i2 = getelementptr inbounds float, float* %a, i64 15
   store float %1, float* %arrayidx.i2, align 4, !noalias !6
 
   %2 = load float, float* %c, align 4, !alias.scope !6
   %arrayidx.i3 = getelementptr inbounds float, float* %a, i64 16
-  store float %2, float* %arrayidx.i3, align 4, !noalias !5
+  store float %2, float* %arrayidx.i3, align 4, !noalias !7
 
   ret void
 }
 
 attributes #0 = { nounwind uwtable }
 
-!0 = !{!0, !"some domain"}
-!1 = !{!1, !"some other domain"}
+!2 = !{!2, !"some domain"}
+!5 = !{!5, !"some other domain"}
 
 ; Two scopes (which must be self-referential to avoid being "uniqued"):
-!2 = !{!2, !0, !"a scope in dom0"}
-!3 = !{!2}
+!1 = !{!1, !2, !"a scope in dom0"}
 
-!4 = !{!4, !0, !"another scope in dom0"}
-!5 = !{!4}
+!3 = !{!3, !2, !"another scope in dom0"}
+!7 = !{!3}
 
 ; A list of the two scopes.
-!6 = !{!2, !4}
+!6 = !{!1, !3}
 
 ; Another scope in the second domain
-!7 = !{!7, !1, !"another scope in dom1"}
-!8 = !{!7}
+!4 = !{!4, !5, !"another scope in dom1"}
 
 ; A list of scopes from both domains.
-!9 = !{!2, !4, !7}
+!0 = !{!1, !3, !4}
 
 ; CHECK: NoAlias:   %0 = load float, float* %c, align 4, !alias.scope !0 <->   store float %0, float* %arrayidx.i, align 4, !noalias !6
 ; CHECK: NoAlias:   %0 = load float, float* %c, align 4, !alias.scope !0 <->   store float %1, float* %arrayidx.i2, align 4, !noalias !6
diff --git a/test/Analysis/ValueTracking/assume.ll b/test/Analysis/ValueTracking/assume.ll
new file mode 100644
index 00000000000..4bffe8ef790
--- /dev/null
+++ b/test/Analysis/ValueTracking/assume.ll
@@ -0,0 +1,14 @@
+; RUN: opt < %s -instcombine -S | FileCheck %s
+
+define i32 @assume_add(i32 %a, i32 %b) {
+; CHECK-LABEL: @assume_add(
+  %1 = add i32 %a, %b
+  %last_two_digits = and i32 %1, 3
+  %2 = icmp eq i32 %last_two_digits, 0
+  call void @llvm.assume(i1 %2)
+  %3 = add i32 %1, 3
+; CHECK: %3 = or i32 %1, 3
+  ret i32 %3
+}
+
+declare void @llvm.assume(i1)
diff --git a/test/Analysis/ValueTracking/dom-cond.ll b/test/Analysis/ValueTracking/dom-cond.ll
new file mode 100644
index 00000000000..c0cafdd0ade
--- /dev/null
+++ b/test/Analysis/ValueTracking/dom-cond.ll
@@ -0,0 +1,18 @@
+; RUN: opt < %s -instcombine -value-tracking-dom-conditions -S | FileCheck %s
+
+define i32 @dom_cond(i32 %a, i32 %b) {
+; CHECK-LABEL: @dom_cond(
+entry:
+  %v = add i32 %a, %b
+  %cond = icmp ule i32 %v, 7
+  br i1 %cond, label %then, label %exit
+
+then:
+  %v2 = add i32 %v, 8
+; CHECK: or i32 %v, 8
+  br label %exit
+
+exit:
+  %v3 = phi i32 [ %v, %entry ], [ %v2, %then ]
+  ret i32 %v3
+}
diff --git a/test/Analysis/ValueTracking/memory-dereferenceable.ll b/test/Analysis/ValueTracking/memory-dereferenceable.ll
index bfee5c78df6..f49f4f77f40 100644
--- a/test/Analysis/ValueTracking/memory-dereferenceable.ll
+++ b/test/Analysis/ValueTracking/memory-dereferenceable.ll
@@ -8,6 +8,10 @@ target datalayout = "e"
 declare zeroext i1 @return_i1()
 
 @globalstr = global [6 x i8] c"hello\00"
+@globali32ptr = external global i32*
+
+%struct.A = type { [8 x i8], [5 x i8] }
+@globalstruct = external global %struct.A
 
 define void @test(i32 addrspace(1)* dereferenceable(8) %dparam) gc "statepoint-example" {
 ; CHECK: The following are dereferenceable:
@@ -16,19 +20,59 @@ define void @test(i32 addrspace(1)* dereferenceable(8) %dparam) gc "statepoint-e
 ; CHECK: %dparam
 ; CHECK: %relocate
 ; CHECK-NOT: %nparam
+; CHECK-NOT: %nd_load
+; CHECK: %d4_load
+; CHECK-NOT: %d2_load
+; CHECK-NOT: %d_or_null_load
+; CHECK: %d_or_null_non_null_load
+; CHECK: %within_allocation
+; CHECK-NOT: %outside_allocation
 entry:
     %globalptr = getelementptr inbounds [6 x i8], [6 x i8]* @globalstr, i32 0, i32 0
     %load1 = load i8, i8* %globalptr
     %alloca = alloca i1
     %load2 = load i1, i1* %alloca
     %load3 = load i32, i32 addrspace(1)* %dparam
-    %tok = tail call i32 (i1 ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_i1f(i1 ()* @return_i1, i32 0, i32 0, i32 0, i32 addrspace(1)* %dparam)
-    %relocate = call i32 addrspace(1)* @llvm.experimental.gc.relocate.p1i32(i32 %tok, i32 4, i32 4)
+    %tok = tail call i32 (i64, i32, i1 ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_i1f(i64 0, i32 0, i1 ()* @return_i1, i32 0, i32 0, i32 0, i32 0, i32 addrspace(1)* %dparam)
+    %relocate = call i32 addrspace(1)* @llvm.experimental.gc.relocate.p1i32(i32 %tok, i32 7, i32 7)
     %load4 = load i32, i32 addrspace(1)* %relocate
     %nparam = getelementptr i32, i32 addrspace(1)* %dparam, i32 5
     %load5 = load i32, i32 addrspace(1)* %nparam
+
+    ; Load from a non-dereferenceable load
+    %nd_load = load i32*, i32** @globali32ptr
+    %load6 = load i32, i32* %nd_load
+
+    ; Load from a dereferenceable load
+    %d4_load = load i32*, i32** @globali32ptr, !dereferenceable !0
+    %load7 = load i32, i32* %d4_load
+
+    ; Load from an offset not covered by the dereferenceable portion
+    %d2_load = load i32*, i32** @globali32ptr, !dereferenceable !1
+    %load8 = load i32, i32* %d2_load
+
+    ; Load from a potentially null pointer with dereferenceable_or_null
+    %d_or_null_load = load i32*, i32** @globali32ptr, !dereferenceable_or_null !0
+    %load9 = load i32, i32* %d_or_null_load
+
+    ; Load from a non-null pointer with dereferenceable_or_null
+    %d_or_null_non_null_load = load i32*, i32** @globali32ptr, !nonnull !2, !dereferenceable_or_null !0
+    %load10 = load i32, i32* %d_or_null_non_null_load
+
+    ; It's OK to overrun static array size as long as we stay within underlying object size
+    %within_allocation = getelementptr inbounds %struct.A, %struct.A* @globalstruct, i64 0, i32 0, i64 10
+    %load11 = load i8, i8* %within_allocation
+
+    ; GEP is outside the underlying object size
+    %outside_allocation = getelementptr inbounds %struct.A, %struct.A* @globalstruct, i64 0, i32 1, i64 10
+    %load12 = load i8, i8* %outside_allocation
+
     ret void
 }
 
-declare i32 @llvm.experimental.gc.statepoint.p0f_i1f(i1 ()*, i32, i32, ...)
+declare i32 @llvm.experimental.gc.statepoint.p0f_i1f(i64, i32, i1 ()*, i32, i32, ...)
 declare i32 addrspace(1)* @llvm.experimental.gc.relocate.p1i32(i32, i32, i32)
+
+!0 = !{i64 4}
+!1 = !{i64 2}
+!2 = !{}
diff --git a/test/Assembler/2010-02-05-FunctionLocalMetadataBecomesNull.ll b/test/Assembler/2010-02-05-FunctionLocalMetadataBecomesNull.ll
index 24d17139c85..8a6735fbca4 100644
--- a/test/Assembler/2010-02-05-FunctionLocalMetadataBecomesNull.ll
+++ b/test/Assembler/2010-02-05-FunctionLocalMetadataBecomesNull.ll
@@ -13,7 +13,7 @@ target triple = "x86_64-apple-darwin10.2"
 define i32 @main() nounwind readonly {
   %diff1 = alloca i64                             ; <i64*> [#uses=2]
 ; CHECK: call void @llvm.dbg.value(metadata i64 72,
-  call void @llvm.dbg.declare(metadata i64* %diff1, metadata !0, metadata !MDExpression()), !dbg !MDLocation(scope: !1)
+  call void @llvm.dbg.declare(metadata i64* %diff1, metadata !0, metadata !DIExpression()), !dbg !DILocation(scope: !1)
   store i64 72, i64* %diff1, align 8
   %v1 = load %struct.test*, %struct.test** @TestArrayPtr, align 8 ; <%struct.test*> [#uses=1]
   %v2 = ptrtoint %struct.test* %v1 to i64 ; <i64> [#uses=1]
@@ -25,14 +25,14 @@ define i32 @main() nounwind readonly {
 declare void @llvm.dbg.declare(metadata, metadata, metadata) nounwind readnone
 
 !7 = !{!1}
-!6 = !MDCompileUnit(language: DW_LANG_C99, producer: "clang version 3.0 (trunk 131941)", isOptimized: true, emissionKind: 0, file: !8, enums: !9, retainedTypes: !9, subprograms: !7)
-!0 = !MDLocalVariable(tag: DW_TAG_auto_variable, name: "c", line: 2, scope: !1, file: !2, type: !5)
-!1 = !MDSubprogram(name: "main", line: 1, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 1, file: !8, scope: !2, type: !3, function: i32 ()* @main)
-!2 = !MDFile(filename: "/d/j/debug-test.c", directory: "/Volumes/Data/b")
-!3 = !MDSubroutineType(types: !4)
+!6 = !DICompileUnit(language: DW_LANG_C99, producer: "clang version 3.0 (trunk 131941)", isOptimized: true, emissionKind: 0, file: !8, enums: !9, retainedTypes: !9, subprograms: !7)
+!0 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "c", line: 2, scope: !1, file: !2, type: !5)
+!1 = !DISubprogram(name: "main", line: 1, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 1, file: !8, scope: !2, type: !3, function: i32 ()* @main)
+!2 = !DIFile(filename: "/d/j/debug-test.c", directory: "/Volumes/Data/b")
+!3 = !DISubroutineType(types: !4)
 !4 = !{!5}
-!5 = !MDBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
-!8 = !MDFile(filename: "/d/j/debug-test.c", directory: "/Volumes/Data/b")
+!5 = !DIBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
+!8 = !DIFile(filename: "/d/j/debug-test.c", directory: "/Volumes/Data/b")
 !9 = !{i32 0}
 
 !llvm.module.flags = !{!10}
diff --git a/test/Assembler/debug-info.ll b/test/Assembler/debug-info.ll
index 417c479a8be..91dfe561a2f 100644
--- a/test/Assembler/debug-info.ll
+++ b/test/Assembler/debug-info.ll
@@ -4,69 +4,69 @@
 ; CHECK: !named = !{!0, !0, !1, !2, !3, !4, !5, !6, !7, !8, !8, !9, !10, !11, !12, !13, !14, !15, !16, !17, !18, !19, !20, !21, !22, !23, !24, !25, !26, !27, !27}
 !named = !{!0, !1, !2, !3, !4, !5, !6, !7, !8, !9, !10, !11, !12, !13, !14, !15, !16, !17, !18, !19, !20, !21, !22, !23, !24, !25, !26, !27, !28, !29, !30}
 
-; CHECK:      !0 = !MDSubrange(count: 3)
-; CHECK-NEXT: !1 = !MDSubrange(count: 3, lowerBound: 4)
-; CHECK-NEXT: !2 = !MDSubrange(count: 3, lowerBound: -5)
-!0 = !MDSubrange(count: 3)
-!1 = !MDSubrange(count: 3, lowerBound: 0)
+; CHECK:      !0 = !DISubrange(count: 3)
+; CHECK-NEXT: !1 = !DISubrange(count: 3, lowerBound: 4)
+; CHECK-NEXT: !2 = !DISubrange(count: 3, lowerBound: -5)
+!0 = !DISubrange(count: 3)
+!1 = !DISubrange(count: 3, lowerBound: 0)
 
-!2 = !MDSubrange(count: 3, lowerBound: 4)
-!3 = !MDSubrange(count: 3, lowerBound: -5)
+!2 = !DISubrange(count: 3, lowerBound: 4)
+!3 = !DISubrange(count: 3, lowerBound: -5)
 
-; CHECK-NEXT: !3 = !MDEnumerator(name: "seven", value: 7)
-; CHECK-NEXT: !4 = !MDEnumerator(name: "negeight", value: -8)
-; CHECK-NEXT: !5 = !MDEnumerator(name: "", value: 0)
-!4 = !MDEnumerator(name: "seven", value: 7)
-!5 = !MDEnumerator(name: "negeight", value: -8)
-!6 = !MDEnumerator(name: "", value: 0)
+; CHECK-NEXT: !3 = !DIEnumerator(name: "seven", value: 7)
+; CHECK-NEXT: !4 = !DIEnumerator(name: "negeight", value: -8)
+; CHECK-NEXT: !5 = !DIEnumerator(name: "", value: 0)
+!4 = !DIEnumerator(name: "seven", value: 7)
+!5 = !DIEnumerator(name: "negeight", value: -8)
+!6 = !DIEnumerator(name: "", value: 0)
 
-; CHECK-NEXT: !6 = !MDBasicType(name: "name", size: 1, align: 2, encoding: DW_ATE_unsigned_char)
-; CHECK-NEXT: !7 = !MDBasicType(tag: DW_TAG_unspecified_type, name: "decltype(nullptr)")
-; CHECK-NEXT: !8 = !MDBasicType()
-!7 = !MDBasicType(tag: DW_TAG_base_type, name: "name", size: 1, align: 2, encoding: DW_ATE_unsigned_char)
-!8 = !MDBasicType(tag: DW_TAG_unspecified_type, name: "decltype(nullptr)")
-!9 = !MDBasicType()
-!10 = !MDBasicType(tag: DW_TAG_base_type, name: "", size: 0, align: 0, encoding: 0)
+; CHECK-NEXT: !6 = !DIBasicType(name: "name", size: 1, align: 2, encoding: DW_ATE_unsigned_char)
+; CHECK-NEXT: !7 = !DIBasicType(tag: DW_TAG_unspecified_type, name: "decltype(nullptr)")
+; CHECK-NEXT: !8 = !DIBasicType()
+!7 = !DIBasicType(tag: DW_TAG_base_type, name: "name", size: 1, align: 2, encoding: DW_ATE_unsigned_char)
+!8 = !DIBasicType(tag: DW_TAG_unspecified_type, name: "decltype(nullptr)")
+!9 = !DIBasicType()
+!10 = !DIBasicType(tag: DW_TAG_base_type, name: "", size: 0, align: 0, encoding: 0)
 
-; CHECK-NEXT: !9 = !MDTemplateTypeParameter(type: !6)
-; CHECK-NEXT: !10 = !MDFile(filename: "path/to/file", directory: "/path/to/dir")
+; CHECK-NEXT: !9 = !DITemplateTypeParameter(type: !6)
+; CHECK-NEXT: !10 = !DIFile(filename: "path/to/file", directory: "/path/to/dir")
 ; CHECK-NEXT: !11 = distinct !{}
-; CHECK-NEXT: !12 = !MDFile(filename: "", directory: "")
-!11 = !MDTemplateTypeParameter(type: !7)
-!12 = !MDFile(filename: "path/to/file", directory: "/path/to/dir")
+; CHECK-NEXT: !12 = !DIFile(filename: "", directory: "")
+!11 = !DITemplateTypeParameter(type: !7)
+!12 = !DIFile(filename: "path/to/file", directory: "/path/to/dir")
 !13 = distinct !{}
-!14 = !MDFile(filename: "", directory: "")
+!14 = !DIFile(filename: "", directory: "")
 
-; CHECK-NEXT: !13 = !MDDerivedType(tag: DW_TAG_pointer_type, baseType: !6, size: 32, align: 32)
-!15 = !MDDerivedType(tag: DW_TAG_pointer_type, baseType: !7, size: 32, align: 32)
+; CHECK-NEXT: !13 = !DIDerivedType(tag: DW_TAG_pointer_type, baseType: !6, size: 32, align: 32)
+!15 = !DIDerivedType(tag: DW_TAG_pointer_type, baseType: !7, size: 32, align: 32)
 
-; CHECK-NEXT: !14 = !MDCompositeType(tag: DW_TAG_structure_type, name: "MyType", file: !10, line: 2, size: 32, align: 32, identifier: "MangledMyType")
-; CHECK-NEXT: !15 = distinct !MDCompositeType(tag: DW_TAG_structure_type, name: "Base", scope: !14, file: !10, line: 3, size: 128, align: 32, offset: 64, flags: DIFlagPublic, elements: !16, runtimeLang: DW_LANG_C_plus_plus_11, vtableHolder: !15, templateParams: !18, identifier: "MangledBase")
+; CHECK-NEXT: !14 = !DICompositeType(tag: DW_TAG_structure_type, name: "MyType", file: !10, line: 2, size: 32, align: 32, identifier: "MangledMyType")
+; CHECK-NEXT: !15 = distinct !DICompositeType(tag: DW_TAG_structure_type, name: "Base", scope: !14, file: !10, line: 3, size: 128, align: 32, offset: 64, flags: DIFlagPublic, elements: !16, runtimeLang: DW_LANG_C_plus_plus_11, vtableHolder: !15, templateParams: !18, identifier: "MangledBase")
 ; CHECK-NEXT: !16 = !{!17}
-; CHECK-NEXT: !17 = !MDDerivedType(tag: DW_TAG_member, name: "field", scope: !15, file: !10, line: 4, baseType: !6, size: 32, align: 32, offset: 32, flags: DIFlagPublic)
+; CHECK-NEXT: !17 = !DIDerivedType(tag: DW_TAG_member, name: "field", scope: !15, file: !10, line: 4, baseType: !6, size: 32, align: 32, offset: 32, flags: DIFlagPublic)
 ; CHECK-NEXT: !18 = !{!9}
-; CHECK-NEXT: !19 = !MDCompositeType(tag: DW_TAG_structure_type, name: "Derived", scope: !14, file: !10, line: 3, baseType: !15, size: 128, align: 32, offset: 64, flags: DIFlagPublic, elements: !20, runtimeLang: DW_LANG_C_plus_plus_11, vtableHolder: !15, templateParams: !18, identifier: "MangledBase")
+; CHECK-NEXT: !19 = !DICompositeType(tag: DW_TAG_structure_type, name: "Derived", scope: !14, file: !10, line: 3, baseType: !15, size: 128, align: 32, offset: 64, flags: DIFlagPublic, elements: !20, runtimeLang: DW_LANG_C_plus_plus_11, vtableHolder: !15, templateParams: !18, identifier: "MangledBase")
 ; CHECK-NEXT: !20 = !{!21}
-; CHECK-NEXT: !21 = !MDDerivedType(tag: DW_TAG_inheritance, scope: !19, baseType: !15)
-; CHECK-NEXT: !22 = !MDDerivedType(tag: DW_TAG_ptr_to_member_type, baseType: !6, size: 32, align: 32, extraData: !15)
-; CHECK-NEXT: !23 = !MDCompositeType(tag: DW_TAG_structure_type)
-; CHECK-NEXT: !24 = !MDCompositeType(tag: DW_TAG_structure_type, runtimeLang: DW_LANG_Cobol85)
-!16 = !MDCompositeType(tag: DW_TAG_structure_type, name: "MyType", file: !12, line: 2, size: 32, align: 32, identifier: "MangledMyType")
-!17 = !MDCompositeType(tag: DW_TAG_structure_type, name: "Base", scope: !16, file: !12, line: 3, size: 128, align: 32, offset: 64, flags: DIFlagPublic, elements: !18, runtimeLang: DW_LANG_C_plus_plus_11, vtableHolder: !17, templateParams: !20, identifier: "MangledBase")
+; CHECK-NEXT: !21 = !DIDerivedType(tag: DW_TAG_inheritance, scope: !19, baseType: !15)
+; CHECK-NEXT: !22 = !DIDerivedType(tag: DW_TAG_ptr_to_member_type, baseType: !6, size: 32, align: 32, extraData: !15)
+; CHECK-NEXT: !23 = !DICompositeType(tag: DW_TAG_structure_type)
+; CHECK-NEXT: !24 = !DICompositeType(tag: DW_TAG_structure_type, runtimeLang: DW_LANG_Cobol85)
+!16 = !DICompositeType(tag: DW_TAG_structure_type, name: "MyType", file: !12, line: 2, size: 32, align: 32, identifier: "MangledMyType")
+!17 = !DICompositeType(tag: DW_TAG_structure_type, name: "Base", scope: !16, file: !12, line: 3, size: 128, align: 32, offset: 64, flags: DIFlagPublic, elements: !18, runtimeLang: DW_LANG_C_plus_plus_11, vtableHolder: !17, templateParams: !20, identifier: "MangledBase")
 !18 = !{!19}
-!19 = !MDDerivedType(tag: DW_TAG_member, name: "field", scope: !17, file: !12, line: 4, baseType: !7, size: 32, align: 32, offset: 32, flags: DIFlagPublic)
+!19 = !DIDerivedType(tag: DW_TAG_member, name: "field", scope: !17, file: !12, line: 4, baseType: !7, size: 32, align: 32, offset: 32, flags: DIFlagPublic)
 !20 = !{!11}
-!21 = !MDCompositeType(tag: DW_TAG_structure_type, name: "Derived", scope: !16, file: !12, line: 3, baseType: !17, size: 128, align: 32, offset: 64, flags: DIFlagPublic, elements: !22, runtimeLang: DW_LANG_C_plus_plus_11, vtableHolder: !17, templateParams: !20, identifier: "MangledBase")
+!21 = !DICompositeType(tag: DW_TAG_structure_type, name: "Derived", scope: !16, file: !12, line: 3, baseType: !17, size: 128, align: 32, offset: 64, flags: DIFlagPublic, elements: !22, runtimeLang: DW_LANG_C_plus_plus_11, vtableHolder: !17, templateParams: !20, identifier: "MangledBase")
 !22 = !{!23}
-!23 = !MDDerivedType(tag: DW_TAG_inheritance, scope: !21, baseType: !17)
-!24 = !MDDerivedType(tag: DW_TAG_ptr_to_member_type, baseType: !7, size: 32, align: 32, extraData: !17)
-!25 = !MDCompositeType(tag: DW_TAG_structure_type)
-!26 = !MDCompositeType(tag: DW_TAG_structure_type, runtimeLang: 6)
+!23 = !DIDerivedType(tag: DW_TAG_inheritance, scope: !21, baseType: !17)
+!24 = !DIDerivedType(tag: DW_TAG_ptr_to_member_type, baseType: !7, size: 32, align: 32, extraData: !17)
+!25 = !DICompositeType(tag: DW_TAG_structure_type)
+!26 = !DICompositeType(tag: DW_TAG_structure_type, runtimeLang: 6)
 
 ; !25 = !{!7, !7}
-; !26 = !MDSubroutineType(flags: DIFlagPublic | DIFlagStaticMember, types: !25)
-; !27 = !MDSubroutineType(types: !25)
+; !26 = !DISubroutineType(flags: DIFlagPublic | DIFlagStaticMember, types: !25)
+; !27 = !DISubroutineType(types: !25)
 !27 = !{!7, !7}
-!28 = !MDSubroutineType(flags: DIFlagPublic | DIFlagStaticMember, types: !27)
-!29 = !MDSubroutineType(flags: 0, types: !27)
-!30 = !MDSubroutineType(types: !27)
+!28 = !DISubroutineType(flags: DIFlagPublic | DIFlagStaticMember, types: !27)
+!29 = !DISubroutineType(flags: 0, types: !27)
+!30 = !DISubroutineType(types: !27)
diff --git a/test/Assembler/dicompileunit.ll b/test/Assembler/dicompileunit.ll
new file mode 100644
index 00000000000..dc136f0b83e
--- /dev/null
+++ b/test/Assembler/dicompileunit.ll
@@ -0,0 +1,31 @@
+; RUN: llvm-as < %s | llvm-dis | llvm-as | llvm-dis | FileCheck %s
+; RUN: verify-uselistorder %s
+
+; CHECK: !named = !{!0, !1, !2, !3, !4, !5, !6, !7, !7, !8, !8}
+!named = !{!0, !1, !2, !3, !4, !5, !6, !7, !8, !9, !10}
+
+!0 = distinct !{}
+!1 = !DIFile(filename: "path/to/file", directory: "/path/to/dir")
+!2 = distinct !{}
+!3 = distinct !{}
+!4 = distinct !{}
+!5 = distinct !{}
+!6 = distinct !{}
+
+; CHECK: !7 = !DICompileUnit(language: DW_LANG_C99, file: !1, producer: "clang", isOptimized: true, flags: "-O2", runtimeVersion: 2, splitDebugFilename: "abc.debug", emissionKind: 3, enums: !2, retainedTypes: !3, subprograms: !4, globals: !5, imports: !6, dwoId: 42)
+!7 = !DICompileUnit(language: DW_LANG_C99, file: !1, producer: "clang",
+                    isOptimized: true, flags: "-O2", runtimeVersion: 2,
+                    splitDebugFilename: "abc.debug", emissionKind: 3,
+                    enums: !2, retainedTypes: !3, subprograms: !4,
+                    globals: !5, imports: !6, dwoId: 42)
+!8 = !DICompileUnit(language: 12, file: !1, producer: "clang",
+                    isOptimized: true, flags: "-O2", runtimeVersion: 2,
+                    splitDebugFilename: "abc.debug", emissionKind: 3,
+                    enums: !2, retainedTypes: !3, subprograms: !4,
+                    globals: !5, imports: !6, dwoId: 42)
+
+; CHECK: !8 = !DICompileUnit(language: DW_LANG_C99, file: !1, isOptimized: false, runtimeVersion: 0, emissionKind: 0)
+!9 = !DICompileUnit(language: 12, file: !1, producer: "",
+                    isOptimized: false, flags: "", runtimeVersion: 0,
+                    splitDebugFilename: "", emissionKind: 0)
+!10 = !DICompileUnit(language: 12, file: !1)
diff --git a/test/Assembler/diexpression.ll b/test/Assembler/diexpression.ll
new file mode 100644
index 00000000000..31be86cef2d
--- /dev/null
+++ b/test/Assembler/diexpression.ll
@@ -0,0 +1,16 @@
+; RUN: llvm-as < %s | llvm-dis | llvm-as | llvm-dis | FileCheck %s
+; RUN: verify-uselistorder %s
+
+; CHECK: !named = !{!0, !1, !2, !3, !4}
+!named = !{!0, !1, !2, !3, !4}
+
+; CHECK:      !0 = !DIExpression()
+; CHECK-NEXT: !1 = !DIExpression(DW_OP_deref)
+; CHECK-NEXT: !2 = !DIExpression(DW_OP_plus, 3)
+; CHECK-NEXT: !3 = !DIExpression(DW_OP_bit_piece, 3, 7)
+; CHECK-NEXT: !4 = !DIExpression(DW_OP_deref, DW_OP_plus, 3, DW_OP_bit_piece, 3, 7)
+!0 = !DIExpression()
+!1 = !DIExpression(DW_OP_deref)
+!2 = !DIExpression(DW_OP_plus, 3)
+!3 = !DIExpression(DW_OP_bit_piece, 3, 7)
+!4 = !DIExpression(DW_OP_deref, DW_OP_plus, 3, DW_OP_bit_piece, 3, 7)
diff --git a/test/Assembler/difile-escaped-chars.ll b/test/Assembler/difile-escaped-chars.ll
new file mode 100644
index 00000000000..5900fdca061
--- /dev/null
+++ b/test/Assembler/difile-escaped-chars.ll
@@ -0,0 +1,9 @@
+; RUN: llvm-as < %s | llvm-dis | llvm-as | llvm-dis | FileCheck %s
+; RUN: verify-uselistorder %s
+
+; Spot-check that metadata string fields are correctly escaped.
+
+!named = !{!0}
+
+; CHECK: !0 = !DIFile(filename: "\00\01\02\80\81\82\FD\FE\FF", directory: "/dir")
+!0 = !DIFile(filename: "\00\01\02\80\81\82\FD\FE\FF", directory: "/dir")
diff --git a/test/Assembler/diglobalvariable.ll b/test/Assembler/diglobalvariable.ll
new file mode 100644
index 00000000000..0d027d310f6
--- /dev/null
+++ b/test/Assembler/diglobalvariable.ll
@@ -0,0 +1,27 @@
+; RUN: llvm-as < %s | llvm-dis | llvm-as | llvm-dis | FileCheck %s
+; RUN: verify-uselistorder %s
+
+@foo = global i32 0
+
+; CHECK: !named = !{!0, !1, !2, !3, !4, !5, !6, !7, !8, !9}
+!named = !{!0, !1, !2, !3, !4, !5, !6, !7, !8, !9}
+
+!0 = !DIFile(filename: "scope.h", directory: "/path/to/dir")
+!1 = distinct !{}
+!2 = !DIFile(filename: "path/to/file", directory: "/path/to/dir")
+!3 = !DIBasicType(name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
+!4 = distinct !{}
+
+; CHECK: !5 = !DIGlobalVariable(name: "foo", linkageName: "foo", scope: !0, file: !2, line: 7, type: !3, isLocal: true, isDefinition: false, variable: i32* @foo)
+!5 = !DIGlobalVariable(name: "foo", linkageName: "foo", scope: !0,
+                       file: !2, line: 7, type: !3, isLocal: true,
+                       isDefinition: false, variable: i32* @foo)
+
+; CHECK: !6 = !DIGlobalVariable(name: "foo", scope: !0, isLocal: false, isDefinition: true)
+!6 = !DIGlobalVariable(name: "foo", scope: !0)
+
+!7 = !DICompositeType(tag: DW_TAG_structure_type, name: "Class", size: 8, align: 8)
+!8 = !DIDerivedType(tag: DW_TAG_member, name: "mem", flags: DIFlagStaticMember, scope: !7, baseType: !3)
+
+; CHECK: !9 = !DIGlobalVariable(name: "mem", scope: !0, isLocal: false, isDefinition: true, declaration: !8)
+!9 = !DIGlobalVariable(name: "mem", scope: !0, declaration: !8)
diff --git a/test/Assembler/diimportedentity.ll b/test/Assembler/diimportedentity.ll
new file mode 100644
index 00000000000..929267e3b25
--- /dev/null
+++ b/test/Assembler/diimportedentity.ll
@@ -0,0 +1,20 @@
+; RUN: llvm-as < %s | llvm-dis | llvm-as | llvm-dis | FileCheck %s
+; RUN: verify-uselistorder %s
+
+; CHECK: !named = !{!0, !1, !2, !3, !3}
+!named = !{!0, !1, !2, !3, !4}
+
+; CHECK:      !0 = !DISubprogram({{.*}})
+; CHECK-NEXT: !1 = !DICompositeType({{.*}})
+!0 = !DISubprogram(name: "foo")
+!1 = !DICompositeType(tag: DW_TAG_structure_type, name: "Class", size: 32, align: 32)
+
+; CHECK-NEXT: !2 = !DIImportedEntity(tag: DW_TAG_imported_module, name: "foo", scope: !0, entity: !1, line: 7)
+!2 = !DIImportedEntity(tag: DW_TAG_imported_module, name: "foo", scope: !0,
+                       entity: !1, line: 7)
+
+; CHECK-NEXT: !3 = !DIImportedEntity(tag: DW_TAG_imported_module, scope: !0)
+!3 = !DIImportedEntity(tag: DW_TAG_imported_module, scope: !0)
+!4 = !DIImportedEntity(tag: DW_TAG_imported_module, name: "", scope: !0, entity: null,
+                       line: 0)
+
diff --git a/test/Assembler/dilexicalblock.ll b/test/Assembler/dilexicalblock.ll
new file mode 100644
index 00000000000..2cab372384c
--- /dev/null
+++ b/test/Assembler/dilexicalblock.ll
@@ -0,0 +1,25 @@
+; RUN: llvm-as < %s | llvm-dis | llvm-as | llvm-dis | FileCheck %s
+; RUN: verify-uselistorder %s
+
+; CHECK: !named = !{!0, !1, !2, !3, !4, !4, !5, !6, !7, !7}
+!named = !{!0, !1, !2, !3, !4, !5, !6, !7, !8, !9}
+
+!0 = distinct !{}
+!1 = !DISubprogram(name: "foo", scope: !2)
+!2 = !DIFile(filename: "path/to/file", directory: "/path/to/dir")
+
+; CHECK: !3 = !DILexicalBlock(scope: !1, file: !2, line: 7, column: 35)
+!3 = !DILexicalBlock(scope: !1, file: !2, line: 7, column: 35)
+
+; CHECK: !4 = !DILexicalBlock(scope: !1)
+!4 = !DILexicalBlock(scope: !1)
+!5 = !DILexicalBlock(scope: !1, file: null, line: 0, column: 0)
+
+; CHECK: !5 = !DILexicalBlockFile(scope: !3, file: !2, discriminator: 0)
+; CHECK: !6 = !DILexicalBlockFile(scope: !3, file: !2, discriminator: 1)
+!6 = !DILexicalBlockFile(scope: !3, file: !2, discriminator: 0)
+!7 = !DILexicalBlockFile(scope: !3, file: !2, discriminator: 1)
+
+; CHECK: !7 = !DILexicalBlockFile(scope: !3, discriminator: 7)
+!8 = !DILexicalBlockFile(scope: !3, discriminator: 7)
+!9 = !DILexicalBlockFile(scope: !3, file: null, discriminator: 7)
diff --git a/test/Assembler/dilocalvariable-arg-large.ll b/test/Assembler/dilocalvariable-arg-large.ll
new file mode 100644
index 00000000000..7788186a54a
--- /dev/null
+++ b/test/Assembler/dilocalvariable-arg-large.ll
@@ -0,0 +1,10 @@
+; RUN: llvm-as < %s | llvm-dis | llvm-as | llvm-dis | FileCheck %s
+; RUN: verify-uselistorder %s
+
+; CHECK: !named = !{!0, !1}
+!named = !{!0, !1}
+
+!0 = distinct !DISubprogram()
+
+; CHECK: !1 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "foo", arg: 65535, scope: !0)
+!1 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "foo", arg: 65535, scope: !0)
diff --git a/test/Assembler/dilocalvariable.ll b/test/Assembler/dilocalvariable.ll
new file mode 100644
index 00000000000..affe6105e1c
--- /dev/null
+++ b/test/Assembler/dilocalvariable.ll
@@ -0,0 +1,26 @@
+; RUN: llvm-as < %s | llvm-dis | llvm-as | llvm-dis | FileCheck %s
+; RUN: verify-uselistorder %s
+
+@foo = global i32 0
+
+; CHECK: !named = !{!0, !1, !2, !3, !4, !5, !6, !7, !8}
+!named = !{!0, !1, !2, !3, !4, !5, !6, !7, !8}
+
+!0 = distinct !DISubprogram()
+!1 = distinct !{}
+!2 = !DIFile(filename: "path/to/file", directory: "/path/to/dir")
+!3 = !DIBasicType(name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
+!4 = !DILocation(scope: !0)
+
+; CHECK: !5 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "foo", arg: 3, scope: !0, file: !2, line: 7, type: !3, flags: DIFlagArtificial)
+; CHECK: !6 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "foo", scope: !0, file: !2, line: 7, type: !3, flags: DIFlagArtificial)
+!5 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "foo", arg: 3,
+                      scope: !0, file: !2, line: 7, type: !3,
+                      flags: DIFlagArtificial)
+!6 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "foo", scope: !0,
+                      file: !2, line: 7, type: !3, flags: DIFlagArtificial)
+
+; CHECK: !7 = !DILocalVariable(tag: DW_TAG_arg_variable, arg: 1, scope: !0)
+; CHECK: !8 = !DILocalVariable(tag: DW_TAG_auto_variable, scope: !0)
+!7 = !DILocalVariable(tag: DW_TAG_arg_variable, scope: !0, arg: 1)
+!8 = !DILocalVariable(tag: DW_TAG_auto_variable, scope: !0)
diff --git a/test/Assembler/dilocation.ll b/test/Assembler/dilocation.ll
new file mode 100644
index 00000000000..a468f8abe9c
--- /dev/null
+++ b/test/Assembler/dilocation.ll
@@ -0,0 +1,23 @@
+; RUN: llvm-as < %s | llvm-dis | FileCheck %s
+; RUN: verify-uselistorder %s
+
+; CHECK: !named = !{!0, !1, !1, !2, !2, !3, !3, !4}
+!named = !{!0, !1, !2, !3, !4, !5, !6, !7}
+
+; CHECK: !0 = !DISubprogram(
+!0 = !DISubprogram()
+
+; CHECK-NEXT: !1 = !DILocation(line: 3, column: 7, scope: !0)
+!1 = !DILocation(line: 3, column: 7, scope: !0)
+!2 = !DILocation(scope: !0, column: 7, line: 3)
+
+; CHECK-NEXT: !2 = !DILocation(line: 3, column: 7, scope: !0, inlinedAt: !1)
+!3 = !DILocation(scope: !0, inlinedAt: !1, column: 7, line: 3)
+!4 = !DILocation(column: 7, line: 3, scope: !0, inlinedAt: !1)
+
+; CHECK-NEXT: !3 = !DILocation(line: 0, scope: !0)
+!5 = !DILocation(scope: !0)
+!6 = !DILocation(scope: !0, column: 0, line: 0)
+
+; CHECK-NEXT: !4 = !DILocation(line: 4294967295, column: 65535, scope: !0)
+!7 = !DILocation(line: 4294967295, column: 65535, scope: !0)
diff --git a/test/Assembler/dimodule.ll b/test/Assembler/dimodule.ll
new file mode 100644
index 00000000000..994bc12b681
--- /dev/null
+++ b/test/Assembler/dimodule.ll
@@ -0,0 +1,15 @@
+; RUN: llvm-as < %s | llvm-dis | llvm-as | llvm-dis | FileCheck %s
+; RUN: verify-uselistorder %s
+
+; CHECK: !named = !{!0, !1, !2, !1}
+!named = !{!0, !1, !2, !3}
+
+!0 = distinct !{}
+
+; CHECK: !1 = !DIModule(scope: !0, name: "Module")
+!1 = !DIModule(scope: !0, name: "Module")
+
+; CHECK: !2 = !DIModule(scope: !0, name: "Module", configMacros: "-DNDEBUG", includePath: "/usr/include", isysroot: "/")
+!2 = !DIModule(scope: !0, name: "Module", configMacros: "-DNDEBUG", includePath: "/usr/include", isysroot: "/")
+
+!3 = !DIModule(scope: !0, name: "Module", configMacros: "")
diff --git a/test/Assembler/dinamespace.ll b/test/Assembler/dinamespace.ll
new file mode 100644
index 00000000000..5d8b6b3fa7d
--- /dev/null
+++ b/test/Assembler/dinamespace.ll
@@ -0,0 +1,16 @@
+; RUN: llvm-as < %s | llvm-dis | llvm-as | llvm-dis | FileCheck %s
+; RUN: verify-uselistorder %s
+
+; CHECK: !named = !{!0, !1, !2, !3, !4, !4}
+!named = !{!0, !1, !2, !3, !4, !5}
+
+!0 = !DIFile(filename: "file.cpp", directory: "/path/to/dir")
+!1 = distinct !{}
+!2 = !DIFile(filename: "path/to/file", directory: "/path/to/dir")
+
+; CHECK: !3 = !DINamespace(name: "Namespace", scope: !0, file: !2, line: 7)
+!3 = !DINamespace(name: "Namespace", scope: !0, file: !2, line: 7)
+
+; CHECK: !4 = !DINamespace(scope: !0)
+!4 = !DINamespace(name: "", scope: !0, file: null, line: 0)
+!5 = !DINamespace(scope: !0)
diff --git a/test/Assembler/diobjcproperty.ll b/test/Assembler/diobjcproperty.ll
new file mode 100644
index 00000000000..ca13e27662d
--- /dev/null
+++ b/test/Assembler/diobjcproperty.ll
@@ -0,0 +1,19 @@
+; RUN: llvm-as < %s | llvm-dis | llvm-as | llvm-dis | FileCheck %s
+; RUN: verify-uselistorder %s
+
+; CHECK: !named = !{!0, !1, !2, !3, !4, !4}
+!named = !{!0, !1, !2, !3, !4, !5}
+
+!0 = distinct !{}
+!1 = !DIFile(filename: "path/to/file", directory: "/path/to/dir")
+!2 = !DICompositeType(tag: DW_TAG_structure_type, name: "Object")
+
+; CHECK: !2 = !DICompositeType({{.*}})
+; CHECK-NEXT: !3 = !DIObjCProperty(name: "foo", file: !1, line: 7, setter: "setFoo", getter: "getFoo", attributes: 7, type: !2)
+!3 = !DIObjCProperty(name: "foo", file: !1, line: 7, setter: "setFoo",
+                     getter: "getFoo", attributes: 7, type: !2)
+
+; CHECK-NEXT: !4 = !DIObjCProperty()
+!4 = !DIObjCProperty(name: "", file: null, line: 0, setter: "", getter: "",
+                     attributes: 0, type: null)
+!5 = !DIObjCProperty()
diff --git a/test/Assembler/disubprogram.ll b/test/Assembler/disubprogram.ll
new file mode 100644
index 00000000000..3fa1081889f
--- /dev/null
+++ b/test/Assembler/disubprogram.ll
@@ -0,0 +1,28 @@
+; RUN: llvm-as < %s | llvm-dis | llvm-as | llvm-dis | FileCheck %s
+; RUN: verify-uselistorder %s
+
+declare void @_Z3foov()
+
+; CHECK: !named = !{!0, !1, !2, !3, !4, !5, !6, !7, !8, !9}
+!named = !{!0, !1, !2, !3, !4, !5, !6, !7, !8, !9}
+
+!0 = !{null}
+!1 = distinct !DICompositeType(tag: DW_TAG_structure_type)
+!2 = !DIFile(filename: "path/to/file", directory: "/path/to/dir")
+!3 = !DISubroutineType(types: !0)
+!4 = distinct !DICompositeType(tag: DW_TAG_structure_type)
+!5 = distinct !{}
+!6 = distinct !DISubprogram(isDefinition: false)
+!7 = distinct !{}
+
+; CHECK: !8 = !DISubprogram(name: "foo", linkageName: "_Zfoov", scope: !1, file: !2, line: 7, type: !3, isLocal: true, isDefinition: false, scopeLine: 8, containingType: !4, virtuality: DW_VIRTUALITY_pure_virtual, virtualIndex: 10, flags: DIFlagPrototyped, isOptimized: true, function: void ()* @_Z3foov, templateParams: !5, declaration: !6, variables: !7)
+!8 = !DISubprogram(name: "foo", linkageName: "_Zfoov", scope: !1,
+                   file: !2, line: 7, type: !3, isLocal: true,
+                   isDefinition: false, scopeLine: 8, containingType: !4,
+                   virtuality: DW_VIRTUALITY_pure_virtual, virtualIndex: 10,
+                   flags: DIFlagPrototyped, isOptimized: true, function: void ()* @_Z3foov,
+                   templateParams: !5, declaration: !6, variables: !7)
+
+; CHECK: !9 = !DISubprogram(scope: null, isLocal: false, isDefinition: true, isOptimized: false)
+!9 = !DISubprogram()
+
diff --git a/test/Assembler/disubrange-empty-array.ll b/test/Assembler/disubrange-empty-array.ll
new file mode 100644
index 00000000000..7b5279e3d3c
--- /dev/null
+++ b/test/Assembler/disubrange-empty-array.ll
@@ -0,0 +1,14 @@
+; RUN: llvm-as < %s | llvm-dis | llvm-as | llvm-dis | FileCheck %s
+; RUN: verify-uselistorder %s
+
+; CHECK: !named = !{!0, !0, !1, !2}
+!named = !{!0, !1, !2, !3}
+
+; CHECK:      !0 = !DISubrange(count: -1)
+; CHECK-NEXT: !1 = !DISubrange(count: -1, lowerBound: 4)
+; CHECK-NEXT: !2 = !DISubrange(count: -1, lowerBound: -5)
+!0 = !DISubrange(count: -1)
+!1 = !DISubrange(count: -1, lowerBound: 0)
+
+!2 = !DISubrange(count: -1, lowerBound: 4)
+!3 = !DISubrange(count: -1, lowerBound: -5)
diff --git a/test/Assembler/disubroutinetype.ll b/test/Assembler/disubroutinetype.ll
new file mode 100644
index 00000000000..4ec2be7de3b
--- /dev/null
+++ b/test/Assembler/disubroutinetype.ll
@@ -0,0 +1,23 @@
+; RUN: llvm-as < %s | llvm-dis | llvm-as | llvm-dis | FileCheck %s
+; RUN: verify-uselistorder %s
+
+; CHECK: !named = !{!0, !1, !2, !3, !4, !5, !6, !7, !8}
+!named = !{!0, !1, !2, !3, !4, !5, !6, !7, !8}
+
+!0 = !DIBasicType(name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
+!1 = !{null}
+!2 = !{null, !0}
+!3 = !{!0, !0, !0}
+
+
+; CHECK: !4 = !DISubroutineType(types: !1)
+; CHECK: !5 = !DISubroutineType(types: !2)
+; CHECK: !6 = !DISubroutineType(types: !3)
+; CHECK: !7 = !DISubroutineType(flags: DIFlagLValueReference, types: !3)
+!4 = !DISubroutineType(types: !1)
+!5 = !DISubroutineType(types: !2)
+!6 = !DISubroutineType(types: !3)
+!7 = !DISubroutineType(flags: DIFlagLValueReference, types: !3)
+
+; CHECK: !8 = !DISubroutineType(types: null)
+!8 = !DISubroutineType(types: null)
diff --git a/test/Assembler/ditemplateparameter.ll b/test/Assembler/ditemplateparameter.ll
new file mode 100644
index 00000000000..a356ad4e7bc
--- /dev/null
+++ b/test/Assembler/ditemplateparameter.ll
@@ -0,0 +1,24 @@
+; RUN: llvm-as < %s | llvm-dis | llvm-as | llvm-dis | FileCheck %s
+; RUN: verify-uselistorder %s
+
+; CHECK: !named = !{!0, !1, !2, !3, !3, !4, !5, !5, !6}
+!named = !{!0, !1, !2, !3, !4, !5, !6, !7, !8}
+
+!0 = distinct !{}
+!1 = !DIBasicType(name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
+; CHECK: !1 = !DIBasicType(name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
+
+; CHECK-NEXT: !2 = !DITemplateTypeParameter(name: "Ty", type: !1)
+; CHECK-NEXT: !3 = !DITemplateTypeParameter(type: !1)
+!2 = !DITemplateTypeParameter(name: "Ty", type: !1)
+!3 = !DITemplateTypeParameter(type: !1)
+!4 = !DITemplateTypeParameter(name: "", type: !1)
+
+; CHECK-NEXT: !4 = !DITemplateValueParameter(name: "V", type: !1, value: i32 7)
+; CHECK-NEXT: !5 = !DITemplateValueParameter(type: !1, value: i32 7)
+; CHECK-NEXT: !6 = !DITemplateValueParameter(tag: DW_TAG_GNU_template_template_param, name: "param", type: !1, value: !"template")
+!5 = !DITemplateValueParameter(name: "V", type: !1, value: i32 7)
+!6 = !DITemplateValueParameter(type: !1, value: i32 7)
+!7 = !DITemplateValueParameter(tag: DW_TAG_template_value_parameter,
+                               name: "", type: !1, value: i32 7)
+!8 = !DITemplateValueParameter(tag: DW_TAG_GNU_template_template_param, name: "param", type: !1, value: !"template")
diff --git a/test/Assembler/ditype-large-values.ll b/test/Assembler/ditype-large-values.ll
new file mode 100644
index 00000000000..a371ac68f90
--- /dev/null
+++ b/test/Assembler/ditype-large-values.ll
@@ -0,0 +1,12 @@
+; RUN: llvm-as < %s | llvm-dis | llvm-as | llvm-dis | FileCheck %s
+; RUN: verify-uselistorder %s
+
+; CHECK: !named = !{!0, !1, !2}
+!named = !{!0, !1, !2}
+
+; CHECK:      !0 = !DIBasicType(name: "name", size: 18446744073709551615, align: 18446744073709551614, encoding: DW_ATE_unsigned_char)
+; CHECK-NEXT: !1 = !DIDerivedType(tag: DW_TAG_pointer_type, baseType: !0, size: 18446744073709551615, align: 18446744073709551614, offset: 18446744073709551613)
+; CHECK-NEXT: !2 = !DICompositeType(tag: DW_TAG_array_type, baseType: !0, size: 18446744073709551615, align: 18446744073709551614, offset: 18446744073709551613)
+!0 = !DIBasicType(tag: DW_TAG_base_type, name: "name", size: 18446744073709551615, align: 18446744073709551614, encoding: DW_ATE_unsigned_char)
+!1 = !DIDerivedType(tag: DW_TAG_pointer_type, baseType: !0, size: 18446744073709551615, align: 18446744073709551614, offset: 18446744073709551613)
+!2 = !DICompositeType(tag: DW_TAG_array_type, baseType: !0, size: 18446744073709551615, align: 18446744073709551614, offset: 18446744073709551613)
diff --git a/test/Assembler/drop-debug-info.ll b/test/Assembler/drop-debug-info.ll
index 698975d8998..44a7b78b004 100644
--- a/test/Assembler/drop-debug-info.ll
+++ b/test/Assembler/drop-debug-info.ll
@@ -12,17 +12,17 @@ entry:
 !llvm.dbg.cu = !{!0}
 !llvm.module.flags = !{!9}
 
-!0 = !MDCompileUnit(language: DW_LANG_C99, producer: "clang version 3.5 (trunk 195495) (llvm/trunk 195495:195504M)", isOptimized: false, emissionKind: 0, file: !1, enums: !2, retainedTypes: !2, subprograms: !3, globals: !2, imports: !2)
-!1 = !MDFile(filename: "../llvm/tools/clang/test/CodeGen/debug-info-version.c", directory: "/Users/manmanren/llvm_gmail/release")
+!0 = !DICompileUnit(language: DW_LANG_C99, producer: "clang version 3.5 (trunk 195495) (llvm/trunk 195495:195504M)", isOptimized: false, emissionKind: 0, file: !1, enums: !2, retainedTypes: !2, subprograms: !3, globals: !2, imports: !2)
+!1 = !DIFile(filename: "../llvm/tools/clang/test/CodeGen/debug-info-version.c", directory: "/Users/manmanren/llvm_gmail/release")
 !2 = !{i32 0}
 !3 = !{!4}
-!4 = !MDSubprogram(name: "main", line: 3, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 3, file: !1, scope: !5, type: !6, function: i32 ()* @main, variables: !2)
-!5 = !MDFile(filename: "../llvm/tools/clang/test/CodeGen/debug-info-version.c", directory: "/Users/manmanren/llvm_gmail/release")
-!6 = !MDSubroutineType(types: !7)
+!4 = !DISubprogram(name: "main", line: 3, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 3, file: !1, scope: !5, type: !6, function: i32 ()* @main, variables: !2)
+!5 = !DIFile(filename: "../llvm/tools/clang/test/CodeGen/debug-info-version.c", directory: "/Users/manmanren/llvm_gmail/release")
+!6 = !DISubroutineType(types: !7)
 !7 = !{!8}
-!8 = !MDBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
+!8 = !DIBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
 !9 = !{i32 2, !"Dwarf Version", i32 2}
-!12 = !MDLocation(line: 4, scope: !4)
+!12 = !DILocation(line: 4, scope: !4)
 
 ; WARN: warning: ignoring debug info with an invalid version (0)
 ; CHECK-NOT: !dbg
diff --git a/test/Assembler/generic-debug-node.ll b/test/Assembler/generic-debug-node.ll
index cb7222b043d..5fb72faa18e 100644
--- a/test/Assembler/generic-debug-node.ll
+++ b/test/Assembler/generic-debug-node.ll
@@ -7,21 +7,21 @@
 ; CHECK: !0 = !{}
 !0 = !{}
 
-; CHECK-NEXT: !1 = !GenericDebugNode(tag: DW_TAG_entry_point, header: "some\00header", operands: {!0, !2, !2})
-!1 = !GenericDebugNode(tag: 3, header: "some\00header", operands: {!0, !3, !4})
-!2 = !GenericDebugNode(tag: 3, header: "some\00header", operands: {!{}, !3, !4})
+; CHECK-NEXT: !1 = !GenericDINode(tag: DW_TAG_entry_point, header: "some\00header", operands: {!0, !2, !2})
+!1 = !GenericDINode(tag: 3, header: "some\00header", operands: {!0, !3, !4})
+!2 = !GenericDINode(tag: 3, header: "some\00header", operands: {!{}, !3, !4})
 
-; CHECK-NEXT: !2 = !GenericDebugNode(tag: DW_TAG_entry_point)
-!3 = !GenericDebugNode(tag: 3)
-!4 = !GenericDebugNode(tag: 3, header: "")
-!5 = !GenericDebugNode(tag: 3, operands: {})
-!6 = !GenericDebugNode(tag: 3, header: "", operands: {})
+; CHECK-NEXT: !2 = !GenericDINode(tag: DW_TAG_entry_point)
+!3 = !GenericDINode(tag: 3)
+!4 = !GenericDINode(tag: 3, header: "")
+!5 = !GenericDINode(tag: 3, operands: {})
+!6 = !GenericDINode(tag: 3, header: "", operands: {})
 
-; CHECK-NEXT: !3 = distinct !GenericDebugNode(tag: DW_TAG_entry_point)
-!7 = distinct !GenericDebugNode(tag: 3)
+; CHECK-NEXT: !3 = distinct !GenericDINode(tag: DW_TAG_entry_point)
+!7 = distinct !GenericDINode(tag: 3)
 
-; CHECK-NEXT: !4 = !GenericDebugNode(tag: 65535)
-!8 = !GenericDebugNode(tag: 65535)
+; CHECK-NEXT: !4 = !GenericDINode(tag: 65535)
+!8 = !GenericDINode(tag: 65535)
 
 ; CHECK-NOT: !
-!9 = !GenericDebugNode(tag: DW_TAG_entry_point)
+!9 = !GenericDINode(tag: DW_TAG_entry_point)
diff --git a/test/Assembler/getelementptr.ll b/test/Assembler/getelementptr.ll
index ee88346eb00..0a0fddfc4dd 100644
--- a/test/Assembler/getelementptr.ll
+++ b/test/Assembler/getelementptr.ll
@@ -19,6 +19,10 @@
 @y = global i32* getelementptr ({ i32, i32 }, { i32, i32 }* @x, i16 42, i32 0)
 ; CHECK: @y = global i32* getelementptr ({ i32, i32 }, { i32, i32 }* @x, i16 42, i32 0)
 
+@PR23753_a = external global i8
+@PR23753_b = global i8* getelementptr (i8, i8* @PR23753_a, i64 ptrtoint (i8* @PR23753_a to i64))
+; CHECK: @PR23753_b = global i8* getelementptr (i8, i8* @PR23753_a, i64 ptrtoint (i8* @PR23753_a to i64))
+
 ; See if i92 indices work too.
 define i32 *@test({i32, i32}* %t, i92 %n) {
 ; CHECK: @test
diff --git a/test/Assembler/getelementptr_vec_idx1.ll b/test/Assembler/getelementptr_vec_idx1.ll
index 084a31e7234..12160574d4e 100644
--- a/test/Assembler/getelementptr_vec_idx1.ll
+++ b/test/Assembler/getelementptr_vec_idx1.ll
@@ -1,8 +1,8 @@
 ; RUN: not llvm-as < %s >/dev/null 2> %t
 ; RUN: FileCheck %s < %t
-; Test that a vector index is only used with a vector pointer.
+; Test that a vector GEP may be used with a scalar base, the result is a vector of pointers
 
-; CHECK: getelementptr index type missmatch
+; CHECK: '%w' defined with type '<2 x i32*>
 
 define i32 @test(i32* %a) {
   %w = getelementptr i32, i32* %a, <2 x i32> <i32 5, i32 9>
diff --git a/test/Assembler/getelementptr_vec_idx2.ll b/test/Assembler/getelementptr_vec_idx2.ll
index 638fcb8b67d..be294098c9e 100644
--- a/test/Assembler/getelementptr_vec_idx2.ll
+++ b/test/Assembler/getelementptr_vec_idx2.ll
@@ -1,10 +1,24 @@
 ; RUN: not llvm-as < %s >/dev/null 2> %t
 ; RUN: FileCheck %s < %t
-; Test that a vector pointer is only used with a vector index.
+; Test that a vector pointer may be used with a scalar index.
+; Test that a vector pointer and vector index should have the same vector width
 
-; CHECK: getelementptr index type missmatch
-
-define <2 x i32> @test(<2 x i32*> %a) {
+; This code is correct
+define <2 x i32*> @test2(<2 x i32*> %a) {
   %w = getelementptr i32, <2 x i32*> %a, i32 2
+  ret <2 x i32*> %w
+}
+
+; This code is correct
+define <2 x i32*> @test3(i32* %a) {
+  %w = getelementptr i32, i32* %a, <2 x i32> <i32 2, i32 2>
+  ret <2 x i32*> %w
+}
+
+; CHECK: getelementptr vector index has a wrong number of elements
+
+define <2 x i32> @test1(<2 x i32*> %a) {
+  %w = getelementptr i32, <2 x i32*> %a, <4 x i32><i32 2, i32 2, i32 2, i32 2>
   ret <2 x i32> %w
 }
+
diff --git a/test/Assembler/getelementptr_vec_idx3.ll b/test/Assembler/getelementptr_vec_idx3.ll
index ac94459e23d..767c817cc62 100644
--- a/test/Assembler/getelementptr_vec_idx3.ll
+++ b/test/Assembler/getelementptr_vec_idx3.ll
@@ -1,8 +1,8 @@
 ; RUN: not llvm-as < %s >/dev/null 2> %t
 ; RUN: FileCheck %s < %t
-; Test that vector indices have the same number of elements as the pointer.
+; Test that a vector GEP may be used with a scalar base, the result is a vector of pointers
 
-; CHECK: getelementptr index type missmatch
+; CHECK: '%w' defined with type '<2 x <4 x i32>*>'
 
 define <4 x i32> @test(<4 x i32>* %a) {
   %w = getelementptr <4 x i32>, <4 x i32>* %a, <2 x i32> <i32 5, i32 9>
diff --git a/test/Assembler/invalid-dicompileunit-language-bad.ll b/test/Assembler/invalid-dicompileunit-language-bad.ll
new file mode 100644
index 00000000000..e6f49f3fba4
--- /dev/null
+++ b/test/Assembler/invalid-dicompileunit-language-bad.ll
@@ -0,0 +1,5 @@
+; RUN: not llvm-as < %s -disable-output 2>&1 | FileCheck %s
+
+; CHECK: <stdin>:[[@LINE+1]]:31: error: invalid DWARF language 'DW_LANG_NoSuchLanguage'
+!0 = !DICompileUnit(language: DW_LANG_NoSuchLanguage,
+                    file: !DIFile(filename: "a", directory: "b"))
diff --git a/test/Assembler/invalid-dicompileunit-language-overflow.ll b/test/Assembler/invalid-dicompileunit-language-overflow.ll
new file mode 100644
index 00000000000..c558f7aaa25
--- /dev/null
+++ b/test/Assembler/invalid-dicompileunit-language-overflow.ll
@@ -0,0 +1,9 @@
+; RUN: not llvm-as < %s -disable-output 2>&1 | FileCheck %s
+
+; CHECK-NOT: error:
+!0 = !DICompileUnit(language: 65535,
+                    file: !DIFile(filename: "a", directory: "b"))
+
+; CHECK: <stdin>:[[@LINE+1]]:31: error: value for 'language' too large, limit is 65535
+!1 = !DICompileUnit(language: 65536,
+                    file: !DIFile(filename: "a", directory: "b"))
diff --git a/test/Assembler/invalid-dicompileunit-missing-language.ll b/test/Assembler/invalid-dicompileunit-missing-language.ll
new file mode 100644
index 00000000000..15631b7f640
--- /dev/null
+++ b/test/Assembler/invalid-dicompileunit-missing-language.ll
@@ -0,0 +1,4 @@
+; RUN: not llvm-as < %s -disable-output 2>&1 | FileCheck %s
+
+; CHECK: <stdin>:[[@LINE+1]]:65: error: missing required field 'language'
+!0 = !DICompileUnit(file: !DIFile(filename: "a", directory: "b"))
diff --git a/test/Assembler/invalid-dicompileunit-null-file.ll b/test/Assembler/invalid-dicompileunit-null-file.ll
new file mode 100644
index 00000000000..cc1892e9146
--- /dev/null
+++ b/test/Assembler/invalid-dicompileunit-null-file.ll
@@ -0,0 +1,4 @@
+; RUN: not llvm-as < %s -disable-output 2>&1 | FileCheck %s
+
+; CHECK: <stdin>:[[@LINE+1]]:27: error: 'file' cannot be null
+!0 = !DICompileUnit(file: null)
diff --git a/test/Assembler/invalid-dicompositetype-missing-tag.ll b/test/Assembler/invalid-dicompositetype-missing-tag.ll
new file mode 100644
index 00000000000..e68c0147990
--- /dev/null
+++ b/test/Assembler/invalid-dicompositetype-missing-tag.ll
@@ -0,0 +1,4 @@
+; RUN: not llvm-as < %s -disable-output 2>&1 | FileCheck %s
+
+; CHECK: [[@LINE+1]]:36: error: missing required field 'tag'
+!25 = !DICompositeType(name: "Type")
diff --git a/test/Assembler/invalid-diderivedtype-missing-basetype.ll b/test/Assembler/invalid-diderivedtype-missing-basetype.ll
new file mode 100644
index 00000000000..308c2ea90ef
--- /dev/null
+++ b/test/Assembler/invalid-diderivedtype-missing-basetype.ll
@@ -0,0 +1,4 @@
+; RUN: not llvm-as < %s -disable-output 2>&1 | FileCheck %s
+
+; CHECK: [[@LINE+1]]:45: error: missing required field 'baseType'
+!0 = !DIDerivedType(tag: DW_TAG_pointer_type)
diff --git a/test/Assembler/invalid-diderivedtype-missing-tag.ll b/test/Assembler/invalid-diderivedtype-missing-tag.ll
new file mode 100644
index 00000000000..fd286f4c296
--- /dev/null
+++ b/test/Assembler/invalid-diderivedtype-missing-tag.ll
@@ -0,0 +1,4 @@
+; RUN: not llvm-as < %s -disable-output 2>&1 | FileCheck %s
+
+; CHECK: [[@LINE+1]]:34: error: missing required field 'tag'
+!0 = !DIDerivedType(baseType: !{})
diff --git a/test/Assembler/invalid-dienumerator-missing-name.ll b/test/Assembler/invalid-dienumerator-missing-name.ll
new file mode 100644
index 00000000000..656d3439ba2
--- /dev/null
+++ b/test/Assembler/invalid-dienumerator-missing-name.ll
@@ -0,0 +1,4 @@
+; RUN: not llvm-as < %s -disable-output 2>&1 | FileCheck %s
+
+; CHECK: [[@LINE+1]]:28: error: missing required field 'name'
+!0 = !DIEnumerator(value: 7)
diff --git a/test/Assembler/invalid-dienumerator-missing-value.ll b/test/Assembler/invalid-dienumerator-missing-value.ll
new file mode 100644
index 00000000000..0eee3be41b6
--- /dev/null
+++ b/test/Assembler/invalid-dienumerator-missing-value.ll
@@ -0,0 +1,4 @@
+; RUN: not llvm-as < %s -disable-output 2>&1 | FileCheck %s
+
+; CHECK: [[@LINE+1]]:32: error: missing required field 'value'
+!0 = !DIEnumerator(name: "name")
diff --git a/test/Assembler/invalid-diexpression-large.ll b/test/Assembler/invalid-diexpression-large.ll
new file mode 100644
index 00000000000..32b77ee5138
--- /dev/null
+++ b/test/Assembler/invalid-diexpression-large.ll
@@ -0,0 +1,7 @@
+; RUN: not llvm-as < %s -disable-output 2>&1 | FileCheck %s
+
+; CHECK-NOT: error:
+!0 = !DIExpression(18446744073709551615)
+
+; CHECK: <stdin>:[[@LINE+1]]:20: error: element too large, limit is 18446744073709551615
+!1 = !DIExpression(18446744073709551616)
diff --git a/test/Assembler/invalid-diexpression-verify.ll b/test/Assembler/invalid-diexpression-verify.ll
new file mode 100644
index 00000000000..50d6943dead
--- /dev/null
+++ b/test/Assembler/invalid-diexpression-verify.ll
@@ -0,0 +1,9 @@
+; RUN: not llvm-as -disable-output < %s 2>&1 | FileCheck -check-prefix VERIFY %s
+; RUN: llvm-as -disable-verify < %s | llvm-dis | FileCheck -check-prefix NOVERIFY %s
+
+; NOVERIFY: !named = !{!0}
+!named = !{!0}
+
+; NOVERIFY: !0 = !DIExpression(0, 1, 9, 7, 2)
+; VERIFY: assembly parsed, but does not verify
+!0 = !DIExpression(0, 1, 9, 7, 2)
diff --git a/test/Assembler/invalid-difile-missing-directory.ll b/test/Assembler/invalid-difile-missing-directory.ll
new file mode 100644
index 00000000000..a54a22f2e18
--- /dev/null
+++ b/test/Assembler/invalid-difile-missing-directory.ll
@@ -0,0 +1,4 @@
+; RUN: not llvm-as < %s -disable-output 2>&1 | FileCheck %s
+
+; CHECK: [[@LINE+1]]:30: error: missing required field 'directory'
+!0 = !DIFile(filename: "file")
diff --git a/test/Assembler/invalid-difile-missing-filename.ll b/test/Assembler/invalid-difile-missing-filename.ll
new file mode 100644
index 00000000000..17bf3da3f7a
--- /dev/null
+++ b/test/Assembler/invalid-difile-missing-filename.ll
@@ -0,0 +1,4 @@
+; RUN: not llvm-as < %s -disable-output 2>&1 | FileCheck %s
+
+; CHECK: [[@LINE+1]]:30: error: missing required field 'filename'
+!0 = !DIFile(directory: "dir")
diff --git a/test/Assembler/invalid-diglobalvariable-empty-name.ll b/test/Assembler/invalid-diglobalvariable-empty-name.ll
new file mode 100644
index 00000000000..a4e69f3c8b7
--- /dev/null
+++ b/test/Assembler/invalid-diglobalvariable-empty-name.ll
@@ -0,0 +1,4 @@
+; RUN: not llvm-as < %s -disable-output 2>&1 | FileCheck %s
+
+; CHECK: <stdin>:[[@LINE+1]]:30: error: 'name' cannot be empty
+!0 = !DIGlobalVariable(name: "")
diff --git a/test/Assembler/invalid-diglobalvariable-missing-name.ll b/test/Assembler/invalid-diglobalvariable-missing-name.ll
new file mode 100644
index 00000000000..baf4d73d94f
--- /dev/null
+++ b/test/Assembler/invalid-diglobalvariable-missing-name.ll
@@ -0,0 +1,4 @@
+; RUN: not llvm-as < %s -disable-output 2>&1 | FileCheck %s
+
+; CHECK: <stdin>:[[@LINE+1]]:24: error: missing required field 'name'
+!0 = !DIGlobalVariable()
diff --git a/test/Assembler/invalid-diimportedentity-missing-scope.ll b/test/Assembler/invalid-diimportedentity-missing-scope.ll
new file mode 100644
index 00000000000..06164e88d9b
--- /dev/null
+++ b/test/Assembler/invalid-diimportedentity-missing-scope.ll
@@ -0,0 +1,4 @@
+; RUN: not llvm-as < %s -disable-output 2>&1 | FileCheck %s
+
+; CHECK: [[@LINE+1]]:51: error: missing required field 'scope'
+!3 = !DIImportedEntity(tag: DW_TAG_imported_module)
diff --git a/test/Assembler/invalid-diimportedentity-missing-tag.ll b/test/Assembler/invalid-diimportedentity-missing-tag.ll
new file mode 100644
index 00000000000..996b1ffdc9b
--- /dev/null
+++ b/test/Assembler/invalid-diimportedentity-missing-tag.ll
@@ -0,0 +1,4 @@
+; RUN: not llvm-as < %s -disable-output 2>&1 | FileCheck %s
+
+; CHECK: [[@LINE+1]]:33: error: missing required field 'tag'
+!3 = !DIImportedEntity(scope: !0)
diff --git a/test/Assembler/invalid-dilexicalblock-missing-scope.ll b/test/Assembler/invalid-dilexicalblock-missing-scope.ll
new file mode 100644
index 00000000000..6108e692b21
--- /dev/null
+++ b/test/Assembler/invalid-dilexicalblock-missing-scope.ll
@@ -0,0 +1,4 @@
+; RUN: not llvm-as < %s -disable-output 2>&1 | FileCheck %s
+
+; CHECK: [[@LINE+1]]:29: error: missing required field 'scope'
+!0 = !DILexicalBlock(line: 7)
diff --git a/test/Assembler/invalid-dilexicalblock-null-scope.ll b/test/Assembler/invalid-dilexicalblock-null-scope.ll
new file mode 100644
index 00000000000..7457b6085d6
--- /dev/null
+++ b/test/Assembler/invalid-dilexicalblock-null-scope.ll
@@ -0,0 +1,4 @@
+; RUN: not llvm-as < %s -disable-output 2>&1 | FileCheck %s
+
+; CHECK: <stdin>:[[@LINE+1]]:29: error: 'scope' cannot be null
+!0 = !DILexicalBlock(scope: null)
diff --git a/test/Assembler/invalid-dilexicalblockfile-missing-discriminator.ll b/test/Assembler/invalid-dilexicalblockfile-missing-discriminator.ll
new file mode 100644
index 00000000000..bd8395ccf4d
--- /dev/null
+++ b/test/Assembler/invalid-dilexicalblockfile-missing-discriminator.ll
@@ -0,0 +1,4 @@
+; RUN: not llvm-as < %s -disable-output 2>&1 | FileCheck %s
+
+; CHECK: [[@LINE+1]]:36: error: missing required field 'discriminator'
+!0 = !DILexicalBlockFile(scope: !{})
diff --git a/test/Assembler/invalid-dilexicalblockfile-missing-scope.ll b/test/Assembler/invalid-dilexicalblockfile-missing-scope.ll
new file mode 100644
index 00000000000..fd037501c40
--- /dev/null
+++ b/test/Assembler/invalid-dilexicalblockfile-missing-scope.ll
@@ -0,0 +1,4 @@
+; RUN: not llvm-as < %s -disable-output 2>&1 | FileCheck %s
+
+; CHECK: [[@LINE+1]]:42: error: missing required field 'scope'
+!0 = !DILexicalBlockFile(discriminator: 0)
diff --git a/test/Assembler/invalid-dilexicalblockfile-null-scope.ll b/test/Assembler/invalid-dilexicalblockfile-null-scope.ll
new file mode 100644
index 00000000000..ae1a5d432e3
--- /dev/null
+++ b/test/Assembler/invalid-dilexicalblockfile-null-scope.ll
@@ -0,0 +1,4 @@
+; RUN: not llvm-as < %s -disable-output 2>&1 | FileCheck %s
+
+; CHECK: <stdin>:[[@LINE+1]]:33: error: 'scope' cannot be null
+!0 = !DILexicalBlockFile(scope: null)
diff --git a/test/Assembler/invalid-dilocalvariable-arg-large.ll b/test/Assembler/invalid-dilocalvariable-arg-large.ll
new file mode 100644
index 00000000000..d62da601e13
--- /dev/null
+++ b/test/Assembler/invalid-dilocalvariable-arg-large.ll
@@ -0,0 +1,6 @@
+; RUN: not llvm-as < %s 2>&1 | FileCheck %s
+
+!0 = !DILocalVariable(tag: DW_TAG_arg_variable, scope: !{}, arg: 65535)
+
+; CHECK: <stdin>:[[@LINE+1]]:66: error: value for 'arg' too large, limit is 65535
+!1 = !DILocalVariable(tag: DW_TAG_arg_variable, scope: !{}, arg: 65536)
diff --git a/test/Assembler/invalid-dilocalvariable-arg-negative.ll b/test/Assembler/invalid-dilocalvariable-arg-negative.ll
new file mode 100644
index 00000000000..ff90d66fa11
--- /dev/null
+++ b/test/Assembler/invalid-dilocalvariable-arg-negative.ll
@@ -0,0 +1,7 @@
+; RUN: not llvm-as < %s 2>&1 | FileCheck %s
+
+!0 = !DILocalVariable(tag: DW_TAG_arg_variable, scope: !{}, arg: 1)
+!1 = !DILocalVariable(tag: DW_TAG_auto_variable, scope: !{}, arg: 0)
+
+; CHECK: <stdin>:[[@LINE+1]]:66: error: expected unsigned integer
+!2 = !DILocalVariable(tag: DW_TAG_arg_variable, scope: !{}, arg: -1)
diff --git a/test/Assembler/invalid-dilocalvariable-missing-scope.ll b/test/Assembler/invalid-dilocalvariable-missing-scope.ll
new file mode 100644
index 00000000000..45dcad71360
--- /dev/null
+++ b/test/Assembler/invalid-dilocalvariable-missing-scope.ll
@@ -0,0 +1,4 @@
+; RUN: not llvm-as < %s -disable-output 2>&1 | FileCheck %s
+
+; CHECK: <stdin>:[[@LINE+1]]:48: error: missing required field 'scope'
+!0 = !DILocalVariable(tag: DW_TAG_auto_variable)
diff --git a/test/Assembler/invalid-dilocalvariable-missing-tag.ll b/test/Assembler/invalid-dilocalvariable-missing-tag.ll
new file mode 100644
index 00000000000..18062edf75b
--- /dev/null
+++ b/test/Assembler/invalid-dilocalvariable-missing-tag.ll
@@ -0,0 +1,4 @@
+; RUN: not llvm-as < %s -disable-output 2>&1 | FileCheck %s
+
+; CHECK: <stdin>:[[@LINE+1]]:45: error: missing required field 'tag'
+!0 = !DILocalVariable(scope: !DISubprogram())
diff --git a/test/Assembler/invalid-dilocalvariable-null-scope.ll b/test/Assembler/invalid-dilocalvariable-null-scope.ll
new file mode 100644
index 00000000000..859412a2568
--- /dev/null
+++ b/test/Assembler/invalid-dilocalvariable-null-scope.ll
@@ -0,0 +1,4 @@
+; RUN: not llvm-as < %s -disable-output 2>&1 | FileCheck %s
+
+; CHECK: <stdin>:[[@LINE+1]]:30: error: 'scope' cannot be null
+!0 = !DILocalVariable(scope: null)
diff --git a/test/Assembler/invalid-dilocation-field-bad.ll b/test/Assembler/invalid-dilocation-field-bad.ll
new file mode 100644
index 00000000000..e68aa45b03e
--- /dev/null
+++ b/test/Assembler/invalid-dilocation-field-bad.ll
@@ -0,0 +1,4 @@
+; RUN: not llvm-as < %s -disable-output 2>&1 | FileCheck %s
+
+; CHECK: <stdin>:[[@LINE+1]]:18: error: invalid field 'bad'
+!0 = !DILocation(bad: 0)
diff --git a/test/Assembler/invalid-dilocation-field-twice.ll b/test/Assembler/invalid-dilocation-field-twice.ll
new file mode 100644
index 00000000000..d1dab68246c
--- /dev/null
+++ b/test/Assembler/invalid-dilocation-field-twice.ll
@@ -0,0 +1,6 @@
+; RUN: not llvm-as < %s -disable-output 2>&1 | FileCheck %s
+
+!0 = !{}
+
+; CHECK: <stdin>:[[@LINE+1]]:38: error: field 'line' cannot be specified more than once
+!1 = !DILocation(line: 3, scope: !0, line: 3)
diff --git a/test/Assembler/invalid-dilocation-missing-scope-2.ll b/test/Assembler/invalid-dilocation-missing-scope-2.ll
new file mode 100644
index 00000000000..380e3fde94f
--- /dev/null
+++ b/test/Assembler/invalid-dilocation-missing-scope-2.ll
@@ -0,0 +1,4 @@
+; RUN: not llvm-as < %s -disable-output 2>&1 | FileCheck %s
+
+; CHECK: <stdin>:[[@LINE+1]]:25: error: missing required field 'scope'
+!0 = !DILocation(line: 7)
diff --git a/test/Assembler/invalid-dilocation-missing-scope.ll b/test/Assembler/invalid-dilocation-missing-scope.ll
new file mode 100644
index 00000000000..fb0cd866dc1
--- /dev/null
+++ b/test/Assembler/invalid-dilocation-missing-scope.ll
@@ -0,0 +1,4 @@
+; RUN: not llvm-as < %s -disable-output 2>&1 | FileCheck %s
+
+; CHECK: <stdin>:[[@LINE+1]]:18: error: missing required field 'scope'
+!0 = !DILocation()
diff --git a/test/Assembler/invalid-dilocation-null-scope.ll b/test/Assembler/invalid-dilocation-null-scope.ll
new file mode 100644
index 00000000000..38c59e15d15
--- /dev/null
+++ b/test/Assembler/invalid-dilocation-null-scope.ll
@@ -0,0 +1,4 @@
+; RUN: not llvm-as < %s -disable-output 2>&1 | FileCheck %s
+
+; CHECK: <stdin>:[[@LINE+1]]:25: error: 'scope' cannot be null
+!0 = !DILocation(scope: null)
diff --git a/test/Assembler/invalid-dilocation-overflow-column.ll b/test/Assembler/invalid-dilocation-overflow-column.ll
new file mode 100644
index 00000000000..043f84d3ab2
--- /dev/null
+++ b/test/Assembler/invalid-dilocation-overflow-column.ll
@@ -0,0 +1,9 @@
+; RUN: not llvm-as < %s -disable-output 2>&1 | FileCheck %s
+
+!0 = !{}
+
+; CHECK-NOT: error:
+!1 = !DILocation(column: 65535, scope: !0)
+
+; CHECK: <stdin>:[[@LINE+1]]:26: error: value for 'column' too large, limit is 65535
+!2 = !DILocation(column: 65536, scope: !0)
diff --git a/test/Assembler/invalid-dilocation-overflow-line.ll b/test/Assembler/invalid-dilocation-overflow-line.ll
new file mode 100644
index 00000000000..7a6774231a9
--- /dev/null
+++ b/test/Assembler/invalid-dilocation-overflow-line.ll
@@ -0,0 +1,9 @@
+; RUN: not llvm-as < %s -disable-output 2>&1 | FileCheck %s
+
+!0 = !{}
+
+; CHECK-NOT: error:
+!1 = !DILocation(line: 4294967295, scope: !0)
+
+; CHECK: <stdin>:[[@LINE+1]]:24: error: value for 'line' too large, limit is 4294967295
+!2 = !DILocation(line: 4294967296, scope: !0)
diff --git a/test/Assembler/invalid-dinamespace-missing-namespace.ll b/test/Assembler/invalid-dinamespace-missing-namespace.ll
new file mode 100644
index 00000000000..c29391b125d
--- /dev/null
+++ b/test/Assembler/invalid-dinamespace-missing-namespace.ll
@@ -0,0 +1,4 @@
+; RUN: not llvm-as < %s -disable-output 2>&1 | FileCheck %s
+
+; CHECK: [[@LINE+1]]:36: error: missing required field 'scope'
+!0 = !DINamespace(name: "Namespace")
diff --git a/test/Assembler/invalid-disubrange-count-large.ll b/test/Assembler/invalid-disubrange-count-large.ll
new file mode 100644
index 00000000000..003274f3a07
--- /dev/null
+++ b/test/Assembler/invalid-disubrange-count-large.ll
@@ -0,0 +1,7 @@
+; RUN: not llvm-as < %s -disable-output 2>&1 | FileCheck %s
+
+; CHECK-NOT: error:
+!0 = !DISubrange(count: 9223372036854775807)
+
+; CHECK: <stdin>:[[@LINE+1]]:25: error: value for 'count' too large, limit is 9223372036854775807
+!1 = !DISubrange(count: 9223372036854775808)
diff --git a/test/Assembler/invalid-disubrange-count-missing.ll b/test/Assembler/invalid-disubrange-count-missing.ll
new file mode 100644
index 00000000000..8fc4487117f
--- /dev/null
+++ b/test/Assembler/invalid-disubrange-count-missing.ll
@@ -0,0 +1,4 @@
+; RUN: not llvm-as < %s -disable-output 2>&1 | FileCheck %s
+
+; CHECK: [[@LINE+1]]:32: error: missing required field 'count'
+!0 = !DISubrange(lowerBound: -3)
diff --git a/test/Assembler/invalid-disubrange-count-negative.ll b/test/Assembler/invalid-disubrange-count-negative.ll
new file mode 100644
index 00000000000..f2ad4c302cb
--- /dev/null
+++ b/test/Assembler/invalid-disubrange-count-negative.ll
@@ -0,0 +1,7 @@
+; RUN: not llvm-as < %s -disable-output 2>&1 | FileCheck %s
+
+; CHECK-NOT: error:
+!0 = !DISubrange(count: -1)
+
+; CHECK: <stdin>:[[@LINE+1]]:25: error: value for 'count' too small, limit is -1
+!0 = !DISubrange(count: -2)
diff --git a/test/Assembler/invalid-disubrange-lowerBound-max.ll b/test/Assembler/invalid-disubrange-lowerBound-max.ll
new file mode 100644
index 00000000000..e163dc47804
--- /dev/null
+++ b/test/Assembler/invalid-disubrange-lowerBound-max.ll
@@ -0,0 +1,4 @@
+; RUN: not llvm-as < %s -disable-output 2>&1 | FileCheck %s
+
+; CHECK: [[@LINE+1]]:41: error: value for 'lowerBound' too large, limit is 9223372036854775807
+!0 = !DISubrange(count: 30, lowerBound: 9223372036854775808)
diff --git a/test/Assembler/invalid-disubrange-lowerBound-min.ll b/test/Assembler/invalid-disubrange-lowerBound-min.ll
new file mode 100644
index 00000000000..1dc3a141a57
--- /dev/null
+++ b/test/Assembler/invalid-disubrange-lowerBound-min.ll
@@ -0,0 +1,4 @@
+; RUN: not llvm-as < %s -disable-output 2>&1 | FileCheck %s
+
+; CHECK: [[@LINE+1]]:41: error: value for 'lowerBound' too small, limit is -9223372036854775808
+!0 = !DISubrange(count: 30, lowerBound: -9223372036854775809)
diff --git a/test/Assembler/invalid-disubroutinetype-missing-types.ll b/test/Assembler/invalid-disubroutinetype-missing-types.ll
new file mode 100644
index 00000000000..086e5cc593e
--- /dev/null
+++ b/test/Assembler/invalid-disubroutinetype-missing-types.ll
@@ -0,0 +1,4 @@
+; RUN: not llvm-as < %s -disable-output 2>&1 | FileCheck %s
+
+; CHECK: [[@LINE+1]]:65: error: missing required field 'types'
+!29 = !DISubroutineType(flags: DIFlagPublic | DIFlagStaticMember)
diff --git a/test/Assembler/invalid-ditemplatetypeparameter-missing-type.ll b/test/Assembler/invalid-ditemplatetypeparameter-missing-type.ll
new file mode 100644
index 00000000000..797abd2c304
--- /dev/null
+++ b/test/Assembler/invalid-ditemplatetypeparameter-missing-type.ll
@@ -0,0 +1,4 @@
+; RUN: not llvm-as < %s -disable-output 2>&1 | FileCheck %s
+
+; CHECK: [[@LINE+1]]:44: error: missing required field 'type'
+!0 = !DITemplateTypeParameter(name: "param")
diff --git a/test/Assembler/invalid-ditemplatevalueparameter-missing-value.ll b/test/Assembler/invalid-ditemplatevalueparameter-missing-value.ll
new file mode 100644
index 00000000000..883bea1c9ad
--- /dev/null
+++ b/test/Assembler/invalid-ditemplatevalueparameter-missing-value.ll
@@ -0,0 +1,5 @@
+; RUN: not llvm-as < %s -disable-output 2>&1 | FileCheck %s
+
+; CHECK: [[@LINE+2]]:41: error: missing required field 'value'
+!0 = !DITemplateValueParameter(tag: DW_TAG_template_value_parameter,
+                               type: !{})
diff --git a/test/Assembler/invalid-generic-debug-node-tag-bad.ll b/test/Assembler/invalid-generic-debug-node-tag-bad.ll
index 98c10b6a878..8da140f33ce 100644
--- a/test/Assembler/invalid-generic-debug-node-tag-bad.ll
+++ b/test/Assembler/invalid-generic-debug-node-tag-bad.ll
@@ -1,4 +1,4 @@
 ; RUN: not llvm-as < %s -disable-output 2>&1 | FileCheck %s
 
-; CHECK: <stdin>:[[@LINE+1]]:29: error: invalid DWARF tag 'DW_TAG_badtag'
-!0 = !GenericDebugNode(tag: DW_TAG_badtag)
+; CHECK: <stdin>:[[@LINE+1]]:26: error: invalid DWARF tag 'DW_TAG_badtag'
+!0 = !GenericDINode(tag: DW_TAG_badtag)
diff --git a/test/Assembler/invalid-generic-debug-node-tag-missing.ll b/test/Assembler/invalid-generic-debug-node-tag-missing.ll
index 5372cbb0880..0ae2c7194a7 100644
--- a/test/Assembler/invalid-generic-debug-node-tag-missing.ll
+++ b/test/Assembler/invalid-generic-debug-node-tag-missing.ll
@@ -1,4 +1,4 @@
 ; RUN: not llvm-as < %s -disable-output 2>&1 | FileCheck %s
 
-; CHECK: <stdin>:[[@LINE+1]]:47: error: missing required field 'tag'
-!0 = !GenericDebugNode(header: "some\00header")
+; CHECK: <stdin>:[[@LINE+1]]:44: error: missing required field 'tag'
+!0 = !GenericDINode(header: "some\00header")
diff --git a/test/Assembler/invalid-generic-debug-node-tag-overflow.ll b/test/Assembler/invalid-generic-debug-node-tag-overflow.ll
index b5189469f07..1349dc0a2e2 100644
--- a/test/Assembler/invalid-generic-debug-node-tag-overflow.ll
+++ b/test/Assembler/invalid-generic-debug-node-tag-overflow.ll
@@ -1,7 +1,7 @@
 ; RUN: not llvm-as < %s -disable-output 2>&1 | FileCheck %s
 
 ; CHECK-NOT: error:
-!0 = !GenericDebugNode(tag: 65535)
+!0 = !GenericDINode(tag: 65535)
 
-; CHECK: <stdin>:[[@LINE+1]]:29: error: value for 'tag' too large, limit is 65535
-!1 = !GenericDebugNode(tag: 65536)
+; CHECK: <stdin>:[[@LINE+1]]:26: error: value for 'tag' too large, limit is 65535
+!1 = !GenericDINode(tag: 65536)
diff --git a/test/Assembler/invalid-generic-debug-node-tag-wrong-type.ll b/test/Assembler/invalid-generic-debug-node-tag-wrong-type.ll
index fca24a70bf0..7d0c04af914 100644
--- a/test/Assembler/invalid-generic-debug-node-tag-wrong-type.ll
+++ b/test/Assembler/invalid-generic-debug-node-tag-wrong-type.ll
@@ -1,4 +1,4 @@
 ; RUN: not llvm-as < %s -disable-output 2>&1 | FileCheck %s
 
-; CHECK: <stdin>:[[@LINE+1]]:29: error: expected DWARF tag
-!0 = !GenericDebugNode(tag: "string")
+; CHECK: <stdin>:[[@LINE+1]]:26: error: expected DWARF tag
+!0 = !GenericDINode(tag: "string")
diff --git a/test/Assembler/invalid-landingpad.ll b/test/Assembler/invalid-landingpad.ll
index c8eab4ac6ae..44bf8db1768 100644
--- a/test/Assembler/invalid-landingpad.ll
+++ b/test/Assembler/invalid-landingpad.ll
@@ -2,6 +2,6 @@
 
 ; CHECK: clause argument must be a constant
 
-define void @test(i32 %in) {
-  landingpad {} personality void()* null filter i32 %in
+define void @test(i32 %in) personality void()* null {
+  landingpad {} filter i32 %in
 }
diff --git a/test/Assembler/invalid-mdcompileunit-language-bad.ll b/test/Assembler/invalid-mdcompileunit-language-bad.ll
deleted file mode 100644
index cf2da20061c..00000000000
--- a/test/Assembler/invalid-mdcompileunit-language-bad.ll
+++ /dev/null
@@ -1,5 +0,0 @@
-; RUN: not llvm-as < %s -disable-output 2>&1 | FileCheck %s
-
-; CHECK: <stdin>:[[@LINE+1]]:31: error: invalid DWARF language 'DW_LANG_NoSuchLanguage'
-!0 = !MDCompileUnit(language: DW_LANG_NoSuchLanguage,
-                    file: !MDFile(filename: "a", directory: "b"))
diff --git a/test/Assembler/invalid-mdcompileunit-language-overflow.ll b/test/Assembler/invalid-mdcompileunit-language-overflow.ll
deleted file mode 100644
index 1849867edae..00000000000
--- a/test/Assembler/invalid-mdcompileunit-language-overflow.ll
+++ /dev/null
@@ -1,9 +0,0 @@
-; RUN: not llvm-as < %s -disable-output 2>&1 | FileCheck %s
-
-; CHECK-NOT: error:
-!0 = !MDCompileUnit(language: 65535,
-                    file: !MDFile(filename: "a", directory: "b"))
-
-; CHECK: <stdin>:[[@LINE+1]]:31: error: value for 'language' too large, limit is 65535
-!1 = !MDCompileUnit(language: 65536,
-                    file: !MDFile(filename: "a", directory: "b"))
diff --git a/test/Assembler/invalid-mdcompileunit-missing-language.ll b/test/Assembler/invalid-mdcompileunit-missing-language.ll
deleted file mode 100644
index 57a9a3efb66..00000000000
--- a/test/Assembler/invalid-mdcompileunit-missing-language.ll
+++ /dev/null
@@ -1,4 +0,0 @@
-; RUN: not llvm-as < %s -disable-output 2>&1 | FileCheck %s
-
-; CHECK: <stdin>:[[@LINE+1]]:65: error: missing required field 'language'
-!0 = !MDCompileUnit(file: !MDFile(filename: "a", directory: "b"))
diff --git a/test/Assembler/invalid-mdcompileunit-null-file.ll b/test/Assembler/invalid-mdcompileunit-null-file.ll
deleted file mode 100644
index 613948f24ef..00000000000
--- a/test/Assembler/invalid-mdcompileunit-null-file.ll
+++ /dev/null
@@ -1,4 +0,0 @@
-; RUN: not llvm-as < %s -disable-output 2>&1 | FileCheck %s
-
-; CHECK: <stdin>:[[@LINE+1]]:27: error: 'file' cannot be null
-!0 = !MDCompileUnit(file: null)
diff --git a/test/Assembler/invalid-mdcompositetype-missing-tag.ll b/test/Assembler/invalid-mdcompositetype-missing-tag.ll
deleted file mode 100644
index a3b1418d9bc..00000000000
--- a/test/Assembler/invalid-mdcompositetype-missing-tag.ll
+++ /dev/null
@@ -1,4 +0,0 @@
-; RUN: not llvm-as < %s -disable-output 2>&1 | FileCheck %s
-
-; CHECK: [[@LINE+1]]:36: error: missing required field 'tag'
-!25 = !MDCompositeType(name: "Type")
diff --git a/test/Assembler/invalid-mdderivedtype-missing-basetype.ll b/test/Assembler/invalid-mdderivedtype-missing-basetype.ll
deleted file mode 100644
index 24fa5850d9f..00000000000
--- a/test/Assembler/invalid-mdderivedtype-missing-basetype.ll
+++ /dev/null
@@ -1,4 +0,0 @@
-; RUN: not llvm-as < %s -disable-output 2>&1 | FileCheck %s
-
-; CHECK: [[@LINE+1]]:45: error: missing required field 'baseType'
-!0 = !MDDerivedType(tag: DW_TAG_pointer_type)
diff --git a/test/Assembler/invalid-mdderivedtype-missing-tag.ll b/test/Assembler/invalid-mdderivedtype-missing-tag.ll
deleted file mode 100644
index 36206281aeb..00000000000
--- a/test/Assembler/invalid-mdderivedtype-missing-tag.ll
+++ /dev/null
@@ -1,4 +0,0 @@
-; RUN: not llvm-as < %s -disable-output 2>&1 | FileCheck %s
-
-; CHECK: [[@LINE+1]]:34: error: missing required field 'tag'
-!0 = !MDDerivedType(baseType: !{})
diff --git a/test/Assembler/invalid-mdenumerator-missing-name.ll b/test/Assembler/invalid-mdenumerator-missing-name.ll
deleted file mode 100644
index 709c6a556e6..00000000000
--- a/test/Assembler/invalid-mdenumerator-missing-name.ll
+++ /dev/null
@@ -1,4 +0,0 @@
-; RUN: not llvm-as < %s -disable-output 2>&1 | FileCheck %s
-
-; CHECK: [[@LINE+1]]:28: error: missing required field 'name'
-!0 = !MDEnumerator(value: 7)
diff --git a/test/Assembler/invalid-mdenumerator-missing-value.ll b/test/Assembler/invalid-mdenumerator-missing-value.ll
deleted file mode 100644
index a8501680688..00000000000
--- a/test/Assembler/invalid-mdenumerator-missing-value.ll
+++ /dev/null
@@ -1,4 +0,0 @@
-; RUN: not llvm-as < %s -disable-output 2>&1 | FileCheck %s
-
-; CHECK: [[@LINE+1]]:32: error: missing required field 'value'
-!0 = !MDEnumerator(name: "name")
diff --git a/test/Assembler/invalid-mdexpression-large.ll b/test/Assembler/invalid-mdexpression-large.ll
deleted file mode 100644
index fb304137ed7..00000000000
--- a/test/Assembler/invalid-mdexpression-large.ll
+++ /dev/null
@@ -1,7 +0,0 @@
-; RUN: not llvm-as < %s -disable-output 2>&1 | FileCheck %s
-
-; CHECK-NOT: error:
-!0 = !MDExpression(18446744073709551615)
-
-; CHECK: <stdin>:[[@LINE+1]]:20: error: element too large, limit is 18446744073709551615
-!1 = !MDExpression(18446744073709551616)
diff --git a/test/Assembler/invalid-mdexpression-verify.ll b/test/Assembler/invalid-mdexpression-verify.ll
deleted file mode 100644
index e573ef32155..00000000000
--- a/test/Assembler/invalid-mdexpression-verify.ll
+++ /dev/null
@@ -1,9 +0,0 @@
-; RUN: not llvm-as -disable-output < %s 2>&1 | FileCheck -check-prefix VERIFY %s
-; RUN: llvm-as -disable-verify < %s | llvm-dis | FileCheck -check-prefix NOVERIFY %s
-
-; NOVERIFY: !named = !{!0}
-!named = !{!0}
-
-; NOVERIFY: !0 = !MDExpression(0, 1, 9, 7, 2)
-; VERIFY: assembly parsed, but does not verify
-!0 = !MDExpression(0, 1, 9, 7, 2)
diff --git a/test/Assembler/invalid-mdfile-missing-directory.ll b/test/Assembler/invalid-mdfile-missing-directory.ll
deleted file mode 100644
index 825db089656..00000000000
--- a/test/Assembler/invalid-mdfile-missing-directory.ll
+++ /dev/null
@@ -1,4 +0,0 @@
-; RUN: not llvm-as < %s -disable-output 2>&1 | FileCheck %s
-
-; CHECK: [[@LINE+1]]:30: error: missing required field 'directory'
-!0 = !MDFile(filename: "file")
diff --git a/test/Assembler/invalid-mdfile-missing-filename.ll b/test/Assembler/invalid-mdfile-missing-filename.ll
deleted file mode 100644
index 0dd71179105..00000000000
--- a/test/Assembler/invalid-mdfile-missing-filename.ll
+++ /dev/null
@@ -1,4 +0,0 @@
-; RUN: not llvm-as < %s -disable-output 2>&1 | FileCheck %s
-
-; CHECK: [[@LINE+1]]:30: error: missing required field 'filename'
-!0 = !MDFile(directory: "dir")
diff --git a/test/Assembler/invalid-mdglobalvariable-empty-name.ll b/test/Assembler/invalid-mdglobalvariable-empty-name.ll
deleted file mode 100644
index 77a9f4d773e..00000000000
--- a/test/Assembler/invalid-mdglobalvariable-empty-name.ll
+++ /dev/null
@@ -1,4 +0,0 @@
-; RUN: not llvm-as < %s -disable-output 2>&1 | FileCheck %s
-
-; CHECK: <stdin>:[[@LINE+1]]:30: error: 'name' cannot be empty
-!0 = !MDGlobalVariable(name: "")
diff --git a/test/Assembler/invalid-mdglobalvariable-missing-name.ll b/test/Assembler/invalid-mdglobalvariable-missing-name.ll
deleted file mode 100644
index d57d71e2d95..00000000000
--- a/test/Assembler/invalid-mdglobalvariable-missing-name.ll
+++ /dev/null
@@ -1,4 +0,0 @@
-; RUN: not llvm-as < %s -disable-output 2>&1 | FileCheck %s
-
-; CHECK: <stdin>:[[@LINE+1]]:24: error: missing required field 'name'
-!0 = !MDGlobalVariable()
diff --git a/test/Assembler/invalid-mdimportedentity-missing-scope.ll b/test/Assembler/invalid-mdimportedentity-missing-scope.ll
deleted file mode 100644
index 710a0276480..00000000000
--- a/test/Assembler/invalid-mdimportedentity-missing-scope.ll
+++ /dev/null
@@ -1,4 +0,0 @@
-; RUN: not llvm-as < %s -disable-output 2>&1 | FileCheck %s
-
-; CHECK: [[@LINE+1]]:51: error: missing required field 'scope'
-!3 = !MDImportedEntity(tag: DW_TAG_imported_module)
diff --git a/test/Assembler/invalid-mdimportedentity-missing-tag.ll b/test/Assembler/invalid-mdimportedentity-missing-tag.ll
deleted file mode 100644
index 63cf0d25650..00000000000
--- a/test/Assembler/invalid-mdimportedentity-missing-tag.ll
+++ /dev/null
@@ -1,4 +0,0 @@
-; RUN: not llvm-as < %s -disable-output 2>&1 | FileCheck %s
-
-; CHECK: [[@LINE+1]]:33: error: missing required field 'tag'
-!3 = !MDImportedEntity(scope: !0)
diff --git a/test/Assembler/invalid-mdlexicalblock-missing-scope.ll b/test/Assembler/invalid-mdlexicalblock-missing-scope.ll
deleted file mode 100644
index cdd12afde58..00000000000
--- a/test/Assembler/invalid-mdlexicalblock-missing-scope.ll
+++ /dev/null
@@ -1,4 +0,0 @@
-; RUN: not llvm-as < %s -disable-output 2>&1 | FileCheck %s
-
-; CHECK: [[@LINE+1]]:29: error: missing required field 'scope'
-!0 = !MDLexicalBlock(line: 7)
diff --git a/test/Assembler/invalid-mdlexicalblock-null-scope.ll b/test/Assembler/invalid-mdlexicalblock-null-scope.ll
deleted file mode 100644
index 59a99684797..00000000000
--- a/test/Assembler/invalid-mdlexicalblock-null-scope.ll
+++ /dev/null
@@ -1,4 +0,0 @@
-; RUN: not llvm-as < %s -disable-output 2>&1 | FileCheck %s
-
-; CHECK: <stdin>:[[@LINE+1]]:29: error: 'scope' cannot be null
-!0 = !MDLexicalBlock(scope: null)
diff --git a/test/Assembler/invalid-mdlexicalblockfile-missing-discriminator.ll b/test/Assembler/invalid-mdlexicalblockfile-missing-discriminator.ll
deleted file mode 100644
index b71eed82cc4..00000000000
--- a/test/Assembler/invalid-mdlexicalblockfile-missing-discriminator.ll
+++ /dev/null
@@ -1,4 +0,0 @@
-; RUN: not llvm-as < %s -disable-output 2>&1 | FileCheck %s
-
-; CHECK: [[@LINE+1]]:36: error: missing required field 'discriminator'
-!0 = !MDLexicalBlockFile(scope: !{})
diff --git a/test/Assembler/invalid-mdlexicalblockfile-missing-scope.ll b/test/Assembler/invalid-mdlexicalblockfile-missing-scope.ll
deleted file mode 100644
index 1c901e25cb5..00000000000
--- a/test/Assembler/invalid-mdlexicalblockfile-missing-scope.ll
+++ /dev/null
@@ -1,4 +0,0 @@
-; RUN: not llvm-as < %s -disable-output 2>&1 | FileCheck %s
-
-; CHECK: [[@LINE+1]]:42: error: missing required field 'scope'
-!0 = !MDLexicalBlockFile(discriminator: 0)
diff --git a/test/Assembler/invalid-mdlexicalblockfile-null-scope.ll b/test/Assembler/invalid-mdlexicalblockfile-null-scope.ll
deleted file mode 100644
index 708136c6328..00000000000
--- a/test/Assembler/invalid-mdlexicalblockfile-null-scope.ll
+++ /dev/null
@@ -1,4 +0,0 @@
-; RUN: not llvm-as < %s -disable-output 2>&1 | FileCheck %s
-
-; CHECK: <stdin>:[[@LINE+1]]:33: error: 'scope' cannot be null
-!0 = !MDLexicalBlockFile(scope: null)
diff --git a/test/Assembler/invalid-mdlocalvariable-missing-scope.ll b/test/Assembler/invalid-mdlocalvariable-missing-scope.ll
deleted file mode 100644
index d8ee8a97307..00000000000
--- a/test/Assembler/invalid-mdlocalvariable-missing-scope.ll
+++ /dev/null
@@ -1,4 +0,0 @@
-; RUN: not llvm-as < %s -disable-output 2>&1 | FileCheck %s
-
-; CHECK: <stdin>:[[@LINE+1]]:48: error: missing required field 'scope'
-!0 = !MDLocalVariable(tag: DW_TAG_auto_variable)
diff --git a/test/Assembler/invalid-mdlocalvariable-missing-tag.ll b/test/Assembler/invalid-mdlocalvariable-missing-tag.ll
deleted file mode 100644
index d3ca10a2013..00000000000
--- a/test/Assembler/invalid-mdlocalvariable-missing-tag.ll
+++ /dev/null
@@ -1,4 +0,0 @@
-; RUN: not llvm-as < %s -disable-output 2>&1 | FileCheck %s
-
-; CHECK: <stdin>:[[@LINE+1]]:45: error: missing required field 'tag'
-!0 = !MDLocalVariable(scope: !MDSubprogram())
diff --git a/test/Assembler/invalid-mdlocalvariable-null-scope.ll b/test/Assembler/invalid-mdlocalvariable-null-scope.ll
deleted file mode 100644
index 53ee94de79b..00000000000
--- a/test/Assembler/invalid-mdlocalvariable-null-scope.ll
+++ /dev/null
@@ -1,4 +0,0 @@
-; RUN: not llvm-as < %s -disable-output 2>&1 | FileCheck %s
-
-; CHECK: <stdin>:[[@LINE+1]]:30: error: 'scope' cannot be null
-!0 = !MDLocalVariable(scope: null)
diff --git a/test/Assembler/invalid-mdlocation-field-bad.ll b/test/Assembler/invalid-mdlocation-field-bad.ll
deleted file mode 100644
index 6ec7c64bb4d..00000000000
--- a/test/Assembler/invalid-mdlocation-field-bad.ll
+++ /dev/null
@@ -1,4 +0,0 @@
-; RUN: not llvm-as < %s -disable-output 2>&1 | FileCheck %s
-
-; CHECK: <stdin>:[[@LINE+1]]:18: error: invalid field 'bad'
-!0 = !MDLocation(bad: 0)
diff --git a/test/Assembler/invalid-mdlocation-field-twice.ll b/test/Assembler/invalid-mdlocation-field-twice.ll
deleted file mode 100644
index 2335c935a5f..00000000000
--- a/test/Assembler/invalid-mdlocation-field-twice.ll
+++ /dev/null
@@ -1,6 +0,0 @@
-; RUN: not llvm-as < %s -disable-output 2>&1 | FileCheck %s
-
-!0 = !{}
-
-; CHECK: <stdin>:[[@LINE+1]]:38: error: field 'line' cannot be specified more than once
-!1 = !MDLocation(line: 3, scope: !0, line: 3)
diff --git a/test/Assembler/invalid-mdlocation-missing-scope-2.ll b/test/Assembler/invalid-mdlocation-missing-scope-2.ll
deleted file mode 100644
index 3b267c9199b..00000000000
--- a/test/Assembler/invalid-mdlocation-missing-scope-2.ll
+++ /dev/null
@@ -1,4 +0,0 @@
-; RUN: not llvm-as < %s -disable-output 2>&1 | FileCheck %s
-
-; CHECK: <stdin>:[[@LINE+1]]:25: error: missing required field 'scope'
-!0 = !MDLocation(line: 7)
diff --git a/test/Assembler/invalid-mdlocation-missing-scope.ll b/test/Assembler/invalid-mdlocation-missing-scope.ll
deleted file mode 100644
index 87f41b31799..00000000000
--- a/test/Assembler/invalid-mdlocation-missing-scope.ll
+++ /dev/null
@@ -1,4 +0,0 @@
-; RUN: not llvm-as < %s -disable-output 2>&1 | FileCheck %s
-
-; CHECK: <stdin>:[[@LINE+1]]:18: error: missing required field 'scope'
-!0 = !MDLocation()
diff --git a/test/Assembler/invalid-mdlocation-null-scope.ll b/test/Assembler/invalid-mdlocation-null-scope.ll
deleted file mode 100644
index e359c9fe424..00000000000
--- a/test/Assembler/invalid-mdlocation-null-scope.ll
+++ /dev/null
@@ -1,4 +0,0 @@
-; RUN: not llvm-as < %s -disable-output 2>&1 | FileCheck %s
-
-; CHECK: <stdin>:[[@LINE+1]]:25: error: 'scope' cannot be null
-!0 = !MDLocation(scope: null)
diff --git a/test/Assembler/invalid-mdlocation-overflow-column.ll b/test/Assembler/invalid-mdlocation-overflow-column.ll
deleted file mode 100644
index 1c06697d01e..00000000000
--- a/test/Assembler/invalid-mdlocation-overflow-column.ll
+++ /dev/null
@@ -1,9 +0,0 @@
-; RUN: not llvm-as < %s -disable-output 2>&1 | FileCheck %s
-
-!0 = !{}
-
-; CHECK-NOT: error:
-!1 = !MDLocation(column: 65535, scope: !0)
-
-; CHECK: <stdin>:[[@LINE+1]]:26: error: value for 'column' too large, limit is 65535
-!2 = !MDLocation(column: 65536, scope: !0)
diff --git a/test/Assembler/invalid-mdlocation-overflow-line.ll b/test/Assembler/invalid-mdlocation-overflow-line.ll
deleted file mode 100644
index 571a555daef..00000000000
--- a/test/Assembler/invalid-mdlocation-overflow-line.ll
+++ /dev/null
@@ -1,9 +0,0 @@
-; RUN: not llvm-as < %s -disable-output 2>&1 | FileCheck %s
-
-!0 = !{}
-
-; CHECK-NOT: error:
-!1 = !MDLocation(line: 4294967295, scope: !0)
-
-; CHECK: <stdin>:[[@LINE+1]]:24: error: value for 'line' too large, limit is 4294967295
-!2 = !MDLocation(line: 4294967296, scope: !0)
diff --git a/test/Assembler/invalid-mdnamespace-missing-namespace.ll b/test/Assembler/invalid-mdnamespace-missing-namespace.ll
deleted file mode 100644
index da2f5114221..00000000000
--- a/test/Assembler/invalid-mdnamespace-missing-namespace.ll
+++ /dev/null
@@ -1,4 +0,0 @@
-; RUN: not llvm-as < %s -disable-output 2>&1 | FileCheck %s
-
-; CHECK: [[@LINE+1]]:36: error: missing required field 'scope'
-!0 = !MDNamespace(name: "Namespace")
diff --git a/test/Assembler/invalid-mdsubrange-count-large.ll b/test/Assembler/invalid-mdsubrange-count-large.ll
deleted file mode 100644
index 733c4afa34a..00000000000
--- a/test/Assembler/invalid-mdsubrange-count-large.ll
+++ /dev/null
@@ -1,7 +0,0 @@
-; RUN: not llvm-as < %s -disable-output 2>&1 | FileCheck %s
-
-; CHECK-NOT: error:
-!0 = !MDSubrange(count: 9223372036854775807)
-
-; CHECK: <stdin>:[[@LINE+1]]:25: error: value for 'count' too large, limit is 9223372036854775807
-!1 = !MDSubrange(count: 9223372036854775808)
diff --git a/test/Assembler/invalid-mdsubrange-count-missing.ll b/test/Assembler/invalid-mdsubrange-count-missing.ll
deleted file mode 100644
index bf9cb9a5e3d..00000000000
--- a/test/Assembler/invalid-mdsubrange-count-missing.ll
+++ /dev/null
@@ -1,4 +0,0 @@
-; RUN: not llvm-as < %s -disable-output 2>&1 | FileCheck %s
-
-; CHECK: [[@LINE+1]]:32: error: missing required field 'count'
-!0 = !MDSubrange(lowerBound: -3)
diff --git a/test/Assembler/invalid-mdsubrange-count-negative.ll b/test/Assembler/invalid-mdsubrange-count-negative.ll
deleted file mode 100644
index 9bfe784ab5d..00000000000
--- a/test/Assembler/invalid-mdsubrange-count-negative.ll
+++ /dev/null
@@ -1,7 +0,0 @@
-; RUN: not llvm-as < %s -disable-output 2>&1 | FileCheck %s
-
-; CHECK-NOT: error:
-!0 = !MDSubrange(count: -1)
-
-; CHECK: <stdin>:[[@LINE+1]]:25: error: value for 'count' too small, limit is -1
-!0 = !MDSubrange(count: -2)
diff --git a/test/Assembler/invalid-mdsubrange-lowerBound-max.ll b/test/Assembler/invalid-mdsubrange-lowerBound-max.ll
deleted file mode 100644
index 1c68e984b17..00000000000
--- a/test/Assembler/invalid-mdsubrange-lowerBound-max.ll
+++ /dev/null
@@ -1,4 +0,0 @@
-; RUN: not llvm-as < %s -disable-output 2>&1 | FileCheck %s
-
-; CHECK: [[@LINE+1]]:41: error: value for 'lowerBound' too large, limit is 9223372036854775807
-!0 = !MDSubrange(count: 30, lowerBound: 9223372036854775808)
diff --git a/test/Assembler/invalid-mdsubrange-lowerBound-min.ll b/test/Assembler/invalid-mdsubrange-lowerBound-min.ll
deleted file mode 100644
index b3b2a03efd8..00000000000
--- a/test/Assembler/invalid-mdsubrange-lowerBound-min.ll
+++ /dev/null
@@ -1,4 +0,0 @@
-; RUN: not llvm-as < %s -disable-output 2>&1 | FileCheck %s
-
-; CHECK: [[@LINE+1]]:41: error: value for 'lowerBound' too small, limit is -9223372036854775808
-!0 = !MDSubrange(count: 30, lowerBound: -9223372036854775809)
diff --git a/test/Assembler/invalid-mdsubroutinetype-missing-types.ll b/test/Assembler/invalid-mdsubroutinetype-missing-types.ll
deleted file mode 100644
index 7342417d1ca..00000000000
--- a/test/Assembler/invalid-mdsubroutinetype-missing-types.ll
+++ /dev/null
@@ -1,4 +0,0 @@
-; RUN: not llvm-as < %s -disable-output 2>&1 | FileCheck %s
-
-; CHECK: [[@LINE+1]]:65: error: missing required field 'types'
-!29 = !MDSubroutineType(flags: DIFlagPublic | DIFlagStaticMember)
diff --git a/test/Assembler/invalid-mdtemplatetypeparameter-missing-type.ll b/test/Assembler/invalid-mdtemplatetypeparameter-missing-type.ll
deleted file mode 100644
index 62bee70d947..00000000000
--- a/test/Assembler/invalid-mdtemplatetypeparameter-missing-type.ll
+++ /dev/null
@@ -1,4 +0,0 @@
-; RUN: not llvm-as < %s -disable-output 2>&1 | FileCheck %s
-
-; CHECK: [[@LINE+1]]:44: error: missing required field 'type'
-!0 = !MDTemplateTypeParameter(name: "param")
diff --git a/test/Assembler/invalid-mdtemplatevalueparameter-missing-value.ll b/test/Assembler/invalid-mdtemplatevalueparameter-missing-value.ll
deleted file mode 100644
index e1e3f81d269..00000000000
--- a/test/Assembler/invalid-mdtemplatevalueparameter-missing-value.ll
+++ /dev/null
@@ -1,5 +0,0 @@
-; RUN: not llvm-as < %s -disable-output 2>&1 | FileCheck %s
-
-; CHECK: [[@LINE+2]]:41: error: missing required field 'value'
-!0 = !MDTemplateValueParameter(tag: DW_TAG_template_value_parameter,
-                               type: !{})
diff --git a/test/Assembler/invalid-safestack-param.ll b/test/Assembler/invalid-safestack-param.ll
new file mode 100644
index 00000000000..b9046c4c33b
--- /dev/null
+++ b/test/Assembler/invalid-safestack-param.ll
@@ -0,0 +1,4 @@
+; RUN: not llvm-as -o /dev/null %s 2>&1 | FileCheck %s
+
+; CHECK: error: invalid use of function-only attribute
+declare void @foo(i32 safestack %x)
diff --git a/test/Assembler/invalid-safestack-return.ll b/test/Assembler/invalid-safestack-return.ll
new file mode 100644
index 00000000000..605e72e84c8
--- /dev/null
+++ b/test/Assembler/invalid-safestack-return.ll
@@ -0,0 +1,4 @@
+; RUN: not llvm-as -o /dev/null %s 2>&1 | FileCheck %s
+
+; CHECK: error: invalid use of function-only attribute
+declare safestack void @foo()
diff --git a/test/Assembler/mdcompileunit.ll b/test/Assembler/mdcompileunit.ll
deleted file mode 100644
index ce0052367f4..00000000000
--- a/test/Assembler/mdcompileunit.ll
+++ /dev/null
@@ -1,31 +0,0 @@
-; RUN: llvm-as < %s | llvm-dis | llvm-as | llvm-dis | FileCheck %s
-; RUN: verify-uselistorder %s
-
-; CHECK: !named = !{!0, !1, !2, !3, !4, !5, !6, !7, !7, !8, !8}
-!named = !{!0, !1, !2, !3, !4, !5, !6, !7, !8, !9, !10}
-
-!0 = distinct !{}
-!1 = !MDFile(filename: "path/to/file", directory: "/path/to/dir")
-!2 = distinct !{}
-!3 = distinct !{}
-!4 = distinct !{}
-!5 = distinct !{}
-!6 = distinct !{}
-
-; CHECK: !7 = !MDCompileUnit(language: DW_LANG_C99, file: !1, producer: "clang", isOptimized: true, flags: "-O2", runtimeVersion: 2, splitDebugFilename: "abc.debug", emissionKind: 3, enums: !2, retainedTypes: !3, subprograms: !4, globals: !5, imports: !6)
-!7 = !MDCompileUnit(language: DW_LANG_C99, file: !1, producer: "clang",
-                    isOptimized: true, flags: "-O2", runtimeVersion: 2,
-                    splitDebugFilename: "abc.debug", emissionKind: 3,
-                    enums: !2, retainedTypes: !3, subprograms: !4,
-                    globals: !5, imports: !6)
-!8 = !MDCompileUnit(language: 12, file: !1, producer: "clang",
-                    isOptimized: true, flags: "-O2", runtimeVersion: 2,
-                    splitDebugFilename: "abc.debug", emissionKind: 3,
-                    enums: !2, retainedTypes: !3, subprograms: !4,
-                    globals: !5, imports: !6)
-
-; CHECK: !8 = !MDCompileUnit(language: DW_LANG_C99, file: !1, isOptimized: false, runtimeVersion: 0, emissionKind: 0)
-!9 = !MDCompileUnit(language: 12, file: !1, producer: "",
-                    isOptimized: false, flags: "", runtimeVersion: 0,
-                    splitDebugFilename: "", emissionKind: 0)
-!10 = !MDCompileUnit(language: 12, file: !1)
diff --git a/test/Assembler/mdexpression.ll b/test/Assembler/mdexpression.ll
deleted file mode 100644
index 42bbfe02d67..00000000000
--- a/test/Assembler/mdexpression.ll
+++ /dev/null
@@ -1,16 +0,0 @@
-; RUN: llvm-as < %s | llvm-dis | llvm-as | llvm-dis | FileCheck %s
-; RUN: verify-uselistorder %s
-
-; CHECK: !named = !{!0, !1, !2, !3, !4}
-!named = !{!0, !1, !2, !3, !4}
-
-; CHECK:      !0 = !MDExpression()
-; CHECK-NEXT: !1 = !MDExpression(DW_OP_deref)
-; CHECK-NEXT: !2 = !MDExpression(DW_OP_plus, 3)
-; CHECK-NEXT: !3 = !MDExpression(DW_OP_bit_piece, 3, 7)
-; CHECK-NEXT: !4 = !MDExpression(DW_OP_deref, DW_OP_plus, 3, DW_OP_bit_piece, 3, 7)
-!0 = !MDExpression()
-!1 = !MDExpression(DW_OP_deref)
-!2 = !MDExpression(DW_OP_plus, 3)
-!3 = !MDExpression(DW_OP_bit_piece, 3, 7)
-!4 = !MDExpression(DW_OP_deref, DW_OP_plus, 3, DW_OP_bit_piece, 3, 7)
diff --git a/test/Assembler/mdfile-escaped-chars.ll b/test/Assembler/mdfile-escaped-chars.ll
deleted file mode 100644
index 5dfd12caf9c..00000000000
--- a/test/Assembler/mdfile-escaped-chars.ll
+++ /dev/null
@@ -1,9 +0,0 @@
-; RUN: llvm-as < %s | llvm-dis | llvm-as | llvm-dis | FileCheck %s
-; RUN: verify-uselistorder %s
-
-; Spot-check that metadata string fields are correctly escaped.
-
-!named = !{!0}
-
-; CHECK: !0 = !MDFile(filename: "\00\01\02\80\81\82\FD\FE\FF", directory: "/dir")
-!0 = !MDFile(filename: "\00\01\02\80\81\82\FD\FE\FF", directory: "/dir")
diff --git a/test/Assembler/mdglobalvariable.ll b/test/Assembler/mdglobalvariable.ll
deleted file mode 100644
index 2726f4f3510..00000000000
--- a/test/Assembler/mdglobalvariable.ll
+++ /dev/null
@@ -1,27 +0,0 @@
-; RUN: llvm-as < %s | llvm-dis | llvm-as | llvm-dis | FileCheck %s
-; RUN: verify-uselistorder %s
-
-@foo = global i32 0
-
-; CHECK: !named = !{!0, !1, !2, !3, !4, !5, !6, !7, !8, !9}
-!named = !{!0, !1, !2, !3, !4, !5, !6, !7, !8, !9}
-
-!0 = !MDFile(filename: "scope.h", directory: "/path/to/dir")
-!1 = distinct !{}
-!2 = !MDFile(filename: "path/to/file", directory: "/path/to/dir")
-!3 = !MDBasicType(name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
-!4 = distinct !{}
-
-; CHECK: !5 = !MDGlobalVariable(name: "foo", linkageName: "foo", scope: !0, file: !2, line: 7, type: !3, isLocal: true, isDefinition: false, variable: i32* @foo)
-!5 = !MDGlobalVariable(name: "foo", linkageName: "foo", scope: !0,
-                       file: !2, line: 7, type: !3, isLocal: true,
-                       isDefinition: false, variable: i32* @foo)
-
-; CHECK: !6 = !MDGlobalVariable(name: "foo", scope: !0, isLocal: false, isDefinition: true)
-!6 = !MDGlobalVariable(name: "foo", scope: !0)
-
-!7 = !MDCompositeType(tag: DW_TAG_structure_type, name: "Class", size: 8, align: 8)
-!8 = !MDDerivedType(tag: DW_TAG_member, name: "mem", flags: DIFlagStaticMember, scope: !7, baseType: !3)
-
-; CHECK: !9 = !MDGlobalVariable(name: "mem", scope: !0, isLocal: false, isDefinition: true, declaration: !8)
-!9 = !MDGlobalVariable(name: "mem", scope: !0, declaration: !8)
diff --git a/test/Assembler/mdimportedentity.ll b/test/Assembler/mdimportedentity.ll
deleted file mode 100644
index 8e98c5de212..00000000000
--- a/test/Assembler/mdimportedentity.ll
+++ /dev/null
@@ -1,20 +0,0 @@
-; RUN: llvm-as < %s | llvm-dis | llvm-as | llvm-dis | FileCheck %s
-; RUN: verify-uselistorder %s
-
-; CHECK: !named = !{!0, !1, !2, !3, !3}
-!named = !{!0, !1, !2, !3, !4}
-
-; CHECK:      !0 = !MDSubprogram({{.*}})
-; CHECK-NEXT: !1 = !MDCompositeType({{.*}})
-!0 = !MDSubprogram(name: "foo")
-!1 = !MDCompositeType(tag: DW_TAG_structure_type, name: "Class", size: 32, align: 32)
-
-; CHECK-NEXT: !2 = !MDImportedEntity(tag: DW_TAG_imported_module, name: "foo", scope: !0, entity: !1, line: 7)
-!2 = !MDImportedEntity(tag: DW_TAG_imported_module, name: "foo", scope: !0,
-                       entity: !1, line: 7)
-
-; CHECK-NEXT: !3 = !MDImportedEntity(tag: DW_TAG_imported_module, scope: !0)
-!3 = !MDImportedEntity(tag: DW_TAG_imported_module, scope: !0)
-!4 = !MDImportedEntity(tag: DW_TAG_imported_module, name: "", scope: !0, entity: null,
-                       line: 0)
-
diff --git a/test/Assembler/mdlexicalblock.ll b/test/Assembler/mdlexicalblock.ll
deleted file mode 100644
index d88eae747ac..00000000000
--- a/test/Assembler/mdlexicalblock.ll
+++ /dev/null
@@ -1,25 +0,0 @@
-; RUN: llvm-as < %s | llvm-dis | llvm-as | llvm-dis | FileCheck %s
-; RUN: verify-uselistorder %s
-
-; CHECK: !named = !{!0, !1, !2, !3, !4, !4, !5, !6, !7, !7}
-!named = !{!0, !1, !2, !3, !4, !5, !6, !7, !8, !9}
-
-!0 = distinct !{}
-!1 = !MDSubprogram(name: "foo", scope: !2)
-!2 = !MDFile(filename: "path/to/file", directory: "/path/to/dir")
-
-; CHECK: !3 = !MDLexicalBlock(scope: !1, file: !2, line: 7, column: 35)
-!3 = !MDLexicalBlock(scope: !1, file: !2, line: 7, column: 35)
-
-; CHECK: !4 = !MDLexicalBlock(scope: !1)
-!4 = !MDLexicalBlock(scope: !1)
-!5 = !MDLexicalBlock(scope: !1, file: null, line: 0, column: 0)
-
-; CHECK: !5 = !MDLexicalBlockFile(scope: !3, file: !2, discriminator: 0)
-; CHECK: !6 = !MDLexicalBlockFile(scope: !3, file: !2, discriminator: 1)
-!6 = !MDLexicalBlockFile(scope: !3, file: !2, discriminator: 0)
-!7 = !MDLexicalBlockFile(scope: !3, file: !2, discriminator: 1)
-
-; CHECK: !7 = !MDLexicalBlockFile(scope: !3, discriminator: 7)
-!8 = !MDLexicalBlockFile(scope: !3, discriminator: 7)
-!9 = !MDLexicalBlockFile(scope: !3, file: null, discriminator: 7)
diff --git a/test/Assembler/mdlocalvariable.ll b/test/Assembler/mdlocalvariable.ll
deleted file mode 100644
index 2c0b35a3a57..00000000000
--- a/test/Assembler/mdlocalvariable.ll
+++ /dev/null
@@ -1,26 +0,0 @@
-; RUN: llvm-as < %s | llvm-dis | llvm-as | llvm-dis | FileCheck %s
-; RUN: verify-uselistorder %s
-
-@foo = global i32 0
-
-; CHECK: !named = !{!0, !1, !2, !3, !4, !5, !6, !7, !8}
-!named = !{!0, !1, !2, !3, !4, !5, !6, !7, !8}
-
-!0 = distinct !MDSubprogram()
-!1 = distinct !{}
-!2 = !MDFile(filename: "path/to/file", directory: "/path/to/dir")
-!3 = !MDBasicType(name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
-!4 = !MDLocation(scope: !0)
-
-; CHECK: !5 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "foo", arg: 3, scope: !0, file: !2, line: 7, type: !3, flags: DIFlagArtificial)
-; CHECK: !6 = !MDLocalVariable(tag: DW_TAG_auto_variable, name: "foo", scope: !0, file: !2, line: 7, type: !3, flags: DIFlagArtificial)
-!5 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "foo", arg: 3,
-                      scope: !0, file: !2, line: 7, type: !3,
-                      flags: DIFlagArtificial)
-!6 = !MDLocalVariable(tag: DW_TAG_auto_variable, name: "foo", scope: !0,
-                      file: !2, line: 7, type: !3, flags: DIFlagArtificial)
-
-; CHECK: !7 = !MDLocalVariable(tag: DW_TAG_arg_variable, arg: 0, scope: !0)
-; CHECK: !8 = !MDLocalVariable(tag: DW_TAG_auto_variable, scope: !0)
-!7 = !MDLocalVariable(tag: DW_TAG_arg_variable, scope: !0)
-!8 = !MDLocalVariable(tag: DW_TAG_auto_variable, scope: !0)
diff --git a/test/Assembler/mdlocation.ll b/test/Assembler/mdlocation.ll
deleted file mode 100644
index 47e6e983d42..00000000000
--- a/test/Assembler/mdlocation.ll
+++ /dev/null
@@ -1,23 +0,0 @@
-; RUN: llvm-as < %s | llvm-dis | FileCheck %s
-; RUN: verify-uselistorder %s
-
-; CHECK: !named = !{!0, !1, !1, !2, !2, !3, !3, !4}
-!named = !{!0, !1, !2, !3, !4, !5, !6, !7}
-
-; CHECK: !0 = !MDSubprogram(
-!0 = !MDSubprogram()
-
-; CHECK-NEXT: !1 = !MDLocation(line: 3, column: 7, scope: !0)
-!1 = !MDLocation(line: 3, column: 7, scope: !0)
-!2 = !MDLocation(scope: !0, column: 7, line: 3)
-
-; CHECK-NEXT: !2 = !MDLocation(line: 3, column: 7, scope: !0, inlinedAt: !1)
-!3 = !MDLocation(scope: !0, inlinedAt: !1, column: 7, line: 3)
-!4 = !MDLocation(column: 7, line: 3, scope: !0, inlinedAt: !1)
-
-; CHECK-NEXT: !3 = !MDLocation(line: 0, scope: !0)
-!5 = !MDLocation(scope: !0)
-!6 = !MDLocation(scope: !0, column: 0, line: 0)
-
-; CHECK-NEXT: !4 = !MDLocation(line: 4294967295, column: 65535, scope: !0)
-!7 = !MDLocation(line: 4294967295, column: 65535, scope: !0)
diff --git a/test/Assembler/mdnamespace.ll b/test/Assembler/mdnamespace.ll
deleted file mode 100644
index e708a3c5faf..00000000000
--- a/test/Assembler/mdnamespace.ll
+++ /dev/null
@@ -1,16 +0,0 @@
-; RUN: llvm-as < %s | llvm-dis | llvm-as | llvm-dis | FileCheck %s
-; RUN: verify-uselistorder %s
-
-; CHECK: !named = !{!0, !1, !2, !3, !4, !4}
-!named = !{!0, !1, !2, !3, !4, !5}
-
-!0 = !MDFile(filename: "file.cpp", directory: "/path/to/dir")
-!1 = distinct !{}
-!2 = !MDFile(filename: "path/to/file", directory: "/path/to/dir")
-
-; CHECK: !3 = !MDNamespace(name: "Namespace", scope: !0, file: !2, line: 7)
-!3 = !MDNamespace(name: "Namespace", scope: !0, file: !2, line: 7)
-
-; CHECK: !4 = !MDNamespace(scope: !0)
-!4 = !MDNamespace(name: "", scope: !0, file: null, line: 0)
-!5 = !MDNamespace(scope: !0)
diff --git a/test/Assembler/mdobjcproperty.ll b/test/Assembler/mdobjcproperty.ll
deleted file mode 100644
index a3b20e5d99d..00000000000
--- a/test/Assembler/mdobjcproperty.ll
+++ /dev/null
@@ -1,19 +0,0 @@
-; RUN: llvm-as < %s | llvm-dis | llvm-as | llvm-dis | FileCheck %s
-; RUN: verify-uselistorder %s
-
-; CHECK: !named = !{!0, !1, !2, !3, !4, !4}
-!named = !{!0, !1, !2, !3, !4, !5}
-
-!0 = distinct !{}
-!1 = !MDFile(filename: "path/to/file", directory: "/path/to/dir")
-!2 = !MDCompositeType(tag: DW_TAG_structure_type, name: "Object")
-
-; CHECK: !2 = !MDCompositeType({{.*}})
-; CHECK-NEXT: !3 = !MDObjCProperty(name: "foo", file: !1, line: 7, setter: "setFoo", getter: "getFoo", attributes: 7, type: !2)
-!3 = !MDObjCProperty(name: "foo", file: !1, line: 7, setter: "setFoo",
-                     getter: "getFoo", attributes: 7, type: !2)
-
-; CHECK-NEXT: !4 = !MDObjCProperty()
-!4 = !MDObjCProperty(name: "", file: null, line: 0, setter: "", getter: "",
-                     attributes: 0, type: null)
-!5 = !MDObjCProperty()
diff --git a/test/Assembler/mdsubprogram.ll b/test/Assembler/mdsubprogram.ll
deleted file mode 100644
index 93000893c16..00000000000
--- a/test/Assembler/mdsubprogram.ll
+++ /dev/null
@@ -1,28 +0,0 @@
-; RUN: llvm-as < %s | llvm-dis | llvm-as | llvm-dis | FileCheck %s
-; RUN: verify-uselistorder %s
-
-declare void @_Z3foov()
-
-; CHECK: !named = !{!0, !1, !2, !3, !4, !5, !6, !7, !8, !9}
-!named = !{!0, !1, !2, !3, !4, !5, !6, !7, !8, !9}
-
-!0 = !{null}
-!1 = distinct !MDCompositeType(tag: DW_TAG_structure_type)
-!2 = !MDFile(filename: "path/to/file", directory: "/path/to/dir")
-!3 = !MDSubroutineType(types: !0)
-!4 = distinct !MDCompositeType(tag: DW_TAG_structure_type)
-!5 = distinct !{}
-!6 = distinct !MDSubprogram(isDefinition: false)
-!7 = distinct !{}
-
-; CHECK: !8 = !MDSubprogram(name: "foo", linkageName: "_Zfoov", scope: !1, file: !2, line: 7, type: !3, isLocal: true, isDefinition: false, scopeLine: 8, containingType: !4, virtuality: DW_VIRTUALITY_pure_virtual, virtualIndex: 10, flags: DIFlagPrototyped, isOptimized: true, function: void ()* @_Z3foov, templateParams: !5, declaration: !6, variables: !7)
-!8 = !MDSubprogram(name: "foo", linkageName: "_Zfoov", scope: !1,
-                   file: !2, line: 7, type: !3, isLocal: true,
-                   isDefinition: false, scopeLine: 8, containingType: !4,
-                   virtuality: DW_VIRTUALITY_pure_virtual, virtualIndex: 10,
-                   flags: DIFlagPrototyped, isOptimized: true, function: void ()* @_Z3foov,
-                   templateParams: !5, declaration: !6, variables: !7)
-
-; CHECK: !9 = !MDSubprogram(scope: null, isLocal: false, isDefinition: true, isOptimized: false)
-!9 = !MDSubprogram()
-
diff --git a/test/Assembler/mdsubrange-empty-array.ll b/test/Assembler/mdsubrange-empty-array.ll
deleted file mode 100644
index fa05582a9a9..00000000000
--- a/test/Assembler/mdsubrange-empty-array.ll
+++ /dev/null
@@ -1,14 +0,0 @@
-; RUN: llvm-as < %s | llvm-dis | llvm-as | llvm-dis | FileCheck %s
-; RUN: verify-uselistorder %s
-
-; CHECK: !named = !{!0, !0, !1, !2}
-!named = !{!0, !1, !2, !3}
-
-; CHECK:      !0 = !MDSubrange(count: -1)
-; CHECK-NEXT: !1 = !MDSubrange(count: -1, lowerBound: 4)
-; CHECK-NEXT: !2 = !MDSubrange(count: -1, lowerBound: -5)
-!0 = !MDSubrange(count: -1)
-!1 = !MDSubrange(count: -1, lowerBound: 0)
-
-!2 = !MDSubrange(count: -1, lowerBound: 4)
-!3 = !MDSubrange(count: -1, lowerBound: -5)
diff --git a/test/Assembler/mdsubroutinetype.ll b/test/Assembler/mdsubroutinetype.ll
deleted file mode 100644
index 936cbf4ec7b..00000000000
--- a/test/Assembler/mdsubroutinetype.ll
+++ /dev/null
@@ -1,23 +0,0 @@
-; RUN: llvm-as < %s | llvm-dis | llvm-as | llvm-dis | FileCheck %s
-; RUN: verify-uselistorder %s
-
-; CHECK: !named = !{!0, !1, !2, !3, !4, !5, !6, !7, !8}
-!named = !{!0, !1, !2, !3, !4, !5, !6, !7, !8}
-
-!0 = !MDBasicType(name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
-!1 = !{null}
-!2 = !{null, !0}
-!3 = !{!0, !0, !0}
-
-
-; CHECK: !4 = !MDSubroutineType(types: !1)
-; CHECK: !5 = !MDSubroutineType(types: !2)
-; CHECK: !6 = !MDSubroutineType(types: !3)
-; CHECK: !7 = !MDSubroutineType(flags: DIFlagLValueReference, types: !3)
-!4 = !MDSubroutineType(types: !1)
-!5 = !MDSubroutineType(types: !2)
-!6 = !MDSubroutineType(types: !3)
-!7 = !MDSubroutineType(flags: DIFlagLValueReference, types: !3)
-
-; CHECK: !8 = !MDSubroutineType(types: null)
-!8 = !MDSubroutineType(types: null)
diff --git a/test/Assembler/mdtemplateparameter.ll b/test/Assembler/mdtemplateparameter.ll
deleted file mode 100644
index 5bcef3f21e9..00000000000
--- a/test/Assembler/mdtemplateparameter.ll
+++ /dev/null
@@ -1,24 +0,0 @@
-; RUN: llvm-as < %s | llvm-dis | llvm-as | llvm-dis | FileCheck %s
-; RUN: verify-uselistorder %s
-
-; CHECK: !named = !{!0, !1, !2, !3, !3, !4, !5, !5, !6}
-!named = !{!0, !1, !2, !3, !4, !5, !6, !7, !8}
-
-!0 = distinct !{}
-!1 = !MDBasicType(name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
-; CHECK: !1 = !MDBasicType(name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
-
-; CHECK-NEXT: !2 = !MDTemplateTypeParameter(name: "Ty", type: !1)
-; CHECK-NEXT: !3 = !MDTemplateTypeParameter(type: !1)
-!2 = !MDTemplateTypeParameter(name: "Ty", type: !1)
-!3 = !MDTemplateTypeParameter(type: !1)
-!4 = !MDTemplateTypeParameter(name: "", type: !1)
-
-; CHECK-NEXT: !4 = !MDTemplateValueParameter(name: "V", type: !1, value: i32 7)
-; CHECK-NEXT: !5 = !MDTemplateValueParameter(type: !1, value: i32 7)
-; CHECK-NEXT: !6 = !MDTemplateValueParameter(tag: DW_TAG_GNU_template_template_param, name: "param", type: !1, value: !"template")
-!5 = !MDTemplateValueParameter(name: "V", type: !1, value: i32 7)
-!6 = !MDTemplateValueParameter(type: !1, value: i32 7)
-!7 = !MDTemplateValueParameter(tag: DW_TAG_template_value_parameter,
-                               name: "", type: !1, value: i32 7)
-!8 = !MDTemplateValueParameter(tag: DW_TAG_GNU_template_template_param, name: "param", type: !1, value: !"template")
diff --git a/test/Assembler/mdtype-large-values.ll b/test/Assembler/mdtype-large-values.ll
deleted file mode 100644
index b2b0a87abb8..00000000000
--- a/test/Assembler/mdtype-large-values.ll
+++ /dev/null
@@ -1,12 +0,0 @@
-; RUN: llvm-as < %s | llvm-dis | llvm-as | llvm-dis | FileCheck %s
-; RUN: verify-uselistorder %s
-
-; CHECK: !named = !{!0, !1, !2}
-!named = !{!0, !1, !2}
-
-; CHECK:      !0 = !MDBasicType(name: "name", size: 18446744073709551615, align: 18446744073709551614, encoding: DW_ATE_unsigned_char)
-; CHECK-NEXT: !1 = !MDDerivedType(tag: DW_TAG_pointer_type, baseType: !0, size: 18446744073709551615, align: 18446744073709551614, offset: 18446744073709551613)
-; CHECK-NEXT: !2 = !MDCompositeType(tag: DW_TAG_array_type, baseType: !0, size: 18446744073709551615, align: 18446744073709551614, offset: 18446744073709551613)
-!0 = !MDBasicType(tag: DW_TAG_base_type, name: "name", size: 18446744073709551615, align: 18446744073709551614, encoding: DW_ATE_unsigned_char)
-!1 = !MDDerivedType(tag: DW_TAG_pointer_type, baseType: !0, size: 18446744073709551615, align: 18446744073709551614, offset: 18446744073709551613)
-!2 = !MDCompositeType(tag: DW_TAG_array_type, baseType: !0, size: 18446744073709551615, align: 18446744073709551614, offset: 18446744073709551613)
diff --git a/test/Assembler/metadata-null-operands.ll b/test/Assembler/metadata-null-operands.ll
index 7e27eba45fe..2d013d0e912 100644
--- a/test/Assembler/metadata-null-operands.ll
+++ b/test/Assembler/metadata-null-operands.ll
@@ -5,7 +5,7 @@
 ; require non-null in the assembler and rework this test to check for that ala
 ; test/Assembler/invalid-mdcompileunit-null-file.ll.
 !named = !{!0}
-!0 = !MDDerivedType(tag: DW_TAG_pointer_type, baseType: null)
+!0 = !DIDerivedType(tag: DW_TAG_pointer_type, baseType: null)
 
 ; CHECK: !named = !{!0}
-; CHECK: !0 = !MDDerivedType({{.*}}baseType: null{{.*}})
+; CHECK: !0 = !DIDerivedType({{.*}}baseType: null{{.*}})
diff --git a/test/Assembler/metadata.ll b/test/Assembler/metadata.ll
index b483fc3f4cf..052ac1b5097 100644
--- a/test/Assembler/metadata.ll
+++ b/test/Assembler/metadata.ll
@@ -1,7 +1,7 @@
 ; RUN: llvm-as < %s | llvm-dis | llvm-as | llvm-dis | FileCheck %s
 ; RUN: verify-uselistorder %s
 
-; CHECK: @test
+; CHECK-LABEL: @test
 ; CHECK: ret void, !bar !1, !foo !0
 define void @test() {
   add i32 2, 1, !bar !0
@@ -11,8 +11,29 @@ define void @test() {
   ret void, !foo !0, !bar !1
 }
 
-!0 = !MDLocation(line: 662302, column: 26, scope: !1)
-!1 = !MDSubprogram(name: "foo")
+; CHECK-LABEL: define void @test2() !foo !2 !baz !3
+define void @test2() !foo !2 !baz !3 {
+  unreachable
+}
+
+; CHECK-LABEL: define void @test3() !bar !3
+; CHECK: unreachable, !bar !4
+define void @test3() !bar !3 {
+  unreachable, !bar !4
+}
+
+; CHECK-LABEL: define void @test_attachment_name() {
+; CHECK:   unreachable, !\342abc !4
+define void @test_attachment_name() {
+  ;; Escape the first character when printing text IR, since it's a digit
+  unreachable, !\34\32abc !4
+}
+
+!0 = !DILocation(line: 662302, column: 26, scope: !1)
+!1 = !DISubprogram(name: "foo")
+!2 = distinct !{}
+!3 = distinct !{}
+!4 = distinct !{}
 
 declare void @llvm.dbg.func.start(metadata) nounwind readnone
 
diff --git a/test/Assembler/unnamed-alias.ll b/test/Assembler/unnamed-alias.ll
new file mode 100644
index 00000000000..8ae1c45d312
--- /dev/null
+++ b/test/Assembler/unnamed-alias.ll
@@ -0,0 +1,11 @@
+; RUN: llvm-as < %s | llvm-dis | FileCheck %s
+
+@0 = private constant i32 0
+; CHECK: @0 = private constant i32 0
+@1 = private constant i32 1
+; CHECK: @1 = private constant i32 1
+
+@2 = private alias i32* @0
+; CHECK: @2 = private alias i32* @0
+@3 = private alias i32* @1
+; CHECK: @3 = private alias i32* @1
diff --git a/test/Bindings/Go/go.test b/test/Bindings/Go/go.test
index 9b4ce8fe54c..14eb3281cc4 100644
--- a/test/Bindings/Go/go.test
+++ b/test/Bindings/Go/go.test
@@ -1,14 +1,3 @@
-; @LOCALMOD-BEGIN
-; Disable this for now to allow cmake builds with libc++.
-; The bindings test tries build a bunch of things and then link against
-; the LLVM libraries, but the build script that it uses within the test
-; does not pick up changes to CMAKE_C*_FLAGS so if we change that then
-; the link commandline is wrong:
-; https://llvm.org/bugs/show_bug.cgi?id=21552
-; https://code.google.com/p/nativeclient/issues/detail?id=4119
-; Add "-x" to the below commandline to get more verbose output.
-; XFAIL: *
-; @LOCALMOD-END
 ; RUN: llvm-go test llvm.org/llvm/bindings/go/llvm
 
 ; REQUIRES: shell, not_ubsan
diff --git a/test/Bindings/OCaml/analysis.ml b/test/Bindings/OCaml/analysis.ml
index e935ee83805..22ef153b372 100644
--- a/test/Bindings/OCaml/analysis.ml
+++ b/test/Bindings/OCaml/analysis.ml
@@ -1,7 +1,7 @@
 (* RUN: cp %s %T/analysis.ml
- * RUN: %ocamlc -g -warn-error A -package llvm.analysis -linkpkg %T/analysis.ml -o %t
+ * RUN: %ocamlc -g -w +A -package llvm.analysis -linkpkg %T/analysis.ml -o %t
  * RUN: %t
- * RUN: %ocamlopt -g -warn-error A -package llvm.analysis -linkpkg %T/analysis.ml -o %t
+ * RUN: %ocamlopt -g -w +A -package llvm.analysis -linkpkg %T/analysis.ml -o %t
  * RUN: %t
  * XFAIL: vg_leak
  *)
diff --git a/test/Bindings/OCaml/bitreader.ml b/test/Bindings/OCaml/bitreader.ml
index 57cfd043b2d..3fda34ab22e 100644
--- a/test/Bindings/OCaml/bitreader.ml
+++ b/test/Bindings/OCaml/bitreader.ml
@@ -1,7 +1,7 @@
 (* RUN: cp %s %T/bitreader.ml
- * RUN: %ocamlc -g -warn-error A -package llvm.bitreader -package llvm.bitwriter -linkpkg %T/bitreader.ml -o %t
+ * RUN: %ocamlc -g -w +A -package llvm.bitreader -package llvm.bitwriter -linkpkg %T/bitreader.ml -o %t
  * RUN: %t %t.bc
- * RUN: %ocamlopt -g -warn-error A -package llvm.bitreader -package llvm.bitwriter -linkpkg %T/bitreader.ml -o %t
+ * RUN: %ocamlopt -g -w +A -package llvm.bitreader -package llvm.bitwriter -linkpkg %T/bitreader.ml -o %t
  * RUN: %t %t.bc
  * RUN: llvm-dis < %t.bc
  * XFAIL: vg_leak
diff --git a/test/Bindings/OCaml/bitwriter.ml b/test/Bindings/OCaml/bitwriter.ml
index 7c803f6d551..b051b8c61a2 100644
--- a/test/Bindings/OCaml/bitwriter.ml
+++ b/test/Bindings/OCaml/bitwriter.ml
@@ -1,7 +1,7 @@
 (* RUN: cp %s %T/bitwriter.ml
- * RUN: %ocamlc -g -w -3 -warn-error A -package llvm.bitreader -package llvm.bitwriter -linkpkg %T/bitwriter.ml -o %t
+ * RUN: %ocamlc -g -w -3 -w +A -package llvm.bitreader -package llvm.bitwriter -linkpkg %T/bitwriter.ml -o %t
  * RUN: %t %t.bc
- * RUN: %ocamlopt -g -w -3 -warn-error A -package llvm.bitreader -package llvm.bitwriter -linkpkg %T/bitwriter.ml -o %t
+ * RUN: %ocamlopt -g -w -3 -w +A -package llvm.bitreader -package llvm.bitwriter -linkpkg %T/bitwriter.ml -o %t
  * RUN: %t %t.bc
  * RUN: llvm-dis < %t.bc
  * XFAIL: vg_leak
diff --git a/test/Bindings/OCaml/core.ml b/test/Bindings/OCaml/core.ml
index ce63a6d5ed9..73e9956bc56 100644
--- a/test/Bindings/OCaml/core.ml
+++ b/test/Bindings/OCaml/core.ml
@@ -1,7 +1,7 @@
 (* RUN: cp %s %T/core.ml
- * RUN: %ocamlc -g -warn-error A -package llvm.analysis -package llvm.bitwriter -linkpkg %T/core.ml -o %t
+ * RUN: %ocamlc -g -w +A -package llvm.analysis -package llvm.bitwriter -linkpkg %T/core.ml -o %t
  * RUN: %t %t.bc
- * RUN: %ocamlopt -g -warn-error A -package llvm.analysis -package llvm.bitwriter -linkpkg %T/core.ml -o %t
+ * RUN: %ocamlopt -g -w +A -package llvm.analysis -package llvm.bitwriter -linkpkg %T/core.ml -o %t
  * RUN: %t %t.bc
  * RUN: llvm-dis < %t.bc > %t.ll
  * RUN: FileCheck %s < %t.ll
@@ -1445,7 +1445,7 @@ let test_builder () =
  * CHECK: !llvm.module.flags = !{!0}
  * CHECK: !0 = !{i32 1, !"Debug Info Version", i32 3}
  * CHECK: !1 = !{i32 1, !"metadata test"}
- * CHECK: !2 = !MDLocation(line: 2, column: 3, scope: !3, inlinedAt: !3)
+ * CHECK: !2 = !DILocation(line: 2, column: 3, scope: !3, inlinedAt: !3)
  *)
 
 (*===-- Pass Managers -----------------------------------------------------===*)
diff --git a/test/Bindings/OCaml/executionengine.ml b/test/Bindings/OCaml/executionengine.ml
index 1de2cfb7fef..6d9abe72880 100644
--- a/test/Bindings/OCaml/executionengine.ml
+++ b/test/Bindings/OCaml/executionengine.ml
@@ -1,7 +1,7 @@
 (* RUN: cp %s %T/executionengine.ml
- * RUN: %ocamlc -g -warn-error A -package llvm.executionengine -linkpkg %T/executionengine.ml -o %t
+ * RUN: %ocamlc -g -w +A -package llvm.executionengine -linkpkg %T/executionengine.ml -o %t
  * RUN: %t
- * RUN: %ocamlopt -g -warn-error A -package llvm.executionengine -linkpkg %T/executionengine.ml -o %t
+ * RUN: %ocamlopt -g -w +A -package llvm.executionengine -linkpkg %T/executionengine.ml -o %t
  * RUN: %t
  * REQUIRES: native, object-emission
  * XFAIL: vg_leak
diff --git a/test/Bindings/OCaml/ext_exc.ml b/test/Bindings/OCaml/ext_exc.ml
index 2b44803f517..a24a28b1f52 100644
--- a/test/Bindings/OCaml/ext_exc.ml
+++ b/test/Bindings/OCaml/ext_exc.ml
@@ -1,7 +1,7 @@
 (* RUN: cp %s %T/ext_exc.ml
- * RUN: %ocamlc -g -warn-error A -package llvm.bitreader -linkpkg %T/ext_exc.ml -o %t
+ * RUN: %ocamlc -g -w +A -package llvm.bitreader -linkpkg %T/ext_exc.ml -o %t
  * RUN: %t
- * RUN: %ocamlopt -g -warn-error A -package llvm.bitreader -linkpkg %T/ext_exc.ml -o %t
+ * RUN: %ocamlopt -g -w +A -package llvm.bitreader -linkpkg %T/ext_exc.ml -o %t
  * RUN: %t
  * XFAIL: vg_leak
  *)
diff --git a/test/Bindings/OCaml/ipo.ml b/test/Bindings/OCaml/ipo.ml
index fc728b92ff3..4a7c73e891b 100644
--- a/test/Bindings/OCaml/ipo.ml
+++ b/test/Bindings/OCaml/ipo.ml
@@ -1,7 +1,7 @@
 (* RUN: cp %s %T/ipo_opts.ml
- * RUN: %ocamlc -g -warn-error A -package llvm.ipo -linkpkg %T/ipo_opts.ml -o %t
+ * RUN: %ocamlc -g -w +A -package llvm.ipo -linkpkg %T/ipo_opts.ml -o %t
  * RUN: %t %t.bc
- * RUN: %ocamlopt -g -warn-error A -package llvm.ipo -linkpkg %T/ipo_opts.ml -o %t
+ * RUN: %ocamlopt -g -w +A -package llvm.ipo -linkpkg %T/ipo_opts.ml -o %t
  * RUN: %t %t.bc
  * XFAIL: vg_leak
  *)
diff --git a/test/Bindings/OCaml/irreader.ml b/test/Bindings/OCaml/irreader.ml
index e1771e75dd4..b5142334d32 100644
--- a/test/Bindings/OCaml/irreader.ml
+++ b/test/Bindings/OCaml/irreader.ml
@@ -1,7 +1,7 @@
 (* RUN: cp %s %T/irreader.ml
- * RUN: %ocamlc -g -warn-error A -package llvm.irreader -linkpkg %T/irreader.ml -o %t
+ * RUN: %ocamlc -g -w +A -package llvm.irreader -linkpkg %T/irreader.ml -o %t
  * RUN: %t
- * RUN: %ocamlopt -g -warn-error A -package llvm.irreader -linkpkg %T/irreader.ml -o %t
+ * RUN: %ocamlopt -g -w +A -package llvm.irreader -linkpkg %T/irreader.ml -o %t
  * RUN: %t
  * XFAIL: vg_leak
  *)
diff --git a/test/Bindings/OCaml/linker.ml b/test/Bindings/OCaml/linker.ml
index 1ea0be9d3dc..275a143d178 100644
--- a/test/Bindings/OCaml/linker.ml
+++ b/test/Bindings/OCaml/linker.ml
@@ -1,7 +1,7 @@
 (* RUN: cp %s %T/linker.ml
- * RUN: %ocamlc -g -warn-error A -package llvm.linker -linkpkg %T/linker.ml -o %t
+ * RUN: %ocamlc -g -w +A -package llvm.linker -linkpkg %T/linker.ml -o %t
  * RUN: %t
- * RUN: %ocamlopt -g -warn-error A -package llvm.linker -linkpkg %T/linker.ml -o %t
+ * RUN: %ocamlopt -g -w +A -package llvm.linker -linkpkg %T/linker.ml -o %t
  * RUN: %t
  * XFAIL: vg_leak
  *)
diff --git a/test/Bindings/OCaml/passmgr_builder.ml b/test/Bindings/OCaml/passmgr_builder.ml
index 5dd9d4e00e8..3104736d66a 100644
--- a/test/Bindings/OCaml/passmgr_builder.ml
+++ b/test/Bindings/OCaml/passmgr_builder.ml
@@ -1,7 +1,7 @@
 (* RUN: cp %s %T/passmgr_builder.ml
- * RUN: %ocamlc -g -warn-error A -package llvm.passmgr_builder -linkpkg %T/passmgr_builder.ml -o %t
+ * RUN: %ocamlc -g -w +A -package llvm.passmgr_builder -linkpkg %T/passmgr_builder.ml -o %t
  * RUN: %t %t.bc
- * RUN: %ocamlopt -g -warn-error A -package llvm.passmgr_builder -linkpkg %T/passmgr_builder.ml -o %t
+ * RUN: %ocamlopt -g -w +A -package llvm.passmgr_builder -linkpkg %T/passmgr_builder.ml -o %t
  * RUN: %t %t.bc
  * XFAIL: vg_leak
  *)
diff --git a/test/Bindings/OCaml/scalar_opts.ml b/test/Bindings/OCaml/scalar_opts.ml
index 3017fb13112..9ffcf6915c0 100644
--- a/test/Bindings/OCaml/scalar_opts.ml
+++ b/test/Bindings/OCaml/scalar_opts.ml
@@ -1,7 +1,7 @@
 (* RUN: cp %s %T/scalar_opts.ml
- * RUN: %ocamlc -g -warn-error A -package llvm.scalar_opts -linkpkg %T/scalar_opts.ml -o %t
+ * RUN: %ocamlc -g -w +A -package llvm.scalar_opts -linkpkg %T/scalar_opts.ml -o %t
  * RUN: %t %t.bc
- * RUN: %ocamlopt -g -warn-error A -package llvm.scalar_opts -linkpkg %T/scalar_opts.ml -o %t
+ * RUN: %ocamlopt -g -w +A -package llvm.scalar_opts -linkpkg %T/scalar_opts.ml -o %t
  * RUN: %t %t.bc
  * XFAIL: vg_leak
  *)
diff --git a/test/Bindings/OCaml/target.ml b/test/Bindings/OCaml/target.ml
index 41faefa8ea6..68ba75d04f2 100644
--- a/test/Bindings/OCaml/target.ml
+++ b/test/Bindings/OCaml/target.ml
@@ -1,6 +1,6 @@
 (* RUN: cp %s %T/target.ml
- * RUN: %ocamlc -g -warn-error A -package llvm.target -package llvm.all_backends -linkpkg %T/target.ml -o %t
- * RUN: %ocamlopt -g -warn-error A -package llvm.target -package llvm.all_backends -linkpkg %T/target.ml -o %t
+ * RUN: %ocamlc -g -w +A -package llvm.target -package llvm.all_backends -linkpkg %T/target.ml -o %t
+ * RUN: %ocamlopt -g -w +A -package llvm.target -package llvm.all_backends -linkpkg %T/target.ml -o %t
  * RUN: %t %t.bc
  * XFAIL: vg_leak
  *)
diff --git a/test/Bindings/OCaml/transform_utils.ml b/test/Bindings/OCaml/transform_utils.ml
index 6b46df173b0..dc14effc57e 100644
--- a/test/Bindings/OCaml/transform_utils.ml
+++ b/test/Bindings/OCaml/transform_utils.ml
@@ -1,7 +1,7 @@
 (* RUN: cp %s %T/transform_utils.ml
- * RUN: %ocamlc -g -warn-error A -package llvm.transform_utils -linkpkg %T/transform_utils.ml -o %t
+ * RUN: %ocamlc -g -w +A -package llvm.transform_utils -linkpkg %T/transform_utils.ml -o %t
  * RUN: %t
- * RUN: %ocamlopt -g -warn-error A -package llvm.transform_utils -linkpkg %T/transform_utils.ml -o %t
+ * RUN: %ocamlopt -g -w +A -package llvm.transform_utils -linkpkg %T/transform_utils.ml -o %t
  * RUN: %t
  * XFAIL: vg_leak
  *)
diff --git a/test/Bindings/OCaml/vectorize.ml b/test/Bindings/OCaml/vectorize.ml
index c5b03b52537..a668654de74 100644
--- a/test/Bindings/OCaml/vectorize.ml
+++ b/test/Bindings/OCaml/vectorize.ml
@@ -1,7 +1,7 @@
 (* RUN: cp %s %T/vectorize_opts.ml
- * RUN: %ocamlc -g -warn-error A -package llvm.vectorize -linkpkg %T/vectorize_opts.ml -o %t
+ * RUN: %ocamlc -g -w +A -package llvm.vectorize -linkpkg %T/vectorize_opts.ml -o %t
  * RUN: %t %t.bc
- * RUN: %ocamlopt -g -warn-error A -package llvm.vectorize -linkpkg %T/vectorize_opts.ml -o %t
+ * RUN: %ocamlopt -g -w +A -package llvm.vectorize -linkpkg %T/vectorize_opts.ml -o %t
  * RUN: %t %t.bc
  * XFAIL: vg_leak
  *)
diff --git a/test/Bindings/llvm-c/ARM/disassemble.test b/test/Bindings/llvm-c/ARM/disassemble.test
new file mode 100644
index 00000000000..ffa7ebfe120
--- /dev/null
+++ b/test/Bindings/llvm-c/ARM/disassemble.test
@@ -0,0 +1,21 @@
+; RUN: llvm-c-test --disassemble < %s | FileCheck %s
+
+armv8-linux-gnu     +crypto 02 00 81 e0 02 03 b0 f3
+;CHECK: triple: armv8-linux-gnu, features: +crypto
+;CHECK: 02 00 81 e0                  add r0, r1, r2
+;CHECK: 02 03 b0 f3                  aese.8 q0, q1
+
+armv8-linux-gnu     -crypto 02 00 81 e0 02 03 b0 f3
+;CHECK: triple: armv8-linux-gnu, features: -crypto
+;CHECK: 02 00 81 e0                  add r0, r1, r2
+;CHECK: 02                           ???
+;CHECK: 03                           ???
+;CHECK: b0                           ???
+;CHECK: f3                           ???
+
+arm-linux-android     NULL  44 26 1f e5 0c 10 4b e2 02 20 81 e0
+;CHECK: triple: arm-linux-android, features: NULL
+;CHECK: ldr	r2, [pc, #-1604]
+;CHECK: sub	r1, r11, #12
+;CHECK: 02 20 81 e0
+;CHECK: add	r2, r1, r2
diff --git a/test/Bindings/llvm-c/ARM/lit.local.cfg b/test/Bindings/llvm-c/ARM/lit.local.cfg
new file mode 100644
index 00000000000..7c23e4f9801
--- /dev/null
+++ b/test/Bindings/llvm-c/ARM/lit.local.cfg
@@ -0,0 +1,2 @@
+if not "ARM" in config.root.targets:
+    config.unsupported = True
diff --git a/test/Bindings/llvm-c/X86/disassemble.test b/test/Bindings/llvm-c/X86/disassemble.test
new file mode 100644
index 00000000000..465b370a94f
--- /dev/null
+++ b/test/Bindings/llvm-c/X86/disassemble.test
@@ -0,0 +1,23 @@
+; RUN: llvm-c-test --disassemble < %s | FileCheck %s
+
+x86_64-linux-unknown  NULL  48 83 c4 38 5b 5d 41 5c 41 5d 41 5e 41 5f c3
+;CHECK: triple: x86_64-linux-unknown, features: NULL
+;CHECK: addq	$56, %rsp
+;CHECK: popq	%rbx
+;CHECK: popq	%rbp
+;CHECK: popq	%r12
+;CHECK: popq	%r13
+;CHECK: popq	%r14
+;CHECK: popq	%r15
+;CHECK: ret
+
+i686-apple-darwin     NULL  0f b7 4c 24 0a e8 29 ce ff ff
+;CHECK: triple: i686-apple-darwin, features: NULL
+;CHECK: movzwl	10(%esp), %ecx
+;CHECK: calll	-12759
+
+i686-linux-unknown    NULL  dd 44 24 04 d9 e1 c3
+;CHECK: triple: i686-linux-unknown, features: NULL
+;CHECK: fldl	4(%esp)
+;CHECK: fabs
+;CHECK: ret
diff --git a/test/Bindings/llvm-c/X86/lit.local.cfg b/test/Bindings/llvm-c/X86/lit.local.cfg
new file mode 100644
index 00000000000..42bf50dcc13
--- /dev/null
+++ b/test/Bindings/llvm-c/X86/lit.local.cfg
@@ -0,0 +1,2 @@
+if not "X86" in config.root.targets:
+    config.unsupported = True
diff --git a/test/Bindings/llvm-c/disassemble.test b/test/Bindings/llvm-c/disassemble.test
deleted file mode 100644
index bb7a9a01ab9..00000000000
--- a/test/Bindings/llvm-c/disassemble.test
+++ /dev/null
@@ -1,43 +0,0 @@
-; RUN: llvm-c-test --disassemble < %s | FileCheck %s
-
-armv8-linux-gnu     +crypto 02 00 81 e0 02 03 b0 f3
-;CHECK: triple: armv8-linux-gnu, features: +crypto
-;CHECK: 02 00 81 e0                  add r0, r1, r2
-;CHECK: 02 03 b0 f3                  aese.8 q0, q1
-
-armv8-linux-gnu     -crypto 02 00 81 e0 02 03 b0 f3
-;CHECK: triple: armv8-linux-gnu, features: -crypto
-;CHECK: 02 00 81 e0                  add r0, r1, r2
-;CHECK: 02                           ???
-;CHECK: 03                           ???
-;CHECK: b0                           ???
-;CHECK: f3                           ???
-
-arm-linux-android     NULL  44 26 1f e5 0c 10 4b e2 02 20 81 e0
-;CHECK: triple: arm-linux-android, features: NULL
-;CHECK: ldr	r2, [pc, #-1604]
-;CHECK: sub	r1, r11, #12
-;CHECK: 02 20 81 e0
-;CHECK: add	r2, r1, r2
-
-x86_64-linux-unknown  NULL  48 83 c4 38 5b 5d 41 5c 41 5d 41 5e 41 5f c3
-;CHECK: triple: x86_64-linux-unknown, features: NULL
-;CHECK: addq	$56, %rsp
-;CHECK: popq	%rbx
-;CHECK: popq	%rbp
-;CHECK: popq	%r12
-;CHECK: popq	%r13
-;CHECK: popq	%r14
-;CHECK: popq	%r15
-;CHECK: ret
-
-i686-apple-darwin     NULL  0f b7 4c 24 0a e8 29 ce ff ff
-;CHECK: triple: i686-apple-darwin, features: NULL
-;CHECK: movzwl	10(%esp), %ecx
-;CHECK: calll	-12759
-
-i686-linux-unknown    NULL  dd 44 24 04 d9 e1 c3
-;CHECK: triple: i686-linux-unknown, features: NULL
-;CHECK: fldl	4(%esp)
-;CHECK: fabs
-;CHECK: ret
diff --git a/test/Bindings/llvm-c/lit.local.cfg b/test/Bindings/llvm-c/lit.local.cfg
deleted file mode 100644
index 75b22c06fb2..00000000000
--- a/test/Bindings/llvm-c/lit.local.cfg
+++ /dev/null
@@ -1,4 +0,0 @@
-if not "X86" in config.root.targets:
-    config.unsupported = True
-if not "ARM" in config.root.targets:
-    config.unsupported = True
diff --git a/test/Bitcode/DICompileUnit-no-DWOId.ll b/test/Bitcode/DICompileUnit-no-DWOId.ll
new file mode 100644
index 00000000000..4771c37264f
--- /dev/null
+++ b/test/Bitcode/DICompileUnit-no-DWOId.ll
@@ -0,0 +1,9 @@
+; The input uses the older (r236428) form without a dwoId field.  This should
+; default to 0, which is not displayed at all in the textual representation.
+;
+; RUN: llvm-dis %s.bc -o - | FileCheck %s
+; CHECK: !DICompileUnit
+; CHECK-NOT: dwoId:
+!named = !{!0}
+!0 = !DICompileUnit(language: 12, file: !1)
+!1 = !DIFile(filename: "path/to/file", directory: "/path/to/dir")
diff --git a/test/Bitcode/DICompileUnit-no-DWOId.ll.bc b/test/Bitcode/DICompileUnit-no-DWOId.ll.bc
new file mode 100644
index 00000000000..1a3c99b5806
Binary files /dev/null and b/test/Bitcode/DICompileUnit-no-DWOId.ll.bc differ
diff --git a/test/Bitcode/Inputs/PR23310.bc b/test/Bitcode/Inputs/PR23310.bc
new file mode 100644
index 00000000000..cd1202f69f3
Binary files /dev/null and b/test/Bitcode/Inputs/PR23310.bc differ
diff --git a/test/Bitcode/Inputs/invalid-GCTable-overflow.bc b/test/Bitcode/Inputs/invalid-GCTable-overflow.bc
new file mode 100644
index 00000000000..a9e9c52e1c0
Binary files /dev/null and b/test/Bitcode/Inputs/invalid-GCTable-overflow.bc differ
diff --git a/test/Bitcode/Inputs/invalid-abbrev-fixed-size-too-big.bc b/test/Bitcode/Inputs/invalid-abbrev-fixed-size-too-big.bc
new file mode 100644
index 00000000000..8fa28960d72
Binary files /dev/null and b/test/Bitcode/Inputs/invalid-abbrev-fixed-size-too-big.bc differ
diff --git a/test/Bitcode/Inputs/invalid-abbrev-no-operands.bc b/test/Bitcode/Inputs/invalid-abbrev-no-operands.bc
new file mode 100644
index 00000000000..29af122e94e
Binary files /dev/null and b/test/Bitcode/Inputs/invalid-abbrev-no-operands.bc differ
diff --git a/test/Bitcode/Inputs/invalid-abbrev-vbr-size-too-big.bc b/test/Bitcode/Inputs/invalid-abbrev-vbr-size-too-big.bc
new file mode 100644
index 00000000000..35d00ba154b
Binary files /dev/null and b/test/Bitcode/Inputs/invalid-abbrev-vbr-size-too-big.bc differ
diff --git a/test/Bitcode/Inputs/invalid-alias-type-mismatch.bc b/test/Bitcode/Inputs/invalid-alias-type-mismatch.bc
new file mode 100644
index 00000000000..5c429894480
Binary files /dev/null and b/test/Bitcode/Inputs/invalid-alias-type-mismatch.bc differ
diff --git a/test/Bitcode/Inputs/invalid-array-element-type.bc b/test/Bitcode/Inputs/invalid-array-element-type.bc
new file mode 100644
index 00000000000..3ce4ba2f77d
Binary files /dev/null and b/test/Bitcode/Inputs/invalid-array-element-type.bc differ
diff --git a/test/Bitcode/Inputs/invalid-array-op-not-2nd-to-last.bc b/test/Bitcode/Inputs/invalid-array-op-not-2nd-to-last.bc
new file mode 100644
index 00000000000..a4b94f8213d
Binary files /dev/null and b/test/Bitcode/Inputs/invalid-array-op-not-2nd-to-last.bc differ
diff --git a/test/Bitcode/Inputs/invalid-array-operand-encoding.bc b/test/Bitcode/Inputs/invalid-array-operand-encoding.bc
new file mode 100644
index 00000000000..f7ec2eb700a
Binary files /dev/null and b/test/Bitcode/Inputs/invalid-array-operand-encoding.bc differ
diff --git a/test/Bitcode/Inputs/invalid-array-type.bc b/test/Bitcode/Inputs/invalid-array-type.bc
new file mode 100644
index 00000000000..3a4b635dd0e
Binary files /dev/null and b/test/Bitcode/Inputs/invalid-array-type.bc differ
diff --git a/test/Bitcode/Inputs/invalid-call-mismatched-explicit-type.bc b/test/Bitcode/Inputs/invalid-call-mismatched-explicit-type.bc
new file mode 100644
index 00000000000..802d28edbf4
Binary files /dev/null and b/test/Bitcode/Inputs/invalid-call-mismatched-explicit-type.bc differ
diff --git a/test/Bitcode/Inputs/invalid-call-non-function-explicit-type.bc b/test/Bitcode/Inputs/invalid-call-non-function-explicit-type.bc
new file mode 100644
index 00000000000..d080689e567
Binary files /dev/null and b/test/Bitcode/Inputs/invalid-call-non-function-explicit-type.bc differ
diff --git a/test/Bitcode/Inputs/invalid-code-len-width.bc b/test/Bitcode/Inputs/invalid-code-len-width.bc
new file mode 100644
index 00000000000..c8e8c4f6653
Binary files /dev/null and b/test/Bitcode/Inputs/invalid-code-len-width.bc differ
diff --git a/test/Bitcode/Inputs/invalid-extract-0-indices.bc b/test/Bitcode/Inputs/invalid-extract-0-indices.bc
new file mode 100644
index 00000000000..bfde5264502
Binary files /dev/null and b/test/Bitcode/Inputs/invalid-extract-0-indices.bc differ
diff --git a/test/Bitcode/Inputs/invalid-fixme-streaming-blob.bc b/test/Bitcode/Inputs/invalid-fixme-streaming-blob.bc
new file mode 100644
index 00000000000..7e32f8b0774
Binary files /dev/null and b/test/Bitcode/Inputs/invalid-fixme-streaming-blob.bc differ
diff --git a/test/Bitcode/Inputs/invalid-fp-shift.bc b/test/Bitcode/Inputs/invalid-fp-shift.bc
new file mode 100644
index 00000000000..aeba1bff968
Binary files /dev/null and b/test/Bitcode/Inputs/invalid-fp-shift.bc differ
diff --git a/test/Bitcode/Inputs/invalid-function-argument-type.bc b/test/Bitcode/Inputs/invalid-function-argument-type.bc
new file mode 100644
index 00000000000..b00fb03cd31
Binary files /dev/null and b/test/Bitcode/Inputs/invalid-function-argument-type.bc differ
diff --git a/test/Bitcode/Inputs/invalid-function-comdat-id.bc b/test/Bitcode/Inputs/invalid-function-comdat-id.bc
new file mode 100644
index 00000000000..d0ad8234bc8
Binary files /dev/null and b/test/Bitcode/Inputs/invalid-function-comdat-id.bc differ
diff --git a/test/Bitcode/Inputs/invalid-fwdref-type-mismatch-2.bc b/test/Bitcode/Inputs/invalid-fwdref-type-mismatch-2.bc
new file mode 100644
index 00000000000..8e38340eede
Binary files /dev/null and b/test/Bitcode/Inputs/invalid-fwdref-type-mismatch-2.bc differ
diff --git a/test/Bitcode/Inputs/invalid-fwdref-type-mismatch.bc b/test/Bitcode/Inputs/invalid-fwdref-type-mismatch.bc
new file mode 100644
index 00000000000..d1c9560257a
Binary files /dev/null and b/test/Bitcode/Inputs/invalid-fwdref-type-mismatch.bc differ
diff --git a/test/Bitcode/Inputs/invalid-global-var-comdat-id.bc b/test/Bitcode/Inputs/invalid-global-var-comdat-id.bc
new file mode 100644
index 00000000000..93d6ba2169b
Binary files /dev/null and b/test/Bitcode/Inputs/invalid-global-var-comdat-id.bc differ
diff --git a/test/Bitcode/Inputs/invalid-insert-0-indices.bc b/test/Bitcode/Inputs/invalid-insert-0-indices.bc
new file mode 100644
index 00000000000..d7a91e1e448
Binary files /dev/null and b/test/Bitcode/Inputs/invalid-insert-0-indices.bc differ
diff --git a/test/Bitcode/Inputs/invalid-inserted-value-type-mismatch.bc b/test/Bitcode/Inputs/invalid-inserted-value-type-mismatch.bc
new file mode 100644
index 00000000000..81333cd6be6
Binary files /dev/null and b/test/Bitcode/Inputs/invalid-inserted-value-type-mismatch.bc differ
diff --git a/test/Bitcode/Inputs/invalid-invoke-mismatched-explicit-type.bc b/test/Bitcode/Inputs/invalid-invoke-mismatched-explicit-type.bc
new file mode 100644
index 00000000000..bce88529508
Binary files /dev/null and b/test/Bitcode/Inputs/invalid-invoke-mismatched-explicit-type.bc differ
diff --git a/test/Bitcode/Inputs/invalid-invoke-non-function-explicit-type.bc b/test/Bitcode/Inputs/invalid-invoke-non-function-explicit-type.bc
new file mode 100644
index 00000000000..d7810bb964b
Binary files /dev/null and b/test/Bitcode/Inputs/invalid-invoke-non-function-explicit-type.bc differ
diff --git a/test/Bitcode/Inputs/invalid-load-pointer-type.bc b/test/Bitcode/Inputs/invalid-load-pointer-type.bc
new file mode 100644
index 00000000000..b6a56c55b3b
Binary files /dev/null and b/test/Bitcode/Inputs/invalid-load-pointer-type.bc differ
diff --git a/test/Bitcode/Inputs/invalid-load-ptr-type.bc b/test/Bitcode/Inputs/invalid-load-ptr-type.bc
new file mode 100644
index 00000000000..5207ed9156e
Binary files /dev/null and b/test/Bitcode/Inputs/invalid-load-ptr-type.bc differ
diff --git a/test/Bitcode/Inputs/invalid-metadata-not-followed-named-node.bc b/test/Bitcode/Inputs/invalid-metadata-not-followed-named-node.bc
new file mode 100644
index 00000000000..42a2c3e65fe
Binary files /dev/null and b/test/Bitcode/Inputs/invalid-metadata-not-followed-named-node.bc differ
diff --git a/test/Bitcode/Inputs/invalid-non-vector-extractelement.bc b/test/Bitcode/Inputs/invalid-non-vector-extractelement.bc
new file mode 100644
index 00000000000..6fee7edad87
Binary files /dev/null and b/test/Bitcode/Inputs/invalid-non-vector-extractelement.bc differ
diff --git a/test/Bitcode/Inputs/invalid-non-vector-insertelement.bc b/test/Bitcode/Inputs/invalid-non-vector-insertelement.bc
new file mode 100644
index 00000000000..36271657791
Binary files /dev/null and b/test/Bitcode/Inputs/invalid-non-vector-insertelement.bc differ
diff --git a/test/Bitcode/Inputs/invalid-non-vector-shufflevector.bc b/test/Bitcode/Inputs/invalid-non-vector-shufflevector.bc
new file mode 100644
index 00000000000..6c83a4dcb76
Binary files /dev/null and b/test/Bitcode/Inputs/invalid-non-vector-shufflevector.bc differ
diff --git a/test/Bitcode/Inputs/invalid-pointer-element-type.bc b/test/Bitcode/Inputs/invalid-pointer-element-type.bc
new file mode 100644
index 00000000000..f9649e66429
Binary files /dev/null and b/test/Bitcode/Inputs/invalid-pointer-element-type.bc differ
diff --git a/test/Bitcode/Inputs/invalid-too-big-fwdref.bc b/test/Bitcode/Inputs/invalid-too-big-fwdref.bc
new file mode 100644
index 00000000000..d1d51a634fc
Binary files /dev/null and b/test/Bitcode/Inputs/invalid-too-big-fwdref.bc differ
diff --git a/test/Bitcode/Inputs/invalid-vector-element-type.bc b/test/Bitcode/Inputs/invalid-vector-element-type.bc
new file mode 100644
index 00000000000..9c6c625c918
Binary files /dev/null and b/test/Bitcode/Inputs/invalid-vector-element-type.bc differ
diff --git a/test/Bitcode/Inputs/invalid-vector-length.bc b/test/Bitcode/Inputs/invalid-vector-length.bc
new file mode 100644
index 00000000000..269df83a07a
Binary files /dev/null and b/test/Bitcode/Inputs/invalid-vector-length.bc differ
diff --git a/test/Bitcode/Inputs/padding-garbage.bc b/test/Bitcode/Inputs/padding-garbage.bc
new file mode 100644
index 00000000000..b44b584f4dc
Binary files /dev/null and b/test/Bitcode/Inputs/padding-garbage.bc differ
diff --git a/test/Bitcode/Inputs/padding.bc b/test/Bitcode/Inputs/padding.bc
new file mode 100644
index 00000000000..7654a3e5da2
Binary files /dev/null and b/test/Bitcode/Inputs/padding.bc differ
diff --git a/test/Bitcode/PR23310.test b/test/Bitcode/PR23310.test
new file mode 100644
index 00000000000..6b794716c3f
--- /dev/null
+++ b/test/Bitcode/PR23310.test
@@ -0,0 +1 @@
+RUN: llvm-dis -disable-output %p/Inputs/PR23310.bc
diff --git a/test/Bitcode/attributes.ll b/test/Bitcode/attributes.ll
index 9fbdfeb0ef0..a0bc66642f7 100644
--- a/test/Bitcode/attributes.ll
+++ b/test/Bitcode/attributes.ll
@@ -204,7 +204,7 @@ define void @f34()
 ; CHECK: define void @f34()
 {
         call void @nobuiltin() nobuiltin
-; CHECK: call void @nobuiltin() #25
+; CHECK: call void @nobuiltin() #27
         ret void;
 }
 
@@ -251,6 +251,17 @@ define dereferenceable_or_null(8) i8* @f42(i8* dereferenceable_or_null(8) %foo)
   ret i8* %foo
 }
 
+; CHECK: define void @f43() #25
+define void @f43() convergent {
+  ret void
+}
+
+define void @f44() argmemonly
+; CHECK: define void @f44() #26
+{
+        ret void;
+}
+
 ; CHECK: attributes #0 = { noreturn }
 ; CHECK: attributes #1 = { nounwind }
 ; CHECK: attributes #2 = { readnone }
@@ -276,4 +287,6 @@ define dereferenceable_or_null(8) i8* @f42(i8* dereferenceable_or_null(8) %foo)
 ; CHECK: attributes #22 = { minsize }
 ; CHECK: attributes #23 = { noinline optnone }
 ; CHECK: attributes #24 = { jumptable }
-; CHECK: attributes #25 = { nobuiltin }
+; CHECK: attributes #25 = { convergent }
+; CHECK: attributes #26 = { argmemonly }
+; CHECK: attributes #27 = { nobuiltin }
diff --git a/test/Bitcode/cmpxchg.3.6.ll b/test/Bitcode/cmpxchg.3.6.ll
new file mode 100644
index 00000000000..bec51a128c3
--- /dev/null
+++ b/test/Bitcode/cmpxchg.3.6.ll
@@ -0,0 +1,13 @@
+; RUN: llvm-dis < %s.bc | FileCheck %s
+
+define void @f2(i32* %x, i32 %y.orig, i32 %z) {
+entry:
+  br label %a
+b:
+  cmpxchg i32* %x, i32 %y, i32 %z acquire acquire
+; CHECK: cmpxchg i32* %x, i32 %y, i32 %z acquire acquire
+  ret void
+a:
+  %y = add i32 %y.orig, 1
+  br label %a
+}
diff --git a/test/Bitcode/cmpxchg.3.6.ll.bc b/test/Bitcode/cmpxchg.3.6.ll.bc
new file mode 100644
index 00000000000..1c66f934324
Binary files /dev/null and b/test/Bitcode/cmpxchg.3.6.ll.bc differ
diff --git a/test/Bitcode/debug-loc-again.ll b/test/Bitcode/debug-loc-again.ll
new file mode 100644
index 00000000000..6dbea16121a
--- /dev/null
+++ b/test/Bitcode/debug-loc-again.ll
@@ -0,0 +1,36 @@
+; RUN: llvm-as < %s | llvm-bcanalyzer -dump | FileCheck %s -check-prefix=BC
+; PR23436: Actually emit DEBUG_LOC_AGAIN records.
+
+; BC: <DEBUG_LOC op
+; BC: <DEBUG_LOC_AGAIN/>
+; BC: <DEBUG_LOC op
+; BC: <DEBUG_LOC_AGAIN/>
+
+; RUN: llvm-as < %s | llvm-dis | FileCheck %s
+; RUN: verify-uselistorder %s
+; Check that this round-trips correctly.
+
+define void @foo() {
+entry:
+  %a = add i32 0, 0, !dbg !3
+  %b = add i32 0, 1, !dbg !3
+  %c = add i32 0, 2, !dbg !4
+  ret void, !dbg !4
+}
+
+; CHECK-LABEL: entry:
+; CHECK-NEXT: %a = add i32 0, 0, !dbg ![[LINE1:[0-9]+]]
+; CHECK-NEXT: %b = add i32 0, 1, !dbg ![[LINE1]]
+; CHECK-NEXT: %c = add i32 0, 2, !dbg ![[LINE2:[0-9]+]]
+; CHECK-NEXT: ret void, !dbg ![[LINE2]]
+; CHECK: ![[LINE1]] = !DILocation(line: 1,
+; CHECK: ![[LINE2]] = !DILocation(line: 2,
+
+!llvm.module.flags = !{!0}
+
+!0 = !{i32 2, !"Debug Info Version", i32 3}
+!1 = !DICompileUnit(language: DW_LANG_C99, file: !DIFile(filename: "f", directory: "/d"),
+                    subprograms: !{!2})
+!2 = !DISubprogram(name: "foo")
+!3 = !DILocation(line: 1, scope: !2)
+!4 = !DILocation(line: 2, scope: !2)
diff --git a/test/Bitcode/fcmp-fast.ll b/test/Bitcode/fcmp-fast.ll
new file mode 100644
index 00000000000..126e3652e3a
--- /dev/null
+++ b/test/Bitcode/fcmp-fast.ll
@@ -0,0 +1,23 @@
+; RUN: llvm-as < %s | llvm-dis > %t0
+; RUN: opt -S < %s > %t1
+; RUN: diff %t0 %t1
+; RUN: FileCheck < %t1 %s
+
+; Make sure flags on fcmp instructions are serialized/deserialized properly.
+
+define i1 @foo(float %a, float %b, double %c, double %d) {
+  ; CHECK:   %plain = fcmp ueq float %a, %b
+  %plain = fcmp ueq float %a, %b
+  ; CHECK:   %fast = fcmp fast olt float %a, %b
+  %fast = fcmp fast olt float %a, %b
+  ; CHECK:   %nsz = fcmp nsz uge float %a, %b
+  %nsz = fcmp nsz uge float %a, %b
+  ; CHECK:   %nnan = fcmp nnan nsz oge double %c, %d
+  %nnan = fcmp nnan nsz oge double %c, %d
+
+  %dce1 = or i1 %plain, %fast
+  %dce2 = or i1 %dce1, %nsz
+  %dce3 = or i1 %dce2, %nnan
+
+  ret i1 %dce3
+}
diff --git a/test/Bitcode/invalid.test b/test/Bitcode/invalid.test
index 9cab227ab19..0aab553bb61 100644
--- a/test/Bitcode/invalid.test
+++ b/test/Bitcode/invalid.test
@@ -18,6 +18,14 @@ RUN: not llvm-dis -disable-output %p/Inputs/invalid-load-mismatched-explicit-typ
 RUN:   FileCheck --check-prefix=MISMATCHED-EXPLICIT-LOAD %s
 RUN: not llvm-dis -disable-output %p/Inputs/invalid-gep-operator-mismatched-explicit-type.bc 2>&1 | \
 RUN:   FileCheck --check-prefix=MISMATCHED-EXPLICIT-GEP-OPERATOR %s
+RUN: not llvm-dis -disable-output %p/Inputs/invalid-call-mismatched-explicit-type.bc 2>&1 | \
+RUN:   FileCheck --check-prefix=MISMATCHED-EXPLICIT-CALL %s
+RUN: not llvm-dis -disable-output %p/Inputs/invalid-call-non-function-explicit-type.bc 2>&1 | \
+RUN:   FileCheck --check-prefix=NON-FUNCTION-EXPLICIT-CALL %s
+RUN: not llvm-dis -disable-output %p/Inputs/invalid-invoke-mismatched-explicit-type.bc 2>&1 | \
+RUN:   FileCheck --check-prefix=MISMATCHED-EXPLICIT-INVOKE %s
+RUN: not llvm-dis -disable-output %p/Inputs/invalid-invoke-non-function-explicit-type.bc 2>&1 | \
+RUN:   FileCheck --check-prefix=NON-FUNCTION-EXPLICIT-INVOKE %s
 
 INVALID-ENCODING: Invalid encoding
 BAD-ABBREV: Abbreviation starts with an Array or a Blob
@@ -27,8 +35,12 @@ BAD-TYPE-TABLE-FORWARD-REF: Invalid TYPE table: Only named structs can be forwar
 BAD-BITWIDTH: Bitwidth for integer type out of range
 BAD-ALIGN: Invalid alignment value
 MISMATCHED-EXPLICIT-GEP: Explicit gep type does not match pointee type of pointer operand
-MISMATCHED-EXPLICIT-LOAD: Explicit load type does not match pointee type of pointer operand
+MISMATCHED-EXPLICIT-LOAD: Explicit load/store type does not match pointee type of pointer operand
 MISMATCHED-EXPLICIT-GEP-OPERATOR: Explicit gep operator type does not match pointee type of pointer operand
+MISMATCHED-EXPLICIT-CALL: Explicit call type does not match pointee type of callee operand
+NON-FUNCTION-EXPLICIT-CALL: Explicit call type is not a function type
+MISMATCHED-EXPLICIT-INVOKE: Explicit invoke type does not match pointee type of callee operand
+NON-FUNCTION-EXPLICIT-INVOKE: Explicit invoke type is not a function type
 
 RUN: not llvm-dis -disable-output %p/Inputs/invalid-extractval-array-idx.bc 2>&1 | \
 RUN:   FileCheck --check-prefix=EXTRACT-ARRAY %s
@@ -55,3 +67,138 @@ RUN: not llvm-dis -disable-output %p/Inputs/invalid-no-proper-module.bc 2>&1 | \
 RUN:   FileCheck --check-prefix=NO-MODULE %s
 
 NO-MODULE: Malformed IR file
+
+RUN: not llvm-dis -disable-output %p/Inputs/invalid-fp-shift.bc 2>&1 | \
+RUN:   FileCheck --check-prefix=FP-SHIFT %s
+
+FP-SHIFT: Invalid record
+
+RUN: not llvm-dis -disable-output %p/Inputs/invalid-abbrev-vbr-size-too-big.bc 2>&1 | \
+RUN:   FileCheck --check-prefix=HUGE-ABBREV-OP %s
+RUN: not llvm-dis -disable-output %p/Inputs/invalid-abbrev-fixed-size-too-big.bc 2>&1 | \
+RUN:   FileCheck --check-prefix=HUGE-ABBREV-OP %s
+
+HUGE-ABBREV-OP: Fixed or VBR abbrev record with size > MaxChunkData
+
+RUN: not llvm-dis -disable-output %p/Inputs/invalid-array-type.bc 2>&1 | \
+RUN:   FileCheck --check-prefix=ARRAY-TYPE %s
+
+ARRAY-TYPE: Array element type can't be an Array or a Blob
+
+RUN: not llvm-dis -disable-output %p/Inputs/invalid-non-vector-extractelement.bc 2>&1 | \
+RUN:   FileCheck --check-prefix=INVALID-TYPE %s
+RUN: not llvm-dis -disable-output %p/Inputs/invalid-non-vector-insertelement.bc 2>&1 | \
+RUN:   FileCheck --check-prefix=INVALID-TYPE %s
+RUN: not llvm-dis -disable-output %p/Inputs/invalid-non-vector-shufflevector.bc 2>&1 | \
+RUN:   FileCheck --check-prefix=INVALID-TYPE %s
+
+INVALID-TYPE: Invalid type for value
+
+RUN: not llvm-dis -disable-output %p/Inputs/invalid-fwdref-type-mismatch.bc 2>&1 | \
+RUN:   FileCheck --check-prefix=FWDREF-TYPE %s
+
+FWDREF-TYPE: Invalid record
+
+RUN: not llvm-dis -disable-output %p/Inputs/invalid-fwdref-type-mismatch-2.bc 2>&1 | \
+RUN:   FileCheck --check-prefix=FWDREF-TYPE-MISMATCH %s
+
+FWDREF-TYPE-MISMATCH: Type mismatch in constant table!
+
+RUN: not llvm-dis -disable-output %p/Inputs/invalid-array-element-type.bc 2>&1 | \
+RUN:   FileCheck --check-prefix=ELEMENT-TYPE %s
+RUN: not llvm-dis -disable-output %p/Inputs/invalid-vector-element-type.bc 2>&1 | \
+RUN:   FileCheck --check-prefix=ELEMENT-TYPE %s
+RUN: not llvm-dis -disable-output %p/Inputs/invalid-pointer-element-type.bc 2>&1 | \
+RUN:   FileCheck --check-prefix=ELEMENT-TYPE %s
+
+ELEMENT-TYPE: Invalid type
+
+RUN: not llvm-dis -disable-output %p/Inputs/invalid-array-op-not-2nd-to-last.bc 2>&1 | \
+RUN:   FileCheck --check-prefix=ARRAY-NOT-2LAST %s
+
+ARRAY-NOT-2LAST: Array op not second to last
+
+RUN: not llvm-dis -disable-output %p/Inputs/invalid-too-big-fwdref.bc 2>&1 | \
+RUN:   FileCheck --check-prefix=HUGE-FWDREF %s
+
+HUGE-FWDREF: Invalid record
+
+RUN: not llvm-dis -disable-output %p/Inputs/invalid-load-pointer-type.bc 2>&1 | \
+RUN:   FileCheck --check-prefix=LOAD-BAD-TYPE %s
+
+LOAD-BAD-TYPE: Load/Store operand is not a pointer type
+
+RUN: not llvm-dis -disable-output %p/Inputs/invalid-GCTable-overflow.bc 2>&1 | \
+RUN:   FileCheck --check-prefix=GCTABLE-OFLOW %s
+
+GCTABLE-OFLOW: Invalid ID
+
+RUN: not llvm-dis -disable-output %p/Inputs/invalid-insert-0-indices.bc 2>&1 | \
+RUN:   FileCheck --check-prefix=INSERT-0-IDXS %s
+
+INSERT-0-IDXS: INSERTVAL: Invalid instruction with 0 indices
+
+RUN: not llvm-dis -disable-output %p/Inputs/invalid-extract-0-indices.bc 2>&1 | \
+RUN:   FileCheck --check-prefix=EXTRACT-0-IDXS %s
+
+EXTRACT-0-IDXS: EXTRACTVAL: Invalid instruction with 0 indices
+
+RUN: not llvm-dis -disable-output %p/Inputs/invalid-load-ptr-type.bc 2>&1 | \
+RUN:   FileCheck --check-prefix=BAD-LOAD-PTR-TYPE %s
+
+BAD-LOAD-PTR-TYPE: Cannot load/store from pointer
+
+RUN: not llvm-dis -disable-output %p/Inputs/invalid-inserted-value-type-mismatch.bc 2>&1 | \
+RUN:   FileCheck --check-prefix=INSERT-TYPE-MISMATCH %s
+
+INSERT-TYPE-MISMATCH: Inserted value type doesn't match aggregate type
+
+RUN: not llvm-dis -disable-output %p/Inputs/invalid-code-len-width.bc 2>&1 | \
+RUN:   FileCheck --check-prefix=INVALID-CODELENWIDTH %s
+
+INVALID-CODELENWIDTH: Malformed block
+
+RUN: not llvm-dis -disable-output %p/Inputs/invalid-function-argument-type.bc 2>&1 | \
+RUN:   FileCheck --check-prefix=INVALID-ARGUMENT-TYPE %s
+
+INVALID-ARGUMENT-TYPE: Invalid function argument type
+
+RUN: not llvm-dis -disable-output %p/Inputs/invalid-fixme-streaming-blob.bc 2>&1 | \
+RUN:   FileCheck --check-prefix=STREAMING-BLOB %s
+
+STREAMING-BLOB: getPointer in streaming memory objects not allowed
+
+RUN: not llvm-dis -disable-output %p/Inputs/invalid-function-comdat-id.bc 2>&1 | \
+RUN:   FileCheck --check-prefix=INVALID-FCOMDAT-ID %s
+
+INVALID-FCOMDAT-ID: Invalid function comdat ID
+
+RUN: not llvm-dis -disable-output %p/Inputs/invalid-global-var-comdat-id.bc 2>&1 | \
+RUN:   FileCheck --check-prefix=INVALID-GVCOMDAT-ID %s
+
+INVALID-GVCOMDAT-ID: Invalid global variable comdat ID
+
+RUN: not llvm-dis -disable-output %p/Inputs/invalid-abbrev-no-operands.bc 2>&1 | \
+RUN:   FileCheck --check-prefix=ABBREV-NO-OPS %s
+
+ABBREV-NO-OPS: Abbrev record with no operands
+
+RUN: not llvm-dis -disable-output %p/Inputs/invalid-array-operand-encoding.bc 2>&1 | \
+RUN:   FileCheck --check-prefix=ARRAY-OP-ENC %s
+
+ARRAY-OP-ENC: Array element type has to be an encoding of a type
+
+RUN: not llvm-dis -disable-output %p/Inputs/invalid-metadata-not-followed-named-node.bc 2>&1 | \
+RUN:   FileCheck --check-prefix=META-NOT-FOLLOWED-BY-NAMED-META %s
+
+META-NOT-FOLLOWED-BY-NAMED-META: METADATA_NAME not followed by METADATA_NAMED_NODE
+
+RUN: not llvm-dis -disable-output %p/Inputs/invalid-vector-length.bc 2>&1 | \
+RUN:   FileCheck --check-prefix=VECTOR-LENGTH %s
+
+VECTOR-LENGTH: Invalid vector length
+
+RUN: not llvm-dis -disable-output %p/Inputs/invalid-alias-type-mismatch.bc 2>&1 | \
+RUN:   FileCheck --check-prefix=ALIAS-TYPE-MISMATCH %s
+
+ALIAS-TYPE-MISMATCH: Alias and aliasee types don't match
diff --git a/test/Bitcode/miscInstructions.3.2.ll b/test/Bitcode/miscInstructions.3.2.ll
index bed26c22147..6b899510726 100644
--- a/test/Bitcode/miscInstructions.3.2.ll
+++ b/test/Bitcode/miscInstructions.3.2.ll
@@ -13,27 +13,33 @@ entry:
   ret i32 0
 }
 
+; CHECK-LABEL: define void @landingpadInstr1
+; CHECK-SAME: personality i32 (...)* @__gxx_personality_v0
 define void @landingpadInstr1(i1 %cond1, <2 x i1> %cond2, <2 x i8> %x1, <2 x i8> %x2){
 entry:
-; CHECK: %res = landingpad { i8*, i32 } personality i32 (...)* @__gxx_personality_v0
+; CHECK: %res = landingpad { i8*, i32 }
   %res = landingpad { i8*, i32 } personality i32 (...)* @__gxx_personality_v0 
 ; CHECK: catch i8** @_ZTIi
   catch i8** @_ZTIi
   ret void
 }
 
+; CHECK-LABEL: define void @landingpadInstr2
+; CHECK-SAME: personality i32 (...)* @__gxx_personality_v0
 define void @landingpadInstr2(i1 %cond1, <2 x i1> %cond2, <2 x i8> %x1, <2 x i8> %x2){
 entry:
-; CHECK: %res = landingpad { i8*, i32 } personality i32 (...)* @__gxx_personality_v0
+; CHECK: %res = landingpad { i8*, i32 }
   %res = landingpad { i8*, i32 } personality i32 (...)* @__gxx_personality_v0
 ; CHECK: cleanup
   cleanup
   ret void
 }
 
+; CHECK-LABEL: define void @landingpadInstr3
+; CHECK-SAME: personality i32 (...)* @__gxx_personality_v0
 define void @landingpadInstr3(i1 %cond1, <2 x i1> %cond2, <2 x i8> %x1, <2 x i8> %x2){
 entry:
-; CHECK: %res = landingpad { i8*, i32 } personality i32 (...)* @__gxx_personality_v0 
+; CHECK: %res = landingpad { i8*, i32 }
   %res = landingpad { i8*, i32 } personality i32 (...)* @__gxx_personality_v0
 ; CHECK: catch i8** @_ZTIi
   catch i8** @_ZTIi
diff --git a/test/Bitcode/padding.test b/test/Bitcode/padding.test
new file mode 100644
index 00000000000..63b2fec2b59
--- /dev/null
+++ b/test/Bitcode/padding.test
@@ -0,0 +1,18 @@
+Test that both llvm-dis (uses a data streamer) and opt (no data streamer)
+handle a .bc file with any padding.
+
+A file padded with '\n' can be produced under a peculiar situation:
+
+* A .bc is produced os OS X, but without a darwin triple, so it has no
+  wrapper.
+* It is included in a .a archive
+* ranlib is ran on that archive. It will pad the members to make them multiple
+  of 8 bytes.
+
+and there is no reason to not handle the general case.
+
+RUN: llvm-dis -disable-output %p/Inputs/padding.bc
+RUN: opt -disable-output %p/Inputs/padding.bc
+
+RUN: llvm-dis -disable-output %p/Inputs/padding-garbage.bc
+RUN: opt -disable-output %p/Inputs/padding-garbage.bc
diff --git a/test/BugPoint/metadata.ll b/test/BugPoint/metadata.ll
index c706d9036d7..d8aca524ee2 100644
--- a/test/BugPoint/metadata.ll
+++ b/test/BugPoint/metadata.ll
@@ -5,10 +5,10 @@
 ; Bugpoint should keep the call's metadata attached to the call.
 
 ; CHECK: call void @foo(), !dbg ![[LOC:[0-9]+]], !attach ![[CALL:[0-9]+]]
-; CHECK: ![[LOC]] = !MDLocation(line: 104, column: 105, scope: ![[SCOPE:[0-9]+]])
-; CHECK: ![[SCOPE]] = !MDSubprogram(name: "test"
+; CHECK: ![[LOC]] = !DILocation(line: 104, column: 105, scope: ![[SCOPE:[0-9]+]])
+; CHECK: ![[SCOPE]] = !DISubprogram(name: "test"
 ; CHECK-SAME:                       file: ![[FILE:[0-9]+]]
-; CHECK: ![[FILE]] = !MDFile(filename: "source.c", directory: "/dir")
+; CHECK: ![[FILE]] = !DIFile(filename: "source.c", directory: "/dir")
 ; CHECK: ![[CALL]] = !{!"the call to foo"}
 
 %rust_task = type {}
@@ -31,12 +31,12 @@ declare void @foo()
 !3 = !{!"noise"}
 !4 = !{!"filler"}
 
-!9 = !MDSubprogram(name: "test", file: !15)
-!10 = !MDLocation(line: 100, column: 101, scope: !9)
-!11 = !MDLocation(line: 102, column: 103, scope: !9)
-!12 = !MDLocation(line: 104, column: 105, scope: !9)
-!13 = !MDLocation(line: 106, column: 107, scope: !9)
-!14 = !MDLocation(line: 108, column: 109, scope: !9)
-!15 = !MDFile(filename: "source.c", directory: "/dir")
+!9 = !DISubprogram(name: "test", file: !15)
+!10 = !DILocation(line: 100, column: 101, scope: !9)
+!11 = !DILocation(line: 102, column: 103, scope: !9)
+!12 = !DILocation(line: 104, column: 105, scope: !9)
+!13 = !DILocation(line: 106, column: 107, scope: !9)
+!14 = !DILocation(line: 108, column: 109, scope: !9)
+!15 = !DIFile(filename: "source.c", directory: "/dir")
 !16 = !{}
 !17 = !{i32 1, !"Debug Info Version", i32 3}
diff --git a/test/BugPoint/remove_arguments_test.ll b/test/BugPoint/remove_arguments_test.ll
index 29a03b83107..fee514e0c40 100644
--- a/test/BugPoint/remove_arguments_test.ll
+++ b/test/BugPoint/remove_arguments_test.ll
@@ -2,8 +2,8 @@
 ; RUN: llvm-dis %t-reduced-simplified.bc -o - | FileCheck %s
 ; REQUIRES: loadable_module
 
-; Test to make sure that arguments are removed from the function if they are 
-; unnecessary. And clean up any types that that frees up too.
+; Test to make sure that arguments are removed from the function if they are
+; unnecessary. And clean up any types that frees up too.
 
 ; CHECK: target triple
 ; CHECK-NOT: struct.anon
diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt
index dc13f7937a7..f49df542f4e 100644
--- a/test/CMakeLists.txt
+++ b/test/CMakeLists.txt
@@ -34,6 +34,7 @@ set(LLVM_TEST_DEPENDS
           llvm-dsymutil
           llvm-dwarfdump
           llvm-extract
+          llvm-lib
           llvm-link
           llvm-lto
           llvm-mc
@@ -49,14 +50,10 @@ set(LLVM_TEST_DEPENDS
           llvm-tblgen
           macho-dump
           opt
-          pnacl-bcanalyzer
-          pnacl-bcdis
-          pnacl-llc
-          pnacl-freeze
-          pnacl-thaw
           FileCheck
           count
           not
+          yaml-bench
           yaml2obj
           obj2yaml
           verify-uselistorder
diff --git a/test/CodeGen/AArch64/aarch-multipart.ll b/test/CodeGen/AArch64/aarch-multipart.ll
new file mode 100644
index 00000000000..fd42d6e8cd8
--- /dev/null
+++ b/test/CodeGen/AArch64/aarch-multipart.ll
@@ -0,0 +1,18 @@
+; RUN: llc < %s -o - | FileCheck %s
+
+target datalayout = "e-m:o-i64:64-i128:128-n32:64-S128"
+target triple = "arm64-apple-os"
+
+declare <4 x double> @user_func(<4 x double>) #1
+
+; Make sure we are not crashing on this code.
+; CHECK-LABEL: caller_function
+; CHECK: ret
+define void @caller_function(<4 x double>, <4 x double>, <4 x double>, <4 x double>, <4 x double>) #1 {
+entry:
+  %r = call <4 x double> @user_func(<4 x double> %4)
+  ret void
+}
+
+attributes #1 = { nounwind readnone }
+
diff --git a/test/CodeGen/AArch64/aarch64-2014-08-11-MachineCombinerCrash.ll b/test/CodeGen/AArch64/aarch64-2014-08-11-MachineCombinerCrash.ll
index 8d606708cc2..b075573cc67 100644
--- a/test/CodeGen/AArch64/aarch64-2014-08-11-MachineCombinerCrash.ll
+++ b/test/CodeGen/AArch64/aarch64-2014-08-11-MachineCombinerCrash.ll
@@ -16,7 +16,7 @@ for.body:                                         ; preds = %for.body, %entry
   %add53 = add nsw i64 %n1, 0, !dbg !52
   %add55 = add nsw i64 %n1, 0, !dbg !53
   %mul63 = mul nsw i64 %add53, -20995, !dbg !54
-  tail call void @llvm.dbg.value(metadata i64 %mul63, i64 0, metadata !30, metadata !MDExpression()), !dbg !55
+  tail call void @llvm.dbg.value(metadata i64 %mul63, i64 0, metadata !30, metadata !DIExpression()), !dbg !55
   %mul65 = mul nsw i64 %add55, -3196, !dbg !56
   %add67 = add nsw i64 0, %mul65, !dbg !57
   %add80 = add i64 0, 1024, !dbg !58
@@ -44,60 +44,60 @@ attributes #1 = { nounwind readnone }
 !llvm.module.flags = !{!36, !37}
 !llvm.ident = !{!38}
 
-!0 = !MDCompileUnit(language: DW_LANG_C99, producer: "clang version 3.6.0 ", isOptimized: true, emissionKind: 1, file: !1, enums: !2, retainedTypes: !2, subprograms: !3, globals: !2, imports: !2)
-!1 = !MDFile(filename: "test.c", directory: "")
+!0 = !DICompileUnit(language: DW_LANG_C99, producer: "clang version 3.6.0 ", isOptimized: true, emissionKind: 1, file: !1, enums: !2, retainedTypes: !2, subprograms: !3, globals: !2, imports: !2)
+!1 = !DIFile(filename: "test.c", directory: "")
 !2 = !{}
 !3 = !{!4}
-!4 = !MDSubprogram(name: "", line: 140, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: true, scopeLine: 141, file: !1, scope: !1, type: !6, function: void ()* @test, variables: !12)
-!6 = !MDSubroutineType(types: !7)
+!4 = !DISubprogram(name: "", line: 140, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: true, scopeLine: 141, file: !1, scope: !1, type: !6, function: void ()* @test, variables: !12)
+!6 = !DISubroutineType(types: !7)
 !7 = !{null, !8}
-!8 = !MDDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, baseType: !9)
-!9 = !MDDerivedType(tag: DW_TAG_typedef, line: 30, file: !1, baseType: !11)
-!11 = !MDBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
+!8 = !DIDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, baseType: !9)
+!9 = !DIDerivedType(tag: DW_TAG_typedef, line: 30, file: !1, baseType: !11)
+!11 = !DIBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
 !12 = !{!13, !14, !18, !19, !20, !21, !22, !23, !24, !25, !26, !27, !28, !29, !30, !31, !32, !33, !34, !35}
-!13 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "", line: 140, arg: 1, scope: !4, file: !1, type: !8)
-!14 = !MDLocalVariable(tag: DW_TAG_auto_variable, name: "", line: 142, scope: !4, file: !1, type: !15)
-!15 = !MDDerivedType(tag: DW_TAG_typedef, line: 183, file: !1, baseType: !17)
-!17 = !MDBasicType(tag: DW_TAG_base_type, size: 64, align: 64, encoding: DW_ATE_signed)
-!18 = !MDLocalVariable(tag: DW_TAG_auto_variable, name: "", line: 142, scope: !4, file: !1, type: !15)
-!19 = !MDLocalVariable(tag: DW_TAG_auto_variable, name: "", line: 142, scope: !4, file: !1, type: !15)
-!20 = !MDLocalVariable(tag: DW_TAG_auto_variable, name: "", line: 142, scope: !4, file: !1, type: !15)
-!21 = !MDLocalVariable(tag: DW_TAG_auto_variable, name: "", line: 142, scope: !4, file: !1, type: !15)
-!22 = !MDLocalVariable(tag: DW_TAG_auto_variable, name: "", line: 142, scope: !4, file: !1, type: !15)
-!23 = !MDLocalVariable(tag: DW_TAG_auto_variable, name: "", line: 142, scope: !4, file: !1, type: !15)
-!24 = !MDLocalVariable(tag: DW_TAG_auto_variable, name: "", line: 142, scope: !4, file: !1, type: !15)
-!25 = !MDLocalVariable(tag: DW_TAG_auto_variable, name: "", line: 143, scope: !4, file: !1, type: !15)
-!26 = !MDLocalVariable(tag: DW_TAG_auto_variable, name: "", line: 143, scope: !4, file: !1, type: !15)
-!27 = !MDLocalVariable(tag: DW_TAG_auto_variable, name: "", line: 143, scope: !4, file: !1, type: !15)
-!28 = !MDLocalVariable(tag: DW_TAG_auto_variable, name: "", line: 143, scope: !4, file: !1, type: !15)
-!29 = !MDLocalVariable(tag: DW_TAG_auto_variable, name: "", line: 144, scope: !4, file: !1, type: !15)
-!30 = !MDLocalVariable(tag: DW_TAG_auto_variable, name: "", line: 144, scope: !4, file: !1, type: !15)
-!31 = !MDLocalVariable(tag: DW_TAG_auto_variable, name: "", line: 144, scope: !4, file: !1, type: !15)
-!32 = !MDLocalVariable(tag: DW_TAG_auto_variable, name: "", line: 144, scope: !4, file: !1, type: !15)
-!33 = !MDLocalVariable(tag: DW_TAG_auto_variable, name: "", line: 144, scope: !4, file: !1, type: !15)
-!34 = !MDLocalVariable(tag: DW_TAG_auto_variable, name: "", line: 145, scope: !4, file: !1, type: !8)
-!35 = !MDLocalVariable(tag: DW_TAG_auto_variable, name: "", line: 146, scope: !4, file: !1, type: !11)
+!13 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "", line: 140, arg: 1, scope: !4, file: !1, type: !8)
+!14 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "", line: 142, scope: !4, file: !1, type: !15)
+!15 = !DIDerivedType(tag: DW_TAG_typedef, line: 183, file: !1, baseType: !17)
+!17 = !DIBasicType(tag: DW_TAG_base_type, size: 64, align: 64, encoding: DW_ATE_signed)
+!18 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "", line: 142, scope: !4, file: !1, type: !15)
+!19 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "", line: 142, scope: !4, file: !1, type: !15)
+!20 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "", line: 142, scope: !4, file: !1, type: !15)
+!21 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "", line: 142, scope: !4, file: !1, type: !15)
+!22 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "", line: 142, scope: !4, file: !1, type: !15)
+!23 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "", line: 142, scope: !4, file: !1, type: !15)
+!24 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "", line: 142, scope: !4, file: !1, type: !15)
+!25 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "", line: 143, scope: !4, file: !1, type: !15)
+!26 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "", line: 143, scope: !4, file: !1, type: !15)
+!27 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "", line: 143, scope: !4, file: !1, type: !15)
+!28 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "", line: 143, scope: !4, file: !1, type: !15)
+!29 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "", line: 144, scope: !4, file: !1, type: !15)
+!30 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "", line: 144, scope: !4, file: !1, type: !15)
+!31 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "", line: 144, scope: !4, file: !1, type: !15)
+!32 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "", line: 144, scope: !4, file: !1, type: !15)
+!33 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "", line: 144, scope: !4, file: !1, type: !15)
+!34 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "", line: 145, scope: !4, file: !1, type: !8)
+!35 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "", line: 146, scope: !4, file: !1, type: !11)
 !36 = !{i32 2, !"Dwarf Version", i32 4}
 !37 = !{i32 2, !"Debug Info Version", i32 3}
 !38 = !{!"clang version 3.6.0 "}
-!39 = !MDLocation(line: 154, column: 8, scope: !40)
-!40 = distinct !MDLexicalBlock(line: 154, column: 8, file: !1, scope: !41)
-!41 = distinct !MDLexicalBlock(line: 154, column: 8, file: !1, scope: !42)
-!42 = distinct !MDLexicalBlock(line: 154, column: 3, file: !1, scope: !4)
-!43 = !MDLocation(line: 157, column: 5, scope: !44)
-!44 = distinct !MDLexicalBlock(line: 154, column: 42, file: !1, scope: !42)
-!45 = !MDLocation(line: 159, column: 5, scope: !44)
+!39 = !DILocation(line: 154, column: 8, scope: !40)
+!40 = distinct !DILexicalBlock(line: 154, column: 8, file: !1, scope: !41)
+!41 = distinct !DILexicalBlock(line: 154, column: 8, file: !1, scope: !42)
+!42 = distinct !DILexicalBlock(line: 154, column: 3, file: !1, scope: !4)
+!43 = !DILocation(line: 157, column: 5, scope: !44)
+!44 = distinct !DILexicalBlock(line: 154, column: 42, file: !1, scope: !42)
+!45 = !DILocation(line: 159, column: 5, scope: !44)
 !46 = !{!47, !47, i64 0}
 !47 = !{!"int", !48, i64 0}
 !48 = !{!"omnipotent char", !49, i64 0}
 !49 = !{!"Simple C/C++ TBAA"}
-!50 = !MDLocation(line: 160, column: 5, scope: !44)
-!51 = !MDLocation(line: 161, column: 5, scope: !44)
-!52 = !MDLocation(line: 188, column: 5, scope: !44)
-!53 = !MDLocation(line: 190, column: 5, scope: !44)
-!54 = !MDLocation(line: 198, column: 5, scope: !44)
-!55 = !MDLocation(line: 144, column: 13, scope: !4)
-!56 = !MDLocation(line: 200, column: 5, scope: !44)
-!57 = !MDLocation(line: 203, column: 5, scope: !44)
-!58 = !MDLocation(line: 207, column: 5, scope: !44)
-!59 = !MDLocation(line: 208, column: 5, scope: !44)
+!50 = !DILocation(line: 160, column: 5, scope: !44)
+!51 = !DILocation(line: 161, column: 5, scope: !44)
+!52 = !DILocation(line: 188, column: 5, scope: !44)
+!53 = !DILocation(line: 190, column: 5, scope: !44)
+!54 = !DILocation(line: 198, column: 5, scope: !44)
+!55 = !DILocation(line: 144, column: 13, scope: !4)
+!56 = !DILocation(line: 200, column: 5, scope: !44)
+!57 = !DILocation(line: 203, column: 5, scope: !44)
+!58 = !DILocation(line: 207, column: 5, scope: !44)
+!59 = !DILocation(line: 208, column: 5, scope: !44)
diff --git a/test/CodeGen/AArch64/aarch64-deferred-spilling.ll b/test/CodeGen/AArch64/aarch64-deferred-spilling.ll
new file mode 100644
index 00000000000..ff280c7f729
--- /dev/null
+++ b/test/CodeGen/AArch64/aarch64-deferred-spilling.ll
@@ -0,0 +1,512 @@
+;RUN: llc < %s -mtriple=aarch64--linux-android -regalloc=greedy -enable-deferred-spilling=true -mcpu=cortex-a57 | FileCheck %s --check-prefix=CHECK --check-prefix=DEFERRED
+;RUN: llc < %s -mtriple=aarch64--linux-android -regalloc=greedy -enable-deferred-spilling=false -mcpu=cortex-a57 | FileCheck %s --check-prefix=CHECK --check-prefix=REGULAR
+
+; Check that we do not end up with useless spill code.
+;
+; Move to the basic block we are interested in.
+;
+; CHECK: // %if.then.120
+;
+; REGULAR: str w22, [sp, #[[OFFSET:[0-9]+]]] // 4-byte Folded Spill
+; Check that w22 wouldn't need to be spilled since it is never reused.
+; REGULAR-NOT: {{[wx]}}22{{,?}}
+;
+; Check that w22 is used to carry a value through the call.
+; DEFERRED-NOT: str {{[wx]}}22,
+; DEFERRED: mov {{[wx]}}22,
+; DEFERRED-NOT: str {{[wx]}}22,
+;
+; CHECK:        bl      fprintf
+;
+; DEFERRED-NOT: ldr {{[wx]}}22,
+; DEFERRED: mov {{[wx][0-9]+}}, {{[wx]}}22
+; DEFERRED-NOT: ldr {{[wx]}}22,
+;
+; REGULAR-NOT: {{[wx]}}22{{,?}}
+; REGUAL: ldr w22, [sp, #[[OFFSET]]] // 4-byte Folded Reload
+;
+; End of the basic block we are interested in.
+; CHECK:        b
+; CHECK: {{[^:]+}}: // %sw.bb.123
+
+%struct.__sFILE = type { i8*, i32, i32, i32, i32, %struct.__sbuf, i32, i8*, i32 (i8*)*, i32 (i8*, i8*, i32)*, i64 (i8*, i64, i32)*, i32 (i8*, i8*, i32)*, %struct.__sbuf, i8*, i32, [3 x i8], [1 x i8], %struct.__sbuf, i32, i64 }
+%struct.__sbuf = type { i8*, i64 }
+%struct.DState = type { %struct.bz_stream*, i32, i8, i32, i8, i32, i32, i32, i32, i32, i8, i32, i32, i32, i32, i32, [256 x i32], i32, [257 x i32], [257 x i32], i32*, i16*, i8*, i32, i32, i32, i32, i32, [256 x i8], [16 x i8], [256 x i8], [4096 x i8], [16 x i32], [18002 x i8], [18002 x i8], [6 x [258 x i8]], [6 x [258 x i32]], [6 x [258 x i32]], [6 x [258 x i32]], [6 x i32], i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32*, i32*, i32* }
+%struct.bz_stream = type { i8*, i32, i32, i32, i8*, i32, i32, i32, i8*, i8* (i8*, i32, i32)*, void (i8*, i8*)*, i8* }
+
+@__sF = external global [0 x %struct.__sFILE], align 8
+@.str = private unnamed_addr constant [20 x i8] c"\0A    [%d: stuff+mf \00", align 1
+
+declare i32 @fprintf(%struct.__sFILE* nocapture, i8* nocapture readonly, ...)
+
+declare void @bar(i32)
+
+declare void @llvm.memset.p0i8.i64(i8* nocapture, i8, i64, i32, i1)
+
+define i32 @foo(%struct.DState* %s) {
+entry:
+  %state = getelementptr inbounds %struct.DState, %struct.DState* %s, i64 0, i32 1
+  %tmp = load i32, i32* %state, align 4
+  %cmp = icmp eq i32 %tmp, 10
+  %save_i = getelementptr inbounds %struct.DState, %struct.DState* %s, i64 0, i32 40
+  br i1 %cmp, label %if.end.thread, label %if.end
+
+if.end.thread:                                    ; preds = %entry
+  %save_j = getelementptr inbounds %struct.DState, %struct.DState* %s, i64 0, i32 41
+  %save_t = getelementptr inbounds %struct.DState, %struct.DState* %s, i64 0, i32 42
+  %save_alphaSize = getelementptr inbounds %struct.DState, %struct.DState* %s, i64 0, i32 43
+  %save_nGroups = getelementptr inbounds %struct.DState, %struct.DState* %s, i64 0, i32 44
+  %save_nSelectors = getelementptr inbounds %struct.DState, %struct.DState* %s, i64 0, i32 45
+  %save_EOB = getelementptr inbounds %struct.DState, %struct.DState* %s, i64 0, i32 46
+  %save_groupNo = getelementptr inbounds %struct.DState, %struct.DState* %s, i64 0, i32 47
+  %save_groupPos = getelementptr inbounds %struct.DState, %struct.DState* %s, i64 0, i32 48
+  %save_nextSym = getelementptr inbounds %struct.DState, %struct.DState* %s, i64 0, i32 49
+  %save_nblockMAX = getelementptr inbounds %struct.DState, %struct.DState* %s, i64 0, i32 50
+  %save_nblock = getelementptr inbounds %struct.DState, %struct.DState* %s, i64 0, i32 51
+  %save_es = getelementptr inbounds %struct.DState, %struct.DState* %s, i64 0, i32 52
+  %save_N = getelementptr inbounds %struct.DState, %struct.DState* %s, i64 0, i32 53
+  %save_curr = getelementptr inbounds %struct.DState, %struct.DState* %s, i64 0, i32 54
+  %save_zt = getelementptr inbounds %struct.DState, %struct.DState* %s, i64 0, i32 55
+  %save_zn = getelementptr inbounds %struct.DState, %struct.DState* %s, i64 0, i32 56
+  %save_zvec = getelementptr inbounds %struct.DState, %struct.DState* %s, i64 0, i32 57
+  %save_zj = getelementptr inbounds %struct.DState, %struct.DState* %s, i64 0, i32 58
+  %tmp1 = bitcast i32* %save_i to i8*
+  call void @llvm.memset.p0i8.i64(i8* %tmp1, i8 0, i64 108, i32 4, i1 false)
+  br label %sw.default
+
+if.end:                                           ; preds = %entry
+  %.pre = load i32, i32* %save_i, align 4
+  %save_j3.phi.trans.insert = getelementptr inbounds %struct.DState, %struct.DState* %s, i64 0, i32 41
+  %.pre406 = load i32, i32* %save_j3.phi.trans.insert, align 4
+  %save_t4.phi.trans.insert = getelementptr inbounds %struct.DState, %struct.DState* %s, i64 0, i32 42
+  %.pre407 = load i32, i32* %save_t4.phi.trans.insert, align 4
+  %save_alphaSize5.phi.trans.insert = getelementptr inbounds %struct.DState, %struct.DState* %s, i64 0, i32 43
+  %.pre408 = load i32, i32* %save_alphaSize5.phi.trans.insert, align 4
+  %save_nGroups6.phi.trans.insert = getelementptr inbounds %struct.DState, %struct.DState* %s, i64 0, i32 44
+  %.pre409 = load i32, i32* %save_nGroups6.phi.trans.insert, align 4
+  %save_nSelectors7.phi.trans.insert = getelementptr inbounds %struct.DState, %struct.DState* %s, i64 0, i32 45
+  %.pre410 = load i32, i32* %save_nSelectors7.phi.trans.insert, align 4
+  %save_EOB8.phi.trans.insert = getelementptr inbounds %struct.DState, %struct.DState* %s, i64 0, i32 46
+  %.pre411 = load i32, i32* %save_EOB8.phi.trans.insert, align 4
+  %save_groupNo9.phi.trans.insert = getelementptr inbounds %struct.DState, %struct.DState* %s, i64 0, i32 47
+  %.pre412 = load i32, i32* %save_groupNo9.phi.trans.insert, align 4
+  %save_groupPos10.phi.trans.insert = getelementptr inbounds %struct.DState, %struct.DState* %s, i64 0, i32 48
+  %.pre413 = load i32, i32* %save_groupPos10.phi.trans.insert, align 4
+  %save_nextSym11.phi.trans.insert = getelementptr inbounds %struct.DState, %struct.DState* %s, i64 0, i32 49
+  %.pre414 = load i32, i32* %save_nextSym11.phi.trans.insert, align 4
+  %save_nblockMAX12.phi.trans.insert = getelementptr inbounds %struct.DState, %struct.DState* %s, i64 0, i32 50
+  %.pre415 = load i32, i32* %save_nblockMAX12.phi.trans.insert, align 4
+  %save_nblock13.phi.trans.insert = getelementptr inbounds %struct.DState, %struct.DState* %s, i64 0, i32 51
+  %.pre416 = load i32, i32* %save_nblock13.phi.trans.insert, align 4
+  %save_es14.phi.trans.insert = getelementptr inbounds %struct.DState, %struct.DState* %s, i64 0, i32 52
+  %.pre417 = load i32, i32* %save_es14.phi.trans.insert, align 4
+  %save_N15.phi.trans.insert = getelementptr inbounds %struct.DState, %struct.DState* %s, i64 0, i32 53
+  %.pre418 = load i32, i32* %save_N15.phi.trans.insert, align 4
+  %save_curr16.phi.trans.insert = getelementptr inbounds %struct.DState, %struct.DState* %s, i64 0, i32 54
+  %.pre419 = load i32, i32* %save_curr16.phi.trans.insert, align 4
+  %save_zt17.phi.trans.insert = getelementptr inbounds %struct.DState, %struct.DState* %s, i64 0, i32 55
+  %.pre420 = load i32, i32* %save_zt17.phi.trans.insert, align 4
+  %save_zn18.phi.trans.insert = getelementptr inbounds %struct.DState, %struct.DState* %s, i64 0, i32 56
+  %.pre421 = load i32, i32* %save_zn18.phi.trans.insert, align 4
+  %save_zvec19.phi.trans.insert = getelementptr inbounds %struct.DState, %struct.DState* %s, i64 0, i32 57
+  %.pre422 = load i32, i32* %save_zvec19.phi.trans.insert, align 4
+  %save_zj20.phi.trans.insert = getelementptr inbounds %struct.DState, %struct.DState* %s, i64 0, i32 58
+  %.pre423 = load i32, i32* %save_zj20.phi.trans.insert, align 4
+  switch i32 %tmp, label %sw.default [
+    i32 13, label %sw.bb
+    i32 14, label %if.end.sw.bb.65_crit_edge
+    i32 25, label %if.end.sw.bb.123_crit_edge
+  ]
+
+if.end.sw.bb.123_crit_edge:                       ; preds = %if.end
+  %.pre433 = getelementptr inbounds %struct.DState, %struct.DState* %s, i64 0, i32 8
+  br label %sw.bb.123
+
+if.end.sw.bb.65_crit_edge:                        ; preds = %if.end
+  %bsLive69.phi.trans.insert = getelementptr inbounds %struct.DState, %struct.DState* %s, i64 0, i32 8
+  %.pre426 = load i32, i32* %bsLive69.phi.trans.insert, align 4
+  br label %sw.bb.65
+
+sw.bb:                                            ; preds = %if.end
+  %sunkaddr = ptrtoint %struct.DState* %s to i64
+  %sunkaddr485 = add i64 %sunkaddr, 8
+  %sunkaddr486 = inttoptr i64 %sunkaddr485 to i32*
+  store i32 13, i32* %sunkaddr486, align 4
+  %bsLive = getelementptr inbounds %struct.DState, %struct.DState* %s, i64 0, i32 8
+  %tmp2 = load i32, i32* %bsLive, align 4
+  %cmp28.400 = icmp sgt i32 %tmp2, 7
+  br i1 %cmp28.400, label %sw.bb.if.then.29_crit_edge, label %if.end.33.lr.ph
+
+sw.bb.if.then.29_crit_edge:                       ; preds = %sw.bb
+  %sunkaddr487 = ptrtoint %struct.DState* %s to i64
+  %sunkaddr488 = add i64 %sunkaddr487, 32
+  %sunkaddr489 = inttoptr i64 %sunkaddr488 to i32*
+  %.pre425 = load i32, i32* %sunkaddr489, align 4
+  br label %if.then.29
+
+if.end.33.lr.ph:                                  ; preds = %sw.bb
+  %tmp3 = bitcast %struct.DState* %s to %struct.bz_stream**
+  %.pre424 = load %struct.bz_stream*, %struct.bz_stream** %tmp3, align 8
+  %avail_in.phi.trans.insert = getelementptr inbounds %struct.bz_stream, %struct.bz_stream* %.pre424, i64 0, i32 1
+  %.pre430 = load i32, i32* %avail_in.phi.trans.insert, align 4
+  %tmp4 = add i32 %.pre430, -1
+  br label %if.end.33
+
+if.then.29:                                       ; preds = %while.body.backedge, %sw.bb.if.then.29_crit_edge
+  %tmp5 = phi i32 [ %.pre425, %sw.bb.if.then.29_crit_edge ], [ %or, %while.body.backedge ]
+  %.lcssa393 = phi i32 [ %tmp2, %sw.bb.if.then.29_crit_edge ], [ %add, %while.body.backedge ]
+  %sub = add nsw i32 %.lcssa393, -8
+  %shr = lshr i32 %tmp5, %sub
+  %and = and i32 %shr, 255
+  %sunkaddr491 = ptrtoint %struct.DState* %s to i64
+  %sunkaddr492 = add i64 %sunkaddr491, 36
+  %sunkaddr493 = inttoptr i64 %sunkaddr492 to i32*
+  store i32 %sub, i32* %sunkaddr493, align 4
+  %blockSize100k = getelementptr inbounds %struct.DState, %struct.DState* %s, i64 0, i32 9
+  store i32 %and, i32* %blockSize100k, align 4
+  %and.off = add nsw i32 %and, -49
+  %tmp6 = icmp ugt i32 %and.off, 8
+  br i1 %tmp6, label %save_state_and_return, label %if.end.62
+
+if.end.33:                                        ; preds = %while.body.backedge, %if.end.33.lr.ph
+  %lsr.iv482 = phi i32 [ %tmp4, %if.end.33.lr.ph ], [ %lsr.iv.next483, %while.body.backedge ]
+  %tmp7 = phi i32 [ %tmp2, %if.end.33.lr.ph ], [ %add, %while.body.backedge ]
+  %cmp35 = icmp eq i32 %lsr.iv482, -1
+  br i1 %cmp35, label %save_state_and_return, label %if.end.37
+
+if.end.37:                                        ; preds = %if.end.33
+  %tmp8 = bitcast %struct.bz_stream* %.pre424 to i8**
+  %sunkaddr494 = ptrtoint %struct.DState* %s to i64
+  %sunkaddr495 = add i64 %sunkaddr494, 32
+  %sunkaddr496 = inttoptr i64 %sunkaddr495 to i32*
+  %tmp9 = load i32, i32* %sunkaddr496, align 4
+  %shl = shl i32 %tmp9, 8
+  %tmp10 = load i8*, i8** %tmp8, align 8
+  %tmp11 = load i8, i8* %tmp10, align 1
+  %conv = zext i8 %tmp11 to i32
+  %or = or i32 %conv, %shl
+  store i32 %or, i32* %sunkaddr496, align 4
+  %add = add nsw i32 %tmp7, 8
+  %sunkaddr497 = ptrtoint %struct.DState* %s to i64
+  %sunkaddr498 = add i64 %sunkaddr497, 36
+  %sunkaddr499 = inttoptr i64 %sunkaddr498 to i32*
+  store i32 %add, i32* %sunkaddr499, align 4
+  %incdec.ptr = getelementptr inbounds i8, i8* %tmp10, i64 1
+  store i8* %incdec.ptr, i8** %tmp8, align 8
+  %sunkaddr500 = ptrtoint %struct.bz_stream* %.pre424 to i64
+  %sunkaddr501 = add i64 %sunkaddr500, 8
+  %sunkaddr502 = inttoptr i64 %sunkaddr501 to i32*
+  store i32 %lsr.iv482, i32* %sunkaddr502, align 4
+  %sunkaddr503 = ptrtoint %struct.bz_stream* %.pre424 to i64
+  %sunkaddr504 = add i64 %sunkaddr503, 12
+  %sunkaddr505 = inttoptr i64 %sunkaddr504 to i32*
+  %tmp12 = load i32, i32* %sunkaddr505, align 4
+  %inc = add i32 %tmp12, 1
+  store i32 %inc, i32* %sunkaddr505, align 4
+  %cmp49 = icmp eq i32 %inc, 0
+  br i1 %cmp49, label %if.then.51, label %while.body.backedge
+
+if.then.51:                                       ; preds = %if.end.37
+  %sunkaddr506 = ptrtoint %struct.bz_stream* %.pre424 to i64
+  %sunkaddr507 = add i64 %sunkaddr506, 16
+  %sunkaddr508 = inttoptr i64 %sunkaddr507 to i32*
+  %tmp13 = load i32, i32* %sunkaddr508, align 4
+  %inc53 = add i32 %tmp13, 1
+  store i32 %inc53, i32* %sunkaddr508, align 4
+  br label %while.body.backedge
+
+while.body.backedge:                              ; preds = %if.then.51, %if.end.37
+  %lsr.iv.next483 = add i32 %lsr.iv482, -1
+  %cmp28 = icmp sgt i32 %add, 7
+  br i1 %cmp28, label %if.then.29, label %if.end.33
+
+if.end.62:                                        ; preds = %if.then.29
+  %sub64 = add nsw i32 %and, -48
+  %sunkaddr509 = ptrtoint %struct.DState* %s to i64
+  %sunkaddr510 = add i64 %sunkaddr509, 40
+  %sunkaddr511 = inttoptr i64 %sunkaddr510 to i32*
+  store i32 %sub64, i32* %sunkaddr511, align 4
+  br label %sw.bb.65
+
+sw.bb.65:                                         ; preds = %if.end.62, %if.end.sw.bb.65_crit_edge
+  %bsLive69.pre-phi = phi i32* [ %bsLive69.phi.trans.insert, %if.end.sw.bb.65_crit_edge ], [ %bsLive, %if.end.62 ]
+  %tmp14 = phi i32 [ %.pre426, %if.end.sw.bb.65_crit_edge ], [ %sub, %if.end.62 ]
+  %sunkaddr512 = ptrtoint %struct.DState* %s to i64
+  %sunkaddr513 = add i64 %sunkaddr512, 8
+  %sunkaddr514 = inttoptr i64 %sunkaddr513 to i32*
+  store i32 14, i32* %sunkaddr514, align 4
+  %cmp70.397 = icmp sgt i32 %tmp14, 7
+  br i1 %cmp70.397, label %if.then.72, label %if.end.82.lr.ph
+
+if.end.82.lr.ph:                                  ; preds = %sw.bb.65
+  %tmp15 = bitcast %struct.DState* %s to %struct.bz_stream**
+  %.pre427 = load %struct.bz_stream*, %struct.bz_stream** %tmp15, align 8
+  %avail_in84.phi.trans.insert = getelementptr inbounds %struct.bz_stream, %struct.bz_stream* %.pre427, i64 0, i32 1
+  %.pre431 = load i32, i32* %avail_in84.phi.trans.insert, align 4
+  %tmp16 = add i32 %.pre431, -1
+  br label %if.end.82
+
+if.then.72:                                       ; preds = %while.body.68.backedge, %sw.bb.65
+  %.lcssa390 = phi i32 [ %tmp14, %sw.bb.65 ], [ %add97, %while.body.68.backedge ]
+  %sub76 = add nsw i32 %.lcssa390, -8
+  %sunkaddr516 = ptrtoint %struct.DState* %s to i64
+  %sunkaddr517 = add i64 %sunkaddr516, 36
+  %sunkaddr518 = inttoptr i64 %sunkaddr517 to i32*
+  store i32 %sub76, i32* %sunkaddr518, align 4
+  %currBlockNo = getelementptr inbounds %struct.DState, %struct.DState* %s, i64 0, i32 11
+  %tmp17 = load i32, i32* %currBlockNo, align 4
+  %inc117 = add nsw i32 %tmp17, 1
+  store i32 %inc117, i32* %currBlockNo, align 4
+  %verbosity = getelementptr inbounds %struct.DState, %struct.DState* %s, i64 0, i32 12
+  %tmp18 = load i32, i32* %verbosity, align 4
+  %cmp118 = icmp sgt i32 %tmp18, 1
+  br i1 %cmp118, label %if.then.120, label %sw.bb.123
+
+if.end.82:                                        ; preds = %while.body.68.backedge, %if.end.82.lr.ph
+  %lsr.iv480 = phi i32 [ %tmp16, %if.end.82.lr.ph ], [ %lsr.iv.next481, %while.body.68.backedge ]
+  %tmp19 = phi i32 [ %tmp14, %if.end.82.lr.ph ], [ %add97, %while.body.68.backedge ]
+  %cmp85 = icmp eq i32 %lsr.iv480, -1
+  br i1 %cmp85, label %save_state_and_return, label %if.end.88
+
+if.end.88:                                        ; preds = %if.end.82
+  %tmp20 = bitcast %struct.bz_stream* %.pre427 to i8**
+  %sunkaddr519 = ptrtoint %struct.DState* %s to i64
+  %sunkaddr520 = add i64 %sunkaddr519, 32
+  %sunkaddr521 = inttoptr i64 %sunkaddr520 to i32*
+  %tmp21 = load i32, i32* %sunkaddr521, align 4
+  %shl90 = shl i32 %tmp21, 8
+  %tmp22 = load i8*, i8** %tmp20, align 8
+  %tmp23 = load i8, i8* %tmp22, align 1
+  %conv93 = zext i8 %tmp23 to i32
+  %or94 = or i32 %conv93, %shl90
+  store i32 %or94, i32* %sunkaddr521, align 4
+  %add97 = add nsw i32 %tmp19, 8
+  %sunkaddr522 = ptrtoint %struct.DState* %s to i64
+  %sunkaddr523 = add i64 %sunkaddr522, 36
+  %sunkaddr524 = inttoptr i64 %sunkaddr523 to i32*
+  store i32 %add97, i32* %sunkaddr524, align 4
+  %incdec.ptr100 = getelementptr inbounds i8, i8* %tmp22, i64 1
+  store i8* %incdec.ptr100, i8** %tmp20, align 8
+  %sunkaddr525 = ptrtoint %struct.bz_stream* %.pre427 to i64
+  %sunkaddr526 = add i64 %sunkaddr525, 8
+  %sunkaddr527 = inttoptr i64 %sunkaddr526 to i32*
+  store i32 %lsr.iv480, i32* %sunkaddr527, align 4
+  %sunkaddr528 = ptrtoint %struct.bz_stream* %.pre427 to i64
+  %sunkaddr529 = add i64 %sunkaddr528, 12
+  %sunkaddr530 = inttoptr i64 %sunkaddr529 to i32*
+  %tmp24 = load i32, i32* %sunkaddr530, align 4
+  %inc106 = add i32 %tmp24, 1
+  store i32 %inc106, i32* %sunkaddr530, align 4
+  %cmp109 = icmp eq i32 %inc106, 0
+  br i1 %cmp109, label %if.then.111, label %while.body.68.backedge
+
+if.then.111:                                      ; preds = %if.end.88
+  %sunkaddr531 = ptrtoint %struct.bz_stream* %.pre427 to i64
+  %sunkaddr532 = add i64 %sunkaddr531, 16
+  %sunkaddr533 = inttoptr i64 %sunkaddr532 to i32*
+  %tmp25 = load i32, i32* %sunkaddr533, align 4
+  %inc114 = add i32 %tmp25, 1
+  store i32 %inc114, i32* %sunkaddr533, align 4
+  br label %while.body.68.backedge
+
+while.body.68.backedge:                           ; preds = %if.then.111, %if.end.88
+  %lsr.iv.next481 = add i32 %lsr.iv480, -1
+  %cmp70 = icmp sgt i32 %add97, 7
+  br i1 %cmp70, label %if.then.72, label %if.end.82
+
+if.then.120:                                      ; preds = %if.then.72
+  %call = tail call i32 (%struct.__sFILE*, i8*, ...) @fprintf(%struct.__sFILE* getelementptr inbounds ([0 x %struct.__sFILE], [0 x %struct.__sFILE]* @__sF, i64 0, i64 2), i8* getelementptr inbounds ([20 x i8], [20 x i8]* @.str, i64 0, i64 0), i32 %inc117)
+  br label %sw.bb.123
+
+sw.bb.123:                                        ; preds = %if.then.120, %if.then.72, %if.end.sw.bb.123_crit_edge
+  %bsLive127.pre-phi = phi i32* [ %.pre433, %if.end.sw.bb.123_crit_edge ], [ %bsLive69.pre-phi, %if.then.72 ], [ %bsLive69.pre-phi, %if.then.120 ]
+  %sunkaddr534 = ptrtoint %struct.DState* %s to i64
+  %sunkaddr535 = add i64 %sunkaddr534, 8
+  %sunkaddr536 = inttoptr i64 %sunkaddr535 to i32*
+  store i32 25, i32* %sunkaddr536, align 4
+  %tmp26 = load i32, i32* %bsLive127.pre-phi, align 4
+  %cmp128.395 = icmp sgt i32 %tmp26, 7
+  br i1 %cmp128.395, label %sw.bb.123.if.then.130_crit_edge, label %if.end.140.lr.ph
+
+sw.bb.123.if.then.130_crit_edge:                  ; preds = %sw.bb.123
+  %sunkaddr537 = ptrtoint %struct.DState* %s to i64
+  %sunkaddr538 = add i64 %sunkaddr537, 32
+  %sunkaddr539 = inttoptr i64 %sunkaddr538 to i32*
+  %.pre429 = load i32, i32* %sunkaddr539, align 4
+  br label %if.then.130
+
+if.end.140.lr.ph:                                 ; preds = %sw.bb.123
+  %tmp27 = bitcast %struct.DState* %s to %struct.bz_stream**
+  %.pre428 = load %struct.bz_stream*, %struct.bz_stream** %tmp27, align 8
+  %avail_in142.phi.trans.insert = getelementptr inbounds %struct.bz_stream, %struct.bz_stream* %.pre428, i64 0, i32 1
+  %.pre432 = load i32, i32* %avail_in142.phi.trans.insert, align 4
+  %tmp28 = add i32 %.pre432, -1
+  br label %if.end.140
+
+if.then.130:                                      ; preds = %while.body.126.backedge, %sw.bb.123.if.then.130_crit_edge
+  %tmp29 = phi i32 [ %.pre429, %sw.bb.123.if.then.130_crit_edge ], [ %or152, %while.body.126.backedge ]
+  %.lcssa = phi i32 [ %tmp26, %sw.bb.123.if.then.130_crit_edge ], [ %add155, %while.body.126.backedge ]
+  %sub134 = add nsw i32 %.lcssa, -8
+  %shr135 = lshr i32 %tmp29, %sub134
+  store i32 %sub134, i32* %bsLive127.pre-phi, align 4
+  %origPtr = getelementptr inbounds %struct.DState, %struct.DState* %s, i64 0, i32 13
+  %tmp30 = load i32, i32* %origPtr, align 4
+  %shl175 = shl i32 %tmp30, 8
+  %conv176 = and i32 %shr135, 255
+  %or177 = or i32 %shl175, %conv176
+  store i32 %or177, i32* %origPtr, align 4
+  %nInUse = getelementptr inbounds %struct.DState, %struct.DState* %s, i64 0, i32 27
+  %tmp31 = load i32, i32* %nInUse, align 4
+  %add179 = add nsw i32 %tmp31, 2
+  br label %save_state_and_return
+
+if.end.140:                                       ; preds = %while.body.126.backedge, %if.end.140.lr.ph
+  %lsr.iv = phi i32 [ %tmp28, %if.end.140.lr.ph ], [ %lsr.iv.next, %while.body.126.backedge ]
+  %tmp32 = phi i32 [ %tmp26, %if.end.140.lr.ph ], [ %add155, %while.body.126.backedge ]
+  %cmp143 = icmp eq i32 %lsr.iv, -1
+  br i1 %cmp143, label %save_state_and_return, label %if.end.146
+
+if.end.146:                                       ; preds = %if.end.140
+  %tmp33 = bitcast %struct.bz_stream* %.pre428 to i8**
+  %sunkaddr541 = ptrtoint %struct.DState* %s to i64
+  %sunkaddr542 = add i64 %sunkaddr541, 32
+  %sunkaddr543 = inttoptr i64 %sunkaddr542 to i32*
+  %tmp34 = load i32, i32* %sunkaddr543, align 4
+  %shl148 = shl i32 %tmp34, 8
+  %tmp35 = load i8*, i8** %tmp33, align 8
+  %tmp36 = load i8, i8* %tmp35, align 1
+  %conv151 = zext i8 %tmp36 to i32
+  %or152 = or i32 %conv151, %shl148
+  store i32 %or152, i32* %sunkaddr543, align 4
+  %add155 = add nsw i32 %tmp32, 8
+  store i32 %add155, i32* %bsLive127.pre-phi, align 4
+  %incdec.ptr158 = getelementptr inbounds i8, i8* %tmp35, i64 1
+  store i8* %incdec.ptr158, i8** %tmp33, align 8
+  %sunkaddr544 = ptrtoint %struct.bz_stream* %.pre428 to i64
+  %sunkaddr545 = add i64 %sunkaddr544, 8
+  %sunkaddr546 = inttoptr i64 %sunkaddr545 to i32*
+  store i32 %lsr.iv, i32* %sunkaddr546, align 4
+  %sunkaddr547 = ptrtoint %struct.bz_stream* %.pre428 to i64
+  %sunkaddr548 = add i64 %sunkaddr547, 12
+  %sunkaddr549 = inttoptr i64 %sunkaddr548 to i32*
+  %tmp37 = load i32, i32* %sunkaddr549, align 4
+  %inc164 = add i32 %tmp37, 1
+  store i32 %inc164, i32* %sunkaddr549, align 4
+  %cmp167 = icmp eq i32 %inc164, 0
+  br i1 %cmp167, label %if.then.169, label %while.body.126.backedge
+
+if.then.169:                                      ; preds = %if.end.146
+  %sunkaddr550 = ptrtoint %struct.bz_stream* %.pre428 to i64
+  %sunkaddr551 = add i64 %sunkaddr550, 16
+  %sunkaddr552 = inttoptr i64 %sunkaddr551 to i32*
+  %tmp38 = load i32, i32* %sunkaddr552, align 4
+  %inc172 = add i32 %tmp38, 1
+  store i32 %inc172, i32* %sunkaddr552, align 4
+  br label %while.body.126.backedge
+
+while.body.126.backedge:                          ; preds = %if.then.169, %if.end.146
+  %lsr.iv.next = add i32 %lsr.iv, -1
+  %cmp128 = icmp sgt i32 %add155, 7
+  br i1 %cmp128, label %if.then.130, label %if.end.140
+
+sw.default:                                       ; preds = %if.end, %if.end.thread
+  %tmp39 = phi i32 [ 0, %if.end.thread ], [ %.pre, %if.end ]
+  %tmp40 = phi i32 [ 0, %if.end.thread ], [ %.pre406, %if.end ]
+  %tmp41 = phi i32 [ 0, %if.end.thread ], [ %.pre407, %if.end ]
+  %tmp42 = phi i32 [ 0, %if.end.thread ], [ %.pre408, %if.end ]
+  %tmp43 = phi i32 [ 0, %if.end.thread ], [ %.pre409, %if.end ]
+  %tmp44 = phi i32 [ 0, %if.end.thread ], [ %.pre410, %if.end ]
+  %tmp45 = phi i32 [ 0, %if.end.thread ], [ %.pre411, %if.end ]
+  %tmp46 = phi i32 [ 0, %if.end.thread ], [ %.pre412, %if.end ]
+  %tmp47 = phi i32 [ 0, %if.end.thread ], [ %.pre413, %if.end ]
+  %tmp48 = phi i32 [ 0, %if.end.thread ], [ %.pre414, %if.end ]
+  %tmp49 = phi i32 [ 0, %if.end.thread ], [ %.pre415, %if.end ]
+  %tmp50 = phi i32 [ 0, %if.end.thread ], [ %.pre416, %if.end ]
+  %tmp51 = phi i32 [ 0, %if.end.thread ], [ %.pre417, %if.end ]
+  %tmp52 = phi i32 [ 0, %if.end.thread ], [ %.pre418, %if.end ]
+  %tmp53 = phi i32 [ 0, %if.end.thread ], [ %.pre419, %if.end ]
+  %tmp54 = phi i32 [ 0, %if.end.thread ], [ %.pre420, %if.end ]
+  %tmp55 = phi i32 [ 0, %if.end.thread ], [ %.pre421, %if.end ]
+  %tmp56 = phi i32 [ 0, %if.end.thread ], [ %.pre422, %if.end ]
+  %tmp57 = phi i32 [ 0, %if.end.thread ], [ %.pre423, %if.end ]
+  %save_j3.pre-phi469 = phi i32* [ %save_j, %if.end.thread ], [ %save_j3.phi.trans.insert, %if.end ]
+  %save_t4.pre-phi467 = phi i32* [ %save_t, %if.end.thread ], [ %save_t4.phi.trans.insert, %if.end ]
+  %save_alphaSize5.pre-phi465 = phi i32* [ %save_alphaSize, %if.end.thread ], [ %save_alphaSize5.phi.trans.insert, %if.end ]
+  %save_nGroups6.pre-phi463 = phi i32* [ %save_nGroups, %if.end.thread ], [ %save_nGroups6.phi.trans.insert, %if.end ]
+  %save_nSelectors7.pre-phi461 = phi i32* [ %save_nSelectors, %if.end.thread ], [ %save_nSelectors7.phi.trans.insert, %if.end ]
+  %save_EOB8.pre-phi459 = phi i32* [ %save_EOB, %if.end.thread ], [ %save_EOB8.phi.trans.insert, %if.end ]
+  %save_groupNo9.pre-phi457 = phi i32* [ %save_groupNo, %if.end.thread ], [ %save_groupNo9.phi.trans.insert, %if.end ]
+  %save_groupPos10.pre-phi455 = phi i32* [ %save_groupPos, %if.end.thread ], [ %save_groupPos10.phi.trans.insert, %if.end ]
+  %save_nextSym11.pre-phi453 = phi i32* [ %save_nextSym, %if.end.thread ], [ %save_nextSym11.phi.trans.insert, %if.end ]
+  %save_nblockMAX12.pre-phi451 = phi i32* [ %save_nblockMAX, %if.end.thread ], [ %save_nblockMAX12.phi.trans.insert, %if.end ]
+  %save_nblock13.pre-phi449 = phi i32* [ %save_nblock, %if.end.thread ], [ %save_nblock13.phi.trans.insert, %if.end ]
+  %save_es14.pre-phi447 = phi i32* [ %save_es, %if.end.thread ], [ %save_es14.phi.trans.insert, %if.end ]
+  %save_N15.pre-phi445 = phi i32* [ %save_N, %if.end.thread ], [ %save_N15.phi.trans.insert, %if.end ]
+  %save_curr16.pre-phi443 = phi i32* [ %save_curr, %if.end.thread ], [ %save_curr16.phi.trans.insert, %if.end ]
+  %save_zt17.pre-phi441 = phi i32* [ %save_zt, %if.end.thread ], [ %save_zt17.phi.trans.insert, %if.end ]
+  %save_zn18.pre-phi439 = phi i32* [ %save_zn, %if.end.thread ], [ %save_zn18.phi.trans.insert, %if.end ]
+  %save_zvec19.pre-phi437 = phi i32* [ %save_zvec, %if.end.thread ], [ %save_zvec19.phi.trans.insert, %if.end ]
+  %save_zj20.pre-phi435 = phi i32* [ %save_zj, %if.end.thread ], [ %save_zj20.phi.trans.insert, %if.end ]
+  tail call void @bar(i32 4001)
+  br label %save_state_and_return
+
+save_state_and_return:                            ; preds = %sw.default, %if.end.140, %if.then.130, %if.end.82, %if.end.33, %if.then.29
+  %tmp58 = phi i32 [ %tmp39, %sw.default ], [ %.pre, %if.then.29 ], [ %.pre, %if.then.130 ], [ %.pre, %if.end.140 ], [ %.pre, %if.end.82 ], [ %.pre, %if.end.33 ]
+  %tmp59 = phi i32 [ %tmp40, %sw.default ], [ %.pre406, %if.then.29 ], [ %.pre406, %if.then.130 ], [ %.pre406, %if.end.140 ], [ %.pre406, %if.end.82 ], [ %.pre406, %if.end.33 ]
+  %tmp60 = phi i32 [ %tmp41, %sw.default ], [ %.pre407, %if.then.29 ], [ %.pre407, %if.then.130 ], [ %.pre407, %if.end.140 ], [ %.pre407, %if.end.82 ], [ %.pre407, %if.end.33 ]
+  %tmp61 = phi i32 [ %tmp43, %sw.default ], [ %.pre409, %if.then.29 ], [ %.pre409, %if.then.130 ], [ %.pre409, %if.end.140 ], [ %.pre409, %if.end.82 ], [ %.pre409, %if.end.33 ]
+  %tmp62 = phi i32 [ %tmp44, %sw.default ], [ %.pre410, %if.then.29 ], [ %.pre410, %if.then.130 ], [ %.pre410, %if.end.140 ], [ %.pre410, %if.end.82 ], [ %.pre410, %if.end.33 ]
+  %tmp63 = phi i32 [ %tmp45, %sw.default ], [ %.pre411, %if.then.29 ], [ %.pre411, %if.then.130 ], [ %.pre411, %if.end.140 ], [ %.pre411, %if.end.82 ], [ %.pre411, %if.end.33 ]
+  %tmp64 = phi i32 [ %tmp46, %sw.default ], [ %.pre412, %if.then.29 ], [ %.pre412, %if.then.130 ], [ %.pre412, %if.end.140 ], [ %.pre412, %if.end.82 ], [ %.pre412, %if.end.33 ]
+  %tmp65 = phi i32 [ %tmp47, %sw.default ], [ %.pre413, %if.then.29 ], [ %.pre413, %if.then.130 ], [ %.pre413, %if.end.140 ], [ %.pre413, %if.end.82 ], [ %.pre413, %if.end.33 ]
+  %tmp66 = phi i32 [ %tmp48, %sw.default ], [ %.pre414, %if.then.29 ], [ %.pre414, %if.then.130 ], [ %.pre414, %if.end.140 ], [ %.pre414, %if.end.82 ], [ %.pre414, %if.end.33 ]
+  %tmp67 = phi i32 [ %tmp49, %sw.default ], [ %.pre415, %if.then.29 ], [ %.pre415, %if.then.130 ], [ %.pre415, %if.end.140 ], [ %.pre415, %if.end.82 ], [ %.pre415, %if.end.33 ]
+  %tmp68 = phi i32 [ %tmp51, %sw.default ], [ %.pre417, %if.then.29 ], [ %.pre417, %if.then.130 ], [ %.pre417, %if.end.140 ], [ %.pre417, %if.end.82 ], [ %.pre417, %if.end.33 ]
+  %tmp69 = phi i32 [ %tmp52, %sw.default ], [ %.pre418, %if.then.29 ], [ %.pre418, %if.then.130 ], [ %.pre418, %if.end.140 ], [ %.pre418, %if.end.82 ], [ %.pre418, %if.end.33 ]
+  %tmp70 = phi i32 [ %tmp53, %sw.default ], [ %.pre419, %if.then.29 ], [ %.pre419, %if.then.130 ], [ %.pre419, %if.end.140 ], [ %.pre419, %if.end.82 ], [ %.pre419, %if.end.33 ]
+  %tmp71 = phi i32 [ %tmp54, %sw.default ], [ %.pre420, %if.then.29 ], [ %.pre420, %if.then.130 ], [ %.pre420, %if.end.140 ], [ %.pre420, %if.end.82 ], [ %.pre420, %if.end.33 ]
+  %tmp72 = phi i32 [ %tmp55, %sw.default ], [ %.pre421, %if.then.29 ], [ %.pre421, %if.then.130 ], [ %.pre421, %if.end.140 ], [ %.pre421, %if.end.82 ], [ %.pre421, %if.end.33 ]
+  %tmp73 = phi i32 [ %tmp56, %sw.default ], [ %.pre422, %if.then.29 ], [ %.pre422, %if.then.130 ], [ %.pre422, %if.end.140 ], [ %.pre422, %if.end.82 ], [ %.pre422, %if.end.33 ]
+  %tmp74 = phi i32 [ %tmp57, %sw.default ], [ %.pre423, %if.then.29 ], [ %.pre423, %if.then.130 ], [ %.pre423, %if.end.140 ], [ %.pre423, %if.end.82 ], [ %.pre423, %if.end.33 ]
+  %save_j3.pre-phi468 = phi i32* [ %save_j3.pre-phi469, %sw.default ], [ %save_j3.phi.trans.insert, %if.then.29 ], [ %save_j3.phi.trans.insert, %if.then.130 ], [ %save_j3.phi.trans.insert, %if.end.140 ], [ %save_j3.phi.trans.insert, %if.end.82 ], [ %save_j3.phi.trans.insert, %if.end.33 ]
+  %save_t4.pre-phi466 = phi i32* [ %save_t4.pre-phi467, %sw.default ], [ %save_t4.phi.trans.insert, %if.then.29 ], [ %save_t4.phi.trans.insert, %if.then.130 ], [ %save_t4.phi.trans.insert, %if.end.140 ], [ %save_t4.phi.trans.insert, %if.end.82 ], [ %save_t4.phi.trans.insert, %if.end.33 ]
+  %save_alphaSize5.pre-phi464 = phi i32* [ %save_alphaSize5.pre-phi465, %sw.default ], [ %save_alphaSize5.phi.trans.insert, %if.then.29 ], [ %save_alphaSize5.phi.trans.insert, %if.then.130 ], [ %save_alphaSize5.phi.trans.insert, %if.end.140 ], [ %save_alphaSize5.phi.trans.insert, %if.end.82 ], [ %save_alphaSize5.phi.trans.insert, %if.end.33 ]
+  %save_nGroups6.pre-phi462 = phi i32* [ %save_nGroups6.pre-phi463, %sw.default ], [ %save_nGroups6.phi.trans.insert, %if.then.29 ], [ %save_nGroups6.phi.trans.insert, %if.then.130 ], [ %save_nGroups6.phi.trans.insert, %if.end.140 ], [ %save_nGroups6.phi.trans.insert, %if.end.82 ], [ %save_nGroups6.phi.trans.insert, %if.end.33 ]
+  %save_nSelectors7.pre-phi460 = phi i32* [ %save_nSelectors7.pre-phi461, %sw.default ], [ %save_nSelectors7.phi.trans.insert, %if.then.29 ], [ %save_nSelectors7.phi.trans.insert, %if.then.130 ], [ %save_nSelectors7.phi.trans.insert, %if.end.140 ], [ %save_nSelectors7.phi.trans.insert, %if.end.82 ], [ %save_nSelectors7.phi.trans.insert, %if.end.33 ]
+  %save_EOB8.pre-phi458 = phi i32* [ %save_EOB8.pre-phi459, %sw.default ], [ %save_EOB8.phi.trans.insert, %if.then.29 ], [ %save_EOB8.phi.trans.insert, %if.then.130 ], [ %save_EOB8.phi.trans.insert, %if.end.140 ], [ %save_EOB8.phi.trans.insert, %if.end.82 ], [ %save_EOB8.phi.trans.insert, %if.end.33 ]
+  %save_groupNo9.pre-phi456 = phi i32* [ %save_groupNo9.pre-phi457, %sw.default ], [ %save_groupNo9.phi.trans.insert, %if.then.29 ], [ %save_groupNo9.phi.trans.insert, %if.then.130 ], [ %save_groupNo9.phi.trans.insert, %if.end.140 ], [ %save_groupNo9.phi.trans.insert, %if.end.82 ], [ %save_groupNo9.phi.trans.insert, %if.end.33 ]
+  %save_groupPos10.pre-phi454 = phi i32* [ %save_groupPos10.pre-phi455, %sw.default ], [ %save_groupPos10.phi.trans.insert, %if.then.29 ], [ %save_groupPos10.phi.trans.insert, %if.then.130 ], [ %save_groupPos10.phi.trans.insert, %if.end.140 ], [ %save_groupPos10.phi.trans.insert, %if.end.82 ], [ %save_groupPos10.phi.trans.insert, %if.end.33 ]
+  %save_nextSym11.pre-phi452 = phi i32* [ %save_nextSym11.pre-phi453, %sw.default ], [ %save_nextSym11.phi.trans.insert, %if.then.29 ], [ %save_nextSym11.phi.trans.insert, %if.then.130 ], [ %save_nextSym11.phi.trans.insert, %if.end.140 ], [ %save_nextSym11.phi.trans.insert, %if.end.82 ], [ %save_nextSym11.phi.trans.insert, %if.end.33 ]
+  %save_nblockMAX12.pre-phi450 = phi i32* [ %save_nblockMAX12.pre-phi451, %sw.default ], [ %save_nblockMAX12.phi.trans.insert, %if.then.29 ], [ %save_nblockMAX12.phi.trans.insert, %if.then.130 ], [ %save_nblockMAX12.phi.trans.insert, %if.end.140 ], [ %save_nblockMAX12.phi.trans.insert, %if.end.82 ], [ %save_nblockMAX12.phi.trans.insert, %if.end.33 ]
+  %save_nblock13.pre-phi448 = phi i32* [ %save_nblock13.pre-phi449, %sw.default ], [ %save_nblock13.phi.trans.insert, %if.then.29 ], [ %save_nblock13.phi.trans.insert, %if.then.130 ], [ %save_nblock13.phi.trans.insert, %if.end.140 ], [ %save_nblock13.phi.trans.insert, %if.end.82 ], [ %save_nblock13.phi.trans.insert, %if.end.33 ]
+  %save_es14.pre-phi446 = phi i32* [ %save_es14.pre-phi447, %sw.default ], [ %save_es14.phi.trans.insert, %if.then.29 ], [ %save_es14.phi.trans.insert, %if.then.130 ], [ %save_es14.phi.trans.insert, %if.end.140 ], [ %save_es14.phi.trans.insert, %if.end.82 ], [ %save_es14.phi.trans.insert, %if.end.33 ]
+  %save_N15.pre-phi444 = phi i32* [ %save_N15.pre-phi445, %sw.default ], [ %save_N15.phi.trans.insert, %if.then.29 ], [ %save_N15.phi.trans.insert, %if.then.130 ], [ %save_N15.phi.trans.insert, %if.end.140 ], [ %save_N15.phi.trans.insert, %if.end.82 ], [ %save_N15.phi.trans.insert, %if.end.33 ]
+  %save_curr16.pre-phi442 = phi i32* [ %save_curr16.pre-phi443, %sw.default ], [ %save_curr16.phi.trans.insert, %if.then.29 ], [ %save_curr16.phi.trans.insert, %if.then.130 ], [ %save_curr16.phi.trans.insert, %if.end.140 ], [ %save_curr16.phi.trans.insert, %if.end.82 ], [ %save_curr16.phi.trans.insert, %if.end.33 ]
+  %save_zt17.pre-phi440 = phi i32* [ %save_zt17.pre-phi441, %sw.default ], [ %save_zt17.phi.trans.insert, %if.then.29 ], [ %save_zt17.phi.trans.insert, %if.then.130 ], [ %save_zt17.phi.trans.insert, %if.end.140 ], [ %save_zt17.phi.trans.insert, %if.end.82 ], [ %save_zt17.phi.trans.insert, %if.end.33 ]
+  %save_zn18.pre-phi438 = phi i32* [ %save_zn18.pre-phi439, %sw.default ], [ %save_zn18.phi.trans.insert, %if.then.29 ], [ %save_zn18.phi.trans.insert, %if.then.130 ], [ %save_zn18.phi.trans.insert, %if.end.140 ], [ %save_zn18.phi.trans.insert, %if.end.82 ], [ %save_zn18.phi.trans.insert, %if.end.33 ]
+  %save_zvec19.pre-phi436 = phi i32* [ %save_zvec19.pre-phi437, %sw.default ], [ %save_zvec19.phi.trans.insert, %if.then.29 ], [ %save_zvec19.phi.trans.insert, %if.then.130 ], [ %save_zvec19.phi.trans.insert, %if.end.140 ], [ %save_zvec19.phi.trans.insert, %if.end.82 ], [ %save_zvec19.phi.trans.insert, %if.end.33 ]
+  %save_zj20.pre-phi434 = phi i32* [ %save_zj20.pre-phi435, %sw.default ], [ %save_zj20.phi.trans.insert, %if.then.29 ], [ %save_zj20.phi.trans.insert, %if.then.130 ], [ %save_zj20.phi.trans.insert, %if.end.140 ], [ %save_zj20.phi.trans.insert, %if.end.82 ], [ %save_zj20.phi.trans.insert, %if.end.33 ]
+  %nblock.1 = phi i32 [ %tmp50, %sw.default ], [ %.pre416, %if.then.29 ], [ 0, %if.then.130 ], [ %.pre416, %if.end.140 ], [ %.pre416, %if.end.82 ], [ %.pre416, %if.end.33 ]
+  %alphaSize.1 = phi i32 [ %tmp42, %sw.default ], [ %.pre408, %if.then.29 ], [ %add179, %if.then.130 ], [ %.pre408, %if.end.140 ], [ %.pre408, %if.end.82 ], [ %.pre408, %if.end.33 ]
+  %retVal.0 = phi i32 [ 0, %sw.default ], [ -5, %if.then.29 ], [ -4, %if.then.130 ], [ 0, %if.end.140 ], [ 0, %if.end.82 ], [ 0, %if.end.33 ]
+  store i32 %tmp58, i32* %save_i, align 4
+  store i32 %tmp59, i32* %save_j3.pre-phi468, align 4
+  store i32 %tmp60, i32* %save_t4.pre-phi466, align 4
+  store i32 %alphaSize.1, i32* %save_alphaSize5.pre-phi464, align 4
+  store i32 %tmp61, i32* %save_nGroups6.pre-phi462, align 4
+  store i32 %tmp62, i32* %save_nSelectors7.pre-phi460, align 4
+  store i32 %tmp63, i32* %save_EOB8.pre-phi458, align 4
+  store i32 %tmp64, i32* %save_groupNo9.pre-phi456, align 4
+  store i32 %tmp65, i32* %save_groupPos10.pre-phi454, align 4
+  store i32 %tmp66, i32* %save_nextSym11.pre-phi452, align 4
+  store i32 %tmp67, i32* %save_nblockMAX12.pre-phi450, align 4
+  store i32 %nblock.1, i32* %save_nblock13.pre-phi448, align 4
+  store i32 %tmp68, i32* %save_es14.pre-phi446, align 4
+  store i32 %tmp69, i32* %save_N15.pre-phi444, align 4
+  store i32 %tmp70, i32* %save_curr16.pre-phi442, align 4
+  store i32 %tmp71, i32* %save_zt17.pre-phi440, align 4
+  store i32 %tmp72, i32* %save_zn18.pre-phi438, align 4
+  store i32 %tmp73, i32* %save_zvec19.pre-phi436, align 4
+  store i32 %tmp74, i32* %save_zj20.pre-phi434, align 4
+  ret i32 %retVal.0
+}
diff --git a/test/CodeGen/AArch64/aarch64-dynamic-stack-layout.ll b/test/CodeGen/AArch64/aarch64-dynamic-stack-layout.ll
index a31c66bad4b..83b9d0a30ae 100644
--- a/test/CodeGen/AArch64/aarch64-dynamic-stack-layout.ll
+++ b/test/CodeGen/AArch64/aarch64-dynamic-stack-layout.ll
@@ -252,11 +252,11 @@ entry:
 ; CHECK: ldr	w[[IARG:[0-9]+]], [x29, #24]
 ; CHECK: ldr	d[[DARG:[0-9]+]], [x29, #40]
 ;   Check correct reservation of 16-byte aligned VLA (size in w0) on stack
-; CHECK: ubfx	x9, x0, #0, #32
+; CHECK: mov	w9, w0
+; CHECK: mov	 x10, sp
 ; CHECK: lsl	x9, x9, #2
 ; CHECK: add	x9, x9, #15
-; CHECK: and	x9, x9, #0xfffffffffffffff0
-; CHECK: mov	 x10, sp
+; CHECK: and	x9, x9, #0x7fffffff0
 ; CHECK: sub	 x[[VLASPTMP:[0-9]+]], x10, x9
 ; CHECK: mov	 sp, x[[VLASPTMP]]
 ;   Check correct access to local variable, through frame pointer
@@ -299,11 +299,11 @@ entry:
 ; CHECK: ldr	w[[IARG:[0-9]+]], [x29, #24]
 ; CHECK: ldr	d[[DARG:[0-9]+]], [x29, #40]
 ;   Check correct reservation of 16-byte aligned VLA (size in w0) on stack
-; CHECK: ubfx	x9, x0, #0, #32
+; CHECK: mov	w9, w0
+; CHECK: mov	 x10, sp
 ; CHECK: lsl	x9, x9, #2
 ; CHECK: add	x9, x9, #15
-; CHECK: and	x9, x9, #0xfffffffffffffff0
-; CHECK: mov	 x10, sp
+; CHECK: and	x9, x9, #0x7fffffff0
 ; CHECK: sub	 x[[VLASPTMP:[0-9]+]], x10, x9
 ; CHECK: mov	 sp, x[[VLASPTMP]]
 ;   Check correct access to local variable, through frame pointer
@@ -361,11 +361,11 @@ entry:
 ; CHECK: ldr	d[[DARG:[0-9]+]], [x29, #40]
 ;   Check correct reservation of 16-byte aligned VLA (size in w0) on stack
 ;   and set-up of base pointer (x19).
-; CHECK: ubfx	x9, x0, #0, #32
+; CHECK: mov	w9, w0
+; CHECK: mov	 x10, sp
 ; CHECK: lsl	x9, x9, #2
 ; CHECK: add	x9, x9, #15
-; CHECK: and	x9, x9, #0xfffffffffffffff0
-; CHECK: mov	 x10, sp
+; CHECK: and	x9, x9, #0x7fffffff0
 ; CHECK: sub	 x[[VLASPTMP:[0-9]+]], x10, x9
 ; CHECK: mov	 sp, x[[VLASPTMP]]
 ;   Check correct access to local variable, through base pointer
@@ -414,11 +414,11 @@ entry:
 ; CHECK: ldr	d[[DARG:[0-9]+]], [x29, #40]
 ;   Check correct reservation of 16-byte aligned VLA (size in w0) on stack
 ;   and set-up of base pointer (x19).
-; CHECK: ubfx	x9, x0, #0, #32
+; CHECK: mov	w9, w0
+; CHECK: mov	 x10, sp
 ; CHECK: lsl	x9, x9, #2
 ; CHECK: add	x9, x9, #15
-; CHECK: and	x9, x9, #0xfffffffffffffff0
-; CHECK: mov	 x10, sp
+; CHECK: and	x9, x9, #0x7fffffff0
 ; CHECK: sub	 x[[VLASPTMP:[0-9]+]], x10, x9
 ; CHECK: mov	 sp, x[[VLASPTMP]]
 ;   Check correct access to local variable, through base pointer
@@ -465,11 +465,11 @@ entry:
 ; CHECK: ldr	d[[DARG:[0-9]+]], [x29, #40]
 ;   Check correct reservation of 16-byte aligned VLA (size in w0) on stack
 ;   and set-up of base pointer (x19).
-; CHECK: ubfx	x9, x0, #0, #32
+; CHECK: mov	w9, w0
+; CHECK: mov	 x10, sp
 ; CHECK: lsl	x9, x9, #2
 ; CHECK: add	x9, x9, #15
-; CHECK: and	x9, x9, #0xfffffffffffffff0
-; CHECK: mov	 x10, sp
+; CHECK: and	x9, x9, #0x7fffffff0
 ; CHECK: sub	 x[[VLASPTMP:[0-9]+]], x10, x9
 ; CHECK: mov	 sp, x[[VLASPTMP]]
 ;   Check correct access to local variable, through base pointer
@@ -482,6 +482,56 @@ entry:
 ; CHECK: ldp	x20, x19, [sp], #32
 ; CHECK: ret
 
+
+define void @realign_conditional(i1 %b) {
+entry:
+  br i1 %b, label %bb0, label %bb1
+
+bb0:
+  %MyAlloca = alloca i8, i64 64, align 32
+  br label %bb1
+
+bb1:
+  ret void
+}
+
+; CHECK-LABEL: realign_conditional
+; No realignment in the prologue.
+; CHECK-NOT:  and
+; CHECK-NOT:  0xffffffffffffffe0
+; CHECK:  tbz  {{.*}} .[[LABEL:.*]]
+; Stack is realigned in a non-entry BB.
+; CHECK:  sub  [[REG:x[01-9]+]], sp, #64
+; CHECK:  and  sp, [[REG]], #0xffffffffffffffe0
+; CHECK:  .[[LABEL]]:
+; CHECK:  ret
+
+
+define void @realign_conditional2(i1 %b) {
+entry:
+  %tmp = alloca i8, i32 4
+  br i1 %b, label %bb0, label %bb1
+
+bb0:
+  %MyAlloca = alloca i8, i64 64, align 32
+  br label %bb1
+
+bb1:
+  ret void
+}
+
+; CHECK-LABEL: realign_conditional2
+; Extra realignment in the prologue (performance issue).
+; CHECK:  sub  x9, sp, #32            // =32
+; CHECK:  and  sp, x9, #0xffffffffffffffe0
+; CHECK:  mov   x19, sp
+; CHECK:  tbz  {{.*}} .[[LABEL:.*]]
+; Stack is realigned in a non-entry BB.
+; CHECK:  sub  [[REG:x[01-9]+]], sp, #64
+; CHECK:  and  sp, [[REG]], #0xffffffffffffffe0
+; CHECK:  .[[LABEL]]:
+; CHECK:  ret
+
 attributes #0 = { "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
 attributes #1 = { nounwind "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
 
diff --git a/test/CodeGen/AArch64/aarch64-gep-opt.ll b/test/CodeGen/AArch64/aarch64-gep-opt.ll
index ce6c8a06c64..93e2ff14ac7 100644
--- a/test/CodeGen/AArch64/aarch64-gep-opt.ll
+++ b/test/CodeGen/AArch64/aarch64-gep-opt.ll
@@ -1,6 +1,6 @@
-; RUN: llc -O3 -verify-machineinstrs %s -o - | FileCheck %s
-; RUN: llc -O3 -print-after=codegenprepare -mcpu=cyclone < %s >%t 2>&1 && FileCheck --check-prefix=CHECK-NoAA <%t %s
-; RUN: llc -O3 -print-after=codegenprepare -mcpu=cortex-a53 < %s >%t 2>&1 && FileCheck --check-prefix=CHECK-UseAA <%t %s
+; RUN: llc -O3 -aarch64-gep-opt=true -verify-machineinstrs %s -o - | FileCheck %s
+; RUN: llc -O3 -aarch64-gep-opt=true -print-after=codegenprepare -mcpu=cyclone < %s >%t 2>&1 && FileCheck --check-prefix=CHECK-NoAA <%t %s
+; RUN: llc -O3 -aarch64-gep-opt=true -print-after=codegenprepare -mcpu=cortex-a53 < %s >%t 2>&1 && FileCheck --check-prefix=CHECK-UseAA <%t %s
 target datalayout = "e-m:e-i64:64-i128:128-n32:64-S128"
 target triple = "aarch64-linux-gnueabi"
 
diff --git a/test/CodeGen/AArch64/aarch64-interleaved-accesses.ll b/test/CodeGen/AArch64/aarch64-interleaved-accesses.ll
new file mode 100644
index 00000000000..ea3b8fa5573
--- /dev/null
+++ b/test/CodeGen/AArch64/aarch64-interleaved-accesses.ll
@@ -0,0 +1,197 @@
+; RUN: llc -march=aarch64 -aarch64-neon-syntax=generic -lower-interleaved-accesses=true < %s | FileCheck %s
+
+; CHECK-LABEL: load_factor2:
+; CHECK: ld2 { v0.8b, v1.8b }, [x0]
+define <8 x i8> @load_factor2(<16 x i8>* %ptr) {
+  %wide.vec = load <16 x i8>, <16 x i8>* %ptr, align 4
+  %strided.v0 = shufflevector <16 x i8> %wide.vec, <16 x i8> undef, <8 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14>
+  %strided.v1 = shufflevector <16 x i8> %wide.vec, <16 x i8> undef, <8 x i32> <i32 1, i32 3, i32 5, i32 7, i32 9, i32 11, i32 13, i32 15>
+  %add = add nsw <8 x i8> %strided.v0, %strided.v1
+  ret <8 x i8> %add
+}
+
+; CHECK-LABEL: load_factor3:
+; CHECK: ld3 { v0.4s, v1.4s, v2.4s }, [x0]
+define <4 x i32> @load_factor3(i32* %ptr) {
+  %base = bitcast i32* %ptr to <12 x i32>*
+  %wide.vec = load <12 x i32>, <12 x i32>* %base, align 4
+  %strided.v2 = shufflevector <12 x i32> %wide.vec, <12 x i32> undef, <4 x i32> <i32 2, i32 5, i32 8, i32 11>
+  %strided.v1 = shufflevector <12 x i32> %wide.vec, <12 x i32> undef, <4 x i32> <i32 1, i32 4, i32 7, i32 10>
+  %add = add nsw <4 x i32> %strided.v2, %strided.v1
+  ret <4 x i32> %add
+}
+
+; CHECK-LABEL: load_factor4:
+; CHECK: ld4 { v0.4s, v1.4s, v2.4s, v3.4s }, [x0]
+define <4 x i32> @load_factor4(i32* %ptr) {
+  %base = bitcast i32* %ptr to <16 x i32>*
+  %wide.vec = load <16 x i32>, <16 x i32>* %base, align 4
+  %strided.v0 = shufflevector <16 x i32> %wide.vec, <16 x i32> undef, <4 x i32> <i32 0, i32 4, i32 8, i32 12>
+  %strided.v2 = shufflevector <16 x i32> %wide.vec, <16 x i32> undef, <4 x i32> <i32 2, i32 6, i32 10, i32 14>
+  %add = add nsw <4 x i32> %strided.v0, %strided.v2
+  ret <4 x i32> %add
+}
+
+; CHECK-LABEL: store_factor2:
+; CHECK: st2 { v0.8b, v1.8b }, [x0]
+define void @store_factor2(<16 x i8>* %ptr, <8 x i8> %v0, <8 x i8> %v1) {
+  %interleaved.vec = shufflevector <8 x i8> %v0, <8 x i8> %v1, <16 x i32> <i32 0, i32 8, i32 1, i32 9, i32 2, i32 10, i32 3, i32 11, i32 4, i32 12, i32 5, i32 13, i32 6, i32 14, i32 7, i32 15>
+  store <16 x i8> %interleaved.vec, <16 x i8>* %ptr, align 4
+  ret void
+}
+
+; CHECK-LABEL: store_factor3:
+; CHECK: st3 { v0.4s, v1.4s, v2.4s }, [x0]
+define void @store_factor3(i32* %ptr, <4 x i32> %v0, <4 x i32> %v1, <4 x i32> %v2) {
+  %base = bitcast i32* %ptr to <12 x i32>*
+  %v0_v1 = shufflevector <4 x i32> %v0, <4 x i32> %v1, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+  %v2_u = shufflevector <4 x i32> %v2, <4 x i32> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef>
+  %interleaved.vec = shufflevector <8 x i32> %v0_v1, <8 x i32> %v2_u, <12 x i32> <i32 0, i32 4, i32 8, i32 1, i32 5, i32 9, i32 2, i32 6, i32 10, i32 3, i32 7, i32 11>
+  store <12 x i32> %interleaved.vec, <12 x i32>* %base, align 4
+  ret void
+}
+
+; CHECK-LABEL: store_factor4:
+; CHECK: st4 { v0.4s, v1.4s, v2.4s, v3.4s }, [x0]
+define void @store_factor4(i32* %ptr, <4 x i32> %v0, <4 x i32> %v1, <4 x i32> %v2, <4 x i32> %v3) {
+  %base = bitcast i32* %ptr to <16 x i32>*
+  %v0_v1 = shufflevector <4 x i32> %v0, <4 x i32> %v1, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+  %v2_v3 = shufflevector <4 x i32> %v2, <4 x i32> %v3, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+  %interleaved.vec = shufflevector <8 x i32> %v0_v1, <8 x i32> %v2_v3, <16 x i32> <i32 0, i32 4, i32 8, i32 12, i32 1, i32 5, i32 9, i32 13, i32 2, i32 6, i32 10, i32 14, i32 3, i32 7, i32 11, i32 15>
+  store <16 x i32> %interleaved.vec, <16 x i32>* %base, align 4
+  ret void
+}
+
+; The following cases test that interleaved access of pointer vectors can be
+; matched to ldN/stN instruction.
+
+; CHECK-LABEL: load_ptrvec_factor2:
+; CHECK: ld2 { v0.2d, v1.2d }, [x0]
+define <2 x i32*> @load_ptrvec_factor2(i32** %ptr) {
+  %base = bitcast i32** %ptr to <4 x i32*>*
+  %wide.vec = load <4 x i32*>, <4 x i32*>* %base, align 4
+  %strided.v0 = shufflevector <4 x i32*> %wide.vec, <4 x i32*> undef, <2 x i32> <i32 0, i32 2>
+  ret <2 x i32*> %strided.v0
+}
+
+; CHECK-LABEL: load_ptrvec_factor3:
+; CHECK: ld3 { v0.2d, v1.2d, v2.2d }, [x0]
+define void @load_ptrvec_factor3(i32** %ptr, <2 x i32*>* %ptr1, <2 x i32*>* %ptr2) {
+  %base = bitcast i32** %ptr to <6 x i32*>*
+  %wide.vec = load <6 x i32*>, <6 x i32*>* %base, align 4
+  %strided.v2 = shufflevector <6 x i32*> %wide.vec, <6 x i32*> undef, <2 x i32> <i32 2, i32 5>
+  store <2 x i32*> %strided.v2, <2 x i32*>* %ptr1
+  %strided.v1 = shufflevector <6 x i32*> %wide.vec, <6 x i32*> undef, <2 x i32> <i32 1, i32 4>
+  store <2 x i32*> %strided.v1, <2 x i32*>* %ptr2
+  ret void
+}
+
+; CHECK-LABEL: load_ptrvec_factor4:
+; CHECK: ld4 { v0.2d, v1.2d, v2.2d, v3.2d }, [x0]
+define void @load_ptrvec_factor4(i32** %ptr, <2 x i32*>* %ptr1, <2 x i32*>* %ptr2) {
+  %base = bitcast i32** %ptr to <8 x i32*>*
+  %wide.vec = load <8 x i32*>, <8 x i32*>* %base, align 4
+  %strided.v1 = shufflevector <8 x i32*> %wide.vec, <8 x i32*> undef, <2 x i32> <i32 1, i32 5>
+  %strided.v3 = shufflevector <8 x i32*> %wide.vec, <8 x i32*> undef, <2 x i32> <i32 3, i32 7>
+  store <2 x i32*> %strided.v1, <2 x i32*>* %ptr1
+  store <2 x i32*> %strided.v3, <2 x i32*>* %ptr2
+  ret void
+}
+
+; CHECK-LABEL: store_ptrvec_factor2:
+; CHECK: st2 { v0.2d, v1.2d }, [x0]
+define void @store_ptrvec_factor2(i32** %ptr, <2 x i32*> %v0, <2 x i32*> %v1) {
+  %base = bitcast i32** %ptr to <4 x i32*>*
+  %interleaved.vec = shufflevector <2 x i32*> %v0, <2 x i32*> %v1, <4 x i32> <i32 0, i32 2, i32 1, i32 3>
+  store <4 x i32*> %interleaved.vec, <4 x i32*>* %base, align 4
+  ret void
+}
+
+; CHECK-LABEL: store_ptrvec_factor3:
+; CHECK: st3 { v0.2d, v1.2d, v2.2d }, [x0]
+define void @store_ptrvec_factor3(i32** %ptr, <2 x i32*> %v0, <2 x i32*> %v1, <2 x i32*> %v2) {
+  %base = bitcast i32** %ptr to <6 x i32*>*
+  %v0_v1 = shufflevector <2 x i32*> %v0, <2 x i32*> %v1, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %v2_u = shufflevector <2 x i32*> %v2, <2 x i32*> undef, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
+  %interleaved.vec = shufflevector <4 x i32*> %v0_v1, <4 x i32*> %v2_u, <6 x i32> <i32 0, i32 2, i32 4, i32 1, i32 3, i32 5>
+  store <6 x i32*> %interleaved.vec, <6 x i32*>* %base, align 4
+  ret void
+}
+
+; CHECK-LABEL: store_ptrvec_factor4:
+; CHECK: st4 { v0.2d, v1.2d, v2.2d, v3.2d }, [x0]
+define void @store_ptrvec_factor4(i32* %ptr, <2 x i32*> %v0, <2 x i32*> %v1, <2 x i32*> %v2, <2 x i32*> %v3) {
+  %base = bitcast i32* %ptr to <8 x i32*>*
+  %v0_v1 = shufflevector <2 x i32*> %v0, <2 x i32*> %v1, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %v2_v3 = shufflevector <2 x i32*> %v2, <2 x i32*> %v3, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %interleaved.vec = shufflevector <4 x i32*> %v0_v1, <4 x i32*> %v2_v3, <8 x i32> <i32 0, i32 2, i32 4, i32 6, i32 1, i32 3, i32 5, i32 7>
+  store <8 x i32*> %interleaved.vec, <8 x i32*>* %base, align 4
+  ret void
+}
+
+; Following cases check that shuffle maskes with undef indices can be matched
+; into ldN/stN instruction.
+
+; CHECK-LABEL: load_undef_mask_factor2:
+; CHECK: ld2 { v0.4s, v1.4s }, [x0]
+define <4 x i32> @load_undef_mask_factor2(i32* %ptr) {
+  %base = bitcast i32* %ptr to <8 x i32>*
+  %wide.vec = load <8 x i32>, <8 x i32>* %base, align 4
+  %strided.v0 = shufflevector <8 x i32> %wide.vec, <8 x i32> undef, <4 x i32> <i32 undef, i32 2, i32 undef, i32 6>
+  %strided.v1 = shufflevector <8 x i32> %wide.vec, <8 x i32> undef, <4 x i32> <i32 undef, i32 3, i32 undef, i32 7>
+  %add = add nsw <4 x i32> %strided.v0, %strided.v1
+  ret <4 x i32> %add
+}
+
+; CHECK-LABEL: load_undef_mask_factor3:
+; CHECK: ld3 { v0.4s, v1.4s, v2.4s }, [x0]
+define <4 x i32> @load_undef_mask_factor3(i32* %ptr) {
+  %base = bitcast i32* %ptr to <12 x i32>*
+  %wide.vec = load <12 x i32>, <12 x i32>* %base, align 4
+  %strided.v2 = shufflevector <12 x i32> %wide.vec, <12 x i32> undef, <4 x i32> <i32 2, i32 undef, i32 undef, i32 undef>
+  %strided.v1 = shufflevector <12 x i32> %wide.vec, <12 x i32> undef, <4 x i32> <i32 1, i32 4, i32 7, i32 10>
+  %add = add nsw <4 x i32> %strided.v2, %strided.v1
+  ret <4 x i32> %add
+}
+
+; CHECK-LABEL: load_undef_mask_factor4:
+; CHECK: ld4 { v0.4s, v1.4s, v2.4s, v3.4s }, [x0]
+define <4 x i32> @load_undef_mask_factor4(i32* %ptr) {
+  %base = bitcast i32* %ptr to <16 x i32>*
+  %wide.vec = load <16 x i32>, <16 x i32>* %base, align 4
+  %strided.v0 = shufflevector <16 x i32> %wide.vec, <16 x i32> undef, <4 x i32> <i32 0, i32 4, i32 undef, i32 undef>
+  %strided.v2 = shufflevector <16 x i32> %wide.vec, <16 x i32> undef, <4 x i32> <i32 2, i32 6, i32 undef, i32 undef>
+  %add = add nsw <4 x i32> %strided.v0, %strided.v2
+  ret <4 x i32> %add
+}
+
+; CHECK-LABEL: store_undef_mask_factor2:
+; CHECK: st2 { v0.4s, v1.4s }, [x0]
+define void @store_undef_mask_factor2(i32* %ptr, <4 x i32> %v0, <4 x i32> %v1) {
+  %base = bitcast i32* %ptr to <8 x i32>*
+  %interleaved.vec = shufflevector <4 x i32> %v0, <4 x i32> %v1, <8 x i32> <i32 undef, i32 undef, i32 undef, i32 undef, i32 2, i32 6, i32 3, i32 7>
+  store <8 x i32> %interleaved.vec, <8 x i32>* %base, align 4
+  ret void
+}
+
+; CHECK-LABEL: store_undef_mask_factor3:
+; CHECK: st3 { v0.4s, v1.4s, v2.4s }, [x0]
+define void @store_undef_mask_factor3(i32* %ptr, <4 x i32> %v0, <4 x i32> %v1, <4 x i32> %v2) {
+  %base = bitcast i32* %ptr to <12 x i32>*
+  %v0_v1 = shufflevector <4 x i32> %v0, <4 x i32> %v1, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+  %v2_u = shufflevector <4 x i32> %v2, <4 x i32> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef>
+  %interleaved.vec = shufflevector <8 x i32> %v0_v1, <8 x i32> %v2_u, <12 x i32> <i32 0, i32 4, i32 undef, i32 1, i32 undef, i32 9, i32 2, i32 6, i32 10, i32 3, i32 7, i32 11>
+  store <12 x i32> %interleaved.vec, <12 x i32>* %base, align 4
+  ret void
+}
+
+; CHECK-LABEL: store_undef_mask_factor4:
+; CHECK: st4 { v0.4s, v1.4s, v2.4s, v3.4s }, [x0]
+define void @store_undef_mask_factor4(i32* %ptr, <4 x i32> %v0, <4 x i32> %v1, <4 x i32> %v2, <4 x i32> %v3) {
+  %base = bitcast i32* %ptr to <16 x i32>*
+  %v0_v1 = shufflevector <4 x i32> %v0, <4 x i32> %v1, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+  %v2_v3 = shufflevector <4 x i32> %v2, <4 x i32> %v3, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+  %interleaved.vec = shufflevector <8 x i32> %v0_v1, <8 x i32> %v2_v3, <16 x i32> <i32 0, i32 4, i32 8, i32 undef, i32 undef, i32 5, i32 9, i32 13, i32 2, i32 6, i32 10, i32 14, i32 3, i32 7, i32 11, i32 15>
+  store <16 x i32> %interleaved.vec, <16 x i32>* %base, align 4
+  ret void
+}
diff --git a/test/CodeGen/AArch64/addsub_ext.ll b/test/CodeGen/AArch64/addsub_ext.ll
index f0c7572ebf1..f30ab89f238 100644
--- a/test/CodeGen/AArch64/addsub_ext.ll
+++ b/test/CodeGen/AArch64/addsub_ext.ll
@@ -80,6 +80,64 @@ end:
     ret void
 }
 
+define void @sub_i8rhs() minsize {
+; CHECK-LABEL: sub_i8rhs:
+    %val8_tmp = load i8, i8* @var8
+    %lhs32 = load i32, i32* @var32
+    %lhs64 = load i64, i64* @var64
+
+    ; Need this to prevent extension upon load and give a vanilla i8 operand.
+    %val8 = add i8 %val8_tmp, 123
+
+
+; Zero-extending to 32-bits
+    %rhs32_zext = zext i8 %val8 to i32
+    %res32_zext = sub i32 %lhs32, %rhs32_zext
+    store volatile i32 %res32_zext, i32* @var32
+; CHECK: sub {{w[0-9]+}}, {{w[0-9]+}}, {{w[0-9]+}}, uxtb
+
+   %rhs32_zext_shift = shl i32 %rhs32_zext, 3
+   %res32_zext_shift = sub i32 %lhs32, %rhs32_zext_shift
+   store volatile i32 %res32_zext_shift, i32* @var32
+; CHECK: sub {{w[0-9]+}}, {{w[0-9]+}}, {{w[0-9]+}}, uxtb #3
+
+
+; Zero-extending to 64-bits
+    %rhs64_zext = zext i8 %val8 to i64
+    %res64_zext = sub i64 %lhs64, %rhs64_zext
+    store volatile i64 %res64_zext, i64* @var64
+; CHECK: sub {{x[0-9]+}}, {{x[0-9]+}}, {{w[0-9]+}}, uxtb
+
+   %rhs64_zext_shift = shl i64 %rhs64_zext, 1
+   %res64_zext_shift = sub i64 %lhs64, %rhs64_zext_shift
+   store volatile i64 %res64_zext_shift, i64* @var64
+; CHECK: sub {{x[0-9]+}}, {{x[0-9]+}}, {{w[0-9]+}}, uxtb #1
+
+; Sign-extending to 32-bits
+    %rhs32_sext = sext i8 %val8 to i32
+    %res32_sext = sub i32 %lhs32, %rhs32_sext
+    store volatile i32 %res32_sext, i32* @var32
+; CHECK: sub {{w[0-9]+}}, {{w[0-9]+}}, {{w[0-9]+}}, sxtb
+
+   %rhs32_sext_shift = shl i32 %rhs32_sext, 1
+   %res32_sext_shift = sub i32 %lhs32, %rhs32_sext_shift
+   store volatile i32 %res32_sext_shift, i32* @var32
+; CHECK: sub {{w[0-9]+}}, {{w[0-9]+}}, {{w[0-9]+}}, sxtb #1
+
+; Sign-extending to 64-bits
+    %rhs64_sext = sext i8 %val8 to i64
+    %res64_sext = sub i64 %lhs64, %rhs64_sext
+    store volatile i64 %res64_sext, i64* @var64
+; CHECK: sub {{x[0-9]+}}, {{x[0-9]+}}, {{w[0-9]+}}, sxtb
+
+   %rhs64_sext_shift = shl i64 %rhs64_sext, 4
+   %res64_sext_shift = sub i64 %lhs64, %rhs64_sext_shift
+   store volatile i64 %res64_sext_shift, i64* @var64
+; CHECK: sub {{x[0-9]+}}, {{x[0-9]+}}, {{w[0-9]+}}, sxtb #4
+
+    ret void
+}
+
 define void @addsub_i16rhs() minsize {
 ; CHECK-LABEL: addsub_i16rhs:
     %val16_tmp = load i16, i16* @var16
@@ -155,6 +213,64 @@ end:
     ret void
 }
 
+define void @sub_i16rhs() minsize {
+; CHECK-LABEL: sub_i16rhs:
+    %val16_tmp = load i16, i16* @var16
+    %lhs32 = load i32, i32* @var32
+    %lhs64 = load i64, i64* @var64
+
+    ; Need this to prevent extension upon load and give a vanilla i16 operand.
+    %val16 = add i16 %val16_tmp, 123
+
+
+; Zero-extending to 32-bits
+    %rhs32_zext = zext i16 %val16 to i32
+    %res32_zext = sub i32 %lhs32, %rhs32_zext
+    store volatile i32 %res32_zext, i32* @var32
+; CHECK: sub {{w[0-9]+}}, {{w[0-9]+}}, {{w[0-9]+}}, uxth
+
+   %rhs32_zext_shift = shl i32 %rhs32_zext, 3
+   %res32_zext_shift = sub i32 %lhs32, %rhs32_zext_shift
+   store volatile i32 %res32_zext_shift, i32* @var32
+; CHECK: sub {{w[0-9]+}}, {{w[0-9]+}}, {{w[0-9]+}}, uxth #3
+
+
+; Zero-extending to 64-bits
+    %rhs64_zext = zext i16 %val16 to i64
+    %res64_zext = sub i64 %lhs64, %rhs64_zext
+    store volatile i64 %res64_zext, i64* @var64
+; CHECK: sub {{x[0-9]+}}, {{x[0-9]+}}, {{w[0-9]+}}, uxth
+
+   %rhs64_zext_shift = shl i64 %rhs64_zext, 1
+   %res64_zext_shift = sub i64 %lhs64, %rhs64_zext_shift
+   store volatile i64 %res64_zext_shift, i64* @var64
+; CHECK: sub {{x[0-9]+}}, {{x[0-9]+}}, {{w[0-9]+}}, uxth #1
+
+; Sign-extending to 32-bits
+    %rhs32_sext = sext i16 %val16 to i32
+    %res32_sext = sub i32 %lhs32, %rhs32_sext
+    store volatile i32 %res32_sext, i32* @var32
+; CHECK: sub {{w[0-9]+}}, {{w[0-9]+}}, {{w[0-9]+}}, sxth
+
+   %rhs32_sext_shift = shl i32 %rhs32_sext, 1
+   %res32_sext_shift = sub i32 %lhs32, %rhs32_sext_shift
+   store volatile i32 %res32_sext_shift, i32* @var32
+; CHECK: sub {{w[0-9]+}}, {{w[0-9]+}}, {{w[0-9]+}}, sxth #1
+
+; Sign-extending to 64-bits
+    %rhs64_sext = sext i16 %val16 to i64
+    %res64_sext = sub i64 %lhs64, %rhs64_sext
+    store volatile i64 %res64_sext, i64* @var64
+; CHECK: sub {{x[0-9]+}}, {{x[0-9]+}}, {{w[0-9]+}}, sxth
+
+   %rhs64_sext_shift = shl i64 %rhs64_sext, 4
+   %res64_sext_shift = sub i64 %lhs64, %rhs64_sext_shift
+   store volatile i64 %res64_sext_shift, i64* @var64
+; CHECK: sub {{x[0-9]+}}, {{x[0-9]+}}, {{w[0-9]+}}, sxth #4
+
+    ret void
+}
+
 ; N.b. we could probably check more here ("add w2, w3, w1, uxtw" for
 ; example), but the remaining instructions are probably not idiomatic
 ; in the face of "add/sub (shifted register)" so I don't intend to.
@@ -187,3 +303,33 @@ define void @addsub_i32rhs() minsize {
 
     ret void
 }
+
+define void @sub_i32rhs() minsize {
+; CHECK-LABEL: sub_i32rhs:
+    %val32_tmp = load i32, i32* @var32
+    %lhs64 = load i64, i64* @var64
+
+    %val32 = add i32 %val32_tmp, 123
+
+    %rhs64_zext = zext i32 %val32 to i64
+    %res64_zext = sub i64 %lhs64, %rhs64_zext
+    store volatile i64 %res64_zext, i64* @var64
+; CHECK: sub {{x[0-9]+}}, {{x[0-9]+}}, {{w[0-9]+}}, uxtw
+
+    %rhs64_zext_shift = shl i64 %rhs64_zext, 2
+    %res64_zext_shift = sub i64 %lhs64, %rhs64_zext_shift
+    store volatile i64 %res64_zext_shift, i64* @var64
+; CHECK: sub {{x[0-9]+}}, {{x[0-9]+}}, {{w[0-9]+}}, uxtw #2
+
+    %rhs64_sext = sext i32 %val32 to i64
+    %res64_sext = sub i64 %lhs64, %rhs64_sext
+    store volatile i64 %res64_sext, i64* @var64
+; CHECK: sub {{x[0-9]+}}, {{x[0-9]+}}, {{w[0-9]+}}, sxtw
+
+    %rhs64_sext_shift = shl i64 %rhs64_sext, 2
+    %res64_sext_shift = sub i64 %lhs64, %rhs64_sext_shift
+    store volatile i64 %res64_sext_shift, i64* @var64
+; CHECK: sub {{x[0-9]+}}, {{x[0-9]+}}, {{w[0-9]+}}, sxtw #2
+
+    ret void
+}
diff --git a/test/CodeGen/AArch64/arm64-2011-03-17-AsmPrinterCrash.ll b/test/CodeGen/AArch64/arm64-2011-03-17-AsmPrinterCrash.ll
index 330fccc4201..173a440326a 100644
--- a/test/CodeGen/AArch64/arm64-2011-03-17-AsmPrinterCrash.ll
+++ b/test/CodeGen/AArch64/arm64-2011-03-17-AsmPrinterCrash.ll
@@ -11,7 +11,7 @@ if.then24:                                        ; preds = %entry
   unreachable
 
 if.else295:                                       ; preds = %entry
-  call void @llvm.dbg.declare(metadata i32* %do_tab_convert, metadata !16, metadata !MDExpression()), !dbg !18
+  call void @llvm.dbg.declare(metadata i32* %do_tab_convert, metadata !16, metadata !DIExpression()), !dbg !18
   store i32 0, i32* %do_tab_convert, align 4, !dbg !19
   unreachable
 }
@@ -21,25 +21,25 @@ declare void @llvm.dbg.declare(metadata, metadata, metadata) nounwind readnone
 !llvm.dbg.gv = !{!0}
 !llvm.dbg.sp = !{!1, !7, !10, !11, !12}
 
-!0 = !MDGlobalVariable(name: "vsplive", line: 617, isLocal: true, isDefinition: true, scope: !1, file: !2, type: !6)
-!1 = !MDSubprogram(name: "drt_vsprintf", line: 616, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, file: !20, scope: !2, type: !4)
-!2 = !MDFile(filename: "print.i", directory: "/Volumes/Ebi/echeng/radars/r9146594")
-!3 = !MDCompileUnit(language: DW_LANG_C99, producer: "clang version 3.0 (http://llvm.org/git/clang.git git:/git/puzzlebox/clang.git/ c4d1aea01c4444eb81bdbf391f1be309127c3cf1)", isOptimized: true, emissionKind: 0, file: !20, enums: !21, retainedTypes: !21)
-!4 = !MDSubroutineType(types: !5)
+!0 = !DIGlobalVariable(name: "vsplive", line: 617, isLocal: true, isDefinition: true, scope: !1, file: !2, type: !6)
+!1 = !DISubprogram(name: "drt_vsprintf", line: 616, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, file: !20, scope: !2, type: !4)
+!2 = !DIFile(filename: "print.i", directory: "/Volumes/Ebi/echeng/radars/r9146594")
+!3 = !DICompileUnit(language: DW_LANG_C99, producer: "clang version 3.0 (http://llvm.org/git/clang.git git:/git/puzzlebox/clang.git/ c4d1aea01c4444eb81bdbf391f1be309127c3cf1)", isOptimized: true, emissionKind: 0, file: !20, enums: !21, retainedTypes: !21)
+!4 = !DISubroutineType(types: !5)
 !5 = !{!6}
-!6 = !MDBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
-!7 = !MDSubprogram(name: "putc_mem", line: 30, isLocal: true, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, file: !20, scope: !2, type: !8)
-!8 = !MDSubroutineType(types: !9)
+!6 = !DIBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
+!7 = !DISubprogram(name: "putc_mem", line: 30, isLocal: true, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, file: !20, scope: !2, type: !8)
+!8 = !DISubroutineType(types: !9)
 !9 = !{null}
-!10 = !MDSubprogram(name: "print_double", line: 203, isLocal: true, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, file: !20, scope: !2, type: !4)
-!11 = !MDSubprogram(name: "print_number", line: 75, isLocal: true, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, file: !20, scope: !2, type: !4)
-!12 = !MDSubprogram(name: "get_flags", line: 508, isLocal: true, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, file: !20, scope: !2, type: !8)
-!13 = !MDLocation(line: 653, column: 5, scope: !14)
-!14 = distinct !MDLexicalBlock(line: 652, column: 35, file: !20, scope: !15)
-!15 = distinct !MDLexicalBlock(line: 616, column: 1, file: !20, scope: !1)
-!16 = !MDLocalVariable(tag: DW_TAG_auto_variable, name: "do_tab_convert", line: 853, scope: !17, file: !2, type: !6)
-!17 = distinct !MDLexicalBlock(line: 850, column: 12, file: !20, scope: !14)
-!18 = !MDLocation(line: 853, column: 11, scope: !17)
-!19 = !MDLocation(line: 853, column: 29, scope: !17)
-!20 = !MDFile(filename: "print.i", directory: "/Volumes/Ebi/echeng/radars/r9146594")
+!10 = !DISubprogram(name: "print_double", line: 203, isLocal: true, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, file: !20, scope: !2, type: !4)
+!11 = !DISubprogram(name: "print_number", line: 75, isLocal: true, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, file: !20, scope: !2, type: !4)
+!12 = !DISubprogram(name: "get_flags", line: 508, isLocal: true, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, file: !20, scope: !2, type: !8)
+!13 = !DILocation(line: 653, column: 5, scope: !14)
+!14 = distinct !DILexicalBlock(line: 652, column: 35, file: !20, scope: !15)
+!15 = distinct !DILexicalBlock(line: 616, column: 1, file: !20, scope: !1)
+!16 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "do_tab_convert", line: 853, scope: !17, file: !2, type: !6)
+!17 = distinct !DILexicalBlock(line: 850, column: 12, file: !20, scope: !14)
+!18 = !DILocation(line: 853, column: 11, scope: !17)
+!19 = !DILocation(line: 853, column: 29, scope: !17)
+!20 = !DIFile(filename: "print.i", directory: "/Volumes/Ebi/echeng/radars/r9146594")
 !21 = !{i32 0}
diff --git a/test/CodeGen/AArch64/arm64-aapcs.ll b/test/CodeGen/AArch64/arm64-aapcs.ll
index 390a3c75ff8..f345acf453d 100644
--- a/test/CodeGen/AArch64/arm64-aapcs.ll
+++ b/test/CodeGen/AArch64/arm64-aapcs.ll
@@ -2,6 +2,7 @@
 
 @var = global i32 0, align 4
 
+; CHECK-LABEL: @test_i128_align
 define i128 @test_i128_align(i32, i128 %arg, i32 %after) {
   store i32 %after, i32* @var, align 4
 ; CHECK: str w4, [{{x[0-9]+}}, :lo12:var]
@@ -11,6 +12,16 @@ define i128 @test_i128_align(i32, i128 %arg, i32 %after) {
 ; CHECK: mov x1, x3
 }
 
+; CHECK-LABEL: @test_i64x2_align
+define [2 x i64] @test_i64x2_align(i32, [2 x i64] %arg, i32 %after) {
+  store i32 %after, i32* @var, align 4
+; CHECK: str w3, [{{x[0-9]+}}, :lo12:var]
+
+  ret [2 x i64] %arg
+; CHECK: mov x0, x1
+; CHECK: mov x1, x2
+}
+
 @var64 = global i64 0, align 8
 
   ; Check stack slots are 64-bit at all times.
@@ -67,8 +78,8 @@ define void @test_extension(i1 %bool, i8 %char, i16 %short, i32 %int) {
 
   %ext_int = zext i32 %int to i64
   store volatile i64 %ext_int, i64* @var64
-; CHECK: ubfx [[EXT:x[0-9]+]], x3, #0, #32
-; CHECK: str [[EXT]], [{{x[0-9]+}}, :lo12:var64]
+; CHECK: mov w[[EXT:[0-9]+]], w3
+; CHECK: str x[[EXT]], [{{x[0-9]+}}, :lo12:var64]
 
   ret void
 }
diff --git a/test/CodeGen/AArch64/arm64-atomic.ll b/test/CodeGen/AArch64/arm64-atomic.ll
index 451e7e02f99..0824bd881a9 100644
--- a/test/CodeGen/AArch64/arm64-atomic.ll
+++ b/test/CodeGen/AArch64/arm64-atomic.ll
@@ -1,37 +1,65 @@
-; RUN: llc < %s -march=arm64 -verify-machineinstrs -mcpu=cyclone | FileCheck %s
+; RUN: llc < %s -march=arm64 -asm-verbose=false -verify-machineinstrs -mcpu=cyclone | FileCheck %s
 
-define i32 @val_compare_and_swap(i32* %p) {
+define i32 @val_compare_and_swap(i32* %p, i32 %cmp, i32 %new) #0 {
 ; CHECK-LABEL: val_compare_and_swap:
-; CHECK: orr    [[NEWVAL_REG:w[0-9]+]], wzr, #0x4
-; CHECK: [[LABEL:.?LBB[0-9]+_[0-9]+]]:
-; CHECK: ldaxr   [[RESULT:w[0-9]+]], [x0]
-; CHECK: cmp    [[RESULT]], #7
-; CHECK: b.ne   [[LABEL2:.?LBB[0-9]+_[0-9]+]]
-; CHECK: stxr   [[SCRATCH_REG:w[0-9]+]], [[NEWVAL_REG]], [x0]
-; CHECK: cbnz   [[SCRATCH_REG]], [[LABEL]]
-; CHECK: [[LABEL2]]:
-  %pair = cmpxchg i32* %p, i32 7, i32 4 acquire acquire
+; CHECK-NEXT: [[LABEL:.?LBB[0-9]+_[0-9]+]]:
+; CHECK-NEXT: ldaxr  [[RESULT:w[0-9]+]], [x0]
+; CHECK-NEXT: cmp    [[RESULT]], w1
+; CHECK-NEXT: b.ne   [[LABEL2:.?LBB[0-9]+_[0-9]+]]
+; CHECK-NEXT: stxr   [[SCRATCH_REG:w[0-9]+]], w2, [x0]
+; CHECK-NEXT: cbnz   [[SCRATCH_REG]], [[LABEL]]
+; CHECK-NEXT: [[LABEL2]]:
+  %pair = cmpxchg i32* %p, i32 %cmp, i32 %new acquire acquire
   %val = extractvalue { i32, i1 } %pair, 0
   ret i32 %val
 }
 
-define i64 @val_compare_and_swap_64(i64* %p) {
+define i32 @val_compare_and_swap_from_load(i32* %p, i32 %cmp, i32* %pnew) #0 {
+; CHECK-LABEL: val_compare_and_swap_from_load:
+; CHECK-NEXT: ldr    [[NEW:w[0-9]+]], [x2]
+; CHECK-NEXT: [[LABEL:.?LBB[0-9]+_[0-9]+]]:
+; CHECK-NEXT: ldaxr  [[RESULT:w[0-9]+]], [x0]
+; CHECK-NEXT: cmp    [[RESULT]], w1
+; CHECK-NEXT: b.ne   [[LABEL2:.?LBB[0-9]+_[0-9]+]]
+; CHECK-NEXT: stxr   [[SCRATCH_REG:w[0-9]+]], [[NEW]], [x0]
+; CHECK-NEXT: cbnz   [[SCRATCH_REG]], [[LABEL]]
+; CHECK-NEXT: [[LABEL2]]:
+  %new = load i32, i32* %pnew
+  %pair = cmpxchg i32* %p, i32 %cmp, i32 %new acquire acquire
+  %val = extractvalue { i32, i1 } %pair, 0
+  ret i32 %val
+}
+
+define i32 @val_compare_and_swap_rel(i32* %p, i32 %cmp, i32 %new) #0 {
+; CHECK-LABEL: val_compare_and_swap_rel:
+; CHECK-NEXT: [[LABEL:.?LBB[0-9]+_[0-9]+]]:
+; CHECK-NEXT: ldaxr  [[RESULT:w[0-9]+]], [x0]
+; CHECK-NEXT: cmp    [[RESULT]], w1
+; CHECK-NEXT: b.ne   [[LABEL2:.?LBB[0-9]+_[0-9]+]]
+; CHECK-NEXT: stlxr  [[SCRATCH_REG:w[0-9]+]], w2, [x0]
+; CHECK-NEXT: cbnz   [[SCRATCH_REG]], [[LABEL]]
+; CHECK-NEXT: [[LABEL2]]:
+  %pair = cmpxchg i32* %p, i32 %cmp, i32 %new acq_rel monotonic
+  %val = extractvalue { i32, i1 } %pair, 0
+  ret i32 %val
+}
+
+define i64 @val_compare_and_swap_64(i64* %p, i64 %cmp, i64 %new) #0 {
 ; CHECK-LABEL: val_compare_and_swap_64:
-; CHECK: orr    w[[NEWVAL_REG:[0-9]+]], wzr, #0x4
-; CHECK: [[LABEL:.?LBB[0-9]+_[0-9]+]]:
-; CHECK: ldxr   [[RESULT:x[0-9]+]], [x0]
-; CHECK: cmp    [[RESULT]], #7
-; CHECK: b.ne   [[LABEL2:.?LBB[0-9]+_[0-9]+]]
-; CHECK-NOT: stxr x[[NEWVAL_REG]], x[[NEWVAL_REG]]
-; CHECK: stxr   [[SCRATCH_REG:w[0-9]+]], x[[NEWVAL_REG]], [x0]
-; CHECK: cbnz   [[SCRATCH_REG]], [[LABEL]]
-; CHECK: [[LABEL2]]:
-  %pair = cmpxchg i64* %p, i64 7, i64 4 monotonic monotonic
+; CHECK-NEXT: mov    x[[ADDR:[0-9]+]], x0
+; CHECK-NEXT: [[LABEL:.?LBB[0-9]+_[0-9]+]]:
+; CHECK-NEXT: ldxr   [[RESULT:x[0-9]+]], [x[[ADDR]]]
+; CHECK-NEXT: cmp    [[RESULT]], x1
+; CHECK-NEXT: b.ne   [[LABEL2:.?LBB[0-9]+_[0-9]+]]
+; CHECK-NEXT: stxr   [[SCRATCH_REG:w[0-9]+]], x2, [x[[ADDR]]]
+; CHECK-NEXT: cbnz   [[SCRATCH_REG]], [[LABEL]]
+; CHECK-NEXT: [[LABEL2]]:
+  %pair = cmpxchg i64* %p, i64 %cmp, i64 %new monotonic monotonic
   %val = extractvalue { i64, i1 } %pair, 0
   ret i64 %val
 }
 
-define i32 @fetch_and_nand(i32* %p) {
+define i32 @fetch_and_nand(i32* %p) #0 {
 ; CHECK-LABEL: fetch_and_nand:
 ; CHECK: [[LABEL:.?LBB[0-9]+_[0-9]+]]:
 ; CHECK: ldxr   w[[DEST_REG:[0-9]+]], [x0]
@@ -45,7 +73,7 @@ define i32 @fetch_and_nand(i32* %p) {
   ret i32 %val
 }
 
-define i64 @fetch_and_nand_64(i64* %p) {
+define i64 @fetch_and_nand_64(i64* %p) #0 {
 ; CHECK-LABEL: fetch_and_nand_64:
 ; CHECK: mov    x[[ADDR:[0-9]+]], x0
 ; CHECK: [[LABEL:.?LBB[0-9]+_[0-9]+]]:
@@ -59,7 +87,7 @@ define i64 @fetch_and_nand_64(i64* %p) {
   ret i64 %val
 }
 
-define i32 @fetch_and_or(i32* %p) {
+define i32 @fetch_and_or(i32* %p) #0 {
 ; CHECK-LABEL: fetch_and_or:
 ; CHECK: movz   [[OLDVAL_REG:w[0-9]+]], #0x5
 ; CHECK: [[LABEL:.?LBB[0-9]+_[0-9]+]]:
@@ -73,7 +101,7 @@ define i32 @fetch_and_or(i32* %p) {
   ret i32 %val
 }
 
-define i64 @fetch_and_or_64(i64* %p) {
+define i64 @fetch_and_or_64(i64* %p) #0 {
 ; CHECK: fetch_and_or_64:
 ; CHECK: mov    x[[ADDR:[0-9]+]], x0
 ; CHECK: [[LABEL:.?LBB[0-9]+_[0-9]+]]:
@@ -85,35 +113,35 @@ define i64 @fetch_and_or_64(i64* %p) {
   ret i64 %val
 }
 
-define void @acquire_fence() {
+define void @acquire_fence() #0 {
    fence acquire
    ret void
    ; CHECK-LABEL: acquire_fence:
    ; CHECK: dmb ishld
 }
 
-define void @release_fence() {
+define void @release_fence() #0 {
    fence release
    ret void
    ; CHECK-LABEL: release_fence:
    ; CHECK: dmb ish{{$}}
 }
 
-define void @seq_cst_fence() {
+define void @seq_cst_fence() #0 {
    fence seq_cst
    ret void
    ; CHECK-LABEL: seq_cst_fence:
    ; CHECK: dmb ish{{$}}
 }
 
-define i32 @atomic_load(i32* %p) {
+define i32 @atomic_load(i32* %p) #0 {
    %r = load atomic i32, i32* %p seq_cst, align 4
    ret i32 %r
    ; CHECK-LABEL: atomic_load:
    ; CHECK: ldar
 }
 
-define i8 @atomic_load_relaxed_8(i8* %p, i32 %off32) {
+define i8 @atomic_load_relaxed_8(i8* %p, i32 %off32) #0 {
 ; CHECK-LABEL: atomic_load_relaxed_8:
   %ptr_unsigned = getelementptr i8, i8* %p, i32 4095
   %val_unsigned = load atomic i8, i8* %ptr_unsigned monotonic, align 1
@@ -138,7 +166,7 @@ define i8 @atomic_load_relaxed_8(i8* %p, i32 %off32) {
   ret i8 %tot3
 }
 
-define i16 @atomic_load_relaxed_16(i16* %p, i32 %off32) {
+define i16 @atomic_load_relaxed_16(i16* %p, i32 %off32) #0 {
 ; CHECK-LABEL: atomic_load_relaxed_16:
   %ptr_unsigned = getelementptr i16, i16* %p, i32 4095
   %val_unsigned = load atomic i16, i16* %ptr_unsigned monotonic, align 2
@@ -163,7 +191,7 @@ define i16 @atomic_load_relaxed_16(i16* %p, i32 %off32) {
   ret i16 %tot3
 }
 
-define i32 @atomic_load_relaxed_32(i32* %p, i32 %off32) {
+define i32 @atomic_load_relaxed_32(i32* %p, i32 %off32) #0 {
 ; CHECK-LABEL: atomic_load_relaxed_32:
   %ptr_unsigned = getelementptr i32, i32* %p, i32 4095
   %val_unsigned = load atomic i32, i32* %ptr_unsigned monotonic, align 4
@@ -188,7 +216,7 @@ define i32 @atomic_load_relaxed_32(i32* %p, i32 %off32) {
   ret i32 %tot3
 }
 
-define i64 @atomic_load_relaxed_64(i64* %p, i32 %off32) {
+define i64 @atomic_load_relaxed_64(i64* %p, i32 %off32) #0 {
 ; CHECK-LABEL: atomic_load_relaxed_64:
   %ptr_unsigned = getelementptr i64, i64* %p, i32 4095
   %val_unsigned = load atomic i64, i64* %ptr_unsigned monotonic, align 8
@@ -214,14 +242,14 @@ define i64 @atomic_load_relaxed_64(i64* %p, i32 %off32) {
 }
 
 
-define void @atomc_store(i32* %p) {
+define void @atomc_store(i32* %p) #0 {
    store atomic i32 4, i32* %p seq_cst, align 4
    ret void
    ; CHECK-LABEL: atomc_store:
    ; CHECK: stlr
 }
 
-define void @atomic_store_relaxed_8(i8* %p, i32 %off32, i8 %val) {
+define void @atomic_store_relaxed_8(i8* %p, i32 %off32, i8 %val) #0 {
 ; CHECK-LABEL: atomic_store_relaxed_8:
   %ptr_unsigned = getelementptr i8, i8* %p, i32 4095
   store atomic i8 %val, i8* %ptr_unsigned monotonic, align 1
@@ -243,7 +271,7 @@ define void @atomic_store_relaxed_8(i8* %p, i32 %off32, i8 %val) {
   ret void
 }
 
-define void @atomic_store_relaxed_16(i16* %p, i32 %off32, i16 %val) {
+define void @atomic_store_relaxed_16(i16* %p, i32 %off32, i16 %val) #0 {
 ; CHECK-LABEL: atomic_store_relaxed_16:
   %ptr_unsigned = getelementptr i16, i16* %p, i32 4095
   store atomic i16 %val, i16* %ptr_unsigned monotonic, align 2
@@ -265,7 +293,7 @@ define void @atomic_store_relaxed_16(i16* %p, i32 %off32, i16 %val) {
   ret void
 }
 
-define void @atomic_store_relaxed_32(i32* %p, i32 %off32, i32 %val) {
+define void @atomic_store_relaxed_32(i32* %p, i32 %off32, i32 %val) #0 {
 ; CHECK-LABEL: atomic_store_relaxed_32:
   %ptr_unsigned = getelementptr i32, i32* %p, i32 4095
   store atomic i32 %val, i32* %ptr_unsigned monotonic, align 4
@@ -287,7 +315,7 @@ define void @atomic_store_relaxed_32(i32* %p, i32 %off32, i32 %val) {
   ret void
 }
 
-define void @atomic_store_relaxed_64(i64* %p, i32 %off32, i64 %val) {
+define void @atomic_store_relaxed_64(i64* %p, i32 %off32, i64 %val) #0 {
 ; CHECK-LABEL: atomic_store_relaxed_64:
   %ptr_unsigned = getelementptr i64, i64* %p, i32 4095
   store atomic i64 %val, i64* %ptr_unsigned monotonic, align 8
@@ -333,3 +361,5 @@ return:                                           ; preds = %if.else, %entry
   %retval.0 = phi i32 [ %add.i2, %if.else ], [ %add.i, %entry ]
   ret i32 %retval.0
 }
+
+attributes #0 = { nounwind }
diff --git a/test/CodeGen/AArch64/arm64-big-endian-eh.ll b/test/CodeGen/AArch64/arm64-big-endian-eh.ll
index a51703a8fc4..77d52e32d3a 100644
--- a/test/CodeGen/AArch64/arm64-big-endian-eh.ll
+++ b/test/CodeGen/AArch64/arm64-big-endian-eh.ll
@@ -14,13 +14,13 @@
 ; }
 ;}
 
-define void @_Z4testii(i32 %a, i32 %b) #0 {
+define void @_Z4testii(i32 %a, i32 %b) #0 personality i8* bitcast (i32 (...)* @__gxx_personality_v0 to i8*) {
 entry:
   invoke void @_Z3fooi(i32 %a)
           to label %try.cont unwind label %lpad
 
 lpad:                                             ; preds = %entry
-  %0 = landingpad { i8*, i32 } personality i8* bitcast (i32 (...)* @__gxx_personality_v0 to i8*)
+  %0 = landingpad { i8*, i32 }
           catch i8* null
   %1 = extractvalue { i8*, i32 } %0, 0
   %2 = tail call i8* @__cxa_begin_catch(i8* %1) #2
@@ -35,7 +35,7 @@ try.cont:                                         ; preds = %entry, %invoke.cont
   ret void
 
 lpad1:                                            ; preds = %lpad
-  %3 = landingpad { i8*, i32 } personality i8* bitcast (i32 (...)* @__gxx_personality_v0 to i8*)
+  %3 = landingpad { i8*, i32 }
           cleanup
   invoke void @__cxa_end_catch()
           to label %eh.resume unwind label %terminate.lpad
@@ -44,7 +44,7 @@ eh.resume:                                        ; preds = %lpad1
   resume { i8*, i32 } %3
 
 terminate.lpad:                                   ; preds = %lpad1
-  %4 = landingpad { i8*, i32 } personality i8* bitcast (i32 (...)* @__gxx_personality_v0 to i8*)
+  %4 = landingpad { i8*, i32 }
           catch i8* null
   %5 = extractvalue { i8*, i32 } %4, 0
   tail call void @__clang_call_terminate(i8* %5) #3
diff --git a/test/CodeGen/AArch64/arm64-builtins-linux.ll b/test/CodeGen/AArch64/arm64-builtins-linux.ll
new file mode 100644
index 00000000000..34fa1b47156
--- /dev/null
+++ b/test/CodeGen/AArch64/arm64-builtins-linux.ll
@@ -0,0 +1,11 @@
+; RUN: llc < %s -march=aarch64 -mtriple=aarch64-linux-gnu | FileCheck %s
+
+; Function Attrs: nounwind readnone
+declare i8* @llvm.aarch64.thread.pointer() #1
+
+define i8* @thread_pointer() {
+; CHECK: thread_pointer:
+; CHECK: mrs {{x[0-9]+}}, TPIDR_EL0
+  %1 = tail call i8* @llvm.aarch64.thread.pointer()
+  ret i8* %1
+}
diff --git a/test/CodeGen/AArch64/arm64-ccmp.ll b/test/CodeGen/AArch64/arm64-ccmp.ll
index ff18f736433..446e5ce8cc4 100644
--- a/test/CodeGen/AArch64/arm64-ccmp.ll
+++ b/test/CodeGen/AArch64/arm64-ccmp.ll
@@ -104,11 +104,14 @@ if.end:                                           ; preds = %if.then, %lor.lhs.f
 ; Speculatively execute division by zero.
 ; The sdiv/udiv instructions do not trap when the divisor is zero, so they are
 ; safe to speculate.
-; CHECK: speculate_division
-; CHECK-NOT: cmp
-; CHECK: sdiv
-; CHECK: cmp
-; CHECK-NEXT: ccmp
+; CHECK-LABEL: speculate_division:
+; CHECK: cmp w0, #1
+; CHECK: sdiv [[DIVRES:w[0-9]+]], w1, w0
+; CHECK: ccmp [[DIVRES]], #16, #0, ge
+; CHECK: b.gt [[BLOCK:LBB[0-9_]+]]
+; CHECK: bl _foo
+; CHECK: [[BLOCK]]:
+; CHECK: orr w0, wzr, #0x7
 define i32 @speculate_division(i32 %a, i32 %b) nounwind ssp {
 entry:
   %cmp = icmp sgt i32 %a, 0
@@ -287,3 +290,99 @@ sw.bb.i.i:
   %code1.i.i.phi.trans.insert = getelementptr inbounds %str1, %str1* %0, i64 0, i32 0, i32 0, i64 16
   br label %sw.bb.i.i
 }
+
+; CHECK-LABEL: select_and
+define i64 @select_and(i32 %w0, i32 %w1, i64 %x2, i64 %x3) {
+; CHECK: cmp w1, #5
+; CHECK-NEXT: ccmp w0, w1, #0, ne
+; CHECK-NEXT: csel x0, x2, x3, lt
+; CHECK-NEXT: ret
+  %1 = icmp slt i32 %w0, %w1
+  %2 = icmp ne i32 5, %w1
+  %3 = and i1 %1, %2
+  %sel = select i1 %3, i64 %x2, i64 %x3
+  ret i64 %sel
+}
+
+; CHECK-LABEL: select_or
+define i64 @select_or(i32 %w0, i32 %w1, i64 %x2, i64 %x3) {
+; CHECK: cmp w1, #5
+; CHECK-NEXT: ccmp w0, w1, #8, eq
+; CHECK-NEXT: csel x0, x2, x3, lt
+; CHECK-NEXT: ret
+  %1 = icmp slt i32 %w0, %w1
+  %2 = icmp ne i32 5, %w1
+  %3 = or i1 %1, %2
+  %sel = select i1 %3, i64 %x2, i64 %x3
+  ret i64 %sel
+}
+
+; CHECK-LABEL: select_complicated
+define i16 @select_complicated(double %v1, double %v2, i16 %a, i16 %b) {
+; CHECK: ldr [[REG:d[0-9]+]],
+; CHECK: fcmp d0, d2
+; CHECK-NEXT: fmov d2, #13.00000000
+; CHECK-NEXT: fccmp d1, d2, #4, ne
+; CHECK-NEXT: fccmp d0, d1, #1, ne
+; CHECK-NEXT: fccmp d0, d1, #4, vc
+; CEHCK-NEXT: csel w0, w0, w1, eq
+  %1 = fcmp one double %v1, %v2
+  %2 = fcmp oeq double %v2, 13.0
+  %3 = fcmp oeq double %v1, 42.0
+  %or0 = or i1 %2, %3
+  %or1 = or i1 %1, %or0
+  %sel = select i1 %or1, i16 %a, i16 %b
+  ret i16 %sel
+}
+
+; CHECK-LABEL: gccbug
+define i64 @gccbug(i64 %x0, i64 %x1) {
+; CHECK: cmp x1, #0
+; CHECK-NEXT: ccmp x0, #2, #0, eq
+; CHECK-NEXT: ccmp x0, #4, #4, ne
+; CHECK-NEXT: orr w[[REGNUM:[0-9]+]], wzr, #0x1
+; CHECK-NEXT: cinc x0, x[[REGNUM]], eq
+; CHECK-NEXT: ret
+  %cmp0 = icmp eq i64 %x1, 0
+  %cmp1 = icmp eq i64 %x0, 2
+  %cmp2 = icmp eq i64 %x0, 4
+
+  %or = or i1 %cmp2, %cmp1
+  %and = and i1 %or, %cmp0
+
+  %sel = select i1 %and, i64 2, i64 1
+  ret i64 %sel
+}
+
+; CHECK-LABEL: select_ororand
+define i32 @select_ororand(i32 %w0, i32 %w1, i32 %w2, i32 %w3) {
+; CHECK: cmp w3, #4
+; CHECK-NEXT: ccmp w2, #2, #0, gt
+; CHECK-NEXT: ccmp w1, #13, #2, ge
+; CHECK-NEXT: ccmp w0, #0, #4, ls
+; CHECK-NEXT: csel w0, w3, wzr, eq
+; CHECK-NEXT: ret
+  %c0 = icmp eq i32 %w0, 0
+  %c1 = icmp ugt i32 %w1, 13
+  %c2 = icmp slt i32 %w2, 2
+  %c4 = icmp sgt i32 %w3, 4
+  %or = or i1 %c0, %c1
+  %and = and i1 %c2, %c4
+  %or1 = or i1 %or, %and
+  %sel = select i1 %or1, i32 %w3, i32 0
+  ret i32 %sel
+}
+
+; CHECK-LABEL: select_noccmp
+define i64 @select_noccmp(i64 %v1, i64 %v2, i64 %v3, i64 %r) {
+; CHECK-NOT: CCMP
+  %c0 = icmp slt i64 %v1, 0
+  %c1 = icmp sgt i64 %v1, 13
+  %c2 = icmp slt i64 %v3, 2
+  %c4 = icmp sgt i64 %v3, 4
+  %and0 = and i1 %c0, %c1
+  %and1 = and i1 %c2, %c4
+  %or = or i1 %and0, %and1
+  %sel = select i1 %or, i64 0, i64 %r
+  ret i64 %sel
+}
diff --git a/test/CodeGen/AArch64/arm64-coalescing-MOVi32imm.ll b/test/CodeGen/AArch64/arm64-coalescing-MOVi32imm.ll
new file mode 100644
index 00000000000..2adbcdf3795
--- /dev/null
+++ b/test/CodeGen/AArch64/arm64-coalescing-MOVi32imm.ll
@@ -0,0 +1,18 @@
+; RUN: llc < %s | FileCheck %s
+
+; CHECK:        orr     w0, wzr, #0x1
+; CHECK-NEXT :  bl      foo
+; CHECK-NEXT :  orr     w0, wzr, #0x1
+; CHECK-NEXT :  bl      foo
+
+target triple = "aarch64--linux-android"
+declare i32 @foo(i32)
+
+; Function Attrs: nounwind uwtable
+define i32 @main() {
+entry:
+  %call = tail call i32 @foo(i32 1)
+  %call1 = tail call i32 @foo(i32 1)
+  ret i32 0
+}
+
diff --git a/test/CodeGen/AArch64/arm64-codegen-prepare-extload.ll b/test/CodeGen/AArch64/arm64-codegen-prepare-extload.ll
index f0b8299a66e..c9f668f2c42 100644
--- a/test/CodeGen/AArch64/arm64-codegen-prepare-extload.ll
+++ b/test/CodeGen/AArch64/arm64-codegen-prepare-extload.ll
@@ -24,7 +24,7 @@ false:
 }
 
 ; Check that we manage to form a zextload is an operation with only one
-; argument to explicitly extend is in the the way.
+; argument to explicitly extend is in the way.
 ; OPTALL-LABEL: @promoteOneArg
 ; OPTALL: [[LD:%[a-zA-Z_0-9-]+]] = load i8, i8* %p
 ; OPT-NEXT: [[ZEXT:%[a-zA-Z_0-9-]+]] = zext i8 [[LD]] to i32
@@ -49,7 +49,7 @@ false:
 }
 
 ; Check that we manage to form a sextload is an operation with only one
-; argument to explicitly extend is in the the way.
+; argument to explicitly extend is in the way.
 ; Version with sext.
 ; OPTALL-LABEL: @promoteOneArgSExt
 ; OPTALL: [[LD:%[a-zA-Z_0-9-]+]] = load i8, i8* %p
@@ -74,7 +74,7 @@ false:
 }
 
 ; Check that we manage to form a zextload is an operation with two
-; arguments to explicitly extend is in the the way.
+; arguments to explicitly extend is in the way.
 ; Extending %add will create two extensions:
 ; 1. One for %b.
 ; 2. One for %t.
@@ -113,7 +113,7 @@ false:
 }
 
 ; Check that we manage to form a sextload is an operation with two
-; arguments to explicitly extend is in the the way.
+; arguments to explicitly extend is in the way.
 ; Version with sext.
 ; OPTALL-LABEL: @promoteTwoArgSExt
 ; OPTALL: [[LD:%[a-zA-Z_0-9-]+]] = load i8, i8* %p
diff --git a/test/CodeGen/AArch64/arm64-convert-v4f64.ll b/test/CodeGen/AArch64/arm64-convert-v4f64.ll
index c4e3e4eae63..b8da3991031 100644
--- a/test/CodeGen/AArch64/arm64-convert-v4f64.ll
+++ b/test/CodeGen/AArch64/arm64-convert-v4f64.ll
@@ -31,3 +31,36 @@ define <8 x i8> @fptosi_v4f64_to_v4i8(<8 x double>* %ptr) {
   ret <8 x i8> %tmp2
 }
 
+define <4 x half> @uitofp_v4i64_to_v4f16(<4 x i64>* %ptr) {
+; CHECK: uitofp_v4i64_to_v4f16
+; CHECK-DAG: ucvtf  v[[LHS:[0-9]+]].2d, v0.2d
+; CHECK-DAG: ucvtf  v[[RHS:[0-9]+]].2d, v1.2d
+; CHECK-DAG: fcvtn  v[[MID:[0-9]+]].2s, v[[LHS]].2d
+; CHECK-DAG: fcvtn2  v[[MID]].4s, v[[RHS]].2d
+; CHECK:     fcvtn  v0.4h, v[[MID]].4s
+  %tmp1 = load <4 x i64>, <4 x i64>* %ptr
+  %tmp2 = uitofp <4 x i64> %tmp1 to <4 x half>
+  ret <4 x half> %tmp2
+}
+
+define <4 x i16> @trunc_v4i64_to_v4i16(<4 x i64>* %ptr) {
+; CHECK: trunc_v4i64_to_v4i16
+; CHECK: xtn
+; CHECK: xtn2
+; CHECK: xtn
+  %tmp1 = load <4 x i64>, <4 x i64>* %ptr
+  %tmp2 = trunc <4 x i64> %tmp1 to <4 x i16>
+  ret <4 x i16> %tmp2
+}
+
+define <4 x i16> @fptoui_v4f64_to_v4i16(<4 x double>* %ptr) {
+; CHECK: fptoui_v4f64_to_v4i16
+; CHECK-DAG: fcvtzu  v[[LHS:[0-9]+]].2d, v0.2d
+; CHECK-DAG: fcvtzu  v[[RHS:[0-9]+]].2d, v1.2d
+; CHECK-DAG: xtn  v[[MID:[0-9]+]].2s, v[[LHS]].2d
+; CHECK-DAG: xtn2  v[[MID]].4s, v[[RHS]].2d
+; CHECK:     xtn  v0.4h, v[[MID]].4s
+  %tmp1 = load <4 x double>, <4 x double>* %ptr
+  %tmp2 = fptoui <4 x double> %tmp1 to <4 x i16>
+  ret <4 x i16> %tmp2
+}
diff --git a/test/CodeGen/AArch64/arm64-fast-isel-conversion.ll b/test/CodeGen/AArch64/arm64-fast-isel-conversion.ll
index 1b688652331..a1b7ab02dbb 100644
--- a/test/CodeGen/AArch64/arm64-fast-isel-conversion.ll
+++ b/test/CodeGen/AArch64/arm64-fast-isel-conversion.ll
@@ -363,7 +363,8 @@ entry:
 define i32 @i64_trunc_i32(i64 %a) nounwind ssp {
 entry:
 ; CHECK-LABEL: i64_trunc_i32
-; CHECK: mov x1, x0
+; CHECK:       mov [[REG:x[0-9]+]], x0
+; CHECK-NEXT:  mov x0, [[REG]]
   %conv = trunc i64 %a to i32
   ret i32 %conv
 }
@@ -371,9 +372,9 @@ entry:
 define zeroext i16 @i64_trunc_i16(i64 %a) nounwind ssp {
 entry:
 ; CHECK-LABEL: i64_trunc_i16
-; CHECK: mov x[[REG:[0-9]+]], x0
-; CHECK: and [[REG2:w[0-9]+]], w[[REG]], #0xffff
-; CHECK: uxth w0, [[REG2]]
+; CHECK:       mov x[[REG:[0-9]+]], x0
+; CHECK-NEXT:  and [[REG2:w[0-9]+]], w[[REG]], #0xffff
+; CHECK-NEXT:  uxth w0, [[REG2]]
   %conv = trunc i64 %a to i16
   ret i16 %conv
 }
@@ -381,9 +382,9 @@ entry:
 define zeroext i8 @i64_trunc_i8(i64 %a) nounwind ssp {
 entry:
 ; CHECK-LABEL: i64_trunc_i8
-; CHECK: mov x[[REG:[0-9]+]], x0
-; CHECK: and [[REG2:w[0-9]+]], w[[REG]], #0xff
-; CHECK: uxtb w0, [[REG2]]
+; CHECK:       mov x[[REG:[0-9]+]], x0
+; CHECK-NEXT:  and [[REG2:w[0-9]+]], w[[REG]], #0xff
+; CHECK-NEXT:  uxtb w0, [[REG2]]
   %conv = trunc i64 %a to i8
   ret i8 %conv
 }
@@ -391,13 +392,67 @@ entry:
 define zeroext i1 @i64_trunc_i1(i64 %a) nounwind ssp {
 entry:
 ; CHECK-LABEL: i64_trunc_i1
-; CHECK: mov x[[REG:[0-9]+]], x0
-; CHECK: and [[REG2:w[0-9]+]], w[[REG]], #0x1
-; CHECK: and w0, [[REG2]], #0x1
+; CHECK:       mov x[[REG:[0-9]+]], x0
+; CHECK-NEXT:  and [[REG2:w[0-9]+]], w[[REG]], #0x1
+; CHECK-NEXT:  and w0, [[REG2]], #0x1
   %conv = trunc i64 %a to i1
   ret i1 %conv
 }
 
+define zeroext i16 @i32_trunc_i16(i32 %a) nounwind ssp {
+entry:
+; CHECK-LABEL: i32_trunc_i16
+; CHECK:       and [[REG:w[0-9]+]], w0, #0xffff
+; CHECK-NEXT:  uxth w0, [[REG]]
+  %conv = trunc i32 %a to i16
+  ret i16 %conv
+}
+
+define zeroext i8 @i32_trunc_i8(i32 %a) nounwind ssp {
+entry:
+; CHECK-LABEL: i32_trunc_i8
+; CHECK:       and [[REG:w[0-9]+]], w0, #0xff
+; CHECK-NEXT:  uxtb w0, [[REG]]
+  %conv = trunc i32 %a to i8
+  ret i8 %conv
+}
+
+define zeroext i1 @i32_trunc_i1(i32 %a) nounwind ssp {
+entry:
+; CHECK-LABEL: i32_trunc_i1
+; CHECK:       and [[REG:w[0-9]+]], w0, #0x1
+; CHECK-NEXT:  and w0, [[REG]], #0x1
+  %conv = trunc i32 %a to i1
+  ret i1 %conv
+}
+
+define zeroext i8 @i16_trunc_i8(i16 zeroext %a) nounwind ssp {
+entry:
+; CHECK-LABEL: i16_trunc_i8
+; CHECK:       and [[REG:w[0-9]+]], w0, #0xff
+; CHECK-NEXT:  uxtb w0, [[REG]]
+  %conv = trunc i16 %a to i8
+  ret i8 %conv
+}
+
+define zeroext i1 @i16_trunc_i1(i16 zeroext %a) nounwind ssp {
+entry:
+; CHECK-LABEL: i16_trunc_i1
+; CHECK:       and [[REG:w[0-9]+]], w0, #0x1
+; CHECK-NEXT:  and w0, [[REG]], #0x1
+  %conv = trunc i16 %a to i1
+  ret i1 %conv
+}
+
+define zeroext i1 @i8_trunc_i1(i8 zeroext %a) nounwind ssp {
+entry:
+; CHECK-LABEL: i8_trunc_i1
+; CHECK:       and [[REG:w[0-9]+]], w0, #0x1
+; CHECK-NEXT:  and w0, [[REG]], #0x1
+  %conv = trunc i8 %a to i1
+  ret i1 %conv
+}
+
 ; rdar://15101939
 define void @stack_trunc() nounwind {
 ; CHECK-LABEL: stack_trunc
diff --git a/test/CodeGen/AArch64/arm64-fast-isel.ll b/test/CodeGen/AArch64/arm64-fast-isel.ll
index a4d08f9016a..9f83a9c359a 100644
--- a/test/CodeGen/AArch64/arm64-fast-isel.ll
+++ b/test/CodeGen/AArch64/arm64-fast-isel.ll
@@ -91,3 +91,45 @@ define void @t6() nounwind {
 }
 
 declare void @llvm.trap() nounwind
+
+define void @ands(i32* %addr) {
+; CHECK-LABEL: ands:
+; CHECK: tst [[COND:w[0-9]+]], #0x1
+; CHECK-NEXT: csel [[COND]],
+entry:
+  %cond91 = select i1 undef, i32 1, i32 2
+  store i32 %cond91, i32* %addr, align 4
+  ret void
+}
+
+define i64 @mul_umul(i64 %arg) {
+; CHECK-LABEL: mul_umul:
+; CHECK: mul x{{[0-9]+}}, [[ARG1:x[0-9]+]], [[ARG2:x[0-9]+]]
+; CHECK-NEXT: umulh x{{[0-9]+}}, [[ARG1]], [[ARG2]]
+entry:
+  %sub.ptr.div = sdiv exact i64 %arg, 8
+  %tmp = call { i64, i1 } @llvm.umul.with.overflow.i64(i64 %sub.ptr.div, i64 8)
+  %tmp1 = extractvalue { i64, i1 } %tmp, 0
+  ret i64 %tmp1
+}
+
+declare { i64, i1 } @llvm.umul.with.overflow.i64(i64, i64)
+
+define void @logicalReg() {
+; Make sure we generate a logical reg = reg, reg instruction without any
+; machine verifier errors.
+; CHECK-LABEL: logicalReg:
+; CHECK: orr w{{[0-9]+}}, w{{[0-9]+}}, w{{[0-9]+}}
+; CHECK: ret
+entry:
+  br i1 undef, label %cond.end, label %cond.false
+
+cond.false:
+  %cond = select i1 undef, i1 true, i1 false
+  br label %cond.end
+
+cond.end:
+  %cond13 = phi i1 [ %cond, %cond.false ], [ true, %entry ]
+  ret void
+}
+
diff --git a/test/CodeGen/AArch64/arm64-fmax.ll b/test/CodeGen/AArch64/arm64-fmax.ll
index 94b745437bd..ea281528b84 100644
--- a/test/CodeGen/AArch64/arm64-fmax.ll
+++ b/test/CodeGen/AArch64/arm64-fmax.ll
@@ -1,29 +1,49 @@
 ; RUN: llc -march=arm64 -enable-no-nans-fp-math < %s | FileCheck %s
+; RUN: llc -march=arm64 < %s | FileCheck %s --check-prefix=CHECK-SAFE
 
-define double @test_direct(float %in) #1 {
+define double @test_direct(float %in) {
 ; CHECK-LABEL: test_direct:
+; CHECK-SAFE-LABEL: test_direct:
   %cmp = fcmp olt float %in, 0.000000e+00
   %longer = fpext float %in to double
   %val = select i1 %cmp, double 0.000000e+00, double %longer
   ret double %val
 
 ; CHECK: fmax
+; CHECK-SAFE: fmax
 }
 
-define double @test_cross(float %in) #1 {
+define double @test_cross(float %in) {
 ; CHECK-LABEL: test_cross:
+; CHECK-SAFE-LABEL: test_cross:
+  %cmp = fcmp ult float %in, 0.000000e+00
+  %longer = fpext float %in to double
+  %val = select i1 %cmp, double %longer, double 0.000000e+00
+  ret double %val
+
+; CHECK: fmin
+; CHECK-SAFE: fmin
+}
+
+; Same as previous, but with ordered comparison;
+; can't be converted in safe-math mode.
+define double @test_cross_fail_nan(float %in) {
+; CHECK-LABEL: test_cross_fail_nan:
+; CHECK-SAFE-LABEL: test_cross_fail_nan:
   %cmp = fcmp olt float %in, 0.000000e+00
   %longer = fpext float %in to double
   %val = select i1 %cmp, double %longer, double 0.000000e+00
   ret double %val
 
 ; CHECK: fmin
+; CHECK-SAFE: fcsel d0, d1, d0, mi
 }
 
 ; This isn't a min or a max, but passes the first condition for swapping the
 ; results. Make sure they're put back before we resort to the normal fcsel.
 define float @test_cross_fail(float %lhs, float %rhs) {
 ; CHECK-LABEL: test_cross_fail:
+; CHECK-SAFE-LABEL: test_cross_fail:
   %tst = fcmp une float %lhs, %rhs
   %res = select i1 %tst, float %rhs, float %lhs
   ret float %res
@@ -31,4 +51,12 @@ define float @test_cross_fail(float %lhs, float %rhs) {
   ; The register allocator would have to decide to be deliberately obtuse before
   ; other register were used.
 ; CHECK: fcsel s0, s1, s0, ne
-}
\ No newline at end of file
+; CHECK-SAFE: fcsel s0, s1, s0, ne
+}
+
+; Make sure the transformation isn't triggered for integers
+define i64 @test_integer(i64  %in) {
+  %cmp = icmp slt i64 %in, 0
+  %val = select i1 %cmp, i64 0, i64 %in
+  ret i64 %val
+}
diff --git a/test/CodeGen/AArch64/arm64-fp128.ll b/test/CodeGen/AArch64/arm64-fp128.ll
index aaef39fcf51..097fe2ca6ed 100644
--- a/test/CodeGen/AArch64/arm64-fp128.ll
+++ b/test/CodeGen/AArch64/arm64-fp128.ll
@@ -148,14 +148,9 @@ define i1 @test_setcc2() {
 ; CHECK: ldr q1, [{{x[0-9]+}}, :lo12:rhs]
 
   %val = fcmp ugt fp128 %lhs, %rhs
-; CHECK: bl      __gttf2
+; CHECK: bl      __letf2
 ; CHECK: cmp     w0, #0
-; CHECK: cset   [[GT:w[0-9]+]], gt
-
-; CHECK: bl      __unordtf2
-; CHECK: cmp     w0, #0
-; CHECK: cset   [[UNORDERED:w[0-9]+]], ne
-; CHECK: orr     w0, [[UNORDERED]], [[GT]]
+; CHECK: cset    w0, gt
 
   ret i1 %val
 ; CHECK: ret
@@ -169,31 +164,21 @@ define i32 @test_br_cc() {
 ; CHECK: ldr q0, [{{x[0-9]+}}, :lo12:lhs]
 ; CHECK: ldr q1, [{{x[0-9]+}}, :lo12:rhs]
 
-  ; olt == !uge, which LLVM unfortunately "optimizes" this to.
+  ; olt == !uge, which LLVM optimizes this to.
   %cond = fcmp olt fp128 %lhs, %rhs
-; CHECK: bl      __getf2
-; CHECK: cmp     w0, #0
-; CHECK: cset   [[OGE:w[0-9]+]], ge
-
-; CHECK: bl      __unordtf2
-; CHECK: cmp     w0, #0
-; CHECK: cset   [[UNORDERED:w[0-9]+]], ne
-
-; CHECK: orr     [[UGE:w[0-9]+]], [[UNORDERED]], [[OGE]]
-; CHECK: cbnz [[UGE]], [[RET29:.LBB[0-9]+_[0-9]+]]
+; CHECK: bl      __lttf2
+; CHECK-NEXT: cmp     w0, #0
+; CHECK-NEXT: b.ge {{.LBB[0-9]+_[0-9]+}}
   br i1 %cond, label %iftrue, label %iffalse
 
 iftrue:
   ret i32 42
 ; CHECK-NEXT: BB#
 ; CHECK-NEXT: movz w0, #0x2a
-; CHECK-NEXT: b [[REALRET:.LBB[0-9]+_[0-9]+]]
-
+; CHECK: ret
 iffalse:
   ret i32 29
-; CHECK: [[RET29]]:
-; CHECK-NEXT: movz w0, #0x1d
-; CHECK-NEXT: [[REALRET]]:
+; CHECK: movz w0, #0x1d
 ; CHECK: ret
 }
 
diff --git a/test/CodeGen/AArch64/arm64-indexed-vector-ldst.ll b/test/CodeGen/AArch64/arm64-indexed-vector-ldst.ll
index 706871e8292..ba31513172d 100644
--- a/test/CodeGen/AArch64/arm64-indexed-vector-ldst.ll
+++ b/test/CodeGen/AArch64/arm64-indexed-vector-ldst.ll
@@ -6171,4 +6171,47 @@ define <2 x double> @test_v2f64_post_reg_ld1lane(double* %bar, double** %ptr, i6
   %tmp3 = getelementptr double, double* %bar, i64 %inc
   store double* %tmp3, double** %ptr
   ret <2 x double> %tmp2
-}
\ No newline at end of file
+}
+
+; Check for dependencies between the vector and the scalar load.
+define <4 x float> @test_v4f32_post_reg_ld1lane_dep_vec_on_load(float* %bar, float** %ptr, i64 %inc, <4 x float>* %dep_ptr_1, <4 x float>* %dep_ptr_2) {
+; CHECK-LABEL: test_v4f32_post_reg_ld1lane_dep_vec_on_load:
+; CHECK: BB#0:
+; CHECK-NEXT: ldr s[[LD:[0-9]+]], [x0]
+; CHECK-NEXT: movi.2d v0, #0000000000000000
+; CHECK-NEXT: str q0, [x3]
+; CHECK-NEXT: ldr q0, [x4]
+; CHECK-NEXT: ins.s v0[1], v[[LD]][0]
+; CHECK-NEXT: add [[POST:x[0-9]]], x0, x2, lsl #2
+; CHECK-NEXT: str [[POST]], [x1]
+; CHECK-NEXT: ret
+  %tmp1 = load float, float* %bar
+  store <4 x float> zeroinitializer, <4 x float>* %dep_ptr_1, align 16
+  %A = load <4 x float>, <4 x float>* %dep_ptr_2, align 16
+  %tmp2 = insertelement <4 x float> %A, float %tmp1, i32 1
+  %tmp3 = getelementptr float, float* %bar, i64 %inc
+  store float* %tmp3, float** %ptr
+  ret <4 x float> %tmp2
+}
+
+; Make sure that we test the narrow V64 code path.
+; The tests above don't, because there, 64-bit insert_vector_elt nodes will be
+; widened to 128-bit before the LD1LANEpost combine has the chance to run,
+; making it avoid narrow vector types.
+; One way to trick that combine into running early is to force the vector ops
+; legalizer to run.  We achieve that using the ctpop.
+; PR23265
+define <4 x i16> @test_v4i16_post_reg_ld1lane_forced_narrow(i16* %bar, i16** %ptr, i64 %inc, <4 x i16> %A, <2 x i32>* %d) {
+; CHECK-LABEL: test_v4i16_post_reg_ld1lane_forced_narrow:
+; CHECK: ld1.h  { v0 }[1], [x0], x{{[0-9]+}}
+  %tmp1 = load i16, i16* %bar
+  %tmp2 = insertelement <4 x i16> %A, i16 %tmp1, i32 1
+  %tmp3 = getelementptr i16, i16* %bar, i64 %inc
+  store i16* %tmp3, i16** %ptr
+  %dl =  load <2 x i32>,  <2 x i32>* %d
+  %dr = call <2 x i32> @llvm.ctpop.v2i32(<2 x i32> %dl)
+  store <2 x i32> %dr, <2 x i32>* %d
+  ret <4 x i16> %tmp2
+}
+
+declare <2 x i32> @llvm.ctpop.v2i32(<2 x i32>)
diff --git a/test/CodeGen/AArch64/arm64-ldp-aa.ll b/test/CodeGen/AArch64/arm64-ldp-aa.ll
new file mode 100644
index 00000000000..ad5c01cfe34
--- /dev/null
+++ b/test/CodeGen/AArch64/arm64-ldp-aa.ll
@@ -0,0 +1,60 @@
+; RUN: llc < %s -march=arm64 -enable-misched=false -verify-machineinstrs | FileCheck %s
+
+; The next set of tests makes sure we can combine the second instruction into
+; the first.
+
+; CHECK-LABEL: ldp_int_aa
+; CHECK: ldp w8, w9, [x1]
+; CHECK: str w0, [x1, #8]
+; CHECK: ret
+define i32 @ldp_int_aa(i32 %a, i32* %p) nounwind {
+  %tmp = load i32, i32* %p, align 4
+  %str.ptr = getelementptr inbounds i32, i32* %p, i64 2
+  store i32 %a, i32* %str.ptr, align 4
+  %add.ptr = getelementptr inbounds i32, i32* %p, i64 1
+  %tmp1 = load i32, i32* %add.ptr, align 4
+  %add = add nsw i32 %tmp1, %tmp
+  ret i32 %add
+}
+
+; CHECK-LABEL: ldp_long_aa
+; CHECK: ldp x8, x9, [x1]
+; CHECK: str x0, [x1, #16]
+; CHECK: ret
+define i64 @ldp_long_aa(i64 %a, i64* %p) nounwind {
+  %tmp = load i64, i64* %p, align 8
+  %str.ptr = getelementptr inbounds i64, i64* %p, i64 2
+  store i64 %a, i64* %str.ptr, align 4
+  %add.ptr = getelementptr inbounds i64, i64* %p, i64 1
+  %tmp1 = load i64, i64* %add.ptr, align 8
+  %add = add nsw i64 %tmp1, %tmp
+  ret i64 %add
+}
+
+; CHECK-LABEL: ldp_float_aa
+; CHECK: str s0, [x0, #8]
+; CHECK: ldp s1, s0, [x0]
+; CHECK: ret
+define float @ldp_float_aa(float %a, float* %p) nounwind {
+  %tmp = load float, float* %p, align 4
+  %str.ptr = getelementptr inbounds float, float* %p, i64 2
+  store float %a, float* %str.ptr, align 4
+  %add.ptr = getelementptr inbounds float, float* %p, i64 1
+  %tmp1 = load float, float* %add.ptr, align 4
+  %add = fadd float %tmp, %tmp1
+  ret float %add
+}
+
+; CHECK-LABEL: ldp_double_aa
+; CHECK: str d0, [x0, #16]
+; CHECK: ldp d1, d0, [x0]
+; CHECK: ret
+define double @ldp_double_aa(double %a, double* %p) nounwind {
+  %tmp = load double, double* %p, align 8
+  %str.ptr = getelementptr inbounds double, double* %p, i64 2
+  store double %a, double* %str.ptr, align 4
+  %add.ptr = getelementptr inbounds double, double* %p, i64 1
+  %tmp1 = load double, double* %add.ptr, align 8
+  %add = fadd double %tmp, %tmp1
+  ret double %add
+}
diff --git a/test/CodeGen/AArch64/arm64-misaligned-memcpy-inline.ll b/test/CodeGen/AArch64/arm64-misaligned-memcpy-inline.ll
index 5bc4d71501b..85572f2cf0f 100644
--- a/test/CodeGen/AArch64/arm64-misaligned-memcpy-inline.ll
+++ b/test/CodeGen/AArch64/arm64-misaligned-memcpy-inline.ll
@@ -1,4 +1,4 @@
-; RUN: llc -mtriple=arm64-apple-ios -aarch64-strict-align < %s | FileCheck %s
+; RUN: llc -mtriple=arm64-apple-ios -mattr=+strict-align < %s | FileCheck %s
 
 ; Small (16-bytes here) unaligned memcpys should stay memcpy calls if
 ; strict-alignment is turned on.
diff --git a/test/CodeGen/AArch64/arm64-named-reg-alloc.ll b/test/CodeGen/AArch64/arm64-named-reg-alloc.ll
index 0c564544a53..5d48c17e128 100644
--- a/test/CodeGen/AArch64/arm64-named-reg-alloc.ll
+++ b/test/CodeGen/AArch64/arm64-named-reg-alloc.ll
@@ -4,7 +4,7 @@
 define i32 @get_stack() nounwind {
 entry:
 ; FIXME: Include an allocatable-specific error message
-; CHECK: Invalid register name global variable
+; CHECK: Invalid register name "x5".
 	%sp = call i32 @llvm.read_register.i32(metadata !0)
   ret i32 %sp
 }
diff --git a/test/CodeGen/AArch64/arm64-named-reg-notareg.ll b/test/CodeGen/AArch64/arm64-named-reg-notareg.ll
index 759bc15807b..8a5fd6f1ac8 100644
--- a/test/CodeGen/AArch64/arm64-named-reg-notareg.ll
+++ b/test/CodeGen/AArch64/arm64-named-reg-notareg.ll
@@ -3,7 +3,7 @@
 
 define i32 @get_stack() nounwind {
 entry:
-; CHECK: Invalid register name global variable
+; CHECK: Invalid register name "notareg".
 	%sp = call i32 @llvm.read_register.i32(metadata !0)
   ret i32 %sp
 }
diff --git a/test/CodeGen/AArch64/arm64-neon-2velem-high.ll b/test/CodeGen/AArch64/arm64-neon-2velem-high.ll
index 58df094d192..3ff1e61d029 100644
--- a/test/CodeGen/AArch64/arm64-neon-2velem-high.ll
+++ b/test/CodeGen/AArch64/arm64-neon-2velem-high.ll
@@ -1,270 +1,484 @@
-; RUN: llc < %s -verify-machineinstrs -mtriple=arm64-none-linux-gnu -mattr=+neon -fp-contract=fast | FileCheck %s
+; RUN: llc -mtriple=arm64-none-linux-gnu -mattr=+neon -fp-contract=fast \
+; RUN:     < %s -verify-machineinstrs -asm-verbose=false | FileCheck %s
 
-declare <2 x float> @llvm.fma.v2f32(<2 x float>, <2 x float>, <2 x float>)
-
-declare <4 x float> @llvm.fma.v4f32(<4 x float>, <4 x float>, <4 x float>)
-
-declare <2 x i64> @llvm.aarch64.neon.sqdmull.v2i64(<2 x i32>, <2 x i32>)
-
-declare <2 x i64> @llvm.aarch64.neon.sqsub.v2i64(<2 x i64>, <2 x i64>)
-
-declare <4 x i32> @llvm.aarch64.neon.sqdmull.v4i32(<4 x i16>, <4 x i16>)
-
-declare <4 x i32> @llvm.aarch64.neon.sqsub.v4i32(<4 x i32>, <4 x i32>)
-
-declare <2 x i64> @llvm.aarch64.neon.sqadd.v2i64(<2 x i64>, <2 x i64>)
-
-declare <4 x i32> @llvm.aarch64.neon.sqadd.v4i32(<4 x i32>, <4 x i32>)
-
-declare <2 x i64> @llvm.aarch64.neon.umull.v2i64(<2 x i32>, <2 x i32>)
-
-declare <4 x i32> @llvm.aarch64.neon.umull.v4i32(<4 x i16>, <4 x i16>)
-
-declare <2 x i64> @llvm.aarch64.neon.smull.v2i64(<2 x i32>, <2 x i32>)
-
-declare <4 x i32> @llvm.aarch64.neon.smull.v4i32(<4 x i16>, <4 x i16>)
-
-define <4 x i32> @test_vmull_high_n_s16(<8 x i16> %a, i16 %b) {
+define <4 x i32> @test_vmull_high_n_s16(<8 x i16> %a, i16 %b) #0 {
 ; CHECK-LABEL: test_vmull_high_n_s16:
-; CHECK: dup [[REPLICATE:v[0-9]+]].8h, w0
-; CHECK: smull2 {{v[0-9]+}}.4s, {{v[0-9]+}}.8h, [[REPLICATE]].8h
+; CHECK-NEXT: dup [[REPLICATE:v[0-9]+]].8h, w0
+; CHECK-NEXT: smull2 {{v[0-9]+}}.4s, {{v[0-9]+}}.8h, [[REPLICATE]].8h
+; CHECK-NEXT: ret
 entry:
   %shuffle.i.i = shufflevector <8 x i16> %a, <8 x i16> undef, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
   %vecinit.i.i = insertelement <4 x i16> undef, i16 %b, i32 0
   %vecinit1.i.i = insertelement <4 x i16> %vecinit.i.i, i16 %b, i32 1
   %vecinit2.i.i = insertelement <4 x i16> %vecinit1.i.i, i16 %b, i32 2
   %vecinit3.i.i = insertelement <4 x i16> %vecinit2.i.i, i16 %b, i32 3
-  %vmull15.i.i = tail call <4 x i32> @llvm.aarch64.neon.smull.v4i32(<4 x i16> %shuffle.i.i, <4 x i16> %vecinit3.i.i)
+  %vmull15.i.i = call <4 x i32> @llvm.aarch64.neon.smull.v4i32(<4 x i16> %shuffle.i.i, <4 x i16> %vecinit3.i.i)
   ret <4 x i32> %vmull15.i.i
 }
 
-define <2 x i64> @test_vmull_high_n_s32(<4 x i32> %a, i32 %b) {
+define <4 x i32> @test_vmull_high_n_s16_imm(<8 x i16> %a) #0 {
+; CHECK-LABEL: test_vmull_high_n_s16_imm:
+; CHECK-NEXT: movi [[REPLICATE:v[0-9]+]].8h, #0x1d
+; CHECK-NEXT: smull2 {{v[0-9]+}}.4s, {{v[0-9]+}}.8h, [[REPLICATE]].8h
+; CHECK-NEXT: ret
+entry:
+  %shuffle.i.i = shufflevector <8 x i16> %a, <8 x i16> undef, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
+  %vmull15.i.i = call <4 x i32> @llvm.aarch64.neon.smull.v4i32(<4 x i16> %shuffle.i.i, <4 x i16> <i16 29, i16 29, i16 29, i16 29>)
+  ret <4 x i32> %vmull15.i.i
+}
+
+define <2 x i64> @test_vmull_high_n_s32(<4 x i32> %a, i32 %b) #0 {
 ; CHECK-LABEL: test_vmull_high_n_s32:
-; CHECK: dup [[REPLICATE:v[0-9]+]].4s, w0
-; CHECK: smull2 {{v[0-9]+}}.2d, {{v[0-9]+}}.4s, [[REPLICATE]].4s
+; CHECK-NEXT: dup [[REPLICATE:v[0-9]+]].4s, w0
+; CHECK-NEXT: smull2 {{v[0-9]+}}.2d, {{v[0-9]+}}.4s, [[REPLICATE]].4s
+; CHECK-NEXT: ret
 entry:
   %shuffle.i.i = shufflevector <4 x i32> %a, <4 x i32> undef, <2 x i32> <i32 2, i32 3>
   %vecinit.i.i = insertelement <2 x i32> undef, i32 %b, i32 0
   %vecinit1.i.i = insertelement <2 x i32> %vecinit.i.i, i32 %b, i32 1
-  %vmull9.i.i = tail call <2 x i64> @llvm.aarch64.neon.smull.v2i64(<2 x i32> %shuffle.i.i, <2 x i32> %vecinit1.i.i)
+  %vmull9.i.i = call <2 x i64> @llvm.aarch64.neon.smull.v2i64(<2 x i32> %shuffle.i.i, <2 x i32> %vecinit1.i.i)
+  ret <2 x i64> %vmull9.i.i
+}
+
+define <2 x i64> @test_vmull_high_n_s32_imm(<4 x i32> %a) #0 {
+; CHECK-LABEL: test_vmull_high_n_s32_imm:
+; CHECK-NEXT: movi [[REPLICATE:v[0-9]+]].4s, #0x1, msl #8
+; CHECK-NEXT: smull2 {{v[0-9]+}}.2d, {{v[0-9]+}}.4s, [[REPLICATE]].4s
+; CHECK-NEXT: ret
+entry:
+  %shuffle.i.i = shufflevector <4 x i32> %a, <4 x i32> undef, <2 x i32> <i32 2, i32 3>
+  %vmull9.i.i = call <2 x i64> @llvm.aarch64.neon.smull.v2i64(<2 x i32> %shuffle.i.i, <2 x i32> <i32 511, i32 511>)
   ret <2 x i64> %vmull9.i.i
 }
 
-define <4 x i32> @test_vmull_high_n_u16(<8 x i16> %a, i16 %b) {
+define <4 x i32> @test_vmull_high_n_u16(<8 x i16> %a, i16 %b) #0 {
 ; CHECK-LABEL: test_vmull_high_n_u16:
-; CHECK: dup [[REPLICATE:v[0-9]+]].8h, w0
-; CHECK: umull2 {{v[0-9]+}}.4s, {{v[0-9]+}}.8h, [[REPLICATE]].8h
+; CHECK-NEXT: dup [[REPLICATE:v[0-9]+]].8h, w0
+; CHECK-NEXT: umull2 {{v[0-9]+}}.4s, {{v[0-9]+}}.8h, [[REPLICATE]].8h
+; CHECK-NEXT: ret
 entry:
   %shuffle.i.i = shufflevector <8 x i16> %a, <8 x i16> undef, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
   %vecinit.i.i = insertelement <4 x i16> undef, i16 %b, i32 0
   %vecinit1.i.i = insertelement <4 x i16> %vecinit.i.i, i16 %b, i32 1
   %vecinit2.i.i = insertelement <4 x i16> %vecinit1.i.i, i16 %b, i32 2
   %vecinit3.i.i = insertelement <4 x i16> %vecinit2.i.i, i16 %b, i32 3
-  %vmull15.i.i = tail call <4 x i32> @llvm.aarch64.neon.umull.v4i32(<4 x i16> %shuffle.i.i, <4 x i16> %vecinit3.i.i)
+  %vmull15.i.i = call <4 x i32> @llvm.aarch64.neon.umull.v4i32(<4 x i16> %shuffle.i.i, <4 x i16> %vecinit3.i.i)
   ret <4 x i32> %vmull15.i.i
 }
 
-define <2 x i64> @test_vmull_high_n_u32(<4 x i32> %a, i32 %b) {
+define <4 x i32> @test_vmull_high_n_u16_imm(<8 x i16> %a) #0 {
+; CHECK-LABEL: test_vmull_high_n_u16_imm:
+; CHECK-NEXT: movi [[REPLICATE:v[0-9]+]].8h, #0x11, lsl #8
+; CHECK-NEXT: umull2 {{v[0-9]+}}.4s, {{v[0-9]+}}.8h, [[REPLICATE]].8h
+; CHECK-NEXT: ret
+entry:
+  %shuffle.i.i = shufflevector <8 x i16> %a, <8 x i16> undef, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
+  %vmull15.i.i = call <4 x i32> @llvm.aarch64.neon.umull.v4i32(<4 x i16> %shuffle.i.i, <4 x i16> <i16 4352, i16 4352, i16 4352, i16 4352>)
+  ret <4 x i32> %vmull15.i.i
+}
+
+define <2 x i64> @test_vmull_high_n_u32(<4 x i32> %a, i32 %b) #0 {
 ; CHECK-LABEL: test_vmull_high_n_u32:
-; CHECK: dup [[REPLICATE:v[0-9]+]].4s, w0
-; CHECK: umull2 {{v[0-9]+}}.2d, {{v[0-9]+}}.4s, [[REPLICATE]].4s
+; CHECK-NEXT: dup [[REPLICATE:v[0-9]+]].4s, w0
+; CHECK-NEXT: umull2 {{v[0-9]+}}.2d, {{v[0-9]+}}.4s, [[REPLICATE]].4s
+; CHECK-NEXT: ret
 entry:
   %shuffle.i.i = shufflevector <4 x i32> %a, <4 x i32> undef, <2 x i32> <i32 2, i32 3>
   %vecinit.i.i = insertelement <2 x i32> undef, i32 %b, i32 0
   %vecinit1.i.i = insertelement <2 x i32> %vecinit.i.i, i32 %b, i32 1
-  %vmull9.i.i = tail call <2 x i64> @llvm.aarch64.neon.umull.v2i64(<2 x i32> %shuffle.i.i, <2 x i32> %vecinit1.i.i)
+  %vmull9.i.i = call <2 x i64> @llvm.aarch64.neon.umull.v2i64(<2 x i32> %shuffle.i.i, <2 x i32> %vecinit1.i.i)
   ret <2 x i64> %vmull9.i.i
 }
 
-define <4 x i32> @test_vqdmull_high_n_s16(<8 x i16> %a, i16 %b) {
+define <2 x i64> @test_vmull_high_n_u32_imm(<4 x i32> %a) #0 {
+; CHECK-LABEL: test_vmull_high_n_u32_imm:
+; CHECK-NEXT: mvni [[REPLICATE:v[0-9]+]].4s, #0x1, msl #8
+; CHECK-NEXT: umull2 {{v[0-9]+}}.2d, {{v[0-9]+}}.4s, [[REPLICATE]].4s
+; CHECK-NEXT: ret
+entry:
+  %shuffle.i.i = shufflevector <4 x i32> %a, <4 x i32> undef, <2 x i32> <i32 2, i32 3>
+  %vmull9.i.i = call <2 x i64> @llvm.aarch64.neon.umull.v2i64(<2 x i32> %shuffle.i.i, <2 x i32> <i32 4294966784, i32 4294966784>)
+  ret <2 x i64> %vmull9.i.i
+}
+
+define <4 x i32> @test_vqdmull_high_n_s16(<8 x i16> %a, i16 %b) #0 {
 ; CHECK-LABEL: test_vqdmull_high_n_s16:
-; CHECK: dup [[REPLICATE:v[0-9]+]].8h, w0
-; CHECK: sqdmull2 {{v[0-9]+}}.4s, {{v[0-9]+}}.8h, [[REPLICATE]].8h
+; CHECK-NEXT: dup [[REPLICATE:v[0-9]+]].8h, w0
+; CHECK-NEXT: sqdmull2 {{v[0-9]+}}.4s, {{v[0-9]+}}.8h, [[REPLICATE]].8h
+; CHECK-NEXT: ret
 entry:
   %shuffle.i.i = shufflevector <8 x i16> %a, <8 x i16> undef, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
   %vecinit.i.i = insertelement <4 x i16> undef, i16 %b, i32 0
   %vecinit1.i.i = insertelement <4 x i16> %vecinit.i.i, i16 %b, i32 1
   %vecinit2.i.i = insertelement <4 x i16> %vecinit1.i.i, i16 %b, i32 2
   %vecinit3.i.i = insertelement <4 x i16> %vecinit2.i.i, i16 %b, i32 3
-  %vqdmull15.i.i = tail call <4 x i32> @llvm.aarch64.neon.sqdmull.v4i32(<4 x i16> %shuffle.i.i, <4 x i16> %vecinit3.i.i)
+  %vqdmull15.i.i = call <4 x i32> @llvm.aarch64.neon.sqdmull.v4i32(<4 x i16> %shuffle.i.i, <4 x i16> %vecinit3.i.i)
+  ret <4 x i32> %vqdmull15.i.i
+}
+
+define <4 x i32> @test_vqdmull_high_n_s16_imm(<8 x i16> %a) #0 {
+; CHECK-LABEL: test_vqdmull_high_n_s16_imm:
+; CHECK-NEXT: mvni [[REPLICATE:v[0-9]+]].8h, #0x11, lsl #8
+; CHECK-NEXT: sqdmull2 {{v[0-9]+}}.4s, {{v[0-9]+}}.8h, [[REPLICATE]].8h
+; CHECK-NEXT: ret
+entry:
+  %shuffle.i.i = shufflevector <8 x i16> %a, <8 x i16> undef, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
+  %vqdmull15.i.i = call <4 x i32> @llvm.aarch64.neon.sqdmull.v4i32(<4 x i16> %shuffle.i.i, <4 x i16> <i16 61183, i16 61183, i16 61183, i16 61183>)
   ret <4 x i32> %vqdmull15.i.i
 }
 
-define <2 x i64> @test_vqdmull_high_n_s32(<4 x i32> %a, i32 %b) {
+define <2 x i64> @test_vqdmull_high_n_s32(<4 x i32> %a, i32 %b) #0 {
 ; CHECK-LABEL: test_vqdmull_high_n_s32:
-; CHECK: dup [[REPLICATE:v[0-9]+]].4s, w0
-; CHECK: sqdmull2 {{v[0-9]+}}.2d, {{v[0-9]+}}.4s, [[REPLICATE]].4s
+; CHECK-NEXT: dup [[REPLICATE:v[0-9]+]].4s, w0
+; CHECK-NEXT: sqdmull2 {{v[0-9]+}}.2d, {{v[0-9]+}}.4s, [[REPLICATE]].4s
+; CHECK-NEXT: ret
 entry:
   %shuffle.i.i = shufflevector <4 x i32> %a, <4 x i32> undef, <2 x i32> <i32 2, i32 3>
   %vecinit.i.i = insertelement <2 x i32> undef, i32 %b, i32 0
   %vecinit1.i.i = insertelement <2 x i32> %vecinit.i.i, i32 %b, i32 1
-  %vqdmull9.i.i = tail call <2 x i64> @llvm.aarch64.neon.sqdmull.v2i64(<2 x i32> %shuffle.i.i, <2 x i32> %vecinit1.i.i)
+  %vqdmull9.i.i = call <2 x i64> @llvm.aarch64.neon.sqdmull.v2i64(<2 x i32> %shuffle.i.i, <2 x i32> %vecinit1.i.i)
+  ret <2 x i64> %vqdmull9.i.i
+}
+
+define <2 x i64> @test_vqdmull_high_n_s32_imm(<4 x i32> %a) #0 {
+; CHECK-LABEL: test_vqdmull_high_n_s32_imm:
+; CHECK-NEXT: movi [[REPLICATE:v[0-9]+]].4s, #0x1d
+; CHECK-NEXT: sqdmull2 {{v[0-9]+}}.2d, {{v[0-9]+}}.4s, [[REPLICATE]].4s
+; CHECK-NEXT: ret
+entry:
+  %shuffle.i.i = shufflevector <4 x i32> %a, <4 x i32> undef, <2 x i32> <i32 2, i32 3>
+  %vqdmull9.i.i = call <2 x i64> @llvm.aarch64.neon.sqdmull.v2i64(<2 x i32> %shuffle.i.i, <2 x i32> <i32 29, i32 29>)
   ret <2 x i64> %vqdmull9.i.i
 }
 
-define <4 x i32> @test_vmlal_high_n_s16(<4 x i32> %a, <8 x i16> %b, i16 %c) {
+define <4 x i32> @test_vmlal_high_n_s16(<4 x i32> %a, <8 x i16> %b, i16 %c) #0 {
 ; CHECK-LABEL: test_vmlal_high_n_s16:
-; CHECK: dup [[REPLICATE:v[0-9]+]].8h, w0
-; CHECK: smlal2 {{v[0-9]+}}.4s, {{v[0-9]+}}.8h, [[REPLICATE]].8h
+; CHECK-NEXT: dup [[REPLICATE:v[0-9]+]].8h, w0
+; CHECK-NEXT: smlal2 {{v[0-9]+}}.4s, {{v[0-9]+}}.8h, [[REPLICATE]].8h
+; CHECK-NEXT: ret
 entry:
   %shuffle.i.i = shufflevector <8 x i16> %b, <8 x i16> undef, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
   %vecinit.i.i = insertelement <4 x i16> undef, i16 %c, i32 0
   %vecinit1.i.i = insertelement <4 x i16> %vecinit.i.i, i16 %c, i32 1
   %vecinit2.i.i = insertelement <4 x i16> %vecinit1.i.i, i16 %c, i32 2
   %vecinit3.i.i = insertelement <4 x i16> %vecinit2.i.i, i16 %c, i32 3
-  %vmull2.i.i.i = tail call <4 x i32> @llvm.aarch64.neon.smull.v4i32(<4 x i16> %shuffle.i.i, <4 x i16> %vecinit3.i.i)
+  %vmull2.i.i.i = call <4 x i32> @llvm.aarch64.neon.smull.v4i32(<4 x i16> %shuffle.i.i, <4 x i16> %vecinit3.i.i)
+  %add.i.i = add <4 x i32> %vmull2.i.i.i, %a
+  ret <4 x i32> %add.i.i
+}
+
+define <4 x i32> @test_vmlal_high_n_s16_imm(<4 x i32> %a, <8 x i16> %b) #0 {
+; CHECK-LABEL: test_vmlal_high_n_s16_imm:
+; CHECK-NEXT: movi [[REPLICATE:v[0-9]+]].8h, #0x1d
+; CHECK-NEXT: smlal2 {{v[0-9]+}}.4s, {{v[0-9]+}}.8h, [[REPLICATE]].8h
+; CHECK-NEXT: ret
+entry:
+  %shuffle.i.i = shufflevector <8 x i16> %b, <8 x i16> undef, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
+  %vmull2.i.i.i = call <4 x i32> @llvm.aarch64.neon.smull.v4i32(<4 x i16> %shuffle.i.i, <4 x i16> <i16 29, i16 29, i16 29, i16 29>)
   %add.i.i = add <4 x i32> %vmull2.i.i.i, %a
   ret <4 x i32> %add.i.i
 }
 
-define <2 x i64> @test_vmlal_high_n_s32(<2 x i64> %a, <4 x i32> %b, i32 %c) {
+define <2 x i64> @test_vmlal_high_n_s32(<2 x i64> %a, <4 x i32> %b, i32 %c) #0 {
 ; CHECK-LABEL: test_vmlal_high_n_s32:
-; CHECK: dup [[REPLICATE:v[0-9]+]].4s, w0
-; CHECK: smlal2 {{v[0-9]+}}.2d, {{v[0-9]+}}.4s, [[REPLICATE]].4s
+; CHECK-NEXT: dup [[REPLICATE:v[0-9]+]].4s, w0
+; CHECK-NEXT: smlal2 {{v[0-9]+}}.2d, {{v[0-9]+}}.4s, [[REPLICATE]].4s
+; CHECK-NEXT: ret
 entry:
   %shuffle.i.i = shufflevector <4 x i32> %b, <4 x i32> undef, <2 x i32> <i32 2, i32 3>
   %vecinit.i.i = insertelement <2 x i32> undef, i32 %c, i32 0
   %vecinit1.i.i = insertelement <2 x i32> %vecinit.i.i, i32 %c, i32 1
-  %vmull2.i.i.i = tail call <2 x i64> @llvm.aarch64.neon.smull.v2i64(<2 x i32> %shuffle.i.i, <2 x i32> %vecinit1.i.i)
+  %vmull2.i.i.i = call <2 x i64> @llvm.aarch64.neon.smull.v2i64(<2 x i32> %shuffle.i.i, <2 x i32> %vecinit1.i.i)
   %add.i.i = add <2 x i64> %vmull2.i.i.i, %a
   ret <2 x i64> %add.i.i
 }
 
-define <4 x i32> @test_vmlal_high_n_u16(<4 x i32> %a, <8 x i16> %b, i16 %c) {
+define <2 x i64> @test_vmlal_high_n_s32_imm(<2 x i64> %a, <4 x i32> %b) #0 {
+; CHECK-LABEL: test_vmlal_high_n_s32_imm:
+; CHECK-NEXT: movi [[REPLICATE:v[0-9]+]].4s, #0x1d
+; CHECK-NEXT: smlal2 {{v[0-9]+}}.2d, {{v[0-9]+}}.4s, [[REPLICATE]].4s
+; CHECK-NEXT: ret
+entry:
+  %shuffle.i.i = shufflevector <4 x i32> %b, <4 x i32> undef, <2 x i32> <i32 2, i32 3>
+  %vmull2.i.i.i = call <2 x i64> @llvm.aarch64.neon.smull.v2i64(<2 x i32> %shuffle.i.i, <2 x i32> <i32 29, i32 29>)
+  %add.i.i = add <2 x i64> %vmull2.i.i.i, %a
+  ret <2 x i64> %add.i.i
+}
+
+define <4 x i32> @test_vmlal_high_n_u16(<4 x i32> %a, <8 x i16> %b, i16 %c) #0 {
 ; CHECK-LABEL: test_vmlal_high_n_u16:
-; CHECK: dup [[REPLICATE:v[0-9]+]].8h, w0
-; CHECK: umlal2 {{v[0-9]+}}.4s, {{v[0-9]+}}.8h, [[REPLICATE]].8h
+; CHECK-NEXT: dup [[REPLICATE:v[0-9]+]].8h, w0
+; CHECK-NEXT: umlal2 {{v[0-9]+}}.4s, {{v[0-9]+}}.8h, [[REPLICATE]].8h
+; CHECK-NEXT: ret
 entry:
   %shuffle.i.i = shufflevector <8 x i16> %b, <8 x i16> undef, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
   %vecinit.i.i = insertelement <4 x i16> undef, i16 %c, i32 0
   %vecinit1.i.i = insertelement <4 x i16> %vecinit.i.i, i16 %c, i32 1
   %vecinit2.i.i = insertelement <4 x i16> %vecinit1.i.i, i16 %c, i32 2
   %vecinit3.i.i = insertelement <4 x i16> %vecinit2.i.i, i16 %c, i32 3
-  %vmull2.i.i.i = tail call <4 x i32> @llvm.aarch64.neon.umull.v4i32(<4 x i16> %shuffle.i.i, <4 x i16> %vecinit3.i.i)
+  %vmull2.i.i.i = call <4 x i32> @llvm.aarch64.neon.umull.v4i32(<4 x i16> %shuffle.i.i, <4 x i16> %vecinit3.i.i)
+  %add.i.i = add <4 x i32> %vmull2.i.i.i, %a
+  ret <4 x i32> %add.i.i
+}
+
+define <4 x i32> @test_vmlal_high_n_u16_imm(<4 x i32> %a, <8 x i16> %b) #0 {
+; CHECK-LABEL: test_vmlal_high_n_u16_imm:
+; CHECK-NEXT: movi [[REPLICATE:v[0-9]+]].8h, #0x1d
+; CHECK-NEXT: umlal2 {{v[0-9]+}}.4s, {{v[0-9]+}}.8h, [[REPLICATE]].8h
+; CHECK-NEXT: ret
+entry:
+  %shuffle.i.i = shufflevector <8 x i16> %b, <8 x i16> undef, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
+  %vmull2.i.i.i = call <4 x i32> @llvm.aarch64.neon.umull.v4i32(<4 x i16> %shuffle.i.i, <4 x i16> <i16 29, i16 29, i16 29, i16 29>)
   %add.i.i = add <4 x i32> %vmull2.i.i.i, %a
   ret <4 x i32> %add.i.i
 }
 
-define <2 x i64> @test_vmlal_high_n_u32(<2 x i64> %a, <4 x i32> %b, i32 %c) {
+define <2 x i64> @test_vmlal_high_n_u32(<2 x i64> %a, <4 x i32> %b, i32 %c) #0 {
 ; CHECK-LABEL: test_vmlal_high_n_u32:
-; CHECK: dup [[REPLICATE:v[0-9]+]].4s, w0
-; CHECK: umlal2 {{v[0-9]+}}.2d, {{v[0-9]+}}.4s, [[REPLICATE]].4s
+; CHECK-NEXT: dup [[REPLICATE:v[0-9]+]].4s, w0
+; CHECK-NEXT: umlal2 {{v[0-9]+}}.2d, {{v[0-9]+}}.4s, [[REPLICATE]].4s
+; CHECK-NEXT: ret
 entry:
   %shuffle.i.i = shufflevector <4 x i32> %b, <4 x i32> undef, <2 x i32> <i32 2, i32 3>
   %vecinit.i.i = insertelement <2 x i32> undef, i32 %c, i32 0
   %vecinit1.i.i = insertelement <2 x i32> %vecinit.i.i, i32 %c, i32 1
-  %vmull2.i.i.i = tail call <2 x i64> @llvm.aarch64.neon.umull.v2i64(<2 x i32> %shuffle.i.i, <2 x i32> %vecinit1.i.i)
+  %vmull2.i.i.i = call <2 x i64> @llvm.aarch64.neon.umull.v2i64(<2 x i32> %shuffle.i.i, <2 x i32> %vecinit1.i.i)
   %add.i.i = add <2 x i64> %vmull2.i.i.i, %a
   ret <2 x i64> %add.i.i
 }
 
-define <4 x i32> @test_vqdmlal_high_n_s16(<4 x i32> %a, <8 x i16> %b, i16 %c) {
+define <2 x i64> @test_vmlal_high_n_u32_imm(<2 x i64> %a, <4 x i32> %b) #0 {
+; CHECK-LABEL: test_vmlal_high_n_u32_imm:
+; CHECK-NEXT: movi [[REPLICATE:v[0-9]+]].4s, #0x1d
+; CHECK-NEXT: umlal2 {{v[0-9]+}}.2d, {{v[0-9]+}}.4s, [[REPLICATE]].4s
+; CHECK-NEXT: ret
+entry:
+  %shuffle.i.i = shufflevector <4 x i32> %b, <4 x i32> undef, <2 x i32> <i32 2, i32 3>
+  %vmull2.i.i.i = call <2 x i64> @llvm.aarch64.neon.umull.v2i64(<2 x i32> %shuffle.i.i, <2 x i32> <i32 29, i32 29>)
+  %add.i.i = add <2 x i64> %vmull2.i.i.i, %a
+  ret <2 x i64> %add.i.i
+}
+
+define <4 x i32> @test_vqdmlal_high_n_s16(<4 x i32> %a, <8 x i16> %b, i16 %c) #0 {
 ; CHECK-LABEL: test_vqdmlal_high_n_s16:
-; CHECK: sqdmlal2 {{v[0-9]+}}.4s, {{v[0-9]+}}.8h, {{v[0-9]+}}.8h
+; CHECK-NEXT: dup [[REPLICATE:v[0-9]+]].8h, w0
+; CHECK-NEXT: sqdmlal2 {{v[0-9]+}}.4s, {{v[0-9]+}}.8h, [[REPLICATE]].8h
+; CHECK-NEXT: ret
 entry:
   %shuffle.i.i = shufflevector <8 x i16> %b, <8 x i16> undef, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
   %vecinit.i.i = insertelement <4 x i16> undef, i16 %c, i32 0
   %vecinit1.i.i = insertelement <4 x i16> %vecinit.i.i, i16 %c, i32 1
   %vecinit2.i.i = insertelement <4 x i16> %vecinit1.i.i, i16 %c, i32 2
   %vecinit3.i.i = insertelement <4 x i16> %vecinit2.i.i, i16 %c, i32 3
-  %vqdmlal15.i.i = tail call <4 x i32> @llvm.aarch64.neon.sqdmull.v4i32(<4 x i16> %shuffle.i.i, <4 x i16> %vecinit3.i.i)
-  %vqdmlal17.i.i = tail call <4 x i32> @llvm.aarch64.neon.sqadd.v4i32(<4 x i32> %a, <4 x i32> %vqdmlal15.i.i)
+  %vqdmlal15.i.i = call <4 x i32> @llvm.aarch64.neon.sqdmull.v4i32(<4 x i16> %shuffle.i.i, <4 x i16> %vecinit3.i.i)
+  %vqdmlal17.i.i = call <4 x i32> @llvm.aarch64.neon.sqadd.v4i32(<4 x i32> %a, <4 x i32> %vqdmlal15.i.i)
+  ret <4 x i32> %vqdmlal17.i.i
+}
+
+define <4 x i32> @test_vqdmlal_high_n_s16_imm(<4 x i32> %a, <8 x i16> %b) #0 {
+; CHECK-LABEL: test_vqdmlal_high_n_s16_imm:
+; CHECK-NEXT: movi [[REPLICATE:v[0-9]+]].8h, #0x1d
+; CHECK-NEXT: sqdmlal2 {{v[0-9]+}}.4s, {{v[0-9]+}}.8h, [[REPLICATE]].8h
+; CHECK-NEXT: ret
+entry:
+  %shuffle.i.i = shufflevector <8 x i16> %b, <8 x i16> undef, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
+  %vqdmlal15.i.i = call <4 x i32> @llvm.aarch64.neon.sqdmull.v4i32(<4 x i16> %shuffle.i.i, <4 x i16> <i16 29, i16 29, i16 29, i16 29>)
+  %vqdmlal17.i.i = call <4 x i32> @llvm.aarch64.neon.sqadd.v4i32(<4 x i32> %a, <4 x i32> %vqdmlal15.i.i)
   ret <4 x i32> %vqdmlal17.i.i
 }
 
-define <2 x i64> @test_vqdmlal_high_n_s32(<2 x i64> %a, <4 x i32> %b, i32 %c) {
+define <2 x i64> @test_vqdmlal_high_n_s32(<2 x i64> %a, <4 x i32> %b, i32 %c) #0 {
 ; CHECK-LABEL: test_vqdmlal_high_n_s32:
-; CHECK: sqdmlal2 {{v[0-9]+}}.2d, {{v[0-9]+}}.4s, {{v[0-9]+}}.4s
+; CHECK-NEXT: dup [[REPLICATE:v[0-9]+]].4s, w0
+; CHECK-NEXT: sqdmlal2 {{v[0-9]+}}.2d, {{v[0-9]+}}.4s, [[REPLICATE]].4s
+; CHECK-NEXT: ret
 entry:
   %shuffle.i.i = shufflevector <4 x i32> %b, <4 x i32> undef, <2 x i32> <i32 2, i32 3>
   %vecinit.i.i = insertelement <2 x i32> undef, i32 %c, i32 0
   %vecinit1.i.i = insertelement <2 x i32> %vecinit.i.i, i32 %c, i32 1
-  %vqdmlal9.i.i = tail call <2 x i64> @llvm.aarch64.neon.sqdmull.v2i64(<2 x i32> %shuffle.i.i, <2 x i32> %vecinit1.i.i)
-  %vqdmlal11.i.i = tail call <2 x i64> @llvm.aarch64.neon.sqadd.v2i64(<2 x i64> %a, <2 x i64> %vqdmlal9.i.i)
+  %vqdmlal9.i.i = call <2 x i64> @llvm.aarch64.neon.sqdmull.v2i64(<2 x i32> %shuffle.i.i, <2 x i32> %vecinit1.i.i)
+  %vqdmlal11.i.i = call <2 x i64> @llvm.aarch64.neon.sqadd.v2i64(<2 x i64> %a, <2 x i64> %vqdmlal9.i.i)
+  ret <2 x i64> %vqdmlal11.i.i
+}
+
+define <2 x i64> @test_vqdmlal_high_n_s32_imm(<2 x i64> %a, <4 x i32> %b) #0 {
+; CHECK-LABEL: test_vqdmlal_high_n_s32_imm:
+; CHECK-NEXT: movi [[REPLICATE:v[0-9]+]].4s, #0x1d
+; CHECK-NEXT: sqdmlal2 {{v[0-9]+}}.2d, {{v[0-9]+}}.4s, [[REPLICATE]].4s
+; CHECK-NEXT: ret
+entry:
+  %shuffle.i.i = shufflevector <4 x i32> %b, <4 x i32> undef, <2 x i32> <i32 2, i32 3>
+  %vqdmlal9.i.i = call <2 x i64> @llvm.aarch64.neon.sqdmull.v2i64(<2 x i32> %shuffle.i.i, <2 x i32> <i32 29, i32 29>)
+  %vqdmlal11.i.i = call <2 x i64> @llvm.aarch64.neon.sqadd.v2i64(<2 x i64> %a, <2 x i64> %vqdmlal9.i.i)
   ret <2 x i64> %vqdmlal11.i.i
 }
 
-define <4 x i32> @test_vmlsl_high_n_s16(<4 x i32> %a, <8 x i16> %b, i16 %c) {
+define <4 x i32> @test_vmlsl_high_n_s16(<4 x i32> %a, <8 x i16> %b, i16 %c) #0 {
 ; CHECK-LABEL: test_vmlsl_high_n_s16:
-; CHECK: smlsl2 {{v[0-9]+}}.4s, {{v[0-9]+}}.8h, {{v[0-9]+}}.8h
+; CHECK-NEXT: dup [[REPLICATE:v[0-9]+]].8h, w0
+; CHECK-NEXT: smlsl2 {{v[0-9]+}}.4s, {{v[0-9]+}}.8h, [[REPLICATE]].8h
+; CHECK-NEXT: ret
 entry:
   %shuffle.i.i = shufflevector <8 x i16> %b, <8 x i16> undef, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
   %vecinit.i.i = insertelement <4 x i16> undef, i16 %c, i32 0
   %vecinit1.i.i = insertelement <4 x i16> %vecinit.i.i, i16 %c, i32 1
   %vecinit2.i.i = insertelement <4 x i16> %vecinit1.i.i, i16 %c, i32 2
   %vecinit3.i.i = insertelement <4 x i16> %vecinit2.i.i, i16 %c, i32 3
-  %vmull2.i.i.i = tail call <4 x i32> @llvm.aarch64.neon.smull.v4i32(<4 x i16> %shuffle.i.i, <4 x i16> %vecinit3.i.i)
+  %vmull2.i.i.i = call <4 x i32> @llvm.aarch64.neon.smull.v4i32(<4 x i16> %shuffle.i.i, <4 x i16> %vecinit3.i.i)
+  %sub.i.i = sub <4 x i32> %a, %vmull2.i.i.i
+  ret <4 x i32> %sub.i.i
+}
+
+define <4 x i32> @test_vmlsl_high_n_s16_imm(<4 x i32> %a, <8 x i16> %b) #0 {
+; CHECK-LABEL: test_vmlsl_high_n_s16_imm:
+; CHECK-NEXT: movi [[REPLICATE:v[0-9]+]].8h, #0x1d
+; CHECK-NEXT: smlsl2 {{v[0-9]+}}.4s, {{v[0-9]+}}.8h, [[REPLICATE]].8h
+; CHECK-NEXT: ret
+entry:
+  %shuffle.i.i = shufflevector <8 x i16> %b, <8 x i16> undef, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
+  %vmull2.i.i.i = call <4 x i32> @llvm.aarch64.neon.smull.v4i32(<4 x i16> %shuffle.i.i, <4 x i16> <i16 29, i16 29, i16 29, i16 29>)
   %sub.i.i = sub <4 x i32> %a, %vmull2.i.i.i
   ret <4 x i32> %sub.i.i
 }
 
-define <2 x i64> @test_vmlsl_high_n_s32(<2 x i64> %a, <4 x i32> %b, i32 %c) {
+define <2 x i64> @test_vmlsl_high_n_s32(<2 x i64> %a, <4 x i32> %b, i32 %c) #0 {
 ; CHECK-LABEL: test_vmlsl_high_n_s32:
-; CHECK: smlsl2 {{v[0-9]+}}.2d, {{v[0-9]+}}.4s, {{v[0-9]+}}.4s
+; CHECK-NEXT: dup [[REPLICATE:v[0-9]+]].4s, w0
+; CHECK-NEXT: smlsl2 {{v[0-9]+}}.2d, {{v[0-9]+}}.4s, [[REPLICATE]].4s
+; CHECK-NEXT: ret
 entry:
   %shuffle.i.i = shufflevector <4 x i32> %b, <4 x i32> undef, <2 x i32> <i32 2, i32 3>
   %vecinit.i.i = insertelement <2 x i32> undef, i32 %c, i32 0
   %vecinit1.i.i = insertelement <2 x i32> %vecinit.i.i, i32 %c, i32 1
-  %vmull2.i.i.i = tail call <2 x i64> @llvm.aarch64.neon.smull.v2i64(<2 x i32> %shuffle.i.i, <2 x i32> %vecinit1.i.i)
+  %vmull2.i.i.i = call <2 x i64> @llvm.aarch64.neon.smull.v2i64(<2 x i32> %shuffle.i.i, <2 x i32> %vecinit1.i.i)
   %sub.i.i = sub <2 x i64> %a, %vmull2.i.i.i
   ret <2 x i64> %sub.i.i
 }
 
-define <4 x i32> @test_vmlsl_high_n_u16(<4 x i32> %a, <8 x i16> %b, i16 %c) {
+define <2 x i64> @test_vmlsl_high_n_s32_imm(<2 x i64> %a, <4 x i32> %b) #0 {
+; CHECK-LABEL: test_vmlsl_high_n_s32_imm:
+; CHECK-NEXT: movi [[REPLICATE:v[0-9]+]].4s, #0x1d
+; CHECK-NEXT: smlsl2 {{v[0-9]+}}.2d, {{v[0-9]+}}.4s, [[REPLICATE]].4s
+; CHECK-NEXT: ret
+entry:
+  %shuffle.i.i = shufflevector <4 x i32> %b, <4 x i32> undef, <2 x i32> <i32 2, i32 3>
+  %vmull2.i.i.i = call <2 x i64> @llvm.aarch64.neon.smull.v2i64(<2 x i32> %shuffle.i.i, <2 x i32> <i32 29, i32 29>)
+  %sub.i.i = sub <2 x i64> %a, %vmull2.i.i.i
+  ret <2 x i64> %sub.i.i
+}
+
+define <4 x i32> @test_vmlsl_high_n_u16(<4 x i32> %a, <8 x i16> %b, i16 %c) #0 {
 ; CHECK-LABEL: test_vmlsl_high_n_u16:
-; CHECK: umlsl2 {{v[0-9]+}}.4s, {{v[0-9]+}}.8h, {{v[0-9]+}}.8h
+; CHECK-NEXT: dup [[REPLICATE:v[0-9]+]].8h, w0
+; CHECK-NEXT: umlsl2 {{v[0-9]+}}.4s, {{v[0-9]+}}.8h, [[REPLICATE]].8h
+; CHECK-NEXT: ret
 entry:
   %shuffle.i.i = shufflevector <8 x i16> %b, <8 x i16> undef, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
   %vecinit.i.i = insertelement <4 x i16> undef, i16 %c, i32 0
   %vecinit1.i.i = insertelement <4 x i16> %vecinit.i.i, i16 %c, i32 1
   %vecinit2.i.i = insertelement <4 x i16> %vecinit1.i.i, i16 %c, i32 2
   %vecinit3.i.i = insertelement <4 x i16> %vecinit2.i.i, i16 %c, i32 3
-  %vmull2.i.i.i = tail call <4 x i32> @llvm.aarch64.neon.umull.v4i32(<4 x i16> %shuffle.i.i, <4 x i16> %vecinit3.i.i)
+  %vmull2.i.i.i = call <4 x i32> @llvm.aarch64.neon.umull.v4i32(<4 x i16> %shuffle.i.i, <4 x i16> %vecinit3.i.i)
+  %sub.i.i = sub <4 x i32> %a, %vmull2.i.i.i
+  ret <4 x i32> %sub.i.i
+}
+
+define <4 x i32> @test_vmlsl_high_n_u16_imm(<4 x i32> %a, <8 x i16> %b) #0 {
+; CHECK-LABEL: test_vmlsl_high_n_u16_imm:
+; CHECK-NEXT: movi [[REPLICATE:v[0-9]+]].8h, #0x1d
+; CHECK-NEXT: umlsl2 {{v[0-9]+}}.4s, {{v[0-9]+}}.8h, [[REPLICATE]].8h
+; CHECK-NEXT: ret
+entry:
+  %shuffle.i.i = shufflevector <8 x i16> %b, <8 x i16> undef, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
+  %vmull2.i.i.i = call <4 x i32> @llvm.aarch64.neon.umull.v4i32(<4 x i16> %shuffle.i.i, <4 x i16> <i16 29, i16 29, i16 29, i16 29>)
   %sub.i.i = sub <4 x i32> %a, %vmull2.i.i.i
   ret <4 x i32> %sub.i.i
 }
 
-define <2 x i64> @test_vmlsl_high_n_u32(<2 x i64> %a, <4 x i32> %b, i32 %c) {
+define <2 x i64> @test_vmlsl_high_n_u32(<2 x i64> %a, <4 x i32> %b, i32 %c) #0 {
 ; CHECK-LABEL: test_vmlsl_high_n_u32:
-; CHECK: umlsl2 {{v[0-9]+}}.2d, {{v[0-9]+}}.4s, {{v[0-9]+}}.4s
+; CHECK-NEXT: dup [[REPLICATE:v[0-9]+]].4s, w0
+; CHECK-NEXT: umlsl2 {{v[0-9]+}}.2d, {{v[0-9]+}}.4s, [[REPLICATE]].4s
+; CHECK-NEXT: ret
 entry:
   %shuffle.i.i = shufflevector <4 x i32> %b, <4 x i32> undef, <2 x i32> <i32 2, i32 3>
   %vecinit.i.i = insertelement <2 x i32> undef, i32 %c, i32 0
   %vecinit1.i.i = insertelement <2 x i32> %vecinit.i.i, i32 %c, i32 1
-  %vmull2.i.i.i = tail call <2 x i64> @llvm.aarch64.neon.umull.v2i64(<2 x i32> %shuffle.i.i, <2 x i32> %vecinit1.i.i)
+  %vmull2.i.i.i = call <2 x i64> @llvm.aarch64.neon.umull.v2i64(<2 x i32> %shuffle.i.i, <2 x i32> %vecinit1.i.i)
   %sub.i.i = sub <2 x i64> %a, %vmull2.i.i.i
   ret <2 x i64> %sub.i.i
 }
 
-define <4 x i32> @test_vqdmlsl_high_n_s16(<4 x i32> %a, <8 x i16> %b, i16 %c) {
+define <2 x i64> @test_vmlsl_high_n_u32_imm(<2 x i64> %a, <4 x i32> %b) #0 {
+; CHECK-LABEL: test_vmlsl_high_n_u32_imm:
+; CHECK-NEXT: movi [[REPLICATE:v[0-9]+]].4s, #0x1d
+; CHECK-NEXT: umlsl2 {{v[0-9]+}}.2d, {{v[0-9]+}}.4s, [[REPLICATE]].4s
+; CHECK-NEXT: ret
+entry:
+  %shuffle.i.i = shufflevector <4 x i32> %b, <4 x i32> undef, <2 x i32> <i32 2, i32 3>
+  %vmull2.i.i.i = call <2 x i64> @llvm.aarch64.neon.umull.v2i64(<2 x i32> %shuffle.i.i, <2 x i32> <i32 29, i32 29>)
+  %sub.i.i = sub <2 x i64> %a, %vmull2.i.i.i
+  ret <2 x i64> %sub.i.i
+}
+
+define <4 x i32> @test_vqdmlsl_high_n_s16(<4 x i32> %a, <8 x i16> %b, i16 %c) #0 {
 ; CHECK-LABEL: test_vqdmlsl_high_n_s16:
-; CHECK: sqdmlsl2 {{v[0-9]+}}.4s, {{v[0-9]+}}.8h, {{v[0-9]+}}.8h
+; CHECK-NEXT: dup [[REPLICATE:v[0-9]+]].8h, w0
+; CHECK-NEXT: sqdmlsl2 {{v[0-9]+}}.4s, {{v[0-9]+}}.8h, [[REPLICATE]].8h
+; CHECK-NEXT: ret
 entry:
   %shuffle.i.i = shufflevector <8 x i16> %b, <8 x i16> undef, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
   %vecinit.i.i = insertelement <4 x i16> undef, i16 %c, i32 0
   %vecinit1.i.i = insertelement <4 x i16> %vecinit.i.i, i16 %c, i32 1
   %vecinit2.i.i = insertelement <4 x i16> %vecinit1.i.i, i16 %c, i32 2
   %vecinit3.i.i = insertelement <4 x i16> %vecinit2.i.i, i16 %c, i32 3
-  %vqdmlsl15.i.i = tail call <4 x i32> @llvm.aarch64.neon.sqdmull.v4i32(<4 x i16> %shuffle.i.i, <4 x i16> %vecinit3.i.i)
-  %vqdmlsl17.i.i = tail call <4 x i32> @llvm.aarch64.neon.sqsub.v4i32(<4 x i32> %a, <4 x i32> %vqdmlsl15.i.i)
+  %vqdmlsl15.i.i = call <4 x i32> @llvm.aarch64.neon.sqdmull.v4i32(<4 x i16> %shuffle.i.i, <4 x i16> %vecinit3.i.i)
+  %vqdmlsl17.i.i = call <4 x i32> @llvm.aarch64.neon.sqsub.v4i32(<4 x i32> %a, <4 x i32> %vqdmlsl15.i.i)
+  ret <4 x i32> %vqdmlsl17.i.i
+}
+
+define <4 x i32> @test_vqdmlsl_high_n_s16_imm(<4 x i32> %a, <8 x i16> %b) #0 {
+; CHECK-LABEL: test_vqdmlsl_high_n_s16_imm:
+; CHECK-NEXT: movi [[REPLICATE:v[0-9]+]].8h, #0x1d
+; CHECK-NEXT: sqdmlsl2 {{v[0-9]+}}.4s, {{v[0-9]+}}.8h, [[REPLICATE]].8h
+; CHECK-NEXT: ret
+entry:
+  %shuffle.i.i = shufflevector <8 x i16> %b, <8 x i16> undef, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
+  %vqdmlsl15.i.i = call <4 x i32> @llvm.aarch64.neon.sqdmull.v4i32(<4 x i16> %shuffle.i.i, <4 x i16> <i16 29, i16 29, i16 29, i16 29>)
+  %vqdmlsl17.i.i = call <4 x i32> @llvm.aarch64.neon.sqsub.v4i32(<4 x i32> %a, <4 x i32> %vqdmlsl15.i.i)
   ret <4 x i32> %vqdmlsl17.i.i
 }
 
-define <2 x i64> @test_vqdmlsl_high_n_s32(<2 x i64> %a, <4 x i32> %b, i32 %c) {
+define <2 x i64> @test_vqdmlsl_high_n_s32(<2 x i64> %a, <4 x i32> %b, i32 %c) #0 {
 ; CHECK-LABEL: test_vqdmlsl_high_n_s32:
-; CHECK: sqdmlsl2 {{v[0-9]+}}.2d, {{v[0-9]+}}.4s, {{v[0-9]+}}.4s
+; CHECK-NEXT: dup [[REPLICATE:v[0-9]+]].4s, w0
+; CHECK-NEXT: sqdmlsl2 {{v[0-9]+}}.2d, {{v[0-9]+}}.4s, [[REPLICATE]].4s
+; CHECK-NEXT: ret
 entry:
   %shuffle.i.i = shufflevector <4 x i32> %b, <4 x i32> undef, <2 x i32> <i32 2, i32 3>
   %vecinit.i.i = insertelement <2 x i32> undef, i32 %c, i32 0
   %vecinit1.i.i = insertelement <2 x i32> %vecinit.i.i, i32 %c, i32 1
-  %vqdmlsl9.i.i = tail call <2 x i64> @llvm.aarch64.neon.sqdmull.v2i64(<2 x i32> %shuffle.i.i, <2 x i32> %vecinit1.i.i)
-  %vqdmlsl11.i.i = tail call <2 x i64> @llvm.aarch64.neon.sqsub.v2i64(<2 x i64> %a, <2 x i64> %vqdmlsl9.i.i)
+  %vqdmlsl9.i.i = call <2 x i64> @llvm.aarch64.neon.sqdmull.v2i64(<2 x i32> %shuffle.i.i, <2 x i32> %vecinit1.i.i)
+  %vqdmlsl11.i.i = call <2 x i64> @llvm.aarch64.neon.sqsub.v2i64(<2 x i64> %a, <2 x i64> %vqdmlsl9.i.i)
+  ret <2 x i64> %vqdmlsl11.i.i
+}
+
+define <2 x i64> @test_vqdmlsl_high_n_s32_imm(<2 x i64> %a, <4 x i32> %b) #0 {
+; CHECK-LABEL: test_vqdmlsl_high_n_s32_imm:
+; CHECK-NEXT: movi [[REPLICATE:v[0-9]+]].4s, #0x1d
+; CHECK-NEXT: sqdmlsl2 {{v[0-9]+}}.2d, {{v[0-9]+}}.4s, [[REPLICATE]].4s
+; CHECK-NEXT: ret
+entry:
+  %shuffle.i.i = shufflevector <4 x i32> %b, <4 x i32> undef, <2 x i32> <i32 2, i32 3>
+  %vqdmlsl9.i.i = call <2 x i64> @llvm.aarch64.neon.sqdmull.v2i64(<2 x i32> %shuffle.i.i, <2 x i32> <i32 29, i32 29>)
+  %vqdmlsl11.i.i = call <2 x i64> @llvm.aarch64.neon.sqsub.v2i64(<2 x i64> %a, <2 x i64> %vqdmlsl9.i.i)
   ret <2 x i64> %vqdmlsl11.i.i
 }
 
-define <2 x float> @test_vmul_n_f32(<2 x float> %a, float %b) {
+define <2 x float> @test_vmul_n_f32(<2 x float> %a, float %b) #0 {
 ; CHECK-LABEL: test_vmul_n_f32:
-; CHECK: fmul {{v[0-9]+}}.2s, {{v[0-9]+}}.2s, {{v[0-9]+}}.s[0]
+; CHECK-NEXT: fmul {{v[0-9]+}}.2s, {{v[0-9]+}}.2s, {{v[0-9]+}}.s[0]
+; CHECK-NEXT: ret
 entry:
   %vecinit.i = insertelement <2 x float> undef, float %b, i32 0
   %vecinit1.i = insertelement <2 x float> %vecinit.i, float %b, i32 1
@@ -272,9 +486,10 @@ entry:
   ret <2 x float> %mul.i
 }
 
-define <4 x float> @test_vmulq_n_f32(<4 x float> %a, float %b) {
+define <4 x float> @test_vmulq_n_f32(<4 x float> %a, float %b) #0 {
 ; CHECK-LABEL: test_vmulq_n_f32:
-; CHECK: fmul {{v[0-9]+}}.4s, {{v[0-9]+}}.4s, {{v[0-9]+}}.s[0]
+; CHECK-NEXT: fmul {{v[0-9]+}}.4s, {{v[0-9]+}}.4s, {{v[0-9]+}}.s[0]
+; CHECK-NEXT: ret
 entry:
   %vecinit.i = insertelement <4 x float> undef, float %b, i32 0
   %vecinit1.i = insertelement <4 x float> %vecinit.i, float %b, i32 1
@@ -284,9 +499,10 @@ entry:
   ret <4 x float> %mul.i
 }
 
-define <2 x double> @test_vmulq_n_f64(<2 x double> %a, double %b) {
+define <2 x double> @test_vmulq_n_f64(<2 x double> %a, double %b) #0 {
 ; CHECK-LABEL: test_vmulq_n_f64:
-; CHECK: fmul {{v[0-9]+}}.2d, {{v[0-9]+}}.2d, {{v[0-9]+}}.d[0]
+; CHECK-NEXT: fmul {{v[0-9]+}}.2d, {{v[0-9]+}}.2d, {{v[0-9]+}}.d[0]
+; CHECK-NEXT: ret
 entry:
   %vecinit.i = insertelement <2 x double> undef, double %b, i32 0
   %vecinit1.i = insertelement <2 x double> %vecinit.i, double %b, i32 1
@@ -294,48 +510,67 @@ entry:
   ret <2 x double> %mul.i
 }
 
-define <2 x float> @test_vfma_n_f32(<2 x float> %a, <2 x float> %b, float %n) {
+define <2 x float> @test_vfma_n_f32(<2 x float> %a, <2 x float> %b, float %n) #0 {
 ; CHECK-LABEL: test_vfma_n_f32:
-; CHECK: fmla {{v[0-9]+}}.2s, {{v[0-9]+}}.2s, {{v[0-9]+}}.s[{{[0-9]+}}]
+; CHECK-NEXT: fmla {{v[0-9]+}}.2s, {{v[0-9]+}}.2s, {{v[0-9]+}}.s[{{[0-9]+}}]
+; CHECK-NEXT: ret
 entry:
   %vecinit.i = insertelement <2 x float> undef, float %n, i32 0
   %vecinit1.i = insertelement <2 x float> %vecinit.i, float %n, i32 1
-  %0 = tail call <2 x float> @llvm.fma.v2f32(<2 x float> %b, <2 x float> %vecinit1.i, <2 x float> %a)
+  %0 = call <2 x float> @llvm.fma.v2f32(<2 x float> %b, <2 x float> %vecinit1.i, <2 x float> %a)
   ret <2 x float> %0
 }
 
-define <4 x float> @test_vfmaq_n_f32(<4 x float> %a, <4 x float> %b, float %n) {
+define <4 x float> @test_vfmaq_n_f32(<4 x float> %a, <4 x float> %b, float %n) #0 {
 ; CHECK-LABEL: test_vfmaq_n_f32:
-; CHECK: fmla {{v[0-9]+}}.4s, {{v[0-9]+}}.4s, {{v[0-9]+}}.s[{{[0-9]+}}]
+; CHECK-NEXT: fmla {{v[0-9]+}}.4s, {{v[0-9]+}}.4s, {{v[0-9]+}}.s[{{[0-9]+}}]
+; CHECK-NEXT: ret
 entry:
   %vecinit.i = insertelement <4 x float> undef, float %n, i32 0
   %vecinit1.i = insertelement <4 x float> %vecinit.i, float %n, i32 1
   %vecinit2.i = insertelement <4 x float> %vecinit1.i, float %n, i32 2
   %vecinit3.i = insertelement <4 x float> %vecinit2.i, float %n, i32 3
-  %0 = tail call <4 x float> @llvm.fma.v4f32(<4 x float> %b, <4 x float> %vecinit3.i, <4 x float> %a)
+  %0 = call <4 x float> @llvm.fma.v4f32(<4 x float> %b, <4 x float> %vecinit3.i, <4 x float> %a)
   ret <4 x float> %0
 }
 
-define <2 x float> @test_vfms_n_f32(<2 x float> %a, <2 x float> %b, float %n) {
+define <2 x float> @test_vfms_n_f32(<2 x float> %a, <2 x float> %b, float %n) #0 {
 ; CHECK-LABEL: test_vfms_n_f32:
-; CHECK: fmls {{v[0-9]+}}.2s, {{v[0-9]+}}.2s, {{v[0-9]+}}.s[{{[0-9]+}}]
+; CHECK-NEXT: fmls {{v[0-9]+}}.2s, {{v[0-9]+}}.2s, {{v[0-9]+}}.s[{{[0-9]+}}]
+; CHECK-NEXT: ret
 entry:
   %vecinit.i = insertelement <2 x float> undef, float %n, i32 0
   %vecinit1.i = insertelement <2 x float> %vecinit.i, float %n, i32 1
   %0 = fsub <2 x float> <float -0.000000e+00, float -0.000000e+00>, %b
-  %1 = tail call <2 x float> @llvm.fma.v2f32(<2 x float> %0, <2 x float> %vecinit1.i, <2 x float> %a)
+  %1 = call <2 x float> @llvm.fma.v2f32(<2 x float> %0, <2 x float> %vecinit1.i, <2 x float> %a)
   ret <2 x float> %1
 }
 
-define <4 x float> @test_vfmsq_n_f32(<4 x float> %a, <4 x float> %b, float %n) {
+define <4 x float> @test_vfmsq_n_f32(<4 x float> %a, <4 x float> %b, float %n) #0 {
 ; CHECK-LABEL: test_vfmsq_n_f32:
-; CHECK: fmls {{v[0-9]+}}.4s, {{v[0-9]+}}.4s, {{v[0-9]+}}.s[{{[0-9]+}}]
+; CHECK-NEXT: fmls {{v[0-9]+}}.4s, {{v[0-9]+}}.4s, {{v[0-9]+}}.s[{{[0-9]+}}]
+; CHECK-NEXT: ret
 entry:
   %vecinit.i = insertelement <4 x float> undef, float %n, i32 0
   %vecinit1.i = insertelement <4 x float> %vecinit.i, float %n, i32 1
   %vecinit2.i = insertelement <4 x float> %vecinit1.i, float %n, i32 2
   %vecinit3.i = insertelement <4 x float> %vecinit2.i, float %n, i32 3
   %0 = fsub <4 x float> <float -0.000000e+00, float -0.000000e+00, float -0.000000e+00, float -0.000000e+00>, %b
-  %1 = tail call <4 x float> @llvm.fma.v4f32(<4 x float> %0, <4 x float> %vecinit3.i, <4 x float> %a)
+  %1 = call <4 x float> @llvm.fma.v4f32(<4 x float> %0, <4 x float> %vecinit3.i, <4 x float> %a)
   ret <4 x float> %1
 }
+
+attributes #0 = { nounwind }
+
+declare <4 x i32> @llvm.aarch64.neon.smull.v4i32(<4 x i16>, <4 x i16>)
+declare <2 x i64> @llvm.aarch64.neon.smull.v2i64(<2 x i32>, <2 x i32>)
+declare <4 x i32> @llvm.aarch64.neon.umull.v4i32(<4 x i16>, <4 x i16>)
+declare <2 x i64> @llvm.aarch64.neon.umull.v2i64(<2 x i32>, <2 x i32>)
+declare <4 x i32> @llvm.aarch64.neon.sqdmull.v4i32(<4 x i16>, <4 x i16>)
+declare <2 x i64> @llvm.aarch64.neon.sqdmull.v2i64(<2 x i32>, <2 x i32>)
+declare <4 x i32> @llvm.aarch64.neon.sqsub.v4i32(<4 x i32>, <4 x i32>)
+declare <2 x i64> @llvm.aarch64.neon.sqsub.v2i64(<2 x i64>, <2 x i64>)
+declare <4 x i32> @llvm.aarch64.neon.sqadd.v4i32(<4 x i32>, <4 x i32>)
+declare <2 x i64> @llvm.aarch64.neon.sqadd.v2i64(<2 x i64>, <2 x i64>)
+declare <4 x float> @llvm.fma.v4f32(<4 x float>, <4 x float>, <4 x float>)
+declare <2 x float> @llvm.fma.v2f32(<2 x float>, <2 x float>, <2 x float>)
diff --git a/test/CodeGen/AArch64/arm64-neon-select_cc.ll b/test/CodeGen/AArch64/arm64-neon-select_cc.ll
index d334c0846ac..fe765f4ef98 100644
--- a/test/CodeGen/AArch64/arm64-neon-select_cc.ll
+++ b/test/CodeGen/AArch64/arm64-neon-select_cc.ll
@@ -1,4 +1,5 @@
-; RUN: llc < %s -verify-machineinstrs -mtriple=arm64-none-linux-gnu -mattr=+neon -fp-contract=fast | FileCheck %s
+; RUN: llc  -mtriple=arm64-none-linux-gnu -mattr=+neon -fp-contract=fast \
+; RUN:        < %s -verify-machineinstrs -asm-verbose=false | FileCheck %s
 
 define <8x i8> @test_select_cc_v8i8_i8(i8 %a, i8 %b, <8x i8> %c, <8x i8> %d ) {
 ; CHECK-LABEL: test_select_cc_v8i8_i8:
@@ -219,3 +220,30 @@ define <2 x i32> @test_select_cc_v2i32_icmpi1(i1 %cc, <2 x i32> %a, <2 x i32> %b
   %e = select i1 %cmp, <2 x i32> %a, <2 x i32> %b
   ret <2 x i32> %e
 }
+
+; Also make sure we support irregular/non-power-of-2 types such as v3f32.
+define <3 x float> @test_select_cc_v3f32_fcmp_f32(<3 x float> %a, <3 x float> %b, float %c1, float %c2) #0 {
+; CHECK-LABEL: test_select_cc_v3f32_fcmp_f32:
+; CHECK-NEXT: fcmeq [[MASK:v[0-9]+]].4s, v2.4s, v3.4s
+; CHECK-NEXT: dup [[DUPMASK:v[0-9]+]].4s, [[MASK]].s[0]
+; CHECK-NEXT: bsl [[DUPMASK:v[0-9]+]].16b, v0.16b, v1.16b
+; CHECK-NEXT: mov v0.16b, [[DUPMASK]].16b
+; CHECK-NEXT: ret
+  %cc = fcmp oeq float %c1, %c2
+  %r = select i1 %cc, <3 x float> %a, <3 x float> %b
+  ret <3 x float> %r
+}
+
+define <3 x float> @test_select_cc_v3f32_fcmp_f64(<3 x float> %a, <3 x float> %b, double %c1, double %c2) #0 {
+; CHECK-LABEL: test_select_cc_v3f32_fcmp_f64:
+; CHECK-NEXT: fcmeq [[MASK:v[0-9]+]].2d, v2.2d, v3.2d
+; CHECK-NEXT: dup [[DUPMASK:v[0-9]+]].2d, [[MASK]].d[0]
+; CHECK-NEXT: bsl [[DUPMASK:v[0-9]+]].16b, v0.16b, v1.16b
+; CHECK-NEXT: mov v0.16b, [[DUPMASK]].16b
+; CHECK-NEXT: ret
+  %cc = fcmp oeq double %c1, %c2
+  %r = select i1 %cc, <3 x float> %a, <3 x float> %b
+  ret <3 x float> %r
+}
+
+attributes #0 = { nounwind}
diff --git a/test/CodeGen/AArch64/arm64-neon-v1i1-setcc.ll b/test/CodeGen/AArch64/arm64-neon-v1i1-setcc.ll
index 74e3af8206f..c739e9dcd90 100644
--- a/test/CodeGen/AArch64/arm64-neon-v1i1-setcc.ll
+++ b/test/CodeGen/AArch64/arm64-neon-v1i1-setcc.ll
@@ -50,9 +50,15 @@ define <1 x double> @test_select_v1i1_2(<1 x i64> %v1, <1 x i64> %v2, <1 x doubl
   ret <1 x double> %res
 }
 
+; For v1i64, it's not clear which of the vector or scalar compare is better.
+; Let's stick to the vector form, like for all other vector selects fed by a
+; scalar setcc.  If anything, it exposes more ILP.
 define <1 x i64> @test_select_v1i1_3(i64 %lhs, i64 %rhs, <1 x i64> %v3) {
 ; CHECK-LABEL: test_select_v1i1_3:
-; CHECK: cmp {{x[0-9]+}}, {{x[0-9]+}}
+; CHECK: fmov d{{[0-9]+}}, x{{[0-9]+}}
+; CHECK: fmov d{{[0-9]+}}, x{{[0-9]+}}
+; CHECK: cmeq d{{[0-9]+}}, d{{[0-9]+}}, d{{[0-9]+}}
+; CHECK: bic v{{[0-9]+}}.8b, v{{[0-9]+}}.8b, v{{[0-9]+}}.8b
   %tst = icmp eq i64 %lhs, %rhs
   %evil = insertelement <1 x i1> undef, i1 %tst, i32 0
   %res = select <1 x i1> %evil, <1 x i64> zeroinitializer, <1 x i64> %v3
diff --git a/test/CodeGen/AArch64/arm64-nvcast.ll b/test/CodeGen/AArch64/arm64-nvcast.ll
new file mode 100644
index 00000000000..3cb1bf25fc3
--- /dev/null
+++ b/test/CodeGen/AArch64/arm64-nvcast.ll
@@ -0,0 +1,29 @@
+; RUN: llc < %s -mtriple=arm64-apple-ios | FileCheck %s
+
+; CHECK-LABEL: _test:
+; CHECK:  fmov.2d v0, #2.00000000
+; CHECK:  str  q0, [sp]
+; CHECK:  mov  x8, sp
+; CHECK:  ldr s0, [x8, w1, sxtw #2]
+; CHECK:  str  s0, [x0]
+
+define void @test(float * %p1, i32 %v1) {
+entry:
+  %v2 = extractelement <3 x float> <float 0.000000e+00, float 2.000000e+00, float 0.000000e+00>, i32 %v1
+  store float %v2, float* %p1, align 4
+  ret void
+}
+
+; CHECK-LABEL: _test2
+; CHECK: movi.16b  v0, #0x3f
+; CHECK: str  q0, [sp]
+; CHECK: mov  x8, sp
+; CHECK: ldr s0, [x8, w1, sxtw #2]
+; CHECK: str  s0, [x0]
+
+define void @test2(float * %p1, i32 %v1) {
+entry:
+  %v2 = extractelement <3 x float> <float 0.7470588088035583, float 0.7470588088035583, float 0.7470588088035583>, i32 %v1
+  store float %v2, float* %p1, align 4
+  ret void
+}
diff --git a/test/CodeGen/AArch64/arm64-platform-reg.ll b/test/CodeGen/AArch64/arm64-platform-reg.ll
index 60672aa3848..f3af01a7355 100644
--- a/test/CodeGen/AArch64/arm64-platform-reg.ll
+++ b/test/CodeGen/AArch64/arm64-platform-reg.ll
@@ -1,5 +1,5 @@
-; RUN: llc -mtriple=arm64-apple-ios -o - %s | FileCheck %s --check-prefix=CHECK-RESERVE-X18
-; RUN: llc -mtriple=arm64-freebsd-gnu -aarch64-reserve-x18 -o - %s | FileCheck %s --check-prefix=CHECK-RESERVE-X18
+; RUN: llc -mtriple=arm64-apple-ios -mattr=+reserve-x18 -o - %s | FileCheck %s --check-prefix=CHECK-RESERVE-X18
+; RUN: llc -mtriple=arm64-freebsd-gnu -mattr=+reserve-x18 -o - %s | FileCheck %s --check-prefix=CHECK-RESERVE-X18
 ; RUN: llc -mtriple=arm64-linux-gnu -o - %s | FileCheck %s
 
 ; x18 is reserved as a platform register on Darwin but not on other
diff --git a/test/CodeGen/AArch64/arm64-popcnt.ll b/test/CodeGen/AArch64/arm64-popcnt.ll
index b0b529a13f4..d6c9471b7a1 100644
--- a/test/CodeGen/AArch64/arm64-popcnt.ll
+++ b/test/CodeGen/AArch64/arm64-popcnt.ll
@@ -4,8 +4,8 @@
 define i32 @cnt32_advsimd(i32 %x) nounwind readnone {
   %cnt = tail call i32 @llvm.ctpop.i32(i32 %x)
   ret i32 %cnt
-; CHECK: ubfx	x{{[0-9]+}}
-; CHECK: fmov	d0, x{{[0-9]+}}
+; CHECK: mov w[[IN64:[0-9]+]], w0
+; CHECK: fmov	d0, x[[IN64]]
 ; CHECK: cnt.8b	v0, v0
 ; CHECK: uaddlv.8b	h0, v0
 ; CHECK: fmov w0, s0
diff --git a/test/CodeGen/AArch64/arm64-promote-const.ll b/test/CodeGen/AArch64/arm64-promote-const.ll
index 5dd92a7d34e..0be2f5c08c0 100644
--- a/test/CodeGen/AArch64/arm64-promote-const.ll
+++ b/test/CodeGen/AArch64/arm64-promote-const.ll
@@ -40,7 +40,7 @@ define <16 x i8> @test2(<16 x i8> %arg) {
 entry:
 ; PROMOTED-LABEL: test2:
 ; In stress mode, constant vector are promoted
-; PROMOTED: adrp [[PAGEADDR:x[0-9]+]], [[CSTV1:__PromotedConst[0-9]+]]@PAGE
+; PROMOTED: adrp [[PAGEADDR:x[0-9]+]], [[CSTV1:__PromotedConst.[0-9]+]]@PAGE
 ; PROMOTED: ldr q[[REGNUM:[0-9]+]], {{\[}}[[PAGEADDR]], [[CSTV1]]@PAGEOFF]
 ; Destination register is defined by ABI
 ; PROMOTED-NEXT: add.16b v0, v0, v[[REGNUM]]
diff --git a/test/CodeGen/AArch64/arm64-rounding.ll b/test/CodeGen/AArch64/arm64-rounding.ll
index 931114447ad..b2686b6b05f 100644
--- a/test/CodeGen/AArch64/arm64-rounding.ll
+++ b/test/CodeGen/AArch64/arm64-rounding.ll
@@ -1,10 +1,13 @@
-; RUN: llc -O3 < %s -mcpu=cyclone | FileCheck %s
-target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-n32:64"
-target triple = "arm64-apple-ios6.0.0"
+; RUN: llc -O3 < %s -mtriple=arm64-apple-ios7.0 | FileCheck %s --check-prefix=CHECK-INEXACT
+; RUN: llc -O3 < %s -mtriple=aarch64-linux-gnu | FileCheck %s --check-prefix=CHECK-FAST
 
-; CHECK: test1
-; CHECK: frintx
-; CHECK: frintm
+; CHECK-INEXACT-LABEL: test1:
+; CHECK-INEXACT-DAG: frintm
+; CHECK-INEXACT-DAG: frintx
+
+; CHECK-FAST-LABEL: test1:
+; CHECK-FAST: frintm
+; CHECK-FAST-NOT: frintx
 define float @test1(float %a) #0 {
 entry:
   %call = tail call float @floorf(float %a) nounwind readnone
@@ -13,9 +16,13 @@ entry:
 
 declare float @floorf(float) nounwind readnone
 
-; CHECK: test2
-; CHECK: frintx
-; CHECK: frintm
+; CHECK-INEXACT-LABEL: test2:
+; CHECK-INEXACT: frintm
+; CHECK-INEXACT: frintx
+
+; CHECK-FAST-LABEL: test2:
+; CHECK-FAST: frintm
+; CHECK-FAST-NOT: frintx
 define double @test2(double %a) #0 {
 entry:
   %call = tail call double @floor(double %a) nounwind readnone
@@ -24,8 +31,11 @@ entry:
 
 declare double @floor(double) nounwind readnone
 
-; CHECK: test3
-; CHECK: frinti
+; CHECK-INEXACT-LABEL: test3:
+; CHECK-INEXACT: frinti
+
+; CHECK-FAST-LABEL: test3:
+; CHECK-FAST: frinti
 define float @test3(float %a) #0 {
 entry:
   %call = tail call float @nearbyintf(float %a) nounwind readnone
@@ -34,8 +44,11 @@ entry:
 
 declare float @nearbyintf(float) nounwind readnone
 
-; CHECK: test4
-; CHECK: frinti
+; CHECK-INEXACT-LABEL: test4:
+; CHECK-INEXACT: frinti
+
+; CHECK-FAST-LABEL: test4:
+; CHECK-FAST: frinti
 define double @test4(double %a) #0 {
 entry:
   %call = tail call double @nearbyint(double %a) nounwind readnone
@@ -44,9 +57,13 @@ entry:
 
 declare double @nearbyint(double) nounwind readnone
 
-; CHECK: test5
-; CHECK: frintx
-; CHECK: frintp
+; CHECK-INEXACT-LABEL: test5:
+; CHECK-INEXACT: frintp
+; CHECK-INEXACT: frintx
+
+; CHECK-FAST-LABEL: test5:
+; CHECK-FAST: frintp
+; CHECK-FAST-NOT: frintx
 define float @test5(float %a) #0 {
 entry:
   %call = tail call float @ceilf(float %a) nounwind readnone
@@ -55,9 +72,13 @@ entry:
 
 declare float @ceilf(float) nounwind readnone
 
-; CHECK: test6
-; CHECK: frintx
-; CHECK: frintp
+; CHECK-INEXACT-LABEL: test6:
+; CHECK-INEXACT: frintp
+; CHECK-INEXACT: frintx
+
+; CHECK-FAST-LABEL: test6:
+; CHECK-FAST: frintp
+; CHECK-FAST-NOT: frintx
 define double @test6(double %a) #0 {
 entry:
   %call = tail call double @ceil(double %a) nounwind readnone
@@ -66,8 +87,11 @@ entry:
 
 declare double @ceil(double) nounwind readnone
 
-; CHECK: test7
-; CHECK: frintx
+; CHECK-INEXACT-LABEL: test7:
+; CHECK-INEXACT: frintx
+
+; CHECK-FAST-LABEL: test7:
+; CHECK-FAST: frintx
 define float @test7(float %a) #0 {
 entry:
   %call = tail call float @rintf(float %a) nounwind readnone
@@ -76,8 +100,11 @@ entry:
 
 declare float @rintf(float) nounwind readnone
 
-; CHECK: test8
-; CHECK: frintx
+; CHECK-INEXACT-LABEL: test8:
+; CHECK-INEXACT: frintx
+
+; CHECK-FAST-LABEL: test8:
+; CHECK-FAST: frintx
 define double @test8(double %a) #0 {
 entry:
   %call = tail call double @rint(double %a) nounwind readnone
@@ -86,9 +113,13 @@ entry:
 
 declare double @rint(double) nounwind readnone
 
-; CHECK: test9
-; CHECK: frintx
-; CHECK: frintz
+; CHECK-INEXACT-LABEL: test9:
+; CHECK-INEXACT: frintz
+; CHECK-INEXACT: frintx
+
+; CHECK-FAST-LABEL: test9:
+; CHECK-FAST: frintz
+; CHECK-FAST-NOT: frintx
 define float @test9(float %a) #0 {
 entry:
   %call = tail call float @truncf(float %a) nounwind readnone
@@ -97,9 +128,13 @@ entry:
 
 declare float @truncf(float) nounwind readnone
 
-; CHECK: test10
-; CHECK: frintx
-; CHECK: frintz
+; CHECK-INEXACT-LABEL: test10:
+; CHECK-INEXACT: frintz
+; CHECK-INEXACT: frintx
+
+; CHECK-FAST-LABEL: test10:
+; CHECK-FAST: frintz
+; CHECK-FAST-NOT: frintx
 define double @test10(double %a) #0 {
 entry:
   %call = tail call double @trunc(double %a) nounwind readnone
@@ -108,9 +143,13 @@ entry:
 
 declare double @trunc(double) nounwind readnone
 
-; CHECK: test11
-; CHECK: frintx
-; CHECK: frinta
+; CHECK-INEXACT-LABEL: test11:
+; CHECK-INEXACT: frinta
+; CHECK-INEXACT: frintx
+
+; CHECK-FAST-LABEL: test11:
+; CHECK-FAST: frinta
+; CHECK-FAST-NOT: frintx
 define float @test11(float %a) #0 {
 entry:
   %call = tail call float @roundf(float %a) nounwind readnone
@@ -119,9 +158,13 @@ entry:
 
 declare float @roundf(float %a) nounwind readnone
 
-; CHECK: test12
-; CHECK: frintx
-; CHECK: frinta
+; CHECK-INEXACT-LABEL: test12:
+; CHECK-INEXACT: frinta
+; CHECK-INEXACT: frintx
+
+; CHECK-FAST-LABEL: test12:
+; CHECK-FAST: frinta
+; CHECK-FAST-NOT: frintx
 define double @test12(double %a) #0 {
 entry:
   %call = tail call double @round(double %a) nounwind readnone
@@ -130,72 +173,104 @@ entry:
 
 declare double @round(double %a) nounwind readnone
 
-; CHECK: test13
-; CHECK-NOT: frintx
-; CHECK: frintm
+; CHECK-INEXACT-LABEL: test13:
+; CHECK-INEXACT-NOT: frintx
+; CHECK-INEXACT: frintm
+
+; CHECK-FAST-LABEL: test13:
+; CHECK-FAST-NOT: frintx
+; CHECK-FAST: frintm
 define float @test13(float %a) #1 {
 entry:
   %call = tail call float @floorf(float %a) nounwind readnone
   ret float %call
 }
 
-; CHECK: test14
-; CHECK-NOT: frintx
-; CHECK: frintm
+; CHECK-INEXACT-LABEL: test14:
+; CHECK-INEXACT-NOT: frintx
+; CHECK-INEXACT: frintm
+
+; CHECK-FAST-LABEL: test14:
+; CHECK-FAST-NOT: frintx
+; CHECK-FAST: frintm
 define double @test14(double %a) #1 {
 entry:
   %call = tail call double @floor(double %a) nounwind readnone
   ret double %call
 }
 
-; CHECK: test15
-; CHECK-NOT: frintx
-; CHECK: frintp
+; CHECK-INEXACT-LABEL: test15:
+; CHECK-INEXACT-NOT: frintx
+; CHECK-INEXACT: frintp
+
+; CHECK-FAST-LABEL: test15:
+; CHECK-FAST-NOT: frintx
+; CHECK-FAST: frintp
 define float @test15(float %a) #1 {
 entry:
   %call = tail call float @ceilf(float %a) nounwind readnone
   ret float %call
 }
 
-; CHECK: test16
-; CHECK-NOT: frintx
-; CHECK: frintp
+; CHECK-INEXACT-LABEL: test16:
+; CHECK-INEXACT-NOT: frintx
+; CHECK-INEXACT: frintp
+
+; CHECK-FAST-LABEL: test16:
+; CHECK-FAST-NOT: frintx
+; CHECK-FAST: frintp
 define double @test16(double %a) #1 {
 entry:
   %call = tail call double @ceil(double %a) nounwind readnone
   ret double %call
 }
 
-; CHECK: test17
-; CHECK-NOT: frintx
-; CHECK: frintz
+; CHECK-INEXACT-LABEL: test17:
+; CHECK-INEXACT-NOT: frintx
+; CHECK-INEXACT: frintz
+
+; CHECK-FAST-LABEL: test17:
+; CHECK-FAST-NOT: frintx
+; CHECK-FAST: frintz
 define float @test17(float %a) #1 {
 entry:
   %call = tail call float @truncf(float %a) nounwind readnone
   ret float %call
 }
 
-; CHECK: test18
-; CHECK-NOT: frintx
-; CHECK: frintz
+; CHECK-INEXACT-LABEL: test18:
+; CHECK-INEXACT-NOT: frintx
+; CHECK-INEXACT: frintz
+
+; CHECK-FAST-LABEL: test18:
+; CHECK-FAST-NOT: frintx
+; CHECK-FAST: frintz
 define double @test18(double %a) #1 {
 entry:
   %call = tail call double @trunc(double %a) nounwind readnone
   ret double %call
 }
 
-; CHECK: test19
-; CHECK-NOT: frintx
-; CHECK: frinta
+; CHECK-INEXACT-LABEL: test19:
+; CHECK-INEXACT-NOT: frintx
+; CHECK-INEXACT: frinta
+
+; CHECK-FAST-LABEL: test19:
+; CHECK-FAST-NOT: frintx
+; CHECK-FAST: frinta
 define float @test19(float %a) #1 {
 entry:
   %call = tail call float @roundf(float %a) nounwind readnone
   ret float %call
 }
 
-; CHECK: test20
-; CHECK-NOT: frintx
-; CHECK: frinta
+; CHECK-INEXACT-LABEL: test20:
+; CHECK-INEXACT-NOT: frintx
+; CHECK-INEXACT: frinta
+
+; CHECK-FAST-LABEL: test20:
+; CHECK-FAST-NOT: frintx
+; CHECK-FAST: frinta
 define double @test20(double %a) #1 {
 entry:
   %call = tail call double @round(double %a) nounwind readnone
diff --git a/test/CodeGen/AArch64/arm64-shrink-wrapping.ll b/test/CodeGen/AArch64/arm64-shrink-wrapping.ll
new file mode 100644
index 00000000000..599712be401
--- /dev/null
+++ b/test/CodeGen/AArch64/arm64-shrink-wrapping.ll
@@ -0,0 +1,541 @@
+; RUN: llc %s -o - -enable-shrink-wrap=true | FileCheck %s --check-prefix=CHECK --check-prefix=ENABLE
+; RUN: llc %s -o - -enable-shrink-wrap=false | FileCheck %s --check-prefix=CHECK --check-prefix=DISABLE
+target datalayout = "e-m:o-i64:64-i128:128-n32:64-S128"
+target triple = "arm64-apple-ios"
+
+
+; Initial motivating example: Simple diamond with a call just on one side.
+; CHECK-LABEL: foo:
+;
+; Compare the arguments and jump to exit.
+; No prologue needed.
+; ENABLE: cmp w0, w1
+; ENABLE-NEXT: b.ge [[EXIT_LABEL:LBB[0-9_]+]]
+;
+; Prologue code.
+; CHECK: stp [[SAVE_SP:x[0-9]+]], [[CSR:x[0-9]+]], [sp, #-16]!
+; CHECK-NEXT: mov [[SAVE_SP]], sp
+; CHECK-NEXT: sub sp, sp, #16
+;
+; Compare the arguments and jump to exit.
+; After the prologue is set.
+; DISABLE: cmp w0, w1
+; DISABLE-NEXT: b.ge [[EXIT_LABEL:LBB[0-9_]+]]
+;
+; Store %a in the alloca.
+; CHECK: stur w0, {{\[}}[[SAVE_SP]], #-4]
+; Set the alloca address in the second argument.
+; CHECK-NEXT: sub x1, [[SAVE_SP]], #4
+; Set the first argument to zero.
+; CHECK-NEXT: mov w0, wzr
+; CHECK-NEXT: bl _doSomething
+; 
+; Without shrink-wrapping, epilogue is in the exit block.
+; DISABLE: [[EXIT_LABEL]]:
+; Epilogue code.
+; CHECK-NEXT: mov sp, [[SAVE_SP]]
+; CHECK-NEXT: ldp [[SAVE_SP]], [[CSR]], [sp], #16
+;
+; With shrink-wrapping, exit block is a simple return.
+; ENABLE: [[EXIT_LABEL]]:
+; CHECK-NEXT: ret
+define i32 @foo(i32 %a, i32 %b) {
+  %tmp = alloca i32, align 4
+  %tmp2 = icmp slt i32 %a, %b
+  br i1 %tmp2, label %true, label %false
+
+true:
+  store i32 %a, i32* %tmp, align 4
+  %tmp4 = call i32 @doSomething(i32 0, i32* %tmp)
+  br label %false
+
+false:
+  %tmp.0 = phi i32 [ %tmp4, %true ], [ %a, %0 ]
+  ret i32 %tmp.0
+}
+
+; Function Attrs: optsize
+declare i32 @doSomething(i32, i32*)
+
+
+; Check that we do not perform the restore inside the loop whereas the save
+; is outside.
+; CHECK-LABEL: freqSaveAndRestoreOutsideLoop:
+;
+; Shrink-wrapping allows to skip the prologue in the else case.
+; ENABLE: cbz w0, [[ELSE_LABEL:LBB[0-9_]+]]
+;
+; Prologue code.
+; CHECK: stp [[CSR1:x[0-9]+]], [[CSR2:x[0-9]+]], [sp, #-32]!
+; CHECK-NEXT: stp [[CSR3:x[0-9]+]], [[CSR4:x[0-9]+]], [sp, #16]
+; CHECK-NEXT: add [[NEW_SP:x[0-9]+]], sp, #16
+;
+; DISABLE: cbz w0, [[ELSE_LABEL:LBB[0-9_]+]]
+;
+; CHECK: mov [[SUM:w[0-9]+]], wzr
+; CHECK-NEXT: movz [[IV:w[0-9]+]], #0xa
+;
+; Next BB.
+; CHECK: [[LOOP:LBB[0-9_]+]]: ; %for.body
+; CHECK: bl _something
+; CHECK-NEXT: add [[SUM]], w0, [[SUM]]
+; CHECK-NEXT: sub [[IV]], [[IV]], #1
+; CHECK-NEXT: cbnz [[IV]], [[LOOP]]
+;
+; Next BB.
+; Copy SUM into the returned register + << 3.
+; CHECK: lsl w0, [[SUM]], #3
+;
+; Jump to epilogue.
+; DISABLE: b [[EPILOG_BB:LBB[0-9_]+]]
+;
+; DISABLE: [[ELSE_LABEL]]: ; %if.else
+; Shift second argument by one and store into returned register.
+; DISABLE: lsl w0, w1, #1
+; DISABLE: [[EPILOG_BB]]: ; %if.end
+;
+; Epilogue code.
+; CHECK: ldp [[CSR3]], [[CSR4]], [sp, #16]
+; CHECK-NEXT: ldp [[CSR1]], [[CSR2]], [sp], #32
+; CHECK-NEXT: ret
+;
+; ENABLE: [[ELSE_LABEL]]: ; %if.else
+; Shift second argument by one and store into returned register.
+; ENABLE: lsl w0, w1, #1
+; ENABLE: ret
+define i32 @freqSaveAndRestoreOutsideLoop(i32 %cond, i32 %N) {
+entry:
+  %tobool = icmp eq i32 %cond, 0
+  br i1 %tobool, label %if.else, label %for.body
+
+for.body:                                         ; preds = %entry, %for.body
+  %i.05 = phi i32 [ %inc, %for.body ], [ 0, %entry ]
+  %sum.04 = phi i32 [ %add, %for.body ], [ 0, %entry ]
+  %call = tail call i32 bitcast (i32 (...)* @something to i32 ()*)()
+  %add = add nsw i32 %call, %sum.04
+  %inc = add nuw nsw i32 %i.05, 1
+  %exitcond = icmp eq i32 %inc, 10
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:                                          ; preds = %for.body
+  %shl = shl i32 %add, 3
+  br label %if.end
+
+if.else:                                          ; preds = %entry
+  %mul = shl nsw i32 %N, 1
+  br label %if.end
+
+if.end:                                           ; preds = %if.else, %for.end
+  %sum.1 = phi i32 [ %shl, %for.end ], [ %mul, %if.else ]
+  ret i32 %sum.1
+}
+
+declare i32 @something(...)
+
+; Check that we do not perform the shrink-wrapping inside the loop even
+; though that would be legal. The cost model must prevent that.
+; CHECK-LABEL: freqSaveAndRestoreOutsideLoop2:
+; Prologue code.
+; CHECK: stp [[CSR1:x[0-9]+]], [[CSR2:x[0-9]+]], [sp, #-32]!
+; CHECK-NEXT: stp [[CSR3:x[0-9]+]], [[CSR4:x[0-9]+]], [sp, #16]
+; CHECK-NEXT: add [[NEW_SP:x[0-9]+]], sp, #16
+; CHECK: mov [[SUM:w[0-9]+]], wzr
+; CHECK-NEXT: movz [[IV:w[0-9]+]], #0xa
+; Next BB.
+; CHECK: [[LOOP_LABEL:LBB[0-9_]+]]: ; %for.body
+; CHECK: bl _something
+; CHECK-NEXT: add [[SUM]], w0, [[SUM]]
+; CHECK-NEXT: sub [[IV]], [[IV]], #1
+; CHECK-NEXT: cbnz [[IV]], [[LOOP_LABEL]]
+; Next BB.
+; CHECK: ; %for.end
+; CHECK: mov w0, [[SUM]]
+; CHECK-NEXT: ldp [[CSR3]], [[CSR4]], [sp, #16]
+; CHECK-NEXT: ldp [[CSR1]], [[CSR2]], [sp], #32
+; CHECK-NEXT: ret
+define i32 @freqSaveAndRestoreOutsideLoop2(i32 %cond) {
+entry:
+  br label %for.body
+
+for.body:                                         ; preds = %for.body, %entry
+  %i.04 = phi i32 [ 0, %entry ], [ %inc, %for.body ]
+  %sum.03 = phi i32 [ 0, %entry ], [ %add, %for.body ]
+  %call = tail call i32 bitcast (i32 (...)* @something to i32 ()*)()
+  %add = add nsw i32 %call, %sum.03
+  %inc = add nuw nsw i32 %i.04, 1
+  %exitcond = icmp eq i32 %inc, 10
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:                                          ; preds = %for.body
+  ret i32 %add
+}
+
+; Check with a more complex case that we do not have save within the loop and
+; restore outside.
+; CHECK-LABEL: loopInfoSaveOutsideLoop:
+;
+; ENABLE: cbz w0, [[ELSE_LABEL:LBB[0-9_]+]]
+;
+; Prologue code.
+; CHECK: stp [[CSR1:x[0-9]+]], [[CSR2:x[0-9]+]], [sp, #-32]!
+; CHECK-NEXT: stp [[CSR3:x[0-9]+]], [[CSR4:x[0-9]+]], [sp, #16]
+; CHECK-NEXT: add [[NEW_SP:x[0-9]+]], sp, #16
+;
+; DISABLE: cbz w0, [[ELSE_LABEL:LBB[0-9_]+]]
+;
+; CHECK: mov [[SUM:w[0-9]+]], wzr
+; CHECK-NEXT: movz [[IV:w[0-9]+]], #0xa
+;
+; CHECK: [[LOOP_LABEL:LBB[0-9_]+]]: ; %for.body
+; CHECK: bl _something
+; CHECK-NEXT: add [[SUM]], w0, [[SUM]]
+; CHECK-NEXT: sub [[IV]], [[IV]], #1
+; CHECK-NEXT: cbnz [[IV]], [[LOOP_LABEL]]
+; Next BB.
+; CHECK: bl _somethingElse
+; CHECK-NEXT: lsl w0, [[SUM]], #3
+;
+; Jump to epilogue.
+; DISABLE: b [[EPILOG_BB:LBB[0-9_]+]]
+;
+; DISABLE: [[ELSE_LABEL]]: ; %if.else
+; Shift second argument by one and store into returned register.
+; DISABLE: lsl w0, w1, #1
+; DISABLE: [[EPILOG_BB]]: ; %if.end
+; Epilogue code.
+; CHECK-NEXT: ldp [[CSR3]], [[CSR4]], [sp, #16]
+; CHECK-NEXT: ldp [[CSR1]], [[CSR2]], [sp], #32
+; CHECK-NEXT: ret
+;
+; ENABLE: [[ELSE_LABEL]]: ; %if.else
+; Shift second argument by one and store into returned register.
+; ENABLE: lsl w0, w1, #1
+; ENABLE: ret
+define i32 @loopInfoSaveOutsideLoop(i32 %cond, i32 %N) {
+entry:
+  %tobool = icmp eq i32 %cond, 0
+  br i1 %tobool, label %if.else, label %for.body
+
+for.body:                                         ; preds = %entry, %for.body
+  %i.05 = phi i32 [ %inc, %for.body ], [ 0, %entry ]
+  %sum.04 = phi i32 [ %add, %for.body ], [ 0, %entry ]
+  %call = tail call i32 bitcast (i32 (...)* @something to i32 ()*)()
+  %add = add nsw i32 %call, %sum.04
+  %inc = add nuw nsw i32 %i.05, 1
+  %exitcond = icmp eq i32 %inc, 10
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:                                          ; preds = %for.body
+  tail call void bitcast (void (...)* @somethingElse to void ()*)()
+  %shl = shl i32 %add, 3
+  br label %if.end
+
+if.else:                                          ; preds = %entry
+  %mul = shl nsw i32 %N, 1
+  br label %if.end
+
+if.end:                                           ; preds = %if.else, %for.end
+  %sum.1 = phi i32 [ %shl, %for.end ], [ %mul, %if.else ]
+  ret i32 %sum.1
+}
+
+declare void @somethingElse(...)
+
+; Check with a more complex case that we do not have restore within the loop and
+; save outside.
+; CHECK-LABEL: loopInfoRestoreOutsideLoop:
+;
+; ENABLE: cbz w0, [[ELSE_LABEL:LBB[0-9_]+]]
+;
+; CHECK: stp [[CSR1:x[0-9]+]], [[CSR2:x[0-9]+]], [sp, #-32]!
+; CHECK-NEXT: stp [[CSR3:x[0-9]+]], [[CSR4:x[0-9]+]], [sp, #16]
+; CHECK-NEXT: add [[NEW_SP:x[0-9]+]], sp, #16
+;
+; DISABLE: cbz w0, [[ELSE_LABEL:LBB[0-9_]+]]
+;
+; CHECK: bl _somethingElse
+; CHECK-NEXT: mov [[SUM:w[0-9]+]], wzr
+; CHECK-NEXT: movz [[IV:w[0-9]+]], #0xa
+;
+; CHECK: [[LOOP_LABEL:LBB[0-9_]+]]: ; %for.body
+; CHECK: bl _something
+; CHECK-NEXT: add [[SUM]], w0, [[SUM]]
+; CHECK-NEXT: sub [[IV]], [[IV]], #1
+; CHECK-NEXT: cbnz [[IV]], [[LOOP_LABEL]]
+; Next BB.
+; CHECK: lsl w0, [[SUM]], #3
+;
+; Jump to epilogue.
+; DISABLE: b [[EPILOG_BB:LBB[0-9_]+]]
+;
+; DISABLE: [[ELSE_LABEL]]: ; %if.else
+; Shift second argument by one and store into returned register.
+; DISABLE: lsl w0, w1, #1
+; DISABLE: [[EPILOG_BB]]: ; %if.end
+; Epilogue code.
+; CHECK: ldp [[CSR3]], [[CSR4]], [sp, #16]
+; CHECK-NEXT: ldp [[CSR1]], [[CSR2]], [sp], #32
+; CHECK-NEXT: ret
+;
+; ENABLE: [[ELSE_LABEL]]: ; %if.else
+; Shift second argument by one and store into returned register.
+; ENABLE: lsl w0, w1, #1
+; ENABLE: ret
+define i32 @loopInfoRestoreOutsideLoop(i32 %cond, i32 %N) #0 {
+entry:
+  %tobool = icmp eq i32 %cond, 0
+  br i1 %tobool, label %if.else, label %if.then
+
+if.then:                                          ; preds = %entry
+  tail call void bitcast (void (...)* @somethingElse to void ()*)()
+  br label %for.body
+
+for.body:                                         ; preds = %for.body, %if.then
+  %i.05 = phi i32 [ 0, %if.then ], [ %inc, %for.body ]
+  %sum.04 = phi i32 [ 0, %if.then ], [ %add, %for.body ]
+  %call = tail call i32 bitcast (i32 (...)* @something to i32 ()*)()
+  %add = add nsw i32 %call, %sum.04
+  %inc = add nuw nsw i32 %i.05, 1
+  %exitcond = icmp eq i32 %inc, 10
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:                                          ; preds = %for.body
+  %shl = shl i32 %add, 3
+  br label %if.end
+
+if.else:                                          ; preds = %entry
+  %mul = shl nsw i32 %N, 1
+  br label %if.end
+
+if.end:                                           ; preds = %if.else, %for.end
+  %sum.1 = phi i32 [ %shl, %for.end ], [ %mul, %if.else ]
+  ret i32 %sum.1
+}
+
+; Check that we handle function with no frame information correctly.
+; CHECK-LABEL: emptyFrame:
+; CHECK: ; %entry
+; CHECK-NEXT: mov w0, wzr
+; CHECK-NEXT: ret
+define i32 @emptyFrame() {
+entry:
+  ret i32 0
+}
+
+; Check that we handle variadic function correctly.
+; CHECK-LABEL: variadicFunc:
+;
+; ENABLE: cbz w0, [[ELSE_LABEL:LBB[0-9_]+]]
+;
+; Prologue code.
+; CHECK: sub sp, sp, #16
+; DISABLE: cbz w0, [[ELSE_LABEL:LBB[0-9_]+]]
+;
+; Sum is merged with the returned register.
+; CHECK: mov [[SUM:w0]], wzr
+; CHECK-NEXT: add [[VA_BASE:x[0-9]+]], sp, #16
+; CHECK-NEXT: str [[VA_BASE]], [sp, #8]
+; CHECK-NEXT: cmp w1, #1
+; CHECK-NEXT: b.lt [[IFEND_LABEL:LBB[0-9_]+]]
+;
+; CHECK: [[LOOP_LABEL:LBB[0-9_]+]]: ; %for.body
+; CHECK: ldr [[VA_ADDR:x[0-9]+]], [sp, #8]
+; CHECK-NEXT: add [[NEXT_VA_ADDR:x[0-9]+]], [[VA_ADDR]], #8
+; CHECK-NEXT: str [[NEXT_VA_ADDR]], [sp, #8]
+; CHECK-NEXT: ldr [[VA_VAL:w[0-9]+]], {{\[}}[[VA_ADDR]]]
+; CHECK-NEXT: add [[SUM]], [[SUM]], [[VA_VAL]]
+; CHECK-NEXT: sub w1, w1, #1
+; CHECK-NEXT: cbnz w1, [[LOOP_LABEL]]
+;
+; DISABLE-NEXT: b [[IFEND_LABEL]]
+; DISABLE: [[ELSE_LABEL]]: ; %if.else
+; DISABLE: lsl w0, w1, #1
+;
+; CHECK: [[IFEND_LABEL]]:
+; Epilogue code.
+; CHECK: add sp, sp, #16
+; CHECK-NEXT: ret
+;
+; ENABLE: [[ELSE_LABEL]]: ; %if.else
+; ENABLE: lsl w0, w1, #1
+; ENABLE-NEXT: ret
+define i32 @variadicFunc(i32 %cond, i32 %count, ...) #0 {
+entry:
+  %ap = alloca i8*, align 8
+  %tobool = icmp eq i32 %cond, 0
+  br i1 %tobool, label %if.else, label %if.then
+
+if.then:                                          ; preds = %entry
+  %ap1 = bitcast i8** %ap to i8*
+  call void @llvm.va_start(i8* %ap1)
+  %cmp6 = icmp sgt i32 %count, 0
+  br i1 %cmp6, label %for.body, label %for.end
+
+for.body:                                         ; preds = %if.then, %for.body
+  %i.08 = phi i32 [ %inc, %for.body ], [ 0, %if.then ]
+  %sum.07 = phi i32 [ %add, %for.body ], [ 0, %if.then ]
+  %0 = va_arg i8** %ap, i32
+  %add = add nsw i32 %sum.07, %0
+  %inc = add nuw nsw i32 %i.08, 1
+  %exitcond = icmp eq i32 %inc, %count
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:                                          ; preds = %for.body, %if.then
+  %sum.0.lcssa = phi i32 [ 0, %if.then ], [ %add, %for.body ]
+  call void @llvm.va_end(i8* %ap1)
+  br label %if.end
+
+if.else:                                          ; preds = %entry
+  %mul = shl nsw i32 %count, 1
+  br label %if.end
+
+if.end:                                           ; preds = %if.else, %for.end
+  %sum.1 = phi i32 [ %sum.0.lcssa, %for.end ], [ %mul, %if.else ]
+  ret i32 %sum.1
+}
+
+declare void @llvm.va_start(i8*)
+
+declare void @llvm.va_end(i8*)
+
+; Check that we handle inline asm correctly.
+; CHECK-LABEL: inlineAsm:
+;
+; ENABLE: cbz w0, [[ELSE_LABEL:LBB[0-9_]+]]
+;
+; Prologue code.
+; Make sure we save the CSR used in the inline asm: x19.
+; CHECK: stp [[CSR1:x[0-9]+]], [[CSR2:x19]], [sp, #-16]!
+;
+; DISABLE: cbz w0, [[ELSE_LABEL:LBB[0-9_]+]]
+;
+; CHECK: movz [[IV:w[0-9]+]], #0xa
+;
+; CHECK: [[LOOP_LABEL:LBB[0-9_]+]]: ; %for.body
+; Inline asm statement.
+; CHECK: add x19, x19, #1
+; CHECK: sub [[IV]], [[IV]], #1
+; CHECK-NEXT: cbnz [[IV]], [[LOOP_LABEL]]
+; Next BB.
+; CHECK: mov w0, wzr
+; Epilogue code.
+; CHECK-NEXT: ldp [[CSR1]], [[CSR2]], [sp], #16
+; CHECK-NEXT: ret
+; Next BB.
+; CHECK: [[ELSE_LABEL]]: ; %if.else
+; CHECK-NEXT: lsl w0, w1, #1
+; Epilogue code.
+; DISABLE-NEXT: ldp [[CSR1]], [[CSR2]], [sp], #16
+; CHECK-NEXT: ret
+define i32 @inlineAsm(i32 %cond, i32 %N) {
+entry:
+  %tobool = icmp eq i32 %cond, 0
+  br i1 %tobool, label %if.else, label %for.body
+
+for.body:                                         ; preds = %entry, %for.body
+  %i.03 = phi i32 [ %inc, %for.body ], [ 0, %entry ]
+  tail call void asm sideeffect "add x19, x19, #1", "~{x19}"()
+  %inc = add nuw nsw i32 %i.03, 1
+  %exitcond = icmp eq i32 %inc, 10
+  br i1 %exitcond, label %if.end, label %for.body
+
+if.else:                                          ; preds = %entry
+  %mul = shl nsw i32 %N, 1
+  br label %if.end
+
+if.end:                                           ; preds = %for.body, %if.else
+  %sum.0 = phi i32 [ %mul, %if.else ], [ 0, %for.body ]
+  ret i32 %sum.0
+}
+
+; Check that we handle calls to variadic functions correctly.
+; CHECK-LABEL: callVariadicFunc:
+;
+; ENABLE: cbz w0, [[ELSE_LABEL:LBB[0-9_]+]]
+;
+; Prologue code.
+; CHECK: stp [[CSR1:x[0-9]+]], [[CSR2:x[0-9]+]], [sp, #-16]!
+; CHECK-NEXT: mov [[NEW_SP:x[0-9]+]], sp
+; CHECK-NEXT: sub sp, sp, #48
+;
+; DISABLE: cbz w0, [[ELSE_LABEL:LBB[0-9_]+]]
+; Setup of the varags.
+; CHECK: stp x1, x1, [sp, #32]
+; CHECK-NEXT: stp x1, x1, [sp, #16]
+; CHECK-NEXT: stp x1, x1, [sp]
+; CHECK-NEXT: mov w0, w1
+; CHECK-NEXT: bl _someVariadicFunc
+; CHECK-NEXT: lsl w0, w0, #3
+;
+; DISABLE: b [[IFEND_LABEL:LBB[0-9_]+]]
+; DISABLE: [[ELSE_LABEL]]: ; %if.else
+; DISABLE-NEXT: lsl w0, w1, #1
+; DISABLE: [[IFEND_LABEL]]: ; %if.end
+;
+; Epilogue code.
+; CHECK: mov sp, [[NEW_SP]]
+; CHECK-NEXT: ldp [[CSR1]], [[CSR2]], [sp], #16
+; CHECK-NEXT: ret
+;
+; ENABLE: [[ELSE_LABEL]]: ; %if.else
+; ENABLE-NEXT: lsl w0, w1, #1
+; ENABLE-NEXT: ret
+define i32 @callVariadicFunc(i32 %cond, i32 %N) {
+entry:
+  %tobool = icmp eq i32 %cond, 0
+  br i1 %tobool, label %if.else, label %if.then
+
+if.then:                                          ; preds = %entry
+  %call = tail call i32 (i32, ...) @someVariadicFunc(i32 %N, i32 %N, i32 %N, i32 %N, i32 %N, i32 %N, i32 %N)
+  %shl = shl i32 %call, 3
+  br label %if.end
+
+if.else:                                          ; preds = %entry
+  %mul = shl nsw i32 %N, 1
+  br label %if.end
+
+if.end:                                           ; preds = %if.else, %if.then
+  %sum.0 = phi i32 [ %shl, %if.then ], [ %mul, %if.else ]
+  ret i32 %sum.0
+}
+
+declare i32 @someVariadicFunc(i32, ...)
+
+; Make sure we do not insert unreachable code after noreturn function.
+; Although this is not incorrect to insert such code, it is useless
+; and it hurts the binary size.
+;
+; CHECK-LABEL: noreturn:
+; DISABLE: stp
+;
+; CHECK: and [[TEST:w[0-9]+]], w0, #0xff
+; CHECK-NEXT: cbnz [[TEST]], [[ABORT:LBB[0-9_]+]]
+;
+; CHECK: movz w0, #0x2a
+;
+; DISABLE-NEXT: ldp
+;
+; CHECK-NEXT: ret
+;
+; CHECK: [[ABORT]]: ; %if.abort
+;
+; ENABLE: stp
+;
+; CHECK: bl _abort
+; ENABLE-NOT: ldp
+define i32 @noreturn(i8 signext %bad_thing) {
+entry:
+  %tobool = icmp eq i8 %bad_thing, 0
+  br i1 %tobool, label %if.end, label %if.abort
+
+if.abort:
+  tail call void @abort() #0
+  unreachable
+
+if.end:
+  ret i32 42
+}
+
+declare void @abort() #0
+
+attributes #0 = { noreturn nounwind }
diff --git a/test/CodeGen/AArch64/arm64-spill-lr.ll b/test/CodeGen/AArch64/arm64-spill-lr.ll
index 88109088a2f..2ea5d7810a1 100644
--- a/test/CodeGen/AArch64/arm64-spill-lr.ll
+++ b/test/CodeGen/AArch64/arm64-spill-lr.ll
@@ -1,9 +1,9 @@
 ; RUN: llc -mtriple=arm64-apple-ios < %s
 @bar = common global i32 0, align 4
 
-; Leaf function which uses all callee-saved registers and allocates >= 256 bytes on the stack
-; this will cause processFunctionBeforeCalleeSavedScan() to spill LR as an additional scratch
-; register.
+; Leaf function which uses all callee-saved registers and allocates >= 256 bytes
+; on the stack this will cause determineCalleeSaves() to spill LR as an
+; additional scratch register.
 ;
 ; This is a crash-only regression test for rdar://15124582.
 define i32 @foo(i32 %a, i32 %b, i32 %c, i32 %d, i32 %e, i32 %f, i32 %g, i32 %h) nounwind {
diff --git a/test/CodeGen/AArch64/arm64-stackmap.ll b/test/CodeGen/AArch64/arm64-stackmap.ll
index 1a4df7a6f2d..3eb1d275300 100644
--- a/test/CodeGen/AArch64/arm64-stackmap.ll
+++ b/test/CodeGen/AArch64/arm64-stackmap.ll
@@ -1,5 +1,5 @@
-; RUN: llc -mtriple=arm64-apple-darwin                             < %s | FileCheck %s
-; RUN: llc -mtriple=arm64-apple-darwin -fast-isel -fast-isel-abort=1 < %s | FileCheck %s
+; RUN: llc -mtriple=arm64-apple-darwin -mattr=+reserve-x18                             < %s | FileCheck %s
+; RUN: llc -mtriple=arm64-apple-darwin -mattr=+reserve-x18 -fast-isel -fast-isel-abort=1 < %s | FileCheck %s
 ;
 ; Note: Print verbose stackmaps using -debug-only=stackmaps.
 
diff --git a/test/CodeGen/AArch64/arm64-stp-aa.ll b/test/CodeGen/AArch64/arm64-stp-aa.ll
new file mode 100644
index 00000000000..82d343d976b
--- /dev/null
+++ b/test/CodeGen/AArch64/arm64-stp-aa.ll
@@ -0,0 +1,111 @@
+; RUN: llc < %s -march=arm64 -enable-misched=false -aarch64-stp-suppress=false -verify-machineinstrs | FileCheck %s
+
+; The next set of tests makes sure we can combine the second instruction into
+; the first.
+
+; CHECK-LABEL: stp_int_aa
+; CHECK: stp w0, w1, [x2]
+; CHECK: ldr w0, [x2, #8]
+; CHECK: ret
+define i32 @stp_int_aa(i32 %a, i32 %b, i32* nocapture %p) nounwind {
+  store i32 %a, i32* %p, align 4
+  %ld.ptr = getelementptr inbounds i32, i32* %p, i64 2
+  %tmp = load i32, i32* %ld.ptr, align 4
+  %add.ptr = getelementptr inbounds i32, i32* %p, i64 1
+  store i32 %b, i32* %add.ptr, align 4
+  ret i32 %tmp
+}
+
+; CHECK-LABEL: stp_long_aa
+; CHECK: stp x0, x1, [x2]
+; CHECK: ldr x0, [x2, #16]
+; CHECK: ret
+define i64 @stp_long_aa(i64 %a, i64 %b, i64* nocapture %p) nounwind {
+  store i64 %a, i64* %p, align 8
+  %ld.ptr = getelementptr inbounds i64, i64* %p, i64 2
+  %tmp = load i64, i64* %ld.ptr, align 4
+  %add.ptr = getelementptr inbounds i64, i64* %p, i64 1
+  store i64 %b, i64* %add.ptr, align 8
+  ret i64 %tmp
+}
+
+; CHECK-LABEL: stp_float_aa
+; CHECK: stp s0, s1, [x0]
+; CHECK: ldr s0, [x0, #8]
+; CHECK: ret
+define float @stp_float_aa(float %a, float %b, float* nocapture %p) nounwind {
+  store float %a, float* %p, align 4
+  %ld.ptr = getelementptr inbounds float, float* %p, i64 2
+  %tmp = load float, float* %ld.ptr, align 4
+  %add.ptr = getelementptr inbounds float, float* %p, i64 1
+  store float %b, float* %add.ptr, align 4
+  ret float %tmp
+}
+
+; CHECK-LABEL: stp_double_aa
+; CHECK: stp d0, d1, [x0]
+; CHECK: ldr d0, [x0, #16]
+; CHECK: ret
+define double @stp_double_aa(double %a, double %b, double* nocapture %p) nounwind {
+  store double %a, double* %p, align 8
+  %ld.ptr = getelementptr inbounds double, double* %p, i64 2
+  %tmp = load double, double* %ld.ptr, align 4
+  %add.ptr = getelementptr inbounds double, double* %p, i64 1
+  store double %b, double* %add.ptr, align 8
+  ret double %tmp
+}
+
+; The next set of tests makes sure we can combine the first instruction into
+; the second.
+
+; CHECK-LABEL: stp_int_aa_after
+; CHECK: ldr w0, [x3, #4]
+; CHECK: stp w1, w2, [x3]
+; CHECK: ret
+define i32 @stp_int_aa_after(i32 %w0, i32 %a, i32 %b, i32* nocapture %p) nounwind {
+  store i32 %a, i32* %p, align 4
+  %ld.ptr = getelementptr inbounds i32, i32* %p, i64 1
+  %tmp = load i32, i32* %ld.ptr, align 4
+  %add.ptr = getelementptr inbounds i32, i32* %p, i64 1
+  store i32 %b, i32* %add.ptr, align 4
+  ret i32 %tmp
+}
+
+; CHECK-LABEL: stp_long_aa_after
+; CHECK: ldr x0, [x3, #8]
+; CHECK: stp x1, x2, [x3]
+; CHECK: ret
+define i64 @stp_long_aa_after(i64 %x0, i64 %a, i64 %b, i64* nocapture %p) nounwind {
+  store i64 %a, i64* %p, align 8
+  %ld.ptr = getelementptr inbounds i64, i64* %p, i64 1
+  %tmp = load i64, i64* %ld.ptr, align 4
+  %add.ptr = getelementptr inbounds i64, i64* %p, i64 1
+  store i64 %b, i64* %add.ptr, align 8
+  ret i64 %tmp
+}
+
+; CHECK-LABEL: stp_float_aa_after
+; CHECK: ldr s0, [x0, #4]
+; CHECK: stp s1, s2, [x0]
+; CHECK: ret
+define float @stp_float_aa_after(float %s0, float %a, float %b, float* nocapture %p) nounwind {
+  store float %a, float* %p, align 4
+  %ld.ptr = getelementptr inbounds float, float* %p, i64 1
+  %tmp = load float, float* %ld.ptr, align 4
+  %add.ptr = getelementptr inbounds float, float* %p, i64 1
+  store float %b, float* %add.ptr, align 4
+  ret float %tmp
+}
+
+; CHECK-LABEL: stp_double_aa_after
+; CHECK: ldr d0, [x0, #8]
+; CHECK: stp d1, d2, [x0]
+; CHECK: ret
+define double @stp_double_aa_after(double %d0, double %a, double %b, double* nocapture %p) nounwind {
+  store double %a, double* %p, align 8
+  %ld.ptr = getelementptr inbounds double, double* %p, i64 1
+  %tmp = load double, double* %ld.ptr, align 4
+  %add.ptr = getelementptr inbounds double, double* %p, i64 1
+  store double %b, double* %add.ptr, align 8
+  ret double %tmp
+}
diff --git a/test/CodeGen/AArch64/arm64-stp.ll b/test/CodeGen/AArch64/arm64-stp.ll
index 4d76396471a..72561aac6e8 100644
--- a/test/CodeGen/AArch64/arm64-stp.ll
+++ b/test/CodeGen/AArch64/arm64-stp.ll
@@ -99,3 +99,35 @@ entry:
   store <4 x i32> %p20, <4 x i32>* %p21, align 4
   ret void
 }
+
+; Read of %b to compute %tmp2 shouldn't prevent formation of stp
+; CHECK-LABEL: stp_int_rar_hazard
+; CHECK: stp w0, w1, [x2]
+; CHECK: ldr [[REG:w[0-9]+]], [x2, #8]
+; CHECK: add w0, [[REG]], w1
+; CHECK: ret
+define i32 @stp_int_rar_hazard(i32 %a, i32 %b, i32* nocapture %p) nounwind {
+  store i32 %a, i32* %p, align 4
+  %ld.ptr = getelementptr inbounds i32, i32* %p, i64 2
+  %tmp = load i32, i32* %ld.ptr, align 4
+  %tmp2 = add i32 %tmp, %b
+  %add.ptr = getelementptr inbounds i32, i32* %p, i64 1
+  store i32 %b, i32* %add.ptr, align 4
+  ret i32 %tmp2
+}
+
+; Read of %b to compute %tmp2 shouldn't prevent formation of stp
+; CHECK-LABEL: stp_int_rar_hazard_after
+; CHECK: ldr [[REG:w[0-9]+]], [x3, #4]
+; CHECK: add w0, [[REG]], w2
+; CHECK: stp w1, w2, [x3]
+; CHECK: ret
+define i32 @stp_int_rar_hazard_after(i32 %w0, i32 %a, i32 %b, i32* nocapture %p) nounwind {
+  store i32 %a, i32* %p, align 4
+  %ld.ptr = getelementptr inbounds i32, i32* %p, i64 1
+  %tmp = load i32, i32* %ld.ptr, align 4
+  %tmp2 = add i32 %tmp, %b
+  %add.ptr = getelementptr inbounds i32, i32* %p, i64 1
+  store i32 %b, i32* %add.ptr, align 4
+  ret i32 %tmp2
+}
diff --git a/test/CodeGen/AArch64/arm64-strict-align.ll b/test/CodeGen/AArch64/arm64-strict-align.ll
index b707527f3c0..28c158f7a2e 100644
--- a/test/CodeGen/AArch64/arm64-strict-align.ll
+++ b/test/CodeGen/AArch64/arm64-strict-align.ll
@@ -1,6 +1,6 @@
 ; RUN: llc < %s -mtriple=arm64-apple-darwin | FileCheck %s
-; RUN: llc < %s -mtriple=arm64-apple-darwin -aarch64-no-strict-align | FileCheck %s
-; RUN: llc < %s -mtriple=arm64-apple-darwin -aarch64-strict-align | FileCheck %s --check-prefix=CHECK-STRICT
+; RUN: llc < %s -mtriple=arm64-apple-darwin -mattr=+strict-align | FileCheck %s --check-prefix=CHECK-STRICT
+; RUN: llc < %s -mtriple=arm64-apple-darwin -mattr=+strict-align -fast-isel | FileCheck %s --check-prefix=CHECK-STRICT
 
 define i32 @f0(i32* nocapture %p) nounwind {
 ; CHECK-STRICT: ldrh [[HIGH:w[0-9]+]], [x0, #2]
diff --git a/test/CodeGen/AArch64/arm64-tls-dynamic-together.ll b/test/CodeGen/AArch64/arm64-tls-dynamic-together.ll
index f94f88a1183..c95eca062ff 100644
--- a/test/CodeGen/AArch64/arm64-tls-dynamic-together.ll
+++ b/test/CodeGen/AArch64/arm64-tls-dynamic-together.ll
@@ -1,4 +1,7 @@
-; RUN: llc -O0 -mtriple=arm64-none-linux-gnu -relocation-model=pic -verify-machineinstrs < %s | FileCheck %s
+; RUN: llc -O0 -mtriple=arm64-none-linux-gnu -relocation-model=pic \
+; RUN:     -verify-machineinstrs < %s | FileCheck -check-prefix=CHECK -check-prefix=NOEMU %s
+; RUN: llc -emulated-tls -O0 -mtriple=arm64-none-linux-gnu -relocation-model=pic \
+; RUN:     -verify-machineinstrs < %s | FileCheck -check-prefix=CHECK -check-prefix=EMU %s
 
 ; If the .tlsdesccall and blr parts are emitted completely separately (even with
 ; glue) then LLVM will separate them quite happily (with a spill at O0, hence
@@ -13,6 +16,40 @@ define i32 @test_generaldynamic() {
   %val = load i32, i32* @general_dynamic_var
   ret i32 %val
 
-; CHECK: .tlsdesccall general_dynamic_var
-; CHECK-NEXT: blr {{x[0-9]+}}
+; NOEMU: .tlsdesccall general_dynamic_var
+; NOEMU-NEXT: blr {{x[0-9]+}}
+; NOEMU-NOT: __emutls_v.general_dynamic_var:
+
+; EMU: adrp{{.+}}__emutls_v.general_dynamic_var
+; EMU: bl __emutls_get_address
+
+; EMU-NOT: __emutls_v.general_dynamic_var
+; EMU-NOT: __emutls_t.general_dynamic_var
+}
+
+@emulated_init_var = thread_local global i32 37, align 8
+
+define i32 @test_emulated_init() {
+; COMMON-LABEL: test_emulated_init:
+
+  %val = load i32, i32* @emulated_init_var
+  ret i32 %val
+
+; EMU: adrp{{.+}}__emutls_v.emulated_init_var
+; EMU: bl __emutls_get_address
+
+; EMU-NOT: __emutls_v.general_dynamic_var:
+
+; EMU:      .align 3
+; EMU-LABEL: __emutls_v.emulated_init_var:
+; EMU-NEXT: .xword 4
+; EMU-NEXT: .xword 8
+; EMU-NEXT: .xword 0
+; EMU-NEXT: .xword __emutls_t.emulated_init_var
+
+; EMU-LABEL: __emutls_t.emulated_init_var:
+; EMU-NEXT: .word 37
 }
+
+; CHECK-NOT: __emutls_v.general_dynamic_var:
+; EMU-NOT: __emutls_t.general_dynamic_var
diff --git a/test/CodeGen/AArch64/bitfield.ll b/test/CodeGen/AArch64/bitfield.ll
index 78399c80b5d..e1e4f62f662 100644
--- a/test/CodeGen/AArch64/bitfield.ll
+++ b/test/CodeGen/AArch64/bitfield.ll
@@ -60,7 +60,7 @@ define void @test_extendw(i32 %var) {
 
   %uxt64 = zext i32 %var to i64
   store volatile i64 %uxt64, i64* @var64
-; CHECK: ubfx {{x[0-9]+}}, {{x[0-9]+}}, #0, #32
+; CHECK: mov {{w[0-9]+}}, w0
   ret void
 }
 
diff --git a/test/CodeGen/AArch64/br-to-eh-lpad.ll b/test/CodeGen/AArch64/br-to-eh-lpad.ll
index f304ba4ca28..2ac9e904333 100644
--- a/test/CodeGen/AArch64/br-to-eh-lpad.ll
+++ b/test/CodeGen/AArch64/br-to-eh-lpad.ll
@@ -7,12 +7,12 @@
 ; that case, the machine verifier, which relies on analyzing branches for this
 ; kind of verification, is unable to check anything, so accepts the CFG.
 
-define void @test_branch_to_landingpad() {
+define void @test_branch_to_landingpad() personality i8* bitcast (i32 (...)* @__objc_personality_v0 to i8*) {
 entry:
   br i1 undef, label %if.end50.thread, label %if.then6
 
 lpad:
-  %0 = landingpad { i8*, i32 } personality i8* bitcast (i32 (...)* @__objc_personality_v0 to i8*)
+  %0 = landingpad { i8*, i32 }
           catch %struct._objc_typeinfo.12.129.194.285.350.493.519.532.571.597.623.765* @"OBJC_EHTYPE_$_NSString"
           catch %struct._objc_typeinfo.12.129.194.285.350.493.519.532.571.597.623.765* @OBJC_EHTYPE_id
           catch i8* null
@@ -46,7 +46,7 @@ invoke.cont43:
   unreachable
 
 lpad40:
-  %1 = landingpad { i8*, i32 } personality i8* bitcast (i32 (...)* @__objc_personality_v0 to i8*)
+  %1 = landingpad { i8*, i32 }
           catch i8* null
   br label %finally.catchall
 
diff --git a/test/CodeGen/AArch64/combine-comparisons-by-cse.ll b/test/CodeGen/AArch64/combine-comparisons-by-cse.ll
index c78fabac618..004267f4e4e 100644
--- a/test/CodeGen/AArch64/combine-comparisons-by-cse.ll
+++ b/test/CodeGen/AArch64/combine-comparisons-by-cse.ll
@@ -403,6 +403,32 @@ return:                                           ; preds = %land.lhs.true, %con
   ret i32 %retval.0
 }
 
+define void @cmp_shifted(i32 %in, i32 %lhs, i32 %rhs) {
+; CHECK-LABEL: cmp_shifted:
+; CHECK: cmp w0, #1
+; [...]
+; CHECK: cmp w0, #2, lsl #12
+
+  %tst_low = icmp sgt i32 %in, 0
+  br i1 %tst_low, label %true, label %false
+
+true:
+  call i32 @zoo(i32 128)
+  ret void
+
+false:
+  %tst = icmp sgt i32 %in, 8191
+  br i1 %tst, label %truer, label %falser
+
+truer:
+  call i32 @zoo(i32 42)
+  ret void
+
+falser:
+  call i32 @zoo(i32 1)
+  ret void
+}
+
 declare i32 @zoo(i32)
 
 declare double @yoo(i32)
diff --git a/test/CodeGen/AArch64/emutls.ll b/test/CodeGen/AArch64/emutls.ll
new file mode 100644
index 00000000000..40e7b88a0f8
--- /dev/null
+++ b/test/CodeGen/AArch64/emutls.ll
@@ -0,0 +1,368 @@
+; RUN: llc -emulated-tls -mtriple=arm-linux-android \
+; RUN:     -relocation-model=pic < %s | FileCheck -check-prefix=ARM32 %s
+; RUN: llc -emulated-tls -mtriple=aarch64-linux-android \
+; RUN:     -relocation-model=pic < %s | FileCheck -check-prefix=ARM64 %s
+
+; Copied from X86/emutls.ll
+
+; Use my_emutls_get_address like __emutls_get_address.
+@my_emutls_v_xyz = external global i8*, align 4
+declare i8* @my_emutls_get_address(i8*)
+
+define i32 @my_get_xyz() {
+; ARM32-LABEL: my_get_xyz:
+; ARM32:        ldr r0,
+; ARM32-NEXT:   ldr r1,
+; ARM32:        add r0, pc, r0
+; ARM32-NEXT:   ldr r0, [r1, r0]
+; ARM32-NEXT:   bl my_emutls_get_address(PLT)
+; ARM32-NEXT:   ldr r0, [r0]
+; ARM64-LABEL: my_get_xyz:
+; ARM64:        adrp x0, :got:my_emutls_v_xyz
+; ARM64-NEXT:   ldr x0, [x0, :got_lo12:my_emutls_v_xyz]
+; ARM64-NEXT:   bl my_emutls_get_address
+; ARM64-NEXT:   ldr  w0, [x0]
+; ARM64-NEXT:   ldp x29, x30, [sp]
+
+entry:
+  %call = call i8* @my_emutls_get_address(i8* bitcast (i8** @my_emutls_v_xyz to i8*))
+  %0 = bitcast i8* %call to i32*
+  %1 = load i32, i32* %0, align 4
+  ret i32 %1
+}
+
+@i1 = thread_local global i32 15
+@i2 = external thread_local global i32
+@i3 = internal thread_local global i32 15
+@i4 = hidden thread_local global i32 15
+@i5 = external hidden thread_local global i32
+@s1 = thread_local global i16 15
+@b1 = thread_local global i8 0
+
+define i32 @f1() {
+; ARM32-LABEL: f1:
+; ARM32:        ldr r0,
+; ARM32-NEXT:   ldr r1,
+; ARM32:        add r0, pc, r0
+; ARM32-NEXT:   ldr r0, [r1, r0]
+; ARM32-NEXT:   bl __emutls_get_address(PLT)
+; ARM32-NEXT:   ldr r0, [r0]
+; ARM64-LABEL: f1:
+; ARM64:        adrp x0, :got:__emutls_v.i1
+; ARM64-NEXT:   ldr x0, [x0, :got_lo12:__emutls_v.i1]
+; ARM64-NEXT:   bl __emutls_get_address
+; ARM64-NEXT:   ldr  w0, [x0]
+; ARM64-NEXT:   ldp x29, x30, [sp]
+
+entry:
+  %tmp1 = load i32, i32* @i1
+  ret i32 %tmp1
+}
+
+define i32* @f2() {
+; ARM32-LABEL: f2:
+; ARM32:        ldr r0,
+; ARM32-NEXT:   ldr r1,
+; ARM32:        add r0, pc, r0
+; ARM32-NEXT:   ldr r0, [r1, r0]
+; ARM32-NEXT:   bl __emutls_get_address(PLT)
+; ARM32-NEXT:   pop
+; ARM64-LABEL: f2:
+; ARM64:        adrp x0, :got:__emutls_v.i1
+; ARM64-NEXT:   ldr x0, [x0, :got_lo12:__emutls_v.i1]
+; ARM64-NEXT:   bl __emutls_get_address
+; ARM64-NEXT:   ldp x29, x30, [sp]
+
+entry:
+  ret i32* @i1
+}
+
+define i32 @f3() nounwind {
+; ARM32-LABEL: f3:
+; ARM32:        ldr r0,
+; ARM32-NEXT:   ldr r1,
+; ARM32:        add r0, pc, r0
+; ARM32-NEXT:   ldr r0, [r1, r0]
+; ARM32-NEXT:   bl __emutls_get_address(PLT)
+; ARM32-NEXT:   ldr r0, [r0]
+
+entry:
+  %tmp1 = load i32, i32* @i2
+  ret i32 %tmp1
+}
+
+define i32* @f4() {
+; ARM32-LABEL: f4:
+; ARM32:        ldr r0,
+; ARM32-NEXT:   ldr r1,
+; ARM32:        add r0, pc, r0
+; ARM32-NEXT:   ldr r0, [r1, r0]
+; ARM32-NEXT:   bl __emutls_get_address(PLT)
+; ARM32-NEXT:   pop
+
+entry:
+  ret i32* @i2
+}
+
+define i32 @f5() nounwind {
+; ARM32-LABEL: f5:
+; ARM32:        ldr r0,
+; ARM32-NEXT:   ldr r1,
+; ARM32:        add r0, pc, r0
+; ARM32-NEXT:   ldr r0, [r1, r0]
+; ARM32-NEXT:   bl __emutls_get_address(PLT)
+; ARM32-NEXT:   ldr r0, [r0]
+
+entry:
+  %tmp1 = load i32, i32* @i3
+  ret i32 %tmp1
+}
+
+define i32* @f6() {
+; ARM32-LABEL: f6:
+; ARM32:        ldr r0,
+; ARM32-NEXT:   ldr r1,
+; ARM32:        add r0, pc, r0
+; ARM32-NEXT:   ldr r0, [r1, r0]
+; ARM32-NEXT:   bl __emutls_get_address(PLT)
+; ARM32-NEXT:   pop
+
+entry:
+  ret i32* @i3
+}
+
+define i32 @f7() {
+; ARM32-LABEL: f7:
+; ARM32:        ldr r0,
+; ARM32-NEXT:   ldr r1,
+; ARM32:        add r0, pc, r0
+; ARM32-NEXT:   ldr r0, [r1, r0]
+; ARM32-NEXT:   bl __emutls_get_address(PLT)
+; ARM32-NEXT:   ldr r0, [r0]
+
+entry:
+  %tmp1 = load i32, i32* @i4
+  ret i32 %tmp1
+}
+
+define i32* @f8() {
+; ARM32-LABEL: f8:
+; ARM32:        ldr r0,
+; ARM32-NEXT:   ldr r1,
+; ARM32:        add r0, pc, r0
+; ARM32-NEXT:   ldr r0, [r1, r0]
+; ARM32-NEXT:   bl __emutls_get_address(PLT)
+; ARM32-NEXT:   pop
+
+entry:
+  ret i32* @i4
+}
+
+define i32 @f9() {
+; ARM32-LABEL: f9:
+; ARM32:        ldr r0,
+; ARM32-NEXT:   ldr r1,
+; ARM32:        add r0, pc, r0
+; ARM32-NEXT:   ldr r0, [r1, r0]
+; ARM32-NEXT:   bl __emutls_get_address(PLT)
+; ARM32-NEXT:   ldr r0, [r0]
+
+entry:
+  %tmp1 = load i32, i32* @i5
+  ret i32 %tmp1
+}
+
+define i32* @f10() {
+; ARM32-LABEL: f10:
+; ARM32:        ldr r0,
+; ARM32-NEXT:   ldr r1,
+; ARM32:        add r0, pc, r0
+; ARM32-NEXT:   ldr r0, [r1, r0]
+; ARM32-NEXT:   bl __emutls_get_address(PLT)
+; ARM32-NEXT:   pop
+
+entry:
+  ret i32* @i5
+}
+
+define i16 @f11() {
+; ARM32-LABEL: f11:
+; ARM32:        ldr r0,
+; ARM32-NEXT:   ldr r1,
+; ARM32:        add r0, pc, r0
+; ARM32-NEXT:   ldr r0, [r1, r0]
+; ARM32-NEXT:   bl __emutls_get_address(PLT)
+; ARM32-NEXT:   ldrh r0, [r0]
+
+entry:
+  %tmp1 = load i16, i16* @s1
+  ret i16 %tmp1
+}
+
+define i32 @f12() {
+; ARM32-LABEL: f12:
+; ARM32:        ldr r0,
+; ARM32-NEXT:   ldr r1,
+; ARM32:        add r0, pc, r0
+; ARM32-NEXT:   ldr r0, [r1, r0]
+; ARM32-NEXT:   bl __emutls_get_address(PLT)
+; ARM32-NEXT:   ldrsh r0, [r0]
+
+entry:
+  %tmp1 = load i16, i16* @s1
+  %tmp2 = sext i16 %tmp1 to i32
+  ret i32 %tmp2
+}
+
+define i8 @f13() {
+; ARM32-LABEL: f13:
+; ARM32:        ldr r0,
+; ARM32-NEXT:   ldr r1,
+; ARM32:        add r0, pc, r0
+; ARM32-NEXT:   ldr r0, [r1, r0]
+; ARM32-NEXT:   bl __emutls_get_address(PLT)
+; ARM32-NEXT:   ldrb r0, [r0]
+; ARM32-NEXT: pop
+
+entry:
+  %tmp1 = load i8, i8* @b1
+  ret i8 %tmp1
+}
+
+define i32 @f14() {
+; ARM32-LABEL: f14:
+; ARM32:        ldr r0,
+; ARM32-NEXT:   ldr r1,
+; ARM32:        add r0, pc, r0
+; ARM32-NEXT:   ldr r0, [r1, r0]
+; ARM32-NEXT:   bl __emutls_get_address(PLT)
+; ARM32-NEXT:   ldrsb r0, [r0]
+; ARM32-NEXT: pop
+
+entry:
+  %tmp1 = load i8, i8* @b1
+  %tmp2 = sext i8 %tmp1 to i32
+  ret i32 %tmp2
+}
+
+;;;;;;;;;;;;;; 32-bit __emutls_v. and __emutls_t.
+
+; ARM32       .section .data.rel.local,
+; ARM32-LABEL: __emutls_v.i1:
+; ARM32-NEXT: .long 4
+; ARM32-NEXT: .long 4
+; ARM32-NEXT: .long 0
+; ARM32-NEXT: .long __emutls_t.i1
+
+; ARM32       .section .rodata,
+; ARM32-LABEL: __emutls_t.i1:
+; ARM32-NEXT: .long 15
+
+; ARM32-NOT:   __emutls_v.i2
+
+; ARM32       .section .data.rel.local,
+; ARM32-LABEL: __emutls_v.i3:
+; ARM32-NEXT: .long 4
+; ARM32-NEXT: .long 4
+; ARM32-NEXT: .long 0
+; ARM32-NEXT: .long __emutls_t.i3
+
+; ARM32       .section .rodata,
+; ARM32-LABEL: __emutls_t.i3:
+; ARM32-NEXT: .long 15
+
+; ARM32       .section .data.rel.local,
+; ARM32-LABEL: __emutls_v.i4:
+; ARM32-NEXT: .long 4
+; ARM32-NEXT: .long 4
+; ARM32-NEXT: .long 0
+; ARM32-NEXT: .long __emutls_t.i4
+
+; ARM32       .section .rodata,
+; ARM32-LABEL: __emutls_t.i4:
+; ARM32-NEXT: .long 15
+
+; ARM32-NOT:   __emutls_v.i5:
+; ARM32       .hidden __emutls_v.i5
+; ARM32-NOT:   __emutls_v.i5:
+
+; ARM32 .section .data.rel.local,
+; ARM32-LABEL: __emutls_v.s1:
+; ARM32-NEXT: .long 2
+; ARM32-NEXT: .long 2
+; ARM32-NEXT: .long 0
+; ARM32-NEXT: .long __emutls_t.s1
+
+; ARM32 .section .rodata,
+; ARM32-LABEL: __emutls_t.s1:
+; ARM32-NEXT: .short 15
+
+; ARM32 .section .data.rel.local,
+; ARM32-LABEL: __emutls_v.b1:
+; ARM32-NEXT: .long 1
+; ARM32-NEXT: .long 1
+; ARM32-NEXT: .long 0
+; ARM32-NEXT: .long 0
+
+; ARM32-NOT:   __emutls_t.b1
+
+;;;;;;;;;;;;;; 64-bit __emutls_v. and __emutls_t.
+
+; ARM64       .section .data.rel.local,
+; ARM64-LABEL: __emutls_v.i1:
+; ARM64-NEXT: .xword 4
+; ARM64-NEXT: .xword 4
+; ARM64-NEXT: .xword 0
+; ARM64-NEXT: .xword __emutls_t.i1
+
+; ARM64       .section .rodata,
+; ARM64-LABEL: __emutls_t.i1:
+; ARM64-NEXT: .word 15
+
+; ARM64-NOT:   __emutls_v.i2
+
+; ARM64       .section .data.rel.local,
+; ARM64-LABEL: __emutls_v.i3:
+; ARM64-NEXT: .xword 4
+; ARM64-NEXT: .xword 4
+; ARM64-NEXT: .xword 0
+; ARM64-NEXT: .xword __emutls_t.i3
+
+; ARM64       .section .rodata,
+; ARM64-LABEL: __emutls_t.i3:
+; ARM64-NEXT: .word 15
+
+; ARM64       .section .data.rel.local,
+; ARM64-LABEL: __emutls_v.i4:
+; ARM64-NEXT: .xword 4
+; ARM64-NEXT: .xword 4
+; ARM64-NEXT: .xword 0
+; ARM64-NEXT: .xword __emutls_t.i4
+
+; ARM64       .section .rodata,
+; ARM64-LABEL: __emutls_t.i4:
+; ARM64-NEXT: .word 15
+
+; ARM64-NOT:   __emutls_v.i5:
+; ARM64       .hidden __emutls_v.i5
+; ARM64-NOT:   __emutls_v.i5:
+
+; ARM64       .section .data.rel.local,
+; ARM64-LABEL: __emutls_v.s1:
+; ARM64-NEXT: .xword 2
+; ARM64-NEXT: .xword 2
+; ARM64-NEXT: .xword 0
+; ARM64-NEXT: .xword __emutls_t.s1
+
+; ARM64       .section .rodata,
+; ARM64-LABEL: __emutls_t.s1:
+; ARM64-NEXT: .hword 15
+
+; ARM64       .section .data.rel.local,
+; ARM64-LABEL: __emutls_v.b1:
+; ARM64-NEXT: .xword 1
+; ARM64-NEXT: .xword 1
+; ARM64-NEXT: .xword 0
+; ARM64-NEXT: .xword 0
+
+; ARM64-NOT:  __emutls_t.b1
diff --git a/test/CodeGen/AArch64/emutls_generic.ll b/test/CodeGen/AArch64/emutls_generic.ll
new file mode 100644
index 00000000000..d02bf72d80e
--- /dev/null
+++ b/test/CodeGen/AArch64/emutls_generic.ll
@@ -0,0 +1,59 @@
+; RUN: llc < %s -emulated-tls -mtriple=aarch64-linux-android -relocation-model=pic \
+; RUN:     | FileCheck -check-prefix=ARM_64 %s
+; RUN: llc < %s -emulated-tls -mtriple=aarch64-linux-android -relocation-model=pic -O3 \
+; RUN:     | FileCheck -check-prefix=ARM_64 %s
+; RUN: llc < %s -emulated-tls -mtriple=aarch64-linux-android -O3 \
+; RUN:     | FileCheck -check-prefix=ARM_64 %s
+
+; Make sure that TLS symbols are emitted in expected order.
+
+@external_x = external thread_local global i32, align 8
+@external_y = thread_local global i8 7, align 2
+@internal_y = internal thread_local global i64 9, align 16
+
+define i32* @get_external_x() {
+entry:
+  ret i32* @external_x
+}
+
+define i8* @get_external_y() {
+entry:
+  ret i8* @external_y
+}
+
+define i64* @get_internal_y() {
+entry:
+  ret i64* @internal_y
+}
+
+; ARM_64-LABEL:  get_external_x:
+; ARM_64:      __emutls_v.external_x
+; ARM_64:      __emutls_get_address
+; ARM_64-LABEL:  get_external_y:
+; ARM_64:      __emutls_v.external_y
+; ARM_64:      __emutls_get_address
+; ARM_64-LABEL:  get_internal_y:
+; ARM_64:      __emutls_v.internal_y
+; ARM_64:      __emutls_get_address
+; ARM_64-NOT:   __emutls_t.external_x
+; ARM_64-NOT:   __emutls_v.external_x:
+; ARM_64:        .align 3
+; ARM_64-LABEL:  __emutls_v.external_y:
+; ARM_64-NEXT:   .xword 1
+; ARM_64-NEXT:   .xword 2
+; ARM_64-NEXT:   .xword 0
+; ARM_64-NEXT:   .xword __emutls_t.external_y
+; ARM_64-NOT:    __emutls_v.external_x:
+; ARM_64:        .section .rodata,
+; ARM_64-LABEL:  __emutls_t.external_y:
+; ARM_64-NEXT:   .byte 7
+; ARM_64:        .section .data.rel.local
+; ARM_64:        .align 3
+; ARM_64-LABEL:  __emutls_v.internal_y:
+; ARM_64-NEXT:   .xword 8
+; ARM_64-NEXT:   .xword 16
+; ARM_64-NEXT:   .xword 0
+; ARM_64-NEXT:   .xword __emutls_t.internal_y
+; ARM_64:        .section .rodata,
+; ARM_64-LABEL:  __emutls_t.internal_y:
+; ARM_64-NEXT:   .xword 9
diff --git a/test/CodeGen/AArch64/eon.ll b/test/CodeGen/AArch64/eon.ll
new file mode 100644
index 00000000000..ea61ce34c05
--- /dev/null
+++ b/test/CodeGen/AArch64/eon.ll
@@ -0,0 +1,29 @@
+; RUN: llc -mtriple=aarch64-none-linux-gnu < %s | FileCheck %s
+
+; Check that the eon instruction is generated instead of eor,movn
+define i64 @test1(i64 %a, i64 %b, i64 %c) {
+; CHECK-LABEL: test1:
+; CHECK: eon
+; CHECK: ret
+entry:
+  %shl = shl i64 %b, 4
+  %neg = xor i64 %a, -1
+  %xor = xor i64 %shl, %neg
+  ret i64 %xor
+}
+
+; Same check with mutliple uses of %neg
+define i64 @test2(i64 %a, i64 %b, i64 %c) {
+; CHECK-LABEL: test2:
+; CHECK: eon
+; CHECK: eon
+; CHECK: lsl
+; CHECK: ret
+entry:
+  %shl = shl i64 %b, 4
+  %neg = xor i64 %shl, -1
+  %xor = xor i64 %neg, %a
+  %xor1 = xor i64 %c, %neg
+  %shl2 = shl i64 %xor, %xor1
+  ret i64 %shl2
+}
diff --git a/test/CodeGen/AArch64/f16-instructions.ll b/test/CodeGen/AArch64/f16-instructions.ll
index be5e2e51385..234447c0789 100644
--- a/test/CodeGen/AArch64/f16-instructions.ll
+++ b/test/CodeGen/AArch64/f16-instructions.ll
@@ -684,10 +684,9 @@ define half @test_copysign(half %a, half %b) #0 {
 }
 
 ; CHECK-LABEL: test_floor:
-; CHECK-NEXT: fcvt s1, h0
-; CHECK-NEXT: frintm s0, s1
-; CHECK-NEXT: fcvt h0, s0
-; CHECK-NEXT: frintx s1, s1
+; CHECK-NEXT: fcvt [[FLOAT32:s[0-9]+]], h0
+; CHECK-NEXT: frintm [[INT32:s[0-9]+]], [[FLOAT32]]
+; CHECK-NEXT: fcvt h0, [[INT32]]
 ; CHECK-NEXT: ret
 define half @test_floor(half %a) #0 {
   %r = call half @llvm.floor.f16(half %a)
@@ -695,10 +694,9 @@ define half @test_floor(half %a) #0 {
 }
 
 ; CHECK-LABEL: test_ceil:
-; CHECK-NEXT: fcvt s1, h0
-; CHECK-NEXT: frintp s0, s1
-; CHECK-NEXT: fcvt h0, s0
-; CHECK-NEXT: frintx s1, s1
+; CHECK-NEXT: fcvt [[FLOAT32:s[0-9]+]], h0
+; CHECK-NEXT: frintp [[INT32:s[0-9]+]], [[FLOAT32]]
+; CHECK-NEXT: fcvt h0, [[INT32]]
 ; CHECK-NEXT: ret
 define half @test_ceil(half %a) #0 {
   %r = call half @llvm.ceil.f16(half %a)
@@ -706,10 +704,9 @@ define half @test_ceil(half %a) #0 {
 }
 
 ; CHECK-LABEL: test_trunc:
-; CHECK-NEXT: fcvt s1, h0
-; CHECK-NEXT: frintz s0, s1
-; CHECK-NEXT: fcvt h0, s0
-; CHECK-NEXT: frintx s1, s1
+; CHECK-NEXT: fcvt [[FLOAT32:s[0-9]+]], h0
+; CHECK-NEXT: frintz [[INT32:s[0-9]+]], [[FLOAT32]]
+; CHECK-NEXT: fcvt h0, [[INT32]]
 ; CHECK-NEXT: ret
 define half @test_trunc(half %a) #0 {
   %r = call half @llvm.trunc.f16(half %a)
@@ -737,10 +734,9 @@ define half @test_nearbyint(half %a) #0 {
 }
 
 ; CHECK-LABEL: test_round:
-; CHECK-NEXT: fcvt s1, h0
-; CHECK-NEXT: frinta s0, s1
-; CHECK-NEXT: fcvt h0, s0
-; CHECK-NEXT: frintx s1, s1
+; CHECK-NEXT: fcvt [[FLOAT32:s[0-9]+]], h0
+; CHECK-NEXT: frinta [[INT32:s[0-9]+]], [[FLOAT32]]
+; CHECK-NEXT: fcvt h0, [[INT32]]
 ; CHECK-NEXT: ret
 define half @test_round(half %a) #0 {
   %r = call half @llvm.round.f16(half %a)
diff --git a/test/CodeGen/AArch64/fast-isel-address-extends.ll b/test/CodeGen/AArch64/fast-isel-address-extends.ll
new file mode 100644
index 00000000000..f6f79fd1a70
--- /dev/null
+++ b/test/CodeGen/AArch64/fast-isel-address-extends.ll
@@ -0,0 +1,39 @@
+; RUN: llc %s -o - -O2 -verify-machineinstrs -fast-isel=true | FileCheck %s
+
+target datalayout = "e-m:o-i64:64-i128:128-n32:64-S128"
+target triple = "arm64-apple-ios8.0.0"
+
+; This test was trying to fold the sext %tmp142 in to the address arithmetic in %sunkaddr1.
+; This was incorrect as %.mux isn't available in the last bb.
+
+; CHECK: sxtw [[REG:x[0-9]+]]
+; CHECK: strh wzr, {{\[}}{{.*}}, [[REG]], lsl #1]
+
+; Function Attrs: nounwind optsize ssp
+define void @EdgeLoop(i32 %dir, i32 %edge, i32 %width, i16* %tmp89, i32 %tmp136, i16 %tmp144) #0 {
+bb:
+  %tmp2 = icmp eq i32 %dir, 0
+  %.mux = select i1 %tmp2, i32 %width, i32 1
+  %tmp142 = sext i32 %.mux to i64
+  %tmp151 = shl nsw i64 %tmp142, 1
+  %tmp153 = getelementptr inbounds i16, i16* %tmp89, i64 %tmp151
+  %tmp154 = load i16, i16* %tmp153, align 2
+  %tmp155 = zext i16 %tmp154 to i32
+  br i1 %tmp2, label %bb225, label %bb212
+
+bb212:                                            ; preds = %bb
+  store i16 %tmp144, i16* %tmp89, align 2
+  ret void
+
+bb225:                                            ; preds = %bb
+  %tmp248 = trunc i32 %tmp155 to i16
+  store i16 %tmp248, i16* %tmp89, align 2
+  %sunkaddr = ptrtoint i16* %tmp89 to i64
+  %sunkaddr1 = mul i64 %tmp142, 2
+  %sunkaddr2 = add i64 %sunkaddr, %sunkaddr1
+  %sunkaddr3 = inttoptr i64 %sunkaddr2 to i16*
+  store i16 0, i16* %sunkaddr3, align 2
+  ret void
+}
+
+attributes #0 = { nounwind optsize ssp }
diff --git a/test/CodeGen/AArch64/fp16-v16-instructions.ll b/test/CodeGen/AArch64/fp16-v16-instructions.ll
new file mode 100644
index 00000000000..1af2bd10912
--- /dev/null
+++ b/test/CodeGen/AArch64/fp16-v16-instructions.ll
@@ -0,0 +1,105 @@
+; RUN: llc < %s -mtriple=aarch64-none-eabi | FileCheck %s
+
+
+define <16 x half> @sitofp_i32(<16 x i32> %a) #0 {
+; CHECK-LABEL: sitofp_i32:
+; CHECK-DAG: scvtf [[S0:v[0-9]+\.4s]], v0.4s
+; CHECK-DAG: scvtf [[S1:v[0-9]+\.4s]], v1.4s
+; CHECK-DAG: scvtf [[S2:v[0-9]+\.4s]], v2.4s
+; CHECK-DAG: scvtf [[S3:v[0-9]+\.4s]], v3.4s
+; CHECK-DAG: fcvtn v0.4h, [[S0]]
+; CHECK-DAG: fcvtn v1.4h, [[S2]]
+; CHECK-DAG: v[[R1:[0-9]+]].4h, [[S1]]
+; CHECK-DAG: v[[R3:[0-9]+]].4h, [[S3]]
+; CHECK-DAg: ins v0.d[1], v[[R1]].d[0]
+; CHECK-DAG: ins v1.d[1], v[[R3]].d[0]
+
+  %1 = sitofp <16 x i32> %a to <16 x half>
+  ret <16 x half> %1
+}
+
+
+define <16 x half> @sitofp_i64(<16 x i64> %a) #0 {
+; CHECK-LABEL: sitofp_i64:
+; CHECK-DAG: scvtf [[D0:v[0-9]+\.2d]], v0.2d
+; CHECK-DAG: scvtf [[D1:v[0-9]+\.2d]], v1.2d
+; CHECK-DAG: scvtf [[D2:v[0-9]+\.2d]], v2.2d
+; CHECK-DAG: scvtf [[D3:v[0-9]+\.2d]], v3.2d
+; CHECK-DAG: scvtf [[D4:v[0-9]+\.2d]], v4.2d
+; CHECK-DAG: scvtf [[D5:v[0-9]+\.2d]], v5.2d
+; CHECK-DAG: scvtf [[D6:v[0-9]+\.2d]], v6.2d
+; CHECK-DAG: scvtf [[D7:v[0-9]+\.2d]], v7.2d
+
+; CHECK-DAG: fcvtn [[S0:v[0-9]+]].2s, [[D0]]
+; CHECK-DAG: fcvtn [[S1:v[0-9]+]].2s, [[D2]]
+; CHECK-DAG: fcvtn [[S2:v[0-9]+]].2s, [[D4]]
+; CHECK-DAG: fcvtn [[S3:v[0-9]+]].2s, [[D6]]
+
+; CHECK-DAG: fcvtn2 [[S0]].4s, [[D1]]
+; CHECK-DAG: fcvtn2 [[S1]].4s, [[D3]]
+; CHECK-DAG: fcvtn2 [[S2]].4s, [[D5]]
+; CHECK-DAG: fcvtn2 [[S3]].4s, [[D7]]
+
+; CHECK-DAG: fcvtn v0.4h, [[S0]].4s
+; CHECK-DAG: fcvtn v1.4h, [[S2]].4s
+; CHECK-DAG: fcvtn v[[R1:[0-9]+]].4h, [[S1]].4s
+; CHECK-DAG: fcvtn v[[R3:[0-9]+]].4h, [[S3]].4s
+; CHECK-DAG: ins v0.d[1], v[[R1]].d[0]
+; CHECK-DAG: ins v1.d[1], v[[R3]].d[0]
+
+  %1 = sitofp <16 x i64> %a to <16 x half>
+  ret <16 x half> %1
+}
+
+
+define <16 x half> @uitofp_i32(<16 x i32> %a) #0 {
+; CHECK-LABEL: uitofp_i32:
+; CHECK-DAG: ucvtf [[S0:v[0-9]+\.4s]], v0.4s
+; CHECK-DAG: ucvtf [[S1:v[0-9]+\.4s]], v1.4s
+; CHECK-DAG: ucvtf [[S2:v[0-9]+\.4s]], v2.4s
+; CHECK-DAG: ucvtf [[S3:v[0-9]+\.4s]], v3.4s
+; CHECK-DAG: fcvtn v0.4h, [[S0]]
+; CHECK-DAG: fcvtn v1.4h, [[S2]]
+; CHECK-DAG: v[[R1:[0-9]+]].4h, [[S1]]
+; CHECK-DAG: v[[R3:[0-9]+]].4h, [[S3]]
+; CHECK-DAg: ins v0.d[1], v[[R1]].d[0]
+; CHECK-DAG: ins v1.d[1], v[[R3]].d[0]
+
+  %1 = uitofp <16 x i32> %a to <16 x half>
+  ret <16 x half> %1
+}
+
+
+define <16 x half> @uitofp_i64(<16 x i64> %a) #0 {
+; CHECK-LABEL: uitofp_i64:
+; CHECK-DAG: ucvtf [[D0:v[0-9]+\.2d]], v0.2d
+; CHECK-DAG: ucvtf [[D1:v[0-9]+\.2d]], v1.2d
+; CHECK-DAG: ucvtf [[D2:v[0-9]+\.2d]], v2.2d
+; CHECK-DAG: ucvtf [[D3:v[0-9]+\.2d]], v3.2d
+; CHECK-DAG: ucvtf [[D4:v[0-9]+\.2d]], v4.2d
+; CHECK-DAG: ucvtf [[D5:v[0-9]+\.2d]], v5.2d
+; CHECK-DAG: ucvtf [[D6:v[0-9]+\.2d]], v6.2d
+; CHECK-DAG: ucvtf [[D7:v[0-9]+\.2d]], v7.2d
+
+; CHECK-DAG: fcvtn [[S0:v[0-9]+]].2s, [[D0]]
+; CHECK-DAG: fcvtn [[S1:v[0-9]+]].2s, [[D2]]
+; CHECK-DAG: fcvtn [[S2:v[0-9]+]].2s, [[D4]]
+; CHECK-DAG: fcvtn [[S3:v[0-9]+]].2s, [[D6]]
+
+; CHECK-DAG: fcvtn2 [[S0]].4s, [[D1]]
+; CHECK-DAG: fcvtn2 [[S1]].4s, [[D3]]
+; CHECK-DAG: fcvtn2 [[S2]].4s, [[D5]]
+; CHECK-DAG: fcvtn2 [[S3]].4s, [[D7]]
+
+; CHECK-DAG: fcvtn v0.4h, [[S0]].4s
+; CHECK-DAG: fcvtn v1.4h, [[S2]].4s
+; CHECK-DAG: fcvtn v[[R1:[0-9]+]].4h, [[S1]].4s
+; CHECK-DAG: fcvtn v[[R3:[0-9]+]].4h, [[S3]].4s
+; CHECK-DAG: ins v0.d[1], v[[R1]].d[0]
+; CHECK-DAG: ins v1.d[1], v[[R3]].d[0]
+
+  %1 = uitofp <16 x i64> %a to <16 x half>
+  ret <16 x half> %1
+}
+
+attributes #0 = { nounwind }
diff --git a/test/CodeGen/AArch64/fp16-v4-instructions.ll b/test/CodeGen/AArch64/fp16-v4-instructions.ll
index 6db4e9745da..0dbda152fca 100644
--- a/test/CodeGen/AArch64/fp16-v4-instructions.ll
+++ b/test/CodeGen/AArch64/fp16-v4-instructions.ll
@@ -1,4 +1,4 @@
-; RUN: llc < %s -mtriple=aarch64-none-eabi | FileCheck %s
+; RUN: llc < %s -asm-verbose=false -mtriple=aarch64-none-eabi | FileCheck %s
 
 define <4 x half> @add_h(<4 x half> %a, <4 x half> %b) {
 entry:
@@ -129,3 +129,93 @@ define <4 x i16> @bitcast_h_to_i(float, <4 x half> %a) {
   %2 = bitcast <4 x half> %a to <4 x i16>
   ret <4 x i16> %2
 }
+
+
+define <4 x half> @sitofp_i8(<4 x i8> %a) #0 {
+; CHECK-LABEL: sitofp_i8:
+; CHECK-NEXT: shl [[OP1:v[0-9]+\.4h]], v0.4h, #8
+; CHECK-NEXT: sshr [[OP2:v[0-9]+\.4h]], [[OP1]], #8
+; CHECK-NEXT: sshll [[OP3:v[0-9]+\.4s]], [[OP2]], #0
+; CHECK-NEXT: scvtf [[OP4:v[0-9]+\.4s]], [[OP3]]
+; CHECK-NEXT: fcvtn v0.4h, [[OP4]]
+; CHECK-NEXT: ret
+  %1 = sitofp <4 x i8> %a to <4 x half>
+  ret <4 x half> %1
+}
+
+
+define <4 x half> @sitofp_i16(<4 x i16> %a) #0 {
+; CHECK-LABEL: sitofp_i16:
+; CHECK-NEXT: sshll [[OP1:v[0-9]+\.4s]], v0.4h, #0
+; CHECK-NEXT: scvtf [[OP2:v[0-9]+\.4s]], [[OP1]]
+; CHECK-NEXT: fcvtn v0.4h, [[OP2]]
+; CHECK-NEXT: ret
+  %1 = sitofp <4 x i16> %a to <4 x half>
+  ret <4 x half> %1
+}
+
+
+define <4 x half> @sitofp_i32(<4 x i32> %a) #0 {
+; CHECK-LABEL: sitofp_i32:
+; CHECK-NEXT: scvtf [[OP1:v[0-9]+\.4s]], v0.4s
+; CHECK-NEXT: fcvtn v0.4h, [[OP1]]
+  %1 = sitofp <4 x i32> %a to <4 x half>
+  ret <4 x half> %1
+}
+
+
+define <4 x half> @sitofp_i64(<4 x i64> %a) #0 {
+; CHECK-LABEL: sitofp_i64:
+; CHECK-DAG: scvtf [[OP1:v[0-9]+\.2d]], v0.2d
+; CHECK-DAG: scvtf [[OP2:v[0-9]+\.2d]], v1.2d
+; CHECK-DAG: fcvtn [[OP3:v[0-9]+]].2s, [[OP1]]
+; CHECK-NEXT: fcvtn2 [[OP3]].4s, [[OP2]]
+; CHECK-NEXT: fcvtn v0.4h, [[OP3]].4s
+  %1 = sitofp <4 x i64> %a to <4 x half>
+  ret <4 x half> %1
+}
+
+define <4 x half> @uitofp_i8(<4 x i8> %a) #0 {
+; CHECK-LABEL: uitofp_i8:
+; CHECK-NEXT: bic v0.4h, #0xff, lsl #8
+; CHECK-NEXT: ushll [[OP1:v[0-9]+\.4s]], v0.4h, #0
+; CHECK-NEXT: ucvtf [[OP2:v[0-9]+\.4s]], [[OP1]]
+; CHECK-NEXT: fcvtn v0.4h, [[OP2]]
+; CHECK-NEXT: ret
+  %1 = uitofp <4 x i8> %a to <4 x half>
+  ret <4 x half> %1
+}
+
+
+define <4 x half> @uitofp_i16(<4 x i16> %a) #0 {
+; CHECK-LABEL: uitofp_i16:
+; CHECK-NEXT: ushll [[OP1:v[0-9]+\.4s]], v0.4h, #0
+; CHECK-NEXT: ucvtf [[OP2:v[0-9]+\.4s]], [[OP1]]
+; CHECK-NEXT: fcvtn v0.4h, [[OP2]]
+; CHECK-NEXT: ret
+  %1 = uitofp <4 x i16> %a to <4 x half>
+  ret <4 x half> %1
+}
+
+
+define <4 x half> @uitofp_i32(<4 x i32> %a) #0 {
+; CHECK-LABEL: uitofp_i32:
+; CHECK-NEXT: ucvtf [[OP1:v[0-9]+\.4s]], v0.4s
+; CHECK-NEXT: fcvtn v0.4h, [[OP1]]
+  %1 = uitofp <4 x i32> %a to <4 x half>
+  ret <4 x half> %1
+}
+
+
+define <4 x half> @uitofp_i64(<4 x i64> %a) #0 {
+; CHECK-LABEL: uitofp_i64:
+; CHECK-DAG: ucvtf [[OP1:v[0-9]+\.2d]], v0.2d
+; CHECK-DAG: ucvtf [[OP2:v[0-9]+\.2d]], v1.2d
+; CHECK-DAG: fcvtn [[OP3:v[0-9]+]].2s, [[OP1]]
+; CHECK-NEXT: fcvtn2 [[OP3]].4s, [[OP2]]
+; CHECK-NEXT: fcvtn v0.4h, [[OP3]].4s
+  %1 = uitofp <4 x i64> %a to <4 x half>
+  ret <4 x half> %1
+}
+
+attributes #0 = { nounwind }
diff --git a/test/CodeGen/AArch64/fp16-v8-instructions.ll b/test/CodeGen/AArch64/fp16-v8-instructions.ll
index e51c0c5f504..10a8c22d6f7 100644
--- a/test/CodeGen/AArch64/fp16-v8-instructions.ll
+++ b/test/CodeGen/AArch64/fp16-v8-instructions.ll
@@ -1,4 +1,4 @@
-; RUN: llc < %s -mtriple=aarch64-none-eabi | FileCheck %s
+; RUN: llc < %s -asm-verbose=false -mtriple=aarch64-none-eabi | FileCheck %s
 
 define <8 x half> @add_h(<8 x half> %a, <8 x half> %b) {
 entry:
@@ -253,3 +253,109 @@ define <8 x i16> @bitcast_h_to_i(float, <8 x half> %a) {
   ret <8 x i16> %2
 }
 
+
+define <8 x half> @sitofp_i8(<8 x i8> %a) #0 {
+; CHECK-LABEL: sitofp_i8:
+; CHECK-NEXT: sshll v[[REG1:[0-9]+]].8h, v0.8b, #0
+; CHECK-NEXT: sshll2 [[LO:v[0-9]+\.4s]], v[[REG1]].8h, #0
+; CHECK-NEXT: sshll  [[HI:v[0-9]+\.4s]], v[[REG1]].4h, #0
+; CHECK-DAG: scvtf [[HIF:v[0-9]+\.4s]], [[HI]]
+; CHECK-DAG: scvtf [[LOF:v[0-9]+\.4s]], [[LO]]
+; CHECK-DAG: fcvtn v[[LOREG:[0-9]+]].4h, [[LOF]]
+; CHECK-DAG: fcvtn v0.4h, [[HIF]]
+; CHECK: ins v0.d[1], v[[LOREG]].d[0]
+  %1 = sitofp <8 x i8> %a to <8 x half>
+  ret <8 x half> %1
+}
+
+
+define <8 x half> @sitofp_i16(<8 x i16> %a) #0 {
+; CHECK-LABEL: sitofp_i16:
+; CHECK-NEXT: sshll2 [[LO:v[0-9]+\.4s]], v0.8h, #0
+; CHECK-NEXT: sshll  [[HI:v[0-9]+\.4s]], v0.4h, #0
+; CHECK-DAG: scvtf [[HIF:v[0-9]+\.4s]], [[HI]]
+; CHECK-DAG: scvtf [[LOF:v[0-9]+\.4s]], [[LO]]
+; CHECK-DAG: fcvtn v[[LOREG:[0-9]+]].4h, [[LOF]]
+; CHECK-DAG: fcvtn v0.4h, [[HIF]]
+; CHECK: ins v0.d[1], v[[LOREG]].d[0]
+  %1 = sitofp <8 x i16> %a to <8 x half>
+  ret <8 x half> %1
+}
+
+
+define <8 x half> @sitofp_i32(<8 x i32> %a) #0 {
+; CHECK-LABEL: sitofp_i32:
+; CHECK-DAG: scvtf [[OP1:v[0-9]+\.4s]], v0.4s
+; CHECK-DAG: scvtf [[OP2:v[0-9]+\.4s]], v1.4s
+; CHECK-DAG: fcvtn v[[REG:[0-9]+]].4h, [[OP2]]
+; CHECK-DAG: fcvtn v0.4h, [[OP1]]
+; CHECK: ins v0.d[1], v[[REG]].d[0]
+  %1 = sitofp <8 x i32> %a to <8 x half>
+  ret <8 x half> %1
+}
+
+
+define <8 x half> @sitofp_i64(<8 x i64> %a) #0 {
+; CHECK-LABEL: sitofp_i64:
+; CHECK-DAG: scvtf [[OP1:v[0-9]+\.2d]], v0.2d
+; CHECK-DAG: scvtf [[OP2:v[0-9]+\.2d]], v1.2d
+; CHECK-DAG: fcvtn [[OP3:v[0-9]+]].2s, [[OP1]]
+; CHECK-DAG: fcvtn2 [[OP3]].4s, [[OP2]]
+; CHECK: fcvtn v0.4h, [[OP3]].4s
+  %1 = sitofp <8 x i64> %a to <8 x half>
+  ret <8 x half> %1
+}
+
+define <8 x half> @uitofp_i8(<8 x i8> %a) #0 {
+; CHECK-LABEL: uitofp_i8:
+; CHECK-NEXT: ushll v[[REG1:[0-9]+]].8h, v0.8b, #0
+; CHECK-NEXT: ushll2 [[LO:v[0-9]+\.4s]], v[[REG1]].8h, #0
+; CHECK-NEXT: ushll  [[HI:v[0-9]+\.4s]], v[[REG1]].4h, #0
+; CHECK-DAG: ucvtf [[HIF:v[0-9]+\.4s]], [[HI]]
+; CHECK-DAG: ucvtf [[LOF:v[0-9]+\.4s]], [[LO]]
+; CHECK-DAG: fcvtn v[[LOREG:[0-9]+]].4h, [[LOF]]
+; CHECK-DAG: fcvtn v0.4h, [[HIF]]
+; CHECK: ins v0.d[1], v[[LOREG]].d[0]
+  %1 = uitofp <8 x i8> %a to <8 x half>
+  ret <8 x half> %1
+}
+
+
+define <8 x half> @uitofp_i16(<8 x i16> %a) #0 {
+; CHECK-LABEL: uitofp_i16:
+; CHECK-NEXT: ushll2 [[LO:v[0-9]+\.4s]], v0.8h, #0
+; CHECK-NEXT: ushll  [[HI:v[0-9]+\.4s]], v0.4h, #0
+; CHECK-DAG: ucvtf [[HIF:v[0-9]+\.4s]], [[HI]]
+; CHECK-DAG: ucvtf [[LOF:v[0-9]+\.4s]], [[LO]]
+; CHECK-DAG: fcvtn v[[LOREG:[0-9]+]].4h, [[LOF]]
+; CHECK-DAG: fcvtn v0.4h, [[HIF]]
+; CHECK: ins v0.d[1], v[[LOREG]].d[0]
+  %1 = uitofp <8 x i16> %a to <8 x half>
+  ret <8 x half> %1
+}
+
+
+define <8 x half> @uitofp_i32(<8 x i32> %a) #0 {
+; CHECK-LABEL: uitofp_i32:
+; CHECK-DAG: ucvtf [[OP1:v[0-9]+\.4s]], v0.4s
+; CHECK-DAG: ucvtf [[OP2:v[0-9]+\.4s]], v1.4s
+; CHECK-DAG: fcvtn v[[REG:[0-9]+]].4h, [[OP2]]
+; CHECK-DAG: fcvtn v0.4h, [[OP1]]
+; CHECK: ins v0.d[1], v[[REG]].d[0]
+  %1 = uitofp <8 x i32> %a to <8 x half>
+  ret <8 x half> %1
+}
+
+
+define <8 x half> @uitofp_i64(<8 x i64> %a) #0 {
+; CHECK-LABEL: uitofp_i64:
+; CHECK-DAG: ucvtf [[OP1:v[0-9]+\.2d]], v0.2d
+; CHECK-DAG: ucvtf [[OP2:v[0-9]+\.2d]], v1.2d
+; CHECK-DAG: fcvtn [[OP3:v[0-9]+]].2s, [[OP1]]
+; CHECK-DAG: fcvtn2 [[OP3]].4s, [[OP2]]
+; CHECK: fcvtn v0.4h, [[OP3]].4s
+  %1 = uitofp <8 x i64> %a to <8 x half>
+  ret <8 x half> %1
+}
+
+attributes #0 = { nounwind }
diff --git a/test/CodeGen/AArch64/fp16-vector-nvcast.ll b/test/CodeGen/AArch64/fp16-vector-nvcast.ll
new file mode 100644
index 00000000000..83e0df74c3d
--- /dev/null
+++ b/test/CodeGen/AArch64/fp16-vector-nvcast.ll
@@ -0,0 +1,89 @@
+; RUN: llc < %s -asm-verbose=false -mtriple=aarch64-none-eabi | FileCheck %s
+
+; Test pattern (v4f16 (AArch64NvCast (v2i32 FPR64:$src)))
+define void @nvcast_v2i32(<4 x half>* %a) #0 {
+; CHECK-LABEL: nvcast_v2i32:
+; CHECK-NEXT: movi v[[REG:[0-9]+]].2s, #0xab, lsl #16
+; CHECK-NEXT: str d[[REG]], [x0]
+; CHECK-NEXT: ret
+  store volatile <4 x half> <half 0xH0000, half 0xH00AB, half 0xH0000, half 0xH00AB>, <4 x half>* %a
+  ret void
+}
+
+
+; Test pattern (v4f16 (AArch64NvCast (v4i16 FPR64:$src)))
+define void @nvcast_v4i16(<4 x half>* %a) #0 {
+; CHECK-LABEL: nvcast_v4i16:
+; CHECK-NEXT: movi v[[REG:[0-9]+]].4h, #0xab
+; CHECK-NEXT: str d[[REG]], [x0]
+; CHECK-NEXT: ret
+  store volatile <4 x half> <half 0xH00AB, half 0xH00AB, half 0xH00AB, half 0xH00AB>, <4 x half>* %a
+  ret void
+}
+
+
+; Test pattern (v4f16 (AArch64NvCast (v8i8 FPR64:$src)))
+define void @nvcast_v8i8(<4 x half>* %a) #0 {
+; CHECK-LABEL: nvcast_v8i8:
+; CHECK-NEXT: movi v[[REG:[0-9]+]].8b, #0xab
+; CHECK-NEXT: str d[[REG]], [x0]
+; CHECK-NEXT: ret
+  store volatile <4 x half> <half 0xHABAB, half 0xHABAB, half 0xHABAB, half 0xHABAB>, <4 x half>* %a
+  ret void
+}
+
+
+; Test pattern (v4f16 (AArch64NvCast (f64 FPR64:$src)))
+define void @nvcast_f64(<4 x half>* %a) #0 {
+; CHECK-LABEL: nvcast_f64:
+; CHECK-NEXT: movi d[[REG:[0-9]+]], #0000000000000000
+; CHECK-NEXT: str d[[REG]], [x0]
+; CHECK-NEXT: ret
+  store volatile <4 x half> zeroinitializer, <4 x half>* %a
+  ret void
+}
+
+; Test pattern (v8f16 (AArch64NvCast (v4i32 FPR128:$src)))
+define void @nvcast_v4i32(<8 x half>* %a) #0 {
+; CHECK-LABEL: nvcast_v4i32:
+; CHECK-NEXT: movi v[[REG:[0-9]+]].4s, #0xab, lsl #16
+; CHECK-NEXT: str q[[REG]], [x0]
+; CHECK-NEXT: ret
+  store volatile <8 x half> <half 0xH0000, half 0xH00AB, half 0xH0000, half 0xH00AB, half 0xH0000, half 0xH00AB, half 0xH0000, half 0xH00AB>, <8 x half>* %a
+  ret void
+}
+
+
+; Test pattern (v8f16 (AArch64NvCast (v8i16 FPR128:$src)))
+define void @nvcast_v8i16(<8 x half>* %a) #0 {
+; CHECK-LABEL: nvcast_v8i16:
+; CHECK-NEXT: movi v[[REG:[0-9]+]].8h, #0xab
+; CHECK-NEXT: str q[[REG]], [x0]
+; CHECK-NEXT: ret
+  store volatile <8 x half> <half 0xH00AB, half 0xH00AB, half 0xH00AB, half 0xH00AB, half 0xH00AB, half 0xH00AB, half 0xH00AB, half 0xH00AB>, <8 x half>* %a
+  ret void
+}
+
+
+; Test pattern (v8f16 (AArch64NvCast (v16i8 FPR128:$src)))
+define void @nvcast_v16i8(<8 x half>* %a) #0 {
+; CHECK-LABEL: nvcast_v16i8:
+; CHECK-NEXT: movi v[[REG:[0-9]+]].16b, #0xab
+; CHECK-NEXT: str q[[REG]], [x0]
+; CHECK-NEXT: ret
+  store volatile <8 x half> <half 0xHABAB, half 0xHABAB, half 0xHABAB, half 0xHABAB, half 0xHABAB, half 0xHABAB, half 0xHABAB, half 0xHABAB>, <8 x half>* %a
+  ret void
+}
+
+
+; Test pattern (v8f16 (AArch64NvCast (v2i64 FPR128:$src)))
+define void @nvcast_v2i64(<8 x half>* %a) #0 {
+; CHECK-LABEL: nvcast_v2i64:
+; CHECK-NEXT: movi v[[REG:[0-9]+]].2d, #0000000000000000
+; CHECK-NEXT: str q[[REG]], [x0]
+; CHECK-NEXT: ret
+  store volatile <8 x half> zeroinitializer, <8 x half>* %a
+  ret void
+}
+
+attributes #0 = { nounwind }
diff --git a/test/CodeGen/AArch64/global-merge-2.ll b/test/CodeGen/AArch64/global-merge-2.ll
index af684039bf1..cdbdb8aed2e 100644
--- a/test/CodeGen/AArch64/global-merge-2.ll
+++ b/test/CodeGen/AArch64/global-merge-2.ll
@@ -35,10 +35,13 @@ define void @g1(i32 %a1, i32 %a2) {
 
 ;CHECK:	.globl	x
 ;CHECK: x = _MergedGlobals_x
+;CHECK: .size x, 4
 ;CHECK:	.globl	y
 ;CHECK: y = _MergedGlobals_x+4
+;CHECK: .size y, 4
 ;CHECK:	.globl	z
 ;CHECK: z = _MergedGlobals_x+8
+;CHECK: .size z, 4
 
 ;CHECK-APPLE-IOS: .globl	__MergedGlobals_x       ; @_MergedGlobals_x
 ;CHECK-APPLE-IOS: .zerofill __DATA,__common,__MergedGlobals_x,12,3
diff --git a/test/CodeGen/AArch64/global-merge-3.ll b/test/CodeGen/AArch64/global-merge-3.ll
index 925108308e5..2e459a069bb 100644
--- a/test/CodeGen/AArch64/global-merge-3.ll
+++ b/test/CodeGen/AArch64/global-merge-3.ll
@@ -42,8 +42,10 @@ define void @f1(i32 %a1, i32 %a2, i32 %a3) {
 
 ;CHECK:	.globl	x
 ;CHECK: x = _MergedGlobals_x+4
+;CHECK: .size x, 4000
 ;CHECK:	.globl	y
 ;CHECK: y = _MergedGlobals_y
+;CHECK: .size y, 4000
 
 ;CHECK-APPLE-IOS:.globl	_x
 ;CHECK-APPLE-IOS: _x = __MergedGlobals_x+4
diff --git a/test/CodeGen/AArch64/global-merge-group-by-use.ll b/test/CodeGen/AArch64/global-merge-group-by-use.ll
new file mode 100644
index 00000000000..ddc044ed9e0
--- /dev/null
+++ b/test/CodeGen/AArch64/global-merge-group-by-use.ll
@@ -0,0 +1,94 @@
+; RUN: llc -mtriple=aarch64-apple-ios -asm-verbose=false -aarch64-collect-loh=false \
+; RUN:   -aarch64-global-merge -global-merge-group-by-use -global-merge-ignore-single-use=false \
+; RUN:   %s -o - | FileCheck %s
+
+; We assume that globals of the same size aren't reordered inside a set.
+
+; Check that we create two MergedGlobal instances for two functions using
+; disjoint sets of globals
+
+@m1 = internal global i32 0, align 4
+@n1 = internal global i32 0, align 4
+
+; CHECK-LABEL: f1:
+define void @f1(i32 %a1, i32 %a2) #0 {
+; CHECK-NEXT: adrp x8, [[SET1:__MergedGlobals.[0-9]*]]@PAGE
+; CHECK-NEXT: add x8, x8, [[SET1]]@PAGEOFF
+; CHECK-NEXT: stp w0, w1, [x8]
+; CHECK-NEXT: ret
+  store i32 %a1, i32* @m1, align 4
+  store i32 %a2, i32* @n1, align 4
+  ret void
+}
+
+@m2 = internal global i32 0, align 4
+@n2 = internal global i32 0, align 4
+@o2 = internal global i32 0, align 4
+
+; CHECK-LABEL: f2:
+define void @f2(i32 %a1, i32 %a2, i32 %a3) #0 {
+; CHECK-NEXT: adrp x8, [[SET2:__MergedGlobals.[0-9]*]]@PAGE
+; CHECK-NEXT: add x8, x8, [[SET2]]@PAGEOFF
+; CHECK-NEXT: stp w0, w1, [x8]
+; CHECK-NEXT: str w2, [x8, #8]
+; CHECK-NEXT: ret
+  store i32 %a1, i32* @m2, align 4
+  store i32 %a2, i32* @n2, align 4
+  store i32 %a3, i32* @o2, align 4
+  ret void
+}
+
+; Sanity-check (don't worry about cost models) that we pick the biggest subset
+; of all global used "together" directly or indirectly.  Here, that means
+; merging n3, m4, and n4 together, but ignoring m3.
+
+@m3 = internal global i32 0, align 4
+@n3 = internal global i32 0, align 4
+
+; CHECK-LABEL: f3:
+define void @f3(i32 %a1, i32 %a2) #0 {
+; CHECK-NEXT: adrp x8, _m3@PAGE
+; CHECK-NEXT: adrp x9, [[SET3:__MergedGlobals[0-9]*]]@PAGE
+; CHECK-NEXT: str w0, [x8, _m3@PAGEOFF]
+; CHECK-NEXT: str w1, [x9, [[SET3]]@PAGEOFF]
+; CHECK-NEXT: ret
+  store i32 %a1, i32* @m3, align 4
+  store i32 %a2, i32* @n3, align 4
+  ret void
+}
+
+@m4 = internal global i32 0, align 4
+@n4 = internal global i32 0, align 4
+
+; CHECK-LABEL: f4:
+define void @f4(i32 %a1, i32 %a2, i32 %a3) #0 {
+; CHECK-NEXT: adrp x8, [[SET3]]@PAGE
+; CHECK-NEXT: add x8, x8, [[SET3]]@PAGEOFF
+; CHECK-NEXT: stp w0, w1, [x8, #4]
+; CHECK-NEXT: str w2, [x8]
+; CHECK-NEXT: ret
+  store i32 %a1, i32* @m4, align 4
+  store i32 %a2, i32* @n4, align 4
+  store i32 %a3, i32* @n3, align 4
+  ret void
+}
+
+; Finally, check that we don't do anything with one-element global sets.
+@o5 = internal global i32 0, align 4
+
+; CHECK-LABEL: f5:
+define void @f5(i32 %a1) #0 {
+; CHECK-NEXT: adrp x8, _o5@PAGE
+; CHECK-NEXT: str w0, [x8, _o5@PAGEOFF]
+; CHECK-NEXT: ret
+  store i32 %a1, i32* @o5, align 4
+  ret void
+}
+
+; CHECK-DAG: .zerofill __DATA,__bss,_o5,4,2
+
+; CHECK-DAG: .zerofill __DATA,__bss,[[SET1]],8,3
+; CHECK-DAG: .zerofill __DATA,__bss,[[SET2]],12,3
+; CHECK-DAG: .zerofill __DATA,__bss,[[SET3]],12,3
+
+attributes #0 = { nounwind }
diff --git a/test/CodeGen/AArch64/global-merge-ignore-single-use-minsize.ll b/test/CodeGen/AArch64/global-merge-ignore-single-use-minsize.ll
new file mode 100644
index 00000000000..e83cbab140a
--- /dev/null
+++ b/test/CodeGen/AArch64/global-merge-ignore-single-use-minsize.ll
@@ -0,0 +1,74 @@
+; RUN: llc -mtriple=aarch64-apple-ios -asm-verbose=false -aarch64-collect-loh=false \
+; RUN:   -O1 -global-merge-group-by-use -global-merge-ignore-single-use \
+; RUN:   %s -o - | FileCheck %s
+
+; Check that, at -O1, we only merge globals used in minsize functions.
+; We assume that globals of the same size aren't reordered inside a set.
+; We use -global-merge-ignore-single-use, and thus only expect one merged set.
+
+@m1 = internal global i32 0, align 4
+@n1 = internal global i32 0, align 4
+
+; CHECK-LABEL: f1:
+define void @f1(i32 %a1, i32 %a2) minsize nounwind {
+; CHECK-NEXT: adrp x8, [[SET:__MergedGlobals]]@PAGE
+; CHECK-NEXT: add x8, x8, [[SET]]@PAGEOFF
+; CHECK-NEXT: stp w0, w1, [x8]
+; CHECK-NEXT: ret
+  store i32 %a1, i32* @m1, align 4
+  store i32 %a2, i32* @n1, align 4
+  ret void
+}
+
+@m2 = internal global i32 0, align 4
+@n2 = internal global i32 0, align 4
+
+; CHECK-LABEL: f2:
+define void @f2(i32 %a1, i32 %a2) nounwind {
+; CHECK-NEXT: adrp x8, _m2@PAGE
+; CHECK-NEXT: adrp x9, _n2@PAGE
+; CHECK-NEXT: str w0, [x8, _m2@PAGEOFF]
+; CHECK-NEXT: str w1, [x9, _n2@PAGEOFF]
+; CHECK-NEXT: ret
+  store i32 %a1, i32* @m2, align 4
+  store i32 %a2, i32* @n2, align 4
+  ret void
+}
+
+; If we have use sets partially overlapping between a minsize and a non-minsize
+; function, explicitly check that we only consider the globals used in the
+; minsize function for merging.
+
+@m3 = internal global i32 0, align 4
+@n3 = internal global i32 0, align 4
+
+; CHECK-LABEL: f3:
+define void @f3(i32 %a1, i32 %a2) minsize nounwind {
+; CHECK-NEXT: adrp x8, [[SET]]@PAGE
+; CHECK-NEXT: add x8, x8, [[SET]]@PAGEOFF
+; CHECK-NEXT: stp w0, w1, [x8, #8]
+; CHECK-NEXT: ret
+  store i32 %a1, i32* @m3, align 4
+  store i32 %a2, i32* @n3, align 4
+  ret void
+}
+
+@n4 = internal global i32 0, align 4
+
+; CHECK-LABEL: f4:
+define void @f4(i32 %a1, i32 %a2) nounwind {
+; CHECK-NEXT: adrp x8, [[SET]]@PAGE
+; CHECK-NEXT: add x8, x8, [[SET]]@PAGEOFF
+; CHECK-NEXT: adrp x9, _n4@PAGE
+; CHECK-NEXT: str w0, [x8, #8]
+; CHECK-NEXT: str w1, [x9, _n4@PAGEOFF]
+; CHECK-NEXT: ret
+  store i32 %a1, i32* @m3, align 4
+  store i32 %a2, i32* @n4, align 4
+  ret void
+}
+
+; CHECK-DAG: .zerofill __DATA,__bss,[[SET]],16,3
+; CHECK-DAG: .zerofill __DATA,__bss,_m2,4,2
+; CHECK-DAG: .zerofill __DATA,__bss,_n2,4,2
+; CHECK-DAG: .zerofill __DATA,__bss,_n4,4,2
diff --git a/test/CodeGen/AArch64/global-merge-ignore-single-use.ll b/test/CodeGen/AArch64/global-merge-ignore-single-use.ll
new file mode 100644
index 00000000000..e6de4699132
--- /dev/null
+++ b/test/CodeGen/AArch64/global-merge-ignore-single-use.ll
@@ -0,0 +1,63 @@
+; RUN: llc -mtriple=aarch64-apple-ios -asm-verbose=false -aarch64-collect-loh=false \
+; RUN:   -aarch64-global-merge -global-merge-group-by-use -global-merge-ignore-single-use \
+; RUN:   %s -o - | FileCheck %s
+
+; We assume that globals of the same size aren't reordered inside a set.
+
+@m1 = internal global i32 0, align 4
+@n1 = internal global i32 0, align 4
+@o1 = internal global i32 0, align 4
+
+; CHECK-LABEL: f1:
+define void @f1(i32 %a1, i32 %a2) #0 {
+; CHECK-NEXT: adrp x8, [[SET:__MergedGlobals]]@PAGE
+; CHECK-NEXT: add x8, x8, [[SET]]@PAGEOFF
+; CHECK-NEXT: stp w0, w1, [x8]
+; CHECK-NEXT: ret
+  store i32 %a1, i32* @m1, align 4
+  store i32 %a2, i32* @n1, align 4
+  ret void
+}
+
+@m2 = internal global i32 0, align 4
+@n2 = internal global i32 0, align 4
+
+; CHECK-LABEL: f2:
+define void @f2(i32 %a1, i32 %a2, i32 %a3) #0 {
+; CHECK-NEXT: adrp x8, [[SET]]@PAGE
+; CHECK-NEXT: add x8, x8, [[SET]]@PAGEOFF
+; CHECK-NEXT: stp w0, w1, [x8]
+; CHECK-NEXT: str w2, [x8, #8]
+; CHECK-NEXT: ret
+  store i32 %a1, i32* @m1, align 4
+  store i32 %a2, i32* @n1, align 4
+  store i32 %a3, i32* @o1, align 4
+  ret void
+}
+
+; CHECK-LABEL: f3:
+define void @f3(i32 %a1, i32 %a2) #0 {
+; CHECK-NEXT: adrp x8, [[SET]]@PAGE
+; CHECK-NEXT: add x8, x8, [[SET]]@PAGEOFF
+; CHECK-NEXT: stp w0, w1, [x8, #12]
+; CHECK-NEXT: ret
+  store i32 %a1, i32* @m2, align 4
+  store i32 %a2, i32* @n2, align 4
+  ret void
+}
+
+@o2 = internal global i32 0, align 4
+
+; CHECK-LABEL: f4:
+define void @f4(i32 %a1) #0 {
+; CHECK-NEXT: adrp x8, _o2@PAGE
+; CHECK-NEXT: str w0, [x8, _o2@PAGEOFF]
+; CHECK-NEXT: ret
+  store i32 %a1, i32* @o2, align 4
+  ret void
+}
+
+; CHECK-DAG: .zerofill __DATA,__bss,[[SET]],20,4
+; CHECK-DAG: .zerofill __DATA,__bss,_o2,4,2
+
+attributes #0 = { nounwind }
diff --git a/test/CodeGen/AArch64/ifcvt-select.ll b/test/CodeGen/AArch64/ifcvt-select.ll
new file mode 100644
index 00000000000..4e024d963f2
--- /dev/null
+++ b/test/CodeGen/AArch64/ifcvt-select.ll
@@ -0,0 +1,41 @@
+; RUN: llc -mtriple=arm64-apple-ios -mcpu=cyclone < %s | FileCheck %s
+; Do not generate redundant select in early if-converstion pass. 
+
+define i32 @foo(i32 %a, i32 %b)  {
+entry:
+;CHECK-LABEL: foo:
+;CHECK: csinc
+;CHECK-NOT: csel
+  %sub = sub nsw i32 %b, %a
+  %cmp10 = icmp sgt i32 %a, 0
+  br i1 %cmp10, label %while.body.lr.ph, label %while.end
+
+while.body.lr.ph:
+  br label %while.body
+
+while.body:                                  
+  %j.012 = phi i32 [ %sub, %while.body.lr.ph ], [ %inc, %if.then ], [ %inc, %if.else ]
+  %i.011 = phi i32 [ %a, %while.body.lr.ph ], [ %inc2, %if.then ], [ %dec, %if.else ]
+  %cmp1 = icmp slt i32 %i.011, %j.012
+  br i1 %cmp1, label %while.end, label %while.cond
+
+while.cond:
+  %inc = add nsw i32 %j.012, 5
+  %cmp2 = icmp slt i32 %inc, %b
+  br i1 %cmp2, label %if.then, label %if.else
+
+if.then:
+  %inc2 = add nsw i32 %i.011, 1
+  br label %while.body
+
+if.else:
+  %dec = add nsw i32 %i.011, -1
+  br label %while.body
+
+while.end:
+  %j.0.lcssa = phi i32 [ %j.012, %while.body ], [ %sub, %entry ]
+  %i.0.lcssa = phi i32 [ %i.011, %while.body ], [ %a, %entry ]
+  %add = add nsw i32 %j.0.lcssa, %i.0.lcssa
+  ret i32 %add
+}
+
diff --git a/test/CodeGen/AArch64/lit.local.cfg b/test/CodeGen/AArch64/lit.local.cfg
index 125995cebf1..f4626c7e0a4 100644
--- a/test/CodeGen/AArch64/lit.local.cfg
+++ b/test/CodeGen/AArch64/lit.local.cfg
@@ -6,5 +6,5 @@ if not 'AArch64' in config.root.targets:
     config.unsupported = True
 
 # For now we don't test arm64-win32.
-if re.search(r'cygwin|mingw32|win32', config.target_triple):
+if re.search(r'cygwin|mingw32|win32|windows-gnu|windows-msvc', config.target_triple):
     config.unsupported = True
diff --git a/test/CodeGen/AArch64/machine-sink-kill-flags.ll b/test/CodeGen/AArch64/machine-sink-kill-flags.ll
new file mode 100644
index 00000000000..590e1692ef8
--- /dev/null
+++ b/test/CodeGen/AArch64/machine-sink-kill-flags.ll
@@ -0,0 +1,29 @@
+; RUN: llc %s -o - -fast-isel=true -O1 -verify-machineinstrs | FileCheck %s
+
+target datalayout = "e-m:o-i64:64-i128:128-n32:64-S128"
+target triple = "arm64-apple-ios8.0.0"
+
+; The machine verifier was asserting on this test because the AND instruction was
+; sunk below the test which killed %tmp340.
+; The kill flags on the test had to be cleared because the AND was going to read
+; registers in a BB after the test instruction.
+
+; CHECK: %bb343
+; CHECK: and
+
+define i32 @test(i32* %ptr) {
+bb:
+  br label %.thread
+
+.thread:                                          ; preds = %.thread, %bb
+  %loc = phi i32 [ %next_iter, %.thread ], [ 0, %bb ]
+  %next_iter = lshr i32 %loc, 1
+  %tmp340 = sub i32 %loc, 1
+  %tmp341 = and i32 %tmp340, 1
+  %tmp342 = icmp eq i32 %tmp341, 0
+  br i1 %tmp342, label %bb343, label %.thread
+
+bb343:                                            ; preds = %.thread
+  store i32 %tmp341, i32* %ptr, align 4
+  ret i32 -1
+}
diff --git a/test/CodeGen/AArch64/minmax.ll b/test/CodeGen/AArch64/minmax.ll
new file mode 100644
index 00000000000..df4912ca1f7
--- /dev/null
+++ b/test/CodeGen/AArch64/minmax.ll
@@ -0,0 +1,107 @@
+; RUN: llc < %s -mtriple=aarch64-linux-gnu | FileCheck %s
+
+; CHECK-LABEL: t1
+; CHECK: smax
+define <4 x i32> @t1(<4 x i32> %a, <4 x i32> %b) {
+  %t1 = icmp sgt <4 x i32> %a, %b
+  %t2 = select <4 x i1> %t1, <4 x i32> %a, <4 x i32> %b
+  ret <4 x i32> %t2
+}
+
+; CHECK-LABEL: t2
+; CHECK: smin
+define <4 x i32> @t2(<4 x i32> %a, <4 x i32> %b) {
+  %t1 = icmp slt <4 x i32> %a, %b
+  %t2 = select <4 x i1> %t1, <4 x i32> %a, <4 x i32> %b
+  ret <4 x i32> %t2
+}
+
+; CHECK-LABEL: t3
+; CHECK: umax
+define <4 x i32> @t3(<4 x i32> %a, <4 x i32> %b) {
+  %t1 = icmp ugt <4 x i32> %a, %b
+  %t2 = select <4 x i1> %t1, <4 x i32> %a, <4 x i32> %b
+  ret <4 x i32> %t2
+}
+
+; CHECK-LABEL: t4
+; CHECK: umin
+define <8 x i8> @t4(<8 x i8> %a, <8 x i8> %b) {
+  %t1 = icmp ult <8 x i8> %a, %b
+  %t2 = select <8 x i1> %t1, <8 x i8> %a, <8 x i8> %b
+  ret <8 x i8> %t2
+}
+
+; CHECK-LABEL: t5
+; CHECK: smin
+define <4 x i16> @t5(<4 x i16> %a, <4 x i16> %b) {
+  %t1 = icmp sgt <4 x i16> %b, %a
+  %t2 = select <4 x i1> %t1, <4 x i16> %a, <4 x i16> %b
+  ret <4 x i16> %t2
+}
+
+; CHECK-LABEL: t6
+; CHECK: smax
+define <2 x i32> @t6(<2 x i32> %a, <2 x i32> %b) {
+  %t1 = icmp slt <2 x i32> %b, %a
+  %t2 = select <2 x i1> %t1, <2 x i32> %a, <2 x i32> %b
+  ret <2 x i32> %t2
+}
+
+; CHECK-LABEL: t7
+; CHECK: umin
+define <16 x i8> @t7(<16 x i8> %a, <16 x i8> %b) {
+  %t1 = icmp ugt <16 x i8> %b, %a
+  %t2 = select <16 x i1> %t1, <16 x i8> %a, <16 x i8> %b
+  ret <16 x i8> %t2
+}
+
+; CHECK-LABEL: t8
+; CHECK: umax
+define <8 x i16> @t8(<8 x i16> %a, <8 x i16> %b) {
+  %t1 = icmp ult <8 x i16> %b, %a
+  %t2 = select <8 x i1> %t1, <8 x i16> %a, <8 x i16> %b
+  ret <8 x i16> %t2
+}
+
+; CHECK-LABEL: t9
+; CHECK: umin
+; CHECK: smax
+define <4 x i32> @t9(<4 x i32> %a, <4 x i32> %b, <4 x i32> %c) {
+  %t1 = icmp ugt <4 x i32> %b, %a
+  %t2 = select <4 x i1> %t1, <4 x i32> %a, <4 x i32> %b
+  %t3 = icmp sge <4 x i32> %t2, %c
+  %t4 = select <4 x i1> %t3, <4 x i32> %t2, <4 x i32> %c
+  ret <4 x i32> %t4
+}
+
+; CHECK-LABEL: t10
+; CHECK: smax
+; CHECK: smax
+define <8 x i32> @t10(<8 x i32> %a, <8 x i32> %b) {
+  %t1 = icmp sgt <8 x i32> %a, %b
+  %t2 = select <8 x i1> %t1, <8 x i32> %a, <8 x i32> %b
+  ret <8 x i32> %t2
+}
+
+; CHECK-LABEL: t11
+; CHECK: smin
+; CHECK: smin
+; CHECK: smin
+; CHECK: smin
+define <16 x i32> @t11(<16 x i32> %a, <16 x i32> %b) {
+  %t1 = icmp sle <16 x i32> %a, %b
+  %t2 = select <16 x i1> %t1, <16 x i32> %a, <16 x i32> %b
+  ret <16 x i32> %t2
+}
+
+; CHECK-LABEL: t12
+; CHECK-NOT: umin
+; The icmp is used by two instructions, so don't produce a umin node.
+define <16 x i8> @t12(<16 x i8> %a, <16 x i8> %b) {
+  %t1 = icmp ugt <16 x i8> %b, %a
+  %t2 = select <16 x i1> %t1, <16 x i8> %a, <16 x i8> %b
+  %t3 = zext <16 x i1> %t1 to <16 x i8>
+  %t4 = add <16 x i8> %t3, %t2
+  ret <16 x i8> %t4
+}
diff --git a/test/CodeGen/AArch64/misched-fusion.ll b/test/CodeGen/AArch64/misched-fusion.ll
new file mode 100644
index 00000000000..d3886932903
--- /dev/null
+++ b/test/CodeGen/AArch64/misched-fusion.ll
@@ -0,0 +1,34 @@
+; RUN: llc -o - %s -mcpu=cyclone | FileCheck %s
+target triple = "arm64-apple-ios"
+
+declare void @foobar(i32 %v0, i32 %v1)
+
+; Make sure sub is scheduled in front of cbnz
+; CHECK-LABEL: test_sub_cbz:
+; CHECK: add w[[ADDRES:[0-9]+]], w1, #7
+; CHECK: sub w[[SUBRES:[0-9]+]], w0, #13
+; CHECK-NEXT: cbnz w[[SUBRES]], [[SKIPBLOCK:LBB[0-9_]+]]
+; CHECK: mov x0, x[[ADDRES]]
+; CHECK: mov x1, x[[SUBRES]]
+; CHECK: bl _foobar
+; CHECK: [[SKIPBLOCK]]:
+; CHECK: mov x0, x[[SUBRES]]
+; CHECK: mov x1, x[[ADDRES]]
+; CHECK: bl _foobar
+define void @test_sub_cbz(i32 %a0, i32 %a1) {
+entry:
+  ; except for the fusion opportunity the sub/add should be equal so the
+  ; scheduler would leave them in source order if it weren't for the scheduling
+  %v0 = sub i32 %a0, 13
+  %cond = icmp eq i32 %v0, 0
+  %v1 = add i32 %a1, 7
+  br i1 %cond, label %if, label %exit
+
+if:
+  call void @foobar(i32 %v1, i32 %v0)
+  br label %exit
+
+exit:
+  call void @foobar(i32 %v0, i32 %v1)
+  ret void
+}
diff --git a/test/CodeGen/AArch64/nest-register.ll b/test/CodeGen/AArch64/nest-register.ll
new file mode 100644
index 00000000000..9c659fb74ec
--- /dev/null
+++ b/test/CodeGen/AArch64/nest-register.ll
@@ -0,0 +1,23 @@
+; RUN: llc -verify-machineinstrs < %s -mtriple=aarch64-none-linux-gnu | FileCheck %s
+
+; Tests that the 'nest' parameter attribute causes the relevant parameter to be
+; passed in the right register.
+
+define i8* @nest_receiver(i8* nest %arg) nounwind {
+; CHECK-LABEL: nest_receiver:
+; CHECK-NEXT: // BB#0:
+; CHECK-NEXT: mov x0, x18
+; CHECK-NEXT: ret
+
+  ret i8* %arg
+}
+
+define i8* @nest_caller(i8* %arg) nounwind {
+; CHECK-LABEL: nest_caller:
+; CHECK: mov x18, x0
+; CHECK-NEXT: bl nest_receiver
+; CHECK: ret
+
+  %result = call i8* @nest_receiver(i8* nest %arg)
+  ret i8* %result
+}
diff --git a/test/CodeGen/AArch64/pic-eh-stubs.ll b/test/CodeGen/AArch64/pic-eh-stubs.ll
index f761a87783c..143558f7b2c 100644
--- a/test/CodeGen/AArch64/pic-eh-stubs.ll
+++ b/test/CodeGen/AArch64/pic-eh-stubs.ll
@@ -21,13 +21,13 @@
 
 @_ZTIi = external constant i8*
 
-define i32 @_Z3barv() {
+define i32 @_Z3barv() personality i8* bitcast (i32 (...)* @__gxx_personality_v0 to i8*) {
 entry:
   invoke void @_Z3foov()
           to label %return unwind label %lpad
 
 lpad:                                             ; preds = %entry
-  %0 = landingpad { i8*, i32 } personality i8* bitcast (i32 (...)* @__gxx_personality_v0 to i8*)
+  %0 = landingpad { i8*, i32 }
           catch i8* bitcast (i8** @_ZTIi to i8*)
   %1 = extractvalue { i8*, i32 } %0, 1
   %2 = tail call i32 @llvm.eh.typeid.for(i8* bitcast (i8** @_ZTIi to i8*)) nounwind
diff --git a/test/CodeGen/AArch64/round-conv.ll b/test/CodeGen/AArch64/round-conv.ll
new file mode 100644
index 00000000000..1dcdb3f4935
--- /dev/null
+++ b/test/CodeGen/AArch64/round-conv.ll
@@ -0,0 +1,461 @@
+; RUN: llc < %s -mtriple=arm64-apple-ios7.0 | FileCheck %s --check-prefix=CHECK-INEXACT
+; RUN: llc < %s -mtriple=arm64-apple-ios7.0 -enable-unsafe-fp-math | FileCheck %s --check-prefix=CHECK-FAST
+; RUN: llc < %s -mtriple=aarch64-linux-gnu | FileCheck %s --check-prefix=CHECK-FAST
+
+; CHECK-INEXACT-LABEL: testmsws:
+; CHECK-INEXACT: fcvtms w0, s0
+; CHECK-INEXACT: frintx {{s[0-9]+}}, s0
+
+; CHECK-FAST-LABEL: testmsws:
+; CHECK-FAST: fcvtms w0, s0
+; CHECK-FAST-NOT: frintx {{s[0-9]+}}, s0
+define i32 @testmsws(float %a) {
+entry:
+  %call = call float @floorf(float %a) nounwind readnone
+  %conv = fptosi float %call to i32
+  ret i32 %conv
+}
+
+; CHECK-INEXACT-LABEL: testmsxs:
+; CHECK-INEXACT: fcvtms x0, s0
+; CHECK-INEXACT: frintx {{s[0-9]+}}, s0
+
+; CHECK-FAST-LABEL: testmsxs:
+; CHECK-FAST: fcvtms x0, s0
+; CHECK-FAST-NOT: frintx {{s[0-9]+}}, s0
+define i64 @testmsxs(float %a) {
+entry:
+  %call = call float @floorf(float %a) nounwind readnone
+  %conv = fptosi float %call to i64
+  ret i64 %conv
+}
+
+; CHECK-INEXACT-LABEL: testmswd:
+; CHECK-INEXACT: fcvtms w0, d0
+; CHECK-INEXACT: frintx {{d[0-9]+}}, d0
+
+; CHECK-FAST-LABEL: testmswd:
+; CHECK-FAST: fcvtms w0, d0
+; CHECK-FAST-NOT: frintx {{d[0-9]+}}, d0
+define i32 @testmswd(double %a) {
+entry:
+  %call = call double @floor(double %a) nounwind readnone
+  %conv = fptosi double %call to i32
+  ret i32 %conv
+}
+
+; CHECK-INEXACT-LABEL: testmsxd:
+; CHECK-INEXACT: fcvtms x0, d0
+; CHECK-INEXACT: frintx {{d[0-9]+}}, d0
+
+; CHECK-FAST-LABEL: testmsxd:
+; CHECK-FAST: fcvtms x0, d0
+; CHECK-FAST-NOT: frintx {{d[0-9]+}}, d0
+define i64 @testmsxd(double %a) {
+entry:
+  %call = call double @floor(double %a) nounwind readnone
+  %conv = fptosi double %call to i64
+  ret i64 %conv
+}
+
+; CHECK-INEXACT-LABEL: testmuws:
+; CHECK-INEXACT: fcvtmu w0, s0
+; CHECK-INEXACT: frintx {{s[0-9]+}}, s0
+
+; CHECK-FAST-LABEL: testmuws:
+; CHECK-FAST: fcvtmu w0, s0
+; CHECK-FAST-NOT: frintx {{s[0-9]+}}, s0
+define i32 @testmuws(float %a) {
+entry:
+  %call = call float @floorf(float %a) nounwind readnone
+  %conv = fptoui float %call to i32
+  ret i32 %conv
+}
+
+; CHECK-INEXACT-LABEL: testmuxs:
+; CHECK-INEXACT: fcvtmu x0, s0
+; CHECK-INEXACT: frintx {{s[0-9]+}}, s0
+
+; CHECK-FAST-LABEL: testmuxs:
+; CHECK-FAST: fcvtmu x0, s0
+; CHECK-FAST-NOT: frintx {{s[0-9]+}}, s0
+define i64 @testmuxs(float %a) {
+entry:
+  %call = call float @floorf(float %a) nounwind readnone
+  %conv = fptoui float %call to i64
+  ret i64 %conv
+}
+
+; CHECK-INEXACT-LABEL: testmuwd:
+; CHECK-INEXACT: fcvtmu w0, d0
+; CHECK-INEXACT: frintx {{d[0-9]+}}, d0
+
+; CHECK-FAST-LABEL: testmuwd:
+; CHECK-FAST: fcvtmu w0, d0
+; CHECK-FAST-NOT: frintx {{d[0-9]+}}, d0
+define i32 @testmuwd(double %a) {
+entry:
+  %call = call double @floor(double %a) nounwind readnone
+  %conv = fptoui double %call to i32
+  ret i32 %conv
+}
+
+; CHECK-INEXACT-LABEL: testmuxd:
+; CHECK-INEXACT: fcvtmu x0, d0
+; CHECK-INEXACT: frintx {{d[0-9]+}}, d0
+
+; CHECK-FAST-LABEL: testmuxd:
+; CHECK-FAST: fcvtmu x0, d0
+; CHECK-FAST-NOT: frintx {{d[0-9]+}}, d0
+define i64 @testmuxd(double %a) {
+entry:
+  %call = call double @floor(double %a) nounwind readnone
+  %conv = fptoui double %call to i64
+  ret i64 %conv
+}
+
+; CHECK-INEXACT-LABEL: testpsws:
+; CHECK-INEXACT: fcvtps w0, s0
+; CHECK-INEXACT: frintx {{s[0-9]+}}, s0
+
+; CHECK-FAST-LABEL: testpsws:
+; CHECK-FAST: fcvtps w0, s0
+; CHECK-FAST-NOT: frintx {{s[0-9]+}}, s0
+define i32 @testpsws(float %a) {
+entry:
+  %call = call float @ceilf(float %a) nounwind readnone
+  %conv = fptosi float %call to i32
+  ret i32 %conv
+}
+
+; CHECK-INEXACT-LABEL: testpsxs:
+; CHECK-INEXACT: fcvtps x0, s0
+; CHECK-INEXACT: frintx {{s[0-9]+}}, s0
+
+; CHECK-FAST-LABEL: testpsxs:
+; CHECK-FAST: fcvtps x0, s0
+; CHECK-FAST-NOT: frintx {{s[0-9]+}}, s0
+define i64 @testpsxs(float %a) {
+entry:
+  %call = call float @ceilf(float %a) nounwind readnone
+  %conv = fptosi float %call to i64
+  ret i64 %conv
+}
+
+; CHECK-INEXACT-LABEL: testpswd:
+; CHECK-INEXACT: fcvtps w0, d0
+; CHECK-INEXACT: frintx {{d[0-9]+}}, d0
+
+; CHECK-FAST-LABEL: testpswd:
+; CHECK-FAST: fcvtps w0, d0
+; CHECK-FAST-NOT: frintx {{d[0-9]+}}, d0
+define i32 @testpswd(double %a) {
+entry:
+  %call = call double @ceil(double %a) nounwind readnone
+  %conv = fptosi double %call to i32
+  ret i32 %conv
+}
+
+; CHECK-INEXACT-LABEL: testpsxd:
+; CHECK-INEXACT: fcvtps x0, d0
+; CHECK-INEXACT: frintx {{d[0-9]+}}, d0
+
+; CHECK-FAST-LABEL: testpsxd:
+; CHECK-FAST: fcvtps x0, d0
+; CHECK-FAST-NOT: frintx {{d[0-9]+}}, d0
+define i64 @testpsxd(double %a) {
+entry:
+  %call = call double @ceil(double %a) nounwind readnone
+  %conv = fptosi double %call to i64
+  ret i64 %conv
+}
+
+; CHECK-INEXACT-LABEL: testpuws:
+; CHECK-INEXACT: fcvtpu w0, s0
+; CHECK-INEXACT: frintx {{s[0-9]+}}, s0
+
+; CHECK-FAST-LABEL: testpuws:
+; CHECK-FAST: fcvtpu w0, s0
+; CHECK-FAST-NOT: frintx {{s[0-9]+}}, s0
+define i32 @testpuws(float %a) {
+entry:
+  %call = call float @ceilf(float %a) nounwind readnone
+  %conv = fptoui float %call to i32
+  ret i32 %conv
+}
+
+; CHECK-INEXACT-LABEL: testpuxs:
+; CHECK-INEXACT: fcvtpu x0, s0
+; CHECK-INEXACT: frintx {{s[0-9]+}}, s0
+
+; CHECK-FAST-LABEL: testpuxs:
+; CHECK-FAST: fcvtpu x0, s0
+; CHECK-FAST-NOT: frintx {{s[0-9]+}}, s0
+define i64 @testpuxs(float %a) {
+entry:
+  %call = call float @ceilf(float %a) nounwind readnone
+  %conv = fptoui float %call to i64
+  ret i64 %conv
+}
+
+; CHECK-INEXACT-LABEL: testpuwd:
+; CHECK-INEXACT: fcvtpu w0, d0
+; CHECK-INEXACT: frintx {{d[0-9]+}}, d0
+
+; CHECK-FAST-LABEL: testpuwd:
+; CHECK-FAST: fcvtpu w0, d0
+; CHECK-FAST-NOT: frintx {{d[0-9]+}}, d0
+define i32 @testpuwd(double %a) {
+entry:
+  %call = call double @ceil(double %a) nounwind readnone
+  %conv = fptoui double %call to i32
+  ret i32 %conv
+}
+
+; CHECK-INEXACT-LABEL: testpuxd:
+; CHECK-INEXACT: fcvtpu x0, d0
+; CHECK-INEXACT: frintx {{d[0-9]+}}, d0
+
+; CHECK-FAST-LABEL: testpuxd:
+; CHECK-FAST: fcvtpu x0, d0
+; CHECK-FAST-NOT: frintx {{d[0-9]+}}, d0
+define i64 @testpuxd(double %a) {
+entry:
+  %call = call double @ceil(double %a) nounwind readnone
+  %conv = fptoui double %call to i64
+  ret i64 %conv
+}
+
+; CHECK-INEXACT-LABEL: testzsws:
+; CHECK-INEXACT: fcvtzs w0, s0
+; CHECK-INEXACT: frintx {{s[0-9]+}}, s0
+
+; CHECK-FAST-LABEL: testzsws:
+; CHECK-FAST: fcvtzs w0, s0
+; CHECK-FAST-NOT: frintx {{s[0-9]+}}, s0
+define i32 @testzsws(float %a) {
+entry:
+  %call = call float @truncf(float %a) nounwind readnone
+  %conv = fptosi float %call to i32
+  ret i32 %conv
+}
+
+; CHECK-INEXACT-LABEL: testzsxs:
+; CHECK-INEXACT: fcvtzs x0, s0
+; CHECK-INEXACT: frintx {{s[0-9]+}}, s0
+
+; CHECK-FAST-LABEL: testzsxs:
+; CHECK-FAST: fcvtzs x0, s0
+; CHECK-FAST-NOT: frintx {{s[0-9]+}}, s0
+define i64 @testzsxs(float %a) {
+entry:
+  %call = call float @truncf(float %a) nounwind readnone
+  %conv = fptosi float %call to i64
+  ret i64 %conv
+}
+
+; CHECK-INEXACT-LABEL: testzswd:
+; CHECK-INEXACT: fcvtzs w0, d0
+; CHECK-INEXACT: frintx {{d[0-9]+}}, d0
+
+; CHECK-FAST-LABEL: testzswd:
+; CHECK-FAST: fcvtzs w0, d0
+; CHECK-FAST-NOT: frintx {{d[0-9]+}}, d0
+define i32 @testzswd(double %a) {
+entry:
+  %call = call double @trunc(double %a) nounwind readnone
+  %conv = fptosi double %call to i32
+  ret i32 %conv
+}
+
+; CHECK-INEXACT-LABEL: testzsxd:
+; CHECK-INEXACT: fcvtzs x0, d0
+; CHECK-INEXACT: frintx {{d[0-9]+}}, d0
+
+; CHECK-FAST-LABEL: testzsxd:
+; CHECK-FAST: fcvtzs x0, d0
+; CHECK-FAST-NOT: frintx {{d[0-9]+}}, d0
+define i64 @testzsxd(double %a) {
+entry:
+  %call = call double @trunc(double %a) nounwind readnone
+  %conv = fptosi double %call to i64
+  ret i64 %conv
+}
+
+; CHECK-INEXACT-LABEL: testzuws:
+; CHECK-INEXACT: fcvtzu w0, s0
+; CHECK-INEXACT: frintx {{s[0-9]+}}, s0
+
+; CHECK-FAST-LABEL: testzuws:
+; CHECK-FAST: fcvtzu w0, s0
+; CHECK-FAST-NOT: frintx {{s[0-9]+}}, s0
+define i32 @testzuws(float %a) {
+entry:
+  %call = call float @truncf(float %a) nounwind readnone
+  %conv = fptoui float %call to i32
+  ret i32 %conv
+}
+
+; CHECK-INEXACT-LABEL: testzuxs:
+; CHECK-INEXACT: fcvtzu x0, s0
+; CHECK-INEXACT: frintx {{s[0-9]+}}, s0
+
+; CHECK-FAST-LABEL: testzuxs:
+; CHECK-FAST: fcvtzu x0, s0
+; CHECK-FAST-NOT: frintx {{s[0-9]+}}, s0
+define i64 @testzuxs(float %a) {
+entry:
+  %call = call float @truncf(float %a) nounwind readnone
+  %conv = fptoui float %call to i64
+  ret i64 %conv
+}
+
+; CHECK-INEXACT-LABEL: testzuwd:
+; CHECK-INEXACT: fcvtzu w0, d0
+; CHECK-INEXACT: frintx {{d[0-9]+}}, d0
+
+; CHECK-FAST-LABEL: testzuwd:
+; CHECK-FAST: fcvtzu w0, d0
+; CHECK-FAST-NOT: frintx {{d[0-9]+}}, d0
+define i32 @testzuwd(double %a) {
+entry:
+  %call = call double @trunc(double %a) nounwind readnone
+  %conv = fptoui double %call to i32
+  ret i32 %conv
+}
+
+; CHECK-INEXACT-LABEL: testzuxd:
+; CHECK-INEXACT: fcvtzu x0, d0
+; CHECK-INEXACT: frintx {{d[0-9]+}}, d0
+
+; CHECK-FAST-LABEL: testzuxd:
+; CHECK-FAST: fcvtzu x0, d0
+; CHECK-FAST-NOT: frintx {{d[0-9]+}}, d0
+define i64 @testzuxd(double %a) {
+entry:
+  %call = call double @trunc(double %a) nounwind readnone
+  %conv = fptoui double %call to i64
+  ret i64 %conv
+}
+
+; CHECK-INEXACT-LABEL: testasws:
+; CHECK-INEXACT: fcvtas w0, s0
+; CHECK-INEXACT: frintx {{s[0-9]+}}, s0
+
+; CHECK-FAST-LABEL: testasws:
+; CHECK-FAST: fcvtas w0, s0
+; CHECK-FAST-NOT: frintx {{s[0-9]+}}, s0
+define i32 @testasws(float %a) {
+entry:
+  %call = call float @roundf(float %a) nounwind readnone
+  %conv = fptosi float %call to i32
+  ret i32 %conv
+}
+
+; CHECK-INEXACT-LABEL: testasxs:
+; CHECK-INEXACT: fcvtas x0, s0
+; CHECK-INEXACT: frintx {{s[0-9]+}}, s0
+
+; CHECK-FAST-LABEL: testasxs:
+; CHECK-FAST: fcvtas x0, s0
+; CHECK-FAST-NOT: frintx {{s[0-9]+}}, s0
+define i64 @testasxs(float %a) {
+entry:
+  %call = call float @roundf(float %a) nounwind readnone
+  %conv = fptosi float %call to i64
+  ret i64 %conv
+}
+
+; CHECK-INEXACT-LABEL: testaswd:
+; CHECK-INEXACT: fcvtas w0, d0
+; CHECK-INEXACT: frintx {{d[0-9]+}}, d0
+
+; CHECK-FAST-LABEL: testaswd:
+; CHECK-FAST: fcvtas w0, d0
+; CHECK-FAST-NOT: frintx {{d[0-9]+}}, d0
+define i32 @testaswd(double %a) {
+entry:
+  %call = call double @round(double %a) nounwind readnone
+  %conv = fptosi double %call to i32
+  ret i32 %conv
+}
+
+; CHECK-INEXACT-LABEL: testasxd:
+; CHECK-INEXACT: fcvtas x0, d0
+; CHECK-INEXACT: frintx {{d[0-9]+}}, d0
+
+; CHECK-FAST-LABEL: testasxd:
+; CHECK-FAST: fcvtas x0, d0
+; CHECK-FAST-NOT: frintx {{d[0-9]+}}, d0
+define i64 @testasxd(double %a) {
+entry:
+  %call = call double @round(double %a) nounwind readnone
+  %conv = fptosi double %call to i64
+  ret i64 %conv
+}
+
+; CHECK-INEXACT-LABEL: testauws:
+; CHECK-INEXACT: fcvtau w0, s0
+; CHECK-INEXACT: frintx {{s[0-9]+}}, s0
+
+; CHECK-FAST-LABEL: testauws:
+; CHECK-FAST: fcvtau w0, s0
+; CHECK-FAST-NOT: frintx {{s[0-9]+}}, s0
+define i32 @testauws(float %a) {
+entry:
+  %call = call float @roundf(float %a) nounwind readnone
+  %conv = fptoui float %call to i32
+  ret i32 %conv
+}
+
+; CHECK-INEXACT-LABEL: testauxs:
+; CHECK-INEXACT: fcvtau x0, s0
+; CHECK-INEXACT: frintx {{s[0-9]+}}, s0
+
+; CHECK-FAST-LABEL: testauxs:
+; CHECK-FAST: fcvtau x0, s0
+; CHECK-FAST-NOT: frintx {{s[0-9]+}}, s0
+define i64 @testauxs(float %a) {
+entry:
+  %call = call float @roundf(float %a) nounwind readnone
+  %conv = fptoui float %call to i64
+  ret i64 %conv
+}
+
+; CHECK-INEXACT-LABEL: testauwd:
+; CHECK-INEXACT: fcvtau w0, d0
+; CHECK-INEXACT: frintx {{d[0-9]+}}, d0
+
+; CHECK-FAST-LABEL: testauwd:
+; CHECK-FAST: fcvtau w0, d0
+; CHECK-FAST-NOT: frintx {{d[0-9]+}}, d0
+define i32 @testauwd(double %a) {
+entry:
+  %call = call double @round(double %a) nounwind readnone
+  %conv = fptoui double %call to i32
+  ret i32 %conv
+}
+
+; CHECK-INEXACT-LABEL: testauxd:
+; CHECK-INEXACT: fcvtau x0, d0
+; CHECK-INEXACT: frintx {{d[0-9]+}}, d0
+
+; CHECK-FAST-LABEL: testauxd:
+; CHECK-FAST: fcvtau x0, d0
+; CHECK-FAST-NOT: frintx {{d[0-9]+}}, d0
+define i64 @testauxd(double %a) {
+entry:
+  %call = call double @round(double %a) nounwind readnone
+  %conv = fptoui double %call to i64
+  ret i64 %conv
+}
+
+
+declare float @floorf(float) nounwind readnone
+declare double @floor(double) nounwind readnone
+declare float @ceilf(float) nounwind readnone
+declare double @ceil(double) nounwind readnone
+declare float @truncf(float) nounwind readnone
+declare double @trunc(double) nounwind readnone
+declare float @roundf(float) nounwind readnone
+declare double @round(double) nounwind readnone
diff --git a/test/CodeGen/AArch64/simple-macho.ll b/test/CodeGen/AArch64/simple-macho.ll
new file mode 100644
index 00000000000..e9dd98e230d
--- /dev/null
+++ b/test/CodeGen/AArch64/simple-macho.ll
@@ -0,0 +1,12 @@
+; RUN: llc -mtriple=arm64-macho -o - %s | FileCheck %s
+; RUN: llc -mtriple=arm64-macho -filetype=obj -o %t %s
+; RUN: llvm-objdump -triple=arm64-macho -d %t | FileCheck --check-prefix=CHECK-OBJ %s
+
+define void @foo() {
+; CHECK-LABEL: _foo:
+; CHECK: ret
+
+; CHECK-OBJ: 0: c0 03 5f d6 ret
+
+  ret void
+}
\ No newline at end of file
diff --git a/test/CodeGen/AArch64/special-reg.ll b/test/CodeGen/AArch64/special-reg.ll
new file mode 100644
index 00000000000..91c32158d42
--- /dev/null
+++ b/test/CodeGen/AArch64/special-reg.ll
@@ -0,0 +1,48 @@
+; RUN: llc < %s -mtriple=aarch64-none-eabi -mcpu=cortex-a57 2>&1 | FileCheck %s
+
+define i64 @read_encoded_register() nounwind {
+entry:
+; CHECK-LABEL: read_encoded_register:
+; CHECK: mrs x0, S1_2_C3_C4_5
+  %reg = call i64 @llvm.read_register.i64(metadata !0)
+  ret i64 %reg
+}
+
+define i64 @read_daif() nounwind {
+entry:
+; CHECK-LABEL: read_daif:
+; CHECK: mrs x0, DAIF
+  %reg = call i64 @llvm.read_register.i64(metadata !1)
+  ret i64 %reg
+}
+
+define void @write_encoded_register(i64 %x) nounwind {
+entry:
+; CHECK-LABEL: write_encoded_register:
+; CHECK: msr S1_2_C3_C4_5, x0
+  call void @llvm.write_register.i64(metadata !0, i64 %x)
+  ret void
+}
+
+define void @write_daif(i64 %x) nounwind {
+entry:
+; CHECK-LABEL: write_daif:
+; CHECK: msr DAIF, x0
+  call void @llvm.write_register.i64(metadata !1, i64 %x)
+  ret void
+}
+
+define void @write_daifset() nounwind {
+entry:
+; CHECK-LABEL: write_daifset:
+; CHECK: msr DAIFSET, #2
+  call void @llvm.write_register.i64(metadata !2, i64 2)
+  ret void
+}
+
+declare i64 @llvm.read_register.i64(metadata) nounwind
+declare void @llvm.write_register.i64(metadata, i64) nounwind
+
+!0 = !{!"1:2:3:4:5"}
+!1 = !{!"daif"}
+!2 = !{!"daifset"}
diff --git a/test/CodeGen/AArch64/tail-call.ll b/test/CodeGen/AArch64/tail-call.ll
index 7fb39545a32..e5766154bb4 100644
--- a/test/CodeGen/AArch64/tail-call.ll
+++ b/test/CodeGen/AArch64/tail-call.ll
@@ -103,3 +103,41 @@ define fastcc void @caller_weak() {
   tail call void @callee_weak()
   ret void
 }
+
+declare { [2 x float] } @get_vec2()
+
+define { [3 x float] } @test_add_elem() {
+; CHECK-LABEL: test_add_elem:
+; CHECK: bl get_vec2
+; CHECK: fmov s2, #1.0
+; CHECK: ret
+
+  %call = tail call { [2 x float] } @get_vec2()
+  %arr = extractvalue { [2 x float] } %call, 0
+  %arr.0 = extractvalue [2 x float] %arr, 0
+  %arr.1 = extractvalue [2 x float] %arr, 1
+
+  %res.0 = insertvalue { [3 x float] } undef, float %arr.0, 0, 0
+  %res.01 = insertvalue { [3 x float] } %res.0, float %arr.1, 0, 1
+  %res.012 = insertvalue { [3 x float] } %res.01, float 1.000000e+00, 0, 2
+  ret { [3 x float] } %res.012
+}
+
+declare double @get_double()
+define { double, [2 x double] } @test_mismatched_insert() {
+; CHECK-LABEL: test_mismatched_insert:
+; CHECK: bl get_double
+; CHECK: bl get_double
+; CHECK: bl get_double
+; CHECK: ret
+
+  %val0 = call double @get_double()
+  %val1 = call double @get_double()
+  %val2 = tail call double @get_double()
+
+  %res.0 = insertvalue { double, [2 x double] } undef, double %val0, 0
+  %res.01 = insertvalue { double, [2 x double] } %res.0, double %val1, 1, 0
+  %res.012 = insertvalue { double, [2 x double] } %res.01, double %val2, 1, 1
+
+  ret { double, [2 x double] } %res.012
+}
diff --git a/test/CodeGen/AArch64/tailcall_misched_graph.ll b/test/CodeGen/AArch64/tailcall_misched_graph.ll
new file mode 100644
index 00000000000..343ffab57e3
--- /dev/null
+++ b/test/CodeGen/AArch64/tailcall_misched_graph.ll
@@ -0,0 +1,42 @@
+; RUN: llc -mcpu=cyclone -debug-only=misched < %s 2>&1 | FileCheck %s
+
+; REQUIRES: asserts
+
+target datalayout = "e-m:o-i64:64-i128:128-n32:64-S128"
+target triple = "arm64-apple-ios7.0.0"
+
+define void @caller2(i8* %a0, i8* %a1, i8* %a2, i8* %a3, i8* %a4, i8* %a5, i8* %a6, i8* %a7, i8* %a8, i8* %a9) {
+entry:
+  tail call void @callee2(i8* %a1, i8* %a2, i8* %a3, i8* %a4, i8* %a5, i8* %a6, i8* %a7, i8* %a8, i8* %a9, i8* %a0)
+  ret void
+}
+
+declare void @callee2(i8*, i8*, i8*, i8*, i8*,
+                      i8*, i8*, i8*, i8*, i8*)
+
+; Make sure there is a dependence between the load and store to the same stack
+; location during a tail call. Tail calls clobber the incoming argument area and
+; therefore it is not safe to assume argument locations are invariant.
+; PR23459 has a test case that we where miscompiling because of this at the
+; time.
+
+; CHECK: Frame Objects
+; CHECK:  fi#-4: {{.*}} fixed, at location [SP+8]
+; CHECK:  fi#-3: {{.*}} fixed, at location [SP]
+; CHECK:  fi#-2: {{.*}} fixed, at location [SP+8]
+; CHECK:  fi#-1: {{.*}} fixed, at location [SP]
+
+; CHECK:  [[VRA:%vreg.*]]<def> = LDRXui <fi#-1>
+; CHECK:  [[VRB:%vreg.*]]<def> = LDRXui <fi#-2>
+; CHECK:  STRXui %vreg{{.*}}, <fi#-4>
+; CHECK:  STRXui [[VRB]], <fi#-3>
+
+; Make sure that there is an dependence edge between fi#-2 and fi#-4.
+; Without this edge the scheduler would be free to move the store accross the load.
+
+; CHECK: SU({{.*}}):   [[VRB]]<def> = LDRXui <fi#-2>
+; CHECK-NOT: SU
+; CHECK:  Successors:
+; CHECK:   ch  SU([[DEPSTORE:.*]]): Latency=0
+
+; CHECK: SU([[DEPSTORE]]):   STRXui %vreg0, <fi#-4>
diff --git a/test/CodeGen/AArch64/xbfiz.ll b/test/CodeGen/AArch64/xbfiz.ll
new file mode 100644
index 00000000000..f763400d7f6
--- /dev/null
+++ b/test/CodeGen/AArch64/xbfiz.ll
@@ -0,0 +1,33 @@
+; RUN: llc -mtriple=arm64-apple-ios < %s | FileCheck %s
+
+define i64 @sbfiz64(i64 %v) {
+; CHECK-LABEL: sbfiz64:
+; CHECK: sbfiz	x0, x0, #1, #16
+  %shl = shl i64 %v, 48
+  %shr = ashr i64 %shl, 47
+  ret i64 %shr
+}
+
+define i32 @sbfiz32(i32 %v) {
+; CHECK-LABEL: sbfiz32:
+; CHECK: sbfiz	w0, w0, #1, #14
+  %shl = shl i32 %v, 18
+  %shr = ashr i32 %shl, 17
+  ret i32 %shr
+}
+
+define i64 @ubfiz64(i64 %v) {
+; CHECK-LABEL: ubfiz64:
+; CHECK: ubfiz	x0, x0, #36, #11
+  %shl = shl i64 %v, 53
+  %shr = lshr i64 %shl, 17
+  ret i64 %shr
+}
+
+define i32 @ubfiz32(i32 %v) {
+; CHECK-LABEL: ubfiz32:
+; CHECK: ubfiz	w0, w0, #6, #24
+  %shl = shl i32 %v, 8
+  %shr = lshr i32 %shl, 2
+  ret i32 %shr
+}
diff --git a/test/CodeGen/R600/32-bit-local-address-space.ll b/test/CodeGen/AMDGPU/32-bit-local-address-space.ll
similarity index 100%
rename from test/CodeGen/R600/32-bit-local-address-space.ll
rename to test/CodeGen/AMDGPU/32-bit-local-address-space.ll
diff --git a/test/CodeGen/R600/README b/test/CodeGen/AMDGPU/README
similarity index 100%
rename from test/CodeGen/R600/README
rename to test/CodeGen/AMDGPU/README
diff --git a/test/CodeGen/R600/add-debug.ll b/test/CodeGen/AMDGPU/add-debug.ll
similarity index 100%
rename from test/CodeGen/R600/add-debug.ll
rename to test/CodeGen/AMDGPU/add-debug.ll
diff --git a/test/CodeGen/R600/add.ll b/test/CodeGen/AMDGPU/add.ll
similarity index 86%
rename from test/CodeGen/R600/add.ll
rename to test/CodeGen/AMDGPU/add.ll
index 70271616796..655e75dbc1a 100644
--- a/test/CodeGen/R600/add.ll
+++ b/test/CodeGen/AMDGPU/add.ll
@@ -62,6 +62,7 @@ define void @test4(<4 x i32> addrspace(1)* %out, <4 x i32> addrspace(1)* %in) {
 ; EG: ADD_INT
 ; EG: ADD_INT
 ; EG: ADD_INT
+
 ; SI: s_add_i32
 ; SI: s_add_i32
 ; SI: s_add_i32
@@ -94,6 +95,7 @@ entry:
 ; EG: ADD_INT
 ; EG: ADD_INT
 ; EG: ADD_INT
+
 ; SI: s_add_i32
 ; SI: s_add_i32
 ; SI: s_add_i32
@@ -120,6 +122,14 @@ entry:
 ; FUNC-LABEL: {{^}}add64:
 ; SI: s_add_u32
 ; SI: s_addc_u32
+
+; EG: MEM_RAT_CACHELESS STORE_RAW [[LO:T[0-9]+\.[XYZW]]]
+; EG: MEM_RAT_CACHELESS STORE_RAW [[HI:T[0-9]+\.[XYZW]]]
+; EG-DAG: ADD_INT {{[* ]*}}[[LO]]
+; EG-DAG: ADDC_UINT
+; EG-DAG: ADD_INT
+; EG-DAG: ADD_INT {{[* ]*}}[[HI]]
+; EG-NOT: SUB
 define void @add64(i64 addrspace(1)* %out, i64 %a, i64 %b) {
 entry:
   %0 = add i64 %a, %b
@@ -134,6 +144,14 @@ entry:
 
 ; FUNC-LABEL: {{^}}add64_sgpr_vgpr:
 ; SI-NOT: v_addc_u32_e32 s
+
+; EG: MEM_RAT_CACHELESS STORE_RAW [[LO:T[0-9]+\.[XYZW]]]
+; EG: MEM_RAT_CACHELESS STORE_RAW [[HI:T[0-9]+\.[XYZW]]]
+; EG-DAG: ADD_INT {{[* ]*}}[[LO]]
+; EG-DAG: ADDC_UINT
+; EG-DAG: ADD_INT
+; EG-DAG: ADD_INT {{[* ]*}}[[HI]]
+; EG-NOT: SUB
 define void @add64_sgpr_vgpr(i64 addrspace(1)* %out, i64 %a, i64 addrspace(1)* %in) {
 entry:
   %0 = load i64, i64 addrspace(1)* %in
@@ -146,6 +164,14 @@ entry:
 ; FUNC-LABEL: {{^}}add64_in_branch:
 ; SI: s_add_u32
 ; SI: s_addc_u32
+
+; EG: MEM_RAT_CACHELESS STORE_RAW [[LO:T[0-9]+\.[XYZW]]]
+; EG: MEM_RAT_CACHELESS STORE_RAW [[HI:T[0-9]+\.[XYZW]]]
+; EG-DAG: ADD_INT {{[* ]*}}[[LO]]
+; EG-DAG: ADDC_UINT
+; EG-DAG: ADD_INT
+; EG-DAG: ADD_INT {{[* ]*}}[[HI]]
+; EG-NOT: SUB
 define void @add64_in_branch(i64 addrspace(1)* %out, i64 addrspace(1)* %in, i64 %a, i64 %b, i64 %c) {
 entry:
   %0 = icmp eq i64 %a, 0
diff --git a/test/CodeGen/R600/add_i64.ll b/test/CodeGen/AMDGPU/add_i64.ll
similarity index 100%
rename from test/CodeGen/R600/add_i64.ll
rename to test/CodeGen/AMDGPU/add_i64.ll
diff --git a/test/CodeGen/R600/address-space.ll b/test/CodeGen/AMDGPU/address-space.ll
similarity index 100%
rename from test/CodeGen/R600/address-space.ll
rename to test/CodeGen/AMDGPU/address-space.ll
diff --git a/test/CodeGen/R600/and.ll b/test/CodeGen/AMDGPU/and.ll
similarity index 100%
rename from test/CodeGen/R600/and.ll
rename to test/CodeGen/AMDGPU/and.ll
diff --git a/test/CodeGen/R600/anyext.ll b/test/CodeGen/AMDGPU/anyext.ll
similarity index 100%
rename from test/CodeGen/R600/anyext.ll
rename to test/CodeGen/AMDGPU/anyext.ll
diff --git a/test/CodeGen/R600/array-ptr-calc-i32.ll b/test/CodeGen/AMDGPU/array-ptr-calc-i32.ll
similarity index 100%
rename from test/CodeGen/R600/array-ptr-calc-i32.ll
rename to test/CodeGen/AMDGPU/array-ptr-calc-i32.ll
diff --git a/test/CodeGen/R600/array-ptr-calc-i64.ll b/test/CodeGen/AMDGPU/array-ptr-calc-i64.ll
similarity index 91%
rename from test/CodeGen/R600/array-ptr-calc-i64.ll
rename to test/CodeGen/AMDGPU/array-ptr-calc-i64.ll
index eae095eb844..a3ae3c3aea1 100644
--- a/test/CodeGen/R600/array-ptr-calc-i64.ll
+++ b/test/CodeGen/AMDGPU/array-ptr-calc-i64.ll
@@ -3,8 +3,9 @@
 declare i32 @llvm.SI.tid() readnone
 
 ; SI-LABEL: {{^}}test_array_ptr_calc:
-; SI: v_mul_lo_i32
-; SI: v_mul_hi_i32
+; SI-DAG: v_mul_lo_i32
+; SI-DAG: v_mul_hi_i32
+; SI: s_endpgm
 define void @test_array_ptr_calc(i32 addrspace(1)* noalias %out, [1025 x i32] addrspace(1)* noalias %inA, i32 addrspace(1)* noalias %inB) {
   %tid = call i32 @llvm.SI.tid() readnone
   %a_ptr = getelementptr [1025 x i32], [1025 x i32] addrspace(1)* %inA, i32 %tid, i32 0
diff --git a/test/CodeGen/R600/atomic_cmp_swap_local.ll b/test/CodeGen/AMDGPU/atomic_cmp_swap_local.ll
similarity index 100%
rename from test/CodeGen/R600/atomic_cmp_swap_local.ll
rename to test/CodeGen/AMDGPU/atomic_cmp_swap_local.ll
diff --git a/test/CodeGen/R600/atomic_load_add.ll b/test/CodeGen/AMDGPU/atomic_load_add.ll
similarity index 100%
rename from test/CodeGen/R600/atomic_load_add.ll
rename to test/CodeGen/AMDGPU/atomic_load_add.ll
diff --git a/test/CodeGen/R600/atomic_load_sub.ll b/test/CodeGen/AMDGPU/atomic_load_sub.ll
similarity index 100%
rename from test/CodeGen/R600/atomic_load_sub.ll
rename to test/CodeGen/AMDGPU/atomic_load_sub.ll
diff --git a/test/CodeGen/R600/basic-branch.ll b/test/CodeGen/AMDGPU/basic-branch.ll
similarity index 100%
rename from test/CodeGen/R600/basic-branch.ll
rename to test/CodeGen/AMDGPU/basic-branch.ll
diff --git a/test/CodeGen/R600/basic-loop.ll b/test/CodeGen/AMDGPU/basic-loop.ll
similarity index 100%
rename from test/CodeGen/R600/basic-loop.ll
rename to test/CodeGen/AMDGPU/basic-loop.ll
diff --git a/test/CodeGen/R600/bfe_uint.ll b/test/CodeGen/AMDGPU/bfe_uint.ll
similarity index 100%
rename from test/CodeGen/R600/bfe_uint.ll
rename to test/CodeGen/AMDGPU/bfe_uint.ll
diff --git a/test/CodeGen/R600/bfi_int.ll b/test/CodeGen/AMDGPU/bfi_int.ll
similarity index 100%
rename from test/CodeGen/R600/bfi_int.ll
rename to test/CodeGen/AMDGPU/bfi_int.ll
diff --git a/test/CodeGen/R600/big_alu.ll b/test/CodeGen/AMDGPU/big_alu.ll
similarity index 100%
rename from test/CodeGen/R600/big_alu.ll
rename to test/CodeGen/AMDGPU/big_alu.ll
diff --git a/test/CodeGen/R600/bitcast.ll b/test/CodeGen/AMDGPU/bitcast.ll
similarity index 100%
rename from test/CodeGen/R600/bitcast.ll
rename to test/CodeGen/AMDGPU/bitcast.ll
diff --git a/test/CodeGen/R600/bswap.ll b/test/CodeGen/AMDGPU/bswap.ll
similarity index 100%
rename from test/CodeGen/R600/bswap.ll
rename to test/CodeGen/AMDGPU/bswap.ll
diff --git a/test/CodeGen/R600/build_vector.ll b/test/CodeGen/AMDGPU/build_vector.ll
similarity index 100%
rename from test/CodeGen/R600/build_vector.ll
rename to test/CodeGen/AMDGPU/build_vector.ll
diff --git a/test/CodeGen/AMDGPU/call.ll b/test/CodeGen/AMDGPU/call.ll
new file mode 100644
index 00000000000..e769fd11c28
--- /dev/null
+++ b/test/CodeGen/AMDGPU/call.ll
@@ -0,0 +1,33 @@
+; RUN: not llc -march=amdgcn -mcpu=SI -verify-machineinstrs< %s 2>&1 | FileCheck %s
+; RUN: not llc -march=amdgcn -mcpu=tonga -verify-machineinstrs< %s 2>&1 | FileCheck %s
+; RUN: not llc -march=r600 -mcpu=cypress < %s 2>&1 | FileCheck %s
+
+; CHECK: error: unsupported call to function external_function in test_call_external
+
+
+declare i32 @external_function(i32) nounwind
+
+define void @test_call_external(i32 addrspace(1)* %out, i32 addrspace(1)* %in) {
+  %b_ptr = getelementptr i32, i32 addrspace(1)* %in, i32 1
+  %a = load i32, i32 addrspace(1)* %in
+  %b = load i32, i32 addrspace(1)* %b_ptr
+  %c = call i32 @external_function(i32 %b) nounwind
+  %result = add i32 %a, %c
+  store i32 %result, i32 addrspace(1)* %out
+  ret void
+}
+
+define i32 @defined_function(i32 %x) nounwind noinline {
+  %y = add i32 %x, 8
+  ret i32 %y
+}
+
+define void @test_call(i32 addrspace(1)* %out, i32 addrspace(1)* %in) {
+  %b_ptr = getelementptr i32, i32 addrspace(1)* %in, i32 1
+  %a = load i32, i32 addrspace(1)* %in
+  %b = load i32, i32 addrspace(1)* %b_ptr
+  %c = call i32 @defined_function(i32 %b) nounwind
+  %result = add i32 %a, %c
+  store i32 %result, i32 addrspace(1)* %out
+  ret void
+}
diff --git a/test/CodeGen/R600/call_fs.ll b/test/CodeGen/AMDGPU/call_fs.ll
similarity index 100%
rename from test/CodeGen/R600/call_fs.ll
rename to test/CodeGen/AMDGPU/call_fs.ll
diff --git a/test/CodeGen/R600/cayman-loop-bug.ll b/test/CodeGen/AMDGPU/cayman-loop-bug.ll
similarity index 100%
rename from test/CodeGen/R600/cayman-loop-bug.ll
rename to test/CodeGen/AMDGPU/cayman-loop-bug.ll
diff --git a/test/CodeGen/R600/cf-stack-bug.ll b/test/CodeGen/AMDGPU/cf-stack-bug.ll
similarity index 85%
rename from test/CodeGen/R600/cf-stack-bug.ll
rename to test/CodeGen/AMDGPU/cf-stack-bug.ll
index 02c87d76bb2..75b87e48622 100644
--- a/test/CodeGen/R600/cf-stack-bug.ll
+++ b/test/CodeGen/AMDGPU/cf-stack-bug.ll
@@ -1,12 +1,29 @@
-; RUN: llc -march=r600 -mcpu=redwood -debug-only=r600cf %s -o - 2>&1 | FileCheck %s --check-prefix=BUG64 --check-prefix=FUNC
-; RUN: llc -march=r600 -mcpu=sumo -debug-only=r600cf %s -o - 2>&1 | FileCheck %s --check-prefix=BUG64 --check-prefix=FUNC
-; RUN: llc -march=r600 -mcpu=barts -debug-only=r600cf %s -o - 2>&1 | FileCheck %s --check-prefix=BUG64 --check-prefix=FUNC
-; RUN: llc -march=r600 -mcpu=turks -debug-only=r600cf %s -o - 2>&1 | FileCheck %s --check-prefix=BUG64 --check-prefix=FUNC
-; RUN: llc -march=r600 -mcpu=caicos -debug-only=r600cf %s -o - 2>&1 | FileCheck %s --check-prefix=BUG64 --check-prefix=FUNC
-; RUN: llc -march=r600 -mcpu=cedar -debug-only=r600cf %s -o - 2>&1 | FileCheck %s --check-prefix=BUG32 --check-prefix=FUNC
-; RUN: llc -march=r600 -mcpu=juniper -debug-only=r600cf %s -o - 2>&1 | FileCheck %s --check-prefix=NOBUG --check-prefix=FUNC
-; RUN: llc -march=r600 -mcpu=cypress -debug-only=r600cf %s -o - 2>&1 | FileCheck %s --check-prefix=NOBUG --check-prefix=FUNC
-; RUN: llc -march=r600 -mcpu=cayman -debug-only=r600cf %s -o - 2>&1 | FileCheck %s --check-prefix=NOBUG --check-prefix=FUNC
+; RUN: llc -march=r600 -mcpu=redwood -debug-only=r600cf %s -o - 2>%t | FileCheck %s --check-prefix=FUNC
+; RUN: FileCheck --check-prefix=BUG64 %s < %t
+
+; RUN: llc -march=r600 -mcpu=sumo -debug-only=r600cf %s -o - 2>%t | FileCheck %s --check-prefix=FUNC
+; RUN: FileCheck --check-prefix=BUG64 %s < %t
+
+; RUN: llc -march=r600 -mcpu=barts -debug-only=r600cf %s -o - 2>%t | FileCheck %s --check-prefix=FUNC
+; RUN: FileCheck --check-prefix=BUG64 %s < %t
+
+; RUN: llc -march=r600 -mcpu=turks -debug-only=r600cf %s -o - 2>%t | FileCheck %s --check-prefix=FUNC
+; RUN: FileCheck --check-prefix=BUG64 %s < %t
+
+; RUN: llc -march=r600 -mcpu=caicos -debug-only=r600cf %s -o - 2>%t | FileCheck %s --check-prefix=FUNC
+; RUN: FileCheck --check-prefix=BUG64 %s < %t
+
+; RUN: llc -march=r600 -mcpu=cedar -debug-only=r600cf %s -o - 2>%t | FileCheck %s --check-prefix=FUNC
+; RUN: FileCheck --check-prefix=BUG32 %s < %t
+
+; RUN: llc -march=r600 -mcpu=juniper -debug-only=r600cf %s -o - 2>%t | FileCheck %s --check-prefix=FUNC
+; RUN: FileCheck --check-prefix=NOBUG %s < %t
+
+; RUN: llc -march=r600 -mcpu=cypress -debug-only=r600cf %s -o - 2>%t | FileCheck %s --check-prefix=FUNC
+; RUN: FileCheck --check-prefix=NOBUG %s < %t
+
+; RUN: llc -march=r600 -mcpu=cayman -debug-only=r600cf %s -o - 2>%t | FileCheck %s --check-prefix=FUNC
+; RUN: FileCheck --check-prefix=NOBUG %s < %t
 
 ; REQUIRES: asserts
 
diff --git a/test/CodeGen/R600/cf_end.ll b/test/CodeGen/AMDGPU/cf_end.ll
similarity index 100%
rename from test/CodeGen/R600/cf_end.ll
rename to test/CodeGen/AMDGPU/cf_end.ll
diff --git a/test/CodeGen/AMDGPU/cgp-addressing-modes.ll b/test/CodeGen/AMDGPU/cgp-addressing-modes.ll
new file mode 100644
index 00000000000..a68d110fdc9
--- /dev/null
+++ b/test/CodeGen/AMDGPU/cgp-addressing-modes.ll
@@ -0,0 +1,248 @@
+; RUN: opt -S -codegenprepare -mtriple=amdgcn-unknown-unknown -mcpu=bonaire < %s | FileCheck -check-prefix=OPT -check-prefix=OPT-CI %s
+; RUN: opt -S -codegenprepare -mtriple=amdgcn-unknown-unknown -mcpu=tonga < %s | FileCheck -check-prefix=OPT -check-prefix=OPT-VI %s
+; RUN: llc -march=amdgcn -mcpu=bonaire -mattr=-promote-alloca < %s | FileCheck -check-prefix=GCN -check-prefix=CI %s
+; RUN: llc -march=amdgcn -mcpu=tonga -mattr=-promote-alloca < %s | FileCheck -check-prefix=GCN -check-prefix=VI %s
+
+declare i32 @llvm.r600.read.tidig.x() #0
+
+; OPT-LABEL: @test_sink_global_small_offset_i32(
+; OPT-CI-NOT: getelementptr i32, i32 addrspace(1)* %in
+; OPT-VI: getelementptr i32, i32 addrspace(1)* %in
+; OPT: br i1
+; OPT-CI: ptrtoint
+
+; GCN-LABEL: {{^}}test_sink_global_small_offset_i32:
+; GCN: {{^}}BB0_2:
+define void @test_sink_global_small_offset_i32(i32 addrspace(1)* %out, i32 addrspace(1)* %in, i32 %cond) {
+entry:
+  %out.gep = getelementptr i32, i32 addrspace(1)* %out, i64 999999
+  %in.gep = getelementptr i32, i32 addrspace(1)* %in, i64 7
+  %tmp0 = icmp eq i32 %cond, 0
+  br i1 %tmp0, label %endif, label %if
+
+if:
+  %tmp1 = load i32, i32 addrspace(1)* %in.gep
+  br label %endif
+
+endif:
+  %x = phi i32 [ %tmp1, %if ], [ 0, %entry ]
+  store i32 %x, i32 addrspace(1)* %out.gep
+  br label %done
+
+done:
+  ret void
+}
+
+; OPT-LABEL: @test_sink_global_small_max_i32_ds_offset(
+; OPT: %in.gep = getelementptr i8, i8 addrspace(1)* %in, i64 65535
+; OPT: br i1
+
+; GCN-LABEL: {{^}}test_sink_global_small_max_i32_ds_offset:
+; GCN: s_and_saveexec_b64
+; GCN: buffer_load_sbyte {{v[0-9]+}}, {{s\[[0-9]+:[0-9]+\]}}, s{{[0-9]+$}}
+; GCN: {{^}}BB1_2:
+; GCN: s_or_b64 exec
+define void @test_sink_global_small_max_i32_ds_offset(i32 addrspace(1)* %out, i8 addrspace(1)* %in, i32 %cond) {
+entry:
+  %out.gep = getelementptr i32, i32 addrspace(1)* %out, i64 99999
+  %in.gep = getelementptr i8, i8 addrspace(1)* %in, i64 65535
+  %tmp0 = icmp eq i32 %cond, 0
+  br i1 %tmp0, label %endif, label %if
+
+if:
+  %tmp1 = load i8, i8 addrspace(1)* %in.gep
+  %tmp2 = sext i8 %tmp1 to i32
+  br label %endif
+
+endif:
+  %x = phi i32 [ %tmp2, %if ], [ 0, %entry ]
+  store i32 %x, i32 addrspace(1)* %out.gep
+  br label %done
+
+done:
+  ret void
+}
+
+; GCN-LABEL: {{^}}test_sink_global_small_max_mubuf_offset:
+; GCN: s_and_saveexec_b64
+; GCN: buffer_load_sbyte {{v[0-9]+}}, {{s\[[0-9]+:[0-9]+\]}}, 0 offset:4095{{$}}
+; GCN: {{^}}BB2_2:
+; GCN: s_or_b64 exec
+define void @test_sink_global_small_max_mubuf_offset(i32 addrspace(1)* %out, i8 addrspace(1)* %in, i32 %cond) {
+entry:
+  %out.gep = getelementptr i32, i32 addrspace(1)* %out, i32 1024
+  %in.gep = getelementptr i8, i8 addrspace(1)* %in, i64 4095
+  %tmp0 = icmp eq i32 %cond, 0
+  br i1 %tmp0, label %endif, label %if
+
+if:
+  %tmp1 = load i8, i8 addrspace(1)* %in.gep
+  %tmp2 = sext i8 %tmp1 to i32
+  br label %endif
+
+endif:
+  %x = phi i32 [ %tmp2, %if ], [ 0, %entry ]
+  store i32 %x, i32 addrspace(1)* %out.gep
+  br label %done
+
+done:
+  ret void
+}
+
+; GCN-LABEL: {{^}}test_sink_global_small_max_plus_1_mubuf_offset:
+; GCN: s_and_saveexec_b64
+; GCN: buffer_load_sbyte {{v[0-9]+}}, {{s\[[0-9]+:[0-9]+\]}}, s{{[0-9]+$}}
+; GCN: {{^}}BB3_2:
+; GCN: s_or_b64 exec
+define void @test_sink_global_small_max_plus_1_mubuf_offset(i32 addrspace(1)* %out, i8 addrspace(1)* %in, i32 %cond) {
+entry:
+  %out.gep = getelementptr i32, i32 addrspace(1)* %out, i64 99999
+  %in.gep = getelementptr i8, i8 addrspace(1)* %in, i64 4096
+  %tmp0 = icmp eq i32 %cond, 0
+  br i1 %tmp0, label %endif, label %if
+
+if:
+  %tmp1 = load i8, i8 addrspace(1)* %in.gep
+  %tmp2 = sext i8 %tmp1 to i32
+  br label %endif
+
+endif:
+  %x = phi i32 [ %tmp2, %if ], [ 0, %entry ]
+  store i32 %x, i32 addrspace(1)* %out.gep
+  br label %done
+
+done:
+  ret void
+}
+
+; OPT-LABEL: @test_no_sink_flat_small_offset_i32(
+; OPT: getelementptr i32, i32 addrspace(4)* %in
+; OPT: br i1
+; OPT-NOT: ptrtoint
+
+; GCN-LABEL: {{^}}test_no_sink_flat_small_offset_i32:
+; GCN: flat_load_dword
+; GCN: {{^}}BB4_2:
+
+define void @test_no_sink_flat_small_offset_i32(i32 addrspace(4)* %out, i32 addrspace(4)* %in, i32 %cond) {
+entry:
+  %out.gep = getelementptr i32, i32 addrspace(4)* %out, i64 999999
+  %in.gep = getelementptr i32, i32 addrspace(4)* %in, i64 7
+  %tmp0 = icmp eq i32 %cond, 0
+  br i1 %tmp0, label %endif, label %if
+
+if:
+  %tmp1 = load i32, i32 addrspace(4)* %in.gep
+  br label %endif
+
+endif:
+  %x = phi i32 [ %tmp1, %if ], [ 0, %entry ]
+  store i32 %x, i32 addrspace(4)* %out.gep
+  br label %done
+
+done:
+  ret void
+}
+
+; OPT-LABEL: @test_sink_scratch_small_offset_i32(
+; OPT-NOT:  getelementptr [512 x i32]
+; OPT: br i1
+; OPT: ptrtoint
+
+; GCN-LABEL: {{^}}test_sink_scratch_small_offset_i32:
+; GCN: s_and_saveexec_b64
+; GCN: buffer_store_dword {{v[0-9]+}}, {{v[0-9]+}}, {{s\[[0-9]+:[0-9]+\]}}, {{s[0-9]+}} offen offset:4092{{$}}
+; GCN: buffer_load_dword {{v[0-9]+}}, {{v[0-9]+}}, {{s\[[0-9]+:[0-9]+\]}}, {{s[0-9]+}} offen offset:4092{{$}}
+; GCN: {{^}}BB5_2:
+define void @test_sink_scratch_small_offset_i32(i32 addrspace(1)* %out, i32 addrspace(1)* %in, i32 %cond, i32 %arg) {
+entry:
+  %alloca = alloca [512 x i32], align 4
+  %out.gep.0 = getelementptr i32, i32 addrspace(1)* %out, i64 999998
+  %out.gep.1 = getelementptr i32, i32 addrspace(1)* %out, i64 999999
+  %add.arg = add i32 %arg, 8
+  %alloca.gep = getelementptr [512 x i32], [512 x i32]* %alloca, i32 0, i32 1023
+  %tmp0 = icmp eq i32 %cond, 0
+  br i1 %tmp0, label %endif, label %if
+
+if:
+  store volatile i32 123, i32* %alloca.gep
+  %tmp1 = load volatile i32, i32* %alloca.gep
+  br label %endif
+
+endif:
+  %x = phi i32 [ %tmp1, %if ], [ 0, %entry ]
+  store i32 %x, i32 addrspace(1)* %out.gep.0
+  %load = load volatile i32, i32* %alloca.gep
+  store i32 %load, i32 addrspace(1)* %out.gep.1
+  br label %done
+
+done:
+  ret void
+}
+
+; OPT-LABEL: @test_no_sink_scratch_large_offset_i32(
+; OPT: %alloca.gep = getelementptr [512 x i32], [512 x i32]* %alloca, i32 0, i32 1024
+; OPT: br i1
+; OPT-NOT: ptrtoint
+
+; GCN-LABEL: {{^}}test_no_sink_scratch_large_offset_i32:
+; GCN: s_and_saveexec_b64
+; GCN: buffer_store_dword {{v[0-9]+}}, {{v[0-9]+}}, {{s\[[0-9]+:[0-9]+\]}}, {{s[0-9]+}} offen{{$}}
+; GCN: buffer_load_dword {{v[0-9]+}}, {{v[0-9]+}}, {{s\[[0-9]+:[0-9]+\]}}, {{s[0-9]+}} offen{{$}}
+; GCN: {{^}}BB6_2:
+define void @test_no_sink_scratch_large_offset_i32(i32 addrspace(1)* %out, i32 addrspace(1)* %in, i32 %cond, i32 %arg) {
+entry:
+  %alloca = alloca [512 x i32], align 4
+  %out.gep.0 = getelementptr i32, i32 addrspace(1)* %out, i64 999998
+  %out.gep.1 = getelementptr i32, i32 addrspace(1)* %out, i64 999999
+  %add.arg = add i32 %arg, 8
+  %alloca.gep = getelementptr [512 x i32], [512 x i32]* %alloca, i32 0, i32 1024
+  %tmp0 = icmp eq i32 %cond, 0
+  br i1 %tmp0, label %endif, label %if
+
+if:
+  store volatile i32 123, i32* %alloca.gep
+  %tmp1 = load volatile i32, i32* %alloca.gep
+  br label %endif
+
+endif:
+  %x = phi i32 [ %tmp1, %if ], [ 0, %entry ]
+  store i32 %x, i32 addrspace(1)* %out.gep.0
+  %load = load volatile i32, i32* %alloca.gep
+  store i32 %load, i32 addrspace(1)* %out.gep.1
+  br label %done
+
+done:
+  ret void
+}
+
+; GCN-LABEL: {{^}}test_sink_global_vreg_sreg_i32:
+; VI-DAG: s_movk_i32 flat_scratch_lo, 0x0
+; VI-DAG: s_movk_i32 flat_scratch_hi, 0x0
+; GCN: s_and_saveexec_b64
+; CI: buffer_load_dword {{v[0-9]+}}, {{v\[[0-9]+:[0-9]+\]}}, {{s\[[0-9]+:[0-9]+\]}}, 0 addr64{{$}}
+; VI: flat_load_dword v{{[0-9]+}}, v[{{[0-9]+:[0-9]+}}]
+; GCN: {{^}}BB7_2:
+define void @test_sink_global_vreg_sreg_i32(i32 addrspace(1)* %out, i32 addrspace(1)* %in, i32 %offset, i32 %cond) {
+entry:
+  %offset.ext = zext i32 %offset to i64
+  %out.gep = getelementptr i32, i32 addrspace(1)* %out, i64 999999
+  %in.gep = getelementptr i32, i32 addrspace(1)* %in, i64 %offset.ext
+  %tmp0 = icmp eq i32 %cond, 0
+  br i1 %tmp0, label %endif, label %if
+
+if:
+  %tmp1 = load i32, i32 addrspace(1)* %in.gep
+  br label %endif
+
+endif:
+  %x = phi i32 [ %tmp1, %if ], [ 0, %entry ]
+  store i32 %x, i32 addrspace(1)* %out.gep
+  br label %done
+
+done:
+  ret void
+}
+
+attributes #0 = { nounwind readnone }
+attributes #1 = { nounwind }
diff --git a/test/CodeGen/AMDGPU/coalescer_remat.ll b/test/CodeGen/AMDGPU/coalescer_remat.ll
new file mode 100644
index 00000000000..96730bcf2e8
--- /dev/null
+++ b/test/CodeGen/AMDGPU/coalescer_remat.ll
@@ -0,0 +1,57 @@
+; RUN: llc -march=amdgcn -verify-machineinstrs -mtriple=amdgcn-- -o - %s | FileCheck %s
+
+declare float @llvm.fma.f32(float, float, float)
+
+; This checks that rematerialization support of the coalescer does not
+; unnecessarily widen the register class. Without those fixes > 20 VGprs
+; are used here
+; Also check that some rematerialization of the 0 constant happened.
+; CHECK-LABEL: foobar
+; CHECK:  v_mov_b32_e32 v{{[0-9]+}}, 0
+; CHECK:  v_mov_b32_e32 v{{[0-9]+}}, 0
+; CHECK:  v_mov_b32_e32 v{{[0-9]+}}, 0
+; CHECK:  v_mov_b32_e32 v{{[0-9]+}}, 0
+; It's probably OK if this is slightly higher:
+; CHECK: ; NumVgprs: 9
+define void @foobar(<4 x float> addrspace(1)* %out, <4 x float> addrspace(1)* %in, i32 %flag) {
+entry:
+  %cmpflag = icmp eq i32 %flag, 1
+  br i1 %cmpflag, label %loop, label %exit
+
+loop:
+  %c = phi i32 [0, %entry], [%cnext, %loop]
+  %v0 = phi float [0.0, %entry], [%fma.0, %loop]
+  %v1 = phi float [0.0, %entry], [%fma.1, %loop]
+  %v2 = phi float [0.0, %entry], [%fma.2, %loop]
+  %v3 = phi float [0.0, %entry], [%fma.3, %loop]
+
+  ; Try to get the 0 constant to get coalesced into a wide register
+  %blup = insertelement <4 x float> undef, float %v0, i32 0
+  store <4 x float> %blup, <4 x float> addrspace(1)* %out
+
+  %load = load <4 x float>, <4 x float> addrspace(1)* %in
+  %load.0 = extractelement <4 x float> %load, i32 0
+  %load.1 = extractelement <4 x float> %load, i32 1
+  %load.2 = extractelement <4 x float> %load, i32 2
+  %load.3 = extractelement <4 x float> %load, i32 3
+  %fma.0 = call float @llvm.fma.f32(float %v0, float %load.0, float %v0)
+  %fma.1 = call float @llvm.fma.f32(float %v1, float %load.1, float %v1)
+  %fma.2 = call float @llvm.fma.f32(float %v2, float %load.2, float %v2)
+  %fma.3 = call float @llvm.fma.f32(float %v3, float %load.3, float %v3)
+
+  %cnext = add nsw i32 %c, 1
+  %cmp = icmp eq i32 %cnext, 42
+  br i1 %cmp, label %exit, label %loop
+
+exit:
+  %ev0 = phi float [0.0, %entry], [%fma.0, %loop]
+  %ev1 = phi float [0.0, %entry], [%fma.1, %loop]
+  %ev2 = phi float [0.0, %entry], [%fma.2, %loop]
+  %ev3 = phi float [0.0, %entry], [%fma.3, %loop]
+  %dst.0 = insertelement <4 x float> undef,  float %ev0, i32 0
+  %dst.1 = insertelement <4 x float> %dst.0, float %ev1, i32 1
+  %dst.2 = insertelement <4 x float> %dst.1, float %ev2, i32 2
+  %dst.3 = insertelement <4 x float> %dst.2, float %ev3, i32 3
+  store <4 x float> %dst.3, <4 x float> addrspace(1)* %out
+  ret void
+}
diff --git a/test/CodeGen/AMDGPU/codegen-prepare-addrmode-sext.ll b/test/CodeGen/AMDGPU/codegen-prepare-addrmode-sext.ll
new file mode 100644
index 00000000000..58517209267
--- /dev/null
+++ b/test/CodeGen/AMDGPU/codegen-prepare-addrmode-sext.ll
@@ -0,0 +1,18 @@
+; RUN: opt -mtriple=amdgcn-- -codegenprepare -S < %s | FileCheck -check-prefix=OPT %s
+; RUN: llc -march=amdgcn -mcpu=verde -verify-machineinstrs < %s | FileCheck -check-prefix=SI-LLC %s
+
+; OPT-LABEL: @test(
+; OPT: mul nsw i32
+; OPT-NEXT: sext
+
+; SI-LLC-LABEL: {{^}}test:
+; SI-LLC: s_mul_i32
+; SI-LLC-NOT: mul
+define void @test(i8 addrspace(1)* nocapture readonly %in, i32 %a, i8 %b) {
+entry:
+  %0 = mul nsw i32 %a, 3
+  %1 = sext i32 %0 to i64
+  %2 = getelementptr i8, i8 addrspace(1)* %in, i64 %1
+  store i8 %b, i8 addrspace(1)* %2
+  ret void
+}
diff --git a/test/CodeGen/R600/combine_vloads.ll b/test/CodeGen/AMDGPU/combine_vloads.ll
similarity index 100%
rename from test/CodeGen/R600/combine_vloads.ll
rename to test/CodeGen/AMDGPU/combine_vloads.ll
diff --git a/test/CodeGen/R600/commute-compares.ll b/test/CodeGen/AMDGPU/commute-compares.ll
similarity index 100%
rename from test/CodeGen/R600/commute-compares.ll
rename to test/CodeGen/AMDGPU/commute-compares.ll
diff --git a/test/CodeGen/AMDGPU/commute-shifts.ll b/test/CodeGen/AMDGPU/commute-shifts.ll
new file mode 100644
index 00000000000..f88cf6470c4
--- /dev/null
+++ b/test/CodeGen/AMDGPU/commute-shifts.ll
@@ -0,0 +1,33 @@
+; RUN: llc -march=amdgcn -mcpu=verde -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=SI %s
+; RUN: llc -march=amdgcn -mcpu=tonga -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=VI %s
+
+; GCN-LABEL: {{^}}main:
+; SI: v_lshl_b32_e32 v{{[0-9]+}}, 1, v{{[0-9]+}}
+; VI: v_lshlrev_b32_e64 v{{[0-9]+}}, v{{[0-9]+}}, 1
+
+define void @main() #0 {
+main_body:
+  %0 = fptosi float undef to i32
+  %1 = call <4 x i32> @llvm.SI.imageload.v4i32(<4 x i32> undef, <32 x i8> undef, i32 2)
+  %2 = extractelement <4 x i32> %1, i32 0
+  %3 = and i32 %0, 7
+  %4 = shl i32 1, %3
+  %5 = and i32 %2, %4
+  %6 = icmp eq i32 %5, 0
+  %.10 = select i1 %6, float 0.000000e+00, float undef
+  %7 = call i32 @llvm.SI.packf16(float undef, float %.10)
+  %8 = bitcast i32 %7 to float
+  call void @llvm.SI.export(i32 15, i32 1, i32 1, i32 0, i32 1, float undef, float %8, float undef, float %8)
+  ret void
+}
+
+; Function Attrs: nounwind readnone
+declare <4 x i32> @llvm.SI.imageload.v4i32(<4 x i32>, <32 x i8>, i32) #1
+
+; Function Attrs: nounwind readnone
+declare i32 @llvm.SI.packf16(float, float) #1
+
+declare void @llvm.SI.export(i32, i32, i32, i32, i32, float, float, float, float)
+
+attributes #0 = { "ShaderType"="0" "enable-no-nans-fp-math"="true" }
+attributes #1 = { nounwind readnone }
diff --git a/test/CodeGen/R600/commute_modifiers.ll b/test/CodeGen/AMDGPU/commute_modifiers.ll
similarity index 100%
rename from test/CodeGen/R600/commute_modifiers.ll
rename to test/CodeGen/AMDGPU/commute_modifiers.ll
diff --git a/test/CodeGen/R600/complex-folding.ll b/test/CodeGen/AMDGPU/complex-folding.ll
similarity index 100%
rename from test/CodeGen/R600/complex-folding.ll
rename to test/CodeGen/AMDGPU/complex-folding.ll
diff --git a/test/CodeGen/R600/concat_vectors.ll b/test/CodeGen/AMDGPU/concat_vectors.ll
similarity index 96%
rename from test/CodeGen/R600/concat_vectors.ll
rename to test/CodeGen/AMDGPU/concat_vectors.ll
index 6b3fae384db..a09ed1f7385 100644
--- a/test/CodeGen/R600/concat_vectors.ll
+++ b/test/CodeGen/AMDGPU/concat_vectors.ll
@@ -283,3 +283,14 @@ define void @test_concat_v16i16(<32 x i16> addrspace(1)* %out, <16 x i16> %a, <1
   store <32 x i16> %concat, <32 x i16> addrspace(1)* %out, align 64
   ret void
 }
+
+; FUNC-LABEL: {{^}}concat_vector_crash:
+; SI: s_endpgm
+define void @concat_vector_crash(<8 x float> addrspace(1)* %out, <2 x float> addrspace(1)* %in) {
+bb:
+  %tmp = load <2 x float>, <2 x float> addrspace(1)* %in, align 4
+  %tmp1 = shufflevector <2 x float> %tmp, <2 x float> undef, <8 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+  %tmp2 = shufflevector <8 x float> undef, <8 x float> %tmp1, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 8, i32 9>
+  store <8 x float> %tmp2, <8 x float> addrspace(1)* %out, align 32
+  ret void
+}
diff --git a/test/CodeGen/R600/copy-illegal-type.ll b/test/CodeGen/AMDGPU/copy-illegal-type.ll
similarity index 100%
rename from test/CodeGen/R600/copy-illegal-type.ll
rename to test/CodeGen/AMDGPU/copy-illegal-type.ll
diff --git a/test/CodeGen/R600/copy-to-reg.ll b/test/CodeGen/AMDGPU/copy-to-reg.ll
similarity index 100%
rename from test/CodeGen/R600/copy-to-reg.ll
rename to test/CodeGen/AMDGPU/copy-to-reg.ll
diff --git a/test/CodeGen/R600/ctlz_zero_undef.ll b/test/CodeGen/AMDGPU/ctlz_zero_undef.ll
similarity index 100%
rename from test/CodeGen/R600/ctlz_zero_undef.ll
rename to test/CodeGen/AMDGPU/ctlz_zero_undef.ll
diff --git a/test/CodeGen/R600/ctpop.ll b/test/CodeGen/AMDGPU/ctpop.ll
similarity index 100%
rename from test/CodeGen/R600/ctpop.ll
rename to test/CodeGen/AMDGPU/ctpop.ll
diff --git a/test/CodeGen/R600/ctpop64.ll b/test/CodeGen/AMDGPU/ctpop64.ll
similarity index 100%
rename from test/CodeGen/R600/ctpop64.ll
rename to test/CodeGen/AMDGPU/ctpop64.ll
diff --git a/test/CodeGen/R600/cttz_zero_undef.ll b/test/CodeGen/AMDGPU/cttz_zero_undef.ll
similarity index 100%
rename from test/CodeGen/R600/cttz_zero_undef.ll
rename to test/CodeGen/AMDGPU/cttz_zero_undef.ll
diff --git a/test/CodeGen/R600/cvt_f32_ubyte.ll b/test/CodeGen/AMDGPU/cvt_f32_ubyte.ll
similarity index 100%
rename from test/CodeGen/R600/cvt_f32_ubyte.ll
rename to test/CodeGen/AMDGPU/cvt_f32_ubyte.ll
diff --git a/test/CodeGen/R600/cvt_flr_i32_f32.ll b/test/CodeGen/AMDGPU/cvt_flr_i32_f32.ll
similarity index 100%
rename from test/CodeGen/R600/cvt_flr_i32_f32.ll
rename to test/CodeGen/AMDGPU/cvt_flr_i32_f32.ll
diff --git a/test/CodeGen/R600/cvt_rpi_i32_f32.ll b/test/CodeGen/AMDGPU/cvt_rpi_i32_f32.ll
similarity index 100%
rename from test/CodeGen/R600/cvt_rpi_i32_f32.ll
rename to test/CodeGen/AMDGPU/cvt_rpi_i32_f32.ll
diff --git a/test/CodeGen/R600/dagcombiner-bug-illegal-vec4-int-to-fp.ll b/test/CodeGen/AMDGPU/dagcombiner-bug-illegal-vec4-int-to-fp.ll
similarity index 100%
rename from test/CodeGen/R600/dagcombiner-bug-illegal-vec4-int-to-fp.ll
rename to test/CodeGen/AMDGPU/dagcombiner-bug-illegal-vec4-int-to-fp.ll
diff --git a/test/CodeGen/AMDGPU/debug.ll b/test/CodeGen/AMDGPU/debug.ll
new file mode 100644
index 00000000000..a2e0e878b74
--- /dev/null
+++ b/test/CodeGen/AMDGPU/debug.ll
@@ -0,0 +1,10 @@
+; RUN: llc < %s -march=amdgcn -mcpu=verde -verify-machineinstrs -mattr=dumpcode -filetype=obj | FileCheck --check-prefix=SI --check-prefix=FUNC %s
+; RUN: llc < %s -march=amdgcn -mcpu=tonga -verify-machineinstrs -mattr=dumpcode -filetype=obj | FileCheck --check-prefix=SI --check-prefix=FUNC %s
+
+; Test for a crash in the custom assembly dump code.
+
+; SI: s_endpgm
+define void @test(i32 addrspace(1)* %out) {
+  store i32 0, i32 addrspace(1)* %out
+  ret void
+}
diff --git a/test/CodeGen/R600/default-fp-mode.ll b/test/CodeGen/AMDGPU/default-fp-mode.ll
similarity index 100%
rename from test/CodeGen/R600/default-fp-mode.ll
rename to test/CodeGen/AMDGPU/default-fp-mode.ll
diff --git a/test/CodeGen/R600/disconnected-predset-break-bug.ll b/test/CodeGen/AMDGPU/disconnected-predset-break-bug.ll
similarity index 100%
rename from test/CodeGen/R600/disconnected-predset-break-bug.ll
rename to test/CodeGen/AMDGPU/disconnected-predset-break-bug.ll
diff --git a/test/CodeGen/R600/dot4-folding.ll b/test/CodeGen/AMDGPU/dot4-folding.ll
similarity index 100%
rename from test/CodeGen/R600/dot4-folding.ll
rename to test/CodeGen/AMDGPU/dot4-folding.ll
diff --git a/test/CodeGen/R600/ds-negative-offset-addressing-mode-loop.ll b/test/CodeGen/AMDGPU/ds-negative-offset-addressing-mode-loop.ll
similarity index 91%
rename from test/CodeGen/R600/ds-negative-offset-addressing-mode-loop.ll
rename to test/CodeGen/AMDGPU/ds-negative-offset-addressing-mode-loop.ll
index e7e13d6178c..5e4654abd91 100644
--- a/test/CodeGen/R600/ds-negative-offset-addressing-mode-loop.ll
+++ b/test/CodeGen/AMDGPU/ds-negative-offset-addressing-mode-loop.ll
@@ -1,5 +1,6 @@
-; RUN: llc -march=amdgcn -mcpu=SI -verify-machineinstrs -mattr=+load-store-opt -enable-misched < %s | FileCheck -check-prefix=SI --check-prefix=CHECK %s
-; RUN: llc -march=amdgcn -mcpu=bonaire -verify-machineinstrs -mattr=+load-store-opt -enable-misched < %s | FileCheck -check-prefix=CI --check-prefix=CHECK %s
+; RUN: llc -march=amdgcn -mcpu=SI -verify-machineinstrs -mattr=+load-store-opt < %s | FileCheck -check-prefix=SI --check-prefix=CHECK %s
+; RUN: llc -march=amdgcn -mcpu=bonaire -verify-machineinstrs -mattr=+load-store-opt < %s | FileCheck -check-prefix=CI --check-prefix=CHECK %s
+; RUN: llc -march=amdgcn -mcpu=SI -verify-machineinstrs -mattr=+load-store-opt,+unsafe-ds-offset-folding < %s | FileCheck -check-prefix=CI --check-prefix=CHECK %s
 
 declare i32 @llvm.r600.read.tidig.x() #0
 declare void @llvm.AMDGPU.barrier.local() #1
diff --git a/test/CodeGen/R600/ds_read2.ll b/test/CodeGen/AMDGPU/ds_read2.ll
similarity index 99%
rename from test/CodeGen/R600/ds_read2.ll
rename to test/CodeGen/AMDGPU/ds_read2.ll
index 5929898f8bd..ec04f8b1acd 100644
--- a/test/CodeGen/R600/ds_read2.ll
+++ b/test/CodeGen/AMDGPU/ds_read2.ll
@@ -1,10 +1,10 @@
-; RUN: llc -march=amdgcn -mcpu=bonaire -verify-machineinstrs -mattr=+load-store-opt -enable-misched < %s | FileCheck -strict-whitespace -check-prefix=SI %s
+; RUN: llc -march=amdgcn -mcpu=bonaire -verify-machineinstrs -mattr=+load-store-opt < %s | FileCheck -strict-whitespace -check-prefix=SI %s
 
 ; FIXME: We don't get cases where the address was an SGPR because we
 ; get a copy to the address register for each one.
 
 @lds = addrspace(3) global [512 x float] undef, align 4
- @lds.f64 = addrspace(3) global [512 x double] undef, align 8
+@lds.f64 = addrspace(3) global [512 x double] undef, align 8
 
 ; SI-LABEL: @simple_read2_f32
 ; SI: ds_read2_b32 v{{\[}}[[LO_VREG:[0-9]+]]:[[HI_VREG:[0-9]+]]{{\]}}, v{{[0-9]+}} offset1:8
diff --git a/test/CodeGen/R600/ds_read2_offset_order.ll b/test/CodeGen/AMDGPU/ds_read2_offset_order.ll
similarity index 85%
rename from test/CodeGen/R600/ds_read2_offset_order.ll
rename to test/CodeGen/AMDGPU/ds_read2_offset_order.ll
index 9ea9a5a2617..d362c46bbf9 100644
--- a/test/CodeGen/R600/ds_read2_offset_order.ll
+++ b/test/CodeGen/AMDGPU/ds_read2_offset_order.ll
@@ -1,16 +1,17 @@
-; RUN: llc -march=amdgcn -mcpu=bonaire -verify-machineinstrs -mattr=+load-store-opt -enable-misched < %s | FileCheck -strict-whitespace -check-prefix=SI %s
-; RUN: llc -march=amdgcn -mcpu=tonga -verify-machineinstrs -mattr=+load-store-opt -enable-misched < %s | FileCheck -strict-whitespace -check-prefix=SI %s
+; RUN: llc -march=amdgcn -mcpu=bonaire -verify-machineinstrs -mattr=+load-store-opt < %s | FileCheck -strict-whitespace -check-prefix=SI %s
+; RUN: llc -march=amdgcn -mcpu=tonga -verify-machineinstrs -mattr=+load-store-opt < %s | FileCheck -strict-whitespace -check-prefix=SI %s
 
-; XFAIL: *
 
 @lds = addrspace(3) global [512 x float] undef, align 4
 
+; offset0 is larger than offset1
+
 ; SI-LABEL: {{^}}offset_order:
 
-; SI: ds_read_b32 v{{[0-9]+}}, v{{[0-9]+}} offset:56
-; SI: ds_read2st64_b32 v[{{[0-9]+}}:{{[0-9]+}}], v{{[0-9]+}} offset0:0 offset1:4
-; SI: ds_read2_b32 v[{{[0-9]+}}:{{[0-9]+}}], v{{[0-9]+}} offset0:2 offset1:3
-; SI: ds_read2_b32 v[{{[0-9]+}}:{{[0-9]+}}], v{{[0-9]+}} offset0:11 offset1:1
+; SI: ds_read2st64_b32 v[{{[0-9]+}}:{{[0-9]+}}], v{{[0-9]+}} offset1:4{{$}}
+; SI: ds_read2_b32 v[{{[0-9]+}}:{{[0-9]+}}], v{{[0-9]+}} offset0:3 offset1:2
+; SI: ds_read2_b32 v[{{[0-9]+}}:{{[0-9]+}}], v{{[0-9]+}} offset0:12 offset1:14
+; SI: ds_read_b32 v{{[0-9]+}}, v{{[0-9]+}} offset:44
 
 define void @offset_order(float addrspace(1)* %out) {
 entry:
diff --git a/test/CodeGen/AMDGPU/ds_read2_superreg.ll b/test/CodeGen/AMDGPU/ds_read2_superreg.ll
new file mode 100644
index 00000000000..842c2d8bc33
--- /dev/null
+++ b/test/CodeGen/AMDGPU/ds_read2_superreg.ll
@@ -0,0 +1,246 @@
+; RUN: llc -march=amdgcn -mcpu=bonaire -verify-machineinstrs -mattr=+load-store-opt < %s | FileCheck -check-prefix=CI %s
+
+@lds = addrspace(3) global [512 x float] undef, align 4
+@lds.v2 = addrspace(3) global [512 x <2 x float>] undef, align 4
+@lds.v3 = addrspace(3) global [512 x <3 x float>] undef, align 4
+@lds.v4 = addrspace(3) global [512 x <4 x float>] undef, align 4
+@lds.v8 = addrspace(3) global [512 x <8 x float>] undef, align 4
+@lds.v16 = addrspace(3) global [512 x <16 x float>] undef, align 4
+
+; CI-LABEL: {{^}}simple_read2_v2f32_superreg_align4:
+; CI: ds_read2_b32 [[RESULT:v\[[0-9]+:[0-9]+\]]], v{{[0-9]+}} offset1:1{{$}}
+; CI: s_waitcnt lgkmcnt(0)
+; CI: buffer_store_dwordx2 [[RESULT]]
+; CI: s_endpgm
+define void @simple_read2_v2f32_superreg_align4(<2 x float> addrspace(1)* %out) #0 {
+  %x.i = tail call i32 @llvm.r600.read.tidig.x() #1
+  %arrayidx0 = getelementptr inbounds  [512 x <2 x float>], [512 x <2 x float>] addrspace(3)* @lds.v2, i32 0, i32 %x.i
+  %val0 = load <2 x float>, <2 x float> addrspace(3)* %arrayidx0, align 4
+  %out.gep = getelementptr inbounds <2 x float>, <2 x float> addrspace(1)* %out, i32 %x.i
+  store <2 x float> %val0, <2 x float> addrspace(1)* %out.gep
+  ret void
+}
+
+; CI-LABEL: {{^}}simple_read2_v2f32_superreg:
+; CI: ds_read_b64 [[RESULT:v\[[0-9]+:[0-9]+\]]], v{{[0-9]+}}{{$}}
+; CI: s_waitcnt lgkmcnt(0)
+; CI: buffer_store_dwordx2 [[RESULT]]
+; CI: s_endpgm
+define void @simple_read2_v2f32_superreg(<2 x float> addrspace(1)* %out) #0 {
+  %x.i = tail call i32 @llvm.r600.read.tidig.x() #1
+  %arrayidx0 = getelementptr inbounds [512 x <2 x float>], [512 x <2 x float>] addrspace(3)* @lds.v2, i32 0, i32 %x.i
+  %val0 = load <2 x float>, <2 x float> addrspace(3)* %arrayidx0
+  %out.gep = getelementptr inbounds <2 x float>, <2 x float> addrspace(1)* %out, i32 %x.i
+  store <2 x float> %val0, <2 x float> addrspace(1)* %out.gep
+  ret void
+}
+
+; FIXME: Shuffling to new superregister
+; CI-LABEL: {{^}}simple_read2_v4f32_superreg_align4:
+; CI-DAG: ds_read2_b32 v{{\[}}[[REG_W:[0-9]+]]:[[REG_Z:[0-9]+]]{{\]}}, v{{[0-9]+}} offset0:3 offset1:2{{$}}
+; CI-DAG: ds_read2_b32 v{{\[}}[[REG_Y:[0-9]+]]:[[REG_X:[0-9]+]]{{\]}}, v{{[0-9]+}} offset0:1{{$}}
+; CI-DAG: v_mov_b32_e32 v[[COPY_REG_Y:[0-9]+]], v[[REG_Y]]
+; CI-DAG: v_mov_b32_e32 v[[COPY_REG_Z:[0-9]+]], v[[REG_Z]]
+; CI-DAG: v_add_f32_e32 v[[ADD0:[0-9]+]], v[[COPY_REG_Z]], v[[REG_X]]
+; CI-DAG: v_add_f32_e32 v[[ADD1:[0-9]+]], v[[REG_W]], v[[COPY_REG_Y]]
+; CI: v_add_f32_e32 v[[ADD2:[0-9]+]], v[[ADD1]], v[[ADD0]]
+; CI: buffer_store_dword v[[ADD2]]
+; CI: s_endpgm
+define void @simple_read2_v4f32_superreg_align4(float addrspace(1)* %out) #0 {
+  %x.i = tail call i32 @llvm.r600.read.tidig.x() #1
+  %arrayidx0 = getelementptr inbounds [512 x <4 x float>], [512 x <4 x float>] addrspace(3)* @lds.v4, i32 0, i32 %x.i
+  %val0 = load <4 x float>, <4 x float> addrspace(3)* %arrayidx0, align 4
+  %elt0 = extractelement <4 x float> %val0, i32 0
+  %elt1 = extractelement <4 x float> %val0, i32 1
+  %elt2 = extractelement <4 x float> %val0, i32 2
+  %elt3 = extractelement <4 x float> %val0, i32 3
+
+  %add0 = fadd float %elt0, %elt2
+  %add1 = fadd float %elt1, %elt3
+  %add2 = fadd float %add0, %add1
+
+  %out.gep = getelementptr inbounds float, float addrspace(1)* %out, i32 %x.i
+  store float %add2, float addrspace(1)* %out.gep
+  ret void
+}
+
+; CI-LABEL: {{^}}simple_read2_v3f32_superreg_align4:
+; CI-DAG: ds_read2_b32 v{{\[}}[[REG_X:[0-9]+]]:[[REG_Y:[0-9]+]]{{\]}}, v{{[0-9]+}} offset1:1{{$}}
+; CI-DAG: ds_read_b32 v[[REG_Z:[0-9]+]], v{{[0-9]+}} offset:8{{$}}
+; CI-DAG: v_add_f32_e32 v[[ADD0:[0-9]+]], v[[REG_Z]], v[[REG_X]]
+; CI-DAG: v_add_f32_e32 v[[ADD1:[0-9]+]], v[[REG_Y]], v[[ADD0]]
+; CI: buffer_store_dword v[[ADD1]]
+; CI: s_endpgm
+define void @simple_read2_v3f32_superreg_align4(float addrspace(1)* %out) #0 {
+  %x.i = tail call i32 @llvm.r600.read.tidig.x() #1
+  %arrayidx0 = getelementptr inbounds [512 x <3 x float>], [512 x <3 x float>] addrspace(3)* @lds.v3, i32 0, i32 %x.i
+  %val0 = load <3 x float>, <3 x float> addrspace(3)* %arrayidx0, align 4
+  %elt0 = extractelement <3 x float> %val0, i32 0
+  %elt1 = extractelement <3 x float> %val0, i32 1
+  %elt2 = extractelement <3 x float> %val0, i32 2
+
+  %add0 = fadd float %elt0, %elt2
+  %add1 = fadd float %add0, %elt1
+
+  %out.gep = getelementptr inbounds float, float addrspace(1)* %out, i32 %x.i
+  store float %add1, float addrspace(1)* %out.gep
+  ret void
+}
+
+; CI-LABEL: {{^}}simple_read2_v4f32_superreg_align8:
+; CI-DAG: ds_read2_b32 v{{\[}}[[REG_W:[0-9]+]]:[[REG_Z:[0-9]+]]{{\]}}, v{{[0-9]+}} offset0:3 offset1:2{{$}}
+; CI-DAG: ds_read2_b32 v{{\[}}[[REG_X:[0-9]+]]:[[REG_Y:[0-9]+]]{{\]}}, v{{[0-9]+}} offset0:1{{$}}
+; CI: buffer_store_dwordx4
+; CI: s_endpgm
+define void @simple_read2_v4f32_superreg_align8(<4 x float> addrspace(1)* %out) #0 {
+  %x.i = tail call i32 @llvm.r600.read.tidig.x() #1
+  %arrayidx0 = getelementptr inbounds [512 x <4 x float>], [512 x <4 x float>] addrspace(3)* @lds.v4, i32 0, i32 %x.i
+  %val0 = load <4 x float>, <4 x float> addrspace(3)* %arrayidx0, align 8
+  %out.gep = getelementptr inbounds <4 x float>, <4 x float> addrspace(1)* %out, i32 %x.i
+  store <4 x float> %val0, <4 x float> addrspace(1)* %out.gep
+  ret void
+}
+
+; CI-LABEL: {{^}}simple_read2_v4f32_superreg:
+; CI-DAG: ds_read2_b32 v{{\[}}[[REG_W:[0-9]+]]:[[REG_Z:[0-9]+]]{{\]}}, v{{[0-9]+}} offset0:3 offset1:2{{$}}
+; CI-DAG: ds_read2_b32 v{{\[}}[[REG_X:[0-9]+]]:[[REG_Y:[0-9]+]]{{\]}}, v{{[0-9]+}} offset0:1{{$}}
+; CI: buffer_store_dwordx4
+; CI: s_endpgm
+define void @simple_read2_v4f32_superreg(<4 x float> addrspace(1)* %out) #0 {
+  %x.i = tail call i32 @llvm.r600.read.tidig.x() #1
+  %arrayidx0 = getelementptr inbounds [512 x <4 x float>], [512 x <4 x float>] addrspace(3)* @lds.v4, i32 0, i32 %x.i
+  %val0 = load <4 x float>, <4 x float> addrspace(3)* %arrayidx0
+  %out.gep = getelementptr inbounds <4 x float>, <4 x float> addrspace(1)* %out, i32 %x.i
+  store <4 x float> %val0, <4 x float> addrspace(1)* %out.gep
+  ret void
+}
+
+; CI-LABEL: {{^}}simple_read2_v8f32_superreg:
+; CI-DAG: ds_read2_b32 v{{\[}}[[REG_ELT7:[0-9]+]]:[[REG_ELT6:[0-9]+]]{{\]}}, v{{[0-9]+}} offset0:7 offset1:6{{$}}
+; CI-DAG: ds_read2_b32 v{{\[}}[[REG_ELT5:[0-9]+]]:[[REG_ELT4:[0-9]+]]{{\]}}, v{{[0-9]+}} offset0:5 offset1:4{{$}}
+; CI-DAG: ds_read2_b32 v{{\[}}[[REG_ELT3:[0-9]+]]:[[REG_ELT2:[0-9]+]]{{\]}}, v{{[0-9]+}} offset0:3 offset1:2{{$}}
+; CI-DAG: ds_read2_b32 v{{\[}}[[REG_ELT1:[0-9]+]]:[[REG_ELT0:[0-9]+]]{{\]}}, v{{[0-9]+}} offset0:1{{$}}
+; CI: buffer_store_dword
+; CI: buffer_store_dword
+; CI: buffer_store_dword
+; CI: buffer_store_dword
+; CI: buffer_store_dword
+; CI: buffer_store_dword
+; CI: buffer_store_dword
+; CI: buffer_store_dword
+; CI: s_endpgm
+define void @simple_read2_v8f32_superreg(<8 x float> addrspace(1)* %out) #0 {
+  %x.i = tail call i32 @llvm.r600.read.tidig.x() #1
+  %arrayidx0 = getelementptr inbounds [512 x <8 x float>], [512 x <8 x float>] addrspace(3)* @lds.v8, i32 0, i32 %x.i
+  %val0 = load <8 x float>, <8 x float> addrspace(3)* %arrayidx0
+  %out.gep = getelementptr inbounds <8 x float>, <8 x float> addrspace(1)* %out, i32 %x.i
+  store <8 x float> %val0, <8 x float> addrspace(1)* %out.gep
+  ret void
+}
+
+; CI-LABEL: {{^}}simple_read2_v16f32_superreg:
+; CI-DAG: ds_read2_b32 v{{\[}}[[REG_ELT7:[0-9]+]]:[[REG_ELT6:[0-9]+]]{{\]}}, v{{[0-9]+}} offset0:15 offset1:14{{$}}
+; CI-DAG: ds_read2_b32 v{{\[}}[[REG_ELT7:[0-9]+]]:[[REG_ELT6:[0-9]+]]{{\]}}, v{{[0-9]+}} offset0:13 offset1:12{{$}}
+; CI-DAG: ds_read2_b32 v{{\[}}[[REG_ELT7:[0-9]+]]:[[REG_ELT6:[0-9]+]]{{\]}}, v{{[0-9]+}} offset0:11 offset1:10{{$}}
+; CI-DAG: ds_read2_b32 v{{\[}}[[REG_ELT7:[0-9]+]]:[[REG_ELT6:[0-9]+]]{{\]}}, v{{[0-9]+}} offset0:9 offset1:8{{$}}
+; CI-DAG: ds_read2_b32 v{{\[}}[[REG_ELT7:[0-9]+]]:[[REG_ELT6:[0-9]+]]{{\]}}, v{{[0-9]+}} offset0:7 offset1:6{{$}}
+; CI-DAG: ds_read2_b32 v{{\[}}[[REG_ELT5:[0-9]+]]:[[REG_ELT4:[0-9]+]]{{\]}}, v{{[0-9]+}} offset0:5 offset1:4{{$}}
+; CI-DAG: ds_read2_b32 v{{\[}}[[REG_ELT3:[0-9]+]]:[[REG_ELT2:[0-9]+]]{{\]}}, v{{[0-9]+}} offset0:3 offset1:2{{$}}
+; CI-DAG: ds_read2_b32 v{{\[}}[[REG_ELT1:[0-9]+]]:[[REG_ELT0:[0-9]+]]{{\]}}, v{{[0-9]+}} offset0:1{{$}}
+
+; CI: s_waitcnt lgkmcnt(0)
+; CI: buffer_store_dword
+; CI: buffer_store_dword
+; CI: buffer_store_dword
+; CI: buffer_store_dword
+; CI: buffer_store_dword
+; CI: buffer_store_dword
+; CI: buffer_store_dword
+; CI: buffer_store_dword
+; CI: buffer_store_dword
+; CI: buffer_store_dword
+; CI: buffer_store_dword
+; CI: buffer_store_dword
+; CI: buffer_store_dword
+; CI: buffer_store_dword
+; CI: buffer_store_dword
+; CI: buffer_store_dword
+; CI: s_endpgm
+define void @simple_read2_v16f32_superreg(<16 x float> addrspace(1)* %out) #0 {
+  %x.i = tail call i32 @llvm.r600.read.tidig.x() #1
+  %arrayidx0 = getelementptr inbounds [512 x <16 x float>], [512 x <16 x float>] addrspace(3)* @lds.v16, i32 0, i32 %x.i
+  %val0 = load <16 x float>, <16 x float> addrspace(3)* %arrayidx0
+  %out.gep = getelementptr inbounds <16 x float>, <16 x float> addrspace(1)* %out, i32 %x.i
+  store <16 x float> %val0, <16 x float> addrspace(1)* %out.gep
+  ret void
+}
+
+; Do scalar loads into the super register we need.
+; CI-LABEL: {{^}}simple_read2_v2f32_superreg_scalar_loads_align4:
+; CI-DAG: ds_read2_b32 v{{\[}}[[REG_ELT0:[0-9]+]]:[[REG_ELT1:[0-9]+]]{{\]}}, v{{[0-9]+}} offset1:1{{$}}
+; CI-NOT: v_mov
+; CI: buffer_store_dwordx2 v{{\[}}[[REG_ELT0]]:[[REG_ELT1]]{{\]}}
+; CI: s_endpgm
+define void @simple_read2_v2f32_superreg_scalar_loads_align4(<2 x float> addrspace(1)* %out) #0 {
+  %x.i = tail call i32 @llvm.r600.read.tidig.x() #1
+  %arrayidx0 = getelementptr inbounds [512 x float], [512 x float] addrspace(3)* @lds, i32 0, i32 %x.i
+  %arrayidx1 = getelementptr inbounds float, float addrspace(3)* %arrayidx0, i32 1
+
+  %val0 = load float, float addrspace(3)* %arrayidx0
+  %val1 = load float, float addrspace(3)* %arrayidx1
+
+  %vec.0 = insertelement <2 x float> undef, float %val0, i32 0
+  %vec.1 = insertelement <2 x float> %vec.0, float %val1, i32 1
+
+  %out.gep = getelementptr inbounds <2 x float>, <2 x float> addrspace(1)* %out, i32 %x.i
+  store <2 x float> %vec.1, <2 x float> addrspace(1)* %out.gep
+  ret void
+}
+
+; Do scalar loads into the super register we need.
+; CI-LABEL: {{^}}simple_read2_v4f32_superreg_scalar_loads_align4:
+; CI-DAG: ds_read2_b32 v{{\[}}[[REG_ELT0:[0-9]+]]:[[REG_ELT1:[0-9]+]]{{\]}}, v{{[0-9]+}} offset1:1{{$}}
+; CI-DAG: ds_read2_b32 v{{\[}}[[REG_ELT2:[0-9]+]]:[[REG_ELT3:[0-9]+]]{{\]}}, v{{[0-9]+}} offset0:2 offset1:3{{$}}
+; CI-NOT: v_mov
+; CI: buffer_store_dwordx4 v{{\[}}[[REG_ELT0]]:[[REG_ELT3]]{{\]}}
+; CI: s_endpgm
+define void @simple_read2_v4f32_superreg_scalar_loads_align4(<4 x float> addrspace(1)* %out) #0 {
+  %x.i = tail call i32 @llvm.r600.read.tidig.x() #1
+  %arrayidx0 = getelementptr inbounds [512 x float], [512 x float] addrspace(3)* @lds, i32 0, i32 %x.i
+  %arrayidx1 = getelementptr inbounds float, float addrspace(3)* %arrayidx0, i32 1
+  %arrayidx2 = getelementptr inbounds float, float addrspace(3)* %arrayidx0, i32 2
+  %arrayidx3 = getelementptr inbounds float, float addrspace(3)* %arrayidx0, i32 3
+
+  %val0 = load float, float addrspace(3)* %arrayidx0
+  %val1 = load float, float addrspace(3)* %arrayidx1
+  %val2 = load float, float addrspace(3)* %arrayidx2
+  %val3 = load float, float addrspace(3)* %arrayidx3
+
+  %vec.0 = insertelement <4 x float> undef, float %val0, i32 0
+  %vec.1 = insertelement <4 x float> %vec.0, float %val1, i32 1
+  %vec.2 = insertelement <4 x float> %vec.1, float %val2, i32 2
+  %vec.3 = insertelement <4 x float> %vec.2, float %val3, i32 3
+
+  %out.gep = getelementptr inbounds <4 x float>, <4 x float> addrspace(1)* %out, i32 %x.i
+  store <4 x float> %vec.3, <4 x float> addrspace(1)* %out.gep
+  ret void
+}
+
+; Function Attrs: nounwind readnone
+declare i32 @llvm.r600.read.tgid.x() #1
+
+; Function Attrs: nounwind readnone
+declare i32 @llvm.r600.read.tgid.y() #1
+
+; Function Attrs: nounwind readnone
+declare i32 @llvm.r600.read.tidig.x() #1
+
+; Function Attrs: nounwind readnone
+declare i32 @llvm.r600.read.tidig.y() #1
+
+; Function Attrs: noduplicate nounwind
+declare void @llvm.AMDGPU.barrier.local() #2
+
+attributes #0 = { nounwind "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "no-realign-stack" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #1 = { nounwind readnone }
+attributes #2 = { noduplicate nounwind }
diff --git a/test/CodeGen/R600/ds_read2st64.ll b/test/CodeGen/AMDGPU/ds_read2st64.ll
similarity index 99%
rename from test/CodeGen/R600/ds_read2st64.ll
rename to test/CodeGen/AMDGPU/ds_read2st64.ll
index 54b3b45636d..e2e441214b4 100644
--- a/test/CodeGen/R600/ds_read2st64.ll
+++ b/test/CodeGen/AMDGPU/ds_read2st64.ll
@@ -1,4 +1,4 @@
-; RUN: llc -march=amdgcn -mcpu=bonaire -verify-machineinstrs -mattr=+load-store-opt -enable-misched < %s | FileCheck -check-prefix=SI %s
+; RUN: llc -march=amdgcn -mcpu=bonaire -verify-machineinstrs -mattr=+load-store-opt < %s | FileCheck -check-prefix=SI %s
 
 @lds = addrspace(3) global [512 x float] undef, align 4
 @lds.f64 = addrspace(3) global [512 x double] undef, align 8
diff --git a/test/CodeGen/R600/ds_write2.ll b/test/CodeGen/AMDGPU/ds_write2.ll
similarity index 96%
rename from test/CodeGen/R600/ds_write2.ll
rename to test/CodeGen/AMDGPU/ds_write2.ll
index b553d3459e4..d4973e377b5 100644
--- a/test/CodeGen/R600/ds_write2.ll
+++ b/test/CodeGen/AMDGPU/ds_write2.ll
@@ -1,4 +1,4 @@
-; RUN: llc -march=amdgcn -mcpu=bonaire -verify-machineinstrs -mattr=+load-store-opt -enable-misched < %s | FileCheck -strict-whitespace -check-prefix=SI %s
+; RUN: llc -march=amdgcn -mcpu=bonaire -verify-machineinstrs -mattr=+load-store-opt < %s | FileCheck -strict-whitespace -check-prefix=SI %s
 
 @lds = addrspace(3) global [512 x float] undef, align 4
 @lds.f64 = addrspace(3) global [512 x double] undef, align 8
@@ -25,7 +25,7 @@ define void @simple_write2_one_val_f32(float addrspace(1)* %C, float addrspace(1
 ; SI-DAG: buffer_load_dword [[VAL0:v[0-9]+]], {{v\[[0-9]+:[0-9]+\]}}, {{s\[[0-9]+:[0-9]+\]}}, 0 addr64{{$}}
 ; SI-DAG: buffer_load_dword [[VAL1:v[0-9]+]], {{v\[[0-9]+:[0-9]+\]}}, {{s\[[0-9]+:[0-9]+\]}}, 0 addr64 offset:4
 ; SI-DAG: v_lshlrev_b32_e32 [[VPTR:v[0-9]+]], 2, v{{[0-9]+}}
-; SI: ds_write2_b32 [[VPTR]], [[VAL0]], [[VAL1]] offset1:8 
+; SI: ds_write2_b32 [[VPTR]], [[VAL0]], [[VAL1]] offset1:8
 ; SI: s_endpgm
 define void @simple_write2_two_val_f32(float addrspace(1)* %C, float addrspace(1)* %in) #0 {
   %x.i = tail call i32 @llvm.r600.read.tidig.x() #1
@@ -405,6 +405,19 @@ define void @write2_sgemm_sequence(float addrspace(1)* %C, i32 %lda, i32 %ldb, f
   ret void
 }
 
+; CI-LABEL: {{^}}simple_write2_v4f32_superreg_align4:
+; CI: ds_write2_b32 {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}} offset0:3 offset1:2{{$}}
+; CI: ds_write2_b32 {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}} offset0:1{{$}}
+; CI: s_endpgm
+define void @simple_write2_v4f32_superreg_align4(<4 x float> addrspace(3)* %out, <4 x float> addrspace(1)* %in) #0 {
+  %x.i = tail call i32 @llvm.r600.read.tidig.x() #1
+  %in.gep = getelementptr inbounds <4 x float>, <4 x float> addrspace(1)* %in
+  %val0 = load <4 x float>, <4 x float> addrspace(1)* %in.gep, align 4
+  %out.gep = getelementptr inbounds <4 x float>, <4 x float> addrspace(3)* %out, i32 %x.i
+  store <4 x float> %val0, <4 x float> addrspace(3)* %out.gep, align 4
+  ret void
+}
+
 ; Function Attrs: nounwind readnone
 declare i32 @llvm.r600.read.tgid.x() #1
 
diff --git a/test/CodeGen/R600/ds_write2st64.ll b/test/CodeGen/AMDGPU/ds_write2st64.ll
similarity index 98%
rename from test/CodeGen/R600/ds_write2st64.ll
rename to test/CodeGen/AMDGPU/ds_write2st64.ll
index 1d9d881c5c7..358aa6a9e36 100644
--- a/test/CodeGen/R600/ds_write2st64.ll
+++ b/test/CodeGen/AMDGPU/ds_write2st64.ll
@@ -1,9 +1,7 @@
-; RUN: llc -march=amdgcn -mcpu=bonaire -verify-machineinstrs -mattr=+load-store-opt -enable-misched < %s | FileCheck -check-prefix=SI %s
-
+; RUN: llc -march=amdgcn -mcpu=bonaire -verify-machineinstrs -mattr=+load-store-opt < %s | FileCheck -check-prefix=SI %s
 
 @lds = addrspace(3) global [512 x float] undef, align 4
 
-
 ; SI-LABEL: @simple_write2st64_one_val_f32_0_1
 ; SI-DAG: buffer_load_dword [[VAL:v[0-9]+]]
 ; SI-DAG: v_lshlrev_b32_e32 [[VPTR:v[0-9]+]], 2, v{{[0-9]+}}
diff --git a/test/CodeGen/AMDGPU/elf.ll b/test/CodeGen/AMDGPU/elf.ll
new file mode 100644
index 00000000000..90af6782c4b
--- /dev/null
+++ b/test/CodeGen/AMDGPU/elf.ll
@@ -0,0 +1,36 @@
+; RUN: llc < %s -march=amdgcn -mcpu=SI -verify-machineinstrs -filetype=obj | llvm-readobj -s -symbols -file-headers - | FileCheck --check-prefix=ELF %s
+; RUN: llc < %s -march=amdgcn -mcpu=SI -verify-machineinstrs -o - | FileCheck --check-prefix=CONFIG --check-prefix=TYPICAL %s
+; RUN: llc < %s -march=amdgcn -mcpu=tonga -verify-machineinstrs -filetype=obj | llvm-readobj -s -symbols -file-headers - | FileCheck --check-prefix=ELF %s
+; RUN: llc < %s -march=amdgcn -mcpu=tonga -verify-machineinstrs -o - | FileCheck --check-prefix=CONFIG --check-prefix=TONGA %s
+; RUN: llc < %s -march=amdgcn -mcpu=carrizo -verify-machineinstrs -filetype=obj | llvm-readobj -s -symbols -file-headers - | FileCheck --check-prefix=ELF %s
+; RUN: llc < %s -march=amdgcn -mcpu=carrizo -verify-machineinstrs -o - | FileCheck --check-prefix=CONFIG --check-prefix=TYPICAL %s
+
+; Test that we don't try to produce a COFF file on windows
+; RUN: llc < %s -mtriple=amdgcn-pc-mingw -mcpu=SI -verify-machineinstrs -filetype=obj | llvm-readobj -s -symbols -file-headers - | FileCheck --check-prefix=ELF %s
+
+; ELF: Format: ELF64
+; ELF: OS/ABI: AMDGPU_HSA (0x40)
+; ELF: Machine: EM_AMDGPU (0xE0)
+; ELF: Name: .AMDGPU.config
+; ELF: Type: SHT_PROGBITS
+
+; ELF: Symbol {
+; ELF: Name: test
+; ELF: Binding: Global
+
+; CONFIG: .section .AMDGPU.config
+; CONFIG-NEXT: .long   45096
+; TYPICAL-NEXT: .long   0
+; TONGA-NEXT: .long   576
+; CONFIG: .align 256
+; CONFIG: test:
+define void @test(i32 %p) #0 {
+   %i = add i32 %p, 2
+   %r = bitcast i32 %i to float
+   call void @llvm.SI.export(i32 15, i32 0, i32 1, i32 12, i32 0, float %r, float %r, float %r, float %r)
+   ret void
+}
+
+declare void @llvm.SI.export(i32, i32, i32, i32, i32, float, float, float, float)
+
+attributes #0 = { "ShaderType"="0" } ; Pixel Shader
diff --git a/test/CodeGen/R600/elf.r600.ll b/test/CodeGen/AMDGPU/elf.r600.ll
similarity index 100%
rename from test/CodeGen/R600/elf.r600.ll
rename to test/CodeGen/AMDGPU/elf.r600.ll
diff --git a/test/CodeGen/R600/empty-function.ll b/test/CodeGen/AMDGPU/empty-function.ll
similarity index 100%
rename from test/CodeGen/R600/empty-function.ll
rename to test/CodeGen/AMDGPU/empty-function.ll
diff --git a/test/CodeGen/R600/endcf-loop-header.ll b/test/CodeGen/AMDGPU/endcf-loop-header.ll
similarity index 100%
rename from test/CodeGen/R600/endcf-loop-header.ll
rename to test/CodeGen/AMDGPU/endcf-loop-header.ll
diff --git a/test/CodeGen/R600/extload-private.ll b/test/CodeGen/AMDGPU/extload-private.ll
similarity index 100%
rename from test/CodeGen/R600/extload-private.ll
rename to test/CodeGen/AMDGPU/extload-private.ll
diff --git a/test/CodeGen/R600/extload.ll b/test/CodeGen/AMDGPU/extload.ll
similarity index 100%
rename from test/CodeGen/R600/extload.ll
rename to test/CodeGen/AMDGPU/extload.ll
diff --git a/test/CodeGen/R600/extract_vector_elt_i16.ll b/test/CodeGen/AMDGPU/extract_vector_elt_i16.ll
similarity index 100%
rename from test/CodeGen/R600/extract_vector_elt_i16.ll
rename to test/CodeGen/AMDGPU/extract_vector_elt_i16.ll
diff --git a/test/CodeGen/R600/fabs.f64.ll b/test/CodeGen/AMDGPU/fabs.f64.ll
similarity index 100%
rename from test/CodeGen/R600/fabs.f64.ll
rename to test/CodeGen/AMDGPU/fabs.f64.ll
diff --git a/test/CodeGen/R600/fabs.ll b/test/CodeGen/AMDGPU/fabs.ll
similarity index 100%
rename from test/CodeGen/R600/fabs.ll
rename to test/CodeGen/AMDGPU/fabs.ll
diff --git a/test/CodeGen/R600/fadd.ll b/test/CodeGen/AMDGPU/fadd.ll
similarity index 100%
rename from test/CodeGen/R600/fadd.ll
rename to test/CodeGen/AMDGPU/fadd.ll
diff --git a/test/CodeGen/R600/fadd64.ll b/test/CodeGen/AMDGPU/fadd64.ll
similarity index 100%
rename from test/CodeGen/R600/fadd64.ll
rename to test/CodeGen/AMDGPU/fadd64.ll
diff --git a/test/CodeGen/R600/fceil.ll b/test/CodeGen/AMDGPU/fceil.ll
similarity index 100%
rename from test/CodeGen/R600/fceil.ll
rename to test/CodeGen/AMDGPU/fceil.ll
diff --git a/test/CodeGen/R600/fceil64.ll b/test/CodeGen/AMDGPU/fceil64.ll
similarity index 100%
rename from test/CodeGen/R600/fceil64.ll
rename to test/CodeGen/AMDGPU/fceil64.ll
diff --git a/test/CodeGen/R600/fcmp-cnd.ll b/test/CodeGen/AMDGPU/fcmp-cnd.ll
similarity index 100%
rename from test/CodeGen/R600/fcmp-cnd.ll
rename to test/CodeGen/AMDGPU/fcmp-cnd.ll
diff --git a/test/CodeGen/R600/fcmp-cnde-int-args.ll b/test/CodeGen/AMDGPU/fcmp-cnde-int-args.ll
similarity index 100%
rename from test/CodeGen/R600/fcmp-cnde-int-args.ll
rename to test/CodeGen/AMDGPU/fcmp-cnde-int-args.ll
diff --git a/test/CodeGen/R600/fcmp.ll b/test/CodeGen/AMDGPU/fcmp.ll
similarity index 100%
rename from test/CodeGen/R600/fcmp.ll
rename to test/CodeGen/AMDGPU/fcmp.ll
diff --git a/test/CodeGen/R600/fcmp64.ll b/test/CodeGen/AMDGPU/fcmp64.ll
similarity index 100%
rename from test/CodeGen/R600/fcmp64.ll
rename to test/CodeGen/AMDGPU/fcmp64.ll
diff --git a/test/CodeGen/R600/fconst64.ll b/test/CodeGen/AMDGPU/fconst64.ll
similarity index 100%
rename from test/CodeGen/R600/fconst64.ll
rename to test/CodeGen/AMDGPU/fconst64.ll
diff --git a/test/CodeGen/R600/fcopysign.f32.ll b/test/CodeGen/AMDGPU/fcopysign.f32.ll
similarity index 100%
rename from test/CodeGen/R600/fcopysign.f32.ll
rename to test/CodeGen/AMDGPU/fcopysign.f32.ll
diff --git a/test/CodeGen/R600/fcopysign.f64.ll b/test/CodeGen/AMDGPU/fcopysign.f64.ll
similarity index 100%
rename from test/CodeGen/R600/fcopysign.f64.ll
rename to test/CodeGen/AMDGPU/fcopysign.f64.ll
diff --git a/test/CodeGen/R600/fdiv.f64.ll b/test/CodeGen/AMDGPU/fdiv.f64.ll
similarity index 100%
rename from test/CodeGen/R600/fdiv.f64.ll
rename to test/CodeGen/AMDGPU/fdiv.f64.ll
diff --git a/test/CodeGen/R600/fdiv.ll b/test/CodeGen/AMDGPU/fdiv.ll
similarity index 100%
rename from test/CodeGen/R600/fdiv.ll
rename to test/CodeGen/AMDGPU/fdiv.ll
diff --git a/test/CodeGen/R600/fetch-limits.r600.ll b/test/CodeGen/AMDGPU/fetch-limits.r600.ll
similarity index 100%
rename from test/CodeGen/R600/fetch-limits.r600.ll
rename to test/CodeGen/AMDGPU/fetch-limits.r600.ll
diff --git a/test/CodeGen/R600/fetch-limits.r700+.ll b/test/CodeGen/AMDGPU/fetch-limits.r700+.ll
similarity index 100%
rename from test/CodeGen/R600/fetch-limits.r700+.ll
rename to test/CodeGen/AMDGPU/fetch-limits.r700+.ll
diff --git a/test/CodeGen/R600/ffloor.f64.ll b/test/CodeGen/AMDGPU/ffloor.f64.ll
similarity index 100%
rename from test/CodeGen/R600/ffloor.f64.ll
rename to test/CodeGen/AMDGPU/ffloor.f64.ll
diff --git a/test/CodeGen/R600/ffloor.ll b/test/CodeGen/AMDGPU/ffloor.ll
similarity index 100%
rename from test/CodeGen/R600/ffloor.ll
rename to test/CodeGen/AMDGPU/ffloor.ll
diff --git a/test/CodeGen/R600/flat-address-space.ll b/test/CodeGen/AMDGPU/flat-address-space.ll
similarity index 98%
rename from test/CodeGen/R600/flat-address-space.ll
rename to test/CodeGen/AMDGPU/flat-address-space.ll
index 425d67d5b07..8ceca078f2d 100644
--- a/test/CodeGen/R600/flat-address-space.ll
+++ b/test/CodeGen/AMDGPU/flat-address-space.ll
@@ -8,7 +8,7 @@
 
 
 ; CHECK-LABEL: {{^}}branch_use_flat_i32:
-; CHECK: flat_store_dword {{v[0-9]+}}, {{v\[[0-9]+:[0-9]+\]}}, [M0, FLAT_SCRATCH]
+; CHECK: flat_store_dword {{v[0-9]+}}, {{v\[[0-9]+:[0-9]+\]}}
 ; CHECK: s_endpgm
 define void @branch_use_flat_i32(i32 addrspace(1)* noalias %out, i32 addrspace(1)* %gptr, i32 addrspace(3)* %lptr, i32 %x, i32 %c) #0 {
 entry:
diff --git a/test/CodeGen/R600/floor.ll b/test/CodeGen/AMDGPU/floor.ll
similarity index 100%
rename from test/CodeGen/R600/floor.ll
rename to test/CodeGen/AMDGPU/floor.ll
diff --git a/test/CodeGen/R600/fma-combine.ll b/test/CodeGen/AMDGPU/fma-combine.ll
similarity index 100%
rename from test/CodeGen/R600/fma-combine.ll
rename to test/CodeGen/AMDGPU/fma-combine.ll
diff --git a/test/CodeGen/R600/fma.f64.ll b/test/CodeGen/AMDGPU/fma.f64.ll
similarity index 100%
rename from test/CodeGen/R600/fma.f64.ll
rename to test/CodeGen/AMDGPU/fma.f64.ll
diff --git a/test/CodeGen/R600/fma.ll b/test/CodeGen/AMDGPU/fma.ll
similarity index 100%
rename from test/CodeGen/R600/fma.ll
rename to test/CodeGen/AMDGPU/fma.ll
diff --git a/test/CodeGen/R600/fmad.ll b/test/CodeGen/AMDGPU/fmad.ll
similarity index 100%
rename from test/CodeGen/R600/fmad.ll
rename to test/CodeGen/AMDGPU/fmad.ll
diff --git a/test/CodeGen/R600/fmax.ll b/test/CodeGen/AMDGPU/fmax.ll
similarity index 100%
rename from test/CodeGen/R600/fmax.ll
rename to test/CodeGen/AMDGPU/fmax.ll
diff --git a/test/CodeGen/R600/fmax3.f64.ll b/test/CodeGen/AMDGPU/fmax3.f64.ll
similarity index 100%
rename from test/CodeGen/R600/fmax3.f64.ll
rename to test/CodeGen/AMDGPU/fmax3.f64.ll
diff --git a/test/CodeGen/R600/fmax3.ll b/test/CodeGen/AMDGPU/fmax3.ll
similarity index 100%
rename from test/CodeGen/R600/fmax3.ll
rename to test/CodeGen/AMDGPU/fmax3.ll
diff --git a/test/CodeGen/R600/fmax_legacy.f64.ll b/test/CodeGen/AMDGPU/fmax_legacy.f64.ll
similarity index 100%
rename from test/CodeGen/R600/fmax_legacy.f64.ll
rename to test/CodeGen/AMDGPU/fmax_legacy.f64.ll
diff --git a/test/CodeGen/R600/fmax_legacy.ll b/test/CodeGen/AMDGPU/fmax_legacy.ll
similarity index 100%
rename from test/CodeGen/R600/fmax_legacy.ll
rename to test/CodeGen/AMDGPU/fmax_legacy.ll
diff --git a/test/CodeGen/R600/fmaxnum.f64.ll b/test/CodeGen/AMDGPU/fmaxnum.f64.ll
similarity index 100%
rename from test/CodeGen/R600/fmaxnum.f64.ll
rename to test/CodeGen/AMDGPU/fmaxnum.f64.ll
diff --git a/test/CodeGen/R600/fmaxnum.ll b/test/CodeGen/AMDGPU/fmaxnum.ll
similarity index 100%
rename from test/CodeGen/R600/fmaxnum.ll
rename to test/CodeGen/AMDGPU/fmaxnum.ll
diff --git a/test/CodeGen/R600/fmin.ll b/test/CodeGen/AMDGPU/fmin.ll
similarity index 100%
rename from test/CodeGen/R600/fmin.ll
rename to test/CodeGen/AMDGPU/fmin.ll
diff --git a/test/CodeGen/R600/fmin3.ll b/test/CodeGen/AMDGPU/fmin3.ll
similarity index 100%
rename from test/CodeGen/R600/fmin3.ll
rename to test/CodeGen/AMDGPU/fmin3.ll
diff --git a/test/CodeGen/R600/fmin_legacy.f64.ll b/test/CodeGen/AMDGPU/fmin_legacy.f64.ll
similarity index 100%
rename from test/CodeGen/R600/fmin_legacy.f64.ll
rename to test/CodeGen/AMDGPU/fmin_legacy.f64.ll
diff --git a/test/CodeGen/R600/fmin_legacy.ll b/test/CodeGen/AMDGPU/fmin_legacy.ll
similarity index 100%
rename from test/CodeGen/R600/fmin_legacy.ll
rename to test/CodeGen/AMDGPU/fmin_legacy.ll
diff --git a/test/CodeGen/R600/fminnum.f64.ll b/test/CodeGen/AMDGPU/fminnum.f64.ll
similarity index 100%
rename from test/CodeGen/R600/fminnum.f64.ll
rename to test/CodeGen/AMDGPU/fminnum.f64.ll
diff --git a/test/CodeGen/R600/fminnum.ll b/test/CodeGen/AMDGPU/fminnum.ll
similarity index 100%
rename from test/CodeGen/R600/fminnum.ll
rename to test/CodeGen/AMDGPU/fminnum.ll
diff --git a/test/CodeGen/AMDGPU/fmul-2-combine-multi-use.ll b/test/CodeGen/AMDGPU/fmul-2-combine-multi-use.ll
new file mode 100644
index 00000000000..1ee92b2f7c0
--- /dev/null
+++ b/test/CodeGen/AMDGPU/fmul-2-combine-multi-use.ll
@@ -0,0 +1,102 @@
+; RUN: llc -march=amdgcn -verify-machineinstrs < %s | FileCheck -check-prefix=GCN %s
+
+; Make sure (fmul (fadd x, x), c) -> (fmul x, (fmul 2.0, c)) doesn't
+; make add an instruction if the fadd has more than one use.
+
+declare float @llvm.fabs.f32(float) #1
+
+; GCN-LABEL: {{^}}multiple_fadd_use_test:
+; GCN: v_max_legacy_f32_e64 [[A16:v[0-9]+]],
+; GCN: v_add_f32_e32 [[A17:v[0-9]+]], [[A16]], [[A16]]
+; GCN: v_mul_f32_e32 [[A18:v[0-9]+]], [[A17]], [[A17]]
+; GCN: v_mad_f32 [[A20:v[0-9]+]], -[[A18]], [[A17]], 1.0
+; GCN: buffer_store_dword [[A20]]
+define void @multiple_fadd_use_test(float addrspace(1)* %out, float %x, float %y, float %z) #0 {
+  %a11 = fadd fast float %y, -1.0
+  %a12 = call float @llvm.fabs.f32(float %a11)
+  %a13 = fadd fast float %x, -1.0
+  %a14 = call float @llvm.fabs.f32(float %a13)
+  %a15 = fcmp ogt float %a12, %a14
+  %a16 = select i1 %a15, float %a12, float %a14
+  %a17 = fmul fast float %a16, 2.0
+  %a18 = fmul fast float %a17, %a17
+  %a19 = fmul fast float %a18, %a17
+  %a20 = fsub fast float 1.0, %a19
+  store float %a20, float addrspace(1)* %out
+  ret void
+}
+
+; GCN-LABEL: {{^}}multiple_use_fadd_fmac
+; GCN-DAG: v_add_f32_e64 [[MUL2:v[0-9]+]], [[X:s[0-9]+]], s{{[0-9]+}}
+; GCN-DAG: v_mac_f32_e64 [[MAD:v[0-9]+]], 2.0, [[X]]
+; GCN-DAG: buffer_store_dword [[MUL2]]
+; GCN-DAG: buffer_store_dword [[MAD]]
+; GCN: s_endpgm
+define void @multiple_use_fadd_fmac(float addrspace(1)* %out, float %x, float %y) #0 {
+  %out.gep.1 = getelementptr float, float addrspace(1)* %out, i32 1
+  %mul2 = fmul fast float %x, 2.0
+  %mad = fadd fast float %mul2, %y
+  store float %mul2, float addrspace(1)* %out
+  store float %mad, float addrspace(1)* %out.gep.1
+  ret void
+}
+
+; GCN-LABEL: {{^}}multiple_use_fadd_fmad:
+; GCN-DAG: v_add_f32_e64 [[MUL2:v[0-9]+]], |[[X:s[0-9]+]]|, |s{{[0-9]+}}|
+; GCN-DAG: v_mad_f32 [[MAD:v[0-9]+]], 2.0, |[[X]]|, v{{[0-9]+}}
+; GCN-DAG: buffer_store_dword [[MUL2]]
+; GCN-DAG: buffer_store_dword [[MAD]]
+; GCN: s_endpgm
+define void @multiple_use_fadd_fmad(float addrspace(1)* %out, float %x, float %y) #0 {
+  %out.gep.1 = getelementptr float, float addrspace(1)* %out, i32 1
+  %x.abs = call float @llvm.fabs.f32(float %x)
+  %mul2 = fmul fast float %x.abs, 2.0
+  %mad = fadd fast float %mul2, %y
+  store float %mul2, float addrspace(1)* %out
+  store float %mad, float addrspace(1)* %out.gep.1
+  ret void
+}
+
+; GCN-LABEL: {{^}}multiple_use_fadd_multi_fmad:
+; GCN: v_mad_f32 {{v[0-9]+}}, 2.0, |[[X:s[0-9]+]]|, v{{[0-9]+}}
+; GCN: v_mad_f32 {{v[0-9]+}}, 2.0, |[[X]]|, v{{[0-9]+}}
+define void @multiple_use_fadd_multi_fmad(float addrspace(1)* %out, float %x, float %y, float %z) #0 {
+  %out.gep.1 = getelementptr float, float addrspace(1)* %out, i32 1
+  %x.abs = call float @llvm.fabs.f32(float %x)
+  %mul2 = fmul fast float %x.abs, 2.0
+  %mad0 = fadd fast float %mul2, %y
+  %mad1 = fadd fast float %mul2, %z
+  store float %mad0, float addrspace(1)* %out
+  store float %mad1, float addrspace(1)* %out.gep.1
+  ret void
+}
+
+; GCN-LABEL: {{^}}fmul_x2_xn2:
+; GCN: v_mul_f32_e64 [[TMP0:v[0-9]+]], -4.0, [[X:s[0-9]+]]
+; GCN: v_mul_f32_e32 [[RESULT:v[0-9]+]], [[X]], [[TMP0]]
+; GCN: buffer_store_dword [[RESULT]]
+define void @fmul_x2_xn2(float addrspace(1)* %out, float %x, float %y) #0 {
+  %out.gep.1 = getelementptr float, float addrspace(1)* %out, i32 1
+  %mul2 = fmul fast float %x, 2.0
+  %muln2 = fmul fast float %x, -2.0
+  %mul = fmul fast float %mul2, %muln2
+  store float %mul, float addrspace(1)* %out
+  ret void
+}
+
+; GCN-LABEL: {{^}}fmul_x2_xn3:
+; GCN: v_mov_b32_e32 [[K:v[0-9]+]], 0xc0c00000
+; GCN: v_mul_f32_e32 [[TMP0:v[0-9]+]], [[X:s[0-9]+]], [[K]]
+; GCN: v_mul_f32_e32 [[RESULT:v[0-9]+]], [[X]], [[TMP0]]
+; GCN: buffer_store_dword [[RESULT]]
+define void @fmul_x2_xn3(float addrspace(1)* %out, float %x, float %y) #0 {
+  %out.gep.1 = getelementptr float, float addrspace(1)* %out, i32 1
+  %mul2 = fmul fast float %x, 2.0
+  %muln2 = fmul fast float %x, -3.0
+  %mul = fmul fast float %mul2, %muln2
+  store float %mul, float addrspace(1)* %out
+  ret void
+}
+
+attributes #0 = { nounwind "unsafe-fp-math"="true" }
+attributes #1 = { nounwind readnone }
diff --git a/test/CodeGen/R600/fmul.ll b/test/CodeGen/AMDGPU/fmul.ll
similarity index 83%
rename from test/CodeGen/R600/fmul.ll
rename to test/CodeGen/AMDGPU/fmul.ll
index 68ebc4dedfe..addc409c9eb 100644
--- a/test/CodeGen/R600/fmul.ll
+++ b/test/CodeGen/AMDGPU/fmul.ll
@@ -73,4 +73,20 @@ define void @test_mul_2_k_inv(float addrspace(1)* %out, float %x) #0 {
   ret void
 }
 
+; There should be three multiplies here; %a should be used twice (once
+; negated), not duplicated into mul x, 5.0 and mul x, -5.0.
+; FUNC-LABEL: {{^}}test_mul_twouse:
+; SI: v_mul_f32
+; SI: v_mul_f32
+; SI: v_mul_f32
+; SI-NOT: v_mul_f32
+define void @test_mul_twouse(float addrspace(1)* %out, float %x, float %y) #0 {
+  %a = fmul float %x, 5.0
+  %b = fsub float -0.0, %a
+  %c = fmul float %b, %y
+  %d = fmul float %c, %a
+  store float %d, float addrspace(1)* %out
+  ret void
+}
+
 attributes #0 = { "less-precise-fpmad"="true" "no-infs-fp-math"="true" "no-nans-fp-math"="true" "unsafe-fp-math"="true" }
diff --git a/test/CodeGen/R600/fmul64.ll b/test/CodeGen/AMDGPU/fmul64.ll
similarity index 100%
rename from test/CodeGen/R600/fmul64.ll
rename to test/CodeGen/AMDGPU/fmul64.ll
diff --git a/test/CodeGen/R600/fmuladd.ll b/test/CodeGen/AMDGPU/fmuladd.ll
similarity index 91%
rename from test/CodeGen/R600/fmuladd.ll
rename to test/CodeGen/AMDGPU/fmuladd.ll
index ae84d841021..600f0cb8357 100644
--- a/test/CodeGen/R600/fmuladd.ll
+++ b/test/CodeGen/AMDGPU/fmuladd.ll
@@ -6,7 +6,7 @@ declare i32 @llvm.r600.read.tidig.x() nounwind readnone
 declare float @llvm.fabs.f32(float) nounwind readnone
 
 ; CHECK-LABEL: {{^}}fmuladd_f32:
-; CHECK: v_mad_f32 {{v[0-9]+, v[0-9]+, v[0-9]+, v[0-9]+}}
+; CHECK: v_mac_f32_e32 {{v[0-9]+, v[0-9]+, v[0-9]+}}
 
 define void @fmuladd_f32(float addrspace(1)* %out, float addrspace(1)* %in1,
                          float addrspace(1)* %in2, float addrspace(1)* %in3) {
@@ -34,8 +34,8 @@ define void @fmuladd_f64(double addrspace(1)* %out, double addrspace(1)* %in1,
 ; CHECK-LABEL: {{^}}fmuladd_2.0_a_b_f32
 ; CHECK-DAG: buffer_load_dword [[R1:v[0-9]+]], {{v\[[0-9]+:[0-9]+\]}}, {{s\[[0-9]+:[0-9]+\]}}, 0 addr64{{$}}
 ; CHECK-DAG: buffer_load_dword [[R2:v[0-9]+]], {{v\[[0-9]+:[0-9]+\]}}, {{s\[[0-9]+:[0-9]+\]}}, 0 addr64 offset:4
-; CHECK: v_mad_f32 [[RESULT:v[0-9]+]], 2.0, [[R1]], [[R2]]
-; CHECK: buffer_store_dword [[RESULT]]
+; CHECK: v_mac_f32_e32 [[R2]], 2.0, [[R1]]
+; CHECK: buffer_store_dword [[R2]]
 define void @fmuladd_2.0_a_b_f32(float addrspace(1)* %out, float addrspace(1)* %in) {
   %tid = call i32 @llvm.r600.read.tidig.x() nounwind readnone
   %gep.0 = getelementptr float, float addrspace(1)* %out, i32 %tid
@@ -53,8 +53,8 @@ define void @fmuladd_2.0_a_b_f32(float addrspace(1)* %out, float addrspace(1)* %
 ; CHECK-LABEL: {{^}}fmuladd_a_2.0_b_f32
 ; CHECK-DAG: buffer_load_dword [[R1:v[0-9]+]], {{v\[[0-9]+:[0-9]+\]}}, {{s\[[0-9]+:[0-9]+\]}}, 0 addr64{{$}}
 ; CHECK-DAG: buffer_load_dword [[R2:v[0-9]+]], {{v\[[0-9]+:[0-9]+\]}}, {{s\[[0-9]+:[0-9]+\]}}, 0 addr64 offset:4
-; CHECK: v_mad_f32 [[RESULT:v[0-9]+]], 2.0, [[R1]], [[R2]]
-; CHECK: buffer_store_dword [[RESULT]]
+; CHECK: v_mac_f32_e32 [[R2]], 2.0, [[R1]]
+; CHECK: buffer_store_dword [[R2]]
 define void @fmuladd_a_2.0_b_f32(float addrspace(1)* %out, float addrspace(1)* %in) {
   %tid = call i32 @llvm.r600.read.tidig.x() nounwind readnone
   %gep.0 = getelementptr float, float addrspace(1)* %out, i32 %tid
@@ -72,8 +72,8 @@ define void @fmuladd_a_2.0_b_f32(float addrspace(1)* %out, float addrspace(1)* %
 ; CHECK-LABEL: {{^}}fadd_a_a_b_f32:
 ; CHECK-DAG: buffer_load_dword [[R1:v[0-9]+]], {{v\[[0-9]+:[0-9]+\]}}, {{s\[[0-9]+:[0-9]+\]}}, 0 addr64{{$}}
 ; CHECK-DAG: buffer_load_dword [[R2:v[0-9]+]], {{v\[[0-9]+:[0-9]+\]}}, {{s\[[0-9]+:[0-9]+\]}}, 0 addr64 offset:4
-; CHECK: v_mad_f32 [[RESULT:v[0-9]+]], 2.0, [[R1]], [[R2]]
-; CHECK: buffer_store_dword [[RESULT]]
+; CHECK: v_mac_f32_e32 [[R2]], 2.0, [[R1]]
+; CHECK: buffer_store_dword [[R2]]
 define void @fadd_a_a_b_f32(float addrspace(1)* %out,
                             float addrspace(1)* %in1,
                             float addrspace(1)* %in2) {
@@ -94,8 +94,8 @@ define void @fadd_a_a_b_f32(float addrspace(1)* %out,
 ; CHECK-LABEL: {{^}}fadd_b_a_a_f32:
 ; CHECK-DAG: buffer_load_dword [[R1:v[0-9]+]], {{v\[[0-9]+:[0-9]+\]}}, {{s\[[0-9]+:[0-9]+\]}}, 0 addr64{{$}}
 ; CHECK-DAG: buffer_load_dword [[R2:v[0-9]+]], {{v\[[0-9]+:[0-9]+\]}}, {{s\[[0-9]+:[0-9]+\]}}, 0 addr64 offset:4
-; CHECK: v_mad_f32 [[RESULT:v[0-9]+]], 2.0, [[R1]], [[R2]]
-; CHECK: buffer_store_dword [[RESULT]]
+; CHECK: v_mac_f32_e32 [[R2]], 2.0, [[R1]]
+; CHECK: buffer_store_dword [[R2]]
 define void @fadd_b_a_a_f32(float addrspace(1)* %out,
                             float addrspace(1)* %in1,
                             float addrspace(1)* %in2) {
@@ -116,8 +116,8 @@ define void @fadd_b_a_a_f32(float addrspace(1)* %out,
 ; CHECK-LABEL: {{^}}fmuladd_neg_2.0_a_b_f32
 ; CHECK-DAG: buffer_load_dword [[R1:v[0-9]+]], {{v\[[0-9]+:[0-9]+\]}}, {{s\[[0-9]+:[0-9]+\]}}, 0 addr64{{$}}
 ; CHECK-DAG: buffer_load_dword [[R2:v[0-9]+]], {{v\[[0-9]+:[0-9]+\]}}, {{s\[[0-9]+:[0-9]+\]}}, 0 addr64 offset:4
-; CHECK: v_mad_f32 [[RESULT:v[0-9]+]], -2.0, [[R1]], [[R2]]
-; CHECK: buffer_store_dword [[RESULT]]
+; CHECK: v_mac_f32_e32 [[R2]], -2.0, [[R1]]
+; CHECK: buffer_store_dword [[R2]]
 define void @fmuladd_neg_2.0_a_b_f32(float addrspace(1)* %out, float addrspace(1)* %in) {
   %tid = call i32 @llvm.r600.read.tidig.x() nounwind readnone
   %gep.0 = getelementptr float, float addrspace(1)* %out, i32 %tid
@@ -136,8 +136,8 @@ define void @fmuladd_neg_2.0_a_b_f32(float addrspace(1)* %out, float addrspace(1
 ; CHECK-LABEL: {{^}}fmuladd_neg_2.0_neg_a_b_f32
 ; CHECK-DAG: buffer_load_dword [[R1:v[0-9]+]], {{v\[[0-9]+:[0-9]+\]}}, {{s\[[0-9]+:[0-9]+\]}}, 0 addr64{{$}}
 ; CHECK-DAG: buffer_load_dword [[R2:v[0-9]+]], {{v\[[0-9]+:[0-9]+\]}}, {{s\[[0-9]+:[0-9]+\]}}, 0 addr64 offset:4
-; CHECK: v_mad_f32 [[RESULT:v[0-9]+]], 2.0, [[R1]], [[R2]]
-; CHECK: buffer_store_dword [[RESULT]]
+; CHECK: v_mac_f32_e32 [[R2]], 2.0, [[R1]]
+; CHECK: buffer_store_dword [[R2]]
 define void @fmuladd_neg_2.0_neg_a_b_f32(float addrspace(1)* %out, float addrspace(1)* %in) {
   %tid = call i32 @llvm.r600.read.tidig.x() nounwind readnone
   %gep.0 = getelementptr float, float addrspace(1)* %out, i32 %tid
@@ -158,8 +158,8 @@ define void @fmuladd_neg_2.0_neg_a_b_f32(float addrspace(1)* %out, float addrspa
 ; CHECK-LABEL: {{^}}fmuladd_2.0_neg_a_b_f32
 ; CHECK-DAG: buffer_load_dword [[R1:v[0-9]+]], {{v\[[0-9]+:[0-9]+\]}}, {{s\[[0-9]+:[0-9]+\]}}, 0 addr64{{$}}
 ; CHECK-DAG: buffer_load_dword [[R2:v[0-9]+]], {{v\[[0-9]+:[0-9]+\]}}, {{s\[[0-9]+:[0-9]+\]}}, 0 addr64 offset:4
-; CHECK: v_mad_f32 [[RESULT:v[0-9]+]], -2.0, [[R1]], [[R2]]
-; CHECK: buffer_store_dword [[RESULT]]
+; CHECK: v_mac_f32_e32 [[R2]], -2.0, [[R1]]
+; CHECK: buffer_store_dword [[R2]]
 define void @fmuladd_2.0_neg_a_b_f32(float addrspace(1)* %out, float addrspace(1)* %in) {
   %tid = call i32 @llvm.r600.read.tidig.x() nounwind readnone
   %gep.0 = getelementptr float, float addrspace(1)* %out, i32 %tid
diff --git a/test/CodeGen/R600/fnearbyint.ll b/test/CodeGen/AMDGPU/fnearbyint.ll
similarity index 100%
rename from test/CodeGen/R600/fnearbyint.ll
rename to test/CodeGen/AMDGPU/fnearbyint.ll
diff --git a/test/CodeGen/R600/fneg-fabs.f64.ll b/test/CodeGen/AMDGPU/fneg-fabs.f64.ll
similarity index 100%
rename from test/CodeGen/R600/fneg-fabs.f64.ll
rename to test/CodeGen/AMDGPU/fneg-fabs.f64.ll
diff --git a/test/CodeGen/R600/fneg-fabs.ll b/test/CodeGen/AMDGPU/fneg-fabs.ll
similarity index 100%
rename from test/CodeGen/R600/fneg-fabs.ll
rename to test/CodeGen/AMDGPU/fneg-fabs.ll
diff --git a/test/CodeGen/R600/fneg.f64.ll b/test/CodeGen/AMDGPU/fneg.f64.ll
similarity index 100%
rename from test/CodeGen/R600/fneg.f64.ll
rename to test/CodeGen/AMDGPU/fneg.f64.ll
diff --git a/test/CodeGen/R600/fneg.ll b/test/CodeGen/AMDGPU/fneg.ll
similarity index 100%
rename from test/CodeGen/R600/fneg.ll
rename to test/CodeGen/AMDGPU/fneg.ll
diff --git a/test/CodeGen/R600/fp-classify.ll b/test/CodeGen/AMDGPU/fp-classify.ll
similarity index 100%
rename from test/CodeGen/R600/fp-classify.ll
rename to test/CodeGen/AMDGPU/fp-classify.ll
diff --git a/test/CodeGen/R600/fp16_to_fp.ll b/test/CodeGen/AMDGPU/fp16_to_fp.ll
similarity index 100%
rename from test/CodeGen/R600/fp16_to_fp.ll
rename to test/CodeGen/AMDGPU/fp16_to_fp.ll
diff --git a/test/CodeGen/R600/fp32_to_fp16.ll b/test/CodeGen/AMDGPU/fp32_to_fp16.ll
similarity index 100%
rename from test/CodeGen/R600/fp32_to_fp16.ll
rename to test/CodeGen/AMDGPU/fp32_to_fp16.ll
diff --git a/test/CodeGen/R600/fp_to_sint.f64.ll b/test/CodeGen/AMDGPU/fp_to_sint.f64.ll
similarity index 100%
rename from test/CodeGen/R600/fp_to_sint.f64.ll
rename to test/CodeGen/AMDGPU/fp_to_sint.f64.ll
diff --git a/test/CodeGen/R600/fp_to_sint.ll b/test/CodeGen/AMDGPU/fp_to_sint.ll
similarity index 100%
rename from test/CodeGen/R600/fp_to_sint.ll
rename to test/CodeGen/AMDGPU/fp_to_sint.ll
diff --git a/test/CodeGen/R600/fp_to_uint.f64.ll b/test/CodeGen/AMDGPU/fp_to_uint.f64.ll
similarity index 100%
rename from test/CodeGen/R600/fp_to_uint.f64.ll
rename to test/CodeGen/AMDGPU/fp_to_uint.f64.ll
diff --git a/test/CodeGen/R600/fp_to_uint.ll b/test/CodeGen/AMDGPU/fp_to_uint.ll
similarity index 100%
rename from test/CodeGen/R600/fp_to_uint.ll
rename to test/CodeGen/AMDGPU/fp_to_uint.ll
diff --git a/test/CodeGen/R600/fpext.ll b/test/CodeGen/AMDGPU/fpext.ll
similarity index 100%
rename from test/CodeGen/R600/fpext.ll
rename to test/CodeGen/AMDGPU/fpext.ll
diff --git a/test/CodeGen/R600/fptrunc.ll b/test/CodeGen/AMDGPU/fptrunc.ll
similarity index 100%
rename from test/CodeGen/R600/fptrunc.ll
rename to test/CodeGen/AMDGPU/fptrunc.ll
diff --git a/test/CodeGen/R600/frem.ll b/test/CodeGen/AMDGPU/frem.ll
similarity index 100%
rename from test/CodeGen/R600/frem.ll
rename to test/CodeGen/AMDGPU/frem.ll
diff --git a/test/CodeGen/R600/fsqrt.ll b/test/CodeGen/AMDGPU/fsqrt.ll
similarity index 100%
rename from test/CodeGen/R600/fsqrt.ll
rename to test/CodeGen/AMDGPU/fsqrt.ll
diff --git a/test/CodeGen/R600/fsub.ll b/test/CodeGen/AMDGPU/fsub.ll
similarity index 100%
rename from test/CodeGen/R600/fsub.ll
rename to test/CodeGen/AMDGPU/fsub.ll
diff --git a/test/CodeGen/R600/fsub64.ll b/test/CodeGen/AMDGPU/fsub64.ll
similarity index 100%
rename from test/CodeGen/R600/fsub64.ll
rename to test/CodeGen/AMDGPU/fsub64.ll
diff --git a/test/CodeGen/R600/ftrunc.f64.ll b/test/CodeGen/AMDGPU/ftrunc.f64.ll
similarity index 100%
rename from test/CodeGen/R600/ftrunc.f64.ll
rename to test/CodeGen/AMDGPU/ftrunc.f64.ll
diff --git a/test/CodeGen/R600/ftrunc.ll b/test/CodeGen/AMDGPU/ftrunc.ll
similarity index 100%
rename from test/CodeGen/R600/ftrunc.ll
rename to test/CodeGen/AMDGPU/ftrunc.ll
diff --git a/test/CodeGen/R600/gep-address-space.ll b/test/CodeGen/AMDGPU/gep-address-space.ll
similarity index 100%
rename from test/CodeGen/R600/gep-address-space.ll
rename to test/CodeGen/AMDGPU/gep-address-space.ll
diff --git a/test/CodeGen/R600/global-directive.ll b/test/CodeGen/AMDGPU/global-directive.ll
similarity index 100%
rename from test/CodeGen/R600/global-directive.ll
rename to test/CodeGen/AMDGPU/global-directive.ll
diff --git a/test/CodeGen/R600/global-extload-i1.ll b/test/CodeGen/AMDGPU/global-extload-i1.ll
similarity index 100%
rename from test/CodeGen/R600/global-extload-i1.ll
rename to test/CodeGen/AMDGPU/global-extload-i1.ll
diff --git a/test/CodeGen/R600/global-extload-i16.ll b/test/CodeGen/AMDGPU/global-extload-i16.ll
similarity index 100%
rename from test/CodeGen/R600/global-extload-i16.ll
rename to test/CodeGen/AMDGPU/global-extload-i16.ll
diff --git a/test/CodeGen/R600/global-extload-i32.ll b/test/CodeGen/AMDGPU/global-extload-i32.ll
similarity index 100%
rename from test/CodeGen/R600/global-extload-i32.ll
rename to test/CodeGen/AMDGPU/global-extload-i32.ll
diff --git a/test/CodeGen/R600/global-extload-i8.ll b/test/CodeGen/AMDGPU/global-extload-i8.ll
similarity index 100%
rename from test/CodeGen/R600/global-extload-i8.ll
rename to test/CodeGen/AMDGPU/global-extload-i8.ll
diff --git a/test/CodeGen/R600/global-zero-initializer.ll b/test/CodeGen/AMDGPU/global-zero-initializer.ll
similarity index 100%
rename from test/CodeGen/R600/global-zero-initializer.ll
rename to test/CodeGen/AMDGPU/global-zero-initializer.ll
diff --git a/test/CodeGen/AMDGPU/global_atomics.ll b/test/CodeGen/AMDGPU/global_atomics.ll
new file mode 100644
index 00000000000..146f0a5fbf2
--- /dev/null
+++ b/test/CodeGen/AMDGPU/global_atomics.ll
@@ -0,0 +1,921 @@
+; RUN: llc < %s -march=amdgcn -mcpu=SI -verify-machineinstrs | FileCheck --check-prefix=GCN --check-prefix=SI --check-prefix=FUNC %s
+; RUN: llc < %s -march=amdgcn -mcpu=tonga -verify-machineinstrs | FileCheck --check-prefix=GCN --check-prefix=VI --check-prefix=FUNC %s
+
+
+; FUNC-LABEL: {{^}}atomic_add_i32_offset:
+; GCN: buffer_atomic_add v{{[0-9]+}}, s[{{[0-9]+}}:{{[0-9]+}}], 0 offset:16{{$}}
+define void @atomic_add_i32_offset(i32 addrspace(1)* %out, i32 %in) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
+  %0  = atomicrmw volatile add i32 addrspace(1)* %gep, i32 %in seq_cst
+  ret void
+}
+
+; FUNC-LABEL: {{^}}atomic_add_i32_ret_offset:
+; GCN: buffer_atomic_add [[RET:v[0-9]+]], s[{{[0-9]+}}:{{[0-9]+}}], 0 offset:16 glc {{$}}
+; GCN: buffer_store_dword [[RET]]
+define void @atomic_add_i32_ret_offset(i32 addrspace(1)* %out, i32 addrspace(1)* %out2, i32 %in) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
+  %0  = atomicrmw volatile add i32 addrspace(1)* %gep, i32 %in seq_cst
+  store i32 %0, i32 addrspace(1)* %out2
+  ret void
+}
+
+; FUNC-LABEL: {{^}}atomic_add_i32_addr64_offset:
+; SI: buffer_atomic_add v{{[0-9]+}}, v[{{[0-9]+}}:{{[0-9]+}}], s[{{[0-9]+}}:{{[0-9]+}}], 0 addr64 offset:16{{$}}
+; VI: s_movk_i32 flat_scratch_lo, 0x0
+; VI: s_movk_i32 flat_scratch_hi, 0x0
+; VI: flat_atomic_add v[{{[0-9]+:[0-9]+}}], v{{[0-9]+$}}
+
+define void @atomic_add_i32_addr64_offset(i32 addrspace(1)* %out, i32 %in, i64 %index) {
+entry:
+  %ptr = getelementptr i32, i32 addrspace(1)* %out, i64 %index
+  %gep = getelementptr i32, i32 addrspace(1)* %ptr, i32 4
+  %0  = atomicrmw volatile add i32 addrspace(1)* %gep, i32 %in seq_cst
+  ret void
+}
+
+; FUNC-LABEL: {{^}}atomic_add_i32_ret_addr64_offset:
+; SI: buffer_atomic_add [[RET:v[0-9]+]], v[{{[0-9]+}}:{{[0-9]+}}], s[{{[0-9]+}}:{{[0-9]+}}], 0 addr64 offset:16 glc{{$}}
+; VI: s_movk_i32 flat_scratch_lo, 0x0
+; VI: s_movk_i32 flat_scratch_hi, 0x0
+; VI: flat_atomic_add [[RET:v[0-9]+]], v[{{[0-9]+:[0-9]+}}], v{{[0-9]+}} glc{{$}}
+; GCN: buffer_store_dword [[RET]]
+define void @atomic_add_i32_ret_addr64_offset(i32 addrspace(1)* %out, i32 addrspace(1)* %out2, i32 %in, i64 %index) {
+entry:
+  %ptr = getelementptr i32, i32 addrspace(1)* %out, i64 %index
+  %gep = getelementptr i32, i32 addrspace(1)* %ptr, i32 4
+  %0  = atomicrmw volatile add i32 addrspace(1)* %gep, i32 %in seq_cst
+  store i32 %0, i32 addrspace(1)* %out2
+  ret void
+}
+
+; FUNC-LABEL: {{^}}atomic_add_i32:
+; GCN: buffer_atomic_add v{{[0-9]+}}, s[{{[0-9]+}}:{{[0-9]+}}], 0{{$}}
+define void @atomic_add_i32(i32 addrspace(1)* %out, i32 %in) {
+entry:
+  %0  = atomicrmw volatile add i32 addrspace(1)* %out, i32 %in seq_cst
+  ret void
+}
+
+; FUNC-LABEL: {{^}}atomic_add_i32_ret:
+; GCN: buffer_atomic_add [[RET:v[0-9]+]], s[{{[0-9]+}}:{{[0-9]+}}], 0 glc
+; GCN: buffer_store_dword [[RET]]
+define void @atomic_add_i32_ret(i32 addrspace(1)* %out, i32 addrspace(1)* %out2, i32 %in) {
+entry:
+  %0  = atomicrmw volatile add i32 addrspace(1)* %out, i32 %in seq_cst
+  store i32 %0, i32 addrspace(1)* %out2
+  ret void
+}
+
+; FUNC-LABEL: {{^}}atomic_add_i32_addr64:
+; SI: buffer_atomic_add v{{[0-9]+}}, v[{{[0-9]+}}:{{[0-9]+}}], s[{{[0-9]+}}:{{[0-9]+}}], 0 addr64{{$}}
+; VI: s_movk_i32 flat_scratch_lo, 0x0
+; VI: s_movk_i32 flat_scratch_hi, 0x0
+; VI: flat_atomic_add v[{{[0-9]+:[0-9]+}}], v{{[0-9]+$}}
+define void @atomic_add_i32_addr64(i32 addrspace(1)* %out, i32 %in, i64 %index) {
+entry:
+  %ptr = getelementptr i32, i32 addrspace(1)* %out, i64 %index
+  %0  = atomicrmw volatile add i32 addrspace(1)* %ptr, i32 %in seq_cst
+  ret void
+}
+
+; FUNC-LABEL: {{^}}atomic_add_i32_ret_addr64:
+; SI: buffer_atomic_add [[RET:v[0-9]+]], v[{{[0-9]+}}:{{[0-9]+}}], s[{{[0-9]+}}:{{[0-9]+}}], 0 addr64 glc{{$}}
+; VI: s_movk_i32 flat_scratch_lo, 0x0
+; VI: s_movk_i32 flat_scratch_hi, 0x0
+; VI: flat_atomic_add [[RET:v[0-9]+]], v[{{[0-9]+:[0-9]+}}], v{{[0-9]+}} glc{{$}}
+; GCN: buffer_store_dword [[RET]]
+define void @atomic_add_i32_ret_addr64(i32 addrspace(1)* %out, i32 addrspace(1)* %out2, i32 %in, i64 %index) {
+entry:
+  %ptr = getelementptr i32, i32 addrspace(1)* %out, i64 %index
+  %0  = atomicrmw volatile add i32 addrspace(1)* %ptr, i32 %in seq_cst
+  store i32 %0, i32 addrspace(1)* %out2
+  ret void
+}
+
+; FUNC-LABEL: {{^}}atomic_and_i32_offset:
+; GCN: buffer_atomic_and v{{[0-9]+}}, s[{{[0-9]+}}:{{[0-9]+}}], 0 offset:16{{$}}
+define void @atomic_and_i32_offset(i32 addrspace(1)* %out, i32 %in) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
+  %0  = atomicrmw volatile and i32 addrspace(1)* %gep, i32 %in seq_cst
+  ret void
+}
+
+; FUNC-LABEL: {{^}}atomic_and_i32_ret_offset:
+; GCN: buffer_atomic_and [[RET:v[0-9]+]], s[{{[0-9]+}}:{{[0-9]+}}], 0 offset:16 glc {{$}}
+; GCN: buffer_store_dword [[RET]]
+define void @atomic_and_i32_ret_offset(i32 addrspace(1)* %out, i32 addrspace(1)* %out2, i32 %in) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
+  %0  = atomicrmw volatile and i32 addrspace(1)* %gep, i32 %in seq_cst
+  store i32 %0, i32 addrspace(1)* %out2
+  ret void
+}
+
+; FUNC-LABEL: {{^}}atomic_and_i32_addr64_offset:
+; SI: buffer_atomic_and v{{[0-9]+}}, v[{{[0-9]+}}:{{[0-9]+}}], s[{{[0-9]+}}:{{[0-9]+}}], 0 addr64 offset:16{{$}}
+; VI: s_movk_i32 flat_scratch_lo, 0x0
+; VI: s_movk_i32 flat_scratch_hi, 0x0
+; VI: flat_atomic_and v[{{[0-9]+:[0-9]+}}], v{{[0-9]+$}}
+define void @atomic_and_i32_addr64_offset(i32 addrspace(1)* %out, i32 %in, i64 %index) {
+entry:
+  %ptr = getelementptr i32, i32 addrspace(1)* %out, i64 %index
+  %gep = getelementptr i32, i32 addrspace(1)* %ptr, i32 4
+  %0  = atomicrmw volatile and i32 addrspace(1)* %gep, i32 %in seq_cst
+  ret void
+}
+
+; FUNC-LABEL: {{^}}atomic_and_i32_ret_addr64_offset:
+; SI: buffer_atomic_and [[RET:v[0-9]+]], v[{{[0-9]+}}:{{[0-9]+}}], s[{{[0-9]+}}:{{[0-9]+}}], 0 addr64 offset:16 glc{{$}}
+; VI: s_movk_i32 flat_scratch_lo, 0x0
+; VI: s_movk_i32 flat_scratch_hi, 0x0
+; VI: flat_atomic_and [[RET:v[0-9]]], v[{{[0-9]+:[0-9]+}}], v{{[0-9]+}} glc{{$}}
+; GCN: buffer_store_dword [[RET]]
+define void @atomic_and_i32_ret_addr64_offset(i32 addrspace(1)* %out, i32 addrspace(1)* %out2, i32 %in, i64 %index) {
+entry:
+  %ptr = getelementptr i32, i32 addrspace(1)* %out, i64 %index
+  %gep = getelementptr i32, i32 addrspace(1)* %ptr, i32 4
+  %0  = atomicrmw volatile and i32 addrspace(1)* %gep, i32 %in seq_cst
+  store i32 %0, i32 addrspace(1)* %out2
+  ret void
+}
+
+; FUNC-LABEL: {{^}}atomic_and_i32:
+; GCN: buffer_atomic_and v{{[0-9]+}}, s[{{[0-9]+}}:{{[0-9]+}}], 0{{$}}
+define void @atomic_and_i32(i32 addrspace(1)* %out, i32 %in) {
+entry:
+  %0  = atomicrmw volatile and i32 addrspace(1)* %out, i32 %in seq_cst
+  ret void
+}
+
+; FUNC-LABEL: {{^}}atomic_and_i32_ret:
+; GCN: buffer_atomic_and [[RET:v[0-9]+]], s[{{[0-9]+}}:{{[0-9]+}}], 0 glc
+; GCN: buffer_store_dword [[RET]]
+define void @atomic_and_i32_ret(i32 addrspace(1)* %out, i32 addrspace(1)* %out2, i32 %in) {
+entry:
+  %0  = atomicrmw volatile and i32 addrspace(1)* %out, i32 %in seq_cst
+  store i32 %0, i32 addrspace(1)* %out2
+  ret void
+}
+
+; FUNC-LABEL: {{^}}atomic_and_i32_addr64:
+; SI: buffer_atomic_and v{{[0-9]+}}, v[{{[0-9]+}}:{{[0-9]+}}], s[{{[0-9]+}}:{{[0-9]+}}], 0 addr64{{$}}
+; VI: s_movk_i32 flat_scratch_lo, 0x0
+; VI: s_movk_i32 flat_scratch_hi, 0x0
+; VI: flat_atomic_and v[{{[0-9]+:[0-9]+}}], v{{[0-9]+$}}
+define void @atomic_and_i32_addr64(i32 addrspace(1)* %out, i32 %in, i64 %index) {
+entry:
+  %ptr = getelementptr i32, i32 addrspace(1)* %out, i64 %index
+  %0  = atomicrmw volatile and i32 addrspace(1)* %ptr, i32 %in seq_cst
+  ret void
+}
+
+; FUNC-LABEL: {{^}}atomic_and_i32_ret_addr64:
+; SI: buffer_atomic_and [[RET:v[0-9]+]], v[{{[0-9]+}}:{{[0-9]+}}], s[{{[0-9]+}}:{{[0-9]+}}], 0 addr64 glc{{$}}
+; VI: s_movk_i32 flat_scratch_lo, 0x0
+; VI: s_movk_i32 flat_scratch_hi, 0x0
+; VI: flat_atomic_and [[RET:v[0-9]+]], v[{{[0-9]+:[0-9]+}}], v{{[0-9]+}} glc{{$}}
+; GCN: buffer_store_dword [[RET]]
+define void @atomic_and_i32_ret_addr64(i32 addrspace(1)* %out, i32 addrspace(1)* %out2, i32 %in, i64 %index) {
+entry:
+  %ptr = getelementptr i32, i32 addrspace(1)* %out, i64 %index
+  %0  = atomicrmw volatile and i32 addrspace(1)* %ptr, i32 %in seq_cst
+  store i32 %0, i32 addrspace(1)* %out2
+  ret void
+}
+
+; FUNC-LABEL: {{^}}atomic_sub_i32_offset:
+; GCN: buffer_atomic_sub v{{[0-9]+}}, s[{{[0-9]+}}:{{[0-9]+}}], 0 offset:16{{$}}
+define void @atomic_sub_i32_offset(i32 addrspace(1)* %out, i32 %in) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
+  %0  = atomicrmw volatile sub i32 addrspace(1)* %gep, i32 %in seq_cst
+  ret void
+}
+
+; FUNC-LABEL: {{^}}atomic_sub_i32_ret_offset:
+; GCN: buffer_atomic_sub [[RET:v[0-9]+]], s[{{[0-9]+}}:{{[0-9]+}}], 0 offset:16 glc {{$}}
+; GCN: buffer_store_dword [[RET]]
+define void @atomic_sub_i32_ret_offset(i32 addrspace(1)* %out, i32 addrspace(1)* %out2, i32 %in) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
+  %0  = atomicrmw volatile sub i32 addrspace(1)* %gep, i32 %in seq_cst
+  store i32 %0, i32 addrspace(1)* %out2
+  ret void
+}
+
+; FUNC-LABEL: {{^}}atomic_sub_i32_addr64_offset:
+; SI: buffer_atomic_sub v{{[0-9]+}}, v[{{[0-9]+}}:{{[0-9]+}}], s[{{[0-9]+}}:{{[0-9]+}}], 0 addr64 offset:16{{$}}
+; VI: s_movk_i32 flat_scratch_lo, 0x0
+; VI: s_movk_i32 flat_scratch_hi, 0x0
+; VI: flat_atomic_sub v[{{[0-9]+:[0-9]+}}], v{{[0-9]+$}}
+define void @atomic_sub_i32_addr64_offset(i32 addrspace(1)* %out, i32 %in, i64 %index) {
+entry:
+  %ptr = getelementptr i32, i32 addrspace(1)* %out, i64 %index
+  %gep = getelementptr i32, i32 addrspace(1)* %ptr, i32 4
+  %0  = atomicrmw volatile sub i32 addrspace(1)* %gep, i32 %in seq_cst
+  ret void
+}
+
+; FUNC-LABEL: {{^}}atomic_sub_i32_ret_addr64_offset:
+; SI: buffer_atomic_sub [[RET:v[0-9]+]], v[{{[0-9]+}}:{{[0-9]+}}], s[{{[0-9]+}}:{{[0-9]+}}], 0 addr64 offset:16 glc{{$}}
+; VI: s_movk_i32 flat_scratch_lo, 0x0
+; VI: s_movk_i32 flat_scratch_hi, 0x0
+; VI: flat_atomic_sub [[RET:v[0-9]+]], v[{{[0-9]+:[0-9]+}}], v{{[0-9]+}} glc{{$}}
+; GCN: buffer_store_dword [[RET]]
+define void @atomic_sub_i32_ret_addr64_offset(i32 addrspace(1)* %out, i32 addrspace(1)* %out2, i32 %in, i64 %index) {
+entry:
+  %ptr = getelementptr i32, i32 addrspace(1)* %out, i64 %index
+  %gep = getelementptr i32, i32 addrspace(1)* %ptr, i32 4
+  %0  = atomicrmw volatile sub i32 addrspace(1)* %gep, i32 %in seq_cst
+  store i32 %0, i32 addrspace(1)* %out2
+  ret void
+}
+
+; FUNC-LABEL: {{^}}atomic_sub_i32:
+; GCN: buffer_atomic_sub v{{[0-9]+}}, s[{{[0-9]+}}:{{[0-9]+}}], 0{{$}}
+define void @atomic_sub_i32(i32 addrspace(1)* %out, i32 %in) {
+entry:
+  %0  = atomicrmw volatile sub i32 addrspace(1)* %out, i32 %in seq_cst
+  ret void
+}
+
+; FUNC-LABEL: {{^}}atomic_sub_i32_ret:
+; GCN: buffer_atomic_sub [[RET:v[0-9]+]], s[{{[0-9]+}}:{{[0-9]+}}], 0 glc
+; GCN: buffer_store_dword [[RET]]
+define void @atomic_sub_i32_ret(i32 addrspace(1)* %out, i32 addrspace(1)* %out2, i32 %in) {
+entry:
+  %0  = atomicrmw volatile sub i32 addrspace(1)* %out, i32 %in seq_cst
+  store i32 %0, i32 addrspace(1)* %out2
+  ret void
+}
+
+; FUNC-LABEL: {{^}}atomic_sub_i32_addr64:
+; SI: buffer_atomic_sub v{{[0-9]+}}, v[{{[0-9]+}}:{{[0-9]+}}], s[{{[0-9]+}}:{{[0-9]+}}], 0 addr64{{$}}
+; VI: s_movk_i32 flat_scratch_lo, 0x0
+; VI: s_movk_i32 flat_scratch_hi, 0x0
+; VI: flat_atomic_sub v[{{[0-9]+:[0-9]+}}], v{{[0-9]+$}}
+define void @atomic_sub_i32_addr64(i32 addrspace(1)* %out, i32 %in, i64 %index) {
+entry:
+  %ptr = getelementptr i32, i32 addrspace(1)* %out, i64 %index
+  %0  = atomicrmw volatile sub i32 addrspace(1)* %ptr, i32 %in seq_cst
+  ret void
+}
+
+; FUNC-LABEL: {{^}}atomic_sub_i32_ret_addr64:
+; SI: buffer_atomic_sub [[RET:v[0-9]+]], v[{{[0-9]+}}:{{[0-9]+}}], s[{{[0-9]+}}:{{[0-9]+}}], 0 addr64 glc{{$}}
+; VI: s_movk_i32 flat_scratch_lo, 0x0
+; VI: s_movk_i32 flat_scratch_hi, 0x0
+; VI: flat_atomic_sub [[RET:v[0-9]+]], v[{{[0-9]+:[0-9]+}}], v{{[0-9]+}} glc{{$}}
+; GCN: buffer_store_dword [[RET]]
+define void @atomic_sub_i32_ret_addr64(i32 addrspace(1)* %out, i32 addrspace(1)* %out2, i32 %in, i64 %index) {
+entry:
+  %ptr = getelementptr i32, i32 addrspace(1)* %out, i64 %index
+  %0  = atomicrmw volatile sub i32 addrspace(1)* %ptr, i32 %in seq_cst
+  store i32 %0, i32 addrspace(1)* %out2
+  ret void
+}
+
+; FUNC-LABEL: {{^}}atomic_max_i32_offset:
+; GCN: buffer_atomic_smax v{{[0-9]+}}, s[{{[0-9]+}}:{{[0-9]+}}], 0 offset:16{{$}}
+define void @atomic_max_i32_offset(i32 addrspace(1)* %out, i32 %in) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
+  %0  = atomicrmw volatile max i32 addrspace(1)* %gep, i32 %in seq_cst
+  ret void
+}
+
+; FUNC-LABEL: {{^}}atomic_max_i32_ret_offset:
+; GCN: buffer_atomic_smax [[RET:v[0-9]+]], s[{{[0-9]+}}:{{[0-9]+}}], 0 offset:16 glc {{$}}
+; GCN: buffer_store_dword [[RET]]
+define void @atomic_max_i32_ret_offset(i32 addrspace(1)* %out, i32 addrspace(1)* %out2, i32 %in) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
+  %0  = atomicrmw volatile max i32 addrspace(1)* %gep, i32 %in seq_cst
+  store i32 %0, i32 addrspace(1)* %out2
+  ret void
+}
+
+; FUNC-LABEL: {{^}}atomic_max_i32_addr64_offset:
+; SI: buffer_atomic_smax v{{[0-9]+}}, v[{{[0-9]+}}:{{[0-9]+}}], s[{{[0-9]+}}:{{[0-9]+}}], 0 addr64 offset:16{{$}}
+; VI: s_movk_i32 flat_scratch_lo, 0x0
+; VI: s_movk_i32 flat_scratch_hi, 0x0
+; VI: flat_atomic_smax v[{{[0-9]+:[0-9]+}}], v{{[0-9]+$}}
+define void @atomic_max_i32_addr64_offset(i32 addrspace(1)* %out, i32 %in, i64 %index) {
+entry:
+  %ptr = getelementptr i32, i32 addrspace(1)* %out, i64 %index
+  %gep = getelementptr i32, i32 addrspace(1)* %ptr, i32 4
+  %0  = atomicrmw volatile max i32 addrspace(1)* %gep, i32 %in seq_cst
+  ret void
+}
+
+; FUNC-LABEL: {{^}}atomic_max_i32_ret_addr64_offset:
+; SI: buffer_atomic_smax [[RET:v[0-9]+]], v[{{[0-9]+}}:{{[0-9]+}}], s[{{[0-9]+}}:{{[0-9]+}}], 0 addr64 offset:16 glc{{$}}
+; VI: s_movk_i32 flat_scratch_lo, 0x0
+; VI: s_movk_i32 flat_scratch_hi, 0x0
+; VI: flat_atomic_smax [[RET:v[0-9]+]], v[{{[0-9]+:[0-9]+}}], v{{[0-9]+}} glc{{$}}
+; GCN: buffer_store_dword [[RET]]
+define void @atomic_max_i32_ret_addr64_offset(i32 addrspace(1)* %out, i32 addrspace(1)* %out2, i32 %in, i64 %index) {
+entry:
+  %ptr = getelementptr i32, i32 addrspace(1)* %out, i64 %index
+  %gep = getelementptr i32, i32 addrspace(1)* %ptr, i32 4
+  %0  = atomicrmw volatile max i32 addrspace(1)* %gep, i32 %in seq_cst
+  store i32 %0, i32 addrspace(1)* %out2
+  ret void
+}
+
+; FUNC-LABEL: {{^}}atomic_max_i32:
+; GCN: buffer_atomic_smax v{{[0-9]+}}, s[{{[0-9]+}}:{{[0-9]+}}], 0{{$}}
+define void @atomic_max_i32(i32 addrspace(1)* %out, i32 %in) {
+entry:
+  %0  = atomicrmw volatile max i32 addrspace(1)* %out, i32 %in seq_cst
+  ret void
+}
+
+; FUNC-LABEL: {{^}}atomic_max_i32_ret:
+; GCN: buffer_atomic_smax [[RET:v[0-9]+]], s[{{[0-9]+}}:{{[0-9]+}}], 0 glc
+; GCN: buffer_store_dword [[RET]]
+define void @atomic_max_i32_ret(i32 addrspace(1)* %out, i32 addrspace(1)* %out2, i32 %in) {
+entry:
+  %0  = atomicrmw volatile max i32 addrspace(1)* %out, i32 %in seq_cst
+  store i32 %0, i32 addrspace(1)* %out2
+  ret void
+}
+
+; FUNC-LABEL: {{^}}atomic_max_i32_addr64:
+; SI: buffer_atomic_smax v{{[0-9]+}}, v[{{[0-9]+}}:{{[0-9]+}}], s[{{[0-9]+}}:{{[0-9]+}}], 0 addr64{{$}}
+; VI: s_movk_i32 flat_scratch_lo, 0x0
+; VI: s_movk_i32 flat_scratch_hi, 0x0
+; VI: flat_atomic_smax v[{{[0-9]+:[0-9]+}}], v{{[0-9]+$}}
+define void @atomic_max_i32_addr64(i32 addrspace(1)* %out, i32 %in, i64 %index) {
+entry:
+  %ptr = getelementptr i32, i32 addrspace(1)* %out, i64 %index
+  %0  = atomicrmw volatile max i32 addrspace(1)* %ptr, i32 %in seq_cst
+  ret void
+}
+
+; FUNC-LABEL: {{^}}atomic_max_i32_ret_addr64:
+; SI: buffer_atomic_smax [[RET:v[0-9]+]], v[{{[0-9]+}}:{{[0-9]+}}], s[{{[0-9]+}}:{{[0-9]+}}], 0 addr64 glc{{$}}
+; VI: s_movk_i32 flat_scratch_lo, 0x0
+; VI: s_movk_i32 flat_scratch_hi, 0x0
+; VI: flat_atomic_smax [[RET:v[0-9]+]], v[{{[0-9]+:[0-9]+}}], v{{[0-9]+}} glc{{$}}
+; GCN: buffer_store_dword [[RET]]
+define void @atomic_max_i32_ret_addr64(i32 addrspace(1)* %out, i32 addrspace(1)* %out2, i32 %in, i64 %index) {
+entry:
+  %ptr = getelementptr i32, i32 addrspace(1)* %out, i64 %index
+  %0  = atomicrmw volatile max i32 addrspace(1)* %ptr, i32 %in seq_cst
+  store i32 %0, i32 addrspace(1)* %out2
+  ret void
+}
+
+; FUNC-LABEL: {{^}}atomic_umax_i32_offset:
+; GCN: buffer_atomic_umax v{{[0-9]+}}, s[{{[0-9]+}}:{{[0-9]+}}], 0 offset:16{{$}}
+define void @atomic_umax_i32_offset(i32 addrspace(1)* %out, i32 %in) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
+  %0  = atomicrmw volatile umax i32 addrspace(1)* %gep, i32 %in seq_cst
+  ret void
+}
+
+; FUNC-LABEL: {{^}}atomic_umax_i32_ret_offset:
+; GCN: buffer_atomic_umax [[RET:v[0-9]+]], s[{{[0-9]+}}:{{[0-9]+}}], 0 offset:16 glc {{$}}
+; GCN: buffer_store_dword [[RET]]
+define void @atomic_umax_i32_ret_offset(i32 addrspace(1)* %out, i32 addrspace(1)* %out2, i32 %in) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
+  %0  = atomicrmw volatile umax i32 addrspace(1)* %gep, i32 %in seq_cst
+  store i32 %0, i32 addrspace(1)* %out2
+  ret void
+}
+
+; FUNC-LABEL: {{^}}atomic_umax_i32_addr64_offset:
+; SI: buffer_atomic_umax v{{[0-9]+}}, v[{{[0-9]+}}:{{[0-9]+}}], s[{{[0-9]+}}:{{[0-9]+}}], 0 addr64 offset:16{{$}}
+; VI: s_movk_i32 flat_scratch_lo, 0x0
+; VI: s_movk_i32 flat_scratch_hi, 0x0
+; VI: flat_atomic_umax v[{{[0-9]+:[0-9]+}}], v{{[0-9]+$}}
+define void @atomic_umax_i32_addr64_offset(i32 addrspace(1)* %out, i32 %in, i64 %index) {
+entry:
+  %ptr = getelementptr i32, i32 addrspace(1)* %out, i64 %index
+  %gep = getelementptr i32, i32 addrspace(1)* %ptr, i32 4
+  %0  = atomicrmw volatile umax i32 addrspace(1)* %gep, i32 %in seq_cst
+  ret void
+}
+
+; FUNC-LABEL: {{^}}atomic_umax_i32_ret_addr64_offset:
+; SI: buffer_atomic_umax [[RET:v[0-9]+]], v[{{[0-9]+}}:{{[0-9]+}}], s[{{[0-9]+}}:{{[0-9]+}}], 0 addr64 offset:16 glc{{$}}
+; VI: s_movk_i32 flat_scratch_lo, 0x0
+; VI: s_movk_i32 flat_scratch_hi, 0x0
+; VI: flat_atomic_umax [[RET:v[0-9]+]], v[{{[0-9]+:[0-9]+}}], v{{[0-9]+}} glc{{$}}
+; GCN: buffer_store_dword [[RET]]
+define void @atomic_umax_i32_ret_addr64_offset(i32 addrspace(1)* %out, i32 addrspace(1)* %out2, i32 %in, i64 %index) {
+entry:
+  %ptr = getelementptr i32, i32 addrspace(1)* %out, i64 %index
+  %gep = getelementptr i32, i32 addrspace(1)* %ptr, i32 4
+  %0  = atomicrmw volatile umax i32 addrspace(1)* %gep, i32 %in seq_cst
+  store i32 %0, i32 addrspace(1)* %out2
+  ret void
+}
+
+; FUNC-LABEL: {{^}}atomic_umax_i32:
+; GCN: buffer_atomic_umax v{{[0-9]+}}, s[{{[0-9]+}}:{{[0-9]+}}], 0{{$}}
+define void @atomic_umax_i32(i32 addrspace(1)* %out, i32 %in) {
+entry:
+  %0  = atomicrmw volatile umax i32 addrspace(1)* %out, i32 %in seq_cst
+  ret void
+}
+
+; FUNC-LABEL: {{^}}atomic_umax_i32_ret:
+; GCN: buffer_atomic_umax [[RET:v[0-9]+]], s[{{[0-9]+}}:{{[0-9]+}}], 0 glc
+; GCN: buffer_store_dword [[RET]]
+define void @atomic_umax_i32_ret(i32 addrspace(1)* %out, i32 addrspace(1)* %out2, i32 %in) {
+entry:
+  %0  = atomicrmw volatile umax i32 addrspace(1)* %out, i32 %in seq_cst
+  store i32 %0, i32 addrspace(1)* %out2
+  ret void
+}
+
+; FUNC-LABEL: {{^}}atomic_umax_i32_addr64:
+; SI: buffer_atomic_umax v{{[0-9]+}}, v[{{[0-9]+}}:{{[0-9]+}}], s[{{[0-9]+}}:{{[0-9]+}}], 0 addr64{{$}}
+; VI: s_movk_i32 flat_scratch_lo, 0x0
+; VI: s_movk_i32 flat_scratch_hi, 0x0
+; VI: flat_atomic_umax v[{{[0-9]+:[0-9]+}}], v{{[0-9]+$}}
+define void @atomic_umax_i32_addr64(i32 addrspace(1)* %out, i32 %in, i64 %index) {
+entry:
+  %ptr = getelementptr i32, i32 addrspace(1)* %out, i64 %index
+  %0  = atomicrmw volatile umax i32 addrspace(1)* %ptr, i32 %in seq_cst
+  ret void
+}
+
+; FUNC-LABEL: {{^}}atomic_umax_i32_ret_addr64:
+; SI: buffer_atomic_umax [[RET:v[0-9]+]], v[{{[0-9]+}}:{{[0-9]+}}], s[{{[0-9]+}}:{{[0-9]+}}], 0 addr64 glc{{$}}
+; VI: s_movk_i32 flat_scratch_lo, 0x0
+; VI: s_movk_i32 flat_scratch_hi, 0x0
+; VI: flat_atomic_umax [[RET:v[0-9]+]], v[{{[0-9]+:[0-9]+}}], v{{[0-9]+}} glc{{$}}
+; GCN: buffer_store_dword [[RET]]
+define void @atomic_umax_i32_ret_addr64(i32 addrspace(1)* %out, i32 addrspace(1)* %out2, i32 %in, i64 %index) {
+entry:
+  %ptr = getelementptr i32, i32 addrspace(1)* %out, i64 %index
+  %0  = atomicrmw volatile umax i32 addrspace(1)* %ptr, i32 %in seq_cst
+  store i32 %0, i32 addrspace(1)* %out2
+  ret void
+}
+
+; FUNC-LABEL: {{^}}atomic_min_i32_offset:
+; GCN: buffer_atomic_smin v{{[0-9]+}}, s[{{[0-9]+}}:{{[0-9]+}}], 0 offset:16{{$}}
+define void @atomic_min_i32_offset(i32 addrspace(1)* %out, i32 %in) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
+  %0  = atomicrmw volatile min i32 addrspace(1)* %gep, i32 %in seq_cst
+  ret void
+}
+
+; FUNC-LABEL: {{^}}atomic_min_i32_ret_offset:
+; GCN: buffer_atomic_smin [[RET:v[0-9]+]], s[{{[0-9]+}}:{{[0-9]+}}], 0 offset:16 glc {{$}}
+; GCN: buffer_store_dword [[RET]]
+define void @atomic_min_i32_ret_offset(i32 addrspace(1)* %out, i32 addrspace(1)* %out2, i32 %in) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
+  %0  = atomicrmw volatile min i32 addrspace(1)* %gep, i32 %in seq_cst
+  store i32 %0, i32 addrspace(1)* %out2
+  ret void
+}
+
+; FUNC-LABEL: {{^}}atomic_min_i32_addr64_offset:
+; SI: buffer_atomic_smin v{{[0-9]+}}, v[{{[0-9]+}}:{{[0-9]+}}], s[{{[0-9]+}}:{{[0-9]+}}], 0 addr64 offset:16{{$}}
+; VI: s_movk_i32 flat_scratch_lo, 0x0
+; VI: s_movk_i32 flat_scratch_hi, 0x0
+; VI: flat_atomic_smin v[{{[0-9]+:[0-9]+}}], v{{[0-9]+$}}
+define void @atomic_min_i32_addr64_offset(i32 addrspace(1)* %out, i32 %in, i64 %index) {
+entry:
+  %ptr = getelementptr i32, i32 addrspace(1)* %out, i64 %index
+  %gep = getelementptr i32, i32 addrspace(1)* %ptr, i32 4
+  %0  = atomicrmw volatile min i32 addrspace(1)* %gep, i32 %in seq_cst
+  ret void
+}
+
+; FUNC-LABEL: {{^}}atomic_min_i32_ret_addr64_offset:
+; SI: buffer_atomic_smin [[RET:v[0-9]+]], v[{{[0-9]+}}:{{[0-9]+}}], s[{{[0-9]+}}:{{[0-9]+}}], 0 addr64 offset:16 glc{{$}}
+; VI: s_movk_i32 flat_scratch_lo, 0x0
+; VI: s_movk_i32 flat_scratch_hi, 0x0
+; VI: flat_atomic_smin [[RET:v[0-9]+]], v[{{[0-9]+:[0-9]+}}], v{{[0-9]+}} glc{{$}}
+; GCN: buffer_store_dword [[RET]]
+define void @atomic_min_i32_ret_addr64_offset(i32 addrspace(1)* %out, i32 addrspace(1)* %out2, i32 %in, i64 %index) {
+entry:
+  %ptr = getelementptr i32, i32 addrspace(1)* %out, i64 %index
+  %gep = getelementptr i32, i32 addrspace(1)* %ptr, i32 4
+  %0  = atomicrmw volatile min i32 addrspace(1)* %gep, i32 %in seq_cst
+  store i32 %0, i32 addrspace(1)* %out2
+  ret void
+}
+
+; FUNC-LABEL: {{^}}atomic_min_i32:
+; GCN: buffer_atomic_smin v{{[0-9]+}}, s[{{[0-9]+}}:{{[0-9]+}}], 0{{$}}
+define void @atomic_min_i32(i32 addrspace(1)* %out, i32 %in) {
+entry:
+  %0  = atomicrmw volatile min i32 addrspace(1)* %out, i32 %in seq_cst
+  ret void
+}
+
+; FUNC-LABEL: {{^}}atomic_min_i32_ret:
+; GCN: buffer_atomic_smin [[RET:v[0-9]+]], s[{{[0-9]+}}:{{[0-9]+}}], 0 glc
+; GCN: buffer_store_dword [[RET]]
+define void @atomic_min_i32_ret(i32 addrspace(1)* %out, i32 addrspace(1)* %out2, i32 %in) {
+entry:
+  %0  = atomicrmw volatile min i32 addrspace(1)* %out, i32 %in seq_cst
+  store i32 %0, i32 addrspace(1)* %out2
+  ret void
+}
+
+; FUNC-LABEL: {{^}}atomic_min_i32_addr64:
+; SI: buffer_atomic_smin v{{[0-9]+}}, v[{{[0-9]+}}:{{[0-9]+}}], s[{{[0-9]+}}:{{[0-9]+}}], 0 addr64{{$}}
+; VI: s_movk_i32 flat_scratch_lo, 0x0
+; VI: s_movk_i32 flat_scratch_hi, 0x0
+; VI: flat_atomic_smin v[{{[0-9]+:[0-9]+}}], v{{[0-9]+$}}
+define void @atomic_min_i32_addr64(i32 addrspace(1)* %out, i32 %in, i64 %index) {
+entry:
+  %ptr = getelementptr i32, i32 addrspace(1)* %out, i64 %index
+  %0  = atomicrmw volatile min i32 addrspace(1)* %ptr, i32 %in seq_cst
+  ret void
+}
+
+; FUNC-LABEL: {{^}}atomic_min_i32_ret_addr64:
+; SI: buffer_atomic_smin [[RET:v[0-9]+]], v[{{[0-9]+}}:{{[0-9]+}}], s[{{[0-9]+}}:{{[0-9]+}}], 0 addr64 glc{{$}}
+; VI: s_movk_i32 flat_scratch_lo, 0x0
+; VI: s_movk_i32 flat_scratch_hi, 0x0
+; VI: flat_atomic_smin [[RET:v[0-9]+]], v[{{[0-9]+:[0-9]+}}], v{{[0-9]+}} glc{{$}}
+; GCN: buffer_store_dword [[RET]]
+define void @atomic_min_i32_ret_addr64(i32 addrspace(1)* %out, i32 addrspace(1)* %out2, i32 %in, i64 %index) {
+entry:
+  %ptr = getelementptr i32, i32 addrspace(1)* %out, i64 %index
+  %0  = atomicrmw volatile min i32 addrspace(1)* %ptr, i32 %in seq_cst
+  store i32 %0, i32 addrspace(1)* %out2
+  ret void
+}
+
+; FUNC-LABEL: {{^}}atomic_umin_i32_offset:
+; GCN: buffer_atomic_umin v{{[0-9]+}}, s[{{[0-9]+}}:{{[0-9]+}}], 0 offset:16{{$}}
+define void @atomic_umin_i32_offset(i32 addrspace(1)* %out, i32 %in) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
+  %0  = atomicrmw volatile umin i32 addrspace(1)* %gep, i32 %in seq_cst
+  ret void
+}
+
+; FUNC-LABEL: {{^}}atomic_umin_i32_ret_offset:
+; GCN: buffer_atomic_umin [[RET:v[0-9]+]], s[{{[0-9]+}}:{{[0-9]+}}], 0 offset:16 glc {{$}}
+; GCN: buffer_store_dword [[RET]]
+define void @atomic_umin_i32_ret_offset(i32 addrspace(1)* %out, i32 addrspace(1)* %out2, i32 %in) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
+  %0  = atomicrmw volatile umin i32 addrspace(1)* %gep, i32 %in seq_cst
+  store i32 %0, i32 addrspace(1)* %out2
+  ret void
+}
+
+; FUNC-LABEL: {{^}}atomic_umin_i32_addr64_offset:
+; SI: buffer_atomic_umin v{{[0-9]+}}, v[{{[0-9]+}}:{{[0-9]+}}], s[{{[0-9]+}}:{{[0-9]+}}], 0 addr64 offset:16{{$}}
+; VI: s_movk_i32 flat_scratch_lo, 0x0
+; VI: s_movk_i32 flat_scratch_hi, 0x0
+; VI: flat_atomic_umin v[{{[0-9]+:[0-9]+}}], v{{[0-9]+$}}
+define void @atomic_umin_i32_addr64_offset(i32 addrspace(1)* %out, i32 %in, i64 %index) {
+entry:
+  %ptr = getelementptr i32, i32 addrspace(1)* %out, i64 %index
+  %gep = getelementptr i32, i32 addrspace(1)* %ptr, i32 4
+  %0  = atomicrmw volatile umin i32 addrspace(1)* %gep, i32 %in seq_cst
+  ret void
+}
+
+; FUNC-LABEL: {{^}}atomic_umin_i32_ret_addr64_offset:
+; SI: buffer_atomic_umin [[RET:v[0-9]+]], v[{{[0-9]+}}:{{[0-9]+}}], s[{{[0-9]+}}:{{[0-9]+}}], 0 addr64 offset:16 glc{{$}}
+; VI: s_movk_i32 flat_scratch_lo, 0x0
+; VI: s_movk_i32 flat_scratch_hi, 0x0
+; VI: flat_atomic_umin [[RET:v[0-9]+]], v[{{[0-9]+:[0-9]+}}], v{{[0-9]+}} glc{{$}}
+; GCN: buffer_store_dword [[RET]]
+define void @atomic_umin_i32_ret_addr64_offset(i32 addrspace(1)* %out, i32 addrspace(1)* %out2, i32 %in, i64 %index) {
+entry:
+  %ptr = getelementptr i32, i32 addrspace(1)* %out, i64 %index
+  %gep = getelementptr i32, i32 addrspace(1)* %ptr, i32 4
+  %0  = atomicrmw volatile umin i32 addrspace(1)* %gep, i32 %in seq_cst
+  store i32 %0, i32 addrspace(1)* %out2
+  ret void
+}
+
+; FUNC-LABEL: {{^}}atomic_umin_i32:
+; GCN: buffer_atomic_umin v{{[0-9]+}}, s[{{[0-9]+}}:{{[0-9]+}}], 0{{$}}
+define void @atomic_umin_i32(i32 addrspace(1)* %out, i32 %in) {
+entry:
+  %0  = atomicrmw volatile umin i32 addrspace(1)* %out, i32 %in seq_cst
+  ret void
+}
+
+; FUNC-LABEL: {{^}}atomic_umin_i32_ret:
+; SI: buffer_atomic_umin [[RET:v[0-9]+]], s[{{[0-9]+}}:{{[0-9]+}}], 0 glc
+; GCN: buffer_store_dword [[RET]]
+define void @atomic_umin_i32_ret(i32 addrspace(1)* %out, i32 addrspace(1)* %out2, i32 %in) {
+entry:
+  %0  = atomicrmw volatile umin i32 addrspace(1)* %out, i32 %in seq_cst
+  store i32 %0, i32 addrspace(1)* %out2
+  ret void
+}
+
+; FUNC-LABEL: {{^}}atomic_umin_i32_addr64:
+; SI: buffer_atomic_umin v{{[0-9]+}}, v[{{[0-9]+}}:{{[0-9]+}}], s[{{[0-9]+}}:{{[0-9]+}}], 0 addr64{{$}}
+; VI: s_movk_i32 flat_scratch_lo, 0x0
+; VI: s_movk_i32 flat_scratch_hi, 0x0
+; VI: flat_atomic_umin v[{{[0-9]+:[0-9]+}}], v{{[0-9]+$}}
+define void @atomic_umin_i32_addr64(i32 addrspace(1)* %out, i32 %in, i64 %index) {
+entry:
+  %ptr = getelementptr i32, i32 addrspace(1)* %out, i64 %index
+  %0  = atomicrmw volatile umin i32 addrspace(1)* %ptr, i32 %in seq_cst
+  ret void
+}
+
+; FUNC-LABEL: {{^}}atomic_umin_i32_ret_addr64:
+; SI: buffer_atomic_umin [[RET:v[0-9]+]], v[{{[0-9]+}}:{{[0-9]+}}], s[{{[0-9]+}}:{{[0-9]+}}], 0 addr64 glc{{$}}
+; VI: s_movk_i32 flat_scratch_lo, 0x0
+; VI: s_movk_i32 flat_scratch_hi, 0x0
+; VI: flat_atomic_umin [[RET:v[0-9]+]], v[{{[0-9]+:[0-9]+}}], v{{[0-9]+}} glc{{$}}
+; GCN: buffer_store_dword [[RET]]
+define void @atomic_umin_i32_ret_addr64(i32 addrspace(1)* %out, i32 addrspace(1)* %out2, i32 %in, i64 %index) {
+entry:
+  %ptr = getelementptr i32, i32 addrspace(1)* %out, i64 %index
+  %0  = atomicrmw volatile umin i32 addrspace(1)* %ptr, i32 %in seq_cst
+  store i32 %0, i32 addrspace(1)* %out2
+  ret void
+}
+
+; FUNC-LABEL: {{^}}atomic_or_i32_offset:
+; GCN: buffer_atomic_or v{{[0-9]+}}, s[{{[0-9]+}}:{{[0-9]+}}], 0 offset:16{{$}}
+define void @atomic_or_i32_offset(i32 addrspace(1)* %out, i32 %in) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
+  %0  = atomicrmw volatile or i32 addrspace(1)* %gep, i32 %in seq_cst
+  ret void
+}
+
+; FUNC-LABEL: {{^}}atomic_or_i32_ret_offset:
+; GCN: buffer_atomic_or [[RET:v[0-9]+]], s[{{[0-9]+}}:{{[0-9]+}}], 0 offset:16 glc {{$}}
+; GCN: buffer_store_dword [[RET]]
+define void @atomic_or_i32_ret_offset(i32 addrspace(1)* %out, i32 addrspace(1)* %out2, i32 %in) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
+  %0  = atomicrmw volatile or i32 addrspace(1)* %gep, i32 %in seq_cst
+  store i32 %0, i32 addrspace(1)* %out2
+  ret void
+}
+
+; FUNC-LABEL: {{^}}atomic_or_i32_addr64_offset:
+; SI: buffer_atomic_or v{{[0-9]+}}, v[{{[0-9]+}}:{{[0-9]+}}], s[{{[0-9]+}}:{{[0-9]+}}], 0 addr64 offset:16{{$}}
+; VI: s_movk_i32 flat_scratch_lo, 0x0
+; VI: s_movk_i32 flat_scratch_hi, 0x0
+; VI: flat_atomic_or v[{{[0-9]+:[0-9]+}}], v{{[0-9]+$}}
+define void @atomic_or_i32_addr64_offset(i32 addrspace(1)* %out, i32 %in, i64 %index) {
+entry:
+  %ptr = getelementptr i32, i32 addrspace(1)* %out, i64 %index
+  %gep = getelementptr i32, i32 addrspace(1)* %ptr, i32 4
+  %0  = atomicrmw volatile or i32 addrspace(1)* %gep, i32 %in seq_cst
+  ret void
+}
+
+; FUNC-LABEL: {{^}}atomic_or_i32_ret_addr64_offset:
+; SI: buffer_atomic_or [[RET:v[0-9]+]], v[{{[0-9]+}}:{{[0-9]+}}], s[{{[0-9]+}}:{{[0-9]+}}], 0 addr64 offset:16 glc{{$}}
+; VI: s_movk_i32 flat_scratch_lo, 0x0
+; VI: s_movk_i32 flat_scratch_hi, 0x0
+; VI: flat_atomic_or [[RET:v[0-9]+]], v[{{[0-9]+:[0-9]+}}], v{{[0-9]+}} glc{{$}}
+; GCN: buffer_store_dword [[RET]]
+define void @atomic_or_i32_ret_addr64_offset(i32 addrspace(1)* %out, i32 addrspace(1)* %out2, i32 %in, i64 %index) {
+entry:
+  %ptr = getelementptr i32, i32 addrspace(1)* %out, i64 %index
+  %gep = getelementptr i32, i32 addrspace(1)* %ptr, i32 4
+  %0  = atomicrmw volatile or i32 addrspace(1)* %gep, i32 %in seq_cst
+  store i32 %0, i32 addrspace(1)* %out2
+  ret void
+}
+
+; FUNC-LABEL: {{^}}atomic_or_i32:
+; GCN: buffer_atomic_or v{{[0-9]+}}, s[{{[0-9]+}}:{{[0-9]+}}], 0{{$}}
+define void @atomic_or_i32(i32 addrspace(1)* %out, i32 %in) {
+entry:
+  %0  = atomicrmw volatile or i32 addrspace(1)* %out, i32 %in seq_cst
+  ret void
+}
+
+; FUNC-LABEL: {{^}}atomic_or_i32_ret:
+; GCN: buffer_atomic_or [[RET:v[0-9]+]], s[{{[0-9]+}}:{{[0-9]+}}], 0 glc
+; GCN: buffer_store_dword [[RET]]
+define void @atomic_or_i32_ret(i32 addrspace(1)* %out, i32 addrspace(1)* %out2, i32 %in) {
+entry:
+  %0  = atomicrmw volatile or i32 addrspace(1)* %out, i32 %in seq_cst
+  store i32 %0, i32 addrspace(1)* %out2
+  ret void
+}
+
+; FUNC-LABEL: {{^}}atomic_or_i32_addr64:
+; SI: buffer_atomic_or v{{[0-9]+}}, v[{{[0-9]+}}:{{[0-9]+}}], s[{{[0-9]+}}:{{[0-9]+}}], 0 addr64{{$}}
+; VI: s_movk_i32 flat_scratch_lo, 0x0
+; VI: s_movk_i32 flat_scratch_hi, 0x0
+; VI: flat_atomic_or v[{{[0-9]+:[0-9]+}}], v{{[0-9]+$}}
+define void @atomic_or_i32_addr64(i32 addrspace(1)* %out, i32 %in, i64 %index) {
+entry:
+  %ptr = getelementptr i32, i32 addrspace(1)* %out, i64 %index
+  %0  = atomicrmw volatile or i32 addrspace(1)* %ptr, i32 %in seq_cst
+  ret void
+}
+
+; FUNC-LABEL: {{^}}atomic_or_i32_ret_addr64:
+; SI: buffer_atomic_or [[RET:v[0-9]+]], v[{{[0-9]+}}:{{[0-9]+}}], s[{{[0-9]+}}:{{[0-9]+}}], 0 addr64 glc{{$}}
+; VI: s_movk_i32 flat_scratch_lo, 0x0
+; VI: s_movk_i32 flat_scratch_hi, 0x0
+; VI: flat_atomic_or [[RET:v[0-9]+]], v[{{[0-9]+:[0-9]+}}], v{{[0-9]+}} glc{{$}}
+; GCN: buffer_store_dword [[RET]]
+define void @atomic_or_i32_ret_addr64(i32 addrspace(1)* %out, i32 addrspace(1)* %out2, i32 %in, i64 %index) {
+entry:
+  %ptr = getelementptr i32, i32 addrspace(1)* %out, i64 %index
+  %0  = atomicrmw volatile or i32 addrspace(1)* %ptr, i32 %in seq_cst
+  store i32 %0, i32 addrspace(1)* %out2
+  ret void
+}
+
+; FUNC-LABEL: {{^}}atomic_xchg_i32_offset:
+; GCN: buffer_atomic_swap v{{[0-9]+}}, s[{{[0-9]+}}:{{[0-9]+}}], 0 offset:16{{$}}
+define void @atomic_xchg_i32_offset(i32 addrspace(1)* %out, i32 %in) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
+  %0  = atomicrmw volatile xchg i32 addrspace(1)* %gep, i32 %in seq_cst
+  ret void
+}
+
+; FUNC-LABEL: {{^}}atomic_xchg_i32_ret_offset:
+; GCN: buffer_atomic_swap [[RET:v[0-9]+]], s[{{[0-9]+}}:{{[0-9]+}}], 0 offset:16 glc {{$}}
+; GCN: buffer_store_dword [[RET]]
+define void @atomic_xchg_i32_ret_offset(i32 addrspace(1)* %out, i32 addrspace(1)* %out2, i32 %in) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
+  %0  = atomicrmw volatile xchg i32 addrspace(1)* %gep, i32 %in seq_cst
+  store i32 %0, i32 addrspace(1)* %out2
+  ret void
+}
+
+; FUNC-LABEL: {{^}}atomic_xchg_i32_addr64_offset:
+; SI: buffer_atomic_swap v{{[0-9]+}}, v[{{[0-9]+}}:{{[0-9]+}}], s[{{[0-9]+}}:{{[0-9]+}}], 0 addr64 offset:16{{$}}
+define void @atomic_xchg_i32_addr64_offset(i32 addrspace(1)* %out, i32 %in, i64 %index) {
+entry:
+  %ptr = getelementptr i32, i32 addrspace(1)* %out, i64 %index
+  %gep = getelementptr i32, i32 addrspace(1)* %ptr, i32 4
+  %0  = atomicrmw volatile xchg i32 addrspace(1)* %gep, i32 %in seq_cst
+  ret void
+}
+
+; FUNC-LABEL: {{^}}atomic_xchg_i32_ret_addr64_offset:
+; SI: buffer_atomic_swap [[RET:v[0-9]+]], v[{{[0-9]+}}:{{[0-9]+}}], s[{{[0-9]+}}:{{[0-9]+}}], 0 addr64 offset:16 glc{{$}}
+; VI: s_movk_i32 flat_scratch_lo, 0x0
+; VI: s_movk_i32 flat_scratch_hi, 0x0
+; VI: flat_atomic_swap [[RET:v[0-9]+]], v[{{[0-9]+:[0-9]+}}], v{{[0-9]+}} glc{{$}}
+; GCN: buffer_store_dword [[RET]]
+define void @atomic_xchg_i32_ret_addr64_offset(i32 addrspace(1)* %out, i32 addrspace(1)* %out2, i32 %in, i64 %index) {
+entry:
+  %ptr = getelementptr i32, i32 addrspace(1)* %out, i64 %index
+  %gep = getelementptr i32, i32 addrspace(1)* %ptr, i32 4
+  %0  = atomicrmw volatile xchg i32 addrspace(1)* %gep, i32 %in seq_cst
+  store i32 %0, i32 addrspace(1)* %out2
+  ret void
+}
+
+; FUNC-LABEL: {{^}}atomic_xchg_i32:
+; GCN: buffer_atomic_swap v{{[0-9]+}}, s[{{[0-9]+}}:{{[0-9]+}}], 0{{$}}
+define void @atomic_xchg_i32(i32 addrspace(1)* %out, i32 %in) {
+entry:
+  %0  = atomicrmw volatile xchg i32 addrspace(1)* %out, i32 %in seq_cst
+  ret void
+}
+
+; FUNC-LABEL: {{^}}atomic_xchg_i32_ret:
+; GCN: buffer_atomic_swap [[RET:v[0-9]+]], s[{{[0-9]+}}:{{[0-9]+}}], 0 glc
+; GCN: buffer_store_dword [[RET]]
+define void @atomic_xchg_i32_ret(i32 addrspace(1)* %out, i32 addrspace(1)* %out2, i32 %in) {
+entry:
+  %0  = atomicrmw volatile xchg i32 addrspace(1)* %out, i32 %in seq_cst
+  store i32 %0, i32 addrspace(1)* %out2
+  ret void
+}
+
+; FUNC-LABEL: {{^}}atomic_xchg_i32_addr64:
+; SI: buffer_atomic_swap v{{[0-9]+}}, v[{{[0-9]+}}:{{[0-9]+}}], s[{{[0-9]+}}:{{[0-9]+}}], 0 addr64{{$}}
+; VI: s_movk_i32 flat_scratch_lo, 0x0
+; VI: s_movk_i32 flat_scratch_hi, 0x0
+; VI: flat_atomic_swap v[{{[0-9]+:[0-9]+}}], v{{[0-9]+$}}
+define void @atomic_xchg_i32_addr64(i32 addrspace(1)* %out, i32 %in, i64 %index) {
+entry:
+  %ptr = getelementptr i32, i32 addrspace(1)* %out, i64 %index
+  %0  = atomicrmw volatile xchg i32 addrspace(1)* %ptr, i32 %in seq_cst
+  ret void
+}
+
+; FUNC-LABEL: {{^}}atomic_xchg_i32_ret_addr64:
+; SI: buffer_atomic_swap [[RET:v[0-9]+]], v[{{[0-9]+}}:{{[0-9]+}}], s[{{[0-9]+}}:{{[0-9]+}}], 0 addr64 glc{{$}}
+; VI: s_movk_i32 flat_scratch_lo, 0x0
+; VI: s_movk_i32 flat_scratch_hi, 0x0
+; VI: flat_atomic_swap [[RET:v[0-9]+]],  v[{{[0-9]+:[0-9]+}}], v{{[0-9]+}} glc{{$}}
+; GCN: buffer_store_dword [[RET]]
+define void @atomic_xchg_i32_ret_addr64(i32 addrspace(1)* %out, i32 addrspace(1)* %out2, i32 %in, i64 %index) {
+entry:
+  %ptr = getelementptr i32, i32 addrspace(1)* %out, i64 %index
+  %0  = atomicrmw volatile xchg i32 addrspace(1)* %ptr, i32 %in seq_cst
+  store i32 %0, i32 addrspace(1)* %out2
+  ret void
+}
+
+; FUNC-LABEL: {{^}}atomic_xor_i32_offset:
+; GCN: buffer_atomic_xor v{{[0-9]+}}, s[{{[0-9]+}}:{{[0-9]+}}], 0 offset:16{{$}}
+define void @atomic_xor_i32_offset(i32 addrspace(1)* %out, i32 %in) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
+  %0  = atomicrmw volatile xor i32 addrspace(1)* %gep, i32 %in seq_cst
+  ret void
+}
+
+; FUNC-LABEL: {{^}}atomic_xor_i32_ret_offset:
+; GCN: buffer_atomic_xor [[RET:v[0-9]+]], s[{{[0-9]+}}:{{[0-9]+}}], 0 offset:16 glc {{$}}
+; GCN: buffer_store_dword [[RET]]
+define void @atomic_xor_i32_ret_offset(i32 addrspace(1)* %out, i32 addrspace(1)* %out2, i32 %in) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
+  %0  = atomicrmw volatile xor i32 addrspace(1)* %gep, i32 %in seq_cst
+  store i32 %0, i32 addrspace(1)* %out2
+  ret void
+}
+
+; FUNC-LABEL: {{^}}atomic_xor_i32_addr64_offset:
+; SI: buffer_atomic_xor v{{[0-9]+}}, v[{{[0-9]+}}:{{[0-9]+}}], s[{{[0-9]+}}:{{[0-9]+}}], 0 addr64 offset:16{{$}}
+; VI: s_movk_i32 flat_scratch_lo, 0x0
+; VI: s_movk_i32 flat_scratch_hi, 0x0
+; VI: flat_atomic_xor v[{{[0-9]+:[0-9]+}}], v{{[0-9]+$}}
+define void @atomic_xor_i32_addr64_offset(i32 addrspace(1)* %out, i32 %in, i64 %index) {
+entry:
+  %ptr = getelementptr i32, i32 addrspace(1)* %out, i64 %index
+  %gep = getelementptr i32, i32 addrspace(1)* %ptr, i32 4
+  %0  = atomicrmw volatile xor i32 addrspace(1)* %gep, i32 %in seq_cst
+  ret void
+}
+
+; FUNC-LABEL: {{^}}atomic_xor_i32_ret_addr64_offset:
+; SI: buffer_atomic_xor [[RET:v[0-9]+]], v[{{[0-9]+}}:{{[0-9]+}}], s[{{[0-9]+}}:{{[0-9]+}}], 0 addr64 offset:16 glc{{$}}
+; VI: s_movk_i32 flat_scratch_lo, 0x0
+; VI: s_movk_i32 flat_scratch_hi, 0x0
+; VI: flat_atomic_xor [[RET:v[0-9]+]], v[{{[0-9]+:[0-9]+}}], v{{[0-9]+}} glc{{$}}
+; GCN: buffer_store_dword [[RET]]
+define void @atomic_xor_i32_ret_addr64_offset(i32 addrspace(1)* %out, i32 addrspace(1)* %out2, i32 %in, i64 %index) {
+entry:
+  %ptr = getelementptr i32, i32 addrspace(1)* %out, i64 %index
+  %gep = getelementptr i32, i32 addrspace(1)* %ptr, i32 4
+  %0  = atomicrmw volatile xor i32 addrspace(1)* %gep, i32 %in seq_cst
+  store i32 %0, i32 addrspace(1)* %out2
+  ret void
+}
+
+; FUNC-LABEL: {{^}}atomic_xor_i32:
+; GCN: buffer_atomic_xor v{{[0-9]+}}, s[{{[0-9]+}}:{{[0-9]+}}], 0{{$}}
+define void @atomic_xor_i32(i32 addrspace(1)* %out, i32 %in) {
+entry:
+  %0  = atomicrmw volatile xor i32 addrspace(1)* %out, i32 %in seq_cst
+  ret void
+}
+
+; FUNC-LABEL: {{^}}atomic_xor_i32_ret:
+; GCN: buffer_atomic_xor [[RET:v[0-9]+]], s[{{[0-9]+}}:{{[0-9]+}}], 0 glc
+; GCN: buffer_store_dword [[RET]]
+define void @atomic_xor_i32_ret(i32 addrspace(1)* %out, i32 addrspace(1)* %out2, i32 %in) {
+entry:
+  %0  = atomicrmw volatile xor i32 addrspace(1)* %out, i32 %in seq_cst
+  store i32 %0, i32 addrspace(1)* %out2
+  ret void
+}
+
+; FUNC-LABEL: {{^}}atomic_xor_i32_addr64:
+; SI: buffer_atomic_xor v{{[0-9]+}}, v[{{[0-9]+}}:{{[0-9]+}}], s[{{[0-9]+}}:{{[0-9]+}}], 0 addr64{{$}}
+; VI: s_movk_i32 flat_scratch_lo, 0x0
+; VI: s_movk_i32 flat_scratch_hi, 0x0
+; VI: flat_atomic_xor v[{{[0-9]+:[0-9]+}}], v{{[0-9]+$}}
+define void @atomic_xor_i32_addr64(i32 addrspace(1)* %out, i32 %in, i64 %index) {
+entry:
+  %ptr = getelementptr i32, i32 addrspace(1)* %out, i64 %index
+  %0  = atomicrmw volatile xor i32 addrspace(1)* %ptr, i32 %in seq_cst
+  ret void
+}
+
+; FUNC-LABEL: {{^}}atomic_xor_i32_ret_addr64:
+; SI: buffer_atomic_xor [[RET:v[0-9]+]], v[{{[0-9]+}}:{{[0-9]+}}], s[{{[0-9]+}}:{{[0-9]+}}], 0 addr64 glc{{$}}
+; VI: s_movk_i32 flat_scratch_lo, 0x0
+; VI: s_movk_i32 flat_scratch_hi, 0x0
+; VI: flat_atomic_xor [[RET:v[0-9]+]], v[{{[0-9]+:[0-9]+}}], v{{[0-9]+}} glc{{$}}
+; GCN: buffer_store_dword [[RET]]
+define void @atomic_xor_i32_ret_addr64(i32 addrspace(1)* %out, i32 addrspace(1)* %out2, i32 %in, i64 %index) {
+entry:
+  %ptr = getelementptr i32, i32 addrspace(1)* %out, i64 %index
+  %0  = atomicrmw volatile xor i32 addrspace(1)* %ptr, i32 %in seq_cst
+  store i32 %0, i32 addrspace(1)* %out2
+  ret void
+}
diff --git a/test/CodeGen/R600/gv-const-addrspace-fail.ll b/test/CodeGen/AMDGPU/gv-const-addrspace-fail.ll
similarity index 100%
rename from test/CodeGen/R600/gv-const-addrspace-fail.ll
rename to test/CodeGen/AMDGPU/gv-const-addrspace-fail.ll
diff --git a/test/CodeGen/R600/gv-const-addrspace.ll b/test/CodeGen/AMDGPU/gv-const-addrspace.ll
similarity index 92%
rename from test/CodeGen/R600/gv-const-addrspace.ll
rename to test/CodeGen/AMDGPU/gv-const-addrspace.ll
index 3c1fc6c98f7..d4d13125cfb 100644
--- a/test/CodeGen/R600/gv-const-addrspace.ll
+++ b/test/CodeGen/AMDGPU/gv-const-addrspace.ll
@@ -8,9 +8,7 @@
 @float_gv = internal unnamed_addr addrspace(2) constant [5 x float] [float 0.0, float 1.0, float 2.0, float 3.0, float 4.0], align 4
 
 ; FUNC-LABEL: {{^}}float:
-; FIXME: We should be using s_load_dword here.
-; SI: buffer_load_dword
-; VI: s_load_dword
+; GCN: s_load_dword
 
 ; EG-DAG: MOV {{\** *}}T2.X
 ; EG-DAG: MOV {{\** *}}T3.X
@@ -31,9 +29,7 @@ entry:
 
 ; FUNC-LABEL: {{^}}i32:
 
-; FIXME: We should be using s_load_dword here.
-; SI: buffer_load_dword
-; VI: s_load_dword
+; GCN: s_load_dword
 
 ; EG-DAG: MOV {{\** *}}T2.X
 ; EG-DAG: MOV {{\** *}}T3.X
@@ -71,9 +67,7 @@ define void @struct_foo_gv_load(i32 addrspace(1)* %out, i32 %index) {
                                                                 <1 x i32> <i32 4> ]
 
 ; FUNC-LABEL: {{^}}array_v1_gv_load:
-; FIXME: We should be using s_load_dword here.
-; SI: buffer_load_dword
-; VI: s_load_dword
+; GCN: s_load_dword
 define void @array_v1_gv_load(<1 x i32> addrspace(1)* %out, i32 %index) {
   %gep = getelementptr inbounds [4 x <1 x i32>], [4 x <1 x i32>] addrspace(2)* @array_v1_gv, i32 0, i32 %index
   %load = load <1 x i32>, <1 x i32> addrspace(2)* %gep, align 4
diff --git a/test/CodeGen/AMDGPU/half.ll b/test/CodeGen/AMDGPU/half.ll
new file mode 100644
index 00000000000..bf8f11860b5
--- /dev/null
+++ b/test/CodeGen/AMDGPU/half.ll
@@ -0,0 +1,525 @@
+; RUN: llc -march=amdgcn -mcpu=SI -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=SI %s
+; RUN: llc -march=amdgcn -mcpu=tonga -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=VI %s
+
+; half args should be promoted to float
+
+; GCN-LABEL: {{^}}load_f16_arg:
+; GCN: s_load_dword [[ARG:s[0-9]+]]
+; GCN: v_cvt_f16_f32_e32 [[CVT:v[0-9]+]], [[ARG]]
+; GCN: buffer_store_short [[CVT]]
+define void @load_f16_arg(half addrspace(1)* %out, half %arg) #0 {
+  store half %arg, half addrspace(1)* %out
+  ret void
+}
+
+; GCN-LABEL: {{^}}load_v2f16_arg:
+; GCN-DAG: buffer_load_ushort [[V0:v[0-9]+]], s{{\[[0-9]+:[0-9]+\]}}, 0 offset:44
+; GCN-DAG: buffer_load_ushort [[V1:v[0-9]+]], s{{\[[0-9]+:[0-9]+\]}}, 0 offset:46
+; GCN-DAG: buffer_store_short [[V0]], s{{\[[0-9]+:[0-9]+\]}}, 0{{$}}
+; GCN-DAG: buffer_store_short [[V1]], s{{\[[0-9]+:[0-9]+\]}}, 0 offset:2{{$}}
+; GCN: s_endpgm
+define void @load_v2f16_arg(<2 x half> addrspace(1)* %out, <2 x half> %arg) #0 {
+  store <2 x half> %arg, <2 x half> addrspace(1)* %out
+  ret void
+}
+
+; GCN-LABEL: {{^}}load_v3f16_arg:
+; GCN: buffer_load_ushort
+; GCN: buffer_load_ushort
+; GCN: buffer_load_ushort
+; GCN-NOT: buffer_load
+; GCN-DAG: buffer_store_dword
+; GCN-DAG: buffer_store_short
+; GCN-NOT: buffer_store
+; GCN: s_endpgm
+define void @load_v3f16_arg(<3 x half> addrspace(1)* %out, <3 x half> %arg) #0 {
+  store <3 x half> %arg, <3 x half> addrspace(1)* %out
+  ret void
+}
+
+; GCN-LABEL: {{^}}load_v4f16_arg:
+; GCN: buffer_load_ushort
+; GCN: buffer_load_ushort
+; GCN: buffer_load_ushort
+; GCN: buffer_load_ushort
+; GCN: buffer_store_short
+; GCN: buffer_store_short
+; GCN: buffer_store_short
+; GCN: buffer_store_short
+; GCN: s_endpgm
+define void @load_v4f16_arg(<4 x half> addrspace(1)* %out, <4 x half> %arg) #0 {
+  store <4 x half> %arg, <4 x half> addrspace(1)* %out
+  ret void
+}
+
+; GCN-LABEL: {{^}}load_v8f16_arg:
+define void @load_v8f16_arg(<8 x half> addrspace(1)* %out, <8 x half> %arg) #0 {
+  store <8 x half> %arg, <8 x half> addrspace(1)* %out
+  ret void
+}
+
+; GCN-LABEL: {{^}}extload_v2f16_arg:
+define void @extload_v2f16_arg(<2 x float> addrspace(1)* %out, <2 x half> %in) #0 {
+  %fpext = fpext <2 x half> %in to <2 x float>
+  store <2 x float> %fpext, <2 x float> addrspace(1)* %out
+  ret void
+}
+
+; GCN-LABEL: {{^}}extload_f16_to_f32_arg:
+define void @extload_f16_to_f32_arg(float addrspace(1)* %out, half %arg) #0 {
+  %ext = fpext half %arg to float
+  store float %ext, float addrspace(1)* %out
+  ret void
+}
+
+; GCN-LABEL: {{^}}extload_v2f16_to_v2f32_arg:
+define void @extload_v2f16_to_v2f32_arg(<2 x float> addrspace(1)* %out, <2 x half> %arg) #0 {
+  %ext = fpext <2 x half> %arg to <2 x float>
+  store <2 x float> %ext, <2 x float> addrspace(1)* %out
+  ret void
+}
+
+; GCN-LABEL: {{^}}extload_v3f16_to_v3f32_arg:
+; GCN: buffer_load_ushort
+; GCN: buffer_load_ushort
+; GCN: buffer_load_ushort
+; GCN-NOT: buffer_load
+; GCN: v_cvt_f32_f16_e32
+; GCN: v_cvt_f32_f16_e32
+; GCN: v_cvt_f32_f16_e32
+; GCN-NOT: v_cvt_f32_f16
+; GCN-DAG: buffer_store_dword
+; GCN-DAG: buffer_store_dwordx2
+; GCN: s_endpgm
+define void @extload_v3f16_to_v3f32_arg(<3 x float> addrspace(1)* %out, <3 x half> %arg) #0 {
+  %ext = fpext <3 x half> %arg to <3 x float>
+  store <3 x float> %ext, <3 x float> addrspace(1)* %out
+  ret void
+}
+
+; GCN-LABEL: {{^}}extload_v4f16_to_v4f32_arg:
+define void @extload_v4f16_to_v4f32_arg(<4 x float> addrspace(1)* %out, <4 x half> %arg) #0 {
+  %ext = fpext <4 x half> %arg to <4 x float>
+  store <4 x float> %ext, <4 x float> addrspace(1)* %out
+  ret void
+}
+
+; GCN-LABEL: {{^}}extload_v8f16_to_v8f32_arg:
+define void @extload_v8f16_to_v8f32_arg(<8 x float> addrspace(1)* %out, <8 x half> %arg) #0 {
+  %ext = fpext <8 x half> %arg to <8 x float>
+  store <8 x float> %ext, <8 x float> addrspace(1)* %out
+  ret void
+}
+
+; GCN-LABEL: {{^}}extload_f16_to_f64_arg:
+define void @extload_f16_to_f64_arg(double addrspace(1)* %out, half %arg) #0 {
+  %ext = fpext half %arg to double
+  store double %ext, double addrspace(1)* %out
+  ret void
+}
+; GCN-LABEL: {{^}}extload_v2f16_to_v2f64_arg:
+define void @extload_v2f16_to_v2f64_arg(<2 x double> addrspace(1)* %out, <2 x half> %arg) #0 {
+  %ext = fpext <2 x half> %arg to <2 x double>
+  store <2 x double> %ext, <2 x double> addrspace(1)* %out
+  ret void
+}
+
+; GCN-LABEL: {{^}}extload_v3f16_to_v3f64_arg:
+define void @extload_v3f16_to_v3f64_arg(<3 x double> addrspace(1)* %out, <3 x half> %arg) #0 {
+  %ext = fpext <3 x half> %arg to <3 x double>
+  store <3 x double> %ext, <3 x double> addrspace(1)* %out
+  ret void
+}
+
+; GCN-LABEL: {{^}}extload_v4f16_to_v4f64_arg:
+define void @extload_v4f16_to_v4f64_arg(<4 x double> addrspace(1)* %out, <4 x half> %arg) #0 {
+  %ext = fpext <4 x half> %arg to <4 x double>
+  store <4 x double> %ext, <4 x double> addrspace(1)* %out
+  ret void
+}
+
+; GCN-LABEL: {{^}}extload_v8f16_to_v8f64_arg:
+define void @extload_v8f16_to_v8f64_arg(<8 x double> addrspace(1)* %out, <8 x half> %arg) #0 {
+  %ext = fpext <8 x half> %arg to <8 x double>
+  store <8 x double> %ext, <8 x double> addrspace(1)* %out
+  ret void
+}
+
+; GCN-LABEL: {{^}}global_load_store_f16:
+; GCN: buffer_load_ushort [[TMP:v[0-9]+]]
+; GCN: buffer_store_short [[TMP]]
+define void @global_load_store_f16(half addrspace(1)* %out, half addrspace(1)* %in) #0 {
+  %val = load half, half addrspace(1)* %in
+  store half %val, half addrspace(1)* %out
+  ret void
+}
+
+; GCN-LABEL: {{^}}global_load_store_v2f16:
+; GCN: buffer_load_dword [[TMP:v[0-9]+]]
+; GCN: buffer_store_dword [[TMP]]
+define void @global_load_store_v2f16(<2 x half> addrspace(1)* %out, <2 x half> addrspace(1)* %in) #0 {
+  %val = load <2 x half>, <2 x half> addrspace(1)* %in
+  store <2 x half> %val, <2 x half> addrspace(1)* %out
+  ret void
+}
+
+; GCN-LABEL: {{^}}global_load_store_v4f16:
+; GCN: buffer_load_dwordx2 [[TMP:v\[[0-9]+:[0-9]+\]]]
+; GCN: buffer_store_dwordx2 [[TMP]]
+define void @global_load_store_v4f16(<4 x half> addrspace(1)* %in, <4 x half> addrspace(1)* %out) #0 {
+  %val = load <4 x half>, <4 x half> addrspace(1)* %in
+  store <4 x half> %val, <4 x half> addrspace(1)* %out
+  ret void
+}
+
+; GCN-LABEL: {{^}}global_load_store_v8f16:
+; GCN: buffer_load_dwordx4 [[TMP:v\[[0-9]+:[0-9]+\]]]
+; GCN: buffer_store_dwordx4 [[TMP:v\[[0-9]+:[0-9]+\]]]
+; GCN: s_endpgm
+define void @global_load_store_v8f16(<8 x half> addrspace(1)* %out, <8 x half> addrspace(1)* %in) #0 {
+  %val = load <8 x half>, <8 x half> addrspace(1)* %in
+  store <8 x half> %val, <8 x half> addrspace(1)* %out
+  ret void
+}
+
+; GCN-LABEL: {{^}}global_extload_f16_to_f32:
+; GCN: buffer_load_ushort [[LOAD:v[0-9]+]]
+; GCN: v_cvt_f32_f16_e32 [[CVT:v[0-9]+]], [[LOAD]]
+; GCN: buffer_store_dword [[CVT]]
+define void @global_extload_f16_to_f32(float addrspace(1)* %out, half addrspace(1)* %in) #0 {
+  %val = load half, half addrspace(1)* %in
+  %cvt = fpext half %val to float
+  store float %cvt, float addrspace(1)* %out
+  ret void
+}
+
+; GCN-LABEL: {{^}}global_extload_v2f16_to_v2f32:
+define void @global_extload_v2f16_to_v2f32(<2 x float> addrspace(1)* %out, <2 x half> addrspace(1)* %in) #0 {
+  %val = load <2 x half>, <2 x half> addrspace(1)* %in
+  %cvt = fpext <2 x half> %val to <2 x float>
+  store <2 x float> %cvt, <2 x float> addrspace(1)* %out
+  ret void
+}
+
+; GCN-LABEL: {{^}}global_extload_v3f16_to_v3f32:
+define void @global_extload_v3f16_to_v3f32(<3 x float> addrspace(1)* %out, <3 x half> addrspace(1)* %in) #0 {
+  %val = load <3 x half>, <3 x half> addrspace(1)* %in
+  %cvt = fpext <3 x half> %val to <3 x float>
+  store <3 x float> %cvt, <3 x float> addrspace(1)* %out
+  ret void
+}
+
+; GCN-LABEL: {{^}}global_extload_v4f16_to_v4f32:
+define void @global_extload_v4f16_to_v4f32(<4 x float> addrspace(1)* %out, <4 x half> addrspace(1)* %in) #0 {
+  %val = load <4 x half>, <4 x half> addrspace(1)* %in
+  %cvt = fpext <4 x half> %val to <4 x float>
+  store <4 x float> %cvt, <4 x float> addrspace(1)* %out
+  ret void
+}
+
+; GCN-LABEL: {{^}}global_extload_v8f16_to_v8f32:
+define void @global_extload_v8f16_to_v8f32(<8 x float> addrspace(1)* %out, <8 x half> addrspace(1)* %in) #0 {
+  %val = load <8 x half>, <8 x half> addrspace(1)* %in
+  %cvt = fpext <8 x half> %val to <8 x float>
+  store <8 x float> %cvt, <8 x float> addrspace(1)* %out
+  ret void
+}
+
+; GCN-LABEL: {{^}}global_extload_v16f16_to_v16f32:
+define void @global_extload_v16f16_to_v16f32(<16 x float> addrspace(1)* %out, <16 x half> addrspace(1)* %in) #0 {
+  %val = load <16 x half>, <16 x half> addrspace(1)* %in
+  %cvt = fpext <16 x half> %val to <16 x float>
+  store <16 x float> %cvt, <16 x float> addrspace(1)* %out
+  ret void
+}
+
+; GCN-LABEL: {{^}}global_extload_f16_to_f64:
+; GCN: buffer_load_ushort [[LOAD:v[0-9]+]]
+; GCN: v_cvt_f32_f16_e32 [[CVT0:v[0-9]+]], [[LOAD]]
+; GCN: v_cvt_f64_f32_e32 [[CVT1:v\[[0-9]+:[0-9]+\]]], [[CVT0]]
+; GCN: buffer_store_dwordx2 [[CVT1]]
+define void @global_extload_f16_to_f64(double addrspace(1)* %out, half addrspace(1)* %in) #0 {
+  %val = load half, half addrspace(1)* %in
+  %cvt = fpext half %val to double
+  store double %cvt, double addrspace(1)* %out
+  ret void
+}
+
+; GCN-LABEL: {{^}}global_extload_v2f16_to_v2f64:
+define void @global_extload_v2f16_to_v2f64(<2 x double> addrspace(1)* %out, <2 x half> addrspace(1)* %in) #0 {
+  %val = load <2 x half>, <2 x half> addrspace(1)* %in
+  %cvt = fpext <2 x half> %val to <2 x double>
+  store <2 x double> %cvt, <2 x double> addrspace(1)* %out
+  ret void
+}
+
+; GCN-LABEL: {{^}}global_extload_v3f16_to_v3f64:
+define void @global_extload_v3f16_to_v3f64(<3 x double> addrspace(1)* %out, <3 x half> addrspace(1)* %in) #0 {
+  %val = load <3 x half>, <3 x half> addrspace(1)* %in
+  %cvt = fpext <3 x half> %val to <3 x double>
+  store <3 x double> %cvt, <3 x double> addrspace(1)* %out
+  ret void
+}
+
+; GCN-LABEL: {{^}}global_extload_v4f16_to_v4f64:
+define void @global_extload_v4f16_to_v4f64(<4 x double> addrspace(1)* %out, <4 x half> addrspace(1)* %in) #0 {
+  %val = load <4 x half>, <4 x half> addrspace(1)* %in
+  %cvt = fpext <4 x half> %val to <4 x double>
+  store <4 x double> %cvt, <4 x double> addrspace(1)* %out
+  ret void
+}
+
+; GCN-LABEL: {{^}}global_extload_v8f16_to_v8f64:
+define void @global_extload_v8f16_to_v8f64(<8 x double> addrspace(1)* %out, <8 x half> addrspace(1)* %in) #0 {
+  %val = load <8 x half>, <8 x half> addrspace(1)* %in
+  %cvt = fpext <8 x half> %val to <8 x double>
+  store <8 x double> %cvt, <8 x double> addrspace(1)* %out
+  ret void
+}
+
+; GCN-LABEL: {{^}}global_extload_v16f16_to_v16f64:
+define void @global_extload_v16f16_to_v16f64(<16 x double> addrspace(1)* %out, <16 x half> addrspace(1)* %in) #0 {
+  %val = load <16 x half>, <16 x half> addrspace(1)* %in
+  %cvt = fpext <16 x half> %val to <16 x double>
+  store <16 x double> %cvt, <16 x double> addrspace(1)* %out
+  ret void
+}
+
+; GCN-LABEL: {{^}}global_truncstore_f32_to_f16:
+; GCN: buffer_load_dword [[LOAD:v[0-9]+]]
+; GCN: v_cvt_f16_f32_e32 [[CVT:v[0-9]+]], [[LOAD]]
+; GCN: buffer_store_short [[CVT]]
+define void @global_truncstore_f32_to_f16(half addrspace(1)* %out, float addrspace(1)* %in) #0 {
+  %val = load float, float addrspace(1)* %in
+  %cvt = fptrunc float %val to half
+  store half %cvt, half addrspace(1)* %out
+  ret void
+}
+
+; GCN-LABEL: {{^}}global_truncstore_v2f32_to_v2f16:
+; GCN: buffer_load_dwordx2 v{{\[}}[[LO:[0-9]+]]:[[HI:[0-9]+]]{{\]}}
+; GCN-DAG: v_cvt_f16_f32_e32 [[CVT0:v[0-9]+]], v[[LO]]
+; GCN-DAG: v_cvt_f16_f32_e32 [[CVT1:v[0-9]+]], v[[HI]]
+; GCN-DAG: buffer_store_short [[CVT0]]
+; GCN-DAG: buffer_store_short [[CVT1]]
+; GCN: s_endpgm
+define void @global_truncstore_v2f32_to_v2f16(<2 x half> addrspace(1)* %out, <2 x float> addrspace(1)* %in) #0 {
+  %val = load <2 x float>, <2 x float> addrspace(1)* %in
+  %cvt = fptrunc <2 x float> %val to <2 x half>
+  store <2 x half> %cvt, <2 x half> addrspace(1)* %out
+  ret void
+}
+
+; FIXME: Shouldn't do 4th conversion
+; GCN-LABEL: {{^}}global_truncstore_v3f32_to_v3f16:
+; GCN: buffer_load_dwordx4
+; GCN: v_cvt_f16_f32_e32
+; GCN: v_cvt_f16_f32_e32
+; GCN: v_cvt_f16_f32_e32
+; GCN: v_cvt_f16_f32_e32
+; GCN: buffer_store_short
+; GCN: buffer_store_dword
+; GCN: s_endpgm
+define void @global_truncstore_v3f32_to_v3f16(<3 x half> addrspace(1)* %out, <3 x float> addrspace(1)* %in) #0 {
+  %val = load <3 x float>, <3 x float> addrspace(1)* %in
+  %cvt = fptrunc <3 x float> %val to <3 x half>
+  store <3 x half> %cvt, <3 x half> addrspace(1)* %out
+  ret void
+}
+
+; GCN-LABEL: {{^}}global_truncstore_v4f32_to_v4f16:
+; GCN: buffer_load_dwordx4
+; GCN: v_cvt_f16_f32_e32
+; GCN: v_cvt_f16_f32_e32
+; GCN: v_cvt_f16_f32_e32
+; GCN: v_cvt_f16_f32_e32
+; GCN: buffer_store_short
+; GCN: buffer_store_short
+; GCN: buffer_store_short
+; GCN: buffer_store_short
+; GCN: s_endpgm
+define void @global_truncstore_v4f32_to_v4f16(<4 x half> addrspace(1)* %out, <4 x float> addrspace(1)* %in) #0 {
+  %val = load <4 x float>, <4 x float> addrspace(1)* %in
+  %cvt = fptrunc <4 x float> %val to <4 x half>
+  store <4 x half> %cvt, <4 x half> addrspace(1)* %out
+  ret void
+}
+
+; GCN-LABEL: {{^}}global_truncstore_v8f32_to_v8f16:
+; GCN: buffer_load_dword
+; GCN: buffer_load_dword
+; GCN: buffer_load_dword
+; GCN: buffer_load_dword
+; GCN: buffer_load_dword
+; GCN: buffer_load_dword
+; GCN: buffer_load_dword
+; GCN: buffer_load_dword
+; GCN: v_cvt_f16_f32_e32
+; GCN: v_cvt_f16_f32_e32
+; GCN: v_cvt_f16_f32_e32
+; GCN: v_cvt_f16_f32_e32
+; GCN: v_cvt_f16_f32_e32
+; GCN: v_cvt_f16_f32_e32
+; GCN: v_cvt_f16_f32_e32
+; GCN: v_cvt_f16_f32_e32
+; GCN: buffer_store_short
+; GCN: buffer_store_short
+; GCN: buffer_store_short
+; GCN: buffer_store_short
+; GCN: buffer_store_short
+; GCN: buffer_store_short
+; GCN: buffer_store_short
+; GCN: buffer_store_short
+; GCN: s_endpgm
+define void @global_truncstore_v8f32_to_v8f16(<8 x half> addrspace(1)* %out, <8 x float> addrspace(1)* %in) #0 {
+  %val = load <8 x float>, <8 x float> addrspace(1)* %in
+  %cvt = fptrunc <8 x float> %val to <8 x half>
+  store <8 x half> %cvt, <8 x half> addrspace(1)* %out
+  ret void
+}
+
+; GCN-LABEL: {{^}}global_truncstore_v16f32_to_v16f16:
+; GCN: buffer_load_dword
+; GCN: buffer_load_dword
+; GCN: buffer_load_dword
+; GCN: buffer_load_dword
+; GCN: buffer_load_dword
+; GCN: buffer_load_dword
+; GCN: buffer_load_dword
+; GCN: buffer_load_dword
+; GCN: buffer_load_dword
+; GCN: buffer_load_dword
+; GCN: buffer_load_dword
+; GCN: buffer_load_dword
+; GCN: buffer_load_dword
+; GCN: buffer_load_dword
+; GCN: buffer_load_dword
+; GCN: buffer_load_dword
+; GCN: v_cvt_f16_f32_e32
+; GCN: v_cvt_f16_f32_e32
+; GCN: v_cvt_f16_f32_e32
+; GCN: v_cvt_f16_f32_e32
+; GCN: v_cvt_f16_f32_e32
+; GCN: v_cvt_f16_f32_e32
+; GCN: v_cvt_f16_f32_e32
+; GCN: v_cvt_f16_f32_e32
+; GCN: v_cvt_f16_f32_e32
+; GCN: v_cvt_f16_f32_e32
+; GCN: v_cvt_f16_f32_e32
+; GCN: v_cvt_f16_f32_e32
+; GCN: v_cvt_f16_f32_e32
+; GCN: v_cvt_f16_f32_e32
+; GCN: v_cvt_f16_f32_e32
+; GCN: v_cvt_f16_f32_e32
+; GCN: buffer_store_short
+; GCN: buffer_store_short
+; GCN: buffer_store_short
+; GCN: buffer_store_short
+; GCN: buffer_store_short
+; GCN: buffer_store_short
+; GCN: buffer_store_short
+; GCN: buffer_store_short
+; GCN: buffer_store_short
+; GCN: buffer_store_short
+; GCN: buffer_store_short
+; GCN: buffer_store_short
+; GCN: buffer_store_short
+; GCN: buffer_store_short
+; GCN: buffer_store_short
+; GCN: buffer_store_short
+; GCN: s_endpgm
+define void @global_truncstore_v16f32_to_v16f16(<16 x half> addrspace(1)* %out, <16 x float> addrspace(1)* %in) #0 {
+  %val = load <16 x float>, <16 x float> addrspace(1)* %in
+  %cvt = fptrunc <16 x float> %val to <16 x half>
+  store <16 x half> %cvt, <16 x half> addrspace(1)* %out
+  ret void
+}
+
+; FIXME: Unsafe math should fold conversions away
+; GCN-LABEL: {{^}}fadd_f16:
+; SI-DAG: v_cvt_f32_f16_e32 v{{[0-9]+}},
+; SI-DAG: v_cvt_f32_f16_e32 v{{[0-9]+}},
+; SI-DAG: v_cvt_f32_f16_e32 v{{[0-9]+}},
+; SI-DAG: v_cvt_f32_f16_e32 v{{[0-9]+}},
+; SI: v_add_f32
+; GCN: s_endpgm
+define void @fadd_f16(half addrspace(1)* %out, half %a, half %b) #0 {
+   %add = fadd half %a, %b
+   store half %add, half addrspace(1)* %out, align 4
+   ret void
+}
+
+; GCN-LABEL: {{^}}fadd_v2f16:
+; SI: v_add_f32
+; SI: v_add_f32
+; GCN: s_endpgm
+define void @fadd_v2f16(<2 x half> addrspace(1)* %out, <2 x half> %a, <2 x half> %b) #0 {
+  %add = fadd <2 x half> %a, %b
+  store <2 x half> %add, <2 x half> addrspace(1)* %out, align 8
+  ret void
+}
+
+; GCN-LABEL: {{^}}fadd_v4f16:
+; SI: v_add_f32
+; SI: v_add_f32
+; SI: v_add_f32
+; SI: v_add_f32
+; GCN: s_endpgm
+define void @fadd_v4f16(<4 x half> addrspace(1)* %out, <4 x half> addrspace(1)* %in) #0 {
+  %b_ptr = getelementptr <4 x half>, <4 x half> addrspace(1)* %in, i32 1
+  %a = load <4 x half>, <4 x half> addrspace(1)* %in, align 16
+  %b = load <4 x half>, <4 x half> addrspace(1)* %b_ptr, align 16
+  %result = fadd <4 x half> %a, %b
+  store <4 x half> %result, <4 x half> addrspace(1)* %out, align 16
+  ret void
+}
+
+; GCN-LABEL: {{^}}fadd_v8f16:
+; SI: v_add_f32
+; SI: v_add_f32
+; SI: v_add_f32
+; SI: v_add_f32
+; SI: v_add_f32
+; SI: v_add_f32
+; SI: v_add_f32
+; SI: v_add_f32
+; GCN: s_endpgm
+define void @fadd_v8f16(<8 x half> addrspace(1)* %out, <8 x half> %a, <8 x half> %b) #0 {
+  %add = fadd <8 x half> %a, %b
+  store <8 x half> %add, <8 x half> addrspace(1)* %out, align 32
+  ret void
+}
+
+; GCN-LABEL: {{^}}fsub_f16:
+; GCN: v_subrev_f32_e32
+; GCN: s_endpgm
+define void @fsub_f16(half addrspace(1)* %out, half addrspace(1)* %in) #0 {
+  %b_ptr = getelementptr half, half addrspace(1)* %in, i32 1
+  %a = load half, half addrspace(1)* %in
+  %b = load half, half addrspace(1)* %b_ptr
+  %sub = fsub half %a, %b
+  store half %sub, half addrspace(1)* %out
+  ret void
+}
+
+; GCN-LABEL: {{^}}test_bitcast_from_half:
+; GCN: buffer_load_ushort [[TMP:v[0-9]+]]
+; GCN: buffer_store_short [[TMP]]
+define void @test_bitcast_from_half(half addrspace(1)* %in, i16 addrspace(1)* %out) #0 {
+  %val = load half, half addrspace(1)* %in
+  %val_int = bitcast half %val to i16
+  store i16 %val_int, i16 addrspace(1)* %out
+  ret void
+}
+
+; GCN-LABEL: {{^}}test_bitcast_to_half:
+; GCN: buffer_load_ushort [[TMP:v[0-9]+]]
+; GCN: buffer_store_short [[TMP]]
+define void @test_bitcast_to_half(half addrspace(1)* %out, i16 addrspace(1)* %in) #0 {
+  %val = load i16, i16 addrspace(1)* %in
+  %val_fp = bitcast i16 %val to half
+  store half %val_fp, half addrspace(1)* %out
+  ret void
+}
+
+attributes #0 = { nounwind }
diff --git a/test/CodeGen/AMDGPU/hsa.ll b/test/CodeGen/AMDGPU/hsa.ll
new file mode 100644
index 00000000000..653a6bb1b60
--- /dev/null
+++ b/test/CodeGen/AMDGPU/hsa.ll
@@ -0,0 +1,35 @@
+; RUN: llc < %s -mtriple=amdgcn--amdhsa -mcpu=kaveri | FileCheck --check-prefix=HSA-CI --check-prefix=HSA %s
+; RUN: llc < %s -mtriple=amdgcn--amdhsa -mcpu=carrizo | FileCheck --check-prefix=HSA-VI --check-prefix=HSA %s
+; RUN: llc < %s -mtriple=amdgcn--amdhsa -mcpu=kaveri -filetype=obj  | llvm-readobj -s -sd | FileCheck --check-prefix=ELF %s
+; RUN: llc < %s -mtriple=amdgcn--amdhsa -mcpu=kaveri | llvm-mc -filetype=obj -triple amdgcn--amdhsa -mcpu=kaveri | llvm-readobj -s -sd | FileCheck %s --check-prefix=ELF
+
+; The SHT_NOTE section contains the output from the .hsa_code_object_*
+; directives.
+
+; ELF: SHT_NOTE
+; ELF: 0000: 04000000 08000000 01000000 414D4400
+; ELF: 0010: 01000000 00000000 04000000 1B000000
+; ELF: 0020: 03000000 414D4400 04000700 07000000
+; ELF: 0030: 00000000 00000000 414D4400 414D4447
+; ELF: 0040: 50550000
+
+; HSA: .hsa_code_object_version 1,0
+; HSA-CI: .hsa_code_object_isa 7,0,0,"AMD","AMDGPU"
+; HSA-VI: .hsa_code_object_isa 8,0,1,"AMD","AMDGPU"
+
+; HSA: {{^}}simple:
+; HSA: .amd_kernel_code_t
+; HSA: .end_amd_kernel_code_t
+; HSA: s_load_dwordx2 s[{{[0-9]+:[0-9]+}}], s[0:1], 0x0
+
+; Make sure we are setting the ATC bit:
+; HSA-CI: s_mov_b32 s[[HI:[0-9]]], 0x100f000
+; On VI+ we also need to set MTYPE = 2
+; HSA-VI: s_mov_b32 s[[HI:[0-9]]], 0x1100f000
+; HSA: buffer_store_dword v{{[0-9]+}}, s[0:[[HI]]], 0
+
+define void @simple(i32 addrspace(1)* %out) {
+entry:
+  store i32 0, i32 addrspace(1)* %out
+  ret void
+}
diff --git a/test/CodeGen/R600/i1-copy-implicit-def.ll b/test/CodeGen/AMDGPU/i1-copy-implicit-def.ll
similarity index 100%
rename from test/CodeGen/R600/i1-copy-implicit-def.ll
rename to test/CodeGen/AMDGPU/i1-copy-implicit-def.ll
diff --git a/test/CodeGen/R600/i1-copy-phi.ll b/test/CodeGen/AMDGPU/i1-copy-phi.ll
similarity index 100%
rename from test/CodeGen/R600/i1-copy-phi.ll
rename to test/CodeGen/AMDGPU/i1-copy-phi.ll
diff --git a/test/CodeGen/R600/i8-to-double-to-float.ll b/test/CodeGen/AMDGPU/i8-to-double-to-float.ll
similarity index 100%
rename from test/CodeGen/R600/i8-to-double-to-float.ll
rename to test/CodeGen/AMDGPU/i8-to-double-to-float.ll
diff --git a/test/CodeGen/R600/icmp-select-sete-reverse-args.ll b/test/CodeGen/AMDGPU/icmp-select-sete-reverse-args.ll
similarity index 100%
rename from test/CodeGen/R600/icmp-select-sete-reverse-args.ll
rename to test/CodeGen/AMDGPU/icmp-select-sete-reverse-args.ll
diff --git a/test/CodeGen/R600/icmp64.ll b/test/CodeGen/AMDGPU/icmp64.ll
similarity index 100%
rename from test/CodeGen/R600/icmp64.ll
rename to test/CodeGen/AMDGPU/icmp64.ll
diff --git a/test/CodeGen/R600/imm.ll b/test/CodeGen/AMDGPU/imm.ll
similarity index 99%
rename from test/CodeGen/R600/imm.ll
rename to test/CodeGen/AMDGPU/imm.ll
index 8917cd6dba3..12eed550eb1 100644
--- a/test/CodeGen/R600/imm.ll
+++ b/test/CodeGen/AMDGPU/imm.ll
@@ -36,7 +36,7 @@ define void @store_imm_neg_0.0_i64(i64 addrspace(1) *%out) {
 
 ; CHECK-LABEL: {{^}}store_inline_imm_neg_0.0_i32:
 ; CHECK: v_mov_b32_e32 [[REG:v[0-9]+]], 0x80000000
-; CHECK-NEXT: buffer_store_dword [[REG]]
+; CHECK: buffer_store_dword [[REG]]
 define void @store_inline_imm_neg_0.0_i32(i32 addrspace(1)* %out) {
   store i32 -2147483648, i32 addrspace(1)* %out
   ret void
diff --git a/test/CodeGen/AMDGPU/indirect-addressing-si.ll b/test/CodeGen/AMDGPU/indirect-addressing-si.ll
new file mode 100644
index 00000000000..f551606d63a
--- /dev/null
+++ b/test/CodeGen/AMDGPU/indirect-addressing-si.ll
@@ -0,0 +1,121 @@
+; RUN: llc < %s -march=amdgcn -mcpu=SI -verify-machineinstrs | FileCheck %s
+; RUN: llc < %s -march=amdgcn -mcpu=tonga -verify-machineinstrs | FileCheck %s
+
+; Tests for indirect addressing on SI, which is implemented using dynamic
+; indexing of vectors.
+
+; CHECK-LABEL: {{^}}extract_w_offset:
+; CHECK: s_mov_b32 m0
+; CHECK-NEXT: v_movrels_b32_e32
+define void @extract_w_offset(float addrspace(1)* %out, i32 %in) {
+entry:
+  %0 = add i32 %in, 1
+  %1 = extractelement <4 x float> <float 1.0, float 2.0, float 3.0, float 4.0>, i32 %0
+  store float %1, float addrspace(1)* %out
+  ret void
+}
+
+; CHECK-LABEL: {{^}}extract_wo_offset:
+; CHECK: s_mov_b32 m0
+; CHECK-NEXT: v_movrels_b32_e32
+define void @extract_wo_offset(float addrspace(1)* %out, i32 %in) {
+entry:
+  %0 = extractelement <4 x float> <float 1.0, float 2.0, float 3.0, float 4.0>, i32 %in
+  store float %0, float addrspace(1)* %out
+  ret void
+}
+
+; CHECK-LABEL: {{^}}extract_neg_offset_sgpr:
+; The offset depends on the register that holds the first element of the vector.
+; CHECK: s_add_i32 m0, s{{[0-9]+}}, 0xfffffe{{[0-9a-z]+}}
+; CHECK: v_movrels_b32_e32 v{{[0-9]}}, v0
+define void @extract_neg_offset_sgpr(i32 addrspace(1)* %out, i32 %offset) {
+entry:
+  %index = add i32 %offset, -512
+  %value = extractelement <4 x i32> <i32 0, i32 1, i32 2, i32 3>, i32 %index
+  store i32 %value, i32 addrspace(1)* %out
+  ret void
+}
+
+; CHECK-LABEL: {{^}}extract_neg_offset_vgpr:
+; The offset depends on the register that holds the first element of the vector.
+; CHECK: v_readfirstlane_b32
+; CHECK: s_add_i32 m0, m0, 0xfffffe{{[0-9a-z]+}}
+; CHECK-NEXT: v_movrels_b32_e32 v{{[0-9]}}, v0
+; CHECK: s_cbranch_execnz
+define void @extract_neg_offset_vgpr(i32 addrspace(1)* %out) {
+entry:
+  %id = call i32 @llvm.r600.read.tidig.x() #1
+  %index = add i32 %id, -512
+  %value = extractelement <4 x i32> <i32 0, i32 1, i32 2, i32 3>, i32 %index
+  store i32 %value, i32 addrspace(1)* %out
+  ret void
+}
+
+; CHECK-LABEL: {{^}}insert_w_offset:
+; CHECK: s_mov_b32 m0
+; CHECK-NEXT: v_movreld_b32_e32
+define void @insert_w_offset(float addrspace(1)* %out, i32 %in) {
+entry:
+  %0 = add i32 %in, 1
+  %1 = insertelement <4 x float> <float 1.0, float 2.0, float 3.0, float 4.0>, float 5.0, i32 %0
+  %2 = extractelement <4 x float> %1, i32 2
+  store float %2, float addrspace(1)* %out
+  ret void
+}
+
+; CHECK-LABEL: {{^}}insert_wo_offset:
+; CHECK: s_mov_b32 m0
+; CHECK-NEXT: v_movreld_b32_e32
+define void @insert_wo_offset(float addrspace(1)* %out, i32 %in) {
+entry:
+  %0 = insertelement <4 x float> <float 1.0, float 2.0, float 3.0, float 4.0>, float 5.0, i32 %in
+  %1 = extractelement <4 x float> %0, i32 2
+  store float %1, float addrspace(1)* %out
+  ret void
+}
+
+; CHECK-LABEL: {{^}}insert_neg_offset_sgpr:
+; The offset depends on the register that holds the first element of the vector.
+; CHECK: s_add_i32 m0, s{{[0-9]+}}, 0xfffffe{{[0-9a-z]+}}
+; CHECK: v_movreld_b32_e32 v0, v{{[0-9]}}
+define void @insert_neg_offset_sgpr(i32 addrspace(1)* %in, <4 x i32> addrspace(1)* %out, i32 %offset) {
+entry:
+  %index = add i32 %offset, -512
+  %value = insertelement <4 x i32> <i32 0, i32 1, i32 2, i32 3>, i32 5, i32 %index
+  store <4 x i32> %value, <4 x i32> addrspace(1)* %out
+  ret void
+}
+
+; CHECK-LABEL: {{^}}insert_neg_offset_vgpr:
+; The offset depends on the register that holds the first element of the vector.
+; CHECK: v_readfirstlane_b32
+; CHECK: s_add_i32 m0, m0, 0xfffffe{{[0-9a-z]+}}
+; CHECK-NEXT: v_movreld_b32_e32 v0, v{{[0-9]}}
+; CHECK: s_cbranch_execnz
+define void @insert_neg_offset_vgpr(i32 addrspace(1)* %in, <4 x i32> addrspace(1)* %out) {
+entry:
+  %id = call i32 @llvm.r600.read.tidig.x() #1
+  %index = add i32 %id, -512
+  %value = insertelement <4 x i32> <i32 0, i32 1, i32 2, i32 3>, i32 5, i32 %index
+  store <4 x i32> %value, <4 x i32> addrspace(1)* %out
+  ret void
+}
+
+; CHECK-LABEL: {{^}}insert_neg_inline_offset_vgpr:
+; The offset depends on the register that holds the first element of the vector.
+; CHECK: v_readfirstlane_b32
+; CHECK: s_add_i32 m0, m0, -{{[0-9]+}}
+; CHECK-NEXT: v_movreld_b32_e32 v0, v{{[0-9]}}
+; CHECK: s_cbranch_execnz
+define void @insert_neg_inline_offset_vgpr(i32 addrspace(1)* %in, <4 x i32> addrspace(1)* %out) {
+entry:
+  %id = call i32 @llvm.r600.read.tidig.x() #1
+  %index = add i32 %id, -16
+  %value = insertelement <4 x i32> <i32 0, i32 1, i32 2, i32 3>, i32 5, i32 %index
+  store <4 x i32> %value, <4 x i32> addrspace(1)* %out
+  ret void
+}
+
+declare i32 @llvm.r600.read.tidig.x() #1
+attributes #1 = { nounwind readnone }
diff --git a/test/CodeGen/R600/indirect-private-64.ll b/test/CodeGen/AMDGPU/indirect-private-64.ll
similarity index 100%
rename from test/CodeGen/R600/indirect-private-64.ll
rename to test/CodeGen/AMDGPU/indirect-private-64.ll
diff --git a/test/CodeGen/R600/infinite-loop-evergreen.ll b/test/CodeGen/AMDGPU/infinite-loop-evergreen.ll
similarity index 100%
rename from test/CodeGen/R600/infinite-loop-evergreen.ll
rename to test/CodeGen/AMDGPU/infinite-loop-evergreen.ll
diff --git a/test/CodeGen/R600/infinite-loop.ll b/test/CodeGen/AMDGPU/infinite-loop.ll
similarity index 100%
rename from test/CodeGen/R600/infinite-loop.ll
rename to test/CodeGen/AMDGPU/infinite-loop.ll
diff --git a/test/CodeGen/R600/inline-asm.ll b/test/CodeGen/AMDGPU/inline-asm.ll
similarity index 100%
rename from test/CodeGen/R600/inline-asm.ll
rename to test/CodeGen/AMDGPU/inline-asm.ll
diff --git a/test/CodeGen/R600/inline-calls.ll b/test/CodeGen/AMDGPU/inline-calls.ll
similarity index 100%
rename from test/CodeGen/R600/inline-calls.ll
rename to test/CodeGen/AMDGPU/inline-calls.ll
diff --git a/test/CodeGen/R600/input-mods.ll b/test/CodeGen/AMDGPU/input-mods.ll
similarity index 100%
rename from test/CodeGen/R600/input-mods.ll
rename to test/CodeGen/AMDGPU/input-mods.ll
diff --git a/test/CodeGen/R600/insert_subreg.ll b/test/CodeGen/AMDGPU/insert_subreg.ll
similarity index 100%
rename from test/CodeGen/R600/insert_subreg.ll
rename to test/CodeGen/AMDGPU/insert_subreg.ll
diff --git a/test/CodeGen/R600/insert_vector_elt.ll b/test/CodeGen/AMDGPU/insert_vector_elt.ll
similarity index 100%
rename from test/CodeGen/R600/insert_vector_elt.ll
rename to test/CodeGen/AMDGPU/insert_vector_elt.ll
diff --git a/test/CodeGen/AMDGPU/invariant-load-no-alias-store.ll b/test/CodeGen/AMDGPU/invariant-load-no-alias-store.ll
new file mode 100644
index 00000000000..2a01a621fc4
--- /dev/null
+++ b/test/CodeGen/AMDGPU/invariant-load-no-alias-store.ll
@@ -0,0 +1,35 @@
+; RUN: llc -march=amdgcn -verify-machineinstrs < %s | FileCheck -check-prefix=GCN %s
+
+; GatherAllAliases gives up on trying to analyze cases where the
+; pointer may have been loaded from an aliased store, so make sure
+; that this works and allows moving the stores to a better chain to
+; allow them to be merged merged when it's clear the pointer is loaded
+; from constant/invariant memory.
+
+; GCN-LABEL: {{^}}test_merge_store_constant_i16_invariant_global_pointer_load:
+; GCN: buffer_load_dwordx2 [[PTR:v\[[0-9]+:[0-9]+\]]],
+; GCN: v_mov_b32_e32 [[K:v[0-9]+]], 0x1c8007b
+; GCN: buffer_store_dword [[K]], [[PTR]]
+define void @test_merge_store_constant_i16_invariant_global_pointer_load(i16 addrspace(1)* addrspace(1)* dereferenceable(4096) nonnull %in) #0 {
+  %ptr = load i16 addrspace(1)*, i16 addrspace(1)* addrspace(1)* %in, !invariant.load !0
+  %ptr.1 = getelementptr i16, i16 addrspace(1)* %ptr, i64 1
+  store i16 123, i16 addrspace(1)* %ptr, align 4
+  store i16 456, i16 addrspace(1)* %ptr.1
+  ret void
+}
+
+; GCN-LABEL: {{^}}test_merge_store_constant_i16_invariant_constant_pointer_load:
+; GCN: s_load_dwordx2 s{{\[}}[[SPTR_LO:[0-9]+]]:[[SPTR_HI:[0-9]+]]{{\]}}
+; GCN: v_mov_b32_e32 [[K:v[0-9]+]], 0x1c8007b
+; GCN: buffer_store_dword [[K]], s{{\[}}[[SPTR_LO]]:
+define void @test_merge_store_constant_i16_invariant_constant_pointer_load(i16 addrspace(1)* addrspace(2)* dereferenceable(4096) nonnull %in) #0 {
+  %ptr = load i16 addrspace(1)*, i16 addrspace(1)* addrspace(2)* %in, !invariant.load !0
+  %ptr.1 = getelementptr i16, i16 addrspace(1)* %ptr, i64 1
+  store i16 123, i16 addrspace(1)* %ptr, align 4
+  store i16 456, i16 addrspace(1)* %ptr.1
+  ret void
+}
+
+!0 = !{}
+
+attributes #0 = { nounwind }
\ No newline at end of file
diff --git a/test/CodeGen/R600/jump-address.ll b/test/CodeGen/AMDGPU/jump-address.ll
similarity index 100%
rename from test/CodeGen/R600/jump-address.ll
rename to test/CodeGen/AMDGPU/jump-address.ll
diff --git a/test/CodeGen/R600/kcache-fold.ll b/test/CodeGen/AMDGPU/kcache-fold.ll
similarity index 100%
rename from test/CodeGen/R600/kcache-fold.ll
rename to test/CodeGen/AMDGPU/kcache-fold.ll
diff --git a/test/CodeGen/R600/kernel-args.ll b/test/CodeGen/AMDGPU/kernel-args.ll
similarity index 96%
rename from test/CodeGen/R600/kernel-args.ll
rename to test/CodeGen/AMDGPU/kernel-args.ll
index 5db45ce7e62..1dd7c2cb799 100644
--- a/test/CodeGen/R600/kernel-args.ll
+++ b/test/CodeGen/AMDGPU/kernel-args.ll
@@ -451,6 +451,18 @@ define void @kernel_arg_i64(i64 addrspace(1)* %out, i64 %a) nounwind {
   ret void
 }
 
+; FUNC-LABEL: {{^}}f64_kernel_arg:
+; SI-DAG: s_load_dwordx2 s[{{[0-9]:[0-9]}}], s[0:1], 0x9
+; SI-DAG: s_load_dwordx2 s[{{[0-9]:[0-9]}}], s[0:1], 0xb
+; VI-DAG: s_load_dwordx2 s[{{[0-9]:[0-9]}}], s[0:1], 0x24
+; VI-DAG: s_load_dwordx2 s[{{[0-9]:[0-9]}}], s[0:1], 0x2c
+; GCN: buffer_store_dwordx2
+define void @f64_kernel_arg(double addrspace(1)* %out, double  %in) {
+entry:
+  store double %in, double addrspace(1)* %out
+  ret void
+}
+
 ; XFUNC-LABEL: {{^}}kernel_arg_v1i64:
 ; XGCN: s_load_dwordx2
 ; XGCN: s_load_dwordx2
diff --git a/test/CodeGen/R600/large-alloca.ll b/test/CodeGen/AMDGPU/large-alloca.ll
similarity index 100%
rename from test/CodeGen/R600/large-alloca.ll
rename to test/CodeGen/AMDGPU/large-alloca.ll
diff --git a/test/CodeGen/R600/large-constant-initializer.ll b/test/CodeGen/AMDGPU/large-constant-initializer.ll
similarity index 100%
rename from test/CodeGen/R600/large-constant-initializer.ll
rename to test/CodeGen/AMDGPU/large-constant-initializer.ll
diff --git a/test/CodeGen/R600/lds-initializer.ll b/test/CodeGen/AMDGPU/lds-initializer.ll
similarity index 100%
rename from test/CodeGen/R600/lds-initializer.ll
rename to test/CodeGen/AMDGPU/lds-initializer.ll
diff --git a/test/CodeGen/R600/lds-oqap-crash.ll b/test/CodeGen/AMDGPU/lds-oqap-crash.ll
similarity index 100%
rename from test/CodeGen/R600/lds-oqap-crash.ll
rename to test/CodeGen/AMDGPU/lds-oqap-crash.ll
diff --git a/test/CodeGen/R600/lds-output-queue.ll b/test/CodeGen/AMDGPU/lds-output-queue.ll
similarity index 100%
rename from test/CodeGen/R600/lds-output-queue.ll
rename to test/CodeGen/AMDGPU/lds-output-queue.ll
diff --git a/test/CodeGen/R600/lds-size.ll b/test/CodeGen/AMDGPU/lds-size.ll
similarity index 100%
rename from test/CodeGen/R600/lds-size.ll
rename to test/CodeGen/AMDGPU/lds-size.ll
diff --git a/test/CodeGen/R600/lds-zero-initializer.ll b/test/CodeGen/AMDGPU/lds-zero-initializer.ll
similarity index 100%
rename from test/CodeGen/R600/lds-zero-initializer.ll
rename to test/CodeGen/AMDGPU/lds-zero-initializer.ll
diff --git a/test/CodeGen/R600/legalizedag-bug-expand-setcc.ll b/test/CodeGen/AMDGPU/legalizedag-bug-expand-setcc.ll
similarity index 100%
rename from test/CodeGen/R600/legalizedag-bug-expand-setcc.ll
rename to test/CodeGen/AMDGPU/legalizedag-bug-expand-setcc.ll
diff --git a/test/CodeGen/AMDGPU/lit.local.cfg b/test/CodeGen/AMDGPU/lit.local.cfg
new file mode 100644
index 00000000000..2a665f06be7
--- /dev/null
+++ b/test/CodeGen/AMDGPU/lit.local.cfg
@@ -0,0 +1,2 @@
+if not 'AMDGPU' in config.root.targets:
+    config.unsupported = True
diff --git a/test/CodeGen/R600/literals.ll b/test/CodeGen/AMDGPU/literals.ll
similarity index 100%
rename from test/CodeGen/R600/literals.ll
rename to test/CodeGen/AMDGPU/literals.ll
diff --git a/test/CodeGen/R600/llvm.AMDGPU.abs.ll b/test/CodeGen/AMDGPU/llvm.AMDGPU.abs.ll
similarity index 100%
rename from test/CodeGen/R600/llvm.AMDGPU.abs.ll
rename to test/CodeGen/AMDGPU/llvm.AMDGPU.abs.ll
diff --git a/test/CodeGen/R600/llvm.AMDGPU.barrier.global.ll b/test/CodeGen/AMDGPU/llvm.AMDGPU.barrier.global.ll
similarity index 100%
rename from test/CodeGen/R600/llvm.AMDGPU.barrier.global.ll
rename to test/CodeGen/AMDGPU/llvm.AMDGPU.barrier.global.ll
diff --git a/test/CodeGen/R600/llvm.AMDGPU.barrier.local.ll b/test/CodeGen/AMDGPU/llvm.AMDGPU.barrier.local.ll
similarity index 100%
rename from test/CodeGen/R600/llvm.AMDGPU.barrier.local.ll
rename to test/CodeGen/AMDGPU/llvm.AMDGPU.barrier.local.ll
diff --git a/test/CodeGen/R600/llvm.AMDGPU.bfe.i32.ll b/test/CodeGen/AMDGPU/llvm.AMDGPU.bfe.i32.ll
similarity index 100%
rename from test/CodeGen/R600/llvm.AMDGPU.bfe.i32.ll
rename to test/CodeGen/AMDGPU/llvm.AMDGPU.bfe.i32.ll
diff --git a/test/CodeGen/R600/llvm.AMDGPU.bfe.u32.ll b/test/CodeGen/AMDGPU/llvm.AMDGPU.bfe.u32.ll
similarity index 100%
rename from test/CodeGen/R600/llvm.AMDGPU.bfe.u32.ll
rename to test/CodeGen/AMDGPU/llvm.AMDGPU.bfe.u32.ll
diff --git a/test/CodeGen/R600/llvm.AMDGPU.bfi.ll b/test/CodeGen/AMDGPU/llvm.AMDGPU.bfi.ll
similarity index 100%
rename from test/CodeGen/R600/llvm.AMDGPU.bfi.ll
rename to test/CodeGen/AMDGPU/llvm.AMDGPU.bfi.ll
diff --git a/test/CodeGen/R600/llvm.AMDGPU.bfm.ll b/test/CodeGen/AMDGPU/llvm.AMDGPU.bfm.ll
similarity index 100%
rename from test/CodeGen/R600/llvm.AMDGPU.bfm.ll
rename to test/CodeGen/AMDGPU/llvm.AMDGPU.bfm.ll
diff --git a/test/CodeGen/R600/llvm.AMDGPU.brev.ll b/test/CodeGen/AMDGPU/llvm.AMDGPU.brev.ll
similarity index 100%
rename from test/CodeGen/R600/llvm.AMDGPU.brev.ll
rename to test/CodeGen/AMDGPU/llvm.AMDGPU.brev.ll
diff --git a/test/CodeGen/R600/llvm.AMDGPU.clamp.ll b/test/CodeGen/AMDGPU/llvm.AMDGPU.clamp.ll
similarity index 100%
rename from test/CodeGen/R600/llvm.AMDGPU.clamp.ll
rename to test/CodeGen/AMDGPU/llvm.AMDGPU.clamp.ll
diff --git a/test/CodeGen/R600/llvm.AMDGPU.class.ll b/test/CodeGen/AMDGPU/llvm.AMDGPU.class.ll
similarity index 100%
rename from test/CodeGen/R600/llvm.AMDGPU.class.ll
rename to test/CodeGen/AMDGPU/llvm.AMDGPU.class.ll
diff --git a/test/CodeGen/R600/llvm.AMDGPU.cube.ll b/test/CodeGen/AMDGPU/llvm.AMDGPU.cube.ll
similarity index 100%
rename from test/CodeGen/R600/llvm.AMDGPU.cube.ll
rename to test/CodeGen/AMDGPU/llvm.AMDGPU.cube.ll
diff --git a/test/CodeGen/R600/llvm.AMDGPU.cvt_f32_ubyte.ll b/test/CodeGen/AMDGPU/llvm.AMDGPU.cvt_f32_ubyte.ll
similarity index 100%
rename from test/CodeGen/R600/llvm.AMDGPU.cvt_f32_ubyte.ll
rename to test/CodeGen/AMDGPU/llvm.AMDGPU.cvt_f32_ubyte.ll
diff --git a/test/CodeGen/R600/llvm.AMDGPU.div_fixup.ll b/test/CodeGen/AMDGPU/llvm.AMDGPU.div_fixup.ll
similarity index 100%
rename from test/CodeGen/R600/llvm.AMDGPU.div_fixup.ll
rename to test/CodeGen/AMDGPU/llvm.AMDGPU.div_fixup.ll
diff --git a/test/CodeGen/R600/llvm.AMDGPU.div_fmas.ll b/test/CodeGen/AMDGPU/llvm.AMDGPU.div_fmas.ll
similarity index 98%
rename from test/CodeGen/R600/llvm.AMDGPU.div_fmas.ll
rename to test/CodeGen/AMDGPU/llvm.AMDGPU.div_fmas.ll
index bcb7f870f1f..f948c987b03 100644
--- a/test/CodeGen/R600/llvm.AMDGPU.div_fmas.ll
+++ b/test/CodeGen/AMDGPU/llvm.AMDGPU.div_fmas.ll
@@ -1,5 +1,5 @@
-; RUN: llc -march=amdgcn -mcpu=SI -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=SI %s
-; XUN: llc -march=amdgcn -mcpu=tonga -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=VI %s
+; RUN: llc -march=amdgcn -mcpu=SI -verify-machineinstrs < %s | FileCheck -strict-whitespace -check-prefix=GCN -check-prefix=SI %s
+; XUN: llc -march=amdgcn -mcpu=tonga -verify-machineinstrs < %s | FileCheck -strict-whitespace -check-prefix=GCN -check-prefix=VI %s
 
 ; FIXME: Enable for VI.
 
diff --git a/test/CodeGen/R600/llvm.AMDGPU.div_scale.ll b/test/CodeGen/AMDGPU/llvm.AMDGPU.div_scale.ll
similarity index 100%
rename from test/CodeGen/R600/llvm.AMDGPU.div_scale.ll
rename to test/CodeGen/AMDGPU/llvm.AMDGPU.div_scale.ll
diff --git a/test/CodeGen/R600/llvm.AMDGPU.flbit.i32.ll b/test/CodeGen/AMDGPU/llvm.AMDGPU.flbit.i32.ll
similarity index 100%
rename from test/CodeGen/R600/llvm.AMDGPU.flbit.i32.ll
rename to test/CodeGen/AMDGPU/llvm.AMDGPU.flbit.i32.ll
diff --git a/test/CodeGen/R600/llvm.AMDGPU.fract.f64.ll b/test/CodeGen/AMDGPU/llvm.AMDGPU.fract.f64.ll
similarity index 88%
rename from test/CodeGen/R600/llvm.AMDGPU.fract.f64.ll
rename to test/CodeGen/AMDGPU/llvm.AMDGPU.fract.f64.ll
index e098dd35d6d..6049dca0401 100644
--- a/test/CodeGen/R600/llvm.AMDGPU.fract.f64.ll
+++ b/test/CodeGen/AMDGPU/llvm.AMDGPU.fract.f64.ll
@@ -11,8 +11,8 @@ declare double @llvm.AMDGPU.fract.f64(double) nounwind readnone
 ; SI: v_mov_b32_e32 v[[UPHI:[0-9]+]], 0x3fefffff
 ; SI: v_min_f64 v{{\[}}[[MINLO:[0-9]+]]:[[MINHI:[0-9]+]]], v{{\[}}[[UPLO]]:[[UPHI]]], [[FRC]]
 ; SI: v_cmp_class_f64_e64 [[COND:s\[[0-9]+:[0-9]+\]]], v{{\[}}[[LO]]:[[HI]]], 3
-; SI: v_cndmask_b32_e64 v[[RESLO:[0-9]+]], v[[LO]], v[[MINLO]], [[COND]]
-; SI: v_cndmask_b32_e64 v[[RESHI:[0-9]+]], v[[HI]], v[[MINHI]], [[COND]]
+; SI: v_cndmask_b32_e64 v[[RESLO:[0-9]+]], v[[MINLO]], v[[LO]], [[COND]]
+; SI: v_cndmask_b32_e64 v[[RESHI:[0-9]+]], v[[MINHI]], v[[HI]], [[COND]]
 ; SI: buffer_store_dwordx2 v{{\[}}[[RESLO]]:[[RESHI]]]
 ; CI: buffer_store_dwordx2 [[FRC]]
 define void @fract_f64(double addrspace(1)* %out, double addrspace(1)* %src) nounwind {
@@ -28,8 +28,8 @@ define void @fract_f64(double addrspace(1)* %out, double addrspace(1)* %src) nou
 ; SI: v_mov_b32_e32 v[[UPHI:[0-9]+]], 0x3fefffff
 ; SI: v_min_f64 v{{\[}}[[MINLO:[0-9]+]]:[[MINHI:[0-9]+]]], v{{\[}}[[UPLO]]:[[UPHI]]], [[FRC]]
 ; SI: v_cmp_class_f64_e64 [[COND:s\[[0-9]+:[0-9]+\]]], v{{\[}}[[LO]]:[[HI]]], 3
-; SI: v_cndmask_b32_e64 v[[RESLO:[0-9]+]], v[[LO]], v[[MINLO]], [[COND]]
-; SI: v_cndmask_b32_e64 v[[RESHI:[0-9]+]], v[[HI]], v[[MINHI]], [[COND]]
+; SI: v_cndmask_b32_e64 v[[RESLO:[0-9]+]], v[[MINLO]], v[[LO]], [[COND]]
+; SI: v_cndmask_b32_e64 v[[RESHI:[0-9]+]], v[[MINHI]], v[[HI]], [[COND]]
 ; SI: buffer_store_dwordx2 v{{\[}}[[RESLO]]:[[RESHI]]]
 ; CI: buffer_store_dwordx2 [[FRC]]
 define void @fract_f64_neg(double addrspace(1)* %out, double addrspace(1)* %src) nounwind {
@@ -46,8 +46,8 @@ define void @fract_f64_neg(double addrspace(1)* %out, double addrspace(1)* %src)
 ; SI: v_mov_b32_e32 v[[UPHI:[0-9]+]], 0x3fefffff
 ; SI: v_min_f64 v{{\[}}[[MINLO:[0-9]+]]:[[MINHI:[0-9]+]]], v{{\[}}[[UPLO]]:[[UPHI]]], [[FRC]]
 ; SI: v_cmp_class_f64_e64 [[COND:s\[[0-9]+:[0-9]+\]]], v{{\[}}[[LO]]:[[HI]]], 3
-; SI: v_cndmask_b32_e64 v[[RESLO:[0-9]+]], v[[LO]], v[[MINLO]], [[COND]]
-; SI: v_cndmask_b32_e64 v[[RESHI:[0-9]+]], v[[HI]], v[[MINHI]], [[COND]]
+; SI: v_cndmask_b32_e64 v[[RESLO:[0-9]+]], v[[MINLO]], v[[LO]], [[COND]]
+; SI: v_cndmask_b32_e64 v[[RESHI:[0-9]+]], v[[MINHI]], v[[HI]], [[COND]]
 ; SI: buffer_store_dwordx2 v{{\[}}[[RESLO]]:[[RESHI]]]
 ; CI: buffer_store_dwordx2 [[FRC]]
 define void @fract_f64_neg_abs(double addrspace(1)* %out, double addrspace(1)* %src) nounwind {
diff --git a/test/CodeGen/R600/llvm.AMDGPU.fract.ll b/test/CodeGen/AMDGPU/llvm.AMDGPU.fract.ll
similarity index 100%
rename from test/CodeGen/R600/llvm.AMDGPU.fract.ll
rename to test/CodeGen/AMDGPU/llvm.AMDGPU.fract.ll
diff --git a/test/CodeGen/R600/llvm.AMDGPU.imad24.ll b/test/CodeGen/AMDGPU/llvm.AMDGPU.imad24.ll
similarity index 100%
rename from test/CodeGen/R600/llvm.AMDGPU.imad24.ll
rename to test/CodeGen/AMDGPU/llvm.AMDGPU.imad24.ll
diff --git a/test/CodeGen/R600/llvm.AMDGPU.imax.ll b/test/CodeGen/AMDGPU/llvm.AMDGPU.imax.ll
similarity index 100%
rename from test/CodeGen/R600/llvm.AMDGPU.imax.ll
rename to test/CodeGen/AMDGPU/llvm.AMDGPU.imax.ll
diff --git a/test/CodeGen/R600/llvm.AMDGPU.imin.ll b/test/CodeGen/AMDGPU/llvm.AMDGPU.imin.ll
similarity index 100%
rename from test/CodeGen/R600/llvm.AMDGPU.imin.ll
rename to test/CodeGen/AMDGPU/llvm.AMDGPU.imin.ll
diff --git a/test/CodeGen/R600/llvm.AMDGPU.imul24.ll b/test/CodeGen/AMDGPU/llvm.AMDGPU.imul24.ll
similarity index 100%
rename from test/CodeGen/R600/llvm.AMDGPU.imul24.ll
rename to test/CodeGen/AMDGPU/llvm.AMDGPU.imul24.ll
diff --git a/test/CodeGen/AMDGPU/llvm.AMDGPU.kill.ll b/test/CodeGen/AMDGPU/llvm.AMDGPU.kill.ll
new file mode 100644
index 00000000000..057708e7b5c
--- /dev/null
+++ b/test/CodeGen/AMDGPU/llvm.AMDGPU.kill.ll
@@ -0,0 +1,39 @@
+; RUN: llc -march=amdgcn -mcpu=verde -verify-machineinstrs < %s | FileCheck -check-prefix=SI %s
+; RUN: llc -march=amdgcn -mcpu=tonga -verify-machineinstrs < %s | FileCheck -check-prefix=SI %s
+
+; SI-LABEL: {{^}}kill_gs_const:
+; SI-NOT: v_cmpx_le_f32
+; SI: s_mov_b64 exec, 0
+
+define void @kill_gs_const() #0 {
+main_body:
+  %0 = icmp ule i32 0, 3
+  %1 = select i1 %0, float 1.000000e+00, float -1.000000e+00
+  call void @llvm.AMDGPU.kill(float %1)
+  %2 = icmp ule i32 3, 0
+  %3 = select i1 %2, float 1.000000e+00, float -1.000000e+00
+  call void @llvm.AMDGPU.kill(float %3)
+  ret void
+}
+
+; SI-LABEL: {{^}}kill_vcc_implicit_def:
+; SI-NOT: v_cmp_gt_f32_e32 vcc,
+; SI: v_cmp_gt_f32_e64 [[CMP:s\[[0-9]+:[0-9]+\]]], 0, v{{[0-9]+}}
+; SI: v_cmpx_le_f32_e32 vcc, 0, v{{[0-9]+}}
+; SI: v_cndmask_b32_e64 v{{[0-9]+}}, 0, 1.0, [[CMP]]
+define void @kill_vcc_implicit_def([6 x <16 x i8>] addrspace(2)* byval, [17 x <16 x i8>] addrspace(2)* byval, [17 x <4 x i32>] addrspace(2)* byval, [34 x <8 x i32>] addrspace(2)* byval, float inreg, i32 inreg, <2 x i32>, <2 x i32>, <2 x i32>, <3 x i32>, <2 x i32>, <2 x i32>, <2 x i32>, float, float, float, float, float, float, i32, float, float) #1 {
+entry:
+  %tmp0 = fcmp olt float %13, 0.0
+  call void @llvm.AMDGPU.kill(float %14)
+  %tmp1 = select i1 %tmp0, float 1.0, float 0.0
+  call void @llvm.SI.export(i32 15, i32 1, i32 1, i32 1, i32 1, float %tmp1, float %tmp1, float %tmp1, float %tmp1)
+  ret void
+}
+
+declare void @llvm.AMDGPU.kill(float)
+declare void @llvm.SI.export(i32, i32, i32, i32, i32, float, float, float, float)
+
+attributes #0 = { "ShaderType"="2" }
+attributes #1 = { "ShaderType"="0" }
+
+!0 = !{!"const", null, i32 1}
diff --git a/test/CodeGen/R600/llvm.AMDGPU.ldexp.ll b/test/CodeGen/AMDGPU/llvm.AMDGPU.ldexp.ll
similarity index 100%
rename from test/CodeGen/R600/llvm.AMDGPU.ldexp.ll
rename to test/CodeGen/AMDGPU/llvm.AMDGPU.ldexp.ll
diff --git a/test/CodeGen/R600/llvm.AMDGPU.legacy.rsq.ll b/test/CodeGen/AMDGPU/llvm.AMDGPU.legacy.rsq.ll
similarity index 100%
rename from test/CodeGen/R600/llvm.AMDGPU.legacy.rsq.ll
rename to test/CodeGen/AMDGPU/llvm.AMDGPU.legacy.rsq.ll
diff --git a/test/CodeGen/R600/llvm.AMDGPU.mul.ll b/test/CodeGen/AMDGPU/llvm.AMDGPU.mul.ll
similarity index 100%
rename from test/CodeGen/R600/llvm.AMDGPU.mul.ll
rename to test/CodeGen/AMDGPU/llvm.AMDGPU.mul.ll
diff --git a/test/CodeGen/R600/llvm.AMDGPU.rcp.f64.ll b/test/CodeGen/AMDGPU/llvm.AMDGPU.rcp.f64.ll
similarity index 100%
rename from test/CodeGen/R600/llvm.AMDGPU.rcp.f64.ll
rename to test/CodeGen/AMDGPU/llvm.AMDGPU.rcp.f64.ll
diff --git a/test/CodeGen/R600/llvm.AMDGPU.rcp.ll b/test/CodeGen/AMDGPU/llvm.AMDGPU.rcp.ll
similarity index 100%
rename from test/CodeGen/R600/llvm.AMDGPU.rcp.ll
rename to test/CodeGen/AMDGPU/llvm.AMDGPU.rcp.ll
diff --git a/test/CodeGen/R600/llvm.AMDGPU.rsq.clamped.f64.ll b/test/CodeGen/AMDGPU/llvm.AMDGPU.rsq.clamped.f64.ll
similarity index 100%
rename from test/CodeGen/R600/llvm.AMDGPU.rsq.clamped.f64.ll
rename to test/CodeGen/AMDGPU/llvm.AMDGPU.rsq.clamped.f64.ll
diff --git a/test/CodeGen/R600/llvm.AMDGPU.rsq.clamped.ll b/test/CodeGen/AMDGPU/llvm.AMDGPU.rsq.clamped.ll
similarity index 100%
rename from test/CodeGen/R600/llvm.AMDGPU.rsq.clamped.ll
rename to test/CodeGen/AMDGPU/llvm.AMDGPU.rsq.clamped.ll
diff --git a/test/CodeGen/R600/llvm.AMDGPU.rsq.ll b/test/CodeGen/AMDGPU/llvm.AMDGPU.rsq.ll
similarity index 100%
rename from test/CodeGen/R600/llvm.AMDGPU.rsq.ll
rename to test/CodeGen/AMDGPU/llvm.AMDGPU.rsq.ll
diff --git a/test/CodeGen/R600/llvm.AMDGPU.tex.ll b/test/CodeGen/AMDGPU/llvm.AMDGPU.tex.ll
similarity index 100%
rename from test/CodeGen/R600/llvm.AMDGPU.tex.ll
rename to test/CodeGen/AMDGPU/llvm.AMDGPU.tex.ll
diff --git a/test/CodeGen/R600/llvm.AMDGPU.trig_preop.ll b/test/CodeGen/AMDGPU/llvm.AMDGPU.trig_preop.ll
similarity index 100%
rename from test/CodeGen/R600/llvm.AMDGPU.trig_preop.ll
rename to test/CodeGen/AMDGPU/llvm.AMDGPU.trig_preop.ll
diff --git a/test/CodeGen/R600/llvm.AMDGPU.trunc.ll b/test/CodeGen/AMDGPU/llvm.AMDGPU.trunc.ll
similarity index 100%
rename from test/CodeGen/R600/llvm.AMDGPU.trunc.ll
rename to test/CodeGen/AMDGPU/llvm.AMDGPU.trunc.ll
diff --git a/test/CodeGen/R600/llvm.AMDGPU.umad24.ll b/test/CodeGen/AMDGPU/llvm.AMDGPU.umad24.ll
similarity index 100%
rename from test/CodeGen/R600/llvm.AMDGPU.umad24.ll
rename to test/CodeGen/AMDGPU/llvm.AMDGPU.umad24.ll
diff --git a/test/CodeGen/R600/llvm.AMDGPU.umax.ll b/test/CodeGen/AMDGPU/llvm.AMDGPU.umax.ll
similarity index 100%
rename from test/CodeGen/R600/llvm.AMDGPU.umax.ll
rename to test/CodeGen/AMDGPU/llvm.AMDGPU.umax.ll
diff --git a/test/CodeGen/R600/llvm.AMDGPU.umin.ll b/test/CodeGen/AMDGPU/llvm.AMDGPU.umin.ll
similarity index 100%
rename from test/CodeGen/R600/llvm.AMDGPU.umin.ll
rename to test/CodeGen/AMDGPU/llvm.AMDGPU.umin.ll
diff --git a/test/CodeGen/R600/llvm.AMDGPU.umul24.ll b/test/CodeGen/AMDGPU/llvm.AMDGPU.umul24.ll
similarity index 100%
rename from test/CodeGen/R600/llvm.AMDGPU.umul24.ll
rename to test/CodeGen/AMDGPU/llvm.AMDGPU.umul24.ll
diff --git a/test/CodeGen/AMDGPU/llvm.SI.fs.interp.ll b/test/CodeGen/AMDGPU/llvm.SI.fs.interp.ll
new file mode 100644
index 00000000000..3d05da616e4
--- /dev/null
+++ b/test/CodeGen/AMDGPU/llvm.SI.fs.interp.ll
@@ -0,0 +1,59 @@
+;RUN: llc < %s -march=amdgcn -mcpu=verde -verify-machineinstrs | FileCheck --check-prefix=GCN %s
+;RUN: llc < %s -march=amdgcn -mcpu=kabini -verify-machineinstrs | FileCheck --check-prefix=GCN --check-prefix=16BANK %s
+;RUN: llc < %s -march=amdgcn -mcpu=tonga -verify-machineinstrs | FileCheck --check-prefix=GCN %s
+
+;GCN-LABEL: {{^}}main:
+;GCN-NOT: s_wqm
+;GCN: s_mov_b32
+;GCN-NEXT: v_interp_mov_f32
+;GCN: v_interp_p1_f32
+;GCN: v_interp_p2_f32
+
+define void @main(<16 x i8> addrspace(2)* inreg, <16 x i8> addrspace(2)* inreg, <32 x i8> addrspace(2)* inreg, i32 inreg, <2 x i32>) #0 {
+main_body:
+  %5 = call float @llvm.SI.fs.constant(i32 0, i32 0, i32 %3)
+  %6 = call float @llvm.SI.fs.interp(i32 0, i32 0, i32 %3, <2 x i32> %4)
+  %7 = call float @llvm.SI.fs.interp(i32 1, i32 0, i32 %3, <2 x i32> %4)
+  call void @llvm.SI.export(i32 15, i32 1, i32 1, i32 0, i32 1, float %5, float %6, float %7, float %7)
+  ret void
+}
+
+; Thest that v_interp_p1 uses different source and destination registers
+; on 16 bank LDS chips.
+
+; 16BANK-LABEL: {{^}}v_interp_p1_bank16_bug:
+; 16BANK-NOT: v_interp_p1_f32 [[DST:v[0-9]+]], [[DST]]
+
+define void @v_interp_p1_bank16_bug([6 x <16 x i8>] addrspace(2)* byval, [17 x <16 x i8>] addrspace(2)* byval, [17 x <4 x i32>] addrspace(2)* byval, [34 x <8 x i32>] addrspace(2)* byval, float inreg, i32 inreg, <2 x i32>, <2 x i32>, <2 x i32>, <3 x i32>, <2 x i32>, <2 x i32>, <2 x i32>, float, float, float, float, float, float, i32, float, float) #0 {
+main_body:
+  %22 = call float @llvm.SI.fs.interp(i32 0, i32 0, i32 %5, <2 x i32> %7)
+  %23 = call float @llvm.SI.fs.interp(i32 1, i32 0, i32 %5, <2 x i32> %7)
+  %24 = call float @llvm.SI.fs.interp(i32 2, i32 0, i32 %5, <2 x i32> %7)
+  %25 = call float @fabs(float %22)
+  %26 = call float @fabs(float %23)
+  %27 = call float @fabs(float %24)
+  %28 = call i32 @llvm.SI.packf16(float %25, float %26)
+  %29 = bitcast i32 %28 to float
+  %30 = call i32 @llvm.SI.packf16(float %27, float 1.000000e+00)
+  %31 = bitcast i32 %30 to float
+  call void @llvm.SI.export(i32 15, i32 1, i32 1, i32 0, i32 1, float %29, float %31, float %29, float %31)
+  ret void
+}
+
+; Function Attrs: readnone
+declare float @fabs(float) #2
+
+; Function Attrs: nounwind readnone
+declare i32 @llvm.SI.packf16(float, float) #1
+
+; Function Attrs: nounwind readnone
+declare float @llvm.SI.fs.constant(i32, i32, i32) #1
+
+; Function Attrs: nounwind readnone
+declare float @llvm.SI.fs.interp(i32, i32, i32, <2 x i32>) #1
+
+declare void @llvm.SI.export(i32, i32, i32, i32, i32, float, float, float, float)
+
+attributes #0 = { "ShaderType"="0" }
+attributes #1 = { nounwind readnone }
+attributes #2 = { readnone }
diff --git a/test/CodeGen/R600/llvm.SI.gather4.ll b/test/CodeGen/AMDGPU/llvm.SI.gather4.ll
similarity index 100%
rename from test/CodeGen/R600/llvm.SI.gather4.ll
rename to test/CodeGen/AMDGPU/llvm.SI.gather4.ll
diff --git a/test/CodeGen/R600/llvm.SI.getlod.ll b/test/CodeGen/AMDGPU/llvm.SI.getlod.ll
similarity index 100%
rename from test/CodeGen/R600/llvm.SI.getlod.ll
rename to test/CodeGen/AMDGPU/llvm.SI.getlod.ll
diff --git a/test/CodeGen/R600/llvm.SI.image.ll b/test/CodeGen/AMDGPU/llvm.SI.image.ll
similarity index 100%
rename from test/CodeGen/R600/llvm.SI.image.ll
rename to test/CodeGen/AMDGPU/llvm.SI.image.ll
diff --git a/test/CodeGen/R600/llvm.SI.image.sample.ll b/test/CodeGen/AMDGPU/llvm.SI.image.sample.ll
similarity index 100%
rename from test/CodeGen/R600/llvm.SI.image.sample.ll
rename to test/CodeGen/AMDGPU/llvm.SI.image.sample.ll
diff --git a/test/CodeGen/R600/llvm.SI.image.sample.o.ll b/test/CodeGen/AMDGPU/llvm.SI.image.sample.o.ll
similarity index 100%
rename from test/CodeGen/R600/llvm.SI.image.sample.o.ll
rename to test/CodeGen/AMDGPU/llvm.SI.image.sample.o.ll
diff --git a/test/CodeGen/R600/llvm.SI.imageload.ll b/test/CodeGen/AMDGPU/llvm.SI.imageload.ll
similarity index 100%
rename from test/CodeGen/R600/llvm.SI.imageload.ll
rename to test/CodeGen/AMDGPU/llvm.SI.imageload.ll
diff --git a/test/CodeGen/R600/llvm.SI.load.dword.ll b/test/CodeGen/AMDGPU/llvm.SI.load.dword.ll
similarity index 100%
rename from test/CodeGen/R600/llvm.SI.load.dword.ll
rename to test/CodeGen/AMDGPU/llvm.SI.load.dword.ll
diff --git a/test/CodeGen/R600/llvm.SI.resinfo.ll b/test/CodeGen/AMDGPU/llvm.SI.resinfo.ll
similarity index 100%
rename from test/CodeGen/R600/llvm.SI.resinfo.ll
rename to test/CodeGen/AMDGPU/llvm.SI.resinfo.ll
diff --git a/test/CodeGen/R600/llvm.SI.sample-masked.ll b/test/CodeGen/AMDGPU/llvm.SI.sample-masked.ll
similarity index 100%
rename from test/CodeGen/R600/llvm.SI.sample-masked.ll
rename to test/CodeGen/AMDGPU/llvm.SI.sample-masked.ll
diff --git a/test/CodeGen/R600/llvm.SI.sample.ll b/test/CodeGen/AMDGPU/llvm.SI.sample.ll
similarity index 100%
rename from test/CodeGen/R600/llvm.SI.sample.ll
rename to test/CodeGen/AMDGPU/llvm.SI.sample.ll
diff --git a/test/CodeGen/R600/llvm.SI.sampled.ll b/test/CodeGen/AMDGPU/llvm.SI.sampled.ll
similarity index 100%
rename from test/CodeGen/R600/llvm.SI.sampled.ll
rename to test/CodeGen/AMDGPU/llvm.SI.sampled.ll
diff --git a/test/CodeGen/R600/llvm.SI.sendmsg-m0.ll b/test/CodeGen/AMDGPU/llvm.SI.sendmsg-m0.ll
similarity index 100%
rename from test/CodeGen/R600/llvm.SI.sendmsg-m0.ll
rename to test/CodeGen/AMDGPU/llvm.SI.sendmsg-m0.ll
diff --git a/test/CodeGen/R600/llvm.SI.sendmsg.ll b/test/CodeGen/AMDGPU/llvm.SI.sendmsg.ll
similarity index 92%
rename from test/CodeGen/R600/llvm.SI.sendmsg.ll
rename to test/CodeGen/AMDGPU/llvm.SI.sendmsg.ll
index ce380024195..09675d50335 100644
--- a/test/CodeGen/R600/llvm.SI.sendmsg.ll
+++ b/test/CodeGen/AMDGPU/llvm.SI.sendmsg.ll
@@ -2,6 +2,8 @@
 ;RUN: llc < %s -march=amdgcn -mcpu=tonga -verify-machineinstrs | FileCheck %s
 
 ; CHECK-LABEL: {{^}}main:
+; CHECK: s_mov_b32 m0, 0
+; CHECK-NOT: s_mov_b32 m0
 ; CHECK: s_sendmsg Gs(emit stream 0)
 ; CHECK: s_sendmsg Gs(cut stream 1)
 ; CHECK: s_sendmsg Gs(emit-cut stream 2)
diff --git a/test/CodeGen/R600/llvm.SI.tbuffer.store.ll b/test/CodeGen/AMDGPU/llvm.SI.tbuffer.store.ll
similarity index 100%
rename from test/CodeGen/R600/llvm.SI.tbuffer.store.ll
rename to test/CodeGen/AMDGPU/llvm.SI.tbuffer.store.ll
diff --git a/test/CodeGen/R600/llvm.SI.tid.ll b/test/CodeGen/AMDGPU/llvm.SI.tid.ll
similarity index 100%
rename from test/CodeGen/R600/llvm.SI.tid.ll
rename to test/CodeGen/AMDGPU/llvm.SI.tid.ll
diff --git a/test/CodeGen/R600/llvm.amdgpu.dp4.ll b/test/CodeGen/AMDGPU/llvm.amdgpu.dp4.ll
similarity index 100%
rename from test/CodeGen/R600/llvm.amdgpu.dp4.ll
rename to test/CodeGen/AMDGPU/llvm.amdgpu.dp4.ll
diff --git a/test/CodeGen/R600/llvm.amdgpu.kilp.ll b/test/CodeGen/AMDGPU/llvm.amdgpu.kilp.ll
similarity index 100%
rename from test/CodeGen/R600/llvm.amdgpu.kilp.ll
rename to test/CodeGen/AMDGPU/llvm.amdgpu.kilp.ll
diff --git a/test/CodeGen/R600/llvm.amdgpu.lrp.ll b/test/CodeGen/AMDGPU/llvm.amdgpu.lrp.ll
similarity index 96%
rename from test/CodeGen/R600/llvm.amdgpu.lrp.ll
rename to test/CodeGen/AMDGPU/llvm.amdgpu.lrp.ll
index 4e4c2ec7791..a64dd0ebd2d 100644
--- a/test/CodeGen/R600/llvm.amdgpu.lrp.ll
+++ b/test/CodeGen/AMDGPU/llvm.amdgpu.lrp.ll
@@ -5,7 +5,7 @@ declare float @llvm.AMDGPU.lrp(float, float, float) nounwind readnone
 
 ; FUNC-LABEL: {{^}}test_lrp:
 ; SI: v_sub_f32
-; SI: v_mad_f32
+; SI: v_mac_f32_e32
 define void @test_lrp(float addrspace(1)* %out, float %src0, float %src1, float %src2) nounwind {
   %mad = call float @llvm.AMDGPU.lrp(float %src0, float %src1, float %src2) nounwind readnone
   store float %mad, float addrspace(1)* %out, align 4
diff --git a/test/CodeGen/R600/llvm.cos.ll b/test/CodeGen/AMDGPU/llvm.cos.ll
similarity index 100%
rename from test/CodeGen/R600/llvm.cos.ll
rename to test/CodeGen/AMDGPU/llvm.cos.ll
diff --git a/test/CodeGen/R600/llvm.exp2.ll b/test/CodeGen/AMDGPU/llvm.exp2.ll
similarity index 100%
rename from test/CodeGen/R600/llvm.exp2.ll
rename to test/CodeGen/AMDGPU/llvm.exp2.ll
diff --git a/test/CodeGen/R600/llvm.log2.ll b/test/CodeGen/AMDGPU/llvm.log2.ll
similarity index 100%
rename from test/CodeGen/R600/llvm.log2.ll
rename to test/CodeGen/AMDGPU/llvm.log2.ll
diff --git a/test/CodeGen/R600/llvm.memcpy.ll b/test/CodeGen/AMDGPU/llvm.memcpy.ll
similarity index 100%
rename from test/CodeGen/R600/llvm.memcpy.ll
rename to test/CodeGen/AMDGPU/llvm.memcpy.ll
diff --git a/test/CodeGen/R600/llvm.pow.ll b/test/CodeGen/AMDGPU/llvm.pow.ll
similarity index 100%
rename from test/CodeGen/R600/llvm.pow.ll
rename to test/CodeGen/AMDGPU/llvm.pow.ll
diff --git a/test/CodeGen/R600/llvm.rint.f64.ll b/test/CodeGen/AMDGPU/llvm.rint.f64.ll
similarity index 100%
rename from test/CodeGen/R600/llvm.rint.f64.ll
rename to test/CodeGen/AMDGPU/llvm.rint.f64.ll
diff --git a/test/CodeGen/R600/llvm.rint.ll b/test/CodeGen/AMDGPU/llvm.rint.ll
similarity index 100%
rename from test/CodeGen/R600/llvm.rint.ll
rename to test/CodeGen/AMDGPU/llvm.rint.ll
diff --git a/test/CodeGen/R600/llvm.round.f64.ll b/test/CodeGen/AMDGPU/llvm.round.f64.ll
similarity index 100%
rename from test/CodeGen/R600/llvm.round.f64.ll
rename to test/CodeGen/AMDGPU/llvm.round.f64.ll
diff --git a/test/CodeGen/R600/llvm.round.ll b/test/CodeGen/AMDGPU/llvm.round.ll
similarity index 94%
rename from test/CodeGen/R600/llvm.round.ll
rename to test/CodeGen/AMDGPU/llvm.round.ll
index f5f124d915a..d0e49243ffa 100644
--- a/test/CodeGen/R600/llvm.round.ll
+++ b/test/CodeGen/AMDGPU/llvm.round.ll
@@ -9,8 +9,8 @@
 ; SI: v_sub_f32_e32 [[SUB:v[0-9]+]], [[SX]], [[TRUNC]]
 ; SI: v_mov_b32_e32 [[VX:v[0-9]+]], [[SX]]
 ; SI: v_bfi_b32 [[COPYSIGN:v[0-9]+]], [[K]], 1.0, [[VX]]
-; SI: v_cmp_le_f32_e64 [[CMP:s\[[0-9]+:[0-9]+\]]], 0.5, |[[SUB]]|
-; SI: v_cndmask_b32_e64 [[SEL:v[0-9]+]], 0, [[VX]], [[CMP]]
+; SI: v_cmp_le_f32_e64 vcc, 0.5, |[[SUB]]|
+; SI: v_cndmask_b32_e32 [[SEL:v[0-9]+]], 0, [[VX]]
 ; SI: v_add_f32_e32 [[RESULT:v[0-9]+]], [[SEL]], [[TRUNC]]
 ; SI: buffer_store_dword [[RESULT]]
 
diff --git a/test/CodeGen/R600/llvm.sin.ll b/test/CodeGen/AMDGPU/llvm.sin.ll
similarity index 100%
rename from test/CodeGen/R600/llvm.sin.ll
rename to test/CodeGen/AMDGPU/llvm.sin.ll
diff --git a/test/CodeGen/AMDGPU/llvm.sqrt.ll b/test/CodeGen/AMDGPU/llvm.sqrt.ll
new file mode 100644
index 00000000000..c6da047f539
--- /dev/null
+++ b/test/CodeGen/AMDGPU/llvm.sqrt.ll
@@ -0,0 +1,105 @@
+; RUN: llc < %s -march=r600 --mcpu=redwood | FileCheck %s --check-prefix=R600
+; RUN: llc < %s -march=amdgcn --mcpu=SI -verify-machineinstrs| FileCheck %s --check-prefix=SI
+; RUN: llc < %s -march=amdgcn --mcpu=tonga -verify-machineinstrs| FileCheck %s --check-prefix=SI
+
+; R600-LABEL: {{^}}sqrt_f32:
+; R600: RECIPSQRT_CLAMPED * T{{[0-9]\.[XYZW]}}, KC0[2].Z
+; R600: MUL NON-IEEE T{{[0-9]\.[XYZW]}}, KC0[2].Z, PS
+; SI-LABEL: {{^}}sqrt_f32:
+; SI: v_sqrt_f32_e32
+define void @sqrt_f32(float addrspace(1)* %out, float %in) {
+entry:
+  %0 = call float @llvm.sqrt.f32(float %in)
+  store float %0, float addrspace(1)* %out
+  ret void
+}
+
+; R600-LABEL: {{^}}sqrt_v2f32:
+; R600-DAG: RECIPSQRT_CLAMPED * T{{[0-9]\.[XYZW]}}, KC0[2].W
+; R600-DAG: MUL NON-IEEE T{{[0-9]\.[XYZW]}}, KC0[2].W, PS
+; R600-DAG: RECIPSQRT_CLAMPED * T{{[0-9]\.[XYZW]}}, KC0[3].X
+; R600-DAG: MUL NON-IEEE T{{[0-9]\.[XYZW]}}, KC0[3].X, PS
+; SI-LABEL: {{^}}sqrt_v2f32:
+; SI: v_sqrt_f32_e32
+; SI: v_sqrt_f32_e32
+define void @sqrt_v2f32(<2 x float> addrspace(1)* %out, <2 x float> %in) {
+entry:
+  %0 = call <2 x float> @llvm.sqrt.v2f32(<2 x float> %in)
+  store <2 x float> %0, <2 x float> addrspace(1)* %out
+  ret void
+}
+
+; R600-LABEL: {{^}}sqrt_v4f32:
+; R600-DAG: RECIPSQRT_CLAMPED * T{{[0-9]\.[XYZW]}}, KC0[3].Y
+; R600-DAG: MUL NON-IEEE T{{[0-9]\.[XYZW]}}, KC0[3].Y, PS
+; R600-DAG: RECIPSQRT_CLAMPED * T{{[0-9]\.[XYZW]}}, KC0[3].Z
+; R600-DAG: MUL NON-IEEE T{{[0-9]\.[XYZW]}}, KC0[3].Z, PS
+; R600-DAG: RECIPSQRT_CLAMPED * T{{[0-9]\.[XYZW]}}, KC0[3].W
+; R600-DAG: MUL NON-IEEE T{{[0-9]\.[XYZW]}}, KC0[3].W, PS
+; R600-DAG: RECIPSQRT_CLAMPED * T{{[0-9]\.[XYZW]}}, KC0[4].X
+; R600-DAG: MUL NON-IEEE T{{[0-9]\.[XYZW]}}, KC0[4].X, PS
+; SI-LABEL: {{^}}sqrt_v4f32:
+; SI: v_sqrt_f32_e32
+; SI: v_sqrt_f32_e32
+; SI: v_sqrt_f32_e32
+; SI: v_sqrt_f32_e32
+define void @sqrt_v4f32(<4 x float> addrspace(1)* %out, <4 x float> %in) {
+entry:
+  %0 = call <4 x float> @llvm.sqrt.v4f32(<4 x float> %in)
+  store <4 x float> %0, <4 x float> addrspace(1)* %out
+  ret void
+}
+
+; SI-LABEL: {{^}}elim_redun_check:
+; SI: v_sqrt_f32_e32
+; SI-NOT: v_cndmask
+define void @elim_redun_check(float addrspace(1)* %out, float %in) {
+entry:
+  %sqrt = call float @llvm.sqrt.f32(float %in)
+  %cmp = fcmp olt float %in, -0.000000e+00
+  %res = select i1 %cmp, float 0x7FF8000000000000, float %sqrt
+  store float %res, float addrspace(1)* %out
+  ret void
+}
+
+; SI-LABEL: {{^}}elim_redun_check_ult:
+; SI: v_sqrt_f32_e32
+; SI-NOT: v_cndmask
+define void @elim_redun_check_ult(float addrspace(1)* %out, float %in) {
+entry:
+  %sqrt = call float @llvm.sqrt.f32(float %in)
+  %cmp = fcmp ult float %in, -0.000000e+00
+  %res = select i1 %cmp, float 0x7FF8000000000000, float %sqrt
+  store float %res, float addrspace(1)* %out
+  ret void
+}
+
+; SI-LABEL: {{^}}elim_redun_check_v2:
+; SI: v_sqrt_f32_e32
+; SI: v_sqrt_f32_e32
+; SI-NOT: v_cndmask
+define void @elim_redun_check_v2(<2 x float> addrspace(1)* %out, <2 x float> %in) {
+entry:
+  %sqrt = call <2 x float> @llvm.sqrt.v2f32(<2 x float> %in)
+  %cmp = fcmp olt <2 x float> %in, <float -0.000000e+00, float -0.000000e+00>
+  %res = select <2 x i1> %cmp, <2 x float> <float 0x7FF8000000000000, float 0x7FF8000000000000>, <2 x float> %sqrt
+  store <2 x float> %res, <2 x float> addrspace(1)* %out
+  ret void
+}
+
+; SI-LABEL: {{^}}elim_redun_check_v2_ult
+; SI: v_sqrt_f32_e32
+; SI: v_sqrt_f32_e32
+; SI-NOT: v_cndmask
+define void @elim_redun_check_v2_ult(<2 x float> addrspace(1)* %out, <2 x float> %in) {
+entry:
+  %sqrt = call <2 x float> @llvm.sqrt.v2f32(<2 x float> %in)
+  %cmp = fcmp ult <2 x float> %in, <float -0.000000e+00, float -0.000000e+00>
+  %res = select <2 x i1> %cmp, <2 x float> <float 0x7FF8000000000000, float 0x7FF8000000000000>, <2 x float> %sqrt
+  store <2 x float> %res, <2 x float> addrspace(1)* %out
+  ret void
+}
+
+declare float @llvm.sqrt.f32(float %in)
+declare <2 x float> @llvm.sqrt.v2f32(<2 x float> %in)
+declare <4 x float> @llvm.sqrt.v4f32(<4 x float> %in)
diff --git a/test/CodeGen/R600/load-i1.ll b/test/CodeGen/AMDGPU/load-i1.ll
similarity index 100%
rename from test/CodeGen/R600/load-i1.ll
rename to test/CodeGen/AMDGPU/load-i1.ll
diff --git a/test/CodeGen/R600/load-input-fold.ll b/test/CodeGen/AMDGPU/load-input-fold.ll
similarity index 100%
rename from test/CodeGen/R600/load-input-fold.ll
rename to test/CodeGen/AMDGPU/load-input-fold.ll
diff --git a/test/CodeGen/AMDGPU/load.ll b/test/CodeGen/AMDGPU/load.ll
new file mode 100644
index 00000000000..93b1b51a0d0
--- /dev/null
+++ b/test/CodeGen/AMDGPU/load.ll
@@ -0,0 +1,709 @@
+; RUN: llc < %s -march=r600 -mcpu=redwood | FileCheck --check-prefix=R600 --check-prefix=FUNC %s
+; RUN: llc < %s -march=r600 -mcpu=cayman | FileCheck --check-prefix=R600 --check-prefix=FUNC %s
+; RUN: llc < %s -march=amdgcn -mcpu=SI -verify-machineinstrs | FileCheck --check-prefix=SI --check-prefix=FUNC %s
+; RUN: llc < %s -march=amdgcn -mcpu=tonga -verify-machineinstrs | FileCheck --check-prefix=SI --check-prefix=FUNC %s
+
+;===------------------------------------------------------------------------===;
+; GLOBAL ADDRESS SPACE
+;===------------------------------------------------------------------------===;
+
+; Load an i8 value from the global address space.
+; FUNC-LABEL: {{^}}load_i8:
+; R600: VTX_READ_8 T{{[0-9]+\.X, T[0-9]+\.X}}
+
+; SI: buffer_load_ubyte v{{[0-9]+}},
+define void @load_i8(i32 addrspace(1)* %out, i8 addrspace(1)* %in) {
+  %1 = load i8, i8 addrspace(1)* %in
+  %2 = zext i8 %1 to i32
+  store i32 %2, i32 addrspace(1)* %out
+  ret void
+}
+
+; FUNC-LABEL: {{^}}load_i8_sext:
+; R600: VTX_READ_8 [[DST:T[0-9]\.[XYZW]]], [[DST]]
+; R600: BFE_INT {{[* ]*}}T{{[0-9].[XYZW]}}, [[DST]], 0.0, literal
+; R600: 8
+; SI: buffer_load_sbyte
+define void @load_i8_sext(i32 addrspace(1)* %out, i8 addrspace(1)* %in) {
+entry:
+  %0 = load i8, i8 addrspace(1)* %in
+  %1 = sext i8 %0 to i32
+  store i32 %1, i32 addrspace(1)* %out
+  ret void
+}
+
+; FUNC-LABEL: {{^}}load_v2i8:
+; R600: VTX_READ_8
+; R600: VTX_READ_8
+; SI: buffer_load_ubyte
+; SI: buffer_load_ubyte
+define void @load_v2i8(<2 x i32> addrspace(1)* %out, <2 x i8> addrspace(1)* %in) {
+entry:
+  %0 = load <2 x i8>, <2 x i8> addrspace(1)* %in
+  %1 = zext <2 x i8> %0 to <2 x i32>
+  store <2 x i32> %1, <2 x i32> addrspace(1)* %out
+  ret void
+}
+
+; FUNC-LABEL: {{^}}load_v2i8_sext:
+; R600-DAG: VTX_READ_8 [[DST_X:T[0-9]\.[XYZW]]], [[DST_X]]
+; R600-DAG: VTX_READ_8 [[DST_Y:T[0-9]\.[XYZW]]], [[DST_Y]]
+; R600-DAG: BFE_INT {{[* ]*}}T{{[0-9].[XYZW]}}, [[DST_X]], 0.0, literal
+; R600-DAG: BFE_INT {{[* ]*}}T{{[0-9].[XYZW]}}, [[DST_Y]], 0.0, literal
+; R600-DAG: 8
+; R600-DAG: 8
+
+; SI: buffer_load_sbyte
+; SI: buffer_load_sbyte
+define void @load_v2i8_sext(<2 x i32> addrspace(1)* %out, <2 x i8> addrspace(1)* %in) {
+entry:
+  %0 = load <2 x i8>, <2 x i8> addrspace(1)* %in
+  %1 = sext <2 x i8> %0 to <2 x i32>
+  store <2 x i32> %1, <2 x i32> addrspace(1)* %out
+  ret void
+}
+
+; FUNC-LABEL: {{^}}load_v4i8:
+; R600: VTX_READ_8
+; R600: VTX_READ_8
+; R600: VTX_READ_8
+; R600: VTX_READ_8
+; SI: buffer_load_ubyte
+; SI: buffer_load_ubyte
+; SI: buffer_load_ubyte
+; SI: buffer_load_ubyte
+define void @load_v4i8(<4 x i32> addrspace(1)* %out, <4 x i8> addrspace(1)* %in) {
+entry:
+  %0 = load <4 x i8>, <4 x i8> addrspace(1)* %in
+  %1 = zext <4 x i8> %0 to <4 x i32>
+  store <4 x i32> %1, <4 x i32> addrspace(1)* %out
+  ret void
+}
+
+; FUNC-LABEL: {{^}}load_v4i8_sext:
+; R600-DAG: VTX_READ_8 [[DST_X:T[0-9]\.[XYZW]]], [[DST_X]]
+; R600-DAG: VTX_READ_8 [[DST_Y:T[0-9]\.[XYZW]]], [[DST_Y]]
+; R600-DAG: VTX_READ_8 [[DST_Z:T[0-9]\.[XYZW]]], [[DST_Z]]
+; R600-DAG: VTX_READ_8 [[DST_W:T[0-9]\.[XYZW]]], [[DST_W]]
+; R600-DAG: BFE_INT {{[* ]*}}T{{[0-9].[XYZW]}}, [[DST_X]], 0.0, literal
+; R600-DAG: BFE_INT {{[* ]*}}T{{[0-9].[XYZW]}}, [[DST_Y]], 0.0, literal
+; R600-DAG: BFE_INT {{[* ]*}}T{{[0-9].[XYZW]}}, [[DST_Z]], 0.0, literal
+; R600-DAG: BFE_INT {{[* ]*}}T{{[0-9].[XYZW]}}, [[DST_W]], 0.0, literal
+; R600-DAG: 8
+; R600-DAG: 8
+; R600-DAG: 8
+; R600-DAG: 8
+; SI: buffer_load_sbyte
+; SI: buffer_load_sbyte
+; SI: buffer_load_sbyte
+; SI: buffer_load_sbyte
+define void @load_v4i8_sext(<4 x i32> addrspace(1)* %out, <4 x i8> addrspace(1)* %in) {
+entry:
+  %0 = load <4 x i8>, <4 x i8> addrspace(1)* %in
+  %1 = sext <4 x i8> %0 to <4 x i32>
+  store <4 x i32> %1, <4 x i32> addrspace(1)* %out
+  ret void
+}
+
+; Load an i16 value from the global address space.
+; FUNC-LABEL: {{^}}load_i16:
+; R600: VTX_READ_16 T{{[0-9]+\.X, T[0-9]+\.X}}
+; SI: buffer_load_ushort
+define void @load_i16(i32 addrspace(1)* %out, i16 addrspace(1)* %in) {
+entry:
+  %0 = load i16	, i16	 addrspace(1)* %in
+  %1 = zext i16 %0 to i32
+  store i32 %1, i32 addrspace(1)* %out
+  ret void
+}
+
+; FUNC-LABEL: {{^}}load_i16_sext:
+; R600: VTX_READ_16 [[DST:T[0-9]\.[XYZW]]], [[DST]]
+; R600: BFE_INT {{[* ]*}}T{{[0-9].[XYZW]}}, [[DST]], 0.0, literal
+; R600: 16
+; SI: buffer_load_sshort
+define void @load_i16_sext(i32 addrspace(1)* %out, i16 addrspace(1)* %in) {
+entry:
+  %0 = load i16, i16 addrspace(1)* %in
+  %1 = sext i16 %0 to i32
+  store i32 %1, i32 addrspace(1)* %out
+  ret void
+}
+
+; FUNC-LABEL: {{^}}load_v2i16:
+; R600: VTX_READ_16
+; R600: VTX_READ_16
+; SI: buffer_load_ushort
+; SI: buffer_load_ushort
+define void @load_v2i16(<2 x i32> addrspace(1)* %out, <2 x i16> addrspace(1)* %in) {
+entry:
+  %0 = load <2 x i16>, <2 x i16> addrspace(1)* %in
+  %1 = zext <2 x i16> %0 to <2 x i32>
+  store <2 x i32> %1, <2 x i32> addrspace(1)* %out
+  ret void
+}
+
+; FUNC-LABEL: {{^}}load_v2i16_sext:
+; R600-DAG: VTX_READ_16 [[DST_X:T[0-9]\.[XYZW]]], [[DST_X]]
+; R600-DAG: VTX_READ_16 [[DST_Y:T[0-9]\.[XYZW]]], [[DST_Y]]
+; R600-DAG: BFE_INT {{[* ]*}}T{{[0-9].[XYZW]}}, [[DST_X]], 0.0, literal
+; R600-DAG: BFE_INT {{[* ]*}}T{{[0-9].[XYZW]}}, [[DST_Y]], 0.0, literal
+; R600-DAG: 16
+; R600-DAG: 16
+; SI: buffer_load_sshort
+; SI: buffer_load_sshort
+define void @load_v2i16_sext(<2 x i32> addrspace(1)* %out, <2 x i16> addrspace(1)* %in) {
+entry:
+  %0 = load <2 x i16>, <2 x i16> addrspace(1)* %in
+  %1 = sext <2 x i16> %0 to <2 x i32>
+  store <2 x i32> %1, <2 x i32> addrspace(1)* %out
+  ret void
+}
+
+; FUNC-LABEL: {{^}}load_v4i16:
+; R600: VTX_READ_16
+; R600: VTX_READ_16
+; R600: VTX_READ_16
+; R600: VTX_READ_16
+; SI: buffer_load_ushort
+; SI: buffer_load_ushort
+; SI: buffer_load_ushort
+; SI: buffer_load_ushort
+define void @load_v4i16(<4 x i32> addrspace(1)* %out, <4 x i16> addrspace(1)* %in) {
+entry:
+  %0 = load <4 x i16>, <4 x i16> addrspace(1)* %in
+  %1 = zext <4 x i16> %0 to <4 x i32>
+  store <4 x i32> %1, <4 x i32> addrspace(1)* %out
+  ret void
+}
+
+; FUNC-LABEL: {{^}}load_v4i16_sext:
+; R600-DAG: VTX_READ_16 [[DST_X:T[0-9]\.[XYZW]]], [[DST_X]]
+; R600-DAG: VTX_READ_16 [[DST_Y:T[0-9]\.[XYZW]]], [[DST_Y]]
+; R600-DAG: VTX_READ_16 [[DST_Z:T[0-9]\.[XYZW]]], [[DST_Z]]
+; R600-DAG: VTX_READ_16 [[DST_W:T[0-9]\.[XYZW]]], [[DST_W]]
+; R600-DAG: BFE_INT {{[* ]*}}T{{[0-9].[XYZW]}}, [[DST_X]], 0.0, literal
+; R600-DAG: BFE_INT {{[* ]*}}T{{[0-9].[XYZW]}}, [[DST_Y]], 0.0, literal
+; R600-DAG: BFE_INT {{[* ]*}}T{{[0-9].[XYZW]}}, [[DST_Z]], 0.0, literal
+; R600-DAG: BFE_INT {{[* ]*}}T{{[0-9].[XYZW]}}, [[DST_W]], 0.0, literal
+; R600-DAG: 16
+; R600-DAG: 16
+; R600-DAG: 16
+; R600-DAG: 16
+; SI: buffer_load_sshort
+; SI: buffer_load_sshort
+; SI: buffer_load_sshort
+; SI: buffer_load_sshort
+define void @load_v4i16_sext(<4 x i32> addrspace(1)* %out, <4 x i16> addrspace(1)* %in) {
+entry:
+  %0 = load <4 x i16>, <4 x i16> addrspace(1)* %in
+  %1 = sext <4 x i16> %0 to <4 x i32>
+  store <4 x i32> %1, <4 x i32> addrspace(1)* %out
+  ret void
+}
+
+; load an i32 value from the global address space.
+; FUNC-LABEL: {{^}}load_i32:
+; R600: VTX_READ_32 T{{[0-9]+}}.X, T{{[0-9]+}}.X, 0
+
+; SI: buffer_load_dword v{{[0-9]+}}
+define void @load_i32(i32 addrspace(1)* %out, i32 addrspace(1)* %in) {
+entry:
+  %0 = load i32, i32 addrspace(1)* %in
+  store i32 %0, i32 addrspace(1)* %out
+  ret void
+}
+
+; load a f32 value from the global address space.
+; FUNC-LABEL: {{^}}load_f32:
+; R600: VTX_READ_32 T{{[0-9]+}}.X, T{{[0-9]+}}.X, 0
+
+; SI: buffer_load_dword v{{[0-9]+}}
+define void @load_f32(float addrspace(1)* %out, float addrspace(1)* %in) {
+entry:
+  %0 = load float, float addrspace(1)* %in
+  store float %0, float addrspace(1)* %out
+  ret void
+}
+
+; load a v2f32 value from the global address space
+; FUNC-LABEL: {{^}}load_v2f32:
+; R600: MEM_RAT
+; R600: VTX_READ_64
+; SI: buffer_load_dwordx2
+define void @load_v2f32(<2 x float> addrspace(1)* %out, <2 x float> addrspace(1)* %in) {
+entry:
+  %0 = load <2 x float>, <2 x float> addrspace(1)* %in
+  store <2 x float> %0, <2 x float> addrspace(1)* %out
+  ret void
+}
+
+; FUNC-LABEL: {{^}}load_i64:
+; R600: VTX_READ_64
+; SI: buffer_load_dwordx2
+define void @load_i64(i64 addrspace(1)* %out, i64 addrspace(1)* %in) {
+entry:
+  %0 = load i64, i64 addrspace(1)* %in
+  store i64 %0, i64 addrspace(1)* %out
+  ret void
+}
+
+; FUNC-LABEL: {{^}}load_i64_sext:
+; R600: MEM_RAT
+; R600: MEM_RAT
+; R600: ASHR {{[* ]*}}T{{[0-9]\.[XYZW]}}, T{{[0-9]\.[XYZW]}},  literal.x
+; R600: 31
+; SI: buffer_load_dword
+
+define void @load_i64_sext(i64 addrspace(1)* %out, i32 addrspace(1)* %in) {
+entry:
+  %0 = load i32, i32 addrspace(1)* %in
+  %1 = sext i32 %0 to i64
+  store i64 %1, i64 addrspace(1)* %out
+  ret void
+}
+
+; FUNC-LABEL: {{^}}load_i64_zext:
+; R600: MEM_RAT
+; R600: MEM_RAT
+define void @load_i64_zext(i64 addrspace(1)* %out, i32 addrspace(1)* %in) {
+entry:
+  %0 = load i32, i32 addrspace(1)* %in
+  %1 = zext i32 %0 to i64
+  store i64 %1, i64 addrspace(1)* %out
+  ret void
+}
+
+; FUNC-LABEL: {{^}}load_v8i32:
+; R600: VTX_READ_128
+; R600: VTX_READ_128
+; XXX: We should be using DWORDX4 instructions on SI.
+; SI: buffer_load_dword
+; SI: buffer_load_dword
+; SI: buffer_load_dword
+; SI: buffer_load_dword
+; SI: buffer_load_dword
+; SI: buffer_load_dword
+; SI: buffer_load_dword
+; SI: buffer_load_dword
+define void @load_v8i32(<8 x i32> addrspace(1)* %out, <8 x i32> addrspace(1)* %in) {
+entry:
+  %0 = load <8 x i32>, <8 x i32> addrspace(1)* %in
+  store <8 x i32> %0, <8 x i32> addrspace(1)* %out
+  ret void
+}
+
+; FUNC-LABEL: {{^}}load_v16i32:
+; R600: VTX_READ_128
+; R600: VTX_READ_128
+; R600: VTX_READ_128
+; R600: VTX_READ_128
+; XXX: We should be using DWORDX4 instructions on SI.
+; SI: buffer_load_dword
+; SI: buffer_load_dword
+; SI: buffer_load_dword
+; SI: buffer_load_dword
+; SI: buffer_load_dword
+; SI: buffer_load_dword
+; SI: buffer_load_dword
+; SI: buffer_load_dword
+; SI: buffer_load_dword
+; SI: buffer_load_dword
+; SI: buffer_load_dword
+; SI: buffer_load_dword
+; SI: buffer_load_dword
+; SI: buffer_load_dword
+; SI: buffer_load_dword
+; SI: buffer_load_dword
+define void @load_v16i32(<16 x i32> addrspace(1)* %out, <16 x i32> addrspace(1)* %in) {
+entry:
+  %0 = load <16 x i32>, <16 x i32> addrspace(1)* %in
+  store <16 x i32> %0, <16 x i32> addrspace(1)* %out
+  ret void
+}
+
+;===------------------------------------------------------------------------===;
+; CONSTANT ADDRESS SPACE
+;===------------------------------------------------------------------------===;
+
+; Load a sign-extended i8 value
+; FUNC-LABEL: {{^}}load_const_i8_sext:
+; R600: VTX_READ_8 [[DST:T[0-9]\.[XYZW]]], [[DST]]
+; R600: BFE_INT {{[* ]*}}T{{[0-9].[XYZW]}}, [[DST]], 0.0, literal
+; R600: 8
+; SI: buffer_load_sbyte v{{[0-9]+}},
+define void @load_const_i8_sext(i32 addrspace(1)* %out, i8 addrspace(2)* %in) {
+entry:
+  %0 = load i8, i8 addrspace(2)* %in
+  %1 = sext i8 %0 to i32
+  store i32 %1, i32 addrspace(1)* %out
+  ret void
+}
+
+; Load an aligned i8 value
+; FUNC-LABEL: {{^}}load_const_i8_aligned:
+; R600: VTX_READ_8 T{{[0-9]+\.X, T[0-9]+\.X}}
+; SI: buffer_load_ubyte v{{[0-9]+}},
+define void @load_const_i8_aligned(i32 addrspace(1)* %out, i8 addrspace(2)* %in) {
+entry:
+  %0 = load i8, i8 addrspace(2)* %in
+  %1 = zext i8 %0 to i32
+  store i32 %1, i32 addrspace(1)* %out
+  ret void
+}
+
+; Load an un-aligned i8 value
+; FUNC-LABEL: {{^}}load_const_i8_unaligned:
+; R600: VTX_READ_8 T{{[0-9]+\.X, T[0-9]+\.X}}
+; SI: buffer_load_ubyte v{{[0-9]+}},
+define void @load_const_i8_unaligned(i32 addrspace(1)* %out, i8 addrspace(2)* %in) {
+entry:
+  %0 = getelementptr i8, i8 addrspace(2)* %in, i32 1
+  %1 = load i8, i8 addrspace(2)* %0
+  %2 = zext i8 %1 to i32
+  store i32 %2, i32 addrspace(1)* %out
+  ret void
+}
+
+; Load a sign-extended i16 value
+; FUNC-LABEL: {{^}}load_const_i16_sext:
+; R600: VTX_READ_16 [[DST:T[0-9]\.[XYZW]]], [[DST]]
+; R600: BFE_INT {{[* ]*}}T{{[0-9].[XYZW]}}, [[DST]], 0.0, literal
+; R600: 16
+; SI: buffer_load_sshort
+define void @load_const_i16_sext(i32 addrspace(1)* %out, i16 addrspace(2)* %in) {
+entry:
+  %0 = load i16, i16 addrspace(2)* %in
+  %1 = sext i16 %0 to i32
+  store i32 %1, i32 addrspace(1)* %out
+  ret void
+}
+
+; Load an aligned i16 value
+; FUNC-LABEL: {{^}}load_const_i16_aligned:
+; R600: VTX_READ_16 T{{[0-9]+\.X, T[0-9]+\.X}}
+; SI: buffer_load_ushort
+define void @load_const_i16_aligned(i32 addrspace(1)* %out, i16 addrspace(2)* %in) {
+entry:
+  %0 = load i16, i16 addrspace(2)* %in
+  %1 = zext i16 %0 to i32
+  store i32 %1, i32 addrspace(1)* %out
+  ret void
+}
+
+; Load an un-aligned i16 value
+; FUNC-LABEL: {{^}}load_const_i16_unaligned:
+; R600: VTX_READ_16 T{{[0-9]+\.X, T[0-9]+\.X}}
+; SI: buffer_load_ushort
+define void @load_const_i16_unaligned(i32 addrspace(1)* %out, i16 addrspace(2)* %in) {
+entry:
+  %0 = getelementptr i16, i16 addrspace(2)* %in, i32 1
+  %1 = load i16, i16 addrspace(2)* %0
+  %2 = zext i16 %1 to i32
+  store i32 %2, i32 addrspace(1)* %out
+  ret void
+}
+
+; Load an i32 value from the constant address space.
+; FUNC-LABEL: {{^}}load_const_addrspace_i32:
+; R600: VTX_READ_32 T{{[0-9]+}}.X, T{{[0-9]+}}.X, 0
+
+; SI: s_load_dword s{{[0-9]+}}
+define void @load_const_addrspace_i32(i32 addrspace(1)* %out, i32 addrspace(2)* %in) {
+entry:
+  %0 = load i32, i32 addrspace(2)* %in
+  store i32 %0, i32 addrspace(1)* %out
+  ret void
+}
+
+; Load a f32 value from the constant address space.
+; FUNC-LABEL: {{^}}load_const_addrspace_f32:
+; R600: VTX_READ_32 T{{[0-9]+}}.X, T{{[0-9]+}}.X, 0
+
+; SI: s_load_dword s{{[0-9]+}}
+define void @load_const_addrspace_f32(float addrspace(1)* %out, float addrspace(2)* %in) {
+  %1 = load float, float addrspace(2)* %in
+  store float %1, float addrspace(1)* %out
+  ret void
+}
+
+;===------------------------------------------------------------------------===;
+; LOCAL ADDRESS SPACE
+;===------------------------------------------------------------------------===;
+
+; Load an i8 value from the local address space.
+; FUNC-LABEL: {{^}}load_i8_local:
+; R600: LDS_UBYTE_READ_RET
+; SI-NOT: s_wqm_b64
+; SI: s_mov_b32 m0
+; SI: ds_read_u8
+define void @load_i8_local(i32 addrspace(1)* %out, i8 addrspace(3)* %in) {
+  %1 = load i8, i8 addrspace(3)* %in
+  %2 = zext i8 %1 to i32
+  store i32 %2, i32 addrspace(1)* %out
+  ret void
+}
+
+; FUNC-LABEL: {{^}}load_i8_sext_local:
+; R600: LDS_UBYTE_READ_RET
+; R600: BFE_INT
+; SI-NOT: s_wqm_b64
+; SI: s_mov_b32 m0
+; SI: ds_read_i8
+define void @load_i8_sext_local(i32 addrspace(1)* %out, i8 addrspace(3)* %in) {
+entry:
+  %0 = load i8, i8 addrspace(3)* %in
+  %1 = sext i8 %0 to i32
+  store i32 %1, i32 addrspace(1)* %out
+  ret void
+}
+
+; FUNC-LABEL: {{^}}load_v2i8_local:
+; R600: LDS_UBYTE_READ_RET
+; R600: LDS_UBYTE_READ_RET
+; SI-NOT: s_wqm_b64
+; SI: s_mov_b32 m0
+; SI: ds_read_u8
+; SI: ds_read_u8
+define void @load_v2i8_local(<2 x i32> addrspace(1)* %out, <2 x i8> addrspace(3)* %in) {
+entry:
+  %0 = load <2 x i8>, <2 x i8> addrspace(3)* %in
+  %1 = zext <2 x i8> %0 to <2 x i32>
+  store <2 x i32> %1, <2 x i32> addrspace(1)* %out
+  ret void
+}
+
+; FUNC-LABEL: {{^}}load_v2i8_sext_local:
+; R600-DAG: LDS_UBYTE_READ_RET
+; R600-DAG: LDS_UBYTE_READ_RET
+; R600-DAG: BFE_INT
+; R600-DAG: BFE_INT
+; SI-NOT: s_wqm_b64
+; SI: s_mov_b32 m0
+; SI: ds_read_i8
+; SI: ds_read_i8
+define void @load_v2i8_sext_local(<2 x i32> addrspace(1)* %out, <2 x i8> addrspace(3)* %in) {
+entry:
+  %0 = load <2 x i8>, <2 x i8> addrspace(3)* %in
+  %1 = sext <2 x i8> %0 to <2 x i32>
+  store <2 x i32> %1, <2 x i32> addrspace(1)* %out
+  ret void
+}
+
+; FUNC-LABEL: {{^}}load_v4i8_local:
+; R600: LDS_UBYTE_READ_RET
+; R600: LDS_UBYTE_READ_RET
+; R600: LDS_UBYTE_READ_RET
+; R600: LDS_UBYTE_READ_RET
+; SI-NOT: s_wqm_b64
+; SI: s_mov_b32 m0
+; SI: ds_read_u8
+; SI: ds_read_u8
+; SI: ds_read_u8
+; SI: ds_read_u8
+define void @load_v4i8_local(<4 x i32> addrspace(1)* %out, <4 x i8> addrspace(3)* %in) {
+entry:
+  %0 = load <4 x i8>, <4 x i8> addrspace(3)* %in
+  %1 = zext <4 x i8> %0 to <4 x i32>
+  store <4 x i32> %1, <4 x i32> addrspace(1)* %out
+  ret void
+}
+
+; FUNC-LABEL: {{^}}load_v4i8_sext_local:
+; R600-DAG: LDS_UBYTE_READ_RET
+; R600-DAG: LDS_UBYTE_READ_RET
+; R600-DAG: LDS_UBYTE_READ_RET
+; R600-DAG: LDS_UBYTE_READ_RET
+; R600-DAG: BFE_INT
+; R600-DAG: BFE_INT
+; R600-DAG: BFE_INT
+; R600-DAG: BFE_INT
+; SI-NOT: s_wqm_b64
+; SI: s_mov_b32 m0
+; SI: ds_read_i8
+; SI: ds_read_i8
+; SI: ds_read_i8
+; SI: ds_read_i8
+define void @load_v4i8_sext_local(<4 x i32> addrspace(1)* %out, <4 x i8> addrspace(3)* %in) {
+entry:
+  %0 = load <4 x i8>, <4 x i8> addrspace(3)* %in
+  %1 = sext <4 x i8> %0 to <4 x i32>
+  store <4 x i32> %1, <4 x i32> addrspace(1)* %out
+  ret void
+}
+
+; Load an i16 value from the local address space.
+; FUNC-LABEL: {{^}}load_i16_local:
+; R600: LDS_USHORT_READ_RET
+; SI-NOT: s_wqm_b64
+; SI: s_mov_b32 m0
+; SI: ds_read_u16
+define void @load_i16_local(i32 addrspace(1)* %out, i16 addrspace(3)* %in) {
+entry:
+  %0 = load i16	, i16	 addrspace(3)* %in
+  %1 = zext i16 %0 to i32
+  store i32 %1, i32 addrspace(1)* %out
+  ret void
+}
+
+; FUNC-LABEL: {{^}}load_i16_sext_local:
+; R600: LDS_USHORT_READ_RET
+; R600: BFE_INT
+; SI-NOT: s_wqm_b64
+; SI: s_mov_b32 m0
+; SI: ds_read_i16
+define void @load_i16_sext_local(i32 addrspace(1)* %out, i16 addrspace(3)* %in) {
+entry:
+  %0 = load i16, i16 addrspace(3)* %in
+  %1 = sext i16 %0 to i32
+  store i32 %1, i32 addrspace(1)* %out
+  ret void
+}
+
+; FUNC-LABEL: {{^}}load_v2i16_local:
+; R600: LDS_USHORT_READ_RET
+; R600: LDS_USHORT_READ_RET
+; SI-NOT: s_wqm_b64
+; SI: s_mov_b32 m0
+; SI: ds_read_u16
+; SI: ds_read_u16
+define void @load_v2i16_local(<2 x i32> addrspace(1)* %out, <2 x i16> addrspace(3)* %in) {
+entry:
+  %0 = load <2 x i16>, <2 x i16> addrspace(3)* %in
+  %1 = zext <2 x i16> %0 to <2 x i32>
+  store <2 x i32> %1, <2 x i32> addrspace(1)* %out
+  ret void
+}
+
+; FUNC-LABEL: {{^}}load_v2i16_sext_local:
+; R600-DAG: LDS_USHORT_READ_RET
+; R600-DAG: LDS_USHORT_READ_RET
+; R600-DAG: BFE_INT
+; R600-DAG: BFE_INT
+; SI-NOT: s_wqm_b64
+; SI: s_mov_b32 m0
+; SI: ds_read_i16
+; SI: ds_read_i16
+define void @load_v2i16_sext_local(<2 x i32> addrspace(1)* %out, <2 x i16> addrspace(3)* %in) {
+entry:
+  %0 = load <2 x i16>, <2 x i16> addrspace(3)* %in
+  %1 = sext <2 x i16> %0 to <2 x i32>
+  store <2 x i32> %1, <2 x i32> addrspace(1)* %out
+  ret void
+}
+
+; FUNC-LABEL: {{^}}load_v4i16_local:
+; R600: LDS_USHORT_READ_RET
+; R600: LDS_USHORT_READ_RET
+; R600: LDS_USHORT_READ_RET
+; R600: LDS_USHORT_READ_RET
+; SI-NOT: s_wqm_b64
+; SI: s_mov_b32 m0
+; SI: ds_read_u16
+; SI: ds_read_u16
+; SI: ds_read_u16
+; SI: ds_read_u16
+define void @load_v4i16_local(<4 x i32> addrspace(1)* %out, <4 x i16> addrspace(3)* %in) {
+entry:
+  %0 = load <4 x i16>, <4 x i16> addrspace(3)* %in
+  %1 = zext <4 x i16> %0 to <4 x i32>
+  store <4 x i32> %1, <4 x i32> addrspace(1)* %out
+  ret void
+}
+
+; FUNC-LABEL: {{^}}load_v4i16_sext_local:
+; R600-DAG: LDS_USHORT_READ_RET
+; R600-DAG: LDS_USHORT_READ_RET
+; R600-DAG: LDS_USHORT_READ_RET
+; R600-DAG: LDS_USHORT_READ_RET
+; R600-DAG: BFE_INT
+; R600-DAG: BFE_INT
+; R600-DAG: BFE_INT
+; R600-DAG: BFE_INT
+; SI-NOT: s_wqm_b64
+; SI: s_mov_b32 m0
+; SI: ds_read_i16
+; SI: ds_read_i16
+; SI: ds_read_i16
+; SI: ds_read_i16
+define void @load_v4i16_sext_local(<4 x i32> addrspace(1)* %out, <4 x i16> addrspace(3)* %in) {
+entry:
+  %0 = load <4 x i16>, <4 x i16> addrspace(3)* %in
+  %1 = sext <4 x i16> %0 to <4 x i32>
+  store <4 x i32> %1, <4 x i32> addrspace(1)* %out
+  ret void
+}
+
+; load an i32 value from the local address space.
+; FUNC-LABEL: {{^}}load_i32_local:
+; R600: LDS_READ_RET
+; SI-NOT: s_wqm_b64
+; SI: s_mov_b32 m0
+; SI: ds_read_b32
+define void @load_i32_local(i32 addrspace(1)* %out, i32 addrspace(3)* %in) {
+entry:
+  %0 = load i32, i32 addrspace(3)* %in
+  store i32 %0, i32 addrspace(1)* %out
+  ret void
+}
+
+; load a f32 value from the local address space.
+; FUNC-LABEL: {{^}}load_f32_local:
+; R600: LDS_READ_RET
+; SI: s_mov_b32 m0
+; SI: ds_read_b32
+define void @load_f32_local(float addrspace(1)* %out, float addrspace(3)* %in) {
+entry:
+  %0 = load float, float addrspace(3)* %in
+  store float %0, float addrspace(1)* %out
+  ret void
+}
+
+; load a v2f32 value from the local address space
+; FUNC-LABEL: {{^}}load_v2f32_local:
+; R600: LDS_READ_RET
+; R600: LDS_READ_RET
+; SI: s_mov_b32 m0
+; SI: ds_read_b64
+define void @load_v2f32_local(<2 x float> addrspace(1)* %out, <2 x float> addrspace(3)* %in) {
+entry:
+  %0 = load <2 x float>, <2 x float> addrspace(3)* %in
+  store <2 x float> %0, <2 x float> addrspace(1)* %out
+  ret void
+}
+
+; Test loading a i32 and v2i32 value from the same base pointer.
+; FUNC-LABEL: {{^}}load_i32_v2i32_local:
+; R600: LDS_READ_RET
+; R600: LDS_READ_RET
+; R600: LDS_READ_RET
+; SI-DAG: ds_read_b32
+; SI-DAG: ds_read2_b32
+define void @load_i32_v2i32_local(<2 x i32> addrspace(1)* %out, i32 addrspace(3)* %in) {
+  %scalar = load i32, i32 addrspace(3)* %in
+  %tmp0 = bitcast i32 addrspace(3)* %in to <2 x i32> addrspace(3)*
+  %vec_ptr = getelementptr <2 x i32>, <2 x i32> addrspace(3)* %tmp0, i32 2
+  %vec0 = load <2 x i32>, <2 x i32> addrspace(3)* %vec_ptr, align 4
+  %vec1 = insertelement <2 x i32> <i32 0, i32 0>, i32 %scalar, i32 0
+  %vec = add <2 x i32> %vec0, %vec1
+  store <2 x i32> %vec, <2 x i32> addrspace(1)* %out
+  ret void
+}
+
+
+@lds = addrspace(3) global [512 x i32] undef, align 4
+
+; On SI we need to make sure that the base offset is a register and not
+; an immediate.
+; FUNC-LABEL: {{^}}load_i32_local_const_ptr:
+; SI: v_mov_b32_e32 v[[ZERO:[0-9]+]], 0
+; SI: ds_read_b32 v0, v[[ZERO]] offset:4
+; R600: LDS_READ_RET
+define void @load_i32_local_const_ptr(i32 addrspace(1)* %out, i32 addrspace(3)* %in) {
+entry:
+  %tmp0 = getelementptr [512 x i32], [512 x i32] addrspace(3)* @lds, i32 0, i32 1
+  %tmp1 = load i32, i32 addrspace(3)* %tmp0
+  %tmp2 = getelementptr i32, i32 addrspace(1)* %out, i32 1
+  store i32 %tmp1, i32 addrspace(1)* %tmp2
+  ret void
+}
diff --git a/test/CodeGen/R600/load.vec.ll b/test/CodeGen/AMDGPU/load.vec.ll
similarity index 100%
rename from test/CodeGen/R600/load.vec.ll
rename to test/CodeGen/AMDGPU/load.vec.ll
diff --git a/test/CodeGen/R600/load64.ll b/test/CodeGen/AMDGPU/load64.ll
similarity index 100%
rename from test/CodeGen/R600/load64.ll
rename to test/CodeGen/AMDGPU/load64.ll
diff --git a/test/CodeGen/R600/local-64.ll b/test/CodeGen/AMDGPU/local-64.ll
similarity index 100%
rename from test/CodeGen/R600/local-64.ll
rename to test/CodeGen/AMDGPU/local-64.ll
diff --git a/test/CodeGen/R600/local-atomics.ll b/test/CodeGen/AMDGPU/local-atomics.ll
similarity index 100%
rename from test/CodeGen/R600/local-atomics.ll
rename to test/CodeGen/AMDGPU/local-atomics.ll
diff --git a/test/CodeGen/R600/local-atomics64.ll b/test/CodeGen/AMDGPU/local-atomics64.ll
similarity index 100%
rename from test/CodeGen/R600/local-atomics64.ll
rename to test/CodeGen/AMDGPU/local-atomics64.ll
diff --git a/test/CodeGen/R600/local-memory-two-objects.ll b/test/CodeGen/AMDGPU/local-memory-two-objects.ll
similarity index 97%
rename from test/CodeGen/R600/local-memory-two-objects.ll
rename to test/CodeGen/AMDGPU/local-memory-two-objects.ll
index 06a8b1246e6..f501a7ac627 100644
--- a/test/CodeGen/R600/local-memory-two-objects.ll
+++ b/test/CodeGen/AMDGPU/local-memory-two-objects.ll
@@ -14,7 +14,7 @@
 
 ; EG: {{^}}local_memory_two_objects:
 
-; We would like to check the the lds writes are using different
+; We would like to check the lds writes are using different
 ; addresses, but due to variations in the scheduler, we can't do
 ; this consistently on evergreen GPUs.
 ; EG: LDS_WRITE
diff --git a/test/CodeGen/R600/local-memory.ll b/test/CodeGen/AMDGPU/local-memory.ll
similarity index 100%
rename from test/CodeGen/R600/local-memory.ll
rename to test/CodeGen/AMDGPU/local-memory.ll
diff --git a/test/CodeGen/R600/loop-address.ll b/test/CodeGen/AMDGPU/loop-address.ll
similarity index 76%
rename from test/CodeGen/R600/loop-address.ll
rename to test/CodeGen/AMDGPU/loop-address.ll
index 7fadb8dba7b..f60d574497d 100644
--- a/test/CodeGen/R600/loop-address.ll
+++ b/test/CodeGen/AMDGPU/loop-address.ll
@@ -1,13 +1,10 @@
-;RUN: llc < %s -march=r600 -mcpu=redwood | FileCheck %s
+;RUN: llc < %s -march=r600 -mcpu=redwood < %s | FileCheck %s
 
 ;CHECK: ALU_PUSH
 ;CHECK: LOOP_START_DX10 @11
 ;CHECK: LOOP_BREAK @10
 ;CHECK: POP @10
 
-target datalayout = "e-p:32:32:32-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-v16:16:16-v24:32:32-v32:32:32-v48:64:64-v64:64:64-v96:128:128-v128:128:128-v192:256:256-v256:256:256-v512:512:512-v1024:1024:1024-v2048:2048:2048-n32:64"
-target triple = "r600--"
-
 define void @loop_ge(i32 addrspace(1)* nocapture %out, i32 %iterations) #0 {
 entry:
   %cmp5 = icmp sgt i32 %iterations, 0
diff --git a/test/CodeGen/R600/loop-idiom.ll b/test/CodeGen/AMDGPU/loop-idiom.ll
similarity index 88%
rename from test/CodeGen/R600/loop-idiom.ll
rename to test/CodeGen/AMDGPU/loop-idiom.ll
index 810b34fed86..5fd9806813c 100644
--- a/test/CodeGen/R600/loop-idiom.ll
+++ b/test/CodeGen/AMDGPU/loop-idiom.ll
@@ -2,10 +2,6 @@
 ; RUN: opt -basicaa -loop-idiom -S < %s -march=amdgcn -mcpu=SI -verify-machineinstrs| FileCheck --check-prefix=SI --check-prefix=FUNC %s
 ; RUN: opt -basicaa -loop-idiom -S < %s -march=amdgcn -mcpu=tonga -verify-machineinstrs| FileCheck --check-prefix=SI --check-prefix=FUNC %s
 
-target datalayout = "e-p:32:32-p1:64:64-p2:64:64-p3:32:32-p4:32:32-p5:64:64-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-v2048:2048-n32:64"
-target triple = "r600--"
-
-
 
 ; Make sure loop-idiom doesn't create memcpy or memset.  There are no library
 ; implementations of these for R600.
diff --git a/test/CodeGen/R600/lshl.ll b/test/CodeGen/AMDGPU/lshl.ll
similarity index 100%
rename from test/CodeGen/R600/lshl.ll
rename to test/CodeGen/AMDGPU/lshl.ll
diff --git a/test/CodeGen/R600/lshr.ll b/test/CodeGen/AMDGPU/lshr.ll
similarity index 100%
rename from test/CodeGen/R600/lshr.ll
rename to test/CodeGen/AMDGPU/lshr.ll
diff --git a/test/CodeGen/R600/m0-spill.ll b/test/CodeGen/AMDGPU/m0-spill.ll
similarity index 100%
rename from test/CodeGen/R600/m0-spill.ll
rename to test/CodeGen/AMDGPU/m0-spill.ll
diff --git a/test/CodeGen/R600/mad-combine.ll b/test/CodeGen/AMDGPU/mad-combine.ll
similarity index 96%
rename from test/CodeGen/R600/mad-combine.ll
rename to test/CodeGen/AMDGPU/mad-combine.ll
index bc071628ead..c98f851f2b9 100644
--- a/test/CodeGen/R600/mad-combine.ll
+++ b/test/CodeGen/AMDGPU/mad-combine.ll
@@ -19,7 +19,7 @@ declare float @llvm.fmuladd.f32(float, float, float) #0
 ; SI-DAG: buffer_load_dword [[B:v[0-9]+]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64 offset:4{{$}}
 ; SI-DAG: buffer_load_dword [[C:v[0-9]+]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64 offset:8{{$}}
 
-; SI-STD: v_mad_f32 [[RESULT:v[0-9]+]], [[A]], [[B]], [[C]]
+; SI-STD: v_mac_f32_e32 [[C]], [[B]], [[A]]
 
 ; SI-DENORM: v_fma_f32 [[RESULT:v[0-9]+]], [[A]], [[B]], [[C]]
 
@@ -29,7 +29,8 @@ declare float @llvm.fmuladd.f32(float, float, float) #0
 ; SI-DENORM-SLOWFMAF: v_mul_f32_e32 [[TMP:v[0-9]+]], [[B]], [[A]]
 ; SI-DENORM-SLOWFMAF: v_add_f32_e32 [[RESULT:v[0-9]+]], [[C]], [[TMP]]
 
-; SI: buffer_store_dword [[RESULT]]
+; SI-DENORM: buffer_store_dword [[RESULT]]
+; SI-STD: buffer_store_dword [[C]]
 define void @combine_to_mad_f32_0(float addrspace(1)* noalias %out, float addrspace(1)* noalias %in) #1 {
   %tid = tail call i32 @llvm.r600.read.tidig.x() #0
   %gep.0 = getelementptr float, float addrspace(1)* %in, i32 %tid
@@ -54,8 +55,8 @@ define void @combine_to_mad_f32_0(float addrspace(1)* noalias %out, float addrsp
 ; SI-DAG: buffer_load_dword [[C:v[0-9]+]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64 offset:8{{$}}
 ; SI-DAG: buffer_load_dword [[D:v[0-9]+]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64 offset:12{{$}}
 
-; SI-STD-DAG: v_mad_f32 [[RESULT0:v[0-9]+]], [[A]], [[B]], [[C]]
-; SI-STD-DAG: v_mad_f32 [[RESULT1:v[0-9]+]], [[A]], [[B]], [[D]]
+; SI-STD-DAG: v_mac_f32_e32 [[C]], [[B]], [[A]]
+; SI-STD-DAG: v_mac_f32_e32 [[D]], [[B]], [[A]]
 
 ; SI-DENORM-DAG: v_fma_f32 [[RESULT0:v[0-9]+]], [[A]], [[B]], [[C]]
 ; SI-DENORM-DAG: v_fma_f32 [[RESULT1:v[0-9]+]], [[A]], [[B]], [[D]]
@@ -64,8 +65,10 @@ define void @combine_to_mad_f32_0(float addrspace(1)* noalias %out, float addrsp
 ; SI-DENORM-SLOWFMAF-DAG: v_add_f32_e32 [[RESULT0:v[0-9]+]], [[C]], [[TMP]]
 ; SI-DENORM-SLOWFMAF-DAG: v_add_f32_e32 [[RESULT1:v[0-9]+]], [[D]], [[TMP]]
 
-; SI-DAG: buffer_store_dword [[RESULT0]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64{{$}}
-; SI-DAG: buffer_store_dword [[RESULT1]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64 offset:4{{$}}
+; SI-DENORM-DAG: buffer_store_dword [[RESULT0]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64{{$}}
+; SI-DENORM-DAG: buffer_store_dword [[RESULT1]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64 offset:4{{$}}
+; SI-STD-DAG: buffer_store_dword [[C]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64{{$}}
+; SI-STD-DAG: buffer_store_dword [[D]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64 offset:4{{$}}
 ; SI: s_endpgm
 define void @combine_to_mad_f32_0_2use(float addrspace(1)* noalias %out, float addrspace(1)* noalias %in) #1 {
   %tid = tail call i32 @llvm.r600.read.tidig.x() #0
@@ -96,13 +99,14 @@ define void @combine_to_mad_f32_0_2use(float addrspace(1)* noalias %out, float a
 ; SI-DAG: buffer_load_dword [[B:v[0-9]+]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64 offset:4{{$}}
 ; SI-DAG: buffer_load_dword [[C:v[0-9]+]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64 offset:8{{$}}
 
-; SI-STD: v_mad_f32 [[RESULT:v[0-9]+]], [[A]], [[B]], [[C]]
+; SI-STD: v_mac_f32_e32 [[C]], [[B]], [[A]]
 ; SI-DENORM: v_fma_f32 [[RESULT:v[0-9]+]], [[A]], [[B]], [[C]]
 
 ; SI-DENORM-SLOWFMAF: v_mul_f32_e32 [[TMP:v[0-9]+]], [[B]], [[A]]
 ; SI-DENORM-SLOWFMAF: v_add_f32_e32 [[RESULT:v[0-9]+]], [[TMP]], [[C]]
 
-; SI: buffer_store_dword [[RESULT]]
+; SI-DENORM: buffer_store_dword [[RESULT]]
+; SI-STD: buffer_store_dword [[C]]
 define void @combine_to_mad_f32_1(float addrspace(1)* noalias %out, float addrspace(1)* noalias %in) #1 {
   %tid = tail call i32 @llvm.r600.read.tidig.x() #0
   %gep.0 = getelementptr float, float addrspace(1)* %in, i32 %tid
@@ -482,7 +486,7 @@ define void @aggressive_combine_to_mad_fsub_1_f32(float addrspace(1)* noalias %o
 ; SI-DAG: buffer_load_dword [[E:v[0-9]+]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64 offset:16{{$}}
 
 ; SI-STD: v_mad_f32 [[TMP:v[0-9]+]], [[D]], [[E]], -[[C]]
-; SI-STD: v_mad_f32 [[RESULT:v[0-9]+]], [[A]], [[B]], [[TMP]]
+; SI-STD: v_mac_f32_e32 [[TMP]], [[B]], [[A]]
 
 ; SI-DENORM: v_fma_f32 [[TMP:v[0-9]+]], [[D]], [[E]], -[[C]]
 ; SI-DENORM: v_fma_f32 [[RESULT:v[0-9]+]], [[A]], [[B]], [[TMP]]
@@ -492,7 +496,8 @@ define void @aggressive_combine_to_mad_fsub_1_f32(float addrspace(1)* noalias %o
 ; SI-DENORM-SLOWFMAF: v_add_f32_e32 [[TMP2:v[0-9]+]], [[TMP0]], [[TMP1]]
 ; SI-DENORM-SLOWFMAF: v_subrev_f32_e32 [[RESULT:v[0-9]+]], [[C]], [[TMP2]]
 
-; SI: buffer_store_dword [[RESULT]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64{{$}}
+; SI-DENORM: buffer_store_dword [[RESULT]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64{{$}}
+; SI-STD: buffer_store_dword [[TMP]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64{{$}}
 ; SI: s_endpgm
 define void @aggressive_combine_to_mad_fsub_2_f32(float addrspace(1)* noalias %out, float addrspace(1)* noalias %in) #1 {
   %tid = tail call i32 @llvm.r600.read.tidig.x() #0
diff --git a/test/CodeGen/R600/mad-sub.ll b/test/CodeGen/AMDGPU/mad-sub.ll
similarity index 98%
rename from test/CodeGen/R600/mad-sub.ll
rename to test/CodeGen/AMDGPU/mad-sub.ll
index aa4194ff610..24ff23a4cfc 100644
--- a/test/CodeGen/R600/mad-sub.ll
+++ b/test/CodeGen/AMDGPU/mad-sub.ll
@@ -123,7 +123,7 @@ define void @mad_sub_fabs_inv_f32(float addrspace(1)* noalias nocapture %out, fl
 }
 
 ; FUNC-LABEL: {{^}}neg_neg_mad_f32:
-; SI: v_mad_f32 {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
+; SI: v_mac_f32_e32 {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
 define void @neg_neg_mad_f32(float addrspace(1)* noalias nocapture %out, float addrspace(1)* noalias nocapture readonly %ptr) #1 {
   %tid = tail call i32 @llvm.r600.read.tidig.x() #0
   %tid.ext = sext i32 %tid to i64
@@ -172,8 +172,8 @@ define void @mad_fabs_sub_f32(float addrspace(1)* noalias nocapture %out, float
 ; FUNC-LABEL: {{^}}fsub_c_fadd_a_a:
 ; SI-DAG: buffer_load_dword [[R1:v[0-9]+]], {{v\[[0-9]+:[0-9]+\]}}, {{s\[[0-9]+:[0-9]+\]}}, 0 addr64{{$}}
 ; SI-DAG: buffer_load_dword [[R2:v[0-9]+]], {{v\[[0-9]+:[0-9]+\]}}, {{s\[[0-9]+:[0-9]+\]}}, 0 addr64 offset:4
-; SI: v_mad_f32 [[RESULT:v[0-9]+]], -2.0, [[R1]], [[R2]]
-; SI: buffer_store_dword [[RESULT]]
+; SI: v_mac_f32_e32 [[R2]], -2.0, [[R1]]
+; SI: buffer_store_dword [[R2]]
 define void @fsub_c_fadd_a_a(float addrspace(1)* %out, float addrspace(1)* %in) {
   %tid = call i32 @llvm.r600.read.tidig.x() nounwind readnone
   %gep.0 = getelementptr float, float addrspace(1)* %out, i32 %tid
diff --git a/test/CodeGen/R600/mad_int24.ll b/test/CodeGen/AMDGPU/mad_int24.ll
similarity index 100%
rename from test/CodeGen/R600/mad_int24.ll
rename to test/CodeGen/AMDGPU/mad_int24.ll
diff --git a/test/CodeGen/R600/mad_uint24.ll b/test/CodeGen/AMDGPU/mad_uint24.ll
similarity index 100%
rename from test/CodeGen/R600/mad_uint24.ll
rename to test/CodeGen/AMDGPU/mad_uint24.ll
diff --git a/test/CodeGen/R600/madak.ll b/test/CodeGen/AMDGPU/madak.ll
similarity index 95%
rename from test/CodeGen/R600/madak.ll
rename to test/CodeGen/AMDGPU/madak.ll
index 933bb016d2c..2e90cf10a3b 100644
--- a/test/CodeGen/R600/madak.ll
+++ b/test/CodeGen/AMDGPU/madak.ll
@@ -9,7 +9,7 @@ declare float @llvm.fabs.f32(float) nounwind readnone
 ; GCN-LABEL: {{^}}madak_f32:
 ; GCN: buffer_load_dword [[VA:v[0-9]+]]
 ; GCN: buffer_load_dword [[VB:v[0-9]+]]
-; GCN: v_madak_f32_e32 {{v[0-9]+}}, [[VB]], [[VA]], 0x41200000
+; GCN: v_madak_f32_e32 {{v[0-9]+}}, [[VA]], [[VB]], 0x41200000
 define void @madak_f32(float addrspace(1)* noalias %out, float addrspace(1)* noalias %in.a, float addrspace(1)* noalias %in.b) nounwind {
   %tid = tail call i32 @llvm.r600.read.tidig.x() nounwind readnone
   %in.a.gep = getelementptr float, float addrspace(1)* %in.a, i32 %tid
@@ -34,8 +34,8 @@ define void @madak_f32(float addrspace(1)* noalias %out, float addrspace(1)* noa
 ; GCN-DAG: buffer_load_dword [[VB:v[0-9]+]], {{v\[[0-9]+:[0-9]+\]}}, {{s\[[0-9]+:[0-9]+\]}}, 0 addr64 offset:4
 ; GCN-DAG: buffer_load_dword [[VC:v[0-9]+]], {{v\[[0-9]+:[0-9]+\]}}, {{s\[[0-9]+:[0-9]+\]}}, 0 addr64 offset:8
 ; GCN-DAG: v_mov_b32_e32 [[VK:v[0-9]+]], 0x41200000
-; GCN-DAG: v_mad_f32 {{v[0-9]+}}, [[VA]], [[VB]], [[VK]]
-; GCN-DAG: v_mad_f32 {{v[0-9]+}}, [[VA]], [[VC]], [[VK]]
+; GCN-DAG: v_mad_f32 {{v[0-9]+}}, [[VB]], [[VA]], [[VK]]
+; GCN-DAG: v_mac_f32_e32 [[VK]], [[VC]], [[VA]]
 ; GCN: s_endpgm
 define void @madak_2_use_f32(float addrspace(1)* noalias %out, float addrspace(1)* noalias %in) nounwind {
   %tid = tail call i32 @llvm.r600.read.tidig.x() nounwind readnone
@@ -105,7 +105,7 @@ define void @madak_inline_imm_f32(float addrspace(1)* noalias %out, float addrsp
 ; GCN-DAG: v_mov_b32_e32 [[VK:v[0-9]+]], 0x41200000
 ; GCN-DAG: buffer_load_dword [[VA:v[0-9]+]]
 ; GCN-NOT: v_madak_f32
-; GCN: v_mad_f32 {{v[0-9]+}}, [[SB]], [[VA]], [[VK]]
+; GCN: v_mac_f32_e32 [[VK]], [[SB]], [[VA]]
 define void @s_v_madak_f32(float addrspace(1)* noalias %out, float addrspace(1)* noalias %in.a, float %b) nounwind {
   %tid = tail call i32 @llvm.r600.read.tidig.x() nounwind readnone
   %in.a.gep = getelementptr float, float addrspace(1)* %in.a, i32 %tid
@@ -124,7 +124,7 @@ define void @s_v_madak_f32(float addrspace(1)* noalias %out, float addrspace(1)*
 ; GCN-DAG: v_mov_b32_e32 [[VK:v[0-9]+]], 0x41200000
 ; GCN-DAG: buffer_load_dword [[VA:v[0-9]+]]
 ; GCN-NOT: v_madak_f32
-; GCN: v_mad_f32 {{v[0-9]+}}, [[VA]], [[SB]], [[VK]]
+; GCN: v_mac_f32_e32 [[VK]], [[SB]], [[VA]]
 define void @v_s_madak_f32(float addrspace(1)* noalias %out, float %a, float addrspace(1)* noalias %in.b) nounwind {
   %tid = tail call i32 @llvm.r600.read.tidig.x() nounwind readnone
   %in.b.gep = getelementptr float, float addrspace(1)* %in.b, i32 %tid
@@ -140,7 +140,7 @@ define void @v_s_madak_f32(float addrspace(1)* noalias %out, float %a, float add
 
 ; GCN-LABEL: {{^}}s_s_madak_f32:
 ; GCN-NOT: v_madak_f32
-; GCN: v_mad_f32 {{v[0-9]+}}, {{s[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
+; GCN: v_mac_f32_e32 {{v[0-9]+}}, {{s[0-9]+}}, {{v[0-9]+}}
 define void @s_s_madak_f32(float addrspace(1)* %out, float %a, float %b) nounwind {
   %mul = fmul float %a, %b
   %madak = fadd float %mul, 10.0
diff --git a/test/CodeGen/R600/madmk.ll b/test/CodeGen/AMDGPU/madmk.ll
similarity index 89%
rename from test/CodeGen/R600/madmk.ll
rename to test/CodeGen/AMDGPU/madmk.ll
index 71615e9a025..f8e14e34af6 100644
--- a/test/CodeGen/R600/madmk.ll
+++ b/test/CodeGen/AMDGPU/madmk.ll
@@ -28,8 +28,8 @@ define void @madmk_f32(float addrspace(1)* noalias %out, float addrspace(1)* noa
 ; GCN-DAG: buffer_load_dword [[VB:v[0-9]+]], {{v\[[0-9]+:[0-9]+\]}}, {{s\[[0-9]+:[0-9]+\]}}, 0 addr64 offset:4
 ; GCN-DAG: buffer_load_dword [[VC:v[0-9]+]], {{v\[[0-9]+:[0-9]+\]}}, {{s\[[0-9]+:[0-9]+\]}}, 0 addr64 offset:8
 ; GCN-DAG: v_mov_b32_e32 [[VK:v[0-9]+]], 0x41200000
-; GCN-DAG: v_mad_f32 {{v[0-9]+}}, [[VA]], [[VK]], [[VB]]
-; GCN-DAG: v_mad_f32 {{v[0-9]+}}, [[VA]], [[VK]], [[VC]]
+; GCN-DAG: v_mac_f32_e32 [[VB]], [[VK]], [[VA]]
+; GCN-DAG: v_mac_f32_e32 [[VC]], [[VK]], [[VA]]
 ; GCN: s_endpgm
 define void @madmk_2_use_f32(float addrspace(1)* noalias %out, float addrspace(1)* noalias %in) nounwind {
   %tid = tail call i32 @llvm.r600.read.tidig.x() nounwind readnone
@@ -59,7 +59,7 @@ define void @madmk_2_use_f32(float addrspace(1)* noalias %out, float addrspace(1
 ; GCN-LABEL: {{^}}madmk_inline_imm_f32:
 ; GCN-DAG: buffer_load_dword [[VA:v[0-9]+]], {{v\[[0-9]+:[0-9]+\]}}, {{s\[[0-9]+:[0-9]+\]}}, 0 addr64{{$}}
 ; GCN-DAG: buffer_load_dword [[VB:v[0-9]+]], {{v\[[0-9]+:[0-9]+\]}}, {{s\[[0-9]+:[0-9]+\]}}, 0 addr64 offset:4
-; GCN: v_mad_f32 {{v[0-9]+}}, 4.0, [[VA]], [[VB]]
+; GCN: v_mac_f32_e32 [[VB]], 4.0, [[VA]]
 define void @madmk_inline_imm_f32(float addrspace(1)* noalias %out, float addrspace(1)* noalias %in) nounwind {
   %tid = tail call i32 @llvm.r600.read.tidig.x() nounwind readnone
   %gep.0 = getelementptr float, float addrspace(1)* %in, i32 %tid
@@ -77,7 +77,7 @@ define void @madmk_inline_imm_f32(float addrspace(1)* noalias %out, float addrsp
 
 ; GCN-LABEL: {{^}}s_s_madmk_f32:
 ; GCN-NOT: v_madmk_f32
-; GCN: v_mad_f32
+; GCN: v_mac_f32_e32
 ; GCN: s_endpgm
 define void @s_s_madmk_f32(float addrspace(1)* noalias %out, float %a, float %b) nounwind {
   %tid = tail call i32 @llvm.r600.read.tidig.x() nounwind readnone
@@ -107,7 +107,7 @@ define void @v_s_madmk_f32(float addrspace(1)* noalias %out, float addrspace(1)*
 
 ; GCN-LABEL: {{^}}scalar_vector_madmk_f32:
 ; GCN-NOT: v_madmk_f32
-; GCN: v_mad_f32
+; GCN: v_mac_f32_e32
 ; GCN: s_endpgm
 define void @scalar_vector_madmk_f32(float addrspace(1)* noalias %out, float addrspace(1)* noalias %in, float %a) nounwind {
   %tid = tail call i32 @llvm.r600.read.tidig.x() nounwind readnone
@@ -179,3 +179,27 @@ define void @madmk_add_inline_imm_f32(float addrspace(1)* noalias %out, float ad
   store float %madmk, float addrspace(1)* %out.gep, align 4
   ret void
 }
+
+; SI-LABEL: {{^}}kill_madmk_verifier_error:
+; SI: s_xor_b64
+; SI: v_madmk_f32_e32 {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, 0x472aee8c
+; SI: s_or_b64
+define void @kill_madmk_verifier_error() nounwind {
+bb:
+  br label %bb2
+
+bb1:                                              ; preds = %bb2
+  ret void
+
+bb2:                                              ; preds = %bb6, %bb
+  %tmp = phi float [ undef, %bb ], [ %tmp8, %bb6 ]
+  %tmp3 = fsub float undef, %tmp
+  %tmp5 = fcmp oeq float %tmp3, 1.000000e+04
+  br i1 %tmp5, label %bb1, label %bb6
+
+bb6:                                              ; preds = %bb2
+  %tmp4 = fmul float %tmp, undef
+  %tmp7 = fmul float %tmp4, 0x40E55DD180000000
+  %tmp8 = fadd float %tmp7, undef
+  br label %bb2
+}
diff --git a/test/CodeGen/R600/max-literals.ll b/test/CodeGen/AMDGPU/max-literals.ll
similarity index 100%
rename from test/CodeGen/R600/max-literals.ll
rename to test/CodeGen/AMDGPU/max-literals.ll
diff --git a/test/CodeGen/AMDGPU/max.ll b/test/CodeGen/AMDGPU/max.ll
new file mode 100644
index 00000000000..fef3e2f0a21
--- /dev/null
+++ b/test/CodeGen/AMDGPU/max.ll
@@ -0,0 +1,168 @@
+; RUN: llc -march=amdgcn -mcpu=SI < %s | FileCheck -check-prefix=SI -check-prefix=FUNC %s
+
+declare i32 @llvm.r600.read.tidig.x() nounwind readnone
+
+; FUNC-LABEL: @v_test_imax_sge_i32
+; SI: v_max_i32_e32
+define void @v_test_imax_sge_i32(i32 addrspace(1)* %out, i32 addrspace(1)* %aptr, i32 addrspace(1)* %bptr) nounwind {
+  %tid = call i32 @llvm.r600.read.tidig.x() nounwind readnone
+  %gep0 = getelementptr i32, i32 addrspace(1)* %aptr, i32 %tid
+  %gep1 = getelementptr i32, i32 addrspace(1)* %bptr, i32 %tid
+  %outgep = getelementptr i32, i32 addrspace(1)* %out, i32 %tid
+  %a = load i32, i32 addrspace(1)* %gep0, align 4
+  %b = load i32, i32 addrspace(1)* %gep1, align 4
+  %cmp = icmp sge i32 %a, %b
+  %val = select i1 %cmp, i32 %a, i32 %b
+  store i32 %val, i32 addrspace(1)* %outgep, align 4
+  ret void
+}
+
+; FUNC-LABEL: @s_test_imax_sge_i32
+; SI: s_max_i32
+define void @s_test_imax_sge_i32(i32 addrspace(1)* %out, i32 %a, i32 %b) nounwind {
+  %cmp = icmp sge i32 %a, %b
+  %val = select i1 %cmp, i32 %a, i32 %b
+  store i32 %val, i32 addrspace(1)* %out, align 4
+  ret void
+}
+
+; FUNC-LABEL: {{^}}s_test_imax_sge_imm_i32:
+; SI: s_max_i32 {{s[0-9]+}}, {{s[0-9]+}}, 9
+define void @s_test_imax_sge_imm_i32(i32 addrspace(1)* %out, i32 %a) nounwind {
+  %cmp = icmp sge i32 %a, 9
+  %val = select i1 %cmp, i32 %a, i32 9
+  store i32 %val, i32 addrspace(1)* %out, align 4
+  ret void
+}
+
+; FUNC-LABEL: {{^}}s_test_imax_sgt_imm_i32:
+; SI: s_max_i32 {{s[0-9]+}}, {{s[0-9]+}}, 9
+define void @s_test_imax_sgt_imm_i32(i32 addrspace(1)* %out, i32 %a) nounwind {
+  %cmp = icmp sgt i32 %a, 9
+  %val = select i1 %cmp, i32 %a, i32 9
+  store i32 %val, i32 addrspace(1)* %out, align 4
+  ret void
+}
+
+; FUNC-LABEL: @v_test_imax_sgt_i32
+; SI: v_max_i32_e32
+define void @v_test_imax_sgt_i32(i32 addrspace(1)* %out, i32 addrspace(1)* %aptr, i32 addrspace(1)* %bptr) nounwind {
+  %tid = call i32 @llvm.r600.read.tidig.x() nounwind readnone
+  %gep0 = getelementptr i32, i32 addrspace(1)* %aptr, i32 %tid
+  %gep1 = getelementptr i32, i32 addrspace(1)* %bptr, i32 %tid
+  %outgep = getelementptr i32, i32 addrspace(1)* %out, i32 %tid
+  %a = load i32, i32 addrspace(1)* %gep0, align 4
+  %b = load i32, i32 addrspace(1)* %gep1, align 4
+  %cmp = icmp sgt i32 %a, %b
+  %val = select i1 %cmp, i32 %a, i32 %b
+  store i32 %val, i32 addrspace(1)* %outgep, align 4
+  ret void
+}
+
+; FUNC-LABEL: @s_test_imax_sgt_i32
+; SI: s_max_i32
+define void @s_test_imax_sgt_i32(i32 addrspace(1)* %out, i32 %a, i32 %b) nounwind {
+  %cmp = icmp sgt i32 %a, %b
+  %val = select i1 %cmp, i32 %a, i32 %b
+  store i32 %val, i32 addrspace(1)* %out, align 4
+  ret void
+}
+
+; FUNC-LABEL: @v_test_umax_uge_i32
+; SI: v_max_u32_e32
+define void @v_test_umax_uge_i32(i32 addrspace(1)* %out, i32 addrspace(1)* %aptr, i32 addrspace(1)* %bptr) nounwind {
+  %tid = call i32 @llvm.r600.read.tidig.x() nounwind readnone
+  %gep0 = getelementptr i32, i32 addrspace(1)* %aptr, i32 %tid
+  %gep1 = getelementptr i32, i32 addrspace(1)* %bptr, i32 %tid
+  %outgep = getelementptr i32, i32 addrspace(1)* %out, i32 %tid
+  %a = load i32, i32 addrspace(1)* %gep0, align 4
+  %b = load i32, i32 addrspace(1)* %gep1, align 4
+  %cmp = icmp uge i32 %a, %b
+  %val = select i1 %cmp, i32 %a, i32 %b
+  store i32 %val, i32 addrspace(1)* %outgep, align 4
+  ret void
+}
+
+; FUNC-LABEL: @s_test_umax_uge_i32
+; SI: s_max_u32
+define void @s_test_umax_uge_i32(i32 addrspace(1)* %out, i32 %a, i32 %b) nounwind {
+  %cmp = icmp uge i32 %a, %b
+  %val = select i1 %cmp, i32 %a, i32 %b
+  store i32 %val, i32 addrspace(1)* %out, align 4
+  ret void
+}
+
+; FUNC-LABEL: @v_test_umax_ugt_i32
+; SI: v_max_u32_e32
+define void @v_test_umax_ugt_i32(i32 addrspace(1)* %out, i32 addrspace(1)* %aptr, i32 addrspace(1)* %bptr) nounwind {
+  %tid = call i32 @llvm.r600.read.tidig.x() nounwind readnone
+  %gep0 = getelementptr i32, i32 addrspace(1)* %aptr, i32 %tid
+  %gep1 = getelementptr i32, i32 addrspace(1)* %bptr, i32 %tid
+  %outgep = getelementptr i32, i32 addrspace(1)* %out, i32 %tid
+  %a = load i32, i32 addrspace(1)* %gep0, align 4
+  %b = load i32, i32 addrspace(1)* %gep1, align 4
+  %cmp = icmp ugt i32 %a, %b
+  %val = select i1 %cmp, i32 %a, i32 %b
+  store i32 %val, i32 addrspace(1)* %outgep, align 4
+  ret void
+}
+
+; FUNC-LABEL: @s_test_umax_ugt_i32
+; SI: s_max_u32
+define void @s_test_umax_ugt_i32(i32 addrspace(1)* %out, i32 %a, i32 %b) nounwind {
+  %cmp = icmp ugt i32 %a, %b
+  %val = select i1 %cmp, i32 %a, i32 %b
+  store i32 %val, i32 addrspace(1)* %out, align 4
+  ret void
+}
+
+; Make sure redundant and removed
+; FUNC-LABEL: {{^}}simplify_demanded_bits_test_umax_ugt_i16:
+; SI-DAG: s_load_dword [[A:s[0-9]+]], {{s\[[0-9]+:[0-9]+\]}}, 0xb
+; SI-DAG: s_load_dword [[B:s[0-9]+]], {{s\[[0-9]+:[0-9]+\]}}, 0xc
+; SI: s_max_u32 [[MIN:s[0-9]+]], [[A]], [[B]]
+; SI-NEXT: v_mov_b32_e32 [[VMIN:v[0-9]+]], [[MIN]]
+; SI-NEXT: buffer_store_dword [[VMIN]]
+define void @simplify_demanded_bits_test_umax_ugt_i16(i32 addrspace(1)* %out, i16 zeroext %a, i16 zeroext %b) nounwind {
+  %a.ext = zext i16 %a to i32
+  %b.ext = zext i16 %b to i32
+  %cmp = icmp ugt i32 %a.ext, %b.ext
+  %val = select i1 %cmp, i32 %a.ext, i32 %b.ext
+  %mask = and i32 %val, 65535
+  store i32 %mask, i32 addrspace(1)* %out
+  ret void
+}
+
+; Make sure redundant sign_extend_inreg removed.
+
+; FUNC-LABEL: {{^}}simplify_demanded_bits_test_min_slt_i16:
+; SI-DAG: s_load_dword [[A:s[0-9]+]], {{s\[[0-9]+:[0-9]+\]}}, 0xb
+; SI-DAG: s_load_dword [[B:s[0-9]+]], {{s\[[0-9]+:[0-9]+\]}}, 0xc
+; SI: s_max_i32 [[MIN:s[0-9]+]], [[A]], [[B]]
+; SI-NEXT: v_mov_b32_e32 [[VMIN:v[0-9]+]], [[MIN]]
+; SI-NEXT: buffer_store_dword [[VMIN]]
+define void @simplify_demanded_bits_test_min_slt_i16(i32 addrspace(1)* %out, i16 signext %a, i16 signext %b) nounwind {
+  %a.ext = sext i16 %a to i32
+  %b.ext = sext i16 %b to i32
+  %cmp = icmp sgt i32 %a.ext, %b.ext
+  %val = select i1 %cmp, i32 %a.ext, i32 %b.ext
+  %shl = shl i32 %val, 16
+  %sextinreg = ashr i32 %shl, 16
+  store i32 %sextinreg, i32 addrspace(1)* %out
+  ret void
+}
+
+; FIXME: Should get match min/max through extends inserted by
+; legalization.
+
+; FUNC-LABEL: {{^}}s_test_imin_sge_i16:
+; SI: s_sext_i32_i16
+; SI: s_sext_i32_i16
+; SI: v_cmp_ge_i32_e32
+; SI: v_cndmask_b32
+define void @s_test_imin_sge_i16(i16 addrspace(1)* %out, i16 %a, i16 %b) nounwind {
+  %cmp = icmp sge i16 %a, %b
+  %val = select i1 %cmp, i16 %a, i16 %b
+  store i16 %val, i16 addrspace(1)* %out
+  ret void
+}
diff --git a/test/CodeGen/R600/max3.ll b/test/CodeGen/AMDGPU/max3.ll
similarity index 100%
rename from test/CodeGen/R600/max3.ll
rename to test/CodeGen/AMDGPU/max3.ll
diff --git a/test/CodeGen/AMDGPU/merge-stores.ll b/test/CodeGen/AMDGPU/merge-stores.ll
new file mode 100644
index 00000000000..34a2fc7ffa7
--- /dev/null
+++ b/test/CodeGen/AMDGPU/merge-stores.ll
@@ -0,0 +1,633 @@
+; RUN: llc -march=amdgcn -verify-machineinstrs < %s | FileCheck -check-prefix=SI -check-prefix=GCN %s
+; RUN: llc -march=amdgcn -mcpu=bonaire -verify-machineinstrs < %s | FileCheck -check-prefix=SI -check-prefix=GCN %s
+
+; Run with devices with different unaligned load restrictions.
+
+; TODO: Vector element tests
+; TODO: Non-zero base offset for load and store combinations
+; TODO: Same base addrspacecasted
+
+
+; GCN-LABEL: {{^}}merge_global_store_2_constants_i8:
+; GCN: buffer_store_byte
+; GCN: buffer_store_byte
+; GCN: s_endpgm
+define void @merge_global_store_2_constants_i8(i8 addrspace(1)* %out) #0 {
+  %out.gep.1 = getelementptr i8, i8 addrspace(1)* %out, i32 1
+
+  store i8 123, i8 addrspace(1)* %out.gep.1
+  store i8 456, i8 addrspace(1)* %out, align 2
+  ret void
+}
+
+; GCN-LABEL: {{^}}merge_global_store_2_constants_i8_natural_align:
+; GCN: buffer_store_byte
+; GCN: buffer_store_byte
+; GCN: s_endpgm
+define void @merge_global_store_2_constants_i8_natural_align(i8 addrspace(1)* %out) #0 {
+  %out.gep.1 = getelementptr i8, i8 addrspace(1)* %out, i32 1
+
+  store i8 123, i8 addrspace(1)* %out.gep.1
+  store i8 456, i8 addrspace(1)* %out
+  ret void
+}
+
+; GCN-LABEL: {{^}}merge_global_store_2_constants_i16:
+; GCN: buffer_store_dword v
+define void @merge_global_store_2_constants_i16(i16 addrspace(1)* %out) #0 {
+  %out.gep.1 = getelementptr i16, i16 addrspace(1)* %out, i32 1
+
+  store i16 123, i16 addrspace(1)* %out.gep.1
+  store i16 456, i16 addrspace(1)* %out, align 4
+  ret void
+}
+
+; GCN-LABEL: {{^}}merge_global_store_2_constants_0_i16:
+; GCN: buffer_store_dword v
+define void @merge_global_store_2_constants_0_i16(i16 addrspace(1)* %out) #0 {
+  %out.gep.1 = getelementptr i16, i16 addrspace(1)* %out, i32 1
+
+  store i16 0, i16 addrspace(1)* %out.gep.1
+  store i16 0, i16 addrspace(1)* %out, align 4
+  ret void
+}
+
+; GCN-LABEL: {{^}}merge_global_store_2_constants_i16_natural_align:
+; GCN: buffer_store_short
+; GCN: buffer_store_short
+; GCN: s_endpgm
+define void @merge_global_store_2_constants_i16_natural_align(i16 addrspace(1)* %out) #0 {
+  %out.gep.1 = getelementptr i16, i16 addrspace(1)* %out, i32 1
+
+  store i16 123, i16 addrspace(1)* %out.gep.1
+  store i16 456, i16 addrspace(1)* %out
+  ret void
+}
+
+; GCN-LABEL: {{^}}merge_global_store_2_constants_i32:
+; SI-DAG: s_movk_i32 [[SLO:s[0-9]+]], 0x1c8
+; SI-DAG: s_movk_i32 [[SHI:s[0-9]+]], 0x7b
+; SI-DAG: v_mov_b32_e32 v[[LO:[0-9]+]], [[SLO]]
+; SI-DAG: v_mov_b32_e32 v[[HI:[0-9]+]], [[SHI]]
+; GCN: buffer_store_dwordx2 v{{\[}}[[LO]]:[[HI]]{{\]}}
+define void @merge_global_store_2_constants_i32(i32 addrspace(1)* %out) #0 {
+  %out.gep.1 = getelementptr i32, i32 addrspace(1)* %out, i32 1
+
+  store i32 123, i32 addrspace(1)* %out.gep.1
+  store i32 456, i32 addrspace(1)* %out
+  ret void
+}
+
+; GCN-LABEL: {{^}}merge_global_store_2_constants_i32_f32:
+; GCN: buffer_store_dwordx2
+define void @merge_global_store_2_constants_i32_f32(i32 addrspace(1)* %out) #0 {
+  %out.gep.1 = getelementptr i32, i32 addrspace(1)* %out, i32 1
+  %out.gep.1.bc = bitcast i32 addrspace(1)* %out.gep.1 to float addrspace(1)*
+  store float 1.0, float addrspace(1)* %out.gep.1.bc
+  store i32 456, i32 addrspace(1)* %out
+  ret void
+}
+
+; GCN-LABEL: {{^}}merge_global_store_2_constants_f32_i32:
+; SI-DAG: s_mov_b32 [[SLO:s[0-9]+]], 4.0
+; SI-DAG: s_movk_i32 [[SHI:s[0-9]+]], 0x7b{{$}}
+; SI-DAG: v_mov_b32_e32 v[[VLO:[0-9]+]], [[SLO]]
+; SI-DAG: v_mov_b32_e32 v[[VHI:[0-9]+]], [[SHI]]
+; GCN: buffer_store_dwordx2 v{{\[}}[[VLO]]:[[VHI]]{{\]}}
+define void @merge_global_store_2_constants_f32_i32(float addrspace(1)* %out) #0 {
+  %out.gep.1 = getelementptr float, float addrspace(1)* %out, i32 1
+  %out.gep.1.bc = bitcast float addrspace(1)* %out.gep.1 to i32 addrspace(1)*
+  store i32 123, i32 addrspace(1)* %out.gep.1.bc
+  store float 4.0, float addrspace(1)* %out
+  ret void
+}
+
+; GCN-LABEL: {{^}}merge_global_store_4_constants_i32:
+; GCN-DAG: v_mov_b32_e32 v[[HI:[0-9]+]], 0x14d{{$}}
+; GCN-DAG: v_mov_b32_e32 v{{[0-9]+}}, 0x1c8{{$}}
+; GCN-DAG: v_mov_b32_e32 v{{[0-9]+}}, 0x7b{{$}}
+; GCN-DAG: v_mov_b32_e32 v[[LO:[0-9]+]], 0x4d2{{$}}
+; GCN: buffer_store_dwordx4 v{{\[}}[[LO]]:[[HI]]{{\]}}
+define void @merge_global_store_4_constants_i32(i32 addrspace(1)* %out) #0 {
+  %out.gep.1 = getelementptr i32, i32 addrspace(1)* %out, i32 1
+  %out.gep.2 = getelementptr i32, i32 addrspace(1)* %out, i32 2
+  %out.gep.3 = getelementptr i32, i32 addrspace(1)* %out, i32 3
+
+  store i32 123, i32 addrspace(1)* %out.gep.1
+  store i32 456, i32 addrspace(1)* %out.gep.2
+  store i32 333, i32 addrspace(1)* %out.gep.3
+  store i32 1234, i32 addrspace(1)* %out
+  ret void
+}
+
+; GCN-LABEL: {{^}}merge_global_store_4_constants_f32_order:
+; XGCN: buffer_store_dwordx4
+; GCN: buffer_store_dword v
+; GCN: buffer_store_dword v
+; GCN: buffer_store_dwordx2 v
+define void @merge_global_store_4_constants_f32_order(float addrspace(1)* %out) #0 {
+  %out.gep.1 = getelementptr float, float addrspace(1)* %out, i32 1
+  %out.gep.2 = getelementptr float, float addrspace(1)* %out, i32 2
+  %out.gep.3 = getelementptr float, float addrspace(1)* %out, i32 3
+
+  store float 8.0, float addrspace(1)* %out
+  store float 1.0, float addrspace(1)* %out.gep.1
+  store float 2.0, float addrspace(1)* %out.gep.2
+  store float 4.0, float addrspace(1)* %out.gep.3
+  ret void
+}
+
+; First store is out of order. Because of order of combines, the
+; consecutive store fails because only some of the stores have been
+; replaced with integer constant stores, and then won't merge because
+; the types are different.
+
+; GCN-LABEL: {{^}}merge_global_store_4_constants_f32:
+; XGCN: buffer_store_dwordx4
+; GCN: buffer_store_dword v
+; GCN: buffer_store_dword v
+; GCN: buffer_store_dword v
+; GCN: buffer_store_dword v
+define void @merge_global_store_4_constants_f32(float addrspace(1)* %out) #0 {
+  %out.gep.1 = getelementptr float, float addrspace(1)* %out, i32 1
+  %out.gep.2 = getelementptr float, float addrspace(1)* %out, i32 2
+  %out.gep.3 = getelementptr float, float addrspace(1)* %out, i32 3
+
+  store float 1.0, float addrspace(1)* %out.gep.1
+  store float 2.0, float addrspace(1)* %out.gep.2
+  store float 4.0, float addrspace(1)* %out.gep.3
+  store float 8.0, float addrspace(1)* %out
+  ret void
+}
+
+; GCN-LABEL: {{^}}merge_global_store_3_constants_i32:
+; SI-DAG: buffer_store_dwordx2
+; SI-DAG: buffer_store_dword
+; SI-NOT: buffer_store_dword
+; GCN: s_endpgm
+define void @merge_global_store_3_constants_i32(i32 addrspace(1)* %out) #0 {
+  %out.gep.1 = getelementptr i32, i32 addrspace(1)* %out, i32 1
+  %out.gep.2 = getelementptr i32, i32 addrspace(1)* %out, i32 2
+
+  store i32 123, i32 addrspace(1)* %out.gep.1
+  store i32 456, i32 addrspace(1)* %out.gep.2
+  store i32 1234, i32 addrspace(1)* %out
+  ret void
+}
+
+; GCN-LABEL: {{^}}merge_global_store_2_constants_i64:
+; XGCN: buffer_store_dwordx4
+; GCN: buffer_store_dwordx2
+; GCN: buffer_store_dwordx2
+define void @merge_global_store_2_constants_i64(i64 addrspace(1)* %out) #0 {
+  %out.gep.1 = getelementptr i64, i64 addrspace(1)* %out, i64 1
+
+  store i64 123, i64 addrspace(1)* %out.gep.1
+  store i64 456, i64 addrspace(1)* %out
+  ret void
+}
+
+; GCN-LABEL: {{^}}merge_global_store_4_constants_i64:
+; XGCN: buffer_store_dwordx4
+; XGCN: buffer_store_dwordx4
+
+; GCN: buffer_store_dwordx2
+; GCN: buffer_store_dwordx2
+; GCN: buffer_store_dwordx2
+; GCN: buffer_store_dwordx2
+define void @merge_global_store_4_constants_i64(i64 addrspace(1)* %out) #0 {
+  %out.gep.1 = getelementptr i64, i64 addrspace(1)* %out, i64 1
+  %out.gep.2 = getelementptr i64, i64 addrspace(1)* %out, i64 2
+  %out.gep.3 = getelementptr i64, i64 addrspace(1)* %out, i64 3
+
+  store i64 123, i64 addrspace(1)* %out.gep.1
+  store i64 456, i64 addrspace(1)* %out.gep.2
+  store i64 333, i64 addrspace(1)* %out.gep.3
+  store i64 1234, i64 addrspace(1)* %out
+  ret void
+}
+
+; GCN-LABEL: {{^}}merge_global_store_2_adjacent_loads_i32:
+; GCN: buffer_load_dwordx2 [[LOAD:v\[[0-9]+:[0-9]+\]]]
+; GCN: buffer_store_dwordx2 [[LOAD]]
+define void @merge_global_store_2_adjacent_loads_i32(i32 addrspace(1)* %out, i32 addrspace(1)* %in) #0 {
+  %out.gep.1 = getelementptr i32, i32 addrspace(1)* %out, i32 1
+  %in.gep.1 = getelementptr i32, i32 addrspace(1)* %in, i32 1
+
+  %lo = load i32, i32 addrspace(1)* %in
+  %hi = load i32, i32 addrspace(1)* %in.gep.1
+
+  store i32 %lo, i32 addrspace(1)* %out
+  store i32 %hi, i32 addrspace(1)* %out.gep.1
+  ret void
+}
+
+; GCN-LABEL: {{^}}merge_global_store_2_adjacent_loads_i32_nonzero_base:
+; GCN: buffer_load_dwordx2 [[LOAD:v\[[0-9]+:[0-9]+\]]], s{{\[[0-9]+:[0-9]+\]}}, 0 offset:8
+; GCN: buffer_store_dwordx2 [[LOAD]], s{{\[[0-9]+:[0-9]+\]}}, 0 offset:8
+define void @merge_global_store_2_adjacent_loads_i32_nonzero_base(i32 addrspace(1)* %out, i32 addrspace(1)* %in) #0 {
+  %in.gep.0 = getelementptr i32, i32 addrspace(1)* %in, i32 2
+  %in.gep.1 = getelementptr i32, i32 addrspace(1)* %in, i32 3
+
+  %out.gep.0 = getelementptr i32, i32 addrspace(1)* %out, i32 2
+  %out.gep.1 = getelementptr i32, i32 addrspace(1)* %out, i32 3
+  %lo = load i32, i32 addrspace(1)* %in.gep.0
+  %hi = load i32, i32 addrspace(1)* %in.gep.1
+
+  store i32 %lo, i32 addrspace(1)* %out.gep.0
+  store i32 %hi, i32 addrspace(1)* %out.gep.1
+  ret void
+}
+
+; GCN-LABEL: {{^}}merge_global_store_2_adjacent_loads_shuffle_i32:
+; GCN: buffer_load_dword v
+; GCN: buffer_load_dword v
+; GCN: buffer_store_dword v
+; GCN: buffer_store_dword v
+define void @merge_global_store_2_adjacent_loads_shuffle_i32(i32 addrspace(1)* %out, i32 addrspace(1)* %in) #0 {
+  %out.gep.1 = getelementptr i32, i32 addrspace(1)* %out, i32 1
+  %in.gep.1 = getelementptr i32, i32 addrspace(1)* %in, i32 1
+
+  %lo = load i32, i32 addrspace(1)* %in
+  %hi = load i32, i32 addrspace(1)* %in.gep.1
+
+  store i32 %hi, i32 addrspace(1)* %out
+  store i32 %lo, i32 addrspace(1)* %out.gep.1
+  ret void
+}
+
+; GCN-LABEL: {{^}}merge_global_store_4_adjacent_loads_i32:
+; GCN: buffer_load_dwordx4 [[LOAD:v\[[0-9]+:[0-9]+\]]]
+; GCN: buffer_store_dwordx4 [[LOAD]]
+define void @merge_global_store_4_adjacent_loads_i32(i32 addrspace(1)* %out, i32 addrspace(1)* %in) #0 {
+  %out.gep.1 = getelementptr i32, i32 addrspace(1)* %out, i32 1
+  %out.gep.2 = getelementptr i32, i32 addrspace(1)* %out, i32 2
+  %out.gep.3 = getelementptr i32, i32 addrspace(1)* %out, i32 3
+  %in.gep.1 = getelementptr i32, i32 addrspace(1)* %in, i32 1
+  %in.gep.2 = getelementptr i32, i32 addrspace(1)* %in, i32 2
+  %in.gep.3 = getelementptr i32, i32 addrspace(1)* %in, i32 3
+
+  %x = load i32, i32 addrspace(1)* %in
+  %y = load i32, i32 addrspace(1)* %in.gep.1
+  %z = load i32, i32 addrspace(1)* %in.gep.2
+  %w = load i32, i32 addrspace(1)* %in.gep.3
+
+  store i32 %x, i32 addrspace(1)* %out
+  store i32 %y, i32 addrspace(1)* %out.gep.1
+  store i32 %z, i32 addrspace(1)* %out.gep.2
+  store i32 %w, i32 addrspace(1)* %out.gep.3
+  ret void
+}
+
+; GCN-LABEL: {{^}}merge_global_store_3_adjacent_loads_i32:
+; SI-DAG: buffer_load_dwordx2
+; SI-DAG: buffer_load_dword v
+; GCN: s_waitcnt
+; SI-DAG: buffer_store_dword v
+; SI-DAG: buffer_store_dwordx2 v
+; GCN: s_endpgm
+define void @merge_global_store_3_adjacent_loads_i32(i32 addrspace(1)* %out, i32 addrspace(1)* %in) #0 {
+  %out.gep.1 = getelementptr i32, i32 addrspace(1)* %out, i32 1
+  %out.gep.2 = getelementptr i32, i32 addrspace(1)* %out, i32 2
+  %in.gep.1 = getelementptr i32, i32 addrspace(1)* %in, i32 1
+  %in.gep.2 = getelementptr i32, i32 addrspace(1)* %in, i32 2
+
+  %x = load i32, i32 addrspace(1)* %in
+  %y = load i32, i32 addrspace(1)* %in.gep.1
+  %z = load i32, i32 addrspace(1)* %in.gep.2
+
+  store i32 %x, i32 addrspace(1)* %out
+  store i32 %y, i32 addrspace(1)* %out.gep.1
+  store i32 %z, i32 addrspace(1)* %out.gep.2
+  ret void
+}
+
+; GCN-LABEL: {{^}}merge_global_store_4_adjacent_loads_f32:
+; GCN: buffer_load_dwordx4 [[LOAD:v\[[0-9]+:[0-9]+\]]]
+; GCN: buffer_store_dwordx4 [[LOAD]]
+define void @merge_global_store_4_adjacent_loads_f32(float addrspace(1)* %out, float addrspace(1)* %in) #0 {
+  %out.gep.1 = getelementptr float, float addrspace(1)* %out, i32 1
+  %out.gep.2 = getelementptr float, float addrspace(1)* %out, i32 2
+  %out.gep.3 = getelementptr float, float addrspace(1)* %out, i32 3
+  %in.gep.1 = getelementptr float, float addrspace(1)* %in, i32 1
+  %in.gep.2 = getelementptr float, float addrspace(1)* %in, i32 2
+  %in.gep.3 = getelementptr float, float addrspace(1)* %in, i32 3
+
+  %x = load float, float addrspace(1)* %in
+  %y = load float, float addrspace(1)* %in.gep.1
+  %z = load float, float addrspace(1)* %in.gep.2
+  %w = load float, float addrspace(1)* %in.gep.3
+
+  store float %x, float addrspace(1)* %out
+  store float %y, float addrspace(1)* %out.gep.1
+  store float %z, float addrspace(1)* %out.gep.2
+  store float %w, float addrspace(1)* %out.gep.3
+  ret void
+}
+
+; GCN-LABEL: {{^}}merge_global_store_4_adjacent_loads_i32_nonzero_base:
+; GCN: buffer_load_dwordx4 [[LOAD:v\[[0-9]+:[0-9]+\]]], s{{\[[0-9]+:[0-9]+\]}}, 0 offset:44
+; GCN: buffer_store_dwordx4 [[LOAD]], s{{\[[0-9]+:[0-9]+\]}}, 0 offset:28
+define void @merge_global_store_4_adjacent_loads_i32_nonzero_base(i32 addrspace(1)* %out, i32 addrspace(1)* %in) #0 {
+  %in.gep.0 = getelementptr i32, i32 addrspace(1)* %in, i32 11
+  %in.gep.1 = getelementptr i32, i32 addrspace(1)* %in, i32 12
+  %in.gep.2 = getelementptr i32, i32 addrspace(1)* %in, i32 13
+  %in.gep.3 = getelementptr i32, i32 addrspace(1)* %in, i32 14
+  %out.gep.0 = getelementptr i32, i32 addrspace(1)* %out, i32 7
+  %out.gep.1 = getelementptr i32, i32 addrspace(1)* %out, i32 8
+  %out.gep.2 = getelementptr i32, i32 addrspace(1)* %out, i32 9
+  %out.gep.3 = getelementptr i32, i32 addrspace(1)* %out, i32 10
+
+  %x = load i32, i32 addrspace(1)* %in.gep.0
+  %y = load i32, i32 addrspace(1)* %in.gep.1
+  %z = load i32, i32 addrspace(1)* %in.gep.2
+  %w = load i32, i32 addrspace(1)* %in.gep.3
+
+  store i32 %x, i32 addrspace(1)* %out.gep.0
+  store i32 %y, i32 addrspace(1)* %out.gep.1
+  store i32 %z, i32 addrspace(1)* %out.gep.2
+  store i32 %w, i32 addrspace(1)* %out.gep.3
+  ret void
+}
+
+; GCN-LABEL: {{^}}merge_global_store_4_adjacent_loads_inverse_i32:
+; GCN: buffer_load_dwordx4 [[LOAD:v\[[0-9]+:[0-9]+\]]]
+; GCN: s_barrier
+; GCN: buffer_store_dwordx4 [[LOAD]]
+define void @merge_global_store_4_adjacent_loads_inverse_i32(i32 addrspace(1)* %out, i32 addrspace(1)* %in) #0 {
+  %out.gep.1 = getelementptr i32, i32 addrspace(1)* %out, i32 1
+  %out.gep.2 = getelementptr i32, i32 addrspace(1)* %out, i32 2
+  %out.gep.3 = getelementptr i32, i32 addrspace(1)* %out, i32 3
+  %in.gep.1 = getelementptr i32, i32 addrspace(1)* %in, i32 1
+  %in.gep.2 = getelementptr i32, i32 addrspace(1)* %in, i32 2
+  %in.gep.3 = getelementptr i32, i32 addrspace(1)* %in, i32 3
+
+  %x = load i32, i32 addrspace(1)* %in
+  %y = load i32, i32 addrspace(1)* %in.gep.1
+  %z = load i32, i32 addrspace(1)* %in.gep.2
+  %w = load i32, i32 addrspace(1)* %in.gep.3
+
+  ; Make sure the barrier doesn't stop this
+  tail call void @llvm.AMDGPU.barrier.local() #1
+
+  store i32 %w, i32 addrspace(1)* %out.gep.3
+  store i32 %z, i32 addrspace(1)* %out.gep.2
+  store i32 %y, i32 addrspace(1)* %out.gep.1
+  store i32 %x, i32 addrspace(1)* %out
+
+  ret void
+}
+
+; TODO: Re-packing of loaded register required. Maybe an IR pass
+; should catch this?
+
+; GCN-LABEL: {{^}}merge_global_store_4_adjacent_loads_shuffle_i32:
+; GCN: buffer_load_dword v
+; GCN: buffer_load_dword v
+; GCN: buffer_load_dword v
+; GCN: buffer_load_dword v
+; GCN: s_barrier
+; GCN: buffer_store_dword v
+; GCN: buffer_store_dword v
+; GCN: buffer_store_dword v
+; GCN: buffer_store_dword v
+define void @merge_global_store_4_adjacent_loads_shuffle_i32(i32 addrspace(1)* %out, i32 addrspace(1)* %in) #0 {
+  %out.gep.1 = getelementptr i32, i32 addrspace(1)* %out, i32 1
+  %out.gep.2 = getelementptr i32, i32 addrspace(1)* %out, i32 2
+  %out.gep.3 = getelementptr i32, i32 addrspace(1)* %out, i32 3
+  %in.gep.1 = getelementptr i32, i32 addrspace(1)* %in, i32 1
+  %in.gep.2 = getelementptr i32, i32 addrspace(1)* %in, i32 2
+  %in.gep.3 = getelementptr i32, i32 addrspace(1)* %in, i32 3
+
+  %x = load i32, i32 addrspace(1)* %in
+  %y = load i32, i32 addrspace(1)* %in.gep.1
+  %z = load i32, i32 addrspace(1)* %in.gep.2
+  %w = load i32, i32 addrspace(1)* %in.gep.3
+
+  ; Make sure the barrier doesn't stop this
+  tail call void @llvm.AMDGPU.barrier.local() #1
+
+  store i32 %w, i32 addrspace(1)* %out
+  store i32 %z, i32 addrspace(1)* %out.gep.1
+  store i32 %y, i32 addrspace(1)* %out.gep.2
+  store i32 %x, i32 addrspace(1)* %out.gep.3
+
+  ret void
+}
+
+; GCN-LABEL: {{^}}merge_global_store_4_adjacent_loads_i8:
+; GCN: buffer_load_dword [[LOAD:v[0-9]+]]
+; GCN: buffer_store_dword [[LOAD]]
+; GCN: s_endpgm
+define void @merge_global_store_4_adjacent_loads_i8(i8 addrspace(1)* %out, i8 addrspace(1)* %in) #0 {
+  %out.gep.1 = getelementptr i8, i8 addrspace(1)* %out, i8 1
+  %out.gep.2 = getelementptr i8, i8 addrspace(1)* %out, i8 2
+  %out.gep.3 = getelementptr i8, i8 addrspace(1)* %out, i8 3
+  %in.gep.1 = getelementptr i8, i8 addrspace(1)* %in, i8 1
+  %in.gep.2 = getelementptr i8, i8 addrspace(1)* %in, i8 2
+  %in.gep.3 = getelementptr i8, i8 addrspace(1)* %in, i8 3
+
+  %x = load i8, i8 addrspace(1)* %in, align 4
+  %y = load i8, i8 addrspace(1)* %in.gep.1
+  %z = load i8, i8 addrspace(1)* %in.gep.2
+  %w = load i8, i8 addrspace(1)* %in.gep.3
+
+  store i8 %x, i8 addrspace(1)* %out, align 4
+  store i8 %y, i8 addrspace(1)* %out.gep.1
+  store i8 %z, i8 addrspace(1)* %out.gep.2
+  store i8 %w, i8 addrspace(1)* %out.gep.3
+  ret void
+}
+
+; GCN-LABEL: {{^}}merge_global_store_4_adjacent_loads_i8_natural_align:
+; GCN: buffer_load_ubyte
+; GCN: buffer_load_ubyte
+; GCN: buffer_load_ubyte
+; GCN: buffer_load_ubyte
+; GCN: buffer_store_byte
+; GCN: buffer_store_byte
+; GCN: buffer_store_byte
+; GCN: buffer_store_byte
+; GCN: s_endpgm
+define void @merge_global_store_4_adjacent_loads_i8_natural_align(i8 addrspace(1)* %out, i8 addrspace(1)* %in) #0 {
+  %out.gep.1 = getelementptr i8, i8 addrspace(1)* %out, i8 1
+  %out.gep.2 = getelementptr i8, i8 addrspace(1)* %out, i8 2
+  %out.gep.3 = getelementptr i8, i8 addrspace(1)* %out, i8 3
+  %in.gep.1 = getelementptr i8, i8 addrspace(1)* %in, i8 1
+  %in.gep.2 = getelementptr i8, i8 addrspace(1)* %in, i8 2
+  %in.gep.3 = getelementptr i8, i8 addrspace(1)* %in, i8 3
+
+  %x = load i8, i8 addrspace(1)* %in
+  %y = load i8, i8 addrspace(1)* %in.gep.1
+  %z = load i8, i8 addrspace(1)* %in.gep.2
+  %w = load i8, i8 addrspace(1)* %in.gep.3
+
+  store i8 %x, i8 addrspace(1)* %out
+  store i8 %y, i8 addrspace(1)* %out.gep.1
+  store i8 %z, i8 addrspace(1)* %out.gep.2
+  store i8 %w, i8 addrspace(1)* %out.gep.3
+  ret void
+}
+
+; This works once AA is enabled on the subtarget
+; GCN-LABEL: {{^}}merge_global_store_4_vector_elts_loads_v4i32:
+; GCN: buffer_load_dwordx4 [[LOAD:v\[[0-9]+:[0-9]+\]]]
+; XGCN: buffer_store_dwordx4 [[LOAD]]
+; GCN: buffer_store_dword v
+; GCN: buffer_store_dword v
+; GCN: buffer_store_dword v
+; GCN: buffer_store_dword v
+define void @merge_global_store_4_vector_elts_loads_v4i32(i32 addrspace(1)* %out, <4 x i32> addrspace(1)* %in) #0 {
+  %out.gep.1 = getelementptr i32, i32 addrspace(1)* %out, i32 1
+  %out.gep.2 = getelementptr i32, i32 addrspace(1)* %out, i32 2
+  %out.gep.3 = getelementptr i32, i32 addrspace(1)* %out, i32 3
+  %vec = load <4 x i32>, <4 x i32> addrspace(1)* %in
+
+  %x = extractelement <4 x i32> %vec, i32 0
+  %y = extractelement <4 x i32> %vec, i32 1
+  %z = extractelement <4 x i32> %vec, i32 2
+  %w = extractelement <4 x i32> %vec, i32 3
+
+  store i32 %x, i32 addrspace(1)* %out
+  store i32 %y, i32 addrspace(1)* %out.gep.1
+  store i32 %z, i32 addrspace(1)* %out.gep.2
+  store i32 %w, i32 addrspace(1)* %out.gep.3
+  ret void
+}
+
+; GCN-LABEL: {{^}}merge_local_store_2_constants_i8:
+; GCN: ds_write_b8
+; GCN: ds_write_b8
+; GCN: s_endpgm
+define void @merge_local_store_2_constants_i8(i8 addrspace(3)* %out) #0 {
+  %out.gep.1 = getelementptr i8, i8 addrspace(3)* %out, i32 1
+
+  store i8 123, i8 addrspace(3)* %out.gep.1
+  store i8 456, i8 addrspace(3)* %out, align 2
+  ret void
+}
+
+; GCN-LABEL: {{^}}merge_local_store_2_constants_i32:
+; GCN-DAG: s_movk_i32 [[SLO:s[0-9]+]], 0x1c8
+; GCN-DAG: s_movk_i32 [[SHI:s[0-9]+]], 0x7b
+; GCN-DAG: v_mov_b32_e32 v[[LO:[0-9]+]], [[SLO]]
+; GCN-DAG: v_mov_b32_e32 v[[HI:[0-9]+]], [[SHI]]
+; GCN: ds_write2_b32 v{{[0-9]+}}, v[[LO]], v[[HI]] offset1:1{{$}}
+define void @merge_local_store_2_constants_i32(i32 addrspace(3)* %out) #0 {
+  %out.gep.1 = getelementptr i32, i32 addrspace(3)* %out, i32 1
+
+  store i32 123, i32 addrspace(3)* %out.gep.1
+  store i32 456, i32 addrspace(3)* %out
+  ret void
+}
+
+; GCN-LABEL: {{^}}merge_local_store_4_constants_i32:
+; GCN: ds_write_b32
+; GCN: ds_write_b32
+; GCN: ds_write_b32
+; GCN: ds_write_b32
+define void @merge_local_store_4_constants_i32(i32 addrspace(3)* %out) #0 {
+  %out.gep.1 = getelementptr i32, i32 addrspace(3)* %out, i32 1
+  %out.gep.2 = getelementptr i32, i32 addrspace(3)* %out, i32 2
+  %out.gep.3 = getelementptr i32, i32 addrspace(3)* %out, i32 3
+
+  store i32 123, i32 addrspace(3)* %out.gep.1
+  store i32 456, i32 addrspace(3)* %out.gep.2
+  store i32 333, i32 addrspace(3)* %out.gep.3
+  store i32 1234, i32 addrspace(3)* %out
+  ret void
+}
+
+; GCN-LABEL: {{^}}merge_global_store_5_constants_i32:
+; GCN-DAG: v_mov_b32_e32 v[[LO:[0-9]+]], 9{{$}}
+; GCN-DAG: v_mov_b32_e32 v[[HI4:[0-9]+]], -12{{$}}
+; GCN: buffer_store_dwordx4 v{{\[}}[[LO]]:[[HI4]]{{\]}}
+; GCN: v_mov_b32_e32 v[[HI:[0-9]+]], 11{{$}}
+; GCN: buffer_store_dword v[[HI]]
+define void @merge_global_store_5_constants_i32(i32 addrspace(1)* %out) {
+  store i32 9, i32 addrspace(1)* %out, align 4
+  %idx1 = getelementptr inbounds i32, i32 addrspace(1)* %out, i64 1
+  store i32 12, i32 addrspace(1)* %idx1, align 4
+  %idx2 = getelementptr inbounds i32, i32 addrspace(1)* %out, i64 2
+  store i32 16, i32 addrspace(1)* %idx2, align 4
+  %idx3 = getelementptr inbounds i32, i32 addrspace(1)* %out, i64 3
+  store i32 -12, i32 addrspace(1)* %idx3, align 4
+  %idx4 = getelementptr inbounds i32, i32 addrspace(1)* %out, i64 4
+  store i32 11, i32 addrspace(1)* %idx4, align 4
+  ret void
+}
+
+; GCN-LABEL: {{^}}merge_global_store_6_constants_i32:
+; GCN: buffer_store_dwordx4
+; GCN: buffer_store_dwordx2
+define void @merge_global_store_6_constants_i32(i32 addrspace(1)* %out) {
+  store i32 13, i32 addrspace(1)* %out, align 4
+  %idx1 = getelementptr inbounds i32, i32 addrspace(1)* %out, i64 1
+  store i32 15, i32 addrspace(1)* %idx1, align 4
+  %idx2 = getelementptr inbounds i32, i32 addrspace(1)* %out, i64 2
+  store i32 62, i32 addrspace(1)* %idx2, align 4
+  %idx3 = getelementptr inbounds i32, i32 addrspace(1)* %out, i64 3
+  store i32 63, i32 addrspace(1)* %idx3, align 4
+  %idx4 = getelementptr inbounds i32, i32 addrspace(1)* %out, i64 4
+  store i32 11, i32 addrspace(1)* %idx4, align 4
+  %idx5 = getelementptr inbounds i32, i32 addrspace(1)* %out, i64 5
+  store i32 123, i32 addrspace(1)* %idx5, align 4
+  ret void
+}
+
+; GCN-LABEL: {{^}}merge_global_store_7_constants_i32:
+; GCN: buffer_store_dwordx4
+; GCN: buffer_store_dwordx2
+; GCN: buffer_store_dword v
+define void @merge_global_store_7_constants_i32(i32 addrspace(1)* %out) {
+  store i32 34, i32 addrspace(1)* %out, align 4
+  %idx1 = getelementptr inbounds i32, i32 addrspace(1)* %out, i64 1
+  store i32 999, i32 addrspace(1)* %idx1, align 4
+  %idx2 = getelementptr inbounds i32, i32 addrspace(1)* %out, i64 2
+  store i32 65, i32 addrspace(1)* %idx2, align 4
+  %idx3 = getelementptr inbounds i32, i32 addrspace(1)* %out, i64 3
+  store i32 33, i32 addrspace(1)* %idx3, align 4
+  %idx4 = getelementptr inbounds i32, i32 addrspace(1)* %out, i64 4
+  store i32 98, i32 addrspace(1)* %idx4, align 4
+  %idx5 = getelementptr inbounds i32, i32 addrspace(1)* %out, i64 5
+  store i32 91, i32 addrspace(1)* %idx5, align 4
+  %idx6 = getelementptr inbounds i32, i32 addrspace(1)* %out, i64 6
+  store i32 212, i32 addrspace(1)* %idx6, align 4
+  ret void
+}
+
+; GCN-LABEL: {{^}}merge_global_store_8_constants_i32:
+; XGCN: buffer_store_dwordx4
+; XGCN: buffer_store_dwordx4
+
+; GCN: buffer_store_dword v
+; GCN: buffer_store_dword v
+; GCN: buffer_store_dword v
+; GCN: buffer_store_dword v
+; GCN: buffer_store_dword v
+; GCN: buffer_store_dword v
+; GCN: buffer_store_dword v
+; GCN: buffer_store_dword v
+define void @merge_global_store_8_constants_i32(i32 addrspace(1)* %out) {
+  store i32 34, i32 addrspace(1)* %out, align 4
+  %idx1 = getelementptr inbounds i32, i32 addrspace(1)* %out, i64 1
+  store i32 999, i32 addrspace(1)* %idx1, align 4
+  %idx2 = getelementptr inbounds i32, i32 addrspace(1)* %out, i64 2
+  store i32 65, i32 addrspace(1)* %idx2, align 4
+  %idx3 = getelementptr inbounds i32, i32 addrspace(1)* %out, i64 3
+  store i32 33, i32 addrspace(1)* %idx3, align 4
+  %idx4 = getelementptr inbounds i32, i32 addrspace(1)* %out, i64 4
+  store i32 98, i32 addrspace(1)* %idx4, align 4
+  %idx5 = getelementptr inbounds i32, i32 addrspace(1)* %out, i64 5
+  store i32 91, i32 addrspace(1)* %idx5, align 4
+  %idx6 = getelementptr inbounds i32, i32 addrspace(1)* %out, i64 6
+  store i32 212, i32 addrspace(1)* %idx6, align 4
+  %idx7 = getelementptr inbounds i32, i32 addrspace(1)* %out, i64 7
+  store i32 999, i32 addrspace(1)* %idx7, align 4
+  ret void
+}
+
+declare void @llvm.AMDGPU.barrier.local() #1
+
+attributes #0 = { nounwind }
+attributes #1 = { noduplicate nounwind }
diff --git a/test/CodeGen/AMDGPU/min.ll b/test/CodeGen/AMDGPU/min.ll
new file mode 100644
index 00000000000..0332d1a8e40
--- /dev/null
+++ b/test/CodeGen/AMDGPU/min.ll
@@ -0,0 +1,189 @@
+; RUN: llc -march=amdgcn -mcpu=SI < %s | FileCheck -check-prefix=SI -check-prefix=FUNC %s
+
+declare i32 @llvm.r600.read.tidig.x() nounwind readnone
+
+; FUNC-LABEL: @v_test_imin_sle_i32
+; SI: v_min_i32_e32
+define void @v_test_imin_sle_i32(i32 addrspace(1)* %out, i32 addrspace(1)* %aptr, i32 addrspace(1)* %bptr) nounwind {
+  %tid = call i32 @llvm.r600.read.tidig.x() nounwind readnone
+  %gep0 = getelementptr i32, i32 addrspace(1)* %aptr, i32 %tid
+  %gep1 = getelementptr i32, i32 addrspace(1)* %bptr, i32 %tid
+  %outgep = getelementptr i32, i32 addrspace(1)* %out, i32 %tid
+  %a = load i32, i32 addrspace(1)* %gep0, align 4
+  %b = load i32, i32 addrspace(1)* %gep1, align 4
+  %cmp = icmp sle i32 %a, %b
+  %val = select i1 %cmp, i32 %a, i32 %b
+  store i32 %val, i32 addrspace(1)* %outgep, align 4
+  ret void
+}
+
+; FUNC-LABEL: @s_test_imin_sle_i32
+; SI: s_min_i32
+define void @s_test_imin_sle_i32(i32 addrspace(1)* %out, i32 %a, i32 %b) nounwind {
+  %cmp = icmp sle i32 %a, %b
+  %val = select i1 %cmp, i32 %a, i32 %b
+  store i32 %val, i32 addrspace(1)* %out, align 4
+  ret void
+}
+
+; FUNC-LABEL: @v_test_imin_slt_i32
+; SI: v_min_i32_e32
+define void @v_test_imin_slt_i32(i32 addrspace(1)* %out, i32 addrspace(1)* %aptr, i32 addrspace(1)* %bptr) nounwind {
+  %tid = call i32 @llvm.r600.read.tidig.x() nounwind readnone
+  %gep0 = getelementptr i32, i32 addrspace(1)* %aptr, i32 %tid
+  %gep1 = getelementptr i32, i32 addrspace(1)* %bptr, i32 %tid
+  %outgep = getelementptr i32, i32 addrspace(1)* %out, i32 %tid
+  %a = load i32, i32 addrspace(1)* %gep0, align 4
+  %b = load i32, i32 addrspace(1)* %gep1, align 4
+  %cmp = icmp slt i32 %a, %b
+  %val = select i1 %cmp, i32 %a, i32 %b
+  store i32 %val, i32 addrspace(1)* %outgep, align 4
+  ret void
+}
+
+; FUNC-LABEL: @s_test_imin_slt_i32
+; SI: s_min_i32
+define void @s_test_imin_slt_i32(i32 addrspace(1)* %out, i32 %a, i32 %b) nounwind {
+  %cmp = icmp slt i32 %a, %b
+  %val = select i1 %cmp, i32 %a, i32 %b
+  store i32 %val, i32 addrspace(1)* %out, align 4
+  ret void
+}
+
+; FUNC-LABEL: {{^}}s_test_imin_slt_imm_i32:
+; SI: s_min_i32 {{s[0-9]+}}, {{s[0-9]+}}, 8
+define void @s_test_imin_slt_imm_i32(i32 addrspace(1)* %out, i32 %a) nounwind {
+  %cmp = icmp slt i32 %a, 8
+  %val = select i1 %cmp, i32 %a, i32 8
+  store i32 %val, i32 addrspace(1)* %out, align 4
+  ret void
+}
+
+; FUNC-LABEL: {{^}}s_test_imin_sle_imm_i32:
+; SI: s_min_i32 {{s[0-9]+}}, {{s[0-9]+}}, 8
+define void @s_test_imin_sle_imm_i32(i32 addrspace(1)* %out, i32 %a) nounwind {
+  %cmp = icmp sle i32 %a, 8
+  %val = select i1 %cmp, i32 %a, i32 8
+  store i32 %val, i32 addrspace(1)* %out, align 4
+  ret void
+}
+
+; FUNC-LABEL: @v_test_umin_ule_i32
+; SI: v_min_u32_e32
+define void @v_test_umin_ule_i32(i32 addrspace(1)* %out, i32 addrspace(1)* %aptr, i32 addrspace(1)* %bptr) nounwind {
+  %tid = call i32 @llvm.r600.read.tidig.x() nounwind readnone
+  %gep0 = getelementptr i32, i32 addrspace(1)* %aptr, i32 %tid
+  %gep1 = getelementptr i32, i32 addrspace(1)* %bptr, i32 %tid
+  %outgep = getelementptr i32, i32 addrspace(1)* %out, i32 %tid
+  %a = load i32, i32 addrspace(1)* %gep0, align 4
+  %b = load i32, i32 addrspace(1)* %gep1, align 4
+  %cmp = icmp ule i32 %a, %b
+  %val = select i1 %cmp, i32 %a, i32 %b
+  store i32 %val, i32 addrspace(1)* %outgep, align 4
+  ret void
+}
+
+; FUNC-LABEL: @s_test_umin_ule_i32
+; SI: s_min_u32
+define void @s_test_umin_ule_i32(i32 addrspace(1)* %out, i32 %a, i32 %b) nounwind {
+  %cmp = icmp ule i32 %a, %b
+  %val = select i1 %cmp, i32 %a, i32 %b
+  store i32 %val, i32 addrspace(1)* %out, align 4
+  ret void
+}
+
+; FUNC-LABEL: @v_test_umin_ult_i32
+; SI: v_min_u32_e32
+define void @v_test_umin_ult_i32(i32 addrspace(1)* %out, i32 addrspace(1)* %aptr, i32 addrspace(1)* %bptr) nounwind {
+  %tid = call i32 @llvm.r600.read.tidig.x() nounwind readnone
+  %gep0 = getelementptr i32, i32 addrspace(1)* %aptr, i32 %tid
+  %gep1 = getelementptr i32, i32 addrspace(1)* %bptr, i32 %tid
+  %outgep = getelementptr i32, i32 addrspace(1)* %out, i32 %tid
+  %a = load i32, i32 addrspace(1)* %gep0, align 4
+  %b = load i32, i32 addrspace(1)* %gep1, align 4
+  %cmp = icmp ult i32 %a, %b
+  %val = select i1 %cmp, i32 %a, i32 %b
+  store i32 %val, i32 addrspace(1)* %outgep, align 4
+  ret void
+}
+
+; FUNC-LABEL: @s_test_umin_ult_i32
+; SI: s_min_u32
+define void @s_test_umin_ult_i32(i32 addrspace(1)* %out, i32 %a, i32 %b) nounwind {
+  %cmp = icmp ult i32 %a, %b
+  %val = select i1 %cmp, i32 %a, i32 %b
+  store i32 %val, i32 addrspace(1)* %out, align 4
+  ret void
+}
+
+; FUNC-LABEL: @v_test_umin_ult_i32_multi_use
+; SI-NOT: v_min
+; SI: v_cmp_lt_u32
+; SI-NEXT: v_cndmask_b32
+; SI-NOT: v_min
+; SI: s_endpgm
+define void @v_test_umin_ult_i32_multi_use(i32 addrspace(1)* %out0, i1 addrspace(1)* %out1, i32 addrspace(1)* %aptr, i32 addrspace(1)* %bptr) nounwind {
+  %tid = call i32 @llvm.r600.read.tidig.x() nounwind readnone
+  %gep0 = getelementptr i32, i32 addrspace(1)* %aptr, i32 %tid
+  %gep1 = getelementptr i32, i32 addrspace(1)* %bptr, i32 %tid
+  %outgep0 = getelementptr i32, i32 addrspace(1)* %out0, i32 %tid
+  %outgep1 = getelementptr i1, i1 addrspace(1)* %out1, i32 %tid
+  %a = load i32, i32 addrspace(1)* %gep0, align 4
+  %b = load i32, i32 addrspace(1)* %gep1, align 4
+  %cmp = icmp ult i32 %a, %b
+  %val = select i1 %cmp, i32 %a, i32 %b
+  store i32 %val, i32 addrspace(1)* %outgep0, align 4
+  store i1 %cmp, i1 addrspace(1)* %outgep1
+  ret void
+}
+
+; Make sure redundant and removed
+; FUNC-LABEL: {{^}}simplify_demanded_bits_test_umin_ult_i16:
+; SI-DAG: s_load_dword [[A:s[0-9]+]], {{s\[[0-9]+:[0-9]+\]}}, 0xb
+; SI-DAG: s_load_dword [[B:s[0-9]+]], {{s\[[0-9]+:[0-9]+\]}}, 0xc
+; SI: s_min_u32 [[MIN:s[0-9]+]], [[A]], [[B]]
+; SI-NEXT: v_mov_b32_e32 [[VMIN:v[0-9]+]], [[MIN]]
+; SI-NEXT: buffer_store_dword [[VMIN]]
+define void @simplify_demanded_bits_test_umin_ult_i16(i32 addrspace(1)* %out, i16 zeroext %a, i16 zeroext %b) nounwind {
+  %a.ext = zext i16 %a to i32
+  %b.ext = zext i16 %b to i32
+  %cmp = icmp ult i32 %a.ext, %b.ext
+  %val = select i1 %cmp, i32 %a.ext, i32 %b.ext
+  %mask = and i32 %val, 65535
+  store i32 %mask, i32 addrspace(1)* %out
+  ret void
+}
+
+; Make sure redundant sign_extend_inreg removed.
+
+; FUNC-LABEL: {{^}}simplify_demanded_bits_test_min_slt_i16:
+; SI-DAG: s_load_dword [[A:s[0-9]+]], {{s\[[0-9]+:[0-9]+\]}}, 0xb
+; SI-DAG: s_load_dword [[B:s[0-9]+]], {{s\[[0-9]+:[0-9]+\]}}, 0xc
+; SI: s_min_i32 [[MIN:s[0-9]+]], [[A]], [[B]]
+; SI-NEXT: v_mov_b32_e32 [[VMIN:v[0-9]+]], [[MIN]]
+; SI-NEXT: buffer_store_dword [[VMIN]]
+define void @simplify_demanded_bits_test_min_slt_i16(i32 addrspace(1)* %out, i16 signext %a, i16 signext %b) nounwind {
+  %a.ext = sext i16 %a to i32
+  %b.ext = sext i16 %b to i32
+  %cmp = icmp slt i32 %a.ext, %b.ext
+  %val = select i1 %cmp, i32 %a.ext, i32 %b.ext
+  %shl = shl i32 %val, 16
+  %sextinreg = ashr i32 %shl, 16
+  store i32 %sextinreg, i32 addrspace(1)* %out
+  ret void
+}
+
+; FIXME: Should get match min/max through extends inserted by
+; legalization.
+
+; FUNC-LABEL: {{^}}s_test_imin_sle_i16:
+; SI: s_sext_i32_i16
+; SI: s_sext_i32_i16
+; SI: v_cmp_le_i32_e32
+; SI: v_cndmask_b32
+define void @s_test_imin_sle_i16(i16 addrspace(1)* %out, i16 %a, i16 %b) nounwind {
+  %cmp = icmp sle i16 %a, %b
+  %val = select i1 %cmp, i16 %a, i16 %b
+  store i16 %val, i16 addrspace(1)* %out
+  ret void
+}
diff --git a/test/CodeGen/R600/min3.ll b/test/CodeGen/AMDGPU/min3.ll
similarity index 100%
rename from test/CodeGen/R600/min3.ll
rename to test/CodeGen/AMDGPU/min3.ll
diff --git a/test/CodeGen/R600/missing-store.ll b/test/CodeGen/AMDGPU/missing-store.ll
similarity index 100%
rename from test/CodeGen/R600/missing-store.ll
rename to test/CodeGen/AMDGPU/missing-store.ll
diff --git a/test/CodeGen/R600/mubuf.ll b/test/CodeGen/AMDGPU/mubuf.ll
similarity index 100%
rename from test/CodeGen/R600/mubuf.ll
rename to test/CodeGen/AMDGPU/mubuf.ll
diff --git a/test/CodeGen/R600/mul.ll b/test/CodeGen/AMDGPU/mul.ll
similarity index 100%
rename from test/CodeGen/R600/mul.ll
rename to test/CodeGen/AMDGPU/mul.ll
diff --git a/test/CodeGen/R600/mul_int24.ll b/test/CodeGen/AMDGPU/mul_int24.ll
similarity index 100%
rename from test/CodeGen/R600/mul_int24.ll
rename to test/CodeGen/AMDGPU/mul_int24.ll
diff --git a/test/CodeGen/R600/mul_uint24.ll b/test/CodeGen/AMDGPU/mul_uint24.ll
similarity index 85%
rename from test/CodeGen/R600/mul_uint24.ll
rename to test/CodeGen/AMDGPU/mul_uint24.ll
index e640a7cd69f..8a0e71d739b 100644
--- a/test/CodeGen/R600/mul_uint24.ll
+++ b/test/CodeGen/AMDGPU/mul_uint24.ll
@@ -52,16 +52,18 @@ entry:
 ; FUNC_LABEL: {{^}}mul24_i64:
 ; EG; MUL_UINT24
 ; EG: MULHI
-; SI: v_mul_u32_u24
 ; FIXME: SI support 24-bit mulhi
-; SI: v_mul_hi_u32
-define void @mul24_i64(i64 addrspace(1)* %out, i64 %a, i64 %b) {
+
+; SI-DAG: v_mul_u32_u24
+; SI-DAG: v_mul_hi_u32
+; SI: s_endpgm
+define void @mul24_i64(i64 addrspace(1)* %out, i64 %a, i64 %b, i64 %c) {
 entry:
-  %0 = shl i64 %a, 40
-  %a_24 = lshr i64 %0, 40
-  %1 = shl i64 %b, 40
-  %b_24 = lshr i64 %1, 40
-  %2 = mul i64 %a_24, %b_24
-  store i64 %2, i64 addrspace(1)* %out
+  %tmp0 = shl i64 %a, 40
+  %a_24 = lshr i64 %tmp0, 40
+  %tmp1 = shl i64 %b, 40
+  %b_24 = lshr i64 %tmp1, 40
+  %tmp2 = mul i64 %a_24, %b_24
+  store i64 %tmp2, i64 addrspace(1)* %out
   ret void
 }
diff --git a/test/CodeGen/R600/mulhu.ll b/test/CodeGen/AMDGPU/mulhu.ll
similarity index 100%
rename from test/CodeGen/R600/mulhu.ll
rename to test/CodeGen/AMDGPU/mulhu.ll
diff --git a/test/CodeGen/R600/no-initializer-constant-addrspace.ll b/test/CodeGen/AMDGPU/no-initializer-constant-addrspace.ll
similarity index 100%
rename from test/CodeGen/R600/no-initializer-constant-addrspace.ll
rename to test/CodeGen/AMDGPU/no-initializer-constant-addrspace.ll
diff --git a/test/CodeGen/R600/no-shrink-extloads.ll b/test/CodeGen/AMDGPU/no-shrink-extloads.ll
similarity index 100%
rename from test/CodeGen/R600/no-shrink-extloads.ll
rename to test/CodeGen/AMDGPU/no-shrink-extloads.ll
diff --git a/test/CodeGen/R600/operand-folding.ll b/test/CodeGen/AMDGPU/operand-folding.ll
similarity index 100%
rename from test/CodeGen/R600/operand-folding.ll
rename to test/CodeGen/AMDGPU/operand-folding.ll
diff --git a/test/CodeGen/R600/operand-spacing.ll b/test/CodeGen/AMDGPU/operand-spacing.ll
similarity index 100%
rename from test/CodeGen/R600/operand-spacing.ll
rename to test/CodeGen/AMDGPU/operand-spacing.ll
diff --git a/test/CodeGen/R600/or.ll b/test/CodeGen/AMDGPU/or.ll
similarity index 100%
rename from test/CodeGen/R600/or.ll
rename to test/CodeGen/AMDGPU/or.ll
diff --git a/test/CodeGen/R600/packetizer.ll b/test/CodeGen/AMDGPU/packetizer.ll
similarity index 100%
rename from test/CodeGen/R600/packetizer.ll
rename to test/CodeGen/AMDGPU/packetizer.ll
diff --git a/test/CodeGen/R600/parallelandifcollapse.ll b/test/CodeGen/AMDGPU/parallelandifcollapse.ll
similarity index 100%
rename from test/CodeGen/R600/parallelandifcollapse.ll
rename to test/CodeGen/AMDGPU/parallelandifcollapse.ll
diff --git a/test/CodeGen/R600/parallelorifcollapse.ll b/test/CodeGen/AMDGPU/parallelorifcollapse.ll
similarity index 100%
rename from test/CodeGen/R600/parallelorifcollapse.ll
rename to test/CodeGen/AMDGPU/parallelorifcollapse.ll
diff --git a/test/CodeGen/R600/predicate-dp4.ll b/test/CodeGen/AMDGPU/predicate-dp4.ll
similarity index 100%
rename from test/CodeGen/R600/predicate-dp4.ll
rename to test/CodeGen/AMDGPU/predicate-dp4.ll
diff --git a/test/CodeGen/R600/predicates.ll b/test/CodeGen/AMDGPU/predicates.ll
similarity index 100%
rename from test/CodeGen/R600/predicates.ll
rename to test/CodeGen/AMDGPU/predicates.ll
diff --git a/test/CodeGen/R600/private-memory-atomics.ll b/test/CodeGen/AMDGPU/private-memory-atomics.ll
similarity index 100%
rename from test/CodeGen/R600/private-memory-atomics.ll
rename to test/CodeGen/AMDGPU/private-memory-atomics.ll
diff --git a/test/CodeGen/R600/private-memory-broken.ll b/test/CodeGen/AMDGPU/private-memory-broken.ll
similarity index 100%
rename from test/CodeGen/R600/private-memory-broken.ll
rename to test/CodeGen/AMDGPU/private-memory-broken.ll
diff --git a/test/CodeGen/R600/private-memory.ll b/test/CodeGen/AMDGPU/private-memory.ll
similarity index 99%
rename from test/CodeGen/R600/private-memory.ll
rename to test/CodeGen/AMDGPU/private-memory.ll
index 1c562978050..645dc04f442 100644
--- a/test/CodeGen/R600/private-memory.ll
+++ b/test/CodeGen/AMDGPU/private-memory.ll
@@ -298,7 +298,7 @@ entry:
 ; FUNC-LABEL: ptrtoint:
 ; SI-NOT: ds_write
 ; SI: buffer_store_dword v{{[0-9]+}}, v{{[0-9]+}}, s[{{[0-9]+:[0-9]+}}], s{{[0-9]+}} offen
-; SI: buffer_load_dword v{{[0-9]+}}, v{{[0-9]+}}, s[{{[0-9]+:[0-9]+}}], s{{[0-9]+}} offen offset:5
+; SI: buffer_load_dword v{{[0-9]+}}, v{{[0-9]+}}, s[{{[0-9]+:[0-9]+}}], s{{[0-9]+}} offen ;
 define void @ptrtoint(i32 addrspace(1)* %out, i32 %a, i32 %b) {
   %alloca = alloca [16 x i32]
   %tmp0 = getelementptr [16 x i32], [16 x i32]* %alloca, i32 0, i32 %a
diff --git a/test/CodeGen/AMDGPU/promote-alloca-bitcast-function.ll b/test/CodeGen/AMDGPU/promote-alloca-bitcast-function.ll
new file mode 100644
index 00000000000..10739df0837
--- /dev/null
+++ b/test/CodeGen/AMDGPU/promote-alloca-bitcast-function.ll
@@ -0,0 +1,22 @@
+; RUN: not llc -march=amdgcn < %s 2>&1 | FileCheck %s
+
+; Make sure that AMDGPUPromoteAlloca doesn't crash if the called
+; function is a constantexpr cast of a function.
+
+declare void @foo(float*) #0
+declare void @foo.varargs(...) #0
+
+; CHECK: error: unsupported call to function foo in crash_call_constexpr_cast
+define void @crash_call_constexpr_cast() #0 {
+  %alloca = alloca i32
+  call void bitcast (void (float*)* @foo to void (i32*)*)(i32* %alloca) #0
+  ret void
+}
+
+define void @crash_call_constexpr_cast_varargs() #0 {
+  %alloca = alloca i32
+  call void bitcast (void (...)* @foo.varargs to void (i32*)*)(i32* %alloca) #0
+  ret void
+}
+
+attributes #0 = { nounwind }
diff --git a/test/CodeGen/AMDGPU/promote-alloca-stored-pointer-value.ll b/test/CodeGen/AMDGPU/promote-alloca-stored-pointer-value.ll
new file mode 100644
index 00000000000..2ee98cc3d2d
--- /dev/null
+++ b/test/CodeGen/AMDGPU/promote-alloca-stored-pointer-value.ll
@@ -0,0 +1,52 @@
+; RUN: llc -march=amdgcn < %s | FileCheck -check-prefix=GCN %s
+
+; Pointer value is stored in a candidate for LDS usage.
+
+; GCN-LABEL: {{^}}stored_lds_pointer_value:
+; GCN: buffer_store_dword v
+define void @stored_lds_pointer_value(float* addrspace(1)* %ptr) #0 {
+  %tmp = alloca float
+  store float 0.0, float *%tmp
+  store float* %tmp, float* addrspace(1)* %ptr
+  ret void
+}
+
+; GCN-LABEL: {{^}}stored_lds_pointer_value_gep:
+; GCN-DAG: s_mov_b32 s{{[0-9]+}}, SCRATCH_RSRC_DWORD0
+; GCN-DAG: s_mov_b32 s{{[0-9]+}}, SCRATCH_RSRC_DWORD1
+; GCN: buffer_store_dword v
+; GCN: buffer_store_dword v
+define void @stored_lds_pointer_value_gep(float* addrspace(1)* %ptr, i32 %idx) #0 {
+bb:
+  %tmp = alloca float, i32 16
+  store float 0.0, float* %tmp
+  %tmp2 = getelementptr inbounds float, float* %tmp, i32 %idx
+  store float* %tmp2, float* addrspace(1)* %ptr
+  ret void
+}
+
+; Pointer value is stored in a candidate for vector usage
+; GCN-LABEL: {{^}}stored_vector_pointer_value:
+; GCN-DAG: s_mov_b32 s{{[0-9]+}}, SCRATCH_RSRC_DWORD0
+; GCN-DAG: s_mov_b32 s{{[0-9]+}}, SCRATCH_RSRC_DWORD1
+; GCN: buffer_store_dword
+; GCN: buffer_store_dword
+; GCN: buffer_store_dword
+; GCN: buffer_store_dword
+define void @stored_vector_pointer_value(i32* addrspace(1)* %out, i32 %index) {
+entry:
+  %tmp0 = alloca [4 x i32]
+  %x = getelementptr [4 x i32], [4 x i32]* %tmp0, i32 0, i32 0
+  %y = getelementptr [4 x i32], [4 x i32]* %tmp0, i32 0, i32 1
+  %z = getelementptr [4 x i32], [4 x i32]* %tmp0, i32 0, i32 2
+  %w = getelementptr [4 x i32], [4 x i32]* %tmp0, i32 0, i32 3
+  store i32 0, i32* %x
+  store i32 1, i32* %y
+  store i32 2, i32* %z
+  store i32 3, i32* %w
+  %tmp1 = getelementptr [4 x i32], [4 x i32]* %tmp0, i32 0, i32 %index
+  store i32* %tmp1, i32* addrspace(1)* %out
+  ret void
+}
+
+attributes #0 = { nounwind }
diff --git a/test/CodeGen/R600/pv-packing.ll b/test/CodeGen/AMDGPU/pv-packing.ll
similarity index 100%
rename from test/CodeGen/R600/pv-packing.ll
rename to test/CodeGen/AMDGPU/pv-packing.ll
diff --git a/test/CodeGen/R600/pv.ll b/test/CodeGen/AMDGPU/pv.ll
similarity index 100%
rename from test/CodeGen/R600/pv.ll
rename to test/CodeGen/AMDGPU/pv.ll
diff --git a/test/CodeGen/R600/r600-encoding.ll b/test/CodeGen/AMDGPU/r600-encoding.ll
similarity index 100%
rename from test/CodeGen/R600/r600-encoding.ll
rename to test/CodeGen/AMDGPU/r600-encoding.ll
diff --git a/test/CodeGen/R600/r600-export-fix.ll b/test/CodeGen/AMDGPU/r600-export-fix.ll
similarity index 100%
rename from test/CodeGen/R600/r600-export-fix.ll
rename to test/CodeGen/AMDGPU/r600-export-fix.ll
diff --git a/test/CodeGen/R600/r600-infinite-loop-bug-while-reorganizing-vector.ll b/test/CodeGen/AMDGPU/r600-infinite-loop-bug-while-reorganizing-vector.ll
similarity index 100%
rename from test/CodeGen/R600/r600-infinite-loop-bug-while-reorganizing-vector.ll
rename to test/CodeGen/AMDGPU/r600-infinite-loop-bug-while-reorganizing-vector.ll
diff --git a/test/CodeGen/R600/r600cfg.ll b/test/CodeGen/AMDGPU/r600cfg.ll
similarity index 100%
rename from test/CodeGen/R600/r600cfg.ll
rename to test/CodeGen/AMDGPU/r600cfg.ll
diff --git a/test/CodeGen/R600/reciprocal.ll b/test/CodeGen/AMDGPU/reciprocal.ll
similarity index 100%
rename from test/CodeGen/R600/reciprocal.ll
rename to test/CodeGen/AMDGPU/reciprocal.ll
diff --git a/test/CodeGen/R600/register-count-comments.ll b/test/CodeGen/AMDGPU/register-count-comments.ll
similarity index 100%
rename from test/CodeGen/R600/register-count-comments.ll
rename to test/CodeGen/AMDGPU/register-count-comments.ll
diff --git a/test/CodeGen/R600/reorder-stores.ll b/test/CodeGen/AMDGPU/reorder-stores.ll
similarity index 100%
rename from test/CodeGen/R600/reorder-stores.ll
rename to test/CodeGen/AMDGPU/reorder-stores.ll
diff --git a/test/CodeGen/R600/rotl.i64.ll b/test/CodeGen/AMDGPU/rotl.i64.ll
similarity index 100%
rename from test/CodeGen/R600/rotl.i64.ll
rename to test/CodeGen/AMDGPU/rotl.i64.ll
diff --git a/test/CodeGen/R600/rotl.ll b/test/CodeGen/AMDGPU/rotl.ll
similarity index 100%
rename from test/CodeGen/R600/rotl.ll
rename to test/CodeGen/AMDGPU/rotl.ll
diff --git a/test/CodeGen/R600/rotr.i64.ll b/test/CodeGen/AMDGPU/rotr.i64.ll
similarity index 100%
rename from test/CodeGen/R600/rotr.i64.ll
rename to test/CodeGen/AMDGPU/rotr.i64.ll
diff --git a/test/CodeGen/R600/rotr.ll b/test/CodeGen/AMDGPU/rotr.ll
similarity index 100%
rename from test/CodeGen/R600/rotr.ll
rename to test/CodeGen/AMDGPU/rotr.ll
diff --git a/test/CodeGen/R600/rsq.ll b/test/CodeGen/AMDGPU/rsq.ll
similarity index 100%
rename from test/CodeGen/R600/rsq.ll
rename to test/CodeGen/AMDGPU/rsq.ll
diff --git a/test/CodeGen/R600/rv7x0_count3.ll b/test/CodeGen/AMDGPU/rv7x0_count3.ll
similarity index 100%
rename from test/CodeGen/R600/rv7x0_count3.ll
rename to test/CodeGen/AMDGPU/rv7x0_count3.ll
diff --git a/test/CodeGen/R600/s_movk_i32.ll b/test/CodeGen/AMDGPU/s_movk_i32.ll
similarity index 100%
rename from test/CodeGen/R600/s_movk_i32.ll
rename to test/CodeGen/AMDGPU/s_movk_i32.ll
diff --git a/test/CodeGen/R600/saddo.ll b/test/CodeGen/AMDGPU/saddo.ll
similarity index 100%
rename from test/CodeGen/R600/saddo.ll
rename to test/CodeGen/AMDGPU/saddo.ll
diff --git a/test/CodeGen/R600/salu-to-valu.ll b/test/CodeGen/AMDGPU/salu-to-valu.ll
similarity index 100%
rename from test/CodeGen/R600/salu-to-valu.ll
rename to test/CodeGen/AMDGPU/salu-to-valu.ll
diff --git a/test/CodeGen/R600/scalar_to_vector.ll b/test/CodeGen/AMDGPU/scalar_to_vector.ll
similarity index 100%
rename from test/CodeGen/R600/scalar_to_vector.ll
rename to test/CodeGen/AMDGPU/scalar_to_vector.ll
diff --git a/test/CodeGen/R600/schedule-fs-loop-nested-if.ll b/test/CodeGen/AMDGPU/schedule-fs-loop-nested-if.ll
similarity index 100%
rename from test/CodeGen/R600/schedule-fs-loop-nested-if.ll
rename to test/CodeGen/AMDGPU/schedule-fs-loop-nested-if.ll
diff --git a/test/CodeGen/R600/schedule-fs-loop-nested.ll b/test/CodeGen/AMDGPU/schedule-fs-loop-nested.ll
similarity index 100%
rename from test/CodeGen/R600/schedule-fs-loop-nested.ll
rename to test/CodeGen/AMDGPU/schedule-fs-loop-nested.ll
diff --git a/test/CodeGen/R600/schedule-fs-loop.ll b/test/CodeGen/AMDGPU/schedule-fs-loop.ll
similarity index 100%
rename from test/CodeGen/R600/schedule-fs-loop.ll
rename to test/CodeGen/AMDGPU/schedule-fs-loop.ll
diff --git a/test/CodeGen/R600/schedule-global-loads.ll b/test/CodeGen/AMDGPU/schedule-global-loads.ll
similarity index 100%
rename from test/CodeGen/R600/schedule-global-loads.ll
rename to test/CodeGen/AMDGPU/schedule-global-loads.ll
diff --git a/test/CodeGen/R600/schedule-if-2.ll b/test/CodeGen/AMDGPU/schedule-if-2.ll
similarity index 100%
rename from test/CodeGen/R600/schedule-if-2.ll
rename to test/CodeGen/AMDGPU/schedule-if-2.ll
diff --git a/test/CodeGen/R600/schedule-if.ll b/test/CodeGen/AMDGPU/schedule-if.ll
similarity index 100%
rename from test/CodeGen/R600/schedule-if.ll
rename to test/CodeGen/AMDGPU/schedule-if.ll
diff --git a/test/CodeGen/R600/schedule-kernel-arg-loads.ll b/test/CodeGen/AMDGPU/schedule-kernel-arg-loads.ll
similarity index 100%
rename from test/CodeGen/R600/schedule-kernel-arg-loads.ll
rename to test/CodeGen/AMDGPU/schedule-kernel-arg-loads.ll
diff --git a/test/CodeGen/R600/schedule-vs-if-nested-loop-failure.ll b/test/CodeGen/AMDGPU/schedule-vs-if-nested-loop-failure.ll
similarity index 100%
rename from test/CodeGen/R600/schedule-vs-if-nested-loop-failure.ll
rename to test/CodeGen/AMDGPU/schedule-vs-if-nested-loop-failure.ll
diff --git a/test/CodeGen/R600/schedule-vs-if-nested-loop.ll b/test/CodeGen/AMDGPU/schedule-vs-if-nested-loop.ll
similarity index 100%
rename from test/CodeGen/R600/schedule-vs-if-nested-loop.ll
rename to test/CodeGen/AMDGPU/schedule-vs-if-nested-loop.ll
diff --git a/test/CodeGen/AMDGPU/scratch-buffer.ll b/test/CodeGen/AMDGPU/scratch-buffer.ll
new file mode 100644
index 00000000000..268869daaa3
--- /dev/null
+++ b/test/CodeGen/AMDGPU/scratch-buffer.ll
@@ -0,0 +1,116 @@
+; RUN: llc -verify-machineinstrs -march=amdgcn -mcpu=SI < %s | FileCheck --check-prefix=GCN --check-prefix=DEFAULT-SCRATCH %s
+; RUN: llc -verify-machineinstrs -march=amdgcn -mcpu=tonga < %s | FileCheck --check-prefix=GCN --check-prefix=DEFAULT-SCRATCH %s
+; RUN: llc -verify-machineinstrs -march=amdgcn -mattr=+huge-scratch-buffer -mcpu=SI < %s | FileCheck --check-prefix=GCN --check-prefix=HUGE-SCRATCH %s
+; RUN: llc -verify-machineinstrs -march=amdgcn -mattr=+huge-scratch-buffer -mcpu=tonga < %s | FileCheck --check-prefix=GCN --check-prefix=HUGE-SCRATCH %s
+
+; When a frame index offset is more than 12-bits, make sure we don't store
+; it in mubuf's offset field.
+
+; Also, make sure we use the same register for storing the scratch buffer addresss
+; for both stores. This register is allocated by the register scavenger, so we
+; should be able to reuse the same regiser for each scratch buffer access.
+
+; GCN-LABEL: {{^}}legal_offset_fi:
+; GCN: v_mov_b32_e32 [[OFFSET:v[0-9]+]], 0{{$}}
+; GCN: buffer_store_dword v{{[0-9]+}}, [[OFFSET]], s[{{[0-9]+}}:{{[0-9]+}}], s{{[0-9]+}} offen
+; GCN: v_mov_b32_e32 [[OFFSET]], 0x8000
+; GCN: buffer_store_dword v{{[0-9]+}}, [[OFFSET]], s[{{[0-9]+}}:{{[0-9]+}}], s{{[0-9]+}} offen{{$}}
+
+define void @legal_offset_fi(i32 addrspace(1)* %out, i32 %cond, i32 %if_offset, i32 %else_offset) {
+entry:
+  %scratch0 = alloca [8192 x i32]
+  %scratch1 = alloca [8192 x i32]
+
+  %scratchptr0 = getelementptr [8192 x i32], [8192 x i32]* %scratch0, i32 0, i32 0
+  store i32 1, i32* %scratchptr0
+
+  %scratchptr1 = getelementptr [8192 x i32], [8192 x i32]* %scratch1, i32 0, i32 0
+  store i32 2, i32* %scratchptr1
+
+  %cmp = icmp eq i32 %cond, 0
+  br i1 %cmp, label %if, label %else
+
+if:
+  %if_ptr = getelementptr [8192 x i32], [8192 x i32]* %scratch0, i32 0, i32 %if_offset
+  %if_value = load i32, i32* %if_ptr
+  br label %done
+
+else:
+  %else_ptr = getelementptr [8192 x i32], [8192 x i32]* %scratch1, i32 0, i32 %else_offset
+  %else_value = load i32, i32* %else_ptr
+  br label %done
+
+done:
+  %value = phi i32 [%if_value, %if], [%else_value, %else]
+  store i32 %value, i32 addrspace(1)* %out
+  ret void
+
+  ret void
+
+}
+
+; GCN-LABEL: {{^}}legal_offset_fi_offset
+; GCN: buffer_store_dword v{{[0-9]+}}, v{{[0-9]+}}, s[{{[0-9]+}}:{{[0-9]+}}], s{{[0-9]+}} offen
+; GCN: v_add_i32_e32 [[OFFSET:v[0-9]+]], 0x8000
+; GCN: buffer_store_dword v{{[0-9]+}}, [[OFFSET]], s[{{[0-9]+}}:{{[0-9]+}}], s{{[0-9]+}} offen{{$}}
+
+define void @legal_offset_fi_offset(i32 addrspace(1)* %out, i32 %cond, i32 addrspace(1)* %offsets, i32 %if_offset, i32 %else_offset) {
+entry:
+  %scratch0 = alloca [8192 x i32]
+  %scratch1 = alloca [8192 x i32]
+
+  %offset0 = load i32, i32 addrspace(1)* %offsets
+  %scratchptr0 = getelementptr [8192 x i32], [8192 x i32]* %scratch0, i32 0, i32 %offset0
+  store i32 %offset0, i32* %scratchptr0
+
+  %offsetptr1 = getelementptr i32, i32 addrspace(1)* %offsets, i32 1
+  %offset1 = load i32, i32 addrspace(1)* %offsetptr1
+  %scratchptr1 = getelementptr [8192 x i32], [8192 x i32]* %scratch1, i32 0, i32 %offset1
+  store i32 %offset1, i32* %scratchptr1
+
+  %cmp = icmp eq i32 %cond, 0
+  br i1 %cmp, label %if, label %else
+
+if:
+  %if_ptr = getelementptr [8192 x i32], [8192 x i32]* %scratch0, i32 0, i32 %if_offset
+  %if_value = load i32, i32* %if_ptr
+  br label %done
+
+else:
+  %else_ptr = getelementptr [8192 x i32], [8192 x i32]* %scratch1, i32 0, i32 %else_offset
+  %else_value = load i32, i32* %else_ptr
+  br label %done
+
+done:
+  %value = phi i32 [%if_value, %if], [%else_value, %else]
+  store i32 %value, i32 addrspace(1)* %out
+  ret void
+}
+
+; GCN-LABEL: @neg_vaddr_offset
+; We can't prove %offset is positive, so we must do the computation with the
+; immediate in an add instruction instead of folding offset and the immediate into
+; the store instruction.
+; GCN: buffer_store_dword v{{[0-9]+}}, v{{[0-9]+}}, s[{{[0-9]+:[0-9]+}}], s{{[0-9]+}} offen{{$}}
+define void @neg_vaddr_offset(i32 %offset) {
+entry:
+  %array = alloca [8192 x i32]
+  %ptr_offset = add i32 %offset, 4
+  %ptr = getelementptr [8192 x i32], [8192 x i32]* %array, i32 0, i32 %ptr_offset
+  store i32 0, i32* %ptr
+  ret void
+}
+
+; GCN-LABEL: @pos_vaddr_offse
+; DEFAULT-SCRATCH: buffer_store_dword v{{[0-9]+}}, v{{[0-9]+}}, s[{{[0-9]+:[0-9]+}}], s{{[0-9]+}} offen offset:16
+; HUGE-SCRATCH: buffer_store_dword v{{[0-9]+}}, v{{[0-9]+}}, s[{{[0-9]+:[0-9]+}}], s{{[0-9]+}} offen{{$}}
+define void @pos_vaddr_offset(i32 addrspace(1)* %out, i32 %offset) {
+entry:
+  %array = alloca [8192 x i32]
+  %ptr = getelementptr [8192 x i32], [8192 x i32]* %array, i32 0, i32 4
+  store i32 0, i32* %ptr
+  %load_ptr = getelementptr [8192 x i32], [8192 x i32]* %array, i32 0, i32 %offset
+  %val = load i32, i32* %load_ptr
+  store i32 %val, i32 addrspace(1)* %out
+  ret void
+}
diff --git a/test/CodeGen/R600/sdiv.ll b/test/CodeGen/AMDGPU/sdiv.ll
similarity index 100%
rename from test/CodeGen/R600/sdiv.ll
rename to test/CodeGen/AMDGPU/sdiv.ll
diff --git a/test/CodeGen/R600/sdivrem24.ll b/test/CodeGen/AMDGPU/sdivrem24.ll
similarity index 100%
rename from test/CodeGen/R600/sdivrem24.ll
rename to test/CodeGen/AMDGPU/sdivrem24.ll
diff --git a/test/CodeGen/R600/sdivrem64.ll b/test/CodeGen/AMDGPU/sdivrem64.ll
similarity index 100%
rename from test/CodeGen/R600/sdivrem64.ll
rename to test/CodeGen/AMDGPU/sdivrem64.ll
diff --git a/test/CodeGen/R600/select-i1.ll b/test/CodeGen/AMDGPU/select-i1.ll
similarity index 100%
rename from test/CodeGen/R600/select-i1.ll
rename to test/CodeGen/AMDGPU/select-i1.ll
diff --git a/test/CodeGen/AMDGPU/select-vectors.ll b/test/CodeGen/AMDGPU/select-vectors.ll
new file mode 100644
index 00000000000..94758ad84c1
--- /dev/null
+++ b/test/CodeGen/AMDGPU/select-vectors.ll
@@ -0,0 +1,156 @@
+; RUN: llc -verify-machineinstrs -march=amdgcn -mcpu=SI < %s | FileCheck -check-prefix=SI -check-prefix=FUNC %s
+; RUN: llc -verify-machineinstrs -march=amdgcn -mcpu=tonga < %s | FileCheck -check-prefix=SI -check-prefix=FUNC %s
+
+; Test expansion of scalar selects on vectors.
+; Evergreen not enabled since it seems to be having problems with doubles.
+
+
+; FUNC-LABEL: {{^}}select_v4i8:
+; SI: v_cndmask_b32_e32
+; SI: v_cndmask_b32_e32
+; SI: v_cndmask_b32_e32
+; SI: v_cndmask_b32_e32
+define void @select_v4i8(<4 x i8> addrspace(1)* %out, <4 x i8> %a, <4 x i8> %b, i8 %c) nounwind {
+  %cmp = icmp eq i8 %c, 0
+  %select = select i1 %cmp, <4 x i8> %a, <4 x i8> %b
+  store <4 x i8> %select, <4 x i8> addrspace(1)* %out, align 4
+  ret void
+}
+
+; FUNC-LABEL: {{^}}select_v4i16:
+; SI: v_cndmask_b32_e32
+; SI: v_cndmask_b32_e32
+; SI: v_cndmask_b32_e32
+; SI: v_cndmask_b32_e32
+define void @select_v4i16(<4 x i16> addrspace(1)* %out, <4 x i16> %a, <4 x i16> %b, i32 %c) nounwind {
+  %cmp = icmp eq i32 %c, 0
+  %select = select i1 %cmp, <4 x i16> %a, <4 x i16> %b
+  store <4 x i16> %select, <4 x i16> addrspace(1)* %out, align 4
+  ret void
+}
+
+; FUNC-LABEL: {{^}}select_v2i32:
+; SI: v_cndmask_b32_e32
+; SI: v_cndmask_b32_e32
+; SI: buffer_store_dwordx2
+define void @select_v2i32(<2 x i32> addrspace(1)* %out, <2 x i32> %a, <2 x i32> %b, i32 %c) nounwind {
+  %cmp = icmp eq i32 %c, 0
+  %select = select i1 %cmp, <2 x i32> %a, <2 x i32> %b
+  store <2 x i32> %select, <2 x i32> addrspace(1)* %out, align 8
+  ret void
+}
+
+; FUNC-LABEL: {{^}}select_v4i32:
+; SI: v_cndmask_b32_e32
+; SI: v_cndmask_b32_e32
+; SI: v_cndmask_b32_e32
+; SI: v_cndmask_b32_e32
+; SI: buffer_store_dwordx4
+define void @select_v4i32(<4 x i32> addrspace(1)* %out, <4 x i32> %a, <4 x i32> %b, i32 %c) nounwind {
+  %cmp = icmp eq i32 %c, 0
+  %select = select i1 %cmp, <4 x i32> %a, <4 x i32> %b
+  store <4 x i32> %select, <4 x i32> addrspace(1)* %out, align 16
+  ret void
+}
+
+; FUNC-LABEL: {{^}}select_v8i32:
+; SI: v_cndmask_b32_e32
+; SI: v_cndmask_b32_e32
+; SI: v_cndmask_b32_e32
+; SI: v_cndmask_b32_e32
+; SI: v_cndmask_b32_e32
+; SI: v_cndmask_b32_e32
+; SI: v_cndmask_b32_e32
+; SI: v_cndmask_b32_e32
+define void @select_v8i32(<8 x i32> addrspace(1)* %out, <8 x i32> %a, <8 x i32> %b, i32 %c) nounwind {
+  %cmp = icmp eq i32 %c, 0
+  %select = select i1 %cmp, <8 x i32> %a, <8 x i32> %b
+  store <8 x i32> %select, <8 x i32> addrspace(1)* %out, align 16
+  ret void
+}
+
+; FUNC-LABEL: {{^}}select_v2f32:
+; SI: buffer_store_dwordx2
+define void @select_v2f32(<2 x float> addrspace(1)* %out, <2 x float> %a, <2 x float> %b, i32 %c) nounwind {
+  %cmp = icmp eq i32 %c, 0
+  %select = select i1 %cmp, <2 x float> %a, <2 x float> %b
+  store <2 x float> %select, <2 x float> addrspace(1)* %out, align 16
+  ret void
+}
+
+; FUNC-LABEL: {{^}}select_v4f32:
+; SI: buffer_store_dwordx4
+define void @select_v4f32(<4 x float> addrspace(1)* %out, <4 x float> %a, <4 x float> %b, i32 %c) nounwind {
+  %cmp = icmp eq i32 %c, 0
+  %select = select i1 %cmp, <4 x float> %a, <4 x float> %b
+  store <4 x float> %select, <4 x float> addrspace(1)* %out, align 16
+  ret void
+}
+
+; FUNC-LABEL: {{^}}select_v8f32:
+; SI: v_cndmask_b32_e32
+; SI: v_cndmask_b32_e32
+; SI: v_cndmask_b32_e32
+; SI: v_cndmask_b32_e32
+; SI: v_cndmask_b32_e32
+; SI: v_cndmask_b32_e32
+; SI: v_cndmask_b32_e32
+; SI: v_cndmask_b32_e32
+define void @select_v8f32(<8 x float> addrspace(1)* %out, <8 x float> %a, <8 x float> %b, i32 %c) nounwind {
+  %cmp = icmp eq i32 %c, 0
+  %select = select i1 %cmp, <8 x float> %a, <8 x float> %b
+  store <8 x float> %select, <8 x float> addrspace(1)* %out, align 16
+  ret void
+}
+
+; FUNC-LABEL: {{^}}select_v2f64:
+; SI: v_cndmask_b32_e32
+; SI: v_cndmask_b32_e32
+; SI: v_cndmask_b32_e32
+; SI: v_cndmask_b32_e32
+define void @select_v2f64(<2 x double> addrspace(1)* %out, <2 x double> %a, <2 x double> %b, i32 %c) nounwind {
+  %cmp = icmp eq i32 %c, 0
+  %select = select i1 %cmp, <2 x double> %a, <2 x double> %b
+  store <2 x double> %select, <2 x double> addrspace(1)* %out, align 16
+  ret void
+}
+
+; FUNC-LABEL: {{^}}select_v4f64:
+; SI: v_cndmask_b32_e32
+; SI: v_cndmask_b32_e32
+; SI: v_cndmask_b32_e32
+; SI: v_cndmask_b32_e32
+; SI: v_cndmask_b32_e32
+; SI: v_cndmask_b32_e32
+; SI: v_cndmask_b32_e32
+; SI: v_cndmask_b32_e32
+define void @select_v4f64(<4 x double> addrspace(1)* %out, <4 x double> %a, <4 x double> %b, i32 %c) nounwind {
+  %cmp = icmp eq i32 %c, 0
+  %select = select i1 %cmp, <4 x double> %a, <4 x double> %b
+  store <4 x double> %select, <4 x double> addrspace(1)* %out, align 16
+  ret void
+}
+
+; FUNC-LABEL: {{^}}select_v8f64:
+; SI: v_cndmask_b32_e32
+; SI: v_cndmask_b32_e32
+; SI: v_cndmask_b32_e32
+; SI: v_cndmask_b32_e32
+; SI: v_cndmask_b32_e32
+; SI: v_cndmask_b32_e32
+; SI: v_cndmask_b32_e32
+; SI: v_cndmask_b32_e32
+; SI: v_cndmask_b32_e32
+; SI: v_cndmask_b32_e32
+; SI: v_cndmask_b32_e32
+; SI: v_cndmask_b32_e32
+; SI: v_cndmask_b32_e32
+; SI: v_cndmask_b32_e32
+; SI: v_cndmask_b32_e32
+; SI: v_cndmask_b32_e32
+define void @select_v8f64(<8 x double> addrspace(1)* %out, <8 x double> %a, <8 x double> %b, i32 %c) nounwind {
+  %cmp = icmp eq i32 %c, 0
+  %select = select i1 %cmp, <8 x double> %a, <8 x double> %b
+  store <8 x double> %select, <8 x double> addrspace(1)* %out, align 16
+  ret void
+}
diff --git a/test/CodeGen/R600/select.ll b/test/CodeGen/AMDGPU/select.ll
similarity index 100%
rename from test/CodeGen/R600/select.ll
rename to test/CodeGen/AMDGPU/select.ll
diff --git a/test/CodeGen/R600/select64.ll b/test/CodeGen/AMDGPU/select64.ll
similarity index 94%
rename from test/CodeGen/R600/select64.ll
rename to test/CodeGen/AMDGPU/select64.ll
index 5cebb30dc72..13fb575b2b1 100644
--- a/test/CodeGen/R600/select64.ll
+++ b/test/CodeGen/AMDGPU/select64.ll
@@ -55,8 +55,8 @@ define void @v_select_trunc_i64_2(i32 addrspace(1)* %out, i32 %cond, i64 addrspa
 ; CHECK: s_mov_b32 [[SLO:s[0-9]+]], 0
 ; CHECK-DAG: v_mov_b32_e32 [[VHI:v[0-9]+]], [[SHI]]
 ; CHECK-DAG: v_mov_b32_e32 [[VLO:v[0-9]+]], [[SLO]]
-; CHECK-DAG: v_cndmask_b32_e64 {{v[0-9]+}}, [[VLO]], {{v[0-9]+}}
-; CHECK-DAG: v_cndmask_b32_e64 {{v[0-9]+}}, [[VHI]], {{v[0-9]+}}
+; CHECK-DAG: v_cndmask_b32_e32 {{v[0-9]+}}, [[VLO]], {{v[0-9]+}}
+; CHECK-DAG: v_cndmask_b32_e32 {{v[0-9]+}}, [[VHI]], {{v[0-9]+}}
 ; CHECK: s_endpgm
 define void @v_select_i64_split_imm(i64 addrspace(1)* %out, i32 %cond, i64 addrspace(1)* %aptr, i64 addrspace(1)* %bptr) nounwind {
   %cmp = icmp ugt i32 %cond, 5
diff --git a/test/CodeGen/R600/selectcc-cnd.ll b/test/CodeGen/AMDGPU/selectcc-cnd.ll
similarity index 100%
rename from test/CodeGen/R600/selectcc-cnd.ll
rename to test/CodeGen/AMDGPU/selectcc-cnd.ll
diff --git a/test/CodeGen/R600/selectcc-cnde-int.ll b/test/CodeGen/AMDGPU/selectcc-cnde-int.ll
similarity index 100%
rename from test/CodeGen/R600/selectcc-cnde-int.ll
rename to test/CodeGen/AMDGPU/selectcc-cnde-int.ll
diff --git a/test/CodeGen/R600/selectcc-icmp-select-float.ll b/test/CodeGen/AMDGPU/selectcc-icmp-select-float.ll
similarity index 100%
rename from test/CodeGen/R600/selectcc-icmp-select-float.ll
rename to test/CodeGen/AMDGPU/selectcc-icmp-select-float.ll
diff --git a/test/CodeGen/R600/selectcc-opt.ll b/test/CodeGen/AMDGPU/selectcc-opt.ll
similarity index 100%
rename from test/CodeGen/R600/selectcc-opt.ll
rename to test/CodeGen/AMDGPU/selectcc-opt.ll
diff --git a/test/CodeGen/R600/selectcc.ll b/test/CodeGen/AMDGPU/selectcc.ll
similarity index 100%
rename from test/CodeGen/R600/selectcc.ll
rename to test/CodeGen/AMDGPU/selectcc.ll
diff --git a/test/CodeGen/R600/set-dx10.ll b/test/CodeGen/AMDGPU/set-dx10.ll
similarity index 100%
rename from test/CodeGen/R600/set-dx10.ll
rename to test/CodeGen/AMDGPU/set-dx10.ll
diff --git a/test/CodeGen/R600/setcc-equivalent.ll b/test/CodeGen/AMDGPU/setcc-equivalent.ll
similarity index 100%
rename from test/CodeGen/R600/setcc-equivalent.ll
rename to test/CodeGen/AMDGPU/setcc-equivalent.ll
diff --git a/test/CodeGen/R600/setcc-opt.ll b/test/CodeGen/AMDGPU/setcc-opt.ll
similarity index 100%
rename from test/CodeGen/R600/setcc-opt.ll
rename to test/CodeGen/AMDGPU/setcc-opt.ll
diff --git a/test/CodeGen/R600/setcc.ll b/test/CodeGen/AMDGPU/setcc.ll
similarity index 100%
rename from test/CodeGen/R600/setcc.ll
rename to test/CodeGen/AMDGPU/setcc.ll
diff --git a/test/CodeGen/R600/setcc64.ll b/test/CodeGen/AMDGPU/setcc64.ll
similarity index 100%
rename from test/CodeGen/R600/setcc64.ll
rename to test/CodeGen/AMDGPU/setcc64.ll
diff --git a/test/CodeGen/R600/seto.ll b/test/CodeGen/AMDGPU/seto.ll
similarity index 100%
rename from test/CodeGen/R600/seto.ll
rename to test/CodeGen/AMDGPU/seto.ll
diff --git a/test/CodeGen/R600/setuo.ll b/test/CodeGen/AMDGPU/setuo.ll
similarity index 100%
rename from test/CodeGen/R600/setuo.ll
rename to test/CodeGen/AMDGPU/setuo.ll
diff --git a/test/CodeGen/R600/sext-eliminate.ll b/test/CodeGen/AMDGPU/sext-eliminate.ll
similarity index 100%
rename from test/CodeGen/R600/sext-eliminate.ll
rename to test/CodeGen/AMDGPU/sext-eliminate.ll
diff --git a/test/CodeGen/R600/sext-in-reg.ll b/test/CodeGen/AMDGPU/sext-in-reg.ll
similarity index 99%
rename from test/CodeGen/R600/sext-in-reg.ll
rename to test/CodeGen/AMDGPU/sext-in-reg.ll
index d9ad4935968..5aedda2ce1a 100644
--- a/test/CodeGen/R600/sext-in-reg.ll
+++ b/test/CodeGen/AMDGPU/sext-in-reg.ll
@@ -450,13 +450,10 @@ define void @vgpr_sext_in_reg_v4i16_to_v4i32(<4 x i32> addrspace(1)* %out, <4 x
   ret void
 }
 
-; FIXME: The BFE should really be eliminated. I think it should happen
-; when computeKnownBitsForTargetNode is implemented for imax.
-
 ; FUNC-LABEL: {{^}}sext_in_reg_to_illegal_type:
 ; SI: buffer_load_sbyte
 ; SI: v_max_i32
-; SI: v_bfe_i32
+; SI-NOT: bfe
 ; SI: buffer_store_short
 define void @sext_in_reg_to_illegal_type(i16 addrspace(1)* nocapture %out, i8 addrspace(1)* nocapture %src) nounwind {
   %tmp5 = load i8, i8 addrspace(1)* %src, align 1
diff --git a/test/CodeGen/R600/sgpr-control-flow.ll b/test/CodeGen/AMDGPU/sgpr-control-flow.ll
similarity index 100%
rename from test/CodeGen/R600/sgpr-control-flow.ll
rename to test/CodeGen/AMDGPU/sgpr-control-flow.ll
diff --git a/test/CodeGen/R600/sgpr-copy-duplicate-operand.ll b/test/CodeGen/AMDGPU/sgpr-copy-duplicate-operand.ll
similarity index 100%
rename from test/CodeGen/R600/sgpr-copy-duplicate-operand.ll
rename to test/CodeGen/AMDGPU/sgpr-copy-duplicate-operand.ll
diff --git a/test/CodeGen/R600/sgpr-copy.ll b/test/CodeGen/AMDGPU/sgpr-copy.ll
similarity index 100%
rename from test/CodeGen/R600/sgpr-copy.ll
rename to test/CodeGen/AMDGPU/sgpr-copy.ll
diff --git a/test/CodeGen/R600/shared-op-cycle.ll b/test/CodeGen/AMDGPU/shared-op-cycle.ll
similarity index 100%
rename from test/CodeGen/R600/shared-op-cycle.ll
rename to test/CodeGen/AMDGPU/shared-op-cycle.ll
diff --git a/test/CodeGen/AMDGPU/shl.ll b/test/CodeGen/AMDGPU/shl.ll
new file mode 100644
index 00000000000..6f81a39ed96
--- /dev/null
+++ b/test/CodeGen/AMDGPU/shl.ll
@@ -0,0 +1,212 @@
+; RUN: llc < %s -march=r600 -mcpu=redwood | FileCheck --check-prefix=EG %s
+; RUN: llc < %s -march=amdgcn -mcpu=verde -verify-machineinstrs | FileCheck -check-prefix=GCN -check-prefix=SI %s
+; XUN: llc < %s -march=amdgcn -mcpu=tonga -verify-machineinstrs | FileCheck -check-prefix=GCN -check-prefix=VI %s
+
+declare i32 @llvm.r600.read.tidig.x() #0
+
+
+;EG: {{^}}shl_v2i32:
+;EG: LSHL {{\*? *}}T{{[0-9]+\.[XYZW], T[0-9]+\.[XYZW], T[0-9]+\.[XYZW]}}
+;EG: LSHL {{\*? *}}T{{[0-9]+\.[XYZW], T[0-9]+\.[XYZW], T[0-9]+\.[XYZW]}}
+
+;SI: {{^}}shl_v2i32:
+;SI: v_lshl_b32_e32 v{{[0-9]+, v[0-9]+, v[0-9]+}}
+;SI: v_lshl_b32_e32 v{{[0-9]+, v[0-9]+, v[0-9]+}}
+
+;VI: {{^}}shl_v2i32:
+;VI: v_lshlrev_b32_e32 v{{[0-9]+, v[0-9]+, v[0-9]+}}
+;VI: v_lshlrev_b32_e32 v{{[0-9]+, v[0-9]+, v[0-9]+}}
+
+define void @shl_v2i32(<2 x i32> addrspace(1)* %out, <2 x i32> addrspace(1)* %in) {
+  %b_ptr = getelementptr <2 x i32>, <2 x i32> addrspace(1)* %in, i32 1
+  %a = load <2 x i32>, <2 x i32> addrspace(1) * %in
+  %b = load <2 x i32>, <2 x i32> addrspace(1) * %b_ptr
+  %result = shl <2 x i32> %a, %b
+  store <2 x i32> %result, <2 x i32> addrspace(1)* %out
+  ret void
+}
+
+;EG: {{^}}shl_v4i32:
+;EG: LSHL {{\*? *}}T{{[0-9]+\.[XYZW], T[0-9]+\.[XYZW], T[0-9]+\.[XYZW]}}
+;EG: LSHL {{\*? *}}T{{[0-9]+\.[XYZW], T[0-9]+\.[XYZW], T[0-9]+\.[XYZW]}}
+;EG: LSHL {{\*? *}}T{{[0-9]+\.[XYZW], T[0-9]+\.[XYZW], T[0-9]+\.[XYZW]}}
+;EG: LSHL {{\*? *}}T{{[0-9]+\.[XYZW], T[0-9]+\.[XYZW], T[0-9]+\.[XYZW]}}
+
+;SI: {{^}}shl_v4i32:
+;SI: v_lshl_b32_e32 v{{[0-9]+, v[0-9]+, v[0-9]+}}
+;SI: v_lshl_b32_e32 v{{[0-9]+, v[0-9]+, v[0-9]+}}
+;SI: v_lshl_b32_e32 v{{[0-9]+, v[0-9]+, v[0-9]+}}
+;SI: v_lshl_b32_e32 v{{[0-9]+, v[0-9]+, v[0-9]+}}
+
+;VI: {{^}}shl_v4i32:
+;VI: v_lshlrev_b32_e32 v{{[0-9]+, v[0-9]+, v[0-9]+}}
+;VI: v_lshlrev_b32_e32 v{{[0-9]+, v[0-9]+, v[0-9]+}}
+;VI: v_lshlrev_b32_e32 v{{[0-9]+, v[0-9]+, v[0-9]+}}
+;VI: v_lshlrev_b32_e32 v{{[0-9]+, v[0-9]+, v[0-9]+}}
+
+define void @shl_v4i32(<4 x i32> addrspace(1)* %out, <4 x i32> addrspace(1)* %in) {
+  %b_ptr = getelementptr <4 x i32>, <4 x i32> addrspace(1)* %in, i32 1
+  %a = load <4 x i32>, <4 x i32> addrspace(1) * %in
+  %b = load <4 x i32>, <4 x i32> addrspace(1) * %b_ptr
+  %result = shl <4 x i32> %a, %b
+  store <4 x i32> %result, <4 x i32> addrspace(1)* %out
+  ret void
+}
+
+;EG: {{^}}shl_i64:
+;EG: SUB_INT {{\*? *}}[[COMPSH:T[0-9]+\.[XYZW]]], {{literal.[xy]}}, [[SHIFT:T[0-9]+\.[XYZW]]]
+;EG: LSHR {{\* *}}[[TEMP:T[0-9]+\.[XYZW]]], [[OPLO:T[0-9]+\.[XYZW]]], {{[[COMPSH]]|PV.[XYZW]}}
+;EG: LSHR {{\*? *}}[[OVERF:T[0-9]+\.[XYZW]]], {{[[TEMP]]|PV.[XYZW]}}, 1
+;EG_CHECK-DAG: ADD_INT {{\*? *}}[[BIGSH:T[0-9]+\.[XYZW]]], [[SHIFT]], literal
+;EG-DAG: LSHL {{\*? *}}[[HISMTMP:T[0-9]+\.[XYZW]]], [[OPHI:T[0-9]+\.[XYZW]]], [[SHIFT]]
+;EG-DAG: OR_INT {{\*? *}}[[HISM:T[0-9]+\.[XYZW]]], {{[[HISMTMP]]|PV.[XYZW]}}, {{[[OVERF]]|PV.[XYZW]}}
+;EG-DAG: LSHL {{\*? *}}[[LOSM:T[0-9]+\.[XYZW]]], [[OPLO]], {{PS|[[SHIFT]]}}
+;EG-DAG: SETGT_UINT {{\*? *}}[[RESC:T[0-9]+\.[XYZW]]], [[SHIFT]], literal
+;EG-DAG: CNDE_INT {{\*? *}}[[RESLO:T[0-9]+\.[XYZW]]], {{T[0-9]+\.[XYZW]}}
+;EG-DAG: CNDE_INT {{\*? *}}[[RESHI:T[0-9]+\.[XYZW]]], {{T[0-9]+\.[XYZW], .*}}, 0.0
+
+;SI: {{^}}shl_i64:
+;SI: v_lshl_b64 {{v\[[0-9]+:[0-9]+\], v\[[0-9]+:[0-9]+\], v[0-9]+}}
+
+;VI: {{^}}shl_i64:
+;VI: v_lshlrev_b64 {{v\[[0-9]+:[0-9]+\], v[0-9]+, v\[[0-9]+:[0-9]+\]}}
+
+define void @shl_i64(i64 addrspace(1)* %out, i64 addrspace(1)* %in) {
+  %b_ptr = getelementptr i64, i64 addrspace(1)* %in, i64 1
+  %a = load i64, i64 addrspace(1) * %in
+  %b = load i64, i64 addrspace(1) * %b_ptr
+  %result = shl i64 %a, %b
+  store i64 %result, i64 addrspace(1)* %out
+  ret void
+}
+
+;EG: {{^}}shl_v2i64:
+;EG-DAG: SUB_INT {{\*? *}}[[COMPSHA:T[0-9]+\.[XYZW]]], {{literal.[xy]}}, [[SHA:T[0-9]+\.[XYZW]]]
+;EG-DAG: SUB_INT {{\*? *}}[[COMPSHB:T[0-9]+\.[XYZW]]], {{literal.[xy]}}, [[SHB:T[0-9]+\.[XYZW]]]
+;EG-DAG: LSHR {{\*? *}}[[COMPSHA]]
+;EG-DAG: LSHR {{\*? *}}[[COMPSHB]]
+;EG-DAG: LSHR {{.*}}, 1
+;EG-DAG: LSHR {{.*}}, 1
+;EG-DAG: ADD_INT  {{\*? *}}[[BIGSHA:T[0-9]+\.[XYZW]]]{{.*}}, literal
+;EG-DAG: ADD_INT  {{\*? *}}[[BIGSHB:T[0-9]+\.[XYZW]]]{{.*}}, literal
+;EG-DAG: LSHL {{.*}}, [[SHA]]
+;EG-DAG: LSHL {{.*}}, [[SHB]]
+;EG-DAG: LSHL {{.*}}, [[SHA]]
+;EG-DAG: LSHL {{.*}}, [[SHB]]
+;EG-DAG: LSHL
+;EG-DAG: LSHL
+;EG-DAG: SETGT_UINT {{\*? *T[0-9]\.[XYZW]}}, [[SHA]], literal
+;EG-DAG: SETGT_UINT {{\*? *T[0-9]\.[XYZW]}}, [[SHB]], literal
+;EG-DAG: CNDE_INT {{.*}}, 0.0
+;EG-DAG: CNDE_INT {{.*}}, 0.0
+;EG-DAG: CNDE_INT
+;EG-DAG: CNDE_INT
+
+;SI: {{^}}shl_v2i64:
+;SI: v_lshl_b64 {{v\[[0-9]+:[0-9]+\], v\[[0-9]+:[0-9]+\], v[0-9]+}}
+;SI: v_lshl_b64 {{v\[[0-9]+:[0-9]+\], v\[[0-9]+:[0-9]+\], v[0-9]+}}
+
+;VI: {{^}}shl_v2i64:
+;VI: v_lshlrev_b64 {{v\[[0-9]+:[0-9]+\], v[0-9]+, v\[[0-9]+:[0-9]+\]}}
+;VI: v_lshlrev_b64 {{v\[[0-9]+:[0-9]+\], v[0-9]+, v\[[0-9]+:[0-9]+\]}}
+
+define void @shl_v2i64(<2 x i64> addrspace(1)* %out, <2 x i64> addrspace(1)* %in) {
+  %b_ptr = getelementptr <2 x i64>, <2 x i64> addrspace(1)* %in, i64 1
+  %a = load <2 x i64>, <2 x i64> addrspace(1) * %in
+  %b = load <2 x i64>, <2 x i64> addrspace(1) * %b_ptr
+  %result = shl <2 x i64> %a, %b
+  store <2 x i64> %result, <2 x i64> addrspace(1)* %out
+  ret void
+}
+
+;EG: {{^}}shl_v4i64:
+;EG-DAG: SUB_INT {{\*? *}}[[COMPSHA:T[0-9]+\.[XYZW]]], {{literal.[xy]}}, [[SHA:T[0-9]+\.[XYZW]]]
+;EG-DAG: SUB_INT {{\*? *}}[[COMPSHB:T[0-9]+\.[XYZW]]], {{literal.[xy]}}, [[SHB:T[0-9]+\.[XYZW]]]
+;EG-DAG: SUB_INT {{\*? *}}[[COMPSHC:T[0-9]+\.[XYZW]]], {{literal.[xy]}}, [[SHC:T[0-9]+\.[XYZW]]]
+;EG-DAG: SUB_INT {{\*? *}}[[COMPSHD:T[0-9]+\.[XYZW]]], {{literal.[xy]}}, [[SHD:T[0-9]+\.[XYZW]]]
+;EG-DAG: LSHR {{\*? *}}[[COMPSHA]]
+;EG-DAG: LSHR {{\*? *}}[[COMPSHB]]
+;EG-DAG: LSHR {{\*? *}}[[COMPSHC]]
+;EG-DAG: LSHR {{\*? *}}[[COMPSHD]]
+;EG-DAG: LSHR {{.*}}, 1
+;EG-DAG: LSHR {{.*}}, 1
+;EG-DAG: LSHR {{.*}}, 1
+;EG-DAG: LSHR {{.*}}, 1
+;EG-DAG: ADD_INT  {{\*? *}}[[BIGSHA:T[0-9]+\.[XYZW]]]{{.*}}, literal
+;EG-DAG: ADD_INT  {{\*? *}}[[BIGSHB:T[0-9]+\.[XYZW]]]{{.*}}, literal
+;EG-DAG: ADD_INT  {{\*? *}}[[BIGSHC:T[0-9]+\.[XYZW]]]{{.*}}, literal
+;EG-DAG: ADD_INT  {{\*? *}}[[BIGSHD:T[0-9]+\.[XYZW]]]{{.*}}, literal
+;EG-DAG: LSHL {{.*}}, [[SHA]]
+;EG-DAG: LSHL {{.*}}, [[SHB]]
+;EG-DAG: LSHL {{.*}}, [[SHC]]
+;EG-DAG: LSHL {{.*}}, [[SHD]]
+;EG-DAG: LSHL {{.*}}, [[SHA]]
+;EG-DAG: LSHL {{.*}}, [[SHB]]
+;EG-DAG: LSHL {{.*}}, [[SHC]]
+;EG-DAG: LSHL {{.*}}, [[SHD]]
+;EG-DAG: LSHL
+;EG-DAG: LSHL
+;EG-DAG: LSHL
+;EG-DAG: LSHL
+;EG-DAG: SETGT_UINT {{\*? *T[0-9]\.[XYZW]}}, [[SHA]], literal
+;EG-DAG: SETGT_UINT {{\*? *T[0-9]\.[XYZW]}}, [[SHB]], literal
+;EG-DAG: SETGT_UINT {{\*? *T[0-9]\.[XYZW]}}, [[SHC]], literal
+;EG-DAG: SETGT_UINT {{\*? *T[0-9]\.[XYZW]}}, [[SHD]], literal
+;EG-DAG: CNDE_INT {{.*}}, 0.0
+;EG-DAG: CNDE_INT {{.*}}, 0.0
+;EG-DAG: CNDE_INT {{.*}}, 0.0
+;EG-DAG: CNDE_INT {{.*}}, 0.0
+;EG-DAG: CNDE_INT
+;EG-DAG: CNDE_INT
+;EG-DAG: CNDE_INT
+;EG-DAG: CNDE_INT
+
+;SI: {{^}}shl_v4i64:
+;SI: v_lshl_b64 {{v\[[0-9]+:[0-9]+\], v\[[0-9]+:[0-9]+\], v[0-9]+}}
+;SI: v_lshl_b64 {{v\[[0-9]+:[0-9]+\], v\[[0-9]+:[0-9]+\], v[0-9]+}}
+;SI: v_lshl_b64 {{v\[[0-9]+:[0-9]+\], v\[[0-9]+:[0-9]+\], v[0-9]+}}
+;SI: v_lshl_b64 {{v\[[0-9]+:[0-9]+\], v\[[0-9]+:[0-9]+\], v[0-9]+}}
+
+;VI: {{^}}shl_v4i64:
+;VI: v_lshlrev_b64 {{v\[[0-9]+:[0-9]+\], v[0-9]+, v\[[0-9]+:[0-9]+\]}}
+;VI: v_lshlrev_b64 {{v\[[0-9]+:[0-9]+\], v[0-9]+, v\[[0-9]+:[0-9]+\]}}
+;VI: v_lshlrev_b64 {{v\[[0-9]+:[0-9]+\], v[0-9]+, v\[[0-9]+:[0-9]+\]}}
+;VI: v_lshlrev_b64 {{v\[[0-9]+:[0-9]+\], v[0-9]+, v\[[0-9]+:[0-9]+\]}}
+
+define void @shl_v4i64(<4 x i64> addrspace(1)* %out, <4 x i64> addrspace(1)* %in) {
+  %b_ptr = getelementptr <4 x i64>, <4 x i64> addrspace(1)* %in, i64 1
+  %a = load <4 x i64>, <4 x i64> addrspace(1) * %in
+  %b = load <4 x i64>, <4 x i64> addrspace(1) * %b_ptr
+  %result = shl <4 x i64> %a, %b
+  store <4 x i64> %result, <4 x i64> addrspace(1)* %out
+  ret void
+}
+
+; Make sure load width gets reduced to i32 load.
+; GCN-LABEL: {{^}}s_shl_32_i64:
+; GCN-DAG: s_load_dword [[LO_A:s[0-9]+]], s{{\[[0-9]+:[0-9]+\]}}, 0xb{{$}}
+; GCN-DAG: s_mov_b32 s[[SLO:[0-9]+]], 0{{$}}
+; GCN-DAG: v_mov_b32_e32 v[[VLO:[0-9]+]], s[[SLO]]
+; GCN-DAG: v_mov_b32_e32 v[[VHI:[0-9]+]], [[LO_A]]
+; GCN: buffer_store_dwordx2 v{{\[}}[[VLO]]:[[VHI]]{{\]}}
+define void @s_shl_32_i64(i64 addrspace(1)* %out, i64 %a) {
+  %result = shl i64 %a, 32
+  store i64 %result, i64 addrspace(1)* %out
+  ret void
+}
+
+; GCN-LABEL: {{^}}v_shl_32_i64:
+; GCN-DAG: buffer_load_dword v[[LO_A:[0-9]+]],
+; GCN-DAG: v_mov_b32_e32 v[[VLO:[0-9]+]], 0{{$}}
+; GCN: buffer_store_dwordx2 v{{\[}}[[VLO]]:[[LO_A]]{{\]}}
+define void @v_shl_32_i64(i64 addrspace(1)* %out, i64 addrspace(1)* %in) {
+  %tid = call i32 @llvm.r600.read.tidig.x() #0
+  %gep.in = getelementptr i64, i64 addrspace(1)* %in, i32 %tid
+  %gep.out = getelementptr i64, i64 addrspace(1)* %out, i32 %tid
+  %a = load i64, i64 addrspace(1)* %gep.in
+  %result = shl i64 %a, 32
+  store i64 %result, i64 addrspace(1)* %gep.out
+  ret void
+}
+
+attributes #0 = { nounwind readnone }
diff --git a/test/CodeGen/R600/shl_add_constant.ll b/test/CodeGen/AMDGPU/shl_add_constant.ll
similarity index 100%
rename from test/CodeGen/R600/shl_add_constant.ll
rename to test/CodeGen/AMDGPU/shl_add_constant.ll
diff --git a/test/CodeGen/R600/shl_add_ptr.ll b/test/CodeGen/AMDGPU/shl_add_ptr.ll
similarity index 99%
rename from test/CodeGen/R600/shl_add_ptr.ll
rename to test/CodeGen/AMDGPU/shl_add_ptr.ll
index 066dafb6e8c..6671e909cd1 100644
--- a/test/CodeGen/R600/shl_add_ptr.ll
+++ b/test/CodeGen/AMDGPU/shl_add_ptr.ll
@@ -69,8 +69,8 @@ define void @load_shl_base_lds_max_offset(i8 addrspace(1)* %out, i8 addrspace(3)
 ; pointer can be used with an offset into the second one.
 
 ; SI-LABEL: {{^}}load_shl_base_lds_2:
+; SI: v_lshlrev_b32_e32 [[PTR:v[0-9]+]], 2, {{v[0-9]+}}
 ; SI: s_mov_b32 m0, -1
-; SI-NEXT: v_lshlrev_b32_e32 [[PTR:v[0-9]+]], 2, {{v[0-9]+}}
 ; SI-NEXT: ds_read2st64_b32 {{v\[[0-9]+:[0-9]+\]}}, [[PTR]] offset0:1 offset1:9
 ; SI: s_endpgm
 define void @load_shl_base_lds_2(float addrspace(1)* %out) #0 {
diff --git a/test/CodeGen/R600/si-annotate-cf-assertion.ll b/test/CodeGen/AMDGPU/si-annotate-cf-assertion.ll
similarity index 100%
rename from test/CodeGen/R600/si-annotate-cf-assertion.ll
rename to test/CodeGen/AMDGPU/si-annotate-cf-assertion.ll
diff --git a/test/CodeGen/AMDGPU/si-annotate-cf.ll b/test/CodeGen/AMDGPU/si-annotate-cf.ll
new file mode 100644
index 00000000000..bbcb861f37d
--- /dev/null
+++ b/test/CodeGen/AMDGPU/si-annotate-cf.ll
@@ -0,0 +1,63 @@
+; RUN: llc < %s -march=amdgcn -mcpu=verde -verify-machineinstrs | FileCheck --check-prefix=SI --check-prefix=FUNC %s
+; RUN: llc < %s -march=amdgcn -mcpu=tonga -verify-machineinstrs | FileCheck --check-prefix=SI --check-prefix=FUNC %s
+
+; FUNC-LABEL: {{^}}break_inserted_outside_of_loop:
+
+; SI: [[LOOP_LABEL:[A-Z0-9]+]]:
+; Lowered break instructin:
+; SI: s_or_b64
+; Lowered Loop instruction:
+; SI: s_andn2_b64
+; s_cbranch_execnz [[LOOP_LABEL]]
+; SI: s_endpgm
+define void @break_inserted_outside_of_loop(i32 addrspace(1)* %out, i32 %a, i32 %b) {
+main_body:
+  %0 = and i32 %a, %b
+  %1 = trunc i32 %0 to i1
+  br label %ENDIF
+
+ENDLOOP:
+  store i32 0, i32 addrspace(1)* %out
+  ret void
+
+ENDIF:
+  br i1 %1, label %ENDLOOP, label %ENDIF
+}
+
+
+; FUNC-LABEL: {{^}}phi_cond_outside_loop:
+; FIXME: This could be folded into the s_or_b64 instruction
+; SI: s_mov_b64 [[ZERO:s\[[0-9]+:[0-9]+\]]], 0
+; SI: [[LOOP_LABEL:[A-Z0-9]+]]
+; SI: v_cmp_ne_i32_e32 vcc, 0, v{{[0-9]+}}
+
+; SI_IF_BREAK instruction:
+; SI: s_or_b64 [[BREAK:s\[[0-9]+:[0-9]+\]]], vcc, [[ZERO]]
+
+; SI_LOOP instruction:
+; SI: s_andn2_b64 exec, exec, [[BREAK]]
+; SI: s_cbranch_execnz [[LOOP_LABEL]]
+; SI: s_endpgm
+
+define void @phi_cond_outside_loop(i32 %a, i32 %b) {
+entry:
+  %0 = icmp eq i32 %a , 0
+  br i1 %0, label %if, label %else
+
+if:
+  br label %endif
+
+else:
+  %1 = icmp eq i32 %b, 0
+  br label %endif
+
+endif:
+  %2 = phi i1 [0, %if], [%1, %else]
+  br label %loop
+
+loop:
+  br i1 %2, label %exit, label %loop
+
+exit:
+  ret void
+}
diff --git a/test/CodeGen/R600/si-lod-bias.ll b/test/CodeGen/AMDGPU/si-lod-bias.ll
similarity index 100%
rename from test/CodeGen/R600/si-lod-bias.ll
rename to test/CodeGen/AMDGPU/si-lod-bias.ll
diff --git a/test/CodeGen/R600/si-sgpr-spill.ll b/test/CodeGen/AMDGPU/si-sgpr-spill.ll
similarity index 100%
rename from test/CodeGen/R600/si-sgpr-spill.ll
rename to test/CodeGen/AMDGPU/si-sgpr-spill.ll
diff --git a/test/CodeGen/AMDGPU/si-spill-cf.ll b/test/CodeGen/AMDGPU/si-spill-cf.ll
new file mode 100644
index 00000000000..4b2d8ec6bf0
--- /dev/null
+++ b/test/CodeGen/AMDGPU/si-spill-cf.ll
@@ -0,0 +1,501 @@
+; RUN: llc -march=amdgcn -mcpu=SI < %s -verify-machineinstrs | FileCheck -check-prefix=SI %s
+; RUN: llc -march=amdgcn -mcpu=tonga < %s -verify-machineinstrs | FileCheck -check-prefix=SI %s
+
+; If this occurs it is likely due to reordering and the restore was
+; originally supposed to happen before SI_END_CF.
+; SI: s_or_b64 exec, exec, [[SAVED:s\[[0-9]+:[0-9]+\]|[a-z]+]]
+; SI-NOT: v_readlane_b32 [[SAVED]]
+
+define void @main() #0 {
+main_body:
+  %0 = call float @llvm.SI.load.const(<16 x i8> undef, i32 16)
+  %1 = call float @llvm.SI.load.const(<16 x i8> undef, i32 32)
+  %2 = call float @llvm.SI.load.const(<16 x i8> undef, i32 80)
+  %3 = call float @llvm.SI.load.const(<16 x i8> undef, i32 84)
+  %4 = call float @llvm.SI.load.const(<16 x i8> undef, i32 88)
+  %5 = call float @llvm.SI.load.const(<16 x i8> undef, i32 96)
+  %6 = call float @llvm.SI.load.const(<16 x i8> undef, i32 100)
+  %7 = call float @llvm.SI.load.const(<16 x i8> undef, i32 104)
+  %8 = call float @llvm.SI.load.const(<16 x i8> undef, i32 112)
+  %9 = call float @llvm.SI.load.const(<16 x i8> undef, i32 116)
+  %10 = call float @llvm.SI.load.const(<16 x i8> undef, i32 120)
+  %11 = call float @llvm.SI.load.const(<16 x i8> undef, i32 128)
+  %12 = call float @llvm.SI.load.const(<16 x i8> undef, i32 132)
+  %13 = call float @llvm.SI.load.const(<16 x i8> undef, i32 136)
+  %14 = call float @llvm.SI.load.const(<16 x i8> undef, i32 144)
+  %15 = call float @llvm.SI.load.const(<16 x i8> undef, i32 148)
+  %16 = call float @llvm.SI.load.const(<16 x i8> undef, i32 152)
+  %17 = call float @llvm.SI.load.const(<16 x i8> undef, i32 160)
+  %18 = call float @llvm.SI.load.const(<16 x i8> undef, i32 164)
+  %19 = call float @llvm.SI.load.const(<16 x i8> undef, i32 168)
+  %20 = call float @llvm.SI.load.const(<16 x i8> undef, i32 176)
+  %21 = call float @llvm.SI.load.const(<16 x i8> undef, i32 180)
+  %22 = call float @llvm.SI.load.const(<16 x i8> undef, i32 184)
+  %23 = call float @llvm.SI.load.const(<16 x i8> undef, i32 192)
+  %24 = call float @llvm.SI.load.const(<16 x i8> undef, i32 196)
+  %25 = call float @llvm.SI.load.const(<16 x i8> undef, i32 200)
+  %26 = call float @llvm.SI.load.const(<16 x i8> undef, i32 208)
+  %27 = call float @llvm.SI.load.const(<16 x i8> undef, i32 212)
+  %28 = call float @llvm.SI.load.const(<16 x i8> undef, i32 216)
+  %29 = call float @llvm.SI.load.const(<16 x i8> undef, i32 224)
+  %30 = call float @llvm.SI.load.const(<16 x i8> undef, i32 228)
+  %31 = call float @llvm.SI.load.const(<16 x i8> undef, i32 232)
+  %32 = call float @llvm.SI.load.const(<16 x i8> undef, i32 240)
+  %33 = call float @llvm.SI.load.const(<16 x i8> undef, i32 244)
+  %34 = call float @llvm.SI.load.const(<16 x i8> undef, i32 248)
+  %35 = call float @llvm.SI.load.const(<16 x i8> undef, i32 256)
+  %36 = call float @llvm.SI.load.const(<16 x i8> undef, i32 260)
+  %37 = call float @llvm.SI.load.const(<16 x i8> undef, i32 264)
+  %38 = call float @llvm.SI.load.const(<16 x i8> undef, i32 272)
+  %39 = call float @llvm.SI.load.const(<16 x i8> undef, i32 276)
+  %40 = call float @llvm.SI.load.const(<16 x i8> undef, i32 280)
+  %41 = call float @llvm.SI.load.const(<16 x i8> undef, i32 288)
+  %42 = call float @llvm.SI.load.const(<16 x i8> undef, i32 292)
+  %43 = call float @llvm.SI.load.const(<16 x i8> undef, i32 296)
+  %44 = call float @llvm.SI.load.const(<16 x i8> undef, i32 304)
+  %45 = call float @llvm.SI.load.const(<16 x i8> undef, i32 308)
+  %46 = call float @llvm.SI.load.const(<16 x i8> undef, i32 312)
+  %47 = call float @llvm.SI.load.const(<16 x i8> undef, i32 320)
+  %48 = call float @llvm.SI.load.const(<16 x i8> undef, i32 324)
+  %49 = call float @llvm.SI.load.const(<16 x i8> undef, i32 328)
+  %50 = call float @llvm.SI.load.const(<16 x i8> undef, i32 336)
+  %51 = call float @llvm.SI.load.const(<16 x i8> undef, i32 340)
+  %52 = call float @llvm.SI.load.const(<16 x i8> undef, i32 344)
+  %53 = call float @llvm.SI.load.const(<16 x i8> undef, i32 352)
+  %54 = call float @llvm.SI.load.const(<16 x i8> undef, i32 356)
+  %55 = call float @llvm.SI.load.const(<16 x i8> undef, i32 360)
+  %56 = call float @llvm.SI.load.const(<16 x i8> undef, i32 368)
+  %57 = call float @llvm.SI.load.const(<16 x i8> undef, i32 372)
+  %58 = call float @llvm.SI.load.const(<16 x i8> undef, i32 376)
+  %59 = call float @llvm.SI.load.const(<16 x i8> undef, i32 384)
+  %60 = call float @llvm.SI.load.const(<16 x i8> undef, i32 388)
+  %61 = call float @llvm.SI.load.const(<16 x i8> undef, i32 392)
+  %62 = call float @llvm.SI.load.const(<16 x i8> undef, i32 400)
+  %63 = call float @llvm.SI.load.const(<16 x i8> undef, i32 404)
+  %64 = call float @llvm.SI.load.const(<16 x i8> undef, i32 408)
+  %65 = call float @llvm.SI.load.const(<16 x i8> undef, i32 416)
+  %66 = call float @llvm.SI.load.const(<16 x i8> undef, i32 420)
+  br label %LOOP
+
+LOOP:                                             ; preds = %ENDIF2795, %main_body
+  %temp894.0 = phi float [ 0.000000e+00, %main_body ], [ %temp894.1, %ENDIF2795 ]
+  %temp18.0 = phi float [ undef, %main_body ], [ %temp18.1, %ENDIF2795 ]
+  %67 = icmp sgt i32 undef, 4
+  br i1 %67, label %ENDLOOP, label %ENDIF
+
+ENDLOOP:                                          ; preds = %ELSE2566, %LOOP
+  %68 = call float @llvm.AMDGPU.lrp(float %0, float undef, float undef)
+  call void @llvm.SI.export(i32 15, i32 1, i32 1, i32 0, i32 0, float undef, float %68, float undef, float 1.000000e+00)
+  ret void
+
+ENDIF:                                            ; preds = %LOOP
+  %69 = fsub float %2, undef
+  %70 = fsub float %3, undef
+  %71 = fsub float %4, undef
+  %72 = fmul float %69, 0.000000e+00
+  %73 = fmul float %70, undef
+  %74 = fmul float %71, undef
+  %75 = fsub float %6, undef
+  %76 = fsub float %7, undef
+  %77 = fmul float %75, undef
+  %78 = fmul float %76, 0.000000e+00
+  %79 = call float @llvm.minnum.f32(float %74, float %78)
+  %80 = call float @llvm.maxnum.f32(float %72, float 0.000000e+00)
+  %81 = call float @llvm.maxnum.f32(float %73, float %77)
+  %82 = call float @llvm.maxnum.f32(float undef, float %79)
+  %83 = call float @llvm.minnum.f32(float %80, float %81)
+  %84 = call float @llvm.minnum.f32(float %83, float undef)
+  %85 = fsub float %14, undef
+  %86 = fsub float %15, undef
+  %87 = fsub float %16, undef
+  %88 = fmul float %85, undef
+  %89 = fmul float %86, undef
+  %90 = fmul float %87, undef
+  %91 = fsub float %17, undef
+  %92 = fsub float %18, undef
+  %93 = fsub float %19, undef
+  %94 = fmul float %91, 0.000000e+00
+  %95 = fmul float %92, undef
+  %96 = fmul float %93, undef
+  %97 = call float @llvm.minnum.f32(float %89, float %95)
+  %98 = call float @llvm.maxnum.f32(float %88, float %94)
+  %99 = call float @llvm.maxnum.f32(float %90, float %96)
+  %100 = call float @llvm.maxnum.f32(float undef, float %97)
+  %101 = call float @llvm.maxnum.f32(float %100, float undef)
+  %102 = call float @llvm.minnum.f32(float %98, float undef)
+  %103 = call float @llvm.minnum.f32(float %102, float %99)
+  %104 = fsub float %30, undef
+  %105 = fsub float %31, undef
+  %106 = fmul float %104, 0.000000e+00
+  %107 = fmul float %105, 0.000000e+00
+  %108 = call float @llvm.minnum.f32(float undef, float %106)
+  %109 = call float @llvm.maxnum.f32(float undef, float %107)
+  %110 = call float @llvm.maxnum.f32(float undef, float %108)
+  %111 = call float @llvm.maxnum.f32(float %110, float undef)
+  %112 = call float @llvm.minnum.f32(float undef, float %109)
+  %113 = fsub float %32, undef
+  %114 = fsub float %33, undef
+  %115 = fsub float %34, undef
+  %116 = fmul float %113, 0.000000e+00
+  %117 = fmul float %114, undef
+  %118 = fmul float %115, undef
+  %119 = fsub float %35, undef
+  %120 = fsub float %36, undef
+  %121 = fsub float %37, undef
+  %122 = fmul float %119, undef
+  %123 = fmul float %120, undef
+  %124 = fmul float %121, undef
+  %125 = call float @llvm.minnum.f32(float %116, float %122)
+  %126 = call float @llvm.minnum.f32(float %117, float %123)
+  %127 = call float @llvm.minnum.f32(float %118, float %124)
+  %128 = call float @llvm.maxnum.f32(float %125, float %126)
+  %129 = call float @llvm.maxnum.f32(float %128, float %127)
+  %130 = fsub float %38, undef
+  %131 = fsub float %39, undef
+  %132 = fsub float %40, undef
+  %133 = fmul float %130, 0.000000e+00
+  %134 = fmul float %131, undef
+  %135 = fmul float %132, undef
+  %136 = fsub float %41, undef
+  %137 = fsub float %42, undef
+  %138 = fsub float %43, undef
+  %139 = fmul float %136, undef
+  %140 = fmul float %137, undef
+  %141 = fmul float %138, undef
+  %142 = call float @llvm.minnum.f32(float %133, float %139)
+  %143 = call float @llvm.minnum.f32(float %134, float %140)
+  %144 = call float @llvm.minnum.f32(float %135, float %141)
+  %145 = call float @llvm.maxnum.f32(float %142, float %143)
+  %146 = call float @llvm.maxnum.f32(float %145, float %144)
+  %147 = fsub float %44, undef
+  %148 = fsub float %45, undef
+  %149 = fsub float %46, undef
+  %150 = fmul float %147, 0.000000e+00
+  %151 = fmul float %148, 0.000000e+00
+  %152 = fmul float %149, undef
+  %153 = fsub float %47, undef
+  %154 = fsub float %48, undef
+  %155 = fsub float %49, undef
+  %156 = fmul float %153, undef
+  %157 = fmul float %154, 0.000000e+00
+  %158 = fmul float %155, undef
+  %159 = call float @llvm.minnum.f32(float %150, float %156)
+  %160 = call float @llvm.minnum.f32(float %151, float %157)
+  %161 = call float @llvm.minnum.f32(float %152, float %158)
+  %162 = call float @llvm.maxnum.f32(float %159, float %160)
+  %163 = call float @llvm.maxnum.f32(float %162, float %161)
+  %164 = fsub float %50, undef
+  %165 = fsub float %51, undef
+  %166 = fsub float %52, undef
+  %167 = fmul float %164, undef
+  %168 = fmul float %165, 0.000000e+00
+  %169 = fmul float %166, 0.000000e+00
+  %170 = fsub float %53, undef
+  %171 = fsub float %54, undef
+  %172 = fsub float %55, undef
+  %173 = fdiv float 1.000000e+00, %temp18.0
+  %174 = fmul float %170, undef
+  %175 = fmul float %171, undef
+  %176 = fmul float %172, %173
+  %177 = call float @llvm.minnum.f32(float %167, float %174)
+  %178 = call float @llvm.minnum.f32(float %168, float %175)
+  %179 = call float @llvm.minnum.f32(float %169, float %176)
+  %180 = call float @llvm.maxnum.f32(float %177, float %178)
+  %181 = call float @llvm.maxnum.f32(float %180, float %179)
+  %182 = fsub float %62, undef
+  %183 = fsub float %63, undef
+  %184 = fsub float %64, undef
+  %185 = fmul float %182, 0.000000e+00
+  %186 = fmul float %183, undef
+  %187 = fmul float %184, undef
+  %188 = fsub float %65, undef
+  %189 = fsub float %66, undef
+  %190 = fmul float %188, undef
+  %191 = fmul float %189, undef
+  %192 = call float @llvm.maxnum.f32(float %185, float %190)
+  %193 = call float @llvm.maxnum.f32(float %186, float %191)
+  %194 = call float @llvm.maxnum.f32(float %187, float undef)
+  %195 = call float @llvm.minnum.f32(float %192, float %193)
+  %196 = call float @llvm.minnum.f32(float %195, float %194)
+  %.temp292.7 = select i1 undef, float %163, float undef
+  %temp292.9 = select i1 false, float %181, float %.temp292.7
+  %.temp292.9 = select i1 undef, float undef, float %temp292.9
+  %197 = fcmp ogt float undef, 0.000000e+00
+  %198 = fcmp olt float undef, %196
+  %199 = and i1 %197, %198
+  %200 = fcmp olt float undef, %.temp292.9
+  %201 = and i1 %199, %200
+  %temp292.11 = select i1 %201, float undef, float %.temp292.9
+  br i1 undef, label %IF2565, label %ELSE2566
+
+IF2565:                                           ; preds = %ENDIF
+  br i1 false, label %ENDIF2582, label %ELSE2584
+
+ELSE2566:                                         ; preds = %ENDIF
+  %202 = fcmp oeq float %temp292.11, 1.000000e+04
+  br i1 %202, label %ENDLOOP, label %ELSE2593
+
+ENDIF2564:                                        ; preds = %ENDIF2594, %ENDIF2588
+  %temp894.1 = phi float [ undef, %ENDIF2588 ], [ %temp894.2, %ENDIF2594 ]
+  %temp18.1 = phi float [ %219, %ENDIF2588 ], [ undef, %ENDIF2594 ]
+  %203 = fsub float %5, undef
+  %204 = fmul float %203, undef
+  %205 = call float @llvm.maxnum.f32(float undef, float %204)
+  %206 = call float @llvm.minnum.f32(float %205, float undef)
+  %207 = call float @llvm.minnum.f32(float %206, float undef)
+  %208 = fcmp ogt float undef, 0.000000e+00
+  %209 = fcmp olt float undef, 1.000000e+00
+  %210 = and i1 %208, %209
+  %211 = fcmp olt float undef, %207
+  %212 = and i1 %210, %211
+  br i1 %212, label %ENDIF2795, label %ELSE2797
+
+ELSE2584:                                         ; preds = %IF2565
+  br label %ENDIF2582
+
+ENDIF2582:                                        ; preds = %ELSE2584, %IF2565
+  %213 = fadd float %1, undef
+  %214 = fadd float 0.000000e+00, %213
+  %215 = call float @llvm.AMDIL.fraction.(float %214)
+  br i1 undef, label %IF2589, label %ELSE2590
+
+IF2589:                                           ; preds = %ENDIF2582
+  br label %ENDIF2588
+
+ELSE2590:                                         ; preds = %ENDIF2582
+  br label %ENDIF2588
+
+ENDIF2588:                                        ; preds = %ELSE2590, %IF2589
+  %216 = fsub float 1.000000e+00, %215
+  %217 = call float @llvm.sqrt.f32(float %216)
+  %218 = fmul float %217, undef
+  %219 = fadd float %218, undef
+  br label %ENDIF2564
+
+ELSE2593:                                         ; preds = %ELSE2566
+  %220 = fcmp oeq float %temp292.11, %82
+  %221 = fcmp olt float %82, %84
+  %222 = and i1 %220, %221
+  br i1 %222, label %ENDIF2594, label %ELSE2596
+
+ELSE2596:                                         ; preds = %ELSE2593
+  %223 = fcmp oeq float %temp292.11, %101
+  %224 = fcmp olt float %101, %103
+  %225 = and i1 %223, %224
+  br i1 %225, label %ENDIF2594, label %ELSE2632
+
+ENDIF2594:                                        ; preds = %ELSE2788, %ELSE2785, %ELSE2782, %ELSE2779, %IF2775, %ELSE2761, %ELSE2758, %IF2757, %ELSE2704, %ELSE2686, %ELSE2671, %ELSE2668, %IF2667, %ELSE2632, %ELSE2596, %ELSE2593
+  %temp894.2 = phi float [ 0.000000e+00, %IF2667 ], [ 0.000000e+00, %ELSE2671 ], [ 0.000000e+00, %IF2757 ], [ 0.000000e+00, %ELSE2761 ], [ %temp894.0, %ELSE2758 ], [ 0.000000e+00, %IF2775 ], [ 0.000000e+00, %ELSE2779 ], [ 0.000000e+00, %ELSE2782 ], [ %.2848, %ELSE2788 ], [ 0.000000e+00, %ELSE2785 ], [ 0.000000e+00, %ELSE2593 ], [ 0.000000e+00, %ELSE2632 ], [ 0.000000e+00, %ELSE2704 ], [ 0.000000e+00, %ELSE2686 ], [ 0.000000e+00, %ELSE2668 ], [ 0.000000e+00, %ELSE2596 ]
+  %226 = fmul float %temp894.2, undef
+  br label %ENDIF2564
+
+ELSE2632:                                         ; preds = %ELSE2596
+  br i1 undef, label %ENDIF2594, label %ELSE2650
+
+ELSE2650:                                         ; preds = %ELSE2632
+  %227 = fcmp oeq float %temp292.11, %111
+  %228 = fcmp olt float %111, %112
+  %229 = and i1 %227, %228
+  br i1 %229, label %IF2667, label %ELSE2668
+
+IF2667:                                           ; preds = %ELSE2650
+  br i1 undef, label %ENDIF2594, label %ELSE2671
+
+ELSE2668:                                         ; preds = %ELSE2650
+  %230 = fcmp oeq float %temp292.11, %129
+  %231 = fcmp olt float %129, undef
+  %232 = and i1 %230, %231
+  br i1 %232, label %ENDIF2594, label %ELSE2686
+
+ELSE2671:                                         ; preds = %IF2667
+  br label %ENDIF2594
+
+ELSE2686:                                         ; preds = %ELSE2668
+  %233 = fcmp oeq float %temp292.11, %146
+  %234 = fcmp olt float %146, undef
+  %235 = and i1 %233, %234
+  br i1 %235, label %ENDIF2594, label %ELSE2704
+
+ELSE2704:                                         ; preds = %ELSE2686
+  %236 = fcmp oeq float %temp292.11, %181
+  %237 = fcmp olt float %181, undef
+  %238 = and i1 %236, %237
+  br i1 %238, label %ENDIF2594, label %ELSE2740
+
+ELSE2740:                                         ; preds = %ELSE2704
+  br i1 undef, label %IF2757, label %ELSE2758
+
+IF2757:                                           ; preds = %ELSE2740
+  br i1 undef, label %ENDIF2594, label %ELSE2761
+
+ELSE2758:                                         ; preds = %ELSE2740
+  br i1 undef, label %IF2775, label %ENDIF2594
+
+ELSE2761:                                         ; preds = %IF2757
+  br label %ENDIF2594
+
+IF2775:                                           ; preds = %ELSE2758
+  %239 = fcmp olt float undef, undef
+  br i1 %239, label %ENDIF2594, label %ELSE2779
+
+ELSE2779:                                         ; preds = %IF2775
+  br i1 undef, label %ENDIF2594, label %ELSE2782
+
+ELSE2782:                                         ; preds = %ELSE2779
+  br i1 undef, label %ENDIF2594, label %ELSE2785
+
+ELSE2785:                                         ; preds = %ELSE2782
+  %240 = fcmp olt float undef, 0.000000e+00
+  br i1 %240, label %ENDIF2594, label %ELSE2788
+
+ELSE2788:                                         ; preds = %ELSE2785
+  %241 = fcmp olt float 0.000000e+00, undef
+  %.2848 = select i1 %241, float -1.000000e+00, float 1.000000e+00
+  br label %ENDIF2594
+
+ELSE2797:                                         ; preds = %ENDIF2564
+  %242 = fsub float %8, undef
+  %243 = fsub float %9, undef
+  %244 = fsub float %10, undef
+  %245 = fmul float %242, undef
+  %246 = fmul float %243, undef
+  %247 = fmul float %244, undef
+  %248 = fsub float %11, undef
+  %249 = fsub float %12, undef
+  %250 = fsub float %13, undef
+  %251 = fmul float %248, undef
+  %252 = fmul float %249, undef
+  %253 = fmul float %250, undef
+  %254 = call float @llvm.minnum.f32(float %245, float %251)
+  %255 = call float @llvm.minnum.f32(float %246, float %252)
+  %256 = call float @llvm.maxnum.f32(float %247, float %253)
+  %257 = call float @llvm.maxnum.f32(float %254, float %255)
+  %258 = call float @llvm.maxnum.f32(float %257, float undef)
+  %259 = call float @llvm.minnum.f32(float undef, float %256)
+  %260 = fcmp ogt float %258, 0.000000e+00
+  %261 = fcmp olt float %258, 1.000000e+00
+  %262 = and i1 %260, %261
+  %263 = fcmp olt float %258, %259
+  %264 = and i1 %262, %263
+  br i1 %264, label %ENDIF2795, label %ELSE2800
+
+ENDIF2795:                                        ; preds = %ELSE2824, %ELSE2821, %ELSE2818, %ELSE2815, %ELSE2812, %ELSE2809, %ELSE2806, %ELSE2803, %ELSE2800, %ELSE2797, %ENDIF2564
+  br label %LOOP
+
+ELSE2800:                                         ; preds = %ELSE2797
+  br i1 undef, label %ENDIF2795, label %ELSE2803
+
+ELSE2803:                                         ; preds = %ELSE2800
+  %265 = fsub float %20, undef
+  %266 = fsub float %21, undef
+  %267 = fsub float %22, undef
+  %268 = fmul float %265, undef
+  %269 = fmul float %266, undef
+  %270 = fmul float %267, 0.000000e+00
+  %271 = fsub float %23, undef
+  %272 = fsub float %24, undef
+  %273 = fsub float %25, undef
+  %274 = fmul float %271, undef
+  %275 = fmul float %272, undef
+  %276 = fmul float %273, undef
+  %277 = call float @llvm.minnum.f32(float %268, float %274)
+  %278 = call float @llvm.maxnum.f32(float %269, float %275)
+  %279 = call float @llvm.maxnum.f32(float %270, float %276)
+  %280 = call float @llvm.maxnum.f32(float %277, float undef)
+  %281 = call float @llvm.maxnum.f32(float %280, float undef)
+  %282 = call float @llvm.minnum.f32(float undef, float %278)
+  %283 = call float @llvm.minnum.f32(float %282, float %279)
+  %284 = fcmp ogt float %281, 0.000000e+00
+  %285 = fcmp olt float %281, 1.000000e+00
+  %286 = and i1 %284, %285
+  %287 = fcmp olt float %281, %283
+  %288 = and i1 %286, %287
+  br i1 %288, label %ENDIF2795, label %ELSE2806
+
+ELSE2806:                                         ; preds = %ELSE2803
+  %289 = fsub float %26, undef
+  %290 = fsub float %27, undef
+  %291 = fsub float %28, undef
+  %292 = fmul float %289, undef
+  %293 = fmul float %290, 0.000000e+00
+  %294 = fmul float %291, undef
+  %295 = fsub float %29, undef
+  %296 = fmul float %295, undef
+  %297 = call float @llvm.minnum.f32(float %292, float %296)
+  %298 = call float @llvm.minnum.f32(float %293, float undef)
+  %299 = call float @llvm.maxnum.f32(float %294, float undef)
+  %300 = call float @llvm.maxnum.f32(float %297, float %298)
+  %301 = call float @llvm.maxnum.f32(float %300, float undef)
+  %302 = call float @llvm.minnum.f32(float undef, float %299)
+  %303 = fcmp ogt float %301, 0.000000e+00
+  %304 = fcmp olt float %301, 1.000000e+00
+  %305 = and i1 %303, %304
+  %306 = fcmp olt float %301, %302
+  %307 = and i1 %305, %306
+  br i1 %307, label %ENDIF2795, label %ELSE2809
+
+ELSE2809:                                         ; preds = %ELSE2806
+  br i1 undef, label %ENDIF2795, label %ELSE2812
+
+ELSE2812:                                         ; preds = %ELSE2809
+  br i1 undef, label %ENDIF2795, label %ELSE2815
+
+ELSE2815:                                         ; preds = %ELSE2812
+  br i1 undef, label %ENDIF2795, label %ELSE2818
+
+ELSE2818:                                         ; preds = %ELSE2815
+  br i1 undef, label %ENDIF2795, label %ELSE2821
+
+ELSE2821:                                         ; preds = %ELSE2818
+  %308 = fsub float %56, undef
+  %309 = fsub float %57, undef
+  %310 = fsub float %58, undef
+  %311 = fmul float %308, undef
+  %312 = fmul float %309, 0.000000e+00
+  %313 = fmul float %310, undef
+  %314 = fsub float %59, undef
+  %315 = fsub float %60, undef
+  %316 = fsub float %61, undef
+  %317 = fmul float %314, undef
+  %318 = fmul float %315, undef
+  %319 = fmul float %316, undef
+  %320 = call float @llvm.maxnum.f32(float %311, float %317)
+  %321 = call float @llvm.maxnum.f32(float %312, float %318)
+  %322 = call float @llvm.maxnum.f32(float %313, float %319)
+  %323 = call float @llvm.minnum.f32(float %320, float %321)
+  %324 = call float @llvm.minnum.f32(float %323, float %322)
+  %325 = fcmp ogt float undef, 0.000000e+00
+  %326 = fcmp olt float undef, 1.000000e+00
+  %327 = and i1 %325, %326
+  %328 = fcmp olt float undef, %324
+  %329 = and i1 %327, %328
+  br i1 %329, label %ENDIF2795, label %ELSE2824
+
+ELSE2824:                                         ; preds = %ELSE2821
+  %.2849 = select i1 undef, float 0.000000e+00, float 1.000000e+00
+  br label %ENDIF2795
+}
+
+; Function Attrs: nounwind readnone
+declare float @llvm.SI.load.const(<16 x i8>, i32) #1
+
+; Function Attrs: readnone
+declare float @llvm.AMDIL.fraction.(float) #2
+
+; Function Attrs: nounwind readnone
+declare float @llvm.sqrt.f32(float) #1
+
+; Function Attrs: nounwind readnone
+declare float @llvm.minnum.f32(float, float) #1
+
+; Function Attrs: nounwind readnone
+declare float @llvm.maxnum.f32(float, float) #1
+
+; Function Attrs: readnone
+declare float @llvm.AMDGPU.lrp(float, float, float) #2
+
+declare void @llvm.SI.export(i32, i32, i32, i32, i32, float, float, float, float)
+
+attributes #0 = { "ShaderType"="0" "enable-no-nans-fp-math"="true" }
+attributes #1 = { nounwind readnone }
+attributes #2 = { readnone }
diff --git a/test/CodeGen/R600/si-triv-disjoint-mem-access.ll b/test/CodeGen/AMDGPU/si-triv-disjoint-mem-access.ll
similarity index 100%
rename from test/CodeGen/R600/si-triv-disjoint-mem-access.ll
rename to test/CodeGen/AMDGPU/si-triv-disjoint-mem-access.ll
diff --git a/test/CodeGen/R600/si-vector-hang.ll b/test/CodeGen/AMDGPU/si-vector-hang.ll
similarity index 94%
rename from test/CodeGen/R600/si-vector-hang.ll
rename to test/CodeGen/AMDGPU/si-vector-hang.ll
index 94c47fe3c60..bd427dd3ed4 100644
--- a/test/CodeGen/R600/si-vector-hang.ll
+++ b/test/CodeGen/AMDGPU/si-vector-hang.ll
@@ -11,10 +11,7 @@
 ; CHECK: buffer_store_byte
 ; CHECK: buffer_store_byte
 ; ModuleID = 'radeon'
-target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v16:16:16-v24:32:32-v32:32:32-v48:64:64-v64:64:64-v96:128:128-v128:128:128-v192:256:256-v256:256:256-v512:512:512-v1024:1024:1024-v2048:2048:2048-n32:64"
-target triple = "r600--"
 
-; Function Attrs: nounwind
 define void @test_8_min_char(i8 addrspace(1)* nocapture %out, i8 addrspace(1)* nocapture readonly %in0, i8 addrspace(1)* nocapture readonly %in1) #0 {
 entry:
   %0 = load i8, i8 addrspace(1)* %in0, align 1
diff --git a/test/CodeGen/R600/sign_extend.ll b/test/CodeGen/AMDGPU/sign_extend.ll
similarity index 100%
rename from test/CodeGen/R600/sign_extend.ll
rename to test/CodeGen/AMDGPU/sign_extend.ll
diff --git a/test/CodeGen/R600/simplify-demanded-bits-build-pair.ll b/test/CodeGen/AMDGPU/simplify-demanded-bits-build-pair.ll
similarity index 100%
rename from test/CodeGen/R600/simplify-demanded-bits-build-pair.ll
rename to test/CodeGen/AMDGPU/simplify-demanded-bits-build-pair.ll
diff --git a/test/CodeGen/R600/sint_to_fp.f64.ll b/test/CodeGen/AMDGPU/sint_to_fp.f64.ll
similarity index 91%
rename from test/CodeGen/R600/sint_to_fp.f64.ll
rename to test/CodeGen/AMDGPU/sint_to_fp.f64.ll
index da4e91db3a3..0db7cdc171b 100644
--- a/test/CodeGen/R600/sint_to_fp.f64.ll
+++ b/test/CodeGen/AMDGPU/sint_to_fp.f64.ll
@@ -12,11 +12,11 @@ define void @sint_to_fp_i32_to_f64(double addrspace(1)* %out, i32 %in) {
 
 ; FIXME: select on 0, 0
 ; SI-LABEL: {{^}}sint_to_fp_i1_f64:
-; SI: v_cmp_eq_i32_e64 [[CMP:s\[[0-9]+:[0-9]\]]],
+; SI: v_cmp_eq_i32_e64 vcc,
 ; We can't fold the SGPRs into v_cndmask_b32_e64, because it already
-; uses an SGPR for [[CMP]]
-; SI: v_cndmask_b32_e64 v{{[0-9]+}}, 0, v{{[0-9]+}}, [[CMP]]
-; SI: v_cndmask_b32_e64 v{{[0-9]+}}, 0, 0, [[CMP]]
+; uses an SGPR (implicit vcc).
+; SI: v_cndmask_b32_e32 v{{[0-9]+}}, 0, v{{[0-9]+}}
+; SI: v_cndmask_b32_e64 v{{[0-9]+}}, 0, 0, vcc
 ; SI: buffer_store_dwordx2
 ; SI: s_endpgm
 define void @sint_to_fp_i1_f64(double addrspace(1)* %out, i32 %in) {
diff --git a/test/CodeGen/R600/sint_to_fp.ll b/test/CodeGen/AMDGPU/sint_to_fp.ll
similarity index 100%
rename from test/CodeGen/R600/sint_to_fp.ll
rename to test/CodeGen/AMDGPU/sint_to_fp.ll
diff --git a/test/CodeGen/R600/smrd.ll b/test/CodeGen/AMDGPU/smrd.ll
similarity index 93%
rename from test/CodeGen/R600/smrd.ll
rename to test/CodeGen/AMDGPU/smrd.ll
index b0c18ca5959..0598208e131 100644
--- a/test/CodeGen/R600/smrd.ll
+++ b/test/CodeGen/AMDGPU/smrd.ll
@@ -43,13 +43,7 @@ entry:
 ; GCN-LABEL: {{^}}smrd3:
 ; FIXME: There are too many copies here because we don't fold immediates
 ;        through REG_SEQUENCE
-; SI: s_mov_b32 s[[SLO:[0-9]+]], 0 ;
-; SI: s_mov_b32 s[[SHI:[0-9]+]], 4
-; SI: s_mov_b32 s[[SSLO:[0-9]+]], s[[SLO]]
-; SI-DAG: v_mov_b32_e32 v[[VLO:[0-9]+]], s[[SSLO]]
-; SI-DAG: v_mov_b32_e32 v[[VHI:[0-9]+]], s[[SHI]]
-; FIXME: We should be able to use s_load_dword here
-; SI: buffer_load_dword v{{[0-9]+}}, v{{\[}}[[VLO]]:[[VHI]]{{\]}}, s[{{[0-9]+:[0-9]+}}], 0 addr64
+; SI: s_load_dwordx2 s[{{[0-9]:[0-9]}}], s[{{[0-9]:[0-9]}}], 0xb ; encoding: [0x0b
 ; TODO: Add VI checks
 ; GCN: s_endpgm
 define void @smrd3(i32 addrspace(1)* %out, i32 addrspace(2)* %ptr) {
diff --git a/test/CodeGen/R600/split-scalar-i64-add.ll b/test/CodeGen/AMDGPU/split-scalar-i64-add.ll
similarity index 100%
rename from test/CodeGen/R600/split-scalar-i64-add.ll
rename to test/CodeGen/AMDGPU/split-scalar-i64-add.ll
diff --git a/test/CodeGen/R600/sra.ll b/test/CodeGen/AMDGPU/sra.ll
similarity index 100%
rename from test/CodeGen/R600/sra.ll
rename to test/CodeGen/AMDGPU/sra.ll
diff --git a/test/CodeGen/R600/srem.ll b/test/CodeGen/AMDGPU/srem.ll
similarity index 100%
rename from test/CodeGen/R600/srem.ll
rename to test/CodeGen/AMDGPU/srem.ll
diff --git a/test/CodeGen/R600/srl.ll b/test/CodeGen/AMDGPU/srl.ll
similarity index 85%
rename from test/CodeGen/R600/srl.ll
rename to test/CodeGen/AMDGPU/srl.ll
index 4904d7fa1bd..0dad91e709d 100644
--- a/test/CodeGen/R600/srl.ll
+++ b/test/CodeGen/AMDGPU/srl.ll
@@ -1,7 +1,9 @@
-; RUN: llc -march=amdgcn -mcpu=verde -verify-machineinstrs < %s | FileCheck -check-prefix=SI -check-prefix=FUNC %s
-; RUN: llc -march=amdgcn -mcpu=tonga -verify-machineinstrs < %s | FileCheck -check-prefix=VI -check-prefix=FUNC %s
+; RUN: llc -march=amdgcn -mcpu=verde -verify-machineinstrs < %s | FileCheck -check-prefix=SI -check-prefix=GCN -check-prefix=FUNC %s
+; XUN: llc -march=amdgcn -mcpu=tonga -verify-machineinstrs < %s | FileCheck -check-prefix=VI -check-prefix=GCN -check-prefix=FUNC %s
 ; RUN: llc -march=r600 -mcpu=redwood < %s | FileCheck -check-prefix=EG -check-prefix=FUNC %s
 
+declare i32 @llvm.r600.read.tidig.x() #0
+
 ; FUNC-LABEL: {{^}}lshr_i32:
 ; SI: v_lshrrev_b32_e32 v{{[0-9]+, v[0-9]+, v[0-9]+}}
 ; VI: v_lshrrev_b32_e32 v{{[0-9]+, v[0-9]+, v[0-9]+}}
@@ -184,3 +186,32 @@ define void @lshr_v4i64(<4 x i64> addrspace(1)* %out, <4 x i64> addrspace(1)* %i
   store <4 x i64> %result, <4 x i64> addrspace(1)* %out
   ret void
 }
+
+; Make sure load width gets reduced to i32 load.
+; GCN-LABEL: {{^}}s_lshr_32_i64:
+; GCN-DAG: s_load_dword [[HI_A:s[0-9]+]], s{{\[[0-9]+:[0-9]+\]}}, 0xc{{$}}
+; GCN-DAG: s_mov_b32 s[[SHI:[0-9]+]], 0{{$}}
+; GCN-DAG: v_mov_b32_e32 v[[VHI:[0-9]+]], s[[SHI]]
+; GCN-DAG: v_mov_b32_e32 v[[VLO:[0-9]+]], [[HI_A]]
+; GCN: buffer_store_dwordx2 v{{\[}}[[VLO]]:[[VHI]]{{\]}}
+define void @s_lshr_32_i64(i64 addrspace(1)* %out, i64 %a) {
+  %result = lshr i64 %a, 32
+  store i64 %result, i64 addrspace(1)* %out
+  ret void
+}
+
+; GCN-LABEL: {{^}}v_lshr_32_i64:
+; GCN-DAG: buffer_load_dword v[[HI_A:[0-9]+]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64 offset:4
+; GCN-DAG: v_mov_b32_e32 v[[VHI:[0-9]+]], 0{{$}}
+; GCN: buffer_store_dwordx2 v{{\[}}[[HI_A]]:[[VHI]]{{\]}}
+define void @v_lshr_32_i64(i64 addrspace(1)* %out, i64 addrspace(1)* %in) {
+  %tid = call i32 @llvm.r600.read.tidig.x() #0
+  %gep.in = getelementptr i64, i64 addrspace(1)* %in, i32 %tid
+  %gep.out = getelementptr i64, i64 addrspace(1)* %out, i32 %tid
+  %a = load i64, i64 addrspace(1)* %gep.in
+  %result = lshr i64 %a, 32
+  store i64 %result, i64 addrspace(1)* %gep.out
+  ret void
+}
+
+attributes #0 = { nounwind readnone }
diff --git a/test/CodeGen/R600/ssubo.ll b/test/CodeGen/AMDGPU/ssubo.ll
similarity index 100%
rename from test/CodeGen/R600/ssubo.ll
rename to test/CodeGen/AMDGPU/ssubo.ll
diff --git a/test/CodeGen/R600/store-barrier.ll b/test/CodeGen/AMDGPU/store-barrier.ll
similarity index 100%
rename from test/CodeGen/R600/store-barrier.ll
rename to test/CodeGen/AMDGPU/store-barrier.ll
diff --git a/test/CodeGen/R600/store-v3i32.ll b/test/CodeGen/AMDGPU/store-v3i32.ll
similarity index 100%
rename from test/CodeGen/R600/store-v3i32.ll
rename to test/CodeGen/AMDGPU/store-v3i32.ll
diff --git a/test/CodeGen/R600/store-v3i64.ll b/test/CodeGen/AMDGPU/store-v3i64.ll
similarity index 100%
rename from test/CodeGen/R600/store-v3i64.ll
rename to test/CodeGen/AMDGPU/store-v3i64.ll
diff --git a/test/CodeGen/R600/store-vector-ptrs.ll b/test/CodeGen/AMDGPU/store-vector-ptrs.ll
similarity index 100%
rename from test/CodeGen/R600/store-vector-ptrs.ll
rename to test/CodeGen/AMDGPU/store-vector-ptrs.ll
diff --git a/test/CodeGen/AMDGPU/store.ll b/test/CodeGen/AMDGPU/store.ll
new file mode 100644
index 00000000000..0f89405e073
--- /dev/null
+++ b/test/CodeGen/AMDGPU/store.ll
@@ -0,0 +1,369 @@
+; RUN: llc -march=amdgcn -mcpu=verde -verify-machineinstrs < %s | FileCheck -check-prefix=SI -check-prefix=FUNC %s
+; RUN: llc -march=amdgcn -mcpu=tonga -verify-machineinstrs < %s | FileCheck -check-prefix=SI -check-prefix=FUNC %s
+; RUN: llc -march=r600 -mcpu=redwood < %s | FileCheck -check-prefix=EG -check-prefix=FUNC %s
+; RUN: llc -march=r600 -mcpu=cayman < %s | FileCheck -check-prefix=CM -check-prefix=FUNC %s
+
+;===------------------------------------------------------------------------===;
+; Global Address Space
+;===------------------------------------------------------------------------===;
+; FUNC-LABEL: {{^}}store_i1:
+; EG: MEM_RAT MSKOR
+; SI: buffer_store_byte
+define void @store_i1(i1 addrspace(1)* %out) {
+entry:
+  store i1 true, i1 addrspace(1)* %out
+  ret void
+}
+
+; i8 store
+; FUNC-LABEL: {{^}}store_i8:
+; EG: MEM_RAT MSKOR T[[RW_GPR:[0-9]]].XW, T{{[0-9]}}.X
+
+; IG 0: Get the byte index and truncate the value
+; EG: AND_INT * T{{[0-9]}}.[[BI_CHAN:[XYZW]]], KC0[2].Y, literal.x
+; EG: LSHL T{{[0-9]}}.[[SHIFT_CHAN:[XYZW]]], PV.[[BI_CHAN]], literal.x
+; EG: AND_INT * T{{[0-9]}}.[[TRUNC_CHAN:[XYZW]]], KC0[2].Z, literal.y
+; EG-NEXT: 3(4.203895e-45), 255(3.573311e-43)
+
+
+; IG 1: Truncate the calculated the shift amount for the mask
+
+; IG 2: Shift the value and the mask
+; EG: LSHL T[[RW_GPR]].X, PS, PV.[[SHIFT_CHAN]]
+; EG: LSHL * T[[RW_GPR]].W, literal.x, PV.[[SHIFT_CHAN]]
+; EG-NEXT: 255
+; IG 3: Initialize the Y and Z channels to zero
+;       XXX: An optimal scheduler should merge this into one of the prevous IGs.
+; EG: MOV T[[RW_GPR]].Y, 0.0
+; EG: MOV * T[[RW_GPR]].Z, 0.0
+
+; SI: buffer_store_byte
+
+define void @store_i8(i8 addrspace(1)* %out, i8 %in) {
+entry:
+  store i8 %in, i8 addrspace(1)* %out
+  ret void
+}
+
+; i16 store
+; FUNC-LABEL: {{^}}store_i16:
+; EG: MEM_RAT MSKOR T[[RW_GPR:[0-9]]].XW, T{{[0-9]}}.X
+
+; IG 0: Get the byte index and truncate the value
+
+
+; EG: AND_INT * T{{[0-9]}}.[[BI_CHAN:[XYZW]]], KC0[2].Y, literal.x
+; EG-NEXT: 3(4.203895e-45),
+
+; EG: LSHL T{{[0-9]}}.[[SHIFT_CHAN:[XYZW]]], PV.[[BI_CHAN]], literal.x
+; EG: AND_INT * T{{[0-9]}}.[[TRUNC_CHAN:[XYZW]]], KC0[2].Z, literal.y
+
+; EG-NEXT: 3(4.203895e-45), 65535(9.183409e-41)
+; IG 1: Truncate the calculated the shift amount for the mask
+
+; IG 2: Shift the value and the mask
+; EG: LSHL T[[RW_GPR]].X, PS, PV.[[SHIFT_CHAN]]
+; EG: LSHL * T[[RW_GPR]].W, literal.x, PV.[[SHIFT_CHAN]]
+; EG-NEXT: 65535
+; IG 3: Initialize the Y and Z channels to zero
+;       XXX: An optimal scheduler should merge this into one of the prevous IGs.
+; EG: MOV T[[RW_GPR]].Y, 0.0
+; EG: MOV * T[[RW_GPR]].Z, 0.0
+
+; SI: buffer_store_short
+define void @store_i16(i16 addrspace(1)* %out, i16 %in) {
+entry:
+  store i16 %in, i16 addrspace(1)* %out
+  ret void
+}
+
+; FUNC-LABEL: {{^}}store_v2i8:
+; EG: MEM_RAT MSKOR
+; EG-NOT: MEM_RAT MSKOR
+
+; SI: buffer_store_byte
+; SI: buffer_store_byte
+define void @store_v2i8(<2 x i8> addrspace(1)* %out, <2 x i32> %in) {
+entry:
+  %0 = trunc <2 x i32> %in to <2 x i8>
+  store <2 x i8> %0, <2 x i8> addrspace(1)* %out
+  ret void
+}
+
+
+; FUNC-LABEL: {{^}}store_v2i16:
+; EG: MEM_RAT_CACHELESS STORE_RAW
+
+; CM: MEM_RAT_CACHELESS STORE_DWORD
+
+; SI: buffer_store_short
+; SI: buffer_store_short
+define void @store_v2i16(<2 x i16> addrspace(1)* %out, <2 x i32> %in) {
+entry:
+  %0 = trunc <2 x i32> %in to <2 x i16>
+  store <2 x i16> %0, <2 x i16> addrspace(1)* %out
+  ret void
+}
+
+; FUNC-LABEL: {{^}}store_v4i8:
+; EG: MEM_RAT_CACHELESS STORE_RAW
+
+; CM: MEM_RAT_CACHELESS STORE_DWORD
+
+; SI: buffer_store_byte
+; SI: buffer_store_byte
+; SI: buffer_store_byte
+; SI: buffer_store_byte
+define void @store_v4i8(<4 x i8> addrspace(1)* %out, <4 x i32> %in) {
+entry:
+  %0 = trunc <4 x i32> %in to <4 x i8>
+  store <4 x i8> %0, <4 x i8> addrspace(1)* %out
+  ret void
+}
+
+; floating-point store
+; FUNC-LABEL: {{^}}store_f32:
+; EG: MEM_RAT_CACHELESS STORE_RAW T{{[0-9]+\.X, T[0-9]+\.X}}, 1
+
+; CM: MEM_RAT_CACHELESS STORE_DWORD T{{[0-9]+\.X, T[0-9]+\.X}}
+
+; SI: buffer_store_dword
+
+define void @store_f32(float addrspace(1)* %out, float %in) {
+  store float %in, float addrspace(1)* %out
+  ret void
+}
+
+; FUNC-LABEL: {{^}}store_v4i16:
+; EG: MEM_RAT MSKOR
+; EG: MEM_RAT MSKOR
+; EG: MEM_RAT MSKOR
+; EG: MEM_RAT MSKOR
+; EG-NOT: MEM_RAT MSKOR
+
+; SI: buffer_store_short
+; SI: buffer_store_short
+; SI: buffer_store_short
+; SI: buffer_store_short
+; SI-NOT: buffer_store_byte
+define void @store_v4i16(<4 x i16> addrspace(1)* %out, <4 x i32> %in) {
+entry:
+  %0 = trunc <4 x i32> %in to <4 x i16>
+  store <4 x i16> %0, <4 x i16> addrspace(1)* %out
+  ret void
+}
+
+; vec2 floating-point stores
+; FUNC-LABEL: {{^}}store_v2f32:
+; EG: MEM_RAT_CACHELESS STORE_RAW
+
+; CM: MEM_RAT_CACHELESS STORE_DWORD
+
+; SI: buffer_store_dwordx2
+
+define void @store_v2f32(<2 x float> addrspace(1)* %out, float %a, float %b) {
+entry:
+  %0 = insertelement <2 x float> <float 0.0, float 0.0>, float %a, i32 0
+  %1 = insertelement <2 x float> %0, float %b, i32 1
+  store <2 x float> %1, <2 x float> addrspace(1)* %out
+  ret void
+}
+
+; FUNC-LABEL: {{^}}store_v4i32:
+; EG: MEM_RAT_CACHELESS STORE_RAW
+; EG-NOT: MEM_RAT_CACHELESS STORE_RAW
+
+; CM: MEM_RAT_CACHELESS STORE_DWORD
+; CM-NOT: MEM_RAT_CACHELESS STORE_DWORD
+
+; SI: buffer_store_dwordx4
+define void @store_v4i32(<4 x i32> addrspace(1)* %out, <4 x i32> %in) {
+entry:
+  store <4 x i32> %in, <4 x i32> addrspace(1)* %out
+  ret void
+}
+
+; FUNC-LABEL: {{^}}store_i64_i8:
+; EG: MEM_RAT MSKOR
+; SI: buffer_store_byte
+define void @store_i64_i8(i8 addrspace(1)* %out, i64 %in) {
+entry:
+  %0 = trunc i64 %in to i8
+  store i8 %0, i8 addrspace(1)* %out
+  ret void
+}
+
+; FUNC-LABEL: {{^}}store_i64_i16:
+; EG: MEM_RAT MSKOR
+; SI: buffer_store_short
+define void @store_i64_i16(i16 addrspace(1)* %out, i64 %in) {
+entry:
+  %0 = trunc i64 %in to i16
+  store i16 %0, i16 addrspace(1)* %out
+  ret void
+}
+
+;===------------------------------------------------------------------------===;
+; Local Address Space
+;===------------------------------------------------------------------------===;
+
+; FUNC-LABEL: {{^}}store_local_i1:
+; EG: LDS_BYTE_WRITE
+; SI: ds_write_b8
+define void @store_local_i1(i1 addrspace(3)* %out) {
+entry:
+  store i1 true, i1 addrspace(3)* %out
+  ret void
+}
+
+; FUNC-LABEL: {{^}}store_local_i8:
+; EG: LDS_BYTE_WRITE
+
+; SI: ds_write_b8
+define void @store_local_i8(i8 addrspace(3)* %out, i8 %in) {
+  store i8 %in, i8 addrspace(3)* %out
+  ret void
+}
+
+; FUNC-LABEL: {{^}}store_local_i16:
+; EG: LDS_SHORT_WRITE
+
+; SI: ds_write_b16
+define void @store_local_i16(i16 addrspace(3)* %out, i16 %in) {
+  store i16 %in, i16 addrspace(3)* %out
+  ret void
+}
+
+; FUNC-LABEL: {{^}}store_local_v2i16:
+; EG: LDS_WRITE
+
+; CM: LDS_WRITE
+
+; SI: ds_write_b16
+; SI: ds_write_b16
+define void @store_local_v2i16(<2 x i16> addrspace(3)* %out, <2 x i16> %in) {
+entry:
+  store <2 x i16> %in, <2 x i16> addrspace(3)* %out
+  ret void
+}
+
+; FUNC-LABEL: {{^}}store_local_v4i8:
+; EG: LDS_WRITE
+
+; CM: LDS_WRITE
+
+; SI: ds_write_b8
+; SI: ds_write_b8
+; SI: ds_write_b8
+; SI: ds_write_b8
+define void @store_local_v4i8(<4 x i8> addrspace(3)* %out, <4 x i8> %in) {
+entry:
+  store <4 x i8> %in, <4 x i8> addrspace(3)* %out
+  ret void
+}
+
+; FUNC-LABEL: {{^}}store_local_v2i32:
+; EG: LDS_WRITE
+; EG: LDS_WRITE
+
+; CM: LDS_WRITE
+; CM: LDS_WRITE
+
+; SI: ds_write_b64
+define void @store_local_v2i32(<2 x i32> addrspace(3)* %out, <2 x i32> %in) {
+entry:
+  store <2 x i32> %in, <2 x i32> addrspace(3)* %out
+  ret void
+}
+
+; FUNC-LABEL: {{^}}store_local_v4i32:
+; EG: LDS_WRITE
+; EG: LDS_WRITE
+; EG: LDS_WRITE
+; EG: LDS_WRITE
+
+; CM: LDS_WRITE
+; CM: LDS_WRITE
+; CM: LDS_WRITE
+; CM: LDS_WRITE
+
+; SI: ds_write_b32
+; SI: ds_write_b32
+; SI: ds_write_b32
+; SI: ds_write_b32
+define void @store_local_v4i32(<4 x i32> addrspace(3)* %out, <4 x i32> %in) {
+entry:
+  store <4 x i32> %in, <4 x i32> addrspace(3)* %out
+  ret void
+}
+
+; FUNC-LABEL: {{^}}store_local_i64_i8:
+; EG: LDS_BYTE_WRITE
+; SI: ds_write_b8
+define void @store_local_i64_i8(i8 addrspace(3)* %out, i64 %in) {
+entry:
+  %0 = trunc i64 %in to i8
+  store i8 %0, i8 addrspace(3)* %out
+  ret void
+}
+
+; FUNC-LABEL: {{^}}store_local_i64_i16:
+; EG: LDS_SHORT_WRITE
+; SI: ds_write_b16
+define void @store_local_i64_i16(i16 addrspace(3)* %out, i64 %in) {
+entry:
+  %0 = trunc i64 %in to i16
+  store i16 %0, i16 addrspace(3)* %out
+  ret void
+}
+
+; The stores in this function are combined by the optimizer to create a
+; 64-bit store with 32-bit alignment.  This is legal for SI and the legalizer
+; should not try to split the 64-bit store back into 2 32-bit stores.
+;
+; Evergreen / Northern Islands don't support 64-bit stores yet, so there should
+; be two 32-bit stores.
+
+; FUNC-LABEL: {{^}}vecload2:
+; EG: MEM_RAT_CACHELESS STORE_RAW
+
+; CM: MEM_RAT_CACHELESS STORE_DWORD
+
+; SI: buffer_store_dwordx2
+define void @vecload2(i32 addrspace(1)* nocapture %out, i32 addrspace(2)* nocapture %mem) #0 {
+entry:
+  %0 = load i32, i32 addrspace(2)* %mem, align 4
+  %arrayidx1.i = getelementptr inbounds i32, i32 addrspace(2)* %mem, i64 1
+  %1 = load i32, i32 addrspace(2)* %arrayidx1.i, align 4
+  store i32 %0, i32 addrspace(1)* %out, align 4
+  %arrayidx1 = getelementptr inbounds i32, i32 addrspace(1)* %out, i64 1
+  store i32 %1, i32 addrspace(1)* %arrayidx1, align 4
+  ret void
+}
+
+attributes #0 = { nounwind "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "unsafe-fp-math"="false" "use-soft-float"="false" }
+
+; When i128 was a legal type this program generated cannot select errors:
+
+; FUNC-LABEL: {{^}}"i128-const-store":
+; FIXME: We should be able to to this with one store instruction
+; EG: STORE_RAW
+; EG: STORE_RAW
+; EG: STORE_RAW
+; EG: STORE_RAW
+; CM: STORE_DWORD
+; CM: STORE_DWORD
+; CM: STORE_DWORD
+; CM: STORE_DWORD
+; SI: buffer_store_dwordx4
+define void @i128-const-store(i32 addrspace(1)* %out) {
+entry:
+  store i32 1, i32 addrspace(1)* %out, align 4
+  %arrayidx2 = getelementptr inbounds i32, i32 addrspace(1)* %out, i64 1
+  store i32 1, i32 addrspace(1)* %arrayidx2, align 4
+  %arrayidx4 = getelementptr inbounds i32, i32 addrspace(1)* %out, i64 2
+  store i32 2, i32 addrspace(1)* %arrayidx4, align 4
+  %arrayidx6 = getelementptr inbounds i32, i32 addrspace(1)* %out, i64 3
+  store i32 2, i32 addrspace(1)* %arrayidx6, align 4
+  ret void
+}
diff --git a/test/CodeGen/R600/store.r600.ll b/test/CodeGen/AMDGPU/store.r600.ll
similarity index 100%
rename from test/CodeGen/R600/store.r600.ll
rename to test/CodeGen/AMDGPU/store.r600.ll
diff --git a/test/CodeGen/R600/structurize.ll b/test/CodeGen/AMDGPU/structurize.ll
similarity index 100%
rename from test/CodeGen/R600/structurize.ll
rename to test/CodeGen/AMDGPU/structurize.ll
diff --git a/test/CodeGen/R600/structurize1.ll b/test/CodeGen/AMDGPU/structurize1.ll
similarity index 100%
rename from test/CodeGen/R600/structurize1.ll
rename to test/CodeGen/AMDGPU/structurize1.ll
diff --git a/test/CodeGen/R600/sub.ll b/test/CodeGen/AMDGPU/sub.ll
similarity index 91%
rename from test/CodeGen/R600/sub.ll
rename to test/CodeGen/AMDGPU/sub.ll
index 03303f595de..b7fba0efa5b 100644
--- a/test/CodeGen/R600/sub.ll
+++ b/test/CodeGen/AMDGPU/sub.ll
@@ -58,11 +58,13 @@ define void @test_sub_v4i32(<4 x i32> addrspace(1)* %out, <4 x i32> addrspace(1)
 ; SI: s_sub_u32
 ; SI: s_subb_u32
 
-; EG-DAG: SETGE_UINT
-; EG-DAG: CNDE_INT
-; EG-DAG: SUB_INT
-; EG-DAG: SUB_INT
+; EG: MEM_RAT_CACHELESS STORE_RAW [[LO:T[0-9]+\.[XYZW]]]
+; EG: MEM_RAT_CACHELESS STORE_RAW [[HI:T[0-9]+\.[XYZW]]]
+; EG-DAG: SUB_INT {{[* ]*}}[[LO]]
+; EG-DAG: SUBB_UINT
 ; EG-DAG: SUB_INT
+; EG-DAG: SUB_INT {{[* ]*}}[[HI]]
+; EG-NOT: SUB
 define void @s_sub_i64(i64 addrspace(1)* noalias %out, i64 %a, i64 %b) nounwind {
   %result = sub i64 %a, %b
   store i64 %result, i64 addrspace(1)* %out, align 8
@@ -73,11 +75,13 @@ define void @s_sub_i64(i64 addrspace(1)* noalias %out, i64 %a, i64 %b) nounwind
 ; SI: v_sub_i32_e32
 ; SI: v_subb_u32_e32
 
-; EG-DAG: SETGE_UINT
-; EG-DAG: CNDE_INT
-; EG-DAG: SUB_INT
-; EG-DAG: SUB_INT
+; EG: MEM_RAT_CACHELESS STORE_RAW [[LO:T[0-9]+\.[XYZW]]]
+; EG: MEM_RAT_CACHELESS STORE_RAW [[HI:T[0-9]+\.[XYZW]]]
+; EG-DAG: SUB_INT {{[* ]*}}[[LO]]
+; EG-DAG: SUBB_UINT
 ; EG-DAG: SUB_INT
+; EG-DAG: SUB_INT {{[* ]*}}[[HI]]
+; EG-NOT: SUB
 define void @v_sub_i64(i64 addrspace(1)* noalias %out, i64 addrspace(1)* noalias %inA, i64 addrspace(1)* noalias %inB) nounwind {
   %tid = call i32 @llvm.r600.read.tidig.x() readnone
   %a_ptr = getelementptr i64, i64 addrspace(1)* %inA, i32 %tid
diff --git a/test/CodeGen/R600/subreg-coalescer-crash.ll b/test/CodeGen/AMDGPU/subreg-coalescer-crash.ll
similarity index 100%
rename from test/CodeGen/R600/subreg-coalescer-crash.ll
rename to test/CodeGen/AMDGPU/subreg-coalescer-crash.ll
diff --git a/test/CodeGen/AMDGPU/subreg-coalescer-undef-use.ll b/test/CodeGen/AMDGPU/subreg-coalescer-undef-use.ll
new file mode 100644
index 00000000000..ac9bedb2f8b
--- /dev/null
+++ b/test/CodeGen/AMDGPU/subreg-coalescer-undef-use.ll
@@ -0,0 +1,37 @@
+; RUN: llc -march=amdgcn -mcpu=SI -o - %s | FileCheck %s
+; Don't crash when the use of an undefined value is only detected by the
+; register coalescer because it is hidden with subregister insert/extract.
+target triple="amdgcn--"
+
+; CHECK-LABEL: foobar:
+; CHECK: s_load_dword s2, s[0:1], 0x9
+; CHECK-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0xb
+; CHECK-NEXT: s_waitcnt lgkmcnt(0)
+; CHECK-NEXT: v_mov_b32_e32 v0, s2
+; CHECK-NEXT: s_and_saveexec_b64 s[2:3], s[0:1]
+; CHECK-NEXT: s_xor_b64 s[2:3], exec, s[2:3]
+; BB0_1:
+; CHECK: s_load_dword s6, s[0:1], 0xa
+; CHECK-NEXT: s_waitcnt lgkmcnt(0)
+; CHECK-NEXT: v_mov_b32_e32 v0, s6
+; BB0_2:
+; CHECK: s_or_b64 exec, exec, s[2:3]
+; CHECK-NEXT: s_mov_b32 s7, 0xf000
+; CHECK-NEXT: s_mov_b32 s6, -1
+; CHECK-NEXT: buffer_store_dword v1, s[4:7], 0
+; CHECK-NEXT: s_endpgm
+define void @foobar(float %a0, float %a1, float addrspace(1)* %out) nounwind {
+entry:
+  %v0 = insertelement <4 x float> undef, float %a0, i32 0
+  br i1 undef, label %ift, label %ife
+
+ift:
+  %v1 = insertelement <4 x float> undef, float %a1, i32 0
+  br label %ife
+
+ife:
+  %val = phi <4 x float> [ %v1, %ift ], [ %v0, %entry ]
+  %v2 = extractelement <4 x float> %val, i32 1
+  store float %v2, float addrspace(1)* %out, align 4
+  ret void
+}
diff --git a/test/CodeGen/AMDGPU/subreg-eliminate-dead.ll b/test/CodeGen/AMDGPU/subreg-eliminate-dead.ll
new file mode 100644
index 00000000000..8bd995a8ecb
--- /dev/null
+++ b/test/CodeGen/AMDGPU/subreg-eliminate-dead.ll
@@ -0,0 +1,19 @@
+; RUN: llc -mtriple=amdgcn-- -verify-machineinstrs -o - %s | FileCheck %s
+; LiveRangeEdit::eliminateDeadDef did not update LiveInterval sub ranges
+; properly.
+
+; Just make sure this test doesn't crash.
+; CHECK-LABEL: foobar:
+; CHECK: s_endpgm
+define void @foobar() {
+  %v0 = icmp eq <4 x i32> undef, <i32 0, i32 1, i32 2, i32 3>
+  %v3 = sext <4 x i1> %v0 to <4 x i32>
+  %v4 = extractelement <4 x i32> %v3, i32 1
+  %v5 = icmp ne i32 %v4, 0
+  %v6 = select i1 %v5, i32 undef, i32 0
+  %v15 = insertelement <2 x i32> undef, i32 %v6, i32 1
+  store <2 x i32> %v15, <2 x i32> addrspace(1)* undef, align 8
+  ret void
+}
+
+declare double @llvm.fma.f64(double, double, double)
diff --git a/test/CodeGen/R600/swizzle-export.ll b/test/CodeGen/AMDGPU/swizzle-export.ll
similarity index 100%
rename from test/CodeGen/R600/swizzle-export.ll
rename to test/CodeGen/AMDGPU/swizzle-export.ll
diff --git a/test/CodeGen/R600/tex-clause-antidep.ll b/test/CodeGen/AMDGPU/tex-clause-antidep.ll
similarity index 100%
rename from test/CodeGen/R600/tex-clause-antidep.ll
rename to test/CodeGen/AMDGPU/tex-clause-antidep.ll
diff --git a/test/CodeGen/R600/texture-input-merge.ll b/test/CodeGen/AMDGPU/texture-input-merge.ll
similarity index 100%
rename from test/CodeGen/R600/texture-input-merge.ll
rename to test/CodeGen/AMDGPU/texture-input-merge.ll
diff --git a/test/CodeGen/R600/trunc-cmp-constant.ll b/test/CodeGen/AMDGPU/trunc-cmp-constant.ll
similarity index 100%
rename from test/CodeGen/R600/trunc-cmp-constant.ll
rename to test/CodeGen/AMDGPU/trunc-cmp-constant.ll
diff --git a/test/CodeGen/AMDGPU/trunc-store-f64-to-f16.ll b/test/CodeGen/AMDGPU/trunc-store-f64-to-f16.ll
new file mode 100644
index 00000000000..c29872beef8
--- /dev/null
+++ b/test/CodeGen/AMDGPU/trunc-store-f64-to-f16.ll
@@ -0,0 +1,56 @@
+; XFAIL: *
+; RUN: llc -march=amdgcn -mcpu=SI < %s
+
+; GCN-LABEL: {{^}}global_truncstore_f64_to_f16:
+; GCN: s_endpgm
+define void @global_truncstore_f64_to_f16(half addrspace(1)* %out, double addrspace(1)* %in) #0 {
+  %val = load double, double addrspace(1)* %in
+  %cvt = fptrunc double %val to half
+  store half %cvt, half addrspace(1)* %out
+  ret void
+}
+
+; GCN-LABEL: {{^}}global_truncstore_v2f64_to_v2f16:
+; GCN: s_endpgm
+define void @global_truncstore_v2f64_to_v2f16(<2 x half> addrspace(1)* %out, <2 x double> addrspace(1)* %in) #0 {
+  %val = load <2 x double>, <2 x double> addrspace(1)* %in
+  %cvt = fptrunc <2 x double> %val to <2 x half>
+  store <2 x half> %cvt, <2 x half> addrspace(1)* %out
+  ret void
+}
+
+; GCN-LABEL: {{^}}global_truncstore_v3f64_to_v3f16:
+; GCN: s_endpgm
+define void @global_truncstore_v3f64_to_v3f16(<3 x half> addrspace(1)* %out, <3 x double> addrspace(1)* %in) #0 {
+  %val = load <3 x double>, <3 x double> addrspace(1)* %in
+  %cvt = fptrunc <3 x double> %val to <3 x half>
+  store <3 x half> %cvt, <3 x half> addrspace(1)* %out
+  ret void
+}
+
+; GCN-LABEL: {{^}}global_truncstore_v4f64_to_v4f16:
+; GCN: s_endpgm
+define void @global_truncstore_v4f64_to_v4f16(<4 x half> addrspace(1)* %out, <4 x double> addrspace(1)* %in) #0 {
+  %val = load <4 x double>, <4 x double> addrspace(1)* %in
+  %cvt = fptrunc <4 x double> %val to <4 x half>
+  store <4 x half> %cvt, <4 x half> addrspace(1)* %out
+  ret void
+}
+
+; GCN-LABEL: {{^}}global_truncstore_v8f64_to_v8f16:
+; GCN: s_endpgm
+define void @global_truncstore_v8f64_to_v8f16(<8 x half> addrspace(1)* %out, <8 x double> addrspace(1)* %in) #0 {
+  %val = load <8 x double>, <8 x double> addrspace(1)* %in
+  %cvt = fptrunc <8 x double> %val to <8 x half>
+  store <8 x half> %cvt, <8 x half> addrspace(1)* %out
+  ret void
+}
+
+; GCN-LABEL: {{^}}global_truncstore_v16f64_to_v16f16:
+; GCN: s_endpgm
+define void @global_truncstore_v16f64_to_v16f16(<16 x half> addrspace(1)* %out, <16 x double> addrspace(1)* %in) #0 {
+  %val = load <16 x double>, <16 x double> addrspace(1)* %in
+  %cvt = fptrunc <16 x double> %val to <16 x half>
+  store <16 x half> %cvt, <16 x half> addrspace(1)* %out
+  ret void
+}
diff --git a/test/CodeGen/R600/trunc-store-i1.ll b/test/CodeGen/AMDGPU/trunc-store-i1.ll
similarity index 100%
rename from test/CodeGen/R600/trunc-store-i1.ll
rename to test/CodeGen/AMDGPU/trunc-store-i1.ll
diff --git a/test/CodeGen/AMDGPU/trunc-store.ll b/test/CodeGen/AMDGPU/trunc-store.ll
new file mode 100644
index 00000000000..4ba815f2669
--- /dev/null
+++ b/test/CodeGen/AMDGPU/trunc-store.ll
@@ -0,0 +1,48 @@
+; RUN: llc -march=amdgcn -mcpu=verde -verify-machineinstrs < %s | FileCheck -check-prefix=SI -check-prefix=FUNC %s
+; RUN: llc -march=amdgcn -mcpu=tonga -verify-machineinstrs < %s | FileCheck -check-prefix=SI -check-prefix=FUNC %s
+
+; FUNC-LABEL: {{^}}truncstore_arg_v16i32_to_v16i8:
+; SI: buffer_store_byte
+; SI: buffer_store_byte
+; SI: buffer_store_byte
+; SI: buffer_store_byte
+; SI: buffer_store_byte
+; SI: buffer_store_byte
+; SI: buffer_store_byte
+; SI: buffer_store_byte
+; SI: buffer_store_byte
+; SI: buffer_store_byte
+; SI: buffer_store_byte
+; SI: buffer_store_byte
+; SI: buffer_store_byte
+; SI: buffer_store_byte
+; SI: buffer_store_byte
+; SI: buffer_store_byte
+define void @truncstore_arg_v16i32_to_v16i8(<16 x i8> addrspace(1)* %out, <16 x i32> %in) {
+  %trunc = trunc <16 x i32> %in to <16 x i8>
+  store <16 x i8> %trunc, <16 x i8> addrspace(1)* %out
+  ret void
+}
+
+; FUNC-LABEL: {{^}}truncstore_arg_v16i64_to_v16i8:
+; SI: buffer_store_byte
+; SI: buffer_store_byte
+; SI: buffer_store_byte
+; SI: buffer_store_byte
+; SI: buffer_store_byte
+; SI: buffer_store_byte
+; SI: buffer_store_byte
+; SI: buffer_store_byte
+; SI: buffer_store_byte
+; SI: buffer_store_byte
+; SI: buffer_store_byte
+; SI: buffer_store_byte
+; SI: buffer_store_byte
+; SI: buffer_store_byte
+; SI: buffer_store_byte
+; SI: buffer_store_byte
+define void @truncstore_arg_v16i64_to_v16i8(<16 x i8> addrspace(1)* %out, <16 x i64> %in) {
+  %trunc = trunc <16 x i64> %in to <16 x i8>
+  store <16 x i8> %trunc, <16 x i8> addrspace(1)* %out
+  ret void
+}
diff --git a/test/CodeGen/R600/trunc-vector-store-assertion-failure.ll b/test/CodeGen/AMDGPU/trunc-vector-store-assertion-failure.ll
similarity index 100%
rename from test/CodeGen/R600/trunc-vector-store-assertion-failure.ll
rename to test/CodeGen/AMDGPU/trunc-vector-store-assertion-failure.ll
diff --git a/test/CodeGen/R600/trunc.ll b/test/CodeGen/AMDGPU/trunc.ll
similarity index 100%
rename from test/CodeGen/R600/trunc.ll
rename to test/CodeGen/AMDGPU/trunc.ll
diff --git a/test/CodeGen/R600/tti-unroll-prefs.ll b/test/CodeGen/AMDGPU/tti-unroll-prefs.ll
similarity index 100%
rename from test/CodeGen/R600/tti-unroll-prefs.ll
rename to test/CodeGen/AMDGPU/tti-unroll-prefs.ll
diff --git a/test/CodeGen/R600/uaddo.ll b/test/CodeGen/AMDGPU/uaddo.ll
similarity index 91%
rename from test/CodeGen/R600/uaddo.ll
rename to test/CodeGen/AMDGPU/uaddo.ll
index 9f383654b70..11438f267ad 100644
--- a/test/CodeGen/R600/uaddo.ll
+++ b/test/CodeGen/AMDGPU/uaddo.ll
@@ -1,6 +1,6 @@
 ; RUN: llc -march=amdgcn -mcpu=SI -verify-machineinstrs< %s | FileCheck -check-prefix=SI -check-prefix=FUNC %s
 ; RUN: llc -march=amdgcn -mcpu=tonga -verify-machineinstrs< %s | FileCheck -check-prefix=SI -check-prefix=FUNC %s
-; RUN: llc -march=r600 -mcpu=cypress -verify-machineinstrs< %s
+; RUN: llc -march=r600 -mcpu=cypress -verify-machineinstrs< %s | FileCheck -check-prefix=EG -check-prefix=FUNC %s
 
 declare { i32, i1 } @llvm.uadd.with.overflow.i32(i32, i32) nounwind readnone
 declare { i64, i1 } @llvm.uadd.with.overflow.i64(i64, i64) nounwind readnone
@@ -9,6 +9,9 @@ declare { i64, i1 } @llvm.uadd.with.overflow.i64(i64, i64) nounwind readnone
 ; SI: add
 ; SI: addc
 ; SI: addc
+
+; EG: ADDC_UINT
+; EG: ADDC_UINT
 define void @uaddo_i64_zext(i64 addrspace(1)* %out, i64 %a, i64 %b) nounwind {
   %uadd = call { i64, i1 } @llvm.uadd.with.overflow.i64(i64 %a, i64 %b) nounwind
   %val = extractvalue { i64, i1 } %uadd, 0
@@ -21,6 +24,9 @@ define void @uaddo_i64_zext(i64 addrspace(1)* %out, i64 %a, i64 %b) nounwind {
 
 ; FUNC-LABEL: {{^}}s_uaddo_i32:
 ; SI: s_add_i32
+
+; EG: ADDC_UINT
+; EG: ADD_INT
 define void @s_uaddo_i32(i32 addrspace(1)* %out, i1 addrspace(1)* %carryout, i32 %a, i32 %b) nounwind {
   %uadd = call { i32, i1 } @llvm.uadd.with.overflow.i32(i32 %a, i32 %b) nounwind
   %val = extractvalue { i32, i1 } %uadd, 0
@@ -32,6 +38,9 @@ define void @s_uaddo_i32(i32 addrspace(1)* %out, i1 addrspace(1)* %carryout, i32
 
 ; FUNC-LABEL: {{^}}v_uaddo_i32:
 ; SI: v_add_i32
+
+; EG: ADDC_UINT
+; EG: ADD_INT
 define void @v_uaddo_i32(i32 addrspace(1)* %out, i1 addrspace(1)* %carryout, i32 addrspace(1)* %aptr, i32 addrspace(1)* %bptr) nounwind {
   %a = load i32, i32 addrspace(1)* %aptr, align 4
   %b = load i32, i32 addrspace(1)* %bptr, align 4
@@ -46,6 +55,9 @@ define void @v_uaddo_i32(i32 addrspace(1)* %out, i1 addrspace(1)* %carryout, i32
 ; FUNC-LABEL: {{^}}s_uaddo_i64:
 ; SI: s_add_u32
 ; SI: s_addc_u32
+
+; EG: ADDC_UINT
+; EG: ADD_INT
 define void @s_uaddo_i64(i64 addrspace(1)* %out, i1 addrspace(1)* %carryout, i64 %a, i64 %b) nounwind {
   %uadd = call { i64, i1 } @llvm.uadd.with.overflow.i64(i64 %a, i64 %b) nounwind
   %val = extractvalue { i64, i1 } %uadd, 0
@@ -58,6 +70,9 @@ define void @s_uaddo_i64(i64 addrspace(1)* %out, i1 addrspace(1)* %carryout, i64
 ; FUNC-LABEL: {{^}}v_uaddo_i64:
 ; SI: v_add_i32
 ; SI: v_addc_u32
+
+; EG: ADDC_UINT
+; EG: ADD_INT
 define void @v_uaddo_i64(i64 addrspace(1)* %out, i1 addrspace(1)* %carryout, i64 addrspace(1)* %aptr, i64 addrspace(1)* %bptr) nounwind {
   %a = load i64, i64 addrspace(1)* %aptr, align 4
   %b = load i64, i64 addrspace(1)* %bptr, align 4
diff --git a/test/CodeGen/R600/udiv.ll b/test/CodeGen/AMDGPU/udiv.ll
similarity index 100%
rename from test/CodeGen/R600/udiv.ll
rename to test/CodeGen/AMDGPU/udiv.ll
diff --git a/test/CodeGen/R600/udivrem.ll b/test/CodeGen/AMDGPU/udivrem.ll
similarity index 100%
rename from test/CodeGen/R600/udivrem.ll
rename to test/CodeGen/AMDGPU/udivrem.ll
diff --git a/test/CodeGen/R600/udivrem24.ll b/test/CodeGen/AMDGPU/udivrem24.ll
similarity index 100%
rename from test/CodeGen/R600/udivrem24.ll
rename to test/CodeGen/AMDGPU/udivrem24.ll
diff --git a/test/CodeGen/R600/udivrem64.ll b/test/CodeGen/AMDGPU/udivrem64.ll
similarity index 100%
rename from test/CodeGen/R600/udivrem64.ll
rename to test/CodeGen/AMDGPU/udivrem64.ll
diff --git a/test/CodeGen/R600/uint_to_fp.f64.ll b/test/CodeGen/AMDGPU/uint_to_fp.f64.ll
similarity index 92%
rename from test/CodeGen/R600/uint_to_fp.f64.ll
rename to test/CodeGen/AMDGPU/uint_to_fp.f64.ll
index dfec8eb15cb..6f608df5e9f 100644
--- a/test/CodeGen/R600/uint_to_fp.f64.ll
+++ b/test/CodeGen/AMDGPU/uint_to_fp.f64.ll
@@ -72,11 +72,11 @@ define void @s_uint_to_fp_v4i32_to_v4f64(<4 x double> addrspace(1)* %out, <4 x i
 
 ; FIXME: select on 0, 0
 ; SI-LABEL: {{^}}uint_to_fp_i1_to_f64:
-; SI: v_cmp_eq_i32_e64 [[CMP:s\[[0-9]+:[0-9]\]]],
-; We can't fold the SGPRs into v_cndmask_b32_e64, because it already
-; uses an SGPR for [[CMP]]
-; SI: v_cndmask_b32_e64 v{{[0-9]+}}, 0, v{{[0-9]+}}, [[CMP]]
-; SI: v_cndmask_b32_e64 v{{[0-9]+}}, 0, 0, [[CMP]]
+; SI: v_cmp_eq_i32_e64 vcc
+; We can't fold the SGPRs into v_cndmask_b32_e32, because it already
+; uses an SGPR (implicit vcc).
+; SI: v_cndmask_b32_e32 v{{[0-9]+}}, 0, v{{[0-9]+}}
+; SI: v_cndmask_b32_e64 v{{[0-9]+}}, 0, 0, vcc
 ; SI: buffer_store_dwordx2
 ; SI: s_endpgm
 define void @uint_to_fp_i1_to_f64(double addrspace(1)* %out, i32 %in) {
diff --git a/test/CodeGen/R600/uint_to_fp.ll b/test/CodeGen/AMDGPU/uint_to_fp.ll
similarity index 100%
rename from test/CodeGen/R600/uint_to_fp.ll
rename to test/CodeGen/AMDGPU/uint_to_fp.ll
diff --git a/test/CodeGen/R600/unaligned-load-store.ll b/test/CodeGen/AMDGPU/unaligned-load-store.ll
similarity index 100%
rename from test/CodeGen/R600/unaligned-load-store.ll
rename to test/CodeGen/AMDGPU/unaligned-load-store.ll
diff --git a/test/CodeGen/R600/unhandled-loop-condition-assertion.ll b/test/CodeGen/AMDGPU/unhandled-loop-condition-assertion.ll
similarity index 100%
rename from test/CodeGen/R600/unhandled-loop-condition-assertion.ll
rename to test/CodeGen/AMDGPU/unhandled-loop-condition-assertion.ll
diff --git a/test/CodeGen/AMDGPU/unroll.ll b/test/CodeGen/AMDGPU/unroll.ll
new file mode 100644
index 00000000000..411a15a4b83
--- /dev/null
+++ b/test/CodeGen/AMDGPU/unroll.ll
@@ -0,0 +1,36 @@
+; RUN: opt -mtriple=amdgcn-- -loop-unroll -simplifycfg -sroa %s -S -o - | FileCheck %s
+; RUN: opt -mtriple=r600-- -loop-unroll -simplifycfg -sroa %s -S -o - | FileCheck %s
+
+
+; This test contains a simple loop that initializes an array declared in
+; private memory.  We want to make sure these kinds of loops are always
+; unrolled, because private memory is slow.
+
+; CHECK-LABEL: @test
+; CHECK-NOT: alloca
+; CHECK: store i32 5, i32 addrspace(1)* %out
+define void @test(i32 addrspace(1)* %out) {
+entry:
+  %0 = alloca [32 x i32]
+  br label %loop.header
+
+loop.header:
+  %counter = phi i32 [0, %entry], [%inc, %loop.inc]
+  br label %loop.body
+
+loop.body:
+  %ptr = getelementptr [32 x i32], [32 x i32]* %0, i32 0, i32 %counter
+  store i32 %counter, i32* %ptr
+  br label %loop.inc
+
+loop.inc:
+  %inc = add i32 %counter, 1
+  %1 = icmp sge i32 %counter, 32
+  br i1 %1, label  %exit, label %loop.header
+
+exit:
+  %2 = getelementptr [32 x i32], [32 x i32]* %0, i32 0, i32 5
+  %3 = load i32, i32* %2
+  store i32 %3, i32 addrspace(1)* %out
+  ret void
+}
diff --git a/test/CodeGen/R600/unsupported-cc.ll b/test/CodeGen/AMDGPU/unsupported-cc.ll
similarity index 100%
rename from test/CodeGen/R600/unsupported-cc.ll
rename to test/CodeGen/AMDGPU/unsupported-cc.ll
diff --git a/test/CodeGen/R600/urecip.ll b/test/CodeGen/AMDGPU/urecip.ll
similarity index 100%
rename from test/CodeGen/R600/urecip.ll
rename to test/CodeGen/AMDGPU/urecip.ll
diff --git a/test/CodeGen/R600/urem.ll b/test/CodeGen/AMDGPU/urem.ll
similarity index 100%
rename from test/CodeGen/R600/urem.ll
rename to test/CodeGen/AMDGPU/urem.ll
diff --git a/test/CodeGen/R600/use-sgpr-multiple-times.ll b/test/CodeGen/AMDGPU/use-sgpr-multiple-times.ll
similarity index 100%
rename from test/CodeGen/R600/use-sgpr-multiple-times.ll
rename to test/CodeGen/AMDGPU/use-sgpr-multiple-times.ll
diff --git a/test/CodeGen/R600/usubo.ll b/test/CodeGen/AMDGPU/usubo.ll
similarity index 88%
rename from test/CodeGen/R600/usubo.ll
rename to test/CodeGen/AMDGPU/usubo.ll
index a753ca4e979..3c9b1622a07 100644
--- a/test/CodeGen/R600/usubo.ll
+++ b/test/CodeGen/AMDGPU/usubo.ll
@@ -1,11 +1,14 @@
 ; RUN: llc -march=amdgcn -mcpu=SI -verify-machineinstrs< %s | FileCheck -check-prefix=SI -check-prefix=FUNC %s
 ; RUN: llc -march=amdgcn -mcpu=tonga -verify-machineinstrs< %s | FileCheck -check-prefix=SI -check-prefix=FUNC %s
-; RUN: llc -march=r600 -mcpu=cypress -verify-machineinstrs< %s
+; RUN: llc -march=r600 -mcpu=cypress -verify-machineinstrs< %s | FileCheck -check-prefix=EG -check-prefix=FUNC %s
 
 declare { i32, i1 } @llvm.usub.with.overflow.i32(i32, i32) nounwind readnone
 declare { i64, i1 } @llvm.usub.with.overflow.i64(i64, i64) nounwind readnone
 
 ; FUNC-LABEL: {{^}}usubo_i64_zext:
+
+; EG: SUBB_UINT
+; EG: ADDC_UINT
 define void @usubo_i64_zext(i64 addrspace(1)* %out, i64 %a, i64 %b) nounwind {
   %usub = call { i64, i1 } @llvm.usub.with.overflow.i64(i64 %a, i64 %b) nounwind
   %val = extractvalue { i64, i1 } %usub, 0
@@ -18,6 +21,9 @@ define void @usubo_i64_zext(i64 addrspace(1)* %out, i64 %a, i64 %b) nounwind {
 
 ; FUNC-LABEL: {{^}}s_usubo_i32:
 ; SI: s_sub_i32
+
+; EG-DAG: SUBB_UINT
+; EG-DAG: SUB_INT
 define void @s_usubo_i32(i32 addrspace(1)* %out, i1 addrspace(1)* %carryout, i32 %a, i32 %b) nounwind {
   %usub = call { i32, i1 } @llvm.usub.with.overflow.i32(i32 %a, i32 %b) nounwind
   %val = extractvalue { i32, i1 } %usub, 0
@@ -29,6 +35,9 @@ define void @s_usubo_i32(i32 addrspace(1)* %out, i1 addrspace(1)* %carryout, i32
 
 ; FUNC-LABEL: {{^}}v_usubo_i32:
 ; SI: v_subrev_i32_e32
+
+; EG-DAG: SUBB_UINT
+; EG-DAG: SUB_INT
 define void @v_usubo_i32(i32 addrspace(1)* %out, i1 addrspace(1)* %carryout, i32 addrspace(1)* %aptr, i32 addrspace(1)* %bptr) nounwind {
   %a = load i32, i32 addrspace(1)* %aptr, align 4
   %b = load i32, i32 addrspace(1)* %bptr, align 4
@@ -43,6 +52,11 @@ define void @v_usubo_i32(i32 addrspace(1)* %out, i1 addrspace(1)* %carryout, i32
 ; FUNC-LABEL: {{^}}s_usubo_i64:
 ; SI: s_sub_u32
 ; SI: s_subb_u32
+
+; EG-DAG: SUBB_UINT
+; EG-DAG: SUB_INT
+; EG-DAG: SUB_INT
+; EG: SUB_INT
 define void @s_usubo_i64(i64 addrspace(1)* %out, i1 addrspace(1)* %carryout, i64 %a, i64 %b) nounwind {
   %usub = call { i64, i1 } @llvm.usub.with.overflow.i64(i64 %a, i64 %b) nounwind
   %val = extractvalue { i64, i1 } %usub, 0
@@ -55,6 +69,11 @@ define void @s_usubo_i64(i64 addrspace(1)* %out, i1 addrspace(1)* %carryout, i64
 ; FUNC-LABEL: {{^}}v_usubo_i64:
 ; SI: v_sub_i32
 ; SI: v_subb_u32
+
+; EG-DAG: SUBB_UINT
+; EG-DAG: SUB_INT
+; EG-DAG: SUB_INT
+; EG: SUB_INT
 define void @v_usubo_i64(i64 addrspace(1)* %out, i1 addrspace(1)* %carryout, i64 addrspace(1)* %aptr, i64 addrspace(1)* %bptr) nounwind {
   %a = load i64, i64 addrspace(1)* %aptr, align 4
   %b = load i64, i64 addrspace(1)* %bptr, align 4
diff --git a/test/CodeGen/R600/v1i64-kernel-arg.ll b/test/CodeGen/AMDGPU/v1i64-kernel-arg.ll
similarity index 100%
rename from test/CodeGen/R600/v1i64-kernel-arg.ll
rename to test/CodeGen/AMDGPU/v1i64-kernel-arg.ll
diff --git a/test/CodeGen/R600/v_cndmask.ll b/test/CodeGen/AMDGPU/v_cndmask.ll
similarity index 100%
rename from test/CodeGen/R600/v_cndmask.ll
rename to test/CodeGen/AMDGPU/v_cndmask.ll
diff --git a/test/CodeGen/AMDGPU/v_mac.ll b/test/CodeGen/AMDGPU/v_mac.ll
new file mode 100644
index 00000000000..a4eaec3403c
--- /dev/null
+++ b/test/CodeGen/AMDGPU/v_mac.ll
@@ -0,0 +1,155 @@
+; RUN: llc -march=amdgcn -verify-machineinstrs < %s | FileCheck -check-prefix=SI -check-prefix=GCN %s
+; RUN: llc -march=amdgcn -mcpu=tonga -verify-machineinstrs < %s | FileCheck -check-prefix=VI -check-prefix=GCN %s
+
+; GCN-LABEL: {{^}}mac_vvv:
+; GCN: buffer_load_dword [[A:v[0-9]+]], s[{{[0-9]+:[0-9]+}}], 0{{$}}
+; GCN: buffer_load_dword [[B:v[0-9]+]], s[{{[0-9]+:[0-9]+}}], 0 offset:4
+; GCN: buffer_load_dword [[C:v[0-9]+]], s[{{[0-9]+:[0-9]+}}], 0 offset:8
+; GCN: v_mac_f32_e32 [[C]], [[B]], [[A]]
+; GCN: buffer_store_dword [[C]]
+define void @mac_vvv(float addrspace(1)* %out, float addrspace(1)* %in) {
+entry:
+  %b_ptr = getelementptr float, float addrspace(1)* %in, i32 1
+  %c_ptr = getelementptr float, float addrspace(1)* %in, i32 2
+
+  %a = load float, float addrspace(1)* %in
+  %b = load float, float addrspace(1)* %b_ptr
+  %c = load float, float addrspace(1)* %c_ptr
+
+  %tmp0 = fmul float %a, %b
+  %tmp1 = fadd float %tmp0, %c
+  store float %tmp1, float addrspace(1)* %out
+  ret void
+}
+
+; GCN-LABEL: {{^}}mad_inline_sgpr_inline:
+; GCN-NOT: v_mac_f32
+; GCN: v_mad_f32 v{{[0-9]}}, 0.5, s{{[0-9]+}}, 0.5
+define void @mad_inline_sgpr_inline(float addrspace(1)* %out, float %in) {
+entry:
+  %tmp0 = fmul float 0.5, %in
+  %tmp1 = fadd float %tmp0, 0.5
+  store float %tmp1, float addrspace(1)* %out
+  ret void
+}
+
+; GCN-LABEL: {{^}}mad_vvs:
+; GCN-NOT: v_mac_f32
+; GCN: v_mad_f32 v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}}, s{{[0-9]+}}
+define void @mad_vvs(float addrspace(1)* %out, float addrspace(1)* %in, float %c) {
+entry:
+  %b_ptr = getelementptr float, float addrspace(1)* %in, i32 1
+
+  %a = load float, float addrspace(1)* %in
+  %b = load float, float addrspace(1)* %b_ptr
+
+  %tmp0 = fmul float %a, %b
+  %tmp1 = fadd float %tmp0, %c
+  store float %tmp1, float addrspace(1)* %out
+  ret void
+}
+
+; GCN-LABEL: {{^}}mac_ssv:
+; GCN: v_mac_f32_e64 v{{[0-9]+}}, s{{[0-9]+}}, s{{[0-9]+}}
+define void @mac_ssv(float addrspace(1)* %out, float addrspace(1)* %in, float %a) {
+entry:
+  %c = load float, float addrspace(1)* %in
+
+  %tmp0 = fmul float %a, %a
+  %tmp1 = fadd float %tmp0, %c
+  store float %tmp1, float addrspace(1)* %out
+  ret void
+}
+
+; GCN-LABEL: {{^}}mac_mad_same_add:
+; GCN: v_mad_f32 v{{[0-9]}}, v{{[0-9]+}}, v{{[0-9]+}}, [[ADD:v[0-9]+]]
+; GCN: v_mac_f32_e32 [[ADD]], v{{[0-9]+}}, v{{[0-9]+}}
+define void @mac_mad_same_add(float addrspace(1)* %out, float addrspace(1)* %in) {
+entry:
+  %b_ptr = getelementptr float, float addrspace(1)* %in, i32 1
+  %c_ptr = getelementptr float, float addrspace(1)* %in, i32 2
+  %d_ptr = getelementptr float, float addrspace(1)* %in, i32 3
+  %e_ptr = getelementptr float, float addrspace(1)* %in, i32 4
+
+  %a = load float, float addrspace(1)* %in
+  %b = load float, float addrspace(1)* %b_ptr
+  %c = load float, float addrspace(1)* %c_ptr
+  %d = load float, float addrspace(1)* %d_ptr
+  %e = load float, float addrspace(1)* %e_ptr
+
+  %tmp0 = fmul float %a, %b
+  %tmp1 = fadd float %tmp0, %c
+
+  %tmp2 = fmul float %d, %e
+  %tmp3 = fadd float %tmp2, %c
+
+  %out1 = getelementptr float, float addrspace(1)* %out, i32 1
+  store float %tmp1, float addrspace(1)* %out
+  store float %tmp3, float addrspace(1)* %out1
+  ret void
+}
+
+; There is no advantage to using v_mac when one of the operands is negated
+; and v_mad accepts more operand types.
+
+; GCN-LABEL: {{^}}mad_neg_src0:
+; GCN-NOT: v_mac_f32
+; GCN: v_mad_f32 v{{[0-9]+}}, -v{{[0-9]+}}, v{{[0-9]+}}, v{{[-0-9]}}
+define void @mad_neg_src0(float addrspace(1)* %out, float addrspace(1)* %in) #0 {
+entry:
+  %b_ptr = getelementptr float, float addrspace(1)* %in, i32 1
+  %c_ptr = getelementptr float, float addrspace(1)* %in, i32 2
+
+  %a = load float, float addrspace(1)* %in
+  %b = load float, float addrspace(1)* %b_ptr
+  %c = load float, float addrspace(1)* %c_ptr
+
+  %neg_a = fsub float 0.0, %a
+  %tmp0 = fmul float %neg_a, %b
+  %tmp1 = fadd float %tmp0, %c
+
+  store float %tmp1, float addrspace(1)* %out
+  ret void
+}
+
+; GCN-LABEL: {{^}}mad_neg_src1:
+; GCN-NOT: v_mac_f32
+; GCN: v_mad_f32 v{{[0-9]+}}, -v{{[0-9]+}}, v{{[0-9]+}}, v{{[-0-9]}}
+define void @mad_neg_src1(float addrspace(1)* %out, float addrspace(1)* %in) #0 {
+entry:
+  %b_ptr = getelementptr float, float addrspace(1)* %in, i32 1
+  %c_ptr = getelementptr float, float addrspace(1)* %in, i32 2
+
+  %a = load float, float addrspace(1)* %in
+  %b = load float, float addrspace(1)* %b_ptr
+  %c = load float, float addrspace(1)* %c_ptr
+
+  %neg_b = fsub float 0.0, %b
+  %tmp0 = fmul float %a, %neg_b
+  %tmp1 = fadd float %tmp0, %c
+
+  store float %tmp1, float addrspace(1)* %out
+  ret void
+}
+
+; GCN-LABEL: {{^}}mad_neg_src2:
+; GCN-NOT: v_mac
+; GCN: v_mad_f32 v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}}, -v{{[-0-9]}}
+define void @mad_neg_src2(float addrspace(1)* %out, float addrspace(1)* %in) #0 {
+entry:
+  %b_ptr = getelementptr float, float addrspace(1)* %in, i32 1
+  %c_ptr = getelementptr float, float addrspace(1)* %in, i32 2
+
+  %a = load float, float addrspace(1)* %in
+  %b = load float, float addrspace(1)* %b_ptr
+  %c = load float, float addrspace(1)* %c_ptr
+
+  %neg_c = fsub float 0.0, %c
+  %tmp0 = fmul float %a, %b
+  %tmp1 = fadd float %tmp0, %neg_c
+
+  store float %tmp1, float addrspace(1)* %out
+  ret void
+}
+
+attributes #0 = { "true" "unsafe-fp-math"="true" }
diff --git a/test/CodeGen/R600/valu-i1.ll b/test/CodeGen/AMDGPU/valu-i1.ll
similarity index 100%
rename from test/CodeGen/R600/valu-i1.ll
rename to test/CodeGen/AMDGPU/valu-i1.ll
diff --git a/test/CodeGen/R600/vector-alloca.ll b/test/CodeGen/AMDGPU/vector-alloca.ll
similarity index 100%
rename from test/CodeGen/R600/vector-alloca.ll
rename to test/CodeGen/AMDGPU/vector-alloca.ll
diff --git a/test/CodeGen/R600/vertex-fetch-encoding.ll b/test/CodeGen/AMDGPU/vertex-fetch-encoding.ll
similarity index 100%
rename from test/CodeGen/R600/vertex-fetch-encoding.ll
rename to test/CodeGen/AMDGPU/vertex-fetch-encoding.ll
diff --git a/test/CodeGen/R600/vop-shrink.ll b/test/CodeGen/AMDGPU/vop-shrink.ll
similarity index 100%
rename from test/CodeGen/R600/vop-shrink.ll
rename to test/CodeGen/AMDGPU/vop-shrink.ll
diff --git a/test/CodeGen/R600/vselect.ll b/test/CodeGen/AMDGPU/vselect.ll
similarity index 86%
rename from test/CodeGen/R600/vselect.ll
rename to test/CodeGen/AMDGPU/vselect.ll
index a3014b03d2b..dc1f1ea11b0 100644
--- a/test/CodeGen/R600/vselect.ll
+++ b/test/CodeGen/AMDGPU/vselect.ll
@@ -1,14 +1,14 @@
-;RUN: llc < %s -march=r600 -mcpu=redwood | FileCheck --check-prefix=EG %s
-;RUN: llc < %s -march=amdgcn -mcpu=SI -verify-machineinstrs | FileCheck --check-prefix=SI %s
-;RUN: llc < %s -march=amdgcn -mcpu=tonga -verify-machineinstrs | FileCheck --check-prefix=SI %s
+;RUN: llc < %s -march=r600 -mcpu=redwood | FileCheck --check-prefix=EG --check-prefix=FUNC %s
+;RUN: llc < %s -march=amdgcn -mcpu=SI -verify-machineinstrs | FileCheck --check-prefix=SI --check-prefix=FUNC %s
+;RUN: llc < %s -march=amdgcn -mcpu=tonga -verify-machineinstrs | FileCheck --check-prefix=SI --check-prefix=VI --check-prefix=FUNC %s
+
+;FUNC-LABEL: {{^}}test_select_v2i32:
 
-;EG: {{^}}test_select_v2i32:
 ;EG: CNDE_INT {{\** *}}T{{[0-9]+\.[XYZW], PV\.[XYZW], T[0-9]+\.[XYZW], T[0-9]+\.[XYZW]}}
 ;EG: CNDE_INT {{\** *}}T{{[0-9]+\.[XYZW], PV\.[XYZW], T[0-9]+\.[XYZW], T[0-9]+\.[XYZW]}}
 
-;SI: {{^}}test_select_v2i32:
-;SI: v_cndmask_b32_e64
 ;SI: v_cndmask_b32_e64
+;SI: v_cndmask_b32_e32
 
 define void @test_select_v2i32(<2 x i32> addrspace(1)* %out, <2 x i32> addrspace(1)* %in0, <2 x i32> addrspace(1)* %in1) {
 entry:
@@ -20,13 +20,13 @@ entry:
   ret void
 }
 
-;EG: {{^}}test_select_v2f32:
+;FUNC-LABEL: {{^}}test_select_v2f32:
+
 ;EG: CNDE_INT {{\** *}}T{{[0-9]+\.[XYZW], PV\.[XYZW], T[0-9]+\.[XYZW], T[0-9]+\.[XYZW]}}
 ;EG: CNDE_INT {{\** *}}T{{[0-9]+\.[XYZW], PV\.[XYZW], T[0-9]+\.[XYZW], T[0-9]+\.[XYZW]}}
 
-;SI: {{^}}test_select_v2f32:
-;SI: v_cndmask_b32_e64
 ;SI: v_cndmask_b32_e64
+;SI: v_cndmask_b32_e32
 
 define void @test_select_v2f32(<2 x float> addrspace(1)* %out, <2 x float> addrspace(1)* %in0, <2 x float> addrspace(1)* %in1) {
 entry:
@@ -38,17 +38,19 @@ entry:
   ret void
 }
 
-;EG: {{^}}test_select_v4i32:
+;FUNC-LABEL: {{^}}test_select_v4i32:
+
 ;EG: CNDE_INT {{\** *}}T{{[0-9]+\.[XYZW], PV\.[XYZW], T[0-9]+\.[XYZW], T[0-9]+\.[XYZW]}}
 ;EG: CNDE_INT {{\** *}}T{{[0-9]+\.[XYZW], T[0-9]+\.[XYZW], T[0-9]+\.[XYZW], T[0-9]+\.[XYZW]}}
 ;EG: CNDE_INT {{\** *}}T{{[0-9]+\.[XYZW], PV\.[XYZW], T[0-9]+\.[XYZW], T[0-9]+\.[XYZW]}}
 ;EG: CNDE_INT {{\** *}}T{{[0-9]+\.[XYZW], PV\.[XYZW], T[0-9]+\.[XYZW], T[0-9]+\.[XYZW]}}
 
-;SI: {{^}}test_select_v4i32:
-;SI: v_cndmask_b32_e64
-;SI: v_cndmask_b32_e64
-;SI: v_cndmask_b32_e64
-;SI: v_cndmask_b32_e64
+; FIXME: The shrinking does not happen on tonga
+
+;SI: v_cndmask_b32
+;SI: v_cndmask_b32
+;SI: v_cndmask_b32
+;SI: v_cndmask_b32
 
 define void @test_select_v4i32(<4 x i32> addrspace(1)* %out, <4 x i32> addrspace(1)* %in0, <4 x i32> addrspace(1)* %in1) {
 entry:
@@ -60,7 +62,7 @@ entry:
   ret void
 }
 
-;EG: {{^}}test_select_v4f32:
+;FUNC-LABEL: {{^}}test_select_v4f32:
 ;EG: CNDE_INT {{\** *}}T{{[0-9]+\.[XYZW], PV\.[XYZW], T[0-9]+\.[XYZW], T[0-9]+\.[XYZW]}}
 ;EG: CNDE_INT {{\** *}}T{{[0-9]+\.[XYZW], T[0-9]+\.[XYZW], T[0-9]+\.[XYZW], T[0-9]+\.[XYZW]}}
 ;EG: CNDE_INT {{\** *}}T{{[0-9]+\.[XYZW], PV\.[XYZW], T[0-9]+\.[XYZW], T[0-9]+\.[XYZW]}}
diff --git a/test/CodeGen/R600/vselect64.ll b/test/CodeGen/AMDGPU/vselect64.ll
similarity index 100%
rename from test/CodeGen/R600/vselect64.ll
rename to test/CodeGen/AMDGPU/vselect64.ll
diff --git a/test/CodeGen/R600/vtx-fetch-branch.ll b/test/CodeGen/AMDGPU/vtx-fetch-branch.ll
similarity index 100%
rename from test/CodeGen/R600/vtx-fetch-branch.ll
rename to test/CodeGen/AMDGPU/vtx-fetch-branch.ll
diff --git a/test/CodeGen/R600/vtx-schedule.ll b/test/CodeGen/AMDGPU/vtx-schedule.ll
similarity index 100%
rename from test/CodeGen/R600/vtx-schedule.ll
rename to test/CodeGen/AMDGPU/vtx-schedule.ll
diff --git a/test/CodeGen/R600/wait.ll b/test/CodeGen/AMDGPU/wait.ll
similarity index 100%
rename from test/CodeGen/R600/wait.ll
rename to test/CodeGen/AMDGPU/wait.ll
diff --git a/test/CodeGen/R600/work-item-intrinsics.ll b/test/CodeGen/AMDGPU/work-item-intrinsics.ll
similarity index 100%
rename from test/CodeGen/R600/work-item-intrinsics.ll
rename to test/CodeGen/AMDGPU/work-item-intrinsics.ll
diff --git a/test/CodeGen/R600/wrong-transalu-pos-fix.ll b/test/CodeGen/AMDGPU/wrong-transalu-pos-fix.ll
similarity index 88%
rename from test/CodeGen/R600/wrong-transalu-pos-fix.ll
rename to test/CodeGen/AMDGPU/wrong-transalu-pos-fix.ll
index 5ab465338e1..8b383e4c393 100644
--- a/test/CodeGen/R600/wrong-transalu-pos-fix.ll
+++ b/test/CodeGen/AMDGPU/wrong-transalu-pos-fix.ll
@@ -1,14 +1,9 @@
-; RUN: llc < %s -march=r600 -mcpu=redwood | FileCheck %s
+; RUN: llc -march=r600 -mcpu=redwood -mtriple=r600-- < %s | FileCheck %s
 
 ; We want all MULLO_INT inst to be last in their instruction group
 ;CHECK: {{^}}fill3d:
 ;CHECK-NOT: MULLO_INT T[0-9]+
 
-; ModuleID = 'radeon'
-target datalayout = "e-p:32:32:32-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-v16:16:16-v24:32:32-v32:32:32-v48:64:64-v64:64:64-v96:128:128-v128:128:128-v192:256:256-v256:256:256-v512:512:512-v1024:1024:1024-v2048:2048:2048-n32:64"
-target triple = "r600--"
-
-; Function Attrs: nounwind
 define void @fill3d(i32 addrspace(1)* nocapture %out) #0 {
 entry:
   %x.i = tail call i32 @llvm.r600.read.global.size.x() #1
diff --git a/test/CodeGen/R600/xor.ll b/test/CodeGen/AMDGPU/xor.ll
similarity index 97%
rename from test/CodeGen/R600/xor.ll
rename to test/CodeGen/AMDGPU/xor.ll
index 089db59eabc..ddb920af29d 100644
--- a/test/CodeGen/R600/xor.ll
+++ b/test/CodeGen/AMDGPU/xor.ll
@@ -42,8 +42,8 @@ define void @xor_v4i32(<4 x i32> addrspace(1)* %out, <4 x i32> addrspace(1)* %in
 
 ; SI-DAG: v_cmp_le_f32_e32 [[CMP0:vcc]], 0, {{v[0-9]+}}
 ; SI-DAG: v_cmp_le_f32_e64 [[CMP1:s\[[0-9]+:[0-9]+\]]], 1.0, {{v[0-9]+}}
-; SI: s_xor_b64 [[XOR:s\[[0-9]+:[0-9]+\]]], [[CMP0]], [[CMP1]]
-; SI: v_cndmask_b32_e64 [[RESULT:v[0-9]+]], {{v[0-9]+}}, {{v[0-9]+}}, [[XOR]]
+; SI: s_xor_b64 [[XOR:vcc]], [[CMP0]], [[CMP1]]
+; SI: v_cndmask_b32_e32 [[RESULT:v[0-9]+]], {{v[0-9]+}}, {{v[0-9]+}}
 ; SI: buffer_store_dword [[RESULT]]
 ; SI: s_endpgm
 define void @xor_i1(float addrspace(1)* %out, float addrspace(1)* %in0, float addrspace(1)* %in1) {
diff --git a/test/CodeGen/R600/zero_extend.ll b/test/CodeGen/AMDGPU/zero_extend.ll
similarity index 100%
rename from test/CodeGen/R600/zero_extend.ll
rename to test/CodeGen/AMDGPU/zero_extend.ll
diff --git a/test/CodeGen/ARM/2007-03-13-InstrSched.ll b/test/CodeGen/ARM/2007-03-13-InstrSched.ll
index 9c0143be06c..81a6bb64971 100644
--- a/test/CodeGen/ARM/2007-03-13-InstrSched.ll
+++ b/test/CodeGen/ARM/2007-03-13-InstrSched.ll
@@ -2,7 +2,7 @@
 ; RUN: llc < %s -mtriple=arm-apple-darwin -relocation-model=pic \
 ; RUN:   -mattr=+v6 | grep r9
 ; RUN: llc < %s -mtriple=arm-apple-darwin -relocation-model=pic \
-; RUN:   -mattr=+v6 -arm-reserve-r9 -ifcvt-limit=0 -stats 2>&1 | grep asm-printer
+; RUN:   -mattr=+v6,+reserve-r9 -ifcvt-limit=0 -stats 2>&1 | grep asm-printer
 ; | grep 35
 
 define void @test(i32 %tmp56222, i32 %tmp36224, i32 %tmp46223, i32 %i.0196.0.ph, i32 %tmp8, i32* %tmp1011, i32** %tmp1, i32* %d2.1.out, i32* %d3.1.out, i32* %d0.1.out, i32* %d1.1.out) {
diff --git a/test/CodeGen/ARM/2009-08-31-LSDA-Name.ll b/test/CodeGen/ARM/2009-08-31-LSDA-Name.ll
index 2f6e428351d..d402c16ccac 100644
--- a/test/CodeGen/ARM/2009-08-31-LSDA-Name.ll
+++ b/test/CodeGen/ARM/2009-08-31-LSDA-Name.ll
@@ -7,7 +7,7 @@
 
 %struct.A = type { i32* }
 
-define void @"\01-[MyFunction Name:]"() {
+define void @"\01-[MyFunction Name:]"() personality i8* bitcast (i32 (...)* @__gxx_personality_sj0 to i8*) {
 entry:
   %save_filt.1 = alloca i32
   %save_eptr.0 = alloca i8*
@@ -39,7 +39,7 @@ return:                                           ; preds = %invcont
   ret void
 
 lpad:                                             ; preds = %entry
-  %exn = landingpad {i8*, i32} personality i8* bitcast (i32 (...)* @__gxx_personality_sj0 to i8*)
+  %exn = landingpad {i8*, i32}
            cleanup
   %eh_ptr = extractvalue {i8*, i32} %exn, 0
   store i8* %eh_ptr, i8** %eh_exception
diff --git a/test/CodeGen/ARM/2009-10-16-Scope.ll b/test/CodeGen/ARM/2009-10-16-Scope.ll
index b2b3bbe29d0..3f47488372b 100644
--- a/test/CodeGen/ARM/2009-10-16-Scope.ll
+++ b/test/CodeGen/ARM/2009-10-16-Scope.ll
@@ -9,7 +9,7 @@ entry:
   br label %do.body, !dbg !0
 
 do.body:                                          ; preds = %entry
-  call void @llvm.dbg.declare(metadata i32* %count_, metadata !4, metadata !MDExpression()), !dbg !MDLocation(scope: !5)
+  call void @llvm.dbg.declare(metadata i32* %count_, metadata !4, metadata !DIExpression()), !dbg !DILocation(scope: !5)
   %conv = ptrtoint i32* %count_ to i32, !dbg !0   ; <i32> [#uses=1]
   %call = call i32 @foo(i32 %conv) ssp, !dbg !0   ; <i32> [#uses=0]
   br label %do.end, !dbg !0
@@ -22,13 +22,13 @@ declare void @llvm.dbg.declare(metadata, metadata, metadata) nounwind readnone
 
 declare i32 @foo(i32) ssp
 
-!0 = !MDLocation(line: 5, column: 2, scope: !1)
-!1 = distinct !MDLexicalBlock(line: 1, column: 1, file: null, scope: !2)
-!2 = !MDSubprogram(name: "bar", linkageName: "bar", line: 4, isLocal: false, isDefinition: true, virtualIndex: 6, isOptimized: false, scope: !3)
-!3 = !MDCompileUnit(language: DW_LANG_C99, producer: "clang 1.1", isOptimized: true, emissionKind: 0, file: !8, retainedTypes: !9)
-!4 = !MDLocalVariable(tag: DW_TAG_auto_variable, name: "count_", line: 5, scope: !5, file: !3, type: !6)
-!5 = distinct !MDLexicalBlock(line: 1, column: 1, file: null, scope: !1)
-!6 = !MDBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
-!7 = !MDLocation(line: 6, column: 1, scope: !2)
-!8 = !MDFile(filename: "genmodes.i", directory: "/Users/yash/Downloads")
+!0 = !DILocation(line: 5, column: 2, scope: !1)
+!1 = distinct !DILexicalBlock(line: 1, column: 1, file: null, scope: !2)
+!2 = !DISubprogram(name: "bar", linkageName: "bar", line: 4, isLocal: false, isDefinition: true, virtualIndex: 6, isOptimized: false, scope: !3)
+!3 = !DICompileUnit(language: DW_LANG_C99, producer: "clang 1.1", isOptimized: true, emissionKind: 0, file: !8, retainedTypes: !9)
+!4 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "count_", line: 5, scope: !5, file: !3, type: !6)
+!5 = distinct !DILexicalBlock(line: 1, column: 1, file: null, scope: !1)
+!6 = !DIBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
+!7 = !DILocation(line: 6, column: 1, scope: !2)
+!8 = !DIFile(filename: "genmodes.i", directory: "/Users/yash/Downloads")
 !9 = !{i32 0}
diff --git a/test/CodeGen/ARM/2010-04-15-ScavengerDebugValue.ll b/test/CodeGen/ARM/2010-04-15-ScavengerDebugValue.ll
index 312cccddde3..86e17205fc9 100644
--- a/test/CodeGen/ARM/2010-04-15-ScavengerDebugValue.ll
+++ b/test/CodeGen/ARM/2010-04-15-ScavengerDebugValue.ll
@@ -5,7 +5,7 @@ target triple = "armv4t-apple-darwin10"
 
 define hidden i32 @__addvsi3(i32 %a, i32 %b) nounwind {
 entry:
-  tail call void @llvm.dbg.value(metadata i32 %b, i64 0, metadata !0, metadata !MDExpression()), !dbg !MDLocation(scope: !1)
+  tail call void @llvm.dbg.value(metadata i32 %b, i64 0, metadata !0, metadata !DIExpression()), !dbg !DILocation(scope: !1)
   %0 = add nsw i32 %b, %a, !dbg !9                ; <i32> [#uses=1]
   ret i32 %0, !dbg !11
 }
@@ -14,19 +14,19 @@ declare void @llvm.dbg.value(metadata, i64, metadata, metadata) nounwind readnon
 
 !llvm.dbg.cu = !{!3}
 !llvm.module.flags = !{!15}
-!0 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "b", line: 93, arg: 0, scope: !1, file: !2, type: !6)
-!1 = !MDSubprogram(name: "__addvsi3", linkageName: "__addvsi3", line: 94, isLocal: false, isDefinition: true, virtualIndex: 6, isOptimized: false, file: !12, scope: null, type: !4)
-!2 = !MDFile(filename: "libgcc2.c", directory: "/Users/bwilson/local/nightly/test-2010-04-14/build/llvmgcc.roots/llvmgcc~obj/src/gcc")
-!12 = !MDFile(filename: "libgcc2.c", directory: "/Users/bwilson/local/nightly/test-2010-04-14/build/llvmgcc.roots/llvmgcc~obj/src/gcc")
-!3 = !MDCompileUnit(language: DW_LANG_C89, producer: "4.2.1 (Based on Apple Inc. build 5658) (LLVM build 00)", isOptimized: true, emissionKind: 0, file: !12, enums: !13, retainedTypes: !13, subprograms: !14)
-!4 = !MDSubroutineType(types: !5)
+!0 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "b", line: 93, arg: 2, scope: !1, file: !2, type: !6)
+!1 = !DISubprogram(name: "__addvsi3", linkageName: "__addvsi3", line: 94, isLocal: false, isDefinition: true, virtualIndex: 6, isOptimized: false, file: !12, scope: null, type: !4)
+!2 = !DIFile(filename: "libgcc2.c", directory: "/Users/bwilson/local/nightly/test-2010-04-14/build/llvmgcc.roots/llvmgcc~obj/src/gcc")
+!12 = !DIFile(filename: "libgcc2.c", directory: "/Users/bwilson/local/nightly/test-2010-04-14/build/llvmgcc.roots/llvmgcc~obj/src/gcc")
+!3 = !DICompileUnit(language: DW_LANG_C89, producer: "4.2.1 (Based on Apple Inc. build 5658) (LLVM build 00)", isOptimized: true, emissionKind: 0, file: !12, enums: !13, retainedTypes: !13, subprograms: !14)
+!4 = !DISubroutineType(types: !5)
 !5 = !{!6, !6, !6}
-!6 = !MDDerivedType(tag: DW_TAG_typedef, name: "SItype", line: 152, file: !12, baseType: !8)
-!7 = !MDFile(filename: "libgcc2.h", directory: "/Users/bwilson/local/nightly/test-2010-04-14/build/llvmgcc.roots/llvmgcc~obj/src/gcc")
-!8 = !MDBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
-!9 = !MDLocation(line: 95, scope: !10)
-!10 = distinct !MDLexicalBlock(line: 94, column: 0, file: !12, scope: !1)
-!11 = !MDLocation(line: 100, scope: !10)
+!6 = !DIDerivedType(tag: DW_TAG_typedef, name: "SItype", line: 152, file: !12, baseType: !8)
+!7 = !DIFile(filename: "libgcc2.h", directory: "/Users/bwilson/local/nightly/test-2010-04-14/build/llvmgcc.roots/llvmgcc~obj/src/gcc")
+!8 = !DIBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
+!9 = !DILocation(line: 95, scope: !10)
+!10 = distinct !DILexicalBlock(line: 94, column: 0, file: !12, scope: !1)
+!11 = !DILocation(line: 100, scope: !10)
 !13 = !{}
 !14 = !{!1}
 !15 = !{i32 1, !"Debug Info Version", i32 3}
diff --git a/test/CodeGen/ARM/2010-06-25-Thumb2ITInvalidIterator.ll b/test/CodeGen/ARM/2010-06-25-Thumb2ITInvalidIterator.ll
index 52c39a48091..5637d7aa934 100644
--- a/test/CodeGen/ARM/2010-06-25-Thumb2ITInvalidIterator.ll
+++ b/test/CodeGen/ARM/2010-06-25-Thumb2ITInvalidIterator.ll
@@ -7,16 +7,16 @@ target triple = "thumbv7-apple-darwin3.0.0-iphoneos"
 
 define void @x0(i8* nocapture %buf, i32 %nbytes) nounwind optsize {
 entry:
-  tail call void @llvm.dbg.value(metadata i8* %buf, i64 0, metadata !0, metadata !MDExpression()), !dbg !15
-  tail call void @llvm.dbg.value(metadata i32 %nbytes, i64 0, metadata !8, metadata !MDExpression()), !dbg !16
+  tail call void @llvm.dbg.value(metadata i8* %buf, i64 0, metadata !0, metadata !DIExpression()), !dbg !15
+  tail call void @llvm.dbg.value(metadata i32 %nbytes, i64 0, metadata !8, metadata !DIExpression()), !dbg !16
   %tmp = load i32, i32* @length, !dbg !17              ; <i32> [#uses=3]
   %cmp = icmp eq i32 %tmp, -1, !dbg !17           ; <i1> [#uses=1]
   %cmp.not = xor i1 %cmp, true                    ; <i1> [#uses=1]
   %cmp3 = icmp ult i32 %tmp, %nbytes, !dbg !17    ; <i1> [#uses=1]
   %or.cond = and i1 %cmp.not, %cmp3               ; <i1> [#uses=1]
-  tail call void @llvm.dbg.value(metadata i32 %tmp, i64 0, metadata !8, metadata !MDExpression()), !dbg !17
+  tail call void @llvm.dbg.value(metadata i32 %tmp, i64 0, metadata !8, metadata !DIExpression()), !dbg !17
   %nbytes.addr.0 = select i1 %or.cond, i32 %tmp, i32 %nbytes ; <i32> [#uses=1]
-  tail call void @llvm.dbg.value(metadata i32 0, i64 0, metadata !10, metadata !MDExpression()), !dbg !19
+  tail call void @llvm.dbg.value(metadata i32 0, i64 0, metadata !10, metadata !DIExpression()), !dbg !19
   br label %while.cond, !dbg !20
 
 while.cond:                                       ; preds = %while.body, %entry
@@ -47,30 +47,30 @@ declare void @llvm.dbg.value(metadata, i64, metadata, metadata) nounwind readnon
 !llvm.dbg.lv.fn = !{!0, !8, !10, !12}
 !llvm.dbg.gv = !{!14}
 
-!0 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "buf", line: 4, arg: 0, scope: !1, file: !2, type: !6)
-!1 = !MDSubprogram(name: "x0", linkageName: "x0", line: 5, isLocal: false, isDefinition: true, virtualIndex: 6, isOptimized: false, file: !26, scope: null, type: !4)
-!2 = !MDFile(filename: "t.c", directory: "/private/tmp")
-!3 = !MDCompileUnit(language: DW_LANG_C99, producer: "clang 2.0", isOptimized: true, file: !26)
-!4 = !MDSubroutineType(types: !5)
+!0 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "buf", line: 4, arg: 1, scope: !1, file: !2, type: !6)
+!1 = !DISubprogram(name: "x0", linkageName: "x0", line: 5, isLocal: false, isDefinition: true, virtualIndex: 6, isOptimized: false, file: !26, scope: null, type: !4)
+!2 = !DIFile(filename: "t.c", directory: "/private/tmp")
+!3 = !DICompileUnit(language: DW_LANG_C99, producer: "clang 2.0", isOptimized: true, file: !26)
+!4 = !DISubroutineType(types: !5)
 !5 = !{null}
-!6 = !MDDerivedType(tag: DW_TAG_pointer_type, size: 32, align: 32, file: !26, scope: !2, baseType: !7)
-!7 = !MDBasicType(tag: DW_TAG_base_type, name: "unsigned char", size: 8, align: 8, encoding: DW_ATE_unsigned_char)
-!8 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "nbytes", line: 4, arg: 0, scope: !1, file: !2, type: !9)
-!9 = !MDBasicType(tag: DW_TAG_base_type, name: "unsigned long", size: 32, align: 32, encoding: DW_ATE_unsigned)
-!10 = !MDLocalVariable(tag: DW_TAG_auto_variable, name: "nread", line: 6, scope: !11, file: !2, type: !9)
-!11 = distinct !MDLexicalBlock(line: 5, column: 1, file: !26, scope: !1)
-!12 = !MDLocalVariable(tag: DW_TAG_auto_variable, name: "c", line: 7, scope: !11, file: !2, type: !13)
-!13 = !MDBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
-!14 = !MDGlobalVariable(name: "length", linkageName: "length", line: 1, isLocal: false, isDefinition: true, scope: !2, file: !2, type: !13, variable: i32* @length)
-!15 = !MDLocation(line: 4, column: 24, scope: !1)
-!16 = !MDLocation(line: 4, column: 43, scope: !1)
-!17 = !MDLocation(line: 9, column: 2, scope: !11)
+!6 = !DIDerivedType(tag: DW_TAG_pointer_type, size: 32, align: 32, file: !26, scope: !2, baseType: !7)
+!7 = !DIBasicType(tag: DW_TAG_base_type, name: "unsigned char", size: 8, align: 8, encoding: DW_ATE_unsigned_char)
+!8 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "nbytes", line: 4, arg: 2, scope: !1, file: !2, type: !9)
+!9 = !DIBasicType(tag: DW_TAG_base_type, name: "unsigned long", size: 32, align: 32, encoding: DW_ATE_unsigned)
+!10 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "nread", line: 6, scope: !11, file: !2, type: !9)
+!11 = distinct !DILexicalBlock(line: 5, column: 1, file: !26, scope: !1)
+!12 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "c", line: 7, scope: !11, file: !2, type: !13)
+!13 = !DIBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
+!14 = !DIGlobalVariable(name: "length", linkageName: "length", line: 1, isLocal: false, isDefinition: true, scope: !2, file: !2, type: !13, variable: i32* @length)
+!15 = !DILocation(line: 4, column: 24, scope: !1)
+!16 = !DILocation(line: 4, column: 43, scope: !1)
+!17 = !DILocation(line: 9, column: 2, scope: !11)
 !18 = !{i32 0}
-!19 = !MDLocation(line: 10, column: 2, scope: !11)
-!20 = !MDLocation(line: 11, column: 2, scope: !11)
-!21 = !MDLocation(line: 12, column: 3, scope: !22)
-!22 = distinct !MDLexicalBlock(line: 11, column: 45, file: !26, scope: !11)
-!23 = !MDLocation(line: 13, column: 3, scope: !22)
-!24 = !MDLocation(line: 14, column: 2, scope: !22)
-!25 = !MDLocation(line: 15, column: 1, scope: !11)
-!26 = !MDFile(filename: "t.c", directory: "/private/tmp")
+!19 = !DILocation(line: 10, column: 2, scope: !11)
+!20 = !DILocation(line: 11, column: 2, scope: !11)
+!21 = !DILocation(line: 12, column: 3, scope: !22)
+!22 = distinct !DILexicalBlock(line: 11, column: 45, file: !26, scope: !11)
+!23 = !DILocation(line: 13, column: 3, scope: !22)
+!24 = !DILocation(line: 14, column: 2, scope: !22)
+!25 = !DILocation(line: 15, column: 1, scope: !11)
+!26 = !DIFile(filename: "t.c", directory: "/private/tmp")
diff --git a/test/CodeGen/ARM/2010-07-26-GlobalMerge.ll b/test/CodeGen/ARM/2010-07-26-GlobalMerge.ll
index b02efea929f..a876d998e75 100644
--- a/test/CodeGen/ARM/2010-07-26-GlobalMerge.ll
+++ b/test/CodeGen/ARM/2010-07-26-GlobalMerge.ll
@@ -40,7 +40,7 @@ entry:
 
 declare void @__cxa_throw(i8*, i8*, i8*)
 
-define i32 @main() ssp {
+define i32 @main() ssp personality i8* bitcast (i32 (...)* @__gxx_personality_sj0 to i8*) {
 entry:
   %puts.i = tail call i32 @puts(i8* getelementptr inbounds ([14 x i8], [14 x i8]* @str, i32 0, i32 0)) ; <i32> [#uses=0]
   %exception.i = tail call i8* @__cxa_allocate_exception(i32 4) nounwind ; <i8*> [#uses=2]
@@ -71,7 +71,7 @@ try.cont:                                         ; preds = %lpad
   ret i32 %conv
 
 lpad:                                             ; preds = %entry
-  %exn.ptr = landingpad { i8*, i32 } personality i8* bitcast (i32 (...)* @__gxx_personality_sj0 to i8*)
+  %exn.ptr = landingpad { i8*, i32 }
            catch i8* bitcast (%0* @_ZTI1A to i8*)
            catch i8* null
   %exn = extractvalue { i8*, i32 } %exn.ptr, 0
diff --git a/test/CodeGen/ARM/2010-08-04-EHCrash.ll b/test/CodeGen/ARM/2010-08-04-EHCrash.ll
index 13214c52153..69482cc8b35 100644
--- a/test/CodeGen/ARM/2010-08-04-EHCrash.ll
+++ b/test/CodeGen/ARM/2010-08-04-EHCrash.ll
@@ -1,7 +1,7 @@
 ; RUN: llc < %s -mtriple=thumbv7-apple-darwin10
 ; <rdar://problem/8264008>
 
-define linkonce_odr arm_apcscc void @func1() {
+define linkonce_odr arm_apcscc void @func1() personality i8* bitcast (i32 (...)* @__gxx_personality_sj0 to i8*) {
 entry:
   %save_filt.936 = alloca i32                     ; <i32*> [#uses=2]
   %save_eptr.935 = alloca i8*                     ; <i8**> [#uses=2]
@@ -34,7 +34,7 @@ return:                                           ; preds = %entry
   ret void
 
 lpad:                                             ; preds = %bb
-  %eh_ptr = landingpad { i8*, i32 } personality i8* bitcast (i32 (...)* @__gxx_personality_sj0 to i8*)
+  %eh_ptr = landingpad { i8*, i32 }
               cleanup
   %exn = extractvalue { i8*, i32 } %eh_ptr, 0
   store i8* %exn, i8** %eh_exception
diff --git a/test/CodeGen/ARM/2010-08-04-StackVariable.ll b/test/CodeGen/ARM/2010-08-04-StackVariable.ll
index a48aef33b16..f59d9a48a30 100644
--- a/test/CodeGen/ARM/2010-08-04-StackVariable.ll
+++ b/test/CodeGen/ARM/2010-08-04-StackVariable.ll
@@ -6,8 +6,8 @@
 define i32 @_Z3fooi4SVal(i32 %i, %struct.SVal* noalias %location) nounwind ssp {
 entry:
   %"alloca point" = bitcast i32 0 to i32          ; <i32> [#uses=0]
-  call void @llvm.dbg.value(metadata i32 %i, i64 0, metadata !23, metadata !MDExpression()), !dbg !24
-  call void @llvm.dbg.value(metadata %struct.SVal* %location, i64 0, metadata !25, metadata !MDExpression()), !dbg !24
+  call void @llvm.dbg.value(metadata i32 %i, i64 0, metadata !23, metadata !DIExpression()), !dbg !24
+  call void @llvm.dbg.value(metadata %struct.SVal* %location, i64 0, metadata !25, metadata !DIExpression()), !dbg !24
   %0 = icmp ne i32 %i, 0, !dbg !27                ; <i1> [#uses=1]
   br i1 %0, label %bb, label %bb1, !dbg !27
 
@@ -34,7 +34,7 @@ return:                                           ; preds = %bb2
 define linkonce_odr void @_ZN4SValC1Ev(%struct.SVal* %this) nounwind ssp align 2  {
 entry:
   %"alloca point" = bitcast i32 0 to i32          ; <i32> [#uses=0]
-  call void @llvm.dbg.value(metadata %struct.SVal* %this, i64 0, metadata !31, metadata !MDExpression()), !dbg !34
+  call void @llvm.dbg.value(metadata %struct.SVal* %this, i64 0, metadata !31, metadata !DIExpression()), !dbg !34
   %0 = getelementptr inbounds %struct.SVal, %struct.SVal* %this, i32 0, i32 0, !dbg !34 ; <i8**> [#uses=1]
   store i8* null, i8** %0, align 8, !dbg !34
   %1 = getelementptr inbounds %struct.SVal, %struct.SVal* %this, i32 0, i32 1, !dbg !34 ; <i32*> [#uses=1]
@@ -52,7 +52,7 @@ entry:
   %0 = alloca %struct.SVal                        ; <%struct.SVal*> [#uses=3]
   %v = alloca %struct.SVal                        ; <%struct.SVal*> [#uses=4]
   %"alloca point" = bitcast i32 0 to i32          ; <i32> [#uses=0]
-  call void @llvm.dbg.declare(metadata %struct.SVal* %v, metadata !38, metadata !MDExpression()), !dbg !41
+  call void @llvm.dbg.declare(metadata %struct.SVal* %v, metadata !38, metadata !DIExpression()), !dbg !41
   call void @_ZN4SValC1Ev(%struct.SVal* %v) nounwind, !dbg !41
   %1 = getelementptr inbounds %struct.SVal, %struct.SVal* %v, i32 0, i32 1, !dbg !42 ; <i32*> [#uses=1]
   store i32 1, i32* %1, align 8, !dbg !42
@@ -65,7 +65,7 @@ entry:
   %7 = load i32, i32* %6, align 8, !dbg !43            ; <i32> [#uses=1]
   store i32 %7, i32* %5, align 8, !dbg !43
   %8 = call i32 @_Z3fooi4SVal(i32 2, %struct.SVal* noalias %0) nounwind, !dbg !43 ; <i32> [#uses=0]
-  call void @llvm.dbg.value(metadata i32 %8, i64 0, metadata !44, metadata !MDExpression()), !dbg !43
+  call void @llvm.dbg.value(metadata i32 %8, i64 0, metadata !44, metadata !DIExpression()), !dbg !43
   br label %return, !dbg !45
 
 return:                                           ; preds = %entry
@@ -77,53 +77,53 @@ declare void @llvm.dbg.value(metadata, i64, metadata, metadata) nounwind readnon
 !llvm.dbg.cu = !{!3}
 !llvm.module.flags = !{!49}
 
-!0 = !MDSubprogram(name: "SVal", line: 11, isLocal: false, isDefinition: false, virtualIndex: 6, isOptimized: false, file: !48, scope: !1, type: !14)
-!1 = !MDCompositeType(tag: DW_TAG_structure_type, name: "SVal", line: 1, size: 128, align: 64, file: !48, elements: !4)
-!2 = !MDFile(filename: "small.cc", directory: "/Users/manav/R8248330")
-!3 = !MDCompileUnit(language: DW_LANG_C_plus_plus, producer: "4.2.1 (Based on Apple Inc. build 5658) (LLVM build)", isOptimized: false, emissionKind: 1, file: !48, enums: !47, retainedTypes: !47, subprograms: !46, globals: !47, imports:  !47)
+!0 = !DISubprogram(name: "SVal", line: 11, isLocal: false, isDefinition: false, virtualIndex: 6, isOptimized: false, file: !48, scope: !1, type: !14)
+!1 = !DICompositeType(tag: DW_TAG_structure_type, name: "SVal", line: 1, size: 128, align: 64, file: !48, elements: !4)
+!2 = !DIFile(filename: "small.cc", directory: "/Users/manav/R8248330")
+!3 = !DICompileUnit(language: DW_LANG_C_plus_plus, producer: "4.2.1 (Based on Apple Inc. build 5658) (LLVM build)", isOptimized: false, emissionKind: 1, file: !48, enums: !47, retainedTypes: !47, subprograms: !46, globals: !47, imports:  !47)
 !4 = !{!5, !7, !0, !9}
-!5 = !MDDerivedType(tag: DW_TAG_member, name: "Data", line: 7, size: 64, align: 64, file: !48, scope: !1, baseType: !6)
-!6 = !MDDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, file: !48, baseType: null)
-!7 = !MDDerivedType(tag: DW_TAG_member, name: "Kind", line: 8, size: 32, align: 32, offset: 64, file: !48, scope: !1, baseType: !8)
-!8 = !MDBasicType(tag: DW_TAG_base_type, name: "unsigned int", size: 32, align: 32, encoding: DW_ATE_unsigned)
-!9 = !MDSubprogram(name: "~SVal", line: 12, isLocal: false, isDefinition: false, virtualIndex: 6, isOptimized: false, file: !48, scope: !1, type: !10)
-!10 = !MDSubroutineType(types: !11)
+!5 = !DIDerivedType(tag: DW_TAG_member, name: "Data", line: 7, size: 64, align: 64, file: !48, scope: !1, baseType: !6)
+!6 = !DIDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, file: !48, baseType: null)
+!7 = !DIDerivedType(tag: DW_TAG_member, name: "Kind", line: 8, size: 32, align: 32, offset: 64, file: !48, scope: !1, baseType: !8)
+!8 = !DIBasicType(tag: DW_TAG_base_type, name: "unsigned int", size: 32, align: 32, encoding: DW_ATE_unsigned)
+!9 = !DISubprogram(name: "~SVal", line: 12, isLocal: false, isDefinition: false, virtualIndex: 6, isOptimized: false, file: !48, scope: !1, type: !10)
+!10 = !DISubroutineType(types: !11)
 !11 = !{null, !12, !13}
-!12 = !MDDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, flags: DIFlagArtificial, file: !48, baseType: !1)
-!13 = !MDBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
-!14 = !MDSubroutineType(types: !15)
+!12 = !DIDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, flags: DIFlagArtificial, file: !48, baseType: !1)
+!13 = !DIBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
+!14 = !DISubroutineType(types: !15)
 !15 = !{null, !12}
-!16 = !MDSubprogram(name: "SVal", linkageName: "_ZN4SValC1Ev", line: 11, isLocal: false, isDefinition: true, virtualIndex: 6, isOptimized: false, file: !48, scope: !1, type: !14, function: void (%struct.SVal*)* @_ZN4SValC1Ev)
-!17 = !MDSubprogram(name: "foo", linkageName: "_Z3fooi4SVal", line: 16, isLocal: false, isDefinition: true, virtualIndex: 6, isOptimized: false, file: !48, scope: !2, type: !18, function: i32 (i32, %struct.SVal*)* @_Z3fooi4SVal)
-!18 = !MDSubroutineType(types: !19)
+!16 = !DISubprogram(name: "SVal", linkageName: "_ZN4SValC1Ev", line: 11, isLocal: false, isDefinition: true, virtualIndex: 6, isOptimized: false, file: !48, scope: !1, type: !14, function: void (%struct.SVal*)* @_ZN4SValC1Ev)
+!17 = !DISubprogram(name: "foo", linkageName: "_Z3fooi4SVal", line: 16, isLocal: false, isDefinition: true, virtualIndex: 6, isOptimized: false, file: !48, scope: !2, type: !18, function: i32 (i32, %struct.SVal*)* @_Z3fooi4SVal)
+!18 = !DISubroutineType(types: !19)
 !19 = !{!13, !13, !1}
-!20 = !MDSubprogram(name: "main", linkageName: "main", line: 23, isLocal: false, isDefinition: true, virtualIndex: 6, isOptimized: false, file: !48, scope: !2, type: !21, function: i32 ()* @main)
-!21 = !MDSubroutineType(types: !22)
+!20 = !DISubprogram(name: "main", linkageName: "main", line: 23, isLocal: false, isDefinition: true, virtualIndex: 6, isOptimized: false, file: !48, scope: !2, type: !21, function: i32 ()* @main)
+!21 = !DISubroutineType(types: !22)
 !22 = !{!13}
-!23 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "i", line: 16, arg: 0, scope: !17, file: !2, type: !13)
-!24 = !MDLocation(line: 16, scope: !17)
-!25 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "location", line: 16, arg: 0, scope: !17, file: !2, type: !26)
-!26 = !MDDerivedType(tag: DW_TAG_reference_type, name: "SVal", size: 64, align: 64, file: !48, scope: !2, baseType: !1)
-!27 = !MDLocation(line: 17, scope: !28)
-!28 = distinct !MDLexicalBlock(line: 16, column: 0, file: !2, scope: !17)
-!29 = !MDLocation(line: 18, scope: !28)
-!30 = !MDLocation(line: 20, scope: !28)
-!31 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "this", line: 11, arg: 0, scope: !16, file: !2, type: !32)
-!32 = !MDDerivedType(tag: DW_TAG_const_type, size: 64, align: 64, flags: DIFlagArtificial, file: !48, scope: !2, baseType: !33)
-!33 = !MDDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, file: !48, scope: !2, baseType: !1)
-!34 = !MDLocation(line: 11, scope: !16)
-!35 = !MDLocation(line: 11, scope: !36)
-!36 = distinct !MDLexicalBlock(line: 11, column: 0, file: !48, scope: !37)
-!37 = distinct !MDLexicalBlock(line: 11, column: 0, file: !48, scope: !16)
-!38 = !MDLocalVariable(tag: DW_TAG_auto_variable, name: "v", line: 24, scope: !39, file: !2, type: !1)
-!39 = distinct !MDLexicalBlock(line: 23, column: 0, file: !48, scope: !40)
-!40 = distinct !MDLexicalBlock(line: 23, column: 0, file: !48, scope: !20)
-!41 = !MDLocation(line: 24, scope: !39)
-!42 = !MDLocation(line: 25, scope: !39)
-!43 = !MDLocation(line: 26, scope: !39)
-!44 = !MDLocalVariable(tag: DW_TAG_auto_variable, name: "k", line: 26, scope: !39, file: !2, type: !13)
-!45 = !MDLocation(line: 27, scope: !39)
+!23 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "i", line: 16, arg: 1, scope: !17, file: !2, type: !13)
+!24 = !DILocation(line: 16, scope: !17)
+!25 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "location", line: 16, arg: 2, scope: !17, file: !2, type: !26)
+!26 = !DIDerivedType(tag: DW_TAG_reference_type, name: "SVal", size: 64, align: 64, file: !48, scope: !2, baseType: !1)
+!27 = !DILocation(line: 17, scope: !28)
+!28 = distinct !DILexicalBlock(line: 16, column: 0, file: !2, scope: !17)
+!29 = !DILocation(line: 18, scope: !28)
+!30 = !DILocation(line: 20, scope: !28)
+!31 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "this", line: 11, arg: 1, scope: !16, file: !2, type: !32)
+!32 = !DIDerivedType(tag: DW_TAG_const_type, size: 64, align: 64, flags: DIFlagArtificial, file: !48, scope: !2, baseType: !33)
+!33 = !DIDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, file: !48, scope: !2, baseType: !1)
+!34 = !DILocation(line: 11, scope: !16)
+!35 = !DILocation(line: 11, scope: !36)
+!36 = distinct !DILexicalBlock(line: 11, column: 0, file: !48, scope: !37)
+!37 = distinct !DILexicalBlock(line: 11, column: 0, file: !48, scope: !16)
+!38 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "v", line: 24, scope: !39, file: !2, type: !1)
+!39 = distinct !DILexicalBlock(line: 23, column: 0, file: !48, scope: !40)
+!40 = distinct !DILexicalBlock(line: 23, column: 0, file: !48, scope: !20)
+!41 = !DILocation(line: 24, scope: !39)
+!42 = !DILocation(line: 25, scope: !39)
+!43 = !DILocation(line: 26, scope: !39)
+!44 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "k", line: 26, scope: !39, file: !2, type: !13)
+!45 = !DILocation(line: 27, scope: !39)
 !46 = !{!16, !17, !20}
 !47 = !{}
-!48 = !MDFile(filename: "small.cc", directory: "/Users/manav/R8248330")
+!48 = !DIFile(filename: "small.cc", directory: "/Users/manav/R8248330")
 !49 = !{i32 1, !"Debug Info Version", i32 3}
diff --git a/test/CodeGen/ARM/2010-12-15-elf-lcomm.ll b/test/CodeGen/ARM/2010-12-15-elf-lcomm.ll
index cb91890a9fd..f04e053b2e0 100644
--- a/test/CodeGen/ARM/2010-12-15-elf-lcomm.ll
+++ b/test/CodeGen/ARM/2010-12-15-elf-lcomm.ll
@@ -1,6 +1,6 @@
-; RUN: llc  %s -mtriple=armv7-linux-gnueabi -O3 -filetype=obj -o - | \
+; RUN: llc  %s -mtriple=armv7-linux-gnueabi -filetype=obj -o - | \
 ; RUN:    llvm-readobj -s -t | FileCheck  -check-prefix=OBJ %s
-; RUN: llc  %s -mtriple=armv7-linux-gnueabi -O3 -o - | \
+; RUN: llc  %s -mtriple=armv7-linux-gnueabi -o - | \
 ; RUN:    FileCheck  -check-prefix=ASM %s
 
 
@@ -12,7 +12,7 @@
 ; ASM:          .type   array00,%object         @ @array00
 ; ASM-NEXT:     .local  array00
 ; ASM-NEXT:     .comm   array00,80,1
-; ASM-NEXT:     .type   _MergedGlobals,%object  @ @_MergedGlobals
+; ASM-NEXT:     .type   sum,%object  @ @sum
 
 
 ; OBJ:      Symbols [
diff --git a/test/CodeGen/ARM/2011-01-19-MergedGlobalDbg.ll b/test/CodeGen/ARM/2011-01-19-MergedGlobalDbg.ll
index 7196d2d37bd..05fd2832525 100644
--- a/test/CodeGen/ARM/2011-01-19-MergedGlobalDbg.ll
+++ b/test/CodeGen/ARM/2011-01-19-MergedGlobalDbg.ll
@@ -1,4 +1,4 @@
-; RUN: llc -O3 -filetype=obj < %s | llvm-dwarfdump -debug-dump=info - | FileCheck %s
+; RUN: llc -arm-global-merge -global-merge-group-by-use=false -filetype=obj < %s | llvm-dwarfdump -debug-dump=info - | FileCheck %s
 
 target datalayout = "e-p:32:32:32-i1:8:32-i8:8:32-i16:16:32-i32:32:32-i64:32:64-f32:32:32-f64:32:64-v64:32:64-v128:32:128-a0:0:32-n32"
 target triple = "thumbv7-apple-darwin10"
@@ -30,9 +30,9 @@ target triple = "thumbv7-apple-darwin10"
 
 define zeroext i8 @get1(i8 zeroext %a) nounwind optsize {
 entry:
-  tail call void @llvm.dbg.value(metadata i8 %a, i64 0, metadata !10, metadata !MDExpression()), !dbg !30
+  tail call void @llvm.dbg.value(metadata i8 %a, i64 0, metadata !10, metadata !DIExpression()), !dbg !30
   %0 = load i8, i8* @x1, align 4, !dbg !30
-  tail call void @llvm.dbg.value(metadata i8 %0, i64 0, metadata !11, metadata !MDExpression()), !dbg !30
+  tail call void @llvm.dbg.value(metadata i8 %0, i64 0, metadata !11, metadata !DIExpression()), !dbg !30
   store i8 %a, i8* @x1, align 4, !dbg !30
   ret i8 %0, !dbg !31
 }
@@ -41,36 +41,36 @@ declare void @llvm.dbg.value(metadata, i64, metadata, metadata) nounwind readnon
 
 define zeroext i8 @get2(i8 zeroext %a) nounwind optsize {
 entry:
-  tail call void @llvm.dbg.value(metadata i8 %a, i64 0, metadata !18, metadata !MDExpression()), !dbg !32
+  tail call void @llvm.dbg.value(metadata i8 %a, i64 0, metadata !18, metadata !DIExpression()), !dbg !32
   %0 = load i8, i8* @x2, align 4, !dbg !32
-  tail call void @llvm.dbg.value(metadata i8 %0, i64 0, metadata !19, metadata !MDExpression()), !dbg !32
+  tail call void @llvm.dbg.value(metadata i8 %0, i64 0, metadata !19, metadata !DIExpression()), !dbg !32
   store i8 %a, i8* @x2, align 4, !dbg !32
   ret i8 %0, !dbg !33
 }
 
 define zeroext i8 @get3(i8 zeroext %a) nounwind optsize {
 entry:
-  tail call void @llvm.dbg.value(metadata i8 %a, i64 0, metadata !21, metadata !MDExpression()), !dbg !34
+  tail call void @llvm.dbg.value(metadata i8 %a, i64 0, metadata !21, metadata !DIExpression()), !dbg !34
   %0 = load i8, i8* @x3, align 4, !dbg !34
-  tail call void @llvm.dbg.value(metadata i8 %0, i64 0, metadata !22, metadata !MDExpression()), !dbg !34
+  tail call void @llvm.dbg.value(metadata i8 %0, i64 0, metadata !22, metadata !DIExpression()), !dbg !34
   store i8 %a, i8* @x3, align 4, !dbg !34
   ret i8 %0, !dbg !35
 }
 
 define zeroext i8 @get4(i8 zeroext %a) nounwind optsize {
 entry:
-  tail call void @llvm.dbg.value(metadata i8 %a, i64 0, metadata !24, metadata !MDExpression()), !dbg !36
+  tail call void @llvm.dbg.value(metadata i8 %a, i64 0, metadata !24, metadata !DIExpression()), !dbg !36
   %0 = load i8, i8* @x4, align 4, !dbg !36
-  tail call void @llvm.dbg.value(metadata i8 %0, i64 0, metadata !25, metadata !MDExpression()), !dbg !36
+  tail call void @llvm.dbg.value(metadata i8 %0, i64 0, metadata !25, metadata !DIExpression()), !dbg !36
   store i8 %a, i8* @x4, align 4, !dbg !36
   ret i8 %0, !dbg !37
 }
 
 define zeroext i8 @get5(i8 zeroext %a) nounwind optsize {
 entry:
-  tail call void @llvm.dbg.value(metadata i8 %a, i64 0, metadata !27, metadata !MDExpression()), !dbg !38
+  tail call void @llvm.dbg.value(metadata i8 %a, i64 0, metadata !27, metadata !DIExpression()), !dbg !38
   %0 = load i8, i8* @x5, align 4, !dbg !38
-  tail call void @llvm.dbg.value(metadata i8 %0, i64 0, metadata !28, metadata !MDExpression()), !dbg !38
+  tail call void @llvm.dbg.value(metadata i8 %0, i64 0, metadata !28, metadata !DIExpression()), !dbg !38
   store i8 %a, i8* @x5, align 4, !dbg !38
   ret i8 %0, !dbg !39
 }
@@ -78,46 +78,46 @@ entry:
 !llvm.dbg.cu = !{!2}
 !llvm.module.flags = !{!49}
 
-!0 = !MDSubprogram(name: "get1", linkageName: "get1", line: 4, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: true, scopeLine: 4, file: !47, scope: !1, type: !3, function: i8 (i8)* @get1, variables: !42)
-!1 = !MDFile(filename: "foo.c", directory: "/tmp/")
-!2 = !MDCompileUnit(language: DW_LANG_C89, producer: "4.2.1 (Based on Apple Inc. build 5658) (LLVM build 2369.8)", isOptimized: true, emissionKind: 0, file: !47, enums: !48, retainedTypes: !48, subprograms: !40, globals: !41, imports:  !48)
-!3 = !MDSubroutineType(types: !4)
+!0 = !DISubprogram(name: "get1", linkageName: "get1", line: 4, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: true, scopeLine: 4, file: !47, scope: !1, type: !3, function: i8 (i8)* @get1, variables: !42)
+!1 = !DIFile(filename: "foo.c", directory: "/tmp/")
+!2 = !DICompileUnit(language: DW_LANG_C89, producer: "4.2.1 (Based on Apple Inc. build 5658) (LLVM build 2369.8)", isOptimized: true, emissionKind: 0, file: !47, enums: !48, retainedTypes: !48, subprograms: !40, globals: !41, imports:  !48)
+!3 = !DISubroutineType(types: !4)
 !4 = !{!5, !5}
-!5 = !MDBasicType(tag: DW_TAG_base_type, name: "_Bool", size: 8, align: 8, encoding: DW_ATE_boolean)
-!6 = !MDSubprogram(name: "get2", linkageName: "get2", line: 7, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: true, scopeLine: 7, file: !47, scope: !1, type: !3, function: i8 (i8)* @get2, variables: !43)
-!7 = !MDSubprogram(name: "get3", linkageName: "get3", line: 10, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: true, scopeLine: 10, file: !47, scope: !1, type: !3, function: i8 (i8)* @get3, variables: !44)
-!8 = !MDSubprogram(name: "get4", linkageName: "get4", line: 13, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: true, scopeLine: 13, file: !47, scope: !1, type: !3, function: i8 (i8)* @get4, variables: !45)
-!9 = !MDSubprogram(name: "get5", linkageName: "get5", line: 16, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: true, scopeLine: 16, file: !47, scope: !1, type: !3, function: i8 (i8)* @get5, variables: !46)
-!10 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "a", line: 4, arg: 0, scope: !0, file: !1, type: !5)
-!11 = !MDLocalVariable(tag: DW_TAG_auto_variable, name: "b", line: 4, scope: !12, file: !1, type: !5)
-!12 = distinct !MDLexicalBlock(line: 4, column: 0, file: !47, scope: !0)
-!13 = !MDGlobalVariable(name: "x1", line: 3, isLocal: true, isDefinition: true, scope: !1, file: !1, type: !5, variable: i8* @x1)
-!14 = !MDGlobalVariable(name: "x2", line: 6, isLocal: true, isDefinition: true, scope: !1, file: !1, type: !5, variable: i8* @x2)
-!15 = !MDGlobalVariable(name: "x3", line: 9, isLocal: true, isDefinition: true, scope: !1, file: !1, type: !5, variable: i8* @x3)
-!16 = !MDGlobalVariable(name: "x4", line: 12, isLocal: true, isDefinition: true, scope: !1, file: !1, type: !5, variable: i8* @x4)
-!17 = !MDGlobalVariable(name: "x5", line: 15, isLocal: false, isDefinition: true, scope: !1, file: !1, type: !5, variable: i8* @x5)
-!18 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "a", line: 7, arg: 0, scope: !6, file: !1, type: !5)
-!19 = !MDLocalVariable(tag: DW_TAG_auto_variable, name: "b", line: 7, scope: !20, file: !1, type: !5)
-!20 = distinct !MDLexicalBlock(line: 7, column: 0, file: !47, scope: !6)
-!21 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "a", line: 10, arg: 0, scope: !7, file: !1, type: !5)
-!22 = !MDLocalVariable(tag: DW_TAG_auto_variable, name: "b", line: 10, scope: !23, file: !1, type: !5)
-!23 = distinct !MDLexicalBlock(line: 10, column: 0, file: !47, scope: !7)
-!24 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "a", line: 13, arg: 0, scope: !8, file: !1, type: !5)
-!25 = !MDLocalVariable(tag: DW_TAG_auto_variable, name: "b", line: 13, scope: !26, file: !1, type: !5)
-!26 = distinct !MDLexicalBlock(line: 13, column: 0, file: !47, scope: !8)
-!27 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "a", line: 16, arg: 0, scope: !9, file: !1, type: !5)
-!28 = !MDLocalVariable(tag: DW_TAG_auto_variable, name: "b", line: 16, scope: !29, file: !1, type: !5)
-!29 = distinct !MDLexicalBlock(line: 16, column: 0, file: !47, scope: !9)
-!30 = !MDLocation(line: 4, scope: !0)
-!31 = !MDLocation(line: 4, scope: !12)
-!32 = !MDLocation(line: 7, scope: !6)
-!33 = !MDLocation(line: 7, scope: !20)
-!34 = !MDLocation(line: 10, scope: !7)
-!35 = !MDLocation(line: 10, scope: !23)
-!36 = !MDLocation(line: 13, scope: !8)
-!37 = !MDLocation(line: 13, scope: !26)
-!38 = !MDLocation(line: 16, scope: !9)
-!39 = !MDLocation(line: 16, scope: !29)
+!5 = !DIBasicType(tag: DW_TAG_base_type, name: "_Bool", size: 8, align: 8, encoding: DW_ATE_boolean)
+!6 = !DISubprogram(name: "get2", linkageName: "get2", line: 7, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: true, scopeLine: 7, file: !47, scope: !1, type: !3, function: i8 (i8)* @get2, variables: !43)
+!7 = !DISubprogram(name: "get3", linkageName: "get3", line: 10, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: true, scopeLine: 10, file: !47, scope: !1, type: !3, function: i8 (i8)* @get3, variables: !44)
+!8 = !DISubprogram(name: "get4", linkageName: "get4", line: 13, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: true, scopeLine: 13, file: !47, scope: !1, type: !3, function: i8 (i8)* @get4, variables: !45)
+!9 = !DISubprogram(name: "get5", linkageName: "get5", line: 16, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: true, scopeLine: 16, file: !47, scope: !1, type: !3, function: i8 (i8)* @get5, variables: !46)
+!10 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "a", line: 4, arg: 1, scope: !0, file: !1, type: !5)
+!11 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "b", line: 4, scope: !12, file: !1, type: !5)
+!12 = distinct !DILexicalBlock(line: 4, column: 0, file: !47, scope: !0)
+!13 = !DIGlobalVariable(name: "x1", line: 3, isLocal: true, isDefinition: true, scope: !1, file: !1, type: !5, variable: i8* @x1)
+!14 = !DIGlobalVariable(name: "x2", line: 6, isLocal: true, isDefinition: true, scope: !1, file: !1, type: !5, variable: i8* @x2)
+!15 = !DIGlobalVariable(name: "x3", line: 9, isLocal: true, isDefinition: true, scope: !1, file: !1, type: !5, variable: i8* @x3)
+!16 = !DIGlobalVariable(name: "x4", line: 12, isLocal: true, isDefinition: true, scope: !1, file: !1, type: !5, variable: i8* @x4)
+!17 = !DIGlobalVariable(name: "x5", line: 15, isLocal: false, isDefinition: true, scope: !1, file: !1, type: !5, variable: i8* @x5)
+!18 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "a", line: 7, arg: 1, scope: !6, file: !1, type: !5)
+!19 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "b", line: 7, scope: !20, file: !1, type: !5)
+!20 = distinct !DILexicalBlock(line: 7, column: 0, file: !47, scope: !6)
+!21 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "a", line: 10, arg: 1, scope: !7, file: !1, type: !5)
+!22 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "b", line: 10, scope: !23, file: !1, type: !5)
+!23 = distinct !DILexicalBlock(line: 10, column: 0, file: !47, scope: !7)
+!24 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "a", line: 13, arg: 1, scope: !8, file: !1, type: !5)
+!25 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "b", line: 13, scope: !26, file: !1, type: !5)
+!26 = distinct !DILexicalBlock(line: 13, column: 0, file: !47, scope: !8)
+!27 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "a", line: 16, arg: 1, scope: !9, file: !1, type: !5)
+!28 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "b", line: 16, scope: !29, file: !1, type: !5)
+!29 = distinct !DILexicalBlock(line: 16, column: 0, file: !47, scope: !9)
+!30 = !DILocation(line: 4, scope: !0)
+!31 = !DILocation(line: 4, scope: !12)
+!32 = !DILocation(line: 7, scope: !6)
+!33 = !DILocation(line: 7, scope: !20)
+!34 = !DILocation(line: 10, scope: !7)
+!35 = !DILocation(line: 10, scope: !23)
+!36 = !DILocation(line: 13, scope: !8)
+!37 = !DILocation(line: 13, scope: !26)
+!38 = !DILocation(line: 16, scope: !9)
+!39 = !DILocation(line: 16, scope: !29)
 !40 = !{!0, !6, !7, !8, !9}
 !41 = !{!13, !14, !15, !16, !17}
 !42 = !{!10, !11}
@@ -125,6 +125,6 @@ entry:
 !44 = !{!21, !22}
 !45 = !{!24, !25}
 !46 = !{!27, !28}
-!47 = !MDFile(filename: "foo.c", directory: "/tmp/")
+!47 = !DIFile(filename: "foo.c", directory: "/tmp/")
 !48 = !{}
 !49 = !{i32 1, !"Debug Info Version", i32 3}
diff --git a/test/CodeGen/ARM/2011-05-04-MultipleLandingPadSuccs.ll b/test/CodeGen/ARM/2011-05-04-MultipleLandingPadSuccs.ll
index 2af3e3e6bd4..559b027fb11 100644
--- a/test/CodeGen/ARM/2011-05-04-MultipleLandingPadSuccs.ll
+++ b/test/CodeGen/ARM/2011-05-04-MultipleLandingPadSuccs.ll
@@ -3,7 +3,7 @@
 target datalayout = "e-p:32:32:32-i1:8:32-i8:8:32-i16:16:32-i32:32:32-i64:32:32-f32:32:32-f64:32:32-v64:64:64-v128:128:128-a0:0:32-n32"
 target triple = "thumbv7-apple-darwin"
 
-define void @func() unnamed_addr align 2 {
+define void @func() unnamed_addr align 2 personality i8* bitcast (i32 (...)* @__gxx_personality_sj0 to i8*) {
 entry:
   br label %for.cond
 
@@ -35,13 +35,13 @@ for.cond.backedge:
   br label %for.cond
 
 lpad:
-  %exn = landingpad { i8*, i32 } personality i8* bitcast (i32 (...)* @__gxx_personality_sj0 to i8*)
+  %exn = landingpad { i8*, i32 }
            catch i8* null
   invoke void @foo()
           to label %eh.resume unwind label %terminate.lpad
 
 lpad26:
-  %exn27 = landingpad { i8*, i32 } personality i8* bitcast (i32 (...)* @__gxx_personality_sj0 to i8*)
+  %exn27 = landingpad { i8*, i32 }
            catch i8* null
   invoke void @foo()
           to label %eh.resume unwind label %terminate.lpad
@@ -57,7 +57,7 @@ call8.i.i.i.noexc:
   ret void
 
 lpad44:
-  %exn45 = landingpad { i8*, i32 } personality i8* bitcast (i32 (...)* @__gxx_personality_sj0 to i8*)
+  %exn45 = landingpad { i8*, i32 }
            catch i8* null
   invoke void @foo()
           to label %eh.resume unwind label %terminate.lpad
@@ -67,7 +67,7 @@ eh.resume:
   resume { i8*, i32 } %exn.slot.0
 
 terminate.lpad:
-  %exn51 = landingpad { i8*, i32 } personality i8* bitcast (i32 (...)* @__gxx_personality_sj0 to i8*)
+  %exn51 = landingpad { i8*, i32 }
            catch i8* null
   tail call void @_ZSt9terminatev() noreturn nounwind
   unreachable
diff --git a/test/CodeGen/ARM/2011-06-29-MergeGlobalsAlign.ll b/test/CodeGen/ARM/2011-06-29-MergeGlobalsAlign.ll
index 236dc1c0dbc..aac8f7b3a02 100644
--- a/test/CodeGen/ARM/2011-06-29-MergeGlobalsAlign.ll
+++ b/test/CodeGen/ARM/2011-06-29-MergeGlobalsAlign.ll
@@ -1,4 +1,4 @@
-; RUN: llc < %s -mtriple=thumbv7-apple-darwin10 -O3 | FileCheck %s
+; RUN: llc < %s -mtriple=thumbv7-apple-darwin10 -arm-global-merge -global-merge-group-by-use=false | FileCheck %s
 ; CHECK: .zerofill __DATA,__bss,__MergedGlobals,16,2
 
 @prev = external global [0 x i16]
diff --git a/test/CodeGen/ARM/2011-08-02-MergedGlobalDbg.ll b/test/CodeGen/ARM/2011-08-02-MergedGlobalDbg.ll
index 1614e8d0c67..067c719f491 100644
--- a/test/CodeGen/ARM/2011-08-02-MergedGlobalDbg.ll
+++ b/test/CodeGen/ARM/2011-08-02-MergedGlobalDbg.ll
@@ -1,4 +1,4 @@
-; RUN: llc -O3 -filetype=obj < %s | llvm-dwarfdump -debug-dump=info - | FileCheck %s
+; RUN: llc -arm-global-merge -global-merge-group-by-use=false -filetype=obj < %s | llvm-dwarfdump -debug-dump=info - | FileCheck %s
 
 ; Check debug info output for merged global.
 ; DW_AT_location
@@ -29,41 +29,41 @@ target triple = "thumbv7-apple-macosx10.7.0"
 @x5 = global i32 0, align 4
 
 define i32 @get1(i32 %a) nounwind optsize ssp {
-  tail call void @llvm.dbg.value(metadata i32 %a, i64 0, metadata !10, metadata !MDExpression()), !dbg !30
+  tail call void @llvm.dbg.value(metadata i32 %a, i64 0, metadata !10, metadata !DIExpression()), !dbg !30
   %1 = load i32, i32* @x1, align 4, !dbg !31
-  tail call void @llvm.dbg.value(metadata i32 %1, i64 0, metadata !11, metadata !MDExpression()), !dbg !31
+  tail call void @llvm.dbg.value(metadata i32 %1, i64 0, metadata !11, metadata !DIExpression()), !dbg !31
   store i32 %a, i32* @x1, align 4, !dbg !31
   ret i32 %1, !dbg !31
 }
 
 define i32 @get2(i32 %a) nounwind optsize ssp {
-  tail call void @llvm.dbg.value(metadata i32 %a, i64 0, metadata !13, metadata !MDExpression()), !dbg !32
+  tail call void @llvm.dbg.value(metadata i32 %a, i64 0, metadata !13, metadata !DIExpression()), !dbg !32
   %1 = load i32, i32* @x2, align 4, !dbg !33
-  tail call void @llvm.dbg.value(metadata i32 %1, i64 0, metadata !14, metadata !MDExpression()), !dbg !33
+  tail call void @llvm.dbg.value(metadata i32 %1, i64 0, metadata !14, metadata !DIExpression()), !dbg !33
   store i32 %a, i32* @x2, align 4, !dbg !33
   ret i32 %1, !dbg !33
 }
 
 define i32 @get3(i32 %a) nounwind optsize ssp {
-  tail call void @llvm.dbg.value(metadata i32 %a, i64 0, metadata !16, metadata !MDExpression()), !dbg !34
+  tail call void @llvm.dbg.value(metadata i32 %a, i64 0, metadata !16, metadata !DIExpression()), !dbg !34
   %1 = load i32, i32* @x3, align 4, !dbg !35
-  tail call void @llvm.dbg.value(metadata i32 %1, i64 0, metadata !17, metadata !MDExpression()), !dbg !35
+  tail call void @llvm.dbg.value(metadata i32 %1, i64 0, metadata !17, metadata !DIExpression()), !dbg !35
   store i32 %a, i32* @x3, align 4, !dbg !35
   ret i32 %1, !dbg !35
 }
 
 define i32 @get4(i32 %a) nounwind optsize ssp {
-  tail call void @llvm.dbg.value(metadata i32 %a, i64 0, metadata !19, metadata !MDExpression()), !dbg !36
+  tail call void @llvm.dbg.value(metadata i32 %a, i64 0, metadata !19, metadata !DIExpression()), !dbg !36
   %1 = load i32, i32* @x4, align 4, !dbg !37
-  tail call void @llvm.dbg.value(metadata i32 %1, i64 0, metadata !20, metadata !MDExpression()), !dbg !37
+  tail call void @llvm.dbg.value(metadata i32 %1, i64 0, metadata !20, metadata !DIExpression()), !dbg !37
   store i32 %a, i32* @x4, align 4, !dbg !37
   ret i32 %1, !dbg !37
 }
 
 define i32 @get5(i32 %a) nounwind optsize ssp {
-  tail call void @llvm.dbg.value(metadata i32 %a, i64 0, metadata !27, metadata !MDExpression()), !dbg !38
+  tail call void @llvm.dbg.value(metadata i32 %a, i64 0, metadata !27, metadata !DIExpression()), !dbg !38
   %1 = load i32, i32* @x5, align 4, !dbg !39
-  tail call void @llvm.dbg.value(metadata i32 %1, i64 0, metadata !28, metadata !MDExpression()), !dbg !39
+  tail call void @llvm.dbg.value(metadata i32 %1, i64 0, metadata !28, metadata !DIExpression()), !dbg !39
   store i32 %a, i32* @x5, align 4, !dbg !39
   ret i32 %1, !dbg !39
 }
@@ -73,43 +73,43 @@ declare void @llvm.dbg.value(metadata, i64, metadata, metadata) nounwind readnon
 !llvm.dbg.cu = !{!0}
 !llvm.module.flags = !{!49}
 
-!0 = !MDCompileUnit(language: DW_LANG_C99, producer: "clang", isOptimized: true, emissionKind: 1, file: !47, enums: !48, retainedTypes: !48, subprograms: !40, globals: !41, imports:  !48)
-!1 = !MDSubprogram(name: "get1", line: 5, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: true, scopeLine: 5, file: !47, scope: !2, type: !3, function: i32 (i32)* @get1, variables: !42)
-!2 = !MDFile(filename: "ss3.c", directory: "/private/tmp")
-!3 = !MDSubroutineType(types: !4)
+!0 = !DICompileUnit(language: DW_LANG_C99, producer: "clang", isOptimized: true, emissionKind: 1, file: !47, enums: !48, retainedTypes: !48, subprograms: !40, globals: !41, imports:  !48)
+!1 = !DISubprogram(name: "get1", line: 5, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: true, scopeLine: 5, file: !47, scope: !2, type: !3, function: i32 (i32)* @get1, variables: !42)
+!2 = !DIFile(filename: "ss3.c", directory: "/private/tmp")
+!3 = !DISubroutineType(types: !4)
 !4 = !{!5}
-!5 = !MDBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
-!6 = !MDSubprogram(name: "get2", line: 8, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: true, scopeLine: 8, file: !47, scope: !2, type: !3, function: i32 (i32)* @get2, variables: !43)
-!7 = !MDSubprogram(name: "get3", line: 11, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: true, scopeLine: 11, file: !47, scope: !2, type: !3, function: i32 (i32)* @get3, variables: !44)
-!8 = !MDSubprogram(name: "get4", line: 14, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: true, scopeLine: 14, file: !47, scope: !2, type: !3, function: i32 (i32)* @get4, variables: !45)
-!9 = !MDSubprogram(name: "get5", line: 17, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: true, scopeLine: 17, file: !47, scope: !2, type: !3, function: i32 (i32)* @get5, variables: !46)
-!10 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "a", line: 5, arg: 1, scope: !1, file: !2, type: !5)
-!11 = !MDLocalVariable(tag: DW_TAG_auto_variable, name: "b", line: 5, scope: !12, file: !2, type: !5)
-!12 = distinct !MDLexicalBlock(line: 5, column: 19, file: !47, scope: !1)
-!13 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "a", line: 8, arg: 1, scope: !6, file: !2, type: !5)
-!14 = !MDLocalVariable(tag: DW_TAG_auto_variable, name: "b", line: 8, scope: !15, file: !2, type: !5)
-!15 = distinct !MDLexicalBlock(line: 8, column: 17, file: !47, scope: !6)
-!16 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "a", line: 11, arg: 1, scope: !7, file: !2, type: !5)
-!17 = !MDLocalVariable(tag: DW_TAG_auto_variable, name: "b", line: 11, scope: !18, file: !2, type: !5)
-!18 = distinct !MDLexicalBlock(line: 11, column: 19, file: !47, scope: !7)
-!19 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "a", line: 14, arg: 1, scope: !8, file: !2, type: !5)
-!20 = !MDLocalVariable(tag: DW_TAG_auto_variable, name: "b", line: 14, scope: !21, file: !2, type: !5)
-!21 = distinct !MDLexicalBlock(line: 14, column: 19, file: !47, scope: !8)
-!25 = !MDGlobalVariable(name: "x1", line: 4, isLocal: true, isDefinition: true, scope: !0, file: !2, type: !5, variable: i32* @x1)
-!26 = !MDGlobalVariable(name: "x2", line: 7, isLocal: true, isDefinition: true, scope: !0, file: !2, type: !5, variable: i32* @x2)
-!27 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "a", line: 17, arg: 1, scope: !9, file: !2, type: !5)
-!28 = !MDLocalVariable(tag: DW_TAG_auto_variable, name: "b", line: 17, scope: !29, file: !2, type: !5)
-!29 = distinct !MDLexicalBlock(line: 17, column: 19, file: !47, scope: !9)
-!30 = !MDLocation(line: 5, column: 16, scope: !1)
-!31 = !MDLocation(line: 5, column: 32, scope: !12)
-!32 = !MDLocation(line: 8, column: 14, scope: !6)
-!33 = !MDLocation(line: 8, column: 29, scope: !15)
-!34 = !MDLocation(line: 11, column: 16, scope: !7)
-!35 = !MDLocation(line: 11, column: 32, scope: !18)
-!36 = !MDLocation(line: 14, column: 16, scope: !8)
-!37 = !MDLocation(line: 14, column: 32, scope: !21)
-!38 = !MDLocation(line: 17, column: 16, scope: !9)
-!39 = !MDLocation(line: 17, column: 32, scope: !29)
+!5 = !DIBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
+!6 = !DISubprogram(name: "get2", line: 8, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: true, scopeLine: 8, file: !47, scope: !2, type: !3, function: i32 (i32)* @get2, variables: !43)
+!7 = !DISubprogram(name: "get3", line: 11, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: true, scopeLine: 11, file: !47, scope: !2, type: !3, function: i32 (i32)* @get3, variables: !44)
+!8 = !DISubprogram(name: "get4", line: 14, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: true, scopeLine: 14, file: !47, scope: !2, type: !3, function: i32 (i32)* @get4, variables: !45)
+!9 = !DISubprogram(name: "get5", line: 17, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: true, scopeLine: 17, file: !47, scope: !2, type: !3, function: i32 (i32)* @get5, variables: !46)
+!10 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "a", line: 5, arg: 1, scope: !1, file: !2, type: !5)
+!11 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "b", line: 5, scope: !12, file: !2, type: !5)
+!12 = distinct !DILexicalBlock(line: 5, column: 19, file: !47, scope: !1)
+!13 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "a", line: 8, arg: 1, scope: !6, file: !2, type: !5)
+!14 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "b", line: 8, scope: !15, file: !2, type: !5)
+!15 = distinct !DILexicalBlock(line: 8, column: 17, file: !47, scope: !6)
+!16 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "a", line: 11, arg: 1, scope: !7, file: !2, type: !5)
+!17 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "b", line: 11, scope: !18, file: !2, type: !5)
+!18 = distinct !DILexicalBlock(line: 11, column: 19, file: !47, scope: !7)
+!19 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "a", line: 14, arg: 1, scope: !8, file: !2, type: !5)
+!20 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "b", line: 14, scope: !21, file: !2, type: !5)
+!21 = distinct !DILexicalBlock(line: 14, column: 19, file: !47, scope: !8)
+!25 = !DIGlobalVariable(name: "x1", line: 4, isLocal: true, isDefinition: true, scope: !0, file: !2, type: !5, variable: i32* @x1)
+!26 = !DIGlobalVariable(name: "x2", line: 7, isLocal: true, isDefinition: true, scope: !0, file: !2, type: !5, variable: i32* @x2)
+!27 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "a", line: 17, arg: 1, scope: !9, file: !2, type: !5)
+!28 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "b", line: 17, scope: !29, file: !2, type: !5)
+!29 = distinct !DILexicalBlock(line: 17, column: 19, file: !47, scope: !9)
+!30 = !DILocation(line: 5, column: 16, scope: !1)
+!31 = !DILocation(line: 5, column: 32, scope: !12)
+!32 = !DILocation(line: 8, column: 14, scope: !6)
+!33 = !DILocation(line: 8, column: 29, scope: !15)
+!34 = !DILocation(line: 11, column: 16, scope: !7)
+!35 = !DILocation(line: 11, column: 32, scope: !18)
+!36 = !DILocation(line: 14, column: 16, scope: !8)
+!37 = !DILocation(line: 14, column: 32, scope: !21)
+!38 = !DILocation(line: 17, column: 16, scope: !9)
+!39 = !DILocation(line: 17, column: 32, scope: !29)
 !40 = !{!1, !6, !7, !8, !9}
 !41 = !{!25, !26}
 !42 = !{!10, !11}
@@ -117,6 +117,6 @@ declare void @llvm.dbg.value(metadata, i64, metadata, metadata) nounwind readnon
 !44 = !{!16, !17}
 !45 = !{!19, !20}
 !46 = !{!27, !28}
-!47 = !MDFile(filename: "ss3.c", directory: "/private/tmp")
+!47 = !DIFile(filename: "ss3.c", directory: "/private/tmp")
 !48 = !{}
 !49 = !{i32 1, !"Debug Info Version", i32 3}
diff --git a/test/CodeGen/ARM/2011-10-26-memset-inline.ll b/test/CodeGen/ARM/2011-10-26-memset-inline.ll
index 17bd291a6b5..5df439389cd 100644
--- a/test/CodeGen/ARM/2011-10-26-memset-inline.ll
+++ b/test/CodeGen/ARM/2011-10-26-memset-inline.ll
@@ -1,5 +1,5 @@
 ; Make sure short memsets on ARM lower to stores, even when optimizing for size.
-; RUN: llc -march=arm < %s | FileCheck %s -check-prefix=CHECK-GENERIC
+; RUN: llc -march=arm -mattr=+strict-align < %s | FileCheck %s -check-prefix=CHECK-GENERIC
 ; RUN: llc -march=arm -mcpu=cortex-a8 < %s | FileCheck %s -check-prefix=CHECK-UNALIGNED
 
 target datalayout = "e-p:32:32:32-i1:8:32-i8:8:32-i16:16:32-i32:32:32-i64:32:64-f32:32:32-f64:32:64-v64:32:64-v128:32:128-a0:0:32-n32-S32"
diff --git a/test/CodeGen/ARM/2011-12-19-sjlj-clobber.ll b/test/CodeGen/ARM/2011-12-19-sjlj-clobber.ll
index 40d1f628aaa..b00cc51d984 100644
--- a/test/CodeGen/ARM/2011-12-19-sjlj-clobber.ll
+++ b/test/CodeGen/ARM/2011-12-19-sjlj-clobber.ll
@@ -8,7 +8,7 @@
 %0 = type opaque
 %struct.NSConstantString = type { i32*, i32, i8*, i32 }
 
-define i32 @asdf(i32 %a, i32 %b, i8** %c, i8* %d) {
+define i32 @asdf(i32 %a, i32 %b, i8** %c, i8* %d) personality i8* bitcast (i32 (...)* @__objc_personality_v0 to i8*) {
 bb:
   %tmp = alloca i32, align 4
   %tmp1 = alloca i32, align 4
@@ -37,7 +37,7 @@ bb14:                                             ; preds = %bb11
   unreachable
 
 bb15:                                             ; preds = %bb11, %bb
-  %tmp16 = landingpad { i8*, i32 } personality i8* bitcast (i32 (...)* @__objc_personality_v0 to i8*)
+  %tmp16 = landingpad { i8*, i32 }
           catch i8* null
   %tmp17 = extractvalue { i8*, i32 } %tmp16, 0
   store i8* %tmp17, i8** %tmp4
diff --git a/test/CodeGen/ARM/2012-04-24-SplitEHCriticalEdge.ll b/test/CodeGen/ARM/2012-04-24-SplitEHCriticalEdge.ll
index 69d72bd8339..ce0dcc70952 100644
--- a/test/CodeGen/ARM/2012-04-24-SplitEHCriticalEdge.ll
+++ b/test/CodeGen/ARM/2012-04-24-SplitEHCriticalEdge.ll
@@ -25,13 +25,13 @@ declare void @__cxa_end_catch()
 
 declare void @_ZSt9terminatev()
 
-define hidden double @t(%0* %self, i8* nocapture %_cmd) optsize ssp {
+define hidden double @t(%0* %self, i8* nocapture %_cmd) optsize ssp personality i8* bitcast (i32 (...)* @__gxx_personality_sj0 to i8*) {
 entry:
   %call = invoke double undef(%class.FunctionInterpreter.3.15.31* undef) optsize
           to label %try.cont unwind label %lpad
 
 lpad:                                             ; preds = %entry
-  %0 = landingpad { i8*, i32 } personality i8* bitcast (i32 (...)* @__gxx_personality_sj0 to i8*)
+  %0 = landingpad { i8*, i32 }
           catch i8* bitcast ({ i8*, i8* }* @_ZTI13ParseErrorMsg to i8*)
   br i1 undef, label %catch, label %eh.resume
 
@@ -47,7 +47,7 @@ try.cont:                                         ; preds = %invoke.cont2, %entr
   ret double %value.0
 
 lpad1:                                            ; preds = %catch
-  %1 = landingpad { i8*, i32 } personality i8* bitcast (i32 (...)* @__gxx_personality_sj0 to i8*)
+  %1 = landingpad { i8*, i32 }
           cleanup
   invoke void @__cxa_end_catch()
           to label %eh.resume unwind label %terminate.lpad
@@ -56,7 +56,7 @@ eh.resume:                                        ; preds = %lpad1, %lpad
   resume { i8*, i32 } undef
 
 terminate.lpad:                                   ; preds = %lpad1
-  %2 = landingpad { i8*, i32 } personality i8* bitcast (i32 (...)* @__gxx_personality_sj0 to i8*)
+  %2 = landingpad { i8*, i32 }
           catch i8* null
   unreachable
 }
diff --git a/test/CodeGen/ARM/2013-04-16-AAPCS-C5-vs-VFP.ll b/test/CodeGen/ARM/2013-04-16-AAPCS-C5-vs-VFP.ll
index 446403d79ca..5e82b0925b2 100644
--- a/test/CodeGen/ARM/2013-04-16-AAPCS-C5-vs-VFP.ll
+++ b/test/CodeGen/ARM/2013-04-16-AAPCS-C5-vs-VFP.ll
@@ -33,7 +33,7 @@ define void @foo2(double %p0, ; --> D0
                   %struct_t* byval %p10) ; --> Stack+8
 {
 entry:
-;CHECK:     push.w {r11, lr}
+;CHECK:     push {r7, lr}
 ;CHECK-NOT: stm
 ;CHECK:     add r0, sp, #16
 ;CHECK:     bl fooUseStruct
diff --git a/test/CodeGen/ARM/2013-05-02-AAPCS-ByVal-Structs-C4-C5-VFP.ll b/test/CodeGen/ARM/2013-05-02-AAPCS-ByVal-Structs-C4-C5-VFP.ll
index 68b918d2d7d..ac5b6f9c970 100644
--- a/test/CodeGen/ARM/2013-05-02-AAPCS-ByVal-Structs-C4-C5-VFP.ll
+++ b/test/CodeGen/ARM/2013-05-02-AAPCS-ByVal-Structs-C4-C5-VFP.ll
@@ -22,11 +22,10 @@ define void @foo(double %vfp0,     ; --> D0,     NSAA=SP
                  i32 %p3) #0 {     ; --> SP+4,   NSAA=SP+12
 entry:
   ;CHECK: sub sp, #12
-  ;CHECK: push.w {r11, lr}
+  ;CHECK: push {r7, lr}
   ;CHECK: sub sp, #4
   ;CHECK: add r0, sp, #12
-  ;CHECK: str r2, [sp, #16]
-  ;CHECK: str r1, [sp, #12]
+  ;CHECK: strd r1, r2, [sp, #12]
   ;CHECK: bl  fooUseStruct
   call void @fooUseStruct(%st_t* %p1)
   ret void
diff --git a/test/CodeGen/ARM/2013-05-02-AAPCS-ByVal-Structs-C4-C5-VFP2.ll b/test/CodeGen/ARM/2013-05-02-AAPCS-ByVal-Structs-C4-C5-VFP2.ll
index 212bbc2ee9c..944a60c0610 100644
--- a/test/CodeGen/ARM/2013-05-02-AAPCS-ByVal-Structs-C4-C5-VFP2.ll
+++ b/test/CodeGen/ARM/2013-05-02-AAPCS-ByVal-Structs-C4-C5-VFP2.ll
@@ -19,7 +19,7 @@ define void @foo(double %vfp0,     ; --> D0,              NSAA=SP
 		 i32 %p2) #0 {     ; --> SP+24,           NSAA=SP+24 
                  
 entry:
-  ;CHECK:  push.w {r11, lr}
+  ;CHECK:  push {r7, lr}
   ;CHECK:  ldr    r0, [sp, #32]
   ;CHECK:  bl     fooUseI32
   call void @fooUseI32(i32 %p2)
diff --git a/test/CodeGen/ARM/2013-05-13-AAPCS-byval-padding2.ll b/test/CodeGen/ARM/2013-05-13-AAPCS-byval-padding2.ll
index 438b021a040..d3aa2331d45 100644
--- a/test/CodeGen/ARM/2013-05-13-AAPCS-byval-padding2.ll
+++ b/test/CodeGen/ARM/2013-05-13-AAPCS-byval-padding2.ll
@@ -9,8 +9,8 @@ define void @foo(%struct4bytes* byval %p0, ; --> R0
 ) {
 ;CHECK:  sub  sp, sp, #16
 ;CHECK:  push  {r11, lr}
-;CHECK:  add  r11, sp, #8
-;CHECK:  stm  r11, {r0, r1, r2, r3}
+;CHECK:  add  r12, sp, #8
+;CHECK:  stm  r12, {r0, r1, r2, r3}
 ;CHECK:  add  r0, sp, #12
 ;CHECK:  bl  useInt
 ;CHECK:  pop  {r11, lr}
diff --git a/test/CodeGen/ARM/2014-05-14-DwarfEHCrash.ll b/test/CodeGen/ARM/2014-05-14-DwarfEHCrash.ll
index 1e40e4afe5c..feae48646cd 100644
--- a/test/CodeGen/ARM/2014-05-14-DwarfEHCrash.ll
+++ b/test/CodeGen/ARM/2014-05-14-DwarfEHCrash.ll
@@ -8,13 +8,13 @@ target triple = "armv4t--linux-androideabi"
 
 @_ZTIi = external constant i8*
 
-define void @_Z3fn2v() #0 {
+define void @_Z3fn2v() #0 personality i8* bitcast (i32 (...)* @__gxx_personality_v0 to i8*) {
 entry:
   invoke void @_Z3fn1v()
           to label %try.cont unwind label %lpad
 
 lpad:                                             ; preds = %entry
-  %0 = landingpad { i8*, i32 } personality i8* bitcast (i32 (...)* @__gxx_personality_v0 to i8*)
+  %0 = landingpad { i8*, i32 }
           catch i8* bitcast (i8** @_ZTIi to i8*)
   %1 = extractvalue { i8*, i32 } %0, 1
   %2 = tail call i32 @llvm.eh.typeid.for(i8* bitcast (i8** @_ZTIi to i8*)) #2
diff --git a/test/CodeGen/ARM/Windows/hard-float.ll b/test/CodeGen/ARM/Windows/hard-float.ll
index f7b7ec273ce..1ce02813dfc 100644
--- a/test/CodeGen/ARM/Windows/hard-float.ll
+++ b/test/CodeGen/ARM/Windows/hard-float.ll
@@ -1,4 +1,8 @@
-; RUN: llc -mtriple=thumbv7-windows-itanium -mcpu=cortex-a9 -o - %s | FileCheck %s
+; RUN: llc -mtriple=thumbv7-windows-itanium -mcpu=cortex-a9 -o - %s \
+; RUN:   | FileCheck %s -check-prefix CHECK-WIN
+
+; RUN: llc -mtriple=thumbv7-windows-gnu -mcpu=cortex-a9 -o - %s \
+; RUN:   | FileCheck %s -check-prefix CHECK-GNU
 
 define float @function(float %f, float %g) nounwind {
 entry:
@@ -6,5 +10,7 @@ entry:
   ret float %h
 }
 
-; CHECK: vadd.f32 s0, s0, s1
+; CHECK-WIN: vadd.f32 s0, s0, s1
+
+; CHECK-GNU: vadd.f32 s0, s0, s1
 
diff --git a/test/CodeGen/ARM/Windows/long-calls.ll b/test/CodeGen/ARM/Windows/long-calls.ll
index 21c95fac91c..4e5bdce146f 100644
--- a/test/CodeGen/ARM/Windows/long-calls.ll
+++ b/test/CodeGen/ARM/Windows/long-calls.ll
@@ -1,4 +1,4 @@
-; RUN: llc -mtriple=thumbv7-windows -mcpu=cortex-a9 -arm-long-calls -o - %s \
+; RUN: llc -mtriple=thumbv7-windows -mcpu=cortex-a9 -mattr=+long-calls -o - %s \
 ; RUN:    | FileCheck %s
 
 declare arm_aapcs_vfpcc void @callee()
diff --git a/test/CodeGen/ARM/Windows/no-arm-mode.ll b/test/CodeGen/ARM/Windows/no-arm-mode.ll
index 6db031fc916..30353640a4c 100644
--- a/test/CodeGen/ARM/Windows/no-arm-mode.ll
+++ b/test/CodeGen/ARM/Windows/no-arm-mode.ll
@@ -1,5 +1,10 @@
 ; RUN: not llc -mtriple=armv7-windows-itanium -mcpu=cortex-a9 -o /dev/null %s 2>&1 \
-; RUN:  | FileCheck %s
+; RUN:   | FileCheck %s -check-prefix CHECK-WIN
 
-; CHECK: does not support ARM mode execution
+; RUN: not llc -mtriple=armv7-windows-gnu -mcpu=cortex-a9 -o /dev/null %s 2>&1 \
+; RUN:   | FileCheck %s -check-prefix CHECK-GNU
+
+; CHECK-WIN: does not support ARM mode execution
+
+; CHECK-GNU: does not support ARM mode execution
 
diff --git a/test/CodeGen/ARM/Windows/pic.ll b/test/CodeGen/ARM/Windows/pic.ll
index 9ef7c35c553..df4c400035a 100644
--- a/test/CodeGen/ARM/Windows/pic.ll
+++ b/test/CodeGen/ARM/Windows/pic.ll
@@ -1,5 +1,8 @@
 ; RUN: llc -mtriple thumbv7-windows-itanium -relocation-model pic -filetype asm -o - %s \
-; RUN:    | FileCheck %s
+; RUN:   | FileCheck %s -check-prefix CHECK-WIN
+
+; RUN: llc -mtriple thumbv7-windows-gnu -relocation-model pic -filetype asm -o - %s \
+; RUN:   | FileCheck %s -check-prefix CHECK-GNU
 
 @external = external global i8
 
@@ -9,8 +12,12 @@ entry:
   ret i8 %0
 }
 
-; CHECK-LABEL: return_external
-; CHECK: movw r0, :lower16:external
-; CHECK: movt r0, :upper16:external
-; CHECK: ldrb r0, [r0]
+; CHECK-WIN-LABEL: return_external
+; CHECK-WIN: movw r0, :lower16:external
+; CHECK-WIN: movt r0, :upper16:external
+; CHECK-WIN: ldrb r0, [r0]
 
+; CHECK-GNU-LABEL: return_external
+; CHECK-GNU: movw r0, :lower16:external
+; CHECK-GNU: movt r0, :upper16:external
+; CHECK-GNU: ldrb r0, [r0]
diff --git a/test/CodeGen/ARM/Windows/structors.ll b/test/CodeGen/ARM/Windows/structors.ll
index 874b5bf35b8..eff1c7f4b38 100644
--- a/test/CodeGen/ARM/Windows/structors.ll
+++ b/test/CodeGen/ARM/Windows/structors.ll
@@ -1,4 +1,8 @@
-; RUN: llc -mtriple thumbv7-windows-itanium -o - %s | FileCheck %s
+; RUN: llc -mtriple thumbv7-windows-itanium -o - %s \
+; RUN:   | FileCheck %s -check-prefix CHECK-WIN
+
+; RUN: llc -mtriple thumbv7-windows-gnu -o - %s \
+; RUN:   | FileCheck %s -check-prefix CHECK-GNU
 
 @llvm.global_ctors = appending global [1 x { i32, void ()*, i8* }] [{ i32, void ()*, i8* } { i32 65535, void ()* @function, i8* null }]
 
@@ -7,6 +11,8 @@ entry:
   ret void
 }
 
-; CHECK: .section .CRT$XCU,"dr"
-; CHECK: .long function
+; CHECK-WIN: .section .CRT$XCU,"dr"
+; CHECK-WIN: .long function
 
+; CHECK-GNU: .section .ctors,"dw"
+; CHECK-GNU: .long function
diff --git a/test/CodeGen/ARM/Windows/trivial-gnu-object.ll b/test/CodeGen/ARM/Windows/trivial-gnu-object.ll
new file mode 100644
index 00000000000..a242f39601c
--- /dev/null
+++ b/test/CodeGen/ARM/Windows/trivial-gnu-object.ll
@@ -0,0 +1,10 @@
+; RUN: llc -mtriple=thumbv7-windows-itanium -filetype=obj -o - %s | llvm-objdump -d - | FileCheck %s
+; RUN: llc -mtriple=thumbv7-windows-gnu -filetype=obj -o - %s | llvm-objdump -d - | FileCheck %s
+
+define void @foo() {
+; CHECK: file format COFF-ARM
+
+; CHECK-LABEL: foo:
+; CHECK: bx lr
+  ret void
+}
diff --git a/test/CodeGen/ARM/adv-copy-opt.ll b/test/CodeGen/ARM/adv-copy-opt.ll
index f71bf78b62c..395be345720 100644
--- a/test/CodeGen/ARM/adv-copy-opt.ll
+++ b/test/CodeGen/ARM/adv-copy-opt.ll
@@ -11,25 +11,25 @@
 ; r0 = r0 / r2
 ; r1 = r1 / r3
 ;
-; NOOPT: vmov	[[B:d[0-9]+]], r2, r3
-; NOOPT-NEXT: vmov	[[A:d[0-9]+]], r0, r1
+; NOOPT: vmov	[[A:d[0-9]+]], r0, r1
+; NOOPT-NEXT: vmov	[[B:d[0-9]+]], r2, r3
 ; Move the low part of B into a register.
 ; Unfortunately, we cannot express that the 's' register is the low
 ; part of B, i.e., sIdx == BIdx x 2. E.g., B = d1, B_low = s2.
 ; NOOPT-NEXT: vmov	[[B_LOW:r[0-9]+]], s{{[0-9]+}}
-; NOOPT-NEXT: vmov	[[A_LOW:r[0-9]+]], s{{[0-9]+}}
-; NOOPT-NEXT: udiv	[[RES_LOW:r[0-9]+]], [[A_LOW]], [[B_LOW]]
 ; NOOPT-NEXT: vmov	[[B_HIGH:r[0-9]+]], s{{[0-9]+}}
+; NOOPT-NEXT: vmov	[[A_LOW:r[0-9]+]], s{{[0-9]+}}
 ; NOOPT-NEXT: vmov	[[A_HIGH:r[0-9]+]], s{{[0-9]+}}
-; NOOPT-NEXT: udiv	[[RES_HIGH:r[0-9]+]], [[A_HIGH]], [[B_HIGH]]
+; NOOPT-NEXT: udiv	[[RES_LOW:r[0-9]+]], [[A_LOW]], [[B_LOW]]
 ; NOOPT-NEXT: vmov.32	[[RES:d[0-9]+]][0], [[RES_LOW]]
+; NOOPT-NEXT: udiv	[[RES_HIGH:r[0-9]+]], [[A_HIGH]], [[B_HIGH]]
 ; NOOPT-NEXT: vmov.32	[[RES]][1], [[RES_HIGH]]
 ; NOOPT-NEXT: vmov	r0, r1, [[RES]]
 ; NOOPT-NEXT: bx	lr
 ;
 ; OPT-NOT: vmov
-; OPT: 	udiv	r0, r0, r2
-; OPT-NEXT: udiv	r1, r1, r3
+; OPT: udiv	r1, r1, r3
+; OPT-NEXT: 	udiv	r0, r0, r2
 ; OPT-NEXT: bx	lr
 define <2 x i32> @simpleVectorDiv(<2 x i32> %A, <2 x i32> %B) nounwind {
 entry:
diff --git a/test/CodeGen/ARM/aliases.ll b/test/CodeGen/ARM/aliases.ll
index 04ca3e87548..8d0d57db44b 100644
--- a/test/CodeGen/ARM/aliases.ll
+++ b/test/CodeGen/ARM/aliases.ll
@@ -2,19 +2,34 @@
 
 ; CHECK: .globl	test
 
+; CHECK: .globl structvar
+; CHECK: .size structvar, 8
+
 ; CHECK: .globl	foo1
 ; CHECK: foo1 = bar
+; CHECK: .size foo1, 4
 
 ; CHECK: .globl	foo2
 ; CHECK: foo2 = bar
+; CHECK: .size foo2, 4
 
 ; CHECK: .weak	bar_f
 ; CHECK: bar_f = foo_f
 
 ; CHECK: bar_i = bar
+; CHECK: .size bar_i, 4
 
 ; CHECK: .globl	A
 ; CHECK: A = bar
+; CHECK: .size A, 8
+
+; CHECK: .globl elem0
+; CHECK: elem0 = structvar
+; CHECK: .size elem0, 4
+
+; CHECK: .globl elem1
+; CHECK: elem1 = structvar+4
+; CHECK: .size elem1, 4
 
 @bar = global i32 42
 @foo1 = alias i32* @bar
@@ -31,6 +46,10 @@ define i32 @foo_f() {
 
 @A = alias bitcast (i32* @bar to i64*)
 
+@structvar = global {i32, i32} {i32 1, i32 2}
+@elem0 = alias getelementptr({i32, i32}, {i32, i32}*  @structvar, i32 0, i32 0)
+@elem1 = alias getelementptr({i32, i32}, {i32, i32}*  @structvar, i32 0, i32 1)
+
 define i32 @test() {
 entry:
    %tmp = load i32, i32* @foo1
diff --git a/test/CodeGen/ARM/arm-and-tst-peephole.ll b/test/CodeGen/ARM/arm-and-tst-peephole.ll
index 7d7fae95bfc..04eae8f9afe 100644
--- a/test/CodeGen/ARM/arm-and-tst-peephole.ll
+++ b/test/CodeGen/ARM/arm-and-tst-peephole.ll
@@ -8,9 +8,9 @@
 
 %struct.Foo = type { i8* }
 
-; ARM:   foo
-; THUMB: foo
-; T2:    foo
+; ARM-LABEL:   foo:
+; THUMB-LABEL: foo:
+; T2-LABEL:    foo:
 define %struct.Foo* @foo(%struct.Foo* %this, i32 %acc) nounwind readonly align 2 {
 entry:
   %scevgep = getelementptr %struct.Foo, %struct.Foo* %this, i32 1
@@ -49,14 +49,14 @@ tailrecurse.switch:                               ; preds = %tailrecurse
 ; V8-NEXT: beq
 ; V8-NEXT: %tailrecurse.switch
 ; V8: cmp
-; V8-NEXT: beq
+; V8-NEXT: bne
 ; V8-NEXT: b	
 ; The trailing space in the last line checks that the branch is unconditional
   switch i32 %and, label %sw.epilog [
     i32 1, label %sw.bb
     i32 3, label %sw.bb6
     i32 2, label %sw.bb8
-  ]
+  ], !prof !1
 
 sw.bb:                                            ; preds = %tailrecurse.switch, %tailrecurse
   %shl = shl i32 %acc.tr, 1
@@ -83,9 +83,9 @@ sw.epilog:                                        ; preds = %tailrecurse.switch
 
 %struct.S = type { i8* (i8*)*, [1 x i8] }
 
-; ARM: bar
-; THUMB: bar
-; T2: bar
+; ARM-LABEL: bar:
+; THUMB-LABEL: bar:
+; T2-LABEL: bar:
 ; V8-LABEL: bar:
 define internal zeroext i8 @bar(%struct.S* %x, %struct.S* nocapture %y) nounwind readonly {
 entry:
@@ -134,3 +134,27 @@ bb4:                                              ; preds = %bb2
 return:                                           ; preds = %bb2, %bb, %entry
   ret i8 1
 }
+
+
+; We were looking through multiple COPY instructions to find an AND we might
+; fold into a TST, but in doing so we changed the register being tested allowing
+; folding of unrelated tests (in this case, a TST against r1 was eliminated in
+; favour of an AND of r0).
+
+; ARM-LABEL: test_tst_assessment:
+; THUMB-LABEL: test_tst_assessment:
+; T2-LABEL: test_tst_assessment:
+; V8-LABEL: test_tst_assessment:
+define i32 @test_tst_assessment(i1 %lhs, i1 %rhs) {
+  %lhs32 = zext i1 %lhs to i32
+  %rhs32 = zext i1 %rhs to i32
+  %diff = sub nsw i32 %lhs32, %rhs32
+; ARM: tst r1, #1
+; THUMB: movs [[RTMP:r[0-9]+]], #1
+; THUMB: tst r1, [[RTMP]]
+; T2: tst.w r1, #1
+; V8: tst.w r1, #1
+  ret i32 %diff
+}
+
+!1 = !{!"branch_weights", i32 1, i32 1, i32 3, i32 2 }
diff --git a/test/CodeGen/ARM/arm-interleaved-accesses.ll b/test/CodeGen/ARM/arm-interleaved-accesses.ll
new file mode 100644
index 00000000000..9a9885ccdd0
--- /dev/null
+++ b/test/CodeGen/ARM/arm-interleaved-accesses.ll
@@ -0,0 +1,204 @@
+; RUN: llc -mtriple=arm-eabi -mattr=+neon -lower-interleaved-accesses=true < %s | FileCheck %s
+
+; CHECK-LABEL: load_factor2:
+; CHECK: vld2.8 {d16, d17}, [r0]
+define <8 x i8> @load_factor2(<16 x i8>* %ptr) {
+  %wide.vec = load <16 x i8>, <16 x i8>* %ptr, align 4
+  %strided.v0 = shufflevector <16 x i8> %wide.vec, <16 x i8> undef, <8 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14>
+  %strided.v1 = shufflevector <16 x i8> %wide.vec, <16 x i8> undef, <8 x i32> <i32 1, i32 3, i32 5, i32 7, i32 9, i32 11, i32 13, i32 15>
+  %add = add nsw <8 x i8> %strided.v0, %strided.v1
+  ret <8 x i8> %add
+}
+
+; CHECK-LABEL: load_factor3:
+; CHECK: vld3.32 {d16, d17, d18}, [r0]
+define <2 x i32> @load_factor3(i32* %ptr) {
+  %base = bitcast i32* %ptr to <6 x i32>*
+  %wide.vec = load <6 x i32>, <6 x i32>* %base, align 4
+  %strided.v2 = shufflevector <6 x i32> %wide.vec, <6 x i32> undef, <2 x i32> <i32 2, i32 5>
+  %strided.v1 = shufflevector <6 x i32> %wide.vec, <6 x i32> undef, <2 x i32> <i32 1, i32 4>
+  %add = add nsw <2 x i32> %strided.v2, %strided.v1
+  ret <2 x i32> %add
+}
+
+; CHECK-LABEL: load_factor4:
+; CHECK: vld4.32 {d16, d18, d20, d22}, [r0]!
+; CHECK: vld4.32 {d17, d19, d21, d23}, [r0]
+define <4 x i32> @load_factor4(i32* %ptr) {
+  %base = bitcast i32* %ptr to <16 x i32>*
+  %wide.vec = load <16 x i32>, <16 x i32>* %base, align 4
+  %strided.v0 = shufflevector <16 x i32> %wide.vec, <16 x i32> undef, <4 x i32> <i32 0, i32 4, i32 8, i32 12>
+  %strided.v2 = shufflevector <16 x i32> %wide.vec, <16 x i32> undef, <4 x i32> <i32 2, i32 6, i32 10, i32 14>
+  %add = add nsw <4 x i32> %strided.v0, %strided.v2
+  ret <4 x i32> %add
+}
+
+; CHECK-LABEL: store_factor2:
+; CHECK: vst2.8 {d16, d17}, [r0]
+define void @store_factor2(<16 x i8>* %ptr, <8 x i8> %v0, <8 x i8> %v1) {
+  %interleaved.vec = shufflevector <8 x i8> %v0, <8 x i8> %v1, <16 x i32> <i32 0, i32 8, i32 1, i32 9, i32 2, i32 10, i32 3, i32 11, i32 4, i32 12, i32 5, i32 13, i32 6, i32 14, i32 7, i32 15>
+  store <16 x i8> %interleaved.vec, <16 x i8>* %ptr, align 4
+  ret void
+}
+
+; CHECK-LABEL: store_factor3:
+; CHECK: vst3.32 {d16, d18, d20}, [r0]!
+; CHECK: vst3.32 {d17, d19, d21}, [r0]
+define void @store_factor3(i32* %ptr, <4 x i32> %v0, <4 x i32> %v1, <4 x i32> %v2) {
+  %base = bitcast i32* %ptr to <12 x i32>*
+  %v0_v1 = shufflevector <4 x i32> %v0, <4 x i32> %v1, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+  %v2_u = shufflevector <4 x i32> %v2, <4 x i32> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef>
+  %interleaved.vec = shufflevector <8 x i32> %v0_v1, <8 x i32> %v2_u, <12 x i32> <i32 0, i32 4, i32 8, i32 1, i32 5, i32 9, i32 2, i32 6, i32 10, i32 3, i32 7, i32 11>
+  store <12 x i32> %interleaved.vec, <12 x i32>* %base, align 4
+  ret void
+}
+
+; CHECK-LABEL: store_factor4:
+; CHECK: vst4.32 {d16, d18, d20, d22}, [r0]!
+; CHECK: vst4.32 {d17, d19, d21, d23}, [r0]
+define void @store_factor4(i32* %ptr, <4 x i32> %v0, <4 x i32> %v1, <4 x i32> %v2, <4 x i32> %v3) {
+  %base = bitcast i32* %ptr to <16 x i32>*
+  %v0_v1 = shufflevector <4 x i32> %v0, <4 x i32> %v1, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+  %v2_v3 = shufflevector <4 x i32> %v2, <4 x i32> %v3, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+  %interleaved.vec = shufflevector <8 x i32> %v0_v1, <8 x i32> %v2_v3, <16 x i32> <i32 0, i32 4, i32 8, i32 12, i32 1, i32 5, i32 9, i32 13, i32 2, i32 6, i32 10, i32 14, i32 3, i32 7, i32 11, i32 15>
+  store <16 x i32> %interleaved.vec, <16 x i32>* %base, align 4
+  ret void
+}
+
+; The following cases test that interleaved access of pointer vectors can be
+; matched to ldN/stN instruction.
+
+; CHECK-LABEL: load_ptrvec_factor2:
+; CHECK: vld2.32 {d16, d17}, [r0]
+define <2 x i32*> @load_ptrvec_factor2(i32** %ptr) {
+  %base = bitcast i32** %ptr to <4 x i32*>*
+  %wide.vec = load <4 x i32*>, <4 x i32*>* %base, align 4
+  %strided.v0 = shufflevector <4 x i32*> %wide.vec, <4 x i32*> undef, <2 x i32> <i32 0, i32 2>
+  ret <2 x i32*> %strided.v0
+}
+
+; CHECK-LABEL: load_ptrvec_factor3:
+; CHECK: vld3.32 {d16, d17, d18}, [r0]
+define void @load_ptrvec_factor3(i32** %ptr, <2 x i32*>* %ptr1, <2 x i32*>* %ptr2) {
+  %base = bitcast i32** %ptr to <6 x i32*>*
+  %wide.vec = load <6 x i32*>, <6 x i32*>* %base, align 4
+  %strided.v2 = shufflevector <6 x i32*> %wide.vec, <6 x i32*> undef, <2 x i32> <i32 2, i32 5>
+  store <2 x i32*> %strided.v2, <2 x i32*>* %ptr1
+  %strided.v1 = shufflevector <6 x i32*> %wide.vec, <6 x i32*> undef, <2 x i32> <i32 1, i32 4>
+  store <2 x i32*> %strided.v1, <2 x i32*>* %ptr2
+  ret void
+}
+
+; CHECK-LABEL: load_ptrvec_factor4:
+; CHECK: vld4.32 {d16, d17, d18, d19}, [r0]
+define void @load_ptrvec_factor4(i32** %ptr, <2 x i32*>* %ptr1, <2 x i32*>* %ptr2) {
+  %base = bitcast i32** %ptr to <8 x i32*>*
+  %wide.vec = load <8 x i32*>, <8 x i32*>* %base, align 4
+  %strided.v1 = shufflevector <8 x i32*> %wide.vec, <8 x i32*> undef, <2 x i32> <i32 1, i32 5>
+  %strided.v3 = shufflevector <8 x i32*> %wide.vec, <8 x i32*> undef, <2 x i32> <i32 3, i32 7>
+  store <2 x i32*> %strided.v1, <2 x i32*>* %ptr1
+  store <2 x i32*> %strided.v3, <2 x i32*>* %ptr2
+  ret void
+}
+
+; CHECK-LABEL: store_ptrvec_factor2:
+; CHECK: vst2.32 {d16, d17}, [r0]
+define void @store_ptrvec_factor2(i32** %ptr, <2 x i32*> %v0, <2 x i32*> %v1) {
+  %base = bitcast i32** %ptr to <4 x i32*>*
+  %interleaved.vec = shufflevector <2 x i32*> %v0, <2 x i32*> %v1, <4 x i32> <i32 0, i32 2, i32 1, i32 3>
+  store <4 x i32*> %interleaved.vec, <4 x i32*>* %base, align 4
+  ret void
+}
+
+; CHECK-LABEL: store_ptrvec_factor3:
+; CHECK: vst3.32 {d16, d17, d18}, [r0]
+define void @store_ptrvec_factor3(i32** %ptr, <2 x i32*> %v0, <2 x i32*> %v1, <2 x i32*> %v2) {
+  %base = bitcast i32** %ptr to <6 x i32*>*
+  %v0_v1 = shufflevector <2 x i32*> %v0, <2 x i32*> %v1, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %v2_u = shufflevector <2 x i32*> %v2, <2 x i32*> undef, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
+  %interleaved.vec = shufflevector <4 x i32*> %v0_v1, <4 x i32*> %v2_u, <6 x i32> <i32 0, i32 2, i32 4, i32 1, i32 3, i32 5>
+  store <6 x i32*> %interleaved.vec, <6 x i32*>* %base, align 4
+  ret void
+}
+
+; CHECK-LABEL: store_ptrvec_factor4:
+; CHECK: vst4.32 {d16, d17, d18, d19}, [r0]
+define void @store_ptrvec_factor4(i32* %ptr, <2 x i32*> %v0, <2 x i32*> %v1, <2 x i32*> %v2, <2 x i32*> %v3) {
+  %base = bitcast i32* %ptr to <8 x i32*>*
+  %v0_v1 = shufflevector <2 x i32*> %v0, <2 x i32*> %v1, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %v2_v3 = shufflevector <2 x i32*> %v2, <2 x i32*> %v3, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %interleaved.vec = shufflevector <4 x i32*> %v0_v1, <4 x i32*> %v2_v3, <8 x i32> <i32 0, i32 2, i32 4, i32 6, i32 1, i32 3, i32 5, i32 7>
+  store <8 x i32*> %interleaved.vec, <8 x i32*>* %base, align 4
+  ret void
+}
+
+; Following cases check that shuffle maskes with undef indices can be matched
+; into ldN/stN instruction.
+
+; CHECK-LABEL: load_undef_mask_factor2:
+; CHECK: vld2.32 {d16, d17, d18, d19}, [r0]
+define <4 x i32> @load_undef_mask_factor2(i32* %ptr) {
+  %base = bitcast i32* %ptr to <8 x i32>*
+  %wide.vec = load <8 x i32>, <8 x i32>* %base, align 4
+  %strided.v0 = shufflevector <8 x i32> %wide.vec, <8 x i32> undef, <4 x i32> <i32 undef, i32 2, i32 undef, i32 6>
+  %strided.v1 = shufflevector <8 x i32> %wide.vec, <8 x i32> undef, <4 x i32> <i32 undef, i32 3, i32 undef, i32 7>
+  %add = add nsw <4 x i32> %strided.v0, %strided.v1
+  ret <4 x i32> %add
+}
+
+; CHECK-LABEL: load_undef_mask_factor3:
+; CHECK: vld3.32 {d16, d18, d20}, [r0]!
+; CHECK: vld3.32 {d17, d19, d21}, [r0]
+define <4 x i32> @load_undef_mask_factor3(i32* %ptr) {
+  %base = bitcast i32* %ptr to <12 x i32>*
+  %wide.vec = load <12 x i32>, <12 x i32>* %base, align 4
+  %strided.v2 = shufflevector <12 x i32> %wide.vec, <12 x i32> undef, <4 x i32> <i32 2, i32 undef, i32 undef, i32 undef>
+  %strided.v1 = shufflevector <12 x i32> %wide.vec, <12 x i32> undef, <4 x i32> <i32 1, i32 4, i32 7, i32 10>
+  %add = add nsw <4 x i32> %strided.v2, %strided.v1
+  ret <4 x i32> %add
+}
+
+; CHECK-LABEL: load_undef_mask_factor4:
+; CHECK: vld4.32 {d16, d18, d20, d22}, [r0]!
+; CHECK: vld4.32 {d17, d19, d21, d23}, [r0]
+define <4 x i32> @load_undef_mask_factor4(i32* %ptr) {
+  %base = bitcast i32* %ptr to <16 x i32>*
+  %wide.vec = load <16 x i32>, <16 x i32>* %base, align 4
+  %strided.v0 = shufflevector <16 x i32> %wide.vec, <16 x i32> undef, <4 x i32> <i32 0, i32 4, i32 undef, i32 undef>
+  %strided.v2 = shufflevector <16 x i32> %wide.vec, <16 x i32> undef, <4 x i32> <i32 2, i32 6, i32 undef, i32 undef>
+  %add = add nsw <4 x i32> %strided.v0, %strided.v2
+  ret <4 x i32> %add
+}
+
+; CHECK-LABEL: store_undef_mask_factor2:
+; CHECK: vst2.32 {d16, d17, d18, d19}, [r0]
+define void @store_undef_mask_factor2(i32* %ptr, <4 x i32> %v0, <4 x i32> %v1) {
+  %base = bitcast i32* %ptr to <8 x i32>*
+  %interleaved.vec = shufflevector <4 x i32> %v0, <4 x i32> %v1, <8 x i32> <i32 undef, i32 undef, i32 undef, i32 undef, i32 2, i32 6, i32 3, i32 7>
+  store <8 x i32> %interleaved.vec, <8 x i32>* %base, align 4
+  ret void
+}
+
+; CHECK-LABEL: store_undef_mask_factor3:
+; CHECK: vst3.32 {d16, d18, d20}, [r0]!
+; CHECK: vst3.32 {d17, d19, d21}, [r0]
+define void @store_undef_mask_factor3(i32* %ptr, <4 x i32> %v0, <4 x i32> %v1, <4 x i32> %v2) {
+  %base = bitcast i32* %ptr to <12 x i32>*
+  %v0_v1 = shufflevector <4 x i32> %v0, <4 x i32> %v1, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+  %v2_u = shufflevector <4 x i32> %v2, <4 x i32> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef>
+  %interleaved.vec = shufflevector <8 x i32> %v0_v1, <8 x i32> %v2_u, <12 x i32> <i32 0, i32 4, i32 undef, i32 1, i32 undef, i32 9, i32 2, i32 6, i32 10, i32 3, i32 7, i32 11>
+  store <12 x i32> %interleaved.vec, <12 x i32>* %base, align 4
+  ret void
+}
+
+; CHECK-LABEL: store_undef_mask_factor4:
+; CHECK: vst4.32 {d16, d18, d20, d22}, [r0]!
+; CHECK: vst4.32 {d17, d19, d21, d23}, [r0]
+define void @store_undef_mask_factor4(i32* %ptr, <4 x i32> %v0, <4 x i32> %v1, <4 x i32> %v2, <4 x i32> %v3) {
+  %base = bitcast i32* %ptr to <16 x i32>*
+  %v0_v1 = shufflevector <4 x i32> %v0, <4 x i32> %v1, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+  %v2_v3 = shufflevector <4 x i32> %v2, <4 x i32> %v3, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+  %interleaved.vec = shufflevector <8 x i32> %v0_v1, <8 x i32> %v2_v3, <16 x i32> <i32 0, i32 4, i32 8, i32 undef, i32 undef, i32 5, i32 9, i32 13, i32 2, i32 6, i32 10, i32 14, i32 3, i32 7, i32 11, i32 15>
+  store <16 x i32> %interleaved.vec, <16 x i32>* %base, align 4
+  ret void
+}
diff --git a/test/CodeGen/ARM/arm-returnaddr.ll b/test/CodeGen/ARM/arm-returnaddr.ll
index 4266572b077..26f8c67bb15 100644
--- a/test/CodeGen/ARM/arm-returnaddr.ll
+++ b/test/CodeGen/ARM/arm-returnaddr.ll
@@ -8,7 +8,6 @@
 define i8* @rt0(i32 %x) nounwind readnone {
 entry:
 ; CHECK-LABEL: rt0:
-; CHECK: {r7, lr}
 ; CHECK: mov r0, lr
   %0 = tail call i8* @llvm.returnaddress(i32 0)
   ret i8* %0
@@ -17,10 +16,9 @@ entry:
 define i8* @rt2() nounwind readnone {
 entry:
 ; CHECK-LABEL: rt2:
-; CHECK: {r7, lr}
 ; CHECK: ldr r[[R0:[0-9]+]], [r7]
-; CHECK: ldr r0, [r0]
-; CHECK: ldr r0, [r0, #4]
+; CHECK: ldr r0, [r[[R0]]]
+; CHECK: ldr r0, [r[[R0]], #4]
   %0 = tail call i8* @llvm.returnaddress(i32 2)
   ret i8* %0
 }
diff --git a/test/CodeGen/ARM/arm-shrink-wrapping.ll b/test/CodeGen/ARM/arm-shrink-wrapping.ll
new file mode 100644
index 00000000000..f033ac47309
--- /dev/null
+++ b/test/CodeGen/ARM/arm-shrink-wrapping.ll
@@ -0,0 +1,536 @@
+; RUN: llc %s -o - -enable-shrink-wrap=true -ifcvt-fn-start=1 -ifcvt-fn-stop=0 -mtriple=armv7-apple-ios \
+; RUN:      | FileCheck %s --check-prefix=CHECK --check-prefix=ARM --check-prefix=ENABLE --check-prefix=ARM-ENABLE
+; RUN: llc %s -o - -enable-shrink-wrap=false -ifcvt-fn-start=1 -ifcvt-fn-stop=0 -mtriple=armv7-apple-ios \
+; RUN:      | FileCheck %s --check-prefix=CHECK --check-prefix=ARM --check-prefix=DISABLE --check-prefix=ARM-DISABLE
+; RUN: llc %s -o - -enable-shrink-wrap=true -ifcvt-fn-start=1 -ifcvt-fn-stop=0 -mtriple=thumbv7-apple-ios \
+; RUN:      | FileCheck %s --check-prefix=CHECK --check-prefix=THUMB --check-prefix=ENABLE --check-prefix=THUMB-ENABLE
+; RUN: llc %s -o - -enable-shrink-wrap=false -ifcvt-fn-start=1 -ifcvt-fn-stop=0 -mtriple=thumbv7-apple-ios \
+; RUN:      | FileCheck %s --check-prefix=CHECK --check-prefix=THUMB --check-prefix=DISABLE --check-prefix=THUMB-DISABLE
+
+;
+; Note: Lots of tests use inline asm instead of regular calls.
+; This allows to have a better control on what the allocation will do.
+; Otherwise, we may have spill right in the entry block, defeating
+; shrink-wrapping. Moreover, some of the inline asm statements (nop)
+; are here to ensure that the related paths do not end up as critical
+; edges.
+; Also disable the late if-converter as it makes harder to reason on
+; the diffs.
+
+; Initial motivating example: Simple diamond with a call just on one side.
+; CHECK-LABEL: foo:
+;
+; Compare the arguments and jump to exit.
+; No prologue needed.
+; ENABLE: cmp r0, r1
+; ENABLE-NEXT: bge [[EXIT_LABEL:LBB[0-9_]+]]
+;
+; Prologue code.
+; CHECK: push {r7, lr}
+; CHECK-NEXT: mov r7, sp
+;;
+; Compare the arguments and jump to exit.
+; After the prologue is set.
+; DISABLE: sub sp
+; DISABLE: cmp r0, r1
+; DISABLE-NEXT: bge [[EXIT_LABEL:LBB[0-9_]+]]
+;
+; Store %a in the alloca.
+; ARM-ENABLE: push {r0}
+; THUMB-ENABLE: str r0, [sp, #-4]
+; DISABLE: str r0, [sp]
+; Set the alloca address in the second argument.
+; CHECK-NEXT: mov r1, sp
+; Set the first argument to zero.
+; CHECK-NEXT: mov{{s?}} r0, #0
+; CHECK-NEXT: bl{{x?}} _doSomething
+;
+; With shrink-wrapping, epilogue is just after the call.
+; ARM-ENABLE-NEXT: mov sp, r7
+; THUMB-ENABLE-NEXT: add sp, #4
+; ENABLE-NEXT: pop{{(\.w)?}} {r7, lr}
+;
+; CHECK: [[EXIT_LABEL]]:
+;
+; Without shrink-wrapping, epilogue is in the exit block.
+; Epilogue code. (What we pop does not matter.)
+; ARM-DISABLE: mov sp, r7
+; THUMB-DISABLE: add sp, 
+; DISABLE-NEXT: pop {r7, pc}
+;
+; ENABLE-NEXT: bx lr
+define i32 @foo(i32 %a, i32 %b) {
+  %tmp = alloca i32, align 4
+  %tmp2 = icmp slt i32 %a, %b
+  br i1 %tmp2, label %true, label %false
+
+true:
+  store i32 %a, i32* %tmp, align 4
+  %tmp4 = call i32 @doSomething(i32 0, i32* %tmp)
+  br label %false
+
+false:
+  %tmp.0 = phi i32 [ %tmp4, %true ], [ %a, %0 ]
+  ret i32 %tmp.0
+}
+
+; Function Attrs: optsize
+declare i32 @doSomething(i32, i32*)
+
+
+; Check that we do not perform the restore inside the loop whereas the save
+; is outside.
+; CHECK-LABEL: freqSaveAndRestoreOutsideLoop:
+;
+; Shrink-wrapping allows to skip the prologue in the else case.
+; ARM-ENABLE: cmp r0, #0
+; ARM-ENABLE-NEXT: beq [[ELSE_LABEL:LBB[0-9_]+]]
+; THUMB-ENABLE: cbz r0, [[ELSE_LABEL:LBB[0-9_]+]]
+;
+; Prologue code.
+; Make sure we save the CSR used in the inline asm: r4.
+; CHECK: push {r4, r7, lr}
+; CHECK-NEXT: add r7, sp, #4
+;
+; ARM-DISABLE: cmp r0, #0
+; ARM-DISABLE-NEXT: beq [[ELSE_LABEL:LBB[0-9_]+]]
+; THUMB-DISABLE: cbz r0, [[ELSE_LABEL:LBB[0-9_]+]]
+;
+; SUM is in r0 because it is coalesced with the second
+; argument on the else path.
+; CHECK: mov{{s?}} [[SUM:r0]], #0
+; CHECK-NEXT: mov{{s?}} [[IV:r[0-9]+]], #10
+;
+; Next BB.
+; CHECK: [[LOOP:LBB[0-9_]+]]: @ %for.body
+; CHECK: mov{{(\.w)?}} [[TMP:r[0-9]+]], #1
+; ARM: subs [[IV]], [[IV]], #1
+; THUMB: subs [[IV]], #1
+; ARM-NEXT: add [[SUM]], [[TMP]], [[SUM]]
+; THUMB-NEXT: add [[SUM]], [[TMP]]
+; CHECK-NEXT: bne [[LOOP]]
+;
+; Next BB.
+; SUM << 3.
+; CHECK: lsl{{s?}} [[SUM]], [[SUM]], #3
+; ENABLE-NEXT: pop {r4, r7, pc}
+;
+; Duplicated epilogue.
+; DISABLE: pop {r4, r7, pc}
+;
+; CHECK: [[ELSE_LABEL]]: @ %if.else
+; Shift second argument by one and store into returned register.
+; CHECK: lsl{{s?}} r0, r1, #1
+; DISABLE-NEXT: pop {r4, r7, pc}
+;
+; ENABLE-NEXT: bx lr
+define i32 @freqSaveAndRestoreOutsideLoop(i32 %cond, i32 %N) {
+entry:
+  %tobool = icmp eq i32 %cond, 0
+  br i1 %tobool, label %if.else, label %for.preheader
+
+for.preheader:
+  tail call void asm "nop", ""()
+  br label %for.body
+
+for.body:                                         ; preds = %entry, %for.body
+  %i.05 = phi i32 [ %inc, %for.body ], [ 0, %for.preheader ]
+  %sum.04 = phi i32 [ %add, %for.body ], [ 0, %for.preheader ]
+  %call = tail call i32 asm sideeffect "mov $0, #1", "=r,~{r4}"()
+  %add = add nsw i32 %call, %sum.04
+  %inc = add nuw nsw i32 %i.05, 1
+  %exitcond = icmp eq i32 %inc, 10
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:                                          ; preds = %for.body
+  %shl = shl i32 %add, 3
+  br label %if.end
+
+if.else:                                          ; preds = %entry
+  %mul = shl nsw i32 %N, 1
+  br label %if.end
+
+if.end:                                           ; preds = %if.else, %for.end
+  %sum.1 = phi i32 [ %shl, %for.end ], [ %mul, %if.else ]
+  ret i32 %sum.1
+}
+
+declare i32 @something(...)
+
+; Check that we do not perform the shrink-wrapping inside the loop even
+; though that would be legal. The cost model must prevent that.
+; CHECK-LABEL: freqSaveAndRestoreOutsideLoop2:
+; Prologue code.
+; Make sure we save the CSR used in the inline asm: r4.
+; CHECK: push {r4
+; CHECK: mov{{s?}} [[SUM:r0]], #0
+; CHECK-NEXT: mov{{s?}} [[IV:r[0-9]+]], #10
+; CHECK: nop
+; Next BB.
+; CHECK: [[LOOP_LABEL:LBB[0-9_]+]]: @ %for.body
+; CHECK: mov{{(\.w)?}} [[TMP:r[0-9]+]], #1
+; ARM: subs [[IV]], [[IV]], #1
+; THUMB: subs [[IV]], #1
+; ARM: add [[SUM]], [[TMP]], [[SUM]]
+; THUMB: add [[SUM]], [[TMP]]
+; CHECK-NEXT: bne [[LOOP_LABEL]]
+; Next BB.
+; CHECK: @ %for.exit
+; CHECK: nop
+; CHECK: pop {r4
+define i32 @freqSaveAndRestoreOutsideLoop2(i32 %cond) {
+entry:
+  br label %for.preheader
+
+for.preheader:
+  tail call void asm "nop", ""()
+  br label %for.body
+
+for.body:                                         ; preds = %for.body, %entry
+  %i.04 = phi i32 [ 0, %for.preheader ], [ %inc, %for.body ]
+  %sum.03 = phi i32 [ 0, %for.preheader ], [ %add, %for.body ]
+  %call = tail call i32 asm sideeffect "mov $0, #1", "=r,~{r4}"()
+  %add = add nsw i32 %call, %sum.03
+  %inc = add nuw nsw i32 %i.04, 1
+  %exitcond = icmp eq i32 %inc, 10
+  br i1 %exitcond, label %for.exit, label %for.body
+
+for.exit:
+  tail call void asm "nop", ""()
+  br label %for.end
+
+for.end:                                          ; preds = %for.body
+  ret i32 %add
+}
+
+; Check with a more complex case that we do not have save within the loop and
+; restore outside.
+; CHECK-LABEL: loopInfoSaveOutsideLoop:
+;
+; ARM-ENABLE: cmp r0, #0
+; ARM-ENABLE-NEXT: beq [[ELSE_LABEL:LBB[0-9_]+]]
+; THUMB-ENABLE: cbz r0, [[ELSE_LABEL:LBB[0-9_]+]]
+;
+; Prologue code.
+; Make sure we save the CSR used in the inline asm: r4.
+; CHECK: push {r4, r7, lr}
+; CHECK-NEXT: add r7, sp, #4
+;
+; ARM-DISABLE: cmp r0, #0
+; ARM-DISABLE-NEXT: beq [[ELSE_LABEL:LBB[0-9_]+]]
+; THUMB-DISABLE: cbz r0, [[ELSE_LABEL:LBB[0-9_]+]]
+;
+; SUM is in r0 because it is coalesced with the second
+; argument on the else path.
+; CHECK: mov{{s?}} [[SUM:r0]], #0
+; CHECK-NEXT: mov{{s?}} [[IV:r[0-9]+]], #10
+;
+; Next BB.
+; CHECK: [[LOOP:LBB[0-9_]+]]: @ %for.body
+; CHECK: mov{{(\.w)?}} [[TMP:r[0-9]+]], #1
+; ARM: subs [[IV]], [[IV]], #1
+; THUMB: subs [[IV]], #1
+; ARM-NEXT: add [[SUM]], [[TMP]], [[SUM]]
+; THUMB-NEXT: add [[SUM]], [[TMP]]
+; CHECK-NEXT: bne [[LOOP]]
+;
+; Next BB.
+; SUM << 3.
+; CHECK: lsl{{s?}} [[SUM]], [[SUM]], #3
+; ENABLE: pop {r4, r7, pc}
+;
+; Duplicated epilogue.
+; DISABLE: pop {r4, r7, pc}
+;
+; CHECK: [[ELSE_LABEL]]: @ %if.else
+; Shift second argument by one and store into returned register.
+; CHECK: lsl{{s?}} r0, r1, #1
+; DISABLE-NEXT: pop {r4, r7, pc}
+;
+; ENABLE-NEXT: bx lr
+define i32 @loopInfoSaveOutsideLoop(i32 %cond, i32 %N) {
+entry:
+  %tobool = icmp eq i32 %cond, 0
+  br i1 %tobool, label %if.else, label %for.preheader
+
+for.preheader:
+  tail call void asm "nop", ""()
+  br label %for.body
+
+for.body:                                         ; preds = %entry, %for.body
+  %i.05 = phi i32 [ %inc, %for.body ], [ 0, %for.preheader ]
+  %sum.04 = phi i32 [ %add, %for.body ], [ 0, %for.preheader ]
+  %call = tail call i32 asm sideeffect "mov $0, #1", "=r,~{r4}"()
+  %add = add nsw i32 %call, %sum.04
+  %inc = add nuw nsw i32 %i.05, 1
+  %exitcond = icmp eq i32 %inc, 10
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:                                          ; preds = %for.body
+  tail call void asm "nop", "~{r4}"()
+  %shl = shl i32 %add, 3
+  br label %if.end
+
+if.else:                                          ; preds = %entry
+  %mul = shl nsw i32 %N, 1
+  br label %if.end
+
+if.end:                                           ; preds = %if.else, %for.end
+  %sum.1 = phi i32 [ %shl, %for.end ], [ %mul, %if.else ]
+  ret i32 %sum.1
+}
+
+declare void @somethingElse(...)
+
+; Check with a more complex case that we do not have restore within the loop and
+; save outside.
+; CHECK-LABEL: loopInfoRestoreOutsideLoop:
+;
+; ARM-ENABLE: cmp r0, #0
+; ARM-ENABLE-NEXT: beq [[ELSE_LABEL:LBB[0-9_]+]]
+; THUMB-ENABLE: cbz r0, [[ELSE_LABEL:LBB[0-9_]+]]
+;
+; Prologue code.
+; Make sure we save the CSR used in the inline asm: r4.
+; CHECK: push {r4, r7, lr}
+; CHECK-NEXT: add r7, sp, #4
+;
+; ARM-DISABLE: cmp r0, #0
+; ARM-DISABLE-NEXT: beq [[ELSE_LABEL:LBB[0-9_]+]]
+; THUMB-DISABLE: cbz r0, [[ELSE_LABEL:LBB[0-9_]+]]
+;
+; SUM is in r0 because it is coalesced with the second
+; argument on the else path.
+; CHECK: mov{{s?}} [[SUM:r0]], #0
+; CHECK-NEXT: mov{{s?}} [[IV:r[0-9]+]], #10
+;
+; Next BB.
+; CHECK: [[LOOP:LBB[0-9_]+]]: @ %for.body
+; CHECK: mov{{(\.w)?}} [[TMP:r[0-9]+]], #1
+; ARM: subs [[IV]], [[IV]], #1
+; THUMB: subs [[IV]], #1
+; ARM-NEXT: add [[SUM]], [[TMP]], [[SUM]]
+; THUMB-NEXT: add [[SUM]], [[TMP]]
+; CHECK-NEXT: bne [[LOOP]]
+;
+; Next BB.
+; SUM << 3.
+; CHECK: lsl{{s?}} [[SUM]], [[SUM]], #3
+; ENABLE-NEXT: pop {r4, r7, pc}
+;
+; Duplicated epilogue.
+; DISABLE: pop {r4, r7, pc}
+;
+; CHECK: [[ELSE_LABEL]]: @ %if.else
+; Shift second argument by one and store into returned register.
+; CHECK: lsl{{s?}} r0, r1, #1
+; DISABLE-NEXT: pop {r4, r7, pc}
+;
+; ENABLE-NEXT: bx lr
+define i32 @loopInfoRestoreOutsideLoop(i32 %cond, i32 %N) #0 {
+entry:
+  %tobool = icmp eq i32 %cond, 0
+  br i1 %tobool, label %if.else, label %if.then
+
+if.then:                                          ; preds = %entry
+  tail call void asm "nop", "~{r4}"()
+  br label %for.body
+
+for.body:                                         ; preds = %for.body, %if.then
+  %i.05 = phi i32 [ 0, %if.then ], [ %inc, %for.body ]
+  %sum.04 = phi i32 [ 0, %if.then ], [ %add, %for.body ]
+  %call = tail call i32 asm sideeffect "mov $0, #1", "=r,~{r4}"()
+  %add = add nsw i32 %call, %sum.04
+  %inc = add nuw nsw i32 %i.05, 1
+  %exitcond = icmp eq i32 %inc, 10
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:                                          ; preds = %for.body
+  %shl = shl i32 %add, 3
+  br label %if.end
+
+if.else:                                          ; preds = %entry
+  %mul = shl nsw i32 %N, 1
+  br label %if.end
+
+if.end:                                           ; preds = %if.else, %for.end
+  %sum.1 = phi i32 [ %shl, %for.end ], [ %mul, %if.else ]
+  ret i32 %sum.1
+}
+
+; Check that we handle function with no frame information correctly.
+; CHECK-LABEL: emptyFrame:
+; CHECK: @ %entry
+; CHECK-NEXT: mov{{s?}} r0, #0
+; CHECK-NEXT: bx lr
+define i32 @emptyFrame() {
+entry:
+  ret i32 0
+}
+
+; Check that we handle inline asm correctly.
+; CHECK-LABEL: inlineAsm:
+;
+; ARM-ENABLE: cmp r0, #0
+; ARM-ENABLE-NEXT: beq [[ELSE_LABEL:LBB[0-9_]+]]
+; THUMB-ENABLE: cbz r0, [[ELSE_LABEL:LBB[0-9_]+]]
+;
+; Prologue code.
+; Make sure we save the CSR used in the inline asm: r4.
+; CHECK: push {r4, r7, lr}
+; CHECK-NEXT: add r7, sp, #4
+;
+; ARM-DISABLE: cmp r0, #0
+; ARM-DISABLE-NEXT: beq [[ELSE_LABEL:LBB[0-9_]+]]
+; THUMB-DISABLE: cbz r0, [[ELSE_LABEL:LBB[0-9_]+]]
+;
+; CHECK: mov{{s?}} [[IV:r[0-9]+]], #10
+;
+; Next BB.
+; CHECK: [[LOOP:LBB[0-9_]+]]: @ %for.body
+; ARM: subs [[IV]], [[IV]], #1
+; THUMB: subs [[IV]], #1
+; CHECK: add{{(\.w)?}} r4, r4, #1
+; CHECK: bne [[LOOP]]
+;
+; Next BB.
+; CHECK: mov{{s?}} r0, #0
+;
+; Duplicated epilogue.
+; DISABLE: pop {r4, r7, pc}
+;
+; CHECK: [[ELSE_LABEL]]: @ %if.else
+; Shift second argument by one and store into returned register.
+; CHECK: lsl{{s?}} r0, r1, #1
+; DISABLE-NEXT: pop {r4, r7, pc}
+;
+; ENABLE-NEXT: bx lr
+define i32 @inlineAsm(i32 %cond, i32 %N) {
+entry:
+  %tobool = icmp eq i32 %cond, 0
+  br i1 %tobool, label %if.else, label %for.preheader
+
+for.preheader:
+  tail call void asm "nop", ""()
+  br label %for.body
+
+for.body:                                         ; preds = %entry, %for.body
+  %i.03 = phi i32 [ %inc, %for.body ], [ 0, %for.preheader ]
+  tail call void asm sideeffect "add r4, #1", "~{r4}"()
+  %inc = add nuw nsw i32 %i.03, 1
+  %exitcond = icmp eq i32 %inc, 10
+  br i1 %exitcond, label %for.exit, label %for.body
+
+for.exit:
+  tail call void asm "nop", ""()
+  br label %if.end
+
+if.else:                                          ; preds = %entry
+  %mul = shl nsw i32 %N, 1
+  br label %if.end
+
+if.end:                                           ; preds = %for.body, %if.else
+  %sum.0 = phi i32 [ %mul, %if.else ], [ 0, %for.exit ]
+  ret i32 %sum.0
+}
+
+; Check that we handle calls to variadic functions correctly.
+; CHECK-LABEL: callVariadicFunc:
+;
+; ARM-ENABLE: cmp r0, #0
+; ARM-ENABLE-NEXT: beq [[ELSE_LABEL:LBB[0-9_]+]]
+; THUMB-ENABLE: cbz r0, [[ELSE_LABEL:LBB[0-9_]+]]
+;
+; Prologue code.
+; CHECK: push {r7, lr}
+; CHECK-NEXT: mov r7, sp
+; CHECK-NEXT: sub sp, {{(sp, )?}}#12
+;
+; ARM-DISABLE: cmp r0, #0
+; ARM-DISABLE-NEXT: beq [[ELSE_LABEL:LBB[0-9_]+]]
+; THUMB-DISABLE-NEXT: cbz r0, [[ELSE_LABEL:LBB[0-9_]+]]
+;
+; Setup of the varags.
+; CHECK: mov r0, r1
+; CHECK-NEXT: mov r2, r1
+; CHECK-NEXT: mov r3, r1
+; ARM-NEXT: str r1, [sp]
+; ARM-NEXT: str r1, [sp, #4]
+; THUMB-NEXT: strd r1, r1, [sp]
+; CHECK-NEXT: str r1, [sp, #8]
+; CHECK-NEXT: bl{{x?}} _someVariadicFunc
+; CHECK-NEXT: lsl{{s?}} r0, r0, #3
+; ARM-NEXT: mov sp, r7
+; THUMB-NEXT: add sp, #12
+; CHECK-NEXT: pop {r7, pc}
+;
+; CHECK: [[ELSE_LABEL]]: @ %if.else
+; Shift second argument by one and store into returned register.
+; CHECK: lsl{{s?}} r0, r1, #1
+;
+; Epilogue code.
+; ENABLE-NEXT: bx lr
+;
+; ARM-DISABLE-NEXT: mov sp, r7
+; THUMB-DISABLE-NEXT: add sp, #12
+; DISABLE-NEXT: pop {r7, pc}
+define i32 @callVariadicFunc(i32 %cond, i32 %N) {
+entry:
+  %tobool = icmp eq i32 %cond, 0
+  br i1 %tobool, label %if.else, label %if.then
+
+if.then:                                          ; preds = %entry
+  %call = tail call i32 (i32, ...) @someVariadicFunc(i32 %N, i32 %N, i32 %N, i32 %N, i32 %N, i32 %N, i32 %N)
+  %shl = shl i32 %call, 3
+  br label %if.end
+
+if.else:                                          ; preds = %entry
+  %mul = shl nsw i32 %N, 1
+  br label %if.end
+
+if.end:                                           ; preds = %if.else, %if.then
+  %sum.0 = phi i32 [ %shl, %if.then ], [ %mul, %if.else ]
+  ret i32 %sum.0
+}
+
+declare i32 @someVariadicFunc(i32, ...)
+
+; Make sure we do not insert unreachable code after noreturn function.
+; Although this is not incorrect to insert such code, it is useless
+; and it hurts the binary size.
+;
+; CHECK-LABEL: noreturn:
+; DISABLE: push
+;
+; CHECK: tst{{(\.w)?}}  r0, #255
+; CHECK-NEXT: bne      [[ABORT:LBB[0-9_]+]]
+;
+; CHECK: mov{{s?}} r0, #42
+;
+; ENABLE-NEXT: bx lr
+;
+; DISABLE-NEXT: pop
+;;
+; CHECK: [[ABORT]]: @ %if.abort
+;
+; ENABLE: push
+;
+; CHECK: bl{{x?}} _abort
+; ENABLE-NOT: pop
+define i32 @noreturn(i8 signext %bad_thing) {
+entry:
+  %tobool = icmp eq i8 %bad_thing, 0
+  br i1 %tobool, label %if.end, label %if.abort
+
+if.abort:
+  %call = tail call i32 asm sideeffect "mov $0, #1", "=r,~{r4}"()
+  tail call void @abort() #0
+  unreachable
+
+if.end:
+  ret i32 42
+}
+
+declare void @abort() #0
+
+attributes #0 = { noreturn nounwind }
diff --git a/test/CodeGen/ARM/arm-ttype-target2.ll b/test/CodeGen/ARM/arm-ttype-target2.ll
index 754a16d9087..ef19d24d7d4 100644
--- a/test/CodeGen/ARM/arm-ttype-target2.ll
+++ b/test/CodeGen/ARM/arm-ttype-target2.ll
@@ -4,13 +4,13 @@
 @_ZTS3Foo = linkonce_odr constant [5 x i8] c"3Foo\00"
 @_ZTI3Foo = linkonce_odr unnamed_addr constant { i8*, i8* } { i8* bitcast (i8** getelementptr inbounds (i8*, i8** @_ZTVN10__cxxabiv117__class_type_infoE, i32 2) to i8*), i8* getelementptr inbounds ([5 x i8], [5 x i8]* @_ZTS3Foo, i32 0, i32 0) }
 
-define i32 @main() {
+define i32 @main() personality i8* bitcast (i32 (...)* @__gxx_personality_v0 to i8*) {
 entry:
   invoke void @_Z3foov()
           to label %return unwind label %lpad
 
 lpad:                                             ; preds = %entry
-  %0 = landingpad { i8*, i32 } personality i8* bitcast (i32 (...)* @__gxx_personality_v0 to i8*)
+  %0 = landingpad { i8*, i32 }
           catch i8* bitcast ({ i8*, i8* }* @_ZTI3Foo to i8*)
   %1 = extractvalue { i8*, i32 } %0, 1
   %2 = tail call i32 @llvm.eh.typeid.for(i8* bitcast ({ i8*, i8* }* @_ZTI3Foo to i8*)) nounwind
diff --git a/test/CodeGen/ARM/atomic-16bit-nacl.ll b/test/CodeGen/ARM/atomic-16bit-nacl.ll
deleted file mode 100644
index b368e80a99c..00000000000
--- a/test/CodeGen/ARM/atomic-16bit-nacl.ll
+++ /dev/null
@@ -1,24 +0,0 @@
-; RUN: llc -mtriple=armv7 < %s | FileCheck %s
-; RUN: llc -mtriple=armv7-unknown-nacl < %s | FileCheck %s -check-prefix=NACL
-
-; Make sure NaCl doesn't generate register-register loads, which are disallowed
-; by its sandbox.
-
-@i16_large = internal global [516 x i8] undef
-
-define void @test() {
-  %i = ptrtoint [516 x i8]* @i16_large to i32
-  %a = add i32 %i, 512
-  %a.asptr = inttoptr i32 %a to i16*
-  %loaded = load atomic i16, i16* %a.asptr seq_cst, align 2
-  ret void
-}
-; CHECK-LABEL: test:
-; CHECK: ldrh	r0, [r1, r0]
-
-; NACL-LABEL: test:
-; NACL: movw	r0, :lower16:i16_large
-; NACL: movt	r0, :upper16:i16_large
-; NACL: add	r0, r0, #512
-; NACL: ldrh	r0, [r0]
-; NACL: dmb	ish
diff --git a/test/CodeGen/ARM/atomic-ops-v8.ll b/test/CodeGen/ARM/atomic-ops-v8.ll
index db5007b0758..86287c1178d 100644
--- a/test/CodeGen/ARM/atomic-ops-v8.ll
+++ b/test/CodeGen/ARM/atomic-ops-v8.ll
@@ -664,7 +664,7 @@ define void @test_atomic_load_min_i64(i64 %offset) nounwind {
 ; CHECK: movt r[[ADDR]], :upper16:var64
 
 ; CHECK: .LBB{{[0-9]+}}_1:
-; CHECK: ldaexd [[OLD1:r[0-9]+]], [[OLD2:r[0-9]+]], [r[[ADDR]]]
+; CHECK: ldaexd [[OLD1:r[0-9]+|lr]], [[OLD2:r[0-9]+|lr]], [r[[ADDR]]]
   ; r0, r1 below is a reasonable guess but could change: it certainly comes into the
   ; function there.
 ; CHECK-ARM: mov [[LOCARRY:r[0-9]+|lr]], #0
@@ -782,7 +782,7 @@ define void @test_atomic_load_max_i64(i64 %offset) nounwind {
 ; CHECK: movt r[[ADDR]], :upper16:var64
 
 ; CHECK: .LBB{{[0-9]+}}_1:
-; CHECK: ldrexd [[OLD1:r[0-9]+]], [[OLD2:r[0-9]+]], [r[[ADDR]]]
+; CHECK: ldrexd [[OLD1:r[0-9]+]], [[OLD2:r[0-9]+|lr]], [r[[ADDR]]]
   ; r0, r1 below is a reasonable guess but could change: it certainly comes into the
   ; function there.
 ; CHECK-ARM: mov [[LOCARRY:r[0-9]+|lr]], #0
@@ -900,7 +900,7 @@ define void @test_atomic_load_umin_i64(i64 %offset) nounwind {
 ; CHECK: movt r[[ADDR]], :upper16:var64
 
 ; CHECK: .LBB{{[0-9]+}}_1:
-; CHECK: ldaexd [[OLD1:r[0-9]+]], [[OLD2:r[0-9]+]], [r[[ADDR]]]
+; CHECK: ldaexd [[OLD1:r[0-9]+|lr]], [[OLD2:r[0-9]+|lr]], [r[[ADDR]]]
   ; r0, r1 below is a reasonable guess but could change: it certainly comes into the
   ; function there.
 ; CHECK-ARM: mov [[LOCARRY:r[0-9]+|lr]], #0
@@ -1018,7 +1018,7 @@ define void @test_atomic_load_umax_i64(i64 %offset) nounwind {
 ; CHECK: movt r[[ADDR]], :upper16:var64
 
 ; CHECK: .LBB{{[0-9]+}}_1:
-; CHECK: ldaexd [[OLD1:r[0-9]+]], [[OLD2:r[0-9]+]], [r[[ADDR]]]
+; CHECK: ldaexd [[OLD1:r[0-9]+|lr]], [[OLD2:r[0-9]+|lr]], [r[[ADDR]]]
   ; r0, r1 below is a reasonable guess but could change: it certainly comes into the
   ; function there.
 ; CHECK-ARM: mov [[LOCARRY:r[0-9]+|lr]], #0
@@ -1146,10 +1146,12 @@ define void @test_atomic_cmpxchg_i64(i64 %wanted, i64 %new) nounwind {
   ; function there.
 ; CHECK-LE-DAG: eor{{(\.w)?}} [[MISMATCH_LO:r[0-9]+|lr]], [[OLD1]], r0
 ; CHECK-LE-DAG: eor{{(\.w)?}} [[MISMATCH_HI:r[0-9]+|lr]], [[OLD2]], r1
-; CHECK-LE: orrs{{(\.w)?}} {{r[0-9]+}}, [[MISMATCH_LO]], [[MISMATCH_HI]]
+; CHECK-ARM-LE: orrs{{(\.w)?}} {{r[0-9]+}}, [[MISMATCH_LO]], [[MISMATCH_HI]]
+; CHECK-THUMB-LE: orrs{{(\.w)?}} {{(r[0-9]+, )?}}[[MISMATCH_HI]], [[MISMATCH_LO]]
 ; CHECK-BE-DAG: eor{{(\.w)?}} [[MISMATCH_HI:r[0-9]+|lr]], [[OLD2]], r1
 ; CHECK-BE-DAG: eor{{(\.w)?}} [[MISMATCH_LO:r[0-9]+|lr]], [[OLD1]], r0
-; CHECK-BE: orrs{{(\.w)?}} {{r[0-9]+}}, [[MISMATCH_HI]], [[MISMATCH_LO]]
+; CHECK-ARM-BE: orrs{{(\.w)?}} {{r[0-9]+}}, [[MISMATCH_HI]], [[MISMATCH_LO]]
+; CHECK-THUMB-BE: orrs{{(\.w)?}} {{(r[0-9]+, )?}}[[MISMATCH_LO]], [[MISMATCH_HI]]
 ; CHECK-NEXT: bne .LBB{{[0-9]+}}_3
 ; CHECK-NEXT: BB#2:
   ; As above, r2, r3 is a reasonable guess.
diff --git a/test/CodeGen/ARM/avoid-cpsr-rmw.ll b/test/CodeGen/ARM/avoid-cpsr-rmw.ll
index c3de07e03b6..79e8e68e2f5 100644
--- a/test/CodeGen/ARM/avoid-cpsr-rmw.ll
+++ b/test/CodeGen/ARM/avoid-cpsr-rmw.ll
@@ -1,5 +1,5 @@
-; RUN: llc < %s -mtriple=thumbv7-apple-darwin -mcpu=cortex-a9 | FileCheck %s
-; RUN: llc < %s -mtriple=thumbv7-apple-darwin -mcpu=swift     | FileCheck %s
+; RUN: llc < %s -mtriple=thumbv7-apple-darwin -mcpu=cortex-a9 | FileCheck %s --check-prefix=CHECK --check-prefix=CHECK-CORTEX
+; RUN: llc < %s -mtriple=thumbv7-apple-darwin -mcpu=swift     | FileCheck %s --check-prefix=CHECK --check-prefix=CHECK-SWIFT
 ; Avoid some 's' 16-bit instruction which partially update CPSR (and add false
 ; dependency) when it isn't dependent on last CPSR defining instruction.
 ; rdar://8928208
@@ -7,8 +7,10 @@
 define i32 @t1(i32 %a, i32 %b, i32 %c, i32 %d) nounwind readnone {
  entry:
 ; CHECK-LABEL: t1:
-; CHECK: muls [[REG:(r[0-9]+)]], r3, r2
-; CHECK-NEXT: mul  [[REG2:(r[0-9]+)]], r1, r0
+; CHECK-CORTEX: muls [[REG:(r[0-9]+)]], r3, r2
+; CHECK-CORTEX-NEXT: mul  [[REG2:(r[0-9]+)]], r1, r0
+; CHECK-SWIFT: muls  [[REG2:(r[0-9]+)]], r1, r0
+; CHECK-SWIFT-NEXT: mul [[REG:(r[0-9]+)]], r2, r3
 ; CHECK-NEXT: muls r0, [[REG]], [[REG2]]
   %0 = mul nsw i32 %a, %b
   %1 = mul nsw i32 %c, %d
@@ -21,8 +23,7 @@ define i32 @t1(i32 %a, i32 %b, i32 %c, i32 %d) nounwind readnone {
 define void @t2(i32* nocapture %ptr1, i32* %ptr2, i32 %c) nounwind {
 entry:
 ; CHECK-LABEL: t2:
-  %tobool7 = icmp eq i32* %ptr2, null
-  br i1 %tobool7, label %while.end, label %while.body
+  br label %while.body
 
 while.body:
 ; CHECK: while.body
@@ -55,8 +56,7 @@ while.end:
 define void @t3(i32* nocapture %ptr1, i32* %ptr2, i32 %c) nounwind minsize {
 entry:
 ; CHECK-LABEL: t3:
-  %tobool7 = icmp eq i32* %ptr2, null
-  br i1 %tobool7, label %while.end, label %while.body
+  br label %while.body
 
 while.body:
 ; CHECK: while.body
diff --git a/test/CodeGen/ARM/big-endian-eh-unwind.ll b/test/CodeGen/ARM/big-endian-eh-unwind.ll
index 630dfed4467..7df5f30570e 100644
--- a/test/CodeGen/ARM/big-endian-eh-unwind.ll
+++ b/test/CodeGen/ARM/big-endian-eh-unwind.ll
@@ -14,13 +14,13 @@
 ; }
 ;}
 
-define void @_Z4testii(i32 %a, i32 %b) #0 {
+define void @_Z4testii(i32 %a, i32 %b) #0 personality i8* bitcast (i32 (...)* @__gxx_personality_v0 to i8*) {
 entry:
   invoke void @_Z3fooi(i32 %a)
           to label %try.cont unwind label %lpad
 
 lpad:                                             ; preds = %entry
-  %0 = landingpad { i8*, i32 } personality i8* bitcast (i32 (...)* @__gxx_personality_v0 to i8*)
+  %0 = landingpad { i8*, i32 }
           catch i8* null
   %1 = extractvalue { i8*, i32 } %0, 0
   %2 = tail call i8* @__cxa_begin_catch(i8* %1) #2
@@ -35,7 +35,7 @@ try.cont:                                         ; preds = %entry, %invoke.cont
   ret void
 
 lpad1:                                            ; preds = %lpad
-  %3 = landingpad { i8*, i32 } personality i8* bitcast (i32 (...)* @__gxx_personality_v0 to i8*)
+  %3 = landingpad { i8*, i32 }
           cleanup
   invoke void @__cxa_end_catch()
           to label %eh.resume unwind label %terminate.lpad
@@ -44,7 +44,7 @@ eh.resume:                                        ; preds = %lpad1
   resume { i8*, i32 } %3
 
 terminate.lpad:                                   ; preds = %lpad1
-  %4 = landingpad { i8*, i32 } personality i8* bitcast (i32 (...)* @__gxx_personality_v0 to i8*)
+  %4 = landingpad { i8*, i32 }
           catch i8* null
   %5 = extractvalue { i8*, i32 } %4, 0
   tail call void @__clang_call_terminate(i8* %5) #3
diff --git a/test/CodeGen/ARM/bswap16.ll b/test/CodeGen/ARM/bswap16.ll
index be2235000be..dc0e468b72d 100644
--- a/test/CodeGen/ARM/bswap16.ll
+++ b/test/CodeGen/ARM/bswap16.ll
@@ -1,9 +1,6 @@
 ; RUN: llc -mtriple=arm-darwin  -mattr=v6 < %s | FileCheck %s
 ; RUN: llc -mtriple=thumb-darwin  -mattr=v6 < %s | FileCheck %s
 
-; @LOCALMOD Change to test that NaCl doesn't merge the pattern.
-; RUN: llc -mtriple=armv7-unknown-nacl < %s | FileCheck %s -check-prefix=NACL
-
 
 define void @test1(i16* nocapture %data) {
 entry:
@@ -16,12 +13,6 @@ entry:
   ; CHECK: ldrh r[[R1:[0-9]+]], [r0]
   ; CHECK: rev16 r[[R1]], r[[R1]]
   ; CHECK: strh r[[R1]], [r0]
-
-  ; NACL-LABEL: test1:
-  ; NACL: ldrh r[[R1:[0-9]+]], [r0]
-  ; NACL: rev r[[R1]], r[[R1]]
-  ; NACL: lsr r[[R1]], r[[R1]], #16
-  ; NACL: strh r[[R1]], [r0]
 }
 
 
@@ -34,10 +25,6 @@ entry:
   ; CHECK-LABEL: test2:
   ; CHECK: rev16 r[[R1:[0-9]+]], r1
   ; CHECK: strh r[[R1]], [r0]
-
-  ; NACL-LABEL: test2:
-  ; NACL: rev16 r[[R1:[0-9]+]], r1
-  ; NACL: strh r[[R1]], [r0]
 }
 
 
@@ -50,35 +37,6 @@ entry:
   ; CHECK-LABEL: test3:
   ; CHECK: ldrh r[[R0:[0-9]+]], [r0]
   ; CHECK: rev16 r[[R0]], r0
-
-  ; NACL-LABEL: test3:
-  ; NACL: ldrh r[[R0:[0-9]+]], [r0]
-  ; NACL: rev r[[R0]], r0
-  ; NACL: lsr r[[R0]], r0, #16
-}
-
-define i16 @test4(i32 %in) {
-  %1 = add i32 %in, 256
-  %2 = inttoptr i32 %1 to i32*
-  %3 = load i32, i32* %2, align 2
-  %4 = trunc i32 %3 to i16
-  %5 = tail call i16 @llvm.bswap.i16(i16 %4)
-  %6 = add i32 %in, 258
-  %7 = inttoptr i32 %6 to i16*
-  store i16 %5, i16* %7, align 2
-  ret i16 %5
-
-  ; CHECK-LABEL: test4:
-  ; CHECK: ldrh r[[R1:[0-9]+]], [r0, r1]
-  ; CHECK: rev16 r[[R2:[0-9]+]], r[[R1]]
-  ; CHECK: strh r[[R2]], [r0, r3]
-
-  ; NACL-LABEL: test4:
-  ; NACL: add r[[R0:[0-9]+]], r0, #256
-  ; NACL: ldrh r[[R0]], [r0]
-  ; NACL: rev r[[R0]], r[[R0]]
-  ; NACL: lsr r[[R0]], r[[R0]], #16
-  ; NACL: strh r[[R0]], [r1]
 }
 
 declare i16 @llvm.bswap.i16(i16)
diff --git a/test/CodeGen/ARM/build-attributes.ll b/test/CodeGen/ARM/build-attributes.ll
index 1982fa98ef4..11aa1950b87 100644
--- a/test/CodeGen/ARM/build-attributes.ll
+++ b/test/CodeGen/ARM/build-attributes.ll
@@ -1,17 +1,17 @@
 ; This tests that MC/asm header conversion is smooth and that the
 ; build attributes are correct
 
-; RUN: llc < %s -mtriple=thumbv5-linux-gnueabi -mcpu=xscale | FileCheck %s --check-prefix=XSCALE
-; RUN: llc < %s -mtriple=armv6-linux-gnueabi | FileCheck %s --check-prefix=V6
-; RUN: llc < %s -mtriple=armv6-linux-gnueabi  -enable-unsafe-fp-math -disable-fp-elim -enable-no-infs-fp-math -enable-no-nans-fp-math -fp-contract=fast | FileCheck %s --check-prefix=V6-FAST
-; RUN: llc < %s -mtriple=armv6-linux-gnueabi -enable-sign-dependent-rounding-fp-math | FileCheck %s --check-prefix=DYN-ROUNDING
-; RUN: llc < %s -mtriple=thumbv6m-linux-gnueabi | FileCheck %s --check-prefix=V6M
-; RUN: llc < %s -mtriple=thumbv6m-linux-gnueabi  -enable-unsafe-fp-math -disable-fp-elim -enable-no-infs-fp-math -enable-no-nans-fp-math -fp-contract=fast | FileCheck %s --check-prefix=V6M-FAST
-; RUN: llc < %s -mtriple=thumbv6sm-linux-gnueabi | FileCheck %s --check-prefix=V6M
-; RUN: llc < %s -mtriple=thumbv6sm-linux-gnueabi  -enable-unsafe-fp-math -disable-fp-elim -enable-no-infs-fp-math -enable-no-nans-fp-math -fp-contract=fast | FileCheck %s --check-prefix=V6M-FAST
-; RUN: llc < %s -mtriple=armv6-linux-gnueabi -mcpu=arm1156t2f-s | FileCheck %s --check-prefix=ARM1156T2F-S
-; RUN: llc < %s -mtriple=armv6-linux-gnueabi -mcpu=arm1156t2f-s  -enable-unsafe-fp-math -disable-fp-elim -enable-no-infs-fp-math -enable-no-nans-fp-math -fp-contract=fast  | FileCheck %s --check-prefix=ARM1156T2F-S-FAST
-; RUN: llc < %s -mtriple=armv6-linux-gnueabi -mcpu=arm1156t2f-s -enable-sign-dependent-rounding-fp-math | FileCheck %s --check-prefix=DYN-ROUNDING
+; RUN: llc < %s -mtriple=thumbv5-linux-gnueabi -mcpu=xscale -mattr=+strict-align | FileCheck %s --check-prefix=XSCALE
+; RUN: llc < %s -mtriple=armv6-linux-gnueabi -mattr=+strict-align | FileCheck %s --check-prefix=V6
+; RUN: llc < %s -mtriple=armv6-linux-gnueabi -mattr=+strict-align  -enable-unsafe-fp-math -disable-fp-elim -enable-no-infs-fp-math -enable-no-nans-fp-math -fp-contract=fast | FileCheck %s --check-prefix=V6-FAST
+; RUN: llc < %s -mtriple=armv6-linux-gnueabi -mattr=+strict-align -enable-sign-dependent-rounding-fp-math | FileCheck %s --check-prefix=DYN-ROUNDING
+; RUN: llc < %s -mtriple=thumbv6m-linux-gnueabi -mattr=+strict-align | FileCheck %s --check-prefix=V6M
+; RUN: llc < %s -mtriple=thumbv6m-linux-gnueabi -mattr=+strict-align  -enable-unsafe-fp-math -disable-fp-elim -enable-no-infs-fp-math -enable-no-nans-fp-math -fp-contract=fast | FileCheck %s --check-prefix=V6M-FAST
+; RUN: llc < %s -mtriple=thumbv6sm-linux-gnueabi -mattr=+strict-align | FileCheck %s --check-prefix=V6M
+; RUN: llc < %s -mtriple=thumbv6sm-linux-gnueabi -mattr=+strict-align -enable-unsafe-fp-math -disable-fp-elim -enable-no-infs-fp-math -enable-no-nans-fp-math -fp-contract=fast | FileCheck %s --check-prefix=V6M-FAST
+; RUN: llc < %s -mtriple=armv6-linux-gnueabi -mcpu=arm1156t2f-s -mattr=+strict-align | FileCheck %s --check-prefix=ARM1156T2F-S
+; RUN: llc < %s -mtriple=armv6-linux-gnueabi -mcpu=arm1156t2f-s -mattr=+strict-align  -enable-unsafe-fp-math -disable-fp-elim -enable-no-infs-fp-math -enable-no-nans-fp-math -fp-contract=fast  | FileCheck %s --check-prefix=ARM1156T2F-S-FAST
+; RUN: llc < %s -mtriple=armv6-linux-gnueabi -mcpu=arm1156t2f-s -mattr=+strict-align -enable-sign-dependent-rounding-fp-math | FileCheck %s --check-prefix=DYN-ROUNDING
 ; RUN: llc < %s -mtriple=thumbv7m-linux-gnueabi | FileCheck %s --check-prefix=V7M
 ; RUN: llc < %s -mtriple=thumbv7m-linux-gnueabi  -enable-unsafe-fp-math -disable-fp-elim -enable-no-infs-fp-math -enable-no-nans-fp-math -fp-contract=fast | FileCheck %s --check-prefix=V7M-FAST
 ; RUN: llc < %s -mtriple=thumbv7m-linux-gnueabi -enable-sign-dependent-rounding-fp-math | FileCheck %s --check-prefix=DYN-ROUNDING
@@ -51,19 +51,26 @@
 ; RUN: llc < %s -mtriple=armv7-linux-gnueabi -mcpu=cortex-a17  -enable-unsafe-fp-math -disable-fp-elim -enable-no-infs-fp-math -enable-no-nans-fp-math -fp-contract=fast | FileCheck %s --check-prefix=CORTEX-A17-FAST
 ; RUN: llc < %s -mtriple=armv7-linux-gnueabi -mcpu=cortex-a17 -mattr=-vfp2 | FileCheck %s --check-prefix=CORTEX-A17-NOFPU
 ; RUN: llc < %s -mtriple=armv7-linux-gnueabi -mcpu=cortex-a17 -mattr=-vfp2  -enable-unsafe-fp-math -disable-fp-elim -enable-no-infs-fp-math -enable-no-nans-fp-math -fp-contract=fast | FileCheck %s --check-prefix=CORTEX-A17-NOFPU-FAST
+
+; RUN: llc < %s -mtriple=armv7-linux-gnueabi -mattr=-neon,+vfp3,+fp16 | FileCheck %s --check-prefix=GENERIC-FPU-VFPV3-FP16
+; RUN: llc < %s -mtriple=armv7-linux-gnueabi -mattr=-neon,+vfp3,+d16,+fp16 | FileCheck %s --check-prefix=GENERIC-FPU-VFPV3-D16-FP16
+; RUN: llc < %s -mtriple=armv7-linux-gnueabi -mattr=-neon,+vfp3,+fp-only-sp,+d16 | FileCheck %s --check-prefix=GENERIC-FPU-VFPV3XD
+; RUN: llc < %s -mtriple=armv7-linux-gnueabi -mattr=-neon,+vfp3,+fp-only-sp,+d16,+fp16 | FileCheck %s --check-prefix=GENERIC-FPU-VFPV3XD-FP16
+; RUN: llc < %s -mtriple=armv7-linux-gnueabi -mattr=+neon,+fp16 | FileCheck %s --check-prefix=GENERIC-FPU-NEON-FP16
+
 ; RUN: llc < %s -mtriple=armv7-linux-gnueabi -mcpu=cortex-a17 -enable-sign-dependent-rounding-fp-math | FileCheck %s --check-prefix=DYN-ROUNDING
-; RUN: llc < %s -mtriple=thumbv6m-linux-gnueabi -mcpu=cortex-m0 | FileCheck %s --check-prefix=CORTEX-M0
-; RUN: llc < %s -mtriple=thumbv6m-linux-gnueabi -mcpu=cortex-m0  -enable-unsafe-fp-math -disable-fp-elim -enable-no-infs-fp-math -enable-no-nans-fp-math -fp-contract=fast | FileCheck %s --check-prefix=CORTEX-M0-FAST
-; RUN: llc < %s -mtriple=thumbv6m-linux-gnueabi -mcpu=cortex-m0 -enable-sign-dependent-rounding-fp-math | FileCheck %s --check-prefix=DYN-ROUNDING
-; RUN: llc < %s -mtriple=thumbv6m-linux-gnueabi -mcpu=cortex-m0plus | FileCheck %s --check-prefix=CORTEX-M0PLUS
-; RUN: llc < %s -mtriple=thumbv6m-linux-gnueabi -mcpu=cortex-m0plus  -enable-unsafe-fp-math -disable-fp-elim -enable-no-infs-fp-math -enable-no-nans-fp-math -fp-contract=fast | FileCheck %s --check-prefix=CORTEX-M0PLUS-FAST
-; RUN: llc < %s -mtriple=thumbv6m-linux-gnueabi -mcpu=cortex-m0plus -enable-sign-dependent-rounding-fp-math | FileCheck %s --check-prefix=DYN-ROUNDING
-; RUN: llc < %s -mtriple=thumbv6m-linux-gnueabi -mcpu=cortex-m1 | FileCheck %s --check-prefix=CORTEX-M1
-; RUN: llc < %s -mtriple=thumbv6m-linux-gnueabi -mcpu=cortex-m1  -enable-unsafe-fp-math -disable-fp-elim -enable-no-infs-fp-math -enable-no-nans-fp-math -fp-contract=fast | FileCheck %s --check-prefix=CORTEX-M1-FAST
-; RUN: llc < %s -mtriple=thumbv6m-linux-gnueabi -mcpu=cortex-m1 -enable-sign-dependent-rounding-fp-math | FileCheck %s --check-prefix=DYN-ROUNDING
-; RUN: llc < %s -mtriple=thumbv6m-linux-gnueabi -mcpu=sc000 | FileCheck %s --check-prefix=SC000
-; RUN: llc < %s -mtriple=thumbv6m-linux-gnueabi -mcpu=sc000  -enable-unsafe-fp-math -disable-fp-elim -enable-no-infs-fp-math -enable-no-nans-fp-math -fp-contract=fast | FileCheck %s --check-prefix=SC000-FAST
-; RUN: llc < %s -mtriple=thumbv6m-linux-gnueabi -mcpu=sc000 -enable-sign-dependent-rounding-fp-math | FileCheck %s --check-prefix=DYN-ROUNDING
+; RUN: llc < %s -mtriple=thumbv6m-linux-gnueabi -mcpu=cortex-m0 -mattr=+strict-align | FileCheck %s --check-prefix=CORTEX-M0
+; RUN: llc < %s -mtriple=thumbv6m-linux-gnueabi -mcpu=cortex-m0 -mattr=+strict-align  -enable-unsafe-fp-math -disable-fp-elim -enable-no-infs-fp-math -enable-no-nans-fp-math -fp-contract=fast | FileCheck %s --check-prefix=CORTEX-M0-FAST
+; RUN: llc < %s -mtriple=thumbv6m-linux-gnueabi -mcpu=cortex-m0 -mattr=+strict-align -enable-sign-dependent-rounding-fp-math | FileCheck %s --check-prefix=DYN-ROUNDING
+; RUN: llc < %s -mtriple=thumbv6m-linux-gnueabi -mcpu=cortex-m0plus -mattr=+strict-align | FileCheck %s --check-prefix=CORTEX-M0PLUS
+; RUN: llc < %s -mtriple=thumbv6m-linux-gnueabi -mcpu=cortex-m0plus -mattr=+strict-align  -enable-unsafe-fp-math -disable-fp-elim -enable-no-infs-fp-math -enable-no-nans-fp-math -fp-contract=fast | FileCheck %s --check-prefix=CORTEX-M0PLUS-FAST
+; RUN: llc < %s -mtriple=thumbv6m-linux-gnueabi -mcpu=cortex-m0plus -mattr=+strict-align -enable-sign-dependent-rounding-fp-math | FileCheck %s --check-prefix=DYN-ROUNDING
+; RUN: llc < %s -mtriple=thumbv6m-linux-gnueabi -mcpu=cortex-m1 -mattr=+strict-align | FileCheck %s --check-prefix=CORTEX-M1
+; RUN: llc < %s -mtriple=thumbv6m-linux-gnueabi -mcpu=cortex-m1 -mattr=+strict-align  -enable-unsafe-fp-math -disable-fp-elim -enable-no-infs-fp-math -enable-no-nans-fp-math -fp-contract=fast | FileCheck %s --check-prefix=CORTEX-M1-FAST
+; RUN: llc < %s -mtriple=thumbv6m-linux-gnueabi -mcpu=cortex-m1 -mattr=+strict-align -enable-sign-dependent-rounding-fp-math | FileCheck %s --check-prefix=DYN-ROUNDING
+; RUN: llc < %s -mtriple=thumbv6m-linux-gnueabi -mcpu=sc000 -mattr=+strict-align | FileCheck %s --check-prefix=SC000
+; RUN: llc < %s -mtriple=thumbv6m-linux-gnueabi -mcpu=sc000 -mattr=+strict-align  -enable-unsafe-fp-math -disable-fp-elim -enable-no-infs-fp-math -enable-no-nans-fp-math -fp-contract=fast | FileCheck %s --check-prefix=SC000-FAST
+; RUN: llc < %s -mtriple=thumbv6m-linux-gnueabi -mcpu=sc000 -mattr=+strict-align -enable-sign-dependent-rounding-fp-math | FileCheck %s --check-prefix=DYN-ROUNDING
 ; RUN: llc < %s -mtriple=thumbv7m-linux-gnueabi -mcpu=cortex-m3 | FileCheck %s --check-prefix=CORTEX-M3
 ; RUN: llc < %s -mtriple=thumbv7m-linux-gnueabi -mcpu=cortex-m3  -enable-unsafe-fp-math -disable-fp-elim -enable-no-infs-fp-math -enable-no-nans-fp-math -fp-contract=fast | FileCheck %s --check-prefix=CORTEX-M3-FAST
 ; RUN: llc < %s -mtriple=thumbv7m-linux-gnueabi -mcpu=cortex-m3 -enable-sign-dependent-rounding-fp-math | FileCheck %s --check-prefix=DYN-ROUNDING
@@ -103,64 +110,54 @@
 ; RUN: llc < %s -mtriple=armv8.1a-linux-gnueabi -enable-sign-dependent-rounding-fp-math | FileCheck %s --check-prefix=DYN-ROUNDING
 ; RUN: llc < %s -mtriple=armv7-none-linux-gnueabi -mcpu=cortex-a7 | FileCheck %s  --check-prefix=CORTEX-A7-CHECK
 ; RUN: llc < %s -mtriple=armv7-none-linux-gnueabi -mcpu=cortex-a7  -enable-unsafe-fp-math -disable-fp-elim -enable-no-infs-fp-math -enable-no-nans-fp-math -fp-contract=fast | FileCheck %s  --check-prefix=CORTEX-A7-CHECK-FAST
-; RUN: llc < %s -mtriple=armv7-none-linux-gnueabi -mcpu=cortex-a7 -mattr=-vfp2,-vfp3,-vfp4,-neon | FileCheck %s --check-prefix=CORTEX-A7-NOFPU
-; RUN: llc < %s -mtriple=armv7-none-linux-gnueabi -mcpu=cortex-a7 -mattr=-vfp2,-vfp3,-vfp4,-neon  -enable-unsafe-fp-math -disable-fp-elim -enable-no-infs-fp-math -enable-no-nans-fp-math -fp-contract=fast | FileCheck %s --check-prefix=CORTEX-A7-NOFPU-FAST
+; RUN: llc < %s -mtriple=armv7-none-linux-gnueabi -mcpu=cortex-a7 -mattr=-vfp2,-vfp3,-vfp4,-neon,-fp16 | FileCheck %s --check-prefix=CORTEX-A7-NOFPU
+; RUN: llc < %s -mtriple=armv7-none-linux-gnueabi -mcpu=cortex-a7 -mattr=-vfp2,-vfp3,-vfp4,-neon,-fp16  -enable-unsafe-fp-math -disable-fp-elim -enable-no-infs-fp-math -enable-no-nans-fp-math -fp-contract=fast | FileCheck %s --check-prefix=CORTEX-A7-NOFPU-FAST
 ; RUN: llc < %s -mtriple=armv7-none-linux-gnueabi -mcpu=cortex-a7 -mattr=+vfp4,-neon | FileCheck %s --check-prefix=CORTEX-A7-FPUV4
 ; RUN: llc < %s -mtriple=armv7-none-linux-gnueabi -mcpu=cortex-a7 -enable-sign-dependent-rounding-fp-math | FileCheck %s --check-prefix=DYN-ROUNDING
 ; RUN: llc < %s -mtriple=armv7-none-linux-gnueabi -mcpu=cortex-a7 -mattr=+vfp4,-neon  -enable-unsafe-fp-math -disable-fp-elim -enable-no-infs-fp-math -enable-no-nans-fp-math -fp-contract=fast | FileCheck %s --check-prefix=CORTEX-A7-FPUV4-FAST
 ; RUN: llc < %s -mtriple=armv7-none-linux-gnueabi -mcpu=cortex-a7 -mattr=+vfp4,,+d16,-neon | FileCheck %s --check-prefix=CORTEX-A7-FPUV4
-; RUN: llc < %s -mtriple=arm-none-linux-gnueabi -relocation-model=pic | FileCheck %s --check-prefix=RELOC-PIC
-; RUN: llc < %s -mtriple=arm-none-linux-gnueabi -relocation-model=static | FileCheck %s --check-prefix=RELOC-OTHER
-; RUN: llc < %s -mtriple=arm-none-linux-gnueabi -relocation-model=default | FileCheck %s --check-prefix=RELOC-OTHER
-; RUN: llc < %s -mtriple=arm-none-linux-gnueabi -relocation-model=dynamic-no-pic | FileCheck %s --check-prefix=RELOC-OTHER
-; RUN: llc < %s -mtriple=arm-none-linux-gnueabi | FileCheck %s --check-prefix=RELOC-OTHER
-; RUN: llc < %s -mtriple=arm-none-linux-gnueabi | FileCheck %s --check-prefix=PCS-R9-USE
-; RUN: llc < %s -mtriple=arm-none-linux-gnueabi -arm-reserve-r9 | FileCheck %s --check-prefix=PCS-R9-RESERVE
+; RUN: llc < %s -mtriple=arm-none-linux-gnueabi -mattr=+strict-align -relocation-model=pic | FileCheck %s --check-prefix=RELOC-PIC
+; RUN: llc < %s -mtriple=arm-none-linux-gnueabi -mattr=+strict-align -relocation-model=static | FileCheck %s --check-prefix=RELOC-OTHER
+; RUN: llc < %s -mtriple=arm-none-linux-gnueabi -mattr=+strict-align -relocation-model=default | FileCheck %s --check-prefix=RELOC-OTHER
+; RUN: llc < %s -mtriple=arm-none-linux-gnueabi -mattr=+strict-align -relocation-model=dynamic-no-pic | FileCheck %s --check-prefix=RELOC-OTHER
+; RUN: llc < %s -mtriple=arm-none-linux-gnueabi -mattr=+strict-align | FileCheck %s --check-prefix=RELOC-OTHER
+; RUN: llc < %s -mtriple=arm-none-linux-gnueabi -mattr=+strict-align | FileCheck %s --check-prefix=PCS-R9-USE
+; RUN: llc < %s -mtriple=arm-none-linux-gnueabi -mattr=+reserve-r9,+strict-align | FileCheck %s --check-prefix=PCS-R9-RESERVE
 
 ; ARMv8.1a (AArch32)
-; RUN: llc < %s -mtriple=armv8.1a-none-linux-gnueabi -arm-no-strict-align | FileCheck %s --check-prefix=NO-STRICT-ALIGN
-; RUN: llc < %s -mtriple=armv8.1a-none-linux-gnueabi -arm-strict-align | FileCheck %s --check-prefix=STRICT-ALIGN
+; RUN: llc < %s -mtriple=armv8.1a-none-linux-gnueabi | FileCheck %s --check-prefix=NO-STRICT-ALIGN
+; RUN: llc < %s -mtriple=armv8.1a-none-linux-gnueabi -mattr=+strict-align | FileCheck %s --check-prefix=STRICT-ALIGN
 ; RUN: llc < %s -mtriple=armv8.1a-none-linux-gnueabi | FileCheck %s --check-prefix=NO-STRICT-ALIGN
 ; ARMv8a (AArch32)
-; RUN: llc < %s -mtriple=armv8-none-linux-gnueabi -mcpu=cortex-a57 -arm-no-strict-align | FileCheck %s --check-prefix=NO-STRICT-ALIGN
-; RUN: llc < %s -mtriple=armv8-none-linux-gnueabi -mcpu=cortex-a57 -arm-strict-align | FileCheck %s --check-prefix=STRICT-ALIGN
 ; RUN: llc < %s -mtriple=armv8-none-linux-gnueabi -mcpu=cortex-a57 | FileCheck %s --check-prefix=NO-STRICT-ALIGN
-; RUN: llc < %s -mtriple=armv8-none-linux-gnueabi -mcpu=cortex-a72 -arm-no-strict-align | FileCheck %s --check-prefix=NO-STRICT-ALIGN
-; RUN: llc < %s -mtriple=armv8-none-linux-gnueabi -mcpu=cortex-a72 -arm-strict-align | FileCheck %s --check-prefix=STRICT-ALIGN
+; RUN: llc < %s -mtriple=armv8-none-linux-gnueabi -mcpu=cortex-a57 -mattr=+strict-align | FileCheck %s --check-prefix=STRICT-ALIGN
 ; RUN: llc < %s -mtriple=armv8-none-linux-gnueabi -mcpu=cortex-a72 | FileCheck %s --check-prefix=NO-STRICT-ALIGN
+; RUN: llc < %s -mtriple=armv8-none-linux-gnueabi -mcpu=cortex-a72 -mattr=+strict-align | FileCheck %s --check-prefix=STRICT-ALIGN
 ; ARMv7a
-; RUN: llc < %s -mtriple=armv7-none-linux-gnueabi -mcpu=cortex-a7 -arm-no-strict-align | FileCheck %s --check-prefix=NO-STRICT-ALIGN
-; RUN: llc < %s -mtriple=armv7-none-linux-gnueabi -mcpu=cortex-a7 -arm-strict-align | FileCheck %s --check-prefix=STRICT-ALIGN
 ; RUN: llc < %s -mtriple=armv7-none-linux-gnueabi -mcpu=cortex-a7 | FileCheck %s --check-prefix=NO-STRICT-ALIGN
+; RUN: llc < %s -mtriple=armv7-none-linux-gnueabi -mcpu=cortex-a7 -mattr=+strict-align | FileCheck %s --check-prefix=STRICT-ALIGN
 ; ARMv7r
-; RUN: llc < %s -mtriple=armv7r-none-linux-gnueabi -mcpu=cortex-r5 -arm-no-strict-align | FileCheck %s --check-prefix=NO-STRICT-ALIGN
-; RUN: llc < %s -mtriple=armv7r-none-linux-gnueabi -mcpu=cortex-r5 -arm-strict-align | FileCheck %s --check-prefix=STRICT-ALIGN
 ; RUN: llc < %s -mtriple=armv7r-none-linux-gnueabi -mcpu=cortex-r5 | FileCheck %s --check-prefix=NO-STRICT-ALIGN
+; RUN: llc < %s -mtriple=armv7r-none-linux-gnueabi -mcpu=cortex-r5 -mattr=+strict-align | FileCheck %s --check-prefix=STRICT-ALIGN
 ; ARMv7m
-; RUN: llc < %s -mtriple=thumbv7m-none-linux-gnueabi -mcpu=cortex-m3 -arm-no-strict-align | FileCheck %s --check-prefix=NO-STRICT-ALIGN
-; RUN: llc < %s -mtriple=thumbv7m-none-linux-gnueabi -mcpu=cortex-m3 -arm-strict-align | FileCheck %s --check-prefix=STRICT-ALIGN
 ; RUN: llc < %s -mtriple=thumbv7m-none-linux-gnueabi -mcpu=cortex-m3 | FileCheck %s --check-prefix=NO-STRICT-ALIGN
+; RUN: llc < %s -mtriple=thumbv7m-none-linux-gnueabi -mcpu=cortex-m3 -mattr=+strict-align | FileCheck %s --check-prefix=STRICT-ALIGN
 ; ARMv6
 ; RUN: llc < %s -mtriple=armv6-none-netbsd-gnueabi -mcpu=arm1136j-s | FileCheck %s --check-prefix=NO-STRICT-ALIGN
-; RUN: llc < %s -mtriple=armv6-none-linux-gnueabi -mcpu=arm1136j-s | FileCheck %s --check-prefix=STRICT-ALIGN
-; RUN: llc < %s -mtriple=armv6-none-linux-gnueabi -mcpu=arm1136j-s -arm-no-strict-align | FileCheck %s --check-prefix=NO-STRICT-ALIGN
-; RUN: llc < %s -mtriple=armv6-none-linux-gnueabi -mcpu=arm1136j-s -arm-strict-align | FileCheck %s --check-prefix=STRICT-ALIGN
+; RUN: llc < %s -mtriple=armv6-none-linux-gnueabi -mcpu=arm1136j-s -mattr=+strict-align | FileCheck %s --check-prefix=STRICT-ALIGN
+; RUN: llc < %s -mtriple=armv6-none-linux-gnueabi -mcpu=arm1136j-s | FileCheck %s --check-prefix=NO-STRICT-ALIGN
 ; ARMv6k
 ; RUN: llc < %s -mtriple=armv6k-none-netbsd-gnueabi -mcpu=arm1176j-s | FileCheck %s --check-prefix=NO-STRICT-ALIGN
-; RUN: llc < %s -mtriple=armv6k-none-linux-gnueabi -mcpu=arm1176j-s | FileCheck %s --check-prefix=STRICT-ALIGN
-; RUN: llc < %s -mtriple=armv6k-none-linux-gnueabi -mcpu=arm1176j-s -arm-no-strict-align | FileCheck %s --check-prefix=NO-STRICT-ALIGN
-; RUN: llc < %s -mtriple=armv6k-none-linux-gnueabi -mcpu=arm1176j-s -arm-strict-align | FileCheck %s --check-prefix=STRICT-ALIGN
+; RUN: llc < %s -mtriple=armv6k-none-linux-gnueabi -mcpu=arm1176j-s -mattr=+strict-align | FileCheck %s --check-prefix=STRICT-ALIGN
+; RUN: llc < %s -mtriple=armv6k-none-linux-gnueabi -mcpu=arm1176j-s | FileCheck %s --check-prefix=NO-STRICT-ALIGN
 ; ARMv6m
-; RUN: llc < %s -mtriple=thumb-none-linux-gnueabi -arm-no-strict-align -mcpu=cortex-m0 | FileCheck %s --check-prefix=STRICT-ALIGN
-; RUN: llc < %s -mtriple=thumb-none-linux-gnueabi -arm-strict-align -mcpu=cortex-m0 | FileCheck %s --check-prefix=STRICT-ALIGN
-; RUN: llc < %s -mtriple=thumbv6m-none-linux-gnueabi -arm-no-strict-align | FileCheck %s --check-prefix=STRICT-ALIGN
-; RUN: llc < %s -mtriple=thumbv6m-none-linux-gnueabi -arm-strict-align | FileCheck %s --check-prefix=STRICT-ALIGN
-; RUN: llc < %s -mtriple=thumb-none-linux-gnueabi -mcpu=cortex-m0 | FileCheck %s --check-prefix=STRICT-ALIGN
-; RUN: llc < %s -mtriple=thumb-none-linux-gnueabi -mcpu=cortex-m0 | FileCheck %s --check-prefix=STRICT-ALIGN
+; RUN: llc < %s -mtriple=thumb-none-linux-gnueabi -mcpu=cortex-m0 -mattr=+strict-align | FileCheck %s --check-prefix=STRICT-ALIGN
+; RUN: llc < %s -mtriple=thumb-none-linux-gnueabi -mattr=+strict-align -mcpu=cortex-m0 | FileCheck %s --check-prefix=STRICT-ALIGN
+; RUN: llc < %s -mtriple=thumbv6m-none-linux-gnueabi -mattr=+strict-align | FileCheck %s --check-prefix=STRICT-ALIGN
+; RUN: llc < %s -mtriple=thumb-none-linux-gnueabi -mcpu=cortex-m0 -mattr=+strict-align | FileCheck %s --check-prefix=STRICT-ALIGN
 ; ARMv5
-; RUN: llc < %s -mtriple=armv5-none-linux-gnueabi -mcpu=arm1022e -arm-no-strict-align | FileCheck %s --check-prefix=NO-STRICT-ALIGN
-; RUN: llc < %s -mtriple=armv5-none-linux-gnueabi -mcpu=arm1022e -arm-strict-align | FileCheck %s --check-prefix=STRICT-ALIGN
-; RUN: llc < %s -mtriple=armv5-none-linux-gnueabi -mcpu=arm1022e | FileCheck %s --check-prefix=STRICT-ALIGN
+; RUN: llc < %s -mtriple=armv5-none-linux-gnueabi -mcpu=arm1022e | FileCheck %s --check-prefix=NO-STRICT-ALIGN
+; RUN: llc < %s -mtriple=armv5-none-linux-gnueabi -mcpu=arm1022e -mattr=+strict-align | FileCheck %s --check-prefix=STRICT-ALIGN
 
 ; XSCALE:      .eabi_attribute 6, 5
 ; XSCALE:      .eabi_attribute 8, 1
@@ -436,7 +433,7 @@
 
 ; Tag_FP_HP_extension
 ; CORTEX-A7-CHECK: .eabi_attribute      36, 1
-; CORTEX-A7-NOFPU: .eabi_attribute      36, 1
+; CORTEX-A7-NOFPU-NOT: .eabi_attribute  36
 ; CORTEX-A7-FPUV4: .eabi_attribute      36, 1
 
 ; Tag_FP_16bit_format
@@ -741,6 +738,7 @@
 ; CORTEX-M0:  .eabi_attribute 21, 1
 ; CORTEX-M0-NOT:  .eabi_attribute 22
 ; CORTEX-M0:  .eabi_attribute 23, 3
+; CORTEX-M0: .eabi_attribute 34, 0
 ; CORTEX-M0:  .eabi_attribute 24, 1
 ; CORTEX-M0:  .eabi_attribute 25, 1
 ; CORTEX-M0-NOT:  .eabi_attribute 27
@@ -923,7 +921,7 @@
 ; CORTEX-M4-SOFT:  .eabi_attribute 7, 77
 ; CORTEX-M4-SOFT:  .eabi_attribute 8, 0
 ; CORTEX-M4-SOFT:  .eabi_attribute 9, 2
-; CORTEX-M4-SOFT:  .fpu vfpv4-d16
+; CORTEX-M4-SOFT:  .fpu fpv4-sp-d16
 ; CORTEX-M4-SOFT-NOT:   .eabi_attribute 19
 ;; We default to IEEE 754 compliance
 ; CORTEX-M4-SOFT:  .eabi_attribute 20, 1
@@ -953,7 +951,7 @@
 ; CORTEX-M4-HARD:  .eabi_attribute 7, 77
 ; CORTEX-M4-HARD:  .eabi_attribute 8, 0
 ; CORTEX-M4-HARD:  .eabi_attribute 9, 2
-; CORTEX-M4-HARD:  .fpu vfpv4-d16
+; CORTEX-M4-HARD:  .fpu fpv4-sp-d16
 ; CORTEX-M4-HARD-NOT:   .eabi_attribute 19
 ;; We default to IEEE 754 compliance
 ; CORTEX-M4-HARD:  .eabi_attribute 20, 1
@@ -984,7 +982,7 @@
 ; CORTEX-M7:  .eabi_attribute 8, 0
 ; CORTEX-M7:  .eabi_attribute 9, 2
 ; CORTEX-M7-SOFT-NOT: .fpu
-; CORTEX-M7-SINGLE:  .fpu fpv5-d16
+; CORTEX-M7-SINGLE:  .fpu fpv5-sp-d16
 ; CORTEX-M7-DOUBLE:  .fpu fpv5-d16
 ; CORTEX-M7:  .eabi_attribute 17, 1
 ; CORTEX-M7-NOT:   .eabi_attribute 19
@@ -1049,7 +1047,7 @@
 ; CORTEX-R4F:  .eabi_attribute 23, 3
 ; CORTEX-R4F:  .eabi_attribute 24, 1
 ; CORTEX-R4F:  .eabi_attribute 25, 1
-; CORTEX-R4F:  .eabi_attribute 27, 1
+; CORTEX-R4F-NOT:  .eabi_attribute 27, 1
 ; CORTEX-R4F-NOT:  .eabi_attribute 28
 ; CORTEX-R4F-NOT:  .eabi_attribute 36
 ; CORTEX-R4F:  .eabi_attribute 38, 1
@@ -1071,7 +1069,7 @@
 ; CORTEX-R5:  .eabi_attribute 23, 3
 ; CORTEX-R5:  .eabi_attribute 24, 1
 ; CORTEX-R5:  .eabi_attribute 25, 1
-; CORTEX-R5:  .eabi_attribute 27, 1
+; CORTEX-R5-NOT:  .eabi_attribute 27, 1
 ; CORTEX-R5-NOT:  .eabi_attribute 28
 ; CORTEX-R5-NOT:  .eabi_attribute 36
 ; CORTEX-R5:  .eabi_attribute 38, 1
@@ -1091,7 +1089,7 @@
 ; CORTEX-R7:  .eabi_attribute 7, 82
 ; CORTEX-R7:  .eabi_attribute 8, 1
 ; CORTEX-R7:  .eabi_attribute 9, 2
-; CORTEX-R7:  .fpu vfpv3-d16
+; CORTEX-R7:  .fpu vfpv3xd
 ; CORTEX-R7-NOT:   .eabi_attribute 19
 ;; We default to IEEE 754 compliance
 ; CORTEX-R7:  .eabi_attribute 20, 1
@@ -1205,6 +1203,12 @@
 ; CORTEX-A72-FAST-NOT:  .eabi_attribute 22
 ; CORTEX-A72-FAST:  .eabi_attribute 23, 1
 
+; GENERIC-FPU-VFPV3-FP16: .fpu vfpv3-fp16
+; GENERIC-FPU-VFPV3-D16-FP16: .fpu vfpv3-d16-fp16
+; GENERIC-FPU-VFPV3XD: .fpu vfpv3xd
+; GENERIC-FPU-VFPV3XD-FP16: .fpu vfpv3xd-fp16
+; GENERIC-FPU-NEON-FP16: .fpu neon-fp16
+
 ; GENERIC-ARMV8_1-A:  .eabi_attribute 6, 14
 ; GENERIC-ARMV8_1-A:  .eabi_attribute 7, 65
 ; GENERIC-ARMV8_1-A:  .eabi_attribute 8, 1
diff --git a/test/CodeGen/ARM/byval-align.ll b/test/CodeGen/ARM/byval-align.ll
index a26b5a79575..8a506280dd5 100644
--- a/test/CodeGen/ARM/byval-align.ll
+++ b/test/CodeGen/ARM/byval-align.ll
@@ -28,8 +28,7 @@ define i32 @test_align8(i8*, [4 x i32]* byval align 8 %b) {
 ; CHECK: push {r4, r7, lr}
 ; CHECK: add r7, sp, #4
 
-; CHECK-DAG: str r2, [r7, #8]
-; CHECK-DAG: str r3, [r7, #12]
+; CHECK: strd r2, r3, [r7, #8]
 
 ; CHECK: ldr r0, [r7, #8]
 
diff --git a/test/CodeGen/ARM/cmpxchg-idioms.ll b/test/CodeGen/ARM/cmpxchg-idioms.ll
index fb88575cab3..81e05acfef7 100644
--- a/test/CodeGen/ARM/cmpxchg-idioms.ll
+++ b/test/CodeGen/ARM/cmpxchg-idioms.ll
@@ -15,14 +15,14 @@ define i32 @test_return(i32* %p, i32 %oldval, i32 %newval) {
 ; CHECK: bne [[LOOP]]
 
 ; CHECK-NOT: cmp {{r[0-9]+}}, {{r[0-9]+}}
-; CHECK: movs r0, #1
 ; CHECK: dmb ish
+; CHECK: movs r0, #1
 ; CHECK: bx lr
 
 ; CHECK: [[FAILED]]:
 ; CHECK-NOT: cmp {{r[0-9]+}}, {{r[0-9]+}}
-; CHECK: movs r0, #0
 ; CHECK: dmb ish
+; CHECK: movs r0, #0
 ; CHECK: bx lr
 
   %pair = cmpxchg i32* %p, i32 %oldval, i32 %newval seq_cst seq_cst
@@ -34,8 +34,8 @@ define i32 @test_return(i32* %p, i32 %oldval, i32 %newval) {
 define i1 @test_return_bool(i8* %value, i8 %oldValue, i8 %newValue) {
 ; CHECK-LABEL: test_return_bool:
 
-; CHECK: uxtb [[OLDBYTE:r[0-9]+]], r1
 ; CHECK: dmb ishst
+; CHECK: uxtb [[OLDBYTE:r[0-9]+]], r1
 
 ; CHECK: [[LOOP:LBB[0-9]+_[0-9]+]]:
 ; CHECK: ldrexb [[LOADED:r[0-9]+]], [r0]
diff --git a/test/CodeGen/ARM/coalesce-dbgvalue.ll b/test/CodeGen/ARM/coalesce-dbgvalue.ll
index 9ec9c6ce1cc..cd2ab257207 100644
--- a/test/CodeGen/ARM/coalesce-dbgvalue.ll
+++ b/test/CodeGen/ARM/coalesce-dbgvalue.ll
@@ -27,11 +27,11 @@ for.cond1:                                        ; preds = %for.end9, %for.cond
 
 for.body2:                                        ; preds = %for.cond1
   store i32 %storemerge11, i32* @b, align 4, !dbg !26
-  tail call void @llvm.dbg.value(metadata i32* null, i64 0, metadata !11, metadata !MDExpression()), !dbg !28
+  tail call void @llvm.dbg.value(metadata i32* null, i64 0, metadata !11, metadata !DIExpression()), !dbg !28
   %0 = load i64, i64* @a, align 8, !dbg !29
   %xor = xor i64 %0, %e.1.ph, !dbg !29
   %conv3 = trunc i64 %xor to i32, !dbg !29
-  tail call void @llvm.dbg.value(metadata i32 %conv3, i64 0, metadata !10, metadata !MDExpression()), !dbg !29
+  tail call void @llvm.dbg.value(metadata i32 %conv3, i64 0, metadata !10, metadata !DIExpression()), !dbg !29
   %tobool4 = icmp eq i32 %conv3, 0, !dbg !29
   br i1 %tobool4, label %land.end, label %land.rhs, !dbg !29
 
@@ -79,33 +79,33 @@ attributes #3 = { nounwind }
 !llvm.dbg.cu = !{!0}
 !llvm.module.flags = !{!33}
 
-!0 = !MDCompileUnit(language: DW_LANG_C99, producer: "clang version 3.4 (trunk 182024) (llvm/trunk 182023)", isOptimized: true, emissionKind: 0, file: !1, enums: !2, retainedTypes: !2, subprograms: !3, globals: !15, imports: !2)
-!1 = !MDFile(filename: "pr16110.c", directory: "/d/b")
+!0 = !DICompileUnit(language: DW_LANG_C99, producer: "clang version 3.4 (trunk 182024) (llvm/trunk 182023)", isOptimized: true, emissionKind: 0, file: !1, enums: !2, retainedTypes: !2, subprograms: !3, globals: !15, imports: !2)
+!1 = !DIFile(filename: "pr16110.c", directory: "/d/b")
 !2 = !{}
 !3 = !{!4}
-!4 = !MDSubprogram(name: "pr16110", line: 7, isLocal: false, isDefinition: true, virtualIndex: 6, isOptimized: true, scopeLine: 7, file: !1, scope: !5, type: !6, function: i32 ()* @pr16110, variables: !9)
-!5 = !MDFile(filename: "pr16110.c", directory: "/d/b")
-!6 = !MDSubroutineType(types: !7)
+!4 = !DISubprogram(name: "pr16110", line: 7, isLocal: false, isDefinition: true, virtualIndex: 6, isOptimized: true, scopeLine: 7, file: !1, scope: !5, type: !6, function: i32 ()* @pr16110, variables: !9)
+!5 = !DIFile(filename: "pr16110.c", directory: "/d/b")
+!6 = !DISubroutineType(types: !7)
 !7 = !{!8}
-!8 = !MDBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
+!8 = !DIBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
 !9 = !{!10, !11}
-!10 = !MDLocalVariable(tag: DW_TAG_auto_variable, name: "e", line: 8, scope: !4, file: !5, type: !8)
-!11 = !MDLocalVariable(tag: DW_TAG_auto_variable, name: "f", line: 13, scope: !12, file: !5, type: !14)
-!12 = distinct !MDLexicalBlock(line: 12, column: 0, file: !1, scope: !13)
-!13 = distinct !MDLexicalBlock(line: 12, column: 0, file: !1, scope: !4)
-!14 = !MDDerivedType(tag: DW_TAG_pointer_type, size: 32, align: 32, baseType: !8)
+!10 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "e", line: 8, scope: !4, file: !5, type: !8)
+!11 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "f", line: 13, scope: !12, file: !5, type: !14)
+!12 = distinct !DILexicalBlock(line: 12, column: 0, file: !1, scope: !13)
+!13 = distinct !DILexicalBlock(line: 12, column: 0, file: !1, scope: !4)
+!14 = !DIDerivedType(tag: DW_TAG_pointer_type, size: 32, align: 32, baseType: !8)
 !15 = !{!16, !18, !19, !20}
-!16 = !MDGlobalVariable(name: "a", line: 1, isLocal: false, isDefinition: true, scope: null, file: !5, type: !17, variable: i64* @a)
-!17 = !MDBasicType(tag: DW_TAG_base_type, name: "long long int", size: 64, align: 32, encoding: DW_ATE_signed)
-!18 = !MDGlobalVariable(name: "b", line: 2, isLocal: false, isDefinition: true, scope: null, file: !5, type: !8, variable: i32* @b)
-!19 = !MDGlobalVariable(name: "c", line: 3, isLocal: false, isDefinition: true, scope: null, file: !5, type: !8, variable: i32* @c)
-!20 = !MDGlobalVariable(name: "d", line: 4, isLocal: false, isDefinition: true, scope: null, file: !5, type: !8, variable: i32* @d)
-!21 = !MDLocation(line: 10, scope: !22)
-!22 = distinct !MDLexicalBlock(line: 10, column: 0, file: !1, scope: !4)
-!26 = !MDLocation(line: 12, scope: !13)
+!16 = !DIGlobalVariable(name: "a", line: 1, isLocal: false, isDefinition: true, scope: null, file: !5, type: !17, variable: i64* @a)
+!17 = !DIBasicType(tag: DW_TAG_base_type, name: "long long int", size: 64, align: 32, encoding: DW_ATE_signed)
+!18 = !DIGlobalVariable(name: "b", line: 2, isLocal: false, isDefinition: true, scope: null, file: !5, type: !8, variable: i32* @b)
+!19 = !DIGlobalVariable(name: "c", line: 3, isLocal: false, isDefinition: true, scope: null, file: !5, type: !8, variable: i32* @c)
+!20 = !DIGlobalVariable(name: "d", line: 4, isLocal: false, isDefinition: true, scope: null, file: !5, type: !8, variable: i32* @d)
+!21 = !DILocation(line: 10, scope: !22)
+!22 = distinct !DILexicalBlock(line: 10, column: 0, file: !1, scope: !4)
+!26 = !DILocation(line: 12, scope: !13)
 !27 = !{i32* null}
-!28 = !MDLocation(line: 13, scope: !12)
-!29 = !MDLocation(line: 14, scope: !12)
-!31 = !MDLocation(line: 16, scope: !4)
-!32 = !MDLocation(line: 18, scope: !4)
+!28 = !DILocation(line: 13, scope: !12)
+!29 = !DILocation(line: 14, scope: !12)
+!31 = !DILocation(line: 16, scope: !4)
+!32 = !DILocation(line: 18, scope: !4)
 !33 = !{i32 1, !"Debug Info Version", i32 3}
diff --git a/test/CodeGen/ARM/combine-movc-sub.ll b/test/CodeGen/ARM/combine-movc-sub.ll
new file mode 100644
index 00000000000..6a5f7bdd38e
--- /dev/null
+++ b/test/CodeGen/ARM/combine-movc-sub.ll
@@ -0,0 +1,63 @@
+; RUN: llc %s -o - -verify-machineinstrs | FileCheck %s
+
+target datalayout = "e-m:o-p:32:32-f64:32:64-v64:32:64-v128:32:128-a:0:32-n32-S32"
+target triple = "thumbv7s-apple-unknown"
+
+; The subtract instruction %3 will be optimized (combined and predicated) with the select
+; inside the loop.  In this case, the kill flag on the subtract should be removed or else
+; it will fail verification.
+
+%struct.PROOFSEARCH_HELP = type { %struct.LIST_HELP*, %struct.LIST_HELP*, %struct.LIST_HELP*, %struct.LIST_HELP*, %struct.SHARED_INDEX_NODE*, %struct.LIST_HELP*, %struct.SHARED_INDEX_NODE*, %struct.LIST_HELP*, %struct.SORTTHEORY_HELP*, %struct.SORTTHEORY_HELP*, %struct.SORTTHEORY_HELP*, %struct.SHARED_INDEX_NODE*, %struct.LIST_HELP*, i32*, i32*, %struct.LIST_HELP*, i32, i32, i32, i32, i32, i32, i32, i32 }
+%struct.SORTTHEORY_HELP = type { %struct.st*, [4000 x %struct.NODE_HELP*], %struct.LIST_HELP*, %struct.LIST_HELP*, i32 }
+%struct.st = type { %struct.subst*, %struct.LIST_HELP*, %struct.LIST_HELP*, i16, i16 }
+%struct.subst = type { %struct.subst*, i32, %struct.term* }
+%struct.term = type { i32, %union.anon, %struct.LIST_HELP*, i32, i32 }
+%union.anon = type { %struct.LIST_HELP* }
+%struct.NODE_HELP = type { %struct.LIST_HELP*, i32, i32, i32, %struct.LIST_HELP*, i32 }
+%struct.SHARED_INDEX_NODE = type { %struct.st*, [3001 x %struct.term*], [4000 x %struct.term*], i32 }
+%struct.LIST_HELP = type { %struct.LIST_HELP*, i8* }
+%struct.CLAUSE_HELP = type { i32, i32, i32, i32, i32*, i32, %struct.LIST_HELP*, %struct.LIST_HELP*, i32, i32, %struct.LITERAL_HELP**, i32, i32, i32, i32 }
+%struct.LITERAL_HELP = type { i32, i32, i32, %struct.CLAUSE_HELP*, %struct.term* }
+
+declare void @foo(%struct.PROOFSEARCH_HELP*, %struct.CLAUSE_HELP*)
+
+; CHECK-LABEL: @test
+; CHECK: it
+; CHECK-NEXT: sub
+
+define hidden fastcc %struct.LIST_HELP* @test(%struct.PROOFSEARCH_HELP* %Search, %struct.LIST_HELP* %ClauseList, i32 %Level, %struct.LIST_HELP** nocapture %New) {
+entry:
+  %cmp4.i.i = icmp ugt i32 %Level, 31
+  %0 = add i32 %Level, -32
+  %1 = lshr i32 %0, 5
+  %2 = shl nuw i32 %1, 5
+  %3 = sub i32 %0, %2
+  %4 = add nuw nsw i32 %1, 1
+  br label %for.body
+
+for.body:                                         ; preds = %for.inc, %entry
+  %Scan.038 = phi %struct.LIST_HELP* [ %ClauseList, %entry ], [ %9, %for.inc ]
+  %car.i33 = getelementptr inbounds %struct.LIST_HELP, %struct.LIST_HELP* %Scan.038, i32 0, i32 1
+  %5 = bitcast i8** %car.i33 to %struct.CLAUSE_HELP**
+  %6 = load %struct.CLAUSE_HELP*, %struct.CLAUSE_HELP** %5, align 4
+  %. = add i32 %4, 10
+  %.Level = select i1 %cmp4.i.i, i32 %3, i32 %Level
+  %splitfield.i = getelementptr inbounds %struct.CLAUSE_HELP, %struct.CLAUSE_HELP* %6, i32 0, i32 4
+  %7 = load i32*, i32** %splitfield.i, align 4
+  %arrayidx.i = getelementptr inbounds i32, i32* %7, i32 %.
+  %8 = load i32, i32* %arrayidx.i, align 4
+  %shl.i = shl i32 1, %.Level
+  %and.i = and i32 %8, %shl.i
+  %cmp4.i = icmp eq i32 %and.i, 0
+  br i1 %cmp4.i, label %for.inc, label %if.then
+
+if.then:                                          ; preds = %for.body
+  tail call void @foo(%struct.PROOFSEARCH_HELP* %Search, %struct.CLAUSE_HELP* %6)
+  store i8* null, i8** %car.i33, align 4
+  br label %for.inc
+
+for.inc:                                          ; preds = %if.then, %for.body
+  %cdr.i = getelementptr inbounds %struct.LIST_HELP, %struct.LIST_HELP* %Scan.038, i32 0, i32 0
+  %9 = load %struct.LIST_HELP*, %struct.LIST_HELP** %cdr.i, align 4
+  br label %for.body
+}
diff --git a/test/CodeGen/ARM/crash.ll b/test/CodeGen/ARM/crash.ll
index 3b01d8113b9..3b7a897e10c 100644
--- a/test/CodeGen/ARM/crash.ll
+++ b/test/CodeGen/ARM/crash.ll
@@ -74,7 +74,7 @@ bb:
 %A = type { %B }
 %B = type { i32 }
 
-define void @_Z3Foov() ssp {
+define void @_Z3Foov() ssp personality i8* bitcast (i32 (...)* @__gxx_personality_sj0 to i8*) {
 entry:
   br i1 true, label %exit, label %false
 
@@ -83,7 +83,7 @@ false:
           to label %exit unwind label %lpad
 
 lpad:
-  %0 = landingpad { i8*, i32 } personality i8* bitcast (i32 (...)* @__gxx_personality_sj0 to i8*)
+  %0 = landingpad { i8*, i32 }
           catch i8* null
   unreachable
 
diff --git a/test/CodeGen/ARM/cttz.ll b/test/CodeGen/ARM/cttz.ll
new file mode 100644
index 00000000000..dacfca50593
--- /dev/null
+++ b/test/CodeGen/ARM/cttz.ll
@@ -0,0 +1,90 @@
+; RUN: llc < %s -mtriple arm-eabi -mattr=+v6t2 | FileCheck %s
+; RUN: llc < %s -mtriple arm-eabi -mattr=+v6t2 -mattr=+neon | FileCheck %s
+
+; This test checks the @llvm.cttz.* intrinsics for integers.
+
+declare i8 @llvm.cttz.i8(i8, i1)
+declare i16 @llvm.cttz.i16(i16, i1)
+declare i32 @llvm.cttz.i32(i32, i1)
+declare i64 @llvm.cttz.i64(i64, i1)
+
+;------------------------------------------------------------------------------
+
+define i8 @test_i8(i8 %a) {
+; CHECK-LABEL: test_i8:
+; CHECK: orr [[REG:r[0-9]+]], [[REG]], #256
+; CHECK: rbit
+; CHECK: clz
+  %tmp = call i8 @llvm.cttz.i8(i8 %a, i1 false)
+  ret i8 %tmp
+}
+
+define i16 @test_i16(i16 %a) {
+; CHECK-LABEL: test_i16:
+; CHECK: orr [[REG:r[0-9]+]], [[REG]], #65536
+; CHECK: rbit
+; CHECK: clz
+  %tmp = call i16 @llvm.cttz.i16(i16 %a, i1 false)
+  ret i16 %tmp
+}
+
+define i32 @test_i32(i32 %a) {
+; CHECK-LABEL: test_i32:
+; CHECK: rbit
+; CHECK: clz
+  %tmp = call i32 @llvm.cttz.i32(i32 %a, i1 false)
+  ret i32 %tmp
+}
+
+define i64 @test_i64(i64 %a) {
+; CHECK-LABEL: test_i64:
+; CHECK: rbit
+; CHECK: rbit
+; CHECK: cmp
+; CHECK: clz
+; CHECK: add
+; CHECK: clzne
+  %tmp = call i64 @llvm.cttz.i64(i64 %a, i1 false)
+  ret i64 %tmp
+}
+
+;------------------------------------------------------------------------------
+
+define i8 @test_i8_zero_undef(i8 %a) {
+; CHECK-LABEL: test_i8_zero_undef:
+; CHECK-NOT: orr
+; CHECK: rbit
+; CHECK: clz
+  %tmp = call i8 @llvm.cttz.i8(i8 %a, i1 true)
+  ret i8 %tmp
+}
+
+define i16 @test_i16_zero_undef(i16 %a) {
+; CHECK-LABEL: test_i16_zero_undef:
+; CHECK-NOT: orr
+; CHECK: rbit
+; CHECK: clz
+  %tmp = call i16 @llvm.cttz.i16(i16 %a, i1 true)
+  ret i16 %tmp
+}
+
+
+define i32 @test_i32_zero_undef(i32 %a) {
+; CHECK-LABEL: test_i32_zero_undef:
+; CHECK: rbit
+; CHECK: clz
+  %tmp = call i32 @llvm.cttz.i32(i32 %a, i1 true)
+  ret i32 %tmp
+}
+
+define i64 @test_i64_zero_undef(i64 %a) {
+; CHECK-LABEL: test_i64_zero_undef:
+; CHECK: rbit
+; CHECK: rbit
+; CHECK: cmp
+; CHECK: clz
+; CHECK: add
+; CHECK: clzne
+  %tmp = call i64 @llvm.cttz.i64(i64 %a, i1 true)
+  ret i64 %tmp
+}
diff --git a/test/CodeGen/ARM/cttz_vector.ll b/test/CodeGen/ARM/cttz_vector.ll
new file mode 100644
index 00000000000..9480d75db47
--- /dev/null
+++ b/test/CodeGen/ARM/cttz_vector.ll
@@ -0,0 +1,383 @@
+; RUN: llc < %s -mtriple armv7-linux-gnueabihf -mattr=+neon | FileCheck %s
+
+; This test checks the @llvm.cttz.* intrinsics for vectors.
+
+declare <1 x i8> @llvm.cttz.v1i8(<1 x i8>, i1)
+declare <2 x i8> @llvm.cttz.v2i8(<2 x i8>, i1)
+declare <4 x i8> @llvm.cttz.v4i8(<4 x i8>, i1)
+declare <8 x i8> @llvm.cttz.v8i8(<8 x i8>, i1)
+declare <16 x i8> @llvm.cttz.v16i8(<16 x i8>, i1)
+
+declare <1 x i16> @llvm.cttz.v1i16(<1 x i16>, i1)
+declare <2 x i16> @llvm.cttz.v2i16(<2 x i16>, i1)
+declare <4 x i16> @llvm.cttz.v4i16(<4 x i16>, i1)
+declare <8 x i16> @llvm.cttz.v8i16(<8 x i16>, i1)
+
+declare <1 x i32> @llvm.cttz.v1i32(<1 x i32>, i1)
+declare <2 x i32> @llvm.cttz.v2i32(<2 x i32>, i1)
+declare <4 x i32> @llvm.cttz.v4i32(<4 x i32>, i1)
+
+declare <1 x i64> @llvm.cttz.v1i64(<1 x i64>, i1)
+declare <2 x i64> @llvm.cttz.v2i64(<2 x i64>, i1)
+
+;------------------------------------------------------------------------------
+
+define void @test_v1i8(<1 x i8>* %p) {
+; CHECK-LABEL: test_v1i8
+  %a = load <1 x i8>, <1 x i8>* %p
+  %tmp = call <1 x i8> @llvm.cttz.v1i8(<1 x i8> %a, i1 false)
+  store <1 x i8> %tmp, <1 x i8>* %p
+  ret void
+}
+
+define void @test_v2i8(<2 x i8>* %p) {
+; CHECK-LABEL: test_v2i8:
+  %a = load <2 x i8>, <2 x i8>* %p
+  %tmp = call <2 x i8> @llvm.cttz.v2i8(<2 x i8> %a, i1 false)
+  store <2 x i8> %tmp, <2 x i8>* %p
+  ret void
+}
+
+define void @test_v4i8(<4 x i8>* %p) {
+; CHECK-LABEL: test_v4i8:
+  %a = load <4 x i8>, <4 x i8>* %p
+  %tmp = call <4 x i8> @llvm.cttz.v4i8(<4 x i8> %a, i1 false)
+  store <4 x i8> %tmp, <4 x i8>* %p
+  ret void
+}
+
+define void @test_v8i8(<8 x i8>* %p) {
+; CHECK-LABEL: test_v8i8:
+; CHECK: vldr		[[D1:d[0-9]+]], [r0]
+; CHECK: vmov.i8	[[D2:d[0-9]+]], #0x1
+; CHECK: vneg.s8	[[D3:d[0-9]+]], [[D1]]
+; CHECK: vand		[[D1]], [[D1]], [[D3]]
+; CHECK: vsub.i8	[[D1]], [[D1]], [[D2]]
+; CHECK: vcnt.8		[[D1]], [[D1]]
+; CHECK: vstr		[[D1]], [r0]
+  %a = load <8 x i8>, <8 x i8>* %p
+  %tmp = call <8 x i8> @llvm.cttz.v8i8(<8 x i8> %a, i1 false)
+  store <8 x i8> %tmp, <8 x i8>* %p
+  ret void
+}
+
+define void @test_v16i8(<16 x i8>* %p) {
+; CHECK-LABEL: test_v16i8:
+; CHECK: vld1.64	{[[D1:d[0-9]+]], [[D2:d[0-9]+]]}, [r0]
+; CHECK: vmov.i8	[[Q2:q[0-9]+]], #0x1
+; CHECK: vneg.s8	[[Q3:q[0-9]+]], [[Q1:q[0-9]+]]
+; CHECK: vand		[[Q1]], [[Q1]], [[Q3]]
+; CHECK: vsub.i8	[[Q1]], [[Q1]], [[Q2]]
+; CHECK: vcnt.8		[[Q1]], [[Q1]]
+; CHECK: vst1.64	{[[D1]], [[D2]]}, [r0]
+  %a = load <16 x i8>, <16 x i8>* %p
+  %tmp = call <16 x i8> @llvm.cttz.v16i8(<16 x i8> %a, i1 false)
+  store <16 x i8> %tmp, <16 x i8>* %p
+  ret void
+}
+
+define void @test_v1i16(<1 x i16>* %p) {
+; CHECK-LABEL: test_v1i16:
+  %a = load <1 x i16>, <1 x i16>* %p
+  %tmp = call <1 x i16> @llvm.cttz.v1i16(<1 x i16> %a, i1 false)
+  store <1 x i16> %tmp, <1 x i16>* %p
+  ret void
+}
+
+define void @test_v2i16(<2 x i16>* %p) {
+; CHECK-LABEL: test_v2i16:
+  %a = load <2 x i16>, <2 x i16>* %p
+  %tmp = call <2 x i16> @llvm.cttz.v2i16(<2 x i16> %a, i1 false)
+  store <2 x i16> %tmp, <2 x i16>* %p
+  ret void
+}
+
+define void @test_v4i16(<4 x i16>* %p) {
+; CHECK-LABEL: test_v4i16:
+; CHECK: vldr		[[D1:d[0-9]+]], [r0]
+; CHECK: vmov.i16	[[D2:d[0-9]+]], #0x1
+; CHECK: vneg.s16	[[D3:d[0-9]+]], [[D1]]
+; CHECK: vand		[[D1]], [[D1]], [[D3]]
+; CHECK: vsub.i16	[[D1]], [[D1]], [[D2]]
+; CHECK: vcnt.8		[[D1]], [[D1]]
+; CHECK: vpaddl.u8	[[D1]], [[D1]]
+; CHECK: vstr		[[D1]], [r0]
+  %a = load <4 x i16>, <4 x i16>* %p
+  %tmp = call <4 x i16> @llvm.cttz.v4i16(<4 x i16> %a, i1 false)
+  store <4 x i16> %tmp, <4 x i16>* %p
+  ret void
+}
+
+define void @test_v8i16(<8 x i16>* %p) {
+; CHECK-LABEL: test_v8i16:
+; CHECK: vld1.64	{[[D1:d[0-9]+]], [[D2:d[0-9]+]]}, [r0]
+; CHECK: vmov.i16	[[Q2:q[0-9]+]], #0x1
+; CHECK: vneg.s16	[[Q3:q[0-9]+]], [[Q1:q[0-9]+]]
+; CHECK: vand		[[Q1]], [[Q1]], [[Q3]]
+; CHECK: vsub.i16	[[Q1]], [[Q1]], [[Q2]]
+; CHECK: vcnt.8		[[Q1]], [[Q1]]
+; CHECK: vpaddl.u8	[[Q1]], [[Q1]]
+; CHECK: vst1.64	{[[D1]], [[D2]]}, [r0]
+  %a = load <8 x i16>, <8 x i16>* %p
+  %tmp = call <8 x i16> @llvm.cttz.v8i16(<8 x i16> %a, i1 false)
+  store <8 x i16> %tmp, <8 x i16>* %p
+  ret void
+}
+
+define void @test_v1i32(<1 x i32>* %p) {
+; CHECK-LABEL: test_v1i32:
+  %a = load <1 x i32>, <1 x i32>* %p
+  %tmp = call <1 x i32> @llvm.cttz.v1i32(<1 x i32> %a, i1 false)
+  store <1 x i32> %tmp, <1 x i32>* %p
+  ret void
+}
+
+define void @test_v2i32(<2 x i32>* %p) {
+; CHECK-LABEL: test_v2i32:
+; CHECK: vldr		[[D1:d[0-9]+]], [r0]
+; CHECK: vmov.i32	[[D2:d[0-9]+]], #0x1
+; CHECK: vneg.s32	[[D3:d[0-9]+]], [[D1]]
+; CHECK: vand		[[D1]], [[D1]], [[D3]]
+; CHECK: vsub.i32	[[D1]], [[D1]], [[D2]]
+; CHECK: vcnt.8		[[D1]], [[D1]]
+; CHECK: vpaddl.u8	[[D1]], [[D1]]
+; CHECK: vpaddl.u16	[[D1]], [[D1]]
+; CHECK: vstr		[[D1]], [r0]
+  %a = load <2 x i32>, <2 x i32>* %p
+  %tmp = call <2 x i32> @llvm.cttz.v2i32(<2 x i32> %a, i1 false)
+  store <2 x i32> %tmp, <2 x i32>* %p
+  ret void
+}
+
+define void @test_v4i32(<4 x i32>* %p) {
+; CHECK-LABEL: test_v4i32:
+; CHECK: vld1.64	{[[D1:d[0-9]+]], [[D2:d[0-9]+]]}, [r0]
+; CHECK: vmov.i32	[[Q2:q[0-9]+]], #0x1
+; CHECK: vneg.s32	[[Q3:q[0-9]+]], [[Q1:q[0-9]+]]
+; CHECK: vand		[[Q1]], [[Q1]], [[Q3]]
+; CHECK: vsub.i32	[[Q1]], [[Q1]], [[Q2]]
+; CHECK: vcnt.8		[[Q1]], [[Q1]]
+; CHECK: vpaddl.u8	[[Q1]], [[Q1]]
+; CHECK: vpaddl.u16	[[Q1]], [[Q1]]
+; CHECK: vst1.64	{[[D1]], [[D2]]}, [r0]
+  %a = load <4 x i32>, <4 x i32>* %p
+  %tmp = call <4 x i32> @llvm.cttz.v4i32(<4 x i32> %a, i1 false)
+  store <4 x i32> %tmp, <4 x i32>* %p
+  ret void
+}
+
+define void @test_v1i64(<1 x i64>* %p) {
+; CHECK-LABEL: test_v1i64:
+; CHECK: vldr		[[D1:d[0-9]+]], [r0]
+; CHECK: vmov.i32	[[D2:d[0-9]+]], #0x0
+; CHECK: vmov.i64	[[D3:d[0-9]+]], #0xffffffffffffffff
+; CHECK: vsub.i64	[[D2]], [[D2]], [[D1]]
+; CHECK: vand		[[D1]], [[D1]], [[D2]]
+; CHECK: vadd.i64	[[D1]], [[D1]], [[D3]]
+; CHECK: vcnt.8		[[D1]], [[D1]]
+; CHECK: vpaddl.u8	[[D1]], [[D1]]
+; CHECK: vpaddl.u16	[[D1]], [[D1]]
+; CHECK: vpaddl.u32	[[D1]], [[D1]]
+; CHECK: vstr		[[D1]], [r0]
+  %a = load <1 x i64>, <1 x i64>* %p
+  %tmp = call <1 x i64> @llvm.cttz.v1i64(<1 x i64> %a, i1 false)
+  store <1 x i64> %tmp, <1 x i64>* %p
+  ret void
+}
+
+define void @test_v2i64(<2 x i64>* %p) {
+; CHECK-LABEL: test_v2i64:
+; CHECK: vld1.64	{[[D1:d[0-9]+]], [[D2:d[0-9]+]]}, [r0]
+; CHECK: vmov.i32	[[Q2:q[0-9]+]], #0x0
+; CHECK: vmov.i64	[[Q3:q[0-9]+]], #0xffffffffffffffff
+; CHECK: vsub.i64	[[Q2]], [[Q2]], [[Q1:q[0-9]+]]
+; CHECK: vand		[[Q1]], [[Q1]], [[Q2]]
+; CHECK: vadd.i64	[[Q1]], [[Q1]], [[Q3]]
+; CHECK: vcnt.8		[[Q1]], [[Q1]]
+; CHECK: vpaddl.u8	[[Q1]], [[Q1]]
+; CHECK: vpaddl.u16	[[Q1]], [[Q1]]
+; CHECK: vpaddl.u32	[[Q1]], [[Q1]]
+; CHECK: vst1.64	{[[D1]], [[D2]]}, [r0]
+  %a = load <2 x i64>, <2 x i64>* %p
+  %tmp = call <2 x i64> @llvm.cttz.v2i64(<2 x i64> %a, i1 false)
+  store <2 x i64> %tmp, <2 x i64>* %p
+  ret void
+}
+
+;------------------------------------------------------------------------------
+
+define void @test_v1i8_zero_undef(<1 x i8>* %p) {
+; CHECK-LABEL: test_v1i8_zero_undef
+  %a = load <1 x i8>, <1 x i8>* %p
+  %tmp = call <1 x i8> @llvm.cttz.v1i8(<1 x i8> %a, i1 true)
+  store <1 x i8> %tmp, <1 x i8>* %p
+  ret void
+}
+
+define void @test_v2i8_zero_undef(<2 x i8>* %p) {
+; CHECK-LABEL: test_v2i8_zero_undef:
+  %a = load <2 x i8>, <2 x i8>* %p
+  %tmp = call <2 x i8> @llvm.cttz.v2i8(<2 x i8> %a, i1 true)
+  store <2 x i8> %tmp, <2 x i8>* %p
+  ret void
+}
+
+define void @test_v4i8_zero_undef(<4 x i8>* %p) {
+; CHECK-LABEL: test_v4i8_zero_undef:
+  %a = load <4 x i8>, <4 x i8>* %p
+  %tmp = call <4 x i8> @llvm.cttz.v4i8(<4 x i8> %a, i1 true)
+  store <4 x i8> %tmp, <4 x i8>* %p
+  ret void
+}
+
+define void @test_v8i8_zero_undef(<8 x i8>* %p) {
+; CHECK-LABEL: test_v8i8_zero_undef:
+; CHECK: vldr		[[D1:d[0-9]+]], [r0]
+; CHECK: vmov.i8	[[D2:d[0-9]+]], #0x1
+; CHECK: vneg.s8	[[D3:d[0-9]+]], [[D1]]
+; CHECK: vand		[[D1]], [[D1]], [[D3]]
+; CHECK: vsub.i8	[[D1]], [[D1]], [[D2]]
+; CHECK: vcnt.8		[[D1]], [[D1]]
+; CHECK: vstr		[[D1]], [r0]
+  %a = load <8 x i8>, <8 x i8>* %p
+  %tmp = call <8 x i8> @llvm.cttz.v8i8(<8 x i8> %a, i1 true)
+  store <8 x i8> %tmp, <8 x i8>* %p
+  ret void
+}
+
+define void @test_v16i8_zero_undef(<16 x i8>* %p) {
+; CHECK-LABEL: test_v16i8_zero_undef:
+; CHECK: vld1.64	{[[D1:d[0-9]+]], [[D2:d[0-9]+]]}, [r0]
+; CHECK: vmov.i8	[[Q2:q[0-9]+]], #0x1
+; CHECK: vneg.s8	[[Q3:q[0-9]+]], [[Q1:q[0-9]+]]
+; CHECK: vand		[[Q1]], [[Q1]], [[Q3]]
+; CHECK: vsub.i8	[[Q1]], [[Q1]], [[Q2]]
+; CHECK: vcnt.8		[[Q1]], [[Q1]]
+; CHECK: vst1.64	{[[D1]], [[D2]]}, [r0]
+  %a = load <16 x i8>, <16 x i8>* %p
+  %tmp = call <16 x i8> @llvm.cttz.v16i8(<16 x i8> %a, i1 true)
+  store <16 x i8> %tmp, <16 x i8>* %p
+  ret void
+}
+
+define void @test_v1i16_zero_undef(<1 x i16>* %p) {
+; CHECK-LABEL: test_v1i16_zero_undef:
+  %a = load <1 x i16>, <1 x i16>* %p
+  %tmp = call <1 x i16> @llvm.cttz.v1i16(<1 x i16> %a, i1 true)
+  store <1 x i16> %tmp, <1 x i16>* %p
+  ret void
+}
+
+define void @test_v2i16_zero_undef(<2 x i16>* %p) {
+; CHECK-LABEL: test_v2i16_zero_undef:
+  %a = load <2 x i16>, <2 x i16>* %p
+  %tmp = call <2 x i16> @llvm.cttz.v2i16(<2 x i16> %a, i1 true)
+  store <2 x i16> %tmp, <2 x i16>* %p
+  ret void
+}
+
+define void @test_v4i16_zero_undef(<4 x i16>* %p) {
+; CHECK-LABEL: test_v4i16_zero_undef:
+; CHECK: vldr		[[D1:d[0-9]+]], [r0]
+; CHECK: vneg.s16	[[D2:d[0-9]+]], [[D1]]
+; CHECK: vand		[[D1]], [[D1]], [[D2]]
+; CHECK: vmov.i16	[[D3:d[0-9]+]], #0xf
+; CHECK: vclz.i16	[[D1]], [[D1]]
+; CHECK: vsub.i16	[[D1]], [[D3]], [[D1]]
+; CHECK: vstr		[[D1]], [r0]
+  %a = load <4 x i16>, <4 x i16>* %p
+  %tmp = call <4 x i16> @llvm.cttz.v4i16(<4 x i16> %a, i1 true)
+  store <4 x i16> %tmp, <4 x i16>* %p
+  ret void
+}
+
+define void @test_v8i16_zero_undef(<8 x i16>* %p) {
+; CHECK-LABEL: test_v8i16_zero_undef:
+; CHECK: vld1.64	{[[D1:d[0-9]+]], [[D2:d[0-9]+]]}, [r0]
+; CHECK: vneg.s16	[[Q2:q[0-9]+]], [[Q1:q[0-9]+]]
+; CHECK: vand		[[Q1]], [[Q1]], [[Q2]]
+; CHECK: vmov.i16	[[Q3:q[0-9]+]], #0xf
+; CHECK: vclz.i16	[[Q1]], [[Q1]]
+; CHECK: vsub.i16	[[Q1]], [[Q3]], [[Q1]]
+; CHECK: vst1.64	{[[D1]], [[D2]]}, [r0]
+  %a = load <8 x i16>, <8 x i16>* %p
+  %tmp = call <8 x i16> @llvm.cttz.v8i16(<8 x i16> %a, i1 true)
+  store <8 x i16> %tmp, <8 x i16>* %p
+  ret void
+}
+
+define void @test_v1i32_zero_undef(<1 x i32>* %p) {
+; CHECK-LABEL: test_v1i32_zero_undef:
+  %a = load <1 x i32>, <1 x i32>* %p
+  %tmp = call <1 x i32> @llvm.cttz.v1i32(<1 x i32> %a, i1 true)
+  store <1 x i32> %tmp, <1 x i32>* %p
+  ret void
+}
+
+define void @test_v2i32_zero_undef(<2 x i32>* %p) {
+; CHECK-LABEL: test_v2i32_zero_undef:
+; CHECK: vldr		[[D1:d[0-9]+]], [r0]
+; CHECK: vneg.s32	[[D2:d[0-9]+]], [[D1]]
+; CHECK: vand		[[D1]], [[D1]], [[D2]]
+; CHECK: vmov.i32	[[D3:d[0-9]+]], #0x1f
+; CHECK: vclz.i32	[[D1]], [[D1]]
+; CHECK: vsub.i32	[[D1]], [[D3]], [[D1]]
+; CHECK: vstr		[[D1]], [r0]
+  %a = load <2 x i32>, <2 x i32>* %p
+  %tmp = call <2 x i32> @llvm.cttz.v2i32(<2 x i32> %a, i1 true)
+  store <2 x i32> %tmp, <2 x i32>* %p
+  ret void
+}
+
+define void @test_v4i32_zero_undef(<4 x i32>* %p) {
+; CHECK-LABEL: test_v4i32_zero_undef:
+; CHECK: vld1.64	{[[D1:d[0-9]+]], [[D2:d[0-9]+]]}, [r0]
+; CHECK: vneg.s32	[[Q2:q[0-9]+]], [[Q1:q[0-9]+]]
+; CHECK: vand		[[Q1]], [[Q1]], [[Q2]]
+; CHECK: vmov.i32	[[Q3:q[0-9]+]], #0x1f
+; CHECK: vclz.i32	[[Q1]], [[Q1]]
+; CHECK: vsub.i32	[[Q1]], [[Q3]], [[Q1]]
+; CHECK: vst1.64	{[[D1]], [[D2]]}, [r0]
+  %a = load <4 x i32>, <4 x i32>* %p
+  %tmp = call <4 x i32> @llvm.cttz.v4i32(<4 x i32> %a, i1 true)
+  store <4 x i32> %tmp, <4 x i32>* %p
+  ret void
+}
+
+define void @test_v1i64_zero_undef(<1 x i64>* %p) {
+; CHECK-LABEL: test_v1i64_zero_undef:
+; CHECK: vldr		[[D1:d[0-9]+]], [r0]
+; CHECK: vmov.i32	[[D2:d[0-9]+]], #0x0
+; CHECK: vmov.i64	[[D3:d[0-9]+]], #0xffffffffffffffff
+; CHECK: vsub.i64	[[D2]], [[D2]], [[D1]]
+; CHECK: vand		[[D1]], [[D1]], [[D2]]
+; CHECK: vadd.i64	[[D1]], [[D1]], [[D3]]
+; CHECK: vcnt.8		[[D1]], [[D1]]
+; CHECK: vpaddl.u8	[[D1]], [[D1]]
+; CHECK: vpaddl.u16	[[D1]], [[D1]]
+; CHECK: vpaddl.u32	[[D1]], [[D1]]
+; CHECK: vstr		[[D1]], [r0]
+  %a = load <1 x i64>, <1 x i64>* %p
+  %tmp = call <1 x i64> @llvm.cttz.v1i64(<1 x i64> %a, i1 true)
+  store <1 x i64> %tmp, <1 x i64>* %p
+  ret void
+}
+
+define void @test_v2i64_zero_undef(<2 x i64>* %p) {
+; CHECK-LABEL: test_v2i64_zero_undef:
+; CHECK: vld1.64	{[[D1:d[0-9]+]], [[D2:d[0-9]+]]}, [r0]
+; CHECK: vmov.i32	[[Q2:q[0-9]+]], #0x0
+; CHECK: vmov.i64	[[Q3:q[0-9]+]], #0xffffffffffffffff
+; CHECK: vsub.i64	[[Q2]], [[Q2]], [[Q1:q[0-9]+]]
+; CHECK: vand		[[Q1]], [[Q1]], [[Q2]]
+; CHECK: vadd.i64	[[Q1]], [[Q1]], [[Q3]]
+; CHECK: vcnt.8		[[Q1]], [[Q1]]
+; CHECK: vpaddl.u8	[[Q1]], [[Q1]]
+; CHECK: vpaddl.u16	[[Q1]], [[Q1]]
+; CHECK: vpaddl.u32	[[Q1]], [[Q1]]
+; CHECK: vst1.64	{[[D1]], [[D2]]}, [r0]
+  %a = load <2 x i64>, <2 x i64>* %p
+  %tmp = call <2 x i64> @llvm.cttz.v2i64(<2 x i64> %a, i1 true)
+  store <2 x i64> %tmp, <2 x i64>* %p
+  ret void
+}
diff --git a/test/CodeGen/ARM/ctz.ll b/test/CodeGen/ARM/ctz.ll
deleted file mode 100644
index 2d88b0351cf..00000000000
--- a/test/CodeGen/ARM/ctz.ll
+++ /dev/null
@@ -1,11 +0,0 @@
-; RUN: llc -mtriple=arm-eabi -mattr=+v6t2 %s -o - | FileCheck %s
-
-declare i32 @llvm.cttz.i32(i32, i1)
-
-define i32 @f1(i32 %a) {
-; CHECK-LABEL: f1:
-; CHECK: rbit
-; CHECK: clz
-  %tmp = call i32 @llvm.cttz.i32( i32 %a, i1 true )
-  ret i32 %tmp
-}
diff --git a/test/CodeGen/ARM/debug-frame-no-debug.ll b/test/CodeGen/ARM/debug-frame-no-debug.ll
index 81702c6e749..8a07f261f41 100644
--- a/test/CodeGen/ARM/debug-frame-no-debug.ll
+++ b/test/CodeGen/ARM/debug-frame-no-debug.ll
@@ -34,14 +34,13 @@ declare void @_Z5printddddd(double, double, double, double, double)
 
 define void @_Z4testiiiiiddddd(i32 %a, i32 %b, i32 %c, i32 %d, i32 %e,
                                double %m, double %n, double %p,
-                               double %q, double %r) {
+                               double %q, double %r) personality i8* bitcast (i32 (...)* @__gxx_personality_v0 to i8*) {
 entry:
   invoke void @_Z5printiiiii(i32 %a, i32 %b, i32 %c, i32 %d, i32 %e)
           to label %try.cont unwind label %lpad
 
 lpad:
   %0 = landingpad { i8*, i32 }
-          personality i8* bitcast (i32 (...)* @__gxx_personality_v0 to i8*)
           catch i8* null
   %1 = extractvalue { i8*, i32 } %0, 0
   %2 = tail call i8* @__cxa_begin_catch(i8* %1)
@@ -58,7 +57,6 @@ try.cont:
 
 lpad1:
   %3 = landingpad { i8*, i32 }
-          personality i8* bitcast (i32 (...)* @__gxx_personality_v0 to i8*)
           cleanup
   invoke void @__cxa_end_catch()
           to label %eh.resume unwind label %terminate.lpad
@@ -68,7 +66,6 @@ eh.resume:
 
 terminate.lpad:
   %4 = landingpad { i8*, i32 }
-          personality i8* bitcast (i32 (...)* @__gxx_personality_v0 to i8*)
           catch i8* null
   %5 = extractvalue { i8*, i32 } %4, 0
   tail call void @__clang_call_terminate(i8* %5)
diff --git a/test/CodeGen/ARM/debug-frame-vararg.ll b/test/CodeGen/ARM/debug-frame-vararg.ll
index 4ff3fa03d65..c1eff0a5bd6 100644
--- a/test/CodeGen/ARM/debug-frame-vararg.ll
+++ b/test/CodeGen/ARM/debug-frame-vararg.ll
@@ -4,7 +4,7 @@
 ; RUN: llc -mtriple thumb-unknown-linux-gnueabi -filetype asm -o - %s -disable-fp-elim | FileCheck %s --check-prefix=CHECK-THUMB-FP-ELIM
 
 ; Tests that the initial space allocated to the varargs on the stack is
-; taken into account in the the .cfi_ directives.
+; taken into account in the .cfi_ directives.
 
 ; Generated from the C program:
 ; #include <stdarg.h>
@@ -25,40 +25,40 @@
 !llvm.module.flags = !{!9, !10}
 !llvm.ident = !{!11}
 
-!0 = !MDCompileUnit(language: DW_LANG_C99, producer: "clang version 3.5 ", isOptimized: false, emissionKind: 0, file: !1, enums: !2, retainedTypes: !2, subprograms: !3, globals: !2, imports: !2)
-!1 = !MDFile(filename: "var.c", directory: "/tmp")
+!0 = !DICompileUnit(language: DW_LANG_C99, producer: "clang version 3.5 ", isOptimized: false, emissionKind: 0, file: !1, enums: !2, retainedTypes: !2, subprograms: !3, globals: !2, imports: !2)
+!1 = !DIFile(filename: "var.c", directory: "/tmp")
 !2 = !{}
 !3 = !{!4}
-!4 = !MDSubprogram(name: "sum", line: 5, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 5, file: !1, scope: !5, type: !6, function: i32 (i32, ...)* @sum, variables: !2)
-!5 = !MDFile(filename: "var.c", directory: "/tmp")
-!6 = !MDSubroutineType(types: !7)
+!4 = !DISubprogram(name: "sum", line: 5, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 5, file: !1, scope: !5, type: !6, function: i32 (i32, ...)* @sum, variables: !2)
+!5 = !DIFile(filename: "var.c", directory: "/tmp")
+!6 = !DISubroutineType(types: !7)
 !7 = !{!8, !8}
-!8 = !MDBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
+!8 = !DIBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
 !9 = !{i32 2, !"Dwarf Version", i32 4}
 !10 = !{i32 1, !"Debug Info Version", i32 3}
 !11 = !{!"clang version 3.5 "}
-!12 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "count", line: 5, arg: 1, scope: !4, file: !5, type: !8)
-!13 = !MDLocation(line: 5, scope: !4)
-!14 = !MDLocalVariable(tag: DW_TAG_auto_variable, name: "vl", line: 6, scope: !4, file: !5, type: !15)
-!15 = !MDDerivedType(tag: DW_TAG_typedef, name: "va_list", line: 30, file: !16, baseType: !17)
-!16 = !MDFile(filename: "/linux-x86_64-high/gcc_4.7.2/dbg/llvm/bin/../lib/clang/3.5/include/stdarg.h", directory: "/tmp")
-!17 = !MDDerivedType(tag: DW_TAG_typedef, name: "__builtin_va_list", line: 6, file: !1, baseType: !18)
-!18 = !MDCompositeType(tag: DW_TAG_structure_type, name: "__va_list", line: 6, size: 32, align: 32, file: !1, elements: !19)
+!12 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "count", line: 5, arg: 1, scope: !4, file: !5, type: !8)
+!13 = !DILocation(line: 5, scope: !4)
+!14 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "vl", line: 6, scope: !4, file: !5, type: !15)
+!15 = !DIDerivedType(tag: DW_TAG_typedef, name: "va_list", line: 30, file: !16, baseType: !17)
+!16 = !DIFile(filename: "/linux-x86_64-high/gcc_4.7.2/dbg/llvm/bin/../lib/clang/3.5/include/stdarg.h", directory: "/tmp")
+!17 = !DIDerivedType(tag: DW_TAG_typedef, name: "__builtin_va_list", line: 6, file: !1, baseType: !18)
+!18 = !DICompositeType(tag: DW_TAG_structure_type, name: "__va_list", line: 6, size: 32, align: 32, file: !1, elements: !19)
 !19 = !{!20}
-!20 = !MDDerivedType(tag: DW_TAG_member, name: "__ap", line: 6, size: 32, align: 32, file: !1, scope: !18, baseType: !21)
-!21 = !MDDerivedType(tag: DW_TAG_pointer_type, size: 32, align: 32, baseType: null)
-!22 = !MDLocation(line: 6, scope: !4)
-!23 = !MDLocation(line: 7, scope: !4)
-!24 = !MDLocalVariable(tag: DW_TAG_auto_variable, name: "sum", line: 8, scope: !4, file: !5, type: !8)
-!25 = !MDLocation(line: 8, scope: !4)
-!26 = !MDLocalVariable(tag: DW_TAG_auto_variable, name: "i", line: 9, scope: !27, file: !5, type: !8)
-!27 = distinct !MDLexicalBlock(line: 9, column: 0, file: !1, scope: !4)
-!28 = !MDLocation(line: 9, scope: !27)
-!29 = !MDLocation(line: 10, scope: !30)
-!30 = distinct !MDLexicalBlock(line: 9, column: 0, file: !1, scope: !27)
-!31 = !MDLocation(line: 11, scope: !30)
-!32 = !MDLocation(line: 12, scope: !4)
-!33 = !MDLocation(line: 13, scope: !4)
+!20 = !DIDerivedType(tag: DW_TAG_member, name: "__ap", line: 6, size: 32, align: 32, file: !1, scope: !18, baseType: !21)
+!21 = !DIDerivedType(tag: DW_TAG_pointer_type, size: 32, align: 32, baseType: null)
+!22 = !DILocation(line: 6, scope: !4)
+!23 = !DILocation(line: 7, scope: !4)
+!24 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "sum", line: 8, scope: !4, file: !5, type: !8)
+!25 = !DILocation(line: 8, scope: !4)
+!26 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "i", line: 9, scope: !27, file: !5, type: !8)
+!27 = distinct !DILexicalBlock(line: 9, column: 0, file: !1, scope: !4)
+!28 = !DILocation(line: 9, scope: !27)
+!29 = !DILocation(line: 10, scope: !30)
+!30 = distinct !DILexicalBlock(line: 9, column: 0, file: !1, scope: !27)
+!31 = !DILocation(line: 11, scope: !30)
+!32 = !DILocation(line: 12, scope: !4)
+!33 = !DILocation(line: 13, scope: !4)
 
 ; CHECK-FP-LABEL: sum
 ; CHECK-FP: .cfi_startproc
diff --git a/test/CodeGen/ARM/debug-frame.ll b/test/CodeGen/ARM/debug-frame.ll
index 9b115edc74d..cc07400c2e1 100644
--- a/test/CodeGen/ARM/debug-frame.ll
+++ b/test/CodeGen/ARM/debug-frame.ll
@@ -73,14 +73,13 @@ declare void @_Z5printddddd(double, double, double, double, double)
 
 define void @_Z4testiiiiiddddd(i32 %a, i32 %b, i32 %c, i32 %d, i32 %e,
                                double %m, double %n, double %p,
-                               double %q, double %r) {
+                               double %q, double %r) personality i8* bitcast (i32 (...)* @__gxx_personality_v0 to i8*) {
 entry:
   invoke void @_Z5printiiiii(i32 %a, i32 %b, i32 %c, i32 %d, i32 %e)
           to label %try.cont unwind label %lpad
 
 lpad:
   %0 = landingpad { i8*, i32 }
-          personality i8* bitcast (i32 (...)* @__gxx_personality_v0 to i8*)
           catch i8* null
   %1 = extractvalue { i8*, i32 } %0, 0
   %2 = tail call i8* @__cxa_begin_catch(i8* %1)
@@ -97,7 +96,6 @@ try.cont:
 
 lpad1:
   %3 = landingpad { i8*, i32 }
-          personality i8* bitcast (i32 (...)* @__gxx_personality_v0 to i8*)
           cleanup
   invoke void @__cxa_end_catch()
           to label %eh.resume unwind label %terminate.lpad
@@ -107,7 +105,6 @@ eh.resume:
 
 terminate.lpad:
   %4 = landingpad { i8*, i32 }
-          personality i8* bitcast (i32 (...)* @__gxx_personality_v0 to i8*)
           catch i8* null
   %5 = extractvalue { i8*, i32 } %4, 0
   tail call void @__clang_call_terminate(i8* %5)
@@ -128,41 +125,41 @@ declare void @_ZSt9terminatev()
 !llvm.module.flags = !{!10, !11}
 !llvm.ident = !{!12}
 
-!0 = !MDCompileUnit(language: DW_LANG_C_plus_plus, producer: "clang version 3.5 ", isOptimized: false, emissionKind: 0, file: !1, enums: !2, retainedTypes: !2, subprograms: !3, globals: !2, imports: !2)
-!1 = !MDFile(filename: "exp.cpp", directory: "/tmp")
+!0 = !DICompileUnit(language: DW_LANG_C_plus_plus, producer: "clang version 3.5 ", isOptimized: false, emissionKind: 0, file: !1, enums: !2, retainedTypes: !2, subprograms: !3, globals: !2, imports: !2)
+!1 = !DIFile(filename: "exp.cpp", directory: "/tmp")
 !2 = !{}
 !3 = !{!4}
-!4 = !MDSubprogram(name: "test", linkageName: "_Z4testiiiiiddddd", line: 4, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 5, file: !1, scope: !5, type: !6, function: void (i32, i32, i32, i32, i32, double, double, double, double, double)* @_Z4testiiiiiddddd, variables: !2)
-!5 = !MDFile(filename: "exp.cpp", directory: "/tmp")
-!6 = !MDSubroutineType(types: !7)
+!4 = !DISubprogram(name: "test", linkageName: "_Z4testiiiiiddddd", line: 4, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 5, file: !1, scope: !5, type: !6, function: void (i32, i32, i32, i32, i32, double, double, double, double, double)* @_Z4testiiiiiddddd, variables: !2)
+!5 = !DIFile(filename: "exp.cpp", directory: "/tmp")
+!6 = !DISubroutineType(types: !7)
 !7 = !{null, !8, !8, !8, !8, !8, !9, !9, !9, !9, !9}
-!8 = !MDBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
-!9 = !MDBasicType(tag: DW_TAG_base_type, name: "double", size: 64, align: 64, encoding: DW_ATE_float)
+!8 = !DIBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
+!9 = !DIBasicType(tag: DW_TAG_base_type, name: "double", size: 64, align: 64, encoding: DW_ATE_float)
 !10 = !{i32 2, !"Dwarf Version", i32 4}
 !11 = !{i32 1, !"Debug Info Version", i32 3}
 !12 = !{!"clang version 3.5 "}
-!13 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "a", line: 4, arg: 1, scope: !4, file: !5, type: !8)
-!14 = !MDLocation(line: 4, scope: !4)
-!15 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "b", line: 4, arg: 2, scope: !4, file: !5, type: !8)
-!16 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "c", line: 4, arg: 3, scope: !4, file: !5, type: !8)
-!17 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "d", line: 4, arg: 4, scope: !4, file: !5, type: !8)
-!18 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "e", line: 4, arg: 5, scope: !4, file: !5, type: !8)
-!19 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "m", line: 5, arg: 6, scope: !4, file: !5, type: !9)
-!20 = !MDLocation(line: 5, scope: !4)
-!21 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "n", line: 5, arg: 7, scope: !4, file: !5, type: !9)
-!22 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "p", line: 5, arg: 8, scope: !4, file: !5, type: !9)
-!23 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "q", line: 5, arg: 9, scope: !4, file: !5, type: !9)
-!24 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "r", line: 5, arg: 10, scope: !4, file: !5, type: !9)
-!25 = !MDLocation(line: 7, scope: !26)
-!26 = distinct !MDLexicalBlock(line: 6, column: 0, file: !1, scope: !4)
-!27 = !MDLocation(line: 8, scope: !26)
-!28 = !MDLocation(line: 11, scope: !26)
-!29 = !MDLocation(line: 9, scope: !30)
-!30 = distinct !MDLexicalBlock(line: 8, column: 0, file: !1, scope: !4)
-!31 = !MDLocation(line: 10, scope: !30)
-!32 = !MDLocation(line: 10, scope: !4)
-!33 = !MDLocation(line: 11, scope: !4)
-!34 = !MDLocation(line: 11, scope: !30)
+!13 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "a", line: 4, arg: 1, scope: !4, file: !5, type: !8)
+!14 = !DILocation(line: 4, scope: !4)
+!15 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "b", line: 4, arg: 2, scope: !4, file: !5, type: !8)
+!16 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "c", line: 4, arg: 3, scope: !4, file: !5, type: !8)
+!17 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "d", line: 4, arg: 4, scope: !4, file: !5, type: !8)
+!18 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "e", line: 4, arg: 5, scope: !4, file: !5, type: !8)
+!19 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "m", line: 5, arg: 6, scope: !4, file: !5, type: !9)
+!20 = !DILocation(line: 5, scope: !4)
+!21 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "n", line: 5, arg: 7, scope: !4, file: !5, type: !9)
+!22 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "p", line: 5, arg: 8, scope: !4, file: !5, type: !9)
+!23 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "q", line: 5, arg: 9, scope: !4, file: !5, type: !9)
+!24 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "r", line: 5, arg: 10, scope: !4, file: !5, type: !9)
+!25 = !DILocation(line: 7, scope: !26)
+!26 = distinct !DILexicalBlock(line: 6, column: 0, file: !1, scope: !4)
+!27 = !DILocation(line: 8, scope: !26)
+!28 = !DILocation(line: 11, scope: !26)
+!29 = !DILocation(line: 9, scope: !30)
+!30 = distinct !DILexicalBlock(line: 8, column: 0, file: !1, scope: !4)
+!31 = !DILocation(line: 10, scope: !30)
+!32 = !DILocation(line: 10, scope: !4)
+!33 = !DILocation(line: 11, scope: !4)
+!34 = !DILocation(line: 11, scope: !30)
 
 ; CHECK-FP-LABEL: _Z4testiiiiiddddd:
 ; CHECK-FP:   .cfi_startproc
@@ -179,7 +176,7 @@ declare void @_ZSt9terminatev()
 ; CHECK-FP:   .cfi_offset r4, -36
 ; CHECK-FP:   add    r11, sp, #28
 ; CHECK-FP:   .cfi_def_cfa r11, 8
-; CHECK-FP:   sub    sp, sp, #28
+; CHECK-FP:   sub    sp, sp, #44
 ; CHECK-FP:   .cfi_endproc
 
 ; CHECK-FP-ELIM-LABEL: _Z4testiiiiiddddd:
@@ -195,8 +192,8 @@ declare void @_ZSt9terminatev()
 ; CHECK-FP-ELIM:   .cfi_offset r6, -28
 ; CHECK-FP-ELIM:   .cfi_offset r5, -32
 ; CHECK-FP-ELIM:   .cfi_offset r4, -36
-; CHECK-FP-ELIM:   sub   sp, sp, #28
-; CHECK-FP-ELIM:   .cfi_def_cfa_offset 64
+; CHECK-FP-ELIM:   sub   sp, sp, #36
+; CHECK-FP-ELIM:   .cfi_def_cfa_offset 72
 ; CHECK-FP-ELIM:   .cfi_endproc
 
 ; CHECK-V7-FP-LABEL: _Z4testiiiiiddddd:
@@ -264,14 +261,14 @@ declare void @_ZSt9terminatev()
 
 ; CHECK-THUMB-V7-FP-LABEL: _Z4testiiiiiddddd:
 ; CHECK-THUMB-V7-FP:   .cfi_startproc
-; CHECK-THUMB-V7-FP:   push.w   {r4, r7, r11, lr}
+; CHECK-THUMB-V7-FP:   push   {r4, r6, r7, lr}
 ; CHECK-THUMB-V7-FP:   .cfi_def_cfa_offset 16
 ; CHECK-THUMB-V7-FP:   .cfi_offset lr, -4
-; CHECK-THUMB-V7-FP:   .cfi_offset r11, -8
-; CHECK-THUMB-V7-FP:   .cfi_offset r7, -12
+; CHECK-THUMB-V7-FP:   .cfi_offset r7, -8
+; CHECK-THUMB-V7-FP:   .cfi_offset r6, -12
 ; CHECK-THUMB-V7-FP:   .cfi_offset r4, -16
-; CHECK-THUMB-V7-FP:   add    r7, sp, #4
-; CHECK-THUMB-V7-FP:   .cfi_def_cfa r7, 12
+; CHECK-THUMB-V7-FP:   add    r7, sp, #8
+; CHECK-THUMB-V7-FP:   .cfi_def_cfa r7, 8
 ; CHECK-THUMB-V7-FP:   vpush  {d8, d9, d10, d11, d12}
 ; CHECK-THUMB-V7-FP:   .cfi_offset d12, -24
 ; CHECK-THUMB-V7-FP:   .cfi_offset d11, -32
@@ -300,14 +297,14 @@ declare void @_ZSt9terminatev()
 
 ; CHECK-THUMB-V7-FP-NOIAS-LABEL: _Z4testiiiiiddddd:
 ; CHECK-THUMB-V7-FP-NOIAS:   .cfi_startproc
-; CHECK-THUMB-V7-FP-NOIAS:   push.w   {r4, r7, r11, lr}
+; CHECK-THUMB-V7-FP-NOIAS:   push   {r4, r6, r7, lr}
 ; CHECK-THUMB-V7-FP-NOIAS:   .cfi_def_cfa_offset 16
 ; CHECK-THUMB-V7-FP-NOIAS:   .cfi_offset 14, -4
-; CHECK-THUMB-V7-FP-NOIAS:   .cfi_offset 11, -8
-; CHECK-THUMB-V7-FP-NOIAS:   .cfi_offset 7, -12
+; CHECK-THUMB-V7-FP-NOIAS:   .cfi_offset 7, -8
+; CHECK-THUMB-V7-FP-NOIAS:   .cfi_offset 6, -12
 ; CHECK-THUMB-V7-FP-NOIAS:   .cfi_offset 4, -16
-; CHECK-THUMB-V7-FP-NOIAS:   add    r7, sp, #4
-; CHECK-THUMB-V7-FP-NOIAS:   .cfi_def_cfa 7, 12
+; CHECK-THUMB-V7-FP-NOIAS:   add    r7, sp, #8
+; CHECK-THUMB-V7-FP-NOIAS:   .cfi_def_cfa 7, 8
 ; CHECK-THUMB-V7-FP-NOIAS:   vpush  {d8, d9, d10, d11, d12}
 ; CHECK-THUMB-V7-FP-NOIAS:   .cfi_offset 268, -24
 ; CHECK-THUMB-V7-FP-NOIAS:   .cfi_offset 267, -32
@@ -404,11 +401,11 @@ entry:
 
 ; CHECK-THUMB-V7-FP-ELIM-LABEL: test2:
 ; CHECK-THUMB-V7-FP-ELIM:   .cfi_startproc
-; CHECK-THUMB-V7-FP-ELIM:   push.w  {r11, lr}
+; CHECK-THUMB-V7-FP-ELIM:   push  {r7, lr}
 ; CHECK-THUMB-V7-FP-ELIM:   .cfi_def_cfa_offset 8
 ; CHECK-THUMB-V7-FP-ELIM:   .cfi_offset lr, -4
-; CHECK-THUMB-V7-FP-ELIM:   .cfi_offset r11, -8
-; CHECK-THUMB-V7-FP-ELIM:   pop.w   {r11, pc}
+; CHECK-THUMB-V7-FP-ELIM:   .cfi_offset r7, -8
+; CHECK-THUMB-V7-FP-ELIM:   pop   {r7, pc}
 ; CHECK-THUMB-V7-FP-ELIM:   .cfi_endproc
 
 
@@ -522,13 +519,13 @@ entry:
 
 ; CHECK-THUMB-V7-FP-ELIM-LABEL: test3:
 ; CHECK-THUMB-V7-FP-ELIM:   .cfi_startproc
-; CHECK-THUMB-V7-FP-ELIM:   push.w  {r4, r5, r11, lr}
+; CHECK-THUMB-V7-FP-ELIM:   push  {r4, r5, r7, lr}
 ; CHECK-THUMB-V7-FP-ELIM:   .cfi_def_cfa_offset 16
 ; CHECK-THUMB-V7-FP-ELIM:   .cfi_offset lr, -4
-; CHECK-THUMB-V7-FP-ELIM:   .cfi_offset r11, -8
+; CHECK-THUMB-V7-FP-ELIM:   .cfi_offset r7, -8
 ; CHECK-THUMB-V7-FP-ELIM:   .cfi_offset r5, -12
 ; CHECK-THUMB-V7-FP-ELIM:   .cfi_offset r4, -16
-; CHECK-THUMB-V7-FP-ELIM:   pop.w   {r4, r5, r11, pc}
+; CHECK-THUMB-V7-FP-ELIM:   pop   {r4, r5, r7, pc}
 ; CHECK-THUMB-V7-FP-ELIM:   .cfi_endproc
 
 
diff --git a/test/CodeGen/ARM/debug-info-arg.ll b/test/CodeGen/ARM/debug-info-arg.ll
index c75c63061f7..84eae77794a 100644
--- a/test/CodeGen/ARM/debug-info-arg.ll
+++ b/test/CodeGen/ARM/debug-info-arg.ll
@@ -7,13 +7,13 @@ target triple = "thumbv7-apple-ios"
 %struct.tag_s = type { i32, i32, i32 }
 
 define void @foo(%struct.tag_s* nocapture %this, %struct.tag_s* %c, i64 %x, i64 %y, %struct.tag_s* nocapture %ptr1, %struct.tag_s* nocapture %ptr2) nounwind ssp {
-  tail call void @llvm.dbg.value(metadata %struct.tag_s* %this, i64 0, metadata !5, metadata !MDExpression()), !dbg !20
-  tail call void @llvm.dbg.value(metadata %struct.tag_s* %c, i64 0, metadata !13, metadata !MDExpression()), !dbg !21
-  tail call void @llvm.dbg.value(metadata i64 %x, i64 0, metadata !14, metadata !MDExpression()), !dbg !22
-  tail call void @llvm.dbg.value(metadata i64 %y, i64 0, metadata !17, metadata !MDExpression()), !dbg !23
+  tail call void @llvm.dbg.value(metadata %struct.tag_s* %this, i64 0, metadata !5, metadata !DIExpression()), !dbg !20
+  tail call void @llvm.dbg.value(metadata %struct.tag_s* %c, i64 0, metadata !13, metadata !DIExpression()), !dbg !21
+  tail call void @llvm.dbg.value(metadata i64 %x, i64 0, metadata !14, metadata !DIExpression()), !dbg !22
+  tail call void @llvm.dbg.value(metadata i64 %y, i64 0, metadata !17, metadata !DIExpression()), !dbg !23
 ;CHECK:	@DEBUG_VALUE: foo:y <- [R7+8]
-  tail call void @llvm.dbg.value(metadata %struct.tag_s* %ptr1, i64 0, metadata !18, metadata !MDExpression()), !dbg !24
-  tail call void @llvm.dbg.value(metadata %struct.tag_s* %ptr2, i64 0, metadata !19, metadata !MDExpression()), !dbg !25
+  tail call void @llvm.dbg.value(metadata %struct.tag_s* %ptr1, i64 0, metadata !18, metadata !DIExpression()), !dbg !24
+  tail call void @llvm.dbg.value(metadata %struct.tag_s* %ptr2, i64 0, metadata !19, metadata !DIExpression()), !dbg !25
   %1 = icmp eq %struct.tag_s* %c, null, !dbg !26
   br i1 %1, label %3, label %2, !dbg !26
 
@@ -32,37 +32,37 @@ declare void @llvm.dbg.value(metadata, i64, metadata, metadata) nounwind readnon
 !llvm.dbg.cu = !{!0}
 !llvm.module.flags = !{!33}
 
-!0 = !MDCompileUnit(language: DW_LANG_C99, producer: "Apple clang version 3.0 (tags/Apple/clang-211.10.1) (based on LLVM 3.0svn)", isOptimized: true, emissionKind: 1, file: !32, enums: !{}, retainedTypes: !{}, subprograms: !30, imports:  null)
-!1 = !MDSubprogram(name: "foo", line: 11, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: true, scopeLine: 11, file: !2, scope: !2, type: !3, function: void (%struct.tag_s*, %struct.tag_s*, i64, i64, %struct.tag_s*, %struct.tag_s*)* @foo, variables: !31)
-!2 = !MDFile(filename: "one.c", directory: "/Volumes/Athwagate/R10048772")
-!3 = !MDSubroutineType(types: !4)
+!0 = !DICompileUnit(language: DW_LANG_C99, producer: "Apple clang version 3.0 (tags/Apple/clang-211.10.1) (based on LLVM 3.0svn)", isOptimized: true, emissionKind: 1, file: !32, enums: !{}, retainedTypes: !{}, subprograms: !30, imports:  null)
+!1 = !DISubprogram(name: "foo", line: 11, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: true, scopeLine: 11, file: !2, scope: !2, type: !3, function: void (%struct.tag_s*, %struct.tag_s*, i64, i64, %struct.tag_s*, %struct.tag_s*)* @foo, variables: !31)
+!2 = !DIFile(filename: "one.c", directory: "/Volumes/Athwagate/R10048772")
+!3 = !DISubroutineType(types: !4)
 !4 = !{null}
-!5 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "this", line: 11, arg: 1, scope: !1, file: !2, type: !6)
-!6 = !MDDerivedType(tag: DW_TAG_pointer_type, size: 32, align: 32, scope: !0, baseType: !7)
-!7 = !MDCompositeType(tag: DW_TAG_structure_type, name: "tag_s", line: 5, size: 96, align: 32, file: !32, scope: !0, elements: !8)
+!5 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "this", line: 11, arg: 1, scope: !1, file: !2, type: !6)
+!6 = !DIDerivedType(tag: DW_TAG_pointer_type, size: 32, align: 32, scope: !0, baseType: !7)
+!7 = !DICompositeType(tag: DW_TAG_structure_type, name: "tag_s", line: 5, size: 96, align: 32, file: !32, scope: !0, elements: !8)
 !8 = !{!9, !11, !12}
-!9 = !MDDerivedType(tag: DW_TAG_member, name: "x", line: 6, size: 32, align: 32, file: !32, scope: !7, baseType: !10)
-!10 = !MDBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
-!11 = !MDDerivedType(tag: DW_TAG_member, name: "y", line: 7, size: 32, align: 32, offset: 32, file: !32, scope: !7, baseType: !10)
-!12 = !MDDerivedType(tag: DW_TAG_member, name: "z", line: 8, size: 32, align: 32, offset: 64, file: !32, scope: !7, baseType: !10)
-!13 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "c", line: 11, arg: 2, scope: !1, file: !2, type: !6)
-!14 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "x", line: 11, arg: 3, scope: !1, file: !2, type: !15)
-!15 = !MDDerivedType(tag: DW_TAG_typedef, name: "UInt64", line: 1, file: !32, scope: !0, baseType: !16)
-!16 = !MDBasicType(tag: DW_TAG_base_type, name: "long long unsigned int", size: 64, align: 32, encoding: DW_ATE_unsigned)
-!17 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "y", line: 11, arg: 4, scope: !1, file: !2, type: !15)
-!18 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "ptr1", line: 11, arg: 5, scope: !1, file: !2, type: !6)
-!19 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "ptr2", line: 11, arg: 6, scope: !1, file: !2, type: !6)
-!20 = !MDLocation(line: 11, column: 24, scope: !1)
-!21 = !MDLocation(line: 11, column: 44, scope: !1)
-!22 = !MDLocation(line: 11, column: 54, scope: !1)
-!23 = !MDLocation(line: 11, column: 64, scope: !1)
-!24 = !MDLocation(line: 11, column: 81, scope: !1)
-!25 = !MDLocation(line: 11, column: 101, scope: !1)
-!26 = !MDLocation(line: 12, column: 3, scope: !27)
-!27 = distinct !MDLexicalBlock(line: 11, column: 107, file: !2, scope: !1)
-!28 = !MDLocation(line: 13, column: 5, scope: !27)
-!29 = !MDLocation(line: 14, column: 1, scope: !27)
+!9 = !DIDerivedType(tag: DW_TAG_member, name: "x", line: 6, size: 32, align: 32, file: !32, scope: !7, baseType: !10)
+!10 = !DIBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
+!11 = !DIDerivedType(tag: DW_TAG_member, name: "y", line: 7, size: 32, align: 32, offset: 32, file: !32, scope: !7, baseType: !10)
+!12 = !DIDerivedType(tag: DW_TAG_member, name: "z", line: 8, size: 32, align: 32, offset: 64, file: !32, scope: !7, baseType: !10)
+!13 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "c", line: 11, arg: 2, scope: !1, file: !2, type: !6)
+!14 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "x", line: 11, arg: 3, scope: !1, file: !2, type: !15)
+!15 = !DIDerivedType(tag: DW_TAG_typedef, name: "UInt64", line: 1, file: !32, scope: !0, baseType: !16)
+!16 = !DIBasicType(tag: DW_TAG_base_type, name: "long long unsigned int", size: 64, align: 32, encoding: DW_ATE_unsigned)
+!17 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "y", line: 11, arg: 4, scope: !1, file: !2, type: !15)
+!18 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "ptr1", line: 11, arg: 5, scope: !1, file: !2, type: !6)
+!19 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "ptr2", line: 11, arg: 6, scope: !1, file: !2, type: !6)
+!20 = !DILocation(line: 11, column: 24, scope: !1)
+!21 = !DILocation(line: 11, column: 44, scope: !1)
+!22 = !DILocation(line: 11, column: 54, scope: !1)
+!23 = !DILocation(line: 11, column: 64, scope: !1)
+!24 = !DILocation(line: 11, column: 81, scope: !1)
+!25 = !DILocation(line: 11, column: 101, scope: !1)
+!26 = !DILocation(line: 12, column: 3, scope: !27)
+!27 = distinct !DILexicalBlock(line: 11, column: 107, file: !2, scope: !1)
+!28 = !DILocation(line: 13, column: 5, scope: !27)
+!29 = !DILocation(line: 14, column: 1, scope: !27)
 !30 = !{!1}
 !31 = !{!5, !13, !14, !17, !18, !19}
-!32 = !MDFile(filename: "one.c", directory: "/Volumes/Athwagate/R10048772")
+!32 = !DIFile(filename: "one.c", directory: "/Volumes/Athwagate/R10048772")
 !33 = !{i32 1, !"Debug Info Version", i32 3}
diff --git a/test/CodeGen/ARM/debug-info-blocks.ll b/test/CodeGen/ARM/debug-info-blocks.ll
index 4e499c6531f..36610275517 100644
--- a/test/CodeGen/ARM/debug-info-blocks.ll
+++ b/test/CodeGen/ARM/debug-info-blocks.ll
@@ -31,17 +31,17 @@ define hidden void @foobar_func_block_invoke_0(i8* %.block_descriptor, %0* %load
   %1 = alloca %0*, align 4
   %bounds = alloca %struct.CR, align 4
   %data = alloca %struct.CR, align 4
-  call void @llvm.dbg.value(metadata i8* %.block_descriptor, i64 0, metadata !27, metadata !MDExpression()), !dbg !129
+  call void @llvm.dbg.value(metadata i8* %.block_descriptor, i64 0, metadata !27, metadata !DIExpression()), !dbg !129
   store %0* %loadedMydata, %0** %1, align 4
-  call void @llvm.dbg.declare(metadata %0** %1, metadata !130, metadata !MDExpression()), !dbg !131
+  call void @llvm.dbg.declare(metadata %0** %1, metadata !130, metadata !DIExpression()), !dbg !131
   %2 = bitcast %struct.CR* %bounds to %1*
   %3 = getelementptr %1, %1* %2, i32 0, i32 0
   store [4 x i32] %bounds.coerce0, [4 x i32]* %3
-  call void @llvm.dbg.declare(metadata %struct.CR* %bounds, metadata !132, metadata !MDExpression()), !dbg !133
+  call void @llvm.dbg.declare(metadata %struct.CR* %bounds, metadata !132, metadata !DIExpression()), !dbg !133
   %4 = bitcast %struct.CR* %data to %1*
   %5 = getelementptr %1, %1* %4, i32 0, i32 0
   store [4 x i32] %data.coerce0, [4 x i32]* %5
-  call void @llvm.dbg.declare(metadata %struct.CR* %data, metadata !134, metadata !MDExpression()), !dbg !135
+  call void @llvm.dbg.declare(metadata %struct.CR* %data, metadata !134, metadata !DIExpression()), !dbg !135
   %6 = bitcast i8* %.block_descriptor to %2*
   %7 = getelementptr inbounds %2, %2* %6, i32 0, i32 6
   call void @llvm.dbg.declare(metadata %2* %6, metadata !136, metadata !163), !dbg !137
@@ -95,169 +95,169 @@ define hidden void @foobar_func_block_invoke_0(i8* %.block_descriptor, %0* %load
 !llvm.dbg.cu = !{!0}
 !llvm.module.flags = !{!162}
 
-!0 = !MDCompileUnit(language: DW_LANG_ObjC, producer: "Apple clang version 2.1", isOptimized: false, runtimeVersion: 2, emissionKind: 1, file: !153, enums: !147, retainedTypes: !{}, subprograms: !148)
-!1 = !MDCompositeType(tag: DW_TAG_enumeration_type, line: 248, size: 32, align: 32, file: !160, scope: !0, elements: !3)
-!2 = !MDFile(filename: "header.h", directory: "/Volumes/Sandbox/llvm")
+!0 = !DICompileUnit(language: DW_LANG_ObjC, producer: "Apple clang version 2.1", isOptimized: false, runtimeVersion: 2, emissionKind: 1, file: !153, enums: !147, retainedTypes: !{}, subprograms: !148)
+!1 = !DICompositeType(tag: DW_TAG_enumeration_type, line: 248, size: 32, align: 32, file: !160, scope: !0, elements: !3)
+!2 = !DIFile(filename: "header.h", directory: "/Volumes/Sandbox/llvm")
 !3 = !{!4}
-!4 = !MDEnumerator(name: "Ver1", value: 0) ; [ DW_TAG_enumerator ]
-!5 = !MDCompositeType(tag: DW_TAG_enumeration_type, name: "Mode", line: 79, size: 32, align: 32, file: !160, scope: !0, elements: !7)
-!6 = !MDFile(filename: "header2.h", directory: "/Volumes/Sandbox/llvm")
+!4 = !DIEnumerator(name: "Ver1", value: 0) ; [ DW_TAG_enumerator ]
+!5 = !DICompositeType(tag: DW_TAG_enumeration_type, name: "Mode", line: 79, size: 32, align: 32, file: !160, scope: !0, elements: !7)
+!6 = !DIFile(filename: "header2.h", directory: "/Volumes/Sandbox/llvm")
 !7 = !{!8}
-!8 = !MDEnumerator(name: "One", value: 0) ; [ DW_TAG_enumerator ]
-!9 = !MDCompositeType(tag: DW_TAG_enumeration_type, line: 15, size: 32, align: 32, file: !149, scope: !0, elements: !11)
-!10 = !MDFile(filename: "header3.h", directory: "/Volumes/Sandbox/llvm")
+!8 = !DIEnumerator(name: "One", value: 0) ; [ DW_TAG_enumerator ]
+!9 = !DICompositeType(tag: DW_TAG_enumeration_type, line: 15, size: 32, align: 32, file: !149, scope: !0, elements: !11)
+!10 = !DIFile(filename: "header3.h", directory: "/Volumes/Sandbox/llvm")
 !11 = !{!12, !13}
-!12 = !MDEnumerator(name: "Unknown", value: 0) ; [ DW_TAG_enumerator ]
-!13 = !MDEnumerator(name: "Known", value: 1) ; [ DW_TAG_enumerator ]
-!14 = !MDCompositeType(tag: DW_TAG_enumeration_type, line: 20, size: 32, align: 32, file: !150, scope: !0, elements: !16)
-!15 = !MDFile(filename: "Private.h", directory: "/Volumes/Sandbox/llvm")
+!12 = !DIEnumerator(name: "Unknown", value: 0) ; [ DW_TAG_enumerator ]
+!13 = !DIEnumerator(name: "Known", value: 1) ; [ DW_TAG_enumerator ]
+!14 = !DICompositeType(tag: DW_TAG_enumeration_type, line: 20, size: 32, align: 32, file: !150, scope: !0, elements: !16)
+!15 = !DIFile(filename: "Private.h", directory: "/Volumes/Sandbox/llvm")
 !16 = !{!17, !18}
-!17 = !MDEnumerator(name: "Single", value: 0) ; [ DW_TAG_enumerator ]
-!18 = !MDEnumerator(name: "Double", value: 1) ; [ DW_TAG_enumerator ]
-!19 = !MDCompositeType(tag: DW_TAG_enumeration_type, line: 14, size: 32, align: 32, file: !151, scope: !0, elements: !21)
-!20 = !MDFile(filename: "header4.h", directory: "/Volumes/Sandbox/llvm")
+!17 = !DIEnumerator(name: "Single", value: 0) ; [ DW_TAG_enumerator ]
+!18 = !DIEnumerator(name: "Double", value: 1) ; [ DW_TAG_enumerator ]
+!19 = !DICompositeType(tag: DW_TAG_enumeration_type, line: 14, size: 32, align: 32, file: !151, scope: !0, elements: !21)
+!20 = !DIFile(filename: "header4.h", directory: "/Volumes/Sandbox/llvm")
 !21 = !{!22}
-!22 = !MDEnumerator(name: "Eleven", value: 0) ; [ DW_TAG_enumerator ]
-!23 = !MDSubprogram(name: "foobar_func_block_invoke_0", line: 609, isLocal: true, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 609, file: !152, scope: !24, type: !25, function: void (i8*, %0*, [4 x i32], [4 x i32])* @foobar_func_block_invoke_0)
-!24 = !MDFile(filename: "MyLibrary.m", directory: "/Volumes/Sandbox/llvm")
-!25 = !MDSubroutineType(types: !26)
+!22 = !DIEnumerator(name: "Eleven", value: 0) ; [ DW_TAG_enumerator ]
+!23 = !DISubprogram(name: "foobar_func_block_invoke_0", line: 609, isLocal: true, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 609, file: !152, scope: !24, type: !25, function: void (i8*, %0*, [4 x i32], [4 x i32])* @foobar_func_block_invoke_0)
+!24 = !DIFile(filename: "MyLibrary.m", directory: "/Volumes/Sandbox/llvm")
+!25 = !DISubroutineType(types: !26)
 !26 = !{null}
-!27 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: ".block_descriptor", line: 609, arg: 1, flags: DIFlagArtificial, scope: !23, file: !24, type: !28)
-!28 = !MDDerivedType(tag: DW_TAG_pointer_type, size: 32, scope: !0, baseType: !29)
-!29 = !MDCompositeType(tag: DW_TAG_structure_type, name: "__block_literal_14", line: 609, size: 256, align: 32, file: !152, scope: !24, elements: !30)
+!27 = !DILocalVariable(tag: DW_TAG_arg_variable, name: ".block_descriptor", line: 609, arg: 1, flags: DIFlagArtificial, scope: !23, file: !24, type: !28)
+!28 = !DIDerivedType(tag: DW_TAG_pointer_type, size: 32, scope: !0, baseType: !29)
+!29 = !DICompositeType(tag: DW_TAG_structure_type, name: "__block_literal_14", line: 609, size: 256, align: 32, file: !152, scope: !24, elements: !30)
 !30 = !{!31, !33, !35, !36, !37, !48, !89, !124}
-!31 = !MDDerivedType(tag: DW_TAG_member, name: "__isa", line: 609, size: 32, align: 32, file: !152, scope: !24, baseType: !32)
-!32 = !MDDerivedType(tag: DW_TAG_pointer_type, size: 32, align: 32, scope: !0, baseType: null)
-!33 = !MDDerivedType(tag: DW_TAG_member, name: "__flags", line: 609, size: 32, align: 32, offset: 32, file: !152, scope: !24, baseType: !34)
-!34 = !MDBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
-!35 = !MDDerivedType(tag: DW_TAG_member, name: "__reserved", line: 609, size: 32, align: 32, offset: 64, file: !152, scope: !24, baseType: !34)
-!36 = !MDDerivedType(tag: DW_TAG_member, name: "__FuncPtr", line: 609, size: 32, align: 32, offset: 96, file: !152, scope: !24, baseType: !32)
-!37 = !MDDerivedType(tag: DW_TAG_member, name: "__descriptor", line: 609, size: 32, align: 32, offset: 128, file: !152, scope: !24, baseType: !38)
-!38 = !MDDerivedType(tag: DW_TAG_pointer_type, size: 32, align: 32, scope: !0, baseType: !39)
-!39 = !MDCompositeType(tag: DW_TAG_structure_type, name: "__block_descriptor_withcopydispose", line: 307, size: 128, align: 32, file: !153, scope: !0, elements: !41)
-!40 = !MDFile(filename: "MyLibrary.i", directory: "/Volumes/Sandbox/llvm")
+!31 = !DIDerivedType(tag: DW_TAG_member, name: "__isa", line: 609, size: 32, align: 32, file: !152, scope: !24, baseType: !32)
+!32 = !DIDerivedType(tag: DW_TAG_pointer_type, size: 32, align: 32, scope: !0, baseType: null)
+!33 = !DIDerivedType(tag: DW_TAG_member, name: "__flags", line: 609, size: 32, align: 32, offset: 32, file: !152, scope: !24, baseType: !34)
+!34 = !DIBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
+!35 = !DIDerivedType(tag: DW_TAG_member, name: "__reserved", line: 609, size: 32, align: 32, offset: 64, file: !152, scope: !24, baseType: !34)
+!36 = !DIDerivedType(tag: DW_TAG_member, name: "__FuncPtr", line: 609, size: 32, align: 32, offset: 96, file: !152, scope: !24, baseType: !32)
+!37 = !DIDerivedType(tag: DW_TAG_member, name: "__descriptor", line: 609, size: 32, align: 32, offset: 128, file: !152, scope: !24, baseType: !38)
+!38 = !DIDerivedType(tag: DW_TAG_pointer_type, size: 32, align: 32, scope: !0, baseType: !39)
+!39 = !DICompositeType(tag: DW_TAG_structure_type, name: "__block_descriptor_withcopydispose", line: 307, size: 128, align: 32, file: !153, scope: !0, elements: !41)
+!40 = !DIFile(filename: "MyLibrary.i", directory: "/Volumes/Sandbox/llvm")
 !41 = !{!42, !44, !45, !47}
-!42 = !MDDerivedType(tag: DW_TAG_member, name: "reserved", line: 307, size: 32, align: 32, file: !153, scope: !40, baseType: !43)
-!43 = !MDBasicType(tag: DW_TAG_base_type, name: "long unsigned int", size: 32, align: 32, encoding: DW_ATE_unsigned)
-!44 = !MDDerivedType(tag: DW_TAG_member, name: "Size", line: 307, size: 32, align: 32, offset: 32, file: !153, scope: !40, baseType: !43)
-!45 = !MDDerivedType(tag: DW_TAG_member, name: "CopyFuncPtr", line: 307, size: 32, align: 32, offset: 64, file: !153, scope: !40, baseType: !46)
-!46 = !MDDerivedType(tag: DW_TAG_pointer_type, size: 32, align: 32, scope: !0, baseType: !32)
-!47 = !MDDerivedType(tag: DW_TAG_member, name: "DestroyFuncPtr", line: 307, size: 32, align: 32, offset: 96, file: !153, scope: !40, baseType: !46)
-!48 = !MDDerivedType(tag: DW_TAG_member, name: "mydata", line: 609, size: 32, align: 32, offset: 160, file: !152, scope: !24, baseType: !49)
-!49 = !MDDerivedType(tag: DW_TAG_pointer_type, size: 32, scope: !0, baseType: !50)
-!50 = !MDCompositeType(tag: DW_TAG_structure_type, size: 224, flags: DIFlagBlockByrefStruct, file: !152, scope: !24, elements: !51)
+!42 = !DIDerivedType(tag: DW_TAG_member, name: "reserved", line: 307, size: 32, align: 32, file: !153, scope: !40, baseType: !43)
+!43 = !DIBasicType(tag: DW_TAG_base_type, name: "long unsigned int", size: 32, align: 32, encoding: DW_ATE_unsigned)
+!44 = !DIDerivedType(tag: DW_TAG_member, name: "Size", line: 307, size: 32, align: 32, offset: 32, file: !153, scope: !40, baseType: !43)
+!45 = !DIDerivedType(tag: DW_TAG_member, name: "CopyFuncPtr", line: 307, size: 32, align: 32, offset: 64, file: !153, scope: !40, baseType: !46)
+!46 = !DIDerivedType(tag: DW_TAG_pointer_type, size: 32, align: 32, scope: !0, baseType: !32)
+!47 = !DIDerivedType(tag: DW_TAG_member, name: "DestroyFuncPtr", line: 307, size: 32, align: 32, offset: 96, file: !153, scope: !40, baseType: !46)
+!48 = !DIDerivedType(tag: DW_TAG_member, name: "mydata", line: 609, size: 32, align: 32, offset: 160, file: !152, scope: !24, baseType: !49)
+!49 = !DIDerivedType(tag: DW_TAG_pointer_type, size: 32, scope: !0, baseType: !50)
+!50 = !DICompositeType(tag: DW_TAG_structure_type, size: 224, flags: DIFlagBlockByrefStruct, file: !152, scope: !24, elements: !51)
 !51 = !{!52, !53, !54, !55, !56, !57, !58}
-!52 = !MDDerivedType(tag: DW_TAG_member, name: "__isa", size: 32, align: 32, file: !152, scope: !24, baseType: !32)
-!53 = !MDDerivedType(tag: DW_TAG_member, name: "__forwarding", size: 32, align: 32, offset: 32, file: !152, scope: !24, baseType: !32)
-!54 = !MDDerivedType(tag: DW_TAG_member, name: "__flags", size: 32, align: 32, offset: 64, file: !152, scope: !24, baseType: !34)
-!55 = !MDDerivedType(tag: DW_TAG_member, name: "__size", size: 32, align: 32, offset: 96, file: !152, scope: !24, baseType: !34)
-!56 = !MDDerivedType(tag: DW_TAG_member, name: "__copy_helper", size: 32, align: 32, offset: 128, file: !152, scope: !24, baseType: !32)
-!57 = !MDDerivedType(tag: DW_TAG_member, name: "__destroy_helper", size: 32, align: 32, offset: 160, file: !152, scope: !24, baseType: !32)
-!58 = !MDDerivedType(tag: DW_TAG_member, name: "mydata", size: 32, align: 32, offset: 192, file: !152, scope: !24, baseType: !59)
-!59 = !MDDerivedType(tag: DW_TAG_pointer_type, size: 32, align: 32, scope: !0, baseType: !60)
-!60 = !MDCompositeType(tag: DW_TAG_structure_type, name: "UIMydata", line: 26, size: 128, align: 32, runtimeLang: DW_LANG_ObjC, file: !154, scope: !24, elements: !62)
-!61 = !MDFile(filename: "header11.h", directory: "/Volumes/Sandbox/llvm")
+!52 = !DIDerivedType(tag: DW_TAG_member, name: "__isa", size: 32, align: 32, file: !152, scope: !24, baseType: !32)
+!53 = !DIDerivedType(tag: DW_TAG_member, name: "__forwarding", size: 32, align: 32, offset: 32, file: !152, scope: !24, baseType: !32)
+!54 = !DIDerivedType(tag: DW_TAG_member, name: "__flags", size: 32, align: 32, offset: 64, file: !152, scope: !24, baseType: !34)
+!55 = !DIDerivedType(tag: DW_TAG_member, name: "__size", size: 32, align: 32, offset: 96, file: !152, scope: !24, baseType: !34)
+!56 = !DIDerivedType(tag: DW_TAG_member, name: "__copy_helper", size: 32, align: 32, offset: 128, file: !152, scope: !24, baseType: !32)
+!57 = !DIDerivedType(tag: DW_TAG_member, name: "__destroy_helper", size: 32, align: 32, offset: 160, file: !152, scope: !24, baseType: !32)
+!58 = !DIDerivedType(tag: DW_TAG_member, name: "mydata", size: 32, align: 32, offset: 192, file: !152, scope: !24, baseType: !59)
+!59 = !DIDerivedType(tag: DW_TAG_pointer_type, size: 32, align: 32, scope: !0, baseType: !60)
+!60 = !DICompositeType(tag: DW_TAG_structure_type, name: "UIMydata", line: 26, size: 128, align: 32, runtimeLang: DW_LANG_ObjC, file: !154, scope: !24, elements: !62)
+!61 = !DIFile(filename: "header11.h", directory: "/Volumes/Sandbox/llvm")
 !62 = !{!63, !71, !75, !79}
-!63 = !MDDerivedType(tag: DW_TAG_inheritance, file: !61, baseType: !64)
-!64 = !MDCompositeType(tag: DW_TAG_structure_type, name: "NSO", line: 66, size: 32, align: 32, runtimeLang: DW_LANG_ObjC, file: !155, scope: !40, elements: !66)
-!65 = !MDFile(filename: "NSO.h", directory: "/Volumes/Sandbox/llvm")
+!63 = !DIDerivedType(tag: DW_TAG_inheritance, file: !61, baseType: !64)
+!64 = !DICompositeType(tag: DW_TAG_structure_type, name: "NSO", line: 66, size: 32, align: 32, runtimeLang: DW_LANG_ObjC, file: !155, scope: !40, elements: !66)
+!65 = !DIFile(filename: "NSO.h", directory: "/Volumes/Sandbox/llvm")
 !66 = !{!67}
-!67 = !MDDerivedType(tag: DW_TAG_member, name: "isa", line: 67, size: 32, align: 32, flags: DIFlagProtected, file: !155, scope: !65, baseType: !68, extraData: !"")
-!68 = !MDDerivedType(tag: DW_TAG_typedef, name: "Class", line: 197, file: !153, scope: !0, baseType: !69)
-!69 = !MDDerivedType(tag: DW_TAG_pointer_type, size: 32, align: 32, scope: !0, baseType: !70)
-!70 = !MDCompositeType(tag: DW_TAG_structure_type, name: "objc_class", flags: DIFlagFwdDecl, file: !153, scope: !0)
-!71 = !MDDerivedType(tag: DW_TAG_member, name: "_mydataRef", line: 28, size: 32, align: 32, offset: 32, file: !154, scope: !61, baseType: !72, extraData: !"")
-!72 = !MDDerivedType(tag: DW_TAG_typedef, name: "CFTypeRef", line: 313, file: !152, scope: !0, baseType: !73)
-!73 = !MDDerivedType(tag: DW_TAG_pointer_type, size: 32, align: 32, scope: !0, baseType: !74)
-!74 = !MDDerivedType(tag: DW_TAG_const_type, scope: !0, baseType: null)
-!75 = !MDDerivedType(tag: DW_TAG_member, name: "_scale", line: 29, size: 32, align: 32, offset: 64, file: !154, scope: !61, baseType: !76, extraData: !"")
-!76 = !MDDerivedType(tag: DW_TAG_typedef, name: "Float", line: 89, file: !156, scope: !0, baseType: !78)
-!77 = !MDFile(filename: "header12.h", directory: "/Volumes/Sandbox/llvm")
-!78 = !MDBasicType(tag: DW_TAG_base_type, name: "float", size: 32, align: 32, encoding: DW_ATE_float)
-!79 = !MDDerivedType(tag: DW_TAG_member, name: "_mydataFlags", line: 37, size: 8, align: 8, offset: 96, file: !154, scope: !61, baseType: !80, extraData: !"")
-!80 = !MDCompositeType(tag: DW_TAG_structure_type, line: 30, size: 8, align: 8, file: !154, scope: !0, elements: !81)
+!67 = !DIDerivedType(tag: DW_TAG_member, name: "isa", line: 67, size: 32, align: 32, flags: DIFlagProtected, file: !155, scope: !65, baseType: !68, extraData: !"")
+!68 = !DIDerivedType(tag: DW_TAG_typedef, name: "Class", line: 197, file: !153, scope: !0, baseType: !69)
+!69 = !DIDerivedType(tag: DW_TAG_pointer_type, size: 32, align: 32, scope: !0, baseType: !70)
+!70 = !DICompositeType(tag: DW_TAG_structure_type, name: "objc_class", flags: DIFlagFwdDecl, file: !153, scope: !0)
+!71 = !DIDerivedType(tag: DW_TAG_member, name: "_mydataRef", line: 28, size: 32, align: 32, offset: 32, file: !154, scope: !61, baseType: !72, extraData: !"")
+!72 = !DIDerivedType(tag: DW_TAG_typedef, name: "CFTypeRef", line: 313, file: !152, scope: !0, baseType: !73)
+!73 = !DIDerivedType(tag: DW_TAG_pointer_type, size: 32, align: 32, scope: !0, baseType: !74)
+!74 = !DIDerivedType(tag: DW_TAG_const_type, scope: !0, baseType: null)
+!75 = !DIDerivedType(tag: DW_TAG_member, name: "_scale", line: 29, size: 32, align: 32, offset: 64, file: !154, scope: !61, baseType: !76, extraData: !"")
+!76 = !DIDerivedType(tag: DW_TAG_typedef, name: "Float", line: 89, file: !156, scope: !0, baseType: !78)
+!77 = !DIFile(filename: "header12.h", directory: "/Volumes/Sandbox/llvm")
+!78 = !DIBasicType(tag: DW_TAG_base_type, name: "float", size: 32, align: 32, encoding: DW_ATE_float)
+!79 = !DIDerivedType(tag: DW_TAG_member, name: "_mydataFlags", line: 37, size: 8, align: 8, offset: 96, file: !154, scope: !61, baseType: !80, extraData: !"")
+!80 = !DICompositeType(tag: DW_TAG_structure_type, line: 30, size: 8, align: 8, file: !154, scope: !0, elements: !81)
 !81 = !{!82, !84, !85, !86, !87, !88}
-!82 = !MDDerivedType(tag: DW_TAG_member, name: "named", line: 31, size: 1, align: 32, file: !154, scope: !61, baseType: !83)
-!83 = !MDBasicType(tag: DW_TAG_base_type, name: "unsigned int", size: 32, align: 32, encoding: DW_ATE_unsigned)
-!84 = !MDDerivedType(tag: DW_TAG_member, name: "mydataO", line: 32, size: 3, align: 32, offset: 1, file: !154, scope: !61, baseType: !83)
-!85 = !MDDerivedType(tag: DW_TAG_member, name: "cached", line: 33, size: 1, align: 32, offset: 4, file: !154, scope: !61, baseType: !83)
-!86 = !MDDerivedType(tag: DW_TAG_member, name: "hasBeenCached", line: 34, size: 1, align: 32, offset: 5, file: !154, scope: !61, baseType: !83)
-!87 = !MDDerivedType(tag: DW_TAG_member, name: "hasPattern", line: 35, size: 1, align: 32, offset: 6, file: !154, scope: !61, baseType: !83)
-!88 = !MDDerivedType(tag: DW_TAG_member, name: "isCIMydata", line: 36, size: 1, align: 32, offset: 7, file: !154, scope: !61, baseType: !83)
-!89 = !MDDerivedType(tag: DW_TAG_member, name: "self", line: 609, size: 32, align: 32, offset: 192, file: !152, scope: !24, baseType: !90)
-!90 = !MDDerivedType(tag: DW_TAG_pointer_type, size: 32, align: 32, scope: !0, baseType: !91)
-!91 = !MDCompositeType(tag: DW_TAG_structure_type, name: "MyWork", line: 36, size: 384, align: 32, runtimeLang: DW_LANG_ObjC, file: !152, scope: !40, elements: !92)
+!82 = !DIDerivedType(tag: DW_TAG_member, name: "named", line: 31, size: 1, align: 32, file: !154, scope: !61, baseType: !83)
+!83 = !DIBasicType(tag: DW_TAG_base_type, name: "unsigned int", size: 32, align: 32, encoding: DW_ATE_unsigned)
+!84 = !DIDerivedType(tag: DW_TAG_member, name: "mydataO", line: 32, size: 3, align: 32, offset: 1, file: !154, scope: !61, baseType: !83)
+!85 = !DIDerivedType(tag: DW_TAG_member, name: "cached", line: 33, size: 1, align: 32, offset: 4, file: !154, scope: !61, baseType: !83)
+!86 = !DIDerivedType(tag: DW_TAG_member, name: "hasBeenCached", line: 34, size: 1, align: 32, offset: 5, file: !154, scope: !61, baseType: !83)
+!87 = !DIDerivedType(tag: DW_TAG_member, name: "hasPattern", line: 35, size: 1, align: 32, offset: 6, file: !154, scope: !61, baseType: !83)
+!88 = !DIDerivedType(tag: DW_TAG_member, name: "isCIMydata", line: 36, size: 1, align: 32, offset: 7, file: !154, scope: !61, baseType: !83)
+!89 = !DIDerivedType(tag: DW_TAG_member, name: "self", line: 609, size: 32, align: 32, offset: 192, file: !152, scope: !24, baseType: !90)
+!90 = !DIDerivedType(tag: DW_TAG_pointer_type, size: 32, align: 32, scope: !0, baseType: !91)
+!91 = !DICompositeType(tag: DW_TAG_structure_type, name: "MyWork", line: 36, size: 384, align: 32, runtimeLang: DW_LANG_ObjC, file: !152, scope: !40, elements: !92)
 !92 = !{!93, !98, !101, !107, !123}
-!93 = !MDDerivedType(tag: DW_TAG_inheritance, file: !152, scope: !91, baseType: !94)
-!94 = !MDCompositeType(tag: DW_TAG_structure_type, name: "twork", line: 43, size: 32, align: 32, runtimeLang: DW_LANG_ObjC, file: !157, scope: !40, elements: !96)
-!95 = !MDFile(filename: "header13.h", directory: "/Volumes/Sandbox/llvm")
+!93 = !DIDerivedType(tag: DW_TAG_inheritance, file: !152, scope: !91, baseType: !94)
+!94 = !DICompositeType(tag: DW_TAG_structure_type, name: "twork", line: 43, size: 32, align: 32, runtimeLang: DW_LANG_ObjC, file: !157, scope: !40, elements: !96)
+!95 = !DIFile(filename: "header13.h", directory: "/Volumes/Sandbox/llvm")
 !96 = !{!97}
-!97 = !MDDerivedType(tag: DW_TAG_inheritance, file: !95, baseType: !64)
-!98 = !MDDerivedType(tag: DW_TAG_member, name: "_itemID", line: 38, size: 64, align: 32, offset: 32, flags: DIFlagPrivate, file: !152, scope: !24, baseType: !99, extraData: !"")
-!99 = !MDDerivedType(tag: DW_TAG_typedef, name: "uint64_t", line: 55, file: !153, scope: !0, baseType: !100)
-!100 = !MDBasicType(tag: DW_TAG_base_type, name: "long long unsigned int", size: 64, align: 32, encoding: DW_ATE_unsigned)
-!101 = !MDDerivedType(tag: DW_TAG_member, name: "_library", line: 39, size: 32, align: 32, offset: 96, flags: DIFlagPrivate, file: !152, scope: !24, baseType: !102, extraData: !"")
-!102 = !MDDerivedType(tag: DW_TAG_pointer_type, size: 32, align: 32, scope: !0, baseType: !103)
-!103 = !MDCompositeType(tag: DW_TAG_structure_type, name: "MyLibrary2", line: 22, size: 32, align: 32, runtimeLang: DW_LANG_ObjC, file: !158, scope: !40, elements: !105)
-!104 = !MDFile(filename: "header14.h", directory: "/Volumes/Sandbox/llvm")
+!97 = !DIDerivedType(tag: DW_TAG_inheritance, file: !95, baseType: !64)
+!98 = !DIDerivedType(tag: DW_TAG_member, name: "_itemID", line: 38, size: 64, align: 32, offset: 32, flags: DIFlagPrivate, file: !152, scope: !24, baseType: !99, extraData: !"")
+!99 = !DIDerivedType(tag: DW_TAG_typedef, name: "uint64_t", line: 55, file: !153, scope: !0, baseType: !100)
+!100 = !DIBasicType(tag: DW_TAG_base_type, name: "long long unsigned int", size: 64, align: 32, encoding: DW_ATE_unsigned)
+!101 = !DIDerivedType(tag: DW_TAG_member, name: "_library", line: 39, size: 32, align: 32, offset: 96, flags: DIFlagPrivate, file: !152, scope: !24, baseType: !102, extraData: !"")
+!102 = !DIDerivedType(tag: DW_TAG_pointer_type, size: 32, align: 32, scope: !0, baseType: !103)
+!103 = !DICompositeType(tag: DW_TAG_structure_type, name: "MyLibrary2", line: 22, size: 32, align: 32, runtimeLang: DW_LANG_ObjC, file: !158, scope: !40, elements: !105)
+!104 = !DIFile(filename: "header14.h", directory: "/Volumes/Sandbox/llvm")
 !105 = !{!106}
-!106 = !MDDerivedType(tag: DW_TAG_inheritance, file: !104, baseType: !64)
-!107 = !MDDerivedType(tag: DW_TAG_member, name: "_bounds", line: 40, size: 128, align: 32, offset: 128, flags: DIFlagPrivate, file: !152, scope: !24, baseType: !108, extraData: !"")
-!108 = !MDDerivedType(tag: DW_TAG_typedef, name: "CR", line: 33, file: !153, scope: !0, baseType: !109)
-!109 = !MDCompositeType(tag: DW_TAG_structure_type, name: "CR", line: 29, size: 128, align: 32, file: !156, scope: !0, elements: !110)
+!106 = !DIDerivedType(tag: DW_TAG_inheritance, file: !104, baseType: !64)
+!107 = !DIDerivedType(tag: DW_TAG_member, name: "_bounds", line: 40, size: 128, align: 32, offset: 128, flags: DIFlagPrivate, file: !152, scope: !24, baseType: !108, extraData: !"")
+!108 = !DIDerivedType(tag: DW_TAG_typedef, name: "CR", line: 33, file: !153, scope: !0, baseType: !109)
+!109 = !DICompositeType(tag: DW_TAG_structure_type, name: "CR", line: 29, size: 128, align: 32, file: !156, scope: !0, elements: !110)
 !110 = !{!111, !117}
-!111 = !MDDerivedType(tag: DW_TAG_member, name: "origin", line: 30, size: 64, align: 32, file: !156, scope: !77, baseType: !112)
-!112 = !MDDerivedType(tag: DW_TAG_typedef, name: "CP", line: 17, file: !156, scope: !0, baseType: !113)
-!113 = !MDCompositeType(tag: DW_TAG_structure_type, name: "CP", line: 13, size: 64, align: 32, file: !156, scope: !0, elements: !114)
+!111 = !DIDerivedType(tag: DW_TAG_member, name: "origin", line: 30, size: 64, align: 32, file: !156, scope: !77, baseType: !112)
+!112 = !DIDerivedType(tag: DW_TAG_typedef, name: "CP", line: 17, file: !156, scope: !0, baseType: !113)
+!113 = !DICompositeType(tag: DW_TAG_structure_type, name: "CP", line: 13, size: 64, align: 32, file: !156, scope: !0, elements: !114)
 !114 = !{!115, !116}
-!115 = !MDDerivedType(tag: DW_TAG_member, name: "x", line: 14, size: 32, align: 32, file: !156, scope: !77, baseType: !76)
-!116 = !MDDerivedType(tag: DW_TAG_member, name: "y", line: 15, size: 32, align: 32, offset: 32, file: !156, scope: !77, baseType: !76)
-!117 = !MDDerivedType(tag: DW_TAG_member, name: "size", line: 31, size: 64, align: 32, offset: 64, file: !156, scope: !77, baseType: !118)
-!118 = !MDDerivedType(tag: DW_TAG_typedef, name: "Size", line: 25, file: !156, scope: !0, baseType: !119)
-!119 = !MDCompositeType(tag: DW_TAG_structure_type, name: "Size", line: 21, size: 64, align: 32, file: !156, scope: !0, elements: !120)
+!115 = !DIDerivedType(tag: DW_TAG_member, name: "x", line: 14, size: 32, align: 32, file: !156, scope: !77, baseType: !76)
+!116 = !DIDerivedType(tag: DW_TAG_member, name: "y", line: 15, size: 32, align: 32, offset: 32, file: !156, scope: !77, baseType: !76)
+!117 = !DIDerivedType(tag: DW_TAG_member, name: "size", line: 31, size: 64, align: 32, offset: 64, file: !156, scope: !77, baseType: !118)
+!118 = !DIDerivedType(tag: DW_TAG_typedef, name: "Size", line: 25, file: !156, scope: !0, baseType: !119)
+!119 = !DICompositeType(tag: DW_TAG_structure_type, name: "Size", line: 21, size: 64, align: 32, file: !156, scope: !0, elements: !120)
 !120 = !{!121, !122}
-!121 = !MDDerivedType(tag: DW_TAG_member, name: "width", line: 22, size: 32, align: 32, file: !156, scope: !77, baseType: !76)
-!122 = !MDDerivedType(tag: DW_TAG_member, name: "height", line: 23, size: 32, align: 32, offset: 32, file: !156, scope: !77, baseType: !76)
-!123 = !MDDerivedType(tag: DW_TAG_member, name: "_data", line: 40, size: 128, align: 32, offset: 256, flags: DIFlagPrivate, file: !152, scope: !24, baseType: !108, extraData: !"")
-!124 = !MDDerivedType(tag: DW_TAG_member, name: "semi", line: 609, size: 32, align: 32, offset: 224, file: !152, scope: !24, baseType: !125)
-!125 = !MDDerivedType(tag: DW_TAG_typedef, name: "d_t", line: 35, file: !152, scope: !0, baseType: !126)
-!126 = !MDDerivedType(tag: DW_TAG_pointer_type, size: 32, align: 32, scope: !0, baseType: !127)
-!127 = !MDCompositeType(tag: DW_TAG_structure_type, name: "my_struct", line: 49, flags: DIFlagFwdDecl, file: !159, scope: !0)
-!128 = !MDFile(filename: "header15.h", directory: "/Volumes/Sandbox/llvm")
-!129 = !MDLocation(line: 609, column: 144, scope: !23)
-!130 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "loadedMydata", line: 609, arg: 2, scope: !23, file: !24, type: !59)
-!131 = !MDLocation(line: 609, column: 155, scope: !23)
-!132 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "bounds", line: 609, arg: 3, scope: !23, file: !24, type: !108)
-!133 = !MDLocation(line: 609, column: 175, scope: !23)
-!134 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "data", line: 609, arg: 4, scope: !23, file: !24, type: !108)
-!135 = !MDLocation(line: 609, column: 190, scope: !23)
-!136 = !MDLocalVariable(tag: DW_TAG_auto_variable, name: "mydata", line: 604, scope: !23, file: !24, type: !50)
-!137 = !MDLocation(line: 604, column: 49, scope: !23)
-!138 = !MDLocalVariable(tag: DW_TAG_auto_variable, name: "self", line: 604, scope: !23, file: !40, type: !90)
-!139 = !MDLocalVariable(tag: DW_TAG_auto_variable, name: "semi", line: 607, scope: !23, file: !24, type: !125)
-!140 = !MDLocation(line: 607, column: 30, scope: !23)
-!141 = !MDLocation(line: 610, column: 17, scope: !142)
-!142 = distinct !MDLexicalBlock(line: 609, column: 200, file: !152, scope: !23)
-!143 = !MDLocation(line: 611, column: 17, scope: !142)
-!144 = !MDLocation(line: 612, column: 17, scope: !142)
-!145 = !MDLocation(line: 613, column: 17, scope: !142)
-!146 = !MDLocation(line: 615, column: 13, scope: !142)
+!121 = !DIDerivedType(tag: DW_TAG_member, name: "width", line: 22, size: 32, align: 32, file: !156, scope: !77, baseType: !76)
+!122 = !DIDerivedType(tag: DW_TAG_member, name: "height", line: 23, size: 32, align: 32, offset: 32, file: !156, scope: !77, baseType: !76)
+!123 = !DIDerivedType(tag: DW_TAG_member, name: "_data", line: 40, size: 128, align: 32, offset: 256, flags: DIFlagPrivate, file: !152, scope: !24, baseType: !108, extraData: !"")
+!124 = !DIDerivedType(tag: DW_TAG_member, name: "semi", line: 609, size: 32, align: 32, offset: 224, file: !152, scope: !24, baseType: !125)
+!125 = !DIDerivedType(tag: DW_TAG_typedef, name: "d_t", line: 35, file: !152, scope: !0, baseType: !126)
+!126 = !DIDerivedType(tag: DW_TAG_pointer_type, size: 32, align: 32, scope: !0, baseType: !127)
+!127 = !DICompositeType(tag: DW_TAG_structure_type, name: "my_struct", line: 49, flags: DIFlagFwdDecl, file: !159, scope: !0)
+!128 = !DIFile(filename: "header15.h", directory: "/Volumes/Sandbox/llvm")
+!129 = !DILocation(line: 609, column: 144, scope: !23)
+!130 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "loadedMydata", line: 609, arg: 2, scope: !23, file: !24, type: !59)
+!131 = !DILocation(line: 609, column: 155, scope: !23)
+!132 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "bounds", line: 609, arg: 3, scope: !23, file: !24, type: !108)
+!133 = !DILocation(line: 609, column: 175, scope: !23)
+!134 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "data", line: 609, arg: 4, scope: !23, file: !24, type: !108)
+!135 = !DILocation(line: 609, column: 190, scope: !23)
+!136 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "mydata", line: 604, scope: !23, file: !24, type: !50)
+!137 = !DILocation(line: 604, column: 49, scope: !23)
+!138 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "self", line: 604, scope: !23, file: !40, type: !90)
+!139 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "semi", line: 607, scope: !23, file: !24, type: !125)
+!140 = !DILocation(line: 607, column: 30, scope: !23)
+!141 = !DILocation(line: 610, column: 17, scope: !142)
+!142 = distinct !DILexicalBlock(line: 609, column: 200, file: !152, scope: !23)
+!143 = !DILocation(line: 611, column: 17, scope: !142)
+!144 = !DILocation(line: 612, column: 17, scope: !142)
+!145 = !DILocation(line: 613, column: 17, scope: !142)
+!146 = !DILocation(line: 615, column: 13, scope: !142)
 !147 = !{!1, !1, !5, !5, !9, !14, !19, !19, !14, !14, !14, !19, !19, !19}
 !148 = !{!23}
-!149 = !MDFile(filename: "header3.h", directory: "/Volumes/Sandbox/llvm")
-!150 = !MDFile(filename: "Private.h", directory: "/Volumes/Sandbox/llvm")
-!151 = !MDFile(filename: "header4.h", directory: "/Volumes/Sandbox/llvm")
-!152 = !MDFile(filename: "MyLibrary.m", directory: "/Volumes/Sandbox/llvm")
-!153 = !MDFile(filename: "MyLibrary.i", directory: "/Volumes/Sandbox/llvm")
-!154 = !MDFile(filename: "header11.h", directory: "/Volumes/Sandbox/llvm")
-!155 = !MDFile(filename: "NSO.h", directory: "/Volumes/Sandbox/llvm")
-!156 = !MDFile(filename: "header12.h", directory: "/Volumes/Sandbox/llvm")
-!157 = !MDFile(filename: "header13.h", directory: "/Volumes/Sandbox/llvm")
-!158 = !MDFile(filename: "header14.h", directory: "/Volumes/Sandbox/llvm")
-!159 = !MDFile(filename: "header15.h", directory: "/Volumes/Sandbox/llvm")
-!160 = !MDFile(filename: "header.h", directory: "/Volumes/Sandbox/llvm")
+!149 = !DIFile(filename: "header3.h", directory: "/Volumes/Sandbox/llvm")
+!150 = !DIFile(filename: "Private.h", directory: "/Volumes/Sandbox/llvm")
+!151 = !DIFile(filename: "header4.h", directory: "/Volumes/Sandbox/llvm")
+!152 = !DIFile(filename: "MyLibrary.m", directory: "/Volumes/Sandbox/llvm")
+!153 = !DIFile(filename: "MyLibrary.i", directory: "/Volumes/Sandbox/llvm")
+!154 = !DIFile(filename: "header11.h", directory: "/Volumes/Sandbox/llvm")
+!155 = !DIFile(filename: "NSO.h", directory: "/Volumes/Sandbox/llvm")
+!156 = !DIFile(filename: "header12.h", directory: "/Volumes/Sandbox/llvm")
+!157 = !DIFile(filename: "header13.h", directory: "/Volumes/Sandbox/llvm")
+!158 = !DIFile(filename: "header14.h", directory: "/Volumes/Sandbox/llvm")
+!159 = !DIFile(filename: "header15.h", directory: "/Volumes/Sandbox/llvm")
+!160 = !DIFile(filename: "header.h", directory: "/Volumes/Sandbox/llvm")
 !161 = !{!"header2.h", !"/Volumes/Sandbox/llvm"}
 !162 = !{i32 1, !"Debug Info Version", i32 3}
-!163 = !MDExpression(DW_OP_plus, 20, DW_OP_deref, DW_OP_plus, 4, DW_OP_deref, DW_OP_plus, 24)
-!164 = !MDExpression(DW_OP_deref, DW_OP_plus, 24)
-!165 = !MDExpression(DW_OP_deref, DW_OP_plus, 28)
+!163 = !DIExpression(DW_OP_plus, 20, DW_OP_deref, DW_OP_plus, 4, DW_OP_deref, DW_OP_plus, 24)
+!164 = !DIExpression(DW_OP_deref, DW_OP_plus, 24)
+!165 = !DIExpression(DW_OP_deref, DW_OP_plus, 28)
diff --git a/test/CodeGen/ARM/debug-info-branch-folding.ll b/test/CodeGen/ARM/debug-info-branch-folding.ll
index cb57efa7767..03b4d6b3815 100644
--- a/test/CodeGen/ARM/debug-info-branch-folding.ll
+++ b/test/CodeGen/ARM/debug-info-branch-folding.ll
@@ -20,9 +20,9 @@ entry:
 
 for.body9:                                        ; preds = %for.body9, %entry
   %add19 = fadd <4 x float> undef, <float 0.000000e+00, float 0.000000e+00, float 0.000000e+00, float 1.000000e+00>, !dbg !39
-  tail call void @llvm.dbg.value(metadata <4 x float> %add19, i64 0, metadata !27, metadata !MDExpression()), !dbg !39
+  tail call void @llvm.dbg.value(metadata <4 x float> %add19, i64 0, metadata !27, metadata !DIExpression()), !dbg !39
   %add20 = fadd <4 x float> undef, <float 0.000000e+00, float 0.000000e+00, float 0.000000e+00, float 1.000000e+00>, !dbg !39
-  tail call void @llvm.dbg.value(metadata <4 x float> %add20, i64 0, metadata !28, metadata !MDExpression()), !dbg !39
+  tail call void @llvm.dbg.value(metadata <4 x float> %add20, i64 0, metadata !28, metadata !DIExpression()), !dbg !39
   br i1 %cond, label %for.end54, label %for.body9, !dbg !44
 
 for.end54:                                        ; preds = %for.body9
@@ -42,60 +42,60 @@ declare void @llvm.dbg.value(metadata, i64, metadata, metadata) nounwind readnon
 !llvm.module.flags = !{!56}
 !llvm.dbg.cu = !{!2}
 
-!0 = !MDSubprogram(name: "test0001", line: 3, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: true, file: !54, scope: null, type: !3, function: <4 x float> (float)* @test0001, variables: !51)
-!1 = !MDFile(filename: "build2.c", directory: "/private/tmp")
-!2 = !MDCompileUnit(language: DW_LANG_C99, producer: "clang version 3.0 (trunk 129915)", isOptimized: true, emissionKind: 1, file: !54, enums: !{}, retainedTypes: !{}, subprograms: !50, imports:  null)
-!3 = !MDSubroutineType(types: !4)
+!0 = !DISubprogram(name: "test0001", line: 3, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: true, file: !54, scope: null, type: !3, function: <4 x float> (float)* @test0001, variables: !51)
+!1 = !DIFile(filename: "build2.c", directory: "/private/tmp")
+!2 = !DICompileUnit(language: DW_LANG_C99, producer: "clang version 3.0 (trunk 129915)", isOptimized: true, emissionKind: 1, file: !54, enums: !{}, retainedTypes: !{}, subprograms: !50, imports:  null)
+!3 = !DISubroutineType(types: !4)
 !4 = !{!5}
-!5 = !MDDerivedType(tag: DW_TAG_typedef, name: "v4f32", line: 14, file: !54, scope: !2, baseType: !6)
-!6 = !MDCompositeType(tag: DW_TAG_array_type, size: 128, align: 128, file: !54, scope: !2, baseType: !7, elements: !8)
-!7 = !MDBasicType(tag: DW_TAG_base_type, name: "float", size: 32, align: 32, encoding: DW_ATE_float)
+!5 = !DIDerivedType(tag: DW_TAG_typedef, name: "v4f32", line: 14, file: !54, scope: !2, baseType: !6)
+!6 = !DICompositeType(tag: DW_TAG_array_type, size: 128, align: 128, file: !54, scope: !2, baseType: !7, elements: !8)
+!7 = !DIBasicType(tag: DW_TAG_base_type, name: "float", size: 32, align: 32, encoding: DW_ATE_float)
 !8 = !{!9}
-!9 = !MDSubrange(count: 4)
-!10 = !MDSubprogram(name: "main", line: 59, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: true, file: !54, scope: null, type: !11, function: i32 (i32, i8**, i1)* @main, variables: !52)
-!11 = !MDSubroutineType(types: !12)
+!9 = !DISubrange(count: 4)
+!10 = !DISubprogram(name: "main", line: 59, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: true, file: !54, scope: null, type: !11, function: i32 (i32, i8**, i1)* @main, variables: !52)
+!11 = !DISubroutineType(types: !12)
 !12 = !{!13}
-!13 = !MDBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
-!14 = !MDSubprogram(name: "printFV", line: 41, isLocal: true, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: true, file: !55, scope: null, type: !16, variables: !53)
-!15 = !MDFile(filename: "/Volumes/Lalgate/work/llvm/projects/llvm-test/SingleSource/UnitTests/Vector/helpers.h", directory: "/private/tmp")
-!16 = !MDSubroutineType(types: !17)
+!13 = !DIBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
+!14 = !DISubprogram(name: "printFV", line: 41, isLocal: true, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: true, file: !55, scope: null, type: !16, variables: !53)
+!15 = !DIFile(filename: "/Volumes/Lalgate/work/llvm/projects/llvm-test/SingleSource/UnitTests/Vector/helpers.h", directory: "/private/tmp")
+!16 = !DISubroutineType(types: !17)
 !17 = !{null}
-!18 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "a", line: 3, arg: 1, scope: !0, file: !1, type: !7)
-!19 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "argc", line: 59, arg: 1, scope: !10, file: !1, type: !13)
-!20 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "argv", line: 59, arg: 2, scope: !10, file: !1, type: !21)
-!21 = !MDDerivedType(tag: DW_TAG_pointer_type, size: 32, align: 32, scope: !2, baseType: !22)
-!22 = !MDDerivedType(tag: DW_TAG_pointer_type, size: 32, align: 32, scope: !2, baseType: !23)
-!23 = !MDBasicType(tag: DW_TAG_base_type, name: "char", size: 8, align: 8, encoding: DW_ATE_signed_char)
-!24 = !MDLocalVariable(tag: DW_TAG_auto_variable, name: "i", line: 60, scope: !25, file: !1, type: !13)
-!25 = distinct !MDLexicalBlock(line: 59, column: 33, file: !1, scope: !10)
-!26 = !MDLocalVariable(tag: DW_TAG_auto_variable, name: "j", line: 60, scope: !25, file: !1, type: !13)
-!27 = !MDLocalVariable(tag: DW_TAG_auto_variable, name: "x", line: 61, scope: !25, file: !1, type: !5)
-!28 = !MDLocalVariable(tag: DW_TAG_auto_variable, name: "y", line: 62, scope: !25, file: !1, type: !5)
-!29 = !MDLocalVariable(tag: DW_TAG_auto_variable, name: "z", line: 63, scope: !25, file: !1, type: !5)
-!30 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "F", line: 41, arg: 1, scope: !14, file: !15, type: !31)
-!31 = !MDDerivedType(tag: DW_TAG_pointer_type, size: 32, align: 32, scope: !2, baseType: !32)
-!32 = !MDDerivedType(tag: DW_TAG_typedef, name: "FV", line: 25, file: !55, scope: !2, baseType: !33)
-!33 = !MDCompositeType(tag: DW_TAG_union_type, line: 22, size: 128, align: 128, file: !55, scope: !2, elements: !34)
+!18 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "a", line: 3, arg: 1, scope: !0, file: !1, type: !7)
+!19 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "argc", line: 59, arg: 1, scope: !10, file: !1, type: !13)
+!20 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "argv", line: 59, arg: 2, scope: !10, file: !1, type: !21)
+!21 = !DIDerivedType(tag: DW_TAG_pointer_type, size: 32, align: 32, scope: !2, baseType: !22)
+!22 = !DIDerivedType(tag: DW_TAG_pointer_type, size: 32, align: 32, scope: !2, baseType: !23)
+!23 = !DIBasicType(tag: DW_TAG_base_type, name: "char", size: 8, align: 8, encoding: DW_ATE_signed_char)
+!24 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "i", line: 60, scope: !25, file: !1, type: !13)
+!25 = distinct !DILexicalBlock(line: 59, column: 33, file: !1, scope: !10)
+!26 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "j", line: 60, scope: !25, file: !1, type: !13)
+!27 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "x", line: 61, scope: !25, file: !1, type: !5)
+!28 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "y", line: 62, scope: !25, file: !1, type: !5)
+!29 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "z", line: 63, scope: !25, file: !1, type: !5)
+!30 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "F", line: 41, arg: 1, scope: !14, file: !15, type: !31)
+!31 = !DIDerivedType(tag: DW_TAG_pointer_type, size: 32, align: 32, scope: !2, baseType: !32)
+!32 = !DIDerivedType(tag: DW_TAG_typedef, name: "FV", line: 25, file: !55, scope: !2, baseType: !33)
+!33 = !DICompositeType(tag: DW_TAG_union_type, line: 22, size: 128, align: 128, file: !55, scope: !2, elements: !34)
 !34 = !{!35, !37}
-!35 = !MDDerivedType(tag: DW_TAG_member, name: "V", line: 23, size: 128, align: 128, file: !55, scope: !15, baseType: !36)
-!36 = !MDDerivedType(tag: DW_TAG_typedef, name: "v4sf", line: 3, file: !55, scope: !2, baseType: !6)
-!37 = !MDDerivedType(tag: DW_TAG_member, name: "A", line: 24, size: 128, align: 32, file: !55, scope: !15, baseType: !38)
-!38 = !MDCompositeType(tag: DW_TAG_array_type, size: 128, align: 32, scope: !2, baseType: !7, elements: !8)
-!39 = !MDLocation(line: 79, column: 7, scope: !40)
-!40 = distinct !MDLexicalBlock(line: 75, column: 35, file: !1, scope: !41)
-!41 = distinct !MDLexicalBlock(line: 75, column: 5, file: !1, scope: !42)
-!42 = distinct !MDLexicalBlock(line: 71, column: 32, file: !1, scope: !43)
-!43 = distinct !MDLexicalBlock(line: 71, column: 3, file: !1, scope: !25)
-!44 = !MDLocation(line: 75, column: 5, scope: !42)
-!45 = !MDLocation(line: 42, column: 2, scope: !46, inlinedAt: !48)
-!46 = distinct !MDLexicalBlock(line: 42, column: 2, file: !15, scope: !47)
-!47 = distinct !MDLexicalBlock(line: 41, column: 28, file: !15, scope: !14)
-!48 = !MDLocation(line: 95, column: 3, scope: !25)
-!49 = !MDLocation(line: 99, column: 3, scope: !25)
+!35 = !DIDerivedType(tag: DW_TAG_member, name: "V", line: 23, size: 128, align: 128, file: !55, scope: !15, baseType: !36)
+!36 = !DIDerivedType(tag: DW_TAG_typedef, name: "v4sf", line: 3, file: !55, scope: !2, baseType: !6)
+!37 = !DIDerivedType(tag: DW_TAG_member, name: "A", line: 24, size: 128, align: 32, file: !55, scope: !15, baseType: !38)
+!38 = !DICompositeType(tag: DW_TAG_array_type, size: 128, align: 32, scope: !2, baseType: !7, elements: !8)
+!39 = !DILocation(line: 79, column: 7, scope: !40)
+!40 = distinct !DILexicalBlock(line: 75, column: 35, file: !1, scope: !41)
+!41 = distinct !DILexicalBlock(line: 75, column: 5, file: !1, scope: !42)
+!42 = distinct !DILexicalBlock(line: 71, column: 32, file: !1, scope: !43)
+!43 = distinct !DILexicalBlock(line: 71, column: 3, file: !1, scope: !25)
+!44 = !DILocation(line: 75, column: 5, scope: !42)
+!45 = !DILocation(line: 42, column: 2, scope: !46, inlinedAt: !48)
+!46 = distinct !DILexicalBlock(line: 42, column: 2, file: !15, scope: !47)
+!47 = distinct !DILexicalBlock(line: 41, column: 28, file: !15, scope: !14)
+!48 = !DILocation(line: 95, column: 3, scope: !25)
+!49 = !DILocation(line: 99, column: 3, scope: !25)
 !50 = !{!0, !10, !14}
 !51 = !{!18}
 !52 = !{!19, !20, !24, !26, !27, !28, !29}
 !53 = !{!30}
-!54 = !MDFile(filename: "build2.c", directory: "/private/tmp")
-!55 = !MDFile(filename: "/Volumes/Lalgate/work/llvm/projects/llvm-test/SingleSource/UnitTests/Vector/helpers.h", directory: "/private/tmp")
+!54 = !DIFile(filename: "build2.c", directory: "/private/tmp")
+!55 = !DIFile(filename: "/Volumes/Lalgate/work/llvm/projects/llvm-test/SingleSource/UnitTests/Vector/helpers.h", directory: "/private/tmp")
 !56 = !{i32 1, !"Debug Info Version", i32 3}
diff --git a/test/CodeGen/ARM/debug-info-d16-reg.ll b/test/CodeGen/ARM/debug-info-d16-reg.ll
index 034d0f46723..b0cf4d10337 100644
--- a/test/CodeGen/ARM/debug-info-d16-reg.ll
+++ b/test/CodeGen/ARM/debug-info-d16-reg.ll
@@ -12,9 +12,9 @@ target triple = "thumbv7-apple-darwin10"
 
 define i32 @inlineprinter(i8* %ptr, double %val, i8 zeroext %c) nounwind optsize {
 entry:
-  tail call void @llvm.dbg.value(metadata i8* %ptr, i64 0, metadata !19, metadata !MDExpression()), !dbg !26
-  tail call void @llvm.dbg.value(metadata double %val, i64 0, metadata !20, metadata !MDExpression()), !dbg !26
-  tail call void @llvm.dbg.value(metadata i8 %c, i64 0, metadata !21, metadata !MDExpression()), !dbg !26
+  tail call void @llvm.dbg.value(metadata i8* %ptr, i64 0, metadata !19, metadata !DIExpression()), !dbg !26
+  tail call void @llvm.dbg.value(metadata double %val, i64 0, metadata !20, metadata !DIExpression()), !dbg !26
+  tail call void @llvm.dbg.value(metadata i8 %c, i64 0, metadata !21, metadata !DIExpression()), !dbg !26
   %0 = zext i8 %c to i32, !dbg !27
   %1 = tail call i32 (i8*, ...) @printf(i8* getelementptr inbounds ([11 x i8], [11 x i8]* @.str, i32 0, i32 0), i8* %ptr, double %val, i32 %0) nounwind, !dbg !27
   ret i32 0, !dbg !29
@@ -22,9 +22,9 @@ entry:
 
 define i32 @printer(i8* %ptr, double %val, i8 zeroext %c) nounwind optsize noinline {
 entry:
-  tail call void @llvm.dbg.value(metadata i8* %ptr, i64 0, metadata !16, metadata !MDExpression()), !dbg !30
-  tail call void @llvm.dbg.value(metadata double %val, i64 0, metadata !17, metadata !MDExpression()), !dbg !30
-  tail call void @llvm.dbg.value(metadata i8 %c, i64 0, metadata !18, metadata !MDExpression()), !dbg !30
+  tail call void @llvm.dbg.value(metadata i8* %ptr, i64 0, metadata !16, metadata !DIExpression()), !dbg !30
+  tail call void @llvm.dbg.value(metadata double %val, i64 0, metadata !17, metadata !DIExpression()), !dbg !30
+  tail call void @llvm.dbg.value(metadata i8 %c, i64 0, metadata !18, metadata !DIExpression()), !dbg !30
   %0 = zext i8 %c to i32, !dbg !31
   %1 = tail call i32 (i8*, ...) @printf(i8* getelementptr inbounds ([11 x i8], [11 x i8]* @.str, i32 0, i32 0), i8* %ptr, double %val, i32 %0) nounwind, !dbg !31
   ret i32 0, !dbg !33
@@ -36,18 +36,18 @@ declare void @llvm.dbg.value(metadata, i64, metadata, metadata) nounwind readnon
 
 define i32 @main(i32 %argc, i8** nocapture %argv) nounwind optsize {
 entry:
-  tail call void @llvm.dbg.value(metadata i32 %argc, i64 0, metadata !22, metadata !MDExpression()), !dbg !34
-  tail call void @llvm.dbg.value(metadata i8** %argv, i64 0, metadata !23, metadata !MDExpression()), !dbg !34
+  tail call void @llvm.dbg.value(metadata i32 %argc, i64 0, metadata !22, metadata !DIExpression()), !dbg !34
+  tail call void @llvm.dbg.value(metadata i8** %argv, i64 0, metadata !23, metadata !DIExpression()), !dbg !34
   %0 = sitofp i32 %argc to double, !dbg !35
   %1 = fadd double %0, 5.555552e+05, !dbg !35
-  tail call void @llvm.dbg.value(metadata double %1, i64 0, metadata !24, metadata !MDExpression()), !dbg !35
+  tail call void @llvm.dbg.value(metadata double %1, i64 0, metadata !24, metadata !DIExpression()), !dbg !35
   %2 = tail call i32 @puts(i8* getelementptr inbounds ([6 x i8], [6 x i8]* @.str1, i32 0, i32 0)) nounwind, !dbg !36
   %3 = getelementptr inbounds i8, i8* bitcast (i32 (i32, i8**)* @main to i8*), i32 %argc, !dbg !37
   %4 = trunc i32 %argc to i8, !dbg !37
   %5 = add i8 %4, 97, !dbg !37
-  tail call void @llvm.dbg.value(metadata i8* %3, i64 0, metadata !49, metadata !MDExpression()) nounwind, !dbg !38
-  tail call void @llvm.dbg.value(metadata double %1, i64 0, metadata !50, metadata !MDExpression()) nounwind, !dbg !38
-  tail call void @llvm.dbg.value(metadata i8 %5, i64 0, metadata !51, metadata !MDExpression()) nounwind, !dbg !38
+  tail call void @llvm.dbg.value(metadata i8* %3, i64 0, metadata !49, metadata !DIExpression()) nounwind, !dbg !38
+  tail call void @llvm.dbg.value(metadata double %1, i64 0, metadata !50, metadata !DIExpression()) nounwind, !dbg !38
+  tail call void @llvm.dbg.value(metadata i8 %5, i64 0, metadata !51, metadata !DIExpression()) nounwind, !dbg !38
   %6 = zext i8 %5 to i32, !dbg !39
   %7 = tail call i32 (i8*, ...) @printf(i8* getelementptr inbounds ([11 x i8], [11 x i8]* @.str, i32 0, i32 0), i8* %3, double %1, i32 %6) nounwind, !dbg !39
   %8 = tail call i32 @printer(i8* %3, double %1, i8 zeroext %5) nounwind, !dbg !40
@@ -59,57 +59,57 @@ declare i32 @puts(i8* nocapture) nounwind
 !llvm.dbg.cu = !{!2}
 !llvm.module.flags = !{!48}
 
-!0 = !MDSubprogram(name: "printer", linkageName: "printer", line: 12, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: true, scopeLine: 12, file: !46, scope: !1, type: !3, function: i32 (i8*, double, i8)* @printer, variables: !43)
-!1 = !MDFile(filename: "a.c", directory: "/tmp/")
-!2 = !MDCompileUnit(language: DW_LANG_C89, producer: "(LLVM build 00)", isOptimized: true, emissionKind: 1, file: !46, enums: !47, retainedTypes: !47, subprograms: !42, imports:  null)
-!3 = !MDSubroutineType(types: !4)
+!0 = !DISubprogram(name: "printer", linkageName: "printer", line: 12, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: true, scopeLine: 12, file: !46, scope: !1, type: !3, function: i32 (i8*, double, i8)* @printer, variables: !43)
+!1 = !DIFile(filename: "a.c", directory: "/tmp/")
+!2 = !DICompileUnit(language: DW_LANG_C89, producer: "(LLVM build 00)", isOptimized: true, emissionKind: 1, file: !46, enums: !47, retainedTypes: !47, subprograms: !42, imports:  null)
+!3 = !DISubroutineType(types: !4)
 !4 = !{!5, !6, !7, !8}
-!5 = !MDBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
-!6 = !MDDerivedType(tag: DW_TAG_pointer_type, size: 32, align: 32, file: !46, scope: !1, baseType: null)
-!7 = !MDBasicType(tag: DW_TAG_base_type, name: "double", size: 64, align: 32, encoding: DW_ATE_float)
-!8 = !MDBasicType(tag: DW_TAG_base_type, name: "unsigned char", size: 8, align: 8, encoding: DW_ATE_unsigned_char)
-!9 = !MDSubprogram(name: "inlineprinter", linkageName: "inlineprinter", line: 5, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: true, scopeLine: 5, file: !46, scope: !1, type: !3, function: i32 (i8*, double, i8)* @inlineprinter, variables: !44)
-!10 = !MDSubprogram(name: "main", linkageName: "main", line: 18, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: true, scopeLine: 18, file: !46, scope: !1, type: !11, function: i32 (i32, i8**)* @main, variables: !45)
-!11 = !MDSubroutineType(types: !12)
+!5 = !DIBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
+!6 = !DIDerivedType(tag: DW_TAG_pointer_type, size: 32, align: 32, file: !46, scope: !1, baseType: null)
+!7 = !DIBasicType(tag: DW_TAG_base_type, name: "double", size: 64, align: 32, encoding: DW_ATE_float)
+!8 = !DIBasicType(tag: DW_TAG_base_type, name: "unsigned char", size: 8, align: 8, encoding: DW_ATE_unsigned_char)
+!9 = !DISubprogram(name: "inlineprinter", linkageName: "inlineprinter", line: 5, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: true, scopeLine: 5, file: !46, scope: !1, type: !3, function: i32 (i8*, double, i8)* @inlineprinter, variables: !44)
+!10 = !DISubprogram(name: "main", linkageName: "main", line: 18, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: true, scopeLine: 18, file: !46, scope: !1, type: !11, function: i32 (i32, i8**)* @main, variables: !45)
+!11 = !DISubroutineType(types: !12)
 !12 = !{!5, !5, !13}
-!13 = !MDDerivedType(tag: DW_TAG_pointer_type, size: 32, align: 32, file: !46, scope: !1, baseType: !14)
-!14 = !MDDerivedType(tag: DW_TAG_pointer_type, size: 32, align: 32, file: !46, scope: !1, baseType: !15)
-!15 = !MDBasicType(tag: DW_TAG_base_type, name: "char", size: 8, align: 8, encoding: DW_ATE_signed_char)
-!16 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "ptr", line: 11, arg: 1, scope: !0, file: !1, type: !6)
-!17 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "val", line: 11, arg: 2, scope: !0, file: !1, type: !7)
-!18 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "c", line: 11, arg: 3, scope: !0, file: !1, type: !8)
-!19 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "ptr", line: 4, arg: 1, scope: !9, file: !1, type: !6)
-!20 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "val", line: 4, arg: 2, scope: !9, file: !1, type: !7)
-!21 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "c", line: 4, arg: 3, scope: !9, file: !1, type: !8)
+!13 = !DIDerivedType(tag: DW_TAG_pointer_type, size: 32, align: 32, file: !46, scope: !1, baseType: !14)
+!14 = !DIDerivedType(tag: DW_TAG_pointer_type, size: 32, align: 32, file: !46, scope: !1, baseType: !15)
+!15 = !DIBasicType(tag: DW_TAG_base_type, name: "char", size: 8, align: 8, encoding: DW_ATE_signed_char)
+!16 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "ptr", line: 11, arg: 1, scope: !0, file: !1, type: !6)
+!17 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "val", line: 11, arg: 2, scope: !0, file: !1, type: !7)
+!18 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "c", line: 11, arg: 3, scope: !0, file: !1, type: !8)
+!19 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "ptr", line: 4, arg: 1, scope: !9, file: !1, type: !6)
+!20 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "val", line: 4, arg: 2, scope: !9, file: !1, type: !7)
+!21 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "c", line: 4, arg: 3, scope: !9, file: !1, type: !8)
 
-!49 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "ptr", line: 4, arg: 1, scope: !9, file: !1, type: !6)
-!50 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "val", line: 4, arg: 2, scope: !9, file: !1, type: !7)
-!51 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "c", line: 4, arg: 2, scope: !9, file: !1, type: !8)
+!49 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "ptr", line: 4, arg: 1, scope: !9, file: !1, type: !6)
+!50 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "val", line: 4, arg: 2, scope: !9, file: !1, type: !7)
+!51 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "c", line: 4, arg: 2, scope: !9, file: !1, type: !8)
 
-!22 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "argc", line: 17, arg: 0, scope: !10, file: !1, type: !5)
-!23 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "argv", line: 17, arg: 0, scope: !10, file: !1, type: !13)
-!24 = !MDLocalVariable(tag: DW_TAG_auto_variable, name: "dval", line: 19, scope: !25, file: !1, type: !7)
-!25 = distinct !MDLexicalBlock(line: 18, column: 0, file: !46, scope: !10)
-!26 = !MDLocation(line: 4, scope: !9)
-!27 = !MDLocation(line: 6, scope: !28)
-!28 = distinct !MDLexicalBlock(line: 5, column: 0, file: !46, scope: !9)
-!29 = !MDLocation(line: 7, scope: !28)
-!30 = !MDLocation(line: 11, scope: !0)
-!31 = !MDLocation(line: 13, scope: !32)
-!32 = distinct !MDLexicalBlock(line: 12, column: 0, file: !46, scope: !0)
-!33 = !MDLocation(line: 14, scope: !32)
-!34 = !MDLocation(line: 17, scope: !10)
-!35 = !MDLocation(line: 19, scope: !25)
-!36 = !MDLocation(line: 20, scope: !25)
-!37 = !MDLocation(line: 21, scope: !25)
-!38 = !MDLocation(line: 4, scope: !9, inlinedAt: !37)
-!39 = !MDLocation(line: 6, scope: !28, inlinedAt: !37)
-!40 = !MDLocation(line: 22, scope: !25)
-!41 = !MDLocation(line: 23, scope: !25)
+!22 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "argc", line: 17, arg: 1, scope: !10, file: !1, type: !5)
+!23 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "argv", line: 17, arg: 2, scope: !10, file: !1, type: !13)
+!24 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "dval", line: 19, scope: !25, file: !1, type: !7)
+!25 = distinct !DILexicalBlock(line: 18, column: 0, file: !46, scope: !10)
+!26 = !DILocation(line: 4, scope: !9)
+!27 = !DILocation(line: 6, scope: !28)
+!28 = distinct !DILexicalBlock(line: 5, column: 0, file: !46, scope: !9)
+!29 = !DILocation(line: 7, scope: !28)
+!30 = !DILocation(line: 11, scope: !0)
+!31 = !DILocation(line: 13, scope: !32)
+!32 = distinct !DILexicalBlock(line: 12, column: 0, file: !46, scope: !0)
+!33 = !DILocation(line: 14, scope: !32)
+!34 = !DILocation(line: 17, scope: !10)
+!35 = !DILocation(line: 19, scope: !25)
+!36 = !DILocation(line: 20, scope: !25)
+!37 = !DILocation(line: 21, scope: !25)
+!38 = !DILocation(line: 4, scope: !9, inlinedAt: !37)
+!39 = !DILocation(line: 6, scope: !28, inlinedAt: !37)
+!40 = !DILocation(line: 22, scope: !25)
+!41 = !DILocation(line: 23, scope: !25)
 !42 = !{!0, !9, !10}
 !43 = !{!16, !17, !18}
 !44 = !{!19, !20, !21}
 !45 = !{!22, !23, !24}
-!46 = !MDFile(filename: "a.c", directory: "/tmp/")
+!46 = !DIFile(filename: "a.c", directory: "/tmp/")
 !47 = !{}
 !48 = !{i32 1, !"Debug Info Version", i32 3}
diff --git a/test/CodeGen/ARM/debug-info-no-frame.ll b/test/CodeGen/ARM/debug-info-no-frame.ll
index 418a074b9e1..e00563cc47c 100644
--- a/test/CodeGen/ARM/debug-info-no-frame.ll
+++ b/test/CodeGen/ARM/debug-info-no-frame.ll
@@ -21,16 +21,16 @@ attributes #1 = { nounwind readnone }
 !llvm.dbg.cu = !{!0}
 !llvm.module.flags = !{!7, !8}
 
-!0 = !MDCompileUnit(language: DW_LANG_C99, file: !1, producer: "", isOptimized: false)
-!1 = !MDFile(filename: "file.c", directory: "/dir")
+!0 = !DICompileUnit(language: DW_LANG_C99, file: !1, producer: "", isOptimized: false)
+!1 = !DIFile(filename: "file.c", directory: "/dir")
 !2 = !{}
-!3 = !MDSubprogram(name: "need_cfi_def_cfa_offset", scope: !1, file: !1, line: 1, type: !4, isLocal: false, isDefinition: true, scopeLine: 2, isOptimized: false, function: void ()* @need_cfi_def_cfa_offset, variables: !2)
-!4 = !MDSubroutineType(types: !5)
+!3 = !DISubprogram(name: "need_cfi_def_cfa_offset", scope: !1, file: !1, line: 1, type: !4, isLocal: false, isDefinition: true, scopeLine: 2, isOptimized: false, function: void ()* @need_cfi_def_cfa_offset, variables: !2)
+!4 = !DISubroutineType(types: !5)
 !5 = !{null}
-!6 = !MDBasicType(name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
+!6 = !DIBasicType(name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
 !7 = !{i32 2, !"Dwarf Version", i32 4}
 !8 = !{i32 2, !"Debug Info Version", i32 3}
-!9 = !MDLocalVariable(tag: DW_TAG_auto_variable, name: "Depth", scope: !3, file: !1, line: 3, type: !6)
-!10 = !MDExpression()
-!11 = !MDLocation(line: 3, column: 9, scope: !3)
-!12 = !MDLocation(line: 7, column: 5, scope: !3)
+!9 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "Depth", scope: !3, file: !1, line: 3, type: !6)
+!10 = !DIExpression()
+!11 = !DILocation(line: 3, column: 9, scope: !3)
+!12 = !DILocation(line: 7, column: 5, scope: !3)
diff --git a/test/CodeGen/ARM/debug-info-qreg.ll b/test/CodeGen/ARM/debug-info-qreg.ll
index 9cfd67d2317..665818fc0b2 100644
--- a/test/CodeGen/ARM/debug-info-qreg.ll
+++ b/test/CodeGen/ARM/debug-info-qreg.ll
@@ -24,7 +24,7 @@ for.body9:                                        ; preds = %for.body9, %entry
   br i1 undef, label %for.end54, label %for.body9, !dbg !44
 
 for.end54:                                        ; preds = %for.body9
-  tail call void @llvm.dbg.value(metadata <4 x float> %add19, i64 0, metadata !27, metadata !MDExpression()), !dbg !39
+  tail call void @llvm.dbg.value(metadata <4 x float> %add19, i64 0, metadata !27, metadata !DIExpression()), !dbg !39
   %tmp115 = extractelement <4 x float> %add19, i32 1
   %conv6.i75 = fpext float %tmp115 to double, !dbg !45
   %call.i82 = tail call i32 (i8*, ...) @printf(i8* getelementptr inbounds ([13 x i8], [13 x i8]* @.str, i32 0, i32 0), double undef, double %conv6.i75, double undef, double undef) nounwind, !dbg !45
@@ -38,60 +38,60 @@ declare void @llvm.dbg.value(metadata, i64, metadata, metadata) nounwind readnon
 !llvm.dbg.cu = !{!2}
 !llvm.module.flags = !{!56}
 
-!0 = !MDSubprogram(name: "test0001", line: 3, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: true, scopeLine: 3, file: !54, scope: !1, type: !3, function: <4 x float> (float)* @test0001, variables: !51)
-!1 = !MDFile(filename: "build2.c", directory: "/private/tmp")
-!2 = !MDCompileUnit(language: DW_LANG_C99, producer: "clang version 3.0 (trunk 129915)", isOptimized: true, emissionKind: 1, file: !54, enums: !{}, retainedTypes: !{}, subprograms: !50, imports:  null)
-!3 = !MDSubroutineType(types: !4)
+!0 = !DISubprogram(name: "test0001", line: 3, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: true, scopeLine: 3, file: !54, scope: !1, type: !3, function: <4 x float> (float)* @test0001, variables: !51)
+!1 = !DIFile(filename: "build2.c", directory: "/private/tmp")
+!2 = !DICompileUnit(language: DW_LANG_C99, producer: "clang version 3.0 (trunk 129915)", isOptimized: true, emissionKind: 1, file: !54, enums: !{}, retainedTypes: !{}, subprograms: !50, imports:  null)
+!3 = !DISubroutineType(types: !4)
 !4 = !{!5}
-!5 = !MDDerivedType(tag: DW_TAG_typedef, name: "v4f32", line: 14, file: !54, scope: !2, baseType: !6)
-!6 = !MDCompositeType(tag: DW_TAG_array_type, size: 128, align: 128, file: !1, baseType: !7, elements: !8)
-!7 = !MDBasicType(tag: DW_TAG_base_type, name: "float", size: 32, align: 32, encoding: DW_ATE_float)
+!5 = !DIDerivedType(tag: DW_TAG_typedef, name: "v4f32", line: 14, file: !54, scope: !2, baseType: !6)
+!6 = !DICompositeType(tag: DW_TAG_array_type, size: 128, align: 128, file: !1, baseType: !7, elements: !8)
+!7 = !DIBasicType(tag: DW_TAG_base_type, name: "float", size: 32, align: 32, encoding: DW_ATE_float)
 !8 = !{!9}
-!9 = !MDSubrange(count: 4)
-!10 = !MDSubprogram(name: "main", line: 59, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: true, scopeLine: 59, file: !54, scope: !1, type: !11, function: i32 (i32, i8**)* @main, variables: !52)
-!11 = !MDSubroutineType(types: !12)
+!9 = !DISubrange(count: 4)
+!10 = !DISubprogram(name: "main", line: 59, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: true, scopeLine: 59, file: !54, scope: !1, type: !11, function: i32 (i32, i8**)* @main, variables: !52)
+!11 = !DISubroutineType(types: !12)
 !12 = !{!13}
-!13 = !MDBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
-!14 = !MDSubprogram(name: "printFV", line: 41, isLocal: true, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: true, scopeLine: 41, file: !55, scope: !15, type: !16, variables: !53)
-!15 = !MDFile(filename: "/Volumes/Lalgate/work/llvm/projects/llvm-test/SingleSource/UnitTests/Vector/helpers.h", directory: "/private/tmp")
-!16 = !MDSubroutineType(types: !17)
+!13 = !DIBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
+!14 = !DISubprogram(name: "printFV", line: 41, isLocal: true, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: true, scopeLine: 41, file: !55, scope: !15, type: !16, variables: !53)
+!15 = !DIFile(filename: "/Volumes/Lalgate/work/llvm/projects/llvm-test/SingleSource/UnitTests/Vector/helpers.h", directory: "/private/tmp")
+!16 = !DISubroutineType(types: !17)
 !17 = !{null}
-!18 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "a", line: 3, arg: 1, scope: !0, file: !1, type: !7)
-!19 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "argc", line: 59, arg: 1, scope: !10, file: !1, type: !13)
-!20 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "argv", line: 59, arg: 2, scope: !10, file: !1, type: !21)
-!21 = !MDDerivedType(tag: DW_TAG_pointer_type, size: 32, align: 32, scope: !2, baseType: !22)
-!22 = !MDDerivedType(tag: DW_TAG_pointer_type, size: 32, align: 32, scope: !2, baseType: !23)
-!23 = !MDBasicType(tag: DW_TAG_base_type, name: "char", size: 8, align: 8, encoding: DW_ATE_signed_char)
-!24 = !MDLocalVariable(tag: DW_TAG_auto_variable, name: "i", line: 60, scope: !25, file: !1, type: !13)
-!25 = distinct !MDLexicalBlock(line: 59, column: 33, file: !54, scope: !10)
-!26 = !MDLocalVariable(tag: DW_TAG_auto_variable, name: "j", line: 60, scope: !25, file: !1, type: !13)
-!27 = !MDLocalVariable(tag: DW_TAG_auto_variable, name: "x", line: 61, scope: !25, file: !1, type: !5)
-!28 = !MDLocalVariable(tag: DW_TAG_auto_variable, name: "y", line: 62, scope: !25, file: !1, type: !5)
-!29 = !MDLocalVariable(tag: DW_TAG_auto_variable, name: "z", line: 63, scope: !25, file: !1, type: !5)
-!30 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "F", line: 41, arg: 1, scope: !14, file: !15, type: !31)
-!31 = !MDDerivedType(tag: DW_TAG_pointer_type, size: 32, align: 32, scope: !2, baseType: !32)
-!32 = !MDDerivedType(tag: DW_TAG_typedef, name: "FV", line: 25, file: !55, scope: !2, baseType: !33)
-!33 = !MDCompositeType(tag: DW_TAG_union_type, line: 22, size: 128, align: 128, file: !55, scope: !2, elements: !34)
+!18 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "a", line: 3, arg: 1, scope: !0, file: !1, type: !7)
+!19 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "argc", line: 59, arg: 1, scope: !10, file: !1, type: !13)
+!20 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "argv", line: 59, arg: 2, scope: !10, file: !1, type: !21)
+!21 = !DIDerivedType(tag: DW_TAG_pointer_type, size: 32, align: 32, scope: !2, baseType: !22)
+!22 = !DIDerivedType(tag: DW_TAG_pointer_type, size: 32, align: 32, scope: !2, baseType: !23)
+!23 = !DIBasicType(tag: DW_TAG_base_type, name: "char", size: 8, align: 8, encoding: DW_ATE_signed_char)
+!24 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "i", line: 60, scope: !25, file: !1, type: !13)
+!25 = distinct !DILexicalBlock(line: 59, column: 33, file: !54, scope: !10)
+!26 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "j", line: 60, scope: !25, file: !1, type: !13)
+!27 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "x", line: 61, scope: !25, file: !1, type: !5)
+!28 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "y", line: 62, scope: !25, file: !1, type: !5)
+!29 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "z", line: 63, scope: !25, file: !1, type: !5)
+!30 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "F", line: 41, arg: 1, scope: !14, file: !15, type: !31)
+!31 = !DIDerivedType(tag: DW_TAG_pointer_type, size: 32, align: 32, scope: !2, baseType: !32)
+!32 = !DIDerivedType(tag: DW_TAG_typedef, name: "FV", line: 25, file: !55, scope: !2, baseType: !33)
+!33 = !DICompositeType(tag: DW_TAG_union_type, line: 22, size: 128, align: 128, file: !55, scope: !2, elements: !34)
 !34 = !{!35, !37}
-!35 = !MDDerivedType(tag: DW_TAG_member, name: "V", line: 23, size: 128, align: 128, file: !55, scope: !15, baseType: !36)
-!36 = !MDDerivedType(tag: DW_TAG_typedef, name: "v4sf", line: 3, file: !55, scope: !2, baseType: !6)
-!37 = !MDDerivedType(tag: DW_TAG_member, name: "A", line: 24, size: 128, align: 32, file: !55, scope: !15, baseType: !38)
-!38 = !MDCompositeType(tag: DW_TAG_array_type, size: 128, align: 32, scope: !2, baseType: !7, elements: !8)
-!39 = !MDLocation(line: 79, column: 7, scope: !40)
-!40 = distinct !MDLexicalBlock(line: 75, column: 35, file: !54, scope: !41)
-!41 = distinct !MDLexicalBlock(line: 75, column: 5, file: !54, scope: !42)
-!42 = distinct !MDLexicalBlock(line: 71, column: 32, file: !54, scope: !43)
-!43 = distinct !MDLexicalBlock(line: 71, column: 3, file: !54, scope: !25)
-!44 = !MDLocation(line: 75, column: 5, scope: !42)
-!45 = !MDLocation(line: 42, column: 2, scope: !46, inlinedAt: !48)
-!46 = distinct !MDLexicalBlock(line: 42, column: 2, file: !55, scope: !47)
-!47 = distinct !MDLexicalBlock(line: 41, column: 28, file: !55, scope: !14)
-!48 = !MDLocation(line: 95, column: 3, scope: !25)
-!49 = !MDLocation(line: 99, column: 3, scope: !25)
+!35 = !DIDerivedType(tag: DW_TAG_member, name: "V", line: 23, size: 128, align: 128, file: !55, scope: !15, baseType: !36)
+!36 = !DIDerivedType(tag: DW_TAG_typedef, name: "v4sf", line: 3, file: !55, scope: !2, baseType: !6)
+!37 = !DIDerivedType(tag: DW_TAG_member, name: "A", line: 24, size: 128, align: 32, file: !55, scope: !15, baseType: !38)
+!38 = !DICompositeType(tag: DW_TAG_array_type, size: 128, align: 32, scope: !2, baseType: !7, elements: !8)
+!39 = !DILocation(line: 79, column: 7, scope: !40)
+!40 = distinct !DILexicalBlock(line: 75, column: 35, file: !54, scope: !41)
+!41 = distinct !DILexicalBlock(line: 75, column: 5, file: !54, scope: !42)
+!42 = distinct !DILexicalBlock(line: 71, column: 32, file: !54, scope: !43)
+!43 = distinct !DILexicalBlock(line: 71, column: 3, file: !54, scope: !25)
+!44 = !DILocation(line: 75, column: 5, scope: !42)
+!45 = !DILocation(line: 42, column: 2, scope: !46, inlinedAt: !48)
+!46 = distinct !DILexicalBlock(line: 42, column: 2, file: !55, scope: !47)
+!47 = distinct !DILexicalBlock(line: 41, column: 28, file: !55, scope: !14)
+!48 = !DILocation(line: 95, column: 3, scope: !25)
+!49 = !DILocation(line: 99, column: 3, scope: !25)
 !50 = !{!0, !10, !14}
 !51 = !{!18}
 !52 = !{!19, !20, !24, !26, !27, !28, !29}
 !53 = !{!30}
-!54 = !MDFile(filename: "build2.c", directory: "/private/tmp")
-!55 = !MDFile(filename: "/Volumes/Lalgate/work/llvm/projects/llvm-test/SingleSource/UnitTests/Vector/helpers.h", directory: "/private/tmp")
+!54 = !DIFile(filename: "build2.c", directory: "/private/tmp")
+!55 = !DIFile(filename: "/Volumes/Lalgate/work/llvm/projects/llvm-test/SingleSource/UnitTests/Vector/helpers.h", directory: "/private/tmp")
 !56 = !{i32 1, !"Debug Info Version", i32 3}
diff --git a/test/CodeGen/ARM/debug-info-s16-reg.ll b/test/CodeGen/ARM/debug-info-s16-reg.ll
index 3cd2837714e..ec080f20db9 100644
--- a/test/CodeGen/ARM/debug-info-s16-reg.ll
+++ b/test/CodeGen/ARM/debug-info-s16-reg.ll
@@ -14,9 +14,9 @@ target triple = "thumbv7-apple-macosx10.6.7"
 
 define i32 @inlineprinter(i8* %ptr, float %val, i8 zeroext %c) nounwind optsize ssp {
 entry:
-  tail call void @llvm.dbg.value(metadata i8* %ptr, i64 0, metadata !8, metadata !MDExpression()), !dbg !24
-  tail call void @llvm.dbg.value(metadata float %val, i64 0, metadata !10, metadata !MDExpression()), !dbg !25
-  tail call void @llvm.dbg.value(metadata i8 %c, i64 0, metadata !12, metadata !MDExpression()), !dbg !26
+  tail call void @llvm.dbg.value(metadata i8* %ptr, i64 0, metadata !8, metadata !DIExpression()), !dbg !24
+  tail call void @llvm.dbg.value(metadata float %val, i64 0, metadata !10, metadata !DIExpression()), !dbg !25
+  tail call void @llvm.dbg.value(metadata i8 %c, i64 0, metadata !12, metadata !DIExpression()), !dbg !26
   %conv = fpext float %val to double, !dbg !27
   %conv3 = zext i8 %c to i32, !dbg !27
   %call = tail call i32 (i8*, ...) @printf(i8* getelementptr inbounds ([11 x i8], [11 x i8]* @.str, i32 0, i32 0), i8* %ptr, double %conv, i32 %conv3) nounwind optsize, !dbg !27
@@ -27,9 +27,9 @@ declare i32 @printf(i8* nocapture, ...) nounwind optsize
 
 define i32 @printer(i8* %ptr, float %val, i8 zeroext %c) nounwind optsize noinline ssp {
 entry:
-  tail call void @llvm.dbg.value(metadata i8* %ptr, i64 0, metadata !14, metadata !MDExpression()), !dbg !30
-  tail call void @llvm.dbg.value(metadata float %val, i64 0, metadata !15, metadata !MDExpression()), !dbg !31
-  tail call void @llvm.dbg.value(metadata i8 %c, i64 0, metadata !16, metadata !MDExpression()), !dbg !32
+  tail call void @llvm.dbg.value(metadata i8* %ptr, i64 0, metadata !14, metadata !DIExpression()), !dbg !30
+  tail call void @llvm.dbg.value(metadata float %val, i64 0, metadata !15, metadata !DIExpression()), !dbg !31
+  tail call void @llvm.dbg.value(metadata i8 %c, i64 0, metadata !16, metadata !DIExpression()), !dbg !32
   %conv = fpext float %val to double, !dbg !33
   %conv3 = zext i8 %c to i32, !dbg !33
   %call = tail call i32 (i8*, ...) @printf(i8* getelementptr inbounds ([11 x i8], [11 x i8]* @.str, i32 0, i32 0), i8* %ptr, double %conv, i32 %conv3) nounwind optsize, !dbg !33
@@ -38,19 +38,19 @@ entry:
 
 define i32 @main(i32 %argc, i8** nocapture %argv) nounwind optsize ssp {
 entry:
-  tail call void @llvm.dbg.value(metadata i32 %argc, i64 0, metadata !17, metadata !MDExpression()), !dbg !36
-  tail call void @llvm.dbg.value(metadata i8** %argv, i64 0, metadata !18, metadata !MDExpression()), !dbg !37
+  tail call void @llvm.dbg.value(metadata i32 %argc, i64 0, metadata !17, metadata !DIExpression()), !dbg !36
+  tail call void @llvm.dbg.value(metadata i8** %argv, i64 0, metadata !18, metadata !DIExpression()), !dbg !37
   %conv = sitofp i32 %argc to double, !dbg !38
   %add = fadd double %conv, 5.555552e+05, !dbg !38
   %conv1 = fptrunc double %add to float, !dbg !38
-  tail call void @llvm.dbg.value(metadata float %conv1, i64 0, metadata !22, metadata !MDExpression()), !dbg !38
+  tail call void @llvm.dbg.value(metadata float %conv1, i64 0, metadata !22, metadata !DIExpression()), !dbg !38
   %call = tail call i32 @puts(i8* getelementptr inbounds ([6 x i8], [6 x i8]* @.str1, i32 0, i32 0)) nounwind optsize, !dbg !39
   %add.ptr = getelementptr i8, i8* bitcast (i32 (i32, i8**)* @main to i8*), i32 %argc, !dbg !40
   %add5 = add nsw i32 %argc, 97, !dbg !40
   %conv6 = trunc i32 %add5 to i8, !dbg !40
-  tail call void @llvm.dbg.value(metadata i8* %add.ptr, i64 0, metadata !58, metadata !MDExpression()) nounwind, !dbg !41
-  tail call void @llvm.dbg.value(metadata float %conv1, i64 0, metadata !60, metadata !MDExpression()) nounwind, !dbg !42
-  tail call void @llvm.dbg.value(metadata i8 %conv6, i64 0, metadata !62, metadata !MDExpression()) nounwind, !dbg !43
+  tail call void @llvm.dbg.value(metadata i8* %add.ptr, i64 0, metadata !58, metadata !DIExpression()) nounwind, !dbg !41
+  tail call void @llvm.dbg.value(metadata float %conv1, i64 0, metadata !60, metadata !DIExpression()) nounwind, !dbg !42
+  tail call void @llvm.dbg.value(metadata i8 %conv6, i64 0, metadata !62, metadata !DIExpression()) nounwind, !dbg !43
   %conv.i = fpext float %conv1 to double, !dbg !44
   %conv3.i = and i32 %add5, 255, !dbg !44
   %call.i = tail call i32 (i8*, ...) @printf(i8* getelementptr inbounds ([11 x i8], [11 x i8]* @.str, i32 0, i32 0), i8* %add.ptr, double %conv.i, i32 %conv3.i) nounwind optsize, !dbg !44
@@ -65,62 +65,62 @@ declare void @llvm.dbg.value(metadata, i64, metadata, metadata) nounwind readnon
 !llvm.dbg.cu = !{!2}
 !llvm.module.flags = !{!53}
 
-!0 = !MDSubprogram(name: "inlineprinter", line: 5, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: true, scopeLine: 5, file: !51, scope: !1, type: !3, function: i32 (i8*, float, i8)* @inlineprinter, variables: !48)
-!1 = !MDFile(filename: "a.c", directory: "/private/tmp")
-!2 = !MDCompileUnit(language: DW_LANG_C99, producer: "clang version 3.0 (trunk 129915)", isOptimized: true, emissionKind: 1, file: !51, enums: !52, retainedTypes: !52, subprograms: !47, imports:  null)
-!3 = !MDSubroutineType(types: !4)
+!0 = !DISubprogram(name: "inlineprinter", line: 5, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: true, scopeLine: 5, file: !51, scope: !1, type: !3, function: i32 (i8*, float, i8)* @inlineprinter, variables: !48)
+!1 = !DIFile(filename: "a.c", directory: "/private/tmp")
+!2 = !DICompileUnit(language: DW_LANG_C99, producer: "clang version 3.0 (trunk 129915)", isOptimized: true, emissionKind: 1, file: !51, enums: !52, retainedTypes: !52, subprograms: !47, imports:  null)
+!3 = !DISubroutineType(types: !4)
 !4 = !{!5}
-!5 = !MDBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
-!6 = !MDSubprogram(name: "printer", line: 12, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: true, scopeLine: 12, file: !51, scope: !1, type: !3, function: i32 (i8*, float, i8)* @printer, variables: !49)
-!7 = !MDSubprogram(name: "main", line: 18, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: true, scopeLine: 18, file: !51, scope: !1, type: !3, function: i32 (i32, i8**)* @main, variables: !50)
-!8 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "ptr", line: 4, arg: 1, scope: !0, file: !1, type: !9)
-!9 = !MDDerivedType(tag: DW_TAG_pointer_type, size: 32, align: 32, scope: !2, baseType: null)
-!10 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "val", line: 4, arg: 2, scope: !0, file: !1, type: !11)
-!11 = !MDBasicType(tag: DW_TAG_base_type, name: "float", size: 32, align: 32, encoding: DW_ATE_float)
-!12 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "c", line: 4, arg: 3, scope: !0, file: !1, type: !13)
-!13 = !MDBasicType(tag: DW_TAG_base_type, name: "unsigned char", size: 8, align: 8, encoding: DW_ATE_unsigned_char)
+!5 = !DIBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
+!6 = !DISubprogram(name: "printer", line: 12, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: true, scopeLine: 12, file: !51, scope: !1, type: !3, function: i32 (i8*, float, i8)* @printer, variables: !49)
+!7 = !DISubprogram(name: "main", line: 18, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: true, scopeLine: 18, file: !51, scope: !1, type: !3, function: i32 (i32, i8**)* @main, variables: !50)
+!8 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "ptr", line: 4, arg: 1, scope: !0, file: !1, type: !9)
+!9 = !DIDerivedType(tag: DW_TAG_pointer_type, size: 32, align: 32, scope: !2, baseType: null)
+!10 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "val", line: 4, arg: 2, scope: !0, file: !1, type: !11)
+!11 = !DIBasicType(tag: DW_TAG_base_type, name: "float", size: 32, align: 32, encoding: DW_ATE_float)
+!12 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "c", line: 4, arg: 3, scope: !0, file: !1, type: !13)
+!13 = !DIBasicType(tag: DW_TAG_base_type, name: "unsigned char", size: 8, align: 8, encoding: DW_ATE_unsigned_char)
 
-!58 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "ptr", line: 4, arg: 1, scope: !0, file: !1, type: !9)
-!60 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "val", line: 4, arg: 2, scope: !0, file: !1, type: !11)
-!62 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "c", line: 4, arg: 3, scope: !0, file: !1, type: !13)
+!58 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "ptr", line: 4, arg: 1, scope: !0, file: !1, type: !9)
+!60 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "val", line: 4, arg: 2, scope: !0, file: !1, type: !11)
+!62 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "c", line: 4, arg: 3, scope: !0, file: !1, type: !13)
 
-!14 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "ptr", line: 11, arg: 1, scope: !6, file: !1, type: !9)
-!15 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "val", line: 11, arg: 2, scope: !6, file: !1, type: !11)
-!16 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "c", line: 11, arg: 3, scope: !6, file: !1, type: !13)
-!17 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "argc", line: 17, arg: 1, scope: !7, file: !1, type: !5)
-!18 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "argv", line: 17, arg: 2, scope: !7, file: !1, type: !19)
-!19 = !MDDerivedType(tag: DW_TAG_pointer_type, size: 32, align: 32, scope: !2, baseType: !20)
-!20 = !MDDerivedType(tag: DW_TAG_pointer_type, size: 32, align: 32, scope: !2, baseType: !21)
-!21 = !MDBasicType(tag: DW_TAG_base_type, name: "char", size: 8, align: 8, encoding: DW_ATE_signed_char)
-!22 = !MDLocalVariable(tag: DW_TAG_auto_variable, name: "dval", line: 19, scope: !23, file: !1, type: !11)
-!23 = distinct !MDLexicalBlock(line: 18, column: 1, file: !51, scope: !7)
-!24 = !MDLocation(line: 4, column: 22, scope: !0)
-!25 = !MDLocation(line: 4, column: 33, scope: !0)
-!26 = !MDLocation(line: 4, column: 52, scope: !0)
-!27 = !MDLocation(line: 6, column: 3, scope: !28)
-!28 = distinct !MDLexicalBlock(line: 5, column: 1, file: !51, scope: !0)
-!29 = !MDLocation(line: 7, column: 3, scope: !28)
-!30 = !MDLocation(line: 11, column: 42, scope: !6)
-!31 = !MDLocation(line: 11, column: 53, scope: !6)
-!32 = !MDLocation(line: 11, column: 72, scope: !6)
-!33 = !MDLocation(line: 13, column: 3, scope: !34)
-!34 = distinct !MDLexicalBlock(line: 12, column: 1, file: !51, scope: !6)
-!35 = !MDLocation(line: 14, column: 3, scope: !34)
-!36 = !MDLocation(line: 17, column: 15, scope: !7)
-!37 = !MDLocation(line: 17, column: 28, scope: !7)
-!38 = !MDLocation(line: 19, column: 31, scope: !23)
-!39 = !MDLocation(line: 20, column: 3, scope: !23)
-!40 = !MDLocation(line: 21, column: 3, scope: !23)
-!41 = !MDLocation(line: 4, column: 22, scope: !0, inlinedAt: !40)
-!42 = !MDLocation(line: 4, column: 33, scope: !0, inlinedAt: !40)
-!43 = !MDLocation(line: 4, column: 52, scope: !0, inlinedAt: !40)
-!44 = !MDLocation(line: 6, column: 3, scope: !28, inlinedAt: !40)
-!45 = !MDLocation(line: 22, column: 3, scope: !23)
-!46 = !MDLocation(line: 23, column: 1, scope: !23)
+!14 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "ptr", line: 11, arg: 1, scope: !6, file: !1, type: !9)
+!15 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "val", line: 11, arg: 2, scope: !6, file: !1, type: !11)
+!16 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "c", line: 11, arg: 3, scope: !6, file: !1, type: !13)
+!17 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "argc", line: 17, arg: 1, scope: !7, file: !1, type: !5)
+!18 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "argv", line: 17, arg: 2, scope: !7, file: !1, type: !19)
+!19 = !DIDerivedType(tag: DW_TAG_pointer_type, size: 32, align: 32, scope: !2, baseType: !20)
+!20 = !DIDerivedType(tag: DW_TAG_pointer_type, size: 32, align: 32, scope: !2, baseType: !21)
+!21 = !DIBasicType(tag: DW_TAG_base_type, name: "char", size: 8, align: 8, encoding: DW_ATE_signed_char)
+!22 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "dval", line: 19, scope: !23, file: !1, type: !11)
+!23 = distinct !DILexicalBlock(line: 18, column: 1, file: !51, scope: !7)
+!24 = !DILocation(line: 4, column: 22, scope: !0)
+!25 = !DILocation(line: 4, column: 33, scope: !0)
+!26 = !DILocation(line: 4, column: 52, scope: !0)
+!27 = !DILocation(line: 6, column: 3, scope: !28)
+!28 = distinct !DILexicalBlock(line: 5, column: 1, file: !51, scope: !0)
+!29 = !DILocation(line: 7, column: 3, scope: !28)
+!30 = !DILocation(line: 11, column: 42, scope: !6)
+!31 = !DILocation(line: 11, column: 53, scope: !6)
+!32 = !DILocation(line: 11, column: 72, scope: !6)
+!33 = !DILocation(line: 13, column: 3, scope: !34)
+!34 = distinct !DILexicalBlock(line: 12, column: 1, file: !51, scope: !6)
+!35 = !DILocation(line: 14, column: 3, scope: !34)
+!36 = !DILocation(line: 17, column: 15, scope: !7)
+!37 = !DILocation(line: 17, column: 28, scope: !7)
+!38 = !DILocation(line: 19, column: 31, scope: !23)
+!39 = !DILocation(line: 20, column: 3, scope: !23)
+!40 = !DILocation(line: 21, column: 3, scope: !23)
+!41 = !DILocation(line: 4, column: 22, scope: !0, inlinedAt: !40)
+!42 = !DILocation(line: 4, column: 33, scope: !0, inlinedAt: !40)
+!43 = !DILocation(line: 4, column: 52, scope: !0, inlinedAt: !40)
+!44 = !DILocation(line: 6, column: 3, scope: !28, inlinedAt: !40)
+!45 = !DILocation(line: 22, column: 3, scope: !23)
+!46 = !DILocation(line: 23, column: 1, scope: !23)
 !47 = !{!0, !6, !7}
 !48 = !{!8, !10, !12}
 !49 = !{!14, !15, !16}
 !50 = !{!17, !18, !22}
-!51 = !MDFile(filename: "a.c", directory: "/private/tmp")
+!51 = !DIFile(filename: "a.c", directory: "/private/tmp")
 !52 = !{}
 !53 = !{i32 1, !"Debug Info Version", i32 3}
diff --git a/test/CodeGen/ARM/debug-info-sreg2.ll b/test/CodeGen/ARM/debug-info-sreg2.ll
index e5f7a27f098..f22559efad4 100644
--- a/test/CodeGen/ARM/debug-info-sreg2.ll
+++ b/test/CodeGen/ARM/debug-info-sreg2.ll
@@ -15,7 +15,7 @@ target triple = "thumbv7-apple-macosx10.6.7"
 define void @_Z3foov() optsize ssp {
 entry:
   %call = tail call float @_Z3barv() optsize, !dbg !11
-  tail call void @llvm.dbg.value(metadata float %call, i64 0, metadata !5, metadata !MDExpression()), !dbg !11
+  tail call void @llvm.dbg.value(metadata float %call, i64 0, metadata !5, metadata !DIExpression()), !dbg !11
   %call16 = tail call float @_Z2f2v() optsize, !dbg !12
   %cmp7 = fcmp olt float %call, %call16, !dbg !12
   br i1 %cmp7, label %for.body, label %for.end, !dbg !12
@@ -43,24 +43,24 @@ declare void @llvm.dbg.value(metadata, i64, metadata, metadata) nounwind readnon
 !llvm.dbg.cu = !{!0}
 !llvm.module.flags = !{!20}
 
-!0 = !MDCompileUnit(language: DW_LANG_C_plus_plus, producer: "clang version 3.0 (trunk 130845)", isOptimized: true, emissionKind: 1, file: !18, enums: !19, retainedTypes: !19, subprograms: !16, imports:  null)
-!1 = !MDSubprogram(name: "foo", linkageName: "_Z3foov", line: 5, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: true, scopeLine: 5, file: !18, scope: !2, type: !3, function: void ()* @_Z3foov, variables: !17)
-!2 = !MDFile(filename: "k.cc", directory: "/private/tmp")
-!3 = !MDSubroutineType(types: !4)
+!0 = !DICompileUnit(language: DW_LANG_C_plus_plus, producer: "clang version 3.0 (trunk 130845)", isOptimized: true, emissionKind: 1, file: !18, enums: !19, retainedTypes: !19, subprograms: !16, imports:  null)
+!1 = !DISubprogram(name: "foo", linkageName: "_Z3foov", line: 5, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: true, scopeLine: 5, file: !18, scope: !2, type: !3, function: void ()* @_Z3foov, variables: !17)
+!2 = !DIFile(filename: "k.cc", directory: "/private/tmp")
+!3 = !DISubroutineType(types: !4)
 !4 = !{null}
-!5 = !MDLocalVariable(tag: DW_TAG_auto_variable, name: "k", line: 6, scope: !6, file: !2, type: !7)
-!6 = distinct !MDLexicalBlock(line: 5, column: 12, file: !18, scope: !1)
-!7 = !MDBasicType(tag: DW_TAG_base_type, name: "float", size: 32, align: 32, encoding: DW_ATE_float)
-!8 = !MDLocalVariable(tag: DW_TAG_auto_variable, name: "y", line: 8, scope: !9, file: !2, type: !7)
-!9 = distinct !MDLexicalBlock(line: 7, column: 25, file: !18, scope: !10)
-!10 = distinct !MDLexicalBlock(line: 7, column: 3, file: !18, scope: !6)
-!11 = !MDLocation(line: 6, column: 18, scope: !6)
-!12 = !MDLocation(line: 7, column: 3, scope: !6)
-!13 = !MDLocation(line: 8, column: 20, scope: !9)
-!14 = !MDLocation(line: 7, column: 20, scope: !10)
-!15 = !MDLocation(line: 10, column: 1, scope: !6)
+!5 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "k", line: 6, scope: !6, file: !2, type: !7)
+!6 = distinct !DILexicalBlock(line: 5, column: 12, file: !18, scope: !1)
+!7 = !DIBasicType(tag: DW_TAG_base_type, name: "float", size: 32, align: 32, encoding: DW_ATE_float)
+!8 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "y", line: 8, scope: !9, file: !2, type: !7)
+!9 = distinct !DILexicalBlock(line: 7, column: 25, file: !18, scope: !10)
+!10 = distinct !DILexicalBlock(line: 7, column: 3, file: !18, scope: !6)
+!11 = !DILocation(line: 6, column: 18, scope: !6)
+!12 = !DILocation(line: 7, column: 3, scope: !6)
+!13 = !DILocation(line: 8, column: 20, scope: !9)
+!14 = !DILocation(line: 7, column: 20, scope: !10)
+!15 = !DILocation(line: 10, column: 1, scope: !6)
 !16 = !{!1}
 !17 = !{!5, !8}
-!18 = !MDFile(filename: "k.cc", directory: "/private/tmp")
+!18 = !DIFile(filename: "k.cc", directory: "/private/tmp")
 !19 = !{}
 !20 = !{i32 1, !"Debug Info Version", i32 3}
diff --git a/test/CodeGen/ARM/debug-segmented-stacks.ll b/test/CodeGen/ARM/debug-segmented-stacks.ll
index 72027dedec1..47d366e49de 100644
--- a/test/CodeGen/ARM/debug-segmented-stacks.ll
+++ b/test/CodeGen/ARM/debug-segmented-stacks.ll
@@ -39,40 +39,40 @@ define void @test_basic() #0 {
 ; ARM-linux       .cfi_same_value r5
 }
 
-!0 = !MDCompileUnit(language: DW_LANG_C99, producer: "clang version 3.5 ", isOptimized: false, emissionKind: 0, file: !1, enums: !2, retainedTypes: !2, subprograms: !3, globals: !2, imports: !2)
-!1 = !MDFile(filename: "var.c", directory: "/tmp")
+!0 = !DICompileUnit(language: DW_LANG_C99, producer: "clang version 3.5 ", isOptimized: false, emissionKind: 0, file: !1, enums: !2, retainedTypes: !2, subprograms: !3, globals: !2, imports: !2)
+!1 = !DIFile(filename: "var.c", directory: "/tmp")
 !2 = !{}
 !3 = !{!4}
-!4 = !MDSubprogram(name: "test_basic", line: 5, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 5, file: !1, scope: !5, type: !6, function: void ()* @test_basic, variables: !2)
-!5 = !MDFile(filename: "var.c", directory: "/tmp")
-!6 = !MDSubroutineType(types: !7)
+!4 = !DISubprogram(name: "test_basic", line: 5, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 5, file: !1, scope: !5, type: !6, function: void ()* @test_basic, variables: !2)
+!5 = !DIFile(filename: "var.c", directory: "/tmp")
+!6 = !DISubroutineType(types: !7)
 !7 = !{!8, !8}
-!8 = !MDBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
+!8 = !DIBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
 !9 = !{i32 2, !"Dwarf Version", i32 4}
 !10 = !{i32 1, !"Debug Info Version", i32 3}
 !11 = !{!"clang version 3.5 "}
-!12 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "count", line: 5, arg: 1, scope: !4, file: !5, type: !8)
-!13 = !MDLocation(line: 5, scope: !4)
-!14 = !MDLocalVariable(tag: DW_TAG_auto_variable, name: "vl", line: 6, scope: !4, file: !5, type: !15)
-!15 = !MDDerivedType(tag: DW_TAG_typedef, name: "va_list", line: 30, file: !16, baseType: !17)
-!16 = !MDFile(filename: "/linux-x86_64-high/gcc_4.7.2/dbg/llvm/bin/../lib/clang/3.5/include/stdarg.h", directory: "/tmp")
-!17 = !MDDerivedType(tag: DW_TAG_typedef, name: "__builtin_va_list", line: 6, file: !1, baseType: !18)
-!18 = !MDCompositeType(tag: DW_TAG_structure_type, name: "__va_list", line: 6, size: 32, align: 32, file: !1, elements: !19)
+!12 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "count", line: 5, arg: 1, scope: !4, file: !5, type: !8)
+!13 = !DILocation(line: 5, scope: !4)
+!14 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "vl", line: 6, scope: !4, file: !5, type: !15)
+!15 = !DIDerivedType(tag: DW_TAG_typedef, name: "va_list", line: 30, file: !16, baseType: !17)
+!16 = !DIFile(filename: "/linux-x86_64-high/gcc_4.7.2/dbg/llvm/bin/../lib/clang/3.5/include/stdarg.h", directory: "/tmp")
+!17 = !DIDerivedType(tag: DW_TAG_typedef, name: "__builtin_va_list", line: 6, file: !1, baseType: !18)
+!18 = !DICompositeType(tag: DW_TAG_structure_type, name: "__va_list", line: 6, size: 32, align: 32, file: !1, elements: !19)
 !19 = !{!20}
-!20 = !MDDerivedType(tag: DW_TAG_member, name: "__ap", line: 6, size: 32, align: 32, file: !1, scope: !18, baseType: !21)
-!21 = !MDDerivedType(tag: DW_TAG_pointer_type, size: 32, align: 32, baseType: null)
-!22 = !MDLocation(line: 6, scope: !4)
-!23 = !MDLocation(line: 7, scope: !4)
-!24 = !MDLocalVariable(tag: DW_TAG_auto_variable, name: "test_basic", line: 8, scope: !4, file: !5, type: !8)
-!25 = !MDLocation(line: 8, scope: !4)
-!26 = !MDLocalVariable(tag: DW_TAG_auto_variable, name: "i", line: 9, scope: !27, file: !5, type: !8)
-!27 = distinct !MDLexicalBlock(line: 9, column: 0, file: !1, scope: !4)
-!28 = !MDLocation(line: 9, scope: !27)
-!29 = !MDLocation(line: 10, scope: !30)
-!30 = distinct !MDLexicalBlock(line: 9, column: 0, file: !1, scope: !27)
-!31 = !MDLocation(line: 11, scope: !30)
-!32 = !MDLocation(line: 12, scope: !4)
-!33 = !MDLocation(line: 13, scope: !4)
+!20 = !DIDerivedType(tag: DW_TAG_member, name: "__ap", line: 6, size: 32, align: 32, file: !1, scope: !18, baseType: !21)
+!21 = !DIDerivedType(tag: DW_TAG_pointer_type, size: 32, align: 32, baseType: null)
+!22 = !DILocation(line: 6, scope: !4)
+!23 = !DILocation(line: 7, scope: !4)
+!24 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "test_basic", line: 8, scope: !4, file: !5, type: !8)
+!25 = !DILocation(line: 8, scope: !4)
+!26 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "i", line: 9, scope: !27, file: !5, type: !8)
+!27 = distinct !DILexicalBlock(line: 9, column: 0, file: !1, scope: !4)
+!28 = !DILocation(line: 9, scope: !27)
+!29 = !DILocation(line: 10, scope: !30)
+!30 = distinct !DILexicalBlock(line: 9, column: 0, file: !1, scope: !27)
+!31 = !DILocation(line: 11, scope: !30)
+!32 = !DILocation(line: 12, scope: !4)
+!33 = !DILocation(line: 13, scope: !4)
 
 ; Just to prevent the alloca from being optimized away
 declare void @dummy_use(i32*, i32)
diff --git a/test/CodeGen/ARM/disable-fp-elim.ll b/test/CodeGen/ARM/disable-fp-elim.ll
new file mode 100644
index 00000000000..dafeda2ac76
--- /dev/null
+++ b/test/CodeGen/ARM/disable-fp-elim.ll
@@ -0,0 +1,25 @@
+; RUN: llc < %s -mtriple armv7-none-linux-gnueabi -O1 | FileCheck %s --check-prefix=DISABLE-FP-ELIM
+; RUN: llc < %s -mtriple armv7-none-linux-gnueabi -disable-fp-elim -O1 | FileCheck %s --check-prefix=DISABLE-FP-ELIM
+; RUN: llc < %s -mtriple armv7-none-linux-gnueabi -disable-fp-elim=false -O1 | FileCheck %s --check-prefix=ENABLE-FP-ELIM
+; RUN: llc < %s -mtriple armv7-none-linux-gnueabi -disable-fp-elim=false -O0 | FileCheck %s --check-prefix=DISABLE-FP-ELIM
+
+; Check that command line option "-disable-fp-elim" overrides function attribute
+; "no-frame-pointer-elim". Also, check frame pointer elimination is disabled
+; when fast-isel is used.
+
+; ENABLE-FP-ELIM-NOT: .setfp
+; DISABLE-FP-ELIM: .setfp r11, sp
+
+define i32 @foo1(i32 %a, i32 %b, i32 %c, i32 %d, i32 %e) #0 {
+entry:
+  %call = tail call i32 @foo2(i32 %a)
+  %add = add i32 %c, %b
+  %add1 = add i32 %add, %d
+  %add2 = add i32 %add1, %e
+  %add3 = add i32 %add2, %call
+  ret i32 %add3
+}
+
+declare i32 @foo2(i32)
+
+attributes #0 = { nounwind "no-frame-pointer-elim"="true" }
diff --git a/test/CodeGen/ARM/disable-tail-calls.ll b/test/CodeGen/ARM/disable-tail-calls.ll
new file mode 100644
index 00000000000..ab3731a839a
--- /dev/null
+++ b/test/CodeGen/ARM/disable-tail-calls.ll
@@ -0,0 +1,40 @@
+; RUN: llc < %s -mtriple=arm-unknown-unknown | FileCheck %s --check-prefix=NO-OPTION
+; RUN: llc < %s -mtriple=arm-unknown-unknown -disable-tail-calls | FileCheck %s --check-prefix=DISABLE-TRUE
+; RUN: llc < %s -mtriple=arm-unknown-unknown -disable-tail-calls=false | FileCheck %s --check-prefix=DISABLE-FALSE
+
+; Check that command line option "-disable-tail-calls" overrides function
+; attribute "disable-tail-calls".
+
+; NO-OPTION-LABEL: {{\_?}}func_attr
+; NO-OPTION: bl {{\_?}}callee
+
+; DISABLE-FALSE-LABEL: {{\_?}}func_attr
+; DISABLE-FALSE: b {{\_?}}callee
+
+; DISABLE-TRUE-LABEL: {{\_?}}func_attr
+; DISABLE-TRUE: bl {{\_?}}callee
+
+define i32 @func_attr(i32 %a) #0 {
+entry:
+  %call = tail call i32 @callee(i32 %a)
+  ret i32 %call
+}
+
+; NO-OPTION-LABEL: {{\_?}}func_noattr
+; NO-OPTION: b {{\_?}}callee
+
+; DISABLE-FALSE-LABEL: {{\_?}}func_noattr
+; DISABLE-FALSE: b {{\_?}}callee
+
+; DISABLE-TRUE-LABEL: {{\_?}}func_noattr
+; DISABLE-TRUE: bl {{\_?}}callee
+
+define i32 @func_noattr(i32 %a) {
+entry:
+  %call = tail call i32 @callee(i32 %a)
+  ret i32 %call
+}
+
+declare i32 @callee(i32)
+
+attributes #0 = { "disable-tail-calls"="true" }
diff --git a/test/CodeGen/ARM/divmod-eabi.ll b/test/CodeGen/ARM/divmod-eabi.ll
index 7f72048d391..e630a95368e 100644
--- a/test/CodeGen/ARM/divmod-eabi.ll
+++ b/test/CodeGen/ARM/divmod-eabi.ll
@@ -1,5 +1,7 @@
 ; RUN: llc -mtriple armv7-none-eabi %s -o - | FileCheck %s --check-prefix=EABI
 ; RUN: llc -mtriple armv7-none-eabihf %s -o - | FileCheck %s --check-prefix=EABI
+; Both "none-eabi" and "androideabi" must lower SREM/UREM to __aeabi_{u,i}divmod
+; RUN: llc -mtriple armv7-linux-androideabi %s -o - | FileCheck %s --check-prefix=EABI
 ; RUN: llc -mtriple armv7-linux-gnueabi %s -o - | FileCheck %s --check-prefix=GNU
 ; RUN: llc -mtriple armv7-apple-darwin %s -o - | FileCheck %s --check-prefix=DARWIN
 ; FIXME: long-term, we will use "-apple-macho" and won't need this exception:
diff --git a/test/CodeGen/ARM/dwarf-eh.ll b/test/CodeGen/ARM/dwarf-eh.ll
index c890206b353..68f8e95b5e7 100644
--- a/test/CodeGen/ARM/dwarf-eh.ll
+++ b/test/CodeGen/ARM/dwarf-eh.ll
@@ -17,7 +17,7 @@ target triple = "armv5e--netbsd-eabi"
 @_ZTS9exception = linkonce_odr constant [11 x i8] c"9exception\00"
 @_ZTI9exception = linkonce_odr unnamed_addr constant { i8*, i8* } { i8* bitcast (i8** getelementptr inbounds (i8*, i8** @_ZTVN10__cxxabiv117__class_type_infoE, i32 2) to i8*), i8* getelementptr inbounds ([11 x i8], [11 x i8]* @_ZTS9exception, i32 0, i32 0) }
 
-define void @f() uwtable {
+define void @f() uwtable personality i8* bitcast (i32 (...)* @__gxx_personality_v0 to i8*) {
   %1 = alloca i8*
   %2 = alloca i32
   %e = alloca %struct.exception*, align 4
@@ -26,7 +26,7 @@ define void @f() uwtable {
 
   br label %16
 
-  %5 = landingpad { i8*, i32 } personality i8* bitcast (i32 (...)* @__gxx_personality_v0 to i8*)
+  %5 = landingpad { i8*, i32 }
           catch i8* bitcast ({ i8*, i8* }* @_ZTI9exception to i8*)
   %6 = extractvalue { i8*, i32 } %5, 0
   store i8* %6, i8** %1
diff --git a/test/CodeGen/ARM/eh-dispcont.ll b/test/CodeGen/ARM/eh-dispcont.ll
index 57ab15feca5..e9871aa7dc7 100644
--- a/test/CodeGen/ARM/eh-dispcont.ll
+++ b/test/CodeGen/ARM/eh-dispcont.ll
@@ -7,7 +7,7 @@
 
 @_ZTIi = external constant i8*
 
-define i32 @main() #0 {
+define i32 @main() #0 personality i8* bitcast (i32 (...)* @__gxx_personality_sj0 to i8*) {
 entry:
   %exception = tail call i8* @__cxa_allocate_exception(i32 4) #1
   %0 = bitcast i8* %exception to i32*
@@ -16,7 +16,7 @@ entry:
           to label %unreachable unwind label %lpad
 
 lpad:                                             ; preds = %entry
-  %1 = landingpad { i8*, i32 } personality i8* bitcast (i32 (...)* @__gxx_personality_sj0 to i8*)
+  %1 = landingpad { i8*, i32 }
           catch i8* null
   %2 = extractvalue { i8*, i32 } %1, 0
   %3 = tail call i8* @__cxa_begin_catch(i8* %2) #1
diff --git a/test/CodeGen/ARM/eh-resume-darwin.ll b/test/CodeGen/ARM/eh-resume-darwin.ll
index d1252f4c986..0cd49775cfb 100644
--- a/test/CodeGen/ARM/eh-resume-darwin.ll
+++ b/test/CodeGen/ARM/eh-resume-darwin.ll
@@ -5,7 +5,7 @@ declare void @func()
 
 declare i32 @__gxx_personality_sj0(...)
 
-define void @test0() {
+define void @test0() personality i8* bitcast (i32 (...)* @__gxx_personality_sj0 to i8*) {
 entry:
   invoke void @func()
     to label %cont unwind label %lpad
@@ -14,7 +14,7 @@ cont:
   ret void
 
 lpad:
-  %exn = landingpad { i8*, i32 } personality i8* bitcast (i32 (...)* @__gxx_personality_sj0 to i8*)
+  %exn = landingpad { i8*, i32 }
            cleanup
   resume { i8*, i32 } %exn
 }
diff --git a/test/CodeGen/ARM/ehabi-filters.ll b/test/CodeGen/ARM/ehabi-filters.ll
index f86b66c30c5..4faa29e2038 100644
--- a/test/CodeGen/ARM/ehabi-filters.ll
+++ b/test/CodeGen/ARM/ehabi-filters.ll
@@ -14,7 +14,7 @@ declare void @__cxa_throw(i8*, i8*, i8*)
 
 declare void @__cxa_call_unexpected(i8*)
 
-define i32 @main() {
+define i32 @main() personality i8* bitcast (i32 (...)* @__gxx_personality_v0 to i8*) {
 ; CHECK-LABEL: main:
 entry:
   %exception.i = tail call i8* @__cxa_allocate_exception(i32 4) nounwind
@@ -24,7 +24,7 @@ entry:
           to label %unreachable.i unwind label %lpad.i
 
 lpad.i:                                           ; preds = %entry
-  %1 = landingpad { i8*, i32 } personality i8* bitcast (i32 (...)* @__gxx_personality_v0 to i8*)
+  %1 = landingpad { i8*, i32 }
           filter [1 x i8*] [i8* bitcast (i8** @_ZTIi to i8*)]
           catch i8* bitcast (i8** @_ZTIi to i8*)
 ; CHECK: .long	_ZTIi(target2)          @ TypeInfo 1
@@ -45,7 +45,7 @@ unreachable.i:                                    ; preds = %entry
   unreachable
 
 lpad:                                             ; preds = %ehspec.unexpected.i
-  %4 = landingpad { i8*, i32 } personality i8* bitcast (i32 (...)* @__gxx_personality_v0 to i8*)
+  %4 = landingpad { i8*, i32 }
           catch i8* bitcast (i8** @_ZTIi to i8*)
   br label %lpad.body
 
diff --git a/test/CodeGen/ARM/ehabi-handlerdata-nounwind.ll b/test/CodeGen/ARM/ehabi-handlerdata-nounwind.ll
index 42ca9888abb..3d380bf8f22 100644
--- a/test/CodeGen/ARM/ehabi-handlerdata-nounwind.ll
+++ b/test/CodeGen/ARM/ehabi-handlerdata-nounwind.ll
@@ -25,12 +25,12 @@ declare i8* @__cxa_begin_catch(i8*)
 
 declare void @__cxa_end_catch()
 
-define void @test1() nounwind {
+define void @test1() nounwind personality i8* bitcast (i32 (...)* @__gxx_personality_v0 to i8*) {
 entry:
   invoke void @throw_exception() to label %try.cont unwind label %lpad
 
 lpad:
-  %0 = landingpad { i8*, i32 } personality i8* bitcast (i32 (...)* @__gxx_personality_v0 to i8*)
+  %0 = landingpad { i8*, i32 }
           catch i8* null
   %1 = extractvalue { i8*, i32 } %0, 0
   %2 = tail call i8* @__cxa_begin_catch(i8* %1)
diff --git a/test/CodeGen/ARM/ehabi-handlerdata.ll b/test/CodeGen/ARM/ehabi-handlerdata.ll
index 7045902f99c..c53b36ffe18 100644
--- a/test/CodeGen/ARM/ehabi-handlerdata.ll
+++ b/test/CodeGen/ARM/ehabi-handlerdata.ll
@@ -23,12 +23,12 @@ declare i8* @__cxa_begin_catch(i8*)
 
 declare void @__cxa_end_catch()
 
-define void @test1() {
+define void @test1() personality i8* bitcast (i32 (...)* @__gxx_personality_v0 to i8*) {
 entry:
   invoke void @throw_exception() to label %try.cont unwind label %lpad
 
 lpad:
-  %0 = landingpad { i8*, i32 } personality i8* bitcast (i32 (...)* @__gxx_personality_v0 to i8*)
+  %0 = landingpad { i8*, i32 }
           catch i8* null
   %1 = extractvalue { i8*, i32 } %0, 0
   %2 = tail call i8* @__cxa_begin_catch(i8* %1)
diff --git a/test/CodeGen/ARM/ehabi.ll b/test/CodeGen/ARM/ehabi.ll
index ebf0c2a0033..923cffcf653 100644
--- a/test/CodeGen/ARM/ehabi.ll
+++ b/test/CodeGen/ARM/ehabi.ll
@@ -89,14 +89,13 @@ declare void @_Z5printddddd(double, double, double, double, double)
 
 define void @_Z4testiiiiiddddd(i32 %a, i32 %b, i32 %c, i32 %d, i32 %e,
                                double %m, double %n, double %p,
-                               double %q, double %r) {
+                               double %q, double %r) personality i8* bitcast (i32 (...)* @__gxx_personality_v0 to i8*) {
 entry:
   invoke void @_Z5printiiiii(i32 %a, i32 %b, i32 %c, i32 %d, i32 %e)
           to label %try.cont unwind label %lpad
 
 lpad:
   %0 = landingpad { i8*, i32 }
-          personality i8* bitcast (i32 (...)* @__gxx_personality_v0 to i8*)
           catch i8* null
   %1 = extractvalue { i8*, i32 } %0, 0
   %2 = tail call i8* @__cxa_begin_catch(i8* %1)
@@ -113,7 +112,6 @@ try.cont:
 
 lpad1:
   %3 = landingpad { i8*, i32 }
-          personality i8* bitcast (i32 (...)* @__gxx_personality_v0 to i8*)
           cleanup
   invoke void @__cxa_end_catch()
           to label %eh.resume unwind label %terminate.lpad
@@ -123,7 +121,6 @@ eh.resume:
 
 terminate.lpad:
   %4 = landingpad { i8*, i32 }
-          personality i8* bitcast (i32 (...)* @__gxx_personality_v0 to i8*)
           catch i8* null
   %5 = extractvalue { i8*, i32 } %4, 0
   tail call void @__clang_call_terminate(i8* %5)
@@ -146,8 +143,8 @@ declare void @_ZSt9terminatev()
 ; CHECK-FP:   push   {r4, r5, r6, r7, r8, r9, r10, r11, lr}
 ; CHECK-FP:   .setfp r11, sp, #28
 ; CHECK-FP:   add    r11, sp, #28
-; CHECK-FP:   .pad   #28
-; CHECK-FP:   sub    sp, sp, #28
+; CHECK-FP:   .pad   #44
+; CHECK-FP:   sub    sp, sp, #44
 ; CHECK-FP:   .personality __gxx_personality_v0
 ; CHECK-FP:   .handlerdata
 ; CHECK-FP:   .fnend
@@ -156,8 +153,8 @@ declare void @_ZSt9terminatev()
 ; CHECK-FP-ELIM:   .fnstart
 ; CHECK-FP-ELIM:   .save {r4, r5, r6, r7, r8, r9, r10, r11, lr}
 ; CHECK-FP-ELIM:   push  {r4, r5, r6, r7, r8, r9, r10, r11, lr}
-; CHECK-FP-ELIM:   .pad  #28
-; CHECK-FP-ELIM:   sub   sp, sp, #28
+; CHECK-FP-ELIM:   .pad  #36
+; CHECK-FP-ELIM:   sub   sp, sp, #36
 ; CHECK-FP-ELIM:   .personality __gxx_personality_v0
 ; CHECK-FP-ELIM:   .handlerdata
 ; CHECK-FP-ELIM:   .fnend
@@ -205,7 +202,7 @@ declare void @_ZSt9terminatev()
 ; DWARF-FP:    .cfi_offset r4, -36
 ; DWARF-FP:    add r11, sp, #28
 ; DWARF-FP:    .cfi_def_cfa r11, 8
-; DWARF-FP:    sub sp, sp, #28
+; DWARF-FP:    sub sp, sp, #44
 ; DWARF-FP:    sub sp, r11, #28
 ; DWARF-FP:    pop {r4, r5, r6, r7, r8, r9, r10, r11, lr}
 ; DWARF-FP:    mov pc, lr
@@ -226,9 +223,9 @@ declare void @_ZSt9terminatev()
 ; DWARF-FP-ELIM:    .cfi_offset r6, -28
 ; DWARF-FP-ELIM:    .cfi_offset r5, -32
 ; DWARF-FP-ELIM:    .cfi_offset r4, -36
-; DWARF-FP-ELIM:    sub sp, sp, #28
-; DWARF-FP-ELIM:    .cfi_def_cfa_offset 64
-; DWARF-FP-ELIM:    add sp, sp, #28
+; DWARF-FP-ELIM:    sub sp, sp, #36
+; DWARF-FP-ELIM:    .cfi_def_cfa_offset 72
+; DWARF-FP-ELIM:    add sp, sp, #36
 ; DWARF-FP-ELIM:    pop {r4, r5, r6, r7, r8, r9, r10, r11, lr}
 ; DWARF-FP-ELIM:    mov pc, lr
 ; DWARF-FP-ELIM:    .cfi_endproc
diff --git a/test/CodeGen/ARM/emutls1.ll b/test/CodeGen/ARM/emutls1.ll
new file mode 100644
index 00000000000..d4ba7eced66
--- /dev/null
+++ b/test/CodeGen/ARM/emutls1.ll
@@ -0,0 +1,31 @@
+; RUN: llc < %s -emulated-tls -march=arm -mtriple=arm-linux-androideabi \
+; RUN:     | FileCheck %s
+; RUN: llc < %s -emulated-tls -march=arm -mtriple=arm-linux-androideabi \
+; RUN:     -relocation-model=pic | FileCheck %s --check-prefix=PIC
+
+; Compared with tls1.ll, emulated mode should not use __aeabi_read_tp or __tls_get_addr.
+
+; CHECK-NOT: _aeabi_read_tp
+; CHECK-NOT: _tls_get_addr
+; CHECK:     __emutls_get_addr
+; CHECK-NOT: __aeabi_read_tp
+; CHECK-NOT: _tls_get_addr
+
+; PIC-NOT: _aeabi_read_tp
+; PIC-NOT: _tls_get_addr
+; PIC:     __emutls_get_addr
+; PIC-NOT: _aeabi_read_tp
+; PIC-NOT: _tls_get_addr
+
+@i = thread_local global i32 15 ; <i32*> [#uses=2]
+
+define i32 @f() {
+entry:
+ %tmp1 = load i32, i32* @i ; <i32> [#uses=1]
+ ret i32 %tmp1
+}
+
+define i32* @g() {
+entry:
+ ret i32* @i
+}
diff --git a/test/CodeGen/ARM/emutls_generic.ll b/test/CodeGen/ARM/emutls_generic.ll
new file mode 100644
index 00000000000..864daae568f
--- /dev/null
+++ b/test/CodeGen/ARM/emutls_generic.ll
@@ -0,0 +1,61 @@
+; RUN: llc < %s -emulated-tls -mtriple=arm-linux-android -relocation-model=pic \
+; RUN:     | FileCheck -check-prefix=ARM_32 %s
+; RUN: llc < %s -emulated-tls -mtriple=arm-linux-androidabi -relocation-model=pic \
+; RUN:     | FileCheck -check-prefix=ARM_32 %s
+; RUN: llc < %s -emulated-tls -mtriple=arm-linux-androidabi -relocation-model=pic -O3 \
+; RUN:     | FileCheck -check-prefix=ARM_32 %s
+; RUN: llc < %s -emulated-tls -mtriple=arm-linux-androidabi -O3 \
+; RUN:     | FileCheck -check-prefix=ARM_32 %s
+
+; Make sure that TLS symbols are emitted in expected order.
+
+@external_x = external thread_local global i32, align 8
+@external_y = thread_local global i8 7, align 2
+@internal_y = internal thread_local global i64 9, align 16
+
+define i32* @get_external_x() {
+entry:
+  ret i32* @external_x
+}
+
+define i8* @get_external_y() {
+entry:
+  ret i8* @external_y
+}
+
+define i64* @get_internal_y() {
+entry:
+  ret i64* @internal_y
+}
+
+; ARM_32-LABEL:  get_external_x:
+; ARM_32:        bl __emutls_get_address
+; ARM_32:        .long __emutls_v.external_x
+; ARM_32-LABEL:  get_external_y:
+; ARM_32:        bl __emutls_get_address
+; ARM_32:        .long __emutls_v.external_y
+; ARM_32-LABEL:  get_internal_y:
+; ARM_32:      bl __emutls_get_address
+; ARM_32:      .long __emutls_v.internal_y
+; ARM_32-NOT:   __emutls_t.external_x
+; ARM_32-NOT:   __emutls_v.external_x:
+; ARM_32:        .section .data.rel.local
+; ARM_32:        .align 2
+; ARM_32-LABEL:  __emutls_v.external_y:
+; ARM_32-NEXT:   .long 1
+; ARM_32-NEXT:   .long 2
+; ARM_32-NEXT:   .long 0
+; ARM_32-NEXT:   .long __emutls_t.external_y
+; ARM_32:        .section .rodata,
+; ARM_32-LABEL:  __emutls_t.external_y:
+; ARM_32-NEXT:   .byte 7
+; ARM_32:        .section .data.rel.local
+; ARM_32:        .align 2
+; ARM_32-LABEL:  __emutls_v.internal_y:
+; ARM_32-NEXT:   .long 8
+; ARM_32-NEXT:   .long 16
+; ARM_32-NEXT:   .long 0
+; ARM_32-NEXT:   .long __emutls_t.internal_y
+; ARM_32-LABEL:  __emutls_t.internal_y:
+; ARM_32-NEXT:   .long 9
+; ARM_32-NEXT:   .long 0
diff --git a/test/CodeGen/ARM/fast-isel-align.ll b/test/CodeGen/ARM/fast-isel-align.ll
index 124a78dada5..701884e926a 100644
--- a/test/CodeGen/ARM/fast-isel-align.ll
+++ b/test/CodeGen/ARM/fast-isel-align.ll
@@ -1,23 +1,22 @@
 ; RUN: llc < %s -O0 -fast-isel-abort=1 -relocation-model=dynamic-no-pic -mtriple=armv7-apple-ios -verify-machineinstrs | FileCheck %s --check-prefix=ARM
 ; RUN: llc < %s -O0 -fast-isel-abort=1 -relocation-model=dynamic-no-pic -mtriple=thumbv7-apple-ios -verify-machineinstrs | FileCheck %s --check-prefix=THUMB
-; RUN: llc < %s -O0 -arm-strict-align -relocation-model=dynamic-no-pic -mtriple=armv7-apple-ios -verify-machineinstrs | FileCheck %s --check-prefix=ARM-STRICT-ALIGN
-; RUN: llc < %s -O0 -arm-strict-align -relocation-model=dynamic-no-pic -mtriple=thumbv7-apple-ios -verify-machineinstrs | FileCheck %s --check-prefix=THUMB-STRICT-ALIGN
+; RUN: llc < %s -O0 -mattr=+strict-align -relocation-model=dynamic-no-pic -mtriple=armv7-apple-ios -verify-machineinstrs | FileCheck %s --check-prefix=ARM-STRICT-ALIGN
+; RUN: llc < %s -O0 -mattr=+strict-align -relocation-model=dynamic-no-pic -mtriple=thumbv7-apple-ios -verify-machineinstrs | FileCheck %s --check-prefix=THUMB-STRICT-ALIGN
 
 ; RUN: llc < %s -O0 -fast-isel-abort=1 -relocation-model=dynamic-no-pic -mtriple=armv7-linux-gnueabi -verify-machineinstrs | FileCheck %s --check-prefix=ARM
 ; RUN: llc < %s -O0 -fast-isel-abort=1 -relocation-model=dynamic-no-pic -mtriple=thumbv7-linux-gnueabi -verify-machineinstrs | FileCheck %s --check-prefix=THUMB
-; RUN: llc < %s -O0 -arm-strict-align -relocation-model=dynamic-no-pic -mtriple=armv7-linux-gnueabi -verify-machineinstrs | FileCheck %s --check-prefix=ARM-STRICT-ALIGN
-; RUN: llc < %s -O0 -arm-strict-align -relocation-model=dynamic-no-pic -mtriple=thumbv7-linux-gnueabi -verify-machineinstrs | FileCheck %s --check-prefix=THUMB-STRICT-ALIGN
-
-; LOCALMOD: use -relocation-model=static for NaCl to keep fast-isel simple. MERGETODO(dschuff): remove this localmod?
-; RUN: llc < %s -O0 -fast-isel-abort=1 -relocation-model=static -mtriple=armv7-unknown-nacl -verify-machineinstrs | FileCheck %s --check-prefix=ARM
-; RUN: llc < %s -O0 -arm-strict-align -relocation-model=static -mtriple=armv7-unknown-nacl -verify-machineinstrs | FileCheck %s --check-prefix=ARM-STRICT-ALIGN
-
-; RUN: llc < %s -O0  -fast-isel-abort=1 -relocation-model=dynamic-no-pic -mtriple=armv7-unknown-unknown -verify-machineinstrs | FileCheck %s --check-prefix=ARM-STRICT-ALIGN
-; RUN: llc < %s -O0  -fast-isel-abort=1 -relocation-model=dynamic-no-pic -mtriple=thumbv7-unknown-unknown -verify-machineinstrs | FileCheck %s --check-prefix=THUMB-STRICT-ALIGN
-; RUN: llc < %s -O0 -arm-no-strict-align -fast-isel-abort=1 -relocation-model=dynamic-no-pic -mtriple=armv7-unknown-unknown -verify-machineinstrs | FileCheck %s --check-prefix=ARM
-; RUN: llc < %s -O0 -arm-no-strict-align -fast-isel-abort=1 -relocation-model=dynamic-no-pic -mtriple=thumbv7-unknown-unknown -verify-machineinstrs | FileCheck %s --check-prefix=THUMB
-; RUN: llc < %s -O0 -relocation-model=dynamic-no-pic -mtriple=armv7-unknown-unknown -verify-machineinstrs | FileCheck %s --check-prefix=ARM-STRICT-ALIGN
-; RUN: llc < %s -O0 -relocation-model=dynamic-no-pic -mtriple=thumbv7-unknown-unknown -verify-machineinstrs | FileCheck %s --check-prefix=THUMB-STRICT-ALIGN
+; RUN: llc < %s -O0 -mattr=+strict-align -relocation-model=dynamic-no-pic -mtriple=armv7-linux-gnueabi -verify-machineinstrs | FileCheck %s --check-prefix=ARM-STRICT-ALIGN
+; RUN: llc < %s -O0 -mattr=+strict-align -relocation-model=dynamic-no-pic -mtriple=thumbv7-linux-gnueabi -verify-machineinstrs | FileCheck %s --check-prefix=THUMB-STRICT-ALIGN
+
+; RUN: llc < %s -O0 -fast-isel-abort=1 -relocation-model=dynamic-no-pic -mtriple=armv7-unknown-nacl -verify-machineinstrs | FileCheck %s --check-prefix=ARM
+; RUN: llc < %s -O0 -mattr=+strict-align -relocation-model=dynamic-no-pic -mtriple=armv7-unknown-nacl -verify-machineinstrs | FileCheck %s --check-prefix=ARM-STRICT-ALIGN
+
+; RUN: llc < %s -O0 -mattr=+strict-align -fast-isel-abort=1 -relocation-model=dynamic-no-pic -mtriple=armv7-unknown-unknown -verify-machineinstrs | FileCheck %s --check-prefix=ARM-STRICT-ALIGN
+; RUN: llc < %s -O0  -fast-isel-abort=1 -relocation-model=dynamic-no-pic -mtriple=thumbv7-unknown-unknown -mattr=+strict-align -verify-machineinstrs | FileCheck %s --check-prefix=THUMB-STRICT-ALIGN
+; RUN: llc < %s -O0 -fast-isel-abort=1 -relocation-model=dynamic-no-pic -mtriple=armv7-unknown-unknown -verify-machineinstrs | FileCheck %s --check-prefix=ARM
+; RUN: llc < %s -O0 -fast-isel-abort=1 -relocation-model=dynamic-no-pic -mtriple=thumbv7-unknown-unknown -verify-machineinstrs | FileCheck %s --check-prefix=THUMB
+; RUN: llc < %s -O0 -relocation-model=dynamic-no-pic -mtriple=armv7-unknown-unknown -mattr=+strict-align -verify-machineinstrs | FileCheck %s --check-prefix=ARM-STRICT-ALIGN
+; RUN: llc < %s -O0 -relocation-model=dynamic-no-pic -mtriple=thumbv7-unknown-unknown -mattr=+strict-align -verify-machineinstrs | FileCheck %s --check-prefix=THUMB-STRICT-ALIGN
 
 ; Check unaligned stores
 %struct.anon = type <{ float }>
diff --git a/test/CodeGen/ARM/fast-isel-call.ll b/test/CodeGen/ARM/fast-isel-call.ll
index bd170f30d97..e382e78a995 100644
--- a/test/CodeGen/ARM/fast-isel-call.ll
+++ b/test/CodeGen/ARM/fast-isel-call.ll
@@ -1,9 +1,9 @@
 ; RUN: llc < %s -O0 -verify-machineinstrs -fast-isel-abort=1 -relocation-model=dynamic-no-pic -mtriple=armv7-apple-ios | FileCheck %s --check-prefix=ARM
 ; RUN: llc < %s -O0 -verify-machineinstrs -fast-isel-abort=1 -relocation-model=dynamic-no-pic -mtriple=armv7-linux-gnueabi | FileCheck %s --check-prefix=ARM
 ; RUN: llc < %s -O0 -verify-machineinstrs -fast-isel-abort=1 -relocation-model=dynamic-no-pic -mtriple=thumbv7-apple-ios | FileCheck %s --check-prefix=THUMB
-; RUN: llc < %s -O0 -verify-machineinstrs -fast-isel-abort=1 -relocation-model=dynamic-no-pic -mtriple=armv7-apple-ios -arm-long-calls | FileCheck %s --check-prefix=ARM-LONG
-; RUN: llc < %s -O0 -verify-machineinstrs -fast-isel-abort=1 -relocation-model=dynamic-no-pic -mtriple=armv7-linux-gnueabi -arm-long-calls | FileCheck %s --check-prefix=ARM-LONG
-; RUN: llc < %s -O0 -verify-machineinstrs -fast-isel-abort=1 -relocation-model=dynamic-no-pic -mtriple=thumbv7-apple-ios -arm-long-calls | FileCheck %s --check-prefix=THUMB-LONG
+; RUN: llc < %s -O0 -verify-machineinstrs -fast-isel-abort=1 -relocation-model=dynamic-no-pic -mtriple=armv7-apple-ios -mattr=+long-calls | FileCheck %s --check-prefix=ARM-LONG
+; RUN: llc < %s -O0 -verify-machineinstrs -fast-isel-abort=1 -relocation-model=dynamic-no-pic -mtriple=armv7-linux-gnueabi -mattr=+long-calls | FileCheck %s --check-prefix=ARM-LONG
+; RUN: llc < %s -O0 -verify-machineinstrs -fast-isel-abort=1 -relocation-model=dynamic-no-pic -mtriple=thumbv7-apple-ios -mattr=+long-calls | FileCheck %s --check-prefix=THUMB-LONG
 ; RUN: llc < %s -O0 -verify-machineinstrs -fast-isel-abort=1 -relocation-model=dynamic-no-pic -mtriple=armv7-apple-ios -mattr=-vfp2 | FileCheck %s --check-prefix=ARM-NOVFP
 ; RUN: llc < %s -O0 -verify-machineinstrs -fast-isel-abort=1 -relocation-model=dynamic-no-pic -mtriple=armv7-linux-gnueabi -mattr=-vfp2 | FileCheck %s --check-prefix=ARM-NOVFP
 ; RUN: llc < %s -O0 -verify-machineinstrs -fast-isel-abort=1 -relocation-model=dynamic-no-pic -mtriple=thumbv7-apple-ios -mattr=-vfp2 | FileCheck %s --check-prefix=THUMB-NOVFP
diff --git a/test/CodeGen/ARM/fast-isel-intrinsic.ll b/test/CodeGen/ARM/fast-isel-intrinsic.ll
index 6b434b74ca7..1c7ff687938 100644
--- a/test/CodeGen/ARM/fast-isel-intrinsic.ll
+++ b/test/CodeGen/ARM/fast-isel-intrinsic.ll
@@ -1,9 +1,9 @@
 ; RUN: llc < %s -O0 -fast-isel-abort=1 -relocation-model=dynamic-no-pic -mtriple=armv7-apple-ios -verify-machineinstrs | FileCheck %s --check-prefix=ARM
 ; RUN: llc < %s -O0 -fast-isel-abort=1 -relocation-model=dynamic-no-pic -mtriple=armv7-linux-gnueabi -verify-machineinstrs | FileCheck %s --check-prefix=ARM
 ; RUN: llc < %s -O0 -fast-isel-abort=1 -relocation-model=dynamic-no-pic -mtriple=thumbv7-apple-ios -verify-machineinstrs | FileCheck %s --check-prefix=THUMB
-; RUN: llc < %s -O0 -fast-isel-abort=1 -relocation-model=dynamic-no-pic -mtriple=armv7-apple-ios -arm-long-calls -verify-machineinstrs | FileCheck %s --check-prefix=ARM-LONG
-; RUN: llc < %s -O0 -fast-isel-abort=1 -relocation-model=dynamic-no-pic -mtriple=armv7-linux-gnueabi -arm-long-calls -verify-machineinstrs | FileCheck %s --check-prefix=ARM-LONG
-; RUN: llc < %s -O0 -fast-isel-abort=1 -relocation-model=dynamic-no-pic -mtriple=thumbv7-apple-ios -arm-long-calls -verify-machineinstrs | FileCheck %s --check-prefix=THUMB-LONG
+; RUN: llc < %s -O0 -fast-isel-abort=1 -relocation-model=dynamic-no-pic -mtriple=armv7-apple-ios -mattr=+long-calls -verify-machineinstrs | FileCheck %s --check-prefix=ARM-LONG
+; RUN: llc < %s -O0 -fast-isel-abort=1 -relocation-model=dynamic-no-pic -mtriple=armv7-linux-gnueabi -mattr=+long-calls -verify-machineinstrs | FileCheck %s --check-prefix=ARM-LONG
+; RUN: llc < %s -O0 -fast-isel-abort=1 -relocation-model=dynamic-no-pic -mtriple=thumbv7-apple-ios -mattr=+long-calls -verify-machineinstrs | FileCheck %s --check-prefix=THUMB-LONG
 
 ; Note that some of these tests assume that relocations are either
 ; movw/movt or constant pool loads. Different platforms will select
diff --git a/test/CodeGen/ARM/fast-isel-mvn.ll b/test/CodeGen/ARM/fast-isel-mvn.ll
index b744dd05f28..34bb7225854 100644
--- a/test/CodeGen/ARM/fast-isel-mvn.ll
+++ b/test/CodeGen/ARM/fast-isel-mvn.ll
@@ -1,16 +1,13 @@
-; RUN: llc -O0 -verify-machineinstrs -fast-isel-abort=1 -relocation-model=dynamic-no-pic -arm-use-movt=false -mtriple=armv7-apple-ios     < %s | FileCheck %s --check-prefix=CHECK --check-prefix=ARM
-; RUN: llc -O0 -verify-machineinstrs -fast-isel-abort=1 -relocation-model=dynamic-no-pic -arm-use-movt=false -mtriple=armv7-linux-gnueabi < %s | FileCheck %s --check-prefix=CHECK --check-prefix=ARM
-; RUN: llc -O0 -verify-machineinstrs -fast-isel-abort=1 -relocation-model=dynamic-no-pic -arm-use-movt=false -mtriple=thumbv7-apple-ios   < %s | FileCheck %s --check-prefix=CHECK --check-prefix=ARM
-; RUN: llc -O0 -verify-machineinstrs -fast-isel-abort=1 -relocation-model=dynamic-no-pic -arm-use-movt=true  -mtriple=thumbv7-apple-ios   < %s | FileCheck %s --check-prefix=CHECK --check-prefix=THUMB
-; RUN: llc -O0 -verify-machineinstrs -fast-isel-abort=1 -relocation-model=dynamic-no-pic -arm-use-movt=true  -mtriple=armv7-apple-ios     < %s | FileCheck %s --check-prefix=MOVT
-; @LOCALMOD: check that we use movt in t10
-; RUN: llc -O0 -verify-machineinstrs -fast-isel-abort=1 -relocation-model=dynamic-no-pic -arm-use-movt=true  -mtriple=armv7-unknown-nacl-gnueabi     < %s | FileCheck %s --check-prefix=CHECK --check-prefix=NACL
+; RUN: llc -O0 -verify-machineinstrs -fast-isel-abort=1 -relocation-model=dynamic-no-pic -mattr=+no-movt -mtriple=armv7-apple-ios     < %s | FileCheck %s --check-prefix=CHECK --check-prefix=ARM
+; RUN: llc -O0 -verify-machineinstrs -fast-isel-abort=1 -relocation-model=dynamic-no-pic -mattr=+no-movt -mtriple=armv7-linux-gnueabi < %s | FileCheck %s --check-prefix=CHECK --check-prefix=ARM
+; RUN: llc -O0 -verify-machineinstrs -fast-isel-abort=1 -relocation-model=dynamic-no-pic -mattr=+no-movt -mtriple=thumbv7-apple-ios   < %s | FileCheck %s --check-prefix=CHECK --check-prefix=ARM
+; RUN: llc -O0 -verify-machineinstrs -fast-isel-abort=1 -relocation-model=dynamic-no-pic -mtriple=thumbv7-apple-ios   < %s | FileCheck %s --check-prefix=CHECK --check-prefix=THUMB
+; RUN: llc -O0 -verify-machineinstrs -fast-isel-abort=1 -relocation-model=dynamic-no-pic -mtriple=armv7-apple-ios     < %s | FileCheck %s --check-prefix=MOVT
 ; rdar://10412592
 
 define void @t1() nounwind {
 entry:
-; @LOCALMOD
-; CHECK-LABEL: t1:
+; CHECK-LABEL: t1
 ; CHECK:       mvn r0, #0
   call void @foo(i32 -1)
   ret void
@@ -90,9 +87,6 @@ entry:
 define i32 @t10(i32 %a) {
 ; MOVT-LABEL: t10
 ; MOVT:       ldr
-; NACL-LABEL:  t10
-; NACL:        movw [[REG:r.*]],  #52257
-; NACL:        movt [[REG]],  #35037
   %1 = xor i32 -1998730207, %a
   ret i32 %1
 }
diff --git a/test/CodeGen/ARM/fast-isel-remat-same-constant.ll b/test/CodeGen/ARM/fast-isel-remat-same-constant.ll
new file mode 100644
index 00000000000..d64cf8022ed
--- /dev/null
+++ b/test/CodeGen/ARM/fast-isel-remat-same-constant.ll
@@ -0,0 +1,29 @@
+; RUN: llc %s -o - -fast-isel=true -O0 -verify-machineinstrs | FileCheck %s
+
+target datalayout = "e-m:o-p:32:32-f64:32:64-v64:32:64-v128:32:128-a:0:32-n32-S32"
+target triple = "thumbv7-apple-ios8.0.0"
+
+; This test failed with verify machine instrs due to incorrect kill flags on the add instructions
+; generated by the GEPs.  The first add generated killed the vreg for the #6680 constant which should
+; be correct.  However, the second add is also a constant expression and the local value save area grows
+; down.  This meant the next use of the vreg for #6680 was after the first which had killed it.
+
+; CHECK: #6680
+
+%struct.RD_8x8DATA = type { i32, [16 x [16 x i32]], [16 x [16 x i32]], [16 x [16 x i32]], [3 x [16 x [16 x i32]]], [4 x i16], [4 x i8], [4 x i8], [4 x i8], [16 x [16 x i16]], [16 x [16 x i16]], [16 x [16 x i32]] }
+
+@tr8x8 = external global %struct.RD_8x8DATA, align 4
+@tr4x4 = external global %struct.RD_8x8DATA, align 4
+
+; Function Attrs: noreturn
+declare void @foo(i16*, i16*) #0
+
+; Function Attrs: minsize
+define i32 @test() #1 {
+bb:
+  call void @foo(i16* getelementptr inbounds (%struct.RD_8x8DATA, %struct.RD_8x8DATA* @tr8x8, i32 0, i32 10, i32 0, i32 0), i16* getelementptr inbounds (%struct.RD_8x8DATA, %struct.RD_8x8DATA* @tr4x4, i32 0, i32 10, i32 0, i32 0))
+  unreachable
+}
+
+attributes #0 = { noreturn }
+attributes #1 = { minsize }
diff --git a/test/CodeGen/ARM/fast-isel-shift-materialize.ll b/test/CodeGen/ARM/fast-isel-shift-materialize.ll
new file mode 100644
index 00000000000..ab36a979d71
--- /dev/null
+++ b/test/CodeGen/ARM/fast-isel-shift-materialize.ll
@@ -0,0 +1,25 @@
+; RUN: llc %s -o - -verify-machineinstrs -fast-isel=true | FileCheck %s
+
+target datalayout = "e-m:o-p:32:32-f64:32:64-v64:32:64-v128:32:128-a:0:32-n32-S32"
+target triple = "thumbv7-apple-ios8.0.0"
+
+; When materializing the '2' for the shifts below, the second shift kills the vreg
+; we materialize in to.  However, the first shift was also killing that vreg.
+; This fails the machine verifier because the second use then thinks its reading an
+; undefined register.
+
+; CHECK: lsr.w
+; CHECK: lsr.w
+
+declare void @foo(i32, i32)
+
+; Function Attrs: minsize
+define i32 @test(i32 %tmp3, i32 %tmp9) #0 {
+bb:
+  %tmp4 = lshr i32 %tmp3, 2
+  %tmp10 = lshr i32 %tmp9, 2
+  call void @foo(i32 %tmp10, i32 %tmp4)
+  unreachable
+}
+
+attributes #0 = { minsize }
diff --git a/test/CodeGen/ARM/fast-isel-static.ll b/test/CodeGen/ARM/fast-isel-static.ll
index c3980cb51f6..200387cf892 100644
--- a/test/CodeGen/ARM/fast-isel-static.ll
+++ b/test/CodeGen/ARM/fast-isel-static.ll
@@ -1,5 +1,5 @@
-; RUN: llc < %s -mtriple=thumbv7-apple-ios -O0 -verify-machineinstrs -fast-isel-abort=1 -relocation-model=static -arm-long-calls | FileCheck -check-prefix=CHECK-LONG %s
-; RUN: llc < %s -mtriple=armv7-linux-gnueabi -O0 -verify-machineinstrs -fast-isel-abort=1 -relocation-model=static -arm-long-calls | FileCheck -check-prefix=CHECK-LONG %s
+; RUN: llc < %s -mtriple=thumbv7-apple-ios -O0 -verify-machineinstrs -fast-isel-abort=1 -relocation-model=static -mattr=+long-calls | FileCheck -check-prefix=CHECK-LONG %s
+; RUN: llc < %s -mtriple=armv7-linux-gnueabi -O0 -verify-machineinstrs -fast-isel-abort=1 -relocation-model=static -mattr=+long-calls | FileCheck -check-prefix=CHECK-LONG %s
 ; RUN: llc < %s -mtriple=thumbv7-apple-ios -O0 -verify-machineinstrs -fast-isel-abort=1 -relocation-model=static | FileCheck -check-prefix=CHECK-NORM %s
 ; RUN: llc < %s -mtriple=armv7-linux-gnueabi -O0 -verify-machineinstrs -fast-isel-abort=1 -relocation-model=static | FileCheck -check-prefix=CHECK-NORM %s
 
diff --git a/test/CodeGen/ARM/fast-isel-update-valuemap-for-extract.ll b/test/CodeGen/ARM/fast-isel-update-valuemap-for-extract.ll
new file mode 100644
index 00000000000..18d60fee189
--- /dev/null
+++ b/test/CodeGen/ARM/fast-isel-update-valuemap-for-extract.ll
@@ -0,0 +1,24 @@
+; RUN: llc %s -o - -fast-isel=true -O0 -verify-machineinstrs | FileCheck %s
+
+target datalayout = "e-m:o-p:32:32-f64:32:64-v64:32:64-v128:32:128-a:0:32-n32-S32"
+target triple = "thumbv7-apple-ios8.0.0"
+
+; This test ensures that when fast-isel rewrites uses of the vreg for %tmp29, it also
+; updates kill flags on the shift instruction generated as part of the gep.
+; This was failing instruction verification.
+
+; CHECK-LABEL: @test
+
+%struct.node = type opaque
+
+declare void @foo([4 x i32], %struct.node*)
+
+define void @test([4 x i32] %xpic.coerce, %struct.node* %t) {
+bb:
+  %tmp29 = extractvalue [4 x i32] %xpic.coerce, 0
+  %tmp40 = bitcast %struct.node* %t to [8 x %struct.node*]*
+  %tmp41 = getelementptr inbounds [8 x %struct.node*], [8 x %struct.node*]* %tmp40, i32 0, i32 %tmp29
+  %tmp42 = load %struct.node*, %struct.node** %tmp41, align 4
+  call void @foo([4 x i32] %xpic.coerce, %struct.node* %tmp42)
+  ret void
+}
diff --git a/test/CodeGen/ARM/fast-isel-vaddd.ll b/test/CodeGen/ARM/fast-isel-vaddd.ll
new file mode 100644
index 00000000000..2aa269a9774
--- /dev/null
+++ b/test/CodeGen/ARM/fast-isel-vaddd.ll
@@ -0,0 +1,33 @@
+; RUN: llc %s -o - -verify-machineinstrs -fast-isel=true -mattr=+vfp4 -mattr=+neon | FileCheck %s
+
+target datalayout = "e-m:o-p:32:32-f64:32:64-v64:32:64-v128:32:128-a:0:32-n32-S32"
+target triple = "thumbv7s-apple-ios8.0.0"
+
+%union.DV = type { <2 x double> }
+
+; Fast-ISel was incorrectly trying to codegen <2 x double> adds and returning only a single vadds
+; Check that we generate the 2 vaddd's we expect
+
+; CHECK: vadd.f64
+; CHECK: vadd.f64
+
+define i32 @main(i32 %argc, i8** nocapture readnone %Argv, <2 x double> %tmp31) {
+bb:
+  %Ad = alloca %union.DV, align 16
+  %tmp32 = getelementptr inbounds %union.DV, %union.DV* %Ad, i32 0, i32 0
+  %tmp33 = fadd <2 x double> %tmp31, %tmp31
+  br label %bb37
+
+bb37:                                             ; preds = %bb37, %bb
+  %i.02 = phi i32 [ 0, %bb ], [ %tmp38, %bb37 ]
+  store <2 x double> %tmp33, <2 x double>* %tmp32, align 16
+  %tmp38 = add nuw nsw i32 %i.02, 1
+  %exitcond = icmp eq i32 %tmp38, 500000
+  br i1 %exitcond, label %bb39, label %bb37
+
+bb39:                                             ; preds = %bb37
+  call fastcc void @printDV(%union.DV* %Ad)
+  ret i32 0
+}
+
+declare hidden fastcc void @printDV(%union.DV* nocapture readonly)
diff --git a/test/CodeGen/ARM/fnattr-trap.ll b/test/CodeGen/ARM/fnattr-trap.ll
new file mode 100644
index 00000000000..492e31b4b9d
--- /dev/null
+++ b/test/CodeGen/ARM/fnattr-trap.ll
@@ -0,0 +1,40 @@
+; RUN: llc < %s -mtriple=arm-unknown-unknown | FileCheck %s -check-prefix=NOOPTION
+; RUN: llc < %s -mtriple=arm-unknown-unknown -trap-func=trap_llc | FileCheck %s -check-prefix=TRAP
+
+; NOOPTION-LABEL: {{\_?}}foo0:
+; NOOPTION: trap{{$}}
+
+; TRAP-LABEL: {{\_?}}foo0:
+; TRAP: bl {{\_?}}trap_llc
+
+define void @foo0() {
+  call void @llvm.trap()
+  unreachable
+}
+
+; NOOPTION-LABEL: {{\_?}}foo1:
+; NOOPTION: bl {{\_?}}trap_func_attr0
+
+; TRAP-LABEL: {{\_?}}foo1:
+; TRAP: bl {{\_?}}trap_llc
+
+define void @foo1() {
+  call void @llvm.trap() #0
+  unreachable
+}
+
+; NOOPTION-LABEL: {{\_?}}foo2:
+; NOOPTION: bl {{\_?}}trap_func_attr1
+
+; TRAP-LABEL: {{\_?}}foo2:
+; TRAP: bl {{\_?}}trap_llc
+
+define void @foo2() {
+  call void @llvm.trap() #1
+  unreachable
+}
+
+declare void @llvm.trap() nounwind
+
+attributes #0 = { "trap-func-name"="trap_func_attr0" }
+attributes #1 = { "trap-func-name"="trap_func_attr1" }
diff --git a/test/CodeGen/ARM/fold-stack-adjust.ll b/test/CodeGen/ARM/fold-stack-adjust.ll
index aff79a1c113..b62d0dfee07 100644
--- a/test/CodeGen/ARM/fold-stack-adjust.ll
+++ b/test/CodeGen/ARM/fold-stack-adjust.ll
@@ -12,11 +12,11 @@ declare void @bar(i8*)
 
 define void @check_simple() minsize {
 ; CHECK-LABEL: check_simple:
-; CHECK: push.w {r7, r8, r9, r10, r11, lr}
+; CHECK: push {r3, r4, r5, r6, r7, lr}
 ; CHECK-NOT: sub sp, sp,
 ; ...
 ; CHECK-NOT: add sp, sp,
-; CHECK: pop.w {r0, r1, r2, r3, r11, pc}
+; CHECK: pop {r0, r1, r2, r3, r7, pc}
 
 ; CHECK-T1-LABEL: check_simple:
 ; CHECK-T1: push {r3, r4, r5, r6, r7, lr}
@@ -44,11 +44,11 @@ define void @check_simple() minsize {
 
 define void @check_simple_too_big() minsize {
 ; CHECK-LABEL: check_simple_too_big:
-; CHECK: push.w {r11, lr}
+; CHECK: push {r7, lr}
 ; CHECK: sub sp,
 ; ...
 ; CHECK: add sp,
-; CHECK: pop.w {r11, pc}
+; CHECK: pop {r7, pc}
   %var = alloca i8, i32 64
   call void @bar(i8* %var)
   ret void
@@ -93,11 +93,11 @@ define void @check_vfp_fold() minsize {
 ; folded in except that doing so would clobber the value being returned.
 define i64 @check_no_return_clobber() minsize {
 ; CHECK-LABEL: check_no_return_clobber:
-; CHECK: push.w {r5, r6, r7, r8, r9, r10, r11, lr}
+; CHECK: push {r1, r2, r3, r4, r5, r6, r7, lr}
 ; CHECK-NOT: sub sp,
 ; ...
 ; CHECK: add sp, #24
-; CHECK: pop.w {r11, pc}
+; CHECK: pop {r7, pc}
 
   ; Just to keep iOS FileCheck within previous function:
 ; CHECK-IOS-LABEL: check_no_return_clobber:
@@ -170,15 +170,15 @@ define void @test_varsize(...) minsize {
 ; CHECK-T1: push	{r5, r6, r7, lr}
 ; ...
 ; CHECK-T1: pop	{r2, r3, r7}
-; CHECK-T1: pop	{r3}
+; CHECK-T1: pop {[[POP_REG:r[0-3]]]}
 ; CHECK-T1: add	sp, #16
-; CHECK-T1: bx	r3
+; CHECK-T1: bx	[[POP_REG]]
 
 ; CHECK-LABEL: test_varsize:
 ; CHECK: sub	sp, #16
-; CHECK: push.w {r9, r10, r11, lr}
+; CHECK: push	{r5, r6, r7, lr}
 ; ...
-; CHECK: pop.w	{r2, r3, r11, lr}
+; CHECK: pop.w	{r2, r3, r7, lr}
 ; CHECK: add	sp, #16
 ; CHECK: bx	lr
 
diff --git a/test/CodeGen/ARM/fp16-promote.ll b/test/CodeGen/ARM/fp16-promote.ll
new file mode 100644
index 00000000000..e691c2bb8a9
--- /dev/null
+++ b/test/CodeGen/ARM/fp16-promote.ll
@@ -0,0 +1,903 @@
+; RUN: llc -asm-verbose=false < %s -mattr=+vfp3,+fp16 | FileCheck %s -check-prefix=CHECK-FP16 -check-prefix=CHECK-ALL
+; RUN: llc -asm-verbose=false < %s | FileCheck %s -check-prefix=CHECK-LIBCALL -check-prefix=CHECK-ALL
+
+target datalayout = "e-p:32:32:32-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-n32"
+target triple = "armv7-eabihf"
+
+; CHECK-FP16-LABEL: test_fadd:
+; CHECK-FP16: vcvtb.f32.f16
+; CHECK-FP16: vcvtb.f32.f16
+; CHECK-FP16: vadd.f32
+; CHECK-FP16: vcvtb.f16.f32
+; CHECK-LIBCALL-LABEL: test_fadd:
+; CHECK-LIBCALL: bl __gnu_h2f_ieee
+; CHECK-LIBCALL: bl __gnu_h2f_ieee
+; CHECK-LIBCALL: vadd.f32
+; CHECK-LIBCALL: bl __gnu_f2h_ieee
+define void @test_fadd(half* %p, half* %q) #0 {
+  %a = load half, half* %p, align 2
+  %b = load half, half* %q, align 2
+  %r = fadd half %a, %b
+  store half %r, half* %p
+  ret void
+}
+
+; CHECK-FP16-LABEL: test_fsub:
+; CHECK-FP16: vcvtb.f32.f16
+; CHECK-FP16: vcvtb.f32.f16
+; CHECK-FP16: vsub.f32
+; CHECK-FP16: vcvtb.f16.f32
+; CHECK-LIBCALL-LABEL: test_fsub:
+; CHECK-LIBCALL: bl __gnu_h2f_ieee
+; CHECK-LIBCALL: bl __gnu_h2f_ieee
+; CHECK-LIBCALL: vsub.f32
+; CHECK-LIBCALL: bl __gnu_f2h_ieee
+define void @test_fsub(half* %p, half* %q) #0 {
+  %a = load half, half* %p, align 2
+  %b = load half, half* %q, align 2
+  %r = fsub half %a, %b
+  store half %r, half* %p
+  ret void
+}
+
+; CHECK-FP16-LABEL: test_fmul:
+; CHECK-FP16: vcvtb.f32.f16
+; CHECK-FP16: vcvtb.f32.f16
+; CHECK-FP16: vmul.f32
+; CHECK-FP16: vcvtb.f16.f32
+; CHECK-LIBCALL-LABEL: test_fmul
+; CHECK-LIBCALL: bl __gnu_h2f_ieee
+; CHECK-LIBCALL: bl __gnu_h2f_ieee
+; CHECK-LIBCALL: vmul.f32
+; CHECK-LIBCALL: bl __gnu_f2h_ieee
+define void @test_fmul(half* %p, half* %q) #0 {
+  %a = load half, half* %p, align 2
+  %b = load half, half* %q, align 2
+  %r = fmul half %a, %b
+  store half %r, half* %p
+  ret void
+}
+
+; CHECK-FP16-LABEL: test_fdiv:
+; CHECK-FP16: vcvtb.f32.f16
+; CHECK-FP16: vcvtb.f32.f16
+; CHECK-FP16: vdiv.f32
+; CHECK-FP16: vcvtb.f16.f32
+; CHECK-LIBCALL-LABEL: test_fdiv
+; CHECK-LIBCALL: bl __gnu_h2f_ieee
+; CHECK-LIBCALL: bl __gnu_h2f_ieee
+; CHECK-LIBCALL: vdiv.f32
+; CHECK-LIBCALL: bl __gnu_f2h_ieee
+define void @test_fdiv(half* %p, half* %q) #0 {
+  %a = load half, half* %p, align 2
+  %b = load half, half* %q, align 2
+  %r = fdiv half %a, %b
+  store half %r, half* %p
+  ret void
+}
+
+; CHECK-FP16-LABEL: test_frem:
+; CHECK-FP16: vcvtb.f32.f16
+; CHECK-FP16: vcvtb.f32.f16
+; CHECK-FP16: bl fmodf
+; CHECK-FP16: vcvtb.f16.f32
+; CHECK-LIBCALL-LABEL: test_frem
+; CHECK-LIBCALL: bl __gnu_h2f_ieee
+; CHECK-LIBCALL: bl __gnu_h2f_ieee
+; CHECK-LIBCALL: bl fmodf
+; CHECK-LIBCALL: bl __gnu_f2h_ieee
+define void @test_frem(half* %p, half* %q) #0 {
+  %a = load half, half* %p, align 2
+  %b = load half, half* %q, align 2
+  %r = frem half %a, %b
+  store half %r, half* %p
+  ret void
+}
+
+; CHECK-ALL-LABEL: test_load_store:
+; CHECK-ALL-NEXT: .fnstart
+; CHECK-ALL-NEXT: ldrh r0, [r0]
+; CHECK-ALL-NEXT: strh r0, [r1]
+; CHECK-ALL-NEXT: bx lr
+define void @test_load_store(half* %p, half* %q) #0 {
+  %a = load half, half* %p, align 2
+  store half %a, half* %q
+  ret void
+}
+
+; Testing only successfull compilation of function calls.  In ARM ABI, half
+; args and returns are handled as f32.
+
+declare half @test_callee(half %a, half %b) #0
+
+; CHECK-ALL-LABEL: test_call:
+; CHECK-ALL-NEXT: .fnstart
+; CHECK-ALL-NEXT: push {r11, lr}
+; CHECK-ALL-NEXT: bl test_callee
+; CHECK-ALL-NEXT: pop {r11, pc}
+define half @test_call(half %a, half %b) #0 {
+  %r = call half @test_callee(half %a, half %b)
+  ret half %r
+}
+
+; CHECK-ALL-LABEL: test_call_flipped:
+; CHECK-ALL-NEXT: .fnstart
+; CHECK-ALL-NEXT: push {r11, lr}
+; CHECK-ALL-NEXT: mov r2, r0
+; CHECK-ALL-NEXT: mov r0, r1
+; CHECK-ALL-NEXT: mov r1, r2
+; CHECK-ALL-NEXT: bl test_callee
+; CHECK-ALL-NEXT: pop {r11, pc}
+define half @test_call_flipped(half %a, half %b) #0 {
+  %r = call half @test_callee(half %b, half %a)
+  ret half %r
+}
+
+; CHECK-ALL-LABEL: test_tailcall_flipped:
+; CHECK-ALL-NEXT: .fnstart
+; CHECK-ALL-NEXT: mov r2, r0
+; CHECK-ALL-NEXT: mov r0, r1
+; CHECK-ALL-NEXT: mov r1, r2
+; CHECK-ALL-NEXT: b test_callee
+define half @test_tailcall_flipped(half %a, half %b) #0 {
+  %r = tail call half @test_callee(half %b, half %a)
+  ret half %r
+}
+
+; Optimizer picks %p or %q based on %c and only loads that value
+; No conversion is needed
+; CHECK-ALL-LABEL: test_select:
+; CHECK-ALL-NEXT: .fnstart
+; CHECK-ALL-NEXT: cmp r2, #0
+; CHECK-ALL-NEXT: movne r1, r0
+; CHECK-ALL-NEXT: ldrh r1, [r1]
+; CHECK-ALL-NEXT: strh r1, [r0]
+; CHECK-ALL-NEXT: bx lr
+define void @test_select(half* %p, half* %q, i1 zeroext %c) #0 {
+  %a = load half, half* %p, align 2
+  %b = load half, half* %q, align 2
+  %r = select i1 %c, half %a, half %b
+  store half %r, half* %p
+  ret void
+}
+
+; Test only two variants of fcmp.  These get translated to f32 vcmpe
+; instructions anyway.
+; CHECK-FP16-LABEL: test_fcmp_une:
+; CHECK-FP16: vcvtb.f32.f16
+; CHECK-FP16: vcvtb.f32.f16
+; CHECK-FP16: vcmpe.f32
+; CHECK-FP16: vmrs APSR_nzcv, fpscr
+; CHECK-FP16: movwne
+; CHECK-LIBCALL-LABEL: test_fcmp_une:
+; CHECK-LIBCALL: bl __gnu_h2f_ieee
+; CHECK-LIBCALL: bl __gnu_h2f_ieee
+; CHECK-LIBCALL: vcmpe.f32
+; CHECK-LIBCALL: movwne
+define i1 @test_fcmp_une(half* %p, half* %q) #0 {
+  %a = load half, half* %p, align 2
+  %b = load half, half* %q, align 2
+  %r = fcmp une half %a, %b
+  ret i1 %r
+}
+
+; CHECK-FP16-LABEL: test_fcmp_ueq:
+; CHECK-FP16: vcvtb.f32.f16
+; CHECK-FP16: vcvtb.f32.f16
+; CHECK-FP16: vcmpe.f32
+; CHECK-FP16: vmrs APSR_nzcv, fpscr
+; CHECK-FP16: movweq
+; CHECK-FP16: movwvs
+; CHECK-LIBCALL-LABEL: test_fcmp_ueq:
+; CHECK-LIBCALL: bl __gnu_h2f_ieee
+; CHECK-LIBCALL: bl __gnu_h2f_ieee
+; CHECK-LIBCALL: vcmpe.f32
+; CHECK-LIBCALL: movweq
+define i1 @test_fcmp_ueq(half* %p, half* %q) #0 {
+  %a = load half, half* %p, align 2
+  %b = load half, half* %q, align 2
+  %r = fcmp ueq half %a, %b
+  ret i1 %r
+}
+
+; CHECK-FP16-LABEL: test_br_cc:
+; CHECK-FP16: vcvtb.f32.f16
+; CHECK-FP16: vcvtb.f32.f16
+; CHECK-FP16: vcmpe.f32
+; CHECK-FP16: vmrs APSR_nzcv, fpscr
+; CHECK-FP16: strmi
+; CHECK-FP16: strpl
+; CHECK-LIBCALL-LABEL: test_br_cc:
+; CHECK-LIBCALL: bl __gnu_h2f_ieee
+; CHECK-LIBCALL: bl __gnu_h2f_ieee
+; CHECK-LIBCALL: vcmpe.f32
+; CHECK-LIBCALL: strmi
+; CHECK-LIBCALL: strpl
+define void @test_br_cc(half* %p, half* %q, i32* %p1, i32* %p2) #0 {
+  %a = load half, half* %p, align 2
+  %b = load half, half* %q, align 2
+  %c = fcmp uge half %a, %b
+  br i1 %c, label %then, label %else
+then:
+  store i32 0, i32* %p1
+  ret void
+else:
+  store i32 0, i32* %p2
+  ret void
+}
+
+declare i1 @test_dummy(half* %p) #0
+; CHECK-FP16-LABEL: test_phi:
+; CHECK-FP16: vcvtb.f32.f16
+; CHECK-FP16: [[LOOP:.LBB[1-9_]+]]:
+; CHECK-FP16: vcvtb.f32.f16
+; CHECK-FP16: bl      test_dummy
+; CHECK-FP16: bne     [[LOOP]]
+; CHECK-FP16: vcvtb.f16.f32
+; CHECK-LIBCALL-LABEL: test_phi:
+; CHECK-LIBCALL: bl __gnu_h2f_ieee
+; CHECK-LIBCALL: [[LOOP:.LBB[1-9_]+]]:
+; CHECK-LIBCALL: bl __gnu_h2f_ieee
+; CHECK-LIBCALL: bl test_dummy
+; CHECK-LIBCALL: bne     [[LOOP]]
+; CHECK-LIBCALL: bl __gnu_f2h_ieee
+define void @test_phi(half* %p) #0 {
+entry:
+  %a = load half, half* %p
+  br label %loop
+loop:
+  %r = phi half [%a, %entry], [%b, %loop]
+  %b = load half, half* %p
+  %c = call i1 @test_dummy(half* %p)
+  br i1 %c, label %loop, label %return
+return:
+  store half %r, half* %p
+  ret void
+}
+
+; CHECK-FP16-LABEL: test_fptosi_i32:
+; CHECK-FP16: vcvtb.f32.f16
+; CHECK-FP16: vcvt.s32.f32
+; CHECK-LIBCALL-LABEL: test_fptosi_i32:
+; CHECK-LIBCALL: bl __gnu_h2f_ieee
+; CHECK-LIBCALL: vcvt.s32.f32
+define i32 @test_fptosi_i32(half* %p) #0 {
+  %a = load half, half* %p, align 2
+  %r = fptosi half %a to i32
+  ret i32 %r
+}
+
+; CHECK-FP16-LABEL: test_fptosi_i64:
+; CHECK-FP16: vcvtb.f32.f16
+; CHECK-FP16: bl __aeabi_f2lz
+; CHECK-LIBCALL-LABEL: test_fptosi_i64:
+; CHECK-LIBCALL: bl __gnu_h2f_ieee
+; CHECK-LIBCALL: bl __aeabi_f2lz
+define i64 @test_fptosi_i64(half* %p) #0 {
+  %a = load half, half* %p, align 2
+  %r = fptosi half %a to i64
+  ret i64 %r
+}
+
+; CHECK-FP16-LABEL: test_fptoui_i32:
+; CHECK-FP16: vcvtb.f32.f16
+; CHECK-FP16: vcvt.u32.f32
+; CHECK-LIBCALL-LABEL: test_fptoui_i32:
+; CHECK-LIBCALL: bl __gnu_h2f_ieee
+; CHECK-LIBCALL: vcvt.u32.f32
+define i32 @test_fptoui_i32(half* %p) #0 {
+  %a = load half, half* %p, align 2
+  %r = fptoui half %a to i32
+  ret i32 %r
+}
+
+; CHECK-FP16-LABEL: test_fptoui_i64:
+; CHECK-FP16: vcvtb.f32.f16
+; CHECK-FP16: bl __aeabi_f2ulz
+; CHECK-LIBCALL-LABEL: test_fptoui_i64:
+; CHECK-LIBCALL: bl __gnu_h2f_ieee
+; CHECK-LIBCALL: bl __aeabi_f2ulz
+define i64 @test_fptoui_i64(half* %p) #0 {
+  %a = load half, half* %p, align 2
+  %r = fptoui half %a to i64
+  ret i64 %r
+}
+
+; CHECK-FP16-LABEL: test_sitofp_i32:
+; CHECK-FP16: vcvt.f32.s32
+; CHECK-FP16: vcvtb.f16.f32
+; CHECK-LIBCALL-LABEL: test_sitofp_i32:
+; CHECK-LIBCALL: vcvt.f32.s32
+; CHECK-LIBCALL: bl __gnu_f2h_ieee
+define void @test_sitofp_i32(i32 %a, half* %p) #0 {
+  %r = sitofp i32 %a to half
+  store half %r, half* %p
+  ret void
+}
+
+; CHECK-FP16-LABEL: test_uitofp_i32:
+; CHECK-FP16: vcvt.f32.u32
+; CHECK-FP16: vcvtb.f16.f32
+; CHECK-LIBCALL-LABEL: test_uitofp_i32:
+; CHECK-LIBCALL: vcvt.f32.u32
+; CHECK-LIBCALL: bl __gnu_f2h_ieee
+define void @test_uitofp_i32(i32 %a, half* %p) #0 {
+  %r = uitofp i32 %a to half
+  store half %r, half* %p
+  ret void
+}
+
+; CHECK-FP16-LABEL: test_sitofp_i64:
+; CHECK-FP16: bl __aeabi_l2f
+; CHECK-FP16: vcvtb.f16.f32
+; CHECK-LIBCALL-LABEL: test_sitofp_i64:
+; CHECK-LIBCALL: bl __aeabi_l2f
+; CHECK-LIBCALL: bl __gnu_f2h_ieee
+define void @test_sitofp_i64(i64 %a, half* %p) #0 {
+  %r = sitofp i64 %a to half
+  store half %r, half* %p
+  ret void
+}
+
+; CHECK-FP16-LABEL: test_uitofp_i64:
+; CHECK-FP16: bl __aeabi_ul2f
+; CHECK-FP16: vcvtb.f16.f32
+; CHECK-LIBCALL-LABEL: test_uitofp_i64:
+; CHECK-LIBCALL: bl __aeabi_ul2f
+; CHECK-LIBCALL: bl __gnu_f2h_ieee
+define void @test_uitofp_i64(i64 %a, half* %p) #0 {
+  %r = uitofp i64 %a to half
+  store half %r, half* %p
+  ret void
+}
+
+; CHECK-FP16-LABEL: test_fptrunc_float:
+; CHECK-FP16: vcvtb.f16.f32
+; CHECK-LIBCALL-LABEL: test_fptrunc_float:
+; CHECK-LIBCALL: bl __gnu_f2h_ieee
+define void @test_fptrunc_float(float %f, half* %p) #0 {
+  %a = fptrunc float %f to half
+  store half %a, half* %p
+  ret void
+}
+
+; CHECK-FP16-LABEL: test_fptrunc_double:
+; CHECK-FP16: bl __aeabi_d2h
+; CHECK-LIBCALL-LABEL: test_fptrunc_double:
+; CHECK-LIBCALL: bl __aeabi_d2h
+define void @test_fptrunc_double(double %d, half* %p) #0 {
+  %a = fptrunc double %d to half
+  store half %a, half* %p
+  ret void
+}
+
+; CHECK-FP16-LABEL: test_fpextend_float:
+; CHECK-FP16: vcvtb.f32.f16
+; CHECK-LIBCALL-LABEL: test_fpextend_float:
+; CHECK-LIBCALL: b __gnu_h2f_ieee
+define float @test_fpextend_float(half* %p) {
+  %a = load half, half* %p, align 2
+  %r = fpext half %a to float
+  ret float %r
+}
+
+; CHECK-FP16-LABEL: test_fpextend_double:
+; CHECK-FP16: vcvtb.f32.f16
+; CHECK-FP16: vcvt.f64.f32
+; CHECK-LIBCALL-LABEL: test_fpextend_double:
+; CHECK-LIBCALL: bl __gnu_h2f_ieee
+; CHECK-LIBCALL: vcvt.f64.f32
+define double @test_fpextend_double(half* %p) {
+  %a = load half, half* %p, align 2
+  %r = fpext half %a to double
+  ret double %r
+}
+
+; CHECK-ALL-LABEL: test_bitcast_halftoi16:
+; CHECK-ALL-NEXT: .fnstart
+; CHECK-ALL-NEXT: ldrh r0, [r0]
+; CHECK-ALL-NEXT: bx lr
+define i16 @test_bitcast_halftoi16(half* %p) #0 {
+  %a = load half, half* %p, align 2
+  %r = bitcast half %a to i16
+  ret i16 %r
+}
+
+; CHECK-ALL-LABEL: test_bitcast_i16tohalf:
+; CHECK-ALL-NEXT: .fnstart
+; CHECK-ALL-NEXT: strh r0, [r1]
+; CHECK-ALL-NEXT: bx lr
+define void @test_bitcast_i16tohalf(i16 %a, half* %p) #0 {
+  %r = bitcast i16 %a to half
+  store half %r, half* %p
+  ret void
+}
+
+declare half @llvm.sqrt.f16(half %a) #0
+declare half @llvm.powi.f16(half %a, i32 %b) #0
+declare half @llvm.sin.f16(half %a) #0
+declare half @llvm.cos.f16(half %a) #0
+declare half @llvm.pow.f16(half %a, half %b) #0
+declare half @llvm.exp.f16(half %a) #0
+declare half @llvm.exp2.f16(half %a) #0
+declare half @llvm.log.f16(half %a) #0
+declare half @llvm.log10.f16(half %a) #0
+declare half @llvm.log2.f16(half %a) #0
+declare half @llvm.fma.f16(half %a, half %b, half %c) #0
+declare half @llvm.fabs.f16(half %a) #0
+declare half @llvm.minnum.f16(half %a, half %b) #0
+declare half @llvm.maxnum.f16(half %a, half %b) #0
+declare half @llvm.copysign.f16(half %a, half %b) #0
+declare half @llvm.floor.f16(half %a) #0
+declare half @llvm.ceil.f16(half %a) #0
+declare half @llvm.trunc.f16(half %a) #0
+declare half @llvm.rint.f16(half %a) #0
+declare half @llvm.nearbyint.f16(half %a) #0
+declare half @llvm.round.f16(half %a) #0
+declare half @llvm.fmuladd.f16(half %a, half %b, half %c) #0
+
+; CHECK-FP16-LABEL: test_sqrt:
+; CHECK-FP16: vcvtb.f32.f16
+; CHECK-FP16: vsqrt.f32
+; CHECK-FP16: vcvtb.f16.f32
+; CHECK-LIBCALL-LABEL: test_sqrt:
+; CHECK-LIBCALL: bl __gnu_h2f_ieee
+; CHECK-LIBCALL: vsqrt.f32
+; CHECK-LIBCALL: bl __gnu_f2h_ieee
+define void @test_sqrt(half* %p) #0 {
+  %a = load half, half* %p, align 2
+  %r = call half @llvm.sqrt.f16(half %a)
+  store half %r, half* %p
+  ret void
+}
+
+; CHECK-FP16-LABEL: test_fpowi:
+; CHECK-FP16: vcvtb.f32.f16
+; CHECK-FP16: bl __powisf2
+; CHECK-FP16: vcvtb.f16.f32
+; CHECK-LIBCALL-LABEL: test_fpowi:
+; CHECK-LIBCALL: bl __gnu_h2f_ieee
+; CHECK-LIBCALL: bl __powisf2
+; CHECK-LIBCALL: bl __gnu_f2h_ieee
+define void @test_fpowi(half* %p, i32 %b) #0 {
+  %a = load half, half* %p, align 2
+  %r = call half @llvm.powi.f16(half %a, i32 %b)
+  store half %r, half* %p
+  ret void
+}
+
+; CHECK-FP16-LABEL: test_sin:
+; CHECK-FP16: vcvtb.f32.f16
+; CHECK-FP16: bl sinf
+; CHECK-FP16: vcvtb.f16.f32
+; CHECK-LIBCALL-LABEL: test_sin:
+; CHECK-LIBCALL: bl __gnu_h2f_ieee
+; CHECK-LIBCALL: bl sinf
+; CHECK-LIBCALL: bl __gnu_f2h_ieee
+define void @test_sin(half* %p) #0 {
+  %a = load half, half* %p, align 2
+  %r = call half @llvm.sin.f16(half %a)
+  store half %r, half* %p
+  ret void
+}
+
+; CHECK-FP16-LABEL: test_cos:
+; CHECK-FP16: vcvtb.f32.f16
+; CHECK-FP16: bl cosf
+; CHECK-FP16: vcvtb.f16.f32
+; CHECK-LIBCALL-LABEL: test_cos:
+; CHECK-LIBCALL: bl __gnu_h2f_ieee
+; CHECK-LIBCALL: bl cosf
+; CHECK-LIBCALL: bl __gnu_f2h_ieee
+define void @test_cos(half* %p) #0 {
+  %a = load half, half* %p, align 2
+  %r = call half @llvm.cos.f16(half %a)
+  store half %r, half* %p
+  ret void
+}
+
+; CHECK-FP16-LABEL: test_pow:
+; CHECK-FP16: vcvtb.f32.f16
+; CHECK-FP16: vcvtb.f32.f16
+; CHECK-FP16: bl powf
+; CHECK-FP16: vcvtb.f16.f32
+; CHECK-LIBCALL-LABEL: test_pow:
+; CHECK-LIBCALL: bl __gnu_h2f_ieee
+; CHECK-LIBCALL: bl __gnu_h2f_ieee
+; CHECK-LIBCALL: bl powf
+; CHECK-LIBCALL: bl __gnu_f2h_ieee
+define void @test_pow(half* %p, half* %q) #0 {
+  %a = load half, half* %p, align 2
+  %b = load half, half* %q, align 2
+  %r = call half @llvm.pow.f16(half %a, half %b)
+  store half %r, half* %p
+  ret void
+}
+
+; CHECK-FP16-LABEL: test_exp:
+; CHECK-FP16: vcvtb.f32.f16
+; CHECK-FP16: bl expf
+; CHECK-FP16: vcvtb.f16.f32
+; CHECK-LIBCALL-LABEL: test_exp:
+; CHECK-LIBCALL: bl __gnu_h2f_ieee
+; CHECK-LIBCALL: bl expf
+; CHECK-LIBCALL: bl __gnu_f2h_ieee
+define void @test_exp(half* %p) #0 {
+  %a = load half, half* %p, align 2
+  %r = call half @llvm.exp.f16(half %a)
+  store half %r, half* %p
+  ret void
+}
+
+; CHECK-FP16-LABEL: test_exp2:
+; CHECK-FP16: vcvtb.f32.f16
+; CHECK-FP16: bl exp2f
+; CHECK-FP16: vcvtb.f16.f32
+; CHECK-LIBCALL-LABEL: test_exp2:
+; CHECK-LIBCALL: bl __gnu_h2f_ieee
+; CHECK-LIBCALL: bl exp2f
+; CHECK-LIBCALL: bl __gnu_f2h_ieee
+define void @test_exp2(half* %p) #0 {
+  %a = load half, half* %p, align 2
+  %r = call half @llvm.exp2.f16(half %a)
+  store half %r, half* %p
+  ret void
+}
+
+; CHECK-FP16-LABEL: test_log:
+; CHECK-FP16: vcvtb.f32.f16
+; CHECK-FP16: bl logf
+; CHECK-FP16: vcvtb.f16.f32
+; CHECK-LIBCALL-LABEL: test_log:
+; CHECK-LIBCALL: bl __gnu_h2f_ieee
+; CHECK-LIBCALL: bl logf
+; CHECK-LIBCALL: bl __gnu_f2h_ieee
+define void @test_log(half* %p) #0 {
+  %a = load half, half* %p, align 2
+  %r = call half @llvm.log.f16(half %a)
+  store half %r, half* %p
+  ret void
+}
+
+; CHECK-FP16-LABEL: test_log10:
+; CHECK-FP16: vcvtb.f32.f16
+; CHECK-FP16: bl log10f
+; CHECK-FP16: vcvtb.f16.f32
+; CHECK-LIBCALL-LABEL: test_log10:
+; CHECK-LIBCALL: bl __gnu_h2f_ieee
+; CHECK-LIBCALL: bl log10f
+; CHECK-LIBCALL: bl __gnu_f2h_ieee
+define void @test_log10(half* %p) #0 {
+  %a = load half, half* %p, align 2
+  %r = call half @llvm.log10.f16(half %a)
+  store half %r, half* %p
+  ret void
+}
+
+; CHECK-FP16-LABEL: test_log2:
+; CHECK-FP16: vcvtb.f32.f16
+; CHECK-FP16: bl log2f
+; CHECK-FP16: vcvtb.f16.f32
+; CHECK-LIBCALL-LABEL: test_log2:
+; CHECK-LIBCALL: bl __gnu_h2f_ieee
+; CHECK-LIBCALL: bl log2f
+; CHECK-LIBCALL: bl __gnu_f2h_ieee
+define void @test_log2(half* %p) #0 {
+  %a = load half, half* %p, align 2
+  %r = call half @llvm.log2.f16(half %a)
+  store half %r, half* %p
+  ret void
+}
+
+; CHECK-FP16-LABEL: test_fma:
+; CHECK-FP16: vcvtb.f32.f16
+; CHECK-FP16: vcvtb.f32.f16
+; CHECK-FP16: vcvtb.f32.f16
+; CHECK-FP16: bl fmaf
+; CHECK-FP16: vcvtb.f16.f32
+; CHECK-LIBCALL-LABEL: test_fma:
+; CHECK-LIBCALL: bl __gnu_h2f_ieee
+; CHECK-LIBCALL: bl __gnu_h2f_ieee
+; CHECK-LIBCALL: bl __gnu_h2f_ieee
+; CHECK-LIBCALL: bl fmaf
+; CHECK-LIBCALL: bl __gnu_f2h_ieee
+define void @test_fma(half* %p, half* %q, half* %r) #0 {
+  %a = load half, half* %p, align 2
+  %b = load half, half* %q, align 2
+  %c = load half, half* %r, align 2
+  %v = call half @llvm.fma.f16(half %a, half %b, half %c)
+  store half %v, half* %p
+  ret void
+}
+
+; CHECK-FP16-LABEL: test_fabs:
+; CHECK-FP16: vcvtb.f32.f16
+; CHECK-FP16: vabs.f32
+; CHECK-FP16: vcvtb.f16.f32
+; CHECK-LIBCALL-LABEL: test_fabs:
+; CHECK-LIBCALL: bl __gnu_h2f_ieee
+; CHECK-LIBCALL: bfc
+; CHECK-LIBCALL: bl __gnu_f2h_ieee
+define void @test_fabs(half* %p) {
+  %a = load half, half* %p, align 2
+  %r = call half @llvm.fabs.f16(half %a)
+  store half %r, half* %p
+  ret void
+}
+
+; CHECK-FP16-LABEL: test_minnum:
+; CHECK-FP16: vcvtb.f32.f16
+; CHECK-FP16: vcvtb.f32.f16
+; CHECK-FP16: bl fminf
+; CHECK-FP16: vcvtb.f16.f32
+; CHECK-LIBCALL-LABEL: test_minnum:
+; CHECK-LIBCALL: bl __gnu_h2f_ieee
+; CHECK-LIBCALL: bl __gnu_h2f_ieee
+; CHECK-LIBCALL: bl fminf
+; CHECK-LIBCALL: bl __gnu_f2h_ieee
+define void @test_minnum(half* %p, half* %q) #0 {
+  %a = load half, half* %p, align 2
+  %b = load half, half* %q, align 2
+  %r = call half @llvm.minnum.f16(half %a, half %b)
+  store half %r, half* %p
+  ret void
+}
+
+; CHECK-FP16-LABEL: test_maxnum:
+; CHECK-FP16: vcvtb.f32.f16
+; CHECK-FP16: vcvtb.f32.f16
+; CHECK-FP16: bl fmaxf
+; CHECK-FP16: vcvtb.f16.f32
+; CHECK-LIBCALL-LABEL: test_maxnum:
+; CHECK-LIBCALL: bl __gnu_h2f_ieee
+; CHECK-LIBCALL: bl __gnu_h2f_ieee
+; CHECK-LIBCALL: bl fmaxf
+; CHECK-LIBCALL: bl __gnu_f2h_ieee
+define void @test_maxnum(half* %p, half* %q) #0 {
+  %a = load half, half* %p, align 2
+  %b = load half, half* %q, align 2
+  %r = call half @llvm.maxnum.f16(half %a, half %b)
+  store half %r, half* %p
+  ret void
+}
+
+; CHECK-FP16-LABEL: test_copysign:
+; CHECK-FP16: vcvtb.f32.f16
+; CHECK-FP16: vcvtb.f32.f16
+; CHECK-FP16: vbsl
+; CHECK-FP16: vcvtb.f16.f32
+; CHECK-LIBCALL-LABEL: test_copysign:
+; CHECK-LIBCALL: bl __gnu_h2f_ieee
+; CHECK-LIBCALL: bl __gnu_h2f_ieee
+; CHECK-LIBCALL: vbsl
+; CHECK-LIBCALL: bl __gnu_f2h_ieee
+define void @test_copysign(half* %p, half* %q) #0 {
+  %a = load half, half* %p, align 2
+  %b = load half, half* %q, align 2
+  %r = call half @llvm.copysign.f16(half %a, half %b)
+  store half %r, half* %p
+  ret void
+}
+
+; CHECK-FP16-LABEL: test_floor:
+; CHECK-FP16: vcvtb.f32.f16
+; CHECK-FP16: bl floorf
+; CHECK-FP16: vcvtb.f16.f32
+; CHECK-LIBCALL-LABEL: test_floor:
+; CHECK-LIBCALL: bl __gnu_h2f_ieee
+; CHECK-LIBCALL: bl floorf
+; CHECK-LIBCALL: bl __gnu_f2h_ieee
+define void @test_floor(half* %p) {
+  %a = load half, half* %p, align 2
+  %r = call half @llvm.floor.f16(half %a)
+  store half %r, half* %p
+  ret void
+}
+
+; CHECK-FP16-LABEL: test_ceil:
+; CHECK-FP16: vcvtb.f32.f16
+; CHECK-FP16: bl ceilf
+; CHECK-FP16: vcvtb.f16.f32
+; CHECK-LIBCALL-LABEL: test_ceil:
+; CHECK-LIBCALL: bl __gnu_h2f_ieee
+; CHECK-LIBCALL: bl ceilf
+; CHECK-LIBCALL: bl __gnu_f2h_ieee
+define void @test_ceil(half* %p) {
+  %a = load half, half* %p, align 2
+  %r = call half @llvm.ceil.f16(half %a)
+  store half %r, half* %p
+  ret void
+}
+
+; CHECK-FP16-LABEL: test_trunc:
+; CHECK-FP16: vcvtb.f32.f16
+; CHECK-FP16: bl truncf
+; CHECK-FP16: vcvtb.f16.f32
+; CHECK-LIBCALL-LABEL: test_trunc:
+; CHECK-LIBCALL: bl __gnu_h2f_ieee
+; CHECK-LIBCALL: bl truncf
+; CHECK-LIBCALL: bl __gnu_f2h_ieee
+define void @test_trunc(half* %p) {
+  %a = load half, half* %p, align 2
+  %r = call half @llvm.trunc.f16(half %a)
+  store half %r, half* %p
+  ret void
+}
+
+; CHECK-FP16-LABEL: test_rint:
+; CHECK-FP16: vcvtb.f32.f16
+; CHECK-FP16: bl rintf
+; CHECK-FP16: vcvtb.f16.f32
+; CHECK-LIBCALL-LABEL: test_rint:
+; CHECK-LIBCALL: bl __gnu_h2f_ieee
+; CHECK-LIBCALL: bl rintf
+; CHECK-LIBCALL: bl __gnu_f2h_ieee
+define void @test_rint(half* %p) {
+  %a = load half, half* %p, align 2
+  %r = call half @llvm.rint.f16(half %a)
+  store half %r, half* %p
+  ret void
+}
+
+; CHECK-FP16-LABEL: test_nearbyint:
+; CHECK-FP16: vcvtb.f32.f16
+; CHECK-FP16: bl nearbyintf
+; CHECK-FP16: vcvtb.f16.f32
+; CHECK-LIBCALL-LABEL: test_nearbyint:
+; CHECK-LIBCALL: bl __gnu_h2f_ieee
+; CHECK-LIBCALL: bl nearbyintf
+; CHECK-LIBCALL: bl __gnu_f2h_ieee
+define void @test_nearbyint(half* %p) {
+  %a = load half, half* %p, align 2
+  %r = call half @llvm.nearbyint.f16(half %a)
+  store half %r, half* %p
+  ret void
+}
+
+; CHECK-FP16-LABEL: test_round:
+; CHECK-FP16: vcvtb.f32.f16
+; CHECK-FP16: bl roundf
+; CHECK-FP16: vcvtb.f16.f32
+; CHECK-LIBCALL-LABEL: test_round:
+; CHECK-LIBCALL: bl __gnu_h2f_ieee
+; CHECK-LIBCALL: bl roundf
+; CHECK-LIBCALL: bl __gnu_f2h_ieee
+define void @test_round(half* %p) {
+  %a = load half, half* %p, align 2
+  %r = call half @llvm.round.f16(half %a)
+  store half %r, half* %p
+  ret void
+}
+
+; CHECK-FP16-LABEL: test_fmuladd:
+; CHECK-FP16: vcvtb.f32.f16
+; CHECK-FP16: vcvtb.f32.f16
+; CHECK-FP16: vcvtb.f32.f16
+; CHECK-FP16: vmla.f32
+; CHECK-FP16: vcvtb.f16.f32
+; CHECK-LIBCALL-LABEL: test_fmuladd:
+; CHECK-LIBCALL: bl __gnu_h2f_ieee
+; CHECK-LIBCALL: bl __gnu_h2f_ieee
+; CHECK-LIBCALL: bl __gnu_h2f_ieee
+; CHECK-LIBCALL: vmla.f32
+; CHECK-LIBCALL: bl __gnu_f2h_ieee
+define void @test_fmuladd(half* %p, half* %q, half* %r) #0 {
+  %a = load half, half* %p, align 2
+  %b = load half, half* %q, align 2
+  %c = load half, half* %r, align 2
+  %v = call half @llvm.fmuladd.f16(half %a, half %b, half %c)
+  store half %v, half* %p
+  ret void
+}
+
+; f16 vectors are not legal in the backend.  Vector elements are not assigned
+; to the register, but are stored in the stack instead.  Hence insertelement
+; and extractelement have these extra loads and stores.
+
+; CHECK-ALL-LABEL: test_insertelement:
+; CHECK-ALL-NEXT: .fnstart
+; CHECK-ALL-NEXT: sub sp, sp, #8
+; CHECK-ALL-NEXT: ldrh
+; CHECK-ALL-NEXT: strh
+; CHECK-ALL-NEXT: ldrh
+; CHECK-ALL-NEXT: strh
+; CHECK-ALL-NEXT: ldrh
+; CHECK-ALL-NEXT: strh
+; CHECK-ALL-NEXT: ldrh
+; CHECK-ALL-NEXT: strh
+; CHECK-ALL-NEXT: mov
+; CHECK-ALL-NEXT: ldrh
+; CHECK-ALL-NEXT: add
+; CHECK-ALL-NEXT: strh
+; CHECK-ALL-NEXT: ldrh
+; CHECK-ALL-NEXT: strh
+; CHECK-ALL-NEXT: ldrh
+; CHECK-ALL-NEXT: strh
+; CHECK-ALL-NEXT: ldrh
+; CHECK-ALL-NEXT: strh
+; CHECK-ALL-NEXT: ldrh
+; CHECK-ALL-NEXT: strh
+; CHECK-ALL-NEXT: add sp, sp, #8
+; CHECK-ALL-NEXT: bx lr
+define void @test_insertelement(half* %p, <4 x half>* %q, i32 %i) #0 {
+  %a = load half, half* %p, align 2
+  %b = load <4 x half>, <4 x half>* %q, align 8
+  %c = insertelement <4 x half> %b, half %a, i32 %i
+  store <4 x half> %c, <4 x half>* %q
+  ret void
+}
+
+; CHECK-ALL-LABEL: test_extractelement:
+; CHECK-ALL-NEXT: .fnstart
+; CHECK-ALL-NEXT: sub sp, sp, #8
+; CHECK-ALL-NEXT: ldrh
+; CHECK-ALL-NEXT: ldrh
+; CHECK-ALL-NEXT: orr
+; CHECK-ALL-NEXT: str
+; CHECK-ALL-NEXT: ldrh
+; CHECK-ALL-NEXT: ldrh
+; CHECK-ALL-NEXT: orr
+; CHECK-ALL-NEXT: str
+; CHECK-ALL-NEXT: mov
+; CHECK-ALL-NEXT: add
+; CHECK-ALL-NEXT: ldrh
+; CHECK-ALL-NEXT: strh
+; CHECK-ALL-NEXT: add sp, sp, #8
+; CHECK-ALL-NEXT: bx lr
+define void @test_extractelement(half* %p, <4 x half>* %q, i32 %i) #0 {
+  %a = load <4 x half>, <4 x half>* %q, align 8
+  %b = extractelement <4 x half> %a, i32 %i
+  store half %b, half* %p
+  ret void
+}
+
+; test struct operations
+
+%struct.dummy = type { i32, half }
+
+; CHECK-ALL-LABEL: test_insertvalue:
+; CHECK-ALL-NEXT: .fnstart
+; CHECK-ALL-NEXT: ldr
+; CHECK-ALL-NEXT: ldrh
+; CHECK-ALL-NEXT: strh
+; CHECK-ALL-NEXT: str
+; CHECK-ALL-NEXT: bx lr
+define void @test_insertvalue(%struct.dummy* %p, half* %q) {
+  %a = load %struct.dummy, %struct.dummy* %p
+  %b = load half, half* %q
+  %c = insertvalue %struct.dummy %a, half %b, 1
+  store %struct.dummy %c, %struct.dummy* %p
+  ret void
+}
+
+; CHECK-ALL-LABEL: test_extractvalue:
+; CHECK-ALL-NEXT: .fnstart
+; CHECK-ALL-NEXT: ldrh
+; CHECK-ALL-NEXT: strh
+; CHECK-ALL-NEXT: bx lr
+define void @test_extractvalue(%struct.dummy* %p, half* %q) {
+  %a = load %struct.dummy, %struct.dummy* %p
+  %b = extractvalue %struct.dummy %a, 1
+  store half %b, half* %q
+  ret void
+}
+
+; CHECK-FP16-LABEL: test_struct_return:
+; CHECK-FP16: vcvtb.f32.f16
+; CHECK-LIBCALL-LABEL: test_struct_return:
+; CHECK-LIBCALL: bl __gnu_h2f_ieee
+define %struct.dummy @test_struct_return(%struct.dummy* %p) {
+  %a = load %struct.dummy, %struct.dummy* %p
+  ret %struct.dummy %a
+}
+
+; CHECK-ALL-LABEL: test_struct_arg:
+; CHECK-ALL-NEXT: .fnstart
+; CHECK-ALL-NEXT: mov r0, r1
+; CHECK-ALL-NEXT: bx lr
+define half @test_struct_arg(%struct.dummy %p) {
+  %a = extractvalue %struct.dummy %p, 1
+  ret half %a
+}
+
+attributes #0 = { nounwind }
diff --git a/test/CodeGen/ARM/global-merge-addrspace.ll b/test/CodeGen/ARM/global-merge-addrspace.ll
index 7b71e89bbe7..8f40a4c28a7 100644
--- a/test/CodeGen/ARM/global-merge-addrspace.ll
+++ b/test/CodeGen/ARM/global-merge-addrspace.ll
@@ -1,4 +1,4 @@
-; RUN: llc < %s -mtriple=thumb-apple-darwin -O3 | FileCheck %s
+; RUN: llc < %s -mtriple=thumb-apple-darwin -arm-global-merge -global-merge-group-by-use=false | FileCheck %s
 ; Test the GlobalMerge pass. Check that the pass does not crash when using
 ; multiple address spaces.
 
@@ -7,6 +7,6 @@
 @g2 = internal addrspace(1) global i32 2
 
 
-; CHECK: _MergedGlobals1:
+; CHECK: _MergedGlobals.1:
 @g3 = internal addrspace(2) global i32 3
 @g4 = internal addrspace(2) global i32 4
diff --git a/test/CodeGen/ARM/global-merge.ll b/test/CodeGen/ARM/global-merge.ll
index d535ea7c0ce..fd06f1eeca7 100644
--- a/test/CodeGen/ARM/global-merge.ll
+++ b/test/CodeGen/ARM/global-merge.ll
@@ -1,4 +1,4 @@
-; RUN: llc < %s -mtriple=thumb-apple-darwin -O3 -global-merge-on-const=true | FileCheck %s
+; RUN: llc < %s -mtriple=thumb-apple-darwin -arm-global-merge -global-merge-group-by-use=false -global-merge-on-const=true | FileCheck %s
 ; Test the ARMGlobalMerge pass.  Use -march=thumb because it has a small
 ; value for the maximum offset (127).
 
@@ -15,13 +15,13 @@
 ; CHECK: ZTIi
 @_ZTIi = internal global i8* null
 
-define i32 @_Z9exceptioni(i32 %arg) {
+define i32 @_Z9exceptioni(i32 %arg) personality i8* bitcast (i32 (...)* @__gxx_personality_sj0 to i8*) {
 bb:
   %tmp = invoke i32 @_Z14throwSomethingi(i32 %arg)
           to label %bb9 unwind label %bb1
 
 bb1:                                              ; preds = %bb
-  %tmp2 = landingpad { i8*, i32 } personality i8* bitcast (i32 (...)* @__gxx_personality_sj0 to i8*)
+  %tmp2 = landingpad { i8*, i32 }
           catch i8* bitcast (i8** @_ZTIi to i8*)
   %tmp3 = extractvalue { i8*, i32 } %tmp2, 1
   %tmp4 = tail call i32 @llvm.eh.typeid.for(i8* bitcast (i8** @_ZTIi to i8*))
@@ -59,16 +59,16 @@ declare void @__cxa_end_catch()
 ; Make sure that the complete variable fits within the range of the maximum
 ; offset.  Having the starting offset in range is not sufficient.
 ; When this works properly, @g3 is placed in a separate chunk of merged globals.
-; CHECK: _MergedGlobals1:
+; CHECK: _MergedGlobals.1:
 @g3 = internal global [30 x i32] [ i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10 ], align 4
 
 ; Global variables that can be placed in BSS should be kept together in a
 ; separate pool of merged globals.
-; CHECK: _MergedGlobals2
+; CHECK: _MergedGlobals.2
 @g4 = internal global i32 0
 @g5 = internal global i32 0
 
 ; Global variables that are constant can be merged together
-; CHECK: _MergedGlobals3
+; CHECK: _MergedGlobals.3
 @g6 = internal constant [12 x i32] zeroinitializer, align 4
 @g7 = internal constant [12 x i32] zeroinitializer, align 4
diff --git a/test/CodeGen/ARM/gv-stubs-crash.ll b/test/CodeGen/ARM/gv-stubs-crash.ll
index 6e82afeacf8..b1e6e4f7b17 100644
--- a/test/CodeGen/ARM/gv-stubs-crash.ll
+++ b/test/CodeGen/ARM/gv-stubs-crash.ll
@@ -3,7 +3,7 @@
 
 @Exn = external hidden unnamed_addr constant { i8*, i8* }
 
-define hidden void @func(i32* %this, i32* %e) optsize align 2 {
+define hidden void @func(i32* %this, i32* %e) optsize align 2 personality i8* bitcast (i32 (...)* @__gxx_personality_sj0 to i8*) {
   %e.ld = load i32, i32* %e, align 4
   %inv = invoke zeroext i1 @func2(i32* %this, i32 %e.ld) optsize
           to label %ret unwind label %lpad
@@ -12,7 +12,7 @@ ret:
   ret void
 
 lpad:
-  %lp = landingpad { i8*, i32 } personality i8* bitcast (i32 (...)* @__gxx_personality_sj0 to i8*)
+  %lp = landingpad { i8*, i32 }
           catch i8* bitcast ({ i8*, i8* }* @Exn to i8*)
   br label %.loopexit4
 
diff --git a/test/CodeGen/ARM/half.ll b/test/CodeGen/ARM/half.ll
index 777aff2f007..b40eaf63851 100644
--- a/test/CodeGen/ARM/half.ll
+++ b/test/CodeGen/ARM/half.ll
@@ -30,7 +30,7 @@ define void @test_bitcast_to_half(half* %addr, i16 %in) {
 define float @test_extend32(half* %addr) {
 ; CHECK-LABEL: test_extend32:
 
-; CHECK-OLD: b.w ___gnu_h2f_ieee
+; CHECK-OLD: b.w ___extendhfsf2
 ; CHECK-F16: vcvtb.f32.f16
 ; CHECK-V8: vcvtb.f32.f16
   %val16 = load half, half* %addr
@@ -41,7 +41,7 @@ define float @test_extend32(half* %addr) {
 define double @test_extend64(half* %addr) {
 ; CHECK-LABEL: test_extend64:
 
-; CHECK-OLD: blx ___gnu_h2f_ieee
+; CHECK-OLD: blx ___extendhfsf2
 ; CHECK-OLD: vcvt.f64.f32
 ; CHECK-F16: vcvtb.f32.f16
 ; CHECK-F16: vcvt.f64.f32
@@ -54,7 +54,7 @@ define double @test_extend64(half* %addr) {
 define void @test_trunc32(float %in, half* %addr) {
 ; CHECK-LABEL: test_trunc32:
 
-; CHECK-OLD: blx ___gnu_f2h_ieee
+; CHECK-OLD: blx ___truncsfhf2
 ; CHECK-F16: vcvtb.f16.f32
 ; CHECK-V8: vcvtb.f16.f32
   %val16 = fptrunc float %in to half
diff --git a/test/CodeGen/ARM/ifcvt-callback.ll b/test/CodeGen/ARM/ifcvt-callback.ll
new file mode 100644
index 00000000000..62a66e745b3
--- /dev/null
+++ b/test/CodeGen/ARM/ifcvt-callback.ll
@@ -0,0 +1,22 @@
+; RUN: llc -march thumb %s -o - | FileCheck %s
+
+; This test checks that if-conversion pass is unconditionally added to the pass
+; pipeline and is conditionally executed based on the per-function targert-cpu
+; attribute.
+ 
+; CHECK: ite eq
+
+define i32 @test_ifcvt(i32 %a, i32 %b) #0 {
+  %tmp2 = icmp eq i32 %a, 0
+  br i1 %tmp2, label %cond_false, label %cond_true
+
+cond_true:
+  %tmp5 = add i32 %b, 1
+  ret i32 %tmp5
+
+cond_false:
+  %tmp7 = add i32 %b, -1
+  ret i32 %tmp7
+}
+
+attributes #0 = { "target-cpu"="cortex-a8" }
diff --git a/test/CodeGen/ARM/ifcvt-dead-def.ll b/test/CodeGen/ARM/ifcvt-dead-def.ll
new file mode 100644
index 00000000000..77a3f5c0961
--- /dev/null
+++ b/test/CodeGen/ARM/ifcvt-dead-def.ll
@@ -0,0 +1,55 @@
+; RUN: llc %s -o - -verify-machineinstrs | FileCheck %s
+
+target datalayout = "e-m:o-p:32:32-f64:32:64-v64:32:64-v128:32:128-a:0:32-n32-S32"
+target triple = "thumbv7-unknown-unknown"
+
+%struct.ref_s = type { %union.v, i16, i16 }
+%union.v = type { i32 }
+%struct.gs_color_s = type { i16, i16, i16, i16, i8, i8 }
+
+; In this case, the if converter was cloning the return instruction so that we had
+;   r2<def> = ...
+;   return [pred] r2<dead,def>
+;   ldr <r2, kill>
+;   return
+; The problem here was that the dead def on the first return was making the machine verifier
+; think that the load read from an undefined register.  We need to remove the dead flag from
+; the return, and also add an implicit use of the prior value of r2.
+
+; CHECK: ldrh
+
+; Function Attrs: minsize nounwind optsize ssp
+define i32 @test(%struct.ref_s* %pref1, %struct.ref_s* %pref2, %struct.gs_color_s** %tmp152) #0 {
+bb:
+  %nref = alloca %struct.ref_s, align 4
+  %tmp46 = call %struct.ref_s* @name_string_ref(%struct.ref_s* %pref1, %struct.ref_s* %nref) #2
+  %tmp153 = load %struct.gs_color_s*, %struct.gs_color_s** %tmp152, align 4
+  %tmp154 = bitcast %struct.ref_s* %pref2 to %struct.gs_color_s**
+  %tmp155 = load %struct.gs_color_s*, %struct.gs_color_s** %tmp154, align 4
+  %tmp162 = getelementptr inbounds %struct.gs_color_s, %struct.gs_color_s* %tmp153, i32 0, i32 1
+  %tmp163 = load i16, i16* %tmp162, align 2
+  %tmp164 = getelementptr inbounds %struct.gs_color_s, %struct.gs_color_s* %tmp155, i32 0, i32 1
+  %tmp165 = load i16, i16* %tmp164, align 2
+  %tmp166 = icmp eq i16 %tmp163, %tmp165
+  br i1 %tmp166, label %bb167, label %bb173
+
+bb167:                                            ; preds = %bb
+  %tmp168 = getelementptr inbounds %struct.gs_color_s, %struct.gs_color_s* %tmp153, i32 0, i32 2
+  %tmp169 = load i16, i16* %tmp168, align 2
+  %tmp170 = getelementptr inbounds %struct.gs_color_s, %struct.gs_color_s* %tmp155, i32 0, i32 2
+  %tmp171 = load i16, i16* %tmp170, align 2
+  %tmp172 = icmp eq i16 %tmp169, %tmp171
+  br label %bb173
+
+bb173:                                            ; preds = %bb167, %bb
+  %tmp174 = phi i1 [ false, %bb ], [ %tmp172, %bb167 ]
+  %tmp175 = zext i1 %tmp174 to i32
+  ret i32 %tmp175
+}
+
+; Function Attrs: minsize optsize
+declare %struct.ref_s* @name_string_ref(%struct.ref_s*, %struct.ref_s*) #1
+
+attributes #0 = { minsize nounwind optsize }
+attributes #1 = { minsize optsize }
+attributes #2 = { minsize nounwind optsize }
diff --git a/test/CodeGen/ARM/ifcvt-regmask-noreturn.ll b/test/CodeGen/ARM/ifcvt-regmask-noreturn.ll
new file mode 100644
index 00000000000..cf04fc90504
--- /dev/null
+++ b/test/CodeGen/ARM/ifcvt-regmask-noreturn.ll
@@ -0,0 +1,45 @@
+; RUN: llc %s -o - -verify-machineinstrs | FileCheck %s
+
+target datalayout = "e-m:o-p:32:32-f64:32:64-v64:32:64-v128:32:128-a:0:32-n32-S32"
+target triple = "thumbv7s-apple-ios8.0.0"
+
+@debw = external global i8*, align 4
+
+; This test ensures that the stack_chk call correctly puts implicit uses/defs for the regsiters
+; live across it when if converting.  This will be R0 which is passed to the call to free at the end
+; of the function.
+; Prior to this change, the stack_chk call (which does not return) would clobber R0 in its regmask,
+; leading to verifier errors because the later use of R0 in free() is not live.
+
+; CHECK-LABEL: @test
+; CHECK: stack_chk_fail
+
+; Function Attrs: ssp
+define void @test(i32 %argc, i8** nocapture readonly %argv, i32* %ptr, i32 %val) #0 {
+entry:
+  %count.i = alloca [256 x i32], align 4
+  %cmp284.i = icmp eq i32 %val, 0
+  br i1 %cmp284.i, label %for.end31.i, label %for.body21.i
+
+for.body21.i:                                     ; preds = %entry
+  %arrayidx23.i = getelementptr inbounds [256 x i32], [256 x i32]* %count.i, i32 0, i32 1
+  %tmp20 = load i32, i32* %arrayidx23.i, align 4, !tbaa !0
+  store i32 %tmp20, i32* %ptr, align 4, !tbaa !0
+  br label %for.end31.i
+
+for.end31.i:                                      ; preds = %for.body21.i, %entry
+  %tmp21 = load i8*, i8** @debw, align 4, !tbaa !4
+  tail call void @free(i8* %tmp21)
+  ret void
+}
+
+declare void @free(i8* nocapture)
+
+attributes #0 = { ssp "stack-protector-buffer-size"="8" }
+
+!0 = !{!1, !1, i64 0}
+!1 = !{!"int", !2, i64 0}
+!2 = !{!"omnipotent char", !3, i64 0}
+!3 = !{!"Simple C/C++ TBAA"}
+!4 = !{!5, !5, i64 0}
+!5 = !{!"any pointer", !2, i64 0}
diff --git a/test/CodeGen/ARM/ifcvt3.ll b/test/CodeGen/ARM/ifcvt3.ll
index 5da63dc5f02..e53d989ad52 100644
--- a/test/CodeGen/ARM/ifcvt3.ll
+++ b/test/CodeGen/ARM/ifcvt3.ll
@@ -4,8 +4,8 @@
 
 define i32 @t1(i32 %a, i32 %b, i32 %c, i32 %d) {
 ; CHECK-LABEL: t1:
-; CHECK: cmp r2, #1
-; CHECK: cmpne r2, #7
+; CHECK: cmp r2, #7
+; CHECK: cmpne r2, #1
 	switch i32 %c, label %cond_next [
 		 i32 1, label %cond_true
 		 i32 7, label %cond_true
diff --git a/test/CodeGen/ARM/interrupt-attr.ll b/test/CodeGen/ARM/interrupt-attr.ll
index 95ada085b0d..43bd5815a55 100644
--- a/test/CodeGen/ARM/interrupt-attr.ll
+++ b/test/CodeGen/ARM/interrupt-attr.ll
@@ -35,7 +35,7 @@ define arm_aapcscc void @irq_fn() alignstack(8) "interrupt"="IRQ" {
   ; Normal AAPCS function (r0-r3 pushed onto stack by hardware, lr set to
   ; appropriate sentinel so no special return needed).
 ; CHECK-M-LABEL: irq_fn:
-; CHECK-M: push.w {r4, r10, r11, lr}
+; CHECK-M: push.w {r4, r7, r11, lr}
 ; CHECK-M: add.w r11, sp, #8
 ; CHECK-M: mov r4, sp
 ; CHECK-M: bfc r4, #0, #3
@@ -43,7 +43,7 @@ define arm_aapcscc void @irq_fn() alignstack(8) "interrupt"="IRQ" {
 ; CHECK-M: bl _bar
 ; CHECK-M: sub.w r4, r11, #8
 ; CHECK-M: mov sp, r4
-; CHECK-M: pop.w {r4, r10, r11, pc}
+; CHECK-M: pop.w {r4, r7, r11, pc}
 
   call arm_aapcscc void @bar()
   ret void
diff --git a/test/CodeGen/ARM/invoke-donothing-assert.ll b/test/CodeGen/ARM/invoke-donothing-assert.ll
index aab3556c547..c6489e3a4ce 100644
--- a/test/CodeGen/ARM/invoke-donothing-assert.ll
+++ b/test/CodeGen/ARM/invoke-donothing-assert.ll
@@ -4,7 +4,7 @@
 ; <rdar://problem/13228754> & <rdar://problem/13316637>
 
 ; CHECK: .globl  _foo
-define void @foo() {
+define void @foo() personality i8* bitcast (i32 (...)* @__gxx_personality_sj0 to i8*) {
 invoke.cont:
   invoke void @callA() 
           to label %invoke.cont25 unwind label %lpad2
@@ -20,12 +20,12 @@ invoke.cont75:
   ret void
 
 lpad2:
-  %0 = landingpad { i8*, i32 } personality i8* bitcast (i32 (...)* @__gxx_personality_sj0 to i8*)
+  %0 = landingpad { i8*, i32 }
           cleanup
   br label %eh.resume
 
 lpad15:
-  %1 = landingpad { i8*, i32 } personality i8* bitcast (i32 (...)* @__gxx_personality_sj0 to i8*)
+  %1 = landingpad { i8*, i32 }
           cleanup
   br label %eh.resume
 
@@ -34,7 +34,7 @@ eh.resume:
 }
 
 ; CHECK: .globl _bar
-define linkonce_odr void @bar(i32* %a) {
+define linkonce_odr void @bar(i32* %a) personality i8* bitcast (i32 (...)* @__gxx_personality_sj0 to i8*) {
 if.end.i.i.i:
   invoke void @llvm.donothing()
           to label %call.i.i.i.noexc unwind label %eh.resume
@@ -58,7 +58,7 @@ _ZN3lol5ArrayIivvvvvvvED1Ev.exit:
   ret void
 
 eh.resume:
-  %1 = landingpad { i8*, i32 } personality i8* bitcast (i32 (...)* @__gxx_personality_sj0 to i8*)
+  %1 = landingpad { i8*, i32 }
           cleanup
   %2 = extractvalue { i8*, i32 } %1, 0
   %3 = extractvalue { i8*, i32 } %1, 1
diff --git a/test/CodeGen/ARM/jump-table-islands-split.ll b/test/CodeGen/ARM/jump-table-islands-split.ll
new file mode 100644
index 00000000000..deba21b4dbb
--- /dev/null
+++ b/test/CodeGen/ARM/jump-table-islands-split.ll
@@ -0,0 +1,52 @@
+; RUN: llc -mtriple=thumbv7s-apple-ios8.0 -o - %s | FileCheck %s
+
+declare void @foo(double)
+declare i32 @llvm.arm.space(i32, i32)
+
+; The constpool entry used to call @foo should be directly between where we want
+; the tbb and its table. Fortunately, the flow is simple enough that we can
+; eliminate the entry calculation (ADD) and use the ADR as the base.
+;
+; I'm hoping this won't be fragile, but if it does break the most likely fix is
+; adjusting the @llvm.arm.space call slightly. If this happens too many times
+; the test should probably be removed.
+define i32 @test_jumptable_not_adjacent(i1 %tst, i32 %sw, i32 %l) {
+; CHECK-LABEL: test_jumptable_not_adjacent:
+; CHECK:     vldr {{d[0-9]+}}, [[DBL_CONST:LCPI[0-9]+_[0-9]+]]
+; [...]
+; CHECK:     adr.w r[[BASE:[0-9]+]], [[JUMP_TABLE:LJTI[0-9]+_[0-9]+]]
+; CHECK-NOT: r[[BASE]]
+
+; CHECK: [[TBB_KEY:LCPI[0-9]+_[0-9]+]]:
+; CHECK-NEXT:     tbb [r[[BASE]], {{r[0-9]+}}]
+
+; CHECK: [[DBL_CONST]]:
+; CHECK:     .long
+; CHECK:     .long
+; CHECK: [[JUMP_TABLE]]:
+; CHECK:     .byte (LBB{{[0-9]+}}_{{[0-9]+}}-([[TBB_KEY]]+4)
+
+  br label %complex
+
+complex:
+  call void @foo(double 12345.0)
+  call i32 @llvm.arm.space(i32 970, i32 undef)
+  switch i32 %sw, label %second [ i32 0, label %other
+                                  i32 1, label %third
+                                  i32 2, label %end
+                                  i32 3, label %other ]
+
+second:
+  ret i32 43
+third:
+  ret i32 0
+
+other:
+  call void @bar()
+  unreachable
+
+end:
+  ret i32 42
+}
+
+declare void @bar()
diff --git a/test/CodeGen/ARM/jump-table-islands.ll b/test/CodeGen/ARM/jump-table-islands.ll
new file mode 100644
index 00000000000..6b4f174c092
--- /dev/null
+++ b/test/CodeGen/ARM/jump-table-islands.ll
@@ -0,0 +1,40 @@
+; RUN: llc -mtriple=armv7-apple-ios8.0 -o - %s | FileCheck %s
+
+%BigInt = type i5500
+
+define %BigInt @test_moved_jumptable(i1 %tst, i32 %sw, %BigInt %l) {
+; CHECK-LABEL: test_moved_jumptable:
+
+; CHECK:   adr {{r[0-9]+}}, [[JUMP_TABLE:LJTI[0-9]+_[0-9]+]]
+; CHECK:   b [[SKIP_TABLE:LBB[0-9]+_[0-9]+]]
+
+; CHECK: [[JUMP_TABLE]]:
+; CHECK:   .data_region jt32
+; CHECK:   .long LBB{{[0-9]+_[0-9]+}}-[[JUMP_TABLE]]
+
+; CHECK: [[SKIP_TABLE]]:
+; CHECK:   add pc, {{r[0-9]+}}, {{r[0-9]+}}
+  br i1 %tst, label %simple, label %complex
+
+simple:
+  br label %end
+
+complex:
+  switch i32 %sw, label %simple [ i32 0, label %other
+                                  i32 1, label %third
+                                  i32 5, label %end
+                                  i32 6, label %other ]
+
+third:
+  ret %BigInt 0
+
+other:
+  call void @bar()
+  unreachable
+
+end:
+  %val = phi %BigInt [ %l, %complex ], [ -1, %simple ]
+  ret %BigInt %val
+}
+
+declare void @bar()
diff --git a/test/CodeGen/ARM/jumptable-label.ll b/test/CodeGen/ARM/jumptable-label.ll
index 49d698672f8..2ba90dc9736 100644
--- a/test/CodeGen/ARM/jumptable-label.ll
+++ b/test/CodeGen/ARM/jumptable-label.ll
@@ -2,8 +2,8 @@
 
 ; test that we print the label of a bb that is only used in a jump table.
 
-; CHECK:	.long	LBB0_2
-; CHECK: LBB0_2:
+; CHECK:	.long	[[JUMPTABLE_DEST:LBB[0-9]+_[0-9]+]]
+; CHECK: [[JUMPTABLE_DEST]]:
 
 define i32 @calculate()  {
 entry:
diff --git a/test/CodeGen/ARM/ldrd.ll b/test/CodeGen/ARM/ldrd.ll
index 7ce846844e0..56cdcaedf90 100644
--- a/test/CodeGen/ARM/ldrd.ll
+++ b/test/CodeGen/ARM/ldrd.ll
@@ -3,26 +3,28 @@
 ; rdar://6949835
 ; RUN: llc < %s -mtriple=thumbv7-apple-ios -mcpu=cortex-a8 -regalloc=basic | FileCheck %s -check-prefix=BASIC -check-prefix=CHECK
 ; RUN: llc < %s -mtriple=thumbv7-apple-ios -mcpu=cortex-a8 -regalloc=greedy | FileCheck %s -check-prefix=GREEDY -check-prefix=CHECK
+; RUN: llc < %s -mtriple=thumbv7-apple-ios -mcpu=swift | FileCheck %s -check-prefix=SWIFT -check-prefix=CHECK
 
 ; Magic ARM pair hints works best with linearscan / fast.
 
-; Cortex-M3 errata 602117: LDRD with base in list may result in incorrect base
-; register when interrupted or faulted.
-
 @b = external global i64*
 
-define i64 @t(i64 %a) nounwind readonly {
-entry:
-; A8-LABEL: t:
-; A8:   ldrd r2, r3, [r2]
+; We use the following two to force values into specific registers.
+declare i64* @get_ptr()
+declare void @use_i64(i64 %v)
 
-; M3-LABEL: t:
-; M3-NOT: ldrd
-
-	%0 = load i64*, i64** @b, align 4
-	%1 = load i64, i64* %0, align 4
-	%2 = mul i64 %1, %a
-	ret i64 %2
+define void @test_ldrd(i64 %a) nounwind readonly {
+; CHECK-LABEL: test_ldrd:
+; CHECK: bl{{x?}} _get_ptr
+; A8: ldrd r0, r1, [r0]
+; Cortex-M3 errata 602117: LDRD with base in list may result in incorrect base
+; register when interrupted or faulted.
+; M3-NOT: ldrd r[[REGNUM:[0-9]+]], {{r[0-9]+}}, [r[[REGNUM]]]
+; CHECK: bl{{x?}} _use_i64
+  %ptr = call i64* @get_ptr()
+  %v = load i64, i64* %ptr, align 8
+  call void @use_i64(i64 %v)
+  ret void
 }
 
 ; rdar://10435045 mixed LDRi8/LDRi12
@@ -92,6 +94,90 @@ entry:
   ret void
 }
 
+declare void @extfunc(i32, i32, i32, i32)
+
+; CHECK-LABEL: Func2:
+; A8: ldrd
+; A8: blx
+; A8: pop
+define void @Func2(i32* %p) {
+entry:
+  %addr0 = getelementptr i32, i32* %p, i32 0
+  %addr1 = getelementptr i32, i32* %p, i32 1
+  %v0 = load i32, i32* %addr0
+  %v1 = load i32, i32* %addr1
+  ; try to force %v0/%v1 into non-adjacent registers
+  call void @extfunc(i32 %v0, i32 0, i32 0, i32 %v1)
+  ret void
+}
+
+; CHECK-LABEL: strd_spill_ldrd_reload:
+; A8: strd r1, r0, [sp, #-8]!
+; M3: strd r1, r0, [sp, #-8]!
+; BASIC: strd r1, r0, [sp, #-8]!
+; GREEDY: strd r0, r1, [sp, #-8]!
+; CHECK: @ InlineAsm Start
+; CHECK: @ InlineAsm End
+; A8: ldrd r2, r1, [sp]
+; M3: ldrd r2, r1, [sp]
+; BASIC: ldrd r2, r1, [sp]
+; GREEDY: ldrd r1, r2, [sp]
+; CHECK: bl{{x?}} _extfunc
+define void @strd_spill_ldrd_reload(i32 %v0, i32 %v1) {
+  ; force %v0 and %v1 to be spilled
+  call void asm sideeffect "", "~{r0},~{r1},~{r2},~{r3},~{r4},~{r5},~{r6},~{r7},~{r8},~{r9},~{r10},~{r11},~{r12},~{lr}"()
+  ; force the reloaded %v0, %v1 into different registers
+  call void @extfunc(i32 0, i32 %v0, i32 %v1, i32 7)
+  ret void
+}
+
+declare void @extfunc2(i32*, i32, i32)
+
+; CHECK-LABEL: ldrd_postupdate_dec:
+; CHECK: ldrd r1, r2, [r0], #-8
+; CHECK-NEXT: bl{{x?}} _extfunc
+define void @ldrd_postupdate_dec(i32* %p0) {
+  %p0.1 = getelementptr i32, i32* %p0, i32 1
+  %v0 = load i32, i32* %p0
+  %v1 = load i32, i32* %p0.1
+  %p1 = getelementptr i32, i32* %p0, i32 -2
+  call void @extfunc2(i32* %p1, i32 %v0, i32 %v1)
+  ret void
+}
+
+; CHECK-LABEL: ldrd_postupdate_inc:
+; CHECK: ldrd r1, r2, [r0], #8
+; CHECK-NEXT: bl{{x?}} _extfunc
+define void @ldrd_postupdate_inc(i32* %p0) {
+  %p0.1 = getelementptr i32, i32* %p0, i32 1
+  %v0 = load i32, i32* %p0
+  %v1 = load i32, i32* %p0.1
+  %p1 = getelementptr i32, i32* %p0, i32 2
+  call void @extfunc2(i32* %p1, i32 %v0, i32 %v1)
+  ret void
+}
+
+; CHECK-LABEL: strd_postupdate_dec:
+; CHECK: strd r1, r2, [r0], #-8
+; CHECK-NEXT: bx lr
+define i32* @strd_postupdate_dec(i32* %p0, i32 %v0, i32 %v1) {
+  %p0.1 = getelementptr i32, i32* %p0, i32 1
+  store i32 %v0, i32* %p0
+  store i32 %v1, i32* %p0.1
+  %p1 = getelementptr i32, i32* %p0, i32 -2
+  ret i32* %p1
+}
+
+; CHECK-LABEL: strd_postupdate_inc:
+; CHECK: strd r1, r2, [r0], #8
+; CHECK-NEXT: bx lr
+define i32* @strd_postupdate_inc(i32* %p0, i32 %v0, i32 %v1) {
+  %p0.1 = getelementptr i32, i32* %p0, i32 1
+  store i32 %v0, i32* %p0
+  store i32 %v1, i32* %p0.1
+  %p1 = getelementptr i32, i32* %p0, i32 2
+  ret i32* %p1
+}
 
 declare void @llvm.lifetime.start(i64, i8* nocapture) nounwind
 declare void @llvm.lifetime.end(i64, i8* nocapture) nounwind
diff --git a/test/CodeGen/ARM/load-store-flags.ll b/test/CodeGen/ARM/load-store-flags.ll
new file mode 100644
index 00000000000..5825a30109d
--- /dev/null
+++ b/test/CodeGen/ARM/load-store-flags.ll
@@ -0,0 +1,43 @@
+; RUN: llc -mtriple=thumbv7-apple-ios7.0 -o - %s -verify-machineinstrs | FileCheck %s
+
+; The base register for the store is killed by the last instruction, but is
+; actually also used during as part of the store itself. If an extra ADD is
+; inserted, it should not kill the base.
+define void @test_base_kill(i32 %v0, i32 %v1, i32* %addr) {
+; CHECK-LABEL: test_base_kill:
+; CHECK: adds [[NEWBASE:r[0-9]+]], r2, #4
+; CHECK: stm.w [[NEWBASE]], {r0, r1, r2}
+
+  %addr.1 = getelementptr i32, i32* %addr, i32 1
+  store i32 %v0, i32* %addr.1
+
+  %addr.2 = getelementptr i32, i32* %addr, i32 2
+  store i32 %v1, i32* %addr.2
+
+  %addr.3 = getelementptr i32, i32* %addr, i32 3
+  %val = ptrtoint i32* %addr to i32
+  store i32 %val, i32* %addr.3
+
+  ret void
+}
+
+; Similar, but it's not sufficient to look at just the last instruction (where
+; liveness of the base is determined). An intervening instruction might be moved
+; past it to form the STM.
+define void @test_base_kill_mid(i32 %v0, i32* %addr, i32 %v1) {
+; CHECK-LABEL: test_base_kill_mid:
+; CHECK: adds [[NEWBASE:r[0-9]+]], r1, #4
+; CHECK: stm.w [[NEWBASE]], {r0, r1, r2}
+
+  %addr.1 = getelementptr i32, i32* %addr, i32 1
+  store i32 %v0, i32* %addr.1
+
+  %addr.2 = getelementptr i32, i32* %addr, i32 2
+  %val = ptrtoint i32* %addr to i32
+  store i32 %val, i32* %addr.2
+
+  %addr.3 = getelementptr i32, i32* %addr, i32 3
+  store i32 %v1, i32* %addr.3
+
+  ret void
+}
diff --git a/test/CodeGen/ARM/memcpy-inline.ll b/test/CodeGen/ARM/memcpy-inline.ll
index 4ea26e1c59a..d874884dcb3 100644
--- a/test/CodeGen/ARM/memcpy-inline.ll
+++ b/test/CodeGen/ARM/memcpy-inline.ll
@@ -1,5 +1,5 @@
 ; RUN: llc < %s -mtriple=thumbv7-apple-ios -mcpu=cortex-a8 -pre-RA-sched=source -disable-post-ra | FileCheck %s
-; RUN: llc < %s -mtriple=thumbv6m-apple-ios -mcpu=cortex-m0 -pre-RA-sched=source -disable-post-ra | FileCheck %s -check-prefix=CHECK-T1
+; RUN: llc < %s -mtriple=thumbv6m-apple-ios -mcpu=cortex-m0 -pre-RA-sched=source -disable-post-ra -mattr=+strict-align | FileCheck %s -check-prefix=CHECK-T1
 %struct.x = type { i8, i8, i8, i8, i8, i8, i8, i8, i8, i8, i8 }
 
 @src = external global %struct.x
diff --git a/test/CodeGen/ARM/memfunc.ll b/test/CodeGen/ARM/memfunc.ll
index c2143361ffb..5223983a7f3 100644
--- a/test/CodeGen/ARM/memfunc.ll
+++ b/test/CodeGen/ARM/memfunc.ll
@@ -18,13 +18,64 @@ entry:
   call void @llvm.memcpy.p0i8.p0i8.i32(i8* %dest, i8* %src, i32 500, i32 0, i1 false)
 
   ; EABI memset swaps arguments
+  ; CHECK-IOS: mov r1, #1
+  ; CHECK-IOS: memset
+  ; CHECK-DARWIN: movs r1, #1
+  ; CHECK-DARWIN: memset
+  ; CHECK-EABI: mov r2, #1
+  ; CHECK-EABI: __aeabi_memset
+  call void @llvm.memset.p0i8.i32(i8* %dest, i8 1, i32 500, i32 0, i1 false)
+
+  ; EABI uses memclr if value set to 0
   ; CHECK-IOS: mov r1, #0
   ; CHECK-IOS: memset
   ; CHECK-DARWIN: movs r1, #0
   ; CHECK-DARWIN: memset
-  ; CHECK-EABI: mov r2, #0
-  ; CHECK-EABI: __aeabi_memset
+  ; CHECK-EABI: __aeabi_memclr
   call void @llvm.memset.p0i8.i32(i8* %dest, i8 0, i32 500, i32 0, i1 false)
+  
+  ; EABI uses aligned function variants if possible
+
+  ; CHECK-IOS: memmove
+  ; CHECK-DARWIN: memmove
+  ; CHECK-EABI: __aeabi_memmove4
+  call void @llvm.memmove.p0i8.p0i8.i32(i8* %dest, i8* %src, i32 500, i32 4, i1 false)
+
+  ; CHECK-IOS: memcpy
+  ; CHECK-DARWIN: memcpy
+  ; CHECK-EABI: __aeabi_memcpy4
+  call void @llvm.memcpy.p0i8.p0i8.i32(i8* %dest, i8* %src, i32 500, i32 4, i1 false)
+
+  ; CHECK-IOS: memset
+  ; CHECK-DARWIN: memset
+  ; CHECK-EABI: __aeabi_memset4
+  call void @llvm.memset.p0i8.i32(i8* %dest, i8 1, i32 500, i32 4, i1 false)
+
+  ; CHECK-IOS: memset
+  ; CHECK-DARWIN: memset
+  ; CHECK-EABI: __aeabi_memclr4
+  call void @llvm.memset.p0i8.i32(i8* %dest, i8 0, i32 500, i32 4, i1 false)
+
+  ; CHECK-IOS: memmove
+  ; CHECK-DARWIN: memmove
+  ; CHECK-EABI: __aeabi_memmove8
+  call void @llvm.memmove.p0i8.p0i8.i32(i8* %dest, i8* %src, i32 500, i32 8, i1 false)
+
+  ; CHECK-IOS: memcpy
+  ; CHECK-DARWIN: memcpy
+  ; CHECK-EABI: __aeabi_memcpy8
+  call void @llvm.memcpy.p0i8.p0i8.i32(i8* %dest, i8* %src, i32 500, i32 8, i1 false)
+
+  ; CHECK-IOS: memset
+  ; CHECK-DARWIN: memset
+  ; CHECK-EABI: __aeabi_memset8
+  call void @llvm.memset.p0i8.i32(i8* %dest, i8 1, i32 500, i32 8, i1 false)
+
+  ; CHECK-IOS: memset
+  ; CHECK-DARWIN: memset
+  ; CHECK-EABI: __aeabi_memclr8
+  call void @llvm.memset.p0i8.i32(i8* %dest, i8 0, i32 500, i32 8, i1 false)
+
   unreachable
 }
 
@@ -53,17 +104,17 @@ entry:
   call void @llvm.memcpy.p0i8.p0i8.i32(i8* %dest, i8* %1, i32 %n, i32 0, i1 false)
 
   ; CHECK-IOS: mov r0, sp
-  ; CHECK-IOS: mov r1, #0
+  ; CHECK-IOS: mov r1, #1
   ; CHECK-IOS: memset
-  ; CHECK-DARINW: add r0, sp, #4
-  ; CHECK-DARWIN: movs r1, #0
+  ; CHECK-DARWIN: add r0, sp, #4
+  ; CHECK-DARWIN: movs r1, #1
   ; CHECK-DARWIN: memset
   ; CHECK-EABI: add r0, sp, #4
-  ; CHECK-EABI: mov r2, #0
+  ; CHECK-EABI: mov r2, #1
   ; CHECK-EABI: __aeabi_memset
   %arr2 = alloca [9 x i8], align 1
   %2 = bitcast [9 x i8]* %arr2 to i8*
-  call void @llvm.memset.p0i8.i32(i8* %2, i8 0, i32 %n, i32 0, i1 false)
+  call void @llvm.memset.p0i8.i32(i8* %2, i8 1, i32 %n, i32 0, i1 false)
 
   unreachable
 }
@@ -90,15 +141,15 @@ entry:
   call void @llvm.memcpy.p0i8.p0i8.i32(i8* %dest, i8* %1, i32 %n, i32 0, i1 false)
 
   ; CHECK: {{add(.w)? r0, sp, #3}}
-  ; CHECK-IOS: mov r1, #0
+  ; CHECK-IOS: mov r1, #1
   ; CHECK-IOS: memset
-  ; CHECK-DARWIN: movs r1, #0
+  ; CHECK-DARWIN: movs r1, #1
   ; CHECK-DARWIN: memset
-  ; CHECK-EABI: mov r2, #0
+  ; CHECK-EABI: mov r2, #1
   ; CHECK-EABI: __aeabi_memset
   %arr2 = alloca [7 x i8], align 1
   %2 = bitcast [7 x i8]* %arr2 to i8*
-  call void @llvm.memset.p0i8.i32(i8* %2, i8 0, i32 %n, i32 0, i1 false)
+  call void @llvm.memset.p0i8.i32(i8* %2, i8 1, i32 %n, i32 0, i1 false)
 
   unreachable
 }
@@ -125,15 +176,15 @@ entry:
   call void @llvm.memcpy.p0i8.p0i8.i32(i8* %dest, i8* %1, i32 %n, i32 0, i1 false)
 
   ; CHECK: {{add(.w)? r., sp, #(1|5)}}
-  ; CHECK-IOS: mov r1, #0
+  ; CHECK-IOS: mov r1, #1
   ; CHECK-IOS: memset
-  ; CHECK-DARWIN: movs r1, #0
+  ; CHECK-DARWIN: movs r1, #1
   ; CHECK-DARWIN: memset
-  ; CHECK-EABI: mov r2, #0
+  ; CHECK-EABI: mov r2, #1
   ; CHECK-EABI: __aeabi_memset
   %arr2 = alloca [9 x i8], align 1
   %2 = getelementptr inbounds [9 x i8], [9 x i8]* %arr2, i32 0, i32 4
-  call void @llvm.memset.p0i8.i32(i8* %2, i8 0, i32 %n, i32 0, i1 false)
+  call void @llvm.memset.p0i8.i32(i8* %2, i8 1, i32 %n, i32 0, i1 false)
 
   unreachable
 }
@@ -160,15 +211,15 @@ entry:
   call void @llvm.memcpy.p0i8.p0i8.i32(i8* %dest, i8* %1, i32 %n, i32 0, i1 false)
 
   ; CHECK: {{add(.w)? r., sp, #(1|5)}}
-  ; CHECK-IOS: mov r1, #0
+  ; CHECK-IOS: mov r1, #1
   ; CHECK-IOS: memset
-  ; CHECK-DARWIN: movs r1, #0
+  ; CHECK-DARWIN: movs r1, #1
   ; CHECK-DARWIN: memset
-  ; CHECK-EABI: mov r2, #0
+  ; CHECK-EABI: mov r2, #1
   ; CHECK-EABI: __aeabi_memset
   %arr2 = alloca [13 x i8], align 1
   %2 = getelementptr inbounds [13 x i8], [13 x i8]* %arr2, i32 0, i32 1
-  call void @llvm.memset.p0i8.i32(i8* %2, i8 0, i32 %n, i32 0, i1 false)
+  call void @llvm.memset.p0i8.i32(i8* %2, i8 1, i32 %n, i32 0, i1 false)
 
   unreachable
 }
@@ -195,15 +246,15 @@ entry:
   call void @llvm.memcpy.p0i8.p0i8.i32(i8* %dest, i8* %1, i32 %n, i32 0, i1 false)
 
   ; CHECK: {{add(.w)? r., sp, #(1|5)}}
-  ; CHECK-IOS: mov r1, #0
+  ; CHECK-IOS: mov r1, #1
   ; CHECK-IOS: memset
-  ; CHECK-DARWIN: movs r1, #0
+  ; CHECK-DARWIN: movs r1, #1
   ; CHECK-DARWIN: memset
-  ; CHECK-EABI: mov r2, #0
+  ; CHECK-EABI: mov r2, #1
   ; CHECK-EABI: __aeabi_memset
   %arr2 = alloca [13 x i8], align 1
   %2 = getelementptr inbounds [13 x i8], [13 x i8]* %arr2, i32 0, i32 %i
-  call void @llvm.memset.p0i8.i32(i8* %2, i8 0, i32 %n, i32 0, i1 false)
+  call void @llvm.memset.p0i8.i32(i8* %2, i8 1, i32 %n, i32 0, i1 false)
 
   unreachable
 }
@@ -230,15 +281,15 @@ entry:
   call void @llvm.memcpy.p0i8.p0i8.i32(i8* %dest, i8* %1, i32 %n, i32 0, i1 false)
 
   ; CHECK: {{add(.w)? r., sp, #(1|5)}}
-  ; CHECK-IOS: mov r1, #0
+  ; CHECK-IOS: mov r1, #1
   ; CHECK-IOS: memset
-  ; CHECK-DARWIN: movs r1, #0
+  ; CHECK-DARWIN: movs r1, #1
   ; CHECK-DARWIN: memset
-  ; CHECK-EABI: mov r2, #0
+  ; CHECK-EABI: mov r2, #1
   ; CHECK-EABI: __aeabi_memset
   %arr2 = alloca [13 x i8], align 1
   %2 = getelementptr [13 x i8], [13 x i8]* %arr2, i32 0, i32 4
-  call void @llvm.memset.p0i8.i32(i8* %2, i8 0, i32 %n, i32 0, i1 false)
+  call void @llvm.memset.p0i8.i32(i8* %2, i8 1, i32 %n, i32 0, i1 false)
 
   unreachable
 }
@@ -265,15 +316,15 @@ entry:
   call void @llvm.memcpy.p0i8.p0i8.i32(i8* %dest, i8* %1, i32 %n, i32 0, i1 false)
 
   ; CHECK: {{add(.w)? r., sp, #(1|5)}}
-  ; CHECK-IOS: mov r1, #0
+  ; CHECK-IOS: mov r1, #1
   ; CHECK-IOS: memset
-  ; CHECK-DARWIN: movs r1, #0
+  ; CHECK-DARWIN: movs r1, #1
   ; CHECK-DARWIN: memset
-  ; CHECK-EABI: mov r2, #0
+  ; CHECK-EABI: mov r2, #1
   ; CHECK-EABI: __aeabi_memset
   %arr2 = alloca [13 x i8], align 1
   %2 = getelementptr inbounds [13 x i8], [13 x i8]* %arr2, i32 0, i32 16
-  call void @llvm.memset.p0i8.i32(i8* %2, i8 0, i32 %n, i32 0, i1 false)
+  call void @llvm.memset.p0i8.i32(i8* %2, i8 1, i32 %n, i32 0, i1 false)
 
   unreachable
 }
diff --git a/test/CodeGen/ARM/memset-inline.ll b/test/CodeGen/ARM/memset-inline.ll
index 191db1e20a2..f6f8d562350 100644
--- a/test/CodeGen/ARM/memset-inline.ll
+++ b/test/CodeGen/ARM/memset-inline.ll
@@ -4,8 +4,7 @@ define void @t1(i8* nocapture %c) nounwind optsize {
 entry:
 ; CHECK-LABEL: t1:
 ; CHECK: movs r1, #0
-; CHECK: str r1, [r0]
-; CHECK: str r1, [r0, #4]
+; CHECK: strd r1, r1, [r0]
 ; CHECK: str r1, [r0, #8]
   call void @llvm.memset.p0i8.i64(i8* %c, i8 0, i64 12, i32 8, i1 false)
   ret void
diff --git a/test/CodeGen/ARM/named-reg-alloc.ll b/test/CodeGen/ARM/named-reg-alloc.ll
index 380cf39734f..d41fa64882c 100644
--- a/test/CodeGen/ARM/named-reg-alloc.ll
+++ b/test/CodeGen/ARM/named-reg-alloc.ll
@@ -4,7 +4,7 @@
 define i32 @get_stack() nounwind {
 entry:
 ; FIXME: Include an allocatable-specific error message
-; CHECK: Invalid register name global variable
+; CHECK: Invalid register name "r5".
 	%sp = call i32 @llvm.read_register.i32(metadata !0)
   ret i32 %sp
 }
diff --git a/test/CodeGen/ARM/named-reg-notareg.ll b/test/CodeGen/ARM/named-reg-notareg.ll
index 3ac03f4fdaa..45cb38f30f3 100644
--- a/test/CodeGen/ARM/named-reg-notareg.ll
+++ b/test/CodeGen/ARM/named-reg-notareg.ll
@@ -3,7 +3,7 @@
 
 define i32 @get_stack() nounwind {
 entry:
-; CHECK: Invalid register name global variable
+; CHECK: Invalid register name "notareg".
 	%sp = call i32 @llvm.read_register.i32(metadata !0)
   ret i32 %sp
 }
diff --git a/test/CodeGen/ARM/neon_vshl_minint.ll b/test/CodeGen/ARM/neon_vshl_minint.ll
new file mode 100644
index 00000000000..769eff845fd
--- /dev/null
+++ b/test/CodeGen/ARM/neon_vshl_minint.ll
@@ -0,0 +1,13 @@
+; RUN: llc < %s -mtriple=arm-none-eabi -mcpu=cortex-a8 2>&1 | FileCheck %s
+; RUN: llc < %s -mtriple=thumb-none-eabi -mcpu=cortex-a8 2>&1 | FileCheck %s
+
+define <1 x i64> @vshl_minint() #0 {
+  entry:
+    ; CHECK-LABEL: vshl_minint
+    ; CHECK: vldr
+    ; CHECK: vshl.u64
+    %vshl.i = tail call <1 x i64> @llvm.arm.neon.vshiftu.v1i64(<1 x i64> undef, <1 x i64> <i64 -9223372036854775808>)
+    ret <1 x i64> %vshl.i
+}
+
+declare <1 x i64> @llvm.arm.neon.vshiftu.v1i64(<1 x i64>, <1 x i64>)
diff --git a/test/CodeGen/ARM/nest-register.ll b/test/CodeGen/ARM/nest-register.ll
new file mode 100644
index 00000000000..6b8c3dc47db
--- /dev/null
+++ b/test/CodeGen/ARM/nest-register.ll
@@ -0,0 +1,21 @@
+; RUN: llc -mtriple=arm-eabi %s -o - | FileCheck %s
+
+; Tests that the 'nest' parameter attribute causes the relevant parameter to be
+; passed in the right register.
+
+define i8* @nest_receiver(i8* nest %arg) nounwind {
+; CHECK-LABEL: nest_receiver:
+; CHECK: @ BB#0:
+; CHECK-NEXT: mov r0, r12
+; CHECK-NEXT: mov pc, lr
+        ret i8* %arg
+}
+
+define i8* @nest_caller(i8* %arg) nounwind {
+; CHECK-LABEL: nest_caller:
+; CHECK: mov r12, r0
+; CHECK-NEXT: bl nest_receiver
+; CHECK: mov pc, lr
+        %result = call i8* @nest_receiver(i8* nest %arg)
+        ret i8* %result
+}
diff --git a/test/CodeGen/ARM/noopt-dmb-v7.ll b/test/CodeGen/ARM/noopt-dmb-v7.ll
new file mode 100644
index 00000000000..56a29c8a17e
--- /dev/null
+++ b/test/CodeGen/ARM/noopt-dmb-v7.ll
@@ -0,0 +1,15 @@
+; Ensure that adjacent duplicated barriers are not removed at -O0.
+; RUN: llc -O0 < %s -mtriple=armv7 -mattr=+db | FileCheck %s
+
+define i32 @t1() {
+entry:
+  fence seq_cst
+  fence seq_cst
+  fence seq_cst
+  ret i32 0
+}
+
+; CHECK: @ BB#0: @ %entry
+; CHECK-NEXT: dmb ish
+; CHECK-NEXT: dmb ish
+; CHECK-NEXT: dmb ish
diff --git a/test/CodeGen/ARM/optimize-dmbs-v7.ll b/test/CodeGen/ARM/optimize-dmbs-v7.ll
index 64f5e202d36..34a55aa718a 100644
--- a/test/CodeGen/ARM/optimize-dmbs-v7.ll
+++ b/test/CodeGen/ARM/optimize-dmbs-v7.ll
@@ -1,4 +1,4 @@
-; RUN: llc < %s -mtriple=armv7 -mattr=+db | FileCheck %s
+; RUN: llc -O1 < %s -mtriple=armv7 -mattr=+db | FileCheck %s
 
 @x1 = global i32 0, align 4
 @x2 = global i32 0, align 4
diff --git a/test/CodeGen/ARM/sched-it-debug-nodes.ll b/test/CodeGen/ARM/sched-it-debug-nodes.ll
new file mode 100644
index 00000000000..7906e7c7581
--- /dev/null
+++ b/test/CodeGen/ARM/sched-it-debug-nodes.ll
@@ -0,0 +1,88 @@
+; RUN: llc -mtriple thumbv7 -print-before=post-RA-sched -print-after=post-RA-sched %s -o /dev/null 2>&1 | FileCheck %s
+
+; ModuleID = '<stdin>'
+target datalayout = "e-m:e-p:32:32-i64:64-v128:64:128-a:0:32-n32-S64"
+target triple = "thumbv7"
+
+%struct.s = type opaque
+
+; Function Attrs: nounwind
+define arm_aapcscc i32 @f(%struct.s* %s, i32 %u, i8* %b, i32 %n) #0 {
+entry:
+  tail call void @llvm.dbg.value(metadata %struct.s* %s, i64 0, metadata !19, metadata !28), !dbg !29
+  tail call void @llvm.dbg.value(metadata i32 %u, i64 0, metadata !20, metadata !28), !dbg !29
+  tail call void @llvm.dbg.value(metadata i8* %b, i64 0, metadata !21, metadata !28), !dbg !29
+  tail call void @llvm.dbg.value(metadata i32 %n, i64 0, metadata !22, metadata !28), !dbg !29
+  %cmp = icmp ult i32 %n, 4, !dbg !30
+  br i1 %cmp, label %return, label %if.end, !dbg !32
+
+if.end:                                           ; preds = %entry
+  tail call arm_aapcscc void @g(%struct.s* %s, i8* %b, i32 %n) #3, !dbg !33
+  br label %return, !dbg !34
+
+return:                                           ; preds = %entry, %if.end
+  %retval.0 = phi i32 [ 0, %if.end ], [ -1, %entry ]
+  ret i32 %retval.0, !dbg !35
+}
+
+; NOTE: This is checking that the register in the DEBUG_VALUE node is not
+; accidentally being marked as KILL.  The DBG_VALUE node gets introduced in
+; If-Conversion, and gets bundled into the IT block.  The Post RA Scheduler
+; attempts to schedule the Machine Instr, and tries to tag the register in the
+; debug value as KILL'ed, resulting in a DEBUG_VALUE node changing codegen!  (or
+; hopefully, triggering an assert).
+
+; CHECK: BUNDLE %ITSTATE<imp-def,dead>
+; CHECK:  * DBG_VALUE %R1, %noreg, !"u"
+; CHECK-NOT:  * DBG_VALUE %R1<kill>, %noreg, !"u"
+
+declare arm_aapcscc void @g(%struct.s*, i8*, i32) #1
+
+; Function Attrs: nounwind readnone
+declare void @llvm.dbg.value(metadata, i64, metadata, metadata) #2
+
+attributes #0 = { nounwind "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "no-realign-stack" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #1 = { "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "no-realign-stack" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #2 = { nounwind readnone }
+attributes #3 = { nounwind }
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!23, !24, !25, !26}
+!llvm.ident = !{!27}
+
+!0 = !DICompileUnit(language: DW_LANG_C99, file: !1, producer: "clang version 3.7.0  (llvm/trunk 237059)", isOptimized: true, runtimeVersion: 0, emissionKind: 1, enums: !2, retainedTypes: !2, subprograms: !3, globals: !2, imports: !2)
+!1 = !DIFile(filename: "<stdin>", directory: "/Users/compnerd/Source/llvm")
+!2 = !{}
+!3 = !{!4}
+!4 = !DISubprogram(name: "f", scope: !5, file: !5, line: 9, type: !6, isLocal: false, isDefinition: true, scopeLine: 9, flags: DIFlagPrototyped, isOptimized: true, function: i32 (%struct.s*, i32, i8*, i32)* @f, variables: !18)
+!5 = !DIFile(filename: "<stdin>", directory: "/Users/compnerd/Source/llvm")
+!6 = !DISubroutineType(types: !7)
+!7 = !{!8, !9, !12, !13, !17}
+!8 = !DIBasicType(name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
+!9 = !DIDerivedType(tag: DW_TAG_pointer_type, baseType: !10, size: 32, align: 32)
+!10 = !DIDerivedType(tag: DW_TAG_typedef, name: "s", file: !5, line: 5, baseType: !11)
+!11 = !DICompositeType(tag: DW_TAG_structure_type, name: "s", file: !5, line: 5, flags: DIFlagFwdDecl)
+!12 = !DIBasicType(name: "unsigned int", size: 32, align: 32, encoding: DW_ATE_unsigned)
+!13 = !DIDerivedType(tag: DW_TAG_pointer_type, baseType: !14, size: 32, align: 32)
+!14 = !DIDerivedType(tag: DW_TAG_const_type, baseType: !15)
+!15 = !DIDerivedType(tag: DW_TAG_typedef, name: "uint8_t", file: !5, line: 2, baseType: !16)
+!16 = !DIBasicType(name: "unsigned char", size: 8, align: 8, encoding: DW_ATE_unsigned_char)
+!17 = !DIDerivedType(tag: DW_TAG_typedef, name: "size_t", file: !5, line: 3, baseType: !12)
+!18 = !{!19, !20, !21, !22}
+!19 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "s", arg: 1, scope: !4, file: !5, line: 9, type: !9)
+!20 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "u", arg: 2, scope: !4, file: !5, line: 9, type: !12)
+!21 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "b", arg: 3, scope: !4, file: !5, line: 9, type: !13)
+!22 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "n", arg: 4, scope: !4, file: !5, line: 9, type: !17)
+!23 = !{i32 2, !"Dwarf Version", i32 4}
+!24 = !{i32 2, !"Debug Info Version", i32 3}
+!25 = !{i32 1, !"wchar_size", i32 4}
+!26 = !{i32 1, !"min_enum_size", i32 4}
+!27 = !{!"clang version 3.7.0  (llvm/trunk 237059)"}
+!28 = !DIExpression()
+!29 = !DILocation(line: 9, scope: !4)
+!30 = !DILocation(line: 10, scope: !31)
+!31 = distinct !DILexicalBlock(scope: !4, file: !5, line: 10)
+!32 = !DILocation(line: 10, scope: !4)
+!33 = !DILocation(line: 13, scope: !4)
+!34 = !DILocation(line: 14, scope: !4)
+!35 = !DILocation(line: 15, scope: !4)
diff --git a/test/CodeGen/ARM/setjmp_longjmp.ll b/test/CodeGen/ARM/setjmp_longjmp.ll
new file mode 100644
index 00000000000..7100175a97a
--- /dev/null
+++ b/test/CodeGen/ARM/setjmp_longjmp.ll
@@ -0,0 +1,113 @@
+; RUN: llc %s -o - | FileCheck %s
+target triple = "armv7-apple-ios"
+
+declare i32 @llvm.eh.sjlj.setjmp(i8*)
+declare void @llvm.eh.sjlj.longjmp(i8*)
+@g = external global i32
+
+declare void @may_throw()
+declare i32 @__gxx_personality_sj0(...)
+declare i8* @__cxa_begin_catch(i8*)
+declare void @__cxa_end_catch()
+declare i32 @llvm.eh.typeid.for(i8*)
+declare i8* @llvm.frameaddress(i32)
+declare i8* @llvm.stacksave()
+@_ZTIPKc = external constant i8*
+
+; CHECK-LABEL: foobar
+;
+; setjmp sequence:
+; CHECK: add [[PCREG:r[0-9]+]], pc, #8
+; CHECK-NEXT: str [[PCREG]], {{\[}}[[BUFREG:r[0-9]+]], #4]
+; CHECK-NEXT: mov r0, #0
+; CHECK-NEXT: add pc, pc, #0
+; CHECK-NEXT: mov r0, #1
+;
+; longjmp sequence:
+; CHECK: ldr sp, [{{\s*}}[[BUFREG:r[0-9]+]], #8]
+; CHECK-NEXT: ldr [[DESTREG:r[0-9]+]], {{\[}}[[BUFREG]], #4]
+; CHECK-NEXT: ldr r7, {{\[}}[[BUFREG]]{{\]}}
+; CHECK-NEXT: bx [[DESTREG]]
+define void @foobar() {
+entry:
+  %buf = alloca [5 x i8*], align 4
+  %arraydecay = getelementptr inbounds [5 x i8*], [5 x i8*]* %buf, i32 0, i32 0
+  %bufptr = bitcast i8** %arraydecay to i8*
+  ; Note: This is simplified, in reality you have to store the framepointer +
+  ; stackpointer in the buffer as well for this to be legal!
+  %setjmpres = call i32 @llvm.eh.sjlj.setjmp(i8* %bufptr)
+  %tobool = icmp ne i32 %setjmpres, 0
+  br i1 %tobool, label %if.then, label %if.else
+
+if.then:
+  store volatile i32 1, i32* @g, align 4
+  br label %if.end
+
+if.else:
+  store volatile i32 0, i32* @g, align 4
+  call void @llvm.eh.sjlj.longjmp(i8* %bufptr)
+  unreachable
+
+if.end:
+  ret void
+}
+
+; CHECK-LABEL: combine_sjlj_eh_and_setjmp_longjmp
+; Check that we can mix sjlj exception handling with __builtin_setjmp
+; and __builtin_longjmp.
+;
+; setjmp sequence:
+; CHECK: add [[PCREG:r[0-9]+]], pc, #8
+; CHECK-NEXT: str [[PCREG]], {{\[}}[[BUFREG:r[0-9]+]], #4]
+; CHECK-NEXT: mov r0, #0
+; CHECK-NEXT: add pc, pc, #0
+; CHECK-NEXT: mov r0, #1
+;
+; longjmp sequence:
+; CHECK: ldr sp, [{{\s*}}[[BUFREG:r[0-9]+]], #8]
+; CHECK-NEXT: ldr [[DESTREG:r[0-9]+]], {{\[}}[[BUFREG]], #4]
+; CHECK-NEXT: ldr r7, {{\[}}[[BUFREG]]{{\]}}
+; CHECK-NEXT: bx [[DESTREG]]
+define void @combine_sjlj_eh_and_setjmp_longjmp() personality i8* bitcast (i32 (...)* @__gxx_personality_sj0 to i8*) {
+entry:
+  %buf = alloca [5 x i8*], align 4
+  invoke void @may_throw() to label %try.cont unwind label %lpad
+
+lpad:
+  %0 = landingpad { i8*, i32 } catch i8* bitcast (i8** @_ZTIPKc to i8*)
+  %1 = extractvalue { i8*, i32 } %0, 1
+  %2 = tail call i32 @llvm.eh.typeid.for(i8* bitcast (i8** @_ZTIPKc to i8*)) #3
+  %matches = icmp eq i32 %1, %2
+  br i1 %matches, label %catch, label %eh.resume
+
+catch:
+  %3 = extractvalue { i8*, i32 } %0, 0
+  %4 = tail call i8* @__cxa_begin_catch(i8* %3) #3
+  store volatile i32 0, i32* @g, align 4
+  %5 = bitcast [5 x i8*]* %buf to i8*
+  %arraydecay = getelementptr inbounds [5 x i8*], [5 x i8*]* %buf, i64 0, i64 0
+  %6 = tail call i8* @llvm.frameaddress(i32 0)
+  store i8* %6, i8** %arraydecay, align 16
+  %7 = tail call i8* @llvm.stacksave()
+  %8 = getelementptr [5 x i8*], [5 x i8*]* %buf, i64 0, i64 2
+  store i8* %7, i8** %8, align 16
+  %9 = call i32 @llvm.eh.sjlj.setjmp(i8* %5)
+  %tobool = icmp eq i32 %9, 0
+  br i1 %tobool, label %if.else, label %if.then
+
+if.then:
+  store volatile i32 2, i32* @g, align 4
+  call void @__cxa_end_catch() #3
+  br label %try.cont
+
+if.else:
+  store volatile i32 1, i32* @g, align 4
+  call void @llvm.eh.sjlj.longjmp(i8* %5)
+  unreachable
+
+eh.resume:
+  resume { i8*, i32 } %0
+
+try.cont:
+  ret void
+}
diff --git a/test/CodeGen/ARM/sjlj-prepare-critical-edge.ll b/test/CodeGen/ARM/sjlj-prepare-critical-edge.ll
index dd0a0a0803d..c7f47b0962d 100644
--- a/test/CodeGen/ARM/sjlj-prepare-critical-edge.ll
+++ b/test/CodeGen/ARM/sjlj-prepare-critical-edge.ll
@@ -6,7 +6,7 @@
 
 declare void @bar(%struct.__CFString*, %struct.__CFString*)
 
-define noalias i8* @foo(i8* nocapture %inRefURL) noreturn ssp {
+define noalias i8* @foo(i8* nocapture %inRefURL) noreturn ssp personality i8* bitcast (i32 (...)* @__gxx_personality_sj0 to i8*) {
 entry:
   %call = tail call %struct.__CFString* @bar3()
   %call2 = invoke i8* @bar2()
@@ -17,14 +17,14 @@ for.cond:                                         ; preds = %entry, %for.cond
           to label %for.cond unwind label %lpad5
 
 lpad:                                             ; preds = %entry
-  %0 = landingpad { i8*, i32 } personality i8* bitcast (i32 (...)* @__gxx_personality_sj0 to i8*)
+  %0 = landingpad { i8*, i32 }
           cleanup
   %1 = extractvalue { i8*, i32 } %0, 0
   %2 = extractvalue { i8*, i32 } %0, 1
   br label %ehcleanup
 
 lpad5:                                            ; preds = %for.cond
-  %3 = landingpad { i8*, i32 } personality i8* bitcast (i32 (...)* @__gxx_personality_sj0 to i8*)
+  %3 = landingpad { i8*, i32 }
           cleanup
   %4 = extractvalue { i8*, i32 } %3, 0
   %5 = extractvalue { i8*, i32 } %3, 1
@@ -32,7 +32,7 @@ lpad5:                                            ; preds = %for.cond
           to label %ehcleanup unwind label %terminate.lpad.i.i16
 
 terminate.lpad.i.i16:                             ; preds = %lpad5
-  %6 = landingpad { i8*, i32 } personality i8* bitcast (i32 (...)* @__gxx_personality_sj0 to i8*)
+  %6 = landingpad { i8*, i32 }
           catch i8* null
   tail call void @terminatev() noreturn nounwind
   unreachable
@@ -45,7 +45,7 @@ ehcleanup:                                        ; preds = %lpad5, %lpad
           to label %_ZN5SmartIPK10__CFStringED1Ev.exit unwind label %terminate.lpad.i.i
 
 terminate.lpad.i.i:                               ; preds = %ehcleanup
-  %8 = landingpad { i8*, i32 } personality i8* bitcast (i32 (...)* @__gxx_personality_sj0 to i8*)
+  %8 = landingpad { i8*, i32 }
           catch i8* null
   tail call void @terminatev() noreturn nounwind
   unreachable
@@ -79,7 +79,7 @@ declare void @terminatev()
 ; CHECK: str r0, [sp, [[OFFSET:#[0-9]+]]]
 ; CHECK: ldr [[R0:r[0-9]+]], [sp, [[OFFSET]]]
 ; CHECK: {{.*}}@ %do.body.i.i.i
-; CHECK: cmp [[R0]], #0
+; CHECK: cbz [[R0]]
 
 %"class.std::__1::basic_string" = type { %"class.std::__1::__compressed_pair" }
 %"class.std::__1::__compressed_pair" = type { %"class.std::__1::__libcpp_compressed_pair_imp" }
@@ -90,7 +90,7 @@ declare void @terminatev()
 
 @.str = private unnamed_addr constant [12 x i8] c"some_string\00", align 1
 
-define void @_Z4foo1c(i8 signext %a) {
+define void @_Z4foo1c(i8 signext %a) personality i8* bitcast (i32 (...)* @__gxx_personality_sj0 to i8*) {
 entry:
   %s1 = alloca %"class.std::__1::basic_string", align 4
   call void @_ZNSt3__112basic_stringIcNS_11char_traitsIcEENS_9allocatorIcEEE6__initEPKcm(%"class.std::__1::basic_string"* %s1, i8* getelementptr inbounds ([12 x i8], [12 x i8]* @.str, i32 0, i32 0), i32 11)
@@ -131,14 +131,14 @@ invoke.cont6:                                     ; preds = %_ZNSt3__113__vector
   ret void
 
 lpad.body:                                        ; preds = %entry
-  %2 = landingpad { i8*, i32 } personality i8* bitcast (i32 (...)* @__gxx_personality_sj0 to i8*)
+  %2 = landingpad { i8*, i32 }
           cleanup
   %3 = extractvalue { i8*, i32 } %2, 0
   %4 = extractvalue { i8*, i32 } %2, 1
   br label %ehcleanup
 
 lpad2:                                            ; preds = %invoke.cont
-  %5 = landingpad { i8*, i32 } personality i8* bitcast (i32 (...)* @__gxx_personality_sj0 to i8*)
+  %5 = landingpad { i8*, i32 }
           cleanup
   %6 = extractvalue { i8*, i32 } %5, 0
   %7 = extractvalue { i8*, i32 } %5, 1
@@ -161,7 +161,7 @@ eh.resume:                                        ; preds = %ehcleanup
   resume { i8*, i32 } %lpad.val13
 
 terminate.lpad:                                   ; preds = %ehcleanup
-  %8 = landingpad { i8*, i32 } personality i8* bitcast (i32 (...)* @__gxx_personality_sj0 to i8*)
+  %8 = landingpad { i8*, i32 }
           catch i8* null
   %9 = extractvalue { i8*, i32 } %8, 0
   call void @__clang_call_terminate(i8* %9)
diff --git a/test/CodeGen/ARM/sjljehprepare-lower-empty-struct.ll b/test/CodeGen/ARM/sjljehprepare-lower-empty-struct.ll
index 3cf2a08fe35..5d015738623 100644
--- a/test/CodeGen/ARM/sjljehprepare-lower-empty-struct.ll
+++ b/test/CodeGen/ARM/sjljehprepare-lower-empty-struct.ll
@@ -10,7 +10,7 @@
 ; __Unwind_SjLj_Register and actual @bar invocation
 
 
-define i8* @foo(i8 %a, {} %c) {
+define i8* @foo(i8 %a, {} %c) personality i8* bitcast (i32 (...)* @baz to i8*) {
 entry:
 ; CHECK: bl __Unwind_SjLj_Register
 ; CHECK-NEXT: {{[A-Z][a-zA-Z0-9]*}}:
@@ -22,7 +22,7 @@ unreachable:
   unreachable
 
 handler:
-  %tmp = landingpad { i8*, i32 } personality i8* bitcast (i32 (...)* @baz to i8*)
+  %tmp = landingpad { i8*, i32 }
   cleanup
   resume { i8*, i32 } undef
 }
diff --git a/test/CodeGen/ARM/special-reg-acore.ll b/test/CodeGen/ARM/special-reg-acore.ll
new file mode 100644
index 00000000000..3d65ff44bfb
--- /dev/null
+++ b/test/CodeGen/ARM/special-reg-acore.ll
@@ -0,0 +1,78 @@
+; RUN: llc < %s -mtriple=arm-none-eabi -mcpu=cortex-a8 2>&1 | FileCheck %s --check-prefix=ACORE
+; RUN: not llc < %s -mtriple=thumb-none-eabi -mcpu=cortex-m4 2>&1 | FileCheck %s --check-prefix=MCORE
+
+; MCORE: LLVM ERROR: Invalid register name "cpsr".
+
+define i32 @read_cpsr() nounwind {
+  ; ACORE-LABEL: read_cpsr:
+  ; ACORE: mrs r0, apsr
+  %reg = call i32 @llvm.read_register.i32(metadata !1)
+  ret i32 %reg
+}
+
+define i32 @read_aclass_registers() nounwind {
+entry:
+  ; ACORE-LABEL: read_aclass_registers:
+  ; ACORE: mrs r0, apsr
+  ; ACORE: mrs r1, spsr
+
+  %0 = call i32 @llvm.read_register.i32(metadata !0)
+  %1 = call i32 @llvm.read_register.i32(metadata !1)
+  %add1 = add i32 %1, %0
+  %2 = call i32 @llvm.read_register.i32(metadata !2)
+  %add2 = add i32 %add1, %2
+  ret i32 %add2
+}
+
+define void @write_aclass_registers(i32 %x) nounwind {
+entry:
+  ; ACORE-LABEL: write_aclass_registers:
+  ; ACORE:   msr APSR_nzcvq, r0
+  ; ACORE:   msr APSR_g, r0
+  ; ACORE:   msr APSR_nzcvqg, r0
+  ; ACORE:   msr CPSR_c, r0
+  ; ACORE:   msr CPSR_x, r0
+  ; ACORE:   msr APSR_g, r0
+  ; ACORE:   msr APSR_nzcvq, r0
+  ; ACORE:   msr CPSR_fsxc, r0
+  ; ACORE:   msr SPSR_c, r0
+  ; ACORE:   msr SPSR_x, r0
+  ; ACORE:   msr SPSR_s, r0
+  ; ACORE:   msr SPSR_f, r0
+  ; ACORE:   msr SPSR_fsxc, r0
+
+  call void @llvm.write_register.i32(metadata !3, i32 %x)
+  call void @llvm.write_register.i32(metadata !4, i32 %x)
+  call void @llvm.write_register.i32(metadata !5, i32 %x)
+  call void @llvm.write_register.i32(metadata !6, i32 %x)
+  call void @llvm.write_register.i32(metadata !7, i32 %x)
+  call void @llvm.write_register.i32(metadata !8, i32 %x)
+  call void @llvm.write_register.i32(metadata !9, i32 %x)
+  call void @llvm.write_register.i32(metadata !10, i32 %x)
+  call void @llvm.write_register.i32(metadata !11, i32 %x)
+  call void @llvm.write_register.i32(metadata !12, i32 %x)
+  call void @llvm.write_register.i32(metadata !13, i32 %x)
+  call void @llvm.write_register.i32(metadata !14, i32 %x)
+  call void @llvm.write_register.i32(metadata !15, i32 %x)
+  ret void
+}
+
+declare i32 @llvm.read_register.i32(metadata) nounwind
+declare void @llvm.write_register.i32(metadata, i32) nounwind
+
+!0 = !{!"apsr"}
+!1 = !{!"cpsr"}
+!2 = !{!"spsr"}
+!3 = !{!"apsr_nzcvq"}
+!4 = !{!"apsr_g"}
+!5 = !{!"apsr_nzcvqg"}
+!6 = !{!"cpsr_c"}
+!7 = !{!"cpsr_x"}
+!8 = !{!"cpsr_s"}
+!9 = !{!"cpsr_f"}
+!10 = !{!"cpsr_cxsf"}
+!11 = !{!"spsr_c"}
+!12 = !{!"spsr_x"}
+!13 = !{!"spsr_s"}
+!14 = !{!"spsr_f"}
+!15 = !{!"spsr_cxsf"}
diff --git a/test/CodeGen/ARM/special-reg-mcore.ll b/test/CodeGen/ARM/special-reg-mcore.ll
new file mode 100644
index 00000000000..686da0f6b83
--- /dev/null
+++ b/test/CodeGen/ARM/special-reg-mcore.ll
@@ -0,0 +1,143 @@
+; RUN: llc < %s -mtriple=thumb-none-eabi -mcpu=cortex-m4 2>&1 | FileCheck %s --check-prefix=MCORE
+; RUN: not llc < %s -mtriple=thumb-none-eabi -mcpu=cortex-m3 2>&1 | FileCheck %s --check-prefix=M3CORE
+; RUN: not llc < %s -mtriple=arm-none-eabi -mcpu=cortex-a8 2>&1 | FileCheck %s --check-prefix=ACORE
+
+; ACORE: LLVM ERROR: Invalid register name "control".
+; M3CORE: LLVM ERROR: Invalid register name "control".
+
+define i32 @read_mclass_registers() nounwind {
+entry:
+  ; MCORE-LABEL: read_mclass_registers:
+  ; MCORE:   mrs r0, apsr
+  ; MCORE:   mrs r1, iapsr
+  ; MCORE:   mrs r1, eapsr
+  ; MCORE:   mrs r1, xpsr
+  ; MCORE:   mrs r1, ipsr
+  ; MCORE:   mrs r1, epsr
+  ; MCORE:   mrs r1, iepsr
+  ; MCORE:   mrs r1, msp
+  ; MCORE:   mrs r1, psp
+  ; MCORE:   mrs r1, primask
+  ; MCORE:   mrs r1, basepri
+  ; MCORE:   mrs r1, basepri_max
+  ; MCORE:   mrs r1, faultmask
+  ; MCORE:   mrs r1, control
+
+  %0 = call i32 @llvm.read_register.i32(metadata !0)
+  %1 = call i32 @llvm.read_register.i32(metadata !4)
+  %add1 = add i32 %1, %0
+  %2 = call i32 @llvm.read_register.i32(metadata !8)
+  %add2 = add i32 %add1, %2
+  %3 = call i32 @llvm.read_register.i32(metadata !12)
+  %add3 = add i32 %add2, %3
+  %4 = call i32 @llvm.read_register.i32(metadata !16)
+  %add4 = add i32 %add3, %4
+  %5 = call i32 @llvm.read_register.i32(metadata !17)
+  %add5 = add i32 %add4, %5
+  %6 = call i32 @llvm.read_register.i32(metadata !18)
+  %add6 = add i32 %add5, %6
+  %7 = call i32 @llvm.read_register.i32(metadata !19)
+  %add7 = add i32 %add6, %7
+  %8 = call i32 @llvm.read_register.i32(metadata !20)
+  %add8 = add i32 %add7, %8
+  %9 = call i32 @llvm.read_register.i32(metadata !21)
+  %add9 = add i32 %add8, %9
+  %10 = call i32 @llvm.read_register.i32(metadata !22)
+  %add10 = add i32 %add9, %10
+  %11 = call i32 @llvm.read_register.i32(metadata !23)
+  %add11 = add i32 %add10, %11
+  %12 = call i32 @llvm.read_register.i32(metadata !24)
+  %add12 = add i32 %add11, %12
+  %13 = call i32 @llvm.read_register.i32(metadata !25)
+  %add13 = add i32 %add12, %13
+  ret i32 %add13
+}
+
+define void @write_mclass_registers(i32 %x) nounwind {
+entry:
+  ; MCORE-LABEL: write_mclass_registers:
+  ; MCORE:   msr apsr_nzcvqg, r0
+  ; MCORE:   msr apsr_nzcvq, r0
+  ; MCORE:   msr apsr_g, r0
+  ; MCORE:   msr apsr_nzcvqg, r0
+  ; MCORE:   msr iapsr_nzcvqg, r0
+  ; MCORE:   msr iapsr_nzcvq, r0
+  ; MCORE:   msr iapsr_g, r0
+  ; MCORE:   msr iapsr_nzcvqg, r0
+  ; MCORE:   msr eapsr_nzcvqg, r0
+  ; MCORE:   msr eapsr_nzcvq, r0
+  ; MCORE:   msr eapsr_g, r0
+  ; MCORE:   msr eapsr_nzcvqg, r0
+  ; MCORE:   msr xpsr_nzcvqg, r0
+  ; MCORE:   msr xpsr_nzcvq, r0
+  ; MCORE:   msr xpsr_g, r0
+  ; MCORE:   msr xpsr_nzcvqg, r0
+  ; MCORE:   msr ipsr, r0
+  ; MCORE:   msr epsr, r0
+  ; MCORE:   msr iepsr, r0
+  ; MCORE:   msr msp, r0
+  ; MCORE:   msr psp, r0
+  ; MCORE:   msr primask, r0
+  ; MCORE:   msr basepri, r0
+  ; MCORE:   msr basepri_max, r0
+  ; MCORE:   msr faultmask, r0
+  ; MCORE:   msr control, r0
+
+  call void @llvm.write_register.i32(metadata !0, i32 %x)
+  call void @llvm.write_register.i32(metadata !1, i32 %x)
+  call void @llvm.write_register.i32(metadata !2, i32 %x)
+  call void @llvm.write_register.i32(metadata !3, i32 %x)
+  call void @llvm.write_register.i32(metadata !4, i32 %x)
+  call void @llvm.write_register.i32(metadata !5, i32 %x)
+  call void @llvm.write_register.i32(metadata !6, i32 %x)
+  call void @llvm.write_register.i32(metadata !7, i32 %x)
+  call void @llvm.write_register.i32(metadata !8, i32 %x)
+  call void @llvm.write_register.i32(metadata !9, i32 %x)
+  call void @llvm.write_register.i32(metadata !10, i32 %x)
+  call void @llvm.write_register.i32(metadata !11, i32 %x)
+  call void @llvm.write_register.i32(metadata !12, i32 %x)
+  call void @llvm.write_register.i32(metadata !13, i32 %x)
+  call void @llvm.write_register.i32(metadata !14, i32 %x)
+  call void @llvm.write_register.i32(metadata !15, i32 %x)
+  call void @llvm.write_register.i32(metadata !16, i32 %x)
+  call void @llvm.write_register.i32(metadata !17, i32 %x)
+  call void @llvm.write_register.i32(metadata !18, i32 %x)
+  call void @llvm.write_register.i32(metadata !19, i32 %x)
+  call void @llvm.write_register.i32(metadata !20, i32 %x)
+  call void @llvm.write_register.i32(metadata !21, i32 %x)
+  call void @llvm.write_register.i32(metadata !22, i32 %x)
+  call void @llvm.write_register.i32(metadata !23, i32 %x)
+  call void @llvm.write_register.i32(metadata !24, i32 %x)
+  call void @llvm.write_register.i32(metadata !25, i32 %x)
+  ret void
+}
+
+declare i32 @llvm.read_register.i32(metadata) nounwind
+declare void @llvm.write_register.i32(metadata, i32) nounwind
+
+!0 = !{!"apsr"}
+!1 = !{!"apsr_nzcvq"}
+!2 = !{!"apsr_g"}
+!3 = !{!"apsr_nzcvqg"}
+!4 = !{!"iapsr"}
+!5 = !{!"iapsr_nzcvq"}
+!6 = !{!"iapsr_g"}
+!7 = !{!"iapsr_nzcvqg"}
+!8 = !{!"eapsr"}
+!9 = !{!"eapsr_nzcvq"}
+!10 = !{!"eapsr_g"}
+!11 = !{!"eapsr_nzcvqg"}
+!12 = !{!"xpsr"}
+!13 = !{!"xpsr_nzcvq"}
+!14 = !{!"xpsr_g"}
+!15 = !{!"xpsr_nzcvqg"}
+!16 = !{!"ipsr"}
+!17 = !{!"epsr"}
+!18 = !{!"iepsr"}
+!19 = !{!"msp"}
+!20 = !{!"psp"}
+!21 = !{!"primask"}
+!22 = !{!"basepri"}
+!23 = !{!"basepri_max"}
+!24 = !{!"faultmask"}
+!25 = !{!"control"}
diff --git a/test/CodeGen/ARM/special-reg.ll b/test/CodeGen/ARM/special-reg.ll
new file mode 100644
index 00000000000..7ccb490f5d4
--- /dev/null
+++ b/test/CodeGen/ARM/special-reg.ll
@@ -0,0 +1,78 @@
+; RUN: llc < %s -mtriple=arm-none-eabi -mcpu=cortex-a8 2>&1 | FileCheck %s --check-prefix=ARM --check-prefix=ACORE
+; RUN: llc < %s -mtriple=thumb-none-eabi -mcpu=cortex-m4 2>&1 | FileCheck %s --check-prefix=ARM --check-prefix=MCORE
+
+define i32 @read_i32_encoded_register() nounwind {
+entry:
+; ARM-LABEL: read_i32_encoded_register:
+; ARM: mrc p1, #2, r0, c3, c4, #5
+  %reg = call i32 @llvm.read_register.i32(metadata !0)
+  ret i32 %reg
+}
+
+define i64 @read_i64_encoded_register() nounwind {
+entry:
+; ARM-LABEL: read_i64_encoded_register:
+; ARM: mrrc p1, #2, r0, r1, c3
+  %reg = call i64 @llvm.read_register.i64(metadata !1)
+  ret i64 %reg
+}
+
+define i32 @read_apsr() nounwind {
+entry:
+; ARM-LABEL: read_apsr:
+; ARM: mrs r0, apsr
+  %reg = call i32 @llvm.read_register.i32(metadata !2)
+  ret i32 %reg
+}
+
+define i32 @read_fpscr() nounwind {
+entry:
+; ARM-LABEL: read_fpscr:
+; ARM: vmrs r0, fpscr
+  %reg = call i32 @llvm.read_register.i32(metadata !3)
+  ret i32 %reg
+}
+
+define void @write_i32_encoded_register(i32 %x) nounwind {
+entry:
+; ARM-LABEL: write_i32_encoded_register:
+; ARM: mcr p1, #2, r0, c3, c4, #5
+  call void @llvm.write_register.i32(metadata !0, i32 %x)
+  ret void
+}
+
+define void @write_i64_encoded_register(i64 %x) nounwind {
+entry:
+; ARM-LABEL: write_i64_encoded_register:
+; ARM: mcrr p1, #2, r0, r1, c3
+  call void @llvm.write_register.i64(metadata !1, i64 %x)
+  ret void
+}
+
+define void @write_apsr(i32 %x) nounwind {
+entry:
+; ARM-LABEL: write_apsr:
+; ACORE: msr APSR_nzcvq, r0
+; MCORE: msr apsr_nzcvq, r0
+  call void @llvm.write_register.i32(metadata !4, i32 %x)
+  ret void
+}
+
+define void @write_fpscr(i32 %x) nounwind {
+entry:
+; ARM-LABEL: write_fpscr:
+; ARM: vmsr fpscr, r0
+  call void @llvm.write_register.i32(metadata !3, i32 %x)
+  ret void
+}
+
+declare i32 @llvm.read_register.i32(metadata) nounwind
+declare i64 @llvm.read_register.i64(metadata) nounwind
+declare void @llvm.write_register.i32(metadata, i32) nounwind
+declare void @llvm.write_register.i64(metadata, i64) nounwind
+
+!0 = !{!"cp1:2:c3:c4:5"}
+!1 = !{!"cp1:2:c3"}
+!2 = !{!"apsr"}
+!3 = !{!"fpscr"}
+!4 = !{!"apsr_nzcvq"}
diff --git a/test/CodeGen/ARM/struct-byval-frame-index.ll b/test/CodeGen/ARM/struct-byval-frame-index.ll
index bca797d6dce..52f70fe1e0f 100644
--- a/test/CodeGen/ARM/struct-byval-frame-index.ll
+++ b/test/CodeGen/ARM/struct-byval-frame-index.ll
@@ -194,7 +194,7 @@ lor.lhs.false459:                                 ; preds = %if.end454
   %18 = load i32, i32* %mb_type, align 4
   switch i32 %18, label %for.inc503 [
     i32 9, label %if.then475
-    i32 10, label %if.then475
+    i32 11, label %if.then475
     i32 13, label %if.then475
     i32 14, label %if.then475
   ]
diff --git a/test/CodeGen/ARM/struct_byval.ll b/test/CodeGen/ARM/struct_byval.ll
index cb533a37e73..d7b9b477ec1 100644
--- a/test/CodeGen/ARM/struct_byval.ll
+++ b/test/CodeGen/ARM/struct_byval.ll
@@ -5,10 +5,6 @@
 
 ; NOMOVT-NOT: movt
 
-; For NaCl (where we always want movt) ensure no const pool loads are generated.
-; RUN: llc < %s -mtriple=armv7-unknown-nacl-gnueabi | FileCheck %s -check-prefix=NACL
-; NACL-NOT: .LCPI
-
 ; rdar://9877866
 %struct.SmallStruct = type { i32, [8 x i32], [37 x i8] }
 %struct.LargeStruct = type { i32, [1001 x i8], [300 x i32] }
diff --git a/test/CodeGen/ARM/subtarget-features-long-calls.ll b/test/CodeGen/ARM/subtarget-features-long-calls.ll
new file mode 100644
index 00000000000..430ae3d1330
--- /dev/null
+++ b/test/CodeGen/ARM/subtarget-features-long-calls.ll
@@ -0,0 +1,49 @@
+; RUN: llc -march thumb -mcpu=cortex-a8 -relocation-model=static %s -o - | FileCheck -check-prefix=NO-OPTION %s
+; RUN: llc -march thumb -mcpu=cortex-a8 -relocation-model=static %s -o - -mattr=+long-calls | FileCheck -check-prefix=LONGCALL %s
+; RUN: llc -march thumb -mcpu=cortex-a8 -relocation-model=static %s -o - -mattr=-long-calls | FileCheck -check-prefix=NO-LONGCALL %s
+; RUN: llc -march thumb -mcpu=cortex-a8 -relocation-model=static %s -o - -O0 | FileCheck -check-prefix=NO-OPTION %s
+; RUN: llc -march thumb -mcpu=cortex-a8 -relocation-model=static %s -o - -O0 -mattr=+long-calls | FileCheck -check-prefix=LONGCALL %s
+; RUN: llc -march thumb -mcpu=cortex-a8 -relocation-model=static %s -o - -O0 -mattr=-long-calls | FileCheck -check-prefix=NO-LONGCALL %s
+
+; NO-OPTION-LABEL: {{_?}}caller0
+; NO-OPTION: ldr [[R0:r[0-9]+]], [[L0:.*]] 
+; NO-OPTION: blx [[R0]]
+; NO-OPTION: [[L0]]:
+; NO-OPTION: .long {{_?}}callee0
+
+; LONGCALL-LABEL: {{_?}}caller0
+; LONGCALL: ldr [[R0:r[0-9]+]], [[L0:.*]]
+; LONGCALL: blx [[R0]]
+; LONGCALL: [[L0]]:
+; LONGCALL: .long {{_?}}callee0
+
+; NO-LONGCALL-LABEL: {{_?}}caller0
+; NO-LONGCALL: bl {{_?}}callee0
+
+define i32 @caller0() #0 {
+entry:
+  tail call void @callee0()
+  ret i32 0
+}
+
+; NO-OPTION-LABEL: {{_?}}caller1
+; NO-OPTION: bl {{_?}}callee0
+
+; LONGCALL-LABEL: {{_?}}caller1
+; LONGCALL: ldr [[R0:r[0-9]+]], [[L0:.*]]
+; LONGCALL: blx [[R0]]
+; LONGCALL: [[L0]]:
+; LONGCALL: .long {{_?}}callee0
+
+; NO-LONGCALL-LABEL: {{_?}}caller1
+; NO-LONGCALL: bl {{_?}}callee0
+
+define i32 @caller1() {
+entry:
+  tail call void @callee0()
+  ret i32 0
+}
+
+declare void @callee0()
+
+attributes #0 = { "target-features"="+long-calls" }
diff --git a/test/CodeGen/ARM/subtarget-no-movt.ll b/test/CodeGen/ARM/subtarget-no-movt.ll
new file mode 100644
index 00000000000..cb61bde3f9c
--- /dev/null
+++ b/test/CodeGen/ARM/subtarget-no-movt.ll
@@ -0,0 +1,45 @@
+; RUN: llc -march thumb -mcpu=cortex-a8 -relocation-model=static %s -o - | FileCheck -check-prefix=NO-OPTION %s
+; RUN: llc -march thumb -mcpu=cortex-a8 -relocation-model=static %s -o - -mattr=-no-movt | FileCheck -check-prefix=USE-MOVT %s
+; RUN: llc -march thumb -mcpu=cortex-a8 -relocation-model=static %s -o - -mattr=+no-movt | FileCheck -check-prefix=NO-USE-MOVT %s
+; RUN: llc -march thumb -mcpu=cortex-a8 -relocation-model=static %s -o - -O0 | FileCheck -check-prefix=NO-OPTION %s
+; RUN: llc -march thumb -mcpu=cortex-a8 -relocation-model=static %s -o - -O0 -mattr=-no-movt | FileCheck -check-prefix=USE-MOVT %s
+; RUN: llc -march thumb -mcpu=cortex-a8 -relocation-model=static %s -o - -O0 -mattr=+no-movt | FileCheck -check-prefix=NO-USE-MOVT %s
+
+; NO-OPTION-LABEL: {{_?}}foo0
+; NO-OPTION: ldr [[R0:r[0-9]+]], [[L0:.*]]
+; NO-OPTION: [[L0]]:
+; NO-OPTION: .long 2296237089
+
+; USE-MOVT-LABEL: {{_?}}foo0
+; USE-MOVT: movw [[R0:r[0-9]+]], #52257
+; USE-MOVT: movt [[R0]], #35037
+
+; NO-USE-MOVT-LABEL: {{_?}}foo0
+; NO-USE-MOVT: ldr [[R0:r[0-9]+]], [[L0:.*]]
+; NO-USE-MOVT: [[L0]]:
+; NO-USE-MOVT: .long 2296237089
+
+define i32 @foo0(i32 %a) #0 {
+  %1 = xor i32 -1998730207, %a
+  ret i32 %1
+}
+
+; NO-OPTION-LABEL: {{_?}}foo1
+; NO-OPTION: movw [[R0:r[0-9]+]], #52257
+; NO-OPTION: movt [[R0]], #35037
+
+; USE-MOVT-LABEL: {{_?}}foo1
+; USE-MOVT: movw [[R0:r[0-9]+]], #52257
+; USE-MOVT: movt [[R0]], #35037
+
+; NO-USE-MOVT-LABEL: {{_?}}foo1
+; NO-USE-MOVT: ldr [[R0:r[0-9]+]], [[L0:.*]]
+; NO-USE-MOVT: [[L0]]:
+; NO-USE-MOVT: .long 2296237089
+
+define i32 @foo1(i32 %a) {
+  %1 = xor i32 -1998730207, %a
+  ret i32 %1
+}
+
+attributes #0 = { "target-features"="+no-movt" }
diff --git a/test/CodeGen/ARM/t2abs-killflags.ll b/test/CodeGen/ARM/t2abs-killflags.ll
new file mode 100644
index 00000000000..eed8f6719ae
--- /dev/null
+++ b/test/CodeGen/ARM/t2abs-killflags.ll
@@ -0,0 +1,23 @@
+; RUN: llc %s -o - -verify-machineinstrs | FileCheck %s
+
+target datalayout = "e-m:o-p:32:32-f64:32:64-v64:32:64-v128:32:128-a:0:32-n32-S32"
+target triple = "thumbv7s-unknown-unknown"
+
+; Test that we don't crash the machine verifier when expanding t2ABS.
+; It was applying the kill flag to its source, even if the original t2ABS didn't
+; kill the operand prior to pseudo expansion.
+
+; Function Attrs: noreturn
+declare void @foo(i32, i32) #0
+
+; CHECK-LABEL: @test
+; CHECK: rsbmi
+define void @test(i32 %arg) {
+  %cmp58 = icmp slt i32 %arg, 0
+  %sub62 = sub nsw i32 0, %arg
+  %l.1 = select i1 %cmp58, i32 %sub62, i32 %arg
+  call void @foo(i32 %l.1, i32 %arg) #0
+  unreachable
+}
+
+attributes #0 = { noreturn }
diff --git a/test/CodeGen/ARM/tail-dup-kill-flags.ll b/test/CodeGen/ARM/tail-dup-kill-flags.ll
new file mode 100644
index 00000000000..bce6cdc4e28
--- /dev/null
+++ b/test/CodeGen/ARM/tail-dup-kill-flags.ll
@@ -0,0 +1,54 @@
+; RUN: llc %s -o - -fast-isel=true -O1 -verify-machineinstrs | FileCheck %s
+
+target datalayout = "e-m:o-p:32:32-f64:32:64-v64:32:64-v128:32:128-a:0:32-n32-S32"
+target triple = "thumbv7-apple-ios8.0.0"
+
+; Tail duplication was incorrectly leaving kill flags on the duplicated instructions.
+; The machine verifier is able to spot this error, so this test should pass if it passes verification.
+
+; CHECK-LABEL: @test
+
+%struct.cdiff_ctx = type { i8*, %struct.cdiff_node*, %struct.cdiff_node*, %struct.cdiff_node*, %struct.cdiff_node* }
+%struct.cdiff_node = type { i32, i8*, i8*, %struct.cdiff_node* }
+
+declare i32 @logg(i32)
+
+define hidden i32 @test(%struct.cdiff_ctx* nocapture %ctx, %struct.cdiff_node* %tmp10) {
+bb:
+  br label %.outer
+
+bb33:                                             ; preds = %bb92, %.outer
+  %lines.0 = phi i32 [ %tmp37, %bb92 ], [ %lines.0.ph, %.outer ]
+  %tmp37 = add i32 %lines.0, 1
+  %tmp39 = load i32, i32* %tmp57, align 4
+  %tmp40 = icmp eq i32 %tmp39, %tmp37
+  br i1 %tmp40, label %bb41, label %bb92
+
+bb41:                                             ; preds = %bb33
+  %tmp45 = call i32 @strncmp()
+  %tmp46 = icmp eq i32 %tmp45, 0
+  br i1 %tmp46, label %bb53, label %bb47
+
+bb47:                                             ; preds = %bb41
+  %tmp52 = call i32 @logg(i32 %tmp37)
+  ret i32 -1
+
+bb53:                                             ; preds = %bb41
+  %tmp54 = getelementptr inbounds %struct.cdiff_node, %struct.cdiff_node* %del.0.ph, i32 0, i32 3
+  %tmp55 = load %struct.cdiff_node*, %struct.cdiff_node** %tmp54, align 4
+  br label %.outer
+
+.outer:                                           ; preds = %bb53, %bb
+  %del.0.ph = phi %struct.cdiff_node* [ %tmp55, %bb53 ], [ null, %bb ]
+  %lines.0.ph = phi i32 [ 1, %bb53 ], [ 0, %bb ]
+  %tmp57 = getelementptr inbounds %struct.cdiff_node, %struct.cdiff_node* %del.0.ph, i32 0, i32 0
+  br label %bb33
+
+bb92:                                             ; preds = %bb33
+  %tmp93 = call i32 @puts()
+  br label %bb33
+}
+
+declare i32 @strncmp()
+
+declare i32 @puts()
diff --git a/test/CodeGen/ARM/test-sharedidx.ll b/test/CodeGen/ARM/test-sharedidx.ll
index 377996c4c3c..77d0f30485d 100644
--- a/test/CodeGen/ARM/test-sharedidx.ll
+++ b/test/CodeGen/ARM/test-sharedidx.ll
@@ -20,8 +20,8 @@ entry:
 
 for.body:                                         ; preds = %entry, %for.body.3
 ; CHECK: %for.body
-; CHECK: ldrb {{r[0-9]|lr}}, [{{r[0-9]|lr}}, {{r[0-9]|lr}}]!
-; CHECK: ldrb {{r[0-9]|lr}}, [{{r[0-9]|lr}}, {{r[0-9]|lr}}]!
+; CHECK: ldrb {{r[0-9]+|lr}}, [{{r[0-9]+|lr}}, {{r[0-9]+|lr}}]!
+; CHECK: ldrb {{r[0-9]+|lr}}, [{{r[0-9]+|lr}}, {{r[0-9]+|lr}}]!
   %i.09 = phi i32 [ %add5.3, %for.body.3 ], [ 0, %entry ]
   %arrayidx = getelementptr inbounds i8, i8* %a, i32 %i.09
   %0 = load i8, i8* %arrayidx, align 1
@@ -42,8 +42,8 @@ for.end:                                          ; preds = %for.body, %for.body
 
 for.body.1:                                       ; preds = %for.body
 ; CHECK: %for.body.1
-; CHECK: ldrb {{r[0-9]|lr}}, [{{r[0-9]|lr}}, {{r[0-9]|lr}}]!
-; CHECK: ldrb {{r[0-9]|lr}}, [{{r[0-9]|lr}}, {{r[0-9]|lr}}]!
+; CHECK: ldrb {{r[0-9]+|lr}}, [{{r[0-9]+|lr}}, {{r[0-9]+|lr}}]!
+; CHECK: ldrb {{r[0-9]+|lr}}, [{{r[0-9]+|lr}}, {{r[0-9]+|lr}}]!
   %arrayidx.1 = getelementptr inbounds i8, i8* %a, i32 %add5
   %2 = load i8, i8* %arrayidx.1, align 1
   %conv6.1 = zext i8 %2 to i32
@@ -60,8 +60,8 @@ for.body.1:                                       ; preds = %for.body
 
 for.body.2:                                       ; preds = %for.body.1
 ; CHECK: %for.body.2
-; CHECK: ldrb {{r[0-9]|lr}}, [{{r[0-9]|lr}}, {{r[0-9]|lr}}]!
-; CHECK: ldrb {{r[0-9]|lr}}, [{{r[0-9]|lr}}, {{r[0-9]|lr}}]!
+; CHECK: ldrb {{r[0-9]+|lr}}, [{{r[0-9]+|lr}}, {{r[0-9]+|lr}}]!
+; CHECK: ldrb {{r[0-9]+|lr}}, [{{r[0-9]+|lr}}, {{r[0-9]+|lr}}]!
   %arrayidx.2 = getelementptr inbounds i8, i8* %a, i32 %add5.1
   %4 = load i8, i8* %arrayidx.2, align 1
   %conv6.2 = zext i8 %4 to i32
@@ -78,8 +78,8 @@ for.body.2:                                       ; preds = %for.body.1
 
 for.body.3:                                       ; preds = %for.body.2
 ; CHECK: %for.body.3
-; CHECK: ldrb {{r[0-9]|lr}}, [{{r[0-9]|lr}}, {{r[0-9]|lr}}]!
-; CHECK: ldrb {{r[0-9]|lr}}, [{{r[0-9]|lr}}, {{r[0-9]|lr}}]!
+; CHECK: ldrb {{r[0-9]+|lr}}, [{{r[0-9]+|lr}}, {{r[0-9]+|lr}}]!
+; CHECK: ldrb {{r[0-9]+|lr}}, [{{r[0-9]+|lr}}, {{r[0-9]+|lr}}]!
   %arrayidx.3 = getelementptr inbounds i8, i8* %a, i32 %add5.2
   %6 = load i8, i8* %arrayidx.3, align 1
   %conv6.3 = zext i8 %6 to i32
diff --git a/test/CodeGen/ARM/thumb-alignment.ll b/test/CodeGen/ARM/thumb-alignment.ll
new file mode 100644
index 00000000000..c11d4b6da3c
--- /dev/null
+++ b/test/CodeGen/ARM/thumb-alignment.ll
@@ -0,0 +1,54 @@
+; RUN: llc -relocation-model=pic -mtriple=thumbv7-unknown-linux -o - %s | FileCheck %s
+
+@x = external global i32
+
+; CHECK: .globl	foo
+; CHECK-NEXT: .align	2
+define i32* @foo() {
+  ret i32* @x
+}
+
+; CHECK: .globl	bar
+; CHECK-NEXT: .align	1
+define i32* @bar() {
+  ret i32* zeroinitializer
+}
+
+@a = external global i32
+@b = external global i32
+@c = external global i32
+@d = external global i32
+
+; Create a Thumb-2 jump table, which should force alignment to 4 bytes.
+
+; CHECK: .globl	baz
+; CHECK-NEXT: .align	2
+; CHECK: adr.w
+define i32 @baz() {
+  %1 = load i32, i32* @c, align 4
+  switch i32 %1, label %7 [
+    i32 1, label %2
+    i32 4, label %5
+    i32 9, label %5
+    i32 3, label %8
+  ]
+
+; <label>:2
+  %3 = load i32, i32* @a, align 4
+  %4 = tail call i32 bitcast (i32 (...)* @fn2 to i32 (i32 (...)*, i32, i32)*)(i32 (...)* bitcast (i32 ()* @baz to i32 (...)*), i32 0, i32 %3) #2
+  br label %8
+
+; <label>:5
+  %6 = load i32, i32* @d, align 4
+  store i32 %6, i32* @b, align 4
+  br label %8
+
+; <label>:7
+  br label %8
+
+; <label>:8
+  %e.0 = phi i32 [ 1, %7 ], [ 1, %2 ], [ 0, %0 ], [ 0, %5 ]
+  ret i32 %e.0
+}
+
+declare i32 @fn2(...)
diff --git a/test/CodeGen/ARM/thumb-big-stack.ll b/test/CodeGen/ARM/thumb-big-stack.ll
new file mode 100644
index 00000000000..e5cbb9747a7
--- /dev/null
+++ b/test/CodeGen/ARM/thumb-big-stack.ll
@@ -0,0 +1,6844 @@
+; RUN: llc %s -O0 -verify-machineinstrs -o - | FileCheck %s
+; This file uses to trigger a machine verifier error because we
+; were generating a stack adjustement with SP as second argument,
+; which is unpredictable behavior for t2ADDrr.
+; This file has been generated from the constpool test of the test-suite.
+target datalayout = "e-m:o-p:32:32-f64:32:64-v64:32:64-v128:32:128-a:0:32-n32-S32"
+target triple = "thumbv7s-apple-ios"
+
+@.str = external unnamed_addr constant [21 x i8], align 1
+
+; CHECK-LABEL: f:
+; CHECK: movw [[ADDR:(r[0-9]+|lr)]], #
+; CHECK-NEXT: add [[ADDR]], sp
+; CHECK-NEXT: vst1.64 {d{{[0-9]+}}, d{{[0-9]+}}}, {{\[}}[[ADDR]]:128]
+define <4 x float> @f(<4 x float> %x) {
+entry:
+  %.compoundliteral7837 = alloca <4 x float>, align 16
+  %.compoundliteral7865 = alloca <4 x float>, align 16
+  %.compoundliteral7991 = alloca <4 x float>, align 16
+  %.compoundliteral8019 = alloca <4 x float>, align 16
+  %.compoundliteral8061 = alloca <4 x float>, align 16
+  %.compoundliteral8075 = alloca <4 x float>, align 16
+  %.compoundliteral8089 = alloca <4 x float>, align 16
+  %.compoundliteral8103 = alloca <4 x float>, align 16
+  %.compoundliteral8117 = alloca <4 x float>, align 16
+  %.compoundliteral8145 = alloca <4 x float>, align 16
+  %.compoundliteral8243 = alloca <4 x float>, align 16
+  %.compoundliteral8285 = alloca <4 x float>, align 16
+  %.compoundliteral8299 = alloca <4 x float>, align 16
+  %.compoundliteral8313 = alloca <4 x float>, align 16
+  %.compoundliteral8327 = alloca <4 x float>, align 16
+  %.compoundliteral9601 = alloca <4 x float>, align 16
+  %.compoundliteral9615 = alloca <4 x float>, align 16
+  %.compoundliteral9629 = alloca <4 x float>, align 16
+  %.compoundliteral9657 = alloca <4 x float>, align 16
+  %.compoundliteral9755 = alloca <4 x float>, align 16
+  %.compoundliteral9769 = alloca <4 x float>, align 16
+  %.compoundliteral9853 = alloca <4 x float>, align 16
+  %.compoundliteral9867 = alloca <4 x float>, align 16
+  %.compoundliteral9895 = alloca <4 x float>, align 16
+  %.compoundliteral9909 = alloca <4 x float>, align 16
+  %.compoundliteral9923 = alloca <4 x float>, align 16
+  %.compoundliteral9937 = alloca <4 x float>, align 16
+  %.compoundliteral9951 = alloca <4 x float>, align 16
+  %.compoundliteral9979 = alloca <4 x float>, align 16
+  %.compoundliteral10021 = alloca <4 x float>, align 16
+  %.compoundliteral10049 = alloca <4 x float>, align 16
+  %.compoundliteral10063 = alloca <4 x float>, align 16
+  %.compoundliteral10077 = alloca <4 x float>, align 16
+  %.compoundliteral10091 = alloca <4 x float>, align 16
+  %.compoundliteral10119 = alloca <4 x float>, align 16
+  %.compoundliteral10133 = alloca <4 x float>, align 16
+  %.compoundliteral10147 = alloca <4 x float>, align 16
+  %.compoundliteral10161 = alloca <4 x float>, align 16
+  %.compoundliteral10203 = alloca <4 x float>, align 16
+  %.compoundliteral10231 = alloca <4 x float>, align 16
+  %.compoundliteral10385 = alloca <4 x float>, align 16
+  %.compoundliteral10399 = alloca <4 x float>, align 16
+  %.compoundliteral10413 = alloca <4 x float>, align 16
+  %.compoundliteral10539 = alloca <4 x float>, align 16
+  %.compoundliteral10553 = alloca <4 x float>, align 16
+  %.compoundliteral10567 = alloca <4 x float>, align 16
+  %.compoundliteral10581 = alloca <4 x float>, align 16
+  %.compoundliteral10595 = alloca <4 x float>, align 16
+  %.compoundliteral10609 = alloca <4 x float>, align 16
+  %.compoundliteral10623 = alloca <4 x float>, align 16
+  %.compoundliteral10637 = alloca <4 x float>, align 16
+  %.compoundliteral10665 = alloca <4 x float>, align 16
+  %.compoundliteral10693 = alloca <4 x float>, align 16
+  %.compoundliteral10707 = alloca <4 x float>, align 16
+  %.compoundliteral10721 = alloca <4 x float>, align 16
+  %.compoundliteral10735 = alloca <4 x float>, align 16
+  %.compoundliteral10749 = alloca <4 x float>, align 16
+  %.compoundliteral10763 = alloca <4 x float>, align 16
+  %.compoundliteral10945 = alloca <4 x float>, align 16
+  %.compoundliteral10959 = alloca <4 x float>, align 16
+  %.compoundliteral10987 = alloca <4 x float>, align 16
+  %.compoundliteral11001 = alloca <4 x float>, align 16
+  %.compoundliteral11015 = alloca <4 x float>, align 16
+  %.compoundliteral11197 = alloca <4 x float>, align 16
+  %.compoundliteral11421 = alloca <4 x float>, align 16
+  %.compoundliteral11435 = alloca <4 x float>, align 16
+  %.compoundliteral11463 = alloca <4 x float>, align 16
+  %.compoundliteral11477 = alloca <4 x float>, align 16
+  %.compoundliteral11491 = alloca <4 x float>, align 16
+  %.compoundliteral11519 = alloca <4 x float>, align 16
+  %.compoundliteral11533 = alloca <4 x float>, align 16
+  %.compoundliteral11547 = alloca <4 x float>, align 16
+  %.compoundliteral11631 = alloca <4 x float>, align 16
+  %.compoundliteral11645 = alloca <4 x float>, align 16
+  %.compoundliteral11659 = alloca <4 x float>, align 16
+  %.compoundliteral11701 = alloca <4 x float>, align 16
+  %.compoundliteral11743 = alloca <4 x float>, align 16
+  %.compoundliteral11757 = alloca <4 x float>, align 16
+  %.compoundliteral11771 = alloca <4 x float>, align 16
+  %.compoundliteral11785 = alloca <4 x float>, align 16
+  %.compoundliteral11799 = alloca <4 x float>, align 16
+  %.compoundliteral11827 = alloca <4 x float>, align 16
+  %.compoundliteral11841 = alloca <4 x float>, align 16
+  %.compoundliteral11855 = alloca <4 x float>, align 16
+  %.compoundliteral11869 = alloca <4 x float>, align 16
+  %.compoundliteral11939 = alloca <4 x float>, align 16
+  %.compoundliteral11953 = alloca <4 x float>, align 16
+  %.compoundliteral11967 = alloca <4 x float>, align 16
+  %.compoundliteral11981 = alloca <4 x float>, align 16
+  %.compoundliteral11995 = alloca <4 x float>, align 16
+  %.compoundliteral12023 = alloca <4 x float>, align 16
+  %.compoundliteral12051 = alloca <4 x float>, align 16
+  %.compoundliteral12065 = alloca <4 x float>, align 16
+  %.compoundliteral12247 = alloca <4 x float>, align 16
+  %.compoundliteral12261 = alloca <4 x float>, align 16
+  %.compoundliteral12275 = alloca <4 x float>, align 16
+  %.compoundliteral12499 = alloca <4 x float>, align 16
+  %.compoundliteral12541 = alloca <4 x float>, align 16
+  %.compoundliteral12555 = alloca <4 x float>, align 16
+  %.compoundliteral12751 = alloca <4 x float>, align 16
+  %.compoundliteral12891 = alloca <4 x float>, align 16
+  %.compoundliteral12905 = alloca <4 x float>, align 16
+  %.compoundliteral12919 = alloca <4 x float>, align 16
+  %.compoundliteral12933 = alloca <4 x float>, align 16
+  %.compoundliteral12947 = alloca <4 x float>, align 16
+  %.compoundliteral12961 = alloca <4 x float>, align 16
+  %.compoundliteral12975 = alloca <4 x float>, align 16
+  %.compoundliteral12989 = alloca <4 x float>, align 16
+  %.compoundliteral13003 = alloca <4 x float>, align 16
+  %.compoundliteral13017 = alloca <4 x float>, align 16
+  %.compoundliteral13031 = alloca <4 x float>, align 16
+  %.compoundliteral13423 = alloca <4 x float>, align 16
+  %.compoundliteral13437 = alloca <4 x float>, align 16
+  %.compoundliteral13493 = alloca <4 x float>, align 16
+  %.compoundliteral13535 = alloca <4 x float>, align 16
+  %.compoundliteral13549 = alloca <4 x float>, align 16
+  %.compoundliteral13647 = alloca <4 x float>, align 16
+  %.compoundliteral13675 = alloca <4 x float>, align 16
+  %.compoundliteral13689 = alloca <4 x float>, align 16
+  %.compoundliteral13703 = alloca <4 x float>, align 16
+  %.compoundliteral13717 = alloca <4 x float>, align 16
+  %.compoundliteral13745 = alloca <4 x float>, align 16
+  %.compoundliteral13759 = alloca <4 x float>, align 16
+  %.compoundliteral13773 = alloca <4 x float>, align 16
+  %.compoundliteral13787 = alloca <4 x float>, align 16
+  %.compoundliteral13941 = alloca <4 x float>, align 16
+  %.compoundliteral13969 = alloca <4 x float>, align 16
+  %.compoundliteral13983 = alloca <4 x float>, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> <float 0x40746999A0000000, float 0xC0719B3340000000, float 0xC070B66660000000, float 0xC07404CCC0000000>, <4 x float>* undef
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> <float 0x40701B3340000000, float 0x405B866660000000, float 0xC0763999A0000000, float 4.895000e+02>, <4 x float>* undef
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp = load <4 x float>, <4 x float>* undef
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp1 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add68 = fadd <4 x float> %tmp1, %tmp
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %add68, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp2 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add76 = fadd float undef, 0x4074C999A0000000
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp3 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins77 = insertelement <4 x float> %tmp3, float %add76, i32 2
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %vecins77, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp4 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecext78 = extractelement <4 x float> %tmp4, i32 3
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add79 = fadd float %vecext78, 0x40776E6660000000
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp5 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins80 = insertelement <4 x float> %tmp5, float %add79, i32 3
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %vecins80, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> <float 0x40678CCCC0000000, float 0xC03E4CCCC0000000, float -4.170000e+02, float -1.220000e+02>, <4 x float>* undef
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp6 = load <4 x float>, <4 x float>* undef
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add82 = fadd <4 x float> undef, %tmp6
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %add82, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp7 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecext83 = extractelement <4 x float> %tmp7, i32 0
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add84 = fadd float %vecext83, 1.300000e+02
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp8 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins85 = insertelement <4 x float> %tmp8, float %add84, i32 0
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %vecins85, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp9 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecext86 = extractelement <4 x float> %tmp9, i32 1
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add93 = fadd float undef, 0xC076C66660000000
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp10 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins94 = insertelement <4 x float> %tmp10, float %add93, i32 3
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> <float 0x406C2999A0000000, float 8.050000e+01, float 0xC0794999A0000000, float 0xC073E4CCC0000000>, <4 x float>* undef
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp11 = load <4 x float>, <4 x float>* undef
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp12 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add96 = fadd <4 x float> %tmp12, %tmp11
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp13 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecext97 = extractelement <4 x float> %tmp13, i32 0
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add98 = fadd float %vecext97, 0x4079E66660000000
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp14 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins102 = insertelement <4 x float> undef, float undef, i32 1
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %vecins102, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp15 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add104 = fadd float undef, 0x406AB999A0000000
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp16 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> <float 0xC0531999A0000000, float 0xC0737999A0000000, float 0x407CB33340000000, float 0xC06DCCCCC0000000>, <4 x float>* undef
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecext579 = extractelement <4 x float> undef, i32 2
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add580 = fadd float %vecext579, 0xC07424CCC0000000
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp17 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins581 = insertelement <4 x float> %tmp17, float %add580, i32 2
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %vecins581, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp18 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecext582 = extractelement <4 x float> %tmp18, i32 3
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add583 = fadd float %vecext582, 0x40444CCCC0000000
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> undef, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp19 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecext590 = extractelement <4 x float> %tmp19, i32 1
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add591 = fadd float %vecext590, 1.725000e+02
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins592 = insertelement <4 x float> undef, float %add591, i32 1
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %vecins592, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp20 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add594 = fadd float undef, 0xC05B466660000000
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add605 = fadd float undef, 0x407164CCC0000000
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp21 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add616 = fadd float undef, 1.885000e+02
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp22 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp23 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins620 = insertelement <4 x float> undef, float undef, i32 1
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %vecins620, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecext621 = extractelement <4 x float> undef, i32 2
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add622 = fadd float %vecext621, 0x40709B3340000000
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins623 = insertelement <4 x float> undef, float %add622, i32 2
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %vecins623, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp24 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecext624 = extractelement <4 x float> %tmp24, i32 3
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add625 = fadd float %vecext624, 0xC064033340000000
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp25 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins626 = insertelement <4 x float> %tmp25, float %add625, i32 3
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %vecins626, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> <float 0x404D0CCCC0000000, float 3.955000e+02, float 0xC0334CCCC0000000, float 0x40754E6660000000>, <4 x float>* undef
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp26 = load <4 x float>, <4 x float>* undef
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp27 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add628 = fadd <4 x float> %tmp27, %tmp26
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %add628, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp28 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecext629 = extractelement <4 x float> %tmp28, i32 0
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add630 = fadd float %vecext629, 0x40730CCCC0000000
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp29 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins631 = insertelement <4 x float> %tmp29, float %add630, i32 0
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %vecins631, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp30 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecext632 = extractelement <4 x float> %tmp30, i32 1
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add633 = fadd float %vecext632, 0xC0630999A0000000
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp31 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins634 = insertelement <4 x float> %tmp31, float %add633, i32 1
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %vecins634, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp32 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecext635 = extractelement <4 x float> %tmp32, i32 2
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add636 = fadd float %vecext635, 0xC078833340000000
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp33 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp34 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp35 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add658 = fadd float undef, 0xC04A4CCCC0000000
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecext663 = extractelement <4 x float> undef, i32 2
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp36 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins665 = insertelement <4 x float> %tmp36, float undef, i32 2
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecext694 = extractelement <4 x float> undef, i32 3
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add695 = fadd float %vecext694, 0xC03CCCCCC0000000
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp37 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins696 = insertelement <4 x float> %tmp37, float %add695, i32 3
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %vecins696, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> <float 0xC069FCCCC0000000, float 0xC07C6E6660000000, float 0x4067E33340000000, float 0x4078DB3340000000>, <4 x float>* undef
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp38 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> undef, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecext699 = extractelement <4 x float> undef, i32 0
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add703 = fadd float undef, 0x4068F33340000000
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins704 = insertelement <4 x float> undef, float %add703, i32 1
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %vecins704, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp39 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp40 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins710 = insertelement <4 x float> %tmp40, float undef, i32 3
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %vecins710, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> <float 0xC05D9999A0000000, float 0x405D6CCCC0000000, float 0x40765CCCC0000000, float 0xC07C64CCC0000000>, <4 x float>* undef
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp41 = load <4 x float>, <4 x float>* undef
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp42 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add712 = fadd <4 x float> %tmp42, %tmp41
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %add712, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp43 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecext713 = extractelement <4 x float> %tmp43, i32 0
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp44 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins715 = insertelement <4 x float> %tmp44, float undef, i32 0
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp45 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecext716 = extractelement <4 x float> %tmp45, i32 1
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add717 = fadd float %vecext716, -4.315000e+02
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp46 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins718 = insertelement <4 x float> %tmp46, float %add717, i32 1
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %vecins718, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp47 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecext719 = extractelement <4 x float> %tmp47, i32 2
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add723 = fadd float undef, 0xC06A6CCCC0000000
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins724 = insertelement <4 x float> undef, float %add723, i32 3
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add726 = fadd <4 x float> undef, undef
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> undef, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecext730 = extractelement <4 x float> undef, i32 1
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add731 = fadd float %vecext730, 0xC0759CCCC0000000
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp48 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins732 = insertelement <4 x float> %tmp48, float %add731, i32 1
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %vecins732, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp49 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecext733 = extractelement <4 x float> %tmp49, i32 2
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> undef, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp50 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins738 = insertelement <4 x float> %tmp50, float undef, i32 3
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> <float 0x406E6CCCC0000000, float 0xC07A766660000000, float 0xC0608CCCC0000000, float 0xC063333340000000>, <4 x float>* undef
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp51 = load <4 x float>, <4 x float>* undef
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add740 = fadd <4 x float> undef, %tmp51
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp52 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecext741 = extractelement <4 x float> %tmp52, i32 0
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add742 = fadd float %vecext741, 0xC07984CCC0000000
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp53 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins743 = insertelement <4 x float> %tmp53, float %add742, i32 0
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %vecins743, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp54 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp55 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add754 = fadd <4 x float> %tmp55, undef
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %add754, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp56 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecext755 = extractelement <4 x float> %tmp56, i32 0
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add756 = fadd float %vecext755, 0xC070ACCCC0000000
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp57 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins757 = insertelement <4 x float> %tmp57, float %add756, i32 0
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add765 = fadd float undef, 0x405BA66660000000
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp58 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins766 = insertelement <4 x float> %tmp58, float %add765, i32 3
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp59 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecext769 = extractelement <4 x float> %tmp59, i32 0
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add770 = fadd float %vecext769, 0x40797199A0000000
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp60 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins771 = insertelement <4 x float> %tmp60, float %add770, i32 0
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %vecins771, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp61 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add776 = fadd float undef, 0xC055F33340000000
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins777 = insertelement <4 x float> undef, float %add776, i32 2
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp62 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp63 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add782 = fadd <4 x float> %tmp63, undef
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %add782, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp64 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecext783 = extractelement <4 x float> %tmp64, i32 0
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add784 = fadd float %vecext783, -3.455000e+02
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> <float 0xC07A866660000000, float 0xC05CF999A0000000, float 0xC0757199A0000000, float -3.845000e+02>, <4 x float>* undef
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add796 = fadd <4 x float> undef, undef
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %add796, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp65 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add801 = fadd float undef, 3.045000e+02
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp66 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins802 = insertelement <4 x float> %tmp66, float %add801, i32 1
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %vecins802, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecext803 = extractelement <4 x float> undef, i32 2
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> undef, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp67 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp68 = load <4 x float>, <4 x float>* undef
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add810 = fadd <4 x float> undef, %tmp68
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %add810, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp69 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecext811 = extractelement <4 x float> %tmp69, i32 0
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp70 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins813 = insertelement <4 x float> %tmp70, float undef, i32 0
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecext817 = extractelement <4 x float> undef, i32 2
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add818 = fadd float %vecext817, -4.830000e+02
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins822 = insertelement <4 x float> undef, float undef, i32 3
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %vecins822, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> <float 2.700000e+01, float 0xC05F666660000000, float 0xC07D0199A0000000, float 0x407A6CCCC0000000>, <4 x float>* undef
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp71 = load <4 x float>, <4 x float>* undef
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp72 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add824 = fadd <4 x float> %tmp72, %tmp71
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add838 = fadd <4 x float> undef, undef
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %add838, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp73 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecext839 = extractelement <4 x float> %tmp73, i32 0
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add849 = fadd float undef, 0xC07C266660000000
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> undef, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> <float 0xC07D566660000000, float 0xC06D233340000000, float 0x4068B33340000000, float 0xC07ADCCCC0000000>, <4 x float>* undef
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp74 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add852 = fadd <4 x float> %tmp74, undef
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecext856 = extractelement <4 x float> undef, i32 1
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add857 = fadd float %vecext856, 0xC070666660000000
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp75 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp76 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecext859 = extractelement <4 x float> %tmp76, i32 2
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add860 = fadd float %vecext859, 4.705000e+02
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp77 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins861 = insertelement <4 x float> %tmp77, float %add860, i32 2
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins889 = insertelement <4 x float> undef, float undef, i32 2
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %vecins889, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp78 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecext890 = extractelement <4 x float> %tmp78, i32 3
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add891 = fadd float %vecext890, 0xC070633340000000
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp79 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins892 = insertelement <4 x float> %tmp79, float %add891, i32 3
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %vecins892, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> <float 0x4063D33340000000, float 0xC076433340000000, float 0x407C966660000000, float 0xC07B5199A0000000>, <4 x float>* undef
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp80 = load <4 x float>, <4 x float>* undef
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp81 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add894 = fadd <4 x float> %tmp81, %tmp80
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %add894, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecext895 = extractelement <4 x float> undef, i32 0
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add896 = fadd float %vecext895, 0xC070F33340000000
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins897 = insertelement <4 x float> undef, float %add896, i32 0
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp82 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecext898 = extractelement <4 x float> %tmp82, i32 1
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add899 = fadd float %vecext898, 0xC076F33340000000
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins900 = insertelement <4 x float> undef, float %add899, i32 1
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp83 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecext901 = extractelement <4 x float> %tmp83, i32 2
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add902 = fadd float %vecext901, 0xC054ECCCC0000000
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp84 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins903 = insertelement <4 x float> %tmp84, float %add902, i32 2
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %vecins903, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecext904 = extractelement <4 x float> undef, i32 3
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add905 = fadd float %vecext904, 0x4056A66660000000
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp85 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins906 = insertelement <4 x float> %tmp85, float %add905, i32 3
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> <float 0xC07EFCCCC0000000, float 1.795000e+02, float 0x407E3E6660000000, float 0x4070633340000000>, <4 x float>* undef
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp86 = load <4 x float>, <4 x float>* undef
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp87 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add908 = fadd <4 x float> %tmp87, %tmp86
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %add908, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp88 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp89 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> undef, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp90 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecext912 = extractelement <4 x float> %tmp90, i32 1
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add913 = fadd float %vecext912, 2.575000e+02
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins914 = insertelement <4 x float> undef, float %add913, i32 1
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp91 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecext915 = extractelement <4 x float> %tmp91, i32 2
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add916 = fadd float %vecext915, -3.115000e+02
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp92 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins917 = insertelement <4 x float> %tmp92, float %add916, i32 2
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %vecins917, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp93 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecext918 = extractelement <4 x float> %tmp93, i32 3
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add919 = fadd float %vecext918, 2.950000e+01
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp94 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins920 = insertelement <4 x float> %tmp94, float %add919, i32 3
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %vecins920, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp95 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins925 = insertelement <4 x float> %tmp95, float undef, i32 0
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %vecins925, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp96 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add927 = fadd float undef, 0xC0501999A0000000
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp97 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins928 = insertelement <4 x float> %tmp97, float %add927, i32 1
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecext929 = extractelement <4 x float> undef, i32 2
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add930 = fadd float %vecext929, 0xC07C8B3340000000
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp98 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins931 = insertelement <4 x float> %tmp98, float %add930, i32 2
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> <float 0xC047B33340000000, float 0x404ACCCCC0000000, float 0x40708E6660000000, float 0x4060F999A0000000>, <4 x float>* undef
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp99 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp100 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecext937 = extractelement <4 x float> %tmp100, i32 0
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add941 = fadd float undef, -4.665000e+02
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins942 = insertelement <4 x float> undef, float %add941, i32 1
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %vecins942, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp101 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecext943 = extractelement <4 x float> %tmp101, i32 2
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add944 = fadd float %vecext943, 4.580000e+02
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp102 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins945 = insertelement <4 x float> %tmp102, float %add944, i32 2
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %vecins945, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp103 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add947 = fadd float undef, 0xC051933340000000
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp104 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins948 = insertelement <4 x float> %tmp104, float %add947, i32 3
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %vecins948, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> <float 0x4060CCCCC0000000, float 0xC07BAB3340000000, float 0xC061233340000000, float 0xC076C199A0000000>, <4 x float>* undef
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp105 = load <4 x float>, <4 x float>* undef
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add955 = fadd float undef, 0x4077F4CCC0000000
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp106 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins956 = insertelement <4 x float> %tmp106, float %add955, i32 1
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %vecins956, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecext971 = extractelement <4 x float> undef, i32 2
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add972 = fadd float %vecext971, 0x4024333340000000
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp107 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins973 = insertelement <4 x float> %tmp107, float %add972, i32 2
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %vecins973, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp108 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecext974 = extractelement <4 x float> %tmp108, i32 3
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins976 = insertelement <4 x float> undef, float undef, i32 3
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %vecins976, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> <float 0x407E266660000000, float -1.225000e+02, float 0x407EB199A0000000, float 0x407BA199A0000000>, <4 x float>* undef
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp109 = load <4 x float>, <4 x float>* undef
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp110 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add978 = fadd <4 x float> %tmp110, %tmp109
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp111 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp112 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> undef, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecext982 = extractelement <4 x float> undef, i32 1
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add983 = fadd float %vecext982, 0x407E1B3340000000
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins984 = insertelement <4 x float> undef, float %add983, i32 1
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %vecins984, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp113 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecext985 = extractelement <4 x float> %tmp113, i32 2
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add986 = fadd float %vecext985, 0x406C8CCCC0000000
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp114 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins987 = insertelement <4 x float> %tmp114, float %add986, i32 2
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %vecins987, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp115 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp116 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins995 = insertelement <4 x float> %tmp116, float undef, i32 0
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %vecins995, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp117 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add997 = fadd float undef, 0xC0798999A0000000
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp118 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins998 = insertelement <4 x float> %tmp118, float %add997, i32 1
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %vecins998, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp119 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecext1013 = extractelement <4 x float> undef, i32 2
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add1014 = fadd float %vecext1013, 3.105000e+02
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp120 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> undef, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp121 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecext1016 = extractelement <4 x float> %tmp121, i32 3
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add1017 = fadd float %vecext1016, 0x406A1999A0000000
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp122 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecext1030 = extractelement <4 x float> undef, i32 3
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add1031 = fadd float %vecext1030, 2.010000e+02
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> undef, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp123 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp124 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins1085 = insertelement <4 x float> %tmp124, float undef, i32 2
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp125 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecext1086 = extractelement <4 x float> %tmp125, i32 3
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add1087 = fadd float %vecext1086, -1.575000e+02
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp126 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins1088 = insertelement <4 x float> %tmp126, float %add1087, i32 3
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %vecins1088, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp127 = load <4 x float>, <4 x float>* undef
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add1090 = fadd <4 x float> undef, %tmp127
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> undef, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp128 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecext1094 = extractelement <4 x float> %tmp128, i32 1
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add1095 = fadd float %vecext1094, 0x4072C999A0000000
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp129 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins1096 = insertelement <4 x float> %tmp129, float %add1095, i32 1
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %vecins1096, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp130 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecext1097 = extractelement <4 x float> %tmp130, i32 2
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add1098 = fadd float %vecext1097, 0xC073E999A0000000
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp131 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins1099 = insertelement <4 x float> %tmp131, float %add1098, i32 2
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %vecins1099, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp132 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecext1100 = extractelement <4 x float> %tmp132, i32 3
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add1101 = fadd float %vecext1100, 2.885000e+02
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp133 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins1102 = insertelement <4 x float> %tmp133, float %add1101, i32 3
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %vecins1102, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> <float 0x4059866660000000, float 0x4072466660000000, float 0xC078FE6660000000, float 0xC058ACCCC0000000>, <4 x float>* undef
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp134 = load <4 x float>, <4 x float>* undef
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add1104 = fadd <4 x float> undef, %tmp134
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp135 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecext1105 = extractelement <4 x float> %tmp135, i32 0
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add1106 = fadd float %vecext1105, 0xC078A999A0000000
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp136 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins1107 = insertelement <4 x float> %tmp136, float %add1106, i32 0
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecext1108 = extractelement <4 x float> undef, i32 1
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp137 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins1110 = insertelement <4 x float> %tmp137, float undef, i32 1
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %vecins1110, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp138 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecext1111 = extractelement <4 x float> %tmp138, i32 2
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add1112 = fadd float %vecext1111, 0x407D566660000000
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp139 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins1113 = insertelement <4 x float> %tmp139, float %add1112, i32 2
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %vecins1113, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add1115 = fadd float undef, 0x4072B33340000000
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins1116 = insertelement <4 x float> undef, float %add1115, i32 3
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %vecins1116, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> <float 0xC0721999A0000000, float 0x4075633340000000, float 0x40794199A0000000, float 0x4061066660000000>, <4 x float>* undef
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp140 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add1118 = fadd <4 x float> %tmp140, undef
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %add1118, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp141 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecext1119 = extractelement <4 x float> %tmp141, i32 0
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add1120 = fadd float %vecext1119, 0xC065A66660000000
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins1121 = insertelement <4 x float> undef, float %add1120, i32 0
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %vecins1121, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp142 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecext1122 = extractelement <4 x float> %tmp142, i32 1
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add1123 = fadd float %vecext1122, 0x4072533340000000
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp143 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins1124 = insertelement <4 x float> %tmp143, float %add1123, i32 1
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecext1125 = extractelement <4 x float> undef, i32 2
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins1127 = insertelement <4 x float> undef, float undef, i32 2
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %vecins1127, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp144 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecext1128 = extractelement <4 x float> %tmp144, i32 3
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add1129 = fadd float %vecext1128, 0x405C866660000000
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp145 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins1130 = insertelement <4 x float> %tmp145, float %add1129, i32 3
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> <float 0xC06D6CCCC0000000, float 0xC032E66660000000, float -1.005000e+02, float 0x40765B3340000000>, <4 x float>* undef
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp146 = load <4 x float>, <4 x float>* undef
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp147 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add1132 = fadd <4 x float> %tmp147, %tmp146
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp148 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecext1133 = extractelement <4 x float> %tmp148, i32 0
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add1134 = fadd float %vecext1133, 0xC07EB999A0000000
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp149 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins1135 = insertelement <4 x float> %tmp149, float %add1134, i32 0
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %vecins1135, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp150 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecext1136 = extractelement <4 x float> %tmp150, i32 1
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp151 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins1138 = insertelement <4 x float> %tmp151, float undef, i32 1
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %vecins1138, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp152 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add1140 = fadd float undef, 0x407AE999A0000000
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp153 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins1141 = insertelement <4 x float> %tmp153, float %add1140, i32 2
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecext1142 = extractelement <4 x float> undef, i32 3
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add1143 = fadd float %vecext1142, 0x407A24CCC0000000
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp154 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins1144 = insertelement <4 x float> %tmp154, float %add1143, i32 3
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %vecins1144, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp155 = load <4 x float>, <4 x float>* undef
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp156 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add1146 = fadd <4 x float> %tmp156, %tmp155
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %add1146, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp157 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add1148 = fadd float undef, 4.145000e+02
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp158 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins1158 = insertelement <4 x float> undef, float undef, i32 3
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %vecins1158, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> <float 0x40603999A0000000, float -9.150000e+01, float 0xC051E66660000000, float -4.825000e+02>, <4 x float>* undef
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add1218 = fadd float undef, 0xC078733340000000
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins1219 = insertelement <4 x float> undef, float %add1218, i32 0
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> <float 0xC0655CCCC0000000, float -4.900000e+01, float -4.525000e+02, float 4.205000e+02>, <4 x float>* undef
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> undef, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp159 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecext1279 = extractelement <4 x float> %tmp159, i32 2
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add1280 = fadd float %vecext1279, 0xC062D999A0000000
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp160 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins1281 = insertelement <4 x float> %tmp160, float %add1280, i32 2
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %vecins1281, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp161 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecext1282 = extractelement <4 x float> %tmp161, i32 3
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add1283 = fadd float %vecext1282, 4.365000e+02
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp162 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins1284 = insertelement <4 x float> %tmp162, float %add1283, i32 3
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %vecins1284, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp163 = load <4 x float>, <4 x float>* undef
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp164 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add1286 = fadd <4 x float> %tmp164, %tmp163
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %add1286, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp165 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add1288 = fadd float undef, 0xC0731199A0000000
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp166 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> undef, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp167 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecext1444 = extractelement <4 x float> %tmp167, i32 1
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins1460 = insertelement <4 x float> undef, float undef, i32 1
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %vecins1460, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp168 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add1462 = fadd float undef, -1.670000e+02
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins1463 = insertelement <4 x float> undef, float %add1462, i32 2
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp169 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecext1464 = extractelement <4 x float> %tmp169, i32 3
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add1465 = fadd float %vecext1464, 0xC066333340000000
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp170 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins1466 = insertelement <4 x float> %tmp170, float %add1465, i32 3
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %vecins1466, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> <float 3.885000e+02, float 0x4054266660000000, float -9.500000e+01, float 8.500000e+01>, <4 x float>* undef
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp171 = load <4 x float>, <4 x float>* undef
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp172 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add1468 = fadd <4 x float> %tmp172, %tmp171
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %add1468, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp173 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add1470 = fadd float undef, 0x4033B33340000000
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp174 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins1471 = insertelement <4 x float> %tmp174, float %add1470, i32 0
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %vecins1471, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp175 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecext1472 = extractelement <4 x float> %tmp175, i32 1
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add1473 = fadd float %vecext1472, 0xC05F666660000000
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp176 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins1474 = insertelement <4 x float> %tmp176, float %add1473, i32 1
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp177 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecext1475 = extractelement <4 x float> %tmp177, i32 2
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp178 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins1477 = insertelement <4 x float> %tmp178, float undef, i32 2
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %vecins1477, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp179 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecext1478 = extractelement <4 x float> %tmp179, i32 3
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add1479 = fadd float %vecext1478, 0x407E2E6660000000
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp180 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins1480 = insertelement <4 x float> %tmp180, float %add1479, i32 3
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %vecins1480, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> <float 0xC061B33340000000, float 3.290000e+02, float 0xC067766660000000, float 0x407DB33340000000>, <4 x float>* undef
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp181 = load <4 x float>, <4 x float>* undef
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp182 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> undef, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp183 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecext1483 = extractelement <4 x float> %tmp183, i32 0
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add1484 = fadd float %vecext1483, 0xC053D999A0000000
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp184 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp185 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecext1486 = extractelement <4 x float> %tmp185, i32 1
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins1502 = insertelement <4 x float> undef, float undef, i32 1
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %vecins1502, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecext1503 = extractelement <4 x float> undef, i32 2
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add1504 = fadd float %vecext1503, -2.475000e+02
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp186 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins1505 = insertelement <4 x float> %tmp186, float %add1504, i32 2
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %vecins1505, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp187 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecext1506 = extractelement <4 x float> %tmp187, i32 3
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add1507 = fadd float %vecext1506, 0x40715199A0000000
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp188 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins1508 = insertelement <4 x float> %tmp188, float %add1507, i32 3
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %vecins1508, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> <float 0x40762B3340000000, float 0xC074566660000000, float 0xC07C74CCC0000000, float 0xC053F999A0000000>, <4 x float>* undef
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp189 = load <4 x float>, <4 x float>* undef
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp190 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add1510 = fadd <4 x float> %tmp190, %tmp189
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %add1510, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp191 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp192 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecext1654 = extractelement <4 x float> %tmp192, i32 1
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add1655 = fadd float %vecext1654, 0xC07D8CCCC0000000
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp193 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins1656 = insertelement <4 x float> %tmp193, float %add1655, i32 1
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %vecins1656, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add1658 = fadd float undef, 0x40709999A0000000
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp194 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> undef, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecext1660 = extractelement <4 x float> undef, i32 3
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add1661 = fadd float %vecext1660, 0xC06F166660000000
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp195 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins1662 = insertelement <4 x float> %tmp195, float %add1661, i32 3
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %vecins1662, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> <float 0xC075266660000000, float 0xC072C4CCC0000000, float 0x407C4E6660000000, float -4.485000e+02>, <4 x float>* undef
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins1676 = insertelement <4 x float> undef, float undef, i32 3
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> undef, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp196 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add1692 = fadd <4 x float> %tmp196, undef
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %add1692, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp197 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecext1693 = extractelement <4 x float> %tmp197, i32 0
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add1694 = fadd float %vecext1693, 0x407A1999A0000000
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp198 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins1695 = insertelement <4 x float> %tmp198, float %add1694, i32 0
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %vecins1695, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp199 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecext1696 = extractelement <4 x float> %tmp199, i32 1
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add1697 = fadd float %vecext1696, 2.850000e+01
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp200 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins1698 = insertelement <4 x float> %tmp200, float %add1697, i32 1
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %vecins1698, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp201 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecext1699 = extractelement <4 x float> %tmp201, i32 2
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp202 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins1701 = insertelement <4 x float> %tmp202, float undef, i32 2
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %vecins1701, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp203 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins1704 = insertelement <4 x float> undef, float undef, i32 3
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> <float 0xC075933340000000, float 0xC0489999A0000000, float 0xC078AB3340000000, float 0x406DFCCCC0000000>, <4 x float>* undef
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp204 = load <4 x float>, <4 x float>* undef
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp205 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp206 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins1709 = insertelement <4 x float> %tmp206, float undef, i32 0
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %vecins1709, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp207 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecext1713 = extractelement <4 x float> undef, i32 2
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add1714 = fadd float %vecext1713, 0xC0703199A0000000
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins1723 = insertelement <4 x float> undef, float undef, i32 0
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp208 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> undef, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecext1730 = extractelement <4 x float> undef, i32 3
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add1731 = fadd float %vecext1730, 4.130000e+02
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp209 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins1732 = insertelement <4 x float> %tmp209, float %add1731, i32 3
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %vecins1732, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> <float 0x40551999A0000000, float 0xC0708999A0000000, float 0xC054F33340000000, float 0xC07C5999A0000000>, <4 x float>* undef
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp210 = load <4 x float>, <4 x float>* undef
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add1734 = fadd <4 x float> undef, %tmp210
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp211 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add1736 = fadd float undef, 0x407C3999A0000000
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp212 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins1737 = insertelement <4 x float> %tmp212, float %add1736, i32 0
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp213 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecext1738 = extractelement <4 x float> %tmp213, i32 1
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add1739 = fadd float %vecext1738, 0xC0711E6660000000
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp214 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins1740 = insertelement <4 x float> %tmp214, float %add1739, i32 1
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %vecins1740, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp215 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecext1741 = extractelement <4 x float> %tmp215, i32 2
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add1742 = fadd float %vecext1741, -2.545000e+02
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp216 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins1743 = insertelement <4 x float> %tmp216, float %add1742, i32 2
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %vecins1743, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecext1744 = extractelement <4 x float> undef, i32 3
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp217 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins1746 = insertelement <4 x float> %tmp217, float undef, i32 3
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> <float 0xC076466660000000, float 0x4060BCCCC0000000, float 0x405EF999A0000000, float 0x4074766660000000>, <4 x float>* undef
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp218 = load <4 x float>, <4 x float>* undef
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add1748 = fadd <4 x float> undef, %tmp218
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %add1748, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp219 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add1750 = fadd float undef, 0x407C6B3340000000
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins1751 = insertelement <4 x float> undef, float %add1750, i32 0
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp220 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecext1752 = extractelement <4 x float> %tmp220, i32 1
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add1753 = fadd float %vecext1752, 0x40730CCCC0000000
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp221 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins1754 = insertelement <4 x float> %tmp221, float %add1753, i32 1
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp222 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecext1755 = extractelement <4 x float> %tmp222, i32 2
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add1756 = fadd float %vecext1755, 0xC059F33340000000
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp223 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add1759 = fadd float undef, 0x40678999A0000000
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp224 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins1760 = insertelement <4 x float> %tmp224, float %add1759, i32 3
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %vecins1760, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> <float 0x405E333340000000, float 0x40571999A0000000, float 0xC02E333340000000, float 0x4053A66660000000>, <4 x float>* undef
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp225 = load <4 x float>, <4 x float>* undef
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add1762 = fadd <4 x float> undef, %tmp225
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %add1762, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp226 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecext1763 = extractelement <4 x float> %tmp226, i32 0
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add1764 = fadd float %vecext1763, 0xC0299999A0000000
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp227 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins1765 = insertelement <4 x float> %tmp227, float %add1764, i32 0
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %vecins1765, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp228 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecext1766 = extractelement <4 x float> %tmp228, i32 1
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add1767 = fadd float %vecext1766, 0x407DDE6660000000
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp229 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins1768 = insertelement <4 x float> %tmp229, float %add1767, i32 1
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %vecins1768, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecext1769 = extractelement <4 x float> undef, i32 2
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add1770 = fadd float %vecext1769, 0x407A1B3340000000
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp230 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins1771 = insertelement <4 x float> %tmp230, float %add1770, i32 2
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %vecins1771, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp231 = load <4 x float>, <4 x float>* undef
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp232 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp233 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp234 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins1779 = insertelement <4 x float> %tmp234, float undef, i32 0
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %vecins1779, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp235 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> undef, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp236 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecext1783 = extractelement <4 x float> %tmp236, i32 2
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add1784 = fadd float %vecext1783, 0x405E933340000000
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins1785 = insertelement <4 x float> undef, float %add1784, i32 2
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %vecins1785, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> <float 0xC07074CCC0000000, float 0xC04D666660000000, float 3.235000e+02, float 0xC0724199A0000000>, <4 x float>* undef
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp237 = load <4 x float>, <4 x float>* undef
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add1790 = fadd <4 x float> undef, %tmp237
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp238 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecext1791 = extractelement <4 x float> %tmp238, i32 0
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add1792 = fadd float %vecext1791, 0x4077DE6660000000
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp239 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins1793 = insertelement <4 x float> %tmp239, float %add1792, i32 0
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %vecins1793, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp240 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add1795 = fadd float undef, 0x4055266660000000
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp241 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins1796 = insertelement <4 x float> %tmp241, float %add1795, i32 1
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins1799 = insertelement <4 x float> undef, float undef, i32 2
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecext1800 = extractelement <4 x float> undef, i32 3
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp242 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> undef, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> <float -6.600000e+01, float 0xC07B2199A0000000, float 0x4011333340000000, float 0xC0635CCCC0000000>, <4 x float>* undef
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp243 = load <4 x float>, <4 x float>* undef
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp244 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp245 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp246 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add1865 = fadd float undef, -2.235000e+02
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp247 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins1866 = insertelement <4 x float> %tmp247, float %add1865, i32 1
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp248 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp249 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins1872 = insertelement <4 x float> %tmp249, float undef, i32 3
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> <float 0x406B8999A0000000, float 0xC0696CCCC0000000, float 0xC07A34CCC0000000, float 0x407654CCC0000000>, <4 x float>* undef
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp250 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add1874 = fadd <4 x float> %tmp250, undef
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %add1874, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecext1875 = extractelement <4 x float> undef, i32 0
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp251 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins1894 = insertelement <4 x float> %tmp251, float undef, i32 1
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp252 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecext1895 = extractelement <4 x float> %tmp252, i32 2
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins1900 = insertelement <4 x float> undef, float undef, i32 3
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %vecins1900, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins1905 = insertelement <4 x float> undef, float undef, i32 0
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %vecins1905, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp253 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecext1906 = extractelement <4 x float> %tmp253, i32 1
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add1907 = fadd float %vecext1906, 0xC07E5E6660000000
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins1908 = insertelement <4 x float> undef, float %add1907, i32 1
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %vecins1908, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecext1909 = extractelement <4 x float> undef, i32 2
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp254 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecext1912 = extractelement <4 x float> %tmp254, i32 3
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add1913 = fadd float %vecext1912, 0xC063ECCCC0000000
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp255 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp256 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add1916 = fadd <4 x float> %tmp256, undef
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %add1916, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> undef, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecext1923 = extractelement <4 x float> undef, i32 2
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp257 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add1927 = fadd float undef, 0x40761999A0000000
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp258 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins1928 = insertelement <4 x float> %tmp258, float %add1927, i32 3
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %vecins1928, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> <float 7.100000e+01, float 0xC0634999A0000000, float 0x407B0B3340000000, float 0xC07DE999A0000000>, <4 x float>* undef
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp259 = load <4 x float>, <4 x float>* undef
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp260 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add1930 = fadd <4 x float> %tmp260, %tmp259
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp261 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp262 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins1933 = insertelement <4 x float> %tmp262, float undef, i32 0
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %vecins1933, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp263 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecext1934 = extractelement <4 x float> %tmp263, i32 1
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add1935 = fadd float %vecext1934, 0xC07D3199A0000000
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp264 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins1936 = insertelement <4 x float> %tmp264, float %add1935, i32 1
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecext1940 = extractelement <4 x float> undef, i32 3
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins1942 = insertelement <4 x float> undef, float undef, i32 3
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> <float -8.200000e+01, float 0xC04C733340000000, float 0xC077ACCCC0000000, float 0x4074566660000000>, <4 x float>* undef
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp265 = load <4 x float>, <4 x float>* undef
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp266 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> undef, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp267 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecext1945 = extractelement <4 x float> %tmp267, i32 0
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add1946 = fadd float %vecext1945, 0xC074866660000000
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins1953 = insertelement <4 x float> undef, float undef, i32 2
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %vecins1953, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp268 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> undef, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp269 = load <4 x float>, <4 x float>* undef
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp270 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add1958 = fadd <4 x float> %tmp270, %tmp269
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp271 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecext1959 = extractelement <4 x float> %tmp271, i32 0
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add1960 = fadd float %vecext1959, 0x4065ACCCC0000000
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecext1962 = extractelement <4 x float> undef, i32 1
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add1963 = fadd float %vecext1962, 0xC07134CCC0000000
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp272 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins1964 = insertelement <4 x float> %tmp272, float %add1963, i32 1
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %vecins1964, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecext1965 = extractelement <4 x float> undef, i32 2
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp273 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins1967 = insertelement <4 x float> %tmp273, float undef, i32 2
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %vecins1967, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp274 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecext1968 = extractelement <4 x float> %tmp274, i32 3
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add1969 = fadd float %vecext1968, 7.100000e+01
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp275 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins1970 = insertelement <4 x float> %tmp275, float %add1969, i32 3
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %vecins1970, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> <float 0x402E9999A0000000, float 0x407344CCC0000000, float -4.165000e+02, float 0x4078FCCCC0000000>, <4 x float>* undef
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp276 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp277 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp278 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins1975 = insertelement <4 x float> %tmp278, float undef, i32 0
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %vecins1975, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp279 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecext1976 = extractelement <4 x float> %tmp279, i32 1
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins1978 = insertelement <4 x float> undef, float undef, i32 1
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %vecins1978, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecext1979 = extractelement <4 x float> undef, i32 2
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins1981 = insertelement <4 x float> undef, float undef, i32 2
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %vecins1981, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins1984 = insertelement <4 x float> undef, float undef, i32 3
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %vecins1984, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> <float 0xC06A766660000000, float 0xC07CE4CCC0000000, float -1.055000e+02, float 0x40786E6660000000>, <4 x float>* undef
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> undef, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecext1990 = extractelement <4 x float> undef, i32 1
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecext1996 = extractelement <4 x float> undef, i32 3
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add1997 = fadd float %vecext1996, -1.400000e+02
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp280 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins1998 = insertelement <4 x float> %tmp280, float %add1997, i32 3
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %vecins1998, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> <float 0xC0794E6660000000, float 0xC073CCCCC0000000, float 0x407994CCC0000000, float 6.500000e+01>, <4 x float>* undef
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> undef, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecext2004 = extractelement <4 x float> undef, i32 1
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add2005 = fadd float %vecext2004, -1.970000e+02
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp281 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins2006 = insertelement <4 x float> %tmp281, float %add2005, i32 1
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %vecins2006, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp282 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecext2007 = extractelement <4 x float> %tmp282, i32 2
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp283 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins2009 = insertelement <4 x float> %tmp283, float undef, i32 2
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp284 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecext2010 = extractelement <4 x float> %tmp284, i32 3
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add2011 = fadd float %vecext2010, 0xC074533340000000
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp285 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins2012 = insertelement <4 x float> %tmp285, float %add2011, i32 3
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %vecins2012, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> <float 0xC04E733340000000, float 0xC074566660000000, float 0x4079F66660000000, float 0xC0705B3340000000>, <4 x float>* undef
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp286 = load <4 x float>, <4 x float>* undef
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp287 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> undef, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp288 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecext2015 = extractelement <4 x float> %tmp288, i32 0
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add2016 = fadd float %vecext2015, 0xC060633340000000
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp289 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins2017 = insertelement <4 x float> %tmp289, float %add2016, i32 0
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add2022 = fadd float undef, 8.350000e+01
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp290 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins2023 = insertelement <4 x float> %tmp290, float %add2022, i32 2
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp291 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecext2024 = extractelement <4 x float> %tmp291, i32 3
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp292 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add2028 = fadd <4 x float> %tmp292, undef
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %add2028, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecext2029 = extractelement <4 x float> undef, i32 0
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add2030 = fadd float %vecext2029, -9.450000e+01
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp293 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> undef, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp294 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add2036 = fadd float undef, 0x407DE66660000000
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp295 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp296 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp297 = load <4 x float>, <4 x float>* undef
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp298 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp299 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins2045 = insertelement <4 x float> %tmp299, float undef, i32 0
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %vecins2045, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp300 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecext2046 = extractelement <4 x float> %tmp300, i32 1
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add2047 = fadd float %vecext2046, 0xC065433340000000
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> undef, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecext2052 = extractelement <4 x float> undef, i32 3
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp301 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins2054 = insertelement <4 x float> %tmp301, float undef, i32 3
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %vecins2054, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> <float 0x4024666660000000, float 0x4079366660000000, float 0x40721B3340000000, float 0x406E533340000000>, <4 x float>* undef
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp302 = load <4 x float>, <4 x float>* undef
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add2056 = fadd <4 x float> undef, %tmp302
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %add2056, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp303 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp304 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins2062 = insertelement <4 x float> %tmp304, float undef, i32 1
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %vecins2062, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp305 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> undef, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp306 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecext2066 = extractelement <4 x float> %tmp306, i32 3
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add2067 = fadd float %vecext2066, 0x40690999A0000000
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins2068 = insertelement <4 x float> undef, float %add2067, i32 3
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %vecins2068, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> <float 0xC07EFCCCC0000000, float -3.420000e+02, float 0xC07BC999A0000000, float 0x40751999A0000000>, <4 x float>* undef
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp307 = load <4 x float>, <4 x float>* undef
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp308 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add2070 = fadd <4 x float> %tmp308, %tmp307
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %add2070, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp309 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecext2071 = extractelement <4 x float> %tmp309, i32 0
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add2072 = fadd float %vecext2071, 0x4057733340000000
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp310 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins2073 = insertelement <4 x float> %tmp310, float %add2072, i32 0
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %vecins2073, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp311 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecext2074 = extractelement <4 x float> %tmp311, i32 1
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp312 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins2076 = insertelement <4 x float> %tmp312, float undef, i32 1
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp313 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecext2077 = extractelement <4 x float> %tmp313, i32 2
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add2078 = fadd float %vecext2077, 0x4061F999A0000000
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp314 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins2079 = insertelement <4 x float> %tmp314, float %add2078, i32 2
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %vecins2079, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp315 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecext2080 = extractelement <4 x float> %tmp315, i32 3
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add2081 = fadd float %vecext2080, 0x407A1B3340000000
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp316 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins2082 = insertelement <4 x float> %tmp316, float %add2081, i32 3
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %vecins2082, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> <float 0x40764E6660000000, float 0x40501999A0000000, float 0xC079A4CCC0000000, float 0x4050533340000000>, <4 x float>* undef
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp317 = load <4 x float>, <4 x float>* undef
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp318 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> undef, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp319 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecext2085 = extractelement <4 x float> %tmp319, i32 0
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add2086 = fadd float %vecext2085, 0x406E666660000000
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins2087 = insertelement <4 x float> undef, float %add2086, i32 0
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %vecins2087, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecext2480 = extractelement <4 x float> undef, i32 1
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add2481 = fadd float %vecext2480, 0x4039666660000000
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecext2483 = extractelement <4 x float> undef, i32 2
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add2484 = fadd float %vecext2483, 0xC06A3999A0000000
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp320 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins2485 = insertelement <4 x float> %tmp320, float %add2484, i32 2
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %vecins2485, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp321 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add2487 = fadd float undef, 2.030000e+02
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp322 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> undef, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> <float 0x4073DE6660000000, float 0x4067CCCCC0000000, float 0xC03F1999A0000000, float 4.350000e+01>, <4 x float>* undef
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecext2491 = extractelement <4 x float> undef, i32 0
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp323 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> undef, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp324 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecext2494 = extractelement <4 x float> %tmp324, i32 1
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add2495 = fadd float %vecext2494, 0xC0743CCCC0000000
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp325 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins2499 = insertelement <4 x float> undef, float undef, i32 2
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %vecins2499, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecext2500 = extractelement <4 x float> undef, i32 3
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add2501 = fadd float %vecext2500, 0x40796E6660000000
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp326 = load <4 x float>, <4 x float>* undef
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp327 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecext2508 = extractelement <4 x float> undef, i32 1
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add2518 = fadd <4 x float> undef, undef
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp328 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecext2519 = extractelement <4 x float> %tmp328, i32 0
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add2520 = fadd float %vecext2519, 0xC0399999A0000000
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp329 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add2534 = fadd float undef, 0x4072C66660000000
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecext2536 = extractelement <4 x float> undef, i32 1
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add2537 = fadd float %vecext2536, 0x407D066660000000
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp330 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins2538 = insertelement <4 x float> %tmp330, float %add2537, i32 1
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %vecins2538, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecext2539 = extractelement <4 x float> undef, i32 2
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add2540 = fadd float %vecext2539, 0x406F9999A0000000
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins2580 = insertelement <4 x float> undef, float undef, i32 1
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %vecins2580, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp331 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecext2581 = extractelement <4 x float> %tmp331, i32 2
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add2582 = fadd float %vecext2581, 0x406BE66660000000
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins2583 = insertelement <4 x float> undef, float %add2582, i32 2
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %vecins2583, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecext2584 = extractelement <4 x float> undef, i32 3
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add2585 = fadd float %vecext2584, 3.585000e+02
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp332 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> undef, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> <float 0x40773199A0000000, float 0x407D7999A0000000, float 0xC0717199A0000000, float 0xC07E9CCCC0000000>, <4 x float>* undef
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add2590 = fadd float undef, 0x407B1999A0000000
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp333 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp334 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> undef, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add2672 = fadd <4 x float> undef, undef
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %add2672, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp335 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecext2676 = extractelement <4 x float> undef, i32 1
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add2677 = fadd float %vecext2676, 0x406D6999A0000000
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp336 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins2678 = insertelement <4 x float> %tmp336, float %add2677, i32 1
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %vecins2678, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp337 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecext2679 = extractelement <4 x float> %tmp337, i32 2
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins2681 = insertelement <4 x float> undef, float undef, i32 2
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %vecins2681, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp338 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecext2682 = extractelement <4 x float> %tmp338, i32 3
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins2684 = insertelement <4 x float> undef, float undef, i32 3
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp339 = load <4 x float>, <4 x float>* undef
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp340 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> undef, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp341 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add2688 = fadd float undef, 0x4063266660000000
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> undef, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins2692 = insertelement <4 x float> undef, float undef, i32 1
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %vecins2692, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp342 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecext2696 = extractelement <4 x float> undef, i32 3
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add2697 = fadd float %vecext2696, 4.140000e+02
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp343 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins2698 = insertelement <4 x float> %tmp343, float %add2697, i32 3
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %vecins2698, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> <float 0x40547999A0000000, float 0xC060633340000000, float 0x4075766660000000, float 0x4072D33340000000>, <4 x float>* undef
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp344 = load <4 x float>, <4 x float>* undef
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp345 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add2700 = fadd <4 x float> %tmp345, %tmp344
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %add2700, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp346 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp347 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp348 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecext2704 = extractelement <4 x float> %tmp348, i32 1
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add2705 = fadd float %vecext2704, 4.700000e+02
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp349 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> undef, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecext3121 = extractelement <4 x float> undef, i32 0
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add3125 = fadd float undef, 0xC06F266660000000
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins3126 = insertelement <4 x float> undef, float %add3125, i32 1
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %vecins3126, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp350 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecext3127 = extractelement <4 x float> %tmp350, i32 2
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add3128 = fadd float %vecext3127, 0x40638999A0000000
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp351 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins3129 = insertelement <4 x float> %tmp351, float %add3128, i32 2
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %vecins3129, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp352 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add3131 = fadd float undef, 3.215000e+02
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp353 = load <4 x float>, <4 x float>* undef
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp354 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add3134 = fadd <4 x float> %tmp354, %tmp353
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %add3134, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp355 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add3136 = fadd float undef, 0x4074333340000000
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins3140 = insertelement <4 x float> undef, float undef, i32 1
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %vecins3140, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp356 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecext3141 = extractelement <4 x float> %tmp356, i32 2
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add3142 = fadd float %vecext3141, 2.425000e+02
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp357 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins3143 = insertelement <4 x float> %tmp357, float %add3142, i32 2
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %vecins3143, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp358 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecext3144 = extractelement <4 x float> %tmp358, i32 3
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add3145 = fadd float %vecext3144, -3.760000e+02
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp359 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins3146 = insertelement <4 x float> %tmp359, float %add3145, i32 3
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %vecins3146, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp360 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins3272 = insertelement <4 x float> undef, float undef, i32 3
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %vecins3272, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> <float 0x407B4999A0000000, float 0x40695CCCC0000000, float 0xC05C0CCCC0000000, float 0x407EB33340000000>, <4 x float>* undef
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp361 = load <4 x float>, <4 x float>* undef
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp362 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add3274 = fadd <4 x float> %tmp362, %tmp361
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %add3274, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp363 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecext3275 = extractelement <4 x float> %tmp363, i32 0
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add3276 = fadd float %vecext3275, 0x4058066660000000
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp364 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins3277 = insertelement <4 x float> %tmp364, float %add3276, i32 0
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %vecins3277, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp365 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecext3278 = extractelement <4 x float> %tmp365, i32 1
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add3279 = fadd float %vecext3278, 0xC053666660000000
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins3280 = insertelement <4 x float> undef, float %add3279, i32 1
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %vecins3280, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp366 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecext3281 = extractelement <4 x float> %tmp366, i32 2
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add3282 = fadd float %vecext3281, 0xC0650CCCC0000000
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp367 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins3283 = insertelement <4 x float> %tmp367, float %add3282, i32 2
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %vecins3283, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp368 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecext3284 = extractelement <4 x float> %tmp368, i32 3
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add3285 = fadd float %vecext3284, 0x4062533340000000
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins3286 = insertelement <4 x float> undef, float %add3285, i32 3
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp369 = load <4 x float>, <4 x float>* undef
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> undef, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp370 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecext3289 = extractelement <4 x float> %tmp370, i32 0
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add3290 = fadd float %vecext3289, 0xC07E133340000000
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp371 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins3291 = insertelement <4 x float> %tmp371, float %add3290, i32 0
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %vecins3291, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecext3292 = extractelement <4 x float> undef, i32 1
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp372 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp373 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins3328 = insertelement <4 x float> %tmp373, float undef, i32 3
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add3330 = fadd <4 x float> undef, undef
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %add3330, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecext3331 = extractelement <4 x float> undef, i32 0
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add3332 = fadd float %vecext3331, 0x4061633340000000
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp374 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins3333 = insertelement <4 x float> %tmp374, float %add3332, i32 0
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %vecins3333, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecext3334 = extractelement <4 x float> undef, i32 1
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add3335 = fadd float %vecext3334, 0x401B333340000000
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins3336 = insertelement <4 x float> undef, float %add3335, i32 1
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp375 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecext3337 = extractelement <4 x float> %tmp375, i32 2
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add3338 = fadd float %vecext3337, 0x403C4CCCC0000000
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp376 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins3339 = insertelement <4 x float> %tmp376, float %add3338, i32 2
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %vecins3339, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp377 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecext3340 = extractelement <4 x float> %tmp377, i32 3
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp378 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins3342 = insertelement <4 x float> %tmp378, float undef, i32 3
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp379 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add3344 = fadd <4 x float> %tmp379, undef
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %add3344, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp380 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecext3345 = extractelement <4 x float> %tmp380, i32 0
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add3346 = fadd float %vecext3345, 0x407E7E6660000000
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp381 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins3347 = insertelement <4 x float> %tmp381, float %add3346, i32 0
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecext3348 = extractelement <4 x float> undef, i32 1
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add3349 = fadd float %vecext3348, 0xC05F666660000000
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp382 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins3350 = insertelement <4 x float> %tmp382, float %add3349, i32 1
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %vecins3350, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add3352 = fadd float undef, 0xC06ACCCCC0000000
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp383 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins3423 = insertelement <4 x float> undef, float undef, i32 2
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %vecins3423, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecext3424 = extractelement <4 x float> undef, i32 3
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add3425 = fadd float %vecext3424, 0xC05DB33340000000
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp384 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins3426 = insertelement <4 x float> %tmp384, float %add3425, i32 3
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %vecins3426, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> <float 2.795000e+02, float -4.065000e+02, float 0xC05CD999A0000000, float 1.825000e+02>, <4 x float>* undef
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp385 = load <4 x float>, <4 x float>* undef
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp386 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add3428 = fadd <4 x float> %tmp386, %tmp385
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp387 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecext3429 = extractelement <4 x float> %tmp387, i32 0
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add3430 = fadd float %vecext3429, 0x40695CCCC0000000
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp388 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins3431 = insertelement <4 x float> %tmp388, float %add3430, i32 0
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %vecins3431, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp389 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecext3432 = extractelement <4 x float> %tmp389, i32 1
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add3433 = fadd float %vecext3432, 0x4052A66660000000
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp390 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins3434 = insertelement <4 x float> %tmp390, float %add3433, i32 1
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %vecins3434, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecext3435 = extractelement <4 x float> undef, i32 2
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp391 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins3437 = insertelement <4 x float> %tmp391, float undef, i32 2
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %vecins3437, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp392 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecext3438 = extractelement <4 x float> %tmp392, i32 3
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add3439 = fadd float %vecext3438, 0xC071D999A0000000
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> <float 0xC0798199A0000000, float -3.385000e+02, float 0xC050066660000000, float 0xC075E999A0000000>, <4 x float>* undef
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp393 = load <4 x float>, <4 x float>* undef
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp394 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add3442 = fadd <4 x float> %tmp394, %tmp393
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %add3442, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecext3443 = extractelement <4 x float> undef, i32 0
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add3444 = fadd float %vecext3443, 0xC07CF999A0000000
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp395 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecext3446 = extractelement <4 x float> %tmp395, i32 1
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add3447 = fadd float %vecext3446, 0xC06E4999A0000000
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp396 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins3448 = insertelement <4 x float> %tmp396, float %add3447, i32 1
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %vecins3448, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp397 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecext3449 = extractelement <4 x float> %tmp397, i32 2
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add3450 = fadd float %vecext3449, 0x40779B3340000000
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp398 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins3451 = insertelement <4 x float> %tmp398, float %add3450, i32 2
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %vecins3451, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add3453 = fadd float undef, 0xC07ADCCCC0000000
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp399 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins3454 = insertelement <4 x float> %tmp399, float %add3453, i32 3
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %vecins3454, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp400 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecext3457 = extractelement <4 x float> %tmp400, i32 0
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add3458 = fadd float %vecext3457, -4.440000e+02
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins3459 = insertelement <4 x float> undef, float %add3458, i32 0
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %vecins3459, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp401 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecext3460 = extractelement <4 x float> %tmp401, i32 1
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp402 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins3462 = insertelement <4 x float> %tmp402, float undef, i32 1
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %vecins3462, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp403 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add3464 = fadd float undef, 0xC057B999A0000000
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp404 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins3465 = insertelement <4 x float> %tmp404, float %add3464, i32 2
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %vecins3465, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp405 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecext3466 = extractelement <4 x float> %tmp405, i32 3
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add3467 = fadd float %vecext3466, 0xC07A9CCCC0000000
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp406 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> undef, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> <float 0x405C3999A0000000, float 0xC07C6B3340000000, float 0x407ACB3340000000, float 0xC06E0999A0000000>, <4 x float>* undef
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp407 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> undef, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp408 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecext3477 = extractelement <4 x float> %tmp408, i32 2
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins3479 = insertelement <4 x float> undef, float undef, i32 2
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %vecins3479, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecext3480 = extractelement <4 x float> undef, i32 3
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add3481 = fadd float %vecext3480, 0xC053F33340000000
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp409 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins3482 = insertelement <4 x float> %tmp409, float %add3481, i32 3
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %vecins3482, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> <float 3.565000e+02, float 0xC0464CCCC0000000, float 0x4037666660000000, float 0xC0788CCCC0000000>, <4 x float>* undef
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp410 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add3484 = fadd <4 x float> %tmp410, undef
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %add3484, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp411 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add3486 = fadd float undef, -1.415000e+02
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins3487 = insertelement <4 x float> undef, float %add3486, i32 0
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %vecins3487, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp412 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecext3488 = extractelement <4 x float> %tmp412, i32 1
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add3489 = fadd float %vecext3488, 0x405A1999A0000000
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp413 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins3490 = insertelement <4 x float> %tmp413, float %add3489, i32 1
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %vecins3490, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add3492 = fadd float undef, 0x4078066660000000
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp414 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins3493 = insertelement <4 x float> %tmp414, float %add3492, i32 2
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %vecins3493, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp415 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add3495 = fadd float undef, 0xC0798999A0000000
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp416 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins3496 = insertelement <4 x float> %tmp416, float %add3495, i32 3
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %vecins3496, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp417 = load <4 x float>, <4 x float>* undef
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp418 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add3498 = fadd <4 x float> %tmp418, %tmp417
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %add3498, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecext3499 = extractelement <4 x float> undef, i32 0
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add3500 = fadd float %vecext3499, -1.605000e+02
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins3501 = insertelement <4 x float> undef, float %add3500, i32 0
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp419 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecext3502 = extractelement <4 x float> %tmp419, i32 1
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add3503 = fadd float %vecext3502, 0x4058C66660000000
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp420 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add3506 = fadd float undef, 0xC074DB3340000000
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp421 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins3507 = insertelement <4 x float> %tmp421, float %add3506, i32 2
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %vecins3507, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add3509 = fadd float undef, 0xC066033340000000
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp422 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> undef, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> <float 0x404B333340000000, float 4.680000e+02, float 0x40577999A0000000, float 0xC07D9999A0000000>, <4 x float>* undef
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp423 = load <4 x float>, <4 x float>* undef
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> undef, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecext3513 = extractelement <4 x float> undef, i32 0
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add3514 = fadd float %vecext3513, 2.300000e+02
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp424 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp425 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecext3516 = extractelement <4 x float> %tmp425, i32 1
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins5414 = insertelement <4 x float> undef, float undef, i32 3
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %vecins5414, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp426 = load <4 x float>, <4 x float>* undef
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp427 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add5416 = fadd <4 x float> %tmp427, %tmp426
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %add5416, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp428 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add5418 = fadd float undef, 0xC07ED999A0000000
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp429 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins5419 = insertelement <4 x float> %tmp429, float %add5418, i32 0
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins5624 = insertelement <4 x float> undef, float undef, i32 3
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %vecins5624, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> <float 0xC07B4999A0000000, float 0x4078B33340000000, float 0xC07674CCC0000000, float 0xC07C533340000000>, <4 x float>* undef
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add5626 = fadd <4 x float> undef, undef
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %add5626, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecext5627 = extractelement <4 x float> undef, i32 0
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp430 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins5629 = insertelement <4 x float> %tmp430, float undef, i32 0
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %vecins5629, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp431 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecext5630 = extractelement <4 x float> %tmp431, i32 1
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add5631 = fadd float %vecext5630, 0x405EECCCC0000000
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins5632 = insertelement <4 x float> undef, float %add5631, i32 1
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %vecins5632, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp432 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins5688 = insertelement <4 x float> %tmp432, float undef, i32 1
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %vecins5688, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp433 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecext5689 = extractelement <4 x float> %tmp433, i32 2
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp434 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins5691 = insertelement <4 x float> %tmp434, float undef, i32 2
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %vecins5691, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecext5692 = extractelement <4 x float> undef, i32 3
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> <float -4.350000e+02, float 0xC0775CCCC0000000, float 0xC0714999A0000000, float 0xC0661999A0000000>, <4 x float>* undef
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp435 = load <4 x float>, <4 x float>* undef
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add5696 = fadd <4 x float> undef, %tmp435
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %add5696, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add5701 = fadd float undef, 0x4077D4CCC0000000
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp436 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins5702 = insertelement <4 x float> %tmp436, float %add5701, i32 1
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %vecins5702, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp437 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp438 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins5705 = insertelement <4 x float> %tmp438, float undef, i32 2
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %vecins5705, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp439 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecext5706 = extractelement <4 x float> %tmp439, i32 3
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add5707 = fadd float %vecext5706, 0xC0780B3340000000
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp440 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins5708 = insertelement <4 x float> %tmp440, float %add5707, i32 3
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %vecins5708, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> <float 0x405D666660000000, float 0xC069333340000000, float 0x407B6B3340000000, float 0xC06EB33340000000>, <4 x float>* undef
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp441 = load <4 x float>, <4 x float>* undef
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp442 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add5710 = fadd <4 x float> %tmp442, %tmp441
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %add5710, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp443 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecext5711 = extractelement <4 x float> %tmp443, i32 0
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add5712 = fadd float %vecext5711, 1.850000e+01
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp444 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins5713 = insertelement <4 x float> %tmp444, float %add5712, i32 0
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %vecins5713, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp445 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp446 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins5716 = insertelement <4 x float> %tmp446, float undef, i32 1
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp447 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add5724 = fadd <4 x float> %tmp447, undef
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %add5724, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp448 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecext5748 = extractelement <4 x float> undef, i32 3
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp449 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins5750 = insertelement <4 x float> %tmp449, float undef, i32 3
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> <float 0x40692999A0000000, float 0xC07C4CCCC0000000, float 0x407D1E6660000000, float 0x407B4199A0000000>, <4 x float>* undef
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp450 = load <4 x float>, <4 x float>* undef
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add5752 = fadd <4 x float> undef, %tmp450
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add5754 = fadd float undef, 0xC064033340000000
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp451 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins5755 = insertelement <4 x float> %tmp451, float %add5754, i32 0
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %vecins5755, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp452 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecext5756 = extractelement <4 x float> %tmp452, i32 1
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add5757 = fadd float %vecext5756, 0x40787B3340000000
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp453 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins5758 = insertelement <4 x float> %tmp453, float %add5757, i32 1
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %vecins5758, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp454 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecext5759 = extractelement <4 x float> %tmp454, i32 2
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp455 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins5761 = insertelement <4 x float> %tmp455, float undef, i32 2
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %vecins5761, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp456 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecext5762 = extractelement <4 x float> %tmp456, i32 3
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add5763 = fadd float %vecext5762, 0x40703E6660000000
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp457 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins5764 = insertelement <4 x float> %tmp457, float %add5763, i32 3
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %vecins5764, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> <float 0x407A6B3340000000, float 0x40470CCCC0000000, float 0xC076F4CCC0000000, float 0x40791999A0000000>, <4 x float>* undef
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add5766 = fadd <4 x float> undef, undef
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %add5766, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp458 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecext5767 = extractelement <4 x float> %tmp458, i32 0
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add5768 = fadd float %vecext5767, 0x4065533340000000
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp459 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins5769 = insertelement <4 x float> %tmp459, float %add5768, i32 0
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %vecins5769, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add5771 = fadd float undef, 8.000000e+00
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp460 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins5772 = insertelement <4 x float> %tmp460, float %add5771, i32 1
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp461 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add5796 = fadd float undef, 0x4058ECCCC0000000
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins5797 = insertelement <4 x float> undef, float %add5796, i32 0
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %vecins5797, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp462 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecext5798 = extractelement <4 x float> %tmp462, i32 1
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp463 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins5800 = insertelement <4 x float> %tmp463, float undef, i32 1
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp464 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecext5801 = extractelement <4 x float> %tmp464, i32 2
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add5802 = fadd float %vecext5801, 0xC072A199A0000000
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp465 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins5803 = insertelement <4 x float> %tmp465, float %add5802, i32 2
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %vecins5803, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp466 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecext5804 = extractelement <4 x float> %tmp466, i32 3
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add5805 = fadd float %vecext5804, 0x40785999A0000000
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp467 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins5806 = insertelement <4 x float> %tmp467, float %add5805, i32 3
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %vecins5806, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp468 = load <4 x float>, <4 x float>* undef
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> undef, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp469 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecext5809 = extractelement <4 x float> %tmp469, i32 0
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add5810 = fadd float %vecext5809, 0x407B7B3340000000
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp470 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> undef, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp471 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecext5818 = extractelement <4 x float> undef, i32 3
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add5819 = fadd float %vecext5818, 0x4071733340000000
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp472 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins5820 = insertelement <4 x float> %tmp472, float %add5819, i32 3
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %vecins5820, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> <float 0x40514CCCC0000000, float 0x406A7999A0000000, float 0xC078766660000000, float 0xC0522CCCC0000000>, <4 x float>* undef
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp473 = load <4 x float>, <4 x float>* undef
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp474 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add5822 = fadd <4 x float> %tmp474, %tmp473
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %add5822, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp475 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecext5823 = extractelement <4 x float> %tmp475, i32 0
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp476 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins5825 = insertelement <4 x float> %tmp476, float undef, i32 0
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp477 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecext5826 = extractelement <4 x float> %tmp477, i32 1
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add5827 = fadd float %vecext5826, 0x407F14CCC0000000
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp478 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins5828 = insertelement <4 x float> %tmp478, float %add5827, i32 1
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %vecins5828, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp479 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecext5829 = extractelement <4 x float> %tmp479, i32 2
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add5830 = fadd float %vecext5829, 3.350000e+02
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp480 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins5831 = insertelement <4 x float> %tmp480, float %add5830, i32 2
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> undef, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> <float -3.370000e+02, float 0xC072DE6660000000, float -2.670000e+02, float 0x4062333340000000>, <4 x float>* undef
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> undef, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp481 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecext5837 = extractelement <4 x float> %tmp481, i32 0
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins5839 = insertelement <4 x float> undef, float undef, i32 0
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %vecins5839, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp482 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecext5840 = extractelement <4 x float> %tmp482, i32 1
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp483 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins5842 = insertelement <4 x float> %tmp483, float undef, i32 1
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %vecins5842, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp484 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp485 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins5845 = insertelement <4 x float> %tmp485, float undef, i32 2
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %vecins5845, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> <float 0xC06EC999A0000000, float 0x406D5999A0000000, float 0x4056F33340000000, float 0xC07E14CCC0000000>, <4 x float>* undef
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add5850 = fadd <4 x float> undef, undef
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %add5850, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp486 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add5852 = fadd float undef, 2.985000e+02
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp487 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins5853 = insertelement <4 x float> %tmp487, float %add5852, i32 0
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %vecins5853, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp488 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecext5854 = extractelement <4 x float> %tmp488, i32 1
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add5855 = fadd float %vecext5854, 0xC053F999A0000000
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp489 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins5856 = insertelement <4 x float> %tmp489, float %add5855, i32 1
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %vecins5856, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp490 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add5858 = fadd float undef, 0x4071666660000000
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp491 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins5859 = insertelement <4 x float> %tmp491, float %add5858, i32 2
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %vecins5859, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp492 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecext5860 = extractelement <4 x float> %tmp492, i32 3
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp493 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecext5899 = extractelement <4 x float> %tmp493, i32 2
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add5900 = fadd float %vecext5899, -2.700000e+01
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp494 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins5901 = insertelement <4 x float> %tmp494, float %add5900, i32 2
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %vecins5901, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add5914 = fadd float undef, 0x40786E6660000000
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins5918 = insertelement <4 x float> undef, float undef, i32 3
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %vecins5918, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> <float 0x406F266660000000, float 7.900000e+01, float -4.695000e+02, float -4.880000e+02>, <4 x float>* undef
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add5920 = fadd <4 x float> undef, undef
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %add5920, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add5934 = fadd <4 x float> undef, undef
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecext5935 = extractelement <4 x float> undef, i32 0
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add5936 = fadd float %vecext5935, 0xC056B999A0000000
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp495 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> undef, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp496 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecext5994 = extractelement <4 x float> undef, i32 1
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add5995 = fadd float %vecext5994, 0x4051666660000000
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins5996 = insertelement <4 x float> undef, float %add5995, i32 1
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %vecins5996, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp497 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecext5997 = extractelement <4 x float> %tmp497, i32 2
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> undef, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp498 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecext6000 = extractelement <4 x float> %tmp498, i32 3
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add6001 = fadd float %vecext6000, -7.600000e+01
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp499 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins6002 = insertelement <4 x float> %tmp499, float %add6001, i32 3
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %vecins6002, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> <float 0xC07EA199A0000000, float 0x407DC33340000000, float 0xC0753199A0000000, float -3.895000e+02>, <4 x float>* undef
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp500 = load <4 x float>, <4 x float>* undef
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add6004 = fadd <4 x float> undef, %tmp500
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %add6004, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp501 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecext6005 = extractelement <4 x float> %tmp501, i32 0
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp502 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins6007 = insertelement <4 x float> %tmp502, float undef, i32 0
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp503 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecext6008 = extractelement <4 x float> %tmp503, i32 1
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp504 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins6024 = insertelement <4 x float> %tmp504, float undef, i32 1
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %vecins6024, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp505 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecext6025 = extractelement <4 x float> %tmp505, i32 2
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add6026 = fadd float %vecext6025, 3.700000e+02
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp506 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins6027 = insertelement <4 x float> %tmp506, float %add6026, i32 2
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %vecins6027, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecext6028 = extractelement <4 x float> undef, i32 3
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add6029 = fadd float %vecext6028, 0x4071666660000000
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp507 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins6030 = insertelement <4 x float> %tmp507, float %add6029, i32 3
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %vecins6030, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> <float 0xC0527999A0000000, float 0xC06AD999A0000000, float 0x3FF6666660000000, float 0xC03F666660000000>, <4 x float>* undef
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp508 = load <4 x float>, <4 x float>* undef
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp509 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> undef, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp510 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecext6033 = extractelement <4 x float> %tmp510, i32 0
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp511 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> undef, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecext6036 = extractelement <4 x float> undef, i32 1
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add6037 = fadd float %vecext6036, 0xC075CB3340000000
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins6038 = insertelement <4 x float> undef, float %add6037, i32 1
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %vecins6038, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp512 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add6040 = fadd float undef, 0x4071ECCCC0000000
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp513 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins6041 = insertelement <4 x float> %tmp513, float %add6040, i32 2
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %vecins6041, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp514 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecext6042 = extractelement <4 x float> %tmp514, i32 3
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add6043 = fadd float %vecext6042, 0xC07DD33340000000
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp515 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins6044 = insertelement <4 x float> %tmp515, float %add6043, i32 3
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %vecins6044, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> <float 0xC065FCCCC0000000, float 0x40767CCCC0000000, float 0x4079D4CCC0000000, float 0xC07314CCC0000000>, <4 x float>* undef
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp516 = load <4 x float>, <4 x float>* undef
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp517 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add6046 = fadd <4 x float> %tmp517, %tmp516
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %add6046, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecext6047 = extractelement <4 x float> undef, i32 0
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp518 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins6049 = insertelement <4 x float> %tmp518, float undef, i32 0
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %vecins6049, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp519 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecext6050 = extractelement <4 x float> %tmp519, i32 1
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add6051 = fadd float %vecext6050, 0x407E4E6660000000
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins6055 = insertelement <4 x float> undef, float undef, i32 2
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecext6056 = extractelement <4 x float> undef, i32 3
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp520 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> undef, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecext6061 = extractelement <4 x float> undef, i32 0
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp521 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> undef, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp522 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecext6067 = extractelement <4 x float> %tmp522, i32 2
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add6068 = fadd float %vecext6067, 0x40768E6660000000
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecext6070 = extractelement <4 x float> undef, i32 3
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add6071 = fadd float %vecext6070, 0xC07C6CCCC0000000
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins6072 = insertelement <4 x float> undef, float %add6071, i32 3
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %vecins6072, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> <float 0x40546CCCC0000000, float 0x4067D66660000000, float 0xC060E33340000000, float 0x4061533340000000>, <4 x float>* undef
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp523 = load <4 x float>, <4 x float>* undef
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp524 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add6074 = fadd <4 x float> %tmp524, %tmp523
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %add6074, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp525 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecext6075 = extractelement <4 x float> %tmp525, i32 0
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add6076 = fadd float %vecext6075, 0x405D733340000000
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp526 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins6077 = insertelement <4 x float> %tmp526, float %add6076, i32 0
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %vecins6077, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp527 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add6079 = fadd float undef, 0xC07E9B3340000000
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp528 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> undef, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp529 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add6082 = fadd float undef, 0x407DCE6660000000
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins6083 = insertelement <4 x float> undef, float %add6082, i32 2
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %vecins6083, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp530 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecext6084 = extractelement <4 x float> %tmp530, i32 3
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add6085 = fadd float %vecext6084, 0xC061A33340000000
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins6086 = insertelement <4 x float> undef, float %add6085, i32 3
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %vecins6086, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> <float 0x4055C66660000000, float 0x40735199A0000000, float 0xC0713199A0000000, float 0x40729B3340000000>, <4 x float>* undef
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp531 = load <4 x float>, <4 x float>* undef
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp532 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add6088 = fadd <4 x float> %tmp532, %tmp531
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %add6088, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp533 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecext6089 = extractelement <4 x float> %tmp533, i32 0
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add6107 = fadd float undef, 0xC06A166660000000
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp534 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins6108 = insertelement <4 x float> %tmp534, float %add6107, i32 1
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %vecins6108, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp535 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecext6109 = extractelement <4 x float> %tmp535, i32 2
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add6110 = fadd float %vecext6109, 0x4070FB3340000000
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp536 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> undef, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp537 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecext6112 = extractelement <4 x float> %tmp537, i32 3
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add6113 = fadd float %vecext6112, 0xC04AF33340000000
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp538 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp539 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecext6117 = extractelement <4 x float> %tmp539, i32 0
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add6118 = fadd float %vecext6117, 0x407AB33340000000
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp540 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins6119 = insertelement <4 x float> %tmp540, float %add6118, i32 0
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %vecins6119, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp541 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecext6120 = extractelement <4 x float> %tmp541, i32 1
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add6121 = fadd float %vecext6120, 0x405AE66660000000
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp542 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins6122 = insertelement <4 x float> %tmp542, float %add6121, i32 1
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %vecins6122, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecext6123 = extractelement <4 x float> undef, i32 2
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add6124 = fadd float %vecext6123, -4.385000e+02
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp543 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> undef, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecext6126 = extractelement <4 x float> undef, i32 3
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp544 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins6128 = insertelement <4 x float> %tmp544, float undef, i32 3
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %vecins6128, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> <float -2.980000e+02, float 0xC06F0CCCC0000000, float 0xC054A66660000000, float 0xC040CCCCC0000000>, <4 x float>* undef
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp545 = load <4 x float>, <4 x float>* undef
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp546 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add6130 = fadd <4 x float> %tmp546, %tmp545
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp547 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecext6131 = extractelement <4 x float> %tmp547, i32 0
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add6132 = fadd float %vecext6131, 0x407BDE6660000000
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins6133 = insertelement <4 x float> undef, float %add6132, i32 0
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %vecins6133, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecext6134 = extractelement <4 x float> undef, i32 1
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add6135 = fadd float %vecext6134, 0xC06B7999A0000000
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp548 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins6136 = insertelement <4 x float> %tmp548, float %add6135, i32 1
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecext6137 = extractelement <4 x float> undef, i32 2
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add6138 = fadd float %vecext6137, 0x40752199A0000000
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp549 = load <4 x float>, <4 x float>* undef
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add6172 = fadd <4 x float> undef, %tmp549
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp550 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp551 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins6178 = insertelement <4 x float> %tmp551, float undef, i32 1
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %vecins6178, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp552 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecext6179 = extractelement <4 x float> %tmp552, i32 2
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add6180 = fadd float %vecext6179, -3.905000e+02
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp553 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins6181 = insertelement <4 x float> %tmp553, float %add6180, i32 2
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp554 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecext6182 = extractelement <4 x float> %tmp554, i32 3
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add6183 = fadd float %vecext6182, 1.515000e+02
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp555 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins6184 = insertelement <4 x float> %tmp555, float %add6183, i32 3
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %vecins6184, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp556 = load <4 x float>, <4 x float>* undef
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins6189 = insertelement <4 x float> undef, float undef, i32 0
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %vecins6189, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp557 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecext6190 = extractelement <4 x float> %tmp557, i32 1
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add6191 = fadd float %vecext6190, 0xC07BD33340000000
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp558 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins6192 = insertelement <4 x float> %tmp558, float %add6191, i32 1
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %vecins6192, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp559 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp560 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecext6196 = extractelement <4 x float> undef, i32 3
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add6197 = fadd float %vecext6196, -4.070000e+02
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp561 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins6198 = insertelement <4 x float> %tmp561, float %add6197, i32 3
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> <float 0x407904CCC0000000, float 0x406A833340000000, float 4.895000e+02, float 0x40648999A0000000>, <4 x float>* undef
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp562 = load <4 x float>, <4 x float>* undef
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp563 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add6200 = fadd <4 x float> %tmp563, %tmp562
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %add6200, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp564 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecext6201 = extractelement <4 x float> %tmp564, i32 0
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp565 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins6203 = insertelement <4 x float> %tmp565, float undef, i32 0
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp566 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecext6204 = extractelement <4 x float> %tmp566, i32 1
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add6205 = fadd float %vecext6204, 1.740000e+02
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp567 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins6206 = insertelement <4 x float> %tmp567, float %add6205, i32 1
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp568 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins6209 = insertelement <4 x float> %tmp568, float undef, i32 2
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %vecins6209, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp569 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecext6210 = extractelement <4 x float> %tmp569, i32 3
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp570 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add6219 = fadd float undef, 0xC0596CCCC0000000
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp571 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins6220 = insertelement <4 x float> %tmp571, float %add6219, i32 1
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecext6224 = extractelement <4 x float> undef, i32 3
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add6225 = fadd float %vecext6224, 0xC074533340000000
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp572 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add6228 = fadd <4 x float> undef, undef
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %add6228, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecext6229 = extractelement <4 x float> undef, i32 0
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add6230 = fadd float %vecext6229, 1.695000e+02
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp573 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins6231 = insertelement <4 x float> %tmp573, float %add6230, i32 0
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %vecins6231, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp574 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecext6232 = extractelement <4 x float> %tmp574, i32 1
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add6233 = fadd float %vecext6232, 0x4079C33340000000
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp575 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins6234 = insertelement <4 x float> %tmp575, float %add6233, i32 1
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %vecins6234, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecext6235 = extractelement <4 x float> undef, i32 2
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add6236 = fadd float %vecext6235, 0xC07D8199A0000000
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins6237 = insertelement <4 x float> undef, float %add6236, i32 2
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %vecins6237, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp576 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins6245 = insertelement <4 x float> undef, float undef, i32 0
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %vecins6245, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp577 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecext6246 = extractelement <4 x float> %tmp577, i32 1
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add6247 = fadd float %vecext6246, 0x40631999A0000000
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp578 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins6251 = insertelement <4 x float> undef, float undef, i32 2
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp579 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add6253 = fadd float undef, 0xC0692999A0000000
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins6254 = insertelement <4 x float> undef, float %add6253, i32 3
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %vecins6254, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> <float 4.600000e+02, float 0xC0777B3340000000, float 0x40351999A0000000, float 0xC06E433340000000>, <4 x float>* undef
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp580 = load <4 x float>, <4 x float>* undef
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp581 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add6256 = fadd <4 x float> %tmp581, %tmp580
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %add6256, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp582 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecext6257 = extractelement <4 x float> %tmp582, i32 0
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add6258 = fadd float %vecext6257, 4.670000e+02
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp583 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins6259 = insertelement <4 x float> %tmp583, float %add6258, i32 0
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %vecins6259, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp584 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecext6260 = extractelement <4 x float> %tmp584, i32 1
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add6261 = fadd float %vecext6260, 0xC05F733340000000
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp585 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins6262 = insertelement <4 x float> %tmp585, float %add6261, i32 1
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %vecins6262, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp586 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecext6263 = extractelement <4 x float> %tmp586, i32 2
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp587 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins6265 = insertelement <4 x float> %tmp587, float undef, i32 2
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %vecins6265, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp588 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecext6266 = extractelement <4 x float> %tmp588, i32 3
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add6267 = fadd float %vecext6266, 0x407174CCC0000000
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp589 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins6268 = insertelement <4 x float> %tmp589, float %add6267, i32 3
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %vecins6268, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> <float -3.130000e+02, float 0xC079733340000000, float -4.660000e+02, float 0xC064E66660000000>, <4 x float>* undef
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp590 = load <4 x float>, <4 x float>* undef
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp591 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add6270 = fadd <4 x float> %tmp591, %tmp590
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %add6270, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp592 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecext6271 = extractelement <4 x float> %tmp592, i32 0
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add6272 = fadd float %vecext6271, 1.765000e+02
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp593 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins6273 = insertelement <4 x float> %tmp593, float %add6272, i32 0
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %vecins6273, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp594 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecext6274 = extractelement <4 x float> %tmp594, i32 1
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add6275 = fadd float %vecext6274, 0x402C666660000000
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp595 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins6276 = insertelement <4 x float> %tmp595, float %add6275, i32 1
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %vecins6276, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp596 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecext6277 = extractelement <4 x float> %tmp596, i32 2
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add6278 = fadd float %vecext6277, -8.450000e+01
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp597 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins6279 = insertelement <4 x float> %tmp597, float %add6278, i32 2
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %vecins6279, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp598 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecext6280 = extractelement <4 x float> %tmp598, i32 3
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add6281 = fadd float %vecext6280, 0xC07A133340000000
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins6282 = insertelement <4 x float> undef, float %add6281, i32 3
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %vecins6282, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> <float 0x4067ECCCC0000000, float 0xC040CCCCC0000000, float 0xC0762E6660000000, float -4.750000e+02>, <4 x float>* undef
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add6284 = fadd <4 x float> undef, undef
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecext6285 = extractelement <4 x float> undef, i32 0
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add6289 = fadd float undef, 0xC0738999A0000000
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp599 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins6293 = insertelement <4 x float> %tmp599, float undef, i32 2
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %vecins6293, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp600 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecext6294 = extractelement <4 x float> %tmp600, i32 3
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add6295 = fadd float %vecext6294, 0xC01CCCCCC0000000
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins6296 = insertelement <4 x float> undef, float %add6295, i32 3
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %vecins6296, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> <float 0x40704199A0000000, float 0x40753CCCC0000000, float 0xC07E2199A0000000, float 0xC068833340000000>, <4 x float>* undef
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp601 = load <4 x float>, <4 x float>* undef
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add6298 = fadd <4 x float> undef, %tmp601
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %add6298, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp602 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecext6299 = extractelement <4 x float> %tmp602, i32 0
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add6300 = fadd float %vecext6299, 0x4074B33340000000
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp603 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins6301 = insertelement <4 x float> %tmp603, float %add6300, i32 0
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %vecins6301, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp604 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecext6302 = extractelement <4 x float> %tmp604, i32 1
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add6303 = fadd float %vecext6302, 0xC05B333340000000
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp605 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins6304 = insertelement <4 x float> %tmp605, float %add6303, i32 1
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %vecins6304, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp606 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecext6305 = extractelement <4 x float> %tmp606, i32 2
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add6306 = fadd float %vecext6305, 0x4077E999A0000000
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins6307 = insertelement <4 x float> undef, float %add6306, i32 2
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %vecins6307, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp607 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecext6308 = extractelement <4 x float> %tmp607, i32 3
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add6309 = fadd float %vecext6308, 0x40707E6660000000
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp608 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins6310 = insertelement <4 x float> %tmp608, float %add6309, i32 3
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %vecins6310, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> <float 0x407A233340000000, float 0x406DA33340000000, float 3.725000e+02, float 0x40761199A0000000>, <4 x float>* undef
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp609 = load <4 x float>, <4 x float>* undef
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp610 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add6312 = fadd <4 x float> %tmp610, %tmp609
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %add6312, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp611 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecext6313 = extractelement <4 x float> %tmp611, i32 0
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add6314 = fadd float %vecext6313, 0xC07CF33340000000
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp612 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins6315 = insertelement <4 x float> %tmp612, float %add6314, i32 0
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp613 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecext6655 = extractelement <4 x float> %tmp613, i32 2
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add6656 = fadd float %vecext6655, 2.185000e+02
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp614 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins6657 = insertelement <4 x float> %tmp614, float %add6656, i32 2
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %vecins6657, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins6660 = insertelement <4 x float> undef, float undef, i32 3
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %vecins6660, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> <float 0xC064E33340000000, float 0xC064833340000000, float 0xC0673CCCC0000000, float 0xC074266660000000>, <4 x float>* undef
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp615 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecext6663 = extractelement <4 x float> undef, i32 0
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add6664 = fadd float %vecext6663, 0xC05B7999A0000000
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp616 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins6665 = insertelement <4 x float> %tmp616, float %add6664, i32 0
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> undef, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp617 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecext6669 = extractelement <4 x float> %tmp617, i32 2
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp618 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> undef, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> <float 0xC07CC4CCC0000000, float 0x404EE66660000000, float 0xC0754CCCC0000000, float 0xC0744B3340000000>, <4 x float>* undef
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp619 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add6676 = fadd <4 x float> %tmp619, undef
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %add6676, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp620 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecext6677 = extractelement <4 x float> %tmp620, i32 0
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add6678 = fadd float %vecext6677, 0x4077F4CCC0000000
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp621 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins6679 = insertelement <4 x float> %tmp621, float %add6678, i32 0
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecext6680 = extractelement <4 x float> undef, i32 1
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add6681 = fadd float %vecext6680, 0x4061766660000000
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp622 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> undef, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp623 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecext6683 = extractelement <4 x float> %tmp623, i32 2
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add6684 = fadd float %vecext6683, 0x40718999A0000000
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp624 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins6685 = insertelement <4 x float> %tmp624, float %add6684, i32 2
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %vecins6685, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp625 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecext6686 = extractelement <4 x float> %tmp625, i32 3
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add6687 = fadd float %vecext6686, 0x4076D66660000000
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp626 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins6688 = insertelement <4 x float> %tmp626, float %add6687, i32 3
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %vecins6688, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> <float 7.500000e+00, float 0x4077E33340000000, float 0xC0596CCCC0000000, float 0xC07D4E6660000000>, <4 x float>* undef
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp627 = load <4 x float>, <4 x float>* undef
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add6690 = fadd <4 x float> undef, %tmp627
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %add6690, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp628 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecext6691 = extractelement <4 x float> %tmp628, i32 0
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add6692 = fadd float %vecext6691, 3.250000e+02
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp629 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins6693 = insertelement <4 x float> %tmp629, float %add6692, i32 0
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %vecins6693, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp630 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecext6694 = extractelement <4 x float> %tmp630, i32 1
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add6695 = fadd float %vecext6694, 0x407DF999A0000000
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp631 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins6696 = insertelement <4 x float> %tmp631, float %add6695, i32 1
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %vecins6696, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp632 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecext6697 = extractelement <4 x float> %tmp632, i32 2
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add6698 = fadd float %vecext6697, 0xC075FE6660000000
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp633 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins6699 = insertelement <4 x float> %tmp633, float %add6698, i32 2
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %vecins6699, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp634 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecext6700 = extractelement <4 x float> %tmp634, i32 3
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add6701 = fadd float %vecext6700, 0xC07BCE6660000000
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp635 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins6702 = insertelement <4 x float> %tmp635, float %add6701, i32 3
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %vecins6702, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> <float 0x40772CCCC0000000, float 0xC0625CCCC0000000, float 6.200000e+01, float 0xC06ADCCCC0000000>, <4 x float>* undef
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp636 = load <4 x float>, <4 x float>* undef
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp637 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins6707 = insertelement <4 x float> undef, float undef, i32 0
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %vecins6707, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp638 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecext6708 = extractelement <4 x float> %tmp638, i32 1
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp639 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> undef, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp640 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecext6714 = extractelement <4 x float> %tmp640, i32 3
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add6715 = fadd float %vecext6714, 0xC0537999A0000000
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp641 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins6716 = insertelement <4 x float> %tmp641, float %add6715, i32 3
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecext6719 = extractelement <4 x float> undef, i32 0
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add6720 = fadd float %vecext6719, 2.870000e+02
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp642 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins6721 = insertelement <4 x float> %tmp642, float %add6720, i32 0
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp643 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecext6722 = extractelement <4 x float> %tmp643, i32 1
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add6723 = fadd float %vecext6722, 0xC07704CCC0000000
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp644 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins6724 = insertelement <4 x float> %tmp644, float %add6723, i32 1
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp645 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add6726 = fadd float undef, 0x4059B999A0000000
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp646 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins6727 = insertelement <4 x float> %tmp646, float %add6726, i32 2
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %vecins6727, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecext6728 = extractelement <4 x float> undef, i32 3
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add6729 = fadd float %vecext6728, 0xC073466660000000
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> undef, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> <float 0xC0309999A0000000, float -2.715000e+02, float 1.620000e+02, float 0x40674CCCC0000000>, <4 x float>* undef
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp647 = load <4 x float>, <4 x float>* undef
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp648 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add6732 = fadd <4 x float> %tmp648, %tmp647
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %add6732, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp649 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecext6733 = extractelement <4 x float> %tmp649, i32 0
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add6734 = fadd float %vecext6733, 0x4040733340000000
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp650 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins6735 = insertelement <4 x float> %tmp650, float %add6734, i32 0
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %vecins6735, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp651 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecext6736 = extractelement <4 x float> %tmp651, i32 1
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add6737 = fadd float %vecext6736, 0xC07B74CCC0000000
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp652 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins6738 = insertelement <4 x float> %tmp652, float %add6737, i32 1
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %vecins6738, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp653 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecext6739 = extractelement <4 x float> %tmp653, i32 2
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add6740 = fadd float %vecext6739, 0x40699CCCC0000000
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp654 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins6741 = insertelement <4 x float> %tmp654, float %add6740, i32 2
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %vecins6741, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp655 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecext6742 = extractelement <4 x float> %tmp655, i32 3
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add6743 = fadd float %vecext6742, 0x4078533340000000
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp656 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins6744 = insertelement <4 x float> %tmp656, float %add6743, i32 3
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %vecins6744, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp657 = load <4 x float>, <4 x float>* undef
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp658 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add6746 = fadd <4 x float> %tmp658, %tmp657
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %add6746, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp659 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins6749 = insertelement <4 x float> undef, float undef, i32 0
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %vecins6749, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp660 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add6751 = fadd float undef, 0x4075DE6660000000
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins6752 = insertelement <4 x float> undef, float %add6751, i32 1
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %vecins6752, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp661 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecext6753 = extractelement <4 x float> %tmp661, i32 2
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add6754 = fadd float %vecext6753, 0xC008CCCCC0000000
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins6755 = insertelement <4 x float> undef, float %add6754, i32 2
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %vecins6755, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp662 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecext6756 = extractelement <4 x float> %tmp662, i32 3
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add6757 = fadd float %vecext6756, 0x406CA999A0000000
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp663 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins6758 = insertelement <4 x float> %tmp663, float %add6757, i32 3
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %vecins6758, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> <float 0x403D1999A0000000, float 0xC05F533340000000, float 3.945000e+02, float 3.950000e+01>, <4 x float>* undef
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp664 = load <4 x float>, <4 x float>* undef
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add6760 = fadd <4 x float> undef, %tmp664
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %add6760, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp665 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecext6761 = extractelement <4 x float> %tmp665, i32 0
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add6762 = fadd float %vecext6761, 2.860000e+02
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins6763 = insertelement <4 x float> undef, float %add6762, i32 0
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp666 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> undef, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> <float 0xC079BE6660000000, float 4.930000e+02, float 0x406CC33340000000, float 0xC062E999A0000000>, <4 x float>* undef
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp667 = load <4 x float>, <4 x float>* undef
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add6774 = fadd <4 x float> undef, %tmp667
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp668 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecext6775 = extractelement <4 x float> %tmp668, i32 0
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add6776 = fadd float %vecext6775, 0x407B8199A0000000
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp669 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins6777 = insertelement <4 x float> %tmp669, float %add6776, i32 0
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %vecins6777, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp670 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecext6778 = extractelement <4 x float> %tmp670, i32 1
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add6779 = fadd float %vecext6778, 0x401C666660000000
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp671 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecext6784 = extractelement <4 x float> %tmp671, i32 3
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins6875 = insertelement <4 x float> undef, float undef, i32 0
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %vecins6875, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp672 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecext6876 = extractelement <4 x float> %tmp672, i32 1
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add6877 = fadd float %vecext6876, 0x4073A66660000000
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins6878 = insertelement <4 x float> undef, float %add6877, i32 1
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %vecins6878, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add6888 = fadd float undef, 0x4057CCCCC0000000
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp673 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins6889 = insertelement <4 x float> %tmp673, float %add6888, i32 0
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %vecins6889, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp674 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecext6890 = extractelement <4 x float> %tmp674, i32 1
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add6891 = fadd float %vecext6890, -4.430000e+02
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp675 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins6892 = insertelement <4 x float> %tmp675, float %add6891, i32 1
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %vecins6892, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp676 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecext6893 = extractelement <4 x float> %tmp676, i32 2
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add6894 = fadd float %vecext6893, -3.280000e+02
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp677 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins6895 = insertelement <4 x float> %tmp677, float %add6894, i32 2
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %vecins6895, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp678 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp679 = load <4 x float>, <4 x float>* undef
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp680 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add6900 = fadd <4 x float> %tmp680, %tmp679
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %add6900, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp681 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecext6901 = extractelement <4 x float> %tmp681, i32 0
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add6902 = fadd float %vecext6901, 0x4079DCCCC0000000
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp682 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins6903 = insertelement <4 x float> %tmp682, float %add6902, i32 0
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %vecins6903, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add6905 = fadd float undef, 0x4031B33340000000
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp683 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins6906 = insertelement <4 x float> %tmp683, float %add6905, i32 1
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp684 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins6912 = insertelement <4 x float> %tmp684, float undef, i32 3
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> <float 3.315000e+02, float 0xC066C999A0000000, float 0xC061F33340000000, float 0x4071166660000000>, <4 x float>* undef
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp685 = load <4 x float>, <4 x float>* undef
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp686 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add6914 = fadd <4 x float> %tmp686, %tmp685
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %add6914, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecext6915 = extractelement <4 x float> undef, i32 0
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins6920 = insertelement <4 x float> undef, float undef, i32 1
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %vecins6920, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecext6921 = extractelement <4 x float> undef, i32 2
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add6922 = fadd float %vecext6921, 0xC064066660000000
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp687 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins6926 = insertelement <4 x float> %tmp687, float undef, i32 3
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %vecins6926, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> <float 0xC03C4CCCC0000000, float 0xC07E5199A0000000, float -8.250000e+01, float 0xC043B33340000000>, <4 x float>* undef
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp688 = load <4 x float>, <4 x float>* undef
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp689 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add6928 = fadd <4 x float> %tmp689, %tmp688
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %add6928, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add6930 = fadd float undef, -4.590000e+02
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins6931 = insertelement <4 x float> undef, float %add6930, i32 0
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %vecins6931, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp690 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecext6932 = extractelement <4 x float> %tmp690, i32 1
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add6933 = fadd float %vecext6932, 0xC063F999A0000000
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp691 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> undef, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp692 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecext6935 = extractelement <4 x float> %tmp692, i32 2
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add6936 = fadd float %vecext6935, -3.335000e+02
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp693 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins6937 = insertelement <4 x float> %tmp693, float %add6936, i32 2
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp694 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecext6938 = extractelement <4 x float> %tmp694, i32 3
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add6939 = fadd float %vecext6938, 0x405F3999A0000000
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add6942 = fadd <4 x float> undef, undef
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecext6943 = extractelement <4 x float> undef, i32 0
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add6944 = fadd float %vecext6943, 0x40530CCCC0000000
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp695 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> undef, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add6950 = fadd float undef, 0xC078F33340000000
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp696 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins6951 = insertelement <4 x float> %tmp696, float %add6950, i32 2
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %vecins6951, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp697 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecext6952 = extractelement <4 x float> %tmp697, i32 3
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add6953 = fadd float %vecext6952, 0xC06E5999A0000000
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp698 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins6954 = insertelement <4 x float> %tmp698, float %add6953, i32 3
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %vecins6954, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp699 = load <4 x float>, <4 x float>* undef
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp700 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add6956 = fadd <4 x float> %tmp700, %tmp699
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %add6956, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp701 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecext6957 = extractelement <4 x float> %tmp701, i32 0
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add6958 = fadd float %vecext6957, 0xC077633340000000
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp702 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins6959 = insertelement <4 x float> %tmp702, float %add6958, i32 0
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %vecins6959, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp703 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecext6963 = extractelement <4 x float> %tmp703, i32 2
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add6964 = fadd float %vecext6963, 0x4068666660000000
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp704 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins6965 = insertelement <4 x float> %tmp704, float %add6964, i32 2
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %vecins6965, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add6975 = fadd float undef, 0x406AF33340000000
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp705 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins6976 = insertelement <4 x float> %tmp705, float %add6975, i32 1
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %vecins6976, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp706 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp707 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add6984 = fadd <4 x float> %tmp707, undef
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %add6984, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp708 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecext6985 = extractelement <4 x float> %tmp708, i32 0
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add6986 = fadd float %vecext6985, 0xC05E266660000000
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp709 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins6987 = insertelement <4 x float> %tmp709, float %add6986, i32 0
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %vecins6987, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp710 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecext6988 = extractelement <4 x float> %tmp710, i32 1
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add6989 = fadd float %vecext6988, 0x40706E6660000000
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp711 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins6996 = insertelement <4 x float> %tmp711, float undef, i32 3
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %vecins6996, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> <float 0x4077A4CCC0000000, float 0xC0757199A0000000, float 0xC072F4CCC0000000, float 0xC071DCCCC0000000>, <4 x float>* undef
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp712 = load <4 x float>, <4 x float>* undef
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp713 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add6998 = fadd <4 x float> %tmp713, %tmp712
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %add6998, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp714 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecext6999 = extractelement <4 x float> %tmp714, i32 0
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add7000 = fadd float %vecext6999, 0x4076233340000000
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp715 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins7001 = insertelement <4 x float> %tmp715, float %add7000, i32 0
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %vecins7001, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp716 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecext7002 = extractelement <4 x float> %tmp716, i32 1
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add7003 = fadd float %vecext7002, 0x403BCCCCC0000000
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp717 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins7004 = insertelement <4 x float> %tmp717, float %add7003, i32 1
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> undef, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp718 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add7140 = fadd float undef, 0x403D333340000000
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins7141 = insertelement <4 x float> undef, float %add7140, i32 0
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecext7142 = extractelement <4 x float> undef, i32 1
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add7143 = fadd float %vecext7142, 0xC058F999A0000000
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins7144 = insertelement <4 x float> undef, float %add7143, i32 1
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> undef, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp719 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecext7148 = extractelement <4 x float> %tmp719, i32 3
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add7149 = fadd float %vecext7148, 0x4075333340000000
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp720 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins7150 = insertelement <4 x float> %tmp720, float %add7149, i32 3
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %vecins7150, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> <float 1.700000e+02, float 0xC077B4CCC0000000, float 0x40625999A0000000, float 0x406C166660000000>, <4 x float>* undef
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp721 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add7152 = fadd <4 x float> %tmp721, undef
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %add7152, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecext7156 = extractelement <4 x float> undef, i32 1
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add7157 = fadd float %vecext7156, 0xC05F533340000000
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp722 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins7158 = insertelement <4 x float> %tmp722, float %add7157, i32 1
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %vecins7158, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp723 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecext7159 = extractelement <4 x float> %tmp723, i32 2
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add7160 = fadd float %vecext7159, 0x407A5999A0000000
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp724 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins7161 = insertelement <4 x float> %tmp724, float %add7160, i32 2
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %vecins7161, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add7168 = fadd float undef, 0xC072F199A0000000
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp725 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> undef, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecext7170 = extractelement <4 x float> undef, i32 1
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add7171 = fadd float %vecext7170, 0x406AACCCC0000000
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins7172 = insertelement <4 x float> undef, float %add7171, i32 1
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %vecins7172, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecext7173 = extractelement <4 x float> undef, i32 2
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> undef, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp726 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecext7419 = extractelement <4 x float> %tmp726, i32 0
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add7420 = fadd float %vecext7419, 0x404EA66660000000
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins7421 = insertelement <4 x float> undef, float %add7420, i32 0
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %vecins7421, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp727 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecext7422 = extractelement <4 x float> %tmp727, i32 1
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add7423 = fadd float %vecext7422, 4.800000e+02
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp728 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins7424 = insertelement <4 x float> %tmp728, float %add7423, i32 1
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %vecins7424, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp729 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecext7425 = extractelement <4 x float> %tmp729, i32 2
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add7426 = fadd float %vecext7425, 0xC072C999A0000000
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp730 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins7427 = insertelement <4 x float> %tmp730, float %add7426, i32 2
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %vecins7427, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecext7428 = extractelement <4 x float> undef, i32 3
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> undef, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp731 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecext7568 = extractelement <4 x float> %tmp731, i32 3
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add7569 = fadd float %vecext7568, 1.090000e+02
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp732 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins7570 = insertelement <4 x float> %tmp732, float %add7569, i32 3
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %vecins7570, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> <float 0x40745199A0000000, float 0xC0411999A0000000, float -5.650000e+01, float -4.005000e+02>, <4 x float>* undef
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp733 = load <4 x float>, <4 x float>* undef
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp734 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add7572 = fadd <4 x float> %tmp734, %tmp733
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %add7572, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecext7573 = extractelement <4 x float> undef, i32 0
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add7574 = fadd float %vecext7573, -3.920000e+02
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp735 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins7575 = insertelement <4 x float> %tmp735, float %add7574, i32 0
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %vecins7575, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp736 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add7577 = fadd float undef, 0xC051666660000000
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp737 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp738 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecext7579 = extractelement <4 x float> %tmp738, i32 2
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add7580 = fadd float %vecext7579, 0x407E9199A0000000
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins7581 = insertelement <4 x float> undef, float %add7580, i32 2
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %vecins7581, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp739 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecext7582 = extractelement <4 x float> %tmp739, i32 3
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add7583 = fadd float %vecext7582, 2.760000e+02
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp740 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins7584 = insertelement <4 x float> %tmp740, float %add7583, i32 3
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> <float 0xC057533340000000, float 0x4060A33340000000, float 0x40791E6660000000, float 2.455000e+02>, <4 x float>* undef
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp741 = load <4 x float>, <4 x float>* undef
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp742 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add7586 = fadd <4 x float> %tmp742, %tmp741
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %add7586, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp743 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecext7587 = extractelement <4 x float> %tmp743, i32 0
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add7588 = fadd float %vecext7587, 6.100000e+01
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp744 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> undef, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp745 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecext7590 = extractelement <4 x float> %tmp745, i32 1
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add7591 = fadd float %vecext7590, -3.935000e+02
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp746 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins7592 = insertelement <4 x float> %tmp746, float %add7591, i32 1
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %vecins7592, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp747 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecext7593 = extractelement <4 x float> %tmp747, i32 2
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins7595 = insertelement <4 x float> undef, float undef, i32 2
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %vecins7595, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp748 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecext7596 = extractelement <4 x float> %tmp748, i32 3
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add7597 = fadd float %vecext7596, 0x407E666660000000
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> <float 0x406A766660000000, float 0xBFC99999A0000000, float 0xC0751B3340000000, float -4.075000e+02>, <4 x float>* undef
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp749 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add7616 = fadd float undef, 0xC04DE66660000000
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp750 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins7617 = insertelement <4 x float> %tmp750, float %add7616, i32 0
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %vecins7617, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp751 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecext7618 = extractelement <4 x float> %tmp751, i32 1
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add7619 = fadd float %vecext7618, 6.050000e+01
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp752 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins7620 = insertelement <4 x float> %tmp752, float %add7619, i32 1
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %vecins7620, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp753 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add7622 = fadd float undef, 0xC054B999A0000000
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp754 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins7626 = insertelement <4 x float> undef, float undef, i32 3
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %vecins7626, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp755 = load <4 x float>, <4 x float>* undef
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp756 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add7628 = fadd <4 x float> %tmp756, %tmp755
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %add7628, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp757 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecext7629 = extractelement <4 x float> %tmp757, i32 0
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add7630 = fadd float %vecext7629, 0xC05E2CCCC0000000
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp758 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins7631 = insertelement <4 x float> %tmp758, float %add7630, i32 0
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add7639 = fadd float undef, 0x407C5999A0000000
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp759 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins7640 = insertelement <4 x float> %tmp759, float %add7639, i32 3
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> <float 0x406AA66660000000, float 0x4067C66660000000, float 0xC054866660000000, float -2.400000e+01>, <4 x float>* undef
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp760 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add7642 = fadd <4 x float> %tmp760, undef
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp761 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add7644 = fadd float undef, 0xC0758999A0000000
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> undef, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp762 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecext7646 = extractelement <4 x float> %tmp762, i32 1
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add7647 = fadd float %vecext7646, 0xC07A3B3340000000
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp763 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins7648 = insertelement <4 x float> %tmp763, float %add7647, i32 1
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %vecins7648, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp764 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecext7649 = extractelement <4 x float> %tmp764, i32 2
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add7650 = fadd float %vecext7649, 0x40760CCCC0000000
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp765 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins7651 = insertelement <4 x float> %tmp765, float %add7650, i32 2
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %vecins7651, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp766 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecext7652 = extractelement <4 x float> %tmp766, i32 3
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add7653 = fadd float %vecext7652, 0x40620CCCC0000000
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp767 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins7654 = insertelement <4 x float> %tmp767, float %add7653, i32 3
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %vecins7654, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp768 = load <4 x float>, <4 x float>* undef
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp769 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add7656 = fadd <4 x float> %tmp769, %tmp768
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %add7656, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp770 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecext7657 = extractelement <4 x float> %tmp770, i32 0
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add7658 = fadd float %vecext7657, 0xC06EF999A0000000
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp771 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins7659 = insertelement <4 x float> %tmp771, float %add7658, i32 0
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %vecins7659, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp772 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecext7660 = extractelement <4 x float> %tmp772, i32 1
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add7661 = fadd float %vecext7660, 0x404B9999A0000000
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp773 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins7662 = insertelement <4 x float> %tmp773, float %add7661, i32 1
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %vecins7662, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp774 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecext7663 = extractelement <4 x float> %tmp774, i32 2
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add7664 = fadd float %vecext7663, 0x4074B66660000000
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp775 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins7665 = insertelement <4 x float> %tmp775, float %add7664, i32 2
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %vecins7665, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp776 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecext7666 = extractelement <4 x float> %tmp776, i32 3
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add7667 = fadd float %vecext7666, 0x4074166660000000
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins7668 = insertelement <4 x float> undef, float %add7667, i32 3
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %vecins7668, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp777 = load <4 x float>, <4 x float>* undef
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp778 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add7670 = fadd <4 x float> %tmp778, %tmp777
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp779 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecext7671 = extractelement <4 x float> %tmp779, i32 0
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add7672 = fadd float %vecext7671, 0x406F166660000000
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add7726 = fadd <4 x float> undef, undef
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp780 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecext7727 = extractelement <4 x float> %tmp780, i32 0
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp781 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> undef, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp782 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add7731 = fadd float undef, 1.900000e+02
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp783 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins7732 = insertelement <4 x float> %tmp783, float %add7731, i32 1
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %vecins7732, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp784 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins7735 = insertelement <4 x float> %tmp784, float undef, i32 2
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %vecins7735, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp785 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecext7736 = extractelement <4 x float> %tmp785, i32 3
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add7737 = fadd float %vecext7736, 0xC06AF66660000000
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins7850 = insertelement <4 x float> undef, float undef, i32 3
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %vecins7850, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> <float 0x4062A33340000000, float 2.290000e+02, float 0x40509999A0000000, float 0xC078BE6660000000>, <4 x float>* undef
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp786 = load <4 x float>, <4 x float>* undef
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp787 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add7852 = fadd <4 x float> %tmp787, %tmp786
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %add7852, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp788 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecext9396 = extractelement <4 x float> undef, i32 1
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add9397 = fadd float %vecext9396, 0xC074533340000000
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp789 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins9398 = insertelement <4 x float> %tmp789, float %add9397, i32 1
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %vecins9398, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecext9399 = extractelement <4 x float> undef, i32 2
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp790 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins9401 = insertelement <4 x float> %tmp790, float undef, i32 2
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp791 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecext9402 = extractelement <4 x float> %tmp791, i32 3
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add9403 = fadd float %vecext9402, 0xC03E4CCCC0000000
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp792 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins9404 = insertelement <4 x float> %tmp792, float %add9403, i32 3
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %vecins9404, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp793 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> undef, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp794 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecext9407 = extractelement <4 x float> %tmp794, i32 0
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add9408 = fadd float %vecext9407, 0x407B2999A0000000
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp795 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecext9410 = extractelement <4 x float> %tmp795, i32 1
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add9411 = fadd float %vecext9410, 0x40726E6660000000
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp796 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> undef, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp797 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecext9413 = extractelement <4 x float> %tmp797, i32 2
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add9414 = fadd float %vecext9413, 0xC057ECCCC0000000
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp798 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins9415 = insertelement <4 x float> %tmp798, float %add9414, i32 2
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %vecins9415, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp799 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecext9416 = extractelement <4 x float> %tmp799, i32 3
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add9417 = fadd float %vecext9416, 0x406B0CCCC0000000
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp800 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins9418 = insertelement <4 x float> %tmp800, float %add9417, i32 3
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %vecins9418, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> <float 3.555000e+02, float 0xC062E33340000000, float 0x4065C66660000000, float -3.645000e+02>, <4 x float>* undef
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp801 = load <4 x float>, <4 x float>* undef
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp802 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add9420 = fadd <4 x float> %tmp802, %tmp801
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %add9420, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp803 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecext9421 = extractelement <4 x float> %tmp803, i32 0
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp804 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins9423 = insertelement <4 x float> %tmp804, float undef, i32 0
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %vecins9423, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp805 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecext9424 = extractelement <4 x float> %tmp805, i32 1
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add9425 = fadd float %vecext9424, 0x4079C199A0000000
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp806 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins9426 = insertelement <4 x float> %tmp806, float %add9425, i32 1
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %vecins9426, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp807 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add9428 = fadd float undef, 0xC065466660000000
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp808 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins9429 = insertelement <4 x float> %tmp808, float %add9428, i32 2
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %vecins9429, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp809 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecext9430 = extractelement <4 x float> %tmp809, i32 3
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add9431 = fadd float %vecext9430, 0xC0742CCCC0000000
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp810 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins9432 = insertelement <4 x float> %tmp810, float %add9431, i32 3
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> <float 0xC07C7E6660000000, float 1.205000e+02, float 0x4050D999A0000000, float 0xC06B233340000000>, <4 x float>* undef
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp811 = load <4 x float>, <4 x float>* undef
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp812 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add9434 = fadd <4 x float> %tmp812, %tmp811
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add9436 = fadd float undef, -3.185000e+02
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp813 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins9437 = insertelement <4 x float> %tmp813, float %add9436, i32 0
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp814 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> undef, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp815 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecext9441 = extractelement <4 x float> %tmp815, i32 2
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add9442 = fadd float %vecext9441, 0xC079CE6660000000
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp816 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins9443 = insertelement <4 x float> %tmp816, float %add9442, i32 2
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %vecins9443, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp817 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecext9444 = extractelement <4 x float> %tmp817, i32 3
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add9445 = fadd float %vecext9444, 0xC06F533340000000
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp818 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins9446 = insertelement <4 x float> %tmp818, float %add9445, i32 3
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %vecins9446, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp819 = load <4 x float>, <4 x float>* undef
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp820 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add9448 = fadd <4 x float> %tmp820, %tmp819
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %add9448, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add9450 = fadd float undef, 0xC0718199A0000000
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp821 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins9451 = insertelement <4 x float> %tmp821, float %add9450, i32 0
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %vecins9451, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp822 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp823 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins9454 = insertelement <4 x float> %tmp823, float undef, i32 1
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %vecins9454, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp824 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecext9455 = extractelement <4 x float> %tmp824, i32 2
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add9456 = fadd float %vecext9455, -3.380000e+02
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp825 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins9457 = insertelement <4 x float> %tmp825, float %add9456, i32 2
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %vecins9457, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecext9458 = extractelement <4 x float> undef, i32 3
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp826 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins9460 = insertelement <4 x float> %tmp826, float undef, i32 3
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %vecins9460, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> <float 0x407B5E6660000000, float 0x40648999A0000000, float 0xC06B966660000000, float 0x40341999A0000000>, <4 x float>* undef
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp827 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add9462 = fadd <4 x float> %tmp827, undef
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %add9462, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp828 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecext9463 = extractelement <4 x float> %tmp828, i32 0
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp829 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins9465 = insertelement <4 x float> %tmp829, float undef, i32 0
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add9467 = fadd float undef, 0x405D666660000000
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp830 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins9468 = insertelement <4 x float> %tmp830, float %add9467, i32 1
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %vecins9468, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp831 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add9470 = fadd float undef, 0x4077033340000000
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp832 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> undef, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecext9472 = extractelement <4 x float> undef, i32 3
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add9473 = fadd float %vecext9472, 0x402DCCCCC0000000
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp833 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins9474 = insertelement <4 x float> %tmp833, float %add9473, i32 3
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %vecins9474, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> <float 0x404F733340000000, float 0x407AB4CCC0000000, float 0x40605999A0000000, float 0xC03E4CCCC0000000>, <4 x float>* undef
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp834 = load <4 x float>, <4 x float>* undef
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp835 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add9476 = fadd <4 x float> %tmp835, %tmp834
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %add9476, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp836 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecext9477 = extractelement <4 x float> %tmp836, i32 0
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add9478 = fadd float %vecext9477, 0xC07F266660000000
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp837 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins9479 = insertelement <4 x float> %tmp837, float %add9478, i32 0
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %vecins9479, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp838 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add9481 = fadd float undef, 0x407BE33340000000
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp839 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins9482 = insertelement <4 x float> %tmp839, float %add9481, i32 1
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %vecins9482, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecext9483 = extractelement <4 x float> undef, i32 2
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add9484 = fadd float %vecext9483, 0xC073E999A0000000
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp840 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins9485 = insertelement <4 x float> %tmp840, float %add9484, i32 2
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %vecins9485, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp841 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecext9486 = extractelement <4 x float> %tmp841, i32 3
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add9487 = fadd float %vecext9486, 0x4076E66660000000
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp842 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> undef, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> <float 0xC076B999A0000000, float 0xC0706CCCC0000000, float 0x407904CCC0000000, float 0x407EE199A0000000>, <4 x float>* undef
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp843 = load <4 x float>, <4 x float>* undef
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> undef, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp844 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecext9491 = extractelement <4 x float> %tmp844, i32 0
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add9492 = fadd float %vecext9491, 0x407C166660000000
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> undef, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add9495 = fadd float undef, 0x407DBB3340000000
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp845 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins9496 = insertelement <4 x float> %tmp845, float %add9495, i32 1
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %vecins9496, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp846 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecext9497 = extractelement <4 x float> %tmp846, i32 2
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add9498 = fadd float %vecext9497, 0x4042CCCCC0000000
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp847 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins9499 = insertelement <4 x float> %tmp847, float %add9498, i32 2
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %vecins9499, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp848 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add9501 = fadd float undef, 0x407D5CCCC0000000
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp849 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins9502 = insertelement <4 x float> %tmp849, float %add9501, i32 3
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %vecins9502, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp850 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add9504 = fadd <4 x float> %tmp850, undef
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %add9504, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp851 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add9506 = fadd float undef, 0x4076EE6660000000
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp852 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins9507 = insertelement <4 x float> %tmp852, float %add9506, i32 0
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %vecins9507, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp853 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add9509 = fadd float undef, 0xC0535999A0000000
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp854 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> undef, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp855 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecext9511 = extractelement <4 x float> %tmp855, i32 2
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add9512 = fadd float %vecext9511, 0xC076766660000000
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp856 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins9513 = insertelement <4 x float> %tmp856, float %add9512, i32 2
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %vecins9513, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp857 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecext9514 = extractelement <4 x float> %tmp857, i32 3
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp858 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins9516 = insertelement <4 x float> %tmp858, float undef, i32 3
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %vecins9516, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> <float 0x407254CCC0000000, float 0x407844CCC0000000, float 0xC04D9999A0000000, float 0xC0550CCCC0000000>, <4 x float>* undef
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp859 = load <4 x float>, <4 x float>* undef
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp860 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add9518 = fadd <4 x float> %tmp860, %tmp859
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp861 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp862 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins9521 = insertelement <4 x float> %tmp862, float undef, i32 0
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %vecins9521, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp863 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecext9522 = extractelement <4 x float> %tmp863, i32 1
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add9523 = fadd float %vecext9522, 0x4029333340000000
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp864 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins9524 = insertelement <4 x float> %tmp864, float %add9523, i32 1
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %vecins9524, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp865 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add9526 = fadd float undef, 0x4072833340000000
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp866 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins9527 = insertelement <4 x float> %tmp866, float %add9526, i32 2
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %vecins9527, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp867 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins9530 = insertelement <4 x float> undef, float undef, i32 3
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %vecins9530, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> <float 0x4072F4CCC0000000, float 0x4065CCCCC0000000, float 0x4051D33340000000, float 0x40680CCCC0000000>, <4 x float>* undef
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp868 = load <4 x float>, <4 x float>* undef
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp869 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add9532 = fadd <4 x float> %tmp869, %tmp868
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecext9533 = extractelement <4 x float> undef, i32 0
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp870 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins9535 = insertelement <4 x float> %tmp870, float undef, i32 0
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %vecins9535, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp871 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecext9536 = extractelement <4 x float> %tmp871, i32 1
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add9537 = fadd float %vecext9536, 0xC079F199A0000000
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp872 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins9538 = insertelement <4 x float> %tmp872, float %add9537, i32 1
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %vecins9538, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp873 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecext9542 = extractelement <4 x float> %tmp873, i32 3
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add9543 = fadd float %vecext9542, 0x4050D999A0000000
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add9576 = fadd float undef, 0x40219999A0000000
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins9577 = insertelement <4 x float> undef, float %add9576, i32 0
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %vecins9577, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp874 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins9580 = insertelement <4 x float> undef, float undef, i32 1
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %vecins9580, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp875 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecext9581 = extractelement <4 x float> %tmp875, i32 2
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add9582 = fadd float %vecext9581, 0xC07EF33340000000
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp876 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins9583 = insertelement <4 x float> %tmp876, float %add9582, i32 2
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %vecins9583, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp877 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> undef, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecext9673 = extractelement <4 x float> undef, i32 0
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add9674 = fadd float %vecext9673, 0xC04CF33340000000
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp878 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins9675 = insertelement <4 x float> %tmp878, float %add9674, i32 0
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %vecins9675, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecext9676 = extractelement <4 x float> undef, i32 1
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add9677 = fadd float %vecext9676, 1.455000e+02
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp879 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins9678 = insertelement <4 x float> %tmp879, float %add9677, i32 1
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp880 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecext9679 = extractelement <4 x float> %tmp880, i32 2
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add9680 = fadd float %vecext9679, 0x4073A33340000000
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp881 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins9681 = insertelement <4 x float> %tmp881, float %add9680, i32 2
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %vecins9681, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp882 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecext9682 = extractelement <4 x float> %tmp882, i32 3
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp883 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add9686 = fadd <4 x float> %tmp883, undef
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %add9686, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp884 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecext9687 = extractelement <4 x float> %tmp884, i32 0
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add9688 = fadd float %vecext9687, 0xC046666660000000
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp885 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins9689 = insertelement <4 x float> %tmp885, float %add9688, i32 0
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecext9690 = extractelement <4 x float> undef, i32 1
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add9691 = fadd float %vecext9690, 0x4034CCCCC0000000
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp886 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins9692 = insertelement <4 x float> %tmp886, float %add9691, i32 1
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp887 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecext9693 = extractelement <4 x float> %tmp887, i32 2
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add9694 = fadd float %vecext9693, -3.710000e+02
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp888 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins9695 = insertelement <4 x float> %tmp888, float %add9694, i32 2
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %vecins9695, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp889 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add9697 = fadd float undef, 0x4058D33340000000
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp890 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins9698 = insertelement <4 x float> %tmp890, float %add9697, i32 3
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %vecins9698, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> <float 0x4062CCCCC0000000, float 0x407AD999A0000000, float 0x40582CCCC0000000, float 0xC0712B3340000000>, <4 x float>* undef
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp891 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add9700 = fadd <4 x float> %tmp891, undef
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp892 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecext9701 = extractelement <4 x float> %tmp892, i32 0
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add9702 = fadd float %vecext9701, 0x406DC33340000000
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp893 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins9703 = insertelement <4 x float> %tmp893, float %add9702, i32 0
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %vecins9703, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp894 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecext9704 = extractelement <4 x float> %tmp894, i32 1
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add9705 = fadd float %vecext9704, 0xC073B33340000000
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp895 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins9706 = insertelement <4 x float> %tmp895, float %add9705, i32 1
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %vecins9706, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecext9707 = extractelement <4 x float> undef, i32 2
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add9708 = fadd float %vecext9707, 0xC0729999A0000000
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp896 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins9709 = insertelement <4 x float> %tmp896, float %add9708, i32 2
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %vecins9709, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp897 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecext9710 = extractelement <4 x float> %tmp897, i32 3
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins9712 = insertelement <4 x float> undef, float undef, i32 3
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %vecins9712, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> <float 0x4069F33340000000, float 0xC048266660000000, float 0x40638CCCC0000000, float 0xC07EC199A0000000>, <4 x float>* undef
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp898 = load <4 x float>, <4 x float>* undef
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add9714 = fadd <4 x float> undef, %tmp898
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %add9714, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp899 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecext9715 = extractelement <4 x float> %tmp899, i32 0
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp900 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins9717 = insertelement <4 x float> %tmp900, float undef, i32 0
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %vecins9717, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp901 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecext9718 = extractelement <4 x float> %tmp901, i32 1
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add9719 = fadd float %vecext9718, 0x406BC66660000000
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp902 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins9720 = insertelement <4 x float> %tmp902, float %add9719, i32 1
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %vecins9720, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp903 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecext9721 = extractelement <4 x float> %tmp903, i32 2
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add9722 = fadd float %vecext9721, -3.860000e+02
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp904 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins9723 = insertelement <4 x float> %tmp904, float %add9722, i32 2
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %vecins9723, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp905 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecext9724 = extractelement <4 x float> %tmp905, i32 3
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add9725 = fadd float %vecext9724, 0x407CF199A0000000
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp906 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins9726 = insertelement <4 x float> %tmp906, float %add9725, i32 3
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %vecins9726, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> <float -4.575000e+02, float 0x40713E6660000000, float 0x407D133340000000, float -1.425000e+02>, <4 x float>* undef
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp907 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add9728 = fadd <4 x float> %tmp907, undef
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %add9728, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp908 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecext9729 = extractelement <4 x float> %tmp908, i32 0
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add9730 = fadd float %vecext9729, 0x4079FB3340000000
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp909 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins9731 = insertelement <4 x float> %tmp909, float %add9730, i32 0
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %vecins9731, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp910 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add9733 = fadd float undef, 0xC050F33340000000
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp911 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins9734 = insertelement <4 x float> %tmp911, float %add9733, i32 1
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %vecins9734, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp912 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecext9735 = extractelement <4 x float> %tmp912, i32 2
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add9736 = fadd float %vecext9735, 0x40582CCCC0000000
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp913 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins9737 = insertelement <4 x float> %tmp913, float %add9736, i32 2
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %vecins9737, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp914 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecext9738 = extractelement <4 x float> %tmp914, i32 3
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins9740 = insertelement <4 x float> undef, float undef, i32 3
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %vecins9740, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> <float 2.150000e+02, float 0x405A2CCCC0000000, float 2.310000e+02, float 0x404E1999A0000000>, <4 x float>* undef
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp915 = load <4 x float>, <4 x float>* undef
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp916 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> undef, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp917 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecext9743 = extractelement <4 x float> %tmp917, i32 0
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add9744 = fadd float %vecext9743, -2.510000e+02
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins9745 = insertelement <4 x float> undef, float %add9744, i32 0
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %vecins9745, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp918 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecext9746 = extractelement <4 x float> %tmp918, i32 1
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add9747 = fadd float %vecext9746, 4.685000e+02
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp919 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins9748 = insertelement <4 x float> %tmp919, float %add9747, i32 1
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %vecins9748, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp920 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecext9749 = extractelement <4 x float> %tmp920, i32 2
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add9750 = fadd float %vecext9749, 1.600000e+01
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp921 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins9751 = insertelement <4 x float> %tmp921, float %add9750, i32 2
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %vecins9751, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp922 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecext9752 = extractelement <4 x float> %tmp922, i32 3
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add9753 = fadd float %vecext9752, -2.600000e+01
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp923 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins9754 = insertelement <4 x float> %tmp923, float %add9753, i32 3
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %vecins9754, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> <float 2.590000e+02, float 0x407B7199A0000000, float 0xC07ED199A0000000, float 0xC064FCCCC0000000>, <4 x float>* %.compoundliteral9755
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp924 = load <4 x float>, <4 x float>* %.compoundliteral9755
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp925 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add9756 = fadd <4 x float> %tmp925, %tmp924
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp926 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecext9757 = extractelement <4 x float> %tmp926, i32 0
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add9758 = fadd float %vecext9757, -1.810000e+02
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp927 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins9759 = insertelement <4 x float> %tmp927, float %add9758, i32 0
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %vecins9759, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp928 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecext9760 = extractelement <4 x float> %tmp928, i32 1
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add9761 = fadd float %vecext9760, 0xC07C3E6660000000
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp929 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins9762 = insertelement <4 x float> %tmp929, float %add9761, i32 1
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %vecins9762, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp930 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add9764 = fadd float undef, 0xC060E66660000000
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp931 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins9765 = insertelement <4 x float> %tmp931, float %add9764, i32 2
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %vecins9765, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp932 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecext9766 = extractelement <4 x float> %tmp932, i32 3
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add9767 = fadd float %vecext9766, 0xC0753E6660000000
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp933 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins9768 = insertelement <4 x float> %tmp933, float %add9767, i32 3
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %vecins9768, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> <float 0x4032CCCCC0000000, float -9.600000e+01, float -5.000000e+02, float 0x4078EE6660000000>, <4 x float>* %.compoundliteral9769
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp934 = load <4 x float>, <4 x float>* %.compoundliteral9769
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp935 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add9770 = fadd <4 x float> %tmp935, %tmp934
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %add9770, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp936 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecext9771 = extractelement <4 x float> %tmp936, i32 0
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add9772 = fadd float %vecext9771, 0xC0733E6660000000
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp937 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins9773 = insertelement <4 x float> %tmp937, float %add9772, i32 0
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %vecins9773, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp938 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecext9774 = extractelement <4 x float> %tmp938, i32 1
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add9775 = fadd float %vecext9774, 1.715000e+02
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp939 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins9776 = insertelement <4 x float> %tmp939, float %add9775, i32 1
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %vecins9776, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecext9816 = extractelement <4 x float> undef, i32 1
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp940 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins9818 = insertelement <4 x float> %tmp940, float undef, i32 1
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> undef, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp941 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add10388 = fadd float undef, 4.755000e+02
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp942 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins10389 = insertelement <4 x float> %tmp942, float %add10388, i32 0
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %vecins10389, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp943 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecext10390 = extractelement <4 x float> %tmp943, i32 1
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add10391 = fadd float %vecext10390, 0xC05AECCCC0000000
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp944 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins10392 = insertelement <4 x float> %tmp944, float %add10391, i32 1
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %vecins10392, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp945 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp946 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add10405 = fadd float undef, -5.650000e+01
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp947 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins10406 = insertelement <4 x float> %tmp947, float %add10405, i32 1
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %vecins10406, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp948 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecext10407 = extractelement <4 x float> %tmp948, i32 2
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add10408 = fadd float %vecext10407, 0xC06A633340000000
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp949 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins10409 = insertelement <4 x float> %tmp949, float %add10408, i32 2
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %vecins10409, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp950 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecext10410 = extractelement <4 x float> %tmp950, i32 3
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add10411 = fadd float %vecext10410, 0xC078D66660000000
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp951 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> undef, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> <float -2.340000e+02, float -4.720000e+02, float 4.350000e+02, float 0xC059A66660000000>, <4 x float>* %.compoundliteral10413
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp952 = load <4 x float>, <4 x float>* %.compoundliteral10413
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp953 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add10414 = fadd <4 x float> %tmp953, %tmp952
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %add10414, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp954 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecext10415 = extractelement <4 x float> %tmp954, i32 0
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add10416 = fadd float %vecext10415, 3.450000e+02
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp955 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins10417 = insertelement <4 x float> %tmp955, float %add10416, i32 0
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %vecins10417, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp956 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecext10418 = extractelement <4 x float> %tmp956, i32 1
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add10419 = fadd float %vecext10418, -6.000000e+00
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp957 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins10420 = insertelement <4 x float> %tmp957, float %add10419, i32 1
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %vecins10420, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add10422 = fadd float undef, 0xC0662CCCC0000000
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecext10424 = extractelement <4 x float> undef, i32 3
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> undef, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> <float 0x402B333340000000, float 0x40735E6660000000, float 0xC0567999A0000000, float 2.050000e+02>, <4 x float>* undef
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp958 = load <4 x float>, <4 x float>* undef
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp959 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add10428 = fadd <4 x float> %tmp959, %tmp958
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %add10428, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp960 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecext10429 = extractelement <4 x float> %tmp960, i32 0
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add10430 = fadd float %vecext10429, 0xC075166660000000
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp961 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add10436 = fadd float undef, 0xC06AF33340000000
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp962 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins10437 = insertelement <4 x float> %tmp962, float %add10436, i32 2
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %vecins10437, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecext10438 = extractelement <4 x float> undef, i32 3
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add10439 = fadd float %vecext10438, 0x405C7999A0000000
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp963 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins10440 = insertelement <4 x float> %tmp963, float %add10439, i32 3
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %vecins10440, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> <float 0xC065E999A0000000, float 0x4067D33340000000, float 0xC070133340000000, float 0x406B666660000000>, <4 x float>* undef
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp964 = load <4 x float>, <4 x float>* undef
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp965 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecext10443 = extractelement <4 x float> %tmp965, i32 0
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add10444 = fadd float %vecext10443, 0xC06CA999A0000000
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp966 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins10445 = insertelement <4 x float> %tmp966, float %add10444, i32 0
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %vecins10445, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp967 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecext10446 = extractelement <4 x float> %tmp967, i32 1
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add10447 = fadd float %vecext10446, 0x4064B999A0000000
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp968 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins10448 = insertelement <4 x float> %tmp968, float %add10447, i32 1
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %vecins10448, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp969 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecext10449 = extractelement <4 x float> %tmp969, i32 2
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add10450 = fadd float %vecext10449, 0x407B3CCCC0000000
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp970 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins10451 = insertelement <4 x float> %tmp970, float %add10450, i32 2
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %vecins10451, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp971 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecext10452 = extractelement <4 x float> %tmp971, i32 3
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add10453 = fadd float %vecext10452, -2.225000e+02
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins10454 = insertelement <4 x float> undef, float %add10453, i32 3
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> <float 0x406AFCCCC0000000, float 0xC07604CCC0000000, float 6.900000e+01, float 0xC060A66660000000>, <4 x float>* undef
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp972 = load <4 x float>, <4 x float>* undef
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp973 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add10456 = fadd <4 x float> %tmp973, %tmp972
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %add10456, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp974 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecext10457 = extractelement <4 x float> %tmp974, i32 0
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add10458 = fadd float %vecext10457, 2.375000e+02
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins10459 = insertelement <4 x float> undef, float %add10458, i32 0
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %vecins10459, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp975 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecext10460 = extractelement <4 x float> %tmp975, i32 1
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add10461 = fadd float %vecext10460, 0xC06B3999A0000000
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp976 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins10462 = insertelement <4 x float> %tmp976, float %add10461, i32 1
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp977 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecext10463 = extractelement <4 x float> %tmp977, i32 2
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add10464 = fadd float %vecext10463, 0x40655999A0000000
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp978 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins10465 = insertelement <4 x float> %tmp978, float %add10464, i32 2
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %vecins10465, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp979 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecext10466 = extractelement <4 x float> %tmp979, i32 3
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add10467 = fadd float %vecext10466, 0xC07B6999A0000000
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp980 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins10468 = insertelement <4 x float> %tmp980, float %add10467, i32 3
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %vecins10468, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> <float 0x4078833340000000, float 0x40786CCCC0000000, float 0xC0468CCCC0000000, float 0xC0793199A0000000>, <4 x float>* undef
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp981 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add10470 = fadd <4 x float> %tmp981, undef
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp982 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecext10471 = extractelement <4 x float> %tmp982, i32 0
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add10472 = fadd float %vecext10471, 0x40710CCCC0000000
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp983 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins10473 = insertelement <4 x float> %tmp983, float %add10472, i32 0
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %vecins10473, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp984 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecext10474 = extractelement <4 x float> %tmp984, i32 1
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add10475 = fadd float %vecext10474, 0x40709B3340000000
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp985 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins10476 = insertelement <4 x float> %tmp985, float %add10475, i32 1
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %vecins10476, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add10489 = fadd float undef, 0x4074666660000000
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp986 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins10490 = insertelement <4 x float> %tmp986, float %add10489, i32 1
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %vecins10490, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp987 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp988 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecext10508 = extractelement <4 x float> %tmp988, i32 3
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add10509 = fadd float %vecext10508, 0xC027333340000000
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp989 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins10510 = insertelement <4 x float> %tmp989, float %add10509, i32 3
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %vecins10510, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> <float 0x40656999A0000000, float 0xC073766660000000, float 1.685000e+02, float 0x40765199A0000000>, <4 x float>* undef
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp990 = load <4 x float>, <4 x float>* undef
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add10512 = fadd <4 x float> undef, %tmp990
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp991 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecext10513 = extractelement <4 x float> %tmp991, i32 0
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add10514 = fadd float %vecext10513, 0x405BB999A0000000
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp992 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins10515 = insertelement <4 x float> %tmp992, float %add10514, i32 0
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %vecins10515, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp993 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add10562 = fadd float undef, 2.035000e+02
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp994 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins10563 = insertelement <4 x float> %tmp994, float %add10562, i32 2
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %vecins10563, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp995 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecext10564 = extractelement <4 x float> %tmp995, i32 3
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add10565 = fadd float %vecext10564, 0x407AE4CCC0000000
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp996 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins10566 = insertelement <4 x float> %tmp996, float %add10565, i32 3
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %vecins10566, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> <float 0xC068B999A0000000, float 0xC050E66660000000, float 0xC0725999A0000000, float 0xC054D33340000000>, <4 x float>* %.compoundliteral10567
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp997 = load <4 x float>, <4 x float>* %.compoundliteral10567
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp998 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add10568 = fadd <4 x float> %tmp998, %tmp997
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp999 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecext10569 = extractelement <4 x float> %tmp999, i32 0
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add10570 = fadd float %vecext10569, 0x4074C33340000000
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp1000 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins10571 = insertelement <4 x float> %tmp1000, float %add10570, i32 0
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %vecins10571, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp1001 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecext10572 = extractelement <4 x float> %tmp1001, i32 1
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %add10573 = fadd float %vecext10572, 0x407DF33340000000
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %tmp1002 = load <4 x float>, <4 x float>* undef, align 16
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  %vecins10574 = insertelement <4 x float> %tmp1002, float %add10573, i32 1
+  tail call void asm sideeffect "", "~{q0}{q1}{q2}{q3}{q4}{q5}{q6}{q7}{q8}{q9}{q10}{q11}{q12}{q13}{q14}{q15}"()
+  store <4 x float> %vecins10574, <4 x float>* undef, align 16
+  %tmp1003 = load <4 x float>, <4 x float>* undef, align 16
+  %vecext10575 = extractelement <4 x float> %tmp1003, i32 2
+  %tmp1004 = load <4 x float>, <4 x float>* undef, align 16
+  %vecins10577 = insertelement <4 x float> %tmp1004, float undef, i32 2
+  store <4 x float> %vecins10577, <4 x float>* undef, align 16
+  %tmp1005 = load <4 x float>, <4 x float>* undef, align 16
+  %vecext10578 = extractelement <4 x float> %tmp1005, i32 3
+  %add10579 = fadd float %vecext10578, 0x4076566660000000
+  %tmp1006 = load <4 x float>, <4 x float>* undef, align 16
+  %vecins10580 = insertelement <4 x float> %tmp1006, float %add10579, i32 3
+  store <4 x float> %vecins10580, <4 x float>* undef, align 16
+  store <4 x float> <float 0x407CAB3340000000, float 1.685000e+02, float 0xC07B866660000000, float 0xC061ACCCC0000000>, <4 x float>* %.compoundliteral10581
+  %tmp1007 = load <4 x float>, <4 x float>* %.compoundliteral10581
+  store <4 x float> undef, <4 x float>* undef, align 16
+  %tmp1008 = load <4 x float>, <4 x float>* undef, align 16
+  %vecext10583 = extractelement <4 x float> %tmp1008, i32 0
+  %add10584 = fadd float %vecext10583, 0xC060533340000000
+  %tmp1009 = load <4 x float>, <4 x float>* undef, align 16
+  %vecins10585 = insertelement <4 x float> %tmp1009, float %add10584, i32 0
+  store <4 x float> %vecins10585, <4 x float>* undef, align 16
+  %tmp1010 = load <4 x float>, <4 x float>* undef, align 16
+  %vecext10586 = extractelement <4 x float> %tmp1010, i32 1
+  %add10587 = fadd float %vecext10586, 0xC0694CCCC0000000
+  %tmp1011 = load <4 x float>, <4 x float>* undef, align 16
+  %vecins10588 = insertelement <4 x float> %tmp1011, float %add10587, i32 1
+  store <4 x float> %vecins10588, <4 x float>* undef, align 16
+  %tmp1012 = load <4 x float>, <4 x float>* undef, align 16
+  %vecext10589 = extractelement <4 x float> %tmp1012, i32 2
+  %add10590 = fadd float %vecext10589, 0xC0541999A0000000
+  %tmp1013 = load <4 x float>, <4 x float>* undef, align 16
+  %vecins10591 = insertelement <4 x float> %tmp1013, float %add10590, i32 2
+  store <4 x float> %vecins10591, <4 x float>* undef, align 16
+  %tmp1014 = load <4 x float>, <4 x float>* undef, align 16
+  %vecext10592 = extractelement <4 x float> %tmp1014, i32 3
+  %add10593 = fadd float %vecext10592, 0xC06C566660000000
+  %tmp1015 = load <4 x float>, <4 x float>* undef, align 16
+  %vecins10594 = insertelement <4 x float> %tmp1015, float %add10593, i32 3
+  store <4 x float> %vecins10594, <4 x float>* undef, align 16
+  store <4 x float> <float 0x407A3199A0000000, float 0xC0659999A0000000, float 0x407E0999A0000000, float 0xC0334CCCC0000000>, <4 x float>* %.compoundliteral10595
+  %tmp1016 = load <4 x float>, <4 x float>* %.compoundliteral10595
+  %tmp1017 = load <4 x float>, <4 x float>* undef, align 16
+  %add10596 = fadd <4 x float> %tmp1017, %tmp1016
+  store <4 x float> %add10596, <4 x float>* undef, align 16
+  %tmp1018 = load <4 x float>, <4 x float>* undef, align 16
+  %vecext10597 = extractelement <4 x float> %tmp1018, i32 0
+  %add10598 = fadd float %vecext10597, 0x40640999A0000000
+  %tmp1019 = load <4 x float>, <4 x float>* undef, align 16
+  %vecins10599 = insertelement <4 x float> %tmp1019, float %add10598, i32 0
+  store <4 x float> %vecins10599, <4 x float>* undef, align 16
+  %tmp1020 = load <4 x float>, <4 x float>* undef, align 16
+  %vecext10600 = extractelement <4 x float> %tmp1020, i32 1
+  %add10601 = fadd float %vecext10600, 0xC073966660000000
+  %tmp1021 = load <4 x float>, <4 x float>* undef, align 16
+  %vecins10602 = insertelement <4 x float> %tmp1021, float %add10601, i32 1
+  %tmp1022 = load <4 x float>, <4 x float>* undef, align 16
+  %vecext10603 = extractelement <4 x float> %tmp1022, i32 2
+  %add10604 = fadd float %vecext10603, 1.780000e+02
+  %tmp1023 = load <4 x float>, <4 x float>* undef, align 16
+  %vecins10605 = insertelement <4 x float> %tmp1023, float %add10604, i32 2
+  store <4 x float> %vecins10605, <4 x float>* undef, align 16
+  %tmp1024 = load <4 x float>, <4 x float>* undef, align 16
+  %add10607 = fadd float undef, 0x4070A33340000000
+  %tmp1025 = load <4 x float>, <4 x float>* undef, align 16
+  store <4 x float> <float 0x407C5999A0000000, float 0x4046733340000000, float 0xC06E6CCCC0000000, float 0xC063C33340000000>, <4 x float>* %.compoundliteral10609
+  %tmp1026 = load <4 x float>, <4 x float>* %.compoundliteral10609
+  %tmp1027 = load <4 x float>, <4 x float>* undef, align 16
+  %tmp1028 = load <4 x float>, <4 x float>* undef, align 16
+  %vecext10611 = extractelement <4 x float> %tmp1028, i32 0
+  %add10612 = fadd float %vecext10611, 0x40757199A0000000
+  %vecins10613 = insertelement <4 x float> undef, float %add10612, i32 0
+  store <4 x float> %vecins10613, <4 x float>* undef, align 16
+  %tmp1029 = load <4 x float>, <4 x float>* undef, align 16
+  %vecext10614 = extractelement <4 x float> %tmp1029, i32 1
+  %add10615 = fadd float %vecext10614, 0x40740CCCC0000000
+  %tmp1030 = load <4 x float>, <4 x float>* undef, align 16
+  %vecins10616 = insertelement <4 x float> %tmp1030, float %add10615, i32 1
+  store <4 x float> %vecins10616, <4 x float>* undef, align 16
+  %tmp1031 = load <4 x float>, <4 x float>* undef, align 16
+  %vecext10617 = extractelement <4 x float> %tmp1031, i32 2
+  %add10618 = fadd float %vecext10617, 0xC012CCCCC0000000
+  %tmp1032 = load <4 x float>, <4 x float>* undef, align 16
+  %vecins10619 = insertelement <4 x float> %tmp1032, float %add10618, i32 2
+  store <4 x float> %vecins10619, <4 x float>* undef, align 16
+  %tmp1033 = load <4 x float>, <4 x float>* undef, align 16
+  %vecext10620 = extractelement <4 x float> %tmp1033, i32 3
+  %add10621 = fadd float %vecext10620, 0x406E566660000000
+  %tmp1034 = load <4 x float>, <4 x float>* undef, align 16
+  store <4 x float> <float 0x407B2199A0000000, float 0xC07D9CCCC0000000, float -4.350000e+01, float 0xC07D3B3340000000>, <4 x float>* %.compoundliteral10623
+  %tmp1035 = load <4 x float>, <4 x float>* %.compoundliteral10623
+  %add10624 = fadd <4 x float> undef, %tmp1035
+  %tmp1036 = load <4 x float>, <4 x float>* undef, align 16
+  %vecext10625 = extractelement <4 x float> %tmp1036, i32 0
+  %tmp1037 = load <4 x float>, <4 x float>* undef, align 16
+  %vecins10627 = insertelement <4 x float> %tmp1037, float undef, i32 0
+  store <4 x float> %vecins10627, <4 x float>* undef, align 16
+  %tmp1038 = load <4 x float>, <4 x float>* undef, align 16
+  %vecext10628 = extractelement <4 x float> %tmp1038, i32 1
+  %add10629 = fadd float %vecext10628, 0x407E3CCCC0000000
+  %tmp1039 = load <4 x float>, <4 x float>* undef, align 16
+  %vecins10630 = insertelement <4 x float> %tmp1039, float %add10629, i32 1
+  store <4 x float> %vecins10630, <4 x float>* undef, align 16
+  %tmp1040 = load <4 x float>, <4 x float>* undef, align 16
+  %vecext10631 = extractelement <4 x float> %tmp1040, i32 2
+  %tmp1041 = load <4 x float>, <4 x float>* undef, align 16
+  %tmp1042 = load <4 x float>, <4 x float>* undef, align 16
+  %vecext10634 = extractelement <4 x float> %tmp1042, i32 3
+  %add10635 = fadd float %vecext10634, 0xC067533340000000
+  %tmp1043 = load <4 x float>, <4 x float>* undef, align 16
+  %vecins10636 = insertelement <4 x float> %tmp1043, float %add10635, i32 3
+  store <4 x float> %vecins10636, <4 x float>* undef, align 16
+  store <4 x float> <float 1.950000e+02, float 0x407E8E6660000000, float 0x407D7CCCC0000000, float 0x407E166660000000>, <4 x float>* %.compoundliteral10637
+  %tmp1044 = load <4 x float>, <4 x float>* undef, align 16
+  %add10638 = fadd <4 x float> %tmp1044, undef
+  %tmp1045 = load <4 x float>, <4 x float>* undef, align 16
+  %vecext10639 = extractelement <4 x float> %tmp1045, i32 0
+  %add10640 = fadd float %vecext10639, 0x406CA33340000000
+  %tmp1046 = load <4 x float>, <4 x float>* undef, align 16
+  %vecins10641 = insertelement <4 x float> %tmp1046, float %add10640, i32 0
+  store <4 x float> %vecins10641, <4 x float>* undef, align 16
+  %tmp1047 = load <4 x float>, <4 x float>* undef, align 16
+  %vecext10642 = extractelement <4 x float> %tmp1047, i32 1
+  %add10643 = fadd float %vecext10642, 0xC07C8999A0000000
+  %tmp1048 = load <4 x float>, <4 x float>* undef, align 16
+  %vecins10644 = insertelement <4 x float> %tmp1048, float %add10643, i32 1
+  store <4 x float> %vecins10644, <4 x float>* undef, align 16
+  %tmp1049 = load <4 x float>, <4 x float>* undef, align 16
+  %vecext10645 = extractelement <4 x float> %tmp1049, i32 2
+  %tmp1050 = load <4 x float>, <4 x float>* undef, align 16
+  %tmp1051 = load <4 x float>, <4 x float>* undef, align 16
+  %vecins10748 = insertelement <4 x float> undef, float undef, i32 3
+  %tmp1052 = load <4 x float>, <4 x float>* %.compoundliteral10749
+  %add10750 = fadd <4 x float> undef, %tmp1052
+  store <4 x float> %add10750, <4 x float>* undef, align 16
+  %tmp1053 = load <4 x float>, <4 x float>* undef, align 16
+  %vecext10751 = extractelement <4 x float> %tmp1053, i32 0
+  %add10752 = fadd float %vecext10751, 0x4071B33340000000
+  %tmp1054 = load <4 x float>, <4 x float>* undef, align 16
+  %vecins10753 = insertelement <4 x float> %tmp1054, float %add10752, i32 0
+  store <4 x float> %vecins10753, <4 x float>* undef, align 16
+  %tmp1055 = load <4 x float>, <4 x float>* undef, align 16
+  %vecext10754 = extractelement <4 x float> %tmp1055, i32 1
+  %add10755 = fadd float %vecext10754, 0xC076A66660000000
+  %tmp1056 = load <4 x float>, <4 x float>* undef, align 16
+  %vecins10756 = insertelement <4 x float> %tmp1056, float %add10755, i32 1
+  store <4 x float> %vecins10756, <4 x float>* undef, align 16
+  %tmp1057 = load <4 x float>, <4 x float>* undef, align 16
+  %vecext10757 = extractelement <4 x float> %tmp1057, i32 2
+  %add10758 = fadd float %vecext10757, 3.800000e+01
+  %tmp1058 = load <4 x float>, <4 x float>* undef, align 16
+  %vecins10759 = insertelement <4 x float> %tmp1058, float %add10758, i32 2
+  store <4 x float> %vecins10759, <4 x float>* undef, align 16
+  %tmp1059 = load <4 x float>, <4 x float>* undef, align 16
+  %vecext10760 = extractelement <4 x float> %tmp1059, i32 3
+  store <4 x float> undef, <4 x float>* undef, align 16
+  store <4 x float> <float 0xC075BB3340000000, float 0x4074D4CCC0000000, float 0xC07A466660000000, float 0xC0691CCCC0000000>, <4 x float>* %.compoundliteral10763
+  %tmp1060 = load <4 x float>, <4 x float>* %.compoundliteral10763
+  %tmp1061 = load <4 x float>, <4 x float>* undef, align 16
+  %tmp1062 = load <4 x float>, <4 x float>* undef, align 16
+  %add10985 = fadd float undef, 0x405E933340000000
+  %tmp1063 = load <4 x float>, <4 x float>* undef, align 16
+  %vecins10986 = insertelement <4 x float> %tmp1063, float %add10985, i32 3
+  store <4 x float> %vecins10986, <4 x float>* undef, align 16
+  store <4 x float> <float 0xC0721E6660000000, float -4.180000e+02, float 0x406F366660000000, float 0xC055F999A0000000>, <4 x float>* %.compoundliteral10987
+  %tmp1064 = load <4 x float>, <4 x float>* %.compoundliteral10987
+  %tmp1065 = load <4 x float>, <4 x float>* undef, align 16
+  %vecins10994 = insertelement <4 x float> %tmp1065, float undef, i32 1
+  %tmp1066 = load <4 x float>, <4 x float>* undef, align 16
+  %vecext10995 = extractelement <4 x float> %tmp1066, i32 2
+  %add10996 = fadd float %vecext10995, 0x406F9999A0000000
+  %tmp1067 = load <4 x float>, <4 x float>* undef, align 16
+  %vecins10997 = insertelement <4 x float> %tmp1067, float %add10996, i32 2
+  store <4 x float> %vecins10997, <4 x float>* undef, align 16
+  %tmp1068 = load <4 x float>, <4 x float>* undef, align 16
+  %vecext10998 = extractelement <4 x float> %tmp1068, i32 3
+  %add10999 = fadd float %vecext10998, -2.765000e+02
+  %tmp1069 = load <4 x float>, <4 x float>* undef, align 16
+  %vecins11000 = insertelement <4 x float> %tmp1069, float %add10999, i32 3
+  store <4 x float> %vecins11000, <4 x float>* undef, align 16
+  store <4 x float> <float 0x4078F999A0000000, float 0xC06D166660000000, float 0x40501999A0000000, float 0x406FC999A0000000>, <4 x float>* %.compoundliteral11001
+  %tmp1070 = load <4 x float>, <4 x float>* undef, align 16
+  %add11002 = fadd <4 x float> %tmp1070, undef
+  %vecext11003 = extractelement <4 x float> undef, i32 0
+  %vecext11009 = extractelement <4 x float> undef, i32 2
+  %tmp1071 = load <4 x float>, <4 x float>* undef, align 16
+  %vecins11033 = insertelement <4 x float> %tmp1071, float undef, i32 0
+  store <4 x float> %vecins11033, <4 x float>* undef, align 16
+  %tmp1072 = load <4 x float>, <4 x float>* undef, align 16
+  %vecext11034 = extractelement <4 x float> %tmp1072, i32 1
+  %add11035 = fadd float %vecext11034, 0x4056D33340000000
+  %tmp1073 = load <4 x float>, <4 x float>* undef, align 16
+  %vecins11036 = insertelement <4 x float> %tmp1073, float %add11035, i32 1
+  store <4 x float> %vecins11036, <4 x float>* undef, align 16
+  %tmp1074 = load <4 x float>, <4 x float>* undef, align 16
+  %vecext11037 = extractelement <4 x float> %tmp1074, i32 2
+  %add11038 = fadd float %vecext11037, 0xC06EA33340000000
+  %tmp1075 = load <4 x float>, <4 x float>* undef, align 16
+  store <4 x float> undef, <4 x float>* undef, align 16
+  %tmp1076 = load <4 x float>, <4 x float>* undef, align 16
+  %vecext11040 = extractelement <4 x float> %tmp1076, i32 3
+  %add11041 = fadd float %vecext11040, 0x40746CCCC0000000
+  %tmp1077 = load <4 x float>, <4 x float>* undef, align 16
+  %vecins11042 = insertelement <4 x float> %tmp1077, float %add11041, i32 3
+  store <4 x float> <float 0x405DD999A0000000, float -3.775000e+02, float -1.265000e+02, float 0xC065C66660000000>, <4 x float>* undef
+  %tmp1078 = load <4 x float>, <4 x float>* undef, align 16
+  %add11044 = fadd <4 x float> %tmp1078, undef
+  store <4 x float> %add11044, <4 x float>* undef, align 16
+  %tmp1079 = load <4 x float>, <4 x float>* undef, align 16
+  %vecext11045 = extractelement <4 x float> %tmp1079, i32 0
+  %add11046 = fadd float %vecext11045, 0xC076E66660000000
+  %tmp1080 = load <4 x float>, <4 x float>* undef, align 16
+  %vecins11047 = insertelement <4 x float> %tmp1080, float %add11046, i32 0
+  %tmp1081 = load <4 x float>, <4 x float>* undef, align 16
+  %vecext11048 = extractelement <4 x float> %tmp1081, i32 1
+  %add11049 = fadd float %vecext11048, 4.100000e+02
+  %vecins11064 = insertelement <4 x float> undef, float undef, i32 1
+  %add11074 = fadd float undef, 0xC06FF999A0000000
+  %tmp1082 = load <4 x float>, <4 x float>* undef, align 16
+  %vecins11075 = insertelement <4 x float> %tmp1082, float %add11074, i32 0
+  store <4 x float> %vecins11075, <4 x float>* undef, align 16
+  %add11077 = fadd float undef, 0xC075D33340000000
+  %tmp1083 = load <4 x float>, <4 x float>* undef, align 16
+  %tmp1084 = load <4 x float>, <4 x float>* undef, align 16
+  store <4 x float> undef, <4 x float>* undef, align 16
+  %tmp1085 = load <4 x float>, <4 x float>* undef, align 16
+  %vecext11093 = extractelement <4 x float> %tmp1085, i32 2
+  %add11094 = fadd float %vecext11093, 0xC07CD66660000000
+  %tmp1086 = load <4 x float>, <4 x float>* undef, align 16
+  %vecins11095 = insertelement <4 x float> %tmp1086, float %add11094, i32 2
+  store <4 x float> %vecins11095, <4 x float>* undef, align 16
+  store <4 x float> undef, <4 x float>* undef, align 16
+  store <4 x float> <float 0x4061F66660000000, float 0xC076DB3340000000, float 0xC055A66660000000, float 2.415000e+02>, <4 x float>* undef
+  %tmp1087 = load <4 x float>, <4 x float>* undef
+  store <4 x float> undef, <4 x float>* undef, align 16
+  %tmp1088 = load <4 x float>, <4 x float>* undef, align 16
+  %vecext11513 = extractelement <4 x float> %tmp1088, i32 2
+  %add11514 = fadd float %vecext11513, 0xC07C7199A0000000
+  %vecins11515 = insertelement <4 x float> undef, float %add11514, i32 2
+  store <4 x float> %vecins11515, <4 x float>* undef, align 16
+  %add11520 = fadd <4 x float> undef, undef
+  store <4 x float> %add11520, <4 x float>* undef, align 16
+  %vecext11521 = extractelement <4 x float> undef, i32 0
+  %add11522 = fadd float %vecext11521, 0x4041733340000000
+  %tmp1089 = load <4 x float>, <4 x float>* undef, align 16
+  store <4 x float> undef, <4 x float>* undef, align 16
+  %tmp1090 = load <4 x float>, <4 x float>* undef
+  %tmp1091 = load <4 x float>, <4 x float>* undef, align 16
+  %add11562 = fadd <4 x float> %tmp1091, %tmp1090
+  %tmp1092 = load <4 x float>, <4 x float>* undef, align 16
+  %add11564 = fadd float undef, 0xC0411999A0000000
+  %tmp1093 = load <4 x float>, <4 x float>* undef, align 16
+  %vecins11565 = insertelement <4 x float> %tmp1093, float %add11564, i32 0
+  store <4 x float> undef, <4 x float>* undef, align 16
+  %vecext11586 = extractelement <4 x float> undef, i32 3
+  %add11587 = fadd float %vecext11586, 3.760000e+02
+  %tmp1094 = load <4 x float>, <4 x float>* undef, align 16
+  store <4 x float> undef, <4 x float>* undef, align 16
+  store <4 x float> <float 0xC06ED999A0000000, float 1.380000e+02, float 0xC073AB3340000000, float 0x4078A66660000000>, <4 x float>* undef
+  %tmp1095 = load <4 x float>, <4 x float>* undef
+  %tmp1096 = load <4 x float>, <4 x float>* undef, align 16
+  %tmp1097 = load <4 x float>, <4 x float>* undef, align 16
+  %tmp1098 = load <4 x float>, <4 x float>* undef, align 16
+  %vecins11593 = insertelement <4 x float> %tmp1098, float undef, i32 0
+  %vecext11594 = extractelement <4 x float> undef, i32 1
+  %tmp1099 = load <4 x float>, <4 x float>* undef, align 16
+  %vecins11596 = insertelement <4 x float> %tmp1099, float undef, i32 1
+  store <4 x float> %vecins11596, <4 x float>* undef, align 16
+  %tmp1100 = load <4 x float>, <4 x float>* undef, align 16
+  %vecext11597 = extractelement <4 x float> %tmp1100, i32 2
+  %add11598 = fadd float %vecext11597, 0x40430CCCC0000000
+  %tmp1101 = load <4 x float>, <4 x float>* undef, align 16
+  %vecins11599 = insertelement <4 x float> %tmp1101, float %add11598, i32 2
+  %tmp1102 = load <4 x float>, <4 x float>* undef, align 16
+  %vecext11600 = extractelement <4 x float> %tmp1102, i32 3
+  %tmp1103 = load <4 x float>, <4 x float>* undef, align 16
+  %vecins11602 = insertelement <4 x float> %tmp1103, float undef, i32 3
+  store <4 x float> %vecins11602, <4 x float>* undef, align 16
+  %tmp1104 = load <4 x float>, <4 x float>* undef
+  %tmp1105 = load <4 x float>, <4 x float>* undef, align 16
+  %add11604 = fadd <4 x float> %tmp1105, %tmp1104
+  %tmp1106 = load <4 x float>, <4 x float>* undef, align 16
+  %vecext11605 = extractelement <4 x float> %tmp1106, i32 0
+  %tmp1107 = load <4 x float>, <4 x float>* undef, align 16
+  %vecins11607 = insertelement <4 x float> %tmp1107, float undef, i32 0
+  %vecins11621 = insertelement <4 x float> undef, float undef, i32 0
+  %vecins11630 = insertelement <4 x float> undef, float undef, i32 3
+  store <4 x float> %vecins11630, <4 x float>* undef, align 16
+  store <4 x float> <float -1.190000e+02, float 0x402F666660000000, float 0xC07BD33340000000, float -1.595000e+02>, <4 x float>* %.compoundliteral11631
+  %tmp1108 = load <4 x float>, <4 x float>* %.compoundliteral11631
+  %tmp1109 = load <4 x float>, <4 x float>* undef, align 16
+  store <4 x float> undef, <4 x float>* undef, align 16
+  %add11634 = fadd float undef, -1.075000e+02
+  %vecext11647 = extractelement <4 x float> undef, i32 0
+  %add11648 = fadd float %vecext11647, 0x40775999A0000000
+  %tmp1110 = load <4 x float>, <4 x float>* undef, align 16
+  %vecext11650 = extractelement <4 x float> undef, i32 1
+  %tmp1111 = load <4 x float>, <4 x float>* undef, align 16
+  %vecins11784 = insertelement <4 x float> %tmp1111, float undef, i32 3
+  store <4 x float> %vecins11784, <4 x float>* undef, align 16
+  store <4 x float> <float 1.605000e+02, float 0x4068366660000000, float 2.820000e+02, float 0x407CF66660000000>, <4 x float>* %.compoundliteral11785
+  %tmp1112 = load <4 x float>, <4 x float>* %.compoundliteral11785
+  %add11786 = fadd <4 x float> undef, %tmp1112
+  store <4 x float> %add11786, <4 x float>* undef, align 16
+  %tmp1113 = load <4 x float>, <4 x float>* undef, align 16
+  %vecext11787 = extractelement <4 x float> %tmp1113, i32 0
+  %vecext11807 = extractelement <4 x float> undef, i32 2
+  %add11808 = fadd float %vecext11807, 4.535000e+02
+  %tmp1114 = load <4 x float>, <4 x float>* undef, align 16
+  %vecext11810 = extractelement <4 x float> undef, i32 3
+  %add11811 = fadd float %vecext11810, 0x4068F66660000000
+  %tmp1115 = load <4 x float>, <4 x float>* undef, align 16
+  %vecins11812 = insertelement <4 x float> %tmp1115, float %add11811, i32 3
+  store <4 x float> %vecins11812, <4 x float>* undef, align 16
+  %tmp1116 = load <4 x float>, <4 x float>* undef
+  %tmp1117 = load <4 x float>, <4 x float>* undef, align 16
+  %vecext11958 = extractelement <4 x float> undef, i32 1
+  store <4 x float> undef, <4 x float>* undef, align 16
+  %vecext11961 = extractelement <4 x float> undef, i32 2
+  %add11962 = fadd float %vecext11961, -3.680000e+02
+  %tmp1118 = load <4 x float>, <4 x float>* undef, align 16
+  store <4 x float> undef, <4 x float>* undef, align 16
+  %add11965 = fadd float undef, 0x4061133340000000
+  store <4 x float> undef, <4 x float>* undef, align 16
+  %tmp1119 = load <4 x float>, <4 x float>* undef, align 16
+  %vecext11975 = extractelement <4 x float> %tmp1119, i32 2
+  %tmp1120 = load <4 x float>, <4 x float>* undef, align 16
+  %vecins11977 = insertelement <4 x float> %tmp1120, float undef, i32 2
+  store <4 x float> %vecins11977, <4 x float>* undef, align 16
+  %vecext11978 = extractelement <4 x float> undef, i32 3
+  %add11979 = fadd float %vecext11978, 0xC0688999A0000000
+  %tmp1121 = load <4 x float>, <4 x float>* undef, align 16
+  %vecins11980 = insertelement <4 x float> %tmp1121, float %add11979, i32 3
+  store <4 x float> %vecins11980, <4 x float>* undef, align 16
+  %add11982 = fadd <4 x float> undef, undef
+  store <4 x float> %add11982, <4 x float>* undef, align 16
+  %tmp1122 = load <4 x float>, <4 x float>* undef, align 16
+  %vecext11983 = extractelement <4 x float> %tmp1122, i32 0
+  %add11984 = fadd float %vecext11983, 0xC075966660000000
+  %tmp1123 = load <4 x float>, <4 x float>* undef, align 16
+  %vecins12005 = insertelement <4 x float> undef, float undef, i32 2
+  store <4 x float> %vecins12005, <4 x float>* undef, align 16
+  %tmp1124 = load <4 x float>, <4 x float>* undef, align 16
+  %add12007 = fadd float undef, 0xC07124CCC0000000
+  %vecins12008 = insertelement <4 x float> undef, float %add12007, i32 3
+  store <4 x float> %vecins12008, <4 x float>* undef, align 16
+  %tmp1125 = load <4 x float>, <4 x float>* undef, align 16
+  store <4 x float> undef, <4 x float>* undef, align 16
+  %tmp1126 = load <4 x float>, <4 x float>* undef, align 16
+  %add12012 = fadd float undef, 0xC0750CCCC0000000
+  %tmp1127 = load <4 x float>, <4 x float>* undef, align 16
+  %vecins12013 = insertelement <4 x float> %tmp1127, float %add12012, i32 0
+  store <4 x float> %vecins12013, <4 x float>* undef, align 16
+  %tmp1128 = load <4 x float>, <4 x float>* undef, align 16
+  %add12015 = fadd float undef, 0x4079CE6660000000
+  %tmp1129 = load <4 x float>, <4 x float>* undef, align 16
+  %vecins12016 = insertelement <4 x float> %tmp1129, float %add12015, i32 1
+  store <4 x float> %vecins12016, <4 x float>* undef, align 16
+  %add12018 = fadd float undef, 3.555000e+02
+  %tmp1130 = load <4 x float>, <4 x float>* undef, align 16
+  %vecins12019 = insertelement <4 x float> %tmp1130, float %add12018, i32 2
+  %tmp1131 = load <4 x float>, <4 x float>* undef, align 16
+  %vecext12020 = extractelement <4 x float> %tmp1131, i32 3
+  store <4 x float> undef, <4 x float>* undef, align 16
+  %vecext12028 = extractelement <4 x float> undef, i32 1
+  store <4 x float> undef, <4 x float>* undef, align 16
+  store <4 x float> <float 0x40791999A0000000, float 0x407C7CCCC0000000, float 0x4070F33340000000, float 0xC056ECCCC0000000>, <4 x float>* undef
+  %tmp1132 = load <4 x float>, <4 x float>* undef, align 16
+  %add12038 = fadd <4 x float> %tmp1132, undef
+  %tmp1133 = load <4 x float>, <4 x float>* undef, align 16
+  %vecext12042 = extractelement <4 x float> %tmp1133, i32 1
+  %add12043 = fadd float %vecext12042, 0x402F9999A0000000
+  %tmp1134 = load <4 x float>, <4 x float>* undef, align 16
+  %vecins12044 = insertelement <4 x float> %tmp1134, float %add12043, i32 1
+  store <4 x float> %vecins12044, <4 x float>* undef, align 16
+  %vecext12045 = extractelement <4 x float> undef, i32 2
+  %add12046 = fadd float %vecext12045, 0xC07EF33340000000
+  %tmp1135 = load <4 x float>, <4 x float>* undef, align 16
+  %vecins12047 = insertelement <4 x float> %tmp1135, float %add12046, i32 2
+  store <4 x float> %vecins12047, <4 x float>* undef, align 16
+  store <4 x float> undef, <4 x float>* undef, align 16
+  %tmp1136 = load <4 x float>, <4 x float>* undef, align 16
+  %vecext12112 = extractelement <4 x float> %tmp1136, i32 1
+  %tmp1137 = load <4 x float>, <4 x float>* undef, align 16
+  store <4 x float> undef, <4 x float>* undef, align 16
+  %add12116 = fadd float undef, 0xC074F4CCC0000000
+  %tmp1138 = load <4 x float>, <4 x float>* undef, align 16
+  %vecins12117 = insertelement <4 x float> %tmp1138, float %add12116, i32 2
+  store <4 x float> %vecins12117, <4 x float>* undef, align 16
+  %tmp1139 = load <4 x float>, <4 x float>* undef, align 16
+  %vecext12118 = extractelement <4 x float> %tmp1139, i32 3
+  %add12119 = fadd float %vecext12118, 0xC0638CCCC0000000
+  %tmp1140 = load <4 x float>, <4 x float>* undef, align 16
+  %vecins12120 = insertelement <4 x float> %tmp1140, float %add12119, i32 3
+  %add12152 = fadd float undef, 0x4039333340000000
+  %tmp1141 = load <4 x float>, <4 x float>* undef, align 16
+  %vecins12153 = insertelement <4 x float> %tmp1141, float %add12152, i32 0
+  %vecext12154 = extractelement <4 x float> undef, i32 1
+  %add12155 = fadd float %vecext12154, 0xC07BBB3340000000
+  %tmp1142 = load <4 x float>, <4 x float>* undef, align 16
+  %vecins12156 = insertelement <4 x float> %tmp1142, float %add12155, i32 1
+  %tmp1143 = load <4 x float>, <4 x float>* undef, align 16
+  %vecext12157 = extractelement <4 x float> %tmp1143, i32 2
+  %add12158 = fadd float %vecext12157, 0xC0428CCCC0000000
+  %tmp1144 = load <4 x float>, <4 x float>* undef, align 16
+  %vecins12159 = insertelement <4 x float> %tmp1144, float %add12158, i32 2
+  %tmp1145 = load <4 x float>, <4 x float>* undef, align 16
+  %vecext12160 = extractelement <4 x float> %tmp1145, i32 3
+  %add12161 = fadd float %vecext12160, 0x407B1999A0000000
+  %tmp1146 = load <4 x float>, <4 x float>* undef, align 16
+  %vecins12162 = insertelement <4 x float> %tmp1146, float %add12161, i32 3
+  store <4 x float> %vecins12162, <4 x float>* undef, align 16
+  %tmp1147 = load <4 x float>, <4 x float>* undef
+  %tmp1148 = load <4 x float>, <4 x float>* undef, align 16
+  %tmp1149 = load <4 x float>, <4 x float>* undef, align 16
+  %vecext12182 = extractelement <4 x float> %tmp1149, i32 1
+  %tmp1150 = load <4 x float>, <4 x float>* undef, align 16
+  store <4 x float> undef, <4 x float>* undef, align 16
+  store <4 x float> <float 0x4061833340000000, float 0x405CA66660000000, float -1.275000e+02, float 0x405BC66660000000>, <4 x float>* undef
+  %add12208 = fadd float undef, 0x407854CCC0000000
+  %tmp1151 = load <4 x float>, <4 x float>* undef, align 16
+  store <4 x float> undef, <4 x float>* undef, align 16
+  %tmp1152 = load <4 x float>, <4 x float>* undef, align 16
+  %tmp1153 = load <4 x float>, <4 x float>* undef, align 16
+  %vecins12218 = insertelement <4 x float> undef, float undef, i32 3
+  store <4 x float> %vecins12218, <4 x float>* undef, align 16
+  store <4 x float> <float 0x407C3CCCC0000000, float 0xC057C66660000000, float 2.605000e+02, float 0xC07974CCC0000000>, <4 x float>* undef
+  %tmp1154 = load <4 x float>, <4 x float>* undef
+  %tmp1155 = load <4 x float>, <4 x float>* undef, align 16
+  %add12220 = fadd <4 x float> %tmp1155, %tmp1154
+  %tmp1156 = load <4 x float>, <4 x float>* undef, align 16
+  %tmp1157 = load <4 x float>, <4 x float>* undef, align 16
+  %vecins12223 = insertelement <4 x float> %tmp1157, float undef, i32 0
+  store <4 x float> %vecins12223, <4 x float>* undef, align 16
+  %tmp1158 = load <4 x float>, <4 x float>* undef, align 16
+  %add12242 = fadd float undef, 0x4067E33340000000
+  %tmp1159 = load <4 x float>, <4 x float>* undef, align 16
+  %vecins12243 = insertelement <4 x float> %tmp1159, float %add12242, i32 2
+  store <4 x float> %vecins12243, <4 x float>* undef, align 16
+  %tmp1160 = load <4 x float>, <4 x float>* undef, align 16
+  %vecext12244 = extractelement <4 x float> %tmp1160, i32 3
+  %add12245 = fadd float %vecext12244, 0x4071AE6660000000
+  %tmp1161 = load <4 x float>, <4 x float>* undef, align 16
+  %vecins12246 = insertelement <4 x float> %tmp1161, float %add12245, i32 3
+  store <4 x float> %vecins12246, <4 x float>* undef, align 16
+  store <4 x float> <float -4.880000e+02, float 0xC079966660000000, float -8.450000e+01, float 0xC0464CCCC0000000>, <4 x float>* %.compoundliteral12247
+  %tmp1162 = load <4 x float>, <4 x float>* %.compoundliteral12247
+  %tmp1163 = load <4 x float>, <4 x float>* undef, align 16
+  %add12248 = fadd <4 x float> %tmp1163, %tmp1162
+  store <4 x float> %add12248, <4 x float>* undef, align 16
+  %tmp1164 = load <4 x float>, <4 x float>* undef, align 16
+  %vecext12249 = extractelement <4 x float> %tmp1164, i32 0
+  %add12250 = fadd float %vecext12249, 1.075000e+02
+  %tmp1165 = load <4 x float>, <4 x float>* undef, align 16
+  store <4 x float> undef, <4 x float>* undef, align 16
+  %tmp1166 = load <4 x float>, <4 x float>* undef, align 16
+  %vecext12252 = extractelement <4 x float> %tmp1166, i32 1
+  %add12253 = fadd float %vecext12252, 0xC0662CCCC0000000
+  %tmp1167 = load <4 x float>, <4 x float>* undef, align 16
+  %vecins12254 = insertelement <4 x float> %tmp1167, float %add12253, i32 1
+  store <4 x float> %vecins12254, <4 x float>* undef, align 16
+  %tmp1168 = load <4 x float>, <4 x float>* undef, align 16
+  %vecext12255 = extractelement <4 x float> %tmp1168, i32 2
+  %add12256 = fadd float %vecext12255, 0x40554CCCC0000000
+  store <4 x float> undef, <4 x float>* undef, align 16
+  %add13141 = fadd float undef, 0x40768999A0000000
+  %tmp1169 = load <4 x float>, <4 x float>* undef, align 16
+  %vecins13142 = insertelement <4 x float> %tmp1169, float %add13141, i32 3
+  store <4 x float> %vecins13142, <4 x float>* undef, align 16
+  %tmp1170 = load <4 x float>, <4 x float>* undef
+  %add13144 = fadd <4 x float> undef, %tmp1170
+  store <4 x float> %add13144, <4 x float>* undef, align 16
+  %tmp1171 = load <4 x float>, <4 x float>* undef, align 16
+  %vecext13145 = extractelement <4 x float> %tmp1171, i32 0
+  %add13146 = fadd float %vecext13145, 3.975000e+02
+  %tmp1172 = load <4 x float>, <4 x float>* undef, align 16
+  %vecext13378 = extractelement <4 x float> %tmp1172, i32 3
+  %add13379 = fadd float %vecext13378, 0xC053B33340000000
+  %tmp1173 = load <4 x float>, <4 x float>* undef, align 16
+  %vecins13380 = insertelement <4 x float> %tmp1173, float %add13379, i32 3
+  store <4 x float> %vecins13380, <4 x float>* undef, align 16
+  %tmp1174 = load <4 x float>, <4 x float>* undef, align 16
+  %vecins13408 = insertelement <4 x float> %tmp1174, float undef, i32 3
+  store <4 x float> %vecins13408, <4 x float>* undef, align 16
+  store <4 x float> <float 0xC0455999A0000000, float 0xC07D366660000000, float 4.240000e+02, float -1.670000e+02>, <4 x float>* undef
+  %tmp1175 = load <4 x float>, <4 x float>* undef
+  %tmp1176 = load <4 x float>, <4 x float>* undef, align 16
+  %add13410 = fadd <4 x float> %tmp1176, %tmp1175
+  store <4 x float> %add13410, <4 x float>* undef, align 16
+  %tmp1177 = load <4 x float>, <4 x float>* undef, align 16
+  %add13412 = fadd float undef, 0xC0708999A0000000
+  %tmp1178 = load <4 x float>, <4 x float>* undef, align 16
+  %vecins13413 = insertelement <4 x float> %tmp1178, float %add13412, i32 0
+  store <4 x float> undef, <4 x float>* undef, align 16
+  %vecext13428 = extractelement <4 x float> undef, i32 1
+  %add13429 = fadd float %vecext13428, 0xC063BCCCC0000000
+  %tmp1179 = load <4 x float>, <4 x float>* undef, align 16
+  %vecins13430 = insertelement <4 x float> %tmp1179, float %add13429, i32 1
+  store <4 x float> %vecins13430, <4 x float>* undef, align 16
+  %tmp1180 = load <4 x float>, <4 x float>* undef, align 16
+  %vecext13431 = extractelement <4 x float> %tmp1180, i32 2
+  %vecins13433 = insertelement <4 x float> undef, float undef, i32 2
+  store <4 x float> undef, <4 x float>* undef, align 16
+  %add13449 = fadd float undef, 4.590000e+02
+  %tmp1181 = load <4 x float>, <4 x float>* undef, align 16
+  %vecins13450 = insertelement <4 x float> %tmp1181, float %add13449, i32 3
+  store <4 x float> %vecins13450, <4 x float>* undef, align 16
+  store <4 x float> <float 0xC073A66660000000, float 0xC041B33340000000, float 0x4066233340000000, float 0x4071C33340000000>, <4 x float>* undef
+  %tmp1182 = load <4 x float>, <4 x float>* undef
+  %tmp1183 = load <4 x float>, <4 x float>* undef, align 16
+  %add13452 = fadd <4 x float> %tmp1183, %tmp1182
+  store <4 x float> %add13452, <4 x float>* undef, align 16
+  %tmp1184 = load <4 x float>, <4 x float>* undef, align 16
+  %vecext13453 = extractelement <4 x float> %tmp1184, i32 0
+  %add13454 = fadd float %vecext13453, 0xC072866660000000
+  %tmp1185 = load <4 x float>, <4 x float>* undef, align 16
+  %vecins13455 = insertelement <4 x float> %tmp1185, float %add13454, i32 0
+  %add13471 = fadd float undef, 0xC0556CCCC0000000
+  %tmp1186 = load <4 x float>, <4 x float>* undef, align 16
+  %vecins13472 = insertelement <4 x float> %tmp1186, float %add13471, i32 1
+  store <4 x float> %vecins13472, <4 x float>* undef, align 16
+  %tmp1187 = load <4 x float>, <4 x float>* undef, align 16
+  %vecext13473 = extractelement <4 x float> %tmp1187, i32 2
+  %add13474 = fadd float %vecext13473, 0xC0786999A0000000
+  %tmp1188 = load <4 x float>, <4 x float>* undef, align 16
+  %vecins13475 = insertelement <4 x float> %tmp1188, float %add13474, i32 2
+  store <4 x float> %vecins13475, <4 x float>* undef, align 16
+  %add13477 = fadd float undef, 0xC07C3E6660000000
+  %tmp1189 = load <4 x float>, <4 x float>* undef, align 16
+  %vecins13478 = insertelement <4 x float> %tmp1189, float %add13477, i32 3
+  store <4 x float> %vecins13478, <4 x float>* undef, align 16
+  store <4 x float> <float -4.740000e+02, float 0x4023CCCCC0000000, float 0xC05C266660000000, float 0x407B7199A0000000>, <4 x float>* undef
+  %tmp1190 = load <4 x float>, <4 x float>* undef, align 16
+  %add13480 = fadd <4 x float> %tmp1190, undef
+  store <4 x float> %add13480, <4 x float>* undef, align 16
+  %tmp1191 = load <4 x float>, <4 x float>* undef, align 16
+  %vecext13481 = extractelement <4 x float> %tmp1191, i32 0
+  %add13482 = fadd float %vecext13481, 0xC07BA4CCC0000000
+  %tmp1192 = load <4 x float>, <4 x float>* undef, align 16
+  %vecins13483 = insertelement <4 x float> %tmp1192, float %add13482, i32 0
+  store <4 x float> %vecins13483, <4 x float>* undef, align 16
+  %tmp1193 = load <4 x float>, <4 x float>* undef, align 16
+  %add13485 = fadd float undef, 0x406B1999A0000000
+  %tmp1194 = load <4 x float>, <4 x float>* undef, align 16
+  %vecins13486 = insertelement <4 x float> %tmp1194, float %add13485, i32 1
+  store <4 x float> %vecins13486, <4 x float>* undef, align 16
+  %tmp1195 = load <4 x float>, <4 x float>* undef, align 16
+  %vecext13487 = extractelement <4 x float> %tmp1195, i32 2
+  %add13488 = fadd float %vecext13487, 0x40647999A0000000
+  %tmp1196 = load <4 x float>, <4 x float>* undef, align 16
+  %vecins13489 = insertelement <4 x float> %tmp1196, float %add13488, i32 2
+  store <4 x float> %vecins13489, <4 x float>* undef, align 16
+  %tmp1197 = load <4 x float>, <4 x float>* undef, align 16
+  %vecext13490 = extractelement <4 x float> %tmp1197, i32 3
+  %tmp1198 = load <4 x float>, <4 x float>* undef, align 16
+  %vecins13492 = insertelement <4 x float> %tmp1198, float undef, i32 3
+  store <4 x float> %vecins13492, <4 x float>* undef, align 16
+  %tmp1199 = load <4 x float>, <4 x float>* %.compoundliteral13493
+  %tmp1200 = load <4 x float>, <4 x float>* undef, align 16
+  store <4 x float> undef, <4 x float>* undef, align 16
+  %vecins13548 = insertelement <4 x float> undef, float undef, i32 3
+  store <4 x float> <float 4.540000e+02, float 3.760000e+02, float 0x406EA33340000000, float 0x405AACCCC0000000>, <4 x float>* %.compoundliteral13549
+  %tmp1201 = load <4 x float>, <4 x float>* undef, align 16
+  %add13552 = fadd float undef, 3.230000e+02
+  %tmp1202 = load <4 x float>, <4 x float>* undef, align 16
+  %vecins13553 = insertelement <4 x float> %tmp1202, float %add13552, i32 0
+  %tmp1203 = load <4 x float>, <4 x float>* undef, align 16
+  %vecext13554 = extractelement <4 x float> %tmp1203, i32 1
+  %tmp1204 = load <4 x float>, <4 x float>* undef, align 16
+  %vecins13556 = insertelement <4 x float> %tmp1204, float undef, i32 1
+  store <4 x float> %vecins13556, <4 x float>* undef, align 16
+  %tmp1205 = load <4 x float>, <4 x float>* undef, align 16
+  %add13558 = fadd float undef, 2.625000e+02
+  %tmp1206 = load <4 x float>, <4 x float>* undef, align 16
+  %vecins13559 = insertelement <4 x float> %tmp1206, float %add13558, i32 2
+  store <4 x float> %vecins13559, <4 x float>* undef, align 16
+  %add13575 = fadd float undef, -4.725000e+02
+  %tmp1207 = load <4 x float>, <4 x float>* undef, align 16
+  %vecins13576 = insertelement <4 x float> %tmp1207, float %add13575, i32 3
+  store <4 x float> %vecins13576, <4 x float>* undef, align 16
+  store <4 x float> <float 0x40334CCCC0000000, float 0xC0785CCCC0000000, float 0xC078D66660000000, float 3.745000e+02>, <4 x float>* undef
+  %tmp1208 = load <4 x float>, <4 x float>* undef
+  %tmp1209 = load <4 x float>, <4 x float>* undef, align 16
+  %add13578 = fadd <4 x float> %tmp1209, %tmp1208
+  store <4 x float> %add13578, <4 x float>* undef, align 16
+  %tmp1210 = load <4 x float>, <4 x float>* undef, align 16
+  %tmp1211 = load <4 x float>, <4 x float>* undef, align 16
+  %add13592 = fadd <4 x float> %tmp1211, undef
+  store <4 x float> %add13592, <4 x float>* undef, align 16
+  %tmp1212 = load <4 x float>, <4 x float>* undef, align 16
+  %vecext13593 = extractelement <4 x float> %tmp1212, i32 0
+  %add13594 = fadd float %vecext13593, 0xC0708B3340000000
+  %tmp1213 = load <4 x float>, <4 x float>* undef, align 16
+  store <4 x float> undef, <4 x float>* undef, align 16
+  %tmp1214 = load <4 x float>, <4 x float>* undef, align 16
+  %vecext13596 = extractelement <4 x float> %tmp1214, i32 1
+  %add13597 = fadd float %vecext13596, 0x40660999A0000000
+  %vecins13604 = insertelement <4 x float> undef, float undef, i32 3
+  store <4 x float> %vecins13604, <4 x float>* undef, align 16
+  store <4 x float> <float 0x407B4999A0000000, float 0xC067F66660000000, float 0xC068F999A0000000, float 0xC079233340000000>, <4 x float>* undef
+  %tmp1215 = load <4 x float>, <4 x float>* undef, align 16
+  %add13606 = fadd <4 x float> %tmp1215, undef
+  %tmp1216 = load <4 x float>, <4 x float>* undef, align 16
+  %vecext13607 = extractelement <4 x float> %tmp1216, i32 0
+  %vecins13609 = insertelement <4 x float> undef, float undef, i32 0
+  %tmp1217 = load <4 x float>, <4 x float>* undef, align 16
+  store <4 x float> undef, <4 x float>* undef, align 16
+  %tmp1218 = load <4 x float>, <4 x float>* undef, align 16
+  %add13622 = fadd float undef, -3.390000e+02
+  %vecins13623 = insertelement <4 x float> undef, float %add13622, i32 0
+  store <4 x float> %vecins13623, <4 x float>* undef, align 16
+  %tmp1219 = load <4 x float>, <4 x float>* undef, align 16
+  %vecext13624 = extractelement <4 x float> %tmp1219, i32 1
+  %add13625 = fadd float %vecext13624, 0x405C3999A0000000
+  %vecext13627 = extractelement <4 x float> undef, i32 2
+  %add13628 = fadd float %vecext13627, 0xC067033340000000
+  %tmp1220 = load <4 x float>, <4 x float>* undef, align 16
+  %tmp1221 = load <4 x float>, <4 x float>* undef, align 16
+  %vecext13630 = extractelement <4 x float> %tmp1221, i32 3
+  %add13631 = fadd float %vecext13630, 0xC060333340000000
+  %tmp1222 = load <4 x float>, <4 x float>* undef, align 16
+  %vecins13632 = insertelement <4 x float> %tmp1222, float %add13631, i32 3
+  store <4 x float> %vecins13632, <4 x float>* undef, align 16
+  store <4 x float> <float 0x4078D66660000000, float 0x4048B33340000000, float 0x4051466660000000, float -2.965000e+02>, <4 x float>* undef
+  %tmp1223 = load <4 x float>, <4 x float>* undef
+  %tmp1224 = load <4 x float>, <4 x float>* undef, align 16
+  %add13634 = fadd <4 x float> %tmp1224, %tmp1223
+  store <4 x float> %add13634, <4 x float>* undef, align 16
+  %vecext13635 = extractelement <4 x float> undef, i32 0
+  %add13636 = fadd float %vecext13635, 0x406A5999A0000000
+  %tmp1225 = load <4 x float>, <4 x float>* undef, align 16
+  %vecins13637 = insertelement <4 x float> %tmp1225, float %add13636, i32 0
+  store <4 x float> %vecins13637, <4 x float>* undef, align 16
+  %tmp1226 = load <4 x float>, <4 x float>* undef, align 16
+  %tmp1227 = load <4 x float>, <4 x float>* undef, align 16
+  %vecins13643 = insertelement <4 x float> %tmp1227, float undef, i32 2
+  store <4 x float> undef, <4 x float>* undef, align 16
+  %tmp1228 = load <4 x float>, <4 x float>* undef, align 16
+  %add13785 = fadd float undef, 0x4068866660000000
+  %tmp1229 = load <4 x float>, <4 x float>* undef, align 16
+  %vecins13786 = insertelement <4 x float> %tmp1229, float %add13785, i32 3
+  store <4 x float> %vecins13786, <4 x float>* undef, align 16
+  store <4 x float> <float 0x407704CCC0000000, float 0x4047B33340000000, float 0x40797B3340000000, float 0xC0652CCCC0000000>, <4 x float>* %.compoundliteral13787
+  %tmp1230 = load <4 x float>, <4 x float>* undef, align 16
+  %add13788 = fadd <4 x float> %tmp1230, undef
+  %tmp1231 = load <4 x float>, <4 x float>* undef
+  %tmp1232 = load <4 x float>, <4 x float>* undef, align 16
+  %add13802 = fadd <4 x float> %tmp1232, %tmp1231
+  store <4 x float> %add13802, <4 x float>* undef, align 16
+  %tmp1233 = load <4 x float>, <4 x float>* undef, align 16
+  %vecext13803 = extractelement <4 x float> %tmp1233, i32 0
+  %add13804 = fadd float %vecext13803, -2.900000e+01
+  %tmp1234 = load <4 x float>, <4 x float>* undef, align 16
+  %vecins13805 = insertelement <4 x float> %tmp1234, float %add13804, i32 0
+  store <4 x float> %vecins13805, <4 x float>* undef, align 16
+  %tmp1235 = load <4 x float>, <4 x float>* undef, align 16
+  %add13807 = fadd float undef, 6.400000e+01
+  %tmp1236 = load <4 x float>, <4 x float>* undef, align 16
+  %tmp1237 = load <4 x float>, <4 x float>* undef, align 16
+  %vecext13809 = extractelement <4 x float> %tmp1237, i32 2
+  %tmp1238 = load <4 x float>, <4 x float>* undef, align 16
+  %vecext13812 = extractelement <4 x float> %tmp1238, i32 3
+  %add13813 = fadd float %vecext13812, -3.615000e+02
+  %vecins13814 = insertelement <4 x float> undef, float %add13813, i32 3
+  store <4 x float> %vecins13814, <4 x float>* undef, align 16
+  store <4 x float> <float -2.270000e+02, float -1.500000e+01, float 0x407084CCC0000000, float -1.425000e+02>, <4 x float>* undef
+  %tmp1239 = load <4 x float>, <4 x float>* undef
+  store <4 x float> undef, <4 x float>* undef, align 16
+  %tmp1240 = load <4 x float>, <4 x float>* undef, align 16
+  %vecext13817 = extractelement <4 x float> %tmp1240, i32 0
+  %vecins13856 = insertelement <4 x float> undef, float undef, i32 3
+  store <4 x float> %vecins13856, <4 x float>* undef, align 16
+  store <4 x float> <float 0x40656CCCC0000000, float 0xC0656999A0000000, float 0x40778E6660000000, float 0x407ECE6660000000>, <4 x float>* undef
+  %tmp1241 = load <4 x float>, <4 x float>* undef
+  %tmp1242 = load <4 x float>, <4 x float>* undef, align 16
+  store <4 x float> undef, <4 x float>* undef, align 16
+  %tmp1243 = load <4 x float>, <4 x float>* undef, align 16
+  %vecext13859 = extractelement <4 x float> %tmp1243, i32 0
+  %tmp1244 = load <4 x float>, <4 x float>* undef, align 16
+  %vecins13861 = insertelement <4 x float> %tmp1244, float undef, i32 0
+  %tmp1245 = load <4 x float>, <4 x float>* undef, align 16
+  %vecext13862 = extractelement <4 x float> %tmp1245, i32 1
+  %add13863 = fadd float %vecext13862, -1.380000e+02
+  %vecins13864 = insertelement <4 x float> undef, float %add13863, i32 1
+  %vecins13867 = insertelement <4 x float> undef, float undef, i32 2
+  store <4 x float> %vecins13867, <4 x float>* undef, align 16
+  %tmp1246 = load <4 x float>, <4 x float>* undef, align 16
+  %tmp1247 = load <4 x float>, <4 x float>* undef, align 16
+  ret <4 x float> undef
+}
+
+declare i32 @printf(i8*, ...)
diff --git a/test/CodeGen/ARM/thumb1_return_sequence.ll b/test/CodeGen/ARM/thumb1_return_sequence.ll
index 9c62faeaa68..5b9c19ab5eb 100644
--- a/test/CodeGen/ARM/thumb1_return_sequence.ll
+++ b/test/CodeGen/ARM/thumb1_return_sequence.ll
@@ -23,11 +23,9 @@ entry:
 ; --------
 ; CHECK-V4T:         add sp,
 ; CHECK-V4T-NEXT:    pop {[[SAVED]]}
-; CHECK-V4T-NEXT:    mov r12, r3
-; CHECK-V4T-NEXT:    pop {r3}
-; CHECK-V4T-NEXT:    mov lr, r3
-; CHECK-V4T-NEXT:    mov r3, r12
-; CHECK-V4T:         bx  lr
+; We do not have any SP update to insert so we can just optimize
+; the pop sequence.
+; CHECK-V4T-NEXT:    pop {pc}
 ; CHECK-V5T:         pop {[[SAVED]], pc}
 }
 
@@ -53,19 +51,19 @@ entry:
 ; Epilogue
 ; --------
 ; CHECK-V4T:         pop {[[SAVED]]}
-; CHECK-V4T-NEXT:    mov r12, r3
-; CHECK-V4T-NEXT:    pop {r3}
+; CHECK-V4T-NEXT:    mov r12, [[POP_REG:r[0-7]]]
+; CHECK-V4T-NEXT:    pop {[[POP_REG]]}
 ; CHECK-V4T-NEXT:    add sp,
-; CHECK-V4T-NEXT:    mov lr, r3
-; CHECK-V4T-NEXT:    mov r3, r12
+; CHECK-V4T-NEXT:    mov lr, [[POP_REG]]
+; CHECK-V4T-NEXT:    mov [[POP_REG]], r12
 ; CHECK-V4T:         bx  lr
 ; CHECK-V5T:         add sp,
 ; CHECK-V5T-NEXT:    pop {[[SAVED]]}
-; CHECK-V5T-NEXT:    mov r12, r3
-; CHECK-V5T-NEXT:    pop {r3}
+; CHECK-V5T-NEXT:    mov r12, [[POP_REG:r[0-7]]]
+; CHECK-V5T-NEXT:    pop {[[POP_REG]]}
 ; CHECK-V5T-NEXT:    add sp,
-; CHECK-V5T-NEXT:    mov lr, r3
-; CHECK-V5T-NEXT:    mov r3, r12
+; CHECK-V5T-NEXT:    mov lr, [[POP_REG]]
+; CHECK-V5T-NEXT:    mov [[POP_REG]], r12
 ; CHECK-V5T-NEXT:    bx lr
 }
 
@@ -95,8 +93,7 @@ entry:
 ; Epilogue
 ; --------
 ; CHECK-V4T:    pop {[[SAVED]]}
-; CHECK-V4T:    pop {r3}
-; CHECK-V4T:    bx r3
+; CHECK-V4T:    pop {pc}
 ; CHECK-V5T:    pop {[[SAVED]], pc}
 }
 
@@ -148,14 +145,18 @@ entry:
 ; --------
 ; CHECK-V4T:         add sp,
 ; CHECK-V4T-NEXT:    pop {[[SAVED]]}
-; CHECK-V4T-NEXT:    pop {r3}
+; Only r1 to r3 are available to pop LR.
+; r0 is used for the return value.
+; CHECK-V4T-NEXT:    pop {[[POP_REG:r[1-3]]]}
 ; CHECK-V4T-NEXT:    add sp,
-; CHECK-V4T-NEXT:    bx r3
+; CHECK-V4T-NEXT:    bx [[POP_REG]]
 ; CHECK-V5T:         add sp,
 ; CHECK-V5T-NEXT:    pop {[[SAVED]]}
-; CHECK-V5T-NEXT:    pop {r3}
+; Only r1 to r3 are available to pop LR.
+; r0 is used for the return value.
+; CHECK-V5T-NEXT:    pop {[[POP_REG:r[1-3]]]}
 ; CHECK-V5T-NEXT:    add sp,
-; CHECK-V5T-NEXT:    bx r3
+; CHECK-V5T-NEXT:    bx [[POP_REG]]
 }
 
 ; CHECK-V4T-LABEL: noframe
@@ -191,13 +192,17 @@ entry:
 ; Epilogue
 ; --------
 ; CHECK-V4T:         pop {[[SAVED]]}
-; CHECK-V4T-NEXT:    pop {r3}
+; Only r1 to r3 are available to pop LR.
+; r0 is used for the return value.
+; CHECK-V4T-NEXT:    pop {[[POP_REG:r[1-3]]]}
 ; CHECK-V4T-NEXT:    add sp,
-; CHECK-V4T-NEXT:    bx r3
+; CHECK-V4T-NEXT:    bx [[POP_REG]]
 ; CHECK-V5T:         pop {[[SAVED]]}
-; CHECK-V5T-NEXT:    pop {r3}
+; Only r1 to r3 are available to pop LR.
+; r0 is used for the return value.
+; CHECK-V5T-NEXT:    pop {[[POP_REG:r[1-3]]]}
 ; CHECK-V5T-NEXT:    add sp,
-; CHECK-V5T-NEXT:    bx r3
+; CHECK-V5T-NEXT:    bx [[POP_REG]]
 }
 
 declare void @llvm.va_start(i8*) nounwind
diff --git a/test/CodeGen/ARM/thumb2-size-reduction-internal-flags.ll b/test/CodeGen/ARM/thumb2-size-reduction-internal-flags.ll
new file mode 100644
index 00000000000..578777f97c5
--- /dev/null
+++ b/test/CodeGen/ARM/thumb2-size-reduction-internal-flags.ll
@@ -0,0 +1,173 @@
+; RUN: llc %s -o - -verify-machineinstrs | FileCheck %s
+
+target datalayout = "e-m:o-p:32:32-f64:32:64-v64:32:64-v128:32:128-a:0:32-n32-S32"
+target triple = "thumbv7s-apple-ios8.0.0"
+
+%struct.cells = type { i32, i32, %struct.cells* }
+
+@reg_len = external global i32, align 4
+
+; The thumb2 size reduction pass commutes arguments to make the first src of an add the same as the dest.
+; It needs to also move the internal flag when commuting arguments.
+
+; CHECK-LABEL: @simulate
+
+; Function Attrs: nounwind optsize ssp
+define i32 @simulate(i32 %iterations, %struct.cells* nocapture %present, double %prob, i8* nocapture readonly %structure) {
+entry:
+  %0 = load i32, i32* @reg_len, align 4, !tbaa !3
+  %sub = add nsw i32 %0, -1
+  %div = sdiv i32 %sub, 31
+  %rem2 = srem i32 %sub, 31
+  %cmp35202 = icmp sgt i32 %rem2, 0
+  br label %for.cond3.preheader
+
+for.cond3.preheader:                              ; preds = %if.end85, %entry
+  %call192 = tail call i32 @lrand48() #2
+  br label %for.cond6.preheader
+
+for.cond34.preheader:                             ; preds = %for.inc30
+  br i1 %cmp35202, label %for.body37, label %for.end73
+
+for.cond6.preheader:                              ; preds = %for.inc30, %for.cond3.preheader
+  %call197 = phi i32 [ %call, %for.inc30 ], [ %call192, %for.cond3.preheader ]
+  %i.0196 = phi i32 [ %inc31, %for.inc30 ], [ 0, %for.cond3.preheader ]
+  %temp.1195 = phi %struct.cells* [ %5, %for.inc30 ], [ %present, %for.cond3.preheader ]
+  %savefaulty.0194 = phi i32 [ %add12, %for.inc30 ], [ 0, %for.cond3.preheader ]
+  %savef_free.0193 = phi i32 [ %add11, %for.inc30 ], [ 0, %for.cond3.preheader ]
+  br label %for.body8
+
+for.body8:                                        ; preds = %for.body8, %for.cond6.preheader
+  %randv.0190 = phi i32 [ %call197, %for.cond6.preheader ], [ %shr, %for.body8 ]
+  %j.0189 = phi i32 [ 0, %for.cond6.preheader ], [ %inc, %for.body8 ]
+  %temp.2188 = phi %struct.cells* [ %temp.1195, %for.cond6.preheader ], [ %5, %for.body8 ]
+  %savefaulty.1187 = phi i32 [ %savefaulty.0194, %for.cond6.preheader ], [ %add12, %for.body8 ]
+  %savef_free.1186 = phi i32 [ %savef_free.0193, %for.cond6.preheader ], [ %add11, %for.body8 ]
+  %f_free = getelementptr inbounds %struct.cells, %struct.cells* %temp.2188, i32 0, i32 0
+  %1 = load i32, i32* %f_free, align 4, !tbaa !7
+  %add11 = add nsw i32 %1, %savef_free.1186
+  %faulty = getelementptr inbounds %struct.cells, %struct.cells* %temp.2188, i32 0, i32 1
+  %2 = load i32, i32* %faulty, align 4, !tbaa !10
+  %add12 = add nsw i32 %2, %savefaulty.1187
+  %next = getelementptr inbounds %struct.cells, %struct.cells* %temp.2188, i32 0, i32 2
+  %3 = load %struct.cells*, %struct.cells** %next, align 4, !tbaa !11
+  %f_free13 = getelementptr inbounds %struct.cells, %struct.cells* %3, i32 0, i32 0
+  %4 = load i32, i32* %f_free13, align 4, !tbaa !7
+  %add14 = add nsw i32 %4, %randv.0190
+  %and = and i32 %add14, 1
+  store i32 %and, i32* %f_free, align 4, !tbaa !7
+  %call16 = tail call i32 @lrand48() #2
+  %rem17 = srem i32 %call16, 1000
+  %conv18 = sitofp i32 %rem17 to double
+  %div19 = fdiv double %conv18, 1.000000e+03
+  %cmp20 = fcmp olt double %div19, %prob
+  %xor = zext i1 %cmp20 to i32
+  %randv.1 = xor i32 %xor, %randv.0190
+  %5 = load %struct.cells*, %struct.cells** %next, align 4, !tbaa !11
+  %faulty25 = getelementptr inbounds %struct.cells, %struct.cells* %5, i32 0, i32 1
+  %6 = load i32, i32* %faulty25, align 4, !tbaa !10
+  %add26 = add nsw i32 %randv.1, %6
+  %and27 = and i32 %add26, 1
+  store i32 %and27, i32* %faulty, align 4, !tbaa !10
+  %shr = ashr i32 %randv.0190, 1
+  %inc = add nuw nsw i32 %j.0189, 1
+  %exitcond = icmp eq i32 %inc, 31
+  br i1 %exitcond, label %for.inc30, label %for.body8
+
+for.inc30:                                        ; preds = %for.body8
+  %inc31 = add nuw nsw i32 %i.0196, 1
+  %cmp4 = icmp slt i32 %inc31, %div
+  %call = tail call i32 @lrand48() #2
+  br i1 %cmp4, label %for.cond6.preheader, label %for.cond34.preheader
+
+for.body37:                                       ; preds = %for.body37, %for.cond34.preheader
+  %randv.2207 = phi i32 [ %shr70, %for.body37 ], [ %call, %for.cond34.preheader ]
+  %temp.3205 = phi %struct.cells* [ %9, %for.body37 ], [ %5, %for.cond34.preheader ]
+  %f_free45 = getelementptr inbounds %struct.cells, %struct.cells* %temp.3205, i32 0, i32 0
+  %.pre220 = getelementptr inbounds %struct.cells, %struct.cells* %temp.3205, i32 0, i32 1
+  %next50 = getelementptr inbounds %struct.cells, %struct.cells* %temp.3205, i32 0, i32 2
+  %7 = load %struct.cells*, %struct.cells** %next50, align 4, !tbaa !11
+  %f_free51 = getelementptr inbounds %struct.cells, %struct.cells* %7, i32 0, i32 0
+  %8 = load i32, i32* %f_free51, align 4, !tbaa !7
+  %add52 = add nsw i32 %8, %randv.2207
+  %and53 = and i32 %add52, 1
+  store i32 %and53, i32* %f_free45, align 4, !tbaa !7
+  %call55 = tail call i32 @lrand48() #2
+  %rem56 = srem i32 %call55, 1000
+  %conv57 = sitofp i32 %rem56 to double
+  %div58 = fdiv double %conv57, 1.000000e+03
+  %cmp59 = fcmp olt double %div58, %prob
+  %xor62 = zext i1 %cmp59 to i32
+  %randv.3 = xor i32 %xor62, %randv.2207
+  %9 = load %struct.cells*, %struct.cells** %next50, align 4, !tbaa !11
+  %faulty65 = getelementptr inbounds %struct.cells, %struct.cells* %9, i32 0, i32 1
+  %10 = load i32, i32* %faulty65, align 4, !tbaa !10
+  %add66 = add nsw i32 %randv.3, %10
+  %and67 = and i32 %add66, 1
+  store i32 %and67, i32* %.pre220, align 4, !tbaa !10
+  %shr70 = ashr i32 %randv.2207, 1
+  br label %for.body37
+
+for.end73:                                        ; preds = %for.cond34.preheader
+  %call74 = tail call i32 @lrand48() #2
+  %11 = load i32, i32* @reg_len, align 4, !tbaa !3
+  %sub75 = add nsw i32 %11, -1
+  %arrayidx76 = getelementptr inbounds i8, i8* %structure, i32 %sub75
+  %12 = load i8, i8* %arrayidx76, align 1, !tbaa !12
+  %cmp78 = icmp eq i8 %12, 49
+  %f_free81 = getelementptr inbounds %struct.cells, %struct.cells* %5, i32 0, i32 0
+  br i1 %cmp78, label %if.then80, label %for.end73.if.end85_crit_edge
+
+for.end73.if.end85_crit_edge:                     ; preds = %for.end73
+  %.pre222 = getelementptr inbounds %struct.cells, %struct.cells* %5, i32 0, i32 1
+  br label %if.end85
+
+if.then80:                                        ; preds = %for.end73
+  %13 = load i32, i32* %f_free81, align 4, !tbaa !7
+  %add82 = add nsw i32 %13, %add11
+  %faulty83 = getelementptr inbounds %struct.cells, %struct.cells* %5, i32 0, i32 1
+  %14 = load i32, i32* %faulty83, align 4, !tbaa !10
+  %add84 = add nsw i32 %14, %add12
+  br label %if.end85
+
+if.end85:                                         ; preds = %if.then80, %for.end73.if.end85_crit_edge
+  %faulty100.pre-phi = phi i32* [ %.pre222, %for.end73.if.end85_crit_edge ], [ %faulty83, %if.then80 ]
+  %savef_free.5 = phi i32 [ %add11, %for.end73.if.end85_crit_edge ], [ %add82, %if.then80 ]
+  %savefaulty.5 = phi i32 [ %add12, %for.end73.if.end85_crit_edge ], [ %add84, %if.then80 ]
+  %add86 = add nsw i32 %savef_free.5, %call74
+  %and87 = and i32 %add86, 1
+  store i32 %and87, i32* %f_free81, align 4, !tbaa !7
+  %call89 = tail call i32 @lrand48() #2
+  %rem90 = srem i32 %call89, 10000
+  %conv91 = sitofp i32 %rem90 to double
+  %div92 = fdiv double %conv91, 1.000000e+04
+  %cmp93 = fcmp olt double %div92, %prob
+  %xor96 = zext i1 %cmp93 to i32
+  %randv.4 = xor i32 %xor96, %call74
+  %add98 = add nsw i32 %randv.4, %savefaulty.5
+  %and99 = and i32 %add98, 1
+  store i32 %and99, i32* %faulty100.pre-phi, align 4, !tbaa !10
+  br label %for.cond3.preheader
+}
+
+; Function Attrs: optsize
+declare i32 @lrand48()
+
+attributes #2 = { nounwind optsize }
+
+!llvm.module.flags = !{!0, !1}
+!llvm.ident = !{!2}
+
+!0 = !{i32 1, !"wchar_size", i32 4}
+!1 = !{i32 1, !"min_enum_size", i32 4}
+!2 = !{!"clang version 3.7.0 (trunk 236243)"}
+!3 = !{!4, !4, i64 0}
+!4 = !{!"int", !5, i64 0}
+!5 = !{!"omnipotent char", !6, i64 0}
+!6 = !{!"Simple C/C++ TBAA"}
+!7 = !{!8, !4, i64 0}
+!8 = !{!"cells", !4, i64 0, !4, i64 4, !9, i64 8}
+!9 = !{!"any pointer", !5, i64 0}
+!10 = !{!8, !4, i64 4}
+!11 = !{!8, !9, i64 8}
+!12 = !{!5, !5, i64 0}
diff --git a/test/CodeGen/ARM/tls-models.ll b/test/CodeGen/ARM/tls-models.ll
index 42c1ba91102..f3c58f74ebf 100644
--- a/test/CodeGen/ARM/tls-models.ll
+++ b/test/CodeGen/ARM/tls-models.ll
@@ -1,5 +1,11 @@
-; RUN: llc -march=arm -mtriple=arm-linux-gnueabi < %s | FileCheck -check-prefix=CHECK-NONPIC %s
-; RUN: llc -march=arm -mtriple=arm-linux-gnueabi -relocation-model=pic < %s | FileCheck -check-prefix=CHECK-PIC %s
+; RUN: llc -march=arm -mtriple=arm-linux-gnueabi < %s \
+; RUN:     | FileCheck -check-prefix=CHECK-NONPIC -check-prefix=COMMON %s
+; RUN: llc -march=arm -mtriple=arm-linux-gnueabi -relocation-model=pic < %s \
+; RUN:     | FileCheck -check-prefix=CHECK-PIC  -check-prefix=COMMON %s
+; RUN: llc -emulated-tls -march=arm -mtriple=arm-linux-gnueabi < %s \
+; RUN:     | FileCheck -check-prefix=EMUNONPIC -check-prefix=EMU -check-prefix=COMMON %s
+; RUN: llc -emulated-tls -march=arm -mtriple=arm-linux-gnueabi -relocation-model=pic < %s \
+; RUN:     | FileCheck -check-prefix=EMUPIC -check-prefix=EMU -check-prefix=COMMON %s
 
 
 @external_gd = external thread_local global i32
@@ -20,23 +26,23 @@ define i32* @f1() {
 entry:
   ret i32* @external_gd
 
+  ; COMMON-LABEL:   f1:
   ; Non-PIC code can use initial-exec, PIC code has to use general dynamic.
-  ; CHECK-NONPIC-LABEL:   f1:
   ; CHECK-NONPIC:   external_gd(GOTTPOFF)
-  ; CHECK-PIC-LABEL:      f1:
   ; CHECK-PIC:      external_gd(TLSGD)
+  ; EMU:            __emutls_get_address
 }
 
 define i32* @f2() {
 entry:
   ret i32* @internal_gd
 
+  ; COMMON-LABEL:   f2:
   ; Non-PIC code can use local exec, PIC code can use local dynamic,
   ; but that is not implemented, so falls back to general dynamic.
-  ; CHECK-NONPIC-LABEL:   f2:
   ; CHECK-NONPIC:   internal_gd(TPOFF)
-  ; CHECK-PIC-LABEL:      f2:
   ; CHECK-PIC:      internal_gd(TLSGD)
+  ; EMU:            __emutls_get_address
 }
 
 
@@ -46,24 +52,24 @@ define i32* @f3() {
 entry:
   ret i32* @external_ld
 
+  ; COMMON-LABEL:   f3:
   ; Non-PIC code can use initial exec, PIC should use local dynamic,
   ; but that is not implemented, so falls back to general dynamic.
-  ; CHECK-NONPIC-LABEL:   f3:
   ; CHECK-NONPIC:   external_ld(GOTTPOFF)
-  ; CHECK-PIC-LABEL:      f3:
   ; CHECK-PIC:      external_ld(TLSGD)
+  ; EMU:            __emutls_get_address
 }
 
 define i32* @f4() {
 entry:
   ret i32* @internal_ld
 
+  ; COMMON-LABEL:   f4:
   ; Non-PIC code can use local exec, PIC code can use local dynamic,
   ; but that is not implemented, so it falls back to general dynamic.
-  ; CHECK-NONPIC-LABEL:   f4:
   ; CHECK-NONPIC:   internal_ld(TPOFF)
-  ; CHECK-PIC-LABEL:      f4:
   ; CHECK-PIC:      internal_ld(TLSGD)
+  ; EMU:            __emutls_get_address
 }
 
 
@@ -73,22 +79,22 @@ define i32* @f5() {
 entry:
   ret i32* @external_ie
 
+  ; COMMON-LABEL:   f5:
   ; Non-PIC and PIC code will use initial exec as specified.
-  ; CHECK-NONPIC-LABEL:   f5:
   ; CHECK-NONPIC:   external_ie(GOTTPOFF)
-  ; CHECK-PIC-LABEL:      f5:
   ; CHECK-PIC:      external_ie(GOTTPOFF)
+  ; EMU:            __emutls_get_address
 }
 
 define i32* @f6() {
 entry:
   ret i32* @internal_ie
 
+  ; COMMON-LABEL:   f6:
   ; Non-PIC code can use local exec, PIC code use initial exec as specified.
-  ; CHECK-NONPIC-LABEL:   f6:
   ; CHECK-NONPIC:   internal_ie(TPOFF)
-  ; CHECK-PIC-LABEL:      f6:
   ; CHECK-PIC:      internal_ie(GOTTPOFF)
+  ; EMU:            __emutls_get_address
 }
 
 
@@ -98,20 +104,52 @@ define i32* @f7() {
 entry:
   ret i32* @external_le
 
+  ; COMMON-LABEL:   f7:
   ; Non-PIC and PIC code will use local exec as specified.
-  ; CHECK-NONPIC-LABEL:   f7:
   ; CHECK-NONPIC:   external_le(TPOFF)
-  ; CHECK-PIC-LABEL:      f7:
   ; CHECK-PIC:      external_le(TPOFF)
+  ; EMU:            __emutls_get_address
 }
 
 define i32* @f8() {
 entry:
   ret i32* @internal_le
 
+  ; COMMON-LABEL:   f8:
   ; Non-PIC and PIC code will use local exec as specified.
-  ; CHECK-NONPIC-LABEL:   f8:
   ; CHECK-NONPIC:   internal_le(TPOFF)
-  ; CHECK-PIC-LABEL:      f8:
   ; CHECK-PIC:      internal_le(TPOFF)
+  ; EMU:            __emutls_get_address
 }
+
+
+; ----- emulated specified -----
+
+; External declaration has no initializer.
+; Internal definition has initializer.
+
+; EMU-NOT:   __emutls_t.external_gd
+; EMU-NOT:   __emutls_v.external_gd
+; EMU:       .align 2
+; EMU-LABEL: __emutls_v.internal_gd:
+; EMU-NEXT:  .long 4
+; EMU-NEXT:  .long 4
+; EMU-NEXT:  .long 0
+; EMU-NEXT:  .long __emutls_t.internal_gd
+; EMU-LABEL: __emutls_t.internal_gd:
+; EMU-NEXT:  .long 42
+; EMU-NOT:   __emutls_t.external_gd
+
+; __emutls_t and __emutls_v are the same for PIC and non-PIC modes.
+
+; EMU-NOT:   __emutls_t.external_gd
+; EMU-NOT:   __emutls_v.external_gd
+; EMU:       .align 2
+; EMU-LABEL: __emutls_v.internal_le:
+; EMU-NEXT:  .long 4
+; EMU-NEXT:  .long 4
+; EMU-NEXT:  .long 0
+; EMU-NEXT:  .long __emutls_t.internal_le
+; EMU-LABEL: __emutls_t.internal_le:
+; EMU-NEXT:  .long 42
+; EMU-NOT:   __emutls_t.external_le
diff --git a/test/CodeGen/ARM/tls3.ll b/test/CodeGen/ARM/tls3.ll
index 7e17b13a3c9..94cadeedd93 100644
--- a/test/CodeGen/ARM/tls3.ll
+++ b/test/CodeGen/ARM/tls3.ll
@@ -1,11 +1,34 @@
 ; RUN: llc < %s -march=arm -mtriple=arm-linux-gnueabi | \
 ; RUN:     grep "tbss"
+; RUN: llc < %s -march=arm -mtriple=arm-linux-gnueabi | \
+; RUN:     FileCheck %s -check-prefix=CHECK -check-prefix=NOEMU
+; RUN: llc < %s -emulated-tls -march=arm -mtriple=arm-linux-gnueabi | \
+; RUN:     FileCheck %s -check-prefix=CHECK -check-prefix=EMU
 
 %struct.anon = type { i32, i32 }
-@teste = internal thread_local global %struct.anon zeroinitializer		; <%struct.anon*> [#uses=1]
+@teste = internal thread_local global %struct.anon zeroinitializer ; <%struct.anon*> [#uses=1]
 
 define i32 @main() {
 entry:
-	%tmp2 = load i32, i32* getelementptr (%struct.anon, %struct.anon* @teste, i32 0, i32 0), align 8		; <i32> [#uses=1]
-	ret i32 %tmp2
+  %tmp2 = load i32, i32* getelementptr (%struct.anon, %struct.anon* @teste, i32 0, i32 0), align 8 ; <i32> [#uses=1]
+  ret i32 %tmp2
 }
+
+; CHECK-LABEL: main:
+; NOEMU-NOT:   __emutls_get_address
+
+; NOEMU:       .section .tbss
+; NOEMU-LABEL: teste:
+; NOEMU-NEXT:  .zero 8
+
+; CHECK-NOT: __emutls_t.teste
+
+; EMU:       .align 2
+; EMU-LABEL: __emutls_v.teste:
+; EMU-NEXT:  .long 8
+; EMU-NEXT:  .long 4
+; EMU-NEXT:  .long 0
+; EMU-NEXT:  .long 0
+
+; CHECK-NOT: teste:
+; CHECK-NOT: __emutls_t.teste
diff --git a/test/CodeGen/ARM/unaligned_load_store.ll b/test/CodeGen/ARM/unaligned_load_store.ll
index 0be3917ffa2..4e16bda6c4d 100644
--- a/test/CodeGen/ARM/unaligned_load_store.ll
+++ b/test/CodeGen/ARM/unaligned_load_store.ll
@@ -1,7 +1,7 @@
-; RUN: llc -mtriple=arm-eabi -pre-RA-sched=source %s -o - \
+; RUN: llc -mtriple=arm-eabi -pre-RA-sched=source -mattr=+strict-align %s -o - \
 ; RUN:	| FileCheck %s -check-prefix=EXPANDED
 
-; RUN: llc -mtriple=armv6-apple-darwin -mcpu=cortex-a8 -mattr=-neon -arm-strict-align -pre-RA-sched=source %s -o - \
+; RUN: llc -mtriple=armv6-apple-darwin -mcpu=cortex-a8 -mattr=-neon,+strict-align -pre-RA-sched=source %s -o - \
 ; RUN:	| FileCheck %s -check-prefix=EXPANDED
 
 ; RUN: llc -mtriple=armv6-apple-darwin -mcpu=cortex-a8 %s -o - \
diff --git a/test/CodeGen/ARM/varargs-spill-stack-align-nacl.ll b/test/CodeGen/ARM/varargs-spill-stack-align-nacl.ll
index c3d1bb243e7..4879d73894d 100644
--- a/test/CodeGen/ARM/varargs-spill-stack-align-nacl.ll
+++ b/test/CodeGen/ARM/varargs-spill-stack-align-nacl.ll
@@ -1,6 +1,4 @@
-; RUN: llc < %s -mtriple=armv7-nacl-gnueabi | FileCheck %s
-; @LOCALMOD: change to armv7, (or alternatively +v6t2) since NaCl needs
-; movw/movt to materialize the @va_list global address.
+; RUN: llc < %s -mtriple=arm-nacl-gnueabi | FileCheck %s
 
 declare void @llvm.va_start(i8*)
 declare void @external_func(i8*)
@@ -29,8 +27,5 @@ define void @varargs_func(i32 %arg1, ...) {
 ; CHECK: sub sp, sp, #12
 ; Calculate the address of the varargs save area and save varargs
 ; arguments into it.
-; @LOCALMOD: Adjust test expectation, removing NEXT to allow CFI
-; directive to intervene in the output.
-; CHECK: add r0, sp, #20
-; @LOCALMOD: Adjust test expectation, removing NEXT to allow sandboxing.
-; CHECK: stm r0, {r1, r2, r3}
+; CHECK-NEXT: add r0, sp, #20
+; CHECK-NEXT: stm r0, {r1, r2, r3}
diff --git a/test/CodeGen/ARM/vector-load.ll b/test/CodeGen/ARM/vector-load.ll
index 17f134f458a..a638c2bdb9b 100644
--- a/test/CodeGen/ARM/vector-load.ll
+++ b/test/CodeGen/ARM/vector-load.ll
@@ -238,12 +238,12 @@ define <4 x i32> @zextload_v8i8tov8i32(<4 x i8>** %ptr) {
 
 define <4 x i32> @zextload_v8i8tov8i32_fake_update(<4 x i8>** %ptr) {
 ;CHECK-LABEL: zextload_v8i8tov8i32_fake_update:
-;CHECK: ldr.w   r[[PTRREG:[0-9]+]], [r0]
+;CHECK: ldr   r[[PTRREG:[0-9]+]], [r0]
 ;CHECK: vld1.32 {{{d[0-9]+}}[0]}, [r[[PTRREG]]:32]
 ;CHECK: add.w   r[[INCREG:[0-9]+]], r[[PTRREG]], #16
-;CHECK: str.w   r[[INCREG]], [r0]
 ;CHECK: vmovl.u8        {{q[0-9]+}}, {{d[0-9]+}}
 ;CHECK: vmovl.u16       {{q[0-9]+}}, {{d[0-9]+}}
+;CHECK: str   r[[INCREG]], [r0]
 	%A = load <4 x i8>*, <4 x i8>** %ptr
 	%lA = load <4 x i8>, <4 x i8>* %A, align 4
 	%inc = getelementptr <4 x i8>, <4 x i8>* %A, i38 4
diff --git a/test/CodeGen/ARM/vector-store.ll b/test/CodeGen/ARM/vector-store.ll
index 30baa9a20dd..161bbf1d0fd 100644
--- a/test/CodeGen/ARM/vector-store.ll
+++ b/test/CodeGen/ARM/vector-store.ll
@@ -228,9 +228,9 @@ define void @truncstore_v4i32tov4i8(<4 x i8>** %ptr, <4 x i32> %val) {
 ;CHECK: ldr.w   r9, [sp]
 ;CHECK: vmov    {{d[0-9]+}}, r3, r9
 ;CHECK: vmov    {{d[0-9]+}}, r1, r2
+;CHECK: ldr     r[[PTRREG:[0-9]+]], [r0]
 ;CHECK: vmovn.i32       [[VECLO:d[0-9]+]], {{q[0-9]+}}
 ;CHECK: vuzp.8  [[VECLO]], {{d[0-9]+}}
-;CHECK: ldr     r[[PTRREG:[0-9]+]], [r0]
 ;CHECK: vst1.32 {[[VECLO]][0]}, [r[[PTRREG]]:32]
 	%A = load <4 x i8>*, <4 x i8>** %ptr
         %trunc = trunc <4 x i32> %val to <4 x i8>
@@ -243,10 +243,10 @@ define void @truncstore_v4i32tov4i8_fake_update(<4 x i8>** %ptr, <4 x i32> %val)
 ;CHECK: ldr.w   r9, [sp]
 ;CHECK: vmov    {{d[0-9]+}}, r3, r9
 ;CHECK: vmov    {{d[0-9]+}}, r1, r2
-;CHECK: movs    [[IMM16:r[0-9]+]], #16
+;CHECK: ldr     r[[PTRREG:[0-9]+]], [r0]
 ;CHECK: vmovn.i32       [[VECLO:d[0-9]+]], {{q[0-9]+}}
 ;CHECK: vuzp.8  [[VECLO]], {{d[0-9]+}}
-;CHECK: ldr     r[[PTRREG:[0-9]+]], [r0]
+;CHECK: movs    [[IMM16:r[0-9]+]], #16
 ;CHECK: vst1.32 {[[VECLO]][0]}, [r[[PTRREG]]:32], [[IMM16]]
 ;CHECK: str     r[[PTRREG]], [r0]
 	%A = load <4 x i8>*, <4 x i8>** %ptr
diff --git a/test/CodeGen/ARM/vext.ll b/test/CodeGen/ARM/vext.ll
index 72ecf0ef062..61342022ad5 100644
--- a/test/CodeGen/ARM/vext.ll
+++ b/test/CodeGen/ARM/vext.ll
@@ -196,3 +196,35 @@ define arm_aapcscc void @test_elem_mismatch(<2 x i64>* nocapture %src, <4 x i16>
   store <4 x i16> %tmp7, <4 x i16>* %dest, align 4
   ret void
 }
+
+define <4 x i32> @test_reverse_and_extract(<2 x i32>* %A) {
+entry:
+  ; CHECK-LABEL: test_reverse_and_extract
+  ; CHECK-NOT: vtrn
+  ; CHECK: vrev
+  ; CHECK: vext
+	%tmp1 = load <2 x i32>, <2 x i32>* %A
+  %0 = shufflevector <2 x i32> %tmp1, <2 x i32> undef, <4 x i32> <i32 undef, i32 undef, i32 1, i32 0>
+  ret <4 x i32> %0
+}
+
+define <4 x i32> @test_dup_and_extract(<2 x i32>* %A) {
+entry:
+  ; CHECK-LABEL: test_dup_and_extract
+  ; CHECK-NOT: vtrn
+  ; CHECK: vdup
+  ; CHECK: vext
+	%tmp1 = load <2 x i32>, <2 x i32>* %A
+  %0 = shufflevector <2 x i32> %tmp1, <2 x i32> undef, <4 x i32> <i32 0, i32 0, i32 0, i32 1>
+  ret <4 x i32> %0
+}
+
+define <4 x i32> @test_zip_and_extract(<2 x i32>* %A) {
+entry:
+  ; CHECK-LABEL: test_zip_and_extract
+  ; CHECK: vzip
+  ; CHECK: vext
+	%tmp1 = load <2 x i32>, <2 x i32>* %A
+  %0 = shufflevector <2 x i32> %tmp1, <2 x i32> undef, <4 x i32> <i32 1, i32 1, i32 0, i32 1>
+  ret <4 x i32> %0
+}
diff --git a/test/CodeGen/ARM/vfp-libcalls.ll b/test/CodeGen/ARM/vfp-libcalls.ll
index 9d4e194e90e..b08073ab62b 100644
--- a/test/CodeGen/ARM/vfp-libcalls.ll
+++ b/test/CodeGen/ARM/vfp-libcalls.ll
@@ -1,6 +1,6 @@
 ; RUN: llc -mtriple=armv6-apple-ios -mcpu=arm1136jf-s -o - %s | FileCheck %s --check-prefix=CHECK-HARD
 ; RUN: llc -mtriple=thumbv6-apple-ios -mcpu=arm1136jf-s -o - %s | FileCheck %s --check-prefix=CHECK-SOFTISH
-; RUN: llc -mtriple=armv7s-apple-ios -soft-float -mcpu=arm1136jf-s -o - %s | FileCheck %s --check-prefix=CHECK-SOFT
+; RUN: llc -mtriple=armv7s-apple-ios -mattr=+soft-float -mcpu=arm1136jf-s -o - %s | FileCheck %s --check-prefix=CHECK-SOFT
 
 define float @test_call(float %a, float %b) {
 ; CHECK-HARD: vadd.f32 {{s[0-9]+}}, {{s[0-9]+}}, {{s[0-9]+}}
@@ -8,4 +8,4 @@ define float @test_call(float %a, float %b) {
 ; CHECK-SOFT: bl ___addsf3{{$}}
   %sum = fadd float %a, %b
   ret float %sum
-}
\ No newline at end of file
+}
diff --git a/test/CodeGen/ARM/vfp-regs-dwarf.ll b/test/CodeGen/ARM/vfp-regs-dwarf.ll
index 70c1e20c5ef..eca0c26e556 100644
--- a/test/CodeGen/ARM/vfp-regs-dwarf.ll
+++ b/test/CodeGen/ARM/vfp-regs-dwarf.ll
@@ -31,13 +31,13 @@ define void @stack_offsets() {
 !llvm.dbg.cu = !{!0}
 !llvm.module.flags = !{!8, !9}
 
-!0 = !MDCompileUnit(language: DW_LANG_C99, producer: "clang version 3.5.0 ", isOptimized: false, emissionKind: 1, file: !1, enums: !2, retainedTypes: !2, subprograms: !3, globals: !2, imports: !2)
-!1 = !MDFile(filename: "tmp.c", directory: "/Users/tim/llvm/build")
+!0 = !DICompileUnit(language: DW_LANG_C99, producer: "clang version 3.5.0 ", isOptimized: false, emissionKind: 1, file: !1, enums: !2, retainedTypes: !2, subprograms: !3, globals: !2, imports: !2)
+!1 = !DIFile(filename: "tmp.c", directory: "/Users/tim/llvm/build")
 !2 = !{}
 !3 = !{!4}
-!4 = !MDSubprogram(name: "bar", line: 1, isLocal: false, isDefinition: true, virtualIndex: 6, isOptimized: false, scopeLine: 1, file: !1, scope: !5, type: !6, function: void ()* @stack_offsets, variables: !2)
-!5 = !MDFile(filename: "tmp.c", directory: "/Users/tim/llvm/build")
-!6 = !MDSubroutineType(types: !7)
+!4 = !DISubprogram(name: "bar", line: 1, isLocal: false, isDefinition: true, virtualIndex: 6, isOptimized: false, scopeLine: 1, file: !1, scope: !5, type: !6, function: void ()* @stack_offsets, variables: !2)
+!5 = !DIFile(filename: "tmp.c", directory: "/Users/tim/llvm/build")
+!6 = !DISubroutineType(types: !7)
 !7 = !{null}
 !8 = !{i32 2, !"Dwarf Version", i32 4}
 !9 = !{i32 1, !"Debug Info Version", i32 3}
diff --git a/test/CodeGen/ARM/vminmaxnm.ll b/test/CodeGen/ARM/vminmaxnm.ll
index 2e2648d11f7..3632ffd0021 100644
--- a/test/CodeGen/ARM/vminmaxnm.ll
+++ b/test/CodeGen/ARM/vminmaxnm.ll
@@ -218,6 +218,291 @@ define double @fp-armv8_vmaxnm_ule_rev(double %a, double %b) {
   ret double %cond
 }
 
+; known non-NaNs
+
+define float @fp-armv8_vminnm_NNNo(float %a) {
+; CHECK-FAST-LABEL: "fp-armv8_vminnm_NNNo":
+; CHECK-FAST: vminnm.f32
+; CHECK-FAST: vminnm.f32
+; CHECK-LABEL: "fp-armv8_vminnm_NNNo":
+; CHECK: vminnm.f32
+; CHECK-NOT: vminnm.f32
+  %cmp1 = fcmp olt float %a, 12.
+  %cond1 = select i1 %cmp1, float %a, float 12.
+  %cmp2 = fcmp olt float 34., %cond1
+  %cond2 = select i1 %cmp2, float 34., float %cond1
+  ret float %cond2
+}
+
+define double @fp-armv8_vminnm_NNNole(double %a) {
+; CHECK-FAST-LABEL: "fp-armv8_vminnm_NNNole":
+; CHECK-FAST: vminnm.f64
+; CHECK-FAST: vminnm.f64
+; CHECK-LABEL: "fp-armv8_vminnm_NNNole":
+; CHECK: vminnm.f64
+; CHECK-NOT: vminnm.f64
+  %cmp1 = fcmp ole double %a, 34.
+  %cond1 = select i1 %cmp1, double %a, double 34.
+  %cmp2 = fcmp ole double 56., %cond1
+  %cond2 = select i1 %cmp2, double 56., double %cond1
+  ret double %cond2
+}
+
+define float @fp-armv8_vminnm_NNNo_rev(float %a) {
+; CHECK-FAST-LABEL: "fp-armv8_vminnm_NNNo_rev":
+; CHECK-FAST: vminnm.f32
+; CHECK-FAST: vminnm.f32
+; CHECK-LABEL: "fp-armv8_vminnm_NNNo_rev":
+; CHECK: vminnm.f32
+; CHECK-NOT: vminnm.f32
+  %cmp1 = fcmp ogt float %a, 56.
+  %cond1 = select i1 %cmp1, float 56., float %a
+  %cmp2 = fcmp ogt float 78., %cond1
+  %cond2 = select i1 %cmp2, float %cond1, float 78.
+  ret float %cond2
+}
+
+define double @fp-armv8_vminnm_NNNoge_rev(double %a) {
+; CHECK-FAST-LABEL: "fp-armv8_vminnm_NNNoge_rev":
+; CHECK-FAST: vminnm.f64
+; CHECK-FAST: vminnm.f64
+; CHECK-LABEL: "fp-armv8_vminnm_NNNoge_rev":
+; CHECK: vminnm.f64
+; CHECK-NOT: vminnm.f64
+  %cmp1 = fcmp oge double %a, 78.
+  %cond1 = select i1 %cmp1, double 78., double %a
+  %cmp2 = fcmp oge double 90., %cond1
+  %cond2 = select i1 %cmp2, double %cond1, double 90.
+  ret double %cond2
+}
+
+define float @fp-armv8_vminnm_NNNu(float %b) {
+; CHECK-FAST-LABEL: "fp-armv8_vminnm_NNNu":
+; CHECK-FAST: vminnm.f32
+; CHECK-FAST: vminnm.f32
+; CHECK-LABEL: "fp-armv8_vminnm_NNNu":
+; CHECK: vminnm.f32
+; CHECK-NOT: vminnm.f32
+  %cmp1 = fcmp ult float 12., %b
+  %cond1 = select i1 %cmp1, float 12., float %b
+  %cmp2 = fcmp ult float %cond1, 34.
+  %cond2 = select i1 %cmp2, float %cond1, float 34.
+  ret float %cond2
+}
+
+define float @fp-armv8_vminnm_NNNule(float %b) {
+; CHECK-FAST-LABEL: "fp-armv8_vminnm_NNNule":
+; CHECK-FAST: vminnm.f32
+; CHECK-FAST: vminnm.f32
+; CHECK-LABEL: "fp-armv8_vminnm_NNNule":
+; CHECK: vminnm.f32
+; CHECK-NOT: vminnm.f32
+  %cmp1 = fcmp ule float 34., %b
+  %cond1 = select i1 %cmp1, float 34., float %b
+  %cmp2 = fcmp ule float %cond1, 56.
+  %cond2 = select i1 %cmp2, float %cond1, float 56.
+  ret float %cond2
+}
+
+define float @fp-armv8_vminnm_NNNu_rev(float %b) {
+; CHECK-FAST-LABEL: "fp-armv8_vminnm_NNNu_rev":
+; CHECK-FAST: vminnm.f32
+; CHECK-FAST: vminnm.f32
+; CHECK-LABEL: "fp-armv8_vminnm_NNNu_rev":
+; CHECK: vminnm.f32
+; CHECK-NOT: vminnm.f32
+  %cmp1 = fcmp ugt float 56., %b
+  %cond1 = select i1 %cmp1, float %b, float 56.
+  %cmp2 = fcmp ugt float %cond1, 78.
+  %cond2 = select i1 %cmp2, float 78., float %cond1
+  ret float %cond2
+}
+
+define double @fp-armv8_vminnm_NNNuge_rev(double %b) {
+; CHECK-FAST-LABEL: "fp-armv8_vminnm_NNNuge_rev":
+; CHECK-FAST: vminnm.f64
+; CHECK-FAST: vminnm.f64
+; CHECK-LABEL: "fp-armv8_vminnm_NNNuge_rev":
+; CHECK: vminnm.f64
+; CHECK-NOT: vminnm.f64
+  %cmp1 = fcmp uge double 78., %b
+  %cond1 = select i1 %cmp1, double %b, double 78.
+  %cmp2 = fcmp uge double %cond1, 90.
+  %cond2 = select i1 %cmp2, double 90., double %cond1
+  ret double %cond2
+}
+
+define float @fp-armv8_vmaxnm_NNNo(float %a) {
+; CHECK-FAST-LABEL: "fp-armv8_vmaxnm_NNNo":
+; CHECK-FAST: vmaxnm.f32
+; CHECK-FAST: vmaxnm.f32
+; CHECK-LABEL: "fp-armv8_vmaxnm_NNNo":
+; CHECK: vmaxnm.f32
+; CHECK-NOT: vmaxnm.f32
+  %cmp1 = fcmp ogt float %a, 12.
+  %cond1 = select i1 %cmp1, float %a, float 12.
+  %cmp2 = fcmp ogt float 34., %cond1
+  %cond2 = select i1 %cmp2, float 34., float %cond1
+  ret float %cond2
+}
+
+define float @fp-armv8_vmaxnm_NNNoge(float %a) {
+; CHECK-FAST-LABEL: "fp-armv8_vmaxnm_NNNoge":
+; CHECK-FAST: vmaxnm.f32
+; CHECK-FAST: vmaxnm.f32
+; CHECK-LABEL: "fp-armv8_vmaxnm_NNNoge":
+; CHECK: vmaxnm.f32
+; CHECK-NOT: vmaxnm.f32
+  %cmp1 = fcmp oge float %a, 34.
+  %cond1 = select i1 %cmp1, float %a, float 34.
+  %cmp2 = fcmp oge float 56., %cond1
+  %cond2 = select i1 %cmp2, float 56., float %cond1
+  ret float %cond2
+}
+
+define float @fp-armv8_vmaxnm_NNNo_rev(float %a) {
+; CHECK-FAST-LABEL: "fp-armv8_vmaxnm_NNNo_rev":
+; CHECK-FAST: vmaxnm.f32
+; CHECK-FAST: vmaxnm.f32
+; CHECK-LABEL: "fp-armv8_vmaxnm_NNNo_rev":
+; CHECK: vmaxnm.f32
+; CHECK-NOT: vmaxnm.f32
+  %cmp1 = fcmp olt float %a, 56.
+  %cond1 = select i1 %cmp1, float 56., float %a
+  %cmp2 = fcmp olt float 78., %cond1
+  %cond2 = select i1 %cmp2, float %cond1, float 78.
+  ret float %cond2
+}
+
+define float @fp-armv8_vmaxnm_NNNole_rev(float %a) {
+; CHECK-FAST-LABEL: "fp-armv8_vmaxnm_NNNole_rev":
+; CHECK-FAST: vmaxnm.f32
+; CHECK-FAST: vmaxnm.f32
+; CHECK-LABEL: "fp-armv8_vmaxnm_NNNole_rev":
+; CHECK: vmaxnm.f32
+; CHECK-NOT: vmaxnm.f32
+  %cmp1 = fcmp ole float %a, 78.
+  %cond1 = select i1 %cmp1, float 78., float %a
+  %cmp2 = fcmp ole float 90., %cond1
+  %cond2 = select i1 %cmp2, float %cond1, float 90.
+  ret float %cond2
+}
+
+define float @fp-armv8_vmaxnm_NNNu(float %b) {
+; CHECK-FAST-LABEL: "fp-armv8_vmaxnm_NNNu":
+; CHECK-FAST: vmaxnm.f32
+; CHECK-FAST: vmaxnm.f32
+; CHECK-LABEL: "fp-armv8_vmaxnm_NNNu":
+; CHECK: vmaxnm.f32
+; CHEC-NOT: vmaxnm.f32
+  %cmp1 = fcmp ugt float 12., %b
+  %cond1 = select i1 %cmp1, float 12., float %b
+  %cmp2 = fcmp ugt float %cond1, 34.
+  %cond2 = select i1 %cmp2, float %cond1, float 34.
+  ret float %cond2
+}
+
+define float @fp-armv8_vmaxnm_NNNuge(float %b) {
+; CHECK-FAST-LABEL: "fp-armv8_vmaxnm_NNNuge":
+; CHECK-FAST: vmaxnm.f32
+; CHECK-FAST: vmaxnm.f32
+; CHECK-LABEL: "fp-armv8_vmaxnm_NNNuge":
+; CHECK: vmaxnm.f32
+; CHECK-NOT: vmaxnm.f32
+  %cmp1 = fcmp uge float 34., %b
+  %cond1 = select i1 %cmp1, float 34., float %b
+  %cmp2 = fcmp uge float %cond1, 56.
+  %cond2 = select i1 %cmp2, float %cond1, float 56.
+  ret float %cond2
+}
+
+define float @fp-armv8_vmaxnm_NNNu_rev(float %b) {
+; CHECK-FAST-LABEL: "fp-armv8_vmaxnm_NNNu_rev":
+; CHECK-FAST: vmaxnm.f32
+; CHECK-FAST: vmaxnm.f32
+; CHECK-LABEL: "fp-armv8_vmaxnm_NNNu_rev":
+; CHECK: vmaxnm.f32
+; CHECK-NOT: vmaxnm.f32
+  %cmp1 = fcmp ult float 56., %b
+  %cond1 = select i1 %cmp1, float %b, float 56.
+  %cmp2 = fcmp ult float %cond1, 78.
+  %cond2 = select i1 %cmp2, float 78., float %cond1
+  ret float %cond2
+}
+
+define double @fp-armv8_vmaxnm_NNNule_rev( double %b) {
+; CHECK-FAST-LABEL: "fp-armv8_vmaxnm_NNNule_rev":
+; CHECK-FAST: vmaxnm.f64
+; CHECK-FAST: vmaxnm.f64
+; CHECK-LABEL: "fp-armv8_vmaxnm_NNNule_rev":
+; CHECK: vmaxnm.f64
+; CHECK-NOT: vmaxnm.f64
+  %cmp1 = fcmp ule double 78., %b
+  %cond1 = select i1 %cmp1, double %b, double 78.
+  %cmp2 = fcmp ule double %cond1, 90.
+  %cond2 = select i1 %cmp2, double 90., double %cond1
+  ret double %cond2
+}
+
+define float @fp-armv8_vminmaxnm_0(float %a) {
+; CHECK-FAST-LABEL: "fp-armv8_vminmaxnm_0":
+; CHECK-FAST-NOT: vcmp
+; CHECK-FAST: vminnm.f32
+; CHECK-FAST: vmaxnm.f32
+; CHECK-LABEL: "fp-armv8_vminmaxnm_0":
+; CHECK-NOT: vminnm.f32
+; CHECK: vmaxnm.f32
+  %cmp1 = fcmp olt float %a, 0.
+  %cond1 = select i1 %cmp1, float %a, float 0.
+  %cmp2 = fcmp ogt float %cond1, 0.
+  %cond2 = select i1 %cmp2, float %cond1, float 0.
+  ret float %cond2
+}
+
+define float @fp-armv8_vminmaxnm_neg0(float %a) {
+; CHECK-FAST-LABEL: "fp-armv8_vminmaxnm_neg0":
+; CHECK-FAST-NOT: vcmp
+; CHECK-FAST: vminnm.f32
+; CHECK-FAST: vmaxnm.f32
+; CHECK-LABEL: "fp-armv8_vminmaxnm_neg0":
+; CHECK: vminnm.f32
+; CHECK-NOT: vmaxnm.f32
+  %cmp1 = fcmp olt float %a, -0.
+  %cond1 = select i1 %cmp1, float %a, float -0.
+  %cmp2 = fcmp ogt float %cond1, -0.
+  %cond2 = select i1 %cmp2, float %cond1, float -0.
+  ret float %cond2
+}
+
+define float @fp-armv8_vminmaxnm_e_0(float %a) {
+; CHECK-FAST-LABEL: "fp-armv8_vminmaxnm_e_0":
+; CHECK-FAST-NOT: vcmp
+; CHECK-FAST: vminnm.f32
+; CHECK-FAST: vmaxnm.f32
+; CHECK-LABEL: "fp-armv8_vminmaxnm_e_0":
+; CHECK-NOT: vminnm.f32
+; CHECK: vmaxnm.f32
+  %cmp1 = fcmp ule float 0., %a
+  %cond1 = select i1 %cmp1, float 0., float %a
+  %cmp2 = fcmp uge float 0., %cond1
+  %cond2 = select i1 %cmp2, float 0., float %cond1
+  ret float %cond2
+}
+
+define float @fp-armv8_vminmaxnm_e_neg0(float %a) {
+; CHECK-FAST-LABEL: "fp-armv8_vminmaxnm_e_neg0":
+; CHECK-FAST-NOT: vcmp
+; CHECK-FAST: vminnm.f32
+; CHECK-FAST: vmaxnm.f32
+; CHECK-LABEL: "fp-armv8_vminmaxnm_e_neg0":
+; CHECK: vminnm.f32
+; CHECK-NOT: vmaxnm.f32
+  %cmp1 = fcmp ule float -0., %a
+  %cond1 = select i1 %cmp1, float -0., float %a
+  %cmp2 = fcmp uge float -0., %cond1
+  %cond2 = select i1 %cmp2, float -0., float %cond1
+  ret float %cond2
+}
 
 declare <4 x float> @llvm.arm.neon.vminnm.v4f32(<4 x float>, <4 x float>) nounwind readnone
 declare <2 x float> @llvm.arm.neon.vminnm.v2f32(<2 x float>, <2 x float>) nounwind readnone
diff --git a/test/CodeGen/ARM/vtrn.ll b/test/CodeGen/ARM/vtrn.ll
index caa5becac1d..40b50499318 100644
--- a/test/CodeGen/ARM/vtrn.ll
+++ b/test/CodeGen/ARM/vtrn.ll
@@ -1,9 +1,14 @@
 ; RUN: llc -mtriple=arm-eabi -mattr=+neon %s -o - | FileCheck %s
 
 define <8 x i8> @vtrni8(<8 x i8>* %A, <8 x i8>* %B) nounwind {
-;CHECK-LABEL: vtrni8:
-;CHECK: vtrn.8
-;CHECK-NEXT: vadd.i8
+; CHECK-LABEL: vtrni8:
+; CHECK:       @ BB#0:
+; CHECK-NEXT:    vldr d16, [r1]
+; CHECK-NEXT:    vldr d17, [r0]
+; CHECK-NEXT:    vtrn.8 d17, d16
+; CHECK-NEXT:    vadd.i8 d16, d17, d16
+; CHECK-NEXT:    vmov r0, r1, d16
+; CHECK-NEXT:    mov pc, lr
 	%tmp1 = load <8 x i8>, <8 x i8>* %A
 	%tmp2 = load <8 x i8>, <8 x i8>* %B
 	%tmp3 = shufflevector <8 x i8> %tmp1, <8 x i8> %tmp2, <8 x i32> <i32 0, i32 8, i32 2, i32 10, i32 4, i32 12, i32 6, i32 14>
@@ -12,10 +17,30 @@ define <8 x i8> @vtrni8(<8 x i8>* %A, <8 x i8>* %B) nounwind {
 	ret <8 x i8> %tmp5
 }
 
+define <16 x i8> @vtrni8_Qres(<8 x i8>* %A, <8 x i8>* %B) nounwind {
+; CHECK-LABEL: vtrni8_Qres:
+; CHECK:       @ BB#0:
+; CHECK-NEXT:    vldr d17, [r1]
+; CHECK-NEXT:    vldr d16, [r0]
+; CHECK-NEXT:    vtrn.8 d16, d17
+; CHECK-NEXT:    vmov r0, r1, d16
+; CHECK-NEXT:    vmov r2, r3, d17
+; CHECK-NEXT:    mov pc, lr
+	%tmp1 = load <8 x i8>, <8 x i8>* %A
+	%tmp2 = load <8 x i8>, <8 x i8>* %B
+	%tmp3 = shufflevector <8 x i8> %tmp1, <8 x i8> %tmp2, <16 x i32> <i32 0, i32 8, i32 2, i32 10, i32 4, i32 12, i32 6, i32 14, i32 1, i32 9, i32 3, i32 11, i32 5, i32 13, i32 7, i32 15>
+	ret <16 x i8> %tmp3
+}
+
 define <4 x i16> @vtrni16(<4 x i16>* %A, <4 x i16>* %B) nounwind {
-;CHECK-LABEL: vtrni16:
-;CHECK: vtrn.16
-;CHECK-NEXT: vadd.i16
+; CHECK-LABEL: vtrni16:
+; CHECK:       @ BB#0:
+; CHECK-NEXT:    vldr d16, [r1]
+; CHECK-NEXT:    vldr d17, [r0]
+; CHECK-NEXT:    vtrn.16 d17, d16
+; CHECK-NEXT:    vadd.i16 d16, d17, d16
+; CHECK-NEXT:    vmov r0, r1, d16
+; CHECK-NEXT:    mov pc, lr
 	%tmp1 = load <4 x i16>, <4 x i16>* %A
 	%tmp2 = load <4 x i16>, <4 x i16>* %B
 	%tmp3 = shufflevector <4 x i16> %tmp1, <4 x i16> %tmp2, <4 x i32> <i32 0, i32 4, i32 2, i32 6>
@@ -24,10 +49,30 @@ define <4 x i16> @vtrni16(<4 x i16>* %A, <4 x i16>* %B) nounwind {
 	ret <4 x i16> %tmp5
 }
 
+define <8 x i16> @vtrni16_Qres(<4 x i16>* %A, <4 x i16>* %B) nounwind {
+; CHECK-LABEL: vtrni16_Qres:
+; CHECK:       @ BB#0:
+; CHECK-NEXT:    vldr d17, [r1]
+; CHECK-NEXT:    vldr d16, [r0]
+; CHECK-NEXT:    vtrn.16 d16, d17
+; CHECK-NEXT:    vmov r0, r1, d16
+; CHECK-NEXT:    vmov r2, r3, d17
+; CHECK-NEXT:    mov pc, lr
+	%tmp1 = load <4 x i16>, <4 x i16>* %A
+	%tmp2 = load <4 x i16>, <4 x i16>* %B
+	%tmp3 = shufflevector <4 x i16> %tmp1, <4 x i16> %tmp2, <8 x i32> <i32 0, i32 4, i32 2, i32 6, i32 1, i32 5, i32 3, i32 7>
+	ret <8 x i16> %tmp3
+}
+
 define <2 x i32> @vtrni32(<2 x i32>* %A, <2 x i32>* %B) nounwind {
-;CHECK-LABEL: vtrni32:
-;CHECK: vtrn.32
-;CHECK-NEXT: vadd.i32
+; CHECK-LABEL: vtrni32:
+; CHECK:       @ BB#0:
+; CHECK-NEXT:    vldr d16, [r1]
+; CHECK-NEXT:    vldr d17, [r0]
+; CHECK-NEXT:    vtrn.32 d17, d16
+; CHECK-NEXT:    vadd.i32 d16, d17, d16
+; CHECK-NEXT:    vmov r0, r1, d16
+; CHECK-NEXT:    mov pc, lr
 	%tmp1 = load <2 x i32>, <2 x i32>* %A
 	%tmp2 = load <2 x i32>, <2 x i32>* %B
 	%tmp3 = shufflevector <2 x i32> %tmp1, <2 x i32> %tmp2, <2 x i32> <i32 0, i32 2>
@@ -36,10 +81,30 @@ define <2 x i32> @vtrni32(<2 x i32>* %A, <2 x i32>* %B) nounwind {
 	ret <2 x i32> %tmp5
 }
 
+define <4 x i32> @vtrni32_Qres(<2 x i32>* %A, <2 x i32>* %B) nounwind {
+; CHECK-LABEL: vtrni32_Qres:
+; CHECK:       @ BB#0:
+; CHECK-NEXT:    vldr d17, [r1]
+; CHECK-NEXT:    vldr d16, [r0]
+; CHECK-NEXT:    vtrn.32 d16, d17
+; CHECK-NEXT:    vmov r0, r1, d16
+; CHECK-NEXT:    vmov r2, r3, d17
+; CHECK-NEXT:    mov pc, lr
+	%tmp1 = load <2 x i32>, <2 x i32>* %A
+	%tmp2 = load <2 x i32>, <2 x i32>* %B
+	%tmp3 = shufflevector <2 x i32> %tmp1, <2 x i32> %tmp2, <4 x i32> <i32 0, i32 2, i32 1, i32 3>
+	ret <4 x i32> %tmp3
+}
+
 define <2 x float> @vtrnf(<2 x float>* %A, <2 x float>* %B) nounwind {
-;CHECK-LABEL: vtrnf:
-;CHECK: vtrn.32
-;CHECK-NEXT: vadd.f32
+; CHECK-LABEL: vtrnf:
+; CHECK:       @ BB#0:
+; CHECK-NEXT:    vldr d16, [r1]
+; CHECK-NEXT:    vldr d17, [r0]
+; CHECK-NEXT:    vtrn.32 d17, d16
+; CHECK-NEXT:    vadd.f32 d16, d17, d16
+; CHECK-NEXT:    vmov r0, r1, d16
+; CHECK-NEXT:    mov pc, lr
 	%tmp1 = load <2 x float>, <2 x float>* %A
 	%tmp2 = load <2 x float>, <2 x float>* %B
 	%tmp3 = shufflevector <2 x float> %tmp1, <2 x float> %tmp2, <2 x i32> <i32 0, i32 2>
@@ -48,10 +113,31 @@ define <2 x float> @vtrnf(<2 x float>* %A, <2 x float>* %B) nounwind {
 	ret <2 x float> %tmp5
 }
 
+define <4 x float> @vtrnf_Qres(<2 x float>* %A, <2 x float>* %B) nounwind {
+; CHECK-LABEL: vtrnf_Qres:
+; CHECK:       @ BB#0:
+; CHECK-NEXT:    vldr d17, [r1]
+; CHECK-NEXT:    vldr d16, [r0]
+; CHECK-NEXT:    vtrn.32 d16, d17
+; CHECK-NEXT:    vmov r0, r1, d16
+; CHECK-NEXT:    vmov r2, r3, d17
+; CHECK-NEXT:    mov pc, lr
+	%tmp1 = load <2 x float>, <2 x float>* %A
+	%tmp2 = load <2 x float>, <2 x float>* %B
+	%tmp3 = shufflevector <2 x float> %tmp1, <2 x float> %tmp2, <4 x i32> <i32 0, i32 2, i32 1, i32 3>
+	ret <4 x float> %tmp3
+}
+
 define <16 x i8> @vtrnQi8(<16 x i8>* %A, <16 x i8>* %B) nounwind {
-;CHECK-LABEL: vtrnQi8:
-;CHECK: vtrn.8
-;CHECK-NEXT: vadd.i8
+; CHECK-LABEL: vtrnQi8:
+; CHECK:       @ BB#0:
+; CHECK-NEXT:    vld1.64 {d16, d17}, [r1]
+; CHECK-NEXT:    vld1.64 {d18, d19}, [r0]
+; CHECK-NEXT:    vtrn.8 q9, q8
+; CHECK-NEXT:    vadd.i8 q8, q9, q8
+; CHECK-NEXT:    vmov r0, r1, d16
+; CHECK-NEXT:    vmov r2, r3, d17
+; CHECK-NEXT:    mov pc, lr
 	%tmp1 = load <16 x i8>, <16 x i8>* %A
 	%tmp2 = load <16 x i8>, <16 x i8>* %B
 	%tmp3 = shufflevector <16 x i8> %tmp1, <16 x i8> %tmp2, <16 x i32> <i32 0, i32 16, i32 2, i32 18, i32 4, i32 20, i32 6, i32 22, i32 8, i32 24, i32 10, i32 26, i32 12, i32 28, i32 14, i32 30>
@@ -60,10 +146,31 @@ define <16 x i8> @vtrnQi8(<16 x i8>* %A, <16 x i8>* %B) nounwind {
 	ret <16 x i8> %tmp5
 }
 
+define <32 x i8> @vtrnQi8_QQres(<16 x i8>* %A, <16 x i8>* %B) nounwind {
+; CHECK-LABEL: vtrnQi8_QQres:
+; CHECK:       @ BB#0:
+; CHECK-NEXT:    vld1.64 {d16, d17}, [r2]
+; CHECK-NEXT:    vld1.64 {d18, d19}, [r1]
+; CHECK-NEXT:    vtrn.8 q9, q8
+; CHECK-NEXT:    vst1.8 {d18, d19}, [r0:128]!
+; CHECK-NEXT:    vst1.64 {d16, d17}, [r0:128]
+; CHECK-NEXT:    mov pc, lr
+	%tmp1 = load <16 x i8>, <16 x i8>* %A
+	%tmp2 = load <16 x i8>, <16 x i8>* %B
+	%tmp3 = shufflevector <16 x i8> %tmp1, <16 x i8> %tmp2, <32 x i32> <i32 0, i32 16, i32 2, i32 18, i32 4, i32 20, i32 6, i32 22, i32 8, i32 24, i32 10, i32 26, i32 12, i32 28, i32 14, i32 30, i32 1, i32 17, i32 3, i32 19, i32 5, i32 21, i32 7, i32 23, i32 9, i32 25, i32 11, i32 27, i32 13, i32 29, i32 15, i32 31>
+	ret <32 x i8> %tmp3
+}
+
 define <8 x i16> @vtrnQi16(<8 x i16>* %A, <8 x i16>* %B) nounwind {
-;CHECK-LABEL: vtrnQi16:
-;CHECK: vtrn.16
-;CHECK-NEXT: vadd.i16
+; CHECK-LABEL: vtrnQi16:
+; CHECK:       @ BB#0:
+; CHECK-NEXT:    vld1.64 {d16, d17}, [r1]
+; CHECK-NEXT:    vld1.64 {d18, d19}, [r0]
+; CHECK-NEXT:    vtrn.16 q9, q8
+; CHECK-NEXT:    vadd.i16 q8, q9, q8
+; CHECK-NEXT:    vmov r0, r1, d16
+; CHECK-NEXT:    vmov r2, r3, d17
+; CHECK-NEXT:    mov pc, lr
 	%tmp1 = load <8 x i16>, <8 x i16>* %A
 	%tmp2 = load <8 x i16>, <8 x i16>* %B
 	%tmp3 = shufflevector <8 x i16> %tmp1, <8 x i16> %tmp2, <8 x i32> <i32 0, i32 8, i32 2, i32 10, i32 4, i32 12, i32 6, i32 14>
@@ -72,10 +179,31 @@ define <8 x i16> @vtrnQi16(<8 x i16>* %A, <8 x i16>* %B) nounwind {
 	ret <8 x i16> %tmp5
 }
 
+define <16 x i16> @vtrnQi16_QQres(<8 x i16>* %A, <8 x i16>* %B) nounwind {
+; CHECK-LABEL: vtrnQi16_QQres:
+; CHECK:       @ BB#0:
+; CHECK-NEXT:    vld1.64 {d16, d17}, [r2]
+; CHECK-NEXT:    vld1.64 {d18, d19}, [r1]
+; CHECK-NEXT:    vtrn.16 q9, q8
+; CHECK-NEXT:    vst1.16 {d18, d19}, [r0:128]!
+; CHECK-NEXT:    vst1.64 {d16, d17}, [r0:128]
+; CHECK-NEXT:    mov pc, lr
+	%tmp1 = load <8 x i16>, <8 x i16>* %A
+	%tmp2 = load <8 x i16>, <8 x i16>* %B
+	%tmp3 = shufflevector <8 x i16> %tmp1, <8 x i16> %tmp2, <16 x i32> <i32 0, i32 8, i32 2, i32 10, i32 4, i32 12, i32 6, i32 14, i32 1, i32 9, i32 3, i32 11, i32 5, i32 13, i32 7, i32 15>
+	ret <16 x i16> %tmp3
+}
+
 define <4 x i32> @vtrnQi32(<4 x i32>* %A, <4 x i32>* %B) nounwind {
-;CHECK-LABEL: vtrnQi32:
-;CHECK: vtrn.32
-;CHECK-NEXT: vadd.i32
+; CHECK-LABEL: vtrnQi32:
+; CHECK:       @ BB#0:
+; CHECK-NEXT:    vld1.64 {d16, d17}, [r1]
+; CHECK-NEXT:    vld1.64 {d18, d19}, [r0]
+; CHECK-NEXT:    vtrn.32 q9, q8
+; CHECK-NEXT:    vadd.i32 q8, q9, q8
+; CHECK-NEXT:    vmov r0, r1, d16
+; CHECK-NEXT:    vmov r2, r3, d17
+; CHECK-NEXT:    mov pc, lr
 	%tmp1 = load <4 x i32>, <4 x i32>* %A
 	%tmp2 = load <4 x i32>, <4 x i32>* %B
 	%tmp3 = shufflevector <4 x i32> %tmp1, <4 x i32> %tmp2, <4 x i32> <i32 0, i32 4, i32 2, i32 6>
@@ -84,10 +212,31 @@ define <4 x i32> @vtrnQi32(<4 x i32>* %A, <4 x i32>* %B) nounwind {
 	ret <4 x i32> %tmp5
 }
 
+define <8 x i32> @vtrnQi32_QQres(<4 x i32>* %A, <4 x i32>* %B) nounwind {
+; CHECK-LABEL: vtrnQi32_QQres:
+; CHECK:       @ BB#0:
+; CHECK-NEXT:    vld1.64 {d16, d17}, [r2]
+; CHECK-NEXT:    vld1.64 {d18, d19}, [r1]
+; CHECK-NEXT:    vtrn.32 q9, q8
+; CHECK-NEXT:    vst1.32 {d18, d19}, [r0:128]!
+; CHECK-NEXT:    vst1.64 {d16, d17}, [r0:128]
+; CHECK-NEXT:    mov pc, lr
+	%tmp1 = load <4 x i32>, <4 x i32>* %A
+	%tmp2 = load <4 x i32>, <4 x i32>* %B
+	%tmp3 = shufflevector <4 x i32> %tmp1, <4 x i32> %tmp2, <8 x i32> <i32 0, i32 4, i32 2, i32 6, i32 1, i32 5, i32 3, i32 7>
+	ret <8 x i32> %tmp3
+}
+
 define <4 x float> @vtrnQf(<4 x float>* %A, <4 x float>* %B) nounwind {
-;CHECK-LABEL: vtrnQf:
-;CHECK: vtrn.32
-;CHECK-NEXT: vadd.f32
+; CHECK-LABEL: vtrnQf:
+; CHECK:       @ BB#0:
+; CHECK-NEXT:    vld1.64 {d16, d17}, [r1]
+; CHECK-NEXT:    vld1.64 {d18, d19}, [r0]
+; CHECK-NEXT:    vtrn.32 q9, q8
+; CHECK-NEXT:    vadd.f32 q8, q9, q8
+; CHECK-NEXT:    vmov r0, r1, d16
+; CHECK-NEXT:    vmov r2, r3, d17
+; CHECK-NEXT:    mov pc, lr
 	%tmp1 = load <4 x float>, <4 x float>* %A
 	%tmp2 = load <4 x float>, <4 x float>* %B
 	%tmp3 = shufflevector <4 x float> %tmp1, <4 x float> %tmp2, <4 x i32> <i32 0, i32 4, i32 2, i32 6>
@@ -96,12 +245,31 @@ define <4 x float> @vtrnQf(<4 x float>* %A, <4 x float>* %B) nounwind {
 	ret <4 x float> %tmp5
 }
 
-; Undef shuffle indices should not prevent matching to VTRN:
+define <8 x float> @vtrnQf_QQres(<4 x float>* %A, <4 x float>* %B) nounwind {
+; CHECK-LABEL: vtrnQf_QQres:
+; CHECK:       @ BB#0:
+; CHECK-NEXT:    vld1.64 {d16, d17}, [r2]
+; CHECK-NEXT:    vld1.64 {d18, d19}, [r1]
+; CHECK-NEXT:    vtrn.32 q9, q8
+; CHECK-NEXT:    vst1.32 {d18, d19}, [r0:128]!
+; CHECK-NEXT:    vst1.64 {d16, d17}, [r0:128]
+; CHECK-NEXT:    mov pc, lr
+	%tmp1 = load <4 x float>, <4 x float>* %A
+	%tmp2 = load <4 x float>, <4 x float>* %B
+	%tmp3 = shufflevector <4 x float> %tmp1, <4 x float> %tmp2, <8 x i32> <i32 0, i32 4, i32 2, i32 6, i32 1, i32 5, i32 3, i32 7>
+	ret <8 x float> %tmp3
+}
+
 
 define <8 x i8> @vtrni8_undef(<8 x i8>* %A, <8 x i8>* %B) nounwind {
-;CHECK-LABEL: vtrni8_undef:
-;CHECK: vtrn.8
-;CHECK-NEXT: vadd.i8
+; CHECK-LABEL: vtrni8_undef:
+; CHECK:       @ BB#0:
+; CHECK-NEXT:    vldr d16, [r1]
+; CHECK-NEXT:    vldr d17, [r0]
+; CHECK-NEXT:    vtrn.8 d17, d16
+; CHECK-NEXT:    vadd.i8 d16, d17, d16
+; CHECK-NEXT:    vmov r0, r1, d16
+; CHECK-NEXT:    mov pc, lr
 	%tmp1 = load <8 x i8>, <8 x i8>* %A
 	%tmp2 = load <8 x i8>, <8 x i8>* %B
 	%tmp3 = shufflevector <8 x i8> %tmp1, <8 x i8> %tmp2, <8 x i32> <i32 0, i32 undef, i32 2, i32 10, i32 undef, i32 12, i32 6, i32 14>
@@ -110,10 +278,31 @@ define <8 x i8> @vtrni8_undef(<8 x i8>* %A, <8 x i8>* %B) nounwind {
 	ret <8 x i8> %tmp5
 }
 
+define <16 x i8> @vtrni8_undef_Qres(<8 x i8>* %A, <8 x i8>* %B) nounwind {
+; CHECK-LABEL: vtrni8_undef_Qres:
+; CHECK:       @ BB#0:
+; CHECK-NEXT:    vldr d17, [r1]
+; CHECK-NEXT:    vldr d16, [r0]
+; CHECK-NEXT:    vtrn.8 d16, d17
+; CHECK-NEXT:    vmov r0, r1, d16
+; CHECK-NEXT:    vmov r2, r3, d17
+; CHECK-NEXT:    mov pc, lr
+	%tmp1 = load <8 x i8>, <8 x i8>* %A
+	%tmp2 = load <8 x i8>, <8 x i8>* %B
+	%tmp3 = shufflevector <8 x i8> %tmp1, <8 x i8> %tmp2, <16 x i32> <i32 0, i32 undef, i32 2, i32 10, i32 undef, i32 12, i32 6, i32 14, i32 1, i32 9, i32 3, i32 11, i32 5, i32 undef, i32 undef, i32 15>
+	ret <16 x i8> %tmp3
+}
+
 define <8 x i16> @vtrnQi16_undef(<8 x i16>* %A, <8 x i16>* %B) nounwind {
-;CHECK-LABEL: vtrnQi16_undef:
-;CHECK: vtrn.16
-;CHECK-NEXT: vadd.i16
+; CHECK-LABEL: vtrnQi16_undef:
+; CHECK:       @ BB#0:
+; CHECK-NEXT:    vld1.64 {d16, d17}, [r1]
+; CHECK-NEXT:    vld1.64 {d18, d19}, [r0]
+; CHECK-NEXT:    vtrn.16 q9, q8
+; CHECK-NEXT:    vadd.i16 q8, q9, q8
+; CHECK-NEXT:    vmov r0, r1, d16
+; CHECK-NEXT:    vmov r2, r3, d17
+; CHECK-NEXT:    mov pc, lr
 	%tmp1 = load <8 x i16>, <8 x i16>* %A
 	%tmp2 = load <8 x i16>, <8 x i16>* %B
 	%tmp3 = shufflevector <8 x i16> %tmp1, <8 x i16> %tmp2, <8 x i32> <i32 0, i32 8, i32 undef, i32 undef, i32 4, i32 12, i32 6, i32 14>
@@ -122,3 +311,27 @@ define <8 x i16> @vtrnQi16_undef(<8 x i16>* %A, <8 x i16>* %B) nounwind {
 	ret <8 x i16> %tmp5
 }
 
+define <16 x i16> @vtrnQi16_undef_QQres(<8 x i16>* %A, <8 x i16>* %B) nounwind {
+; CHECK-LABEL: vtrnQi16_undef_QQres:
+; CHECK:       @ BB#0:
+; CHECK-NEXT:    vld1.64 {d16, d17}, [r2]
+; CHECK-NEXT:    vld1.64 {d18, d19}, [r1]
+; CHECK-NEXT:    vtrn.16 q9, q8
+; CHECK-NEXT:    vst1.16 {d18, d19}, [r0:128]!
+; CHECK-NEXT:    vst1.64 {d16, d17}, [r0:128]
+; CHECK-NEXT:    mov pc, lr
+	%tmp1 = load <8 x i16>, <8 x i16>* %A
+	%tmp2 = load <8 x i16>, <8 x i16>* %B
+	%tmp3 = shufflevector <8 x i16> %tmp1, <8 x i16> %tmp2, <16 x i32> <i32 0, i32 8, i32 undef, i32 undef, i32 4, i32 12, i32 6, i32 14, i32 1, i32 undef, i32 3, i32 11, i32 5, i32 13, i32 undef, i32 undef>
+	ret <16 x i16> %tmp3
+}
+
+define <8 x i16> @vtrn_lower_shufflemask_undef(<4 x i16>* %A, <4 x i16>* %B) {
+entry:
+  ; CHECK-LABEL: vtrn_lower_shufflemask_undef
+  ; CHECK: vtrn
+	%tmp1 = load <4 x i16>, <4 x i16>* %A
+	%tmp2 = load <4 x i16>, <4 x i16>* %B
+  %0 = shufflevector <4 x i16> %tmp1, <4 x i16> %tmp2, <8 x i32> <i32 undef, i32 undef, i32 undef, i32 undef, i32 1, i32 5, i32 3, i32 7>
+  ret <8 x i16> %0
+}
diff --git a/test/CodeGen/ARM/vuzp.ll b/test/CodeGen/ARM/vuzp.ll
index 7a7306a2659..8e71ad81264 100644
--- a/test/CodeGen/ARM/vuzp.ll
+++ b/test/CodeGen/ARM/vuzp.ll
@@ -1,9 +1,14 @@
 ; RUN: llc -mtriple=arm-eabi -mattr=+neon %s -o - | FileCheck %s
 
 define <8 x i8> @vuzpi8(<8 x i8>* %A, <8 x i8>* %B) nounwind {
-;CHECK-LABEL: vuzpi8:
-;CHECK: vuzp.8
-;CHECK-NEXT: vadd.i8
+; CHECK-LABEL: vuzpi8:
+; CHECK:       @ BB#0:
+; CHECK-NEXT:    vldr d16, [r1]
+; CHECK-NEXT:    vldr d17, [r0]
+; CHECK-NEXT:    vuzp.8 d17, d16
+; CHECK-NEXT:    vadd.i8 d16, d17, d16
+; CHECK-NEXT:    vmov r0, r1, d16
+; CHECK-NEXT:    mov pc, lr
 	%tmp1 = load <8 x i8>, <8 x i8>* %A
 	%tmp2 = load <8 x i8>, <8 x i8>* %B
 	%tmp3 = shufflevector <8 x i8> %tmp1, <8 x i8> %tmp2, <8 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14>
@@ -12,10 +17,30 @@ define <8 x i8> @vuzpi8(<8 x i8>* %A, <8 x i8>* %B) nounwind {
 	ret <8 x i8> %tmp5
 }
 
+define <16 x i8> @vuzpi8_Qres(<8 x i8>* %A, <8 x i8>* %B) nounwind {
+; CHECK-LABEL: vuzpi8_Qres:
+; CHECK:       @ BB#0:
+; CHECK-NEXT:    vldr d17, [r1]
+; CHECK-NEXT:    vldr d16, [r0]
+; CHECK-NEXT:    vuzp.8 d16, d17
+; CHECK-NEXT:    vmov r0, r1, d16
+; CHECK-NEXT:    vmov r2, r3, d17
+; CHECK-NEXT:    mov pc, lr
+	%tmp1 = load <8 x i8>, <8 x i8>* %A
+	%tmp2 = load <8 x i8>, <8 x i8>* %B
+	%tmp3 = shufflevector <8 x i8> %tmp1, <8 x i8> %tmp2, <16 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14, i32 1, i32 3, i32 5, i32 7, i32 9, i32 11, i32 13, i32 15>
+	ret <16 x i8> %tmp3
+}
+
 define <4 x i16> @vuzpi16(<4 x i16>* %A, <4 x i16>* %B) nounwind {
-;CHECK-LABEL: vuzpi16:
-;CHECK: vuzp.16
-;CHECK-NEXT: vadd.i16
+; CHECK-LABEL: vuzpi16:
+; CHECK:       @ BB#0:
+; CHECK-NEXT:    vldr d16, [r1]
+; CHECK-NEXT:    vldr d17, [r0]
+; CHECK-NEXT:    vuzp.16 d17, d16
+; CHECK-NEXT:    vadd.i16 d16, d17, d16
+; CHECK-NEXT:    vmov r0, r1, d16
+; CHECK-NEXT:    mov pc, lr
 	%tmp1 = load <4 x i16>, <4 x i16>* %A
 	%tmp2 = load <4 x i16>, <4 x i16>* %B
 	%tmp3 = shufflevector <4 x i16> %tmp1, <4 x i16> %tmp2, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
@@ -24,12 +49,33 @@ define <4 x i16> @vuzpi16(<4 x i16>* %A, <4 x i16>* %B) nounwind {
 	ret <4 x i16> %tmp5
 }
 
+define <8 x i16> @vuzpi16_Qres(<4 x i16>* %A, <4 x i16>* %B) nounwind {
+; CHECK-LABEL: vuzpi16_Qres:
+; CHECK:       @ BB#0:
+; CHECK-NEXT:    vldr d17, [r1]
+; CHECK-NEXT:    vldr d16, [r0]
+; CHECK-NEXT:    vuzp.16 d16, d17
+; CHECK-NEXT:    vmov r0, r1, d16
+; CHECK-NEXT:    vmov r2, r3, d17
+; CHECK-NEXT:    mov pc, lr
+	%tmp1 = load <4 x i16>, <4 x i16>* %A
+	%tmp2 = load <4 x i16>, <4 x i16>* %B
+	%tmp3 = shufflevector <4 x i16> %tmp1, <4 x i16> %tmp2, <8 x i32> <i32 0, i32 2, i32 4, i32 6, i32 1, i32 3, i32 5, i32 7>
+	ret <8 x i16> %tmp3
+}
+
 ; VUZP.32 is equivalent to VTRN.32 for 64-bit vectors.
 
 define <16 x i8> @vuzpQi8(<16 x i8>* %A, <16 x i8>* %B) nounwind {
-;CHECK-LABEL: vuzpQi8:
-;CHECK: vuzp.8
-;CHECK-NEXT: vadd.i8
+; CHECK-LABEL: vuzpQi8:
+; CHECK:       @ BB#0:
+; CHECK-NEXT:    vld1.64 {d16, d17}, [r1]
+; CHECK-NEXT:    vld1.64 {d18, d19}, [r0]
+; CHECK-NEXT:    vuzp.8 q9, q8
+; CHECK-NEXT:    vadd.i8 q8, q9, q8
+; CHECK-NEXT:    vmov r0, r1, d16
+; CHECK-NEXT:    vmov r2, r3, d17
+; CHECK-NEXT:    mov pc, lr
 	%tmp1 = load <16 x i8>, <16 x i8>* %A
 	%tmp2 = load <16 x i8>, <16 x i8>* %B
 	%tmp3 = shufflevector <16 x i8> %tmp1, <16 x i8> %tmp2, <16 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14, i32 16, i32 18, i32 20, i32 22, i32 24, i32 26, i32 28, i32 30>
@@ -38,10 +84,31 @@ define <16 x i8> @vuzpQi8(<16 x i8>* %A, <16 x i8>* %B) nounwind {
 	ret <16 x i8> %tmp5
 }
 
+define <32 x i8> @vuzpQi8_QQres(<16 x i8>* %A, <16 x i8>* %B) nounwind {
+; CHECK-LABEL: vuzpQi8_QQres:
+; CHECK:       @ BB#0:
+; CHECK-NEXT:    vld1.64 {d16, d17}, [r2]
+; CHECK-NEXT:    vld1.64 {d18, d19}, [r1]
+; CHECK-NEXT:    vuzp.8 q9, q8
+; CHECK-NEXT:    vst1.8 {d18, d19}, [r0:128]!
+; CHECK-NEXT:    vst1.64 {d16, d17}, [r0:128]
+; CHECK-NEXT:    mov pc, lr
+	%tmp1 = load <16 x i8>, <16 x i8>* %A
+	%tmp2 = load <16 x i8>, <16 x i8>* %B
+	%tmp3 = shufflevector <16 x i8> %tmp1, <16 x i8> %tmp2, <32 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14, i32 16, i32 18, i32 20, i32 22, i32 24, i32 26, i32 28, i32 30, i32 1, i32 3, i32 5, i32 7, i32 9, i32 11, i32 13, i32 15, i32 17, i32 19, i32 21, i32 23, i32 25, i32 27, i32 29, i32 31>
+	ret <32 x i8> %tmp3
+}
+
 define <8 x i16> @vuzpQi16(<8 x i16>* %A, <8 x i16>* %B) nounwind {
-;CHECK-LABEL: vuzpQi16:
-;CHECK: vuzp.16
-;CHECK-NEXT: vadd.i16
+; CHECK-LABEL: vuzpQi16:
+; CHECK:       @ BB#0:
+; CHECK-NEXT:    vld1.64 {d16, d17}, [r1]
+; CHECK-NEXT:    vld1.64 {d18, d19}, [r0]
+; CHECK-NEXT:    vuzp.16 q9, q8
+; CHECK-NEXT:    vadd.i16 q8, q9, q8
+; CHECK-NEXT:    vmov r0, r1, d16
+; CHECK-NEXT:    vmov r2, r3, d17
+; CHECK-NEXT:    mov pc, lr
 	%tmp1 = load <8 x i16>, <8 x i16>* %A
 	%tmp2 = load <8 x i16>, <8 x i16>* %B
 	%tmp3 = shufflevector <8 x i16> %tmp1, <8 x i16> %tmp2, <8 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14>
@@ -50,10 +117,31 @@ define <8 x i16> @vuzpQi16(<8 x i16>* %A, <8 x i16>* %B) nounwind {
 	ret <8 x i16> %tmp5
 }
 
+define <16 x i16> @vuzpQi16_QQres(<8 x i16>* %A, <8 x i16>* %B) nounwind {
+; CHECK-LABEL: vuzpQi16_QQres:
+; CHECK:       @ BB#0:
+; CHECK-NEXT:    vld1.64 {d16, d17}, [r2]
+; CHECK-NEXT:    vld1.64 {d18, d19}, [r1]
+; CHECK-NEXT:    vuzp.16 q9, q8
+; CHECK-NEXT:    vst1.16 {d18, d19}, [r0:128]!
+; CHECK-NEXT:    vst1.64 {d16, d17}, [r0:128]
+; CHECK-NEXT:    mov pc, lr
+	%tmp1 = load <8 x i16>, <8 x i16>* %A
+	%tmp2 = load <8 x i16>, <8 x i16>* %B
+	%tmp3 = shufflevector <8 x i16> %tmp1, <8 x i16> %tmp2, <16 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14, i32 1, i32 3, i32 5, i32 7, i32 9, i32 11, i32 13, i32 15>
+	ret <16 x i16> %tmp3
+}
+
 define <4 x i32> @vuzpQi32(<4 x i32>* %A, <4 x i32>* %B) nounwind {
-;CHECK-LABEL: vuzpQi32:
-;CHECK: vuzp.32
-;CHECK-NEXT: vadd.i32
+; CHECK-LABEL: vuzpQi32:
+; CHECK:       @ BB#0:
+; CHECK-NEXT:    vld1.64 {d16, d17}, [r1]
+; CHECK-NEXT:    vld1.64 {d18, d19}, [r0]
+; CHECK-NEXT:    vuzp.32 q9, q8
+; CHECK-NEXT:    vadd.i32 q8, q9, q8
+; CHECK-NEXT:    vmov r0, r1, d16
+; CHECK-NEXT:    vmov r2, r3, d17
+; CHECK-NEXT:    mov pc, lr
 	%tmp1 = load <4 x i32>, <4 x i32>* %A
 	%tmp2 = load <4 x i32>, <4 x i32>* %B
 	%tmp3 = shufflevector <4 x i32> %tmp1, <4 x i32> %tmp2, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
@@ -62,10 +150,31 @@ define <4 x i32> @vuzpQi32(<4 x i32>* %A, <4 x i32>* %B) nounwind {
 	ret <4 x i32> %tmp5
 }
 
+define <8 x i32> @vuzpQi32_QQres(<4 x i32>* %A, <4 x i32>* %B) nounwind {
+; CHECK-LABEL: vuzpQi32_QQres:
+; CHECK:       @ BB#0:
+; CHECK-NEXT:    vld1.64 {d16, d17}, [r2]
+; CHECK-NEXT:    vld1.64 {d18, d19}, [r1]
+; CHECK-NEXT:    vuzp.32 q9, q8
+; CHECK-NEXT:    vst1.32 {d18, d19}, [r0:128]!
+; CHECK-NEXT:    vst1.64 {d16, d17}, [r0:128]
+; CHECK-NEXT:    mov pc, lr
+	%tmp1 = load <4 x i32>, <4 x i32>* %A
+	%tmp2 = load <4 x i32>, <4 x i32>* %B
+	%tmp3 = shufflevector <4 x i32> %tmp1, <4 x i32> %tmp2, <8 x i32> <i32 0, i32 2, i32 4, i32 6, i32 1, i32 3, i32 5, i32 7>
+	ret <8 x i32> %tmp3
+}
+
 define <4 x float> @vuzpQf(<4 x float>* %A, <4 x float>* %B) nounwind {
-;CHECK-LABEL: vuzpQf:
-;CHECK: vuzp.32
-;CHECK-NEXT: vadd.f32
+; CHECK-LABEL: vuzpQf:
+; CHECK:       @ BB#0:
+; CHECK-NEXT:    vld1.64 {d16, d17}, [r1]
+; CHECK-NEXT:    vld1.64 {d18, d19}, [r0]
+; CHECK-NEXT:    vuzp.32 q9, q8
+; CHECK-NEXT:    vadd.f32 q8, q9, q8
+; CHECK-NEXT:    vmov r0, r1, d16
+; CHECK-NEXT:    vmov r2, r3, d17
+; CHECK-NEXT:    mov pc, lr
 	%tmp1 = load <4 x float>, <4 x float>* %A
 	%tmp2 = load <4 x float>, <4 x float>* %B
 	%tmp3 = shufflevector <4 x float> %tmp1, <4 x float> %tmp2, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
@@ -74,12 +183,32 @@ define <4 x float> @vuzpQf(<4 x float>* %A, <4 x float>* %B) nounwind {
 	ret <4 x float> %tmp5
 }
 
+define <8 x float> @vuzpQf_QQres(<4 x float>* %A, <4 x float>* %B) nounwind {
+; CHECK-LABEL: vuzpQf_QQres:
+; CHECK:       @ BB#0:
+; CHECK-NEXT:    vld1.64 {d16, d17}, [r2]
+; CHECK-NEXT:    vld1.64 {d18, d19}, [r1]
+; CHECK-NEXT:    vuzp.32 q9, q8
+; CHECK-NEXT:    vst1.32 {d18, d19}, [r0:128]!
+; CHECK-NEXT:    vst1.64 {d16, d17}, [r0:128]
+; CHECK-NEXT:    mov pc, lr
+	%tmp1 = load <4 x float>, <4 x float>* %A
+	%tmp2 = load <4 x float>, <4 x float>* %B
+	%tmp3 = shufflevector <4 x float> %tmp1, <4 x float> %tmp2, <8 x i32> <i32 0, i32 2, i32 4, i32 6, i32 1, i32 3, i32 5, i32 7>
+	ret <8 x float> %tmp3
+}
+
 ; Undef shuffle indices should not prevent matching to VUZP:
 
 define <8 x i8> @vuzpi8_undef(<8 x i8>* %A, <8 x i8>* %B) nounwind {
-;CHECK-LABEL: vuzpi8_undef:
-;CHECK: vuzp.8
-;CHECK-NEXT: vadd.i8
+; CHECK-LABEL: vuzpi8_undef:
+; CHECK:       @ BB#0:
+; CHECK-NEXT:    vldr d16, [r1]
+; CHECK-NEXT:    vldr d17, [r0]
+; CHECK-NEXT:    vuzp.8 d17, d16
+; CHECK-NEXT:    vadd.i8 d16, d17, d16
+; CHECK-NEXT:    vmov r0, r1, d16
+; CHECK-NEXT:    mov pc, lr
 	%tmp1 = load <8 x i8>, <8 x i8>* %A
 	%tmp2 = load <8 x i8>, <8 x i8>* %B
 	%tmp3 = shufflevector <8 x i8> %tmp1, <8 x i8> %tmp2, <8 x i32> <i32 0, i32 2, i32 undef, i32 undef, i32 8, i32 10, i32 12, i32 14>
@@ -88,10 +217,31 @@ define <8 x i8> @vuzpi8_undef(<8 x i8>* %A, <8 x i8>* %B) nounwind {
 	ret <8 x i8> %tmp5
 }
 
+define <16 x i8> @vuzpi8_undef_Qres(<8 x i8>* %A, <8 x i8>* %B) nounwind {
+; CHECK-LABEL: vuzpi8_undef_Qres:
+; CHECK:       @ BB#0:
+; CHECK-NEXT:    vldr d17, [r1]
+; CHECK-NEXT:    vldr d16, [r0]
+; CHECK-NEXT:    vuzp.8 d16, d17
+; CHECK-NEXT:    vmov r0, r1, d16
+; CHECK-NEXT:    vmov r2, r3, d17
+; CHECK-NEXT:    mov pc, lr
+	%tmp1 = load <8 x i8>, <8 x i8>* %A
+	%tmp2 = load <8 x i8>, <8 x i8>* %B
+	%tmp3 = shufflevector <8 x i8> %tmp1, <8 x i8> %tmp2, <16 x i32> <i32 0, i32 2, i32 undef, i32 undef, i32 8, i32 10, i32 12, i32 14, i32 1, i32 3, i32 5, i32 7, i32 undef, i32 undef, i32 13, i32 15>
+	ret <16 x i8> %tmp3
+}
+
 define <8 x i16> @vuzpQi16_undef(<8 x i16>* %A, <8 x i16>* %B) nounwind {
-;CHECK-LABEL: vuzpQi16_undef:
-;CHECK: vuzp.16
-;CHECK-NEXT: vadd.i16
+; CHECK-LABEL: vuzpQi16_undef:
+; CHECK:       @ BB#0:
+; CHECK-NEXT:    vld1.64 {d16, d17}, [r1]
+; CHECK-NEXT:    vld1.64 {d18, d19}, [r0]
+; CHECK-NEXT:    vuzp.16 q9, q8
+; CHECK-NEXT:    vadd.i16 q8, q9, q8
+; CHECK-NEXT:    vmov r0, r1, d16
+; CHECK-NEXT:    vmov r2, r3, d17
+; CHECK-NEXT:    mov pc, lr
 	%tmp1 = load <8 x i16>, <8 x i16>* %A
 	%tmp2 = load <8 x i16>, <8 x i16>* %B
 	%tmp3 = shufflevector <8 x i16> %tmp1, <8 x i16> %tmp2, <8 x i32> <i32 0, i32 undef, i32 4, i32 undef, i32 8, i32 10, i32 12, i32 14>
@@ -100,3 +250,38 @@ define <8 x i16> @vuzpQi16_undef(<8 x i16>* %A, <8 x i16>* %B) nounwind {
 	ret <8 x i16> %tmp5
 }
 
+define <16 x i16> @vuzpQi16_undef_QQres(<8 x i16>* %A, <8 x i16>* %B) nounwind {
+; CHECK-LABEL: vuzpQi16_undef_QQres:
+; CHECK:       @ BB#0:
+; CHECK-NEXT:    vld1.64 {d16, d17}, [r2]
+; CHECK-NEXT:    vld1.64 {d18, d19}, [r1]
+; CHECK-NEXT:    vuzp.16 q9, q8
+; CHECK-NEXT:    vst1.16 {d18, d19}, [r0:128]!
+; CHECK-NEXT:    vst1.64 {d16, d17}, [r0:128]
+; CHECK-NEXT:    mov pc, lr
+	%tmp1 = load <8 x i16>, <8 x i16>* %A
+	%tmp2 = load <8 x i16>, <8 x i16>* %B
+	%tmp3 = shufflevector <8 x i16> %tmp1, <8 x i16> %tmp2, <16 x i32> <i32 0, i32 undef, i32 4, i32 undef, i32 8, i32 10, i32 12, i32 14, i32 1, i32 3, i32 5, i32 undef, i32 undef, i32 11, i32 13, i32 15>
+	ret <16 x i16> %tmp3
+}
+
+define <8 x i16> @vuzp_lower_shufflemask_undef(<4 x i16>* %A, <4 x i16>* %B) {
+entry:
+  ; CHECK-LABEL: vuzp_lower_shufflemask_undef
+  ; CHECK: vuzp
+	%tmp1 = load <4 x i16>, <4 x i16>* %A
+	%tmp2 = load <4 x i16>, <4 x i16>* %B
+  %0 = shufflevector <4 x i16> %tmp1, <4 x i16> %tmp2, <8 x i32> <i32 undef, i32 undef, i32 undef, i32 undef, i32 1, i32 3, i32 5, i32 7>
+  ret <8 x i16> %0
+}
+
+define <4 x i32> @vuzp_lower_shufflemask_zeroed(<2 x i32>* %A, <2 x i32>* %B) {
+entry:
+  ; CHECK-LABEL: vuzp_lower_shufflemask_zeroed
+  ; CHECK-NOT: vtrn
+  ; CHECK: vuzp
+  %tmp1 = load <2 x i32>, <2 x i32>* %A
+	%tmp2 = load <2 x i32>, <2 x i32>* %B
+  %0 = shufflevector <2 x i32> %tmp1, <2 x i32> %tmp2, <4 x i32> <i32 0, i32 0, i32 1, i32 3>
+  ret <4 x i32> %0
+}
diff --git a/test/CodeGen/ARM/vzip.ll b/test/CodeGen/ARM/vzip.ll
index a1b5b4549ac..e7b7da333f4 100644
--- a/test/CodeGen/ARM/vzip.ll
+++ b/test/CodeGen/ARM/vzip.ll
@@ -1,9 +1,14 @@
 ; RUN: llc -mtriple=arm-eabi -mattr=+neon %s -o - | FileCheck %s
 
 define <8 x i8> @vzipi8(<8 x i8>* %A, <8 x i8>* %B) nounwind {
-;CHECK-LABEL: vzipi8:
-;CHECK: vzip.8
-;CHECK-NEXT: vadd.i8
+; CHECK-LABEL: vzipi8:
+; CHECK:       @ BB#0:
+; CHECK-NEXT:    vldr d16, [r1]
+; CHECK-NEXT:    vldr d17, [r0]
+; CHECK-NEXT:    vzip.8 d17, d16
+; CHECK-NEXT:    vadd.i8 d16, d17, d16
+; CHECK-NEXT:    vmov r0, r1, d16
+; CHECK-NEXT:    mov pc, lr
 	%tmp1 = load <8 x i8>, <8 x i8>* %A
 	%tmp2 = load <8 x i8>, <8 x i8>* %B
 	%tmp3 = shufflevector <8 x i8> %tmp1, <8 x i8> %tmp2, <8 x i32> <i32 0, i32 8, i32 1, i32 9, i32 2, i32 10, i32 3, i32 11>
@@ -12,10 +17,30 @@ define <8 x i8> @vzipi8(<8 x i8>* %A, <8 x i8>* %B) nounwind {
 	ret <8 x i8> %tmp5
 }
 
+define <16 x i8> @vzipi8_Qres(<8 x i8>* %A, <8 x i8>* %B) nounwind {
+; CHECK-LABEL: vzipi8_Qres:
+; CHECK:       @ BB#0:
+; CHECK-NEXT:    vldr d17, [r1]
+; CHECK-NEXT:    vldr d16, [r0]
+; CHECK-NEXT:    vzip.8 d16, d17
+; CHECK-NEXT:    vmov r0, r1, d16
+; CHECK-NEXT:    vmov r2, r3, d17
+; CHECK-NEXT:    mov pc, lr
+	%tmp1 = load <8 x i8>, <8 x i8>* %A
+	%tmp2 = load <8 x i8>, <8 x i8>* %B
+	%tmp3 = shufflevector <8 x i8> %tmp1, <8 x i8> %tmp2, <16 x i32> <i32 0, i32 8, i32 1, i32 9, i32 2, i32 10, i32 3, i32 11, i32 4, i32 12, i32 5, i32 13, i32 6, i32 14, i32 7, i32 15>
+	ret <16 x i8> %tmp3
+}
+
 define <4 x i16> @vzipi16(<4 x i16>* %A, <4 x i16>* %B) nounwind {
-;CHECK-LABEL: vzipi16:
-;CHECK: vzip.16
-;CHECK-NEXT: vadd.i16
+; CHECK-LABEL: vzipi16:
+; CHECK:       @ BB#0:
+; CHECK-NEXT:    vldr d16, [r1]
+; CHECK-NEXT:    vldr d17, [r0]
+; CHECK-NEXT:    vzip.16 d17, d16
+; CHECK-NEXT:    vadd.i16 d16, d17, d16
+; CHECK-NEXT:    vmov r0, r1, d16
+; CHECK-NEXT:    mov pc, lr
 	%tmp1 = load <4 x i16>, <4 x i16>* %A
 	%tmp2 = load <4 x i16>, <4 x i16>* %B
 	%tmp3 = shufflevector <4 x i16> %tmp1, <4 x i16> %tmp2, <4 x i32> <i32 0, i32 4, i32 1, i32 5>
@@ -24,12 +49,33 @@ define <4 x i16> @vzipi16(<4 x i16>* %A, <4 x i16>* %B) nounwind {
 	ret <4 x i16> %tmp5
 }
 
+define <8 x i16> @vzipi16_Qres(<4 x i16>* %A, <4 x i16>* %B) nounwind {
+; CHECK-LABEL: vzipi16_Qres:
+; CHECK:       @ BB#0:
+; CHECK-NEXT:    vldr d17, [r1]
+; CHECK-NEXT:    vldr d16, [r0]
+; CHECK-NEXT:    vzip.16 d16, d17
+; CHECK-NEXT:    vmov r0, r1, d16
+; CHECK-NEXT:    vmov r2, r3, d17
+; CHECK-NEXT:    mov pc, lr
+	%tmp1 = load <4 x i16>, <4 x i16>* %A
+	%tmp2 = load <4 x i16>, <4 x i16>* %B
+	%tmp3 = shufflevector <4 x i16> %tmp1, <4 x i16> %tmp2, <8 x i32> <i32 0, i32 4, i32 1, i32 5, i32 2, i32 6, i32 3, i32 7>
+	ret <8 x i16> %tmp3
+}
+
 ; VZIP.32 is equivalent to VTRN.32 for 64-bit vectors.
 
 define <16 x i8> @vzipQi8(<16 x i8>* %A, <16 x i8>* %B) nounwind {
-;CHECK-LABEL: vzipQi8:
-;CHECK: vzip.8
-;CHECK-NEXT: vadd.i8
+; CHECK-LABEL: vzipQi8:
+; CHECK:       @ BB#0:
+; CHECK-NEXT:    vld1.64 {d16, d17}, [r1]
+; CHECK-NEXT:    vld1.64 {d18, d19}, [r0]
+; CHECK-NEXT:    vzip.8 q9, q8
+; CHECK-NEXT:    vadd.i8 q8, q9, q8
+; CHECK-NEXT:    vmov r0, r1, d16
+; CHECK-NEXT:    vmov r2, r3, d17
+; CHECK-NEXT:    mov pc, lr
 	%tmp1 = load <16 x i8>, <16 x i8>* %A
 	%tmp2 = load <16 x i8>, <16 x i8>* %B
 	%tmp3 = shufflevector <16 x i8> %tmp1, <16 x i8> %tmp2, <16 x i32> <i32 0, i32 16, i32 1, i32 17, i32 2, i32 18, i32 3, i32 19, i32 4, i32 20, i32 5, i32 21, i32 6, i32 22, i32 7, i32 23>
@@ -38,10 +84,31 @@ define <16 x i8> @vzipQi8(<16 x i8>* %A, <16 x i8>* %B) nounwind {
 	ret <16 x i8> %tmp5
 }
 
+define <32 x i8> @vzipQi8_QQres(<16 x i8>* %A, <16 x i8>* %B) nounwind {
+; CHECK-LABEL: vzipQi8_QQres:
+; CHECK:       @ BB#0:
+; CHECK-NEXT:    vld1.64 {d16, d17}, [r2]
+; CHECK-NEXT:    vld1.64 {d18, d19}, [r1]
+; CHECK-NEXT:    vzip.8 q9, q8
+; CHECK-NEXT:    vst1.8 {d18, d19}, [r0:128]!
+; CHECK-NEXT:    vst1.64 {d16, d17}, [r0:128]
+; CHECK-NEXT:    mov pc, lr
+	%tmp1 = load <16 x i8>, <16 x i8>* %A
+	%tmp2 = load <16 x i8>, <16 x i8>* %B
+	%tmp3 = shufflevector <16 x i8> %tmp1, <16 x i8> %tmp2, <32 x i32> <i32 0, i32 16, i32 1, i32 17, i32 2, i32 18, i32 3, i32 19, i32 4, i32 20, i32 5, i32 21, i32 6, i32 22, i32 7, i32 23, i32 8, i32 24, i32 9, i32 25, i32 10, i32 26, i32 11, i32 27, i32 12, i32 28, i32 13, i32 29, i32 14, i32 30, i32 15, i32 31>
+	ret <32 x i8> %tmp3
+}
+
 define <8 x i16> @vzipQi16(<8 x i16>* %A, <8 x i16>* %B) nounwind {
-;CHECK-LABEL: vzipQi16:
-;CHECK: vzip.16
-;CHECK-NEXT: vadd.i16
+; CHECK-LABEL: vzipQi16:
+; CHECK:       @ BB#0:
+; CHECK-NEXT:    vld1.64 {d16, d17}, [r1]
+; CHECK-NEXT:    vld1.64 {d18, d19}, [r0]
+; CHECK-NEXT:    vzip.16 q9, q8
+; CHECK-NEXT:    vadd.i16 q8, q9, q8
+; CHECK-NEXT:    vmov r0, r1, d16
+; CHECK-NEXT:    vmov r2, r3, d17
+; CHECK-NEXT:    mov pc, lr
 	%tmp1 = load <8 x i16>, <8 x i16>* %A
 	%tmp2 = load <8 x i16>, <8 x i16>* %B
 	%tmp3 = shufflevector <8 x i16> %tmp1, <8 x i16> %tmp2, <8 x i32> <i32 0, i32 8, i32 1, i32 9, i32 2, i32 10, i32 3, i32 11>
@@ -50,10 +117,31 @@ define <8 x i16> @vzipQi16(<8 x i16>* %A, <8 x i16>* %B) nounwind {
 	ret <8 x i16> %tmp5
 }
 
+define <16 x i16> @vzipQi16_QQres(<8 x i16>* %A, <8 x i16>* %B) nounwind {
+; CHECK-LABEL: vzipQi16_QQres:
+; CHECK:       @ BB#0:
+; CHECK-NEXT:    vld1.64 {d16, d17}, [r2]
+; CHECK-NEXT:    vld1.64 {d18, d19}, [r1]
+; CHECK-NEXT:    vzip.16 q9, q8
+; CHECK-NEXT:    vst1.16 {d18, d19}, [r0:128]!
+; CHECK-NEXT:    vst1.64 {d16, d17}, [r0:128]
+; CHECK-NEXT:    mov pc, lr
+	%tmp1 = load <8 x i16>, <8 x i16>* %A
+	%tmp2 = load <8 x i16>, <8 x i16>* %B
+	%tmp3 = shufflevector <8 x i16> %tmp1, <8 x i16> %tmp2, <16 x i32> <i32 0, i32 8, i32 1, i32 9, i32 2, i32 10, i32 3, i32 11, i32 4, i32 12, i32 5, i32 13, i32 6, i32 14, i32 7, i32 15>
+	ret <16 x i16> %tmp3
+}
+
 define <4 x i32> @vzipQi32(<4 x i32>* %A, <4 x i32>* %B) nounwind {
-;CHECK-LABEL: vzipQi32:
-;CHECK: vzip.32
-;CHECK-NEXT: vadd.i32
+; CHECK-LABEL: vzipQi32:
+; CHECK:       @ BB#0:
+; CHECK-NEXT:    vld1.64 {d16, d17}, [r1]
+; CHECK-NEXT:    vld1.64 {d18, d19}, [r0]
+; CHECK-NEXT:    vzip.32 q9, q8
+; CHECK-NEXT:    vadd.i32 q8, q9, q8
+; CHECK-NEXT:    vmov r0, r1, d16
+; CHECK-NEXT:    vmov r2, r3, d17
+; CHECK-NEXT:    mov pc, lr
 	%tmp1 = load <4 x i32>, <4 x i32>* %A
 	%tmp2 = load <4 x i32>, <4 x i32>* %B
 	%tmp3 = shufflevector <4 x i32> %tmp1, <4 x i32> %tmp2, <4 x i32> <i32 0, i32 4, i32 1, i32 5>
@@ -62,10 +150,31 @@ define <4 x i32> @vzipQi32(<4 x i32>* %A, <4 x i32>* %B) nounwind {
 	ret <4 x i32> %tmp5
 }
 
+define <8 x i32> @vzipQi32_QQres(<4 x i32>* %A, <4 x i32>* %B) nounwind {
+; CHECK-LABEL: vzipQi32_QQres:
+; CHECK:       @ BB#0:
+; CHECK-NEXT:    vld1.64 {d16, d17}, [r2]
+; CHECK-NEXT:    vld1.64 {d18, d19}, [r1]
+; CHECK-NEXT:    vzip.32 q9, q8
+; CHECK-NEXT:    vst1.32 {d18, d19}, [r0:128]!
+; CHECK-NEXT:    vst1.64 {d16, d17}, [r0:128]
+; CHECK-NEXT:    mov pc, lr
+	%tmp1 = load <4 x i32>, <4 x i32>* %A
+	%tmp2 = load <4 x i32>, <4 x i32>* %B
+	%tmp3 = shufflevector <4 x i32> %tmp1, <4 x i32> %tmp2, <8 x i32> <i32 0, i32 4, i32 1, i32 5, i32 2, i32 6, i32 3, i32 7>
+	ret <8 x i32> %tmp3
+}
+
 define <4 x float> @vzipQf(<4 x float>* %A, <4 x float>* %B) nounwind {
-;CHECK-LABEL: vzipQf:
-;CHECK: vzip.32
-;CHECK-NEXT: vadd.f32
+; CHECK-LABEL: vzipQf:
+; CHECK:       @ BB#0:
+; CHECK-NEXT:    vld1.64 {d16, d17}, [r1]
+; CHECK-NEXT:    vld1.64 {d18, d19}, [r0]
+; CHECK-NEXT:    vzip.32 q9, q8
+; CHECK-NEXT:    vadd.f32 q8, q9, q8
+; CHECK-NEXT:    vmov r0, r1, d16
+; CHECK-NEXT:    vmov r2, r3, d17
+; CHECK-NEXT:    mov pc, lr
 	%tmp1 = load <4 x float>, <4 x float>* %A
 	%tmp2 = load <4 x float>, <4 x float>* %B
 	%tmp3 = shufflevector <4 x float> %tmp1, <4 x float> %tmp2, <4 x i32> <i32 0, i32 4, i32 1, i32 5>
@@ -74,12 +183,32 @@ define <4 x float> @vzipQf(<4 x float>* %A, <4 x float>* %B) nounwind {
 	ret <4 x float> %tmp5
 }
 
+define <8 x float> @vzipQf_QQres(<4 x float>* %A, <4 x float>* %B) nounwind {
+; CHECK-LABEL: vzipQf_QQres:
+; CHECK:       @ BB#0:
+; CHECK-NEXT:    vld1.64 {d16, d17}, [r2]
+; CHECK-NEXT:    vld1.64 {d18, d19}, [r1]
+; CHECK-NEXT:    vzip.32 q9, q8
+; CHECK-NEXT:    vst1.32 {d18, d19}, [r0:128]!
+; CHECK-NEXT:    vst1.64 {d16, d17}, [r0:128]
+; CHECK-NEXT:    mov pc, lr
+	%tmp1 = load <4 x float>, <4 x float>* %A
+	%tmp2 = load <4 x float>, <4 x float>* %B
+	%tmp3 = shufflevector <4 x float> %tmp1, <4 x float> %tmp2, <8 x i32> <i32 0, i32 4, i32 1, i32 5, i32 2, i32 6, i32 3, i32 7>
+	ret <8 x float> %tmp3
+}
+
 ; Undef shuffle indices should not prevent matching to VZIP:
 
 define <8 x i8> @vzipi8_undef(<8 x i8>* %A, <8 x i8>* %B) nounwind {
-;CHECK-LABEL: vzipi8_undef:
-;CHECK: vzip.8
-;CHECK-NEXT: vadd.i8
+; CHECK-LABEL: vzipi8_undef:
+; CHECK:       @ BB#0:
+; CHECK-NEXT:    vldr d16, [r1]
+; CHECK-NEXT:    vldr d17, [r0]
+; CHECK-NEXT:    vzip.8 d17, d16
+; CHECK-NEXT:    vadd.i8 d16, d17, d16
+; CHECK-NEXT:    vmov r0, r1, d16
+; CHECK-NEXT:    mov pc, lr
 	%tmp1 = load <8 x i8>, <8 x i8>* %A
 	%tmp2 = load <8 x i8>, <8 x i8>* %B
 	%tmp3 = shufflevector <8 x i8> %tmp1, <8 x i8> %tmp2, <8 x i32> <i32 0, i32 undef, i32 1, i32 9, i32 undef, i32 10, i32 3, i32 11>
@@ -88,10 +217,31 @@ define <8 x i8> @vzipi8_undef(<8 x i8>* %A, <8 x i8>* %B) nounwind {
 	ret <8 x i8> %tmp5
 }
 
+define <16 x i8> @vzipi8_undef_Qres(<8 x i8>* %A, <8 x i8>* %B) nounwind {
+; CHECK-LABEL: vzipi8_undef_Qres:
+; CHECK:       @ BB#0:
+; CHECK-NEXT:    vldr d17, [r1]
+; CHECK-NEXT:    vldr d16, [r0]
+; CHECK-NEXT:    vzip.8 d16, d17
+; CHECK-NEXT:    vmov r0, r1, d16
+; CHECK-NEXT:    vmov r2, r3, d17
+; CHECK-NEXT:    mov pc, lr
+	%tmp1 = load <8 x i8>, <8 x i8>* %A
+	%tmp2 = load <8 x i8>, <8 x i8>* %B
+	%tmp3 = shufflevector <8 x i8> %tmp1, <8 x i8> %tmp2, <16 x i32> <i32 0, i32 undef, i32 1, i32 9, i32 undef, i32 10, i32 3, i32 11, i32 4, i32 12, i32 5, i32 13, i32 6, i32 undef, i32 undef, i32 15>
+	ret <16 x i8> %tmp3
+}
+
 define <16 x i8> @vzipQi8_undef(<16 x i8>* %A, <16 x i8>* %B) nounwind {
-;CHECK-LABEL: vzipQi8_undef:
-;CHECK: vzip.8
-;CHECK-NEXT: vadd.i8
+; CHECK-LABEL: vzipQi8_undef:
+; CHECK:       @ BB#0:
+; CHECK-NEXT:    vld1.64 {d16, d17}, [r1]
+; CHECK-NEXT:    vld1.64 {d18, d19}, [r0]
+; CHECK-NEXT:    vzip.8 q9, q8
+; CHECK-NEXT:    vadd.i8 q8, q9, q8
+; CHECK-NEXT:    vmov r0, r1, d16
+; CHECK-NEXT:    vmov r2, r3, d17
+; CHECK-NEXT:    mov pc, lr
 	%tmp1 = load <16 x i8>, <16 x i8>* %A
 	%tmp2 = load <16 x i8>, <16 x i8>* %B
 	%tmp3 = shufflevector <16 x i8> %tmp1, <16 x i8> %tmp2, <16 x i32> <i32 0, i32 16, i32 1, i32 undef, i32 undef, i32 undef, i32 3, i32 19, i32 4, i32 20, i32 5, i32 21, i32 6, i32 22, i32 7, i32 23>
@@ -100,3 +250,48 @@ define <16 x i8> @vzipQi8_undef(<16 x i8>* %A, <16 x i8>* %B) nounwind {
 	ret <16 x i8> %tmp5
 }
 
+define <32 x i8> @vzipQi8_undef_QQres(<16 x i8>* %A, <16 x i8>* %B) nounwind {
+; CHECK-LABEL: vzipQi8_undef_QQres:
+; CHECK:       @ BB#0:
+; CHECK-NEXT:    vld1.64 {d16, d17}, [r2]
+; CHECK-NEXT:    vld1.64 {d18, d19}, [r1]
+; CHECK-NEXT:    vzip.8 q9, q8
+; CHECK-NEXT:    vst1.8 {d18, d19}, [r0:128]!
+; CHECK-NEXT:    vst1.64 {d16, d17}, [r0:128]
+; CHECK-NEXT:    mov pc, lr
+	%tmp1 = load <16 x i8>, <16 x i8>* %A
+	%tmp2 = load <16 x i8>, <16 x i8>* %B
+	%tmp3 = shufflevector <16 x i8> %tmp1, <16 x i8> %tmp2, <32 x i32> <i32 0, i32 16, i32 1, i32 undef, i32 undef, i32 undef, i32 3, i32 19, i32 4, i32 20, i32 5, i32 21, i32 6, i32 22, i32 7, i32 23, i32 8, i32 24, i32 9, i32 undef, i32 10, i32 26, i32 11, i32 27, i32 12, i32 28, i32 13, i32 undef, i32 14, i32 30, i32 undef, i32 31>
+	ret <32 x i8> %tmp3
+}
+
+define <8 x i16> @vzip_lower_shufflemask_undef(<4 x i16>* %A, <4 x i16>* %B) {
+entry:
+  ; CHECK-LABEL: vzip_lower_shufflemask_undef
+  ; CHECK: vzip
+	%tmp1 = load <4 x i16>, <4 x i16>* %A
+	%tmp2 = load <4 x i16>, <4 x i16>* %B
+  %0 = shufflevector <4 x i16> %tmp1, <4 x i16> %tmp2, <8 x i32> <i32 undef, i32 undef, i32 undef, i32 undef, i32 2, i32 6, i32 3, i32 7>
+  ret <8 x i16> %0
+}
+
+define <4 x i32> @vzip_lower_shufflemask_zeroed(<2 x i32>* %A) {
+entry:
+  ; CHECK-LABEL: vzip_lower_shufflemask_zeroed
+  ; CHECK-NOT: vtrn
+  ; CHECK: vzip
+  %tmp1 = load <2 x i32>, <2 x i32>* %A
+  %0 = shufflevector <2 x i32> %tmp1, <2 x i32> %tmp1, <4 x i32> <i32 0, i32 0, i32 1, i32 0>
+  ret <4 x i32> %0
+}
+
+define <4 x i32> @vzip_lower_shufflemask_vuzp(<2 x i32>* %A) {
+entry:
+  ; CHECK-LABEL: vzip_lower_shufflemask_vuzp
+  ; CHECK-NOT: vuzp
+  ; CHECK: vzip
+  %tmp1 = load <2 x i32>, <2 x i32>* %A
+  %0 = shufflevector <2 x i32> %tmp1, <2 x i32> %tmp1, <4 x i32> <i32 0, i32 2, i32 1, i32 0>
+  ret <4 x i32> %0
+}
+
diff --git a/test/CodeGen/ARM/wrong-t2stmia-size-opt.ll b/test/CodeGen/ARM/wrong-t2stmia-size-opt.ll
index 4b274d2aedc..fe335df7a1a 100644
--- a/test/CodeGen/ARM/wrong-t2stmia-size-opt.ll
+++ b/test/CodeGen/ARM/wrong-t2stmia-size-opt.ll
@@ -1,20 +1,24 @@
-; RUN: llc -mcpu=cortex-a9 -O1 -filetype=obj %s -o - | llvm-objdump -arch thumb -mcpu=cortex-a9 -d - | FileCheck %s
+; RUN: llc -mtriple=thumbv7-- -mcpu=cortex-a9 -O1 -filetype=obj %s -o - | llvm-objdump -triple=thumbv7-- -mcpu=cortex-a9 -d - | FileCheck %s
 
 target datalayout = "e-m:e-p:32:32-i1:8:32-i8:8:32-i16:16:32-i64:64-v128:64:128-a:0:32-n32-S64"
 target triple = "thumbv7--linux-gnueabi"
 
 declare i8* @llvm.returnaddress(i32)
 
-define i32* @wrong-t2stmia-size-reduction(i32* %addr, i32 %val0) minsize {
+define i32* @wrong-t2stmia-size-reduction(i32* %addr, i32 %val0, i32 %val1) minsize {
   store i32 %val0, i32* %addr
   %addr1 = getelementptr i32, i32* %addr, i32 1
+  %addr2 = getelementptr i32, i32* %addr, i32 2
   %lr = call i8* @llvm.returnaddress(i32 0)
   %lr32 = ptrtoint i8* %lr to i32
-  store i32 %lr32, i32* %addr1
-  %addr2 = getelementptr i32, i32* %addr1, i32 1
-  ret i32* %addr2
+  store i32 %val1, i32* %addr1
+  store i32 %lr32, i32* %addr2
+
+  %addr3 = getelementptr i32, i32* %addr, i32 3
+  ret i32* %addr3
 }
 
-; Check that stm writes two registers.  The bug caused one of registers (LR,
+; Check that stm writes three registers.  The bug caused one of registers (LR,
 ; which invalid for Thumb1 form of STMIA instruction) to be dropped.
-; CHECK: stm{{[^,]*}}, {{{.*,.*}}}
+; CHECK-LABEL: wrong-t2stmia-size-reduction:
+; CHECK: stm{{[^,]*}}, {{{.*,.*,.*}}}
diff --git a/test/CodeGen/BPF/alu8.ll b/test/CodeGen/BPF/alu8.ll
index 0233225f81b..c1c2bd29f24 100644
--- a/test/CodeGen/BPF/alu8.ll
+++ b/test/CodeGen/BPF/alu8.ll
@@ -1,5 +1,4 @@
-; RUN: llc -march=bpf -show-mc-encoding < %s | FileCheck %s
-; test little endian only for now
+; RUN: llc -march=bpfel -show-mc-encoding < %s | FileCheck %s
 
 define i8 @mov(i8 %a, i8 %b) nounwind {
 ; CHECK-LABEL: mov:
diff --git a/test/CodeGen/BPF/atomics.ll b/test/CodeGen/BPF/atomics.ll
index 2f9730dddde..a2e17d29196 100644
--- a/test/CodeGen/BPF/atomics.ll
+++ b/test/CodeGen/BPF/atomics.ll
@@ -1,5 +1,4 @@
-; RUN: llc < %s -march=bpf -verify-machineinstrs -show-mc-encoding | FileCheck %s
-; test little endian only for now
+; RUN: llc < %s -march=bpfel -verify-machineinstrs -show-mc-encoding | FileCheck %s
 
 ; CHECK-LABEL: test_load_add_32
 ; CHECK: xadd32
diff --git a/test/CodeGen/BPF/basictest.ll b/test/CodeGen/BPF/basictest.ll
index 2a2d49878a6..82feb43d005 100644
--- a/test/CodeGen/BPF/basictest.ll
+++ b/test/CodeGen/BPF/basictest.ll
@@ -1,4 +1,4 @@
-; RUN: llc < %s -march=bpf | FileCheck %s
+; RUN: llc < %s -march=bpfel | FileCheck %s
 
 define i32 @test0(i32 %X) {
   %tmp.1 = add i32 %X, 1
diff --git a/test/CodeGen/BPF/cc_args.ll b/test/CodeGen/BPF/cc_args.ll
index 5085fe5684e..8e3f8604ac8 100644
--- a/test/CodeGen/BPF/cc_args.ll
+++ b/test/CodeGen/BPF/cc_args.ll
@@ -1,5 +1,4 @@
-; RUN: llc < %s -march=bpf -show-mc-encoding | FileCheck %s
-; test little endian only for now
+; RUN: llc < %s -march=bpfel -show-mc-encoding | FileCheck %s
 
 define void @test() #0 {
 entry:
diff --git a/test/CodeGen/BPF/cc_args_be.ll b/test/CodeGen/BPF/cc_args_be.ll
new file mode 100644
index 00000000000..59a7439728b
--- /dev/null
+++ b/test/CodeGen/BPF/cc_args_be.ll
@@ -0,0 +1,96 @@
+; RUN: llc < %s -march=bpfeb -show-mc-encoding | FileCheck %s
+; test big endian
+
+define void @test() #0 {
+entry:
+; CHECK: test:
+
+; CHECK: mov  r1, 123 # encoding: [0xb7,0x10,0x00,0x00,0x00,0x00,0x00,0x7b]
+; CHECK: call f_i16
+  call void @f_i16(i16 123)
+
+; CHECK: mov  r1, 12345678 # encoding: [0xb7,0x10,0x00,0x00,0x00,0xbc,0x61,0x4e]
+; CHECK: call f_i32
+  call void @f_i32(i32 12345678)
+
+; CHECK: ld_64 r1, 72623859790382856 # encoding: [0x18,0x10,0x00,0x00,0x05,0x06,0x07,0x08,0x00,0x00,0x00,0x00,0x01,0x02,0x03,0x04]
+; CHECK: call f_i64
+  call void @f_i64(i64 72623859790382856)
+
+; CHECK: mov  r1, 1234
+; CHECK: mov  r2, 5678
+; CHECK: call f_i32_i32
+  call void @f_i32_i32(i32 1234, i32 5678)
+
+; CHECK: mov  r1, 2
+; CHECK: mov  r2, 3
+; CHECK: mov  r3, 4
+; CHECK: call f_i16_i32_i16
+  call void @f_i16_i32_i16(i16 2, i32 3, i16 4)
+
+; CHECK: mov  r1, 5
+; CHECK: ld_64 r2, 7262385979038285
+; CHECK: mov  r3, 6
+; CHECK: call f_i16_i64_i16
+  call void @f_i16_i64_i16(i16 5, i64 7262385979038285, i16 6)
+
+  ret void
+}
+
+@g_i16 = common global i16 0, align 2
+@g_i32 = common global i32 0, align 2
+@g_i64 = common global i64 0, align 4
+
+define void @f_i16(i16 %a) #0 {
+; CHECK: f_i16:
+; CHECK: sth 0(r2), r1 # encoding: [0x6b,0x21,0x00,0x00,0x00,0x00,0x00,0x00]
+  store volatile i16 %a, i16* @g_i16, align 2
+  ret void
+}
+
+define void @f_i32(i32 %a) #0 {
+; CHECK: f_i32:
+; CHECK: sth 2(r2), r1 # encoding: [0x6b,0x21,0x00,0x02,0x00,0x00,0x00,0x00]
+; CHECK: sth 0(r2), r1 # encoding: [0x6b,0x21,0x00,0x00,0x00,0x00,0x00,0x00]
+  store volatile i32 %a, i32* @g_i32, align 2
+  ret void
+}
+
+define void @f_i64(i64 %a) #0 {
+; CHECK: f_i64:
+; CHECK: stw 4(r2), r1 # encoding: [0x63,0x21,0x00,0x04,0x00,0x00,0x00,0x00]
+; CHECK: stw 0(r2), r1
+  store volatile i64 %a, i64* @g_i64, align 2
+  ret void
+}
+
+define void @f_i32_i32(i32 %a, i32 %b) #0 {
+; CHECK: f_i32_i32:
+; CHECK: stw 0(r3), r1
+  store volatile i32 %a, i32* @g_i32, align 4
+; CHECK: stw 0(r3), r2
+  store volatile i32 %b, i32* @g_i32, align 4
+  ret void
+}
+
+define void @f_i16_i32_i16(i16 %a, i32 %b, i16 %c) #0 {
+; CHECK: f_i16_i32_i16:
+; CHECK: sth 0(r4), r1
+  store volatile i16 %a, i16* @g_i16, align 2
+; CHECK: stw 0(r1), r2
+  store volatile i32 %b, i32* @g_i32, align 4
+; CHECK: sth 0(r4), r3
+  store volatile i16 %c, i16* @g_i16, align 2
+  ret void
+}
+
+define void @f_i16_i64_i16(i16 %a, i64 %b, i16 %c) #0 {
+; CHECK: f_i16_i64_i16:
+; CHECK: sth 0(r4), r1
+  store volatile i16 %a, i16* @g_i16, align 2
+; CHECK: std 0(r1), r2 # encoding: [0x7b,0x12,0x00,0x00,0x00,0x00,0x00,0x00]
+  store volatile i64 %b, i64* @g_i64, align 8
+; CHECK: sth 0(r4), r3
+  store volatile i16 %c, i16* @g_i16, align 2
+  ret void
+}
diff --git a/test/CodeGen/BPF/cc_ret.ll b/test/CodeGen/BPF/cc_ret.ll
index e32b17bcc61..09574922f32 100644
--- a/test/CodeGen/BPF/cc_ret.ll
+++ b/test/CodeGen/BPF/cc_ret.ll
@@ -1,4 +1,4 @@
-; RUN: llc < %s -march=bpf | FileCheck %s
+; RUN: llc < %s -march=bpfel | FileCheck %s
 
 define void @test() #0 {
 entry:
diff --git a/test/CodeGen/BPF/ex1.ll b/test/CodeGen/BPF/ex1.ll
index be038e9a3d8..546e5d49da6 100644
--- a/test/CodeGen/BPF/ex1.ll
+++ b/test/CodeGen/BPF/ex1.ll
@@ -1,4 +1,4 @@
-; RUN: llc < %s -march=bpf | FileCheck %s
+; RUN: llc < %s -march=bpfel | FileCheck %s
 
 %struct.bpf_context = type { i64, i64, i64, i64, i64, i64, i64 }
 %struct.sk_buff = type { i64, i64, i64, i64, i64, i64, i64 }
diff --git a/test/CodeGen/BPF/intrinsics.ll b/test/CodeGen/BPF/intrinsics.ll
index 98b57deb7c8..483473e922f 100644
--- a/test/CodeGen/BPF/intrinsics.ll
+++ b/test/CodeGen/BPF/intrinsics.ll
@@ -1,4 +1,4 @@
-; RUN: llc < %s -march=bpf -show-mc-encoding | FileCheck %s
+; RUN: llc < %s -march=bpfel -show-mc-encoding | FileCheck %s
 
 ; Function Attrs: nounwind uwtable
 define i32 @ld_b(i64 %foo, i64* nocapture %bar, i8* %ctx, i8* %ctx2) #0 {
diff --git a/test/CodeGen/BPF/load.ll b/test/CodeGen/BPF/load.ll
index 03fb17c965b..d4ba315b5f1 100644
--- a/test/CodeGen/BPF/load.ll
+++ b/test/CodeGen/BPF/load.ll
@@ -1,4 +1,4 @@
-; RUN: llc < %s -march=bpf | FileCheck %s
+; RUN: llc < %s -march=bpfel | FileCheck %s
 
 define i16 @am1(i16* %a) nounwind {
   %1 = load i16, i16* %a
diff --git a/test/CodeGen/BPF/loops.ll b/test/CodeGen/BPF/loops.ll
index 4798d78842c..00be54b3bac 100644
--- a/test/CodeGen/BPF/loops.ll
+++ b/test/CodeGen/BPF/loops.ll
@@ -1,4 +1,4 @@
-; RUN: llc < %s -march=bpf | FileCheck %s
+; RUN: llc < %s -march=bpfel | FileCheck %s
 
 define zeroext i16 @add(i16* nocapture %a, i16 zeroext %n) nounwind readonly {
 entry:
diff --git a/test/CodeGen/BPF/sanity.ll b/test/CodeGen/BPF/sanity.ll
index 09a6b65d085..7f0ef889ff9 100644
--- a/test/CodeGen/BPF/sanity.ll
+++ b/test/CodeGen/BPF/sanity.ll
@@ -1,4 +1,4 @@
-; RUN: llc < %s -march=bpf | FileCheck %s
+; RUN: llc < %s -march=bpfel | FileCheck %s
 
 @foo_printf.fmt = private unnamed_addr constant [9 x i8] c"hello  \0A\00", align 1
 
diff --git a/test/CodeGen/BPF/setcc.ll b/test/CodeGen/BPF/setcc.ll
index eabb6c9bf2d..f6c6db6c683 100644
--- a/test/CodeGen/BPF/setcc.ll
+++ b/test/CodeGen/BPF/setcc.ll
@@ -1,4 +1,4 @@
-; RUN: llc -march=bpf < %s | FileCheck %s
+; RUN: llc -march=bpfel < %s | FileCheck %s
 
 define i16 @sccweqand(i16 %a, i16 %b) nounwind {
   %t1 = and i16 %a, %b
diff --git a/test/CodeGen/BPF/shifts.ll b/test/CodeGen/BPF/shifts.ll
index 898ae2d4612..cb000b92fcd 100644
--- a/test/CodeGen/BPF/shifts.ll
+++ b/test/CodeGen/BPF/shifts.ll
@@ -1,5 +1,4 @@
-; RUN: llc < %s -march=bpf -show-mc-encoding | FileCheck %s
-; test little endian only for now
+; RUN: llc < %s -march=bpfel -show-mc-encoding | FileCheck %s
 
 define zeroext i8 @lshr8(i8 zeroext %a, i8 zeroext %cnt) nounwind readnone {
 entry:
diff --git a/test/CodeGen/BPF/sockex2.ll b/test/CodeGen/BPF/sockex2.ll
index 6ae5e1c8d6b..d372a5982f6 100644
--- a/test/CodeGen/BPF/sockex2.ll
+++ b/test/CodeGen/BPF/sockex2.ll
@@ -1,5 +1,4 @@
-; RUN: llc < %s -march=bpf -show-mc-encoding | FileCheck %s
-; test little endian only for now
+; RUN: llc < %s -march=bpfel -show-mc-encoding | FileCheck %s
 
 %struct.bpf_map_def = type { i32, i32, i32, i32 }
 %struct.sk_buff = type opaque
diff --git a/test/CodeGen/Generic/2007-02-25-invoke.ll b/test/CodeGen/Generic/2007-02-25-invoke.ll
index 7850cec35f9..4ca280d1587 100644
--- a/test/CodeGen/Generic/2007-02-25-invoke.ll
+++ b/test/CodeGen/Generic/2007-02-25-invoke.ll
@@ -3,12 +3,12 @@
 ; PR1224
 
 declare i32 @test()
-define i32 @test2() {
+define i32 @test2() personality i32 (...)* @__gxx_personality_v0 {
         %A = invoke i32 @test() to label %invcont unwind label %blat
 invcont:
         ret i32 %A
 blat:
-  %lpad = landingpad { i8*, i32 } personality i32 (...)* @__gxx_personality_v0
+  %lpad = landingpad { i8*, i32 }
             cleanup
   ret i32 0
 }
diff --git a/test/CodeGen/Generic/2007-04-30-LandingPadBranchFolding.ll b/test/CodeGen/Generic/2007-04-30-LandingPadBranchFolding.ll
index 407696f4c2f..f0259ec6b12 100644
--- a/test/CodeGen/Generic/2007-04-30-LandingPadBranchFolding.ll
+++ b/test/CodeGen/Generic/2007-04-30-LandingPadBranchFolding.ll
@@ -7,7 +7,7 @@
 	%"struct.std::locale::facet" = type { i32 (...)**, i32 }
 	%"struct.std::string" = type { %"struct.std::basic_string<char,std::char_traits<char>,std::allocator<char> >::_Alloc_hider" }
 
-define void @_ZNKSt6locale4nameEv(%"struct.std::string"* %agg.result) {
+define void @_ZNKSt6locale4nameEv(%"struct.std::string"* %agg.result) personality i32 (...)* @__gxx_personality_v0 {
 entry:
 	%tmp105 = icmp eq i8* null, null		; <i1> [#uses=1]
 	br i1 %tmp105, label %cond_true, label %cond_true222
@@ -45,7 +45,7 @@ cond_next1328:		; preds = %cond_true235, %cond_true
 	ret void
 
 cond_true1402:		; preds = %invcont282, %cond_false280, %cond_true235, %cond_true
-  %lpad = landingpad { i8*, i32 } personality i32 (...)* @__gxx_personality_v0
+  %lpad = landingpad { i8*, i32 }
             cleanup
   ret void
 }
diff --git a/test/CodeGen/Generic/2007-12-17-InvokeAsm.ll b/test/CodeGen/Generic/2007-12-17-InvokeAsm.ll
index 03ccbdfaf0c..fe7f463159a 100644
--- a/test/CodeGen/Generic/2007-12-17-InvokeAsm.ll
+++ b/test/CodeGen/Generic/2007-12-17-InvokeAsm.ll
@@ -1,6 +1,6 @@
 ; RUN: llc -no-integrated-as < %s
 
-define fastcc void @bc__support__high_resolution_time__initialize_clock_rate() {
+define fastcc void @bc__support__high_resolution_time__initialize_clock_rate() personality i32 (...)* @__gxx_personality_v0 {
 entry:
   invoke void asm "rdtsc\0A\09movl %eax, $0\0A\09movl %edx, $1", "=*imr,=*imr,~{dirflag},~{fpsr},~{flags},~{dx},~{ax}"( i32* null, i32* null )
       to label %.noexc unwind label %cleanup144
@@ -9,7 +9,7 @@ entry:
   ret void
 
 cleanup144:		; preds = %entry
-  %exn = landingpad {i8*, i32} personality i32 (...)* @__gxx_personality_v0
+  %exn = landingpad {i8*, i32}
             cleanup
   resume { i8*, i32 } %exn
 }
diff --git a/test/CodeGen/Generic/2007-12-31-UnusedSelector.ll b/test/CodeGen/Generic/2007-12-31-UnusedSelector.ll
index d67559e4d85..59a7b64e0df 100644
--- a/test/CodeGen/Generic/2007-12-31-UnusedSelector.ll
+++ b/test/CodeGen/Generic/2007-12-31-UnusedSelector.ll
@@ -5,7 +5,7 @@
 	%struct.__type_info_pseudo = type { i8*, i8* }
 @_ZTI2e1 = external constant %struct.__class_type_info_pseudo		; <%struct.__class_type_info_pseudo*> [#uses=1]
 
-define void @_Z7ex_testv() {
+define void @_Z7ex_testv() personality i32 (...)* @__gxx_personality_v0 {
 entry:
 	invoke void @__cxa_throw( i8* null, i8* bitcast (%struct.__class_type_info_pseudo* @_ZTI2e1 to i8*), void (i8*)* null ) noreturn 
 			to label %UnifiedUnreachableBlock unwind label %lpad
@@ -14,13 +14,13 @@ bb14:		; preds = %lpad
 	unreachable
 
 lpad:		; preds = %entry
-        %lpad1 = landingpad { i8*, i32 } personality i32 (...)* @__gxx_personality_v0
+        %lpad1 = landingpad { i8*, i32 }
                   catch i8* null
 	invoke void @__cxa_end_catch( )
 			to label %bb14 unwind label %lpad17
 
 lpad17:		; preds = %lpad
-        %lpad2 = landingpad { i8*, i32 } personality i32 (...)* @__gxx_personality_v0
+        %lpad2 = landingpad { i8*, i32 }
                   catch i8* null
 	unreachable
 
diff --git a/test/CodeGen/Generic/2009-03-29-SoftFloatVectorExtract.ll b/test/CodeGen/Generic/2009-03-29-SoftFloatVectorExtract.ll
index c18e3c9009a..f614db00da4 100644
--- a/test/CodeGen/Generic/2009-03-29-SoftFloatVectorExtract.ll
+++ b/test/CodeGen/Generic/2009-03-29-SoftFloatVectorExtract.ll
@@ -1,10 +1,14 @@
-; RUN: llc < %s -soft-float
+; RUN: llc < %s
 ; PR3899
 
 @m = external global <2 x double>
 
-define double @vector_ex() nounwind {
+define double @vector_ex() nounwind #0 {
        %v = load <2 x double>, <2 x double>* @m
        %x = extractelement <2 x double> %v, i32 1
        ret double %x
 }
+
+; Soft-float attribute so that targets that pay attention to soft float will
+; make floating point types illegal and we'll exercise the legalizer code.
+attributes #0 = { "use-soft-float" = "true" }
diff --git a/test/CodeGen/Generic/2009-11-16-BadKillsCrash.ll b/test/CodeGen/Generic/2009-11-16-BadKillsCrash.ll
index b483009976c..a0455cfe1aa 100644
--- a/test/CodeGen/Generic/2009-11-16-BadKillsCrash.ll
+++ b/test/CodeGen/Generic/2009-11-16-BadKillsCrash.ll
@@ -19,7 +19,7 @@ declare i8* @__cxa_begin_catch(i8*) nounwind
 
 declare %"struct.std::ctype<char>"* @_ZSt9use_facetISt5ctypeIcEERKT_RKSt6locale(%"struct.std::locale"*)
 
-define %"struct.std::basic_istream<char,std::char_traits<char> >"* @_ZStrsIcSt11char_traitsIcEERSt13basic_istreamIT_T0_ES6_PS3_(%"struct.std::basic_istream<char,std::char_traits<char> >"* %__in, i8* nocapture %__s) {
+define %"struct.std::basic_istream<char,std::char_traits<char> >"* @_ZStrsIcSt11char_traitsIcEERSt13basic_istreamIT_T0_ES6_PS3_(%"struct.std::basic_istream<char,std::char_traits<char> >"* %__in, i8* nocapture %__s) personality i32 (...)* @__gxx_personality_v0 {
 entry:
   %0 = invoke %"struct.std::ctype<char>"* @_ZSt9use_facetISt5ctypeIcEERKT_RKSt6locale(%"struct.std::locale"* undef)
           to label %invcont8 unwind label %lpad74 ; <%"struct.std::ctype<char>"*> [#uses=0]
@@ -62,14 +62,14 @@ invcont38:                                        ; preds = %invcont25, %bb1.i,
 
 lpad:                                             ; preds = %bb.i93, %invcont24, %bb1.i, %invcont8
   %__extracted.1 = phi i32 [ 0, %invcont8 ], [ %2, %bb1.i ], [ undef, %bb.i93 ], [ undef, %invcont24 ] ; <i32> [#uses=0]
-  %lpad1 = landingpad { i8*, i32 } personality i32 (...)* @__gxx_personality_v0
+  %lpad1 = landingpad { i8*, i32 }
             catch i8* null
   %eh_ptr = extractvalue { i8*, i32 } %lpad1, 0
   %6 = call i8* @__cxa_begin_catch(i8* %eh_ptr) nounwind ; <i8*> [#uses=0]
   unreachable
 
 lpad74:                                           ; preds = %entry
-  %lpad2 = landingpad { i8*, i32 } personality i32 (...)* @__gxx_personality_v0
+  %lpad2 = landingpad { i8*, i32 }
             cleanup
   unreachable
 }
diff --git a/test/CodeGen/Generic/ForceStackAlign.ll b/test/CodeGen/Generic/ForceStackAlign.ll
new file mode 100644
index 00000000000..f798fca06d7
--- /dev/null
+++ b/test/CodeGen/Generic/ForceStackAlign.ll
@@ -0,0 +1,27 @@
+; Check that stack alignment can be forced. Individual targets should test their
+; specific implementation details.
+
+; RUN: llc < %s -force-align-stack -stack-alignment=32 | FileCheck %s
+; CHECK-LABEL: @f
+; CHECK-LABEL: @g
+
+define i32 @f(i8* %p) nounwind {
+entry:
+  %0 = load i8, i8* %p
+  %conv = sext i8 %0 to i32
+  ret i32 %conv
+}
+
+define i64 @g(i32 %i) nounwind {
+entry:
+  br label %if.then
+
+if.then:
+  %0 = alloca i8, i32 %i
+  call void @llvm.memset.p0i8.i32(i8* %0, i8 0, i32 %i, i32 1, i1 false)
+  %call = call i32 @f(i8* %0)
+  %conv = sext i32 %call to i64
+  ret i64 %conv
+}
+
+declare void @llvm.memset.p0i8.i32(i8*, i8, i32, i32, i1) nounwind
diff --git a/test/CodeGen/Generic/MachineBranchProb.ll b/test/CodeGen/Generic/MachineBranchProb.ll
index 83277c98989..8288e45ee50 100644
--- a/test/CodeGen/Generic/MachineBranchProb.ll
+++ b/test/CodeGen/Generic/MachineBranchProb.ll
@@ -1,11 +1,12 @@
 ; RUN: llc < %s -print-machineinstrs=expand-isel-pseudos -o /dev/null 2>&1 | FileCheck %s
 
 ; ARM & AArch64 run an extra SimplifyCFG which disrupts this test.
-; XFAIL: arm,aarch64
+; Hexagon crashes (PR23377)
+; XFAIL: arm,aarch64,hexagon
 
 ; Make sure we have the correct weight attached to each successor.
 define i32 @test2(i32 %x) nounwind uwtable readnone ssp {
-; CHECK: Machine code for function test2:
+; CHECK-LABEL: Machine code for function test2:
 entry:
   %conv = sext i32 %x to i64
   switch i64 %conv, label %return [
@@ -33,3 +34,41 @@ return:
 }
 
 !0 = !{!"branch_weights", i32 7, i32 6, i32 4, i32 4, i32 64}
+
+
+declare void @g(i32)
+define void @left_leaning_weight_balanced_tree(i32 %x) {
+entry:
+  switch i32 %x, label %return [
+    i32 0,  label %bb0
+    i32 10, label %bb1
+    i32 20, label %bb2
+    i32 30, label %bb3
+    i32 40, label %bb4
+    i32 50, label %bb5
+  ], !prof !1
+bb0: tail call void @g(i32 0) br label %return
+bb1: tail call void @g(i32 1) br label %return
+bb2: tail call void @g(i32 2) br label %return
+bb3: tail call void @g(i32 3) br label %return
+bb4: tail call void @g(i32 4) br label %return
+bb5: tail call void @g(i32 5) br label %return
+return: ret void
+
+; Check that we set branch weights on the pivot cmp instruction correctly.
+; Cases {0,10,20,30} go on the left with weight 13; cases {40,50} go on the
+; right with weight 20.
+;
+; CHECK-LABEL: Machine code for function left_leaning_weight_balanced_tree:
+; CHECK: BB#0: derived from LLVM BB %entry
+; CHECK-NOT: Successors
+; CHECK: Successors according to CFG: BB#8(13) BB#9(20)
+}
+
+!1 = !{!"branch_weights",
+  ; Default:
+  i32 1,
+  ; Case 0, 10, 20:
+  i32 10, i32 1, i32 1,
+  ; Case 30, 40, 50:
+  i32 1, i32 10, i32 10}
diff --git a/test/CodeGen/Generic/dbg_value.ll b/test/CodeGen/Generic/dbg_value.ll
index 2ee667c3e02..9fe4bf33087 100644
--- a/test/CodeGen/Generic/dbg_value.ll
+++ b/test/CodeGen/Generic/dbg_value.ll
@@ -4,11 +4,11 @@
 %0 = type { i32, i32 }
 
 define void @t(%0*, i32, i32, i32, i32) nounwind {
-  tail call void @llvm.dbg.value(metadata %0* %0, i64 0, metadata !0, metadata !MDExpression()), !dbg !MDLocation(scope: !MDSubprogram())
+  tail call void @llvm.dbg.value(metadata %0* %0, i64 0, metadata !0, metadata !DIExpression()), !dbg !DILocation(scope: !DISubprogram())
   unreachable
 }
 
 declare void @llvm.dbg.value(metadata, i64, metadata, metadata) nounwind readnone
 
 ; !0 should conform to the format of DIVariable.
-!0 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "a", arg: 0, scope: !MDSubprogram())
+!0 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "a", arg: 1, scope: !DISubprogram())
diff --git a/test/CodeGen/Generic/donothing.ll b/test/CodeGen/Generic/donothing.ll
index 3727b60a1a4..59ccf6be092 100644
--- a/test/CodeGen/Generic/donothing.ll
+++ b/test/CodeGen/Generic/donothing.ll
@@ -5,7 +5,7 @@ declare void @__cxa_call_unexpected(i8*)
 declare void @llvm.donothing() readnone
 
 ; CHECK: f1
-define void @f1() nounwind uwtable ssp {
+define void @f1() nounwind uwtable ssp personality i8* bitcast (i32 (...)* @__gxx_personality_v0 to i8*) {
 entry:
 ; CHECK-NOT: donothing
   invoke void @llvm.donothing()
@@ -15,7 +15,7 @@ invoke.cont:
   ret void
 
 lpad:
-  %0 = landingpad { i8*, i32 } personality i8* bitcast (i32 (...)* @__gxx_personality_v0 to i8*)
+  %0 = landingpad { i8*, i32 }
           filter [0 x i8*] zeroinitializer
   %1 = extractvalue { i8*, i32 } %0, 0
   tail call void @__cxa_call_unexpected(i8* %1) noreturn nounwind
diff --git a/test/CodeGen/Generic/exception-handling.ll b/test/CodeGen/Generic/exception-handling.ll
index 376e1f196d6..6e2a8678e9a 100644
--- a/test/CodeGen/Generic/exception-handling.ll
+++ b/test/CodeGen/Generic/exception-handling.ll
@@ -2,7 +2,7 @@
 ; PR10733
 declare void @_Znam()
 
-define void @_ZNK14gIndexOdometer15AfterExcisionOfERi() uwtable align 2 {
+define void @_ZNK14gIndexOdometer15AfterExcisionOfERi() uwtable align 2 personality i32 (i32, i64, i8*, i8*)* @__gxx_personality_v0 {
 _ZN6Gambit5ArrayIiEC2Ej.exit36:
   br label %"9"
 
@@ -19,7 +19,7 @@ _ZN6Gambit5ArrayIiEC2Ej.exit36:
 
 lpad27:                                           ; preds = %"10", %"9"
   %0 = phi i32 [ undef, %"9" ], [ %tmp, %"10" ]
-  %1 = landingpad { i8*, i32 } personality i32 (i32, i64, i8*, i8*)* @__gxx_personality_v0
+  %1 = landingpad { i8*, i32 }
           cleanup
   resume { i8*, i32 } zeroinitializer
 }
diff --git a/test/CodeGen/Generic/icmp-illegal.ll b/test/CodeGen/Generic/icmp-illegal.ll
new file mode 100644
index 00000000000..23d20c04652
--- /dev/null
+++ b/test/CodeGen/Generic/icmp-illegal.ll
@@ -0,0 +1,50 @@
+
+; RUN: llc < %s | FileCheck %s
+
+; CHECK-LABEL: test_ult
+define i1 @test_ult(i256 %a) nounwind {
+  %1 = icmp ult i256 %a, -6432394258550908438
+  ret i1 %1
+}
+
+; CHECK-LABEL: test_ule
+define i1 @test_ule(i256 %a) nounwind {
+  %1 = icmp ule i256 %a, -6432394258550908438
+  ret i1 %1
+}
+
+; CHECK-LABEL: test_ugt
+define i1 @test_ugt(i256 %a) nounwind {
+  %1 = icmp ugt i256 %a, -6432394258550908438
+  ret i1 %1
+}
+
+; CHECK-LABEL: test_uge
+define i1 @test_uge(i256 %a) nounwind {
+  %1 = icmp uge i256 %a, -6432394258550908438
+  ret i1 %1
+}
+
+; CHECK-LABEL: test_slt
+define i1 @test_slt(i256 %a) nounwind {
+  %1 = icmp slt i256 %a, -6432394258550908438
+  ret i1 %1
+}
+
+; CHECK-LABEL: test_sle
+define i1 @test_sle(i256 %a) nounwind {
+  %1 = icmp sle i256 %a, -6432394258550908438
+  ret i1 %1
+}
+
+; CHECK-LABEL: test_sgt
+define i1 @test_sgt(i256 %a) nounwind {
+  %1 = icmp sgt i256 %a, -6432394258550908438
+  ret i1 %1
+}
+
+; CHECK-LABEL: test_sge
+define i1 @test_sge(i256 %a) nounwind {
+  %1 = icmp sge i256 %a, -6432394258550908438
+  ret i1 %1
+}
diff --git a/test/CodeGen/Generic/multiple-return-values-cross-block-with-invoke.ll b/test/CodeGen/Generic/multiple-return-values-cross-block-with-invoke.ll
index a135c625fcc..a21906cf6dc 100644
--- a/test/CodeGen/Generic/multiple-return-values-cross-block-with-invoke.ll
+++ b/test/CodeGen/Generic/multiple-return-values-cross-block-with-invoke.ll
@@ -2,7 +2,7 @@
 ; XFAIL: hexagon
 declare { i64, double } @wild()
 
-define void @foo(i64* %p, double* %q) nounwind {
+define void @foo(i64* %p, double* %q) nounwind personality i32 (...)* @__gxx_personality_v0 {
         %t = invoke { i64, double } @wild() to label %normal unwind label %handler
 
 normal:
@@ -13,7 +13,7 @@ normal:
 	ret void
   
 handler:
-        %exn = landingpad {i8*, i32} personality i32 (...)* @__gxx_personality_v0
+        %exn = landingpad {i8*, i32}
                  catch i8* null
 	ret void
 }
diff --git a/test/CodeGen/Generic/overloaded-intrinsic-name.ll b/test/CodeGen/Generic/overloaded-intrinsic-name.ll
index 915ba9fae51..979bc772f75 100644
--- a/test/CodeGen/Generic/overloaded-intrinsic-name.ll
+++ b/test/CodeGen/Generic/overloaded-intrinsic-name.ll
@@ -13,29 +13,29 @@
 
 ; function and integer
 define i32* @test_iAny(i32* %v) gc "statepoint-example" {
-       %tok = call i32 (i1 ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_i1f(i1 ()* @return_i1, i32 0, i32 0, i32 0, i32* %v)
-       %v-new = call i32* @llvm.experimental.gc.relocate.p0i32(i32 %tok, i32 4, i32 4)
+       %tok = call i32 (i64, i32, i1 ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_i1f(i64 0, i32 0, i1 ()* @return_i1, i32 0, i32 0, i32 0, i32 0, i32* %v)
+       %v-new = call i32* @llvm.experimental.gc.relocate.p0i32(i32 %tok,  i32 7, i32 7)
        ret i32* %v-new
 }
 
 ; float
 define float* @test_fAny(float* %v) gc "statepoint-example" {
-       %tok = call i32 (i1 ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_i1f(i1 ()* @return_i1, i32 0, i32 0, i32 0, float* %v)
-       %v-new = call float* @llvm.experimental.gc.relocate.p0f32(i32 %tok, i32 4, i32 4)
+       %tok = call i32 (i64, i32, i1 ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_i1f(i64 0, i32 0, i1 ()* @return_i1, i32 0, i32 0, i32 0, i32 0, float* %v)
+       %v-new = call float* @llvm.experimental.gc.relocate.p0f32(i32 %tok,  i32 7, i32 7)
        ret float* %v-new
 }
 
 ; array of integers
 define [3 x i32]* @test_aAny([3 x i32]* %v) gc "statepoint-example" {
-       %tok = call i32 (i1 ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_i1f(i1 ()* @return_i1, i32 0, i32 0, i32 0, [3 x i32]* %v)
-       %v-new = call [3 x i32]* @llvm.experimental.gc.relocate.p0a3i32(i32 %tok, i32 4, i32 4)
+       %tok = call i32 (i64, i32, i1 ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_i1f(i64 0, i32 0, i1 ()* @return_i1, i32 0, i32 0, i32 0, i32 0, [3 x i32]* %v)
+       %v-new = call [3 x i32]* @llvm.experimental.gc.relocate.p0a3i32(i32 %tok,  i32 7, i32 7)
        ret [3 x i32]* %v-new
 }
 
 ; vector of integers
 define <3 x i32>* @test_vAny(<3 x i32>* %v) gc "statepoint-example" {
-       %tok = call i32 (i1 ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_i1f(i1 ()* @return_i1, i32 0, i32 0, i32 0, <3 x i32>* %v)
-       %v-new = call <3 x i32>* @llvm.experimental.gc.relocate.p0v3i32(i32 %tok, i32 4, i32 4)
+       %tok = call i32 (i64, i32, i1 ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_i1f(i64 0, i32 0, i1 ()* @return_i1, i32 0, i32 0, i32 0, i32 0, <3 x i32>* %v)
+       %v-new = call <3 x i32>* @llvm.experimental.gc.relocate.p0v3i32(i32 %tok,  i32 7, i32 7)
        ret <3 x i32>* %v-new
 }
 
@@ -43,13 +43,13 @@ define <3 x i32>* @test_vAny(<3 x i32>* %v) gc "statepoint-example" {
 
 ; struct
 define %struct.test* @test_struct(%struct.test* %v) gc "statepoint-example" {
-       %tok = call i32 (i1 ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_i1f(i1 ()* @return_i1, i32 0, i32 0, i32 0, %struct.test* %v)
-       %v-new = call %struct.test* @llvm.experimental.gc.relocate.p0struct.test(i32 %tok, i32 4, i32 4)
+       %tok = call i32 (i64, i32, i1 ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_i1f(i64 0, i32 0, i1 ()* @return_i1, i32 0, i32 0, i32 0, i32 0, %struct.test* %v)
+       %v-new = call %struct.test* @llvm.experimental.gc.relocate.p0struct.test(i32 %tok,  i32 7, i32 7)
        ret %struct.test* %v-new
 }
 
 declare zeroext i1 @return_i1()
-declare i32 @llvm.experimental.gc.statepoint.p0f_i1f(i1 ()*, i32, i32, ...)
+declare i32 @llvm.experimental.gc.statepoint.p0f_i1f(i64, i32, i1 ()*, i32, i32, ...)
 declare i32* @llvm.experimental.gc.relocate.p0i32(i32, i32, i32)
 declare float* @llvm.experimental.gc.relocate.p0f32(i32, i32, i32)
 declare [3 x i32]* @llvm.experimental.gc.relocate.p0a3i32(i32, i32, i32)
diff --git a/test/CodeGen/Generic/run-pass.ll b/test/CodeGen/Generic/run-pass.ll
new file mode 100644
index 00000000000..55d62ec1864
--- /dev/null
+++ b/test/CodeGen/Generic/run-pass.ll
@@ -0,0 +1,7 @@
+; RUN: llc < %s -debug-pass=Structure -run-pass=gc-lowering -o /dev/null 2>&1 | FileCheck %s
+
+; CHECK: -gc-lowering
+; CHECK: FunctionPass Manager
+; CHECK-NEXT: Lower Garbage Collection Instructions
+; CHECK-NEXT: Machine Function Analysis
+; CHECK-NEXT: MIR Printing Pass
diff --git a/test/CodeGen/Generic/stop-after.ll b/test/CodeGen/Generic/stop-after.ll
index 557e097840a..791378c3737 100644
--- a/test/CodeGen/Generic/stop-after.ll
+++ b/test/CodeGen/Generic/stop-after.ll
@@ -1,9 +1,10 @@
 ; RUN: llc < %s -debug-pass=Structure -stop-after=loop-reduce -o /dev/null 2>&1 | FileCheck %s -check-prefix=STOP
 ; RUN: llc < %s -debug-pass=Structure -start-after=loop-reduce -o /dev/null 2>&1 | FileCheck %s -check-prefix=START
 
-; STOP: -loop-reduce -print-module
+; STOP: -loop-reduce
 ; STOP: Loop Strength Reduction
 ; STOP-NEXT: Machine Function Analysis
+; STOP-NEXT: MIR Printing Pass
 
 ; START: -machine-branch-prob -gc-lowering
 ; START: FunctionPass Manager
diff --git a/test/CodeGen/Generic/vector-casts.ll b/test/CodeGen/Generic/vector-casts.ll
index fee72b6b658..0afbb8cfadc 100644
--- a/test/CodeGen/Generic/vector-casts.ll
+++ b/test/CodeGen/Generic/vector-casts.ll
@@ -2,44 +2,44 @@
 ; PR2671
 
 define void @a(<2 x double>* %p, <2 x i8>* %q) {
-        %t = load <2 x double>, <2 x double>* %p
-	%r = fptosi <2 x double> %t to <2 x i8>
-        store <2 x i8> %r, <2 x i8>* %q
-	ret void
+  %t = load <2 x double>, <2 x double>* %p
+  %r = fptosi <2 x double> %t to <2 x i8>
+  store <2 x i8> %r, <2 x i8>* %q
+  ret void
 }
 define void @b(<2 x double>* %p, <2 x i8>* %q) {
-        %t = load <2 x double>, <2 x double>* %p
-	%r = fptoui <2 x double> %t to <2 x i8>
-        store <2 x i8> %r, <2 x i8>* %q
-	ret void
+  %t = load <2 x double>, <2 x double>* %p
+  %r = fptoui <2 x double> %t to <2 x i8>
+  store <2 x i8> %r, <2 x i8>* %q
+  ret void
 }
 define void @c(<2 x i8>* %p, <2 x double>* %q) {
-        %t = load <2 x i8>, <2 x i8>* %p
-	%r = sitofp <2 x i8> %t to <2 x double>
-        store <2 x double> %r, <2 x double>* %q
-	ret void
+  %t = load <2 x i8>, <2 x i8>* %p
+  %r = sitofp <2 x i8> %t to <2 x double>
+  store <2 x double> %r, <2 x double>* %q
+  ret void
 }
 define void @d(<2 x i8>* %p, <2 x double>* %q) {
-        %t = load <2 x i8>, <2 x i8>* %p
-	%r = uitofp <2 x i8> %t to <2 x double>
-        store <2 x double> %r, <2 x double>* %q
-	ret void
+  %t = load <2 x i8>, <2 x i8>* %p
+  %r = uitofp <2 x i8> %t to <2 x double>
+  store <2 x double> %r, <2 x double>* %q
+  ret void
 }
 define void @e(<2 x i8>* %p, <2 x i16>* %q) {
-        %t = load <2 x i8>, <2 x i8>* %p
-	%r = sext <2 x i8> %t to <2 x i16>
-        store <2 x i16> %r, <2 x i16>* %q
-	ret void
+  %t = load <2 x i8>, <2 x i8>* %p
+  %r = sext <2 x i8> %t to <2 x i16>
+  store <2 x i16> %r, <2 x i16>* %q
+  ret void
 }
 define void @f(<2 x i8>* %p, <2 x i16>* %q) {
-        %t = load <2 x i8>, <2 x i8>* %p
-	%r = zext <2 x i8> %t to <2 x i16>
-        store <2 x i16> %r, <2 x i16>* %q
-	ret void
+  %t = load <2 x i8>, <2 x i8>* %p
+  %r = zext <2 x i8> %t to <2 x i16>
+  store <2 x i16> %r, <2 x i16>* %q
+  ret void
 }
 define void @g(<2 x i16>* %p, <2 x i8>* %q) {
-        %t = load <2 x i16>, <2 x i16>* %p
-	%r = trunc <2 x i16> %t to <2 x i8>
-        store <2 x i8> %r, <2 x i8>* %q
-	ret void
+  %t = load <2 x i16>, <2 x i16>* %p
+  %r = trunc <2 x i16> %t to <2 x i8>
+  store <2 x i8> %r, <2 x i8>* %q
+  ret void
 }
diff --git a/test/CodeGen/Hexagon/Atomics.ll b/test/CodeGen/Hexagon/Atomics.ll
new file mode 100644
index 00000000000..bbac5d73c86
--- /dev/null
+++ b/test/CodeGen/Hexagon/Atomics.ll
@@ -0,0 +1,71 @@
+; RUN: llc < %s -march=hexagon
+
+@si = common global i32 0, align 4
+@sll = common global i64 0, align 8
+
+define void @test_op_ignore() nounwind {
+entry:
+  %t00 = atomicrmw add i32* @si, i32 1 monotonic
+  %t01 = atomicrmw add i64* @sll, i64 1 monotonic
+  %t10 = atomicrmw sub i32* @si, i32 1 monotonic
+  %t11 = atomicrmw sub i64* @sll, i64 1 monotonic
+  %t20 = atomicrmw or i32* @si, i32 1 monotonic
+  %t21 = atomicrmw or i64* @sll, i64 1 monotonic
+  %t30 = atomicrmw xor i32* @si, i32 1 monotonic
+  %t31 = atomicrmw xor i64* @sll, i64 1 monotonic
+  %t40 = atomicrmw and i32* @si, i32 1 monotonic
+  %t41 = atomicrmw and i64* @sll, i64 1 monotonic
+  %t50 = atomicrmw nand i32* @si, i32 1 monotonic
+  %t51 = atomicrmw nand i64* @sll, i64 1 monotonic
+  br label %return
+
+return:                                           ; preds = %entry
+  ret void
+}
+
+define void @test_fetch_and_op() nounwind {
+entry:
+  %t00 = atomicrmw add i32* @si, i32 11 monotonic
+  store i32 %t00, i32* @si, align 4
+  %t01 = atomicrmw add i64* @sll, i64 11 monotonic
+  store i64 %t01, i64* @sll, align 8
+  %t10 = atomicrmw sub i32* @si, i32 11 monotonic
+  store i32 %t10, i32* @si, align 4
+  %t11 = atomicrmw sub i64* @sll, i64 11 monotonic
+  store i64 %t11, i64* @sll, align 8
+  %t20 = atomicrmw or i32* @si, i32 11 monotonic
+  store i32 %t20, i32* @si, align 4
+  %t21 = atomicrmw or i64* @sll, i64 11 monotonic
+  store i64 %t21, i64* @sll, align 8
+  %t30 = atomicrmw xor i32* @si, i32 11 monotonic
+  store i32 %t30, i32* @si, align 4
+  %t31 = atomicrmw xor i64* @sll, i64 11 monotonic
+  store i64 %t31, i64* @sll, align 8
+  %t40 = atomicrmw and i32* @si, i32 11 monotonic
+  store i32 %t40, i32* @si, align 4
+  %t41 = atomicrmw and i64* @sll, i64 11 monotonic
+  store i64 %t41, i64* @sll, align 8
+  %t50 = atomicrmw nand i32* @si, i32 11 monotonic
+  store i32 %t50, i32* @si, align 4
+  %t51 = atomicrmw nand i64* @sll, i64 11 monotonic
+  store i64 %t51, i64* @sll, align 8
+  br label %return
+
+return:                                           ; preds = %entry
+  ret void
+}
+
+define void @test_lock() nounwind {
+entry:
+  %t00 = atomicrmw xchg i32* @si, i32 1 monotonic
+  store i32 %t00, i32* @si, align 4
+  %t01 = atomicrmw xchg i64* @sll, i64 1 monotonic
+  store i64 %t01, i64* @sll, align 8
+  fence seq_cst
+  store volatile i32 0, i32* @si, align 4
+  store volatile i64 0, i64* @sll, align 8
+  br label %return
+
+return:                                           ; preds = %entry
+  ret void
+}
diff --git a/test/CodeGen/Hexagon/BranchPredict.ll b/test/CodeGen/Hexagon/BranchPredict.ll
index 0cd616b31a3..17d169974e5 100644
--- a/test/CodeGen/Hexagon/BranchPredict.ll
+++ b/test/CodeGen/Hexagon/BranchPredict.ll
@@ -1,4 +1,4 @@
-; RUN: llc -march=hexagon -mcpu=hexagonv4 < %s | FileCheck %s
+; RUN: llc -march=hexagon -ifcvt-limit=0 < %s | FileCheck %s
 
 ; Check if the branch probabilities are reflected in the instructions:
 ; The basic block placement pass should place the more probable successor
diff --git a/test/CodeGen/Hexagon/absaddr-store.ll b/test/CodeGen/Hexagon/absaddr-store.ll
index 3be4b1cc261..dac8607d88d 100644
--- a/test/CodeGen/Hexagon/absaddr-store.ll
+++ b/test/CodeGen/Hexagon/absaddr-store.ll
@@ -1,39 +1,42 @@
-; RUN: llc -march=hexagon -mcpu=hexagonv4 < %s | FileCheck %s
+; RUN: llc -march=hexagon -hexagon-small-data-threshold=0 < %s | FileCheck %s
 ; Check that we generate load instructions with absolute addressing mode.
 
-@a = external global i32
-@b = external global i8
-@c = external global i16
+@a0 = external global i32
+@a1 = external global i32
+@b0 = external global i8
+@b1 = external global i8
+@c0 = external global i16
+@c1 = external global i16
 @d = external global i64
 
 define zeroext i8 @absStoreByte() nounwind {
-; CHECK: memb(##b){{ *}}={{ *}}r{{[0-9]+}}
+; CHECK: memb(##b1){{ *}}={{ *}}r{{[0-9]+}}
 entry:
-  %0 = load i8, i8* @b, align 1
+  %0 = load i8, i8* @b0, align 1
   %conv = zext i8 %0 to i32
   %mul = mul nsw i32 100, %conv
   %conv1 = trunc i32 %mul to i8
-  store i8 %conv1, i8* @b, align 1
+  store i8 %conv1, i8* @b1, align 1
   ret i8 %conv1
 }
 
 define signext i16 @absStoreHalf() nounwind {
-; CHECK: memh(##c){{ *}}={{ *}}r{{[0-9]+}}
+; CHECK: memh(##c1){{ *}}={{ *}}r{{[0-9]+}}
 entry:
-  %0 = load i16, i16* @c, align 2
+  %0 = load i16, i16* @c0, align 2
   %conv = sext i16 %0 to i32
   %mul = mul nsw i32 100, %conv
   %conv1 = trunc i32 %mul to i16
-  store i16 %conv1, i16* @c, align 2
+  store i16 %conv1, i16* @c1, align 2
   ret i16 %conv1
 }
 
 define i32 @absStoreWord() nounwind {
-; CHECK: memw(##a){{ *}}={{ *}}r{{[0-9]+}}
+; CHECK: memw(##a1){{ *}}={{ *}}r{{[0-9]+}}
 entry:
-  %0 = load i32, i32* @a, align 4
+  %0 = load i32, i32* @a0, align 4
   %mul = mul nsw i32 100, %0
-  store i32 %mul, i32* @a, align 4
+  store i32 %mul, i32* @a1, align 4
   ret i32 %mul
 }
 
diff --git a/test/CodeGen/Hexagon/absimm.ll b/test/CodeGen/Hexagon/absimm.ll
index 07adb3fe49d..e67af5e8fef 100644
--- a/test/CodeGen/Hexagon/absimm.ll
+++ b/test/CodeGen/Hexagon/absimm.ll
@@ -1,4 +1,4 @@
-; RUN: llc -march=hexagon -mcpu=hexagonv4 < %s | FileCheck %s
+; RUN: llc -march=hexagon < %s | FileCheck %s
 ; Check that we generate absolute addressing mode instructions
 ; with immediate value.
 
diff --git a/test/CodeGen/Hexagon/adde.ll b/test/CodeGen/Hexagon/adde.ll
index 5a8345c477c..7b29e7ad8a0 100644
--- a/test/CodeGen/Hexagon/adde.ll
+++ b/test/CodeGen/Hexagon/adde.ll
@@ -1,7 +1,7 @@
 ; RUN: llc -march=hexagon -hexagon-expand-condsets=0 < %s | FileCheck %s
 
-; CHECK: r{{[0-9]+:[0-9]+}} = #0
 ; CHECK: r{{[0-9]+:[0-9]+}} = #1
+; CHECK: r{{[0-9]+:[0-9]+}} = #0
 ; CHECK: r{{[0-9]+:[0-9]+}} = add(r{{[0-9]+:[0-9]+}}, r{{[0-9]+:[0-9]+}})
 ; CHECK: p{{[0-9]+}} = cmp.gtu(r{{[0-9]+:[0-9]+}}, r{{[0-9]+:[0-9]+}})
 ; CHECK: p{{[0-9]+}} = cmp.gtu(r{{[0-9]+:[0-9]+}}, r{{[0-9]+:[0-9]+}})
diff --git a/test/CodeGen/Hexagon/addh-sext-trunc.ll b/test/CodeGen/Hexagon/addh-sext-trunc.ll
new file mode 100644
index 00000000000..094932933fb
--- /dev/null
+++ b/test/CodeGen/Hexagon/addh-sext-trunc.ll
@@ -0,0 +1,43 @@
+; RUN: llc -march=hexagon < %s | FileCheck %s
+; CHECK: r{{[0-9]+}} = add(r{{[0-9]+}}.{{L|l}}, r{{[0-9]+}}.{{H|h}})
+
+target datalayout = "e-p:32:32:32-i64:64:64-i32:32:32-i16:16:16-i1:32:32-f64:64:64-f32:32:32-v64:64:64-v32:32:32-a0:0-n16:32"
+target triple = "hexagon-unknown-none"
+
+%struct.aDataType = type { i16, i16, i16, i16, i16, i16*, i16*, i16*, i8*, i16*, i16*, i16*, i8* }
+
+define i8* @a_get_score(%struct.aDataType* nocapture %pData, i16 signext %gmmModelIndex, i16* nocapture %pGmmScoreL16Q4) #0 {
+entry:
+  %numSubVector = getelementptr inbounds %struct.aDataType, %struct.aDataType* %pData, i32 0, i32 3
+  %0 = load i16, i16* %numSubVector, align 2, !tbaa !0
+  %and = and i16 %0, -4
+  %b = getelementptr inbounds %struct.aDataType, %struct.aDataType* %pData, i32 0, i32 8
+  %1 = load i8*, i8** %b, align 4, !tbaa !3
+  %conv3 = sext i16 %and to i32
+  %cmp21 = icmp sgt i16 %and, 0
+  br i1 %cmp21, label %for.inc.preheader, label %for.end
+
+for.inc.preheader:                                ; preds = %entry
+  br label %for.inc
+
+for.inc:                                          ; preds = %for.inc.preheader, %for.inc
+  %j.022 = phi i32 [ %phitmp, %for.inc ], [ 0, %for.inc.preheader ]
+  %add13 = mul i32 %j.022, 65536
+  %sext = add i32 %add13, 262144
+  %phitmp = ashr exact i32 %sext, 16
+  %cmp = icmp slt i32 %phitmp, %conv3
+  br i1 %cmp, label %for.inc, label %for.end.loopexit
+
+for.end.loopexit:                                 ; preds = %for.inc
+  br label %for.end
+
+for.end:                                          ; preds = %for.end.loopexit, %entry
+  ret i8* %1
+}
+
+attributes #0 = { nounwind readonly "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-frame-pointer-elim-non-leaf"="true" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "unsafe-fp-math"="false" "use-soft-float"="false" }
+
+!0 = !{!"short", !1}
+!1 = !{!"omnipotent char", !2}
+!2 = !{!"Simple C/C++ TBAA"}
+!3 = !{!"any pointer", !1}
diff --git a/test/CodeGen/Hexagon/addh-shifted.ll b/test/CodeGen/Hexagon/addh-shifted.ll
new file mode 100644
index 00000000000..eb263521b42
--- /dev/null
+++ b/test/CodeGen/Hexagon/addh-shifted.ll
@@ -0,0 +1,21 @@
+; RUN: llc -march=hexagon < %s | FileCheck %s
+; CHECK: r{{[0-9]+}} = add(r{{[0-9]+}}.{{L|l}}, r{{[0-9]+}}.{{L|l}}):<<16
+
+define i64 @test_cast(i64 %arg0, i16 zeroext %arg1, i16 zeroext %arg2) nounwind readnone {
+entry:
+  %conv.i = zext i16 %arg1 to i32
+  %conv1.i = zext i16 %arg2 to i32
+  %sub.i = add nsw i32 %conv.i, %conv1.i
+  %sext.i = shl i32 %sub.i, 16
+  %cmp.i = icmp slt i32 %sext.i, 65536
+  %0 = ashr exact i32 %sext.i, 16
+  %conv7.i = select i1 %cmp.i, i32 1, i32 %0
+  %cmp8.i = icmp sgt i32 %conv7.i, 4
+  %conv7.op.i = add i32 %conv7.i, 65535
+  %shl = shl i64 %arg0, 2
+  %.mask = and i32 %conv7.op.i, 65535
+  %1 = zext i32 %.mask to i64
+  %conv = select i1 %cmp8.i, i64 3, i64 %1
+  %or = or i64 %conv, %shl
+  ret i64 %or
+}
diff --git a/test/CodeGen/Hexagon/addh.ll b/test/CodeGen/Hexagon/addh.ll
new file mode 100644
index 00000000000..c2b536c4669
--- /dev/null
+++ b/test/CodeGen/Hexagon/addh.ll
@@ -0,0 +1,21 @@
+; RUN: llc -march=hexagon < %s | FileCheck %s
+; CHECK: r{{[0-9]+}} = add(r{{[0-9]+}}.{{L|l}}, r{{[0-9]+}}.{{L|l}})
+
+define i64 @test_cast(i64 %arg0, i16 zeroext %arg1, i16 zeroext %arg2) nounwind readnone {
+entry:
+  %conv.i = zext i16 %arg1 to i32
+  %conv1.i = zext i16 %arg2 to i32
+  %sub.i = add nsw i32 %conv.i, %conv1.i
+  %sext.i = shl i32 %sub.i, 16
+  %cmp.i = icmp slt i32 %sext.i, 65536
+  %0 = ashr exact i32 %sext.i, 16
+  %conv7.i = select i1 %cmp.i, i32 1, i32 %0
+  %cmp8.i = icmp sgt i32 %conv7.i, 4
+  %conv7.op.i = add i32 %conv7.i, 65535
+  %shl = shl i64 %arg0, 2
+  %.mask = and i32 %conv7.op.i, 65535
+  %1 = zext i32 %.mask to i64
+  %conv = select i1 %cmp8.i, i64 3, i64 %1
+  %or = or i64 %conv, %shl
+  ret i64 %or
+}
diff --git a/test/CodeGen/Hexagon/addrmode-indoff.ll b/test/CodeGen/Hexagon/addrmode-indoff.ll
new file mode 100644
index 00000000000..6ea2b3d95da
--- /dev/null
+++ b/test/CodeGen/Hexagon/addrmode-indoff.ll
@@ -0,0 +1,74 @@
+; RUN: llc -march=hexagon < %s | FileCheck %s
+;
+; Bug 6840. Use absolute+index addressing.
+
+@ga = common global [1024 x i8] zeroinitializer, align 8
+@gb = common global [1024 x i8] zeroinitializer, align 8
+
+; CHECK: memub(r{{[0-9]+}}{{ *}}<<{{ *}}#0{{ *}}+{{ *}}##ga)
+define zeroext i8 @lf2(i32 %i) nounwind readonly {
+entry:
+  %arrayidx = getelementptr inbounds [1024 x i8], [1024 x i8]* @ga, i32 0, i32 %i
+  %0 = load i8, i8* %arrayidx, align 1
+  ret i8 %0
+}
+
+; CHECK: memb(r{{[0-9]+}}{{ *}}<<{{ *}}#0{{ *}}+{{ *}}##gb)
+define signext i8 @lf2s(i32 %i) nounwind readonly {
+entry:
+  %arrayidx = getelementptr inbounds [1024 x i8], [1024 x i8]* @gb, i32 0, i32 %i
+  %0 = load i8, i8* %arrayidx, align 1
+  ret i8 %0
+}
+
+; CHECK: memub(r{{[0-9]+}}{{ *}}<<{{ *}}#2{{ *}}+{{ *}}##ga)
+define zeroext i8 @lf3(i32 %i) nounwind readonly {
+entry:
+  %mul = shl nsw i32 %i, 2
+  %arrayidx = getelementptr inbounds [1024 x i8], [1024 x i8]* @ga, i32 0, i32 %mul
+  %0 = load i8, i8* %arrayidx, align 1
+  ret i8 %0
+}
+
+; CHECK: memb(r{{[0-9]+}}{{ *}}<<{{ *}}#2{{ *}}+{{ *}}##gb)
+define signext i8 @lf3s(i32 %i) nounwind readonly {
+entry:
+  %mul = shl nsw i32 %i, 2
+  %arrayidx = getelementptr inbounds [1024 x i8], [1024 x i8]* @gb, i32 0, i32 %mul
+  %0 = load i8, i8* %arrayidx, align 1
+  ret i8 %0
+}
+
+; CHECK: memb(r{{[0-9]+}}{{ *}}<<{{ *}}#0{{ *}}+{{ *}}##ga)
+define void @sf4(i32 %i, i8 zeroext %j) nounwind {
+entry:
+  %arrayidx = getelementptr inbounds [1024 x i8], [1024 x i8]* @ga, i32 0, i32 %i
+  store i8 %j, i8* %arrayidx, align 1
+  ret void
+}
+
+; CHECK: memb(r{{[0-9]+}}{{ *}}<<{{ *}}#0{{ *}}+{{ *}}##gb)
+define void @sf4s(i32 %i, i8 signext %j) nounwind {
+entry:
+  %arrayidx = getelementptr inbounds [1024 x i8], [1024 x i8]* @gb, i32 0, i32 %i
+  store i8 %j, i8* %arrayidx, align 1
+  ret void
+}
+
+; CHECK: memb(r{{[0-9]+}}{{ *}}<<{{ *}}#2{{ *}}+{{ *}}##ga)
+define void @sf5(i32 %i, i8 zeroext %j) nounwind {
+entry:
+  %mul = shl nsw i32 %i, 2
+  %arrayidx = getelementptr inbounds [1024 x i8], [1024 x i8]* @ga, i32 0, i32 %mul
+  store i8 %j, i8* %arrayidx, align 1
+  ret void
+}
+
+; CHECK: memb(r{{[0-9]+}}{{ *}}<<{{ *}}#2{{ *}}+{{ *}}##gb)
+define void @sf5s(i32 %i, i8 signext %j) nounwind {
+entry:
+  %mul = shl nsw i32 %i, 2
+  %arrayidx = getelementptr inbounds [1024 x i8], [1024 x i8]* @gb, i32 0, i32 %mul
+  store i8 %j, i8* %arrayidx, align 1
+  ret void
+}
diff --git a/test/CodeGen/Hexagon/alu64.ll b/test/CodeGen/Hexagon/alu64.ll
new file mode 100644
index 00000000000..d0824a4ecad
--- /dev/null
+++ b/test/CodeGen/Hexagon/alu64.ll
@@ -0,0 +1,599 @@
+; RUN: llc -march=hexagon < %s | FileCheck %s
+
+; CHECK-LABEL: @test00
+; CHECK: p0 = cmp.eq(r1:0, r3:2)
+define i32 @test00(i64 %Rs, i64 %Rt) #0 {
+entry:
+  %0 = tail call i32 @llvm.hexagon.C2.cmpeqp(i64 %Rs, i64 %Rt)
+  ret i32 %0
+}
+
+; CHECK-LABEL: @test01
+; CHECK: p0 = cmp.gt(r1:0, r3:2)
+define i32 @test01(i64 %Rs, i64 %Rt) #0 {
+entry:
+  %0 = tail call i32 @llvm.hexagon.C2.cmpgtp(i64 %Rs, i64 %Rt)
+  ret i32 %0
+}
+
+; CHECK-LABEL: @test02
+; CHECK: p0 = cmp.gtu(r1:0, r3:2)
+define i32 @test02(i64 %Rs, i64 %Rt) #0 {
+entry:
+  %0 = tail call i32 @llvm.hexagon.C2.cmpgtup(i64 %Rs, i64 %Rt)
+  ret i32 %0
+}
+
+; CHECK-LABEL: @test10
+; CHECK: r0 = cmp.eq(r0, r1)
+define i32 @test10(i32 %Rs, i32 %Rt) #0 {
+entry:
+  %0 = tail call i32 @llvm.hexagon.A4.rcmpeq(i32 %Rs, i32 %Rt)
+  ret i32 %0
+}
+
+; CHECK-LABEL: @test11
+; CHECK: r0 = !cmp.eq(r0, r1)
+define i32 @test11(i32 %Rs, i32 %Rt) #0 {
+entry:
+  %0 = tail call i32 @llvm.hexagon.A4.rcmpneq(i32 %Rs, i32 %Rt)
+  ret i32 %0
+}
+
+; CHECK-LABEL: @test12
+; CHECK: r0 = cmp.eq(r0, #23)
+define i32 @test12(i32 %Rs) #0 {
+entry:
+  %0 = tail call i32 @llvm.hexagon.A4.rcmpeqi(i32 %Rs, i32 23)
+  ret i32 %0
+}
+
+; CHECK-LABEL: @test13
+; CHECK: r0 = !cmp.eq(r0, #47)
+define i32 @test13(i32 %Rs) #0 {
+entry:
+  %0 = tail call i32 @llvm.hexagon.A4.rcmpneqi(i32 %Rs, i32 47)
+  ret i32 %0
+}
+
+; CHECK-LABEL: @test20
+; CHECK: p0 = cmpb.eq(r0, r1)
+define i32 @test20(i32 %Rs, i32 %Rt) #0 {
+entry:
+  %0 = tail call i32 @llvm.hexagon.A4.cmpbeq(i32 %Rs, i32 %Rt)
+  ret i32 %0
+}
+
+; CHECK-LABEL: @test21
+; CHECK: p0 = cmpb.gt(r0, r1)
+define i32 @test21(i32 %Rs, i32 %Rt) #0 {
+entry:
+  %0 = tail call i32 @llvm.hexagon.A4.cmpbgt(i32 %Rs, i32 %Rt)
+  ret i32 %0
+}
+
+; CHECK-LABEL: @test22
+; CHECK: p0 = cmpb.gtu(r0, r1)
+define i32 @test22(i32 %Rs, i32 %Rt) #0 {
+entry:
+  %0 = tail call i32 @llvm.hexagon.A4.cmpbgtu(i32 %Rs, i32 %Rt)
+  ret i32 %0
+}
+
+; CHECK-LABEL: @test23
+; CHECK: p0 = cmpb.eq(r0, #56)
+define i32 @test23(i32 %Rs) #0 {
+entry:
+  %0 = tail call i32 @llvm.hexagon.A4.cmpbeqi(i32 %Rs, i32 56)
+  ret i32 %0
+}
+
+; CHECK-LABEL: @test24
+; CHECK: p0 = cmpb.gt(r0, #29)
+define i32 @test24(i32 %Rs) #0 {
+entry:
+  %0 = tail call i32 @llvm.hexagon.A4.cmpbgti(i32 %Rs, i32 29)
+  ret i32 %0
+}
+
+; CHECK-LABEL: @test25
+; CHECK: p0 = cmpb.gtu(r0, #111)
+define i32 @test25(i32 %Rs) #0 {
+entry:
+  %0 = tail call i32 @llvm.hexagon.A4.cmpbgtui(i32 %Rs, i32 111)
+  ret i32 %0
+}
+
+; CHECK-LABEL: @test30
+; CHECK: p0 = cmph.eq(r0, r1)
+define i32 @test30(i32 %Rs, i32 %Rt) #0 {
+entry:
+  %0 = tail call i32 @llvm.hexagon.A4.cmpheq(i32 %Rs, i32 %Rt)
+  ret i32 %0
+}
+
+; CHECK-LABEL: @test31
+; CHECK: p0 = cmph.gt(r0, r1)
+define i32 @test31(i32 %Rs, i32 %Rt) #0 {
+entry:
+  %0 = tail call i32 @llvm.hexagon.A4.cmphgt(i32 %Rs, i32 %Rt)
+  ret i32 %0
+}
+
+; CHECK-LABEL: @test32
+; CHECK: p0 = cmph.gtu(r0, r1)
+define i32 @test32(i32 %Rs, i32 %Rt) #0 {
+entry:
+  %0 = tail call i32 @llvm.hexagon.A4.cmphgtu(i32 %Rs, i32 %Rt)
+  ret i32 %0
+}
+
+; CHECK-LABEL: @test33
+; CHECK: p0 = cmph.eq(r0, #-123)
+define i32 @test33(i32 %Rs) #0 {
+entry:
+  %0 = tail call i32 @llvm.hexagon.A4.cmpheqi(i32 %Rs, i32 -123)
+  ret i32 %0
+}
+
+; CHECK-LABEL: @test34
+; CHECK: p0 = cmph.gt(r0, #-3)
+define i32 @test34(i32 %Rs) #0 {
+entry:
+  %0 = tail call i32 @llvm.hexagon.A4.cmphgti(i32 %Rs, i32 -3)
+  ret i32 %0
+}
+
+; CHECK-LABEL: @test35
+; CHECK: p0 = cmph.gtu(r0, #13)
+define i32 @test35(i32 %Rs) #0 {
+entry:
+  %0 = tail call i32 @llvm.hexagon.A4.cmphgtui(i32 %Rs, i32 13)
+  ret i32 %0
+}
+
+; CHECK-LABEL: @test40
+; CHECK: r1:0 = vmux(p0, r3:2, r5:4)
+define i64 @test40(i32 %Pu, i64 %Rs, i64 %Rt) #0 {
+entry:
+  %0 = tail call i64 @llvm.hexagon.C2.vmux(i32 %Pu, i64 %Rs, i64 %Rt)
+  ret i64 %0
+}
+
+; CHECK-LABEL: @test41
+; CHECK: p0 = any8(vcmpb.eq(r1:0, r3:2))
+define i32 @test41(i64 %Rs, i64 %Rt) #0 {
+entry:
+  %0 = tail call i32 @llvm.hexagon.A4.vcmpbeq.any(i64 %Rs, i64 %Rt)
+  ret i32 %0
+}
+
+; CHECK-LABEL: @test50
+; CHECK: r1:0 = add(r1:0, r3:2)
+define i64 @test50(i64 %Rs, i64 %Rt) #0 {
+entry:
+  %0 = tail call i64 @llvm.hexagon.A2.addp(i64 %Rs, i64 %Rt)
+  ret i64 %0
+}
+
+; CHECK-LABEL: @test51
+; CHECK: r1:0 = add(r1:0, r3:2):sat
+define i64 @test51(i64 %Rs, i64 %Rt) #0 {
+entry:
+  %0 = tail call i64 @llvm.hexagon.A2.addpsat(i64 %Rs, i64 %Rt)
+  ret i64 %0
+}
+
+; CHECK-LABEL: @test52
+; CHECK: r1:0 = sub(r1:0, r3:2)
+define i64 @test52(i64 %Rs, i64 %Rt) #0 {
+entry:
+  %0 = tail call i64 @llvm.hexagon.A2.subp(i64 %Rs, i64 %Rt)
+  ret i64 %0
+}
+
+; CHECK-LABEL: @test53
+; CHECK: r1:0 = add(r0, r3:2)
+define i64 @test53(i32 %Rs, i64 %Rt) #0 {
+entry:
+  %0 = tail call i64 @llvm.hexagon.A2.addsp(i32 %Rs, i64 %Rt)
+  ret i64 %0
+}
+
+; CHECK-LABEL: @test54
+; CHECK: r1:0 = and(r1:0, r3:2)
+define i64 @test54(i64 %Rs, i64 %Rt) #0 {
+entry:
+  %0 = tail call i64 @llvm.hexagon.A2.andp(i64 %Rs, i64 %Rt)
+  ret i64 %0
+}
+
+; CHECK-LABEL: @test55
+; CHECK: r1:0 = or(r1:0, r3:2)
+define i64 @test55(i64 %Rs, i64 %Rt) #0 {
+entry:
+  %0 = tail call i64 @llvm.hexagon.A2.orp(i64 %Rs, i64 %Rt)
+  ret i64 %0
+}
+
+; CHECK-LABEL: @test56
+; CHECK: r1:0 = xor(r1:0, r3:2)
+define i64 @test56(i64 %Rs, i64 %Rt) #0 {
+entry:
+  %0 = tail call i64 @llvm.hexagon.A2.xorp(i64 %Rs, i64 %Rt)
+  ret i64 %0
+}
+
+; CHECK-LABEL: @test57
+; CHECK: r1:0 = and(r1:0, ~r3:2)
+define i64 @test57(i64 %Rs, i64 %Rt) #0 {
+entry:
+  %0 = tail call i64 @llvm.hexagon.A4.andnp(i64 %Rs, i64 %Rt)
+  ret i64 %0
+}
+
+; CHECK-LABEL: @test58
+; CHECK: r1:0 = or(r1:0, ~r3:2)
+define i64 @test58(i64 %Rs, i64 %Rt) #0 {
+entry:
+  %0 = tail call i64 @llvm.hexagon.A4.ornp(i64 %Rs, i64 %Rt)
+  ret i64 %0
+}
+
+; CHECK-LABEL: @test60
+; CHECK: r0 = add(r0.l, r1.l)
+define i32 @test60(i32 %Rs, i32 %Rt) #0 {
+entry:
+  %0 = tail call i32 @llvm.hexagon.A2.addh.l16.ll(i32 %Rs, i32 %Rt)
+  ret i32 %0
+}
+
+; CHECK-LABEL: @test61
+; CHECK: r0 = add(r0.l, r1.h)
+define i32 @test61(i32 %Rs, i32 %Rt) #0 {
+entry:
+  %0 = tail call i32 @llvm.hexagon.A2.addh.l16.hl(i32 %Rs, i32 %Rt)
+  ret i32 %0
+}
+
+; CHECK-LABEL: @test62
+; CHECK: r0 = add(r0.l, r1.l):sat
+define i32 @test62(i32 %Rs, i32 %Rt) #0 {
+entry:
+  %0 = tail call i32 @llvm.hexagon.A2.addh.l16.sat.ll(i32 %Rs, i32 %Rt)
+  ret i32 %0
+}
+
+; CHECK-LABEL: @test63
+; CHECK: r0 = add(r0.l, r1.h):sat
+define i32 @test63(i32 %Rs, i32 %Rt) #0 {
+entry:
+  %0 = tail call i32 @llvm.hexagon.A2.addh.l16.sat.hl(i32 %Rs, i32 %Rt)
+  ret i32 %0
+}
+
+; CHECK-LABEL: @test64
+; CHECK: r0 = add(r0.l, r1.l):<<16
+define i32 @test64(i32 %Rs, i32 %Rt) #0 {
+entry:
+  %0 = tail call i32 @llvm.hexagon.A2.addh.h16.ll(i32 %Rs, i32 %Rt)
+  ret i32 %0
+}
+
+; CHECK-LABEL: @test65
+; CHECK: r0 = add(r0.l, r1.h):<<16
+define i32 @test65(i32 %Rs, i32 %Rt) #0 {
+entry:
+  %0 = tail call i32 @llvm.hexagon.A2.addh.h16.lh(i32 %Rs, i32 %Rt)
+  ret i32 %0
+}
+
+; CHECK-LABEL: @test66
+; CHECK: r0 = add(r0.h, r1.l):<<16
+define i32 @test66(i32 %Rs, i32 %Rt) #0 {
+entry:
+  %0 = tail call i32 @llvm.hexagon.A2.addh.h16.hl(i32 %Rs, i32 %Rt)
+  ret i32 %0
+}
+
+; CHECK-LABEL: @test67
+; CHECK: r0 = add(r0.h, r1.h):<<16
+define i32 @test67(i32 %Rs, i32 %Rt) #0 {
+entry:
+  %0 = tail call i32 @llvm.hexagon.A2.addh.h16.hh(i32 %Rs, i32 %Rt)
+  ret i32 %0
+}
+
+; CHECK-LABEL: @test68
+; CHECK: r0 = add(r0.l, r1.l):sat:<<16
+define i32 @test68(i32 %Rs, i32 %Rt) #0 {
+entry:
+  %0 = tail call i32 @llvm.hexagon.A2.addh.h16.sat.ll(i32 %Rs, i32 %Rt)
+  ret i32 %0
+}
+
+; CHECK-LABEL: @test69
+; CHECK: r0 = add(r0.l, r1.h):sat:<<16
+define i32 @test69(i32 %Rs, i32 %Rt) #0 {
+entry:
+  %0 = tail call i32 @llvm.hexagon.A2.addh.h16.sat.lh(i32 %Rs, i32 %Rt)
+  ret i32 %0
+}
+
+; CHECK-LABEL: @test6A
+; CHECK: r0 = add(r0.h, r1.l):sat:<<16
+define i32 @test6A(i32 %Rs, i32 %Rt) #0 {
+entry:
+  %0 = tail call i32 @llvm.hexagon.A2.addh.h16.sat.hl(i32 %Rs, i32 %Rt)
+  ret i32 %0
+}
+
+; CHECK-LABEL: @test6B
+; CHECK: r0 = add(r0.h, r1.h):sat:<<16
+define i32 @test6B(i32 %Rs, i32 %Rt) #0 {
+entry:
+  %0 = tail call i32 @llvm.hexagon.A2.addh.h16.sat.hh(i32 %Rs, i32 %Rt)
+  ret i32 %0
+}
+
+; CHECK-LABEL: @test70
+; CHECK: r0 = sub(r0.l, r1.l)
+define i32 @test70(i32 %Rs, i32 %Rt) #0 {
+entry:
+  %0 = tail call i32 @llvm.hexagon.A2.subh.l16.ll(i32 %Rs, i32 %Rt)
+  ret i32 %0
+}
+
+; CHECK-LABEL: @test71
+; CHECK: r0 = sub(r0.l, r1.h)
+define i32 @test71(i32 %Rs, i32 %Rt) #0 {
+entry:
+  %0 = tail call i32 @llvm.hexagon.A2.subh.l16.hl(i32 %Rs, i32 %Rt)
+  ret i32 %0
+}
+
+; CHECK-LABEL: @test72
+; CHECK: r0 = sub(r0.l, r1.l):sat
+define i32 @test72(i32 %Rs, i32 %Rt) #0 {
+entry:
+  %0 = tail call i32 @llvm.hexagon.A2.subh.l16.sat.ll(i32 %Rs, i32 %Rt)
+  ret i32 %0
+}
+
+; CHECK-LABEL: @test73
+; CHECK: r0 = sub(r0.l, r1.h):sat
+define i32 @test73(i32 %Rs, i32 %Rt) #0 {
+entry:
+  %0 = tail call i32 @llvm.hexagon.A2.subh.l16.sat.hl(i32 %Rs, i32 %Rt)
+  ret i32 %0
+}
+
+; CHECK-LABEL: @test74
+; CHECK: r0 = sub(r0.l, r1.l):<<16
+define i32 @test74(i32 %Rs, i32 %Rt) #0 {
+entry:
+  %0 = tail call i32 @llvm.hexagon.A2.subh.h16.ll(i32 %Rs, i32 %Rt)
+  ret i32 %0
+}
+
+; CHECK-LABEL: @test75
+; CHECK: r0 = sub(r0.l, r1.h):<<16
+define i32 @test75(i32 %Rs, i32 %Rt) #0 {
+entry:
+  %0 = tail call i32 @llvm.hexagon.A2.subh.h16.lh(i32 %Rs, i32 %Rt)
+  ret i32 %0
+}
+
+; CHECK-LABEL: @test76
+; CHECK: r0 = sub(r0.h, r1.l):<<16
+define i32 @test76(i32 %Rs, i32 %Rt) #0 {
+entry:
+  %0 = tail call i32 @llvm.hexagon.A2.subh.h16.hl(i32 %Rs, i32 %Rt)
+  ret i32 %0
+}
+
+; CHECK-LABEL: @test77
+; CHECK: r0 = sub(r0.h, r1.h):<<16
+define i32 @test77(i32 %Rs, i32 %Rt) #0 {
+entry:
+  %0 = tail call i32 @llvm.hexagon.A2.subh.h16.hh(i32 %Rs, i32 %Rt)
+  ret i32 %0
+}
+
+; CHECK-LABEL: @test78
+; CHECK: r0 = sub(r0.l, r1.l):sat:<<16
+define i32 @test78(i32 %Rs, i32 %Rt) #0 {
+entry:
+  %0 = tail call i32 @llvm.hexagon.A2.subh.h16.sat.ll(i32 %Rs, i32 %Rt)
+  ret i32 %0
+}
+
+; CHECK-LABEL: @test79
+; CHECK: r0 = sub(r0.l, r1.h):sat:<<16
+define i32 @test79(i32 %Rs, i32 %Rt) #0 {
+entry:
+  %0 = tail call i32 @llvm.hexagon.A2.subh.h16.sat.lh(i32 %Rs, i32 %Rt)
+  ret i32 %0
+}
+
+; CHECK-LABEL: @test7A
+; CHECK: r0 = sub(r0.h, r1.l):sat:<<16
+define i32 @test7A(i32 %Rs, i32 %Rt) #0 {
+entry:
+  %0 = tail call i32 @llvm.hexagon.A2.subh.h16.sat.hl(i32 %Rs, i32 %Rt)
+  ret i32 %0
+}
+
+; CHECK-LABEL: @test7B
+; CHECK: r0 = sub(r0.h, r1.h):sat:<<16
+define i32 @test7B(i32 %Rs, i32 %Rt) #0 {
+entry:
+  %0 = tail call i32 @llvm.hexagon.A2.subh.h16.sat.hh(i32 %Rs, i32 %Rt)
+  ret i32 %0
+}
+
+; CHECK-LABEL: @test90
+; CHECK: r0 = and(#1, asl(r0, #2))
+define i32 @test90(i32 %Rs) #0 {
+entry:
+  %0 = tail call i32 @llvm.hexagon.S4.andi.asl.ri(i32 1, i32 %Rs, i32 2)
+  ret i32 %0
+}
+
+; CHECK-LABEL: @test91
+; CHECK: r0 = or(#1, asl(r0, #2))
+define i32 @test91(i32 %Rs) #0 {
+entry:
+  %0 = tail call i32 @llvm.hexagon.S4.ori.asl.ri(i32 1, i32 %Rs, i32 2)
+  ret i32 %0
+}
+
+; CHECK-LABEL: @test92
+; CHECK: r0 = add(#1, asl(r0, #2))
+define i32 @test92(i32 %Rs) #0 {
+entry:
+  %0 = tail call i32 @llvm.hexagon.S4.addi.asl.ri(i32 1, i32 %Rs, i32 2)
+  ret i32 %0
+}
+
+; CHECK-LABEL: @test93
+; CHECK: r0 = sub(#1, asl(r0, #2))
+define i32 @test93(i32 %Rs) #0 {
+entry:
+  %0 = tail call i32 @llvm.hexagon.S4.subi.asl.ri(i32 1, i32 %Rs, i32 2)
+  ret i32 %0
+}
+
+; CHECK-LABEL: @test94
+; CHECK: r0 = and(#1, lsr(r0, #2))
+define i32 @test94(i32 %Rs) #0 {
+entry:
+  %0 = tail call i32 @llvm.hexagon.S4.andi.lsr.ri(i32 1, i32 %Rs, i32 2)
+  ret i32 %0
+}
+
+; CHECK-LABEL: @test95
+; CHECK: r0 = or(#1, lsr(r0, #2))
+define i32 @test95(i32 %Rs) #0 {
+entry:
+  %0 = tail call i32 @llvm.hexagon.S4.ori.lsr.ri(i32 1, i32 %Rs, i32 2)
+  ret i32 %0
+}
+
+; CHECK-LABEL: @test96
+; CHECK: r0 = add(#1, lsr(r0, #2))
+define i32 @test96(i32 %Rs) #0 {
+entry:
+  %0 = tail call i32 @llvm.hexagon.S4.addi.lsr.ri(i32 1, i32 %Rs, i32 2)
+  ret i32 %0
+}
+
+; CHECK-LABEL: @test97
+; CHECK: r0 = sub(#1, lsr(r0, #2))
+define i32 @test97(i32 %Rs) #0 {
+entry:
+  %0 = tail call i32 @llvm.hexagon.S4.subi.lsr.ri(i32 1, i32 %Rs, i32 2)
+  ret i32 %0
+}
+
+; CHECK-LABEL: @test100
+; CHECK: r1:0 = bitsplit(r0, r1)
+define i64 @test100(i32 %Rs, i32 %Rt) #0 {
+entry:
+  %0 = tail call i64 @llvm.hexagon.A4.bitsplit(i32 %Rs, i32 %Rt)
+  ret i64 %0
+}
+
+; CHECK-LABEL: @test101
+; CHECK: r0 = modwrap(r0, r1)
+define i32 @test101(i32 %Rs, i32 %Rt) #0 {
+entry:
+  %0 = tail call i32 @llvm.hexagon.A4.modwrapu(i32 %Rs, i32 %Rt)
+  ret i32 %0
+}
+
+; CHECK-LABEL: @test102
+; CHECK: r0 = parity(r1:0, r3:2)
+define i32 @test102(i64 %Rs, i64 %Rt) #0 {
+entry:
+  %0 = tail call i32 @llvm.hexagon.S2.parityp(i64 %Rs, i64 %Rt)
+  ret i32 %0
+}
+
+; CHECK-LABEL: @test103
+; CHECK: r0 = parity(r0, r1)
+define i32 @test103(i32 %Rs, i32 %Rt) #0 {
+entry:
+  %0 = tail call i32 @llvm.hexagon.S4.parity(i32 %Rs, i32 %Rt)
+  ret i32 %0
+}
+
+declare i32 @llvm.hexagon.C2.cmpeqp(i64, i64) #1
+declare i32 @llvm.hexagon.C2.cmpgtp(i64, i64) #1
+declare i32 @llvm.hexagon.C2.cmpgtup(i64, i64) #1
+declare i32 @llvm.hexagon.A4.rcmpeq(i32, i32) #1
+declare i32 @llvm.hexagon.A4.rcmpneq(i32, i32) #1
+declare i32 @llvm.hexagon.A4.rcmpeqi(i32, i32) #1
+declare i32 @llvm.hexagon.A4.rcmpneqi(i32, i32) #1
+declare i32 @llvm.hexagon.A4.cmpbeq(i32, i32) #1
+declare i32 @llvm.hexagon.A4.cmpbgt(i32, i32) #1
+declare i32 @llvm.hexagon.A4.cmpbgtu(i32, i32) #1
+declare i32 @llvm.hexagon.A4.cmpbeqi(i32, i32) #1
+declare i32 @llvm.hexagon.A4.cmpbgti(i32, i32) #1
+declare i32 @llvm.hexagon.A4.cmpbgtui(i32, i32) #1
+declare i32 @llvm.hexagon.A4.cmpheq(i32, i32) #1
+declare i32 @llvm.hexagon.A4.cmphgt(i32, i32) #1
+declare i32 @llvm.hexagon.A4.cmphgtu(i32, i32) #1
+declare i32 @llvm.hexagon.A4.cmpheqi(i32, i32) #1
+declare i32 @llvm.hexagon.A4.cmphgti(i32, i32) #1
+declare i32 @llvm.hexagon.A4.cmphgtui(i32, i32) #1
+declare i64 @llvm.hexagon.C2.vmux(i32, i64, i64) #1
+declare i32 @llvm.hexagon.A4.vcmpbeq.any(i64, i64) #1
+declare i64 @llvm.hexagon.A2.addp(i64, i64) #1
+declare i64 @llvm.hexagon.A2.addpsat(i64, i64) #1
+declare i64 @llvm.hexagon.A2.subp(i64, i64) #1
+declare i64 @llvm.hexagon.A2.addsp(i32, i64) #1
+declare i64 @llvm.hexagon.A2.andp(i64, i64) #1
+declare i64 @llvm.hexagon.A2.orp(i64, i64) #1
+declare i64 @llvm.hexagon.A2.xorp(i64, i64) #1
+declare i64 @llvm.hexagon.A4.ornp(i64, i64) #1
+declare i64 @llvm.hexagon.A4.andnp(i64, i64) #1
+declare i32 @llvm.hexagon.A2.addh.l16.ll(i32, i32) #1
+declare i32 @llvm.hexagon.A2.addh.l16.hl(i32, i32) #1
+declare i32 @llvm.hexagon.A2.addh.l16.sat.ll(i32, i32) #1
+declare i32 @llvm.hexagon.A2.addh.l16.sat.hl(i32, i32) #1
+declare i32 @llvm.hexagon.A2.addh.h16.ll(i32, i32) #1
+declare i32 @llvm.hexagon.A2.addh.h16.lh(i32, i32) #1
+declare i32 @llvm.hexagon.A2.addh.h16.hl(i32, i32) #1
+declare i32 @llvm.hexagon.A2.addh.h16.hh(i32, i32) #1
+declare i32 @llvm.hexagon.A2.addh.h16.sat.ll(i32, i32) #1
+declare i32 @llvm.hexagon.A2.addh.h16.sat.lh(i32, i32) #1
+declare i32 @llvm.hexagon.A2.addh.h16.sat.hl(i32, i32) #1
+declare i32 @llvm.hexagon.A2.addh.h16.sat.hh(i32, i32) #1
+declare i32 @llvm.hexagon.A2.subh.l16.ll(i32, i32) #1
+declare i32 @llvm.hexagon.A2.subh.l16.hl(i32, i32) #1
+declare i32 @llvm.hexagon.A2.subh.l16.sat.ll(i32, i32) #1
+declare i32 @llvm.hexagon.A2.subh.l16.sat.hl(i32, i32) #1
+declare i32 @llvm.hexagon.A2.subh.h16.ll(i32, i32) #1
+declare i32 @llvm.hexagon.A2.subh.h16.lh(i32, i32) #1
+declare i32 @llvm.hexagon.A2.subh.h16.hl(i32, i32) #1
+declare i32 @llvm.hexagon.A2.subh.h16.hh(i32, i32) #1
+declare i32 @llvm.hexagon.A2.subh.h16.sat.ll(i32, i32) #1
+declare i32 @llvm.hexagon.A2.subh.h16.sat.lh(i32, i32) #1
+declare i32 @llvm.hexagon.A2.subh.h16.sat.hl(i32, i32) #1
+declare i32 @llvm.hexagon.A2.subh.h16.sat.hh(i32, i32) #1
+declare i64 @llvm.hexagon.A4.bitsplit(i32, i32) #1
+declare i32 @llvm.hexagon.A4.modwrapu(i32, i32) #1
+declare i32 @llvm.hexagon.S2.parityp(i64, i64) #1
+declare i32 @llvm.hexagon.S4.parity(i32, i32) #1
+declare i32 @llvm.hexagon.S4.andi.asl.ri(i32, i32, i32) #1
+declare i32 @llvm.hexagon.S4.ori.asl.ri(i32, i32, i32) #1
+declare i32 @llvm.hexagon.S4.addi.asl.ri(i32, i32, i32) #1
+declare i32 @llvm.hexagon.S4.subi.asl.ri(i32, i32, i32) #1
+declare i32 @llvm.hexagon.S4.andi.lsr.ri(i32, i32, i32) #1
+declare i32 @llvm.hexagon.S4.ori.lsr.ri(i32, i32, i32) #1
+declare i32 @llvm.hexagon.S4.addi.lsr.ri(i32, i32, i32) #1
+declare i32 @llvm.hexagon.S4.subi.lsr.ri(i32, i32, i32) #1
+
+attributes #0 = { nounwind readnone "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #1 = { nounwind readnone }
diff --git a/test/CodeGen/Hexagon/always-ext.ll b/test/CodeGen/Hexagon/always-ext.ll
index 8b4b2f5bf4f..3bf465b6a51 100644
--- a/test/CodeGen/Hexagon/always-ext.ll
+++ b/test/CodeGen/Hexagon/always-ext.ll
@@ -1,5 +1,4 @@
-; XFAIL:
-; RUN: llc -march=hexagon -mcpu=hexagonv4 < %s | FileCheck %s
+; RUN: llc -march=hexagon < %s | FileCheck %s
 
 ; Check that we don't generate an invalid packet with too many instructions
 ; due to a store that has a must-extend operand.
@@ -8,7 +7,7 @@
 ; CHECK: {
 ; CHECK-NOT: call abort
 ; CHECK: memw(##0)
-; CHECK: memw(r{{[0-9+]}}<<#2 + ##4)
+; CHECK: memw(r{{[0-9+]}}{{ *}}<<{{ *}}#2{{ *}}+{{ *}}##4)
 ; CHECK: }
 
 %struct.CuTest.1.28.31.37.40.43.52.55.67.85.111 = type { i8*, void (%struct.CuTest.1.28.31.37.40.43.52.55.67.85.111*)*, i32, i32, i8*, [23 x i32]* }
diff --git a/test/CodeGen/Hexagon/args.ll b/test/CodeGen/Hexagon/args.ll
index aea4ffe2eee..3bfb8b15955 100644
--- a/test/CodeGen/Hexagon/args.ll
+++ b/test/CodeGen/Hexagon/args.ll
@@ -1,8 +1,8 @@
-; RUN: llc -march=hexagon -mcpu=hexagonv4 -disable-dfa-sched -disable-hexagon-misched < %s | FileCheck %s
-; CHECK: memw(r29{{ *}}+{{ *}}#0){{ *}}={{ *}}#7
-; CHECK: r1:0 = combine(#2, #1)
-; CHECK: r3:2 = combine(#4, #3)
+; RUN: llc -march=hexagon < %s | FileCheck %s
 ; CHECK: r5:4 = combine(#6, #5)
+; CHECK: r3:2 = combine(#4, #3)
+; CHECK: r1:0 = combine(#2, #1)
+; CHECK: memw(r29+#0)=#7
 
 
 define void @foo() nounwind {
diff --git a/test/CodeGen/Hexagon/ashift-left-right.ll b/test/CodeGen/Hexagon/ashift-left-right.ll
index 7c41bc7bbf3..bc3e813220d 100644
--- a/test/CodeGen/Hexagon/ashift-left-right.ll
+++ b/test/CodeGen/Hexagon/ashift-left-right.ll
@@ -1,4 +1,4 @@
-; RUN: llc -march=hexagon -mcpu=hexagonv4 < %s | FileCheck %s
+; RUN: llc -march=hexagon < %s | FileCheck %s
 
 define i32 @foo(i32 %a, i32 %b) nounwind readnone {
 ; CHECK: lsl
diff --git a/test/CodeGen/Hexagon/barrier-flag.ll b/test/CodeGen/Hexagon/barrier-flag.ll
new file mode 100644
index 00000000000..e70a56bae02
--- /dev/null
+++ b/test/CodeGen/Hexagon/barrier-flag.ll
@@ -0,0 +1,125 @@
+; RUN: llc -O2 < %s
+; Check for successful compilation. It originally caused an abort due to
+; the "isBarrier" flag set on instructions that were not meant to have it.
+
+target datalayout = "e-m:e-p:32:32-i1:32-i64:64-a:0-v32:32-n16:32"
+target triple = "hexagon"
+
+; Function Attrs: nounwind optsize readnone
+define void @dummy() #0 {
+entry:
+  ret void
+}
+
+; Function Attrs: nounwind optsize
+define void @conv3x3(i8* nocapture readonly %inp, i8* nocapture readonly %mask, i32 %shift, i8* nocapture %outp, i32 %width) #1 {
+entry:
+  %cmp381 = icmp sgt i32 %width, 0
+  %arrayidx16.gep = getelementptr i8, i8* %mask, i32 4
+  %arrayidx19.gep = getelementptr i8, i8* %mask, i32 8
+  br label %for.body
+
+for.body:                                         ; preds = %for.inc48, %entry
+  %i.086 = phi i32 [ 0, %entry ], [ %inc49, %for.inc48 ]
+  %mul = mul nsw i32 %i.086, %width
+  %arrayidx.sum = add i32 %mul, %width
+  br i1 %cmp381, label %for.cond5.preheader.lr.ph, label %for.inc48
+
+for.cond5.preheader.lr.ph:                        ; preds = %for.body
+  %add.ptr.sum = add i32 %arrayidx.sum, %width
+  %add.ptr1 = getelementptr inbounds i8, i8* %inp, i32 %add.ptr.sum
+  %add.ptr = getelementptr inbounds i8, i8* %inp, i32 %arrayidx.sum
+  %arrayidx = getelementptr inbounds i8, i8* %inp, i32 %mul
+  %arrayidx44.gep = getelementptr i8, i8* %outp, i32 %mul
+  br label %for.cond5.preheader
+
+for.cond5.preheader:                              ; preds = %if.end40, %for.cond5.preheader.lr.ph
+  %arrayidx44.phi = phi i8* [ %arrayidx44.gep, %for.cond5.preheader.lr.ph ], [ %arrayidx44.inc, %if.end40 ]
+  %j.085 = phi i32 [ 0, %for.cond5.preheader.lr.ph ], [ %inc46, %if.end40 ]
+  %IN1.084 = phi i8* [ %arrayidx, %for.cond5.preheader.lr.ph ], [ %incdec.ptr, %if.end40 ]
+  %IN2.083 = phi i8* [ %add.ptr, %for.cond5.preheader.lr.ph ], [ %incdec.ptr33, %if.end40 ]
+  %IN3.082 = phi i8* [ %add.ptr1, %for.cond5.preheader.lr.ph ], [ %incdec.ptr34, %if.end40 ]
+  br label %for.body7
+
+for.body7:                                        ; preds = %for.body7, %for.cond5.preheader
+  %arrayidx8.phi = phi i8* [ %IN1.084, %for.cond5.preheader ], [ %arrayidx8.inc, %for.body7 ]
+  %arrayidx9.phi = phi i8* [ %IN2.083, %for.cond5.preheader ], [ %arrayidx9.inc, %for.body7 ]
+  %arrayidx11.phi = phi i8* [ %IN3.082, %for.cond5.preheader ], [ %arrayidx11.inc, %for.body7 ]
+  %arrayidx13.phi = phi i8* [ %mask, %for.cond5.preheader ], [ %arrayidx13.inc, %for.body7 ]
+  %arrayidx16.phi = phi i8* [ %arrayidx16.gep, %for.cond5.preheader ], [ %arrayidx16.inc, %for.body7 ]
+  %arrayidx19.phi = phi i8* [ %arrayidx19.gep, %for.cond5.preheader ], [ %arrayidx19.inc, %for.body7 ]
+  %k.080 = phi i32 [ 0, %for.cond5.preheader ], [ %inc, %for.body7 ]
+  %sum.079 = phi i32 [ 0, %for.cond5.preheader ], [ %add32, %for.body7 ]
+  %0 = load i8, i8* %arrayidx8.phi, align 1, !tbaa !1
+  %1 = load i8, i8* %arrayidx9.phi, align 1, !tbaa !1
+  %2 = load i8, i8* %arrayidx11.phi, align 1, !tbaa !1
+  %3 = load i8, i8* %arrayidx13.phi, align 1, !tbaa !1
+  %4 = load i8, i8* %arrayidx16.phi, align 1, !tbaa !1
+  %5 = load i8, i8* %arrayidx19.phi, align 1, !tbaa !1
+  %conv21 = zext i8 %0 to i32
+  %conv22 = sext i8 %3 to i32
+  %mul23 = mul nsw i32 %conv22, %conv21
+  %conv24 = zext i8 %1 to i32
+  %conv25 = sext i8 %4 to i32
+  %mul26 = mul nsw i32 %conv25, %conv24
+  %conv27 = zext i8 %2 to i32
+  %conv28 = sext i8 %5 to i32
+  %mul29 = mul nsw i32 %conv28, %conv27
+  %add30 = add i32 %mul23, %sum.079
+  %add31 = add i32 %add30, %mul26
+  %add32 = add i32 %add31, %mul29
+  %inc = add nsw i32 %k.080, 1
+  %exitcond = icmp eq i32 %inc, 3
+  %arrayidx8.inc = getelementptr i8, i8* %arrayidx8.phi, i32 1
+  %arrayidx9.inc = getelementptr i8, i8* %arrayidx9.phi, i32 1
+  %arrayidx11.inc = getelementptr i8, i8* %arrayidx11.phi, i32 1
+  %arrayidx13.inc = getelementptr i8, i8* %arrayidx13.phi, i32 1
+  %arrayidx16.inc = getelementptr i8, i8* %arrayidx16.phi, i32 1
+  %arrayidx19.inc = getelementptr i8, i8* %arrayidx19.phi, i32 1
+  br i1 %exitcond, label %for.end, label %for.body7
+
+for.end:                                          ; preds = %for.body7
+  %incdec.ptr = getelementptr inbounds i8, i8* %IN1.084, i32 1
+  %incdec.ptr33 = getelementptr inbounds i8, i8* %IN2.083, i32 1
+  %incdec.ptr34 = getelementptr inbounds i8, i8* %IN3.082, i32 1
+  %shr = ashr i32 %add32, %shift
+  %cmp35 = icmp slt i32 %shr, 0
+  br i1 %cmp35, label %if.end40, label %if.end
+
+if.end:                                           ; preds = %for.end
+  %cmp37 = icmp sgt i32 %shr, 255
+  br i1 %cmp37, label %if.then39, label %if.end40
+
+if.then39:                                        ; preds = %if.end
+  br label %if.end40
+
+if.end40:                                         ; preds = %for.end, %if.then39, %if.end
+  %sum.2 = phi i32 [ 255, %if.then39 ], [ %shr, %if.end ], [ 0, %for.end ]
+  %conv41 = trunc i32 %sum.2 to i8
+  store i8 %conv41, i8* %arrayidx44.phi, align 1, !tbaa !1
+  %inc46 = add nsw i32 %j.085, 1
+  %exitcond87 = icmp eq i32 %inc46, %width
+  %arrayidx44.inc = getelementptr i8, i8* %arrayidx44.phi, i32 1
+  br i1 %exitcond87, label %for.inc48.loopexit, label %for.cond5.preheader
+
+for.inc48.loopexit:                               ; preds = %if.end40
+  br label %for.inc48
+
+for.inc48:                                        ; preds = %for.inc48.loopexit, %for.body
+  %inc49 = add nsw i32 %i.086, 1
+  %exitcond88 = icmp eq i32 %inc49, 2
+  br i1 %exitcond88, label %for.end50, label %for.body
+
+for.end50:                                        ; preds = %for.inc48
+  ret void
+}
+
+attributes #0 = { nounwind optsize readnone "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #1 = { nounwind optsize "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+
+!llvm.ident = !{!0}
+
+!0 = !{!"Clang 3.1"}
+!1 = !{!2, !2, i64 0}
+!2 = !{!"omnipotent char", !3, i64 0}
+!3 = !{!"Simple C/C++ TBAA"}
diff --git a/test/CodeGen/Hexagon/base-offset-addr.ll b/test/CodeGen/Hexagon/base-offset-addr.ll
new file mode 100644
index 00000000000..30410fe9254
--- /dev/null
+++ b/test/CodeGen/Hexagon/base-offset-addr.ll
@@ -0,0 +1,15 @@
+; RUN: llc -march=hexagon -enable-aa-sched-mi < %s
+; REQUIRES: asserts
+
+; Make sure the base is a register and not an address.
+
+define fastcc void @Get_lsp_pol(i32* nocapture %f) #0 {
+entry:
+  %f5 = alloca i32, align 4
+  %arrayidx103 = getelementptr inbounds i32, i32* %f, i32 4
+  store i32 0, i32* %arrayidx103, align 4
+  %f5.0.load185 = load volatile i32, i32* %f5, align 4
+  ret void
+}
+
+attributes #0 = { nounwind "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf"="true" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "unsafe-fp-math"="false" "use-soft-float"="false" }
diff --git a/test/CodeGen/Hexagon/base-offset-post.ll b/test/CodeGen/Hexagon/base-offset-post.ll
new file mode 100644
index 00000000000..a6e4cdd34a0
--- /dev/null
+++ b/test/CodeGen/Hexagon/base-offset-post.ll
@@ -0,0 +1,30 @@
+; RUN: llc -march=hexagon -mcpu=hexagonv5 < %s
+; REQUIRES: asserts
+
+; Test that the accessSize is set on a post-increment store. If not, an assert
+; is triggered in getBaseAndOffset()
+
+%struct.A = type { i8, i32, i32, i32, [10 x i32], [10 x i32], [80 x i32], [80 x i32], [8 x i32], i32, i16, i16, i16, i16, i16, i16, i16, i16, i16, i16, i16 }
+
+; Function Attrs: nounwind
+define fastcc void @Decoder_amr(i8 zeroext %mode) #0 {
+entry:
+  br label %for.cond64.preheader.i
+
+for.cond64.preheader.i:
+  %i.1984.i = phi i32 [ 0, %entry ], [ %inc166.i.1, %for.cond64.preheader.i ]
+  %inc166.i = add nsw i32 %i.1984.i, 1
+  %arrayidx71.i1422.1 = getelementptr inbounds %struct.A, %struct.A* undef, i32 0, i32 7, i32 %inc166.i
+  %storemerge800.i.1 = select i1 undef, i32 1310, i32 undef
+  %sub156.i.1 = sub nsw i32 0, %storemerge800.i.1
+  %sub156.storemerge800.i.1 = select i1 undef, i32 %storemerge800.i.1, i32 %sub156.i.1
+  store i32 %sub156.storemerge800.i.1, i32* %arrayidx71.i1422.1, align 4
+  store i32 0, i32* undef, align 4
+  %inc166.i.1 = add nsw i32 %i.1984.i, 2
+  br label %for.cond64.preheader.i
+
+if.end:
+  ret void
+}
+
+attributes #0 = { nounwind "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf"="true" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "unsafe-fp-math"="false" "use-soft-float"="false" }
diff --git a/test/CodeGen/Hexagon/block-addr.ll b/test/CodeGen/Hexagon/block-addr.ll
index 902765e42ef..eda167a67f2 100644
--- a/test/CodeGen/Hexagon/block-addr.ll
+++ b/test/CodeGen/Hexagon/block-addr.ll
@@ -1,7 +1,8 @@
 ; RUN: llc -march=hexagon < %s | FileCheck %s
 
-; CHECK: r{{[0-9]+}} = CONST32(#.LJTI{{[0-9]+_[0-9]+}})
-; CHECK: r{{[0-9]+}} = memw(r{{[0-9]+}} + r{{[0-9]+<<#[0-9]+}})
+; Allow combine(..##JTI..):
+; CHECK: r{{[0-9]+}}{{.*}} = {{.*}}#.LJTI
+; CHECK: r{{[0-9]+}} = memw(r{{[0-9]+}}{{ *}}+{{ *}}r{{[0-9]+<<#[0-9]+}})
 ; CHECK: jumpr r{{[0-9]+}}
 
 define void @main() #0 {
diff --git a/test/CodeGen/Hexagon/bugAsmHWloop.ll b/test/CodeGen/Hexagon/bugAsmHWloop.ll
new file mode 100644
index 00000000000..c7e95ed0566
--- /dev/null
+++ b/test/CodeGen/Hexagon/bugAsmHWloop.ll
@@ -0,0 +1,71 @@
+; RUN: llc -march=hexagon < %s | FileCheck %s
+
+; CHECK: {
+; CHECK: loop0(.LBB
+; CHECK-NOT: loop0(##.LBB
+
+target datalayout = "e-p:32:32:32-i64:64:64-i32:32:32-i16:16:16-i1:32:32-f64:64:64-f32:32:32-v64:64:64-v32:32:32-a0:0-n16:32"
+target triple = "hexagon"
+
+define i32 @q6zip_uncompress(i8* %out_buf, i32* %out_buf_size, i8* %in_buf, i32 %in_buf_size, i8* nocapture %dict, i32 %dict_size) nounwind {
+entry:
+  %0 = bitcast i8* %in_buf to i32*
+  %incdec.ptr = getelementptr inbounds i8, i8* %in_buf, i32 4
+  %1 = load i32, i32* %0, align 4, !tbaa !0
+  %2 = ptrtoint i8* %incdec.ptr to i32
+  %and.i = and i32 %2, 31
+  %sub.i = sub i32 %2, %and.i
+  %3 = inttoptr i32 %sub.i to i8*
+  %add.i = add i32 %in_buf_size, 31
+  %sub2.i = add i32 %add.i, %and.i
+  %div.i = lshr i32 %sub2.i, 5
+  %4 = tail call i32 @llvm.hexagon.A2.combine.ll(i32 32, i32 %div.i) nounwind
+  %5 = tail call i64 @llvm.hexagon.A4.combineir(i32 32, i32 %4) nounwind
+  tail call void asm sideeffect "l2fetch($0,$1)", "r,r,~{memory}"(i8* %3, i64 %5) nounwind, !srcloc !3
+  %6 = ptrtoint i8* %out_buf to i32
+  br label %for.body.i
+
+for.body.i:                                       ; preds = %for.body.i, %entry
+  %i.02.i = phi i32 [ 0, %entry ], [ %inc.i, %for.body.i ]
+  %addr.addr.01.i = phi i32 [ %6, %entry ], [ %add.i14, %for.body.i ]
+  tail call void asm sideeffect "dczeroa($0)", "r"(i32 %addr.addr.01.i) nounwind, !srcloc !4
+  %add.i14 = add i32 %addr.addr.01.i, 32
+  %inc.i = add i32 %i.02.i, 1
+  %exitcond.i = icmp eq i32 %inc.i, 128
+  br i1 %exitcond.i, label %while.cond.preheader, label %for.body.i
+
+while.cond.preheader:                             ; preds = %for.body.i
+  %and = and i32 %1, 3
+  switch i32 %and, label %infloop.preheader [
+    i32 0, label %exit_inflate.split
+    i32 2, label %if.then.preheader
+  ]
+
+if.then.preheader:                                ; preds = %while.cond.preheader
+  br label %if.then
+
+infloop.preheader:                                ; preds = %while.cond.preheader
+  br label %infloop
+
+if.then:                                          ; preds = %if.then.preheader, %if.then
+  tail call void @llvm.prefetch(i8* %incdec.ptr, i32 0, i32 3, i32 1)
+  br label %if.then
+
+exit_inflate.split:                               ; preds = %while.cond.preheader
+  ret i32 0
+
+infloop:                                          ; preds = %infloop.preheader, %infloop
+  br label %infloop
+}
+
+declare void @llvm.prefetch(i8* nocapture, i32, i32, i32) nounwind
+
+declare i64 @llvm.hexagon.A4.combineir(i32, i32) nounwind readnone
+
+declare i32 @llvm.hexagon.A2.combine.ll(i32, i32) nounwind readnone
+
+!0 = !{!"long", !1}
+!1 = !{!"omnipotent char", !2}
+!2 = !{!"Simple C/C++ TBAA"}
+!3 = !{i32 18362}
+!4 = !{i32 18893}
diff --git a/test/CodeGen/Hexagon/calling-conv-2.ll b/test/CodeGen/Hexagon/calling-conv-2.ll
new file mode 100644
index 00000000000..3c68c88bd71
--- /dev/null
+++ b/test/CodeGen/Hexagon/calling-conv-2.ll
@@ -0,0 +1,13 @@
+; RUN: llc -march=hexagon -mcpu=hexagonv5 <%s | \
+; RUN:   FileCheck %s --check-prefix=CHECK-ONE
+
+%struct.test_struct = type { i32, i8, i64 }
+
+; CHECK-ONE:    r1 = #45
+define void @foo(%struct.test_struct* noalias nocapture sret %agg.result, i32 %a) #0 {
+entry:
+  call void @bar(%struct.test_struct* sret %agg.result, i32 45) #2
+  ret void
+}
+
+declare void @bar(%struct.test_struct* sret, i32) #1
diff --git a/test/CodeGen/Hexagon/cext-valid-packet1.ll b/test/CodeGen/Hexagon/cext-valid-packet1.ll
index a479d37e4ae..36abc59f5e3 100644
--- a/test/CodeGen/Hexagon/cext-valid-packet1.ll
+++ b/test/CodeGen/Hexagon/cext-valid-packet1.ll
@@ -1,4 +1,4 @@
-; RUN: llc -march=hexagon -mcpu=hexagonv4 < %s | FileCheck %s
+; RUN: llc -march=hexagon < %s | FileCheck %s
 
 ; Check that the packetizer generates valid packets with constant
 ; extended instructions.
diff --git a/test/CodeGen/Hexagon/cext-valid-packet2.ll b/test/CodeGen/Hexagon/cext-valid-packet2.ll
index 2eba7432996..9f03ef1309e 100644
--- a/test/CodeGen/Hexagon/cext-valid-packet2.ll
+++ b/test/CodeGen/Hexagon/cext-valid-packet2.ll
@@ -1,43 +1,16 @@
-; RUN: llc -march=hexagon -mcpu=hexagonv4 < %s | FileCheck %s
+; RUN: llc -march=hexagon < %s | FileCheck %s
 ; Check that the packetizer generates valid packets with constant
 ; extended add and base+offset store instructions.
 
-; CHECK: {
-; CHECK-NEXT: r{{[0-9]+}}{{ *}}={{ *}}add(r{{[0-9]+}}, ##{{[0-9]+}})
-; CHECK-NEXT: memw(r{{[0-9]+}}+{{ *}}##{{[0-9]+}}){{ *}}={{ *}}r{{[0-9]+}}.new
+; CHECK: r{{[0-9]+}}{{ *}}={{ *}}add(r{{[0-9]+}},{{ *}}##200000)
+; CHECK-NEXT: memw(r{{[0-9]+}}{{ *}}+{{ *}}##12000){{ *}}={{ *}}r{{[0-9]+}}.new
 ; CHECK-NEXT: }
 
-define i32 @test(i32* nocapture %a, i32* nocapture %b, i32 %c) nounwind {
+define void @test(i32* nocapture %a, i32* nocapture %b, i32 %c) nounwind {
 entry:
-  %add = add nsw i32 %c, 200002
   %0 = load i32, i32* %a, align 4
   %add1 = add nsw i32 %0, 200000
   %arrayidx2 = getelementptr inbounds i32, i32* %a, i32 3000
   store i32 %add1, i32* %arrayidx2, align 4
-  %1 = load i32, i32* %b, align 4
-  %add4 = add nsw i32 %1, 200001
-  %arrayidx5 = getelementptr inbounds i32, i32* %a, i32 1
-  store i32 %add4, i32* %arrayidx5, align 4
-  %arrayidx7 = getelementptr inbounds i32, i32* %b, i32 1
-  %2 = load i32, i32* %arrayidx7, align 4
-  %cmp = icmp sgt i32 %add4, %2
-  br i1 %cmp, label %if.then, label %if.else
-
-if.then:                                          ; preds = %entry
-  %arrayidx8 = getelementptr inbounds i32, i32* %a, i32 2
-  %3 = load i32, i32* %arrayidx8, align 4
-  %arrayidx9 = getelementptr inbounds i32, i32* %b, i32 2000
-  %4 = load i32, i32* %arrayidx9, align 4
-  %sub = sub nsw i32 %3, %4
-  %arrayidx10 = getelementptr inbounds i32, i32* %a, i32 4000
-  store i32 %sub, i32* %arrayidx10, align 4
-  br label %if.end
-
-if.else:                                          ; preds = %entry
-  %arrayidx11 = getelementptr inbounds i32, i32* %b, i32 3200
-  store i32 %add, i32* %arrayidx11, align 4
-  br label %if.end
-
-if.end:                                           ; preds = %if.else, %if.then
-  ret i32 %add
+  ret void
 }
diff --git a/test/CodeGen/Hexagon/cext.ll b/test/CodeGen/Hexagon/cext.ll
new file mode 100644
index 00000000000..6daba8cc959
--- /dev/null
+++ b/test/CodeGen/Hexagon/cext.ll
@@ -0,0 +1,16 @@
+; RUN: llc -march=hexagon < %s | FileCheck %s
+; CHECK: memub(r{{[0-9]+}}{{ *}}<<{{ *}}#1{{ *}}+{{ *}}##a)
+
+@a = external global [5 x [2 x i8]]
+
+define zeroext i8 @foo(i8 zeroext %l) nounwind readonly {
+for.end:
+  %idxprom = zext i8 %l to i32
+  %arrayidx1 = getelementptr inbounds [5 x [2 x i8]], [5 x [2 x i8]]* @a, i32 0, i32 %idxprom, i32 0
+  %0 = load i8, i8* %arrayidx1, align 1
+  %conv = zext i8 %0 to i32
+  %mul = mul nsw i32 %conv, 20
+  %conv2 = trunc i32 %mul to i8
+  ret i8 %conv2
+}
+
diff --git a/test/CodeGen/Hexagon/cexti16.ll b/test/CodeGen/Hexagon/cexti16.ll
new file mode 100644
index 00000000000..465cfe40071
--- /dev/null
+++ b/test/CodeGen/Hexagon/cexti16.ll
@@ -0,0 +1,16 @@
+; RUN: llc -march=hexagon < %s | FileCheck %s
+; CHECK: memuh(r{{[0-9]+}}{{ *}}<<{{ *}}#2{{ *}}+{{ *}}##a)
+
+@a = external global [5 x [2 x i16]]
+
+define signext i16 @foo(i16 zeroext %l) nounwind readonly {
+for.end:
+  %idxprom = zext i16 %l to i32
+  %arrayidx1 = getelementptr inbounds [5 x [2 x i16]], [5 x [2 x i16]]* @a, i32 0, i32 %idxprom, i32 0
+  %0 = load i16, i16* %arrayidx1, align 2
+  %conv = zext i16 %0 to i32
+  %mul = mul nsw i32 %conv, 20
+  %conv2 = trunc i32 %mul to i16
+  ret i16 %conv2
+}
+
diff --git a/test/CodeGen/Hexagon/checktabs.ll b/test/CodeGen/Hexagon/checktabs.ll
new file mode 100644
index 00000000000..740433bf824
--- /dev/null
+++ b/test/CodeGen/Hexagon/checktabs.ll
@@ -0,0 +1,8 @@
+; RUN: llc -march=hexagon < %s | FileCheck --strict-whitespace %s
+; Make sure we are emitting tabs as formatting.
+; CHECK:	{
+; CHECK-NEXT:		{{jump|r}}
+define i32 @foobar(i32 %a, i32 %b) {
+  %1 = add i32 %a, %b
+  ret i32 %1
+}
diff --git a/test/CodeGen/Hexagon/cmp-extend.ll b/test/CodeGen/Hexagon/cmp-extend.ll
new file mode 100644
index 00000000000..0bd1fca7394
--- /dev/null
+++ b/test/CodeGen/Hexagon/cmp-extend.ll
@@ -0,0 +1,40 @@
+; RUN: llc -march=hexagon -mcpu=hexagonv5 < %s | FileCheck %s
+
+%struct.RESULTS_S.A = type { i16, i16, i16, [4 x i8*], i32, i32, i32, %struct.list_head_s.B*, %struct.MAT_PARAMS_S.D, i16, i16, i16, i16, i16, %struct.CORE_PORTABLE_S.E }
+%struct.list_head_s.B = type { %struct.list_head_s.B*, %struct.list_data_s.C* }
+%struct.list_data_s.C = type { i16, i16 }
+%struct.MAT_PARAMS_S.D = type { i32, i16*, i16*, i32* }
+%struct.CORE_PORTABLE_S.E = type { i8 }
+
+; Test that we don't generate a zero extend in this case. Instead we generate
+; a single sign extend instead of two zero extends.
+
+; CHECK-NOT: zxth
+
+; Function Attrs: nounwind
+define void @core_bench_list(%struct.RESULTS_S.A* %res) #0 {
+entry:
+  %seed3 = getelementptr inbounds %struct.RESULTS_S.A, %struct.RESULTS_S.A* %res, i32 0, i32 2
+  %0 = load i16, i16* %seed3, align 2
+  %cmp364 = icmp sgt i16 %0, 0
+  br i1 %cmp364, label %for.body, label %while.body19.i160
+
+for.body:
+  %i.0370 = phi i16 [ %inc50, %if.then ], [ 0, %entry ]
+  br i1 undef, label %if.then, label %while.body.i273
+
+while.body.i273:
+  %tobool.i272 = icmp eq %struct.list_head_s.B* undef, null
+  br i1 %tobool.i272, label %if.then, label %while.body.i273
+
+if.then:
+  %inc50 = add i16 %i.0370, 1
+  %exitcond = icmp eq i16 %inc50, %0
+  br i1 %exitcond, label %while.body19.i160, label %for.body
+
+while.body19.i160:
+  br label %while.body19.i160
+}
+
+attributes #0 = { nounwind "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+
diff --git a/test/CodeGen/Hexagon/cmp-promote.ll b/test/CodeGen/Hexagon/cmp-promote.ll
new file mode 100644
index 00000000000..7811b7e729c
--- /dev/null
+++ b/test/CodeGen/Hexagon/cmp-promote.ll
@@ -0,0 +1,72 @@
+; RUN: llc -march=hexagon < %s | FileCheck %s
+;
+; Bug 6714. Use sign-extend to promote the arguments for compare
+; equal/not-equal for 8- and 16-bit types with negative constants.
+
+; CHECK: cmp.eq{{.*}}#-16
+define i32 @foo1(i16 signext %q) nounwind readnone {
+entry:
+  %not.cmp = icmp ne i16 %q, -16
+  %res.0 = zext i1 %not.cmp to i32
+  ret i32 %res.0
+}
+
+; CHECK: cmp.eq{{.*}}#-14
+define i32 @foo2(i16 signext %q) nounwind readnone {
+entry:
+  %cmp = icmp eq i16 %q, -14
+  %res.0 = select i1 %cmp, i32 2, i32 0
+  ret i32 %res.0
+}
+
+; CHECK: cmp.eq{{.*}}#-8
+define i32 @foo3(i8 signext %r) nounwind readnone {
+entry:
+  %cmp = icmp eq i8 %r, -8
+  %res.0 = select i1 %cmp, i32 0, i32 3
+  ret i32 %res.0
+}
+
+; CHECK: cmp.eq{{.*}}#-6
+define i32 @foo4(i8 signext %r) nounwind readnone {
+entry:
+  %cmp = icmp eq i8 %r, -6
+  %res.0 = select i1 %cmp, i32 4, i32 0
+  ret i32 %res.0
+}
+
+; CHECK: cmp.eq{{.*}}#-20
+define i32 @foo5(i32 %s) nounwind readnone {
+entry:
+  %cmp = icmp eq i32 %s, -20
+  %res.0 = select i1 %cmp, i32 0, i32 5
+  ret i32 %res.0
+}
+
+; CHECK: cmp.eq{{.*}}#-18
+define i32 @foo6(i32 %s) nounwind readnone {
+entry:
+  %cmp = icmp eq i32 %s, -18
+  %res.0 = select i1 %cmp, i32 6, i32 0
+  ret i32 %res.0
+}
+
+; CHECK: cmp.eq{{.*}}#10
+define i32 @foo7(i16 signext %q) nounwind readnone {
+entry:
+  %cmp = icmp eq i16 %q, 10
+  %res.0 = select i1 %cmp, i32 7, i32 0
+  ret i32 %res.0
+}
+
+@g = external global i16
+
+; CHECK: cmp.eq{{.*}}#-12
+define i32 @foo8() nounwind readonly {
+entry:
+  %0 = load i16, i16* @g, align 2
+  %cmp = icmp eq i16 %0, -12
+  %res.0 = select i1 %cmp, i32 0, i32 8
+  ret i32 %res.0
+}
+
diff --git a/test/CodeGen/Hexagon/cmp-to-genreg.ll b/test/CodeGen/Hexagon/cmp-to-genreg.ll
index 97cf51ce1a2..d0df1681513 100644
--- a/test/CodeGen/Hexagon/cmp-to-genreg.ll
+++ b/test/CodeGen/Hexagon/cmp-to-genreg.ll
@@ -1,4 +1,4 @@
-; RUN: llc -march=hexagon -mcpu=hexagonv4 < %s | FileCheck %s
+; RUN: llc -march=hexagon < %s | FileCheck %s
 ; Check that we generate compare to general register.
 
 define i32 @compare1(i32 %a) nounwind {
diff --git a/test/CodeGen/Hexagon/cmp-to-predreg.ll b/test/CodeGen/Hexagon/cmp-to-predreg.ll
index 2b65343ab2c..c97a736f10a 100644
--- a/test/CodeGen/Hexagon/cmp-to-predreg.ll
+++ b/test/CodeGen/Hexagon/cmp-to-predreg.ll
@@ -1,4 +1,4 @@
-; RUN: llc -march=hexagon -mcpu=hexagonv4 < %s | FileCheck %s
+; RUN: llc -march=hexagon < %s | FileCheck %s
 ; Check that we generate compare to predicate register.
 
 define i32 @compare1(i32 %a, i32 %b) nounwind {
diff --git a/test/CodeGen/Hexagon/cmp.ll b/test/CodeGen/Hexagon/cmp.ll
new file mode 100644
index 00000000000..c274a787249
--- /dev/null
+++ b/test/CodeGen/Hexagon/cmp.ll
@@ -0,0 +1,161 @@
+; RUN: llc -march=hexagon --filetype=obj < %s  -o - | llvm-objdump -d - | FileCheck %s
+
+; Function Attrs: nounwind
+define i32 @cmpeq(i32 %i) #0 {
+entry:
+  %i.addr = alloca i32, align 4
+  store i32 %i, i32* %i.addr, align 4
+  %0 = load i32, i32* %i.addr, align 4
+  %1 = call i32 @llvm.hexagon.C2.cmpeq(i32 %0, i32 1)
+  ret i32 %1
+}
+; CHECK: { p{{[0-3]}} = cmp.eq(r{{[0-9]}}, r{{[0-9]}})
+
+; Function Attrs: nounwind readnone
+declare i32 @llvm.hexagon.C2.cmpeq(i32, i32) #1
+
+; Function Attrs: nounwind
+define i32 @cmpgt(i32 %i) #0 {
+entry:
+  %i.addr = alloca i32, align 4
+  store i32 %i, i32* %i.addr, align 4
+  %0 = load i32, i32* %i.addr, align 4
+  %1 = call i32 @llvm.hexagon.C2.cmpgt(i32 %0, i32 2)
+  ret i32 %1
+}
+; CHECK: { p{{[0-3]}} = cmp.gt(r{{[0-9]}}, r{{[0-9]}})
+
+; Function Attrs: nounwind readnone
+declare i32 @llvm.hexagon.C2.cmpgt(i32, i32) #1
+
+; Function Attrs: nounwind
+define i32 @cmpgtu(i32 %i) #0 {
+entry:
+  %i.addr = alloca i32, align 4
+  store i32 %i, i32* %i.addr, align 4
+  %0 = load i32, i32* %i.addr, align 4
+  %1 = call i32 @llvm.hexagon.C2.cmpgtu(i32 %0, i32 3)
+  ret i32 %1
+}
+; CHECK: { p{{[0-3]}} = cmp.gtu(r{{[0-9]}}, r{{[0-9]}})
+
+; Function Attrs: nounwind readnone
+declare i32 @llvm.hexagon.C2.cmpgtu(i32, i32) #1
+
+; Function Attrs: nounwind
+define i32 @cmplt(i32 %i) #0 {
+entry:
+  %i.addr = alloca i32, align 4
+  store i32 %i, i32* %i.addr, align 4
+  %0 = load i32, i32* %i.addr, align 4
+  %1 = call i32 @llvm.hexagon.C2.cmplt(i32 %0, i32 4)
+  ret i32 %1
+}
+; CHECK: { p{{[0-3]}} = cmp.gt(r{{[0-9]}}, r{{[0-9]}})
+
+; Function Attrs: nounwind readnone
+declare i32 @llvm.hexagon.C2.cmplt(i32, i32) #1
+
+; Function Attrs: nounwind
+define i32 @cmpltu(i32 %i) #0 {
+entry:
+  %i.addr = alloca i32, align 4
+  store i32 %i, i32* %i.addr, align 4
+  %0 = load i32, i32* %i.addr, align 4
+  %1 = call i32 @llvm.hexagon.C2.cmpltu(i32 %0, i32 5)
+  ret i32 %1
+}
+; CHECK: { p{{[0-3]}} = cmp.gtu(r{{[0-9]}}, r{{[0-9]}})
+
+; Function Attrs: nounwind readnone
+declare i32 @llvm.hexagon.C2.cmpltu(i32, i32) #1
+
+; Function Attrs: nounwind
+define i32 @cmpeqi(i32 %i) #0 {
+entry:
+  %i.addr = alloca i32, align 4
+  store i32 %i, i32* %i.addr, align 4
+  %0 = load i32, i32* %i.addr, align 4
+  %1 = call i32 @llvm.hexagon.C2.cmpeqi(i32 %0, i32 10)
+  ret i32 %1
+}
+; CHECK: { p{{[0-3]}} = cmp.eq(r{{[0-9]}}, {{.*}}#10)
+
+; Function Attrs: nounwind readnone
+declare i32 @llvm.hexagon.C2.cmpeqi(i32, i32) #1
+
+; Function Attrs: nounwind
+define i32 @cmpgti(i32 %i) #0 {
+entry:
+  %i.addr = alloca i32, align 4
+  store i32 %i, i32* %i.addr, align 4
+  %0 = load i32, i32* %i.addr, align 4
+  %1 = call i32 @llvm.hexagon.C2.cmpgti(i32 %0, i32 20)
+  ret i32 %1
+}
+; CHECK: { p{{[0-3]}} = cmp.gt(r{{[0-9]}}, {{.*}}#20)
+
+; Function Attrs: nounwind readnone
+declare i32 @llvm.hexagon.C2.cmpgti(i32, i32) #1
+
+; Function Attrs: nounwind
+define i32 @cmpgtui(i32 %i) #0 {
+entry:
+  %i.addr = alloca i32, align 4
+  store i32 %i, i32* %i.addr, align 4
+  %0 = load i32, i32* %i.addr, align 4
+  %1 = call i32 @llvm.hexagon.C2.cmpgtui(i32 %0, i32 40)
+  ret i32 %1
+}
+; CHECK: { p{{[0-3]}} = cmp.gtu(r{{[0-9]}}, {{.*}}#40)
+
+; Function Attrs: nounwind readnone
+declare i32 @llvm.hexagon.C2.cmpgtui(i32, i32) #1
+
+; Function Attrs: nounwind
+define i32 @cmpgei(i32 %i) #0 {
+entry:
+  %i.addr = alloca i32, align 4
+  store i32 %i, i32* %i.addr, align 4
+  %0 = load i32, i32* %i.addr, align 4
+  %1 = call i32 @llvm.hexagon.C2.cmpgei(i32 %0, i32 3)
+  ret i32 %1
+}
+; CHECK: { p{{[0-3]}} = cmp.gt(r{{[0-9]}}, {{.*}}#2)
+
+; Function Attrs: nounwind readnone
+declare i32 @llvm.hexagon.C2.cmpgei(i32, i32) #1
+
+; Function Attrs: nounwind
+define i32 @cmpgeu(i32 %i) #0 {
+entry:
+  %i.addr = alloca i32, align 4
+  store i32 %i, i32* %i.addr, align 4
+  %0 = load i32, i32* %i.addr, align 4
+  %1 = call i32 @llvm.hexagon.C2.cmpgeui(i32 %0, i32 3)
+  ret i32 %1
+}
+; CHECK: { p{{[0-3]}} = cmp.gtu(r{{[0-9]}}, {{.*}}#2)
+
+; Function Attrs: nounwind readnone
+declare i32 @llvm.hexagon.C2.cmpgeui(i32, i32) #1
+
+; Function Attrs: nounwind
+define i32 @cmpgeu0(i32 %i) #0 {
+entry:
+  %i.addr = alloca i32, align 4
+  store i32 %i, i32* %i.addr, align 4
+  %0 = load i32, i32* %i.addr, align 4
+  %1 = call i32 @llvm.hexagon.C2.cmpgeui(i32 %0, i32 0)
+  ret i32 %1
+}
+; CHECK: { p{{[0-3]}} = cmp.eq(r{{[0-9]}}, r{{[0-9]}})
+
+
+attributes #0 = { nounwind "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #1 = { nounwind readnone }
+
+!llvm.ident = !{!0}
+
+!0 = !{!"Clang 3.1"}
+
diff --git a/test/CodeGen/Hexagon/cmp_pred.ll b/test/CodeGen/Hexagon/cmp_pred.ll
index 39549a1f2d5..ee3f5ddf1f1 100644
--- a/test/CodeGen/Hexagon/cmp_pred.ll
+++ b/test/CodeGen/Hexagon/cmp_pred.ll
@@ -1,4 +1,3 @@
-; XFAIL:
 ; RUN: llc -march=hexagon -mcpu=hexagonv5  < %s | FileCheck %s
 ; Generate various cmpb instruction followed by if (p0) .. if (!p0)...
 target triple = "hexagon"
@@ -61,7 +60,7 @@ entry:
 
 define i32 @Func_3gt(i32 %Enum_Par_Val, i32 %pv2) nounwind readnone {
 entry:
-; CHECK: mux
+; CHECK-NOT: mux
   %cmp = icmp sgt i32 %Enum_Par_Val, %pv2
   %selv = zext i1 %cmp to i32
   ret i32 %selv
diff --git a/test/CodeGen/Hexagon/cmp_pred_reg.ll b/test/CodeGen/Hexagon/cmp_pred_reg.ll
index 39549a1f2d5..ee3f5ddf1f1 100644
--- a/test/CodeGen/Hexagon/cmp_pred_reg.ll
+++ b/test/CodeGen/Hexagon/cmp_pred_reg.ll
@@ -1,4 +1,3 @@
-; XFAIL:
 ; RUN: llc -march=hexagon -mcpu=hexagonv5  < %s | FileCheck %s
 ; Generate various cmpb instruction followed by if (p0) .. if (!p0)...
 target triple = "hexagon"
@@ -61,7 +60,7 @@ entry:
 
 define i32 @Func_3gt(i32 %Enum_Par_Val, i32 %pv2) nounwind readnone {
 entry:
-; CHECK: mux
+; CHECK-NOT: mux
   %cmp = icmp sgt i32 %Enum_Par_Val, %pv2
   %selv = zext i1 %cmp to i32
   ret i32 %selv
diff --git a/test/CodeGen/Hexagon/cmpb-eq.ll b/test/CodeGen/Hexagon/cmpb-eq.ll
new file mode 100644
index 00000000000..e59ed3e51c3
--- /dev/null
+++ b/test/CodeGen/Hexagon/cmpb-eq.ll
@@ -0,0 +1,53 @@
+; RUN: llc -march=hexagon < %s | FileCheck %s
+; CHECK-NOT: cmpb.eq(r{{[0-9]+}}, #-1)
+
+target datalayout = "e-p:32:32:32-i64:64:64-i32:32:32-i16:16:16-i1:32:32-f64:64:64-f32:32:32-v64:64:64-v32:32:32-a0:0-n16:32"
+target triple = "hexagon"
+
+%struct.wms_address_s = type { i32, i32, i32, i32, i8, [48 x i8] }
+
+define zeroext i8 @qmi_wmsi_bin_to_addr(i8* %str, i8 zeroext %len, %struct.wms_address_s* %addr) nounwind optsize {
+entry:
+  %cmp = icmp eq i8* %str, null
+  %cmp2 = icmp eq %struct.wms_address_s* %addr, null
+  %or.cond = or i1 %cmp, %cmp2
+  br i1 %or.cond, label %if.then12, label %if.then
+
+if.then:                                          ; preds = %entry
+  %dec = add i8 %len, -1
+  %cmp3 = icmp ugt i8 %dec, 24
+  %tobool27 = icmp eq i8 %dec, 0
+  %or.cond31 = or i1 %cmp3, %tobool27
+  br i1 %or.cond31, label %if.then12, label %for.body.lr.ph
+
+for.body.lr.ph:                                   ; preds = %if.then
+  %dec626 = add i8 %len, -2
+  br label %for.body
+
+for.body:                                         ; preds = %for.body.lr.ph, %if.end21
+  %indvars.iv = phi i32 [ 0, %for.body.lr.ph ], [ %indvars.iv.next, %if.end21 ]
+  %dec630 = phi i8 [ %dec626, %for.body.lr.ph ], [ %dec6, %if.end21 ]
+  %str.pn = phi i8* [ %str, %for.body.lr.ph ], [ %str.addr.029, %if.end21 ]
+  %str.addr.029 = getelementptr inbounds i8, i8* %str.pn, i32 1
+  %0 = load i8, i8* %str.addr.029, align 1, !tbaa !0
+  %cmp10 = icmp ugt i8 %0, -49
+  br i1 %cmp10, label %if.then12.loopexit, label %if.end21
+
+if.then12.loopexit:                               ; preds = %if.end21, %for.body
+  br label %if.then12
+
+if.then12:                                        ; preds = %if.then12.loopexit, %if.then, %entry
+  ret i8 0
+
+if.end21:                                         ; preds = %for.body
+  %shr24 = lshr i8 %0, 4
+  %arrayidx = getelementptr inbounds %struct.wms_address_s, %struct.wms_address_s* %addr, i32 0, i32 5, i32 %indvars.iv
+  store i8 %shr24, i8* %arrayidx, align 1, !tbaa !0
+  %dec6 = add i8 %dec630, -1
+  %tobool = icmp eq i8 %dec630, 0
+  %indvars.iv.next = add i32 %indvars.iv, 1
+  br i1 %tobool, label %if.then12.loopexit, label %for.body
+}
+
+!0 = !{!"omnipotent char", !1}
+!1 = !{!"Simple C/C++ TBAA"}
diff --git a/test/CodeGen/Hexagon/cmpb_pred.ll b/test/CodeGen/Hexagon/cmpb_pred.ll
index 1a43e629169..d5a76ff129e 100644
--- a/test/CodeGen/Hexagon/cmpb_pred.ll
+++ b/test/CodeGen/Hexagon/cmpb_pred.ll
@@ -1,4 +1,3 @@
-; XFAIL:
 ; RUN: llc -march=hexagon -mcpu=hexagonv5  < %s | FileCheck %s
 ; Generate various cmpb instruction followed by if (p0) .. if (!p0)...
 target triple = "hexagon"
@@ -64,7 +63,7 @@ entry:
 
 define i32 @Func_3g(i32) nounwind readnone {
 entry:
-; CHECK: mux
+; CHECK-NOT: mux
   %conv = and i32 %0, 255
   %cmp = icmp ult i32 %conv, 3
   %selv = zext i1 %cmp to i32
diff --git a/test/CodeGen/Hexagon/common-gep-basic.ll b/test/CodeGen/Hexagon/common-gep-basic.ll
new file mode 100644
index 00000000000..317bf868d0f
--- /dev/null
+++ b/test/CodeGen/Hexagon/common-gep-basic.ll
@@ -0,0 +1,37 @@
+; RUN: llc -O2 -march=hexagon < %s | FileCheck %s
+; CHECK: mpyi
+; CHECK-NOT: mpyi
+; The mpyis from the two GEPs should be commoned out.
+
+target datalayout = "e-m:e-p:32:32-i64:64-a:0-v32:32-n16:32"
+target triple = "hexagon-unknown--elf"
+
+%struct.s_t = type { %struct.anon, i32 }
+%struct.anon = type { i32, [5 x i32] }
+
+@g = common global [100 x %struct.s_t] zeroinitializer, align 8
+
+; Function Attrs: nounwind
+define void @foo(i32 %x) #0 {
+entry:
+  %cmp = icmp slt i32 %x, 90
+  br i1 %cmp, label %if.then, label %if.else
+
+if.then:                                          ; preds = %entry
+  %arrayidx1 = getelementptr inbounds [100 x %struct.s_t], [100 x %struct.s_t]* @g, i32 0, i32 %x, i32 0, i32 1, i32 2
+  tail call void @bar(i32* %arrayidx1) #0
+  br label %if.end
+
+if.else:                                          ; preds = %entry
+  %arrayidx5 = getelementptr inbounds [100 x %struct.s_t], [100 x %struct.s_t]* @g, i32 0, i32 %x, i32 0, i32 1, i32 3
+  tail call void @bar(i32* %arrayidx5) #0
+  br label %if.end
+
+if.end:                                           ; preds = %if.else, %if.then
+  ret void
+}
+
+declare void @bar(i32*) #0
+
+attributes #0 = { nounwind }
+
diff --git a/test/CodeGen/Hexagon/common-gep-icm.ll b/test/CodeGen/Hexagon/common-gep-icm.ll
new file mode 100644
index 00000000000..bc5719dfe1d
--- /dev/null
+++ b/test/CodeGen/Hexagon/common-gep-icm.ll
@@ -0,0 +1,76 @@
+; RUN: llc -O2 -march=hexagon < %s | FileCheck %s
+; Rely on the comments generated by llc. Make sure there are no add/addasl
+; instructions in while.body13 (before the loads).
+; CHECK: while.body13
+; CHECK-NOT: add
+; CHECK: memw
+
+%struct.1 = type { i32, i32 }
+%struct.2 = type { [24 x i32], [24 x i32], [24 x i32], [24 x i32], [24 x i32], [24 x i32], [24 x i32], [24 x i32], [24 x i32], [24 x i32], [24 x i32], [24 x i32], [24 x i32], [24 x i32], [24 x i32], [24 x i32], [24 x i32], [3 x i32], [24 x i32], [8 x %struct.1], [5 x i32] }
+
+@A1 = global i64 zeroinitializer
+@A2 = global i64 zeroinitializer
+@B1 = global i32 zeroinitializer
+@B2 = global i32 zeroinitializer
+@C1 = global i8 zeroinitializer
+
+declare i32 @llvm.hexagon.S2.cl0(i32) nounwind readnone
+declare i32 @llvm.hexagon.S2.setbit.r(i32, i32) nounwind readnone
+declare i64 @llvm.hexagon.M2.vmpy2s.s0(i32, i32) nounwind readnone
+declare i64 @llvm.hexagon.M2.vmac2s.s0(i64, i32, i32) nounwind readnone
+declare i64 @llvm.hexagon.A2.vaddws(i64, i64) nounwind readnone
+declare i64 @llvm.hexagon.A2.vsubws(i64, i64) nounwind readnone
+declare i32 @llvm.hexagon.A4.modwrapu(i32, i32) nounwind readnone
+
+define void @foo(i32 %n) nounwind {
+entry:
+  br label %while.body
+
+while.body:
+  %count = phi i32 [ 0, %entry ], [ %next, %while.end ]
+  %idx = phi i32 [ 0, %entry ], [ %15, %while.end ]
+  %0 = load i32, i32* @B1, align 4
+  %1 = load i32, i32* @B2, align 8
+  %2 = and i32 %1, %0
+  br label %while.body13
+
+while.body13:                                     ; preds = %while.body, %if.end
+  %3 = phi i64 [ %13, %if.end ], [ 0, %while.body ]
+  %4 = phi i64 [ %14, %if.end ], [ 0, %while.body ]
+  %m = phi i32 [ %6, %if.end ], [ %2, %while.body ]
+  %5 = tail call i32 @llvm.hexagon.S2.cl0(i32 %m)
+  %6 = tail call i32 @llvm.hexagon.S2.setbit.r(i32 %m, i32 %5)
+  %cgep85 = getelementptr [10 x %struct.2], [10 x %struct.2]* inttoptr (i32 -121502345 to [10 x %struct.2]*), i32 0, i32 %idx
+  %cgep90 = getelementptr %struct.2, %struct.2* %cgep85, i32 0, i32 12, i32 %5
+  %7 = load i32, i32* %cgep90, align 4
+  %8 = tail call i64 @llvm.hexagon.M2.vmpy2s.s0(i32 %7, i32 %7)
+  %cgep91 = getelementptr %struct.2, %struct.2* %cgep85, i32 0, i32 13, i32 %5
+  %9 = load i32, i32* %cgep91, align 4
+  %10 = tail call i64 @llvm.hexagon.M2.vmac2s.s0(i64 %8, i32 %9, i32 %9)
+  %11 = load i8, i8* @C1, align 1
+  %and24 = and i8 %11, 1
+  %cmp = icmp eq i8 %and24, 0
+  br i1 %cmp, label %if.then, label %if.end
+
+if.then:                                          ; preds = %while.body13
+  %12 = tail call i64 @llvm.hexagon.A2.vaddws(i64 %3, i64 %10)
+  store i64 %12, i64* @A1, align 8
+  br label %if.end
+
+if.end:                                           ; preds = %if.then, %while.body13
+  %13 = phi i64 [ %12, %if.then ], [ %3, %while.body13 ]
+  %14 = tail call i64 @llvm.hexagon.A2.vsubws(i64 %4, i64 %10)
+  %tobool12 = icmp eq i32 %6, 0
+  br i1 %tobool12, label %while.end, label %while.body13
+
+while.end:
+  %add40 = add i32 %idx, 1
+  %15 = tail call i32 @llvm.hexagon.A4.modwrapu(i32 %add40, i32 10) nounwind
+  %next = add i32 %count, 1
+  %cc = icmp eq i32 %next, %n
+  br i1 %cc, label %end, label %while.body
+
+end:
+  store i64 %10, i64* @A2, align 8
+  ret void
+}
diff --git a/test/CodeGen/Hexagon/compound.ll b/test/CodeGen/Hexagon/compound.ll
new file mode 100644
index 00000000000..f8d36b8b77d
--- /dev/null
+++ b/test/CodeGen/Hexagon/compound.ll
@@ -0,0 +1,17 @@
+; RUN: llc -march=hexagon -filetype=obj -o - %s | llvm-objdump -d - | FileCheck %s
+
+; CHECK: p0 = cmp.gt(r0,#-1); if (!p0.new) jump:nt
+
+declare void @a()
+declare void @b()
+
+define void @foo(i32 %a) {
+%b = icmp sgt i32 %a, -1
+br i1 %b, label %x, label %y
+x:
+call void @a()
+ret void
+y:
+call void @b()
+ret void
+}
\ No newline at end of file
diff --git a/test/CodeGen/Hexagon/ctlz-cttz-ctpop.ll b/test/CodeGen/Hexagon/ctlz-cttz-ctpop.ll
index e942f8d0c5d..b8f483298f8 100644
--- a/test/CodeGen/Hexagon/ctlz-cttz-ctpop.ll
+++ b/test/CodeGen/Hexagon/ctlz-cttz-ctpop.ll
@@ -1,8 +1,10 @@
 ; RUN: llc -march=hexagon -mcpu=hexagonv4 < %s | FileCheck %s
 
-; CHECK: r{{[0-9]+}}:{{[0-9]+}} |= lsr(r{{[0-9]+}}:{{[0-9]+}}, #4)
-; CHECK: r{{[0-9]+}}:{{[0-9]+}} &= lsr(r{{[0-9]+}}:{{[0-9]+}}, #2)
-; CHECK: r{{[0-9]+}} += lsr(r{{[0-9]+}}, #4)
+; CHECK-DAG: ct0({{r[0-9]*:[0-9]*}})
+; CHECK-DAG: cl0({{r[0-9]*:[0-9]*}})
+; CHECK-DAG: ct0({{r[0-9]*}})
+; CHECK-DAG: cl0({{r[0-9]*}})
+; CHECK-DAG: r{{[0-9]+}} += lsr(r{{[0-9]+}}, #4)
 
 define i32 @foo(i64 %a, i32 %b) nounwind  {
 entry:
diff --git a/test/CodeGen/Hexagon/dualstore.ll b/test/CodeGen/Hexagon/dualstore.ll
index 33d9ce9b935..9f4569d6459 100644
--- a/test/CodeGen/Hexagon/dualstore.ll
+++ b/test/CodeGen/Hexagon/dualstore.ll
@@ -1,12 +1,11 @@
-; RUN: llc -march=hexagon -disable-hexagon-misched < %s | FileCheck %s
+; RUN: llc -march=hexagon -filetype=obj %s -o - | llvm-objdump -d - | FileCheck %s
 ; Check that we generate dual stores in one packet in V4
 
-; CHECK: memw(r{{[0-9]+}}{{ *}}+{{ *}}#{{[0-9]+}}){{ *}}=
-; CHECK-NEXT: memw(r{{[0-9]+}}{{ *}}+{{ *}}#{{[0-9]+}}){{ *}}=
+; CHECK: 00 40 9f 52 529f4000
+; CHECK: 10 10 00 f0 f0001010
 
-define i32 @main(i32 %v, i32* %p1, i32* %p2) nounwind {
-entry:
-  store i32 %v, i32* %p1, align 4
-  store i32 %v, i32* %p2, align 4
-  ret i32 0
+define void @foo(i32* %a, i32* %b) {
+  store i32 0, i32* %a
+  store i32 0, i32* %b
+  ret void
 }
diff --git a/test/CodeGen/Hexagon/duplex.ll b/test/CodeGen/Hexagon/duplex.ll
new file mode 100644
index 00000000000..80fe61ceccc
--- /dev/null
+++ b/test/CodeGen/Hexagon/duplex.ll
@@ -0,0 +1,7 @@
+; RUN: llc -march=hexagon -filetype=obj -o - %s | llvm-objdump -d - | FileCheck %s
+
+; CHECK: c0 3f 00 48 48003fc0
+
+define i32 @foo() {
+ret i32 0
+}
\ No newline at end of file
diff --git a/test/CodeGen/Hexagon/eh_return.ll b/test/CodeGen/Hexagon/eh_return.ll
new file mode 100644
index 00000000000..67649a07afc
--- /dev/null
+++ b/test/CodeGen/Hexagon/eh_return.ll
@@ -0,0 +1,48 @@
+; RUN: llc -O0 -march=hexagon < %s | FileCheck %s
+; Make sure we generate an exception handling return.
+
+; CHECK:         deallocframe
+; CHECK-NEXT:  }
+; CHECK-NEXT:  {
+; CHECK-NEXT:    r29 = add(r29, r28)
+; CHECK-NEXT:  }
+; CHECK-NEXT:  {
+; CHECK-NEXT:    jumpr r31
+; CHECK-NEXT:  }
+
+target datalayout = "e-p:32:32:32-i64:64:64-i32:32:32-i16:16:16-i1:32:32-f64:64:64-f32:32:32-a0:0-n32"
+target triple = "hexagon-unknown-linux-gnu"
+
+%struct.Data = type { i32, i8* }
+
+define i32 @test_eh_return(i32 %a, i32 %b) nounwind {
+entry:
+  %a.addr = alloca i32, align 4
+  %b.addr = alloca i32, align 4
+  %d = alloca %struct.Data, align 4
+  store i32 %a, i32* %a.addr, align 4
+  store i32 %b, i32* %b.addr, align 4
+  %0 = load i32, i32* %a.addr, align 4
+  %1 = load i32, i32* %b.addr, align 4
+  %cmp = icmp sgt i32 %0, %1
+  br i1 %cmp, label %if.then, label %if.else
+
+if.then:                                          ; preds = %entry
+  %2 = load i32, i32* %a.addr, align 4
+  %3 = load i32, i32* %b.addr, align 4
+  %add = add nsw i32 %2, %3
+  ret i32 %add
+
+if.else:                                          ; preds = %entry
+  %call = call i32 @setup(%struct.Data* %d)
+  %_d1 = getelementptr inbounds %struct.Data, %struct.Data* %d, i32 0, i32 0
+  %4 = load i32, i32* %_d1, align 4
+  %_d2 = getelementptr inbounds %struct.Data, %struct.Data* %d, i32 0, i32 1
+  %5 = load i8*, i8** %_d2, align 4
+  call void @llvm.eh.return.i32(i32 %4, i8* %5)
+  unreachable
+}
+
+declare i32 @setup(%struct.Data*)
+
+declare void @llvm.eh.return.i32(i32, i8*) nounwind
diff --git a/test/CodeGen/Hexagon/extract-basic.ll b/test/CodeGen/Hexagon/extract-basic.ll
new file mode 100644
index 00000000000..c75125cedd3
--- /dev/null
+++ b/test/CodeGen/Hexagon/extract-basic.ll
@@ -0,0 +1,76 @@
+; RUN: llc -O2 -march=hexagon < %s | FileCheck %s
+
+; CHECK-DAG: extractu(r{{[0-9]*}}, #3, #4)
+; CHECK-DAG: extractu(r{{[0-9]*}}, #8, #7)
+; CHECK-DAG: extractu(r{{[0-9]*}}, #8, #16)
+
+; C source:
+; typedef struct {
+;   unsigned x1:3;
+;   unsigned x2:7;
+;   unsigned x3:8;
+;   unsigned x4:12;
+;   unsigned x5:2;
+; } structx_t;
+;
+; typedef struct {
+;   unsigned y1:4;
+;   unsigned y2:3;
+;   unsigned y3:9;
+;   unsigned y4:8;
+;   unsigned y5:8;
+; } structy_t;
+;
+; void foo(structx_t *px, structy_t *py) {
+;   px->x1 = py->y1;
+;   px->x2 = py->y2;
+;   px->x3 = py->y3;
+;   px->x4 = py->y4;
+;   px->x5 = py->y5;
+; }
+
+target datalayout = "e-p:32:32:32-i64:64:64-i32:32:32-i16:16:16-i1:32:32-f64:64:64-f32:32:32-v64:64:64-v32:32:32-a0:0-n16:32"
+target triple = "hexagon"
+
+%struct.structx_t = type { i8, i8, i8, i8 }
+%struct.structy_t = type { i8, i8, i8, i8 }
+
+define void @foo(%struct.structx_t* nocapture %px, %struct.structy_t* nocapture %py) nounwind {
+entry:
+  %0 = bitcast %struct.structy_t* %py to i32*
+  %1 = load i32, i32* %0, align 4
+  %bf.value = and i32 %1, 7
+  %2 = bitcast %struct.structx_t* %px to i32*
+  %3 = load i32, i32* %2, align 4
+  %4 = and i32 %3, -8
+  %5 = or i32 %4, %bf.value
+  store i32 %5, i32* %2, align 4
+  %6 = load i32, i32* %0, align 4
+  %7 = lshr i32 %6, 4
+  %bf.clear1 = shl nuw nsw i32 %7, 3
+  %8 = and i32 %bf.clear1, 56
+  %9 = and i32 %5, -1017
+  %10 = or i32 %8, %9
+  store i32 %10, i32* %2, align 4
+  %11 = load i32, i32* %0, align 4
+  %12 = lshr i32 %11, 7
+  %bf.value4 = shl i32 %12, 10
+  %13 = and i32 %bf.value4, 261120
+  %14 = and i32 %10, -262081
+  %15 = or i32 %14, %13
+  store i32 %15, i32* %2, align 4
+  %16 = load i32, i32* %0, align 4
+  %17 = lshr i32 %16, 16
+  %bf.clear5 = shl i32 %17, 18
+  %18 = and i32 %bf.clear5, 66846720
+  %19 = and i32 %15, -1073480641
+  %20 = or i32 %19, %18
+  store i32 %20, i32* %2, align 4
+  %21 = load i32, i32* %0, align 4
+  %22 = lshr i32 %21, 24
+  %23 = shl i32 %22, 30
+  %24 = and i32 %20, 67107903
+  %25 = or i32 %24, %23
+  store i32 %25, i32* %2, align 4
+  ret void
+}
diff --git a/test/CodeGen/Hexagon/fusedandshift.ll b/test/CodeGen/Hexagon/fusedandshift.ll
index 59a1e1d84fc..414574aec40 100644
--- a/test/CodeGen/Hexagon/fusedandshift.ll
+++ b/test/CodeGen/Hexagon/fusedandshift.ll
@@ -1,5 +1,6 @@
-; RUN: llc -march=hexagon -mcpu=hexagonv4  < %s | FileCheck %s
+; RUN: llc -march=hexagon -hexagon-extract=0 < %s | FileCheck %s
 ; Check that we generate fused logical and with shift instruction.
+; Disable "extract" generation, since it may eliminate the and/lsr.
 
 ; CHECK: r{{[0-9]+}} = and(#15, lsr(r{{[0-9]+}}, #{{[0-9]+}})
 
diff --git a/test/CodeGen/Hexagon/hwloop-crit-edge.ll b/test/CodeGen/Hexagon/hwloop-crit-edge.ll
new file mode 100644
index 00000000000..4de4540c142
--- /dev/null
+++ b/test/CodeGen/Hexagon/hwloop-crit-edge.ll
@@ -0,0 +1,58 @@
+; RUN: llc -O3 -march=hexagon -mcpu=hexagonv5 < %s | FileCheck %s
+;
+; Generate hardware loop when loop 'latch' block is different
+; from the loop 'exiting' block.
+
+; CHECK: loop0(.LBB{{.}}_{{.}}, r{{[0-9]+}})
+; CHECK: endloop0
+
+define void @test(i32* nocapture %pFL, i16 signext %nBS, i16* nocapture readonly %pHT) #0 {
+entry:
+  %0 = load i32, i32* %pFL, align 4
+  %1 = tail call i64 @llvm.hexagon.M2.dpmpyss.s0(i32 %0, i32 246)
+  %2 = tail call i64 @llvm.hexagon.S2.asl.r.p(i64 %1, i32 -13)
+  %3 = tail call i32 @llvm.hexagon.A2.sat(i64 %2)
+  store i32 %3, i32* %pFL, align 4
+  %cmp16 = icmp sgt i16 %nBS, 0
+  br i1 %cmp16, label %for.body.lr.ph, label %for.end
+
+for.body.lr.ph:
+  %4 = sext i16 %nBS to i32
+  br label %for.body
+
+for.body:
+  %5 = phi i32 [ %3, %for.body.lr.ph ], [ %.pre, %for.body.for.body_crit_edge ]
+  %arrayidx3.phi = phi i32* [ %pFL, %for.body.lr.ph ], [ %arrayidx3.inc, %for.body.for.body_crit_edge ]
+  %arrayidx5.phi = phi i16* [ %pHT, %for.body.lr.ph ], [ %arrayidx5.inc, %for.body.for.body_crit_edge ]
+  %i.017.pmt = phi i32 [ 1, %for.body.lr.ph ], [ %phitmp, %for.body.for.body_crit_edge ]
+  %6 = load i16, i16* %arrayidx5.phi, align 2
+  %conv6 = sext i16 %6 to i32
+  %7 = tail call i64 @llvm.hexagon.M2.dpmpyss.s0(i32 %5, i32 %conv6)
+  %8 = tail call i64 @llvm.hexagon.S2.asl.r.p(i64 %7, i32 -13)
+  %9 = tail call i32 @llvm.hexagon.A2.sat(i64 %8)
+  store i32 %9, i32* %arrayidx3.phi, align 4
+  %exitcond = icmp eq i32 %i.017.pmt, %4
+  %arrayidx3.inc = getelementptr i32, i32* %arrayidx3.phi, i32 1
+  br i1 %exitcond, label %for.end.loopexit, label %for.body.for.body_crit_edge
+
+for.body.for.body_crit_edge:
+  %arrayidx5.inc = getelementptr i16, i16* %arrayidx5.phi, i32 1
+  %.pre = load i32, i32* %arrayidx3.inc, align 4
+  %phitmp = add i32 %i.017.pmt, 1
+  br label %for.body
+
+for.end.loopexit:
+  br label %for.end
+
+for.end:
+  ret void
+}
+
+declare i32 @llvm.hexagon.A2.sat(i64) #1
+
+declare i64 @llvm.hexagon.S2.asl.r.p(i64, i32) #1
+
+declare i64 @llvm.hexagon.M2.dpmpyss.s0(i32, i32) #1
+
+attributes #0 = { nounwind "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf"="true" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "ssp-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #1 = { nounwind readnone }
diff --git a/test/CodeGen/Hexagon/hwloop-dbg.ll b/test/CodeGen/Hexagon/hwloop-dbg.ll
index 2fa7f529801..66c6662f735 100644
--- a/test/CodeGen/Hexagon/hwloop-dbg.ll
+++ b/test/CodeGen/Hexagon/hwloop-dbg.ll
@@ -5,9 +5,9 @@ target triple = "hexagon"
 
 define void @foo(i32* nocapture %a, i32* nocapture %b) nounwind {
 entry:
-  tail call void @llvm.dbg.value(metadata i32* %a, i64 0, metadata !13, metadata !MDExpression()), !dbg !17
-  tail call void @llvm.dbg.value(metadata i32* %b, i64 0, metadata !14, metadata !MDExpression()), !dbg !18
-  tail call void @llvm.dbg.value(metadata i32 0, i64 0, metadata !15, metadata !MDExpression()), !dbg !19
+  tail call void @llvm.dbg.value(metadata i32* %a, i64 0, metadata !13, metadata !DIExpression()), !dbg !17
+  tail call void @llvm.dbg.value(metadata i32* %b, i64 0, metadata !14, metadata !DIExpression()), !dbg !18
+  tail call void @llvm.dbg.value(metadata i32 0, i64 0, metadata !15, metadata !DIExpression()), !dbg !19
   br label %for.body, !dbg !19
 
 for.body:                                         ; preds = %for.body, %entry
@@ -18,11 +18,11 @@ for.body:                                         ; preds = %for.body, %entry
   %i.02 = phi i32 [ 0, %entry ], [ %inc, %for.body ]
   %b.addr.01 = phi i32* [ %b, %entry ], [ %incdec.ptr, %for.body ]
   %incdec.ptr = getelementptr inbounds i32, i32* %b.addr.01, i32 1, !dbg !21
-  tail call void @llvm.dbg.value(metadata i32* %incdec.ptr, i64 0, metadata !14, metadata !MDExpression()), !dbg !21
+  tail call void @llvm.dbg.value(metadata i32* %incdec.ptr, i64 0, metadata !14, metadata !DIExpression()), !dbg !21
   %0 = load i32, i32* %b.addr.01, align 4, !dbg !21
   store i32 %0, i32* %arrayidx.phi, align 4, !dbg !21
   %inc = add nsw i32 %i.02, 1, !dbg !26
-  tail call void @llvm.dbg.value(metadata i32 %inc, i64 0, metadata !15, metadata !MDExpression()), !dbg !26
+  tail call void @llvm.dbg.value(metadata i32 %inc, i64 0, metadata !15, metadata !DIExpression()), !dbg !26
   %exitcond = icmp eq i32 %inc, 10, !dbg !19
   %arrayidx.inc = getelementptr i32, i32* %arrayidx.phi, i32 1
   br i1 %exitcond, label %for.end, label %for.body, !dbg !19
@@ -37,28 +37,28 @@ declare void @llvm.dbg.value(metadata, i64, metadata, metadata) nounwind readnon
 !llvm.dbg.cu = !{!0}
 !llvm.module.flags = !{!29}
 
-!0 = !MDCompileUnit(language: DW_LANG_C99, producer: "QuIC LLVM Hexagon Clang version 6.1-pre-unknown, (git://git-hexagon-aus.quicinc.com/llvm/clang-mainline.git e9382867661454cdf44addb39430741578e9765c) (llvm/llvm-mainline.git 36412bb1fcf03ed426d4437b41198bae066675ac)", isOptimized: true, emissionKind: 1, file: !28, enums: !2, retainedTypes: !2, subprograms: !3, globals: !2)
+!0 = !DICompileUnit(language: DW_LANG_C99, producer: "QuIC LLVM Hexagon Clang version 6.1-pre-unknown, (git://git-hexagon-aus.quicinc.com/llvm/clang-mainline.git e9382867661454cdf44addb39430741578e9765c) (llvm/llvm-mainline.git 36412bb1fcf03ed426d4437b41198bae066675ac)", isOptimized: true, emissionKind: 1, file: !28, enums: !2, retainedTypes: !2, subprograms: !3, globals: !2)
 !2 = !{}
 !3 = !{!5}
-!5 = !MDSubprogram(name: "foo", line: 1, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: true, scopeLine: 1, file: !28, scope: null, type: !7, function: void (i32*, i32*)* @foo, variables: !11)
-!6 = !MDFile(filename: "hwloop-dbg.c", directory: "/usr2/kparzysz/s.hex/t")
-!7 = !MDSubroutineType(types: !8)
+!5 = !DISubprogram(name: "foo", line: 1, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: true, scopeLine: 1, file: !28, scope: null, type: !7, function: void (i32*, i32*)* @foo, variables: !11)
+!6 = !DIFile(filename: "hwloop-dbg.c", directory: "/usr2/kparzysz/s.hex/t")
+!7 = !DISubroutineType(types: !8)
 !8 = !{null, !9, !9}
-!9 = !MDDerivedType(tag: DW_TAG_pointer_type, size: 32, align: 32, baseType: !10)
-!10 = !MDBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
+!9 = !DIDerivedType(tag: DW_TAG_pointer_type, size: 32, align: 32, baseType: !10)
+!10 = !DIBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
 !11 = !{!13, !14, !15}
-!13 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "a", line: 1, arg: 1, scope: !5, file: !6, type: !9)
-!14 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "b", line: 1, arg: 2, scope: !5, file: !6, type: !9)
-!15 = !MDLocalVariable(tag: DW_TAG_auto_variable, name: "i", line: 2, scope: !16, file: !6, type: !10)
-!16 = distinct !MDLexicalBlock(line: 1, column: 26, file: !28, scope: !5)
-!17 = !MDLocation(line: 1, column: 15, scope: !5)
-!18 = !MDLocation(line: 1, column: 23, scope: !5)
-!19 = !MDLocation(line: 3, column: 8, scope: !20)
-!20 = distinct !MDLexicalBlock(line: 3, column: 3, file: !28, scope: !16)
-!21 = !MDLocation(line: 4, column: 5, scope: !22)
-!22 = distinct !MDLexicalBlock(line: 3, column: 28, file: !28, scope: !20)
-!26 = !MDLocation(line: 3, column: 23, scope: !20)
-!27 = !MDLocation(line: 6, column: 1, scope: !16)
-!28 = !MDFile(filename: "hwloop-dbg.c", directory: "/usr2/kparzysz/s.hex/t")
+!13 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "a", line: 1, arg: 1, scope: !5, file: !6, type: !9)
+!14 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "b", line: 1, arg: 2, scope: !5, file: !6, type: !9)
+!15 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "i", line: 2, scope: !16, file: !6, type: !10)
+!16 = distinct !DILexicalBlock(line: 1, column: 26, file: !28, scope: !5)
+!17 = !DILocation(line: 1, column: 15, scope: !5)
+!18 = !DILocation(line: 1, column: 23, scope: !5)
+!19 = !DILocation(line: 3, column: 8, scope: !20)
+!20 = distinct !DILexicalBlock(line: 3, column: 3, file: !28, scope: !16)
+!21 = !DILocation(line: 4, column: 5, scope: !22)
+!22 = distinct !DILexicalBlock(line: 3, column: 28, file: !28, scope: !20)
+!26 = !DILocation(line: 3, column: 23, scope: !20)
+!27 = !DILocation(line: 6, column: 1, scope: !16)
+!28 = !DIFile(filename: "hwloop-dbg.c", directory: "/usr2/kparzysz/s.hex/t")
 !29 = !{i32 1, !"Debug Info Version", i32 3}
 !30 = !{i32 0}
diff --git a/test/CodeGen/Hexagon/hwloop-loop1.ll b/test/CodeGen/Hexagon/hwloop-loop1.ll
new file mode 100644
index 00000000000..8b02736e037
--- /dev/null
+++ b/test/CodeGen/Hexagon/hwloop-loop1.ll
@@ -0,0 +1,68 @@
+; RUN: llc -march=hexagon -mcpu=hexagonv5 < %s | FileCheck %s
+;
+; Generate loop1 instruction for double loop sequence.
+
+; CHECK: loop0(.LBB{{.}}_{{.}}, #100)
+; CHECK: endloop0
+; CHECK: loop1(.LBB{{.}}_{{.}}, #100)
+; CHECK: loop0(.LBB{{.}}_{{.}}, #100)
+; CHECK: endloop0
+; CHECK: endloop1
+
+define i32 @main() #0 {
+entry:
+  %array = alloca [100 x i32], align 8
+  %doublearray = alloca [100 x [100 x i32]], align 8
+  %0 = bitcast [100 x i32]* %array to i8*
+  call void @llvm.lifetime.start(i64 400, i8* %0) #1
+  %1 = bitcast [100 x [100 x i32]]* %doublearray to i8*
+  call void @llvm.lifetime.start(i64 40000, i8* %1) #1
+  %arrayidx1 = getelementptr inbounds [100 x [100 x i32]], [100 x [100 x i32]]* %doublearray, i32 0, i32 10, i32 10
+  %arrayidx2.gep = getelementptr [100 x i32], [100 x i32]* %array, i32 0, i32 0
+  br label %for.body
+
+for.body:
+  %2 = phi i32 [ undef, %entry ], [ %.pre, %for.body.for.body_crit_edge ]
+  %sum.031 = phi i32 [ undef, %entry ], [ %add, %for.body.for.body_crit_edge ]
+  %arrayidx2.phi = phi i32* [ %arrayidx2.gep, %entry ], [ %arrayidx2.inc, %for.body.for.body_crit_edge ]
+  %i.030 = phi i32 [ 1, %entry ], [ %phitmp, %for.body.for.body_crit_edge ]
+  %add = add nsw i32 %2, %sum.031
+  %exitcond33 = icmp eq i32 %i.030, 100
+  %arrayidx2.inc = getelementptr i32, i32* %arrayidx2.phi, i32 1
+  br i1 %exitcond33, label %for.cond7.preheader.preheader, label %for.body.for.body_crit_edge
+
+for.cond7.preheader.preheader:
+  br label %for.cond7.preheader
+
+for.body.for.body_crit_edge:
+  %.pre = load i32, i32* %arrayidx2.inc, align 4
+  %phitmp = add i32 %i.030, 1
+  br label %for.body
+
+for.cond7.preheader:
+  %i.129 = phi i32 [ %inc16, %for.inc15 ], [ 0, %for.cond7.preheader.preheader ]
+  br label %for.body9
+
+for.body9:
+  %j.028 = phi i32 [ 0, %for.cond7.preheader ], [ %inc13, %for.body9 ]
+  %arrayidx11 = getelementptr inbounds [100 x [100 x i32]], [100 x [100 x i32]]* %doublearray, i32 0, i32 %i.129, i32 %j.028
+  store i32 %add, i32* %arrayidx11, align 4
+  %inc13 = add nsw i32 %j.028, 1
+  %exitcond = icmp eq i32 %inc13, 100
+  br i1 %exitcond, label %for.inc15, label %for.body9
+
+for.inc15:
+  %inc16 = add nsw i32 %i.129, 1
+  %exitcond32 = icmp eq i32 %inc16, 100
+  br i1 %exitcond32, label %for.end17, label %for.cond7.preheader
+
+for.end17:
+  %3 = load i32, i32* %arrayidx1, align 8
+  call void @llvm.lifetime.end(i64 40000, i8* %1) #1
+  call void @llvm.lifetime.end(i64 400, i8* %0) #1
+  ret i32 %3
+}
+
+declare void @llvm.lifetime.start(i64, i8* nocapture) #1
+
+declare void @llvm.lifetime.end(i64, i8* nocapture) #1
diff --git a/test/CodeGen/Hexagon/hwloop-lt.ll b/test/CodeGen/Hexagon/hwloop-lt.ll
index 804f76456e2..8919f265abf 100644
--- a/test/CodeGen/Hexagon/hwloop-lt.ll
+++ b/test/CodeGen/Hexagon/hwloop-lt.ll
@@ -1,7 +1,6 @@
-; RUN: llc -march=hexagon -mcpu=hexagonv4 -O3 < %s | FileCheck %s
+; RUN: llc -march=hexagon -O3 < %s | FileCheck %s
 
-
-; CHECK: test_pos1_ir_slt
+; CHECK-LABEL: @test_pos1_ir_slt
 ; CHECK: loop0
 ; a < b
 define void @test_pos1_ir_slt(i8* nocapture %p, i32 %a, i32 %b) nounwind {
@@ -9,10 +8,10 @@ entry:
   %cmp3 = icmp slt i32 8531, %b
   br i1 %cmp3, label %for.body.lr.ph, label %for.end
 
-for.body.lr.ph:                                   ; preds = %entry
+for.body.lr.ph:
   br label %for.body
 
-for.body:                                         ; preds = %for.body.lr.ph, %for.body
+for.body:
   %i.04 = phi i32 [ 8531, %for.body.lr.ph ], [ %inc, %for.body ]
   %arrayidx = getelementptr inbounds i8, i8* %p, i32 %i.04
   %0 = load i8, i8* %arrayidx, align 1
@@ -24,13 +23,11 @@ for.body:                                         ; preds = %for.body.lr.ph, %fo
   %cmp = icmp slt i32 %inc, %b
   br i1 %cmp, label %for.body, label %for.end
 
-for.end:                                          ; preds = %for.body, %entry
+for.end:
   ret void
 }
 
-
-
-; CHECK: test_pos2_ir_slt
+; CHECK-LABEL: @test_pos2_ir_slt
 ; CHECK: loop0
 ; a < b
 define void @test_pos2_ir_slt(i8* nocapture %p, i32 %a, i32 %b) nounwind {
@@ -38,10 +35,10 @@ entry:
   %cmp3 = icmp slt i32 9152, %b
   br i1 %cmp3, label %for.body.lr.ph, label %for.end
 
-for.body.lr.ph:                                   ; preds = %entry
+for.body.lr.ph:
   br label %for.body
 
-for.body:                                         ; preds = %for.body.lr.ph, %for.body
+for.body:
   %i.04 = phi i32 [ 9152, %for.body.lr.ph ], [ %inc, %for.body ]
   %arrayidx = getelementptr inbounds i8, i8* %p, i32 %i.04
   %0 = load i8, i8* %arrayidx, align 1
@@ -53,13 +50,11 @@ for.body:                                         ; preds = %for.body.lr.ph, %fo
   %cmp = icmp slt i32 %inc, %b
   br i1 %cmp, label %for.body, label %for.end
 
-for.end:                                          ; preds = %for.body, %entry
+for.end:
   ret void
 }
 
-
-
-; CHECK: test_pos4_ir_slt
+; CHECK-LABEL: @test_pos4_ir_slt
 ; CHECK: loop0
 ; a < b
 define void @test_pos4_ir_slt(i8* nocapture %p, i32 %a, i32 %b) nounwind {
@@ -67,10 +62,10 @@ entry:
   %cmp3 = icmp slt i32 18851, %b
   br i1 %cmp3, label %for.body.lr.ph, label %for.end
 
-for.body.lr.ph:                                   ; preds = %entry
+for.body.lr.ph:
   br label %for.body
 
-for.body:                                         ; preds = %for.body.lr.ph, %for.body
+for.body:
   %i.04 = phi i32 [ 18851, %for.body.lr.ph ], [ %inc, %for.body ]
   %arrayidx = getelementptr inbounds i8, i8* %p, i32 %i.04
   %0 = load i8, i8* %arrayidx, align 1
@@ -82,13 +77,11 @@ for.body:                                         ; preds = %for.body.lr.ph, %fo
   %cmp = icmp slt i32 %inc, %b
   br i1 %cmp, label %for.body, label %for.end
 
-for.end:                                          ; preds = %for.body, %entry
+for.end:
   ret void
 }
 
-
-
-; CHECK: test_pos8_ir_slt
+; CHECK-LABEL: @test_pos8_ir_slt
 ; CHECK: loop0
 ; a < b
 define void @test_pos8_ir_slt(i8* nocapture %p, i32 %a, i32 %b) nounwind {
@@ -96,10 +89,10 @@ entry:
   %cmp3 = icmp slt i32 25466, %b
   br i1 %cmp3, label %for.body.lr.ph, label %for.end
 
-for.body.lr.ph:                                   ; preds = %entry
+for.body.lr.ph:
   br label %for.body
 
-for.body:                                         ; preds = %for.body.lr.ph, %for.body
+for.body:
   %i.04 = phi i32 [ 25466, %for.body.lr.ph ], [ %inc, %for.body ]
   %arrayidx = getelementptr inbounds i8, i8* %p, i32 %i.04
   %0 = load i8, i8* %arrayidx, align 1
@@ -111,13 +104,11 @@ for.body:                                         ; preds = %for.body.lr.ph, %fo
   %cmp = icmp slt i32 %inc, %b
   br i1 %cmp, label %for.body, label %for.end
 
-for.end:                                          ; preds = %for.body, %entry
+for.end:
   ret void
 }
 
-
-
-; CHECK: test_pos16_ir_slt
+; CHECK-LABEL: @test_pos16_ir_slt
 ; CHECK: loop0
 ; a < b
 define void @test_pos16_ir_slt(i8* nocapture %p, i32 %a, i32 %b) nounwind {
@@ -125,10 +116,10 @@ entry:
   %cmp3 = icmp slt i32 9295, %b
   br i1 %cmp3, label %for.body.lr.ph, label %for.end
 
-for.body.lr.ph:                                   ; preds = %entry
+for.body.lr.ph:
   br label %for.body
 
-for.body:                                         ; preds = %for.body.lr.ph, %for.body
+for.body:
   %i.04 = phi i32 [ 9295, %for.body.lr.ph ], [ %inc, %for.body ]
   %arrayidx = getelementptr inbounds i8, i8* %p, i32 %i.04
   %0 = load i8, i8* %arrayidx, align 1
@@ -140,13 +131,11 @@ for.body:                                         ; preds = %for.body.lr.ph, %fo
   %cmp = icmp slt i32 %inc, %b
   br i1 %cmp, label %for.body, label %for.end
 
-for.end:                                          ; preds = %for.body, %entry
+for.end:
   ret void
 }
 
-
-
-; CHECK: test_pos1_ri_slt
+; CHECK-LABEL: @test_pos1_ri_slt
 ; CHECK: loop0
 ; a < b
 define void @test_pos1_ri_slt(i8* nocapture %p, i32 %a, i32 %b) nounwind {
@@ -154,10 +143,10 @@ entry:
   %cmp3 = icmp slt i32 %a, 31236
   br i1 %cmp3, label %for.body.lr.ph, label %for.end
 
-for.body.lr.ph:                                   ; preds = %entry
+for.body.lr.ph:
   br label %for.body
 
-for.body:                                         ; preds = %for.body.lr.ph, %for.body
+for.body:
   %i.04 = phi i32 [ %a, %for.body.lr.ph ], [ %inc, %for.body ]
   %arrayidx = getelementptr inbounds i8, i8* %p, i32 %i.04
   %0 = load i8, i8* %arrayidx, align 1
@@ -169,13 +158,11 @@ for.body:                                         ; preds = %for.body.lr.ph, %fo
   %cmp = icmp slt i32 %inc, 31236
   br i1 %cmp, label %for.body, label %for.end
 
-for.end:                                          ; preds = %for.body, %entry
+for.end:
   ret void
 }
 
-
-
-; CHECK: test_pos2_ri_slt
+; CHECK-LABEL: @test_pos2_ri_slt
 ; CHECK: loop0
 ; a < b
 define void @test_pos2_ri_slt(i8* nocapture %p, i32 %a, i32 %b) nounwind {
@@ -183,10 +170,10 @@ entry:
   %cmp3 = icmp slt i32 %a, 22653
   br i1 %cmp3, label %for.body.lr.ph, label %for.end
 
-for.body.lr.ph:                                   ; preds = %entry
+for.body.lr.ph:
   br label %for.body
 
-for.body:                                         ; preds = %for.body.lr.ph, %for.body
+for.body:
   %i.04 = phi i32 [ %a, %for.body.lr.ph ], [ %inc, %for.body ]
   %arrayidx = getelementptr inbounds i8, i8* %p, i32 %i.04
   %0 = load i8, i8* %arrayidx, align 1
@@ -198,13 +185,11 @@ for.body:                                         ; preds = %for.body.lr.ph, %fo
   %cmp = icmp slt i32 %inc, 22653
   br i1 %cmp, label %for.body, label %for.end
 
-for.end:                                          ; preds = %for.body, %entry
+for.end:
   ret void
 }
 
-
-
-; CHECK: test_pos4_ri_slt
+; CHECK-LABEL: @test_pos4_ri_slt
 ; CHECK: loop0
 ; a < b
 define void @test_pos4_ri_slt(i8* nocapture %p, i32 %a, i32 %b) nounwind {
@@ -212,10 +197,10 @@ entry:
   %cmp3 = icmp slt i32 %a, 1431
   br i1 %cmp3, label %for.body.lr.ph, label %for.end
 
-for.body.lr.ph:                                   ; preds = %entry
+for.body.lr.ph:
   br label %for.body
 
-for.body:                                         ; preds = %for.body.lr.ph, %for.body
+for.body:
   %i.04 = phi i32 [ %a, %for.body.lr.ph ], [ %inc, %for.body ]
   %arrayidx = getelementptr inbounds i8, i8* %p, i32 %i.04
   %0 = load i8, i8* %arrayidx, align 1
@@ -227,13 +212,11 @@ for.body:                                         ; preds = %for.body.lr.ph, %fo
   %cmp = icmp slt i32 %inc, 1431
   br i1 %cmp, label %for.body, label %for.end
 
-for.end:                                          ; preds = %for.body, %entry
+for.end:
   ret void
 }
 
-
-
-; CHECK: test_pos8_ri_slt
+; CHECK-LABEL: @test_pos8_ri_slt
 ; CHECK: loop0
 ; a < b
 define void @test_pos8_ri_slt(i8* nocapture %p, i32 %a, i32 %b) nounwind {
@@ -241,10 +224,10 @@ entry:
   %cmp3 = icmp slt i32 %a, 22403
   br i1 %cmp3, label %for.body.lr.ph, label %for.end
 
-for.body.lr.ph:                                   ; preds = %entry
+for.body.lr.ph:
   br label %for.body
 
-for.body:                                         ; preds = %for.body.lr.ph, %for.body
+for.body:
   %i.04 = phi i32 [ %a, %for.body.lr.ph ], [ %inc, %for.body ]
   %arrayidx = getelementptr inbounds i8, i8* %p, i32 %i.04
   %0 = load i8, i8* %arrayidx, align 1
@@ -256,13 +239,11 @@ for.body:                                         ; preds = %for.body.lr.ph, %fo
   %cmp = icmp slt i32 %inc, 22403
   br i1 %cmp, label %for.body, label %for.end
 
-for.end:                                          ; preds = %for.body, %entry
+for.end:
   ret void
 }
 
-
-
-; CHECK: test_pos16_ri_slt
+; CHECK-LABEL: @test_pos16_ri_slt
 ; CHECK: loop0
 ; a < b
 define void @test_pos16_ri_slt(i8* nocapture %p, i32 %a, i32 %b) nounwind {
@@ -270,10 +251,10 @@ entry:
   %cmp3 = icmp slt i32 %a, 21715
   br i1 %cmp3, label %for.body.lr.ph, label %for.end
 
-for.body.lr.ph:                                   ; preds = %entry
+for.body.lr.ph:
   br label %for.body
 
-for.body:                                         ; preds = %for.body.lr.ph, %for.body
+for.body:
   %i.04 = phi i32 [ %a, %for.body.lr.ph ], [ %inc, %for.body ]
   %arrayidx = getelementptr inbounds i8, i8* %p, i32 %i.04
   %0 = load i8, i8* %arrayidx, align 1
@@ -285,13 +266,11 @@ for.body:                                         ; preds = %for.body.lr.ph, %fo
   %cmp = icmp slt i32 %inc, 21715
   br i1 %cmp, label %for.body, label %for.end
 
-for.end:                                          ; preds = %for.body, %entry
+for.end:
   ret void
 }
 
-
-
-; CHECK: test_pos1_rr_slt
+; CHECK-LABEL: @test_pos1_rr_slt
 ; CHECK: loop0
 ; a < b
 define void @test_pos1_rr_slt(i8* nocapture %p, i32 %a, i32 %b) nounwind {
@@ -299,10 +278,10 @@ entry:
   %cmp3 = icmp slt i32 %a, %b
   br i1 %cmp3, label %for.body.lr.ph, label %for.end
 
-for.body.lr.ph:                                   ; preds = %entry
+for.body.lr.ph:
   br label %for.body
 
-for.body:                                         ; preds = %for.body.lr.ph, %for.body
+for.body:
   %i.04 = phi i32 [ %a, %for.body.lr.ph ], [ %inc, %for.body ]
   %arrayidx = getelementptr inbounds i8, i8* %p, i32 %i.04
   %0 = load i8, i8* %arrayidx, align 1
@@ -314,13 +293,11 @@ for.body:                                         ; preds = %for.body.lr.ph, %fo
   %cmp = icmp slt i32 %inc, %b
   br i1 %cmp, label %for.body, label %for.end
 
-for.end:                                          ; preds = %for.body, %entry
+for.end:
   ret void
 }
 
-
-
-; CHECK: test_pos2_rr_slt
+; CHECK-LABEL: @test_pos2_rr_slt
 ; CHECK: loop0
 ; a < b
 define void @test_pos2_rr_slt(i8* nocapture %p, i32 %a, i32 %b) nounwind {
@@ -328,10 +305,10 @@ entry:
   %cmp3 = icmp slt i32 %a, %b
   br i1 %cmp3, label %for.body.lr.ph, label %for.end
 
-for.body.lr.ph:                                   ; preds = %entry
+for.body.lr.ph:
   br label %for.body
 
-for.body:                                         ; preds = %for.body.lr.ph, %for.body
+for.body:
   %i.04 = phi i32 [ %a, %for.body.lr.ph ], [ %inc, %for.body ]
   %arrayidx = getelementptr inbounds i8, i8* %p, i32 %i.04
   %0 = load i8, i8* %arrayidx, align 1
@@ -343,13 +320,11 @@ for.body:                                         ; preds = %for.body.lr.ph, %fo
   %cmp = icmp slt i32 %inc, %b
   br i1 %cmp, label %for.body, label %for.end
 
-for.end:                                          ; preds = %for.body, %entry
+for.end:
   ret void
 }
 
-
-
-; CHECK: test_pos4_rr_slt
+; CHECK-LABEL: @test_pos4_rr_slt
 ; CHECK: loop0
 ; a < b
 define void @test_pos4_rr_slt(i8* nocapture %p, i32 %a, i32 %b) nounwind {
@@ -357,10 +332,10 @@ entry:
   %cmp3 = icmp slt i32 %a, %b
   br i1 %cmp3, label %for.body.lr.ph, label %for.end
 
-for.body.lr.ph:                                   ; preds = %entry
+for.body.lr.ph:
   br label %for.body
 
-for.body:                                         ; preds = %for.body.lr.ph, %for.body
+for.body:
   %i.04 = phi i32 [ %a, %for.body.lr.ph ], [ %inc, %for.body ]
   %arrayidx = getelementptr inbounds i8, i8* %p, i32 %i.04
   %0 = load i8, i8* %arrayidx, align 1
@@ -372,13 +347,11 @@ for.body:                                         ; preds = %for.body.lr.ph, %fo
   %cmp = icmp slt i32 %inc, %b
   br i1 %cmp, label %for.body, label %for.end
 
-for.end:                                          ; preds = %for.body, %entry
+for.end:
   ret void
 }
 
-
-
-; CHECK: test_pos8_rr_slt
+; CHECK-LABEL: @test_pos8_rr_slt
 ; CHECK: loop0
 ; a < b
 define void @test_pos8_rr_slt(i8* nocapture %p, i32 %a, i32 %b) nounwind {
@@ -386,10 +359,10 @@ entry:
   %cmp3 = icmp slt i32 %a, %b
   br i1 %cmp3, label %for.body.lr.ph, label %for.end
 
-for.body.lr.ph:                                   ; preds = %entry
+for.body.lr.ph:
   br label %for.body
 
-for.body:                                         ; preds = %for.body.lr.ph, %for.body
+for.body:
   %i.04 = phi i32 [ %a, %for.body.lr.ph ], [ %inc, %for.body ]
   %arrayidx = getelementptr inbounds i8, i8* %p, i32 %i.04
   %0 = load i8, i8* %arrayidx, align 1
@@ -401,13 +374,11 @@ for.body:                                         ; preds = %for.body.lr.ph, %fo
   %cmp = icmp slt i32 %inc, %b
   br i1 %cmp, label %for.body, label %for.end
 
-for.end:                                          ; preds = %for.body, %entry
+for.end:
   ret void
 }
 
-
-
-; CHECK: test_pos16_rr_slt
+; CHECK-LABEL: @test_pos16_rr_slt
 ; CHECK: loop0
 ; a < b
 define void @test_pos16_rr_slt(i8* nocapture %p, i32 %a, i32 %b) nounwind {
@@ -415,10 +386,10 @@ entry:
   %cmp3 = icmp slt i32 %a, %b
   br i1 %cmp3, label %for.body.lr.ph, label %for.end
 
-for.body.lr.ph:                                   ; preds = %entry
+for.body.lr.ph:
   br label %for.body
 
-for.body:                                         ; preds = %for.body.lr.ph, %for.body
+for.body:
   %i.04 = phi i32 [ %a, %for.body.lr.ph ], [ %inc, %for.body ]
   %arrayidx = getelementptr inbounds i8, i8* %p, i32 %i.04
   %0 = load i8, i8* %arrayidx, align 1
@@ -430,7 +401,7 @@ for.body:                                         ; preds = %for.body.lr.ph, %fo
   %cmp = icmp slt i32 %inc, %b
   br i1 %cmp, label %for.body, label %for.end
 
-for.end:                                          ; preds = %for.body, %entry
+for.end:
   ret void
 }
 
diff --git a/test/CodeGen/Hexagon/hwloop-lt1.ll b/test/CodeGen/Hexagon/hwloop-lt1.ll
index 16fe728fa7b..cf97fffce40 100644
--- a/test/CodeGen/Hexagon/hwloop-lt1.ll
+++ b/test/CodeGen/Hexagon/hwloop-lt1.ll
@@ -1,4 +1,4 @@
-; RUN: llc -march=hexagon -mcpu=hexagonv4 < %s | FileCheck %s
+; RUN: llc -march=hexagon < %s | FileCheck %s
 ; Check that we generate a hardware loop instruction.
 ; CHECK: endloop0
 
diff --git a/test/CodeGen/Hexagon/hwloop-missed.ll b/test/CodeGen/Hexagon/hwloop-missed.ll
new file mode 100644
index 00000000000..bcc80065229
--- /dev/null
+++ b/test/CodeGen/Hexagon/hwloop-missed.ll
@@ -0,0 +1,49 @@
+; RUN: llc -march=hexagon -hexagon-hwloop-preheader < %s | FileCheck %s
+
+; Generate hardware loops when we also need to add a new preheader.
+; we should generate two hardware loops for this test case.
+
+; CHECK: loop0
+; CHECK: endloop0
+; CHECK: loop0
+; CHECK: endloop0
+
+@g = external global i32
+
+define void @test(i32* nocapture %a, i32* nocapture %b, i32 %n) nounwind {
+entry:
+  %tobool = icmp eq i32 %n, 0
+  br i1 %tobool, label %for.body4.preheader, label %for.body.preheader
+
+for.body.preheader:
+  br label %for.body
+
+for.body:
+  %arrayidx.phi = phi i32* [ %arrayidx.inc, %for.body ], [ %a, %for.body.preheader ]
+  %i.014 = phi i32 [ %inc, %for.body ], [ 0, %for.body.preheader ]
+  %0 = load i32, i32* @g, align 4
+  store i32 %0, i32* %arrayidx.phi, align 4
+  %inc = add nsw i32 %i.014, 1
+  %exitcond15 = icmp eq i32 %inc, 3
+  %arrayidx.inc = getelementptr i32, i32* %arrayidx.phi, i32 1
+  br i1 %exitcond15, label %for.body4.preheader.loopexit, label %for.body
+
+for.body4.preheader.loopexit:
+  br label %for.body4.preheader
+
+for.body4.preheader:
+  br label %for.body4
+
+for.body4:
+  %arrayidx5.phi = phi i32* [ %arrayidx5.inc, %for.body4 ], [ %b, %for.body4.preheader ]
+  %i1.013 = phi i32 [ %inc7, %for.body4 ], [ 0, %for.body4.preheader ]
+  %1 = load i32, i32* @g, align 4
+  store i32 %1, i32* %arrayidx5.phi, align 4
+  %inc7 = add nsw i32 %i1.013, 1
+  %exitcond = icmp eq i32 %inc7, 3
+  %arrayidx5.inc = getelementptr i32, i32* %arrayidx5.phi, i32 1
+  br i1 %exitcond, label %for.end8, label %for.body4
+
+for.end8:
+  ret void
+}
diff --git a/test/CodeGen/Hexagon/hwloop-ph-deadcode.ll b/test/CodeGen/Hexagon/hwloop-ph-deadcode.ll
new file mode 100644
index 00000000000..06e6db420f8
--- /dev/null
+++ b/test/CodeGen/Hexagon/hwloop-ph-deadcode.ll
@@ -0,0 +1,23 @@
+; RUN: llc -march=hexagon -mcpu=hexagonv5 -O2 -disable-block-placement=0 < %s | FileCheck %s
+
+; Test that there is no redundant register assignment in the hardware loop
+; preheader.
+
+; CHECK-NOT: r{{.*}} = #5
+
+@g = external global i32
+
+define void @foo() #0 {
+entry:
+  br i1 undef, label %if.end38, label %for.body
+
+for.body:
+  %loopIdx.051 = phi i32 [ %inc, %for.body ], [ 0, %entry ]
+  store i32 1, i32* @g, align 4
+  %inc = add i32 %loopIdx.051, 1
+  %cmp9 = icmp ult i32 %inc, 5
+  br i1 %cmp9, label %for.body, label %if.end38
+
+if.end38:
+  ret void
+}
diff --git a/test/CodeGen/Hexagon/hwloop-pos-ivbump1.ll b/test/CodeGen/Hexagon/hwloop-pos-ivbump1.ll
new file mode 100644
index 00000000000..7c5ea031ffa
--- /dev/null
+++ b/test/CodeGen/Hexagon/hwloop-pos-ivbump1.ll
@@ -0,0 +1,45 @@
+;RUN: llc -march=hexagon < %s | FileCheck %s
+
+; Test that a hardware loop is not generaetd due to a potential
+; underflow.
+
+; CHECK-NOT: loop0
+
+define i32 @main() #0 {
+entry:
+  br label %while.cond.outer
+
+while.cond.outer.loopexit:
+  %.lcssa = phi i32 [ %0, %for.body.preheader ]
+  br label %while.cond.outer
+
+while.cond.outer:
+  %i.0.ph = phi i32 [ 0, %entry ], [ 3, %while.cond.outer.loopexit ]
+  %j.0.ph = phi i32 [ 0, %entry ], [ %.lcssa, %while.cond.outer.loopexit ]
+  %k.0.ph = phi i32 [ 0, %entry ], [ 1, %while.cond.outer.loopexit ]
+  br label %while.cond
+
+while.cond:
+  %i.0 = phi i32 [ %i.0.ph, %while.cond.outer ], [ %inc, %for.body.preheader ]
+  %j.0 = phi i32 [ %j.0.ph, %while.cond.outer ], [ %0, %for.body.preheader ]
+  %inc = add nsw i32 %i.0, 1
+  %cmp = icmp slt i32 %i.0, 4
+  br i1 %cmp, label %for.body.preheader, label %while.end
+
+for.body.preheader:
+  %0 = add i32 %j.0, 3
+  %cmp5 = icmp eq i32 %inc, 3
+  br i1 %cmp5, label %while.cond.outer.loopexit, label %while.cond
+
+while.end:
+  %k.0.ph.lcssa = phi i32 [ %k.0.ph, %while.cond ]
+  %inc.lcssa = phi i32 [ %inc, %while.cond ]
+  %j.0.lcssa = phi i32 [ %j.0, %while.cond ]
+  %cmp6 = icmp ne i32 %inc.lcssa, 5
+  %cmp7 = icmp ne i32 %j.0.lcssa, 12
+  %or.cond = or i1 %cmp6, %cmp7
+  %cmp9 = icmp ne i32 %k.0.ph.lcssa, 1
+  %or.cond12 = or i1 %or.cond, %cmp9
+  %locflg.0 = zext i1 %or.cond12 to i32
+  ret i32 %locflg.0
+}
diff --git a/test/CodeGen/Hexagon/hwloop-preheader.ll b/test/CodeGen/Hexagon/hwloop-preheader.ll
new file mode 100644
index 00000000000..66efd2089fc
--- /dev/null
+++ b/test/CodeGen/Hexagon/hwloop-preheader.ll
@@ -0,0 +1,40 @@
+; RUN: llc -march=hexagon -mcpu=hexagonv5 -hexagon-hwloop-preheader < %s
+; REQUIRES: asserts
+
+; Test that the preheader is added to the parent loop, otherwise
+; we generate an invalid hardware loop.
+
+; Function Attrs: nounwind readonly
+define void @test(i16 signext %n) #0 {
+entry:
+  br i1 undef, label %for.cond4.preheader.preheader.split.us, label %for.end22
+
+for.cond4.preheader.preheader.split.us:
+  %0 = sext i16 %n to i32
+  br label %for.body9.preheader.us
+
+for.body9.us:
+  %indvars.iv = phi i32 [ %indvars.iv.next.7, %for.body9.us ], [ 0, %for.body9.preheader.us ]
+  %indvars.iv.next.7 = add i32 %indvars.iv, 8
+  %lftr.wideiv.7 = trunc i32 %indvars.iv.next.7 to i16
+  %exitcond.7 = icmp slt i16 %lftr.wideiv.7, 0
+  br i1 %exitcond.7, label %for.body9.us, label %for.body9.us.ur
+
+for.body9.preheader.us:
+  %i.030.us.pmt = phi i32 [ %inc21.us.pmt, %for.end.loopexit.us ], [ 0, %for.cond4.preheader.preheader.split.us ]
+  br i1 undef, label %for.body9.us, label %for.body9.us.ur
+
+for.body9.us.ur:
+  %exitcond.ur.old = icmp eq i16 undef, %n
+  br i1 %exitcond.ur.old, label %for.end.loopexit.us, label %for.body9.us.ur
+
+for.end.loopexit.us:
+  %inc21.us.pmt = add i32 %i.030.us.pmt, 1
+  %exitcond33 = icmp eq i32 %inc21.us.pmt, %0
+  br i1 %exitcond33, label %for.end22, label %for.body9.preheader.us
+
+for.end22:
+  ret void
+}
+
+attributes #0 = { nounwind readonly "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf"="true" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "unsafe-fp-math"="false" "use-soft-float"="false" }
diff --git a/test/CodeGen/Hexagon/hwloop-range.ll b/test/CodeGen/Hexagon/hwloop-range.ll
new file mode 100644
index 00000000000..5e6fe78d0e0
--- /dev/null
+++ b/test/CodeGen/Hexagon/hwloop-range.ll
@@ -0,0 +1,36 @@
+; RUN: llc -march=hexagon -hexagon-loop-range=0 < %s | FileCheck %s
+
+; Test that the loop start address operand uses a constant extender
+; if the offset is out of range.
+
+; CHECK: loop0(##.LBB
+; CHECK: endloop0
+
+@g = external global i32, align 4
+
+define void @test(i32* nocapture %a, i32* nocapture readonly %b, i32 %n) #0 {
+entry:
+  %cmp6 = icmp slt i32 %n, 1
+  br i1 %cmp6, label %for.end, label %for.body.preheader
+
+for.body.preheader:
+  br label %for.body
+
+for.body:
+  %i.07 = phi i32 [ %inc, %for.body ], [ 0, %for.body.preheader ]
+  %arrayidx = getelementptr inbounds i32, i32* %b, i32 %i.07
+  %0 = load i32, i32* %arrayidx, align 4
+  %1 = load i32, i32* @g, align 4
+  %mul = mul nsw i32 %1, %0
+  %arrayidx1 = getelementptr inbounds i32, i32* %a, i32 %i.07
+  store i32 %mul, i32* %arrayidx1, align 4
+  %inc = add nuw nsw i32 %i.07, 1
+  %exitcond = icmp eq i32 %inc, %n
+  br i1 %exitcond, label %for.end.loopexit, label %for.body
+
+for.end.loopexit:
+  br label %for.end
+
+for.end:
+  ret void
+}
diff --git a/test/CodeGen/Hexagon/hwloop-recursion.ll b/test/CodeGen/Hexagon/hwloop-recursion.ll
new file mode 100644
index 00000000000..8ab2dc37d02
--- /dev/null
+++ b/test/CodeGen/Hexagon/hwloop-recursion.ll
@@ -0,0 +1,64 @@
+; RUN: llc -O2 -march=hexagon -mcpu=hexagonv5 < %s
+; REQUIRES: asserts
+; Check for successful compilation.
+
+@c = common global i32 0, align 4
+@e = common global i32 0, align 4
+@g = common global i32* null, align 4
+@a = common global i32 0, align 4
+@b = common global i32 0, align 4
+@h = common global i32* null, align 4
+@d = common global i32 0, align 4
+@f = common global i32 0, align 4
+
+define i32 @fn1([0 x i32]* nocapture readnone %p1) #0 {
+entry:
+  %0 = load i32*, i32** @h, align 4
+  %1 = load i32*, i32** @g, align 4
+  %.pre = load i32, i32* @c, align 4
+  br label %for.cond
+
+for.cond:
+  %2 = phi i32 [ %10, %if.end ], [ %.pre, %entry ]
+  store i32 %2, i32* @e, align 4
+  %tobool5 = icmp eq i32 %2, 0
+  br i1 %tobool5, label %for.end, label %for.body.lr.ph
+
+for.body.lr.ph:
+  %3 = sub i32 -5, %2
+  %4 = urem i32 %3, 5
+  %5 = sub i32 %3, %4
+  br label %for.body
+
+for.body:
+  %add6 = phi i32 [ %2, %for.body.lr.ph ], [ %add, %for.body ]
+  %6 = load i32, i32* %1, align 4
+  store i32 %6, i32* @a, align 4
+  %add = add nsw i32 %add6, 5
+  %tobool = icmp eq i32 %add, 0
+  br i1 %tobool, label %for.cond1.for.end_crit_edge, label %for.body
+
+for.cond1.for.end_crit_edge:
+  %7 = add i32 %2, 5
+  %8 = add i32 %7, %5
+  store i32 %8, i32* @e, align 4
+  br label %for.end
+
+for.end:
+  %9 = load i32, i32* @b, align 4
+  %tobool2 = icmp eq i32 %9, 0
+  br i1 %tobool2, label %if.end, label %if.then
+
+if.then:
+  store i32 0, i32* %0, align 4
+  %.pre7 = load i32, i32* @c, align 4
+  br label %if.end
+
+if.end:
+  %10 = phi i32 [ %2, %for.end ], [ %.pre7, %if.then ]
+  store i32 %10, i32* @d, align 4
+  %11 = load i32, i32* @f, align 4
+  %inc = add nsw i32 %11, 1
+  store i32 %inc, i32* @f, align 4
+  br label %for.cond
+}
diff --git a/test/CodeGen/Hexagon/hwloop-wrap.ll b/test/CodeGen/Hexagon/hwloop-wrap.ll
new file mode 100644
index 00000000000..e0f6a87fd2e
--- /dev/null
+++ b/test/CodeGen/Hexagon/hwloop-wrap.ll
@@ -0,0 +1,22 @@
+; RUN: llc -march=hexagon -mcpu=hexagonv5 < %s | FileCheck %s
+
+; We shouldn't generate a hardware loop in this case because the initial
+; value may be zero, which means the endloop instruction will not decrement
+; the loop counter, and the loop will execute only once.
+
+; CHECK-NOT: loop0
+
+define void @foo(i32 %count, i32 %v) #0 {
+entry:
+  br label %do.body
+
+do.body:
+  %count.addr.0 = phi i32 [ %count, %entry ], [ %dec, %do.body ]
+  tail call void asm sideeffect "nop", ""() #1
+  %dec = add i32 %count.addr.0, -1
+  %cmp = icmp eq i32 %dec, 0
+  br i1 %cmp, label %do.end, label %do.body
+
+do.end:
+  ret void
+}
diff --git a/test/CodeGen/Hexagon/hwloop-wrap2.ll b/test/CodeGen/Hexagon/hwloop-wrap2.ll
new file mode 100644
index 00000000000..50675d6b681
--- /dev/null
+++ b/test/CodeGen/Hexagon/hwloop-wrap2.ll
@@ -0,0 +1,67 @@
+; RUN: llc -march=hexagon -mcpu=hexagonv5 -O3 < %s | FileCheck %s
+
+; Test that we do not generate a hardware loop due to a potential underflow.
+
+; CHECK-NOT: loop0
+
+%struct.3 = type { i8*, i8, i8, i32, i32, i16, i16, i16, i16, i16, i16, i16, %struct.2* }
+%struct.2 = type { i16, i16, i16, i16, %struct.1* }
+%struct.1 = type { %struct.1*, %struct.0*, i32, i32, i16, [2 x i16], [2 x i16], i16 }
+%struct.0 = type { %struct.0*, i32, i32, i32, i32, i32, i32, i16, i16, i16, i8, i8, i8, i8 }
+
+@pairArray = external global i32**
+@carray = external global %struct.3**
+
+define void @test() #0 {
+entry:
+  %0 = load i32**, i32*** @pairArray, align 4
+  %1 = load %struct.3**, %struct.3*** @carray, align 4
+  br i1 undef, label %for.end110, label %for.body
+
+for.body:
+  %row.0199 = phi i32 [ %inc109, %for.inc108 ], [ 1, %entry ]
+  %arrayidx = getelementptr inbounds i32*, i32** %0, i32 %row.0199
+  %2 = load i32*, i32** %arrayidx, align 4
+  br i1 undef, label %for.body48, label %for.inc108
+
+for.cond45:
+  %cmp46 = icmp sgt i32 %dec58, 0
+  br i1 %cmp46, label %for.body48, label %for.inc108
+
+for.body48:
+  %i.1190 = phi i32 [ %dec58, %for.cond45 ], [ 0, %for.body ]
+  %arrayidx50 = getelementptr inbounds i32, i32* %2, i32 %i.1190
+  %3 = load i32, i32* %arrayidx50, align 4
+  %cmp53 = icmp slt i32 %3, 0
+  %dec58 = add nsw i32 %i.1190, -1
+  br i1 %cmp53, label %for.end59, label %for.cond45
+
+for.end59:
+  %cmp60 = icmp slt i32 %i.1190, 0
+  br i1 %cmp60, label %if.then65, label %for.inc108
+
+if.then65:
+  br label %for.body80
+
+for.body80:
+  %j.1196.in = phi i32 [ %j.1196, %for.body80 ], [ %i.1190, %if.then65 ]
+  %j.1196 = add nsw i32 %j.1196.in, 1
+  %arrayidx81 = getelementptr inbounds i32, i32* %2, i32 %j.1196
+  %4 = load i32, i32* %arrayidx81, align 4
+  %arrayidx82 = getelementptr inbounds %struct.3*, %struct.3** %1, i32 %4
+  %5 = load %struct.3*, %struct.3** %arrayidx82, align 4
+  %cxcenter83 = getelementptr inbounds %struct.3, %struct.3* %5, i32 0, i32 3
+  store i32 0, i32* %cxcenter83, align 4
+  %6 = load i32, i32* %arrayidx81, align 4
+  %arrayidx87 = getelementptr inbounds i32, i32* %2, i32 %j.1196.in
+  store i32 %6, i32* %arrayidx87, align 4
+  %exitcond = icmp eq i32 %j.1196, 0
+  br i1 %exitcond, label %for.inc108, label %for.body80
+
+for.inc108:
+  %inc109 = add nsw i32 %row.0199, 1
+  br i1 undef, label %for.body, label %for.end110
+
+for.end110:
+  ret void
+}
diff --git a/test/CodeGen/Hexagon/hwloop1.ll b/test/CodeGen/Hexagon/hwloop1.ll
new file mode 100644
index 00000000000..97b779cf962
--- /dev/null
+++ b/test/CodeGen/Hexagon/hwloop1.ll
@@ -0,0 +1,161 @@
+; RUN: llc -march=hexagon < %s | FileCheck %s
+; Check that we generate hardware loop instructions.
+
+; Case 1 : Loop with a constant number of iterations.
+; CHECK-LABEL: @hwloop1
+; CHECK: loop0(.LBB{{.}}_{{.}}, #10)
+; CHECK: endloop0
+
+@a = common global [10 x i32] zeroinitializer, align 4
+define i32 @hwloop1() nounwind {
+entry:
+  br label %for.body
+for.body:
+  %i.01 = phi i32 [ 0, %entry ], [ %inc, %for.body ]
+  %arrayidx = getelementptr inbounds [10 x i32], [10 x i32]* @a, i32 0, i32 %i.01
+  store i32 %i.01, i32* %arrayidx, align 4
+  %inc = add nsw i32 %i.01, 1
+  %exitcond = icmp eq i32 %inc, 10
+  br i1 %exitcond, label %for.end, label %for.body
+for.end:
+  ret i32 0
+}
+
+; Case 2 : Loop with a run-time number of iterations.
+; CHECK-LABEL: @hwloop2
+; CHECK: loop0(.LBB{{.}}_{{.}}, r{{[0-9]+}})
+; CHECK: endloop0
+
+define i32 @hwloop2(i32 %n, i32* nocapture %b) nounwind {
+entry:
+  %cmp1 = icmp sgt i32 %n, 0
+  br i1 %cmp1, label %for.body.preheader, label %for.end
+
+for.body.preheader:
+  br label %for.body
+
+for.body:
+  %a.03 = phi i32 [ %add, %for.body ], [ 0, %for.body.preheader ]
+  %i.02 = phi i32 [ %inc, %for.body ], [ 0, %for.body.preheader ]
+  %arrayidx = getelementptr inbounds i32, i32* %b, i32 %i.02
+  %0 = load i32, i32* %arrayidx, align 4
+  %add = add nsw i32 %0, %a.03
+  %inc = add nsw i32 %i.02, 1
+  %exitcond = icmp eq i32 %inc, %n
+  br i1 %exitcond, label %for.end.loopexit, label %for.body
+
+for.end.loopexit:
+  br label %for.end
+
+for.end:
+  %a.0.lcssa = phi i32 [ 0, %entry ], [ %add, %for.end.loopexit ]
+  ret i32 %a.0.lcssa
+}
+
+; Case 3 : Induction variable increment more than 1.
+; CHECK-LABEL: @hwloop3
+; CHECK: lsr(r{{[0-9]+}}, #2)
+; CHECK: loop0(.LBB{{.}}_{{.}}, r{{[0-9]+}})
+; CHECK: endloop0
+
+define i32 @hwloop3(i32 %n, i32* nocapture %b) nounwind {
+entry:
+  %cmp1 = icmp sgt i32 %n, 0
+  br i1 %cmp1, label %for.body.preheader, label %for.end
+
+for.body.preheader:
+  br label %for.body
+
+for.body:
+  %a.03 = phi i32 [ %add, %for.body ], [ 0, %for.body.preheader ]
+  %i.02 = phi i32 [ %inc, %for.body ], [ 0, %for.body.preheader ]
+  %arrayidx = getelementptr inbounds i32, i32* %b, i32 %i.02
+  %0 = load i32, i32* %arrayidx, align 4
+  %add = add nsw i32 %0, %a.03
+  %inc = add nsw i32 %i.02, 4
+  %exitcond = icmp eq i32 %inc, %n
+  br i1 %exitcond, label %for.end.loopexit, label %for.body
+
+for.end.loopexit:
+  br label %for.end
+
+for.end:
+  %a.0.lcssa = phi i32 [ 0, %entry ], [ %add, %for.end.loopexit ]
+  ret i32 %a.0.lcssa
+}
+
+; Case 4 : Loop exit compare uses register instead of immediate value.
+; CHECK-LABEL: @hwloop4
+; CHECK: loop0(.LBB{{.}}_{{.}}, r{{[0-9]+}})
+; CHECK: endloop0
+
+define i32 @hwloop4(i32 %n, i32* nocapture %b) nounwind {
+entry:
+  %cmp1 = icmp sgt i32 %n, 0
+  br i1 %cmp1, label %for.body.preheader, label %for.end
+
+for.body.preheader:
+  br label %for.body
+
+for.body:
+  %i.02 = phi i32 [ %inc, %for.body ], [ 0, %for.body.preheader ]
+  %arrayidx = getelementptr inbounds i32, i32* %b, i32 %i.02
+  store i32 %i.02, i32* %arrayidx, align 4
+  %inc = add nsw i32 %i.02, 1
+  %exitcond = icmp eq i32 %inc, %n
+  br i1 %exitcond, label %for.end.loopexit, label %for.body
+
+for.end.loopexit:
+  br label %for.end
+
+for.end:
+  ret i32 0
+}
+
+; Case 5: After LSR, the initial value is 100 and the iv decrements to 0.
+; CHECK-LABEL: @hwloop5
+; CHECK: loop0(.LBB{{.}}_{{.}}, #100)
+; CHECK: endloop0
+
+define void @hwloop5(i32* nocapture %a, i32* nocapture %res) nounwind {
+entry:
+  br label %for.body
+
+for.body:
+  %i.03 = phi i32 [ 0, %entry ], [ %inc, %for.body ]
+  %arrayidx = getelementptr inbounds i32, i32* %a, i32 %i.03
+  %0 = load i32, i32* %arrayidx, align 4
+  %mul = mul nsw i32 %0, %0
+  %arrayidx2 = getelementptr inbounds i32, i32* %res, i32 %i.03
+  store i32 %mul, i32* %arrayidx2, align 4
+  %inc = add nsw i32 %i.03, 1
+  %exitcond = icmp eq i32 %inc, 100
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:
+  ret void
+}
+
+; Case 6: Large immediate offset
+; CHECK-LABEL: @hwloop6
+; CHECK-NOT: loop0(.LBB{{.}}_{{.}}, #1024)
+; CHECK: loop0(.LBB{{.}}_{{.}}, r{{[0-9]+}})
+; CHECK: endloop0
+
+define void @hwloop6(i32* nocapture %a, i32* nocapture %res) nounwind {
+entry:
+  br label %for.body
+
+for.body:
+  %i.02 = phi i32 [ 0, %entry ], [ %inc, %for.body ]
+  %arrayidx = getelementptr inbounds i32, i32* %a, i32 %i.02
+  %0 = load i32, i32* %arrayidx, align 4
+  %arrayidx1 = getelementptr inbounds i32, i32* %res, i32 %i.02
+  store i32 %0, i32* %arrayidx1, align 4
+  %inc = add nsw i32 %i.02, 1
+  %exitcond = icmp eq i32 %inc, 1024
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:
+  ret void
+}
diff --git a/test/CodeGen/Hexagon/hwloop2.ll b/test/CodeGen/Hexagon/hwloop2.ll
new file mode 100644
index 00000000000..d411d979904
--- /dev/null
+++ b/test/CodeGen/Hexagon/hwloop2.ll
@@ -0,0 +1,37 @@
+; RUN: llc -disable-lsr -march=hexagon < %s | FileCheck %s
+
+; Test for multiple phis with induction variables.
+
+; CHECK: loop0(.LBB{{.}}_{{.}}, r{{[0-9]+}})
+; CHECK: endloop0
+
+define i32 @hwloop4(i32* nocapture %s, i32* nocapture %a, i32 %n) {
+entry:
+  %cmp3 = icmp eq i32 %n, 0
+  br i1 %cmp3, label %for.end, label %for.body.lr.ph
+
+for.body.lr.ph:
+  %.pre = load i32, i32* %s, align 4
+  br label %for.body
+
+for.body:
+  %0 = phi i32 [ %.pre, %for.body.lr.ph ], [ %add1, %for.body ]
+  %j.05 = phi i32 [ 0, %for.body.lr.ph ], [ %add2, %for.body ]
+  %lsr.iv = phi i32 [ %lsr.iv.next, %for.body ], [ %n, %for.body.lr.ph ]
+  %lsr.iv1 = phi i32* [ %scevgep, %for.body ], [ %a, %for.body.lr.ph ]
+  %1 = load i32, i32* %lsr.iv1, align 4
+  %add1 = add nsw i32 %0, %1
+  store i32 %add1, i32* %s, align 4
+  %add2 = add nsw i32 %j.05, 1
+  %lsr.iv.next = add i32 %lsr.iv, -1
+  %scevgep = getelementptr i32, i32* %lsr.iv1, i32 1
+  %cmp = icmp eq i32 %lsr.iv.next, 0
+  br i1 %cmp, label %for.end.loopexit, label %for.body
+
+for.end.loopexit:
+  br label %for.end
+
+for.end:
+  %j.0.lcssa = phi i32 [ 0, %entry ], [ %add2, %for.end.loopexit ]
+  ret i32 %j.0.lcssa
+}
diff --git a/test/CodeGen/Hexagon/hwloop3.ll b/test/CodeGen/Hexagon/hwloop3.ll
new file mode 100644
index 00000000000..1135e06a0c4
--- /dev/null
+++ b/test/CodeGen/Hexagon/hwloop3.ll
@@ -0,0 +1,27 @@
+; RUN: llc -march=hexagon < %s | FileCheck %s
+;
+; Remove the unconditional jump to following instruction.
+
+; CHECK: endloop0
+; CHECK-NOT: jump [[L1:.]]
+; CHECK-NOT: [[L1]]
+
+define void @test(i32* nocapture %a, i32 %n) nounwind {
+entry:
+  br label %for.body
+
+for.body:
+  %arrayidx.phi = phi i32* [ %a, %entry ], [ %arrayidx.inc, %for.body ]
+  %i.02 = phi i32 [ 0, %entry ], [ %inc, %for.body ]
+  %0 = load i32, i32* %arrayidx.phi, align 4
+  %add = add nsw i32 %0, 1
+  store i32 %add, i32* %arrayidx.phi, align 4
+  %inc = add nsw i32 %i.02, 1
+  %exitcond = icmp eq i32 %inc, 100
+  %arrayidx.inc = getelementptr i32, i32* %arrayidx.phi, i32 1
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:
+  ret void
+}
+
diff --git a/test/CodeGen/Hexagon/hwloop4.ll b/test/CodeGen/Hexagon/hwloop4.ll
new file mode 100644
index 00000000000..d159c45e3fb
--- /dev/null
+++ b/test/CodeGen/Hexagon/hwloop4.ll
@@ -0,0 +1,76 @@
+; RUN: llc -march=hexagon -mcpu=hexagonv5 < %s | FileCheck %s
+;
+; Remove the unnecessary 'add' instruction used for the hardware loop setup.
+
+; CHECK: [[OP0:r[0-9]+]] = add([[OP1:r[0-9]+]], #-[[OP2:[0-9]+]]
+; CHECK-NOT: add([[OP0]], #[[OP2]])
+; CHECK: lsr([[OP1]], #{{[0-9]+}})
+; CHECK: loop0
+
+define void @matrix_mul_matrix(i32 %N, i32* nocapture %C, i16* nocapture readnone %A, i16* nocapture readnone %B) #0 {
+entry:
+  %cmp4 = icmp eq i32 %N, 0
+  br i1 %cmp4, label %for.end, label %for.body.preheader
+
+for.body.preheader:
+  %maxval = add i32 %N, -7
+  %0 = icmp sgt i32 %maxval, 0
+  br i1 %0, label %for.body.preheader9, label %for.body.ur.preheader
+
+for.body.preheader9:
+  br label %for.body
+
+for.body:
+  %arrayidx.phi = phi i32* [ %arrayidx.inc.7, %for.body ], [ %C, %for.body.preheader9 ]
+  %i.05 = phi i32 [ %inc.7, %for.body ], [ 0, %for.body.preheader9 ]
+  store i32 %i.05, i32* %arrayidx.phi, align 4
+  %inc = add i32 %i.05, 1
+  %arrayidx.inc = getelementptr i32, i32* %arrayidx.phi, i32 1
+  store i32 %inc, i32* %arrayidx.inc, align 4
+  %inc.1 = add i32 %i.05, 2
+  %arrayidx.inc.1 = getelementptr i32, i32* %arrayidx.phi, i32 2
+  store i32 %inc.1, i32* %arrayidx.inc.1, align 4
+  %inc.2 = add i32 %i.05, 3
+  %arrayidx.inc.2 = getelementptr i32, i32* %arrayidx.phi, i32 3
+  store i32 %inc.2, i32* %arrayidx.inc.2, align 4
+  %inc.3 = add i32 %i.05, 4
+  %arrayidx.inc.3 = getelementptr i32, i32* %arrayidx.phi, i32 4
+  store i32 %inc.3, i32* %arrayidx.inc.3, align 4
+  %inc.4 = add i32 %i.05, 5
+  %arrayidx.inc.4 = getelementptr i32, i32* %arrayidx.phi, i32 5
+  store i32 %inc.4, i32* %arrayidx.inc.4, align 4
+  %inc.5 = add i32 %i.05, 6
+  %arrayidx.inc.5 = getelementptr i32, i32* %arrayidx.phi, i32 6
+  store i32 %inc.5, i32* %arrayidx.inc.5, align 4
+  %inc.6 = add i32 %i.05, 7
+  %arrayidx.inc.6 = getelementptr i32, i32* %arrayidx.phi, i32 7
+  store i32 %inc.6, i32* %arrayidx.inc.6, align 4
+  %inc.7 = add i32 %i.05, 8
+  %exitcond.7 = icmp slt i32 %inc.7, %maxval
+  %arrayidx.inc.7 = getelementptr i32, i32* %arrayidx.phi, i32 8
+  br i1 %exitcond.7, label %for.body, label %for.end.loopexit.ur-lcssa
+
+for.end.loopexit.ur-lcssa:
+  %1 = icmp eq i32 %inc.7, %N
+  br i1 %1, label %for.end, label %for.body.ur.preheader
+
+for.body.ur.preheader:
+  %arrayidx.phi.ur.ph = phi i32* [ %C, %for.body.preheader ], [ %arrayidx.inc.7, %for.end.loopexit.ur-lcssa ]
+  %i.05.ur.ph = phi i32 [ 0, %for.body.preheader ], [ %inc.7, %for.end.loopexit.ur-lcssa ]
+  br label %for.body.ur
+
+for.body.ur:
+  %arrayidx.phi.ur = phi i32* [ %arrayidx.inc.ur, %for.body.ur ], [ %arrayidx.phi.ur.ph, %for.body.ur.preheader ]
+  %i.05.ur = phi i32 [ %inc.ur, %for.body.ur ], [ %i.05.ur.ph, %for.body.ur.preheader ]
+  store i32 %i.05.ur, i32* %arrayidx.phi.ur, align 4
+  %inc.ur = add i32 %i.05.ur, 1
+  %exitcond.ur = icmp eq i32 %inc.ur, %N
+  %arrayidx.inc.ur = getelementptr i32, i32* %arrayidx.phi.ur, i32 1
+  br i1 %exitcond.ur, label %for.end.loopexit, label %for.body.ur
+
+for.end.loopexit:
+  br label %for.end
+
+for.end:
+  ret void
+}
diff --git a/test/CodeGen/Hexagon/hwloop5.ll b/test/CodeGen/Hexagon/hwloop5.ll
new file mode 100644
index 00000000000..0886b03cc75
--- /dev/null
+++ b/test/CodeGen/Hexagon/hwloop5.ll
@@ -0,0 +1,93 @@
+; RUN: llc -O3 -march=hexagon -mcpu=hexagonv5 < %s | FileCheck %s
+;
+; Generate hardware loop when unknown trip count loop is vectorized.
+
+; CHECK: loop0(.LBB{{[0-9]*}}_{{[0-9]*}}, r{{[0-9]+}})
+; CHECK: endloop0
+; CHECK: loop0(.LBB{{[0-9]*}}_{{[0-9]*}}, r{{[0-9]+}})
+; CHECK: endloop0
+
+@A = common global [1000 x i32] zeroinitializer, align 8
+@B = common global [1000 x i32] zeroinitializer, align 8
+
+define i32 @dotprod2(i32 %count) #0 {
+entry.split:
+  %cmp6 = icmp sgt i32 %count, 0
+  br i1 %cmp6, label %polly.cond, label %for.end
+
+for.end.loopexit:
+  br label %for.end
+
+for.end:
+  %sum.0.lcssa.reg2mem.0.load37 = phi i32 [ 0, %entry.split ], [ %p_add34, %polly.loop_if13 ], [ %p_add, %for.end.loopexit ]
+  ret i32 %sum.0.lcssa.reg2mem.0.load37
+
+polly.cond:
+  %0 = icmp sgt i32 %count, 1
+  br i1 %0, label %polly.loop_if, label %polly.loop_if13
+
+polly.loop_exit.loopexit:
+  br label %polly.loop_exit
+
+polly.loop_exit:
+  %1 = phi <2 x i32> [ zeroinitializer, %polly.loop_if ], [ %addp_vec, %polly.loop_exit.loopexit ]
+  %2 = extractelement <2 x i32> %1, i32 0
+  %3 = extractelement <2 x i32> %1, i32 1
+  %add_sum = add i32 %2, %3
+  br label %polly.loop_if13
+
+polly.loop_if:
+  %4 = add i32 %count, -1
+  %leftover_lb = and i32 %4, -2
+  %polly.loop_guard = icmp eq i32 %leftover_lb, 0
+  br i1 %polly.loop_guard, label %polly.loop_exit, label %polly.loop_preheader
+
+polly.stmt.for.body:
+  %addp_vec28 = phi <2 x i32> [ zeroinitializer, %polly.loop_preheader ], [ %addp_vec, %polly.stmt.for.body ]
+  %scevgep.phi = phi i32* [ getelementptr inbounds ([1000 x i32], [1000 x i32]* @A, i32 0, i32 0), %polly.loop_preheader ], [ %scevgep.inc, %polly.stmt.for.body ]
+  %scevgep9.phi = phi i32* [ getelementptr inbounds ([1000 x i32], [1000 x i32]* @B, i32 0, i32 0), %polly.loop_preheader ], [ %scevgep9.inc, %polly.stmt.for.body ]
+  %polly.indvar = phi i32 [ 0, %polly.loop_preheader ], [ %polly.indvar_next, %polly.stmt.for.body ]
+  %vector_ptr = bitcast i32* %scevgep.phi to <2 x i32>*
+  %_p_vec_full = load <2 x i32>, <2 x i32>* %vector_ptr, align 8
+  %vector_ptr10 = bitcast i32* %scevgep9.phi to <2 x i32>*
+  %_p_vec_full11 = load <2 x i32>, <2 x i32>* %vector_ptr10, align 8
+  %mulp_vec = mul <2 x i32> %_p_vec_full11, %_p_vec_full
+  %addp_vec = add <2 x i32> %mulp_vec, %addp_vec28
+  %polly.indvar_next = add nsw i32 %polly.indvar, 2
+  %polly.loop_cond = icmp eq i32 %polly.indvar, %polly.adjust_ub
+  %scevgep.inc = getelementptr i32, i32* %scevgep.phi, i32 2
+  %scevgep9.inc = getelementptr i32, i32* %scevgep9.phi, i32 2
+  br i1 %polly.loop_cond, label %polly.loop_exit.loopexit, label %polly.stmt.for.body
+
+polly.loop_preheader:
+  %polly.adjust_ub = add i32 %leftover_lb, -2
+  br label %polly.stmt.for.body
+
+polly.loop_if13:
+  %p_add34 = phi i32 [ 0, %polly.cond ], [ %add_sum, %polly.loop_exit ]
+  %merge.lb = phi i32 [ 0, %polly.cond ], [ %leftover_lb, %polly.loop_exit ]
+  %polly.loop_guard17 = icmp slt i32 %merge.lb, %count
+  br i1 %polly.loop_guard17, label %polly.loop_preheader15, label %for.end
+
+polly.stmt.for.body22:
+  %p_add30 = phi i32 [ %p_add34, %polly.loop_preheader15 ], [ %p_add, %polly.stmt.for.body22 ]
+  %polly.indvar18 = phi i32 [ %merge.lb, %polly.loop_preheader15 ], [ %polly.indvar_next19, %polly.stmt.for.body22 ]
+  %5 = tail call i32 @llvm.annotation.i32(i32 %polly.indvar18, i8* null, i8* null, i32 0), !polly.loop.smallTripCount !0
+  %scevgep23 = getelementptr [1000 x i32], [1000 x i32]* @A, i32 0, i32 %polly.indvar18
+  %_p_scalar_ = load i32, i32* %scevgep23, align 4
+  %scevgep24 = getelementptr [1000 x i32], [1000 x i32]* @B, i32 0, i32 %polly.indvar18
+  %_p_scalar_25 = load i32, i32* %scevgep24, align 4
+  %p_mul = mul nsw i32 %_p_scalar_25, %_p_scalar_
+  %p_add = add nsw i32 %p_mul, %p_add30
+  %polly.indvar_next19 = add nsw i32 %polly.indvar18, 1
+  %polly.loop_cond21 = icmp slt i32 %polly.indvar18, %polly.adjust_ub20
+  br i1 %polly.loop_cond21, label %polly.stmt.for.body22, label %for.end.loopexit
+
+polly.loop_preheader15:
+  %polly.adjust_ub20 = add i32 %count, -1
+  br label %polly.stmt.for.body22
+}
+
+declare i32 @llvm.annotation.i32(i32, i8*, i8*, i32) #1
+
+!0 = !{}
diff --git a/test/CodeGen/Hexagon/insert-basic.ll b/test/CodeGen/Hexagon/insert-basic.ll
new file mode 100644
index 00000000000..e941c063d9e
--- /dev/null
+++ b/test/CodeGen/Hexagon/insert-basic.ll
@@ -0,0 +1,66 @@
+; RUN: llc -O2 -march=hexagon < %s | FileCheck %s
+; CHECK-DAG: insert(r{{[0-9]*}}, #17, #0)
+; CHECK-DAG: insert(r{{[0-9]*}}, #18, #0)
+; CHECK-DAG: insert(r{{[0-9]*}}, #22, #0)
+; CHECK-DAG: insert(r{{[0-9]*}}, #12, #0)
+
+; C source:
+; typedef struct {
+;   unsigned x1:23;
+;   unsigned x2:17;
+;   unsigned x3:18;
+;   unsigned x4:22;
+;   unsigned x5:12;
+; } structx_t;
+;
+; void foo(structx_t *px, int y1, int y2, int y3, int y4, int y5) {
+;   px->x1 = y1;
+;   px->x2 = y2;
+;   px->x3 = y3;
+;   px->x4 = y4;
+;   px->x5 = y5;
+; }
+
+target datalayout = "e-p:32:32:32-i64:64:64-i32:32:32-i16:16:16-i1:32:32-f64:64:64-f32:32:32-v64:64:64-v32:32:32-a0:0-n16:32"
+target triple = "hexagon"
+
+%struct.structx_t = type { [3 x i8], i8, [3 x i8], i8, [3 x i8], i8, [3 x i8], i8, [2 x i8], [2 x i8] }
+
+define void @foo(%struct.structx_t* nocapture %px, i32 %y1, i32 %y2, i32 %y3, i32 %y4, i32 %y5) nounwind {
+entry:
+  %bf.value = and i32 %y1, 8388607
+  %0 = bitcast %struct.structx_t* %px to i32*
+  %1 = load i32, i32* %0, align 4
+  %2 = and i32 %1, -8388608
+  %3 = or i32 %2, %bf.value
+  store i32 %3, i32* %0, align 4
+  %bf.value1 = and i32 %y2, 131071
+  %bf.field.offs = getelementptr %struct.structx_t, %struct.structx_t* %px, i32 0, i32 0, i32 4
+  %4 = bitcast i8* %bf.field.offs to i32*
+  %5 = load i32, i32* %4, align 4
+  %6 = and i32 %5, -131072
+  %7 = or i32 %6, %bf.value1
+  store i32 %7, i32* %4, align 4
+  %bf.value2 = and i32 %y3, 262143
+  %bf.field.offs3 = getelementptr %struct.structx_t, %struct.structx_t* %px, i32 0, i32 0, i32 8
+  %8 = bitcast i8* %bf.field.offs3 to i32*
+  %9 = load i32, i32* %8, align 4
+  %10 = and i32 %9, -262144
+  %11 = or i32 %10, %bf.value2
+  store i32 %11, i32* %8, align 4
+  %bf.value4 = and i32 %y4, 4194303
+  %bf.field.offs5 = getelementptr %struct.structx_t, %struct.structx_t* %px, i32 0, i32 0, i32 12
+  %12 = bitcast i8* %bf.field.offs5 to i32*
+  %13 = load i32, i32* %12, align 4
+  %14 = and i32 %13, -4194304
+  %15 = or i32 %14, %bf.value4
+  store i32 %15, i32* %12, align 4
+  %bf.value6 = and i32 %y5, 4095
+  %bf.field.offs7 = getelementptr %struct.structx_t, %struct.structx_t* %px, i32 0, i32 0, i32 16
+  %16 = bitcast i8* %bf.field.offs7 to i32*
+  %17 = load i32, i32* %16, align 4
+  %18 = and i32 %17, -4096
+  %19 = or i32 %18, %bf.value6
+  store i32 %19, i32* %16, align 4
+  ret void
+}
diff --git a/test/CodeGen/Hexagon/intrinsics/alu32_alu.ll b/test/CodeGen/Hexagon/intrinsics/alu32_alu.ll
index 37f9f4007b6..fcf80b08181 100644
--- a/test/CodeGen/Hexagon/intrinsics/alu32_alu.ll
+++ b/test/CodeGen/Hexagon/intrinsics/alu32_alu.ll
@@ -1,27 +1,30 @@
 ; RUN: llc -march=hexagon -O0 < %s | FileCheck %s
+; RUN: llc -march=hexagon -O0 < %s | FileCheck -check-prefix=CHECK-CALL %s
 ; Hexagon Programmer's Reference Manual 11.1.1 ALU32/ALU
 
+; CHECK-CALL-NOT: call
+
 ; Add
 declare i32 @llvm.hexagon.A2.addi(i32, i32)
 define i32 @A2_addi(i32 %a) {
   %z = call i32 @llvm.hexagon.A2.addi(i32 %a, i32 0)
   ret i32 %z
 }
-; CHECK: r0 = add(r0, #0)
+; CHECK: = add({{.*}}, #0)
 
 declare i32 @llvm.hexagon.A2.add(i32, i32)
 define i32 @A2_add(i32 %a, i32 %b) {
   %z = call i32 @llvm.hexagon.A2.add(i32 %a, i32 %b)
   ret i32 %z
 }
-; CHECK: r0 = add(r0, r1)
+; CHECK: = add({{.*}}, {{.*}})
 
 declare i32 @llvm.hexagon.A2.addsat(i32, i32)
 define i32 @A2_addsat(i32 %a, i32 %b) {
   %z = call i32 @llvm.hexagon.A2.addsat(i32 %a, i32 %b)
   ret i32 %z
 }
-; CHECK: r0 = add(r0, r1):sat
+; CHECK: = add({{.*}}, {{.*}}):sat
 
 ; Logical operations
 declare i32 @llvm.hexagon.A2.and(i32, i32)
@@ -29,43 +32,35 @@ define i32 @A2_and(i32 %a, i32 %b) {
   %z = call i32 @llvm.hexagon.A2.and(i32 %a, i32 %b)
   ret i32 %z
 }
-; CHECK: r0 = and(r0, r1)
+; CHECK: = and({{.*}}, {{.*}})
 
 declare i32 @llvm.hexagon.A2.or(i32, i32)
 define i32 @A2_or(i32 %a, i32 %b) {
   %z = call i32 @llvm.hexagon.A2.or(i32 %a, i32 %b)
   ret i32 %z
 }
-; CHECK: r0 = or(r0, r1)
+; CHECK: = or({{.*}}, {{.*}})
 
 declare i32 @llvm.hexagon.A2.xor(i32, i32)
 define i32 @A2_xor(i32 %a, i32 %b) {
   %z = call i32 @llvm.hexagon.A2.xor(i32 %a, i32 %b)
   ret i32 %z
 }
-; CHECK: r0 = xor(r0, r1)
+; CHECK: = xor({{.*}}, {{.*}})
 
 declare i32 @llvm.hexagon.A4.andn(i32, i32)
 define i32 @A4_andn(i32 %a, i32 %b) {
   %z = call i32 @llvm.hexagon.A4.andn(i32 %a, i32 %b)
   ret i32 %z
 }
-; CHECK: r0 = and(r0, ~r1)
+; CHECK: = and({{.*}}, ~{{.*}})
 
 declare i32 @llvm.hexagon.A4.orn(i32, i32)
 define i32 @A4_orn(i32 %a, i32 %b) {
   %z = call i32 @llvm.hexagon.A4.orn(i32 %a, i32 %b)
   ret i32 %z
 }
-; CHECK: r0 = or(r0, ~r1)
-
-; Nop
-declare void @llvm.hexagon.A2.nop()
-define void @A2_nop(i32 %a, i32 %b) {
-  call void @llvm.hexagon.A2.nop()
-  ret void
-}
-; CHECK: nop
+; CHECK: = or({{.*}}, ~{{.*}})
 
 ; Subtract
 declare i32 @llvm.hexagon.A2.sub(i32, i32)
@@ -73,14 +68,14 @@ define i32 @A2_sub(i32 %a, i32 %b) {
   %z = call i32 @llvm.hexagon.A2.sub(i32 %a, i32 %b)
   ret i32 %z
 }
-; CHECK: r0 = sub(r0, r1)
+; CHECK: = sub({{.*}}, {{.*}})
 
 declare i32 @llvm.hexagon.A2.subsat(i32, i32)
 define i32 @A2_subsat(i32 %a, i32 %b) {
   %z = call i32 @llvm.hexagon.A2.subsat(i32 %a, i32 %b)
   ret i32 %z
 }
-; CHECK: r0 = sub(r0, r1):sat
+; CHECK: = sub({{.*}}, {{.*}}):sat
 
 ; Sign extend
 declare i32 @llvm.hexagon.A2.sxtb(i32)
@@ -88,14 +83,14 @@ define i32 @A2_sxtb(i32 %a) {
   %z = call i32 @llvm.hexagon.A2.sxtb(i32 %a)
   ret i32 %z
 }
-; CHECK: r0 = sxtb(r0)
+; CHECK: = sxtb({{.*}})
 
 declare i32 @llvm.hexagon.A2.sxth(i32)
 define i32 @A2_sxth(i32 %a) {
   %z = call i32 @llvm.hexagon.A2.sxth(i32 %a)
   ret i32 %z
 }
-; CHECK: r0 = sxth(r0)
+; CHECK: = sxth({{.*}})
 
 ; Transfer immediate
 declare i32 @llvm.hexagon.A2.tfril(i32, i32)
@@ -103,21 +98,21 @@ define i32 @A2_tfril(i32 %a) {
   %z = call i32 @llvm.hexagon.A2.tfril(i32 %a, i32 0)
   ret i32 %z
 }
-; CHECK: r0.l = #0
+; CHECK: = #0
 
 declare i32 @llvm.hexagon.A2.tfrih(i32, i32)
 define i32 @A2_tfrih(i32 %a) {
   %z = call i32 @llvm.hexagon.A2.tfrih(i32 %a, i32 0)
   ret i32 %z
 }
-; CHECK: r0.h = #0
+; CHECK: = #0
 
 declare i32 @llvm.hexagon.A2.tfrsi(i32)
 define i32 @A2_tfrsi() {
   %z = call i32 @llvm.hexagon.A2.tfrsi(i32 0)
   ret i32 %z
 }
-; CHECK: r0 = #0
+; CHECK: = #0
 
 ; Transfer register
 declare i32 @llvm.hexagon.A2.tfr(i32)
@@ -125,7 +120,7 @@ define i32 @A2_tfr(i32 %a) {
   %z = call i32 @llvm.hexagon.A2.tfr(i32 %a)
   ret i32 %z
 }
-; CHECK: r0 = r0
+; CHECK: =
 
 ; Vector add halfwords
 declare i32 @llvm.hexagon.A2.svaddh(i32, i32)
@@ -133,21 +128,21 @@ define i32 @A2_svaddh(i32 %a, i32 %b) {
   %z = call i32 @llvm.hexagon.A2.svaddh(i32 %a, i32 %b)
   ret i32 %z
 }
-; CHECK: r0 = vaddh(r0, r1)
+; CHECK: = vaddh({{.*}}, {{.*}})
 
 declare i32 @llvm.hexagon.A2.svaddhs(i32, i32)
 define i32 @A2_svaddhs(i32 %a, i32 %b) {
   %z = call i32 @llvm.hexagon.A2.svaddhs(i32 %a, i32 %b)
   ret i32 %z
 }
-; CHECK: r0 = vaddh(r0, r1):sat
+; CHECK: = vaddh({{.*}}, {{.*}}):sat
 
 declare i32 @llvm.hexagon.A2.svadduhs(i32, i32)
 define i32 @A2_svadduhs(i32 %a, i32 %b) {
   %z = call i32 @llvm.hexagon.A2.svadduhs(i32 %a, i32 %b)
   ret i32 %z
 }
-; CHECK: r0 = vadduh(r0, r1):sat
+; CHECK: = vadduh({{.*}}, {{.*}}):sat
 
 ; Vector average halfwords
 declare i32 @llvm.hexagon.A2.svavgh(i32, i32)
@@ -155,21 +150,21 @@ define i32 @A2_svavgh(i32 %a, i32 %b) {
   %z = call i32 @llvm.hexagon.A2.svavgh(i32 %a, i32 %b)
   ret i32 %z
 }
-; CHECK: r0 = vavgh(r0, r1)
+; CHECK: = vavgh({{.*}}, {{.*}})
 
 declare i32 @llvm.hexagon.A2.svavghs(i32, i32)
 define i32 @A2_svavghs(i32 %a, i32 %b) {
   %z = call i32 @llvm.hexagon.A2.svavghs(i32 %a, i32 %b)
   ret i32 %z
 }
-; CHECK: r0 = vavgh(r0, r1):rnd
+; CHECK: = vavgh({{.*}}, {{.*}}):rnd
 
 declare i32 @llvm.hexagon.A2.svnavgh(i32, i32)
 define i32 @A2_svnavgh(i32 %a, i32 %b) {
   %z = call i32 @llvm.hexagon.A2.svnavgh(i32 %a, i32 %b)
   ret i32 %z
 }
-; CHECK: r0 = vnavgh(r0, r1)
+; CHECK: = vnavgh({{.*}}, {{.*}})
 
 ; Vector subtract halfwords
 declare i32 @llvm.hexagon.A2.svsubh(i32, i32)
@@ -177,21 +172,21 @@ define i32 @A2_svsubh(i32 %a, i32 %b) {
   %z = call i32 @llvm.hexagon.A2.svsubh(i32 %a, i32 %b)
   ret i32 %z
 }
-; CHECK: r0 = vsubh(r0, r1)
+; CHECK: = vsubh({{.*}}, {{.*}})
 
 declare i32 @llvm.hexagon.A2.svsubhs(i32, i32)
 define i32 @A2_svsubhs(i32 %a, i32 %b) {
   %z = call i32 @llvm.hexagon.A2.svsubhs(i32 %a, i32 %b)
   ret i32 %z
 }
-; CHECK: r0 = vsubh(r0, r1):sat
+; CHECK: = vsubh({{.*}}, {{.*}}):sat
 
 declare i32 @llvm.hexagon.A2.svsubuhs(i32, i32)
 define i32 @A2_svsubuhs(i32 %a, i32 %b) {
   %z = call i32 @llvm.hexagon.A2.svsubuhs(i32 %a, i32 %b)
   ret i32 %z
 }
-; CHECK: r0 = vsubuh(r0, r1):sat
+; CHECK: = vsubuh({{.*}}, {{.*}}):sat
 
 ; Zero extend
 declare i32 @llvm.hexagon.A2.zxth(i32)
@@ -199,4 +194,4 @@ define i32 @A2_zxth(i32 %a) {
   %z = call i32 @llvm.hexagon.A2.zxth(i32 %a)
   ret i32 %z
 }
-; CHECK: r0 = zxth(r0)
+; CHECK: = zxth({{.*}})
diff --git a/test/CodeGen/Hexagon/intrinsics/alu32_perm.ll b/test/CodeGen/Hexagon/intrinsics/alu32_perm.ll
index a9cc01c5dcb..c9fb0afe078 100644
--- a/test/CodeGen/Hexagon/intrinsics/alu32_perm.ll
+++ b/test/CodeGen/Hexagon/intrinsics/alu32_perm.ll
@@ -1,62 +1,65 @@
 ; RUN: llc -march=hexagon -O0 < %s | FileCheck %s
+; RUN: llc -march=hexagon -O0 < %s | FileCheck -check-prefix=CHECK-CALL %s
 ; Hexagon Programmer's Reference Manual 11.1.2 ALU32/PERM
 
+; CHECK-CALL-NOT: call
+
 ; Combine words into doubleword
 declare i64 @llvm.hexagon.A4.combineri(i32, i32)
 define i64 @A4_combineri(i32 %a) {
   %z = call i64 @llvm.hexagon.A4.combineri(i32 %a, i32 0)
   ret i64 %z
 }
-; CHECK:  = combine(r0, #0)
+; CHECK: = combine({{.*}}, #0)
 
 declare i64 @llvm.hexagon.A4.combineir(i32, i32)
 define i64 @A4_combineir(i32 %a) {
   %z = call i64 @llvm.hexagon.A4.combineir(i32 0, i32 %a)
   ret i64 %z
 }
-; CHECK:  = combine(#0, r0)
+; CHECK: = combine(#0, {{.*}})
 
 declare i64 @llvm.hexagon.A2.combineii(i32, i32)
 define i64 @A2_combineii() {
   %z = call i64 @llvm.hexagon.A2.combineii(i32 0, i32 0)
   ret i64 %z
 }
-; CHECK: r1:0 = combine(#0, #0)
+; CHECK: = combine(#0, #0)
 
 declare i32 @llvm.hexagon.A2.combine.hh(i32, i32)
 define i32 @A2_combine_hh(i32 %a, i32 %b) {
   %z = call i32 @llvm.hexagon.A2.combine.hh(i32 %a, i32 %b)
   ret i32 %z
 }
-; CHECK: r0 = combine(r0.h, r1.h)
+; CHECK: = combine({{.*}}, {{.*}})
 
 declare i32 @llvm.hexagon.A2.combine.hl(i32, i32)
 define i32 @A2_combine_hl(i32 %a, i32 %b) {
   %z = call i32 @llvm.hexagon.A2.combine.hl(i32 %a, i32 %b)
   ret i32 %z
 }
-; CHECK: r0 = combine(r0.h, r1.l)
+; CHECK: = combine({{.*}}, {{.*}})
 
 declare i32 @llvm.hexagon.A2.combine.lh(i32, i32)
 define i32 @A2_combine_lh(i32 %a, i32 %b) {
   %z = call i32 @llvm.hexagon.A2.combine.lh(i32 %a, i32 %b)
   ret i32 %z
 }
-; CHECK: r0 = combine(r0.l, r1.h)
+; CHECK: = combine({{.*}}, {{.*}})
 
 declare i32 @llvm.hexagon.A2.combine.ll(i32, i32)
 define i32 @A2_combine_ll(i32 %a, i32 %b) {
   %z = call i32 @llvm.hexagon.A2.combine.ll(i32 %a, i32 %b)
   ret i32 %z
 }
-; CHECK: r0 = combine(r0.l, r1.l)
+; CHECK: = combine({{.*}}, {{.*}})
 
 declare i64 @llvm.hexagon.A2.combinew(i32, i32)
 define i64 @A2_combinew(i32 %a, i32 %b) {
   %z = call i64 @llvm.hexagon.A2.combinew(i32 %a, i32 %b)
   ret i64 %z
 }
-; CHECK: r1:0 = combine(r0, r1)
+; CHECK: = combine({{.*}}, {{.*}})
 
 ; Mux
 declare i32 @llvm.hexagon.C2.muxri(i32, i32, i32)
@@ -64,21 +67,21 @@ define i32 @C2_muxri(i32 %a, i32 %b) {
   %z = call i32 @llvm.hexagon.C2.muxri(i32 %a, i32 0, i32 %b)
   ret i32 %z
 }
-; CHECK: r0 = mux(p0, #0, r1)
+; CHECK: = mux({{.*}}, #0, {{.*}})
 
 declare i32 @llvm.hexagon.C2.muxir(i32, i32, i32)
 define i32 @C2_muxir(i32 %a, i32 %b) {
   %z = call i32 @llvm.hexagon.C2.muxir(i32 %a, i32 %b, i32 0)
   ret i32 %z
 }
-; CHECK: r0 = mux(p0, r1, #0)
+; CHECK: = mux({{.*}}, {{.*}}, #0)
 
 declare i32 @llvm.hexagon.C2.mux(i32, i32, i32)
 define i32 @C2_mux(i32 %a, i32 %b, i32 %c) {
   %z = call i32 @llvm.hexagon.C2.mux(i32 %a, i32 %b, i32 %c)
   ret i32 %z
 }
-; CHECK: r0 = mux(p0, r1, r2)
+; CHECK: = mux({{.*}}, {{.*}}, {{.*}})
 
 ; Shift word by 16
 declare i32 @llvm.hexagon.A2.aslh(i32)
@@ -86,14 +89,14 @@ define i32 @A2_aslh(i32 %a) {
   %z = call i32 @llvm.hexagon.A2.aslh(i32 %a)
   ret i32 %z
 }
-; CHECK: r0 = aslh(r0)
+; CHECK: = aslh({{.*}})
 
 declare i32 @llvm.hexagon.A2.asrh(i32)
 define i32 @A2_asrh(i32 %a) {
   %z = call i32 @llvm.hexagon.A2.asrh(i32 %a)
   ret i32 %z
 }
-; CHECK: r0 = asrh(r0)
+; CHECK: = asrh({{.*}})
 
 ; Pack high and low halfwords
 declare i64 @llvm.hexagon.S2.packhl(i32, i32)
@@ -101,4 +104,4 @@ define i64 @S2_packhl(i32 %a, i32 %b) {
   %z = call i64 @llvm.hexagon.S2.packhl(i32 %a, i32 %b)
   ret i64 %z
 }
-; CHECK: r1:0 = packhl(r0, r1)
+; CHECK: = packhl({{.*}}, {{.*}})
diff --git a/test/CodeGen/Hexagon/intrinsics/cr.ll b/test/CodeGen/Hexagon/intrinsics/cr.ll
index 9bdcb253fe2..f308ef8e566 100644
--- a/test/CodeGen/Hexagon/intrinsics/cr.ll
+++ b/test/CodeGen/Hexagon/intrinsics/cr.ll
@@ -1,20 +1,23 @@
 ; RUN: llc -march=hexagon < %s | FileCheck %s
+; RUN: llc -march=hexagon -O0 < %s | FileCheck -check-prefix=CHECK-CALL %s
 ; Hexagon Programmer's Reference Manual 11.2 CR
 
+; CHECK-CALL-NOT: call
+
 ; Corner detection acceleration
 declare i32 @llvm.hexagon.C4.fastcorner9(i32, i32)
 define i32 @C4_fastcorner9(i32 %a, i32 %b) {
   %z = call i32@llvm.hexagon.C4.fastcorner9(i32 %a, i32 %b)
   ret i32 %z
 }
-; CHECK: p0 = fastcorner9(p0, p1)
+; CHECK: = fastcorner9({{.*}}, {{.*}})
 
 declare i32 @llvm.hexagon.C4.fastcorner9.not(i32, i32)
 define i32 @C4_fastcorner9_not(i32 %a, i32 %b) {
   %z = call i32@llvm.hexagon.C4.fastcorner9.not(i32 %a, i32 %b)
   ret i32 %z
 }
-; CHECK: p0 = !fastcorner9(p0, p1)
+; CHECK: = !fastcorner9({{.*}}, {{.*}})
 
 ; Logical reductions on predicates
 declare i32 @llvm.hexagon.C2.any8(i32)
@@ -22,7 +25,7 @@ define i32 @C2_any8(i32 %a) {
   %z = call i32@llvm.hexagon.C2.any8(i32 %a)
   ret i32 %z
 }
-; CHECK: p0 = any8(p0)
+; CHECK: = any8({{.*}})
 
 declare i32 @llvm.hexagon.C2.all8(i32)
 define i32 @C2_all8(i32 %a) {
@@ -30,7 +33,7 @@ define i32 @C2_all8(i32 %a) {
   ret i32 %z
 }
 
-; CHECK: p0 = all8(p0)
+; CHECK: = all8({{.*}})
 
 ; Logical operations on predicates
 declare i32 @llvm.hexagon.C2.and(i32, i32)
@@ -38,95 +41,95 @@ define i32 @C2_and(i32 %a, i32 %b) {
   %z = call i32@llvm.hexagon.C2.and(i32 %a, i32 %b)
   ret i32 %z
 }
-; CHECK: p0 = and(p0, p1)
+; CHECK: = and({{.*}}, {{.*}})
 
 declare i32 @llvm.hexagon.C4.and.and(i32, i32, i32)
 define i32 @C4_and_and(i32 %a, i32 %b, i32 %c) {
   %z = call i32@llvm.hexagon.C4.and.and(i32 %a, i32 %b, i32 %c)
   ret i32 %z
 }
-; CHECK: p0 = and(p0, and(p1, p2))
+; CHECK: = and({{.*}}, and({{.*}}, {{.*}}))
 
 declare i32 @llvm.hexagon.C2.or(i32, i32)
 define i32 @C2_or(i32 %a, i32 %b) {
   %z = call i32@llvm.hexagon.C2.or(i32 %a, i32 %b)
   ret i32 %z
 }
-; CHECK: p0 = or(p0, p1)
+; CHECK: = or({{.*}}, {{.*}})
 
 declare i32 @llvm.hexagon.C4.and.or(i32, i32, i32)
 define i32 @C4_and_or(i32 %a, i32 %b, i32 %c) {
   %z = call i32@llvm.hexagon.C4.and.or(i32 %a, i32 %b, i32 %c)
   ret i32 %z
 }
-; CHECK: p0 = and(p0, or(p1, p2))
+; CHECK: = and({{.*}}, or({{.*}}, {{.*}}))
 
 declare i32 @llvm.hexagon.C2.xor(i32, i32)
 define i32 @C2_xor(i32 %a, i32 %b) {
   %z = call i32@llvm.hexagon.C2.xor(i32 %a, i32 %b)
   ret i32 %z
 }
-; CHECK: p0 = xor(p0, p1)
+; CHECK: = xor({{.*}}, {{.*}})
 
 declare i32 @llvm.hexagon.C4.or.and(i32, i32, i32)
 define i32 @C4_or_and(i32 %a, i32 %b, i32 %c) {
   %z = call i32@llvm.hexagon.C4.or.and(i32 %a, i32 %b, i32 %c)
   ret i32 %z
 }
-; CHECK: p0 = or(p0, and(p1, p2))
+; CHECK: = or({{.*}}, and({{.*}}, {{.*}}))
 
 declare i32 @llvm.hexagon.C2.andn(i32, i32)
 define i32 @C2_andn(i32 %a, i32 %b) {
   %z = call i32@llvm.hexagon.C2.andn(i32 %a, i32 %b)
   ret i32 %z
 }
-; CHECK: p0 = and(p0, !p1)
+; CHECK: = and({{.*}}, !{{.*}})
 
 declare i32 @llvm.hexagon.C4.or.or(i32, i32, i32)
 define i32 @C4_or_or(i32 %a, i32 %b, i32 %c) {
   %z = call i32@llvm.hexagon.C4.or.or(i32 %a, i32 %b, i32 %c)
   ret i32 %z
 }
-; CHECK: p0 = or(p0, or(p1, p2))
+; CHECK: = or({{.*}}, or({{.*}}, {{.*}}))
 
 declare i32 @llvm.hexagon.C4.and.andn(i32, i32, i32)
 define i32 @C4_and_andn(i32 %a, i32 %b, i32 %c) {
   %z = call i32@llvm.hexagon.C4.and.andn(i32 %a, i32 %b, i32 %c)
   ret i32 %z
 }
-; CHECK: p0 = and(p0, and(p1, !p2))
+; CHECK: = and({{.*}}, and({{.*}}, !{{.*}}))
 
 declare i32 @llvm.hexagon.C4.and.orn(i32, i32, i32)
 define i32 @C4_and_orn(i32 %a, i32 %b, i32 %c) {
   %z = call i32@llvm.hexagon.C4.and.orn(i32 %a, i32 %b, i32 %c)
   ret i32 %z
 }
-; CHECK: p0 = and(p0, or(p1, !p2))
+; CHECK: = and({{.*}}, or({{.*}}, !{{.*}}))
 
 declare i32 @llvm.hexagon.C2.not(i32)
 define i32 @C2_not(i32 %a) {
   %z = call i32@llvm.hexagon.C2.not(i32 %a)
   ret i32 %z
 }
-; CHECK: p0 = not(p0)
+; CHECK: = not({{.*}})
 
 declare i32 @llvm.hexagon.C4.or.andn(i32, i32, i32)
 define i32 @C4_or_andn(i32 %a, i32 %b, i32 %c) {
   %z = call i32@llvm.hexagon.C4.or.andn(i32 %a, i32 %b, i32 %c)
   ret i32 %z
 }
-; CHECK: p0 = or(p0, and(p1, !p2))
+; CHECK: = or({{.*}}, and({{.*}}, !{{.*}}))
 
 declare i32 @llvm.hexagon.C2.orn(i32, i32)
 define i32 @C2_orn(i32 %a, i32 %b) {
   %z = call i32@llvm.hexagon.C2.orn(i32 %a, i32 %b)
   ret i32 %z
 }
-; CHECK: p0 = or(p0, !p1)
+; CHECK: = or({{.*}}, !{{.*}})
 
 declare i32 @llvm.hexagon.C4.or.orn(i32, i32, i32)
 define i32 @C4_or_orn(i32 %a, i32 %b, i32 %c) {
   %z = call i32@llvm.hexagon.C4.or.orn(i32 %a, i32 %b, i32 %c)
   ret i32 %z
 }
-; CHECK: p0 = or(p0, or(p1, !p2))
+; CHECK: = or({{.*}}, or({{.*}}, !{{.*}}))
diff --git a/test/CodeGen/Hexagon/intrinsics/xtype_alu.ll b/test/CodeGen/Hexagon/intrinsics/xtype_alu.ll
index 4a11112d73a..c5c23c22bde 100644
--- a/test/CodeGen/Hexagon/intrinsics/xtype_alu.ll
+++ b/test/CodeGen/Hexagon/intrinsics/xtype_alu.ll
@@ -1,13 +1,17 @@
 ; RUN: llc -march=hexagon -mcpu=hexagonv5 -O0 < %s | FileCheck %s
+; RUN: llc -march=hexagon -mcpu=hexagonv5 -O0 < %s | \
+; RUN: FileCheck -check-prefix=CHECK-CALL %s
 ; Hexagon Programmer's Reference Manual 11.10.1 XTYPE/ALU
 
+; CHECK-CALL-NOT: call
+
 ; Absolute value doubleword
 declare i64 @llvm.hexagon.A2.absp(i64)
 define i64 @A2_absp(i64 %a) {
   %z = call i64 @llvm.hexagon.A2.absp(i64 %a)
   ret i64 %z
 }
-; CHECK: r1:0 = abs(r1:0)
+; CHECK: = abs({{.*}})
 
 ; Absolute value word
 declare i32 @llvm.hexagon.A2.abs(i32)
@@ -15,14 +19,14 @@ define i32 @A2_abs(i32 %a) {
   %z = call i32 @llvm.hexagon.A2.abs(i32 %a)
   ret i32 %z
 }
-; CHECK: r0 = abs(r0)
+; CHECK: = abs({{.*}})
 
 declare i32 @llvm.hexagon.A2.abssat(i32)
 define i32 @A2_abssat(i32 %a) {
   %z = call i32 @llvm.hexagon.A2.abssat(i32 %a)
   ret i32 %z
 }
-; CHECK: r0 = abs(r0):sat
+; CHECK: = abs({{.*}}):sat
 
 ; Add and accumulate
 declare i32 @llvm.hexagon.S4.addaddi(i32, i32, i32)
@@ -30,42 +34,42 @@ define i32 @S4_addaddi(i32 %a, i32 %b) {
   %z = call i32 @llvm.hexagon.S4.addaddi(i32 %a, i32 %b, i32 0)
   ret i32 %z
 }
-; CHECK: r0 = add(r0, add(r1, #0))
+; CHECK: = add({{.*}}, add({{.*}}, #0))
 
 declare i32 @llvm.hexagon.S4.subaddi(i32, i32, i32)
 define i32 @S4_subaddi(i32 %a, i32 %b) {
   %z = call i32 @llvm.hexagon.S4.subaddi(i32 %a, i32 0, i32 %b)
   ret i32 %z
 }
-; CHECK: r0 = add(r0, sub(#0, r1))
+; CHECK: = add({{.*}}, sub(#0, {{.*}}))
 
 declare i32 @llvm.hexagon.M2.accii(i32, i32, i32)
 define i32 @M2_accii(i32 %a, i32 %b) {
   %z = call i32 @llvm.hexagon.M2.accii(i32 %a, i32 %b, i32 0)
   ret i32 %z
 }
-; CHECK: r0 += add(r1, #0)
+; CHECK: += add({{.*}}, #0)
 
 declare i32 @llvm.hexagon.M2.naccii(i32, i32, i32)
 define i32 @M2_naccii(i32 %a, i32 %b) {
   %z = call i32 @llvm.hexagon.M2.naccii(i32 %a, i32 %b, i32 0)
   ret i32 %z
 }
-; CHECK: r0 -= add(r1, #0)
+; CHECK: -= add({{.*}}, #0)
 
 declare i32 @llvm.hexagon.M2.acci(i32, i32, i32)
 define i32 @M2_acci(i32 %a, i32 %b, i32 %c) {
   %z = call i32 @llvm.hexagon.M2.acci(i32 %a, i32 %b, i32 %c)
   ret i32 %z
 }
-; CHECK: r0 += add(r1, r2)
+; CHECK: += add({{.*}}, {{.*}})
 
 declare i32 @llvm.hexagon.M2.nacci(i32, i32, i32)
 define i32 @M2_nacci(i32 %a, i32 %b, i32 %c) {
   %z = call i32 @llvm.hexagon.M2.nacci(i32 %a, i32 %b, i32 %c)
   ret i32 %z
 }
-; CHECK: r0 -= add(r1, r2)
+; CHECK: -= add({{.*}}, {{.*}})
 
 ; Add doublewords
 declare i64 @llvm.hexagon.A2.addp(i64, i64)
@@ -73,14 +77,14 @@ define i64 @A2_addp(i64 %a, i64 %b) {
   %z = call i64 @llvm.hexagon.A2.addp(i64 %a, i64 %b)
   ret i64 %z
 }
-; CHECK: r1:0 = add(r1:0, r3:2)
+; CHECK: = add({{.*}}, {{.*}})
 
 declare i64 @llvm.hexagon.A2.addpsat(i64, i64)
 define i64 @A2_addpsat(i64 %a, i64 %b) {
   %z = call i64 @llvm.hexagon.A2.addpsat(i64 %a, i64 %b)
   ret i64 %z
 }
-; CHECK: r1:0 = add(r1:0, r3:2):sat
+; CHECK: = add({{.*}}, {{.*}}):sat
 
 ; Add halfword
 declare i32 @llvm.hexagon.A2.addh.l16.ll(i32, i32)
@@ -88,84 +92,84 @@ define i32 @A2_addh_l16_ll(i32 %a, i32 %b) {
   %z = call i32 @llvm.hexagon.A2.addh.l16.ll(i32 %a, i32 %b)
   ret i32 %z
 }
-; CHECK: r0 = add(r0.l, r1.l)
+; CHECK: = add({{.*}}.l, {{.*}}.l)
 
 declare i32 @llvm.hexagon.A2.addh.l16.hl(i32, i32)
 define i32 @A2_addh_l16_hl(i32 %a, i32 %b) {
   %z = call i32 @llvm.hexagon.A2.addh.l16.hl(i32 %a, i32 %b)
   ret i32 %z
 }
-; CHECK: r0 = add(r0.l, r1.h)
+; CHECK: = add({{.*}}.l, {{.*}}.h)
 
 declare i32 @llvm.hexagon.A2.addh.l16.sat.ll(i32, i32)
 define i32 @A2_addh_l16_sat.ll(i32 %a, i32 %b) {
   %z = call i32 @llvm.hexagon.A2.addh.l16.sat.ll(i32 %a, i32 %b)
   ret i32 %z
 }
-; CHECK: r0 = add(r0.l, r1.l):sat
+; CHECK: = add({{.*}}.l, {{.*}}.l):sat
 
 declare i32 @llvm.hexagon.A2.addh.l16.sat.hl(i32, i32)
 define i32 @A2_addh_l16_sat.hl(i32 %a, i32 %b) {
   %z = call i32 @llvm.hexagon.A2.addh.l16.sat.hl(i32 %a, i32 %b)
   ret i32 %z
 }
-; CHECK: r0 = add(r0.l, r1.h):sat
+; CHECK: = add({{.*}}.l, {{.*}}.h):sat
 
 declare i32 @llvm.hexagon.A2.addh.h16.ll(i32, i32)
 define i32 @A2_addh_h16_ll(i32 %a, i32 %b) {
   %z = call i32 @llvm.hexagon.A2.addh.h16.ll(i32 %a, i32 %b)
   ret i32 %z
 }
-; CHECK: r0 = add(r0.l, r1.l):<<16
+; CHECK: = add({{.*}}.l, {{.*}}.l):<<16
 
 declare i32 @llvm.hexagon.A2.addh.h16.lh(i32, i32)
 define i32 @A2_addh_h16_lh(i32 %a, i32 %b) {
   %z = call i32 @llvm.hexagon.A2.addh.h16.lh(i32 %a, i32 %b)
   ret i32 %z
 }
-; CHECK: r0 = add(r0.l, r1.h):<<16
+; CHECK: = add({{.*}}.l, {{.*}}.h):<<16
 
 declare i32 @llvm.hexagon.A2.addh.h16.hl(i32, i32)
 define i32 @A2_addh_h16_hl(i32 %a, i32 %b) {
   %z = call i32 @llvm.hexagon.A2.addh.h16.hl(i32 %a, i32 %b)
   ret i32 %z
 }
-; CHECK: r0 = add(r0.h, r1.l):<<16
+; CHECK: = add({{.*}}.h, {{.*}}.l):<<16
 
 declare i32 @llvm.hexagon.A2.addh.h16.hh(i32, i32)
 define i32 @A2_addh_h16_hh(i32 %a, i32 %b) {
   %z = call i32 @llvm.hexagon.A2.addh.h16.hh(i32 %a, i32 %b)
   ret i32 %z
 }
-; CHECK: r0 = add(r0.h, r1.h):<<16
+; CHECK: = add({{.*}}.h, {{.*}}.h):<<16
 
 declare i32 @llvm.hexagon.A2.addh.h16.sat.ll(i32, i32)
 define i32 @A2_addh_h16_sat_ll(i32 %a, i32 %b) {
   %z = call i32 @llvm.hexagon.A2.addh.h16.sat.ll(i32 %a, i32 %b)
   ret i32 %z
 }
-; CHECK: r0 = add(r0.l, r1.l):sat:<<16
+; CHECK: = add({{.*}}.l, {{.*}}.l):sat:<<16
 
 declare i32 @llvm.hexagon.A2.addh.h16.sat.lh(i32, i32)
 define i32 @A2_addh_h16_sat_lh(i32 %a, i32 %b) {
   %z = call i32 @llvm.hexagon.A2.addh.h16.sat.lh(i32 %a, i32 %b)
   ret i32 %z
 }
-; CHECK: r0 = add(r0.l, r1.h):sat:<<16
+; CHECK: = add({{.*}}.l, {{.*}}.h):sat:<<16
 
 declare i32 @llvm.hexagon.A2.addh.h16.sat.hl(i32, i32)
 define i32 @A2_addh_h16_sat_hl(i32 %a, i32 %b) {
   %z = call i32 @llvm.hexagon.A2.addh.h16.sat.hl(i32 %a, i32 %b)
   ret i32 %z
 }
-; CHECK: r0 = add(r0.h, r1.l):sat:<<16
+; CHECK: = add({{.*}}.h, {{.*}}.l):sat:<<16
 
 declare i32 @llvm.hexagon.A2.addh.h16.sat.hh(i32, i32)
 define i32 @A2_addh_h16_sat_hh(i32 %a, i32 %b) {
   %z = call i32 @llvm.hexagon.A2.addh.h16.sat.hh(i32 %a, i32 %b)
   ret i32 %z
 }
-; CHECK: r0 = add(r0.h, r1.h):sat:<<16
+; CHECK: = add({{.*}}.h, {{.*}}.h):sat:<<16
 
 ; Logical doublewords
 declare i64 @llvm.hexagon.A2.notp(i64)
@@ -173,42 +177,42 @@ define i64 @A2_notp(i64 %a) {
   %z = call i64 @llvm.hexagon.A2.notp(i64 %a)
   ret i64 %z
 }
-; CHECK: r1:0 = not(r1:0)
+; CHECK: = not({{.*}})
 
 declare i64 @llvm.hexagon.A2.andp(i64, i64)
 define i64 @A2_andp(i64 %a, i64 %b) {
   %z = call i64 @llvm.hexagon.A2.andp(i64 %a, i64 %b)
   ret i64 %z
 }
-; CHECK: r1:0 = and(r1:0, r3:2)
+; CHECK: = and({{.*}}, {{.*}})
 
 declare i64 @llvm.hexagon.A4.andnp(i64, i64)
 define i64 @A2_andnp(i64 %a, i64 %b) {
   %z = call i64 @llvm.hexagon.A4.andnp(i64 %a, i64 %b)
   ret i64 %z
 }
-; CHECK: r1:0 = and(r1:0, ~r3:2)
+; CHECK: = and({{.*}}, ~{{.*}})
 
 declare i64 @llvm.hexagon.A2.orp(i64, i64)
 define i64 @A2_orp(i64 %a, i64 %b) {
   %z = call i64 @llvm.hexagon.A2.orp(i64 %a, i64 %b)
   ret i64 %z
 }
-; CHECK: r1:0 = or(r1:0, r3:2)
+; CHECK: = or({{.*}}, {{.*}})
 
 declare i64 @llvm.hexagon.A4.ornp(i64, i64)
 define i64 @A2_ornp(i64 %a, i64 %b) {
   %z = call i64 @llvm.hexagon.A4.ornp(i64 %a, i64 %b)
   ret i64 %z
 }
-; CHECK: r1:0 = or(r1:0, ~r3:2)
+; CHECK: = or({{.*}}, ~{{.*}})
 
 declare i64 @llvm.hexagon.A2.xorp(i64, i64)
 define i64 @A2_xorp(i64 %a, i64 %b) {
   %z = call i64 @llvm.hexagon.A2.xorp(i64 %a, i64 %b)
   ret i64 %z
 }
-; CHECK: r1:0 = xor(r1:0, r3:2)
+; CHECK: = xor({{.*}}, {{.*}})
 
 ; Logical-logical doublewords
 declare i64 @llvm.hexagon.M4.xor.xacc(i64, i64, i64)
@@ -216,7 +220,7 @@ define i64 @M4_xor_xacc(i64 %a, i64 %b, i64 %c) {
   %z = call i64 @llvm.hexagon.M4.xor.xacc(i64 %a, i64 %b, i64 %c)
   ret i64 %z
 }
-; CHECK: r1:0 ^= xor(r3:2, r5:4)
+; CHECK: ^= xor({{.*}}, {{.*}})
 
 ; Logical-logical words
 declare i32 @llvm.hexagon.S4.or.andi(i32, i32, i32)
@@ -224,91 +228,91 @@ define i32 @S4_or_andi(i32 %a, i32 %b) {
   %z = call i32 @llvm.hexagon.S4.or.andi(i32 %a, i32 %b, i32 0)
   ret i32 %z
 }
-; CHECK: r0 |= and(r1, #0)
+; CHECK: |= and({{.*}}, #0)
 
 declare i32 @llvm.hexagon.S4.or.andix(i32, i32, i32)
 define i32 @S4_or_andix(i32 %a, i32 %b) {
   %z = call i32 @llvm.hexagon.S4.or.andix(i32 %a, i32 %b, i32 0)
   ret i32 %z
 }
-; CHECK: r1 = or(r0, and(r1, #0))
+; CHECK: = or({{.*}}, and({{.*}}, #0))
 
 declare i32 @llvm.hexagon.M4.or.andn(i32, i32, i32)
 define i32 @M4_or_andn(i32 %a, i32 %b, i32 %c) {
   %z = call i32 @llvm.hexagon.M4.or.andn(i32 %a, i32 %b, i32 %c)
   ret i32 %z
 }
-; CHECK: r0 |= and(r1, ~r2)
+; CHECK: |= and({{.*}}, ~{{.*}})
 
 declare i32 @llvm.hexagon.M4.and.andn(i32, i32, i32)
 define i32 @M4_and_andn(i32 %a, i32 %b, i32 %c) {
   %z = call i32 @llvm.hexagon.M4.and.andn(i32 %a, i32 %b, i32 %c)
   ret i32 %z
 }
-; CHECK: r0 &= and(r1, ~r2)
+; CHECK: &= and({{.*}}, ~{{.*}})
 
 declare i32 @llvm.hexagon.M4.xor.andn(i32, i32, i32)
 define i32 @M4_xor_andn(i32 %a, i32 %b, i32 %c) {
   %z = call i32 @llvm.hexagon.M4.xor.andn(i32 %a, i32 %b, i32 %c)
   ret i32 %z
 }
-; CHECK: r0 ^= and(r1, ~r2)
+; CHECK: ^= and({{.*}}, ~{{.*}})
 
 declare i32 @llvm.hexagon.M4.and.and(i32, i32, i32)
 define i32 @M4_and_and(i32 %a, i32 %b, i32 %c) {
   %z = call i32 @llvm.hexagon.M4.and.and(i32 %a, i32 %b, i32 %c)
   ret i32 %z
 }
-; CHECK: r0 &= and(r1, r2)
+; CHECK: &= and({{.*}}, {{.*}})
 
 declare i32 @llvm.hexagon.M4.and.or(i32, i32, i32)
 define i32 @M4_and_or(i32 %a, i32 %b, i32 %c) {
   %z = call i32 @llvm.hexagon.M4.and.or(i32 %a, i32 %b, i32 %c)
   ret i32 %z
 }
-; CHECK: r0 &= or(r1, r2)
+; CHECK: &= or({{.*}}, {{.*}})
 
 declare i32 @llvm.hexagon.M4.and.xor(i32, i32, i32)
 define i32 @M4_and_xor(i32 %a, i32 %b, i32 %c) {
   %z = call i32 @llvm.hexagon.M4.and.xor(i32 %a, i32 %b, i32 %c)
   ret i32 %z
 }
-; CHECK: r0 &= xor(r1, r2)
+; CHECK: &= xor({{.*}}, {{.*}})
 
 declare i32 @llvm.hexagon.M4.or.and(i32, i32, i32)
 define i32 @M4_or_and(i32 %a, i32 %b, i32 %c) {
   %z = call i32 @llvm.hexagon.M4.or.and(i32 %a, i32 %b, i32 %c)
   ret i32 %z
 }
-; CHECK: r0 |= and(r1, r2)
+; CHECK: |= and({{.*}}, {{.*}})
 
 declare i32 @llvm.hexagon.M4.or.or(i32, i32, i32)
 define i32 @M4_or_or(i32 %a, i32 %b, i32 %c) {
   %z = call i32 @llvm.hexagon.M4.or.or(i32 %a, i32 %b, i32 %c)
   ret i32 %z
 }
-; CHECK: r0 |= or(r1, r2)
+; CHECK: |= or({{.*}}, {{.*}})
 
 declare i32 @llvm.hexagon.M4.or.xor(i32, i32, i32)
 define i32 @M4_or_xor(i32 %a, i32 %b, i32 %c) {
   %z = call i32 @llvm.hexagon.M4.or.xor(i32 %a, i32 %b, i32 %c)
   ret i32 %z
 }
-; CHECK: r0 |= xor(r1, r2)
+; CHECK: |= xor({{.*}}, {{.*}})
 
 declare i32 @llvm.hexagon.M4.xor.and(i32, i32, i32)
 define i32 @M4_xor_and(i32 %a, i32 %b, i32 %c) {
   %z = call i32 @llvm.hexagon.M4.xor.and(i32 %a, i32 %b, i32 %c)
   ret i32 %z
 }
-; CHECK: r0 ^= and(r1, r2)
+; CHECK: ^= and({{.*}}, {{.*}})
 
 declare i32 @llvm.hexagon.M4.xor.or(i32, i32, i32)
 define i32 @M4_xor_or(i32 %a, i32 %b, i32 %c) {
   %z = call i32 @llvm.hexagon.M4.xor.or(i32 %a, i32 %b, i32 %c)
   ret i32 %z
 }
-; CHECK: r0 ^= or(r1, r2)
+; CHECK: ^= or({{.*}}, {{.*}})
 
 ; Maximum words
 declare i32 @llvm.hexagon.A2.max(i32, i32)
@@ -316,14 +320,14 @@ define i32 @A2_max(i32 %a, i32 %b) {
   %z = call i32 @llvm.hexagon.A2.max(i32 %a, i32 %b)
   ret i32 %z
 }
-; CHECK: r0 = max(r0, r1)
+; CHECK: = max({{.*}}, {{.*}})
 
 declare i32 @llvm.hexagon.A2.maxu(i32, i32)
 define i32 @A2_maxu(i32 %a, i32 %b) {
   %z = call i32 @llvm.hexagon.A2.maxu(i32 %a, i32 %b)
   ret i32 %z
 }
-; CHECK: r0 = maxu(r0, r1)
+; CHECK: = maxu({{.*}}, {{.*}})
 
 ; Maximum doublewords
 declare i64 @llvm.hexagon.A2.maxp(i64, i64)
@@ -331,14 +335,14 @@ define i64 @A2_maxp(i64 %a, i64 %b) {
   %z = call i64 @llvm.hexagon.A2.maxp(i64 %a, i64 %b)
   ret i64 %z
 }
-; CHECK: r1:0 = max(r1:0, r3:2)
+; CHECK: = max({{.*}}, {{.*}})
 
 declare i64 @llvm.hexagon.A2.maxup(i64, i64)
 define i64 @A2_maxup(i64 %a, i64 %b) {
   %z = call i64 @llvm.hexagon.A2.maxup(i64 %a, i64 %b)
   ret i64 %z
 }
-; CHECK: r1:0 = maxu(r1:0, r3:2)
+; CHECK: = maxu({{.*}}, {{.*}})
 
 ; Minimum words
 declare i32 @llvm.hexagon.A2.min(i32, i32)
@@ -346,14 +350,14 @@ define i32 @A2_min(i32 %a, i32 %b) {
   %z = call i32 @llvm.hexagon.A2.min(i32 %a, i32 %b)
   ret i32 %z
 }
-; CHECK: r0 = min(r0, r1)
+; CHECK: = min({{.*}}, {{.*}})
 
 declare i32 @llvm.hexagon.A2.minu(i32, i32)
 define i32 @A2_minu(i32 %a, i32 %b) {
   %z = call i32 @llvm.hexagon.A2.minu(i32 %a, i32 %b)
   ret i32 %z
 }
-; CHECK: r0 = minu(r0, r1)
+; CHECK: = minu({{.*}}, {{.*}})
 
 ; Minimum doublewords
 declare i64 @llvm.hexagon.A2.minp(i64, i64)
@@ -361,14 +365,14 @@ define i64 @A2_minp(i64 %a, i64 %b) {
   %z = call i64 @llvm.hexagon.A2.minp(i64 %a, i64 %b)
   ret i64 %z
 }
-; CHECK: r1:0 = min(r1:0, r3:2)
+; CHECK: = min({{.*}}, {{.*}})
 
 declare i64 @llvm.hexagon.A2.minup(i64, i64)
 define i64 @A2_minup(i64 %a, i64 %b) {
   %z = call i64 @llvm.hexagon.A2.minup(i64 %a, i64 %b)
   ret i64 %z
 }
-; CHECK: r1:0 = minu(r1:0, r3:2)
+; CHECK: = minu({{.*}}, {{.*}})
 
 ; Module wrap
 declare i32 @llvm.hexagon.A4.modwrapu(i32, i32)
@@ -376,7 +380,7 @@ define i32 @A4_modwrapu(i32 %a, i32 %b) {
   %z = call i32 @llvm.hexagon.A4.modwrapu(i32 %a, i32 %b)
   ret i32 %z
 }
-; CHECK: r0 = modwrap(r0, r1)
+; CHECK: = modwrap({{.*}}, {{.*}})
 
 ; Negate
 declare i64 @llvm.hexagon.A2.negp(i64)
@@ -384,14 +388,14 @@ define i64 @A2_negp(i64 %a) {
   %z = call i64 @llvm.hexagon.A2.negp(i64 %a)
   ret i64 %z
 }
-; CHECK: r1:0 = neg(r1:0)
+; CHECK: = neg({{.*}})
 
 declare i32 @llvm.hexagon.A2.negsat(i32)
 define i32 @A2_negsat(i32 %a) {
   %z = call i32 @llvm.hexagon.A2.negsat(i32 %a)
   ret i32 %z
 }
-; CHECK: r0 = neg(r0):sat
+; CHECK: = neg({{.*}}):sat
 
 ; Round
 declare i32 @llvm.hexagon.A2.roundsat(i64)
@@ -399,49 +403,49 @@ define i32 @A2_roundsat(i64 %a) {
   %z = call i32 @llvm.hexagon.A2.roundsat(i64 %a)
   ret i32 %z
 }
-; CHECK: r0 = round(r1:0):sat
+; CHECK: = round({{.*}}):sat
 
 declare i32 @llvm.hexagon.A4.cround.ri(i32, i32)
 define i32 @A4_cround_ri(i32 %a) {
   %z = call i32 @llvm.hexagon.A4.cround.ri(i32 %a, i32 0)
   ret i32 %z
 }
-; CHECK: r0 = cround(r0, #0)
+; CHECK: = cround({{.*}}, #0)
 
 declare i32 @llvm.hexagon.A4.round.ri(i32, i32)
 define i32 @A4_round_ri(i32 %a) {
   %z = call i32 @llvm.hexagon.A4.round.ri(i32 %a, i32 0)
   ret i32 %z
 }
-; CHECK: r0 = round(r0, #0)
+; CHECK: = round({{.*}}, #0)
 
 declare i32 @llvm.hexagon.A4.round.ri.sat(i32, i32)
 define i32 @A4_round_ri_sat(i32 %a) {
   %z = call i32 @llvm.hexagon.A4.round.ri.sat(i32 %a, i32 0)
   ret i32 %z
 }
-; CHECK: r0 = round(r0, #0):sat
+; CHECK: = round({{.*}}, #0):sat
 
 declare i32 @llvm.hexagon.A4.cround.rr(i32, i32)
 define i32 @A4_cround_rr(i32 %a, i32 %b) {
   %z = call i32 @llvm.hexagon.A4.cround.rr(i32 %a, i32 %b)
   ret i32 %z
 }
-; CHECK: r0 = cround(r0, r1)
+; CHECK: = cround({{.*}}, {{.*}})
 
 declare i32 @llvm.hexagon.A4.round.rr(i32, i32)
 define i32 @A4_round_rr(i32 %a, i32 %b) {
   %z = call i32 @llvm.hexagon.A4.round.rr(i32 %a, i32 %b)
   ret i32 %z
 }
-; CHECK: r0 = round(r0, r1)
+; CHECK: = round({{.*}}, {{.*}})
 
 declare i32 @llvm.hexagon.A4.round.rr.sat(i32, i32)
 define i32 @A4_round_rr_sat(i32 %a, i32 %b) {
   %z = call i32 @llvm.hexagon.A4.round.rr.sat(i32 %a, i32 %b)
   ret i32 %z
 }
-; CHECK: r0 = round(r0, r1):sat
+; CHECK: = round({{.*}}, {{.*}}):sat
 
 ; Subtract doublewords
 declare i64 @llvm.hexagon.A2.subp(i64, i64)
@@ -449,7 +453,7 @@ define i64 @A2_subp(i64 %a, i64 %b) {
   %z = call i64 @llvm.hexagon.A2.subp(i64 %a, i64 %b)
   ret i64 %z
 }
-; CHECK: r1:0 = sub(r1:0, r3:2)
+; CHECK: = sub({{.*}}, {{.*}})
 
 ; Subtract and accumulate
 declare i32 @llvm.hexagon.M2.subacc(i32, i32, i32)
@@ -457,7 +461,7 @@ define i32 @M2_subacc(i32 %a, i32 %b, i32 %c) {
   %z = call i32 @llvm.hexagon.M2.subacc(i32 %a, i32 %b, i32 %c)
   ret i32 %z
 }
-; CHECK: r0 += sub(r1, r2)
+; CHECK: += sub({{.*}}, {{.*}})
 
 ; Subtract halfwords
 declare i32 @llvm.hexagon.A2.subh.l16.ll(i32, i32)
@@ -465,84 +469,84 @@ define i32 @A2_subh_l16_ll(i32 %a, i32 %b) {
   %z = call i32 @llvm.hexagon.A2.subh.l16.ll(i32 %a, i32 %b)
   ret i32 %z
 }
-; CHECK: r0 = sub(r0.l, r1.l)
+; CHECK: = sub({{.*}}.l, {{.*}}.l)
 
 declare i32 @llvm.hexagon.A2.subh.l16.hl(i32, i32)
 define i32 @A2_subh_l16_hl(i32 %a, i32 %b) {
   %z = call i32 @llvm.hexagon.A2.subh.l16.hl(i32 %a, i32 %b)
   ret i32 %z
 }
-; CHECK: r0 = sub(r0.l, r1.h)
+; CHECK: = sub({{.*}}.l, {{.*}}.h)
 
 declare i32 @llvm.hexagon.A2.subh.l16.sat.ll(i32, i32)
 define i32 @A2_subh_l16_sat.ll(i32 %a, i32 %b) {
   %z = call i32 @llvm.hexagon.A2.subh.l16.sat.ll(i32 %a, i32 %b)
   ret i32 %z
 }
-; CHECK: r0 = sub(r0.l, r1.l):sat
+; CHECK: = sub({{.*}}.l, {{.*}}.l):sat
 
 declare i32 @llvm.hexagon.A2.subh.l16.sat.hl(i32, i32)
 define i32 @A2_subh_l16_sat.hl(i32 %a, i32 %b) {
   %z = call i32 @llvm.hexagon.A2.subh.l16.sat.hl(i32 %a, i32 %b)
   ret i32 %z
 }
-; CHECK: r0 = sub(r0.l, r1.h):sat
+; CHECK: = sub({{.*}}.l, {{.*}}.h):sat
 
 declare i32 @llvm.hexagon.A2.subh.h16.ll(i32, i32)
 define i32 @A2_subh_h16_ll(i32 %a, i32 %b) {
   %z = call i32 @llvm.hexagon.A2.subh.h16.ll(i32 %a, i32 %b)
   ret i32 %z
 }
-; CHECK: r0 = sub(r0.l, r1.l):<<16
+; CHECK: = sub({{.*}}.l, {{.*}}.l):<<16
 
 declare i32 @llvm.hexagon.A2.subh.h16.lh(i32, i32)
 define i32 @A2_subh_h16_lh(i32 %a, i32 %b) {
   %z = call i32 @llvm.hexagon.A2.subh.h16.lh(i32 %a, i32 %b)
   ret i32 %z
 }
-; CHECK: r0 = sub(r0.l, r1.h):<<16
+; CHECK: = sub({{.*}}.l, {{.*}}.h):<<16
 
 declare i32 @llvm.hexagon.A2.subh.h16.hl(i32, i32)
 define i32 @A2_subh_h16_hl(i32 %a, i32 %b) {
   %z = call i32 @llvm.hexagon.A2.subh.h16.hl(i32 %a, i32 %b)
   ret i32 %z
 }
-; CHECK: r0 = sub(r0.h, r1.l):<<16
+; CHECK: = sub({{.*}}.h, {{.*}}.l):<<16
 
 declare i32 @llvm.hexagon.A2.subh.h16.hh(i32, i32)
 define i32 @A2_subh_h16_hh(i32 %a, i32 %b) {
   %z = call i32 @llvm.hexagon.A2.subh.h16.hh(i32 %a, i32 %b)
   ret i32 %z
 }
-; CHECK: r0 = sub(r0.h, r1.h):<<16
+; CHECK: = sub({{.*}}.h, {{.*}}.h):<<16
 
 declare i32 @llvm.hexagon.A2.subh.h16.sat.ll(i32, i32)
 define i32 @A2_subh_h16_sat_ll(i32 %a, i32 %b) {
   %z = call i32 @llvm.hexagon.A2.subh.h16.sat.ll(i32 %a, i32 %b)
   ret i32 %z
 }
-; CHECK: r0 = sub(r0.l, r1.l):sat:<<16
+; CHECK: = sub({{.*}}.l, {{.*}}.l):sat:<<16
 
 declare i32 @llvm.hexagon.A2.subh.h16.sat.lh(i32, i32)
 define i32 @A2_subh_h16_sat_lh(i32 %a, i32 %b) {
   %z = call i32 @llvm.hexagon.A2.subh.h16.sat.lh(i32 %a, i32 %b)
   ret i32 %z
 }
-; CHECK: r0 = sub(r0.l, r1.h):sat:<<16
+; CHECK: = sub({{.*}}.l, {{.*}}.h):sat:<<16
 
 declare i32 @llvm.hexagon.A2.subh.h16.sat.hl(i32, i32)
 define i32 @A2_subh_h16_sat_hl(i32 %a, i32 %b) {
   %z = call i32 @llvm.hexagon.A2.subh.h16.sat.hl(i32 %a, i32 %b)
   ret i32 %z
 }
-; CHECK: r0 = sub(r0.h, r1.l):sat:<<16
+; CHECK: = sub({{.*}}.h, {{.*}}.l):sat:<<16
 
 declare i32 @llvm.hexagon.A2.subh.h16.sat.hh(i32, i32)
 define i32 @A2_subh_h16_sat_hh(i32 %a, i32 %b) {
   %z = call i32 @llvm.hexagon.A2.subh.h16.sat.hh(i32 %a, i32 %b)
   ret i32 %z
 }
-; CHECK: r0 = sub(r0.h, r1.h):sat:<<16
+; CHECK: = sub({{.*}}.h, {{.*}}.h):sat:<<16
 
 ; Sign extend word to doubleword
 declare i64 @llvm.hexagon.A2.sxtw(i32)
@@ -550,7 +554,7 @@ define i64 @A2_sxtw(i32 %a) {
   %z = call i64 @llvm.hexagon.A2.sxtw(i32 %a)
   ret i64 %z
 }
-; CHECK:  = sxtw(r0)
+; CHECK: = sxtw({{.*}})
 
 ; Vector absolute value halfwords
 declare i64 @llvm.hexagon.A2.vabsh(i64)
@@ -558,14 +562,14 @@ define i64 @A2_vabsh(i64 %a) {
   %z = call i64 @llvm.hexagon.A2.vabsh(i64 %a)
   ret i64 %z
 }
-; CHECK: r1:0 = vabsh(r1:0)
+; CHECK: = vabsh({{.*}})
 
 declare i64 @llvm.hexagon.A2.vabshsat(i64)
 define i64 @A2_vabshsat(i64 %a) {
   %z = call i64 @llvm.hexagon.A2.vabshsat(i64 %a)
   ret i64 %z
 }
-; CHECK: r1:0 = vabsh(r1:0):sat
+; CHECK: = vabsh({{.*}}):sat
 
 ; Vector absolute value words
 declare i64 @llvm.hexagon.A2.vabsw(i64)
@@ -573,14 +577,14 @@ define i64 @A2_vabsw(i64 %a) {
   %z = call i64 @llvm.hexagon.A2.vabsw(i64 %a)
   ret i64 %z
 }
-; CHECK: r1:0 = vabsw(r1:0)
+; CHECK: = vabsw({{.*}})
 
 declare i64 @llvm.hexagon.A2.vabswsat(i64)
 define i64 @A2_vabswsat(i64 %a) {
   %z = call i64 @llvm.hexagon.A2.vabswsat(i64 %a)
   ret i64 %z
 }
-; CHECK: r1:0 = vabsw(r1:0):sat
+; CHECK: = vabsw({{.*}}):sat
 
 ; Vector absolute difference halfwords
 declare i64 @llvm.hexagon.M2.vabsdiffh(i64, i64)
@@ -588,7 +592,7 @@ define i64 @M2_vabsdiffh(i64 %a, i64 %b) {
   %z = call i64 @llvm.hexagon.M2.vabsdiffh(i64 %a, i64 %b)
   ret i64 %z
 }
-; CHECK: r1:0 = vabsdiffh(r1:0, r3:2)
+; CHECK: = vabsdiffh({{.*}}, {{.*}})
 
 ; Vector absolute difference words
 declare i64 @llvm.hexagon.M2.vabsdiffw(i64, i64)
@@ -596,7 +600,7 @@ define i64 @M2_vabsdiffw(i64 %a, i64 %b) {
   %z = call i64 @llvm.hexagon.M2.vabsdiffw(i64 %a, i64 %b)
   ret i64 %z
 }
-; CHECK: r1:0 = vabsdiffw(r1:0, r3:2)
+; CHECK: = vabsdiffw({{.*}}, {{.*}})
 
 ; Vector add halfwords
 declare i64 @llvm.hexagon.A2.vaddh(i64, i64)
@@ -604,21 +608,21 @@ define i64 @A2_vaddh(i64 %a, i64 %b) {
   %z = call i64 @llvm.hexagon.A2.vaddh(i64 %a, i64 %b)
   ret i64 %z
 }
-; CHECK: r1:0 = vaddh(r1:0, r3:2)
+; CHECK: = vaddh({{.*}}, {{.*}})
 
 declare i64 @llvm.hexagon.A2.vaddhs(i64, i64)
 define i64 @A2_vaddhs(i64 %a, i64 %b) {
   %z = call i64 @llvm.hexagon.A2.vaddhs(i64 %a, i64 %b)
   ret i64 %z
 }
-; CHECK: r1:0 = vaddh(r1:0, r3:2):sat
+; CHECK: = vaddh({{.*}}, {{.*}}):sat
 
 declare i64 @llvm.hexagon.A2.vadduhs(i64, i64)
 define i64 @A2_vadduhs(i64 %a, i64 %b) {
   %z = call i64 @llvm.hexagon.A2.vadduhs(i64 %a, i64 %b)
   ret i64 %z
 }
-; CHECK: r1:0 = vadduh(r1:0, r3:2):sat
+; CHECK: = vadduh({{.*}}, {{.*}}):sat
 
 ; Vector add halfwords with saturate and pack to unsigned bytes
 declare i32 @llvm.hexagon.A5.vaddhubs(i64, i64)
@@ -626,7 +630,7 @@ define i32 @A5_vaddhubs(i64 %a, i64 %b) {
   %z = call i32 @llvm.hexagon.A5.vaddhubs(i64 %a, i64 %b)
   ret i32 %z
 }
-; CHECK: r0 = vaddhub(r1:0, r3:2):sat
+; CHECK: = vaddhub({{.*}}, {{.*}}):sat
 
 ; Vector reduce add unsigned bytes
 declare i64 @llvm.hexagon.A2.vraddub(i64, i64)
@@ -634,14 +638,14 @@ define i64 @A2_vraddub(i64 %a, i64 %b) {
   %z = call i64 @llvm.hexagon.A2.vraddub(i64 %a, i64 %b)
   ret i64 %z
 }
-; CHECK: r1:0 = vraddub(r1:0, r3:2)
+; CHECK: = vraddub({{.*}}, {{.*}})
 
 declare i64 @llvm.hexagon.A2.vraddub.acc(i64, i64, i64)
 define i64 @A2_vraddub_acc(i64 %a, i64 %b, i64 %c) {
   %z = call i64 @llvm.hexagon.A2.vraddub.acc(i64 %a, i64 %b, i64 %c)
   ret i64 %z
 }
-; CHECK: r1:0 += vraddub(r3:2, r5:4)
+; CHECK: += vraddub({{.*}}, {{.*}})
 
 ; Vector reduce add halfwords
 declare i32 @llvm.hexagon.M2.vradduh(i64, i64)
@@ -649,14 +653,14 @@ define i32 @M2_vradduh(i64 %a, i64 %b) {
   %z = call i32 @llvm.hexagon.M2.vradduh(i64 %a, i64 %b)
   ret i32 %z
 }
-; CHECK: r0 = vradduh(r1:0, r3:2)
+; CHECK: = vradduh({{.*}}, {{.*}})
 
 declare i32 @llvm.hexagon.M2.vraddh(i64, i64)
 define i32 @M2_vraddh(i64 %a, i64 %b) {
   %z = call i32 @llvm.hexagon.M2.vraddh(i64 %a, i64 %b)
   ret i32 %z
 }
-; CHECK: r0 = vraddh(r1:0, r3:2)
+; CHECK: = vraddh({{.*}}, {{.*}})
 
 ; Vector add bytes
 declare i64 @llvm.hexagon.A2.vaddub(i64, i64)
@@ -664,14 +668,14 @@ define i64 @A2_vaddub(i64 %a, i64 %b) {
   %z = call i64 @llvm.hexagon.A2.vaddub(i64 %a, i64 %b)
   ret i64 %z
 }
-; CHECK: r1:0 = vaddub(r1:0, r3:2)
+; CHECK: = vaddub({{.*}}, {{.*}})
 
 declare i64 @llvm.hexagon.A2.vaddubs(i64, i64)
 define i64 @A2_vaddubs(i64 %a, i64 %b) {
   %z = call i64 @llvm.hexagon.A2.vaddubs(i64 %a, i64 %b)
   ret i64 %z
 }
-; CHECK: r1:0 = vaddub(r1:0, r3:2):sat
+; CHECK: = vaddub({{.*}}, {{.*}}):sat
 
 ; Vector add words
 declare i64 @llvm.hexagon.A2.vaddw(i64, i64)
@@ -679,14 +683,14 @@ define i64 @A2_vaddw(i64 %a, i64 %b) {
   %z = call i64 @llvm.hexagon.A2.vaddw(i64 %a, i64 %b)
   ret i64 %z
 }
-; CHECK: r1:0 = vaddw(r1:0, r3:2)
+; CHECK: = vaddw({{.*}}, {{.*}})
 
 declare i64 @llvm.hexagon.A2.vaddws(i64, i64)
 define i64 @A2_vaddws(i64 %a, i64 %b) {
   %z = call i64 @llvm.hexagon.A2.vaddws(i64 %a, i64 %b)
   ret i64 %z
 }
-; CHECK: r1:0 = vaddw(r1:0, r3:2):sat
+; CHECK: = vaddw({{.*}}, {{.*}}):sat
 
 ; Vector average halfwords
 declare i64 @llvm.hexagon.A2.vavgh(i64, i64)
@@ -694,56 +698,56 @@ define i64 @A2_vavgh(i64 %a, i64 %b) {
   %z = call i64 @llvm.hexagon.A2.vavgh(i64 %a, i64 %b)
   ret i64 %z
 }
-; CHECK: r1:0 = vavgh(r1:0, r3:2)
+; CHECK: = vavgh({{.*}}, {{.*}})
 
 declare i64 @llvm.hexagon.A2.vavghr(i64, i64)
 define i64 @A2_vavghr(i64 %a, i64 %b) {
   %z = call i64 @llvm.hexagon.A2.vavghr(i64 %a, i64 %b)
   ret i64 %z
 }
-; CHECK: r1:0 = vavgh(r1:0, r3:2):rnd
+; CHECK: = vavgh({{.*}}, {{.*}}):rnd
 
 declare i64 @llvm.hexagon.A2.vavghcr(i64, i64)
 define i64 @A2_vavghcr(i64 %a, i64 %b) {
   %z = call i64 @llvm.hexagon.A2.vavghcr(i64 %a, i64 %b)
   ret i64 %z
 }
-; CHECK: r1:0 = vavgh(r1:0, r3:2):crnd
+; CHECK: = vavgh({{.*}}, {{.*}}):crnd
 
 declare i64 @llvm.hexagon.A2.vavguh(i64, i64)
 define i64 @A2_vavguh(i64 %a, i64 %b) {
   %z = call i64 @llvm.hexagon.A2.vavguh(i64 %a, i64 %b)
   ret i64 %z
 }
-; CHECK: r1:0 = vavguh(r1:0, r3:2)
+; CHECK: = vavguh({{.*}}, {{.*}})
 
 declare i64 @llvm.hexagon.A2.vavguhr(i64, i64)
 define i64 @A2_vavguhr(i64 %a, i64 %b) {
   %z = call i64 @llvm.hexagon.A2.vavguhr(i64 %a, i64 %b)
   ret i64 %z
 }
-; CHECK: r1:0 = vavguh(r1:0, r3:2):rnd
+; CHECK: = vavguh({{.*}}, {{.*}}):rnd
 
 declare i64 @llvm.hexagon.A2.vnavgh(i64, i64)
 define i64 @A2_vnavgh(i64 %a, i64 %b) {
   %z = call i64 @llvm.hexagon.A2.vnavgh(i64 %a, i64 %b)
   ret i64 %z
 }
-; CHECK: r1:0 = vnavgh(r1:0, r3:2)
+; CHECK: = vnavgh({{.*}}, {{.*}})
 
 declare i64 @llvm.hexagon.A2.vnavghr(i64, i64)
 define i64 @A2_vnavghr(i64 %a, i64 %b) {
   %z = call i64 @llvm.hexagon.A2.vnavghr(i64 %a, i64 %b)
   ret i64 %z
 }
-; CHECK: r1:0 = vnavgh(r1:0, r3:2):rnd
+; CHECK: = vnavgh({{.*}}, {{.*}}):rnd
 
 declare i64 @llvm.hexagon.A2.vnavghcr(i64, i64)
 define i64 @A2_vnavghcr(i64 %a, i64 %b) {
   %z = call i64 @llvm.hexagon.A2.vnavghcr(i64 %a, i64 %b)
   ret i64 %z
 }
-; CHECK: r1:0 = vnavgh(r1:0, r3:2):crnd
+; CHECK: = vnavgh({{.*}}, {{.*}}):crnd
 
 ; Vector average unsigned bytes
 declare i64 @llvm.hexagon.A2.vavgub(i64, i64)
@@ -751,14 +755,14 @@ define i64 @A2_vavgub(i64 %a, i64 %b) {
   %z = call i64 @llvm.hexagon.A2.vavgub(i64 %a, i64 %b)
   ret i64 %z
 }
-; CHECK: r1:0 = vavgub(r1:0, r3:2)
+; CHECK:  vavgub({{.*}}, {{.*}})
 
 declare i64 @llvm.hexagon.A2.vavgubr(i64, i64)
 define i64 @A2_vavgubr(i64 %a, i64 %b) {
   %z = call i64 @llvm.hexagon.A2.vavgubr(i64 %a, i64 %b)
   ret i64 %z
 }
-; CHECK: r1:0 = vavgub(r1:0, r3:2):rnd
+; CHECK: = vavgub({{.*}}, {{.*}}):rnd
 
 ; Vector average words
 declare i64 @llvm.hexagon.A2.vavgw(i64, i64)
@@ -766,56 +770,56 @@ define i64 @A2_vavgw(i64 %a, i64 %b) {
   %z = call i64 @llvm.hexagon.A2.vavgw(i64 %a, i64 %b)
   ret i64 %z
 }
-; CHECK: r1:0 = vavgw(r1:0, r3:2)
+; CHECK: = vavgw({{.*}}, {{.*}})
 
 declare i64 @llvm.hexagon.A2.vavgwr(i64, i64)
 define i64 @A2_vavgwr(i64 %a, i64 %b) {
   %z = call i64 @llvm.hexagon.A2.vavgwr(i64 %a, i64 %b)
   ret i64 %z
 }
-; CHECK: r1:0 = vavgw(r1:0, r3:2):rnd
+; CHECK: = vavgw({{.*}}, {{.*}}):rnd
 
 declare i64 @llvm.hexagon.A2.vavgwcr(i64, i64)
 define i64 @A2_vavgwcr(i64 %a, i64 %b) {
   %z = call i64 @llvm.hexagon.A2.vavgwcr(i64 %a, i64 %b)
   ret i64 %z
 }
-; CHECK: r1:0 = vavgw(r1:0, r3:2):crnd
+; CHECK: = vavgw({{.*}}, {{.*}}):crnd
 
 declare i64 @llvm.hexagon.A2.vavguw(i64, i64)
 define i64 @A2_vavguw(i64 %a, i64 %b) {
   %z = call i64 @llvm.hexagon.A2.vavguw(i64 %a, i64 %b)
   ret i64 %z
 }
-; CHECK: r1:0 = vavguw(r1:0, r3:2)
+; CHECK: = vavguw({{.*}}, {{.*}})
 
 declare i64 @llvm.hexagon.A2.vavguwr(i64, i64)
 define i64 @A2_vavguwr(i64 %a, i64 %b) {
   %z = call i64 @llvm.hexagon.A2.vavguwr(i64 %a, i64 %b)
   ret i64 %z
 }
-; CHECK: r1:0 = vavguw(r1:0, r3:2):rnd
+; CHECK: = vavguw({{.*}}, {{.*}}):rnd
 
 declare i64 @llvm.hexagon.A2.vnavgw(i64, i64)
 define i64 @A2_vnavgw(i64 %a, i64 %b) {
   %z = call i64 @llvm.hexagon.A2.vnavgw(i64 %a, i64 %b)
   ret i64 %z
 }
-; CHECK: r1:0 = vnavgw(r1:0, r3:2)
+; CHECK: = vnavgw({{.*}}, {{.*}})
 
 declare i64 @llvm.hexagon.A2.vnavgwr(i64, i64)
 define i64 @A2_vnavgwr(i64 %a, i64 %b) {
   %z = call i64 @llvm.hexagon.A2.vnavgwr(i64 %a, i64 %b)
   ret i64 %z
 }
-; CHECK: r1:0 = vnavgw(r1:0, r3:2):rnd
+; CHECK: = vnavgw({{.*}}, {{.*}}):rnd
 
 declare i64 @llvm.hexagon.A2.vnavgwcr(i64, i64)
 define i64 @A2_vnavgwcr(i64 %a, i64 %b) {
   %z = call i64 @llvm.hexagon.A2.vnavgwcr(i64 %a, i64 %b)
   ret i64 %z
 }
-; CHECK: r1:0 = vnavgw(r1:0, r3:2):crnd
+; CHECK: = vnavgw({{.*}}, {{.*}}):crnd
 
 ; Vector conditional negate
 declare i64 @llvm.hexagon.S2.vcnegh(i64, i32)
@@ -823,14 +827,14 @@ define i64 @S2_vcnegh(i64 %a, i32 %b) {
   %z = call i64 @llvm.hexagon.S2.vcnegh(i64 %a, i32 %b)
   ret i64 %z
 }
-; CHECK: r1:0 = vcnegh(r1:0, r2)
+; CHECK: = vcnegh({{.*}}, {{.*}})
 
 declare i64 @llvm.hexagon.S2.vrcnegh(i64, i64, i32)
 define i64 @S2_vrcnegh(i64 %a, i64 %b, i32 %c) {
   %z = call i64 @llvm.hexagon.S2.vrcnegh(i64 %a, i64 %b, i32 %c)
   ret i64 %z
 }
-; CHECK: r1:0 += vrcnegh(r3:2, r4)
+; CHECK: += vrcnegh({{.*}}, {{.*}})
 
 ; Vector maximum bytes
 declare i64 @llvm.hexagon.A2.vmaxub(i64, i64)
@@ -838,14 +842,14 @@ define i64 @A2_vmaxub(i64 %a, i64 %b) {
   %z = call i64 @llvm.hexagon.A2.vmaxub(i64 %a, i64 %b)
   ret i64 %z
 }
-; CHECK: r1:0 = vmaxub(r1:0, r3:2)
+; CHECK: = vmaxub({{.*}}, {{.*}})
 
 declare i64 @llvm.hexagon.A2.vmaxb(i64, i64)
 define i64 @A2_vmaxb(i64 %a, i64 %b) {
   %z = call i64 @llvm.hexagon.A2.vmaxb(i64 %a, i64 %b)
   ret i64 %z
 }
-; CHECK: r1:0 = vmaxb(r1:0, r3:2)
+; CHECK: = vmaxb({{.*}}, {{.*}})
 
 ; Vector maximum halfwords
 declare i64 @llvm.hexagon.A2.vmaxh(i64, i64)
@@ -853,14 +857,14 @@ define i64 @A2_vmaxh(i64 %a, i64 %b) {
   %z = call i64 @llvm.hexagon.A2.vmaxh(i64 %a, i64 %b)
   ret i64 %z
 }
-; CHECK: r1:0 = vmaxh(r1:0, r3:2)
+; CHECK: = vmaxh({{.*}}, {{.*}})
 
 declare i64 @llvm.hexagon.A2.vmaxuh(i64, i64)
 define i64 @A2_vmaxuh(i64 %a, i64 %b) {
   %z = call i64 @llvm.hexagon.A2.vmaxuh(i64 %a, i64 %b)
   ret i64 %z
 }
-; CHECK: r1:0 = vmaxuh(r1:0, r3:2)
+; CHECK: = vmaxuh({{.*}}, {{.*}})
 
 ; Vector reduce maximum halfwords
 declare i64 @llvm.hexagon.A4.vrmaxh(i64, i64, i32)
@@ -868,14 +872,14 @@ define i64 @A4_vrmaxh(i64 %a, i64 %b, i32 %c) {
   %z = call i64 @llvm.hexagon.A4.vrmaxh(i64 %a, i64 %b, i32 %c)
   ret i64 %z
 }
-; CHECK: r1:0 = vrmaxh(r3:2, r4)
+; CHECK: = vrmaxh({{.*}}, {{.*}})
 
 declare i64 @llvm.hexagon.A4.vrmaxuh(i64, i64, i32)
 define i64 @A4_vrmaxuh(i64 %a, i64 %b, i32 %c) {
   %z = call i64 @llvm.hexagon.A4.vrmaxuh(i64 %a, i64 %b, i32 %c)
   ret i64 %z
 }
-; CHECK: r1:0 = vrmaxuh(r3:2, r4)
+; CHECK: = vrmaxuh({{.*}}, {{.*}})
 
 ; Vector reduce maximum words
 declare i64 @llvm.hexagon.A4.vrmaxw(i64, i64, i32)
@@ -883,14 +887,14 @@ define i64 @A4_vrmaxw(i64 %a, i64 %b, i32 %c) {
   %z = call i64 @llvm.hexagon.A4.vrmaxw(i64 %a, i64 %b, i32 %c)
   ret i64 %z
 }
-; CHECK: r1:0 = vrmaxw(r3:2, r4)
+; CHECK: = vrmaxw({{.*}}, {{.*}})
 
 declare i64 @llvm.hexagon.A4.vrmaxuw(i64, i64, i32)
 define i64 @A4_vrmaxuw(i64 %a, i64 %b, i32 %c) {
   %z = call i64 @llvm.hexagon.A4.vrmaxuw(i64 %a, i64 %b, i32 %c)
   ret i64 %z
 }
-; CHECK: r1:0 = vrmaxuw(r3:2, r4)
+; CHECK:  vrmaxuw({{.*}}, {{.*}})
 
 ; Vector minimum bytes
 declare i64 @llvm.hexagon.A2.vminub(i64, i64)
@@ -898,14 +902,14 @@ define i64 @A2_vminub(i64 %a, i64 %b) {
   %z = call i64 @llvm.hexagon.A2.vminub(i64 %a, i64 %b)
   ret i64 %z
 }
-; CHECK: r1:0 = vminub(r1:0, r3:2)
+; CHECK: = vminub({{.*}}, {{.*}})
 
 declare i64 @llvm.hexagon.A2.vminb(i64, i64)
 define i64 @A2_vminb(i64 %a, i64 %b) {
   %z = call i64 @llvm.hexagon.A2.vminb(i64 %a, i64 %b)
   ret i64 %z
 }
-; CHECK: r1:0 = vminb(r1:0, r3:2)
+; CHECK: = vminb({{.*}}, {{.*}})
 
 ; Vector minimum halfwords
 declare i64 @llvm.hexagon.A2.vminh(i64, i64)
@@ -913,14 +917,14 @@ define i64 @A2_vminh(i64 %a, i64 %b) {
   %z = call i64 @llvm.hexagon.A2.vminh(i64 %a, i64 %b)
   ret i64 %z
 }
-; CHECK: r1:0 = vminh(r1:0, r3:2)
+; CHECK: = vminh({{.*}}, {{.*}})
 
 declare i64 @llvm.hexagon.A2.vminuh(i64, i64)
 define i64 @A2_vminuh(i64 %a, i64 %b) {
   %z = call i64 @llvm.hexagon.A2.vminuh(i64 %a, i64 %b)
   ret i64 %z
 }
-; CHECK: r1:0 = vminuh(r1:0, r3:2)
+; CHECK: = vminuh({{.*}}, {{.*}})
 
 ; Vector reduce minimum halfwords
 declare i64 @llvm.hexagon.A4.vrminh(i64, i64, i32)
@@ -928,14 +932,14 @@ define i64 @A4_vrminh(i64 %a, i64 %b, i32 %c) {
   %z = call i64 @llvm.hexagon.A4.vrminh(i64 %a, i64 %b, i32 %c)
   ret i64 %z
 }
-; CHECK: r1:0 = vrminh(r3:2, r4)
+; CHECK: = vrminh({{.*}}, {{.*}})
 
 declare i64 @llvm.hexagon.A4.vrminuh(i64, i64, i32)
 define i64 @A4_vrminuh(i64 %a, i64 %b, i32 %c) {
   %z = call i64 @llvm.hexagon.A4.vrminuh(i64 %a, i64 %b, i32 %c)
   ret i64 %z
 }
-; CHECK: r1:0 = vrminuh(r3:2, r4)
+; CHECK: = vrminuh({{.*}}, {{.*}})
 
 ; Vector reduce minimum words
 declare i64 @llvm.hexagon.A4.vrminw(i64, i64, i32)
@@ -943,14 +947,14 @@ define i64 @A4_vrminw(i64 %a, i64 %b, i32 %c) {
   %z = call i64 @llvm.hexagon.A4.vrminw(i64 %a, i64 %b, i32 %c)
   ret i64 %z
 }
-; CHECK: r1:0 = vrminw(r3:2, r4)
+; CHECK: = vrminw({{.*}}, {{.*}})
 
 declare i64 @llvm.hexagon.A4.vrminuw(i64, i64, i32)
 define i64 @A4_vrminuw(i64 %a, i64 %b, i32 %c) {
   %z = call i64 @llvm.hexagon.A4.vrminuw(i64 %a, i64 %b, i32 %c)
   ret i64 %z
 }
-; CHECK: r1:0 = vrminuw(r3:2, r4)
+; CHECK: = vrminuw({{.*}}, {{.*}})
 
 ; Vector sum of absolute differences unsigned bytes
 declare i64 @llvm.hexagon.A2.vrsadub(i64, i64)
@@ -958,14 +962,14 @@ define i64 @A2_vrsadub(i64 %a, i64 %b) {
   %z = call i64 @llvm.hexagon.A2.vrsadub(i64 %a, i64 %b)
   ret i64 %z
 }
-; CHECK: r1:0 = vrsadub(r1:0, r3:2)
+; CHECK: = vrsadub({{.*}}, {{.*}})
 
 declare i64 @llvm.hexagon.A2.vrsadub.acc(i64, i64, i64)
 define i64 @A2_vrsadub_acc(i64 %a, i64 %b, i64 %c) {
   %z = call i64 @llvm.hexagon.A2.vrsadub.acc(i64 %a, i64 %b, i64 %c)
   ret i64 %z
 }
-; CHECK: r1:0 += vrsadub(r3:2, r5:4)
+; CHECK: += vrsadub({{.*}}, {{.*}})
 
 ; Vector subtract halfwords
 declare i64 @llvm.hexagon.A2.vsubh(i64, i64)
@@ -973,21 +977,21 @@ define i64 @A2_vsubh(i64 %a, i64 %b) {
   %z = call i64 @llvm.hexagon.A2.vsubh(i64 %a, i64 %b)
   ret i64 %z
 }
-; CHECK: r1:0 = vsubh(r1:0, r3:2)
+; CHECK: = vsubh({{.*}}, {{.*}})
 
 declare i64 @llvm.hexagon.A2.vsubhs(i64, i64)
 define i64 @A2_vsubhs(i64 %a, i64 %b) {
   %z = call i64 @llvm.hexagon.A2.vsubhs(i64 %a, i64 %b)
   ret i64 %z
 }
-; CHECK: r1:0 = vsubh(r1:0, r3:2):sat
+; CHECK: = vsubh({{.*}}, {{.*}}):sat
 
 declare i64 @llvm.hexagon.A2.vsubuhs(i64, i64)
 define i64 @A2_vsubuhs(i64 %a, i64 %b) {
   %z = call i64 @llvm.hexagon.A2.vsubuhs(i64 %a, i64 %b)
   ret i64 %z
 }
-; CHECK: r1:0 = vsubuh(r1:0, r3:2):sat
+; CHECK: = vsubuh({{.*}}, {{.*}}):sat
 
 ; Vector subtract bytes
 declare i64 @llvm.hexagon.A2.vsubub(i64, i64)
@@ -995,14 +999,14 @@ define i64 @A2_vsubub(i64 %a, i64 %b) {
   %z = call i64 @llvm.hexagon.A2.vsubub(i64 %a, i64 %b)
   ret i64 %z
 }
-; CHECK: r1:0 = vsubub(r1:0, r3:2)
+; CHECK: = vsubub({{.*}}, {{.*}})
 
 declare i64 @llvm.hexagon.A2.vsububs(i64, i64)
 define i64 @A2_vsububs(i64 %a, i64 %b) {
   %z = call i64 @llvm.hexagon.A2.vsububs(i64 %a, i64 %b)
   ret i64 %z
 }
-; CHECK: r1:0 = vsubub(r1:0, r3:2):sat
+; CHECK: = vsubub({{.*}}, {{.*}}):sat
 
 ; Vector subtract words
 declare i64 @llvm.hexagon.A2.vsubw(i64, i64)
@@ -1010,11 +1014,11 @@ define i64 @A2_vsubw(i64 %a, i64 %b) {
   %z = call i64 @llvm.hexagon.A2.vsubw(i64 %a, i64 %b)
   ret i64 %z
 }
-; CHECK: r1:0 = vsubw(r1:0, r3:2)
+; CHECK: = vsubw({{.*}}, {{.*}})
 
 declare i64 @llvm.hexagon.A2.vsubws(i64, i64)
 define i64 @A2_vsubws(i64 %a, i64 %b) {
   %z = call i64 @llvm.hexagon.A2.vsubws(i64 %a, i64 %b)
   ret i64 %z
 }
-; CHECK: r1:0 = vsubw(r1:0, r3:2):sat
+; CHECK: = vsubw({{.*}}, {{.*}}):sat
diff --git a/test/CodeGen/Hexagon/intrinsics/xtype_bit.ll b/test/CodeGen/Hexagon/intrinsics/xtype_bit.ll
index 8531b2f9334..e8f83d01820 100644
--- a/test/CodeGen/Hexagon/intrinsics/xtype_bit.ll
+++ b/test/CodeGen/Hexagon/intrinsics/xtype_bit.ll
@@ -1,69 +1,72 @@
 ; RUN: llc -march=hexagon -O0 < %s | FileCheck %s
+; RUN: llc -march=hexagon -O0 < %s | FileCheck -check-prefix=CHECK-CALL %s
 ; Hexagon Programmer's Reference Manual 11.10.2 XTYPE/BIT
 
+; CHECK-CALL-NOT: call
+
 ; Count leading
 declare i32 @llvm.hexagon.S2.clbp(i64)
 define i32 @S2_clbp(i64 %a) {
   %z = call i32 @llvm.hexagon.S2.clbp(i64 %a)
   ret i32 %z
 }
-; CHECK: r0 = clb(r1:0)
+; CHECK: = clb({{.*}})
 
 declare i32 @llvm.hexagon.S2.cl0p(i64)
 define i32 @S2_cl0p(i64 %a) {
   %z = call i32 @llvm.hexagon.S2.cl0p(i64 %a)
   ret i32 %z
 }
-; CHECK: r0 = cl0(r1:0)
+; CHECK: = cl0({{.*}})
 
 declare i32 @llvm.hexagon.S2.cl1p(i64)
 define i32 @S2_cl1p(i64 %a) {
   %z = call i32 @llvm.hexagon.S2.cl1p(i64 %a)
   ret i32 %z
 }
-; CHECK: r0 = cl1(r1:0)
+; CHECK: = cl1({{.*}})
 
 declare i32 @llvm.hexagon.S4.clbpnorm(i64)
 define i32 @S4_clbpnorm(i64 %a) {
   %z = call i32 @llvm.hexagon.S4.clbpnorm(i64 %a)
   ret i32 %z
 }
-; CHECK: r0 = normamt(r1:0)
+; CHECK: = normamt({{.*}})
 
 declare i32 @llvm.hexagon.S4.clbpaddi(i64, i32)
 define i32 @S4_clbpaddi(i64 %a) {
   %z = call i32 @llvm.hexagon.S4.clbpaddi(i64 %a, i32 0)
   ret i32 %z
 }
-; CHECK: r0 = add(clb(r1:0), #0)
+; CHECK: = add(clb({{.*}}), #0)
 
 declare i32 @llvm.hexagon.S4.clbaddi(i32, i32)
 define i32 @S4_clbaddi(i32 %a) {
   %z = call i32 @llvm.hexagon.S4.clbaddi(i32 %a, i32 0)
   ret i32 %z
 }
-; CHECK: r0 = add(clb(r0), #0)
+; CHECK: = add(clb({{.*}}), #0)
 
 declare i32 @llvm.hexagon.S2.cl0(i32)
 define i32 @S2_cl0(i32 %a) {
   %z = call i32 @llvm.hexagon.S2.cl0(i32 %a)
   ret i32 %z
 }
-; CHECK: r0 = cl0(r0)
+; CHECK: = cl0({{.*}})
 
 declare i32 @llvm.hexagon.S2.cl1(i32)
 define i32 @S2_cl1(i32 %a) {
   %z = call i32 @llvm.hexagon.S2.cl1(i32 %a)
   ret i32 %z
 }
-; CHECK: r0 = cl1(r0)
+; CHECK: = cl1({{.*}})
 
 declare i32 @llvm.hexagon.S2.clbnorm(i32)
 define i32 @S4_clbnorm(i32 %a) {
   %z = call i32 @llvm.hexagon.S2.clbnorm(i32 %a)
   ret i32 %z
 }
-; CHECK: r0 = normamt(r0)
+; CHECK: = normamt({{.*}})
 
 ; Count population
 declare i32 @llvm.hexagon.S5.popcountp(i64)
@@ -71,7 +74,7 @@ define i32 @S5_popcountp(i64 %a) {
   %z = call i32 @llvm.hexagon.S5.popcountp(i64 %a)
   ret i32 %z
 }
-; CHECK: r0 = popcount(r1:0)
+; CHECK: = popcount({{.*}})
 
 ; Count trailing
 declare i32 @llvm.hexagon.S2.ct0p(i64)
@@ -79,28 +82,28 @@ define i32 @S2_ct0p(i64 %a) {
   %z = call i32 @llvm.hexagon.S2.ct0p(i64 %a)
   ret i32 %z
 }
-; CHECK: r0 = ct0(r1:0)
+; CHECK: = ct0({{.*}})
 
 declare i32 @llvm.hexagon.S2.ct1p(i64)
 define i32 @S2_ct1p(i64 %a) {
   %z = call i32 @llvm.hexagon.S2.ct1p(i64 %a)
   ret i32 %z
 }
-; CHECK: r0 = ct1(r1:0)
+; CHECK: = ct1({{.*}})
 
 declare i32 @llvm.hexagon.S2.ct0(i32)
 define i32 @S2_ct0(i32 %a) {
   %z = call i32 @llvm.hexagon.S2.ct0(i32 %a)
   ret i32 %z
 }
-; CHECK: r0 = ct0(r0)
+; CHECK: = ct0({{.*}})
 
 declare i32 @llvm.hexagon.S2.ct1(i32)
 define i32 @S2_ct1(i32 %a) {
   %z = call i32 @llvm.hexagon.S2.ct1(i32 %a)
   ret i32 %z
 }
-; CHECK: r0 = ct1(r0)
+; CHECK: = ct1({{.*}})
 
 ; Extract bitfield
 declare i64 @llvm.hexagon.S2.extractup(i64, i32, i32)
@@ -108,56 +111,56 @@ define i64 @S2_extractup(i64 %a) {
   %z = call i64 @llvm.hexagon.S2.extractup(i64 %a, i32 0, i32 0)
   ret i64 %z
 }
-; CHECK: r1:0 = extractu(r1:0, #0, #0)
+; CHECK: = extractu({{.*}}, #0, #0)
 
 declare i64 @llvm.hexagon.S4.extractp(i64, i32, i32)
 define i64 @S2_extractp(i64 %a) {
   %z = call i64 @llvm.hexagon.S4.extractp(i64 %a, i32 0, i32 0)
   ret i64 %z
 }
-; CHECK: r1:0 = extract(r1:0, #0, #0)
+; CHECK: = extract({{.*}}, #0, #0)
 
 declare i32 @llvm.hexagon.S2.extractu(i32, i32, i32)
 define i32 @S2_extractu(i32 %a) {
   %z = call i32 @llvm.hexagon.S2.extractu(i32 %a, i32 0, i32 0)
   ret i32 %z
 }
-; CHECK: r0 = extractu(r0, #0, #0)
+; CHECK: = extractu({{.*}}, #0, #0)
 
 declare i32 @llvm.hexagon.S4.extract(i32, i32, i32)
 define i32 @S2_extract(i32 %a) {
   %z = call i32 @llvm.hexagon.S4.extract(i32 %a, i32 0, i32 0)
   ret i32 %z
 }
-; CHECK: r0 = extract(r0, #0, #0)
+; CHECK: = extract({{.*}}, #0, #0)
 
 declare i64 @llvm.hexagon.S2.extractup.rp(i64, i64)
 define i64 @S2_extractup_rp(i64 %a, i64 %b) {
   %z = call i64 @llvm.hexagon.S2.extractup.rp(i64 %a, i64 %b)
   ret i64 %z
 }
-; CHECK: r1:0 = extractu(r1:0, r3:2)
+; CHECK: = extractu({{.*}}, {{.*}})
 
 declare i64 @llvm.hexagon.S4.extractp.rp(i64, i64)
 define i64 @S4_extractp_rp(i64 %a, i64 %b) {
   %z = call i64 @llvm.hexagon.S4.extractp.rp(i64 %a, i64 %b)
   ret i64 %z
 }
-; CHECK: r1:0 = extract(r1:0, r3:2)
+; CHECK: = extract({{.*}}, {{.*}})
 
 declare i32 @llvm.hexagon.S2.extractu.rp(i32, i64)
 define i32 @S2_extractu_rp(i32 %a, i64 %b) {
   %z = call i32 @llvm.hexagon.S2.extractu.rp(i32 %a, i64 %b)
   ret i32 %z
 }
-; CHECK: r0 = extractu(r0, r3:2)
+; CHECK: = extractu({{.*}}, {{.*}})
 
 declare i32 @llvm.hexagon.S4.extract.rp(i32, i64)
 define i32 @S4_extract_rp(i32 %a, i64 %b) {
   %z = call i32 @llvm.hexagon.S4.extract.rp(i32 %a, i64 %b)
   ret i32 %z
 }
-; CHECK: r0 = extract(r0, r3:2)
+; CHECK: = extract({{.*}}, {{.*}})
 
 ; Insert bitfield
 declare i64 @llvm.hexagon.S2.insertp(i64, i64, i32, i32)
@@ -165,28 +168,28 @@ define i64 @S2_insertp(i64 %a, i64 %b) {
   %z = call i64 @llvm.hexagon.S2.insertp(i64 %a, i64 %b, i32 0, i32 0)
   ret i64 %z
 }
-; CHECK: r1:0 = insert(r3:2, #0, #0)
+; CHECK: = insert({{.*}}, #0, #0)
 
 declare i32 @llvm.hexagon.S2.insert(i32, i32, i32, i32)
 define i32 @S2_insert(i32 %a, i32 %b) {
   %z = call i32 @llvm.hexagon.S2.insert(i32 %a, i32 %b, i32 0, i32 0)
   ret i32 %z
 }
-; CHECK: r0 = insert(r1, #0, #0)
+; CHECK: = insert({{.*}}, #0, #0)
 
 declare i32 @llvm.hexagon.S2.insert.rp(i32, i32, i64)
 define i32 @S2_insert_rp(i32 %a, i32 %b, i64 %c) {
   %z = call i32 @llvm.hexagon.S2.insert.rp(i32 %a, i32 %b, i64 %c)
   ret i32 %z
 }
-; CHECK: r0 = insert(r1, r3:2)
+; CHECK: = insert({{.*}}, {{.*}})
 
 declare i64 @llvm.hexagon.S2.insertp.rp(i64, i64, i64)
 define i64 @S2_insertp_rp(i64 %a, i64 %b, i64 %c) {
   %z = call i64 @llvm.hexagon.S2.insertp.rp(i64 %a, i64 %b, i64 %c)
   ret i64 %z
 }
-; CHECK: r1:0 = insert(r3:2, r5:4)
+; CHECK: = insert({{.*}}, r5:4)
 
 ; Interleave/deinterleave
 declare i64 @llvm.hexagon.S2.deinterleave(i64)
@@ -194,14 +197,14 @@ define i64 @S2_deinterleave(i64 %a) {
   %z = call i64 @llvm.hexagon.S2.deinterleave(i64 %a)
   ret i64 %z
 }
-; CHECK: r1:0 = deinterleave(r1:0)
+; CHECK: = deinterleave({{.*}})
 
 declare i64 @llvm.hexagon.S2.interleave(i64)
 define i64 @S2_interleave(i64 %a) {
   %z = call i64 @llvm.hexagon.S2.interleave(i64 %a)
   ret i64 %z
 }
-; CHECK: r1:0 = interleave(r1:0)
+; CHECK: = interleave({{.*}})
 
 ; Linear feedback-shift operation
 declare i64 @llvm.hexagon.S2.lfsp(i64, i64)
@@ -209,7 +212,7 @@ define i64 @S2_lfsp(i64 %a, i64 %b) {
   %z = call i64 @llvm.hexagon.S2.lfsp(i64 %a, i64 %b)
   ret i64 %z
 }
-; CHECK: r1:0 = lfs(r1:0, r3:2)
+; CHECK: = lfs({{.*}}, {{.*}})
 
 ; Masked parity
 declare i32 @llvm.hexagon.S2.parityp(i64, i64)
@@ -217,14 +220,14 @@ define i32 @S2_parityp(i64 %a, i64 %b) {
   %z = call i32 @llvm.hexagon.S2.parityp(i64 %a, i64 %b)
   ret i32 %z
 }
-; CHECK: r0 = parity(r1:0, r3:2)
+; CHECK: = parity({{.*}}, {{.*}})
 
 declare i32 @llvm.hexagon.S4.parity(i32, i32)
 define i32 @S4_parity(i32 %a, i32 %b) {
   %z = call i32 @llvm.hexagon.S4.parity(i32 %a, i32 %b)
   ret i32 %z
 }
-; CHECK: r0 = parity(r0, r1)
+; CHECK: = parity({{.*}}, {{.*}})
 
 ; Bit reverse
 declare i64 @llvm.hexagon.S2.brevp(i64)
@@ -232,14 +235,14 @@ define i64 @S2_brevp(i64 %a) {
   %z = call i64 @llvm.hexagon.S2.brevp(i64 %a)
   ret i64 %z
 }
-; CHECK: r1:0 = brev(r1:0)
+; CHECK: = brev({{.*}})
 
 declare i32 @llvm.hexagon.S2.brev(i32)
 define i32 @S2_brev(i32 %a) {
   %z = call i32 @llvm.hexagon.S2.brev(i32 %a)
   ret i32 %z
 }
-; CHECK: r0 = brev(r0)
+; CHECK: = brev({{.*}})
 
 ; Set/clear/toggle bit
 declare i32 @llvm.hexagon.S2.setbit.i(i32, i32)
@@ -247,42 +250,42 @@ define i32 @S2_setbit_i(i32 %a) {
   %z = call i32 @llvm.hexagon.S2.setbit.i(i32 %a, i32 0)
   ret i32 %z
 }
-; CHECK: r0 = setbit(r0, #0)
+; CHECK: = setbit({{.*}}, #0)
 
 declare i32 @llvm.hexagon.S2.clrbit.i(i32, i32)
 define i32 @S2_clrbit_i(i32 %a) {
   %z = call i32 @llvm.hexagon.S2.clrbit.i(i32 %a, i32 0)
   ret i32 %z
 }
-; CHECK: r0 = clrbit(r0, #0)
+; CHECK: = clrbit({{.*}}, #0)
 
 declare i32 @llvm.hexagon.S2.togglebit.i(i32, i32)
 define i32 @S2_togglebit_i(i32 %a) {
   %z = call i32 @llvm.hexagon.S2.togglebit.i(i32 %a, i32 0)
   ret i32 %z
 }
-; CHECK: r0 = togglebit(r0, #0)
+; CHECK: = togglebit({{.*}}, #0)
 
 declare i32 @llvm.hexagon.S2.setbit.r(i32, i32)
 define i32 @S2_setbit_r(i32 %a, i32 %b) {
   %z = call i32 @llvm.hexagon.S2.setbit.r(i32 %a, i32 %b)
   ret i32 %z
 }
-; CHECK: r0 = setbit(r0, r1)
+; CHECK: = setbit({{.*}}, {{.*}})
 
 declare i32 @llvm.hexagon.S2.clrbit.r(i32, i32)
 define i32 @S2_clrbit_r(i32 %a, i32 %b) {
   %z = call i32 @llvm.hexagon.S2.clrbit.r(i32 %a, i32 %b)
   ret i32 %z
 }
-; CHECK: r0 = clrbit(r0, r1)
+; CHECK: = clrbit({{.*}}, {{.*}})
 
 declare i32 @llvm.hexagon.S2.togglebit.r(i32, i32)
 define i32 @S2_togglebit_r(i32 %a, i32 %b) {
   %z = call i32 @llvm.hexagon.S2.togglebit.r(i32 %a, i32 %b)
   ret i32 %z
 }
-; CHECK: r0 = togglebit(r0, r1)
+; CHECK: = togglebit({{.*}}, {{.*}})
 
 ; Split bitfield
 declare i64 @llvm.hexagon.A4.bitspliti(i32, i32)
@@ -290,14 +293,14 @@ define i64 @A4_bitspliti(i32 %a) {
   %z = call i64 @llvm.hexagon.A4.bitspliti(i32 %a, i32 0)
   ret i64 %z
 }
-; CHECK:  = bitsplit(r0, #0)
+; CHECK: = bitsplit({{.*}}, #0)
 
 declare i64 @llvm.hexagon.A4.bitsplit(i32, i32)
 define i64 @A4_bitsplit(i32 %a, i32 %b) {
   %z = call i64 @llvm.hexagon.A4.bitsplit(i32 %a, i32 %b)
   ret i64 %z
 }
-; CHECK: r1:0 = bitsplit(r0, r1)
+; CHECK: = bitsplit({{.*}}, {{.*}})
 
 ; Table index
 declare i32 @llvm.hexagon.S2.tableidxb.goodsyntax(i32, i32, i32, i32)
@@ -305,25 +308,25 @@ define i32 @S2_tableidxb_goodsyntax(i32 %a, i32 %b) {
   %z = call i32 @llvm.hexagon.S2.tableidxb.goodsyntax(i32 %a, i32 %b, i32 0, i32 0)
   ret i32 %z
 }
-; CHECK: r0 = tableidxb(r1, #0, #0)
+; CHECK: = tableidxb({{.*}}, #0, #0)
 
 declare i32 @llvm.hexagon.S2.tableidxh.goodsyntax(i32, i32, i32, i32)
 define i32 @S2_tableidxh_goodsyntax(i32 %a, i32 %b) {
   %z = call i32 @llvm.hexagon.S2.tableidxh.goodsyntax(i32 %a, i32 %b, i32 0, i32 0)
   ret i32 %z
 }
-; CHECK: r0 = tableidxh(r1, #0, #-1)
+; CHECK: = tableidxh({{.*}}, #0, #-1)
 
 declare i32 @llvm.hexagon.S2.tableidxw.goodsyntax(i32, i32, i32, i32)
 define i32 @S2_tableidxw_goodsyntax(i32 %a, i32 %b) {
   %z = call i32 @llvm.hexagon.S2.tableidxw.goodsyntax(i32 %a, i32 %b, i32 0, i32 0)
   ret i32 %z
 }
-; CHECK: r0 = tableidxw(r1, #0, #-2)
+; CHECK: = tableidxw({{.*}}, #0, #-2)
 
 declare i32 @llvm.hexagon.S2.tableidxd.goodsyntax(i32, i32, i32, i32)
 define i32 @S2_tableidxd_goodsyntax(i32 %a, i32 %b) {
   %z = call i32 @llvm.hexagon.S2.tableidxd.goodsyntax(i32 %a, i32 %b, i32 0, i32 0)
   ret i32 %z
 }
-; CHECK: r0 = tableidxd(r1, #0, #-3)
+; CHECK: = tableidxd({{.*}}, #0, #-3)
diff --git a/test/CodeGen/Hexagon/intrinsics/xtype_complex.ll b/test/CodeGen/Hexagon/intrinsics/xtype_complex.ll
index 57b0c5b6db5..0087883573e 100644
--- a/test/CodeGen/Hexagon/intrinsics/xtype_complex.ll
+++ b/test/CodeGen/Hexagon/intrinsics/xtype_complex.ll
@@ -1,34 +1,37 @@
 ; RUN: llc -march=hexagon -O0 < %s | FileCheck %s
+; RUN: llc -march=hexagon -O0 < %s | FileCheck -check-prefix=CHECK-CALL %s
 ; Hexagon Programmer's Reference Manual 11.10.3 XTYPE/COMPLEX
 
+; CHECK-CALL-NOT: call
+
 ; Complex add/sub halfwords
 declare i64 @llvm.hexagon.S4.vxaddsubh(i64, i64)
 define i64 @S4_vxaddsubh(i64 %a, i64 %b) {
   %z = call i64 @llvm.hexagon.S4.vxaddsubh(i64 %a, i64 %b)
   ret i64 %z
 }
-; CHECK: r1:0 = vxaddsubh(r1:0, r3:2):sat
+; CHECK: = vxaddsubh({{.*}}, {{.*}}):sat
 
 declare i64 @llvm.hexagon.S4.vxsubaddh(i64, i64)
 define i64 @S4_vxsubaddh(i64 %a, i64 %b) {
   %z = call i64 @llvm.hexagon.S4.vxsubaddh(i64 %a, i64 %b)
   ret i64 %z
 }
-; CHECK: r1:0 = vxsubaddh(r1:0, r3:2):sat
+; CHECK: = vxsubaddh({{.*}}, {{.*}}):sat
 
 declare i64 @llvm.hexagon.S4.vxaddsubhr(i64, i64)
 define i64 @S4_vxaddsubhr(i64 %a, i64 %b) {
   %z = call i64 @llvm.hexagon.S4.vxaddsubhr(i64 %a, i64 %b)
   ret i64 %z
 }
-; CHECK: r1:0 = vxaddsubh(r1:0, r3:2):rnd:>>1:sat
+; CHECK: = vxaddsubh({{.*}}, {{.*}}):rnd:>>1:sat
 
 declare i64 @llvm.hexagon.S4.vxsubaddhr(i64, i64)
 define i64 @S4_vxsubaddhr(i64 %a, i64 %b) {
   %z = call i64 @llvm.hexagon.S4.vxsubaddhr(i64 %a, i64 %b)
   ret i64 %z
 }
-; CHECK: r1:0 = vxsubaddh(r1:0, r3:2):rnd:>>1:sat
+; CHECK: = vxsubaddh({{.*}}, {{.*}}):rnd:>>1:sat
 
 ; Complex add/sub words
 declare i64 @llvm.hexagon.S4.vxaddsubw(i64, i64)
@@ -36,14 +39,14 @@ define i64 @S4_vxaddsubw(i64 %a, i64 %b) {
   %z = call i64 @llvm.hexagon.S4.vxaddsubw(i64 %a, i64 %b)
   ret i64 %z
 }
-; CHECK: r1:0 = vxaddsubw(r1:0, r3:2):sat
+; CHECK: = vxaddsubw({{.*}}, {{.*}}):sat
 
 declare i64 @llvm.hexagon.S4.vxsubaddw(i64, i64)
 define i64 @S4_vxsubaddw(i64 %a, i64 %b) {
   %z = call i64 @llvm.hexagon.S4.vxsubaddw(i64 %a, i64 %b)
   ret i64 %z
 }
-; CHECK: r1:0 = vxsubaddw(r1:0, r3:2):sat
+; CHECK: = vxsubaddw({{.*}}, {{.*}}):sat
 
 ; Complex multiply
 declare i64 @llvm.hexagon.M2.cmpys.s0(i32, i32)
@@ -51,84 +54,84 @@ define i64 @M2_cmpys_s0(i32 %a, i32 %b) {
   %z = call i64 @llvm.hexagon.M2.cmpys.s0(i32 %a, i32 %b)
   ret i64 %z
 }
-; CHECK: r1:0 = cmpy(r0, r1):sat
+; CHECK: = cmpy({{.*}}, {{.*}}):sat
 
 declare i64 @llvm.hexagon.M2.cmpys.s1(i32, i32)
 define i64 @M2_cmpys_s1(i32 %a, i32 %b) {
   %z = call i64 @llvm.hexagon.M2.cmpys.s1(i32 %a, i32 %b)
   ret i64 %z
 }
-; CHECK: r1:0 = cmpy(r0, r1):<<1:sat
+; CHECK: = cmpy({{.*}}, {{.*}}):<<1:sat
 
 declare i64 @llvm.hexagon.M2.cmpysc.s0(i32, i32)
 define i64 @M2_cmpysc_s0(i32 %a, i32 %b) {
   %z = call i64 @llvm.hexagon.M2.cmpysc.s0(i32 %a, i32 %b)
   ret i64 %z
 }
-; CHECK: r1:0 = cmpy(r0, r1*):sat
+; CHECK: = cmpy({{.*}}, {{.*}}*):sat
 
 declare i64 @llvm.hexagon.M2.cmpysc.s1(i32, i32)
 define i64 @M2_cmpysc_s1(i32 %a, i32 %b) {
   %z = call i64 @llvm.hexagon.M2.cmpysc.s1(i32 %a, i32 %b)
   ret i64 %z
 }
-; CHECK: r1:0 = cmpy(r0, r1*):<<1:sat
+; CHECK: = cmpy({{.*}}, {{.*}}*):<<1:sat
 
 declare i64 @llvm.hexagon.M2.cmacs.s0(i64, i32, i32)
 define i64 @M2_cmacs_s0(i64 %a, i32 %b, i32 %c) {
   %z = call i64 @llvm.hexagon.M2.cmacs.s0(i64 %a, i32 %b, i32 %c)
   ret i64 %z
 }
-; CHECK: r1:0 += cmpy(r2, r3):sat
+; CHECK: += cmpy({{.*}}, {{.*}}):sat
 
 declare i64 @llvm.hexagon.M2.cmacs.s1(i64, i32, i32)
 define i64 @M2_cmacs_s1(i64 %a, i32 %b, i32 %c) {
   %z = call i64 @llvm.hexagon.M2.cmacs.s1(i64 %a, i32 %b, i32 %c)
   ret i64 %z
 }
-; CHECK: r1:0 += cmpy(r2, r3):<<1:sat
+; CHECK: += cmpy({{.*}}, {{.*}}):<<1:sat
 
 declare i64 @llvm.hexagon.M2.cnacs.s0(i64, i32, i32)
 define i64 @M2_cnacs_s0(i64 %a, i32 %b, i32 %c) {
   %z = call i64 @llvm.hexagon.M2.cnacs.s0(i64 %a, i32 %b, i32 %c)
   ret i64 %z
 }
-; CHECK: r1:0 -= cmpy(r2, r3):sat
+; CHECK: -= cmpy({{.*}}, {{.*}}):sat
 
 declare i64 @llvm.hexagon.M2.cnacs.s1(i64, i32, i32)
 define i64 @M2_cnacs_s1(i64 %a, i32 %b, i32 %c) {
   %z = call i64 @llvm.hexagon.M2.cnacs.s1(i64 %a, i32 %b, i32 %c)
   ret i64 %z
 }
-; CHECK: r1:0 -= cmpy(r2, r3):<<1:sat
+; CHECK: -= cmpy({{.*}}, {{.*}}):<<1:sat
 
 declare i64 @llvm.hexagon.M2.cmacsc.s0(i64, i32, i32)
 define i64 @M2_cmacsc_s0(i64 %a, i32 %b, i32 %c) {
   %z = call i64 @llvm.hexagon.M2.cmacsc.s0(i64 %a, i32 %b, i32 %c)
   ret i64 %z
 }
-; CHECK: r1:0 += cmpy(r2, r3*):sat
+; CHECK: += cmpy({{.*}}, {{.*}}*):sat
 
 declare i64 @llvm.hexagon.M2.cmacsc.s1(i64, i32, i32)
 define i64 @M2_cmacsc_s1(i64 %a, i32 %b, i32 %c) {
   %z = call i64 @llvm.hexagon.M2.cmacsc.s1(i64 %a, i32 %b, i32 %c)
   ret i64 %z
 }
-; CHECK: r1:0 += cmpy(r2, r3*):<<1:sat
+; CHECK: += cmpy({{.*}}, {{.*}}*):<<1:sat
 
 declare i64 @llvm.hexagon.M2.cnacsc.s0(i64, i32, i32)
 define i64 @M2_cnacsc_s0(i64 %a, i32 %b, i32 %c) {
   %z = call i64 @llvm.hexagon.M2.cnacsc.s0(i64 %a, i32 %b, i32 %c)
   ret i64 %z
 }
-; CHECK: r1:0 -= cmpy(r2, r3*):sat
+; CHECK: -= cmpy({{.*}}, {{.*}}*):sat
 
 declare i64 @llvm.hexagon.M2.cnacsc.s1(i64, i32, i32)
 define i64 @M2_cnacsc_s1(i64 %a, i32 %b, i32 %c) {
   %z = call i64 @llvm.hexagon.M2.cnacsc.s1(i64 %a, i32 %b, i32 %c)
   ret i64 %z
 }
-; CHECK: r1:0 -= cmpy(r2, r3*):<<1:sat
+; CHECK: -= cmpy({{.*}}, {{.*}}*):<<1:sat
 
 ; Complex multiply real or imaginary
 declare i64 @llvm.hexagon.M2.cmpyi.s0(i32, i32)
@@ -136,28 +139,28 @@ define i64 @M2_cmpyi_s0(i32 %a, i32 %b) {
   %z = call i64 @llvm.hexagon.M2.cmpyi.s0(i32 %a, i32 %b)
   ret i64 %z
 }
-; CHECK: r1:0 = cmpyi(r0, r1)
+; CHECK: = cmpyi({{.*}}, {{.*}})
 
 declare i64 @llvm.hexagon.M2.cmpyr.s0(i32, i32)
 define i64 @M2_cmpyr_s0(i32 %a, i32 %b) {
   %z = call i64 @llvm.hexagon.M2.cmpyr.s0(i32 %a, i32 %b)
   ret i64 %z
 }
-; CHECK: r1:0 = cmpyr(r0, r1)
+; CHECK: = cmpyr({{.*}}, {{.*}})
 
 declare i64 @llvm.hexagon.M2.cmaci.s0(i64, i32, i32)
 define i64 @M2_cmaci_s0(i64 %a, i32 %b, i32 %c) {
   %z = call i64 @llvm.hexagon.M2.cmaci.s0(i64 %a, i32 %b, i32 %c)
   ret i64 %z
 }
-; CHECK: r1:0 += cmpyi(r2, r3)
+; CHECK: += cmpyi({{.*}}, {{.*}})
 
 declare i64 @llvm.hexagon.M2.cmacr.s0(i64, i32, i32)
 define i64 @M2_cmacr_s0(i64 %a, i32 %b, i32 %c) {
   %z = call i64 @llvm.hexagon.M2.cmacr.s0(i64 %a, i32 %b, i32 %c)
   ret i64 %z
 }
-; CHECK: r1:0 += cmpyr(r2, r3)
+; CHECK: += cmpyr({{.*}}, {{.*}})
 
 ; Complex multiply with round and pack
 declare i32 @llvm.hexagon.M2.cmpyrs.s0(i32, i32)
@@ -165,28 +168,28 @@ define i32 @M2_cmpyrs_s0(i32 %a, i32 %b) {
   %z = call i32 @llvm.hexagon.M2.cmpyrs.s0(i32 %a, i32 %b)
   ret i32 %z
 }
-; CHECK: r0 = cmpy(r0, r1):rnd:sat
+; CHECK: = cmpy({{.*}}, {{.*}}):rnd:sat
 
 declare i32 @llvm.hexagon.M2.cmpyrs.s1(i32, i32)
 define i32 @M2_cmpyrs_s1(i32 %a, i32 %b) {
   %z = call i32 @llvm.hexagon.M2.cmpyrs.s1(i32 %a, i32 %b)
   ret i32 %z
 }
-; CHECK: r0 = cmpy(r0, r1):<<1:rnd:sat
+; CHECK: = cmpy({{.*}}, {{.*}}):<<1:rnd:sat
 
 declare i32 @llvm.hexagon.M2.cmpyrsc.s0(i32, i32)
 define i32 @M2_cmpyrsc_s0(i32 %a, i32 %b) {
   %z = call i32 @llvm.hexagon.M2.cmpyrsc.s0(i32 %a, i32 %b)
   ret i32 %z
 }
-; CHECK: r0 = cmpy(r0, r1*):rnd:sat
+; CHECK: = cmpy({{.*}}, {{.*}}*):rnd:sat
 
 declare i32 @llvm.hexagon.M2.cmpyrsc.s1(i32, i32)
 define i32 @M2_cmpyrsc_s1(i32 %a, i32 %b) {
   %z = call i32 @llvm.hexagon.M2.cmpyrsc.s1(i32 %a, i32 %b)
   ret i32 %z
 }
-; CHECK: r0 = cmpy(r0, r1*):<<1:rnd:sat
+; CHECK: = cmpy({{.*}}, {{.*}}*):<<1:rnd:sat
 
 ; Complex multiply 32x16
 declare i32 @llvm.hexagon.M4.cmpyi.wh(i64, i32)
@@ -194,28 +197,28 @@ define i32 @M4_cmpyi_wh(i64 %a, i32 %b) {
   %z = call i32 @llvm.hexagon.M4.cmpyi.wh(i64 %a, i32 %b)
   ret i32 %z
 }
-; CHECK: r0 = cmpyiwh(r1:0, r2):<<1:rnd:sat
+; CHECK: = cmpyiwh({{.*}}, {{.*}}):<<1:rnd:sat
 
 declare i32 @llvm.hexagon.M4.cmpyi.whc(i64, i32)
 define i32 @M4_cmpyi_whc(i64 %a, i32 %b) {
   %z = call i32 @llvm.hexagon.M4.cmpyi.whc(i64 %a, i32 %b)
   ret i32 %z
 }
-; CHECK: r0 = cmpyiwh(r1:0, r2*):<<1:rnd:sat
+; CHECK: = cmpyiwh({{.*}}, {{.*}}*):<<1:rnd:sat
 
 declare i32 @llvm.hexagon.M4.cmpyr.wh(i64, i32)
 define i32 @M4_cmpyr_wh(i64 %a, i32 %b) {
   %z = call i32 @llvm.hexagon.M4.cmpyr.wh(i64 %a, i32 %b)
   ret i32 %z
 }
-; CHECK: r0 = cmpyrwh(r1:0, r2):<<1:rnd:sat
+; CHECK: = cmpyrwh({{.*}}, {{.*}}):<<1:rnd:sat
 
 declare i32 @llvm.hexagon.M4.cmpyr.whc(i64, i32)
 define i32 @M4_cmpyr_whc(i64 %a, i32 %b) {
   %z = call i32 @llvm.hexagon.M4.cmpyr.whc(i64 %a, i32 %b)
   ret i32 %z
 }
-; CHECK: r0 = cmpyrwh(r1:0, r2*):<<1:rnd:sat
+; CHECK: = cmpyrwh({{.*}}, {{.*}}*):<<1:rnd:sat
 
 ; Vector complex multiply real or imaginary
 declare i64 @llvm.hexagon.M2.vcmpy.s0.sat.r(i64, i64)
@@ -223,42 +226,42 @@ define i64 @M2_vcmpy_s0_sat_r(i64 %a, i64 %b) {
   %z = call i64 @llvm.hexagon.M2.vcmpy.s0.sat.r(i64 %a, i64 %b)
   ret i64 %z
 }
-; CHECK: r1:0 = vcmpyr(r1:0, r3:2):sat
+; CHECK: = vcmpyr({{.*}}, {{.*}}):sat
 
 declare i64 @llvm.hexagon.M2.vcmpy.s1.sat.r(i64, i64)
 define i64 @M2_vcmpy_s1_sat_r(i64 %a, i64 %b) {
   %z = call i64 @llvm.hexagon.M2.vcmpy.s1.sat.r(i64 %a, i64 %b)
   ret i64 %z
 }
-; CHECK: r1:0 = vcmpyr(r1:0, r3:2):<<1:sat
+; CHECK: = vcmpyr({{.*}}, {{.*}}):<<1:sat
 
 declare i64 @llvm.hexagon.M2.vcmpy.s0.sat.i(i64, i64)
 define i64 @M2_vcmpy_s0_sat_i(i64 %a, i64 %b) {
   %z = call i64 @llvm.hexagon.M2.vcmpy.s0.sat.i(i64 %a, i64 %b)
   ret i64 %z
 }
-; CHECK: r1:0 = vcmpyi(r1:0, r3:2):sat
+; CHECK: = vcmpyi({{.*}}, {{.*}}):sat
 
 declare i64 @llvm.hexagon.M2.vcmpy.s1.sat.i(i64, i64)
 define i64 @M2_vcmpy_s1_sat_i(i64 %a, i64 %b) {
   %z = call i64 @llvm.hexagon.M2.vcmpy.s1.sat.i(i64 %a, i64 %b)
   ret i64 %z
 }
-; CHECK: r1:0 = vcmpyi(r1:0, r3:2):<<1:sat
+; CHECK: = vcmpyi({{.*}}, {{.*}}):<<1:sat
 
 declare i64 @llvm.hexagon.M2.vcmac.s0.sat.r(i64, i64, i64)
 define i64 @M2_vcmac_s0_sat_r(i64 %a, i64 %b, i64 %c) {
   %z = call i64 @llvm.hexagon.M2.vcmac.s0.sat.r(i64 %a, i64 %b, i64 %c)
   ret i64 %z
 }
-; CHECK: r1:0 += vcmpyr(r3:2, r5:4):sat
+; CHECK: += vcmpyr({{.*}}, r5:4):sat
 
 declare i64 @llvm.hexagon.M2.vcmac.s0.sat.i(i64, i64, i64)
 define i64 @M2_vcmac_s0_sat_i(i64 %a, i64 %b, i64 %c) {
   %z = call i64 @llvm.hexagon.M2.vcmac.s0.sat.i(i64 %a, i64 %b, i64 %c)
   ret i64 %z
 }
-; CHECK: r1:0 += vcmpyi(r3:2, r5:4):sat
+; CHECK: += vcmpyi({{.*}}, r5:4):sat
 
 ; Vector complex conjugate
 declare i64 @llvm.hexagon.A2.vconj(i64)
@@ -266,7 +269,7 @@ define i64 @A2_vconj(i64 %a) {
   %z = call i64 @llvm.hexagon.A2.vconj(i64 %a)
   ret i64 %z
 }
-; CHECK: r1:0 = vconj(r1:0):sat
+; CHECK: = vconj({{.*}}):sat
 
 ; Vector complex rotate
 declare i64 @llvm.hexagon.S2.vcrotate(i64, i32)
@@ -274,7 +277,7 @@ define i64 @S2_vcrotate(i64 %a, i32 %b) {
   %z = call i64 @llvm.hexagon.S2.vcrotate(i64 %a, i32 %b)
   ret i64 %z
 }
-; CHECK: r1:0 = vcrotate(r1:0, r2)
+; CHECK: = vcrotate({{.*}}, {{.*}})
 
 ; Vector reduce complex multiply real or imaginary
 declare i64 @llvm.hexagon.M2.vrcmpyi.s0(i64, i64)
@@ -282,56 +285,56 @@ define i64 @M2_vrcmpyi_s0(i64 %a, i64 %b) {
   %z = call i64 @llvm.hexagon.M2.vrcmpyi.s0(i64 %a, i64 %b)
   ret i64 %z
 }
-; CHECK: r1:0 = vrcmpyi(r1:0, r3:2)
+; CHECK: = vrcmpyi({{.*}}, {{.*}})
 
 declare i64 @llvm.hexagon.M2.vrcmpyr.s0(i64, i64)
 define i64 @M2_vrcmpyr_s0(i64 %a, i64 %b) {
   %z = call i64 @llvm.hexagon.M2.vrcmpyr.s0(i64 %a, i64 %b)
   ret i64 %z
 }
-; CHECK: r1:0 = vrcmpyr(r1:0, r3:2)
+; CHECK: = vrcmpyr({{.*}}, {{.*}})
 
 declare i64 @llvm.hexagon.M2.vrcmpyi.s0c(i64, i64)
 define i64 @M2_vrcmpyi_s0c(i64 %a, i64 %b) {
   %z = call i64 @llvm.hexagon.M2.vrcmpyi.s0c(i64 %a, i64 %b)
   ret i64 %z
 }
-; CHECK: r1:0 = vrcmpyi(r1:0, r3:2*)
+; CHECK: = vrcmpyi({{.*}}, {{.*}}*)
 
 declare i64 @llvm.hexagon.M2.vrcmpyr.s0c(i64, i64)
 define i64 @M2_vrcmpyr_s0c(i64 %a, i64 %b) {
   %z = call i64 @llvm.hexagon.M2.vrcmpyr.s0c(i64 %a, i64 %b)
   ret i64 %z
 }
-; CHECK: r1:0 = vrcmpyr(r1:0, r3:2*)
+; CHECK: = vrcmpyr({{.*}}, {{.*}}*)
 
 declare i64 @llvm.hexagon.M2.vrcmaci.s0(i64, i64, i64)
 define i64 @M2_vrcmaci_s0(i64 %a, i64 %b, i64 %c) {
   %z = call i64 @llvm.hexagon.M2.vrcmaci.s0(i64 %a, i64 %b, i64 %c)
   ret i64 %z
 }
-; CHECK: r1:0 += vrcmpyi(r3:2, r5:4)
+; CHECK: += vrcmpyi({{.*}}, r5:4)
 
 declare i64 @llvm.hexagon.M2.vrcmacr.s0(i64, i64, i64)
 define i64 @M2_vrcmacr_s0(i64 %a, i64 %b, i64 %c) {
   %z = call i64 @llvm.hexagon.M2.vrcmacr.s0(i64 %a, i64 %b, i64 %c)
   ret i64 %z
 }
-; CHECK: r1:0 += vrcmpyr(r3:2, r5:4)
+; CHECK: += vrcmpyr({{.*}}, r5:4)
 
 declare i64 @llvm.hexagon.M2.vrcmaci.s0c(i64, i64, i64)
 define i64 @M2_vrcmaci_s0c(i64 %a, i64 %b, i64 %c) {
   %z = call i64 @llvm.hexagon.M2.vrcmaci.s0c(i64 %a, i64 %b, i64 %c)
   ret i64 %z
 }
-; CHECK: r1:0 += vrcmpyi(r3:2, r5:4*)
+; CHECK: += vrcmpyi({{.*}}, r5:4*)
 
 declare i64 @llvm.hexagon.M2.vrcmacr.s0c(i64, i64, i64)
 define i64 @M2_vrcmacr_s0c(i64 %a, i64 %b, i64 %c) {
   %z = call i64 @llvm.hexagon.M2.vrcmacr.s0c(i64 %a, i64 %b, i64 %c)
   ret i64 %z
 }
-; CHECK: r1:0 += vrcmpyr(r3:2, r5:4*)
+; CHECK: += vrcmpyr({{.*}}, r5:4*)
 
 ; Vector reduce complex rotate
 declare i64 @llvm.hexagon.S4.vrcrotate(i64, i32, i32)
@@ -339,11 +342,11 @@ define i64 @S4_vrcrotate(i64 %a, i32 %b) {
   %z = call i64 @llvm.hexagon.S4.vrcrotate(i64 %a, i32 %b, i32 0)
   ret i64 %z
 }
-; CHECK: r1:0 = vrcrotate(r1:0, r2, #0)
+; CHECK: = vrcrotate({{.*}}, {{.*}}, #0)
 
 declare i64 @llvm.hexagon.S4.vrcrotate.acc(i64, i64, i32, i32)
 define i64 @S4_vrcrotate_acc(i64 %a, i64 %b, i32 %c) {
   %z = call i64 @llvm.hexagon.S4.vrcrotate.acc(i64 %a, i64 %b, i32 %c, i32 0)
   ret i64 %z
 }
-; CHECK: r1:0 += vrcrotate(r3:2, r4, #0)
+; CHECK: += vrcrotate({{.*}}, {{.*}}, #0)
diff --git a/test/CodeGen/Hexagon/intrinsics/xtype_fp.ll b/test/CodeGen/Hexagon/intrinsics/xtype_fp.ll
index aef8127d668..598d0a83206 100644
--- a/test/CodeGen/Hexagon/intrinsics/xtype_fp.ll
+++ b/test/CodeGen/Hexagon/intrinsics/xtype_fp.ll
@@ -1,13 +1,17 @@
 ; RUN: llc -march=hexagon -mcpu=hexagonv5 -O0 < %s | FileCheck %s
+; RUN: llc -march=hexagon -mcpu=hexagonv5 -O0 < %s | \
+; RUN: FileCheck -check-prefix=CHECK-CALL %s
 ; Hexagon Programmer's Reference Manual 11.10.4 XTYPE/FP
 
+; CHECK-CALL-NOT: call
+
 ; Floating point addition
 declare float @llvm.hexagon.F2.sfadd(float, float)
 define float @F2_sfadd(float %a, float %b) {
   %z = call float @llvm.hexagon.F2.sfadd(float %a, float %b)
   ret float %z
 }
-; CHECK: r0 = sfadd(r0, r1)
+; CHECK: = sfadd({{.*}}, {{.*}})
 
 ; Classify floating-point value
 declare i32 @llvm.hexagon.F2.sfclass(float, i32)
@@ -15,14 +19,14 @@ define i32 @F2_sfclass(float %a) {
   %z = call i32 @llvm.hexagon.F2.sfclass(float %a, i32 0)
   ret i32 %z
 }
-; CHECK: p0 = sfclass(r0, #0)
+; CHECK: = sfclass({{.*}}, #0)
 
 declare i32 @llvm.hexagon.F2.dfclass(double, i32)
 define i32 @F2_dfclass(double %a) {
   %z = call i32 @llvm.hexagon.F2.dfclass(double %a, i32 0)
   ret i32 %z
 }
-; CHECK: p0 = dfclass(r1:0, #0)
+; CHECK: = dfclass({{.*}}, #0)
 
 ; Compare floating-point value
 declare i32 @llvm.hexagon.F2.sfcmpge(float, float)
@@ -30,56 +34,56 @@ define i32 @F2_sfcmpge(float %a, float %b) {
   %z = call i32 @llvm.hexagon.F2.sfcmpge(float %a, float %b)
   ret i32 %z
 }
-; CHECK: p0 = sfcmp.ge(r0, r1)
+; CHECK: = sfcmp.ge({{.*}}, {{.*}})
 
 declare i32 @llvm.hexagon.F2.sfcmpuo(float, float)
 define i32 @F2_sfcmpuo(float %a, float %b) {
   %z = call i32 @llvm.hexagon.F2.sfcmpuo(float %a, float %b)
   ret i32 %z
 }
-; CHECK: p0 = sfcmp.uo(r0, r1)
+; CHECK: = sfcmp.uo({{.*}}, {{.*}})
 
 declare i32 @llvm.hexagon.F2.sfcmpeq(float, float)
 define i32 @F2_sfcmpeq(float %a, float %b) {
   %z = call i32 @llvm.hexagon.F2.sfcmpeq(float %a, float %b)
   ret i32 %z
 }
-; CHECK: p0 = sfcmp.eq(r0, r1)
+; CHECK: = sfcmp.eq({{.*}}, {{.*}})
 
 declare i32 @llvm.hexagon.F2.sfcmpgt(float, float)
 define i32 @F2_sfcmpgt(float %a, float %b) {
   %z = call i32 @llvm.hexagon.F2.sfcmpgt(float %a, float %b)
   ret i32 %z
 }
-; CHECK: p0 = sfcmp.gt(r0, r1)
+; CHECK: = sfcmp.gt({{.*}}, {{.*}})
 
 declare i32 @llvm.hexagon.F2.dfcmpge(double, double)
 define i32 @F2_dfcmpge(double %a, double %b) {
   %z = call i32 @llvm.hexagon.F2.dfcmpge(double %a, double %b)
   ret i32 %z
 }
-; CHECK: p0 = dfcmp.ge(r1:0, r3:2)
+; CHECK: = dfcmp.ge({{.*}}, {{.*}})
 
 declare i32 @llvm.hexagon.F2.dfcmpuo(double, double)
 define i32 @F2_dfcmpuo(double %a, double %b) {
   %z = call i32 @llvm.hexagon.F2.dfcmpuo(double %a, double %b)
   ret i32 %z
 }
-; CHECK: p0 = dfcmp.uo(r1:0, r3:2)
+; CHECK: = dfcmp.uo({{.*}}, {{.*}})
 
 declare i32 @llvm.hexagon.F2.dfcmpeq(double, double)
 define i32 @F2_dfcmpeq(double %a, double %b) {
   %z = call i32 @llvm.hexagon.F2.dfcmpeq(double %a, double %b)
   ret i32 %z
 }
-; CHECK: p0 = dfcmp.eq(r1:0, r3:2)
+; CHECK: = dfcmp.eq({{.*}}, {{.*}})
 
 declare i32 @llvm.hexagon.F2.dfcmpgt(double, double)
 define i32 @F2_dfcmpgt(double %a, double %b) {
   %z = call i32 @llvm.hexagon.F2.dfcmpgt(double %a, double %b)
   ret i32 %z
 }
-; CHECK: p0 = dfcmp.gt(r1:0, r3:2)
+; CHECK: = dfcmp.gt({{.*}}, {{.*}})
 
 ; Convert floating-point value to other format
 declare double @llvm.hexagon.F2.conv.sf2df(float)
@@ -87,14 +91,14 @@ define double @F2_conv_sf2df(float %a) {
   %z = call double @llvm.hexagon.F2.conv.sf2df(float %a)
   ret double %z
 }
-; CHECK:  = convert_sf2df(r0)
+; CHECK: = convert_sf2df({{.*}})
 
 declare float @llvm.hexagon.F2.conv.df2sf(double)
 define float @F2_conv_df2sf(double %a) {
   %z = call float @llvm.hexagon.F2.conv.df2sf(double %a)
   ret float %z
 }
-; CHECK: r0 = convert_df2sf(r1:0)
+; CHECK: = convert_df2sf({{.*}})
 
 ; Convert integer to floating-point value
 declare double @llvm.hexagon.F2.conv.ud2df(i64)
@@ -102,56 +106,56 @@ define double @F2_conv_ud2df(i64 %a) {
   %z = call double @llvm.hexagon.F2.conv.ud2df(i64 %a)
   ret double %z
 }
-; CHECK: r1:0 = convert_ud2df(r1:0)
+; CHECK: = convert_ud2df({{.*}})
 
 declare double @llvm.hexagon.F2.conv.d2df(i64)
 define double @F2_conv_d2df(i64 %a) {
   %z = call double @llvm.hexagon.F2.conv.d2df(i64 %a)
   ret double %z
 }
-; CHECK: r1:0 = convert_d2df(r1:0)
+; CHECK: = convert_d2df({{.*}})
 
 declare double @llvm.hexagon.F2.conv.uw2df(i32)
 define double @F2_conv_uw2df(i32 %a) {
   %z = call double @llvm.hexagon.F2.conv.uw2df(i32 %a)
   ret double %z
 }
-; CHECK:  = convert_uw2df(r0)
+; CHECK: = convert_uw2df({{.*}})
 
 declare double @llvm.hexagon.F2.conv.w2df(i32)
 define double @F2_conv_w2df(i32 %a) {
   %z = call double @llvm.hexagon.F2.conv.w2df(i32 %a)
   ret double %z
 }
-; CHECK:  = convert_w2df(r0)
+; CHECK: = convert_w2df({{.*}})
 
 declare float @llvm.hexagon.F2.conv.ud2sf(i64)
 define float @F2_conv_ud2sf(i64 %a) {
   %z = call float @llvm.hexagon.F2.conv.ud2sf(i64 %a)
   ret float %z
 }
-; CHECK: r0 = convert_ud2sf(r1:0)
+; CHECK: = convert_ud2sf({{.*}})
 
 declare float @llvm.hexagon.F2.conv.d2sf(i64)
 define float @F2_conv_d2sf(i64 %a) {
   %z = call float @llvm.hexagon.F2.conv.d2sf(i64 %a)
   ret float %z
 }
-; CHECK: r0 = convert_d2sf(r1:0)
+; CHECK: = convert_d2sf({{.*}})
 
 declare float @llvm.hexagon.F2.conv.uw2sf(i32)
 define float @F2_conv_uw2sf(i32 %a) {
   %z = call float @llvm.hexagon.F2.conv.uw2sf(i32 %a)
   ret float %z
 }
-; CHECK: r0 = convert_uw2sf(r0)
+; CHECK: = convert_uw2sf({{.*}})
 
 declare float @llvm.hexagon.F2.conv.w2sf(i32)
 define float @F2_conv_w2sf(i32 %a) {
   %z = call float @llvm.hexagon.F2.conv.w2sf(i32 %a)
   ret float %z
 }
-; CHECK: r0 = convert_w2sf(r0)
+; CHECK: = convert_w2sf({{.*}})
 
 ; Convert floating-point value to integer
 declare i64 @llvm.hexagon.F2.conv.df2d(double)
@@ -159,112 +163,112 @@ define i64 @F2_conv_df2d(double %a) {
   %z = call i64 @llvm.hexagon.F2.conv.df2d(double %a)
   ret i64 %z
 }
-; CHECK: r1:0 = convert_df2d(r1:0)
+; CHECK: = convert_df2d({{.*}})
 
 declare i64 @llvm.hexagon.F2.conv.df2ud(double)
 define i64 @F2_conv_df2ud(double %a) {
   %z = call i64 @llvm.hexagon.F2.conv.df2ud(double %a)
   ret i64 %z
 }
-; CHECK: r1:0 = convert_df2ud(r1:0)
+; CHECK: {{.*}} = convert_df2ud({{.*}})
 
 declare i64 @llvm.hexagon.F2.conv.df2d.chop(double)
 define i64 @F2_conv_df2d_chop(double %a) {
   %z = call i64 @llvm.hexagon.F2.conv.df2d.chop(double %a)
   ret i64 %z
 }
-; CHECK: r1:0 = convert_df2d(r1:0):chop
+; CHECK: = convert_df2d({{.*}}):chop
 
 declare i64 @llvm.hexagon.F2.conv.df2ud.chop(double)
 define i64 @F2_conv_df2ud_chop(double %a) {
   %z = call i64 @llvm.hexagon.F2.conv.df2ud.chop(double %a)
   ret i64 %z
 }
-; CHECK: r1:0 = convert_df2ud(r1:0):chop
+; CHECK: = convert_df2ud({{.*}}):chop
 
 declare i64 @llvm.hexagon.F2.conv.sf2ud(float)
 define i64 @F2_conv_sf2ud(float %a) {
   %z = call i64 @llvm.hexagon.F2.conv.sf2ud(float %a)
   ret i64 %z
 }
-; CHECK:  = convert_sf2ud(r0)
+; CHECK: = convert_sf2ud({{.*}})
 
 declare i64 @llvm.hexagon.F2.conv.sf2d(float)
 define i64 @F2_conv_sf2d(float %a) {
   %z = call i64 @llvm.hexagon.F2.conv.sf2d(float %a)
   ret i64 %z
 }
-; CHECK:  = convert_sf2d(r0)
+; CHECK: = convert_sf2d({{.*}})
 
 declare i64 @llvm.hexagon.F2.conv.sf2d.chop(float)
 define i64 @F2_conv_sf2d_chop(float %a) {
   %z = call i64 @llvm.hexagon.F2.conv.sf2d.chop(float %a)
   ret i64 %z
 }
-; CHECK:  = convert_sf2d(r0):chop
+; CHECK: = convert_sf2d({{.*}}):chop
 
 declare i64 @llvm.hexagon.F2.conv.sf2ud.chop(float)
 define i64 @F2_conv_sf2ud_chop(float %a) {
   %z = call i64 @llvm.hexagon.F2.conv.sf2ud.chop(float %a)
   ret i64 %z
 }
-; CHECK:  = convert_sf2ud(r0):chop
+; CHECK: = convert_sf2ud({{.*}}):chop
 
 declare i32 @llvm.hexagon.F2.conv.df2uw(double)
 define i32 @F2_conv_df2uw(double %a) {
   %z = call i32 @llvm.hexagon.F2.conv.df2uw(double %a)
   ret i32 %z
 }
-; CHECK: r0 = convert_df2uw(r1:0)
+; CHECK: = convert_df2uw({{.*}})
 
 declare i32 @llvm.hexagon.F2.conv.df2w(double)
 define i32 @F2_conv_df2w(double %a) {
   %z = call i32 @llvm.hexagon.F2.conv.df2w(double %a)
   ret i32 %z
 }
-; CHECK: r0 = convert_df2w(r1:0)
+; CHECK: = convert_df2w({{.*}})
 
 declare i32 @llvm.hexagon.F2.conv.df2w.chop(double)
 define i32 @F2_conv_df2w_chop(double %a) {
   %z = call i32 @llvm.hexagon.F2.conv.df2w.chop(double %a)
   ret i32 %z
 }
-; CHECK: r0 = convert_df2w(r1:0):chop
+; CHECK: = convert_df2w({{.*}}):chop
 
 declare i32 @llvm.hexagon.F2.conv.df2uw.chop(double)
 define i32 @F2_conv_df2uw_chop(double %a) {
   %z = call i32 @llvm.hexagon.F2.conv.df2uw.chop(double %a)
   ret i32 %z
 }
-; CHECK: r0 = convert_df2uw(r1:0):chop
+; CHECK: = convert_df2uw({{.*}}):chop
 
 declare i32 @llvm.hexagon.F2.conv.sf2uw(float)
 define i32 @F2_conv_sf2uw(float %a) {
   %z = call i32 @llvm.hexagon.F2.conv.sf2uw(float %a)
   ret i32 %z
 }
-; CHECK: r0 = convert_sf2uw(r0)
+; CHECK: = convert_sf2uw({{.*}})
 
 declare i32 @llvm.hexagon.F2.conv.sf2uw.chop(float)
 define i32 @F2_conv_sf2uw_chop(float %a) {
   %z = call i32 @llvm.hexagon.F2.conv.sf2uw.chop(float %a)
   ret i32 %z
 }
-; CHECK: r0 = convert_sf2uw(r0):chop
+; CHECK: = convert_sf2uw({{.*}}):chop
 
 declare i32 @llvm.hexagon.F2.conv.sf2w(float)
 define i32 @F2_conv_sf2w(float %a) {
   %z = call i32 @llvm.hexagon.F2.conv.sf2w(float %a)
   ret i32 %z
 }
-; CHECK: r0 = convert_sf2w(r0)
+; CHECK: = convert_sf2w({{.*}})
 
 declare i32 @llvm.hexagon.F2.conv.sf2w.chop(float)
 define i32 @F2_conv_sf2w_chop(float %a) {
   %z = call i32 @llvm.hexagon.F2.conv.sf2w.chop(float %a)
   ret i32 %z
 }
-; CHECK: r0 = convert_sf2w(r0):chop
+; CHECK: = convert_sf2w({{.*}}):chop
 
 ; Floating point extreme value assistance
 declare float @llvm.hexagon.F2.sffixupr(float)
@@ -272,21 +276,21 @@ define float @F2_sffixupr(float %a) {
   %z = call float @llvm.hexagon.F2.sffixupr(float %a)
   ret float %z
 }
-; CHECK: r0 = sffixupr(r0)
+; CHECK: = sffixupr({{.*}})
 
 declare float @llvm.hexagon.F2.sffixupn(float, float)
 define float @F2_sffixupn(float %a, float %b) {
   %z = call float @llvm.hexagon.F2.sffixupn(float %a, float %b)
   ret float %z
 }
-; CHECK: r0 = sffixupn(r0, r1)
+; CHECK: = sffixupn({{.*}}, {{.*}})
 
 declare float @llvm.hexagon.F2.sffixupd(float, float)
 define float @F2_sffixupd(float %a, float %b) {
   %z = call float @llvm.hexagon.F2.sffixupd(float %a, float %b)
   ret float %z
 }
-; CHECK: r0 = sffixupd(r0, r1)
+; CHECK: = sffixupd({{.*}}, {{.*}})
 
 ; Floating point fused multiply-add
 declare float @llvm.hexagon.F2.sffma(float, float, float)
@@ -294,14 +298,14 @@ define float @F2_sffma(float %a, float %b, float %c) {
   %z = call float @llvm.hexagon.F2.sffma(float %a, float %b, float %c)
   ret float %z
 }
-; CHECK: r0 += sfmpy(r1, r2)
+; CHECK: += sfmpy({{.*}}, {{.*}})
 
 declare float @llvm.hexagon.F2.sffms(float, float, float)
 define float @F2_sffms(float %a, float %b, float %c) {
   %z = call float @llvm.hexagon.F2.sffms(float %a, float %b, float %c)
   ret float %z
 }
-; CHECK: r0 -= sfmpy(r1, r2)
+; CHECK: -= sfmpy({{.*}}, {{.*}})
 
 ; Floating point fused multiply-add with scaling
 declare float @llvm.hexagon.F2.sffma.sc(float, float, float, i32)
@@ -309,7 +313,7 @@ define float @F2_sffma_sc(float %a, float %b, float %c, i32 %d) {
   %z = call float @llvm.hexagon.F2.sffma.sc(float %a, float %b, float %c, i32 %d)
   ret float %z
 }
-; CHECK: r0 += sfmpy(r1, r2, p0):scale
+; CHECK: += sfmpy({{.*}}, {{.*}}, {{.*}}):scale
 
 ; Floating point fused multiply-add for library routines
 declare float @llvm.hexagon.F2.sffma.lib(float, float, float)
@@ -317,14 +321,14 @@ define float @F2_sffma_lib(float %a, float %b, float %c) {
   %z = call float @llvm.hexagon.F2.sffma.lib(float %a, float %b, float %c)
   ret float %z
 }
-; CHECK: r0 += sfmpy(r1, r2):lib
+; CHECK: += sfmpy({{.*}}, {{.*}}):lib
 
 declare float @llvm.hexagon.F2.sffms.lib(float, float, float)
 define float @F2_sffms_lib(float %a, float %b, float %c) {
   %z = call float @llvm.hexagon.F2.sffms.lib(float %a, float %b, float %c)
   ret float %z
 }
-; CHECK: r0 -= sfmpy(r1, r2):lib
+; CHECK: -= sfmpy({{.*}}, {{.*}}):lib
 
 ; Create floating-point constant
 declare float @llvm.hexagon.F2.sfimm.p(i32)
@@ -332,28 +336,28 @@ define float @F2_sfimm_p() {
   %z = call float @llvm.hexagon.F2.sfimm.p(i32 0)
   ret float %z
 }
-; CHECK: r0 = sfmake(#0):pos
+; CHECK: = sfmake(#0):pos
 
 declare float @llvm.hexagon.F2.sfimm.n(i32)
 define float @F2_sfimm_n() {
   %z = call float @llvm.hexagon.F2.sfimm.n(i32 0)
   ret float %z
 } 
-; CHECK: r0 = sfmake(#0):neg
+; CHECK: = sfmake(#0):neg
 
 declare double @llvm.hexagon.F2.dfimm.p(i32)
 define double @F2_dfimm_p() {
   %z = call double @llvm.hexagon.F2.dfimm.p(i32 0)
   ret double %z
 }
-; CHECK: r1:0 = dfmake(#0):pos
+; CHECK: = dfmake(#0):pos
 
 declare double @llvm.hexagon.F2.dfimm.n(i32)
 define double @F2_dfimm_n() {
   %z = call double @llvm.hexagon.F2.dfimm.n(i32 0)
   ret double %z
 }
-; CHECK: r1:0 = dfmake(#0):neg
+; CHECK: = dfmake(#0):neg
 
 ; Floating point maximum
 declare float @llvm.hexagon.F2.sfmax(float, float)
@@ -361,7 +365,7 @@ define float @F2_sfmax(float %a, float %b) {
   %z = call float @llvm.hexagon.F2.sfmax(float %a, float %b)
   ret float %z
 }
-; CHECK: r0 = sfmax(r0, r1)
+; CHECK: = sfmax({{.*}}, {{.*}})
 
 ; Floating point minimum
 declare float @llvm.hexagon.F2.sfmin(float, float)
@@ -369,7 +373,7 @@ define float @F2_sfmin(float %a, float %b) {
   %z = call float @llvm.hexagon.F2.sfmin(float %a, float %b)
   ret float %z
 }
-; CHECK: r0 = sfmin(r0, r1)
+; CHECK: = sfmin({{.*}}, {{.*}})
 
 ; Floating point multiply
 declare float @llvm.hexagon.F2.sfmpy(float, float)
@@ -377,7 +381,7 @@ define float @F2_sfmpy(float %a, float %b) {
   %z = call float @llvm.hexagon.F2.sfmpy(float %a, float %b)
   ret float %z
 }
-; CHECK: r0 = sfmpy(r0, r1)
+; CHECK: = sfmpy({{.*}}, {{.*}})
 
 ; Floating point subtraction
 declare float @llvm.hexagon.F2.sfsub(float, float)
@@ -385,4 +389,4 @@ define float @F2_sfsub(float %a, float %b) {
   %z = call float @llvm.hexagon.F2.sfsub(float %a, float %b)
   ret float %z
 }
-; CHECK: r0 = sfsub(r0, r1)
+; CHECK: = sfsub({{.*}}, {{.*}})
diff --git a/test/CodeGen/Hexagon/intrinsics/xtype_mpy.ll b/test/CodeGen/Hexagon/intrinsics/xtype_mpy.ll
index 6409e4e10ca..a1490499fbf 100644
--- a/test/CodeGen/Hexagon/intrinsics/xtype_mpy.ll
+++ b/test/CodeGen/Hexagon/intrinsics/xtype_mpy.ll
@@ -1,41 +1,45 @@
 ; RUN: llc -march=hexagon -mcpu=hexagonv5 -O0 < %s | FileCheck %s
+; RUN: llc -march=hexagon -mcpu=hexagonv5 -O0 < %s | \
+; RUN: FileCheck -check-prefix=CHECK-CALL %s
 ; Hexagon Programmer's Reference Manual 11.10.5 XTYPE/MPY
 
+; CHECK-CALL-NOT: call
+
 ; Multiply and use lower result
 declare i32 @llvm.hexagon.M4.mpyrr.addi(i32, i32, i32)
 define i32 @M4_mpyrr_addi(i32 %a, i32 %b) {
   %z = call i32 @llvm.hexagon.M4.mpyrr.addi(i32 0, i32 %a, i32 %b)
   ret i32 %z
 }
-; CHECK: r0 = add(#0, mpyi(r0, r1))
+; CHECK: = add(#0, mpyi({{.*}}, {{.*}}))
 
 declare i32 @llvm.hexagon.M4.mpyri.addi(i32, i32, i32)
 define i32 @M4_mpyri_addi(i32 %a) {
   %z = call i32 @llvm.hexagon.M4.mpyri.addi(i32 0, i32 %a, i32 0)
   ret i32 %z
 }
-; CHECK: r0 = add(#0, mpyi(r0, #0))
+; CHECK: = add(#0, mpyi({{.*}}, #0))
 
 declare i32 @llvm.hexagon.M4.mpyri.addr.u2(i32, i32, i32)
 define i32 @M4_mpyri_addr_u2(i32 %a, i32 %b) {
   %z = call i32 @llvm.hexagon.M4.mpyri.addr.u2(i32 %a, i32 0, i32 %b)
   ret i32 %z
 }
-; CHECK: r0 = add(r0, mpyi(#0, r1))
+; CHECK: = add({{.*}}, mpyi(#0, {{.*}}))
 
 declare i32 @llvm.hexagon.M4.mpyri.addr(i32, i32, i32)
 define i32 @M4_mpyri_addr(i32 %a, i32 %b) {
   %z = call i32 @llvm.hexagon.M4.mpyri.addr(i32 %a, i32 %b, i32 0)
   ret i32 %z
 }
-; CHECK: r0 = add(r0, mpyi(r1, #0))
+; CHECK: = add({{.*}}, mpyi({{.*}}, #0))
 
 declare i32 @llvm.hexagon.M4.mpyrr.addr(i32, i32, i32)
 define i32 @M4_mpyrr_addr(i32 %a, i32 %b, i32 %c) {
   %z = call i32 @llvm.hexagon.M4.mpyrr.addr(i32 %a, i32 %b, i32 %c)
   ret i32 %z
 }
-; CHECK: r1 = add(r0, mpyi(r1, r2))
+; CHECK: = add({{.*}}, mpyi({{.*}}, {{.*}}))
 
 ; Vector multiply word by signed half (32x16)
 declare i64 @llvm.hexagon.M2.mmpyl.s0(i64, i64)
@@ -43,56 +47,56 @@ define i64 @M2_mmpyl_s0(i64 %a, i64 %b) {
   %z = call i64 @llvm.hexagon.M2.mmpyl.s0(i64 %a, i64 %b)
   ret i64 %z
 }
-; CHECK: r1:0 = vmpyweh(r1:0, r3:2):sat
+; CHECK: = vmpyweh({{.*}}, {{.*}}):sat
 
 declare i64 @llvm.hexagon.M2.mmpyl.s1(i64, i64)
 define i64 @M2_mmpyl_s1(i64 %a, i64 %b) {
   %z = call i64 @llvm.hexagon.M2.mmpyl.s1(i64 %a, i64 %b)
   ret i64 %z
 }
-; CHECK: r1:0 = vmpyweh(r1:0, r3:2):<<1:sat
+; CHECK: = vmpyweh({{.*}}, {{.*}}):<<1:sat
 
 declare i64 @llvm.hexagon.M2.mmpyh.s0(i64, i64)
 define i64 @M2_mmpyh_s0(i64 %a, i64 %b) {
   %z = call i64 @llvm.hexagon.M2.mmpyh.s0(i64 %a, i64 %b)
   ret i64 %z
 }
-; CHECK: r1:0 = vmpywoh(r1:0, r3:2):sat
+; CHECK: = vmpywoh({{.*}}, {{.*}}):sat
 
 declare i64 @llvm.hexagon.M2.mmpyh.s1(i64, i64)
 define i64 @M2_mmpyh_s1(i64 %a, i64 %b) {
   %z = call i64 @llvm.hexagon.M2.mmpyh.s1(i64 %a, i64 %b)
   ret i64 %z
 }
-; CHECK: r1:0 = vmpywoh(r1:0, r3:2):<<1:sat
+; CHECK: = vmpywoh({{.*}}, {{.*}}):<<1:sat
 
 declare i64 @llvm.hexagon.M2.mmpyl.rs0(i64, i64)
 define i64 @M2_mmpyl_rs0(i64 %a, i64 %b) {
   %z = call i64 @llvm.hexagon.M2.mmpyl.rs0(i64 %a, i64 %b)
   ret i64 %z
 }
-; CHECK: r1:0 = vmpyweh(r1:0, r3:2):rnd:sat
+; CHECK: = vmpyweh({{.*}}, {{.*}}):rnd:sat
 
 declare i64 @llvm.hexagon.M2.mmpyl.rs1(i64, i64)
 define i64 @M2_mmpyl_rs1(i64 %a, i64 %b) {
   %z = call i64 @llvm.hexagon.M2.mmpyl.rs1(i64 %a, i64 %b)
   ret i64 %z
 }
-; CHECK: r1:0 = vmpyweh(r1:0, r3:2):<<1:rnd:sat
+; CHECK: = vmpyweh({{.*}}, {{.*}}):<<1:rnd:sat
 
 declare i64 @llvm.hexagon.M2.mmpyh.rs0(i64, i64)
 define i64 @M2_mmpyh_rs0(i64 %a, i64 %b) {
   %z = call i64 @llvm.hexagon.M2.mmpyh.rs0(i64 %a, i64 %b)
   ret i64 %z
 }
-; CHECK: r1:0 = vmpywoh(r1:0, r3:2):rnd:sat
+; CHECK: = vmpywoh({{.*}}, {{.*}}):rnd:sat
 
 declare i64 @llvm.hexagon.M2.mmpyh.rs1(i64, i64)
 define i64 @M2_mmpyh_rs1(i64 %a, i64 %b) {
   %z = call i64 @llvm.hexagon.M2.mmpyh.rs1(i64 %a, i64 %b)
   ret i64 %z
 }
-; CHECK: r1:0 = vmpywoh(r1:0, r3:2):<<1:rnd:sat
+; CHECK: = vmpywoh({{.*}}, {{.*}}):<<1:rnd:sat
 
 ; Vector multiply word by unsigned half (32x16)
 declare i64 @llvm.hexagon.M2.mmpyul.s0(i64, i64)
@@ -100,56 +104,56 @@ define i64 @M2_mmpyul_s0(i64 %a, i64 %b) {
   %z = call i64 @llvm.hexagon.M2.mmpyul.s0(i64 %a, i64 %b)
   ret i64 %z
 }
-; CHECK: r1:0 = vmpyweuh(r1:0, r3:2):sat
+; CHECK: = vmpyweuh({{.*}}, {{.*}}):sat
 
 declare i64 @llvm.hexagon.M2.mmpyul.s1(i64, i64)
 define i64 @M2_mmpyul_s1(i64 %a, i64 %b) {
   %z = call i64 @llvm.hexagon.M2.mmpyul.s1(i64 %a, i64 %b)
   ret i64 %z
 }
-; CHECK: r1:0 = vmpyweuh(r1:0, r3:2):<<1:sat
+; CHECK: = vmpyweuh({{.*}}, {{.*}}):<<1:sat
 
 declare i64 @llvm.hexagon.M2.mmpyuh.s0(i64, i64)
 define i64 @M2_mmpyuh_s0(i64 %a, i64 %b) {
   %z = call i64 @llvm.hexagon.M2.mmpyuh.s0(i64 %a, i64 %b)
   ret i64 %z
 }
-; CHECK: r1:0 = vmpywouh(r1:0, r3:2):sat
+; CHECK: = vmpywouh({{.*}}, {{.*}}):sat
 
 declare i64 @llvm.hexagon.M2.mmpyuh.s1(i64, i64)
 define i64 @M2_mmpyuh_s1(i64 %a, i64 %b) {
   %z = call i64 @llvm.hexagon.M2.mmpyuh.s1(i64 %a, i64 %b)
   ret i64 %z
 }
-; CHECK: r1:0 = vmpywouh(r1:0, r3:2):<<1:sat
+; CHECK: = vmpywouh({{.*}}, {{.*}}):<<1:sat
 
 declare i64 @llvm.hexagon.M2.mmpyul.rs0(i64, i64)
 define i64 @M2_mmpyul_rs0(i64 %a, i64 %b) {
   %z = call i64 @llvm.hexagon.M2.mmpyul.rs0(i64 %a, i64 %b)
   ret i64 %z
 }
-; CHECK: r1:0 = vmpyweuh(r1:0, r3:2):rnd:sat
+; CHECK: = vmpyweuh({{.*}}, {{.*}}):rnd:sat
 
 declare i64 @llvm.hexagon.M2.mmpyul.rs1(i64, i64)
 define i64 @M2_mmpyul_rs1(i64 %a, i64 %b) {
   %z = call i64 @llvm.hexagon.M2.mmpyul.rs1(i64 %a, i64 %b)
   ret i64 %z
 }
-; CHECK: r1:0 = vmpyweuh(r1:0, r3:2):<<1:rnd:sat
+; CHECK: = vmpyweuh({{.*}}, {{.*}}):<<1:rnd:sat
 
 declare i64 @llvm.hexagon.M2.mmpyuh.rs0(i64, i64)
 define i64 @M2_mmpyuh_rs0(i64 %a, i64 %b) {
   %z = call i64 @llvm.hexagon.M2.mmpyuh.rs0(i64 %a, i64 %b)
   ret i64 %z
 }
-; CHECK: r1:0 = vmpywouh(r1:0, r3:2):rnd:sat
+; CHECK: = vmpywouh({{.*}}, {{.*}}):rnd:sat
 
 declare i64 @llvm.hexagon.M2.mmpyuh.rs1(i64, i64)
 define i64 @M2_mmpyuh_rs1(i64 %a, i64 %b) {
   %z = call i64 @llvm.hexagon.M2.mmpyuh.rs1(i64 %a, i64 %b)
   ret i64 %z
 }
-; CHECK: r1:0 = vmpywouh(r1:0, r3:2):<<1:rnd:sat
+; CHECK: = vmpywouh({{.*}}, {{.*}}):<<1:rnd:sat
 
 ; Multiply signed halfwords
 declare i64 @llvm.hexagon.M2.mpyd.ll.s0(i32, i32)
@@ -157,616 +161,616 @@ define i64 @M2_mpyd_ll_s0(i32 %a, i32 %b) {
   %z = call i64 @llvm.hexagon.M2.mpyd.ll.s0(i32 %a, i32 %b)
   ret i64 %z
 }
-; CHECK: r1:0 = mpy(r0.l, r1.l)
+; CHECK: = mpy({{.*}}.l, {{.*}}.l)
 
 declare i64 @llvm.hexagon.M2.mpyd.ll.s1(i32, i32)
 define i64 @M2_mpyd_ll_s1(i32 %a, i32 %b) {
   %z = call i64 @llvm.hexagon.M2.mpyd.ll.s1(i32 %a, i32 %b)
   ret i64 %z
 }
-; CHECK: r1:0 = mpy(r0.l, r1.l):<<1
+; CHECK: = mpy({{.*}}.l, {{.*}}.l):<<1
 
 declare i64 @llvm.hexagon.M2.mpyd.lh.s0(i32, i32)
 define i64 @M2_mpyd_lh_s0(i32 %a, i32 %b) {
   %z = call i64 @llvm.hexagon.M2.mpyd.lh.s0(i32 %a, i32 %b)
   ret i64 %z
 }
-; CHECK: r1:0 = mpy(r0.l, r1.h)
+; CHECK: = mpy({{.*}}.l, {{.*}}.h)
 
 declare i64 @llvm.hexagon.M2.mpyd.lh.s1(i32, i32)
 define i64 @M2_mpyd_lh_s1(i32 %a, i32 %b) {
   %z = call i64 @llvm.hexagon.M2.mpyd.lh.s1(i32 %a, i32 %b)
   ret i64 %z
 }
-; CHECK: r1:0 = mpy(r0.l, r1.h):<<1
+; CHECK: = mpy({{.*}}.l, {{.*}}.h):<<1
 
 declare i64 @llvm.hexagon.M2.mpyd.hl.s0(i32, i32)
 define i64 @M2_mpyd_hl_s0(i32 %a, i32 %b) {
   %z = call i64 @llvm.hexagon.M2.mpyd.hl.s0(i32 %a, i32 %b)
   ret i64 %z
 }
-; CHECK: r1:0 = mpy(r0.h, r1.l)
+; CHECK: = mpy({{.*}}.h, {{.*}}.l)
 
 declare i64 @llvm.hexagon.M2.mpyd.hl.s1(i32, i32)
 define i64 @M2_mpyd_hl_s1(i32 %a, i32 %b) {
   %z = call i64 @llvm.hexagon.M2.mpyd.hl.s1(i32 %a, i32 %b)
   ret i64 %z
 }
-; CHECK: r1:0 = mpy(r0.h, r1.l):<<1
+; CHECK: = mpy({{.*}}.h, {{.*}}.l):<<1
 
 declare i64 @llvm.hexagon.M2.mpyd.hh.s0(i32, i32)
 define i64 @M2_mpyd_hh_s0(i32 %a, i32 %b) {
   %z = call i64 @llvm.hexagon.M2.mpyd.hh.s0(i32 %a, i32 %b)
   ret i64 %z
 }
-; CHECK: r1:0 = mpy(r0.h, r1.h)
+; CHECK: = mpy({{.*}}.h, {{.*}}.h)
 
 declare i64 @llvm.hexagon.M2.mpyd.hh.s1(i32, i32)
 define i64 @M2_mpyd_hh_s1(i32 %a, i32 %b) {
   %z = call i64 @llvm.hexagon.M2.mpyd.hh.s1(i32 %a, i32 %b)
   ret i64 %z
 }
-; CHECK: r1:0 = mpy(r0.h, r1.h):<<1
+; CHECK: = mpy({{.*}}.h, {{.*}}.h):<<1
 
 declare i64 @llvm.hexagon.M2.mpyd.rnd.ll.s0(i32, i32)
 define i64 @M2_mpyd_rnd_ll_s0(i32 %a, i32 %b) {
   %z = call i64 @llvm.hexagon.M2.mpyd.rnd.ll.s0(i32 %a, i32 %b)
   ret i64 %z
 }
-; CHECK: r1:0 = mpy(r0.l, r1.l):rnd
+; CHECK: = mpy({{.*}}.l, {{.*}}.l):rnd
 
 declare i64 @llvm.hexagon.M2.mpyd.rnd.ll.s1(i32, i32)
 define i64 @M2_mpyd_rnd_ll_s1(i32 %a, i32 %b) {
   %z = call i64 @llvm.hexagon.M2.mpyd.rnd.ll.s1(i32 %a, i32 %b)
   ret i64 %z
 }
-; CHECK: r1:0 = mpy(r0.l, r1.l):<<1:rnd
+; CHECK: = mpy({{.*}}.l, {{.*}}.l):<<1:rnd
 
 declare i64 @llvm.hexagon.M2.mpyd.rnd.lh.s0(i32, i32)
 define i64 @M2_mpyd_rnd_lh_s0(i32 %a, i32 %b) {
   %z = call i64 @llvm.hexagon.M2.mpyd.rnd.lh.s0(i32 %a, i32 %b)
   ret i64 %z
 }
-; CHECK: r1:0 = mpy(r0.l, r1.h):rnd
+; CHECK: = mpy({{.*}}.l, {{.*}}.h):rnd
 
 declare i64 @llvm.hexagon.M2.mpyd.rnd.lh.s1(i32, i32)
 define i64 @M2_mpyd_rnd_lh_s1(i32 %a, i32 %b) {
   %z = call i64 @llvm.hexagon.M2.mpyd.rnd.lh.s1(i32 %a, i32 %b)
   ret i64 %z
 }
-; CHECK: r1:0 = mpy(r0.l, r1.h):<<1:rnd
+; CHECK: = mpy({{.*}}.l, {{.*}}.h):<<1:rnd
 
 declare i64 @llvm.hexagon.M2.mpyd.rnd.hl.s0(i32, i32)
 define i64 @M2_mpyd_rnd_hl_s0(i32 %a, i32 %b) {
   %z = call i64 @llvm.hexagon.M2.mpyd.rnd.hl.s0(i32 %a, i32 %b)
   ret i64 %z
 }
-; CHECK: r1:0 = mpy(r0.h, r1.l):rnd
+; CHECK: = mpy({{.*}}.h, {{.*}}.l):rnd
 
 declare i64 @llvm.hexagon.M2.mpyd.rnd.hl.s1(i32, i32)
 define i64 @M2_mpyd_rnd_hl_s1(i32 %a, i32 %b) {
   %z = call i64 @llvm.hexagon.M2.mpyd.rnd.hl.s1(i32 %a, i32 %b)
   ret i64 %z
 }
-; CHECK: r1:0 = mpy(r0.h, r1.l):<<1:rnd
+; CHECK: = mpy({{.*}}.h, {{.*}}.l):<<1:rnd
 
 declare i64 @llvm.hexagon.M2.mpyd.rnd.hh.s0(i32, i32)
 define i64 @M2_mpyd_rnd_hh_s0(i32 %a, i32 %b) {
   %z = call i64 @llvm.hexagon.M2.mpyd.rnd.hh.s0(i32 %a, i32 %b)
   ret i64 %z
 }
-; CHECK: r1:0 = mpy(r0.h, r1.h):rnd
+; CHECK: = mpy({{.*}}.h, {{.*}}.h):rnd
 
 declare i64 @llvm.hexagon.M2.mpyd.rnd.hh.s1(i32, i32)
 define i64 @M2_mpyd_rnd_hh_s1(i32 %a, i32 %b) {
   %z = call i64 @llvm.hexagon.M2.mpyd.rnd.hh.s1(i32 %a, i32 %b)
   ret i64 %z
 }
-; CHECK: r1:0 = mpy(r0.h, r1.h):<<1:rnd
+; CHECK: = mpy({{.*}}.h, {{.*}}.h):<<1:rnd
 
 declare i64 @llvm.hexagon.M2.mpyd.acc.ll.s0(i64, i32, i32)
 define i64 @M2_mpyd_acc_ll_s0(i64 %a, i32 %b, i32 %c) {
   %z = call i64 @llvm.hexagon.M2.mpyd.acc.ll.s0(i64 %a, i32 %b, i32 %c)
   ret i64 %z
 }
-; CHECK: r1:0 += mpy(r2.l, r3.l)
+; CHECK: += mpy({{.*}}.l, {{.*}}.l)
 
 declare i64 @llvm.hexagon.M2.mpyd.acc.ll.s1(i64, i32, i32)
 define i64 @M2_mpyd_acc_ll_s1(i64 %a, i32 %b, i32 %c) {
   %z = call i64 @llvm.hexagon.M2.mpyd.acc.ll.s1(i64 %a, i32 %b, i32 %c)
   ret i64 %z
 }
-; CHECK: r1:0 += mpy(r2.l, r3.l):<<1
+; CHECK: += mpy({{.*}}.l, {{.*}}.l):<<1
 
 declare i64 @llvm.hexagon.M2.mpyd.acc.lh.s0(i64, i32, i32)
 define i64 @M2_mpyd_acc_lh_s0(i64 %a, i32 %b, i32 %c) {
   %z = call i64 @llvm.hexagon.M2.mpyd.acc.lh.s0(i64 %a, i32 %b, i32 %c)
   ret i64 %z
 }
-; CHECK: r1:0 += mpy(r2.l, r3.h)
+; CHECK: += mpy({{.*}}.l, {{.*}}.h)
 
 declare i64 @llvm.hexagon.M2.mpyd.acc.lh.s1(i64, i32, i32)
 define i64 @M2_mpyd_acc_lh_s1(i64 %a, i32 %b, i32 %c) {
   %z = call i64 @llvm.hexagon.M2.mpyd.acc.lh.s1(i64 %a, i32 %b, i32 %c)
   ret i64 %z
 }
-; CHECK: r1:0 += mpy(r2.l, r3.h):<<1
+; CHECK: += mpy({{.*}}.l, {{.*}}.h):<<1
 
 declare i64 @llvm.hexagon.M2.mpyd.acc.hl.s0(i64, i32, i32)
 define i64 @M2_mpyd_acc_hl_s0(i64 %a, i32 %b, i32 %c) {
   %z = call i64 @llvm.hexagon.M2.mpyd.acc.hl.s0(i64 %a, i32 %b, i32 %c)
   ret i64 %z
 }
-; CHECK: r1:0 += mpy(r2.h, r3.l)
+; CHECK: += mpy({{.*}}.h, {{.*}}.l)
 
 declare i64 @llvm.hexagon.M2.mpyd.acc.hl.s1(i64, i32, i32)
 define i64 @M2_mpyd_acc_hl_s1(i64 %a, i32 %b, i32 %c) {
   %z = call i64 @llvm.hexagon.M2.mpyd.acc.hl.s1(i64 %a, i32 %b, i32 %c)
   ret i64 %z
 }
-; CHECK: r1:0 += mpy(r2.h, r3.l):<<1
+; CHECK: += mpy({{.*}}.h, {{.*}}.l):<<1
 
 declare i64 @llvm.hexagon.M2.mpyd.acc.hh.s0(i64, i32, i32)
 define i64 @M2_mpyd_acc_hh_s0(i64 %a, i32 %b, i32 %c) {
   %z = call i64 @llvm.hexagon.M2.mpyd.acc.hh.s0(i64 %a, i32 %b, i32 %c)
   ret i64 %z
 }
-; CHECK: r1:0 += mpy(r2.h, r3.h)
+; CHECK: += mpy({{.*}}.h, {{.*}}.h)
 
 declare i64 @llvm.hexagon.M2.mpyd.acc.hh.s1(i64, i32, i32)
 define i64 @M2_mpyd_acc_hh_s1(i64 %a, i32 %b, i32 %c) {
   %z = call i64 @llvm.hexagon.M2.mpyd.acc.hh.s1(i64 %a, i32 %b, i32 %c)
   ret i64 %z
 }
-; CHECK: r1:0 += mpy(r2.h, r3.h):<<1
+; CHECK: += mpy({{.*}}.h, {{.*}}.h):<<1
 
 declare i64 @llvm.hexagon.M2.mpyd.nac.ll.s0(i64, i32, i32)
 define i64 @M2_mpyd_nac_ll_s0(i64 %a, i32 %b, i32 %c) {
   %z = call i64 @llvm.hexagon.M2.mpyd.nac.ll.s0(i64 %a, i32 %b, i32 %c)
   ret i64 %z
 }
-; CHECK: r1:0 -= mpy(r2.l, r3.l)
+; CHECK: -= mpy({{.*}}.l, {{.*}}.l)
 
 declare i64 @llvm.hexagon.M2.mpyd.nac.ll.s1(i64, i32, i32)
 define i64 @M2_mpyd_nac_ll_s1(i64 %a, i32 %b, i32 %c) {
   %z = call i64 @llvm.hexagon.M2.mpyd.nac.ll.s1(i64 %a, i32 %b, i32 %c)
   ret i64 %z
 }
-; CHECK: r1:0 -= mpy(r2.l, r3.l):<<1
+; CHECK: -= mpy({{.*}}.l, {{.*}}.l):<<1
 
 declare i64 @llvm.hexagon.M2.mpyd.nac.lh.s0(i64, i32, i32)
 define i64 @M2_mpyd_nac_lh_s0(i64 %a, i32 %b, i32 %c) {
   %z = call i64 @llvm.hexagon.M2.mpyd.nac.lh.s0(i64 %a, i32 %b, i32 %c)
   ret i64 %z
 }
-; CHECK: r1:0 -= mpy(r2.l, r3.h)
+; CHECK: -= mpy({{.*}}.l, {{.*}}.h)
 
 declare i64 @llvm.hexagon.M2.mpyd.nac.lh.s1(i64, i32, i32)
 define i64 @M2_mpyd_nac_lh_s1(i64 %a, i32 %b, i32 %c) {
   %z = call i64 @llvm.hexagon.M2.mpyd.nac.lh.s1(i64 %a, i32 %b, i32 %c)
   ret i64 %z
 }
-; CHECK: r1:0 -= mpy(r2.l, r3.h):<<1
+; CHECK: -= mpy({{.*}}.l, {{.*}}.h):<<1
 
 declare i64 @llvm.hexagon.M2.mpyd.nac.hl.s0(i64, i32, i32)
 define i64 @M2_mpyd_nac_hl_s0(i64 %a, i32 %b, i32 %c) {
   %z = call i64 @llvm.hexagon.M2.mpyd.nac.hl.s0(i64 %a, i32 %b, i32 %c)
   ret i64 %z
 }
-; CHECK: r1:0 -= mpy(r2.h, r3.l)
+; CHECK: -= mpy({{.*}}.h, {{.*}}.l)
 
 declare i64 @llvm.hexagon.M2.mpyd.nac.hl.s1(i64, i32, i32)
 define i64 @M2_mpyd_nac_hl_s1(i64 %a, i32 %b, i32 %c) {
   %z = call i64 @llvm.hexagon.M2.mpyd.nac.hl.s1(i64 %a, i32 %b, i32 %c)
   ret i64 %z
 }
-; CHECK: r1:0 -= mpy(r2.h, r3.l):<<1
+; CHECK: -= mpy({{.*}}.h, {{.*}}.l):<<1
 
 declare i64 @llvm.hexagon.M2.mpyd.nac.hh.s0(i64, i32, i32)
 define i64 @M2_mpyd_nac_hh_s0(i64 %a, i32 %b, i32 %c) {
   %z = call i64 @llvm.hexagon.M2.mpyd.nac.hh.s0(i64 %a, i32 %b, i32 %c)
   ret i64 %z
 }
-; CHECK: r1:0 -= mpy(r2.h, r3.h)
+; CHECK: -= mpy({{.*}}.h, {{.*}}.h)
 
 declare i64 @llvm.hexagon.M2.mpyd.nac.hh.s1(i64, i32, i32)
 define i64 @M2_mpyd_nac_hh_s1(i64 %a, i32 %b, i32 %c) {
   %z = call i64 @llvm.hexagon.M2.mpyd.nac.hh.s1(i64 %a, i32 %b, i32 %c)
   ret i64 %z
 }
-; CHECK: r1:0 -= mpy(r2.h, r3.h):<<1
+; CHECK: -= mpy({{.*}}.h, {{.*}}.h):<<1
 
 declare i32 @llvm.hexagon.M2.mpy.ll.s0(i32, i32)
 define i32 @M2_mpy_ll_s0(i32 %a, i32 %b) {
   %z = call i32 @llvm.hexagon.M2.mpy.ll.s0(i32 %a, i32 %b)
   ret i32 %z
 }
-; CHECK: r0 = mpy(r0.l, r1.l)
+; CHECK: = mpy({{.*}}.l, {{.*}}.l)
 
 declare i32 @llvm.hexagon.M2.mpy.ll.s1(i32, i32)
 define i32 @M2_mpy_ll_s1(i32 %a, i32 %b) {
   %z = call i32 @llvm.hexagon.M2.mpy.ll.s1(i32 %a, i32 %b)
   ret i32 %z
 }
-; CHECK: r0 = mpy(r0.l, r1.l):<<1
+; CHECK: = mpy({{.*}}.l, {{.*}}.l):<<1
 
 declare i32 @llvm.hexagon.M2.mpy.lh.s0(i32, i32)
 define i32 @M2_mpy_lh_s0(i32 %a, i32 %b) {
   %z = call i32 @llvm.hexagon.M2.mpy.lh.s0(i32 %a, i32 %b)
   ret i32 %z
 }
-; CHECK: r0 = mpy(r0.l, r1.h)
+; CHECK: = mpy({{.*}}.l, {{.*}}.h)
 
 declare i32 @llvm.hexagon.M2.mpy.lh.s1(i32, i32)
 define i32 @M2_mpy_lh_s1(i32 %a, i32 %b) {
   %z = call i32 @llvm.hexagon.M2.mpy.lh.s1(i32 %a, i32 %b)
   ret i32 %z
 }
-; CHECK: r0 = mpy(r0.l, r1.h):<<1
+; CHECK: = mpy({{.*}}.l, {{.*}}.h):<<1
 
 declare i32 @llvm.hexagon.M2.mpy.hl.s0(i32, i32)
 define i32 @M2_mpy_hl_s0(i32 %a, i32 %b) {
   %z = call i32 @llvm.hexagon.M2.mpy.hl.s0(i32 %a, i32 %b)
   ret i32 %z
 }
-; CHECK: r0 = mpy(r0.h, r1.l)
+; CHECK: = mpy({{.*}}.h, {{.*}}.l)
 
 declare i32 @llvm.hexagon.M2.mpy.hl.s1(i32, i32)
 define i32 @M2_mpy_hl_s1(i32 %a, i32 %b) {
   %z = call i32 @llvm.hexagon.M2.mpy.hl.s1(i32 %a, i32 %b)
   ret i32 %z
 }
-; CHECK: r0 = mpy(r0.h, r1.l):<<1
+; CHECK: = mpy({{.*}}.h, {{.*}}.l):<<1
 
 declare i32 @llvm.hexagon.M2.mpy.hh.s0(i32, i32)
 define i32 @M2_mpy_hh_s0(i32 %a, i32 %b) {
   %z = call i32 @llvm.hexagon.M2.mpy.hh.s0(i32 %a, i32 %b)
   ret i32 %z
 }
-; CHECK: r0 = mpy(r0.h, r1.h)
+; CHECK: = mpy({{.*}}.h, {{.*}}.h)
 
 declare i32 @llvm.hexagon.M2.mpy.hh.s1(i32, i32)
 define i32 @M2_mpy_hh_s1(i32 %a, i32 %b) {
   %z = call i32 @llvm.hexagon.M2.mpy.hh.s1(i32 %a, i32 %b)
   ret i32 %z
 }
-; CHECK: r0 = mpy(r0.h, r1.h):<<1
+; CHECK: = mpy({{.*}}.h, {{.*}}.h):<<1
 
 declare i32 @llvm.hexagon.M2.mpy.sat.ll.s0(i32, i32)
 define i32 @M2_mpy_sat_ll_s0(i32 %a, i32 %b) {
   %z = call i32 @llvm.hexagon.M2.mpy.sat.ll.s0(i32 %a, i32 %b)
   ret i32 %z
 }
-; CHECK: r0 = mpy(r0.l, r1.l):sat
+; CHECK: = mpy({{.*}}.l, {{.*}}.l):sat
 
 declare i32 @llvm.hexagon.M2.mpy.sat.ll.s1(i32, i32)
 define i32 @M2_mpy_sat_ll_s1(i32 %a, i32 %b) {
   %z = call i32 @llvm.hexagon.M2.mpy.sat.ll.s1(i32 %a, i32 %b)
   ret i32 %z
 }
-; CHECK: r0 = mpy(r0.l, r1.l):<<1:sat
+; CHECK: = mpy({{.*}}.l, {{.*}}.l):<<1:sat
 
 declare i32 @llvm.hexagon.M2.mpy.sat.lh.s0(i32, i32)
 define i32 @M2_mpy_sat_lh_s0(i32 %a, i32 %b) {
   %z = call i32 @llvm.hexagon.M2.mpy.sat.lh.s0(i32 %a, i32 %b)
   ret i32 %z
 }
-; CHECK: r0 = mpy(r0.l, r1.h):sat
+; CHECK: = mpy({{.*}}.l, {{.*}}.h):sat
 
 declare i32 @llvm.hexagon.M2.mpy.sat.lh.s1(i32, i32)
 define i32 @M2_mpy_sat_lh_s1(i32 %a, i32 %b) {
   %z = call i32 @llvm.hexagon.M2.mpy.sat.lh.s1(i32 %a, i32 %b)
   ret i32 %z
 }
-; CHECK: r0 = mpy(r0.l, r1.h):<<1:sat
+; CHECK: = mpy({{.*}}.l, {{.*}}.h):<<1:sat
 
 declare i32 @llvm.hexagon.M2.mpy.sat.hl.s0(i32, i32)
 define i32 @M2_mpy_sat_hl_s0(i32 %a, i32 %b) {
   %z = call i32 @llvm.hexagon.M2.mpy.sat.hl.s0(i32 %a, i32 %b)
   ret i32 %z
 }
-; CHECK: r0 = mpy(r0.h, r1.l):sat
+; CHECK: = mpy({{.*}}.h, {{.*}}.l):sat
 
 declare i32 @llvm.hexagon.M2.mpy.sat.hl.s1(i32, i32)
 define i32 @M2_mpy_sat_hl_s1(i32 %a, i32 %b) {
   %z = call i32 @llvm.hexagon.M2.mpy.sat.hl.s1(i32 %a, i32 %b)
   ret i32 %z
 }
-; CHECK: r0 = mpy(r0.h, r1.l):<<1:sat
+; CHECK: = mpy({{.*}}.h, {{.*}}.l):<<1:sat
 
 declare i32 @llvm.hexagon.M2.mpy.sat.hh.s0(i32, i32)
 define i32 @M2_mpy_sat_hh_s0(i32 %a, i32 %b) {
   %z = call i32 @llvm.hexagon.M2.mpy.sat.hh.s0(i32 %a, i32 %b)
   ret i32 %z
 }
-; CHECK: r0 = mpy(r0.h, r1.h):sat
+; CHECK: = mpy({{.*}}.h, {{.*}}.h):sat
 
 declare i32 @llvm.hexagon.M2.mpy.sat.hh.s1(i32, i32)
 define i32 @M2_mpy_sat_hh_s1(i32 %a, i32 %b) {
   %z = call i32 @llvm.hexagon.M2.mpy.sat.hh.s1(i32 %a, i32 %b)
   ret i32 %z
 }
-; CHECK: r0 = mpy(r0.h, r1.h):<<1:sat
+; CHECK: = mpy({{.*}}.h, {{.*}}.h):<<1:sat
 
 declare i32 @llvm.hexagon.M2.mpy.sat.rnd.ll.s0(i32, i32)
 define i32 @M2_mpy_sat_rnd_ll_s0(i32 %a, i32 %b) {
   %z = call i32 @llvm.hexagon.M2.mpy.sat.rnd.ll.s0(i32 %a, i32 %b)
   ret i32 %z
 }
-; CHECK: r0 = mpy(r0.l, r1.l):rnd:sat
+; CHECK: = mpy({{.*}}.l, {{.*}}.l):rnd:sat
 
 declare i32 @llvm.hexagon.M2.mpy.sat.rnd.ll.s1(i32, i32)
 define i32 @M2_mpy_sat_rnd_ll_s1(i32 %a, i32 %b) {
   %z = call i32 @llvm.hexagon.M2.mpy.sat.rnd.ll.s1(i32 %a, i32 %b)
   ret i32 %z
 }
-; CHECK: r0 = mpy(r0.l, r1.l):<<1:rnd:sat
+; CHECK: = mpy({{.*}}.l, {{.*}}.l):<<1:rnd:sat
 
 declare i32 @llvm.hexagon.M2.mpy.sat.rnd.lh.s0(i32, i32)
 define i32 @M2_mpy_sat_rnd_lh_s0(i32 %a, i32 %b) {
   %z = call i32 @llvm.hexagon.M2.mpy.sat.rnd.lh.s0(i32 %a, i32 %b)
   ret i32 %z
 }
-; CHECK: r0 = mpy(r0.l, r1.h):rnd:sat
+; CHECK: = mpy({{.*}}.l, {{.*}}.h):rnd:sat
 
 declare i32 @llvm.hexagon.M2.mpy.sat.rnd.lh.s1(i32, i32)
 define i32 @M2_mpy_sat_rnd_lh_s1(i32 %a, i32 %b) {
   %z = call i32 @llvm.hexagon.M2.mpy.sat.rnd.lh.s1(i32 %a, i32 %b)
   ret i32 %z
 }
-; CHECK: r0 = mpy(r0.l, r1.h):<<1:rnd:sat
+; CHECK: = mpy({{.*}}.l, {{.*}}.h):<<1:rnd:sat
 
 declare i32 @llvm.hexagon.M2.mpy.sat.rnd.hl.s0(i32, i32)
 define i32 @M2_mpy_sat_rnd_hl_s0(i32 %a, i32 %b) {
   %z = call i32 @llvm.hexagon.M2.mpy.sat.rnd.hl.s0(i32 %a, i32 %b)
   ret i32 %z
 }
-; CHECK: r0 = mpy(r0.h, r1.l):rnd:sat
+; CHECK: = mpy({{.*}}.h, {{.*}}.l):rnd:sat
 
 declare i32 @llvm.hexagon.M2.mpy.sat.rnd.hl.s1(i32, i32)
 define i32 @M2_mpy_sat_rnd_hl_s1(i32 %a, i32 %b) {
   %z = call i32 @llvm.hexagon.M2.mpy.sat.rnd.hl.s1(i32 %a, i32 %b)
   ret i32 %z
 }
-; CHECK: r0 = mpy(r0.h, r1.l):<<1:rnd:sat
+; CHECK: = mpy({{.*}}.h, {{.*}}.l):<<1:rnd:sat
 
 declare i32 @llvm.hexagon.M2.mpy.sat.rnd.hh.s0(i32, i32)
 define i32 @M2_mpy_sat_rnd_hh_s0(i32 %a, i32 %b) {
   %z = call i32 @llvm.hexagon.M2.mpy.sat.rnd.hh.s0(i32 %a, i32 %b)
   ret i32 %z
 }
-; CHECK: r0 = mpy(r0.h, r1.h):rnd:sat
+; CHECK: = mpy({{.*}}.h, {{.*}}.h):rnd:sat
 
 declare i32 @llvm.hexagon.M2.mpy.sat.rnd.hh.s1(i32, i32)
 define i32 @M2_mpy_sat_rnd_hh_s1(i32 %a, i32 %b) {
   %z = call i32 @llvm.hexagon.M2.mpy.sat.rnd.hh.s1(i32 %a, i32 %b)
   ret i32 %z
 }
-; CHECK: r0 = mpy(r0.h, r1.h):<<1:rnd:sat
+; CHECK: = mpy({{.*}}.h, {{.*}}.h):<<1:rnd:sat
 
 declare i32 @llvm.hexagon.M2.mpy.acc.ll.s0(i32, i32, i32)
 define i32 @M2_mpy_acc_ll_s0(i32 %a, i32 %b, i32 %c) {
   %z = call i32 @llvm.hexagon.M2.mpy.acc.ll.s0(i32 %a, i32 %b, i32 %c)
   ret i32 %z
 }
-; CHECK: r0 += mpy(r1.l, r2.l)
+; CHECK: += mpy({{.*}}.l, {{.*}}.l)
 
 declare i32 @llvm.hexagon.M2.mpy.acc.ll.s1(i32, i32, i32)
 define i32 @M2_mpy_acc_ll_s1(i32 %a, i32 %b, i32 %c) {
   %z = call i32 @llvm.hexagon.M2.mpy.acc.ll.s1(i32 %a, i32 %b, i32 %c)
   ret i32 %z
 }
-; CHECK: r0 += mpy(r1.l, r2.l):<<1
+; CHECK: += mpy({{.*}}.l, {{.*}}.l):<<1
 
 declare i32 @llvm.hexagon.M2.mpy.acc.lh.s0(i32, i32, i32)
 define i32 @M2_mpy_acc_lh_s0(i32 %a, i32 %b, i32 %c) {
   %z = call i32 @llvm.hexagon.M2.mpy.acc.lh.s0(i32 %a, i32 %b, i32 %c)
   ret i32 %z
 }
-; CHECK: r0 += mpy(r1.l, r2.h)
+; CHECK: += mpy({{.*}}.l, {{.*}}.h)
 
 declare i32 @llvm.hexagon.M2.mpy.acc.lh.s1(i32, i32, i32)
 define i32 @M2_mpy_acc_lh_s1(i32 %a, i32 %b, i32 %c) {
   %z = call i32 @llvm.hexagon.M2.mpy.acc.lh.s1(i32 %a, i32 %b, i32 %c)
   ret i32 %z
 }
-; CHECK: r0 += mpy(r1.l, r2.h):<<1
+; CHECK: += mpy({{.*}}.l, {{.*}}.h):<<1
 
 declare i32 @llvm.hexagon.M2.mpy.acc.hl.s0(i32, i32, i32)
 define i32 @M2_mpy_acc_hl_s0(i32 %a, i32 %b, i32 %c) {
   %z = call i32 @llvm.hexagon.M2.mpy.acc.hl.s0(i32 %a, i32 %b, i32 %c)
   ret i32 %z
 }
-; CHECK: r0 += mpy(r1.h, r2.l)
+; CHECK: += mpy({{.*}}.h, {{.*}}.l)
 
 declare i32 @llvm.hexagon.M2.mpy.acc.hl.s1(i32, i32, i32)
 define i32 @M2_mpy_acc_hl_s1(i32 %a, i32 %b, i32 %c) {
   %z = call i32 @llvm.hexagon.M2.mpy.acc.hl.s1(i32 %a, i32 %b, i32 %c)
   ret i32 %z
 }
-; CHECK: r0 += mpy(r1.h, r2.l):<<1
+; CHECK: += mpy({{.*}}.h, {{.*}}.l):<<1
 
 declare i32 @llvm.hexagon.M2.mpy.acc.hh.s0(i32, i32, i32)
 define i32 @M2_mpy_acc_hh_s0(i32 %a, i32 %b, i32 %c) {
   %z = call i32 @llvm.hexagon.M2.mpy.acc.hh.s0(i32 %a, i32 %b, i32 %c)
   ret i32 %z
 }
-; CHECK: r0 += mpy(r1.h, r2.h)
+; CHECK: += mpy({{.*}}.h, {{.*}}.h)
 
 declare i32 @llvm.hexagon.M2.mpy.acc.hh.s1(i32, i32, i32)
 define i32 @M2_mpy_acc_hh_s1(i32 %a, i32 %b, i32 %c) {
   %z = call i32 @llvm.hexagon.M2.mpy.acc.hh.s1(i32 %a, i32 %b, i32 %c)
   ret i32 %z
 }
-; CHECK: r0 += mpy(r1.h, r2.h):<<1
+; CHECK: += mpy({{.*}}.h, {{.*}}.h):<<1
 
 declare i32 @llvm.hexagon.M2.mpy.acc.sat.ll.s0(i32, i32, i32)
 define i32 @M2_mpy_acc_sat_ll_s0(i32 %a, i32 %b, i32 %c) {
   %z = call i32 @llvm.hexagon.M2.mpy.acc.sat.ll.s0(i32 %a, i32 %b, i32 %c)
   ret i32 %z
 }
-; CHECK: r0 += mpy(r1.l, r2.l):sat
+; CHECK: += mpy({{.*}}.l, {{.*}}.l):sat
 
 declare i32 @llvm.hexagon.M2.mpy.acc.sat.ll.s1(i32, i32, i32)
 define i32 @M2_mpy_acc_sat_ll_s1(i32 %a, i32 %b, i32 %c) {
   %z = call i32 @llvm.hexagon.M2.mpy.acc.sat.ll.s1(i32 %a, i32 %b, i32 %c)
   ret i32 %z
 }
-; CHECK: r0 += mpy(r1.l, r2.l):<<1:sat
+; CHECK: += mpy({{.*}}.l, {{.*}}.l):<<1:sat
 
 declare i32 @llvm.hexagon.M2.mpy.acc.sat.lh.s0(i32, i32, i32)
 define i32 @M2_mpy_acc_sat_lh_s0(i32 %a, i32 %b, i32 %c) {
   %z = call i32 @llvm.hexagon.M2.mpy.acc.sat.lh.s0(i32 %a, i32 %b, i32 %c)
   ret i32 %z
 }
-; CHECK: r0 += mpy(r1.l, r2.h):sat
+; CHECK: += mpy({{.*}}.l, {{.*}}.h):sat
 
 declare i32 @llvm.hexagon.M2.mpy.acc.sat.lh.s1(i32, i32, i32)
 define i32 @M2_mpy_acc_sat_lh_s1(i32 %a, i32 %b, i32 %c) {
   %z = call i32 @llvm.hexagon.M2.mpy.acc.sat.lh.s1(i32 %a, i32 %b, i32 %c)
   ret i32 %z
 }
-; CHECK: r0 += mpy(r1.l, r2.h):<<1:sat
+; CHECK: += mpy({{.*}}.l, {{.*}}.h):<<1:sat
 
 declare i32 @llvm.hexagon.M2.mpy.acc.sat.hl.s0(i32, i32, i32)
 define i32 @M2_mpy_acc_sat_hl_s0(i32 %a, i32 %b, i32 %c) {
   %z = call i32 @llvm.hexagon.M2.mpy.acc.sat.hl.s0(i32 %a, i32 %b, i32 %c)
   ret i32 %z
 }
-; CHECK: r0 += mpy(r1.h, r2.l):sat
+; CHECK: += mpy({{.*}}.h, {{.*}}.l):sat
 
 declare i32 @llvm.hexagon.M2.mpy.acc.sat.hl.s1(i32, i32, i32)
 define i32 @M2_mpy_acc_sat_hl_s1(i32 %a, i32 %b, i32 %c) {
   %z = call i32 @llvm.hexagon.M2.mpy.acc.sat.hl.s1(i32 %a, i32 %b, i32 %c)
   ret i32 %z
 }
-; CHECK: r0 += mpy(r1.h, r2.l):<<1:sat
+; CHECK: += mpy({{.*}}.h, {{.*}}.l):<<1:sat
 
 declare i32 @llvm.hexagon.M2.mpy.acc.sat.hh.s0(i32, i32, i32)
 define i32 @M2_mpy_acc_sat_hh_s0(i32 %a, i32 %b, i32 %c) {
   %z = call i32 @llvm.hexagon.M2.mpy.acc.sat.hh.s0(i32 %a, i32 %b, i32 %c)
   ret i32 %z
 }
-; CHECK: r0 += mpy(r1.h, r2.h):sat
+; CHECK: += mpy({{.*}}.h, {{.*}}.h):sat
 
 declare i32 @llvm.hexagon.M2.mpy.acc.sat.hh.s1(i32, i32, i32)
 define i32 @M2_mpy_acc_sat_hh_s1(i32 %a, i32 %b, i32 %c) {
   %z = call i32 @llvm.hexagon.M2.mpy.acc.sat.hh.s1(i32 %a, i32 %b, i32 %c)
   ret i32 %z
 }
-; CHECK: r0 += mpy(r1.h, r2.h):<<1:sat
+; CHECK: += mpy({{.*}}.h, {{.*}}.h):<<1:sat
 
 declare i32 @llvm.hexagon.M2.mpy.nac.ll.s0(i32, i32, i32)
 define i32 @M2_mpy_nac_ll_s0(i32 %a, i32 %b, i32 %c) {
   %z = call i32 @llvm.hexagon.M2.mpy.nac.ll.s0(i32 %a, i32 %b, i32 %c)
   ret i32 %z
 }
-; CHECK: r0 -= mpy(r1.l, r2.l)
+; CHECK: -= mpy({{.*}}.l, {{.*}}.l)
 
 declare i32 @llvm.hexagon.M2.mpy.nac.ll.s1(i32, i32, i32)
 define i32 @M2_mpy_nac_ll_s1(i32 %a, i32 %b, i32 %c) {
   %z = call i32 @llvm.hexagon.M2.mpy.nac.ll.s1(i32 %a, i32 %b, i32 %c)
   ret i32 %z
 }
-; CHECK: r0 -= mpy(r1.l, r2.l):<<1
+; CHECK: -= mpy({{.*}}.l, {{.*}}.l):<<1
 
 declare i32 @llvm.hexagon.M2.mpy.nac.lh.s0(i32, i32, i32)
 define i32 @M2_mpy_nac_lh_s0(i32 %a, i32 %b, i32 %c) {
   %z = call i32 @llvm.hexagon.M2.mpy.nac.lh.s0(i32 %a, i32 %b, i32 %c)
   ret i32 %z
 }
-; CHECK: r0 -= mpy(r1.l, r2.h)
+; CHECK: -= mpy({{.*}}.l, {{.*}}.h)
 
 declare i32 @llvm.hexagon.M2.mpy.nac.lh.s1(i32, i32, i32)
 define i32 @M2_mpy_nac_lh_s1(i32 %a, i32 %b, i32 %c) {
   %z = call i32 @llvm.hexagon.M2.mpy.nac.lh.s1(i32 %a, i32 %b, i32 %c)
   ret i32 %z
 }
-; CHECK: r0 -= mpy(r1.l, r2.h):<<1
+; CHECK:  -= mpy({{.*}}.l, {{.*}}.h):<<1
 
 declare i32 @llvm.hexagon.M2.mpy.nac.hl.s0(i32, i32, i32)
 define i32 @M2_mpy_nac_hl_s0(i32 %a, i32 %b, i32 %c) {
   %z = call i32 @llvm.hexagon.M2.mpy.nac.hl.s0(i32 %a, i32 %b, i32 %c)
   ret i32 %z
 }
-; CHECK: r0 -= mpy(r1.h, r2.l)
+; CHECK: -= mpy({{.*}}.h, {{.*}}.l)
 
 declare i32 @llvm.hexagon.M2.mpy.nac.hl.s1(i32, i32, i32)
 define i32 @M2_mpy_nac_hl_s1(i32 %a, i32 %b, i32 %c) {
   %z = call i32 @llvm.hexagon.M2.mpy.nac.hl.s1(i32 %a, i32 %b, i32 %c)
   ret i32 %z
 }
-; CHECK: r0 -= mpy(r1.h, r2.l):<<1
+; CHECK: -= mpy({{.*}}.h, {{.*}}.l):<<1
 
 declare i32 @llvm.hexagon.M2.mpy.nac.hh.s0(i32, i32, i32)
 define i32 @M2_mpy_nac_hh_s0(i32 %a, i32 %b, i32 %c) {
   %z = call i32 @llvm.hexagon.M2.mpy.nac.hh.s0(i32 %a, i32 %b, i32 %c)
   ret i32 %z
 }
-; CHECK: r0 -= mpy(r1.h, r2.h)
+; CHECK: -= mpy({{.*}}.h, {{.*}}.h)
 
 declare i32 @llvm.hexagon.M2.mpy.nac.hh.s1(i32, i32, i32)
 define i32 @M2_mpy_nac_hh_s1(i32 %a, i32 %b, i32 %c) {
   %z = call i32 @llvm.hexagon.M2.mpy.nac.hh.s1(i32 %a, i32 %b, i32 %c)
   ret i32 %z
 }
-; CHECK: r0 -= mpy(r1.h, r2.h):<<1
+; CHECK: -= mpy({{.*}}.h, {{.*}}.h):<<1
 
 declare i32 @llvm.hexagon.M2.mpy.nac.sat.ll.s0(i32, i32, i32)
 define i32 @M2_mpy_nac_sat_ll_s0(i32 %a, i32 %b, i32 %c) {
   %z = call i32 @llvm.hexagon.M2.mpy.nac.sat.ll.s0(i32 %a, i32 %b, i32 %c)
   ret i32 %z
 }
-; CHECK: r0 -= mpy(r1.l, r2.l):sat
+; CHECK: -= mpy({{.*}}.l, {{.*}}.l):sat
 
 declare i32 @llvm.hexagon.M2.mpy.nac.sat.ll.s1(i32, i32, i32)
 define i32 @M2_mpy_nac_sat_ll_s1(i32 %a, i32 %b, i32 %c) {
   %z = call i32 @llvm.hexagon.M2.mpy.nac.sat.ll.s1(i32 %a, i32 %b, i32 %c)
   ret i32 %z
 }
-; CHECK: r0 -= mpy(r1.l, r2.l):<<1:sat
+; CHECK: -= mpy({{.*}}.l, {{.*}}.l):<<1:sat
 
 declare i32 @llvm.hexagon.M2.mpy.nac.sat.lh.s0(i32, i32, i32)
 define i32 @M2_mpy_nac_sat_lh_s0(i32 %a, i32 %b, i32 %c) {
   %z = call i32 @llvm.hexagon.M2.mpy.nac.sat.lh.s0(i32 %a, i32 %b, i32 %c)
   ret i32 %z
 }
-; CHECK: r0 -= mpy(r1.l, r2.h):sat
+; CHECK: -= mpy({{.*}}.l, {{.*}}.h):sat
 
 declare i32 @llvm.hexagon.M2.mpy.nac.sat.lh.s1(i32, i32, i32)
 define i32 @M2_mpy_nac_sat_lh_s1(i32 %a, i32 %b, i32 %c) {
   %z = call i32 @llvm.hexagon.M2.mpy.nac.sat.lh.s1(i32 %a, i32 %b, i32 %c)
   ret i32 %z
 }
-; CHECK: r0 -= mpy(r1.l, r2.h):<<1:sat
+; CHECK: -= mpy({{.*}}.l, {{.*}}.h):<<1:sat
 
 declare i32 @llvm.hexagon.M2.mpy.nac.sat.hl.s0(i32, i32, i32)
 define i32 @M2_mpy_nac_sat_hl_s0(i32 %a, i32 %b, i32 %c) {
   %z = call i32 @llvm.hexagon.M2.mpy.nac.sat.hl.s0(i32 %a, i32 %b, i32 %c)
   ret i32 %z
 }
-; CHECK: r0 -= mpy(r1.h, r2.l):sat
+; CHECK: -= mpy({{.*}}.h, {{.*}}.l):sat
 
 declare i32 @llvm.hexagon.M2.mpy.nac.sat.hl.s1(i32, i32, i32)
 define i32 @M2_mpy_nac_sat_hl_s1(i32 %a, i32 %b, i32 %c) {
   %z = call i32 @llvm.hexagon.M2.mpy.nac.sat.hl.s1(i32 %a, i32 %b, i32 %c)
   ret i32 %z
 }
-; CHECK: r0 -= mpy(r1.h, r2.l):<<1:sat
+; CHECK: -= mpy({{.*}}.h, {{.*}}.l):<<1:sat
 
 declare i32 @llvm.hexagon.M2.mpy.nac.sat.hh.s0(i32, i32, i32)
 define i32 @M2_mpy_nac_sat_hh_s0(i32 %a, i32 %b, i32 %c) {
   %z = call i32 @llvm.hexagon.M2.mpy.nac.sat.hh.s0(i32 %a, i32 %b, i32 %c)
   ret i32 %z
 }
-; CHECK: r0 -= mpy(r1.h, r2.h):sat
+; CHECK: -= mpy({{.*}}.h, {{.*}}.h):sat
 
 declare i32 @llvm.hexagon.M2.mpy.nac.sat.hh.s1(i32, i32, i32)
 define i32 @M2_mpy_nac_sat_hh_s1(i32 %a, i32 %b, i32 %c) {
   %z = call i32 @llvm.hexagon.M2.mpy.nac.sat.hh.s1(i32 %a, i32 %b, i32 %c)
   ret i32 %z
 }
-; CHECK: r0 -= mpy(r1.h, r2.h):<<1:sat
+; CHECK: -= mpy({{.*}}.h, {{.*}}.h):<<1:sat
 
 ; Multiply unsigned halfwords
 declare i64 @llvm.hexagon.M2.mpyud.ll.s0(i32, i32)
@@ -774,336 +778,336 @@ define i64 @M2_mpyud_ll_s0(i32 %a, i32 %b) {
   %z = call i64 @llvm.hexagon.M2.mpyud.ll.s0(i32 %a, i32 %b)
   ret i64 %z
 }
-; CHECK: r1:0 = mpyu(r0.l, r1.l)
+; CHECK: = mpyu({{.*}}.l, {{.*}}.l)
 
 declare i64 @llvm.hexagon.M2.mpyud.ll.s1(i32, i32)
 define i64 @M2_mpyud_ll_s1(i32 %a, i32 %b) {
   %z = call i64 @llvm.hexagon.M2.mpyud.ll.s1(i32 %a, i32 %b)
   ret i64 %z
 }
-; CHECK: r1:0 = mpyu(r0.l, r1.l):<<1
+; CHECK: = mpyu({{.*}}.l, {{.*}}.l):<<1
 
 declare i64 @llvm.hexagon.M2.mpyud.lh.s0(i32, i32)
 define i64 @M2_mpyud_lh_s0(i32 %a, i32 %b) {
   %z = call i64 @llvm.hexagon.M2.mpyud.lh.s0(i32 %a, i32 %b)
   ret i64 %z
 }
-; CHECK: r1:0 = mpyu(r0.l, r1.h)
+; CHECK: = mpyu({{.*}}.l, {{.*}}.h)
 
 declare i64 @llvm.hexagon.M2.mpyud.lh.s1(i32, i32)
 define i64 @M2_mpyud_lh_s1(i32 %a, i32 %b) {
   %z = call i64 @llvm.hexagon.M2.mpyud.lh.s1(i32 %a, i32 %b)
   ret i64 %z
 }
-; CHECK: r1:0 = mpyu(r0.l, r1.h):<<1
+; CHECK: = mpyu({{.*}}.l, {{.*}}.h):<<1
 
 declare i64 @llvm.hexagon.M2.mpyud.hl.s0(i32, i32)
 define i64 @M2_mpyud_hl_s0(i32 %a, i32 %b) {
   %z = call i64 @llvm.hexagon.M2.mpyud.hl.s0(i32 %a, i32 %b)
   ret i64 %z
 }
-; CHECK: r1:0 = mpyu(r0.h, r1.l)
+; CHECK: = mpyu({{.*}}.h, {{.*}}.l)
 
 declare i64 @llvm.hexagon.M2.mpyud.hl.s1(i32, i32)
 define i64 @M2_mpyud_hl_s1(i32 %a, i32 %b) {
   %z = call i64 @llvm.hexagon.M2.mpyud.hl.s1(i32 %a, i32 %b)
   ret i64 %z
 }
-; CHECK: r1:0 = mpyu(r0.h, r1.l):<<1
+; CHECK: = mpyu({{.*}}.h, {{.*}}.l):<<1
 
 declare i64 @llvm.hexagon.M2.mpyud.hh.s0(i32, i32)
 define i64 @M2_mpyud_hh_s0(i32 %a, i32 %b) {
   %z = call i64 @llvm.hexagon.M2.mpyud.hh.s0(i32 %a, i32 %b)
   ret i64 %z
 }
-; CHECK: r1:0 = mpyu(r0.h, r1.h)
+; CHECK: = mpyu({{.*}}.h, {{.*}}.h)
 
 declare i64 @llvm.hexagon.M2.mpyud.hh.s1(i32, i32)
 define i64 @M2_mpyud_hh_s1(i32 %a, i32 %b) {
   %z = call i64 @llvm.hexagon.M2.mpyud.hh.s1(i32 %a, i32 %b)
   ret i64 %z
 }
-; CHECK: r1:0 = mpyu(r0.h, r1.h):<<1
+; CHECK: = mpyu({{.*}}.h, {{.*}}.h):<<1
 
 declare i64 @llvm.hexagon.M2.mpyud.acc.ll.s0(i64, i32, i32)
 define i64 @M2_mpyud_acc_ll_s0(i64 %a, i32 %b, i32 %c) {
   %z = call i64 @llvm.hexagon.M2.mpyud.acc.ll.s0(i64 %a, i32 %b, i32 %c)
   ret i64 %z
 }
-; CHECK: r1:0 += mpyu(r2.l, r3.l)
+; CHECK: += mpyu({{.*}}.l, {{.*}}.l)
 
 declare i64 @llvm.hexagon.M2.mpyud.acc.ll.s1(i64, i32, i32)
 define i64 @M2_mpyud_acc_ll_s1(i64 %a, i32 %b, i32 %c) {
   %z = call i64 @llvm.hexagon.M2.mpyud.acc.ll.s1(i64 %a, i32 %b, i32 %c)
   ret i64 %z
 }
-; CHECK: r1:0 += mpyu(r2.l, r3.l):<<1
+; CHECK: += mpyu({{.*}}.l, {{.*}}.l):<<1
 
 declare i64 @llvm.hexagon.M2.mpyud.acc.lh.s0(i64, i32, i32)
 define i64 @M2_mpyud_acc_lh_s0(i64 %a, i32 %b, i32 %c) {
   %z = call i64 @llvm.hexagon.M2.mpyud.acc.lh.s0(i64 %a, i32 %b, i32 %c)
   ret i64 %z
 }
-; CHECK: r1:0 += mpyu(r2.l, r3.h)
+; CHECK: += mpyu({{.*}}.l, {{.*}}.h)
 
 declare i64 @llvm.hexagon.M2.mpyud.acc.lh.s1(i64, i32, i32)
 define i64 @M2_mpyud_acc_lh_s1(i64 %a, i32 %b, i32 %c) {
   %z = call i64 @llvm.hexagon.M2.mpyud.acc.lh.s1(i64 %a, i32 %b, i32 %c)
   ret i64 %z
 }
-; CHECK: r1:0 += mpyu(r2.l, r3.h):<<1
+; CHECK: += mpyu({{.*}}.l, {{.*}}.h):<<1
 
 declare i64 @llvm.hexagon.M2.mpyud.acc.hl.s0(i64, i32, i32)
 define i64 @M2_mpyud_acc_hl_s0(i64 %a, i32 %b, i32 %c) {
   %z = call i64 @llvm.hexagon.M2.mpyud.acc.hl.s0(i64 %a, i32 %b, i32 %c)
   ret i64 %z
 }
-; CHECK: r1:0 += mpyu(r2.h, r3.l)
+; CHECK: += mpyu({{.*}}.h, {{.*}}.l)
 
 declare i64 @llvm.hexagon.M2.mpyud.acc.hl.s1(i64, i32, i32)
 define i64 @M2_mpyud_acc_hl_s1(i64 %a, i32 %b, i32 %c) {
   %z = call i64 @llvm.hexagon.M2.mpyud.acc.hl.s1(i64 %a, i32 %b, i32 %c)
   ret i64 %z
 }
-; CHECK: r1:0 += mpyu(r2.h, r3.l):<<1
+; CHECK: += mpyu({{.*}}.h, {{.*}}.l):<<1
 
 declare i64 @llvm.hexagon.M2.mpyud.acc.hh.s0(i64, i32, i32)
 define i64 @M2_mpyud_acc_hh_s0(i64 %a, i32 %b, i32 %c) {
   %z = call i64 @llvm.hexagon.M2.mpyud.acc.hh.s0(i64 %a, i32 %b, i32 %c)
   ret i64 %z
 }
-; CHECK: r1:0 += mpyu(r2.h, r3.h)
+; CHECK: += mpyu({{.*}}.h, {{.*}}.h)
 
 declare i64 @llvm.hexagon.M2.mpyud.acc.hh.s1(i64, i32, i32)
 define i64 @M2_mpyud_acc_hh_s1(i64 %a, i32 %b, i32 %c) {
   %z = call i64 @llvm.hexagon.M2.mpyud.acc.hh.s1(i64 %a, i32 %b, i32 %c)
   ret i64 %z
 }
-; CHECK: r1:0 += mpyu(r2.h, r3.h):<<1
+; CHECK: += mpyu({{.*}}.h, {{.*}}.h):<<1
 
 declare i64 @llvm.hexagon.M2.mpyud.nac.ll.s0(i64, i32, i32)
 define i64 @M2_mpyud_nac_ll_s0(i64 %a, i32 %b, i32 %c) {
   %z = call i64 @llvm.hexagon.M2.mpyud.nac.ll.s0(i64 %a, i32 %b, i32 %c)
   ret i64 %z
 }
-; CHECK: r1:0 -= mpyu(r2.l, r3.l)
+; CHECK: -= mpyu({{.*}}.l, {{.*}}.l)
 
 declare i64 @llvm.hexagon.M2.mpyud.nac.ll.s1(i64, i32, i32)
 define i64 @M2_mpyud_nac_ll_s1(i64 %a, i32 %b, i32 %c) {
   %z = call i64 @llvm.hexagon.M2.mpyud.nac.ll.s1(i64 %a, i32 %b, i32 %c)
   ret i64 %z
 }
-; CHECK: r1:0 -= mpyu(r2.l, r3.l):<<1
+; CHECK: -= mpyu({{.*}}.l, {{.*}}.l):<<1
 
 declare i64 @llvm.hexagon.M2.mpyud.nac.lh.s0(i64, i32, i32)
 define i64 @M2_mpyud_nac_lh_s0(i64 %a, i32 %b, i32 %c) {
   %z = call i64 @llvm.hexagon.M2.mpyud.nac.lh.s0(i64 %a, i32 %b, i32 %c)
   ret i64 %z
 }
-; CHECK: r1:0 -= mpyu(r2.l, r3.h)
+; CHECK: -= mpyu({{.*}}.l, {{.*}}.h)
 
 declare i64 @llvm.hexagon.M2.mpyud.nac.lh.s1(i64, i32, i32)
 define i64 @M2_mpyud_nac_lh_s1(i64 %a, i32 %b, i32 %c) {
   %z = call i64 @llvm.hexagon.M2.mpyud.nac.lh.s1(i64 %a, i32 %b, i32 %c)
   ret i64 %z
 }
-; CHECK: r1:0 -= mpyu(r2.l, r3.h):<<1
+; CHECK: -= mpyu({{.*}}.l, {{.*}}.h):<<1
 
 declare i64 @llvm.hexagon.M2.mpyud.nac.hl.s0(i64, i32, i32)
 define i64 @M2_mpyud_nac_hl_s0(i64 %a, i32 %b, i32 %c) {
   %z = call i64 @llvm.hexagon.M2.mpyud.nac.hl.s0(i64 %a, i32 %b, i32 %c)
   ret i64 %z
 }
-; CHECK: r1:0 -= mpyu(r2.h, r3.l)
+; CHECK: -= mpyu({{.*}}.h, {{.*}}.l)
 
 declare i64 @llvm.hexagon.M2.mpyud.nac.hl.s1(i64, i32, i32)
 define i64 @M2_mpyud_nac_hl_s1(i64 %a, i32 %b, i32 %c) {
   %z = call i64 @llvm.hexagon.M2.mpyud.nac.hl.s1(i64 %a, i32 %b, i32 %c)
   ret i64 %z
 }
-; CHECK: r1:0 -= mpyu(r2.h, r3.l):<<1
+; CHECK: -= mpyu({{.*}}.h, {{.*}}.l):<<1
 
 declare i64 @llvm.hexagon.M2.mpyud.nac.hh.s0(i64, i32, i32)
 define i64 @M2_mpyud_nac_hh_s0(i64 %a, i32 %b, i32 %c) {
   %z = call i64 @llvm.hexagon.M2.mpyud.nac.hh.s0(i64 %a, i32 %b, i32 %c)
   ret i64 %z
 }
-; CHECK: r1:0 -= mpyu(r2.h, r3.h)
+; CHECK: -= mpyu({{.*}}.h, {{.*}}.h)
 
 declare i64 @llvm.hexagon.M2.mpyud.nac.hh.s1(i64, i32, i32)
 define i64 @M2_mpyud_nac_hh_s1(i64 %a, i32 %b, i32 %c) {
   %z = call i64 @llvm.hexagon.M2.mpyud.nac.hh.s1(i64 %a, i32 %b, i32 %c)
   ret i64 %z
 }
-; CHECK: r1:0 -= mpyu(r2.h, r3.h):<<1
+; CHECK: -= mpyu({{.*}}.h, {{.*}}.h):<<1
 
 declare i32 @llvm.hexagon.M2.mpyu.ll.s0(i32, i32)
 define i32 @M2_mpyu_ll_s0(i32 %a, i32 %b) {
   %z = call i32 @llvm.hexagon.M2.mpyu.ll.s0(i32 %a, i32 %b)
   ret i32 %z
 }
-; CHECK: r0 = mpyu(r0.l, r1.l)
+; CHECK: = mpyu({{.*}}.l, {{.*}}.l)
 
 declare i32 @llvm.hexagon.M2.mpyu.ll.s1(i32, i32)
 define i32 @M2_mpyu_ll_s1(i32 %a, i32 %b) {
   %z = call i32 @llvm.hexagon.M2.mpyu.ll.s1(i32 %a, i32 %b)
   ret i32 %z
 }
-; CHECK: r0 = mpyu(r0.l, r1.l):<<1
+; CHECK: = mpyu({{.*}}.l, {{.*}}.l):<<1
 
 declare i32 @llvm.hexagon.M2.mpyu.lh.s0(i32, i32)
 define i32 @M2_mpyu_lh_s0(i32 %a, i32 %b) {
   %z = call i32 @llvm.hexagon.M2.mpyu.lh.s0(i32 %a, i32 %b)
   ret i32 %z
 }
-; CHECK: r0 = mpyu(r0.l, r1.h)
+; CHECK: = mpyu({{.*}}.l, {{.*}}.h)
 
 declare i32 @llvm.hexagon.M2.mpyu.lh.s1(i32, i32)
 define i32 @M2_mpyu_lh_s1(i32 %a, i32 %b) {
   %z = call i32 @llvm.hexagon.M2.mpyu.lh.s1(i32 %a, i32 %b)
   ret i32 %z
 }
-; CHECK: r0 = mpyu(r0.l, r1.h):<<1
+; CHECK: = mpyu({{.*}}.l, {{.*}}.h):<<1
 
 declare i32 @llvm.hexagon.M2.mpyu.hl.s0(i32, i32)
 define i32 @M2_mpyu_hl_s0(i32 %a, i32 %b) {
   %z = call i32 @llvm.hexagon.M2.mpyu.hl.s0(i32 %a, i32 %b)
   ret i32 %z
 }
-; CHECK: r0 = mpyu(r0.h, r1.l)
+; CHECK: = mpyu({{.*}}.h, {{.*}}.l)
 
 declare i32 @llvm.hexagon.M2.mpyu.hl.s1(i32, i32)
 define i32 @M2_mpyu_hl_s1(i32 %a, i32 %b) {
   %z = call i32 @llvm.hexagon.M2.mpyu.hl.s1(i32 %a, i32 %b)
   ret i32 %z
 }
-; CHECK: r0 = mpyu(r0.h, r1.l):<<1
+; CHECK: = mpyu({{.*}}.h, {{.*}}.l):<<1
 
 declare i32 @llvm.hexagon.M2.mpyu.hh.s0(i32, i32)
 define i32 @M2_mpyu_hh_s0(i32 %a, i32 %b) {
   %z = call i32 @llvm.hexagon.M2.mpyu.hh.s0(i32 %a, i32 %b)
   ret i32 %z
 }
-; CHECK: r0 = mpyu(r0.h, r1.h)
+; CHECK: = mpyu({{.*}}.h, {{.*}}.h)
 
 declare i32 @llvm.hexagon.M2.mpyu.hh.s1(i32, i32)
 define i32 @M2_mpyu_hh_s1(i32 %a, i32 %b) {
   %z = call i32 @llvm.hexagon.M2.mpyu.hh.s1(i32 %a, i32 %b)
   ret i32 %z
 }
-; CHECK: r0 = mpyu(r0.h, r1.h):<<1
+; CHECK: = mpyu({{.*}}.h, {{.*}}.h):<<1
 
 declare i32 @llvm.hexagon.M2.mpyu.acc.ll.s0(i32, i32, i32)
 define i32 @M2_mpyu_acc_ll_s0(i32 %a, i32 %b, i32 %c) {
   %z = call i32 @llvm.hexagon.M2.mpyu.acc.ll.s0(i32 %a, i32 %b, i32 %c)
   ret i32 %z
 }
-; CHECK: r0 += mpyu(r1.l, r2.l)
+; CHECK: += mpyu({{.*}}.l, {{.*}}.l)
 
 declare i32 @llvm.hexagon.M2.mpyu.acc.ll.s1(i32, i32, i32)
 define i32 @M2_mpyu_acc_ll_s1(i32 %a, i32 %b, i32 %c) {
   %z = call i32 @llvm.hexagon.M2.mpyu.acc.ll.s1(i32 %a, i32 %b, i32 %c)
   ret i32 %z
 }
-; CHECK: r0 += mpyu(r1.l, r2.l):<<1
+; CHECK: += mpyu({{.*}}.l, {{.*}}.l):<<1
 
 declare i32 @llvm.hexagon.M2.mpyu.acc.lh.s0(i32, i32, i32)
 define i32 @M2_mpyu_acc_lh_s0(i32 %a, i32 %b, i32 %c) {
   %z = call i32 @llvm.hexagon.M2.mpyu.acc.lh.s0(i32 %a, i32 %b, i32 %c)
   ret i32 %z
 }
-; CHECK: r0 += mpyu(r1.l, r2.h)
+; CHECK: += mpyu({{.*}}.l, {{.*}}.h)
 
 declare i32 @llvm.hexagon.M2.mpyu.acc.lh.s1(i32, i32, i32)
 define i32 @M2_mpyu_acc_lh_s1(i32 %a, i32 %b, i32 %c) {
   %z = call i32 @llvm.hexagon.M2.mpyu.acc.lh.s1(i32 %a, i32 %b, i32 %c)
   ret i32 %z
 }
-; CHECK: r0 += mpyu(r1.l, r2.h):<<1
+; CHECK: += mpyu({{.*}}.l, {{.*}}.h):<<1
 
 declare i32 @llvm.hexagon.M2.mpyu.acc.hl.s0(i32, i32, i32)
 define i32 @M2_mpyu_acc_hl_s0(i32 %a, i32 %b, i32 %c) {
   %z = call i32 @llvm.hexagon.M2.mpyu.acc.hl.s0(i32 %a, i32 %b, i32 %c)
   ret i32 %z
 }
-; CHECK: r0 += mpyu(r1.h, r2.l)
+; CHECK: += mpyu({{.*}}.h, {{.*}}.l)
 
 declare i32 @llvm.hexagon.M2.mpyu.acc.hl.s1(i32, i32, i32)
 define i32 @M2_mpyu_acc_hl_s1(i32 %a, i32 %b, i32 %c) {
   %z = call i32 @llvm.hexagon.M2.mpyu.acc.hl.s1(i32 %a, i32 %b, i32 %c)
   ret i32 %z
 }
-; CHECK: r0 += mpyu(r1.h, r2.l):<<1
+; CHECK: += mpyu({{.*}}.h, {{.*}}.l):<<1
 
 declare i32 @llvm.hexagon.M2.mpyu.acc.hh.s0(i32, i32, i32)
 define i32 @M2_mpyu_acc_hh_s0(i32 %a, i32 %b, i32 %c) {
   %z = call i32 @llvm.hexagon.M2.mpyu.acc.hh.s0(i32 %a, i32 %b, i32 %c)
   ret i32 %z
 }
-; CHECK: r0 += mpyu(r1.h, r2.h)
+; CHECK: += mpyu({{.*}}.h, {{.*}}.h)
 
 declare i32 @llvm.hexagon.M2.mpyu.acc.hh.s1(i32, i32, i32)
 define i32 @M2_mpyu_acc_hh_s1(i32 %a, i32 %b, i32 %c) {
   %z = call i32 @llvm.hexagon.M2.mpyu.acc.hh.s1(i32 %a, i32 %b, i32 %c)
   ret i32 %z
 }
-; CHECK: r0 += mpyu(r1.h, r2.h):<<1
+; CHECK: += mpyu({{.*}}.h, {{.*}}.h):<<1
 
 declare i32 @llvm.hexagon.M2.mpyu.nac.ll.s0(i32, i32, i32)
 define i32 @M2_mpyu_nac_ll_s0(i32 %a, i32 %b, i32 %c) {
   %z = call i32 @llvm.hexagon.M2.mpyu.nac.ll.s0(i32 %a, i32 %b, i32 %c)
   ret i32 %z
 }
-; CHECK: r0 -= mpyu(r1.l, r2.l)
+; CHECK: -= mpyu({{.*}}.l, {{.*}}.l)
 
 declare i32 @llvm.hexagon.M2.mpyu.nac.ll.s1(i32, i32, i32)
 define i32 @M2_mpyu_nac_ll_s1(i32 %a, i32 %b, i32 %c) {
   %z = call i32 @llvm.hexagon.M2.mpyu.nac.ll.s1(i32 %a, i32 %b, i32 %c)
   ret i32 %z
 }
-; CHECK: r0 -= mpyu(r1.l, r2.l):<<1
+; CHECK: -= mpyu({{.*}}.l, {{.*}}.l):<<1
 
 declare i32 @llvm.hexagon.M2.mpyu.nac.lh.s0(i32, i32, i32)
 define i32 @M2_mpyu_nac_lh_s0(i32 %a, i32 %b, i32 %c) {
   %z = call i32 @llvm.hexagon.M2.mpyu.nac.lh.s0(i32 %a, i32 %b, i32 %c)
   ret i32 %z
 }
-; CHECK: r0 -= mpyu(r1.l, r2.h)
+; CHECK: -= mpyu({{.*}}.l, {{.*}}.h)
 
 declare i32 @llvm.hexagon.M2.mpyu.nac.lh.s1(i32, i32, i32)
 define i32 @M2_mpyu_nac_lh_s1(i32 %a, i32 %b, i32 %c) {
   %z = call i32 @llvm.hexagon.M2.mpyu.nac.lh.s1(i32 %a, i32 %b, i32 %c)
   ret i32 %z
 }
-; CHECK: r0 -= mpyu(r1.l, r2.h):<<1
+; CHECK: -= mpyu({{.*}}.l, {{.*}}.h):<<1
 
 declare i32 @llvm.hexagon.M2.mpyu.nac.hl.s0(i32, i32, i32)
 define i32 @M2_mpyu_nac_hl_s0(i32 %a, i32 %b, i32 %c) {
   %z = call i32 @llvm.hexagon.M2.mpyu.nac.hl.s0(i32 %a, i32 %b, i32 %c)
   ret i32 %z
 }
-; CHECK: r0 -= mpyu(r1.h, r2.l)
+; CHECK: -= mpyu({{.*}}.h, {{.*}}.l)
 
 declare i32 @llvm.hexagon.M2.mpyu.nac.hl.s1(i32, i32, i32)
 define i32 @M2_mpyu_nac_hl_s1(i32 %a, i32 %b, i32 %c) {
   %z = call i32 @llvm.hexagon.M2.mpyu.nac.hl.s1(i32 %a, i32 %b, i32 %c)
   ret i32 %z
 }
-; CHECK: r0 -= mpyu(r1.h, r2.l):<<1
+; CHECK: -= mpyu({{.*}}.h, {{.*}}.l):<<1
 
 declare i32 @llvm.hexagon.M2.mpyu.nac.hh.s0(i32, i32, i32)
 define i32 @M2_mpyu_nac_hh_s0(i32 %a, i32 %b, i32 %c) {
   %z = call i32 @llvm.hexagon.M2.mpyu.nac.hh.s0(i32 %a, i32 %b, i32 %c)
   ret i32 %z
 }
-; CHECK: r0 -= mpyu(r1.h, r2.h)
+; CHECK: -= mpyu({{.*}}.h, {{.*}}.h)
 
 declare i32 @llvm.hexagon.M2.mpyu.nac.hh.s1(i32, i32, i32)
 define i32 @M2_mpyu_nac_hh_s1(i32 %a, i32 %b, i32 %c) {
   %z = call i32 @llvm.hexagon.M2.mpyu.nac.hh.s1(i32 %a, i32 %b, i32 %c)
   ret i32 %z
 }
-; CHECK: r0 -= mpyu(r1.h, r2.h):<<1
+; CHECK: -= mpyu({{.*}}.h, {{.*}}.h):<<1
 
 ; Polynomial multiply words
 declare i64 @llvm.hexagon.M4.pmpyw(i32, i32)
@@ -1111,14 +1115,14 @@ define i64 @M4_pmpyw(i32 %a, i32 %b) {
   %z = call i64 @llvm.hexagon.M4.pmpyw(i32 %a, i32 %b)
   ret i64 %z
 }
-; CHECK: r1:0 = pmpyw(r0, r1)
+; CHECK: = pmpyw({{.*}}, {{.*}})
 
 declare i64 @llvm.hexagon.M4.pmpyw.acc(i64, i32, i32)
 define i64 @M4_pmpyw_acc(i64 %a, i32 %b, i32 %c) {
   %z = call i64 @llvm.hexagon.M4.pmpyw.acc(i64 %a, i32 %b, i32 %c)
   ret i64 %z
 }
-; CHECK: r1:0 ^= pmpyw(r2, r3)
+; CHECK: ^= pmpyw({{.*}}, {{.*}})
 
 ; Vector reduce multiply word by signed half
 declare i64 @llvm.hexagon.M4.vrmpyoh.s0(i64, i64)
@@ -1126,56 +1130,56 @@ define i64 @M4_vrmpyoh_s0(i64 %a, i64 %b) {
   %z = call i64 @llvm.hexagon.M4.vrmpyoh.s0(i64 %a, i64 %b)
   ret i64 %z
 }
-; CHECK: r1:0 = vrmpywoh(r1:0, r3:2)
+; CHECK: = vrmpywoh({{.*}}, {{.*}})
 
 declare i64 @llvm.hexagon.M4.vrmpyoh.s1(i64, i64)
 define i64 @M4_vrmpyoh_s1(i64 %a, i64 %b) {
   %z = call i64 @llvm.hexagon.M4.vrmpyoh.s1(i64 %a, i64 %b)
   ret i64 %z
 }
-; CHECK: r1:0 = vrmpywoh(r1:0, r3:2):<<1
+; CHECK: = vrmpywoh({{.*}}, {{.*}}):<<1
 
 declare i64 @llvm.hexagon.M4.vrmpyeh.s0(i64, i64)
 define i64 @M4_vrmpyeh_s0(i64 %a, i64 %b) {
   %z = call i64 @llvm.hexagon.M4.vrmpyeh.s0(i64 %a, i64 %b)
   ret i64 %z
 }
-; CHECK: r1:0 = vrmpyweh(r1:0, r3:2)
+; CHECK: = vrmpyweh({{.*}}, {{.*}})
 
 declare i64 @llvm.hexagon.M4.vrmpyeh.s1(i64, i64)
 define i64 @M4_vrmpyeh_s1(i64 %a, i64 %b) {
   %z = call i64 @llvm.hexagon.M4.vrmpyeh.s1(i64 %a, i64 %b)
   ret i64 %z
 }
-; CHECK: r1:0 = vrmpyweh(r1:0, r3:2):<<1
+; CHECK: = vrmpyweh({{.*}}, {{.*}}):<<1
 
 declare i64 @llvm.hexagon.M4.vrmpyoh.acc.s0(i64, i64, i64)
 define i64 @M4_vrmpyoh_acc_s0(i64 %a, i64 %b, i64 %c) {
   %z = call i64 @llvm.hexagon.M4.vrmpyoh.acc.s0(i64 %a, i64 %b, i64 %c)
   ret i64 %z
 }
-; CHECK: r1:0 += vrmpywoh(r3:2, r5:4)
+; CHECK: += vrmpywoh({{.*}}, r5:4)
 
 declare i64 @llvm.hexagon.M4.vrmpyoh.acc.s1(i64, i64, i64)
 define i64 @M4_vrmpyoh_acc_s1(i64 %a, i64 %b, i64 %c) {
   %z = call i64 @llvm.hexagon.M4.vrmpyoh.acc.s1(i64 %a, i64 %b, i64 %c)
   ret i64 %z
 }
-; CHECK: r1:0 += vrmpywoh(r3:2, r5:4):<<1
+; CHECK: += vrmpywoh({{.*}}, r5:4):<<1
 
 declare i64 @llvm.hexagon.M4.vrmpyeh.acc.s0(i64, i64, i64)
 define i64 @M4_vrmpyeh_acc_s0(i64 %a, i64 %b, i64 %c) {
   %z = call i64 @llvm.hexagon.M4.vrmpyeh.acc.s0(i64 %a, i64 %b, i64 %c)
   ret i64 %z
 }
-; CHECK: r1:0 += vrmpyweh(r3:2, r5:4)
+; CHECK: += vrmpyweh({{.*}}, r5:4)
 
 declare i64 @llvm.hexagon.M4.vrmpyeh.acc.s1(i64, i64, i64)
 define i64 @M4_vrmpyeh_acc_s1(i64 %a, i64 %b, i64 %c) {
   %z = call i64 @llvm.hexagon.M4.vrmpyeh.acc.s1(i64 %a, i64 %b, i64 %c)
   ret i64 %z
 }
-; CHECK: r1:0 += vrmpyweh(r3:2, r5:4):<<1
+; CHECK: += vrmpyweh({{.*}}, r5:4):<<1
 
 ; Multiply and use upper result
 declare i32 @llvm.hexagon.M2.dpmpyss.rnd.s0(i32, i32)
@@ -1183,84 +1187,84 @@ define i32 @M2_dpmpyss_rnd_s0(i32 %a, i32 %b) {
   %z = call i32 @llvm.hexagon.M2.dpmpyss.rnd.s0(i32 %a, i32 %b)
   ret i32 %z
 }
-; CHECK: r0 = mpy(r0, r1):rnd
+; CHECK: = mpy({{.*}}, {{.*}}):rnd
 
 declare i32 @llvm.hexagon.M2.mpyu.up(i32, i32)
 define i32 @M2_mpyu_up(i32 %a, i32 %b) {
   %z = call i32 @llvm.hexagon.M2.mpyu.up(i32 %a, i32 %b)
   ret i32 %z
 }
-; CHECK: r0 = mpyu(r0, r1)
+; CHECK: = mpyu({{.*}}, {{.*}})
 
 declare i32 @llvm.hexagon.M2.mpysu.up(i32, i32)
 define i32 @M2_mpysu_up(i32 %a, i32 %b) {
   %z = call i32 @llvm.hexagon.M2.mpysu.up(i32 %a, i32 %b)
   ret i32 %z
 }
-; CHECK: r0 = mpysu(r0, r1)
+; CHECK: = mpysu({{.*}}, {{.*}})
 
 declare i32 @llvm.hexagon.M2.hmmpyh.s1(i32, i32)
 define i32 @M2_hmmpyh_s1(i32 %a, i32 %b) {
   %z = call i32 @llvm.hexagon.M2.hmmpyh.s1(i32 %a, i32 %b)
   ret i32 %z
 }
-; CHECK: r0 = mpy(r0, r1.h):<<1:sat
+; CHECK: = mpy({{.*}}, {{.*}}.h):<<1:sat
 
 declare i32 @llvm.hexagon.M2.hmmpyl.s1(i32, i32)
 define i32 @M2_hmmpyl_s1(i32 %a, i32 %b) {
   %z = call i32 @llvm.hexagon.M2.hmmpyl.s1(i32 %a, i32 %b)
   ret i32 %z
 }
-; CHECK: r0 = mpy(r0, r1.l):<<1:sat
+; CHECK: = mpy({{.*}}, {{.*}}.l):<<1:sat
 
 declare i32 @llvm.hexagon.M2.hmmpyh.rs1(i32, i32)
 define i32 @M2_hmmpyh_rs1(i32 %a, i32 %b) {
   %z = call i32 @llvm.hexagon.M2.hmmpyh.rs1(i32 %a, i32 %b)
   ret i32 %z
 }
-; CHECK: r0 = mpy(r0, r1.h):<<1:rnd:sat
+; CHECK: = mpy({{.*}}, {{.*}}.h):<<1:rnd:sat
 
 declare i32 @llvm.hexagon.M2.mpy.up.s1.sat(i32, i32)
 define i32 @M2_mpy_up_s1_sat(i32 %a, i32 %b) {
   %z = call i32 @llvm.hexagon.M2.mpy.up.s1.sat(i32 %a, i32 %b)
   ret i32 %z
 }
-; CHECK: r0 = mpy(r0, r1):<<1:sat
+; CHECK: = mpy({{.*}}, {{.*}}):<<1:sat
 
 declare i32 @llvm.hexagon.M2.hmmpyl.rs1(i32, i32)
 define i32 @M2_hmmpyl_rs1(i32 %a, i32 %b) {
   %z = call i32 @llvm.hexagon.M2.hmmpyl.rs1(i32 %a, i32 %b)
   ret i32 %z
 }
-; CHECK: r0 = mpy(r0, r1.l):<<1:rnd:sat
+; CHECK: = mpy({{.*}}, {{.*}}.l):<<1:rnd:sat
 
 declare i32 @llvm.hexagon.M2.mpy.up(i32, i32)
 define i32 @M2_mpy_up(i32 %a, i32 %b) {
   %z = call i32 @llvm.hexagon.M2.mpy.up(i32 %a, i32 %b)
   ret i32 %z
 }
-; CHECK: r0 = mpy(r0, r1)
+; CHECK: = mpy({{.*}}, {{.*}})
 
 declare i32 @llvm.hexagon.M2.mpy.up.s1(i32, i32)
 define i32 @M2_mpy_up_s1(i32 %a, i32 %b) {
   %z = call i32 @llvm.hexagon.M2.mpy.up.s1(i32 %a, i32 %b)
   ret i32 %z
 }
-; CHECK: r0 = mpy(r0, r1):<<1
+; CHECK: = mpy({{.*}}, {{.*}}):<<1
 
 declare i32 @llvm.hexagon.M4.mac.up.s1.sat(i32, i32, i32)
 define i32 @M4_mac_up_s1_sat(i32 %a, i32 %b, i32 %c) {
   %z = call i32 @llvm.hexagon.M4.mac.up.s1.sat(i32 %a, i32 %b, i32 %c)
   ret i32 %z
 }
-; CHECK: r0 += mpy(r1, r2):<<1:sat
+; CHECK: += mpy({{.*}}, {{.*}}):<<1:sat
 
 declare i32 @llvm.hexagon.M4.nac.up.s1.sat(i32, i32, i32)
 define i32 @M4_nac_up_s1_sat(i32 %a, i32 %b, i32 %c) {
   %z = call i32 @llvm.hexagon.M4.nac.up.s1.sat(i32 %a, i32 %b, i32 %c)
   ret i32 %z
 }
-; CHECK: r0 -= mpy(r1, r2):<<1:sat
+; CHECK: -= mpy({{.*}}, {{.*}}):<<1:sat
 
 ; Multiply and use full result
 declare i64 @llvm.hexagon.M2.dpmpyss.s0(i32, i32)
@@ -1268,42 +1272,42 @@ define i64 @M2_dpmpyss_s0(i32 %a, i32 %b) {
   %z = call i64 @llvm.hexagon.M2.dpmpyss.s0(i32 %a, i32 %b)
   ret i64 %z
 }
-; CHECK: r1:0 = mpy(r0, r1)
+; CHECK: = mpy({{.*}}, {{.*}})
 
 declare i64 @llvm.hexagon.M2.dpmpyuu.s0(i32, i32)
 define i64 @M2_dpmpyuu_s0(i32 %a, i32 %b) {
   %z = call i64 @llvm.hexagon.M2.dpmpyuu.s0(i32 %a, i32 %b)
   ret i64 %z
 }
-; CHECK: r1:0 = mpyu(r0, r1)
+; CHECK: = mpyu({{.*}}, {{.*}})
 
 declare i64 @llvm.hexagon.M2.dpmpyss.acc.s0(i64, i32, i32)
 define i64 @M2_dpmpyss_acc_s0(i64 %a, i32 %b, i32 %c) {
   %z = call i64 @llvm.hexagon.M2.dpmpyss.acc.s0(i64 %a, i32 %b, i32 %c)
   ret i64 %z
 }
-; CHECK: r1:0 += mpy(r2, r3)
+; CHECK: += mpy({{.*}}, {{.*}})
 
 declare i64 @llvm.hexagon.M2.dpmpyss.nac.s0(i64, i32, i32)
 define i64 @M2_dpmpyss_nac_s0(i64 %a, i32 %b, i32 %c) {
   %z = call i64 @llvm.hexagon.M2.dpmpyss.nac.s0(i64 %a, i32 %b, i32 %c)
   ret i64 %z
 }
-; CHECK: r1:0 -= mpy(r2, r3)
+; CHECK: -= mpy({{.*}}, {{.*}})
 
 declare i64 @llvm.hexagon.M2.dpmpyuu.acc.s0(i64, i32, i32)
 define i64 @M2_dpmpyuu_acc_s0(i64 %a, i32 %b, i32 %c) {
   %z = call i64 @llvm.hexagon.M2.dpmpyuu.acc.s0(i64 %a, i32 %b, i32 %c)
   ret i64 %z
 }
-; CHECK: r1:0 += mpyu(r2, r3)
+; CHECK: += mpyu({{.*}}, {{.*}})
 
 declare i64 @llvm.hexagon.M2.dpmpyuu.nac.s0(i64, i32, i32)
 define i64 @M2_dpmpyuu_nac_s0(i64 %a, i32 %b, i32 %c) {
   %z = call i64 @llvm.hexagon.M2.dpmpyuu.nac.s0(i64 %a, i32 %b, i32 %c)
   ret i64 %z
 }
-; CHECK: r1:0 -= mpyu(r2, r3)
+; CHECK: -= mpyu({{.*}}, {{.*}})
 
 ; Vector dual multiply
 declare i64 @llvm.hexagon.M2.vdmpys.s0(i64, i64)
@@ -1311,14 +1315,14 @@ define i64 @M2_vdmpys_s0(i64 %a, i64 %b) {
   %z = call i64 @llvm.hexagon.M2.vdmpys.s0(i64 %a, i64 %b)
   ret i64 %z
 }
-; CHECK: r1:0 = vdmpy(r1:0, r3:2):sat
+; CHECK: = vdmpy({{.*}}, {{.*}}):sat
 
 declare i64 @llvm.hexagon.M2.vdmpys.s1(i64, i64)
 define i64 @M2_vdmpys_s1(i64 %a, i64 %b) {
   %z = call i64 @llvm.hexagon.M2.vdmpys.s1(i64 %a, i64 %b)
   ret i64 %z
 }
-; CHECK: r1:0 = vdmpy(r1:0, r3:2):<<1:sat
+; CHECK: = vdmpy({{.*}}, {{.*}}):<<1:sat
 
 ; Vector reduce multiply bytes
 declare i64 @llvm.hexagon.M5.vrmpybuu(i64, i64)
@@ -1326,28 +1330,28 @@ define i64 @M5_vrmpybuu(i64 %a, i64 %b) {
   %z = call i64 @llvm.hexagon.M5.vrmpybuu(i64 %a, i64 %b)
   ret i64 %z
 }
-; CHECK: r1:0 = vrmpybu(r1:0, r3:2)
+; CHECK: = vrmpybu({{.*}}, {{.*}})
 
 declare i64 @llvm.hexagon.M5.vrmpybsu(i64, i64)
 define i64 @M5_vrmpybsu(i64 %a, i64 %b) {
   %z = call i64 @llvm.hexagon.M5.vrmpybsu(i64 %a, i64 %b)
   ret i64 %z
 }
-; CHECK: r1:0 = vrmpybsu(r1:0, r3:2)
+; CHECK: = vrmpybsu({{.*}}, {{.*}})
 
 declare i64 @llvm.hexagon.M5.vrmacbuu(i64, i64, i64)
 define i64 @M5_vrmacbuu(i64 %a, i64 %b, i64 %c) {
   %z = call i64 @llvm.hexagon.M5.vrmacbuu(i64 %a, i64 %b, i64 %c)
   ret i64 %z
 }
-; CHECK: r1:0 += vrmpybu(r3:2, r5:4)
+; CHECK: += vrmpybu({{.*}}, r5:4)
 
 declare i64 @llvm.hexagon.M5.vrmacbsu(i64, i64, i64)
 define i64 @M5_vrmacbsu(i64 %a, i64 %b, i64 %c) {
   %z = call i64 @llvm.hexagon.M5.vrmacbsu(i64 %a, i64 %b, i64 %c)
   ret i64 %z
 }
-; CHECK: r1:0 += vrmpybsu(r3:2, r5:4)
+; CHECK: += vrmpybsu({{.*}}, r5:4)
 
 ; Vector dual multiply signed by unsigned bytes
 declare i64 @llvm.hexagon.M5.vdmpybsu(i64, i64)
@@ -1355,14 +1359,14 @@ define i64 @M5_vdmpybsu(i64 %a, i64 %b) {
   %z = call i64 @llvm.hexagon.M5.vdmpybsu(i64 %a, i64 %b)
   ret i64 %z
 }
-; CHECK: r1:0 = vdmpybsu(r1:0, r3:2):sat
+; CHECK: = vdmpybsu({{.*}}, {{.*}}):sat
 
 declare i64 @llvm.hexagon.M5.vdmacbsu(i64, i64, i64)
 define i64 @M5_vdmacbsu(i64 %a, i64 %b, i64 %c) {
   %z = call i64 @llvm.hexagon.M5.vdmacbsu(i64 %a, i64 %b, i64 %c)
   ret i64 %z
 }
-; CHECK: r1:0 += vdmpybsu(r3:2, r5:4):sat
+; CHECK: += vdmpybsu({{.*}}, r5:4):sat
 
 ; Vector multiply even halfwords
 declare i64 @llvm.hexagon.M2.vmpy2es.s0(i64, i64)
@@ -1370,35 +1374,35 @@ define i64 @M2_vmpy2es_s0(i64 %a, i64 %b) {
   %z = call i64 @llvm.hexagon.M2.vmpy2es.s0(i64 %a, i64 %b)
   ret i64 %z
 }
-; CHECK: r1:0 = vmpyeh(r1:0, r3:2):sat
+; CHECK: = vmpyeh({{.*}}, {{.*}}):sat
 
 declare i64 @llvm.hexagon.M2.vmpy2es.s1(i64, i64)
 define i64 @M2_vmpy2es_s1(i64 %a, i64 %b) {
   %z = call i64 @llvm.hexagon.M2.vmpy2es.s1(i64 %a, i64 %b)
   ret i64 %z
 }
-; CHECK: r1:0 = vmpyeh(r1:0, r3:2):<<1:sat
+; CHECK: = vmpyeh({{.*}}, {{.*}}):<<1:sat
 
 declare i64 @llvm.hexagon.M2.vmac2es(i64, i64, i64)
 define i64 @M2_vmac2es(i64 %a, i64 %b, i64 %c) {
   %z = call i64 @llvm.hexagon.M2.vmac2es(i64 %a, i64 %b, i64 %c)
   ret i64 %z
 }
-; CHECK: r1:0 += vmpyeh(r3:2, r5:4)
+; CHECK: += vmpyeh({{.*}}, r5:4)
 
 declare i64 @llvm.hexagon.M2.vmac2es.s0(i64, i64, i64)
 define i64 @M2_vmac2es_s0(i64 %a, i64 %b, i64 %c) {
   %z = call i64 @llvm.hexagon.M2.vmac2es.s0(i64 %a, i64 %b, i64 %c)
   ret i64 %z
 }
-; CHECK: r1:0 += vmpyeh(r3:2, r5:4):sat
+; CHECK: += vmpyeh({{.*}}, r5:4):sat
 
 declare i64 @llvm.hexagon.M2.vmac2es.s1(i64, i64, i64)
 define i64 @M2_vmac2es_s1(i64 %a, i64 %b, i64 %c) {
   %z = call i64 @llvm.hexagon.M2.vmac2es.s1(i64 %a, i64 %b, i64 %c)
   ret i64 %z
 }
-; CHECK: r1:0 += vmpyeh(r3:2, r5:4):<<1:sat
+; CHECK: += vmpyeh({{.*}}, r5:4):<<1:sat
 
 ; Vector multiply halfwords
 declare i64 @llvm.hexagon.M2.vmpy2s.s0(i32, i32)
@@ -1406,35 +1410,35 @@ define i64 @M2_vmpy2s_s0(i32 %a, i32 %b) {
   %z = call i64 @llvm.hexagon.M2.vmpy2s.s0(i32 %a, i32 %b)
   ret i64 %z
 }
-; CHECK: r1:0 = vmpyh(r0, r1):sat
+; CHECK: = vmpyh({{.*}}, {{.*}}):sat
 
 declare i64 @llvm.hexagon.M2.vmpy2s.s1(i32, i32)
 define i64 @M2_vmpy2s_s1(i32 %a, i32 %b) {
   %z = call i64 @llvm.hexagon.M2.vmpy2s.s1(i32 %a, i32 %b)
   ret i64 %z
 }
-; CHECK: r1:0 = vmpyh(r0, r1):<<1:sat
+; CHECK: = vmpyh({{.*}}, {{.*}}):<<1:sat
 
 declare i64 @llvm.hexagon.M2.vmac2(i64, i32, i32)
 define i64 @M2_vmac2(i64 %a, i32 %b, i32 %c) {
   %z = call i64 @llvm.hexagon.M2.vmac2(i64 %a, i32 %b, i32 %c)
   ret i64 %z
 }
-; CHECK: r1:0 += vmpyh(r2, r3)
+; CHECK: += vmpyh({{.*}}, {{.*}})
 
 declare i64 @llvm.hexagon.M2.vmac2s.s0(i64, i32, i32)
 define i64 @M2_vmac2s_s0(i64 %a, i32 %b, i32 %c) {
   %z = call i64 @llvm.hexagon.M2.vmac2s.s0(i64 %a, i32 %b, i32 %c)
   ret i64 %z
 }
-; CHECK: r1:0 += vmpyh(r2, r3):sat
+; CHECK: += vmpyh({{.*}}, {{.*}}):sat
 
 declare i64 @llvm.hexagon.M2.vmac2s.s1(i64, i32, i32)
 define i64 @M2_vmac2s_s1(i64 %a, i32 %b, i32 %c) {
   %z = call i64 @llvm.hexagon.M2.vmac2s.s1(i64 %a, i32 %b, i32 %c)
   ret i64 %z
 }
-; CHECK: r1:0 += vmpyh(r2, r3):<<1:sat
+; CHECK: += vmpyh({{.*}}, {{.*}}):<<1:sat
 
 ; Vector multiply halfwords signed by unsigned
 declare i64 @llvm.hexagon.M2.vmpy2su.s0(i32, i32)
@@ -1442,28 +1446,28 @@ define i64 @M2_vmpy2su_s0(i32 %a, i32 %b) {
   %z = call i64 @llvm.hexagon.M2.vmpy2su.s0(i32 %a, i32 %b)
   ret i64 %z
 }
-; CHECK: r1:0 = vmpyhsu(r0, r1):sat
+; CHECK: = vmpyhsu({{.*}}, {{.*}}):sat
 
 declare i64 @llvm.hexagon.M2.vmpy2su.s1(i32, i32)
 define i64 @M2_vmpy2su_s1(i32 %a, i32 %b) {
   %z = call i64 @llvm.hexagon.M2.vmpy2su.s1(i32 %a, i32 %b)
   ret i64 %z
 }
-; CHECK: r1:0 = vmpyhsu(r0, r1):<<1:sat
+; CHECK: = vmpyhsu({{.*}}, {{.*}}):<<1:sat
 
 declare i64 @llvm.hexagon.M2.vmac2su.s0(i64, i32, i32)
 define i64 @M2_vmac2su_s0(i64 %a, i32 %b, i32 %c) {
   %z = call i64 @llvm.hexagon.M2.vmac2su.s0(i64 %a, i32 %b, i32 %c)
   ret i64 %z
 }
-; CHECK: r1:0 += vmpyhsu(r2, r3):sat
+; CHECK: += vmpyhsu({{.*}}, {{.*}}):sat
 
 declare i64 @llvm.hexagon.M2.vmac2su.s1(i64, i32, i32)
 define i64 @M2_vmac2su_s1(i64 %a, i32 %b, i32 %c) {
   %z = call i64 @llvm.hexagon.M2.vmac2su.s1(i64 %a, i32 %b, i32 %c)
   ret i64 %z
 }
-; CHECK: r1:0 += vmpyhsu(r2, r3):<<1:sat
+; CHECK: += vmpyhsu({{.*}}, {{.*}}):<<1:sat
 
 ; Vector reduce multiply halfwords
 declare i64 @llvm.hexagon.M2.vrmpy.s0(i64, i64)
@@ -1471,14 +1475,14 @@ define i64 @M2_vrmpy_s0(i64 %a, i64 %b) {
   %z = call i64 @llvm.hexagon.M2.vrmpy.s0(i64 %a, i64 %b)
   ret i64 %z
 }
-; CHECK: r1:0 = vrmpyh(r1:0, r3:2)
+; CHECK: = vrmpyh({{.*}}, {{.*}})
 
 declare i64 @llvm.hexagon.M2.vrmac.s0(i64, i64, i64)
 define i64 @M2_vrmac_s0(i64 %a, i64 %b, i64 %c) {
   %z = call i64 @llvm.hexagon.M2.vrmac.s0(i64 %a, i64 %b, i64 %c)
   ret i64 %z
 }
-; CHECK: r1:0 += vrmpyh(r3:2, r5:4)
+; CHECK: += vrmpyh({{.*}}, r5:4)
 
 ; Vector multiply bytes
 declare i64 @llvm.hexagon.M5.vmpybsu(i32, i32)
@@ -1486,28 +1490,28 @@ define i64 @M2_vmpybsu(i32 %a, i32 %b) {
   %z = call i64 @llvm.hexagon.M5.vmpybsu(i32 %a, i32 %b)
   ret i64 %z
 }
-; CHECK: r1:0 = vmpybsu(r0, r1)
+; CHECK: = vmpybsu({{.*}}, {{.*}})
 
 declare i64 @llvm.hexagon.M5.vmpybuu(i32, i32)
 define i64 @M2_vmpybuu(i32 %a, i32 %b) {
   %z = call i64 @llvm.hexagon.M5.vmpybuu(i32 %a, i32 %b)
   ret i64 %z
 }
-; CHECK: r1:0 = vmpybu(r0, r1)
+; CHECK: = vmpybu({{.*}}, {{.*}})
 
 declare i64 @llvm.hexagon.M5.vmacbuu(i64, i32, i32)
 define i64 @M2_vmacbuu(i64 %a, i32 %b, i32 %c) {
   %z = call i64 @llvm.hexagon.M5.vmacbuu(i64 %a, i32 %b, i32 %c)
   ret i64 %z
 }
-; CHECK: r1:0 += vmpybu(r2, r3)
+; CHECK: += vmpybu({{.*}}, {{.*}})
 
 declare i64 @llvm.hexagon.M5.vmacbsu(i64, i32, i32)
 define i64 @M2_vmacbsu(i64 %a, i32 %b, i32 %c) {
   %z = call i64 @llvm.hexagon.M5.vmacbsu(i64 %a, i32 %b, i32 %c)
   ret i64 %z
 }
-; CHECK: r1:0 += vmpybsu(r2, r3)
+; CHECK: += vmpybsu({{.*}}, {{.*}})
 
 ; Vector polynomial multiply halfwords
 declare i64 @llvm.hexagon.M4.vpmpyh(i32, i32)
@@ -1515,11 +1519,11 @@ define i64 @M4_vpmpyh(i32 %a, i32 %b) {
   %z = call i64 @llvm.hexagon.M4.vpmpyh(i32 %a, i32 %b)
   ret i64 %z
 }
-; CHECK: r1:0 = vpmpyh(r0, r1)
+; CHECK: = vpmpyh({{.*}}, {{.*}})
 
 declare i64 @llvm.hexagon.M4.vpmpyh.acc(i64, i32, i32)
 define i64 @M4_vpmpyh_acc(i64 %a, i32 %b, i32 %c) {
   %z = call i64 @llvm.hexagon.M4.vpmpyh.acc(i64 %a, i32 %b, i32 %c)
   ret i64 %z
 }
-; CHECK: r1:0 ^= vpmpyh(r2, r3)
+; CHECK: ^= vpmpyh({{.*}}, {{.*}})
diff --git a/test/CodeGen/Hexagon/intrinsics/xtype_perm.ll b/test/CodeGen/Hexagon/intrinsics/xtype_perm.ll
index 0b761323e31..3e044e3838d 100644
--- a/test/CodeGen/Hexagon/intrinsics/xtype_perm.ll
+++ b/test/CodeGen/Hexagon/intrinsics/xtype_perm.ll
@@ -1,41 +1,44 @@
 ; RUN: llc -march=hexagon -O0 < %s | FileCheck %s
+; RUN: llc -march=hexagon -O0 < %s | FileCheck -check-prefix=CHECK-CALL %s
 ; Hexagon Programmer's Reference Manual 11.10.6 XTYPE/PERM
 
+; CHECK-CALL-NOT: call
+
 ; Saturate
 declare i32 @llvm.hexagon.A2.sat(i64)
 define i32 @A2_sat(i64 %a) {
   %z = call i32 @llvm.hexagon.A2.sat(i64 %a)
   ret i32 %z
 }
-; CHECK: r0 = sat(r1:0)
+; CHECK: = sat({{.*}})
 
 declare i32 @llvm.hexagon.A2.sath(i32)
 define i32 @A2_sath(i32 %a) {
   %z = call i32 @llvm.hexagon.A2.sath(i32 %a)
   ret i32 %z
 }
-; CHECK: r0 = sath(r0)
+; CHECK: = sath({{.*}})
 
 declare i32 @llvm.hexagon.A2.satuh(i32)
 define i32 @A2_satuh(i32 %a) {
   %z = call i32 @llvm.hexagon.A2.satuh(i32 %a)
   ret i32 %z
 }
-; CHECK: r0 = satuh(r0)
+; CHECK: = satuh({{.*}})
 
 declare i32 @llvm.hexagon.A2.satub(i32)
 define i32 @A2_satub(i32 %a) {
   %z = call i32 @llvm.hexagon.A2.satub(i32 %a)
   ret i32 %z
 }
-; CHECK: r0 = satub(r0)
+; CHECK: = satub({{.*}})
 
 declare i32 @llvm.hexagon.A2.satb(i32)
 define i32 @A2_satb(i32 %a) {
   %z = call i32 @llvm.hexagon.A2.satb(i32 %a)
   ret i32 %z
 }
-; CHECK: r0 = satb(r0)
+; CHECK: = satb({{.*}})
 
 ; Swizzle bytes
 declare i32 @llvm.hexagon.A2.swiz(i32)
@@ -43,7 +46,7 @@ define i32 @A2_swiz(i32 %a) {
   %z = call i32 @llvm.hexagon.A2.swiz(i32 %a)
   ret i32 %z
 }
-; CHECK: r0 = swiz(r0)
+; CHECK: = swiz({{.*}})
 
 ; Vector round and pack
 declare i32 @llvm.hexagon.S2.vrndpackwh(i64)
@@ -51,14 +54,14 @@ define i32 @S2_vrndpackwh(i64 %a) {
   %z = call i32 @llvm.hexagon.S2.vrndpackwh(i64 %a)
   ret i32 %z
 }
-; CHECK: r0 = vrndwh(r1:0)
+; CHECK: = vrndwh({{.*}})
 
 declare i32 @llvm.hexagon.S2.vrndpackwhs(i64)
 define i32 @S2_vrndpackwhs(i64 %a) {
   %z = call i32 @llvm.hexagon.S2.vrndpackwhs(i64 %a)
   ret i32 %z
 }
-; CHECK: r0 = vrndwh(r1:0):sat
+; CHECK: = vrndwh({{.*}}):sat
 
 ; Vector saturate and pack
 declare i32 @llvm.hexagon.S2.vsathub(i64)
@@ -66,42 +69,42 @@ define i32 @S2_vsathub(i64 %a) {
   %z = call i32 @llvm.hexagon.S2.vsathub(i64 %a)
   ret i32 %z
 }
-; CHECK: r0 = vsathub(r1:0)
+; CHECK: = vsathub({{.*}})
 
 declare i32 @llvm.hexagon.S2.vsatwh(i64)
 define i32 @S2_vsatwh(i64 %a) {
   %z = call i32 @llvm.hexagon.S2.vsatwh(i64 %a)
   ret i32 %z
 }
-; CHECK: r0 = vsatwh(r1:0)
+; CHECK: = vsatwh({{.*}})
 
 declare i32 @llvm.hexagon.S2.vsatwuh(i64)
 define i32 @S2_vsatwuh(i64 %a) {
   %z = call i32 @llvm.hexagon.S2.vsatwuh(i64 %a)
   ret i32 %z
 }
-; CHECK: r0 = vsatwuh(r1:0)
+; CHECK: = vsatwuh({{.*}})
 
 declare i32 @llvm.hexagon.S2.vsathb(i64)
 define i32 @S2_vsathb(i64 %a) {
   %z = call i32 @llvm.hexagon.S2.vsathb(i64 %a)
   ret i32 %z
 }
-; CHECK: r0 = vsathb(r1:0)
+; CHECK: = vsathb({{.*}})
 
 declare i32 @llvm.hexagon.S2.svsathb(i32)
 define i32 @S2_svsathb(i32 %a) {
   %z = call i32 @llvm.hexagon.S2.svsathb(i32 %a)
   ret i32 %z
 }
-; CHECK: r0 = vsathb(r0)
+; CHECK: = vsathb({{.*}})
 
 declare i32 @llvm.hexagon.S2.svsathub(i32)
 define i32 @S2_svsathub(i32 %a) {
   %z = call i32 @llvm.hexagon.S2.svsathub(i32 %a)
   ret i32 %z
 }
-; CHECK: r0 = vsathub(r0)
+; CHECK: = vsathub({{.*}})
 
 ; Vector saturate without pack
 declare i64 @llvm.hexagon.S2.vsathub.nopack(i64)
@@ -109,28 +112,28 @@ define i64 @S2_vsathub_nopack(i64 %a) {
   %z = call i64 @llvm.hexagon.S2.vsathub.nopack(i64 %a)
   ret i64 %z
 }
-; CHECK: r1:0 = vsathub(r1:0)
+; CHECK: = vsathub({{.*}})
 
 declare i64 @llvm.hexagon.S2.vsatwuh.nopack(i64)
 define i64 @S2_vsatwuh_nopack(i64 %a) {
   %z = call i64 @llvm.hexagon.S2.vsatwuh.nopack(i64 %a)
   ret i64 %z
 }
-; CHECK: r1:0 = vsatwuh(r1:0)
+; CHECK: = vsatwuh({{.*}})
 
 declare i64 @llvm.hexagon.S2.vsatwh.nopack(i64)
 define i64 @S2_vsatwh_nopack(i64 %a) {
   %z = call i64 @llvm.hexagon.S2.vsatwh.nopack(i64 %a)
   ret i64 %z
 }
-; CHECK: r1:0 = vsatwh(r1:0)
+; CHECK: = vsatwh({{.*}})
 
 declare i64 @llvm.hexagon.S2.vsathb.nopack(i64)
 define i64 @S2_vsathb_nopack(i64 %a) {
   %z = call i64 @llvm.hexagon.S2.vsathb.nopack(i64 %a)
   ret i64 %z
 }
-; CHECK: r1:0 = vsathb(r1:0)
+; CHECK: = vsathb({{.*}})
 
 ; Vector shuffle
 declare i64 @llvm.hexagon.S2.shuffeb(i64, i64)
@@ -138,28 +141,28 @@ define i64 @S2_shuffeb(i64 %a, i64 %b) {
   %z = call i64 @llvm.hexagon.S2.shuffeb(i64 %a, i64 %b)
   ret i64 %z
 }
-; CHECK: r1:0 = shuffeb(r1:0, r3:2)
+; CHECK: = shuffeb({{.*}}, {{.*}})
 
 declare i64 @llvm.hexagon.S2.shuffob(i64, i64)
 define i64 @S2_shuffob(i64 %a, i64 %b) {
   %z = call i64 @llvm.hexagon.S2.shuffob(i64 %a, i64 %b)
   ret i64 %z
 }
-; CHECK: r1:0 = shuffob(r1:0, r3:2)
+; CHECK: = shuffob({{.*}}, {{.*}})
 
 declare i64 @llvm.hexagon.S2.shuffeh(i64, i64)
 define i64 @S2_shuffeh(i64 %a, i64 %b) {
   %z = call i64 @llvm.hexagon.S2.shuffeh(i64 %a, i64 %b)
   ret i64 %z
 }
-; CHECK: r1:0 = shuffeh(r1:0, r3:2)
+; CHECK: = shuffeh({{.*}}, {{.*}})
 
 declare i64 @llvm.hexagon.S2.shuffoh(i64, i64)
 define i64 @S2_shuffoh(i64 %a, i64 %b) {
   %z = call i64 @llvm.hexagon.S2.shuffoh(i64 %a, i64 %b)
   ret i64 %z
 }
-; CHECK: r1:0 = shuffoh(r1:0, r3:2)
+; CHECK: = shuffoh({{.*}}, {{.*}})
 
 ; Vector splat bytes
 declare i32 @llvm.hexagon.S2.vsplatrb(i32)
@@ -167,7 +170,7 @@ define i32 @S2_vsplatrb(i32 %a) {
   %z = call i32 @llvm.hexagon.S2.vsplatrb(i32 %a)
   ret i32 %z
 }
-; CHECK: r0 = vsplatb(r0)
+; CHECK: = vsplatb({{.*}})
 
 ; Vector splat halfwords
 declare i64 @llvm.hexagon.S2.vsplatrh(i32)
@@ -175,7 +178,7 @@ define i64 @S2_vsplatrh(i32 %a) {
   %z = call i64 @llvm.hexagon.S2.vsplatrh(i32 %a)
   ret i64 %z
 }
-; CHECK:  = vsplath(r0)
+; CHECK: = vsplath({{.*}})
 
 ; Vector splice
 declare i64 @llvm.hexagon.S2.vspliceib(i64, i64, i32)
@@ -183,14 +186,14 @@ define i64 @S2_vspliceib(i64 %a, i64 %b) {
   %z = call i64 @llvm.hexagon.S2.vspliceib(i64 %a, i64 %b, i32 0)
   ret i64 %z
 }
-; CHECK: r1:0 = vspliceb(r1:0, r3:2, #0)
+; CHECK: = vspliceb({{.*}}, {{.*}}, #0)
 
 declare i64 @llvm.hexagon.S2.vsplicerb(i64, i64, i32)
 define i64 @S2_vsplicerb(i64 %a, i64 %b, i32 %c) {
   %z = call i64 @llvm.hexagon.S2.vsplicerb(i64 %a, i64 %b, i32 %c)
   ret i64 %z
 }
-; CHECK: r1:0 = vspliceb(r1:0, r3:2, p0)
+; CHECK: = vspliceb({{.*}}, {{.*}}, {{.*}})
 
 ; Vector sign extend
 declare i64 @llvm.hexagon.S2.vsxtbh(i32)
@@ -198,14 +201,14 @@ define i64 @S2_vsxtbh(i32 %a) {
   %z = call i64 @llvm.hexagon.S2.vsxtbh(i32 %a)
   ret i64 %z
 }
-; CHECK:  = vsxtbh(r0)
+; CHECK: = vsxtbh({{.*}})
 
 declare i64 @llvm.hexagon.S2.vsxthw(i32)
 define i64 @S2_vsxthw(i32 %a) {
   %z = call i64 @llvm.hexagon.S2.vsxthw(i32 %a)
   ret i64 %z
 }
-; CHECK:  = vsxthw(r0)
+; CHECK: = vsxthw({{.*}})
 
 ; Vector truncate
 declare i32 @llvm.hexagon.S2.vtrunohb(i64)
@@ -213,28 +216,28 @@ define i32 @S2_vtrunohb(i64 %a) {
   %z = call i32 @llvm.hexagon.S2.vtrunohb(i64 %a)
   ret i32 %z
 }
-; CHECK: r0 = vtrunohb(r1:0)
+; CHECK: = vtrunohb({{.*}})
 
 declare i32 @llvm.hexagon.S2.vtrunehb(i64)
 define i32 @S2_vtrunehb(i64 %a) {
   %z = call i32 @llvm.hexagon.S2.vtrunehb(i64 %a)
   ret i32 %z
 }
-; CHECK: r0 = vtrunehb(r1:0)
+; CHECK: = vtrunehb({{.*}})
 
 declare i64 @llvm.hexagon.S2.vtrunowh(i64, i64)
 define i64 @S2_vtrunowh(i64 %a, i64 %b) {
   %z = call i64 @llvm.hexagon.S2.vtrunowh(i64 %a, i64 %b)
   ret i64 %z
 }
-; CHECK: r1:0 = vtrunowh(r1:0, r3:2)
+; CHECK: = vtrunowh({{.*}}, {{.*}})
 
 declare i64 @llvm.hexagon.S2.vtrunewh(i64, i64)
 define i64 @S2_vtrunewh(i64 %a, i64 %b) {
   %z = call i64 @llvm.hexagon.S2.vtrunewh(i64 %a, i64 %b)
   ret i64 %z
 }
-; CHECK: r1:0 = vtrunewh(r1:0, r3:2)
+; CHECK: = vtrunewh({{.*}}, {{.*}})
 
 ; Vector zero extend
 declare i64 @llvm.hexagon.S2.vzxtbh(i32)
@@ -242,11 +245,11 @@ define i64 @S2_vzxtbh(i32 %a) {
   %z = call i64 @llvm.hexagon.S2.vzxtbh(i32 %a)
   ret i64 %z
 }
-; CHECK:  = vzxtbh(r0)
+; CHECK: = vzxtbh({{.*}})
 
 declare i64 @llvm.hexagon.S2.vzxthw(i32)
 define i64 @S2_vzxthw(i32 %a) {
   %z = call i64 @llvm.hexagon.S2.vzxthw(i32 %a)
   ret i64 %z
 }
-; CHECK:  = vzxthw(r0)
+; CHECK: = vzxthw({{.*}})
diff --git a/test/CodeGen/Hexagon/intrinsics/xtype_pred.ll b/test/CodeGen/Hexagon/intrinsics/xtype_pred.ll
index 96e63d8d779..f06339b9a85 100644
--- a/test/CodeGen/Hexagon/intrinsics/xtype_pred.ll
+++ b/test/CodeGen/Hexagon/intrinsics/xtype_pred.ll
@@ -1,48 +1,51 @@
 ; RUN: llc -march=hexagon -O0 < %s | FileCheck %s
+; RUN: llc -march=hexagon -O0 < %s | FileCheck -check-prefix=CHECK-CALL %s
 ; Hexagon Programmer's Reference Manual 11.10.7 XTYPE/PRED
 
+; CHECK-CALL-NOT: call
+
 ; Compare byte
 declare i32 @llvm.hexagon.A4.cmpbgt(i32, i32)
 define i32 @A4_cmpbgt(i32 %a, i32 %b) {
   %z = call i32 @llvm.hexagon.A4.cmpbgt(i32 %a, i32 %b)
   ret i32 %z
 }
-; CHECK: p0 = cmpb.gt(r0, r1)
+; CHECK: = cmpb.gt({{.*}}, {{.*}})
 
 declare i32 @llvm.hexagon.A4.cmpbeq(i32, i32)
 define i32 @A4_cmpbeq(i32 %a, i32 %b) {
   %z = call i32 @llvm.hexagon.A4.cmpbeq(i32 %a, i32 %b)
   ret i32 %z
 }
-; CHECK: p0 = cmpb.eq(r0, r1)
+; CHECK: = cmpb.eq({{.*}}, {{.*}})
 
 declare i32 @llvm.hexagon.A4.cmpbgtu(i32, i32)
 define i32 @A4_cmpbgtu(i32 %a, i32 %b) {
   %z = call i32 @llvm.hexagon.A4.cmpbgtu(i32 %a, i32 %b)
   ret i32 %z
 }
-; CHECK: p0 = cmpb.gtu(r0, r1)
+; CHECK: = cmpb.gtu({{.*}}, {{.*}})
 
 declare i32 @llvm.hexagon.A4.cmpbgti(i32, i32)
 define i32 @A4_cmpbgti(i32 %a) {
   %z = call i32 @llvm.hexagon.A4.cmpbgti(i32 %a, i32 0)
   ret i32 %z
 }
-; CHECK: p0 = cmpb.gt(r0, #0)
+; CHECK: = cmpb.gt({{.*}}, #0)
 
 declare i32 @llvm.hexagon.A4.cmpbeqi(i32, i32)
 define i32 @A4_cmpbeqi(i32 %a) {
   %z = call i32 @llvm.hexagon.A4.cmpbeqi(i32 %a, i32 0)
   ret i32 %z
 }
-; CHECK: p0 = cmpb.eq(r0, #0)
+; CHECK: = cmpb.eq({{.*}}, #0)
 
 declare i32 @llvm.hexagon.A4.cmpbgtui(i32, i32)
 define i32 @A4_cmpbgtui(i32 %a) {
   %z = call i32 @llvm.hexagon.A4.cmpbgtui(i32 %a, i32 0)
   ret i32 %z
 }
-; CHECK: p0 = cmpb.gtu(r0, #0)
+; CHECK: = cmpb.gtu({{.*}}, #0)
 
 ; Compare half
 declare i32 @llvm.hexagon.A4.cmphgt(i32, i32)
@@ -50,42 +53,42 @@ define i32 @A4_cmphgt(i32 %a, i32 %b) {
   %z = call i32 @llvm.hexagon.A4.cmphgt(i32 %a, i32 %b)
   ret i32 %z
 }
-; CHECK: p0 = cmph.gt(r0, r1)
+; CHECK: = cmph.gt({{.*}}, {{.*}})
 
 declare i32 @llvm.hexagon.A4.cmpheq(i32, i32)
 define i32 @A4_cmpheq(i32 %a, i32 %b) {
   %z = call i32 @llvm.hexagon.A4.cmpheq(i32 %a, i32 %b)
   ret i32 %z
 }
-; CHECK: p0 = cmph.eq(r0, r1)
+; CHECK: = cmph.eq({{.*}}, {{.*}})
 
 declare i32 @llvm.hexagon.A4.cmphgtu(i32, i32)
 define i32 @A4_cmphgtu(i32 %a, i32 %b) {
   %z = call i32 @llvm.hexagon.A4.cmphgtu(i32 %a, i32 %b)
   ret i32 %z
 }
-; CHECK: p0 = cmph.gtu(r0, r1)
+; CHECK: = cmph.gtu({{.*}}, {{.*}})
 
 declare i32 @llvm.hexagon.A4.cmphgti(i32, i32)
 define i32 @A4_cmphgti(i32 %a) {
   %z = call i32 @llvm.hexagon.A4.cmphgti(i32 %a, i32 0)
   ret i32 %z
 }
-; CHECK: p0 = cmph.gt(r0, #0)
+; CHECK: = cmph.gt({{.*}}, #0)
 
 declare i32 @llvm.hexagon.A4.cmpheqi(i32, i32)
 define i32 @A4_cmpheqi(i32 %a) {
   %z = call i32 @llvm.hexagon.A4.cmpheqi(i32 %a, i32 0)
   ret i32 %z
 }
-; CHECK: p0 = cmph.eq(r0, #0)
+; CHECK: = cmph.eq({{.*}}, #0)
 
 declare i32 @llvm.hexagon.A4.cmphgtui(i32, i32)
 define i32 @A4_cmphgtui(i32 %a) {
   %z = call i32 @llvm.hexagon.A4.cmphgtui(i32 %a, i32 0)
   ret i32 %z
 }
-; CHECK: p0 = cmph.gtu(r0, #0)
+; CHECK: = cmph.gtu({{.*}}, #0)
 
 ; Compare doublewords
 declare i32 @llvm.hexagon.C2.cmpgtp(i64, i64)
@@ -93,21 +96,21 @@ define i32 @C2_cmpgtp(i64 %a, i64 %b) {
   %z = call i32 @llvm.hexagon.C2.cmpgtp(i64 %a, i64 %b)
   ret i32 %z
 }
-; CHECK: p0 = cmp.gt(r1:0, r3:2)
+; CHECK: = cmp.gt({{.*}}, {{.*}})
 
 declare i32 @llvm.hexagon.C2.cmpeqp(i64, i64)
 define i32 @C2_cmpeqp(i64 %a, i64 %b) {
   %z = call i32 @llvm.hexagon.C2.cmpeqp(i64 %a, i64 %b)
   ret i32 %z
 }
-; CHECK: p0 = cmp.eq(r1:0, r3:2)
+; CHECK: = cmp.eq({{.*}}, {{.*}})
 
 declare i32 @llvm.hexagon.C2.cmpgtup(i64, i64)
 define i32 @C2_cmpgtup(i64 %a, i64 %b) {
   %z = call i32 @llvm.hexagon.C2.cmpgtup(i64 %a, i64 %b)
   ret i32 %z
 }
-; CHECK: p0 = cmp.gtu(r1:0, r3:2)
+; CHECK: = cmp.gtu({{.*}}, {{.*}})
 
 ; Compare bitmask
 declare i32 @llvm.hexagon.C2.bitsclri(i32, i32)
@@ -115,42 +118,42 @@ define i32 @C2_bitsclri(i32 %a) {
   %z = call i32 @llvm.hexagon.C2.bitsclri(i32 %a, i32 0)
   ret i32 %z
 }
-; CHECK: p0 = bitsclr(r0, #0)
+; CHECK: = bitsclr({{.*}}, #0)
 
 declare i32 @llvm.hexagon.C4.nbitsclri(i32, i32)
 define i32 @C4_nbitsclri(i32 %a) {
   %z = call i32 @llvm.hexagon.C4.nbitsclri(i32 %a, i32 0)
   ret i32 %z
 }
-; CHECK: p0 = !bitsclr(r0, #0)
+; CHECK: = !bitsclr({{.*}}, #0)
 
 declare i32 @llvm.hexagon.C2.bitsset(i32, i32)
 define i32 @C2_bitsset(i32 %a, i32 %b) {
   %z = call i32 @llvm.hexagon.C2.bitsset(i32 %a, i32 %b)
   ret i32 %z
 }
-; CHECK: p0 = bitsset(r0, r1)
+; CHECK: = bitsset({{.*}}, {{.*}})
 
 declare i32 @llvm.hexagon.C4.nbitsset(i32, i32)
 define i32 @C4_nbitsset(i32 %a, i32 %b) {
   %z = call i32 @llvm.hexagon.C4.nbitsset(i32 %a, i32 %b)
   ret i32 %z
 }
-; CHECK: p0 = !bitsset(r0, r1)
+; CHECK: = !bitsset({{.*}}, {{.*}})
 
 declare i32 @llvm.hexagon.C2.bitsclr(i32, i32)
 define i32 @C2_bitsclr(i32 %a, i32 %b) {
   %z = call i32 @llvm.hexagon.C2.bitsclr(i32 %a, i32 %b)
   ret i32 %z
 }
-; CHECK: p0 = bitsclr(r0, r1)
+; CHECK: = bitsclr({{.*}}, {{.*}})
 
 declare i32 @llvm.hexagon.C4.nbitsclr(i32, i32)
 define i32 @C4_nbitsclr(i32 %a, i32 %b) {
   %z = call i32 @llvm.hexagon.C4.nbitsclr(i32 %a, i32 %b)
   ret i32 %z
 }
-; CHECK: p0 = !bitsclr(r0, r1)
+; CHECK: = !bitsclr({{.*}}, {{.*}})
 
 ; Mask generate from predicate
 declare i64 @llvm.hexagon.C2.mask(i32)
@@ -158,7 +161,7 @@ define i64 @C2_mask(i32 %a) {
   %z = call i64 @llvm.hexagon.C2.mask(i32 %a)
   ret i64 %z
 }
-; CHECK:  = mask(p0)
+; CHECK: = mask({{.*}})
 
 ; Check for TLB match
 declare i32 @llvm.hexagon.A4.tlbmatch(i64, i32)
@@ -166,7 +169,7 @@ define i32 @A4_tlbmatch(i64 %a, i32 %b) {
   %z = call i32 @llvm.hexagon.A4.tlbmatch(i64 %a, i32 %b)
   ret i32 %z
 }
-; CHECK: p0 = tlbmatch(r1:0, r2)
+; CHECK: = tlbmatch({{.*}}, {{.*}})
 
 ; Test bit
 declare i32 @llvm.hexagon.S2.tstbit.i(i32, i32)
@@ -174,28 +177,28 @@ define i32 @S2_tstbit_i(i32 %a) {
   %z = call i32 @llvm.hexagon.S2.tstbit.i(i32 %a, i32 0)
   ret i32 %z
 }
-; CHECK: p0 = tstbit(r0, #0)
+; CHECK: = tstbit({{.*}}, #0)
 
 declare i32 @llvm.hexagon.S4.ntstbit.i(i32, i32)
 define i32 @S4_ntstbit_i(i32 %a) {
   %z = call i32 @llvm.hexagon.S4.ntstbit.i(i32 %a, i32 0)
   ret i32 %z
 }
-; CHECK: p0 = !tstbit(r0, #0)
+; CHECK: = !tstbit({{.*}}, #0)
 
 declare i32 @llvm.hexagon.S2.tstbit.r(i32, i32)
 define i32 @S2_tstbit_r(i32 %a, i32 %b) {
   %z = call i32 @llvm.hexagon.S2.tstbit.r(i32 %a, i32 %b)
   ret i32 %z
 }
-; CHECK: p0 = tstbit(r0, r1)
+; CHECK: = tstbit({{.*}}, {{.*}})
 
 declare i32 @llvm.hexagon.S4.ntstbit.r(i32, i32)
 define i32 @S4_ntstbit_r(i32 %a, i32 %b) {
   %z = call i32 @llvm.hexagon.S4.ntstbit.r(i32 %a, i32 %b)
   ret i32 %z
 }
-; CHECK: p0 = !tstbit(r0, r1)
+; CHECK: = !tstbit({{.*}}, {{.*}})
 
 ; Vector compare halfwords
 declare i32 @llvm.hexagon.A2.vcmpheq(i64, i64)
@@ -203,42 +206,42 @@ define i32 @A2_vcmpheq(i64 %a, i64 %b) {
   %z = call i32 @llvm.hexagon.A2.vcmpheq(i64 %a, i64 %b)
   ret i32 %z
 }
-; CHECK: p0 = vcmph.eq(r1:0, r3:2)
+; CHECK: = vcmph.eq({{.*}}, {{.*}})
 
 declare i32 @llvm.hexagon.A2.vcmphgt(i64, i64)
 define i32 @A2_vcmphgt(i64 %a, i64 %b) {
   %z = call i32 @llvm.hexagon.A2.vcmphgt(i64 %a, i64 %b)
   ret i32 %z
 }
-; CHECK: p0 = vcmph.gt(r1:0, r3:2)
+; CHECK: = vcmph.gt({{.*}}, {{.*}})
 
 declare i32 @llvm.hexagon.A2.vcmphgtu(i64, i64)
 define i32 @A2_vcmphgtu(i64 %a, i64 %b) {
   %z = call i32 @llvm.hexagon.A2.vcmphgtu(i64 %a, i64 %b)
   ret i32 %z
 }
-; CHECK: p0 = vcmph.gtu(r1:0, r3:2)
+; CHECK: = vcmph.gtu({{.*}}, {{.*}})
 
 declare i32 @llvm.hexagon.A4.vcmpheqi(i64, i32)
 define i32 @A4_vcmpheqi(i64 %a) {
   %z = call i32 @llvm.hexagon.A4.vcmpheqi(i64 %a, i32 0)
   ret i32 %z
 }
-; CHECK: p0 = vcmph.eq(r1:0, #0)
+; CHECK: = vcmph.eq({{.*}}, #0)
 
 declare i32 @llvm.hexagon.A4.vcmphgti(i64, i32)
 define i32 @A4_vcmphgti(i64 %a) {
   %z = call i32 @llvm.hexagon.A4.vcmphgti(i64 %a, i32 0)
   ret i32 %z
 }
-; CHECK: p0 = vcmph.gt(r1:0, #0)
+; CHECK: = vcmph.gt({{.*}}, #0)
 
 declare i32 @llvm.hexagon.A4.vcmphgtui(i64, i32)
 define i32 @A4_vcmphgtui(i64 %a) {
   %z = call i32 @llvm.hexagon.A4.vcmphgtui(i64 %a, i32 0)
   ret i32 %z
 }
-; CHECK: p0 = vcmph.gtu(r1:0, #0)
+; CHECK: = vcmph.gtu({{.*}}, #0)
 
 ; Vector compare bytes for any match
 declare i32 @llvm.hexagon.A4.vcmpbeq.any(i64, i64)
@@ -246,7 +249,7 @@ define i32 @A4_vcmpbeq_any(i64 %a, i64 %b) {
   %z = call i32 @llvm.hexagon.A4.vcmpbeq.any(i64 %a, i64 %b)
   ret i32 %z
 }
-; CHECK: p0 = any8(vcmpb.eq(r1:0, r3:2))
+; CHECK: = any8(vcmpb.eq({{.*}}, {{.*}}))
 
 ; Vector compare bytes
 declare i32 @llvm.hexagon.A2.vcmpbeq(i64, i64)
@@ -254,42 +257,42 @@ define i32 @A2_vcmpbeq(i64 %a, i64 %b) {
   %z = call i32 @llvm.hexagon.A2.vcmpbeq(i64 %a, i64 %b)
   ret i32 %z
 }
-; CHECK: p0 = vcmpb.eq(r1:0, r3:2)
+; CHECK: = vcmpb.eq({{.*}}, {{.*}})
 
 declare i32 @llvm.hexagon.A2.vcmpbgtu(i64, i64)
 define i32 @A2_vcmpbgtu(i64 %a, i64 %b) {
   %z = call i32 @llvm.hexagon.A2.vcmpbgtu(i64 %a, i64 %b)
   ret i32 %z
 }
-; CHECK: p0 = vcmpb.gtu(r1:0, r3:2)
+; CHECK: = vcmpb.gtu({{.*}}, {{.*}})
 
 declare i32 @llvm.hexagon.A4.vcmpbgt(i64, i64)
 define i32 @A4_vcmpbgt(i64 %a, i64 %b) {
   %z = call i32 @llvm.hexagon.A4.vcmpbgt(i64 %a, i64 %b)
   ret i32 %z
 }
-; CHECK: p0 = vcmpb.gt(r1:0, r3:2)
+; CHECK: = vcmpb.gt({{.*}}, {{.*}})
 
 declare i32 @llvm.hexagon.A4.vcmpbeqi(i64, i32)
 define i32 @A4_vcmpbeqi(i64 %a) {
   %z = call i32 @llvm.hexagon.A4.vcmpbeqi(i64 %a, i32 0)
   ret i32 %z
 }
-; CHECK: p0 = vcmpb.eq(r1:0, #0)
+; CHECK: = vcmpb.eq({{.*}}, #0)
 
 declare i32 @llvm.hexagon.A4.vcmpbgti(i64, i32)
 define i32 @A4_vcmpbgti(i64 %a) {
   %z = call i32 @llvm.hexagon.A4.vcmpbgti(i64 %a, i32 0)
   ret i32 %z
 }
-; CHECK: p0 = vcmpb.gt(r1:0, #0)
+; CHECK: = vcmpb.gt({{.*}}, #0)
 
 declare i32 @llvm.hexagon.A4.vcmpbgtui(i64, i32)
 define i32 @A4_vcmpbgtui(i64 %a) {
   %z = call i32 @llvm.hexagon.A4.vcmpbgtui(i64 %a, i32 0)
   ret i32 %z
 }
-; CHECK: p0 = vcmpb.gtu(r1:0, #0)
+; CHECK: = vcmpb.gtu({{.*}}, #0)
 
 ; Vector compare words
 declare i32 @llvm.hexagon.A2.vcmpweq(i64, i64)
@@ -297,42 +300,42 @@ define i32 @A2_vcmpweq(i64 %a, i64 %b) {
   %z = call i32 @llvm.hexagon.A2.vcmpweq(i64 %a, i64 %b)
   ret i32 %z
 }
-; CHECK: p0 = vcmpw.eq(r1:0, r3:2)
+; CHECK: = vcmpw.eq({{.*}}, {{.*}})
 
 declare i32 @llvm.hexagon.A2.vcmpwgt(i64, i64)
 define i32 @A2_vcmpwgt(i64 %a, i64 %b) {
   %z = call i32 @llvm.hexagon.A2.vcmpwgt(i64 %a, i64 %b)
   ret i32 %z
 }
-; CHECK: p0 = vcmpw.gt(r1:0, r3:2)
+; CHECK: = vcmpw.gt({{.*}}, {{.*}})
 
 declare i32 @llvm.hexagon.A2.vcmpwgtu(i64, i64)
 define i32 @A2_vcmpwgtu(i64 %a, i64 %b) {
   %z = call i32 @llvm.hexagon.A2.vcmpwgtu(i64 %a, i64 %b)
   ret i32 %z
 }
-; CHECK: p0 = vcmpw.gtu(r1:0, r3:2)
+; CHECK: = vcmpw.gtu({{.*}}, {{.*}})
 
 declare i32 @llvm.hexagon.A4.vcmpweqi(i64, i32)
 define i32 @A4_vcmpweqi(i64 %a) {
   %z = call i32 @llvm.hexagon.A4.vcmpweqi(i64 %a, i32 0)
   ret i32 %z
 }
-; CHECK: p0 = vcmpw.eq(r1:0, #0)
+; CHECK: = vcmpw.eq({{.*}}, #0)
 
 declare i32 @llvm.hexagon.A4.vcmpwgti(i64, i32)
 define i32 @A4_vcmpwgti(i64 %a) {
   %z = call i32 @llvm.hexagon.A4.vcmpwgti(i64 %a, i32 0)
   ret i32 %z
 }
-; CHECK: p0 = vcmpw.gt(r1:0, #0)
+; CHECK: = vcmpw.gt({{.*}}, #0)
 
 declare i32 @llvm.hexagon.A4.vcmpwgtui(i64, i32)
 define i32 @A4_vcmpwgtui(i64 %a) {
   %z = call i32 @llvm.hexagon.A4.vcmpwgtui(i64 %a, i32 0)
   ret i32 %z
 }
-; CHECK: p0 = vcmpw.gtu(r1:0, #0)
+; CHECK: = vcmpw.gtu({{.*}}, #0)
 
 ; Viterbi pack even and odd predicate bitsclr
 declare i32 @llvm.hexagon.C2.vitpack(i32, i32)
@@ -340,7 +343,7 @@ define i32 @C2_vitpack(i32 %a, i32 %b) {
   %z = call i32 @llvm.hexagon.C2.vitpack(i32 %a, i32 %b)
   ret i32 %z
 }
-; CHECK: r0 = vitpack(p1, p0)
+; CHECK: = vitpack({{.*}}, {{.*}})
 
 ; Vector mux
 declare i64 @llvm.hexagon.C2.vmux(i32, i64, i64)
@@ -348,4 +351,4 @@ define i64 @C2_vmux(i32 %a, i64 %b, i64 %c) {
   %z = call i64 @llvm.hexagon.C2.vmux(i32 %a, i64 %b, i64 %c)
   ret i64 %z
 }
-; CHECK:  = vmux(p0, r3:2, r5:4)
+; CHECK: = vmux({{.*}}, {{.*}}, {{.*}})
diff --git a/test/CodeGen/Hexagon/intrinsics/xtype_shift.ll b/test/CodeGen/Hexagon/intrinsics/xtype_shift.ll
index c84999bf94f..1a65f44c195 100644
--- a/test/CodeGen/Hexagon/intrinsics/xtype_shift.ll
+++ b/test/CodeGen/Hexagon/intrinsics/xtype_shift.ll
@@ -1,48 +1,51 @@
 ; RUN: llc -march=hexagon -O0 < %s | FileCheck %s
+; RUN: llc -march=hexagon -O0 < %s | FileCheck -check-prefix=CHECK-CALL %s
 ; Hexagon Programmer's Reference Manual 11.10.8 XTYPE/SHIFT
 
+; CHECK-CALL-NOT: call
+
 ; Shift by immediate
 declare i64 @llvm.hexagon.S2.asr.i.p(i64, i32)
 define i64 @S2_asr_i_p(i64 %a) {
   %z = call i64 @llvm.hexagon.S2.asr.i.p(i64 %a, i32 0)
   ret i64 %z
 }
-; CHECK: r1:0 = asr(r1:0, #0)
+; CHECK: = asr({{.*}}, #0)
 
 declare i64 @llvm.hexagon.S2.lsr.i.p(i64, i32)
 define i64 @S2_lsr_i_p(i64 %a) {
   %z = call i64 @llvm.hexagon.S2.lsr.i.p(i64 %a, i32 0)
   ret i64 %z
 }
-; CHECK: r1:0 = lsr(r1:0, #0)
+; CHECK: = lsr({{.*}}, #0)
 
 declare i64 @llvm.hexagon.S2.asl.i.p(i64, i32)
 define i64 @S2_asl_i_p(i64 %a) {
   %z = call i64 @llvm.hexagon.S2.asl.i.p(i64 %a, i32 0)
   ret i64 %z
 }
-; CHECK: r1:0 = asl(r1:0, #0)
+; CHECK: = asl({{.*}}, #0)
 
 declare i32 @llvm.hexagon.S2.asr.i.r(i32, i32)
 define i32 @S2_asr_i_r(i32 %a) {
   %z = call i32 @llvm.hexagon.S2.asr.i.r(i32 %a, i32 0)
   ret i32 %z
 }
-; CHECK: r0 = asr(r0, #0)
+; CHECK: = asr({{.*}}, #0)
 
 declare i32 @llvm.hexagon.S2.lsr.i.r(i32, i32)
 define i32 @S2_lsr_i_r(i32 %a) {
   %z = call i32 @llvm.hexagon.S2.lsr.i.r(i32 %a, i32 0)
   ret i32 %z
 }
-; CHECK: r0 = lsr(r0, #0)
+; CHECK: = lsr({{.*}}, #0)
 
 declare i32 @llvm.hexagon.S2.asl.i.r(i32, i32)
 define i32 @S2_asl_i_r(i32 %a) {
   %z = call i32 @llvm.hexagon.S2.asl.i.r(i32 %a, i32 0)
   ret i32 %z
 }
-; CHECK: r0 = asl(r0, #0)
+; CHECK: = asl({{.*}}, #0)
 
 ; Shift by immediate and accumulate
 declare i64 @llvm.hexagon.S2.asr.i.p.nac(i64, i64, i32)
@@ -50,84 +53,84 @@ define i64 @S2_asr_i_p_nac(i64 %a, i64 %b) {
   %z = call i64 @llvm.hexagon.S2.asr.i.p.nac(i64 %a, i64 %b, i32 0)
   ret i64 %z
 }
-; CHECK: r1:0 -= asr(r3:2, #0)
+; CHECK: -= asr({{.*}}, #0)
 
 declare i64 @llvm.hexagon.S2.lsr.i.p.nac(i64, i64, i32)
 define i64 @S2_lsr_i_p_nac(i64 %a, i64 %b) {
   %z = call i64 @llvm.hexagon.S2.lsr.i.p.nac(i64 %a, i64 %b, i32 0)
   ret i64 %z
 }
-; CHECK: r1:0 -= lsr(r3:2, #0)
+; CHECK: -= lsr({{.*}}, #0)
 
 declare i64 @llvm.hexagon.S2.asl.i.p.nac(i64, i64, i32)
 define i64 @S2_asl_i_p_nac(i64 %a, i64 %b) {
   %z = call i64 @llvm.hexagon.S2.asl.i.p.nac(i64 %a, i64 %b, i32 0)
   ret i64 %z
 }
-; CHECK: r1:0 -= asl(r3:2, #0)
+; CHECK: -= asl({{.*}}, #0)
 
 declare i64 @llvm.hexagon.S2.asr.i.p.acc(i64, i64, i32)
 define i64 @S2_asr_i_p_acc(i64 %a, i64 %b) {
   %z = call i64 @llvm.hexagon.S2.asr.i.p.acc(i64 %a, i64 %b, i32 0)
   ret i64 %z
 }
-; CHECK: r1:0 += asr(r3:2, #0)
+; CHECK: += asr({{.*}}, #0)
 
 declare i64 @llvm.hexagon.S2.lsr.i.p.acc(i64, i64, i32)
 define i64 @S2_lsr_i_p_acc(i64 %a, i64 %b) {
   %z = call i64 @llvm.hexagon.S2.lsr.i.p.acc(i64 %a, i64 %b, i32 0)
   ret i64 %z
 }
-; CHECK: r1:0 += lsr(r3:2, #0)
+; CHECK: += lsr({{.*}}, #0)
 
 declare i64 @llvm.hexagon.S2.asl.i.p.acc(i64, i64, i32)
 define i64 @S2_asl_i_p_acc(i64 %a, i64 %b) {
   %z = call i64 @llvm.hexagon.S2.asl.i.p.acc(i64 %a, i64 %b, i32 0)
   ret i64 %z
 }
-; CHECK: r1:0 += asl(r3:2, #0)
+; CHECK: += asl({{.*}}, #0)
 
 declare i32 @llvm.hexagon.S2.asr.i.r.nac(i32, i32, i32)
 define i32 @S2_asr_i_r_nac(i32 %a, i32 %b) {
   %z = call i32 @llvm.hexagon.S2.asr.i.r.nac(i32 %a, i32 %b, i32 0)
   ret i32 %z
 }
-; CHECK: r0 -= asr(r1, #0)
+; CHECK: -= asr({{.*}}, #0)
 
 declare i32 @llvm.hexagon.S2.lsr.i.r.nac(i32, i32, i32)
 define i32 @S2_lsr_i_r_nac(i32 %a, i32 %b) {
   %z = call i32 @llvm.hexagon.S2.lsr.i.r.nac(i32 %a, i32 %b, i32 0)
   ret i32 %z
 }
-; CHECK: r0 -= lsr(r1, #0)
+; CHECK: -= lsr({{.*}}, #0)
 
 declare i32 @llvm.hexagon.S2.asl.i.r.nac(i32, i32, i32)
 define i32 @S2_asl_i_r_nac(i32 %a, i32 %b) {
   %z = call i32 @llvm.hexagon.S2.asl.i.r.nac(i32 %a, i32 %b, i32 0)
   ret i32 %z
 }
-; CHECK: r0 -= asl(r1, #0)
+; CHECK: -= asl({{.*}}, #0)
 
 declare i32 @llvm.hexagon.S2.asr.i.r.acc(i32, i32, i32)
 define i32 @S2_asr_i_r_acc(i32 %a, i32 %b) {
   %z = call i32 @llvm.hexagon.S2.asr.i.r.acc(i32 %a, i32 %b, i32 0)
   ret i32 %z
 }
-; CHECK: r0 += asr(r1, #0)
+; CHECK: += asr({{.*}}, #0)
 
 declare i32 @llvm.hexagon.S2.lsr.i.r.acc(i32, i32, i32)
 define i32 @S2_lsr_i_r_acc(i32 %a, i32 %b) {
   %z = call i32 @llvm.hexagon.S2.lsr.i.r.acc(i32 %a, i32 %b, i32 0)
   ret i32 %z
 }
-; CHECK: r0 += lsr(r1, #0)
+; CHECK: += lsr({{.*}}, #0)
 
 declare i32 @llvm.hexagon.S2.asl.i.r.acc(i32, i32, i32)
 define i32 @S2_asl_i_r_acc(i32 %a, i32 %b) {
   %z = call i32 @llvm.hexagon.S2.asl.i.r.acc(i32 %a, i32 %b, i32 0)
   ret i32 %z
 }
-; CHECK: r0 += asl(r1, #0)
+; CHECK: += asl({{.*}}, #0)
 
 ; Shift by immediate and add
 declare i32 @llvm.hexagon.S4.addi.asl.ri(i32, i32, i32)
@@ -135,35 +138,35 @@ define i32 @S4_addi_asl_ri(i32 %a) {
   %z = call i32 @llvm.hexagon.S4.addi.asl.ri(i32 0, i32 %a, i32 0)
   ret i32 %z
 }
-; CHECK: r0 = add(#0, asl(r0, #0))
+; CHECK: = add(#0, asl({{.*}}, #0))
 
 declare i32 @llvm.hexagon.S4.subi.asl.ri(i32, i32, i32)
 define i32 @S4_subi_asl_ri(i32 %a) {
   %z = call i32 @llvm.hexagon.S4.subi.asl.ri(i32 0, i32 %a, i32 0)
   ret i32 %z
 }
-; CHECK: r0 = sub(#0, asl(r0, #0))
+; CHECK: = sub(#0, asl({{.*}}, #0))
 
 declare i32 @llvm.hexagon.S4.addi.lsr.ri(i32, i32, i32)
 define i32 @S4_addi_lsr_ri(i32 %a) {
   %z = call i32 @llvm.hexagon.S4.addi.lsr.ri(i32 0, i32 %a, i32 0)
   ret i32 %z
 }
-; CHECK: r0 = add(#0, lsr(r0, #0))
+; CHECK: = add(#0, lsr({{.*}}, #0))
 
 declare i32 @llvm.hexagon.S4.subi.lsr.ri(i32, i32, i32)
 define i32 @S4_subi_lsr_ri(i32 %a) {
   %z = call i32 @llvm.hexagon.S4.subi.lsr.ri(i32 0, i32 %a, i32 0)
   ret i32 %z
 }
-; CHECK: r0 = sub(#0, lsr(r0, #0))
+; CHECK: = sub(#0, lsr({{.*}}, #0))
 
 declare i32 @llvm.hexagon.S2.addasl.rrri(i32, i32, i32)
 define i32 @S2_addasl_rrri(i32 %a, i32 %b) {
   %z = call i32 @llvm.hexagon.S2.addasl.rrri(i32 %a, i32 %b, i32 0)
   ret i32 %z
 }
-; CHECK: r0 = addasl(r0, r1, #0)
+; CHECK: = addasl({{.*}}, {{.*}}, #0)
 
 ; Shift by immediate and logical
 declare i64 @llvm.hexagon.S2.asr.i.p.and(i64, i64, i32)
@@ -171,140 +174,140 @@ define i64 @S2_asr_i_p_and(i64 %a, i64 %b) {
   %z = call i64 @llvm.hexagon.S2.asr.i.p.and(i64 %a, i64 %b, i32 0)
   ret i64 %z
 }
-; CHECK: r1:0 &= asr(r3:2, #0)
+; CHECK: &= asr({{.*}}, #0)
 
 declare i64 @llvm.hexagon.S2.lsr.i.p.and(i64, i64, i32)
 define i64 @S2_lsr_i_p_and(i64 %a, i64 %b) {
   %z = call i64 @llvm.hexagon.S2.lsr.i.p.and(i64 %a, i64 %b, i32 0)
   ret i64 %z
 }
-; CHECK: r1:0 &= lsr(r3:2, #0)
+; CHECK: {{.*}} &= lsr({{.*}}, #0)
 
 declare i64 @llvm.hexagon.S2.asl.i.p.and(i64, i64, i32)
 define i64 @S2_asl_i_p_and(i64 %a, i64 %b) {
   %z = call i64 @llvm.hexagon.S2.asl.i.p.and(i64 %a, i64 %b, i32 0)
   ret i64 %z
 }
-; CHECK: r1:0 &= asl(r3:2, #0)
+; CHECK: &= asl({{.*}}, #0)
 
 declare i64 @llvm.hexagon.S2.asr.i.p.or(i64, i64, i32)
 define i64 @S2_asr_i_p_or(i64 %a, i64 %b) {
   %z = call i64 @llvm.hexagon.S2.asr.i.p.or(i64 %a, i64 %b, i32 0)
   ret i64 %z
 }
-; CHECK: r1:0 |= asr(r3:2, #0)
+; CHECK: |= asr({{.*}}, #0)
 
 declare i64 @llvm.hexagon.S2.lsr.i.p.or(i64, i64, i32)
 define i64 @S2_lsr_i_p_or(i64 %a, i64 %b) {
   %z = call i64 @llvm.hexagon.S2.lsr.i.p.or(i64 %a, i64 %b, i32 0)
   ret i64 %z
 }
-; CHECK: r1:0 |= lsr(r3:2, #0)
+; CHECK: |= lsr({{.*}}, #0)
 
 declare i64 @llvm.hexagon.S2.asl.i.p.or(i64, i64, i32)
 define i64 @S2_asl_i_p_or(i64 %a, i64 %b) {
   %z = call i64 @llvm.hexagon.S2.asl.i.p.or(i64 %a, i64 %b, i32 0)
   ret i64 %z
 }
-; CHECK: r1:0 |= asl(r3:2, #0)
+; CHECK: |= asl({{.*}}, #0)
 
 declare i64 @llvm.hexagon.S2.lsr.i.p.xacc(i64, i64, i32)
 define i64 @S2_lsr_i_p_xacc(i64 %a, i64 %b) {
   %z = call i64 @llvm.hexagon.S2.lsr.i.p.xacc(i64 %a, i64 %b, i32 0)
   ret i64 %z
 }
-; CHECK: r1:0 ^= lsr(r3:2, #0)
+; CHECK: ^= lsr({{.*}}, #0)
 
 declare i64 @llvm.hexagon.S2.asl.i.p.xacc(i64, i64, i32)
 define i64 @S2_asl_i_p_xacc(i64 %a, i64 %b) {
   %z = call i64 @llvm.hexagon.S2.asl.i.p.xacc(i64 %a, i64 %b, i32 0)
   ret i64 %z
 }
-; CHECK: r1:0 ^= asl(r3:2, #0)
+; CHECK: ^= asl({{.*}}, #0)
 
 declare i32 @llvm.hexagon.S2.asr.i.r.and(i32, i32, i32)
 define i32 @S2_asr_i_r_and(i32 %a, i32 %b) {
   %z = call i32 @llvm.hexagon.S2.asr.i.r.and(i32 %a, i32 %b, i32 0)
   ret i32 %z
 }
-; CHECK: r0 &= asr(r1, #0)
+; CHECK: &= asr({{.*}}, #0)
 
 declare i32 @llvm.hexagon.S2.lsr.i.r.and(i32, i32, i32)
 define i32 @S2_lsr_i_r_and(i32 %a, i32 %b) {
   %z = call i32 @llvm.hexagon.S2.lsr.i.r.and(i32 %a, i32 %b, i32 0)
   ret i32 %z
 }
-; CHECK: r0 &= lsr(r1, #0)
+; CHECK: &= lsr({{.*}}, #0)
 
 declare i32 @llvm.hexagon.S2.asl.i.r.and(i32, i32, i32)
 define i32 @S2_asl_i_r_and(i32 %a, i32 %b) {
   %z = call i32 @llvm.hexagon.S2.asl.i.r.and(i32 %a, i32 %b, i32 0)
   ret i32 %z
 }
-; CHECK: r0 &= asl(r1, #0)
+; CHECK: &= asl({{.*}}, #0)
 
 declare i32 @llvm.hexagon.S2.asr.i.r.or(i32, i32, i32)
 define i32 @S2_asr_i_r_or(i32 %a, i32 %b) {
   %z = call i32 @llvm.hexagon.S2.asr.i.r.or(i32 %a, i32 %b, i32 0)
   ret i32 %z
 }
-; CHECK: r0 |= asr(r1, #0)
+; CHECK: |= asr({{.*}}, #0)
 
 declare i32 @llvm.hexagon.S2.lsr.i.r.or(i32, i32, i32)
 define i32 @S2_lsr_i_r_or(i32 %a, i32 %b) {
   %z = call i32 @llvm.hexagon.S2.lsr.i.r.or(i32 %a, i32 %b, i32 0)
   ret i32 %z
 }
-; CHECK: r0 |= lsr(r1, #0)
+; CHECK: |= lsr({{.*}}, #0)
 
 declare i32 @llvm.hexagon.S2.asl.i.r.or(i32, i32, i32)
 define i32 @S2_asl_i_r_or(i32%a, i32 %b) {
   %z = call i32 @llvm.hexagon.S2.asl.i.r.or(i32 %a, i32 %b, i32 0)
   ret i32 %z
 }
-; CHECK: r0 |= asl(r1, #0)
+; CHECK: |= asl({{.*}}, #0)
 
 declare i32 @llvm.hexagon.S2.lsr.i.r.xacc(i32, i32, i32)
 define i32 @S2_lsr_i_r_xacc(i32 %a, i32 %b) {
   %z = call i32 @llvm.hexagon.S2.lsr.i.r.xacc(i32%a, i32 %b, i32 0)
   ret i32 %z
 }
-; CHECK: r0 ^= lsr(r1, #0)
+; CHECK: ^= lsr({{.*}}, #0)
 
 declare i32 @llvm.hexagon.S2.asl.i.r.xacc(i32, i32, i32)
 define i32 @S2_asl_i_r_xacc(i32 %a, i32 %b) {
   %z = call i32 @llvm.hexagon.S2.asl.i.r.xacc(i32 %a, i32 %b, i32 0)
   ret i32 %z
 }
-; CHECK: r0 ^= asl(r1, #0)
+; CHECK: ^= asl({{.*}}, #0)
 
 declare i32 @llvm.hexagon.S4.andi.asl.ri(i32, i32, i32)
 define i32 @S4_andi_asl_ri(i32 %a) {
   %z = call i32 @llvm.hexagon.S4.andi.asl.ri(i32 0, i32 %a, i32 0)
   ret i32 %z
 }
-; CHECK: r0 = and(#0, asl(r0, #0))
+; CHECK: = and(#0, asl({{.*}}, #0))
 
 declare i32 @llvm.hexagon.S4.ori.asl.ri(i32, i32, i32)
 define i32 @S4_ori_asl_ri(i32 %a) {
   %z = call i32 @llvm.hexagon.S4.ori.asl.ri(i32 0, i32 %a, i32 0)
   ret i32 %z
 }
-; CHECK: r0 = or(#0, asl(r0, #0))
+; CHECK: = or(#0, asl({{.*}}, #0))
 
 declare i32 @llvm.hexagon.S4.andi.lsr.ri(i32, i32, i32)
 define i32 @S4_andi_lsr_ri(i32 %a) {
   %z = call i32 @llvm.hexagon.S4.andi.lsr.ri(i32 0, i32 %a, i32 0)
   ret i32 %z
 }
-; CHECK: r0 = and(#0, lsr(r0, #0))
+; CHECK: = and(#0, lsr({{.*}}, #0))
 
 declare i32 @llvm.hexagon.S4.ori.lsr.ri(i32, i32, i32)
 define i32 @S4_ori_lsr_ri(i32 %a) {
   %z = call i32 @llvm.hexagon.S4.ori.lsr.ri(i32 0, i32 %a, i32 0)
   ret i32 %z
 }
-; CHECK: r0 = or(#0, lsr(r0, #0))
+; CHECK: = or(#0, lsr({{.*}}, #0))
 
 ; Shift right by immediate with rounding
 declare i64 @llvm.hexagon.S2.asr.i.p.rnd(i64, i32)
@@ -312,14 +315,14 @@ define i64 @S2_asr_i_p_rnd(i64 %a) {
   %z = call i64 @llvm.hexagon.S2.asr.i.p.rnd(i64 %a, i32 0)
   ret i64 %z
 }
-; CHECK: r1:0 = asr(r1:0, #0):rnd
+; CHECK: = asr({{.*}}, #0):rnd
 
 declare i32 @llvm.hexagon.S2.asr.i.r.rnd(i32, i32)
 define i32 @S2_asr_i_r_rnd(i32 %a) {
   %z = call i32 @llvm.hexagon.S2.asr.i.r.rnd(i32 %a, i32 0)
   ret i32 %z
 }
-; CHECK: r0 = asr(r0, #0):rnd
+; CHECK: = asr({{.*}}, #0):rnd
 
 ; Shift left by immediate with saturation
 declare i32 @llvm.hexagon.S2.asl.i.r.sat(i32, i32)
@@ -327,7 +330,7 @@ define i32 @S2_asl_i_r_sat(i32 %a) {
   %z = call i32 @llvm.hexagon.S2.asl.i.r.sat(i32 %a, i32 0)
   ret i32 %z
 }
-; CHECK: r0 = asl(r0, #0):sat
+; CHECK: = asl({{.*}}, #0):sat
 
 ; Shift by register
 declare i64 @llvm.hexagon.S2.asr.r.p(i64, i32)
@@ -335,63 +338,63 @@ define i64 @S2_asr_r_p(i64 %a, i32 %b) {
   %z = call i64 @llvm.hexagon.S2.asr.r.p(i64 %a, i32 %b)
   ret i64 %z
 }
-; CHECK: r1:0 = asr(r1:0, r2)
+; CHECK: = asr({{.*}}, {{.*}})
 
 declare i64 @llvm.hexagon.S2.lsr.r.p(i64, i32)
 define i64 @S2_lsr_r_p(i64 %a, i32 %b) {
   %z = call i64 @llvm.hexagon.S2.lsr.r.p(i64 %a, i32 %b)
   ret i64 %z
 }
-; CHECK: r1:0 = lsr(r1:0, r2)
+; CHECK: = lsr({{.*}}, {{.*}})
 
 declare i64 @llvm.hexagon.S2.asl.r.p(i64, i32)
 define i64 @S2_asl_r_p(i64 %a, i32 %b) {
   %z = call i64 @llvm.hexagon.S2.asl.r.p(i64 %a, i32 %b)
   ret i64 %z
 }
-; CHECK: r1:0 = asl(r1:0, r2)
+; CHECK: = asl({{.*}}, {{.*}})
 
 declare i64 @llvm.hexagon.S2.lsl.r.p(i64, i32)
 define i64 @S2_lsl_r_p(i64 %a, i32 %b) {
   %z = call i64 @llvm.hexagon.S2.lsl.r.p(i64 %a, i32 %b)
   ret i64 %z
 }
-; CHECK: r1:0 = lsl(r1:0, r2)
+; CHECK: = lsl({{.*}}, {{.*}})
 
 declare i32 @llvm.hexagon.S2.asr.r.r(i32, i32)
 define i32 @S2_asr_r_r(i32 %a, i32 %b) {
   %z = call i32 @llvm.hexagon.S2.asr.r.r(i32 %a, i32 %b)
   ret i32 %z
 }
-; CHECK: r0 = asr(r0, r1)
+; CHECK: = asr({{.*}}, {{.*}})
 
 declare i32 @llvm.hexagon.S2.lsr.r.r(i32, i32)
 define i32 @S2_lsr_r_r(i32 %a, i32 %b) {
   %z = call i32 @llvm.hexagon.S2.lsr.r.r(i32 %a, i32 %b)
   ret i32 %z
 }
-; CHECK: r0 = lsr(r0, r1)
+; CHECK: = lsr({{.*}}, {{.*}})
 
 declare i32 @llvm.hexagon.S2.asl.r.r(i32, i32)
 define i32 @S2_asl_r_r(i32 %a, i32 %b) {
   %z = call i32 @llvm.hexagon.S2.asl.r.r(i32 %a, i32 %b)
   ret i32 %z
 }
-; CHECK: r0 = asl(r0, r1)
+; CHECK: = asl({{.*}}, {{.*}})
 
 declare i32 @llvm.hexagon.S2.lsl.r.r(i32, i32)
 define i32 @S2_lsl_r_r(i32 %a, i32 %b) {
   %z = call i32 @llvm.hexagon.S2.lsl.r.r(i32 %a, i32 %b)
   ret i32 %z
 }
-; CHECK: r0 = lsl(r0, r1)
+; CHECK: = lsl({{.*}}, {{.*}})
 
 declare i32 @llvm.hexagon.S4.lsli(i32, i32)
 define i32 @S4_lsli(i32 %a) {
   %z = call i32 @llvm.hexagon.S4.lsli(i32 0, i32 %a)
   ret i32 %z
 }
-; CHECK: r0 = lsl(#0, r0)
+; CHECK: = lsl(#0, {{.*}})
 
 ; Shift by register and accumulate
 declare i64 @llvm.hexagon.S2.asr.r.p.nac(i64, i64, i32)
@@ -399,112 +402,112 @@ define i64 @S2_asr_r_p_nac(i64 %a, i64 %b, i32 %c) {
   %z = call i64 @llvm.hexagon.S2.asr.r.p.nac(i64 %a, i64 %b, i32 %c)
   ret i64 %z
 }
-; CHECK: r1:0 -= asr(r3:2, r4)
+; CHECK: -= asr({{.*}}, r4)
 
 declare i64 @llvm.hexagon.S2.lsr.r.p.nac(i64, i64, i32)
 define i64 @S2_lsr_r_p_nac(i64 %a, i64 %b, i32 %c) {
   %z = call i64 @llvm.hexagon.S2.lsr.r.p.nac(i64 %a, i64 %b, i32 %c)
   ret i64 %z
 }
-; CHECK: r1:0 -= lsr(r3:2, r4)
+; CHECK: -= lsr({{.*}}, r4)
 
 declare i64 @llvm.hexagon.S2.asl.r.p.nac(i64, i64, i32)
 define i64 @S2_asl_r_p_nac(i64 %a, i64 %b, i32 %c) {
   %z = call i64 @llvm.hexagon.S2.asl.r.p.nac(i64 %a, i64 %b, i32 %c)
   ret i64 %z
 }
-; CHECK: r1:0 -= asl(r3:2, r4)
+; CHECK: -= asl({{.*}}, r4)
 
 declare i64 @llvm.hexagon.S2.lsl.r.p.nac(i64, i64, i32)
 define i64 @S2_lsl_r_p_nac(i64 %a, i64 %b, i32 %c) {
   %z = call i64 @llvm.hexagon.S2.lsl.r.p.nac(i64 %a, i64 %b, i32 %c)
   ret i64 %z
 }
-; CHECK: r1:0 -= lsl(r3:2, r4)
+; CHECK: -= lsl({{.*}}, r4)
 
 declare i64 @llvm.hexagon.S2.asr.r.p.acc(i64, i64, i32)
 define i64 @S2_asr_r_p_acc(i64 %a, i64 %b, i32 %c) {
   %z = call i64 @llvm.hexagon.S2.asr.r.p.acc(i64 %a, i64 %b, i32 %c)
   ret i64 %z
 }
-; CHECK: r1:0 += asr(r3:2, r4)
+; CHECK: += asr({{.*}}, r4)
 
 declare i64 @llvm.hexagon.S2.lsr.r.p.acc(i64, i64, i32)
 define i64 @S2_lsr_r_p_acc(i64 %a, i64 %b, i32 %c) {
   %z = call i64 @llvm.hexagon.S2.lsr.r.p.acc(i64 %a, i64 %b, i32 %c)
   ret i64 %z
 }
-; CHECK: r1:0 += lsr(r3:2, r4)
+; CHECK: += lsr({{.*}}, r4)
 
 declare i64 @llvm.hexagon.S2.asl.r.p.acc(i64, i64, i32)
 define i64 @S2_asl_r_p_acc(i64 %a, i64 %b, i32 %c) {
   %z = call i64 @llvm.hexagon.S2.asl.r.p.acc(i64 %a, i64 %b, i32 %c)
   ret i64 %z
 }
-; CHECK: r1:0 += asl(r3:2, r4)
+; CHECK: += asl({{.*}}, r4)
 
 declare i64 @llvm.hexagon.S2.lsl.r.p.acc(i64, i64, i32)
 define i64 @S2_lsl_r_p_acc(i64 %a, i64 %b, i32 %c) {
   %z = call i64 @llvm.hexagon.S2.lsl.r.p.acc(i64 %a, i64 %b, i32 %c)
   ret i64 %z
 }
-; CHECK: r1:0 += lsl(r3:2, r4)
+; CHECK: += lsl({{.*}}, r4)
 
 declare i32 @llvm.hexagon.S2.asr.r.r.nac(i32, i32, i32)
 define i32 @S2_asr_r_r_nac(i32 %a, i32 %b, i32 %c) {
   %z = call i32 @llvm.hexagon.S2.asr.r.r.nac(i32 %a, i32 %b, i32 %c)
   ret i32 %z
 }
-; CHECK: r0 -= asr(r1, r2)
+; CHECK: -= asr({{.*}}, {{.*}})
 
 declare i32 @llvm.hexagon.S2.lsr.r.r.nac(i32, i32, i32)
 define i32 @S2_lsr_r_r_nac(i32 %a, i32 %b, i32 %c) {
   %z = call i32 @llvm.hexagon.S2.lsr.r.r.nac(i32 %a, i32 %b, i32 %c)
   ret i32 %z
 }
-; CHECK: r0 -= lsr(r1, r2)
+; CHECK: -= lsr({{.*}}, {{.*}})
 
 declare i32 @llvm.hexagon.S2.asl.r.r.nac(i32, i32, i32)
 define i32 @S2_asl_r_r_nac(i32 %a, i32 %b, i32 %c) {
   %z = call i32 @llvm.hexagon.S2.asl.r.r.nac(i32 %a, i32 %b, i32 %c)
   ret i32 %z
 }
-; CHECK: r0 -= asl(r1, r2)
+; CHECK: -= asl({{.*}}, {{.*}})
 
 declare i32 @llvm.hexagon.S2.lsl.r.r.nac(i32, i32, i32)
 define i32 @S2_lsl_r_r_nac(i32 %a, i32 %b, i32 %c) {
   %z = call i32 @llvm.hexagon.S2.lsl.r.r.nac(i32 %a, i32 %b, i32 %c)
   ret i32 %z
 }
-; CHECK: r0 -= lsl(r1, r2)
+; CHECK: -= lsl({{.*}}, {{.*}})
 
 declare i32 @llvm.hexagon.S2.asr.r.r.acc(i32, i32, i32)
 define i32 @S2_asr_r_r_acc(i32 %a, i32 %b, i32 %c) {
   %z = call i32 @llvm.hexagon.S2.asr.r.r.acc(i32 %a, i32 %b, i32 %c)
   ret i32 %z
 }
-; CHECK: r0 += asr(r1, r2)
+; CHECK: += asr({{.*}}, {{.*}})
 
 declare i32 @llvm.hexagon.S2.lsr.r.r.acc(i32, i32, i32)
 define i32 @S2_lsr_r_r_acc(i32 %a, i32 %b, i32 %c) {
   %z = call i32 @llvm.hexagon.S2.lsr.r.r.acc(i32 %a, i32 %b, i32 %c)
   ret i32 %z
 }
-; CHECK: r0 += lsr(r1, r2)
+; CHECK: += lsr({{.*}}, {{.*}})
 
 declare i32 @llvm.hexagon.S2.asl.r.r.acc(i32, i32, i32)
 define i32 @S2_asl_r_r_acc(i32 %a, i32 %b, i32 %c) {
   %z = call i32 @llvm.hexagon.S2.asl.r.r.acc(i32 %a, i32 %b, i32 %c)
   ret i32 %z
 }
-; CHECK: r0 += asl(r1, r2)
+; CHECK: += asl({{.*}}, {{.*}})
 
 declare i32 @llvm.hexagon.S2.lsl.r.r.acc(i32, i32, i32)
 define i32 @S2_lsl_r_r_acc(i32 %a, i32 %b, i32 %c) {
   %z = call i32 @llvm.hexagon.S2.lsl.r.r.acc(i32 %a, i32 %b, i32 %c)
   ret i32 %z
 }
-; CHECK: r0 += lsl(r1, r2)
+; CHECK: += lsl({{.*}}, {{.*}})
 
 ; Shift by register and logical
 declare i64 @llvm.hexagon.S2.asr.r.p.or(i64, i64, i32)
@@ -512,112 +515,112 @@ define i64 @S2_asr_r_p_or(i64 %a, i64 %b, i32 %c) {
   %z = call i64 @llvm.hexagon.S2.asr.r.p.or(i64 %a, i64 %b, i32 %c)
   ret i64 %z
 }
-; CHECK: r1:0 |= asr(r3:2, r4)
+; CHECK: |= asr({{.*}}, r4)
 
 declare i64 @llvm.hexagon.S2.lsr.r.p.or(i64, i64, i32)
 define i64 @S2_lsr_r_p_or(i64 %a, i64 %b, i32 %c) {
   %z = call i64 @llvm.hexagon.S2.lsr.r.p.or(i64 %a, i64 %b, i32 %c)
   ret i64 %z
 }
-; CHECK: r1:0 |= lsr(r3:2, r4)
+; CHECK: |= lsr({{.*}}, r4)
 
 declare i64 @llvm.hexagon.S2.asl.r.p.or(i64, i64, i32)
 define i64 @S2_asl_r_p_or(i64 %a, i64 %b, i32 %c) {
   %z = call i64 @llvm.hexagon.S2.asl.r.p.or(i64 %a, i64 %b, i32 %c)
   ret i64 %z
 }
-; CHECK: r1:0 |= asl(r3:2, r4)
+; CHECK: |= asl({{.*}}, r4)
 
 declare i64 @llvm.hexagon.S2.lsl.r.p.or(i64, i64, i32)
 define i64 @S2_lsl_r_p_or(i64 %a, i64 %b, i32 %c) {
   %z = call i64 @llvm.hexagon.S2.lsl.r.p.or(i64 %a, i64 %b, i32 %c)
   ret i64 %z
 }
-; CHECK: r1:0 |= lsl(r3:2, r4)
+; CHECK: |= lsl({{.*}}, r4)
 
 declare i64 @llvm.hexagon.S2.asr.r.p.and(i64, i64, i32)
 define i64 @S2_asr_r_p_and(i64 %a, i64 %b, i32 %c) {
   %z = call i64 @llvm.hexagon.S2.asr.r.p.and(i64 %a, i64 %b, i32 %c)
   ret i64 %z
 }
-; CHECK: r1:0 &= asr(r3:2, r4)
+; CHECK: &= asr({{.*}}, r4)
 
 declare i64 @llvm.hexagon.S2.lsr.r.p.and(i64, i64, i32)
 define i64 @S2_lsr_r_p_and(i64 %a, i64 %b, i32 %c) {
   %z = call i64 @llvm.hexagon.S2.lsr.r.p.and(i64 %a, i64 %b, i32 %c)
   ret i64 %z
 }
-; CHECK: r1:0 &= lsr(r3:2, r4)
+; CHECK: &= lsr({{.*}}, r4)
 
 declare i64 @llvm.hexagon.S2.asl.r.p.and(i64, i64, i32)
 define i64 @S2_asl_r_p_and(i64 %a, i64 %b, i32 %c) {
   %z = call i64 @llvm.hexagon.S2.asl.r.p.and(i64 %a, i64 %b, i32 %c)
   ret i64 %z
 }
-; CHECK: r1:0 &= asl(r3:2, r4)
+; CHECK: &= asl({{.*}}, r4)
 
 declare i64 @llvm.hexagon.S2.lsl.r.p.and(i64, i64, i32)
 define i64 @S2_lsl_r_p_and(i64 %a, i64 %b, i32 %c) {
   %z = call i64 @llvm.hexagon.S2.lsl.r.p.and(i64 %a, i64 %b, i32 %c)
   ret i64 %z
 }
-; CHECK: r1:0 &= lsl(r3:2, r4)
+; CHECK: &= lsl({{.*}}, r4)
 
 declare i32 @llvm.hexagon.S2.asr.r.r.or(i32, i32, i32)
 define i32 @S2_asr_r_r_or(i32 %a, i32 %b, i32 %c) {
   %z = call i32 @llvm.hexagon.S2.asr.r.r.or(i32 %a, i32 %b, i32 %c)
   ret i32 %z
 }
-; CHECK: r0 |= asr(r1, r2)
+; CHECK: |= asr({{.*}}, {{.*}})
 
 declare i32 @llvm.hexagon.S2.lsr.r.r.or(i32, i32, i32)
 define i32 @S2_lsr_r_r_or(i32 %a, i32 %b, i32 %c) {
   %z = call i32 @llvm.hexagon.S2.lsr.r.r.or(i32 %a, i32 %b, i32 %c)
   ret i32 %z
 }
-; CHECK: r0 |= lsr(r1, r2)
+; CHECK: |= lsr({{.*}}, {{.*}})
 
 declare i32 @llvm.hexagon.S2.asl.r.r.or(i32, i32, i32)
 define i32 @S2_asl_r_r_or(i32%a, i32 %b, i32 %c) {
   %z = call i32 @llvm.hexagon.S2.asl.r.r.or(i32 %a, i32 %b, i32 %c)
   ret i32 %z
 }
-; CHECK: r0 |= asl(r1, r2)
+; CHECK: |= asl({{.*}}, {{.*}})
 
 declare i32 @llvm.hexagon.S2.lsl.r.r.or(i32, i32, i32)
 define i32 @S2_lsl_r_r_or(i32%a, i32 %b, i32 %c) {
   %z = call i32 @llvm.hexagon.S2.lsl.r.r.or(i32 %a, i32 %b, i32 %c)
   ret i32 %z
 }
-; CHECK: r0 |= lsl(r1, r2)
+; CHECK: |= lsl({{.*}}, {{.*}})
 
 declare i32 @llvm.hexagon.S2.asr.r.r.and(i32, i32, i32)
 define i32 @S2_asr_r_r_and(i32 %a, i32 %b, i32 %c) {
   %z = call i32 @llvm.hexagon.S2.asr.r.r.and(i32 %a, i32 %b, i32 %c)
   ret i32 %z
 }
-; CHECK: r0 &= asr(r1, r2)
+; CHECK: &= asr({{.*}}, {{.*}})
 
 declare i32 @llvm.hexagon.S2.lsr.r.r.and(i32, i32, i32)
 define i32 @S2_lsr_r_r_and(i32 %a, i32 %b, i32 %c) {
   %z = call i32 @llvm.hexagon.S2.lsr.r.r.and(i32 %a, i32 %b, i32 %c)
   ret i32 %z
 }
-; CHECK: r0 &= lsr(r1, r2)
+; CHECK: &= lsr({{.*}}, {{.*}})
 
 declare i32 @llvm.hexagon.S2.asl.r.r.and(i32, i32, i32)
 define i32 @S2_asl_r_r_and(i32 %a, i32 %b, i32 %c) {
   %z = call i32 @llvm.hexagon.S2.asl.r.r.and(i32 %a, i32 %b, i32 %c)
   ret i32 %z
 }
-; CHECK: r0 &= asl(r1, r2)
+; CHECK: &= asl({{.*}}, {{.*}})
 
 declare i32 @llvm.hexagon.S2.lsl.r.r.and(i32, i32, i32)
 define i32 @S2_lsl_r_r_and(i32 %a, i32 %b, i32 %c) {
   %z = call i32 @llvm.hexagon.S2.lsl.r.r.and(i32 %a, i32 %b, i32 %c)
   ret i32 %z
 }
-; CHECK: r0 &= lsl(r1, r2)
+; CHECK: &= lsl({{.*}}, {{.*}})
 
 ; Shift by register with saturation
 declare i32 @llvm.hexagon.S2.asr.r.r.sat(i32, i32)
@@ -625,14 +628,14 @@ define i32 @S2_asr_r_r_sat(i32 %a, i32 %b) {
   %z = call i32 @llvm.hexagon.S2.asr.r.r.sat(i32 %a, i32 %b)
   ret i32 %z
 }
-; CHECK: r0 = asr(r0, r1):sat
+; CHECK: = asr({{.*}}, {{.*}}):sat
 
 declare i32 @llvm.hexagon.S2.asl.r.r.sat(i32, i32)
 define i32 @S2_asl_r_r_sat(i32 %a, i32 %b) {
   %z = call i32 @llvm.hexagon.S2.asl.r.r.sat(i32 %a, i32 %b)
   ret i32 %z
 }
-; CHECK: r0 = asl(r0, r1):sat
+; CHECK: = asl({{.*}}, {{.*}}):sat
 
 ; Vector shift halfwords by immediate
 declare i64 @llvm.hexagon.S2.asr.i.vh(i64, i32)
@@ -640,21 +643,21 @@ define i64 @S2_asr_i_vh(i64 %a) {
   %z = call i64 @llvm.hexagon.S2.asr.i.vh(i64 %a, i32 0)
   ret i64 %z
 }
-; CHECK: r1:0 = vasrh(r1:0, #0)
+; CHECK: = vasrh({{.*}}, #0)
 
 declare i64 @llvm.hexagon.S2.lsr.i.vh(i64, i32)
 define i64 @S2_lsr_i_vh(i64 %a) {
   %z = call i64 @llvm.hexagon.S2.lsr.i.vh(i64 %a, i32 0)
   ret i64 %z
 }
-; CHECK: r1:0 = vlsrh(r1:0, #0)
+; CHECK: = vlsrh({{.*}}, #0)
 
 declare i64 @llvm.hexagon.S2.asl.i.vh(i64, i32)
 define i64 @S2_asl_i_vh(i64 %a) {
   %z = call i64 @llvm.hexagon.S2.asl.i.vh(i64 %a, i32 0)
   ret i64 %z
 }
-; CHECK: r1:0 = vaslh(r1:0, #0)
+; CHECK: = vaslh({{.*}}, #0)
 
 ; Vector shift halfwords by register
 declare i64 @llvm.hexagon.S2.asr.r.vh(i64, i32)
@@ -662,28 +665,28 @@ define i64 @S2_asr_r_vh(i64 %a, i32 %b) {
   %z = call i64 @llvm.hexagon.S2.asr.r.vh(i64 %a, i32 %b)
   ret i64 %z
 }
-; CHECK: r1:0 = vasrh(r1:0, r2)
+; CHECK: = vasrh({{.*}}, {{.*}})
 
 declare i64 @llvm.hexagon.S2.lsr.r.vh(i64, i32)
 define i64 @S2_lsr_r_vh(i64 %a, i32 %b) {
   %z = call i64 @llvm.hexagon.S2.lsr.r.vh(i64 %a, i32 %b)
   ret i64 %z
 }
-; CHECK: r1:0 = vlsrh(r1:0, r2)
+; CHECK: = vlsrh({{.*}}, {{.*}})
 
 declare i64 @llvm.hexagon.S2.asl.r.vh(i64, i32)
 define i64 @S2_asl_r_vh(i64 %a, i32 %b) {
   %z = call i64 @llvm.hexagon.S2.asl.r.vh(i64 %a, i32 %b)
   ret i64 %z
 }
-; CHECK: r1:0 = vaslh(r1:0, r2)
+; CHECK: = vaslh({{.*}}, {{.*}})
 
 declare i64 @llvm.hexagon.S2.lsl.r.vh(i64, i32)
 define i64 @S2_lsl_r_vh(i64 %a, i32 %b) {
   %z = call i64 @llvm.hexagon.S2.lsl.r.vh(i64 %a, i32 %b)
   ret i64 %z
 }
-; CHECK: r1:0 = vlslh(r1:0, r2)
+; CHECK: = vlslh({{.*}}, {{.*}})
 
 ; Vector shift words by immediate
 declare i64 @llvm.hexagon.S2.asr.i.vw(i64, i32)
@@ -691,21 +694,21 @@ define i64 @S2_asr_i_vw(i64 %a) {
   %z = call i64 @llvm.hexagon.S2.asr.i.vw(i64 %a, i32 0)
   ret i64 %z
 }
-; CHECK: r1:0 = vasrw(r1:0, #0)
+; CHECK: = vasrw({{.*}}, #0)
 
 declare i64 @llvm.hexagon.S2.lsr.i.vw(i64, i32)
 define i64 @S2_lsr_i_vw(i64 %a) {
   %z = call i64 @llvm.hexagon.S2.lsr.i.vw(i64 %a, i32 0)
   ret i64 %z
 }
-; CHECK: r1:0 = vlsrw(r1:0, #0)
+; CHECK: = vlsrw({{.*}}, #0)
 
 declare i64 @llvm.hexagon.S2.asl.i.vw(i64, i32)
 define i64 @S2_asl_i_vw(i64 %a) {
   %z = call i64 @llvm.hexagon.S2.asl.i.vw(i64 %a, i32 0)
   ret i64 %z
 }
-; CHECK: r1:0 = vaslw(r1:0, #0)
+; CHECK: = vaslw({{.*}}, #0)
 
 ; Vector shift words by with truncate and pack
 declare i32 @llvm.hexagon.S2.asr.i.svw.trun(i64, i32)
@@ -713,11 +716,11 @@ define i32 @S2_asr_i_svw_trun(i64 %a) {
   %z = call i32 @llvm.hexagon.S2.asr.i.svw.trun(i64 %a, i32 0)
   ret i32 %z
 }
-; CHECK: r0 = vasrw(r1:0, #0)
+; CHECK: = vasrw({{.*}}, #0)
 
 declare i32 @llvm.hexagon.S2.asr.r.svw.trun(i64, i32)
 define i32 @S2_asr_r_svw_trun(i64 %a, i32 %b) {
   %z = call i32 @llvm.hexagon.S2.asr.r.svw.trun(i64 %a, i32 %b)
   ret i32 %z
 }
-; CHECK: r0 = vasrw(r1:0, r2)
+; CHECK: = vasrw({{.*}}, {{.*}})
diff --git a/test/CodeGen/Hexagon/loadi1-G0.ll b/test/CodeGen/Hexagon/loadi1-G0.ll
new file mode 100644
index 00000000000..1116341c92b
--- /dev/null
+++ b/test/CodeGen/Hexagon/loadi1-G0.ll
@@ -0,0 +1,43 @@
+; RUN: llc -march=hexagon -mcpu=hexagonv5 -hexagon-small-data-threshold=0 < %s | FileCheck %s
+target datalayout = "e-p:32:32:32-i64:64:64-i32:32:32-i16:16:16-i1:32:32-f64:64:64-f32:32:32-a0:0-n32"
+target triple = "hexagon-unknown-linux-gnu"
+
+
+@flag = external global i1
+
+
+; CHECK-NOT: CONST
+
+define i32 @test_sextloadi1_32() {
+entry:
+  %0 = load i1, i1* @flag, align 4
+  %1 = sext i1 %0 to i32
+  ret i32 %1
+}
+
+
+
+define i16 @test_zextloadi1_16() {
+entry:
+  %0 = load i1, i1* @flag, align 4
+  %1 = zext i1 %0 to i16
+  ret i16 %1
+}
+
+
+define i32 @test_zextloadi1_32() {
+entry:
+  %0 = load i1, i1* @flag, align 4
+  %1 = zext i1 %0 to i32
+  ret i32 %1
+}
+
+
+define i64 @test_zextloadi1_64() {
+entry:
+  %0 = load i1, i1* @flag, align 4
+  %1 = zext i1 %0 to i64
+  ret i64 %1
+}
+
+
diff --git a/test/CodeGen/Hexagon/loadi1-v4-G0.ll b/test/CodeGen/Hexagon/loadi1-v4-G0.ll
new file mode 100644
index 00000000000..b7df1a125fb
--- /dev/null
+++ b/test/CodeGen/Hexagon/loadi1-v4-G0.ll
@@ -0,0 +1,43 @@
+; RUN: llc -march=hexagon -hexagon-small-data-threshold=0 < %s | FileCheck %s
+target datalayout = "e-p:32:32:32-i64:64:64-i32:32:32-i16:16:16-i1:32:32-f64:64:64-f32:32:32-a0:0-n32"
+target triple = "hexagon-unknown-linux-gnu"
+
+
+@flag = external global i1
+
+
+; CHECK-NOT: CONST
+
+define i32 @test_sextloadi1_32() {
+entry:
+  %0 = load i1, i1* @flag, align 4
+  %1 = sext i1 %0 to i32
+  ret i32 %1
+}
+
+
+
+define i16 @test_zextloadi1_16() {
+entry:
+  %0 = load i1, i1* @flag, align 4
+  %1 = zext i1 %0 to i16
+  ret i16 %1
+}
+
+
+define i32 @test_zextloadi1_32() {
+entry:
+  %0 = load i1, i1* @flag, align 4
+  %1 = zext i1 %0 to i32
+  ret i32 %1
+}
+
+
+define i64 @test_zextloadi1_64() {
+entry:
+  %0 = load i1, i1* @flag, align 4
+  %1 = zext i1 %0 to i64
+  ret i64 %1
+}
+
+
diff --git a/test/CodeGen/Hexagon/loadi1-v4.ll b/test/CodeGen/Hexagon/loadi1-v4.ll
new file mode 100644
index 00000000000..15b056040a4
--- /dev/null
+++ b/test/CodeGen/Hexagon/loadi1-v4.ll
@@ -0,0 +1,45 @@
+; RUN: llc -march=hexagon < %s | FileCheck %s
+target datalayout = "e-p:32:32:32-i64:64:64-i32:32:32-i16:16:16-i1:32:32-f64:64:64-f32:32:32-a0:0-n32"
+target triple = "hexagon-unknown-linux-gnu"
+
+
+@flag = external global i1
+
+
+define i32 @test_sextloadi1_32() {
+entry:
+  %0 = load i1, i1* @flag, align 4
+; CHECK: memub
+  %1 = sext i1 %0 to i32
+  ret i32 %1
+}
+
+
+
+define i16 @test_zextloadi1_16() {
+entry:
+  %0 = load i1, i1* @flag, align 4
+; CHECK: memub
+  %1 = zext i1 %0 to i16
+  ret i16 %1
+}
+
+
+define i32 @test_zextloadi1_32() {
+entry:
+  %0 = load i1, i1* @flag, align 4
+; CHECK: memub
+  %1 = zext i1 %0 to i32
+  ret i32 %1
+}
+
+
+define i64 @test_zextloadi1_64() {
+entry:
+  %0 = load i1, i1* @flag, align 4
+; CHECK: memub
+  %1 = zext i1 %0 to i64
+  ret i64 %1
+}
+
+
diff --git a/test/CodeGen/Hexagon/loadi1.ll b/test/CodeGen/Hexagon/loadi1.ll
new file mode 100644
index 00000000000..38c1dfec832
--- /dev/null
+++ b/test/CodeGen/Hexagon/loadi1.ll
@@ -0,0 +1,45 @@
+; RUN: llc -march=hexagon -mcpu=hexagonv5 < %s | FileCheck %s
+target datalayout = "e-p:32:32:32-i64:64:64-i32:32:32-i16:16:16-i1:32:32-f64:64:64-f32:32:32-a0:0-n32"
+target triple = "hexagon-unknown-linux-gnu"
+
+
+@flag = external global i1
+
+
+define i32 @test_sextloadi1_32() {
+entry:
+  %0 = load i1, i1* @flag, align 4
+; CHECK: memub
+  %1 = sext i1 %0 to i32
+  ret i32 %1
+}
+
+
+
+define i16 @test_zextloadi1_16() {
+entry:
+  %0 = load i1, i1* @flag, align 4
+; CHECK: memub
+  %1 = zext i1 %0 to i16
+  ret i16 %1
+}
+
+
+define i32 @test_zextloadi1_32() {
+entry:
+  %0 = load i1, i1* @flag, align 4
+; CHECK: memub
+  %1 = zext i1 %0 to i32
+  ret i32 %1
+}
+
+
+define i64 @test_zextloadi1_64() {
+entry:
+  %0 = load i1, i1* @flag, align 4
+; CHECK: memub
+  %1 = zext i1 %0 to i64
+  ret i64 %1
+}
+
+
diff --git a/test/CodeGen/Hexagon/macint.ll b/test/CodeGen/Hexagon/macint.ll
index 458a537467c..514ba5b9130 100644
--- a/test/CodeGen/Hexagon/macint.ll
+++ b/test/CodeGen/Hexagon/macint.ll
@@ -1,7 +1,7 @@
 ; RUN: llc -march=hexagon -mcpu=hexagonv4  < %s | FileCheck %s
 ; Check that we generate integer multiply accumulate.
 
-; CHECK: r{{[0-9]+}} += mpyi(r{{[0-9]+}}, r{{[0-9]+}})
+; CHECK: r{{[0-9]+}} {{\+|\-}}= mpyi(r{{[0-9]+}},
 
 define i32 @main(i32* %a, i32* %b) nounwind {
   entry:
diff --git a/test/CodeGen/Hexagon/maxd.ll b/test/CodeGen/Hexagon/maxd.ll
new file mode 100644
index 00000000000..7f237fd54e7
--- /dev/null
+++ b/test/CodeGen/Hexagon/maxd.ll
@@ -0,0 +1,9 @@
+; RUN: llc -march=hexagon < %s | FileCheck %s
+; CHECK: max
+
+define i64 @f(i64 %src, i64 %maxval) nounwind readnone {
+entry:
+  %cmp = icmp slt i64 %maxval, %src
+  %cond = select i1 %cmp, i64 %src, i64 %maxval
+  ret i64 %cond
+}
diff --git a/test/CodeGen/Hexagon/maxh.ll b/test/CodeGen/Hexagon/maxh.ll
new file mode 100644
index 00000000000..79b5e922c1b
--- /dev/null
+++ b/test/CodeGen/Hexagon/maxh.ll
@@ -0,0 +1,23 @@
+; RUN: llc -march=hexagon < %s | FileCheck %s
+; The result of max(half-word, half-word) is also half-word.
+; Check that we are not producing a sign extend after the max.
+; CHECK-NOT: sxth
+
+define i64 @test_cast(i64 %arg0, i16 zeroext %arg1, i16 zeroext %arg2) nounwind readnone {
+entry:
+  %conv.i = zext i16 %arg1 to i32
+  %conv1.i = zext i16 %arg2 to i32
+  %sub.i = sub nsw i32 %conv.i, %conv1.i
+  %sext.i = shl i32 %sub.i, 16
+  %cmp.i = icmp slt i32 %sext.i, 65536
+  %0 = ashr exact i32 %sext.i, 16
+  %conv7.i = select i1 %cmp.i, i32 1, i32 %0
+  %cmp8.i = icmp sgt i32 %conv7.i, 4
+  %conv7.op.i = add i32 %conv7.i, 65535
+  %shl = shl i64 %arg0, 2
+  %.mask = and i32 %conv7.op.i, 65535
+  %1 = zext i32 %.mask to i64
+  %conv = select i1 %cmp8.i, i64 3, i64 %1
+  %or = or i64 %conv, %shl
+  ret i64 %or
+}
diff --git a/test/CodeGen/Hexagon/maxud.ll b/test/CodeGen/Hexagon/maxud.ll
new file mode 100644
index 00000000000..eca4faee602
--- /dev/null
+++ b/test/CodeGen/Hexagon/maxud.ll
@@ -0,0 +1,9 @@
+; RUN: llc -march=hexagon < %s | FileCheck %s
+; CHECK: maxu
+
+define i64 @f(i64 %src, i64 %maxval) nounwind readnone {
+entry:
+  %cmp = icmp ult i64 %maxval, %src
+  %cond = select i1 %cmp, i64 %src, i64 %maxval
+  ret i64 %cond
+}
diff --git a/test/CodeGen/Hexagon/maxuw.ll b/test/CodeGen/Hexagon/maxuw.ll
new file mode 100644
index 00000000000..0dba1f5acde
--- /dev/null
+++ b/test/CodeGen/Hexagon/maxuw.ll
@@ -0,0 +1,9 @@
+; RUN: llc -march=hexagon < %s | FileCheck %s
+; CHECK: maxu
+
+define i32 @f(i32 %src, i32 %maxval) nounwind readnone {
+entry:
+  %cmp = icmp ult i32 %maxval, %src
+  %cond = select i1 %cmp, i32 %src, i32 %maxval
+  ret i32 %cond
+}
diff --git a/test/CodeGen/Hexagon/maxw.ll b/test/CodeGen/Hexagon/maxw.ll
new file mode 100644
index 00000000000..e66ca958806
--- /dev/null
+++ b/test/CodeGen/Hexagon/maxw.ll
@@ -0,0 +1,9 @@
+; RUN: llc -march=hexagon < %s | FileCheck %s
+; CHECK: max
+
+define i32 @f(i32 %src, i32 %maxval) nounwind readnone {
+entry:
+  %cmp = icmp slt i32 %maxval, %src
+  %cond = select i1 %cmp, i32 %src, i32 %maxval
+  ret i32 %cond
+}
diff --git a/test/CodeGen/Hexagon/mem-fi-add.ll b/test/CodeGen/Hexagon/mem-fi-add.ll
new file mode 100644
index 00000000000..a46029fdb5e
--- /dev/null
+++ b/test/CodeGen/Hexagon/mem-fi-add.ll
@@ -0,0 +1,29 @@
+; RUN: llc -O2 < %s | FileCheck %s
+; Look for four stores directly via r29.
+; CHECK: memd(r29
+; CHECK: memd(r29
+; CHECK: memd(r29
+; CHECK: memd(r29
+
+target datalayout = "e-m:e-p:32:32-i1:32-i64:64-a:0-v32:32-n16:32"
+target triple = "hexagon"
+
+; Function Attrs: nounwind
+define void @foo() #0 {
+entry:
+  %t = alloca [4 x [2 x i32]], align 8
+  %0 = bitcast [4 x [2 x i32]]* %t to i8*
+  call void @llvm.memset.p0i8.i32(i8* %0, i8 0, i32 32, i32 8, i1 false)
+  %arraydecay = getelementptr inbounds [4 x [2 x i32]], [4 x [2 x i32]]* %t, i32 0, i32 0
+  call void @bar([2 x i32]* %arraydecay) #1
+  ret void
+}
+
+; Function Attrs: nounwind
+declare void @llvm.memset.p0i8.i32(i8* nocapture, i8, i32, i32, i1) #1
+
+declare void @bar([2 x i32]*) #2
+
+attributes #0 = { nounwind "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #1 = { nounwind }
+attributes #2 = { "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
diff --git a/test/CodeGen/Hexagon/mind.ll b/test/CodeGen/Hexagon/mind.ll
new file mode 100644
index 00000000000..610283d97e2
--- /dev/null
+++ b/test/CodeGen/Hexagon/mind.ll
@@ -0,0 +1,9 @@
+; RUN: llc -march=hexagon < %s | FileCheck %s
+; CHECK: min
+
+define i64 @f(i64 %src, i64 %maxval) nounwind readnone {
+entry:
+  %cmp = icmp sgt i64 %maxval, %src
+  %cond = select i1 %cmp, i64 %src, i64 %maxval
+  ret i64 %cond
+}
diff --git a/test/CodeGen/Hexagon/minu-zext-16.ll b/test/CodeGen/Hexagon/minu-zext-16.ll
new file mode 100644
index 00000000000..e27507da3d4
--- /dev/null
+++ b/test/CodeGen/Hexagon/minu-zext-16.ll
@@ -0,0 +1,11 @@
+; RUN: llc -march=hexagon < %s | FileCheck %s
+; CHECK: minu
+
+define zeroext i16 @f(i16* noalias nocapture %src) nounwind readonly {
+entry:
+  %arrayidx = getelementptr inbounds i16, i16* %src, i32 1
+  %0 = load i16, i16* %arrayidx, align 1
+  %cmp = icmp ult i16 %0, 32767
+  %. = select i1 %cmp, i16 %0, i16 32767
+  ret i16 %.
+}
diff --git a/test/CodeGen/Hexagon/minu-zext-8.ll b/test/CodeGen/Hexagon/minu-zext-8.ll
new file mode 100644
index 00000000000..15dc1a16491
--- /dev/null
+++ b/test/CodeGen/Hexagon/minu-zext-8.ll
@@ -0,0 +1,11 @@
+; RUN: llc -march=hexagon < %s | FileCheck %s
+; CHECK: minu
+
+define zeroext i8 @f(i8* noalias nocapture %src) nounwind readonly {
+entry:
+  %arrayidx = getelementptr inbounds i8, i8* %src, i32 1
+  %0 = load i8, i8* %arrayidx, align 1
+  %cmp = icmp ult i8 %0, 127
+  %. = select i1 %cmp, i8 %0, i8 127
+  ret i8 %.
+}
diff --git a/test/CodeGen/Hexagon/minud.ll b/test/CodeGen/Hexagon/minud.ll
new file mode 100644
index 00000000000..29e81005081
--- /dev/null
+++ b/test/CodeGen/Hexagon/minud.ll
@@ -0,0 +1,9 @@
+; RUN: llc -march=hexagon < %s | FileCheck %s
+; CHECK: minu
+
+define i64 @f(i64 %src, i64 %maxval) nounwind readnone {
+entry:
+  %cmp = icmp ugt i64 %maxval, %src
+  %cond = select i1 %cmp, i64 %src, i64 %maxval
+  ret i64 %cond
+}
diff --git a/test/CodeGen/Hexagon/minuw.ll b/test/CodeGen/Hexagon/minuw.ll
new file mode 100644
index 00000000000..a88d1e11603
--- /dev/null
+++ b/test/CodeGen/Hexagon/minuw.ll
@@ -0,0 +1,9 @@
+; RUN: llc -march=hexagon < %s | FileCheck %s
+; CHECK: minu
+
+define i32 @f(i32 %src, i32 %maxval) nounwind readnone {
+entry:
+  %cmp = icmp ugt i32 %maxval, %src
+  %cond = select i1 %cmp, i32 %src, i32 %maxval
+  ret i32 %cond
+}
diff --git a/test/CodeGen/Hexagon/minw.ll b/test/CodeGen/Hexagon/minw.ll
new file mode 100644
index 00000000000..5bfaae09c80
--- /dev/null
+++ b/test/CodeGen/Hexagon/minw.ll
@@ -0,0 +1,9 @@
+; RUN: llc -march=hexagon < %s | FileCheck %s
+; CHECK: min
+
+define i32 @f(i32 %src, i32 %maxval) nounwind readnone {
+entry:
+  %cmp = icmp sgt i32 %maxval, %src
+  %cond = select i1 %cmp, i32 %src, i32 %maxval
+  ret i32 %cond
+}
diff --git a/test/CodeGen/Hexagon/mux-basic.ll b/test/CodeGen/Hexagon/mux-basic.ll
new file mode 100644
index 00000000000..ef1f7cb60e1
--- /dev/null
+++ b/test/CodeGen/Hexagon/mux-basic.ll
@@ -0,0 +1,28 @@
+; RUN: llc -O2 < %s | FileCheck %s
+; We should generate a MUX instruction for one of the selects.
+; CHECK: mux
+target datalayout = "e-p:32:32:32-i64:64:64-i32:32:32-i16:16:16-i1:32:32-f64:64:64-f32:32:32-v64:64:64-v32:32:32-a0:0-n16:32"
+target triple = "hexagon"
+
+%struct.struct_t = type { i32, i32, i32 }
+
+define void @foo(%struct.struct_t* nocapture %p, i32 %x, i32 %y, i32 %z) nounwind {
+entry:
+  %cmp = icmp slt i32 %x, 4660
+  %add = add nsw i32 %x, 1
+  %add.y = select i1 %cmp, i32 %add, i32 %y
+  %x.add.y = select i1 %cmp, i32 %x, i32 %y
+  %. = zext i1 %cmp to i32
+  %b.0 = add nsw i32 %x.add.y, %z
+  %a3 = getelementptr inbounds %struct.struct_t, %struct.struct_t* %p, i32 0, i32 0
+  store i32 %add.y, i32* %a3, align 4, !tbaa !0
+  %b4 = getelementptr inbounds %struct.struct_t, %struct.struct_t* %p, i32 0, i32 1
+  store i32 %b.0, i32* %b4, align 4, !tbaa !0
+  %c5 = getelementptr inbounds %struct.struct_t, %struct.struct_t* %p, i32 0, i32 2
+  store i32 %., i32* %c5, align 4, !tbaa !0
+  ret void
+}
+
+!0 = !{!"int", !1}
+!1 = !{!"omnipotent char", !2}
+!2 = !{!"Simple C/C++ TBAA"}
diff --git a/test/CodeGen/Hexagon/newvaluejump2.ll b/test/CodeGen/Hexagon/newvaluejump2.ll
index 36a0db16cfc..a812a7d9665 100644
--- a/test/CodeGen/Hexagon/newvaluejump2.ll
+++ b/test/CodeGen/Hexagon/newvaluejump2.ll
@@ -1,17 +1,16 @@
-; RUN: llc -march=hexagon -mcpu=hexagonv4 < %s | FileCheck %s
+; RUN: llc -march=hexagon -mcpu=hexagonv5 -disable-hexagon-misched < %s \
+; RUN:    | FileCheck %s
 ; Check that we generate new value jump, both registers, with one 
 ; of the registers as new.
 
-@Reg = common global i8 0, align 1
+@Reg = common global i32 0, align 4
 define i32 @main() nounwind {
 entry:
-; CHECK: if (cmp.gt(r{{[0-9]+}}.new, r{{[0-9]+}})) jump:{{[t|nt]}} .LBB{{[0-9]+}}_{{[0-9]+}}
-  %Reg2 = alloca i8, align 1
-  %0 = load i8, i8* %Reg2, align 1
-  %conv0 = zext i8 %0 to i32
-  %1 = load i8, i8* @Reg, align 1
-  %conv1 = zext i8 %1 to i32
-  %tobool = icmp sle i32 %conv0, %conv1
+; CHECK: if (cmp.gt(r{{[0-9]+}}, r{{[0-9]+}}.new)) jump:{{[t|nt]}} .LBB{{[0-9]+}}_{{[0-9]+}}
+  %Reg2 = alloca i32, align 4
+  %0 = load i32, i32* %Reg2, align 4
+  %1 = load i32, i32* @Reg, align 4
+  %tobool = icmp sle i32 %0, %1
   br i1 %tobool, label %if.then, label %if.else
 
 if.then:
diff --git a/test/CodeGen/Hexagon/postinc-offset.ll b/test/CodeGen/Hexagon/postinc-offset.ll
new file mode 100644
index 00000000000..5e0f4751f30
--- /dev/null
+++ b/test/CodeGen/Hexagon/postinc-offset.ll
@@ -0,0 +1,40 @@
+; RUN: llc -enable-aa-sched-mi -march=hexagon -mcpu=hexagonv5 < %s | FileCheck %s
+
+; CHECK: {
+; CHECK: ={{ *}}memd([[REG0:(r[0-9]+)]]{{ *}}++{{ *}}#8)
+; CHECK-NOT: memw([[REG0]]{{ *}}+{{ *}}#0){{ *}}=
+; CHECK: }
+
+define void @main() #0 {
+cond.end.6:
+  store i32 -1, i32* undef, align 8, !tbaa !0
+  br label %polly.stmt.for.body.i
+
+if.then:
+  unreachable
+
+if.end:
+  ret void
+
+polly.stmt.for.body.i24:
+  %0 = extractelement <2 x i32> %add.ip_vec, i32 1
+  br i1 undef, label %if.end, label %if.then
+
+polly.stmt.for.body.i:
+  %add.ip_vec30 = phi <2 x i32> [ %add.ip_vec, %polly.stmt.for.body.i ], [ zeroinitializer, %cond.end.6 ]
+  %scevgep.phi = phi i32* [ %scevgep.inc, %polly.stmt.for.body.i ], [ undef, %cond.end.6 ]
+  %polly.indvar = phi i32 [ %polly.indvar_next, %polly.stmt.for.body.i ], [ 0, %cond.end.6 ]
+  %vector_ptr = bitcast i32* %scevgep.phi to <2 x i32>*
+  %_p_vec_full = load <2 x i32>, <2 x i32>* %vector_ptr, align 8
+  %add.ip_vec = add <2 x i32> %_p_vec_full, %add.ip_vec30
+  %polly.indvar_next = add nsw i32 %polly.indvar, 2
+  %polly.loop_cond = icmp slt i32 %polly.indvar, 4
+  %scevgep.inc = getelementptr i32, i32* %scevgep.phi, i32 2
+  br i1 %polly.loop_cond, label %polly.stmt.for.body.i, label %polly.stmt.for.body.i24
+}
+
+attributes #0 = { "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf"="true" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "unsafe-fp-math"="false" "use-soft-float"="false" }
+
+!0 = !{!"int", !1}
+!1 = !{!"omnipotent char", !2}
+!2 = !{!"Simple C/C++ TBAA"}
diff --git a/test/CodeGen/Hexagon/predicate-logical.ll b/test/CodeGen/Hexagon/predicate-logical.ll
new file mode 100644
index 00000000000..be2bcb03d6a
--- /dev/null
+++ b/test/CodeGen/Hexagon/predicate-logical.ll
@@ -0,0 +1,30 @@
+; RUN: llc -O2 -march=hexagon < %s | FileCheck %s
+; CHECK: p{{[0-9]}} = or(p{{[0-9]}}, and(p{{[0-9]}}, p{{[0-9]}}))
+
+target triple = "hexagon"
+
+define i32 @foo(i64* nocapture %p, i64* nocapture %q) nounwind readonly {
+entry:
+  %incdec.ptr = getelementptr inbounds i64, i64* %p, i32 1
+  %0 = load i64, i64* %p, align 8, !tbaa !0
+  %incdec.ptr1 = getelementptr inbounds i64, i64* %q, i32 1
+  %1 = load i64, i64* %q, align 8, !tbaa !0
+  %2 = tail call i32 @llvm.hexagon.A2.vcmpwgtu(i64 %0, i64 %1)
+  %incdec.ptr2 = getelementptr inbounds i64, i64* %p, i32 2
+  %3 = load i64, i64* %incdec.ptr, align 8, !tbaa !0
+  %incdec.ptr3 = getelementptr inbounds i64, i64* %q, i32 2
+  %4 = load i64, i64* %incdec.ptr1, align 8, !tbaa !0
+  %5 = tail call i32 @llvm.hexagon.A2.vcmpwgtu(i64 %3, i64 %4)
+  %6 = load i64, i64* %incdec.ptr2, align 8, !tbaa !0
+  %7 = load i64, i64* %incdec.ptr3, align 8, !tbaa !0
+  %8 = tail call i32 @llvm.hexagon.A2.vcmpwgtu(i64 %6, i64 %7)
+  %and = and i32 %5, %2
+  %or = or i32 %8, %and
+  ret i32 %or
+}
+
+declare i32 @llvm.hexagon.A2.vcmpwgtu(i64, i64) nounwind readnone
+
+!0 = !{!"long long", !1}
+!1 = !{!"omnipotent char", !2}
+!2 = !{!"Simple C/C++ TBAA"}
diff --git a/test/CodeGen/Hexagon/predicate-rcmp.ll b/test/CodeGen/Hexagon/predicate-rcmp.ll
new file mode 100644
index 00000000000..45daa88d716
--- /dev/null
+++ b/test/CodeGen/Hexagon/predicate-rcmp.ll
@@ -0,0 +1,19 @@
+; RUN: llc -O2 -march=hexagon < %s | FileCheck %s
+; CHECK: cmp.eq(r{{[0-9]+}}, #0)
+; Check that the result of the builtin is not stored directly, i.e. that
+; there is an instruction that converts it to {0,1} from {0,-1}. Right now
+; the instruction is "r4 = !cmp.eq(r0, #0)".
+
+@var = common global i32 0, align 4
+declare i32 @llvm.hexagon.C2.cmpgtup(i64,i64) nounwind
+
+define void @foo(i64 %a98, i64 %a100) nounwind {
+entry:
+  %a101 = tail call i32 @llvm.hexagon.C2.cmpgtup(i64 %a98, i64 %a100)
+  %tobool250 = icmp eq i32 %a101, 0
+  %a102 = zext i1 %tobool250 to i8
+  %detected.0 = xor i8 %a102, 1
+  %conv253 = zext i8 %detected.0 to i32
+  store i32 %conv253, i32* @var, align 4
+  ret void
+}
diff --git a/test/CodeGen/Hexagon/relax.ll b/test/CodeGen/Hexagon/relax.ll
new file mode 100644
index 00000000000..9823d4d1cd9
--- /dev/null
+++ b/test/CodeGen/Hexagon/relax.ll
@@ -0,0 +1,14 @@
+; RUN: llc -march=hexagon -filetype=obj < %s | llvm-objdump -d -r - | FileCheck %s
+
+declare void @bar()
+
+define void @foo() {
+call void @bar()
+ret void
+}
+
+
+; CHECK: { allocframe(#0) }
+; CHECK: { call 0 }
+; CHECK: 00000004:  R_HEX_B22_PCREL
+; CHECK: { dealloc_return }
\ No newline at end of file
diff --git a/test/CodeGen/Hexagon/remove-endloop.ll b/test/CodeGen/Hexagon/remove-endloop.ll
new file mode 100644
index 00000000000..73e1ad02cd8
--- /dev/null
+++ b/test/CodeGen/Hexagon/remove-endloop.ll
@@ -0,0 +1,56 @@
+; RUN: llc -march=hexagon -O2 < %s | FileCheck %s
+
+define void @foo(i32 %n, i32* nocapture %A, i32* nocapture %B) nounwind optsize {
+entry:
+  %cmp = icmp sgt i32 %n, 100
+  br i1 %cmp, label %for.body.preheader, label %for.cond4.preheader
+
+; CHECK: endloop0
+; CHECK: endloop0
+; CHECK-NOT: endloop0
+
+for.body.preheader:
+  br label %for.body
+
+for.cond4.preheader:
+  %cmp113 = icmp sgt i32 %n, 0
+  br i1 %cmp113, label %for.body7.preheader, label %if.end
+
+for.body7.preheader:
+  br label %for.body7
+
+for.body:
+  %arrayidx.phi = phi i32* [ %arrayidx.inc, %for.body ], [ %B, %for.body.preheader ]
+  %arrayidx3.phi = phi i32* [ %arrayidx3.inc, %for.body ], [ %A, %for.body.preheader ]
+  %i.014 = phi i32 [ %inc, %for.body ], [ 0, %for.body.preheader ]
+  %0 = load i32, i32* %arrayidx.phi, align 4
+  %sub = add nsw i32 %0, -1
+  store i32 %sub, i32* %arrayidx3.phi, align 4
+  %inc = add nsw i32 %i.014, 1
+  %exitcond = icmp eq i32 %inc, %n
+  %arrayidx.inc = getelementptr i32, i32* %arrayidx.phi, i32 1
+  %arrayidx3.inc = getelementptr i32, i32* %arrayidx3.phi, i32 1
+  br i1 %exitcond, label %if.end.loopexit, label %for.body
+
+for.body7:
+  %arrayidx8.phi = phi i32* [ %arrayidx8.inc, %for.body7 ], [ %B, %for.body7.preheader ]
+  %arrayidx9.phi = phi i32* [ %arrayidx9.inc, %for.body7 ], [ %A, %for.body7.preheader ]
+  %i.117 = phi i32 [ %inc11, %for.body7 ], [ 0, %for.body7.preheader ]
+  %1 = load i32, i32* %arrayidx8.phi, align 4
+  %add = add nsw i32 %1, 1
+  store i32 %add, i32* %arrayidx9.phi, align 4
+  %inc11 = add nsw i32 %i.117, 1
+  %exitcond18 = icmp eq i32 %inc11, %n
+  %arrayidx8.inc = getelementptr i32, i32* %arrayidx8.phi, i32 1
+  %arrayidx9.inc = getelementptr i32, i32* %arrayidx9.phi, i32 1
+  br i1 %exitcond18, label %if.end.loopexit21, label %for.body7
+
+if.end.loopexit:
+  br label %if.end
+
+if.end.loopexit21:
+  br label %if.end
+
+if.end:
+  ret void
+}
diff --git a/test/CodeGen/Hexagon/shrink-frame-basic.ll b/test/CodeGen/Hexagon/shrink-frame-basic.ll
new file mode 100644
index 00000000000..50b37885eda
--- /dev/null
+++ b/test/CodeGen/Hexagon/shrink-frame-basic.ll
@@ -0,0 +1,36 @@
+; RUN: llc < %s | FileCheck %s
+; Check for allocframe in a non-entry block LBB0_n.
+; CHECK: LBB0_{{[0-9]+}}:
+; CHECK:   allocframe
+; Deallocframe may be in a different block, but must follow.
+; CHECK: deallocframe
+
+target datalayout = "e-m:e-p:32:32-i1:32-i64:64-a:0-v32:32-n16:32"
+target triple = "hexagon"
+
+; Function Attrs: nounwind
+define i32 @foo(i32 %n, i32* %p) #0 {
+entry:
+  %cmp = icmp eq i32* %p, null
+  br i1 %cmp, label %if.end, label %if.then
+
+if.then:                                          ; preds = %entry
+  %0 = load i32, i32* %p, align 4
+  %inc = add nsw i32 %0, 1
+  store i32 %inc, i32* %p, align 4
+  br label %return
+
+if.end:                                           ; preds = %entry
+  %call = tail call i32 bitcast (i32 (...)* @bar to i32 (i32)*)(i32 %n) #0
+  %add = add nsw i32 %call, 1
+  br label %return
+
+return:                                           ; preds = %if.end, %if.then
+  %retval.0 = phi i32 [ %0, %if.then ], [ %add, %if.end ]
+  ret i32 %retval.0
+}
+
+declare i32 @bar(...) #0
+
+attributes #0 = { nounwind }
+
diff --git a/test/CodeGen/Hexagon/signed_immediates.ll b/test/CodeGen/Hexagon/signed_immediates.ll
new file mode 100644
index 00000000000..a4766313cc6
--- /dev/null
+++ b/test/CodeGen/Hexagon/signed_immediates.ll
@@ -0,0 +1,99 @@
+; RUN: llc -march=hexagon < %s | FileCheck %s
+
+; s4_0Imm
+; CHECK: memb(r0++#-1) = r1
+define i8* @foo1(i8* %a, i8 %b)  {
+  store i8 %b, i8* %a
+  %c = getelementptr i8, i8* %a, i32 -1
+  ret i8* %c
+}
+
+; s4_1Imm
+; CHECK: memh(r0++#-2) = r1
+define i16* @foo2(i16* %a, i16 %b)  {
+  store i16 %b, i16* %a
+  %c = getelementptr i16, i16* %a, i32 -1
+  ret i16* %c
+}
+
+; s4_2Imm
+; CHECK: memw(r0++#-4) = r1
+define i32* @foo3(i32* %a, i32 %b)  {
+  store i32 %b, i32* %a
+  %c = getelementptr i32, i32* %a, i32 -1
+  ret i32* %c
+}
+
+; s4_3Imm
+; CHECK: memd(r0++#-8) = r3:2
+define i64* @foo4(i64* %a, i64 %b)  {
+  store i64 %b, i64* %a
+  %c = getelementptr i64, i64* %a, i32 -1
+  ret i64* %c
+}
+
+; s6Ext
+; CHECK: if (p0.new) memw(r0+#0)=#-1
+define void @foo5(i32* %a, i1 %b) {
+br i1 %b, label %x, label %y
+x:
+  store i32 -1, i32* %a
+  ret void
+y:
+  ret void
+}
+
+; s10Ext
+; CHECK: p0 = cmp.eq(r0, #-1)
+define i1 @foo7(i32 %a) {
+  %b = icmp eq i32 %a, -1
+  ret i1 %b
+}
+
+; s11_0Ext
+; CHECK: memb(r0+#-1) = r1
+define void @foo8(i8* %a, i8 %b) {
+  %c = getelementptr i8, i8* %a, i32 -1
+  store i8 %b, i8* %c
+  ret void
+}
+
+; s11_1Ext
+; CHECK: memh(r0+#-2) = r1
+define void @foo9(i16* %a, i16 %b) {
+  %c = getelementptr i16, i16* %a, i32 -1
+  store i16 %b, i16* %c
+  ret void
+}
+
+; s11_2Ext
+; CHECK: memw(r0+#-4) = r1
+define void @foo10(i32* %a, i32 %b) {
+  %c = getelementptr i32, i32* %a, i32 -1
+  store i32 %b, i32* %c
+  ret void
+}
+
+; s11_3Ext
+; CHECK: memd(r0+#-8) = r3:2
+define void @foo11(i64* %a, i64 %b) {
+  %c = getelementptr i64, i64* %a, i32 -1
+  store i64 %b, i64* %c
+  ret void
+}
+
+; s12Ext
+; CHECK: if (p0.new) r0 = #-1
+define i32 @foo12(i32 %a, i1 %b) {
+br i1 %b, label %x, label %y
+x:
+  ret i32 -1
+y:
+  ret i32 %a
+}
+
+; s16Ext
+; CHECK: r0 = #-2
+define i32 @foo13() {
+  ret i32 -2
+}
\ No newline at end of file
diff --git a/test/CodeGen/Hexagon/simple_addend.ll b/test/CodeGen/Hexagon/simple_addend.ll
new file mode 100644
index 00000000000..ec3a87f1dcc
--- /dev/null
+++ b/test/CodeGen/Hexagon/simple_addend.ll
@@ -0,0 +1,10 @@
+; RUN: llc -march=hexagon -filetype=obj -o - < %s | llvm-readobj -relocations | FileCheck %s
+
+declare void @bar(i32);
+
+define void @foo(i32 %a) {
+  %b = mul i32 %a, 3
+  call void @bar(i32 %b)
+  ret void
+}
+; CHECK:     0x8 R_HEX_B22_PCREL bar 0x4
diff --git a/test/CodeGen/Hexagon/stack-align1.ll b/test/CodeGen/Hexagon/stack-align1.ll
new file mode 100644
index 00000000000..4efa70f5985
--- /dev/null
+++ b/test/CodeGen/Hexagon/stack-align1.ll
@@ -0,0 +1,21 @@
+; RUN: llc -O0 -march=hexagon < %s | FileCheck %s
+; CHECK: and(r29, #-32)
+; CHECK-DAG: add(r29, #0)
+; CHECK-DAG: add(r29, #28)
+
+target triple = "hexagon-unknown-unknown"
+
+; Function Attrs: nounwind uwtable
+define void @foo() #0 {
+entry:
+  %x = alloca i32, align 4
+  %y = alloca i32, align 32
+  %0 = bitcast i32* %x to i8*
+  %1 = bitcast i32* %y to i8*
+  call void @bar(i8* %0, i8* %1)
+  ret void
+}
+
+declare void @bar(i8*, i8*) #0
+
+attributes #0 = { nounwind }
diff --git a/test/CodeGen/Hexagon/stack-align2.ll b/test/CodeGen/Hexagon/stack-align2.ll
new file mode 100644
index 00000000000..1bbd5782032
--- /dev/null
+++ b/test/CodeGen/Hexagon/stack-align2.ll
@@ -0,0 +1,27 @@
+; RUN: llc -O0 -march=hexagon < %s | FileCheck %s
+; CHECK: and(r29, #-128)
+; CHECK-DAG: add(r29, #0)
+; CHECK-DAG: add(r29, #64)
+; CHECK-DAG: add(r29, #96)
+; CHECK-DAG: add(r29, #124)
+
+target triple = "hexagon-unknown-unknown"
+
+; Function Attrs: nounwind uwtable
+define void @foo() #0 {
+entry:
+  %x = alloca i32, align 4
+  %y = alloca i32, align 32
+  %z = alloca i32, align 64
+  %w = alloca i32, align 128
+  %0 = bitcast i32* %x to i8*
+  %1 = bitcast i32* %y to i8*
+  %2 = bitcast i32* %z to i8*
+  %3 = bitcast i32* %w to i8*
+  call void @bar(i8* %0, i8* %1, i8* %2, i8* %3)
+  ret void
+}
+
+declare void @bar(i8*, i8*, i8*, i8*) #0
+
+attributes #0 = { nounwind }
diff --git a/test/CodeGen/Hexagon/stack-alloca1.ll b/test/CodeGen/Hexagon/stack-alloca1.ll
new file mode 100644
index 00000000000..00e9e051aeb
--- /dev/null
+++ b/test/CodeGen/Hexagon/stack-alloca1.ll
@@ -0,0 +1,18 @@
+; RUN: llc -O0 -march=hexagon < %s | FileCheck %s
+; CHECK: sub(r29, r[[REG:[0-9]+]])
+; CHECK: r29 = r[[REG]]
+
+target triple = "hexagon-unknown-unknown"
+
+; Function Attrs: nounwind uwtable
+define void @foo(i32 %n) #0 {
+entry:
+  %x = alloca i32, i32 %n
+  %0 = bitcast i32* %x to i8*
+  call void @bar(i8* %0)
+  ret void
+}
+
+declare void @bar(i8*) #0
+
+attributes #0 = { nounwind }
diff --git a/test/CodeGen/Hexagon/stack-alloca2.ll b/test/CodeGen/Hexagon/stack-alloca2.ll
new file mode 100644
index 00000000000..ad5e13166aa
--- /dev/null
+++ b/test/CodeGen/Hexagon/stack-alloca2.ll
@@ -0,0 +1,23 @@
+; RUN: llc -O0 -march=hexagon < %s | FileCheck %s
+; CHECK-DAG: r[[AP:[0-9]+]] = and(r30, #-32)
+; CHECK-DAG: r1 = add(r[[AP]], #-32)
+
+; CHECK-DAG: sub(r29, r[[SP:[0-9]+]])
+; CHECK-DAG: r29 = r[[SP]]
+
+target triple = "hexagon-unknown-unknown"
+
+; Function Attrs: nounwind uwtable
+define void @foo(i32 %n) #0 {
+entry:
+  %x = alloca i32, i32 %n
+  %y = alloca i32, align 32
+  %0 = bitcast i32* %x to i8*
+  %1 = bitcast i32* %y to i8*
+  call void @bar(i8* %0, i8* %1)
+  ret void
+}
+
+declare void @bar(i8*, i8* %y) #0
+
+attributes #0 = { nounwind }
diff --git a/test/CodeGen/Hexagon/sube.ll b/test/CodeGen/Hexagon/sube.ll
index 1a7882276c8..9735894c419 100644
--- a/test/CodeGen/Hexagon/sube.ll
+++ b/test/CodeGen/Hexagon/sube.ll
@@ -1,12 +1,12 @@
 ; RUN: llc -march=hexagon -hexagon-expand-condsets=0 < %s | FileCheck %s
 
-; CHECK: r{{[0-9]+:[0-9]+}} = #0
 ; CHECK: r{{[0-9]+:[0-9]+}} = #1
+; CHECK: r{{[0-9]+:[0-9]+}} = #0
 ; CHECK: p{{[0-9]+}} = cmp.gtu(r{{[0-9]+:[0-9]+}}, r{{[0-9]+:[0-9]+}})
-; CHECK: r{{[0-9]+}} = mux(p{{[0-9]+}}, r{{[0-9]+}}, r{{[0-9]+}})
-; CHECK: r{{[0-9]+}} = mux(p{{[0-9]+}}, r{{[0-9]+}}, r{{[0-9]+}})
 ; CHECK: r{{[0-9]+:[0-9]+}} = sub(r{{[0-9]+:[0-9]+}}, r{{[0-9]+:[0-9]+}})
 ; CHECK: r{{[0-9]+:[0-9]+}} = sub(r{{[0-9]+:[0-9]+}}, r{{[0-9]+:[0-9]+}})
+; CHECK: r{{[0-9]+}} = mux(p{{[0-9]+}}, r{{[0-9]+}}, r{{[0-9]+}})
+; CHECK: r{{[0-9]+}} = mux(p{{[0-9]+}}, r{{[0-9]+}}, r{{[0-9]+}})
 ; CHECK: r{{[0-9]+:[0-9]+}} = combine(r{{[0-9]+}}, r{{[0-9]+}})
 
 define void @check_sube_subc(i64 %AL, i64 %AH, i64 %BL, i64 %BH, i64* %RL, i64* %RH) {
diff --git a/test/CodeGen/Hexagon/tfr-to-combine.ll b/test/CodeGen/Hexagon/tfr-to-combine.ll
index d22d685f7d6..a257acfeb49 100644
--- a/test/CodeGen/Hexagon/tfr-to-combine.ll
+++ b/test/CodeGen/Hexagon/tfr-to-combine.ll
@@ -27,7 +27,7 @@ entry:
 
 ; Function Attrs: nounwind
 define i64 @test4() #0 {
-; CHECK: combine(#0, ##100)
+; CHECK: combine(#0, #100)
 entry:
   store i16 100, i16* @b, align 2
   store i16 0, i16* @a, align 2
diff --git a/test/CodeGen/Hexagon/usr-ovf-dep.ll b/test/CodeGen/Hexagon/usr-ovf-dep.ll
new file mode 100644
index 00000000000..1f06986f0aa
--- /dev/null
+++ b/test/CodeGen/Hexagon/usr-ovf-dep.ll
@@ -0,0 +1,28 @@
+; RUN: llc -O2 < %s | FileCheck %s
+target datalayout = "e-m:e-p:32:32-i1:32-i64:64-a:0-v32:32-n16:32"
+target triple = "hexagon"
+
+; Check that the two ":sat" instructions are in the same packet.
+; CHECK: foo
+; CHECK: {
+; CHECK: :sat
+; CHECK-NEXT: :sat
+
+target datalayout = "e-m:e-p:32:32-i1:32-i64:64-a:0-v32:32-n16:32"
+target triple = "hexagon"
+
+; Function Attrs: nounwind readnone
+define i32 @foo(i32 %Rs, i32 %Rt, i32 %Ru) #0 {
+entry:
+  %0 = tail call i32 @llvm.hexagon.S2.asr.r.r.sat(i32 %Rs, i32 %Ru)
+  %1 = tail call i32 @llvm.hexagon.S2.asr.r.r.sat(i32 %Rt, i32 %Ru)
+  %add = add nsw i32 %1, %0
+  ret i32 %add
+}
+
+; Function Attrs: nounwind readnone
+declare i32 @llvm.hexagon.S2.asr.r.r.sat(i32, i32) #1
+
+attributes #0 = { nounwind readnone "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #1 = { nounwind readnone }
+
diff --git a/test/CodeGen/Inputs/DbgValueOtherTargets.ll b/test/CodeGen/Inputs/DbgValueOtherTargets.ll
index 0af0293e568..efa1a0849a8 100644
--- a/test/CodeGen/Inputs/DbgValueOtherTargets.ll
+++ b/test/CodeGen/Inputs/DbgValueOtherTargets.ll
@@ -3,7 +3,7 @@
 define i32 @main() nounwind ssp {
 entry:
 ; CHECK: DEBUG_VALUE
-  call void @llvm.dbg.value(metadata i32 0, i64 0, metadata !7, metadata !MDExpression()), !dbg !9
+  call void @llvm.dbg.value(metadata i32 0, i64 0, metadata !7, metadata !DIExpression()), !dbg !9
   ret i32 0, !dbg !10
 }
 
@@ -14,17 +14,17 @@ declare void @llvm.dbg.value(metadata, i64, metadata, metadata) nounwind readnon
 !llvm.dbg.cu = !{!2}
 !llvm.module.flags = !{!13}
 
-!0 = !MDSubprogram(name: "main", line: 2, isLocal: false, isDefinition: true, virtualIndex: 6, isOptimized: false, file: !12, scope: !1, type: !3, function: i32 ()* @main)
-!1 = !MDFile(filename: "/tmp/x.c", directory: "/Users/manav")
-!2 = !MDCompileUnit(language: DW_LANG_C99, producer: "clang version 2.9 (trunk 120996)", isOptimized: false, emissionKind: 0, file: !12, enums: !6, retainedTypes: !6, subprograms: !11)
-!3 = !MDSubroutineType(types: !4)
+!0 = !DISubprogram(name: "main", line: 2, isLocal: false, isDefinition: true, virtualIndex: 6, isOptimized: false, file: !12, scope: !1, type: !3, function: i32 ()* @main)
+!1 = !DIFile(filename: "/tmp/x.c", directory: "/Users/manav")
+!2 = !DICompileUnit(language: DW_LANG_C99, producer: "clang version 2.9 (trunk 120996)", isOptimized: false, emissionKind: 0, file: !12, enums: !6, retainedTypes: !6, subprograms: !11)
+!3 = !DISubroutineType(types: !4)
 !4 = !{!5}
-!5 = !MDBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
+!5 = !DIBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
 !6 = !{}
-!7 = !MDLocalVariable(tag: DW_TAG_auto_variable, name: "i", line: 3, scope: !8, file: !1, type: !5)
-!8 = distinct !MDLexicalBlock(line: 2, column: 12, file: !12, scope: !0)
-!9 = !MDLocation(line: 3, column: 11, scope: !8)
-!10 = !MDLocation(line: 4, column: 2, scope: !8)
+!7 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "i", line: 3, scope: !8, file: !1, type: !5)
+!8 = distinct !DILexicalBlock(line: 2, column: 12, file: !12, scope: !0)
+!9 = !DILocation(line: 3, column: 11, scope: !8)
+!10 = !DILocation(line: 4, column: 2, scope: !8)
 !11 = !{!0}
-!12 = !MDFile(filename: "/tmp/x.c", directory: "/Users/manav")
+!12 = !DIFile(filename: "/tmp/x.c", directory: "/Users/manav")
 !13 = !{i32 1, !"Debug Info Version", i32 3}
diff --git a/test/CodeGen/MIR/AArch64/cfi-def-cfa.mir b/test/CodeGen/MIR/AArch64/cfi-def-cfa.mir
new file mode 100644
index 00000000000..6deff62ec9c
--- /dev/null
+++ b/test/CodeGen/MIR/AArch64/cfi-def-cfa.mir
@@ -0,0 +1,32 @@
+# RUN: llc -mtriple=aarch64-none-linux-gnu -start-after branch-folder -stop-after branch-folder -o /dev/null %s | FileCheck %s
+# This test ensures that the MIR parser parses the .cfi_def_cfa operands
+# correctly.
+
+--- |
+
+  declare void @foo()
+
+  define void @trivial_fp_func() {
+  entry:
+    call void @foo()
+    ret void
+  }
+
+...
+---
+name:            trivial_fp_func
+body:
+  - id:          0
+    name:        entry
+    liveins:     [ '%lr', '%fp', '%lr', '%fp' ]
+    instructions:
+      - '%sp = frame-setup STPXpre killed %fp, killed %lr, %sp, -2'
+      - '%fp = frame-setup ADDXri %sp, 0, 0'
+      # CHECK: CFI_INSTRUCTION .cfi_def_cfa %w29, 16
+      - 'frame-setup CFI_INSTRUCTION .cfi_def_cfa %w29, 16'
+      - 'frame-setup CFI_INSTRUCTION .cfi_offset %w30, -8'
+      - 'frame-setup CFI_INSTRUCTION .cfi_offset %w29, -16'
+      - 'BL @foo, csr_aarch64_aapcs, implicit-def dead %lr, implicit %sp, implicit-def %sp'
+      - '%sp, %fp, %lr = LDPXpost %sp, 2'
+      - RET_ReallyLR
+...
diff --git a/test/CodeGen/MIR/AArch64/lit.local.cfg b/test/CodeGen/MIR/AArch64/lit.local.cfg
new file mode 100644
index 00000000000..f4f77c5aa31
--- /dev/null
+++ b/test/CodeGen/MIR/AArch64/lit.local.cfg
@@ -0,0 +1,8 @@
+import re
+
+if not 'AArch64' in config.root.targets:
+    config.unsupported = True
+
+# For now we don't test arm64-win32.
+if re.search(r'cygwin|mingw32|win32|windows-gnu|windows-msvc', config.target_triple):
+    config.unsupported = True
diff --git a/test/CodeGen/MIR/AArch64/multiple-lhs-operands.mir b/test/CodeGen/MIR/AArch64/multiple-lhs-operands.mir
new file mode 100644
index 00000000000..d47c8ba47f6
--- /dev/null
+++ b/test/CodeGen/MIR/AArch64/multiple-lhs-operands.mir
@@ -0,0 +1,29 @@
+# RUN: llc -mtriple=aarch64-none-linux-gnu -start-after branch-folder -stop-after branch-folder -o /dev/null %s | FileCheck %s
+# This test ensures that the MIR parser can parse multiple register machine
+# operands before '='.
+
+--- |
+
+  declare void @foo()
+
+  define void @trivial_fp_func() {
+  entry:
+    call void @foo()
+    ret void
+  }
+
+...
+---
+name:            trivial_fp_func
+body:
+  - id:          0
+    name:        entry
+    liveins:     [ '%lr', '%fp', '%lr', '%fp' ]
+    instructions:
+      - '%sp = frame-setup STPXpre killed %fp, killed %lr, %sp, -2'
+      - '%fp = frame-setup ADDXri %sp, 0, 0'
+      - 'BL @foo, csr_aarch64_aapcs, implicit-def dead %lr, implicit %sp, implicit-def %sp'
+# CHECK: %sp, %fp, %lr = LDPXpost %sp, 2
+      - '%sp, %fp, %lr = LDPXpost %sp, 2'
+      - RET_ReallyLR
+...
diff --git a/test/CodeGen/MIR/AMDGPU/expected-target-index-name.mir b/test/CodeGen/MIR/AMDGPU/expected-target-index-name.mir
new file mode 100644
index 00000000000..9df6ba5f5d6
--- /dev/null
+++ b/test/CodeGen/MIR/AMDGPU/expected-target-index-name.mir
@@ -0,0 +1,65 @@
+# RUN: not llc -march=amdgcn -mcpu=SI -start-after postrapseudos -stop-after postrapseudos -o /dev/null %s 2>&1 | FileCheck %s
+
+--- |
+
+  %struct.foo = type { float, [5 x i32] }
+
+  @float_gv = internal unnamed_addr addrspace(2) constant [5 x float] [float 0.000000e+00, float 1.000000e+00, float 2.000000e+00, float 3.000000e+00, float 4.000000e+00], align 4
+
+  define void @float(float addrspace(1)* %out, i32 %index) #0 {
+  entry:
+    %0 = getelementptr inbounds [5 x float], [5 x float] addrspace(2)* @float_gv, i32 0, i32 %index
+    %1 = load float, float addrspace(2)* %0
+    store float %1, float addrspace(1)* %out
+    ret void
+  }
+
+  declare { i1, i64 } @llvm.SI.if(i1)
+
+  declare { i1, i64 } @llvm.SI.else(i64)
+
+  declare i64 @llvm.SI.break(i64)
+
+  declare i64 @llvm.SI.if.break(i1, i64)
+
+  declare i64 @llvm.SI.else.break(i64, i64)
+
+  declare i1 @llvm.SI.loop(i64)
+
+  declare void @llvm.SI.end.cf(i64)
+
+  attributes #0 = { "target-cpu"="SI" }
+
+...
+---
+name:            float
+tracksSubRegLiveness: true
+liveins:
+  - { reg: '%sgpr0_sgpr1' }
+frameInfo:
+  maxAlignment:  8
+body:
+  - id:          0
+    name:        entry
+    liveins:     [ '%sgpr0_sgpr1' ]
+    instructions:
+      - '%sgpr2_sgpr3 = S_GETPC_B64'
+# CHECK: [[@LINE+1]]:50: expected the name of the target index
+      - '%sgpr2 = S_ADD_U32 %sgpr2, target-index(0), implicit-def %scc, implicit-def %scc'
+      - '%sgpr3 = S_ADDC_U32 %sgpr3, 0, implicit-def %scc, implicit %scc, implicit-def %scc, implicit %scc'
+      - '%sgpr4_sgpr5 = S_LSHR_B64 %sgpr2_sgpr3, 32, implicit-def dead %scc'
+      - '%sgpr6 = S_LOAD_DWORD_IMM %sgpr0_sgpr1, 11'
+      - '%sgpr7 = S_ASHR_I32 %sgpr6, 31, implicit-def dead %scc'
+      - '%sgpr6_sgpr7 = S_LSHL_B64 %sgpr6_sgpr7, 2, implicit-def dead %scc'
+      - '%sgpr2 = S_ADD_U32 %sgpr2, @float_gv, implicit-def %scc'
+      - '%sgpr3 = S_ADDC_U32 %sgpr4, 0, implicit-def dead %scc, implicit %scc'
+      - '%sgpr4 = S_ADD_U32 %sgpr2, %sgpr6, implicit-def %scc'
+      - '%sgpr5 = S_ADDC_U32 %sgpr3, %sgpr7, implicit-def dead %scc, implicit %scc'
+      - '%sgpr2 = S_LOAD_DWORD_IMM %sgpr4_sgpr5, 0'
+      - '%sgpr4_sgpr5 = S_LOAD_DWORDX2_IMM killed %sgpr0_sgpr1, 9'
+      - '%sgpr7 = S_MOV_B32 61440'
+      - '%sgpr6 = S_MOV_B32 -1'
+      - '%vgpr0 = V_MOV_B32_e32 killed %sgpr2, implicit %exec'
+      - 'BUFFER_STORE_DWORD_OFFSET killed %vgpr0, %sgpr4_sgpr5_sgpr6_sgpr7, 0, 0, 0, 0, 0, implicit %exec'
+      - S_ENDPGM
+...
diff --git a/test/CodeGen/MIR/AMDGPU/invalid-target-index-operand.mir b/test/CodeGen/MIR/AMDGPU/invalid-target-index-operand.mir
new file mode 100644
index 00000000000..aeea9b2e3a5
--- /dev/null
+++ b/test/CodeGen/MIR/AMDGPU/invalid-target-index-operand.mir
@@ -0,0 +1,65 @@
+# RUN: not llc -march=amdgcn -mcpu=SI -start-after postrapseudos -stop-after postrapseudos -o /dev/null %s 2>&1 | FileCheck %s
+
+--- |
+
+  %struct.foo = type { float, [5 x i32] }
+
+  @float_gv = internal unnamed_addr addrspace(2) constant [5 x float] [float 0.000000e+00, float 1.000000e+00, float 2.000000e+00, float 3.000000e+00, float 4.000000e+00], align 4
+
+  define void @float(float addrspace(1)* %out, i32 %index) #0 {
+  entry:
+    %0 = getelementptr inbounds [5 x float], [5 x float] addrspace(2)* @float_gv, i32 0, i32 %index
+    %1 = load float, float addrspace(2)* %0
+    store float %1, float addrspace(1)* %out
+    ret void
+  }
+
+  declare { i1, i64 } @llvm.SI.if(i1)
+
+  declare { i1, i64 } @llvm.SI.else(i64)
+
+  declare i64 @llvm.SI.break(i64)
+
+  declare i64 @llvm.SI.if.break(i1, i64)
+
+  declare i64 @llvm.SI.else.break(i64, i64)
+
+  declare i1 @llvm.SI.loop(i64)
+
+  declare void @llvm.SI.end.cf(i64)
+
+  attributes #0 = { "target-cpu"="SI" }
+
+...
+---
+name:            float
+tracksSubRegLiveness: true
+liveins:
+  - { reg: '%sgpr0_sgpr1' }
+frameInfo:
+  maxAlignment:  8
+body:
+  - id:          0
+    name:        entry
+    liveins:     [ '%sgpr0_sgpr1' ]
+    instructions:
+      - '%sgpr2_sgpr3 = S_GETPC_B64'
+# CHECK: [[@LINE+1]]:50: use of undefined target index 'constdata-start'
+      - '%sgpr2 = S_ADD_U32 %sgpr2, target-index(constdata-start), implicit-def %scc, implicit-def %scc'
+      - '%sgpr3 = S_ADDC_U32 %sgpr3, 0, implicit-def %scc, implicit %scc, implicit-def %scc, implicit %scc'
+      - '%sgpr4_sgpr5 = S_LSHR_B64 %sgpr2_sgpr3, 32, implicit-def dead %scc'
+      - '%sgpr6 = S_LOAD_DWORD_IMM %sgpr0_sgpr1, 11'
+      - '%sgpr7 = S_ASHR_I32 %sgpr6, 31, implicit-def dead %scc'
+      - '%sgpr6_sgpr7 = S_LSHL_B64 %sgpr6_sgpr7, 2, implicit-def dead %scc'
+      - '%sgpr2 = S_ADD_U32 %sgpr2, @float_gv, implicit-def %scc'
+      - '%sgpr3 = S_ADDC_U32 %sgpr4, 0, implicit-def dead %scc, implicit %scc'
+      - '%sgpr4 = S_ADD_U32 %sgpr2, %sgpr6, implicit-def %scc'
+      - '%sgpr5 = S_ADDC_U32 %sgpr3, %sgpr7, implicit-def dead %scc, implicit %scc'
+      - '%sgpr2 = S_LOAD_DWORD_IMM %sgpr4_sgpr5, 0'
+      - '%sgpr4_sgpr5 = S_LOAD_DWORDX2_IMM killed %sgpr0_sgpr1, 9'
+      - '%sgpr7 = S_MOV_B32 61440'
+      - '%sgpr6 = S_MOV_B32 -1'
+      - '%vgpr0 = V_MOV_B32_e32 killed %sgpr2, implicit %exec'
+      - 'BUFFER_STORE_DWORD_OFFSET killed %vgpr0, %sgpr4_sgpr5_sgpr6_sgpr7, 0, 0, 0, 0, 0, implicit %exec'
+      - S_ENDPGM
+...
diff --git a/test/CodeGen/MIR/AMDGPU/lit.local.cfg b/test/CodeGen/MIR/AMDGPU/lit.local.cfg
new file mode 100644
index 00000000000..2a665f06be7
--- /dev/null
+++ b/test/CodeGen/MIR/AMDGPU/lit.local.cfg
@@ -0,0 +1,2 @@
+if not 'AMDGPU' in config.root.targets:
+    config.unsupported = True
diff --git a/test/CodeGen/MIR/AMDGPU/target-index-operands.mir b/test/CodeGen/MIR/AMDGPU/target-index-operands.mir
new file mode 100644
index 00000000000..a130689419a
--- /dev/null
+++ b/test/CodeGen/MIR/AMDGPU/target-index-operands.mir
@@ -0,0 +1,66 @@
+# RUN: llc -march=amdgcn -mcpu=SI -start-after postrapseudos -stop-after postrapseudos -o /dev/null %s | FileCheck %s
+# This test verifies that the MIR parser can parse target index operands.
+
+--- |
+
+  %struct.foo = type { float, [5 x i32] }
+
+  @float_gv = internal unnamed_addr addrspace(2) constant [5 x float] [float 0.000000e+00, float 1.000000e+00, float 2.000000e+00, float 3.000000e+00, float 4.000000e+00], align 4
+
+  define void @float(float addrspace(1)* %out, i32 %index) #0 {
+  entry:
+    %0 = getelementptr inbounds [5 x float], [5 x float] addrspace(2)* @float_gv, i32 0, i32 %index
+    %1 = load float, float addrspace(2)* %0
+    store float %1, float addrspace(1)* %out
+    ret void
+  }
+
+  declare { i1, i64 } @llvm.SI.if(i1)
+
+  declare { i1, i64 } @llvm.SI.else(i64)
+
+  declare i64 @llvm.SI.break(i64)
+
+  declare i64 @llvm.SI.if.break(i1, i64)
+
+  declare i64 @llvm.SI.else.break(i64, i64)
+
+  declare i1 @llvm.SI.loop(i64)
+
+  declare void @llvm.SI.end.cf(i64)
+
+  attributes #0 = { "target-cpu"="SI" }
+
+...
+---
+name:            float
+tracksSubRegLiveness: true
+liveins:
+  - { reg: '%sgpr0_sgpr1' }
+frameInfo:
+  maxAlignment:  8
+body:
+  - id:          0
+    name:        entry
+    liveins:     [ '%sgpr0_sgpr1' ]
+    instructions:
+      - '%sgpr2_sgpr3 = S_GETPC_B64'
+# CHECK: %sgpr2 = S_ADD_U32 %sgpr2, target-index(amdgpu-constdata-start), implicit-def %scc, implicit-def %scc
+      - '%sgpr2 = S_ADD_U32 %sgpr2, target-index(amdgpu-constdata-start), implicit-def %scc, implicit-def %scc'
+      - '%sgpr3 = S_ADDC_U32 %sgpr3, 0, implicit-def %scc, implicit %scc, implicit-def %scc, implicit %scc'
+      - '%sgpr4_sgpr5 = S_LSHR_B64 %sgpr2_sgpr3, 32, implicit-def dead %scc'
+      - '%sgpr6 = S_LOAD_DWORD_IMM %sgpr0_sgpr1, 11'
+      - '%sgpr7 = S_ASHR_I32 %sgpr6, 31, implicit-def dead %scc'
+      - '%sgpr6_sgpr7 = S_LSHL_B64 %sgpr6_sgpr7, 2, implicit-def dead %scc'
+      - '%sgpr2 = S_ADD_U32 %sgpr2, @float_gv, implicit-def %scc'
+      - '%sgpr3 = S_ADDC_U32 %sgpr4, 0, implicit-def dead %scc, implicit %scc'
+      - '%sgpr4 = S_ADD_U32 %sgpr2, %sgpr6, implicit-def %scc'
+      - '%sgpr5 = S_ADDC_U32 %sgpr3, %sgpr7, implicit-def dead %scc, implicit %scc'
+      - '%sgpr2 = S_LOAD_DWORD_IMM %sgpr4_sgpr5, 0'
+      - '%sgpr4_sgpr5 = S_LOAD_DWORDX2_IMM killed %sgpr0_sgpr1, 9'
+      - '%sgpr7 = S_MOV_B32 61440'
+      - '%sgpr6 = S_MOV_B32 -1'
+      - '%vgpr0 = V_MOV_B32_e32 killed %sgpr2, implicit %exec'
+      - 'BUFFER_STORE_DWORD_OFFSET killed %vgpr0, %sgpr4_sgpr5_sgpr6_sgpr7, 0, 0, 0, 0, 0, implicit %exec'
+      - S_ENDPGM
+...
diff --git a/test/CodeGen/MIR/X86/basic-block-liveins.mir b/test/CodeGen/MIR/X86/basic-block-liveins.mir
new file mode 100644
index 00000000000..d749a052442
--- /dev/null
+++ b/test/CodeGen/MIR/X86/basic-block-liveins.mir
@@ -0,0 +1,25 @@
+# RUN: llc -march=x86-64 -start-after branch-folder -stop-after branch-folder -o /dev/null %s | FileCheck %s
+# This test ensures that the MIR parser parses basic block liveins correctly.
+
+--- |
+
+  define i32 @test(i32 %a, i32 %b) {
+  body:
+    %c = add i32 %a, %b
+    ret i32 %c
+  }
+
+...
+---
+name:            test
+body:
+  # CHECK: name: body
+  # CHECK: liveins: [ '%edi', '%esi' ]
+  # CHECK-NEXT: instructions:
+  - id:          0
+    name:        body
+    liveins:     [ '%edi', '%esi' ]
+    instructions:
+      - '%eax = LEA64_32r killed %rdi, 1, killed %rsi, 0, _'
+      - 'RETQ %eax'
+...
diff --git a/test/CodeGen/MIR/X86/block-address-operands.mir b/test/CodeGen/MIR/X86/block-address-operands.mir
new file mode 100644
index 00000000000..a0b17b00d2f
--- /dev/null
+++ b/test/CodeGen/MIR/X86/block-address-operands.mir
@@ -0,0 +1,89 @@
+# RUN: llc -march=x86-64 -start-after branch-folder -stop-after branch-folder -o /dev/null %s | FileCheck %s
+# This test ensures that the MIR parser parses the block address operands
+# correctly.
+
+--- |
+
+  @addr = global i8* null
+
+  define void @test() {
+  entry:
+    store volatile i8* blockaddress(@test, %block), i8** @addr
+    %val = load volatile i8*, i8** @addr
+    indirectbr i8* %val, [label %block]
+
+  block:
+    ret void
+  }
+
+  define void @test2() {
+  entry:
+    store volatile i8* blockaddress(@test2, %"quoted block"), i8** @addr
+    %val = load volatile i8*, i8** @addr
+    indirectbr i8* %val, [label %"quoted block"]
+
+  "quoted block":
+    ret void
+  }
+
+  define void @test3() {
+  entry:
+    store volatile i8* blockaddress(@test3, %0), i8** @addr
+    %val = load volatile i8*, i8** @addr
+    indirectbr i8* %val, [label %0]
+
+    ret void
+  }
+
+...
+---
+name:            test
+body:
+  - id:              0
+    name:            entry
+    successors:      [ '%bb.1.block' ]
+    instructions:
+    # CHECK: %rax = LEA64r %rip, 1, _, blockaddress(@test, %ir-block.block), _
+      - '%rax = LEA64r %rip, 1, _, blockaddress(@test, %ir-block.block), _'
+      - 'MOV64mr %rip, 1, _, @addr, _, killed %rax'
+      - 'JMP64m %rip, 1, _, @addr, _'
+  - id:              1
+    name:            block
+    addressTaken:    true
+    instructions:
+      - RETQ
+...
+---
+name:            test2
+tracksRegLiveness: true
+body:
+  - id:              0
+    name:            entry
+    successors:      [ '%bb.1' ]
+    instructions:
+    # CHECK: %rax = LEA64r %rip, 1, _, blockaddress(@test2, %ir-block."quoted block"), _
+      - '%rax = LEA64r %rip, 1, _, blockaddress(@test2, %ir-block."quoted block"), _'
+      - 'MOV64mr %rip, 1, _, @addr, _, killed %rax'
+      - 'JMP64m %rip, 1, _, @addr, _'
+  - id:              1
+    addressTaken:    true
+    instructions:
+      - RETQ
+...
+---
+name:            test3
+tracksRegLiveness: true
+body:
+  - id:              0
+    name:            entry
+    successors:      [ '%bb.1' ]
+    instructions:
+    # CHECK: %rax = LEA64r %rip, 1, _, blockaddress(@test3, %ir-block.0), _
+      - '%rax = LEA64r %rip, 1, _, blockaddress(@test3, %ir-block.0), _'
+      - 'MOV64mr %rip, 1, _, @addr, _, killed %rax'
+      - 'JMP64m %rip, 1, _, @addr, _'
+  - id:              1
+    addressTaken:    true
+    instructions:
+      - RETQ
+...
diff --git a/test/CodeGen/MIR/X86/callee-saved-info.mir b/test/CodeGen/MIR/X86/callee-saved-info.mir
new file mode 100644
index 00000000000..4ec5b50c364
--- /dev/null
+++ b/test/CodeGen/MIR/X86/callee-saved-info.mir
@@ -0,0 +1,98 @@
+# RUN: llc -march=x86-64 -start-after prologepilog -stop-after prologepilog -o /dev/null %s | FileCheck %s
+# This test ensures that the MIR parser parses callee saved information in the
+# stack objects correctly.
+
+--- |
+
+  define i32 @compute(i32 %a) {
+  body:
+    ret i32 %a
+  }
+
+  define i32 @func(i32 %a) {
+  entry:
+    %b = alloca i32
+    store i32 %a, i32* %b
+    br label %check
+
+  check:
+    %comp = icmp sle i32 %a, 10
+    br i1 %comp, label %loop, label %exit
+
+  loop:
+    %c = load i32, i32* %b
+    %d = call i32 @compute(i32 %c)
+    %e = sub i32 %d, 1
+    store i32 %e, i32* %b
+    br label %check
+
+  exit:
+    ret i32 0
+  }
+
+...
+---
+name:            compute
+tracksRegLiveness: true
+body:
+  - id:          0
+    name:        body
+    liveins:     [ '%edi' ]
+    instructions:
+      - '%eax = COPY killed %edi'
+      - 'RETQ killed %eax'
+...
+---
+name:            func
+tracksRegLiveness: true
+frameInfo:
+  stackSize:       24
+  maxAlignment:    4
+  adjustsStack:    true
+  hasCalls:        true
+# CHECK: fixedStack:
+# CHECK-NEXT: , callee-saved-register: '%rbx' }
+fixedStack:
+  - { id: 0, type: spill-slot, offset: -16, size: 8, alignment: 16, callee-saved-register: '%rbx' }
+# CHECK: stack:
+# CHECK-NEXT: - { id: 0
+# CHECK-NEXT: , callee-saved-register: '%edi' }
+stack:
+  - { id: 0, name: b, offset: -20, size: 4, alignment: 4 }
+  - { id: 1, offset: -24, size: 4, alignment: 4, callee-saved-register: '%edi' }
+body:
+  - id:          0
+    name:        entry
+    successors:  [ '%bb.1.check' ]
+    liveins:     [ '%edi', '%rbx' ]
+    instructions:
+      - 'frame-setup PUSH64r killed %rbx, implicit-def %rsp, implicit %rsp'
+      - '%rsp = frame-setup SUB64ri8 %rsp, 16, implicit-def dead %eflags'
+      - '%ebx = COPY %edi'
+      - 'MOV32mr %rsp, 1, _, 12, _, %ebx'
+  - id:          1
+    name:        check
+    successors:  [ '%bb.2.loop', '%bb.3.exit' ]
+    liveins:     [ '%ebx' ]
+    instructions:
+      - 'CMP32ri8 %ebx, 10, implicit-def %eflags'
+      - 'JG_1 %bb.3.exit, implicit killed %eflags'
+      - 'JMP_1 %bb.2.loop'
+  - id:          2
+    name:        loop
+    successors:  [ '%bb.1.check' ]
+    liveins:     [ '%ebx' ]
+    instructions:
+      - '%edi = MOV32rm %rsp, 1, _, 12, _'
+      - 'CALL64pcrel32 @compute, csr_64, implicit %rsp, implicit %edi, implicit-def %rsp, implicit-def %eax'
+      - '%eax = DEC32r killed %eax, implicit-def dead %eflags'
+      - 'MOV32mr %rsp, 1, _, 12, _, killed %eax'
+      - 'JMP_1 %bb.1.check'
+  - id:          3
+    name:        exit
+    instructions:
+      - '%eax = MOV32r0 implicit-def dead %eflags'
+      - '%rsp = ADD64ri8 %rsp, 16, implicit-def dead %eflags'
+      - '%rbx = POP64r implicit-def %rsp, implicit %rsp'
+      - 'RETQ %eax'
+...
diff --git a/test/CodeGen/MIR/X86/cfi-def-cfa-offset.mir b/test/CodeGen/MIR/X86/cfi-def-cfa-offset.mir
new file mode 100644
index 00000000000..3fb5395602d
--- /dev/null
+++ b/test/CodeGen/MIR/X86/cfi-def-cfa-offset.mir
@@ -0,0 +1,31 @@
+# RUN: llc -march=x86-64 -start-after branch-folder -stop-after branch-folder -o /dev/null %s | FileCheck %s
+# This test ensures that the MIR parser parses the .cfi_def_cfa_offset operands
+# correctly.
+
+--- |
+
+  define void @test() {
+  entry:
+    %tmp = alloca [4168 x i8], align 4
+    ret void
+  }
+
+...
+---
+name:            test
+tracksRegLiveness: true
+frameInfo:
+  stackSize:       4040
+stack:
+  - { id: 0, name: tmp, offset: -4176, size: 4168, alignment: 4 }
+body:
+  - id:              0
+    name:            entry
+    instructions:
+      - '%rsp = SUB64ri32 %rsp, 4040, implicit-def dead %eflags'
+     # CHECK: CFI_INSTRUCTION .cfi_def_cfa_offset 4048
+      - 'CFI_INSTRUCTION .cfi_def_cfa_offset 4048'
+      - '%rsp = ADD64ri32 %rsp, 4040, implicit-def dead %eflags'
+      - 'RETQ'
+...
+
diff --git a/test/CodeGen/MIR/X86/cfi-def-cfa-register.mir b/test/CodeGen/MIR/X86/cfi-def-cfa-register.mir
new file mode 100644
index 00000000000..aad546494d7
--- /dev/null
+++ b/test/CodeGen/MIR/X86/cfi-def-cfa-register.mir
@@ -0,0 +1,33 @@
+# RUN: llc -march=x86-64 -start-after branch-folder -stop-after branch-folder -o /dev/null %s | FileCheck %s
+# This test ensures that the MIR parser parses the .cfi_def_cfa_register
+# operands correctly.
+
+--- |
+
+  define void @func() #0 {
+  entry:
+    unreachable
+  }
+
+  attributes #0 = { "no-frame-pointer-elim"="true" }
+
+...
+---
+name:            func
+tracksRegLiveness: true
+frameInfo:
+  stackSize:     8
+fixedStack:
+  - { id: 0, type: spill-slot, offset: -16, size: 8, alignment: 16 }
+body:
+  - id:          0
+    name:        entry
+    liveins:     [ '%rbp' ]
+    instructions:
+      - 'PUSH64r killed %rbp, implicit-def %rsp, implicit %rsp'
+      - 'CFI_INSTRUCTION .cfi_def_cfa_offset 16'
+      - 'CFI_INSTRUCTION .cfi_offset %rbp, -16'
+      - '%rbp = MOV64rr %rsp'
+      # CHECK: CFI_INSTRUCTION .cfi_def_cfa_register %rbp
+      - 'CFI_INSTRUCTION .cfi_def_cfa_register %rbp'
+...
diff --git a/test/CodeGen/MIR/X86/cfi-offset.mir b/test/CodeGen/MIR/X86/cfi-offset.mir
new file mode 100644
index 00000000000..edfffda8641
--- /dev/null
+++ b/test/CodeGen/MIR/X86/cfi-offset.mir
@@ -0,0 +1,48 @@
+# RUN: llc -march=x86-64 -start-after branch-folder -stop-after branch-folder -o /dev/null %s | FileCheck %s
+# This test ensures that the MIR parser parses the .cfi_offset operands
+# correctly.
+
+--- |
+
+  declare void @foo(i32)
+
+  define i32 @test(i32 %a, i32 %b, i32 %c, i32 %d) {
+  entry:
+    %add = add nsw i32 %b, %a
+    %add1 = add nsw i32 %add, %c
+    %add2 = add nsw i32 %add1, %d
+    tail call void @foo(i32 %add2)
+    %add6 = add nsw i32 %add2, %add2
+    ret i32 %add6
+  }
+
+...
+---
+name:            test
+tracksRegLiveness: true
+frameInfo:
+  stackSize:       8
+  adjustsStack:    true
+  hasCalls:        true
+fixedStack:
+  - { id: 0, type: spill-slot, offset: -16, size: 8, alignment: 16 }
+body:
+  - id:              0
+    name:            entry
+    liveins:         [ '%ecx', '%edi', '%edx', '%esi', '%rbx' ]
+    instructions:
+      - 'PUSH64r killed %rbx, implicit-def %rsp, implicit %rsp'
+      - 'CFI_INSTRUCTION .cfi_def_cfa_offset 16'
+      # CHECK: CFI_INSTRUCTION .cfi_offset %rbx, -16
+      - 'CFI_INSTRUCTION .cfi_offset %rbx, -16'
+      - '%ebx = COPY %edi, implicit-def %rbx'
+      - '%ebx = ADD32rr %ebx, killed %esi, implicit-def dead %eflags'
+      - '%ebx = ADD32rr %ebx, killed %edx, implicit-def dead %eflags'
+      - '%ebx = ADD32rr %ebx, killed %ecx, implicit-def dead %eflags'
+      - '%edi = COPY %ebx'
+      - 'CALL64pcrel32 @foo, csr_64, implicit %rsp, implicit %edi, implicit-def %rsp'
+      - '%eax = LEA64_32r killed %rbx, 1, %rbx, 0, _'
+      - '%rbx = POP64r implicit-def %rsp, implicit %rsp'
+      - 'RETQ %eax'
+...
+
diff --git a/test/CodeGen/MIR/X86/constant-pool-item-redefinition-error.mir b/test/CodeGen/MIR/X86/constant-pool-item-redefinition-error.mir
new file mode 100644
index 00000000000..0ff67302876
--- /dev/null
+++ b/test/CodeGen/MIR/X86/constant-pool-item-redefinition-error.mir
@@ -0,0 +1,27 @@
+# RUN: not llc -march=x86-64 -start-after branch-folder -stop-after branch-folder -o /dev/null %s 2>&1 | FileCheck %s
+
+--- |
+
+  define double @test(double %a, float %b) {
+  entry:
+    %c = fadd double %a, 3.250000e+00
+    ret double %c
+  }
+
+...
+---
+name:            test
+constants:
+  - id:          0
+    value:       'double 3.250000e+00'
+# CHECK: [[@LINE+1]]:18: redefinition of constant pool item '%const.0'
+  - id:          0
+    value:       'double 3.250000e+00'
+body:
+  - id: 0
+    name: entry
+    instructions:
+      - '%xmm0 = ADDSDrm killed %xmm0, %rip, 1, _, %const.0, _'
+      - 'RETQ %xmm0'
+...
+
diff --git a/test/CodeGen/MIR/X86/constant-pool.mir b/test/CodeGen/MIR/X86/constant-pool.mir
new file mode 100644
index 00000000000..57508d1c51d
--- /dev/null
+++ b/test/CodeGen/MIR/X86/constant-pool.mir
@@ -0,0 +1,118 @@
+# RUN: llc -march=x86-64 -start-after branch-folder -stop-after branch-folder -o /dev/null %s | FileCheck %s
+# This test ensures that the MIR parser parses constant pool constants and
+# constant pool operands correctly.
+
+--- |
+
+  define double @test(double %a, float %b) {
+  entry:
+    %c = fadd double %a, 3.250000e+00
+    %d = fadd float %b, 6.250000e+00
+    %e = fpext float %d to double
+    %f = fmul double %c, %e
+    ret double %f
+  }
+
+  define double @test2(double %a, float %b) {
+  entry:
+    %c = fadd double %a, 3.250000e+00
+    %d = fadd float %b, 6.250000e+00
+    %e = fpext float %d to double
+    %f = fmul double %c, %e
+    ret double %f
+  }
+
+  define double @test3(double %a, float %b) {
+  entry:
+    %c = fadd double %a, 3.250000e+00
+    %d = fadd float %b, 6.250000e+00
+    %e = fpext float %d to double
+    %f = fmul double %c, %e
+    ret double %f
+  }
+...
+---
+# CHECK: name: test
+# CHECK: constants:
+# CHECK-NEXT: - id: 0
+# CHECK-NEXT:   value: 'double 3.250000e+00'
+# CHECK-NEXT:   alignment: 8
+# CHECK-NEXT: - id: 1
+# CHECK-NEXT:   value: 'float 6.250000e+00'
+# CHECK-NEXT:   alignment: 4
+name:            test
+constants:
+  - id:          0
+    value:       'double 3.250000e+00'
+    alignment:   8
+  - id:          1
+    value:       'float 6.250000e+00'
+    alignment:   4
+body:
+  - id: 0
+    name: entry
+    instructions:
+      # CHECK:      %xmm0 = ADDSDrm killed %xmm0, %rip, 1, _, %const.0, _
+      # CHECK-NEXT: %xmm1 = ADDSSrm killed %xmm1, %rip, 1, _, %const.1, _
+      - '%xmm0 = ADDSDrm killed %xmm0, %rip, 1, _, %const.0, _'
+      - '%xmm1 = ADDSSrm killed %xmm1, %rip, 1, _, %const.1, _'
+      - '%xmm1 = CVTSS2SDrr killed %xmm1'
+      - '%xmm0 = MULSDrr killed %xmm0, killed %xmm1'
+      - 'RETQ %xmm0'
+...
+---
+# Verify that alignment can be inferred:
+# CHECK: name: test2
+# CHECK: constants:
+# CHECK-NEXT: - id: 0
+# CHECK-NEXT:   value: 'double 3.250000e+00'
+# CHECK-NEXT:   alignment: 8
+# CHECK-NEXT: - id: 1
+# CHECK-NEXT:   value: 'float 6.250000e+00'
+# CHECK-NEXT:   alignment: 4
+name:            test2
+constants:
+  - id:          0
+    value:       'double 3.250000e+00'
+  - id:          1
+    value:       'float 6.250000e+00'
+body:
+  - id: 0
+    name: entry
+    instructions:
+      - '%xmm0 = ADDSDrm killed %xmm0, %rip, 1, _, %const.0, _'
+      - '%xmm1 = ADDSSrm killed %xmm1, %rip, 1, _, %const.1, _'
+      - '%xmm1 = CVTSS2SDrr killed %xmm1'
+      - '%xmm0 = MULSDrr killed %xmm0, killed %xmm1'
+      - 'RETQ %xmm0'
+...
+---
+# Verify that the non-standard alignments are respected:
+# CHECK: name: test3
+# CHECK: constants:
+# CHECK-NEXT: - id: 0
+# CHECK-NEXT:   value: 'double 3.250000e+00'
+# CHECK-NEXT:   alignment: 128
+# CHECK-NEXT: - id: 1
+# CHECK-NEXT:   value: 'float 6.250000e+00'
+# CHECK-NEXT:   alignment: 1
+name:            test3
+constants:
+  - id:          0
+    value:       'double 3.250000e+00'
+    alignment:   128
+  - id:          1
+    value:       'float 6.250000e+00'
+    alignment:   1
+body:
+  - id: 0
+    name: entry
+    instructions:
+      # CHECK:      %xmm0 = ADDSDrm killed %xmm0, %rip, 1, _, %const.0, _
+      # CHECK-NEXT: %xmm1 = ADDSSrm killed %xmm1, %rip, 1, _, %const.1, _
+      - '%xmm0 = ADDSDrm killed %xmm0, %rip, 1, _, %const.0, _'
+      - '%xmm1 = ADDSSrm killed %xmm1, %rip, 1, _, %const.1, _'
+      - '%xmm1 = CVTSS2SDrr killed %xmm1'
+      - '%xmm0 = MULSDrr killed %xmm0, killed %xmm1'
+      - 'RETQ %xmm0'
+...
diff --git a/test/CodeGen/MIR/X86/constant-value-error.mir b/test/CodeGen/MIR/X86/constant-value-error.mir
new file mode 100644
index 00000000000..d6c21107022
--- /dev/null
+++ b/test/CodeGen/MIR/X86/constant-value-error.mir
@@ -0,0 +1,27 @@
+# RUN: not llc -march=x86-64 -start-after branch-folder -stop-after branch-folder -o /dev/null %s 2>&1 | FileCheck %s
+# This test ensures that the MIR parser reports an error when parsing an invalid
+# constant value.
+
+--- |
+
+  define double @test(double %a, float %b) {
+  entry:
+    %c = fadd double %a, 3.250000e+00
+    ret double %c
+  }
+
+...
+---
+name:            test
+constants:
+  - id:          0
+  # CHECK: [[@LINE+1]]:19: expected type
+    value:       'dub 3.250000e+00'
+body:
+  - id: 0
+    name: entry
+    instructions:
+      - '%xmm0 = ADDSDrm killed %xmm0, %rip, 1, _, %const.0, _'
+      - 'RETQ %xmm0'
+...
+
diff --git a/test/CodeGen/MIR/X86/dead-register-flag.mir b/test/CodeGen/MIR/X86/dead-register-flag.mir
new file mode 100644
index 00000000000..988b554659c
--- /dev/null
+++ b/test/CodeGen/MIR/X86/dead-register-flag.mir
@@ -0,0 +1,26 @@
+# RUN: llc -march=x86-64 -start-after branch-folder -stop-after branch-folder -o /dev/null %s | FileCheck %s
+# This test ensures that the MIR parser parses the 'dead' register flags
+# correctly.
+
+--- |
+
+  define i32 @foo(i32 %a) #0 {
+  body:
+    %c = mul i32 %a, 11
+    ret i32 %c
+  }
+
+  attributes #0 = { "no-frame-pointer-elim"="false" }
+
+...
+---
+name:            foo
+body:
+  # CHECK: name: body
+  - id:          0
+    name:        body
+    instructions:
+      # CHECK: - '%eax = IMUL32rri8 %edi, 11, implicit-def dead %eflags'
+      - '%eax = IMUL32rri8 %edi, 11, implicit-def dead %eflags'
+      - 'RETQ %eax'
+...
diff --git a/test/CodeGen/MIR/X86/expected-block-reference-in-blockaddress.mir b/test/CodeGen/MIR/X86/expected-block-reference-in-blockaddress.mir
new file mode 100644
index 00000000000..2d4f5fac60b
--- /dev/null
+++ b/test/CodeGen/MIR/X86/expected-block-reference-in-blockaddress.mir
@@ -0,0 +1,34 @@
+# RUN: not llc -march=x86-64 -start-after branch-folder -stop-after branch-folder -o /dev/null %s 2>&1 | FileCheck %s
+
+--- |
+
+  @addr = global i8* null
+
+  define void @test() {
+  entry:
+    store volatile i8* blockaddress(@test, %block), i8** @addr
+    %val = load volatile i8*, i8** @addr
+    indirectbr i8* %val, [label %block]
+
+  block:
+    ret void
+  }
+
+...
+---
+name:            test
+body:
+  - id:              0
+    name:            entry
+    successors:      [ '%bb.1.block' ]
+    instructions:
+      # CHECK: [[@LINE+1]]:56: expected an IR block reference
+      - '%rax = LEA64r %rip, 1, _, blockaddress(@test, _), _'
+      - 'MOV64mr %rip, 1, _, @addr, _, killed %rax'
+      - 'JMP64m %rip, 1, _, @addr, _'
+  - id:              1
+    name:            block
+    addressTaken:    true
+    instructions:
+      - RETQ
+...
diff --git a/test/CodeGen/MIR/X86/expected-comma-after-cfi-register.mir b/test/CodeGen/MIR/X86/expected-comma-after-cfi-register.mir
new file mode 100644
index 00000000000..8c4eac037fa
--- /dev/null
+++ b/test/CodeGen/MIR/X86/expected-comma-after-cfi-register.mir
@@ -0,0 +1,44 @@
+# RUN: not llc -march=x86-64 -start-after branch-folder -stop-after branch-folder -o /dev/null %s 2>&1 | FileCheck %s
+
+--- |
+
+  declare void @foo(i32)
+
+  define i32 @test(i32 %a, i32 %b, i32 %c, i32 %d) {
+  entry:
+    %add = add nsw i32 %b, %a
+    %add1 = add nsw i32 %add, %c
+    %add2 = add nsw i32 %add1, %d
+    tail call void @foo(i32 %add2)
+    %add6 = add nsw i32 %add2, %add2
+    ret i32 %add6
+  }
+
+...
+---
+name:            test
+tracksRegLiveness: true
+frameInfo:
+  stackSize:       8
+  adjustsStack:    true
+  hasCalls:        true
+fixedStack:
+  - { id: 0, type: spill-slot, offset: -16, size: 8, alignment: 16 }
+body:
+  - id:              0
+    name:            entry
+    instructions:
+      - 'PUSH64r killed %rbx, implicit-def %rsp, implicit %rsp'
+      - 'CFI_INSTRUCTION .cfi_def_cfa_offset 16'
+      # CHECK: [[@LINE+1]]:43: expected ','
+      - 'CFI_INSTRUCTION .cfi_offset %rbx -16'
+      - '%ebx = COPY %edi, implicit-def %rbx'
+      - '%ebx = ADD32rr %ebx, killed %esi, implicit-def dead %eflags'
+      - '%ebx = ADD32rr %ebx, killed %edx, implicit-def dead %eflags'
+      - '%ebx = ADD32rr %ebx, killed %ecx, implicit-def dead %eflags'
+      - '%edi = COPY %ebx'
+      - 'CALL64pcrel32 @foo, csr_64, implicit %rsp, implicit %edi, implicit-def %rsp'
+      - '%eax = LEA64_32r killed %rbx, 1, %rbx, 0, _'
+      - '%rbx = POP64r implicit-def %rsp, implicit %rsp'
+      - 'RETQ %eax'
+...
diff --git a/test/CodeGen/MIR/X86/expected-different-implicit-operand.mir b/test/CodeGen/MIR/X86/expected-different-implicit-operand.mir
new file mode 100644
index 00000000000..c5f5aaca34e
--- /dev/null
+++ b/test/CodeGen/MIR/X86/expected-different-implicit-operand.mir
@@ -0,0 +1,38 @@
+# RUN: not llc -march=x86-64 -start-after branch-folder -stop-after branch-folder -o /dev/null %s 2>&1 | FileCheck %s
+
+--- |
+
+  define i32 @foo(i32* %p) {
+  entry:
+    %a = load i32, i32* %p
+    %0 = icmp sle i32 %a, 10
+    br i1 %0, label %less, label %exit
+
+  less:
+    ret i32 0
+
+  exit:
+    ret i32 %a
+  }
+
+
+...
+---
+name:            foo
+body:
+ - id:              0
+   name:            entry
+   instructions:
+     - '%eax = MOV32rm %rdi, 1, _, 0, _'
+     - 'CMP32ri8 %eax, 10, implicit-def %eflags'
+# CHECK: [[@LINE+1]]:26: expected an implicit register operand 'implicit %eflags'
+     - 'JG_1 %bb.2.exit, implicit %eax'
+ - id:              1
+   name:            less
+   instructions:
+     - '%eax = MOV32r0 implicit-def %eflags'
+ - id:              2
+   name:            exit
+   instructions:
+     - 'RETQ %eax'
+...
diff --git a/test/CodeGen/MIR/X86/expected-different-implicit-register-flag.mir b/test/CodeGen/MIR/X86/expected-different-implicit-register-flag.mir
new file mode 100644
index 00000000000..ecf3a122bf6
--- /dev/null
+++ b/test/CodeGen/MIR/X86/expected-different-implicit-register-flag.mir
@@ -0,0 +1,38 @@
+# RUN: not llc -march=x86-64 -start-after branch-folder -stop-after branch-folder -o /dev/null %s 2>&1 | FileCheck %s
+
+--- |
+
+  define i32 @foo(i32* %p) {
+  entry:
+    %a = load i32, i32* %p
+    %0 = icmp sle i32 %a, 10
+    br i1 %0, label %less, label %exit
+
+  less:
+    ret i32 0
+
+  exit:
+    ret i32 %a
+  }
+
+
+...
+---
+name:            foo
+body:
+ - id:              0
+   name:            entry
+   instructions:
+     - '%eax = MOV32rm %rdi, 1, _, 0, _'
+     - 'CMP32ri8 %eax, 10, implicit-def %eflags'
+# CHECK: [[@LINE+1]]:26: expected an implicit register operand 'implicit %eflags'
+     - 'JG_1 %bb.2.exit, implicit-def %eflags'
+ - id:              1
+   name:            less
+   instructions:
+     - '%eax = MOV32r0 implicit-def %eflags'
+ - id:              2
+   name:            exit
+   instructions:
+     - 'RETQ %eax'
+...
diff --git a/test/CodeGen/MIR/X86/expected-function-reference-after-blockaddress.mir b/test/CodeGen/MIR/X86/expected-function-reference-after-blockaddress.mir
new file mode 100644
index 00000000000..0b8fe36c025
--- /dev/null
+++ b/test/CodeGen/MIR/X86/expected-function-reference-after-blockaddress.mir
@@ -0,0 +1,34 @@
+# RUN: not llc -march=x86-64 -start-after branch-folder -stop-after branch-folder -o /dev/null %s 2>&1 | FileCheck %s
+
+--- |
+
+  @addr = global i8* null
+
+  define void @test() {
+  entry:
+    store volatile i8* blockaddress(@test, %block), i8** @addr
+    %val = load volatile i8*, i8** @addr
+    indirectbr i8* %val, [label %block]
+
+  block:
+    ret void
+  }
+
+...
+---
+name:            test
+body:
+  - id:              0
+    name:            entry
+    successors:      [ '%bb.1.block' ]
+    instructions:
+      # CHECK: [[@LINE+1]]:49: expected an IR function reference
+      - '%rax = LEA64r %rip, 1, _, blockaddress(@addr, %ir-block.block), _'
+      - 'MOV64mr %rip, 1, _, @addr, _, killed %rax'
+      - 'JMP64m %rip, 1, _, @addr, _'
+  - id:              1
+    name:            block
+    addressTaken:    true
+    instructions:
+      - RETQ
+...
diff --git a/test/CodeGen/MIR/X86/expected-global-value-after-blockaddress.mir b/test/CodeGen/MIR/X86/expected-global-value-after-blockaddress.mir
new file mode 100644
index 00000000000..a42a26cb07c
--- /dev/null
+++ b/test/CodeGen/MIR/X86/expected-global-value-after-blockaddress.mir
@@ -0,0 +1,34 @@
+# RUN: not llc -march=x86-64 -start-after branch-folder -stop-after branch-folder -o /dev/null %s 2>&1 | FileCheck %s
+
+--- |
+
+  @addr = global i8* null
+
+  define void @test() {
+  entry:
+    store volatile i8* blockaddress(@test, %block), i8** @addr
+    %val = load volatile i8*, i8** @addr
+    indirectbr i8* %val, [label %block]
+
+  block:
+    ret void
+  }
+
+...
+---
+name:            test
+body:
+  - id:              0
+    name:            entry
+    successors:      [ '%bb.1.block' ]
+    instructions:
+      # CHECK: [[@LINE+1]]:49: expected a global value
+      - '%rax = LEA64r %rip, 1, _, blockaddress(0, %ir-block.block), _'
+      - 'MOV64mr %rip, 1, _, @addr, _, killed %rax'
+      - 'JMP64m %rip, 1, _, @addr, _'
+  - id:              1
+    name:            block
+    addressTaken:    true
+    instructions:
+      - RETQ
+...
diff --git a/test/CodeGen/MIR/X86/expected-machine-operand.mir b/test/CodeGen/MIR/X86/expected-machine-operand.mir
new file mode 100644
index 00000000000..3725c93cd3e
--- /dev/null
+++ b/test/CodeGen/MIR/X86/expected-machine-operand.mir
@@ -0,0 +1,21 @@
+# RUN: not llc -march=x86-64 -start-after branch-folder -stop-after branch-folder -o /dev/null %s 2>&1 | FileCheck %s
+
+--- |
+
+  define i32 @foo() {
+  entry:
+    ret i32 0
+  }
+
+...
+---
+name:            foo
+body:
+ - id:           0
+   name:         entry
+   instructions:
+     # CHECK: [[@LINE+1]]:24: expected a machine operand
+     - '%eax = XOR32rr ='
+     - 'RETQ %eax'
+...
+
diff --git a/test/CodeGen/MIR/X86/expected-metadata-node-after-debug-location.mir b/test/CodeGen/MIR/X86/expected-metadata-node-after-debug-location.mir
new file mode 100644
index 00000000000..14143751096
--- /dev/null
+++ b/test/CodeGen/MIR/X86/expected-metadata-node-after-debug-location.mir
@@ -0,0 +1,61 @@
+# RUN: not llc -march=x86-64 -start-after machine-sink -stop-after machine-sink -o /dev/null %s 2>&1 | FileCheck %s
+
+--- |
+
+  define i32 @test(i32 %x) #0 {
+  entry:
+    %x.addr = alloca i32, align 4
+    store i32 %x, i32* %x.addr, align 4
+    call void @llvm.dbg.declare(metadata i32* %x.addr, metadata !12, metadata !13), !dbg !14
+    %0 = load i32, i32* %x.addr, align 4, !dbg !15
+    ret i32 %0, !dbg !15
+  }
+
+  declare void @llvm.dbg.declare(metadata, metadata, metadata) #1
+
+  attributes #0 = { nounwind "no-frame-pointer-elim"="false" }
+  attributes #1 = { nounwind readnone }
+
+  !llvm.dbg.cu = !{!0}
+  !llvm.module.flags = !{!9, !10}
+  !llvm.ident = !{!11}
+
+  !0 = !DICompileUnit(language: DW_LANG_C99, file: !1, producer: "clang version 3.7.0", isOptimized: false, runtimeVersion: 0, emissionKind: 1, enums: !2, retainedTypes: !2, subprograms: !3, globals: !2, imports: !2)
+  !1 = !DIFile(filename: "test.ll", directory: "")
+  !2 = !{}
+  !3 = !{!4}
+  !4 = !DISubprogram(name: "test", scope: !5, file: !5, line: 4, type: !6, isLocal: false, isDefinition: true, scopeLine: 4, flags: DIFlagPrototyped, isOptimized: false, function: i32 (i32)* @test, variables: !2)
+  !5 = !DIFile(filename: "test.c", directory: "")
+  !6 = !DISubroutineType(types: !7)
+  !7 = !{!8, !8}
+  !8 = !DIBasicType(name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
+  !9 = !{i32 2, !"Dwarf Version", i32 4}
+  !10 = !{i32 2, !"Debug Info Version", i32 3}
+  !11 = !{!"clang version 3.7.0"}
+  !12 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "x", arg: 1, scope: !4, file: !5, line: 4, type: !8)
+  !13 = !DIExpression()
+  !14 = !DILocation(line: 4, scope: !4)
+  !15 = !DILocation(line: 8, scope: !4)
+
+...
+---
+name:            test
+isSSA:           true
+tracksRegLiveness: true
+registers:
+  - { id: 0, class: gr32 }
+frameInfo:
+  maxAlignment:  4
+stack:
+  - { id: 0, name: x.addr, size: 4, alignment: 4 }
+body:
+  - id:          0
+    name:        entry
+    instructions:
+      - '%0 = COPY %edi'
+      # CHECK: [[@LINE+1]]:51: expected a metadata node after 'debug-location'
+      - 'DBG_VALUE _, 0, !12, !13, debug-location 14'
+      - 'MOV32mr %stack.x.addr, 1, _, 0, _, %0'
+      - '%eax = COPY %0'
+      - 'RETQ %eax'
+...
diff --git a/test/CodeGen/MIR/X86/expected-metadata-node-after-exclaim.mir b/test/CodeGen/MIR/X86/expected-metadata-node-after-exclaim.mir
new file mode 100644
index 00000000000..8cbce13b0c0
--- /dev/null
+++ b/test/CodeGen/MIR/X86/expected-metadata-node-after-exclaim.mir
@@ -0,0 +1,61 @@
+# RUN: not llc -march=x86-64 -start-after machine-sink -stop-after machine-sink -o /dev/null %s 2>&1 | FileCheck %s
+
+--- |
+
+  define i32 @test(i32 %x) #0 {
+  entry:
+    %x.addr = alloca i32, align 4
+    store i32 %x, i32* %x.addr, align 4
+    call void @llvm.dbg.declare(metadata i32* %x.addr, metadata !12, metadata !13), !dbg !14
+    %0 = load i32, i32* %x.addr, align 4, !dbg !15
+    ret i32 %0, !dbg !15
+  }
+
+  declare void @llvm.dbg.declare(metadata, metadata, metadata) #1
+
+  attributes #0 = { nounwind "no-frame-pointer-elim"="false" }
+  attributes #1 = { nounwind readnone }
+
+  !llvm.dbg.cu = !{!0}
+  !llvm.module.flags = !{!9, !10}
+  !llvm.ident = !{!11}
+
+  !0 = !DICompileUnit(language: DW_LANG_C99, file: !1, producer: "clang version 3.7.0", isOptimized: false, runtimeVersion: 0, emissionKind: 1, enums: !2, retainedTypes: !2, subprograms: !3, globals: !2, imports: !2)
+  !1 = !DIFile(filename: "test.ll", directory: "")
+  !2 = !{}
+  !3 = !{!4}
+  !4 = !DISubprogram(name: "test", scope: !5, file: !5, line: 4, type: !6, isLocal: false, isDefinition: true, scopeLine: 4, flags: DIFlagPrototyped, isOptimized: false, function: i32 (i32)* @test, variables: !2)
+  !5 = !DIFile(filename: "test.c", directory: "")
+  !6 = !DISubroutineType(types: !7)
+  !7 = !{!8, !8}
+  !8 = !DIBasicType(name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
+  !9 = !{i32 2, !"Dwarf Version", i32 4}
+  !10 = !{i32 2, !"Debug Info Version", i32 3}
+  !11 = !{!"clang version 3.7.0"}
+  !12 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "x", arg: 1, scope: !4, file: !5, line: 4, type: !8)
+  !13 = !DIExpression()
+  !14 = !DILocation(line: 4, scope: !4)
+  !15 = !DILocation(line: 8, scope: !4)
+
+...
+---
+name:            test
+isSSA:           true
+tracksRegLiveness: true
+registers:
+  - { id: 0, class: gr32 }
+frameInfo:
+  maxAlignment:  4
+stack:
+  - { id: 0, name: x.addr, size: 4, alignment: 4 }
+body:
+  - id:          0
+    name:        entry
+    instructions:
+      - '%0 = COPY %edi'
+      # CHECK: [[@LINE+1]]:33: expected metadata id after '!'
+      - 'DBG_VALUE _, 0, !12, ! _'
+      - 'MOV32mr %stack.0.x.addr, 1, _, 0, _, %0'
+      - '%eax = COPY %0'
+      - 'RETQ %eax'
+...
diff --git a/test/CodeGen/MIR/X86/expected-named-register-in-allocation-hint.mir b/test/CodeGen/MIR/X86/expected-named-register-in-allocation-hint.mir
new file mode 100644
index 00000000000..3dca9ab726b
--- /dev/null
+++ b/test/CodeGen/MIR/X86/expected-named-register-in-allocation-hint.mir
@@ -0,0 +1,30 @@
+# RUN: not llc -march=x86-64 -start-after machine-scheduler -stop-after machine-scheduler -o /dev/null %s 2>&1 | FileCheck %s
+
+--- |
+
+  define i32 @test(i32 %a, i32 %b) {
+  body:
+    %c = mul i32 %a, %b
+    ret i32 %c
+  }
+
+...
+---
+name:            test
+tracksRegLiveness: true
+registers:
+  - { id: 0, class: gr32 }
+  # CHECK: [[@LINE+1]]:48: expected a named register
+  - { id: 1, class: gr32, preferred-register: '%0' }
+  - { id: 2, class: gr32, preferred-register: '%edi' }
+body:
+  - id:              0
+    name:            body
+    liveins:         [ '%edi', '%esi' ]
+    instructions:
+      - '%1 = COPY %esi'
+      - '%2 = COPY %edi'
+      - '%2 = IMUL32rr %2, %1, implicit-def dead %eflags'
+      - '%eax = COPY %2'
+      - 'RETQ killed %eax'
+...
diff --git a/test/CodeGen/MIR/X86/expected-named-register-in-callee-saved-register.mir b/test/CodeGen/MIR/X86/expected-named-register-in-callee-saved-register.mir
new file mode 100644
index 00000000000..bd9e70420b5
--- /dev/null
+++ b/test/CodeGen/MIR/X86/expected-named-register-in-callee-saved-register.mir
@@ -0,0 +1,91 @@
+# RUN: not llc -march=x86-64 -start-after prologepilog -stop-after prologepilog -o /dev/null %s 2>&1 | FileCheck %s
+
+--- |
+
+  define i32 @compute(i32 %a) {
+  body:
+    ret i32 %a
+  }
+
+  define i32 @func(i32 %a) {
+  entry:
+    %b = alloca i32
+    store i32 %a, i32* %b
+    br label %check
+
+  check:
+    %comp = icmp sle i32 %a, 10
+    br i1 %comp, label %loop, label %exit
+
+  loop:
+    %c = load i32, i32* %b
+    %d = call i32 @compute(i32 %c)
+    %e = sub i32 %d, 1
+    store i32 %e, i32* %b
+    br label %check
+
+  exit:
+    ret i32 0
+  }
+
+...
+---
+name:            compute
+tracksRegLiveness: true
+body:
+  - id:          0
+    name:        body
+    liveins:     [ '%edi' ]
+    instructions:
+      - '%eax = COPY killed %edi'
+      - 'RETQ killed %eax'
+...
+---
+name:            func
+tracksRegLiveness: true
+frameInfo:
+  stackSize:       24
+  maxAlignment:    4
+  adjustsStack:    true
+  hasCalls:        true
+fixedStack:
+  # CHECK: [[@LINE+1]]:93: expected a named register
+  - { id: 0, type: spill-slot, offset: -16, size: 8, alignment: 16, callee-saved-register: '%0' }
+stack:
+  - { id: 0, name: b, offset: -20, size: 4, alignment: 4 }
+body:
+  - id:          0
+    name:        entry
+    successors:  [ '%bb.1.check' ]
+    liveins:     [ '%edi', '%rbx' ]
+    instructions:
+      - 'frame-setup PUSH64r killed %rbx, implicit-def %rsp, implicit %rsp'
+      - '%rsp = frame-setup SUB64ri8 %rsp, 16, implicit-def dead %eflags'
+      - '%ebx = COPY %edi'
+      - 'MOV32mr %rsp, 1, _, 12, _, %ebx'
+  - id:          1
+    name:        check
+    successors:  [ '%bb.2.loop', '%bb.3.exit' ]
+    liveins:     [ '%ebx' ]
+    instructions:
+      - 'CMP32ri8 %ebx, 10, implicit-def %eflags'
+      - 'JG_1 %bb.3.exit, implicit killed %eflags'
+      - 'JMP_1 %bb.2.loop'
+  - id:          2
+    name:        loop
+    successors:  [ '%bb.1.check' ]
+    liveins:     [ '%ebx' ]
+    instructions:
+      - '%edi = MOV32rm %rsp, 1, _, 12, _'
+      - 'CALL64pcrel32 @compute, csr_64, implicit %rsp, implicit %edi, implicit-def %rsp, implicit-def %eax'
+      - '%eax = DEC32r killed %eax, implicit-def dead %eflags'
+      - 'MOV32mr %rsp, 1, _, 12, _, killed %eax'
+      - 'JMP_1 %bb.1.check'
+  - id:          3
+    name:        exit
+    instructions:
+      - '%eax = MOV32r0 implicit-def dead %eflags'
+      - '%rsp = ADD64ri8 %rsp, 16, implicit-def dead %eflags'
+      - '%rbx = POP64r implicit-def %rsp, implicit %rsp'
+      - 'RETQ %eax'
+...
diff --git a/test/CodeGen/MIR/X86/expected-named-register-in-functions-livein.mir b/test/CodeGen/MIR/X86/expected-named-register-in-functions-livein.mir
new file mode 100644
index 00000000000..b14ff32694a
--- /dev/null
+++ b/test/CodeGen/MIR/X86/expected-named-register-in-functions-livein.mir
@@ -0,0 +1,28 @@
+# RUN: not llc -march=x86-64 -start-after machine-sink -stop-after machine-sink -o /dev/null %s 2>&1 | FileCheck %s
+
+--- |
+
+  define i32 @test(i32 %a) {
+  body:
+    ret i32 %a
+  }
+
+...
+---
+name:            test
+isSSA:           true
+tracksRegLiveness: true
+registers:
+  - { id: 0, class: gr32 }
+liveins:
+  # CHECK: [[@LINE+1]]:13: expected a named register
+  - { reg: '%0' }
+body:
+  - id:          0
+    name:        body
+    liveins:     [ '%edi' ]
+    instructions:
+      - '%0 = COPY %edi'
+      - '%eax = COPY %0'
+      - 'RETQ %eax'
+...
diff --git a/test/CodeGen/MIR/X86/expected-named-register-livein.mir b/test/CodeGen/MIR/X86/expected-named-register-livein.mir
new file mode 100644
index 00000000000..1fbe881c8c7
--- /dev/null
+++ b/test/CodeGen/MIR/X86/expected-named-register-livein.mir
@@ -0,0 +1,21 @@
+# RUN: not llc -march=x86-64 -start-after branch-folder -stop-after branch-folder -o /dev/null %s 2>&1 | FileCheck %s
+
+--- |
+
+  define i32 @test(i32 %a) {
+  body:
+    ret i32 %a
+  }
+
+...
+---
+name:            test
+body:
+  - id:          0
+    name:        body
+    # CHECK: [[@LINE+1]]:21: expected a named register
+    liveins:     [ '%0' ]
+    instructions:
+      - '%eax = COPY %edi'
+      - 'RETQ %eax'
+...
diff --git a/test/CodeGen/MIR/X86/expected-number-after-bb.mir b/test/CodeGen/MIR/X86/expected-number-after-bb.mir
new file mode 100644
index 00000000000..5343a847fbb
--- /dev/null
+++ b/test/CodeGen/MIR/X86/expected-number-after-bb.mir
@@ -0,0 +1,37 @@
+# RUN: not llc -march=x86-64 -start-after branch-folder -stop-after branch-folder -o /dev/null %s 2>&1 | FileCheck %s
+
+--- |
+
+  define i32 @foo(i32* %p) {
+  entry:
+    %a = load i32, i32* %p
+    %b = icmp sle i32 %a, 10
+    br i1 %b, label %yes, label %nah
+
+  yes:
+    ret i32 0
+
+  nah:
+    ret i32 %a
+  }
+
+...
+---
+name:            foo
+body:
+ - id: 0
+   name:   entry
+   instructions:
+     - '%eax = MOV32rm %rdi, 1, _, 0, _'
+     - 'CMP32ri8 %eax, 10, implicit-def %eflags'
+     # CHECK: [[@LINE+1]]:18: expected a number after '%bb.'
+     - 'JG_1 %bb.nah, implicit %eflags'
+ - id: 1
+   name: yes
+   instructions:
+     - '%eax = MOV32r0 implicit-def %eflags'
+ - id: 2
+   name: nah
+   instructions:
+     - 'RETQ %eax'
+...
diff --git a/test/CodeGen/MIR/X86/expected-offset-after-cfi-operand.mir b/test/CodeGen/MIR/X86/expected-offset-after-cfi-operand.mir
new file mode 100644
index 00000000000..42fb1400268
--- /dev/null
+++ b/test/CodeGen/MIR/X86/expected-offset-after-cfi-operand.mir
@@ -0,0 +1,29 @@
+# RUN: not llc -march=x86-64 -start-after branch-folder -stop-after branch-folder -o /dev/null %s 2>&1 | FileCheck %s
+
+--- |
+
+  define void @test() {
+  entry:
+    %tmp = alloca [4168 x i8], align 4
+    ret void
+  }
+
+...
+---
+name:            test
+tracksRegLiveness: true
+frameInfo:
+  stackSize:       4040
+stack:
+  - { id: 0, name: tmp, offset: -4176, size: 4168, alignment: 4 }
+body:
+  - id:              0
+    name:            entry
+    instructions:
+      - '%rsp = SUB64ri32 %rsp, 4040, implicit-def dead %eflags'
+     # CHECK: [[@LINE+1]]:46: expected a cfi offset
+      - 'CFI_INSTRUCTION .cfi_def_cfa_offset _'
+      - '%rsp = ADD64ri32 %rsp, 4040, implicit-def dead %eflags'
+      - 'RETQ'
+...
+
diff --git a/test/CodeGen/MIR/X86/expected-register-after-cfi-operand.mir b/test/CodeGen/MIR/X86/expected-register-after-cfi-operand.mir
new file mode 100644
index 00000000000..7178c1d4568
--- /dev/null
+++ b/test/CodeGen/MIR/X86/expected-register-after-cfi-operand.mir
@@ -0,0 +1,44 @@
+# RUN: not llc -march=x86-64 -start-after branch-folder -stop-after branch-folder -o /dev/null %s 2>&1 | FileCheck %s
+
+--- |
+
+  declare void @foo(i32)
+
+  define i32 @test(i32 %a, i32 %b, i32 %c, i32 %d) {
+  entry:
+    %add = add nsw i32 %b, %a
+    %add1 = add nsw i32 %add, %c
+    %add2 = add nsw i32 %add1, %d
+    tail call void @foo(i32 %add2)
+    %add6 = add nsw i32 %add2, %add2
+    ret i32 %add6
+  }
+
+...
+---
+name:            test
+tracksRegLiveness: true
+frameInfo:
+  stackSize:       8
+  adjustsStack:    true
+  hasCalls:        true
+fixedStack:
+  - { id: 0, type: spill-slot, offset: -16, size: 8, alignment: 16 }
+body:
+  - id:              0
+    name:            entry
+    instructions:
+      - 'PUSH64r killed %rbx, implicit-def %rsp, implicit %rsp'
+      - 'CFI_INSTRUCTION .cfi_def_cfa_offset 16'
+      # CHECK: [[@LINE+1]]:38: expected a cfi register
+      - 'CFI_INSTRUCTION .cfi_offset %0, -16'
+      - '%ebx = COPY %edi, implicit-def %rbx'
+      - '%ebx = ADD32rr %ebx, killed %esi, implicit-def dead %eflags'
+      - '%ebx = ADD32rr %ebx, killed %edx, implicit-def dead %eflags'
+      - '%ebx = ADD32rr %ebx, killed %ecx, implicit-def dead %eflags'
+      - '%edi = COPY %ebx'
+      - 'CALL64pcrel32 @foo, csr_64, implicit %rsp, implicit %edi, implicit-def %rsp'
+      - '%eax = LEA64_32r killed %rbx, 1, %rbx, 0, _'
+      - '%rbx = POP64r implicit-def %rsp, implicit %rsp'
+      - 'RETQ %eax'
+...
diff --git a/test/CodeGen/MIR/X86/expected-register-after-flags.mir b/test/CodeGen/MIR/X86/expected-register-after-flags.mir
new file mode 100644
index 00000000000..111f5496a37
--- /dev/null
+++ b/test/CodeGen/MIR/X86/expected-register-after-flags.mir
@@ -0,0 +1,22 @@
+# RUN: not llc -march=x86-64 -start-after branch-folder -stop-after branch-folder -o /dev/null %s 2>&1 | FileCheck %s
+# This test ensures that an error is reported when a register operand doesn't
+# follow register flags.
+
+--- |
+
+  define i32 @foo() {
+  entry:
+    ret i32 0
+  }
+
+...
+---
+name:            foo
+body:
+ - id:           0
+   name:         entry
+   instructions:
+     # CHECK: [[@LINE+1]]:37: expected a register after register flags
+     - '%eax = MOV32r0 implicit-def 2'
+     - 'RETQ %eax'
+...
diff --git a/test/CodeGen/MIR/X86/expected-subregister-after-colon.mir b/test/CodeGen/MIR/X86/expected-subregister-after-colon.mir
new file mode 100644
index 00000000000..c891a115a18
--- /dev/null
+++ b/test/CodeGen/MIR/X86/expected-subregister-after-colon.mir
@@ -0,0 +1,29 @@
+# RUN: not llc -march=x86-64 -start-after machine-sink -stop-after machine-sink -o /dev/null %s 2>&1 | FileCheck %s
+
+--- |
+
+  define zeroext i1 @t(i1 %c) {
+  entry:
+    ret i1 %c
+  }
+
+...
+---
+name:            t
+isSSA:           true
+tracksRegLiveness: true
+registers:
+  - { id: 0, class: gr32 }
+  - { id: 1, class: gr8 }
+  - { id: 2, class: gr8 }
+body:
+  - name:        entry
+    id:          0
+    instructions:
+      - '%0 = COPY %edi'
+      # CHECK: [[@LINE+1]]:25: expected a subregister index after ':'
+      - '%1 = COPY %0 : 42'
+      - '%2 = AND8ri %1, 1, implicit-def %eflags'
+      - '%al = COPY %2'
+      - 'RETQ %al'
+...
diff --git a/test/CodeGen/MIR/X86/expected-virtual-register-in-functions-livein.mir b/test/CodeGen/MIR/X86/expected-virtual-register-in-functions-livein.mir
new file mode 100644
index 00000000000..edadf42056b
--- /dev/null
+++ b/test/CodeGen/MIR/X86/expected-virtual-register-in-functions-livein.mir
@@ -0,0 +1,28 @@
+# RUN: not llc -march=x86-64 -start-after machine-sink -stop-after machine-sink -o /dev/null %s 2>&1 | FileCheck %s
+
+--- |
+
+  define i32 @test(i32 %a) {
+  body:
+    ret i32 %a
+  }
+
+...
+---
+name:            test
+isSSA:           true
+tracksRegLiveness: true
+registers:
+  - { id: 0, class: gr32 }
+liveins:
+  # CHECK: [[@LINE+1]]:34: expected a virtual register
+  - { reg: '%edi', virtual-reg: '%edi' }
+body:
+  - id:          0
+    name:        body
+    liveins:     [ '%edi' ]
+    instructions:
+      - '%0 = COPY %edi'
+      - '%eax = COPY %0'
+      - 'RETQ %eax'
+...
diff --git a/test/CodeGen/MIR/X86/external-symbol-operands.mir b/test/CodeGen/MIR/X86/external-symbol-operands.mir
new file mode 100644
index 00000000000..78d66459af5
--- /dev/null
+++ b/test/CodeGen/MIR/X86/external-symbol-operands.mir
@@ -0,0 +1,62 @@
+# RUN: llc -march=x86-64 -start-after branch-folder -stop-after branch-folder -o /dev/null %s | FileCheck %s
+# This test ensures that the MIR parser parses the external symbol machine
+# operands correctly.
+
+--- |
+  @__stack_chk_guard = external global i8*
+
+  define i32 @test(i32 %n) #0 {
+  entry:
+    %StackGuardSlot = alloca i8*
+    %StackGuard = load i8*, i8** @__stack_chk_guard
+    call void @llvm.stackprotector(i8* %StackGuard, i8** %StackGuardSlot)
+    %a = alloca [128 x i32], align 16
+    %idxprom = sext i32 %n to i64
+    %arrayidx = getelementptr inbounds [128 x i32], [128 x i32]* %a, i64 0, i64 %idxprom
+    %0 = load i32, i32* %arrayidx, align 4
+    call void @llvm.stackprotectorcheck(i8** @__stack_chk_guard)
+    ret i32 %0
+  }
+
+  declare void @llvm.stackprotector(i8*, i8**) #1
+
+  declare void @llvm.stackprotectorcheck(i8**) #1
+
+  attributes #0 = { ssp "stack-protector-buffer-size"="8" }
+  attributes #1 = { nounwind }
+
+...
+---
+name:            test
+tracksRegLiveness: true
+body:
+  - id:              0
+    name:            entry
+    successors:      [ '%bb.1.entry', '%bb.2.entry' ]
+    liveins:         [ '%edi' ]
+    instructions:
+      - '%rsp = SUB64ri32 %rsp, 520, implicit-def %eflags'
+      - '%rcx = LOAD_STACK_GUARD'
+      - 'MOV64mr %rsp, 1, _, 512, _, %rcx'
+      - '%rax = MOVSX64rr32 %edi'
+      - '%eax = MOV32rm %rsp, 4, %rax, 0, _'
+      - 'CMP64rm %rcx, %rsp, 1, _, 512, _, implicit-def %eflags'
+      - 'JNE_1 %bb.2.entry, implicit %eflags'
+  - id:              1
+    name:            entry
+    liveins:         [ '%eax' ]
+    instructions:
+      - '%rsp = ADD64ri32 %rsp, 520, implicit-def %eflags'
+      - 'RETQ %eax'
+  - id:              2
+    name:            entry
+    instructions:
+      # CHECK:      CALL64pcrel32 $__stack_chk_fail,
+      # CHECK-NEXT: CALL64pcrel32 $__stack_chk_fail.09-_,
+      # CHECK-NEXT: CALL64pcrel32 $"__stack_chk_fail$",
+      # CHECK-NEXT: CALL64pcrel32 $"$Quoted \09 External symbol \11 ",
+      - 'CALL64pcrel32 $__stack_chk_fail, csr_64, implicit %rsp, implicit-def %rsp'
+      - 'CALL64pcrel32 $__stack_chk_fail.09-_, csr_64, implicit %rsp, implicit-def %rsp'
+      - 'CALL64pcrel32 $__stack_chk_fail$, csr_64, implicit %rsp, implicit-def %rsp'
+      - 'CALL64pcrel32 $"$Quoted \09 External symbol \11 ", csr_64, implicit %rsp, implicit-def %rsp'
+...
diff --git a/test/CodeGen/MIR/X86/fixed-stack-objects.mir b/test/CodeGen/MIR/X86/fixed-stack-objects.mir
new file mode 100644
index 00000000000..dcbe6f73a6d
--- /dev/null
+++ b/test/CodeGen/MIR/X86/fixed-stack-objects.mir
@@ -0,0 +1,35 @@
+# RUN: llc -march=x86 -start-after branch-folder -stop-after branch-folder -o /dev/null %s | FileCheck %s
+# This test ensures that the MIR parser parses fixed stack objects correctly.
+
+--- |
+
+  define i32 @test(i32 %a) #0 {
+  entry:
+    %b = alloca i32
+    store i32 %a, i32* %b
+    %c = load i32, i32* %b
+    ret i32 %c
+  }
+
+  attributes #0 = { "no-frame-pointer-elim"="false" }
+
+...
+---
+name:            test
+frameInfo:
+  stackSize:       4
+  maxAlignment:    4
+# CHECK: fixedStack:
+# CHECK-NEXT: - { id: 0, offset: 0, size: 4, alignment: 4, isImmutable: true, isAliased: false }
+fixedStack:
+  - { id: 0, offset: 0, size: 4, alignment: 4, isImmutable: true, isAliased: false }
+stack:
+  - { id: 0, offset: -8, size: 4, alignment: 4 }
+body:
+  - id:          0
+    name:        entry
+    instructions:
+      - '%eax = MOV32rm %esp, 1, _, 8, _'
+      - 'MOV32mr %esp, 1, _, 0, _, %eax'
+      - 'RETL %eax'
+...
diff --git a/test/CodeGen/MIR/X86/frame-info-save-restore-points.mir b/test/CodeGen/MIR/X86/frame-info-save-restore-points.mir
new file mode 100644
index 00000000000..9bf8d0d77a9
--- /dev/null
+++ b/test/CodeGen/MIR/X86/frame-info-save-restore-points.mir
@@ -0,0 +1,72 @@
+# RUN: llc -march=x86-64 -enable-shrink-wrap=true -start-after shrink-wrap -stop-after shrink-wrap -o /dev/null %s | FileCheck %s
+# This test ensures that the MIR parser parses the save and restore points in
+# the machine frame info correctly.
+
+--- |
+
+  define i32 @foo(i32 %a, i32 %b) {
+  entry:
+    %tmp = alloca i32, align 4
+    %tmp2 = icmp slt i32 %a, %b
+    br i1 %tmp2, label %true, label %false
+
+  true:
+    store i32 %a, i32* %tmp, align 4
+    %tmp4 = call i32 @doSomething(i32 0, i32* %tmp)
+    br label %false
+
+  false:
+    %tmp.0 = phi i32 [ %tmp4, %true ], [ %a, %entry ]
+    ret i32 %tmp.0
+  }
+
+  declare i32 @doSomething(i32, i32*)
+
+...
+---
+name:            foo
+tracksRegLiveness: true
+liveins:
+  - { reg: '%edi' }
+  - { reg: '%esi' }
+# CHECK: frameInfo:
+# CHECK:      savePoint: '%bb.2.true'
+# CHECK-NEXT: restorePoint: '%bb.2.true'
+# CHECK: stack
+frameInfo:
+  maxAlignment:  4
+  hasCalls:      true
+  savePoint:     '%bb.2.true'
+  restorePoint:  '%bb.2.true'
+stack:
+  - { id: 0, name: tmp, offset: 0, size: 4, alignment: 4 }
+body:
+  - id:          0
+    successors:  [ '%bb.2.true', '%bb.1' ]
+    liveins:     [ '%edi', '%esi' ]
+    instructions:
+      - '%eax = COPY %edi'
+      - 'CMP32rr %eax, killed %esi, implicit-def %eflags'
+      - 'JL_1 %bb.2.true, implicit killed %eflags'
+  - id:          1
+    successors:  [ '%bb.3.false' ]
+    liveins:     [ '%eax' ]
+    instructions:
+      - 'JMP_1 %bb.3.false'
+  - id:          2
+    name:        'true'
+    successors:  [ '%bb.3.false' ]
+    liveins:     [ '%eax' ]
+    instructions:
+      - 'MOV32mr %stack.0.tmp, 1, _, 0, _, killed %eax'
+      - 'ADJCALLSTACKDOWN64 0, 0, implicit-def %rsp, implicit-def dead %eflags, implicit %rsp'
+      - '%rsi = LEA64r %stack.0.tmp, 1, _, 0, _'
+      - '%edi = MOV32r0 implicit-def dead %eflags'
+      - 'CALL64pcrel32 @doSomething, csr_64, implicit %rsp, implicit %edi, implicit %rsi, implicit-def %rsp, implicit-def %eax'
+      - 'ADJCALLSTACKUP64 0, 0, implicit-def %rsp, implicit-def dead %eflags, implicit %rsp'
+  - id:          3
+    name:        'false'
+    liveins:     [ '%eax' ]
+    instructions:
+      - 'RETQ %eax'
+...
diff --git a/test/CodeGen/MIR/X86/frame-setup-instruction-flag.mir b/test/CodeGen/MIR/X86/frame-setup-instruction-flag.mir
new file mode 100644
index 00000000000..ca34fe1050d
--- /dev/null
+++ b/test/CodeGen/MIR/X86/frame-setup-instruction-flag.mir
@@ -0,0 +1,39 @@
+# RUN: llc -march=x86-64 -start-after branch-folder -stop-after branch-folder -o /dev/null %s | FileCheck %s
+# This test ensures that the MIR parser parses the frame setup instruction flag.
+
+--- |
+
+  define i32 @compute(i32 %a) {
+  body:
+    %c = mul i32 %a, 11
+    ret i32 %c
+  }
+
+  define i32 @foo(i32 %a) {
+  entry:
+    %b = call i32 @compute(i32 %a)
+    ret i32 %b
+  }
+
+...
+---
+name:            compute
+body:
+  - name:        body
+    id:          0
+    instructions:
+      - '%eax = IMUL32rri8 %edi, 11, implicit-def %eflags'
+      - 'RETQ %eax'
+...
+---
+name:            foo
+body:
+  - name:        entry
+    id:          0
+    instructions:
+      # CHECK: frame-setup PUSH64r %rax
+      - 'frame-setup PUSH64r %rax, implicit-def %rsp, implicit %rsp'
+      - 'CALL64pcrel32 @compute, csr_64, implicit %rsp, implicit %edi, implicit-def %rsp, implicit-def %eax'
+      - '%rdx = POP64r implicit-def %rsp, implicit %rsp'
+      - 'RETQ %eax'
+...
diff --git a/test/CodeGen/MIR/X86/function-liveins.mir b/test/CodeGen/MIR/X86/function-liveins.mir
new file mode 100644
index 00000000000..d3c9d3f72cd
--- /dev/null
+++ b/test/CodeGen/MIR/X86/function-liveins.mir
@@ -0,0 +1,38 @@
+# RUN: llc -march=x86-64 -start-after machine-sink -stop-after machine-sink -o /dev/null %s | FileCheck %s
+# This test ensures that the MIR parser parses machine function's liveins
+# correctly.
+
+--- |
+
+  define i32 @test(i32 %a, i32 %b) {
+  body:
+    %c = add i32 %a, %b
+    ret i32 %c
+  }
+
+...
+---
+name:            test
+isSSA:           true
+tracksRegLiveness: true
+registers:
+  - { id: 0, class: gr32 }
+  - { id: 1, class: gr32 }
+  - { id: 2, class: gr32 }
+# CHECK: liveins:
+# CHECK-NEXT: - { reg: '%edi', virtual-reg: '%0' }
+# CHECK-NEXT: - { reg: '%esi', virtual-reg: '%1' }
+liveins:
+  - { reg: '%edi', virtual-reg: '%0' }
+  - { reg: '%esi', virtual-reg: '%1' }
+body:
+  - id:              0
+    name:            body
+    liveins:         [ '%edi', '%esi' ]
+    instructions:
+      - '%1 = COPY %esi'
+      - '%0 = COPY %edi'
+      - '%2 = ADD32rr %0, %1, implicit-def dead %eflags'
+      - '%eax = COPY %2'
+      - 'RETQ %eax'
+...
diff --git a/test/CodeGen/MIR/X86/global-value-operands.mir b/test/CodeGen/MIR/X86/global-value-operands.mir
new file mode 100644
index 00000000000..775a01ba003
--- /dev/null
+++ b/test/CodeGen/MIR/X86/global-value-operands.mir
@@ -0,0 +1,102 @@
+# RUN: llc -march=x86-64 -start-after branch-folder -stop-after branch-folder -o /dev/null %s | FileCheck %s
+# This test ensures that the MIR parser parses global value operands correctly.
+
+--- |
+
+  @G = external global i32
+  @0 = external global i32
+
+  define i32 @inc() {
+  entry:
+    %a = load i32, i32* @G
+    %b = add i32 %a, 1
+    ret i32 %b
+  }
+
+  define i32 @inc2() {
+  entry:
+    %a = load i32, i32* @0
+    %b = add i32 %a, 1
+    ret i32 %b
+  }
+
+  @.$0  = external global i32
+  @-_-  = external global i32
+  @_-_a = external global i32
+  @$.-B = external global i32
+
+  define i32 @test() {
+  entry:
+    %a = load i32, i32* @.$0
+    store i32 %a, i32* @-_-
+    %b = load i32, i32* @_-_a
+    store i32 %b, i32* @$.-B
+    ret i32 %b
+  }
+
+  @"\01Hello@$%09 \\ World," = external global i32
+
+  define i32 @test2() {
+  entry:
+    %a = load i32, i32* @"\01Hello@$%09 \\ World,"
+    ret i32 %a
+  }
+
+...
+---
+# CHECK: name: inc
+name: inc
+body:
+  - id: 0
+    name: entry
+    instructions:
+      # CHECK: - '%rax = MOV64rm %rip, 1, _, @G, _'
+      - '%rax = MOV64rm %rip, 1, _, @G, _'
+      - '%eax = MOV32rm %rax, 1, _, 0, _'
+      - '%eax = INC32r %eax, implicit-def %eflags'
+      - 'RETQ %eax'
+...
+---
+# CHECK: name: inc2
+name: inc2
+body:
+  - id: 0
+    name: entry
+    instructions:
+      # CHECK: - '%rax = MOV64rm %rip, 1, _, @0, _'
+      - '%rax = MOV64rm %rip, 1, _, @0, _'
+      - '%eax = MOV32rm %rax, 1, _, 0, _'
+      - '%eax = INC32r %eax, implicit-def %eflags'
+      - 'RETQ %eax'
+...
+---
+name:            test
+body:
+  - id:              0
+    name:            entry
+    instructions:
+      # CHECK: , @".$0",
+      # CHECK: , @-_-,
+      # CHECK: , @_-_a,
+      # CHECK: , @"$.-B",
+      - '%rax = MOV64rm %rip, 1, _, @.$0, _'
+      - '%eax = MOV32rm killed %rax, 1, _, 0, _'
+      - '%rcx = MOV64rm %rip, 1, _, @-_-, _'
+      - 'MOV32mr killed %rcx, 1, _, 0, _, killed %eax'
+      - '%rax = MOV64rm %rip, 1, _, @_-_a, _'
+      - '%eax = MOV32rm killed %rax, 1, _, 0, _'
+      - '%rcx = MOV64rm %rip, 1, _, @$.-B, _'
+      - 'MOV32mr killed %rcx, 1, _, 0, _, %eax'
+      - 'RETQ %eax'
+...
+---
+name:            test2
+body:
+  - id:              0
+    name:            entry
+    instructions:
+      # CHECK: , @"\01Hello@$%09 \5C World,",
+      - '%rax = MOV64rm %rip, 1, _, @"\01Hello@$%09 \\ World,", _'
+      - '%eax = MOV32rm killed %rax, 1, _, 0, _'
+      - 'RETQ %eax'
+...
diff --git a/test/CodeGen/MIR/X86/immediate-operands.mir b/test/CodeGen/MIR/X86/immediate-operands.mir
new file mode 100644
index 00000000000..5d4956f539d
--- /dev/null
+++ b/test/CodeGen/MIR/X86/immediate-operands.mir
@@ -0,0 +1,40 @@
+# RUN: llc -march=x86-64 -start-after branch-folder -stop-after branch-folder -o /dev/null %s | FileCheck %s
+# This test ensures that the MIR parser parses immediate machine operands.
+
+--- |
+
+  define i32 @foo() {
+  entry:
+    ret i32 42
+  }
+
+  define i32 @bar() {
+  entry:
+    ret i32 -11
+  }
+
+...
+---
+# CHECK: name: foo
+name:            foo
+body:
+ - id:           0
+   name:         entry
+   instructions:
+     # CHECK:      - '%eax = MOV32ri 42'
+     # CHECK-NEXT: - 'RETQ %eax'
+     - '%eax = MOV32ri 42'
+     - 'RETQ %eax'
+...
+---
+# CHECK: name: bar
+name:            bar
+body:
+ - id:           0
+   name:         entry
+   instructions:
+     # CHECK:      - '%eax = MOV32ri -11'
+     # CHECK-NEXT: - 'RETQ %eax'
+     - '%eax = MOV32ri -11'
+     - 'RETQ %eax'
+...
diff --git a/test/CodeGen/MIR/X86/implicit-register-flag.mir b/test/CodeGen/MIR/X86/implicit-register-flag.mir
new file mode 100644
index 00000000000..cf2c930b64b
--- /dev/null
+++ b/test/CodeGen/MIR/X86/implicit-register-flag.mir
@@ -0,0 +1,42 @@
+# RUN: llc -march=x86-64 -start-after branch-folder -stop-after branch-folder -o /dev/null %s | FileCheck %s
+# This test ensures that the MIR parser parses the 'implicit' and 'implicit-def'
+# register flags correctly.
+
+--- |
+
+  define i32 @foo(i32 %a) {
+  entry:
+    %0 = icmp sle i32 %a, 10
+    br i1 %0, label %less, label %exit
+
+  less:
+    ret i32 0
+
+  exit:
+    ret i32 %a
+  }
+
+...
+---
+name:            foo
+body:
+  - id:          0
+    name:        entry
+    successors:  [ '%bb.1.less', '%bb.2.exit' ]
+    instructions:
+      # CHECK:      - 'CMP32ri8 %edi, 10, implicit-def %eflags'
+      # CHECK-NEXT: - 'JG_1 %bb.2.exit, implicit %eflags'
+      - 'CMP32ri8 %edi, 10, implicit-def %eflags'
+      - 'JG_1 %bb.2.exit, implicit %eflags'
+  - id:          1
+    name:        less
+    instructions:
+      # CHECK: - '%eax = MOV32r0 implicit-def %eflags'
+      - '%eax = MOV32r0 implicit-def %eflags'
+      - 'RETQ %eax'
+  - id:          2
+    name:        exit
+    instructions:
+      - '%eax = COPY %edi'
+      - 'RETQ %eax'
+...
diff --git a/test/CodeGen/MIR/X86/instructions-debug-location.mir b/test/CodeGen/MIR/X86/instructions-debug-location.mir
new file mode 100644
index 00000000000..dc8eae1c0c9
--- /dev/null
+++ b/test/CodeGen/MIR/X86/instructions-debug-location.mir
@@ -0,0 +1,66 @@
+# RUN: llc -march=x86-64 -start-after machine-sink -stop-after machine-sink -o /dev/null %s | FileCheck %s
+# This test ensures that the MIR parser parses the machine instruction's
+# debug location metadata correctly.
+
+--- |
+
+  define i32 @test(i32 %x) #0 {
+  entry:
+    %x.addr = alloca i32, align 4
+    store i32 %x, i32* %x.addr, align 4
+    call void @llvm.dbg.declare(metadata i32* %x.addr, metadata !12, metadata !13), !dbg !14
+    %0 = load i32, i32* %x.addr, align 4, !dbg !15
+    ret i32 %0, !dbg !15
+  }
+
+  declare void @llvm.dbg.declare(metadata, metadata, metadata) #1
+
+  attributes #0 = { nounwind "no-frame-pointer-elim"="false" }
+  attributes #1 = { nounwind readnone }
+
+  !llvm.dbg.cu = !{!0}
+  !llvm.module.flags = !{!9, !10}
+  !llvm.ident = !{!11}
+
+  !0 = !DICompileUnit(language: DW_LANG_C99, file: !1, producer: "clang version 3.7.0", isOptimized: false, runtimeVersion: 0, emissionKind: 1, enums: !2, retainedTypes: !2, subprograms: !3, globals: !2, imports: !2)
+  !1 = !DIFile(filename: "test.ll", directory: "")
+  !2 = !{}
+  !3 = !{!4}
+  !4 = !DISubprogram(name: "test", scope: !5, file: !5, line: 4, type: !6, isLocal: false, isDefinition: true, scopeLine: 4, flags: DIFlagPrototyped, isOptimized: false, function: i32 (i32)* @test, variables: !2)
+  !5 = !DIFile(filename: "test.c", directory: "")
+  !6 = !DISubroutineType(types: !7)
+  !7 = !{!8, !8}
+  !8 = !DIBasicType(name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
+  !9 = !{i32 2, !"Dwarf Version", i32 4}
+  !10 = !{i32 2, !"Debug Info Version", i32 3}
+  !11 = !{!"clang version 3.7.0"}
+  !12 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "x", arg: 1, scope: !4, file: !5, line: 4, type: !8)
+  !13 = !DIExpression()
+  !14 = !DILocation(line: 4, scope: !4)
+  !15 = !DILocation(line: 8, scope: !4)
+
+...
+---
+name:            test
+isSSA:           true
+tracksRegLiveness: true
+registers:
+  - { id: 0, class: gr32 }
+frameInfo:
+  maxAlignment:  4
+stack:
+  - { id: 0, name: x.addr, size: 4, alignment: 4 }
+body:
+  - id:           0
+    name:         entry
+    liveins:      [ '%edi' ]
+    instructions:
+      # CHECK: DBG_VALUE _, 0, !12, !13, debug-location !14
+      # CHECK: %eax = COPY %0, debug-location !15
+      # CHECK: RETQ %eax, debug-location !15
+      - '%0 = COPY %edi'
+      - 'DBG_VALUE _, 0, !12, !13, debug-location !14'
+      - 'MOV32mr %stack.0.x.addr, 1, _, 0, _, %0'
+      - '%eax = COPY %0, debug-location !15'
+      - 'RETQ %eax, debug-location !15'
+...
diff --git a/test/CodeGen/MIR/X86/invalid-constant-pool-item.mir b/test/CodeGen/MIR/X86/invalid-constant-pool-item.mir
new file mode 100644
index 00000000000..de2ca4cca75
--- /dev/null
+++ b/test/CodeGen/MIR/X86/invalid-constant-pool-item.mir
@@ -0,0 +1,27 @@
+# RUN: not llc -march=x86-64 -start-after branch-folder -stop-after branch-folder -o /dev/null %s 2>&1 | FileCheck %s
+# This test ensures that the MIR parser reports an error when parsing an invalid
+# constant pool item operand.
+
+--- |
+
+  define double @test(double %a, float %b) {
+  entry:
+    %c = fadd double %a, 3.250000e+00
+    ret double %c
+  }
+
+...
+---
+name:            test
+constants:
+  - id:          0
+    value:       'double 3.250000e+00'
+body:
+  - id: 0
+    name: entry
+    instructions:
+      # CHECK: [[@LINE+1]]:52: use of undefined constant '%const.10'
+      - '%xmm0 = ADDSDrm killed %xmm0, %rip, 1, _, %const.10, _'
+      - 'RETQ %xmm0'
+...
+
diff --git a/test/CodeGen/MIR/X86/jump-table-info.mir b/test/CodeGen/MIR/X86/jump-table-info.mir
new file mode 100644
index 00000000000..25cf9240526
--- /dev/null
+++ b/test/CodeGen/MIR/X86/jump-table-info.mir
@@ -0,0 +1,164 @@
+# RUN: llc -march=x86-64 -start-after branch-folder -stop-after branch-folder -o /dev/null %s | FileCheck %s
+# This test ensures that the MIR parser parses the jump table info and jump
+# table operands correctly.
+
+--- |
+
+  define i32 @test_jumptable(i32 %in) {
+  entry:
+    switch i32 %in, label %def [
+      i32 0, label %lbl1
+      i32 1, label %lbl2
+      i32 2, label %lbl3
+      i32 3, label %lbl4
+    ]
+
+  def:
+    ret i32 0
+
+  lbl1:
+    ret i32 1
+
+  lbl2:
+    ret i32 2
+
+  lbl3:
+    ret i32 4
+
+  lbl4:
+    ret i32 8
+  }
+
+  define i32 @test_jumptable2(i32 %in) {
+  entry:
+    switch i32 %in, label %def [
+      i32 0, label %lbl1
+      i32 1, label %lbl2
+      i32 2, label %lbl3
+      i32 3, label %lbl4
+    ]
+
+  def:
+    ret i32 0
+
+  lbl1:
+    ret i32 1
+
+  lbl2:
+    ret i32 2
+
+  lbl3:
+    ret i32 4
+
+  lbl4:
+    ret i32 8
+  }
+
+...
+---
+name:            test_jumptable
+# CHECK:      jumpTable:
+# CHECK-NEXT: kind: label-difference32
+# CHECK-NEXT: entries:
+# CHECK-NEXT: - id: 0
+# CHECK-NEXT: blocks: [ '%bb.3.lbl1', '%bb.4.lbl2', '%bb.5.lbl3', '%bb.6.lbl4' ]
+# CHECK_NEXT: body:
+jumpTable:
+  kind:          label-difference32
+  entries:
+    - id:        0
+      blocks:    [ '%bb.3.lbl1', '%bb.4.lbl2', '%bb.5.lbl3', '%bb.6.lbl4' ]
+body:
+  - id:          0
+    name:        entry
+    successors:  [ '%bb.2.def', '%bb.1.entry' ]
+    instructions:
+      - '%eax = MOV32rr %edi, implicit-def %rax'
+      - 'CMP32ri8 %edi, 3, implicit-def %eflags'
+      - 'JA_1 %bb.2.def, implicit %eflags'
+  - id:          1
+    name:        entry
+    successors:  [ '%bb.3.lbl1', '%bb.4.lbl2', '%bb.5.lbl3', '%bb.6.lbl4' ]
+    instructions:
+    # CHECK: %rcx = LEA64r %rip, 1, _, %jump-table.0, _
+      - '%rcx = LEA64r %rip, 1, _, %jump-table.0, _'
+      - '%rax = MOVSX64rm32 %rcx, 4, %rax, 0, _'
+      - '%rax = ADD64rr %rax, %rcx, implicit-def %eflags'
+      - 'JMP64r %rax'
+  - id:          2
+    name:        def
+    instructions:
+      - '%eax = MOV32r0 implicit-def %eflags'
+      - 'RETQ %eax'
+  - id:          3
+    name:        lbl1
+    instructions:
+      - '%eax = MOV32ri 1'
+      - 'RETQ %eax'
+  - id:          4
+    name:        lbl2
+    instructions:
+      - '%eax = MOV32ri 2'
+      - 'RETQ %eax'
+  - id:          5
+    name:        lbl3
+    instructions:
+      - '%eax = MOV32ri 4'
+      - 'RETQ %eax'
+  - id:          6
+    name:        lbl4
+    instructions:
+      - '%eax = MOV32ri 8'
+      - 'RETQ %eax'
+...
+---
+name:            test_jumptable2
+jumpTable:
+  kind:          label-difference32
+  entries:
+    - id:        1
+      blocks:    [ '%bb.3.lbl1', '%bb.4.lbl2', '%bb.5.lbl3', '%bb.6.lbl4' ]
+body:
+  - id:          0
+    name:        entry
+    successors:  [ '%bb.2.def', '%bb.1.entry' ]
+    instructions:
+      - '%eax = MOV32rr %edi, implicit-def %rax'
+      - 'CMP32ri8 %edi, 3, implicit-def %eflags'
+      - 'JA_1 %bb.2.def, implicit %eflags'
+  - id:          1
+    name:        entry
+    successors:  [ '%bb.3.lbl1', '%bb.4.lbl2', '%bb.5.lbl3', '%bb.6.lbl4' ]
+    instructions:
+    # Verify that the printer will use an id of 0 for this jump table:
+    # CHECK: %rcx = LEA64r %rip, 1, _, %jump-table.0, _
+      - '%rcx = LEA64r %rip, 1, _, %jump-table.1, _'
+      - '%rax = MOVSX64rm32 %rcx, 4, %rax, 0, _'
+      - '%rax = ADD64rr %rax, %rcx, implicit-def %eflags'
+      - 'JMP64r %rax'
+  - id:         2
+    name:       def
+    instructions:
+      - '%eax = MOV32r0 implicit-def %eflags'
+      - 'RETQ %eax'
+  - id:         3
+    name:       lbl1
+    instructions:
+      - '%eax = MOV32ri 1'
+      - 'RETQ %eax'
+  - id:         4
+    name:       lbl2
+    instructions:
+      - '%eax = MOV32ri 2'
+      - 'RETQ %eax'
+  - id:         5
+    name:       lbl3
+    instructions:
+      - '%eax = MOV32ri 4'
+      - 'RETQ %eax'
+  - id:         6
+    name:       lbl4
+    instructions:
+      - '%eax = MOV32ri 8'
+      - 'RETQ %eax'
+...
diff --git a/test/CodeGen/MIR/X86/killed-register-flag.mir b/test/CodeGen/MIR/X86/killed-register-flag.mir
new file mode 100644
index 00000000000..fd5f881b0fd
--- /dev/null
+++ b/test/CodeGen/MIR/X86/killed-register-flag.mir
@@ -0,0 +1,43 @@
+# RUN: llc -march=x86-64 -start-after branch-folder -stop-after branch-folder -o /dev/null %s | FileCheck %s
+# This test ensures that the MIR parser parses the 'killed' register flags
+# correctly.
+
+--- |
+
+  define i32 @foo(i32 %a) {
+  entry:
+    %0 = icmp sle i32 %a, 10
+    br i1 %0, label %less, label %exit
+
+  less:
+    ret i32 0
+
+  exit:
+    ret i32 %a
+  }
+
+...
+---
+name:            foo
+body:
+  - id:          0
+    name:        entry
+    successors:  [ '%bb.1.less', '%bb.2.exit' ]
+    instructions:
+      - 'CMP32ri8 %edi, 10, implicit-def %eflags'
+      - 'JG_1 %bb.2.exit, implicit %eflags'
+  - id:          1
+    name:        less
+    instructions:
+      # CHECK:      - '%eax = MOV32r0
+      # CHECK-NEXT: - 'RETQ killed %eax
+      - '%eax = MOV32r0 implicit-def %eflags'
+      - 'RETQ killed %eax'
+  - id:          2
+    name:        exit
+    instructions:
+      # CHECK:      - '%eax = COPY killed %edi
+      # CHECK-NEXT: - 'RETQ killed %eax
+      - '%eax = COPY killed %edi'
+      - 'RETQ killed %eax'
+...
diff --git a/test/CodeGen/MIR/X86/large-cfi-offset-number-error.mir b/test/CodeGen/MIR/X86/large-cfi-offset-number-error.mir
new file mode 100644
index 00000000000..5f395b1ef01
--- /dev/null
+++ b/test/CodeGen/MIR/X86/large-cfi-offset-number-error.mir
@@ -0,0 +1,29 @@
+# RUN: not llc -march=x86-64 -start-after branch-folder -stop-after branch-folder -o /dev/null %s 2>&1 | FileCheck %s
+
+--- |
+
+  define void @test() {
+  entry:
+    %tmp = alloca [4168 x i8], align 4
+    ret void
+  }
+
+...
+---
+name:            test
+tracksRegLiveness: true
+frameInfo:
+  stackSize:       4040
+stack:
+  - { id: 0, name: tmp, offset: -4176, size: 4168, alignment: 4 }
+body:
+  - id:              0
+    name:            entry
+    instructions:
+      - '%rsp = SUB64ri32 %rsp, 4040, implicit-def dead %eflags'
+     # CHECK: [[@LINE+1]]:46: expected a 32 bit integer (the cfi offset is too large)
+      - 'CFI_INSTRUCTION .cfi_def_cfa_offset 123456789123456'
+      - '%rsp = ADD64ri32 %rsp, 4040, implicit-def dead %eflags'
+      - 'RETQ'
+...
+
diff --git a/test/CodeGen/MIR/X86/large-index-number-error.mir b/test/CodeGen/MIR/X86/large-index-number-error.mir
new file mode 100644
index 00000000000..fdb25c907f5
--- /dev/null
+++ b/test/CodeGen/MIR/X86/large-index-number-error.mir
@@ -0,0 +1,35 @@
+# RUN: not llc -march=x86-64 -start-after branch-folder -stop-after branch-folder -o /dev/null %s 2>&1 | FileCheck %s
+
+--- |
+
+  define i32 @foo(i32* %p) {
+  entry:
+    %a = load i32, i32* %p
+    %b = icmp sle i32 %a, 10
+    br i1 %b, label %0, label %1
+
+  ; <label>:0
+    ret i32 0
+
+  ; <label>:1
+    ret i32 %a
+  }
+
+...
+---
+name:            foo
+body:
+ - id: 0
+   name: entry
+   instructions:
+     - '%eax = MOV32rm %rdi, 1, _, 0, _'
+     - 'CMP32ri8 %eax, 10, implicit-def %eflags'
+     # CHECK: [[@LINE+1]]:14: expected 32-bit integer (too large)
+     - 'JG_1 %bb.123456789123456, implicit %eflags'
+ - id: 1
+   instructions:
+     - '%eax = MOV32r0 implicit-def %eflags'
+ - id: 2
+   instructions:
+     - 'RETQ %eax'
+...
diff --git a/test/CodeGen/MIR/X86/lit.local.cfg b/test/CodeGen/MIR/X86/lit.local.cfg
new file mode 100644
index 00000000000..c8625f4d9d2
--- /dev/null
+++ b/test/CodeGen/MIR/X86/lit.local.cfg
@@ -0,0 +1,2 @@
+if not 'X86' in config.root.targets:
+    config.unsupported = True
diff --git a/test/CodeGen/MIR/X86/machine-basic-block-operands.mir b/test/CodeGen/MIR/X86/machine-basic-block-operands.mir
new file mode 100644
index 00000000000..25cb7a1211a
--- /dev/null
+++ b/test/CodeGen/MIR/X86/machine-basic-block-operands.mir
@@ -0,0 +1,77 @@
+# RUN: llc -march=x86-64 -start-after branch-folder -stop-after branch-folder -o /dev/null %s | FileCheck %s
+# This test ensures that the MIR parser parses machine basic block operands.
+
+--- |
+
+  define i32 @foo(i32* %p) {
+  entry:
+    %a = load i32, i32* %p
+    %0 = icmp sle i32 %a, 10
+    br i1 %0, label %less, label %exit
+
+  less:
+    ret i32 0
+
+  exit:
+    ret i32 %a
+  }
+
+  define i32 @bar(i32* %p) {
+  entry:
+    %a = load i32, i32* %p
+    %b = icmp sle i32 %a, 10
+    br i1 %b, label %0, label %1
+
+  ; <label>:0
+    ret i32 0
+
+  ; <label>:1
+    ret i32 %a
+  }
+
+...
+---
+# CHECK: name: foo
+name:            foo
+body:
+ # CHECK: name: entry
+ - id:              0
+   name:            entry
+   successors:      [ '%bb.1.less', '%bb.2.exit' ]
+   instructions:
+     - '%eax = MOV32rm %rdi, 1, _, 0, _'
+     # CHECK:      - 'CMP32ri8 %eax, 10
+     # CHECK-NEXT: - 'JG_1 %bb.2.exit
+     - 'CMP32ri8 %eax, 10, implicit-def %eflags'
+     - 'JG_1 %bb.2.exit, implicit %eflags'
+ # CHECK: name: less
+ - id:              1
+   name:            less
+   instructions:
+     - '%eax = MOV32r0 implicit-def %eflags'
+ - id:              2
+   name:            exit
+   instructions:
+     - 'RETQ %eax'
+...
+---
+# CHECK: name: bar
+name:            bar
+body:
+ # CHECK: name: entry
+ - id: 0
+   name: entry
+   successors:  [ '%bb.1', '%bb.3' ]
+   instructions:
+     - '%eax = MOV32rm %rdi, 1, _, 0, _'
+     # CHECK:      - 'CMP32ri8 %eax, 10
+     # CHECK-NEXT: - 'JG_1 %bb.2
+     - 'CMP32ri8 %eax, 10, implicit-def %eflags'
+     - 'JG_1 %bb.3, implicit %eflags'
+ - id: 1
+   instructions:
+     - '%eax = MOV32r0 implicit-def %eflags'
+ - id: 3
+   instructions:
+     - 'RETQ %eax'
+...
diff --git a/test/CodeGen/MIR/X86/machine-instructions.mir b/test/CodeGen/MIR/X86/machine-instructions.mir
new file mode 100644
index 00000000000..c319db0892b
--- /dev/null
+++ b/test/CodeGen/MIR/X86/machine-instructions.mir
@@ -0,0 +1,25 @@
+# RUN: llc -march=x86-64 -start-after branch-folder -stop-after branch-folder -o /dev/null %s | FileCheck %s
+# This test ensures that the MIR parser parses X86 machine instructions
+# correctly.
+
+--- |
+
+  define i32 @inc(i32 %a) {
+  entry:
+    %b = mul i32 %a, 11
+    ret i32 %b
+  }
+
+...
+---
+# CHECK: name: inc
+name:            inc
+body:
+ - id:           0
+   name:         entry
+   instructions:
+     # CHECK:      MOV32rr
+     # CHECK-NEXT: RETQ
+     - '%eax = MOV32rr %eax'
+     - ' RETQ %eax'
+...
diff --git a/test/CodeGen/MIR/X86/machine-verifier.mir b/test/CodeGen/MIR/X86/machine-verifier.mir
new file mode 100644
index 00000000000..608a90ea4de
--- /dev/null
+++ b/test/CodeGen/MIR/X86/machine-verifier.mir
@@ -0,0 +1,23 @@
+# RUN: not llc -march=x86-64 -start-after branch-folder -stop-after branch-folder -o /dev/null %s 2>&1 | FileCheck %s
+# This test ensures that the MIR parser runs the machine verifier after parsing.
+
+--- |
+
+  define i32 @inc(i32 %a) {
+  entry:
+    ret i32 %a
+  }
+
+...
+---
+name:            inc
+body:
+ - id:           0
+   name:         entry
+   liveins:      [ '%edi' ]
+   instructions:
+   # CHECK: *** Bad machine code: Too few operands ***
+   # CHECK: instruction: COPY2 operands expected, but 0 given.
+     - 'COPY'
+     - 'RETQ'
+...
diff --git a/test/CodeGen/MIR/X86/metadata-operands.mir b/test/CodeGen/MIR/X86/metadata-operands.mir
new file mode 100644
index 00000000000..1f622b1098c
--- /dev/null
+++ b/test/CodeGen/MIR/X86/metadata-operands.mir
@@ -0,0 +1,65 @@
+# RUN: llc -march=x86-64 -start-after machine-sink -stop-after machine-sink -o /dev/null %s | FileCheck %s
+# This test ensures that the MIR parser parses the metadata machine operands
+# correctly.
+
+--- |
+
+  define i32 @test(i32 %x) #0 {
+  entry:
+    %x.addr = alloca i32, align 4
+    store i32 %x, i32* %x.addr, align 4
+    call void @llvm.dbg.declare(metadata i32* %x.addr, metadata !12, metadata !13), !dbg !14
+    %0 = load i32, i32* %x.addr, align 4, !dbg !15
+    ret i32 %0, !dbg !15
+  }
+
+  declare void @llvm.dbg.declare(metadata, metadata, metadata) #1
+
+  attributes #0 = { nounwind "no-frame-pointer-elim"="false" }
+  attributes #1 = { nounwind readnone }
+
+  !llvm.dbg.cu = !{!0}
+  !llvm.module.flags = !{!9, !10}
+  !llvm.ident = !{!11}
+
+  !0 = !DICompileUnit(language: DW_LANG_C99, file: !1, producer: "clang version 3.7.0", isOptimized: false, runtimeVersion: 0, emissionKind: 1, enums: !2, retainedTypes: !2, subprograms: !3, globals: !2, imports: !2)
+  !1 = !DIFile(filename: "test.ll", directory: "")
+  !2 = !{}
+  !3 = !{!4}
+  !4 = !DISubprogram(name: "test", scope: !5, file: !5, line: 4, type: !6, isLocal: false, isDefinition: true, scopeLine: 4, flags: DIFlagPrototyped, isOptimized: false, function: i32 (i32)* @test, variables: !2)
+  !5 = !DIFile(filename: "test.c", directory: "")
+  !6 = !DISubroutineType(types: !7)
+  !7 = !{!8, !8}
+  !8 = !DIBasicType(name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
+  !9 = !{i32 2, !"Dwarf Version", i32 4}
+  !10 = !{i32 2, !"Debug Info Version", i32 3}
+  !11 = !{!"clang version 3.7.0"}
+  !12 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "x", arg: 1, scope: !4, file: !5, line: 4, type: !8)
+  !13 = !DIExpression()
+  !14 = !DILocation(line: 4, scope: !4)
+  !15 = !DILocation(line: 8, scope: !4)
+
+...
+---
+name:            test
+isSSA:           true
+tracksRegLiveness: true
+registers:
+  - { id: 0, class: gr32 }
+frameInfo:
+  maxAlignment:  4
+stack:
+  - { id: 0, name: x.addr, size: 4, alignment: 4 }
+body:
+  - id:          0
+    name:        entry
+    liveins:     [ '%edi' ]
+    instructions:
+      # CHECK:      %0 = COPY %edi
+      # CHECK-NEXT: DBG_VALUE _, 0, !12, !13
+      - '%0 = COPY %edi'
+      - 'DBG_VALUE _, 0, !12, ! 13'
+      - 'MOV32mr %stack.0.x.addr, 1, _, 0, _, %0'
+      - '%eax = COPY %0'
+      - 'RETQ %eax'
+...
diff --git a/test/CodeGen/MIR/X86/missing-closing-quote.mir b/test/CodeGen/MIR/X86/missing-closing-quote.mir
new file mode 100644
index 00000000000..f188d3ad8cf
--- /dev/null
+++ b/test/CodeGen/MIR/X86/missing-closing-quote.mir
@@ -0,0 +1,24 @@
+# RUN: not llc -march=x86-64 -start-after branch-folder -stop-after branch-folder -o /dev/null %s 2>&1 | FileCheck %s
+
+--- |
+
+  @"quoted name" = external global i32
+
+  define i32 @test() {
+  entry:
+    %a = load i32, i32* @"quoted name"
+    ret i32 %a
+  }
+
+...
+---
+name:            test
+body:
+  - id:          0
+    name:        entry
+    instructions:
+      # CHECK: [[@LINE+1]]:53: end of machine instruction reached before the closing '"'
+      - '%rax = MOV64rm %rip, 1, _, @"quoted name, _'
+      - '%eax = MOV32rm killed %rax, 1, _, 0, _'
+      - 'RETQ %eax'
+...
diff --git a/test/CodeGen/MIR/X86/missing-comma.mir b/test/CodeGen/MIR/X86/missing-comma.mir
new file mode 100644
index 00000000000..54c67ac6c91
--- /dev/null
+++ b/test/CodeGen/MIR/X86/missing-comma.mir
@@ -0,0 +1,21 @@
+# RUN: not llc -march=x86-64 -start-after branch-folder -stop-after branch-folder -o /dev/null %s 2>&1 | FileCheck %s
+
+--- |
+
+  define i32 @foo() {
+  entry:
+    ret i32 0
+  }
+
+...
+---
+name:            foo
+body:
+ - id:           0
+   name:         entry
+   instructions:
+     # CHECK: [[@LINE+1]]:29: expected ',' before the next machine operand
+     - '%eax = XOR32rr %eax %eflags'
+     - 'RETQ %eax'
+...
+
diff --git a/test/CodeGen/MIR/X86/missing-implicit-operand.mir b/test/CodeGen/MIR/X86/missing-implicit-operand.mir
new file mode 100644
index 00000000000..b610f89d82b
--- /dev/null
+++ b/test/CodeGen/MIR/X86/missing-implicit-operand.mir
@@ -0,0 +1,41 @@
+# RUN: not llc -march=x86-64 -start-after branch-folder -stop-after branch-folder -o /dev/null %s 2>&1 | FileCheck %s
+# This test ensures that the MIR parser reports an error when an instruction
+# is missing one of its implicit register operands.
+
+--- |
+
+  define i32 @foo(i32* %p) {
+  entry:
+    %a = load i32, i32* %p
+    %0 = icmp sle i32 %a, 10
+    br i1 %0, label %less, label %exit
+
+  less:
+    ret i32 0
+
+  exit:
+    ret i32 %a
+  }
+
+
+...
+---
+name:            foo
+body:
+ - id:              0
+   name:            entry
+   successors:      [ '%bb.1.less', '%bb.2.exit' ]
+   instructions:
+     - '%eax = MOV32rm %rdi, 1, _, 0, _'
+     - 'CMP32ri8 %eax, 10, implicit-def %eflags'
+# CHECK: [[@LINE+1]]:24: missing implicit register operand 'implicit %eflags'
+     - 'JG_1 %bb.2.exit'
+ - id:              1
+   name:            less
+   instructions:
+     - '%eax = MOV32r0 implicit-def %eflags'
+ - id:              2
+   name:            exit
+   instructions:
+     - 'RETQ %eax'
+...
diff --git a/test/CodeGen/MIR/X86/missing-instruction.mir b/test/CodeGen/MIR/X86/missing-instruction.mir
new file mode 100644
index 00000000000..8d11ab5eaab
--- /dev/null
+++ b/test/CodeGen/MIR/X86/missing-instruction.mir
@@ -0,0 +1,19 @@
+# RUN: not llc -march=x86-64 -start-after branch-folder -stop-after branch-folder -o /dev/null %s 2>&1 | FileCheck %s
+
+--- |
+
+  define void @foo() {
+  entry:
+    ret void
+  }
+
+...
+---
+name:            foo
+body:
+ - id:           0
+   name:         entry
+   instructions:
+     # CHECK: [[@LINE+1]]:9: expected a machine instruction
+     - ''
+...
diff --git a/test/CodeGen/MIR/X86/named-registers.mir b/test/CodeGen/MIR/X86/named-registers.mir
new file mode 100644
index 00000000000..91ed4856867
--- /dev/null
+++ b/test/CodeGen/MIR/X86/named-registers.mir
@@ -0,0 +1,23 @@
+# RUN: llc -march=x86-64 -start-after branch-folder -stop-after branch-folder -o /dev/null %s | FileCheck %s
+# This test ensures that the MIR parser parses X86 registers correctly.
+
+--- |
+
+  define i32 @foo() {
+  entry:
+    ret i32 0
+  }
+
+...
+---
+# CHECK: name: foo
+name:            foo
+body:
+ - id:           0
+   name:         entry
+   instructions:
+     # CHECK:      - '%eax = MOV32r0
+     # CHECK-NEXT: - 'RETQ %eax
+     - '%eax = MOV32r0 implicit-def %eflags'
+     - 'RETQ %eax'
+...
diff --git a/test/CodeGen/MIR/X86/null-register-operands.mir b/test/CodeGen/MIR/X86/null-register-operands.mir
new file mode 100644
index 00000000000..55c0ceb3a60
--- /dev/null
+++ b/test/CodeGen/MIR/X86/null-register-operands.mir
@@ -0,0 +1,24 @@
+# RUN: llc -march=x86-64 -start-after branch-folder -stop-after branch-folder -o /dev/null %s | FileCheck %s
+# This test ensures that the MIR parser parses null register operands correctly.
+
+--- |
+
+  define i32 @deref(i32* %p) {
+  entry:
+    %a = load i32, i32* %p
+    ret i32 %a
+  }
+
+...
+---
+# CHECK: name: deref
+name:            deref
+body:
+ - id:           0
+   name:         entry
+   instructions:
+     # CHECK:      - '%eax = MOV32rm %rdi, 1, _, 0, _'
+     # CHECK-NEXT: - 'RETQ %eax'
+     - '%eax = MOV32rm %rdi, 1, _, 0, %noreg'
+     - 'RETQ %eax'
+...
diff --git a/test/CodeGen/MIR/X86/register-mask-operands.mir b/test/CodeGen/MIR/X86/register-mask-operands.mir
new file mode 100644
index 00000000000..f4136598ff5
--- /dev/null
+++ b/test/CodeGen/MIR/X86/register-mask-operands.mir
@@ -0,0 +1,43 @@
+# RUN: llc -march=x86-64 -start-after branch-folder -stop-after branch-folder -o /dev/null %s | FileCheck %s
+# This test ensures that the MIR parser parses register mask operands correctly.
+
+--- |
+
+  define i32 @compute(i32 %a) #0 {
+  body:
+    %c = mul i32 %a, 11
+    ret i32 %c
+  }
+
+  define i32 @foo(i32 %a) #0 {
+  entry:
+    %b = call i32 @compute(i32 %a)
+    ret i32 %b
+  }
+
+  attributes #0 = { "no-frame-pointer-elim"="false" }
+
+...
+---
+name:            compute
+body:
+  - id:          0
+    name:        body
+    instructions:
+      - '%eax = IMUL32rri8 %edi, 11, implicit-def %eflags'
+      - 'RETQ %eax'
+...
+---
+# CHECK: name: foo
+name:            foo
+body:
+  - id:          0
+    name:        entry
+    instructions:
+      # CHECK:      - 'PUSH64r %rax
+      # CHECK-NEXT: - 'CALL64pcrel32 @compute, csr_64, implicit %rsp, implicit %edi, implicit-def %rsp, implicit-def %eax'
+      - 'PUSH64r %rax, implicit-def %rsp, implicit %rsp'
+      - 'CALL64pcrel32 @compute, csr_64, implicit %rsp, implicit %edi, implicit-def %rsp, implicit-def %eax'
+      - '%rdx = POP64r implicit-def %rsp, implicit %rsp'
+      - 'RETQ %eax'
+...
diff --git a/test/CodeGen/MIR/X86/simple-register-allocation-hints.mir b/test/CodeGen/MIR/X86/simple-register-allocation-hints.mir
new file mode 100644
index 00000000000..42ed70ee372
--- /dev/null
+++ b/test/CodeGen/MIR/X86/simple-register-allocation-hints.mir
@@ -0,0 +1,35 @@
+# RUN: llc -march=x86-64 -start-after machine-scheduler -stop-after machine-scheduler -o /dev/null %s | FileCheck %s
+# This test ensures that the MIR parser parses simple register allocation hints
+# correctly.
+
+--- |
+
+  define i32 @test(i32 %a, i32 %b) {
+  body:
+    %c = mul i32 %a, %b
+    ret i32 %c
+  }
+
+...
+---
+name:            test
+tracksRegLiveness: true
+# CHECK: registers:
+# CHECK-NEXT:  - { id: 0, class: gr32 }
+# CHECK-NEXT:  - { id: 1, class: gr32, preferred-register: '%esi' }
+# CHECK-NEXT:  - { id: 2, class: gr32, preferred-register: '%edi' }
+registers:
+  - { id: 0, class: gr32 }
+  - { id: 1, class: gr32, preferred-register: '%esi' }
+  - { id: 2, class: gr32, preferred-register: '%edi' }
+body:
+  - id:              0
+    name:            body
+    liveins:         [ '%edi', '%esi' ]
+    instructions:
+      - '%1 = COPY %esi'
+      - '%2 = COPY %edi'
+      - '%2 = IMUL32rr %2, %1, implicit-def dead %eflags'
+      - '%eax = COPY %2'
+      - 'RETQ killed %eax'
+...
diff --git a/test/CodeGen/MIR/X86/spill-slot-fixed-stack-object-aliased.mir b/test/CodeGen/MIR/X86/spill-slot-fixed-stack-object-aliased.mir
new file mode 100644
index 00000000000..67f4bd21cd0
--- /dev/null
+++ b/test/CodeGen/MIR/X86/spill-slot-fixed-stack-object-aliased.mir
@@ -0,0 +1,32 @@
+# RUN: not llc -march=x86-64 -start-after branch-folder -stop-after branch-folder -o /dev/null %s 2>&1 | FileCheck %s
+
+--- |
+
+  define i32 @test(i32 %a) #0 {
+  entry:
+    %b = alloca i32
+    store i32 %a, i32* %b
+    %c = load i32, i32* %b
+    ret i32 %c
+  }
+
+  attributes #0 = { "no-frame-pointer-elim"="false" }
+
+...
+---
+name:            test
+frameInfo:
+  maxAlignment:    4
+fixedStack:
+  # CHECK: [[@LINE+1]]:63: unknown key 'isAliased'
+  - { id: 0, type: spill-slot, offset: 0, size: 4, isAliased: true }
+stack:
+  - { id: 0, offset: -12, size: 4, alignment: 4 }
+body:
+  - id:          0
+    name:        entry
+    instructions:
+      - 'MOV32mr %rsp, 1, _, -4, _, %edi'
+      - '%eax = COPY %edi'
+      - 'RETQ %eax'
+...
diff --git a/test/CodeGen/MIR/X86/spill-slot-fixed-stack-object-immutable.mir b/test/CodeGen/MIR/X86/spill-slot-fixed-stack-object-immutable.mir
new file mode 100644
index 00000000000..1e1b0fdcc8d
--- /dev/null
+++ b/test/CodeGen/MIR/X86/spill-slot-fixed-stack-object-immutable.mir
@@ -0,0 +1,32 @@
+# RUN: not llc -march=x86-64 -start-after branch-folder -stop-after branch-folder -o /dev/null %s 2>&1 | FileCheck %s
+
+--- |
+
+  define i32 @test(i32 %a) #0 {
+  entry:
+    %b = alloca i32
+    store i32 %a, i32* %b
+    %c = load i32, i32* %b
+    ret i32 %c
+  }
+
+  attributes #0 = { "no-frame-pointer-elim"="false" }
+
+...
+---
+name:            test
+frameInfo:
+  maxAlignment:    4
+fixedStack:
+  # CHECK: [[@LINE+1]]:65: unknown key 'isImmutable'
+  - { id: 0, type: spill-slot, offset: 0, size: 4, isImmutable: true }
+stack:
+  - { id: 0, offset: -12, size: 4, alignment: 4 }
+body:
+  - id:          0
+    name:        entry
+    instructions:
+      - 'MOV32mr %rsp, 1, _, -4, _, %edi'
+      - '%eax = COPY %edi'
+      - 'RETQ %eax'
+...
diff --git a/test/CodeGen/MIR/X86/spill-slot-fixed-stack-objects.mir b/test/CodeGen/MIR/X86/spill-slot-fixed-stack-objects.mir
new file mode 100644
index 00000000000..f771f796ec3
--- /dev/null
+++ b/test/CodeGen/MIR/X86/spill-slot-fixed-stack-objects.mir
@@ -0,0 +1,34 @@
+# RUN: llc -march=x86-64 -start-after branch-folder -stop-after branch-folder -o /dev/null %s | FileCheck %s
+# This test ensures that the MIR parser parses fixed stack objects correctly.
+
+--- |
+
+  define i32 @test(i32 %a) #0 {
+  entry:
+    %b = alloca i32
+    store i32 %a, i32* %b
+    %c = load i32, i32* %b
+    ret i32 %c
+  }
+
+  attributes #0 = { "no-frame-pointer-elim"="false" }
+
+...
+---
+name:            test
+frameInfo:
+  maxAlignment:    4
+# CHECK: fixedStack:
+# CHECK-NEXT: - { id: 0, type: spill-slot, offset: 0, size: 4, alignment: 4 }
+fixedStack:
+  - { id: 0, type: spill-slot, offset: 0, size: 4, alignment: 4 }
+stack:
+  - { id: 0, offset: -12, size: 4, alignment: 4 }
+body:
+  - id:          0
+    name:        entry
+    instructions:
+      - 'MOV32mr %rsp, 1, _, -4, _, %edi'
+      - '%eax = COPY %edi'
+      - 'RETQ %eax'
+...
diff --git a/test/CodeGen/MIR/X86/stack-object-invalid-name.mir b/test/CodeGen/MIR/X86/stack-object-invalid-name.mir
new file mode 100644
index 00000000000..2d4f57bad3d
--- /dev/null
+++ b/test/CodeGen/MIR/X86/stack-object-invalid-name.mir
@@ -0,0 +1,30 @@
+# RUN: not llc -march=x86-64 -start-after branch-folder -stop-after branch-folder -o /dev/null %s 2>&1 | FileCheck %s
+# This test ensures that the MIR parser reports an error when it encounters a
+# stack object with a name that can't be associated with an alloca instruction.
+
+--- |
+
+  define i32 @test(i32 %a) {
+  entry:
+    %b = alloca i32
+    store i32 %a, i32* %b
+    %c = load i32, i32* %b
+    ret i32 %c
+  }
+
+...
+---
+name:            test
+frameInfo:
+  maxAlignment:  4
+stack:
+  # CHECK: [[@LINE+1]]:20: alloca instruction named 'x' isn't defined in the function 'test'
+  - { id: 0, name: x, offset: -12, size: 4, alignment: 4 }
+body:
+  - id:          0
+    name:        entry
+    instructions:
+      - 'MOV32mr %rsp, 1, _, -4, _, %edi'
+      - '%eax = MOV32rm %rsp, 1, _, -4, _'
+      - 'RETQ %eax'
+...
diff --git a/test/CodeGen/MIR/X86/stack-object-operand-name-mismatch-error.mir b/test/CodeGen/MIR/X86/stack-object-operand-name-mismatch-error.mir
new file mode 100644
index 00000000000..a6ffb3b93e0
--- /dev/null
+++ b/test/CodeGen/MIR/X86/stack-object-operand-name-mismatch-error.mir
@@ -0,0 +1,35 @@
+# RUN: not llc -march=x86-64 -start-after machine-sink -stop-after machine-sink -o /dev/null %s 2>&1 | FileCheck %s
+# This test ensures that an error is reported when an stack object reference
+# uses a different name then the stack object definition.
+
+--- |
+
+  define i32 @test(i32 %a) {
+  entry:
+    %b = alloca i32
+    store i32 %a, i32* %b
+    %c = load i32, i32* %b
+    ret i32 %c
+  }
+
+...
+---
+name:            test
+isSSA:           true
+tracksRegLiveness: true
+registers:
+  - { id: 0, class: gr32 }
+frameInfo:
+  maxAlignment:  4
+stack:
+  - { id: 0, name: b, size: 4, alignment: 4 }
+body:
+  - id:          0
+    name:        entry
+    instructions:
+      - '%0 = COPY %edi'
+      # CHECK: [[@LINE+1]]:18: the name of the stack object '%stack.0' isn't 'x'
+      - 'MOV32mr %stack.0.x, 1, _, 0, _, %0'
+      - '%eax = COPY %0'
+      - 'RETQ %eax'
+...
diff --git a/test/CodeGen/MIR/X86/stack-object-operands.mir b/test/CodeGen/MIR/X86/stack-object-operands.mir
new file mode 100644
index 00000000000..b1c078eb6a3
--- /dev/null
+++ b/test/CodeGen/MIR/X86/stack-object-operands.mir
@@ -0,0 +1,48 @@
+# RUN: llc -march=x86 -start-after machine-sink -stop-after machine-sink -o /dev/null %s | FileCheck %s
+# This test ensures that the MIR parser parses stack object machine operands
+# correctly.
+
+--- |
+
+  define i32 @test(i32 %a) {
+  entry:
+    %b = alloca i32
+    %0 = alloca i32
+    store i32 %a, i32* %b
+    store i32 2, i32* %0
+    %c = load i32, i32* %b
+    ret i32 %c
+  }
+
+...
+---
+name:            test
+isSSA:           true
+tracksRegLiveness: true
+registers:
+  - { id: 0, class: gr32 }
+  - { id: 1, class: gr32 }
+frameInfo:
+  maxAlignment:    4
+fixedStack:
+  - { id: 0, offset: 0, size: 4, isImmutable: true, isAliased: false }
+stack:
+  - { id: 0, name: b, size: 4, alignment: 4 }
+  - { id: 1, size: 4, alignment: 4 }
+body:
+  # CHECK: name: entry
+  # CHECK: instructions:
+  # CHECK-NEXT: - '%0 = MOV32rm %fixed-stack.0, 1, _, 0, _'
+  # CHECK-NEXT: - 'MOV32mr %stack.0.b, 1, _, 0, _, %0'
+  # CHECK-NEXT: - 'MOV32mi %stack.1, 1, _, 0, _, 2'
+  # CHECK-NEXT: - '%1 = MOV32rm %stack.0.b, 1, _, 0, _'
+  - id:          0
+    name:        entry
+    instructions:
+      - '%0 = MOV32rm %fixed-stack.0, 1, _, 0, _'
+      - 'MOV32mr %stack.0.b, 1, _, 0, _, %0'
+      - 'MOV32mi %stack.1, 1, _, 0, _, 2'
+      - '%1 = MOV32rm %stack.0, 1, _, 0, _'
+      - '%eax = COPY %1'
+      - 'RETL %eax'
+...
diff --git a/test/CodeGen/MIR/X86/stack-objects.mir b/test/CodeGen/MIR/X86/stack-objects.mir
new file mode 100644
index 00000000000..37666300a91
--- /dev/null
+++ b/test/CodeGen/MIR/X86/stack-objects.mir
@@ -0,0 +1,39 @@
+# RUN: llc -march=x86-64 -start-after branch-folder -stop-after branch-folder -o /dev/null %s | FileCheck %s
+# This test ensures that the MIR parser parses stack objects correctly.
+
+--- |
+
+  define i32 @test(i32 %a) #0 {
+  entry:
+    %b = alloca i32
+    %x = alloca i64
+    store i32 %a, i32* %b
+    store i64 2, i64* %x
+    %c = load i32, i32* %b
+    ret i32 %c
+  }
+
+  attributes #0 = { "no-frame-pointer-elim"="false" }
+
+...
+---
+name:            test
+frameInfo:
+  maxAlignment:    8
+# CHECK: stack:
+# CHECK-NEXT: - { id: 0, name: b, offset: -12, size: 4, alignment: 4 }
+# CHECK-NEXT: - { id: 1, name: x, offset: -24, size: 8, alignment: 8 }
+# CHECK-NEXT: - { id: 2, type: spill-slot, offset: -32, size: 4, alignment: 4 }
+stack:
+  - { id: 0, name: b, offset: -12, size: 4, alignment: 4 }
+  - { id: 1, name: x, offset: -24, size: 8, alignment: 8 }
+  - { id: 2, type: spill-slot, offset: -32, size: 4, alignment: 4 }
+body:
+  - id:          0
+    name:        entry
+    instructions:
+      - 'MOV32mr %rsp, 1, _, -4, _, %edi'
+      - 'MOV64mi32 %rsp, 1, _, -16, _, 2'
+      - '%eax = MOV32rm %rsp, 1, _, -4, _'
+      - 'RETQ %eax'
+...
diff --git a/test/CodeGen/MIR/X86/subregister-operands.mir b/test/CodeGen/MIR/X86/subregister-operands.mir
new file mode 100644
index 00000000000..bbdf3fb4a67
--- /dev/null
+++ b/test/CodeGen/MIR/X86/subregister-operands.mir
@@ -0,0 +1,34 @@
+# RUN: llc -march=x86-64 -start-after machine-sink -stop-after machine-sink -o /dev/null %s | FileCheck %s
+# This test ensures that the MIR parser parses subregisters in register operands
+# correctly.
+
+--- |
+
+  define zeroext i1 @t(i1 %c) {
+  entry:
+    ret i1 %c
+  }
+
+...
+---
+name:            t
+isSSA:           true
+tracksRegLiveness: true
+registers:
+  - { id: 0, class: gr32 }
+  - { id: 1, class: gr8 }
+  - { id: 2, class: gr8 }
+body:
+  - name:        entry
+    id:          0
+    liveins:     [ '%edi' ]
+    instructions:
+      # CHECK:      %0 = COPY %edi
+      # CHECK-NEXT: %1 = COPY %0:sub_8bit
+      - '%0 = COPY %edi'
+      - '%1 = COPY %0:sub_8bit'
+      - '%2 = AND8ri %1, 1, implicit-def %eflags'
+      - '%al = COPY %2'
+      - 'RETQ %al'
+...
+
diff --git a/test/CodeGen/MIR/X86/successor-basic-blocks-few-weights.mir b/test/CodeGen/MIR/X86/successor-basic-blocks-few-weights.mir
new file mode 100644
index 00000000000..93d2eead0bb
--- /dev/null
+++ b/test/CodeGen/MIR/X86/successor-basic-blocks-few-weights.mir
@@ -0,0 +1,41 @@
+# RUN: not llc -march=x86-64 -start-after branch-folder -stop-after branch-folder -o /dev/null %s 2>&1 | FileCheck %s
+
+--- |
+
+  define i32 @foo(i32 %a) {
+  entry:
+    %0 = icmp sle i32 %a, 10
+    br i1 %0, label %less, label %exit
+
+  less:
+    ret i32 0
+
+  exit:
+    ret i32 %a
+  }
+
+...
+---
+name:            foo
+body:
+  - id:          0
+    name:        entry
+    successors:  [ '%bb.1.less', '%bb.2.exit' ]
+# CHECK: [[@LINE+1]]:23: too few successor weights, expected 2, have 1
+    weights:     [ 16 ]
+    liveins:     [ '%edi' ]
+    instructions:
+      - 'CMP32ri8 %edi, 10, implicit-def %eflags'
+      - 'JG_1 %bb.2.exit, implicit killed %eflags'
+  - id:          1
+    name:        less
+    instructions:
+      - '%eax = MOV32r0 implicit-def dead %eflags'
+      - 'RETQ killed %eax'
+  - id:          2
+    name:        exit
+    liveins:     [ '%edi' ]
+    instructions:
+      - '%eax = COPY killed %edi'
+      - 'RETQ killed %eax'
+...
diff --git a/test/CodeGen/MIR/X86/successor-basic-blocks-many-weights.mir b/test/CodeGen/MIR/X86/successor-basic-blocks-many-weights.mir
new file mode 100644
index 00000000000..2c0e1932aad
--- /dev/null
+++ b/test/CodeGen/MIR/X86/successor-basic-blocks-many-weights.mir
@@ -0,0 +1,41 @@
+# RUN: not llc -march=x86-64 -start-after branch-folder -stop-after branch-folder -o /dev/null %s 2>&1 | FileCheck %s
+
+--- |
+
+  define i32 @foo(i32 %a) {
+  entry:
+    %0 = icmp sle i32 %a, 10
+    br i1 %0, label %less, label %exit
+
+  less:
+    ret i32 0
+
+  exit:
+    ret i32 %a
+  }
+
+...
+---
+name:            foo
+body:
+  - id:          0
+    name:        entry
+    successors:  [ '%bb.1.less', '%bb.2.exit' ]
+# CHECK: [[@LINE+1]]:28: too many successor weights, expected 2, have 3
+    weights:     [ 16, 16, 16 ]
+    liveins:     [ '%edi' ]
+    instructions:
+      - 'CMP32ri8 %edi, 10, implicit-def %eflags'
+      - 'JG_1 %bb.2.exit, implicit killed %eflags'
+  - id:          1
+    name:        less
+    instructions:
+      - '%eax = MOV32r0 implicit-def dead %eflags'
+      - 'RETQ killed %eax'
+  - id:          2
+    name:        exit
+    liveins:     [ '%edi' ]
+    instructions:
+      - '%eax = COPY killed %edi'
+      - 'RETQ killed %eax'
+...
diff --git a/test/CodeGen/MIR/X86/successor-basic-blocks-weights.mir b/test/CodeGen/MIR/X86/successor-basic-blocks-weights.mir
new file mode 100644
index 00000000000..c7cfeaae90d
--- /dev/null
+++ b/test/CodeGen/MIR/X86/successor-basic-blocks-weights.mir
@@ -0,0 +1,46 @@
+# RUN: llc -march=x86-64 -start-after branch-folder -stop-after branch-folder -o /dev/null %s | FileCheck %s
+# This test ensures that the MIR parser parses basic block successors and
+# weights correctly.
+
+--- |
+
+  define i32 @foo(i32 %a) {
+  entry:
+    %0 = icmp sle i32 %a, 10
+    br i1 %0, label %less, label %exit
+
+  less:
+    ret i32 0
+
+  exit:
+    ret i32 %a
+  }
+
+...
+---
+name:            foo
+body:
+  # CHECK:      name: entry
+  # CHECK:      successors: [ '%bb.1.less', '%bb.2.exit' ]
+  # CHECK-NEXT: weights: [ 16, 32 ]
+  # CHECK:      name: less
+  - id:          0
+    name:        entry
+    successors:  [ '%bb.1.less', '%bb.2.exit' ]
+    weights:     [ 16, 32 ]
+    liveins:     [ '%edi' ]
+    instructions:
+      - 'CMP32ri8 %edi, 10, implicit-def %eflags'
+      - 'JG_1 %bb.2.exit, implicit killed %eflags'
+  - id:          1
+    name:        less
+    instructions:
+      - '%eax = MOV32r0 implicit-def dead %eflags'
+      - 'RETQ killed %eax'
+  - id:          2
+    name:        exit
+    liveins:     [ '%edi' ]
+    instructions:
+      - '%eax = COPY killed %edi'
+      - 'RETQ killed %eax'
+...
diff --git a/test/CodeGen/MIR/X86/successor-basic-blocks.mir b/test/CodeGen/MIR/X86/successor-basic-blocks.mir
new file mode 100644
index 00000000000..de4f84eb210
--- /dev/null
+++ b/test/CodeGen/MIR/X86/successor-basic-blocks.mir
@@ -0,0 +1,80 @@
+# RUN: llc -march=x86-64 -start-after branch-folder -stop-after branch-folder -o /dev/null %s | FileCheck %s
+# This test ensures that the MIR parser parses basic block successors correctly.
+
+--- |
+
+  define i32 @foo(i32 %a) {
+  entry:
+    %0 = icmp sle i32 %a, 10
+    br i1 %0, label %less, label %exit
+
+  less:
+    ret i32 0
+
+  exit:
+    ret i32 %a
+  }
+
+  define i32 @bar(i32 %a) {
+  entry:
+    %b = icmp sle i32 %a, 10
+    br i1 %b, label %0, label %1
+
+  ; <label>:0
+    ret i32 0
+
+  ; <label>:1
+    ret i32 %a
+  }
+
+...
+---
+name:            foo
+body:
+  # CHECK: name: entry
+  # CHECK: successors: [ '%bb.1.less', '%bb.2.exit' ]
+  # CHECK: name: less
+  - id:          0
+    name:        entry
+    successors:  [ '%bb.1.less', '%bb.2.exit' ]
+    liveins:     [ '%edi' ]
+    instructions:
+      - 'CMP32ri8 %edi, 10, implicit-def %eflags'
+      - 'JG_1 %bb.2.exit, implicit killed %eflags'
+  - id:          1
+    name:        less
+    instructions:
+      - '%eax = MOV32r0 implicit-def dead %eflags'
+      - 'RETQ killed %eax'
+  - id:          2
+    name:        exit
+    liveins:     [ '%edi' ]
+    instructions:
+      - '%eax = COPY killed %edi'
+      - 'RETQ killed %eax'
+...
+---
+name:            bar
+body:
+  # CHECK: name: bar
+  # CHECK: name: entry
+  # CHECK: successors: [ '%bb.1', '%bb.2' ]
+  # CHECK: id: 1
+  # CHECK: id: 2
+  - id:           0
+    name:         entry
+    successors:   [ '%bb.1', '%bb.2' ]
+    liveins:      [ '%edi' ]
+    instructions:
+      - 'CMP32ri8 %edi, 10, implicit-def %eflags'
+      - 'JG_1 %bb.2, implicit killed %eflags'
+  - id:           1
+    instructions:
+      - '%eax = MOV32r0 implicit-def dead %eflags'
+      - 'RETQ killed %eax'
+  - id:           2
+    liveins:      [ '%edi' ]
+    instructions:
+      - '%eax = COPY killed %edi'
+      - 'RETQ killed %eax'
+...
diff --git a/test/CodeGen/MIR/X86/undef-register-flag.mir b/test/CodeGen/MIR/X86/undef-register-flag.mir
new file mode 100644
index 00000000000..83b9e10a80d
--- /dev/null
+++ b/test/CodeGen/MIR/X86/undef-register-flag.mir
@@ -0,0 +1,42 @@
+# RUN: llc -march=x86-64 -start-after branch-folder -stop-after branch-folder -o /dev/null %s | FileCheck %s
+# This test ensures that the MIR parser parses the 'undef' register flags
+# correctly.
+
+--- |
+
+  define i32 @compute(i32 %a) #0 {
+  body:
+    %c = mul i32 %a, 11
+    ret i32 %c
+  }
+
+  define i32 @foo(i32 %a) #0 {
+  entry:
+    %b = call i32 @compute(i32 %a)
+    ret i32 %b
+  }
+
+  attributes #0 = { "no-frame-pointer-elim"="false" }
+
+...
+---
+name:            compute
+body:
+  - id:          0
+    name:        body
+    instructions:
+      - '%eax = IMUL32rri8 %edi, 11, implicit-def %eflags'
+      - 'RETQ %eax'
+...
+---
+name:            foo
+body:
+  - id:          0
+    name:        entry
+    instructions:
+      # CHECK: - 'PUSH64r undef %rax
+      - 'PUSH64r undef %rax, implicit-def %rsp, implicit %rsp'
+      - 'CALL64pcrel32 @compute, csr_64, implicit %rsp, implicit %edi, implicit-def %rsp, implicit-def %eax'
+      - '%rdx = POP64r implicit-def %rsp, implicit %rsp'
+      - 'RETQ %eax'
+...
diff --git a/test/CodeGen/MIR/X86/undefined-fixed-stack-object.mir b/test/CodeGen/MIR/X86/undefined-fixed-stack-object.mir
new file mode 100644
index 00000000000..4a2fc80d52a
--- /dev/null
+++ b/test/CodeGen/MIR/X86/undefined-fixed-stack-object.mir
@@ -0,0 +1,40 @@
+# RUN: not llc -march=x86-64 -start-after machine-sink -stop-after machine-sink -o /dev/null %s 2>&1 | FileCheck %s
+--- |
+
+  define i32 @test(i32 %a) {
+  entry:
+    %b = alloca i32
+    %0 = alloca i32
+    store i32 %a, i32* %b
+    store i32 2, i32* %0
+    %c = load i32, i32* %b
+    ret i32 %c
+  }
+
+...
+---
+name:            test
+isSSA:           true
+tracksRegLiveness: true
+registers:
+  - { id: 0, class: gr32 }
+  - { id: 1, class: gr32 }
+frameInfo:
+  maxAlignment:    4
+fixedStack:
+  - { id: 0, offset: 0, size: 4, isImmutable: true, isAliased: false }
+stack:
+  - { id: 0, name: b, size: 4, alignment: 4 }
+  - { id: 1, size: 4, alignment: 4 }
+body:
+  - id:          0
+    name:        entry
+    instructions:
+      # CHECK: [[@LINE+1]]:23: use of undefined fixed stack object '%fixed-stack.11'
+      - '%0 = MOV32rm %fixed-stack.11, 1, _, 0, _'
+      - 'MOV32mr %stack.0, 1, _, 0, _, %0'
+      - 'MOV32mi %stack.1, 1, _, 0, _, 2'
+      - '%1 = MOV32rm %stack.0, 1, _, 0, _'
+      - '%eax = COPY %1'
+      - 'RETL %eax'
+...
diff --git a/test/CodeGen/MIR/X86/undefined-global-value.mir b/test/CodeGen/MIR/X86/undefined-global-value.mir
new file mode 100644
index 00000000000..e41dc0454d2
--- /dev/null
+++ b/test/CodeGen/MIR/X86/undefined-global-value.mir
@@ -0,0 +1,28 @@
+# RUN: not llc -march=x86-64 -start-after branch-folder -stop-after branch-folder -o /dev/null %s 2>&1 | FileCheck %s
+# This test ensures that an error is reported when an invalid global value index
+# is used.
+
+--- |
+
+  @0 = external global i32
+
+  define i32 @inc() {
+  entry:
+    %a = load i32, i32* @0
+    %b = add i32 %a, 1
+    ret i32 %b
+  }
+
+...
+---
+name: inc
+body:
+  - id: 0
+    name: entry
+    instructions:
+      # CHECK: [[@LINE+1]]:37: use of undefined global value '@2'
+      - '%rax = MOV64rm %rip, 1, _, @2, _'
+      - '%eax = MOV32rm %rax, 1, _, 0, _'
+      - '%eax = INC32r %eax'
+      - 'RETQ %eax'
+...
diff --git a/test/CodeGen/MIR/X86/undefined-ir-block-in-blockaddress.mir b/test/CodeGen/MIR/X86/undefined-ir-block-in-blockaddress.mir
new file mode 100644
index 00000000000..fc43df37303
--- /dev/null
+++ b/test/CodeGen/MIR/X86/undefined-ir-block-in-blockaddress.mir
@@ -0,0 +1,34 @@
+# RUN: not llc -march=x86-64 -start-after branch-folder -stop-after branch-folder -o /dev/null %s 2>&1 | FileCheck %s
+
+--- |
+
+  @addr = global i8* null
+
+  define void @test() {
+  entry:
+    store volatile i8* blockaddress(@test, %block), i8** @addr
+    %val = load volatile i8*, i8** @addr
+    indirectbr i8* %val, [label %block]
+
+  block:
+    ret void
+  }
+
+...
+---
+name:            test
+body:
+  - id:              0
+    name:            entry
+    successors:      [ '%bb.1.block' ]
+    instructions:
+      # CHECK: [[@LINE+1]]:56: use of undefined IR block '%ir-block."block "'
+      - '%rax = LEA64r %rip, 1, _, blockaddress(@test, %ir-block."block "), _'
+      - 'MOV64mr %rip, 1, _, @addr, _, killed %rax'
+      - 'JMP64m %rip, 1, _, @addr, _'
+  - id:              1
+    name:            block
+    addressTaken:    true
+    instructions:
+      - RETQ
+...
diff --git a/test/CodeGen/MIR/X86/undefined-ir-block-slot-in-blockaddress.mir b/test/CodeGen/MIR/X86/undefined-ir-block-slot-in-blockaddress.mir
new file mode 100644
index 00000000000..189b0f15ccc
--- /dev/null
+++ b/test/CodeGen/MIR/X86/undefined-ir-block-slot-in-blockaddress.mir
@@ -0,0 +1,32 @@
+# RUN: not llc -march=x86-64 -start-after branch-folder -stop-after branch-folder -o /dev/null %s 2>&1 | FileCheck %s
+
+--- |
+
+  @addr = global i8* null
+
+  define void @test() {
+  entry:
+    store volatile i8* blockaddress(@test, %0), i8** @addr
+    %val = load volatile i8*, i8** @addr
+    indirectbr i8* %val, [label %0]
+
+    ret void
+  }
+
+...
+---
+name:            test
+body:
+  - id:              0
+    name:            entry
+    successors:      [ '%bb.1' ]
+    instructions:
+      # CHECK: [[@LINE+1]]:56: use of undefined IR block '%ir-block.1'
+      - '%rax = LEA64r %rip, 1, _, blockaddress(@test, %ir-block.1), _'
+      - 'MOV64mr %rip, 1, _, @addr, _, killed %rax'
+      - 'JMP64m %rip, 1, _, @addr, _'
+  - id:              1
+    addressTaken:    true
+    instructions:
+      - RETQ
+...
diff --git a/test/CodeGen/MIR/X86/undefined-jump-table-id.mir b/test/CodeGen/MIR/X86/undefined-jump-table-id.mir
new file mode 100644
index 00000000000..f4bc41cd8f7
--- /dev/null
+++ b/test/CodeGen/MIR/X86/undefined-jump-table-id.mir
@@ -0,0 +1,80 @@
+# RUN: not llc -march=x86-64 -start-after branch-folder -stop-after branch-folder -o /dev/null %s 2>&1 | FileCheck %s
+
+--- |
+
+  define i32 @test_jumptable(i32 %in) {
+  entry:
+    switch i32 %in, label %def [
+      i32 0, label %lbl1
+      i32 1, label %lbl2
+      i32 2, label %lbl3
+      i32 3, label %lbl4
+    ]
+
+  def:
+    ret i32 0
+
+  lbl1:
+    ret i32 1
+
+  lbl2:
+    ret i32 2
+
+  lbl3:
+    ret i32 4
+
+  lbl4:
+    ret i32 8
+  }
+
+...
+---
+name:            test_jumptable
+jumpTable:
+  kind:          label-difference32
+  entries:
+    - id:        0
+      blocks:    [ '%bb.3.lbl1', '%bb.4.lbl2', '%bb.5.lbl3', '%bb.6.lbl4' ]
+body:
+  - id:              0
+    name:            entry
+    successors:      [ '%bb.2.def', '%bb.1.entry' ]
+    instructions:
+      - '%eax = MOV32rr %edi, implicit-def %rax'
+      - 'CMP32ri8 %edi, 3, implicit-def %eflags'
+      - 'JA_1 %bb.2.def, implicit %eflags'
+  - id:              1
+    name:            entry
+    successors:      [ '%bb.3.lbl1', '%bb.4.lbl2', '%bb.5.lbl3', '%bb.6.lbl4' ]
+    instructions:
+    # CHECK: [[@LINE+1]]:36: use of undefined jump table '%jump-table.2'
+      - '%rcx = LEA64r %rip, 1, _, %jump-table.2, _'
+      - '%rax = MOVSX64rm32 %rcx, 4, %rax, 0, _'
+      - '%rax = ADD64rr %rax, %rcx, implicit-def %eflags'
+      - 'JMP64r %rax'
+  - id:              2
+    name:            def
+    instructions:
+      - '%eax = MOV32r0 implicit-def %eflags'
+      - 'RETQ %eax'
+  - id:              3
+    name:            lbl1
+    instructions:
+      - '%eax = MOV32ri 1'
+      - 'RETQ %eax'
+  - id:              4
+    name:            lbl2
+    instructions:
+      - '%eax = MOV32ri 2'
+      - 'RETQ %eax'
+  - id:              5
+    name:            lbl3
+    instructions:
+      - '%eax = MOV32ri 4'
+      - 'RETQ %eax'
+  - id:              6
+    name:            lbl4
+    instructions:
+      - '%eax = MOV32ri 8'
+      - 'RETQ %eax'
+...
diff --git a/test/CodeGen/MIR/X86/undefined-named-global-value.mir b/test/CodeGen/MIR/X86/undefined-named-global-value.mir
new file mode 100644
index 00000000000..b40c2ce43b5
--- /dev/null
+++ b/test/CodeGen/MIR/X86/undefined-named-global-value.mir
@@ -0,0 +1,28 @@
+# RUN: not llc -march=x86-64 -start-after branch-folder -stop-after branch-folder -o /dev/null %s 2>&1 | FileCheck %s
+# This test ensures that an error is reported when an undefined global value is
+# used.
+
+--- |
+
+  @G = external global i32
+
+  define i32 @inc() {
+  entry:
+    %a = load i32, i32* @G
+    %b = add i32 %a, 1
+    ret i32 %b
+  }
+
+...
+---
+name: inc
+body:
+  - id: 0
+    name: entry
+    instructions:
+      # CHECK: [[@LINE+1]]:37: use of undefined global value '@GG'
+      - '%rax = MOV64rm %rip, 1, _, @GG, _'
+      - '%eax = MOV32rm %rax, 1, _, 0, _'
+      - '%eax = INC32r %eax'
+      - 'RETQ %eax'
+...
diff --git a/test/CodeGen/MIR/X86/undefined-register-class.mir b/test/CodeGen/MIR/X86/undefined-register-class.mir
new file mode 100644
index 00000000000..a14d2303a7d
--- /dev/null
+++ b/test/CodeGen/MIR/X86/undefined-register-class.mir
@@ -0,0 +1,26 @@
+# RUN: not llc -march=x86-64 -start-after machine-sink -stop-after machine-sink -o /dev/null %s 2>&1 | FileCheck %s
+# This test ensures that the MIR parser reports an error when it encounters an
+# unknown register class.
+
+--- |
+
+  define i32 @test(i32 %a) {
+  entry:
+    ret i32 %a
+  }
+
+...
+---
+name:            test
+isSSA:           true
+tracksRegLiveness: true
+registers:
+  # CHECK: [[@LINE+1]]:20: use of undefined register class 'gr3200'
+  - {id: 0, class: 'gr3200'}
+body:
+  - id:          0
+    name:        entry
+    instructions:
+      - 'RETQ %eax'
+...
+
diff --git a/test/CodeGen/MIR/X86/undefined-stack-object.mir b/test/CodeGen/MIR/X86/undefined-stack-object.mir
new file mode 100644
index 00000000000..bec703a2d11
--- /dev/null
+++ b/test/CodeGen/MIR/X86/undefined-stack-object.mir
@@ -0,0 +1,32 @@
+# RUN: not llc -march=x86-64 -start-after machine-sink -stop-after machine-sink -o /dev/null %s 2>&1 | FileCheck %s
+--- |
+
+  define i32 @test(i32 %a) {
+  entry:
+    %b = alloca i32
+    store i32 %a, i32* %b
+    %c = load i32, i32* %b
+    ret i32 %c
+  }
+
+...
+---
+name:            test
+isSSA:           true
+tracksRegLiveness: true
+registers:
+  - { id: 0, class: gr32 }
+frameInfo:
+  maxAlignment:  4
+stack:
+  - { id: 0, name: b, size: 4, alignment: 4 }
+body:
+  - id:          0
+    name:        entry
+    instructions:
+      - '%0 = COPY %edi'
+      # CHECK: [[@LINE+1]]:18: use of undefined stack object '%stack.2'
+      - 'MOV32mr %stack.2, 1, _, 0, _, %0'
+      - '%eax = COPY %0'
+      - 'RETQ %eax'
+...
diff --git a/test/CodeGen/MIR/X86/undefined-virtual-register.mir b/test/CodeGen/MIR/X86/undefined-virtual-register.mir
new file mode 100644
index 00000000000..12370c80caf
--- /dev/null
+++ b/test/CodeGen/MIR/X86/undefined-virtual-register.mir
@@ -0,0 +1,28 @@
+# RUN: not llc -march=x86-64 -start-after machine-sink -stop-after machine-sink -o /dev/null %s 2>&1 | FileCheck %s
+# This test ensures that the MIR parser reports an error when parsing a
+# reference to an undefined virtual register.
+
+--- |
+
+  define i32 @test(i32 %a) {
+  entry:
+    ret i32 %a
+  }
+
+...
+---
+name:            test
+isSSA:           true
+tracksRegLiveness: true
+registers:
+  - { id: 0, class: gr32 }
+body:
+  - id:          0
+    name:        entry
+    instructions:
+      - '%0 = COPY %edi'
+      # CHECK: [[@LINE+1]]:22: use of undefined virtual register '%10'
+      - '%eax = COPY %10'
+      - 'RETQ %eax'
+...
+
diff --git a/test/CodeGen/MIR/X86/unknown-instruction.mir b/test/CodeGen/MIR/X86/unknown-instruction.mir
new file mode 100644
index 00000000000..4e58ca6bad4
--- /dev/null
+++ b/test/CodeGen/MIR/X86/unknown-instruction.mir
@@ -0,0 +1,21 @@
+# RUN: not llc -march=x86-64 -start-after branch-folder -stop-after branch-folder -o /dev/null %s 2>&1 | FileCheck %s
+# This test ensures that an error is reported when an unknown instruction is
+# encountered.
+
+--- |
+
+  define i32 @foo() {
+  entry:
+    ret i32 0
+  }
+
+...
+---
+name:            foo
+body:
+ - id:           0
+   name:         entry
+   instructions:
+     # CHECK: [[@LINE+1]]:8: unknown machine instruction name 'retJust0'
+     - retJust0
+...
diff --git a/test/CodeGen/MIR/X86/unknown-machine-basic-block.mir b/test/CodeGen/MIR/X86/unknown-machine-basic-block.mir
new file mode 100644
index 00000000000..a82e9a780f5
--- /dev/null
+++ b/test/CodeGen/MIR/X86/unknown-machine-basic-block.mir
@@ -0,0 +1,38 @@
+# RUN: not llc -march=x86-64 -start-after branch-folder -stop-after branch-folder -o /dev/null %s 2>&1 | FileCheck %s
+# This test ensures that an error is reported when an invalid machine basic
+# block index is used.
+
+
+--- |
+
+  define i32 @foo(i32* %p) {
+  entry:
+    %a = load i32, i32* %p
+    %b = icmp sle i32 %a, 10
+    br i1 %b, label %0, label %1
+
+  ; <label>:0
+    ret i32 0
+
+  ; <label>:1
+    ret i32 %a
+  }
+
+...
+---
+name:            foo
+body:
+ - id: 0
+   name:         entry
+   instructions:
+     - '%eax = MOV32rm %rdi, 1, _, 0, _'
+     - 'CMP32ri8 %eax, 10, implicit-def %eflags'
+     # CHECK: [[@LINE+1]]:14: use of undefined machine basic block #4
+     - 'JG_1 %bb.4, implicit %eflags'
+ - id: 1
+   instructions:
+     - '%eax = MOV32r0 implicit-def %eflags'
+ - id: 2
+   instructions:
+     - 'RETQ %eax'
+...
diff --git a/test/CodeGen/MIR/X86/unknown-metadata-node.mir b/test/CodeGen/MIR/X86/unknown-metadata-node.mir
new file mode 100644
index 00000000000..194537fa8e2
--- /dev/null
+++ b/test/CodeGen/MIR/X86/unknown-metadata-node.mir
@@ -0,0 +1,61 @@
+# RUN: not llc -march=x86-64 -start-after machine-sink -stop-after machine-sink -o /dev/null %s 2>&1 | FileCheck %s
+
+--- |
+
+  define i32 @test(i32 %x) #0 {
+  entry:
+    %x.addr = alloca i32, align 4
+    store i32 %x, i32* %x.addr, align 4
+    call void @llvm.dbg.declare(metadata i32* %x.addr, metadata !12, metadata !13), !dbg !14
+    %0 = load i32, i32* %x.addr, align 4, !dbg !15
+    ret i32 %0, !dbg !15
+  }
+
+  declare void @llvm.dbg.declare(metadata, metadata, metadata) #1
+
+  attributes #0 = { nounwind "no-frame-pointer-elim"="false" }
+  attributes #1 = { nounwind readnone }
+
+  !llvm.dbg.cu = !{!0}
+  !llvm.module.flags = !{!9, !10}
+  !llvm.ident = !{!11}
+
+  !0 = !DICompileUnit(language: DW_LANG_C99, file: !1, producer: "clang version 3.7.0", isOptimized: false, runtimeVersion: 0, emissionKind: 1, enums: !2, retainedTypes: !2, subprograms: !3, globals: !2, imports: !2)
+  !1 = !DIFile(filename: "test.ll", directory: "")
+  !2 = !{}
+  !3 = !{!4}
+  !4 = !DISubprogram(name: "test", scope: !5, file: !5, line: 4, type: !6, isLocal: false, isDefinition: true, scopeLine: 4, flags: DIFlagPrototyped, isOptimized: false, function: i32 (i32)* @test, variables: !2)
+  !5 = !DIFile(filename: "test.c", directory: "")
+  !6 = !DISubroutineType(types: !7)
+  !7 = !{!8, !8}
+  !8 = !DIBasicType(name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
+  !9 = !{i32 2, !"Dwarf Version", i32 4}
+  !10 = !{i32 2, !"Debug Info Version", i32 3}
+  !11 = !{!"clang version 3.7.0"}
+  !12 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "x", arg: 1, scope: !4, file: !5, line: 4, type: !8)
+  !13 = !DIExpression()
+  !14 = !DILocation(line: 4, scope: !4)
+  !15 = !DILocation(line: 8, scope: !4)
+
+...
+---
+name:            test
+isSSA:           true
+tracksRegLiveness: true
+registers:
+  - { id: 0, class: gr32 }
+frameInfo:
+  maxAlignment:  4
+stack:
+  - { id: 0, name: x.addr, size: 4, alignment: 4 }
+body:
+  - id:          0
+    name:        entry
+    instructions:
+      - '%0 = COPY %edi'
+      # CHECK: [[@LINE+1]]:26: use of undefined metadata '!42'
+      - 'DBG_VALUE _, 0, !42, !13'
+      - 'MOV32mr %stack.0.x.addr, 1, _, 0, _, %0'
+      - '%eax = COPY %0'
+      - 'RETQ %eax'
+...
diff --git a/test/CodeGen/MIR/X86/unknown-named-machine-basic-block.mir b/test/CodeGen/MIR/X86/unknown-named-machine-basic-block.mir
new file mode 100644
index 00000000000..f304113f40b
--- /dev/null
+++ b/test/CodeGen/MIR/X86/unknown-named-machine-basic-block.mir
@@ -0,0 +1,39 @@
+# RUN: not llc -march=x86-64 -start-after branch-folder -stop-after branch-folder -o /dev/null %s 2>&1 | FileCheck %s
+# This test ensures that an error is reported when an unknown named machine
+# basic block is encountered.
+
+--- |
+
+  define i32 @foo(i32* %p) {
+  entry:
+    %a = load i32, i32* %p
+    %0 = icmp sle i32 %a, 10
+    br i1 %0, label %less, label %exit
+
+  less:
+    ret i32 0
+
+  exit:
+    ret i32 %a
+  }
+
+...
+---
+name:            foo
+body:
+ - id:              0
+   name:            entry
+   instructions:
+     - '%eax = MOV32rm %rdi, 1, _, 0, _'
+     - 'CMP32ri8 %eax, 10, implicit-def %eflags'
+     # CHECK: [[@LINE+1]]:14: the name of machine basic block #2 isn't 'hit'
+     - 'JG_1 %bb.2.hit, implicit %eflags'
+ - id:              1
+   name:            less
+   instructions:
+     - '%eax = MOV32r0 implicit-def %eflags'
+ - id:              2
+   name:            exit
+   instructions:
+     - 'RETQ %eax'
+...
diff --git a/test/CodeGen/MIR/X86/unknown-register.mir b/test/CodeGen/MIR/X86/unknown-register.mir
new file mode 100644
index 00000000000..ce40ee809bf
--- /dev/null
+++ b/test/CodeGen/MIR/X86/unknown-register.mir
@@ -0,0 +1,22 @@
+# RUN: not llc -march=x86-64 -start-after branch-folder -stop-after branch-folder -o /dev/null %s 2>&1 | FileCheck %s
+# This test ensures that an error is reported when an unknown register is
+# encountered.
+
+--- |
+
+  define i32 @foo() {
+  entry:
+    ret i32 0
+  }
+
+...
+---
+name:            foo
+body:
+ - id:           0
+   name:         entry
+   instructions:
+     # CHECK: [[@LINE+1]]:9: unknown register name 'xax'
+     - '%xax = MOV32r0'
+     - 'RETQ %xax'
+...
diff --git a/test/CodeGen/MIR/X86/unknown-subregister-index.mir b/test/CodeGen/MIR/X86/unknown-subregister-index.mir
new file mode 100644
index 00000000000..50461232b62
--- /dev/null
+++ b/test/CodeGen/MIR/X86/unknown-subregister-index.mir
@@ -0,0 +1,31 @@
+# RUN: not llc -march=x86-64 -start-after machine-sink -stop-after machine-sink -o /dev/null %s 2>&1 | FileCheck %s
+# This test ensures that an error is reported when an unknown subregister index
+# is encountered.
+
+--- |
+
+  define zeroext i1 @t(i1 %c) {
+  entry:
+    ret i1 %c
+  }
+
+...
+---
+name:            t
+isSSA:           true
+tracksRegLiveness: true
+registers:
+  - { id: 0, class: gr32 }
+  - { id: 1, class: gr8 }
+  - { id: 2, class: gr8 }
+body:
+  - name:        entry
+    id:          0
+    instructions:
+      - '%0 = COPY %edi'
+      # CHECK: [[@LINE+1]]:23: use of unknown subregister index 'bit8'
+      - '%1 = COPY %0:bit8'
+      - '%2 = AND8ri %1, 1, implicit-def %eflags'
+      - '%al = COPY %2'
+      - 'RETQ %al'
+...
diff --git a/test/CodeGen/MIR/X86/unrecognized-character.mir b/test/CodeGen/MIR/X86/unrecognized-character.mir
new file mode 100644
index 00000000000..3b4fb1a9fc6
--- /dev/null
+++ b/test/CodeGen/MIR/X86/unrecognized-character.mir
@@ -0,0 +1,19 @@
+# RUN: not llc -march=x86-64 -start-after branch-folder -stop-after branch-folder -o /dev/null %s 2>&1 | FileCheck %s
+
+--- |
+
+  define void @foo() {
+  entry:
+    ret void
+  }
+
+...
+---
+name:            foo
+body:
+ - id:           0
+   name:         entry
+   instructions:
+     # CHECK: [[@LINE+1]]:9: unexpected character '`'
+     - '` RETQ'
+...
diff --git a/test/CodeGen/MIR/X86/variable-sized-stack-object-size-error.mir b/test/CodeGen/MIR/X86/variable-sized-stack-object-size-error.mir
new file mode 100644
index 00000000000..8e50c52f5e1
--- /dev/null
+++ b/test/CodeGen/MIR/X86/variable-sized-stack-object-size-error.mir
@@ -0,0 +1,36 @@
+# RUN: not llc -march=x86-64 -start-after branch-folder -stop-after branch-folder -o /dev/null %s 2>&1 | FileCheck %s
+--- |
+
+  define i32 @test(i32 %a) {
+  entry:
+    %b = alloca i32
+    %x = alloca i64
+    %y = alloca i32, i32 %a
+    store i32 %a, i32* %b
+    store i64 2, i64* %x
+    %c = load i32, i32* %b
+    ret i32 %c
+  }
+
+...
+---
+name:            test
+frameInfo:
+  stackSize:       24
+  offsetAdjustment: -16
+  maxAlignment:    8
+  adjustsStack:    true
+stack:
+  - { id: 0, offset: -20, size: 4, alignment: 4 }
+  - { id: 1, offset: -32, size: 8, alignment: 8 }
+  # CHECK: [[@LINE+1]]:55: unknown key 'size'
+  - { id: 2, type: variable-sized, offset: -32, size: 42, alignment: 1 }
+body:
+  - id:          0
+    name:        entry
+    instructions:
+      - 'MOV32mr %rsp, 1, _, -4, _, %edi'
+      - 'MOV64mi32 %rsp, 1, _, -16, _, 2'
+      - '%eax = MOV32rm %rsp, 1, _, -4, _'
+      - 'RETQ %eax'
+...
diff --git a/test/CodeGen/MIR/X86/variable-sized-stack-objects.mir b/test/CodeGen/MIR/X86/variable-sized-stack-objects.mir
new file mode 100644
index 00000000000..5e6467e9cfa
--- /dev/null
+++ b/test/CodeGen/MIR/X86/variable-sized-stack-objects.mir
@@ -0,0 +1,42 @@
+# RUN: llc -march=x86-64 -start-after branch-folder -stop-after branch-folder -o /dev/null %s | FileCheck %s
+# This test ensures that the MIR parser parses variable sized stack objects
+# correctly.
+
+--- |
+
+  define i32 @test(i32 %a) {
+  entry:
+    %b = alloca i32
+    %x = alloca i64
+    %y = alloca i32, i32 %a
+    store i32 %a, i32* %b
+    store i64 2, i64* %x
+    %c = load i32, i32* %b
+    ret i32 %c
+  }
+
+...
+---
+name:            test
+frameInfo:
+  stackSize:       24
+  offsetAdjustment: -16
+  maxAlignment:    8
+  adjustsStack:    true
+# CHECK: stack:
+# CHECK-NEXT: - { id: 0, offset: -20, size: 4, alignment: 4 }
+# CHECK-NEXT: - { id: 1, offset: -32, size: 8, alignment: 8 }
+# CHECK-NEXT: - { id: 2, name: y, type: variable-sized, offset: -32, alignment: 1 }
+stack:
+  - { id: 0, offset: -20, size: 4, alignment: 4 }
+  - { id: 1, offset: -32, size: 8, alignment: 8 }
+  - { id: 2, name: y, type: variable-sized, offset: -32, alignment: 1 }
+body:
+  - id:          0
+    name:        entry
+    instructions:
+      - 'MOV32mr %rsp, 1, _, -4, _, %edi'
+      - 'MOV64mi32 %rsp, 1, _, -16, _, 2'
+      - '%eax = MOV32rm %rsp, 1, _, -4, _'
+      - 'RETQ %eax'
+...
diff --git a/test/CodeGen/MIR/X86/virtual-register-redefinition-error.mir b/test/CodeGen/MIR/X86/virtual-register-redefinition-error.mir
new file mode 100644
index 00000000000..baadeaee3d4
--- /dev/null
+++ b/test/CodeGen/MIR/X86/virtual-register-redefinition-error.mir
@@ -0,0 +1,28 @@
+# RUN: not llc -march=x86-64 -start-after machine-sink -stop-after machine-sink -o /dev/null %s 2>&1 | FileCheck %s
+
+--- |
+
+  define i32 @test(i32 %a) {
+  body:
+    ret i32 %a
+  }
+
+...
+---
+name:            test
+isSSA:           true
+tracksRegLiveness: true
+registers:
+  - { id: 0, class: gr32 }
+# CHECK: [[@LINE+1]]:11: redefinition of virtual register '%0'
+  - { id: 0, class: gr32 }
+body:
+  - id:          0
+    name:        body
+    liveins:     [ '%edi' ]
+    instructions:
+      - '%0 = COPY %edi'
+      - '%eax = COPY %0'
+      - 'RETQ %eax'
+...
+
diff --git a/test/CodeGen/MIR/X86/virtual-registers.mir b/test/CodeGen/MIR/X86/virtual-registers.mir
new file mode 100644
index 00000000000..cde6b988519
--- /dev/null
+++ b/test/CodeGen/MIR/X86/virtual-registers.mir
@@ -0,0 +1,109 @@
+# RUN: llc -march=x86-64 -start-after machine-sink -stop-after machine-sink -o /dev/null %s | FileCheck %s
+# This test ensures that the MIR parser parses virtual register definitions and
+# references correctly.
+
+--- |
+
+  define i32 @bar(i32 %a) {
+  entry:
+    %0 = icmp sle i32 %a, 10
+    br i1 %0, label %less, label %exit
+
+  less:
+    ret i32 0
+
+  exit:
+    ret i32 %a
+  }
+
+  define i32 @foo(i32 %a) {
+  entry:
+    %0 = icmp sle i32 %a, 10
+    br i1 %0, label %less, label %exit
+
+  less:
+    ret i32 0
+
+  exit:
+    ret i32 %a
+  }
+
+...
+---
+name:            bar
+isSSA:           true
+tracksRegLiveness: true
+# CHECK:      registers:
+# CHECK-NEXT:   - { id: 0, class: gr32 }
+# CHECK-NEXT:   - { id: 1, class: gr32 }
+# CHECK-NEXT:   - { id: 2, class: gr32 }
+registers:
+  - { id: 0, class: gr32 }
+  - { id: 1, class: gr32 }
+  - { id: 2, class: gr32 }
+body:
+  - id:          0
+    name:        entry
+    successors:  [ '%bb.2.exit', '%bb.1.less' ]
+    liveins:     [ '%edi' ]
+    # CHECK:      %0 = COPY %edi
+    # CHECK-NEXT: %1 = SUB32ri8 %0, 10
+    instructions:
+      - '%0 = COPY %edi'
+      - '%1 = SUB32ri8 %0, 10, implicit-def %eflags'
+      - 'JG_1 %bb.2.exit, implicit %eflags'
+      - 'JMP_1 %bb.1.less'
+  - id:          1
+    name:        less
+    # CHECK:      %2 = MOV32r0
+    # CHECK-NEXT: %eax = COPY %2
+    instructions:
+      - '%2 = MOV32r0 implicit-def %eflags'
+      - '%eax = COPY %2'
+      - 'RETQ %eax'
+  - id:          2
+    name:        exit
+    instructions:
+      - '%eax = COPY %0'
+      - 'RETQ %eax'
+...
+---
+name:            foo
+isSSA:           true
+tracksRegLiveness: true
+# CHECK: name: foo
+# CHECK:      registers:
+# CHECK-NEXT:   - { id: 0, class: gr32 }
+# CHECK-NEXT:   - { id: 1, class: gr32 }
+# CHECK-NEXT:   - { id: 2, class: gr32 }
+registers:
+  - { id: 2, class: gr32 }
+  - { id: 0, class: gr32 }
+  - { id: 10, class: gr32 }
+body:
+  - id:          0
+    name:        entry
+    successors:  [ '%bb.2.exit', '%bb.1.less' ]
+    liveins:     [ '%edi' ]
+    # CHECK:      %0 = COPY %edi
+    # CHECK-NEXT: %1 = SUB32ri8 %0, 10
+    instructions:
+      - '%2 = COPY %edi'
+      - '%0 = SUB32ri8 %2, 10, implicit-def %eflags'
+      - 'JG_1 %bb.2.exit, implicit %eflags'
+      - 'JMP_1 %bb.1.less'
+  - id:          1
+    name:        less
+    # CHECK:      %2 = MOV32r0
+    # CHECK-NEXT: %eax = COPY %2
+    instructions:
+      - '%10 = MOV32r0 implicit-def %eflags'
+      - '%eax = COPY %10'
+      - 'RETQ %eax'
+  - id:          2
+    name:        exit
+    # CHECK: %eax = COPY %0
+    instructions:
+      - '%eax = COPY %2'
+      - 'RETQ %eax'
+...
diff --git a/test/CodeGen/MIR/basic-blocks.mir b/test/CodeGen/MIR/basic-blocks.mir
new file mode 100644
index 00000000000..17313047576
--- /dev/null
+++ b/test/CodeGen/MIR/basic-blocks.mir
@@ -0,0 +1,49 @@
+# RUN: llc -start-after branch-folder -stop-after branch-folder -o /dev/null %s | FileCheck %s
+# This test ensures that the MIR parser parses machine functions correctly.
+
+--- |
+
+  define i32 @foo() {
+  entry:
+    ret i32 0
+  }
+
+  define i32 @bar() {
+  start:
+    ret i32 0
+  }
+
+...
+---
+# CHECK: name: foo
+# CHECK: body:
+# CHECK-NEXT: - id: 0
+# CHECK-NEXT:   name: entry
+# CHECK-NEXT:   alignment: 0
+# CHECK-NEXT:   isLandingPad: false
+# CHECK-NEXT:   addressTaken: false
+name:            foo
+body:
+ - id:           0
+   name:         entry
+...
+---
+# CHECK: name: bar
+# CHECK: body:
+# CHECK-NEXT: - id: 0
+# CHECK-NEXT:   name: start
+# CHECK-NEXT:   alignment: 4
+# CHECK-NEXT:   isLandingPad: false
+# CHECK-NEXT:   addressTaken: false
+# CHECK-NEXT: - id: 1
+# CHECK-NEXT:   alignment: 0
+# CHECK-NEXT:   isLandingPad: false
+# CHECK-NEXT:   addressTaken: true
+name:            bar
+body:
+ - id:           0
+   name:         start
+   alignment:    4
+ - id:           1
+   addressTaken: true
+...
diff --git a/test/CodeGen/MIR/expected-eof-after-successor-mbb.mir b/test/CodeGen/MIR/expected-eof-after-successor-mbb.mir
new file mode 100644
index 00000000000..25ae5119297
--- /dev/null
+++ b/test/CodeGen/MIR/expected-eof-after-successor-mbb.mir
@@ -0,0 +1,29 @@
+# RUN: not llc -start-after branch-folder -stop-after branch-folder -o /dev/null %s 2>&1 | FileCheck %s
+
+--- |
+
+  define i32 @foo(i32 %a) {
+  entry:
+    %0 = icmp sle i32 %a, 10
+    br i1 %0, label %less, label %exit
+
+  less:
+    ret i32 0
+
+  exit:
+    ret i32 %a
+  }
+
+...
+---
+name:            foo
+body:
+  - id:          0
+    name:        entry
+    # CHECK: [[@LINE+1]]:46: expected end of string after the machine basic block reference
+    successors:  [ '%bb.1.less', '%bb.2.exit 2' ]
+  - id:          1
+    name:        less
+  - id:          2
+    name:        exit
+...
diff --git a/test/CodeGen/MIR/expected-mbb-reference-for-successor-mbb.mir b/test/CodeGen/MIR/expected-mbb-reference-for-successor-mbb.mir
new file mode 100644
index 00000000000..ce9192901d7
--- /dev/null
+++ b/test/CodeGen/MIR/expected-mbb-reference-for-successor-mbb.mir
@@ -0,0 +1,29 @@
+# RUN: not llc -start-after branch-folder -stop-after branch-folder -o /dev/null %s 2>&1 | FileCheck %s
+
+--- |
+
+  define i32 @foo(i32 %a) {
+  entry:
+    %0 = icmp sle i32 %a, 10
+    br i1 %0, label %less, label %exit
+
+  less:
+    ret i32 0
+
+  exit:
+    ret i32 %a
+  }
+
+...
+---
+name:            foo
+body:
+  - id:          0
+    name:        entry
+    # CHECK: [[@LINE+1]]:35: expected a machine basic block reference
+    successors:  [ '%bb.1.less', '2' ]
+  - id:          1
+    name:        less
+  - id:          2
+    name:        exit
+...
diff --git a/test/CodeGen/MIR/frame-info.mir b/test/CodeGen/MIR/frame-info.mir
new file mode 100644
index 00000000000..c5468f94f33
--- /dev/null
+++ b/test/CodeGen/MIR/frame-info.mir
@@ -0,0 +1,91 @@
+# RUN: llc -start-after machine-sink -stop-after machine-sink -o /dev/null %s | FileCheck %s
+# This test ensures that the MIR parser parses machine frame info properties
+# correctly.
+
+--- |
+
+  define i32 @test(i32 %a) {
+  entry:
+    %b = alloca i32
+    store i32 %a, i32* %b
+    %c = load i32, i32* %b
+    ret i32 %c
+  }
+
+  define i32 @test2(i32 %a) {
+  entry:
+    %b = alloca i32
+    store i32 %a, i32* %b
+    %c = load i32, i32* %b
+    ret i32 %c
+  }
+
+...
+---
+name:            test
+isSSA:           true
+tracksRegLiveness: true
+
+# CHECK: frameInfo:
+# CHECK-NEXT: isFrameAddressTaken: false
+# CHECK-NEXT: isReturnAddressTaken: false
+# CHECK-NEXT: hasStackMap: false
+# CHECK-NEXT: hasPatchPoint: false
+# CHECK-NEXT: stackSize: 0
+# CHECK-NEXT: offsetAdjustment: 0
+# Note: max alignment can be target specific when printed.
+# CHECK-NEXT: maxAlignment:
+# CHECK-NEXT: adjustsStack: false
+# CHECK-NEXT: hasCalls: false
+# CHECK-NEXT: maxCallFrameSize: 0
+# CHECK-NEXT: hasOpaqueSPAdjustment: false
+# CHECK-NEXT: hasVAStart: false
+# CHECK-NEXT: hasMustTailInVarArgFunc: false
+# CHECK: body
+frameInfo:
+  maxAlignment:    4
+body:
+  - id:          0
+    name:        entry
+...
+---
+name:            test2
+isSSA:           true
+tracksRegLiveness: true
+
+# CHECK: test2
+# CHECK: frameInfo:
+# CHECK-NEXT: isFrameAddressTaken: true
+# CHECK-NEXT: isReturnAddressTaken: true
+# CHECK-NEXT: hasStackMap: true
+# CHECK-NEXT: hasPatchPoint: true
+# CHECK-NEXT: stackSize: 4
+# CHECK-NEXT: offsetAdjustment: 4
+# Note: max alignment can be target specific when printed.
+# CHECK-NEXT: maxAlignment:
+# CHECK-NEXT: adjustsStack: true
+# CHECK-NEXT: hasCalls: true
+# CHECK-NEXT: maxCallFrameSize: 4
+# CHECK-NEXT: hasOpaqueSPAdjustment: true
+# CHECK-NEXT: hasVAStart: true
+# CHECK-NEXT: hasMustTailInVarArgFunc: true
+# CHECK: body
+frameInfo:
+  isFrameAddressTaken: true
+  isReturnAddressTaken: true
+  hasStackMap:     true
+  hasPatchPoint:   true
+  stackSize:       4
+  offsetAdjustment: 4
+  maxAlignment:    4
+  adjustsStack:    true
+  hasCalls:        true
+  maxCallFrameSize: 4
+  hasOpaqueSPAdjustment: true
+  hasVAStart:      true
+  hasMustTailInVarArgFunc: true
+body:
+  - id:          0
+    name:        entry
+...
+
diff --git a/test/CodeGen/MIR/function-missing-machine-function.mir b/test/CodeGen/MIR/function-missing-machine-function.mir
new file mode 100644
index 00000000000..71b5b284534
--- /dev/null
+++ b/test/CodeGen/MIR/function-missing-machine-function.mir
@@ -0,0 +1,13 @@
+# RUN: not llc -start-after branch-folder -stop-after branch-folder -o /dev/null %s 2>&1 | FileCheck %s
+# This test verifies that an error is reported when a MIR file has some
+# function but is missing a corresponding machine function.
+
+# CHECK: no machine function information for function 'foo' in the MIR file
+
+--- |
+
+  define i32 @foo() {
+    ret i32 0
+  }
+
+...
diff --git a/test/CodeGen/MIR/invalid-jump-table-kind.mir b/test/CodeGen/MIR/invalid-jump-table-kind.mir
new file mode 100644
index 00000000000..4876f795386
--- /dev/null
+++ b/test/CodeGen/MIR/invalid-jump-table-kind.mir
@@ -0,0 +1,54 @@
+# RUN: not llc -start-after branch-folder -stop-after branch-folder -o /dev/null %s 2>&1 | FileCheck %s
+
+--- |
+
+  define i32 @test_jumptable(i32 %in) {
+  entry:
+    switch i32 %in, label %def [
+      i32 0, label %lbl1
+      i32 1, label %lbl2
+      i32 2, label %lbl3
+      i32 3, label %lbl4
+    ]
+
+  def:
+    ret i32 0
+
+  lbl1:
+    ret i32 1
+
+  lbl2:
+    ret i32 2
+
+  lbl3:
+    ret i32 4
+
+  lbl4:
+    ret i32 8
+  }
+
+...
+---
+name:            test_jumptable
+jumpTable:
+  # CHECK: [[@LINE+1]]:18: unknown enumerated scalar
+  kind:          switch
+  entries:
+    - id:        0
+      blocks:    [ '%bb.3.lbl1', '%bb.4.lbl2', '%bb.5.lbl3', '%bb.6.lbl4' ]
+body:
+  - id:          0
+    name:        entry
+  - id:          1
+    name:        entry
+  - id:          2
+    name:        def
+  - id:          3
+    name:        lbl1
+  - id:          4
+    name:        lbl2
+  - id:          5
+    name:        lbl3
+  - id:          6
+    name:        lbl4
+...
diff --git a/test/CodeGen/MIR/lit.local.cfg b/test/CodeGen/MIR/lit.local.cfg
new file mode 100644
index 00000000000..e69aa576535
--- /dev/null
+++ b/test/CodeGen/MIR/lit.local.cfg
@@ -0,0 +1,2 @@
+config.suffixes = ['.mir']
+
diff --git a/test/CodeGen/MIR/llvm-ir-error-reported.mir b/test/CodeGen/MIR/llvm-ir-error-reported.mir
new file mode 100644
index 00000000000..3508c341c44
--- /dev/null
+++ b/test/CodeGen/MIR/llvm-ir-error-reported.mir
@@ -0,0 +1,22 @@
+# RUN: not llc -start-after branch-folder -stop-after branch-folder -o /dev/null %s 2>&1 | FileCheck %s
+# This test ensures an error is reported if the embedded LLVM IR contains an
+# error.
+
+--- |
+  
+  ; CHECK: [[@LINE+3]]:15: use of undefined value '%a'
+  define i32 @foo(i32 %x, i32 %y) {
+    %z = alloca i32, align 4
+    store i32 %a, i32* %z, align 4
+    br label %Test
+  Test:
+    %m = load i32, i32* %z, align 4
+    %cond = icmp eq i32 %y, %m
+    br i1 %cond, label %IfEqual, label %IfUnequal
+  IfEqual:
+    ret i32 1
+  IfUnequal:
+    ret i32 0
+  }
+  
+...
diff --git a/test/CodeGen/MIR/llvmIR.mir b/test/CodeGen/MIR/llvmIR.mir
new file mode 100644
index 00000000000..3c084ad7d39
--- /dev/null
+++ b/test/CodeGen/MIR/llvmIR.mir
@@ -0,0 +1,37 @@
+# RUN: llc -start-after branch-folder -stop-after branch-folder -o /dev/null %s | FileCheck %s
+# This test ensures that the LLVM IR that's embedded with MIR is parsed
+# correctly.
+
+--- |
+  ; CHECK: define i32 @foo(i32 %x, i32 %y)
+  ; CHECK: %z = alloca i32, align 4
+  ; CHECK: store i32 %x, i32* %z, align 4
+  ; CHECK: br label %Test
+  ; CHECK: Test:
+  ; CHECK: %m = load i32, i32* %z, align 4
+  ; CHECK: %cond = icmp eq i32 %y, %m
+  ; CHECK: br i1 %cond, label %IfEqual, label %IfUnequal
+  ; CHECK: IfEqual:
+  ; CHECK: ret i32 1
+  ; CHECK: IfUnequal:
+  ; CHECK: ret i32 0
+  define i32 @foo(i32 %x, i32 %y) {
+    %z = alloca i32, align 4
+    store i32 %x, i32* %z, align 4
+    br label %Test
+  Test:
+    %m = load i32, i32* %z, align 4
+    %cond = icmp eq i32 %y, %m
+    br i1 %cond, label %IfEqual, label %IfUnequal
+  IfEqual:
+    ret i32 1
+  IfUnequal:
+    ret i32 0
+  }
+  
+...
+---
+name: foo
+body:
+  - id: 0
+...
diff --git a/test/CodeGen/MIR/llvmIRMissing.mir b/test/CodeGen/MIR/llvmIRMissing.mir
new file mode 100644
index 00000000000..80cea5a6fda
--- /dev/null
+++ b/test/CodeGen/MIR/llvmIRMissing.mir
@@ -0,0 +1,9 @@
+# RUN: llc -start-after branch-folder -stop-after branch-folder -o /dev/null %s | FileCheck %s
+# This test ensures that the MIR parser accepts files without the LLVM IR.
+
+---
+# CHECK: name: foo
+name: foo
+body:
+  - id: 0
+...
diff --git a/test/CodeGen/MIR/machine-basic-block-expected-ir-block.mir b/test/CodeGen/MIR/machine-basic-block-expected-ir-block.mir
new file mode 100644
index 00000000000..2f3d40af792
--- /dev/null
+++ b/test/CodeGen/MIR/machine-basic-block-expected-ir-block.mir
@@ -0,0 +1,16 @@
+# RUN: not llc -start-after branch-folder -stop-after branch-folder -o /dev/null %s 2>&1 | FileCheck %s
+
+--- |
+
+  define i32 @foo() {
+    ret i32 0
+  }
+
+...
+---
+name:            foo
+body:
+ - id:           0
+# CHECK: [[@LINE+1]]:19: expected an IR block reference
+   ir-block:     '0'
+...
diff --git a/test/CodeGen/MIR/machine-basic-block-ir-block-reference.mir b/test/CodeGen/MIR/machine-basic-block-ir-block-reference.mir
new file mode 100644
index 00000000000..ccbab534388
--- /dev/null
+++ b/test/CodeGen/MIR/machine-basic-block-ir-block-reference.mir
@@ -0,0 +1,19 @@
+# RUN: llc -start-after branch-folder -stop-after branch-folder -o /dev/null %s | FileCheck %s
+# This test ensures that the MIR parser preserves unnamed LLVM IR block
+# references.
+
+--- |
+
+  define i32 @foo() {
+    ret i32 0
+  }
+
+...
+---
+name:            foo
+body:
+# CHECK: id: 0
+# CHECK: ir-block: '%ir-block.0'
+ - id:           0
+   ir-block:     '%ir-block.0'
+...
diff --git a/test/CodeGen/MIR/machine-basic-block-redefinition-error.mir b/test/CodeGen/MIR/machine-basic-block-redefinition-error.mir
new file mode 100644
index 00000000000..deac3b0b69b
--- /dev/null
+++ b/test/CodeGen/MIR/machine-basic-block-redefinition-error.mir
@@ -0,0 +1,17 @@
+# RUN: not llc -start-after branch-folder -stop-after branch-folder -o /dev/null %s 2>&1 | FileCheck %s
+
+--- |
+
+  define i32 @foo() {
+  entry:
+    ret i32 0
+  }
+
+...
+---
+name:            foo
+body:
+  # CHECK: redefinition of machine basic block with id #0
+  - id:       0
+  - id:       0
+...
diff --git a/test/CodeGen/MIR/machine-basic-block-undefined-ir-block.mir b/test/CodeGen/MIR/machine-basic-block-undefined-ir-block.mir
new file mode 100644
index 00000000000..76d561e61fa
--- /dev/null
+++ b/test/CodeGen/MIR/machine-basic-block-undefined-ir-block.mir
@@ -0,0 +1,16 @@
+# RUN: not llc -start-after branch-folder -stop-after branch-folder -o /dev/null %s 2>&1 | FileCheck %s
+
+--- |
+
+  define i32 @foo() {
+    ret i32 0
+  }
+
+...
+---
+name:            foo
+body:
+ - id:           0
+# CHECK: [[@LINE+1]]:19: use of undefined IR block '%ir-block.10'
+   ir-block:     '%ir-block.10'
+...
diff --git a/test/CodeGen/MIR/machine-basic-block-unknown-name.mir b/test/CodeGen/MIR/machine-basic-block-unknown-name.mir
new file mode 100644
index 00000000000..df8eee9d270
--- /dev/null
+++ b/test/CodeGen/MIR/machine-basic-block-unknown-name.mir
@@ -0,0 +1,19 @@
+# RUN: not llc -start-after branch-folder -stop-after branch-folder -o /dev/null %s 2>&1 | FileCheck %s
+# This test ensures that an error is reported whenever the MIR parser can't find
+# a basic block with the machine basis block's name.
+
+--- |
+
+  define i32 @foo() {
+  entry:
+    ret i32 0
+  }
+
+...
+---
+name:            foo
+body:
+  # CHECK: [[@LINE+2]]:18: basic block 'entrie' is not defined in the function 'foo'
+  - id:          0
+    name:        entrie
+...
diff --git a/test/CodeGen/MIR/machine-function-missing-body-error.mir b/test/CodeGen/MIR/machine-function-missing-body-error.mir
new file mode 100644
index 00000000000..0dc7477f627
--- /dev/null
+++ b/test/CodeGen/MIR/machine-function-missing-body-error.mir
@@ -0,0 +1,15 @@
+# RUN: not llc -start-after branch-folder -stop-after branch-folder -o /dev/null %s 2>&1 | FileCheck %s
+# This test ensures that the MIR parser reports an error when it encounters a
+# machine function with an empty body.
+
+--- |
+
+  define i32 @foo() {
+    ret i32 0
+  }
+
+...
+---
+# CHECK: machine function 'foo' requires at least one machine basic block in its body
+name:            foo
+...
diff --git a/test/CodeGen/MIR/machine-function-missing-function.mir b/test/CodeGen/MIR/machine-function-missing-function.mir
new file mode 100644
index 00000000000..424c34aae84
--- /dev/null
+++ b/test/CodeGen/MIR/machine-function-missing-function.mir
@@ -0,0 +1,23 @@
+# RUN: not llc -start-after branch-folder -stop-after branch-folder -o /dev/null %s 2>&1 | FileCheck %s
+# This test ensures that an error is reported when the mir file has LLVM IR and
+# one of the machine functions has a name that doesn't match any function in
+# the LLVM IR.
+
+--- |
+
+  define i32 @foo() {
+    ret i32 0
+  }
+
+...
+---
+name:            foo
+body:
+  - id: 0
+...
+---
+# CHECK: function 'faa' isn't defined in the provided LLVM IR
+name:            faa
+body:
+  - id: 0
+...
diff --git a/test/CodeGen/MIR/machine-function-missing-name.mir b/test/CodeGen/MIR/machine-function-missing-name.mir
new file mode 100644
index 00000000000..a868a65d35f
--- /dev/null
+++ b/test/CodeGen/MIR/machine-function-missing-name.mir
@@ -0,0 +1,26 @@
+# RUN: not llc -start-after branch-folder -stop-after branch-folder -o /dev/null %s 2>&1 | FileCheck %s
+# This test ensures that an error is reported when a machine function doesn't
+# have a name attribute.
+
+--- |
+
+  define i32 @foo() {
+    ret i32 0
+  }
+
+  define i32 @bar() {
+    ret i32 0
+  }
+
+...
+---
+# CHECK: [[@LINE+1]]:1: missing required key 'name'
+nme:             foo
+body:
+  - id: 0
+...
+---
+name:            bar
+body:
+  - id: 0
+...
diff --git a/test/CodeGen/MIR/machine-function-redefinition-error.mir b/test/CodeGen/MIR/machine-function-redefinition-error.mir
new file mode 100644
index 00000000000..be84161b563
--- /dev/null
+++ b/test/CodeGen/MIR/machine-function-redefinition-error.mir
@@ -0,0 +1,10 @@
+# RUN: not llc -start-after branch-folder -stop-after branch-folder -o /dev/null %s 2>&1 | FileCheck %s
+# This test ensures that the machine function errors are reported correctly.
+
+---
+name:            foo
+...
+---
+# CHECK: redefinition of machine function 'foo'
+name:            foo
+...
diff --git a/test/CodeGen/MIR/machine-function.mir b/test/CodeGen/MIR/machine-function.mir
new file mode 100644
index 00000000000..afd10ab02c2
--- /dev/null
+++ b/test/CodeGen/MIR/machine-function.mir
@@ -0,0 +1,66 @@
+# RUN: llc -start-after branch-folder -stop-after branch-folder -o /dev/null %s | FileCheck %s
+# This test ensures that the MIR parser parses machine functions correctly.
+
+--- |
+
+  define i32 @foo() {
+    ret i32 0
+  }
+
+  define i32 @bar() {
+    ret i32 0
+  }
+
+  define i32 @func() {
+    ret i32 0
+  }
+
+  define i32 @func2() {
+    ret i32 0
+  }
+  
+...
+---
+# CHECK: name: foo
+# CHECK-NEXT: alignment:
+# CHECK-NEXT: exposesReturnsTwice: false
+# CHECK-NEXT: hasInlineAsm: false
+# CHECK: ...
+name:            foo
+body:
+  - id: 0
+...
+---
+# CHECK: name: bar
+# CHECK-NEXT: alignment:
+# CHECK-NEXT: exposesReturnsTwice: false
+# CHECK-NEXT: hasInlineAsm: false
+# CHECK: ...
+name:            bar
+body:
+  - id: 0
+...
+---
+# CHECK: name: func
+# CHECK-NEXT: alignment: 8
+# CHECK-NEXT: exposesReturnsTwice: false
+# CHECK-NEXT: hasInlineAsm: false
+# CHECK: ...
+name:            func
+alignment:       8
+body:
+  - id: 0
+...
+---
+# CHECK: name: func2
+# CHECK-NEXT: alignment: 16
+# CHECK-NEXT: exposesReturnsTwice: true
+# CHECK-NEXT: hasInlineAsm: true
+# CHECK: ...
+name:            func2
+alignment:       16
+exposesReturnsTwice: true
+hasInlineAsm:    true
+body:
+  - id: 0
+...
diff --git a/test/CodeGen/MIR/register-info.mir b/test/CodeGen/MIR/register-info.mir
new file mode 100644
index 00000000000..9585faa9622
--- /dev/null
+++ b/test/CodeGen/MIR/register-info.mir
@@ -0,0 +1,40 @@
+# RUN: llc -start-after branch-folder -stop-after branch-folder -o /dev/null %s | FileCheck %s
+# This test ensures that the MIR parser parses machine register info properties
+# correctly.
+
+--- |
+
+  define i32 @foo() {
+  entry:
+    ret i32 0
+  }
+
+  define i32 @bar() {
+  start:
+    ret i32 0
+  }
+
+...
+---
+# CHECK: name: foo
+# CHECK:      isSSA: false
+# CHECK-NEXT: tracksRegLiveness: false
+# CHECK-NEXT: tracksSubRegLiveness: false
+# CHECK: ...
+name:            foo
+body:
+  - id: 0
+...
+---
+# CHECK: name: bar
+# CHECK:      isSSA: false
+# CHECK-NEXT: tracksRegLiveness: true
+# CHECK-NEXT: tracksSubRegLiveness: true
+# CHECK: ...
+name: bar
+isSSA: false
+tracksRegLiveness: true
+tracksSubRegLiveness: true
+body:
+  - id: 0
+...
diff --git a/test/CodeGen/Mips/2008-11-10-xint_to_fp.ll b/test/CodeGen/Mips/2008-11-10-xint_to_fp.ll
index 9cebfcd5012..789f7ee3d4b 100644
--- a/test/CodeGen/Mips/2008-11-10-xint_to_fp.ll
+++ b/test/CodeGen/Mips/2008-11-10-xint_to_fp.ll
@@ -1,4 +1,4 @@
-; RUN: llc -march=mips -soft-float < %s
+; RUN: llc -march=mips -mattr=+soft-float < %s
 ; PR2667
 target datalayout = "e-p:32:32:32-i1:8:8-i8:8:32-i16:16:32-i32:32:32-i64:32:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64"
 	%struct._Bigint = type { %struct._Bigint*, i32, i32, i32, i32, [1 x i32] }
diff --git a/test/CodeGen/Mips/Fast-ISel/br1.ll b/test/CodeGen/Mips/Fast-ISel/br1.ll
index 11842ddc418..a448e90187c 100644
--- a/test/CodeGen/Mips/Fast-ISel/br1.ll
+++ b/test/CodeGen/Mips/Fast-ISel/br1.ll
@@ -1,6 +1,6 @@
-; RUN: llc -march=mipsel -relocation-model=pic -O0 -mips-fast-isel -fast-isel-abort=1 -mcpu=mips32r2 \
+; RUN: llc -march=mipsel -relocation-model=pic -O0 -fast-isel-abort=1 -mcpu=mips32r2 \
 ; RUN:     < %s | FileCheck %s
-; RUN: llc -march=mipsel -relocation-model=pic -O0 -mips-fast-isel -fast-isel-abort=1 -mcpu=mips32 \
+; RUN: llc -march=mipsel -relocation-model=pic -O0 -fast-isel-abort=1 -mcpu=mips32 \
 ; RUN:     < %s | FileCheck %s
 
 @b = global i32 1, align 4
diff --git a/test/CodeGen/Mips/Fast-ISel/bswap1.ll b/test/CodeGen/Mips/Fast-ISel/bswap1.ll
new file mode 100644
index 00000000000..8f1f703ea07
--- /dev/null
+++ b/test/CodeGen/Mips/Fast-ISel/bswap1.ll
@@ -0,0 +1,58 @@
+; RUN: llc < %s -march=mipsel -mcpu=mips32 -O0 -relocation-model=pic \
+; RUN:      -fast-isel-abort=1 | FileCheck %s \
+; RUN:      -check-prefix=ALL -check-prefix=32R1
+; RUN: llc < %s -march=mipsel -mcpu=mips32r2 -O0 -relocation-model=pic \
+; RUN:      -fast-isel-abort=1 | FileCheck %s \
+; RUN:      -check-prefix=ALL -check-prefix=32R2
+
+@a = global i16 -21829, align 2
+@b = global i32 -1430532899, align 4
+@a1 = common global i16 0, align 2
+@b1 = common global i32 0, align 4
+
+declare i16 @llvm.bswap.i16(i16)
+declare i32 @llvm.bswap.i32(i32)
+
+define void @b16() {
+  ; ALL-LABEL:  b16:
+
+  ; ALL:            lw    $[[A_ADDR:[0-9]+]], %got(a)($[[GOT_ADDR:[0-9]+]])
+  ; ALL:            lhu   $[[A_VAL:[0-9]+]], 0($[[A_ADDR]])
+
+  ; 32R1:           sll   $[[TMP1:[0-9]+]], $[[A_VAL]], 8
+  ; 32R1:           srl   $[[TMP2:[0-9]+]], $[[A_VAL]], 8
+  ; 32R1:           or    $[[TMP3:[0-9]+]], $[[TMP1]], $[[TMP2]]
+  ; 32R1:           andi  $[[TMP4:[0-9]+]], $[[TMP3]], 65535
+
+  ; 32R2:           wsbh  $[[RESULT:[0-9]+]], $[[A_VAL]]
+
+  %1 = load i16, i16* @a, align 2
+  %2 = call i16 @llvm.bswap.i16(i16 %1)
+  store i16 %2, i16* @a1, align 2
+  ret void
+}
+
+define void @b32() {
+  ; ALL-LABEL:  b32:
+
+  ; ALL:            lw    $[[B_ADDR:[0-9]+]], %got(b)($[[GOT_ADDR:[0-9]+]])
+  ; ALL:            lw    $[[B_VAL:[0-9]+]], 0($[[B_ADDR]])
+
+  ; 32R1:           srl   $[[TMP1:[0-9]+]], $[[B_VAL]], 8
+  ; 32R1:           srl   $[[TMP2:[0-9]+]], $[[B_VAL]], 24
+  ; 32R1:           andi  $[[TMP3:[0-9]+]], $[[TMP1]], 65280
+  ; 32R1:           or    $[[TMP4:[0-9]+]], $[[TMP2]], $[[TMP3]]
+  ; 32R1:           andi  $[[TMP5:[0-9]+]], $[[B_VAL]], 65280
+  ; 32R1:           sll   $[[TMP6:[0-9]+]], $[[TMP5]], 8
+  ; 32R1:           sll   $[[TMP7:[0-9]+]], $[[B_VAL]], 24
+  ; 32R1:           or    $[[TMP8:[0-9]+]], $[[TMP4]], $[[TMP6]]
+  ; 32R1:           or    $[[RESULT:[0-9]+]], $[[TMP7]], $[[TMP8]]
+
+  ; 32R2:           wsbh  $[[TMP:[0-9]+]], $[[B_VAL]]
+  ; 32R2:           rotr  $[[RESULT:[0-9]+]], $[[TMP]], 16
+
+  %1 = load i32, i32* @b, align 4
+  %2 = call i32 @llvm.bswap.i32(i32 %1)
+  store i32 %2, i32* @b1, align 4
+  ret void
+}
diff --git a/test/CodeGen/Mips/Fast-ISel/callabi.ll b/test/CodeGen/Mips/Fast-ISel/callabi.ll
index f80cb82ca4d..34616a50b1a 100644
--- a/test/CodeGen/Mips/Fast-ISel/callabi.ll
+++ b/test/CodeGen/Mips/Fast-ISel/callabi.ll
@@ -1,477 +1,538 @@
-; RUN: llc -march=mipsel -relocation-model=pic -O0 -mips-fast-isel -fast-isel-abort=1 -mcpu=mips32r2 \
-; RUN:     < %s | FileCheck %s
-; RUN: llc -march=mipsel -relocation-model=pic -O0 -mips-fast-isel -fast-isel-abort=1 -mcpu=mips32 \
-; RUN:     < %s | FileCheck %s
-; RUN: llc -march=mipsel -relocation-model=pic -O0 -mips-fast-isel -fast-isel-abort=1 -mcpu=mips32r2 \
-; RUN:     < %s | FileCheck %s -check-prefix=mips32r2
-; RUN: llc -march=mipsel -relocation-model=pic -O0 -mips-fast-isel -fast-isel-abort=1 -mcpu=mips32 \
-; RUN:     < %s | FileCheck %s -check-prefix=mips32
-; RUN: llc -march=mipsel -relocation-model=pic -O0 -mips-fast-isel -fast-isel-abort=1 -mcpu=mips32r2 \
-; RUN:     < %s | FileCheck %s -check-prefix=CHECK2
-; RUN: llc -march=mipsel -relocation-model=pic -O0 -mips-fast-isel -fast-isel-abort=1 -mcpu=mips32 \
-; RUN:     < %s | FileCheck %s -check-prefix=CHECK2
+; RUN: llc -march=mipsel -mcpu=mips32 -O0 \
+; RUN:     -relocation-model=pic -fast-isel-abort=1 < %s | \
+; RUN:     FileCheck %s -check-prefix=ALL -check-prefix=32R1
+; RUN: llc -march=mipsel -mcpu=mips32r2 -O0 \
+; RUN:     -relocation-model=pic -fast-isel-abort=1 < %s | \
+; RUN:     FileCheck %s -check-prefix=ALL -check-prefix=32R2
 
+declare void @xb(i8)
 
-@c1 = global i8 -45, align 1
-@uc1 = global i8 27, align 1
-@s1 = global i16 -1789, align 2
-@us1 = global i16 1256, align 2
+define void @cxb() {
+  ; ALL-LABEL:    cxb:
 
-; Function Attrs: nounwind
-define void @cxi() #0 {
-entry:
-; CHECK-LABEL:  cxi
-  call void @xi(i32 10)
-; CHECK-DAG:    addiu   $4, $zero, 10
-; CHECK-DAG:    lw      $25, %got(xi)(${{[0-9]+}})
-; CHECK:        jalr    $25
+  ; ALL:            addiu   $[[T0:[0-9]+]], $zero, 10
 
+  ; 32R1:           sll     $[[T1:[0-9]+]], $[[T0]], 24
+  ; 32R1:           sra     $4, $[[T1]], 24
+
+  ; 32R2:           seb     $4, $[[T0]]
+  call void @xb(i8 10)
   ret void
 }
 
-declare void @xi(i32) #1
+declare void @xh(i16)
 
-; Function Attrs: nounwind
-define void @cxii() #0 {
-entry:
-; CHECK-LABEL:  cxii
-  call void @xii(i32 746, i32 892)
-; CHECK-DAG:    addiu   $4, $zero, 746
-; CHECK-DAG:    addiu   $5, $zero, 892
-; CHECK-DAG:    lw      $25, %got(xii)(${{[0-9]+}})
-; CHECK:        jalr    $25
+define void @cxh() {
+  ; ALL-LABEL:    cxh:
+
+  ; ALL:            addiu   $[[T0:[0-9]+]], $zero, 10
 
+  ; 32R1:           sll     $[[T1:[0-9]+]], $[[T0]], 16
+  ; 32R1:           sra     $4, $[[T1]], 16
+
+  ; 32R2:           seh     $4, $[[T0]]
+  call void @xh(i16 10)
   ret void
 }
 
-declare void @xii(i32, i32) #1
+declare void @xi(i32)
 
-; Function Attrs: nounwind
-define void @cxiii() #0 {
-entry:
-; CHECK-LABEL:  cxiii
-  call void @xiii(i32 88, i32 44, i32 11)
-; CHECK-DAG:    addiu   $4, $zero, 88
-; CHECK-DAG:    addiu   $5, $zero, 44
-; CHECK-DAG:    addiu   $6, $zero, 11
-; CHECK-DAG:    lw      $25, %got(xiii)(${{[0-9]+}})
-; CHECK:        jalr    $25
+define void @cxi() {
+  ; ALL-LABEL:    cxi:
+
+  ; ALL-DAG:        addiu   $4, $zero, 10
+  ; ALL-DAG:        lw      $25, %got(xi)(${{[0-9]+}})
+  ; ALL:            jalr    $25
+  call void @xi(i32 10)
   ret void
 }
 
-declare void @xiii(i32, i32, i32) #1
+declare void @xbb(i8, i8)
 
-; Function Attrs: nounwind
-define void @cxiiii() #0 {
-entry:
-; CHECK-LABEL:  cxiiii
-  call void @xiiii(i32 167, i32 320, i32 97, i32 14)
-; CHECK-DAG:    addiu   $4, $zero, 167
-; CHECK-DAG:    addiu   $5, $zero, 320
-; CHECK-DAG:    addiu   $6, $zero, 97
-; CHECK-DAG:    addiu   $7, $zero, 14
-; CHECK-DAG:    lw      $25, %got(xiiii)(${{[0-9]+}})
-; CHECK:        jalr    $25
+define void @cxbb() {
+  ; ALL-LABEL:    cxbb:
+
+  ; ALL-DAG:        addiu   $[[T0:[0-9]+]], $zero, 76
+  ; ALL-DAG:        addiu   $[[T1:[0-9]+]], $zero, 101
+
+  ; 32R1-DAG:       sll     $[[T2:[0-9]+]], $[[T0]], 24
+  ; 32R1-DAG:       sra     $[[T3:[0-9]+]], $[[T2]], 24
+  ; 32R1-DAG:       sll     $[[T4:[0-9]+]], $[[T1]], 24
+  ; 32R1-DAG:       sra     $[[T5:[0-9]+]], $[[T4]], 24
 
+  ; 32R2-DAG:       seb     $4, $[[T0]]
+  ; 32R2-DAG:       seb     $5, $[[T1]]
+  call void @xbb(i8 76, i8 101)
   ret void
 }
 
-declare void @xiiii(i32, i32, i32, i32) #1
-
-; Function Attrs: nounwind
-define void @cxiiiiconv() #0 {
-entry:
-; CHECK-LABEL: cxiiiiconv
-; mips32r2-LABEL:  cxiiiiconv
-; mips32-LABEL:  cxiiiiconv
-  %0 = load i8, i8* @c1, align 1
-  %conv = sext i8 %0 to i32
-  %1 = load i8, i8* @uc1, align 1
-  %conv1 = zext i8 %1 to i32
-  %2 = load i16, i16* @s1, align 2
-  %conv2 = sext i16 %2 to i32
-  %3 = load i16, i16* @us1, align 2
-  %conv3 = zext i16 %3 to i32
-  call void @xiiii(i32 %conv, i32 %conv1, i32 %conv2, i32 %conv3)
-; CHECK:        addu    $[[REG_GP:[0-9]+]], ${{[0-9]+}}, ${{[0-9+]}}
-; mips32r2:     addu    $[[REG_GP:[0-9]+]], ${{[0-9]+}}, ${{[0-9+]}}
-; mips32:       addu    $[[REG_GP:[0-9]+]], ${{[0-9]+}}, ${{[0-9+]}}
-; mips32r2-DAG:         lw      $[[REG_C1_ADDR:[0-9]+]], %got(c1)($[[REG_GP]])
-; mips32r2-DAG: lbu     $[[REG_C1:[0-9]+]], 0($[[REG_C1_ADDR]])
-; mips32r2-DAG  seb     $3, $[[REG_C1]]
-; mips32-DAG:   lw      $[[REG_C1_ADDR:[0-9]+]], %got(c1)($[[REG_GP]])
-; mips32-DAG:   lbu     $[[REG_C1:[0-9]+]], 0($[[REG_C1_ADDR]])
-; mips32-DAG:   sll     $[[REG_C1_1:[0-9]+]], $[[REG_C1]], 24
-; mips32-DAG:   sra     $4, $[[REG_C1_1]], 24
-; CHECK-DAG:    lw      $[[REG_UC1_ADDR:[0-9]+]], %got(uc1)($[[REG_GP]])
-; CHECK-DAG:    lbu     $[[REG_UC1:[0-9]+]], 0($[[REG_UC1_ADDR]])
-; FIXME andi is superfulous
-; CHECK-DAG:    andi    $5, $[[REG_UC1]], 255
-; mips32r2-DAG:         lw      $[[REG_S1_ADDR:[0-9]+]], %got(s1)($[[REG_GP]])
-; mips32r2-DAG: lhu     $[[REG_S1:[0-9]+]], 0($[[REG_S1_ADDR]])
-; mips32r2-DAG: seh     $6, $[[REG_S1]]
-; mips32-DAG:   lw      $[[REG_S1_ADDR:[0-9]+]], %got(s1)($[[REG_GP]])
-; mips32-DAG:   lhu     $[[REG_S1:[0-9]+]], 0($[[REG_S1_ADDR]])
-; mips32-DAG:   sll     $[[REG_S1_1:[0-9]+]], $[[REG_S1]], 16
-; mips32-DAG:   sra     $6, $[[REG_S1_1]], 16
-; CHECK-DAG:    lw      $[[REG_US1_ADDR:[0-9]+]], %got(us1)($[[REG_GP]])
-; CHECK-DAG:    lhu     $[[REG_US1:[0-9]+]], 0($[[REG_US1_ADDR]])
-; FIXME andi is superfulous
-; CHECK-DAG:    andi    $7, $[[REG_US1]], 65535
-; mips32r2:     jalr    $25
-; mips32r2:     jalr    $25
-; CHECK:        jalr    $25
+declare void @xhh(i16, i16)
+
+define void @cxhh() {
+  ; ALL-LABEL:    cxhh:
+
+  ; ALL-DAG:        addiu   $[[T0:[0-9]+]], $zero, 76
+  ; ALL-DAG:        addiu   $[[T1:[0-9]+]], $zero, 101
+
+  ; 32R1-DAG:       sll     $[[T2:[0-9]+]], $[[T0]], 16
+  ; 32R1-DAG:       sra     $[[T3:[0-9]+]], $[[T2]], 16
+  ; 32R1-DAG:       sll     $[[T4:[0-9]+]], $[[T1]], 16
+  ; 32R1-DAG:       sra     $[[T5:[0-9]+]], $[[T4]], 16
+
+  ; 32R2-DAG:       seh     $4, $[[T0]]
+  ; 32R2-DAG:       seh     $5, $[[T1]]
+  call void @xhh(i16 76, i16 101)
   ret void
 }
 
-; Function Attrs: nounwind
-define void @cxf() #0 {
-entry:
-; CHECK-LABEL:  cxf
-  call void @xf(float 0x40BBC85560000000)
-; CHECK:        addu    $[[REG_GP:[0-9]+]], ${{[0-9]+}}, ${{[0-9+]}}
-; CHECK:        lui     $[[REG_FPCONST_1:[0-9]+]], 17886
-; CHECK:        ori     $[[REG_FPCONST:[0-9]+]], $[[REG_FPCONST_1]], 17067
-; CHECK: mtc1   $[[REG_FPCONST]], $f12
-; CHECK:        lw      $25, %got(xf)($[[REG_GP]])
-; CHECK:        jalr    $25
+declare void @xii(i32, i32)
+
+define void @cxii() {
+  ; ALL-LABEL:    cxii:
+
+  ; ALL-DAG:        addiu   $4, $zero, 746
+  ; ALL-DAG:        addiu   $5, $zero, 892
+  ; ALL-DAG:        lw      $25, %got(xii)(${{[0-9]+}})
+  ; ALL:            jalr    $25
+  call void @xii(i32 746, i32 892)
   ret void
 }
 
-declare void @xf(float) #1
+declare void @xccc(i8, i8, i8)
 
-; Function Attrs: nounwind
-define void @cxff() #0 {
-entry:
-; CHECK-LABEL:  cxff
-  call void @xff(float 0x3FF74A6CA0000000, float 0x401A2C0840000000)
-; CHECK:        addu    $[[REG_GP:[0-9]+]], ${{[0-9]+}}, ${{[0-9+]}}
-; CHECK-DAG:    lui     $[[REG_FPCONST_1:[0-9]+]], 16314
-; CHECK-DAG:    ori     $[[REG_FPCONST:[0-9]+]], $[[REG_FPCONST_1]], 21349
-; CHECK-DAG: mtc1       $[[REG_FPCONST]], $f12
-; CHECK-DAG:    lui     $[[REG_FPCONST_2:[0-9]+]], 16593
-; CHECK-DAG:    ori     $[[REG_FPCONST_3:[0-9]+]], $[[REG_FPCONST_2]], 24642
-; CHECK-DAG: mtc1       $[[REG_FPCONST_3]], $f14
-; CHECK:        lw      $25, %got(xff)($[[REG_GP]])
-; CHECK:        jalr    $25
+define void @cxccc() {
+  ; ALL-LABEL:    cxccc:
+
+  ; ALL-DAG:        addiu   $[[T0:[0-9]+]], $zero, 88
+  ; ALL-DAG:        addiu   $[[T1:[0-9]+]], $zero, 44
+  ; ALL-DAG:        addiu   $[[T2:[0-9]+]], $zero, 11
+
+  ; 32R1-DAG:       sll     $[[T3:[0-9]+]], $[[T0]], 24
+  ; 32R1-DAG:       sra     $4, $[[T3]], 24
+  ; 32R1-DAG:       sll     $[[T4:[0-9]+]], $[[T1]], 24
+  ; 32R1-DAG:       sra     $5, $[[T4]], 24
+  ; 32R1-DAG:       sll     $[[T5:[0-9]+]], $[[T2]], 24
+  ; 32R1-DAG:       sra     $6, $[[T5]], 24
+
+  ; 32R2-DAG:       seb     $4, $[[T0]]
+  ; 32R2-DAG:       seb     $5, $[[T1]]
+  ; 32R2-DAG:       seb     $6, $[[T2]]
+  call void @xccc(i8 88, i8 44, i8 11)
   ret void
 }
 
-declare void @xff(float, float) #1
+declare void @xhhh(i16, i16, i16)
 
-; Function Attrs: nounwind
-define void @cxfi() #0 {
-entry:
-; CHECK-LABEL: cxfi
-  call void @xfi(float 0x4013906240000000, i32 102)
-; CHECK:        addu    $[[REG_GP:[0-9]+]], ${{[0-9]+}}, ${{[0-9+]}}
-; CHECK-DAG:    lui     $[[REG_FPCONST_1:[0-9]+]], 16540
-; CHECK-DAG:    ori     $[[REG_FPCONST:[0-9]+]], $[[REG_FPCONST_1]], 33554
-; CHECK-DAG: mtc1       $[[REG_FPCONST]], $f12
-; CHECK-DAG:    addiu   $5, $zero, 102
-; CHECK:        lw      $25, %got(xfi)($[[REG_GP]])
-; CHECK:        jalr    $25
+define void @cxhhh() {
+  ; ALL-LABEL:    cxhhh:
+
+  ; ALL-DAG:        addiu   $[[T0:[0-9]+]], $zero, 88
+  ; ALL-DAG:        addiu   $[[T1:[0-9]+]], $zero, 44
+  ; ALL-DAG:        addiu   $[[T2:[0-9]+]], $zero, 11
 
+  ; 32R1-DAG:       sll     $[[T3:[0-9]+]], $[[T0]], 16
+  ; 32R1-DAG:       sra     $4, $[[T3]], 16
+  ; 32R1-DAG:       sll     $[[T4:[0-9]+]], $[[T1]], 16
+  ; 32R1-DAG:       sra     $5, $[[T4]], 16
+  ; 32R1-DAG:       sll     $[[T5:[0-9]+]], $[[T2]], 16
+  ; 32R1-DAG:       sra     $6, $[[T5]], 16
+
+  ; 32R2-DAG:       seh     $4, $[[T0]]
+  ; 32R2-DAG:       seh     $5, $[[T1]]
+  ; 32R2-DAG:       seh     $6, $[[T2]]
+  call void @xhhh(i16 88, i16 44, i16 11)
   ret void
 }
 
-declare void @xfi(float, i32) #1
+declare void @xiii(i32, i32, i32)
 
-; Function Attrs: nounwind
-define void @cxfii() #0 {
-entry:
-; CHECK-LABEL: cxfii
-  call void @xfii(float 0x405EC7EE00000000, i32 9993, i32 10922)
-; CHECK:        addu    $[[REG_GP:[0-9]+]], ${{[0-9]+}}, ${{[0-9+]}}
-; CHECK-DAG:    lui     $[[REG_FPCONST_1:[0-9]+]], 17142
-; CHECK-DAG:    ori     $[[REG_FPCONST:[0-9]+]], $[[REG_FPCONST_1]], 16240
-; CHECK-DAG: mtc1       $[[REG_FPCONST]], $f12
-; CHECK-DAG:    addiu   $5, $zero, 9993
-; CHECK-DAG:    addiu   $6, $zero, 10922
-; CHECK:        lw      $25, %got(xfii)($[[REG_GP]])
-; CHECK:        jalr    $25
+define void @cxiii() {
+  ; ALL-LABEL:    cxiii:
+
+  ; ALL-DAG:        addiu   $4, $zero, 88
+  ; ALL-DAG:        addiu   $5, $zero, 44
+  ; ALL-DAG:        addiu   $6, $zero, 11
+  ; ALL-DAG:        lw      $25, %got(xiii)(${{[0-9]+}})
+  ; ALL:            jalr    $25
+  call void @xiii(i32 88, i32 44, i32 11)
   ret void
 }
 
-declare void @xfii(float, i32, i32) #1
+declare void @xcccc(i8, i8, i8, i8)
+
+define void @cxcccc() {
+  ; ALL-LABEL:    cxcccc:
+
+  ; ALL-DAG:        addiu   $[[T0:[0-9]+]], $zero, 88
+  ; ALL-DAG:        addiu   $[[T1:[0-9]+]], $zero, 44
+  ; ALL-DAG:        addiu   $[[T2:[0-9]+]], $zero, 11
+  ; ALL-DAG:        addiu   $[[T3:[0-9]+]], $zero, 33
+
+  ; FIXME: We should avoid the unnecessary spill/reload here.
+
+  ; 32R1-DAG:       sll     $[[T4:[0-9]+]], $[[T0]], 24
+  ; 32R1-DAG:       sra     $[[T5:[0-9]+]], $[[T4]], 24
+  ; 32R1-DAG:       sw      $4, 16($sp)
+  ; 32R1-DAG:       move    $4, $[[T5]]
+  ; 32R1-DAG:       sll     $[[T6:[0-9]+]], $[[T1]], 24
+  ; 32R1-DAG:       sra     $5, $[[T6]], 24
+  ; 32R1-DAG:       sll     $[[T7:[0-9]+]], $[[T2]], 24
+  ; 32R1-DAG:       sra     $6, $[[T7]], 24
+  ; 32R1:           lw      $[[T8:[0-9]+]], 16($sp)
+  ; 32R1:           sll     $[[T9:[0-9]+]], $[[T8]], 24
+  ; 32R1:           sra     $7, $[[T9]], 24
+
+  ; 32R2-DAG:       seb     $[[T4:[0-9]+]], $[[T0]]
+  ; 32R2-DAG:       sw      $4, 16($sp)
+  ; 32R2-DAG:       move    $4, $[[T4]]
+  ; 32R2-DAG:       seb     $5, $[[T1]]
+  ; 32R2-DAG:       seb     $6, $[[T2]]
+  ; 32R2-DAG:       lw      $[[T5:[0-9]+]], 16($sp)
+  ; 32R2:           seb     $7, $[[T5]]
+  call void @xcccc(i8 88, i8 44, i8 11, i8 33)
+  ret void
+}
 
-; Function Attrs: nounwind
-define void @cxfiii() #0 {
-entry:
-; CHECK-LABEL: cxfiii
-  call void @xfiii(float 0x405C072B20000000, i32 3948, i32 89011, i32 111222)
-; CHECK:        addu    $[[REG_GP:[0-9]+]], ${{[0-9]+}}, ${{[0-9+]}}
-; CHECK-DAG:    lui     $[[REG_FPCONST_1:[0-9]+]], 17120
-; CHECK-DAG:    ori     $[[REG_FPCONST:[0-9]+]], $[[REG_FPCONST_1]], 14681
-; CHECK-DAG: mtc1       $[[REG_FPCONST]], $f12
-; CHECK-DAG:    addiu   $5, $zero, 3948
-; CHECK-DAG:    lui     $[[REG_I_1:[0-9]+]], 1
-; CHECK-DAG:    ori     $6, $[[REG_I_1]], 23475
-; CHECK-DAG:    lui     $[[REG_I_2:[0-9]+]], 1
-; CHECK-DAG:    ori     $7, $[[REG_I_2]], 45686
-; CHECK:        lw      $25, %got(xfiii)($[[REG_GP]])
-; CHECK:        jalr    $25
+declare void @xhhhh(i16, i16, i16, i16)
+
+define void @cxhhhh() {
+  ; ALL-LABEL:    cxhhhh:
+
+  ; ALL-DAG:        addiu   $[[T0:[0-9]+]], $zero, 88
+  ; ALL-DAG:        addiu   $[[T1:[0-9]+]], $zero, 44
+  ; ALL-DAG:        addiu   $[[T2:[0-9]+]], $zero, 11
+  ; ALL-DAG:        addiu   $[[T3:[0-9]+]], $zero, 33
+
+  ; FIXME: We should avoid the unnecessary spill/reload here.
+
+  ; 32R1-DAG:       sll     $[[T4:[0-9]+]], $[[T0]], 16
+  ; 32R1-DAG:       sra     $[[T5:[0-9]+]], $[[T4]], 16
+  ; 32R1-DAG:       sw      $4, 16($sp)
+  ; 32R1-DAG:       move    $4, $[[T5]]
+  ; 32R1-DAG:       sll     $[[T6:[0-9]+]], $[[T1]], 16
+  ; 32R1-DAG:       sra     $5, $[[T6]], 16
+  ; 32R1-DAG:       sll     $[[T7:[0-9]+]], $[[T2]], 16
+  ; 32R1-DAG:       sra     $6, $[[T7]], 16
+  ; 32R1:           lw      $[[T8:[0-9]+]], 16($sp)
+  ; 32R1:           sll     $[[T9:[0-9]+]], $[[T8]], 16
+  ; 32R1:           sra     $7, $[[T9]], 16
+
+  ; 32R2-DAG:       seh     $[[T4:[0-9]+]], $[[T0]]
+  ; 32R2-DAG:       sw      $4, 16($sp)
+  ; 32R2-DAG:       move    $4, $[[T4]]
+  ; 32R2-DAG:       seh     $5, $[[T1]]
+  ; 32R2-DAG:       seh     $6, $[[T2]]
+  ; 32R2-DAG:       lw      $[[T5:[0-9]+]], 16($sp)
+  ; 32R2:           seh     $7, $[[T5]]
+  call void @xhhhh(i16 88, i16 44, i16 11, i16 33)
   ret void
 }
 
-declare void @xfiii(float, i32, i32, i32) #1
+declare void @xiiii(i32, i32, i32, i32)
 
-; Function Attrs: nounwind
-define void @cxd() #0 {
-entry:
-; mips32r2-LABEL: cxd:
-; mips32-LABEL: cxd:
-  call void @xd(double 5.994560e+02)
-; mips32:       addu    $[[REG_GP:[0-9]+]], ${{[0-9]+}}, ${{[0-9+]}}
-; mips32-DAG:   lui     $[[REG_FPCONST_1:[0-9]+]], 16514
-; mips32-DAG:   ori     $[[REG_FPCONST_2:[0-9]+]], $[[REG_FPCONST_1]], 48037
-; mips32-DAG:   lui     $[[REG_FPCONST_3:[0-9]+]], 58195
-; mips32-DAG:   ori     $[[REG_FPCONST_4:[0-9]+]], $[[REG_FPCONST_3]], 63439
-; mips32-DAG:    mtc1   $[[REG_FPCONST_4]], $f12
-; mips32-DAG:       mtc1        $[[REG_FPCONST_2]], $f13
-; mips32-DAG:   lw      $25, %got(xd)($[[REG_GP]])
-; mips32:       jalr    $25
-; mips32r2:     addu    $[[REG_GP:[0-9]+]], ${{[0-9]+}}, ${{[0-9+]}}
-; mips32r2-DAG: lui     $[[REG_FPCONST_1:[0-9]+]], 16514
-; mips32r2-DAG: ori     $[[REG_FPCONST_2:[0-9]+]], $[[REG_FPCONST_1]], 48037
-; mips32r2-DAG: lui     $[[REG_FPCONST_3:[0-9]+]], 58195
-; mips32r2-DAG: ori     $[[REG_FPCONST_4:[0-9]+]], $[[REG_FPCONST_3]], 63439
-; mips32r2-DAG: mtc1    $[[REG_FPCONST_4]], $f12
-; mips32r2-DAG: mthc1   $[[REG_FPCONST_2]], $f12
-; mips32r2-DAG: lw      $25, %got(xd)($[[REG_GP]])
-; mips32r2 :    jalr    $25
+define void @cxiiii() {
+  ; ALL-LABEL:    cxiiii:
+
+  ; ALL-DAG:        addiu   $4, $zero, 167
+  ; ALL-DAG:        addiu   $5, $zero, 320
+  ; ALL-DAG:        addiu   $6, $zero, 97
+  ; ALL-DAG:        addiu   $7, $zero, 14
+  ; ALL-DAG:        lw      $25, %got(xiiii)(${{[0-9]+}})
+  ; ALL:            jalr    $25
+  call void @xiiii(i32 167, i32 320, i32 97, i32 14)
   ret void
 }
 
-declare void @xd(double) #1
+@c1 = global i8 -45, align 1
+@uc1 = global i8 27, align 1
+@s1 = global i16 -1789, align 2
+@us1 = global i16 1256, align 2
 
-; Function Attrs: nounwind
-define void @cxdd() #0 {
-; mips32r2-LABEL: cxdd:
-; mips32-LABEL: cxdd:
-entry:
-  call void @xdd(double 1.234980e+03, double 0x40F5B331F7CED917)
-; mips32:       addu    $[[REG_GP:[0-9]+]], ${{[0-9]+}}, ${{[0-9+]}}
-; mips32-DAG:   lui     $[[REG_FPCONST_1:[0-9]+]], 16531
-; mips32-DAG:   ori     $[[REG_FPCONST_2:[0-9]+]], $[[REG_FPCONST_1]], 19435
-; mips32-DAG:   lui     $[[REG_FPCONST_3:[0-9]+]], 34078
-; mips32-DAG:   ori     $[[REG_FPCONST_4:[0-9]+]], $[[REG_FPCONST_3]], 47186
-; mips32-DAG:   mtc1    $[[REG_FPCONST_4]], $f12
-; mips32-DAG:   mtc1    $[[REG_FPCONST_2]], $f13
-; mips32-DAG:   lui     $[[REG_FPCONST_1:[0-9]+]], 16629
-; mips32-DAG:   ori     $[[REG_FPCONST_2:[0-9]+]], $[[REG_FPCONST_1]], 45873
-; mips32-DAG:   lui     $[[REG_FPCONST_3:[0-9]+]], 63438
-; mips32-DAG:   ori     $[[REG_FPCONST_4:[0-9]+]], $[[REG_FPCONST_3]], 55575
-; mips32-DAG:   mtc1    $[[REG_FPCONST_4]], $f14
-; mips32-DAG:   mtc1    $[[REG_FPCONST_2]], $f15
-; mips32-DAG:   lw      $25, %got(xdd)($[[REG_GP]])
-; mips32:       jalr    $25
-; mips32r2:     addu    $[[REG_GP:[0-9]+]], ${{[0-9]+}}, ${{[0-9+]}}
-; mips32r2-DAG: lui     $[[REG_FPCONST_1:[0-9]+]], 16531
-; mips32r2-DAG: ori     $[[REG_FPCONST_2:[0-9]+]], $[[REG_FPCONST_1]], 19435
-; mips32r2-DAG: lui     $[[REG_FPCONST_3:[0-9]+]], 34078
-; mips32r2-DAG: ori     $[[REG_FPCONST_4:[0-9]+]], $[[REG_FPCONST_3]], 47186
-; mips32r2-DAG: mtc1    $[[REG_FPCONST_4]], $f12
-; mips32r2-DAG: mthc1   $[[REG_FPCONST_2]], $f12
-; mips32r2-DAG: lui     $[[REG_FPCONST_1:[0-9]+]], 16629
-; mips32r2-DAG: ori     $[[REG_FPCONST_2:[0-9]+]], $[[REG_FPCONST_1]], 45873
-; mips32r2-DAG: lui     $[[REG_FPCONST_3:[0-9]+]], 63438
-; mips32r2-DAG: ori     $[[REG_FPCONST_4:[0-9]+]], $[[REG_FPCONST_3]], 55575
-; mips32r2-DAG: mtc1    $[[REG_FPCONST_4]], $f14
-; mips32r2-DAG: mthc1   $[[REG_FPCONST_2]], $f14
-; mips32r2-DAG: lw      $25, %got(xdd)($[[REG_GP]])
-; mips32r2 :    jalr    $25
+define void @cxiiiiconv() {
+  ; ALL-LABEL:    cxiiiiconv:
+
+  ; ALL:            addu    $[[REG_GP:[0-9]+]], ${{[0-9]+}}, ${{[0-9+]}}
+  ; ALL-DAG:        lw      $[[REG_C1_ADDR:[0-9]+]], %got(c1)($[[REG_GP]])
+  ; ALL-DAG:        lbu     $[[REG_C1:[0-9]+]], 0($[[REG_C1_ADDR]])
+  ; 32R1-DAG:       sll     $[[REG_C1_1:[0-9]+]], $[[REG_C1]], 24
+  ; 32R1-DAG:       sra     $4, $[[REG_C1_1]], 24
+  ; 32R2-DAG:       seb     $4, $[[REG_C1]]
+  ; FIXME: andi is superfulous
+  ; ALL-DAG:        lw      $[[REG_UC1_ADDR:[0-9]+]], %got(uc1)($[[REG_GP]])
+  ; ALL-DAG:        lbu     $[[REG_UC1:[0-9]+]], 0($[[REG_UC1_ADDR]])
+  ; ALL-DAG:        andi    $5, $[[REG_UC1]], 255
+  ; ALL-DAG:        lw      $[[REG_S1_ADDR:[0-9]+]], %got(s1)($[[REG_GP]])
+  ; ALL-DAG:        lhu     $[[REG_S1:[0-9]+]], 0($[[REG_S1_ADDR]])
+  ; 32R1-DAG:       sll     $[[REG_S1_1:[0-9]+]], $[[REG_S1]], 16
+  ; 32R1-DAG:       sra     $6, $[[REG_S1_1]], 16
+  ; 32R2-DAG:       seh     $6, $[[REG_S1]]
+  ; FIXME andi is superfulous
+  ; ALL-DAG:        lw      $[[REG_US1_ADDR:[0-9]+]], %got(us1)($[[REG_GP]])
+  ; ALL-DAG:        lhu     $[[REG_US1:[0-9]+]], 0($[[REG_US1_ADDR]])
+  ; ALL-DAG:        andi    $7, $[[REG_US1]], 65535
+  ; ALL:            jalr    $25
+  %1 = load i8, i8* @c1, align 1
+  %conv = sext i8 %1 to i32
+  %2 = load i8, i8* @uc1, align 1
+  %conv1 = zext i8 %2 to i32
+  %3 = load i16, i16* @s1, align 2
+  %conv2 = sext i16 %3 to i32
+  %4 = load i16, i16* @us1, align 2
+  %conv3 = zext i16 %4 to i32
+  call void @xiiii(i32 %conv, i32 %conv1, i32 %conv2, i32 %conv3)
   ret void
 }
 
-declare void @xdd(double, double) #1
+declare void @xf(float)
 
-; Function Attrs: nounwind
-define void @cxif() #0 {
-entry:
-; CHECK-LABEL: cxif:
-  call void @xif(i32 345, float 0x407BCE5A20000000)
-; CHECK-DAG:    addu    $[[REG_GP:[0-9]+]], ${{[0-9]+}}, ${{[0-9+]}}
-; CHECK-DAG:    addiu   $4, $zero, 345
-; CHECK-DAG:    lui     $[[REGF_1:[0-9]+]], 17374
-; CHECK-DAG:    ori     $[[REGF_2:[0-9]+]], $[[REGF_1]], 29393
-; CHECK-DAG:    mtc1    $[[REGF_2]], $f[[REGF_3:[0-9]+]]
-; CHECK-DAG:    mfc1    $5, $f[[REGF_3]]
-; CHECK-DAG:    lw      $25, %got(xif)($[[REG_GP]])
-; CHECK:        jalr    $25
+define void @cxf() {
+  ; ALL-LABEL:    cxf:
 
+  ; ALL:            addu    $[[REG_GP:[0-9]+]], ${{[0-9]+}}, ${{[0-9+]}}
+  ; ALL:            lui     $[[REG_FPCONST_1:[0-9]+]], 17886
+  ; ALL:            ori     $[[REG_FPCONST:[0-9]+]], $[[REG_FPCONST_1]], 17067
+  ; ALL:            mtc1    $[[REG_FPCONST]], $f12
+  ; ALL:            lw      $25, %got(xf)($[[REG_GP]])
+  ; ALL:            jalr    $25
+  call void @xf(float 0x40BBC85560000000)
   ret void
 }
 
-declare void @xif(i32, float) #1
+declare void @xff(float, float)
 
-; Function Attrs: nounwind
-define void @cxiff() #0 {
-entry:
-; CHECK-LABEL: cxiff:
-; CHECK2-LABEL: cxiff:
-  call void @xiff(i32 12239, float 0x408EDB3340000000, float 0x4013FFE5C0000000)
-; We need to do the two floating point parameters in a separate
-; check because we can't control the ordering of parts of the sequence
-;;
-; CHECK:        addu    $[[REG_GP:[0-9]+]], ${{[0-9]+}}, ${{[0-9+]}}
-; CHECK:        addiu   $4, $zero, 12239
-; CHECK2:       addu    $[[REG_GP:[0-9]+]], ${{[0-9]+}}, ${{[0-9+]}}
-; CHECK2:       addiu   $4, $zero, 12239
-; CHECK:        lui     $[[REGF_1:[0-9]+]], 17526
-; CHECK:        ori     $[[REGF_2:[0-9]+]], $[[REGF_1]], 55706
-; CHECK:        mtc1    $[[REGF_2]], $f[[REGF_3:[0-9]+]]
-; CHECK:        mfc1    $5, $f[[REGF_3]]
-; CHECK2:       lui     $[[REGF2_1:[0-9]+]], 16543
-; CHECK2:       ori     $[[REGF2_2:[0-9]+]], $[[REGF2_1]], 65326
-; CHECK2:       mtc1    $[[REGF2_2]], $f[[REGF2_3:[0-9]+]]
-; CHECK2:       mfc1    $6, $f[[REGF2_3]]
-; CHECK:        lw      $25, %got(xiff)($[[REG_GP]])
-; CHECK2:       lw      $25, %got(xiff)($[[REG_GP]])
-; CHECK:        jalr    $25
-; CHECK2:       jalr    $25
+define void @cxff() {
+  ; ALL-LABEL:    cxff:
+
+  ; ALL:            addu    $[[REG_GP:[0-9]+]], ${{[0-9]+}}, ${{[0-9+]}}
+  ; ALL-DAG:        lui     $[[REG_FPCONST_1:[0-9]+]], 16314
+  ; ALL-DAG:        ori     $[[REG_FPCONST:[0-9]+]], $[[REG_FPCONST_1]], 21349
+  ; ALL-DAG:        mtc1    $[[REG_FPCONST]], $f12
+  ; ALL-DAG:        lui     $[[REG_FPCONST_2:[0-9]+]], 16593
+  ; ALL-DAG:        ori     $[[REG_FPCONST_3:[0-9]+]], $[[REG_FPCONST_2]], 24642
+  ; ALL-DAG:        mtc1    $[[REG_FPCONST_3]], $f14
+  ; ALL-DAG:        lw      $25, %got(xff)($[[REG_GP]])
+  ; ALL:            jalr    $25
+  call void @xff(float 0x3FF74A6CA0000000, float 0x401A2C0840000000)
   ret void
 }
 
-declare void @xiff(i32, float, float) #1
+declare void @xfi(float, i32)
 
-; Function Attrs: nounwind
-define void @cxifi() #0 {
-entry:
-; CHECK: cxifi:
-  call void @xifi(i32 887, float 0x402277CEE0000000, i32 888)
-; CHECK-DAG:    addu    $[[REG_GP:[0-9]+]], ${{[0-9]+}}, ${{[0-9+]}}
-; CHECK-DAG:    addiu   $4, $zero, 887
-; CHECK-DAG:    lui     $[[REGF_1:[0-9]+]], 16659
-; CHECK-DAG:    ori     $[[REGF_2:[0-9]+]], $[[REGF_1]], 48759
-; CHECK-DAG:    mtc1    $[[REGF_2]], $f[[REGF_3:[0-9]+]]
-; CHECK-DAG:    mfc1    $5, $f[[REGF_3]]
-; CHECk-DAG:    addiu   $6, $zero, 888
-; CHECK-DAG:    lw      $25, %got(xifi)($[[REG_GP]])
-; CHECK:        jalr    $25
+define void @cxfi() {
+  ; ALL-LABEL:    cxfi:
 
+  ; ALL:            addu    $[[REG_GP:[0-9]+]], ${{[0-9]+}}, ${{[0-9+]}}
+  ; ALL-DAG:        lui     $[[REG_FPCONST_1:[0-9]+]], 16540
+  ; ALL-DAG:        ori     $[[REG_FPCONST:[0-9]+]], $[[REG_FPCONST_1]], 33554
+  ; ALL-DAG:        mtc1    $[[REG_FPCONST]], $f12
+  ; ALL-DAG:        addiu   $5, $zero, 102
+  ; ALL-DAG:        lw      $25, %got(xfi)($[[REG_GP]])
+  ; ALL:            jalr    $25
+  call void @xfi(float 0x4013906240000000, i32 102)
   ret void
 }
 
-declare void @xifi(i32, float, i32) #1
-
-; Function Attrs: nounwind
-define void @cxifif() #0 {
-entry:
-; CHECK: cxifif:
-; CHECK2: cxifif:
-  call void @xifif(i32 67774, float 0x408EE0FBE0000000, i32 9991, float 0x40B15C8CC0000000)
-; CHECK-DAG:    addu    $[[REG_GP:[0-9]+]], ${{[0-9]+}}, ${{[0-9+]}}
-; CHECK-DAG:    addu    $[[REG_GP:[0-9]+]], ${{[0-9]+}}, ${{[0-9+]}}
-; CHECK-DAG:    lui     $[[REGI:[0-9]+]], 1
-; CHECK-DAG:    ori     $4, $[[REGI]], 2238
-; CHECK-DAG:    lui     $[[REGF_1:[0-9]+]], 17527
-; CHECK-DAG:    ori     $[[REGF_2:[0-9]+]], $[[REGF_1]], 2015
-; CHECK-DAG:    mtc1    $[[REGF_2]], $f[[REGF_3:[0-9]+]]
-; CHECK-DAG:    mfc1    $5, $f[[REGF_3]]
-; CHECk-DAG:    addiu   $6, $zero, 888
-; CHECK2:       lui     $[[REGF2_1:[0-9]+]], 17802
-; CHECK2:       ori     $[[REGF2_2:[0-9]+]], $[[REGF2_1]], 58470
-; CHECK2:       mtc1    $[[REGF2_2]], $f[[REGF2_3:[0-9]+]]
-; CHECK2:       mfc1    $7, $f[[REGF2_3]]
-; CHECK:        lw      $25, %got(xifif)($[[REG_GP]])
-; CHECK2:       lw      $25, %got(xifif)($[[REG_GP]])
-; CHECK2:       jalr    $25
-; CHECK:        jalr    $25
+declare void @xfii(float, i32, i32)
 
+define void @cxfii() {
+  ; ALL-LABEL:    cxfii:
+
+  ; ALL:            addu    $[[REG_GP:[0-9]+]], ${{[0-9]+}}, ${{[0-9+]}}
+  ; ALL-DAG:        lui     $[[REG_FPCONST_1:[0-9]+]], 17142
+  ; ALL-DAG:        ori     $[[REG_FPCONST:[0-9]+]], $[[REG_FPCONST_1]], 16240
+  ; ALL-DAG:        mtc1    $[[REG_FPCONST]], $f12
+  ; ALL-DAG:        addiu   $5, $zero, 9993
+  ; ALL-DAG:        addiu   $6, $zero, 10922
+  ; ALL-DAG:        lw      $25, %got(xfii)($[[REG_GP]])
+  ; ALL:            jalr    $25
+  call void @xfii(float 0x405EC7EE00000000, i32 9993, i32 10922)
   ret void
 }
 
-declare void @xifif(i32, float, i32, float) #1
-
-; Function Attrs: nounwind
-define void @cxiffi() #0 {
-entry:
-; CHECK-label: cxiffi:
-; CHECK2-label: cxiffi:
-  call void @xiffi(i32 45, float 0x3FF6666660000000, float 0x408F333340000000, i32 234)
-; CHECK-DAG:    addu    $[[REG_GP:[0-9]+]], ${{[0-9]+}}, ${{[0-9+]}}
-; CHECK-DAG:    addiu   $4, $zero, 45
-; CHECK2-DAG:   addu    $[[REG_GP:[0-9]+]], ${{[0-9]+}}, ${{[0-9+]}}
-; CHECK2-DAG:   addiu   $4, $zero, 45
-; CHECK-DAG:    lui     $[[REGF_1:[0-9]+]], 16307
-; CHECK-DAG:    ori     $[[REGF_2:[0-9]+]], $[[REGF_1]], 13107
-; CHECK-DAG:    mtc1    $[[REGF_2]], $f[[REGF_3:[0-9]+]]
-; CHECK-DAG:    mfc1    $5, $f[[REGF_3]]
-; CHECK2:       lui     $[[REGF2_1:[0-9]+]], 17529
-; CHECK2:       ori     $[[REGF2_2:[0-9]+]], $[[REGF2_1]], 39322
-; CHECK2:       mtc1    $[[REGF2_2]], $f[[REGF2_3:[0-9]+]]
-; CHECK2:       mfc1    $6, $f[[REGF2_3]]
-; CHECK-DAG:    lw      $25, %got(xiffi)($[[REG_GP]])
-; CHECK-DAG:    addiu   $7, $zero, 234
-; CHECK2-DAG:   lw      $25, %got(xiffi)($[[REG_GP]])
-; CHECK:        jalr    $25
-; CHECK2:       jalr    $25
+declare void @xfiii(float, i32, i32, i32)
+
+define void @cxfiii() {
+  ; ALL-LABEL:    cxfiii:
+
+  ; ALL:            addu    $[[REG_GP:[0-9]+]], ${{[0-9]+}}, ${{[0-9+]}}
+  ; ALL-DAG:        lui     $[[REG_FPCONST_1:[0-9]+]], 17120
+  ; ALL-DAG:        ori     $[[REG_FPCONST:[0-9]+]], $[[REG_FPCONST_1]], 14681
+  ; ALL-DAG:        mtc1    $[[REG_FPCONST]], $f12
+  ; ALL-DAG:        addiu   $5, $zero, 3948
+  ; ALL-DAG:        lui     $[[REG_I_1:[0-9]+]], 1
+  ; ALL-DAG:        ori     $6, $[[REG_I_1]], 23475
+  ; ALL-DAG:        lui     $[[REG_I_2:[0-9]+]], 1
+  ; ALL-DAG:        ori     $7, $[[REG_I_2]], 45686
+  ; ALL-DAG:        lw      $25, %got(xfiii)($[[REG_GP]])
+  ; ALL:            jalr    $25
+  call void @xfiii(float 0x405C072B20000000, i32 3948, i32 89011, i32 111222)
+  ret void
+}
 
+declare void @xd(double)
+
+define void @cxd() {
+  ; ALL-LABEL:    cxd:
+
+  ; ALL:            addu    $[[REG_GP:[0-9]+]], ${{[0-9]+}}, ${{[0-9+]}}
+  ; ALL-DAG:        lui     $[[REG_FPCONST_1:[0-9]+]], 16514
+  ; ALL-DAG:        ori     $[[REG_FPCONST_2:[0-9]+]], $[[REG_FPCONST_1]], 48037
+  ; ALL-DAG:        lui     $[[REG_FPCONST_3:[0-9]+]], 58195
+  ; ALL-DAG:        ori     $[[REG_FPCONST_4:[0-9]+]], $[[REG_FPCONST_3]], 63439
+  ; ALL-DAG:        mtc1    $[[REG_FPCONST_4]], $f12
+  ; 32R1-DAG:       mtc1    $[[REG_FPCONST_2]], $f13
+  ; 32R2-DAG:       mthc1   $[[REG_FPCONST_2]], $f12
+  ; ALL-DAG:        lw      $25, %got(xd)($[[REG_GP]])
+  ; ALL:            jalr    $25
+  call void @xd(double 5.994560e+02)
   ret void
 }
 
-declare void @xiffi(i32, float, float, i32) #1
+declare void @xdd(double, double)
+
+define void @cxdd() {
+  ; ALL-LABEL:    cxdd:
+
+  ; ALL:            addu    $[[REG_GP:[0-9]+]], ${{[0-9]+}}, ${{[0-9+]}}
+  ; ALL-DAG:        lui     $[[REG_FPCONST_1:[0-9]+]], 16531
+  ; ALL-DAG:        ori     $[[REG_FPCONST_2:[0-9]+]], $[[REG_FPCONST_1]], 19435
+  ; ALL-DAG:        lui     $[[REG_FPCONST_3:[0-9]+]], 34078
+  ; ALL-DAG:        ori     $[[REG_FPCONST_4:[0-9]+]], $[[REG_FPCONST_3]], 47186
+  ; ALL-DAG:        mtc1    $[[REG_FPCONST_4]], $f12
+  ; 32R1-DAG:       mtc1    $[[REG_FPCONST_2]], $f13
+  ; 32R2-DAG:       mthc1   $[[REG_FPCONST_2]], $f12
+  ; ALL-DAG:        lui     $[[REG_FPCONST_1:[0-9]+]], 16629
+  ; ALL-DAG:        ori     $[[REG_FPCONST_2:[0-9]+]], $[[REG_FPCONST_1]], 45873
+  ; ALL-DAG:        lui     $[[REG_FPCONST_3:[0-9]+]], 63438
+  ; ALL-DAG:        ori     $[[REG_FPCONST_4:[0-9]+]], $[[REG_FPCONST_3]], 55575
+  ; ALL-DAG:        mtc1    $[[REG_FPCONST_4]], $f14
+  ; 32R1-DAG:       mtc1    $[[REG_FPCONST_2]], $f15
+  ; 32R2-DAG:       mthc1   $[[REG_FPCONST_2]], $f14
+  ; ALL-DAG:        lw      $25, %got(xdd)($[[REG_GP]])
+  ; ALL:            jalr    $25
+  call void @xdd(double 1.234980e+03, double 0x40F5B331F7CED917)
+  ret void
+}
 
-; Function Attrs: nounwind
-define void @cxifii() #0 {
-entry:
-; CHECK-DAG:    cxifii:
-  call void @xifii(i32 12239, float 0x408EDB3340000000, i32 998877, i32 1234)
-; CHECK-DAG:    addu    $[[REG_GP:[0-9]+]], ${{[0-9]+}}, ${{[0-9+]}}
-; CHECK-DAG:    addiu   $4, $zero, 12239
-; CHECK-DAG:    lui     $[[REGF_1:[0-9]+]], 17526
-; CHECK-DAG:    ori     $[[REGF_2:[0-9]+]], $[[REGF_1]], 55706
-; CHECK-DAG:    mtc1    $[[REGF_2]], $f[[REGF_3:[0-9]+]]
-; CHECK-DAG:    mfc1    $5, $f[[REGF_3]]
-; CHECK-DAG:    lui     $[[REGI2:[0-9]+]], 15
-; CHECK-DAG:    ori     $6, $[[REGI2]], 15837
-; CHECk-DAG:    addiu   $7, $zero, 1234
-; CHECK-DAG:    lw      $25, %got(xifii)($[[REG_GP]])
-; CHECK:        jalr    $25
+declare void @xif(i32, float)
+
+define void @cxif() {
+  ; ALL-LABEL:    cxif:
+
+  ; ALL:            addu    $[[REG_GP:[0-9]+]], ${{[0-9]+}}, ${{[0-9+]}}
+  ; ALL-DAG:        addiu   $4, $zero, 345
+  ; ALL-DAG:        lui     $[[REGF_1:[0-9]+]], 17374
+  ; ALL-DAG:        ori     $[[REGF_2:[0-9]+]], $[[REGF_1]], 29393
+  ; ALL-DAG:        mtc1    $[[REGF_2]], $f[[REGF_3:[0-9]+]]
+  ; ALL-DAG:        mfc1    $5, $f[[REGF_3]]
+  ; ALL-DAG:        lw      $25, %got(xif)($[[REG_GP]])
+  ; ALL:            jalr    $25
+  call void @xif(i32 345, float 0x407BCE5A20000000)
   ret void
 }
 
-declare void @xifii(i32, float, i32, i32) #1
-
-; FIXME: this function will not pass yet. 
-; Function Attrs: nounwind
-; define void @cxfid() #0 {
-;entry:
-;  call void @xfid(float 0x4013B851E0000000, i32 811123, double 0x40934BFF487FCB92)
-;  ret void
-;}
-
-declare void @xfid(float, i32, double) #1
-
-; Function Attrs: nounwind
-define void @g() #0 {
-entry:
-  call void @cxi()
-  call void @cxii()
-  call void @cxiii()
-  call void @cxiiii()
-  call void @cxiiiiconv()
-  call void @cxf()
-  call void @cxff()
-  call void @cxd()
-  call void @cxfi()
-  call void @cxfii()
-  call void @cxfiii()
-  call void @cxdd()
-  call void @cxif()
-  call void @cxiff()
-  call void @cxifi()
-  call void @cxifii()
-  call void @cxifif()
-  call void @cxiffi()
+declare void @xiff(i32, float, float)
+
+define void @cxiff() {
+  ; ALL-LABEL:    cxiff:
+
+  ; ALL:            addu    $[[REG_GP:[0-9]+]], ${{[0-9]+}}, ${{[0-9+]}}
+  ; ALL-DAG:        addiu   $4, $zero, 12239
+  ; ALL-DAG:        lui     $[[REGF0_1:[0-9]+]], 17526
+  ; ALL-DAG:        ori     $[[REGF0_2:[0-9]+]], $[[REGF0_1]], 55706
+  ; ALL-DAG:        mtc1    $[[REGF0_2]], $f[[REGF0_3:[0-9]+]]
+  ; ALL-DAG:        lui     $[[REGF1_1:[0-9]+]], 16543
+  ; ALL-DAG:        ori     $[[REGF1_2:[0-9]+]], $[[REGF1_1]], 65326
+  ; ALL:            mtc1    $[[REGF1_2]], $f[[REGF1_3:[0-9]+]]
+  ; ALL-DAG:        mfc1    $5, $f[[REGF0_3]]
+  ; ALL-DAG:        mfc1    $6, $f[[REGF1_3]]
+  ; ALL-DAG:        lw      $25, %got(xiff)($[[REG_GP]])
+  ; ALL:            jalr    $25
+  call void @xiff(i32 12239, float 0x408EDB3340000000, float 0x4013FFE5C0000000)
   ret void
 }
 
+declare void @xifi(i32, float, i32)
 
-attributes #0 = { nounwind "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
-attributes #1 = { "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+define void @cxifi() {
+  ; ALL-LABEL:    cxifi:
 
-!llvm.ident = !{!0}
+  ; ALL:            addu    $[[REG_GP:[0-9]+]], ${{[0-9]+}}, ${{[0-9+]}}
+  ; ALL-DAG:        addiu   $4, $zero, 887
+  ; ALL-DAG:        lui     $[[REGF_1:[0-9]+]], 16659
+  ; ALL-DAG:        ori     $[[REGF_2:[0-9]+]], $[[REGF_1]], 48759
+  ; ALL-DAG:        mtc1    $[[REGF_2]], $f[[REGF_3:[0-9]+]]
+  ; ALL-DAG:        mfc1    $5, $f[[REGF_3]]
+  ; ALL-DAG:        addiu   $6, $zero, 888
+  ; ALL-DAG:        lw      $25, %got(xifi)($[[REG_GP]])
+  ; ALL:            jalr    $25
+  call void @xifi(i32 887, float 0x402277CEE0000000, i32 888)
+  ret void
+}
 
-!0 = !{!"clang version 3.6.0 (gitosis@dmz-portal.mips.com:clang 43992fe7b17de5553ac06d323cb80cc6723a9ae3) (gitosis@dmz-portal.mips.com:llvm.git 0834e6839eb170197c81bb02e916258d1527e312)"}
+declare void @xifif(i32, float, i32, float)
+
+define void @cxifif() {
+  ; ALL-LABEL:    cxifif:
+
+  ; ALL:            addu    $[[REG_GP:[0-9]+]], ${{[0-9]+}}, ${{[0-9+]}}
+  ; ALL-DAG:        lui     $[[REGI:[0-9]+]], 1
+  ; ALL-DAG:        ori     $4, $[[REGI]], 2238
+  ; ALL-DAG:        lui     $[[REGF0_1:[0-9]+]], 17527
+  ; ALL-DAG:        ori     $[[REGF0_2:[0-9]+]], $[[REGF0_1]], 2015
+  ; ALL-DAG:        mtc1    $[[REGF0_2]], $f[[REGF0_3:[0-9]+]]
+  ; ALL-DAG:        addiu   $6, $zero, 9991
+  ; ALL-DAG:        lui     $[[REGF1_1:[0-9]+]], 17802
+  ; ALL-DAG:        ori     $[[REGF1_2:[0-9]+]], $[[REGF1_1]], 58470
+  ; ALL:            mtc1    $[[REGF1_2]], $f[[REGF1_3:[0-9]+]]
+  ; ALL-DAG:        mfc1    $5, $f[[REGF0_3]]
+  ; ALL-DAG:        mfc1    $7, $f[[REGF1_3]]
+  ; ALL-DAG:        lw      $25, %got(xifif)($[[REG_GP]])
+  ; ALL:            jalr    $25
+  call void @xifif(i32 67774, float 0x408EE0FBE0000000,
+                   i32 9991, float 0x40B15C8CC0000000)
+  ret void
+}
+
+declare void @xiffi(i32, float, float, i32)
+
+define void @cxiffi() {
+  ; ALL-LABEL:    cxiffi:
+
+  ; ALL:            addu    $[[REG_GP:[0-9]+]], ${{[0-9]+}}, ${{[0-9+]}}
+  ; ALL-DAG:        addiu   $4, $zero, 45
+  ; ALL-DAG:        lui     $[[REGF0_1:[0-9]+]], 16307
+  ; ALL-DAG:        ori     $[[REGF0_2:[0-9]+]], $[[REGF0_1]], 13107
+  ; ALL-DAG:        mtc1    $[[REGF0_2]], $f[[REGF0_3:[0-9]+]]
+  ; ALL-DAG:        lui     $[[REGF1_1:[0-9]+]], 17529
+  ; ALL-DAG:        ori     $[[REGF1_2:[0-9]+]], $[[REGF1_1]], 39322
+  ; ALL:            mtc1    $[[REGF1_2]], $f[[REGF1_3:[0-9]+]]
+  ; ALL-DAG:        addiu   $7, $zero, 234
+  ; ALL-DAG:        mfc1    $5, $f[[REGF0_3]]
+  ; ALL-DAG:        mfc1    $6, $f[[REGF1_3]]
+  ; ALL-DAG:        lw      $25, %got(xiffi)($[[REG_GP]])
+  ; ALL:            jalr    $25
+  call void @xiffi(i32 45, float 0x3FF6666660000000,
+                   float 0x408F333340000000, i32 234)
+  ret void
+}
+
+declare void @xifii(i32, float, i32, i32)
+
+define void @cxifii() {
+  ; ALL-LABEL:    cxifii:
+
+  ; ALL-DAG:    addu    $[[REG_GP:[0-9]+]], ${{[0-9]+}}, ${{[0-9+]}}
+  ; ALL-DAG:    addiu   $4, $zero, 12239
+  ; ALL-DAG:    lui     $[[REGF_1:[0-9]+]], 17526
+  ; ALL-DAG:    ori     $[[REGF_2:[0-9]+]], $[[REGF_1]], 55706
+  ; ALL-DAG:    mtc1    $[[REGF_2]], $f[[REGF_3:[0-9]+]]
+  ; ALL-DAG:    mfc1    $5, $f[[REGF_3]]
+  ; ALL-DAG:    lui     $[[REGI2:[0-9]+]], 15
+  ; ALL-DAG:    ori     $6, $[[REGI2]], 15837
+  ; ALL-DAG:    addiu   $7, $zero, 1234
+  ; ALL-DAG:    lw      $25, %got(xifii)($[[REG_GP]])
+  ; ALL:        jalr    $25
+  call void @xifii(i32 12239, float 0x408EDB3340000000, i32 998877, i32 1234)
+  ret void
+}
diff --git a/test/CodeGen/Mips/Fast-ISel/constexpr-address.ll b/test/CodeGen/Mips/Fast-ISel/constexpr-address.ll
new file mode 100644
index 00000000000..d6d9074c7c1
--- /dev/null
+++ b/test/CodeGen/Mips/Fast-ISel/constexpr-address.ll
@@ -0,0 +1,18 @@
+; RUN: llc -march=mipsel -mcpu=mips32 -relocation-model=pic \
+; RUN:     -fast-isel=true -fast-isel-abort=1 < %s | FileCheck %s
+; RUN: llc -march=mipsel -mcpu=mips32r2 -relocation-model=pic \
+; RUN:     -fast-isel=true -fast-isel-abort=1 < %s | FileCheck %s
+
+@ARR = external global [10 x i32], align 4
+
+define void @foo() {
+; CHECK-LABEL: foo
+
+; CHECK-DAG:    lw      $[[ARR:[0-9]+]], %got(ARR)({{.*}})
+; CHECK-DAG:    addiu   $[[T0:[0-9]+]], $zero, 12345
+; CHECK:        sw      $[[T0]], 8($[[ARR]])
+
+entry:
+  store i32 12345, i32* getelementptr inbounds ([10 x i32], [10 x i32]* @ARR, i32 0, i32 2), align 4
+  ret void
+}
diff --git a/test/CodeGen/Mips/Fast-ISel/div1.ll b/test/CodeGen/Mips/Fast-ISel/div1.ll
new file mode 100644
index 00000000000..89055aa1280
--- /dev/null
+++ b/test/CodeGen/Mips/Fast-ISel/div1.ll
@@ -0,0 +1,55 @@
+; RUN: llc < %s -march=mipsel -mcpu=mips32 -O0 -relocation-model=pic \
+; RUN:      -fast-isel-abort=1 | FileCheck %s
+; RUN: llc < %s -march=mipsel -mcpu=mips32r2 -O0 -relocation-model=pic \
+; RUN:      -fast-isel-abort=1 | FileCheck %s
+
+@sj = global i32 200000, align 4
+@sk = global i32 -47, align 4
+@uj = global i32 200000, align 4
+@uk = global i32 43, align 4
+@si = common global i32 0, align 4
+@ui = common global i32 0, align 4
+
+define void @divs() {
+  ; CHECK-LABEL:  divs:
+
+  ; CHECK:        lui     $[[GOT1:[0-9]+]], %hi(_gp_disp)
+  ; CHECK:        addiu   $[[GOT2:[0-9]+]], $[[GOT1]], %lo(_gp_disp)
+  ; CHECK:        addu    $[[GOT:[0-9]+]], $[[GOT2:[0-9]+]], $25
+  ; CHECK-DAG:    lw      $[[I_ADDR:[0-9]+]], %got(si)($[[GOT]])
+  ; CHECK-DAG:    lw      $[[K_ADDR:[0-9]+]], %got(sk)($[[GOT]])
+  ; CHECK-DAG:    lw      $[[J_ADDR:[0-9]+]], %got(sj)($[[GOT]])
+  ; CHECK-DAG:    lw      $[[J:[0-9]+]], 0($[[J_ADDR]])
+  ; CHECK-DAG:    lw      $[[K:[0-9]+]], 0($[[K_ADDR]])
+  ; CHECK-DAG:    div     $zero, $[[J]], $[[K]]
+  ; CHECK_DAG:    teq     $[[K]], $zero, 7
+  ; CHECK-DAG:    mflo    $[[RESULT:[0-9]+]]
+  ; CHECK:        sw      $[[RESULT]], 0($[[I_ADDR]])
+  %1 = load i32, i32* @sj, align 4
+  %2 = load i32, i32* @sk, align 4
+  %div = sdiv i32 %1, %2
+  store i32 %div, i32* @si, align 4
+  ret void
+}
+
+define void @divu() {
+  ; CHECK-LABEL:  divu:
+
+  ; CHECK:            lui     $[[GOT1:[0-9]+]], %hi(_gp_disp)
+  ; CHECK:            addiu   $[[GOT2:[0-9]+]], $[[GOT1]], %lo(_gp_disp)
+  ; CHECK:            addu    $[[GOT:[0-9]+]], $[[GOT2:[0-9]+]], $25
+  ; CHECK-DAG:        lw      $[[I_ADDR:[0-9]+]], %got(ui)($[[GOT]])
+  ; CHECK-DAG:        lw      $[[K_ADDR:[0-9]+]], %got(uk)($[[GOT]])
+  ; CHECK-DAG:        lw      $[[J_ADDR:[0-9]+]], %got(uj)($[[GOT]])
+  ; CHECK-DAG:        lw      $[[J:[0-9]+]], 0($[[J_ADDR]])
+  ; CHECK-DAG:        lw      $[[K:[0-9]+]], 0($[[K_ADDR]])
+  ; CHECK-DAG:        divu    $zero, $[[J]], $[[K]]
+  ; CHECK_DAG:        teq     $[[K]], $zero, 7
+  ; CHECK-DAG:        mflo    $[[RESULT:[0-9]+]]
+  ; CHECK:            sw      $[[RESULT]], 0($[[I_ADDR]])
+  %1 = load i32, i32* @uj, align 4
+  %2 = load i32, i32* @uk, align 4
+  %div = udiv i32 %1, %2
+  store i32 %div, i32* @ui, align 4
+  ret void
+}
diff --git a/test/CodeGen/Mips/Fast-ISel/fastalloca.ll b/test/CodeGen/Mips/Fast-ISel/fastalloca.ll
new file mode 100644
index 00000000000..00bc7f485e0
--- /dev/null
+++ b/test/CodeGen/Mips/Fast-ISel/fastalloca.ll
@@ -0,0 +1,32 @@
+; RUN: llc -march=mipsel -relocation-model=pic -O0 -fast-isel-abort=1 -mcpu=mips32r2 \
+; RUN:     < %s | FileCheck %s
+
+%struct.x = type { i32 }
+
+@i = common global i32 0, align 4
+
+define i32 @foobar(i32 signext %x) {
+entry:
+; CHECK-LABEL: foobar:
+  %retval = alloca i32, align 4
+  %x.addr = alloca i32, align 4
+  %a = alloca %struct.x, align 4
+  %c = alloca %struct.x*, align 4
+  store i32 %x, i32* %x.addr, align 4
+  %x1 = getelementptr inbounds %struct.x, %struct.x* %a, i32 0, i32 0
+  %0 = load i32, i32* %x.addr, align 4
+  store i32 %0, i32* %x1, align 4
+  store %struct.x* %a, %struct.x** %c, align 4
+  %1 = load %struct.x*, %struct.x** %c, align 4
+  %x2 = getelementptr inbounds %struct.x, %struct.x* %1, i32 0, i32 0
+  %2 = load i32, i32* %x2, align 4
+  store i32 %2, i32* @i, align 4
+  %3 = load i32, i32* %retval
+; CHECK-DAG:    lw      $[[I_ADDR:[0-9]+]], %got(i)($[[REG_GP:[0-9]+]])
+; CHECK-DAG:    addiu   $[[A_ADDR:[0-9]+]], $sp, 8
+; CHECK-DAG:    sw      $[[A_ADDR]], [[A_ADDR_FI:[0-9]+]]($sp)
+; CHECK-DAG:    lw      $[[A_ADDR2:[0-9]+]], [[A_ADDR_FI]]($sp)
+; CHECK-DAG:    lw      $[[A_X:[0-9]+]], 0($[[A_ADDR2]])
+; CHECK-DAG:    sw      $[[A_X]], 0($[[I_ADDR]])
+  ret i32 %3
+}
diff --git a/test/CodeGen/Mips/Fast-ISel/fastcc-miss.ll b/test/CodeGen/Mips/Fast-ISel/fastcc-miss.ll
new file mode 100644
index 00000000000..d9ce8b3964a
--- /dev/null
+++ b/test/CodeGen/Mips/Fast-ISel/fastcc-miss.ll
@@ -0,0 +1,15 @@
+; RUN: llc < %s -march=mipsel -mcpu=mips32r2 -O0 -relocation-model=pic \
+; RUN:     -fast-isel-verbose 2>&1 | FileCheck %s
+
+; CHECK:      FastISel missed call:
+; CHECK-SAME: %call = call fastcc i32 @foo(i32 signext %a, i32 signext %b)
+
+define internal i32 @bar(i32 signext %a, i32 signext %b) {
+  %s = and i32 %a, %b
+  ret i32 %s
+}
+
+define i32 @foo(i32 signext %a, i32 signext %b) {
+  %call = call fastcc i32 @foo(i32 signext %a, i32 signext %b)
+  ret i32 %call
+}
diff --git a/test/CodeGen/Mips/Fast-ISel/fpcmpa.ll b/test/CodeGen/Mips/Fast-ISel/fpcmpa.ll
index 72de888b26e..e346acfeff1 100644
--- a/test/CodeGen/Mips/Fast-ISel/fpcmpa.ll
+++ b/test/CodeGen/Mips/Fast-ISel/fpcmpa.ll
@@ -1,6 +1,6 @@
-; RUN: llc -march=mipsel -relocation-model=pic -O0 -mips-fast-isel -fast-isel-abort=1 -mcpu=mips32r2 \
+; RUN: llc -march=mipsel -relocation-model=pic -O0 -fast-isel-abort=1 -mcpu=mips32r2 \
 ; RUN:     < %s | FileCheck %s
-; RUN: llc -march=mipsel -relocation-model=pic -O0 -mips-fast-isel -fast-isel-abort=1 -mcpu=mips32 \
+; RUN: llc -march=mipsel -relocation-model=pic -O0 -fast-isel-abort=1 -mcpu=mips32 \
 ; RUN:     < %s | FileCheck %s
 
 @f1 = common global float 0.000000e+00, align 4
diff --git a/test/CodeGen/Mips/Fast-ISel/fpext.ll b/test/CodeGen/Mips/Fast-ISel/fpext.ll
index 5ac22490ff0..f78289f40a0 100644
--- a/test/CodeGen/Mips/Fast-ISel/fpext.ll
+++ b/test/CodeGen/Mips/Fast-ISel/fpext.ll
@@ -1,6 +1,6 @@
-; RUN: llc -march=mipsel -relocation-model=pic -O0 -mips-fast-isel -fast-isel-abort=1 -mcpu=mips32r2 \
+; RUN: llc -march=mipsel -relocation-model=pic -O0 -fast-isel-abort=1 -mcpu=mips32r2 \
 ; RUN:     < %s | FileCheck %s
-; RUN: llc -march=mipsel -relocation-model=pic -O0 -mips-fast-isel -fast-isel-abort=1 -mcpu=mips32 \
+; RUN: llc -march=mipsel -relocation-model=pic -O0 -fast-isel-abort=1 -mcpu=mips32 \
 ; RUN:     < %s | FileCheck %s
 
 @f = global float 0x40147E6B80000000, align 4
diff --git a/test/CodeGen/Mips/Fast-ISel/fpintconv.ll b/test/CodeGen/Mips/Fast-ISel/fpintconv.ll
index a94ef508153..2c022be5b3f 100644
--- a/test/CodeGen/Mips/Fast-ISel/fpintconv.ll
+++ b/test/CodeGen/Mips/Fast-ISel/fpintconv.ll
@@ -1,6 +1,6 @@
-; RUN: llc -march=mipsel -relocation-model=pic -O0 -mips-fast-isel -fast-isel-abort=1 -mcpu=mips32r2 \
+; RUN: llc -march=mipsel -relocation-model=pic -O0 -fast-isel-abort=1 -mcpu=mips32r2 \
 ; RUN:     < %s | FileCheck %s
-; RUN: llc -march=mipsel -relocation-model=pic -O0 -mips-fast-isel -fast-isel-abort=1 -mcpu=mips32 \
+; RUN: llc -march=mipsel -relocation-model=pic -O0 -fast-isel-abort=1 -mcpu=mips32 \
 ; RUN:     < %s | FileCheck %s
 
 
diff --git a/test/CodeGen/Mips/Fast-ISel/fptrunc.ll b/test/CodeGen/Mips/Fast-ISel/fptrunc.ll
index 2eec4c3ef54..89a7bfce5b0 100644
--- a/test/CodeGen/Mips/Fast-ISel/fptrunc.ll
+++ b/test/CodeGen/Mips/Fast-ISel/fptrunc.ll
@@ -1,6 +1,6 @@
-; RUN: llc -march=mipsel -relocation-model=pic -O0 -mips-fast-isel -fast-isel-abort=1 -mcpu=mips32r2 \
+; RUN: llc -march=mipsel -relocation-model=pic -O0 -fast-isel-abort=1 -mcpu=mips32r2 \
 ; RUN:     < %s | FileCheck %s
-; RUN: llc -march=mipsel -relocation-model=pic -O0 -mips-fast-isel -fast-isel-abort=1 -mcpu=mips32 \
+; RUN: llc -march=mipsel -relocation-model=pic -O0 -fast-isel-abort=1 -mcpu=mips32 \
 ; RUN:     < %s | FileCheck %s
 
 @d = global double 0x40147E6B74DF0446, align 8
diff --git a/test/CodeGen/Mips/Fast-ISel/icmpa.ll b/test/CodeGen/Mips/Fast-ISel/icmpa.ll
index 670a8d5cfb4..fc37e118e75 100644
--- a/test/CodeGen/Mips/Fast-ISel/icmpa.ll
+++ b/test/CodeGen/Mips/Fast-ISel/icmpa.ll
@@ -1,6 +1,6 @@
-; RUN: llc -march=mipsel -relocation-model=pic -O0 -mips-fast-isel -fast-isel-abort=1 -mcpu=mips32r2 \
+; RUN: llc -march=mipsel -relocation-model=pic -O0 -fast-isel-abort=1 -mcpu=mips32r2 \
 ; RUN:     < %s | FileCheck %s
-; RUN: llc -march=mipsel -relocation-model=pic -O0 -mips-fast-isel -fast-isel-abort=1 -mcpu=mips32 \
+; RUN: llc -march=mipsel -relocation-model=pic -O0 -fast-isel-abort=1 -mcpu=mips32 \
 ; RUN:     < %s | FileCheck %s
 
 @c = global i32 4, align 4
diff --git a/test/CodeGen/Mips/Fast-ISel/loadstore2.ll b/test/CodeGen/Mips/Fast-ISel/loadstore2.ll
index 3daf03d681c..46f7a42a5fe 100644
--- a/test/CodeGen/Mips/Fast-ISel/loadstore2.ll
+++ b/test/CodeGen/Mips/Fast-ISel/loadstore2.ll
@@ -4,9 +4,9 @@ target triple = "mips--linux-gnu"
 
 @c2 = common global i8 0, align 1
 @c1 = common global i8 0, align 1
-; RUN: llc -march=mipsel -relocation-model=pic -O0 -mips-fast-isel -fast-isel-abort=1 -mcpu=mips32r2 \
+; RUN: llc -march=mipsel -relocation-model=pic -O0 -fast-isel-abort=1 -mcpu=mips32r2 \
 ; RUN:     < %s | FileCheck %s
-; RUN: llc -march=mipsel -relocation-model=pic -O0 -mips-fast-isel -fast-isel-abort=1 -mcpu=mips32 \
+; RUN: llc -march=mipsel -relocation-model=pic -O0 -fast-isel-abort=1 -mcpu=mips32 \
 ; RUN:     < %s | FileCheck %s
 
 @s2 = common global i16 0, align 2
diff --git a/test/CodeGen/Mips/Fast-ISel/loadstoreconv.ll b/test/CodeGen/Mips/Fast-ISel/loadstoreconv.ll
index acba132b28e..09b56d2c87e 100644
--- a/test/CodeGen/Mips/Fast-ISel/loadstoreconv.ll
+++ b/test/CodeGen/Mips/Fast-ISel/loadstoreconv.ll
@@ -1,10 +1,10 @@
-; RUN: llc -march=mipsel -relocation-model=pic -O0 -mips-fast-isel -fast-isel-abort=1 -mcpu=mips32r2 \
+; RUN: llc -march=mipsel -relocation-model=pic -O0 -fast-isel-abort=1 -mcpu=mips32r2 \
 ; RUN:     < %s | FileCheck %s
-; RUN: llc -march=mipsel -relocation-model=pic -O0 -mips-fast-isel -fast-isel-abort=1 -mcpu=mips32 \
+; RUN: llc -march=mipsel -relocation-model=pic -O0 -fast-isel-abort=1 -mcpu=mips32 \
 ; RUN:     < %s | FileCheck %s
-; RUN: llc -march=mipsel -relocation-model=pic -O0 -mips-fast-isel -fast-isel-abort=1 -mcpu=mips32r2 \
+; RUN: llc -march=mipsel -relocation-model=pic -O0 -fast-isel-abort=1 -mcpu=mips32r2 \
 ; RUN:     < %s | FileCheck %s -check-prefix=mips32r2
-; RUN: llc -march=mipsel -relocation-model=pic -O0 -mips-fast-isel -fast-isel-abort=1 -mcpu=mips32 \
+; RUN: llc -march=mipsel -relocation-model=pic -O0 -fast-isel-abort=1 -mcpu=mips32 \
 ; RUN:     < %s | FileCheck %s -check-prefix=mips32
 
 @b2 = global i8 0, align 1
diff --git a/test/CodeGen/Mips/Fast-ISel/loadstrconst.ll b/test/CodeGen/Mips/Fast-ISel/loadstrconst.ll
index 9f644ecd187..1051b2800e5 100644
--- a/test/CodeGen/Mips/Fast-ISel/loadstrconst.ll
+++ b/test/CodeGen/Mips/Fast-ISel/loadstrconst.ll
@@ -1,6 +1,6 @@
-; RUN: llc -march=mipsel -relocation-model=pic -O0 -mips-fast-isel -fast-isel-abort=1 -mcpu=mips32r2 \
+; RUN: llc -march=mipsel -relocation-model=pic -O0 -fast-isel-abort=1 -mcpu=mips32r2 \
 ; RUN:     < %s | FileCheck %s
-; RUN: llc -march=mipsel -relocation-model=pic -O0 -mips-fast-isel -fast-isel-abort=1 -mcpu=mips32 \
+; RUN: llc -march=mipsel -relocation-model=pic -O0 -fast-isel-abort=1 -mcpu=mips32 \
 ; RUN:     < %s | FileCheck %s
 
 @.str = private unnamed_addr constant [6 x i8] c"hello\00", align 1
diff --git a/test/CodeGen/Mips/Fast-ISel/logopm.ll b/test/CodeGen/Mips/Fast-ISel/logopm.ll
index 0f0c3bf9e1d..fec85092fff 100644
--- a/test/CodeGen/Mips/Fast-ISel/logopm.ll
+++ b/test/CodeGen/Mips/Fast-ISel/logopm.ll
@@ -1,5 +1,5 @@
-; RUN: llc -march=mipsel -relocation-model=pic -O0 -fast-isel -mips-fast-isel -fast-isel-abort=1 -mcpu=mips32r2 < %s | FileCheck %s
-; RUN: llc -march=mipsel -relocation-model=pic -O0 -fast-isel -mips-fast-isel -fast-isel-abort=1 -mcpu=mips32 < %s | FileCheck %s
+; RUN: llc -march=mipsel -relocation-model=pic -O0 -fast-isel-abort=1 -mcpu=mips32r2 < %s | FileCheck %s
+; RUN: llc -march=mipsel -relocation-model=pic -O0 -fast-isel-abort=1 -mcpu=mips32 < %s | FileCheck %s
 
 @ub1 = common global i8 0, align 1
 @ub2 = common global i8 0, align 1
@@ -283,8 +283,8 @@ entry:
 ; CHECK-DAG:    lw      $[[UC_ADDR:[0-9]+]], %got(uc)($[[REG_GP]])
 ; CHECK-DAG:    lw      $[[UC1_ADDR:[0-9]+]], %got(uc1)($[[REG_GP]])
 ; CHECK-DAG:    lbu     $[[UC1:[0-9]+]], 0($[[UC1_ADDR]])
-; CHECK-DAG:    addiu   $[[CONST_Neg89:[0-9]+]], $zero, -89
-; CHECK-DAG:    and     $[[RES:[0-9]+]], $[[UC1]], $[[CONST_Neg89]]
+; CHECK-DAG:    addiu   $[[CONST_167:[0-9]+]], $zero, 167
+; CHECK-DAG:    and     $[[RES:[0-9]+]], $[[UC1]], $[[CONST_167]]
 ; CHECK:        sb      $[[RES]], 0($[[UC_ADDR]])
 ; CHECK:        .end    andUc1
   ret void
@@ -345,8 +345,8 @@ entry:
 ; CHECK-DAG:    lw      $[[UC_ADDR:[0-9]+]], %got(uc)($[[REG_GP]])
 ; CHECK-DAG:    lw      $[[UC1_ADDR:[0-9]+]], %got(uc1)($[[REG_GP]])
 ; CHECK-DAG:    lbu     $[[UC1:[0-9]+]], 0($[[UC1_ADDR]])
-; CHECK-DAG:    addiu   $[[CONST_neg18:[0-9]+]], $zero, -18
-; CHECK-DAG:    or      $[[RES:[0-9]+]], $[[UC1]], $[[CONST_neg18]]
+; CHECK-DAG:    addiu   $[[CONST_238:[0-9]+]], $zero, 238
+; CHECK-DAG:    or      $[[RES:[0-9]+]], $[[UC1]], $[[CONST_238]]
 ; CHECK:        sb      $[[RES]], 0($[[UC_ADDR]])
 ; CHECK:        .end    orUc1
   ret void
@@ -469,8 +469,8 @@ entry:
 ; CHECK-DAG:    lw      $[[US_ADDR:[0-9]+]], %got(us)($[[REG_GP]])
 ; CHECK-DAG:    lw      $[[US1_ADDR:[0-9]+]], %got(us1)($[[REG_GP]])
 ; CHECK-DAG:    lhu     $[[US1:[0-9]+]], 0($[[US1_ADDR]])
-; CHECK-DAG:    addiu   $[[CONST_Neg4185:[0-9]+]], $zero, -4185
-; CHECK-DAG:    and     $[[RES:[0-9]+]], $[[US1]], $[[CONST_Neg4185]]
+; CHECK-DAG:    ori     $[[CONST_61351:[0-9]+]], $zero, 61351
+; CHECK-DAG:    and     $[[RES:[0-9]+]], $[[US1]], $[[CONST_61351]]
 ; CHECK:        sh      $[[RES]], 0($[[US_ADDR]])
 ; CHECK:        .end    andUs1
   ret void
@@ -520,8 +520,8 @@ entry:
 ; CHECK-DAG:    lw      $[[US_ADDR:[0-9]+]], %got(us)($[[REG_GP]])
 ; CHECK-DAG:    lw      $[[US1_ADDR:[0-9]+]], %got(us1)($[[REG_GP]])
 ; CHECK-DAG:    lhu     $[[US1:[0-9]+]], 0($[[US1_ADDR]])
-; CHECK-DAG:    addiu   $[[CONST_neg4591:[0-9]+]], $zero, -4591
-; CHECK-DAG:    or      $[[RES:[0-9]+]], $[[US1]], $[[CONST_neg4591]]
+; CHECK-DAG:    ori     $[[CONST_60945:[0-9]+]], $zero, 60945
+; CHECK-DAG:    or      $[[RES:[0-9]+]], $[[US1]], $[[CONST_60945]]
 ; CHECK:        sh      $[[RES]], 0($[[US_ADDR]])
 ; CHECK:        .end    orUs1
   ret void
@@ -583,8 +583,8 @@ entry:
 ; CHECK-DAG:    lw      $[[US_ADDR:[0-9]+]], %got(us)($[[REG_GP]])
 ; CHECK-DAG:    lw      $[[US1_ADDR:[0-9]+]], %got(us1)($[[REG_GP]])
 ; CHECK-DAG:    lhu     $[[US1:[0-9]+]], 0($[[US1_ADDR]])
-; CHECK-DAG:    addiu   $[[CONST_Neg5512:[0-9]+]], $zero, -5512
-; CHECK-DAG:    xor     $[[RES:[0-9]+]], $[[US1]], $[[CONST_Neg5512]]
+; CHECK-DAG:    ori     $[[CONST_60024:[0-9]+]], $zero, 60024
+; CHECK-DAG:    xor     $[[RES:[0-9]+]], $[[US1]], $[[CONST_60024]]
 ; CHECK:        sh      $[[RES]], 0($[[US_ADDR]])
 ; CHECK:        .end    xorUs1
   ret void
diff --git a/test/CodeGen/Mips/Fast-ISel/memtest1.ll b/test/CodeGen/Mips/Fast-ISel/memtest1.ll
new file mode 100644
index 00000000000..b98200d7456
--- /dev/null
+++ b/test/CodeGen/Mips/Fast-ISel/memtest1.ll
@@ -0,0 +1,74 @@
+; RUN: llc < %s -march=mipsel -mcpu=mips32 -O0 -relocation-model=pic \
+; RUN:     -fast-isel-abort=1 | FileCheck %s \
+; RUN:     -check-prefix=ALL -check-prefix=32R1
+; RUN: llc < %s -march=mipsel -mcpu=mips32r2 -O0 -relocation-model=pic \
+; RUN:     -fast-isel-abort=1 | FileCheck %s \
+; RUN:     -check-prefix=ALL -check-prefix=32R2
+
+@str = private unnamed_addr constant [12 x i8] c"hello there\00", align 1
+@src = global i8* getelementptr inbounds ([12 x i8], [12 x i8]* @str, i32 0, i32 0), align 4
+@i = global i32 12, align 4
+@dest = common global [50 x i8] zeroinitializer, align 1
+
+declare void @llvm.memcpy.p0i8.p0i8.i32(i8* nocapture, i8* nocapture readonly, i32, i32, i1)
+declare void @llvm.memmove.p0i8.p0i8.i32(i8* nocapture, i8* nocapture readonly, i32, i32, i1)
+declare void @llvm.memset.p0i8.i32(i8* nocapture, i8, i32, i32, i1)
+
+define void @cpy(i8* %src, i32 %i) {
+  ; ALL-LABEL:  cpy:
+
+  ; ALL-DAG:        lw    $[[T0:[0-9]+]], %got(dest)(${{[0-9]+}})
+  ; ALL-DAG:        sw    $4, 24($sp)
+  ; ALL-DAG:        move  $4, $[[T0]]
+  ; ALL-DAG:        sw    $5, 20($sp)
+  ; ALL-DAG:        lw    $[[T1:[0-9]+]], 24($sp)
+  ; ALL-DAG:        move  $5, $[[T1]]
+  ; ALL-DAG:        lw    $6, 20($sp)
+  ; ALL-DAG:        lw    $[[T2:[0-9]+]], %got(memcpy)(${{[0-9]+}})
+  ; ALL:            jalr  $[[T2]]
+  ; ALL-NEXT:       nop
+  ; ALL-NOT:        {{.*}}$2{{.*}}
+  call void @llvm.memcpy.p0i8.p0i8.i32(i8* getelementptr inbounds ([50 x i8], [50 x i8]* @dest, i32 0, i32 0),
+                                       i8* %src, i32 %i, i32 1, i1 false)
+  ret void
+}
+
+define void @mov(i8* %src, i32 %i) {
+  ; ALL-LABEL:  mov:
+
+
+  ; ALL-DAG:        lw    $[[T0:[0-9]+]], %got(dest)(${{[0-9]+}})
+  ; ALL-DAG:        sw    $4, 24($sp)
+  ; ALL-DAG:        move  $4, $[[T0]]
+  ; ALL-DAG:        sw    $5, 20($sp)
+  ; ALL-DAG:        lw    $[[T1:[0-9]+]], 24($sp)
+  ; ALL-DAG:        move  $5, $[[T1]]
+  ; ALL-DAG:        lw    $6, 20($sp)
+  ; ALL-DAG:        lw    $[[T2:[0-9]+]], %got(memmove)(${{[0-9]+}})
+  ; ALL:            jalr  $[[T2]]
+  ; ALL-NEXT:       nop
+  ; ALL-NOT:        {{.*}}$2{{.*}}
+  call void @llvm.memmove.p0i8.p0i8.i32(i8* getelementptr inbounds ([50 x i8], [50 x i8]* @dest, i32 0, i32 0),
+                                        i8* %src, i32 %i, i32 1, i1 false)
+  ret void
+}
+
+define void @clear(i32 %i) {
+  ; ALL-LABEL:  clear:
+
+  ; ALL-DAG:        lw    $[[T0:[0-9]+]], %got(dest)(${{[0-9]+}})
+  ; ALL-DAG:        sw    $4, 16($sp)
+  ; ALL-DAG:        move  $4, $[[T0]]
+  ; ALL-DAG:        addiu $[[T1:[0-9]+]], $zero, 42
+  ; 32R1-DAG:       sll   $[[T2:[0-9]+]], $[[T1]], 24
+  ; 32R1-DAG:       sra   $5, $[[T2]], 24
+  ; 32R2-DAG:       seb   $5, $[[T1]]
+  ; ALL-DAG:        lw    $6, 16($sp)
+  ; ALL-DAG:        lw    $[[T2:[0-9]+]], %got(memset)(${{[0-9]+}})
+  ; ALL:            jalr  $[[T2]]
+  ; ALL-NEXT:       nop
+  ; ALL-NOT:        {{.*}}$2{{.*}}
+  call void @llvm.memset.p0i8.i32(i8* getelementptr inbounds ([50 x i8], [50 x i8]* @dest, i32 0, i32 0),
+                                  i8 42, i32 %i, i32 1, i1 false)
+  ret void
+}
diff --git a/test/CodeGen/Mips/Fast-ISel/mul1.ll b/test/CodeGen/Mips/Fast-ISel/mul1.ll
new file mode 100644
index 00000000000..8713e7ef1d9
--- /dev/null
+++ b/test/CodeGen/Mips/Fast-ISel/mul1.ll
@@ -0,0 +1,16 @@
+; RUN: llc < %s -march=mipsel -mcpu=mips32 -O0 -relocation-model=pic
+; RUN: llc < %s -march=mipsel -mcpu=mips32r2 -O0 -relocation-model=pic
+
+; The test is just to make sure it is able to allocate
+; registers for this example. There was an issue with allocating AC0
+; after a mul instruction.
+
+declare { i32, i1 } @llvm.smul.with.overflow.i32(i32, i32)
+
+define i32 @foo(i32 %a, i32 %b)  {
+entry:
+  %0 = mul i32 %a, %b
+  %1 = call { i32, i1 } @llvm.smul.with.overflow.i32(i32 %0, i32 %b)
+  %2 = extractvalue { i32, i1 } %1, 0
+  ret i32 %2
+}
diff --git a/test/CodeGen/Mips/Fast-ISel/nullvoid.ll b/test/CodeGen/Mips/Fast-ISel/nullvoid.ll
index 5fa3f13ace4..106015e30c3 100644
--- a/test/CodeGen/Mips/Fast-ISel/nullvoid.ll
+++ b/test/CodeGen/Mips/Fast-ISel/nullvoid.ll
@@ -1,6 +1,6 @@
-; RUN: llc -march=mipsel -relocation-model=pic -O0 -mips-fast-isel -fast-isel-abort=1 -mcpu=mips32r2 \
+; RUN: llc -march=mipsel -relocation-model=pic -O0 -fast-isel-abort=1 -mcpu=mips32r2 \
 ; RUN:     < %s | FileCheck %s
-; RUN: llc -march=mipsel -relocation-model=pic -O0 -mips-fast-isel -fast-isel-abort=1 -mcpu=mips32 \
+; RUN: llc -march=mipsel -relocation-model=pic -O0 -fast-isel-abort=1 -mcpu=mips32 \
 ; RUN:     < %s | FileCheck %s
 
 ; Function Attrs: nounwind
diff --git a/test/CodeGen/Mips/Fast-ISel/overflt.ll b/test/CodeGen/Mips/Fast-ISel/overflt.ll
index 57f991e23d9..37e87b29c58 100644
--- a/test/CodeGen/Mips/Fast-ISel/overflt.ll
+++ b/test/CodeGen/Mips/Fast-ISel/overflt.ll
@@ -1,6 +1,6 @@
-; RUN: llc -march=mipsel -relocation-model=pic -O0 -mips-fast-isel -fast-isel-abort=1 -mcpu=mips32r2 \
+; RUN: llc -march=mipsel -relocation-model=pic -O0 -fast-isel-abort=1 -mcpu=mips32r2 \
 ; RUN:     < %s | FileCheck %s
-; RUN: llc -march=mipsel -relocation-model=pic -O0 -mips-fast-isel -fast-isel-abort=1 -mcpu=mips32 \
+; RUN: llc -march=mipsel -relocation-model=pic -O0 -fast-isel-abort=1 -mcpu=mips32 \
 ; RUN:     < %s | FileCheck %s
 
 @x = common global [128000 x float] zeroinitializer, align 4
diff --git a/test/CodeGen/Mips/Fast-ISel/rem1.ll b/test/CodeGen/Mips/Fast-ISel/rem1.ll
new file mode 100644
index 00000000000..cf709e7e495
--- /dev/null
+++ b/test/CodeGen/Mips/Fast-ISel/rem1.ll
@@ -0,0 +1,56 @@
+; RUN: llc < %s -march=mipsel -mcpu=mips32 -O0 -relocation-model=pic \
+; RUN:      -fast-isel-abort=1 | FileCheck %s
+; RUN: llc < %s -march=mipsel -mcpu=mips32r2 -O0 -relocation-model=pic \
+; RUN:      -fast-isel-abort=1 | FileCheck %s
+
+@sj = global i32 200, align 4
+@sk = global i32 -47, align 4
+@uj = global i32 200, align 4
+@uk = global i32 43, align 4
+@si = common global i32 0, align 4
+@ui = common global i32 0, align 4
+
+define void @rems() {
+  ; CHECK-LABEL:  rems:
+
+  ; CHECK:            lui     $[[GOT1:[0-9]+]], %hi(_gp_disp)
+  ; CHECK:            addiu   $[[GOT2:[0-9]+]], $[[GOT1]], %lo(_gp_disp)
+  ; CHECK:            addu    $[[GOT:[0-9]+]], $[[GOT2:[0-9]+]], $25
+  ; CHECK-DAG:        lw      $[[I_ADDR:[0-9]+]], %got(si)($[[GOT]])
+  ; CHECK-DAG:        lw      $[[K_ADDR:[0-9]+]], %got(sk)($[[GOT]])
+  ; CHECK-DAG:        lw      $[[J_ADDR:[0-9]+]], %got(sj)($[[GOT]])
+  ; CHECK-DAG:        lw      $[[J:[0-9]+]], 0($[[J_ADDR]])
+  ; CHECK-DAG:        lw      $[[K:[0-9]+]], 0($[[K_ADDR]])
+  ; CHECK-DAG:        div     $zero, $[[J]], $[[K]]
+  ; CHECK_DAG:        teq     $[[K]], $zero, 7
+  ; CHECK-DAG:        mfhi    $[[RESULT:[0-9]+]]
+  ; CHECK:            sw      $[[RESULT]], 0($[[I_ADDR]])
+  %1 = load i32, i32* @sj, align 4
+  %2 = load i32, i32* @sk, align 4
+  %rem = srem i32 %1, %2
+  store i32 %rem, i32* @si, align 4
+  ret void
+}
+
+; Function Attrs: noinline nounwind
+define void @remu() {
+  ; CHECK-LABEL:  remu:
+
+  ; CHECK:            lui     $[[GOT1:[0-9]+]], %hi(_gp_disp)
+  ; CHECK:            addiu   $[[GOT2:[0-9]+]], $[[GOT1]], %lo(_gp_disp)
+  ; CHECK:            addu    $[[GOT:[0-9]+]], $[[GOT2:[0-9]+]], $25
+  ; CHECK-DAG:        lw      $[[I_ADDR:[0-9]+]], %got(ui)($[[GOT]])
+  ; CHECK-DAG:        lw      $[[K_ADDR:[0-9]+]], %got(uk)($[[GOT]])
+  ; CHECK-DAG:        lw      $[[J_ADDR:[0-9]+]], %got(uj)($[[GOT]])
+  ; CHECK-DAG:        lw      $[[J:[0-9]+]], 0($[[J_ADDR]])
+  ; CHECK-DAG:        lw      $[[K:[0-9]+]], 0($[[K_ADDR]])
+  ; CHECK-DAG:        divu    $zero, $[[J]], $[[K]]
+  ; CHECK_DAG:        teq     $[[K]], $zero, 7
+  ; CHECK-DAG:        mfhi    $[[RESULT:[0-9]+]]
+  ; CHECK:            sw      $[[RESULT]], 0($[[I_ADDR]])
+  %1 = load i32, i32* @uj, align 4
+  %2 = load i32, i32* @uk, align 4
+  %rem = urem i32 %1, %2
+  store i32 %rem, i32* @ui, align 4
+  ret void
+}
diff --git a/test/CodeGen/Mips/Fast-ISel/retabi.ll b/test/CodeGen/Mips/Fast-ISel/retabi.ll
index ce0ca347ada..20747c4ed20 100644
--- a/test/CodeGen/Mips/Fast-ISel/retabi.ll
+++ b/test/CodeGen/Mips/Fast-ISel/retabi.ll
@@ -1,4 +1,4 @@
-; RUN: llc -march=mipsel -relocation-model=pic -O0 -mips-fast-isel -fast-isel-abort=1 -mcpu=mips32r2 \
+; RUN: llc -march=mipsel -relocation-model=pic -O0 -fast-isel-abort=1 -mcpu=mips32r2 \
 ; RUN:     < %s | FileCheck %s
 
 @i = global i32 75, align 4
@@ -21,6 +21,20 @@ entry:
 ; CHECK:        jr      $ra
 }
 
+; Function Attrs: nounwind
+define i16 @retus() {
+entry:
+; CHECK-LABEL: retus:
+  %0 = load i16, i16* @s, align 2
+  ret i16 %0
+; CHECK:        lui     $[[REG_GPa:[0-9]+]], %hi(_gp_disp)
+; CHECK:        addiu   $[[REG_GPb:[0-9]+]], $[[REG_GPa]], %lo(_gp_disp)
+; CHECK:        addu    $[[REG_GP:[0-9]+]], $[[REG_GPb]], $25
+; CHECK:        lw      $[[REG_S_ADDR:[0-9]+]], %got(s)($[[REG_GP]])
+; CHECK:        lhu     $2, 0($[[REG_S_ADDR]])
+; CHECK:        jr      $ra
+}
+
 ; Function Attrs: nounwind
 define signext i16 @rets() {
 entry:
@@ -36,6 +50,20 @@ entry:
 ; CHECK:        jr      $ra
 }
 
+; Function Attrs: nounwind
+define i8 @retuc() {
+entry:
+; CHECK-LABEL: retuc:
+  %0 = load i8, i8* @c, align 1
+  ret i8 %0
+; CHECK:        lui     $[[REG_GPa:[0-9]+]], %hi(_gp_disp)
+; CHECK:        addiu   $[[REG_GPb:[0-9]+]], $[[REG_GPa]], %lo(_gp_disp)
+; CHECK:        addu    $[[REG_GP:[0-9]+]], $[[REG_GPb]], $25
+; CHECK:        lw      $[[REG_C_ADDR:[0-9]+]], %got(c)($[[REG_GP]])
+; CHECK:        lbu     $2, 0($[[REG_C_ADDR]])
+; CHECK:        jr      $ra
+}
+
 ; Function Attrs: nounwind
 define signext i8 @retc() {
 entry:
diff --git a/test/CodeGen/Mips/Fast-ISel/sel1.ll b/test/CodeGen/Mips/Fast-ISel/sel1.ll
new file mode 100644
index 00000000000..8f762b0ed08
--- /dev/null
+++ b/test/CodeGen/Mips/Fast-ISel/sel1.ll
@@ -0,0 +1,97 @@
+; RUN: llc < %s -march=mipsel -mcpu=mips32r2 -O2 -relocation-model=pic \
+; RUN:          -fast-isel -fast-isel-abort=1 | FileCheck %s
+
+define i1 @sel_i1(i1 %j, i1 %k, i1 %l) {
+entry:
+  ; CHECK-LABEL:  sel_i1:
+
+  ; FIXME: The following instruction is redundant.
+  ; CHECK:            xor     $[[T0:[0-9]+]], $4, $zero
+  ; CHECK-NEXT:       sltu    $[[T1:[0-9]+]], $zero, $[[T0]]
+  ; CHECK-NEXT:       andi    $[[T2:[0-9]+]], $[[T1]], 1
+  ; CHECK-NEXT:       movn    $6, $5, $[[T2]]
+  ; CHECK:            move    $2, $6
+  %cond = icmp ne i1 %j, 0
+  %res = select i1 %cond, i1 %k, i1 %l
+  ret i1 %res
+}
+
+define i8 @sel_i8(i8 %j, i8 %k, i8 %l) {
+entry:
+  ; CHECK-LABEL:  sel_i8:
+
+  ; CHECK-DAG:        seb     $[[T0:[0-9]+]], $4
+  ; FIXME: The following 2 instructions are redundant.
+  ; CHECK-DAG:        seb     $[[T1:[0-9]+]], $zero
+  ; CHECK:            xor     $[[T2:[0-9]+]], $[[T0]], $[[T1]]
+  ; CHECK-NEXT:       sltu    $[[T3:[0-9]+]], $zero, $[[T2]]
+  ; CHECK-NEXT:       andi    $[[T4:[0-9]+]], $[[T3]], 1
+  ; CHECK-NEXT:       movn    $6, $5, $[[T4]]
+  ; CHECK:            move    $2, $6
+  %cond = icmp ne i8 %j, 0
+  %res = select i1 %cond, i8 %k, i8 %l
+  ret i8 %res
+}
+
+define i16 @sel_i16(i16 %j, i16 %k, i16 %l) {
+entry:
+  ; CHECK-LABEL:  sel_i16:
+
+  ; CHECK-DAG:        seh     $[[T0:[0-9]+]], $4
+  ; FIXME: The following 2 instructions are redundant.
+  ; CHECK-DAG:        seh     $[[T1:[0-9]+]], $zero
+  ; CHECK:            xor     $[[T2:[0-9]+]], $[[T0]], $[[T1]]
+  ; CHECK-NEXT:       sltu    $[[T3:[0-9]+]], $zero, $[[T2]]
+  ; CHECK-NEXT:       andi    $[[T4:[0-9]+]], $[[T3]], 1
+  ; CHECK-NEXT:       movn    $6, $5, $[[T4]]
+  ; CHECK:            move    $2, $6
+  %cond = icmp ne i16 %j, 0
+  %res = select i1 %cond, i16 %k, i16 %l
+  ret i16 %res
+}
+
+define i32 @sel_i32(i32 %j, i32 %k, i32 %l) {
+entry:
+  ; CHECK-LABEL:  sel_i32:
+
+  ; FIXME: The following instruction is redundant.
+  ; CHECK:            xor     $[[T0:[0-9]+]], $4, $zero
+  ; CHECK-NEXT:       sltu    $[[T1:[0-9]+]], $zero, $[[T0]]
+  ; CHECK-NEXT:       andi    $[[T2:[0-9]+]], $[[T1]], 1
+  ; CHECK-NEXT:       movn    $6, $5, $[[T2]]
+  ; CHECK:            move    $2, $6
+  %cond = icmp ne i32 %j, 0
+  %res = select i1 %cond, i32 %k, i32 %l
+  ret i32 %res
+}
+
+define float @sel_float(i32 %j, float %k, float %l) {
+entry:
+  ; CHECK-LABEL:  sel_float:
+
+  ; CHECK-DAG:        mtc1    $6, $f0
+  ; CHECK-DAG:        mtc1    $5, $f1
+  ; CHECK-DAG:        xor     $[[T0:[0-9]+]], $4, $zero
+  ; CHECK:            sltu    $[[T1:[0-9]+]], $zero, $[[T0]]
+  ; CHECK-NEXT:       andi    $[[T2:[0-9]+]], $[[T1]], 1
+  ; CHECK:            movn.s  $f0, $f1, $[[T2]]
+  %cond = icmp ne i32 %j, 0
+  %res = select i1 %cond, float %k, float %l
+  ret float %res
+}
+
+define double @sel_double(i32 %j, double %k, double %l) {
+entry:
+  ; CHECK-LABEL:  sel_double:
+
+  ; CHECK-DAG:        mtc1    $6, $f2
+  ; CHECK-DAG:        mthc1   $7, $f2
+  ; CHECK-DAG:        ldc1    $f0, 16($sp)
+  ; CHECK-DAG:        xor     $[[T0:[0-9]+]], $4, $zero
+  ; CHECK:            sltu    $[[T1:[0-9]+]], $zero, $[[T0]]
+  ; CHECK-NEXT:       andi    $[[T2:[0-9]+]], $[[T1]], 1
+  ; CHECK:            movn.d  $f0, $f2, $[[T2]]
+  %cond = icmp ne i32 %j, 0
+  %res = select i1 %cond, double %k, double %l
+  ret double %res
+}
diff --git a/test/CodeGen/Mips/Fast-ISel/shftopm.ll b/test/CodeGen/Mips/Fast-ISel/shftopm.ll
new file mode 100644
index 00000000000..bbea9c5566c
--- /dev/null
+++ b/test/CodeGen/Mips/Fast-ISel/shftopm.ll
@@ -0,0 +1,122 @@
+; RUN: llc -march=mipsel -relocation-model=pic -O0 \
+; RUN:     -fast-isel-abort=1 -mcpu=mips32r2  < %s | FileCheck %s
+; RUN: llc -march=mipsel -relocation-model=pic -O0 \
+; RUN:     -fast-isel-abort=1 -mcpu=mips32 < %s | FileCheck %s
+
+@s1 = global i16 -89, align 2
+@s2 = global i16 4, align 2
+@us1 = global i16 -503, align 2
+@us2 = global i16 5, align 2
+@s3 = common global i16 0, align 2
+@us3 = common global i16 0, align 2
+
+define void @sll() {
+entry:
+  %0 = load i16, i16* @s1, align 2
+  %1 = load i16, i16* @s2, align 2
+  %shl = shl i16 %0, %1
+  store i16 %shl, i16* @s3, align 2
+; CHECK-LABEL:  sll:
+; CHECK:        lui     $[[REG_GPa:[0-9]+]], %hi(_gp_disp)
+; CHECK-DAG:    addiu   $[[REG_GPb:[0-9]+]], $[[REG_GPa]], %lo(_gp_disp)
+; CHECK-DAG:    addu    $[[REG_GP:[0-9]+]], $[[REG_GPb]], $25
+; CHECK-DAG:    lw      $[[S3_ADDR:[0-9]+]], %got(s3)($[[REG_GP]])
+; CHECK-DAG:    lw      $[[S2_ADDR:[0-9]+]], %got(s2)($[[REG_GP]])
+; CHECK-DAG:    lw      $[[S1_ADDR:[0-9]+]], %got(s1)($[[REG_GP]])
+; CHECK-DAG:    lhu     $[[S1:[0-9]+]], 0($[[S1_ADDR]])
+; CHECK-DAG:    lhu     $[[S2:[0-9]+]], 0($[[S2_ADDR]])
+; CHECK:        sllv    $[[RES:[0-9]+]], $[[S1]], $[[S2]]
+; CHECK:        sh      $[[RES]], 0($[[S3_ADDR]])
+  ret void
+}
+
+define void @slli() {
+entry:
+  %0 = load i16, i16* @s1, align 2
+  %shl = shl i16 %0, 5
+  store i16 %shl, i16* @s3, align 2
+; CHECK-LABEL:  slli:
+; CHECK:        lui     $[[REG_GPa:[0-9]+]], %hi(_gp_disp)
+; CHECK-DAG:    addiu   $[[REG_GPb:[0-9]+]], $[[REG_GPa]], %lo(_gp_disp)
+; CHECK-DAG:    addu    $[[REG_GP:[0-9]+]], $[[REG_GPb]], $25
+; CHECK-DAG:    lw      $[[S3_ADDR:[0-9]+]], %got(s3)($[[REG_GP]])
+; CHECK-DAG:    lw      $[[S1_ADDR:[0-9]+]], %got(s1)($[[REG_GP]])
+; CHECK-DAG:    lhu     $[[S1:[0-9]+]], 0($[[S1_ADDR]])
+; CHECK:        sll     $[[RES:[0-9]+]], $[[S1]],  5
+; CHECK:        sh      $[[RES]], 0($[[S3_ADDR]])
+  ret void
+}
+
+define void @srl() {
+entry:
+  %0 = load i16, i16* @us1, align 2
+  %1 = load i16, i16* @us2, align 2
+  %shr = lshr i16 %0, %1
+  store i16 %shr, i16* @us3, align 2
+  ret void
+; CHECK-LABEL:  srl:
+; CHECK:        lui     $[[REG_GPa:[0-9]+]], %hi(_gp_disp)
+; CHECK-DAG:    addiu   $[[REG_GPb:[0-9]+]], $[[REG_GPa]], %lo(_gp_disp)
+; CHECK-DAG:    addu    $[[REG_GP:[0-9]+]], $[[REG_GPb]], $25
+; CHECK-DAG:    lw      $[[US3_ADDR:[0-9]+]], %got(us3)($[[REG_GP]])
+; CHECK-DAG:    lw      $[[US2_ADDR:[0-9]+]], %got(us2)($[[REG_GP]])
+; CHECK-DAG:    lw      $[[US1_ADDR:[0-9]+]], %got(us1)($[[REG_GP]])
+; CHECK-DAG:    lhu     $[[US1:[0-9]+]], 0($[[US1_ADDR]])
+; CHECK-DAG:    lhu     $[[US2:[0-9]+]], 0($[[US2_ADDR]])
+; CHECK:        srlv    $[[RES:[0-9]+]], $[[US1]], $[[US2]]
+; CHECK:        sh      $[[RES]], 0($[[S3_ADDR]])
+}
+
+define void @srli() {
+entry:
+  %0 = load i16, i16* @us1, align 2
+  %shr = lshr i16 %0, 4
+  store i16 %shr, i16* @us3, align 2
+; CHECK-LABEL:  srli:
+; CHECK:        lui     $[[REG_GPa:[0-9]+]], %hi(_gp_disp)
+; CHECK-DAG:    addiu   $[[REG_GPb:[0-9]+]], $[[REG_GPa]], %lo(_gp_disp)
+; CHECK-DAG:    addu    $[[REG_GP:[0-9]+]], $[[REG_GPb]], $25
+; CHECK-DAG:    lw      $[[US3_ADDR:[0-9]+]], %got(us3)($[[REG_GP]])
+; CHECK-DAG:    lw      $[[US1_ADDR:[0-9]+]], %got(us1)($[[REG_GP]])
+; CHECK-DAG:    lhu     $[[US1:[0-9]+]], 0($[[US1_ADDR]])
+; CHECK:        srl     $[[RES:[0-9]+]], $[[US1]], 4
+; CHECK:        sh      $[[RES]], 0($[[S3_ADDR]])
+  ret void
+}
+
+define void @sra() {
+entry:
+  %0 = load i16, i16* @s1, align 2
+  %1 = load i16, i16* @s2, align 2
+  %shr = ashr i16 %0, %1
+  store i16 %shr, i16* @s3, align 2
+; CHECK-LABEL:  sra:
+; CHECK:        lui     $[[REG_GPa:[0-9]+]], %hi(_gp_disp)
+; CHECK-DAG:    addiu   $[[REG_GPb:[0-9]+]], $[[REG_GPa]], %lo(_gp_disp)
+; CHECK-DAG:    addu    $[[REG_GP:[0-9]+]], $[[REG_GPb]], $25
+; CHECK-DAG:    lw      $[[S3_ADDR:[0-9]+]], %got(s3)($[[REG_GP]])
+; CHECK-DAG:    lw      $[[S2_ADDR:[0-9]+]], %got(s2)($[[REG_GP]])
+; CHECK-DAG:    lw      $[[S1_ADDR:[0-9]+]], %got(s1)($[[REG_GP]])
+; CHECK-DAG:    lhu     $[[S1:[0-9]+]], 0($[[S1_ADDR]])
+; CHECK-DAG:    lhu     $[[S2:[0-9]+]], 0($[[S2_ADDR]])
+; CHECK:        srav    $[[RES:[0-9]+]], $[[S1]], $[[S2]]
+; CHECK:        sh      $[[RES]], 0($[[S3_ADDR]])
+  ret void
+}
+
+define void @srai() {
+entry:
+  %0 = load i16, i16* @s1, align 2
+  %shr = ashr i16 %0, 2
+  store i16 %shr, i16* @s3, align 2
+; CHECK-LABEL:  srai:
+; CHECK:        lui     $[[REG_GPa:[0-9]+]], %hi(_gp_disp)
+; CHECK-DAG:    addiu   $[[REG_GPb:[0-9]+]], $[[REG_GPa]], %lo(_gp_disp)
+; CHECK-DAG:    addu    $[[REG_GP:[0-9]+]], $[[REG_GPb]], $25
+; CHECK-DAG:    lw      $[[S3_ADDR:[0-9]+]], %got(s3)($[[REG_GP]])
+; CHECK-DAG:    lw      $[[S1_ADDR:[0-9]+]], %got(s1)($[[REG_GP]])
+; CHECK-DAG:    lhu     $[[S1:[0-9]+]], 0($[[S1_ADDR]])
+; CHECK:        sra     $[[RES:[0-9]+]], $[[S1]], 2
+; CHECK:        sh      $[[RES]], 0($[[S3_ADDR]])
+  ret void
+}
diff --git a/test/CodeGen/Mips/Fast-ISel/shift.ll b/test/CodeGen/Mips/Fast-ISel/shift.ll
index df1c82700d5..9fe694bb582 100644
--- a/test/CodeGen/Mips/Fast-ISel/shift.ll
+++ b/test/CodeGen/Mips/Fast-ISel/shift.ll
@@ -1,4 +1,4 @@
-; RUN: llc -march=mipsel -mcpu=mips32r2 -O1 -fast-isel=true -mips-fast-isel -filetype=obj %s -o - \
+; RUN: llc -march=mipsel -mcpu=mips32r2 -O0 -fast-isel=true -filetype=obj %s -o - \
 ; RUN:   | llvm-objdump -arch mipsel -mcpu=mips32r2 -d - | FileCheck %s
 
 ; This test checks that encoding for srl is correct when fast-isel for mips32r2 is used.
diff --git a/test/CodeGen/Mips/Fast-ISel/simplestore.ll b/test/CodeGen/Mips/Fast-ISel/simplestore.ll
index bcb198b1a82..627a383f597 100644
--- a/test/CodeGen/Mips/Fast-ISel/simplestore.ll
+++ b/test/CodeGen/Mips/Fast-ISel/simplestore.ll
@@ -1,6 +1,6 @@
-; RUN: llc -march=mipsel -relocation-model=pic -O0 -mips-fast-isel -fast-isel-abort=1 -mcpu=mips32r2 \
+; RUN: llc -march=mipsel -relocation-model=pic -O0 -fast-isel-abort=1 -mcpu=mips32r2 \
 ; RUN:     < %s | FileCheck %s
-; RUN: llc -march=mipsel -relocation-model=pic -O0 -mips-fast-isel -fast-isel-abort=1 -mcpu=mips32 \
+; RUN: llc -march=mipsel -relocation-model=pic -O0 -fast-isel-abort=1 -mcpu=mips32 \
 ; RUN:     < %s | FileCheck %s
 
 @abcd = external global i32
diff --git a/test/CodeGen/Mips/Fast-ISel/simplestorefp1.ll b/test/CodeGen/Mips/Fast-ISel/simplestorefp1.ll
index f4b91d85025..62101d8ef7e 100644
--- a/test/CodeGen/Mips/Fast-ISel/simplestorefp1.ll
+++ b/test/CodeGen/Mips/Fast-ISel/simplestorefp1.ll
@@ -1,10 +1,10 @@
-; RUN: llc -march=mipsel -relocation-model=pic -O0 -mips-fast-isel -fast-isel-abort=1 -mcpu=mips32r2 \
+; RUN: llc -march=mipsel -relocation-model=pic -O0 -fast-isel-abort=1 -mcpu=mips32r2 \
 ; RUN:     < %s | FileCheck %s 
-; RUN: llc -march=mipsel -relocation-model=pic -O0 -mips-fast-isel -fast-isel-abort=1 -mcpu=mips32 \
+; RUN: llc -march=mipsel -relocation-model=pic -O0 -fast-isel-abort=1 -mcpu=mips32 \
 ; RUN:     < %s | FileCheck %s
-; RUN: llc -march=mipsel -relocation-model=pic -O0 -mips-fast-isel -fast-isel-abort=1 -mcpu=mips32r2 \
+; RUN: llc -march=mipsel -relocation-model=pic -O0 -fast-isel-abort=1 -mcpu=mips32r2 \
 ; RUN:     < %s | FileCheck %s -check-prefix=mips32r2 
-; RUN: llc -march=mipsel -relocation-model=pic -O0 -mips-fast-isel -fast-isel-abort=1 -mcpu=mips32 \
+; RUN: llc -march=mipsel -relocation-model=pic -O0 -fast-isel-abort=1 -mcpu=mips32 \
 ; RUN:     < %s | FileCheck %s -check-prefix=mips32
 
 @f = common global float 0.000000e+00, align 4
diff --git a/test/CodeGen/Mips/Fast-ISel/simplestorei.ll b/test/CodeGen/Mips/Fast-ISel/simplestorei.ll
index 83ccae0b1de..67541b54bae 100644
--- a/test/CodeGen/Mips/Fast-ISel/simplestorei.ll
+++ b/test/CodeGen/Mips/Fast-ISel/simplestorei.ll
@@ -1,6 +1,6 @@
-; RUN: llc -march=mipsel -relocation-model=pic -O0 -mips-fast-isel -fast-isel-abort=1 -mcpu=mips32r2 \
+; RUN: llc -march=mipsel -relocation-model=pic -O0 -fast-isel-abort=1 -mcpu=mips32r2 \
 ; RUN:     < %s | FileCheck %s
-; RUN: llc -march=mipsel -relocation-model=pic -O0 -mips-fast-isel -fast-isel-abort=1 -mcpu=mips32 \
+; RUN: llc -march=mipsel -relocation-model=pic -O0 -fast-isel-abort=1 -mcpu=mips32 \
 ; RUN:     < %s | FileCheck %s
 
 @ijk = external global i32
@@ -22,9 +22,10 @@ define void @si2_2() #0 {
 entry:
   store i32 -32768, i32* @ijk, align 4
 ; CHECK:        .ent    si2_2
-; CHECK:        addiu   $[[REG1:[0-9]+]], $zero, -32768
-; CHECK:        lw      $[[REG2:[0-9]+]], %got(ijk)(${{[0-9]+}})
-; CHECK:        sw      $[[REG1]], 0($[[REG2]])
+; CHECK:        lui     $[[REG1:[0-9]+]], 65535
+; CHECK:        ori     $[[REG2:[0-9]+]], $[[REG1]], 32768
+; CHECK:        lw      $[[REG3:[0-9]+]], %got(ijk)(${{[0-9]+}})
+; CHECK:        sw      $[[REG2]], 0($[[REG3]])
   ret void
 }
 
diff --git a/test/CodeGen/Mips/beqzc.ll b/test/CodeGen/Mips/beqzc.ll
index afb66a915b3..37bece88421 100644
--- a/test/CodeGen/Mips/beqzc.ll
+++ b/test/CodeGen/Mips/beqzc.ll
@@ -1,4 +1,4 @@
-; RUN: llc -mtriple=mipsel-linux-gnu -march=mipsel -mcpu=mips16 -soft-float -mips16-hard-float -relocation-model=pic -mips16-constant-islands   < %s | FileCheck %s -check-prefix=cond-b-short
+; RUN: llc -mtriple=mipsel-linux-gnu -march=mipsel -mcpu=mips16 -mattr=+soft-float -mips16-hard-float -relocation-model=pic -mips16-constant-islands   < %s | FileCheck %s -check-prefix=cond-b-short
 
 @i = global i32 0, align 4
 @j = common global i32 0, align 4
diff --git a/test/CodeGen/Mips/beqzc1.ll b/test/CodeGen/Mips/beqzc1.ll
index fe0dd2a3ce9..1f5575f099f 100644
--- a/test/CodeGen/Mips/beqzc1.ll
+++ b/test/CodeGen/Mips/beqzc1.ll
@@ -1,4 +1,4 @@
-; RUN: llc -mtriple=mipsel-linux-gnu -march=mipsel -mcpu=mips16 -soft-float -mips16-hard-float -relocation-model=pic -mips16-constant-islands   < %s | FileCheck %s -check-prefix=cond-b-short
+; RUN: llc -mtriple=mipsel-linux-gnu -march=mipsel -mcpu=mips16 -mattr=+soft-float -mips16-hard-float -relocation-model=pic -mips16-constant-islands   < %s | FileCheck %s -check-prefix=cond-b-short
 
 @i = global i32 0, align 4
 @j = common global i32 0, align 4
diff --git a/test/CodeGen/Mips/brsize3.ll b/test/CodeGen/Mips/brsize3.ll
index 3620868bb2f..dad0d841d4c 100644
--- a/test/CodeGen/Mips/brsize3.ll
+++ b/test/CodeGen/Mips/brsize3.ll
@@ -1,6 +1,6 @@
-; RUN: llc -mtriple=mipsel-linux-gnu -march=mipsel -mcpu=mips16 -soft-float -mips16-hard-float -relocation-model=pic -mips16-constant-islands   < %s | FileCheck %s -check-prefix=b-no-short
+; RUN: llc -mtriple=mipsel-linux-gnu -march=mipsel -mcpu=mips16 -mattr=+soft-float -mips16-hard-float -relocation-model=pic -mips16-constant-islands   < %s | FileCheck %s -check-prefix=b-no-short
 
-; RUN: llc -mtriple=mipsel-linux-gnu -march=mipsel -mcpu=mips16 -soft-float -mips16-hard-float -relocation-model=pic -mips16-constant-islands   < %s | FileCheck %s -check-prefix=b-long
+; RUN: llc -mtriple=mipsel-linux-gnu -march=mipsel -mcpu=mips16 -mattr=+soft-float -mips16-hard-float -relocation-model=pic -mips16-constant-islands   < %s | FileCheck %s -check-prefix=b-long
 
 ; ModuleID = 'brsize3.c'
 target datalayout = "E-p:32:32:32-i1:8:8-i8:8:32-i16:16:32-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-n32-S64"
diff --git a/test/CodeGen/Mips/brsize3a.ll b/test/CodeGen/Mips/brsize3a.ll
index f05e2119192..e1cd5893ced 100644
--- a/test/CodeGen/Mips/brsize3a.ll
+++ b/test/CodeGen/Mips/brsize3a.ll
@@ -1,4 +1,4 @@
-; RUN: llc -mtriple=mipsel-linux-gnu -march=mipsel -mcpu=mips16 -soft-float -mips16-hard-float -relocation-model=pic -mips16-constant-islands   < %s | FileCheck %s -check-prefix=b-short
+; RUN: llc -mtriple=mipsel-linux-gnu -march=mipsel -mcpu=mips16 -mattr=+soft-float -mips16-hard-float -relocation-model=pic -mips16-constant-islands   < %s | FileCheck %s -check-prefix=b-short
 
 ; ModuleID = 'brsize3.c'
 target datalayout = "E-p:32:32:32-i1:8:8-i8:8:32-i16:16:32-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-n32-S64"
diff --git a/test/CodeGen/Mips/cconv/arguments-float.ll b/test/CodeGen/Mips/cconv/arguments-float.ll
index 156d6f1ee66..c81c7215e16 100644
--- a/test/CodeGen/Mips/cconv/arguments-float.ll
+++ b/test/CodeGen/Mips/cconv/arguments-float.ll
@@ -1,14 +1,14 @@
-; RUN: llc -march=mips -relocation-model=static -soft-float < %s | FileCheck --check-prefix=ALL --check-prefix=SYM32 --check-prefix=O32 --check-prefix=O32BE %s
-; RUN: llc -march=mipsel -relocation-model=static -soft-float < %s | FileCheck --check-prefix=ALL --check-prefix=SYM32 --check-prefix=O32 --check-prefix=O32LE %s
+; RUN: llc -march=mips -relocation-model=static -mattr=+soft-float < %s | FileCheck --check-prefix=ALL --check-prefix=SYM32 --check-prefix=O32 --check-prefix=O32BE %s
+; RUN: llc -march=mipsel -relocation-model=static -mattr=+soft-float < %s | FileCheck --check-prefix=ALL --check-prefix=SYM32 --check-prefix=O32 --check-prefix=O32LE %s
 
-; RUN-TODO: llc -march=mips64 -relocation-model=static -soft-float -target-abi o32 < %s | FileCheck --check-prefix=ALL --check-prefix=SYM32 --check-prefix=O32 %s
-; RUN-TODO: llc -march=mips64el -relocation-model=static -soft-float -target-abi o32 < %s | FileCheck --check-prefix=ALL --check-prefix=SYM32 --check-prefix=O32 %s
+; RUN-TODO: llc -march=mips64 -relocation-model=static -mattr=+soft-float -target-abi o32 < %s | FileCheck --check-prefix=ALL --check-prefix=SYM32 --check-prefix=O32 %s
+; RUN-TODO: llc -march=mips64el -relocation-model=static -mattr=+soft-float -target-abi o32 < %s | FileCheck --check-prefix=ALL --check-prefix=SYM32 --check-prefix=O32 %s
 
-; RUN: llc -march=mips64 -relocation-model=static -soft-float -target-abi n32 < %s | FileCheck --check-prefix=ALL --check-prefix=SYM32 --check-prefix=NEW %s
-; RUN: llc -march=mips64el -relocation-model=static -soft-float -target-abi n32 < %s | FileCheck --check-prefix=ALL --check-prefix=SYM32 --check-prefix=NEW %s
+; RUN: llc -march=mips64 -relocation-model=static -mattr=+soft-float -target-abi n32 < %s | FileCheck --check-prefix=ALL --check-prefix=SYM32 --check-prefix=NEW %s
+; RUN: llc -march=mips64el -relocation-model=static -mattr=+soft-float -target-abi n32 < %s | FileCheck --check-prefix=ALL --check-prefix=SYM32 --check-prefix=NEW %s
 
-; RUN: llc -march=mips64 -relocation-model=static -soft-float -target-abi n64 < %s | FileCheck --check-prefix=ALL --check-prefix=SYM64 --check-prefix=NEW %s
-; RUN: llc -march=mips64el -relocation-model=static -soft-float -target-abi n64 < %s | FileCheck --check-prefix=ALL --check-prefix=SYM64 --check-prefix=NEW %s
+; RUN: llc -march=mips64 -relocation-model=static -mattr=+soft-float -target-abi n64 < %s | FileCheck --check-prefix=ALL --check-prefix=SYM64 --check-prefix=NEW %s
+; RUN: llc -march=mips64el -relocation-model=static -mattr=+soft-float -target-abi n64 < %s | FileCheck --check-prefix=ALL --check-prefix=SYM64 --check-prefix=NEW %s
 
 ; Test the floating point arguments for all ABI's and byte orders as specified
 ; by section 5 of MD00305 (MIPS ABIs Described).
diff --git a/test/CodeGen/Mips/cconv/arguments-fp128.ll b/test/CodeGen/Mips/cconv/arguments-fp128.ll
index fabc107917f..6c62609396c 100644
--- a/test/CodeGen/Mips/cconv/arguments-fp128.ll
+++ b/test/CodeGen/Mips/cconv/arguments-fp128.ll
@@ -1,8 +1,8 @@
-; RUN: llc -march=mips64 -relocation-model=static -soft-float -target-abi n32 < %s | FileCheck --check-prefix=ALL --check-prefix=SYM32 %s
-; RUN: llc -march=mips64el -relocation-model=static -soft-float -target-abi n32 < %s | FileCheck --check-prefix=ALL --check-prefix=SYM32 %s
+; RUN: llc -march=mips64 -relocation-model=static -mattr=+soft-float -target-abi n32 < %s | FileCheck --check-prefix=ALL --check-prefix=SYM32 %s
+; RUN: llc -march=mips64el -relocation-model=static -mattr=+soft-float -target-abi n32 < %s | FileCheck --check-prefix=ALL --check-prefix=SYM32 %s
 
-; RUN: llc -march=mips64 -relocation-model=static -soft-float -target-abi n64 < %s | FileCheck --check-prefix=ALL --check-prefix=SYM64 %s
-; RUN: llc -march=mips64el -relocation-model=static -soft-float -target-abi n64 < %s | FileCheck --check-prefix=ALL --check-prefix=SYM64 %s
+; RUN: llc -march=mips64 -relocation-model=static -mattr=+soft-float -target-abi n64 < %s | FileCheck --check-prefix=ALL --check-prefix=SYM64 %s
+; RUN: llc -march=mips64el -relocation-model=static -mattr=+soft-float -target-abi n64 < %s | FileCheck --check-prefix=ALL --check-prefix=SYM64 %s
 
 ; Test the fp128 arguments for all ABI's and byte orders as specified
 ; by section 2 of the MIPSpro N32 Handbook.
diff --git a/test/CodeGen/Mips/cconv/callee-saved.ll b/test/CodeGen/Mips/cconv/callee-saved.ll
index d0b1e64cdee..0570ab35fd0 100644
--- a/test/CodeGen/Mips/cconv/callee-saved.ll
+++ b/test/CodeGen/Mips/cconv/callee-saved.ll
@@ -18,7 +18,7 @@
 ; RUN: llc -march=mips64 -target-abi n64 < %s | FileCheck --check-prefix=ALL --check-prefix=N64-INV %s
 ; RUN: llc -march=mips64el -target-abi n64 < %s | FileCheck --check-prefix=ALL --check-prefix=N64-INV %s
 
-; Test the the callee-saved registers are callee-saved as specified by section
+; Test the callee-saved registers are callee-saved as specified by section
 ; 2 of the MIPSpro N32 Handbook and section 3 of the SYSV ABI spec.
 
 define void @gpr_clobber() nounwind {
diff --git a/test/CodeGen/Mips/cconv/return-float.ll b/test/CodeGen/Mips/cconv/return-float.ll
index 4355a55232e..8eb8c411e08 100644
--- a/test/CodeGen/Mips/cconv/return-float.ll
+++ b/test/CodeGen/Mips/cconv/return-float.ll
@@ -1,14 +1,14 @@
-; RUN: llc -mtriple=mips-linux-gnu -soft-float -relocation-model=static < %s | FileCheck --check-prefix=ALL --check-prefix=O32 %s
-; RUN: llc -mtriple=mipsel-linux-gnu -soft-float -relocation-model=static < %s | FileCheck --check-prefix=ALL --check-prefix=O32 %s
+; RUN: llc -mtriple=mips-linux-gnu -mattr=+soft-float -relocation-model=static < %s | FileCheck --check-prefix=ALL --check-prefix=O32 %s
+; RUN: llc -mtriple=mipsel-linux-gnu -mattr=+soft-float -relocation-model=static < %s | FileCheck --check-prefix=ALL --check-prefix=O32 %s
 
-; RUN-TODO: llc -mtriple=mips64-linux-gnu -soft-float -relocation-model=static -target-abi o32 < %s | FileCheck --check-prefix=ALL --check-prefix=O32 %s
-; RUN-TODO: llc -mtriple=mips64el-linux-gnu -soft-float -relocation-model=static -target-abi o32 < %s | FileCheck --check-prefix=ALL --check-prefix=O32 %s
+; RUN-TODO: llc -mtriple=mips64-linux-gnu -mattr=+soft-float -relocation-model=static -target-abi o32 < %s | FileCheck --check-prefix=ALL --check-prefix=O32 %s
+; RUN-TODO: llc -mtriple=mips64el-linux-gnu -mattr=+soft-float -relocation-model=static -target-abi o32 < %s | FileCheck --check-prefix=ALL --check-prefix=O32 %s
 
-; RUN: llc -mtriple=mips64-linux-gnu -soft-float -relocation-model=static -target-abi n32 < %s | FileCheck --check-prefix=ALL --check-prefix=N32 %s
-; RUN: llc -mtriple=mips64el-linux-gnu -soft-float -relocation-model=static -target-abi n32 < %s | FileCheck --check-prefix=ALL --check-prefix=N32 %s
+; RUN: llc -mtriple=mips64-linux-gnu -mattr=+soft-float -relocation-model=static -target-abi n32 < %s | FileCheck --check-prefix=ALL --check-prefix=N32 %s
+; RUN: llc -mtriple=mips64el-linux-gnu -mattr=+soft-float -relocation-model=static -target-abi n32 < %s | FileCheck --check-prefix=ALL --check-prefix=N32 %s
 
-; RUN: llc -mtriple=mips64-linux-gnu -soft-float -relocation-model=static -target-abi n64 < %s | FileCheck --check-prefix=ALL --check-prefix=N64 %s
-; RUN: llc -mtriple=mips64el-linux-gnu -soft-float -relocation-model=static -target-abi n64 < %s | FileCheck --check-prefix=ALL --check-prefix=N64 %s
+; RUN: llc -mtriple=mips64-linux-gnu -mattr=+soft-float -relocation-model=static -target-abi n64 < %s | FileCheck --check-prefix=ALL --check-prefix=N64 %s
+; RUN: llc -mtriple=mips64el-linux-gnu -mattr=+soft-float -relocation-model=static -target-abi n64 < %s | FileCheck --check-prefix=ALL --check-prefix=N64 %s
 
 ; Test the float returns for all ABI's and byte orders as specified by
 ; section 5 of MD00305 (MIPS ABIs Described).
diff --git a/test/CodeGen/Mips/ci2.ll b/test/CodeGen/Mips/ci2.ll
index 63ed68387a2..4687748879a 100644
--- a/test/CodeGen/Mips/ci2.ll
+++ b/test/CodeGen/Mips/ci2.ll
@@ -1,4 +1,4 @@
-; RUN: llc -mtriple=mipsel-linux-gnu -march=mipsel -mcpu=mips16 -soft-float -mips16-hard-float -relocation-model=static -mips16-constant-islands   < %s | FileCheck %s -check-prefix=constisle
+; RUN: llc -mtriple=mipsel-linux-gnu -march=mipsel -mcpu=mips16 -mattr=+soft-float -mips16-hard-float -relocation-model=static -mips16-constant-islands   < %s | FileCheck %s -check-prefix=constisle
 
 @i = common global i32 0, align 4
 @b = common global i32 0, align 4
diff --git a/test/CodeGen/Mips/const1.ll b/test/CodeGen/Mips/const1.ll
index f32ce244cf4..1a5d58bd3f9 100644
--- a/test/CodeGen/Mips/const1.ll
+++ b/test/CodeGen/Mips/const1.ll
@@ -1,4 +1,4 @@
-; RUN: llc -mtriple=mipsel-linux-gnu -march=mipsel -mcpu=mips16 -soft-float -mips16-hard-float -relocation-model=static -mips16-constant-islands < %s | FileCheck %s 
+; RUN: llc -mtriple=mipsel-linux-gnu -march=mipsel -mcpu=mips16 -mattr=+soft-float -mips16-hard-float -relocation-model=static -mips16-constant-islands < %s | FileCheck %s 
 
 ; ModuleID = 'const1.c'
 target datalayout = "e-p:32:32:32-i1:8:8-i8:8:32-i16:16:32-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-n32-S64"
diff --git a/test/CodeGen/Mips/const4a.ll b/test/CodeGen/Mips/const4a.ll
index 9022eb44a8a..c31e54a0103 100644
--- a/test/CodeGen/Mips/const4a.ll
+++ b/test/CodeGen/Mips/const4a.ll
@@ -1,4 +1,4 @@
-; RUN: llc -mtriple=mipsel-linux-gnu -march=mipsel -mcpu=mips16 -soft-float -mips16-hard-float -relocation-model=pic -mips16-constant-islands -mips-constant-islands-no-load-relaxation  < %s | FileCheck %s -check-prefix=no-load-relax
+; RUN: llc -mtriple=mipsel-linux-gnu -march=mipsel -mcpu=mips16 -mattr=+soft-float -mips16-hard-float -relocation-model=pic -mips16-constant-islands -mips-constant-islands-no-load-relaxation  < %s | FileCheck %s -check-prefix=no-load-relax
 
 ; ModuleID = 'const4.c'
 target datalayout = "E-p:32:32:32-i1:8:8-i8:8:32-i16:16:32-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-n32-S64"
diff --git a/test/CodeGen/Mips/const6.ll b/test/CodeGen/Mips/const6.ll
index c26e02f2ebb..49e98ea7870 100644
--- a/test/CodeGen/Mips/const6.ll
+++ b/test/CodeGen/Mips/const6.ll
@@ -1,6 +1,6 @@
-; RUN: llc -mtriple=mipsel-linux-gnu -march=mipsel -mcpu=mips16 -soft-float -mips16-hard-float -relocation-model=pic -mips16-constant-islands   < %s | FileCheck %s -check-prefix=load-relax
+; RUN: llc -mtriple=mipsel-linux-gnu -march=mipsel -mcpu=mips16 -mattr=+soft-float -mips16-hard-float -relocation-model=pic -mips16-constant-islands   < %s | FileCheck %s -check-prefix=load-relax
 
-; RUN: llc -mtriple=mipsel-linux-gnu -march=mipsel -mcpu=mips16 -soft-float -mips16-hard-float -relocation-model=pic -mips16-constant-islands -mips-constant-islands-no-load-relaxation  < %s | FileCheck %s -check-prefix=no-load-relax
+; RUN: llc -mtriple=mipsel-linux-gnu -march=mipsel -mcpu=mips16 -mattr=+soft-float -mips16-hard-float -relocation-model=pic -mips16-constant-islands -mips-constant-islands-no-load-relaxation  < %s | FileCheck %s -check-prefix=no-load-relax
 
 ; ModuleID = 'const6.c'
 target datalayout = "E-p:32:32:32-i1:8:8-i8:8:32-i16:16:32-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-n32-S64"
diff --git a/test/CodeGen/Mips/const6a.ll b/test/CodeGen/Mips/const6a.ll
index aff1357c3a8..54a3f2234dc 100644
--- a/test/CodeGen/Mips/const6a.ll
+++ b/test/CodeGen/Mips/const6a.ll
@@ -1,6 +1,6 @@
-; RUN: llc -mtriple=mipsel-linux-gnu -march=mipsel -mcpu=mips16 -soft-float -mips16-hard-float -relocation-model=pic -mips16-constant-islands   < %s | FileCheck %s -check-prefix=load-relax1
+; RUN: llc -mtriple=mipsel-linux-gnu -march=mipsel -mcpu=mips16 -mattr=+soft-float -mips16-hard-float -relocation-model=pic -mips16-constant-islands   < %s | FileCheck %s -check-prefix=load-relax1
 
-; RUN: llc -mtriple=mipsel-linux-gnu -march=mipsel -mcpu=mips16 -soft-float -mips16-hard-float -relocation-model=pic -mips16-constant-islands   < %s | FileCheck %s -check-prefix=load-relax
+; RUN: llc -mtriple=mipsel-linux-gnu -march=mipsel -mcpu=mips16 -mattr=+soft-float -mips16-hard-float -relocation-model=pic -mips16-constant-islands   < %s | FileCheck %s -check-prefix=load-relax
 
 ; ModuleID = 'const6a.c'
 target datalayout = "E-p:32:32:32-i1:8:8-i8:8:32-i16:16:32-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-n32-S64"
diff --git a/test/CodeGen/Mips/delay-slot-fill-forward.ll b/test/CodeGen/Mips/delay-slot-fill-forward.ll
new file mode 100644
index 00000000000..7fc011da92d
--- /dev/null
+++ b/test/CodeGen/Mips/delay-slot-fill-forward.ll
@@ -0,0 +1,183 @@
+; RUN: llc < %s -march=mips -mcpu=mips32r2 -O2 \
+; RUN:     -disable-mips-df-forward-search=false \
+; RUN:     -disable-mips-df-succbb-search=false \
+; RUN:     -relocation-model=static | FileCheck %s
+
+; This test was generated with bugpoint from
+; MultiSource/Applications/JM/lencod/me_fullsearch.c
+
+%struct.SubImageContainer = type { i16****, [2 x i16****] }
+%struct.storable_picture = type { i32, i32, i32, i32, i32, i32,
+  [6 x [33 x i64]], [6 x [33 x i64]], [6 x [33 x i64]], [6 x [33 x i64]],
+  i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32,
+  i32, i32, i32, i32, i32, i16**, i16****, i16****, i16*****, i16***,
+  i8*, i8***, i64***, i64***, i16****, i8**, i8**, %struct.storable_picture*,
+  %struct.storable_picture*, %struct.storable_picture*,
+  i32, i32, i32, i32, i32, i32, i32 }
+
+@img_height = external global i16, align 2
+@width_pad = external global i32, align 4
+@height_pad = external global i32, align 4
+@mvbits = external global i32*, align 4
+@ref_pic1_sub = external global %struct.SubImageContainer, align 4
+@ref_pic2_sub = external global %struct.SubImageContainer, align 4
+@wbp_weight = external global i32****, align 4
+@weight1 = external global i16, align 2
+@weight2 = external global i16, align 2
+@offsetBi = external global i16, align 2
+@computeBiPred2 = external global [3 x i32 (i16*, i32, i32, i32, i32, i32, i32, i32)*], align 4
+@computeBiPred = external global i32 (i16*, i32, i32, i32, i32, i32, i32, i32)*, align 4
+@bipred2_access_method = external global i32, align 4
+@start_me_refinement_hp = external global i32, align 4
+
+declare i32 @foobar(i16*, i32 signext , i32 signext , i32 signext ,
+                    i32 signext , i32 signext , i32 signext , i32 signext ) #1
+
+define void @SubPelBlockSearchBiPred(i16* %orig_pic, i16 signext %ref,
+    i32 signext %pic_pix_x, i32 signext %pic_pix_y, i16 signext %pred_mv_y,
+    i16* nocapture %mv_x, i16* nocapture %mv_y, i16* nocapture readonly %s_mv_x,
+    i32 signext %search_pos2, i32 signext %min_mcost) #0 {
+; CHECK-LABEL: SubPelBlockSearchBiPred:
+entry:
+  %add40 = shl i32 %pic_pix_x, 2
+  %shl = add i32 %add40, 80
+  %add41 = shl i32 %pic_pix_y, 2
+  %0 = load i32, i32* @start_me_refinement_hp, align 4, !tbaa !1
+  %cond47 = select i1 undef, i32 1, i32 %search_pos2
+  %1 = load i16, i16* %s_mv_x, align 2, !tbaa !5
+  %conv48 = sext i16 %1 to i32
+  %add49 = add nsw i32 %conv48, %shl
+  %idxprom52 = sext i16 %ref to i32
+  %2 = load i32, i32* null, align 4, !tbaa !1
+  store i32 undef, i32* bitcast (%struct.SubImageContainer* @ref_pic1_sub to i32*), align 4, !tbaa !7
+  %3 = load i32, i32* undef, align 4, !tbaa !10
+  store i32 %3, i32* bitcast (%struct.SubImageContainer* @ref_pic2_sub to i32*), align 4, !tbaa !7
+  store i16 0, i16* @img_height, align 2, !tbaa !5
+  %size_x_pad = getelementptr inbounds %struct.storable_picture, %struct.storable_picture* null, i32 0, i32 22
+  %4 = load i32, i32* %size_x_pad, align 4, !tbaa !12
+  store i32 %4, i32* @width_pad, align 4, !tbaa !1
+  %5 = load i32, i32* undef, align 4, !tbaa !13
+  store i32 %5, i32* @height_pad, align 4, !tbaa !1
+  %6 = load i32****, i32***** @wbp_weight, align 4, !tbaa !14
+  %arrayidx75 = getelementptr inbounds i32***, i32**** %6, i32 undef
+  %7 = load i32***, i32**** %arrayidx75, align 4, !tbaa !14
+  %arrayidx76 = getelementptr inbounds i32**, i32*** %7, i32 %idxprom52
+  %8 = load i32**, i32*** %arrayidx76, align 4, !tbaa !14
+  %cond87.in671 = load i32*, i32** %8, align 4
+  %cond87672 = load i32, i32* %cond87.in671, align 4
+  %conv88673 = trunc i32 %cond87672 to i16
+  store i16 %conv88673, i16* @weight1, align 2, !tbaa !5
+  %cond105 = load i32, i32* undef, align 4
+  %conv106 = trunc i32 %cond105 to i16
+  store i16 %conv106, i16* @weight2, align 2, !tbaa !5
+  store i16 0, i16* @offsetBi, align 2, !tbaa !5
+  %storemerge655 = load i32, i32* bitcast (i32 (i16*, i32, i32, i32, i32, i32, i32, i32)** getelementptr inbounds ([3 x i32 (i16*, i32, i32, i32, i32, i32, i32, i32)*], [3 x i32 (i16*, i32, i32, i32, i32, i32, i32, i32)*]* @computeBiPred2, i32 0, i32 1) to i32*), align 4
+  store i32 %storemerge655, i32* bitcast (i32 (i16*, i32, i32, i32, i32, i32, i32, i32)** @computeBiPred to i32*), align 4, !tbaa !14
+  %9 = load i16, i16* %mv_x, align 2, !tbaa !5
+  %cmp270 = icmp sgt i32 undef, 1
+  %or.cond = and i1 %cmp270, false
+  br i1 %or.cond, label %land.lhs.true277, label %if.else289
+
+land.lhs.true277:                                 ; preds = %entry
+  %10 = load i16, i16* %mv_y, align 2, !tbaa !5
+  %conv278 = sext i16 %10 to i32
+  %add279 = add nsw i32 %conv278, 0
+  %cmp280 = icmp sgt i32 %add279, 1
+  %or.cond660 = and i1 %cmp280, undef
+  br i1 %or.cond660, label %if.end290, label %if.else289
+
+if.else289:                                       ; preds = %land.lhs.true277, %entry
+  br label %if.end290
+
+if.end290:                                        ; preds = %if.else289, %land.lhs.true277
+  %storemerge = phi i32 [ 1, %if.else289 ], [ 0, %land.lhs.true277 ]
+  store i32 %storemerge, i32* @bipred2_access_method, align 4, !tbaa !1
+  %cmp315698 = icmp slt i32 %0, %cond47
+  br i1 %cmp315698, label %for.body.lr.ph, label %if.end358
+
+for.body.lr.ph:                                   ; preds = %if.end290
+  %conv328 = sext i16 %pred_mv_y to i32
+  br label %for.body
+
+for.body:                                         ; preds = %for.inc, %for.body.lr.ph
+  %11 = phi i16 [ %9, %for.body.lr.ph ], [ %.pre, %for.inc ]
+  %min_mcost.addr.0701 = phi i32 [ %min_mcost, %for.body.lr.ph ], [ undef, %for.inc ]
+  %pos.0700 = phi i32 [ %0, %for.body.lr.ph ], [ undef, %for.inc ]
+  %best_pos.0699 = phi i32 [ 0, %for.body.lr.ph ], [ %best_pos.1, %for.inc ]
+  %conv317 = sext i16 %11 to i32
+  %add320 = add nsw i32 0, %conv317
+  %12 = load i16, i16* %mv_y, align 2, !tbaa !5
+  %conv321 = sext i16 %12 to i32
+  %add324 = add nsw i32 0, %conv321
+  %13 = load i32*, i32** @mvbits, align 4, !tbaa !14
+  %14 = load i32, i32* undef, align 4, !tbaa !1
+  %sub329 = sub nsw i32 %add324, %conv328
+  %arrayidx330 = getelementptr inbounds i32, i32* %13, i32 %sub329
+  %15 = load i32, i32* %arrayidx330, align 4, !tbaa !1
+  %add331 = add nsw i32 %15, %14
+  %mul = mul nsw i32 %add331, %2
+  %shr332 = ashr i32 %mul, 16
+  %cmp333 = icmp sgt i32 %min_mcost.addr.0701, %shr332
+  br i1 %cmp333, label %if.end336, label %for.inc
+
+if.end336:                                        ; preds = %for.body
+  ; CHECK:      jalr  $25
+  ; CHECK-NOT:  move  $ra, {{.*}}
+  ; CHECK:      j     $BB{{.*}}
+  %add337 = add nsw i32 %add320, %shl
+  %add338 = add nsw i32 %add324, 0
+  %call340 = tail call i32 undef(i16* %orig_pic, i32 signext undef, i32 signext
+                                 undef, i32 signext 0, i32 signext %add49,
+                                 i32 signext undef, i32 signext %add337,
+                                 i32 signext %add338) #1
+  %cmp342 = icmp slt i32 0, %min_mcost.addr.0701
+  %pos.0.best_pos.0 = select i1 %cmp342, i32 %pos.0700, i32 %best_pos.0699
+  br label %for.inc
+
+for.inc:                                          ; preds = %if.end336, %for.body
+  %best_pos.1 = phi i32 [ %best_pos.0699, %for.body ], [ %pos.0.best_pos.0, %if.end336 ]
+  %.pre = load i16, i16* %mv_x, align 2, !tbaa !5
+  br label %for.body
+
+if.end358:                                        ; preds = %if.end290
+  %.min_mcost.addr.0 = select i1 false, i32 2147483647, i32 %min_mcost
+  br i1 undef, label %for.body415.lr.ph, label %if.end461
+
+for.body415.lr.ph:                                ; preds = %if.end358
+  %16 = load i16, i16* %mv_y, align 2, !tbaa !5
+  %conv420 = sext i16 %16 to i32
+  %add423 = add nsw i32 0, %conv420
+  %cmp433 = icmp sgt i32 %.min_mcost.addr.0, 0
+  br i1 %cmp433, label %if.end436, label %if.end461
+
+if.end436:                                        ; preds = %for.body415.lr.ph
+  %add438 = add nsw i32 %add423, 0
+  %call440 = tail call i32 @foobar(i16* %orig_pic, i32 signext undef, i32 signext undef,
+                                 i32 signext 0, i32 signext %add49, i32 signext undef,
+                                 i32 signext undef, i32 signext %add438) #1
+  br label %if.end461
+
+if.end461:                                        ; preds = %if.end436, %for.body415.lr.ph, %if.end358
+  ret void
+}
+
+attributes #0 = { nounwind "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="mips32r2" "target-features"="+mips32r2,+nooddspreg,+fpxx" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #1 = { nounwind }
+
+!llvm.ident = !{!0}
+
+!0 = !{!"clang version 3.7.0 (trunk 236218) (llvm/trunk 236237)"}
+!1 = !{!2, !2, i64 0}
+!2 = !{!"int", !3, i64 0}
+!3 = !{!"omnipotent char", !4, i64 0}
+!4 = !{!"Simple C/C++ TBAA"}
+!5 = !{!6, !6, i64 0}
+!6 = !{!"short", !3, i64 0}
+!7 = !{!8, !9, i64 0}
+!8 = !{!"", !9, i64 0, !3, i64 4}
+!9 = !{!"any pointer", !3, i64 0}
+!10 = !{!11, !9, i64 6440}
+!11 = !{!"storable_picture", !3, i64 0, !2, i64 4, !2, i64 8, !2, i64 12, !2, i64 16, !2, i64 20, !3, i64 24, !3, i64 1608, !3, i64 3192, !3, i64 4776, !2, i64 6360, !2, i64 6364, !2, i64 6368, !2, i64 6372, !2, i64 6376, !2, i64 6380, !2, i64 6384, !2, i64 6388, !2, i64 6392, !2, i64 6396, !2, i64 6400, !2, i64 6404, !2, i64 6408, !2, i64 6412, !2, i64 6416, !2, i64 6420, !2, i64 6424, !2, i64 6428, !2, i64 6432, !9, i64 6436, !9, i64 6440, !9, i64 6444, !9, i64 6448, !9, i64 6452, !9, i64 6456, !9, i64 6460, !9, i64 6464, !9, i64 6468, !9, i64 6472, !9, i64 6476, !9, i64 6480, !9, i64 6484, !9, i64 6488, !9, i64 6492, !2, i64 6496, !2, i64 6500, !2, i64 6504, !2, i64 6508, !2, i64 6512, !2, i64 6516, !2, i64 6520}
+!12 = !{!11, !2, i64 6408}
+!13 = !{!11, !2, i64 6412}
+!14 = !{!9, !9, i64 0}
diff --git a/test/CodeGen/Mips/delay-slot-kill.ll b/test/CodeGen/Mips/delay-slot-kill.ll
new file mode 100644
index 00000000000..57b630303c2
--- /dev/null
+++ b/test/CodeGen/Mips/delay-slot-kill.ll
@@ -0,0 +1,14 @@
+; RUN: llc < %s -march=mips64 -mcpu=mips3 | FileCheck %s
+
+; Currently, the following IR assembly generates a KILL instruction between
+; the bitwise-and instruction and the return instruction. We verify that the
+; delay slot filler ignores such KILL instructions by filling the slot of the
+; return instruction properly.
+define signext i32 @f1(i32 signext %a, i32 signext %b) {
+entry:
+  ; CHECK:          jr      $ra
+  ; CHECK-NEXT:     and     $2, $4, $5
+
+  %r = and i32 %a, %b
+  ret i32 %r
+}
diff --git a/test/CodeGen/Mips/dynamic-stack-realignment.ll b/test/CodeGen/Mips/dynamic-stack-realignment.ll
new file mode 100644
index 00000000000..777930a37ad
--- /dev/null
+++ b/test/CodeGen/Mips/dynamic-stack-realignment.ll
@@ -0,0 +1,299 @@
+; RUN: llc < %s -march=mips -mcpu=mips2 | FileCheck %s \
+; RUN:    --check-prefix=ALL --check-prefix=GP32
+; RUN: llc < %s -march=mips -mcpu=mips32 | FileCheck %s \
+; RUN:    --check-prefix=ALL --check-prefix=GP32
+; RUN: llc < %s -march=mips -mcpu=mips32r6 | FileCheck %s \
+; RUN:    --check-prefix=ALL --check-prefix=GP32
+; RUN: llc < %s -march=mips64 -mcpu=mips3 | FileCheck %s \
+; RUN:    --check-prefix=ALL --check-prefix=GP64 -check-prefix=N64
+; RUN: llc < %s -march=mips64 -mcpu=mips64 | FileCheck %s \
+; RUN:    --check-prefix=ALL --check-prefix=GP64 -check-prefix=N64
+; RUN: llc < %s -march=mips64 -mcpu=mips64r6 | FileCheck %s \
+; RUN:    --check-prefix=ALL --check-prefix=GP64 -check-prefix=N64
+; RUN: llc < %s -march=mips64 -mcpu=mips3 -target-abi n32 | FileCheck %s \
+; RUN:    --check-prefix=ALL --check-prefix=GP64 -check-prefix=N32
+; RUN: llc < %s -march=mips64 -mcpu=mips64 -target-abi n32 | FileCheck %s \
+; RUN:    --check-prefix=ALL --check-prefix=GP64 -check-prefix=N32
+; RUN: llc < %s -march=mips64 -mcpu=mips64r6 -target-abi n32 | FileCheck %s \
+; RUN:    --check-prefix=ALL --check-prefix=GP64 -check-prefix=N32
+
+; Check dynamic stack realignment in functions without variable-sized objects.
+
+declare void @helper_01(i32, i32, i32, i32, i32*)
+
+; O32 ABI
+define void @func_01() {
+entry:
+; GP32-LABEL: func_01:
+
+  ; prologue
+  ; FIXME: We are currently over-allocating stack space. This particular case
+  ;        needs a frame of up to between 16 and 512-bytes but currently
+  ;        allocates between 1024 and 1536 bytes
+  ; GP32:       addiu   $sp, $sp, -1024
+  ; GP32:       sw      $ra, 1020($sp)
+  ; GP32:       sw      $fp, 1016($sp)
+  ;
+  ; GP32:       move    $fp, $sp
+  ; GP32:       addiu   $[[T0:[0-9]+|ra|gp]], $zero, -512
+  ; GP32-NEXT:  and     $sp, $sp, $[[T0]]
+
+  ; body
+  ; GP32:       addiu   $[[T1:[0-9]+]], $sp, 512
+  ; GP32:       sw      $[[T1]], 16($sp)
+
+  ; epilogue
+  ; GP32:       move    $sp, $fp
+  ; GP32:       lw      $fp, 1016($sp)
+  ; GP32:       lw      $ra, 1020($sp)
+  ; GP32:       addiu   $sp, $sp, 1024
+
+  %a = alloca i32, align 512
+  call void @helper_01(i32 0, i32 0, i32 0, i32 0, i32* %a)
+  ret void
+}
+
+declare void @helper_02(i32, i32, i32, i32,
+                        i32, i32, i32, i32, i32*)
+
+; N32/N64 ABIs
+define void @func_02() {
+entry:
+; GP64-LABEL: func_02:
+
+  ; prologue
+  ; FIXME: We are currently over-allocating stack space. This particular case
+  ;        needs a frame of up to between 16 and 512-bytes but currently
+  ;        allocates between 1024 and 1536 bytes
+  ; N32:        addiu   $sp, $sp, -1024
+  ; N64:        daddiu  $sp, $sp, -1024
+  ; GP64:       sd      $ra, 1016($sp)
+  ; GP64:       sd      $fp, 1008($sp)
+  ; N32:        sd      $gp, 1000($sp)
+  ;
+  ; GP64:       move    $fp, $sp
+  ; N32:        addiu   $[[T0:[0-9]+|ra]], $zero, -512
+  ; N64:        daddiu  $[[T0:[0-9]+|ra]], $zero, -512
+  ; GP64-NEXT:  and     $sp, $sp, $[[T0]]
+
+  ; body
+  ; N32:        addiu   $[[T1:[0-9]+]], $sp, 512
+  ; N64:        daddiu  $[[T1:[0-9]+]], $sp, 512
+  ; GP64:       sd      $[[T1]], 0($sp)
+
+  ; epilogue
+  ; GP64:       move    $sp, $fp
+  ; N32:        ld      $gp, 1000($sp)
+  ; GP64:       ld      $fp, 1008($sp)
+  ; GP64:       ld      $ra, 1016($sp)
+  ; N32:        addiu   $sp, $sp, 1024
+  ; N64:        daddiu  $sp, $sp, 1024
+
+  %a = alloca i32, align 512
+  call void @helper_02(i32 0, i32 0, i32 0, i32 0,
+                       i32 0, i32 0, i32 0, i32 0, i32* %a)
+  ret void
+}
+
+; Verify that we use $fp for referencing incoming arguments.
+
+declare void @helper_03(i32, i32, i32, i32, i32*, i32*)
+
+; O32 ABI
+define void @func_03(i32 %p0, i32 %p1, i32 %p2, i32 %p3, i32* %b) {
+entry:
+; GP32-LABEL: func_03:
+
+  ; body
+  ; FIXME: We are currently over-allocating stack space.
+  ; GP32-DAG:   addiu   $[[T0:[0-9]+]], $sp, 512
+  ; GP32-DAG:   sw      $[[T0]], 16($sp)
+  ; GP32-DAG:   lw      $[[T1:[0-9]+]], 1040($fp)
+  ; GP32-DAG:   sw      $[[T1]], 20($sp)
+
+  %a = alloca i32, align 512
+  call void @helper_03(i32 0, i32 0, i32 0, i32 0, i32* %a, i32* %b)
+  ret void
+}
+
+declare void @helper_04(i32, i32, i32, i32,
+                        i32, i32, i32, i32, i32*, i32*)
+
+; N32/N64 ABIs
+define void @func_04(i32 %p0, i32 %p1, i32 %p2, i32 %p3,
+                     i32 %p4, i32 %p5, i32 %p6, i32 %p7,
+                     i32* %b) {
+entry:
+; GP64-LABEL: func_04:
+
+  ; body
+  ; FIXME: We are currently over-allocating stack space.
+  ; N32-DAG:    addiu   $[[T0:[0-9]+]], $sp, 512
+  ; N64-DAG:    daddiu  $[[T0:[0-9]+]], $sp, 512
+  ; GP64-DAG:   sd      $[[T0]], 0($sp)
+  ; GP64-DAG:   ld      $[[T1:[0-9]+]], 1024($fp)
+  ; GP64-DAG:   sd      $[[T1]], 8($sp)
+
+  %a = alloca i32, align 512
+  call void @helper_04(i32 0, i32 0, i32 0, i32 0,
+                       i32 0, i32 0, i32 0, i32 0, i32* %a, i32* %b)
+  ret void
+}
+
+; Check dynamic stack realignment in functions with variable-sized objects.
+
+; O32 ABI
+define void @func_05(i32 %sz) {
+entry:
+; GP32-LABEL: func_05:
+
+  ; prologue
+  ; FIXME: We are currently over-allocating stack space.
+  ; GP32:       addiu   $sp, $sp, -1024
+  ; GP32:       sw      $fp, 1020($sp)
+  ; GP32:       sw      $23, 1016($sp)
+  ;
+  ; GP32:       move    $fp, $sp
+  ; GP32:       addiu   $[[T0:[0-9]+|gp]], $zero, -512
+  ; GP32-NEXT:  and     $sp, $sp, $[[T0]]
+  ; GP32-NEXT:  move    $23, $sp
+
+  ; body
+  ; GP32:       addiu   $[[T1:[0-9]+]], $zero, 222
+  ; GP32:       sw      $[[T1]], 508($23)
+
+  ; epilogue
+  ; GP32:       move    $sp, $fp
+  ; GP32:       lw      $23, 1016($sp)
+  ; GP32:       lw      $fp, 1020($sp)
+  ; GP32:       addiu   $sp, $sp, 1024
+
+  %a0 = alloca i32, i32 %sz, align 512
+  %a1 = alloca i32, align 4
+
+  store volatile i32 111, i32* %a0, align 512
+  store volatile i32 222, i32* %a1, align 4
+
+  ret void
+}
+
+; N32/N64 ABIs
+define void @func_06(i32 %sz) {
+entry:
+; GP64-LABEL: func_06:
+
+  ; prologue
+  ; FIXME: We are currently over-allocating stack space.
+  ; N32:        addiu   $sp, $sp, -1024
+  ; N64:        daddiu  $sp, $sp, -1024
+  ; GP64:       sd      $fp, 1016($sp)
+  ; GP64:       sd      $23, 1008($sp)
+  ;
+  ; GP64:       move    $fp, $sp
+  ; GP64:       addiu   $[[T0:[0-9]+|gp]], $zero, -512
+  ; GP64-NEXT:  and     $sp, $sp, $[[T0]]
+  ; GP64-NEXT:  move    $23, $sp
+
+  ; body
+  ; GP64:       addiu   $[[T1:[0-9]+]], $zero, 222
+  ; GP64:       sw      $[[T1]], 508($23)
+
+  ; epilogue
+  ; GP64:       move    $sp, $fp
+  ; GP64:       ld      $23, 1008($sp)
+  ; GP64:       ld      $fp, 1016($sp)
+  ; N32:        addiu   $sp, $sp, 1024
+  ; N64:        daddiu  $sp, $sp, 1024
+
+  %a0 = alloca i32, i32 %sz, align 512
+  %a1 = alloca i32, align 4
+
+  store volatile i32 111, i32* %a0, align 512
+  store volatile i32 222, i32* %a1, align 4
+
+  ret void
+}
+
+; Verify that we use $fp for referencing incoming arguments and $sp for
+; building outbound arguments for nested function calls.
+
+; O32 ABI
+define void @func_07(i32 %p0, i32 %p1, i32 %p2, i32 %p3, i32 %sz) {
+entry:
+; GP32-LABEL: func_07:
+
+  ; body
+  ; FIXME: We are currently over-allocating stack space.
+  ; GP32-DAG:       lw      $[[T0:[0-9]+]], 1040($fp)
+  ;
+  ; GP32-DAG:       addiu   $[[T1:[0-9]+]], $zero, 222
+  ; GP32-DAG:       sw      $[[T1]], 508($23)
+  ;
+  ; GP32-DAG:       sw      $[[T2:[0-9]+]], 16($sp)
+
+  %a0 = alloca i32, i32 %sz, align 512
+  %a1 = alloca i32, align 4
+
+  store volatile i32 111, i32* %a0, align 512
+  store volatile i32 222, i32* %a1, align 4
+
+  call void @helper_01(i32 0, i32 0, i32 0, i32 0, i32* %a1)
+
+  ret void
+}
+
+; N32/N64 ABIs
+define void @func_08(i32 %p0, i32 %p1, i32 %p2, i32 %p3,
+                     i32 %p4, i32 %p5, i32 %p6, i32 %p7,
+                     i32 %sz) {
+entry:
+; GP64-LABEL: func_08:
+
+  ; body
+  ; FIXME: We are currently over-allocating stack space.
+  ; N32-DAG:        lw      $[[T0:[0-9]+]], 1028($fp)
+  ; N64-DAG:        lwu     $[[T0:[0-9]+]], 1028($fp)
+  ;
+  ; GP64-DAG:       addiu   $[[T1:[0-9]+]], $zero, 222
+  ; GP64-DAG:       sw      $[[T1]], 508($23)
+  ;
+  ; GP64-DAG:       sd      $[[T2:[0-9]+]], 0($sp)
+
+  %a0 = alloca i32, i32 %sz, align 512
+  %a1 = alloca i32, align 4
+
+  store volatile i32 111, i32* %a0, align 512
+  store volatile i32 222, i32* %a1, align 4
+
+  call void @helper_02(i32 0, i32 0, i32 0, i32 0,
+                       i32 0, i32 0, i32 0, i32 0, i32* %a1)
+  ret void
+}
+
+; Check that we do not perform dynamic stack realignment in the presence of
+; the "no-realign-stack" function attribute.
+define void @func_09() "no-realign-stack" {
+entry:
+; ALL-LABEL: func_09:
+
+  ; ALL-NOT:  and     $sp, $sp, $[[T0:[0-9]+|ra|gp]]
+
+  %a = alloca i32, align 512
+  call void @helper_01(i32 0, i32 0, i32 0, i32 0, i32* %a)
+  ret void
+}
+
+define void @func_10(i32 %sz) "no-realign-stack" {
+entry:
+; ALL-LABEL: func_10:
+
+  ; ALL-NOT:  and     $sp, $sp, $[[T0:[0-9]+|ra|gp]]
+
+  %a0 = alloca i32, i32 %sz, align 512
+  %a1 = alloca i32, align 4
+
+  store volatile i32 111, i32* %a0, align 512
+  store volatile i32 222, i32* %a1, align 4
+
+  ret void
+}
diff --git a/test/CodeGen/Mips/eh.ll b/test/CodeGen/Mips/eh.ll
index fcbd99ef737..2f843d9da9a 100644
--- a/test/CodeGen/Mips/eh.ll
+++ b/test/CodeGen/Mips/eh.ll
@@ -4,7 +4,7 @@
 @g1 = global double 0.000000e+00, align 8
 @_ZTId = external constant i8*
 
-define void @_Z1fd(double %i2) {
+define void @_Z1fd(double %i2) personality i32 (...)* @__gxx_personality_v0 {
 entry:
 ; CHECK-EL:  addiu $sp, $sp
 ; CHECK-EL:  .cfi_def_cfa_offset
@@ -26,7 +26,7 @@ lpad:                                             ; preds = %entry
 ; CHECK-EL:  # %lpad
 ; CHECK-EL:  bne $5
 
-  %exn.val = landingpad { i8*, i32 } personality i32 (...)* @__gxx_personality_v0
+  %exn.val = landingpad { i8*, i32 }
            cleanup
            catch i8* bitcast (i8** @_ZTId to i8*)
   %exn = extractvalue { i8*, i32 } %exn.val, 0
diff --git a/test/CodeGen/Mips/ehframe-indirect.ll b/test/CodeGen/Mips/ehframe-indirect.ll
index b4efb40b642..a51cfb7e0fc 100644
--- a/test/CodeGen/Mips/ehframe-indirect.ll
+++ b/test/CodeGen/Mips/ehframe-indirect.ll
@@ -1,39 +1,55 @@
-; RUN: llc -mtriple=mipsel-linux-gnu < %s | FileCheck  -check-prefix=CHECK32 %s
-; RUN: llc -mtriple=mipsel-linux-android < %s | FileCheck -check-prefix=CHECK32 %s
-; RUN: llc -mtriple=mips64el-linux-gnu < %s | FileCheck  -check-prefix=CHECK64 %s
-; RUN: llc -mtriple=mips64el-linux-android < %s | FileCheck -check-prefix=CHECK64 %s
+; RUN: llc -mtriple=mipsel-linux-gnu < %s -asm-verbose | FileCheck -check-prefix=ALL -check-prefix=O32 %s
+; RUN: llc -mtriple=mipsel-linux-android < %s -asm-verbose | FileCheck -check-prefix=ALL -check-prefix=O32 %s
+; RUN: llc -mtriple=mips64el-linux-gnu -target-abi=n32 < %s -asm-verbose | FileCheck -check-prefix=ALL -check-prefix=N32 %s
+; RUN: llc -mtriple=mips64el-linux-android -target-abi=n32 < %s -asm-verbose | FileCheck -check-prefix=ALL -check-prefix=N32 %s
+; RUN: llc -mtriple=mips64el-linux-gnu < %s -asm-verbose | FileCheck -check-prefix=ALL -check-prefix=N64 %s
+; RUN: llc -mtriple=mips64el-linux-android < %s -asm-verbose | FileCheck -check-prefix=ALL -check-prefix=N64 %s
 
-define i32 @main() {
-; CHECK: .cfi_startproc
-; CHECK: .cfi_personality 128, DW.ref.__gxx_personality_v0
+@_ZTISt9exception = external constant i8*
+
+define i32 @main() personality i8* bitcast (i32 (...)* @__gxx_personality_v0 to i8*) {
+; ALL: .cfi_startproc
+; ALL: .cfi_personality 128, DW.ref.__gxx_personality_v0
 
 entry:
   invoke void @foo() to label %cont unwind label %lpad
-; CHECK: foo
-; CHECK: jalr
+; ALL: foo
+; ALL: jalr
 
 lpad:
-  %0 = landingpad { i8*, i32 } personality i8*
-    bitcast (i32 (...)* @__gxx_personality_v0 to i8*) catch i8* null
+  %0 = landingpad { i8*, i32 }
+    catch i8* null
+    catch i8* bitcast (i8** @_ZTISt9exception to i8*)
   ret i32 0
 
 cont:
   ret i32 0
 }
-; CHECK: .cfi_endproc
+; ALL: .cfi_endproc
 
 declare i32 @__gxx_personality_v0(...)
 
 declare void @foo()
 
-; CHECK: .hidden DW.ref.__gxx_personality_v0
-; CHECK: .weak DW.ref.__gxx_personality_v0
-; CHECK: .section .data.DW.ref.__gxx_personality_v0,"aGw",@progbits,DW.ref.__gxx_personality_v0,comdat
-; CHECK32: .align 2
-; CHECK64: .align 3
-; CHECK: .type DW.ref.__gxx_personality_v0,@object
-; CHECK32: .size DW.ref.__gxx_personality_v0, 4
-; CHECK64: .size DW.ref.__gxx_personality_v0, 8
-; CHECK: DW.ref.__gxx_personality_v0:
-; CHECK32: .4byte __gxx_personality_v0
-; CHECK64: .8byte __gxx_personality_v0
+; ALL: GCC_except_table{{[0-9]+}}:
+; ALL: .byte 155 # @TType Encoding = indirect pcrel sdata4
+; ALL: $[[PC_LABEL:tmp[0-9]+]]:
+; ALL: .4byte	($_ZTISt9exception.DW.stub)-($[[PC_LABEL]])
+; ALL: $_ZTISt9exception.DW.stub:
+; O32: .4byte _ZTISt9exception
+; N32: .4byte _ZTISt9exception
+; N64: .8byte _ZTISt9exception
+; ALL: .hidden DW.ref.__gxx_personality_v0
+; ALL: .weak DW.ref.__gxx_personality_v0
+; ALL: .section .data.DW.ref.__gxx_personality_v0,"aGw",@progbits,DW.ref.__gxx_personality_v0,comdat
+; O32: .align 2
+; N32: .align 2
+; N64: .align 3
+; ALL: .type DW.ref.__gxx_personality_v0,@object
+; O32: .size DW.ref.__gxx_personality_v0, 4
+; N32: .size DW.ref.__gxx_personality_v0, 4
+; N64: .size DW.ref.__gxx_personality_v0, 8
+; ALL: DW.ref.__gxx_personality_v0:
+; O32: .4byte __gxx_personality_v0
+; N32: .4byte __gxx_personality_v0
+; N64: .8byte __gxx_personality_v0
diff --git a/test/CodeGen/Mips/emergency-spill-slot-near-fp.ll b/test/CodeGen/Mips/emergency-spill-slot-near-fp.ll
new file mode 100644
index 00000000000..58dd16c9f9c
--- /dev/null
+++ b/test/CodeGen/Mips/emergency-spill-slot-near-fp.ll
@@ -0,0 +1,34 @@
+; Check that register scavenging spill slot is close to $fp.
+; RUN: llc -march=mipsel -O0 -fast-isel=false < %s | FileCheck %s
+
+; CHECK: sw ${{.*}}, 8($sp)
+; CHECK: lw ${{.*}}, 8($sp)
+
+define i32 @main(i32 signext %argc, i8** %argv) #0 {
+entry:
+  %retval = alloca i32, align 4
+  %argc.addr = alloca i32, align 4
+  %argv.addr = alloca i8**, align 4
+  %v0 = alloca <16 x i8>, align 16
+  %.compoundliteral = alloca <16 x i8>, align 16
+  %v1 = alloca <16 x i8>, align 16
+  %.compoundliteral1 = alloca <16 x i8>, align 16
+  %unused_variable = alloca [16384 x i32], align 4
+  %result = alloca <16 x i8>, align 16
+  store i32 0, i32* %retval
+  store i32 %argc, i32* %argc.addr, align 4
+  store i8** %argv, i8*** %argv.addr, align 4
+  store <16 x i8> <i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 8, i8 9, i8 10, i8 11, i8 12, i8 13, i8 14, i8 15, i8 16>, <16 x i8>* %.compoundliteral
+  %0 = load <16 x i8>, <16 x i8>* %.compoundliteral
+  store <16 x i8> %0, <16 x i8>* %v0, align 16
+  store <16 x i8> zeroinitializer, <16 x i8>* %.compoundliteral1
+  %1 = load <16 x i8>, <16 x i8>* %.compoundliteral1
+  store <16 x i8> %1, <16 x i8>* %v1, align 16
+  %2 = load <16 x i8>, <16 x i8>* %v0, align 16
+  %3 = load <16 x i8>, <16 x i8>* %v1, align 16
+  %mul = mul <16 x i8> %2, %3
+  store <16 x i8> %mul, <16 x i8>* %result, align 16
+  ret i32 0
+}
+
+attributes #0 = { noinline optnone "no-frame-pointer-elim"="true" }
diff --git a/test/CodeGen/Mips/emutls_generic.ll b/test/CodeGen/Mips/emutls_generic.ll
new file mode 100644
index 00000000000..84a84f3fd74
--- /dev/null
+++ b/test/CodeGen/Mips/emutls_generic.ll
@@ -0,0 +1,70 @@
+; RUN: llc < %s -emulated-tls -mtriple=mipsel-linux-android -relocation-model=pic \
+; RUN:     | FileCheck -check-prefix=MIPS_32 %s
+; RUN: llc < %s -emulated-tls -mtriple=mips64el-linux-android -relocation-model=pic \
+; RUN:     | FileCheck -check-prefix=MIPS_64 %s
+
+; Make sure that TLS symbols are emitted in expected order.
+
+@external_x = external thread_local global i32, align 8
+@external_y = thread_local global i8 7, align 2
+@internal_y = internal thread_local global i64 9, align 16
+
+define i32* @get_external_x() {
+entry:
+  ret i32* @external_x
+}
+
+define i8* @get_external_y() {
+entry:
+  ret i8* @external_y
+}
+
+define i64* @get_internal_y() {
+entry:
+  ret i64* @internal_y
+}
+
+; MIPS_32-LABEL: get_external_y:
+; MIPS_32-LABEL: get_internal_y:
+; MIPS_32:     lw {{.+}}(__emutls_v.internal_y
+; MIPS_32:     lw {{.+}}call16(__emutls_get_address
+; MIPS_32-NOT:  __emutls_t.external_x
+; MIPS_32-NOT:  __emutls_v.external_x:
+; MIPS_32:       .section .data.rel.local
+; MIPS_32:       .align 2
+; MIPS_32-LABEL: __emutls_v.external_y:
+; MIPS_32:       .section .rodata,
+; MIPS_32-LABEL: __emutls_t.external_y:
+; MIPS_32-NEXT:  .byte 7
+; MIPS_32:       .section .data.rel.local
+; MIPS_32:       .align 2
+; MIPS_32-LABEL: __emutls_v.internal_y:
+; MIPS_32-NEXT:  .4byte 8
+; MIPS_32-NEXT:  .4byte 16
+; MIPS_32-NEXT:  .4byte 0
+; MIPS_32-NEXT:  .4byte __emutls_t.internal_y
+; MIPS_32-LABEL: __emutls_t.internal_y:
+; MIPS_32-NEXT:  .8byte 9
+
+; MIPS_64-LABEL: get_external_x:
+; MIPS_64-LABEL: get_external_y:
+; MIPS_64-LABEL: get_internal_y:
+; MIPS_64:     ld {{.+}}(__emutls_v.internal_y
+; MIPS_64:     ld {{.+}}call16(__emutls_get_address
+; MIPS_64-NOT:  __emutls_t.external_x
+; MIPS_64-NOT:  __emutls_v.external_x:
+; MIPS_64-LABEL: __emutls_v.external_y:
+; MIPS_64-NOT:   __emutls_v.external_x:
+; MIPS_64:       .section .rodata,
+; MIPS_64-LABEL: __emutls_t.external_y:
+; MIPS_64-NEXT:  .byte 7
+; MIPS_64:       .section .data.rel.local
+; MIPS_64:       .align 3
+; MIPS_64-LABEL: __emutls_v.internal_y:
+; MIPS_64-NEXT:  .8byte 8
+; MIPS_64-NEXT:  .8byte 16
+; MIPS_64-NEXT:  .8byte 0
+; MIPS_64-NEXT:  .8byte __emutls_t.internal_y
+; MIPS_64:       .section .rodata,
+; MIPS_64-LABEL: __emutls_t.internal_y:
+; MIPS_64-NEXT:  .8byte 9
diff --git a/test/CodeGen/Mips/fp16-promote.ll b/test/CodeGen/Mips/fp16-promote.ll
new file mode 100644
index 00000000000..2ac46e02807
--- /dev/null
+++ b/test/CodeGen/Mips/fp16-promote.ll
@@ -0,0 +1,98 @@
+; RUN: llc -asm-verbose=false -mtriple=mipsel-linux-gnueabi < %s | FileCheck %s -check-prefix=CHECK-LIBCALL
+
+; CHECK-LIBCALL-LABEL: test_fadd:
+; CHECK-LIBCALL: %call16(__gnu_h2f_ieee)
+; CHECK-LIBCALL: %call16(__gnu_h2f_ieee)
+; CHECK-LIBCALL-DAG: add.s
+; CHECK-LIBCALL-DAG: %call16(__gnu_f2h_ieee)
+define void @test_fadd(half* %p, half* %q) #0 {
+  %a = load half, half* %p, align 2
+  %b = load half, half* %q, align 2
+  %r = fadd half %a, %b
+  store half %r, half* %p
+  ret void
+}
+
+; CHECK-LIBCALL-LABEL: test_fpext_float:
+; CHECK-LIBCALL: %call16(__gnu_h2f_ieee)
+define float @test_fpext_float(half* %p) {
+  %a = load half, half* %p, align 2
+  %r = fpext half %a to float
+  ret float %r
+}
+
+; CHECK-LIBCALL-LABEL: test_fpext_double:
+; CHECK-LIBCALL: %call16(__gnu_h2f_ieee)
+; CHECK-LIBCALL: cvt.d.s
+define double @test_fpext_double(half* %p) {
+  %a = load half, half* %p, align 2
+  %r = fpext half %a to double
+  ret double %r
+}
+
+; CHECK-LIBCALL-LABEL: test_fptrunc_float:
+; CHECK-LIBCALL: %call16(__gnu_f2h_ieee)
+define void @test_fptrunc_float(float %f, half* %p) #0 {
+  %a = fptrunc float %f to half
+  store half %a, half* %p
+  ret void
+}
+
+; CHECK-LIBCALL-LABEL: test_fptrunc_double:
+; CHECK-LIBCALL: %call16(__truncdfhf2)
+define void @test_fptrunc_double(double %d, half* %p) #0 {
+  %a = fptrunc double %d to half
+  store half %a, half* %p
+  ret void
+}
+
+; CHECK-LIBCALL-LABEL: test_vec_fpext_float:
+; CHECK-LIBCALL: %call16(__gnu_h2f_ieee)
+; CHECK-LIBCALL: %call16(__gnu_h2f_ieee)
+; CHECK-LIBCALL: %call16(__gnu_h2f_ieee)
+; CHECK-LIBCALL: %call16(__gnu_h2f_ieee)
+define <4 x float> @test_vec_fpext_float(<4 x half>* %p) #0 {
+  %a = load <4 x half>, <4 x half>* %p, align 8
+  %b = fpext <4 x half> %a to <4 x float>
+  ret <4 x float> %b
+}
+
+; This test is not robust against variations in instruction scheduling.
+; See the discussion in http://reviews.llvm.org/D8804
+; CHECK-LIBCALL-LABEL: test_vec_fpext_double:
+; CHECK-LIBCALL: %call16(__gnu_h2f_ieee)
+; CHECK-LIBCALL: %call16(__gnu_h2f_ieee)
+; CHECK-LIBCALL: %call16(__gnu_h2f_ieee)
+; CHECK-LIBCALL: cvt.d.s
+; CHECK-LIBCALL: cvt.d.s
+; CHECK-LIBCALL: cvt.d.s
+; CHECK-LIBCALL: %call16(__gnu_h2f_ieee)
+; CHECK-LIBCALL: cvt.d.s
+define <4 x double> @test_vec_fpext_double(<4 x half>* %p) #0 {
+  %a = load <4 x half>, <4 x half>* %p, align 8
+  %b = fpext <4 x half> %a to <4 x double>
+  ret <4 x double> %b
+}
+
+; CHECK-LIBCALL-LABEL: test_vec_fptrunc_float:
+; CHECK-LIBCALL: %call16(__gnu_f2h_ieee)
+; CHECK-LIBCALL: %call16(__gnu_f2h_ieee)
+; CHECK-LIBCALL: %call16(__gnu_f2h_ieee)
+; CHECK-LIBCALL: %call16(__gnu_f2h_ieee)
+define void @test_vec_fptrunc_float(<4 x float> %a, <4 x half>* %p) #0 {
+  %b = fptrunc <4 x float> %a to <4 x half>
+  store <4 x half> %b, <4 x half>* %p, align 8
+  ret void
+}
+
+; CHECK-LIBCALL-LABEL: test_vec_fptrunc_double:
+; CHECK-LIBCALL: %call16(__truncdfhf2)
+; CHECK-LIBCALL: %call16(__truncdfhf2)
+; CHECK-LIBCALL: %call16(__truncdfhf2)
+; CHECK-LIBCALL: %call16(__truncdfhf2)
+define void @test_vec_fptrunc_double(<4 x double> %a, <4 x half>* %p) #0 {
+  %b = fptrunc <4 x double> %a to <4 x half>
+  store <4 x half> %b, <4 x half>* %p, align 8
+  ret void
+}
+
diff --git a/test/CodeGen/Mips/insn-zero-size-bb.ll b/test/CodeGen/Mips/insn-zero-size-bb.ll
new file mode 100644
index 00000000000..ea61c994ae1
--- /dev/null
+++ b/test/CodeGen/Mips/insn-zero-size-bb.ll
@@ -0,0 +1,27 @@
+; RUN: llc < %s -march=mips -mcpu=mips32 | FileCheck %s
+; RUN: llc < %s -march=mips -mcpu=mips32r3 -mattr=+micromips | FileCheck %s
+; RUN: llc < %s -march=mips -mcpu=mips16 | FileCheck %s
+
+; Verify that we emit the .insn directive for zero-sized (empty) basic blocks.
+; This only really matters for microMIPS and MIPS16.
+
+declare i32 @foo(...)
+declare void @bar()
+
+define void @main() personality i8* bitcast (i32 (...)* @foo to i8*) {
+entry:
+  invoke void @bar() #0
+          to label %unreachable unwind label %return
+
+unreachable:
+; CHECK:          ${{.*}}: # %unreachable
+; CHECK-NEXT:         .insn
+  unreachable
+
+return:
+  %0 = landingpad { i8*, i32 }
+          catch i8* null
+  ret void
+}
+
+attributes #0 = { noreturn }
diff --git a/test/CodeGen/Mips/l3mc.ll b/test/CodeGen/Mips/l3mc.ll
index 6aeed04f779..c1bff11595c 100644
--- a/test/CodeGen/Mips/l3mc.ll
+++ b/test/CodeGen/Mips/l3mc.ll
@@ -1,22 +1,22 @@
-; RUN: llc -mtriple=mipsel-linux-gnu -march=mipsel -mcpu=mips16 -soft-float -mips16-hard-float -relocation-model=static  < %s | FileCheck  %s -check-prefix=__call_stub_fp___fixunsdfsi 
+; RUN: llc -mtriple=mipsel-linux-gnu -march=mipsel -mcpu=mips16 -mattr=+soft-float -mips16-hard-float -relocation-model=static  < %s | FileCheck  %s -check-prefix=__call_stub_fp___fixunsdfsi 
 
-; RUN: llc -mtriple=mipsel-linux-gnu -march=mipsel -mcpu=mips16 -soft-float -mips16-hard-float -relocation-model=static  < %s | FileCheck %s -check-prefix=__call_stub_fp___floatdidf 
+; RUN: llc -mtriple=mipsel-linux-gnu -march=mipsel -mcpu=mips16 -mattr=+soft-float -mips16-hard-float -relocation-model=static  < %s | FileCheck %s -check-prefix=__call_stub_fp___floatdidf 
 
-; RUN: llc -mtriple=mipsel-linux-gnu -march=mipsel -mcpu=mips16 -soft-float -mips16-hard-float -relocation-model=static  < %s | FileCheck %s -check-prefix=__call_stub_fp___floatdisf 
+; RUN: llc -mtriple=mipsel-linux-gnu -march=mipsel -mcpu=mips16 -mattr=+soft-float -mips16-hard-float -relocation-model=static  < %s | FileCheck %s -check-prefix=__call_stub_fp___floatdisf 
 
-; RUN: llc -mtriple=mipsel-linux-gnu -march=mipsel -mcpu=mips16 -soft-float -mips16-hard-float -relocation-model=static  < %s | FileCheck %s -check-prefix=__call_stub_fp___floatundidf
+; RUN: llc -mtriple=mipsel-linux-gnu -march=mipsel -mcpu=mips16 -mattr=+soft-float -mips16-hard-float -relocation-model=static  < %s | FileCheck %s -check-prefix=__call_stub_fp___floatundidf
 
-; RUN: llc -mtriple=mipsel-linux-gnu -march=mipsel -mcpu=mips16 -soft-float -mips16-hard-float -relocation-model=static  < %s | FileCheck %s -check-prefix=__call_stub_fp___fixsfdi 
+; RUN: llc -mtriple=mipsel-linux-gnu -march=mipsel -mcpu=mips16 -mattr=+soft-float -mips16-hard-float -relocation-model=static  < %s | FileCheck %s -check-prefix=__call_stub_fp___fixsfdi 
 
-; RUN: llc -mtriple=mipsel-linux-gnu -march=mipsel -mcpu=mips16 -soft-float -mips16-hard-float -relocation-model=static  < %s | FileCheck %s -check-prefix=__call_stub_fp___fixunsdfdi 
+; RUN: llc -mtriple=mipsel-linux-gnu -march=mipsel -mcpu=mips16 -mattr=+soft-float -mips16-hard-float -relocation-model=static  < %s | FileCheck %s -check-prefix=__call_stub_fp___fixunsdfdi 
 
-; RUN: llc -mtriple=mipsel-linux-gnu -march=mipsel -mcpu=mips16 -soft-float -mips16-hard-float -relocation-model=static  < %s | FileCheck %s -check-prefix=__call_stub_fp___fixdfdi
+; RUN: llc -mtriple=mipsel-linux-gnu -march=mipsel -mcpu=mips16 -mattr=+soft-float -mips16-hard-float -relocation-model=static  < %s | FileCheck %s -check-prefix=__call_stub_fp___fixdfdi
 
-; RUN: llc -mtriple=mipsel-linux-gnu -march=mipsel -mcpu=mips16 -soft-float -mips16-hard-float -relocation-model=static  < %s | FileCheck %s -check-prefix=__call_stub_fp___fixunssfsi 
+; RUN: llc -mtriple=mipsel-linux-gnu -march=mipsel -mcpu=mips16 -mattr=+soft-float -mips16-hard-float -relocation-model=static  < %s | FileCheck %s -check-prefix=__call_stub_fp___fixunssfsi 
 
-; RUN: llc -mtriple=mipsel-linux-gnu -march=mipsel -mcpu=mips16 -soft-float -mips16-hard-float -relocation-model=static  < %s | FileCheck %s -check-prefix=__call_stub_fp___fixunssfdi 
+; RUN: llc -mtriple=mipsel-linux-gnu -march=mipsel -mcpu=mips16 -mattr=+soft-float -mips16-hard-float -relocation-model=static  < %s | FileCheck %s -check-prefix=__call_stub_fp___fixunssfdi 
 
-; RUN: llc -mtriple=mipsel-linux-gnu -march=mipsel -mcpu=mips16 -soft-float -mips16-hard-float -relocation-model=static  < %s | FileCheck %s -check-prefix=__call_stub_fp___floatundisf 
+; RUN: llc -mtriple=mipsel-linux-gnu -march=mipsel -mcpu=mips16 -mattr=+soft-float -mips16-hard-float -relocation-model=static  < %s | FileCheck %s -check-prefix=__call_stub_fp___floatundisf 
 
 @ll1 = global i64 0, align 8
 @ll2 = global i64 0, align 8
diff --git a/test/CodeGen/Mips/lcb2.ll b/test/CodeGen/Mips/lcb2.ll
index 716a6bbce8f..a6f4968e6d2 100644
--- a/test/CodeGen/Mips/lcb2.ll
+++ b/test/CodeGen/Mips/lcb2.ll
@@ -1,6 +1,6 @@
-; RUN: llc -mtriple=mipsel-linux-gnu -march=mipsel -mcpu=mips16 -soft-float -mips16-hard-float -relocation-model=static -mips16-constant-islands=true   < %s | FileCheck %s -check-prefix=lcb
+; RUN: llc -mtriple=mipsel-linux-gnu -march=mipsel -mcpu=mips16 -mattr=+soft-float -mips16-hard-float -relocation-model=static -mips16-constant-islands=true   < %s | FileCheck %s -check-prefix=lcb
 
-; RUN: llc -mtriple=mipsel-linux-gnu -march=mipsel -mcpu=mips16 -soft-float -mips16-hard-float -relocation-model=static -mips16-constant-islands=true   < %s | FileCheck %s -check-prefix=lcbn
+; RUN: llc -mtriple=mipsel-linux-gnu -march=mipsel -mcpu=mips16 -mattr=+soft-float -mips16-hard-float -relocation-model=static -mips16-constant-islands=true   < %s | FileCheck %s -check-prefix=lcbn
 
 @i = global i32 0, align 4
 @j = common global i32 0, align 4
diff --git a/test/CodeGen/Mips/lcb3c.ll b/test/CodeGen/Mips/lcb3c.ll
index d6e259c7260..4c6f2c036a0 100644
--- a/test/CodeGen/Mips/lcb3c.ll
+++ b/test/CodeGen/Mips/lcb3c.ll
@@ -1,4 +1,4 @@
-; RUN: llc -mtriple=mipsel-linux-gnu -march=mipsel -mcpu=mips16 -soft-float -mips16-hard-float -relocation-model=static -O0    < %s | FileCheck %s -check-prefix=lcb
+; RUN: llc -mtriple=mipsel-linux-gnu -march=mipsel -mcpu=mips16 -mattr=+soft-float -mips16-hard-float -relocation-model=static -O0    < %s | FileCheck %s -check-prefix=lcb
 
 @i = global i32 0, align 4
 @j = common global i32 0, align 4
diff --git a/test/CodeGen/Mips/lcb4a.ll b/test/CodeGen/Mips/lcb4a.ll
index 0285ae17e1d..9e97b5bf143 100644
--- a/test/CodeGen/Mips/lcb4a.ll
+++ b/test/CodeGen/Mips/lcb4a.ll
@@ -1,4 +1,4 @@
-; RUN: llc -mtriple=mipsel-linux-gnu -march=mipsel -mcpu=mips16 -soft-float -mips16-hard-float -relocation-model=static     < %s | FileCheck %s -check-prefix=ci
+; RUN: llc -mtriple=mipsel-linux-gnu -march=mipsel -mcpu=mips16 -mattr=+soft-float -mips16-hard-float -relocation-model=static     < %s | FileCheck %s -check-prefix=ci
 
 @i = global i32 0, align 4
 @j = common global i32 0, align 4
diff --git a/test/CodeGen/Mips/lcb5.ll b/test/CodeGen/Mips/lcb5.ll
index 172ecb3d4da..41878d5f881 100644
--- a/test/CodeGen/Mips/lcb5.ll
+++ b/test/CodeGen/Mips/lcb5.ll
@@ -1,4 +1,4 @@
-; RUN: llc -mtriple=mipsel-linux-gnu -march=mipsel -mcpu=mips16 -soft-float -mips16-hard-float -relocation-model=static     < %s | FileCheck %s -check-prefix=ci
+; RUN: llc -mtriple=mipsel-linux-gnu -march=mipsel -mcpu=mips16 -mattr=+soft-float -mips16-hard-float -relocation-model=static     < %s | FileCheck %s -check-prefix=ci
 
 @i = global i32 0, align 4
 @j = common global i32 0, align 4
diff --git a/test/CodeGen/Mips/llvm-ir/ashr.ll b/test/CodeGen/Mips/llvm-ir/ashr.ll
index 7e1587c9f4b..cad4a39d774 100644
--- a/test/CodeGen/Mips/llvm-ir/ashr.ll
+++ b/test/CodeGen/Mips/llvm-ir/ashr.ll
@@ -131,9 +131,7 @@ entry:
   ; 32R6:       jr        $ra
   ; 32R6:       or        $3, $[[T0]], $[[T11]]
 
-  ; FIXME: The sll instruction below is redundant.
-  ; GP64:       sll       $[[T0:[0-9]+]], $5, 0
-  ; GP64:       dsrav     $2, $4, $[[T0]]
+  ; GP64:       dsrav     $2, $4, $5
 
   %r = ashr i64 %a, %b
   ret i64 %r
@@ -146,11 +144,11 @@ entry:
   ; GP32:           lw        $25, %call16(__ashrti3)($gp)
 
   ; M3:             sll       $[[T0:[0-9]+]], $7, 0
-  ; M3:             dsrav     $[[T1:[0-9]+]], $4, $[[T0]]
-  ; M3:             andi      $[[T2:[0-9]+]], $[[T0]], 32
+  ; M3:             dsrav     $[[T1:[0-9]+]], $4, $7
+  ; M3:             andi      $[[T2:[0-9]+]], $[[T0]], 64
   ; M3:             bnez      $[[T3:[0-9]+]], $[[BB0:BB[0-9_]+]]
   ; M3:             move      $3, $[[T1]]
-  ; M3:             dsrlv     $[[T4:[0-9]+]], $5, $[[T0]]
+  ; M3:             dsrlv     $[[T4:[0-9]+]], $5, $7
   ; M3:             dsll      $[[T5:[0-9]+]], $4, 1
   ; M3:             not       $[[T6:[0-9]+]], $[[T0]]
   ; M3:             dsllv     $[[T7:[0-9]+]], $[[T5]], $[[T6]]
@@ -158,40 +156,39 @@ entry:
   ; M3:             $[[BB0]]:
   ; M3:             beqz      $[[T3]], $[[BB1:BB[0-9_]+]]
   ; M3:             nop
-  ; M3:             dsra      $2, $4, 31
+  ; M3:             dsra      $2, $4, 63
   ; M3:             $[[BB1]]:
   ; M3:             jr        $ra
   ; M3:             nop
 
-  ; GP64-NOT-R6:    sll       $[[T0:[0-9]+]], $7, 0
-  ; GP64-NOT-R6:    dsrlv     $[[T1:[0-9]+]], $5, $[[T0]]
-  ; GP64-NOT-R6:    dsll      $[[T2:[0-9]+]], $4, 1
-  ; GP64-NOT-R6:    not       $[[T3:[0-9]+]], $[[T0]]
-  ; GP64-NOT-R6:    dsllv     $[[T4:[0-9]+]], $[[T2]], $[[T3]]
-  ; GP64-NOT-R6:    or        $3, $[[T4]], $[[T1]]
-  ; GP64-NOT-R6:    dsrav     $2, $4, $[[T0]]
-  ; GP64-NOT-R6:    andi      $[[T5:[0-9]+]], $[[T0]], 32
-
+  ; GP64-NOT-R6:    dsrlv     $[[T0:[0-9]+]], $5, $7
+  ; GP64-NOT-R6:    dsll      $[[T1:[0-9]+]], $4, 1
+  ; GP64-NOT-R6:    sll       $[[T2:[0-9]+]], $7, 0
+  ; GP64-NOT-R6:    not       $[[T3:[0-9]+]], $[[T2]]
+  ; GP64-NOT-R6:    dsllv     $[[T4:[0-9]+]], $[[T1]], $[[T3]]
+  ; GP64-NOT-R6:    or        $3, $[[T4]], $[[T0]]
+  ; GP64-NOT-R6:    dsrav     $2, $4, $7
+  ; GP64-NOT-R6:    andi      $[[T5:[0-9]+]], $[[T2]], 64
   ; GP64-NOT-R6:    movn      $3, $2, $[[T5]]
-  ; GP64-NOT-R6:    dsra      $[[T6:[0-9]+]], $4, 31
+  ; GP64-NOT-R6:    dsra      $[[T6:[0-9]+]], $4, 63
   ; GP64-NOT-R6:    jr        $ra
   ; GP64-NOT-R6:    movn      $2, $[[T6]], $[[T5]]
 
-  ; 64R6:           sll       $[[T0:[0-9]+]], $7, 0
-  ; 64R6:           dsrav     $[[T1:[0-9]+]], $4, $[[T0]]
-  ; 64R6:           andi      $[[T2:[0-9]+]], $[[T0]], 32
+  ; 64R6:           dsrav     $[[T0:[0-9]+]], $4, $7
+  ; 64R6:           sll       $[[T1:[0-9]+]], $7, 0
+  ; 64R6:           andi      $[[T2:[0-9]+]], $[[T1]], 64
   ; 64R6:           sll       $[[T3:[0-9]+]], $[[T2]], 0
-  ; 64R6:           seleqz    $[[T4:[0-9]+]], $[[T1]], $[[T3]]
-  ; 64R6:           dsra      $[[T5:[0-9]+]], $4, 31
+  ; 64R6:           seleqz    $[[T4:[0-9]+]], $[[T0]], $[[T3]]
+  ; 64R6:           dsra      $[[T5:[0-9]+]], $4, 63
   ; 64R6:           selnez    $[[T6:[0-9]+]], $[[T5]], $[[T3]]
   ; 64R6:           or        $2, $[[T6]], $[[T4]]
-  ; 64R6:           dsrlv     $[[T7:[0-9]+]], $5, $[[T0]]
+  ; 64R6:           dsrlv     $[[T7:[0-9]+]], $5, $7
   ; 64R6:           dsll      $[[T8:[0-9]+]], $4, 1
-  ; 64R6:           not       $[[T9:[0-9]+]], $[[T0]]
+  ; 64R6:           not       $[[T9:[0-9]+]], $[[T1]]
   ; 64R6:           dsllv     $[[T10:[0-9]+]], $[[T8]], $[[T9]]
   ; 64R6:           or        $[[T11:[0-9]+]], $[[T10]], $[[T7]]
   ; 64R6:           seleqz    $[[T12:[0-9]+]], $[[T11]], $[[T3]]
-  ; 64R6:           selnez    $[[T13:[0-9]+]], $[[T1]], $[[T3]]
+  ; 64R6:           selnez    $[[T13:[0-9]+]], $[[T0]], $[[T3]]
   ; 64R6:           jr        $ra
   ; 64R6:           or        $3, $[[T13]], $[[T12]]
 
diff --git a/test/CodeGen/Mips/llvm-ir/lshr.ll b/test/CodeGen/Mips/llvm-ir/lshr.ll
index 7344d950cad..3a7029fa5b7 100644
--- a/test/CodeGen/Mips/llvm-ir/lshr.ll
+++ b/test/CodeGen/Mips/llvm-ir/lshr.ll
@@ -125,8 +125,7 @@ entry:
   ; 32R6:       jr        $ra
   ; 32R6:       seleqz    $2, $[[T7]], $[[T5]]
 
-  ; GP64:         sll     $[[T0:[0-9]+]], $5, 0
-  ; GP64:         dsrlv   $2, $4, $[[T0]]
+  ; GP64:         dsrlv   $2, $4, $5
 
   %r = lshr i64 %a, %b
   ret i64 %r
@@ -139,11 +138,11 @@ entry:
   ; GP32:         lw      $25, %call16(__lshrti3)($gp)
 
   ; M3:             sll       $[[T0:[0-9]+]], $7, 0
-  ; M3:             dsrlv     $[[T1:[0-9]+]], $4, $[[T0]]
-  ; M3:             andi      $[[T2:[0-9]+]], $[[T0]], 32
+  ; M3:             dsrlv     $[[T1:[0-9]+]], $4, $7
+  ; M3:             andi      $[[T2:[0-9]+]], $[[T0]], 64
   ; M3:             bnez      $[[T3:[0-9]+]], $[[BB0:BB[0-9_]+]]
   ; M3:             move      $3, $[[T1]]
-  ; M3:             dsrlv     $[[T4:[0-9]+]], $5, $[[T0]]
+  ; M3:             dsrlv     $[[T4:[0-9]+]], $5, $7
   ; M3:             dsll      $[[T5:[0-9]+]], $4, 1
   ; M3:             not       $[[T6:[0-9]+]], $[[T0]]
   ; M3:             dsllv     $[[T7:[0-9]+]], $[[T5]], $[[T6]]
@@ -156,32 +155,32 @@ entry:
   ; M3:             jr        $ra
   ; M3:             nop
 
-  ; GP64-NOT-R6:    sll       $[[T0:[0-9]+]], $7, 0
-  ; GP64-NOT-R6:    dsrlv     $[[T1:[0-9]+]], $5, $[[T0]]
-  ; GP64-NOT-R6:    dsll      $[[T2:[0-9]+]], $4, 1
-  ; GP64-NOT-R6:    not       $[[T3:[0-9]+]], $[[T0]]
-  ; GP64-NOT-R6:    dsllv     $[[T4:[0-9]+]], $[[T2]], $[[T3]]
-  ; GP64-NOT-R6:    or        $3, $[[T4]], $[[T1]]
-  ; GP64-NOT-R6:    dsrlv     $2, $4, $[[T0]]
-  ; GP64-NOT-R6:    andi      $[[T5:[0-9]+]], $[[T0]], 32
+  ; GP64-NOT-R6:    dsrlv     $[[T0:[0-9]+]], $5, $7
+  ; GP64-NOT-R6:    dsll      $[[T1:[0-9]+]], $4, 1
+  ; GP64-NOT-R6:    sll       $[[T2:[0-9]+]], $7, 0
+  ; GP64-NOT-R6:    not       $[[T3:[0-9]+]], $[[T2]]
+  ; GP64-NOT-R6:    dsllv     $[[T4:[0-9]+]], $[[T1]], $[[T3]]
+  ; GP64-NOT-R6:    or        $3, $[[T4]], $[[T0]]
+  ; GP64-NOT-R6:    dsrlv     $2, $4, $7
+  ; GP64-NOT-R6:    andi      $[[T5:[0-9]+]], $[[T2]], 64
   ; GP64-NOT-R6:    movn      $3, $2, $[[T5]]
   ; GP64-NOT-R6:    jr        $ra
   ; GP64-NOT-R6:    movn      $2, $zero, $1
 
-  ; 64R6:           sll       $[[T0:[0-9]+]], $7, 0
-  ; 64R6:           dsrlv     $[[T1:[0-9]+]], $5, $[[T0]]
-  ; 64R6:           dsll      $[[T2:[0-9]+]], $4, 1
-  ; 64R6:           not       $[[T3:[0-9]+]], $[[T0]]
-  ; 64R6:           dsllv     $[[T4:[0-9]+]], $[[T2]], $[[T3]]
-  ; 64R6:           or        $[[T5:[0-9]+]], $[[T4]], $[[T1]]
-  ; 64R6:           andi      $[[T6:[0-9]+]], $[[T0]], 32
+  ; 64R6:           dsrlv     $[[T0:[0-9]+]], $5, $7
+  ; 64R6:           dsll      $[[T1:[0-9]+]], $4, 1
+  ; 64R6:           sll       $[[T2:[0-9]+]], $7, 0
+  ; 64R6:           not       $[[T3:[0-9]+]], $[[T2]]
+  ; 64R6:           dsllv     $[[T4:[0-9]+]], $[[T1]], $[[T3]]
+  ; 64R6:           or        $[[T5:[0-9]+]], $[[T4]], $[[T0]]
+  ; 64R6:           andi      $[[T6:[0-9]+]], $[[T2]], 64
   ; 64R6:           sll       $[[T7:[0-9]+]], $[[T6]], 0
   ; 64R6:           seleqz    $[[T8:[0-9]+]], $[[T5]], $[[T7]]
-  ; 64R6:           dsrlv     $[[T9:[0-9]+]], $4, $[[T0]]
+  ; 64R6:           dsrlv     $[[T9:[0-9]+]], $4, $7
   ; 64R6:           selnez    $[[T10:[0-9]+]], $[[T9]], $[[T7]]
   ; 64R6:           or        $3, $[[T10]], $[[T8]]
   ; 64R6:           jr        $ra
-  ; 64R6:           seleqz    $2, $[[T0]], $[[T7]]
+  ; 64R6:           seleqz    $2, $[[T9]], $[[T7]]
 
   %r = lshr i128 %a, %b
   ret i128 %r
diff --git a/test/CodeGen/Mips/llvm-ir/shl.ll b/test/CodeGen/Mips/llvm-ir/shl.ll
index 664032005e6..bba34c47ea8 100644
--- a/test/CodeGen/Mips/llvm-ir/shl.ll
+++ b/test/CodeGen/Mips/llvm-ir/shl.ll
@@ -137,8 +137,7 @@ entry:
   ; 32R6:       jr        $ra
   ; 32R6:       seleqz    $3, $[[T7]], $[[T5]]
 
-  ; GP64:       sll       $[[T0:[0-9]+]], $5, 0
-  ; GP64:       dsllv     $2, $4, $1
+  ; GP64:       dsllv     $2, $4, $5
 
   %r = shl i64 %a, %b
   ret i64 %r
@@ -151,11 +150,11 @@ entry:
   ; GP32:           lw        $25, %call16(__ashlti3)($gp)
 
   ; M3:             sll       $[[T0:[0-9]+]], $7, 0
-  ; M3:             dsllv     $[[T1:[0-9]+]], $5, $[[T0]]
-  ; M3:             andi      $[[T2:[0-9]+]], $[[T0]], 32
+  ; M3:             dsllv     $[[T1:[0-9]+]], $5, $7
+  ; M3:             andi      $[[T2:[0-9]+]], $[[T0]], 64
   ; M3:             bnez      $[[T3:[0-9]+]], $[[BB0:BB[0-9_]+]]
   ; M3:             move      $2, $[[T1]]
-  ; M3:             dsllv     $[[T4:[0-9]+]], $4, $[[T0]]
+  ; M3:             dsllv     $[[T4:[0-9]+]], $4, $7
   ; M3:             dsrl      $[[T5:[0-9]+]], $5, 1
   ; M3:             not       $[[T6:[0-9]+]], $[[T0]]
   ; M3:             dsrlv     $[[T7:[0-9]+]], $[[T5]], $[[T6]]
@@ -168,32 +167,32 @@ entry:
   ; M3:             jr        $ra
   ; M3:             nop
 
-  ; GP64-NOT-R6:    sll       $[[T0:[0-9]+]], $7, 0
-  ; GP64-NOT-R6:    dsllv     $[[T1:[0-9]+]], $4, $[[T0]]
-  ; GP64-NOT-R6:    dsrl      $[[T2:[0-9]+]], $5, 1
-  ; GP64-NOT-R6:    not       $[[T3:[0-9]+]], $[[T0]]
-  ; GP64-NOT-R6:    dsrlv     $[[T4:[0-9]+]], $[[T2]], $[[T3]]
-  ; GP64-NOT-R6:    or        $2, $[[T1]], $[[T4]]
-  ; GP64-NOT-R6:    dsllv     $3, $5, $[[T0]]
-  ; GP64-NOT-R6:    andi      $[[T5:[0-9]+]], $[[T0]], 32
+  ; GP64-NOT-R6:    dsllv     $[[T0:[0-9]+]], $4, $7
+  ; GP64-NOT-R6:    dsrl      $[[T1:[0-9]+]], $5, 1
+  ; GP64-NOT-R6:    sll       $[[T2:[0-9]+]], $7, 0
+  ; GP64-NOT-R6:    not       $[[T3:[0-9]+]], $[[T2]]
+  ; GP64-NOT-R6:    dsrlv     $[[T4:[0-9]+]], $[[T1]], $[[T3]]
+  ; GP64-NOT-R6:    or        $2, $[[T0]], $[[T4]]
+  ; GP64-NOT-R6:    dsllv     $3, $5, $7
+  ; GP64-NOT-R6:    andi      $[[T5:[0-9]+]], $[[T2]], 64
   ; GP64-NOT-R6:    movn      $2, $3, $[[T5]]
   ; GP64-NOT-R6:    jr        $ra
   ; GP64-NOT-R6:    movn      $3, $zero, $1
 
-  ; 64R6:           sll       $[[T0:[0-9]+]], $7, 0
-  ; 64R6:           dsllv     $[[T1:[0-9]+]], $4, $[[T0]]
-  ; 64R6:           dsrl      $[[T2:[0-9]+]], $5, 1
-  ; 64R6:           not       $[[T3:[0-9]+]], $[[T0]]
-  ; 64R6:           dsrlv     $[[T4:[0-9]+]], $[[T2]], $[[T3]]
-  ; 64R6:           or        $[[T5:[0-9]+]], $[[T1]], $[[T4]]
-  ; 64R6:           andi      $[[T6:[0-9]+]], $[[T0]], 32
+  ; 64R6:           dsllv     $[[T0:[0-9]+]], $4, $7
+  ; 64R6:           dsrl      $[[T1:[0-9]+]], $5, 1
+  ; 64R6:           sll       $[[T2:[0-9]+]], $7, 0
+  ; 64R6:           not       $[[T3:[0-9]+]], $[[T2]]
+  ; 64R6:           dsrlv     $[[T4:[0-9]+]], $[[T1]], $[[T3]]
+  ; 64R6:           or        $[[T5:[0-9]+]], $[[T0]], $[[T4]]
+  ; 64R6:           andi      $[[T6:[0-9]+]], $[[T2]], 64
   ; 64R6:           sll       $[[T7:[0-9]+]], $[[T6]], 0
   ; 64R6:           seleqz    $[[T8:[0-9]+]], $[[T5]], $[[T7]]
-  ; 64R6:           dsllv     $[[T9:[0-9]+]], $5, $[[T0]]
+  ; 64R6:           dsllv     $[[T9:[0-9]+]], $5, $7
   ; 64R6:           selnez    $[[T10:[0-9]+]], $[[T9]], $[[T7]]
   ; 64R6:           or        $2, $[[T10]], $[[T8]]
   ; 64R6:           jr        $ra
-  ; 64R6:           seleqz    $3, $[[T0]], $[[T7]]
+  ; 64R6:           seleqz    $3, $[[T9]], $[[T7]]
 
   %r = shl i128 %a, %b
   ret i128 %r
diff --git a/test/CodeGen/Mips/mbrsize4a.ll b/test/CodeGen/Mips/mbrsize4a.ll
index a99faccc8d2..264d2284afc 100644
--- a/test/CodeGen/Mips/mbrsize4a.ll
+++ b/test/CodeGen/Mips/mbrsize4a.ll
@@ -1,4 +1,4 @@
-; RUN: llc -mtriple=mipsel-linux-gnu -march=mipsel -mcpu=mips16 -soft-float -mips16-hard-float -relocation-model=static -mips16-constant-islands   < %s | FileCheck %s -check-prefix=jal16
+; RUN: llc -mtriple=mipsel-linux-gnu -march=mipsel -mcpu=mips16 -mattr=+soft-float -mips16-hard-float -relocation-model=static -mips16-constant-islands   < %s | FileCheck %s -check-prefix=jal16
 
 @j = global i32 10, align 4
 @.str = private unnamed_addr constant [11 x i8] c"at bottom\0A\00", align 1
diff --git a/test/CodeGen/Mips/mips16ex.ll b/test/CodeGen/Mips/mips16ex.ll
index 0f4dc7eb86c..c3a02261119 100644
--- a/test/CodeGen/Mips/mips16ex.ll
+++ b/test/CodeGen/Mips/mips16ex.ll
@@ -9,7 +9,7 @@
 @_ZTIi = external constant i8*
 @.str1 = private unnamed_addr constant [15 x i8] c"exception %i \0A\00", align 1
 
-define i32 @main() {
+define i32 @main() personality i8* bitcast (i32 (...)* @__gxx_personality_v0 to i8*) {
 entry:
   %retval = alloca i32, align 4
   %exn.slot = alloca i8*
@@ -24,7 +24,7 @@ entry:
           to label %unreachable unwind label %lpad
 
 lpad:                                             ; preds = %entry
-  %1 = landingpad { i8*, i32 } personality i8* bitcast (i32 (...)* @__gxx_personality_v0 to i8*)
+  %1 = landingpad { i8*, i32 }
           catch i8* bitcast (i8** @_ZTIi to i8*)
   %2 = extractvalue { i8*, i32 } %1, 0
   store i8* %2, i8** %exn.slot
@@ -45,7 +45,7 @@ catch:                                            ; preds = %catch.dispatch
   %exn.scalar = load i32, i32* %6
   store i32 %exn.scalar, i32* %e, align 4
   %7 = load i32, i32* %e, align 4
-  %call2 = invoke i32 (i8*, ...)* @printf(i8* getelementptr inbounds ([15 x i8], [15 x i8]* @.str1, i32 0, i32 0), i32 %7)
+  %call2 = invoke i32 (i8*, ...) @printf(i8* getelementptr inbounds ([15 x i8], [15 x i8]* @.str1, i32 0, i32 0), i32 %7)
           to label %invoke.cont unwind label %lpad1
 
 invoke.cont:                                      ; preds = %catch
@@ -56,7 +56,7 @@ try.cont:                                         ; preds = %invoke.cont
   ret i32 0
 
 lpad1:                                            ; preds = %catch
-  %8 = landingpad { i8*, i32 } personality i8* bitcast (i32 (...)* @__gxx_personality_v0 to i8*)
+  %8 = landingpad { i8*, i32 }
           cleanup
   %9 = extractvalue { i8*, i32 } %8, 0
   store i8* %9, i8** %exn.slot
diff --git a/test/CodeGen/Mips/mips16fpe.ll b/test/CodeGen/Mips/mips16fpe.ll
index f8b916da3a4..a9f92c2a8e0 100644
--- a/test/CodeGen/Mips/mips16fpe.ll
+++ b/test/CodeGen/Mips/mips16fpe.ll
@@ -297,7 +297,7 @@ entry:
   %and2 = and i1 %lnot, %cmp1
   %and = zext i1 %and2 to i32
   store i32 %and, i32* @ltsf2_result, align 4
-;16hf:  lw	${{[0-9]+}}, %call16(__mips16_unordsf2)(${{[0-9]+}})
+;16hf:  lw	${{[0-9]+}}, %call16(__mips16_ltsf2)(${{[0-9]+}})
 ;16hf:  lw	${{[0-9]+}}, %call16(__mips16_ltsf2)(${{[0-9]+}})
   ret void
 }
@@ -313,7 +313,7 @@ entry:
   %and2 = and i1 %lnot, %cmp1
   %and = zext i1 %and2 to i32
   store i32 %and, i32* @ltdf2_result, align 4
-;16hf:  lw	${{[0-9]+}}, %call16(__mips16_unorddf2)(${{[0-9]+}})
+;16hf:  lw	${{[0-9]+}}, %call16(__mips16_ltdf2)(${{[0-9]+}})
 ;16hf:  lw	${{[0-9]+}}, %call16(__mips16_ltdf2)(${{[0-9]+}})
   ret void
 }
diff --git a/test/CodeGen/Mips/mips64-f128.ll b/test/CodeGen/Mips/mips64-f128.ll
index 0217c7cefcd..d9c93810438 100644
--- a/test/CodeGen/Mips/mips64-f128.ll
+++ b/test/CodeGen/Mips/mips64-f128.ll
@@ -1,10 +1,10 @@
-; RUN: llc -mtriple=mips64el-unknown-unknown -mcpu=mips4 -soft-float -O1 \
+; RUN: llc -mtriple=mips64el-unknown-unknown -mcpu=mips4 -mattr=+soft-float -O1 \
 ; RUN:     -disable-mips-delay-filler < %s | FileCheck %s -check-prefix=ALL -check-prefix=C_CC_FMT
-; RUN: llc -mtriple=mips64el-unknown-unknown -mcpu=mips64 -soft-float -O1 \
+; RUN: llc -mtriple=mips64el-unknown-unknown -mcpu=mips64 -mattr=+soft-float -O1 \
 ; RUN:     -disable-mips-delay-filler < %s | FileCheck %s -check-prefix=ALL -check-prefix=C_CC_FMT
-; RUN: llc -mtriple=mips64el-unknown-unknown -mcpu=mips64r2 -soft-float -O1 \
+; RUN: llc -mtriple=mips64el-unknown-unknown -mcpu=mips64r2 -mattr=+soft-float -O1 \
 ; RUN:     -disable-mips-delay-filler < %s | FileCheck %s -check-prefix=ALL -check-prefix=C_CC_FMT
-; RUN: llc -mtriple=mips64el-unknown-unknown -mcpu=mips64r6 -soft-float -O1 \
+; RUN: llc -mtriple=mips64el-unknown-unknown -mcpu=mips64r6 -mattr=+soft-float -O1 \
 ; RUN:     -disable-mips-delay-filler < %s | FileCheck %s -check-prefix=ALL -check-prefix=CMP_CC_FMT
 
 @gld0 = external global fp128
diff --git a/test/CodeGen/Mips/mips64-libcall.ll b/test/CodeGen/Mips/mips64-libcall.ll
index 290baafd18b..8512e9fcb72 100644
--- a/test/CodeGen/Mips/mips64-libcall.ll
+++ b/test/CodeGen/Mips/mips64-libcall.ll
@@ -1,6 +1,6 @@
 ; RUN: llc -march=mips64el -mcpu=mips64r2 -O3 < %s |\
 ; RUN: FileCheck %s -check-prefix=HARD
-; RUN: llc -march=mips64el -mcpu=mips64r2 -soft-float < %s |\
+; RUN: llc -march=mips64el -mcpu=mips64r2 -mattr=+soft-float < %s |\
 ; RUN: FileCheck %s -check-prefix=SOFT
 
 ; Check that %add is not passed in an integer register.
diff --git a/test/CodeGen/Mips/mips64shift.ll b/test/CodeGen/Mips/mips64shift.ll
index 45d1c953227..52c6f906639 100644
--- a/test/CodeGen/Mips/mips64shift.ll
+++ b/test/CodeGen/Mips/mips64shift.ll
@@ -65,7 +65,8 @@ entry:
 
 define i64 @f9(i64 %a0, i64 %a1) nounwind readnone {
 entry:
-; CHECK: drotrv
+; CHECK-NOT:  sll
+; CHECK:      drotrv
   %shr = lshr i64 %a0, %a1
   %sub = sub i64 64, %a1
   %shl = shl i64 %a0, %sub
@@ -75,7 +76,8 @@ entry:
 
 define i64 @f10(i64 %a0, i64 %a1) nounwind readnone {
 entry:
-; CHECK: drotrv
+; CHECK-NOT:  sll
+; CHECK:      drotrv
   %shl = shl i64 %a0, %a1
   %sub = sub i64 64, %a1
   %shr = lshr i64 %a0, %sub
diff --git a/test/CodeGen/Mips/mips64signextendsesf.ll b/test/CodeGen/Mips/mips64signextendsesf.ll
index a76388ddda3..d0ce1b86bf5 100644
--- a/test/CodeGen/Mips/mips64signextendsesf.ll
+++ b/test/CodeGen/Mips/mips64signextendsesf.ll
@@ -1,4 +1,4 @@
-; RUN: llc -march=mips64 -mcpu=mips64r2 -soft-float -O2 < %s | FileCheck %s
+; RUN: llc -march=mips64 -mcpu=mips64r2 -mattr=+soft-float -O2 < %s | FileCheck %s
 
 define void @foosf() #0 {
 entry:
diff --git a/test/CodeGen/Mips/mips64sinttofpsf.ll b/test/CodeGen/Mips/mips64sinttofpsf.ll
index 7bd75bbe15b..87f803ecf55 100644
--- a/test/CodeGen/Mips/mips64sinttofpsf.ll
+++ b/test/CodeGen/Mips/mips64sinttofpsf.ll
@@ -1,4 +1,4 @@
-; RUN: llc -march=mips64 -mcpu=mips64r2 -soft-float -O0 < %s | FileCheck %s
+; RUN: llc -march=mips64 -mcpu=mips64r2 -mattr=+soft-float -O0 < %s | FileCheck %s
 
 
 define double @foo() #0 {
diff --git a/test/CodeGen/Mips/msa/basic_operations.ll b/test/CodeGen/Mips/msa/basic_operations.ll
index 97525be0491..2efec291193 100644
--- a/test/CodeGen/Mips/msa/basic_operations.ll
+++ b/test/CodeGen/Mips/msa/basic_operations.ll
@@ -1,5 +1,9 @@
-; RUN: llc -march=mips -mattr=+msa,+fp64 < %s | FileCheck -check-prefix=MIPS32-AE -check-prefix=MIPS32-BE %s
-; RUN: llc -march=mipsel -mattr=+msa,+fp64 < %s | FileCheck -check-prefix=MIPS32-AE -check-prefix=MIPS32-LE %s
+; RUN: llc -march=mips -mattr=+msa,+fp64 < %s | FileCheck -check-prefix=ALL -check-prefix=O32 -check-prefix=MIPS32 -check-prefix=ALL-BE %s
+; RUN: llc -march=mipsel -mattr=+msa,+fp64 < %s | FileCheck -check-prefix=ALL -check-prefix=O32 -check-prefix=MIPS32 -check-prefix=ALL-LE %s
+; RUN: llc -march=mips64 -target-abi n32 -mattr=+msa,+fp64 < %s | FileCheck -check-prefix=ALL -check-prefix=N32 -check-prefix=MIPS64 -check-prefix=ALL-BE %s
+; RUN: llc -march=mips64el -target-abi n32 -mattr=+msa,+fp64 < %s | FileCheck -check-prefix=ALL -check-prefix=N32 -check-prefix=MIPS64 -check-prefix=ALL-LE %s
+; RUN: llc -march=mips64 -mattr=+msa,+fp64 < %s | FileCheck -check-prefix=ALL -check-prefix=N64 -check-prefix=MIPS64 -check-prefix=ALL-BE %s
+; RUN: llc -march=mips64el -mattr=+msa,+fp64 < %s | FileCheck -check-prefix=ALL -check-prefix=N64 -check-prefix=MIPS64 -check-prefix=ALL-LE %s
 
 @v4i8 = global <4 x i8> <i8 0, i8 0, i8 0, i8 0>
 @v16i8 = global <16 x i8> <i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0>
@@ -10,135 +14,153 @@
 @i64 = global i64 0
 
 define void @const_v16i8() nounwind {
-  ; MIPS32-AE-LABEL: const_v16i8:
+  ; ALL-LABEL: const_v16i8:
 
   store volatile <16 x i8> <i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0>, <16 x i8>*@v16i8
-  ; MIPS32-AE: ldi.b [[R1:\$w[0-9]+]], 0
+  ; ALL: ldi.b [[R1:\$w[0-9]+]], 0
 
   store volatile <16 x i8> <i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1>, <16 x i8>*@v16i8
-  ; MIPS32-AE: ldi.b [[R1:\$w[0-9]+]], 1
+  ; ALL: ldi.b [[R1:\$w[0-9]+]], 1
 
   store volatile <16 x i8> <i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 31>, <16 x i8>*@v16i8
-  ; MIPS32-AE: addiu [[G_PTR:\$[0-9]+]], {{.*}}, %lo($
-  ; MIPS32-AE: ld.b  [[R1:\$w[0-9]+]], 0([[G_PTR]])
+  ; O32: addiu [[G_PTR:\$[0-9]+]], {{.*}}, %lo($
+  ; N32: addiu [[G_PTR:\$[0-9]+]], {{.*}}, %got_ofst($
+  ; N64: daddiu [[G_PTR:\$[0-9]+]], {{.*}}, %got_ofst($
+  ; ALL: ld.b  [[R1:\$w[0-9]+]], 0([[G_PTR]])
 
   store volatile <16 x i8> <i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 8, i8 9, i8 0, i8 1, i8 2, i8 3, i8 4, i8 5, i8 6>, <16 x i8>*@v16i8
-  ; MIPS32-AE: addiu [[G_PTR:\$[0-9]+]], {{.*}}, %lo($
-  ; MIPS32-AE: ld.b  [[R1:\$w[0-9]+]], 0([[G_PTR]])
+  ; O32: addiu [[G_PTR:\$[0-9]+]], {{.*}}, %lo($
+  ; N32: addiu [[G_PTR:\$[0-9]+]], {{.*}}, %got_ofst($
+  ; N64: daddiu [[G_PTR:\$[0-9]+]], {{.*}}, %got_ofst($
+  ; ALL: ld.b  [[R1:\$w[0-9]+]], 0([[G_PTR]])
 
   store volatile <16 x i8> <i8 1, i8 0, i8 1, i8 0, i8 1, i8 0, i8 1, i8 0, i8 1, i8 0, i8 1, i8 0, i8 1, i8 0, i8 1, i8 0>, <16 x i8>*@v16i8
-  ; MIPS32-BE: ldi.h [[R1:\$w[0-9]+]], 256
-  ; MIPS32-LE: ldi.h [[R1:\$w[0-9]+]], 1
+  ; ALL-BE: ldi.h [[R1:\$w[0-9]+]], 256
+  ; ALL-LE: ldi.h [[R1:\$w[0-9]+]], 1
 
   store volatile <16 x i8> <i8 1, i8 2, i8 3, i8 4, i8 1, i8 2, i8 3, i8 4, i8 1, i8 2, i8 3, i8 4, i8 1, i8 2, i8 3, i8 4>, <16 x i8>*@v16i8
-  ; MIPS32-BE-DAG: lui [[R2:\$[0-9]+]], 258
-  ; MIPS32-LE-DAG: lui [[R2:\$[0-9]+]], 1027
-  ; MIPS32-BE-DAG: ori [[R2]], [[R2]], 772
-  ; MIPS32-LE-DAG: ori [[R2]], [[R2]], 513
-  ; MIPS32-AE-DAG: fill.w [[R1:\$w[0-9]+]], [[R2]]
+  ; ALL-BE-DAG: lui [[R2:\$[0-9]+]], 258
+  ; ALL-LE-DAG: lui [[R2:\$[0-9]+]], 1027
+  ; ALL-BE-DAG: ori [[R2]], [[R2]], 772
+  ; ALL-LE-DAG: ori [[R2]], [[R2]], 513
+  ; ALL-DAG: fill.w [[R1:\$w[0-9]+]], [[R2]]
 
   store volatile <16 x i8> <i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 8, i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 8>, <16 x i8>*@v16i8
-  ; MIPS32-AE: addiu [[G_PTR:\$[0-9]+]], {{.*}}, %lo($
-  ; MIPS32-AE: ld.b  [[R1:\$w[0-9]+]], 0([[G_PTR]])
+  ; O32: addiu [[G_PTR:\$[0-9]+]], {{.*}}, %lo($
+  ; N32: addiu [[G_PTR:\$[0-9]+]], {{.*}}, %got_ofst($
+  ; N64: daddiu [[G_PTR:\$[0-9]+]], {{.*}}, %got_ofst($
+  ; ALL: ld.b  [[R1:\$w[0-9]+]], 0([[G_PTR]])
 
   ret void
-  ; MIPS32-AE: .size const_v16i8
 }
 
 define void @const_v8i16() nounwind {
-  ; MIPS32-AE-LABEL: const_v8i16:
+  ; ALL-LABEL: const_v8i16:
 
   store volatile <8 x i16> <i16 0, i16 0, i16 0, i16 0, i16 0, i16 0, i16 0, i16 0>, <8 x i16>*@v8i16
-  ; MIPS32-AE: ldi.b [[R1:\$w[0-9]+]], 0
+  ; ALL: ldi.b [[R1:\$w[0-9]+]], 0
 
   store volatile <8 x i16> <i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1>, <8 x i16>*@v8i16
-  ; MIPS32-AE: ldi.h [[R1:\$w[0-9]+]], 1
+  ; ALL: ldi.h [[R1:\$w[0-9]+]], 1
 
   store volatile <8 x i16> <i16 1, i16 1, i16 1, i16 2, i16 1, i16 1, i16 1, i16 31>, <8 x i16>*@v8i16
-  ; MIPS32-AE: addiu [[G_PTR:\$[0-9]+]], {{.*}}, %lo($
-  ; MIPS32-AE: ld.h  [[R1:\$w[0-9]+]], 0([[G_PTR]])
+  ; O32: addiu [[G_PTR:\$[0-9]+]], {{.*}}, %lo($
+  ; N32: addiu [[G_PTR:\$[0-9]+]], {{.*}}, %got_ofst($
+  ; N64: daddiu [[G_PTR:\$[0-9]+]], {{.*}}, %got_ofst($
+  ; ALL: ld.h  [[R1:\$w[0-9]+]], 0([[G_PTR]])
 
   store volatile <8 x i16> <i16 1028, i16 1028, i16 1028, i16 1028, i16 1028, i16 1028, i16 1028, i16 1028>, <8 x i16>*@v8i16
-  ; MIPS32-AE: ldi.b [[R1:\$w[0-9]+]], 4
+  ; ALL: ldi.b [[R1:\$w[0-9]+]], 4
 
   store volatile <8 x i16> <i16 1, i16 2, i16 1, i16 2, i16 1, i16 2, i16 1, i16 2>, <8 x i16>*@v8i16
-  ; MIPS32-BE-DAG: lui [[R2:\$[0-9]+]], 1
-  ; MIPS32-LE-DAG: lui [[R2:\$[0-9]+]], 2
-  ; MIPS32-BE-DAG: ori [[R2]], [[R2]], 2
-  ; MIPS32-LE-DAG: ori [[R2]], [[R2]], 1
-  ; MIPS32-AE-DAG: fill.w [[R1:\$w[0-9]+]], [[R2]]
+  ; ALL-BE-DAG: lui [[R2:\$[0-9]+]], 1
+  ; ALL-LE-DAG: lui [[R2:\$[0-9]+]], 2
+  ; ALL-BE-DAG: ori [[R2]], [[R2]], 2
+  ; ALL-LE-DAG: ori [[R2]], [[R2]], 1
+  ; ALL-DAG: fill.w [[R1:\$w[0-9]+]], [[R2]]
 
   store volatile <8 x i16> <i16 1, i16 2, i16 3, i16 4, i16 1, i16 2, i16 3, i16 4>, <8 x i16>*@v8i16
-  ; MIPS32-AE: addiu [[G_PTR:\$[0-9]+]], {{.*}}, %lo($
-  ; MIPS32-AE: ld.h  [[R1:\$w[0-9]+]], 0([[G_PTR]])
+  ; O32: addiu [[G_PTR:\$[0-9]+]], {{.*}}, %lo($
+  ; N32: addiu [[G_PTR:\$[0-9]+]], {{.*}}, %got_ofst($
+  ; N64: daddiu [[G_PTR:\$[0-9]+]], {{.*}}, %got_ofst($
+  ; ALL: ld.h  [[R1:\$w[0-9]+]], 0([[G_PTR]])
 
   ret void
-  ; MIPS32-AE: .size const_v8i16
 }
 
 define void @const_v4i32() nounwind {
-  ; MIPS32-AE-LABEL: const_v4i32:
+  ; ALL-LABEL: const_v4i32:
 
   store volatile <4 x i32> <i32 0, i32 0, i32 0, i32 0>, <4 x i32>*@v4i32
-  ; MIPS32-AE: ldi.b [[R1:\$w[0-9]+]], 0
+  ; ALL: ldi.b [[R1:\$w[0-9]+]], 0
 
   store volatile <4 x i32> <i32 1, i32 1, i32 1, i32 1>, <4 x i32>*@v4i32
-  ; MIPS32-AE: ldi.w [[R1:\$w[0-9]+]], 1
+  ; ALL: ldi.w [[R1:\$w[0-9]+]], 1
 
   store volatile <4 x i32> <i32 1, i32 1, i32 1, i32 31>, <4 x i32>*@v4i32
-  ; MIPS32-AE: addiu [[G_PTR:\$[0-9]+]], {{.*}}, %lo($
-  ; MIPS32-AE: ld.w  [[R1:\$w[0-9]+]], 0([[G_PTR]])
+  ; O32: addiu [[G_PTR:\$[0-9]+]], {{.*}}, %lo($
+  ; N32: addiu [[G_PTR:\$[0-9]+]], {{.*}}, %got_ofst($
+  ; N64: daddiu [[G_PTR:\$[0-9]+]], {{.*}}, %got_ofst($
+  ; ALL: ld.w  [[R1:\$w[0-9]+]], 0([[G_PTR]])
 
   store volatile <4 x i32> <i32 16843009, i32 16843009, i32 16843009, i32 16843009>, <4 x i32>*@v4i32
-  ; MIPS32-AE: ldi.b [[R1:\$w[0-9]+]], 1
+  ; ALL: ldi.b [[R1:\$w[0-9]+]], 1
 
   store volatile <4 x i32> <i32 65537, i32 65537, i32 65537, i32 65537>, <4 x i32>*@v4i32
-  ; MIPS32-AE: ldi.h [[R1:\$w[0-9]+]], 1
+  ; ALL: ldi.h [[R1:\$w[0-9]+]], 1
 
   store volatile <4 x i32> <i32 1, i32 2, i32 1, i32 2>, <4 x i32>*@v4i32
-  ; MIPS32-AE: addiu [[G_PTR:\$[0-9]+]], {{.*}}, %lo($
-  ; MIPS32-AE: ld.w  [[R1:\$w[0-9]+]], 0([[G_PTR]])
+  ; O32: addiu [[G_PTR:\$[0-9]+]], {{.*}}, %lo($
+  ; N32: addiu [[G_PTR:\$[0-9]+]], {{.*}}, %got_ofst($
+  ; N64: daddiu [[G_PTR:\$[0-9]+]], {{.*}}, %got_ofst($
+  ; ALL: ld.w  [[R1:\$w[0-9]+]], 0([[G_PTR]])
 
   store volatile <4 x i32> <i32 3, i32 4, i32 5, i32 6>, <4 x i32>*@v4i32
-  ; MIPS32-AE: addiu [[G_PTR:\$[0-9]+]], {{.*}}, %lo($
-  ; MIPS32-AE: ld.w  [[R1:\$w[0-9]+]], 0([[G_PTR]])
+  ; O32: addiu [[G_PTR:\$[0-9]+]], {{.*}}, %lo($
+  ; N32: addiu [[G_PTR:\$[0-9]+]], {{.*}}, %got_ofst($
+  ; N64: daddiu [[G_PTR:\$[0-9]+]], {{.*}}, %got_ofst($
+  ; ALL: ld.w  [[R1:\$w[0-9]+]], 0([[G_PTR]])
 
   ret void
-  ; MIPS32-AE: .size const_v4i32
 }
 
 define void @const_v2i64() nounwind {
-  ; MIPS32-AE-LABEL: const_v2i64:
+  ; ALL-LABEL: const_v2i64:
 
   store volatile <2 x i64> <i64 0, i64 0>, <2 x i64>*@v2i64
-  ; MIPS32-AE: ldi.b [[R1:\$w[0-9]+]], 0
+  ; ALL: ldi.b [[R1:\$w[0-9]+]], 0
 
   store volatile <2 x i64> <i64 72340172838076673, i64 72340172838076673>, <2 x i64>*@v2i64
-  ; MIPS32-AE: ldi.b [[R1:\$w[0-9]+]], 1
+  ; ALL: ldi.b [[R1:\$w[0-9]+]], 1
 
   store volatile <2 x i64> <i64 281479271743489, i64 281479271743489>, <2 x i64>*@v2i64
-  ; MIPS32-AE: ldi.h [[R1:\$w[0-9]+]], 1
+  ; ALL: ldi.h [[R1:\$w[0-9]+]], 1
 
   store volatile <2 x i64> <i64 4294967297, i64 4294967297>, <2 x i64>*@v2i64
-  ; MIPS32-AE: ldi.w [[R1:\$w[0-9]+]], 1
+  ; ALL: ldi.w [[R1:\$w[0-9]+]], 1
 
   store volatile <2 x i64> <i64 1, i64 1>, <2 x i64>*@v2i64
-  ; MIPS32-AE: ldi.d [[R1:\$w[0-9]+]], 1
+  ; ALL: ldi.d [[R1:\$w[0-9]+]], 1
 
   store volatile <2 x i64> <i64 1, i64 31>, <2 x i64>*@v2i64
-  ; MIPS32-AE: addiu [[G_PTR:\$[0-9]+]], {{.*}}, %lo($
-  ; MIPS32-AE: ld.w  [[R1:\$w[0-9]+]], 0([[G_PTR]])
+  ; O32: addiu [[G_PTR:\$[0-9]+]], {{.*}}, %lo($
+  ; N32: addiu [[G_PTR:\$[0-9]+]], {{.*}}, %got_ofst($
+  ; N64: daddiu [[G_PTR:\$[0-9]+]], {{.*}}, %got_ofst($
+  ; MIPS32: ld.w [[R1:\$w[0-9]+]], 0([[G_PTR]])
+  ; MIPS64: ld.d [[R1:\$w[0-9]+]], 0([[G_PTR]])
 
   store volatile <2 x i64> <i64 3, i64 4>, <2 x i64>*@v2i64
-  ; MIPS32-AE: addiu [[G_PTR:\$[0-9]+]], {{.*}}, %lo($
-  ; MIPS32-AE: ld.w  [[R1:\$w[0-9]+]], 0([[G_PTR]])
+  ; O32: addiu [[G_PTR:\$[0-9]+]], {{.*}}, %lo($
+  ; N32: addiu [[G_PTR:\$[0-9]+]], {{.*}}, %got_ofst($
+  ; N64: daddiu [[G_PTR:\$[0-9]+]], {{.*}}, %got_ofst($
+  ; MIPS32: ld.w  [[R1:\$w[0-9]+]], 0([[G_PTR]])
+  ; MIPS64: ld.d  [[R1:\$w[0-9]+]], 0([[G_PTR]])
 
   ret void
-  ; MIPS32-AE: .size const_v2i64
 }
 
-define void @nonconst_v16i8(i8 %a, i8 %b, i8 %c, i8 %d, i8 %e, i8 %f, i8 %g, i8 %h) nounwind {
-  ; MIPS32-AE-LABEL: nonconst_v16i8:
+define void @nonconst_v16i8(i8 signext %a, i8 signext %b, i8 signext %c, i8 signext %d, i8 signext %e, i8 signext %f, i8 signext %g, i8 signext %h) nounwind {
+  ; ALL-LABEL: nonconst_v16i8:
 
   %1 = insertelement <16 x i8> undef, i8 %a, i32 0
   %2 = insertelement <16 x i8> %1, i8 %b, i32 1
@@ -156,39 +178,38 @@ define void @nonconst_v16i8(i8 %a, i8 %b, i8 %c, i8 %d, i8 %e, i8 %f, i8 %g, i8
   %14 = insertelement <16 x i8> %13, i8 %h, i32 13
   %15 = insertelement <16 x i8> %14, i8 %h, i32 14
   %16 = insertelement <16 x i8> %15, i8 %h, i32 15
-  ; MIPS32-AE-DAG: insert.b [[R1:\$w[0-9]+]][0], $4
-  ; MIPS32-AE-DAG: insert.b [[R1]][1], $5
-  ; MIPS32-AE-DAG: insert.b [[R1]][2], $6
-  ; MIPS32-AE-DAG: insert.b [[R1]][3], $7
-  ; MIPS32-BE-DAG: lbu [[R2:\$[0-9]+]], 19($sp)
-  ; MIPS32-LE-DAG: lbu [[R2:\$[0-9]+]], 16($sp)
-  ; MIPS32-AE-DAG: insert.b [[R1]][4], [[R2]]
-  ; MIPS32-BE-DAG: lbu [[R3:\$[0-9]+]], 23($sp)
-  ; MIPS32-LE-DAG: lbu [[R3:\$[0-9]+]], 20($sp)
-  ; MIPS32-AE-DAG: insert.b [[R1]][5], [[R3]]
-  ; MIPS32-BE-DAG: lbu [[R4:\$[0-9]+]], 27($sp)
-  ; MIPS32-LE-DAG: lbu [[R4:\$[0-9]+]], 24($sp)
-  ; MIPS32-AE-DAG: insert.b [[R1]][6], [[R4]]
-  ; MIPS32-BE-DAG: lbu [[R5:\$[0-9]+]], 31($sp)
-  ; MIPS32-LE-DAG: lbu [[R5:\$[0-9]+]], 28($sp)
-  ; MIPS32-AE-DAG: insert.b [[R1]][7], [[R5]]
-  ; MIPS32-AE-DAG: insert.b [[R1]][8], [[R5]]
-  ; MIPS32-AE-DAG: insert.b [[R1]][9], [[R5]]
-  ; MIPS32-AE-DAG: insert.b [[R1]][10], [[R5]]
-  ; MIPS32-AE-DAG: insert.b [[R1]][11], [[R5]]
-  ; MIPS32-AE-DAG: insert.b [[R1]][12], [[R5]]
-  ; MIPS32-AE-DAG: insert.b [[R1]][13], [[R5]]
-  ; MIPS32-AE-DAG: insert.b [[R1]][14], [[R5]]
-  ; MIPS32-AE-DAG: insert.b [[R1]][15], [[R5]]
+  ; ALL-DAG: insert.b [[R1:\$w[0-9]+]][0], $4
+  ; ALL-DAG: insert.b [[R1]][1], $5
+  ; ALL-DAG: insert.b [[R1]][2], $6
+  ; ALL-DAG: insert.b [[R1]][3], $7
+  ; MIPS32-DAG: lw [[R2:\$[0-9]+]], 16($sp)
+  ; MIPS32-DAG: insert.b [[R1]][4], [[R2]]
+  ; MIPS64-DAG: insert.b [[R1]][4], $8
+  ; MIPS32-DAG: lw [[R3:\$[0-9]+]], 20($sp)
+  ; MIPS32-DAG: insert.b [[R1]][5], [[R3]]
+  ; MIPS64-DAG: insert.b [[R1]][5], $9
+  ; MIPS32-DAG: lw [[R4:\$[0-9]+]], 24($sp)
+  ; MIPS32-DAG: insert.b [[R1]][6], [[R4]]
+  ; MIPS64-DAG: insert.b [[R1]][6], $10
+  ; MIPS32-DAG: lw [[R5:\$[0-9]+]], 28($sp)
+  ; MIPS32-DAG: insert.b [[R1]][7], [[R5]]
+  ; MIPS64-DAG: insert.b [[R1]][7], [[R5:\$11]]
+  ; ALL-DAG: insert.b [[R1]][8], [[R5]]
+  ; ALL-DAG: insert.b [[R1]][9], [[R5]]
+  ; ALL-DAG: insert.b [[R1]][10], [[R5]]
+  ; ALL-DAG: insert.b [[R1]][11], [[R5]]
+  ; ALL-DAG: insert.b [[R1]][12], [[R5]]
+  ; ALL-DAG: insert.b [[R1]][13], [[R5]]
+  ; ALL-DAG: insert.b [[R1]][14], [[R5]]
+  ; ALL-DAG: insert.b [[R1]][15], [[R5]]
 
   store volatile <16 x i8> %16, <16 x i8>*@v16i8
 
   ret void
-  ; MIPS32-AE: .size nonconst_v16i8
 }
 
-define void @nonconst_v8i16(i16 %a, i16 %b, i16 %c, i16 %d, i16 %e, i16 %f, i16 %g, i16 %h) nounwind {
-  ; MIPS32-AE-LABEL: nonconst_v8i16:
+define void @nonconst_v8i16(i16 signext %a, i16 signext %b, i16 signext %c, i16 signext %d, i16 signext %e, i16 signext %f, i16 signext %g, i16 signext %h) nounwind {
+  ; ALL-LABEL: nonconst_v8i16:
 
   %1 = insertelement <8 x i16> undef, i16 %a, i32 0
   %2 = insertelement <8 x i16> %1, i16 %b, i32 1
@@ -198,610 +219,648 @@ define void @nonconst_v8i16(i16 %a, i16 %b, i16 %c, i16 %d, i16 %e, i16 %f, i16
   %6 = insertelement <8 x i16> %5, i16 %f, i32 5
   %7 = insertelement <8 x i16> %6, i16 %g, i32 6
   %8 = insertelement <8 x i16> %7, i16 %h, i32 7
-  ; MIPS32-AE-DAG: insert.h [[R1:\$w[0-9]+]][0], $4
-  ; MIPS32-AE-DAG: insert.h [[R1]][1], $5
-  ; MIPS32-AE-DAG: insert.h [[R1]][2], $6
-  ; MIPS32-AE-DAG: insert.h [[R1]][3], $7
-  ; MIPS32-BE-DAG: lhu [[R2:\$[0-9]+]], 18($sp)
-  ; MIPS32-LE-DAG: lhu [[R2:\$[0-9]+]], 16($sp)
-  ; MIPS32-AE-DAG: insert.h [[R1]][4], [[R2]]
-  ; MIPS32-BE-DAG: lhu [[R2:\$[0-9]+]], 22($sp)
-  ; MIPS32-LE-DAG: lhu [[R2:\$[0-9]+]], 20($sp)
-  ; MIPS32-AE-DAG: insert.h [[R1]][5], [[R2]]
-  ; MIPS32-BE-DAG: lhu [[R2:\$[0-9]+]], 26($sp)
-  ; MIPS32-LE-DAG: lhu [[R2:\$[0-9]+]], 24($sp)
-  ; MIPS32-AE-DAG: insert.h [[R1]][6], [[R2]]
-  ; MIPS32-BE-DAG: lhu [[R2:\$[0-9]+]], 30($sp)
-  ; MIPS32-LE-DAG: lhu [[R2:\$[0-9]+]], 28($sp)
-  ; MIPS32-AE-DAG: insert.h [[R1]][7], [[R2]]
+  ; ALL-DAG: insert.h [[R1:\$w[0-9]+]][0], $4
+  ; ALL-DAG: insert.h [[R1]][1], $5
+  ; ALL-DAG: insert.h [[R1]][2], $6
+  ; ALL-DAG: insert.h [[R1]][3], $7
+  ; MIPS32-DAG: lw [[R2:\$[0-9]+]], 16($sp)
+  ; MIPS32-DAG: insert.h [[R1]][4], [[R2]]
+  ; MIPS64-DAG: insert.h [[R1]][4], $8
+  ; MIPS32-DAG: lw [[R2:\$[0-9]+]], 20($sp)
+  ; MIPS32-DAG: insert.h [[R1]][5], [[R2]]
+  ; MIPS64-DAG: insert.h [[R1]][5], $9
+  ; MIPS32-DAG: lw [[R2:\$[0-9]+]], 24($sp)
+  ; MIPS32-DAG: insert.h [[R1]][6], [[R2]]
+  ; MIPS64-DAG: insert.h [[R1]][6], $10
+  ; MIPS32-DAG: lw [[R2:\$[0-9]+]], 28($sp)
+  ; MIPS32-DAG: insert.h [[R1]][7], [[R2]]
+  ; MIPS64-DAG: insert.h [[R1]][7], $11
 
   store volatile <8 x i16> %8, <8 x i16>*@v8i16
 
   ret void
-  ; MIPS32-AE: .size nonconst_v8i16
 }
 
-define void @nonconst_v4i32(i32 %a, i32 %b, i32 %c, i32 %d) nounwind {
-  ; MIPS32-AE-LABEL: nonconst_v4i32:
+define void @nonconst_v4i32(i32 signext %a, i32 signext %b, i32 signext %c, i32 signext %d) nounwind {
+  ; ALL-LABEL: nonconst_v4i32:
 
   %1 = insertelement <4 x i32> undef, i32 %a, i32 0
   %2 = insertelement <4 x i32> %1, i32 %b, i32 1
   %3 = insertelement <4 x i32> %2, i32 %c, i32 2
   %4 = insertelement <4 x i32> %3, i32 %d, i32 3
-  ; MIPS32-AE: insert.w [[R1:\$w[0-9]+]][0], $4
-  ; MIPS32-AE: insert.w [[R1]][1], $5
-  ; MIPS32-AE: insert.w [[R1]][2], $6
-  ; MIPS32-AE: insert.w [[R1]][3], $7
+  ; ALL: insert.w [[R1:\$w[0-9]+]][0], $4
+  ; ALL: insert.w [[R1]][1], $5
+  ; ALL: insert.w [[R1]][2], $6
+  ; ALL: insert.w [[R1]][3], $7
 
   store volatile <4 x i32> %4, <4 x i32>*@v4i32
 
   ret void
-  ; MIPS32-AE: .size nonconst_v4i32
 }
 
-define void @nonconst_v2i64(i64 %a, i64 %b) nounwind {
-  ; MIPS32-AE-LABEL: nonconst_v2i64:
+define void @nonconst_v2i64(i64 signext %a, i64 signext %b) nounwind {
+  ; ALL-LABEL: nonconst_v2i64:
 
   %1 = insertelement <2 x i64> undef, i64 %a, i32 0
   %2 = insertelement <2 x i64> %1, i64 %b, i32 1
-  ; MIPS32-AE: insert.w [[R1:\$w[0-9]+]][0], $4
-  ; MIPS32-AE: insert.w [[R1]][1], $5
-  ; MIPS32-AE: insert.w [[R1]][2], $6
-  ; MIPS32-AE: insert.w [[R1]][3], $7
+  ; MIPS32: insert.w [[R1:\$w[0-9]+]][0], $4
+  ; MIPS32: insert.w [[R1]][1], $5
+  ; MIPS32: insert.w [[R1]][2], $6
+  ; MIPS32: insert.w [[R1]][3], $7
+  ; MIPS64: insert.d [[R1:\$w[0-9]+]][0], $4
+  ; MIPS64: insert.d [[R1]][1], $5
 
   store volatile <2 x i64> %2, <2 x i64>*@v2i64
 
   ret void
-  ; MIPS32-AE: .size nonconst_v2i64
 }
 
 define i32 @extract_sext_v16i8() nounwind {
-  ; MIPS32-AE-LABEL: extract_sext_v16i8:
+  ; ALL-LABEL: extract_sext_v16i8:
 
   %1 = load <16 x i8>, <16 x i8>* @v16i8
-  ; MIPS32-AE-DAG: ld.b [[R1:\$w[0-9]+]],
+  ; ALL-DAG: ld.b [[R1:\$w[0-9]+]],
 
   %2 = add <16 x i8> %1, %1
-  ; MIPS32-AE-DAG: addv.b [[R2:\$w[0-9]+]], [[R1]], [[R1]]
+  ; ALL-DAG: addv.b [[R2:\$w[0-9]+]], [[R1]], [[R1]]
 
   %3 = extractelement <16 x i8> %2, i32 1
   %4 = sext i8 %3 to i32
-  ; MIPS32-AE-DAG: copy_s.b [[R3:\$[0-9]+]], [[R1]][1]
-  ; MIPS32-AE-NOT: sll
-  ; MIPS32-AE-NOT: sra
+  ; ALL-DAG: copy_s.b [[R3:\$[0-9]+]], [[R1]][1]
+  ; ALL-NOT: sll
+  ; ALL-NOT: sra
 
   ret i32 %4
-  ; MIPS32-AE: .size extract_sext_v16i8
 }
 
 define i32 @extract_sext_v8i16() nounwind {
-  ; MIPS32-AE-LABEL: extract_sext_v8i16:
+  ; ALL-LABEL: extract_sext_v8i16:
 
   %1 = load <8 x i16>, <8 x i16>* @v8i16
-  ; MIPS32-AE-DAG: ld.h [[R1:\$w[0-9]+]],
+  ; ALL-DAG: ld.h [[R1:\$w[0-9]+]],
 
   %2 = add <8 x i16> %1, %1
-  ; MIPS32-AE-DAG: addv.h [[R2:\$w[0-9]+]], [[R1]], [[R1]]
+  ; ALL-DAG: addv.h [[R2:\$w[0-9]+]], [[R1]], [[R1]]
 
   %3 = extractelement <8 x i16> %2, i32 1
   %4 = sext i16 %3 to i32
-  ; MIPS32-AE-DAG: copy_s.h [[R3:\$[0-9]+]], [[R1]][1]
-  ; MIPS32-AE-NOT: sll
-  ; MIPS32-AE-NOT: sra
+  ; ALL-DAG: copy_s.h [[R3:\$[0-9]+]], [[R1]][1]
+  ; ALL-NOT: sll
+  ; ALL-NOT: sra
 
   ret i32 %4
-  ; MIPS32-AE: .size extract_sext_v8i16
 }
 
 define i32 @extract_sext_v4i32() nounwind {
-  ; MIPS32-AE-LABEL: extract_sext_v4i32:
+  ; ALL-LABEL: extract_sext_v4i32:
 
   %1 = load <4 x i32>, <4 x i32>* @v4i32
-  ; MIPS32-AE-DAG: ld.w [[R1:\$w[0-9]+]],
+  ; ALL-DAG: ld.w [[R1:\$w[0-9]+]],
 
   %2 = add <4 x i32> %1, %1
-  ; MIPS32-AE-DAG: addv.w [[R2:\$w[0-9]+]], [[R1]], [[R1]]
+  ; ALL-DAG: addv.w [[R2:\$w[0-9]+]], [[R1]], [[R1]]
 
   %3 = extractelement <4 x i32> %2, i32 1
-  ; MIPS32-AE-DAG: copy_s.w [[R3:\$[0-9]+]], [[R1]][1]
+  ; ALL-DAG: copy_s.w [[R3:\$[0-9]+]], [[R1]][1]
 
   ret i32 %3
-  ; MIPS32-AE: .size extract_sext_v4i32
 }
 
 define i64 @extract_sext_v2i64() nounwind {
-  ; MIPS32-AE-LABEL: extract_sext_v2i64:
+  ; ALL-LABEL: extract_sext_v2i64:
 
   %1 = load <2 x i64>, <2 x i64>* @v2i64
-  ; MIPS32-AE-DAG: ld.d [[R1:\$w[0-9]+]],
+  ; ALL-DAG: ld.d [[R1:\$w[0-9]+]],
 
   %2 = add <2 x i64> %1, %1
-  ; MIPS32-AE-DAG: addv.d [[R2:\$w[0-9]+]], [[R1]], [[R1]]
+  ; ALL-DAG: addv.d [[R2:\$w[0-9]+]], [[R1]], [[R1]]
 
   %3 = extractelement <2 x i64> %2, i32 1
-  ; MIPS32-AE-DAG: copy_s.w [[R3:\$[0-9]+]], [[R1]][2]
-  ; MIPS32-AE-DAG: copy_s.w [[R4:\$[0-9]+]], [[R1]][3]
-  ; MIPS32-AE-NOT: sll
-  ; MIPS32-AE-NOT: sra
+  ; MIPS32-DAG: copy_s.w [[R3:\$[0-9]+]], [[R1]][2]
+  ; MIPS32-DAG: copy_s.w [[R4:\$[0-9]+]], [[R1]][3]
+  ; MIPS64-DAG: copy_s.d [[R3:\$[0-9]+]], [[R1]][1]
+  ; ALL-NOT: sll
+  ; ALL-NOT: sra
 
   ret i64 %3
-  ; MIPS32-AE: .size extract_sext_v2i64
 }
 
 define i32 @extract_zext_v16i8() nounwind {
-  ; MIPS32-AE-LABEL: extract_zext_v16i8:
+  ; ALL-LABEL: extract_zext_v16i8:
 
   %1 = load <16 x i8>, <16 x i8>* @v16i8
-  ; MIPS32-AE-DAG: ld.b [[R1:\$w[0-9]+]],
+  ; ALL-DAG: ld.b [[R1:\$w[0-9]+]],
 
   %2 = add <16 x i8> %1, %1
-  ; MIPS32-AE-DAG: addv.b [[R2:\$w[0-9]+]], [[R1]], [[R1]]
+  ; ALL-DAG: addv.b [[R2:\$w[0-9]+]], [[R1]], [[R1]]
 
   %3 = extractelement <16 x i8> %2, i32 1
   %4 = zext i8 %3 to i32
-  ; MIPS32-AE-DAG: copy_u.b [[R3:\$[0-9]+]], [[R1]][1]
-  ; MIPS32-AE-NOT: andi
+  ; ALL-DAG: copy_u.b [[R3:\$[0-9]+]], [[R1]][1]
+  ; ALL-NOT: andi
 
   ret i32 %4
-  ; MIPS32-AE: .size extract_zext_v16i8
 }
 
 define i32 @extract_zext_v8i16() nounwind {
-  ; MIPS32-AE-LABEL: extract_zext_v8i16:
+  ; ALL-LABEL: extract_zext_v8i16:
 
   %1 = load <8 x i16>, <8 x i16>* @v8i16
-  ; MIPS32-AE-DAG: ld.h [[R1:\$w[0-9]+]],
+  ; ALL-DAG: ld.h [[R1:\$w[0-9]+]],
 
   %2 = add <8 x i16> %1, %1
-  ; MIPS32-AE-DAG: addv.h [[R2:\$w[0-9]+]], [[R1]], [[R1]]
+  ; ALL-DAG: addv.h [[R2:\$w[0-9]+]], [[R1]], [[R1]]
 
   %3 = extractelement <8 x i16> %2, i32 1
   %4 = zext i16 %3 to i32
-  ; MIPS32-AE-DAG: copy_u.h [[R3:\$[0-9]+]], [[R1]][1]
-  ; MIPS32-AE-NOT: andi
+  ; ALL-DAG: copy_u.h [[R3:\$[0-9]+]], [[R1]][1]
+  ; ALL-NOT: andi
 
   ret i32 %4
-  ; MIPS32-AE: .size extract_zext_v8i16
 }
 
 define i32 @extract_zext_v4i32() nounwind {
-  ; MIPS32-AE-LABEL: extract_zext_v4i32:
+  ; ALL-LABEL: extract_zext_v4i32:
 
   %1 = load <4 x i32>, <4 x i32>* @v4i32
-  ; MIPS32-AE-DAG: ld.w [[R1:\$w[0-9]+]],
+  ; ALL-DAG: ld.w [[R1:\$w[0-9]+]],
 
   %2 = add <4 x i32> %1, %1
-  ; MIPS32-AE-DAG: addv.w [[R2:\$w[0-9]+]], [[R1]], [[R1]]
+  ; ALL-DAG: addv.w [[R2:\$w[0-9]+]], [[R1]], [[R1]]
 
   %3 = extractelement <4 x i32> %2, i32 1
-  ; MIPS32-AE-DAG: copy_{{[su]}}.w [[R3:\$[0-9]+]], [[R1]][1]
+  ; ALL-DAG: copy_{{[su]}}.w [[R3:\$[0-9]+]], [[R1]][1]
 
   ret i32 %3
-  ; MIPS32-AE: .size extract_zext_v4i32
 }
 
 define i64 @extract_zext_v2i64() nounwind {
-  ; MIPS32-AE-LABEL: extract_zext_v2i64:
+  ; ALL-LABEL: extract_zext_v2i64:
 
   %1 = load <2 x i64>, <2 x i64>* @v2i64
-  ; MIPS32-AE-DAG: ld.d [[R1:\$w[0-9]+]],
+  ; ALL-DAG: ld.d [[R1:\$w[0-9]+]],
 
   %2 = add <2 x i64> %1, %1
-  ; MIPS32-AE-DAG: addv.d [[R2:\$w[0-9]+]], [[R1]], [[R1]]
+  ; ALL-DAG: addv.d [[R2:\$w[0-9]+]], [[R1]], [[R1]]
 
   %3 = extractelement <2 x i64> %2, i32 1
-  ; MIPS32-AE-DAG: copy_{{[su]}}.w [[R3:\$[0-9]+]], [[R1]][2]
-  ; MIPS32-AE-DAG: copy_{{[su]}}.w [[R4:\$[0-9]+]], [[R1]][3]
-  ; MIPS32-AE-NOT: andi
+  ; MIPS32-DAG: copy_{{[su]}}.w [[R3:\$[0-9]+]], [[R1]][2]
+  ; MIPS32-DAG: copy_{{[su]}}.w [[R4:\$[0-9]+]], [[R1]][3]
+  ; MIPS64-DAG: copy_{{[su]}}.d [[R3:\$[0-9]+]], [[R1]][1]
+  ; ALL-NOT: andi
 
   ret i64 %3
-  ; MIPS32-AE: .size extract_zext_v2i64
 }
 
 define i32 @extract_sext_v16i8_vidx() nounwind {
-  ; MIPS32-AE-LABEL: extract_sext_v16i8_vidx:
+  ; ALL-LABEL: extract_sext_v16i8_vidx:
 
   %1 = load <16 x i8>, <16 x i8>* @v16i8
-  ; MIPS32-AE-DAG: lw [[PTR_V:\$[0-9]+]], %got(v16i8)(
-  ; MIPS32-AE-DAG: ld.b [[R1:\$w[0-9]+]], 0([[PTR_V]])
+  ; O32-DAG: lw [[PTR_V:\$[0-9]+]], %got(v16i8)(
+  ; N32-DAG: lw [[PTR_V:\$[0-9]+]], %got_disp(v16i8)(
+  ; N64-DAG: ld [[PTR_V:\$[0-9]+]], %got_disp(v16i8)(
+  ; ALL-DAG: ld.b [[R1:\$w[0-9]+]], 0([[PTR_V]])
 
   %2 = add <16 x i8> %1, %1
-  ; MIPS32-AE-DAG: addv.b [[R2:\$w[0-9]+]], [[R1]], [[R1]]
+  ; ALL-DAG: addv.b [[R2:\$w[0-9]+]], [[R1]], [[R1]]
 
   %3 = load i32, i32* @i32
-  ; MIPS32-AE-DAG: lw [[PTR_I:\$[0-9]+]], %got(i32)(
-  ; MIPS32-AE-DAG: lw [[IDX:\$[0-9]+]], 0([[PTR_I]])
+  ; O32-DAG: lw [[PTR_I:\$[0-9]+]], %got(i32)(
+  ; N32-DAG: lw [[PTR_I:\$[0-9]+]], %got_disp(i32)(
+  ; N64-DAG: ld [[PTR_I:\$[0-9]+]], %got_disp(i32)(
+  ; ALL-DAG: lw [[IDX:\$[0-9]+]], 0([[PTR_I]])
 
   %4 = extractelement <16 x i8> %2, i32 %3
   %5 = sext i8 %4 to i32
-  ; MIPS32-AE-DAG: splat.b $w[[R3:[0-9]+]], [[R1]]{{\[}}[[IDX]]]
-  ; MIPS32-AE-DAG: mfc1 [[R5:\$[0-9]+]], $f[[R3]]
-  ; MIPS32-AE-DAG: sra [[R6:\$[0-9]+]], [[R5]], 24
+  ; ALL-DAG: splat.b $w[[R3:[0-9]+]], [[R1]]{{\[}}[[IDX]]]
+  ; ALL-DAG: mfc1 [[R5:\$[0-9]+]], $f[[R3]]
+  ; ALL-DAG: sra [[R6:\$[0-9]+]], [[R5]], 24
 
   ret i32 %5
-  ; MIPS32-AE: .size extract_sext_v16i8_vidx
 }
 
 define i32 @extract_sext_v8i16_vidx() nounwind {
-  ; MIPS32-AE-LABEL: extract_sext_v8i16_vidx:
+  ; ALL-LABEL: extract_sext_v8i16_vidx:
 
   %1 = load <8 x i16>, <8 x i16>* @v8i16
-  ; MIPS32-AE-DAG: lw [[PTR_V:\$[0-9]+]], %got(v8i16)(
-  ; MIPS32-AE-DAG: ld.h [[R1:\$w[0-9]+]], 0([[PTR_V]])
+  ; O32-DAG: lw [[PTR_V:\$[0-9]+]], %got(v8i16)(
+  ; N32-DAG: lw [[PTR_V:\$[0-9]+]], %got_disp(v8i16)(
+  ; N64-DAG: ld [[PTR_V:\$[0-9]+]], %got_disp(v8i16)(
+  ; ALL-DAG: ld.h [[R1:\$w[0-9]+]], 0([[PTR_V]])
 
   %2 = add <8 x i16> %1, %1
-  ; MIPS32-AE-DAG: addv.h [[R2:\$w[0-9]+]], [[R1]], [[R1]]
+  ; ALL-DAG: addv.h [[R2:\$w[0-9]+]], [[R1]], [[R1]]
 
   %3 = load i32, i32* @i32
-  ; MIPS32-AE-DAG: lw [[PTR_I:\$[0-9]+]], %got(i32)(
-  ; MIPS32-AE-DAG: lw [[IDX:\$[0-9]+]], 0([[PTR_I]])
+  ; O32-DAG: lw [[PTR_I:\$[0-9]+]], %got(i32)(
+  ; N32-DAG: lw [[PTR_I:\$[0-9]+]], %got_disp(i32)(
+  ; N64-DAG: ld [[PTR_I:\$[0-9]+]], %got_disp(i32)(
+  ; ALL-DAG: lw [[IDX:\$[0-9]+]], 0([[PTR_I]])
 
   %4 = extractelement <8 x i16> %2, i32 %3
   %5 = sext i16 %4 to i32
-  ; MIPS32-AE-DAG: splat.h $w[[R3:[0-9]+]], [[R1]]{{\[}}[[IDX]]]
-  ; MIPS32-AE-DAG: mfc1 [[R5:\$[0-9]+]], $f[[R3]]
-  ; MIPS32-AE-DAG: sra [[R6:\$[0-9]+]], [[R5]], 16
+  ; ALL-DAG: splat.h $w[[R3:[0-9]+]], [[R1]]{{\[}}[[IDX]]]
+  ; ALL-DAG: mfc1 [[R5:\$[0-9]+]], $f[[R3]]
+  ; ALL-DAG: sra [[R6:\$[0-9]+]], [[R5]], 16
 
   ret i32 %5
-  ; MIPS32-AE: .size extract_sext_v8i16_vidx
 }
 
 define i32 @extract_sext_v4i32_vidx() nounwind {
-  ; MIPS32-AE-LABEL: extract_sext_v4i32_vidx:
+  ; ALL-LABEL: extract_sext_v4i32_vidx:
 
   %1 = load <4 x i32>, <4 x i32>* @v4i32
-  ; MIPS32-AE-DAG: lw [[PTR_V:\$[0-9]+]], %got(v4i32)(
-  ; MIPS32-AE-DAG: ld.w [[R1:\$w[0-9]+]], 0([[PTR_V]])
+  ; O32-DAG: lw [[PTR_V:\$[0-9]+]], %got(v4i32)(
+  ; N32-DAG: lw [[PTR_V:\$[0-9]+]], %got_disp(v4i32)(
+  ; N64-DAG: ld [[PTR_V:\$[0-9]+]], %got_disp(v4i32)(
+  ; ALL-DAG: ld.w [[R1:\$w[0-9]+]], 0([[PTR_V]])
 
   %2 = add <4 x i32> %1, %1
-  ; MIPS32-AE-DAG: addv.w [[R2:\$w[0-9]+]], [[R1]], [[R1]]
+  ; ALL-DAG: addv.w [[R2:\$w[0-9]+]], [[R1]], [[R1]]
 
   %3 = load i32, i32* @i32
-  ; MIPS32-AE-DAG: lw [[PTR_I:\$[0-9]+]], %got(i32)(
-  ; MIPS32-AE-DAG: lw [[IDX:\$[0-9]+]], 0([[PTR_I]])
+  ; O32-DAG: lw [[PTR_I:\$[0-9]+]], %got(i32)(
+  ; N32-DAG: lw [[PTR_I:\$[0-9]+]], %got_disp(i32)(
+  ; N64-DAG: ld [[PTR_I:\$[0-9]+]], %got_disp(i32)(
+  ; ALL-DAG: lw [[IDX:\$[0-9]+]], 0([[PTR_I]])
 
   %4 = extractelement <4 x i32> %2, i32 %3
-  ; MIPS32-AE-DAG: splat.w $w[[R3:[0-9]+]], [[R1]]{{\[}}[[IDX]]]
-  ; MIPS32-AE-DAG: mfc1 [[R5:\$[0-9]+]], $f[[R3]]
-  ; MIPS32-AE-NOT: sra
+  ; ALL-DAG: splat.w $w[[R3:[0-9]+]], [[R1]]{{\[}}[[IDX]]]
+  ; ALL-DAG: mfc1 [[R5:\$[0-9]+]], $f[[R3]]
+  ; ALL-NOT: sra
 
   ret i32 %4
-  ; MIPS32-AE: .size extract_sext_v4i32_vidx
 }
 
 define i64 @extract_sext_v2i64_vidx() nounwind {
-  ; MIPS32-AE-LABEL: extract_sext_v2i64_vidx:
+  ; ALL-LABEL: extract_sext_v2i64_vidx:
 
   %1 = load <2 x i64>, <2 x i64>* @v2i64
-  ; MIPS32-AE-DAG: lw [[PTR_V:\$[0-9]+]], %got(v2i64)(
-  ; MIPS32-AE-DAG: ld.d [[R1:\$w[0-9]+]], 0([[PTR_V]])
+  ; O32-DAG: lw [[PTR_V:\$[0-9]+]], %got(v2i64)(
+  ; N32-DAG: lw [[PTR_V:\$[0-9]+]], %got_disp(v2i64)(
+  ; N64-DAG: ld [[PTR_V:\$[0-9]+]], %got_disp(v2i64)(
+  ; ALL-DAG: ld.d [[R1:\$w[0-9]+]], 0([[PTR_V]])
 
   %2 = add <2 x i64> %1, %1
-  ; MIPS32-AE-DAG: addv.d [[R2:\$w[0-9]+]], [[R1]], [[R1]]
+  ; ALL-DAG: addv.d [[R2:\$w[0-9]+]], [[R1]], [[R1]]
 
   %3 = load i32, i32* @i32
-  ; MIPS32-AE-DAG: lw [[PTR_I:\$[0-9]+]], %got(i32)(
-  ; MIPS32-AE-DAG: lw [[IDX:\$[0-9]+]], 0([[PTR_I]])
+  ; O32-DAG: lw [[PTR_I:\$[0-9]+]], %got(i32)(
+  ; N32-DAG: lw [[PTR_I:\$[0-9]+]], %got_disp(i32)(
+  ; N64-DAG: ld [[PTR_I:\$[0-9]+]], %got_disp(i32)(
+  ; ALL-DAG: lw [[IDX:\$[0-9]+]], 0([[PTR_I]])
 
   %4 = extractelement <2 x i64> %2, i32 %3
-  ; MIPS32-AE-DAG: splat.w $w[[R3:[0-9]+]], [[R1]]{{\[}}[[IDX]]]
-  ; MIPS32-AE-DAG: mfc1 [[R5:\$[0-9]+]], $f[[R3]]
-  ; MIPS32-AE-DAG: splat.w $w[[R4:[0-9]+]], [[R1]]{{\[}}[[IDX]]]
-  ; MIPS32-AE-DAG: mfc1 [[R6:\$[0-9]+]], $f[[R4]]
-  ; MIPS32-AE-NOT: sra
+  ; MIPS32-DAG: splat.w $w[[R3:[0-9]+]], [[R1]]{{\[}}[[IDX]]]
+  ; MIPS32-DAG: mfc1 [[R5:\$[0-9]+]], $f[[R3]]
+  ; MIPS32-DAG: splat.w $w[[R4:[0-9]+]], [[R1]]{{\[}}[[IDX]]]
+  ; MIPS32-DAG: mfc1 [[R6:\$[0-9]+]], $f[[R4]]
+  ; MIPS64-DAG: splat.d $w[[R3:[0-9]+]], [[R1]]{{\[}}[[IDX]]]
+  ; MIPS64-DAG: dmfc1 [[R5:\$[0-9]+]], $f[[R3]]
+  ; ALL-NOT: sra
 
   ret i64 %4
-  ; MIPS32-AE: .size extract_sext_v2i64_vidx
 }
 
 define i32 @extract_zext_v16i8_vidx() nounwind {
-  ; MIPS32-AE-LABEL: extract_zext_v16i8_vidx:
+  ; ALL-LABEL: extract_zext_v16i8_vidx:
 
   %1 = load <16 x i8>, <16 x i8>* @v16i8
-  ; MIPS32-AE-DAG: lw [[PTR_V:\$[0-9]+]], %got(v16i8)(
-  ; MIPS32-AE-DAG: ld.b [[R1:\$w[0-9]+]], 0([[PTR_V]])
+  ; O32-DAG: lw [[PTR_V:\$[0-9]+]], %got(v16i8)(
+  ; N32-DAG: lw [[PTR_V:\$[0-9]+]], %got_disp(v16i8)(
+  ; N64-DAG: ld [[PTR_V:\$[0-9]+]], %got_disp(v16i8)(
+  ; ALL-DAG: ld.b [[R1:\$w[0-9]+]], 0([[PTR_V]])
 
   %2 = add <16 x i8> %1, %1
-  ; MIPS32-AE-DAG: addv.b [[R2:\$w[0-9]+]], [[R1]], [[R1]]
+  ; ALL-DAG: addv.b [[R2:\$w[0-9]+]], [[R1]], [[R1]]
 
   %3 = load i32, i32* @i32
-  ; MIPS32-AE-DAG: lw [[PTR_I:\$[0-9]+]], %got(i32)(
-  ; MIPS32-AE-DAG: lw [[IDX:\$[0-9]+]], 0([[PTR_I]])
+  ; O32-DAG: lw [[PTR_I:\$[0-9]+]], %got(i32)(
+  ; N32-DAG: lw [[PTR_I:\$[0-9]+]], %got_disp(i32)(
+  ; N64-DAG: ld [[PTR_I:\$[0-9]+]], %got_disp(i32)(
+  ; ALL-DAG: lw [[IDX:\$[0-9]+]], 0([[PTR_I]])
 
   %4 = extractelement <16 x i8> %2, i32 %3
   %5 = zext i8 %4 to i32
-  ; MIPS32-AE-DAG: splat.b $w[[R3:[0-9]+]], [[R1]]{{\[}}[[IDX]]]
-  ; MIPS32-AE-DAG: mfc1 [[R5:\$[0-9]+]], $f[[R3]]
-  ; MIPS32-AE-DAG: srl [[R6:\$[0-9]+]], [[R5]], 24
+  ; ALL-DAG: splat.b $w[[R3:[0-9]+]], [[R1]]{{\[}}[[IDX]]]
+  ; ALL-DAG: mfc1 [[R5:\$[0-9]+]], $f[[R3]]
+  ; ALL-DAG: srl [[R6:\$[0-9]+]], [[R5]], 24
 
   ret i32 %5
-  ; MIPS32-AE: .size extract_zext_v16i8_vidx
 }
 
 define i32 @extract_zext_v8i16_vidx() nounwind {
-  ; MIPS32-AE-LABEL: extract_zext_v8i16_vidx:
+  ; ALL-LABEL: extract_zext_v8i16_vidx:
 
   %1 = load <8 x i16>, <8 x i16>* @v8i16
-  ; MIPS32-AE-DAG: lw [[PTR_V:\$[0-9]+]], %got(v8i16)(
-  ; MIPS32-AE-DAG: ld.h [[R1:\$w[0-9]+]], 0([[PTR_V]])
+  ; O32-DAG: lw [[PTR_V:\$[0-9]+]], %got(v8i16)(
+  ; N32-DAG: lw [[PTR_V:\$[0-9]+]], %got_disp(v8i16)(
+  ; N64-DAG: ld [[PTR_V:\$[0-9]+]], %got_disp(v8i16)(
+  ; ALL-DAG: ld.h [[R1:\$w[0-9]+]], 0([[PTR_V]])
 
   %2 = add <8 x i16> %1, %1
-  ; MIPS32-AE-DAG: addv.h [[R2:\$w[0-9]+]], [[R1]], [[R1]]
+  ; ALL-DAG: addv.h [[R2:\$w[0-9]+]], [[R1]], [[R1]]
 
   %3 = load i32, i32* @i32
-  ; MIPS32-AE-DAG: lw [[PTR_I:\$[0-9]+]], %got(i32)(
-  ; MIPS32-AE-DAG: lw [[IDX:\$[0-9]+]], 0([[PTR_I]])
+  ; O32-DAG: lw [[PTR_I:\$[0-9]+]], %got(i32)(
+  ; N32-DAG: lw [[PTR_I:\$[0-9]+]], %got_disp(i32)(
+  ; N64-DAG: ld [[PTR_I:\$[0-9]+]], %got_disp(i32)(
+  ; ALL-DAG: lw [[IDX:\$[0-9]+]], 0([[PTR_I]])
 
   %4 = extractelement <8 x i16> %2, i32 %3
   %5 = zext i16 %4 to i32
-  ; MIPS32-AE-DAG: splat.h $w[[R3:[0-9]+]], [[R1]]{{\[}}[[IDX]]]
-  ; MIPS32-AE-DAG: mfc1 [[R5:\$[0-9]+]], $f[[R3]]
-  ; MIPS32-AE-DAG: srl [[R6:\$[0-9]+]], [[R5]], 16
+  ; ALL-DAG: splat.h $w[[R3:[0-9]+]], [[R1]]{{\[}}[[IDX]]]
+  ; ALL-DAG: mfc1 [[R5:\$[0-9]+]], $f[[R3]]
+  ; ALL-DAG: srl [[R6:\$[0-9]+]], [[R5]], 16
 
   ret i32 %5
-  ; MIPS32-AE: .size extract_zext_v8i16_vidx
 }
 
 define i32 @extract_zext_v4i32_vidx() nounwind {
-  ; MIPS32-AE-LABEL: extract_zext_v4i32_vidx:
+  ; ALL-LABEL: extract_zext_v4i32_vidx:
 
   %1 = load <4 x i32>, <4 x i32>* @v4i32
-  ; MIPS32-AE-DAG: lw [[PTR_V:\$[0-9]+]], %got(v4i32)(
-  ; MIPS32-AE-DAG: ld.w [[R1:\$w[0-9]+]], 0([[PTR_V]])
+  ; O32-DAG: lw [[PTR_V:\$[0-9]+]], %got(v4i32)(
+  ; N32-DAG: lw [[PTR_V:\$[0-9]+]], %got_disp(v4i32)(
+  ; N64-DAG: ld [[PTR_V:\$[0-9]+]], %got_disp(v4i32)(
+  ; ALL-DAG: ld.w [[R1:\$w[0-9]+]], 0([[PTR_V]])
 
   %2 = add <4 x i32> %1, %1
-  ; MIPS32-AE-DAG: addv.w [[R2:\$w[0-9]+]], [[R1]], [[R1]]
+  ; ALL-DAG: addv.w [[R2:\$w[0-9]+]], [[R1]], [[R1]]
 
   %3 = load i32, i32* @i32
-  ; MIPS32-AE-DAG: lw [[PTR_I:\$[0-9]+]], %got(i32)(
-  ; MIPS32-AE-DAG: lw [[IDX:\$[0-9]+]], 0([[PTR_I]])
+  ; O32-DAG: lw [[PTR_I:\$[0-9]+]], %got(i32)(
+  ; N32-DAG: lw [[PTR_I:\$[0-9]+]], %got_disp(i32)(
+  ; N64-DAG: ld [[PTR_I:\$[0-9]+]], %got_disp(i32)(
+  ; ALL-DAG: lw [[IDX:\$[0-9]+]], 0([[PTR_I]])
 
   %4 = extractelement <4 x i32> %2, i32 %3
-  ; MIPS32-AE-DAG: splat.w $w[[R3:[0-9]+]], [[R1]]{{\[}}[[IDX]]]
-  ; MIPS32-AE-DAG: mfc1 [[R5:\$[0-9]+]], $f[[R3]]
-  ; MIPS32-AE-NOT: srl
+  ; ALL-DAG: splat.w $w[[R3:[0-9]+]], [[R1]]{{\[}}[[IDX]]]
+  ; ALL-DAG: mfc1 [[R5:\$[0-9]+]], $f[[R3]]
+  ; ALL-NOT: srl
 
   ret i32 %4
-  ; MIPS32-AE: .size extract_zext_v4i32_vidx
 }
 
 define i64 @extract_zext_v2i64_vidx() nounwind {
-  ; MIPS32-AE-LABEL: extract_zext_v2i64_vidx:
+  ; ALL-LABEL: extract_zext_v2i64_vidx:
 
   %1 = load <2 x i64>, <2 x i64>* @v2i64
-  ; MIPS32-AE-DAG: lw [[PTR_V:\$[0-9]+]], %got(v2i64)(
-  ; MIPS32-AE-DAG: ld.d [[R1:\$w[0-9]+]], 0([[PTR_V]])
+  ; O32-DAG: lw [[PTR_V:\$[0-9]+]], %got(v2i64)(
+  ; N32-DAG: lw [[PTR_V:\$[0-9]+]], %got_disp(v2i64)(
+  ; N64-DAG: ld [[PTR_V:\$[0-9]+]], %got_disp(v2i64)(
+  ; ALL-DAG: ld.d [[R1:\$w[0-9]+]], 0([[PTR_V]])
 
   %2 = add <2 x i64> %1, %1
-  ; MIPS32-AE-DAG: addv.d [[R2:\$w[0-9]+]], [[R1]], [[R1]]
+  ; ALL-DAG: addv.d [[R2:\$w[0-9]+]], [[R1]], [[R1]]
 
   %3 = load i32, i32* @i32
-  ; MIPS32-AE-DAG: lw [[PTR_I:\$[0-9]+]], %got(i32)(
-  ; MIPS32-AE-DAG: lw [[IDX:\$[0-9]+]], 0([[PTR_I]])
+  ; O32-DAG: lw [[PTR_I:\$[0-9]+]], %got(i32)(
+  ; N32-DAG: lw [[PTR_I:\$[0-9]+]], %got_disp(i32)(
+  ; N64-DAG: ld [[PTR_I:\$[0-9]+]], %got_disp(i32)(
+  ; ALL-DAG: lw [[IDX:\$[0-9]+]], 0([[PTR_I]])
 
   %4 = extractelement <2 x i64> %2, i32 %3
-  ; MIPS32-AE-DAG: splat.w $w[[R3:[0-9]+]], [[R1]]{{\[}}[[IDX]]]
-  ; MIPS32-AE-DAG: mfc1 [[R5:\$[0-9]+]], $f[[R3]]
-  ; MIPS32-AE-DAG: splat.w $w[[R4:[0-9]+]], [[R1]]{{\[}}[[IDX]]]
-  ; MIPS32-AE-DAG: mfc1 [[R6:\$[0-9]+]], $f[[R4]]
-  ; MIPS32-AE-NOT: srl
+  ; MIPS32-DAG: splat.w $w[[R3:[0-9]+]], [[R1]]{{\[}}[[IDX]]]
+  ; MIPS32-DAG: mfc1 [[R5:\$[0-9]+]], $f[[R3]]
+  ; MIPS32-DAG: splat.w $w[[R4:[0-9]+]], [[R1]]{{\[}}[[IDX]]]
+  ; MIPS32-DAG: mfc1 [[R6:\$[0-9]+]], $f[[R4]]
+  ; MIPS64-DAG: splat.d $w[[R3:[0-9]+]], [[R1]]{{\[}}[[IDX]]]
+  ; MIPS64-DAG: dmfc1 [[R5:\$[0-9]+]], $f[[R3]]
+  ; ALL-NOT: srl
 
   ret i64 %4
-  ; MIPS32-AE: .size extract_zext_v2i64_vidx
 }
 
-define void @insert_v16i8(i32 %a) nounwind {
-  ; MIPS32-AE-LABEL: insert_v16i8:
+define void @insert_v16i8(i32 signext %a) nounwind {
+  ; ALL-LABEL: insert_v16i8:
 
   %1 = load <16 x i8>, <16 x i8>* @v16i8
-  ; MIPS32-AE-DAG: ld.b [[R1:\$w[0-9]+]],
+  ; ALL-DAG: ld.b [[R1:\$w[0-9]+]],
 
   %a2 = trunc i32 %a to i8
   %a3 = sext i8 %a2 to i32
   %a4 = trunc i32 %a3 to i8
-  ; MIPS32-AE-NOT: andi
-  ; MIPS32-AE-NOT: sra
+  ; ALL-NOT: andi
+  ; ALL-NOT: sra
 
   %2 = insertelement <16 x i8> %1, i8 %a4, i32 1
-  ; MIPS32-AE-DAG: insert.b [[R1]][1], $4
+  ; ALL-DAG: insert.b [[R1]][1], $4
 
   store <16 x i8> %2, <16 x i8>* @v16i8
-  ; MIPS32-AE-DAG: st.b [[R1]]
+  ; ALL-DAG: st.b [[R1]]
 
   ret void
-  ; MIPS32-AE: .size insert_v16i8
 }
 
-define void @insert_v8i16(i32 %a) nounwind {
-  ; MIPS32-AE-LABEL: insert_v8i16:
+define void @insert_v8i16(i32 signext %a) nounwind {
+  ; ALL-LABEL: insert_v8i16:
 
   %1 = load <8 x i16>, <8 x i16>* @v8i16
-  ; MIPS32-AE-DAG: ld.h [[R1:\$w[0-9]+]],
+  ; ALL-DAG: ld.h [[R1:\$w[0-9]+]],
 
   %a2 = trunc i32 %a to i16
   %a3 = sext i16 %a2 to i32
   %a4 = trunc i32 %a3 to i16
-  ; MIPS32-AE-NOT: andi
-  ; MIPS32-AE-NOT: sra
+  ; ALL-NOT: andi
+  ; ALL-NOT: sra
 
   %2 = insertelement <8 x i16> %1, i16 %a4, i32 1
-  ; MIPS32-AE-DAG: insert.h [[R1]][1], $4
+  ; ALL-DAG: insert.h [[R1]][1], $4
 
   store <8 x i16> %2, <8 x i16>* @v8i16
-  ; MIPS32-AE-DAG: st.h [[R1]]
+  ; ALL-DAG: st.h [[R1]]
 
   ret void
-  ; MIPS32-AE: .size insert_v8i16
 }
 
-define void @insert_v4i32(i32 %a) nounwind {
-  ; MIPS32-AE-LABEL: insert_v4i32:
+define void @insert_v4i32(i32 signext %a) nounwind {
+  ; ALL-LABEL: insert_v4i32:
 
   %1 = load <4 x i32>, <4 x i32>* @v4i32
-  ; MIPS32-AE-DAG: ld.w [[R1:\$w[0-9]+]],
+  ; ALL-DAG: ld.w [[R1:\$w[0-9]+]],
 
-  ; MIPS32-AE-NOT: andi
-  ; MIPS32-AE-NOT: sra
+  ; ALL-NOT: andi
+  ; ALL-NOT: sra
 
   %2 = insertelement <4 x i32> %1, i32 %a, i32 1
-  ; MIPS32-AE-DAG: insert.w [[R1]][1], $4
+  ; ALL-DAG: insert.w [[R1]][1], $4
 
   store <4 x i32> %2, <4 x i32>* @v4i32
-  ; MIPS32-AE-DAG: st.w [[R1]]
+  ; ALL-DAG: st.w [[R1]]
 
   ret void
-  ; MIPS32-AE: .size insert_v4i32
 }
 
-define void @insert_v2i64(i64 %a) nounwind {
-  ; MIPS32-AE-LABEL: insert_v2i64:
+define void @insert_v2i64(i64 signext %a) nounwind {
+  ; ALL-LABEL: insert_v2i64:
 
   %1 = load <2 x i64>, <2 x i64>* @v2i64
-  ; MIPS32-AE-DAG: ld.w [[R1:\$w[0-9]+]],
+  ; MIPS32-DAG: ld.w [[R1:\$w[0-9]+]],
+  ; MIPS64-DAG: ld.d [[R1:\$w[0-9]+]],
 
-  ; MIPS32-AE-NOT: andi
-  ; MIPS32-AE-NOT: sra
+  ; ALL-NOT: andi
+  ; ALL-NOT: sra
 
   %2 = insertelement <2 x i64> %1, i64 %a, i32 1
-  ; MIPS32-AE-DAG: insert.w [[R1]][2], $4
-  ; MIPS32-AE-DAG: insert.w [[R1]][3], $5
+  ; MIPS32-DAG: insert.w [[R1]][2], $4
+  ; MIPS32-DAG: insert.w [[R1]][3], $5
+  ; MIPS64-DAG: insert.d [[R1]][1], $4
 
   store <2 x i64> %2, <2 x i64>* @v2i64
-  ; MIPS32-AE-DAG: st.w [[R1]]
+  ; MIPS32-DAG: st.w [[R1]]
+  ; MIPS64-DAG: st.d [[R1]]
 
   ret void
-  ; MIPS32-AE: .size insert_v2i64
 }
 
-define void @insert_v16i8_vidx(i32 %a) nounwind {
-  ; MIPS32-AE: insert_v16i8_vidx:
+define void @insert_v16i8_vidx(i32 signext %a) nounwind {
+  ; ALL-LABEL: insert_v16i8_vidx:
 
   %1 = load <16 x i8>, <16 x i8>* @v16i8
-  ; MIPS32-AE-DAG: ld.b [[R1:\$w[0-9]+]],
+  ; ALL-DAG: ld.b [[R1:\$w[0-9]+]],
 
   %2 = load i32, i32* @i32
-  ; MIPS32-AE-DAG: lw [[PTR_I:\$[0-9]+]], %got(i32)(
-  ; MIPS32-AE-DAG: lw [[IDX:\$[0-9]+]], 0([[PTR_I]])
+  ; O32-DAG: lw [[PTR_I:\$[0-9]+]], %got(i32)(
+  ; N32-DAG: lw [[PTR_I:\$[0-9]+]], %got_disp(i32)(
+  ; N64-DAG: ld [[PTR_I:\$[0-9]+]], %got_disp(i32)(
+  ; ALL-DAG: lw [[IDX:\$[0-9]+]], 0([[PTR_I]])
 
   %a2 = trunc i32 %a to i8
   %a3 = sext i8 %a2 to i32
   %a4 = trunc i32 %a3 to i8
-  ; MIPS32-AE-NOT: andi
-  ; MIPS32-AE-NOT: sra
+  ; ALL-NOT: andi
+  ; ALL-NOT: sra
 
   %3 = insertelement <16 x i8> %1, i8 %a4, i32 %2
-  ; MIPS32-AE-DAG: sld.b [[R1]], [[R1]]{{\[}}[[IDX]]]
-  ; MIPS32-AE-DAG: insert.b [[R1]][0], $4
-  ; MIPS32-AE-DAG: neg [[NIDX:\$[0-9]+]], [[IDX]]
-  ; MIPS32-AE-DAG: sld.b [[R1]], [[R1]]{{\[}}[[NIDX]]]
+  ; ALL-DAG: sld.b [[R1]], [[R1]]{{\[}}[[IDX]]]
+  ; ALL-DAG: insert.b [[R1]][0], $4
+  ; O32-DAG: neg [[NIDX:\$[0-9]+]], [[IDX]]
+  ; N32-DAG: neg [[NIDX:\$[0-9]+]], [[IDX]]
+  ; N64-DAG: dneg [[NIDX:\$[0-9]+]], [[IDX]]
+  ; ALL-DAG: sld.b [[R1]], [[R1]]{{\[}}[[NIDX]]]
 
   store <16 x i8> %3, <16 x i8>* @v16i8
-  ; MIPS32-AE-DAG: st.b [[R1]]
+  ; ALL-DAG: st.b [[R1]]
 
   ret void
-  ; MIPS32-AE: .size insert_v16i8_vidx
 }
 
-define void @insert_v8i16_vidx(i32 %a) nounwind {
-  ; MIPS32-AE: insert_v8i16_vidx:
+define void @insert_v8i16_vidx(i32 signext %a) nounwind {
+  ; ALL-LABEL: insert_v8i16_vidx:
 
   %1 = load <8 x i16>, <8 x i16>* @v8i16
-  ; MIPS32-AE-DAG: ld.h [[R1:\$w[0-9]+]],
+  ; ALL-DAG: ld.h [[R1:\$w[0-9]+]],
 
   %2 = load i32, i32* @i32
-  ; MIPS32-AE-DAG: lw [[PTR_I:\$[0-9]+]], %got(i32)(
-  ; MIPS32-AE-DAG: lw [[IDX:\$[0-9]+]], 0([[PTR_I]])
+  ; O32-DAG: lw [[PTR_I:\$[0-9]+]], %got(i32)(
+  ; N32-DAG: lw [[PTR_I:\$[0-9]+]], %got_disp(i32)(
+  ; N64-DAG: ld [[PTR_I:\$[0-9]+]], %got_disp(i32)(
+  ; ALL-DAG: lw [[IDX:\$[0-9]+]], 0([[PTR_I]])
 
   %a2 = trunc i32 %a to i16
   %a3 = sext i16 %a2 to i32
   %a4 = trunc i32 %a3 to i16
-  ; MIPS32-AE-NOT: andi
-  ; MIPS32-AE-NOT: sra
+  ; ALL-NOT: andi
+  ; ALL-NOT: sra
 
   %3 = insertelement <8 x i16> %1, i16 %a4, i32 %2
-  ; MIPS32-AE-DAG: sll [[BIDX:\$[0-9]+]], [[IDX]], 1
-  ; MIPS32-AE-DAG: sld.b [[R1]], [[R1]]{{\[}}[[BIDX]]]
-  ; MIPS32-AE-DAG: insert.h [[R1]][0], $4
-  ; MIPS32-AE-DAG: neg [[NIDX:\$[0-9]+]], [[BIDX]]
-  ; MIPS32-AE-DAG: sld.b [[R1]], [[R1]]{{\[}}[[NIDX]]]
+  ; ALL-DAG: sll [[BIDX:\$[0-9]+]], [[IDX]], 1
+  ; ALL-DAG: sld.b [[R1]], [[R1]]{{\[}}[[BIDX]]]
+  ; ALL-DAG: insert.h [[R1]][0], $4
+  ; O32-DAG: neg [[NIDX:\$[0-9]+]], [[BIDX]]
+  ; N32-DAG: neg [[NIDX:\$[0-9]+]], [[BIDX]]
+  ; N64-DAG: dneg [[NIDX:\$[0-9]+]], [[BIDX]]
+  ; ALL-DAG: sld.b [[R1]], [[R1]]{{\[}}[[NIDX]]]
 
   store <8 x i16> %3, <8 x i16>* @v8i16
-  ; MIPS32-AE-DAG: st.h [[R1]]
+  ; ALL-DAG: st.h [[R1]]
 
   ret void
-  ; MIPS32-AE: .size insert_v8i16_vidx
 }
 
-define void @insert_v4i32_vidx(i32 %a) nounwind {
-  ; MIPS32-AE: insert_v4i32_vidx:
+define void @insert_v4i32_vidx(i32 signext %a) nounwind {
+  ; ALL-LABEL: insert_v4i32_vidx:
 
   %1 = load <4 x i32>, <4 x i32>* @v4i32
-  ; MIPS32-AE-DAG: ld.w [[R1:\$w[0-9]+]],
+  ; ALL-DAG: ld.w [[R1:\$w[0-9]+]],
 
   %2 = load i32, i32* @i32
-  ; MIPS32-AE-DAG: lw [[PTR_I:\$[0-9]+]], %got(i32)(
-  ; MIPS32-AE-DAG: lw [[IDX:\$[0-9]+]], 0([[PTR_I]])
+  ; O32-DAG: lw [[PTR_I:\$[0-9]+]], %got(i32)(
+  ; N32-DAG: lw [[PTR_I:\$[0-9]+]], %got_disp(i32)(
+  ; N64-DAG: ld [[PTR_I:\$[0-9]+]], %got_disp(i32)(
+  ; ALL-DAG: lw [[IDX:\$[0-9]+]], 0([[PTR_I]])
 
-  ; MIPS32-AE-NOT: andi
-  ; MIPS32-AE-NOT: sra
+  ; ALL-NOT: andi
+  ; ALL-NOT: sra
 
   %3 = insertelement <4 x i32> %1, i32 %a, i32 %2
-  ; MIPS32-AE-DAG: sll [[BIDX:\$[0-9]+]], [[IDX]], 2
-  ; MIPS32-AE-DAG: sld.b [[R1]], [[R1]]{{\[}}[[BIDX]]]
-  ; MIPS32-AE-DAG: insert.w [[R1]][0], $4
-  ; MIPS32-AE-DAG: neg [[NIDX:\$[0-9]+]], [[BIDX]]
-  ; MIPS32-AE-DAG: sld.b [[R1]], [[R1]]{{\[}}[[NIDX]]]
+  ; ALL-DAG: sll [[BIDX:\$[0-9]+]], [[IDX]], 2
+  ; ALL-DAG: sld.b [[R1]], [[R1]]{{\[}}[[BIDX]]]
+  ; ALL-DAG: insert.w [[R1]][0], $4
+  ; O32-DAG: neg [[NIDX:\$[0-9]+]], [[BIDX]]
+  ; N32-DAG: neg [[NIDX:\$[0-9]+]], [[BIDX]]
+  ; N64-DAG: dneg [[NIDX:\$[0-9]+]], [[BIDX]]
+  ; ALL-DAG: sld.b [[R1]], [[R1]]{{\[}}[[NIDX]]]
 
   store <4 x i32> %3, <4 x i32>* @v4i32
-  ; MIPS32-AE-DAG: st.w [[R1]]
+  ; ALL-DAG: st.w [[R1]]
 
   ret void
-  ; MIPS32-AE: .size insert_v4i32_vidx
 }
 
-define void @insert_v2i64_vidx(i64 %a) nounwind {
-  ; MIPS32-AE: insert_v2i64_vidx:
+define void @insert_v2i64_vidx(i64 signext %a) nounwind {
+  ; ALL-LABEL: insert_v2i64_vidx:
 
   %1 = load <2 x i64>, <2 x i64>* @v2i64
-  ; MIPS32-AE-DAG: ld.w [[R1:\$w[0-9]+]],
+  ; MIPS32-DAG: ld.w [[R1:\$w[0-9]+]],
+  ; MIPS64-DAG: ld.d [[R1:\$w[0-9]+]],
 
   %2 = load i32, i32* @i32
-  ; MIPS32-AE-DAG: lw [[PTR_I:\$[0-9]+]], %got(i32)(
-  ; MIPS32-AE-DAG: lw [[IDX:\$[0-9]+]], 0([[PTR_I]])
+  ; O32-DAG: lw [[PTR_I:\$[0-9]+]], %got(i32)(
+  ; N32-DAG: lw [[PTR_I:\$[0-9]+]], %got_disp(i32)(
+  ; N64-DAG: ld [[PTR_I:\$[0-9]+]], %got_disp(i32)(
+  ; ALL-DAG: lw [[IDX:\$[0-9]+]], 0([[PTR_I]])
 
-  ; MIPS32-AE-NOT: andi
-  ; MIPS32-AE-NOT: sra
+  ; ALL-NOT: andi
+  ; ALL-NOT: sra
 
   %3 = insertelement <2 x i64> %1, i64 %a, i32 %2
   ; TODO: This code could be a lot better but it works. The legalizer splits
   ; 64-bit inserts into two 32-bit inserts because there is no i64 type on
   ; MIPS32. The obvious optimisation is to perform both insert.w's at once while
   ; the vector is rotated.
-  ; MIPS32-AE-DAG: sll [[BIDX:\$[0-9]+]], [[IDX]], 2
-  ; MIPS32-AE-DAG: sld.b [[R1]], [[R1]]{{\[}}[[BIDX]]]
-  ; MIPS32-AE-DAG: insert.w [[R1]][0], $4
-  ; MIPS32-AE-DAG: neg [[NIDX:\$[0-9]+]], [[BIDX]]
-  ; MIPS32-AE-DAG: sld.b [[R1]], [[R1]]{{\[}}[[NIDX]]]
-  ; MIPS32-AE-DAG: addiu [[IDX2:\$[0-9]+]], [[IDX]], 1
-  ; MIPS32-AE-DAG: sll [[BIDX:\$[0-9]+]], [[IDX2]], 2
-  ; MIPS32-AE-DAG: sld.b [[R1]], [[R1]]{{\[}}[[BIDX]]]
-  ; MIPS32-AE-DAG: insert.w [[R1]][0], $5
-  ; MIPS32-AE-DAG: neg [[NIDX:\$[0-9]+]], [[BIDX]]
-  ; MIPS32-AE-DAG: sld.b [[R1]], [[R1]]{{\[}}[[NIDX]]]
+  ; MIPS32-DAG: sll [[BIDX:\$[0-9]+]], [[IDX]], 2
+  ; MIPS32-DAG: sld.b [[R1]], [[R1]]{{\[}}[[BIDX]]]
+  ; MIPS32-DAG: insert.w [[R1]][0], $4
+  ; MIPS32-DAG: neg [[NIDX:\$[0-9]+]], [[BIDX]]
+  ; MIPS32-DAG: sld.b [[R1]], [[R1]]{{\[}}[[NIDX]]]
+  ; MIPS32-DAG: addiu [[IDX2:\$[0-9]+]], [[IDX]], 1
+  ; MIPS32-DAG: sll [[BIDX:\$[0-9]+]], [[IDX2]], 2
+  ; MIPS32-DAG: sld.b [[R1]], [[R1]]{{\[}}[[BIDX]]]
+  ; MIPS32-DAG: insert.w [[R1]][0], $5
+  ; MIPS32-DAG: neg [[NIDX:\$[0-9]+]], [[BIDX]]
+  ; MIPS32-DAG: sld.b [[R1]], [[R1]]{{\[}}[[NIDX]]]
+
+  ; MIPS64-DAG: sll [[BIDX:\$[0-9]+]], [[IDX]], 3
+  ; MIPS64-DAG: sld.b [[R1]], [[R1]]{{\[}}[[BIDX]]]
+  ; MIPS64-DAG: insert.d [[R1]][0], $4
+  ; N32-DAG: neg [[NIDX:\$[0-9]+]], [[BIDX]]
+  ; N64-DAG: dneg [[NIDX:\$[0-9]+]], [[BIDX]]
+  ; MIPS64-DAG: sld.b [[R1]], [[R1]]{{\[}}[[NIDX]]]
 
   store <2 x i64> %3, <2 x i64>* @v2i64
-  ; MIPS32-AE-DAG: st.w [[R1]]
+  ; MIPS32-DAG: st.w [[R1]]
+  ; MIPS64-DAG: st.d [[R1]]
 
   ret void
-  ; MIPS32-AE: .size insert_v2i64_vidx
 }
 
 define void @truncstore() nounwind {
-  ; MIPS32-AE-LABEL: truncstore:
+  ; ALL-LABEL: truncstore:
 
   store volatile <4 x i8> <i8 -1, i8 -1, i8 -1, i8 -1>, <4 x i8>*@v4i8
   ; TODO: What code should be emitted?
 
   ret void
-  ; MIPS32-AE: .size truncstore
 }
diff --git a/test/CodeGen/Mips/msa/basic_operations_float.ll b/test/CodeGen/Mips/msa/basic_operations_float.ll
index 53c1f11f3ad..f19cb9b7c2e 100644
--- a/test/CodeGen/Mips/msa/basic_operations_float.ll
+++ b/test/CodeGen/Mips/msa/basic_operations_float.ll
@@ -1,5 +1,9 @@
-; RUN: llc -march=mips -mattr=+msa,+fp64 < %s | FileCheck -check-prefix=MIPS32 %s
-; RUN: llc -march=mipsel -mattr=+msa,+fp64 < %s | FileCheck -check-prefix=MIPS32 %s
+; RUN: llc -march=mips -mattr=+msa,+fp64 < %s | FileCheck -check-prefix=ALL -check-prefix=O32 %s
+; RUN: llc -march=mipsel -mattr=+msa,+fp64 < %s | FileCheck -check-prefix=ALL -check-prefix=O32 %s
+; RUN: llc -march=mips64 -target-abi=n32 -mattr=+msa,+fp64 < %s | FileCheck -check-prefix=ALL -check-prefix=N32 %s
+; RUN: llc -march=mips64el -target-abi=n32 -mattr=+msa,+fp64 < %s | FileCheck -check-prefix=ALL -check-prefix=N32 %s
+; RUN: llc -march=mips64 -mattr=+msa,+fp64 < %s | FileCheck -check-prefix=ALL -check-prefix=N64 %s
+; RUN: llc -march=mips64el -mattr=+msa,+fp64 < %s | FileCheck -check-prefix=ALL -check-prefix=N64 %s
 
 @v4f32 = global <4 x float> <float 0.0, float 0.0, float 0.0, float 0.0>
 @v2f64 = global <2 x double> <double 0.0, double 0.0>
@@ -8,72 +12,88 @@
 @f64 = global double 0.0
 
 define void @const_v4f32() nounwind {
-  ; MIPS32-LABEL: const_v4f32:
+  ; ALL-LABEL: const_v4f32:
 
   store volatile <4 x float> <float 0.0, float 0.0, float 0.0, float 0.0>, <4 x float>*@v4f32
-  ; MIPS32: ldi.b  [[R1:\$w[0-9]+]], 0
+  ; ALL: ldi.b  [[R1:\$w[0-9]+]], 0
 
   store volatile <4 x float> <float 1.0, float 1.0, float 1.0, float 1.0>, <4 x float>*@v4f32
-  ; MIPS32: lui     [[R1:\$[0-9]+]], 16256
-  ; MIPS32: fill.w  [[R2:\$w[0-9]+]], [[R1]]
+  ; ALL: lui     [[R1:\$[0-9]+]], 16256
+  ; ALL: fill.w  [[R2:\$w[0-9]+]], [[R1]]
 
   store volatile <4 x float> <float 1.0, float 1.0, float 1.0, float 31.0>, <4 x float>*@v4f32
-  ; MIPS32: addiu [[G_PTR:\$[0-9]+]], {{.*}}, %lo($
-  ; MIPS32: ld.w  [[R1:\$w[0-9]+]], 0([[G_PTR]])
+  ; O32: addiu [[G_PTR:\$[0-9]+]], {{.*}}, %lo($
+  ; N32: addiu [[G_PTR:\$[0-9]+]], {{.*}}, %got_ofst($
+  ; N64: daddiu [[G_PTR:\$[0-9]+]], {{.*}}, %got_ofst($
+  ; ALL: ld.w  [[R1:\$w[0-9]+]], 0([[G_PTR]])
 
   store volatile <4 x float> <float 65537.0, float 65537.0, float 65537.0, float 65537.0>, <4 x float>*@v4f32
-  ; MIPS32: lui     [[R1:\$[0-9]+]], 18304
-  ; MIPS32: ori     [[R2:\$[0-9]+]], [[R1]], 128
-  ; MIPS32: fill.w  [[R3:\$w[0-9]+]], [[R2]]
+  ; ALL: lui     [[R1:\$[0-9]+]], 18304
+  ; ALL: ori     [[R2:\$[0-9]+]], [[R1]], 128
+  ; ALL: fill.w  [[R3:\$w[0-9]+]], [[R2]]
 
   store volatile <4 x float> <float 1.0, float 2.0, float 1.0, float 2.0>, <4 x float>*@v4f32
-  ; MIPS32: addiu [[G_PTR:\$[0-9]+]], {{.*}}, %lo($
-  ; MIPS32: ld.w  [[R1:\$w[0-9]+]], 0([[G_PTR]])
+  ; O32: addiu [[G_PTR:\$[0-9]+]], {{.*}}, %lo($
+  ; N32: addiu [[G_PTR:\$[0-9]+]], {{.*}}, %got_ofst($
+  ; N64: daddiu [[G_PTR:\$[0-9]+]], {{.*}}, %got_ofst($
+  ; ALL: ld.w  [[R1:\$w[0-9]+]], 0([[G_PTR]])
 
   store volatile <4 x float> <float 3.0, float 4.0, float 5.0, float 6.0>, <4 x float>*@v4f32
-  ; MIPS32: addiu [[G_PTR:\$[0-9]+]], {{.*}}, %lo($
-  ; MIPS32: ld.w  [[R1:\$w[0-9]+]], 0([[G_PTR]])
+  ; O32: addiu [[G_PTR:\$[0-9]+]], {{.*}}, %lo($
+  ; N32: addiu [[G_PTR:\$[0-9]+]], {{.*}}, %got_ofst($
+  ; N64: daddiu [[G_PTR:\$[0-9]+]], {{.*}}, %got_ofst($
+  ; ALL: ld.w  [[R1:\$w[0-9]+]], 0([[G_PTR]])
 
   ret void
-  ; MIPS32: .size const_v4f32
 }
 
 define void @const_v2f64() nounwind {
-  ; MIPS32-LABEL: const_v2f64:
+  ; ALL-LABEL: const_v2f64:
 
   store volatile <2 x double> <double 0.0, double 0.0>, <2 x double>*@v2f64
-  ; MIPS32: ldi.b  [[R1:\$w[0-9]+]], 0
+  ; ALL: ldi.b  [[R1:\$w[0-9]+]], 0
 
   store volatile <2 x double> <double 72340172838076673.0, double 72340172838076673.0>, <2 x double>*@v2f64
-  ; MIPS32: addiu [[G_PTR:\$[0-9]+]], {{.*}}, %lo($
-  ; MIPS32: ld.d  [[R1:\$w[0-9]+]], 0([[G_PTR]])
+  ; O32: addiu [[G_PTR:\$[0-9]+]], {{.*}}, %lo($
+  ; N32: addiu [[G_PTR:\$[0-9]+]], {{.*}}, %got_ofst($
+  ; N64: daddiu [[G_PTR:\$[0-9]+]], {{.*}}, %got_ofst($
+  ; ALL: ld.d  [[R1:\$w[0-9]+]], 0([[G_PTR]])
 
   store volatile <2 x double> <double 281479271743489.0, double 281479271743489.0>, <2 x double>*@v2f64
-  ; MIPS32: addiu [[G_PTR:\$[0-9]+]], {{.*}}, %lo($
-  ; MIPS32: ld.d  [[R1:\$w[0-9]+]], 0([[G_PTR]])
+  ; O32: addiu [[G_PTR:\$[0-9]+]], {{.*}}, %lo($
+  ; N32: addiu [[G_PTR:\$[0-9]+]], {{.*}}, %got_ofst($
+  ; N64: daddiu [[G_PTR:\$[0-9]+]], {{.*}}, %got_ofst($
+  ; ALL: ld.d  [[R1:\$w[0-9]+]], 0([[G_PTR]])
 
   store volatile <2 x double> <double 4294967297.0, double 4294967297.0>, <2 x double>*@v2f64
-  ; MIPS32: addiu [[G_PTR:\$[0-9]+]], {{.*}}, %lo($
-  ; MIPS32: ld.d  [[R1:\$w[0-9]+]], 0([[G_PTR]])
+  ; O32: addiu [[G_PTR:\$[0-9]+]], {{.*}}, %lo($
+  ; N32: addiu [[G_PTR:\$[0-9]+]], {{.*}}, %got_ofst($
+  ; N64: daddiu [[G_PTR:\$[0-9]+]], {{.*}}, %got_ofst($
+  ; ALL: ld.d  [[R1:\$w[0-9]+]], 0([[G_PTR]])
 
   store volatile <2 x double> <double 1.0, double 1.0>, <2 x double>*@v2f64
-  ; MIPS32: addiu [[G_PTR:\$[0-9]+]], {{.*}}, %lo($
-  ; MIPS32: ld.d  [[R1:\$w[0-9]+]], 0([[G_PTR]])
+  ; O32: addiu [[G_PTR:\$[0-9]+]], {{.*}}, %lo($
+  ; N32: addiu [[G_PTR:\$[0-9]+]], {{.*}}, %got_ofst($
+  ; N64: daddiu [[G_PTR:\$[0-9]+]], {{.*}}, %got_ofst($
+  ; ALL: ld.d  [[R1:\$w[0-9]+]], 0([[G_PTR]])
 
   store volatile <2 x double> <double 1.0, double 31.0>, <2 x double>*@v2f64
-  ; MIPS32: addiu [[G_PTR:\$[0-9]+]], {{.*}}, %lo($
-  ; MIPS32: ld.d  [[R1:\$w[0-9]+]], 0([[G_PTR]])
+  ; O32: addiu [[G_PTR:\$[0-9]+]], {{.*}}, %lo($
+  ; N32: addiu [[G_PTR:\$[0-9]+]], {{.*}}, %got_ofst($
+  ; N64: daddiu [[G_PTR:\$[0-9]+]], {{.*}}, %got_ofst($
+  ; ALL: ld.d  [[R1:\$w[0-9]+]], 0([[G_PTR]])
 
   store volatile <2 x double> <double 3.0, double 4.0>, <2 x double>*@v2f64
-  ; MIPS32: addiu [[G_PTR:\$[0-9]+]], {{.*}}, %lo($
-  ; MIPS32: ld.d  [[R1:\$w[0-9]+]], 0([[G_PTR]])
+  ; O32: addiu [[G_PTR:\$[0-9]+]], {{.*}}, %lo($
+  ; N32: addiu [[G_PTR:\$[0-9]+]], {{.*}}, %got_ofst($
+  ; N64: daddiu [[G_PTR:\$[0-9]+]], {{.*}}, %got_ofst($
+  ; ALL: ld.d  [[R1:\$w[0-9]+]], 0([[G_PTR]])
 
   ret void
-  ; MIPS32: .size const_v2f64
 }
 
 define void @nonconst_v4f32() nounwind {
-  ; MIPS32-LABEL: nonconst_v4f32:
+  ; ALL-LABEL: nonconst_v4f32:
 
   %1 = load float , float *@f32
   %2 = insertelement <4 x float> undef, float %1, i32 0
@@ -81,249 +101,252 @@ define void @nonconst_v4f32() nounwind {
   %4 = insertelement <4 x float> %3, float %1, i32 2
   %5 = insertelement <4 x float> %4, float %1, i32 3
   store volatile <4 x float> %5, <4 x float>*@v4f32
-  ; MIPS32: lwc1 $f[[R1:[0-9]+]], 0(
-  ; MIPS32: splati.w [[R2:\$w[0-9]+]], $w[[R1]]
+  ; ALL: lwc1 $f[[R1:[0-9]+]], 0(
+  ; ALL: splati.w [[R2:\$w[0-9]+]], $w[[R1]]
 
   ret void
-  ; MIPS32: .size nonconst_v4f32
 }
 
 define void @nonconst_v2f64() nounwind {
-  ; MIPS32-LABEL: nonconst_v2f64:
+  ; ALL-LABEL: nonconst_v2f64:
 
   %1 = load double , double *@f64
   %2 = insertelement <2 x double> undef, double %1, i32 0
   %3 = insertelement <2 x double> %2, double %1, i32 1
   store volatile <2 x double> %3, <2 x double>*@v2f64
-  ; MIPS32: ldc1 $f[[R1:[0-9]+]], 0(
-  ; MIPS32: splati.d [[R2:\$w[0-9]+]], $w[[R1]]
+  ; ALL: ldc1 $f[[R1:[0-9]+]], 0(
+  ; ALL: splati.d [[R2:\$w[0-9]+]], $w[[R1]]
 
   ret void
-  ; MIPS32: .size nonconst_v2f64
 }
 
 define float @extract_v4f32() nounwind {
-  ; MIPS32-LABEL: extract_v4f32:
+  ; ALL-LABEL: extract_v4f32:
 
   %1 = load <4 x float>, <4 x float>* @v4f32
-  ; MIPS32-DAG: ld.w [[R1:\$w[0-9]+]],
+  ; ALL-DAG: ld.w [[R1:\$w[0-9]+]],
 
   %2 = fadd <4 x float> %1, %1
-  ; MIPS32-DAG: fadd.w [[R2:\$w[0-9]+]], [[R1]], [[R1]]
+  ; ALL-DAG: fadd.w [[R2:\$w[0-9]+]], [[R1]], [[R1]]
 
   %3 = extractelement <4 x float> %2, i32 1
   ; Element 1 can be obtained by splatting it across the vector and extracting
   ; $w0:sub_lo
-  ; MIPS32-DAG: splati.w $w0, [[R1]][1]
+  ; ALL-DAG: splati.w $w0, [[R1]][1]
 
   ret float %3
-  ; MIPS32: .size extract_v4f32
 }
 
 define float @extract_v4f32_elt0() nounwind {
-  ; MIPS32-LABEL: extract_v4f32_elt0:
+  ; ALL-LABEL: extract_v4f32_elt0:
 
   %1 = load <4 x float>, <4 x float>* @v4f32
-  ; MIPS32-DAG: ld.w [[R1:\$w[0-9]+]],
+  ; ALL-DAG: ld.w [[R1:\$w[0-9]+]],
 
   %2 = fadd <4 x float> %1, %1
-  ; MIPS32-DAG: fadd.w $w0, [[R1]], [[R1]]
+  ; ALL-DAG: fadd.w $w0, [[R1]], [[R1]]
 
   %3 = extractelement <4 x float> %2, i32 0
   ; Element 0 can be obtained by extracting $w0:sub_lo ($f0)
-  ; MIPS32-NOT: copy_u.w
-  ; MIPS32-NOT: mtc1
+  ; ALL-NOT: copy_u.w
+  ; ALL-NOT: mtc1
 
   ret float %3
-  ; MIPS32: .size extract_v4f32_elt0
 }
 
 define float @extract_v4f32_elt2() nounwind {
-  ; MIPS32-LABEL: extract_v4f32_elt2:
+  ; ALL-LABEL: extract_v4f32_elt2:
 
   %1 = load <4 x float>, <4 x float>* @v4f32
-  ; MIPS32-DAG: ld.w [[R1:\$w[0-9]+]],
+  ; ALL-DAG: ld.w [[R1:\$w[0-9]+]],
 
   %2 = fadd <4 x float> %1, %1
-  ; MIPS32-DAG: fadd.w [[R2:\$w[0-9]+]], [[R1]], [[R1]]
+  ; ALL-DAG: fadd.w [[R2:\$w[0-9]+]], [[R1]], [[R1]]
 
   %3 = extractelement <4 x float> %2, i32 2
   ; Element 2 can be obtained by splatting it across the vector and extracting
   ; $w0:sub_lo
-  ; MIPS32-DAG: splati.w $w0, [[R1]][2]
+  ; ALL-DAG: splati.w $w0, [[R1]][2]
 
   ret float %3
-  ; MIPS32: .size extract_v4f32_elt2
 }
 
 define float @extract_v4f32_vidx() nounwind {
-  ; MIPS32-LABEL: extract_v4f32_vidx:
+  ; ALL-LABEL: extract_v4f32_vidx:
 
   %1 = load <4 x float>, <4 x float>* @v4f32
-  ; MIPS32-DAG: lw [[PTR_V:\$[0-9]+]], %got(v4f32)(
-  ; MIPS32-DAG: ld.w [[R1:\$w[0-9]+]], 0([[PTR_V]])
+  ; O32-DAG: lw [[PTR_V:\$[0-9]+]], %got(v4f32)(
+  ; N32-DAG: lw [[PTR_V:\$[0-9]+]], %got_disp(v4f32)(
+  ; N64-DAG: ld [[PTR_V:\$[0-9]+]], %got_disp(v4f32)(
+  ; ALL-DAG: ld.w [[R1:\$w[0-9]+]], 0([[PTR_V]])
 
   %2 = fadd <4 x float> %1, %1
-  ; MIPS32-DAG: fadd.w [[R2:\$w[0-9]+]], [[R1]], [[R1]]
+  ; ALL-DAG: fadd.w [[R2:\$w[0-9]+]], [[R1]], [[R1]]
 
   %3 = load i32, i32* @i32
-  ; MIPS32-DAG: lw [[PTR_I:\$[0-9]+]], %got(i32)(
-  ; MIPS32-DAG: lw [[IDX:\$[0-9]+]], 0([[PTR_I]])
+  ; O32-DAG: lw [[PTR_I:\$[0-9]+]], %got(i32)(
+  ; N32-DAG: lw [[PTR_I:\$[0-9]+]], %got_disp(i32)(
+  ; N64-DAG: ld [[PTR_I:\$[0-9]+]], %got_disp(i32)(
+  ; ALL-DAG: lw [[IDX:\$[0-9]+]], 0([[PTR_I]])
 
   %4 = extractelement <4 x float> %2, i32 %3
-  ; MIPS32-DAG: splat.w $w0, [[R1]]{{\[}}[[IDX]]]
+  ; ALL-DAG: splat.w $w0, [[R1]]{{\[}}[[IDX]]]
 
   ret float %4
-  ; MIPS32: .size extract_v4f32_vidx
 }
 
 define double @extract_v2f64() nounwind {
-  ; MIPS32-LABEL: extract_v2f64:
+  ; ALL-LABEL: extract_v2f64:
 
   %1 = load <2 x double>, <2 x double>* @v2f64
-  ; MIPS32-DAG: ld.d [[R1:\$w[0-9]+]],
+  ; ALL-DAG: ld.d [[R1:\$w[0-9]+]],
 
   %2 = fadd <2 x double> %1, %1
-  ; MIPS32-DAG: fadd.d [[R2:\$w[0-9]+]], [[R1]], [[R1]]
+  ; ALL-DAG: fadd.d [[R2:\$w[0-9]+]], [[R1]], [[R1]]
 
   %3 = extractelement <2 x double> %2, i32 1
   ; Element 1 can be obtained by splatting it across the vector and extracting
   ; $w0:sub_64
-  ; MIPS32-DAG: splati.d $w0, [[R1]][1]
-  ; MIPS32-NOT: copy_u.w
-  ; MIPS32-NOT: mtc1
-  ; MIPS32-NOT: mthc1
-  ; MIPS32-NOT: sll
-  ; MIPS32-NOT: sra
+  ; ALL-DAG: splati.d $w0, [[R1]][1]
+  ; ALL-NOT: copy_u.w
+  ; ALL-NOT: mtc1
+  ; ALL-NOT: mthc1
+  ; ALL-NOT: sll
+  ; ALL-NOT: sra
 
   ret double %3
-  ; MIPS32: .size extract_v2f64
 }
 
 define double @extract_v2f64_elt0() nounwind {
-  ; MIPS32-LABEL: extract_v2f64_elt0:
+  ; ALL-LABEL: extract_v2f64_elt0:
 
   %1 = load <2 x double>, <2 x double>* @v2f64
-  ; MIPS32-DAG: ld.d [[R1:\$w[0-9]+]],
+  ; ALL-DAG: ld.d [[R1:\$w[0-9]+]],
 
   %2 = fadd <2 x double> %1, %1
-  ; MIPS32-DAG: fadd.d $w0, [[R1]], [[R1]]
+  ; ALL-DAG: fadd.d $w0, [[R1]], [[R1]]
 
   %3 = extractelement <2 x double> %2, i32 0
   ; Element 0 can be obtained by extracting $w0:sub_64 ($f0)
-  ; MIPS32-NOT: copy_u.w
-  ; MIPS32-NOT: mtc1
-  ; MIPS32-NOT: mthc1
-  ; MIPS32-NOT: sll
-  ; MIPS32-NOT: sra
+  ; ALL-NOT: copy_u.w
+  ; ALL-NOT: mtc1
+  ; ALL-NOT: mthc1
+  ; ALL-NOT: sll
+  ; ALL-NOT: sra
 
   ret double %3
-  ; MIPS32: .size extract_v2f64_elt0
 }
 
 define double @extract_v2f64_vidx() nounwind {
-  ; MIPS32-LABEL: extract_v2f64_vidx:
+  ; ALL-LABEL: extract_v2f64_vidx:
 
   %1 = load <2 x double>, <2 x double>* @v2f64
-  ; MIPS32-DAG: lw [[PTR_V:\$[0-9]+]], %got(v2f64)(
-  ; MIPS32-DAG: ld.d [[R1:\$w[0-9]+]], 0([[PTR_V]])
+  ; O32-DAG: lw [[PTR_V:\$[0-9]+]], %got(v2f64)(
+  ; N32-DAG: lw [[PTR_V:\$[0-9]+]], %got_disp(v2f64)(
+  ; N64-DAG: ld [[PTR_V:\$[0-9]+]], %got_disp(v2f64)(
+  ; ALL-DAG: ld.d [[R1:\$w[0-9]+]], 0([[PTR_V]])
 
   %2 = fadd <2 x double> %1, %1
-  ; MIPS32-DAG: fadd.d [[R2:\$w[0-9]+]], [[R1]], [[R1]]
+  ; ALL-DAG: fadd.d [[R2:\$w[0-9]+]], [[R1]], [[R1]]
 
   %3 = load i32, i32* @i32
-  ; MIPS32-DAG: lw [[PTR_I:\$[0-9]+]], %got(i32)(
-  ; MIPS32-DAG: lw [[IDX:\$[0-9]+]], 0([[PTR_I]])
+  ; O32-DAG: lw [[PTR_I:\$[0-9]+]], %got(i32)(
+  ; N32-DAG: lw [[PTR_I:\$[0-9]+]], %got_disp(i32)(
+  ; N64-DAG: ld [[PTR_I:\$[0-9]+]], %got_disp(i32)(
+  ; ALL-DAG: lw [[IDX:\$[0-9]+]], 0([[PTR_I]])
 
   %4 = extractelement <2 x double> %2, i32 %3
-  ; MIPS32-DAG: splat.d $w0, [[R1]]{{\[}}[[IDX]]]
+  ; ALL-DAG: splat.d $w0, [[R1]]{{\[}}[[IDX]]]
 
   ret double %4
-  ; MIPS32: .size extract_v2f64_vidx
 }
 
 define void @insert_v4f32(float %a) nounwind {
-  ; MIPS32-LABEL: insert_v4f32:
+  ; ALL-LABEL: insert_v4f32:
 
   %1 = load <4 x float>, <4 x float>* @v4f32
-  ; MIPS32-DAG: ld.w [[R1:\$w[0-9]+]],
+  ; ALL-DAG: ld.w [[R1:\$w[0-9]+]],
 
   %2 = insertelement <4 x float> %1, float %a, i32 1
   ; float argument passed in $f12
-  ; MIPS32-DAG: insve.w [[R1]][1], $w12[0]
+  ; ALL-DAG: insve.w [[R1]][1], $w12[0]
 
   store <4 x float> %2, <4 x float>* @v4f32
-  ; MIPS32-DAG: st.w [[R1]]
+  ; ALL-DAG: st.w [[R1]]
 
   ret void
-  ; MIPS32: .size insert_v4f32
 }
 
 define void @insert_v2f64(double %a) nounwind {
-  ; MIPS32-LABEL: insert_v2f64:
+  ; ALL-LABEL: insert_v2f64:
 
   %1 = load <2 x double>, <2 x double>* @v2f64
-  ; MIPS32-DAG: ld.d [[R1:\$w[0-9]+]],
+  ; ALL-DAG: ld.d [[R1:\$w[0-9]+]],
 
   %2 = insertelement <2 x double> %1, double %a, i32 1
   ; double argument passed in $f12
-  ; MIPS32-DAG: insve.d [[R1]][1], $w12[0]
+  ; ALL-DAG: insve.d [[R1]][1], $w12[0]
 
   store <2 x double> %2, <2 x double>* @v2f64
-  ; MIPS32-DAG: st.d [[R1]]
+  ; ALL-DAG: st.d [[R1]]
 
   ret void
-  ; MIPS32: .size insert_v2f64
 }
 
 define void @insert_v4f32_vidx(float %a) nounwind {
-  ; MIPS32-LABEL: insert_v4f32_vidx:
+  ; ALL-LABEL: insert_v4f32_vidx:
 
   %1 = load <4 x float>, <4 x float>* @v4f32
-  ; MIPS32-DAG: lw [[PTR_V:\$[0-9]+]], %got(v4f32)(
-  ; MIPS32-DAG: ld.w [[R1:\$w[0-9]+]], 0([[PTR_V]])
+  ; O32-DAG: lw [[PTR_V:\$[0-9]+]], %got(v4f32)(
+  ; N32-DAG: lw [[PTR_V:\$[0-9]+]], %got_disp(v4f32)(
+  ; N64-DAG: ld [[PTR_V:\$[0-9]+]], %got_disp(v4f32)(
+  ; ALL-DAG: ld.w [[R1:\$w[0-9]+]], 0([[PTR_V]])
 
   %2 = load i32, i32* @i32
-  ; MIPS32-DAG: lw [[PTR_I:\$[0-9]+]], %got(i32)(
-  ; MIPS32-DAG: lw [[IDX:\$[0-9]+]], 0([[PTR_I]])
+  ; O32-DAG: lw [[PTR_I:\$[0-9]+]], %got(i32)(
+  ; N32-DAG: lw [[PTR_I:\$[0-9]+]], %got_disp(i32)(
+  ; N64-DAG: ld [[PTR_I:\$[0-9]+]], %got_disp(i32)(
+  ; ALL-DAG: lw [[IDX:\$[0-9]+]], 0([[PTR_I]])
 
   %3 = insertelement <4 x float> %1, float %a, i32 %2
   ; float argument passed in $f12
-  ; MIPS32-DAG: sll [[BIDX:\$[0-9]+]], [[IDX]], 2
-  ; MIPS32-DAG: sld.b [[R1]], [[R1]]{{\[}}[[BIDX]]]
-  ; MIPS32-DAG: insve.w [[R1]][0], $w12[0]
-  ; MIPS32-DAG: neg [[NIDX:\$[0-9]+]], [[BIDX]]
-  ; MIPS32-DAG: sld.b [[R1]], [[R1]]{{\[}}[[NIDX]]]
+  ; ALL-DAG: sll [[BIDX:\$[0-9]+]], [[IDX]], 2
+  ; ALL-DAG: sld.b [[R1]], [[R1]]{{\[}}[[BIDX]]]
+  ; ALL-DAG: insve.w [[R1]][0], $w12[0]
+  ; ALL-DAG: neg [[NIDX:\$[0-9]+]], [[BIDX]]
+  ; ALL-DAG: sld.b [[R1]], [[R1]]{{\[}}[[NIDX]]]
 
   store <4 x float> %3, <4 x float>* @v4f32
-  ; MIPS32-DAG: st.w [[R1]]
+  ; ALL-DAG: st.w [[R1]]
 
   ret void
-  ; MIPS32: .size insert_v4f32_vidx
 }
 
 define void @insert_v2f64_vidx(double %a) nounwind {
-  ; MIPS32-LABEL: insert_v2f64_vidx:
+  ; ALL-LABEL: insert_v2f64_vidx:
 
   %1 = load <2 x double>, <2 x double>* @v2f64
-  ; MIPS32-DAG: lw [[PTR_V:\$[0-9]+]], %got(v2f64)(
-  ; MIPS32-DAG: ld.d [[R1:\$w[0-9]+]], 0([[PTR_V]])
+  ; O32-DAG: lw [[PTR_V:\$[0-9]+]], %got(v2f64)(
+  ; N32-DAG: lw [[PTR_V:\$[0-9]+]], %got_disp(v2f64)(
+  ; N64-DAG: ld [[PTR_V:\$[0-9]+]], %got_disp(v2f64)(
+  ; ALL-DAG: ld.d [[R1:\$w[0-9]+]], 0([[PTR_V]])
 
   %2 = load i32, i32* @i32
-  ; MIPS32-DAG: lw [[PTR_I:\$[0-9]+]], %got(i32)(
-  ; MIPS32-DAG: lw [[IDX:\$[0-9]+]], 0([[PTR_I]])
+  ; O32-DAG: lw [[PTR_I:\$[0-9]+]], %got(i32)(
+  ; N32-DAG: lw [[PTR_I:\$[0-9]+]], %got_disp(i32)(
+  ; N64-DAG: ld [[PTR_I:\$[0-9]+]], %got_disp(i32)(
+  ; ALL-DAG: lw [[IDX:\$[0-9]+]], 0([[PTR_I]])
 
   %3 = insertelement <2 x double> %1, double %a, i32 %2
   ; double argument passed in $f12
-  ; MIPS32-DAG: sll [[BIDX:\$[0-9]+]], [[IDX]], 3
-  ; MIPS32-DAG: sld.b [[R1]], [[R1]]{{\[}}[[BIDX]]]
-  ; MIPS32-DAG: insve.d [[R1]][0], $w12[0]
-  ; MIPS32-DAG: neg [[NIDX:\$[0-9]+]], [[BIDX]]
-  ; MIPS32-DAG: sld.b [[R1]], [[R1]]{{\[}}[[NIDX]]]
+  ; ALL-DAG: sll [[BIDX:\$[0-9]+]], [[IDX]], 3
+  ; ALL-DAG: sld.b [[R1]], [[R1]]{{\[}}[[BIDX]]]
+  ; ALL-DAG: insve.d [[R1]][0], $w12[0]
+  ; ALL-DAG: neg [[NIDX:\$[0-9]+]], [[BIDX]]
+  ; ALL-DAG: sld.b [[R1]], [[R1]]{{\[}}[[NIDX]]]
 
   store <2 x double> %3, <2 x double>* @v2f64
-  ; MIPS32-DAG: st.d [[R1]]
+  ; ALL-DAG: st.d [[R1]]
 
   ret void
-  ; MIPS32: .size insert_v2f64_vidx
 }
diff --git a/test/CodeGen/Mips/msa/shuffle.ll b/test/CodeGen/Mips/msa/shuffle.ll
index 7feed927026..aadff7d5a5c 100644
--- a/test/CodeGen/Mips/msa/shuffle.ll
+++ b/test/CodeGen/Mips/msa/shuffle.ll
@@ -2,7 +2,7 @@
 ; RUN: llc -march=mipsel -mattr=+msa,+fp64 < %s | FileCheck %s
 
 define void @vshf_v16i8_0(<16 x i8>* %c, <16 x i8>* %a, <16 x i8>* %b) nounwind {
-  ; CHECK: vshf_v16i8_0:
+  ; CHECK-LABEL: vshf_v16i8_0:
 
   %1 = load <16 x i8>, <16 x i8>* %a
   ; CHECK-DAG: ld.b [[R1:\$w[0-9]+]], 0($5)
@@ -14,11 +14,10 @@ define void @vshf_v16i8_0(<16 x i8>* %c, <16 x i8>* %a, <16 x i8>* %b) nounwind
   ; CHECK-DAG: st.b [[R3]], 0($4)
 
   ret void
-  ; CHECK: .size vshf_v16i8_0
 }
 
 define void @vshf_v16i8_1(<16 x i8>* %c, <16 x i8>* %a, <16 x i8>* %b) nounwind {
-  ; CHECK: vshf_v16i8_1:
+  ; CHECK-LABEL: vshf_v16i8_1:
 
   %1 = load <16 x i8>, <16 x i8>* %a
   ; CHECK-DAG: ld.b [[R1:\$w[0-9]+]], 0($5)
@@ -28,11 +27,10 @@ define void @vshf_v16i8_1(<16 x i8>* %c, <16 x i8>* %a, <16 x i8>* %b) nounwind
   ; CHECK-DAG: st.b [[R3]], 0($4)
 
   ret void
-  ; CHECK: .size vshf_v16i8_1
 }
 
 define void @vshf_v16i8_2(<16 x i8>* %c, <16 x i8>* %a, <16 x i8>* %b) nounwind {
-  ; CHECK: vshf_v16i8_2:
+  ; CHECK-LABEL: vshf_v16i8_2:
 
   %1 = load <16 x i8>, <16 x i8>* %a
   %2 = load <16 x i8>, <16 x i8>* %b
@@ -45,11 +43,10 @@ define void @vshf_v16i8_2(<16 x i8>* %c, <16 x i8>* %a, <16 x i8>* %b) nounwind
   ; CHECK-DAG: st.b [[R3]], 0($4)
 
   ret void
-  ; CHECK: .size vshf_v16i8_2
 }
 
 define void @vshf_v16i8_3(<16 x i8>* %c, <16 x i8>* %a, <16 x i8>* %b) nounwind {
-  ; CHECK: vshf_v16i8_3:
+  ; CHECK-LABEL: vshf_v16i8_3:
 
   %1 = load <16 x i8>, <16 x i8>* %a
   ; CHECK-DAG: ld.b [[R1:\$w[0-9]+]], 0($5)
@@ -65,11 +62,10 @@ define void @vshf_v16i8_3(<16 x i8>* %c, <16 x i8>* %a, <16 x i8>* %b) nounwind
   ; CHECK-DAG: st.b [[R3]], 0($4)
 
   ret void
-  ; CHECK: .size vshf_v16i8_3
 }
 
 define void @vshf_v16i8_4(<16 x i8>* %c, <16 x i8>* %a, <16 x i8>* %b) nounwind {
-  ; CHECK: vshf_v16i8_4:
+  ; CHECK-LABEL: vshf_v16i8_4:
 
   %1 = load <16 x i8>, <16 x i8>* %a
   ; CHECK-DAG: ld.b [[R1:\$w[0-9]+]], 0($5)
@@ -79,11 +75,10 @@ define void @vshf_v16i8_4(<16 x i8>* %c, <16 x i8>* %a, <16 x i8>* %b) nounwind
   ; CHECK-DAG: st.b [[R3]], 0($4)
 
   ret void
-  ; CHECK: .size vshf_v16i8_4
 }
 
 define void @vshf_v8i16_0(<8 x i16>* %c, <8 x i16>* %a, <8 x i16>* %b) nounwind {
-  ; CHECK: vshf_v8i16_0:
+  ; CHECK-LABEL: vshf_v8i16_0:
 
   %1 = load <8 x i16>, <8 x i16>* %a
   ; CHECK-DAG: ld.h [[R1:\$w[0-9]+]], 0($5)
@@ -95,11 +90,10 @@ define void @vshf_v8i16_0(<8 x i16>* %c, <8 x i16>* %a, <8 x i16>* %b) nounwind
   ; CHECK-DAG: st.h [[R3]], 0($4)
 
   ret void
-  ; CHECK: .size vshf_v8i16_0
 }
 
 define void @vshf_v8i16_1(<8 x i16>* %c, <8 x i16>* %a, <8 x i16>* %b) nounwind {
-  ; CHECK: vshf_v8i16_1:
+  ; CHECK-LABEL: vshf_v8i16_1:
 
   %1 = load <8 x i16>, <8 x i16>* %a
   ; CHECK-DAG: ld.h [[R1:\$w[0-9]+]], 0($5)
@@ -109,11 +103,10 @@ define void @vshf_v8i16_1(<8 x i16>* %c, <8 x i16>* %a, <8 x i16>* %b) nounwind
   ; CHECK-DAG: st.h [[R3]], 0($4)
 
   ret void
-  ; CHECK: .size vshf_v8i16_1
 }
 
 define void @vshf_v8i16_2(<8 x i16>* %c, <8 x i16>* %a, <8 x i16>* %b) nounwind {
-  ; CHECK: vshf_v8i16_2:
+  ; CHECK-LABEL: vshf_v8i16_2:
 
   %1 = load <8 x i16>, <8 x i16>* %a
   %2 = load <8 x i16>, <8 x i16>* %b
@@ -126,11 +119,10 @@ define void @vshf_v8i16_2(<8 x i16>* %c, <8 x i16>* %a, <8 x i16>* %b) nounwind
   ; CHECK-DAG: st.h [[R3]], 0($4)
 
   ret void
-  ; CHECK: .size vshf_v8i16_2
 }
 
 define void @vshf_v8i16_3(<8 x i16>* %c, <8 x i16>* %a, <8 x i16>* %b) nounwind {
-  ; CHECK: vshf_v8i16_3:
+  ; CHECK-LABEL: vshf_v8i16_3:
 
   %1 = load <8 x i16>, <8 x i16>* %a
   ; CHECK-DAG: ld.h [[R1:\$w[0-9]+]], 0($5)
@@ -146,11 +138,10 @@ define void @vshf_v8i16_3(<8 x i16>* %c, <8 x i16>* %a, <8 x i16>* %b) nounwind
   ; CHECK-DAG: st.h [[R3]], 0($4)
 
   ret void
-  ; CHECK: .size vshf_v8i16_3
 }
 
 define void @vshf_v8i16_4(<8 x i16>* %c, <8 x i16>* %a, <8 x i16>* %b) nounwind {
-  ; CHECK: vshf_v8i16_4:
+  ; CHECK-LABEL: vshf_v8i16_4:
 
   %1 = load <8 x i16>, <8 x i16>* %a
   ; CHECK-DAG: ld.h [[R1:\$w[0-9]+]], 0($5)
@@ -160,14 +151,13 @@ define void @vshf_v8i16_4(<8 x i16>* %c, <8 x i16>* %a, <8 x i16>* %b) nounwind
   ; CHECK-DAG: st.h [[R3]], 0($4)
 
   ret void
-  ; CHECK: .size vshf_v8i16_4
 }
 
 ; Note: v4i32 only has one 4-element set so it's impossible to get a vshf.w
 ; instruction when using a single vector.
 
 define void @vshf_v4i32_0(<4 x i32>* %c, <4 x i32>* %a, <4 x i32>* %b) nounwind {
-  ; CHECK: vshf_v4i32_0:
+  ; CHECK-LABEL: vshf_v4i32_0:
 
   %1 = load <4 x i32>, <4 x i32>* %a
   ; CHECK-DAG: ld.w [[R1:\$w[0-9]+]], 0($5)
@@ -177,25 +167,23 @@ define void @vshf_v4i32_0(<4 x i32>* %c, <4 x i32>* %a, <4 x i32>* %b) nounwind
   ; CHECK-DAG: st.w [[R3]], 0($4)
 
   ret void
-  ; CHECK: .size vshf_v4i32_0
 }
 
 define void @vshf_v4i32_1(<4 x i32>* %c, <4 x i32>* %a, <4 x i32>* %b) nounwind {
-  ; CHECK: vshf_v4i32_1:
+  ; CHECK-LABEL: vshf_v4i32_1:
 
   %1 = load <4 x i32>, <4 x i32>* %a
   ; CHECK-DAG: ld.w [[R1:\$w[0-9]+]], 0($5)
   %2 = shufflevector <4 x i32> %1, <4 x i32> undef, <4 x i32> <i32 1, i32 1, i32 1, i32 1>
-  ; CHECK-DAG: shf.w [[R3:\$w[0-9]+]], [[R1]], 85
+  ; CHECK-DAG: splati.w [[R3:\$w[0-9]+]], [[R1]][1]
   store <4 x i32> %2, <4 x i32>* %c
   ; CHECK-DAG: st.w [[R3]], 0($4)
 
   ret void
-  ; CHECK: .size vshf_v4i32_1
 }
 
 define void @vshf_v4i32_2(<4 x i32>* %c, <4 x i32>* %a, <4 x i32>* %b) nounwind {
-  ; CHECK: vshf_v4i32_2:
+  ; CHECK-LABEL: vshf_v4i32_2:
 
   %1 = load <4 x i32>, <4 x i32>* %a
   %2 = load <4 x i32>, <4 x i32>* %b
@@ -206,11 +194,10 @@ define void @vshf_v4i32_2(<4 x i32>* %c, <4 x i32>* %a, <4 x i32>* %b) nounwind
   ; CHECK-DAG: st.w [[R3]], 0($4)
 
   ret void
-  ; CHECK: .size vshf_v4i32_2
 }
 
 define void @vshf_v4i32_3(<4 x i32>* %c, <4 x i32>* %a, <4 x i32>* %b) nounwind {
-  ; CHECK: vshf_v4i32_3:
+  ; CHECK-LABEL: vshf_v4i32_3:
 
   %1 = load <4 x i32>, <4 x i32>* %a
   ; CHECK-DAG: ld.w [[R1:\$w[0-9]+]], 0($5)
@@ -226,25 +213,24 @@ define void @vshf_v4i32_3(<4 x i32>* %c, <4 x i32>* %a, <4 x i32>* %b) nounwind
   ; CHECK-DAG: st.w [[R3]], 0($4)
 
   ret void
-  ; CHECK: .size vshf_v4i32_3
 }
 
 define void @vshf_v4i32_4(<4 x i32>* %c, <4 x i32>* %a, <4 x i32>* %b) nounwind {
-  ; CHECK: vshf_v4i32_4:
+  ; CHECK-LABEL: vshf_v4i32_4:
 
   %1 = load <4 x i32>, <4 x i32>* %a
   ; CHECK-DAG: ld.w [[R1:\$w[0-9]+]], 0($5)
   %2 = shufflevector <4 x i32> %1, <4 x i32> %1, <4 x i32> <i32 1, i32 5, i32 5, i32 1>
-  ; CHECK-DAG: shf.w [[R3:\$w[0-9]+]], [[R1]], 85
+  ; The two operand vectors are the same so element 1 and 5 are equivalent.
+  ; CHECK-DAG: splati.w [[R3:\$w[0-9]+]], [[R1]][1]
   store <4 x i32> %2, <4 x i32>* %c
   ; CHECK-DAG: st.w [[R3]], 0($4)
 
   ret void
-  ; CHECK: .size vshf_v4i32_4
 }
 
 define void @vshf_v2i64_0(<2 x i64>* %c, <2 x i64>* %a, <2 x i64>* %b) nounwind {
-  ; CHECK: vshf_v2i64_0:
+  ; CHECK-LABEL: vshf_v2i64_0:
 
   %1 = load <2 x i64>, <2 x i64>* %a
   ; CHECK-DAG: ld.d [[R1:\$w[0-9]+]], 0($5)
@@ -256,11 +242,10 @@ define void @vshf_v2i64_0(<2 x i64>* %c, <2 x i64>* %a, <2 x i64>* %b) nounwind
   ; CHECK-DAG: st.d [[R3]], 0($4)
 
   ret void
-  ; CHECK: .size vshf_v2i64_0
 }
 
 define void @vshf_v2i64_1(<2 x i64>* %c, <2 x i64>* %a, <2 x i64>* %b) nounwind {
-  ; CHECK: vshf_v2i64_1:
+  ; CHECK-LABEL: vshf_v2i64_1:
 
   %1 = load <2 x i64>, <2 x i64>* %a
   ; CHECK-DAG: ld.d [[R1:\$w[0-9]+]], 0($5)
@@ -270,11 +255,10 @@ define void @vshf_v2i64_1(<2 x i64>* %c, <2 x i64>* %a, <2 x i64>* %b) nounwind
   ; CHECK-DAG: st.d [[R3]], 0($4)
 
   ret void
-  ; CHECK: .size vshf_v2i64_1
 }
 
 define void @vshf_v2i64_2(<2 x i64>* %c, <2 x i64>* %a, <2 x i64>* %b) nounwind {
-  ; CHECK: vshf_v2i64_2:
+  ; CHECK-LABEL: vshf_v2i64_2:
 
   %1 = load <2 x i64>, <2 x i64>* %a
   %2 = load <2 x i64>, <2 x i64>* %b
@@ -287,11 +271,10 @@ define void @vshf_v2i64_2(<2 x i64>* %c, <2 x i64>* %a, <2 x i64>* %b) nounwind
   ; CHECK-DAG: st.d [[R3]], 0($4)
 
   ret void
-  ; CHECK: .size vshf_v2i64_2
 }
 
 define void @vshf_v2i64_3(<2 x i64>* %c, <2 x i64>* %a, <2 x i64>* %b) nounwind {
-  ; CHECK: vshf_v2i64_3:
+  ; CHECK-LABEL: vshf_v2i64_3:
 
   %1 = load <2 x i64>, <2 x i64>* %a
   ; CHECK-DAG: ld.d [[R1:\$w[0-9]+]], 0($5)
@@ -307,11 +290,10 @@ define void @vshf_v2i64_3(<2 x i64>* %c, <2 x i64>* %a, <2 x i64>* %b) nounwind
   ; CHECK-DAG: st.d [[R3]], 0($4)
 
   ret void
-  ; CHECK: .size vshf_v2i64_3
 }
 
 define void @vshf_v2i64_4(<2 x i64>* %c, <2 x i64>* %a, <2 x i64>* %b) nounwind {
-  ; CHECK: vshf_v2i64_4:
+  ; CHECK-LABEL: vshf_v2i64_4:
 
   %1 = load <2 x i64>, <2 x i64>* %a
   ; CHECK-DAG: ld.d [[R1:\$w[0-9]+]], 0($5)
@@ -321,11 +303,10 @@ define void @vshf_v2i64_4(<2 x i64>* %c, <2 x i64>* %a, <2 x i64>* %b) nounwind
   ; CHECK-DAG: st.d [[R3]], 0($4)
 
   ret void
-  ; CHECK: .size vshf_v2i64_4
 }
 
 define void @shf_v16i8_0(<16 x i8>* %c, <16 x i8>* %a, <16 x i8>* %b) nounwind {
-  ; CHECK: shf_v16i8_0:
+  ; CHECK-LABEL: shf_v16i8_0:
 
   %1 = load <16 x i8>, <16 x i8>* %a
   ; CHECK-DAG: ld.b [[R1:\$w[0-9]+]], 0($5)
@@ -335,11 +316,10 @@ define void @shf_v16i8_0(<16 x i8>* %c, <16 x i8>* %a, <16 x i8>* %b) nounwind {
   ; CHECK-DAG: st.b [[R3]], 0($4)
 
   ret void
-  ; CHECK: .size shf_v16i8_0
 }
 
 define void @shf_v8i16_0(<8 x i16>* %c, <8 x i16>* %a, <8 x i16>* %b) nounwind {
-  ; CHECK: shf_v8i16_0:
+  ; CHECK-LABEL: shf_v8i16_0:
 
   %1 = load <8 x i16>, <8 x i16>* %a
   ; CHECK-DAG: ld.h [[R1:\$w[0-9]+]], 0($5)
@@ -349,11 +329,10 @@ define void @shf_v8i16_0(<8 x i16>* %c, <8 x i16>* %a, <8 x i16>* %b) nounwind {
   ; CHECK-DAG: st.h [[R3]], 0($4)
 
   ret void
-  ; CHECK: .size shf_v8i16_0
 }
 
 define void @shf_v4i32_0(<4 x i32>* %c, <4 x i32>* %a, <4 x i32>* %b) nounwind {
-  ; CHECK: shf_v4i32_0:
+  ; CHECK-LABEL: shf_v4i32_0:
 
   %1 = load <4 x i32>, <4 x i32>* %a
   ; CHECK-DAG: ld.w [[R1:\$w[0-9]+]], 0($5)
@@ -363,13 +342,12 @@ define void @shf_v4i32_0(<4 x i32>* %c, <4 x i32>* %a, <4 x i32>* %b) nounwind {
   ; CHECK-DAG: st.w [[R3]], 0($4)
 
   ret void
-  ; CHECK: .size shf_v4i32_0
 }
 
 ; shf.d does not exist
 
 define void @ilvev_v16i8_0(<16 x i8>* %c, <16 x i8>* %a, <16 x i8>* %b) nounwind {
-  ; CHECK: ilvev_v16i8_0:
+  ; CHECK-LABEL: ilvev_v16i8_0:
 
   %1 = load <16 x i8>, <16 x i8>* %a
   ; CHECK-DAG: ld.b [[R1:\$w[0-9]+]], 0($5)
@@ -377,64 +355,177 @@ define void @ilvev_v16i8_0(<16 x i8>* %c, <16 x i8>* %a, <16 x i8>* %b) nounwind
   ; CHECK-DAG: ld.b [[R2:\$w[0-9]+]], 0($6)
   %3 = shufflevector <16 x i8> %1, <16 x i8> %2,
                      <16 x i32> <i32 0, i32 16, i32 2, i32 18, i32 4, i32 20, i32 6, i32 22, i32 8, i32 24, i32 10, i32 26, i32 12, i32 28, i32 14, i32 30>
-  ; CHECK-DAG: ilvev.b [[R3:\$w[0-9]+]], [[R1]], [[R2]]
+  ; CHECK-DAG: ilvev.b [[R3:\$w[0-9]+]], [[R2]], [[R1]]
   store <16 x i8> %3, <16 x i8>* %c
   ; CHECK-DAG: st.b [[R3]], 0($4)
 
   ret void
-  ; CHECK: .size ilvev_v16i8_0
 }
 
 define void @ilvev_v8i16_0(<8 x i16>* %c, <8 x i16>* %a, <8 x i16>* %b) nounwind {
-  ; CHECK: ilvev_v8i16_0:
+  ; CHECK-LABEL: ilvev_v8i16_0:
 
   %1 = load <8 x i16>, <8 x i16>* %a
   ; CHECK-DAG: ld.h [[R1:\$w[0-9]+]], 0($5)
   %2 = load <8 x i16>, <8 x i16>* %b
   ; CHECK-DAG: ld.h [[R2:\$w[0-9]+]], 0($6)
   %3 = shufflevector <8 x i16> %1, <8 x i16> %2, <8 x i32> <i32 0, i32 8, i32 2, i32 10, i32 4, i32 12, i32 6, i32 14>
-  ; CHECK-DAG: ilvev.h [[R3:\$w[0-9]+]], [[R1]], [[R2]]
+  ; CHECK-DAG: ilvev.h [[R3:\$w[0-9]+]], [[R2]], [[R1]]
   store <8 x i16> %3, <8 x i16>* %c
   ; CHECK-DAG: st.h [[R3]], 0($4)
 
   ret void
-  ; CHECK: .size ilvev_v8i16_0
 }
 
 define void @ilvev_v4i32_0(<4 x i32>* %c, <4 x i32>* %a, <4 x i32>* %b) nounwind {
-  ; CHECK: ilvev_v4i32_0:
+  ; CHECK-LABEL: ilvev_v4i32_0:
 
   %1 = load <4 x i32>, <4 x i32>* %a
   ; CHECK-DAG: ld.w [[R1:\$w[0-9]+]], 0($5)
   %2 = load <4 x i32>, <4 x i32>* %b
   ; CHECK-DAG: ld.w [[R2:\$w[0-9]+]], 0($6)
   %3 = shufflevector <4 x i32> %1, <4 x i32> %2, <4 x i32> <i32 0, i32 4, i32 2, i32 6>
-  ; CHECK-DAG: ilvev.w [[R3:\$w[0-9]+]], [[R1]], [[R2]]
+  ; CHECK-DAG: ilvev.w [[R3:\$w[0-9]+]], [[R2]], [[R1]]
   store <4 x i32> %3, <4 x i32>* %c
   ; CHECK-DAG: st.w [[R3]], 0($4)
 
   ret void
-  ; CHECK: .size ilvev_v4i32_0
 }
 
 define void @ilvev_v2i64_0(<2 x i64>* %c, <2 x i64>* %a, <2 x i64>* %b) nounwind {
-  ; CHECK: ilvev_v2i64_0:
+  ; CHECK-LABEL: ilvev_v2i64_0:
 
   %1 = load <2 x i64>, <2 x i64>* %a
   ; CHECK-DAG: ld.d [[R1:\$w[0-9]+]], 0($5)
   %2 = load <2 x i64>, <2 x i64>* %b
   ; CHECK-DAG: ld.d [[R2:\$w[0-9]+]], 0($6)
   %3 = shufflevector <2 x i64> %1, <2 x i64> %2, <2 x i32> <i32 0, i32 2>
-  ; CHECK-DAG: ilvev.d [[R3:\$w[0-9]+]], [[R1]], [[R2]]
+  ; CHECK-DAG: ilvev.d [[R3:\$w[0-9]+]], [[R2]], [[R1]]
+  store <2 x i64> %3, <2 x i64>* %c
+  ; CHECK-DAG: st.d [[R3]], 0($4)
+
+  ret void
+}
+
+; Interleaving one operand with itself.
+define void @ilvev_v16i8_1(<16 x i8>* %c, <16 x i8>* %a, <16 x i8>* %b) nounwind {
+  ; CHECK-LABEL: ilvev_v16i8_1:
+
+  %1 = load <16 x i8>, <16 x i8>* %a
+  ; CHECK-DAG: ld.b [[R1:\$w[0-9]+]], 0($5)
+  %2 = load <16 x i8>, <16 x i8>* %b
+  %3 = shufflevector <16 x i8> %1, <16 x i8> %2,
+                     <16 x i32> <i32 0, i32 0, i32 2, i32 2, i32 4, i32 4, i32 6, i32 6, i32 8, i32 8, i32 10, i32 10, i32 12, i32 12, i32 14, i32 14>
+  ; CHECK-DAG: ilvev.b [[R3:\$w[0-9]+]], [[R1]], [[R1]]
+  store <16 x i8> %3, <16 x i8>* %c
+  ; CHECK-DAG: st.b [[R3]], 0($4)
+
+  ret void
+}
+
+define void @ilvev_v8i16_1(<8 x i16>* %c, <8 x i16>* %a, <8 x i16>* %b) nounwind {
+  ; CHECK-LABEL: ilvev_v8i16_1:
+
+  %1 = load <8 x i16>, <8 x i16>* %a
+  ; CHECK-DAG: ld.h [[R1:\$w[0-9]+]], 0($5)
+  %2 = load <8 x i16>, <8 x i16>* %b
+  %3 = shufflevector <8 x i16> %1, <8 x i16> %2, <8 x i32> <i32 0, i32 0, i32 2, i32 2, i32 4, i32 4, i32 6, i32 6>
+  ; CHECK-DAG: ilvev.h [[R3:\$w[0-9]+]], [[R1]], [[R1]]
+  store <8 x i16> %3, <8 x i16>* %c
+  ; CHECK-DAG: st.h [[R3]], 0($4)
+
+  ret void
+}
+
+define void @ilvev_v4i32_1(<4 x i32>* %c, <4 x i32>* %a, <4 x i32>* %b) nounwind {
+  ; CHECK-LABEL: ilvev_v4i32_1:
+
+  %1 = load <4 x i32>, <4 x i32>* %a
+  ; CHECK-DAG: ld.w [[R1:\$w[0-9]+]], 0($5)
+  %2 = load <4 x i32>, <4 x i32>* %b
+  %3 = shufflevector <4 x i32> %1, <4 x i32> %2, <4 x i32> <i32 0, i32 0, i32 2, i32 2>
+  ; CHECK-DAG: ilvev.w [[R3:\$w[0-9]+]], [[R1]], [[R1]]
+  store <4 x i32> %3, <4 x i32>* %c
+  ; CHECK-DAG: st.w [[R3]], 0($4)
+
+  ret void
+}
+
+define void @ilvev_v2i64_1(<2 x i64>* %c, <2 x i64>* %a, <2 x i64>* %b) nounwind {
+  ; CHECK-LABEL: ilvev_v2i64_1:
+
+  %1 = load <2 x i64>, <2 x i64>* %a
+  ; CHECK-DAG: ld.d [[R1:\$w[0-9]+]], 0($5)
+  %2 = load <2 x i64>, <2 x i64>* %b
+  %3 = shufflevector <2 x i64> %1, <2 x i64> %2, <2 x i32> <i32 0, i32 0>
+  ; ilvev.d with two identical operands is equivalent to splati.d
+  ; CHECK-DAG: splati.d [[R3:\$w[0-9]+]], [[R1]][0]
+  store <2 x i64> %3, <2 x i64>* %c
+  ; CHECK-DAG: st.d [[R3]], 0($4)
+
+  ret void
+}
+
+define void @ilvev_v16i8_2(<16 x i8>* %c, <16 x i8>* %a, <16 x i8>* %b) nounwind {
+  ; CHECK-LABEL: ilvev_v16i8_2:
+
+  %1 = load <16 x i8>, <16 x i8>* %a
+  %2 = load <16 x i8>, <16 x i8>* %b
+  ; CHECK-DAG: ld.b [[R2:\$w[0-9]+]], 0($6)
+  %3 = shufflevector <16 x i8> %1, <16 x i8> %2,
+                     <16 x i32> <i32 16, i32 16, i32 18, i32 18, i32 20, i32 20, i32 22, i32 22, i32 24, i32 24, i32 26, i32 26, i32 28, i32 28, i32 30, i32 30>
+  ; CHECK-DAG: ilvev.b [[R3:\$w[0-9]+]], [[R2]], [[R2]]
+  store <16 x i8> %3, <16 x i8>* %c
+  ; CHECK-DAG: st.b [[R3]], 0($4)
+
+  ret void
+}
+
+define void @ilvev_v8i16_2(<8 x i16>* %c, <8 x i16>* %a, <8 x i16>* %b) nounwind {
+  ; CHECK-LABEL: ilvev_v8i16_2:
+
+  %1 = load <8 x i16>, <8 x i16>* %a
+  %2 = load <8 x i16>, <8 x i16>* %b
+  ; CHECK-DAG: ld.h [[R2:\$w[0-9]+]], 0($6)
+  %3 = shufflevector <8 x i16> %1, <8 x i16> %2, <8 x i32> <i32 8, i32 8, i32 10, i32 10, i32 12, i32 12, i32 14, i32 14>
+  ; CHECK-DAG: ilvev.h [[R3:\$w[0-9]+]], [[R2]], [[R2]]
+  store <8 x i16> %3, <8 x i16>* %c
+  ; CHECK-DAG: st.h [[R3]], 0($4)
+
+  ret void
+}
+
+define void @ilvev_v4i32_2(<4 x i32>* %c, <4 x i32>* %a, <4 x i32>* %b) nounwind {
+  ; CHECK-LABEL: ilvev_v4i32_2:
+
+  %1 = load <4 x i32>, <4 x i32>* %a
+  %2 = load <4 x i32>, <4 x i32>* %b
+  ; CHECK-DAG: ld.w [[R2:\$w[0-9]+]], 0($6)
+  %3 = shufflevector <4 x i32> %1, <4 x i32> %2, <4 x i32> <i32 4, i32 4, i32 6, i32 6>
+  ; CHECK-DAG: ilvev.w [[R3:\$w[0-9]+]], [[R2]], [[R2]]
+  store <4 x i32> %3, <4 x i32>* %c
+  ; CHECK-DAG: st.w [[R3]], 0($4)
+
+  ret void
+}
+
+define void @ilvev_v2i64_2(<2 x i64>* %c, <2 x i64>* %a, <2 x i64>* %b) nounwind {
+  ; CHECK-LABEL: ilvev_v2i64_2:
+
+  %1 = load <2 x i64>, <2 x i64>* %a
+  %2 = load <2 x i64>, <2 x i64>* %b
+  ; CHECK-DAG: ld.d [[R2:\$w[0-9]+]], 0($6)
+  %3 = shufflevector <2 x i64> %1, <2 x i64> %2, <2 x i32> <i32 2, i32 2>
+  ; ilvev.d with two identical operands is equivalent to splati.d
+  ; CHECK-DAG: splati.d [[R3:\$w[0-9]+]], [[R2]][0]
   store <2 x i64> %3, <2 x i64>* %c
   ; CHECK-DAG: st.d [[R3]], 0($4)
 
   ret void
-  ; CHECK: .size ilvev_v2i64_0
 }
 
 define void @ilvod_v16i8_0(<16 x i8>* %c, <16 x i8>* %a, <16 x i8>* %b) nounwind {
-  ; CHECK: ilvod_v16i8_0:
+  ; CHECK-LABEL: ilvod_v16i8_0:
 
   %1 = load <16 x i8>, <16 x i8>* %a
   ; CHECK-DAG: ld.b [[R1:\$w[0-9]+]], 0($5)
@@ -447,11 +538,10 @@ define void @ilvod_v16i8_0(<16 x i8>* %c, <16 x i8>* %a, <16 x i8>* %b) nounwind
   ; CHECK-DAG: st.b [[R3]], 0($4)
 
   ret void
-  ; CHECK: .size ilvod_v16i8_0
 }
 
 define void @ilvod_v8i16_0(<8 x i16>* %c, <8 x i16>* %a, <8 x i16>* %b) nounwind {
-  ; CHECK: ilvod_v8i16_0:
+  ; CHECK-LABEL: ilvod_v8i16_0:
 
   %1 = load <8 x i16>, <8 x i16>* %a
   ; CHECK-DAG: ld.h [[R1:\$w[0-9]+]], 0($5)
@@ -463,11 +553,10 @@ define void @ilvod_v8i16_0(<8 x i16>* %c, <8 x i16>* %a, <8 x i16>* %b) nounwind
   ; CHECK-DAG: st.h [[R3]], 0($4)
 
   ret void
-  ; CHECK: .size ilvod_v8i16_0
 }
 
 define void @ilvod_v4i32_0(<4 x i32>* %c, <4 x i32>* %a, <4 x i32>* %b) nounwind {
-  ; CHECK: ilvod_v4i32_0:
+  ; CHECK-LABEL: ilvod_v4i32_0:
 
   %1 = load <4 x i32>, <4 x i32>* %a
   ; CHECK-DAG: ld.w [[R1:\$w[0-9]+]], 0($5)
@@ -479,11 +568,10 @@ define void @ilvod_v4i32_0(<4 x i32>* %c, <4 x i32>* %a, <4 x i32>* %b) nounwind
   ; CHECK-DAG: st.w [[R3]], 0($4)
 
   ret void
-  ; CHECK: .size ilvod_v4i32_0
 }
 
 define void @ilvod_v2i64_0(<2 x i64>* %c, <2 x i64>* %a, <2 x i64>* %b) nounwind {
-  ; CHECK: ilvod_v2i64_0:
+  ; CHECK-LABEL: ilvod_v2i64_0:
 
   %1 = load <2 x i64>, <2 x i64>* %a
   ; CHECK-DAG: ld.d [[R1:\$w[0-9]+]], 0($5)
@@ -495,11 +583,126 @@ define void @ilvod_v2i64_0(<2 x i64>* %c, <2 x i64>* %a, <2 x i64>* %b) nounwind
   ; CHECK-DAG: st.d [[R3]], 0($4)
 
   ret void
-  ; CHECK: .size ilvod_v2i64_0
 }
 
-define void @ilvl_v16i8_0(<16 x i8>* %c, <16 x i8>* %a, <16 x i8>* %b) nounwind {
-  ; CHECK: ilvl_v16i8_0:
+define void @ilvod_v16i8_1(<16 x i8>* %c, <16 x i8>* %a, <16 x i8>* %b) nounwind {
+  ; CHECK-LABEL: ilvod_v16i8_1:
+
+  %1 = load <16 x i8>, <16 x i8>* %a
+  ; CHECK-DAG: ld.b [[R1:\$w[0-9]+]], 0($5)
+  %2 = load <16 x i8>, <16 x i8>* %b
+  %3 = shufflevector <16 x i8> %1, <16 x i8> %2,
+                     <16 x i32> <i32 1, i32 1, i32 3, i32 3, i32 5, i32 5, i32 7, i32 7, i32 9, i32 9, i32 11, i32 11, i32 13, i32 13, i32 15, i32 15>
+  ; CHECK-DAG: ilvod.b [[R3:\$w[0-9]+]], [[R1]], [[R1]]
+  store <16 x i8> %3, <16 x i8>* %c
+  ; CHECK-DAG: st.b [[R3]], 0($4)
+
+  ret void
+}
+
+define void @ilvod_v8i16_1(<8 x i16>* %c, <8 x i16>* %a, <8 x i16>* %b) nounwind {
+  ; CHECK-LABEL: ilvod_v8i16_1:
+
+  %1 = load <8 x i16>, <8 x i16>* %a
+  ; CHECK-DAG: ld.h [[R1:\$w[0-9]+]], 0($5)
+  %2 = load <8 x i16>, <8 x i16>* %b
+  %3 = shufflevector <8 x i16> %1, <8 x i16> %2, <8 x i32> <i32 1, i32 1, i32 3, i32 3, i32 5, i32 5, i32 7, i32 7>
+  ; CHECK-DAG: ilvod.h [[R3:\$w[0-9]+]], [[R1]], [[R1]]
+  store <8 x i16> %3, <8 x i16>* %c
+  ; CHECK-DAG: st.h [[R3]], 0($4)
+
+  ret void
+}
+
+define void @ilvod_v4i32_1(<4 x i32>* %c, <4 x i32>* %a, <4 x i32>* %b) nounwind {
+  ; CHECK-LABEL: ilvod_v4i32_1:
+
+  %1 = load <4 x i32>, <4 x i32>* %a
+  ; CHECK-DAG: ld.w [[R1:\$w[0-9]+]], 0($5)
+  %2 = load <4 x i32>, <4 x i32>* %b
+  %3 = shufflevector <4 x i32> %1, <4 x i32> %2, <4 x i32> <i32 1, i32 1, i32 3, i32 3>
+  ; CHECK-DAG: ilvod.w [[R3:\$w[0-9]+]], [[R1]], [[R1]]
+  store <4 x i32> %3, <4 x i32>* %c
+  ; CHECK-DAG: st.w [[R3]], 0($4)
+
+  ret void
+}
+
+define void @ilvod_v2i64_1(<2 x i64>* %c, <2 x i64>* %a, <2 x i64>* %b) nounwind {
+  ; CHECK-LABEL: ilvod_v2i64_1:
+
+  %1 = load <2 x i64>, <2 x i64>* %a
+  ; CHECK-DAG: ld.d [[R1:\$w[0-9]+]], 0($5)
+  %2 = load <2 x i64>, <2 x i64>* %b
+  %3 = shufflevector <2 x i64> %1, <2 x i64> %2, <2 x i32> <i32 1, i32 1>
+  ; ilvod.d with two identical operands is equivalent to splati.d
+  ; CHECK-DAG: splati.d [[R3:\$w[0-9]+]], [[R1]][1]
+  store <2 x i64> %3, <2 x i64>* %c
+  ; CHECK-DAG: st.d [[R3]], 0($4)
+
+  ret void
+}
+
+define void @ilvod_v16i8_2(<16 x i8>* %c, <16 x i8>* %a, <16 x i8>* %b) nounwind {
+  ; CHECK-LABEL: ilvod_v16i8_2:
+
+  %1 = load <16 x i8>, <16 x i8>* %a
+  %2 = load <16 x i8>, <16 x i8>* %b
+  ; CHECK-DAG: ld.b [[R2:\$w[0-9]+]], 0($6)
+  %3 = shufflevector <16 x i8> %1, <16 x i8> %2,
+                     <16 x i32> <i32 17, i32 17, i32 19, i32 19, i32 21, i32 21, i32 23, i32 23, i32 25, i32 25, i32 27, i32 27, i32 29, i32 29, i32 31, i32 31>
+  ; CHECK-DAG: ilvod.b [[R3:\$w[0-9]+]], [[R2]], [[R2]]
+  store <16 x i8> %3, <16 x i8>* %c
+  ; CHECK-DAG: st.b [[R3]], 0($4)
+
+  ret void
+}
+
+define void @ilvod_v8i16_2(<8 x i16>* %c, <8 x i16>* %a, <8 x i16>* %b) nounwind {
+  ; CHECK-LABEL: ilvod_v8i16_2:
+
+  %1 = load <8 x i16>, <8 x i16>* %a
+  %2 = load <8 x i16>, <8 x i16>* %b
+  ; CHECK-DAG: ld.h [[R2:\$w[0-9]+]], 0($6)
+  %3 = shufflevector <8 x i16> %1, <8 x i16> %2, <8 x i32> <i32 9, i32 9, i32 11, i32 11, i32 13, i32 13, i32 15, i32 15>
+  ; CHECK-DAG: ilvod.h [[R3:\$w[0-9]+]], [[R2]], [[R2]]
+  store <8 x i16> %3, <8 x i16>* %c
+  ; CHECK-DAG: st.h [[R3]], 0($4)
+
+  ret void
+}
+
+define void @ilvod_v4i32_2(<4 x i32>* %c, <4 x i32>* %a, <4 x i32>* %b) nounwind {
+  ; CHECK-LABEL: ilvod_v4i32_2:
+
+  %1 = load <4 x i32>, <4 x i32>* %a
+  %2 = load <4 x i32>, <4 x i32>* %b
+  ; CHECK-DAG: ld.w [[R2:\$w[0-9]+]], 0($6)
+  %3 = shufflevector <4 x i32> %1, <4 x i32> %2, <4 x i32> <i32 5, i32 5, i32 7, i32 7>
+  ; CHECK-DAG: ilvod.w [[R3:\$w[0-9]+]], [[R2]], [[R2]]
+  store <4 x i32> %3, <4 x i32>* %c
+  ; CHECK-DAG: st.w [[R3]], 0($4)
+
+  ret void
+}
+
+define void @ilvod_v2i64_2(<2 x i64>* %c, <2 x i64>* %a, <2 x i64>* %b) nounwind {
+  ; CHECK-LABEL: ilvod_v2i64_2:
+
+  %1 = load <2 x i64>, <2 x i64>* %a
+  %2 = load <2 x i64>, <2 x i64>* %b
+  ; CHECK-DAG: ld.d [[R2:\$w[0-9]+]], 0($6)
+  %3 = shufflevector <2 x i64> %1, <2 x i64> %2, <2 x i32> <i32 3, i32 3>
+  ; ilvod.d with two identical operands is equivalent to splati.d
+  ; CHECK-DAG: splati.d [[R3:\$w[0-9]+]], [[R2]][1]
+  store <2 x i64> %3, <2 x i64>* %c
+  ; CHECK-DAG: st.d [[R3]], 0($4)
+
+  ret void
+}
+
+define void @ilvr_v16i8_0(<16 x i8>* %c, <16 x i8>* %a, <16 x i8>* %b) nounwind {
+  ; CHECK-LABEL: ilvr_v16i8_0:
 
   %1 = load <16 x i8>, <16 x i8>* %a
   ; CHECK-DAG: ld.b [[R1:\$w[0-9]+]], 0($5)
@@ -507,65 +710,177 @@ define void @ilvl_v16i8_0(<16 x i8>* %c, <16 x i8>* %a, <16 x i8>* %b) nounwind
   ; CHECK-DAG: ld.b [[R2:\$w[0-9]+]], 0($6)
   %3 = shufflevector <16 x i8> %1, <16 x i8> %2,
                      <16 x i32> <i32 0, i32 16, i32 1, i32 17, i32 2, i32 18, i32 3, i32 19, i32 4, i32 20, i32 5, i32 21, i32 6, i32 22, i32 7, i32 23>
-  ; CHECK-DAG: ilvl.b [[R3:\$w[0-9]+]], [[R1]], [[R2]]
+  ; CHECK-DAG: ilvr.b [[R3:\$w[0-9]+]], [[R2]], [[R1]]
   store <16 x i8> %3, <16 x i8>* %c
   ; CHECK-DAG: st.b [[R3]], 0($4)
 
   ret void
-  ; CHECK: .size ilvl_v16i8_0
 }
 
-define void @ilvl_v8i16_0(<8 x i16>* %c, <8 x i16>* %a, <8 x i16>* %b) nounwind {
-  ; CHECK: ilvl_v8i16_0:
+define void @ilvr_v8i16_0(<8 x i16>* %c, <8 x i16>* %a, <8 x i16>* %b) nounwind {
+  ; CHECK-LABEL: ilvr_v8i16_0:
 
   %1 = load <8 x i16>, <8 x i16>* %a
   ; CHECK-DAG: ld.h [[R1:\$w[0-9]+]], 0($5)
   %2 = load <8 x i16>, <8 x i16>* %b
   ; CHECK-DAG: ld.h [[R2:\$w[0-9]+]], 0($6)
   %3 = shufflevector <8 x i16> %1, <8 x i16> %2, <8 x i32> <i32 0, i32 8, i32 1, i32 9, i32 2, i32 10, i32 3, i32 11>
-  ; CHECK-DAG: ilvl.h [[R3:\$w[0-9]+]], [[R1]], [[R2]]
+  ; CHECK-DAG: ilvr.h [[R3:\$w[0-9]+]], [[R2]], [[R1]]
   store <8 x i16> %3, <8 x i16>* %c
   ; CHECK-DAG: st.h [[R3]], 0($4)
 
   ret void
-  ; CHECK: .size ilvl_v8i16_0
 }
 
-define void @ilvl_v4i32_0(<4 x i32>* %c, <4 x i32>* %a, <4 x i32>* %b) nounwind {
-  ; CHECK: ilvl_v4i32_0:
+define void @ilvr_v4i32_0(<4 x i32>* %c, <4 x i32>* %a, <4 x i32>* %b) nounwind {
+  ; CHECK-LABEL: ilvr_v4i32_0:
 
   %1 = load <4 x i32>, <4 x i32>* %a
   ; CHECK-DAG: ld.w [[R1:\$w[0-9]+]], 0($5)
   %2 = load <4 x i32>, <4 x i32>* %b
   ; CHECK-DAG: ld.w [[R2:\$w[0-9]+]], 0($6)
   %3 = shufflevector <4 x i32> %1, <4 x i32> %2, <4 x i32> <i32 0, i32 4, i32 1, i32 5>
-  ; CHECK-DAG: ilvl.w [[R3:\$w[0-9]+]], [[R1]], [[R2]]
+  ; CHECK-DAG: ilvr.w [[R3:\$w[0-9]+]], [[R2]], [[R1]]
   store <4 x i32> %3, <4 x i32>* %c
   ; CHECK-DAG: st.w [[R3]], 0($4)
 
   ret void
-  ; CHECK: .size ilvl_v4i32_0
 }
 
-define void @ilvl_v2i64_0(<2 x i64>* %c, <2 x i64>* %a, <2 x i64>* %b) nounwind {
-  ; CHECK: ilvl_v2i64_0:
+define void @ilvr_v2i64_0(<2 x i64>* %c, <2 x i64>* %a, <2 x i64>* %b) nounwind {
+  ; CHECK-LABEL: ilvr_v2i64_0:
 
   %1 = load <2 x i64>, <2 x i64>* %a
   ; CHECK-DAG: ld.d [[R1:\$w[0-9]+]], 0($5)
   %2 = load <2 x i64>, <2 x i64>* %b
   ; CHECK-DAG: ld.d [[R2:\$w[0-9]+]], 0($6)
   %3 = shufflevector <2 x i64> %1, <2 x i64> %2, <2 x i32> <i32 0, i32 2>
-  ; ilvl.d and ilvev.d are equivalent for v2i64
-  ; CHECK-DAG: ilvev.d [[R3:\$w[0-9]+]], [[R1]], [[R2]]
+  ; ilvr.d and ilvev.d are equivalent for v2i64
+  ; CHECK-DAG: ilvev.d [[R3:\$w[0-9]+]], [[R2]], [[R1]]
   store <2 x i64> %3, <2 x i64>* %c
   ; CHECK-DAG: st.d [[R3]], 0($4)
 
   ret void
-  ; CHECK: .size ilvl_v2i64_0
 }
 
-define void @ilvr_v16i8_0(<16 x i8>* %c, <16 x i8>* %a, <16 x i8>* %b) nounwind {
-  ; CHECK: ilvr_v16i8_0:
+define void @ilvr_v16i8_1(<16 x i8>* %c, <16 x i8>* %a, <16 x i8>* %b) nounwind {
+  ; CHECK-LABEL: ilvr_v16i8_1:
+
+  %1 = load <16 x i8>, <16 x i8>* %a
+  %2 = load <16 x i8>, <16 x i8>* %b
+  ; CHECK-DAG: ld.b [[R2:\$w[0-9]+]], 0($6)
+  %3 = shufflevector <16 x i8> %1, <16 x i8> %2,
+                     <16 x i32> <i32 16, i32 16, i32 17, i32 17, i32 18, i32 18, i32 19, i32 19, i32 20, i32 20, i32 21, i32 21, i32 22, i32 22, i32 23, i32 23>
+  ; CHECK-DAG: ilvr.b [[R3:\$w[0-9]+]], [[R2]], [[R2]]
+  store <16 x i8> %3, <16 x i8>* %c
+  ; CHECK-DAG: st.b [[R3]], 0($4)
+
+  ret void
+}
+
+define void @ilvr_v8i16_1(<8 x i16>* %c, <8 x i16>* %a, <8 x i16>* %b) nounwind {
+  ; CHECK-LABEL: ilvr_v8i16_1:
+
+  %1 = load <8 x i16>, <8 x i16>* %a
+  %2 = load <8 x i16>, <8 x i16>* %b
+  ; CHECK-DAG: ld.h [[R2:\$w[0-9]+]], 0($6)
+  %3 = shufflevector <8 x i16> %1, <8 x i16> %2, <8 x i32> <i32 8, i32 8, i32 9, i32 9, i32 10, i32 10, i32 11, i32 11>
+  ; CHECK-DAG: ilvr.h [[R3:\$w[0-9]+]], [[R2]], [[R2]]
+  store <8 x i16> %3, <8 x i16>* %c
+  ; CHECK-DAG: st.h [[R3]], 0($4)
+
+  ret void
+}
+
+define void @ilvr_v4i32_1(<4 x i32>* %c, <4 x i32>* %a, <4 x i32>* %b) nounwind {
+  ; CHECK-LABEL: ilvr_v4i32_1:
+
+  %1 = load <4 x i32>, <4 x i32>* %a
+  %2 = load <4 x i32>, <4 x i32>* %b
+  ; CHECK-DAG: ld.w [[R2:\$w[0-9]+]], 0($6)
+  %3 = shufflevector <4 x i32> %1, <4 x i32> %2, <4 x i32> <i32 4, i32 4, i32 5, i32 5>
+  ; CHECK-DAG: ilvr.w [[R3:\$w[0-9]+]], [[R2]], [[R2]]
+  store <4 x i32> %3, <4 x i32>* %c
+  ; CHECK-DAG: st.w [[R3]], 0($4)
+
+  ret void
+}
+
+define void @ilvr_v2i64_1(<2 x i64>* %c, <2 x i64>* %a, <2 x i64>* %b) nounwind {
+  ; CHECK-LABEL: ilvr_v2i64_1:
+
+  %1 = load <2 x i64>, <2 x i64>* %a
+  %2 = load <2 x i64>, <2 x i64>* %b
+  ; CHECK-DAG: ld.d [[R2:\$w[0-9]+]], 0($6)
+  %3 = shufflevector <2 x i64> %1, <2 x i64> %2, <2 x i32> <i32 2, i32 2>
+  ; ilvr.d and splati.d are equivalent for v2i64
+  ; CHECK-DAG: splati.d [[R3:\$w[0-9]+]], [[R2]][0]
+  store <2 x i64> %3, <2 x i64>* %c
+  ; CHECK-DAG: st.d [[R3]], 0($4)
+
+  ret void
+}
+
+define void @ilvr_v16i8_2(<16 x i8>* %c, <16 x i8>* %a, <16 x i8>* %b) nounwind {
+  ; CHECK-LABEL: ilvr_v16i8_2:
+
+  %1 = load <16 x i8>, <16 x i8>* %a
+  ; CHECK-DAG: ld.b [[R1:\$w[0-9]+]], 0($5)
+  %2 = load <16 x i8>, <16 x i8>* %b
+  %3 = shufflevector <16 x i8> %1, <16 x i8> %2,
+                     <16 x i32> <i32 0, i32 0, i32 1, i32 1, i32 2, i32 2, i32 3, i32 3, i32 4, i32 4, i32 5, i32 5, i32 6, i32 6, i32 7, i32 7>
+  ; CHECK-DAG: ilvr.b [[R3:\$w[0-9]+]], [[R1]], [[R1]]
+  store <16 x i8> %3, <16 x i8>* %c
+  ; CHECK-DAG: st.b [[R3]], 0($4)
+
+  ret void
+}
+
+define void @ilvr_v8i16_2(<8 x i16>* %c, <8 x i16>* %a, <8 x i16>* %b) nounwind {
+  ; CHECK-LABEL: ilvr_v8i16_2:
+
+  %1 = load <8 x i16>, <8 x i16>* %a
+  ; CHECK-DAG: ld.h [[R1:\$w[0-9]+]], 0($5)
+  %2 = load <8 x i16>, <8 x i16>* %b
+  %3 = shufflevector <8 x i16> %1, <8 x i16> %2, <8 x i32> <i32 0, i32 0, i32 1, i32 1, i32 2, i32 2, i32 3, i32 3>
+  ; CHECK-DAG: ilvr.h [[R3:\$w[0-9]+]], [[R1]], [[R1]]
+  store <8 x i16> %3, <8 x i16>* %c
+  ; CHECK-DAG: st.h [[R3]], 0($4)
+
+  ret void
+}
+
+define void @ilvr_v4i32_2(<4 x i32>* %c, <4 x i32>* %a, <4 x i32>* %b) nounwind {
+  ; CHECK-LABEL: ilvr_v4i32_2:
+
+  %1 = load <4 x i32>, <4 x i32>* %a
+  ; CHECK-DAG: ld.w [[R1:\$w[0-9]+]], 0($5)
+  %2 = load <4 x i32>, <4 x i32>* %b
+  %3 = shufflevector <4 x i32> %1, <4 x i32> %2, <4 x i32> <i32 0, i32 0, i32 1, i32 1>
+  ; CHECK-DAG: ilvr.w [[R3:\$w[0-9]+]], [[R1]], [[R1]]
+  store <4 x i32> %3, <4 x i32>* %c
+  ; CHECK-DAG: st.w [[R3]], 0($4)
+
+  ret void
+}
+
+define void @ilvr_v2i64_2(<2 x i64>* %c, <2 x i64>* %a, <2 x i64>* %b) nounwind {
+  ; CHECK-LABEL: ilvr_v2i64_2:
+
+  %1 = load <2 x i64>, <2 x i64>* %a
+  ; CHECK-DAG: ld.d [[R1:\$w[0-9]+]], 0($5)
+  %2 = load <2 x i64>, <2 x i64>* %b
+  %3 = shufflevector <2 x i64> %1, <2 x i64> %2, <2 x i32> <i32 0, i32 0>
+  ; ilvr.d and splati.d are equivalent for v2i64
+  ; CHECK-DAG: splati.d [[R3:\$w[0-9]+]], [[R1]][0]
+  store <2 x i64> %3, <2 x i64>* %c
+  ; CHECK-DAG: st.d [[R3]], 0($4)
+
+  ret void
+}
+
+define void @ilvl_v16i8_0(<16 x i8>* %c, <16 x i8>* %a, <16 x i8>* %b) nounwind {
+  ; CHECK-LABEL: ilvl_v16i8_0:
 
   %1 = load <16 x i8>, <16 x i8>* %a
   ; CHECK-DAG: ld.b [[R1:\$w[0-9]+]], 0($5)
@@ -573,65 +888,177 @@ define void @ilvr_v16i8_0(<16 x i8>* %c, <16 x i8>* %a, <16 x i8>* %b) nounwind
   ; CHECK-DAG: ld.b [[R2:\$w[0-9]+]], 0($6)
   %3 = shufflevector <16 x i8> %1, <16 x i8> %2,
                      <16 x i32> <i32 8, i32 24, i32 9, i32 25, i32 10, i32 26, i32 11, i32 27, i32 12, i32 28, i32 13, i32 29, i32 14, i32 30, i32 15, i32 31>
-  ; CHECK-DAG: ilvr.b [[R3:\$w[0-9]+]], [[R1]], [[R2]]
+  ; CHECK-DAG: ilvl.b [[R3:\$w[0-9]+]], [[R2]], [[R1]]
   store <16 x i8> %3, <16 x i8>* %c
   ; CHECK-DAG: st.b [[R3]], 0($4)
 
   ret void
-  ; CHECK: .size ilvr_v16i8_0
 }
 
-define void @ilvr_v8i16_0(<8 x i16>* %c, <8 x i16>* %a, <8 x i16>* %b) nounwind {
-  ; CHECK: ilvr_v8i16_0:
+define void @ilvl_v8i16_0(<8 x i16>* %c, <8 x i16>* %a, <8 x i16>* %b) nounwind {
+  ; CHECK-LABEL: ilvl_v8i16_0:
 
   %1 = load <8 x i16>, <8 x i16>* %a
   ; CHECK-DAG: ld.h [[R1:\$w[0-9]+]], 0($5)
   %2 = load <8 x i16>, <8 x i16>* %b
   ; CHECK-DAG: ld.h [[R2:\$w[0-9]+]], 0($6)
   %3 = shufflevector <8 x i16> %1, <8 x i16> %2, <8 x i32> <i32 4, i32 12, i32 5, i32 13, i32 6, i32 14, i32 7, i32 15>
-  ; CHECK-DAG: ilvr.h [[R3:\$w[0-9]+]], [[R1]], [[R2]]
+  ; CHECK-DAG: ilvl.h [[R3:\$w[0-9]+]], [[R2]], [[R1]]
   store <8 x i16> %3, <8 x i16>* %c
   ; CHECK-DAG: st.h [[R3]], 0($4)
 
   ret void
-  ; CHECK: .size ilvr_v8i16_0
 }
 
-define void @ilvr_v4i32_0(<4 x i32>* %c, <4 x i32>* %a, <4 x i32>* %b) nounwind {
-  ; CHECK: ilvr_v4i32_0:
+define void @ilvl_v4i32_0(<4 x i32>* %c, <4 x i32>* %a, <4 x i32>* %b) nounwind {
+  ; CHECK-LABEL: ilvl_v4i32_0:
 
   %1 = load <4 x i32>, <4 x i32>* %a
   ; CHECK-DAG: ld.w [[R1:\$w[0-9]+]], 0($5)
   %2 = load <4 x i32>, <4 x i32>* %b
   ; CHECK-DAG: ld.w [[R2:\$w[0-9]+]], 0($6)
   %3 = shufflevector <4 x i32> %1, <4 x i32> %2, <4 x i32> <i32 2, i32 6, i32 3, i32 7>
-  ; CHECK-DAG: ilvr.w [[R3:\$w[0-9]+]], [[R1]], [[R2]]
+  ; CHECK-DAG: ilvl.w [[R3:\$w[0-9]+]], [[R2]], [[R1]]
   store <4 x i32> %3, <4 x i32>* %c
   ; CHECK-DAG: st.w [[R3]], 0($4)
 
   ret void
-  ; CHECK: .size ilvr_v4i32_0
 }
 
-define void @ilvr_v2i64_0(<2 x i64>* %c, <2 x i64>* %a, <2 x i64>* %b) nounwind {
-  ; CHECK: ilvr_v2i64_0:
+define void @ilvl_v2i64_0(<2 x i64>* %c, <2 x i64>* %a, <2 x i64>* %b) nounwind {
+  ; CHECK-LABEL: ilvl_v2i64_0:
 
   %1 = load <2 x i64>, <2 x i64>* %a
   ; CHECK-DAG: ld.d [[R1:\$w[0-9]+]], 0($5)
   %2 = load <2 x i64>, <2 x i64>* %b
   ; CHECK-DAG: ld.d [[R2:\$w[0-9]+]], 0($6)
   %3 = shufflevector <2 x i64> %1, <2 x i64> %2, <2 x i32> <i32 1, i32 3>
-  ; ilvr.d and ilvod.d are equivalent for v2i64
+  ; ilvl.d and ilvod.d are equivalent for v2i64
   ; CHECK-DAG: ilvod.d [[R3:\$w[0-9]+]], [[R1]], [[R2]]
   store <2 x i64> %3, <2 x i64>* %c
   ; CHECK-DAG: st.d [[R3]], 0($4)
 
   ret void
-  ; CHECK: .size ilvr_v2i64_0
+}
+
+define void @ilvl_v16i8_1(<16 x i8>* %c, <16 x i8>* %a, <16 x i8>* %b) nounwind {
+  ; CHECK-LABEL: ilvl_v16i8_1:
+
+  %1 = load <16 x i8>, <16 x i8>* %a
+  %2 = load <16 x i8>, <16 x i8>* %b
+  ; CHECK-DAG: ld.b [[R2:\$w[0-9]+]], 0($6)
+  %3 = shufflevector <16 x i8> %1, <16 x i8> %2,
+                     <16 x i32> <i32 24, i32 24, i32 25, i32 25, i32 26, i32 26, i32 27, i32 27, i32 28, i32 28, i32 29, i32 29, i32 30, i32 30, i32 31, i32 31>
+  ; CHECK-DAG: ilvl.b [[R3:\$w[0-9]+]], [[R2]], [[R2]]
+  store <16 x i8> %3, <16 x i8>* %c
+  ; CHECK-DAG: st.b [[R3]], 0($4)
+
+  ret void
+}
+
+define void @ilvl_v8i16_1(<8 x i16>* %c, <8 x i16>* %a, <8 x i16>* %b) nounwind {
+  ; CHECK-LABEL: ilvl_v8i16_1:
+
+  %1 = load <8 x i16>, <8 x i16>* %a
+  %2 = load <8 x i16>, <8 x i16>* %b
+  ; CHECK-DAG: ld.h [[R2:\$w[0-9]+]], 0($6)
+  %3 = shufflevector <8 x i16> %1, <8 x i16> %2, <8 x i32> <i32 12, i32 12, i32 13, i32 13, i32 14, i32 14, i32 15, i32 15>
+  ; CHECK-DAG: ilvl.h [[R3:\$w[0-9]+]], [[R2]], [[R2]]
+  store <8 x i16> %3, <8 x i16>* %c
+  ; CHECK-DAG: st.h [[R3]], 0($4)
+
+  ret void
+}
+
+define void @ilvl_v4i32_1(<4 x i32>* %c, <4 x i32>* %a, <4 x i32>* %b) nounwind {
+  ; CHECK-LABEL: ilvl_v4i32_1:
+
+  %1 = load <4 x i32>, <4 x i32>* %a
+  %2 = load <4 x i32>, <4 x i32>* %b
+  ; CHECK-DAG: ld.w [[R2:\$w[0-9]+]], 0($6)
+  %3 = shufflevector <4 x i32> %1, <4 x i32> %2, <4 x i32> <i32 6, i32 6, i32 7, i32 7>
+  ; CHECK-DAG: ilvl.w [[R3:\$w[0-9]+]], [[R2]], [[R2]]
+  store <4 x i32> %3, <4 x i32>* %c
+  ; CHECK-DAG: st.w [[R3]], 0($4)
+
+  ret void
+}
+
+define void @ilvl_v2i64_1(<2 x i64>* %c, <2 x i64>* %a, <2 x i64>* %b) nounwind {
+  ; CHECK-LABEL: ilvl_v2i64_1:
+
+  %1 = load <2 x i64>, <2 x i64>* %a
+  %2 = load <2 x i64>, <2 x i64>* %b
+  ; CHECK-DAG: ld.d [[R2:\$w[0-9]+]], 0($6)
+  %3 = shufflevector <2 x i64> %1, <2 x i64> %2, <2 x i32> <i32 3, i32 3>
+  ; ilvl.d and splati.d are equivalent for v2i64
+  ; CHECK-DAG: splati.d [[R3:\$w[0-9]+]], [[R2]][1]
+  store <2 x i64> %3, <2 x i64>* %c
+  ; CHECK-DAG: st.d [[R3]], 0($4)
+
+  ret void
+}
+
+define void @ilvl_v16i8_2(<16 x i8>* %c, <16 x i8>* %a, <16 x i8>* %b) nounwind {
+  ; CHECK-LABEL: ilvl_v16i8_2:
+
+  %1 = load <16 x i8>, <16 x i8>* %a
+  ; CHECK-DAG: ld.b [[R1:\$w[0-9]+]], 0($5)
+  %2 = load <16 x i8>, <16 x i8>* %b
+  %3 = shufflevector <16 x i8> %1, <16 x i8> %2,
+                     <16 x i32> <i32 8, i32 8, i32 9, i32 9, i32 10, i32 10, i32 11, i32 11, i32 12, i32 12, i32 13, i32 13, i32 14, i32 14, i32 15, i32 15>
+  ; CHECK-DAG: ilvl.b [[R3:\$w[0-9]+]], [[R1]], [[R1]]
+  store <16 x i8> %3, <16 x i8>* %c
+  ; CHECK-DAG: st.b [[R3]], 0($4)
+
+  ret void
+}
+
+define void @ilvl_v8i16_2(<8 x i16>* %c, <8 x i16>* %a, <8 x i16>* %b) nounwind {
+  ; CHECK-LABEL: ilvl_v8i16_2:
+
+  %1 = load <8 x i16>, <8 x i16>* %a
+  ; CHECK-DAG: ld.h [[R1:\$w[0-9]+]], 0($5)
+  %2 = load <8 x i16>, <8 x i16>* %b
+  %3 = shufflevector <8 x i16> %1, <8 x i16> %2, <8 x i32> <i32 4, i32 4, i32 5, i32 5, i32 6, i32 6, i32 7, i32 7>
+  ; CHECK-DAG: ilvl.h [[R3:\$w[0-9]+]], [[R1]], [[R1]]
+  store <8 x i16> %3, <8 x i16>* %c
+  ; CHECK-DAG: st.h [[R3]], 0($4)
+
+  ret void
+}
+
+define void @ilvl_v4i32_2(<4 x i32>* %c, <4 x i32>* %a, <4 x i32>* %b) nounwind {
+  ; CHECK-LABEL: ilvl_v4i32_2:
+
+  %1 = load <4 x i32>, <4 x i32>* %a
+  ; CHECK-DAG: ld.w [[R1:\$w[0-9]+]], 0($5)
+  %2 = load <4 x i32>, <4 x i32>* %b
+  %3 = shufflevector <4 x i32> %1, <4 x i32> %2, <4 x i32> <i32 2, i32 2, i32 3, i32 3>
+  ; CHECK-DAG: ilvl.w [[R3:\$w[0-9]+]], [[R1]], [[R1]]
+  store <4 x i32> %3, <4 x i32>* %c
+  ; CHECK-DAG: st.w [[R3]], 0($4)
+
+  ret void
+}
+
+define void @ilvl_v2i64_2(<2 x i64>* %c, <2 x i64>* %a, <2 x i64>* %b) nounwind {
+  ; CHECK-LABEL: ilvl_v2i64_2:
+
+  %1 = load <2 x i64>, <2 x i64>* %a
+  ; CHECK-DAG: ld.d [[R1:\$w[0-9]+]], 0($5)
+  %2 = load <2 x i64>, <2 x i64>* %b
+  %3 = shufflevector <2 x i64> %1, <2 x i64> %2, <2 x i32> <i32 1, i32 1>
+  ; ilvl.d and splati.d are equivalent for v2i64
+  ; CHECK-DAG: splati.d [[R3:\$w[0-9]+]], [[R1]][1]
+  store <2 x i64> %3, <2 x i64>* %c
+  ; CHECK-DAG: st.d [[R3]], 0($4)
+
+  ret void
 }
 
 define void @pckev_v16i8_0(<16 x i8>* %c, <16 x i8>* %a, <16 x i8>* %b) nounwind {
-  ; CHECK: pckev_v16i8_0:
+  ; CHECK-LABEL: pckev_v16i8_0:
 
   %1 = load <16 x i8>, <16 x i8>* %a
   ; CHECK-DAG: ld.b [[R1:\$w[0-9]+]], 0($5)
@@ -639,48 +1066,45 @@ define void @pckev_v16i8_0(<16 x i8>* %c, <16 x i8>* %a, <16 x i8>* %b) nounwind
   ; CHECK-DAG: ld.b [[R2:\$w[0-9]+]], 0($6)
   %3 = shufflevector <16 x i8> %1, <16 x i8> %2,
                      <16 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14, i32 16, i32 18, i32 20, i32 22, i32 24, i32 26, i32 28, i32 30>
-  ; CHECK-DAG: pckev.b [[R3:\$w[0-9]+]], [[R1]], [[R2]]
+  ; CHECK-DAG: pckev.b [[R3:\$w[0-9]+]], [[R2]], [[R1]]
   store <16 x i8> %3, <16 x i8>* %c
   ; CHECK-DAG: st.b [[R3]], 0($4)
 
   ret void
-  ; CHECK: .size pckev_v16i8_0
 }
 
 define void @pckev_v8i16_0(<8 x i16>* %c, <8 x i16>* %a, <8 x i16>* %b) nounwind {
-  ; CHECK: pckev_v8i16_0:
+  ; CHECK-LABEL: pckev_v8i16_0:
 
   %1 = load <8 x i16>, <8 x i16>* %a
   ; CHECK-DAG: ld.h [[R1:\$w[0-9]+]], 0($5)
   %2 = load <8 x i16>, <8 x i16>* %b
   ; CHECK-DAG: ld.h [[R2:\$w[0-9]+]], 0($6)
   %3 = shufflevector <8 x i16> %1, <8 x i16> %2, <8 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14>
-  ; CHECK-DAG: pckev.h [[R3:\$w[0-9]+]], [[R1]], [[R2]]
+  ; CHECK-DAG: pckev.h [[R3:\$w[0-9]+]], [[R2]], [[R1]]
   store <8 x i16> %3, <8 x i16>* %c
   ; CHECK-DAG: st.h [[R3]], 0($4)
 
   ret void
-  ; CHECK: .size pckev_v8i16_0
 }
 
 define void @pckev_v4i32_0(<4 x i32>* %c, <4 x i32>* %a, <4 x i32>* %b) nounwind {
-  ; CHECK: pckev_v4i32_0:
+  ; CHECK-LABEL: pckev_v4i32_0:
 
   %1 = load <4 x i32>, <4 x i32>* %a
   ; CHECK-DAG: ld.w [[R1:\$w[0-9]+]], 0($5)
   %2 = load <4 x i32>, <4 x i32>* %b
   ; CHECK-DAG: ld.w [[R2:\$w[0-9]+]], 0($6)
   %3 = shufflevector <4 x i32> %1, <4 x i32> %2, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
-  ; CHECK-DAG: pckev.w [[R3:\$w[0-9]+]], [[R1]], [[R2]]
+  ; CHECK-DAG: pckev.w [[R3:\$w[0-9]+]], [[R2]], [[R1]]
   store <4 x i32> %3, <4 x i32>* %c
   ; CHECK-DAG: st.w [[R3]], 0($4)
 
   ret void
-  ; CHECK: .size pckev_v4i32_0
 }
 
 define void @pckev_v2i64_0(<2 x i64>* %c, <2 x i64>* %a, <2 x i64>* %b) nounwind {
-  ; CHECK: pckev_v2i64_0:
+  ; CHECK-LABEL: pckev_v2i64_0:
 
   %1 = load <2 x i64>, <2 x i64>* %a
   ; CHECK-DAG: ld.d [[R1:\$w[0-9]+]], 0($5)
@@ -688,16 +1112,131 @@ define void @pckev_v2i64_0(<2 x i64>* %c, <2 x i64>* %a, <2 x i64>* %b) nounwind
   ; CHECK-DAG: ld.d [[R2:\$w[0-9]+]], 0($6)
   %3 = shufflevector <2 x i64> %1, <2 x i64> %2, <2 x i32> <i32 0, i32 2>
   ; pckev.d and ilvev.d are equivalent for v2i64
-  ; CHECK-DAG: ilvev.d [[R3:\$w[0-9]+]], [[R1]], [[R2]]
+  ; CHECK-DAG: ilvev.d [[R3:\$w[0-9]+]], [[R2]], [[R1]]
+  store <2 x i64> %3, <2 x i64>* %c
+  ; CHECK-DAG: st.d [[R3]], 0($4)
+
+  ret void
+}
+
+define void @pckev_v16i8_1(<16 x i8>* %c, <16 x i8>* %a, <16 x i8>* %b) nounwind {
+  ; CHECK-LABEL: pckev_v16i8_1:
+
+  %1 = load <16 x i8>, <16 x i8>* %a
+  %2 = load <16 x i8>, <16 x i8>* %b
+  ; CHECK-DAG: ld.b [[R2:\$w[0-9]+]], 0($6)
+  %3 = shufflevector <16 x i8> %1, <16 x i8> %2,
+                     <16 x i32> <i32 16, i32 18, i32 20, i32 22, i32 24, i32 26, i32 28, i32 30, i32 16, i32 18, i32 20, i32 22, i32 24, i32 26, i32 28, i32 30>
+  ; CHECK-DAG: pckev.b [[R3:\$w[0-9]+]], [[R2]], [[R2]]
+  store <16 x i8> %3, <16 x i8>* %c
+  ; CHECK-DAG: st.b [[R3]], 0($4)
+
+  ret void
+}
+
+define void @pckev_v8i16_1(<8 x i16>* %c, <8 x i16>* %a, <8 x i16>* %b) nounwind {
+  ; CHECK-LABEL: pckev_v8i16_1:
+
+  %1 = load <8 x i16>, <8 x i16>* %a
+  %2 = load <8 x i16>, <8 x i16>* %b
+  ; CHECK-DAG: ld.h [[R2:\$w[0-9]+]], 0($6)
+  %3 = shufflevector <8 x i16> %1, <8 x i16> %2, <8 x i32> <i32 8, i32 10, i32 12, i32 14, i32 8, i32 10, i32 12, i32 14>
+  ; CHECK-DAG: pckev.h [[R3:\$w[0-9]+]], [[R2]], [[R2]]
+  store <8 x i16> %3, <8 x i16>* %c
+  ; CHECK-DAG: st.h [[R3]], 0($4)
+
+  ret void
+}
+
+define void @pckev_v4i32_1(<4 x i32>* %c, <4 x i32>* %a, <4 x i32>* %b) nounwind {
+  ; CHECK-LABEL: pckev_v4i32_1:
+
+  %1 = load <4 x i32>, <4 x i32>* %a
+  %2 = load <4 x i32>, <4 x i32>* %b
+  ; CHECK-DAG: ld.w [[R2:\$w[0-9]+]], 0($6)
+  %3 = shufflevector <4 x i32> %1, <4 x i32> %2, <4 x i32> <i32 4, i32 6, i32 4, i32 6>
+  ; CHECK-DAG: pckev.w [[R3:\$w[0-9]+]], [[R2]], [[R2]]
+  store <4 x i32> %3, <4 x i32>* %c
+  ; CHECK-DAG: st.w [[R3]], 0($4)
+
+  ret void
+}
+
+define void @pckev_v2i64_1(<2 x i64>* %c, <2 x i64>* %a, <2 x i64>* %b) nounwind {
+  ; CHECK-LABEL: pckev_v2i64_1:
+
+  %1 = load <2 x i64>, <2 x i64>* %a
+  %2 = load <2 x i64>, <2 x i64>* %b
+  ; CHECK-DAG: ld.d [[R2:\$w[0-9]+]], 0($6)
+  %3 = shufflevector <2 x i64> %1, <2 x i64> %2, <2 x i32> <i32 2, i32 2>
+  ; pckev.d and splati.d are equivalent for v2i64
+  ; CHECK-DAG: splati.d [[R3:\$w[0-9]+]], [[R2]][0]
+  store <2 x i64> %3, <2 x i64>* %c
+  ; CHECK-DAG: st.d [[R3]], 0($4)
+
+  ret void
+}
+
+define void @pckev_v16i8_2(<16 x i8>* %c, <16 x i8>* %a, <16 x i8>* %b) nounwind {
+  ; CHECK-LABEL: pckev_v16i8_2:
+
+  %1 = load <16 x i8>, <16 x i8>* %a
+  ; CHECK-DAG: ld.b [[R1:\$w[0-9]+]], 0($5)
+  %2 = load <16 x i8>, <16 x i8>* %b
+  %3 = shufflevector <16 x i8> %1, <16 x i8> %2,
+                     <16 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14, i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14>
+  ; CHECK-DAG: pckev.b [[R3:\$w[0-9]+]], [[R1]], [[R1]]
+  store <16 x i8> %3, <16 x i8>* %c
+  ; CHECK-DAG: st.b [[R3]], 0($4)
+
+  ret void
+}
+
+define void @pckev_v8i16_2(<8 x i16>* %c, <8 x i16>* %a, <8 x i16>* %b) nounwind {
+  ; CHECK-LABEL: pckev_v8i16_2:
+
+  %1 = load <8 x i16>, <8 x i16>* %a
+  ; CHECK-DAG: ld.h [[R1:\$w[0-9]+]], 0($5)
+  %2 = load <8 x i16>, <8 x i16>* %b
+  %3 = shufflevector <8 x i16> %1, <8 x i16> %2, <8 x i32> <i32 0, i32 2, i32 4, i32 6, i32 0, i32 2, i32 4, i32 6>
+  ; CHECK-DAG: pckev.h [[R3:\$w[0-9]+]], [[R1]], [[R1]]
+  store <8 x i16> %3, <8 x i16>* %c
+  ; CHECK-DAG: st.h [[R3]], 0($4)
+
+  ret void
+}
+
+define void @pckev_v4i32_2(<4 x i32>* %c, <4 x i32>* %a, <4 x i32>* %b) nounwind {
+  ; CHECK-LABEL: pckev_v4i32_2:
+
+  %1 = load <4 x i32>, <4 x i32>* %a
+  ; CHECK-DAG: ld.w [[R1:\$w[0-9]+]], 0($5)
+  %2 = load <4 x i32>, <4 x i32>* %b
+  %3 = shufflevector <4 x i32> %1, <4 x i32> %2, <4 x i32> <i32 0, i32 2, i32 0, i32 2>
+  ; CHECK-DAG: pckev.w [[R3:\$w[0-9]+]], [[R1]], [[R1]]
+  store <4 x i32> %3, <4 x i32>* %c
+  ; CHECK-DAG: st.w [[R3]], 0($4)
+
+  ret void
+}
+
+define void @pckev_v2i64_2(<2 x i64>* %c, <2 x i64>* %a, <2 x i64>* %b) nounwind {
+  ; CHECK-LABEL: pckev_v2i64_2:
+
+  %1 = load <2 x i64>, <2 x i64>* %a
+  ; CHECK-DAG: ld.d [[R1:\$w[0-9]+]], 0($5)
+  %2 = load <2 x i64>, <2 x i64>* %b
+  %3 = shufflevector <2 x i64> %1, <2 x i64> %2, <2 x i32> <i32 0, i32 0>
+  ; pckev.d and splati.d are equivalent for v2i64
+  ; CHECK-DAG: splati.d [[R3:\$w[0-9]+]], [[R1]][0]
   store <2 x i64> %3, <2 x i64>* %c
   ; CHECK-DAG: st.d [[R3]], 0($4)
 
   ret void
-  ; CHECK: .size pckev_v2i64_0
 }
 
 define void @pckod_v16i8_0(<16 x i8>* %c, <16 x i8>* %a, <16 x i8>* %b) nounwind {
-  ; CHECK: pckod_v16i8_0:
+  ; CHECK-LABEL: pckod_v16i8_0:
 
   %1 = load <16 x i8>, <16 x i8>* %a
   ; CHECK-DAG: ld.b [[R1:\$w[0-9]+]], 0($5)
@@ -705,48 +1244,45 @@ define void @pckod_v16i8_0(<16 x i8>* %c, <16 x i8>* %a, <16 x i8>* %b) nounwind
   ; CHECK-DAG: ld.b [[R2:\$w[0-9]+]], 0($6)
   %3 = shufflevector <16 x i8> %1, <16 x i8> %2,
                      <16 x i32> <i32 1, i32 3, i32 5, i32 7, i32 9, i32 11, i32 13, i32 15, i32 17, i32 19, i32 21, i32 23, i32 25, i32 27, i32 29, i32 31>
-  ; CHECK-DAG: pckod.b [[R3:\$w[0-9]+]], [[R1]], [[R2]]
+  ; CHECK-DAG: pckod.b [[R3:\$w[0-9]+]], [[R2]], [[R1]]
   store <16 x i8> %3, <16 x i8>* %c
   ; CHECK-DAG: st.b [[R3]], 0($4)
 
   ret void
-  ; CHECK: .size pckod_v16i8_0
 }
 
 define void @pckod_v8i16_0(<8 x i16>* %c, <8 x i16>* %a, <8 x i16>* %b) nounwind {
-  ; CHECK: pckod_v8i16_0:
+  ; CHECK-LABEL: pckod_v8i16_0:
 
   %1 = load <8 x i16>, <8 x i16>* %a
   ; CHECK-DAG: ld.h [[R1:\$w[0-9]+]], 0($5)
   %2 = load <8 x i16>, <8 x i16>* %b
   ; CHECK-DAG: ld.h [[R2:\$w[0-9]+]], 0($6)
   %3 = shufflevector <8 x i16> %1, <8 x i16> %2, <8 x i32> <i32 1, i32 3, i32 5, i32 7, i32 9, i32 11, i32 13, i32 15>
-  ; CHECK-DAG: pckod.h [[R3:\$w[0-9]+]], [[R1]], [[R2]]
+  ; CHECK-DAG: pckod.h [[R3:\$w[0-9]+]], [[R2]], [[R1]]
   store <8 x i16> %3, <8 x i16>* %c
   ; CHECK-DAG: st.h [[R3]], 0($4)
 
   ret void
-  ; CHECK: .size pckod_v8i16_0
 }
 
 define void @pckod_v4i32_0(<4 x i32>* %c, <4 x i32>* %a, <4 x i32>* %b) nounwind {
-  ; CHECK: pckod_v4i32_0:
+  ; CHECK-LABEL: pckod_v4i32_0:
 
   %1 = load <4 x i32>, <4 x i32>* %a
   ; CHECK-DAG: ld.w [[R1:\$w[0-9]+]], 0($5)
   %2 = load <4 x i32>, <4 x i32>* %b
   ; CHECK-DAG: ld.w [[R2:\$w[0-9]+]], 0($6)
   %3 = shufflevector <4 x i32> %1, <4 x i32> %2, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
-  ; CHECK-DAG: pckod.w [[R3:\$w[0-9]+]], [[R1]], [[R2]]
+  ; CHECK-DAG: pckod.w [[R3:\$w[0-9]+]], [[R2]], [[R1]]
   store <4 x i32> %3, <4 x i32>* %c
   ; CHECK-DAG: st.w [[R3]], 0($4)
 
   ret void
-  ; CHECK: .size pckod_v4i32_0
 }
 
 define void @pckod_v2i64_0(<2 x i64>* %c, <2 x i64>* %a, <2 x i64>* %b) nounwind {
-  ; CHECK: pckod_v2i64_0:
+  ; CHECK-LABEL: pckod_v2i64_0:
 
   %1 = load <2 x i64>, <2 x i64>* %a
   ; CHECK-DAG: ld.d [[R1:\$w[0-9]+]], 0($5)
@@ -759,11 +1295,126 @@ define void @pckod_v2i64_0(<2 x i64>* %c, <2 x i64>* %a, <2 x i64>* %b) nounwind
   ; CHECK-DAG: st.d [[R3]], 0($4)
 
   ret void
-  ; CHECK: .size pckod_v2i64_0
+}
+
+define void @pckod_v16i8_1(<16 x i8>* %c, <16 x i8>* %a, <16 x i8>* %b) nounwind {
+  ; CHECK-LABEL: pckod_v16i8_1:
+
+  %1 = load <16 x i8>, <16 x i8>* %a
+  %2 = load <16 x i8>, <16 x i8>* %b
+  ; CHECK-DAG: ld.b [[R2:\$w[0-9]+]], 0($6)
+  %3 = shufflevector <16 x i8> %1, <16 x i8> %2,
+                     <16 x i32> <i32 17, i32 19, i32 21, i32 23, i32 25, i32 27, i32 29, i32 31, i32 17, i32 19, i32 21, i32 23, i32 25, i32 27, i32 29, i32 31>
+  ; CHECK-DAG: pckod.b [[R3:\$w[0-9]+]], [[R2]], [[R2]]
+  store <16 x i8> %3, <16 x i8>* %c
+  ; CHECK-DAG: st.b [[R3]], 0($4)
+
+  ret void
+}
+
+define void @pckod_v8i16_1(<8 x i16>* %c, <8 x i16>* %a, <8 x i16>* %b) nounwind {
+  ; CHECK-LABEL: pckod_v8i16_1:
+
+  %1 = load <8 x i16>, <8 x i16>* %a
+  %2 = load <8 x i16>, <8 x i16>* %b
+  ; CHECK-DAG: ld.h [[R2:\$w[0-9]+]], 0($6)
+  %3 = shufflevector <8 x i16> %1, <8 x i16> %2, <8 x i32> <i32 9, i32 11, i32 13, i32 15, i32 9, i32 11, i32 13, i32 15>
+  ; CHECK-DAG: pckod.h [[R3:\$w[0-9]+]], [[R2]], [[R2]]
+  store <8 x i16> %3, <8 x i16>* %c
+  ; CHECK-DAG: st.h [[R3]], 0($4)
+
+  ret void
+}
+
+define void @pckod_v4i32_1(<4 x i32>* %c, <4 x i32>* %a, <4 x i32>* %b) nounwind {
+  ; CHECK-LABEL: pckod_v4i32_1:
+
+  %1 = load <4 x i32>, <4 x i32>* %a
+  %2 = load <4 x i32>, <4 x i32>* %b
+  ; CHECK-DAG: ld.w [[R2:\$w[0-9]+]], 0($6)
+  %3 = shufflevector <4 x i32> %1, <4 x i32> %2, <4 x i32> <i32 5, i32 7, i32 5, i32 7>
+  ; CHECK-DAG: pckod.w [[R3:\$w[0-9]+]], [[R2]], [[R2]]
+  store <4 x i32> %3, <4 x i32>* %c
+  ; CHECK-DAG: st.w [[R3]], 0($4)
+
+  ret void
+}
+
+define void @pckod_v2i64_1(<2 x i64>* %c, <2 x i64>* %a, <2 x i64>* %b) nounwind {
+  ; CHECK-LABEL: pckod_v2i64_1:
+
+  %1 = load <2 x i64>, <2 x i64>* %a
+  %2 = load <2 x i64>, <2 x i64>* %b
+  ; CHECK-DAG: ld.d [[R2:\$w[0-9]+]], 0($6)
+  %3 = shufflevector <2 x i64> %1, <2 x i64> %2, <2 x i32> <i32 3, i32 3>
+  ; pckod.d and splati.d are equivalent for v2i64
+  ; CHECK-DAG: splati.d [[R3:\$w[0-9]+]], [[R2]][1]
+  store <2 x i64> %3, <2 x i64>* %c
+  ; CHECK-DAG: st.d [[R3]], 0($4)
+
+  ret void
+}
+
+define void @pckod_v16i8_2(<16 x i8>* %c, <16 x i8>* %a, <16 x i8>* %b) nounwind {
+  ; CHECK-LABEL: pckod_v16i8_2:
+
+  %1 = load <16 x i8>, <16 x i8>* %a
+  ; CHECK-DAG: ld.b [[R1:\$w[0-9]+]], 0($5)
+  %2 = load <16 x i8>, <16 x i8>* %b
+  %3 = shufflevector <16 x i8> %1, <16 x i8> %2,
+                     <16 x i32> <i32 1, i32 3, i32 5, i32 7, i32 9, i32 11, i32 13, i32 15, i32 1, i32 3, i32 5, i32 7, i32 9, i32 11, i32 13, i32 15>
+  ; CHECK-DAG: pckod.b [[R3:\$w[0-9]+]], [[R1]], [[R1]]
+  store <16 x i8> %3, <16 x i8>* %c
+  ; CHECK-DAG: st.b [[R3]], 0($4)
+
+  ret void
+}
+
+define void @pckod_v8i16_2(<8 x i16>* %c, <8 x i16>* %a, <8 x i16>* %b) nounwind {
+  ; CHECK-LABEL: pckod_v8i16_2:
+
+  %1 = load <8 x i16>, <8 x i16>* %a
+  ; CHECK-DAG: ld.h [[R1:\$w[0-9]+]], 0($5)
+  %2 = load <8 x i16>, <8 x i16>* %b
+  %3 = shufflevector <8 x i16> %1, <8 x i16> %2, <8 x i32> <i32 1, i32 3, i32 5, i32 7, i32 1, i32 3, i32 5, i32 7>
+  ; CHECK-DAG: pckod.h [[R3:\$w[0-9]+]], [[R1]], [[R1]]
+  store <8 x i16> %3, <8 x i16>* %c
+  ; CHECK-DAG: st.h [[R3]], 0($4)
+
+  ret void
+}
+
+define void @pckod_v4i32_2(<4 x i32>* %c, <4 x i32>* %a, <4 x i32>* %b) nounwind {
+  ; CHECK-LABEL: pckod_v4i32_2:
+
+  %1 = load <4 x i32>, <4 x i32>* %a
+  ; CHECK-DAG: ld.w [[R1:\$w[0-9]+]], 0($5)
+  %2 = load <4 x i32>, <4 x i32>* %b
+  %3 = shufflevector <4 x i32> %1, <4 x i32> %2, <4 x i32> <i32 1, i32 3, i32 1, i32 3>
+  ; CHECK-DAG: pckod.w [[R3:\$w[0-9]+]], [[R1]], [[R1]]
+  store <4 x i32> %3, <4 x i32>* %c
+  ; CHECK-DAG: st.w [[R3]], 0($4)
+
+  ret void
+}
+
+define void @pckod_v2i64_2(<2 x i64>* %c, <2 x i64>* %a, <2 x i64>* %b) nounwind {
+  ; CHECK-LABEL: pckod_v2i64_2:
+
+  %1 = load <2 x i64>, <2 x i64>* %a
+  ; CHECK-DAG: ld.d [[R1:\$w[0-9]+]], 0($5)
+  %2 = load <2 x i64>, <2 x i64>* %b
+  %3 = shufflevector <2 x i64> %1, <2 x i64> %2, <2 x i32> <i32 1, i32 1>
+  ; pckod.d and splati.d are equivalent for v2i64
+  ; CHECK-DAG: splati.d [[R3:\$w[0-9]+]], [[R1]][1]
+  store <2 x i64> %3, <2 x i64>* %c
+  ; CHECK-DAG: st.d [[R3]], 0($4)
+
+  ret void
 }
 
 define void @splati_v16i8_0(<16 x i8>* %c, <16 x i8>* %a) nounwind {
-  ; CHECK: splati_v16i8_0:
+  ; CHECK-LABEL: splati_v16i8_0:
 
   %1 = load <16 x i8>, <16 x i8>* %a
   ; CHECK-DAG: ld.b [[R1:\$w[0-9]+]], 0($5)
@@ -774,11 +1425,10 @@ define void @splati_v16i8_0(<16 x i8>* %c, <16 x i8>* %a) nounwind {
   ; CHECK-DAG: st.b [[R3]], 0($4)
 
   ret void
-  ; CHECK: .size splati_v16i8_0
 }
 
 define void @splati_v8i16_0(<8 x i16>* %c, <8 x i16>* %a) nounwind {
-  ; CHECK: splati_v8i16_0:
+  ; CHECK-LABEL: splati_v8i16_0:
 
   %1 = load <8 x i16>, <8 x i16>* %a
   ; CHECK-DAG: ld.h [[R1:\$w[0-9]+]], 0($5)
@@ -788,26 +1438,23 @@ define void @splati_v8i16_0(<8 x i16>* %c, <8 x i16>* %a) nounwind {
   ; CHECK-DAG: st.h [[R3]], 0($4)
 
   ret void
-  ; CHECK: .size splati_v8i16_0
 }
 
 define void @splati_v4i32_0(<4 x i32>* %c, <4 x i32>* %a) nounwind {
-  ; CHECK: splati_v4i32_0:
+  ; CHECK-LABEL: splati_v4i32_0:
 
   %1 = load <4 x i32>, <4 x i32>* %a
   ; CHECK-DAG: ld.w [[R1:\$w[0-9]+]], 0($5)
   %2 = shufflevector <4 x i32> %1, <4 x i32> undef, <4 x i32> <i32 3, i32 3, i32 3, i32 3>
-  ; shf.w and splati.w are equivalent
-  ; CHECK-DAG: shf.w [[R3:\$w[0-9]+]], [[R1]], 255
+  ; CHECK-DAG: splati.w [[R3:\$w[0-9]+]], [[R1]][3]
   store <4 x i32> %2, <4 x i32>* %c
   ; CHECK-DAG: st.w [[R3]], 0($4)
 
   ret void
-  ; CHECK: .size splati_v4i32_0
 }
 
 define void @splati_v2i64_0(<2 x i64>* %c, <2 x i64>* %a) nounwind {
-  ; CHECK: splati_v2i64_0:
+  ; CHECK-LABEL: splati_v2i64_0:
 
   %1 = load <2 x i64>, <2 x i64>* %a
   ; CHECK-DAG: ld.d [[R1:\$w[0-9]+]], 0($5)
@@ -817,5 +1464,4 @@ define void @splati_v2i64_0(<2 x i64>* %c, <2 x i64>* %a) nounwind {
   ; CHECK-DAG: st.d [[R3]], 0($4)
 
   ret void
-  ; CHECK: .size splati_v2i64_0
 }
diff --git a/test/CodeGen/Mips/sel1c.ll b/test/CodeGen/Mips/sel1c.ll
index 991228d71fa..6753af106e0 100644
--- a/test/CodeGen/Mips/sel1c.ll
+++ b/test/CodeGen/Mips/sel1c.ll
@@ -1,4 +1,4 @@
-; RUN: llc -mtriple=mipsel-linux-gnu -march=mipsel -mcpu=mips16 -soft-float -mips16-hard-float -relocation-model=pic -mips16-constant-islands   < %s | FileCheck %s -check-prefix=cond-b-short
+; RUN: llc -mtriple=mipsel-linux-gnu -march=mipsel -mcpu=mips16 -mattr=+soft-float -mips16-hard-float -relocation-model=pic -mips16-constant-islands   < %s | FileCheck %s -check-prefix=cond-b-short
 
 @i = global i32 1, align 4
 @j = global i32 2, align 4
diff --git a/test/CodeGen/Mips/sel2c.ll b/test/CodeGen/Mips/sel2c.ll
index ba95a518175..987cccad5bf 100644
--- a/test/CodeGen/Mips/sel2c.ll
+++ b/test/CodeGen/Mips/sel2c.ll
@@ -1,4 +1,4 @@
-; RUN: llc -mtriple=mipsel-linux-gnu -march=mipsel -mcpu=mips16 -soft-float -mips16-hard-float -relocation-model=pic -mips16-constant-islands   < %s | FileCheck %s -check-prefix=cond-b-short
+; RUN: llc -mtriple=mipsel-linux-gnu -march=mipsel -mcpu=mips16 -mattr=+soft-float -mips16-hard-float -relocation-model=pic -mips16-constant-islands   < %s | FileCheck %s -check-prefix=cond-b-short
 
 @i = global i32 1, align 4
 @j = global i32 2, align 4
diff --git a/test/CodeGen/Mips/simplebr.ll b/test/CodeGen/Mips/simplebr.ll
index 08e153a8c6b..2aeacc903fb 100644
--- a/test/CodeGen/Mips/simplebr.ll
+++ b/test/CodeGen/Mips/simplebr.ll
@@ -1,4 +1,4 @@
-; RUN: llc -mtriple=mipsel-linux-gnu -march=mipsel -mcpu=mips16 -mips16-hard-float -soft-float -relocation-model=static < %s | FileCheck %s -check-prefix=CHECK-STATIC16
+; RUN: llc -mtriple=mipsel-linux-gnu -march=mipsel -mcpu=mips16 -mips16-hard-float -mattr=+soft-float -relocation-model=static < %s | FileCheck %s -check-prefix=CHECK-STATIC16
 
 ; ModuleID = 'simplebr.c'
 target datalayout = "E-p:32:32:32-i1:8:8-i8:8:32-i16:16:32-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-n32-S64"
diff --git a/test/CodeGen/Mips/tail16.ll b/test/CodeGen/Mips/tail16.ll
index 4e62e557478..13f27fcc513 100644
--- a/test/CodeGen/Mips/tail16.ll
+++ b/test/CodeGen/Mips/tail16.ll
@@ -1,4 +1,4 @@
-; RUN: llc -mtriple=mipsel-linux-gnu -march=mipsel -mcpu=mips16 -soft-float -mips16-hard-float -relocation-model=pic   < %s | FileCheck %s 
+; RUN: llc -mtriple=mipsel-linux-gnu -march=mipsel -mcpu=mips16 -mattr=+soft-float -mips16-hard-float -relocation-model=pic   < %s | FileCheck %s 
 
 ; Function Attrs: nounwind optsize
 define float @h()  {
diff --git a/test/CodeGen/NVPTX/access-non-generic.ll b/test/CodeGen/NVPTX/access-non-generic.ll
index e709302918f..c1327274a9c 100644
--- a/test/CodeGen/NVPTX/access-non-generic.ll
+++ b/test/CodeGen/NVPTX/access-non-generic.ll
@@ -85,6 +85,44 @@ define i32 @ld_int_from_float() {
   ret i32 %1
 }
 
+define i32 @ld_int_from_global_float(float addrspace(1)* %input, i32 %i, i32 %j) {
+; IR-LABEL: @ld_int_from_global_float(
+; PTX-LABEL: ld_int_from_global_float(
+  %1 = addrspacecast float addrspace(1)* %input to float*
+  %2 = getelementptr float, float* %1, i32 %i
+; IR-NEXT: getelementptr float, float addrspace(1)* %input, i32 %i
+  %3 = getelementptr float, float* %2, i32 %j
+; IR-NEXT: getelementptr float, float addrspace(1)* {{%[^,]+}}, i32 %j
+  %4 = bitcast float* %3 to i32*
+; IR-NEXT: bitcast float addrspace(1)* {{%[^ ]+}} to i32 addrspace(1)*
+  %5 = load i32, i32* %4
+; IR-NEXT: load i32, i32 addrspace(1)* {{%.+}}
+; PTX-LABEL: ld.global
+  ret i32 %5
+}
+
+define void @nested_const_expr() {
+; PTX-LABEL: nested_const_expr(
+  ; store 1 to bitcast(gep(addrspacecast(array), 0, 1))
+  store i32 1, i32* bitcast (float* getelementptr ([10 x float], [10 x float]* addrspacecast ([10 x float] addrspace(3)* @array to [10 x float]*), i64 0, i64 1) to i32*), align 4
+; PTX: mov.u32 %r1, 1;
+; PTX-NEXT: st.shared.u32 [array+4], %r1;
+  ret void
+}
+
+define void @rauw(float addrspace(1)* %input) {
+  %generic_input = addrspacecast float addrspace(1)* %input to float*
+  %addr = getelementptr float, float* %generic_input, i64 10
+  %v = load float, float* %addr
+  store float %v, float* %addr
+  ret void
+; IR-LABEL: @rauw(
+; IR-NEXT: %1 = getelementptr float, float addrspace(1)* %input, i64 10
+; IR-NEXT: %v = load float, float addrspace(1)* %1
+; IR-NEXT: store float %v, float addrspace(1)* %1
+; IR-NEXT: ret void
+}
+
 declare void @llvm.cuda.syncthreads() #3
 
 attributes #3 = { noduplicate nounwind }
diff --git a/test/CodeGen/NVPTX/addrspacecast-gvar.ll b/test/CodeGen/NVPTX/addrspacecast-gvar.ll
index 6afbdb8a429..1e2fde4b858 100644
--- a/test/CodeGen/NVPTX/addrspacecast-gvar.ll
+++ b/test/CodeGen/NVPTX/addrspacecast-gvar.ll
@@ -3,7 +3,11 @@
 ; CHECK: .visible .global .align 4 .u32 g = 42;
 ; CHECK: .visible .global .align 4 .u32 g2 = generic(g);
 ; CHECK: .visible .global .align 4 .u32 g3 = g;
+; CHECK: .visible .global .align 8 .u32 g4[2] = {0, generic(g)};
+; CHECK: .visible .global .align 8 .u32 g5[2] = {0, generic(g)+8};
 
 @g = addrspace(1) global i32 42
 @g2 = addrspace(1) global i32* addrspacecast (i32 addrspace(1)* @g to i32*)
 @g3 = addrspace(1) global i32 addrspace(1)* @g
+@g4 = constant {i32*, i32*} {i32* null, i32* addrspacecast (i32 addrspace(1)* @g to i32*)}
+@g5 = constant {i32*, i32*} {i32* null, i32* addrspacecast (i32 addrspace(1)* getelementptr (i32, i32 addrspace(1)* @g, i32 2) to i32*)}
diff --git a/test/CodeGen/NVPTX/branch-fold.ll b/test/CodeGen/NVPTX/branch-fold.ll
new file mode 100644
index 00000000000..2b9cd0a35d9
--- /dev/null
+++ b/test/CodeGen/NVPTX/branch-fold.ll
@@ -0,0 +1,40 @@
+; RUN: llc < %s -march=nvptx64 -mcpu=sm_35 -disable-cgp | FileCheck %s
+; Disable CGP which also folds branches, so that only BranchFolding is under
+; the spotlight.
+
+target datalayout = "e-i64:64-v16:16-v32:32-n16:32:64"
+target triple = "nvptx64-nvidia-cuda"
+
+define void @foo(i32 %x, float* %output) {
+; CHECK-LABEL: .visible .func foo(
+; CHECK-NOT: bra.uni
+; CHECK-NOT: LBB0_
+  %1 = icmp eq i32 %x, 1
+  br i1 %1, label %then, label %else
+
+then:
+  br label %merge
+
+else:
+  br label %merge
+
+merge:
+  store float 2.0, float* %output
+  ret void
+}
+
+; PR24299. no crash
+define ptx_kernel void @hoge() #0 {
+; CHECK-LABEL: .visible .entry hoge(
+bb:
+  br i1 undef, label %bb1, label %bb4
+
+bb1:                                              ; preds = %bb1, %bb
+  %tmp = phi i64 [ %tmp2, %bb1 ], [ undef, %bb ]
+  %tmp2 = add nsw i64 %tmp, 1
+  %tmp3 = icmp sle i64 %tmp, 0
+  br i1 %tmp3, label %bb1, label %bb4
+
+bb4:                                              ; preds = %bb4, %bb1, %bb
+  br label %bb4
+}
diff --git a/test/CodeGen/NVPTX/bug21465.ll b/test/CodeGen/NVPTX/bug21465.ll
index 76af386c651..2eae41f73a0 100644
--- a/test/CodeGen/NVPTX/bug21465.ll
+++ b/test/CodeGen/NVPTX/bug21465.ll
@@ -1,4 +1,5 @@
-; RUN: opt < %s -nvptx-lower-struct-args -S | FileCheck %s
+; RUN: opt < %s -nvptx-lower-kernel-args -S | FileCheck %s
+; RUN: llc < %s -march=nvptx64 -mcpu=sm_35 | FileCheck %s --check-prefix PTX
 
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v16:16:16-v32:32:32-v64:64:64-v128:128:128-n16:32:64"
 target triple = "nvptx64-unknown-unknown"
@@ -8,12 +9,15 @@ target triple = "nvptx64-unknown-unknown"
 ; Function Attrs: nounwind
 define void @_Z11TakesStruct1SPi(%struct.S* byval nocapture readonly %input, i32* nocapture %output) #0 {
 entry:
-; CHECK-LABEL @_Z22TakesStruct1SPi
-; CHECK:   bitcast %struct.S* %input to i8*
-; CHECK:   call i8 addrspace(101)* @llvm.nvvm.ptr.gen.to.param.p101i8.p0i8
+; CHECK-LABEL: @_Z11TakesStruct1SPi
+; PTX-LABEL: .visible .entry _Z11TakesStruct1SPi(
+; CHECK: addrspacecast %struct.S* %input to %struct.S addrspace(101)*
   %b = getelementptr inbounds %struct.S, %struct.S* %input, i64 0, i32 1
   %0 = load i32, i32* %b, align 4
+; PTX-NOT: ld.param.u32 {{%r[0-9]+}}, [{{%rd[0-9]+}}]
+; PTX: ld.param.u32 [[value:%r[0-9]+]], [{{%rd[0-9]+}}+4]
   store i32 %0, i32* %output, align 4
+; PTX-NEXT: st.global.u32 [{{%rd[0-9]+}}], [[value]]
   ret void
 }
 
diff --git a/test/CodeGen/NVPTX/call-with-alloca-buffer.ll b/test/CodeGen/NVPTX/call-with-alloca-buffer.ll
index 58b19112991..7ca31bbbaf2 100644
--- a/test/CodeGen/NVPTX/call-with-alloca-buffer.ll
+++ b/test/CodeGen/NVPTX/call-with-alloca-buffer.ll
@@ -20,12 +20,15 @@ entry:
   %buf = alloca [16 x i8], align 4
 
 ; CHECK: .local .align 4 .b8 	__local_depot0[16]
-; CHECK: mov.u64 %rd[[BUF_REG:[0-9]+]]
-; CHECK: cvta.local.u64 %SP, %rd[[BUF_REG]]
+; CHECK: mov.u64 %SPL
 
 ; CHECK: ld.param.u64 %rd[[A_REG:[0-9]+]], [kernel_func_param_0]
-; CHECK: ld.f32 %f[[A0_REG:[0-9]+]], [%rd[[A_REG]]]
-; CHECK: st.f32 [%SP+0], %f[[A0_REG]]
+; CHECK: cvta.to.global.u64 %rd[[A1_REG:[0-9]+]], %rd[[A_REG]]
+; FIXME: casting A1_REG to A2_REG is unnecessary; A2_REG is essentially A_REG
+; CHECK: cvta.global.u64 %rd[[A2_REG:[0-9]+]], %rd[[A1_REG]]
+; CHECK: cvta.local.u64 %rd[[SP_REG:[0-9]+]]
+; CHECK: ld.global.f32 %f[[A0_REG:[0-9]+]], [%rd[[A1_REG]]]
+; CHECK: st.local.f32 [{{%rd[0-9]+}}], %f[[A0_REG]]
 
   %0 = load float, float* %a, align 4
   %1 = bitcast [16 x i8]* %buf to float*
@@ -46,9 +49,8 @@ entry:
   %7 = bitcast i8* %arrayidx7 to float*
   store float %6, float* %7, align 4
 
-; CHECK: add.u64 %rd[[SP_REG:[0-9]+]], %SP, 0
 ; CHECK:        .param .b64 param0;
-; CHECK-NEXT:   st.param.b64  [param0+0], %rd[[A_REG]]
+; CHECK-NEXT:   st.param.b64  [param0+0], %rd[[A2_REG]]
 ; CHECK-NEXT:   .param .b64 param1;
 ; CHECK-NEXT:   st.param.b64  [param1+0], %rd[[SP_REG]]
 ; CHECK-NEXT:   call.uni
diff --git a/test/CodeGen/NVPTX/extloadv.ll b/test/CodeGen/NVPTX/extloadv.ll
new file mode 100644
index 00000000000..8c264ae0933
--- /dev/null
+++ b/test/CodeGen/NVPTX/extloadv.ll
@@ -0,0 +1,15 @@
+; RUN: llc < %s -march=nvptx64 -mcpu=sm_35 | FileCheck %s
+
+define void @foo(float* nocapture readonly %x_value, double* nocapture %output) #0 {
+  %1 = bitcast float* %x_value to <4 x float>*
+  %2 = load <4 x float>, <4 x float>* %1, align 16
+  %3 = fpext <4 x float> %2 to <4 x double>
+; CHECK-NOT: ld.v2.f32 {%fd{{[0-9]+}}, %fd{{[0-9]+}}}, [%rd{{[0-9]+}}];
+; CHECK:  cvt.f64.f32
+; CHECK:  cvt.f64.f32
+; CHECK:  cvt.f64.f32
+; CHECK:  cvt.f64.f32
+  %4 = bitcast double* %output to <4 x double>*
+  store <4 x double> %3, <4 x double>* %4
+  ret void
+}
diff --git a/test/CodeGen/NVPTX/globals_init.ll b/test/CodeGen/NVPTX/globals_init.ll
new file mode 100644
index 00000000000..5b45f410156
--- /dev/null
+++ b/test/CodeGen/NVPTX/globals_init.ll
@@ -0,0 +1,23 @@
+; RUN: llc < %s -march=nvptx -mcpu=sm_20 | FileCheck %s
+
+; Make sure the globals constant initializers are not prone to host endianess 
+; issues.
+
+; CHECK-DAG: .b8 Gbli08[2] = {171, 205};
+@Gbli08 = global [2 x i8] [i8 171, i8 205]
+
+; CHECK-DAG: .b8 Gbli16[4] = {205, 171, 1, 239};
+@Gbli16 = global [2 x i16] [i16 43981, i16 61185]
+
+; CHECK-DAG: .b8 Gbli32[8] = {1, 239, 205, 171, 137, 103, 69, 35};
+@Gbli32 = global [2 x i32] [i32 2882400001, i32 591751049]
+
+; CHECK-DAG: .b8 Gbli64[16] = {137, 103, 69, 35, 1, 239, 205, 171, 239, 205, 171, 137, 103, 69, 35, 1};
+@Gbli64 = global [2 x i64] [i64 12379813738877118345, i64 81985529216486895]
+
+; CHECK-DAG: .b8 Gblf32[8] = {192, 225, 100, 75, 0, 96, 106, 69};
+@Gblf32 = global [2 x float] [float 1.5e+7, float 3.75e+3]
+
+; CHECK-DAG: .b8 Gblf64[16] = {116, 10, 181, 48, 134, 62, 230, 58, 106, 222, 138, 98, 204, 250, 200, 75};
+@Gblf64 = global [2 x double] [double 5.75e-25, double 12.25e+56]
+
diff --git a/test/CodeGen/NVPTX/globals_lowering.ll b/test/CodeGen/NVPTX/globals_lowering.ll
new file mode 100644
index 00000000000..84c61ef4033
--- /dev/null
+++ b/test/CodeGen/NVPTX/globals_lowering.ll
@@ -0,0 +1,15 @@
+; RUN: llc < %s -march=nvptx -mcpu=sm_20 -relocation-model=static | FileCheck %s --check-prefix CHK
+
+%MyStruct = type { i32, i32, float }
+@Gbl = internal addrspace(3) global [1024 x %MyStruct] zeroinitializer
+
+; CHK-LABEL: foo
+define void @foo(float %f) {
+entry:
+  ; CHK: ld.shared.f32  %{{[a-zA-Z0-9]+}}, [Gbl+8];
+  %0 = load float, float addrspace(3)* getelementptr inbounds ([1024 x %MyStruct], [1024 x %MyStruct] addrspace(3)* @Gbl, i32 0, i32 0, i32 2)
+  %add = fadd float %0, %f
+  ; CHK: st.shared.f32   [Gbl+8], %{{[a-zA-Z0-9]+}};
+  store float %add, float addrspace(3)* getelementptr inbounds ([1024 x %MyStruct], [1024 x %MyStruct] addrspace(3)* @Gbl, i32 0, i32 0, i32 2)
+  ret void
+}
diff --git a/test/CodeGen/NVPTX/intrin-nocapture.ll b/test/CodeGen/NVPTX/intrin-nocapture.ll
index 55781bb15a0..2dbd29f616f 100644
--- a/test/CodeGen/NVPTX/intrin-nocapture.ll
+++ b/test/CodeGen/NVPTX/intrin-nocapture.ll
@@ -11,7 +11,7 @@ declare i32 addrspace(1)* @llvm.nvvm.ptr.gen.to.global.p1i32.p0i32(i32*)
 ; CHECK: @bar
 define void @bar() {
   %t1 = alloca i32
-; CHECK: call i32 addrspace(1)* @llvm.nvvm.ptr.gen.to.global.p1i32.p0i32(i32* %t1)
+; CHECK: call i32 addrspace(1)* @llvm.nvvm.ptr.gen.to.global.p1i32.p0i32(i32* nonnull %t1)
 ; CHECK-NEXT: store i32 10, i32* %t1
   %t2 = call i32 addrspace(1)* @llvm.nvvm.ptr.gen.to.global.p1i32.p0i32(i32* %t1)
   store i32 10, i32* %t1
diff --git a/test/CodeGen/NVPTX/intrinsics.ll b/test/CodeGen/NVPTX/intrinsics.ll
index 34b671d70e9..06a8712c210 100644
--- a/test/CodeGen/NVPTX/intrinsics.ll
+++ b/test/CodeGen/NVPTX/intrinsics.ll
@@ -16,6 +16,8 @@ define ptx_device double @test_fabs(double %d) {
 }
 
 define float @test_nvvm_sqrt(float %a) {
+; CHECK: sqrt.rn.f32 %f{{[0-9]+}}, %f{{[0-9]+}};
+; CHECK: ret;
   %val = call float @llvm.nvvm.sqrt.f(float %a)
   ret float %val
 }
diff --git a/test/CodeGen/NVPTX/load-with-non-coherent-cache.ll b/test/CodeGen/NVPTX/load-with-non-coherent-cache.ll
new file mode 100644
index 00000000000..d2d1ae67c01
--- /dev/null
+++ b/test/CodeGen/NVPTX/load-with-non-coherent-cache.ll
@@ -0,0 +1,210 @@
+; RUN: llc < %s -march=nvptx64 -mcpu=sm_20 | FileCheck -check-prefix=SM20 %s
+; RUN: llc < %s -march=nvptx64 -mcpu=sm_35 | FileCheck -check-prefix=SM35 %s
+
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v16:16:16-v32:32:32-v64:64:64-v128:128:128-n16:32:64"
+target triple = "nvptx64-unknown-unknown"
+
+; SM20-LABEL: .visible .entry foo1(
+; SM20: ld.global.f32
+; SM35-LABEL: .visible .entry foo1(
+; SM35: ld.global.nc.f32
+define void @foo1(float * noalias readonly %from, float * %to) {
+  %1 = load float, float * %from
+  store float %1, float * %to
+  ret void
+}
+
+; SM20-LABEL: .visible .entry foo2(
+; SM20: ld.global.f64
+; SM35-LABEL: .visible .entry foo2(
+; SM35: ld.global.nc.f64
+define void @foo2(double * noalias readonly %from, double * %to) {
+  %1 = load double, double * %from
+  store double %1, double * %to
+  ret void
+}
+
+; SM20-LABEL: .visible .entry foo3(
+; SM20: ld.global.u16
+; SM35-LABEL: .visible .entry foo3(
+; SM35: ld.global.nc.u16
+define void @foo3(i16 * noalias readonly %from, i16 * %to) {
+  %1 = load i16, i16 * %from
+  store i16 %1, i16 * %to
+  ret void
+}
+
+; SM20-LABEL: .visible .entry foo4(
+; SM20: ld.global.u32
+; SM35-LABEL: .visible .entry foo4(
+; SM35: ld.global.nc.u32
+define void @foo4(i32 * noalias readonly %from, i32 * %to) {
+  %1 = load i32, i32 * %from
+  store i32 %1, i32 * %to
+  ret void
+}
+
+; SM20-LABEL: .visible .entry foo5(
+; SM20: ld.global.u64
+; SM35-LABEL: .visible .entry foo5(
+; SM35: ld.global.nc.u64
+define void @foo5(i64 * noalias readonly %from, i64 * %to) {
+  %1 = load i64, i64 * %from
+  store i64 %1, i64 * %to
+  ret void
+}
+
+; i128 is non standard integer in nvptx64
+; SM20-LABEL: .visible .entry foo6(
+; SM20: ld.global.u64
+; SM20: ld.global.u64
+; SM35-LABEL: .visible .entry foo6(
+; SM35: ld.global.nc.u64
+; SM35: ld.global.nc.u64
+define void @foo6(i128 * noalias readonly %from, i128 * %to) {
+  %1 = load i128, i128 * %from
+  store i128 %1, i128 * %to
+  ret void
+}
+
+; SM20-LABEL: .visible .entry foo7(
+; SM20: ld.global.v2.u8
+; SM35-LABEL: .visible .entry foo7(
+; SM35: ld.global.nc.v2.u8
+define void @foo7(<2 x i8> * noalias readonly %from, <2 x i8> * %to) {
+  %1 = load <2 x i8>, <2 x i8> * %from
+  store <2 x i8> %1, <2 x i8> * %to
+  ret void
+}
+
+; SM20-LABEL: .visible .entry foo8(
+; SM20: ld.global.v2.u16
+; SM35-LABEL: .visible .entry foo8(
+; SM35: ld.global.nc.v2.u16
+define void @foo8(<2 x i16> * noalias readonly %from, <2 x i16> * %to) {
+  %1 = load <2 x i16>, <2 x i16> * %from
+  store <2 x i16> %1, <2 x i16> * %to
+  ret void
+}
+
+; SM20-LABEL: .visible .entry foo9(
+; SM20: ld.global.v2.u32
+; SM35-LABEL: .visible .entry foo9(
+; SM35: ld.global.nc.v2.u32
+define void @foo9(<2 x i32> * noalias readonly %from, <2 x i32> * %to) {
+  %1 = load <2 x i32>, <2 x i32> * %from
+  store <2 x i32> %1, <2 x i32> * %to
+  ret void
+}
+
+; SM20-LABEL: .visible .entry foo10(
+; SM20: ld.global.v2.u64
+; SM35-LABEL: .visible .entry foo10(
+; SM35: ld.global.nc.v2.u64
+define void @foo10(<2 x i64> * noalias readonly %from, <2 x i64> * %to) {
+  %1 = load <2 x i64>, <2 x i64> * %from
+  store <2 x i64> %1, <2 x i64> * %to
+  ret void
+}
+
+; SM20-LABEL: .visible .entry foo11(
+; SM20: ld.global.v2.f32
+; SM35-LABEL: .visible .entry foo11(
+; SM35: ld.global.nc.v2.f32
+define void @foo11(<2 x float> * noalias readonly %from, <2 x float> * %to) {
+  %1 = load <2 x float>, <2 x float> * %from
+  store <2 x float> %1, <2 x float> * %to
+  ret void
+}
+
+; SM20-LABEL: .visible .entry foo12(
+; SM20: ld.global.v2.f64
+; SM35-LABEL: .visible .entry foo12(
+; SM35: ld.global.nc.v2.f64
+define void @foo12(<2 x double> * noalias readonly %from, <2 x double> * %to) {
+  %1 = load <2 x double>, <2 x double> * %from
+  store <2 x double> %1, <2 x double> * %to
+  ret void
+}
+
+; SM20-LABEL: .visible .entry foo13(
+; SM20: ld.global.v4.u8
+; SM35-LABEL: .visible .entry foo13(
+; SM35: ld.global.nc.v4.u8
+define void @foo13(<4 x i8> * noalias readonly %from, <4 x i8> * %to) {
+  %1 = load <4 x i8>, <4 x i8> * %from
+  store <4 x i8> %1, <4 x i8> * %to
+  ret void
+}
+
+; SM20-LABEL: .visible .entry foo14(
+; SM20: ld.global.v4.u16
+; SM35-LABEL: .visible .entry foo14(
+; SM35: ld.global.nc.v4.u16
+define void @foo14(<4 x i16> * noalias readonly %from, <4 x i16> * %to) {
+  %1 = load <4 x i16>, <4 x i16> * %from
+  store <4 x i16> %1, <4 x i16> * %to
+  ret void
+}
+
+; SM20-LABEL: .visible .entry foo15(
+; SM20: ld.global.v4.u32
+; SM35-LABEL: .visible .entry foo15(
+; SM35: ld.global.nc.v4.u32
+define void @foo15(<4 x i32> * noalias readonly %from, <4 x i32> * %to) {
+  %1 = load <4 x i32>, <4 x i32> * %from
+  store <4 x i32> %1, <4 x i32> * %to
+  ret void
+}
+
+; SM20-LABEL: .visible .entry foo16(
+; SM20: ld.global.v4.f32
+; SM35-LABEL: .visible .entry foo16(
+; SM35: ld.global.nc.v4.f32
+define void @foo16(<4 x float> * noalias readonly %from, <4 x float> * %to) {
+  %1 = load <4 x float>, <4 x float> * %from
+  store <4 x float> %1, <4 x float> * %to
+  ret void
+}
+
+; SM20-LABEL: .visible .entry foo17(
+; SM20: ld.global.v2.f64
+; SM20: ld.global.v2.f64
+; SM35-LABEL: .visible .entry foo17(
+; SM35: ld.global.nc.v2.f64
+; SM35: ld.global.nc.v2.f64
+define void @foo17(<4 x double> * noalias readonly %from, <4 x double> * %to) {
+  %1 = load <4 x double>, <4 x double> * %from
+  store <4 x double> %1, <4 x double> * %to
+  ret void
+}
+
+; SM20-LABEL: .visible .entry foo18(
+; SM20: ld.global.u64
+; SM35-LABEL: .visible .entry foo18(
+; SM35: ld.global.nc.u64
+define void @foo18(float ** noalias readonly %from, float ** %to) {
+  %1 = load float *, float ** %from
+  store float * %1, float ** %to
+  ret void
+}
+
+!nvvm.annotations = !{!1 ,!2 ,!3 ,!4 ,!5 ,!6, !7 ,!8 ,!9 ,!10 ,!11 ,!12, !13, !14, !15, !16, !17, !18}
+!1 = !{void (float *, float *)* @foo1, !"kernel", i32 1}
+!2 = !{void (double *, double *)* @foo2, !"kernel", i32 1}
+!3 = !{void (i16 *, i16 *)* @foo3, !"kernel", i32 1}
+!4 = !{void (i32 *, i32 *)* @foo4, !"kernel", i32 1}
+!5 = !{void (i64 *, i64 *)* @foo5, !"kernel", i32 1}
+!6 = !{void (i128 *, i128 *)* @foo6, !"kernel", i32 1}
+!7 = !{void (<2 x i8> *, <2 x i8> *)* @foo7, !"kernel", i32 1}
+!8 = !{void (<2 x i16> *, <2 x i16> *)* @foo8, !"kernel", i32 1}
+!9 = !{void (<2 x i32> *, <2 x i32> *)* @foo9, !"kernel", i32 1}
+!10 = !{void (<2 x i64> *, <2 x i64> *)* @foo10, !"kernel", i32 1}
+!11 = !{void (<2 x float> *, <2 x float> *)* @foo11, !"kernel", i32 1}
+!12 = !{void (<2 x double> *, <2 x double> *)* @foo12, !"kernel", i32 1}
+!13 = !{void (<4 x i8> *, <4 x i8> *)* @foo13, !"kernel", i32 1}
+!14 = !{void (<4 x i16> *, <4 x i16> *)* @foo14, !"kernel", i32 1}
+!15 = !{void (<4 x i32> *, <4 x i32> *)* @foo15, !"kernel", i32 1}
+!16 = !{void (<4 x float> *, <4 x float> *)* @foo16, !"kernel", i32 1}
+!17 = !{void (<4 x double> *, <4 x double> *)* @foo17, !"kernel", i32 1}
+!18 = !{void (float **, float **)* @foo18, !"kernel", i32 1}
diff --git a/test/CodeGen/NVPTX/local-stack-frame.ll b/test/CodeGen/NVPTX/local-stack-frame.ll
index 377eee9170e..ef1b7da6ad0 100644
--- a/test/CodeGen/NVPTX/local-stack-frame.ll
+++ b/test/CodeGen/NVPTX/local-stack-frame.ll
@@ -3,12 +3,12 @@
 
 ; Ensure we access the local stack properly
 
-; PTX32:        mov.u32          %r{{[0-9]+}}, __local_depot{{[0-9]+}};
-; PTX32:        cvta.local.u32   %SP, %r{{[0-9]+}};
+; PTX32:        mov.u32          %SPL, __local_depot{{[0-9]+}};
+; PTX32:        cvta.local.u32   %SP, %SPL;
 ; PTX32:        ld.param.u32     %r{{[0-9]+}}, [foo_param_0];
 ; PTX32:        st.volatile.u32  [%SP+0], %r{{[0-9]+}};
-; PTX64:        mov.u64          %rd{{[0-9]+}}, __local_depot{{[0-9]+}};
-; PTX64:        cvta.local.u64   %SP, %rd{{[0-9]+}};
+; PTX64:        mov.u64          %SPL, __local_depot{{[0-9]+}};
+; PTX64:        cvta.local.u64   %SP, %SPL;
 ; PTX64:        ld.param.u32     %r{{[0-9]+}}, [foo_param_0];
 ; PTX64:        st.volatile.u32  [%SP+0], %r{{[0-9]+}};
 define void @foo(i32 %a) {
@@ -16,3 +16,67 @@ define void @foo(i32 %a) {
   store volatile i32 %a, i32* %local
   ret void
 }
+
+; PTX32:        mov.u32          %SPL, __local_depot{{[0-9]+}};
+; PTX32:        cvta.local.u32   %SP, %SPL;
+; PTX32:        ld.param.u32     %r{{[0-9]+}}, [foo2_param_0];
+; PTX32:        add.u32          %r[[SP_REG:[0-9]+]], %SPL, 0;
+; PTX32:        st.local.u32  [%r[[SP_REG]]], %r{{[0-9]+}};
+; PTX64:        mov.u64          %SPL, __local_depot{{[0-9]+}};
+; PTX64:        cvta.local.u64   %SP, %SPL;
+; PTX64:        ld.param.u32     %r{{[0-9]+}}, [foo2_param_0];
+; PTX64:        add.u64          %rd[[SP_REG:[0-9]+]], %SPL, 0;
+; PTX64:        st.local.u32  [%rd[[SP_REG]]], %r{{[0-9]+}};
+define void @foo2(i32 %a) {
+  %local = alloca i32, align 4
+  store i32 %a, i32* %local
+  call void @bar(i32* %local)
+  ret void
+}
+
+declare void @bar(i32* %a)
+
+!nvvm.annotations = !{!0}
+!0 = !{void (i32)* @foo2, !"kernel", i32 1}
+
+; PTX32:        mov.u32          %SPL, __local_depot{{[0-9]+}};
+; PTX32-NOT:    cvta.local.u32   %SP, %SPL;
+; PTX32:        ld.param.u32     %r{{[0-9]+}}, [foo3_param_0];
+; PTX32:        add.u32          %r{{[0-9]+}}, %SPL, 0;
+; PTX32:        st.local.u32  [%r{{[0-9]+}}], %r{{[0-9]+}};
+; PTX64:        mov.u64          %SPL, __local_depot{{[0-9]+}};
+; PTX64-NOT:    cvta.local.u64   %SP, %SPL;
+; PTX64:        ld.param.u32     %r{{[0-9]+}}, [foo3_param_0];
+; PTX64:        add.u64          %rd{{[0-9]+}}, %SPL, 0;
+; PTX64:        st.local.u32  [%rd{{[0-9]+}}], %r{{[0-9]+}};
+define void @foo3(i32 %a) {
+  %local = alloca [3 x i32], align 4
+  %1 = bitcast [3 x i32]* %local to i32*
+  %2 = getelementptr inbounds i32, i32* %1, i32 %a
+  store i32 %a, i32* %2
+  ret void
+}
+
+; PTX32:        cvta.local.u32   %SP, %SPL;
+; PTX32:        add.u32          {{%r[0-9]+}}, %SP, 0;
+; PTX32:        add.u32          {{%r[0-9]+}}, %SPL, 0;
+; PTX32:        add.u32          {{%r[0-9]+}}, %SP, 4;
+; PTX32:        add.u32          {{%r[0-9]+}}, %SPL, 4;
+; PTX32:        st.local.u32     [{{%r[0-9]+}}], {{%r[0-9]+}}
+; PTX32:        st.local.u32     [{{%r[0-9]+}}], {{%r[0-9]+}}
+; PTX64:        cvta.local.u64   %SP, %SPL;
+; PTX64:        add.u64          {{%rd[0-9]+}}, %SP, 0;
+; PTX64:        add.u64          {{%rd[0-9]+}}, %SPL, 0;
+; PTX64:        add.u64          {{%rd[0-9]+}}, %SP, 4;
+; PTX64:        add.u64          {{%rd[0-9]+}}, %SPL, 4;
+; PTX64:        st.local.u32     [{{%rd[0-9]+}}], {{%r[0-9]+}}
+; PTX64:        st.local.u32     [{{%rd[0-9]+}}], {{%r[0-9]+}}
+define void @foo4() {
+  %A = alloca i32
+  %B = alloca i32
+  store i32 0, i32* %A
+  store i32 0, i32* %B
+  call void @bar(i32* %A)
+  call void @bar(i32* %B)
+  ret void
+}
diff --git a/test/CodeGen/NVPTX/loop-vectorize.ll b/test/CodeGen/NVPTX/loop-vectorize.ll
new file mode 100644
index 00000000000..1b337441ac9
--- /dev/null
+++ b/test/CodeGen/NVPTX/loop-vectorize.ll
@@ -0,0 +1,39 @@
+; RUN: opt < %s -O3 -S | FileCheck %s
+
+target datalayout = "e-i64:64-v16:16-v32:32-n16:32:64"
+target triple = "nvptx64-nvidia-cuda"
+
+define void @no_vectorization(i32 %n, i32 %a, i32 %b) {
+; CHECK-LABEL: no_vectorization(
+; CHECK-NOT: <4 x i32>
+; CHECK-NOT: <4 x i1>
+entry:
+  %cmp.5 = icmp sgt i32 %n, 0
+  br i1 %cmp.5, label %for.body.preheader, label %for.cond.cleanup
+
+for.body.preheader:                               ; preds = %entry
+  br label %for.body
+
+for.cond.cleanup.loopexit:                        ; preds = %for.body
+  br label %for.cond.cleanup
+
+for.cond.cleanup:                                 ; preds = %for.cond.cleanup.loopexit, %entry
+  ret void
+
+for.body:                                         ; preds = %for.body.preheader, %for.body
+  %i.06 = phi i32 [ %inc, %for.body ], [ 0, %for.body.preheader ]
+  %add = add nsw i32 %i.06, %a
+  %mul = mul nsw i32 %add, %b
+  %cmp1 = icmp sgt i32 %mul, -1
+  tail call void @llvm.assume(i1 %cmp1)
+  %inc = add nuw nsw i32 %i.06, 1
+  %exitcond = icmp eq i32 %inc, %n
+  br i1 %exitcond, label %for.cond.cleanup.loopexit, label %for.body
+}
+
+declare void @llvm.assume(i1) #0
+
+attributes #0 = { nounwind }
+
+!nvvm.annotations = !{!0}
+!0 = !{void (i32, i32, i32)* @no_vectorization, !"kernel", i32 1}
diff --git a/test/CodeGen/NVPTX/lower-aggr-copies.ll b/test/CodeGen/NVPTX/lower-aggr-copies.ll
new file mode 100644
index 00000000000..ef570982b80
--- /dev/null
+++ b/test/CodeGen/NVPTX/lower-aggr-copies.ll
@@ -0,0 +1,121 @@
+; RUN: llc < %s -march=nvptx64 -mcpu=sm_35 | FileCheck %s --check-prefix PTX
+; RUN: opt < %s -S -nvptx-lower-aggr-copies | FileCheck %s --check-prefix IR
+
+; Verify that the NVPTXLowerAggrCopies pass works as expected - calls to
+; llvm.mem* intrinsics get lowered to loops.
+
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "nvptx64-unknown-unknown"
+
+declare void @llvm.memcpy.p0i8.p0i8.i64(i8* nocapture, i8* nocapture readonly, i64, i32, i1) #1
+declare void @llvm.memmove.p0i8.p0i8.i64(i8* nocapture, i8* nocapture readonly, i64, i32, i1) #1
+declare void @llvm.memset.p0i8.i64(i8* nocapture, i8, i64, i32, i1) #1
+
+define i8* @memcpy_caller(i8* %dst, i8* %src, i64 %n) #0 {
+entry:
+  tail call void @llvm.memcpy.p0i8.p0i8.i64(i8* %dst, i8* %src, i64 %n, i32 1, i1 false)
+  ret i8* %dst
+
+; IR-LABEL:   @memcpy_caller
+; IR:         loadstoreloop:
+; IR:         [[LOADPTR:%[0-9]+]] = getelementptr inbounds i8, i8* %src, i64
+; IR-NEXT:    [[VAL:%[0-9]+]] = load i8, i8* [[LOADPTR]]
+; IR-NEXT:    [[STOREPTR:%[0-9]+]] = getelementptr inbounds i8, i8* %dst, i64
+; IR-NEXT:    store i8 [[VAL]], i8* [[STOREPTR]]
+
+; PTX-LABEL:  .visible .func (.param .b64 func_retval0) memcpy_caller
+; PTX:        LBB[[LABEL:[_0-9]+]]:
+; PTX:        ld.u8 %rs[[REG:[0-9]+]]
+; PTX:        st.u8 [%rd{{[0-9]+}}], %rs[[REG]]
+; PTX:        add.s64 %rd[[COUNTER:[0-9]+]], %rd[[COUNTER]], 1
+; PTX-NEXT:   setp.lt.u64 %p[[PRED:[0-9]+]], %rd[[COUNTER]], %rd
+; PTX-NEXT:   @%p[[PRED]] bra LBB[[LABEL]]
+}
+
+define i8* @memcpy_volatile_caller(i8* %dst, i8* %src, i64 %n) #0 {
+entry:
+  tail call void @llvm.memcpy.p0i8.p0i8.i64(i8* %dst, i8* %src, i64 %n, i32 1, i1 true)
+  ret i8* %dst
+
+; IR-LABEL:   @memcpy_volatile_caller
+; IR:         load volatile
+; IR:         store volatile
+
+; PTX-LABEL:  .visible .func (.param .b64 func_retval0) memcpy_volatile_caller
+; PTX:        LBB[[LABEL:[_0-9]+]]:
+; PTX:        ld.volatile.u8 %rs[[REG:[0-9]+]]
+; PTX:        st.volatile.u8 [%rd{{[0-9]+}}], %rs[[REG]]
+; PTX:        add.s64 %rd[[COUNTER:[0-9]+]], %rd[[COUNTER]], 1
+; PTX-NEXT:   setp.lt.u64 %p[[PRED:[0-9]+]], %rd[[COUNTER]], %rd
+; PTX-NEXT:   @%p[[PRED]] bra LBB[[LABEL]]
+}
+
+define i8* @memcpy_casting_caller(i32* %dst, i32* %src, i64 %n) #0 {
+entry:
+  %0 = bitcast i32* %dst to i8*
+  %1 = bitcast i32* %src to i8*
+  tail call void @llvm.memcpy.p0i8.p0i8.i64(i8* %0, i8* %1, i64 %n, i32 1, i1 false)
+  ret i8* %0
+
+; Check that casts in calls to memcpy are handled properly
+; IR-LABEL:   @memcpy_casting_caller
+; IR:         [[DSTCAST:%[0-9]+]] = bitcast i32* %dst to i8*
+; IR:         [[SRCCAST:%[0-9]+]] = bitcast i32* %src to i8*
+; IR:         getelementptr inbounds i8, i8* [[SRCCAST]]
+; IR:         getelementptr inbounds i8, i8* [[DSTCAST]]
+}
+
+define i8* @memset_caller(i8* %dst, i32 %c, i64 %n) #0 {
+entry:
+  %0 = trunc i32 %c to i8
+  tail call void @llvm.memset.p0i8.i64(i8* %dst, i8 %0, i64 %n, i32 1, i1 false)
+  ret i8* %dst
+
+; IR-LABEL:   @memset_caller
+; IR:         [[VAL:%[0-9]+]] = trunc i32 %c to i8
+; IR:         loadstoreloop:
+; IR:         [[STOREPTR:%[0-9]+]] = getelementptr inbounds i8, i8* %dst, i64
+; IR-NEXT:    store i8 [[VAL]], i8* [[STOREPTR]]
+
+; PTX-LABEL:  .visible .func (.param .b64 func_retval0) memset_caller(
+; PTX:        ld.param.u8 %rs[[REG:[0-9]+]]
+; PTX:        LBB[[LABEL:[_0-9]+]]:
+; PTX:        st.u8 [%rd{{[0-9]+}}], %rs[[REG]]
+; PTX:        add.s64 %rd[[COUNTER:[0-9]+]], %rd[[COUNTER]], 1
+; PTX-NEXT:   setp.lt.u64 %p[[PRED:[0-9]+]], %rd[[COUNTER]], %rd
+; PTX-NEXT:   @%p[[PRED]] bra LBB[[LABEL]]
+}
+
+define i8* @memmove_caller(i8* %dst, i8* %src, i64 %n) #0 {
+entry:
+  tail call void @llvm.memmove.p0i8.p0i8.i64(i8* %dst, i8* %src, i64 %n, i32 1, i1 false)
+  ret i8* %dst
+
+; IR-LABEL:   @memmove_caller
+; IR:         icmp ult i8* %src, %dst
+; IR:         [[PHIVAL:%[0-9a-zA-Z_]+]] = phi i64
+; IR-NEXT:    %index_ptr = sub i64 [[PHIVAL]], 1
+; IR:         [[FWDPHIVAL:%[0-9a-zA-Z_]+]] = phi i64
+; IR:         {{%[0-9a-zA-Z_]+}} = add i64 [[FWDPHIVAL]], 1
+
+; PTX-LABEL:  .visible .func (.param .b64 func_retval0) memmove_caller(
+; PTX:        ld.param.u64 %rd[[N:[0-9]+]]
+; PTX:        setp.eq.s64 %p[[NEQ0:[0-9]+]], %rd[[N]], 0
+; PTX:        setp.ge.u64 %p[[SRC_GT_THAN_DST:[0-9]+]], %rd{{[0-9]+}}, %rd{{[0-9]+}}
+; PTX-NEXT:   @%p[[SRC_GT_THAN_DST]] bra LBB[[FORWARD_BB:[0-9_]+]]
+; -- this is the backwards copying BB
+; PTX:        @%p[[NEQ0]] bra LBB[[EXIT:[0-9_]+]]
+; PTX:        add.s64 %rd[[N]], %rd[[N]], -1
+; PTX:        ld.u8 %rs[[ELEMENT:[0-9]+]]
+; PTX:        st.u8 [%rd{{[0-9]+}}], %rs[[ELEMENT]]
+; -- this is the forwards copying BB
+; PTX:        LBB[[FORWARD_BB]]:
+; PTX:        @%p[[NEQ0]] bra LBB[[EXIT]]
+; PTX:        ld.u8 %rs[[ELEMENT2:[0-9]+]]
+; PTX:        st.u8 [%rd{{[0-9]+}}], %rs[[ELEMENT2]]
+; PTX:        add.s64 %rd[[INDEX:[0-9]+]], %rd[[INDEX]], 1
+; -- exit block
+; PTX:        LBB[[EXIT]]:
+; PTX-NEXT:   st.param.b64 [func_retval0
+; PTX-NEXT:   ret
+}
diff --git a/test/CodeGen/NVPTX/lower-alloca.ll b/test/CodeGen/NVPTX/lower-alloca.ll
new file mode 100644
index 00000000000..397dc1fc52c
--- /dev/null
+++ b/test/CodeGen/NVPTX/lower-alloca.ll
@@ -0,0 +1,22 @@
+; RUN: opt < %s -S -nvptx-lower-alloca -nvptx-favor-non-generic -dce | FileCheck %s
+; RUN: llc < %s -march=nvptx64 -mcpu=sm_35 | FileCheck %s --check-prefix PTX
+
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v16:16:16-v32:32:32-v64:64:64-v128:128:128-n16:32:64"
+target triple = "nvptx64-unknown-unknown"
+
+define void @kernel() {
+; LABEL: @lower_alloca
+; PTX-LABEL: .visible .entry kernel(
+  %A = alloca i32
+; CHECK: addrspacecast i32* %A to i32 addrspace(5)*
+; CHECK: store i32 0, i32 addrspace(5)* {{%.+}}
+; PTX: st.local.u32 [{{%rd[0-9]+}}], {{%r[0-9]+}}
+  store i32 0, i32* %A
+  call void @callee(i32* %A)
+  ret void
+}
+
+declare void @callee(i32*)
+
+!nvvm.annotations = !{!0}
+!0 = !{void ()* @kernel, !"kernel", i32 1}
diff --git a/test/CodeGen/NVPTX/lower-kernel-ptr-arg.ll b/test/CodeGen/NVPTX/lower-kernel-ptr-arg.ll
new file mode 100644
index 00000000000..0de72c4a1ae
--- /dev/null
+++ b/test/CodeGen/NVPTX/lower-kernel-ptr-arg.ll
@@ -0,0 +1,31 @@
+; RUN: llc < %s -march=nvptx64 -mcpu=sm_20 | FileCheck %s
+
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v16:16:16-v32:32:32-v64:64:64-v128:128:128-n16:32:64"
+target triple = "nvptx64-unknown-unknown"
+
+; Verify that both %input and %output are converted to global pointers and then
+; addrspacecast'ed back to the original type.
+define void @kernel(float* %input, float* %output) {
+; CHECK-LABEL: .visible .entry kernel(
+; CHECK: cvta.to.global.u64
+; CHECK: cvta.to.global.u64
+  %1 = load float, float* %input, align 4
+; CHECK: ld.global.f32
+  store float %1, float* %output, align 4
+; CHECK: st.global.f32
+  ret void
+}
+
+define void @kernel2(float addrspace(1)* %input, float addrspace(1)* %output) {
+; CHECK-LABEL: .visible .entry kernel2(
+; CHECK-NOT: cvta.to.global.u64
+  %1 = load float, float addrspace(1)* %input, align 4
+; CHECK: ld.global.f32
+  store float %1, float addrspace(1)* %output, align 4
+; CHECK: st.global.f32
+  ret void
+}
+
+!nvvm.annotations = !{!0, !1}
+!0 = !{void (float*, float*)* @kernel, !"kernel", i32 1}
+!1 = !{void (float addrspace(1)*, float addrspace(1)*)* @kernel2, !"kernel", i32 1}
diff --git a/test/CodeGen/NVPTX/pr13291-i1-store.ll b/test/CodeGen/NVPTX/pr13291-i1-store.ll
index d4f7c3bd210..934df30a3a7 100644
--- a/test/CodeGen/NVPTX/pr13291-i1-store.ll
+++ b/test/CodeGen/NVPTX/pr13291-i1-store.ll
@@ -3,19 +3,19 @@
 
 define ptx_kernel void @t1(i1* %a) {
 ; PTX32:      mov.u16 %rs{{[0-9]+}}, 0;
-; PTX32-NEXT: st.u8 [%r{{[0-9]+}}], %rs{{[0-9]+}};
+; PTX32-NEXT: st.global.u8 [%r{{[0-9]+}}], %rs{{[0-9]+}};
 ; PTX64:      mov.u16 %rs{{[0-9]+}}, 0;
-; PTX64-NEXT: st.u8 [%rd{{[0-9]+}}], %rs{{[0-9]+}};
+; PTX64-NEXT: st.global.u8 [%rd{{[0-9]+}}], %rs{{[0-9]+}};
   store i1 false, i1* %a
   ret void
 }
 
 
 define ptx_kernel void @t2(i1* %a, i8* %b) {
-; PTX32: ld.u8 %rs{{[0-9]+}}, [%r{{[0-9]+}}]
+; PTX32: ld.global.u8 %rs{{[0-9]+}}, [%r{{[0-9]+}}]
 ; PTX32: and.b16 %rs{{[0-9]+}}, %rs{{[0-9]+}}, 1;
 ; PTX32: setp.eq.b16 %p{{[0-9]+}}, %rs{{[0-9]+}}, 1;
-; PTX64: ld.u8 %rs{{[0-9]+}}, [%rd{{[0-9]+}}]
+; PTX64: ld.global.u8 %rs{{[0-9]+}}, [%rd{{[0-9]+}}]
 ; PTX64: and.b16 %rs{{[0-9]+}}, %rs{{[0-9]+}}, 1;
 ; PTX64: setp.eq.b16 %p{{[0-9]+}}, %rs{{[0-9]+}}, 1;
 
diff --git a/test/CodeGen/NVPTX/surf-read-cuda.ll b/test/CodeGen/NVPTX/surf-read-cuda.ll
index ed021346c0f..c17c71e01d3 100644
--- a/test/CodeGen/NVPTX/surf-read-cuda.ll
+++ b/test/CodeGen/NVPTX/surf-read-cuda.ll
@@ -18,8 +18,8 @@ define void @foo(i64 %img, float* %red, i32 %idx) {
 ; SM20: cvt.rn.f32.s32 %f[[REDF:[0-9]+]], %r[[RED]]
 ; SM30: cvt.rn.f32.s32 %f[[REDF:[0-9]+]], %r[[RED]]
   %ret = sitofp i32 %val to float
-; SM20: st.f32 [%r{{[0-9]+}}], %f[[REDF]]
-; SM30: st.f32 [%r{{[0-9]+}}], %f[[REDF]]
+; SM20: st.global.f32 [%r{{[0-9]+}}], %f[[REDF]]
+; SM30: st.global.f32 [%r{{[0-9]+}}], %f[[REDF]]
   store float %ret, float* %red
   ret void
 }
@@ -37,8 +37,8 @@ define void @bar(float* %red, i32 %idx) {
 ; SM20: cvt.rn.f32.s32 %f[[REDF:[0-9]+]], %r[[RED]]
 ; SM30: cvt.rn.f32.s32 %f[[REDF:[0-9]+]], %r[[RED]]
   %ret = sitofp i32 %val to float
-; SM20: st.f32 [%r{{[0-9]+}}], %f[[REDF]]
-; SM30: st.f32 [%r{{[0-9]+}}], %f[[REDF]]
+; SM20: st.global.f32 [%r{{[0-9]+}}], %f[[REDF]]
+; SM30: st.global.f32 [%r{{[0-9]+}}], %f[[REDF]]
   store float %ret, float* %red
   ret void
 }
diff --git a/test/CodeGen/NVPTX/tex-read-cuda.ll b/test/CodeGen/NVPTX/tex-read-cuda.ll
index c5b5600de87..d5f7c1667f1 100644
--- a/test/CodeGen/NVPTX/tex-read-cuda.ll
+++ b/test/CodeGen/NVPTX/tex-read-cuda.ll
@@ -16,8 +16,8 @@ define void @foo(i64 %img, float* %red, i32 %idx) {
 ; SM30: tex.1d.v4.f32.s32 {%f[[RED:[0-9]+]], %f[[GREEN:[0-9]+]], %f[[BLUE:[0-9]+]], %f[[ALPHA:[0-9]+]]}, [%rd[[TEXREG]], {%r{{[0-9]+}}}]
   %val = tail call { float, float, float, float } @llvm.nvvm.tex.unified.1d.v4f32.s32(i64 %img, i32 %idx)
   %ret = extractvalue { float, float, float, float } %val, 0
-; SM20: st.f32 [%r{{[0-9]+}}], %f[[RED]]
-; SM30: st.f32 [%r{{[0-9]+}}], %f[[RED]]
+; SM20: st.global.f32 [%r{{[0-9]+}}], %f[[RED]]
+; SM30: st.global.f32 [%r{{[0-9]+}}], %f[[RED]]
   store float %ret, float* %red
   ret void
 }
@@ -34,8 +34,8 @@ define void @bar(float* %red, i32 %idx) {
 ; SM30: tex.1d.v4.f32.s32 {%f[[RED:[0-9]+]], %f[[GREEN:[0-9]+]], %f[[BLUE:[0-9]+]], %f[[ALPHA:[0-9]+]]}, [%rd[[TEXHANDLE]], {%r{{[0-9]+}}}]
   %val = tail call { float, float, float, float } @llvm.nvvm.tex.unified.1d.v4f32.s32(i64 %texHandle, i32 %idx)
   %ret = extractvalue { float, float, float, float } %val, 0
-; SM20: st.f32 [%r{{[0-9]+}}], %f[[RED]]
-; SM30: st.f32 [%r{{[0-9]+}}], %f[[RED]]
+; SM20: st.global.f32 [%r{{[0-9]+}}], %f[[RED]]
+; SM30: st.global.f32 [%r{{[0-9]+}}], %f[[RED]]
   store float %ret, float* %red
   ret void
 }
diff --git a/test/CodeGen/PowerPC/2007-01-15-AsmDialect.ll b/test/CodeGen/PowerPC/2007-01-15-AsmDialect.ll
index 57c3531c3da..ca134fa9be4 100644
--- a/test/CodeGen/PowerPC/2007-01-15-AsmDialect.ll
+++ b/test/CodeGen/PowerPC/2007-01-15-AsmDialect.ll
@@ -1,5 +1,5 @@
 ; RUN: llc < %s -march=ppc32 -mtriple=powerpc-apple-darwin8 | \
-; RUN:    grep cntlzw
+; RUN:    grep cntlz
 
 define i32 @foo() nounwind {
 entry:
diff --git a/test/CodeGen/PowerPC/2007-11-16-landingpad-split.ll b/test/CodeGen/PowerPC/2007-11-16-landingpad-split.ll
index 34122912349..bd496704890 100644
--- a/test/CodeGen/PowerPC/2007-11-16-landingpad-split.ll
+++ b/test/CodeGen/PowerPC/2007-11-16-landingpad-split.ll
@@ -19,7 +19,7 @@ target triple = "powerpc64-apple-darwin8"
 ; CHECK: .cfi_endproc
 
 
-define void @Bork(i64 %range.0.0, i64 %range.0.1, i64 %size) {
+define void @Bork(i64 %range.0.0, i64 %range.0.1, i64 %size) personality i32 (...)* @__gxx_personality_v0 {
 entry:
 	%effectiveRange = alloca %struct.Range, align 8		; <%struct.Range*> [#uses=2]
 	%tmp4 = call i8* @llvm.stacksave()		; <i8*> [#uses=1]
@@ -33,7 +33,7 @@ bb30.preheader:		; preds = %entry
 	br label %bb30
 
 unwind:		; preds = %cond_true, %entry
-        %exn = landingpad {i8*, i32} personality i32 (...)* @__gxx_personality_v0
+        %exn = landingpad {i8*, i32}
                  catch i8* null
 	call void @llvm.stackrestore(i8* %tmp4)
         resume { i8*, i32 } %exn
diff --git a/test/CodeGen/PowerPC/2010-02-12-saveCR.ll b/test/CodeGen/PowerPC/2010-02-12-saveCR.ll
index b0c37b80ed2..5932b6d75c0 100644
--- a/test/CodeGen/PowerPC/2010-02-12-saveCR.ll
+++ b/test/CodeGen/PowerPC/2010-02-12-saveCR.ll
@@ -9,13 +9,13 @@ entry:
 ; CHECK: mfcr [[T1:r[0-9]+]]                         ; cr2
 ; CHECK: lis [[T2:r[0-9]+]], 1
 ; CHECK: addi r3, r1, 72
-; CHECK: rlwinm [[T1]], [[T1]], 8, 0, 31
+; CHECK: rotlwi [[T1]], [[T1]], 8
 ; CHECK: ori [[T2]], [[T2]], 34540
 ; CHECK: stwx [[T1]], r1, [[T2]]
 ; CHECK: lis [[T3:r[0-9]+]], 1
 ; CHECK: mfcr [[T4:r[0-9]+]]                         ; cr3
 ; CHECK: ori [[T3]], [[T3]], 34536
-; CHECK: rlwinm [[T4]], [[T4]], 12, 0, 31
+; CHECK: rotlwi [[T4]], [[T4]], 12
 ; CHECK: stwx [[T4]], r1, [[T3]]
   %x = alloca [100000 x i8]                       ; <[100000 x i8]*> [#uses=1]
   %"alloca point" = bitcast i32 0 to i32          ; <i32> [#uses=0]
@@ -28,12 +28,12 @@ return:                                           ; preds = %entry
 ; CHECK: lis [[T1:r[0-9]+]], 1
 ; CHECK: ori [[T1]], [[T1]], 34536
 ; CHECK: lwzx [[T1]], r1, [[T1]]
-; CHECK: rlwinm [[T1]], [[T1]], 20, 0, 31
+; CHECK: rotlwi [[T1]], [[T1]], 20
 ; CHECK: mtcrf 16, [[T1]]
 ; CHECK: lis [[T1]], 1
 ; CHECK: ori [[T1]], [[T1]], 34540
 ; CHECK: lwzx [[T1]], r1, [[T1]]
-; CHECK: rlwinm [[T1]], [[T1]], 24, 0, 31
+; CHECK: rotlwi [[T1]], [[T1]], 24
 ; CHECK: mtcrf 32, [[T1]]
   ret void
 }
diff --git a/test/CodeGen/PowerPC/MergeConsecutiveStores.ll b/test/CodeGen/PowerPC/MergeConsecutiveStores.ll
new file mode 100644
index 00000000000..977b3b701cc
--- /dev/null
+++ b/test/CodeGen/PowerPC/MergeConsecutiveStores.ll
@@ -0,0 +1,68 @@
+; RUN: llc -march=ppc32 -mtriple=powerpc-unknown-linux-gnu -mattr=+altivec < %s | FileCheck %s
+
+;; This test ensures that MergeConsecutiveStores does not attempt to
+;; merge stores or loads when doing so would result in unaligned
+;; memory operations (unless the target supports those, e.g. X86).
+
+;; This issue happen in other situations for other targets, but PPC
+;; with Altivec extensions was chosen for the test because it does not
+;; support unaligned access with AltiVec instructions. If the 4
+;; load/stores get merged to an v4i32 vector type severely bad code
+;; gets generated: it painstakingly copies the values to a temporary
+;; location on the stack, with vector ops, in order to then use
+;; integer ops to load from the temporary stack location and store to
+;; the final location. Yuck!
+
+%struct.X = type { i32, i32, i32, i32 }
+
+@fx = common global %struct.X zeroinitializer, align 4
+@fy = common global %struct.X zeroinitializer, align 4
+
+;; In this test case, lvx and stvx instructions should NOT be
+;; generated, as the alignment is not sufficient for it to be
+;; worthwhile.
+
+;; CHECK-LABEL: f:
+;; CHECK:      lwzu
+;; CHECK-NEXT: lwz
+;; CHECK-NEXT: lwz
+;; CHECK-NEXT: lwz
+;; CHECK-NEXT: stwu
+;; CHECK-NEXT: stw
+;; CHECK-NEXT: stw
+;; CHECK-NEXT: stw
+;; CHECK-NEXT: blr
+define void @f() {
+entry:
+  %0 = load i32, i32* getelementptr inbounds (%struct.X, %struct.X* @fx, i32 0, i32 0), align 4
+  %1 = load i32, i32* getelementptr inbounds (%struct.X, %struct.X* @fx, i32 0, i32 1), align 4
+  %2 = load i32, i32* getelementptr inbounds (%struct.X, %struct.X* @fx, i32 0, i32 2), align 4
+  %3 = load i32, i32* getelementptr inbounds (%struct.X, %struct.X* @fx, i32 0, i32 3), align 4
+  store i32 %0, i32* getelementptr inbounds (%struct.X, %struct.X* @fy, i32 0, i32 0), align 4
+  store i32 %1, i32* getelementptr inbounds (%struct.X, %struct.X* @fy, i32 0, i32 1), align 4
+  store i32 %2, i32* getelementptr inbounds (%struct.X, %struct.X* @fy, i32 0, i32 2), align 4
+  store i32 %3, i32* getelementptr inbounds (%struct.X, %struct.X* @fy, i32 0, i32 3), align 4
+  ret void
+}
+
+@gx = common global %struct.X zeroinitializer, align 16
+@gy = common global %struct.X zeroinitializer, align 16
+
+;; In this test, lvx and stvx instructions SHOULD be generated, as
+;; the 16-byte alignment of the new load/store is acceptable.
+;; CHECK-LABEL: g:
+;; CHECK: lvx
+;; CHECK: stvx
+;; CHECK: blr
+define void @g() {
+entry:
+  %0 = load i32, i32* getelementptr inbounds (%struct.X, %struct.X* @fx, i32 0, i32 0), align 16
+  %1 = load i32, i32* getelementptr inbounds (%struct.X, %struct.X* @fx, i32 0, i32 1), align 4
+  %2 = load i32, i32* getelementptr inbounds (%struct.X, %struct.X* @fx, i32 0, i32 2), align 4
+  %3 = load i32, i32* getelementptr inbounds (%struct.X, %struct.X* @fx, i32 0, i32 3), align 4
+  store i32 %0, i32* getelementptr inbounds (%struct.X, %struct.X* @fy, i32 0, i32 0), align 16
+  store i32 %1, i32* getelementptr inbounds (%struct.X, %struct.X* @fy, i32 0, i32 1), align 4
+  store i32 %2, i32* getelementptr inbounds (%struct.X, %struct.X* @fy, i32 0, i32 2), align 4
+  store i32 %3, i32* getelementptr inbounds (%struct.X, %struct.X* @fy, i32 0, i32 3), align 4
+  ret void
+}
diff --git a/test/CodeGen/PowerPC/anon_aggr.ll b/test/CodeGen/PowerPC/anon_aggr.ll
index e899cfa4ce0..9f9eed019c1 100644
--- a/test/CodeGen/PowerPC/anon_aggr.ll
+++ b/test/CodeGen/PowerPC/anon_aggr.ll
@@ -21,7 +21,7 @@ unequal:
 }
 
 ; CHECK-LABEL: func1:
-; CHECK: cmpld {{[0-9]+}}, 4, 5
+; CHECK: cmpld {{([0-9]+,)?}}4, 5
 ; CHECK-DAG: std 4, -[[OFFSET1:[0-9]+]]
 ; CHECK-DAG: std 5, -[[OFFSET2:[0-9]+]]
 ; CHECK: ld 3, -[[OFFSET1]](1)
@@ -31,7 +31,7 @@ unequal:
 ; DARWIN32: mr
 ; DARWIN32: mr r[[REG1:[0-9]+]], r[[REGA:[0-9]+]]
 ; DARWIN32: mr r[[REG2:[0-9]+]], r[[REGB:[0-9]+]]
-; DARWIN32: cmplw cr{{[0-9]+}}, r[[REGA]], r[[REGB]]
+; DARWIN32: cmplw {{(cr[0-9]+,)?}}r[[REGA]], r[[REGB]]
 ; DARWIN32: stw r[[REG1]], -[[OFFSET1:[0-9]+]]
 ; DARWIN32: stw r[[REG2]], -[[OFFSET2:[0-9]+]]
 ; DARWIN32: lwz r3, -[[OFFSET1]]
@@ -41,7 +41,7 @@ unequal:
 ; DARWIN64: mr
 ; DARWIN64: mr r[[REG1:[0-9]+]], r[[REGA:[0-9]+]]
 ; DARWIN64: mr r[[REG2:[0-9]+]], r[[REGB:[0-9]+]]
-; DARWIN64: cmpld cr{{[0-9]+}}, r[[REGA]], r[[REGB]]
+; DARWIN64: cmpld {{(cr[0-9]+,)?}}r[[REGA]], r[[REGB]]
 ; DARWIN64: std r[[REG1]], -[[OFFSET1:[0-9]+]]
 ; DARWIN64: std r[[REG2]], -[[OFFSET2:[0-9]+]]
 ; DARWIN64: ld r3, -[[OFFSET1]]
@@ -63,7 +63,7 @@ unequal:
 
 ; CHECK-LABEL: func2:
 ; CHECK: ld [[REG2:[0-9]+]], 72(1)
-; CHECK: cmpld {{[0-9]+}}, 4, [[REG2]]
+; CHECK: cmpld {{([0-9]+,)?}}4, [[REG2]]
 ; CHECK-DAG: std [[REG2]], -[[OFFSET1:[0-9]+]]
 ; CHECK-DAG: std 4, -[[OFFSET2:[0-9]+]]
 ; CHECK: ld 3, -[[OFFSET2]](1)
@@ -74,7 +74,7 @@ unequal:
 ; DARWIN32: lwz r[[REG2:[0-9]+]], 44(r[[REGSP]])
 ; DARWIN32: mr
 ; DARWIN32: mr r[[REG3:[0-9]+]], r[[REGA:[0-9]+]]
-; DARWIN32: cmplw cr{{[0-9]+}}, r[[REGA]], r[[REG2]]
+; DARWIN32: cmplw {{(cr[0-9]+,)?}}r[[REGA]], r[[REG2]]
 ; DARWIN32: stw r[[REG3]], -[[OFFSET1:[0-9]+]]
 ; DARWIN32: stw r[[REG2]], -[[OFFSET2:[0-9]+]]
 ; DARWIN32: lwz r3, -[[OFFSET1]]
@@ -84,7 +84,7 @@ unequal:
 ; DARWIN64: ld r[[REG2:[0-9]+]], 72(r1)
 ; DARWIN64: mr
 ; DARWIN64: mr r[[REG3:[0-9]+]], r[[REGA:[0-9]+]]
-; DARWIN64: cmpld cr{{[0-9]+}}, r[[REGA]], r[[REG2]]
+; DARWIN64: cmpld {{(cr[0-9]+,)?}}r[[REGA]], r[[REG2]]
 ; DARWIN64: std r[[REG3]], -[[OFFSET1:[0-9]+]]
 ; DARWIN64: std r[[REG2]], -[[OFFSET2:[0-9]+]]
 ; DARWIN64: ld r3, -[[OFFSET1]]
@@ -108,7 +108,7 @@ unequal:
 ; CHECK-LABEL: func3:
 ; CHECK: ld [[REG3:[0-9]+]], 72(1)
 ; CHECK: ld [[REG4:[0-9]+]], 56(1)
-; CHECK: cmpld {{[0-9]+}}, [[REG4]], [[REG3]]
+; CHECK: cmpld {{([0-9]+,)?}}[[REG4]], [[REG3]]
 ; CHECK: std [[REG3]], -[[OFFSET1:[0-9]+]](1)
 ; CHECK: std [[REG4]], -[[OFFSET2:[0-9]+]](1)
 ; CHECK: ld 3, -[[OFFSET2]](1)
@@ -119,7 +119,7 @@ unequal:
 ; DARWIN32: addi r[[REG2:[0-9]+]], r[[REGSP]], 24
 ; DARWIN32: lwz r[[REG3:[0-9]+]], 44(r[[REGSP]])
 ; DARWIN32: lwz r[[REG4:[0-9]+]], 32(r[[REGSP]])
-; DARWIN32: cmplw cr{{[0-9]+}}, r[[REG4]], r[[REG3]]
+; DARWIN32: cmplw {{(cr[0-9]+,)?}}r[[REG4]], r[[REG3]]
 ; DARWIN32: stw r[[REG3]], -[[OFFSET1:[0-9]+]]
 ; DARWIN32: stw r[[REG4]], -[[OFFSET2:[0-9]+]]
 ; DARWIN32: lwz r3, -[[OFFSET2]]
@@ -128,7 +128,7 @@ unequal:
 ; DARWIN64: _func3:
 ; DARWIN64: ld r[[REG3:[0-9]+]], 72(r1)
 ; DARWIN64: ld r[[REG4:[0-9]+]], 56(r1)
-; DARWIN64: cmpld cr{{[0-9]+}}, r[[REG4]], r[[REG3]]
+; DARWIN64: cmpld {{(cr[0-9]+,)?}}r[[REG4]], r[[REG3]]
 ; DARWIN64: std r[[REG3]], -[[OFFSET1:[0-9]+]]
 ; DARWIN64: std r[[REG4]], -[[OFFSET2:[0-9]+]]
 ; DARWIN64: ld r3, -[[OFFSET2]]
@@ -153,7 +153,7 @@ unequal:
 ; CHECK-LABEL: func4:
 ; CHECK: ld [[REG3:[0-9]+]], 136(1)
 ; CHECK: ld [[REG2:[0-9]+]], 120(1)
-; CHECK: cmpld {{[0-9]+}}, [[REG2]], [[REG3]]
+; CHECK: cmpld {{([0-9]+,)?}}[[REG2]], [[REG3]]
 ; CHECK: std [[REG3]], -[[OFFSET2:[0-9]+]](1)
 ; CHECK: std [[REG2]], -[[OFFSET1:[0-9]+]](1)
 ; CHECK: ld 3, -[[OFFSET1]](1)
@@ -164,7 +164,7 @@ unequal:
 ; DARWIN32: addi r[[REG1:[0-9]+]], r1, 100
 ; DARWIN32: lwz r[[REG3:[0-9]+]], 108(r1)
 ; DARWIN32: mr r[[REG2:[0-9]+]], r[[REG4]]
-; DARWIN32: cmplw cr{{[0-9]+}}, r[[REG4]], r[[REG3]]
+; DARWIN32: cmplw {{(cr[0-9]+,)?}}r[[REG4]], r[[REG3]]
 ; DARWIN32: stw r[[REG2]], -[[OFFSET1:[0-9]+]]
 ; DARWIN32: stw r[[REG3]], -[[OFFSET2:[0-9]+]]
 ; DARWIN32: lwz r[[REG1]], -[[OFFSET1]]
@@ -174,7 +174,7 @@ unequal:
 ; DARWIN64: ld r[[REG2:[0-9]+]], 120(r1)
 ; DARWIN64: ld r[[REG3:[0-9]+]], 136(r1)
 ; DARWIN64: mr r[[REG4:[0-9]+]], r[[REG2]]
-; DARWIN64: cmpld cr{{[0-9]+}}, r[[REG2]], r[[REG3]]
+; DARWIN64: cmpld {{(cr[0-9]+,)?}}r[[REG2]], r[[REG3]]
 ; DARWIN64: std r[[REG4]], -[[OFFSET1:[0-9]+]]
 ; DARWIN64: std r[[REG3]], -[[OFFSET2:[0-9]+]]
 ; DARWIN64: ld r3, -[[OFFSET1]]
diff --git a/test/CodeGen/PowerPC/asm-constraints.ll b/test/CodeGen/PowerPC/asm-constraints.ll
index 1cb00ca30fb..2d9b0eb591d 100644
--- a/test/CodeGen/PowerPC/asm-constraints.ll
+++ b/test/CodeGen/PowerPC/asm-constraints.ll
@@ -31,7 +31,7 @@ entry:
 
 ; CHECK-LABEL: @foo
 ; CHECK: ld [[REG:[0-9]+]], 0(4)
-; CHECK: cmpw 0, [[REG]], [[REG]]
+; CHECK: cmpw [[REG]], [[REG]]
 ; CHECK: bne- 0, .Ltmp[[TMP:[0-9]+]]
 ; CHECK: .Ltmp[[TMP]]:
 ; CHECK: isync
diff --git a/test/CodeGen/PowerPC/atomic-2.ll b/test/CodeGen/PowerPC/atomic-2.ll
index b4b95a2baab..1857d5d697e 100644
--- a/test/CodeGen/PowerPC/atomic-2.ll
+++ b/test/CodeGen/PowerPC/atomic-2.ll
@@ -84,7 +84,7 @@ define void @atomic_store(i64* %mem, i64 %val) nounwind {
 entry:
 ; CHECK: @atomic_store
   store atomic i64 %val, i64* %mem release, align 64
-; CHECK: sync 1
+; CHECK: lwsync
 ; CHECK-NOT: stdcx
 ; CHECK: std
   ret void
@@ -96,7 +96,7 @@ entry:
   %tmp = load atomic i64, i64* %mem acquire, align 64
 ; CHECK-NOT: ldarx
 ; CHECK: ld
-; CHECK: sync 1
+; CHECK: lwsync
   ret i64 %tmp
 }
 
diff --git a/test/CodeGen/PowerPC/atomics-fences.ll b/test/CodeGen/PowerPC/atomics-fences.ll
index 862bd173fda..c015fa6eefb 100644
--- a/test/CodeGen/PowerPC/atomics-fences.ll
+++ b/test/CodeGen/PowerPC/atomics-fences.ll
@@ -5,24 +5,23 @@
 ; Fences
 define void @fence_acquire() {
 ; CHECK-LABEL: fence_acquire
-; CHECK: sync 1
-; PPC440-NOT: sync 1
+; CHECK: lwsync
+; PPC440-NOT: lwsync
 ; PPC440: msync
   fence acquire
   ret void
 }
 define void @fence_release() {
 ; CHECK-LABEL: fence_release
-; CHECK: sync 1
-; PPC440-NOT: sync 1
+; CHECK: lwsync
+; PPC440-NOT: lwsync
 ; PPC440: msync
   fence release
   ret void
 }
 define void @fence_seq_cst() {
 ; CHECK-LABEL: fence_seq_cst
-; CHECK: sync 0
-; PPC440-NOT: sync 0
+; CHECK: sync
 ; PPC440: msync
   fence seq_cst
   ret void
diff --git a/test/CodeGen/PowerPC/atomics-indexed.ll b/test/CodeGen/PowerPC/atomics-indexed.ll
index b9ec3c669ba..7a0dde034d6 100644
--- a/test/CodeGen/PowerPC/atomics-indexed.ll
+++ b/test/CodeGen/PowerPC/atomics-indexed.ll
@@ -9,9 +9,9 @@
 ; Indexed version of loads
 define i8 @load_x_i8_seq_cst([100000 x i8]* %mem) {
 ; CHECK-LABEL: load_x_i8_seq_cst
-; CHECK: sync 0
+; CHECK: sync
 ; CHECK: lbzx
-; CHECK: sync 1
+; CHECK: lwsync
   %ptr = getelementptr inbounds [100000 x i8], [100000 x i8]* %mem, i64 0, i64 90000
   %val = load atomic i8, i8* %ptr seq_cst, align 1
   ret i8 %val
@@ -19,7 +19,7 @@ define i8 @load_x_i8_seq_cst([100000 x i8]* %mem) {
 define i16 @load_x_i16_acquire([100000 x i16]* %mem) {
 ; CHECK-LABEL: load_x_i16_acquire
 ; CHECK: lhzx
-; CHECK: sync 1
+; CHECK: lwsync
   %ptr = getelementptr inbounds [100000 x i16], [100000 x i16]* %mem, i64 0, i64 90000
   %val = load atomic i16, i16* %ptr acquire, align 2
   ret i16 %val
@@ -46,7 +46,7 @@ define i64 @load_x_i64_unordered([100000 x i64]* %mem) {
 ; Indexed version of stores
 define void @store_x_i8_seq_cst([100000 x i8]* %mem) {
 ; CHECK-LABEL: store_x_i8_seq_cst
-; CHECK: sync 0
+; CHECK: sync
 ; CHECK: stbx
   %ptr = getelementptr inbounds [100000 x i8], [100000 x i8]* %mem, i64 0, i64 90000
   store atomic i8 42, i8* %ptr seq_cst, align 1
@@ -54,7 +54,7 @@ define void @store_x_i8_seq_cst([100000 x i8]* %mem) {
 }
 define void @store_x_i16_release([100000 x i16]* %mem) {
 ; CHECK-LABEL: store_x_i16_release
-; CHECK: sync 1
+; CHECK: lwsync
 ; CHECK: sthx
   %ptr = getelementptr inbounds [100000 x i16], [100000 x i16]* %mem, i64 0, i64 90000
   store atomic i16 42, i16* %ptr release, align 2
@@ -70,8 +70,7 @@ define void @store_x_i32_monotonic([100000 x i32]* %mem) {
 }
 define void @store_x_i64_unordered([100000 x i64]* %mem) {
 ; CHECK-LABEL: store_x_i64_unordered
-; CHECK-NOT: sync 0
-; CHECK-NOT: sync 1
+; CHECK-NOT: sync
 ; PPC32: __sync_
 ; PPC64-NOT: __sync_
 ; PPC64: stdx
diff --git a/test/CodeGen/PowerPC/atomics.ll b/test/CodeGen/PowerPC/atomics.ll
index fe4791ec193..2e1eff0f634 100644
--- a/test/CodeGen/PowerPC/atomics.ll
+++ b/test/CodeGen/PowerPC/atomics.ll
@@ -27,17 +27,17 @@ define i32 @load_i32_acquire(i32* %mem) {
 ; CHECK-LABEL: load_i32_acquire
 ; CHECK: lwz
   %val = load atomic i32, i32* %mem acquire, align 4
-; CHECK: sync 1
+; CHECK: lwsync
   ret i32 %val
 }
 define i64 @load_i64_seq_cst(i64* %mem) {
 ; CHECK-LABEL: load_i64_seq_cst
-; CHECK: sync 0
+; CHECK: sync
 ; PPC32: __sync_
 ; PPC64-NOT: __sync_
 ; PPC64: ld
   %val = load atomic i64, i64* %mem seq_cst, align 8
-; CHECK: sync 1
+; CHECK: lwsync
   ret i64 %val
 }
 
@@ -58,14 +58,14 @@ define void @store_i16_monotonic(i16* %mem) {
 }
 define void @store_i32_release(i32* %mem) {
 ; CHECK-LABEL: store_i32_release
-; CHECK: sync 1
+; CHECK: lwsync
 ; CHECK: stw
   store atomic i32 42, i32* %mem release, align 4
   ret void
 }
 define void @store_i64_seq_cst(i64* %mem) {
 ; CHECK-LABEL: store_i64_seq_cst
-; CHECK: sync 0
+; CHECK: sync
 ; PPC32: __sync_
 ; PPC64-NOT: __sync_
 ; PPC64: std
@@ -76,9 +76,9 @@ define void @store_i64_seq_cst(i64* %mem) {
 ; Atomic CmpXchg
 define i8 @cas_strong_i8_sc_sc(i8* %mem) {
 ; CHECK-LABEL: cas_strong_i8_sc_sc
-; CHECK: sync 0
+; CHECK: sync
   %val = cmpxchg i8* %mem, i8 0, i8 1 seq_cst seq_cst
-; CHECK: sync 1
+; CHECK: lwsync
   %loaded = extractvalue { i8, i1} %val, 0
   ret i8 %loaded
 }
@@ -86,21 +86,21 @@ define i16 @cas_weak_i16_acquire_acquire(i16* %mem) {
 ; CHECK-LABEL: cas_weak_i16_acquire_acquire
 ;CHECK-NOT: sync
   %val = cmpxchg weak i16* %mem, i16 0, i16 1 acquire acquire
-; CHECK: sync 1
+; CHECK: lwsync
   %loaded = extractvalue { i16, i1} %val, 0
   ret i16 %loaded
 }
 define i32 @cas_strong_i32_acqrel_acquire(i32* %mem) {
 ; CHECK-LABEL: cas_strong_i32_acqrel_acquire
-; CHECK: sync 1
+; CHECK: lwsync
   %val = cmpxchg i32* %mem, i32 0, i32 1 acq_rel acquire
-; CHECK: sync 1
+; CHECK: lwsync
   %loaded = extractvalue { i32, i1} %val, 0
   ret i32 %loaded
 }
 define i64 @cas_weak_i64_release_monotonic(i64* %mem) {
 ; CHECK-LABEL: cas_weak_i64_release_monotonic
-; CHECK: sync 1
+; CHECK: lwsync
   %val = cmpxchg weak i64* %mem, i64 0, i64 1 release monotonic
 ; CHECK-NOT: [sync ]
   %loaded = extractvalue { i64, i1} %val, 0
@@ -116,21 +116,21 @@ define i8 @add_i8_monotonic(i8* %mem, i8 %operand) {
 }
 define i16 @xor_i16_seq_cst(i16* %mem, i16 %operand) {
 ; CHECK-LABEL: xor_i16_seq_cst
-; CHECK: sync 0
+; CHECK: sync
   %val = atomicrmw xor i16* %mem, i16 %operand seq_cst
-; CHECK: sync 1
+; CHECK: lwsync
   ret i16 %val
 }
 define i32 @xchg_i32_acq_rel(i32* %mem, i32 %operand) {
 ; CHECK-LABEL: xchg_i32_acq_rel
-; CHECK: sync 1
+; CHECK: lwsync
   %val = atomicrmw xchg i32* %mem, i32 %operand acq_rel
-; CHECK: sync 1
+; CHECK: lwsync
   ret i32 %val
 }
 define i64 @and_i64_release(i64* %mem, i64 %operand) {
 ; CHECK-LABEL: and_i64_release
-; CHECK: sync 1
+; CHECK: lwsync
   %val = atomicrmw and i64* %mem, i64 %operand release
 ; CHECK-NOT: [sync ]
   ret i64 %val
diff --git a/test/CodeGen/PowerPC/bperm.ll b/test/CodeGen/PowerPC/bperm.ll
index c489c1f90a8..c5a728859a0 100644
--- a/test/CodeGen/PowerPC/bperm.ll
+++ b/test/CodeGen/PowerPC/bperm.ll
@@ -22,15 +22,15 @@ entry:
   ret i64 %0
 
 ; CHECK-LABEL: @bs8
-; CHECK-DAG: rldicl [[REG1:[0-9]+]], 3, 16, 0
-; CHECK-DAG: rldicl [[REG2:[0-9]+]], 3, 8, 0
-; CHECK-DAG: rldicl [[REG3:[0-9]+]], 3, 24, 0
+; CHECK-DAG: rotldi [[REG1:[0-9]+]], 3, 16
+; CHECK-DAG: rotldi [[REG2:[0-9]+]], 3, 8
+; CHECK-DAG: rotldi [[REG3:[0-9]+]], 3, 24
 ; CHECK-DAG: rldimi [[REG2]], [[REG1]], 8, 48
-; CHECK-DAG: rldicl [[REG4:[0-9]+]], 3, 32, 0
+; CHECK-DAG: rotldi [[REG4:[0-9]+]], 3, 32
 ; CHECK-DAG: rldimi [[REG2]], [[REG3]], 16, 40
-; CHECK-DAG: rldicl [[REG5:[0-9]+]], 3, 48, 0
+; CHECK-DAG: rotldi [[REG5:[0-9]+]], 3, 48
 ; CHECK-DAG: rldimi [[REG2]], [[REG4]], 24, 32
-; CHECK-DAG: rldicl [[REG6:[0-9]+]], 3, 56, 0
+; CHECK-DAG: rotldi [[REG6:[0-9]+]], 3, 56
 ; CHECK-DAG: rldimi [[REG2]], [[REG5]], 40, 16
 ; CHECK-DAG: rldimi [[REG2]], [[REG6]], 48, 8
 ; CHECK-DAG: rldimi [[REG2]], 3, 56, 0
@@ -46,7 +46,7 @@ entry:
 
 ; CHECK-LABEL: @test1
 ; CHECK-DAG: li [[REG1:[0-9]+]], 11375
-; CHECK-DAG: rldicl [[REG3:[0-9]+]], 4, 56, 0
+; CHECK-DAG: rotldi [[REG3:[0-9]+]], 4, 56
 ; CHECK-DAG: sldi [[REG2:[0-9]+]], [[REG1]], 19
 ; CHECK: and 3, [[REG3]], [[REG2]]
 ; CHECK: blr
@@ -60,7 +60,7 @@ entry:
 
 ; CHECK-LABEL: @test2
 ; CHECK-DAG: lis [[REG1:[0-9]+]], 474
-; CHECK-DAG: rldicl [[REG5:[0-9]+]], 4, 58, 0
+; CHECK-DAG: rotldi [[REG5:[0-9]+]], 4, 58
 ; CHECK-DAG: ori [[REG2:[0-9]+]], [[REG1]], 3648
 ; CHECK-DAG: sldi [[REG3:[0-9]+]], [[REG2]], 32
 ; CHECK-DAG: oris [[REG4:[0-9]+]], [[REG3]], 25464
@@ -76,7 +76,7 @@ entry:
 
 ; CHECK-LABEL: @test3
 ; CHECK-DAG: lis [[REG1:[0-9]+]], 170
-; CHECK-DAG: rldicl [[REG4:[0-9]+]], 3, 34, 0
+; CHECK-DAG: rotldi [[REG4:[0-9]+]], 3, 34
 ; CHECK-DAG: ori [[REG2:[0-9]+]], [[REG1]], 22861
 ; CHECK-DAG: sldi [[REG3:[0-9]+]], [[REG2]], 34
 ; CHECK: and 3, [[REG4]], [[REG3]]
@@ -90,7 +90,7 @@ entry:
   ret i64 %and
 
 ; CHECK-LABEL: @test4
-; CHECK: rldicl [[REG1:[0-9]+]], 4, 49, 0
+; CHECK: rotldi [[REG1:[0-9]+]], 4, 49
 ; CHECK: andis. 3, [[REG1]], 888
 ; CHECK: blr
 }
@@ -103,7 +103,7 @@ entry:
 
 ; CHECK-LABEL: @test5
 ; CHECK-DAG: lis [[REG1:[0-9]+]], 3703
-; CHECK-DAG: rldicl [[REG4:[0-9]+]], 4, 12, 0
+; CHECK-DAG: rotldi [[REG4:[0-9]+]], 4, 12
 ; CHECK-DAG: ori [[REG2:[0-9]+]], [[REG1]], 35951
 ; CHECK-DAG: sldi [[REG3:[0-9]+]], [[REG2]], 19
 ; CHECK: and 3, [[REG4]], [[REG3]]
@@ -148,7 +148,7 @@ entry:
 
 ; CHECK-LABEL: @test8
 ; CHECK-DAG: lis [[REG1:[0-9]+]], 4
-; CHECK-DAG: rldicl [[REG4:[0-9]+]], 3, 63, 0
+; CHECK-DAG: rotldi [[REG4:[0-9]+]], 3, 63
 ; CHECK-DAG: ori [[REG2:[0-9]+]], [[REG1]], 60527
 ; CHECK-DAG: sldi [[REG3:[0-9]+]], [[REG2]], 19
 ; CHECK: and 3, [[REG4]], [[REG3]]
@@ -166,8 +166,8 @@ entry:
 
 ; CHECK-LABEL: @test9
 ; CHECK-DAG: lis [[REG1:[0-9]+]], 1440
-; CHECK-DAG: rldicl [[REG5:[0-9]+]], 4, 62, 0
-; CHECK-DAG: rldicl [[REG6:[0-9]+]], 4, 50, 0
+; CHECK-DAG: rotldi [[REG5:[0-9]+]], 4, 62
+; CHECK-DAG: rotldi [[REG6:[0-9]+]], 4, 50
 ; CHECK-DAG: ori [[REG2:[0-9]+]], [[REG1]], 4
 ; CHECK-DAG: rldimi [[REG6]], [[REG5]], 53, 0
 ; CHECK-DAG: sldi [[REG3:[0-9]+]], [[REG2]], 32
@@ -187,8 +187,8 @@ entry:
 
 ; CHECK-LABEL: @test10
 ; CHECK-DAG: lis [[REG1:[0-9]+]], 1
-; CHECK-DAG: rldicl [[REG6:[0-9]+]], 3, 25, 0
-; CHECK-DAG: rldicl [[REG7:[0-9]+]], 3, 37, 0
+; CHECK-DAG: rotldi [[REG6:[0-9]+]], 3, 25
+; CHECK-DAG: rotldi [[REG7:[0-9]+]], 3, 37
 ; CHECK-DAG: ori [[REG2:[0-9]+]], [[REG1]], 8183
 ; CHECK-DAG: ori [[REG3:[0-9]+]], [[REG1]], 50017
 ; CHECK-DAG: sldi [[REG4:[0-9]+]], [[REG2]], 25
diff --git a/test/CodeGen/PowerPC/builtins-ppc-elf2-abi.ll b/test/CodeGen/PowerPC/builtins-ppc-elf2-abi.ll
new file mode 100644
index 00000000000..6013a412924
--- /dev/null
+++ b/test/CodeGen/PowerPC/builtins-ppc-elf2-abi.ll
@@ -0,0 +1,195 @@
+; RUN: llc -mtriple=powerpc64le-unknown-linux-gnu -mcpu=pwr8 < %s | FileCheck %s
+; RUN: llc -mtriple=powerpc64-unknown-linux-gnu -mcpu=pwr8 < %s | FileCheck %s
+
+@vda = common global <2 x double> zeroinitializer, align 16
+@vdb = common global <2 x double> zeroinitializer, align 16
+@vdr = common global <2 x double> zeroinitializer, align 16
+@vfa = common global <4 x float> zeroinitializer, align 16
+@vfb = common global <4 x float> zeroinitializer, align 16
+@vfr = common global <4 x float> zeroinitializer, align 16
+@vbllr = common global <2 x i64> zeroinitializer, align 16
+@vbir = common global <4 x i32> zeroinitializer, align 16
+@vblla = common global <2 x i64> zeroinitializer, align 16
+@vbllb = common global <2 x i64> zeroinitializer, align 16
+@vbia = common global <4 x i32> zeroinitializer, align 16
+@vbib = common global <4 x i32> zeroinitializer, align 16
+
+; Function Attrs: nounwind
+define void @test1() {
+entry:
+  %0 = load <2 x double>, <2 x double>* @vda, align 16
+  %1 = load <2 x double>, <2 x double>* @vdb, align 16
+  %2 = call <2 x double> @llvm.ppc.vsx.xvdivdp(<2 x double> %0, <2 x double> %1)
+  store <2 x double> %2, <2 x double>* @vdr, align 16
+  ret void
+; CHECK-LABEL: @test1
+; CHECK: xvdivdp {{[0-9]+}}, {{[0-9]+}}, {{[0-9]+}}
+}
+
+; Function Attrs: nounwind
+define void @test2() {
+entry:
+  %0 = load <4 x float>, <4 x float>* @vfa, align 16
+  %1 = load <4 x float>, <4 x float>* @vfb, align 16
+  %2 = call <4 x float> @llvm.ppc.vsx.xvdivsp(<4 x float> %0, <4 x float> %1)
+  store <4 x float> %2, <4 x float>* @vfr, align 16
+  ret void
+; CHECK-LABEL: @test2
+; CHECK: xvdivsp {{[0-9]+}}, {{[0-9]+}}, {{[0-9]+}}
+}
+
+; Function Attrs: nounwind
+define void @test3() {
+entry:
+  %0 = load <2 x double>, <2 x double>* @vda, align 16
+  %1 = load <2 x double>, <2 x double>* @vda, align 16
+  %2 = call <2 x double> @llvm.ceil.v2f64(<2 x double> %1)
+  store <2 x double> %2, <2 x double>* @vdr, align 16
+  ret void
+; CHECK-LABEL: @test3
+; CHECK: xvrdpip {{[0-9]+}}, {{[0-9]+}}
+}
+
+; Function Attrs: nounwind
+define void @test4() {
+entry:
+  %0 = load <4 x float>, <4 x float>* @vfa, align 16
+  %1 = load <4 x float>, <4 x float>* @vfa, align 16
+  %2 = call <4 x float> @llvm.ceil.v4f32(<4 x float> %1)
+  store <4 x float> %2, <4 x float>* @vfr, align 16
+  ret void
+; CHECK-LABEL: @test4
+; CHECK: xvrspip {{[0-9]+}}, {{[0-9]+}}
+}
+
+; Function Attrs: nounwind
+define void @test5() {
+entry:
+  %0 = load <2 x double>, <2 x double>* @vda, align 16
+  %1 = load <2 x double>, <2 x double>* @vdb, align 16
+  %2 = call <2 x i64> @llvm.ppc.vsx.xvcmpeqdp(<2 x double> %0, <2 x double> %1)
+  store <2 x i64> %2, <2 x i64>* @vbllr, align 16
+  ret void
+; CHECK-LABEL: @test5
+; CHECK: xvcmpeqdp {{[0-9]+}}, {{[0-9]+}}, {{[0-9]+}}
+}
+
+; Function Attrs: nounwind
+define void @test6() {
+entry:
+  %0 = load <4 x float>, <4 x float>* @vfa, align 16
+  %1 = load <4 x float>, <4 x float>* @vfb, align 16
+  %2 = call <4 x i32> @llvm.ppc.vsx.xvcmpeqsp(<4 x float> %0, <4 x float> %1)
+  store <4 x i32> %2, <4 x i32>* @vbir, align 16
+  ret void
+; CHECK-LABEL: @test6
+; CHECK: xvcmpeqsp {{[0-9]+}}, {{[0-9]+}}, {{[0-9]+}}
+}
+
+; Function Attrs: nounwind
+define void @test7() {
+entry:
+  %0 = load <2 x double>, <2 x double>* @vda, align 16
+  %1 = load <2 x double>, <2 x double>* @vdb, align 16
+  %2 = call <2 x i64> @llvm.ppc.vsx.xvcmpgedp(<2 x double> %0, <2 x double> %1)
+  store <2 x i64> %2, <2 x i64>* @vbllr, align 16
+  ret void
+; CHECK-LABEL: @test7
+; CHECK: xvcmpgedp {{[0-9]+}}, {{[0-9]+}}, {{[0-9]+}}
+}
+
+; Function Attrs: nounwind
+define void @test8() {
+entry:
+  %0 = load <4 x float>, <4 x float>* @vfa, align 16
+  %1 = load <4 x float>, <4 x float>* @vfb, align 16
+  %2 = call <4 x i32> @llvm.ppc.vsx.xvcmpgesp(<4 x float> %0, <4 x float> %1)
+  store <4 x i32> %2, <4 x i32>* @vbir, align 16
+  ret void
+; CHECK-LABEL: @test8
+; CHECK: xvcmpgesp {{[0-9]+}}, {{[0-9]+}}, {{[0-9]+}}
+}
+
+; Function Attrs: nounwind
+define void @test9() {
+entry:
+  %0 = load <2 x double>, <2 x double>* @vda, align 16
+  %1 = load <2 x double>, <2 x double>* @vdb, align 16
+  %2 = call <2 x i64> @llvm.ppc.vsx.xvcmpgtdp(<2 x double> %0, <2 x double> %1)
+  store <2 x i64> %2, <2 x i64>* @vbllr, align 16
+  ret void
+; CHECK-LABEL: @test9
+; CHECK: xvcmpgtdp {{[0-9]+}}, {{[0-9]+}}, {{[0-9]+}}
+}
+
+; Function Attrs: nounwind
+define void @test10() {
+entry:
+  %0 = load <4 x float>, <4 x float>* @vfa, align 16
+  %1 = load <4 x float>, <4 x float>* @vfb, align 16
+  %2 = call <4 x i32> @llvm.ppc.vsx.xvcmpgtsp(<4 x float> %0, <4 x float> %1)
+  store <4 x i32> %2, <4 x i32>* @vbir, align 16
+  ret void
+; CHECK-LABEL: @test10
+; CHECK: xvcmpgtsp {{[0-9]+}}, {{[0-9]+}}, {{[0-9]+}}
+}
+
+; Function Attrs: nounwind
+define <4 x float> @emit_xvresp(<4 x float> %a) {
+entry:
+  %a.addr = alloca <4 x float>, align 16
+  store <4 x float> %a, <4 x float>* %a.addr, align 16
+  %0 = load <4 x float>, <4 x float>* %a.addr, align 16
+  %1 = call <4 x float> @llvm.ppc.vsx.xvresp(<4 x float> %0)
+  ret <4 x float> %1
+; CHECK-LABEL: @emit_xvresp
+; CHECK: xvresp {{[0-9]+}}, {{[0-9]+}}
+}
+
+; Function Attrs: nounwind
+define <2 x double> @emit_xvredp(<2 x double> %a) {
+entry:
+  %a.addr = alloca <2 x double>, align 16
+  store <2 x double> %a, <2 x double>* %a.addr, align 16
+  %0 = load <2 x double>, <2 x double>* %a.addr, align 16
+  %1 = call <2 x double> @llvm.ppc.vsx.xvredp(<2 x double> %0)
+  ret <2 x double> %1
+; CHECK-LABEL: @emit_xvredp
+; CHECK: xvredp {{[0-9]+}}, {{[0-9]+}}
+}
+
+; Function Attrs: nounwind readnone
+declare <4 x float> @llvm.ppc.vsx.xvresp(<4 x float>)
+
+; Function Attrs: nounwind readnone
+declare <2 x double> @llvm.ppc.vsx.xvredp(<2 x double>)
+
+; Function Attrs: nounwind readnone
+declare <2 x double> @llvm.ceil.v2f64(<2 x double>)
+
+; Function Attrs: nounwind readnone
+declare <4 x float> @llvm.ceil.v4f32(<4 x float>)
+
+; Function Attrs: nounwind readnone
+declare <2 x double> @llvm.ppc.vsx.xvdivdp(<2 x double>, <2 x double>)
+
+; Function Attrs: nounwind readnone
+declare <4 x float> @llvm.ppc.vsx.xvdivsp(<4 x float>, <4 x float>)
+
+; Function Attrs: nounwind readnone
+declare <2 x i64> @llvm.ppc.vsx.xvcmpeqdp(<2 x double>, <2 x double>)
+
+; Function Attrs: nounwind readnone
+declare <4 x i32> @llvm.ppc.vsx.xvcmpeqsp(<4 x float>, <4 x float>)
+
+; Function Attrs: nounwind readnone
+declare <2 x i64> @llvm.ppc.vsx.xvcmpgedp(<2 x double>, <2 x double>)
+
+; Function Attrs: nounwind readnone
+declare <4 x i32> @llvm.ppc.vsx.xvcmpgesp(<4 x float>, <4 x float>)
+
+; Function Attrs: nounwind readnone
+declare <2 x i64> @llvm.ppc.vsx.xvcmpgtdp(<2 x double>, <2 x double>)
+
+; Function Attrs: nounwind readnone
+declare <4 x i32> @llvm.ppc.vsx.xvcmpgtsp(<4 x float>, <4 x float>)
diff --git a/test/CodeGen/PowerPC/builtins-ppc-p8vector.ll b/test/CodeGen/PowerPC/builtins-ppc-p8vector.ll
new file mode 100644
index 00000000000..37111ef0d89
--- /dev/null
+++ b/test/CodeGen/PowerPC/builtins-ppc-p8vector.ll
@@ -0,0 +1,91 @@
+; RUN: llc -mtriple=powerpc64-unknown-linux-gnu -mcpu=pwr8 -mattr=-vsx < %s | FileCheck %s
+; RUN: llc -mtriple=powerpc64le-unknown-linux-gnu -mcpu=pwr8 -mattr=-vsx < %s | FileCheck %s
+; RUN: llc -mtriple=powerpc64-unknown-linux-gnu -mcpu=pwr7 -mattr=+power8-vector -mattr=-vsx < %s | FileCheck %s
+; RUN: llc -mtriple=powerpc64le-unknown-linux-gnu -mcpu=pwr8 < %s | FileCheck %s -check-prefix=CHECK-VSX
+
+@vsc = global <16 x i8> <i8 0, i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 8, i8 9, i8 0, i8 1, i8 2, i8 3, i8 4, i8 5>, align 16
+@vuc = global <16 x i8> <i8 0, i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 8, i8 9, i8 0, i8 1, i8 2, i8 3, i8 4, i8 5>, align 16
+@res_vll = common global <2 x i64> zeroinitializer, align 16
+@res_vull = common global <2 x i64> zeroinitializer, align 16
+@res_vsc = common global <16 x i8> zeroinitializer, align 16
+@res_vuc = common global <16 x i8> zeroinitializer, align 16
+
+; Function Attrs: nounwind
+define void @test1() {
+entry:
+  %__a.addr.i = alloca <16 x i8>, align 16
+  %__b.addr.i = alloca <16 x i8>, align 16
+  %0 = load <16 x i8>, <16 x i8>* @vsc, align 16
+  %1 = load <16 x i8>, <16 x i8>* @vsc, align 16
+  store <16 x i8> %0, <16 x i8>* %__a.addr.i, align 16
+  store <16 x i8> %1, <16 x i8>* %__b.addr.i, align 16
+  %2 = load <16 x i8>, <16 x i8>* %__a.addr.i, align 16
+  %3 = load <16 x i8>, <16 x i8>* %__b.addr.i, align 16
+  %4 = call <2 x i64> @llvm.ppc.altivec.vbpermq(<16 x i8> %2, <16 x i8> %3)
+  store <2 x i64> %4, <2 x i64>* @res_vll, align 16
+  ret void
+; CHECK-LABEL: @test1
+; CHECK: lvx [[REG1:[0-9]+]],
+; CHECK: lvx [[REG2:[0-9]+]],
+; CHECK: vbpermq {{[0-9]+}}, [[REG2]], [[REG1]]
+; CHECK-VSX: vbpermq {{[0-9]+}}, {{[0-9]+}}, {{[0-9]+}}
+}
+
+; Function Attrs: nounwind
+define void @test2() {
+entry:
+  %__a.addr.i = alloca <16 x i8>, align 16
+  %__b.addr.i = alloca <16 x i8>, align 16
+  %0 = load <16 x i8>, <16 x i8>* @vuc, align 16
+  %1 = load <16 x i8>, <16 x i8>* @vuc, align 16
+  store <16 x i8> %0, <16 x i8>* %__a.addr.i, align 16
+  store <16 x i8> %1, <16 x i8>* %__b.addr.i, align 16
+  %2 = load <16 x i8>, <16 x i8>* %__a.addr.i, align 16
+  %3 = load <16 x i8>, <16 x i8>* %__b.addr.i, align 16
+  %4 = call <2 x i64> @llvm.ppc.altivec.vbpermq(<16 x i8> %2, <16 x i8> %3)
+  store <2 x i64> %4, <2 x i64>* @res_vull, align 16
+  ret void
+; CHECK-LABEL: @test2
+; CHECK: lvx [[REG1:[0-9]+]],
+; CHECK: lvx [[REG2:[0-9]+]],
+; CHECK: vbpermq {{[0-9]+}}, [[REG2]], [[REG1]]
+; CHECK-VSX: vbpermq {{[0-9]+}}, {{[0-9]+}}, {{[0-9]+}}
+}
+
+; Function Attrs: nounwind
+define void @test3() {
+entry:
+  %__a.addr.i = alloca <16 x i8>, align 16
+  %0 = load <16 x i8>, <16 x i8>* @vsc, align 16
+  store <16 x i8> %0, <16 x i8>* %__a.addr.i, align 16
+  %1 = load <16 x i8>, <16 x i8>* %__a.addr.i, align 16
+  %2 = call <16 x i8> @llvm.ppc.altivec.vgbbd(<16 x i8> %1)
+  store <16 x i8> %2, <16 x i8>* @res_vsc, align 16
+  ret void
+; CHECK-LABEL: @test3
+; CHECK: lvx [[REG1:[0-9]+]],
+; CHECK: vgbbd {{[0-9]+}}, [[REG1]]
+; CHECK-VSX: vgbbd {{[0-9]+}}, {{[0-9]+}}
+}
+
+; Function Attrs: nounwind
+define void @test4() {
+entry:
+  %__a.addr.i = alloca <16 x i8>, align 16
+  %0 = load <16 x i8>, <16 x i8>* @vuc, align 16
+  store <16 x i8> %0, <16 x i8>* %__a.addr.i, align 16
+  %1 = load <16 x i8>, <16 x i8>* %__a.addr.i, align 16
+  %2 = call <16 x i8> @llvm.ppc.altivec.vgbbd(<16 x i8> %1)
+  store <16 x i8> %2, <16 x i8>* @res_vuc, align 16
+  ret void
+; CHECK-LABEL: @test4
+; CHECK: lvx [[REG1:[0-9]+]],
+; CHECK: vgbbd {{[0-9]+}}, [[REG1]]
+; CHECK-VSX: vgbbd {{[0-9]+}}, {{[0-9]+}}
+}
+
+; Function Attrs: nounwind readnone
+declare <2 x i64> @llvm.ppc.altivec.vbpermq(<16 x i8>, <16 x i8>)
+
+; Function Attrs: nounwind readnone
+declare <16 x i8> @llvm.ppc.altivec.vgbbd(<16 x i8>)
diff --git a/test/CodeGen/PowerPC/cmpb-ppc32.ll b/test/CodeGen/PowerPC/cmpb-ppc32.ll
index 639ed887b97..b5cb0935d88 100644
--- a/test/CodeGen/PowerPC/cmpb-ppc32.ll
+++ b/test/CodeGen/PowerPC/cmpb-ppc32.ll
@@ -17,7 +17,7 @@ entry:
 
 ; CHECK-LABEL: @test16
 ; CHECK: cmpb [[REG1:[0-9]+]], 4, 3
-; CHECK: rlwinm 3, [[REG1]], 0, 16, 31
+; CHECK: clrlwi 3, [[REG1]], 16
 ; CHECK: blr
 }
 
diff --git a/test/CodeGen/PowerPC/cmpb.ll b/test/CodeGen/PowerPC/cmpb.ll
index 7d0c0ab3316..d1c951df962 100644
--- a/test/CodeGen/PowerPC/cmpb.ll
+++ b/test/CodeGen/PowerPC/cmpb.ll
@@ -17,7 +17,7 @@ entry:
 
 ; CHECK-LABEL: @test16
 ; CHECK: cmpb [[REG1:[0-9]+]], 4, 3
-; CHECK: rldicl 3, [[REG1]], 0, 48
+; CHECK: clrldi 3, [[REG1]], 48
 ; CHECK: blr
 }
 
@@ -73,7 +73,7 @@ entry:
 
 ; CHECK-LABEL: @test16p3
 ; CHECK: cmpb [[REG1:[0-9]+]], 4, 3
-; CHECK: rldicl [[REG2:[0-9]+]], [[REG1]], 0, 55
+; CHECK: clrldi [[REG2:[0-9]+]], [[REG1]], 55
 ; CHECK: xori 3, [[REG2]], 1280
 ; CHECK: blr
 }
@@ -99,7 +99,7 @@ entry:
 
 ; CHECK-LABEL: @test32
 ; CHECK: cmpb [[REG1:[0-9]+]], 4, 3
-; CHECK: rldicl 3, [[REG1]], 0, 32
+; CHECK: clrldi 3, [[REG1]], 32
 ; CHECK: blr
 }
 
diff --git a/test/CodeGen/PowerPC/compare-simm.ll b/test/CodeGen/PowerPC/compare-simm.ll
index 94c5c0290f5..12eff7bb181 100644
--- a/test/CodeGen/PowerPC/compare-simm.ll
+++ b/test/CodeGen/PowerPC/compare-simm.ll
@@ -1,7 +1,9 @@
-; RUN: llc < %s -march=ppc32 -mtriple=powerpc-apple-darwin8 | \
-; RUN:   grep "cmpwi cr0, r3, -1"
+; RUN: llc < %s -march=ppc32 -mtriple=powerpc-apple-darwin8 | FileCheck %s
 
 define i32 @test(i32 %x) nounwind {
+; CHECK-LABEL: @test
+; CHECK: cmpwi r3, -1
+
         %c = icmp eq i32 %x, -1
 	br i1 %c, label %T, label %F
 T:
diff --git a/test/CodeGen/PowerPC/crbit-asm.ll b/test/CodeGen/PowerPC/crbit-asm.ll
index 373e334f02b..36de3435a08 100644
--- a/test/CodeGen/PowerPC/crbit-asm.ll
+++ b/test/CodeGen/PowerPC/crbit-asm.ll
@@ -12,7 +12,7 @@ entry:
 ; CHECK-LABEL: @testi1
 ; CHECK-DAG: andi. {{[0-9]+}}, 3, 1
 ; CHECK-DAG: li [[REG1:[0-9]+]], 0
-; CHECK-DAG: cror [[REG2:[0-9]+]], 1, 1
+; CHECK-DAG: crmove [[REG2:[0-9]+]], 1
 ; CHECK-DAG: andi. {{[0-9]+}}, 4, 1
 ; CHECK-DAG: crand [[REG3:[0-9]+]], [[REG2]], 1
 ; CHECK-DAG: li [[REG4:[0-9]+]], 1
@@ -31,7 +31,7 @@ entry:
 ; CHECK-LABEL: @testi32
 ; CHECK-DAG: andi. {{[0-9]+}}, 3, 1
 ; CHECK-DAG: li [[REG1:[0-9]+]], 0
-; CHECK-DAG: cror [[REG2:[0-9]+]], 1, 1
+; CHECK-DAG: crmove [[REG2:[0-9]+]], 1
 ; CHECK-DAG: andi. {{[0-9]+}}, 4, 1
 ; CHECK-DAG: crand [[REG3:[0-9]+]], [[REG2]], 1
 ; CHECK-DAG: li [[REG4:[0-9]+]], -1
@@ -47,7 +47,7 @@ entry:
 ; CHECK-LABEL: @testi8
 ; CHECK-DAG: andi. {{[0-9]+}}, 3, 1
 ; CHECK-DAG: li [[REG1:[0-9]+]], 0
-; CHECK-DAG: cror [[REG2:[0-9]+]], 1, 1
+; CHECK-DAG: crmove [[REG2:[0-9]+]], 1
 ; CHECK-DAG: andi. {{[0-9]+}}, 4, 1
 ; CHECK-DAG: crand [[REG3:[0-9]+]], [[REG2]], 1
 ; CHECK-DAG: li [[REG4:[0-9]+]], 1
diff --git a/test/CodeGen/PowerPC/crbits.ll b/test/CodeGen/PowerPC/crbits.ll
index 8873c1bd28c..ab8655c5c8c 100644
--- a/test/CodeGen/PowerPC/crbits.ll
+++ b/test/CodeGen/PowerPC/crbits.ll
@@ -107,7 +107,7 @@ entry:
 ; CHECK-LABEL: @test6
 ; CHECK-DAG: andi. {{[0-9]+}}, 3, 1
 ; CHECK-DAG: cmpwi {{[0-9]+}}, 5, -2
-; CHECK-DAG: cror [[REG1:[0-9]+]], 1, 1
+; CHECK-DAG: crmove [[REG1:[0-9]+]], 1
 ; CHECK-DAG: andi. {{[0-9]+}}, 4, 1
 ; CHECK-DAG: li [[REG2:[0-9]+]], 1
 ; CHECK-DAG: crorc [[REG4:[0-9]+]], 1,
diff --git a/test/CodeGen/PowerPC/cttz.ll b/test/CodeGen/PowerPC/cttz.ll
index 3757fa3e2f2..60de982d91a 100644
--- a/test/CodeGen/PowerPC/cttz.ll
+++ b/test/CodeGen/PowerPC/cttz.ll
@@ -6,7 +6,7 @@ declare i32 @llvm.cttz.i32(i32, i1)
 define i32 @bar(i32 %x) {
 entry:
 ; CHECK: @bar
-; CHECK: cntlzw
+; CHECK: cntlz
         %tmp.1 = call i32 @llvm.cttz.i32( i32 %x, i1 true )              ; <i32> [#uses=1]
         ret i32 %tmp.1
 }
diff --git a/test/CodeGen/PowerPC/dbg.ll b/test/CodeGen/PowerPC/dbg.ll
index f09fc4f6cb8..87914025b73 100644
--- a/test/CodeGen/PowerPC/dbg.ll
+++ b/test/CodeGen/PowerPC/dbg.ll
@@ -6,8 +6,8 @@ target triple = "powerpc64-unknown-linux-gnu"
 
 define i32 @main(i32 %argc, i8** nocapture %argv) nounwind readnone {
 entry:
-  tail call void @llvm.dbg.value(metadata i32 %argc, i64 0, metadata !15, metadata !MDExpression()), !dbg !17
-  tail call void @llvm.dbg.value(metadata i8** %argv, i64 0, metadata !16, metadata !MDExpression()), !dbg !18
+  tail call void @llvm.dbg.value(metadata i32 %argc, i64 0, metadata !15, metadata !DIExpression()), !dbg !17
+  tail call void @llvm.dbg.value(metadata i8** %argv, i64 0, metadata !16, metadata !DIExpression()), !dbg !18
   %add = add nsw i32 %argc, 1, !dbg !19
   ret i32 %add, !dbg !19
 }
@@ -17,23 +17,23 @@ declare void @llvm.dbg.value(metadata, i64, metadata, metadata) nounwind readnon
 !llvm.dbg.cu = !{!0}
 !llvm.module.flags = !{!22}
 
-!0 = !MDCompileUnit(language: DW_LANG_C99, producer: "clang version 3.1", isOptimized: true, emissionKind: 0, file: !21, enums: !1, retainedTypes: !1, subprograms: !3, globals: !1, imports: !1)
+!0 = !DICompileUnit(language: DW_LANG_C99, producer: "clang version 3.1", isOptimized: true, emissionKind: 0, file: !21, enums: !1, retainedTypes: !1, subprograms: !3, globals: !1, imports: !1)
 !1 = !{}
 !3 = !{!5}
-!5 = !MDSubprogram(name: "main", line: 1, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: true, file: !21, scope: null, type: !7, function: i32 (i32, i8**)* @main, variables: !13)
-!6 = !MDFile(filename: "dbg.c", directory: "/src")
-!7 = !MDSubroutineType(types: !8)
+!5 = !DISubprogram(name: "main", line: 1, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: true, file: !21, scope: null, type: !7, function: i32 (i32, i8**)* @main, variables: !13)
+!6 = !DIFile(filename: "dbg.c", directory: "/src")
+!7 = !DISubroutineType(types: !8)
 !8 = !{!9, !9, !10}
-!9 = !MDBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
-!10 = !MDDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, baseType: !11)
-!11 = !MDDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, baseType: !12)
-!12 = !MDBasicType(tag: DW_TAG_base_type, name: "char", size: 8, align: 8, encoding: DW_ATE_unsigned_char)
+!9 = !DIBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
+!10 = !DIDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, baseType: !11)
+!11 = !DIDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, baseType: !12)
+!12 = !DIBasicType(tag: DW_TAG_base_type, name: "char", size: 8, align: 8, encoding: DW_ATE_unsigned_char)
 !13 = !{!15, !16}
-!15 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "argc", line: 1, arg: 1, scope: !5, file: !6, type: !9)
-!16 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "argv", line: 1, arg: 2, scope: !5, file: !6, type: !10)
-!17 = !MDLocation(line: 1, column: 14, scope: !5)
-!18 = !MDLocation(line: 1, column: 26, scope: !5)
-!19 = !MDLocation(line: 2, column: 3, scope: !20)
-!20 = distinct !MDLexicalBlock(line: 1, column: 34, file: !21, scope: !5)
-!21 = !MDFile(filename: "dbg.c", directory: "/src")
+!15 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "argc", line: 1, arg: 1, scope: !5, file: !6, type: !9)
+!16 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "argv", line: 1, arg: 2, scope: !5, file: !6, type: !10)
+!17 = !DILocation(line: 1, column: 14, scope: !5)
+!18 = !DILocation(line: 1, column: 26, scope: !5)
+!19 = !DILocation(line: 2, column: 3, scope: !20)
+!20 = distinct !DILexicalBlock(line: 1, column: 34, file: !21, scope: !5)
+!21 = !DIFile(filename: "dbg.c", directory: "/src")
 !22 = !{i32 1, !"Debug Info Version", i32 3}
diff --git a/test/CodeGen/PowerPC/early-ret.ll b/test/CodeGen/PowerPC/early-ret.ll
index 7d3e225a1e2..52cf464b9fd 100644
--- a/test/CodeGen/PowerPC/early-ret.ll
+++ b/test/CodeGen/PowerPC/early-ret.ll
@@ -1,4 +1,4 @@
-; RUN: llc < %s -mtriple=powerpc64-unknown-linux-gnu -mcpu=pwr7 | FileCheck %s
+; RUN: llc < %s -mtriple=powerpc64-unknown-linux-gnu | FileCheck %s
 target datalayout = "E-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-f128:128:128-v128:128:128-n32:64"
 target triple = "powerpc64-unknown-linux-gnu"
 
@@ -45,4 +45,37 @@ if.end3:                                          ; preds = %if.then, %if.then2,
 ; CHECK: blr
 }
 
+
+@.str0 = private unnamed_addr constant [2 x i8] c"a\00"
+@.str1 = private unnamed_addr constant [2 x i8] c"b\00"
+@.str2 = private unnamed_addr constant [2 x i8] c"c\00"
+@.str3 = private unnamed_addr constant [2 x i8] c"d\00"
+@.str4 = private unnamed_addr constant [2 x i8] c"e\00"
+define i8* @dont_assert(i32 %x) {
+; LLVM would assert due to moving an early return into the jump table block and
+; removing one of its predecessors despite that block ending with an indirect
+; branch.
+entry:
+  switch i32 %x, label %sw.epilog [
+    i32 1, label %return
+    i32 2, label %sw.bb1
+    i32 3, label %sw.bb2
+    i32 4, label %sw.bb3
+    i32 255, label %sw.bb4
+  ]
+sw.bb1: br label %return
+sw.bb2: br label %return
+sw.bb3: br label %return
+sw.bb4: br label %return
+sw.epilog: br label %return
+return:
+  %retval.0 = phi i8* [ null, %sw.epilog ],
+                      [ getelementptr inbounds ([2 x i8], [2 x i8]* @.str4, i64 0, i64 0), %sw.bb4 ],
+                      [ getelementptr inbounds ([2 x i8], [2 x i8]* @.str3, i64 0, i64 0), %sw.bb3 ],
+                      [ getelementptr inbounds ([2 x i8], [2 x i8]* @.str2, i64 0, i64 0), %sw.bb2 ],
+                      [ getelementptr inbounds ([2 x i8], [2 x i8]* @.str1, i64 0, i64 0), %sw.bb1 ],
+                      [ getelementptr inbounds ([2 x i8], [2 x i8]* @.str0, i64 0, i64 0), %entry ]
+  ret i8* %retval.0
+}
+
 attributes #0 = { nounwind }
diff --git a/test/CodeGen/PowerPC/ec-input.ll b/test/CodeGen/PowerPC/ec-input.ll
new file mode 100644
index 00000000000..a57f69be12d
--- /dev/null
+++ b/test/CodeGen/PowerPC/ec-input.ll
@@ -0,0 +1,155 @@
+; RUN: llc -verify-machineinstrs < %s | FileCheck %s
+; This test case used to fail both with and without -verify-machineinstrs
+; (-verify-machineinstrs would catch the problem right after instruction
+; scheduling because the live intervals would not be right for the registers
+; that were both inputs to the inline asm and also early-clobber outputs).
+
+target datalayout = "E-m:e-i64:64-n32:64"
+target triple = "powerpc64-bgq-linux"
+
+%struct._IO_FILE.119.8249.32639.195239.200117.211499.218003.221255.222881.224507.226133.240767.244019.245645.248897.260279.271661.281417.283043.302555.304181.325319.326945.344713 = type { i32, i8*, i8*, i8*, i8*, i8*, i8*, i8*, i8*, i8*, i8*, i8*, %struct._IO_marker.118.8248.32638.195238.200116.211498.218002.221254.222880.224506.226132.240766.244018.245644.248896.260278.271660.281416.283042.302554.304180.325318.326944.344712*, %struct._IO_FILE.119.8249.32639.195239.200117.211499.218003.221255.222881.224507.226133.240767.244019.245645.248897.260279.271661.281417.283043.302555.304181.325319.326945.344713*, i32, i32, i64, i16, i8, [1 x i8], i8*, i64, i8*, i8*, i8*, i8*, i64, i32, [20 x i8] }
+%struct._IO_marker.118.8248.32638.195238.200116.211498.218002.221254.222880.224506.226132.240766.244018.245644.248896.260278.271660.281416.283042.302554.304180.325318.326944.344712 = type { %struct._IO_marker.118.8248.32638.195238.200116.211498.218002.221254.222880.224506.226132.240766.244018.245644.248896.260278.271660.281416.283042.302554.304180.325318.326944.344712*, %struct._IO_FILE.119.8249.32639.195239.200117.211499.218003.221255.222881.224507.226133.240767.244019.245645.248897.260279.271661.281417.283043.302555.304181.325319.326945.344713*, i32 }
+
+@.str236 = external unnamed_addr constant [121 x i8], align 1
+@.str294 = external unnamed_addr constant [49 x i8], align 1
+
+; Function Attrs: nounwind
+declare void @fprintf(%struct._IO_FILE.119.8249.32639.195239.200117.211499.218003.221255.222881.224507.226133.240767.244019.245645.248897.260279.271661.281417.283043.302555.304181.325319.326945.344713* nocapture, i8* nocapture readonly, ...) #0
+
+; Function Attrs: inlinehint nounwind
+define void @_ZN4PAMI6Device2MU15ResourceManager46calculatePerCoreMUResourcesBasedOnAvailabilityEv() #1 align 2 {
+; CHECK-LABEL: @_ZN4PAMI6Device2MU15ResourceManager46calculatePerCoreMUResourcesBasedOnAvailabilityEv
+; CHECK: sc
+
+entry:
+  %numFreeResourcesInSubgroup = alloca i32, align 4
+  %0 = ptrtoint i32* %numFreeResourcesInSubgroup to i64
+  br label %for.cond2.preheader
+
+for.cond2.preheader:                              ; preds = %if.end23.3, %entry
+  %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %if.end23.3 ]
+  %group.098 = phi i32 [ 0, %entry ], [ %inc37, %if.end23.3 ]
+  %minFreeBatIdsPerCore.097 = phi i64 [ 32, %entry ], [ %numFreeBatIdsInGroup.0.minFreeBatIdsPerCore.0, %if.end23.3 ]
+  %minFreeRecFifosPerCore.096 = phi i64 [ 16, %entry ], [ %minFreeRecFifosPerCore.1, %if.end23.3 ]
+  %minFreeInjFifosPerCore.095 = phi i64 [ 32, %entry ], [ %numFreeInjFifosInGroup.0.minFreeInjFifosPerCore.0, %if.end23.3 ]
+  %cmp5 = icmp eq i32 undef, 0
+  br i1 %cmp5, label %if.end, label %if.then
+
+if.then:                                          ; preds = %if.end23.2, %if.end23.1, %if.end23, %for.cond2.preheader
+  unreachable
+
+if.end:                                           ; preds = %for.cond2.preheader
+  %1 = load i32, i32* %numFreeResourcesInSubgroup, align 4
+  %conv = zext i32 %1 to i64
+  %2 = call { i64, i64, i64, i64 } asm sideeffect "sc", "=&{r0},=&{r3},=&{r4},=&{r5},{r0},{r3},{r4},{r5},~{r6},~{r7},~{r8},~{r9},~{r10},~{r11},~{r12},~{cr0},~{memory}"(i64 1034, i64 %indvars.iv, i64 %0, i64 undef) #2
+  %cmp10 = icmp eq i32 0, 0
+  br i1 %cmp10, label %if.end14, label %if.then11
+
+if.then11:                                        ; preds = %if.end.3, %if.end.2, %if.end.1, %if.end
+  unreachable
+
+if.end14:                                         ; preds = %if.end
+  %3 = load i32, i32* %numFreeResourcesInSubgroup, align 4
+  %cmp19 = icmp eq i32 undef, 0
+  br i1 %cmp19, label %if.end23, label %if.then20
+
+if.then20:                                        ; preds = %if.end14.3, %if.end14.2, %if.end14.1, %if.end14
+  %conv4.i65.lcssa = phi i32 [ undef, %if.end14 ], [ 0, %if.end14.1 ], [ %conv4.i65.2, %if.end14.2 ], [ %conv4.i65.3, %if.end14.3 ]
+  call void (%struct._IO_FILE.119.8249.32639.195239.200117.211499.218003.221255.222881.224507.226133.240767.244019.245645.248897.260279.271661.281417.283043.302555.304181.325319.326945.344713*, i8*, ...) @fprintf(%struct._IO_FILE.119.8249.32639.195239.200117.211499.218003.221255.222881.224507.226133.240767.244019.245645.248897.260279.271661.281417.283043.302555.304181.325319.326945.344713* undef, i8* getelementptr inbounds ([121 x i8], [121 x i8]* @.str236, i64 0, i64 0), i32 signext 2503) #3
+  call void (%struct._IO_FILE.119.8249.32639.195239.200117.211499.218003.221255.222881.224507.226133.240767.244019.245645.248897.260279.271661.281417.283043.302555.304181.325319.326945.344713*, i8*, ...) @fprintf(%struct._IO_FILE.119.8249.32639.195239.200117.211499.218003.221255.222881.224507.226133.240767.244019.245645.248897.260279.271661.281417.283043.302555.304181.325319.326945.344713* undef, i8* getelementptr inbounds ([49 x i8], [49 x i8]* @.str294, i64 0, i64 0), i32 signext %conv4.i65.lcssa) #3
+  unreachable
+
+if.end23:                                         ; preds = %if.end14
+  %conv15 = zext i32 %3 to i64
+  %4 = load i32, i32* %numFreeResourcesInSubgroup, align 4
+  %conv24 = zext i32 %4 to i64
+  %5 = call { i64, i64, i64, i64 } asm sideeffect "sc", "=&{r0},=&{r3},=&{r4},=&{r5},{r0},{r3},{r4},{r5},~{r6},~{r7},~{r8},~{r9},~{r10},~{r11},~{r12},~{cr0},~{memory}"(i64 1033, i64 0, i64 %0, i64 undef) #2
+  %cmp5.1 = icmp eq i32 0, 0
+  br i1 %cmp5.1, label %if.end.1, label %if.then
+
+for.end38:                                        ; preds = %if.end23.3
+  ret void
+
+if.end.1:                                         ; preds = %if.end23
+  %6 = load i32, i32* %numFreeResourcesInSubgroup, align 4
+  %conv.1 = zext i32 %6 to i64
+  %add.1 = add nuw nsw i64 %conv.1, %conv
+  %7 = call { i64, i64, i64, i64 } asm sideeffect "sc", "=&{r0},=&{r3},=&{r4},=&{r5},{r0},{r3},{r4},{r5},~{r6},~{r7},~{r8},~{r9},~{r10},~{r11},~{r12},~{cr0},~{memory}"(i64 1034, i64 0, i64 %0, i64 undef) #2
+  %cmp10.1 = icmp eq i32 undef, 0
+  br i1 %cmp10.1, label %if.end14.1, label %if.then11
+
+if.end14.1:                                       ; preds = %if.end.1
+  %8 = load i32, i32* %numFreeResourcesInSubgroup, align 4
+  %cmp19.1 = icmp eq i32 0, 0
+  br i1 %cmp19.1, label %if.end23.1, label %if.then20
+
+if.end23.1:                                       ; preds = %if.end14.1
+  %conv15.1 = zext i32 %8 to i64
+  %add16.1 = add nuw nsw i64 %conv15.1, %conv15
+  %9 = load i32, i32* %numFreeResourcesInSubgroup, align 4
+  %conv24.1 = zext i32 %9 to i64
+  %add25.1 = add nuw nsw i64 %conv24.1, %conv24
+  %cmp5.2 = icmp eq i32 undef, 0
+  br i1 %cmp5.2, label %if.end.2, label %if.then
+
+if.end.2:                                         ; preds = %if.end23.1
+  %10 = load i32, i32* %numFreeResourcesInSubgroup, align 4
+  %conv.2 = zext i32 %10 to i64
+  %add.2 = add nuw nsw i64 %conv.2, %add.1
+  %11 = call { i64, i64, i64, i64 } asm sideeffect "sc", "=&{r0},=&{r3},=&{r4},=&{r5},{r0},{r3},{r4},{r5},~{r6},~{r7},~{r8},~{r9},~{r10},~{r11},~{r12},~{cr0},~{memory}"(i64 1034, i64 undef, i64 %0, i64 undef) #2
+  %cmp10.2 = icmp eq i32 0, 0
+  br i1 %cmp10.2, label %if.end14.2, label %if.then11
+
+if.end14.2:                                       ; preds = %if.end.2
+  %12 = load i32, i32* %numFreeResourcesInSubgroup, align 4
+  %13 = call { i64, i64, i64, i64 } asm sideeffect "sc", "=&{r0},=&{r3},=&{r4},=&{r5},{r0},{r3},{r4},{r5},~{r6},~{r7},~{r8},~{r9},~{r10},~{r11},~{r12},~{cr0},~{memory}"(i64 1035, i64 undef, i64 %0, i64 0) #2
+  %asmresult1.i64.2 = extractvalue { i64, i64, i64, i64 } %13, 1
+  %conv4.i65.2 = trunc i64 %asmresult1.i64.2 to i32
+  %cmp19.2 = icmp eq i32 %conv4.i65.2, 0
+  br i1 %cmp19.2, label %if.end23.2, label %if.then20
+
+if.end23.2:                                       ; preds = %if.end14.2
+  %conv15.2 = zext i32 %12 to i64
+  %add16.2 = add nuw nsw i64 %conv15.2, %add16.1
+  %14 = load i32, i32* %numFreeResourcesInSubgroup, align 4
+  %conv24.2 = zext i32 %14 to i64
+  %add25.2 = add nuw nsw i64 %conv24.2, %add25.1
+  %cmp5.3 = icmp eq i32 0, 0
+  br i1 %cmp5.3, label %if.end.3, label %if.then
+
+if.end.3:                                         ; preds = %if.end23.2
+  %15 = load i32, i32* %numFreeResourcesInSubgroup, align 4
+  %conv.3 = zext i32 %15 to i64
+  %add.3 = add nuw nsw i64 %conv.3, %add.2
+  %cmp10.3 = icmp eq i32 undef, 0
+  br i1 %cmp10.3, label %if.end14.3, label %if.then11
+
+if.end14.3:                                       ; preds = %if.end.3
+  %16 = load i32, i32* %numFreeResourcesInSubgroup, align 4
+  %17 = call { i64, i64, i64, i64 } asm sideeffect "sc", "=&{r0},=&{r3},=&{r4},=&{r5},{r0},{r3},{r4},{r5},~{r6},~{r7},~{r8},~{r9},~{r10},~{r11},~{r12},~{cr0},~{memory}"(i64 1035, i64 0, i64 %0, i64 0) #2
+  %asmresult1.i64.3 = extractvalue { i64, i64, i64, i64 } %17, 1
+  %conv4.i65.3 = trunc i64 %asmresult1.i64.3 to i32
+  %cmp19.3 = icmp eq i32 %conv4.i65.3, 0
+  br i1 %cmp19.3, label %if.end23.3, label %if.then20
+
+if.end23.3:                                       ; preds = %if.end14.3
+  %conv15.3 = zext i32 %16 to i64
+  %add16.3 = add nuw nsw i64 %conv15.3, %add16.2
+  %add25.3 = add nuw nsw i64 0, %add25.2
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 4
+  %cmp27 = icmp ult i64 %add.3, %minFreeInjFifosPerCore.095
+  %numFreeInjFifosInGroup.0.minFreeInjFifosPerCore.0 = select i1 %cmp27, i64 %add.3, i64 %minFreeInjFifosPerCore.095
+  %cmp30 = icmp ult i64 %add16.3, %minFreeRecFifosPerCore.096
+  %minFreeRecFifosPerCore.1 = select i1 %cmp30, i64 %add16.3, i64 %minFreeRecFifosPerCore.096
+  %cmp33 = icmp ult i64 %add25.3, %minFreeBatIdsPerCore.097
+  %numFreeBatIdsInGroup.0.minFreeBatIdsPerCore.0 = select i1 %cmp33, i64 %add25.3, i64 %minFreeBatIdsPerCore.097
+  %inc37 = add nuw nsw i32 %group.098, 1
+  %cmp = icmp ult i32 %inc37, 16
+  br i1 %cmp, label %for.cond2.preheader, label %for.end38
+}
+
+attributes #0 = { nounwind "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "target-cpu"="a2q" }
+attributes #1 = { inlinehint nounwind "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "target-cpu"="a2q" }
+attributes #2 = { nounwind }
+attributes #3 = { cold nounwind }
+
diff --git a/test/CodeGen/PowerPC/emutls_generic.ll b/test/CodeGen/PowerPC/emutls_generic.ll
new file mode 100644
index 00000000000..a2e13a6723f
--- /dev/null
+++ b/test/CodeGen/PowerPC/emutls_generic.ll
@@ -0,0 +1,41 @@
+; RUN: llc < %s -emulated-tls -mtriple=powerpc64-unknown-linux-gnu -relocation-model=pic \
+; RUN:     | FileCheck %s
+; RUN: llc < %s -emulated-tls -mtriple=powerpc-unknown-linux-gnu -relocation-model=pic \
+; RUN:     | FileCheck %s
+
+; Make sure that TLS symbols are emitted in expected order.
+
+@external_x = external thread_local global i32, align 8
+@external_y = thread_local global i8 7, align 2
+@internal_y = internal thread_local global i64 9, align 16
+
+define i32* @get_external_x() {
+entry:
+  ret i32* @external_x
+}
+
+define i8* @get_external_y() {
+entry:
+  ret i8* @external_y
+}
+
+define i64* @get_internal_y() {
+entry:
+  ret i64* @internal_y
+}
+
+; CHECK-LABEL: get_external_x:
+; CHECK-NOT:   _tls_get_address
+; CHECK:       __emutls_get_address
+; CHECK-LABEL: get_external_y:
+; CHECK:       __emutls_get_address
+; CHECK-NOT:   _tls_get_address
+; CHECK-LABEL: get_internal_y:
+; CHECK-NOT:   __emutls_t.external_x:
+; CHECK-NOT:   __emutls_v.external_x:
+; CHECK-LABEL: __emutls_v.external_y:
+; CHECK-LABEL: __emutls_t.external_y:
+; CHECK:       __emutls_t.external_y
+; CHECK-LABEL: __emutls_v.internal_y:
+; CHECK-LABEL: __emutls_t.internal_y:
+; CHECK:       __emutls_t.internal_y
diff --git a/test/CodeGen/PowerPC/extra-toc-reg-deps.ll b/test/CodeGen/PowerPC/extra-toc-reg-deps.ll
new file mode 100644
index 00000000000..488771807ce
--- /dev/null
+++ b/test/CodeGen/PowerPC/extra-toc-reg-deps.ll
@@ -0,0 +1,430 @@
+; RUN: llc < %s | FileCheck %s
+target datalayout = "E-m:e-i64:64-n32:64"
+target triple = "powerpc64-bgq-linux"
+
+%"class.Foam::messageStream.6" = type <{ %"class.Foam::string.5", i32, i32, i32, [4 x i8] }>
+%"class.Foam::string.5" = type { %"class.std::basic_string.4" }
+%"class.std::basic_string.4" = type { %"struct.std::basic_string<char, std::char_traits<char>, std::allocator<char> >::_Alloc_hider.3" }
+%"struct.std::basic_string<char, std::char_traits<char>, std::allocator<char> >::_Alloc_hider.3" = type { i8* }
+%"class.Foam::prefixOSstream.27" = type { %"class.Foam::OSstream.26", i8, %"class.Foam::string.5" }
+%"class.Foam::OSstream.26" = type { %"class.Foam::Ostream.base.9", %"class.Foam::fileName.10", %"class.std::basic_ostream.25"* }
+%"class.Foam::Ostream.base.9" = type <{ %"class.Foam::IOstream.8", i16 }>
+%"class.Foam::IOstream.8" = type { i32 (...)**, i32, [4 x i8], %"class.Foam::IOstream::versionNumber.7", i32, i32, i32, i32 }
+%"class.Foam::IOstream::versionNumber.7" = type <{ double, i32, [4 x i8] }>
+%"class.Foam::fileName.10" = type { %"class.Foam::string.5" }
+%"class.std::basic_ostream.25" = type { i32 (...)**, %"class.std::basic_ios.24" }
+%"class.std::basic_ios.24" = type { %"class.std::ios_base.16", %"class.std::basic_ostream.25"*, i8, i8, %"class.std::basic_streambuf.17"*, %"class.std::ctype.21"*, %"class.std::__gnu_cxx_ldbl128::num_put.22"*, %"class.std::__gnu_cxx_ldbl128::num_get.23"* }
+%"class.std::ios_base.16" = type { i32 (...)**, i64, i64, i32, i32, i32, %"struct.std::ios_base::_Callback_list.11"*, %"struct.std::ios_base::_Words.12", [8 x %"struct.std::ios_base::_Words.12"], i32, %"struct.std::ios_base::_Words.12"*, %"class.std::locale.15" }
+%"struct.std::ios_base::_Callback_list.11" = type { %"struct.std::ios_base::_Callback_list.11"*, void (i32, %"class.std::ios_base.16"*, i32)*, i32, i32 }
+%"struct.std::ios_base::_Words.12" = type { i8*, i64 }
+%"class.std::locale.15" = type { %"class.std::locale::_Impl.14"* }
+%"class.std::locale::_Impl.14" = type { i32, %"class.std::locale::facet.13"**, i64, %"class.std::locale::facet.13"**, i8** }
+%"class.std::locale::facet.13" = type <{ i32 (...)**, i32, [4 x i8] }>
+%"class.std::basic_streambuf.17" = type { i32 (...)**, i8*, i8*, i8*, i8*, i8*, i8*, %"class.std::locale.15" }
+%"class.std::ctype.21" = type <{ %"class.std::locale::facet.base.18", [4 x i8], %struct.__locale_struct.20*, i8, [7 x i8], i32*, i32*, i16*, i8, [256 x i8], [256 x i8], i8, [6 x i8] }>
+%"class.std::locale::facet.base.18" = type <{ i32 (...)**, i32 }>
+%struct.__locale_struct.20 = type { [13 x %struct.__locale_data.19*], i16*, i32*, i32*, [13 x i8*] }
+%struct.__locale_data.19 = type opaque
+%"class.std::__gnu_cxx_ldbl128::num_put.22" = type { %"class.std::locale::facet.base.18", [4 x i8] }
+%"class.std::__gnu_cxx_ldbl128::num_get.23" = type { %"class.std::locale::facet.base.18", [4 x i8] }
+%"class.Foam::primitiveMesh.135" = type { i32 (...)**, i32, i32, i32, i32, i32, i32, i32, i32, i32, %"class.Foam::List.116"*, %"class.Foam::List.0"*, %"class.Foam::List.1"*, %"class.Foam::List.1"*, %"class.Foam::List.1"*, %"class.Foam::List.5"*, %"class.Foam::List.1"*, %"class.Foam::List.1"*, %"class.Foam::List.1"*, %"class.Foam::List.1"*, %"class.Foam::List.1"*, %"class.Foam::List.1"*, %"class.Foam::List.1"*, %"class.Foam::DynamicList.40", %"class.Foam::HashSet.127", %"class.Foam::Field.131"*, %"class.Foam::Field.131"*, %"class.Foam::Field.11"*, %"class.Foam::Field.131"* }
+%"class.Foam::List.116" = type opaque
+%"class.Foam::List.0" = type { %"class.Foam::UList.119" }
+%"class.Foam::UList.119" = type { i32, %"class.Foam::edge.118"* }
+%"class.Foam::edge.118" = type { %"class.Foam::FixedList.117" }
+%"class.Foam::FixedList.117" = type { [2 x i32] }
+%"class.Foam::List.5" = type { %"class.Foam::UList.6" }
+%"class.Foam::UList.6" = type { i32, %"class.Foam::cell.121"* }
+%"class.Foam::cell.121" = type { %"class.Foam::List.3" }
+%"class.Foam::List.3" = type { %"class.Foam::UList.4" }
+%"class.Foam::UList.4" = type { i32, i32* }
+%"class.Foam::List.1" = type { %"class.Foam::UList.2" }
+%"class.Foam::UList.2" = type { i32, %"class.Foam::List.3"* }
+%"class.Foam::DynamicList.40" = type <{ %"class.Foam::List.3", i32, [4 x i8] }>
+%"class.Foam::HashSet.127" = type { %"class.Foam::HashTable.7" }
+%"class.Foam::HashTable.7" = type { i32, i32, %"struct.Foam::HashTable<Foam::nil, int, Foam::Hash<Foam::label> >::hashedEntry.125"** }
+%"struct.Foam::HashTable<Foam::nil, int, Foam::Hash<Foam::label> >::hashedEntry.125" = type <{ i32, [4 x i8], %"struct.Foam::HashTable<Foam::nil, int, Foam::Hash<Foam::label> >::hashedEntry.125"*, %"class.Foam::nil.124", [7 x i8] }>
+%"class.Foam::nil.124" = type { i8 }
+%"class.Foam::Field.11" = type { %"class.Foam::refCount.128", %"class.Foam::List.12" }
+%"class.Foam::refCount.128" = type { i32 }
+%"class.Foam::List.12" = type { %"class.Foam::UList.13" }
+%"class.Foam::UList.13" = type { i32, double* }
+%"class.Foam::Field.131" = type { %"class.Foam::refCount.128", %"class.Foam::List.8" }
+%"class.Foam::List.8" = type { %"class.Foam::UList.9" }
+%"class.Foam::UList.9" = type { i32, %"class.Foam::Vector.29"* }
+%"class.Foam::Vector.29" = type { %"class.Foam::VectorSpace.10" }
+%"class.Foam::VectorSpace.10" = type { [3 x double] }
+%"class.Foam::Ostream.189" = type <{ %"class.Foam::IOstream.8", i16, [6 x i8] }>
+
+@_ZN4Foam4InfoE = external global %"class.Foam::messageStream.6", align 8
+@.str27 = external unnamed_addr constant [24 x i8], align 1
+@.str28 = external unnamed_addr constant [7 x i8], align 1
+@_ZN4Foam4PoutE = external global %"class.Foam::prefixOSstream.27", align 8
+
+define void @_ZN4Foam13checkTopologyERKNS_8polyMeshEbb(i1 zeroext %allTopology) #0 personality i8* bitcast (i32 (...)* @__gxx_personality_v0 to i8*) {
+entry:
+  br i1 undef, label %for.body, label %for.cond.cleanup
+
+; CHECK-LABEL: @_ZN4Foam13checkTopologyERKNS_8polyMeshEbb
+
+; CHECK: addis [[REG1:[0-9]+]], 2, .LC0@toc@ha
+; CHECK: std 2, 40(1)
+; CHECK: ld {{[0-9]+}}, .LC0@toc@l([[REG1]])
+; CHECK: {{mr|ld}} 2,
+; CHECK: mtctr
+; CHECK: bctrl
+; CHECK: ld 2, 40(1)
+
+; CHECK: addis [[REG1:[0-9]+]], 2, .LC0@toc@ha
+; CHECK: std 2, 40(1)
+; CHECK: ld {{[0-9]+}}, .LC0@toc@l([[REG1]])
+; CHECK: {{mr|ld}} 2,
+; CHECK: mtctr
+; CHECK: bctrl
+; CHECK: ld 2, 40(1)
+
+for.cond.cleanup:                                 ; preds = %entry
+  br i1 undef, label %if.then.i, label %if.else.i
+
+if.then.i:                                        ; preds = %for.cond.cleanup
+  br i1 undef, label %if.then.i1435, label %if.else.i1436
+
+if.else.i:                                        ; preds = %for.cond.cleanup
+  unreachable
+
+if.then.i1435:                                    ; preds = %if.then.i
+  br label %_ZN4Foam12returnReduceIiNS_5sumOpIiEEEET_RKS3_RKT0_ii.exit
+
+if.else.i1436:                                    ; preds = %if.then.i
+  br label %_ZN4Foam12returnReduceIiNS_5sumOpIiEEEET_RKS3_RKT0_ii.exit
+
+_ZN4Foam12returnReduceIiNS_5sumOpIiEEEET_RKS3_RKT0_ii.exit: ; preds = %if.else.i1436, %if.then.i1435
+  br i1 undef, label %for.body.i, label %_ZNK4Foam8ZoneMeshINS_8cellZoneENS_8polyMeshEE15checkDefinitionEb.exit
+
+for.body:                                         ; preds = %entry
+  unreachable
+
+for.body.i:                                       ; preds = %_ZN4Foam12returnReduceIiNS_5sumOpIiEEEET_RKS3_RKT0_ii.exit
+  unreachable
+
+_ZNK4Foam8ZoneMeshINS_8cellZoneENS_8polyMeshEE15checkDefinitionEb.exit: ; preds = %_ZN4Foam12returnReduceIiNS_5sumOpIiEEEET_RKS3_RKT0_ii.exit
+  br i1 undef, label %for.body.i1480, label %_ZNK4Foam8ZoneMeshINS_8faceZoneENS_8polyMeshEE15checkDefinitionEb.exit
+
+for.body.i1480:                                   ; preds = %_ZNK4Foam8ZoneMeshINS_8cellZoneENS_8polyMeshEE15checkDefinitionEb.exit
+  unreachable
+
+_ZNK4Foam8ZoneMeshINS_8faceZoneENS_8polyMeshEE15checkDefinitionEb.exit: ; preds = %_ZNK4Foam8ZoneMeshINS_8cellZoneENS_8polyMeshEE15checkDefinitionEb.exit
+  br i1 undef, label %for.body.i1504, label %_ZNK4Foam8ZoneMeshINS_9pointZoneENS_8polyMeshEE15checkDefinitionEb.exit
+
+for.body.i1504:                                   ; preds = %_ZNK4Foam8ZoneMeshINS_8faceZoneENS_8polyMeshEE15checkDefinitionEb.exit
+  unreachable
+
+_ZNK4Foam8ZoneMeshINS_9pointZoneENS_8polyMeshEE15checkDefinitionEb.exit: ; preds = %_ZNK4Foam8ZoneMeshINS_8faceZoneENS_8polyMeshEE15checkDefinitionEb.exit
+  invoke void @_ZN4Foam4word12stripInvalidEv()
+          to label %_ZN4Foam4wordC2EPKcb.exit unwind label %lpad.i
+
+lpad.i:                                           ; preds = %_ZNK4Foam8ZoneMeshINS_9pointZoneENS_8polyMeshEE15checkDefinitionEb.exit
+  %0 = landingpad { i8*, i32 }
+          cleanup
+  resume { i8*, i32 } %0
+
+_ZN4Foam4wordC2EPKcb.exit:                        ; preds = %_ZNK4Foam8ZoneMeshINS_9pointZoneENS_8polyMeshEE15checkDefinitionEb.exit
+  invoke void @_ZN4Foam7cellSetC1ERKNS_8polyMeshERKNS_4wordEiNS_8IOobject11writeOptionE()
+          to label %invoke.cont59 unwind label %lpad
+
+invoke.cont59:                                    ; preds = %_ZN4Foam4wordC2EPKcb.exit
+  br i1 undef, label %_ZNSsD2Ev.exit, label %if.then.i.i, !prof !1
+
+if.then.i.i:                                      ; preds = %invoke.cont59
+  br i1 true, label %if.then.i.i.i1508, label %if.else.i.i.i
+
+if.then.i.i.i1508:                                ; preds = %if.then.i.i
+  br label %_ZN9__gnu_cxxL27__exchange_and_add_dispatchEPii.exit.i.i
+
+if.else.i.i.i:                                    ; preds = %if.then.i.i
+  br label %_ZN9__gnu_cxxL27__exchange_and_add_dispatchEPii.exit.i.i
+
+_ZN9__gnu_cxxL27__exchange_and_add_dispatchEPii.exit.i.i: ; preds = %if.else.i.i.i, %if.then.i.i.i1508
+  br i1 undef, label %if.then4.i.i, label %_ZNSsD2Ev.exit
+
+if.then4.i.i:                                     ; preds = %_ZN9__gnu_cxxL27__exchange_and_add_dispatchEPii.exit.i.i
+  br label %_ZNSsD2Ev.exit
+
+_ZNSsD2Ev.exit:                                   ; preds = %if.then4.i.i, %_ZN9__gnu_cxxL27__exchange_and_add_dispatchEPii.exit.i.i, %invoke.cont59
+  br i1 undef, label %for.body70, label %for.cond.cleanup69
+
+for.cond.cleanup69:                               ; preds = %_ZNSsD2Ev.exit
+  br i1 undef, label %if.then121, label %if.else
+
+lpad:                                             ; preds = %_ZN4Foam4wordC2EPKcb.exit
+  %1 = landingpad { i8*, i32 }
+          cleanup
+  br i1 undef, label %_ZNSsD2Ev.exit1578, label %if.then.i.i1570, !prof !1
+
+if.then.i.i1570:                                  ; preds = %lpad
+  br i1 undef, label %if.then4.i.i1577, label %_ZNSsD2Ev.exit1578
+
+if.then4.i.i1577:                                 ; preds = %if.then.i.i1570
+  unreachable
+
+_ZNSsD2Ev.exit1578:                               ; preds = %if.then.i.i1570, %lpad
+  unreachable
+
+for.body70:                                       ; preds = %_ZNSsD2Ev.exit
+  unreachable
+
+if.then121:                                       ; preds = %for.cond.cleanup69
+  unreachable
+
+if.else:                                          ; preds = %for.cond.cleanup69
+  invoke void @_ZN4Foam4word12stripInvalidEv()
+          to label %_ZN4Foam4wordC2EPKcb.exit1701 unwind label %lpad.i1689
+
+lpad.i1689:                                       ; preds = %if.else
+  %2 = landingpad { i8*, i32 }
+          cleanup
+  unreachable
+
+_ZN4Foam4wordC2EPKcb.exit1701:                    ; preds = %if.else
+  invoke void @_ZN4Foam8pointSetC1ERKNS_8polyMeshERKNS_4wordEiNS_8IOobject11writeOptionE()
+          to label %invoke.cont169 unwind label %lpad165
+
+invoke.cont169:                                   ; preds = %_ZN4Foam4wordC2EPKcb.exit1701
+  %call177 = invoke zeroext i1 undef(%"class.Foam::primitiveMesh.135"* undef, i1 zeroext true, %"class.Foam::HashSet.127"* undef)
+          to label %invoke.cont176 unwind label %lpad175
+
+invoke.cont176:                                   ; preds = %invoke.cont169
+  br i1 %call177, label %if.then178, label %if.end213
+
+if.then178:                                       ; preds = %invoke.cont176
+  unreachable
+
+lpad165:                                          ; preds = %_ZN4Foam4wordC2EPKcb.exit1701
+  %3 = landingpad { i8*, i32 }
+          cleanup
+  unreachable
+
+lpad175:                                          ; preds = %invoke.cont169
+  %4 = landingpad { i8*, i32 }
+          cleanup
+  invoke void @_ZN4Foam8pointSetD1Ev()
+          to label %eh.resume unwind label %terminate.lpad
+
+if.end213:                                        ; preds = %invoke.cont176
+  invoke void @_ZN4Foam4word12stripInvalidEv()
+          to label %_ZN4Foam4wordC2EPKcb.exit1777 unwind label %lpad.i1765
+
+lpad.i1765:                                       ; preds = %if.end213
+  %5 = landingpad { i8*, i32 }
+          cleanup
+  br i1 undef, label %eh.resume.i1776, label %if.then.i.i.i1767, !prof !1
+
+if.then.i.i.i1767:                                ; preds = %lpad.i1765
+  unreachable
+
+eh.resume.i1776:                                  ; preds = %lpad.i1765
+  resume { i8*, i32 } %5
+
+_ZN4Foam4wordC2EPKcb.exit1777:                    ; preds = %if.end213
+  invoke void @_ZN4Foam7faceSetC1ERKNS_8polyMeshERKNS_4wordEiNS_8IOobject11writeOptionE()
+          to label %invoke.cont221 unwind label %lpad217
+
+invoke.cont221:                                   ; preds = %_ZN4Foam4wordC2EPKcb.exit1777
+  br i1 undef, label %_ZNSsD2Ev.exit1792, label %if.then.i.i1784, !prof !1
+
+if.then.i.i1784:                                  ; preds = %invoke.cont221
+  br i1 undef, label %if.then4.i.i1791, label %_ZNSsD2Ev.exit1792
+
+if.then4.i.i1791:                                 ; preds = %if.then.i.i1784
+  br label %_ZNSsD2Ev.exit1792
+
+_ZNSsD2Ev.exit1792:                               ; preds = %if.then4.i.i1791, %if.then.i.i1784, %invoke.cont221
+  %call232 = invoke zeroext i1 undef(%"class.Foam::primitiveMesh.135"* undef, i1 zeroext true, %"class.Foam::HashSet.127"* undef)
+          to label %invoke.cont231 unwind label %lpad230
+
+invoke.cont231:                                   ; preds = %_ZNSsD2Ev.exit1792
+  invoke void @_ZN4Foam6reduceIiNS_5sumOpIiEEEEvRKNS_4ListINS_8UPstream11commsStructEEERT_RKT0_ii()
+          to label %invoke.cont243 unwind label %lpad230
+
+lpad217:                                          ; preds = %_ZN4Foam4wordC2EPKcb.exit1777
+  %6 = landingpad { i8*, i32 }
+          cleanup
+  br label %eh.resume
+
+lpad230:                                          ; preds = %invoke.cont231, %_ZNSsD2Ev.exit1792
+  %7 = landingpad { i8*, i32 }
+          cleanup
+  invoke void @_ZN4Foam7faceSetD1Ev()
+          to label %eh.resume unwind label %terminate.lpad
+
+invoke.cont243:                                   ; preds = %invoke.cont231
+  invoke void @_ZN4Foam4word12stripInvalidEv()
+          to label %_ZN4Foam4wordC2EPKcb.exit1862 unwind label %lpad.i1850
+
+lpad.i1850:                                       ; preds = %invoke.cont243
+  %8 = landingpad { i8*, i32 }
+          cleanup
+  unreachable
+
+_ZN4Foam4wordC2EPKcb.exit1862:                    ; preds = %invoke.cont243
+  invoke void @_ZN4Foam7faceSetC1ERKNS_8polyMeshERKNS_4wordEiNS_8IOobject11writeOptionE()
+          to label %invoke.cont280 unwind label %lpad276
+
+invoke.cont280:                                   ; preds = %_ZN4Foam4wordC2EPKcb.exit1862
+  br i1 undef, label %_ZNSsD2Ev.exit1877, label %if.then.i.i1869, !prof !1
+
+if.then.i.i1869:                                  ; preds = %invoke.cont280
+  unreachable
+
+_ZNSsD2Ev.exit1877:                               ; preds = %invoke.cont280
+  br i1 undef, label %if.then292, label %if.end328
+
+if.then292:                                       ; preds = %_ZNSsD2Ev.exit1877
+  unreachable
+
+lpad276:                                          ; preds = %_ZN4Foam4wordC2EPKcb.exit1862
+  %9 = landingpad { i8*, i32 }
+          cleanup
+  unreachable
+
+if.end328:                                        ; preds = %_ZNSsD2Ev.exit1877
+  br i1 %allTopology, label %if.then331, label %if.end660
+
+if.then331:                                       ; preds = %if.end328
+  unreachable
+
+if.end660:                                        ; preds = %if.end328
+  invoke void @_ZN4Foam13messageStreamcvRNS_8OSstreamEEv()
+          to label %invoke.cont668 unwind label %lpad663
+
+invoke.cont668:                                   ; preds = %if.end660
+  %call671 = invoke dereferenceable(56) %"class.Foam::Ostream.189"* @_ZN4FoamlsERNS_7OstreamEPKc()
+          to label %invoke.cont670 unwind label %lpad663
+
+invoke.cont670:                                   ; preds = %invoke.cont668
+  invoke void @_ZN4FoamlsERNS_7OstreamEi()
+          to label %invoke.cont674 unwind label %lpad663
+
+invoke.cont674:                                   ; preds = %invoke.cont670
+  %call677 = invoke dereferenceable(56) %"class.Foam::Ostream.189"* @_ZN4FoamlsERNS_7OstreamEPKc()
+          to label %invoke.cont676 unwind label %lpad663
+
+invoke.cont676:                                   ; preds = %invoke.cont674
+  invoke void undef(%"class.Foam::Ostream.189"* %call677)
+          to label %if.end878 unwind label %lpad663
+
+lpad663:                                          ; preds = %invoke.cont670, %if.end660, %invoke.cont668, %invoke.cont674, %invoke.cont676
+  %10 = landingpad { i8*, i32 }
+          cleanup
+  br i1 undef, label %_ZN4Foam4ListIiED2Ev.exit.i3073, label %delete.notnull.i.i3071
+
+if.end878:                                        ; preds = %invoke.cont676
+  br i1 undef, label %_ZN4Foam11regionSplitD2Ev.exit, label %delete.notnull.i.i3056
+
+delete.notnull.i.i3056:                           ; preds = %if.end878
+  unreachable
+
+_ZN4Foam11regionSplitD2Ev.exit:                   ; preds = %if.end878
+  br i1 undef, label %if.then883, label %if.else888
+
+if.then883:                                       ; preds = %_ZN4Foam11regionSplitD2Ev.exit
+  unreachable
+
+delete.notnull.i.i3071:                           ; preds = %lpad663
+  unreachable
+
+_ZN4Foam4ListIiED2Ev.exit.i3073:                  ; preds = %lpad663
+  invoke void @_ZN4Foam11regIOobjectD2Ev()
+          to label %eh.resume unwind label %terminate.lpad
+
+if.else888:                                       ; preds = %_ZN4Foam11regionSplitD2Ev.exit
+  invoke void @_ZN4Foam4word12stripInvalidEv()
+          to label %_ZN4Foam4wordC2EPKcb.exit3098 unwind label %lpad.i3086
+
+lpad.i3086:                                       ; preds = %if.else888
+  %11 = landingpad { i8*, i32 }
+          cleanup
+  unreachable
+
+_ZN4Foam4wordC2EPKcb.exit3098:                    ; preds = %if.else888
+  invoke void @_ZN4Foam8pointSetC1ERKNS_8polyMeshERKNS_4wordEiNS_8IOobject11writeOptionE()
+          to label %invoke.cont902 unwind label %lpad898
+
+invoke.cont902:                                   ; preds = %_ZN4Foam4wordC2EPKcb.exit3098
+  br i1 undef, label %_ZNSsD2Ev.exit3113, label %if.then.i.i3105, !prof !1
+
+if.then.i.i3105:                                  ; preds = %invoke.cont902
+  br i1 undef, label %if.then4.i.i3112, label %_ZNSsD2Ev.exit3113
+
+if.then4.i.i3112:                                 ; preds = %if.then.i.i3105
+  unreachable
+
+_ZNSsD2Ev.exit3113:                               ; preds = %if.then.i.i3105, %invoke.cont902
+  %call.i31163117 = invoke zeroext i32 undef(%"class.Foam::IOstream.8"* getelementptr inbounds (%"class.Foam::prefixOSstream.27", %"class.Foam::prefixOSstream.27"* @_ZN4Foam4PoutE, i64 0, i32 0, i32 0, i32 0))
+          to label %call.i3116.noexc unwind label %lpad905.loopexit.split-lp
+
+call.i3116.noexc:                                 ; preds = %_ZNSsD2Ev.exit3113
+  %call5.i3118 = invoke zeroext i32 null(%"class.Foam::IOstream.8"* getelementptr inbounds (%"class.Foam::prefixOSstream.27", %"class.Foam::prefixOSstream.27"* @_ZN4Foam4PoutE, i64 0, i32 0, i32 0, i32 0), i32 zeroext undef)
+          to label %invoke.cont906 unwind label %lpad905.loopexit.split-lp
+
+invoke.cont906:                                   ; preds = %call.i3116.noexc
+  unreachable
+
+lpad898:                                          ; preds = %_ZN4Foam4wordC2EPKcb.exit3098
+  %12 = landingpad { i8*, i32 }
+          cleanup
+  br i1 undef, label %_ZNSsD2Ev.exit3204, label %if.then.i.i3196, !prof !1
+
+if.then.i.i3196:                                  ; preds = %lpad898
+  unreachable
+
+_ZNSsD2Ev.exit3204:                               ; preds = %lpad898
+  unreachable
+
+lpad905.loopexit.split-lp:                        ; preds = %call.i3116.noexc, %_ZNSsD2Ev.exit3113
+  %lpad.loopexit.split-lp = landingpad { i8*, i32 }
+          cleanup
+  invoke void @_ZN4Foam8pointSetD1Ev()
+          to label %eh.resume unwind label %terminate.lpad
+
+eh.resume:                                        ; preds = %_ZN4Foam4ListIiED2Ev.exit.i3073, %lpad230, %lpad175, %lpad905.loopexit.split-lp, %lpad217
+  resume { i8*, i32 } undef
+
+terminate.lpad:                                   ; preds = %_ZN4Foam4ListIiED2Ev.exit.i3073, %lpad230, %lpad175, %lpad905.loopexit.split-lp
+  %13 = landingpad { i8*, i32 }
+          catch i8* null
+  unreachable
+}
+
+declare dereferenceable(56) %"class.Foam::Ostream.189"* @_ZN4FoamlsERNS_7OstreamEPKc() #0
+
+declare void @_ZN4Foam13messageStreamcvRNS_8OSstreamEEv() #0
+
+declare i32 @__gxx_personality_v0(...)
+
+declare void @_ZN4Foam7cellSetC1ERKNS_8polyMeshERKNS_4wordEiNS_8IOobject11writeOptionE() #0
+
+declare void @_ZN4FoamlsERNS_7OstreamEi() #0
+
+declare void @_ZN4Foam8pointSetC1ERKNS_8polyMeshERKNS_4wordEiNS_8IOobject11writeOptionE() #0
+
+declare void @_ZN4Foam8pointSetD1Ev() #0
+
+declare void @_ZN4Foam7faceSetC1ERKNS_8polyMeshERKNS_4wordEiNS_8IOobject11writeOptionE() #0
+
+declare void @_ZN4Foam7faceSetD1Ev() #0
+
+; Function Attrs: inlinehint
+declare void @_ZN4Foam4word12stripInvalidEv() #1 align 2
+
+declare void @_ZN4Foam11regIOobjectD2Ev() #0
+
+declare void @_ZN4Foam6reduceIiNS_5sumOpIiEEEEvRKNS_4ListINS_8UPstream11commsStructEEERT_RKT0_ii() #0
+
+attributes #0 = { "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="a2q" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #1 = { inlinehint "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="a2q" "unsafe-fp-math"="false" "use-soft-float"="false" }
+
+!llvm.module.flags = !{!0}
+
+!0 = !{i32 1, !"PIC Level", i32 2}
+!1 = !{!"branch_weights", i32 64, i32 4}
diff --git a/test/CodeGen/PowerPC/fast-isel-binary.ll b/test/CodeGen/PowerPC/fast-isel-binary.ll
index 2f1513f8aa1..1036689ff44 100644
--- a/test/CodeGen/PowerPC/fast-isel-binary.ll
+++ b/test/CodeGen/PowerPC/fast-isel-binary.ll
@@ -2,7 +2,7 @@
 
 ; Test add with non-legal types
 
-define void @add_i8(i8 %a, i8 %b) nounwind ssp {
+define void @add_i8(i8 %a, i8 %b) nounwind {
 entry:
 ; ELF64: add_i8
   %a.addr = alloca i8, align 4
@@ -12,7 +12,7 @@ entry:
   ret void
 }
 
-define void @add_i8_imm(i8 %a) nounwind ssp {
+define void @add_i8_imm(i8 %a) nounwind {
 entry:
 ; ELF64: add_i8_imm
   %a.addr = alloca i8, align 4
@@ -22,7 +22,7 @@ entry:
   ret void
 }
 
-define void @add_i16(i16 %a, i16 %b) nounwind ssp {
+define void @add_i16(i16 %a, i16 %b) nounwind {
 entry:
 ; ELF64: add_i16
   %a.addr = alloca i16, align 4
@@ -32,7 +32,7 @@ entry:
   ret void
 }
 
-define void @add_i16_imm(i16 %a, i16 %b) nounwind ssp {
+define void @add_i16_imm(i16 %a, i16 %b) nounwind {
 entry:
 ; ELF64: add_i16_imm
   %a.addr = alloca i16, align 4
@@ -44,7 +44,7 @@ entry:
 
 ; Test or with non-legal types
 
-define void @or_i8(i8 %a, i8 %b) nounwind ssp {
+define void @or_i8(i8 %a, i8 %b) nounwind {
 entry:
 ; ELF64: or_i8
   %a.addr = alloca i8, align 4
@@ -54,7 +54,7 @@ entry:
   ret void
 }
 
-define void @or_i8_imm(i8 %a) nounwind ssp {
+define void @or_i8_imm(i8 %a) nounwind {
 entry:
 ; ELF64: or_i8_imm
   %a.addr = alloca i8, align 4
@@ -64,7 +64,7 @@ entry:
   ret void
 }
 
-define void @or_i16(i16 %a, i16 %b) nounwind ssp {
+define void @or_i16(i16 %a, i16 %b) nounwind {
 entry:
 ; ELF64: or_i16
   %a.addr = alloca i16, align 4
@@ -74,7 +74,7 @@ entry:
   ret void
 }
 
-define void @or_i16_imm(i16 %a) nounwind ssp {
+define void @or_i16_imm(i16 %a) nounwind {
 entry:
 ; ELF64: or_i16_imm
   %a.addr = alloca i16, align 4
@@ -86,7 +86,7 @@ entry:
 
 ; Test sub with non-legal types
 
-define void @sub_i8(i8 %a, i8 %b) nounwind ssp {
+define void @sub_i8(i8 %a, i8 %b) nounwind {
 entry:
 ; ELF64: sub_i8
   %a.addr = alloca i8, align 4
@@ -96,7 +96,7 @@ entry:
   ret void
 }
 
-define void @sub_i8_imm(i8 %a) nounwind ssp {
+define void @sub_i8_imm(i8 %a) nounwind {
 entry:
 ; ELF64: sub_i8_imm
   %a.addr = alloca i8, align 4
@@ -106,7 +106,7 @@ entry:
   ret void
 }
 
-define void @sub_i16(i16 %a, i16 %b) nounwind ssp {
+define void @sub_i16(i16 %a, i16 %b) nounwind {
 entry:
 ; ELF64: sub_i16
   %a.addr = alloca i16, align 4
@@ -116,7 +116,7 @@ entry:
   ret void
 }
 
-define void @sub_i16_imm(i16 %a) nounwind ssp {
+define void @sub_i16_imm(i16 %a) nounwind {
 entry:
 ; ELF64: sub_i16_imm
   %a.addr = alloca i16, align 4
@@ -126,7 +126,7 @@ entry:
   ret void
 }
 
-define void @sub_i16_badimm(i16 %a) nounwind ssp {
+define void @sub_i16_badimm(i16 %a) nounwind {
 entry:
 ; ELF64: sub_i16_imm
   %a.addr = alloca i16, align 4
diff --git a/test/CodeGen/PowerPC/fast-isel-br-const.ll b/test/CodeGen/PowerPC/fast-isel-br-const.ll
index 6be7fbf9e02..f411d23fb28 100644
--- a/test/CodeGen/PowerPC/fast-isel-br-const.ll
+++ b/test/CodeGen/PowerPC/fast-isel-br-const.ll
@@ -1,6 +1,6 @@
 ; RUN: llc < %s -O0 -verify-machineinstrs -fast-isel-abort=1 -mtriple=powerpc64-unknown-linux-gnu -mcpu=pwr7 | FileCheck %s --check-prefix=ELF64
 
-define i32 @t1(i32 %a, i32 %b) nounwind uwtable ssp {
+define i32 @t1(i32 %a, i32 %b) nounwind {
 entry:
 ; ELF64: t1
   %x = add i32 %a, %b  
diff --git a/test/CodeGen/PowerPC/fast-isel-call.ll b/test/CodeGen/PowerPC/fast-isel-call.ll
index 64d8f6e7919..5d541e3a01f 100644
--- a/test/CodeGen/PowerPC/fast-isel-call.ll
+++ b/test/CodeGen/PowerPC/fast-isel-call.ll
@@ -56,7 +56,7 @@ declare zeroext i16 @t6();
 declare signext i8 @t7();
 declare zeroext i8 @t8();
 
-define i32 @t10(i32 %argc, i8** nocapture %argv) {
+define i32 @t10(i32 %argc, i8** nocapture %argv) nounwind {
 entry:
 ; ELF64: t10
   %call = call i32 @bar(i8 zeroext 0, i8 zeroext -8, i8 zeroext -69, i8 zeroext 28, i8 zeroext 40, i8 zeroext -70)
@@ -97,9 +97,9 @@ define i32 @bar0(i32 %i) nounwind {
 ;        ret i32 %tmp1
 ;}
 
-declare void @float_foo(float %f) ssp
+declare void @float_foo(float %f)
 
-define void @float_const() ssp {
+define void @float_const() nounwind {
 entry:
 ; ELF64: float_const
   call void @float_foo(float 0x401C666660000000)
@@ -108,7 +108,7 @@ entry:
   ret void
 }
 
-define void @float_reg(float %dummy, float %f) ssp {
+define void @float_reg(float %dummy, float %f) nounwind {
 entry:
 ; ELF64: float_reg
   call void @float_foo(float %f)
@@ -116,9 +116,9 @@ entry:
   ret void
 }
 
-declare void @double_foo(double %d) ssp
+declare void @double_foo(double %d)
 
-define void @double_const() ssp {
+define void @double_const() nounwind {
 entry:
 ; ELF64: double_const
   call void @double_foo(double 0x1397723CCABD0000401C666660000000)
@@ -127,7 +127,7 @@ entry:
   ret void
 }
 
-define void @double_reg(double %dummy, double %d) ssp {
+define void @double_reg(double %dummy, double %d) nounwind {
 entry:
 ; ELF64: double_reg
   call void @double_foo(double %d)
diff --git a/test/CodeGen/PowerPC/fast-isel-cmp-imm.ll b/test/CodeGen/PowerPC/fast-isel-cmp-imm.ll
index 24bd11bda40..5881dc3798a 100644
--- a/test/CodeGen/PowerPC/fast-isel-cmp-imm.ll
+++ b/test/CodeGen/PowerPC/fast-isel-cmp-imm.ll
@@ -3,7 +3,7 @@
 ; When fastisel better supports VSX fix up this test case.
 ;
 ; RUN: llc < %s -O0 -verify-machineinstrs -fast-isel-abort=1 -mtriple=powerpc64-unknown-linux-gnu -mcpu=pwr7 -mattr=-vsx | FileCheck %s --check-prefix=ELF64
-define void @t1a(float %a) uwtable ssp {
+define void @t1a(float %a) nounwind {
 entry:
 ; ELF64: t1a
   %cmp = fcmp oeq float %a, 0.000000e+00
@@ -22,7 +22,7 @@ if.end:                                           ; preds = %if.then, %entry
 
 declare void @foo()
 
-define void @t1b(float %a) uwtable ssp {
+define void @t1b(float %a) nounwind {
 entry:
 ; ELF64: t1b
   %cmp = fcmp oeq float %a, -0.000000e+00
@@ -39,7 +39,7 @@ if.end:                                           ; preds = %if.then, %entry
   ret void
 }
 
-define void @t2a(double %a) uwtable ssp {
+define void @t2a(double %a) nounwind {
 entry:
 ; ELF64: t2a
   %cmp = fcmp oeq double %a, 0.000000e+00
@@ -56,7 +56,7 @@ if.end:                                           ; preds = %if.then, %entry
   ret void
 }
 
-define void @t2b(double %a) uwtable ssp {
+define void @t2b(double %a) nounwind {
 entry:
 ; ELF64: t2b
   %cmp = fcmp oeq double %a, -0.000000e+00
@@ -73,7 +73,7 @@ if.end:                                           ; preds = %if.then, %entry
   ret void
 }
 
-define void @t4(i8 signext %a) uwtable ssp {
+define void @t4(i8 signext %a) nounwind {
 entry:
 ; ELF64: t4
   %cmp = icmp eq i8 %a, -1
@@ -89,7 +89,7 @@ if.end:                                           ; preds = %if.then, %entry
   ret void
 }
 
-define void @t5(i8 zeroext %a) uwtable ssp {
+define void @t5(i8 zeroext %a) nounwind {
 entry:
 ; ELF64: t5
   %cmp = icmp eq i8 %a, 1
@@ -105,7 +105,7 @@ if.end:                                           ; preds = %if.then, %entry
   ret void
 }
 
-define void @t6(i16 signext %a) uwtable ssp {
+define void @t6(i16 signext %a) nounwind {
 entry:
 ; ELF64: t6
   %cmp = icmp eq i16 %a, -1
@@ -121,7 +121,7 @@ if.end:                                           ; preds = %if.then, %entry
   ret void
 }
 
-define void @t7(i16 zeroext %a) uwtable ssp {
+define void @t7(i16 zeroext %a) nounwind {
 entry:
 ; ELF64: t7
   %cmp = icmp eq i16 %a, 1
@@ -137,7 +137,7 @@ if.end:                                           ; preds = %if.then, %entry
   ret void
 }
 
-define void @t8(i32 %a) uwtable ssp {
+define void @t8(i32 %a) nounwind {
 entry:
 ; ELF64: t8
   %cmp = icmp eq i32 %a, -1
@@ -152,7 +152,7 @@ if.end:                                           ; preds = %if.then, %entry
   ret void
 }
 
-define void @t9(i32 %a) uwtable ssp {
+define void @t9(i32 %a) nounwind {
 entry:
 ; ELF64: t9
   %cmp = icmp eq i32 %a, 1
@@ -167,7 +167,7 @@ if.end:                                           ; preds = %if.then, %entry
   ret void
 }
 
-define void @t10(i32 %a) uwtable ssp {
+define void @t10(i32 %a) nounwind {
 entry:
 ; ELF64: t10
   %cmp = icmp eq i32 %a, 384
@@ -182,7 +182,7 @@ if.end:                                           ; preds = %if.then, %entry
   ret void
 }
 
-define void @t11(i32 %a) uwtable ssp {
+define void @t11(i32 %a) nounwind {
 entry:
 ; ELF64: t11
   %cmp = icmp eq i32 %a, 4096
@@ -197,11 +197,11 @@ if.end:                                           ; preds = %if.then, %entry
   ret void
 }
 
-define void @t12(i8 %a) uwtable ssp {
+define void @t12(i8 %a) nounwind {
 entry:
 ; ELF64: t12
   %cmp = icmp ugt i8 %a, -113
-; ELF64: rlwinm
+; ELF64: clrlwi
 ; ELF64: cmplwi
   br i1 %cmp, label %if.then, label %if.end
 
@@ -229,7 +229,7 @@ if.end:                                           ; preds = %entry
   ret void
 }
 
-define void @t14(i64 %a) uwtable ssp {
+define void @t14(i64 %a) nounwind {
 entry:
 ; ELF64: t14
   %cmp = icmp eq i64 %a, -1
@@ -244,7 +244,7 @@ if.end:                                           ; preds = %if.then, %entry
   ret void
 }
 
-define void @t15(i64 %a) uwtable ssp {
+define void @t15(i64 %a) nounwind {
 entry:
 ; ELF64: t15
   %cmp = icmp eq i64 %a, 1
@@ -259,7 +259,7 @@ if.end:                                           ; preds = %if.then, %entry
   ret void
 }
 
-define void @t16(i64 %a) uwtable ssp {
+define void @t16(i64 %a) nounwind {
 entry:
 ; ELF64: t16
   %cmp = icmp eq i64 %a, 384
@@ -274,7 +274,7 @@ if.end:                                           ; preds = %if.then, %entry
   ret void
 }
 
-define void @t17(i64 %a) uwtable ssp {
+define void @t17(i64 %a) nounwind {
 entry:
 ; ELF64: t17
   %cmp = icmp eq i64 %a, 32768
diff --git a/test/CodeGen/PowerPC/fast-isel-const.ll b/test/CodeGen/PowerPC/fast-isel-const.ll
index a751a2be6c6..3987e54a8d1 100644
--- a/test/CodeGen/PowerPC/fast-isel-const.ll
+++ b/test/CodeGen/PowerPC/fast-isel-const.ll
@@ -1,6 +1,6 @@
 ; RUN: llc < %s -O0 -verify-machineinstrs -fast-isel-abort=1 -mtriple=powerpc64-unknown-linux-gnu -mcpu=pwr7 -mattr=-vsx | FileCheck %s --check-prefix=ELF64
 
-define zeroext i1 @testi1(i8 %in) nounwind uwtable ssp {
+define zeroext i1 @testi1(i8 %in) nounwind {
 entry:
   %c = icmp eq i8 %in, 5
   br i1 %c, label %true, label %false
diff --git a/test/CodeGen/PowerPC/fast-isel-conversion-p5.ll b/test/CodeGen/PowerPC/fast-isel-conversion-p5.ll
index cfb934c6ab0..e4cdf8d7a9c 100644
--- a/test/CodeGen/PowerPC/fast-isel-conversion-p5.ll
+++ b/test/CodeGen/PowerPC/fast-isel-conversion-p5.ll
@@ -2,7 +2,7 @@
 
 ; Test sitofp
 
-define void @sitofp_double_i32(i32 %a, double %b) nounwind ssp {
+define void @sitofp_double_i32(i32 %a, double %b) nounwind {
 entry:
 ; ELF64: sitofp_double_i32
   %b.addr = alloca double, align 8
@@ -14,7 +14,7 @@ entry:
   ret void
 }
 
-define void @sitofp_double_i64(i64 %a, double %b) nounwind ssp {
+define void @sitofp_double_i64(i64 %a, double %b) nounwind {
 entry:
 ; ELF64: sitofp_double_i64
   %b.addr = alloca double, align 8
@@ -26,7 +26,7 @@ entry:
   ret void
 }
 
-define void @sitofp_double_i16(i16 %a, double %b) nounwind ssp {
+define void @sitofp_double_i16(i16 %a, double %b) nounwind {
 entry:
 ; ELF64: sitofp_double_i16
   %b.addr = alloca double, align 8
@@ -39,7 +39,7 @@ entry:
   ret void
 }
 
-define void @sitofp_double_i8(i8 %a, double %b) nounwind ssp {
+define void @sitofp_double_i8(i8 %a, double %b) nounwind {
 entry:
 ; ELF64: sitofp_double_i8
   %b.addr = alloca double, align 8
@@ -54,7 +54,7 @@ entry:
 
 ; Test fptosi
 
-define void @fptosi_float_i32(float %a) nounwind ssp {
+define void @fptosi_float_i32(float %a) nounwind {
 entry:
 ; ELF64: fptosi_float_i32
   %b.addr = alloca i32, align 4
@@ -66,7 +66,7 @@ entry:
   ret void
 }
 
-define void @fptosi_float_i64(float %a) nounwind ssp {
+define void @fptosi_float_i64(float %a) nounwind {
 entry:
 ; ELF64: fptosi_float_i64
   %b.addr = alloca i64, align 4
@@ -78,7 +78,7 @@ entry:
   ret void
 }
 
-define void @fptosi_double_i32(double %a) nounwind ssp {
+define void @fptosi_double_i32(double %a) nounwind {
 entry:
 ; ELF64: fptosi_double_i32
   %b.addr = alloca i32, align 8
@@ -90,7 +90,7 @@ entry:
   ret void
 }
 
-define void @fptosi_double_i64(double %a) nounwind ssp {
+define void @fptosi_double_i64(double %a) nounwind {
 entry:
 ; ELF64: fptosi_double_i64
   %b.addr = alloca i64, align 8
@@ -104,7 +104,7 @@ entry:
 
 ; Test fptoui
 
-define void @fptoui_float_i32(float %a) nounwind ssp {
+define void @fptoui_float_i32(float %a) nounwind {
 entry:
 ; ELF64: fptoui_float_i32
   %b.addr = alloca i32, align 4
@@ -116,7 +116,7 @@ entry:
   ret void
 }
 
-define void @fptoui_double_i32(double %a) nounwind ssp {
+define void @fptoui_double_i32(double %a) nounwind {
 entry:
 ; ELF64: fptoui_double_i32
   %b.addr = alloca i32, align 8
diff --git a/test/CodeGen/PowerPC/fast-isel-conversion.ll b/test/CodeGen/PowerPC/fast-isel-conversion.ll
index fe4580441eb..a9324592aea 100644
--- a/test/CodeGen/PowerPC/fast-isel-conversion.ll
+++ b/test/CodeGen/PowerPC/fast-isel-conversion.ll
@@ -11,7 +11,7 @@
 
 ; Test sitofp
 
-define void @sitofp_single_i64(i64 %a, float %b) nounwind ssp {
+define void @sitofp_single_i64(i64 %a, float %b) nounwind {
 entry:
 ; ELF64: sitofp_single_i64
 ; ELF64LE: sitofp_single_i64
@@ -32,7 +32,7 @@ entry:
   ret void
 }
 
-define void @sitofp_single_i32(i32 %a, float %b) nounwind ssp {
+define void @sitofp_single_i32(i32 %a, float %b) nounwind {
 entry:
 ; ELF64: sitofp_single_i32
 ; ELF64LE: sitofp_single_i32
@@ -57,7 +57,7 @@ entry:
   ret void
 }
 
-define void @sitofp_single_i16(i16 %a, float %b) nounwind ssp {
+define void @sitofp_single_i16(i16 %a, float %b) nounwind {
 entry:
 ; ELF64: sitofp_single_i16
 ; ELF64LE: sitofp_single_i16
@@ -81,7 +81,7 @@ entry:
   ret void
 }
 
-define void @sitofp_single_i8(i8 %a) nounwind ssp {
+define void @sitofp_single_i8(i8 %a) nounwind {
 entry:
 ; ELF64: sitofp_single_i8
 ; ELF64LE: sitofp_single_i8
@@ -105,7 +105,7 @@ entry:
   ret void
 }
 
-define void @sitofp_double_i32(i32 %a, double %b) nounwind ssp {
+define void @sitofp_double_i32(i32 %a, double %b) nounwind {
 entry:
 ; ELF64: sitofp_double_i32
 ; ELF64LE: sitofp_double_i32
@@ -129,7 +129,7 @@ entry:
   ret void
 }
 
-define void @sitofp_double_i64(i64 %a, double %b) nounwind ssp {
+define void @sitofp_double_i64(i64 %a, double %b) nounwind {
 entry:
 ; ELF64: sitofp_double_i64
 ; ELF64LE: sitofp_double_i64
@@ -149,7 +149,7 @@ entry:
   ret void
 }
 
-define void @sitofp_double_i16(i16 %a, double %b) nounwind ssp {
+define void @sitofp_double_i16(i16 %a, double %b) nounwind {
 entry:
 ; ELF64: sitofp_double_i16
 ; ELF64LE: sitofp_double_i16
@@ -172,7 +172,7 @@ entry:
   ret void
 }
 
-define void @sitofp_double_i8(i8 %a, double %b) nounwind ssp {
+define void @sitofp_double_i8(i8 %a, double %b) nounwind {
 entry:
 ; ELF64: sitofp_double_i8
 ; ELF64LE: sitofp_double_i8
@@ -197,7 +197,7 @@ entry:
 
 ; Test uitofp
 
-define void @uitofp_single_i64(i64 %a, float %b) nounwind ssp {
+define void @uitofp_single_i64(i64 %a, float %b) nounwind {
 entry:
 ; ELF64: uitofp_single_i64
 ; ELF64LE: uitofp_single_i64
@@ -215,7 +215,7 @@ entry:
   ret void
 }
 
-define void @uitofp_single_i32(i32 %a, float %b) nounwind ssp {
+define void @uitofp_single_i32(i32 %a, float %b) nounwind {
 entry:
 ; ELF64: uitofp_single_i32
 ; ELF64LE: uitofp_single_i32
@@ -238,7 +238,7 @@ entry:
   ret void
 }
 
-define void @uitofp_single_i16(i16 %a, float %b) nounwind ssp {
+define void @uitofp_single_i16(i16 %a, float %b) nounwind {
 entry:
 ; ELF64: uitofp_single_i16
 ; ELF64LE: uitofp_single_i16
@@ -253,7 +253,7 @@ entry:
 ; ELF64LE: std
 ; ELF64LE: lfd
 ; ELF64LE: fcfidus
-; PPC970: rlwinm {{[0-9]+}}, {{[0-9]+}}, 0, 16, 31
+; PPC970: clrlwi {{[0-9]+}}, {{[0-9]+}}, 16
 ; PPC970: std
 ; PPC970: lfd
 ; PPC970: fcfid
@@ -262,7 +262,7 @@ entry:
   ret void
 }
 
-define void @uitofp_single_i8(i8 %a) nounwind ssp {
+define void @uitofp_single_i8(i8 %a) nounwind {
 entry:
 ; ELF64: uitofp_single_i8
 ; ELF64LE: uitofp_single_i8
@@ -277,7 +277,7 @@ entry:
 ; ELF64LE: std
 ; ELF64LE: lfd
 ; ELF64LE: fcfidus
-; PPC970: rlwinm {{[0-9]+}}, {{[0-9]+}}, 0, 24, 31
+; PPC970: clrlwi {{[0-9]+}}, {{[0-9]+}}, 24
 ; PPC970: std
 ; PPC970: lfd
 ; PPC970: fcfid
@@ -286,7 +286,7 @@ entry:
   ret void
 }
 
-define void @uitofp_double_i64(i64 %a, double %b) nounwind ssp {
+define void @uitofp_double_i64(i64 %a, double %b) nounwind {
 entry:
 ; ELF64: uitofp_double_i64
 ; ELF64LE: uitofp_double_i64
@@ -304,7 +304,7 @@ entry:
   ret void
 }
 
-define void @uitofp_double_i32(i32 %a, double %b) nounwind ssp {
+define void @uitofp_double_i32(i32 %a, double %b) nounwind {
 entry:
 ; ELF64: uitofp_double_i32
 ; ELF64LE: uitofp_double_i32
@@ -327,7 +327,7 @@ entry:
   ret void
 }
 
-define void @uitofp_double_i16(i16 %a, double %b) nounwind ssp {
+define void @uitofp_double_i16(i16 %a, double %b) nounwind {
 entry:
 ; ELF64: uitofp_double_i16
 ; ELF64LE: uitofp_double_i16
@@ -342,7 +342,7 @@ entry:
 ; ELF64LE: std
 ; ELF64LE: lfd
 ; ELF64LE: fcfidu
-; PPC970: rlwinm {{[0-9]+}}, {{[0-9]+}}, 0, 16, 31
+; PPC970: clrlwi {{[0-9]+}}, {{[0-9]+}}, 16
 ; PPC970: std
 ; PPC970: lfd
 ; PPC970: fcfid
@@ -350,7 +350,7 @@ entry:
   ret void
 }
 
-define void @uitofp_double_i8(i8 %a, double %b) nounwind ssp {
+define void @uitofp_double_i8(i8 %a, double %b) nounwind {
 entry:
 ; ELF64: uitofp_double_i8
 ; ELF64LE: uitofp_double_i8
@@ -365,7 +365,7 @@ entry:
 ; ELF64LE: std
 ; ELF64LE: lfd
 ; ELF64LE: fcfidu
-; PPC970: rlwinm {{[0-9]+}}, {{[0-9]+}}, 0, 24, 31
+; PPC970: clrlwi {{[0-9]+}}, {{[0-9]+}}, 24
 ; PPC970: std
 ; PPC970: lfd
 ; PPC970: fcfid
@@ -375,7 +375,7 @@ entry:
 
 ; Test fptosi
 
-define void @fptosi_float_i32(float %a) nounwind ssp {
+define void @fptosi_float_i32(float %a) nounwind {
 entry:
 ; ELF64: fptosi_float_i32
 ; ELF64LE: fptosi_float_i32
@@ -395,7 +395,7 @@ entry:
   ret void
 }
 
-define void @fptosi_float_i64(float %a) nounwind ssp {
+define void @fptosi_float_i64(float %a) nounwind {
 entry:
 ; ELF64: fptosi_float_i64
 ; ELF64LE: fptosi_float_i64
@@ -415,7 +415,7 @@ entry:
   ret void
 }
 
-define void @fptosi_double_i32(double %a) nounwind ssp {
+define void @fptosi_double_i32(double %a) nounwind {
 entry:
 ; ELF64: fptosi_double_i32
 ; ELF64LE: fptosi_double_i32
@@ -435,7 +435,7 @@ entry:
   ret void
 }
 
-define void @fptosi_double_i64(double %a) nounwind ssp {
+define void @fptosi_double_i64(double %a) nounwind {
 entry:
 ; ELF64: fptosi_double_i64
 ; ELF64LE: fptosi_double_i64
@@ -457,7 +457,7 @@ entry:
 
 ; Test fptoui
 
-define void @fptoui_float_i32(float %a) nounwind ssp {
+define void @fptoui_float_i32(float %a) nounwind {
 entry:
 ; ELF64: fptoui_float_i32
 ; ELF64LE: fptoui_float_i32
@@ -477,7 +477,7 @@ entry:
   ret void
 }
 
-define void @fptoui_float_i64(float %a) nounwind ssp {
+define void @fptoui_float_i64(float %a) nounwind {
 entry:
 ; ELF64: fptoui_float_i64
 ; ELF64LE: fptoui_float_i64
@@ -495,7 +495,7 @@ entry:
   ret void
 }
 
-define void @fptoui_double_i32(double %a) nounwind ssp {
+define void @fptoui_double_i32(double %a) nounwind {
 entry:
 ; ELF64: fptoui_double_i32
 ; ELF64LE: fptoui_double_i32
@@ -515,7 +515,7 @@ entry:
   ret void
 }
 
-define void @fptoui_double_i64(double %a) nounwind ssp {
+define void @fptoui_double_i64(double %a) nounwind {
 entry:
 ; ELF64: fptoui_double_i64
 ; ELF64LE: fptoui_double_i64
diff --git a/test/CodeGen/PowerPC/fast-isel-crash.ll b/test/CodeGen/PowerPC/fast-isel-crash.ll
index 55e87effcd8..e20ef6bcd5d 100644
--- a/test/CodeGen/PowerPC/fast-isel-crash.ll
+++ b/test/CodeGen/PowerPC/fast-isel-crash.ll
@@ -11,12 +11,12 @@ entry:
   ret void
 }
 
-define internal i32 @_Z13get_global_idj(i32 %dim) nounwind ssp {
+define internal i32 @_Z13get_global_idj(i32 %dim) nounwind {
 entry:
   ret i32 undef
 }
 
-define void @wrap(i8 addrspace(1)* addrspace(1)* %arglist, i32 addrspace(1)* %gtid) nounwind ssp {
+define void @wrap(i8 addrspace(1)* addrspace(1)* %arglist, i32 addrspace(1)* %gtid) nounwind {
 entry:
   call void @stretch(<4 x i8> addrspace(1)* undef, <4 x i8> addrspace(1)* undef, i32 undef, i32 undef, i32 undef, i32 undef, <2 x float> undef, <4 x float> undef)
   ret void
diff --git a/test/CodeGen/PowerPC/fast-isel-ext.ll b/test/CodeGen/PowerPC/fast-isel-ext.ll
index 69fa9945ded..ce8ac440b79 100644
--- a/test/CodeGen/PowerPC/fast-isel-ext.ll
+++ b/test/CodeGen/PowerPC/fast-isel-ext.ll
@@ -2,35 +2,35 @@
 
 ; zext
 
-define i32 @zext_8_32(i8 %a) nounwind ssp {
+define i32 @zext_8_32(i8 %a) nounwind {
 ; ELF64: zext_8_32
   %r = zext i8 %a to i32
-; ELF64: rlwinm {{[0-9]+}}, {{[0-9]+}}, 0, 24, 31
+; ELF64: clrlwi {{[0-9]+}}, {{[0-9]+}}, 24
   ret i32 %r
 }
 
-define i32 @zext_16_32(i16 %a) nounwind ssp {
+define i32 @zext_16_32(i16 %a) nounwind {
 ; ELF64: zext_16_32
   %r = zext i16 %a to i32
-; ELF64: rlwinm {{[0-9]+}}, {{[0-9]+}}, 0, 16, 31
+; ELF64: clrlwi {{[0-9]+}}, {{[0-9]+}}, 16
   ret i32 %r
 }
 
-define i64 @zext_8_64(i8 %a) nounwind ssp {
+define i64 @zext_8_64(i8 %a) nounwind {
 ; ELF64: zext_8_64
   %r = zext i8 %a to i64
 ; ELF64: rldicl {{[0-9]+}}, {{[0-9]+}}, 0, 56
   ret i64 %r
 }
 
-define i64 @zext_16_64(i16 %a) nounwind ssp {
+define i64 @zext_16_64(i16 %a) nounwind {
 ; ELF64: zext_16_64
   %r = zext i16 %a to i64
 ; ELF64: rldicl {{[0-9]+}}, {{[0-9]+}}, 0, 48
   ret i64 %r
 }
 
-define i64 @zext_32_64(i32 %a) nounwind ssp {
+define i64 @zext_32_64(i32 %a) nounwind {
 ; ELF64: zext_32_64
   %r = zext i32 %a to i64
 ; ELF64: rldicl {{[0-9]+}}, {{[0-9]+}}, 0, 32
@@ -39,35 +39,35 @@ define i64 @zext_32_64(i32 %a) nounwind ssp {
 
 ; sext
 
-define i32 @sext_8_32(i8 %a) nounwind ssp {
+define i32 @sext_8_32(i8 %a) nounwind {
 ; ELF64: sext_8_32
   %r = sext i8 %a to i32
 ; ELF64: extsb
   ret i32 %r
 }
 
-define i32 @sext_16_32(i16 %a) nounwind ssp {
+define i32 @sext_16_32(i16 %a) nounwind {
 ; ELF64: sext_16_32
   %r = sext i16 %a to i32
 ; ELF64: extsh
   ret i32 %r
 }
 
-define i64 @sext_8_64(i8 %a) nounwind ssp {
+define i64 @sext_8_64(i8 %a) nounwind {
 ; ELF64: sext_8_64
   %r = sext i8 %a to i64
 ; ELF64: extsb
   ret i64 %r
 }
 
-define i64 @sext_16_64(i16 %a) nounwind ssp {
+define i64 @sext_16_64(i16 %a) nounwind {
 ; ELF64: sext_16_64
   %r = sext i16 %a to i64
 ; ELF64: extsh
   ret i64 %r
 }
 
-define i64 @sext_32_64(i32 %a) nounwind ssp {
+define i64 @sext_32_64(i32 %a) nounwind {
 ; ELF64: sext_32_64
   %r = sext i32 %a to i64
 ; ELF64: extsw
diff --git a/test/CodeGen/PowerPC/fast-isel-fold.ll b/test/CodeGen/PowerPC/fast-isel-fold.ll
index e56101a28e2..24cdca35b0d 100644
--- a/test/CodeGen/PowerPC/fast-isel-fold.ll
+++ b/test/CodeGen/PowerPC/fast-isel-fold.ll
@@ -4,7 +4,7 @@
 @b = global i16 2, align 2
 @c = global i32 4, align 4
 
-define void @t1() nounwind uwtable ssp {
+define void @t1() nounwind {
 ; ELF64: t1
   %1 = load i8, i8* @a, align 1
   call void @foo1(i8 zeroext %1)
@@ -14,7 +14,7 @@ define void @t1() nounwind uwtable ssp {
   ret void
 }
 
-define void @t2() nounwind uwtable ssp {
+define void @t2() nounwind {
 ; ELF64: t2
   %1 = load i16, i16* @b, align 2
   call void @foo2(i16 zeroext %1)
@@ -24,7 +24,7 @@ define void @t2() nounwind uwtable ssp {
   ret void
 }
 
-define void @t2a() nounwind uwtable ssp {
+define void @t2a() nounwind {
 ; ELF64: t2a
   %1 = load i32, i32* @c, align 4
   call void @foo3(i32 zeroext %1)
@@ -38,7 +38,7 @@ declare void @foo1(i8 zeroext)
 declare void @foo2(i16 zeroext)
 declare void @foo3(i32 zeroext)
 
-define i32 @t3() nounwind uwtable ssp {
+define i32 @t3() nounwind {
 ; ELF64: t3
   %1 = load i8, i8* @a, align 1
   %2 = zext i8 %1 to i32
@@ -47,7 +47,7 @@ define i32 @t3() nounwind uwtable ssp {
   ret i32 %2
 }
 
-define i32 @t4() nounwind uwtable ssp {
+define i32 @t4() nounwind {
 ; ELF64: t4
   %1 = load i16, i16* @b, align 2
   %2 = zext i16 %1 to i32
@@ -56,7 +56,7 @@ define i32 @t4() nounwind uwtable ssp {
   ret i32 %2
 }
 
-define i32 @t5() nounwind uwtable ssp {
+define i32 @t5() nounwind {
 ; ELF64: t5
   %1 = load i16, i16* @b, align 2
   %2 = sext i16 %1 to i32
@@ -65,7 +65,7 @@ define i32 @t5() nounwind uwtable ssp {
   ret i32 %2
 }
 
-define i32 @t6() nounwind uwtable ssp {
+define i32 @t6() nounwind {
 ; ELF64: t6
   %1 = load i8, i8* @a, align 2
   %2 = sext i8 %1 to i32
@@ -74,7 +74,7 @@ define i32 @t6() nounwind uwtable ssp {
   ret i32 %2
 }
 
-define i64 @t7() nounwind uwtable ssp {
+define i64 @t7() nounwind {
 ; ELF64: t7
   %1 = load i8, i8* @a, align 1
   %2 = zext i8 %1 to i64
@@ -83,7 +83,7 @@ define i64 @t7() nounwind uwtable ssp {
   ret i64 %2
 }
 
-define i64 @t8() nounwind uwtable ssp {
+define i64 @t8() nounwind {
 ; ELF64: t8
   %1 = load i16, i16* @b, align 2
   %2 = zext i16 %1 to i64
@@ -92,7 +92,7 @@ define i64 @t8() nounwind uwtable ssp {
   ret i64 %2
 }
 
-define i64 @t9() nounwind uwtable ssp {
+define i64 @t9() nounwind {
 ; ELF64: t9
   %1 = load i16, i16* @b, align 2
   %2 = sext i16 %1 to i64
@@ -101,7 +101,7 @@ define i64 @t9() nounwind uwtable ssp {
   ret i64 %2
 }
 
-define i64 @t10() nounwind uwtable ssp {
+define i64 @t10() nounwind {
 ; ELF64: t10
   %1 = load i8, i8* @a, align 2
   %2 = sext i8 %1 to i64
@@ -110,7 +110,7 @@ define i64 @t10() nounwind uwtable ssp {
   ret i64 %2
 }
 
-define i64 @t11() nounwind uwtable ssp {
+define i64 @t11() nounwind {
 ; ELF64: t11
   %1 = load i32, i32* @c, align 4
   %2 = zext i32 %1 to i64
@@ -119,7 +119,7 @@ define i64 @t11() nounwind uwtable ssp {
   ret i64 %2
 }
 
-define i64 @t12() nounwind uwtable ssp {
+define i64 @t12() nounwind {
 ; ELF64: t12
   %1 = load i32, i32* @c, align 4
   %2 = sext i32 %1 to i64
diff --git a/test/CodeGen/PowerPC/fast-isel-icmp-split.ll b/test/CodeGen/PowerPC/fast-isel-icmp-split.ll
new file mode 100644
index 00000000000..e1f22781db3
--- /dev/null
+++ b/test/CodeGen/PowerPC/fast-isel-icmp-split.ll
@@ -0,0 +1,72 @@
+; RUN: llc -O0 -relocation-model=pic < %s | FileCheck %s
+target datalayout = "E-m:e-i64:64-n32:64"
+target triple = "powerpc64-bgq-linux"
+
+%"class.std::__1::__tree_node.130.151" = type { %"class.std::__1::__tree_node_base.base.128.149", %"class.boost::serialization::extended_type_info.129.150"* }
+%"class.std::__1::__tree_node_base.base.128.149" = type <{ %"class.std::__1::__tree_end_node.127.148", %"class.std::__1::__tree_node_base.126.147"*, %"class.std::__1::__tree_node_base.126.147"*, i8 }>
+%"class.std::__1::__tree_end_node.127.148" = type { %"class.std::__1::__tree_node_base.126.147"* }
+%"class.std::__1::__tree_node_base.126.147" = type <{ %"class.std::__1::__tree_end_node.127.148", %"class.std::__1::__tree_node_base.126.147"*, %"class.std::__1::__tree_node_base.126.147"*, i8, [7 x i8] }>
+%"class.boost::serialization::extended_type_info.129.150" = type { i32 (...)**, i32, i8* }
+
+; Function Attrs: noinline
+define void @_ZN5boost13serialization18extended_type_info4findEPKc() #0 align 2 personality i8* bitcast (i32 (...)* @__gxx_personality_v0 to i8*) {
+entry:
+  br i1 undef, label %cond.true, label %cond.false
+
+; CHECK: @_ZN5boost13serialization18extended_type_info4findEPKc
+
+cond.true:                                        ; preds = %entry
+  br label %cond.end
+
+cond.false:                                       ; preds = %entry
+  unreachable
+                                                  ; No predecessors!
+  br label %cond.end
+
+cond.end:                                         ; preds = %0, %cond.true
+  invoke void @_ZNKSt3__16__treeIPKN5boost13serialization18extended_type_infoENS2_6detail11key_compareENS_9allocatorIS5_EEE4findIS5_EENS_21__tree_const_iteratorIS5_PNS_11__tree_nodeIS5_PvEElEERKT_()
+          to label %_ZNKSt3__18multisetIPKN5boost13serialization18extended_type_infoENS2_6detail11key_compareENS_9allocatorIS5_EEE4findERKS5_.exit unwind label %lpad
+
+_ZNKSt3__18multisetIPKN5boost13serialization18extended_type_infoENS2_6detail11key_compareENS_9allocatorIS5_EEE4findERKS5_.exit: ; preds = %cond.end
+  br label %invoke.cont
+
+invoke.cont:                                      ; preds = %_ZNKSt3__18multisetIPKN5boost13serialization18extended_type_infoENS2_6detail11key_compareENS_9allocatorIS5_EEE4findERKS5_.exit
+  %1 = load %"class.std::__1::__tree_node.130.151"*, %"class.std::__1::__tree_node.130.151"** undef, align 8
+  %cmp.i = icmp eq %"class.std::__1::__tree_node.130.151"* undef, %1
+  br label %invoke.cont.2
+
+invoke.cont.2:                                    ; preds = %invoke.cont
+  br i1 %cmp.i, label %if.then, label %if.end
+
+if.then:                                          ; preds = %invoke.cont.2
+  br label %cleanup
+
+lpad:                                             ; preds = %cond.end
+  %2 = landingpad { i8*, i32 }
+          cleanup
+  br label %eh.resume
+
+if.end:                                           ; preds = %invoke.cont.2
+  br label %invoke.cont.4
+
+invoke.cont.4:                                    ; preds = %if.end
+  br label %cleanup
+
+cleanup:                                          ; preds = %invoke.cont.4, %if.then
+  ret void
+
+eh.resume:                                        ; preds = %lpad
+  resume { i8*, i32 } undef
+}
+
+declare i32 @__gxx_personality_v0(...)
+
+; Function Attrs: noinline
+declare void @_ZNKSt3__16__treeIPKN5boost13serialization18extended_type_infoENS2_6detail11key_compareENS_9allocatorIS5_EEE4findIS5_EENS_21__tree_const_iteratorIS5_PNS_11__tree_nodeIS5_PvEElEERKT_() #0 align 2
+
+attributes #0 = { noinline "target-cpu"="a2q" }
+
+!llvm.module.flags = !{!0}
+
+!0 = !{i32 1, !"PIC Level", i32 2}
+
diff --git a/test/CodeGen/PowerPC/fast-isel-indirectbr.ll b/test/CodeGen/PowerPC/fast-isel-indirectbr.ll
index b5477134c51..d66fd1fb752 100644
--- a/test/CodeGen/PowerPC/fast-isel-indirectbr.ll
+++ b/test/CodeGen/PowerPC/fast-isel-indirectbr.ll
@@ -1,6 +1,6 @@
 ; RUN: llc < %s -O0 -verify-machineinstrs -fast-isel-abort=1 -mtriple=powerpc64-unknown-linux-gnu -mcpu=pwr7 | FileCheck %s --check-prefix=ELF64
 
-define void @t1(i8* %x) {
+define void @t1(i8* %x) nounwind {
 entry:
 ; ELF64: t1
   br label %L0
diff --git a/test/CodeGen/PowerPC/fast-isel-load-store.ll b/test/CodeGen/PowerPC/fast-isel-load-store.ll
index f6a55f06b2c..800e5aa6609 100644
--- a/test/CodeGen/PowerPC/fast-isel-load-store.ll
+++ b/test/CodeGen/PowerPC/fast-isel-load-store.ll
@@ -24,7 +24,7 @@
 
 ; load
 
-define i8 @t1() nounwind uwtable ssp {
+define i8 @t1() nounwind {
 ; ELF64: t1
   %1 = load i8, i8* @a, align 1
 ; ELF64: lbz
@@ -33,7 +33,7 @@ define i8 @t1() nounwind uwtable ssp {
   ret i8 %2
 }
 
-define i16 @t2() nounwind uwtable ssp {
+define i16 @t2() nounwind {
 ; ELF64: t2
   %1 = load i16, i16* @b, align 2
 ; ELF64: lhz
@@ -42,7 +42,7 @@ define i16 @t2() nounwind uwtable ssp {
   ret i16 %2
 }
 
-define i32 @t3() nounwind uwtable ssp {
+define i32 @t3() nounwind {
 ; ELF64: t3
   %1 = load i32, i32* @c, align 4
 ; ELF64: lwz
@@ -51,7 +51,7 @@ define i32 @t3() nounwind uwtable ssp {
   ret i32 %2
 }
 
-define i64 @t4() nounwind uwtable ssp {
+define i64 @t4() nounwind {
 ; ELF64: t4
   %1 = load i64, i64* @d, align 4
 ; ELF64: ld
@@ -60,7 +60,7 @@ define i64 @t4() nounwind uwtable ssp {
   ret i64 %2
 }
 
-define float @t5() nounwind uwtable ssp {
+define float @t5() nounwind {
 ; ELF64: t5
   %1 = load float, float* @e, align 4
 ; ELF64: lfs
@@ -69,7 +69,7 @@ define float @t5() nounwind uwtable ssp {
   ret float %2
 }
 
-define double @t6() nounwind uwtable ssp {
+define double @t6() nounwind {
 ; ELF64: t6
   %1 = load double, double* @f, align 8
 ; ELF64: lfd
@@ -80,7 +80,7 @@ define double @t6() nounwind uwtable ssp {
 
 ; store
 
-define void @t7(i8 %v) nounwind uwtable ssp {
+define void @t7(i8 %v) nounwind {
 ; ELF64: t7
   %1 = add nsw i8 %v, 1
   store i8 %1, i8* @a, align 1
@@ -91,7 +91,7 @@ define void @t7(i8 %v) nounwind uwtable ssp {
   ret void
 }
 
-define void @t8(i16 %v) nounwind uwtable ssp {
+define void @t8(i16 %v) nounwind {
 ; ELF64: t8
   %1 = add nsw i16 %v, 1
   store i16 %1, i16* @b, align 2
@@ -102,7 +102,7 @@ define void @t8(i16 %v) nounwind uwtable ssp {
   ret void
 }
 
-define void @t9(i32 %v) nounwind uwtable ssp {
+define void @t9(i32 %v) nounwind {
 ; ELF64: t9
   %1 = add nsw i32 %v, 1
   store i32 %1, i32* @c, align 4
@@ -113,7 +113,7 @@ define void @t9(i32 %v) nounwind uwtable ssp {
   ret void
 }
 
-define void @t10(i64 %v) nounwind uwtable ssp {
+define void @t10(i64 %v) nounwind {
 ; ELF64: t10
   %1 = add nsw i64 %v, 1
   store i64 %1, i64* @d, align 4
@@ -124,7 +124,7 @@ define void @t10(i64 %v) nounwind uwtable ssp {
   ret void
 }
 
-define void @t11(float %v) nounwind uwtable ssp {
+define void @t11(float %v) nounwind {
 ; ELF64: t11
   %1 = fadd float %v, 1.0
   store float %1, float* @e, align 4
@@ -133,7 +133,7 @@ define void @t11(float %v) nounwind uwtable ssp {
   ret void
 }
 
-define void @t12(double %v) nounwind uwtable ssp {
+define void @t12(double %v) nounwind {
 ; ELF64: t12
   %1 = fadd double %v, 1.0
   store double %1, double* @f, align 8
@@ -143,7 +143,7 @@ define void @t12(double %v) nounwind uwtable ssp {
 }
 
 ;; lwa requires an offset divisible by 4, so we need lwax here.
-define i64 @t13() nounwind uwtable ssp {
+define i64 @t13() nounwind {
 ; ELF64: t13
   %1 = load i32, i32* getelementptr inbounds (%struct.s, %struct.s* @g, i32 0, i32 1), align 1
   %2 = sext i32 %1 to i64
@@ -155,7 +155,7 @@ define i64 @t13() nounwind uwtable ssp {
 }
 
 ;; ld requires an offset divisible by 4, so we need ldx here.
-define i64 @t14() nounwind uwtable ssp {
+define i64 @t14() nounwind {
 ; ELF64: t14
   %1 = load i64, i64* getelementptr inbounds (%struct.t, %struct.t* @h, i32 0, i32 1), align 1
 ; ELF64: li
@@ -166,7 +166,7 @@ define i64 @t14() nounwind uwtable ssp {
 }
 
 ;; std requires an offset divisible by 4, so we need stdx here.
-define void @t15(i64 %v) nounwind uwtable ssp {
+define void @t15(i64 %v) nounwind {
 ; ELF64: t15
   %1 = add nsw i64 %v, 1
   store i64 %1, i64* getelementptr inbounds (%struct.t, %struct.t* @h, i32 0, i32 1), align 1
@@ -179,7 +179,7 @@ define void @t15(i64 %v) nounwind uwtable ssp {
 }
 
 ;; ld requires an offset that fits in 16 bits, so we need ldx here.
-define i64 @t16() nounwind uwtable ssp {
+define i64 @t16() nounwind {
 ; ELF64: t16
   %1 = load i64, i64* getelementptr inbounds ([8192 x i64], [8192 x i64]* @i, i32 0, i64 5000), align 8
 ; ELF64: lis
@@ -191,7 +191,7 @@ define i64 @t16() nounwind uwtable ssp {
 }
 
 ;; std requires an offset that fits in 16 bits, so we need stdx here.
-define void @t17(i64 %v) nounwind uwtable ssp {
+define void @t17(i64 %v) nounwind {
 ; ELF64: t17
   %1 = add nsw i64 %v, 1
   store i64 %1, i64* getelementptr inbounds ([8192 x i64], [8192 x i64]* @i, i32 0, i64 5000), align 8
diff --git a/test/CodeGen/PowerPC/fast-isel-redefinition.ll b/test/CodeGen/PowerPC/fast-isel-redefinition.ll
index 60706a6e143..19392c938e1 100644
--- a/test/CodeGen/PowerPC/fast-isel-redefinition.ll
+++ b/test/CodeGen/PowerPC/fast-isel-redefinition.ll
@@ -3,7 +3,7 @@
 ; doesn't crash.  (It crashed formerly on ARM, and proved useful in
 ; discovering a bug on PowerPC as well.)
 
-define i32 @f(i32* %x) nounwind ssp {
+define i32 @f(i32* %x) nounwind {
   %y = getelementptr inbounds i32, i32* %x, i32 5000
   %tmp103 = load i32, i32* %y, align 4
   ret i32 %tmp103
diff --git a/test/CodeGen/PowerPC/fast-isel-ret.ll b/test/CodeGen/PowerPC/fast-isel-ret.ll
index 1e4566d94df..e05ef7d9ab8 100644
--- a/test/CodeGen/PowerPC/fast-isel-ret.ll
+++ b/test/CodeGen/PowerPC/fast-isel-ret.ll
@@ -4,7 +4,7 @@
 ;
 ; RUN: llc < %s -O0 -verify-machineinstrs -fast-isel-abort=1 -mtriple=powerpc64-unknown-linux-gnu -mcpu=pwr7 -mattr=-vsx | FileCheck %s --check-prefix=ELF64
 
-define zeroext i1 @rettrue() nounwind uwtable ssp {
+define zeroext i1 @rettrue() nounwind {
 entry:
 ; ELF64-LABEL: rettrue
 ; ELF64: li 3, 1
@@ -12,7 +12,7 @@ entry:
   ret i1 true
 }
 
-define zeroext i1 @retfalse() nounwind uwtable ssp {
+define zeroext i1 @retfalse() nounwind {
 entry:
 ; ELF64-LABEL: retfalse
 ; ELF64: li 3, 0
@@ -20,7 +20,7 @@ entry:
   ret i1 false
 }
 
-define signext i1 @retstrue() nounwind uwtable ssp {
+define signext i1 @retstrue() nounwind {
 entry:
 ; ELF64-LABEL: retstrue
 ; ELF64: li 3, -1
@@ -28,7 +28,7 @@ entry:
   ret i1 true
 }
 
-define signext i1 @retsfalse() nounwind uwtable ssp {
+define signext i1 @retsfalse() nounwind {
 entry:
 ; ELF64-LABEL: retsfalse
 ; ELF64: li 3, 0
@@ -36,7 +36,7 @@ entry:
   ret i1 false
 }
 
-define signext i8 @ret2(i8 signext %a) nounwind uwtable ssp {
+define signext i8 @ret2(i8 signext %a) nounwind {
 entry:
 ; ELF64-LABEL: ret2
 ; ELF64: extsb
@@ -44,7 +44,7 @@ entry:
   ret i8 %a
 }
 
-define zeroext i8 @ret3(i8 signext %a) nounwind uwtable ssp {
+define zeroext i8 @ret3(i8 signext %a) nounwind {
 entry:
 ; ELF64-LABEL: ret3
 ; ELF64: rldicl {{[0-9]+}}, {{[0-9]+}}, 0, 56
@@ -52,7 +52,7 @@ entry:
   ret i8 %a
 }
 
-define signext i16 @ret4(i16 signext %a) nounwind uwtable ssp {
+define signext i16 @ret4(i16 signext %a) nounwind {
 entry:
 ; ELF64-LABEL: ret4
 ; ELF64: extsh
@@ -60,7 +60,7 @@ entry:
   ret i16 %a
 }
 
-define zeroext i16 @ret5(i16 signext %a) nounwind uwtable ssp {
+define zeroext i16 @ret5(i16 signext %a) nounwind {
 entry:
 ; ELF64-LABEL: ret5
 ; ELF64: rldicl {{[0-9]+}}, {{[0-9]+}}, 0, 48
@@ -68,7 +68,7 @@ entry:
   ret i16 %a
 }
 
-define i16 @ret6(i16 %a) nounwind uwtable ssp {
+define i16 @ret6(i16 %a) nounwind {
 entry:
 ; ELF64-LABEL: ret6
 ; ELF64: rldicl {{[0-9]+}}, {{[0-9]+}}, 0, 48
@@ -76,7 +76,7 @@ entry:
   ret i16 %a
 }
 
-define signext i32 @ret7(i32 signext %a) nounwind uwtable ssp {
+define signext i32 @ret7(i32 signext %a) nounwind {
 entry:
 ; ELF64-LABEL: ret7
 ; ELF64: extsw
@@ -84,7 +84,7 @@ entry:
   ret i32 %a
 }
 
-define zeroext i32 @ret8(i32 signext %a) nounwind uwtable ssp {
+define zeroext i32 @ret8(i32 signext %a) nounwind {
 entry:
 ; ELF64-LABEL: ret8
 ; ELF64: rldicl {{[0-9]+}}, {{[0-9]+}}, 0, 32
@@ -92,7 +92,7 @@ entry:
   ret i32 %a
 }
 
-define i32 @ret9(i32 %a) nounwind uwtable ssp {
+define i32 @ret9(i32 %a) nounwind {
 entry:
 ; ELF64-LABEL: ret9
 ; ELF64: rldicl {{[0-9]+}}, {{[0-9]+}}, 0, 32
@@ -100,7 +100,7 @@ entry:
   ret i32 %a
 }
 
-define i64 @ret10(i64 %a) nounwind uwtable ssp {
+define i64 @ret10(i64 %a) nounwind {
 entry:
 ; ELF64-LABEL: ret10
 ; ELF64-NOT: exts
@@ -109,21 +109,21 @@ entry:
   ret i64 %a
 }
 
-define float @ret11(float %a) nounwind uwtable ssp {
+define float @ret11(float %a) nounwind {
 entry:
 ; ELF64-LABEL: ret11
 ; ELF64: blr
   ret float %a
 }
 
-define double @ret12(double %a) nounwind uwtable ssp {
+define double @ret12(double %a) nounwind {
 entry:
 ; ELF64-LABEL: ret12
 ; ELF64: blr
   ret double %a
 }
 
-define i8 @ret13() nounwind uwtable ssp {
+define i8 @ret13() nounwind {
 entry:
 ; ELF64-LABEL: ret13
 ; ELF64: li
@@ -131,7 +131,7 @@ entry:
   ret i8 15;
 }
 
-define i16 @ret14() nounwind uwtable ssp {
+define i16 @ret14() nounwind {
 entry:
 ; ELF64-LABEL: ret14
 ; ELF64: li
@@ -139,7 +139,7 @@ entry:
   ret i16 -225;
 }
 
-define i32 @ret15() nounwind uwtable ssp {
+define i32 @ret15() nounwind {
 entry:
 ; ELF64-LABEL: ret15
 ; ELF64: lis
@@ -148,7 +148,7 @@ entry:
   ret i32 278135;
 }
 
-define i64 @ret16() nounwind uwtable ssp {
+define i64 @ret16() nounwind {
 entry:
 ; ELF64-LABEL: ret16
 ; ELF64: li
@@ -159,7 +159,7 @@ entry:
   ret i64 27813515225;
 }
 
-define float @ret17() nounwind uwtable ssp {
+define float @ret17() nounwind {
 entry:
 ; ELF64-LABEL: ret17
 ; ELF64: addis
@@ -168,7 +168,7 @@ entry:
   ret float 2.5;
 }
 
-define double @ret18() nounwind uwtable ssp {
+define double @ret18() nounwind {
 entry:
 ; ELF64-LABEL: ret18
 ; ELF64: addis
@@ -176,3 +176,13 @@ entry:
 ; ELF64: blr
   ret double 2.5e-33;
 }
+
+define zeroext i32 @ret19() nounwind {
+entry:
+; ELF64-LABEL: ret19
+; ELF64: li
+; ELF64: oris
+; ELF64: ori
+; ELF64: blr
+  ret i32 -1
+}
diff --git a/test/CodeGen/PowerPC/fast-isel-shifter.ll b/test/CodeGen/PowerPC/fast-isel-shifter.ll
index c18f659dde1..04cb4192060 100644
--- a/test/CodeGen/PowerPC/fast-isel-shifter.ll
+++ b/test/CodeGen/PowerPC/fast-isel-shifter.ll
@@ -1,6 +1,6 @@
 ; RUN: llc < %s -O0 -verify-machineinstrs -fast-isel-abort=1 -mtriple=powerpc64-unknown-linux-gnu -mcpu=pwr7 | FileCheck %s --check-prefix=ELF64
 
-define i32 @shl() nounwind ssp {
+define i32 @shl() nounwind {
 entry:
 ; ELF64: shl
 ; ELF64: slw
@@ -8,7 +8,7 @@ entry:
   ret i32 %shl
 }
 
-define i32 @shl_reg(i32 %src1, i32 %src2) nounwind ssp {
+define i32 @shl_reg(i32 %src1, i32 %src2) nounwind {
 entry:
 ; ELF64: shl_reg
 ; ELF64: slw
@@ -16,7 +16,7 @@ entry:
   ret i32 %shl
 }
 
-define i32 @lshr() nounwind ssp {
+define i32 @lshr() nounwind {
 entry:
 ; ELF64: lshr
 ; ELF64: srw
@@ -24,7 +24,7 @@ entry:
   ret i32 %lshr
 }
 
-define i32 @lshr_reg(i32 %src1, i32 %src2) nounwind ssp {
+define i32 @lshr_reg(i32 %src1, i32 %src2) nounwind {
 entry:
 ; ELF64: lshr_reg
 ; ELF64: srw
@@ -32,7 +32,7 @@ entry:
   ret i32 %lshr
 }
 
-define i32 @ashr() nounwind ssp {
+define i32 @ashr() nounwind {
 entry:
 ; ELF64: ashr
 ; ELF64: srawi
@@ -40,7 +40,7 @@ entry:
   ret i32 %ashr
 }
 
-define i32 @ashr_reg(i32 %src1, i32 %src2) nounwind ssp {
+define i32 @ashr_reg(i32 %src1, i32 %src2) nounwind {
 entry:
 ; ELF64: ashr_reg
 ; ELF64: sraw
diff --git a/test/CodeGen/PowerPC/fastisel-gep-promote-before-add.ll b/test/CodeGen/PowerPC/fastisel-gep-promote-before-add.ll
index 96cf67c869f..32f4c23c2de 100644
--- a/test/CodeGen/PowerPC/fastisel-gep-promote-before-add.ll
+++ b/test/CodeGen/PowerPC/fastisel-gep-promote-before-add.ll
@@ -2,7 +2,7 @@
 ; sext(a) + sext(b) != sext(a + b)
 ; RUN: llc -mtriple=powerpc64-unknown-freebsd10.0 %s -O0 -o - | FileCheck %s
 
-define zeroext i8 @gep_promotion(i8* %ptr) nounwind uwtable ssp {
+define zeroext i8 @gep_promotion(i8* %ptr) nounwind {
 entry:
   %ptr.addr = alloca i8*, align 8
   %add = add i8 64, 64 ; 0x40 + 0x40
diff --git a/test/CodeGen/PowerPC/fma-assoc.ll b/test/CodeGen/PowerPC/fma-assoc.ll
index dc1316e5e24..3044dd09128 100644
--- a/test/CodeGen/PowerPC/fma-assoc.ll
+++ b/test/CodeGen/PowerPC/fma-assoc.ll
@@ -3,11 +3,11 @@
 
 define double @test_FMADD_ASSOC1(double %A, double %B, double %C,
                                  double %D, double %E) {
-	%F = fmul double %A, %B         ; <double> [#uses=1]
-	%G = fmul double %C, %D         ; <double> [#uses=1]
-	%H = fadd double %F, %G         ; <double> [#uses=1]
-	%I = fadd double %H, %E         ; <double> [#uses=1]
-	ret double %I
+  %F = fmul double %A, %B         ; <double> [#uses=1]
+  %G = fmul double %C, %D         ; <double> [#uses=1]
+  %H = fadd double %F, %G         ; <double> [#uses=1]
+  %I = fadd double %H, %E         ; <double> [#uses=1]
+  ret double %I
 ; CHECK-LABEL: test_FMADD_ASSOC1:
 ; CHECK: fmadd
 ; CHECK-NEXT: fmadd
@@ -22,11 +22,11 @@ define double @test_FMADD_ASSOC1(double %A, double %B, double %C,
 
 define double @test_FMADD_ASSOC2(double %A, double %B, double %C,
                                  double %D, double %E) {
-	%F = fmul double %A, %B         ; <double> [#uses=1]
-	%G = fmul double %C, %D         ; <double> [#uses=1]
-	%H = fadd double %F, %G         ; <double> [#uses=1]
-	%I = fadd double %E, %H         ; <double> [#uses=1]
-	ret double %I
+  %F = fmul double %A, %B         ; <double> [#uses=1]
+  %G = fmul double %C, %D         ; <double> [#uses=1]
+  %H = fadd double %F, %G         ; <double> [#uses=1]
+  %I = fadd double %E, %H         ; <double> [#uses=1]
+  ret double %I
 ; CHECK-LABEL: test_FMADD_ASSOC2:
 ; CHECK: fmadd
 ; CHECK-NEXT: fmadd
@@ -41,11 +41,11 @@ define double @test_FMADD_ASSOC2(double %A, double %B, double %C,
 
 define double @test_FMSUB_ASSOC1(double %A, double %B, double %C,
                                  double %D, double %E) {
-	%F = fmul double %A, %B         ; <double> [#uses=1]
-	%G = fmul double %C, %D         ; <double> [#uses=1]
-	%H = fadd double %F, %G         ; <double> [#uses=1]
-	%I = fsub double %H, %E         ; <double> [#uses=1]
-	ret double %I
+  %F = fmul double %A, %B         ; <double> [#uses=1]
+  %G = fmul double %C, %D         ; <double> [#uses=1]
+  %H = fadd double %F, %G         ; <double> [#uses=1]
+  %I = fsub double %H, %E         ; <double> [#uses=1]
+  ret double %I
 ; CHECK-LABEL: test_FMSUB_ASSOC1:
 ; CHECK: fmsub
 ; CHECK-NEXT: fmadd
@@ -60,11 +60,11 @@ define double @test_FMSUB_ASSOC1(double %A, double %B, double %C,
 
 define double @test_FMSUB_ASSOC2(double %A, double %B, double %C,
                                  double %D, double %E) {
-	%F = fmul double %A, %B         ; <double> [#uses=1]
-	%G = fmul double %C, %D         ; <double> [#uses=1]
-	%H = fadd double %F, %G         ; <double> [#uses=1]
-	%I = fsub double %E, %H         ; <double> [#uses=1]
-	ret double %I
+  %F = fmul double %A, %B         ; <double> [#uses=1]
+  %G = fmul double %C, %D         ; <double> [#uses=1]
+  %H = fadd double %F, %G         ; <double> [#uses=1]
+  %I = fsub double %E, %H         ; <double> [#uses=1]
+  ret double %I
 ; CHECK-LABEL: test_FMSUB_ASSOC2:
 ; CHECK: fnmsub
 ; CHECK-NEXT: fnmsub
@@ -77,3 +77,159 @@ define double @test_FMSUB_ASSOC2(double %A, double %B, double %C,
 ; CHECK-VSX-NEXT: blr
 }
 
+define double @test_FMADD_ASSOC_EXT1(float %A, float %B, double %C,
+                                 double %D, double %E) {
+  %F = fmul float %A, %B         ; <float> [#uses=1]
+  %G = fpext float %F to double   ; <double> [#uses=1]
+  %H = fmul double %C, %D         ; <double> [#uses=1]
+  %I = fadd double %H, %G         ; <double> [#uses=1]
+  %J = fadd double %I, %E         ; <double> [#uses=1]
+  ret double %J
+; CHECK-LABEL: test_FMADD_ASSOC_EXT1:
+; CHECK: fmadd
+; CHECK-NEXT: fmadd
+; CHECK-NEXT: blr
+
+; CHECK-VSX-LABEL: test_FMADD_ASSOC_EXT1:
+; CHECK-VSX: xsmaddmdp
+; CHECK-VSX-NEXT: xsmaddadp
+; CHECK-VSX-NEXT: blr
+}
+
+define double @test_FMADD_ASSOC_EXT2(float %A, float %B, float %C,
+                                 float %D, double %E) {
+  %F = fmul float %A, %B         ; <float> [#uses=1]
+  %G = fmul float %C, %D         ; <float> [#uses=1]
+  %H = fadd float %F, %G         ; <float> [#uses=1]
+  %I = fpext float %H to double   ; <double> [#uses=1]
+  %J = fadd double %I, %E         ; <double> [#uses=1]
+  ret double %J
+; CHECK-LABEL: test_FMADD_ASSOC_EXT2:
+; CHECK: fmadd
+; CHECK-NEXT: fmadd
+; CHECK-NEXT: blr
+
+; CHECK-VSX-LABEL: test_FMADD_ASSOC_EXT2:
+; CHECK-VSX: xsmaddmdp
+; CHECK-VSX-NEXT: xsmaddadp
+; CHECK-VSX-NEXT: fmr
+; CHECK-VSX-NEXT: blr
+}
+
+define double @test_FMADD_ASSOC_EXT3(float %A, float %B, double %C,
+                                 double %D, double %E) {
+  %F = fmul float %A, %B          ; <float> [#uses=1]
+  %G = fpext float %F to double   ; <double> [#uses=1]
+  %H = fmul double %C, %D         ; <double> [#uses=1]
+  %I = fadd double %H, %G         ; <double> [#uses=1]
+  %J = fadd double %E, %I         ; <double> [#uses=1]
+  ret double %J
+; CHECK-LABEL: test_FMADD_ASSOC_EXT3:
+; CHECK: fmadd
+; CHECK-NEXT: fmadd
+; CHECK-NEXT: blr
+
+; CHECK-VSX-LABEL: test_FMADD_ASSOC_EXT3:
+; CHECK-VSX: xsmaddmdp
+; CHECK-VSX-NEXT: xsmaddadp
+; CHECK-VSX-NEXT: blr
+}
+
+define double @test_FMADD_ASSOC_EXT4(float %A, float %B, float %C,
+                                 float %D, double %E) {
+  %F = fmul float %A, %B          ; <float> [#uses=1]
+  %G = fmul float %C, %D          ; <float> [#uses=1]
+  %H = fadd float %F, %G          ; <float> [#uses=1]
+  %I = fpext float %H to double   ; <double> [#uses=1]
+  %J = fadd double %E, %I         ; <double> [#uses=1]
+  ret double %J
+; CHECK-LABEL: test_FMADD_ASSOC_EXT4:
+; CHECK: fmadd
+; CHECK-NEXT: fmadd
+; CHECK-NEXT: blr
+
+; CHECK-VSX-LABEL: test_FMADD_ASSOC_EXT4:
+; CHECK-VSX: xsmaddmdp
+; CHECK-VSX-NEXT: xsmaddadp
+; CHECK-VSX-NEXT: fmr
+; CHECK-VSX-NEXT: blr
+}
+
+define double @test_FMSUB_ASSOC_EXT1(float %A, float %B, double %C,
+                                 double %D, double %E) {
+  %F = fmul float %A, %B          ; <float> [#uses=1]
+  %G = fpext float %F to double   ; <double> [#uses=1]
+  %H = fmul double %C, %D         ; <double> [#uses=1]
+  %I = fadd double %H, %G         ; <double> [#uses=1]
+  %J = fsub double %I, %E         ; <double> [#uses=1]
+  ret double %J
+; CHECK-LABEL: test_FMSUB_ASSOC_EXT1:
+; CHECK: fmsub
+; CHECK-NEXT: fmadd
+; CHECK-NEXT: blr
+
+; CHECK-VSX-LABEL: test_FMSUB_ASSOC_EXT1:
+; CHECK-VSX: xsmsubmdp
+; CHECK-VSX-NEXT: xsmaddadp
+; CHECK-VSX-NEXT: blr
+}
+
+define double @test_FMSUB_ASSOC_EXT2(float %A, float %B, float %C,
+                                 float %D, double %E) {
+  %F = fmul float %A, %B          ; <float> [#uses=1]
+  %G = fmul float %C, %D          ; <float> [#uses=1]
+  %H = fadd float %F, %G          ; <float> [#uses=1]
+  %I = fpext float %H to double   ; <double> [#uses=1]
+  %J = fsub double %I, %E         ; <double> [#uses=1]
+  ret double %J
+; CHECK-LABEL: test_FMSUB_ASSOC_EXT2:
+; CHECK: fmsub
+; CHECK-NEXT: fmadd
+; CHECK-NEXT: blr
+
+; CHECK-VSX-LABEL: test_FMSUB_ASSOC_EXT2:
+; CHECK-VSX: xsmsubmdp
+; CHECK-VSX-NEXT: xsmaddadp
+; CHECK-VSX-NEXT: fmr
+; CHECK-VSX-NEXT: blr
+}
+
+define double @test_FMSUB_ASSOC_EXT3(float %A, float %B, double %C,
+                                 double %D, double %E) {
+  %F = fmul float %A, %B          ; <float> [#uses=1]
+  %G = fpext float %F to double   ; <double> [#uses=1]
+  %H = fmul double %C, %D         ; <double> [#uses=1]
+  %I = fadd double %H, %G         ; <double> [#uses=1]
+  %J = fsub double %E, %I         ; <double> [#uses=1]
+  ret double %J
+; CHECK-LABEL: test_FMSUB_ASSOC_EXT3:
+; CHECK: fnmsub
+; CHECK-NEXT: fnmsub
+; CHECK-NEXT: blr
+
+; CHECK-VSX-LABEL: test_FMSUB_ASSOC_EXT3:
+; CHECK-VSX: xsnmsubmdp
+; CHECK-VSX-NEXT: xsnmsubadp
+; CHECK-VSX-NEXT: fmr
+; CHECK-VSX-NEXT: blr
+}
+
+define double @test_FMSUB_ASSOC_EXT4(float %A, float %B, float %C,
+                                 float %D, double %E) {
+  %F = fmul float %A, %B          ; <float> [#uses=1]
+  %G = fmul float %C, %D          ; <float> [#uses=1]
+  %H = fadd float %F, %G          ; <float> [#uses=1]
+  %I = fpext float %H to double   ; <double> [#uses=1]
+  %J = fsub double %E, %I         ; <double> [#uses=1]
+  ret double %J
+; CHECK-LABEL: test_FMSUB_ASSOC_EXT4:
+; CHECK: fnmsub
+; CHECK-NEXT: fnmsub
+; CHECK-NEXT: blr
+
+; CHECK-VSX-LABEL: test_FMSUB_ASSOC_EXT4:
+; CHECK-VSX: xsnmsubmdp
+; CHECK-VSX-NEXT: xsnmsubadp
+; CHECK-VSX-NEXT: fmr
+; CHECK-VSX-NEXT: blr
+}
diff --git a/test/CodeGen/PowerPC/fma-ext.ll b/test/CodeGen/PowerPC/fma-ext.ll
index 56825ce8f22..da7c34ccb9d 100644
--- a/test/CodeGen/PowerPC/fma-ext.ll
+++ b/test/CodeGen/PowerPC/fma-ext.ll
@@ -60,34 +60,34 @@ define double @test_FMSUB_EXT2(float %A, float %B, double %C) {
 
 define double @test_FMSUB_EXT3(float %A, float %B, double %C) {
     %D = fmul float %A, %B          ; <float> [#uses=1]
-    %E = fsub float -0.000000e+00, %D ;		<float> [#uses=1]
+    %E = fsub float -0.000000e+00, %D ;    <float> [#uses=1]
     %F = fpext float %E to double   ; <double> [#uses=1]
     %G = fsub double %F, %C         ; <double> [#uses=1]
     ret double %G
 ; CHECK-LABEL: test_FMSUB_EXT3:
-; CHECK: fneg
-; CHECK-NEXT: fmsub
+; CHECK: fnmadd
+
 ; CHECK-NEXT: blr
                                 
 ; CHECK-VSX-LABEL: test_FMSUB_EXT3:
-; CHECK-VSX: xsnegdp
-; CHECK-VSX-NEXT: xsmsubmdp
+; CHECK-VSX: xsnmaddmdp
+
 ; CHECK-VSX-NEXT: blr
 }
     
 define double @test_FMSUB_EXT4(float %A, float %B, double %C) {
     %D = fmul float %A, %B          ; <float> [#uses=1]
     %E = fpext float %D to double   ; <double> [#uses=1]
-    %F = fsub double -0.000000e+00, %E ;		<double> [#uses=1]
+    %F = fsub double -0.000000e+00, %E ;    <double> [#uses=1]
     %G = fsub double %F, %C         ; <double> [#uses=1]
     ret double %G
 ; CHECK-LABEL: test_FMSUB_EXT4:
-; CHECK: fneg
-; CHECK-NEXT: fmsub
+; CHECK: fnmadd
+
 ; CHECK-NEXT: blr
                                 
 ; CHECK-VSX-LABEL: test_FMSUB_EXT4:
-; CHECK-VSX: xsnegdp
-; CHECK-VSX-NEXT: xsmsubmdp
+; CHECK-VSX: xsnmaddmdp
+
 ; CHECK-VSX-NEXT: blr
-}  
\ No newline at end of file
+}  
diff --git a/test/CodeGen/PowerPC/fma.ll b/test/CodeGen/PowerPC/fma.ll
index ab5251b2a55..9cfef398edf 100644
--- a/test/CodeGen/PowerPC/fma.ll
+++ b/test/CodeGen/PowerPC/fma.ll
@@ -1,9 +1,12 @@
 ; RUN: llc < %s -march=ppc32 -fp-contract=fast -mattr=-vsx | FileCheck %s
 ; RUN: llc < %s -mtriple=powerpc64-unknown-linux-gnu -fp-contract=fast -mattr=+vsx -mcpu=pwr7 | FileCheck -check-prefix=CHECK-VSX %s
+; RUN: llc < %s -mtriple=powerpc64-unknown-linux-gnu -fp-contract=fast -mcpu=pwr8 | FileCheck -check-prefix=CHECK-P8 %s
+; RUN: llc < %s -mtriple=powerpc64le-unknown-linux-gnu -fp-contract=fast -mcpu=pwr8 | FileCheck -check-prefix=CHECK-P8 %s
 
 declare double @dummy1(double) #0
 declare double @dummy2(double, double) #0
 declare double @dummy3(double, double, double) #0
+declare float @dummy4(float, float) #0
 
 define double @test_FMADD1(double %A, double %B, double %C) {
 	%D = fmul double %A, %B		; <double> [#uses=1]
@@ -126,3 +129,83 @@ define float @test_FNMSUBS(float %A, float %B, float %C) {
 ; CHECK-VSX: fnmsubs
 ; CHECK-VSX-NEXT: blr
 }
+
+define float @test_XSMADDMSP(float %A, float %B, float %C) {
+	%D = fmul float %A, %B		; <float> [#uses=1]
+	%E = fadd float %C, %D		; <float> [#uses=1]
+	ret float %E
+; CHECK-P8-LABEL: test_XSMADDMSP:
+; CHECK-P8: xsmaddmsp
+; CHECK-P8-NEXT: blr
+}
+
+define float @test_XSMSUBMSP(float %A, float %B, float %C) {
+	%D = fmul float %A, %B		; <float> [#uses=1]
+	%E = fsub float %D, %C		; <float> [#uses=1]
+	ret float %E
+; CHECK-P8-LABEL: test_XSMSUBMSP:
+; CHECK-P8: xsmsubmsp
+; CHECK-P8-NEXT: blr
+}
+
+define float @test_XSMADDASP(float %A, float %B, float %C, float %D) {
+	%E = fmul float %A, %B 	; <float> [#uses=2]
+	%F = fadd float %E, %C 	; <float> [#uses=1]
+	%G = fsub float %E, %D 	; <float> [#uses=1]
+	%H = call float @dummy4(float %F, float %G)      ; <float> [#uses=1]
+	ret float %H
+; CHECK-P8-LABEL: test_XSMADDASP:
+; CHECK-P8: xsmaddasp
+; CHECK-P8-NEXT: xsmsubmsp
+}
+
+define float @test_XSMSUBASP(float %A, float %B, float %C, float %D) {
+	%E = fmul float %A, %B 	; <float> [#uses=2]
+	%F = fsub float %E, %C 	; <float> [#uses=1]
+	%G = fsub float %E, %D 	; <float> [#uses=1]
+	%H = call float @dummy4(float %F, float %G)      ; <float> [#uses=1]
+	ret float %H
+; CHECK-P8-LABEL: test_XSMSUBASP:
+; CHECK-P8: xsmsubasp
+; CHECK-P8-NEXT: xsmsubmsp
+}
+
+define float @test_XSNMADDMSP(float %A, float %B, float %C) {
+	%D = fmul float %A, %B		; <float> [#uses=1]
+	%E = fadd float %D, %C		; <float> [#uses=1]
+	%F = fsub float -0.000000e+00, %E		; <float> [#uses=1]
+	ret float %F
+; CHECK-P8-LABEL: test_XSNMADDMSP:
+; CHECK-P8: xsnmaddmsp
+; CHECK-P8-NEXT: blr
+}
+
+define float @test_XSNMSUBMSP(float %A, float %B, float %C) {
+	%D = fmul float %A, %B		; <float> [#uses=1]
+	%E = fsub float %D, %C		; <float> [#uses=1]
+	%F = fsub float -0.000000e+00, %E		; <float> [#uses=1]
+	ret float %F
+; CHECK-P8-LABEL: test_XSNMSUBMSP:
+; CHECK-P8: xsnmsubmsp
+; CHECK-P8-NEXT: blr
+}
+
+define float @test_XSNMADDASP(float %A, float %B, float %C) {
+	%D = fmul float %A, %B		; <float> [#uses=1]
+	%E = fadd float %D, %C		; <float> [#uses=1]
+	%F = fsub float -0.000000e+00, %E		; <float> [#uses=1]
+	%H = call float @dummy4(float %E, float %F)      ; <float> [#uses=1]
+	ret float %F
+; CHECK-P8-LABEL: test_XSNMADDASP:
+; CHECK-P8: xsnmaddasp
+}
+
+define float @test_XSNMSUBASP(float %A, float %B, float %C) {
+	%D = fmul float %A, %B		; <float> [#uses=1]
+	%E = fsub float %D, %C		; <float> [#uses=1]
+	%F = fsub float -0.000000e+00, %E		; <float> [#uses=1]
+	%H = call float @dummy4(float %E, float %F)      ; <float> [#uses=1]
+	ret float %F
+; CHECK-P8-LABEL: test_XSNMSUBASP:
+; CHECK-P8: xsnmsubasp
+}
diff --git a/test/CodeGen/PowerPC/glob-comp-aa-crash.ll b/test/CodeGen/PowerPC/glob-comp-aa-crash.ll
index 66df6bb8669..88530a3f303 100644
--- a/test/CodeGen/PowerPC/glob-comp-aa-crash.ll
+++ b/test/CodeGen/PowerPC/glob-comp-aa-crash.ll
@@ -17,7 +17,7 @@ target triple = "powerpc64-bgq-linux"
 declare i32 @__gxx_personality_v0(...)
 
 ; Function Attrs: optsize
-define void @_ZNSt3__117__assoc_sub_state4copyEv(%"class.std::__1::__assoc_sub_state"* %this) #0 align 2 {
+define void @_ZNSt3__117__assoc_sub_state4copyEv(%"class.std::__1::__assoc_sub_state"* %this) #0 align 2 personality i8* bitcast (i32 (...)* @__gxx_personality_v0 to i8*) {
 entry:
   %__lk = alloca %"class.std::__1::unique_lock", align 8
   %ref.tmp = alloca %"class.std::__exception_ptr::exception_ptr", align 8
@@ -50,14 +50,14 @@ invoke.cont4:                                     ; preds = %if.then
   unreachable
 
 lpad:                                             ; preds = %entry
-  %1 = landingpad { i8*, i32 } personality i8* bitcast (i32 (...)* @__gxx_personality_v0 to i8*)
+  %1 = landingpad { i8*, i32 }
           cleanup
   %2 = extractvalue { i8*, i32 } %1, 0
   %3 = extractvalue { i8*, i32 } %1, 1
   br label %ehcleanup
 
 lpad3:                                            ; preds = %if.then
-  %4 = landingpad { i8*, i32 } personality i8* bitcast (i32 (...)* @__gxx_personality_v0 to i8*)
+  %4 = landingpad { i8*, i32 }
           cleanup
   %5 = extractvalue { i8*, i32 } %4, 0
   %6 = extractvalue { i8*, i32 } %4, 1
diff --git a/test/CodeGen/PowerPC/hello-reloc.s b/test/CodeGen/PowerPC/hello-reloc.s
index 97dfbb5362f..12f4315f675 100644
--- a/test/CodeGen/PowerPC/hello-reloc.s
+++ b/test/CodeGen/PowerPC/hello-reloc.s
@@ -2,7 +2,7 @@
 ; which is responsible for writing mach-o relocation entries for (PIC)
 ; PowerPC objects.
 
-; RUN: llvm-mc -filetype=obj -relocation-model=pic -mcpu=g4 -triple=powerpc-apple-darwin8 %s -o - | llvm-readobj -relocations | FileCheck -check-prefix=DARWIN-G4-DUMP %s
+; RUN: llvm-mc -filetype=obj -relocation-model=pic -mcpu=g4 -triple=powerpc-apple-darwin8 %s -o - | llvm-readobj -r --expand-relocs | FileCheck -check-prefix=DARWIN-G4-DUMP %s
 
 	.machine ppc7400
 	.section	__TEXT,__textcoal_nt,coalesced,pure_instructions
@@ -62,19 +62,79 @@ L_.str:                                 ; @.str
 ; DARWIN-G4-DUMP:AddressSize: 32bit
 ; DARWIN-G4-DUMP:Relocations [
 ; DARWIN-G4-DUMP:  Section __text {
-; DARWIN-G4-DUMP:    0x34 1 2 0 PPC_RELOC_BR24 0 0x3
-; DARWIN-G4-DUMP:    0x30 0 2 n/a PPC_RELOC_LO16_SECTDIFF 1 0x74
-; DARWIN-G4-DUMP:    0x0 0 2 n/a PPC_RELOC_PAIR 1 0x14
-; DARWIN-G4-DUMP:    0x2C 0 2 n/a PPC_RELOC_HA16_SECTDIFF 1 0x74
-; DARWIN-G4-DUMP:    0x60 0 2 n/a PPC_RELOC_PAIR 1 0x14
+; DARWIN-G4-DUMP:    Relocation {
+; DARWIN-G4-DUMP:      Offset: 0x34
+; DARWIN-G4-DUMP:      PCRel: 1
+; DARWIN-G4-DUMP:      Length: 2
+; DARWIN-G4-DUMP:      Type: PPC_RELOC_BR24 (3)
+; DARWIN-G4-DUMP:      Section: __picsymbolstub1
+; DARWIN-G4-DUMP:    }
+; DARWIN-G4-DUMP:    Relocation {
+; DARWIN-G4-DUMP:      Offset: 0x30
+; DARWIN-G4-DUMP:      PCRel: 0
+; DARWIN-G4-DUMP:      Length: 2
+; DARWIN-G4-DUMP:      Type: PPC_RELOC_LO16_SECTDIFF (11)
+; DARWIN-G4-DUMP:      Value: 0x74
+; DARWIN-G4-DUMP:    }
+; DARWIN-G4-DUMP:    Relocation {
+; DARWIN-G4-DUMP:      Offset: 0x0
+; DARWIN-G4-DUMP:      PCRel: 0
+; DARWIN-G4-DUMP:      Length: 2
+; DARWIN-G4-DUMP:      Type: PPC_RELOC_PAIR (1)
+; DARWIN-G4-DUMP:      Value: 0x14
+; DARWIN-G4-DUMP:    }
+; DARWIN-G4-DUMP:    Relocation {
+; DARWIN-G4-DUMP:      Offset: 0x2C
+; DARWIN-G4-DUMP:      PCRel: 0
+; DARWIN-G4-DUMP:      Length: 2
+; DARWIN-G4-DUMP:      Type: PPC_RELOC_HA16_SECTDIFF (12)
+; DARWIN-G4-DUMP:      Value: 0x74
+; DARWIN-G4-DUMP:    }
+; DARWIN-G4-DUMP:    Relocation {
+; DARWIN-G4-DUMP:      Offset: 0x60
+; DARWIN-G4-DUMP:      PCRel: 0
+; DARWIN-G4-DUMP:      Length: 2
+; DARWIN-G4-DUMP:      Type: PPC_RELOC_PAIR (1)
+; DARWIN-G4-DUMP:      Value: 0x14
+; DARWIN-G4-DUMP:    }
 ; DARWIN-G4-DUMP:  }
 ; DARWIN-G4-DUMP:  Section __picsymbolstub1 {
-; DARWIN-G4-DUMP:    0x14 0 2 n/a PPC_RELOC_LO16_SECTDIFF 1 0x70
-; DARWIN-G4-DUMP:    0x0 0 2 n/a PPC_RELOC_PAIR 1 0x58
-; DARWIN-G4-DUMP:    0xC 0 2 n/a PPC_RELOC_HA16_SECTDIFF 1 0x70
-; DARWIN-G4-DUMP:    0x18 0 2 n/a PPC_RELOC_PAIR 1 0x58
+; DARWIN-G4-DUMP:    Relocation {
+; DARWIN-G4-DUMP:      Offset: 0x14
+; DARWIN-G4-DUMP:      PCRel: 0
+; DARWIN-G4-DUMP:      Length: 2
+; DARWIN-G4-DUMP:      Type: PPC_RELOC_LO16_SECTDIFF (11)
+; DARWIN-G4-DUMP:      Value: 0x70
+; DARWIN-G4-DUMP:    }
+; DARWIN-G4-DUMP:    Relocation {
+; DARWIN-G4-DUMP:      Offset: 0x0
+; DARWIN-G4-DUMP:      PCRel: 0
+; DARWIN-G4-DUMP:      Length: 2
+; DARWIN-G4-DUMP:      Type: PPC_RELOC_PAIR (1)
+; DARWIN-G4-DUMP:      Value: 0x58
+; DARWIN-G4-DUMP:    }
+; DARWIN-G4-DUMP:    Relocation {
+; DARWIN-G4-DUMP:      Offset: 0xC
+; DARWIN-G4-DUMP:      PCRel: 0
+; DARWIN-G4-DUMP:      Length: 2
+; DARWIN-G4-DUMP:      Type: PPC_RELOC_HA16_SECTDIFF (12)
+; DARWIN-G4-DUMP:      Value: 0x70
+; DARWIN-G4-DUMP:    }
+; DARWIN-G4-DUMP:    Relocation {
+; DARWIN-G4-DUMP:      Offset: 0x18
+; DARWIN-G4-DUMP:      PCRel: 0
+; DARWIN-G4-DUMP:      Length: 2
+; DARWIN-G4-DUMP:      Type: PPC_RELOC_PAIR (1)
+; DARWIN-G4-DUMP:      Value: 0x58
+; DARWIN-G4-DUMP:    }
 ; DARWIN-G4-DUMP:  }
 ; DARWIN-G4-DUMP:  Section __la_symbol_ptr {
-; DARWIN-G4-DUMP:    0x0 0 2 1 PPC_RELOC_VANILLA 0 dyld_stub_binding_helper
+; DARWIN-G4-DUMP:    Relocation {
+; DARWIN-G4-DUMP:      Offset: 0x0
+; DARWIN-G4-DUMP:      PCRel: 0
+; DARWIN-G4-DUMP:      Length: 2
+; DARWIN-G4-DUMP:      Type: PPC_RELOC_VANILLA (0)
+; DARWIN-G4-DUMP:      Symbol: dyld_stub_binding_helper
+; DARWIN-G4-DUMP:    }
 ; DARWIN-G4-DUMP:  }
 ; DARWIN-G4-DUMP:]
diff --git a/test/CodeGen/PowerPC/i64_fp_round.ll b/test/CodeGen/PowerPC/i64_fp_round.ll
index 5770d788caf..2530b8a9282 100644
--- a/test/CodeGen/PowerPC/i64_fp_round.ll
+++ b/test/CodeGen/PowerPC/i64_fp_round.ll
@@ -14,7 +14,7 @@ entry:
 
 ; CHECK: sradi [[REG1:[0-9]+]], 3, 53
 ; CHECK: addi [[REG2:[0-9]+]], [[REG1]], 1
-; CHECK: cmpldi 0, [[REG2]], 1
+; CHECK: cmpldi [[REG2]], 1
 ; CHECK: isel [[REG3:[0-9]+]], {{[0-9]+}}, 3, 1
 ; CHECK: std [[REG3]], -{{[0-9]+}}(1)
 
diff --git a/test/CodeGen/PowerPC/long-compare.ll b/test/CodeGen/PowerPC/long-compare.ll
index 915595f6dba..e53356a5ddf 100644
--- a/test/CodeGen/PowerPC/long-compare.ll
+++ b/test/CodeGen/PowerPC/long-compare.ll
@@ -1,4 +1,4 @@
-; RUN: llc < %s -march=ppc32 | grep cntlzw 
+; RUN: llc < %s -march=ppc32 | grep cntlz
 ; RUN: llc < %s -march=ppc32 | not grep xori 
 ; RUN: llc < %s -march=ppc32 | not grep "li "
 ; RUN: llc < %s -march=ppc32 | not grep "mr "
diff --git a/test/CodeGen/PowerPC/lxvw4x-bug.ll b/test/CodeGen/PowerPC/lxvw4x-bug.ll
new file mode 100644
index 00000000000..1f521a5d533
--- /dev/null
+++ b/test/CodeGen/PowerPC/lxvw4x-bug.ll
@@ -0,0 +1,25 @@
+; RUN: llc -O0 -mcpu=pwr8 -mtriple=powerpc64le-unknown-unknown < %s | FileCheck %s
+; Function Attrs: nounwind
+define void @test() {
+entry:
+  %__a.addr.i = alloca i32, align 4
+  %__b.addr.i = alloca <4 x i32>*, align 8
+  %i = alloca <4 x i32>, align 16
+  %j = alloca <4 x i32>, align 16
+  store <4 x i32> <i32 1, i32 2, i32 3, i32 4>, <4 x i32>* %i, align 16
+  store i32 0, i32* %__a.addr.i, align 4
+  store <4 x i32>* %i, <4 x i32>** %__b.addr.i, align 8
+  %0 = load i32, i32* %__a.addr.i, align 4
+  %1 = load <4 x i32>*, <4 x i32>** %__b.addr.i, align 8
+  %2 = bitcast <4 x i32>* %1 to i8*
+  %3 = getelementptr i8, i8* %2, i32 %0
+  %4 = call <4 x i32> @llvm.ppc.vsx.lxvw4x(i8* %3)
+; CHECK: lwa [[REG0:[0-9]+]],
+; CHECK: lxvd2x [[REG1:[0-9]+]], {{[0-9]+}}, [[REG0]]
+; CHECK: xxswapd [[REG1]], [[REG1]]
+  store <4 x i32> %4, <4 x i32>* %j, align 16
+  ret void
+}
+
+; Function Attrs: nounwind readonly
+declare <4 x i32> @llvm.ppc.vsx.lxvw4x(i8*)
diff --git a/test/CodeGen/PowerPC/machine-combiner.ll b/test/CodeGen/PowerPC/machine-combiner.ll
new file mode 100644
index 00000000000..93fb2020d53
--- /dev/null
+++ b/test/CodeGen/PowerPC/machine-combiner.ll
@@ -0,0 +1,188 @@
+; RUN: llc -O3 -mcpu=pwr7 -enable-unsafe-fp-math < %s | FileCheck %s -check-prefix=CHECK -check-prefix=CHECK-PWR
+; RUN: llc -O3 -mcpu=a2q -enable-unsafe-fp-math < %s | FileCheck %s -check-prefix=CHECK -check-prefix=CHECK-QPX
+target datalayout = "E-m:e-i64:64-n32:64"
+target triple = "powerpc64-unknown-linux-gnu"
+
+; Verify that the first two adds are independent regardless of how the inputs are
+; commuted. The destination registers are used as source registers for the third add.
+
+define float @reassociate_adds1(float %x0, float %x1, float %x2, float %x3) {
+; CHECK-LABEL: reassociate_adds1:
+; CHECK:       # BB#0:
+; CHECK:       fadds [[REG0:[0-9]+]], 1, 2
+; CHECK:       fadds [[REG1:[0-9]+]], 3, 4
+; CHECK:       fadds 1, [[REG0]], [[REG1]]
+; CHECK-NEXT:  blr
+
+  %t0 = fadd float %x0, %x1
+  %t1 = fadd float %t0, %x2
+  %t2 = fadd float %t1, %x3
+  ret float %t2
+}
+
+define float @reassociate_adds2(float %x0, float %x1, float %x2, float %x3) {
+; CHECK-LABEL: reassociate_adds2:
+; CHECK:       # BB#0:
+; CHECK:       fadds [[REG0:[0-9]+]], 1, 2
+; CHECK:       fadds [[REG1:[0-9]+]], 3, 4
+; CHECK:       fadds 1, [[REG0]], [[REG1]]
+; CHECK-NEXT:  blr
+
+  %t0 = fadd float %x0, %x1
+  %t1 = fadd float %x2, %t0
+  %t2 = fadd float %t1, %x3
+  ret float %t2
+}
+
+define float @reassociate_adds3(float %x0, float %x1, float %x2, float %x3) {
+; CHECK-LABEL: reassociate_adds3:
+; CHECK:       # BB#0:
+; CHECK:       fadds [[REG0:[0-9]+]], 1, 2
+; CHECK:       fadds [[REG1:[0-9]+]], 3, 4
+; CHECK:       fadds 1, [[REG0]], [[REG1]]
+; CHECK-NEXT:  blr
+
+  %t0 = fadd float %x0, %x1
+  %t1 = fadd float %t0, %x2
+  %t2 = fadd float %x3, %t1
+  ret float %t2
+}
+
+define float @reassociate_adds4(float %x0, float %x1, float %x2, float %x3) {
+; CHECK-LABEL: reassociate_adds4:
+; CHECK:       # BB#0:
+; CHECK:       fadds [[REG0:[0-9]+]], 1, 2
+; CHECK:       fadds [[REG1:[0-9]+]], 3, 4
+; CHECK:       fadds 1, [[REG0]], [[REG1]]
+; CHECK-NEXT:  blr
+
+  %t0 = fadd float %x0, %x1
+  %t1 = fadd float %x2, %t0
+  %t2 = fadd float %x3, %t1
+  ret float %t2
+}
+
+; Verify that we reassociate some of these ops. The optimal balanced tree of adds is not
+; produced because that would cost more compile time.
+
+define float @reassociate_adds5(float %x0, float %x1, float %x2, float %x3, float %x4, float %x5, float %x6, float %x7) {
+; CHECK-LABEL: reassociate_adds5:
+; CHECK:       # BB#0:
+; CHECK:       fadds [[REG12:[0-9]+]], 5, 6
+; CHECK:       fadds [[REG0:[0-9]+]], 1, 2
+; CHECK:       fadds [[REG11:[0-9]+]], 3, 4
+; CHECK:       fadds [[REG13:[0-9]+]], [[REG12]], 7
+; CHECK:       fadds [[REG1:[0-9]+]], [[REG0]], [[REG11]]
+; CHECK:       fadds [[REG2:[0-9]+]], [[REG1]], [[REG13]]
+; CHECK:       fadds 1, [[REG2]], 8
+; CHECK-NEXT:    blr
+
+  %t0 = fadd float %x0, %x1
+  %t1 = fadd float %t0, %x2
+  %t2 = fadd float %t1, %x3
+  %t3 = fadd float %t2, %x4
+  %t4 = fadd float %t3, %x5
+  %t5 = fadd float %t4, %x6
+  %t6 = fadd float %t5, %x7
+  ret float %t6
+}
+
+; Verify that we reassociate vector instructions too.
+
+define <4 x float> @vector_reassociate_adds1(<4 x float> %x0, <4 x float> %x1, <4 x float> %x2, <4 x float> %x3) {
+; CHECK-LABEL: vector_reassociate_adds1:
+; CHECK:       # BB#0:
+; CHECK-QPX:       qvfadds [[REG0:[0-9]+]], 1, 2
+; CHECK-QPX:       qvfadds [[REG1:[0-9]+]], 3, 4
+; CHECK-QPX:       qvfadds 1, [[REG0]], [[REG1]]
+; CHECK-PWR:       xvaddsp [[REG0:[0-9]+]], 34, 35
+; CHECK-PWR:       xvaddsp [[REG1:[0-9]+]], 36, 37
+; CHECK-PWR:       xvaddsp 34, [[REG0]], [[REG1]]
+; CHECK-NEXT:  blr
+
+  %t0 = fadd <4 x float> %x0, %x1
+  %t1 = fadd <4 x float> %t0, %x2
+  %t2 = fadd <4 x float> %t1, %x3
+  ret <4 x float> %t2
+}
+
+define <4 x float> @vector_reassociate_adds2(<4 x float> %x0, <4 x float> %x1, <4 x float> %x2, <4 x float> %x3) {
+; CHECK-LABEL: vector_reassociate_adds2:
+; CHECK:       # BB#0:
+; CHECK-QPX:       qvfadds [[REG0:[0-9]+]], 1, 2
+; CHECK-QPX:       qvfadds [[REG1:[0-9]+]], 3, 4
+; CHECK-QPX:       qvfadds 1, [[REG0]], [[REG1]]
+; CHECK-PWR:       xvaddsp [[REG0:[0-9]+]], 34, 35
+; CHECK-PWR:       xvaddsp [[REG1:[0-9]+]], 36, 37
+; CHECK-PWR:       xvaddsp 34, [[REG0]], [[REG1]]
+; CHECK-NEXT:  blr
+
+  %t0 = fadd <4 x float> %x0, %x1
+  %t1 = fadd <4 x float> %x2, %t0
+  %t2 = fadd <4 x float> %t1, %x3
+  ret <4 x float> %t2
+}
+
+define <4 x float> @vector_reassociate_adds3(<4 x float> %x0, <4 x float> %x1, <4 x float> %x2, <4 x float> %x3) {
+; CHECK-LABEL: vector_reassociate_adds3:
+; CHECK:       # BB#0:
+; CHECK-QPX:       qvfadds [[REG0:[0-9]+]], 1, 2
+; CHECK-QPX:       qvfadds [[REG1:[0-9]+]], 3, 4
+; CHECK-QPX:       qvfadds 1, [[REG0]], [[REG1]]
+; CHECK-PWR:       xvaddsp [[REG0:[0-9]+]], 34, 35
+; CHECK-PWR:       xvaddsp [[REG1:[0-9]+]], 36, 37
+; CHECK-PWR:       xvaddsp 34, [[REG0]], [[REG1]]
+; CHECK-NEXT:  blr
+
+  %t0 = fadd <4 x float> %x0, %x1
+  %t1 = fadd <4 x float> %t0, %x2
+  %t2 = fadd <4 x float> %x3, %t1
+  ret <4 x float> %t2
+}
+
+define <4 x float> @vector_reassociate_adds4(<4 x float> %x0, <4 x float> %x1, <4 x float> %x2, <4 x float> %x3) {
+; CHECK-LABEL: vector_reassociate_adds4:
+; CHECK:       # BB#0:
+; CHECK-QPX:       qvfadds [[REG0:[0-9]+]], 1, 2
+; CHECK-QPX:       qvfadds [[REG1:[0-9]+]], 3, 4
+; CHECK-QPX:       qvfadds 1, [[REG0]], [[REG1]]
+; CHECK-PWR:       xvaddsp [[REG0:[0-9]+]], 34, 35
+; CHECK-PWR:       xvaddsp [[REG1:[0-9]+]], 36, 37
+; CHECK-PWR:       xvaddsp 34, [[REG0]], [[REG1]]
+; CHECK-NEXT:  blr
+
+  %t0 = fadd <4 x float> %x0, %x1
+  %t1 = fadd <4 x float> %x2, %t0
+  %t2 = fadd <4 x float> %x3, %t1
+  ret <4 x float> %t2
+}
+
+define float @reassociate_adds6(float %x0, float %x1, float %x2, float %x3) {
+  %t0 = fdiv float %x0, %x1
+  %t1 = fadd float %x2, %t0
+  %t2 = fadd float %x3, %t1
+  ret float %t2
+}
+
+define float @reassociate_muls1(float %x0, float %x1, float %x2, float %x3) {
+  %t0 = fdiv float %x0, %x1
+  %t1 = fmul float %x2, %t0
+  %t2 = fmul float %x3, %t1
+  ret float %t2
+}
+
+define double @reassociate_adds_double(double %x0, double %x1, double %x2, double %x3) {
+  %t0 = fdiv double %x0, %x1
+  %t1 = fadd double %x2, %t0
+  %t2 = fadd double %x3, %t1
+  ret double %t2
+}
+
+define double @reassociate_muls_double(double %x0, double %x1, double %x2, double %x3) {
+  %t0 = fdiv double %x0, %x1
+  %t1 = fmul double %x2, %t0
+  %t2 = fmul double %x3, %t1
+  ret double %t2
+}
+
+
diff --git a/test/CodeGen/PowerPC/mcm-5.ll b/test/CodeGen/PowerPC/mcm-5.ll
index 0c258459c91..19adbe5b7d9 100644
--- a/test/CodeGen/PowerPC/mcm-5.ll
+++ b/test/CodeGen/PowerPC/mcm-5.ll
@@ -1,5 +1,5 @@
-; RUN: llc -mcpu=pwr7 -O0 -code-model=medium <%s | FileCheck %s
-; RUN: llc -mcpu=pwr7 -O0 -code-model=large <%s | FileCheck %s
+; RUN: llc -mcpu=pwr7 -code-model=medium <%s | FileCheck %s
+; RUN: llc -mcpu=pwr7 -code-model=large <%s | FileCheck %s
 
 ; Test correct code generation for medium and large code model
 ; for loading the address of a jump table from the TOC.
diff --git a/test/CodeGen/PowerPC/mcm-obj.ll b/test/CodeGen/PowerPC/mcm-obj.ll
index 46295cf3187..1ececf84926 100644
--- a/test/CodeGen/PowerPC/mcm-obj.ll
+++ b/test/CodeGen/PowerPC/mcm-obj.ll
@@ -3,6 +3,12 @@
 ; RUN: llc -O0 -mcpu=pwr7 -code-model=large -filetype=obj -fast-isel=false %s -o - | \
 ; RUN: llvm-readobj -r | FileCheck -check-prefix=LARGE %s
 
+; Run jump table test separately since jump tables aren't generated at -O0.
+; RUN: llc -mcpu=pwr7 -code-model=medium -filetype=obj -fast-isel=false %s -o - | \
+; RUN: llvm-readobj -r | FileCheck -check-prefix=MEDIUM-JT %s
+; RUN: llc -mcpu=pwr7 -code-model=large -filetype=obj -fast-isel=false %s -o - | \
+; RUN: llvm-readobj -r | FileCheck -check-prefix=LARGE-JT %s
+
 ; FIXME: When asm-parse is available, could make this an assembly test.
 
 target datalayout = "E-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-f128:128:128-v128:128:128-n32:64"
@@ -92,6 +98,46 @@ entry:
 ; LARGE-NEXT:      0x{{[0-9,A-F]+}} R_PPC64_TOC16_HA [[SYM4:[^ ]+]]
 ; LARGE-NEXT:      0x{{[0-9,A-F]+}} R_PPC64_TOC16_LO_DS [[SYM4]]
 
+@ti = common global i32 0, align 4
+
+define signext i32 @test_tentative() nounwind {
+entry:
+  %0 = load i32, i32* @ti, align 4
+  %inc = add nsw i32 %0, 1
+  store i32 %inc, i32* @ti, align 4
+  ret i32 %0
+}
+
+; Verify generation of R_PPC64_TOC16_HA and R_PPC64_TOC16_LO_DS for
+; accessing tentatively declared variable ti.
+;
+; MEDIUM-NEXT:     0x{{[0-9,A-F]+}} R_PPC64_TOC16_HA [[SYM6:[^ ]+]]
+; MEDIUM-NEXT:     0x{{[0-9,A-F]+}} R_PPC64_TOC16_LO_DS [[SYM6]]
+;
+; LARGE-NEXT:      0x{{[0-9,A-F]+}} R_PPC64_TOC16_HA [[SYM6:[^ ]+]]
+; LARGE-NEXT:      0x{{[0-9,A-F]+}} R_PPC64_TOC16_LO_DS [[SYM6]]
+
+define i8* @test_fnaddr() nounwind {
+entry:
+  %func = alloca i32 (i32)*, align 8
+  store i32 (i32)* @foo, i32 (i32)** %func, align 8
+  %0 = load i32 (i32)*, i32 (i32)** %func, align 8
+  %1 = bitcast i32 (i32)* %0 to i8*
+  ret i8* %1
+}
+
+declare signext i32 @foo(i32 signext)
+
+; Verify generation of R_PPC64_TOC16_HA and R_PPC64_TOC16_LO_DS for
+; accessing function address foo.
+;
+; MEDIUM-NEXT:     0x{{[0-9,A-F]+}} R_PPC64_TOC16_HA [[SYM7:[^ ]+]]
+; MEDIUM-NEXT:     0x{{[0-9,A-F]+}} R_PPC64_TOC16_LO_DS [[SYM7]]
+;
+; LARGE-NEXT:      0x{{[0-9,A-F]+}} R_PPC64_TOC16_HA [[SYM7:[^ ]+]]
+; LARGE-NEXT:      0x{{[0-9,A-F]+}} R_PPC64_TOC16_LO_DS [[SYM7]]
+
+
 define signext i32 @test_jump_table(i32 signext %i) nounwind {
 entry:
   %i.addr = alloca i32, align 4
@@ -139,47 +185,12 @@ sw.epilog:                                        ; preds = %sw.bb3, %sw.default
 ; Verify generation of R_PPC64_TOC16_HA and R_PPC64_TOC16_LO_DS for
 ; accessing a jump table address.
 ;
-; MEDIUM-NEXT:     0x{{[0-9,A-F]+}} R_PPC64_TOC16_HA [[SYM5:[^ ]+]]
-; MEDIUM-NEXT:     0x{{[0-9,A-F]+}} R_PPC64_TOC16_LO_DS [[SYM5]]
-;
-; LARGE-NEXT:      0x{{[0-9,A-F]+}} R_PPC64_TOC16_HA [[SYM5:[^ ]+]]
-; LARGE-NEXT:      0x{{[0-9,A-F]+}} R_PPC64_TOC16_LO_DS [[SYM5]]
-
-@ti = common global i32 0, align 4
-
-define signext i32 @test_tentative() nounwind {
-entry:
-  %0 = load i32, i32* @ti, align 4
-  %inc = add nsw i32 %0, 1
-  store i32 %inc, i32* @ti, align 4
-  ret i32 %0
-}
-
-; Verify generation of R_PPC64_TOC16_HA and R_PPC64_TOC16_LO_DS for
-; accessing tentatively declared variable ti.
-;
-; MEDIUM-NEXT:     0x{{[0-9,A-F]+}} R_PPC64_TOC16_HA [[SYM6:[^ ]+]]
-; MEDIUM-NEXT:     0x{{[0-9,A-F]+}} R_PPC64_TOC16_LO_DS [[SYM6]]
-;
-; LARGE-NEXT:      0x{{[0-9,A-F]+}} R_PPC64_TOC16_HA [[SYM6:[^ ]+]]
-; LARGE-NEXT:      0x{{[0-9,A-F]+}} R_PPC64_TOC16_LO_DS [[SYM6]]
-
-define i8* @test_fnaddr() nounwind {
-entry:
-  %func = alloca i32 (i32)*, align 8
-  store i32 (i32)* @foo, i32 (i32)** %func, align 8
-  %0 = load i32 (i32)*, i32 (i32)** %func, align 8
-  %1 = bitcast i32 (i32)* %0 to i8*
-  ret i8* %1
-}
-
-declare signext i32 @foo(i32 signext)
-
-; Verify generation of R_PPC64_TOC16_HA and R_PPC64_TOC16_LO_DS for
-; accessing function address foo.
-;
-; MEDIUM-NEXT:     0x{{[0-9,A-F]+}} R_PPC64_TOC16_HA [[SYM7:[^ ]+]]
-; MEDIUM-NEXT:     0x{{[0-9,A-F]+}} R_PPC64_TOC16_LO_DS [[SYM7]]
+; MEDIUM-JT:      Relocations [
+; MEDIUM-JT:        Section ({{.*}}) .rela.text {
+; MEDIUM-JT-NEXT:     0x{{[0-9,A-F]+}} R_PPC64_TOC16_HA [[SYM:[^ ]+]]
+; MEDIUM-JT-NEXT:     0x{{[0-9,A-F]+}} R_PPC64_TOC16_LO_DS [[SYM]]
 ;
-; LARGE-NEXT:      0x{{[0-9,A-F]+}} R_PPC64_TOC16_HA [[SYM7:[^ ]+]]
-; LARGE-NEXT:      0x{{[0-9,A-F]+}} R_PPC64_TOC16_LO_DS [[SYM7]]
+; LARGE-JT:       Relocations [
+; LARGE-JT:         Section ({{.*}}) .rela.text {
+; LARGE-JT-NEXT:      0x{{[0-9,A-F]+}} R_PPC64_TOC16_HA [[SYM:[^ ]+]]
+; LARGE-JT-NEXT:      0x{{[0-9,A-F]+}} R_PPC64_TOC16_LO_DS [[SYM]]
diff --git a/test/CodeGen/PowerPC/mftb.ll b/test/CodeGen/PowerPC/mftb.ll
new file mode 100644
index 00000000000..9ad93267b9d
--- /dev/null
+++ b/test/CodeGen/PowerPC/mftb.ll
@@ -0,0 +1,72 @@
+; Check handling of the mftb instruction.
+; For CPUs 601 and pwr3, the mftb instruction should be emitted.
+; On all other CPUs (including generic, ppc, ppc64), the mfspr instruction 
+; should be used instead. There should no longer be a deprecated warning 
+; message emittedfor this instruction for any CPU.
+
+; RUN: llc -mtriple=powerpc64-unknown-linux-gnu -mcpu=pwr7 < %s 2>&1 \
+; RUN:    | FileCheck %s --check-prefix=CHECK-MFSPR
+; RUN: llc -mtriple=powerpc64-unknown-linux-gnu -mcpu=pwr8 < %s 2>&1 \
+; RUN:    | FileCheck %s --check-prefix=CHECK-MFSPR
+; RUN: llc -mtriple=powerpc64le-unknown-linux-gnu < %s 2>&1 \
+; RUN:    | FileCheck %s --check-prefix=CHECK-MFSPR
+; RUN: llc -mtriple=powerpc-unknown-linux-gnu  < %s 2>&1 \
+; RUN:    | FileCheck %s --check-prefix=CHECK-MFSPR
+; RUN: llc -mtriple=powerpc-unknown-linux-gnu -mcpu=ppc < %s 2>&1 \
+; RUN:    | FileCheck %s --check-prefix=CHECK-MFSPR
+; RUN: llc -mtriple=powerpc-unknown-linux-gnu -mcpu=601 < %s 2>&1 \
+; RUN:    | FileCheck %s --check-prefix=CHECK-MFTB
+; RUN: llc -mtriple=powerpc-unknown-linux-gnu -mcpu=pwr3 < %s 2>&1 \
+; RUN:    | FileCheck %s --check-prefix=CHECK-MFTB
+
+; CHECK-MFSPR-NOT: warning: deprecated
+; CHECK-MFTB-NOT: warning: deprecated
+
+define i32 @get_time() {
+       %time = call i32 asm "mftb $0, 268", "=r"()
+       ret i32 %time
+; CHECK-MFSPR-LABEL: @get_time
+; CHECK-MFSPR: mfspr 3, 268
+; CHECK-MFSPR: blr
+
+; CHECK-MFTB-LABEL: @get_time
+; CHECK-MFTB: mftb 3, 268
+; CHECK-MFTB: blr
+}
+
+define i32 @get_timeu() {
+       %time = call i32 asm "mftb $0, 269", "=r"()
+       ret i32 %time
+; CHECK-MFSPR-LABEL: @get_timeu
+; CHECK-MFSPR: mfspr 3, 269
+; CHECK-MFSPR: blr
+
+; CHECK-MFTB-LABEL: @get_timeu
+; CHECK-MFTB: mftbu 3
+; CHECK-MFTB: blr
+}
+
+define i32 @get_time_e() {
+       %time = call i32 asm "mftb $0", "=r"()
+       ret i32 %time
+; CHECK-MFSPR-LABEL: @get_time_e
+; CHECK-MFSPR: mfspr 3, 268
+; CHECK-MFSPR: blr
+
+; CHECK-MFTB-LABEL: @get_time_e
+; CHECK-MFTB: mftb 3, 268
+; CHECK-MFTB: blr
+}
+
+define i32 @get_timeu_e() {
+       %time = call i32 asm "mftbu $0", "=r"()
+       ret i32 %time
+; CHECK-MFSPR-LABEL: @get_timeu_e
+; CHECK-MFSPR: mfspr 3, 269
+; CHECK-MFSPR: blr
+
+; CHECK-MFTB-LABEL: @get_timeu_e
+; CHECK-MFTB: mftbu 3
+; CHECK-MFTB: blr
+}
+
diff --git a/test/CodeGen/PowerPC/ppc-crbits-onoff.ll b/test/CodeGen/PowerPC/ppc-crbits-onoff.ll
index 88648df5fa3..c69f30017d8 100644
--- a/test/CodeGen/PowerPC/ppc-crbits-onoff.ll
+++ b/test/CodeGen/PowerPC/ppc-crbits-onoff.ll
@@ -15,8 +15,8 @@ entry:
 ; CHECK-DAG: cmplwi {{[0-9]+}}, 3, 0
 ; CHECK-DAG: li [[REG2:[0-9]+]], 1
 ; CHECK-DAG: cntlzw [[REG3:[0-9]+]],
-; CHECK: isel 3, 0, [[REG2]]
-; CHECK: and 3, 3, [[REG3]]
+; CHECK: isel [[REG4:[0-9]+]], 0, [[REG2]]
+; CHECK: and 3, [[REG4]], [[REG3]]
 ; CHECK: blr
 }
 
diff --git a/test/CodeGen/PowerPC/ppc32-cyclecounter.ll b/test/CodeGen/PowerPC/ppc32-cyclecounter.ll
index 9e2cd0b1288..ea50a1bdbdc 100644
--- a/test/CodeGen/PowerPC/ppc32-cyclecounter.ll
+++ b/test/CodeGen/PowerPC/ppc32-cyclecounter.ll
@@ -13,8 +13,8 @@ entry:
 ; CHECK: mfspr 3, 269
 ; CHECK: mfspr 4, 268
 ; CHECK: mfspr [[REG:[0-9]+]], 269
-; CHECK: cmpw [[CR:[0-9]+]], 3, [[REG]]
-; CHECK: bne [[CR]], .LBB
+; CHECK: cmpw 3, [[REG]]
+; CHECK: bne 0, .LBB
 
 declare i64 @llvm.readcyclecounter()
 
diff --git a/test/CodeGen/PowerPC/ppc32-nest.ll b/test/CodeGen/PowerPC/ppc32-nest.ll
new file mode 100644
index 00000000000..ed7bbe2b8f3
--- /dev/null
+++ b/test/CodeGen/PowerPC/ppc32-nest.ll
@@ -0,0 +1,26 @@
+; RUN: llc < %s | FileCheck %s
+target datalayout = "E-m:e-p:32:32-i64:64-n32"
+target triple = "powerpc-unknown-linux-gnu"
+
+; Tests that the 'nest' parameter attribute causes the relevant parameter to be
+; passed in the right register (r11 for PPC).
+
+define i8* @nest_receiver(i8* nest %arg) nounwind {
+; CHECK-LABEL: nest_receiver:
+; CHECK: # BB#0:
+; CHECK-NEXT: mr 3, 11
+; CHECK-NEXT: blr
+
+  ret i8* %arg
+}
+
+define i8* @nest_caller(i8* %arg) nounwind {
+; CHECK-LABEL: nest_caller:
+; CHECK: mr 11, 3
+; CHECK-NEXT: bl nest_receiver
+; CHECK: blr
+
+  %result = call i8* @nest_receiver(i8* nest %arg)
+  ret i8* %result
+}
+
diff --git a/test/CodeGen/PowerPC/ppc64-anyregcc.ll b/test/CodeGen/PowerPC/ppc64-anyregcc.ll
index 66f6a2c790c..ff0768ff47e 100644
--- a/test/CodeGen/PowerPC/ppc64-anyregcc.ll
+++ b/test/CodeGen/PowerPC/ppc64-anyregcc.ll
@@ -82,7 +82,7 @@ target triple = "powerpc64-unknown-linux-gnu"
 ; CHECK-NEXT:   .long 3
 define i64 @test() nounwind ssp uwtable {
 entry:
-  call anyregcc void (i64, i32, i8*, i32, ...) @llvm.experimental.patchpoint.void(i64 0, i32 24, i8* null, i32 2, i32 1, i32 2, i64 3)
+  call anyregcc void (i64, i32, i8*, i32, ...) @llvm.experimental.patchpoint.void(i64 0, i32 40, i8* null, i32 2, i32 1, i32 2, i64 3)
   ret i64 0
 }
 
@@ -104,7 +104,7 @@ entry:
 define i64 @property_access1(i8* %obj) nounwind ssp uwtable {
 entry:
   %f = inttoptr i64 281474417671919 to i8*
-  %ret = call anyregcc i64 (i64, i32, i8*, i32, ...) @llvm.experimental.patchpoint.i64(i64 1, i32 24, i8* %f, i32 1, i8* %obj)
+  %ret = call anyregcc i64 (i64, i32, i8*, i32, ...) @llvm.experimental.patchpoint.i64(i64 1, i32 40, i8* %f, i32 1, i8* %obj)
   ret i64 %ret
 }
 
@@ -127,7 +127,7 @@ define i64 @property_access2() nounwind ssp uwtable {
 entry:
   %obj = alloca i64, align 8
   %f = inttoptr i64 281474417671919 to i8*
-  %ret = call anyregcc i64 (i64, i32, i8*, i32, ...) @llvm.experimental.patchpoint.i64(i64 2, i32 24, i8* %f, i32 1, i64* %obj)
+  %ret = call anyregcc i64 (i64, i32, i8*, i32, ...) @llvm.experimental.patchpoint.i64(i64 2, i32 40, i8* %f, i32 1, i64* %obj)
   ret i64 %ret
 }
 
@@ -150,7 +150,7 @@ define i64 @property_access3() nounwind ssp uwtable {
 entry:
   %obj = alloca i64, align 8
   %f = inttoptr i64 281474417671919 to i8*
-  %ret = call anyregcc i64 (i64, i32, i8*, i32, ...) @llvm.experimental.patchpoint.i64(i64 3, i32 24, i8* %f, i32 0, i64* %obj)
+  %ret = call anyregcc i64 (i64, i32, i8*, i32, ...) @llvm.experimental.patchpoint.i64(i64 3, i32 40, i8* %f, i32 0, i64* %obj)
   ret i64 %ret
 }
 
@@ -232,7 +232,7 @@ entry:
 define i64 @anyreg_test1(i8* %a1, i8* %a2, i8* %a3, i8* %a4, i8* %a5, i8* %a6, i8* %a7, i8* %a8, i8* %a9, i8* %a10, i8* %a11, i8* %a12, i8* %a13) nounwind ssp uwtable {
 entry:
   %f = inttoptr i64 281474417671919 to i8*
-  %ret = call anyregcc i64 (i64, i32, i8*, i32, ...) @llvm.experimental.patchpoint.i64(i64 4, i32 24, i8* %f, i32 13, i8* %a1, i8* %a2, i8* %a3, i8* %a4, i8* %a5, i8* %a6, i8* %a7, i8* %a8, i8* %a9, i8* %a10, i8* %a11, i8* %a12, i8* %a13)
+  %ret = call anyregcc i64 (i64, i32, i8*, i32, ...) @llvm.experimental.patchpoint.i64(i64 4, i32 40, i8* %f, i32 13, i8* %a1, i8* %a2, i8* %a3, i8* %a4, i8* %a5, i8* %a6, i8* %a7, i8* %a8, i8* %a9, i8* %a10, i8* %a11, i8* %a12, i8* %a13)
   ret i64 %ret
 }
 
@@ -314,7 +314,7 @@ entry:
 define i64 @anyreg_test2(i8* %a1, i8* %a2, i8* %a3, i8* %a4, i8* %a5, i8* %a6, i8* %a7, i8* %a8, i8* %a9, i8* %a10, i8* %a11, i8* %a12, i8* %a13) nounwind ssp uwtable {
 entry:
   %f = inttoptr i64 281474417671919 to i8*
-  %ret = call anyregcc i64 (i64, i32, i8*, i32, ...) @llvm.experimental.patchpoint.i64(i64 5, i32 24, i8* %f, i32 8, i8* %a1, i8* %a2, i8* %a3, i8* %a4, i8* %a5, i8* %a6, i8* %a7, i8* %a8, i8* %a9, i8* %a10, i8* %a11, i8* %a12, i8* %a13)
+  %ret = call anyregcc i64 (i64, i32, i8*, i32, ...) @llvm.experimental.patchpoint.i64(i64 5, i32 40, i8* %f, i32 8, i8* %a1, i8* %a2, i8* %a3, i8* %a4, i8* %a5, i8* %a6, i8* %a7, i8* %a8, i8* %a9, i8* %a10, i8* %a11, i8* %a12, i8* %a13)
   ret i64 %ret
 }
 
@@ -342,7 +342,7 @@ entry:
 ; CHECK-NEXT: .long  0
 define i64 @patchpoint_spilldef(i64 %p1, i64 %p2, i64 %p3, i64 %p4) {
 entry:
-  %result = tail call anyregcc i64 (i64, i32, i8*, i32, ...) @llvm.experimental.patchpoint.i64(i64 12, i32 24, i8* inttoptr (i64 0 to i8*), i32 2, i64 %p1, i64 %p2)
+  %result = tail call anyregcc i64 (i64, i32, i8*, i32, ...) @llvm.experimental.patchpoint.i64(i64 12, i32 40, i8* inttoptr (i64 0 to i8*), i32 2, i64 %p1, i64 %p2)
   tail call void asm sideeffect "nop", "~{r0},~{r3},~{r4},~{r5},~{r6},~{r7},~{r8},~{r9},~{r10},~{r11},~{r12},~{r14},~{r15},~{r16},~{r17
 },~{r18},~{r19},~{r20},~{r21},~{r22},~{r23},~{r24},~{r25},~{r26},~{r27},~{r28},~{r29},~{r30},~{r31}"() nounwind
   ret i64 %result
@@ -384,7 +384,7 @@ define i64 @patchpoint_spillargs(i64 %p1, i64 %p2, i64 %p3, i64 %p4) {
 entry:
   tail call void asm sideeffect "nop", "~{r0},~{r3},~{r4},~{r5},~{r6},~{r7},~{r8},~{r9},~{r10},~{r11},~{r12},~{r14},~{r15},~{r16},~{r17
 },~{r18},~{r19},~{r20},~{r21},~{r22},~{r23},~{r24},~{r25},~{r26},~{r27},~{r28},~{r29},~{r30},~{r31}"() nounwind
-  %result = tail call anyregcc i64 (i64, i32, i8*, i32, ...) @llvm.experimental.patchpoint.i64(i64 13, i32 24, i8* inttoptr (i64 0 to i8*), i32 2, i64 %p1, i64 %p2, i64 %p3, i64 %p4)
+  %result = tail call anyregcc i64 (i64, i32, i8*, i32, ...) @llvm.experimental.patchpoint.i64(i64 13, i32 40, i8* inttoptr (i64 0 to i8*), i32 2, i64 %p1, i64 %p2, i64 %p3, i64 %p4)
   ret i64 %result
 }
 
diff --git a/test/CodeGen/PowerPC/ppc64-fastcc-fast-isel.ll b/test/CodeGen/PowerPC/ppc64-fastcc-fast-isel.ll
index f90519836c2..92d6d556738 100644
--- a/test/CodeGen/PowerPC/ppc64-fastcc-fast-isel.ll
+++ b/test/CodeGen/PowerPC/ppc64-fastcc-fast-isel.ll
@@ -35,7 +35,7 @@ define fastcc double @f2(i64 %g1, double %f1, i64 %g2, double %f2, i64 %g3, doub
 }
 
 define void @cg2(i64 %v) #0 {
-  tail call fastcc i64 @g1(i64 0, double 0.0, i64 %v, double 0.0, i64 0, double 0.0, i64 0, double 0.0)
+  call fastcc i64 @g1(i64 0, double 0.0, i64 %v, double 0.0, i64 0, double 0.0, i64 0, double 0.0)
   ret void
 
 ; CHECK-LABEL: @cg2
@@ -44,11 +44,11 @@ define void @cg2(i64 %v) #0 {
 }
 
 define void @cf2(double %v) #0 {
-  tail call fastcc i64 @g1(i64 0, double 0.0, i64 0, double %v, i64 0, double 0.0, i64 0, double 0.0)
+  call fastcc i64 @g1(i64 0, double 0.0, i64 0, double %v, i64 0, double 0.0, i64 0, double 0.0)
   ret void
 
 ; CHECK-LABEL: @cf2
-; CHECK: mr 2, 1
+; CHECK: fmr 2, 1
 ; CHECK: blr
 }
 
diff --git a/test/CodeGen/PowerPC/ppc64-fastcc.ll b/test/CodeGen/PowerPC/ppc64-fastcc.ll
index bb1365a3b67..69e15d104da 100644
--- a/test/CodeGen/PowerPC/ppc64-fastcc.ll
+++ b/test/CodeGen/PowerPC/ppc64-fastcc.ll
@@ -521,8 +521,9 @@ define void @cv13(<4 x i32> %v) #0 {
   ret void
 
 ; CHECK-LABEL: @cv13
-; CHECK: li [[REG1:[0-9]+]], 96
-; CHECK: stvx 2, 1, [[REG1]]
+; CHECK-DAG: li [[REG1:[0-9]+]], 96
+; CHECK-DAG: vor [[REG2:[0-9]+]], 2, 2
+; CHECK: stvx [[REG2]], 1, [[REG1]]
 ; CHECK: blr
 }
 
@@ -531,8 +532,9 @@ define void @cv14(<4 x i32> %v) #0 {
   ret void
 
 ; CHECK-LABEL: @cv14
-; CHECK: li [[REG1:[0-9]+]], 128
-; CHECK: stvx 2, 1, [[REG1]]
+; CHECK-DAG: li [[REG1:[0-9]+]], 128
+; CHECK-DAG: vor [[REG2:[0-9]+]], 2, 2
+; CHECK: stvx [[REG2]], 1, [[REG1]]
 ; CHECK: blr
 }
 
diff --git a/test/CodeGen/PowerPC/ppc64-i128-abi.ll b/test/CodeGen/PowerPC/ppc64-i128-abi.ll
new file mode 100644
index 00000000000..993aec24c8f
--- /dev/null
+++ b/test/CodeGen/PowerPC/ppc64-i128-abi.ll
@@ -0,0 +1,274 @@
+; RUN: llc -mtriple=powerpc64le-unknown-linux-gnu -mcpu=pwr8 < %s | FileCheck %s -check-prefix=CHECK-LE
+; RUN: llc -mtriple=powerpc64-unknown-linux-gnu -mcpu=pwr8 < %s | FileCheck %s -check-prefix=CHECK-BE
+; RUN: llc -mtriple=powerpc64-unknown-linux-gnu -mcpu=pwr8 -mattr=-vsx < %s | FileCheck %s -check-prefix=CHECK-NOVSX
+; RUN: llc -mtriple=powerpc64le-unknown-linux-gnu -mcpu=pwr8 -mattr=-vsx < %s | FileCheck %s -check-prefix=CHECK-NOVSX
+; RUN: llc -mtriple=powerpc64-unknown-linux-gnu -mcpu=pwr8 -mattr=-vsx < %s | FileCheck %s -check-prefix=CHECK-BE-NOVSX
+; RUN: llc -mtriple=powerpc64le-unknown-linux-gnu -mcpu=pwr8 -mattr=-vsx < %s | FileCheck %s -check-prefix=CHECK-LE-NOVSX
+
+@x = common global <1 x i128> zeroinitializer, align 16
+@y = common global <1 x i128> zeroinitializer, align 16
+@a = common global i128 zeroinitializer, align 16
+@b = common global i128 zeroinitializer, align 16
+
+; VSX:
+;   %a is passed in register 34
+;   The value of 1 is stored in the TOC.
+;   On LE, ensure the value of 1 is swapped before being used (using xxswapd).
+; VMX (no VSX): 
+;   %a is passed in register 2
+;   The value of 1 is stored in the TOC.
+;   No swaps are necessary when using P8 Vector instructions on LE
+define <1 x i128> @v1i128_increment_by_one(<1 x i128> %a) nounwind {
+       %tmp = add <1 x i128> %a, <i128 1>
+       ret <1 x i128> %tmp  
+
+; FIXME: Seems a 128-bit literal is materialized by loading from the TOC. There
+;        should be a better way of doing this.
+
+; CHECK-LE-LABEL: @v1i128_increment_by_one
+; CHECK-LE: lxvd2x [[VAL:[0-9]+]], {{[0-9]+}}, {{[0-9]+}}
+; CHECK-LE: xxswapd 35, [[VAL]]
+; CHECK-LE: vadduqm 2, 2, 3
+; CHECK-LE: blr
+
+; CHECK-BE-LABEL: @v1i128_increment_by_one
+; CHECK-BE: lxvd2x 35, {{[0-9]+}}, {{[0-9]+}}
+; CHECK-BE-NOT: xxswapd 
+; CHECK-BE: vadduqm 2, 2, 3 
+; CHECK-BE-NOT: xxswapd 34, {{[0-9]+}}
+; CHECK-BE: blr
+
+; CHECK-NOVSX-LABEL: @v1i128_increment_by_one
+; CHECK-NOVSX-NOT: xxswapd {{[0-9]+}}, {{[0-9]+}}
+; CHECK-NOVSX-NOT: stxvd2x {{[0-9]+}}, {{[0-9]+}}, {{[0-9]+}}
+; CHECK-NOVSX: lvx [[VAL:[0-9]+]], {{[0-9]+}}, {{[0-9]+}}
+; CHECK-NOVSX-NOT: lxvd2x {{[0-9]+}}, {{[0-9]+}}, {{[0-9]+}}
+; CHECK-NOVSX-NOT: xxswapd {{[0-9]+}}, {{[0-9]+}}
+; CHECK-NOVSX: vadduqm 2, 2, [[VAL]]
+; CHECK-NOVSX: blr
+}
+
+; VSX:
+;   %a is passed in register 34
+;   %b is passed in register 35
+;   No swaps are necessary when using P8 Vector instructions on LE
+; VMX (no VSX):
+;   %a is passewd in register 2
+;   %b is passed in register 3
+;   On LE, do not need to swap contents of 2 and 3 because the lvx/stvx 
+;   instructions no not swap elements
+define <1 x i128> @v1i128_increment_by_val(<1 x i128> %a, <1 x i128> %b) nounwind {
+       %tmp = add <1 x i128> %a, %b
+       ret <1 x i128> %tmp
+
+; CHECK-LE-LABEL: @v1i128_increment_by_val
+; CHECK-LE-NOT: xxswapd
+; CHECK-LE: adduqm 2, 2, 3
+; CHECK-LE: blr
+
+; CHECK-BE-LABEL: @v1i128_increment_by_val
+; CHECK-BE-NOT: xxswapd {{[0-9]+}}, 34
+; CHECK-BE-NOT: xxswapd {{[0-9]+}}, 35
+; CHECK-BE-NOT: xxswapd 34, [[RESULT]]
+; CHECK-BE: adduqm 2, 2, 3
+; CHECK-BE: blr
+
+; CHECK-NOVSX-LABEL: @v1i128_increment_by_val
+; CHECK-NOVSX-NOT: xxswapd 34, [[RESULT]]
+; CHECK-NOVSX: adduqm 2, 2, 3
+; CHECK-NOVSX: blr
+}
+
+; Little Endian (VSX and VMX):
+;   Lower 64-bits of %a are passed in register 3
+;   Upper 64-bits of %a are passed in register 4
+;   Increment lower 64-bits using addic (immediate value of 1)
+;   Increment upper 64-bits using add zero extended
+;   Results are placed in registers 3 and 4
+; Big Endian (VSX and VMX)
+;   Lower 64-bits of %a are passed in register 4
+;   Upper 64-bits of %a are passed in register 3
+;   Increment lower 64-bits using addic (immediate value of 1)
+;   Increment upper 64-bits using add zero extended
+;   Results are placed in registers 3 and 4
+define i128 @i128_increment_by_one(i128 %a) nounwind {
+       %tmp =  add i128 %a,  1
+       ret i128 %tmp
+; CHECK-LE-LABEL: @i128_increment_by_one
+; CHECK-LE: addic 3, 3, 1
+; CHECK-LE-NEXT: addze 4, 4
+; CHECK-LE: blr
+
+; CHECK-BE-LABEL: @i128_increment_by_one
+; CHECK-BE: addic 4, 4, 1
+; CHECK-BE-NEXT: addze 3, 3
+; CHECK-BE: blr
+
+; CHECK-LE-NOVSX-LABEL: @i128_increment_by_one
+; CHECK-LE-NOVSX: addic 3, 3, 1
+; CHECK-LE-NOVSX-NEXT: addze 4, 4
+; CHECK-LE-NOVSX: blr
+
+; CHECK-BE-NOVSX-LABEL: @i128_increment_by_one
+; CHECK-BE-NOVSX: addic 4, 4, 1
+; CHECK-BE-NOVSX-NEXT: addze 3, 3
+; CHECK-BE-NOVSX: blr
+}
+
+; Little Endian (VSX and VMX):
+;   Lower 64-bits of %a are passed in register 3
+;   Upper 64-bits of %a are passed in register 4
+;   Lower 64-bits of %b are passed in register 5
+;   Upper 64-bits of %b are passed in register 6
+;   Add the lower 64-bits using addc on registers 3 and 5
+;   Add the upper 64-bits using adde on registers 4 and 6
+;   Registers 3 and 4 should hold the result
+; Big Endian (VSX and VMX):
+;   Upper 64-bits of %a are passed in register 3
+;   Lower 64-bits of %a are passed in register 4
+;   Upper 64-bits of %b are passed in register 5
+;   Lower 64-bits of %b are passed in register 6
+;   Add the lower 64-bits using addc on registers 4 and 6
+;   Add the upper 64-bits using adde on registers 3 and 5
+;   Registers 3 and 4 should hold the result
+define i128 @i128_increment_by_val(i128 %a, i128 %b) nounwind {
+       %tmp =  add i128 %a, %b
+       ret i128 %tmp
+; CHECK-LE-LABEL: @i128_increment_by_val
+; CHECK-LE: addc 3, 3, 5
+; CHECK-LE-NEXT: adde 4, 4, 6
+; CHECK-LE: blr
+
+; CHECK-BE-LABEL: @i128_increment_by_val
+; CHECK-BE: addc 4, 4, 6
+; CHECK-BE-NEXT: adde 3, 3, 5
+; CHECK-BE: blr
+
+; CHECK-LE-NOVSX-LABEL: @i128_increment_by_val
+; CHECK-LE-NOVSX: addc 3, 3, 5
+; CHECK-LE-NOVSX-NEXT: adde 4, 4, 6
+; CHECK-LE-NOVSX: blr
+
+; CHECK-BE-NOVSX-LABEL: @i128_increment_by_val
+; CHECK-BE-NOVSX: addc 4, 4, 6
+; CHECK-BE-NOVSX-NEXT: adde 3, 3, 5
+; CHECK-BE-NOVSX: blr
+}
+
+
+; Callsites for the routines defined above. 
+; Ensure the parameters are loaded in the same order that is expected by the 
+; callee. See comments for individual functions above for details on registers
+; used for parameters.
+define <1 x i128> @call_v1i128_increment_by_one() nounwind {
+       %tmp = load <1 x i128>, <1 x i128>* @x, align 16
+       %ret = call <1 x i128> @v1i128_increment_by_one(<1 x i128> %tmp)
+       ret <1 x i128> %ret
+
+; CHECK-LE-LABEL: @call_v1i128_increment_by_one
+; CHECK-LE: lxvd2x [[PARAM:[0-9]+]], {{[0-9]+}}, {{[0-9]+}}
+; CHECK-LE: xxswapd 34, [[PARAM]]
+; CHECK-LE: bl v1i128_increment_by_one
+; CHECK-LE: blr
+
+; CHECK-BE-LABEL: @call_v1i128_increment_by_one
+; CHECK-BE: lxvw4x 34, {{[0-9]+}}, {{[0-9]+}}
+; CHECK-BE-NOT: xxswapd 34, {{[0-9]+}}
+; CHECK-BE: bl v1i128_increment_by_one
+; CHECK-BE: blr
+
+; CHECK-NOVSX-LABEL: @call_v1i128_increment_by_one
+; CHECK-NOVSX: lvx 2, {{[0-9]+}}, {{[0-9]+}}
+; CHECK-NOVSX-NOT: xxswapd {{[0-9]+}}, {{[0-9]+}}
+; CHECK-NOVSX: bl v1i128_increment_by_one
+; CHECK-NOVSX: blr
+}
+
+define <1 x i128> @call_v1i128_increment_by_val() nounwind {
+       %tmp = load <1 x i128>, <1 x i128>* @x, align 16
+       %tmp2 = load <1 x i128>, <1 x i128>* @y, align 16
+       %ret = call <1 x i128> @v1i128_increment_by_val(<1 x i128> %tmp, <1 x i128> %tmp2)
+       ret <1 x i128> %ret
+
+; CHECK-LE-LABEL: @call_v1i128_increment_by_val
+; CHECK-LE: lxvd2x [[PARAM1:[0-9]+]], {{[0-9]+}}, {{[0-9]+}}
+; CHECK-LE: lxvd2x [[PARAM2:[0-9]+]], {{[0-9]+}}, {{[0-9]+}}
+; CHECK-LE-DAG: xxswapd 34, [[PARAM1]]
+; CHECK-LE-DAG: xxswapd 35, [[PARAM2]]
+; CHECK-LE: bl v1i128_increment_by_val
+; CHECK-LE: blr
+
+; CHECK-BE-LABEL: @call_v1i128_increment_by_val
+
+
+; CHECK-BE-DAG: lxvw4x 35, {{[0-9]+}}, {{[0-9]+}}
+; CHECK-BE-NOT: xxswapd 34, {{[0-9]+}}
+; CHECK-BE-NOT: xxswapd 35, {{[0-9]+}}
+; CHECK-BE: bl v1i128_increment_by_val
+; CHECK-BE: blr
+
+; CHECK-NOVSX-LABEL: @call_v1i128_increment_by_val
+; CHECK-NOVSX-DAG: lvx 2, {{[0-9]+}}, {{[0-9]+}}
+; CHECK-NOVSX-DAG: lvx 3, {{[0-9]+}}, {{[0-9]+}}
+; CHECK-NOVSX-NOT: xxswapd 34, {{[0-9]+}}
+; CHECK-NOVSX-NOT: xxswapd 35, {{[0-9]+}}
+; CHECK-NOVSX: bl v1i128_increment_by_val
+; CHECK-NOVSX: blr
+
+}
+
+define i128 @call_i128_increment_by_one() nounwind {
+       %tmp = load i128, i128* @a, align 16
+       %ret = call i128 @i128_increment_by_one(i128 %tmp)
+       ret i128 %ret
+;       %ret4 = call i128 @i128_increment_by_val(i128 %tmp2, i128 %tmp2)
+; CHECK-LE-LABEL: @call_i128_increment_by_one
+; CHECK-LE-DAG: ld 3, 0([[BASEREG:[0-9]+]])
+; CHECK-LE-DAG: ld 4, 8([[BASEREG]])
+; CHECK-LE: bl i128_increment_by_one
+; CHECK-LE: blr
+
+; CHECK-BE-LABEL: @call_i128_increment_by_one
+; CHECK-BE-DAG: ld 3, 0([[BASEREG:[0-9]+]])
+; CHECK-BE-DAG: ld 4, 8([[BASEREG]])
+; CHECK-BE: bl i128_increment_by_one
+; CHECK-BE: blr
+
+; CHECK-NOVSX-LABEL: @call_i128_increment_by_one
+; CHECK-NOVSX-DAG: ld 3, 0([[BASEREG:[0-9]+]])
+; CHECK-NOVSX-DAG: ld 4, 8([[BASEREG]])
+; CHECK-NOVSX: bl i128_increment_by_one
+; CHECK-NOVSX: blr
+}
+
+define i128 @call_i128_increment_by_val() nounwind {
+       %tmp = load i128, i128* @a, align 16
+       %tmp2 = load i128, i128* @b, align 16
+       %ret = call i128 @i128_increment_by_val(i128 %tmp, i128 %tmp2)
+       ret i128 %ret
+; CHECK-LE-LABEL: @call_i128_increment_by_val
+; CHECK-LE-DAG: ld 3, 0([[P1BASEREG:[0-9]+]])
+; CHECK-LE-DAG: ld 4, 8([[P1BASEREG]])
+; CHECK-LE-DAG: ld 5, 0([[P2BASEREG:[0-9]+]])
+; CHECK-LE-DAG: ld 6, 8([[P2BASEREG]])
+; CHECK-LE: bl i128_increment_by_val
+; CHECK-LE: blr
+
+; CHECK-BE-LABEL: @call_i128_increment_by_val
+; CHECK-BE-DAG: ld 3, 0([[P1BASEREG:[0-9]+]])
+; CHECK-BE-DAG: ld 4, 8([[P1BASEREG]])
+; CHECK-BE-DAG: ld 5, 0([[P2BASEREG:[0-9]+]])
+; CHECK-BE-DAG: ld 6, 8([[P2BASEREG]])
+; CHECK-BE: bl i128_increment_by_val
+; CHECK-BE: blr
+
+; CHECK-NOVSX-LABEL: @call_i128_increment_by_val
+; CHECK-NOVSX-DAG: ld 3, 0([[P1BASEREG:[0-9]+]])
+; CHECK-NOVSX-DAG: ld 4, 8([[P1BASEREG]])
+; CHECK-NOVSX-DAG: ld 5, 0([[P2BASEREG:[0-9]+]])
+; CHECK-NOVSX-DAG: ld 6, 8([[P2BASEREG]])
+; CHECK-NOVSX: bl i128_increment_by_val
+; CHECK-NOVSX: blr
+}
+
+
diff --git a/test/CodeGen/PowerPC/ppc64-nest.ll b/test/CodeGen/PowerPC/ppc64-nest.ll
new file mode 100644
index 00000000000..9dd88db2fb5
--- /dev/null
+++ b/test/CodeGen/PowerPC/ppc64-nest.ll
@@ -0,0 +1,42 @@
+; RUN: llc < %s | FileCheck %s
+target datalayout = "E-m:e-i64:64-n32:64"
+target triple = "powerpc64-unknown-linux-gnu"
+
+; Tests that the 'nest' parameter attribute causes the relevant parameter to be
+; passed in the right register (r11 for PPC).
+
+define i8* @nest_receiver(i8* nest %arg) nounwind {
+; CHECK-LABEL: nest_receiver:
+; CHECK: # BB#0:
+; CHECK-NEXT: mr 3, 11
+; CHECK-NEXT: blr
+
+  ret i8* %arg
+}
+
+define i8* @nest_caller(i8* %arg) nounwind {
+; CHECK-LABEL: nest_caller:
+; CHECK: mr 11, 3
+; CHECK-NEXT: bl nest_receiver
+; CHECK: blr
+
+  %result = call i8* @nest_receiver(i8* nest %arg)
+  ret i8* %result
+}
+
+define void @test_indirect(i32 ()* nocapture %f, i8* %p) {
+entry:
+
+; CHECK-LABEL: test_indirect
+; CHECK-DAG: ld [[DEST:[0-9]+]], 0(3)
+; CHECK-DAG: ld 2, 8(3)
+; CHECK-DAG: mr 11, 4
+; CHECK: mtctr [[DEST]]
+; CHECK: bctrl
+; CHECK: blr
+
+  %callee.knr.cast = bitcast i32 ()* %f to i32 (i8*)*
+  %call = tail call signext i32 %callee.knr.cast(i8* nest %p)
+  ret void
+}
+
diff --git a/test/CodeGen/PowerPC/ppc64-patchpoint.ll b/test/CodeGen/PowerPC/ppc64-patchpoint.ll
index 67b26268a3a..d10ea98cd1a 100644
--- a/test/CodeGen/PowerPC/ppc64-patchpoint.ll
+++ b/test/CodeGen/PowerPC/ppc64-patchpoint.ll
@@ -15,22 +15,34 @@ entry:
 ; CHECK-NEXT: rldic 12, 12, 32, 16
 ; CHECK-NEXT: oris 12, 12, 48879
 ; CHECK-NEXT: ori 12, 12, 51966
+; CHECK-LE-NEXT: std 2, 24(1)
+; CHECK-BE-NEXT: std 2, 40(1)
+; CHECK-BE-NEXT: ld 2, 8(12)
+; CHECK-BE-NEXT: ld 12, 0(12)
 ; CHECK-NEXT: mtctr 12
 ; CHECK-NEXT: bctrl
+; CHECK-LE-NEXT: ld 2, 24(1)
+; CHECK-BE-NEXT: ld 2, 40(1)
 
 ; CHECK: li 12, -8531
 ; CHECK-NEXT: rldic 12, 12, 32, 16
 ; CHECK-NEXT: oris 12, 12, 48879
 ; CHECK-NEXT: ori 12, 12, 51967
+; CHECK-LE-NEXT: std 2, 24(1)
+; CHECK-BE-NEXT: std 2, 40(1)
+; CHECK-BE-NEXT: ld 2, 8(12)
+; CHECK-BE-NEXT: ld 12, 0(12)
 ; CHECK-NEXT: mtctr 12
 ; CHECK-NEXT: bctrl
+; CHECK-LE-NEXT: ld 2, 24(1)
+; CHECK-BE-NEXT: ld 2, 40(1)
 
 ; CHECK: blr
 
   %resolveCall2 = inttoptr i64 244837814094590 to i8*
-  %result = tail call i64 (i64, i32, i8*, i32, ...) @llvm.experimental.patchpoint.i64(i64 2, i32 24, i8* %resolveCall2, i32 4, i64 %p1, i64 %p2, i64 %p3, i64 %p4)
+  %result = tail call i64 (i64, i32, i8*, i32, ...) @llvm.experimental.patchpoint.i64(i64 2, i32 40, i8* %resolveCall2, i32 4, i64 %p1, i64 %p2, i64 %p3, i64 %p4)
   %resolveCall3 = inttoptr i64 244837814094591 to i8*
-  tail call void (i64, i32, i8*, i32, ...) @llvm.experimental.patchpoint.void(i64 3, i32 24, i8* %resolveCall3, i32 2, i64 %p1, i64 %result)
+  tail call void (i64, i32, i8*, i32, ...) @llvm.experimental.patchpoint.void(i64 3, i32 40, i8* %resolveCall3, i32 2, i64 %p1, i64 %result)
   ret i64 %result
 }
 
@@ -65,13 +77,13 @@ entry:
   %tmp81 = inttoptr i64 %tmp80 to i64*
   %tmp82 = load i64, i64* %tmp81, align 8
   tail call void (i64, i32, ...) @llvm.experimental.stackmap(i64 14, i32 8, i64 %arg, i64 %tmp2, i64 %tmp10, i64 %tmp82)
-  tail call void (i64, i32, i8*, i32, ...) @llvm.experimental.patchpoint.void(i64 15, i32 32, i8* null, i32 3, i64 %arg, i64 %tmp10, i64 %tmp82)
+  tail call void (i64, i32, i8*, i32, ...) @llvm.experimental.patchpoint.void(i64 15, i32 48, i8* null, i32 3, i64 %arg, i64 %tmp10, i64 %tmp82)
   %tmp83 = load i64, i64* %tmp33, align 8
   %tmp84 = add i64 %tmp83, -24
   %tmp85 = inttoptr i64 %tmp84 to i64*
   %tmp86 = load i64, i64* %tmp85, align 8
   tail call void (i64, i32, ...) @llvm.experimental.stackmap(i64 17, i32 8, i64 %arg, i64 %tmp10, i64 %tmp86)
-  tail call void (i64, i32, i8*, i32, ...) @llvm.experimental.patchpoint.void(i64 18, i32 32, i8* null, i32 3, i64 %arg, i64 %tmp10, i64 %tmp86)
+  tail call void (i64, i32, i8*, i32, ...) @llvm.experimental.patchpoint.void(i64 18, i32 48, i8* null, i32 3, i64 %arg, i64 %tmp10, i64 %tmp86)
   ret i64 10
 }
 
@@ -91,6 +103,21 @@ entry:
   ret void
 }
 
+; Trivial symbolic patchpoint codegen.
+
+declare i64 @foo(i64 %p1, i64 %p2)
+define i64 @trivial_symbolic_patchpoint_codegen(i64 %p1, i64 %p2) {
+entry:
+; CHECK-LABEL: trivial_symbolic_patchpoint_codegen:
+; CHECK:       bl foo
+; CHECK-NEXT:  nop
+; CHECK-NEXT:  nop
+; CHECK-NOT:   nop
+; CHECK:       blr
+  %result = tail call i64 (i64, i32, i8*, i32, ...) @llvm.experimental.patchpoint.i64(i64 9, i32 12, i8* bitcast (i64 (i64, i64)* @foo to i8*), i32 2, i64 %p1, i64 %p2)
+  ret i64 %result
+}
+
 declare void @llvm.experimental.stackmap(i64, i32, ...)
 declare void @llvm.experimental.patchpoint.void(i64, i32, i8*, i32, ...)
 declare i64 @llvm.experimental.patchpoint.i64(i64, i32, i8*, i32, ...)
diff --git a/test/CodeGen/PowerPC/ppc64-stackmap.ll b/test/CodeGen/PowerPC/ppc64-stackmap.ll
index 917fa742251..a77339f8e47 100644
--- a/test/CodeGen/PowerPC/ppc64-stackmap.ll
+++ b/test/CodeGen/PowerPC/ppc64-stackmap.ll
@@ -112,7 +112,7 @@ target triple = "powerpc64-unknown-linux-gnu"
 define void @constantargs() {
 entry:
   %0 = inttoptr i64 244837814094590 to i8*
-  tail call void (i64, i32, i8*, i32, ...) @llvm.experimental.patchpoint.void(i64 1, i32 24, i8* %0, i32 0, i64 65535, i64 65536, i64 4294967295, i64 4294967296)
+  tail call void (i64, i32, i8*, i32, ...) @llvm.experimental.patchpoint.void(i64 1, i32 40, i8* %0, i32 0, i64 65535, i64 65536, i64 4294967295, i64 4294967296)
   ret void
 }
 
@@ -160,7 +160,7 @@ entry:
 cold:
   ; OSR patchpoint with 12-byte nop-slide and 2 live vars.
   %thunk = inttoptr i64 244837814094590 to i8*
-  call void (i64, i32, i8*, i32, ...) @llvm.experimental.patchpoint.void(i64 4, i32 24, i8* %thunk, i32 0, i64 %a, i64 %b)
+  call void (i64, i32, i8*, i32, ...) @llvm.experimental.patchpoint.void(i64 4, i32 40, i8* %thunk, i32 0, i64 %a, i64 %b)
   unreachable
 ret:
   ret void
@@ -176,7 +176,7 @@ ret:
 define i64 @propertyRead(i64* %obj) {
 entry:
   %resolveRead = inttoptr i64 244837814094590 to i8*
-  %result = call i64 (i64, i32, i8*, i32, ...) @llvm.experimental.patchpoint.i64(i64 5, i32 24, i8* %resolveRead, i32 1, i64* %obj)
+  %result = call i64 (i64, i32, i8*, i32, ...) @llvm.experimental.patchpoint.i64(i64 5, i32 40, i8* %resolveRead, i32 1, i64* %obj)
   %add = add i64 %result, 3
   ret i64 %add
 }
@@ -196,7 +196,7 @@ entry:
 define void @propertyWrite(i64 %dummy1, i64* %obj, i64 %dummy2, i64 %a) {
 entry:
   %resolveWrite = inttoptr i64 244837814094590 to i8*
-  call anyregcc void (i64, i32, i8*, i32, ...) @llvm.experimental.patchpoint.void(i64 6, i32 24, i8* %resolveWrite, i32 2, i64* %obj, i64 %a)
+  call anyregcc void (i64, i32, i8*, i32, ...) @llvm.experimental.patchpoint.void(i64 6, i32 40, i8* %resolveWrite, i32 2, i64* %obj, i64 %a)
   ret void
 }
 
@@ -218,7 +218,7 @@ entry:
 define void @jsVoidCall(i64 %dummy1, i64* %obj, i64 %arg, i64 %l1, i64 %l2) {
 entry:
   %resolveCall = inttoptr i64 244837814094590 to i8*
-  call void (i64, i32, i8*, i32, ...) @llvm.experimental.patchpoint.void(i64 7, i32 24, i8* %resolveCall, i32 2, i64* %obj, i64 %arg, i64 %l1, i64 %l2)
+  call void (i64, i32, i8*, i32, ...) @llvm.experimental.patchpoint.void(i64 7, i32 40, i8* %resolveCall, i32 2, i64* %obj, i64 %arg, i64 %l1, i64 %l2)
   ret void
 }
 
@@ -240,7 +240,7 @@ entry:
 define i64 @jsIntCall(i64 %dummy1, i64* %obj, i64 %arg, i64 %l1, i64 %l2) {
 entry:
   %resolveCall = inttoptr i64 244837814094590 to i8*
-  %result = call i64 (i64, i32, i8*, i32, ...) @llvm.experimental.patchpoint.i64(i64 8, i32 24, i8* %resolveCall, i32 2, i64* %obj, i64 %arg, i64 %l1, i64 %l2)
+  %result = call i64 (i64, i32, i8*, i32, ...) @llvm.experimental.patchpoint.i64(i64 8, i32 40, i8* %resolveCall, i32 2, i64* %obj, i64 %arg, i64 %l1, i64 %l2)
   %add = add i64 %result, 3
   ret i64 %add
 }
@@ -260,7 +260,7 @@ entry:
 ; CHECK-NEXT:   .short 31
 define void @spilledValue(i64 %arg0, i64 %arg1, i64 %arg2, i64 %arg3, i64 %arg4, i64 %l0, i64 %l1, i64 %l2, i64 %l3, i64 %l4, i64 %l5, i64 %l6, i64 %l7, i64 %l8, i64 %l9, i64 %l10, i64 %l11, i64 %l12, i64 %l13, i64 %l14, i64 %l15, i64 %l16, i64 %l17, i64 %l18, i64 %l19, i64 %l20, i64 %l21, i64 %l22, i64 %l23, i64 %l24, i64 %l25, i64 %l26, i64 %l27) {
 entry:
-  call void (i64, i32, i8*, i32, ...) @llvm.experimental.patchpoint.void(i64 11, i32 24, i8* null, i32 5, i64 %arg0, i64 %arg1, i64 %arg2, i64 %arg3, i64 %arg4, i64 %l0, i64 %l1, i64 %l2, i64 %l3, i64 %l4, i64 %l5, i64 %l6, i64 %l7, i64 %l8, i64 %l9, i64 %l10, i64 %l11, i64 %l12, i64 %l13, i64 %l14, i64 %l15, i64 %l16, i64 %l17, i64 %l18, i64 %l19, i64 %l20, i64 %l21, i64 %l22, i64 %l23, i64 %l24, i64 %l25, i64 %l26, i64 %l27)
+  call void (i64, i32, i8*, i32, ...) @llvm.experimental.patchpoint.void(i64 11, i32 40, i8* null, i32 5, i64 %arg0, i64 %arg1, i64 %arg2, i64 %arg3, i64 %arg4, i64 %l0, i64 %l1, i64 %l2, i64 %l3, i64 %l4, i64 %l5, i64 %l6, i64 %l7, i64 %l8, i64 %l9, i64 %l10, i64 %l11, i64 %l12, i64 %l13, i64 %l14, i64 %l15, i64 %l16, i64 %l17, i64 %l18, i64 %l19, i64 %l20, i64 %l21, i64 %l22, i64 %l23, i64 %l24, i64 %l25, i64 %l26, i64 %l27)
   ret void
 }
 
diff --git a/test/CodeGen/PowerPC/ppc64-zext.ll b/test/CodeGen/PowerPC/ppc64-zext.ll
index eb55445cc6c..bbd4856babd 100644
--- a/test/CodeGen/PowerPC/ppc64-zext.ll
+++ b/test/CodeGen/PowerPC/ppc64-zext.ll
@@ -4,7 +4,7 @@ target triple = "powerpc64-unknown-linux"
 
 define i64 @fun(i32 %arg32) nounwind {
 entry:
-; CHECK: rldicl {{[0-9]+}}, {{[0-9]+}}, 0, 32
+; CHECK: clrldi {{[0-9]+}}, {{[0-9]+}}, 32
   %o = zext i32 %arg32 to i64
   ret i64 %o
 }
diff --git a/test/CodeGen/PowerPC/ppc64le-smallarg.ll b/test/CodeGen/PowerPC/ppc64le-smallarg.ll
index 77d066363cd..070a617ffe4 100644
--- a/test/CodeGen/PowerPC/ppc64le-smallarg.ll
+++ b/test/CodeGen/PowerPC/ppc64le-smallarg.ll
@@ -42,7 +42,8 @@ entry:
   ret float %x
 }
 ; CHECK: @callee2
-; CHECK: lfs {{[0-9]+}}, 136(1)
+; CHECK: addi [[TOCREG:[0-9]+]], 1, 136
+; CHECK: lxsspx {{[0-9]+}}, {{[0-9]+}}, [[TOCREG]]
 ; CHECK: blr
 
 define void @caller2() {
@@ -52,7 +53,8 @@ entry:
   ret void
 }
 ; CHECK: @caller2
-; CHECK: stfs {{[0-9]+}}, 136(1)
+; CHECK: li [[TOCOFF:[0-9]+]], 136
+; CHECK: stxsspx {{[0-9]+}}, 1, [[TOCOFF]]
 ; CHECK: bl test2
 
 declare float @test2(float, float, float, float, float, float, float, float, float, float, float, float, float, float)
diff --git a/test/CodeGen/PowerPC/pr17168.ll b/test/CodeGen/PowerPC/pr17168.ll
index 1f61dfb9b78..09689549138 100644
--- a/test/CodeGen/PowerPC/pr17168.ll
+++ b/test/CodeGen/PowerPC/pr17168.ll
@@ -25,7 +25,7 @@ for.cond968.preheader:                            ; preds = %for.cond968.prehead
 for.end1042:                                      ; preds = %for.cond968.preheader, %for.cond964.preheader, %entry
   %0 = phi i32 [ undef, %for.cond964.preheader ], [ undef, %for.cond968.preheader ], [ undef, %entry ]
   %1 = load i32, i32* getelementptr inbounds ([3 x i32], [3 x i32]* @grid_points, i64 0, i64 0), align 4, !dbg !443, !tbaa !444
-  tail call void @llvm.dbg.value(metadata i32 1, i64 0, metadata !119, metadata !MDExpression()), !dbg !448
+  tail call void @llvm.dbg.value(metadata i32 1, i64 0, metadata !119, metadata !DIExpression()), !dbg !448
   %sub10454270 = add nsw i32 %0, -1, !dbg !448
   %cmp10464271 = icmp sgt i32 %sub10454270, 1, !dbg !448
   %sub11134263 = add nsw i32 %1, -1, !dbg !450
@@ -54,468 +54,468 @@ attributes #1 = { nounwind readnone }
 !llvm.dbg.cu = !{!0}
 !llvm.module.flags = !{!438, !464}
 
-!0 = !MDCompileUnit(language: DW_LANG_C99, producer: "clang version 3.4 (trunk 190311)", isOptimized: true, emissionKind: 0, file: !1, enums: !2, retainedTypes: !2, subprograms: !3, globals: !298, imports: !2)
-!1 = !MDFile(filename: "bt.c", directory: "/home/hfinkel/src/NPB2.3-omp-C/BT")
+!0 = !DICompileUnit(language: DW_LANG_C99, producer: "clang version 3.4 (trunk 190311)", isOptimized: true, emissionKind: 0, file: !1, enums: !2, retainedTypes: !2, subprograms: !3, globals: !298, imports: !2)
+!1 = !DIFile(filename: "bt.c", directory: "/home/hfinkel/src/NPB2.3-omp-C/BT")
 !2 = !{}
 !3 = !{!4, !82, !102, !114, !132, !145, !154, !155, !162, !183, !200, !201, !207, !208, !215, !221, !230, !238, !246, !255, !260, !261, !268, !274, !279, !280, !287, !293}
-!4 = !MDSubprogram(name: "main", line: 74, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: true, scopeLine: 74, file: !1, scope: !5, type: !6, variables: !12)
-!5 = !MDFile(filename: "bt.c", directory: "/home/hfinkel/src/NPB2.3-omp-C/BT")
-!6 = !MDSubroutineType(types: !7)
+!4 = !DISubprogram(name: "main", line: 74, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: true, scopeLine: 74, file: !1, scope: !5, type: !6, variables: !12)
+!5 = !DIFile(filename: "bt.c", directory: "/home/hfinkel/src/NPB2.3-omp-C/BT")
+!6 = !DISubroutineType(types: !7)
 !7 = !{!8, !8, !9}
-!8 = !MDBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
-!9 = !MDDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, baseType: !10)
-!10 = !MDDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, baseType: !11)
-!11 = !MDBasicType(tag: DW_TAG_base_type, name: "char", size: 8, align: 8, encoding: DW_ATE_unsigned_char)
+!8 = !DIBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
+!9 = !DIDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, baseType: !10)
+!10 = !DIDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, baseType: !11)
+!11 = !DIBasicType(tag: DW_TAG_base_type, name: "char", size: 8, align: 8, encoding: DW_ATE_unsigned_char)
 !12 = !{!13, !14, !15, !16, !17, !18, !19, !21, !22, !23, !25, !26}
-!13 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "argc", line: 74, arg: 1, scope: !4, file: !5, type: !8)
-!14 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "argv", line: 74, arg: 2, scope: !4, file: !5, type: !9)
-!15 = !MDLocalVariable(tag: DW_TAG_auto_variable, name: "niter", line: 76, scope: !4, file: !5, type: !8)
-!16 = !MDLocalVariable(tag: DW_TAG_auto_variable, name: "step", line: 76, scope: !4, file: !5, type: !8)
-!17 = !MDLocalVariable(tag: DW_TAG_auto_variable, name: "n3", line: 76, scope: !4, file: !5, type: !8)
-!18 = !MDLocalVariable(tag: DW_TAG_auto_variable, name: "nthreads", line: 77, scope: !4, file: !5, type: !8)
-!19 = !MDLocalVariable(tag: DW_TAG_auto_variable, name: "navg", line: 78, scope: !4, file: !5, type: !20)
-!20 = !MDBasicType(tag: DW_TAG_base_type, name: "double", size: 64, align: 64, encoding: DW_ATE_float)
-!21 = !MDLocalVariable(tag: DW_TAG_auto_variable, name: "mflops", line: 78, scope: !4, file: !5, type: !20)
-!22 = !MDLocalVariable(tag: DW_TAG_auto_variable, name: "tmax", line: 80, scope: !4, file: !5, type: !20)
-!23 = !MDLocalVariable(tag: DW_TAG_auto_variable, name: "verified", line: 81, scope: !4, file: !5, type: !24)
-!24 = !MDDerivedType(tag: DW_TAG_typedef, name: "boolean", line: 12, file: !1, baseType: !8)
-!25 = !MDLocalVariable(tag: DW_TAG_auto_variable, name: "class", line: 82, scope: !4, file: !5, type: !11)
-!26 = !MDLocalVariable(tag: DW_TAG_auto_variable, name: "fp", line: 83, scope: !4, file: !5, type: !27)
-!27 = !MDDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, baseType: !28)
-!28 = !MDDerivedType(tag: DW_TAG_typedef, name: "FILE", line: 49, file: !1, baseType: !29)
-!29 = !MDCompositeType(tag: DW_TAG_structure_type, name: "_IO_FILE", line: 271, size: 1728, align: 64, file: !30, elements: !31)
-!30 = !MDFile(filename: "/usr/include/libio.h", directory: "/home/hfinkel/src/NPB2.3-omp-C/BT")
+!13 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "argc", line: 74, arg: 1, scope: !4, file: !5, type: !8)
+!14 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "argv", line: 74, arg: 2, scope: !4, file: !5, type: !9)
+!15 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "niter", line: 76, scope: !4, file: !5, type: !8)
+!16 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "step", line: 76, scope: !4, file: !5, type: !8)
+!17 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "n3", line: 76, scope: !4, file: !5, type: !8)
+!18 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "nthreads", line: 77, scope: !4, file: !5, type: !8)
+!19 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "navg", line: 78, scope: !4, file: !5, type: !20)
+!20 = !DIBasicType(tag: DW_TAG_base_type, name: "double", size: 64, align: 64, encoding: DW_ATE_float)
+!21 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "mflops", line: 78, scope: !4, file: !5, type: !20)
+!22 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "tmax", line: 80, scope: !4, file: !5, type: !20)
+!23 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "verified", line: 81, scope: !4, file: !5, type: !24)
+!24 = !DIDerivedType(tag: DW_TAG_typedef, name: "boolean", line: 12, file: !1, baseType: !8)
+!25 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "class", line: 82, scope: !4, file: !5, type: !11)
+!26 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "fp", line: 83, scope: !4, file: !5, type: !27)
+!27 = !DIDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, baseType: !28)
+!28 = !DIDerivedType(tag: DW_TAG_typedef, name: "FILE", line: 49, file: !1, baseType: !29)
+!29 = !DICompositeType(tag: DW_TAG_structure_type, name: "_IO_FILE", line: 271, size: 1728, align: 64, file: !30, elements: !31)
+!30 = !DIFile(filename: "/usr/include/libio.h", directory: "/home/hfinkel/src/NPB2.3-omp-C/BT")
 !31 = !{!32, !33, !34, !35, !36, !37, !38, !39, !40, !41, !42, !43, !44, !52, !53, !54, !55, !58, !60, !62, !66, !68, !70, !71, !72, !73, !74, !77, !78}
-!32 = !MDDerivedType(tag: DW_TAG_member, name: "_flags", line: 272, size: 32, align: 32, file: !30, scope: !29, baseType: !8)
-!33 = !MDDerivedType(tag: DW_TAG_member, name: "_IO_read_ptr", line: 277, size: 64, align: 64, offset: 64, file: !30, scope: !29, baseType: !10)
-!34 = !MDDerivedType(tag: DW_TAG_member, name: "_IO_read_end", line: 278, size: 64, align: 64, offset: 128, file: !30, scope: !29, baseType: !10)
-!35 = !MDDerivedType(tag: DW_TAG_member, name: "_IO_read_base", line: 279, size: 64, align: 64, offset: 192, file: !30, scope: !29, baseType: !10)
-!36 = !MDDerivedType(tag: DW_TAG_member, name: "_IO_write_base", line: 280, size: 64, align: 64, offset: 256, file: !30, scope: !29, baseType: !10)
-!37 = !MDDerivedType(tag: DW_TAG_member, name: "_IO_write_ptr", line: 281, size: 64, align: 64, offset: 320, file: !30, scope: !29, baseType: !10)
-!38 = !MDDerivedType(tag: DW_TAG_member, name: "_IO_write_end", line: 282, size: 64, align: 64, offset: 384, file: !30, scope: !29, baseType: !10)
-!39 = !MDDerivedType(tag: DW_TAG_member, name: "_IO_buf_base", line: 283, size: 64, align: 64, offset: 448, file: !30, scope: !29, baseType: !10)
-!40 = !MDDerivedType(tag: DW_TAG_member, name: "_IO_buf_end", line: 284, size: 64, align: 64, offset: 512, file: !30, scope: !29, baseType: !10)
-!41 = !MDDerivedType(tag: DW_TAG_member, name: "_IO_save_base", line: 286, size: 64, align: 64, offset: 576, file: !30, scope: !29, baseType: !10)
-!42 = !MDDerivedType(tag: DW_TAG_member, name: "_IO_backup_base", line: 287, size: 64, align: 64, offset: 640, file: !30, scope: !29, baseType: !10)
-!43 = !MDDerivedType(tag: DW_TAG_member, name: "_IO_save_end", line: 288, size: 64, align: 64, offset: 704, file: !30, scope: !29, baseType: !10)
-!44 = !MDDerivedType(tag: DW_TAG_member, name: "_markers", line: 290, size: 64, align: 64, offset: 768, file: !30, scope: !29, baseType: !45)
-!45 = !MDDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, baseType: !46)
-!46 = !MDCompositeType(tag: DW_TAG_structure_type, name: "_IO_marker", line: 186, size: 192, align: 64, file: !30, elements: !47)
+!32 = !DIDerivedType(tag: DW_TAG_member, name: "_flags", line: 272, size: 32, align: 32, file: !30, scope: !29, baseType: !8)
+!33 = !DIDerivedType(tag: DW_TAG_member, name: "_IO_read_ptr", line: 277, size: 64, align: 64, offset: 64, file: !30, scope: !29, baseType: !10)
+!34 = !DIDerivedType(tag: DW_TAG_member, name: "_IO_read_end", line: 278, size: 64, align: 64, offset: 128, file: !30, scope: !29, baseType: !10)
+!35 = !DIDerivedType(tag: DW_TAG_member, name: "_IO_read_base", line: 279, size: 64, align: 64, offset: 192, file: !30, scope: !29, baseType: !10)
+!36 = !DIDerivedType(tag: DW_TAG_member, name: "_IO_write_base", line: 280, size: 64, align: 64, offset: 256, file: !30, scope: !29, baseType: !10)
+!37 = !DIDerivedType(tag: DW_TAG_member, name: "_IO_write_ptr", line: 281, size: 64, align: 64, offset: 320, file: !30, scope: !29, baseType: !10)
+!38 = !DIDerivedType(tag: DW_TAG_member, name: "_IO_write_end", line: 282, size: 64, align: 64, offset: 384, file: !30, scope: !29, baseType: !10)
+!39 = !DIDerivedType(tag: DW_TAG_member, name: "_IO_buf_base", line: 283, size: 64, align: 64, offset: 448, file: !30, scope: !29, baseType: !10)
+!40 = !DIDerivedType(tag: DW_TAG_member, name: "_IO_buf_end", line: 284, size: 64, align: 64, offset: 512, file: !30, scope: !29, baseType: !10)
+!41 = !DIDerivedType(tag: DW_TAG_member, name: "_IO_save_base", line: 286, size: 64, align: 64, offset: 576, file: !30, scope: !29, baseType: !10)
+!42 = !DIDerivedType(tag: DW_TAG_member, name: "_IO_backup_base", line: 287, size: 64, align: 64, offset: 640, file: !30, scope: !29, baseType: !10)
+!43 = !DIDerivedType(tag: DW_TAG_member, name: "_IO_save_end", line: 288, size: 64, align: 64, offset: 704, file: !30, scope: !29, baseType: !10)
+!44 = !DIDerivedType(tag: DW_TAG_member, name: "_markers", line: 290, size: 64, align: 64, offset: 768, file: !30, scope: !29, baseType: !45)
+!45 = !DIDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, baseType: !46)
+!46 = !DICompositeType(tag: DW_TAG_structure_type, name: "_IO_marker", line: 186, size: 192, align: 64, file: !30, elements: !47)
 !47 = !{!48, !49, !51}
-!48 = !MDDerivedType(tag: DW_TAG_member, name: "_next", line: 187, size: 64, align: 64, file: !30, scope: !46, baseType: !45)
-!49 = !MDDerivedType(tag: DW_TAG_member, name: "_sbuf", line: 188, size: 64, align: 64, offset: 64, file: !30, scope: !46, baseType: !50)
-!50 = !MDDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, baseType: !29)
-!51 = !MDDerivedType(tag: DW_TAG_member, name: "_pos", line: 192, size: 32, align: 32, offset: 128, file: !30, scope: !46, baseType: !8)
-!52 = !MDDerivedType(tag: DW_TAG_member, name: "_chain", line: 292, size: 64, align: 64, offset: 832, file: !30, scope: !29, baseType: !50)
-!53 = !MDDerivedType(tag: DW_TAG_member, name: "_fileno", line: 294, size: 32, align: 32, offset: 896, file: !30, scope: !29, baseType: !8)
-!54 = !MDDerivedType(tag: DW_TAG_member, name: "_flags2", line: 298, size: 32, align: 32, offset: 928, file: !30, scope: !29, baseType: !8)
-!55 = !MDDerivedType(tag: DW_TAG_member, name: "_old_offset", line: 300, size: 64, align: 64, offset: 960, file: !30, scope: !29, baseType: !56)
-!56 = !MDDerivedType(tag: DW_TAG_typedef, name: "__off_t", line: 141, file: !30, baseType: !57)
-!57 = !MDBasicType(tag: DW_TAG_base_type, name: "long int", size: 64, align: 64, encoding: DW_ATE_signed)
-!58 = !MDDerivedType(tag: DW_TAG_member, name: "_cur_column", line: 304, size: 16, align: 16, offset: 1024, file: !30, scope: !29, baseType: !59)
-!59 = !MDBasicType(tag: DW_TAG_base_type, name: "unsigned short", size: 16, align: 16, encoding: DW_ATE_unsigned)
-!60 = !MDDerivedType(tag: DW_TAG_member, name: "_vtable_offset", line: 305, size: 8, align: 8, offset: 1040, file: !30, scope: !29, baseType: !61)
-!61 = !MDBasicType(tag: DW_TAG_base_type, name: "signed char", size: 8, align: 8, encoding: DW_ATE_signed_char)
-!62 = !MDDerivedType(tag: DW_TAG_member, name: "_shortbuf", line: 306, size: 8, align: 8, offset: 1048, file: !30, scope: !29, baseType: !63)
-!63 = !MDCompositeType(tag: DW_TAG_array_type, size: 8, align: 8, baseType: !11, elements: !64)
+!48 = !DIDerivedType(tag: DW_TAG_member, name: "_next", line: 187, size: 64, align: 64, file: !30, scope: !46, baseType: !45)
+!49 = !DIDerivedType(tag: DW_TAG_member, name: "_sbuf", line: 188, size: 64, align: 64, offset: 64, file: !30, scope: !46, baseType: !50)
+!50 = !DIDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, baseType: !29)
+!51 = !DIDerivedType(tag: DW_TAG_member, name: "_pos", line: 192, size: 32, align: 32, offset: 128, file: !30, scope: !46, baseType: !8)
+!52 = !DIDerivedType(tag: DW_TAG_member, name: "_chain", line: 292, size: 64, align: 64, offset: 832, file: !30, scope: !29, baseType: !50)
+!53 = !DIDerivedType(tag: DW_TAG_member, name: "_fileno", line: 294, size: 32, align: 32, offset: 896, file: !30, scope: !29, baseType: !8)
+!54 = !DIDerivedType(tag: DW_TAG_member, name: "_flags2", line: 298, size: 32, align: 32, offset: 928, file: !30, scope: !29, baseType: !8)
+!55 = !DIDerivedType(tag: DW_TAG_member, name: "_old_offset", line: 300, size: 64, align: 64, offset: 960, file: !30, scope: !29, baseType: !56)
+!56 = !DIDerivedType(tag: DW_TAG_typedef, name: "__off_t", line: 141, file: !30, baseType: !57)
+!57 = !DIBasicType(tag: DW_TAG_base_type, name: "long int", size: 64, align: 64, encoding: DW_ATE_signed)
+!58 = !DIDerivedType(tag: DW_TAG_member, name: "_cur_column", line: 304, size: 16, align: 16, offset: 1024, file: !30, scope: !29, baseType: !59)
+!59 = !DIBasicType(tag: DW_TAG_base_type, name: "unsigned short", size: 16, align: 16, encoding: DW_ATE_unsigned)
+!60 = !DIDerivedType(tag: DW_TAG_member, name: "_vtable_offset", line: 305, size: 8, align: 8, offset: 1040, file: !30, scope: !29, baseType: !61)
+!61 = !DIBasicType(tag: DW_TAG_base_type, name: "signed char", size: 8, align: 8, encoding: DW_ATE_signed_char)
+!62 = !DIDerivedType(tag: DW_TAG_member, name: "_shortbuf", line: 306, size: 8, align: 8, offset: 1048, file: !30, scope: !29, baseType: !63)
+!63 = !DICompositeType(tag: DW_TAG_array_type, size: 8, align: 8, baseType: !11, elements: !64)
 !64 = !{!65}
-!65 = !MDSubrange(count: 1)
-!66 = !MDDerivedType(tag: DW_TAG_member, name: "_lock", line: 310, size: 64, align: 64, offset: 1088, file: !30, scope: !29, baseType: !67)
-!67 = !MDDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, baseType: null)
-!68 = !MDDerivedType(tag: DW_TAG_member, name: "_offset", line: 319, size: 64, align: 64, offset: 1152, file: !30, scope: !29, baseType: !69)
-!69 = !MDDerivedType(tag: DW_TAG_typedef, name: "__off64_t", line: 142, file: !30, baseType: !57)
-!70 = !MDDerivedType(tag: DW_TAG_member, name: "__pad1", line: 328, size: 64, align: 64, offset: 1216, file: !30, scope: !29, baseType: !67)
-!71 = !MDDerivedType(tag: DW_TAG_member, name: "__pad2", line: 329, size: 64, align: 64, offset: 1280, file: !30, scope: !29, baseType: !67)
-!72 = !MDDerivedType(tag: DW_TAG_member, name: "__pad3", line: 330, size: 64, align: 64, offset: 1344, file: !30, scope: !29, baseType: !67)
-!73 = !MDDerivedType(tag: DW_TAG_member, name: "__pad4", line: 331, size: 64, align: 64, offset: 1408, file: !30, scope: !29, baseType: !67)
-!74 = !MDDerivedType(tag: DW_TAG_member, name: "__pad5", line: 332, size: 64, align: 64, offset: 1472, file: !30, scope: !29, baseType: !75)
-!75 = !MDDerivedType(tag: DW_TAG_typedef, name: "size_t", line: 42, file: !30, baseType: !76)
-!76 = !MDBasicType(tag: DW_TAG_base_type, name: "long unsigned int", size: 64, align: 64, encoding: DW_ATE_unsigned)
-!77 = !MDDerivedType(tag: DW_TAG_member, name: "_mode", line: 334, size: 32, align: 32, offset: 1536, file: !30, scope: !29, baseType: !8)
-!78 = !MDDerivedType(tag: DW_TAG_member, name: "_unused2", line: 336, size: 160, align: 8, offset: 1568, file: !30, scope: !29, baseType: !79)
-!79 = !MDCompositeType(tag: DW_TAG_array_type, size: 160, align: 8, baseType: !11, elements: !80)
+!65 = !DISubrange(count: 1)
+!66 = !DIDerivedType(tag: DW_TAG_member, name: "_lock", line: 310, size: 64, align: 64, offset: 1088, file: !30, scope: !29, baseType: !67)
+!67 = !DIDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, baseType: null)
+!68 = !DIDerivedType(tag: DW_TAG_member, name: "_offset", line: 319, size: 64, align: 64, offset: 1152, file: !30, scope: !29, baseType: !69)
+!69 = !DIDerivedType(tag: DW_TAG_typedef, name: "__off64_t", line: 142, file: !30, baseType: !57)
+!70 = !DIDerivedType(tag: DW_TAG_member, name: "__pad1", line: 328, size: 64, align: 64, offset: 1216, file: !30, scope: !29, baseType: !67)
+!71 = !DIDerivedType(tag: DW_TAG_member, name: "__pad2", line: 329, size: 64, align: 64, offset: 1280, file: !30, scope: !29, baseType: !67)
+!72 = !DIDerivedType(tag: DW_TAG_member, name: "__pad3", line: 330, size: 64, align: 64, offset: 1344, file: !30, scope: !29, baseType: !67)
+!73 = !DIDerivedType(tag: DW_TAG_member, name: "__pad4", line: 331, size: 64, align: 64, offset: 1408, file: !30, scope: !29, baseType: !67)
+!74 = !DIDerivedType(tag: DW_TAG_member, name: "__pad5", line: 332, size: 64, align: 64, offset: 1472, file: !30, scope: !29, baseType: !75)
+!75 = !DIDerivedType(tag: DW_TAG_typedef, name: "size_t", line: 42, file: !30, baseType: !76)
+!76 = !DIBasicType(tag: DW_TAG_base_type, name: "long unsigned int", size: 64, align: 64, encoding: DW_ATE_unsigned)
+!77 = !DIDerivedType(tag: DW_TAG_member, name: "_mode", line: 334, size: 32, align: 32, offset: 1536, file: !30, scope: !29, baseType: !8)
+!78 = !DIDerivedType(tag: DW_TAG_member, name: "_unused2", line: 336, size: 160, align: 8, offset: 1568, file: !30, scope: !29, baseType: !79)
+!79 = !DICompositeType(tag: DW_TAG_array_type, size: 160, align: 8, baseType: !11, elements: !80)
 !80 = !{!81}
-!81 = !MDSubrange(count: 20)
-!82 = !MDSubprogram(name: "verify", line: 2388, isLocal: true, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: true, scopeLine: 2388, file: !1, scope: !5, type: !83, variables: !86)
-!83 = !MDSubroutineType(types: !84)
+!81 = !DISubrange(count: 20)
+!82 = !DISubprogram(name: "verify", line: 2388, isLocal: true, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: true, scopeLine: 2388, file: !1, scope: !5, type: !83, variables: !86)
+!83 = !DISubroutineType(types: !84)
 !84 = !{null, !8, !10, !85}
-!85 = !MDDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, baseType: !24)
+!85 = !DIDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, baseType: !24)
 !86 = !{!87, !88, !89, !90, !94, !95, !96, !97, !98, !99, !100, !101}
-!87 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "no_time_steps", line: 2388, arg: 1, scope: !82, file: !5, type: !8)
-!88 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "class", line: 2388, arg: 2, scope: !82, file: !5, type: !10)
-!89 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "verified", line: 2388, arg: 3, scope: !82, file: !5, type: !85)
-!90 = !MDLocalVariable(tag: DW_TAG_auto_variable, name: "xcrref", line: 2397, scope: !82, file: !5, type: !91)
-!91 = !MDCompositeType(tag: DW_TAG_array_type, size: 320, align: 64, baseType: !20, elements: !92)
+!87 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "no_time_steps", line: 2388, arg: 1, scope: !82, file: !5, type: !8)
+!88 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "class", line: 2388, arg: 2, scope: !82, file: !5, type: !10)
+!89 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "verified", line: 2388, arg: 3, scope: !82, file: !5, type: !85)
+!90 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "xcrref", line: 2397, scope: !82, file: !5, type: !91)
+!91 = !DICompositeType(tag: DW_TAG_array_type, size: 320, align: 64, baseType: !20, elements: !92)
 !92 = !{!93}
-!93 = !MDSubrange(count: 5)
-!94 = !MDLocalVariable(tag: DW_TAG_auto_variable, name: "xceref", line: 2397, scope: !82, file: !5, type: !91)
-!95 = !MDLocalVariable(tag: DW_TAG_auto_variable, name: "xcrdif", line: 2397, scope: !82, file: !5, type: !91)
-!96 = !MDLocalVariable(tag: DW_TAG_auto_variable, name: "xcedif", line: 2397, scope: !82, file: !5, type: !91)
-!97 = !MDLocalVariable(tag: DW_TAG_auto_variable, name: "epsilon", line: 2398, scope: !82, file: !5, type: !20)
-!98 = !MDLocalVariable(tag: DW_TAG_auto_variable, name: "xce", line: 2398, scope: !82, file: !5, type: !91)
-!99 = !MDLocalVariable(tag: DW_TAG_auto_variable, name: "xcr", line: 2398, scope: !82, file: !5, type: !91)
-!100 = !MDLocalVariable(tag: DW_TAG_auto_variable, name: "dtref", line: 2398, scope: !82, file: !5, type: !20)
-!101 = !MDLocalVariable(tag: DW_TAG_auto_variable, name: "m", line: 2399, scope: !82, file: !5, type: !8)
-!102 = !MDSubprogram(name: "rhs_norm", line: 266, isLocal: true, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: true, scopeLine: 266, file: !1, scope: !5, type: !103, variables: !106)
-!103 = !MDSubroutineType(types: !104)
+!93 = !DISubrange(count: 5)
+!94 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "xceref", line: 2397, scope: !82, file: !5, type: !91)
+!95 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "xcrdif", line: 2397, scope: !82, file: !5, type: !91)
+!96 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "xcedif", line: 2397, scope: !82, file: !5, type: !91)
+!97 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "epsilon", line: 2398, scope: !82, file: !5, type: !20)
+!98 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "xce", line: 2398, scope: !82, file: !5, type: !91)
+!99 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "xcr", line: 2398, scope: !82, file: !5, type: !91)
+!100 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "dtref", line: 2398, scope: !82, file: !5, type: !20)
+!101 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "m", line: 2399, scope: !82, file: !5, type: !8)
+!102 = !DISubprogram(name: "rhs_norm", line: 266, isLocal: true, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: true, scopeLine: 266, file: !1, scope: !5, type: !103, variables: !106)
+!103 = !DISubroutineType(types: !104)
 !104 = !{null, !105}
-!105 = !MDDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, baseType: !20)
+!105 = !DIDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, baseType: !20)
 !106 = !{!107, !108, !109, !110, !111, !112, !113}
-!107 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "rms", line: 266, arg: 1, scope: !102, file: !5, type: !105)
-!108 = !MDLocalVariable(tag: DW_TAG_auto_variable, name: "i", line: 271, scope: !102, file: !5, type: !8)
-!109 = !MDLocalVariable(tag: DW_TAG_auto_variable, name: "j", line: 271, scope: !102, file: !5, type: !8)
-!110 = !MDLocalVariable(tag: DW_TAG_auto_variable, name: "k", line: 271, scope: !102, file: !5, type: !8)
-!111 = !MDLocalVariable(tag: DW_TAG_auto_variable, name: "d", line: 271, scope: !102, file: !5, type: !8)
-!112 = !MDLocalVariable(tag: DW_TAG_auto_variable, name: "m", line: 271, scope: !102, file: !5, type: !8)
-!113 = !MDLocalVariable(tag: DW_TAG_auto_variable, name: "add", line: 272, scope: !102, file: !5, type: !20)
-!114 = !MDSubprogram(name: "compute_rhs", line: 1767, isLocal: true, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: true, scopeLine: 1767, file: !1, scope: !5, type: !115, function: void ()* @compute_rhs, variables: !117)
-!115 = !MDSubroutineType(types: !116)
+!107 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "rms", line: 266, arg: 1, scope: !102, file: !5, type: !105)
+!108 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "i", line: 271, scope: !102, file: !5, type: !8)
+!109 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "j", line: 271, scope: !102, file: !5, type: !8)
+!110 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "k", line: 271, scope: !102, file: !5, type: !8)
+!111 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "d", line: 271, scope: !102, file: !5, type: !8)
+!112 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "m", line: 271, scope: !102, file: !5, type: !8)
+!113 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "add", line: 272, scope: !102, file: !5, type: !20)
+!114 = !DISubprogram(name: "compute_rhs", line: 1767, isLocal: true, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: true, scopeLine: 1767, file: !1, scope: !5, type: !115, function: void ()* @compute_rhs, variables: !117)
+!115 = !DISubroutineType(types: !116)
 !116 = !{null}
 !117 = !{!118, !119, !120, !121, !122, !123, !124, !125, !126, !127, !128, !129, !130, !131}
-!118 = !MDLocalVariable(tag: DW_TAG_auto_variable, name: "i", line: 1769, scope: !114, file: !5, type: !8)
-!119 = !MDLocalVariable(tag: DW_TAG_auto_variable, name: "j", line: 1769, scope: !114, file: !5, type: !8)
-!120 = !MDLocalVariable(tag: DW_TAG_auto_variable, name: "k", line: 1769, scope: !114, file: !5, type: !8)
-!121 = !MDLocalVariable(tag: DW_TAG_auto_variable, name: "m", line: 1769, scope: !114, file: !5, type: !8)
-!122 = !MDLocalVariable(tag: DW_TAG_auto_variable, name: "rho_inv", line: 1770, scope: !114, file: !5, type: !20)
-!123 = !MDLocalVariable(tag: DW_TAG_auto_variable, name: "uijk", line: 1770, scope: !114, file: !5, type: !20)
-!124 = !MDLocalVariable(tag: DW_TAG_auto_variable, name: "up1", line: 1770, scope: !114, file: !5, type: !20)
-!125 = !MDLocalVariable(tag: DW_TAG_auto_variable, name: "um1", line: 1770, scope: !114, file: !5, type: !20)
-!126 = !MDLocalVariable(tag: DW_TAG_auto_variable, name: "vijk", line: 1770, scope: !114, file: !5, type: !20)
-!127 = !MDLocalVariable(tag: DW_TAG_auto_variable, name: "vp1", line: 1770, scope: !114, file: !5, type: !20)
-!128 = !MDLocalVariable(tag: DW_TAG_auto_variable, name: "vm1", line: 1770, scope: !114, file: !5, type: !20)
-!129 = !MDLocalVariable(tag: DW_TAG_auto_variable, name: "wijk", line: 1770, scope: !114, file: !5, type: !20)
-!130 = !MDLocalVariable(tag: DW_TAG_auto_variable, name: "wp1", line: 1770, scope: !114, file: !5, type: !20)
-!131 = !MDLocalVariable(tag: DW_TAG_auto_variable, name: "wm1", line: 1770, scope: !114, file: !5, type: !20)
-!132 = !MDSubprogram(name: "error_norm", line: 225, isLocal: true, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: true, scopeLine: 225, file: !1, scope: !5, type: !103, variables: !133)
+!118 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "i", line: 1769, scope: !114, file: !5, type: !8)
+!119 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "j", line: 1769, scope: !114, file: !5, type: !8)
+!120 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "k", line: 1769, scope: !114, file: !5, type: !8)
+!121 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "m", line: 1769, scope: !114, file: !5, type: !8)
+!122 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "rho_inv", line: 1770, scope: !114, file: !5, type: !20)
+!123 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "uijk", line: 1770, scope: !114, file: !5, type: !20)
+!124 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "up1", line: 1770, scope: !114, file: !5, type: !20)
+!125 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "um1", line: 1770, scope: !114, file: !5, type: !20)
+!126 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "vijk", line: 1770, scope: !114, file: !5, type: !20)
+!127 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "vp1", line: 1770, scope: !114, file: !5, type: !20)
+!128 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "vm1", line: 1770, scope: !114, file: !5, type: !20)
+!129 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "wijk", line: 1770, scope: !114, file: !5, type: !20)
+!130 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "wp1", line: 1770, scope: !114, file: !5, type: !20)
+!131 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "wm1", line: 1770, scope: !114, file: !5, type: !20)
+!132 = !DISubprogram(name: "error_norm", line: 225, isLocal: true, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: true, scopeLine: 225, file: !1, scope: !5, type: !103, variables: !133)
 !133 = !{!134, !135, !136, !137, !138, !139, !140, !141, !142, !143, !144}
-!134 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "rms", line: 225, arg: 1, scope: !132, file: !5, type: !105)
-!135 = !MDLocalVariable(tag: DW_TAG_auto_variable, name: "i", line: 232, scope: !132, file: !5, type: !8)
-!136 = !MDLocalVariable(tag: DW_TAG_auto_variable, name: "j", line: 232, scope: !132, file: !5, type: !8)
-!137 = !MDLocalVariable(tag: DW_TAG_auto_variable, name: "k", line: 232, scope: !132, file: !5, type: !8)
-!138 = !MDLocalVariable(tag: DW_TAG_auto_variable, name: "m", line: 232, scope: !132, file: !5, type: !8)
-!139 = !MDLocalVariable(tag: DW_TAG_auto_variable, name: "d", line: 232, scope: !132, file: !5, type: !8)
-!140 = !MDLocalVariable(tag: DW_TAG_auto_variable, name: "xi", line: 233, scope: !132, file: !5, type: !20)
-!141 = !MDLocalVariable(tag: DW_TAG_auto_variable, name: "eta", line: 233, scope: !132, file: !5, type: !20)
-!142 = !MDLocalVariable(tag: DW_TAG_auto_variable, name: "zeta", line: 233, scope: !132, file: !5, type: !20)
-!143 = !MDLocalVariable(tag: DW_TAG_auto_variable, name: "u_exact", line: 233, scope: !132, file: !5, type: !91)
-!144 = !MDLocalVariable(tag: DW_TAG_auto_variable, name: "add", line: 233, scope: !132, file: !5, type: !20)
-!145 = !MDSubprogram(name: "exact_solution", line: 643, isLocal: true, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: true, scopeLine: 644, file: !1, scope: !5, type: !146, variables: !148)
-!146 = !MDSubroutineType(types: !147)
+!134 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "rms", line: 225, arg: 1, scope: !132, file: !5, type: !105)
+!135 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "i", line: 232, scope: !132, file: !5, type: !8)
+!136 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "j", line: 232, scope: !132, file: !5, type: !8)
+!137 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "k", line: 232, scope: !132, file: !5, type: !8)
+!138 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "m", line: 232, scope: !132, file: !5, type: !8)
+!139 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "d", line: 232, scope: !132, file: !5, type: !8)
+!140 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "xi", line: 233, scope: !132, file: !5, type: !20)
+!141 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "eta", line: 233, scope: !132, file: !5, type: !20)
+!142 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "zeta", line: 233, scope: !132, file: !5, type: !20)
+!143 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "u_exact", line: 233, scope: !132, file: !5, type: !91)
+!144 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "add", line: 233, scope: !132, file: !5, type: !20)
+!145 = !DISubprogram(name: "exact_solution", line: 643, isLocal: true, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: true, scopeLine: 644, file: !1, scope: !5, type: !146, variables: !148)
+!146 = !DISubroutineType(types: !147)
 !147 = !{null, !20, !20, !20, !105}
 !148 = !{!149, !150, !151, !152, !153}
-!149 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "xi", line: 643, arg: 1, scope: !145, file: !5, type: !20)
-!150 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "eta", line: 643, arg: 2, scope: !145, file: !5, type: !20)
-!151 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "zeta", line: 643, arg: 3, scope: !145, file: !5, type: !20)
-!152 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "dtemp", line: 644, arg: 4, scope: !145, file: !5, type: !105)
-!153 = !MDLocalVariable(tag: DW_TAG_auto_variable, name: "m", line: 653, scope: !145, file: !5, type: !8)
-!154 = !MDSubprogram(name: "set_constants", line: 2191, isLocal: true, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: true, scopeLine: 2191, file: !1, scope: !5, type: !115, variables: !2)
-!155 = !MDSubprogram(name: "lhsinit", line: 855, isLocal: true, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: true, scopeLine: 855, file: !1, scope: !5, type: !115, variables: !156)
+!149 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "xi", line: 643, arg: 1, scope: !145, file: !5, type: !20)
+!150 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "eta", line: 643, arg: 2, scope: !145, file: !5, type: !20)
+!151 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "zeta", line: 643, arg: 3, scope: !145, file: !5, type: !20)
+!152 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "dtemp", line: 644, arg: 4, scope: !145, file: !5, type: !105)
+!153 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "m", line: 653, scope: !145, file: !5, type: !8)
+!154 = !DISubprogram(name: "set_constants", line: 2191, isLocal: true, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: true, scopeLine: 2191, file: !1, scope: !5, type: !115, variables: !2)
+!155 = !DISubprogram(name: "lhsinit", line: 855, isLocal: true, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: true, scopeLine: 855, file: !1, scope: !5, type: !115, variables: !156)
 !156 = !{!157, !158, !159, !160, !161}
-!157 = !MDLocalVariable(tag: DW_TAG_auto_variable, name: "i", line: 857, scope: !155, file: !5, type: !8)
-!158 = !MDLocalVariable(tag: DW_TAG_auto_variable, name: "j", line: 857, scope: !155, file: !5, type: !8)
-!159 = !MDLocalVariable(tag: DW_TAG_auto_variable, name: "k", line: 857, scope: !155, file: !5, type: !8)
-!160 = !MDLocalVariable(tag: DW_TAG_auto_variable, name: "m", line: 857, scope: !155, file: !5, type: !8)
-!161 = !MDLocalVariable(tag: DW_TAG_auto_variable, name: "n", line: 857, scope: !155, file: !5, type: !8)
-!162 = !MDSubprogram(name: "initialize", line: 669, isLocal: true, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: true, scopeLine: 669, file: !1, scope: !5, type: !115, variables: !163)
+!157 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "i", line: 857, scope: !155, file: !5, type: !8)
+!158 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "j", line: 857, scope: !155, file: !5, type: !8)
+!159 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "k", line: 857, scope: !155, file: !5, type: !8)
+!160 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "m", line: 857, scope: !155, file: !5, type: !8)
+!161 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "n", line: 857, scope: !155, file: !5, type: !8)
+!162 = !DISubprogram(name: "initialize", line: 669, isLocal: true, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: true, scopeLine: 669, file: !1, scope: !5, type: !115, variables: !163)
 !163 = !{!164, !165, !166, !167, !168, !169, !170, !171, !172, !173, !174, !179, !180, !181, !182}
-!164 = !MDLocalVariable(tag: DW_TAG_auto_variable, name: "i", line: 679, scope: !162, file: !5, type: !8)
-!165 = !MDLocalVariable(tag: DW_TAG_auto_variable, name: "j", line: 679, scope: !162, file: !5, type: !8)
-!166 = !MDLocalVariable(tag: DW_TAG_auto_variable, name: "k", line: 679, scope: !162, file: !5, type: !8)
-!167 = !MDLocalVariable(tag: DW_TAG_auto_variable, name: "m", line: 679, scope: !162, file: !5, type: !8)
-!168 = !MDLocalVariable(tag: DW_TAG_auto_variable, name: "ix", line: 679, scope: !162, file: !5, type: !8)
-!169 = !MDLocalVariable(tag: DW_TAG_auto_variable, name: "iy", line: 679, scope: !162, file: !5, type: !8)
-!170 = !MDLocalVariable(tag: DW_TAG_auto_variable, name: "iz", line: 679, scope: !162, file: !5, type: !8)
-!171 = !MDLocalVariable(tag: DW_TAG_auto_variable, name: "xi", line: 680, scope: !162, file: !5, type: !20)
-!172 = !MDLocalVariable(tag: DW_TAG_auto_variable, name: "eta", line: 680, scope: !162, file: !5, type: !20)
-!173 = !MDLocalVariable(tag: DW_TAG_auto_variable, name: "zeta", line: 680, scope: !162, file: !5, type: !20)
-!174 = !MDLocalVariable(tag: DW_TAG_auto_variable, name: "Pface", line: 680, scope: !162, file: !5, type: !175)
-!175 = !MDCompositeType(tag: DW_TAG_array_type, size: 1920, align: 64, baseType: !20, elements: !176)
+!164 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "i", line: 679, scope: !162, file: !5, type: !8)
+!165 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "j", line: 679, scope: !162, file: !5, type: !8)
+!166 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "k", line: 679, scope: !162, file: !5, type: !8)
+!167 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "m", line: 679, scope: !162, file: !5, type: !8)
+!168 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "ix", line: 679, scope: !162, file: !5, type: !8)
+!169 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "iy", line: 679, scope: !162, file: !5, type: !8)
+!170 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "iz", line: 679, scope: !162, file: !5, type: !8)
+!171 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "xi", line: 680, scope: !162, file: !5, type: !20)
+!172 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "eta", line: 680, scope: !162, file: !5, type: !20)
+!173 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "zeta", line: 680, scope: !162, file: !5, type: !20)
+!174 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "Pface", line: 680, scope: !162, file: !5, type: !175)
+!175 = !DICompositeType(tag: DW_TAG_array_type, size: 1920, align: 64, baseType: !20, elements: !176)
 !176 = !{!177, !178, !93}
-!177 = !MDSubrange(count: 2)
-!178 = !MDSubrange(count: 3)
-!179 = !MDLocalVariable(tag: DW_TAG_auto_variable, name: "Pxi", line: 680, scope: !162, file: !5, type: !20)
-!180 = !MDLocalVariable(tag: DW_TAG_auto_variable, name: "Peta", line: 680, scope: !162, file: !5, type: !20)
-!181 = !MDLocalVariable(tag: DW_TAG_auto_variable, name: "Pzeta", line: 680, scope: !162, file: !5, type: !20)
-!182 = !MDLocalVariable(tag: DW_TAG_auto_variable, name: "temp", line: 680, scope: !162, file: !5, type: !91)
-!183 = !MDSubprogram(name: "exact_rhs", line: 301, isLocal: true, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: true, scopeLine: 301, file: !1, scope: !5, type: !115, variables: !184)
+!177 = !DISubrange(count: 2)
+!178 = !DISubrange(count: 3)
+!179 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "Pxi", line: 680, scope: !162, file: !5, type: !20)
+!180 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "Peta", line: 680, scope: !162, file: !5, type: !20)
+!181 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "Pzeta", line: 680, scope: !162, file: !5, type: !20)
+!182 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "temp", line: 680, scope: !162, file: !5, type: !91)
+!183 = !DISubprogram(name: "exact_rhs", line: 301, isLocal: true, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: true, scopeLine: 301, file: !1, scope: !5, type: !115, variables: !184)
 !184 = !{!185, !186, !187, !188, !189, !190, !191, !192, !193, !194, !195, !196, !197, !198, !199}
-!185 = !MDLocalVariable(tag: DW_TAG_auto_variable, name: "dtemp", line: 310, scope: !183, file: !5, type: !91)
-!186 = !MDLocalVariable(tag: DW_TAG_auto_variable, name: "xi", line: 310, scope: !183, file: !5, type: !20)
-!187 = !MDLocalVariable(tag: DW_TAG_auto_variable, name: "eta", line: 310, scope: !183, file: !5, type: !20)
-!188 = !MDLocalVariable(tag: DW_TAG_auto_variable, name: "zeta", line: 310, scope: !183, file: !5, type: !20)
-!189 = !MDLocalVariable(tag: DW_TAG_auto_variable, name: "dtpp", line: 310, scope: !183, file: !5, type: !20)
-!190 = !MDLocalVariable(tag: DW_TAG_auto_variable, name: "m", line: 311, scope: !183, file: !5, type: !8)
-!191 = !MDLocalVariable(tag: DW_TAG_auto_variable, name: "i", line: 311, scope: !183, file: !5, type: !8)
-!192 = !MDLocalVariable(tag: DW_TAG_auto_variable, name: "j", line: 311, scope: !183, file: !5, type: !8)
-!193 = !MDLocalVariable(tag: DW_TAG_auto_variable, name: "k", line: 311, scope: !183, file: !5, type: !8)
-!194 = !MDLocalVariable(tag: DW_TAG_auto_variable, name: "ip1", line: 311, scope: !183, file: !5, type: !8)
-!195 = !MDLocalVariable(tag: DW_TAG_auto_variable, name: "im1", line: 311, scope: !183, file: !5, type: !8)
-!196 = !MDLocalVariable(tag: DW_TAG_auto_variable, name: "jp1", line: 311, scope: !183, file: !5, type: !8)
-!197 = !MDLocalVariable(tag: DW_TAG_auto_variable, name: "jm1", line: 311, scope: !183, file: !5, type: !8)
-!198 = !MDLocalVariable(tag: DW_TAG_auto_variable, name: "km1", line: 311, scope: !183, file: !5, type: !8)
-!199 = !MDLocalVariable(tag: DW_TAG_auto_variable, name: "kp1", line: 311, scope: !183, file: !5, type: !8)
-!200 = !MDSubprogram(name: "adi", line: 210, isLocal: true, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: true, scopeLine: 210, file: !1, scope: !5, type: !115, variables: !2)
-!201 = !MDSubprogram(name: "add", line: 187, isLocal: true, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: true, scopeLine: 187, file: !1, scope: !5, type: !115, variables: !202)
+!185 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "dtemp", line: 310, scope: !183, file: !5, type: !91)
+!186 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "xi", line: 310, scope: !183, file: !5, type: !20)
+!187 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "eta", line: 310, scope: !183, file: !5, type: !20)
+!188 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "zeta", line: 310, scope: !183, file: !5, type: !20)
+!189 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "dtpp", line: 310, scope: !183, file: !5, type: !20)
+!190 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "m", line: 311, scope: !183, file: !5, type: !8)
+!191 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "i", line: 311, scope: !183, file: !5, type: !8)
+!192 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "j", line: 311, scope: !183, file: !5, type: !8)
+!193 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "k", line: 311, scope: !183, file: !5, type: !8)
+!194 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "ip1", line: 311, scope: !183, file: !5, type: !8)
+!195 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "im1", line: 311, scope: !183, file: !5, type: !8)
+!196 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "jp1", line: 311, scope: !183, file: !5, type: !8)
+!197 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "jm1", line: 311, scope: !183, file: !5, type: !8)
+!198 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "km1", line: 311, scope: !183, file: !5, type: !8)
+!199 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "kp1", line: 311, scope: !183, file: !5, type: !8)
+!200 = !DISubprogram(name: "adi", line: 210, isLocal: true, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: true, scopeLine: 210, file: !1, scope: !5, type: !115, variables: !2)
+!201 = !DISubprogram(name: "add", line: 187, isLocal: true, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: true, scopeLine: 187, file: !1, scope: !5, type: !115, variables: !202)
 !202 = !{!203, !204, !205, !206}
-!203 = !MDLocalVariable(tag: DW_TAG_auto_variable, name: "i", line: 193, scope: !201, file: !5, type: !8)
-!204 = !MDLocalVariable(tag: DW_TAG_auto_variable, name: "j", line: 193, scope: !201, file: !5, type: !8)
-!205 = !MDLocalVariable(tag: DW_TAG_auto_variable, name: "k", line: 193, scope: !201, file: !5, type: !8)
-!206 = !MDLocalVariable(tag: DW_TAG_auto_variable, name: "m", line: 193, scope: !201, file: !5, type: !8)
-!207 = !MDSubprogram(name: "z_solve", line: 3457, isLocal: true, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: true, scopeLine: 3457, file: !1, scope: !5, type: !115, variables: !2)
-!208 = !MDSubprogram(name: "z_backsubstitute", line: 3480, isLocal: true, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: true, scopeLine: 3480, file: !1, scope: !5, type: !115, variables: !209)
+!203 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "i", line: 193, scope: !201, file: !5, type: !8)
+!204 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "j", line: 193, scope: !201, file: !5, type: !8)
+!205 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "k", line: 193, scope: !201, file: !5, type: !8)
+!206 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "m", line: 193, scope: !201, file: !5, type: !8)
+!207 = !DISubprogram(name: "z_solve", line: 3457, isLocal: true, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: true, scopeLine: 3457, file: !1, scope: !5, type: !115, variables: !2)
+!208 = !DISubprogram(name: "z_backsubstitute", line: 3480, isLocal: true, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: true, scopeLine: 3480, file: !1, scope: !5, type: !115, variables: !209)
 !209 = !{!210, !211, !212, !213, !214}
-!210 = !MDLocalVariable(tag: DW_TAG_auto_variable, name: "i", line: 3492, scope: !208, file: !5, type: !8)
-!211 = !MDLocalVariable(tag: DW_TAG_auto_variable, name: "j", line: 3492, scope: !208, file: !5, type: !8)
-!212 = !MDLocalVariable(tag: DW_TAG_auto_variable, name: "k", line: 3492, scope: !208, file: !5, type: !8)
-!213 = !MDLocalVariable(tag: DW_TAG_auto_variable, name: "m", line: 3492, scope: !208, file: !5, type: !8)
-!214 = !MDLocalVariable(tag: DW_TAG_auto_variable, name: "n", line: 3492, scope: !208, file: !5, type: !8)
-!215 = !MDSubprogram(name: "z_solve_cell", line: 3512, isLocal: true, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: true, scopeLine: 3512, file: !1, scope: !5, type: !115, variables: !216)
+!210 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "i", line: 3492, scope: !208, file: !5, type: !8)
+!211 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "j", line: 3492, scope: !208, file: !5, type: !8)
+!212 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "k", line: 3492, scope: !208, file: !5, type: !8)
+!213 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "m", line: 3492, scope: !208, file: !5, type: !8)
+!214 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "n", line: 3492, scope: !208, file: !5, type: !8)
+!215 = !DISubprogram(name: "z_solve_cell", line: 3512, isLocal: true, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: true, scopeLine: 3512, file: !1, scope: !5, type: !115, variables: !216)
 !216 = !{!217, !218, !219, !220}
-!217 = !MDLocalVariable(tag: DW_TAG_auto_variable, name: "i", line: 3527, scope: !215, file: !5, type: !8)
-!218 = !MDLocalVariable(tag: DW_TAG_auto_variable, name: "j", line: 3527, scope: !215, file: !5, type: !8)
-!219 = !MDLocalVariable(tag: DW_TAG_auto_variable, name: "k", line: 3527, scope: !215, file: !5, type: !8)
-!220 = !MDLocalVariable(tag: DW_TAG_auto_variable, name: "ksize", line: 3527, scope: !215, file: !5, type: !8)
-!221 = !MDSubprogram(name: "binvrhs", line: 3154, isLocal: true, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: true, scopeLine: 3154, file: !1, scope: !5, type: !222, variables: !225)
-!222 = !MDSubroutineType(types: !223)
+!217 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "i", line: 3527, scope: !215, file: !5, type: !8)
+!218 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "j", line: 3527, scope: !215, file: !5, type: !8)
+!219 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "k", line: 3527, scope: !215, file: !5, type: !8)
+!220 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "ksize", line: 3527, scope: !215, file: !5, type: !8)
+!221 = !DISubprogram(name: "binvrhs", line: 3154, isLocal: true, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: true, scopeLine: 3154, file: !1, scope: !5, type: !222, variables: !225)
+!222 = !DISubroutineType(types: !223)
 !223 = !{null, !224, !105}
-!224 = !MDDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, baseType: !91)
+!224 = !DIDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, baseType: !91)
 !225 = !{!226, !227, !228, !229}
-!226 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "lhs", line: 3154, arg: 1, scope: !221, file: !5, type: !224)
-!227 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "r", line: 3154, arg: 2, scope: !221, file: !5, type: !105)
-!228 = !MDLocalVariable(tag: DW_TAG_auto_variable, name: "pivot", line: 3159, scope: !221, file: !5, type: !20)
-!229 = !MDLocalVariable(tag: DW_TAG_auto_variable, name: "coeff", line: 3159, scope: !221, file: !5, type: !20)
-!230 = !MDSubprogram(name: "matmul_sub", line: 2841, isLocal: true, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: true, scopeLine: 2842, file: !1, scope: !5, type: !231, variables: !233)
-!231 = !MDSubroutineType(types: !232)
+!226 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "lhs", line: 3154, arg: 1, scope: !221, file: !5, type: !224)
+!227 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "r", line: 3154, arg: 2, scope: !221, file: !5, type: !105)
+!228 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "pivot", line: 3159, scope: !221, file: !5, type: !20)
+!229 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "coeff", line: 3159, scope: !221, file: !5, type: !20)
+!230 = !DISubprogram(name: "matmul_sub", line: 2841, isLocal: true, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: true, scopeLine: 2842, file: !1, scope: !5, type: !231, variables: !233)
+!231 = !DISubroutineType(types: !232)
 !232 = !{null, !224, !224, !224}
 !233 = !{!234, !235, !236, !237}
-!234 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "ablock", line: 2841, arg: 1, scope: !230, file: !5, type: !224)
-!235 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "bblock", line: 2841, arg: 2, scope: !230, file: !5, type: !224)
-!236 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "cblock", line: 2842, arg: 3, scope: !230, file: !5, type: !224)
-!237 = !MDLocalVariable(tag: DW_TAG_auto_variable, name: "j", line: 2851, scope: !230, file: !5, type: !8)
-!238 = !MDSubprogram(name: "matvec_sub", line: 2814, isLocal: true, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: true, scopeLine: 2814, file: !1, scope: !5, type: !239, variables: !241)
-!239 = !MDSubroutineType(types: !240)
+!234 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "ablock", line: 2841, arg: 1, scope: !230, file: !5, type: !224)
+!235 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "bblock", line: 2841, arg: 2, scope: !230, file: !5, type: !224)
+!236 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "cblock", line: 2842, arg: 3, scope: !230, file: !5, type: !224)
+!237 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "j", line: 2851, scope: !230, file: !5, type: !8)
+!238 = !DISubprogram(name: "matvec_sub", line: 2814, isLocal: true, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: true, scopeLine: 2814, file: !1, scope: !5, type: !239, variables: !241)
+!239 = !DISubroutineType(types: !240)
 !240 = !{null, !224, !105, !105}
 !241 = !{!242, !243, !244, !245}
-!242 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "ablock", line: 2814, arg: 1, scope: !238, file: !5, type: !224)
-!243 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "avec", line: 2814, arg: 2, scope: !238, file: !5, type: !105)
-!244 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "bvec", line: 2814, arg: 3, scope: !238, file: !5, type: !105)
-!245 = !MDLocalVariable(tag: DW_TAG_auto_variable, name: "i", line: 2823, scope: !238, file: !5, type: !8)
-!246 = !MDSubprogram(name: "binvcrhs", line: 2885, isLocal: true, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: true, scopeLine: 2885, file: !1, scope: !5, type: !247, variables: !249)
-!247 = !MDSubroutineType(types: !248)
+!242 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "ablock", line: 2814, arg: 1, scope: !238, file: !5, type: !224)
+!243 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "avec", line: 2814, arg: 2, scope: !238, file: !5, type: !105)
+!244 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "bvec", line: 2814, arg: 3, scope: !238, file: !5, type: !105)
+!245 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "i", line: 2823, scope: !238, file: !5, type: !8)
+!246 = !DISubprogram(name: "binvcrhs", line: 2885, isLocal: true, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: true, scopeLine: 2885, file: !1, scope: !5, type: !247, variables: !249)
+!247 = !DISubroutineType(types: !248)
 !248 = !{null, !224, !224, !105}
 !249 = !{!250, !251, !252, !253, !254}
-!250 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "lhs", line: 2885, arg: 1, scope: !246, file: !5, type: !224)
-!251 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "c", line: 2885, arg: 2, scope: !246, file: !5, type: !224)
-!252 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "r", line: 2885, arg: 3, scope: !246, file: !5, type: !105)
-!253 = !MDLocalVariable(tag: DW_TAG_auto_variable, name: "pivot", line: 2890, scope: !246, file: !5, type: !20)
-!254 = !MDLocalVariable(tag: DW_TAG_auto_variable, name: "coeff", line: 2890, scope: !246, file: !5, type: !20)
-!255 = !MDSubprogram(name: "lhsz", line: 1475, isLocal: true, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: true, scopeLine: 1475, file: !1, scope: !5, type: !115, variables: !256)
+!250 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "lhs", line: 2885, arg: 1, scope: !246, file: !5, type: !224)
+!251 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "c", line: 2885, arg: 2, scope: !246, file: !5, type: !224)
+!252 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "r", line: 2885, arg: 3, scope: !246, file: !5, type: !105)
+!253 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "pivot", line: 2890, scope: !246, file: !5, type: !20)
+!254 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "coeff", line: 2890, scope: !246, file: !5, type: !20)
+!255 = !DISubprogram(name: "lhsz", line: 1475, isLocal: true, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: true, scopeLine: 1475, file: !1, scope: !5, type: !115, variables: !256)
 !256 = !{!257, !258, !259}
-!257 = !MDLocalVariable(tag: DW_TAG_auto_variable, name: "i", line: 1484, scope: !255, file: !5, type: !8)
-!258 = !MDLocalVariable(tag: DW_TAG_auto_variable, name: "j", line: 1484, scope: !255, file: !5, type: !8)
-!259 = !MDLocalVariable(tag: DW_TAG_auto_variable, name: "k", line: 1484, scope: !255, file: !5, type: !8)
-!260 = !MDSubprogram(name: "y_solve", line: 3299, isLocal: true, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: true, scopeLine: 3299, file: !1, scope: !5, type: !115, variables: !2)
-!261 = !MDSubprogram(name: "y_backsubstitute", line: 3323, isLocal: true, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: true, scopeLine: 3323, file: !1, scope: !5, type: !115, variables: !262)
+!257 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "i", line: 1484, scope: !255, file: !5, type: !8)
+!258 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "j", line: 1484, scope: !255, file: !5, type: !8)
+!259 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "k", line: 1484, scope: !255, file: !5, type: !8)
+!260 = !DISubprogram(name: "y_solve", line: 3299, isLocal: true, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: true, scopeLine: 3299, file: !1, scope: !5, type: !115, variables: !2)
+!261 = !DISubprogram(name: "y_backsubstitute", line: 3323, isLocal: true, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: true, scopeLine: 3323, file: !1, scope: !5, type: !115, variables: !262)
 !262 = !{!263, !264, !265, !266, !267}
-!263 = !MDLocalVariable(tag: DW_TAG_auto_variable, name: "i", line: 3335, scope: !261, file: !5, type: !8)
-!264 = !MDLocalVariable(tag: DW_TAG_auto_variable, name: "j", line: 3335, scope: !261, file: !5, type: !8)
-!265 = !MDLocalVariable(tag: DW_TAG_auto_variable, name: "k", line: 3335, scope: !261, file: !5, type: !8)
-!266 = !MDLocalVariable(tag: DW_TAG_auto_variable, name: "m", line: 3335, scope: !261, file: !5, type: !8)
-!267 = !MDLocalVariable(tag: DW_TAG_auto_variable, name: "n", line: 3335, scope: !261, file: !5, type: !8)
-!268 = !MDSubprogram(name: "y_solve_cell", line: 3355, isLocal: true, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: true, scopeLine: 3355, file: !1, scope: !5, type: !115, variables: !269)
+!263 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "i", line: 3335, scope: !261, file: !5, type: !8)
+!264 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "j", line: 3335, scope: !261, file: !5, type: !8)
+!265 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "k", line: 3335, scope: !261, file: !5, type: !8)
+!266 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "m", line: 3335, scope: !261, file: !5, type: !8)
+!267 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "n", line: 3335, scope: !261, file: !5, type: !8)
+!268 = !DISubprogram(name: "y_solve_cell", line: 3355, isLocal: true, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: true, scopeLine: 3355, file: !1, scope: !5, type: !115, variables: !269)
 !269 = !{!270, !271, !272, !273}
-!270 = !MDLocalVariable(tag: DW_TAG_auto_variable, name: "i", line: 3370, scope: !268, file: !5, type: !8)
-!271 = !MDLocalVariable(tag: DW_TAG_auto_variable, name: "j", line: 3370, scope: !268, file: !5, type: !8)
-!272 = !MDLocalVariable(tag: DW_TAG_auto_variable, name: "k", line: 3370, scope: !268, file: !5, type: !8)
-!273 = !MDLocalVariable(tag: DW_TAG_auto_variable, name: "jsize", line: 3370, scope: !268, file: !5, type: !8)
-!274 = !MDSubprogram(name: "lhsy", line: 1181, isLocal: true, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: true, scopeLine: 1181, file: !1, scope: !5, type: !115, variables: !275)
+!270 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "i", line: 3370, scope: !268, file: !5, type: !8)
+!271 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "j", line: 3370, scope: !268, file: !5, type: !8)
+!272 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "k", line: 3370, scope: !268, file: !5, type: !8)
+!273 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "jsize", line: 3370, scope: !268, file: !5, type: !8)
+!274 = !DISubprogram(name: "lhsy", line: 1181, isLocal: true, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: true, scopeLine: 1181, file: !1, scope: !5, type: !115, variables: !275)
 !275 = !{!276, !277, !278}
-!276 = !MDLocalVariable(tag: DW_TAG_auto_variable, name: "i", line: 1190, scope: !274, file: !5, type: !8)
-!277 = !MDLocalVariable(tag: DW_TAG_auto_variable, name: "j", line: 1190, scope: !274, file: !5, type: !8)
-!278 = !MDLocalVariable(tag: DW_TAG_auto_variable, name: "k", line: 1190, scope: !274, file: !5, type: !8)
-!279 = !MDSubprogram(name: "x_solve", line: 2658, isLocal: true, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: true, scopeLine: 2658, file: !1, scope: !5, type: !115, variables: !2)
-!280 = !MDSubprogram(name: "x_backsubstitute", line: 2684, isLocal: true, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: true, scopeLine: 2684, file: !1, scope: !5, type: !115, variables: !281)
+!276 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "i", line: 1190, scope: !274, file: !5, type: !8)
+!277 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "j", line: 1190, scope: !274, file: !5, type: !8)
+!278 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "k", line: 1190, scope: !274, file: !5, type: !8)
+!279 = !DISubprogram(name: "x_solve", line: 2658, isLocal: true, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: true, scopeLine: 2658, file: !1, scope: !5, type: !115, variables: !2)
+!280 = !DISubprogram(name: "x_backsubstitute", line: 2684, isLocal: true, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: true, scopeLine: 2684, file: !1, scope: !5, type: !115, variables: !281)
 !281 = !{!282, !283, !284, !285, !286}
-!282 = !MDLocalVariable(tag: DW_TAG_auto_variable, name: "i", line: 2696, scope: !280, file: !5, type: !8)
-!283 = !MDLocalVariable(tag: DW_TAG_auto_variable, name: "j", line: 2696, scope: !280, file: !5, type: !8)
-!284 = !MDLocalVariable(tag: DW_TAG_auto_variable, name: "k", line: 2696, scope: !280, file: !5, type: !8)
-!285 = !MDLocalVariable(tag: DW_TAG_auto_variable, name: "m", line: 2696, scope: !280, file: !5, type: !8)
-!286 = !MDLocalVariable(tag: DW_TAG_auto_variable, name: "n", line: 2696, scope: !280, file: !5, type: !8)
-!287 = !MDSubprogram(name: "x_solve_cell", line: 2716, isLocal: true, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: true, scopeLine: 2716, file: !1, scope: !5, type: !115, variables: !288)
+!282 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "i", line: 2696, scope: !280, file: !5, type: !8)
+!283 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "j", line: 2696, scope: !280, file: !5, type: !8)
+!284 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "k", line: 2696, scope: !280, file: !5, type: !8)
+!285 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "m", line: 2696, scope: !280, file: !5, type: !8)
+!286 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "n", line: 2696, scope: !280, file: !5, type: !8)
+!287 = !DISubprogram(name: "x_solve_cell", line: 2716, isLocal: true, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: true, scopeLine: 2716, file: !1, scope: !5, type: !115, variables: !288)
 !288 = !{!289, !290, !291, !292}
-!289 = !MDLocalVariable(tag: DW_TAG_auto_variable, name: "i", line: 2728, scope: !287, file: !5, type: !8)
-!290 = !MDLocalVariable(tag: DW_TAG_auto_variable, name: "j", line: 2728, scope: !287, file: !5, type: !8)
-!291 = !MDLocalVariable(tag: DW_TAG_auto_variable, name: "k", line: 2728, scope: !287, file: !5, type: !8)
-!292 = !MDLocalVariable(tag: DW_TAG_auto_variable, name: "isize", line: 2728, scope: !287, file: !5, type: !8)
-!293 = !MDSubprogram(name: "lhsx", line: 898, isLocal: true, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: true, scopeLine: 898, file: !1, scope: !5, type: !115, variables: !294)
+!289 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "i", line: 2728, scope: !287, file: !5, type: !8)
+!290 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "j", line: 2728, scope: !287, file: !5, type: !8)
+!291 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "k", line: 2728, scope: !287, file: !5, type: !8)
+!292 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "isize", line: 2728, scope: !287, file: !5, type: !8)
+!293 = !DISubprogram(name: "lhsx", line: 898, isLocal: true, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: true, scopeLine: 898, file: !1, scope: !5, type: !115, variables: !294)
 !294 = !{!295, !296, !297}
-!295 = !MDLocalVariable(tag: DW_TAG_auto_variable, name: "i", line: 907, scope: !293, file: !5, type: !8)
-!296 = !MDLocalVariable(tag: DW_TAG_auto_variable, name: "j", line: 907, scope: !293, file: !5, type: !8)
-!297 = !MDLocalVariable(tag: DW_TAG_auto_variable, name: "k", line: 907, scope: !293, file: !5, type: !8)
+!295 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "i", line: 907, scope: !293, file: !5, type: !8)
+!296 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "j", line: 907, scope: !293, file: !5, type: !8)
+!297 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "k", line: 907, scope: !293, file: !5, type: !8)
 !298 = !{!299, !304, !305, !309, !310, !311, !312, !313, !314, !315, !316, !317, !318, !319, !320, !321, !322, !323, !324, !325, !326, !327, !328, !329, !330, !331, !332, !333, !334, !335, !336, !337, !338, !339, !340, !341, !342, !343, !347, !350, !351, !352, !353, !354, !355, !356, !360, !361, !362, !363, !364, !365, !366, !367, !368, !369, !370, !371, !372, !373, !374, !375, !376, !377, !378, !379, !380, !381, !382, !383, !384, !385, !386, !387, !388, !389, !390, !391, !392, !393, !394, !395, !396, !397, !398, !399, !400, !401, !402, !403, !404, !405, !406, !407, !408, !409, !410, !411, !412, !413, !414, !415, !416, !417, !418, !419, !422, !426, !427, !430, !431, !434, !435, !436, !437}
-!299 = !MDGlobalVariable(name: "grid_points", line: 28, isLocal: true, isDefinition: true, scope: null, file: !300, type: !302, variable: [3 x i32]* @grid_points)
-!300 = !MDFile(filename: "./header.h", directory: "/home/hfinkel/src/NPB2.3-omp-C/BT")
+!299 = !DIGlobalVariable(name: "grid_points", line: 28, isLocal: true, isDefinition: true, scope: null, file: !300, type: !302, variable: [3 x i32]* @grid_points)
+!300 = !DIFile(filename: "./header.h", directory: "/home/hfinkel/src/NPB2.3-omp-C/BT")
 !301 = !{!"./header.h", !"/home/hfinkel/src/NPB2.3-omp-C/BT"}
-!302 = !MDCompositeType(tag: DW_TAG_array_type, size: 96, align: 32, baseType: !8, elements: !303)
+!302 = !DICompositeType(tag: DW_TAG_array_type, size: 96, align: 32, baseType: !8, elements: !303)
 !303 = !{!178}
-!304 = !MDGlobalVariable(name: "dt", line: 35, isLocal: true, isDefinition: true, scope: null, file: !300, type: !20)
-!305 = !MDGlobalVariable(name: "rhs", line: 68, isLocal: true, isDefinition: true, scope: null, file: !300, type: !306)
-!306 = !MDCompositeType(tag: DW_TAG_array_type, size: 1385839040, align: 64, baseType: !20, elements: !307)
+!304 = !DIGlobalVariable(name: "dt", line: 35, isLocal: true, isDefinition: true, scope: null, file: !300, type: !20)
+!305 = !DIGlobalVariable(name: "rhs", line: 68, isLocal: true, isDefinition: true, scope: null, file: !300, type: !306)
+!306 = !DICompositeType(tag: DW_TAG_array_type, size: 1385839040, align: 64, baseType: !20, elements: !307)
 !307 = !{!308, !308, !308, !93}
-!308 = !MDSubrange(count: 163)
-!309 = !MDGlobalVariable(name: "zzcon5", line: 42, isLocal: true, isDefinition: true, scope: null, file: !300, type: !20)
-!310 = !MDGlobalVariable(name: "zzcon4", line: 42, isLocal: true, isDefinition: true, scope: null, file: !300, type: !20)
-!311 = !MDGlobalVariable(name: "zzcon3", line: 42, isLocal: true, isDefinition: true, scope: null, file: !300, type: !20)
-!312 = !MDGlobalVariable(name: "dz5tz1", line: 43, isLocal: true, isDefinition: true, scope: null, file: !300, type: !20)
-!313 = !MDGlobalVariable(name: "dz4tz1", line: 43, isLocal: true, isDefinition: true, scope: null, file: !300, type: !20)
-!314 = !MDGlobalVariable(name: "dz3tz1", line: 43, isLocal: true, isDefinition: true, scope: null, file: !300, type: !20)
-!315 = !MDGlobalVariable(name: "zzcon2", line: 42, isLocal: true, isDefinition: true, scope: null, file: !300, type: !20)
-!316 = !MDGlobalVariable(name: "dz2tz1", line: 43, isLocal: true, isDefinition: true, scope: null, file: !300, type: !20)
-!317 = !MDGlobalVariable(name: "tz2", line: 31, isLocal: true, isDefinition: true, scope: null, file: !300, type: !20)
-!318 = !MDGlobalVariable(name: "dz1tz1", line: 43, isLocal: true, isDefinition: true, scope: null, file: !300, type: !20)
-!319 = !MDGlobalVariable(name: "yycon5", line: 40, isLocal: true, isDefinition: true, scope: null, file: !300, type: !20)
-!320 = !MDGlobalVariable(name: "yycon4", line: 40, isLocal: true, isDefinition: true, scope: null, file: !300, type: !20)
-!321 = !MDGlobalVariable(name: "yycon3", line: 40, isLocal: true, isDefinition: true, scope: null, file: !300, type: !20)
-!322 = !MDGlobalVariable(name: "dy5ty1", line: 41, isLocal: true, isDefinition: true, scope: null, file: !300, type: !20)
-!323 = !MDGlobalVariable(name: "dy4ty1", line: 41, isLocal: true, isDefinition: true, scope: null, file: !300, type: !20)
-!324 = !MDGlobalVariable(name: "dy3ty1", line: 41, isLocal: true, isDefinition: true, scope: null, file: !300, type: !20)
-!325 = !MDGlobalVariable(name: "yycon2", line: 40, isLocal: true, isDefinition: true, scope: null, file: !300, type: !20)
-!326 = !MDGlobalVariable(name: "dy2ty1", line: 41, isLocal: true, isDefinition: true, scope: null, file: !300, type: !20)
-!327 = !MDGlobalVariable(name: "ty2", line: 31, isLocal: true, isDefinition: true, scope: null, file: !300, type: !20)
-!328 = !MDGlobalVariable(name: "dy1ty1", line: 41, isLocal: true, isDefinition: true, scope: null, file: !300, type: !20)
-!329 = !MDGlobalVariable(name: "dssp", line: 35, isLocal: true, isDefinition: true, scope: null, file: !300, type: !20)
-!330 = !MDGlobalVariable(name: "c1", line: 45, isLocal: true, isDefinition: true, scope: null, file: !300, type: !20)
-!331 = !MDGlobalVariable(name: "xxcon5", line: 38, isLocal: true, isDefinition: true, scope: null, file: !300, type: !20)
-!332 = !MDGlobalVariable(name: "xxcon4", line: 38, isLocal: true, isDefinition: true, scope: null, file: !300, type: !20)
-!333 = !MDGlobalVariable(name: "xxcon3", line: 38, isLocal: true, isDefinition: true, scope: null, file: !300, type: !20)
-!334 = !MDGlobalVariable(name: "dx5tx1", line: 39, isLocal: true, isDefinition: true, scope: null, file: !300, type: !20)
-!335 = !MDGlobalVariable(name: "dx4tx1", line: 39, isLocal: true, isDefinition: true, scope: null, file: !300, type: !20)
-!336 = !MDGlobalVariable(name: "dx3tx1", line: 39, isLocal: true, isDefinition: true, scope: null, file: !300, type: !20)
-!337 = !MDGlobalVariable(name: "c2", line: 45, isLocal: true, isDefinition: true, scope: null, file: !300, type: !20)
-!338 = !MDGlobalVariable(name: "con43", line: 48, isLocal: true, isDefinition: true, scope: null, file: !300, type: !20)
-!339 = !MDGlobalVariable(name: "xxcon2", line: 38, isLocal: true, isDefinition: true, scope: null, file: !300, type: !20)
-!340 = !MDGlobalVariable(name: "dx2tx1", line: 39, isLocal: true, isDefinition: true, scope: null, file: !300, type: !20)
-!341 = !MDGlobalVariable(name: "tx2", line: 31, isLocal: true, isDefinition: true, scope: null, file: !300, type: !20)
-!342 = !MDGlobalVariable(name: "dx1tx1", line: 39, isLocal: true, isDefinition: true, scope: null, file: !300, type: !20)
-!343 = !MDGlobalVariable(name: "forcing", line: 66, isLocal: true, isDefinition: true, scope: null, file: !300, type: !344)
-!344 = !MDCompositeType(tag: DW_TAG_array_type, size: 1663006848, align: 64, baseType: !20, elements: !345)
+!308 = !DISubrange(count: 163)
+!309 = !DIGlobalVariable(name: "zzcon5", line: 42, isLocal: true, isDefinition: true, scope: null, file: !300, type: !20)
+!310 = !DIGlobalVariable(name: "zzcon4", line: 42, isLocal: true, isDefinition: true, scope: null, file: !300, type: !20)
+!311 = !DIGlobalVariable(name: "zzcon3", line: 42, isLocal: true, isDefinition: true, scope: null, file: !300, type: !20)
+!312 = !DIGlobalVariable(name: "dz5tz1", line: 43, isLocal: true, isDefinition: true, scope: null, file: !300, type: !20)
+!313 = !DIGlobalVariable(name: "dz4tz1", line: 43, isLocal: true, isDefinition: true, scope: null, file: !300, type: !20)
+!314 = !DIGlobalVariable(name: "dz3tz1", line: 43, isLocal: true, isDefinition: true, scope: null, file: !300, type: !20)
+!315 = !DIGlobalVariable(name: "zzcon2", line: 42, isLocal: true, isDefinition: true, scope: null, file: !300, type: !20)
+!316 = !DIGlobalVariable(name: "dz2tz1", line: 43, isLocal: true, isDefinition: true, scope: null, file: !300, type: !20)
+!317 = !DIGlobalVariable(name: "tz2", line: 31, isLocal: true, isDefinition: true, scope: null, file: !300, type: !20)
+!318 = !DIGlobalVariable(name: "dz1tz1", line: 43, isLocal: true, isDefinition: true, scope: null, file: !300, type: !20)
+!319 = !DIGlobalVariable(name: "yycon5", line: 40, isLocal: true, isDefinition: true, scope: null, file: !300, type: !20)
+!320 = !DIGlobalVariable(name: "yycon4", line: 40, isLocal: true, isDefinition: true, scope: null, file: !300, type: !20)
+!321 = !DIGlobalVariable(name: "yycon3", line: 40, isLocal: true, isDefinition: true, scope: null, file: !300, type: !20)
+!322 = !DIGlobalVariable(name: "dy5ty1", line: 41, isLocal: true, isDefinition: true, scope: null, file: !300, type: !20)
+!323 = !DIGlobalVariable(name: "dy4ty1", line: 41, isLocal: true, isDefinition: true, scope: null, file: !300, type: !20)
+!324 = !DIGlobalVariable(name: "dy3ty1", line: 41, isLocal: true, isDefinition: true, scope: null, file: !300, type: !20)
+!325 = !DIGlobalVariable(name: "yycon2", line: 40, isLocal: true, isDefinition: true, scope: null, file: !300, type: !20)
+!326 = !DIGlobalVariable(name: "dy2ty1", line: 41, isLocal: true, isDefinition: true, scope: null, file: !300, type: !20)
+!327 = !DIGlobalVariable(name: "ty2", line: 31, isLocal: true, isDefinition: true, scope: null, file: !300, type: !20)
+!328 = !DIGlobalVariable(name: "dy1ty1", line: 41, isLocal: true, isDefinition: true, scope: null, file: !300, type: !20)
+!329 = !DIGlobalVariable(name: "dssp", line: 35, isLocal: true, isDefinition: true, scope: null, file: !300, type: !20)
+!330 = !DIGlobalVariable(name: "c1", line: 45, isLocal: true, isDefinition: true, scope: null, file: !300, type: !20)
+!331 = !DIGlobalVariable(name: "xxcon5", line: 38, isLocal: true, isDefinition: true, scope: null, file: !300, type: !20)
+!332 = !DIGlobalVariable(name: "xxcon4", line: 38, isLocal: true, isDefinition: true, scope: null, file: !300, type: !20)
+!333 = !DIGlobalVariable(name: "xxcon3", line: 38, isLocal: true, isDefinition: true, scope: null, file: !300, type: !20)
+!334 = !DIGlobalVariable(name: "dx5tx1", line: 39, isLocal: true, isDefinition: true, scope: null, file: !300, type: !20)
+!335 = !DIGlobalVariable(name: "dx4tx1", line: 39, isLocal: true, isDefinition: true, scope: null, file: !300, type: !20)
+!336 = !DIGlobalVariable(name: "dx3tx1", line: 39, isLocal: true, isDefinition: true, scope: null, file: !300, type: !20)
+!337 = !DIGlobalVariable(name: "c2", line: 45, isLocal: true, isDefinition: true, scope: null, file: !300, type: !20)
+!338 = !DIGlobalVariable(name: "con43", line: 48, isLocal: true, isDefinition: true, scope: null, file: !300, type: !20)
+!339 = !DIGlobalVariable(name: "xxcon2", line: 38, isLocal: true, isDefinition: true, scope: null, file: !300, type: !20)
+!340 = !DIGlobalVariable(name: "dx2tx1", line: 39, isLocal: true, isDefinition: true, scope: null, file: !300, type: !20)
+!341 = !DIGlobalVariable(name: "tx2", line: 31, isLocal: true, isDefinition: true, scope: null, file: !300, type: !20)
+!342 = !DIGlobalVariable(name: "dx1tx1", line: 39, isLocal: true, isDefinition: true, scope: null, file: !300, type: !20)
+!343 = !DIGlobalVariable(name: "forcing", line: 66, isLocal: true, isDefinition: true, scope: null, file: !300, type: !344)
+!344 = !DICompositeType(tag: DW_TAG_array_type, size: 1663006848, align: 64, baseType: !20, elements: !345)
 !345 = !{!308, !308, !308, !346}
-!346 = !MDSubrange(count: 6)
-!347 = !MDGlobalVariable(name: "qs", line: 63, isLocal: true, isDefinition: true, scope: null, file: !300, type: !348)
-!348 = !MDCompositeType(tag: DW_TAG_array_type, size: 277167808, align: 64, baseType: !20, elements: !349)
+!346 = !DISubrange(count: 6)
+!347 = !DIGlobalVariable(name: "qs", line: 63, isLocal: true, isDefinition: true, scope: null, file: !300, type: !348)
+!348 = !DICompositeType(tag: DW_TAG_array_type, size: 277167808, align: 64, baseType: !20, elements: !349)
 !349 = !{!308, !308, !308}
-!350 = !MDGlobalVariable(name: "square", line: 65, isLocal: true, isDefinition: true, scope: null, file: !300, type: !348)
-!351 = !MDGlobalVariable(name: "ws", line: 62, isLocal: true, isDefinition: true, scope: null, file: !300, type: !348)
-!352 = !MDGlobalVariable(name: "vs", line: 61, isLocal: true, isDefinition: true, scope: null, file: !300, type: !348)
-!353 = !MDGlobalVariable(name: "us", line: 60, isLocal: true, isDefinition: true, scope: null, file: !300, type: !348)
-!354 = !MDGlobalVariable(name: "rho_i", line: 64, isLocal: true, isDefinition: true, scope: null, file: !300, type: !348)
-!355 = !MDGlobalVariable(name: "u", line: 67, isLocal: true, isDefinition: true, scope: null, file: !300, type: !306)
-!356 = !MDGlobalVariable(name: "ce", line: 36, isLocal: true, isDefinition: true, scope: null, file: !300, type: !357)
-!357 = !MDCompositeType(tag: DW_TAG_array_type, size: 4160, align: 64, baseType: !20, elements: !358)
+!350 = !DIGlobalVariable(name: "square", line: 65, isLocal: true, isDefinition: true, scope: null, file: !300, type: !348)
+!351 = !DIGlobalVariable(name: "ws", line: 62, isLocal: true, isDefinition: true, scope: null, file: !300, type: !348)
+!352 = !DIGlobalVariable(name: "vs", line: 61, isLocal: true, isDefinition: true, scope: null, file: !300, type: !348)
+!353 = !DIGlobalVariable(name: "us", line: 60, isLocal: true, isDefinition: true, scope: null, file: !300, type: !348)
+!354 = !DIGlobalVariable(name: "rho_i", line: 64, isLocal: true, isDefinition: true, scope: null, file: !300, type: !348)
+!355 = !DIGlobalVariable(name: "u", line: 67, isLocal: true, isDefinition: true, scope: null, file: !300, type: !306)
+!356 = !DIGlobalVariable(name: "ce", line: 36, isLocal: true, isDefinition: true, scope: null, file: !300, type: !357)
+!357 = !DICompositeType(tag: DW_TAG_array_type, size: 4160, align: 64, baseType: !20, elements: !358)
 !358 = !{!93, !359}
-!359 = !MDSubrange(count: 13)
-!360 = !MDGlobalVariable(name: "dnzm1", line: 44, isLocal: true, isDefinition: true, scope: null, file: !300, type: !20)
-!361 = !MDGlobalVariable(name: "dnym1", line: 44, isLocal: true, isDefinition: true, scope: null, file: !300, type: !20)
-!362 = !MDGlobalVariable(name: "dnxm1", line: 44, isLocal: true, isDefinition: true, scope: null, file: !300, type: !20)
-!363 = !MDGlobalVariable(name: "zzcon1", line: 42, isLocal: true, isDefinition: true, scope: null, file: !300, type: !20)
-!364 = !MDGlobalVariable(name: "yycon1", line: 40, isLocal: true, isDefinition: true, scope: null, file: !300, type: !20)
-!365 = !MDGlobalVariable(name: "xxcon1", line: 38, isLocal: true, isDefinition: true, scope: null, file: !300, type: !20)
-!366 = !MDGlobalVariable(name: "con16", line: 48, isLocal: true, isDefinition: true, scope: null, file: !300, type: !20)
-!367 = !MDGlobalVariable(name: "c2iv", line: 48, isLocal: true, isDefinition: true, scope: null, file: !300, type: !20)
-!368 = !MDGlobalVariable(name: "c3c4tz3", line: 48, isLocal: true, isDefinition: true, scope: null, file: !300, type: !20)
-!369 = !MDGlobalVariable(name: "c3c4ty3", line: 48, isLocal: true, isDefinition: true, scope: null, file: !300, type: !20)
-!370 = !MDGlobalVariable(name: "c3c4tx3", line: 48, isLocal: true, isDefinition: true, scope: null, file: !300, type: !20)
-!371 = !MDGlobalVariable(name: "comz6", line: 47, isLocal: true, isDefinition: true, scope: null, file: !300, type: !20)
-!372 = !MDGlobalVariable(name: "comz5", line: 47, isLocal: true, isDefinition: true, scope: null, file: !300, type: !20)
-!373 = !MDGlobalVariable(name: "comz4", line: 47, isLocal: true, isDefinition: true, scope: null, file: !300, type: !20)
-!374 = !MDGlobalVariable(name: "comz1", line: 47, isLocal: true, isDefinition: true, scope: null, file: !300, type: !20)
-!375 = !MDGlobalVariable(name: "dtdssp", line: 45, isLocal: true, isDefinition: true, scope: null, file: !300, type: !20)
-!376 = !MDGlobalVariable(name: "c2dttz1", line: 47, isLocal: true, isDefinition: true, scope: null, file: !300, type: !20)
-!377 = !MDGlobalVariable(name: "c2dtty1", line: 47, isLocal: true, isDefinition: true, scope: null, file: !300, type: !20)
-!378 = !MDGlobalVariable(name: "c2dttx1", line: 47, isLocal: true, isDefinition: true, scope: null, file: !300, type: !20)
-!379 = !MDGlobalVariable(name: "dttz2", line: 46, isLocal: true, isDefinition: true, scope: null, file: !300, type: !20)
-!380 = !MDGlobalVariable(name: "dttz1", line: 46, isLocal: true, isDefinition: true, scope: null, file: !300, type: !20)
-!381 = !MDGlobalVariable(name: "dtty2", line: 46, isLocal: true, isDefinition: true, scope: null, file: !300, type: !20)
-!382 = !MDGlobalVariable(name: "dtty1", line: 46, isLocal: true, isDefinition: true, scope: null, file: !300, type: !20)
-!383 = !MDGlobalVariable(name: "dttx2", line: 46, isLocal: true, isDefinition: true, scope: null, file: !300, type: !20)
-!384 = !MDGlobalVariable(name: "dttx1", line: 46, isLocal: true, isDefinition: true, scope: null, file: !300, type: !20)
-!385 = !MDGlobalVariable(name: "c5dssp", line: 45, isLocal: true, isDefinition: true, scope: null, file: !300, type: !20)
-!386 = !MDGlobalVariable(name: "c4dssp", line: 45, isLocal: true, isDefinition: true, scope: null, file: !300, type: !20)
-!387 = !MDGlobalVariable(name: "dzmax", line: 37, isLocal: true, isDefinition: true, scope: null, file: !300, type: !20)
-!388 = !MDGlobalVariable(name: "dymax", line: 37, isLocal: true, isDefinition: true, scope: null, file: !300, type: !20)
-!389 = !MDGlobalVariable(name: "dxmax", line: 37, isLocal: true, isDefinition: true, scope: null, file: !300, type: !20)
-!390 = !MDGlobalVariable(name: "dz5", line: 34, isLocal: true, isDefinition: true, scope: null, file: !300, type: !20)
-!391 = !MDGlobalVariable(name: "dz4", line: 34, isLocal: true, isDefinition: true, scope: null, file: !300, type: !20)
-!392 = !MDGlobalVariable(name: "dz3", line: 34, isLocal: true, isDefinition: true, scope: null, file: !300, type: !20)
-!393 = !MDGlobalVariable(name: "dz2", line: 34, isLocal: true, isDefinition: true, scope: null, file: !300, type: !20)
-!394 = !MDGlobalVariable(name: "dz1", line: 34, isLocal: true, isDefinition: true, scope: null, file: !300, type: !20)
-!395 = !MDGlobalVariable(name: "dy5", line: 33, isLocal: true, isDefinition: true, scope: null, file: !300, type: !20)
-!396 = !MDGlobalVariable(name: "dy4", line: 33, isLocal: true, isDefinition: true, scope: null, file: !300, type: !20)
-!397 = !MDGlobalVariable(name: "dy3", line: 33, isLocal: true, isDefinition: true, scope: null, file: !300, type: !20)
-!398 = !MDGlobalVariable(name: "dy2", line: 33, isLocal: true, isDefinition: true, scope: null, file: !300, type: !20)
-!399 = !MDGlobalVariable(name: "dy1", line: 33, isLocal: true, isDefinition: true, scope: null, file: !300, type: !20)
-!400 = !MDGlobalVariable(name: "dx5", line: 32, isLocal: true, isDefinition: true, scope: null, file: !300, type: !20)
-!401 = !MDGlobalVariable(name: "dx4", line: 32, isLocal: true, isDefinition: true, scope: null, file: !300, type: !20)
-!402 = !MDGlobalVariable(name: "dx3", line: 32, isLocal: true, isDefinition: true, scope: null, file: !300, type: !20)
-!403 = !MDGlobalVariable(name: "dx2", line: 32, isLocal: true, isDefinition: true, scope: null, file: !300, type: !20)
-!404 = !MDGlobalVariable(name: "dx1", line: 32, isLocal: true, isDefinition: true, scope: null, file: !300, type: !20)
-!405 = !MDGlobalVariable(name: "tz3", line: 31, isLocal: true, isDefinition: true, scope: null, file: !300, type: !20)
-!406 = !MDGlobalVariable(name: "tz1", line: 31, isLocal: true, isDefinition: true, scope: null, file: !300, type: !20)
-!407 = !MDGlobalVariable(name: "ty3", line: 31, isLocal: true, isDefinition: true, scope: null, file: !300, type: !20)
-!408 = !MDGlobalVariable(name: "ty1", line: 31, isLocal: true, isDefinition: true, scope: null, file: !300, type: !20)
-!409 = !MDGlobalVariable(name: "tx3", line: 31, isLocal: true, isDefinition: true, scope: null, file: !300, type: !20)
-!410 = !MDGlobalVariable(name: "tx1", line: 31, isLocal: true, isDefinition: true, scope: null, file: !300, type: !20)
-!411 = !MDGlobalVariable(name: "conz1", line: 45, isLocal: true, isDefinition: true, scope: null, file: !300, type: !20)
-!412 = !MDGlobalVariable(name: "c1345", line: 44, isLocal: true, isDefinition: true, scope: null, file: !300, type: !20)
-!413 = !MDGlobalVariable(name: "c3c4", line: 44, isLocal: true, isDefinition: true, scope: null, file: !300, type: !20)
-!414 = !MDGlobalVariable(name: "c1c5", line: 44, isLocal: true, isDefinition: true, scope: null, file: !300, type: !20)
-!415 = !MDGlobalVariable(name: "c1c2", line: 44, isLocal: true, isDefinition: true, scope: null, file: !300, type: !20)
-!416 = !MDGlobalVariable(name: "c5", line: 45, isLocal: true, isDefinition: true, scope: null, file: !300, type: !20)
-!417 = !MDGlobalVariable(name: "c4", line: 45, isLocal: true, isDefinition: true, scope: null, file: !300, type: !20)
-!418 = !MDGlobalVariable(name: "c3", line: 45, isLocal: true, isDefinition: true, scope: null, file: !300, type: !20)
-!419 = !MDGlobalVariable(name: "lhs", line: 69, isLocal: true, isDefinition: true, scope: null, file: !300, type: !420)
-!420 = !MDCompositeType(tag: DW_TAG_array_type, size: 20787585600, align: 64, baseType: !20, elements: !421)
+!359 = !DISubrange(count: 13)
+!360 = !DIGlobalVariable(name: "dnzm1", line: 44, isLocal: true, isDefinition: true, scope: null, file: !300, type: !20)
+!361 = !DIGlobalVariable(name: "dnym1", line: 44, isLocal: true, isDefinition: true, scope: null, file: !300, type: !20)
+!362 = !DIGlobalVariable(name: "dnxm1", line: 44, isLocal: true, isDefinition: true, scope: null, file: !300, type: !20)
+!363 = !DIGlobalVariable(name: "zzcon1", line: 42, isLocal: true, isDefinition: true, scope: null, file: !300, type: !20)
+!364 = !DIGlobalVariable(name: "yycon1", line: 40, isLocal: true, isDefinition: true, scope: null, file: !300, type: !20)
+!365 = !DIGlobalVariable(name: "xxcon1", line: 38, isLocal: true, isDefinition: true, scope: null, file: !300, type: !20)
+!366 = !DIGlobalVariable(name: "con16", line: 48, isLocal: true, isDefinition: true, scope: null, file: !300, type: !20)
+!367 = !DIGlobalVariable(name: "c2iv", line: 48, isLocal: true, isDefinition: true, scope: null, file: !300, type: !20)
+!368 = !DIGlobalVariable(name: "c3c4tz3", line: 48, isLocal: true, isDefinition: true, scope: null, file: !300, type: !20)
+!369 = !DIGlobalVariable(name: "c3c4ty3", line: 48, isLocal: true, isDefinition: true, scope: null, file: !300, type: !20)
+!370 = !DIGlobalVariable(name: "c3c4tx3", line: 48, isLocal: true, isDefinition: true, scope: null, file: !300, type: !20)
+!371 = !DIGlobalVariable(name: "comz6", line: 47, isLocal: true, isDefinition: true, scope: null, file: !300, type: !20)
+!372 = !DIGlobalVariable(name: "comz5", line: 47, isLocal: true, isDefinition: true, scope: null, file: !300, type: !20)
+!373 = !DIGlobalVariable(name: "comz4", line: 47, isLocal: true, isDefinition: true, scope: null, file: !300, type: !20)
+!374 = !DIGlobalVariable(name: "comz1", line: 47, isLocal: true, isDefinition: true, scope: null, file: !300, type: !20)
+!375 = !DIGlobalVariable(name: "dtdssp", line: 45, isLocal: true, isDefinition: true, scope: null, file: !300, type: !20)
+!376 = !DIGlobalVariable(name: "c2dttz1", line: 47, isLocal: true, isDefinition: true, scope: null, file: !300, type: !20)
+!377 = !DIGlobalVariable(name: "c2dtty1", line: 47, isLocal: true, isDefinition: true, scope: null, file: !300, type: !20)
+!378 = !DIGlobalVariable(name: "c2dttx1", line: 47, isLocal: true, isDefinition: true, scope: null, file: !300, type: !20)
+!379 = !DIGlobalVariable(name: "dttz2", line: 46, isLocal: true, isDefinition: true, scope: null, file: !300, type: !20)
+!380 = !DIGlobalVariable(name: "dttz1", line: 46, isLocal: true, isDefinition: true, scope: null, file: !300, type: !20)
+!381 = !DIGlobalVariable(name: "dtty2", line: 46, isLocal: true, isDefinition: true, scope: null, file: !300, type: !20)
+!382 = !DIGlobalVariable(name: "dtty1", line: 46, isLocal: true, isDefinition: true, scope: null, file: !300, type: !20)
+!383 = !DIGlobalVariable(name: "dttx2", line: 46, isLocal: true, isDefinition: true, scope: null, file: !300, type: !20)
+!384 = !DIGlobalVariable(name: "dttx1", line: 46, isLocal: true, isDefinition: true, scope: null, file: !300, type: !20)
+!385 = !DIGlobalVariable(name: "c5dssp", line: 45, isLocal: true, isDefinition: true, scope: null, file: !300, type: !20)
+!386 = !DIGlobalVariable(name: "c4dssp", line: 45, isLocal: true, isDefinition: true, scope: null, file: !300, type: !20)
+!387 = !DIGlobalVariable(name: "dzmax", line: 37, isLocal: true, isDefinition: true, scope: null, file: !300, type: !20)
+!388 = !DIGlobalVariable(name: "dymax", line: 37, isLocal: true, isDefinition: true, scope: null, file: !300, type: !20)
+!389 = !DIGlobalVariable(name: "dxmax", line: 37, isLocal: true, isDefinition: true, scope: null, file: !300, type: !20)
+!390 = !DIGlobalVariable(name: "dz5", line: 34, isLocal: true, isDefinition: true, scope: null, file: !300, type: !20)
+!391 = !DIGlobalVariable(name: "dz4", line: 34, isLocal: true, isDefinition: true, scope: null, file: !300, type: !20)
+!392 = !DIGlobalVariable(name: "dz3", line: 34, isLocal: true, isDefinition: true, scope: null, file: !300, type: !20)
+!393 = !DIGlobalVariable(name: "dz2", line: 34, isLocal: true, isDefinition: true, scope: null, file: !300, type: !20)
+!394 = !DIGlobalVariable(name: "dz1", line: 34, isLocal: true, isDefinition: true, scope: null, file: !300, type: !20)
+!395 = !DIGlobalVariable(name: "dy5", line: 33, isLocal: true, isDefinition: true, scope: null, file: !300, type: !20)
+!396 = !DIGlobalVariable(name: "dy4", line: 33, isLocal: true, isDefinition: true, scope: null, file: !300, type: !20)
+!397 = !DIGlobalVariable(name: "dy3", line: 33, isLocal: true, isDefinition: true, scope: null, file: !300, type: !20)
+!398 = !DIGlobalVariable(name: "dy2", line: 33, isLocal: true, isDefinition: true, scope: null, file: !300, type: !20)
+!399 = !DIGlobalVariable(name: "dy1", line: 33, isLocal: true, isDefinition: true, scope: null, file: !300, type: !20)
+!400 = !DIGlobalVariable(name: "dx5", line: 32, isLocal: true, isDefinition: true, scope: null, file: !300, type: !20)
+!401 = !DIGlobalVariable(name: "dx4", line: 32, isLocal: true, isDefinition: true, scope: null, file: !300, type: !20)
+!402 = !DIGlobalVariable(name: "dx3", line: 32, isLocal: true, isDefinition: true, scope: null, file: !300, type: !20)
+!403 = !DIGlobalVariable(name: "dx2", line: 32, isLocal: true, isDefinition: true, scope: null, file: !300, type: !20)
+!404 = !DIGlobalVariable(name: "dx1", line: 32, isLocal: true, isDefinition: true, scope: null, file: !300, type: !20)
+!405 = !DIGlobalVariable(name: "tz3", line: 31, isLocal: true, isDefinition: true, scope: null, file: !300, type: !20)
+!406 = !DIGlobalVariable(name: "tz1", line: 31, isLocal: true, isDefinition: true, scope: null, file: !300, type: !20)
+!407 = !DIGlobalVariable(name: "ty3", line: 31, isLocal: true, isDefinition: true, scope: null, file: !300, type: !20)
+!408 = !DIGlobalVariable(name: "ty1", line: 31, isLocal: true, isDefinition: true, scope: null, file: !300, type: !20)
+!409 = !DIGlobalVariable(name: "tx3", line: 31, isLocal: true, isDefinition: true, scope: null, file: !300, type: !20)
+!410 = !DIGlobalVariable(name: "tx1", line: 31, isLocal: true, isDefinition: true, scope: null, file: !300, type: !20)
+!411 = !DIGlobalVariable(name: "conz1", line: 45, isLocal: true, isDefinition: true, scope: null, file: !300, type: !20)
+!412 = !DIGlobalVariable(name: "c1345", line: 44, isLocal: true, isDefinition: true, scope: null, file: !300, type: !20)
+!413 = !DIGlobalVariable(name: "c3c4", line: 44, isLocal: true, isDefinition: true, scope: null, file: !300, type: !20)
+!414 = !DIGlobalVariable(name: "c1c5", line: 44, isLocal: true, isDefinition: true, scope: null, file: !300, type: !20)
+!415 = !DIGlobalVariable(name: "c1c2", line: 44, isLocal: true, isDefinition: true, scope: null, file: !300, type: !20)
+!416 = !DIGlobalVariable(name: "c5", line: 45, isLocal: true, isDefinition: true, scope: null, file: !300, type: !20)
+!417 = !DIGlobalVariable(name: "c4", line: 45, isLocal: true, isDefinition: true, scope: null, file: !300, type: !20)
+!418 = !DIGlobalVariable(name: "c3", line: 45, isLocal: true, isDefinition: true, scope: null, file: !300, type: !20)
+!419 = !DIGlobalVariable(name: "lhs", line: 69, isLocal: true, isDefinition: true, scope: null, file: !300, type: !420)
+!420 = !DICompositeType(tag: DW_TAG_array_type, size: 20787585600, align: 64, baseType: !20, elements: !421)
 !421 = !{!308, !308, !308, !178, !93, !93}
-!422 = !MDGlobalVariable(name: "q", line: 73, isLocal: true, isDefinition: true, scope: null, file: !300, type: !423)
-!423 = !MDCompositeType(tag: DW_TAG_array_type, size: 10368, align: 64, baseType: !20, elements: !424)
+!422 = !DIGlobalVariable(name: "q", line: 73, isLocal: true, isDefinition: true, scope: null, file: !300, type: !423)
+!423 = !DICompositeType(tag: DW_TAG_array_type, size: 10368, align: 64, baseType: !20, elements: !424)
 !424 = !{!425}
-!425 = !MDSubrange(count: 162)
-!426 = !MDGlobalVariable(name: "cuf", line: 72, isLocal: true, isDefinition: true, scope: null, file: !300, type: !423)
-!427 = !MDGlobalVariable(name: "buf", line: 75, isLocal: true, isDefinition: true, scope: null, file: !300, type: !428)
-!428 = !MDCompositeType(tag: DW_TAG_array_type, size: 51840, align: 64, baseType: !20, elements: !429)
+!425 = !DISubrange(count: 162)
+!426 = !DIGlobalVariable(name: "cuf", line: 72, isLocal: true, isDefinition: true, scope: null, file: !300, type: !423)
+!427 = !DIGlobalVariable(name: "buf", line: 75, isLocal: true, isDefinition: true, scope: null, file: !300, type: !428)
+!428 = !DICompositeType(tag: DW_TAG_array_type, size: 51840, align: 64, baseType: !20, elements: !429)
 !429 = !{!425, !93}
-!430 = !MDGlobalVariable(name: "ue", line: 74, isLocal: true, isDefinition: true, scope: null, file: !300, type: !428)
-!431 = !MDGlobalVariable(name: "njac", line: 86, isLocal: true, isDefinition: true, scope: null, file: !300, type: !432)
-!432 = !MDCompositeType(tag: DW_TAG_array_type, size: 6886684800, align: 64, baseType: !20, elements: !433)
+!430 = !DIGlobalVariable(name: "ue", line: 74, isLocal: true, isDefinition: true, scope: null, file: !300, type: !428)
+!431 = !DIGlobalVariable(name: "njac", line: 86, isLocal: true, isDefinition: true, scope: null, file: !300, type: !432)
+!432 = !DICompositeType(tag: DW_TAG_array_type, size: 6886684800, align: 64, baseType: !20, elements: !433)
 !433 = !{!308, !308, !425, !93, !93}
-!434 = !MDGlobalVariable(name: "fjac", line: 84, isLocal: true, isDefinition: true, scope: null, file: !300, type: !432)
-!435 = !MDGlobalVariable(name: "tmp3", line: 88, isLocal: true, isDefinition: true, scope: null, file: !300, type: !20)
-!436 = !MDGlobalVariable(name: "tmp2", line: 88, isLocal: true, isDefinition: true, scope: null, file: !300, type: !20)
-!437 = !MDGlobalVariable(name: "tmp1", line: 88, isLocal: true, isDefinition: true, scope: null, file: !300, type: !20)
+!434 = !DIGlobalVariable(name: "fjac", line: 84, isLocal: true, isDefinition: true, scope: null, file: !300, type: !432)
+!435 = !DIGlobalVariable(name: "tmp3", line: 88, isLocal: true, isDefinition: true, scope: null, file: !300, type: !20)
+!436 = !DIGlobalVariable(name: "tmp2", line: 88, isLocal: true, isDefinition: true, scope: null, file: !300, type: !20)
+!437 = !DIGlobalVariable(name: "tmp1", line: 88, isLocal: true, isDefinition: true, scope: null, file: !300, type: !20)
 !438 = !{i32 2, !"Dwarf Version", i32 4}
-!439 = !MDLocation(line: 1898, scope: !440)
-!440 = distinct !MDLexicalBlock(line: 1898, column: 0, file: !1, scope: !114)
-!441 = !MDLocation(line: 1913, scope: !442)
-!442 = distinct !MDLexicalBlock(line: 1913, column: 0, file: !1, scope: !114)
-!443 = !MDLocation(line: 1923, scope: !114)
+!439 = !DILocation(line: 1898, scope: !440)
+!440 = distinct !DILexicalBlock(line: 1898, column: 0, file: !1, scope: !114)
+!441 = !DILocation(line: 1913, scope: !442)
+!442 = distinct !DILexicalBlock(line: 1913, column: 0, file: !1, scope: !114)
+!443 = !DILocation(line: 1923, scope: !114)
 !444 = !{!"int", !445}
 !445 = !{!"omnipotent char", !446}
 !446 = !{!"Simple C/C++ TBAA"}
 !447 = !{i32 1}
-!448 = !MDLocation(line: 1925, scope: !449)
-!449 = distinct !MDLexicalBlock(line: 1925, column: 0, file: !1, scope: !114)
-!450 = !MDLocation(line: 1939, scope: !451)
-!451 = distinct !MDLexicalBlock(line: 1939, column: 0, file: !1, scope: !114)
-!452 = !MDLocation(line: 1940, scope: !453)
-!453 = distinct !MDLexicalBlock(line: 1940, column: 0, file: !1, scope: !454)
-!454 = distinct !MDLexicalBlock(line: 1939, column: 0, file: !1, scope: !451)
-!455 = !MDLocation(line: 1941, scope: !456)
-!456 = distinct !MDLexicalBlock(line: 1941, column: 0, file: !1, scope: !457)
-!457 = distinct !MDLexicalBlock(line: 1940, column: 0, file: !1, scope: !453)
-!458 = !MDLocation(line: 2020, scope: !459)
-!459 = distinct !MDLexicalBlock(line: 2020, column: 0, file: !1, scope: !460)
-!460 = distinct !MDLexicalBlock(line: 2019, column: 0, file: !1, scope: !461)
-!461 = distinct !MDLexicalBlock(line: 2019, column: 0, file: !1, scope: !462)
-!462 = distinct !MDLexicalBlock(line: 2018, column: 0, file: !1, scope: !463)
-!463 = distinct !MDLexicalBlock(line: 2018, column: 0, file: !1, scope: !114)
+!448 = !DILocation(line: 1925, scope: !449)
+!449 = distinct !DILexicalBlock(line: 1925, column: 0, file: !1, scope: !114)
+!450 = !DILocation(line: 1939, scope: !451)
+!451 = distinct !DILexicalBlock(line: 1939, column: 0, file: !1, scope: !114)
+!452 = !DILocation(line: 1940, scope: !453)
+!453 = distinct !DILexicalBlock(line: 1940, column: 0, file: !1, scope: !454)
+!454 = distinct !DILexicalBlock(line: 1939, column: 0, file: !1, scope: !451)
+!455 = !DILocation(line: 1941, scope: !456)
+!456 = distinct !DILexicalBlock(line: 1941, column: 0, file: !1, scope: !457)
+!457 = distinct !DILexicalBlock(line: 1940, column: 0, file: !1, scope: !453)
+!458 = !DILocation(line: 2020, scope: !459)
+!459 = distinct !DILexicalBlock(line: 2020, column: 0, file: !1, scope: !460)
+!460 = distinct !DILexicalBlock(line: 2019, column: 0, file: !1, scope: !461)
+!461 = distinct !DILexicalBlock(line: 2019, column: 0, file: !1, scope: !462)
+!462 = distinct !DILexicalBlock(line: 2018, column: 0, file: !1, scope: !463)
+!463 = distinct !DILexicalBlock(line: 2018, column: 0, file: !1, scope: !114)
 !464 = !{i32 1, !"Debug Info Version", i32 3}
diff --git a/test/CodeGen/PowerPC/pr18663-2.ll b/test/CodeGen/PowerPC/pr18663-2.ll
index 6b54440c4d5..c77291e5145 100644
--- a/test/CodeGen/PowerPC/pr18663-2.ll
+++ b/test/CodeGen/PowerPC/pr18663-2.ll
@@ -46,7 +46,7 @@ declare void @_ZN4Foam7IOerror4exitEi() #0
 ; Function Attrs: inlinehint
 declare void @_ZN4Foam8fileName12stripInvalidEv() #2 align 2
 
-define void @_ZN4Foam3CSVINS_6VectorIdEEE4readEv() #0 align 2 {
+define void @_ZN4Foam3CSVINS_6VectorIdEEE4readEv() #0 align 2 personality i8* bitcast (i32 (...)* @__gxx_personality_v0 to i8*) {
 entry:
   invoke void @_ZN4Foam6string6expandEb()
           to label %invoke.cont unwind label %lpad
@@ -66,7 +66,7 @@ _ZN4Foam6stringC2ERKS0_.exit.i:                   ; preds = %invoke.cont
           to label %invoke.cont2 unwind label %lpad.i
 
 lpad.i:                                           ; preds = %_ZN4Foam6stringC2ERKS0_.exit.i
-  %0 = landingpad { i8*, i32 } personality i8* bitcast (i32 (...)* @__gxx_personality_v0 to i8*)
+  %0 = landingpad { i8*, i32 }
           cleanup
   br label %ehcleanup142
 
@@ -90,17 +90,17 @@ memptr.end.i:                                     ; preds = %invoke.cont8
           to label %if.end unwind label %lpad5
 
 lpad:                                             ; preds = %if.then.i.i.i.i176, %entry
-  %1 = landingpad { i8*, i32 } personality i8* bitcast (i32 (...)* @__gxx_personality_v0 to i8*)
+  %1 = landingpad { i8*, i32 }
           cleanup
   br label %ehcleanup142
 
 lpad3:                                            ; preds = %invoke.cont2
-  %2 = landingpad { i8*, i32 } personality i8* bitcast (i32 (...)* @__gxx_personality_v0 to i8*)
+  %2 = landingpad { i8*, i32 }
           cleanup
   br label %ehcleanup142
 
 lpad5:                                            ; preds = %memptr.end.i, %invoke.cont8, %if.then
-  %3 = landingpad { i8*, i32 } personality i8* bitcast (i32 (...)* @__gxx_personality_v0 to i8*)
+  %3 = landingpad { i8*, i32 }
           cleanup
   br label %ehcleanup142
 
@@ -119,12 +119,12 @@ invoke.cont.i.i.i:                                ; preds = %.noexc205
   unreachable
 
 lpad.i.i.i:                                       ; preds = %.noexc205
-  %4 = landingpad { i8*, i32 } personality i8* bitcast (i32 (...)* @__gxx_personality_v0 to i8*)
+  %4 = landingpad { i8*, i32 }
           cleanup
   br label %ehcleanup142
 
 lpad19:                                           ; preds = %for.body
-  %5 = landingpad { i8*, i32 } personality i8* bitcast (i32 (...)* @__gxx_personality_v0 to i8*)
+  %5 = landingpad { i8*, i32 }
           cleanup
   br label %ehcleanup142
 
diff --git a/test/CodeGen/PowerPC/pr24216.ll b/test/CodeGen/PowerPC/pr24216.ll
new file mode 100644
index 00000000000..4ab41985f5b
--- /dev/null
+++ b/test/CodeGen/PowerPC/pr24216.ll
@@ -0,0 +1,14 @@
+; RUN: llc -mcpu=pwr8 -mtriple=powerpc64le-unknown-linux-gnu < %s | FileCheck %s
+
+; Test case adapted from PR24216.
+
+define void @foo(<16 x i8>* nocapture readonly %in, <16 x i8>* nocapture %out) {
+entry:
+  %0 = load <16 x i8>, <16 x i8>* %in, align 16
+  %1 = shufflevector <16 x i8> %0, <16 x i8> undef, <16 x i32> <i32 2, i32 3, i32 4, i32 5, i32 2, i32 3, i32 4, i32 5, i32 2, i32 3, i32 4, i32 5, i32 2, i32 3, i32 4, i32 5>
+  store <16 x i8> %1, <16 x i8>* %out, align 16
+  ret void
+}
+
+; CHECK: vperm
+; CHECK-NOT: vspltw
diff --git a/test/CodeGen/PowerPC/preinc-ld-sel-crash.ll b/test/CodeGen/PowerPC/preinc-ld-sel-crash.ll
new file mode 100644
index 00000000000..cb2f035e566
--- /dev/null
+++ b/test/CodeGen/PowerPC/preinc-ld-sel-crash.ll
@@ -0,0 +1,63 @@
+; RUN: llc < %s | FileCheck %s
+target datalayout = "E-m:e-i64:64-n32:64"
+target triple = "powerpc64-bgq-linux"
+
+%t1 = type { %t2*, %t3* }
+%t2 = type <{ %t3*, i32, [4 x i8] }>
+%t3 = type { %t3* }
+
+@_ZN4Foam10SLListBase13endConstIter_E = external global %t1
+
+define void @_ZN4FoamrsIbEERNS_7IstreamES2_RNS_4ListIT_EE() #0 {
+entry:
+  switch i32 undef, label %if.else82 [
+    i32 9, label %if.then
+    i32 6, label %invoke.cont10
+    i32 1, label %invoke.cont61
+  ]
+
+if.then:                                          ; preds = %entry
+  unreachable
+
+invoke.cont10:                                    ; preds = %entry
+  unreachable
+
+invoke.cont61:                                    ; preds = %entry
+  br i1 undef, label %if.end75, label %if.then64
+
+if.then64:                                        ; preds = %invoke.cont61
+  unreachable
+
+if.end75:                                         ; preds = %invoke.cont61
+  br i1 undef, label %if.then17.i, label %if.then.i181
+
+if.then.i181:                                     ; preds = %if.end75
+  unreachable
+
+if.then17.i:                                      ; preds = %if.end75
+  %tobool.i.i.i = icmp eq i32 undef, 0
+  %0 = load i64*, i64** undef, align 8
+  %agg.tmp.sroa.3.0.copyload33.in.i = select i1 %tobool.i.i.i, i64* bitcast (%t3** getelementptr inbounds (%t1, %t1* @_ZN4Foam10SLListBase13endConstIter_E, i64 0, i32 1) to i64*), i64* %0
+  %agg.tmp.sroa.3.0.copyload33.i = load i64, i64* %agg.tmp.sroa.3.0.copyload33.in.i, align 8
+  %1 = inttoptr i64 %agg.tmp.sroa.3.0.copyload33.i to %t3*
+  %2 = load %t3*, %t3** getelementptr inbounds (%t1, %t1* @_ZN4Foam10SLListBase13endConstIter_E, i64 0, i32 1), align 8
+  %cmp.i37.i = icmp eq %t3* %1, %2
+  br i1 %cmp.i37.i, label %invoke.cont79, label %for.body.lr.ph.i
+
+; CHECK-LABEL: @_ZN4FoamrsIbEERNS_7IstreamES2_RNS_4ListIT_EE
+
+for.body.lr.ph.i:                                 ; preds = %if.then17.i
+  br label %for.body.i
+
+for.body.i:                                       ; preds = %for.body.i, %for.body.lr.ph.i
+  br i1 undef, label %invoke.cont79, label %for.body.i
+
+invoke.cont79:                                    ; preds = %for.body.i, %if.then17.i
+  unreachable
+
+if.else82:                                        ; preds = %entry
+  ret void
+}
+
+attributes #0 = { "target-cpu"="a2q" }
+
diff --git a/test/CodeGen/PowerPC/preincprep-invoke.ll b/test/CodeGen/PowerPC/preincprep-invoke.ll
index 0e09ff1b774..8dbce9a3a08 100644
--- a/test/CodeGen/PowerPC/preincprep-invoke.ll
+++ b/test/CodeGen/PowerPC/preincprep-invoke.ll
@@ -11,7 +11,7 @@ declare void @_ZN13CStdOutStream5FlushEv()
 
 declare i32 @__gxx_personality_v0(...)
 
-define void @_Z11GetPasswordP13CStdOutStreamb() {
+define void @_Z11GetPasswordP13CStdOutStreamb() personality i8* bitcast (i32 (...)* @__gxx_personality_v0 to i8*) {
 entry:
   br label %for.cond.i.i
 
@@ -41,7 +41,7 @@ for.cond.i.i30:                                   ; preds = %for.cond.i.i30, %in
   br label %for.cond.i.i30
 
 lpad:                                             ; preds = %invoke.cont4, %invoke.cont, %_ZN11CStringBaseIcEC2EPKc.exit.critedge
-  %1 = landingpad { i8*, i32 } personality i8* bitcast (i32 (...)* @__gxx_personality_v0 to i8*)
+  %1 = landingpad { i8*, i32 }
           cleanup
   resume { i8*, i32 } undef
 }
diff --git a/test/CodeGen/PowerPC/recipest.ll b/test/CodeGen/PowerPC/recipest.ll
index cd77548b281..41dcb0f5b3f 100644
--- a/test/CodeGen/PowerPC/recipest.ll
+++ b/test/CodeGen/PowerPC/recipest.ll
@@ -1,4 +1,5 @@
 ; RUN: llc < %s -mtriple=powerpc64-unknown-linux-gnu -mcpu=pwr7 -enable-unsafe-fp-math -mattr=-vsx | FileCheck %s
+; RUN: llc < %s -mtriple=powerpc64-unknown-linux-gnu -mcpu=pwr7 -enable-unsafe-fp-math -mattr=-vsx -recip=sqrtf:0,sqrtd:0 | FileCheck %s -check-prefix=CHECK-NONR
 ; RUN: llc < %s -mtriple=powerpc64-unknown-linux-gnu -mcpu=pwr7 -mattr=-vsx | FileCheck -check-prefix=CHECK-SAFE %s
 target datalayout = "E-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-f128:128:128-v128:128:128-n32:64"
 target triple = "powerpc64-unknown-linux-gnu"
@@ -24,6 +25,13 @@ define double @foo(double %a, double %b) nounwind {
 ; CHECK-NEXT: fmul
 ; CHECK: blr
 
+; CHECK-NONR: @foo
+; CHECK-NONR: frsqrte
+; CHECK-NONR-NOT: fmadd
+; CHECK-NONR: fmul
+; CHECK-NONR-NOT: fmadd
+; CHECK-NONR: blr
+
 ; CHECK-SAFE: @foo
 ; CHECK-SAFE: fsqrt
 ; CHECK-SAFE: fdiv
@@ -90,6 +98,13 @@ define float @goo(float %a, float %b) nounwind {
 ; CHECK-NEXT: fmuls
 ; CHECK-NEXT: blr
 
+; CHECK-NONR: @goo
+; CHECK-NONR: frsqrtes
+; CHECK-NONR-NOT: fmadds
+; CHECK-NONR: fmuls
+; CHECK-NONR-NOT: fmadds
+; CHECK-NONR: blr
+
 ; CHECK-SAFE: @goo
 ; CHECK-SAFE: fsqrts
 ; CHECK-SAFE: fdivs
diff --git a/test/CodeGen/PowerPC/rlwimi-and.ll b/test/CodeGen/PowerPC/rlwimi-and.ll
index b9b0c915a1d..59f704ee16b 100644
--- a/test/CodeGen/PowerPC/rlwimi-and.ll
+++ b/test/CodeGen/PowerPC/rlwimi-and.ll
@@ -29,7 +29,7 @@ codeRepl17:                                       ; preds = %codeRepl4
   unreachable
 
 ; CHECK: @test
-; CHECK: rlwinm [[R1:[0-9]+]], {{[0-9]+}}, 0, 31, 31
+; CHECK: clrlwi [[R1:[0-9]+]], {{[0-9]+}}, 31
 ; CHECK: rlwimi [[R1]], {{[0-9]+}}, 8, 23, 23
 
 codeRepl29:                                       ; preds = %codeRepl1
diff --git a/test/CodeGen/PowerPC/rotl-2.ll b/test/CodeGen/PowerPC/rotl-2.ll
index d32ef59be6c..86539b6c119 100644
--- a/test/CodeGen/PowerPC/rotl-2.ll
+++ b/test/CodeGen/PowerPC/rotl-2.ll
@@ -1,5 +1,6 @@
-; RUN: llc < %s -march=ppc32  | grep rlwinm | count 4
-; RUN: llc < %s -march=ppc32  | grep rlwnm | count 2
+; RUN: llc < %s -march=ppc32  | grep rotlwi | count 2
+; RUN: llc < %s -march=ppc32  | grep clrlwi | count 2
+; RUN: llc < %s -march=ppc32  | grep rotlw | count 4
 ; RUN: llc < %s -march=ppc32  | not grep or
 
 define i32 @rotl32(i32 %A, i8 %Amt) nounwind {
diff --git a/test/CodeGen/PowerPC/rotl-64.ll b/test/CodeGen/PowerPC/rotl-64.ll
index 674c9e4cc95..2ccdc29f2cd 100644
--- a/test/CodeGen/PowerPC/rotl-64.ll
+++ b/test/CodeGen/PowerPC/rotl-64.ll
@@ -1,5 +1,5 @@
-; RUN: llc < %s -march=ppc64 | grep rldicl
-; RUN: llc < %s -march=ppc64 | grep rldcl
+; RUN: llc < %s -march=ppc64 | grep rotld
+; RUN: llc < %s -march=ppc64 | grep rotldi
 ; PR1613
 
 define i64 @t1(i64 %A) {
diff --git a/test/CodeGen/PowerPC/rotl.ll b/test/CodeGen/PowerPC/rotl.ll
index 56fc4a8c911..671f524645a 100644
--- a/test/CodeGen/PowerPC/rotl.ll
+++ b/test/CodeGen/PowerPC/rotl.ll
@@ -1,5 +1,7 @@
-; RUN: llc < %s -march=ppc32 | grep rlwnm | count 2
-; RUN: llc < %s -march=ppc32 | grep rlwinm | count 2
+; RUN: llc < %s -march=ppc32 | grep rotrw: | count 1
+; RUN: llc < %s -march=ppc32 | grep rotlw: | count 1
+; RUN: llc < %s -march=ppc32 | grep rotlwi: | count 1
+; RUN: llc < %s -march=ppc32 | grep rotrwi: | count 1
 
 define i32 @rotlw(i32 %x, i32 %sh) {
 entry:
diff --git a/test/CodeGen/PowerPC/sdag-ppcf128.ll b/test/CodeGen/PowerPC/sdag-ppcf128.ll
index c46bc6b22dd..6d2a04c7293 100644
--- a/test/CodeGen/PowerPC/sdag-ppcf128.ll
+++ b/test/CodeGen/PowerPC/sdag-ppcf128.ll
@@ -5,7 +5,7 @@
 define fastcc void @_D3std4math4sqrtFNaNbNfcZc() {
 entry:
   br i1 undef, label %if, label %else
-; CHECK: cmplwi 0, 3, 0
+; CHECK: cmplwi 3, 0
 if:                                               ; preds = %entry
   store { ppc_fp128, ppc_fp128 } zeroinitializer, { ppc_fp128, ppc_fp128 }* undef
   ret void
diff --git a/test/CodeGen/PowerPC/seteq-0.ll b/test/CodeGen/PowerPC/seteq-0.ll
index b7dd78085eb..4afb8fee177 100644
--- a/test/CodeGen/PowerPC/seteq-0.ll
+++ b/test/CodeGen/PowerPC/seteq-0.ll
@@ -5,7 +5,7 @@ define i32 @eq0(i32 %a) {
         %tmp.2 = zext i1 %tmp.1 to i32          ; <i32> [#uses=1]
         ret i32 %tmp.2
 
-; CHECK: cntlzw [[REG:r[0-9]+]], r3
+; CHECK: cntlz [[REG:r[0-9]+]], r3
 ; CHECK: rlwinm r3, [[REG]], 27, 31, 31
 ; CHECK: blr
 }
diff --git a/test/CodeGen/PowerPC/sjlj.ll b/test/CodeGen/PowerPC/sjlj.ll
index 62403e71196..dcbdd69d5d5 100644
--- a/test/CodeGen/PowerPC/sjlj.ll
+++ b/test/CodeGen/PowerPC/sjlj.ll
@@ -18,10 +18,10 @@ entry:
 ; CHECK: addi [[REG]], [[REG]], env_sigill@toc@l
 ; CHECK: ld 31, 0([[REG]])
 ; CHECK: ld [[REG2:[0-9]+]], 8([[REG]])
-; CHECK: ld 1, 16([[REG]])
-; CHECK: mtctr [[REG2]]
-; CHECK: ld 30, 32([[REG]])
-; CHECK: ld 2, 24([[REG]])
+; CHECK-DAG: ld 1, 16([[REG]])
+; CHECK-DAG: mtctr [[REG2]]
+; CHECK-DAG: ld 30, 32([[REG]])
+; CHECK-DAG: ld 2, 24([[REG]])
 ; CHECK: bctr
 
 return:                                           ; No predecessors!
diff --git a/test/CodeGen/PowerPC/stack-realign.ll b/test/CodeGen/PowerPC/stack-realign.ll
index 00aee56199b..e91b563af72 100644
--- a/test/CodeGen/PowerPC/stack-realign.ll
+++ b/test/CodeGen/PowerPC/stack-realign.ll
@@ -30,7 +30,7 @@ entry:
 ; CHECK-LABEL: @goo
 
 ; CHECK-DAG: mflr 0
-; CHECK-DAG: rldicl [[REG:[0-9]+]], 1, 0, 59
+; CHECK-DAG: clrldi [[REG:[0-9]+]], 1, 59
 ; CHECK-DAG: std 30, -16(1)
 ; CHECK-DAG: mr 30, 1
 ; CHECK-DAG: std 0, 16(1)
@@ -52,7 +52,7 @@ entry:
 ; CHECK-FP-LABEL: @goo
 
 ; CHECK-FP-DAG: mflr 0
-; CHECK-FP-DAG: rldicl [[REG:[0-9]+]], 1, 0, 59
+; CHECK-FP-DAG: clrldi [[REG:[0-9]+]], 1, 59
 ; CHECK-FP-DAG: std 31, -8(1)
 ; CHECK-FP-DAG: std 30, -16(1)
 ; CHECK-FP-DAG: mr 30, 1
@@ -78,7 +78,7 @@ entry:
 
 ; CHECK-32-LABEL: @goo
 ; CHECK-32-DAG: mflr 0
-; CHECK-32-DAG: rlwinm [[REG:[0-9]+]], 1, 0, 27, 31
+; CHECK-32-DAG: clrlwi [[REG:[0-9]+]], 1, 27
 ; CHECK-32-DAG: stw 30, -8(1)
 ; CHECK-32-DAG: mr 30, 1
 ; CHECK-32-DAG: stw 0, 4(1)
@@ -87,7 +87,7 @@ entry:
 
 ; CHECK-32-PIC-LABEL: @goo
 ; CHECK-32-PIC-DAG: mflr 0
-; CHECK-32-PIC-DAG: rlwinm [[REG:[0-9]+]], 1, 0, 27, 31
+; CHECK-32-PIC-DAG: clrlwi [[REG:[0-9]+]], 1, 27
 ; CHECK-32-PIC-DAG: stw 29, -12(1)
 ; CHECK-32-PIC-DAG: mr 29, 1
 ; CHECK-32-PIC-DAG: stw 0, 4(1)
@@ -113,7 +113,7 @@ entry:
 ; CHECK-LABEL: @hoo
 
 ; CHECK-DAG: lis [[REG1:[0-9]+]], -13
-; CHECK-DAG: rldicl [[REG3:[0-9]+]], 1, 0, 59
+; CHECK-DAG: clrldi [[REG3:[0-9]+]], 1, 59
 ; CHECK-DAG: mflr 0
 ; CHECK-DAG: ori [[REG2:[0-9]+]], [[REG1]], 51808
 ; CHECK-DAG: std 30, -16(1)
@@ -129,7 +129,7 @@ entry:
 ; CHECK-32-LABEL: @hoo
 
 ; CHECK-32-DAG: lis [[REG1:[0-9]+]], -13
-; CHECK-32-DAG: rlwinm [[REG3:[0-9]+]], 1, 0, 27, 31
+; CHECK-32-DAG: clrlwi [[REG3:[0-9]+]], 1, 27
 ; CHECK-32-DAG: mflr 0
 ; CHECK-32-DAG: ori [[REG2:[0-9]+]], [[REG1]], 51904
 ; CHECK-32-DAG: stw 30, -8(1)
@@ -143,7 +143,7 @@ entry:
 ; CHECK-32-PIC-LABEL: @hoo
 
 ; CHECK-32-PIC-DAG: lis [[REG1:[0-9]+]], -13
-; CHECK-32-PIC-DAG: rlwinm [[REG3:[0-9]+]], 1, 0, 27, 31
+; CHECK-32-PIC-DAG: clrlwi [[REG3:[0-9]+]], 1, 27
 ; CHECK-32-PIC-DAG: mflr 0
 ; CHECK-32-PIC-DAG: ori [[REG2:[0-9]+]], [[REG1]], 51904
 ; CHECK-32-PIC-DAG: stw 29, -12(1)
@@ -175,7 +175,7 @@ entry:
 ; CHECK-LABEL: @loo
 
 ; CHECK-DAG: mflr 0
-; CHECK-DAG: rldicl [[REG:[0-9]+]], 1, 0, 59
+; CHECK-DAG: clrldi [[REG:[0-9]+]], 1, 59
 ; CHECK-DAG: std 30, -32(1)
 ; CHECK-DAG: mr 30, 1
 ; CHECK-DAG: std 0, 16(1)
@@ -191,7 +191,7 @@ entry:
 ; CHECK-FP-LABEL: @loo
 
 ; CHECK-FP-DAG: mflr 0
-; CHECK-FP-DAG: rldicl [[REG:[0-9]+]], 1, 0, 59
+; CHECK-FP-DAG: clrldi [[REG:[0-9]+]], 1, 59
 ; CHECK-FP-DAG: std 31, -24(1)
 ; CHECK-FP-DAG: std 30, -32(1)
 ; CHECK-FP-DAG: mr 30, 1
diff --git a/test/CodeGen/PowerPC/subreg-postra-2.ll b/test/CodeGen/PowerPC/subreg-postra-2.ll
index 893e4b9e8ae..05153644341 100644
--- a/test/CodeGen/PowerPC/subreg-postra-2.ll
+++ b/test/CodeGen/PowerPC/subreg-postra-2.ll
@@ -160,7 +160,7 @@ while.end418:                                     ; preds = %wait_on_buffer.exit
 
 ; CHECK-LABEL: @jbd2_journal_commit_transaction
 ; CHECK: andi.
-; CHECK: cror [[REG:[0-9]+]], 1, 1
+; CHECK: crmove [[REG:[0-9]+]], 1
 ; CHECK: stdcx.
 ; CHECK: isel {{[0-9]+}}, {{[0-9]+}}, {{[0-9]+}}, [[REG]]
 
diff --git a/test/CodeGen/PowerPC/subreg-postra.ll b/test/CodeGen/PowerPC/subreg-postra.ll
index fa5fd7e1c76..ba1b967cf20 100644
--- a/test/CodeGen/PowerPC/subreg-postra.ll
+++ b/test/CodeGen/PowerPC/subreg-postra.ll
@@ -143,7 +143,7 @@ wait_on_buffer.exit1319:                          ; preds = %while.body392
 
 ; CHECK-LABEL: @jbd2_journal_commit_transaction
 ; CHECK: andi.
-; CHECK: cror [[REG:[0-9]+]], 1, 1
+; CHECK: crmove [[REG:[0-9]+]], 1
 ; CHECK: stdcx.
 ; CHECK: isel {{[0-9]+}}, {{[0-9]+}}, {{[0-9]+}}, [[REG]]
 
diff --git a/test/CodeGen/PowerPC/swaps-le-1.ll b/test/CodeGen/PowerPC/swaps-le-1.ll
new file mode 100644
index 00000000000..0c416316903
--- /dev/null
+++ b/test/CodeGen/PowerPC/swaps-le-1.ll
@@ -0,0 +1,147 @@
+; RUN: llc -O3 -mcpu=pwr8 -mtriple=powerpc64le-unknown-linux-gnu < %s | FileCheck %s
+; RUN: llc -O3 -mcpu=pwr8 -disable-ppc-vsx-swap-removal -mtriple=powerpc64le-unknown-linux-gnu < %s | FileCheck -check-prefix=NOOPTSWAP %s
+
+; This test was generated from the following source:
+;
+; #define N 4096
+; int ca[N] __attribute__((aligned(16)));
+; int cb[N] __attribute__((aligned(16)));
+; int cc[N] __attribute__((aligned(16)));
+; int cd[N] __attribute__((aligned(16)));
+;
+; void foo ()
+; {
+;   int i;
+;   for (i = 0; i < N; i++) {
+;     ca[i] = (cb[i] + cc[i]) * cd[i];
+;   }
+; }
+
+@cb = common global [4096 x i32] zeroinitializer, align 16
+@cc = common global [4096 x i32] zeroinitializer, align 16
+@cd = common global [4096 x i32] zeroinitializer, align 16
+@ca = common global [4096 x i32] zeroinitializer, align 16
+
+define void @foo() {
+entry:
+  br label %vector.body
+
+vector.body:
+  %index = phi i64 [ 0, %entry ], [ %index.next.3, %vector.body ]
+  %0 = getelementptr inbounds [4096 x i32], [4096 x i32]* @cb, i64 0, i64 %index
+  %1 = bitcast i32* %0 to <4 x i32>*
+  %wide.load = load <4 x i32>, <4 x i32>* %1, align 16
+  %2 = getelementptr inbounds [4096 x i32], [4096 x i32]* @cc, i64 0, i64 %index
+  %3 = bitcast i32* %2 to <4 x i32>*
+  %wide.load13 = load <4 x i32>, <4 x i32>* %3, align 16
+  %4 = add nsw <4 x i32> %wide.load13, %wide.load
+  %5 = getelementptr inbounds [4096 x i32], [4096 x i32]* @cd, i64 0, i64 %index
+  %6 = bitcast i32* %5 to <4 x i32>*
+  %wide.load14 = load <4 x i32>, <4 x i32>* %6, align 16
+  %7 = mul nsw <4 x i32> %4, %wide.load14
+  %8 = getelementptr inbounds [4096 x i32], [4096 x i32]* @ca, i64 0, i64 %index
+  %9 = bitcast i32* %8 to <4 x i32>*
+  store <4 x i32> %7, <4 x i32>* %9, align 16
+  %index.next = add nuw nsw i64 %index, 4
+  %10 = getelementptr inbounds [4096 x i32], [4096 x i32]* @cb, i64 0, i64 %index.next
+  %11 = bitcast i32* %10 to <4 x i32>*
+  %wide.load.1 = load <4 x i32>, <4 x i32>* %11, align 16
+  %12 = getelementptr inbounds [4096 x i32], [4096 x i32]* @cc, i64 0, i64 %index.next
+  %13 = bitcast i32* %12 to <4 x i32>*
+  %wide.load13.1 = load <4 x i32>, <4 x i32>* %13, align 16
+  %14 = add nsw <4 x i32> %wide.load13.1, %wide.load.1
+  %15 = getelementptr inbounds [4096 x i32], [4096 x i32]* @cd, i64 0, i64 %index.next
+  %16 = bitcast i32* %15 to <4 x i32>*
+  %wide.load14.1 = load <4 x i32>, <4 x i32>* %16, align 16
+  %17 = mul nsw <4 x i32> %14, %wide.load14.1
+  %18 = getelementptr inbounds [4096 x i32], [4096 x i32]* @ca, i64 0, i64 %index.next
+  %19 = bitcast i32* %18 to <4 x i32>*
+  store <4 x i32> %17, <4 x i32>* %19, align 16
+  %index.next.1 = add nuw nsw i64 %index.next, 4
+  %20 = getelementptr inbounds [4096 x i32], [4096 x i32]* @cb, i64 0, i64 %index.next.1
+  %21 = bitcast i32* %20 to <4 x i32>*
+  %wide.load.2 = load <4 x i32>, <4 x i32>* %21, align 16
+  %22 = getelementptr inbounds [4096 x i32], [4096 x i32]* @cc, i64 0, i64 %index.next.1
+  %23 = bitcast i32* %22 to <4 x i32>*
+  %wide.load13.2 = load <4 x i32>, <4 x i32>* %23, align 16
+  %24 = add nsw <4 x i32> %wide.load13.2, %wide.load.2
+  %25 = getelementptr inbounds [4096 x i32], [4096 x i32]* @cd, i64 0, i64 %index.next.1
+  %26 = bitcast i32* %25 to <4 x i32>*
+  %wide.load14.2 = load <4 x i32>, <4 x i32>* %26, align 16
+  %27 = mul nsw <4 x i32> %24, %wide.load14.2
+  %28 = getelementptr inbounds [4096 x i32], [4096 x i32]* @ca, i64 0, i64 %index.next.1
+  %29 = bitcast i32* %28 to <4 x i32>*
+  store <4 x i32> %27, <4 x i32>* %29, align 16
+  %index.next.2 = add nuw nsw i64 %index.next.1, 4
+  %30 = getelementptr inbounds [4096 x i32], [4096 x i32]* @cb, i64 0, i64 %index.next.2
+  %31 = bitcast i32* %30 to <4 x i32>*
+  %wide.load.3 = load <4 x i32>, <4 x i32>* %31, align 16
+  %32 = getelementptr inbounds [4096 x i32], [4096 x i32]* @cc, i64 0, i64 %index.next.2
+  %33 = bitcast i32* %32 to <4 x i32>*
+  %wide.load13.3 = load <4 x i32>, <4 x i32>* %33, align 16
+  %34 = add nsw <4 x i32> %wide.load13.3, %wide.load.3
+  %35 = getelementptr inbounds [4096 x i32], [4096 x i32]* @cd, i64 0, i64 %index.next.2
+  %36 = bitcast i32* %35 to <4 x i32>*
+  %wide.load14.3 = load <4 x i32>, <4 x i32>* %36, align 16
+  %37 = mul nsw <4 x i32> %34, %wide.load14.3
+  %38 = getelementptr inbounds [4096 x i32], [4096 x i32]* @ca, i64 0, i64 %index.next.2
+  %39 = bitcast i32* %38 to <4 x i32>*
+  store <4 x i32> %37, <4 x i32>* %39, align 16
+  %index.next.3 = add nuw nsw i64 %index.next.2, 4
+  %40 = icmp eq i64 %index.next.3, 4096
+  br i1 %40, label %for.end, label %vector.body
+
+for.end:
+  ret void
+}
+
+; CHECK-LABEL: @foo
+; CHECK-NOT: xxpermdi
+; CHECK-NOT: xxswapd
+
+; CHECK: lxvd2x
+; CHECK: lxvd2x
+; CHECK-DAG: lxvd2x
+; CHECK-DAG: vadduwm
+; CHECK: vmuluwm
+; CHECK: stxvd2x
+
+; CHECK: lxvd2x
+; CHECK: lxvd2x
+; CHECK-DAG: lxvd2x
+; CHECK-DAG: vadduwm
+; CHECK: vmuluwm
+; CHECK: stxvd2x
+
+; CHECK: lxvd2x
+; CHECK: lxvd2x
+; CHECK-DAG: lxvd2x
+; CHECK-DAG: vadduwm
+; CHECK: vmuluwm
+; CHECK: stxvd2x
+
+; CHECK: lxvd2x
+; CHECK: lxvd2x
+; CHECK-DAG: lxvd2x
+; CHECK-DAG: vadduwm
+; CHECK: vmuluwm
+; CHECK: stxvd2x
+
+
+; NOOPTSWAP-LABEL: @foo
+
+; NOOPTSWAP: lxvd2x
+; NOOPTSWAP-DAG: lxvd2x
+; NOOPTSWAP-DAG: lxvd2x
+; NOOPTSWAP-DAG: xxswapd
+; NOOPTSWAP-DAG: xxswapd
+; NOOPTSWAP-DAG: xxswapd
+; NOOPTSWAP-DAG: vadduwm
+; NOOPTSWAP: vmuluwm
+; NOOPTSWAP: xxswapd
+; NOOPTSWAP-DAG: xxswapd
+; NOOPTSWAP-DAG: xxswapd
+; NOOPTSWAP-DAG: stxvd2x
+; NOOPTSWAP-DAG: stxvd2x
+; NOOPTSWAP: stxvd2x
+
diff --git a/test/CodeGen/PowerPC/swaps-le-2.ll b/test/CodeGen/PowerPC/swaps-le-2.ll
new file mode 100644
index 00000000000..08096ed20dd
--- /dev/null
+++ b/test/CodeGen/PowerPC/swaps-le-2.ll
@@ -0,0 +1,91 @@
+; RUN: llc -O3 -mcpu=pwr8 -mtriple=powerpc64le-unknown-linux-gnu < %s | FileCheck %s
+
+; Test swap removal when a vector splat must be adjusted to make it legal.
+;
+; Test generated from following C code:
+;
+; vector char vc = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15};
+; vector char vcr;
+; vector short vs = {0, 1, 2, 3, 4, 5, 6, 7};
+; vector short vsr;
+; vector int vi = {0, 1, 2, 3};
+; vector int vir;
+;
+; void cfoo ()
+; {
+;   vcr = (vector char){vc[5], vc[5], vc[5], vc[5], vc[5], vc[5], vc[5], vc[5],
+;                       vc[5], vc[5], vc[5], vc[5], vc[5], vc[5], vc[5], vc[5]};
+; }
+;
+; void sfoo ()
+; {
+;   vsr = (vector short){vs[6], vs[6], vs[6], vs[6],
+;                        vs[6], vs[6], vs[6], vs[6]};
+; }
+;
+; void ifoo ()
+; {
+;   vir = (vector int){vi[1], vi[1], vi[1], vi[1]};
+; }
+
+@vc = global <16 x i8> <i8 0, i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 8, i8 9, i8 10, i8 11, i8 12, i8 13, i8 14, i8 15>, align 16
+@vs = global <8 x i16> <i16 0, i16 1, i16 2, i16 3, i16 4, i16 5, i16 6, i16 7>, align 16
+@vi = global <4 x i32> <i32 0, i32 1, i32 2, i32 3>, align 16
+@vcr = common global <16 x i8> zeroinitializer, align 16
+@vsr = common global <8 x i16> zeroinitializer, align 16
+@vir = common global <4 x i32> zeroinitializer, align 16
+
+; Function Attrs: nounwind
+define void @cfoo() {
+entry:
+  %0 = load <16 x i8>, <16 x i8>* @vc, align 16
+  %vecinit30 = shufflevector <16 x i8> %0, <16 x i8> undef, <16 x i32> <i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5>
+  store <16 x i8> %vecinit30, <16 x i8>* @vcr, align 16
+  ret void
+}
+
+; Function Attrs: nounwind
+define void @sfoo() {
+entry:
+  %0 = load <8 x i16>, <8 x i16>* @vs, align 16
+  %vecinit14 = shufflevector <8 x i16> %0, <8 x i16> undef, <8 x i32> <i32 6, i32 6, i32 6, i32 6, i32 6, i32 6, i32 6, i32 6>
+  store <8 x i16> %vecinit14, <8 x i16>* @vsr, align 16
+  ret void
+}
+
+; Function Attrs: nounwind
+define void @ifoo() {
+entry:
+  %0 = load <4 x i32>, <4 x i32>* @vi, align 16
+  %vecinit6 = shufflevector <4 x i32> %0, <4 x i32> undef, <4 x i32> <i32 1, i32 1, i32 1, i32 1>
+  store <4 x i32> %vecinit6, <4 x i32>* @vir, align 16
+  ret void
+}
+
+; Justification:
+;  Byte splat of element 5 (BE) becomes element 15-5 = 10 (LE)
+;  which becomes (10+8)%16 = 2 (LE swapped).
+;
+;  Halfword splat of element 6 (BE) becomes element 7-6 = 1 (LE)
+;  which becomes (1+4)%8 = 5 (LE swapped).
+;
+;  Word splat of element 1 (BE) becomes element 3-1 = 2 (LE)
+;  which becomes (2+2)%4 = 0 (LE swapped).
+
+; CHECK-NOT: xxpermdi
+; CHECK-NOT: xxswapd
+
+; CHECK-LABEL: @cfoo
+; CHECK: lxvd2x
+; CHECK: vspltb {{[0-9]+}}, {{[0-9]+}}, 2
+; CHECK: stxvd2x
+
+; CHECK-LABEL: @sfoo
+; CHECK: lxvd2x
+; CHECK: vsplth {{[0-9]+}}, {{[0-9]+}}, 5
+; CHECK: stxvd2x
+
+; CHECK-LABEL: @ifoo
+; CHECK: lxvd2x
+; CHECK: vspltw {{[0-9]+}}, {{[0-9]+}}, 0
+; CHECK: stxvd2x
diff --git a/test/CodeGen/PowerPC/swaps-le-3.ll b/test/CodeGen/PowerPC/swaps-le-3.ll
new file mode 100644
index 00000000000..49b93976d31
--- /dev/null
+++ b/test/CodeGen/PowerPC/swaps-le-3.ll
@@ -0,0 +1,24 @@
+; RUN: llc -mcpu=pwr8 -mtriple=powerpc64le-unknown-linux-gnu -O3 < %s | FileCheck %s
+
+; This test verifies that VSX swap optimization works for the
+; doubleword splat idiom.
+
+@a = external global <2 x double>, align 16
+@b = external global <2 x double>, align 16
+
+define void @test(double %s) {
+entry:
+  %0 = insertelement <2 x double> undef, double %s, i32 0
+  %1 = shufflevector <2 x double> %0, <2 x double> undef, <2 x i32> zeroinitializer
+  %2 = load <2 x double>, <2 x double>* @a, align 16
+  %3 = fadd <2 x double> %0, %2
+  store <2 x double> %3, <2 x double>* @b, align 16
+  ret void
+}
+
+; CHECK-LABEL: @test
+; CHECK-DAG: xxspltd
+; CHECK-DAG: lxvd2x
+; CHECK: xvadddp
+; CHECK: stxvd2x
+; CHECK-NOT: xxswapd
diff --git a/test/CodeGen/PowerPC/swaps-le-4.ll b/test/CodeGen/PowerPC/swaps-le-4.ll
new file mode 100644
index 00000000000..7d8239bd255
--- /dev/null
+++ b/test/CodeGen/PowerPC/swaps-le-4.ll
@@ -0,0 +1,27 @@
+; RUN: llc -mcpu=pwr8 -mtriple=powerpc64le-unknown-linux-gnu -O3 < %s | FileCheck %s
+
+; This test verifies that VSX swap optimization works when an implicit
+; subregister is present (in this case, in the XXPERMDI associated with
+; the store).
+
+define void @bar() {
+entry:
+  %x = alloca <2 x i64>, align 16
+  %0 = bitcast <2 x i64>* %x to i8*
+  call void @llvm.lifetime.start(i64 16, i8* %0)
+  %arrayidx = getelementptr inbounds <2 x i64>, <2 x i64>* %x, i64 0, i64 0
+  store <2 x i64> <i64 0, i64 1>, <2 x i64>* %x, align 16
+  call void @foo(i64* %arrayidx)
+  call void @llvm.lifetime.end(i64 16, i8* %0)
+  ret void
+}
+
+; CHECK-LABEL: @bar
+; CHECK: lxvd2x
+; CHECK: stxvd2x
+; CHECK-NOT: xxswapd
+
+declare void @llvm.lifetime.start(i64, i8* nocapture)
+declare void @foo(i64*)
+declare void @llvm.lifetime.end(i64, i8* nocapture)
+
diff --git a/test/CodeGen/PowerPC/swaps-le-5.ll b/test/CodeGen/PowerPC/swaps-le-5.ll
new file mode 100644
index 00000000000..5cd739a0efa
--- /dev/null
+++ b/test/CodeGen/PowerPC/swaps-le-5.ll
@@ -0,0 +1,70 @@
+; RUN: llc -mcpu=pwr8 -mtriple=powerpc64le-unknown-linux-gnu -O3 < %s | FileCheck %s
+
+; These tests verify that VSX swap optimization works for various
+; manipulations of <2 x double> vectors.
+
+@x = global <2 x double> <double 9.970000e+01, double -1.032220e+02>, align 16
+@z = global <2 x double> <double 2.332000e+01, double 3.111111e+01>, align 16
+
+define void @bar0(double %y) {
+entry:
+  %0 = load <2 x double>, <2 x double>* @x, align 16
+  %vecins = insertelement <2 x double> %0, double %y, i32 0
+  store <2 x double> %vecins, <2 x double>* @z, align 16
+  ret void
+}
+
+; CHECK-LABEL: @bar0
+; CHECK-DAG: xxswapd {{[0-9]+}}, 1
+; CHECK-DAG: lxvd2x [[REG1:[0-9]+]]
+; CHECK-DAG: xxspltd [[REG2:[0-9]+]]
+; CHECK: xxpermdi [[REG3:[0-9]+]], [[REG2]], [[REG1]], 1
+; CHECK: stxvd2x [[REG3]]
+
+define void @bar1(double %y) {
+entry:
+  %0 = load <2 x double>, <2 x double>* @x, align 16
+  %vecins = insertelement <2 x double> %0, double %y, i32 1
+  store <2 x double> %vecins, <2 x double>* @z, align 16
+  ret void
+}
+
+; CHECK-LABEL: @bar1
+; CHECK-DAG: xxswapd {{[0-9]+}}, 1
+; CHECK-DAG: lxvd2x [[REG1:[0-9]+]]
+; CHECK-DAG: xxspltd [[REG2:[0-9]+]]
+; CHECK: xxmrghd [[REG3:[0-9]+]], [[REG1]], [[REG2]]
+; CHECK: stxvd2x [[REG3]]
+
+define void @baz0() {
+entry:
+  %0 = load <2 x double>, <2 x double>* @z, align 16
+  %1 = load <2 x double>, <2 x double>* @x, align 16
+  %vecins = shufflevector <2 x double> %0, <2 x double> %1, <2 x i32> <i32 0, i32 2>
+  store <2 x double> %vecins, <2 x double>* @z, align 16
+  ret void
+}
+
+; CHECK-LABEL: @baz0
+; CHECK: lxvd2x
+; CHECK: lxvd2x
+; CHECK: xxmrghd
+; CHECK: stxvd2x
+; CHECK-NOT: xxswapd
+
+define void @baz1() {
+entry:
+  %0 = load <2 x double>, <2 x double>* @z, align 16
+  %1 = load <2 x double>, <2 x double>* @x, align 16
+  %vecins = shufflevector <2 x double> %0, <2 x double> %1, <2 x i32> <i32 3, i32 1>
+  store <2 x double> %vecins, <2 x double>* @z, align 16
+  ret void
+}
+
+; CHECK-LABEL: @baz1
+; CHECK: lxvd2x
+; CHECK: lxvd2x
+; CHECK: xxmrgld
+; CHECK: stxvd2x
+; CHECK-NOT: xxswapd
+
diff --git a/test/CodeGen/PowerPC/swaps-le-6.ll b/test/CodeGen/PowerPC/swaps-le-6.ll
new file mode 100644
index 00000000000..365aeee2d8f
--- /dev/null
+++ b/test/CodeGen/PowerPC/swaps-le-6.ll
@@ -0,0 +1,44 @@
+; RUN: llc -mcpu=pwr8 -mtriple=powerpc64le-unknown-linux-gnu -O3 < %s | FileCheck %s
+
+; These tests verify that VSX swap optimization works when loading a scalar
+; into a vector register.
+
+
+@x = global <2 x double> <double 9.970000e+01, double -1.032220e+02>, align 16
+@z = global <2 x double> <double 2.332000e+01, double 3.111111e+01>, align 16
+@y = global double 1.780000e+00, align 8
+
+define void @bar0() {
+entry:
+  %0 = load <2 x double>, <2 x double>* @x, align 16
+  %1 = load double, double* @y, align 8
+  %vecins = insertelement <2 x double> %0, double %1, i32 0
+  store <2 x double> %vecins, <2 x double>* @z, align 16
+  ret void
+}
+
+; CHECK-LABEL: @bar0
+; CHECK-DAG: lxvd2x [[REG1:[0-9]+]]
+; CHECK-DAG: lxsdx [[REG2:[0-9]+]]
+; CHECK: xxswapd [[REG3:[0-9]+]], [[REG2]]
+; CHECK: xxspltd [[REG4:[0-9]+]], [[REG3]], 1
+; CHECK: xxpermdi [[REG5:[0-9]+]], [[REG4]], [[REG1]], 1
+; CHECK: stxvd2x [[REG5]]
+
+define void @bar1() {
+entry:
+  %0 = load <2 x double>, <2 x double>* @x, align 16
+  %1 = load double, double* @y, align 8
+  %vecins = insertelement <2 x double> %0, double %1, i32 1
+  store <2 x double> %vecins, <2 x double>* @z, align 16
+  ret void
+}
+
+; CHECK-LABEL: @bar1
+; CHECK-DAG: lxvd2x [[REG1:[0-9]+]]
+; CHECK-DAG: lxsdx [[REG2:[0-9]+]]
+; CHECK: xxswapd [[REG3:[0-9]+]], [[REG2]]
+; CHECK: xxspltd [[REG4:[0-9]+]], [[REG3]], 1
+; CHECK: xxmrghd [[REG5:[0-9]+]], [[REG1]], [[REG4]]
+; CHECK: stxvd2x [[REG5]]
+
diff --git a/test/CodeGen/PowerPC/tls-store2.ll b/test/CodeGen/PowerPC/tls-store2.ll
index e9aa17e8c0f..649508637f4 100644
--- a/test/CodeGen/PowerPC/tls-store2.ll
+++ b/test/CodeGen/PowerPC/tls-store2.ll
@@ -29,6 +29,8 @@ entry:
 ; CHECK: addi 3, {{[0-9]+}}, __once_call@got@tlsgd@l
 ; CHECK: bl __tls_get_addr(__once_call@tlsgd)
 ; CHECK-NEXT: nop
-; CHECK: std {{[0-9]+}}, 0(3)
+; FIXME: We don't really need the copy here either, we could move the store up.
+; CHECK: mr [[REG1:[0-9]+]], 3
+; CHECK: std {{[0-9]+}}, 0([[REG1]])
 
 declare void @__once_call_impl()
diff --git a/test/CodeGen/PowerPC/unwind-dw2-g.ll b/test/CodeGen/PowerPC/unwind-dw2-g.ll
index bea4f9f35da..8bd158867c7 100644
--- a/test/CodeGen/PowerPC/unwind-dw2-g.ll
+++ b/test/CodeGen/PowerPC/unwind-dw2-g.ll
@@ -21,15 +21,15 @@ attributes #0 = { nounwind }
 !llvm.dbg.cu = !{!0}
 !llvm.module.flags = !{!8, !11}
 
-!0 = !MDCompileUnit(language: DW_LANG_C99, producer: "clang version 3.4", isOptimized: false, emissionKind: 0, file: !1, enums: !2, retainedTypes: !2, subprograms: !3, globals: !2, imports: !2)
-!1 = !MDFile(filename: "/tmp/unwind-dw2.c", directory: "/tmp")
+!0 = !DICompileUnit(language: DW_LANG_C99, producer: "clang version 3.4", isOptimized: false, emissionKind: 0, file: !1, enums: !2, retainedTypes: !2, subprograms: !3, globals: !2, imports: !2)
+!1 = !DIFile(filename: "/tmp/unwind-dw2.c", directory: "/tmp")
 !2 = !{}
 !3 = !{!4}
-!4 = !MDSubprogram(name: "foo", line: 1, isLocal: false, isDefinition: true, virtualIndex: 6, isOptimized: false, scopeLine: 1, file: !1, scope: !5, type: !6, function: void ()* @foo, variables: !2)
-!5 = !MDFile(filename: "/tmp/unwind-dw2.c", directory: "/tmp")
-!6 = !MDSubroutineType(types: !7)
+!4 = !DISubprogram(name: "foo", line: 1, isLocal: false, isDefinition: true, virtualIndex: 6, isOptimized: false, scopeLine: 1, file: !1, scope: !5, type: !6, function: void ()* @foo, variables: !2)
+!5 = !DIFile(filename: "/tmp/unwind-dw2.c", directory: "/tmp")
+!6 = !DISubroutineType(types: !7)
 !7 = !{null}
 !8 = !{i32 2, !"Dwarf Version", i32 3}
-!9 = !MDLocation(line: 2, scope: !4)
-!10 = !MDLocation(line: 3, scope: !4)
+!9 = !DILocation(line: 2, scope: !4)
+!10 = !DILocation(line: 3, scope: !4)
 !11 = !{i32 1, !"Debug Info Version", i32 3}
diff --git a/test/CodeGen/PowerPC/vec_add_sub_quadword.ll b/test/CodeGen/PowerPC/vec_add_sub_quadword.ll
new file mode 100644
index 00000000000..f7ebf479755
--- /dev/null
+++ b/test/CodeGen/PowerPC/vec_add_sub_quadword.ll
@@ -0,0 +1,130 @@
+; Check VMX 128-bit integer operations
+;
+; RUN: llc -mtriple=powerpc64-unknown-linux-gnu -mcpu=pwr8 < %s | FileCheck %s
+; RUN: llc -mtriple=powerpc64-unknown-linux-gnu -mcpu=pwr8 -mattr=-vsx < %s | FileCheck %s
+
+define <1 x i128> @test_add(<1 x i128> %x, <1 x i128> %y) nounwind {
+       %result = add <1 x i128> %x, %y
+       ret <1 x i128> %result
+; CHECK-LABEL: @test_add
+; CHECK: vadduqm 2, 2, 3
+}
+
+define <1 x i128> @increment_by_one(<1 x i128> %x) nounwind {
+       %result = add <1 x i128> %x, <i128 1>
+       ret <1 x i128> %result
+; CHECK-LABEL: @increment_by_one
+; CHECK vadduqm 2, 2, 3
+}
+
+define <1 x i128> @increment_by_val(<1 x i128> %x, i128 %val) nounwind {
+       %tmpvec = insertelement <1 x i128> <i128 0>, i128 %val, i32 0
+       %tmpvec2 = insertelement <1 x i128> %tmpvec, i128 %val, i32 1
+       %result = add <1 x i128> %x, %tmpvec2
+       ret <1 x i128> %result
+; CHECK-LABEL: @increment_by_val
+; CHECK: vadduqm 2, 2, 3
+}
+
+define <1 x i128> @test_sub(<1 x i128> %x, <1 x i128> %y) nounwind {
+       %result = sub <1 x i128> %x, %y
+       ret <1 x i128> %result
+; CHECK-LABEL: @test_sub
+; CHECK: vsubuqm 2, 2, 3
+}
+
+define <1 x i128> @decrement_by_one(<1 x i128> %x) nounwind {
+       %result = sub <1 x i128> %x, <i128 1>
+       ret <1 x i128> %result
+; CHECK-LABEL: @decrement_by_one
+; CHECK vsubuqm 2, 2, 3
+}
+
+define <1 x i128> @decrement_by_val(<1 x i128> %x, i128 %val) nounwind {
+       %tmpvec = insertelement <1 x i128> <i128 0>, i128 %val, i32 0
+       %tmpvec2 = insertelement <1 x i128> %tmpvec, i128 %val, i32 1
+       %result = sub <1 x i128> %x, %tmpvec2
+       ret <1 x i128> %result
+; CHECK-LABEL: @decrement_by_val
+; CHECK vsubuqm   2, 2, 3
+}
+
+declare <1 x i128> @llvm.ppc.altivec.vaddeuqm(<1 x i128> %x, 
+                                              <1 x i128> %y, 
+                                              <1 x i128> %z) nounwind readnone
+declare <1 x i128> @llvm.ppc.altivec.vaddcuq(<1 x i128> %x, 
+                                             <1 x i128> %y) nounwind readnone
+declare <1 x i128> @llvm.ppc.altivec.vaddecuq(<1 x i128> %x, 
+                                              <1 x i128> %y, 
+                                              <1 x i128> %z) nounwind readnone
+declare <1 x i128> @llvm.ppc.altivec.vsubeuqm(<1 x i128> %x, 
+                                              <1 x i128> %y, 
+                                              <1 x i128> %z) nounwind readnone
+declare <1 x i128> @llvm.ppc.altivec.vsubcuq(<1 x i128> %x, 
+                                             <1 x i128> %y) nounwind readnone
+declare <1 x i128> @llvm.ppc.altivec.vsubecuq(<1 x i128> %x, 
+                                              <1 x i128> %y, 
+                                              <1 x i128> %z) nounwind readnone
+
+define <1 x i128> @test_vaddeuqm(<1 x i128> %x, 
+       	    	                 <1 x i128> %y, 
+                                 <1 x i128> %z) nounwind {
+  %tmp = tail call <1 x i128> @llvm.ppc.altivec.vaddeuqm(<1 x i128> %x,
+                                                         <1 x i128> %y,
+                                                         <1 x i128> %z)
+  ret <1 x i128> %tmp
+; CHECK-LABEL: @test_vaddeuqm
+; CHECK: vaddeuqm 2, 2, 3, 4
+}
+
+define <1 x i128> @test_vaddcuq(<1 x i128> %x, 
+       	    	                <1 x i128> %y) nounwind {
+  %tmp = tail call <1 x i128> @llvm.ppc.altivec.vaddcuq(<1 x i128> %x,
+                                                        <1 x i128> %y)
+  ret <1 x i128> %tmp
+; CHECK-LABEL: @test_vaddcuq
+; CHECK: vaddcuq 2, 2, 3
+}
+
+define <1 x i128> @test_vaddecuq(<1 x i128> %x, 
+       	    	                 <1 x i128> %y, 
+                                 <1 x i128> %z) nounwind {
+  %tmp = tail call <1 x i128> @llvm.ppc.altivec.vaddecuq(<1 x i128> %x,
+                                                         <1 x i128> %y,
+                                                         <1 x i128> %z)
+  ret <1 x i128> %tmp
+; CHECK-LABEL: @test_vaddecuq
+; CHECK: vaddecuq 2, 2, 3, 4
+}
+
+define <1 x i128> @test_vsubeuqm(<1 x i128> %x, 
+       	    	                 <1 x i128> %y, 
+                                 <1 x i128> %z) nounwind {
+  %tmp = tail call <1 x i128> @llvm.ppc.altivec.vsubeuqm(<1 x i128> %x,
+                                                         <1 x i128> %y,
+                                                         <1 x i128> %z)
+  ret <1 x i128> %tmp
+; CHECK-LABEL: test_vsubeuqm
+; CHECK: vsubeuqm 2, 2, 3, 4
+}
+
+define <1 x i128> @test_vsubcuq(<1 x i128> %x, 
+       	    	                <1 x i128> %y) nounwind {
+  %tmp = tail call <1 x i128> @llvm.ppc.altivec.vsubcuq(<1 x i128> %x,
+                                                        <1 x i128> %y)
+  ret <1 x i128> %tmp
+; CHECK-LABEL: test_vsubcuq
+; CHECK: vsubcuq 2, 2, 3
+}
+
+define <1 x i128> @test_vsubecuq(<1 x i128> %x, 
+       	    	                 <1 x i128> %y, 
+                                 <1 x i128> %z) nounwind {
+  %tmp = tail call <1 x i128> @llvm.ppc.altivec.vsubecuq(<1 x i128> %x,
+                                                         <1 x i128> %y,
+                                                         <1 x i128> %z)
+  ret <1 x i128> %tmp
+; CHECK-LABEL: test_vsubecuq
+; CHECK: vsubecuq 2, 2, 3, 4
+}
+
diff --git a/test/CodeGen/PowerPC/vec_mergeow.ll b/test/CodeGen/PowerPC/vec_mergeow.ll
new file mode 100644
index 00000000000..c7c7448e3ae
--- /dev/null
+++ b/test/CodeGen/PowerPC/vec_mergeow.ll
@@ -0,0 +1,101 @@
+; RUN: llc -mtriple=powerpc64le-unknown-linux-gnu -mcpu=pwr8 < %s | \
+; RUN:   FileCheck %s  -check-prefix=CHECK-LE
+; RUN: llc -mtriple=powerpc64-unknown-linux-gnu -mcpu=pwr8 < %s | \
+; RUN:   FileCheck %s -check-prefix=CHECK-BE
+
+; Check for a vector merge instruction using two inputs
+; The shufflevector specifies the even elements, using big endian element 
+; ordering. If run on a big endian machine, this should produce the vmrgew 
+; instruction. If run on a little endian machine, this should produce the
+; vmrgow instruction. Note also that on little endian the input registers 
+; are swapped also.
+define void @check_merge_even_xy(<16 x i8>* %A, <16 x i8>* %B) {
+entry:
+; CHECK-LE-LABEL: @check_merge_even_xy
+; CHECK-BE-LABEL: @check_merge_even_xy
+        %tmp = load <16 x i8>, <16 x i8>* %A
+	%tmp2 = load <16 x i8>, <16 x i8>* %B
+	%tmp3 = shufflevector <16 x i8> %tmp, <16 x i8> %tmp2, 
+	      		      <16 x i32> <i32 0, i32 1, i32 2, i32 3, 
+			      	    	  i32 16, i32 17, i32 18, i32 19, 
+					  i32 8, i32 9, i32 10, i32 11, 
+					  i32 24, i32 25, i32 26, i32 27>
+; CHECK-LE: vmrgow 2, 3, 2
+; CHECK-BE: vmrgew 2, 2, 3
+      	store <16 x i8> %tmp3, <16 x i8>* %A
+	ret void
+; CHECK-LE: blr
+; CHECK-BE: blr
+}
+
+; Check for a vector merge instruction using a single input. 
+; The shufflevector specifies the even elements, using big endian element 
+; ordering. If run on a big endian machine, this should produce the vmrgew 
+; instruction. If run on a little endian machine, this should produce the
+; vmrgow instruction. 
+define void @check_merge_even_xx(<16 x i8>* %A) {
+entry:
+; CHECK-LE-LABEL: @check_merge_even_xx
+; CHECK-BE-LABEL: @check_merge_even_xx
+        %tmp = load <16 x i8>, <16 x i8>* %A
+	%tmp2 = shufflevector <16 x i8> %tmp, <16 x i8> %tmp, 
+	      		      <16 x i32> <i32 0, i32 1, i32 2, i32 3, 
+			      	          i32 0, i32 1, i32 2, i32 3, 
+					  i32 8, i32 9, i32 10, i32 11, 
+					  i32 8, i32 9, i32 10, i32 11>
+; CHECK-LE: vmrgow 2, 2, 2
+; CHECK-BE: vmrgew 2, 2, 2
+  	store <16 x i8> %tmp2, <16 x i8>* %A
+	ret void
+; CHECK-LE: blr
+; CHECK-BE: blr       
+}
+
+; Check for a vector merge instruction using two inputs.
+; The shufflevector specifies the odd elements, using big endian element 
+; ordering. If run on a big endian machine, this should produce the vmrgow 
+; instruction. If run on a little endian machine, this should produce the
+; vmrgew instruction. Note also that on little endian the input registers 
+; are swapped also.
+define void @check_merge_odd_xy(<16 x i8>* %A, <16 x i8>* %B) {
+entry:
+; CHECK-LE-LABEL: @check_merge_odd_xy
+; CHECK-BE-LABEL: @check_merge_odd_xy
+        %tmp = load <16 x i8>, <16 x i8>* %A
+	%tmp2 = load <16 x i8>, <16 x i8>* %B
+	%tmp3 = shufflevector <16 x i8> %tmp, <16 x i8> %tmp2, 
+	      		      <16 x i32> <i32 4, i32 5, i32 6, i32 7, 
+			      	    	  i32 20, i32 21, i32 22, i32 23, 
+					  i32 12, i32 13, i32 14, i32 15, 
+					  i32 28, i32 29, i32 30, i32 31>
+; CHECK-LE: vmrgew 2, 3, 2
+; CHECK-BE: vmrgow 2, 2, 3
+        store <16 x i8> %tmp3, <16 x i8>* %A
+	ret void
+; CHECK-LE: blr
+; CHECK-BE: blr
+}
+
+; Check for a vector merge instruction using a single input.
+; The shufflevector specifies the odd elements, using big endian element 
+; ordering. If run on a big endian machine, this should produce the vmrgow 
+; instruction. If run on a little endian machine, this should produce the
+; vmrgew instruction. 
+define void @check_merge_odd_xx(<16 x i8>* %A) {
+entry:
+; CHECK-LE-LABEL: @check_merge_odd_xx
+; CHECK-BE-LABEL: @check_merge_odd_xx
+        %tmp = load <16 x i8>, <16 x i8>* %A
+	%tmp2 = shufflevector <16 x i8> %tmp, <16 x i8> %tmp, 
+	      		      <16 x i32> <i32 4, i32 5, i32 6, i32 7, 
+			      	    	  i32 4, i32 5, i32 6, i32 7, 
+					  i32 12, i32 13, i32 14, i32 15, 
+					  i32 12, i32 13, i32 14, i32 15>
+; CHECK-LE: vmrgew 2, 2, 2
+; CHECK-BE: vmrgow 2, 2, 2
+        store <16 x i8> %tmp2, <16 x i8>* %A
+	ret void
+; CHECK-LE: blr
+; CHECK-BE: blr
+}
+
diff --git a/test/CodeGen/PowerPC/vec_shuffle_le.ll b/test/CodeGen/PowerPC/vec_shuffle_le.ll
index 46d451ff157..65c47ada875 100644
--- a/test/CodeGen/PowerPC/vec_shuffle_le.ll
+++ b/test/CodeGen/PowerPC/vec_shuffle_le.ll
@@ -202,7 +202,7 @@ entry:
 ; CHECK: VSLDOI_xx:
         %tmp = load <16 x i8>, <16 x i8>* %A
         %tmp2 = shufflevector <16 x i8> %tmp, <16 x i8> %tmp, <16 x i32> <i32 12, i32 13, i32 14, i32 15, i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11>
-; CHECK: vsldoi
+; CHECK: vsldoi {{[0-9]+}}, [[REG1:[0-9]+]], [[REG1]], 4
         store <16 x i8> %tmp2, <16 x i8>* %A
         ret void
 }
diff --git a/test/CodeGen/PowerPC/vec_shuffle_p8vector.ll b/test/CodeGen/PowerPC/vec_shuffle_p8vector.ll
new file mode 100644
index 00000000000..77802348d8e
--- /dev/null
+++ b/test/CodeGen/PowerPC/vec_shuffle_p8vector.ll
@@ -0,0 +1,54 @@
+; RUN: llc -mcpu=pwr8 -mtriple=powerpc64-unknown-linux-gnu -mattr=+power8-vector < %s | FileCheck %s
+; RUN: llc -mcpu=pwr7 -mtriple=powerpc64-unknown-linux-gnu < %s | FileCheck -check-prefix=CHECK-PWR7 %s
+
+define void @VPKUDUM_unary(<2 x i64>* %A) {
+entry:
+        %tmp = load <2 x i64>, <2 x i64>* %A
+        %tmp2 = bitcast <2 x i64> %tmp to <4 x i32>
+        %tmp3 = extractelement <4 x i32> %tmp2, i32 1
+        %tmp4 = extractelement <4 x i32> %tmp2, i32 3
+        %tmp5 = insertelement <4 x i32> undef, i32 %tmp3, i32 0
+        %tmp6 = insertelement <4 x i32> %tmp5, i32 %tmp4, i32 1
+        %tmp7 = insertelement <4 x i32> %tmp6, i32 %tmp3, i32 2
+        %tmp8 = insertelement <4 x i32> %tmp7, i32 %tmp4, i32 3
+        %tmp9 = bitcast <4 x i32> %tmp8 to <2 x i64>
+        store <2 x i64> %tmp9, <2 x i64>* %A
+        ret void
+}
+
+; CHECK-LABEL: @VPKUDUM_unary
+; CHECK-NOT:   vperm
+; CHECK-NOT:   vmrglw
+; CHECK-NOT:   vmrghw
+; CHECK:       vpkudum
+; CHECK-PWR7:  vmrglw
+; CHECK-PWR7:  vmrghw
+; CHECK-PWR7:  vmrglw
+
+define void @VPKUDUM(<2 x i64>* %A, <2 x i64>* %B) {
+entry:
+        %tmp = load <2 x i64>, <2 x i64>* %A
+        %tmp2 = bitcast <2 x i64> %tmp to <4 x i32>
+        %tmp3 = load <2 x i64>, <2 x i64>* %B
+        %tmp4 = bitcast <2 x i64> %tmp3 to <4 x i32>
+        %tmp5 = extractelement <4 x i32> %tmp2, i32 1
+        %tmp6 = extractelement <4 x i32> %tmp2, i32 3
+        %tmp7 = extractelement <4 x i32> %tmp4, i32 1
+        %tmp8 = extractelement <4 x i32> %tmp4, i32 3
+        %tmp9 = insertelement <4 x i32> undef, i32 %tmp5, i32 0
+        %tmp10 = insertelement <4 x i32> %tmp9, i32 %tmp6, i32 1
+        %tmp11 = insertelement <4 x i32> %tmp10, i32 %tmp7, i32 2
+        %tmp12 = insertelement <4 x i32> %tmp11, i32 %tmp8, i32 3
+        %tmp13 = bitcast <4 x i32> %tmp12 to <2 x i64>
+        store <2 x i64> %tmp13, <2 x i64>* %A
+        ret void
+}
+
+; CHECK-LABEL: @VPKUDUM
+; CHECK-NOT:   vperm
+; CHECK-NOT:   vmrglw
+; CHECK-NOT:   vmrghw
+; CHECK:       vpkudum
+; CHECK-PWR7:  vmrglw
+; CHECK-PWR7:  vmrghw
+; CHECK-PWR7:  vmrglw
diff --git a/test/CodeGen/PowerPC/vec_shuffle_p8vector_le.ll b/test/CodeGen/PowerPC/vec_shuffle_p8vector_le.ll
new file mode 100644
index 00000000000..709388675f6
--- /dev/null
+++ b/test/CodeGen/PowerPC/vec_shuffle_p8vector_le.ll
@@ -0,0 +1,43 @@
+; RUN: llc -mcpu=pwr8 -mtriple=powerpc64le-unknown-linux-gnu -mattr=+power8-vector < %s | FileCheck %s
+
+define void @VPKUDUM_unary(<2 x i64>* %A) {
+entry:
+	%tmp = load <2 x i64>, <2 x i64>* %A
+	%tmp2 = bitcast <2 x i64> %tmp to <4 x i32>
+	%tmp3 = extractelement <4 x i32> %tmp2, i32 0
+	%tmp4 = extractelement <4 x i32> %tmp2, i32 2
+	%tmp5 = insertelement <4 x i32> undef, i32 %tmp3, i32 0
+	%tmp6 = insertelement <4 x i32> %tmp5, i32 %tmp4, i32 1
+	%tmp7 = insertelement <4 x i32> %tmp6, i32 %tmp3, i32 2
+	%tmp8 = insertelement <4 x i32> %tmp7, i32 %tmp4, i32 3
+	%tmp9 = bitcast <4 x i32> %tmp8 to <2 x i64>
+	store <2 x i64> %tmp9, <2 x i64>* %A
+	ret void
+}
+
+; CHECK-LABEL: @VPKUDUM_unary
+; CHECK-NOT:   vperm
+; CHECK:       vpkudum
+
+define void @VPKUDUM(<2 x i64>* %A, <2 x i64>* %B) {
+entry:
+	%tmp = load <2 x i64>, <2 x i64>* %A
+	%tmp2 = bitcast <2 x i64> %tmp to <4 x i32>
+        %tmp3 = load <2 x i64>, <2 x i64>* %B
+        %tmp4 = bitcast <2 x i64> %tmp3 to <4 x i32>
+	%tmp5 = extractelement <4 x i32> %tmp2, i32 0
+	%tmp6 = extractelement <4 x i32> %tmp2, i32 2
+	%tmp7 = extractelement <4 x i32> %tmp4, i32 0
+	%tmp8 = extractelement <4 x i32> %tmp4, i32 2
+	%tmp9 = insertelement <4 x i32> undef, i32 %tmp5, i32 0
+	%tmp10 = insertelement <4 x i32> %tmp9, i32 %tmp6, i32 1
+	%tmp11 = insertelement <4 x i32> %tmp10, i32 %tmp7, i32 2
+	%tmp12 = insertelement <4 x i32> %tmp11, i32 %tmp8, i32 3
+	%tmp13 = bitcast <4 x i32> %tmp12 to <2 x i64>
+	store <2 x i64> %tmp13, <2 x i64>* %A
+	ret void
+}
+
+; CHECK-LABEL: @VPKUDUM
+; CHECK-NOT:   vperm
+; CHECK:       vpkudum
diff --git a/test/CodeGen/PowerPC/vsx-elementary-arith.ll b/test/CodeGen/PowerPC/vsx-elementary-arith.ll
new file mode 100644
index 00000000000..5416f667aef
--- /dev/null
+++ b/test/CodeGen/PowerPC/vsx-elementary-arith.ll
@@ -0,0 +1,151 @@
+; RUN: llc < %s -mtriple=powerpc64-unknown-linux-gnu -mcpu=pwr8 | FileCheck %s
+; RUN: llc < %s -mtriple=powerpc64le-unknown-linux-gnu -mcpu=pwr8 | FileCheck %s
+@a = global float 3.000000e+00, align 4
+@b = global float 4.000000e+00, align 4
+@c = global double 3.000000e+00, align 8
+@d = global double 4.000000e+00, align 8
+
+; Function Attrs: nounwind
+define float @emit_xsaddsp() {
+entry:
+  %0 = load float, float* @a, align 4
+  %1 = load float, float* @b, align 4
+  %add = fadd float %0, %1
+  ret float %add
+; CHECK-LABEL: @emit_xsaddsp
+; CHECK: xsaddsp {{[0-9]+}}
+}
+
+; Function Attrs: nounwind
+define float @emit_xssubsp() {
+entry:
+  %0 = load float, float* @a, align 4
+  %1 = load float, float* @b, align 4
+  %sub = fsub float %0, %1
+  ret float %sub
+; CHECK-LABEL: @emit_xssubsp
+; CHECK: xssubsp {{[0-9]+}}
+}
+
+; Function Attrs: nounwind
+define float @emit_xsdivsp() {
+entry:
+  %0 = load float, float* @a, align 4
+  %1 = load float, float* @b, align 4
+  %div = fdiv float %0, %1
+  ret float %div
+; CHECK-LABEL: @emit_xsdivsp
+; CHECK: xsdivsp {{[0-9]+}}
+}
+
+; Function Attrs: nounwind
+define float @emit_xsmulsp() {
+entry:
+  %0 = load float, float* @a, align 4
+  %1 = load float, float* @b, align 4
+  %mul = fmul float %0, %1
+  ret float %mul
+; CHECK-LABEL: @emit_xsmulsp
+; CHECK: xsmulsp {{[0-9]+}}
+}
+
+; Function Attrs: nounwind
+define float @emit_xssqrtsp() {
+entry:
+  %0 = load float, float* @b, align 4
+  %call = call float @sqrtf(float %0)
+  ret float %call
+; CHECK-LABEL: @emit_xssqrtsp
+; CHECK: xssqrtsp {{[0-9]+}}
+}
+
+; Function Attrs: nounwind
+declare float @sqrtf(float)
+
+; Function Attrs: nounwind
+define double @emit_xsadddp() {
+entry:
+  %0 = load double, double* @c, align 8
+  %1 = load double, double* @d, align 8
+  %add = fadd double %0, %1
+  ret double %add
+; CHECK-LABEL: @emit_xsadddp
+; CHECK: xsadddp {{[0-9]+}}
+}
+
+; Function Attrs: nounwind
+define double @emit_xssubdp() {
+entry:
+  %0 = load double, double* @c, align 8
+  %1 = load double, double* @d, align 8
+  %sub = fsub double %0, %1
+  ret double %sub
+; CHECK-LABEL: @emit_xssubdp
+; CHECK: xssubdp {{[0-9]+}}
+}
+
+; Function Attrs: nounwind
+define double @emit_xsdivdp() {
+entry:
+  %0 = load double, double* @c, align 8
+  %1 = load double, double* @d, align 8
+  %div = fdiv double %0, %1
+  ret double %div
+; CHECK-LABEL: @emit_xsdivdp
+; CHECK: xsdivdp {{[0-9]+}}
+}
+
+; Function Attrs: nounwind
+define double @emit_xsmuldp() {
+entry:
+  %0 = load double, double* @c, align 8
+  %1 = load double, double* @d, align 8
+  %mul = fmul double %0, %1
+  ret double %mul
+; CHECK-LABEL: @emit_xsmuldp
+; CHECK: xsmuldp {{[0-9]+}}
+}
+
+; Function Attrs: nounwind
+define double @emit_xssqrtdp() {
+entry:
+  %0 = load double, double* @d, align 8
+  %call = call double @sqrt(double %0)
+  ret double %call
+; CHECK-LABEL: @emit_xssqrtdp
+; CHECK: xssqrtdp {{[0-9]+}}
+}
+
+; Vector forms
+; Function Attrs: nounwind
+define <4 x float> @emit_xvrsqrtesp() {
+entry:
+; CHECK-LABEL: @emit_xvrsqrtesp
+  %vf = alloca <4 x float>, align 16
+  %vfr = alloca <4 x float>, align 16
+  %0 = load <4 x float>, <4 x float>* %vf, align 16
+  %call = call <4 x float> @llvm.ppc.vsx.xvrsqrtesp(<4 x float> %0)
+; CHECK: xvrsqrtesp {{[0-9]+}}, {{[0-9]+}}
+  ret <4 x float> %call
+}
+
+; Function Attrs: nounwind
+define <2 x double> @emit_xvrsqrtedp() {
+entry:
+; CHECK-LABEL: @emit_xvrsqrtedp
+  %vd = alloca <2 x double>, align 16
+  %vdr = alloca <2 x double>, align 16
+  %0 = load <2 x double>, <2 x double>* %vd, align 16
+  %call = call <2 x double> @llvm.ppc.vsx.xvrsqrtedp(<2 x double> %0)
+  ret <2 x double> %call
+; CHECK: xvrsqrtedp {{[0-9]+}}, {{[0-9]+}}
+}
+
+; Function Attrs: nounwind
+declare double @sqrt(double)
+
+; Function Attrs: nounwind readnone
+declare <4 x float> @llvm.ppc.vsx.xvrsqrtesp(<4 x float>)
+
+; Function Attrs: nounwind readnone
+declare <2 x double> @llvm.ppc.vsx.xvrsqrtedp(<2 x double>)
diff --git a/test/CodeGen/PowerPC/vsx-fma-m.ll b/test/CodeGen/PowerPC/vsx-fma-m.ll
index d85927396e3..4f556b6b79c 100644
--- a/test/CodeGen/PowerPC/vsx-fma-m.ll
+++ b/test/CodeGen/PowerPC/vsx-fma-m.ll
@@ -49,12 +49,13 @@ entry:
 ; CHECK-LABEL: @test2
 ; CHECK-DAG: li [[C1:[0-9]+]], 8
 ; CHECK-DAG: li [[C2:[0-9]+]], 16
-; CHECK-DAG: xsmaddmdp 3, 2, 1
-; CHECK-DAG: xsmaddmdp 4, 2, 1
-; CHECK-DAG: xsmaddadp 1, 2, 5
-; CHECK-DAG: stxsdx 3, 0, 8
-; CHECK-DAG: stxsdx 4, 8, [[C1]]
-; CHECK-DAG: stxsdx 1, 8, [[C2]]
+; FIXME: We no longer get this because of copy ordering at the MI level.
+; CHECX-DAG: xsmaddmdp 3, 2, 1
+; CHECX-DAG: xsmaddmdp 4, 2, 1
+; CHECX-DAG: xsmaddadp 1, 2, 5
+; CHECX-DAG: stxsdx 3, 0, 8
+; CHECX-DAG: stxsdx 4, 8, [[C1]]
+; CHECX-DAG: stxsdx 1, 8, [[C2]]
 ; CHECK: blr
 
 ; CHECK-FISL-LABEL: @test2
@@ -213,14 +214,15 @@ entry:
   ret void
 
 ; CHECK-LABEL: @testv2
-; CHECK-DAG: xvmaddmdp 36, 35, 34
-; CHECK-DAG: xvmaddmdp 37, 35, 34
-; CHECK-DAG: li [[C1:[0-9]+]], 16
-; CHECK-DAG: li [[C2:[0-9]+]], 32
-; CHECK-DAG: xvmaddadp 34, 35, 38
-; CHECK-DAG: stxvd2x 36, 0, 3
-; CHECK-DAG: stxvd2x 37, 3, [[C1:[0-9]+]]
-; CHECK-DAG: stxvd2x 34, 3, [[C2:[0-9]+]]
+; FIXME: We currently don't get this because of copy ordering on the MI level.
+; CHECX-DAG: xvmaddmdp 36, 35, 34
+; CHECX-DAG: xvmaddmdp 37, 35, 34
+; CHECX-DAG: li [[C1:[0-9]+]], 16
+; CHECX-DAG: li [[C2:[0-9]+]], 32
+; CHECX-DAG: xvmaddadp 34, 35, 38
+; CHECX-DAG: stxvd2x 36, 0, 3
+; CHECX-DAG: stxvd2x 37, 3, [[C1:[0-9]+]]
+; CHECX-DAG: stxvd2x 34, 3, [[C2:[0-9]+]]
 ; CHECK: blr
 
 ; CHECK-FISL-LABEL: @testv2
diff --git a/test/CodeGen/PowerPC/vsx-fma-sp.ll b/test/CodeGen/PowerPC/vsx-fma-sp.ll
new file mode 100644
index 00000000000..b4dd2e1627c
--- /dev/null
+++ b/test/CodeGen/PowerPC/vsx-fma-sp.ll
@@ -0,0 +1,168 @@
+; RUN: llc < %s -mtriple=powerpc64le-unknown-linux-gnu -mcpu=pwr8 -mattr=+vsx | FileCheck %s
+; RUN: llc < %s -mtriple=powerpc64le-unknown-linux-gnu -mcpu=pwr8 -mattr=+vsx -fast-isel -O0 | FileCheck -check-prefix=CHECK-FISL %s
+define void @test1sp(float %a, float %b, float %c, float %e, float* nocapture %d) #0 {
+entry:
+  %0 = tail call float @llvm.fma.f32(float %b, float %c, float %a)
+  store float %0, float* %d, align 4
+  %1 = tail call float @llvm.fma.f32(float %b, float %e, float %a)
+  %arrayidx1 = getelementptr inbounds float, float* %d, i64 1
+  store float %1, float* %arrayidx1, align 4
+  ret void
+
+; CHECK-LABEL: @test1sp
+; CHECK-DAG: li [[C1:[0-9]+]], 4
+; CHECK-DAG: xsmaddmsp 3, 2, 1
+; CHECK-DAG: xsmaddasp 1, 2, 4
+; CHECK-DAG: stxsspx 3, 0, 7
+; CHECK-DAG: stxsspx 1, 7, [[C1]]
+; CHECK: blr
+
+; CHECK-FISL-LABEL: @test1sp
+; CHECK-FISL-DAG: fmr 0, 1
+; CHECK-FISL-DAG: xsmaddasp 0, 2, 3
+; CHECK-FISL-DAG: stxsspx 0, 0, 7
+; CHECK-FISL-DAG: xsmaddasp 1, 2, 4
+; CHECK-FISL-DAG: li [[C1:[0-9]+]], 4
+; CHECK-FISL-DAG: stxsspx 1, 7, [[C1]]
+; CHECK-FISL: blr
+}
+
+define void @test2sp(float %a, float %b, float %c, float %e, float %f, float* nocapture %d) #0 {
+entry:
+  %0 = tail call float @llvm.fma.f32(float %b, float %c, float %a)
+  store float %0, float* %d, align 4
+  %1 = tail call float @llvm.fma.f32(float %b, float %e, float %a)
+  %arrayidx1 = getelementptr inbounds float, float* %d, i64 1
+  store float %1, float* %arrayidx1, align 4
+  %2 = tail call float @llvm.fma.f32(float %b, float %f, float %a)
+  %arrayidx2 = getelementptr inbounds float, float* %d, i64 2
+  store float %2, float* %arrayidx2, align 4
+  ret void
+
+; CHECK-LABEL: @test2sp
+; CHECK-DAG: li [[C1:[0-9]+]], 4
+; CHECK-DAG: li [[C2:[0-9]+]], 8
+; FIXME: We now miss this because of copy ordering at the MI level.
+; CHECX-DAG: xsmaddmsp 3, 2, 1
+; CHECX-DAG: xsmaddmsp 4, 2, 1
+; CHECX-DAG: xsmaddasp 1, 2, 5
+; CHECX-DAG: stxsspx 3, 0, 8
+; CHECX-DAG: stxsspx 4, 8, [[C1]]
+; CHECX-DAG: stxsspx 1, 8, [[C2]]
+; CHECK: blr
+
+; CHECK-FISL-LABEL: @test2sp
+; CHECK-FISL-DAG: fmr 0, 1
+; CHECK-FISL-DAG: xsmaddasp 0, 2, 3
+; CHECK-FISL-DAG: stxsspx 0, 0, 8
+; CHECK-FISL-DAG: fmr 0, 1
+; CHECK-FISL-DAG: xsmaddasp 0, 2, 4
+; CHECK-FISL-DAG: li [[C1:[0-9]+]], 4
+; CHECK-FISL-DAG: stxsspx 0, 8, [[C1]]
+; CHECK-FISL-DAG: xsmaddasp 1, 2, 5
+; CHECK-FISL-DAG: li [[C2:[0-9]+]], 8
+; CHECK-FISL-DAG: stxsspx 1, 8, [[C2]]
+; CHECK-FISL: blr
+}
+
+define void @test3sp(float %a, float %b, float %c, float %e, float %f, float* nocapture %d) #0 {
+entry:
+  %0 = tail call float @llvm.fma.f32(float %b, float %c, float %a)
+  store float %0, float* %d, align 4
+  %1 = tail call float @llvm.fma.f32(float %b, float %e, float %a)
+  %2 = tail call float @llvm.fma.f32(float %b, float %c, float %1)
+  %arrayidx1 = getelementptr inbounds float, float* %d, i64 3
+  store float %2, float* %arrayidx1, align 4
+  %3 = tail call float @llvm.fma.f32(float %b, float %f, float %a)
+  %arrayidx2 = getelementptr inbounds float, float* %d, i64 2
+  store float %3, float* %arrayidx2, align 4
+  %arrayidx3 = getelementptr inbounds float, float* %d, i64 1
+  store float %1, float* %arrayidx3, align 4
+  ret void
+
+; CHECK-LABEL: @test3sp
+; CHECK-DAG: fmr [[F1:[0-9]+]], 1
+; CHECK-DAG: li [[C1:[0-9]+]], 12
+; CHECK-DAG: li [[C2:[0-9]+]], 8
+; CHECK-DAG: li [[C3:[0-9]+]], 4
+; CHECK-DAG: xsmaddmsp 4, 2, 1
+; CHECK-DAG: xsmaddasp 1, 2, 5
+
+; Note: We could convert this next FMA to M-type as well, but it would require
+; re-ordering the instructions.
+; CHECK-DAG: xsmaddasp [[F1]], 2, 3
+
+; CHECK-DAG: xsmaddmsp 3, 2, 4
+; CHECK-DAG: stxsspx [[F1]], 0, 8
+; CHECK-DAG: stxsspx 3, 8, [[C1]]
+; CHECK-DAG: stxsspx 1, 8, [[C2]]
+; CHECK-DAG: stxsspx 4, 8, [[C3]]
+; CHECK: blr
+
+; CHECK-FISL-LABEL: @test3sp
+; CHECK-FISL-DAG: fmr [[F1:[0-9]+]], 1
+; CHECK-FISL-DAG: xsmaddasp [[F1]], 2, 4
+; CHECK-FISL-DAG: fmr 4, [[F1]]
+; CHECK-FISL-DAG: xsmaddasp 4, 2, 3
+; CHECK-FISL-DAG: li [[C1:[0-9]+]], 12
+; CHECK-FISL-DAG: stxsspx 4, 8, [[C1]]
+; CHECK-FISL-DAG: xsmaddasp 1, 2, 5
+; CHECK-FISL-DAG: li [[C2:[0-9]+]], 8
+; CHECK-FISL-DAG: stxsspx 1, 8, [[C2]]
+; CHECK-FISL-DAG: li [[C3:[0-9]+]], 4
+; CHECK-FISL-DAG: stxsspx 0, 8, [[C3]]
+; CHECK-FISL: blr
+}
+
+define void @test4sp(float %a, float %b, float %c, float %e, float %f, float* nocapture %d) #0 {
+entry:
+  %0 = tail call float @llvm.fma.f32(float %b, float %c, float %a)
+  store float %0, float* %d, align 4
+  %1 = tail call float @llvm.fma.f32(float %b, float %e, float %a)
+  %arrayidx1 = getelementptr inbounds float, float* %d, i64 1
+  store float %1, float* %arrayidx1, align 4
+  %2 = tail call float @llvm.fma.f32(float %b, float %c, float %1)
+  %arrayidx3 = getelementptr inbounds float, float* %d, i64 3
+  store float %2, float* %arrayidx3, align 4
+  %3 = tail call float @llvm.fma.f32(float %b, float %f, float %a)
+  %arrayidx4 = getelementptr inbounds float, float* %d, i64 2
+  store float %3, float* %arrayidx4, align 4
+  ret void
+
+; CHECK-LABEL: @test4sp
+; CHECK-DAG: fmr [[F1:[0-9]+]], 1
+; CHECK-DAG: li [[C1:[0-9]+]], 4
+; CHECK-DAG: li [[C2:[0-9]+]], 8
+; CHECK-DAG: xsmaddmsp 4, 2, 1
+
+; Note: We could convert this next FMA to M-type as well, but it would require
+; re-ordering the instructions.
+; CHECK-DAG: xsmaddasp 1, 2, 5
+
+; CHECK-DAG: xsmaddasp [[F1]], 2, 3
+; CHECK-DAG: stxsspx [[F1]], 0, 8
+; CHECK-DAG: stxsspx 4, 8, [[C1]]
+; CHECK-DAG: li [[C3:[0-9]+]], 12
+; CHECK-DAG: xsmaddasp 4, 2, 3
+; CHECK-DAG: stxsspx 4, 8, [[C3]]
+; CHECK-DAG: stxsspx 1, 8, [[C2]]
+; CHECK: blr
+
+; CHECK-FISL-LABEL: @test4sp
+; CHECK-FISL-DAG: fmr [[F1:[0-9]+]], 1
+; CHECK-FISL-DAG: xsmaddasp [[F1]], 2, 3
+; CHECK-FISL-DAG: stxsspx 0, 0, 8
+; CHECK-FISL-DAG: fmr [[F1]], 1
+; CHECK-FISL-DAG: xsmaddasp [[F1]], 2, 4
+; CHECK-FISL-DAG: li [[C3:[0-9]+]], 4
+; CHECK-FISL-DAG: stxsspx 0, 8, [[C3]]
+; CHECK-FISL-DAG: xsmaddasp 0, 2, 3
+; CHECK-FISL-DAG: li [[C1:[0-9]+]], 12
+; CHECK-FISL-DAG: stxsspx 0, 8, [[C1]]
+; CHECK-FISL-DAG: xsmaddasp 1, 2, 5
+; CHECK-FISL-DAG: li [[C2:[0-9]+]], 8
+; CHECK-FISL-DAG: stxsspx 1, 8, [[C2]]
+; CHECK-FISL: blr
+}
+
+declare float @llvm.fma.f32(float, float, float) #0
diff --git a/test/CodeGen/PowerPC/vsx-ldst-builtin-le.ll b/test/CodeGen/PowerPC/vsx-ldst-builtin-le.ll
index e6ddf647616..d6940e46df3 100644
--- a/test/CodeGen/PowerPC/vsx-ldst-builtin-le.ll
+++ b/test/CodeGen/PowerPC/vsx-ldst-builtin-le.ll
@@ -1,7 +1,6 @@
 ; RUN: llc -mcpu=pwr8 -mattr=+vsx -O2 -mtriple=powerpc64le-unknown-linux-gnu < %s > %t
 ; RUN: grep lxvd2x < %t | count 18
 ; RUN: grep stxvd2x < %t | count 18
-; RUN: grep xxpermdi < %t | count 36
 
 @vf = global <4 x float> <float -1.500000e+00, float 2.500000e+00, float -3.500000e+00, float 4.500000e+00>, align 16
 @vd = global <2 x double> <double 3.500000e+00, double -7.500000e+00>, align 16
diff --git a/test/CodeGen/PowerPC/vsx-ldst.ll b/test/CodeGen/PowerPC/vsx-ldst.ll
index 4ed91bca420..7f12b0480e1 100644
--- a/test/CodeGen/PowerPC/vsx-ldst.ll
+++ b/test/CodeGen/PowerPC/vsx-ldst.ll
@@ -12,7 +12,6 @@
 ; RUN: llc -mcpu=pwr8 -mattr=+vsx -O2 -mtriple=powerpc64le-unknown-linux-gnu < %s > %t
 ; RUN: grep lxvd2x < %t | count 6
 ; RUN: grep stxvd2x < %t | count 6
-; RUN: grep xxpermdi < %t | count 12
 
 @vsi = global <4 x i32> <i32 -1, i32 2, i32 -3, i32 4>, align 16
 @vui = global <4 x i32> <i32 0, i32 1, i32 2, i32 3>, align 16
diff --git a/test/CodeGen/PowerPC/vsx-recip-est.ll b/test/CodeGen/PowerPC/vsx-recip-est.ll
new file mode 100644
index 00000000000..f589c6c103e
--- /dev/null
+++ b/test/CodeGen/PowerPC/vsx-recip-est.ll
@@ -0,0 +1,62 @@
+; RUN: llc < %s -mtriple=powerpc64-unknown-linux-gnu -mcpu=pwr8 -enable-unsafe-fp-math | FileCheck %s
+; RUN: llc < %s -mtriple=powerpc64le-unknown-linux-gnu -mcpu=pwr8 -enable-unsafe-fp-math | FileCheck %s
+@a = global float 3.000000e+00, align 4
+@b = global float 4.000000e+00, align 4
+@c = global double 3.000000e+00, align 8
+@d = global double 4.000000e+00, align 8
+
+; Function Attrs: nounwind
+define float @emit_xsresp() {
+entry:
+  %0 = load float, float* @a, align 4
+  %1 = load float, float* @b, align 4
+  %div = fdiv fast float %0, %1
+  ret float %div
+; CHECK-LABEL: @emit_xsresp
+; CHECK: xsresp {{[0-9]+}}
+}
+
+; Function Attrs: nounwind
+define float @emit_xsrsqrtesp(float %f) {
+entry:
+  %f.addr = alloca float, align 4
+  store float %f, float* %f.addr, align 4
+  %0 = load float, float* %f.addr, align 4
+  %1 = load float, float* @b, align 4
+  %2 = call float @llvm.sqrt.f32(float %1)
+  %div = fdiv fast float %0, %2
+  ret float %div
+; CHECK-LABEL: @emit_xsrsqrtesp
+; CHECK: xsrsqrtesp {{[0-9]+}}
+}
+
+; Function Attrs: nounwind readnone
+declare float @llvm.sqrt.f32(float)
+
+; Function Attrs: nounwind
+define double @emit_xsredp() {
+entry:
+  %0 = load double, double* @c, align 8
+  %1 = load double, double* @d, align 8
+  %div = fdiv fast double %0, %1
+  ret double %div
+; CHECK-LABEL: @emit_xsredp
+; CHECK: xsredp {{[0-9]+}}
+}
+
+; Function Attrs: nounwind
+define double @emit_xsrsqrtedp(double %f) {
+entry:
+  %f.addr = alloca double, align 8
+  store double %f, double* %f.addr, align 8
+  %0 = load double, double* %f.addr, align 8
+  %1 = load double, double* @d, align 8
+  %2 = call double @llvm.sqrt.f64(double %1)
+  %div = fdiv fast double %0, %2
+  ret double %div
+; CHECK-LABEL: @emit_xsrsqrtedp
+; CHECK: xsrsqrtedp {{[0-9]+}}
+}
+
+; Function Attrs: nounwind readnone
+declare double @llvm.sqrt.f64(double) #1
diff --git a/test/CodeGen/PowerPC/vsx.ll b/test/CodeGen/PowerPC/vsx.ll
index 25cf3d4750a..dceb2516c69 100644
--- a/test/CodeGen/PowerPC/vsx.ll
+++ b/test/CodeGen/PowerPC/vsx.ll
@@ -1,9 +1,8 @@
-; RUN: llc -mcpu=pwr7 -mattr=+vsx < %s | FileCheck %s
-; RUN: llc -mcpu=pwr7 -mattr=+vsx < %s | FileCheck -check-prefix=CHECK-REG %s
-; RUN: llc -mcpu=pwr7 -mattr=+vsx -fast-isel -O0 < %s | FileCheck %s
-; RUN: llc -mcpu=pwr7 -mattr=+vsx -fast-isel -O0 < %s | FileCheck -check-prefix=CHECK-FISL %s
-target datalayout = "E-m:e-i64:64-n32:64"
-target triple = "powerpc64-unknown-linux-gnu"
+; RUN: llc -mcpu=pwr7 -mtriple=powerpc64-unknown-linux-gnu -mattr=+vsx < %s | FileCheck %s
+; RUN: llc -mcpu=pwr7 -mtriple=powerpc64-unknown-linux-gnu -mattr=+vsx < %s | FileCheck -check-prefix=CHECK-REG %s
+; RUN: llc -mcpu=pwr7 -mtriple=powerpc64-unknown-linux-gnu -mattr=+vsx -fast-isel -O0 < %s | FileCheck %s
+; RUN: llc -mcpu=pwr7 -mtriple=powerpc64-unknown-linux-gnu -mattr=+vsx -fast-isel -O0 < %s | FileCheck -check-prefix=CHECK-FISL %s
+; RUN: llc -mcpu=pwr8 -mtriple=powerpc64le-unknown-linux-gnu -mattr=+vsx < %s | FileCheck -check-prefix=CHECK-LE %s
 
 define double @test1(double %a, double %b) {
 entry:
@@ -13,6 +12,10 @@ entry:
 ; CHECK-LABEL: @test1
 ; CHECK: xsmuldp 1, 1, 2
 ; CHECK: blr
+
+; CHECK-LE-LABEL: @test1
+; CHECK-LE: xsmuldp 1, 1, 2
+; CHECK-LE: blr
 }
 
 define double @test2(double %a, double %b) {
@@ -23,6 +26,10 @@ entry:
 ; CHECK-LABEL: @test2
 ; CHECK: xsdivdp 1, 1, 2
 ; CHECK: blr
+
+; CHECK-LE-LABEL: @test2
+; CHECK-LE: xsdivdp 1, 1, 2
+; CHECK-LE: blr
 }
 
 define double @test3(double %a, double %b) {
@@ -33,6 +40,10 @@ entry:
 ; CHECK-LABEL: @test3
 ; CHECK: xsadddp 1, 1, 2
 ; CHECK: blr
+
+; CHECK-LE-LABEL: @test3
+; CHECK-LE: xsadddp 1, 1, 2
+; CHECK-LE: blr
 }
 
 define <2 x double> @test4(<2 x double> %a, <2 x double> %b) {
@@ -43,6 +54,10 @@ entry:
 ; CHECK-LABEL: @test4
 ; CHECK: xvadddp 34, 34, 35
 ; CHECK: blr
+
+; CHECK-LE-LABEL: @test4
+; CHECK-LE: xvadddp 34, 34, 35
+; CHECK-LE: blr
 }
 
 define <4 x i32> @test5(<4 x i32> %a, <4 x i32> %b) {
@@ -55,11 +70,15 @@ entry:
 ; CHECK-REG: blr
 
 ; CHECK-FISL-LABEL: @test5
-; CHECK-FISL: vor 4, 2, 2
-; CHECK-FISL: vor 5, 3, 3
-; CHECK-FISL: xxlxor 36, 36, 37
-; CHECK-FISL: vor 2, 4, 4
+; CHECK-FISL: vor
+; CHECK-FISL: vor
+; CHECK-FISL: xxlxor
+; CHECK-FISL: vor 2
 ; CHECK-FISL: blr
+
+; CHECK-LE-LABEL: @test5
+; CHECK-LE: xxlxor 34, 34, 35
+; CHECK-LE: blr
 }
 
 define <8 x i16> @test6(<8 x i16> %a, <8 x i16> %b) {
@@ -77,6 +96,10 @@ entry:
 ; CHECK-FISL: xxlxor 36, 36, 37
 ; CHECK-FISL: vor 2, 4, 4
 ; CHECK-FISL: blr
+
+; CHECK-LE-LABEL: @test6
+; CHECK-LE: xxlxor 34, 34, 35
+; CHECK-LE: blr
 }
 
 define <16 x i8> @test7(<16 x i8> %a, <16 x i8> %b) {
@@ -94,6 +117,10 @@ entry:
 ; CHECK-FISL: xxlxor 36, 36, 37
 ; CHECK-FISL: vor 2, 4, 4
 ; CHECK-FISL: blr
+
+; CHECK-LE-LABEL: @test7
+; CHECK-LE: xxlxor 34, 34, 35
+; CHECK-LE: blr
 }
 
 define <4 x i32> @test8(<4 x i32> %a, <4 x i32> %b) {
@@ -106,11 +133,15 @@ entry:
 ; CHECK-REG: blr
 
 ; CHECK-FISL-LABEL: @test8
-; CHECK-FISL: vor 4, 2, 2
-; CHECK-FISL: vor 5, 3, 3
-; CHECK-FISL: xxlor 36, 36, 37
-; CHECK-FISL: vor 2, 4, 4
+; CHECK-FISL: vor
+; CHECK-FISL: vor
+; CHECK-FISL: xxlor
+; CHECK-FISL: vor 2
 ; CHECK-FISL: blr
+
+; CHECK-LE-LABEL: @test8
+; CHECK-LE: xxlor 34, 34, 35
+; CHECK-LE: blr
 }
 
 define <8 x i16> @test9(<8 x i16> %a, <8 x i16> %b) {
@@ -128,6 +159,10 @@ entry:
 ; CHECK-FISL: xxlor 36, 36, 37
 ; CHECK-FISL: vor 2, 4, 4
 ; CHECK-FISL: blr
+
+; CHECK-LE-LABEL: @test9
+; CHECK-LE: xxlor 34, 34, 35
+; CHECK-LE: blr
 }
 
 define <16 x i8> @test10(<16 x i8> %a, <16 x i8> %b) {
@@ -145,6 +180,10 @@ entry:
 ; CHECK-FISL: xxlor 36, 36, 37
 ; CHECK-FISL: vor 2, 4, 4
 ; CHECK-FISL: blr
+
+; CHECK-LE-LABEL: @test10
+; CHECK-LE: xxlor 34, 34, 35
+; CHECK-LE: blr
 }
 
 define <4 x i32> @test11(<4 x i32> %a, <4 x i32> %b) {
@@ -157,11 +196,15 @@ entry:
 ; CHECK-REG: blr
 
 ; CHECK-FISL-LABEL: @test11
-; CHECK-FISL: vor 4, 2, 2
-; CHECK-FISL: vor 5, 3, 3
-; CHECK-FISL: xxland 36, 36, 37
-; CHECK-FISL: vor 2, 4, 4
+; CHECK-FISL: vor
+; CHECK-FISL: vor
+; CHECK-FISL: xxland
+; CHECK-FISL: vor 2
 ; CHECK-FISL: blr
+
+; CHECK-LE-LABEL: @test11
+; CHECK-LE: xxland 34, 34, 35
+; CHECK-LE: blr
 }
 
 define <8 x i16> @test12(<8 x i16> %a, <8 x i16> %b) {
@@ -179,6 +222,10 @@ entry:
 ; CHECK-FISL: xxland 36, 36, 37
 ; CHECK-FISL: vor 2, 4, 4
 ; CHECK-FISL: blr
+
+; CHECK-LE-LABEL: @test12
+; CHECK-LE: xxland 34, 34, 35
+; CHECK-LE: blr
 }
 
 define <16 x i8> @test13(<16 x i8> %a, <16 x i8> %b) {
@@ -196,6 +243,10 @@ entry:
 ; CHECK-FISL: xxland 36, 36, 37
 ; CHECK-FISL: vor 2, 4, 4
 ; CHECK-FISL: blr
+
+; CHECK-LE-LABEL: @test13
+; CHECK-LE: xxland 34, 34, 35
+; CHECK-LE: blr
 }
 
 define <4 x i32> @test14(<4 x i32> %a, <4 x i32> %b) {
@@ -209,18 +260,19 @@ entry:
 ; CHECK-REG: blr
 
 ; CHECK-FISL-LABEL: @test14
-; CHECK-FISL: vor 4, 2, 2
-; CHECK-FISL: vor 5, 3, 3
-; CHECK-FISL: xxlor 36, 36, 37
-; CHECK-FISL: vor 0, 4, 4
-; CHECK-FISL: vor 4, 2, 2
-; CHECK-FISL: vor 5, 3, 3
-; CHECK-FISL: xxlnor 36, 36, 37
+; CHECK-FISL: vor 4, 3, 3
+; CHECK-FISL: vor 5, 2, 2
+; CHECK-FISL: xxlor 0, 37, 36
+; CHECK-FISL: xxlnor 36, 37, 36
 ; CHECK-FISL: vor 2, 4, 4
 ; CHECK-FISL: lis 0, -1
 ; CHECK-FISL: ori 0, 0, 65520
-; CHECK-FISL: stvx 0, 1, 0
+; CHECK-FISL: stxvd2x 0, 1, 0
 ; CHECK-FISL: blr
+
+; CHECK-LE-LABEL: @test14
+; CHECK-LE: xxlnor 34, 34, 35
+; CHECK-LE: blr
 }
 
 define <8 x i16> @test15(<8 x i16> %a, <8 x i16> %b) {
@@ -246,6 +298,10 @@ entry:
 ; CHECK-FISL: ori 0, 0, 65520
 ; CHECK-FISL: stvx 0, 1, 0
 ; CHECK-FISL: blr
+
+; CHECK-LE-LABEL: @test15
+; CHECK-LE: xxlnor 34, 34, 35
+; CHECK-LE: blr
 }
 
 define <16 x i8> @test16(<16 x i8> %a, <16 x i8> %b) {
@@ -271,6 +327,10 @@ entry:
 ; CHECK-FISL: ori 0, 0, 65520
 ; CHECK-FISL: stvx 0, 1, 0
 ; CHECK-FISL: blr
+
+; CHECK-LE-LABEL: @test16
+; CHECK-LE: xxlnor 34, 34, 35
+; CHECK-LE: blr
 }
 
 define <4 x i32> @test17(<4 x i32> %a, <4 x i32> %b) {
@@ -284,16 +344,18 @@ entry:
 ; CHECK-REG: blr
 
 ; CHECK-FISL-LABEL: @test17
-; CHECK-FISL: vspltisb 4, -1
-; CHECK-FISL: vor 5, 3, 3
-; CHECK-FISL: vor 0, 4, 4
-; CHECK-FISL: xxlxor 37, 37, 32
-; CHECK-FISL: vor 3, 5, 5
+; CHECK-FISL: vor 4, 3, 3
 ; CHECK-FISL: vor 5, 2, 2
-; CHECK-FISL: vor 0, 3, 3
-; CHECK-FISL: xxland 37, 37, 32
-; CHECK-FISL: vor 2, 5, 5
+; CHECK-FISL: vspltisb 2, -1
+; CHECK-FISL: vor 0, 2, 2
+; CHECK-FISL: xxlxor 36, 36, 32
+; CHECK-FISL: xxland 36, 37, 36
+; CHECK-FISL: vor 2, 4, 4
 ; CHECK-FISL: blr
+
+; CHECK-LE-LABEL: @test17
+; CHECK-LE: xxlandc 34, 34, 35
+; CHECK-LE: blr
 }
 
 define <8 x i16> @test18(<8 x i16> %a, <8 x i16> %b) {
@@ -320,6 +382,10 @@ entry:
 ; CHECK-FISL: ori 0, 0, 65520
 ; CHECK-FISL: stvx 4, 1, 0
 ; CHECK-FISL: blr
+
+; CHECK-LE-LABEL: @test18
+; CHECK-LE: xxlandc 34, 34, 35
+; CHECK-LE: blr
 }
 
 define <16 x i8> @test19(<16 x i8> %a, <16 x i8> %b) {
@@ -346,6 +412,10 @@ entry:
 ; CHECK-FISL: ori 0, 0, 65520
 ; CHECK-FISL: stvx 4, 1, 0
 ; CHECK-FISL: blr
+
+; CHECK-LE-LABEL: @test19
+; CHECK-LE: xxlandc 34, 34, 35
+; CHECK-LE: blr
 }
 
 define <4 x i32> @test20(<4 x i32> %a, <4 x i32> %b, <4 x i32> %c, <4 x i32> %d) {
@@ -359,14 +429,25 @@ entry:
 ; CHECK-REG: xxsel 34, 35, 34, {{[0-9]+}}
 ; CHECK-REG: blr
 
+; FIXME: The fast-isel code is pretty miserable for this one.
+
 ; CHECK-FISL-LABEL: @test20
-; CHECK-FISL: vcmpequw 4, 4, 5
-; CHECK-FISL: vor 0, 3, 3
-; CHECK-FISL: vor 1, 2, 2
-; CHECK-FISL: vor 6, 4, 4
-; CHECK-FISL: xxsel 32, 32, 33, 38
+; CHECK-FISL: vor 0, 5, 5
+; CHECK-FISL: vor 1, 4, 4
+; CHECK-FISL: vor 6, 3, 3
+; CHECK-FISL: vor 7, 2, 2
+; CHECK-FISL: vor 2, 1, 1
+; CHECK-FISL: vor 3, 0, 0
+; CHECK-FISL: vcmpequw 2, 2, 3
+; CHECK-FISL: vor 0, 2, 2
+; CHECK-FISL: xxsel 32, 38, 39, 32
 ; CHECK-FISL: vor 2, 0, 0
 ; CHECK-FISL: blr
+
+; CHECK-LE-LABEL: @test20
+; CHECK-LE: vcmpequw {{[0-9]+}}, 4, 5
+; CHECK-LE: xxsel 34, 35, 34, {{[0-9]+}}
+; CHECK-LE: blr
 }
 
 define <4 x float> @test21(<4 x float> %a, <4 x float> %b, <4 x float> %c, <4 x float> %d) {
@@ -389,6 +470,11 @@ entry:
 ; CHECK-FISL: xxsel 32, 38, 39, 32
 ; CHECK-FISL: vor 2, 0, 0
 ; CHECK-FISL: blr
+
+; CHECK-LE-LABEL: @test21
+; CHECK-LE: xvcmpeqsp [[V1:[0-9]+]], 36, 37
+; CHECK-LE: xxsel 34, 35, 34, [[V1]]
+; CHECK-LE: blr
 }
 
 define <4 x float> @test22(<4 x float> %a, <4 x float> %b, <4 x float> %c, <4 x float> %d) {
@@ -418,6 +504,17 @@ entry:
 ; CHECK-FISL-DAG: xxlor
 ; CHECK-FISL: xxsel 0, 38, 39, {{[0-9]+}}
 ; CHECK-FISL: blr
+
+; CHECK-LE-LABEL: @test22
+; CHECK-LE-DAG: xvcmpeqsp {{[0-9]+}}, 37, 37
+; CHECK-LE-DAG: xvcmpeqsp {{[0-9]+}}, 36, 36
+; CHECK-LE-DAG: xvcmpeqsp {{[0-9]+}}, 36, 37
+; CHECK-LE-DAG: xxlnor
+; CHECK-LE-DAG: xxlnor
+; CHECK-LE-DAG: xxlor
+; CHECK-LE-DAG: xxlor
+; CHECK-LE: xxsel 34, 35, 34, {{[0-9]+}}
+; CHECK-LE: blr
 }
 
 define <8 x i16> @test23(<8 x i16> %a, <8 x i16> %b, <8 x i16> %c, <8 x i16> %d) {
@@ -439,6 +536,11 @@ entry:
 ; CHECK-FISL: xxsel 32, 32, 33, 38
 ; CHECK-FISL: vor 2, 0, 
 ; CHECK-FISL: blr
+
+; CHECK-LE-LABEL: @test23
+; CHECK-LE: vcmpequh {{[0-9]+}}, 4, 5
+; CHECK-LE: xxsel 34, 35, 34, {{[0-9]+}}
+; CHECK-LE: blr
 }
 
 define <16 x i8> @test24(<16 x i8> %a, <16 x i8> %b, <16 x i8> %c, <16 x i8> %d) {
@@ -460,6 +562,11 @@ entry:
 ; CHECK-FISL: xxsel 32, 32, 33, 38
 ; CHECK-FISL: vor 2, 0, 0
 ; CHECK-FISL: blr
+
+; CHECK-LE-LABEL: @test24
+; CHECK-LE: vcmpequb {{[0-9]+}}, 4, 5
+; CHECK-LE: xxsel 34, 35, 34, {{[0-9]+}}
+; CHECK-LE: blr
 }
 
 define <2 x double> @test25(<2 x double> %a, <2 x double> %b, <2 x double> %c, <2 x double> %d) {
@@ -472,6 +579,11 @@ entry:
 ; CHECK: xvcmpeqdp [[V1:[0-9]+]], 36, 37
 ; CHECK: xxsel 34, 35, 34, [[V1]]
 ; CHECK: blr
+
+; CHECK-LE-LABEL: @test25
+; CHECK-LE: xvcmpeqdp [[V1:[0-9]+]], 36, 37
+; CHECK-LE: xxsel 34, 35, 34, [[V1]]
+; CHECK-LE: blr
 }
 
 define <2 x i64> @test26(<2 x i64> %a, <2 x i64> %b) {
@@ -489,6 +601,9 @@ define <2 x i64> @test26(<2 x i64> %a, <2 x i64> %b) {
 ; CHECK: add
 ; CHECK: add
 ; CHECK: blr
+
+; CHECK-LE: vaddudm 2, 2, 3
+; CHECK-LE: blr
 }
 
 define <2 x i64> @test27(<2 x i64> %a, <2 x i64> %b) {
@@ -498,6 +613,10 @@ define <2 x i64> @test27(<2 x i64> %a, <2 x i64> %b) {
 ; CHECK-LABEL: @test27
 ; CHECK: xxland 34, 34, 35
 ; CHECK: blr
+
+; CHECK-LE-LABEL: @test27
+; CHECK-LE: xxland 34, 34, 35
+; CHECK-LE: blr
 }
 
 define <2 x double> @test28(<2 x double>* %a) {
@@ -507,6 +626,11 @@ define <2 x double> @test28(<2 x double>* %a) {
 ; CHECK-LABEL: @test28
 ; CHECK: lxvd2x 34, 0, 3
 ; CHECK: blr
+
+; CHECK-LE-LABEL: @test28
+; CHECK-LE: lxvd2x [[V1:[0-9]+]], 0, 3
+; CHECK-LE: xxswapd 34, [[V1]]
+; CHECK-LE: blr
 }
 
 define void @test29(<2 x double>* %a, <2 x double> %b) {
@@ -516,6 +640,11 @@ define void @test29(<2 x double>* %a, <2 x double> %b) {
 ; CHECK-LABEL: @test29
 ; CHECK: stxvd2x 34, 0, 3
 ; CHECK: blr
+
+; CHECK-LE-LABEL: @test29
+; CHECK-LE: xxswapd [[V1:[0-9]+]], 34
+; CHECK-LE: stxvd2x [[V1]], 0, 3
+; CHECK-LE: blr
 }
 
 define <2 x double> @test28u(<2 x double>* %a) {
@@ -525,6 +654,11 @@ define <2 x double> @test28u(<2 x double>* %a) {
 ; CHECK-LABEL: @test28u
 ; CHECK: lxvd2x 34, 0, 3
 ; CHECK: blr
+
+; CHECK-LE-LABEL: @test28u
+; CHECK-LE: lxvd2x [[V1:[0-9]+]], 0, 3
+; CHECK-LE: xxswapd 34, [[V1]]
+; CHECK-LE: blr
 }
 
 define void @test29u(<2 x double>* %a, <2 x double> %b) {
@@ -534,6 +668,11 @@ define void @test29u(<2 x double>* %a, <2 x double> %b) {
 ; CHECK-LABEL: @test29u
 ; CHECK: stxvd2x 34, 0, 3
 ; CHECK: blr
+
+; CHECK-LE-LABEL: @test29u
+; CHECK-LE: xxswapd [[V1:[0-9]+]], 34
+; CHECK-LE: stxvd2x [[V1]], 0, 3
+; CHECK-LE: blr
 }
 
 define <2 x i64> @test30(<2 x i64>* %a) {
@@ -550,6 +689,11 @@ define <2 x i64> @test30(<2 x i64>* %a) {
 ; CHECK-FISL: vor 3, 2, 2
 ; CHECK-FISL: vor 2, 3, 3
 ; CHECK-FISL: blr
+
+; CHECK-LE-LABEL: @test30
+; CHECK-LE: lxvd2x [[V1:[0-9]+]], 0, 3
+; CHECK-LE: xxswapd 34, [[V1]]
+; CHECK-LE: blr
 }
 
 define void @test31(<2 x i64>* %a, <2 x i64> %b) {
@@ -559,6 +703,11 @@ define void @test31(<2 x i64>* %a, <2 x i64> %b) {
 ; CHECK-LABEL: @test31
 ; CHECK: stxvd2x 34, 0, 3
 ; CHECK: blr
+
+; CHECK-LE-LABEL: @test31
+; CHECK-LE: xxswapd [[V1:[0-9]+]], 34
+; CHECK-LE: stxvd2x [[V1]], 0, 3
+; CHECK-LE: blr
 }
 
 define <4 x float> @test32(<4 x float>* %a) {
@@ -573,6 +722,11 @@ define <4 x float> @test32(<4 x float>* %a) {
 ; CHECK-FISL: lxvw4x 0, 0, 3
 ; CHECK-FISL: xxlor 34, 0, 0
 ; CHECK-FISL: blr
+
+; CHECK-LE-LABEL: @test32
+; CHECK-LE: lxvd2x [[V1:[0-9]+]], 0, 3
+; CHECK-LE: xxswapd 34, [[V1]]
+; CHECK-LE: blr
 }
 
 define void @test33(<4 x float>* %a, <4 x float> %b) {
@@ -587,6 +741,11 @@ define void @test33(<4 x float>* %a, <4 x float> %b) {
 ; CHECK-FISL: vor 3, 2, 2
 ; CHECK-FISL: stxvw4x 35, 0, 3
 ; CHECK-FISL: blr
+
+; CHECK-LE-LABEL: @test33
+; CHECK-LE: xxswapd [[V1:[0-9]+]], 34
+; CHECK-LE: stxvd2x [[V1]], 0, 3
+; CHECK-LE: blr
 }
 
 define <4 x float> @test32u(<4 x float>* %a) {
@@ -599,6 +758,11 @@ define <4 x float> @test32u(<4 x float>* %a) {
 ; CHECK-DAG: lvx
 ; CHECK: vperm 2,
 ; CHECK: blr
+
+; CHECK-LE-LABEL: @test32u
+; CHECK-LE: lxvd2x [[V1:[0-9]+]], 0, 3
+; CHECK-LE: xxswapd 34, [[V1]]
+; CHECK-LE: blr
 }
 
 define void @test33u(<4 x float>* %a, <4 x float> %b) {
@@ -613,6 +777,11 @@ define void @test33u(<4 x float>* %a, <4 x float> %b) {
 ; CHECK-FISL: vor 3, 2, 2
 ; CHECK-FISL: stxvw4x 35, 0, 3
 ; CHECK-FISL: blr
+
+; CHECK-LE-LABEL: @test33u
+; CHECK-LE: xxswapd [[V1:[0-9]+]], 34
+; CHECK-LE: stxvd2x [[V1]], 0, 3
+; CHECK-LE: blr
 }
 
 define <4 x i32> @test34(<4 x i32>* %a) {
@@ -626,9 +795,12 @@ define <4 x i32> @test34(<4 x i32>* %a) {
 ; CHECK-FISL-LABEL: @test34
 ; CHECK-FISL: lxvw4x 0, 0, 3
 ; CHECK-FISL: xxlor 34, 0, 0
-; CHECK-FISL: vor 3, 2, 2
-; CHECK-FISL: vor 2, 3, 3
 ; CHECK-FISL: blr
+
+; CHECK-LE-LABEL: @test34
+; CHECK-LE: lxvd2x [[V1:[0-9]+]], 0, 3
+; CHECK-LE: xxswapd 34, [[V1]]
+; CHECK-LE: blr
 }
 
 define void @test35(<4 x i32>* %a, <4 x i32> %b) {
@@ -643,6 +815,11 @@ define void @test35(<4 x i32>* %a, <4 x i32> %b) {
 ; CHECK-FISL: vor 3, 2, 2
 ; CHECK-FISL: stxvw4x 35, 0, 3
 ; CHECK-FISL: blr
+
+; CHECK-LE-LABEL: @test35
+; CHECK-LE: xxswapd [[V1:[0-9]+]], 34
+; CHECK-LE: stxvd2x [[V1]], 0, 3
+; CHECK-LE: blr
 }
 
 define <2 x double> @test40(<2 x i64> %a) {
@@ -652,6 +829,10 @@ define <2 x double> @test40(<2 x i64> %a) {
 ; CHECK-LABEL: @test40
 ; CHECK: xvcvuxddp 34, 34
 ; CHECK: blr
+
+; CHECK-LE-LABEL: @test40
+; CHECK-LE: xvcvuxddp 34, 34
+; CHECK-LE: blr
 }
 
 define <2 x double> @test41(<2 x i64> %a) {
@@ -661,6 +842,10 @@ define <2 x double> @test41(<2 x i64> %a) {
 ; CHECK-LABEL: @test41
 ; CHECK: xvcvsxddp 34, 34
 ; CHECK: blr
+
+; CHECK-LE-LABEL: @test41
+; CHECK-LE: xvcvsxddp 34, 34
+; CHECK-LE: blr
 }
 
 define <2 x i64> @test42(<2 x double> %a) {
@@ -670,6 +855,10 @@ define <2 x i64> @test42(<2 x double> %a) {
 ; CHECK-LABEL: @test42
 ; CHECK: xvcvdpuxds 34, 34
 ; CHECK: blr
+
+; CHECK-LE-LABEL: @test42
+; CHECK-LE: xvcvdpuxds 34, 34
+; CHECK-LE: blr
 }
 
 define <2 x i64> @test43(<2 x double> %a) {
@@ -679,6 +868,10 @@ define <2 x i64> @test43(<2 x double> %a) {
 ; CHECK-LABEL: @test43
 ; CHECK: xvcvdpsxds 34, 34
 ; CHECK: blr
+
+; CHECK-LE-LABEL: @test43
+; CHECK-LE: xvcvdpsxds 34, 34
+; CHECK-LE: blr
 }
 
 define <2 x float> @test44(<2 x i64> %a) {
@@ -726,6 +919,10 @@ define <2 x double> @test50(double* %a) {
 ; CHECK-LABEL: @test50
 ; CHECK: lxvdsx 34, 0, 3
 ; CHECK: blr
+
+; CHECK-LE-LABEL: @test50
+; CHECK-LE: lxvdsx 34, 0, 3
+; CHECK-LE: blr
 }
 
 define <2 x double> @test51(<2 x double> %a, <2 x double> %b) {
@@ -733,8 +930,12 @@ define <2 x double> @test51(<2 x double> %a, <2 x double> %b) {
   ret <2 x double> %v
 
 ; CHECK-LABEL: @test51
-; CHECK: xxpermdi 34, 34, 34, 0
+; CHECK: xxspltd 34, 34, 0
 ; CHECK: blr
+
+; CHECK-LE-LABEL: @test51
+; CHECK-LE: xxspltd 34, 34, 1
+; CHECK-LE: blr
 }
 
 define <2 x double> @test52(<2 x double> %a, <2 x double> %b) {
@@ -742,8 +943,12 @@ define <2 x double> @test52(<2 x double> %a, <2 x double> %b) {
   ret <2 x double> %v
 
 ; CHECK-LABEL: @test52
-; CHECK: xxpermdi 34, 34, 35, 0
+; CHECK: xxmrghd 34, 34, 35
 ; CHECK: blr
+
+; CHECK-LE-LABEL: @test52
+; CHECK-LE: xxmrgld 34, 35, 34
+; CHECK-LE: blr
 }
 
 define <2 x double> @test53(<2 x double> %a, <2 x double> %b) {
@@ -751,8 +956,12 @@ define <2 x double> @test53(<2 x double> %a, <2 x double> %b) {
   ret <2 x double> %v
 
 ; CHECK-LABEL: @test53
-; CHECK: xxpermdi 34, 35, 34, 0
+; CHECK: xxmrghd 34, 35, 34
 ; CHECK: blr
+
+; CHECK-LE-LABEL: @test53
+; CHECK-LE: xxmrgld 34, 34, 35
+; CHECK-LE: blr
 }
 
 define <2 x double> @test54(<2 x double> %a, <2 x double> %b) {
@@ -762,6 +971,10 @@ define <2 x double> @test54(<2 x double> %a, <2 x double> %b) {
 ; CHECK-LABEL: @test54
 ; CHECK: xxpermdi 34, 34, 35, 2
 ; CHECK: blr
+
+; CHECK-LE-LABEL: @test54
+; CHECK-LE: xxpermdi 34, 35, 34, 2
+; CHECK-LE: blr
 }
 
 define <2 x double> @test55(<2 x double> %a, <2 x double> %b) {
@@ -769,8 +982,12 @@ define <2 x double> @test55(<2 x double> %a, <2 x double> %b) {
   ret <2 x double> %v
 
 ; CHECK-LABEL: @test55
-; CHECK: xxpermdi 34, 34, 35, 3
+; CHECK: xxmrgld 34, 34, 35
 ; CHECK: blr
+
+; CHECK-LE-LABEL: @test55
+; CHECK-LE: xxmrghd 34, 35, 34
+; CHECK-LE: blr
 }
 
 define <2 x i64> @test56(<2 x i64> %a, <2 x i64> %b) {
@@ -778,8 +995,12 @@ define <2 x i64> @test56(<2 x i64> %a, <2 x i64> %b) {
   ret <2 x i64> %v
 
 ; CHECK-LABEL: @test56
-; CHECK: xxpermdi 34, 34, 35, 3
+; CHECK: xxmrgld 34, 34, 35
 ; CHECK: blr
+
+; CHECK-LE-LABEL: @test56
+; CHECK-LE: xxmrghd 34, 35, 34
+; CHECK-LE: blr
 }
 
 define <2 x i64> @test60(<2 x i64> %a, <2 x i64> %b) {
@@ -836,6 +1057,10 @@ define double @test63(<2 x double> %a) {
 ; CHECK-FISL: xxlor 0, 34, 34
 ; CHECK-FISL: fmr 1, 0
 ; CHECK-FISL: blr
+
+; CHECK-LE-LABEL: @test63
+; CHECK-LE: xxswapd 1, 34
+; CHECK-LE: blr
 }
 
 define double @test64(<2 x double> %a) {
@@ -843,14 +1068,17 @@ define double @test64(<2 x double> %a) {
   ret double %v
 
 ; CHECK-REG-LABEL: @test64
-; CHECK-REG: xxpermdi 1, 34, 34, 2
+; CHECK-REG: xxswapd 1, 34
 ; CHECK-REG: blr
 
 ; CHECK-FISL-LABEL: @test64
-; CHECK-FISL: xxpermdi 34, 34, 34, 2
+; CHECK-FISL: xxswapd  34, 34
 ; CHECK-FISL: xxlor 0, 34, 34
 ; CHECK-FISL: fmr 1, 0
 ; CHECK-FISL: blr
+
+; CHECK-LE-LABEL: @test64
+; CHECK-LE: xxlor 1, 34, 34
 }
 
 define <2 x i1> @test65(<2 x i64> %a, <2 x i64> %b) {
@@ -867,6 +1095,10 @@ define <2 x i1> @test65(<2 x i64> %a, <2 x i64> %b) {
 ; CHECK-FISL: vcmpequw 4, 5, 4
 ; CHECK-FISL: vor 2, 4, 4
 ; CHECK-FISL: blr
+
+; CHECK-LE-LABEL: @test65
+; CHECK-LE: vcmpequd 2, 2, 3
+; CHECK-LE: blr
 }
 
 define <2 x i1> @test66(<2 x i64> %a, <2 x i64> %b) {
@@ -882,6 +1114,11 @@ define <2 x i1> @test66(<2 x i64> %a, <2 x i64> %b) {
 ; CHECK-FISL: vcmpequw {{[0-9]+}}, 5, 4
 ; CHECK-FISL: xxlnor 34, {{[0-9]+}}, {{[0-9]+}}
 ; CHECK-FISL: blr
+
+; CHECK-LE-LABEL: @test66
+; CHECK-LE: vcmpequd {{[0-9]+}}, 2, 3
+; CHECK-LE: xxlnor 34, {{[0-9]+}}, {{[0-9]+}}
+; CHECK-LE: blr
 }
 
 define <2 x i1> @test67(<2 x i64> %a, <2 x i64> %b) {
@@ -896,6 +1133,10 @@ define <2 x i1> @test67(<2 x i64> %a, <2 x i64> %b) {
 ; CHECK: cmpld
 ; CHECK: lxvd2x
 ; CHECK: blr
+
+; CHECK-LE-LABEL: @test67
+; CHECK-LE: vcmpgtud 2, 3, 2
+; CHECK-LE: blr
 }
 
 define <2 x double> @test68(<2 x i32> %a) {
@@ -906,6 +1147,11 @@ define <2 x double> @test68(<2 x i32> %a) {
 ; CHECK: xxsldwi [[V1:[0-9]+]], 34, 34, 1
 ; CHECK: xvcvsxwdp 34, [[V1]]
 ; CHECK: blr
+
+; CHECK-LE-LABEL: @test68
+; CHECK-LE: xxsldwi [[V1:[0-9]+]], 34, 34, 1
+; CHECK-LE: xvcvsxwdp 34, [[V1]]
+; CHECK-LE: blr
 }
 
 define <2 x double> @test69(<2 x i16> %a) {
@@ -920,6 +1166,15 @@ define <2 x double> @test69(<2 x i16> %a) {
 ; CHECK: xxsldwi [[V4:[0-9]+]], {{[0-9]+}}, {{[0-9]+}}, 1
 ; CHECK: xvcvsxwdp 34, [[V4]]
 ; CHECK: blr
+
+; CHECK-LE-LABEL: @test69
+; CHECK-LE: vspltisw [[V1:[0-9]+]], 8
+; CHECK-LE: vadduwm [[V2:[0-9]+]], [[V1]], [[V1]]
+; CHECK-LE: vslw [[V3:[0-9]+]], {{[0-9]+}}, [[V2]]
+; CHECK-LE: vsraw {{[0-9]+}}, [[V3]], [[V2]]
+; CHECK-LE: xxsldwi [[V4:[0-9]+]], {{[0-9]+}}, {{[0-9]+}}, 1
+; CHECK-LE: xvcvsxwdp 34, [[V4]]
+; CHECK-LE: blr
 }
 
 define <2 x double> @test70(<2 x i8> %a) {
@@ -934,6 +1189,15 @@ define <2 x double> @test70(<2 x i8> %a) {
 ; CHECK: xxsldwi [[V4:[0-9]+]], {{[0-9]+}}, {{[0-9]+}}, 1
 ; CHECK: xvcvsxwdp 34, [[V4]]
 ; CHECK: blr
+
+; CHECK-LE-LABEL: @test70
+; CHECK-LE: vspltisw [[V1:[0-9]+]], 12
+; CHECK-LE: vadduwm [[V2:[0-9]+]], [[V1]], [[V1]]
+; CHECK-LE: vslw [[V3:[0-9]+]], {{[0-9]+}}, [[V2]]
+; CHECK-LE: vsraw {{[0-9]+}}, [[V3]], [[V2]]
+; CHECK-LE: xxsldwi [[V4:[0-9]+]], {{[0-9]+}}, {{[0-9]+}}, 1
+; CHECK-LE: xvcvsxwdp 34, [[V4]]
+; CHECK-LE: blr
 }
 
 define <2 x i32> @test80(i32 %v) {
@@ -960,6 +1224,16 @@ define <2 x i32> @test80(i32 %v) {
 ; CHECK-FISL-DAG: std [[R3]], -16(1)
 ; CHECK-FISL-DAG: lxvd2x 0, 0, [[R2]]
 ; CHECK-FISL: blr
+
+; CHECK-LE-LABEL: @test80
+; CHECK-LE-DAG: addi [[R1:[0-9]+]], 1, -16
+; CHECK-LE-DAG: addi [[R2:[0-9]+]], {{[0-9]+}}, .LCPI
+; CHECK-LE-DAG: lxvd2x [[V1:[0-9]+]], 0, [[R1]]
+; CHECK-LE-DAG: lxvd2x [[V2:[0-9]+]], 0, [[R2]]
+; CHECK-LE-DAG: xxswapd 34, [[V1]]
+; CHECK-LE-DAG: xxswapd 35, [[V2]]
+; CHECK-LE: vaddudm 2, 2, 3
+; CHECK-LE: blr
 }
 
 define <2 x double> @test81(<4 x float> %b) {
@@ -968,6 +1242,9 @@ define <2 x double> @test81(<4 x float> %b) {
 
 ; CHECK-LABEL: @test81
 ; CHECK: blr
+
+; CHECK-LE-LABEL: @test81
+; CHECK-LE: blr
 }
 
 define double @test82(double %a, double %b, double %c, double %d) {
@@ -983,4 +1260,8 @@ entry:
 ; CHECK-FISL-LABEL: @test82
 ; CHECK-FISL: xscmpudp [[REG:[0-9]+]], 3, 4
 ; CHECK-FISL: beq [[REG]], {{.*}}
+
+; CHECK-LE-LABEL: @test82
+; CHECK-LE: xscmpudp [[REG:[0-9]+]], 3, 4
+; CHECK-LE: beqlr [[REG]]
 }
diff --git a/test/CodeGen/PowerPC/vsx_insert_extract_le.ll b/test/CodeGen/PowerPC/vsx_insert_extract_le.ll
index e00cc4bf9fc..6c89b1092bd 100644
--- a/test/CodeGen/PowerPC/vsx_insert_extract_le.ll
+++ b/test/CodeGen/PowerPC/vsx_insert_extract_le.ll
@@ -8,9 +8,9 @@ define <2 x double> @testi0(<2 x double>* %p1, double* %p2) {
 
 ; CHECK-LABEL: testi0
 ; CHECK: lxvd2x 0, 0, 3
-; CHECK: lxsdx 34, 0, 4
-; CHECK: xxpermdi 0, 0, 0, 2
-; CHECK: xxpermdi 1, 34, 34, 0
+; CHECK: lxsdx 1, 0, 4
+; CHECK: xxswapd 0, 0
+; CHECK: xxspltd 1, 1, 0
 ; CHECK: xxpermdi 34, 0, 1, 1
 }
 
@@ -22,10 +22,10 @@ define <2 x double> @testi1(<2 x double>* %p1, double* %p2) {
 
 ; CHECK-LABEL: testi1
 ; CHECK: lxvd2x 0, 0, 3
-; CHECK: lxsdx 34, 0, 4
-; CHECK: xxpermdi 0, 0, 0, 2
-; CHECK: xxpermdi 1, 34, 34, 0
-; CHECK: xxpermdi 34, 1, 0, 3
+; CHECK: lxsdx 1, 0, 4
+; CHECK: xxswapd 0, 0
+; CHECK: xxspltd 1, 1, 0
+; CHECK: xxmrgld 34, 1, 0
 }
 
 define double @teste0(<2 x double>* %p1) {
@@ -37,8 +37,8 @@ define double @teste0(<2 x double>* %p1) {
 
 ; CHECK-LABEL: teste0
 ; CHECK: lxvd2x 0, 0, 3
-; CHECK: xxpermdi 0, 0, 0, 2
-; CHECK: xxpermdi 1, 0, 0, 2
+; CHECK: xxswapd 0, 0
+; CHECK: xxswapd 1, 0
 }
 
 define double @teste1(<2 x double>* %p1) {
@@ -48,5 +48,5 @@ define double @teste1(<2 x double>* %p1) {
 
 ; CHECK-LABEL: teste1
 ; CHECK: lxvd2x 0, 0, 3
-; CHECK: xxpermdi 1, 0, 0, 2
+; CHECK: xxswapd 1, 0
 }
diff --git a/test/CodeGen/PowerPC/vsx_scalar_ld_st.ll b/test/CodeGen/PowerPC/vsx_scalar_ld_st.ll
new file mode 100644
index 00000000000..10297088596
--- /dev/null
+++ b/test/CodeGen/PowerPC/vsx_scalar_ld_st.ll
@@ -0,0 +1,139 @@
+; RUN: llc < %s -mtriple=powerpc64-unknown-linux-gnu -mcpu=pwr8 -mattr=-direct-move | FileCheck %s
+; RUN: llc < %s -mtriple=powerpc64le-unknown-linux-gnu -mcpu=pwr8 -mattr=-direct-move | FileCheck %s
+
+@d = common global double 0.000000e+00, align 8
+@f = common global float 0.000000e+00, align 4
+@i = common global i32 0, align 4
+@ui = common global i32 0, align 4
+
+; Function Attrs: nounwind
+define void @dblToInt() #0 {
+entry:
+  %ii = alloca i32, align 4
+  %0 = load double, double* @d, align 8
+  %conv = fptosi double %0 to i32
+  store volatile i32 %conv, i32* %ii, align 4
+  ret void
+; CHECK-LABEL: @dblToInt
+; CHECK: xscvdpsxws [[REGCONV1:[0-9]+]],
+; CHECK: stxsiwx [[REGCONV1]],
+}
+
+; Function Attrs: nounwind
+define void @fltToInt() #0 {
+entry:
+  %ii = alloca i32, align 4
+  %0 = load float, float* @f, align 4
+  %conv = fptosi float %0 to i32
+  store volatile i32 %conv, i32* %ii, align 4
+  ret void
+; CHECK-LABEL: @fltToInt
+; CHECK: xscvdpsxws [[REGCONV2:[0-9]+]],
+; CHECK: stxsiwx [[REGCONV2]],
+}
+
+; Function Attrs: nounwind
+define void @intToDbl() #0 {
+entry:
+  %dd = alloca double, align 8
+  %0 = load i32, i32* @i, align 4
+  %conv = sitofp i32 %0 to double
+  store volatile double %conv, double* %dd, align 8
+  ret void
+; CHECK-LABEL: @intToDbl
+; CHECK: lxsiwax [[REGLD1:[0-9]+]],
+; CHECK: xscvsxddp {{[0-9]+}}, [[REGLD1]]
+}
+
+; Function Attrs: nounwind
+define void @intToFlt() #0 {
+entry:
+  %ff = alloca float, align 4
+  %0 = load i32, i32* @i, align 4
+  %conv = sitofp i32 %0 to float
+  store volatile float %conv, float* %ff, align 4
+  ret void
+; CHECK-LABEL: @intToFlt
+; CHECK: lxsiwax [[REGLD2:[0-9]+]],
+; FIXME: the below will change when the VSX form is implemented
+; CHECK: fcfids {{[0-9]}}, [[REGLD2]]
+}
+
+; Function Attrs: nounwind
+define void @dblToUInt() #0 {
+entry:
+  %uiui = alloca i32, align 4
+  %0 = load double, double* @d, align 8
+  %conv = fptoui double %0 to i32
+  store volatile i32 %conv, i32* %uiui, align 4
+  ret void
+; CHECK-LABEL: @dblToUInt
+; CHECK: xscvdpuxws [[REGCONV3:[0-9]+]],
+; CHECK: stxsiwx [[REGCONV3]],
+}
+
+; Function Attrs: nounwind
+define void @fltToUInt() #0 {
+entry:
+  %uiui = alloca i32, align 4
+  %0 = load float, float* @f, align 4
+  %conv = fptoui float %0 to i32
+  store volatile i32 %conv, i32* %uiui, align 4
+  ret void
+; CHECK-LABEL: @fltToUInt
+; CHECK: xscvdpuxws [[REGCONV4:[0-9]+]],
+; CHECK: stxsiwx [[REGCONV4]],
+}
+
+; Function Attrs: nounwind
+define void @uIntToDbl() #0 {
+entry:
+  %dd = alloca double, align 8
+  %0 = load i32, i32* @ui, align 4
+  %conv = uitofp i32 %0 to double
+  store volatile double %conv, double* %dd, align 8
+  ret void
+; CHECK-LABEL: @uIntToDbl
+; CHECK: lxsiwzx [[REGLD3:[0-9]+]],
+; CHECK: xscvuxddp {{[0-9]+}}, [[REGLD3]]
+}
+
+; Function Attrs: nounwind
+define void @uIntToFlt() #0 {
+entry:
+  %ff = alloca float, align 4
+  %0 = load i32, i32* @ui, align 4
+  %conv = uitofp i32 %0 to float
+  store volatile float %conv, float* %ff, align 4
+  ret void
+; CHECK-LABEL: @uIntToFlt
+; CHECK: lxsiwzx [[REGLD4:[0-9]+]],
+; FIXME: the below will change when the VSX form is implemented
+; CHECK: fcfidus {{[0-9]+}}, [[REGLD4]]
+}
+
+; Function Attrs: nounwind
+define void @dblToFloat() #0 {
+entry:
+  %ff = alloca float, align 4
+  %0 = load double, double* @d, align 8
+  %conv = fptrunc double %0 to float
+  store volatile float %conv, float* %ff, align 4
+  ret void
+; CHECK-LABEL: @dblToFloat
+; CHECK: lxsdx [[REGLD5:[0-9]+]],
+; CHECK: stxsspx [[REGLD5]],
+}
+
+; Function Attrs: nounwind
+define void @floatToDbl() #0 {
+entry:
+  %dd = alloca double, align 8
+  %0 = load float, float* @f, align 4
+  %conv = fpext float %0 to double
+  store volatile double %conv, double* %dd, align 8
+  ret void
+; CHECK-LABEL: @floatToDbl
+; CHECK: lxsspx [[REGLD5:[0-9]+]],
+; CHECK: stxsdx [[REGLD5]],
+}
diff --git a/test/CodeGen/PowerPC/vsx_shuffle_le.ll b/test/CodeGen/PowerPC/vsx_shuffle_le.ll
index 6bfde935f1d..dcfa0e78886 100644
--- a/test/CodeGen/PowerPC/vsx_shuffle_le.ll
+++ b/test/CodeGen/PowerPC/vsx_shuffle_le.ll
@@ -8,8 +8,8 @@ define <2 x double> @test00(<2 x double>* %p1, <2 x double>* %p2) {
 
 ; CHECK-LABEL: test00
 ; CHECK: lxvd2x 0, 0, 3
-; CHECK: xxpermdi 0, 0, 0, 2
-; CHECK: xxpermdi 34, 0, 0, 3
+; CHECK: xxswapd 0, 0
+; CHECK: xxspltd 34, 0, 1
 }
 
 define <2 x double> @test01(<2 x double>* %p1, <2 x double>* %p2) {
@@ -20,7 +20,7 @@ define <2 x double> @test01(<2 x double>* %p1, <2 x double>* %p2) {
 
 ; CHECK-LABEL: test01
 ; CHECK: lxvd2x 0, 0, 3
-; CHECK: xxpermdi 34, 0, 0, 2
+; CHECK: xxswapd 34, 0
 }
 
 define <2 x double> @test02(<2 x double>* %p1, <2 x double>* %p2) {
@@ -32,9 +32,9 @@ define <2 x double> @test02(<2 x double>* %p1, <2 x double>* %p2) {
 ; CHECK-LABEL: @test02
 ; CHECK: lxvd2x 0, 0, 3
 ; CHECK: lxvd2x 1, 0, 4
-; CHECK: xxpermdi 0, 0, 0, 2
-; CHECK: xxpermdi 1, 1, 1, 2
-; CHECK: xxpermdi 34, 1, 0, 3
+; CHECK: xxswapd 0, 0
+; CHECK: xxswapd 1, 1
+; CHECK: xxmrgld 34, 1, 0
 }
 
 define <2 x double> @test03(<2 x double>* %p1, <2 x double>* %p2) {
@@ -46,8 +46,8 @@ define <2 x double> @test03(<2 x double>* %p1, <2 x double>* %p2) {
 ; CHECK-LABEL: @test03
 ; CHECK: lxvd2x 0, 0, 3
 ; CHECK: lxvd2x 1, 0, 4
-; CHECK: xxpermdi 0, 0, 0, 2
-; CHECK: xxpermdi 1, 1, 1, 2
+; CHECK: xxswapd 0, 0
+; CHECK: xxswapd 1, 1
 ; CHECK: xxpermdi 34, 1, 0, 1
 }
 
@@ -59,8 +59,8 @@ define <2 x double> @test10(<2 x double>* %p1, <2 x double>* %p2) {
 
 ; CHECK-LABEL: @test10
 ; CHECK: lxvd2x 0, 0, 3
-; CHECK: xxpermdi 0, 0, 0, 2
-; CHECK: xxpermdi 34, 0, 0, 2
+; CHECK: xxswapd 0, 0
+; CHECK: xxswapd 34, 0
 }
 
 define <2 x double> @test11(<2 x double>* %p1, <2 x double>* %p2) {
@@ -71,8 +71,8 @@ define <2 x double> @test11(<2 x double>* %p1, <2 x double>* %p2) {
 
 ; CHECK-LABEL: @test11
 ; CHECK: lxvd2x 0, 0, 3
-; CHECK: xxpermdi 0, 0, 0, 2
-; CHECK: xxpermdi 34, 0, 0, 0
+; CHECK: xxswapd 0, 0
+; CHECK: xxspltd 34, 0, 0
 }
 
 define <2 x double> @test12(<2 x double>* %p1, <2 x double>* %p2) {
@@ -84,8 +84,8 @@ define <2 x double> @test12(<2 x double>* %p1, <2 x double>* %p2) {
 ; CHECK-LABEL: @test12
 ; CHECK: lxvd2x 0, 0, 3
 ; CHECK: lxvd2x 1, 0, 4
-; CHECK: xxpermdi 0, 0, 0, 2
-; CHECK: xxpermdi 1, 1, 1, 2
+; CHECK: xxswapd 0, 0
+; CHECK: xxswapd 1, 1
 ; CHECK: xxpermdi 34, 1, 0, 2
 }
 
@@ -98,9 +98,9 @@ define <2 x double> @test13(<2 x double>* %p1, <2 x double>* %p2) {
 ; CHECK-LABEL: @test13
 ; CHECK: lxvd2x 0, 0, 3
 ; CHECK: lxvd2x 1, 0, 4
-; CHECK: xxpermdi 0, 0, 0, 2
-; CHECK: xxpermdi 1, 1, 1, 2
-; CHECK: xxpermdi 34, 1, 0, 0
+; CHECK: xxswapd 0, 0
+; CHECK: xxswapd 1, 1
+; CHECK: xxmrghd 34, 1, 0
 }
 
 define <2 x double> @test20(<2 x double>* %p1, <2 x double>* %p2) {
@@ -112,9 +112,9 @@ define <2 x double> @test20(<2 x double>* %p1, <2 x double>* %p2) {
 ; CHECK-LABEL: @test20
 ; CHECK: lxvd2x 0, 0, 3
 ; CHECK: lxvd2x 1, 0, 4
-; CHECK: xxpermdi 0, 0, 0, 2
-; CHECK: xxpermdi 1, 1, 1, 2
-; CHECK: xxpermdi 34, 0, 1, 3
+; CHECK: xxswapd 0, 0
+; CHECK: xxswapd 1, 1
+; CHECK: xxmrgld 34, 0, 1
 }
 
 define <2 x double> @test21(<2 x double>* %p1, <2 x double>* %p2) {
@@ -126,8 +126,8 @@ define <2 x double> @test21(<2 x double>* %p1, <2 x double>* %p2) {
 ; CHECK-LABEL: @test21
 ; CHECK: lxvd2x 0, 0, 3
 ; CHECK: lxvd2x 1, 0, 4
-; CHECK: xxpermdi 0, 0, 0, 2
-; CHECK: xxpermdi 1, 1, 1, 2
+; CHECK: xxswapd 0, 0
+; CHECK: xxswapd 1, 1
 ; CHECK: xxpermdi 34, 0, 1, 1
 }
 
@@ -139,8 +139,8 @@ define <2 x double> @test22(<2 x double>* %p1, <2 x double>* %p2) {
 
 ; CHECK-LABEL: @test22
 ; CHECK: lxvd2x 0, 0, 4
-; CHECK: xxpermdi 0, 0, 0, 2
-; CHECK: xxpermdi 34, 0, 0, 3
+; CHECK: xxswapd 0, 0
+; CHECK: xxspltd 34, 0, 1
 }
 
 define <2 x double> @test23(<2 x double>* %p1, <2 x double>* %p2) {
@@ -151,7 +151,7 @@ define <2 x double> @test23(<2 x double>* %p1, <2 x double>* %p2) {
 
 ; CHECK-LABEL: @test23
 ; CHECK: lxvd2x 0, 0, 4
-; CHECK: xxpermdi 34, 0, 0, 2
+; CHECK: xxswapd 34, 0
 }
 
 define <2 x double> @test30(<2 x double>* %p1, <2 x double>* %p2) {
@@ -163,8 +163,8 @@ define <2 x double> @test30(<2 x double>* %p1, <2 x double>* %p2) {
 ; CHECK-LABEL: @test30
 ; CHECK: lxvd2x 0, 0, 3
 ; CHECK: lxvd2x 1, 0, 4
-; CHECK: xxpermdi 0, 0, 0, 2
-; CHECK: xxpermdi 1, 1, 1, 2
+; CHECK: xxswapd 0, 0
+; CHECK: xxswapd 1, 1
 ; CHECK: xxpermdi 34, 0, 1, 2
 }
 
@@ -177,9 +177,9 @@ define <2 x double> @test31(<2 x double>* %p1, <2 x double>* %p2) {
 ; CHECK-LABEL: @test31
 ; CHECK: lxvd2x 0, 0, 3
 ; CHECK: lxvd2x 1, 0, 4
-; CHECK: xxpermdi 0, 0, 0, 2
-; CHECK: xxpermdi 1, 1, 1, 2
-; CHECK: xxpermdi 34, 0, 1, 0
+; CHECK: xxswapd 0, 0
+; CHECK: xxswapd 1, 1
+; CHECK: xxmrghd 34, 0, 1
 }
 
 define <2 x double> @test32(<2 x double>* %p1, <2 x double>* %p2) {
@@ -190,8 +190,8 @@ define <2 x double> @test32(<2 x double>* %p1, <2 x double>* %p2) {
 
 ; CHECK-LABEL: @test32
 ; CHECK: lxvd2x 0, 0, 4
-; CHECK: xxpermdi 0, 0, 0, 2
-; CHECK: xxpermdi 34, 0, 0, 2
+; CHECK: xxswapd 0, 0
+; CHECK: xxswapd 34, 0
 }
 
 define <2 x double> @test33(<2 x double>* %p1, <2 x double>* %p2) {
@@ -202,6 +202,6 @@ define <2 x double> @test33(<2 x double>* %p1, <2 x double>* %p2) {
 
 ; CHECK-LABEL: @test33
 ; CHECK: lxvd2x 0, 0, 4
-; CHECK: xxpermdi 0, 0, 0, 2
-; CHECK: xxpermdi 34, 0, 0, 0
+; CHECK: xxswapd 0, 0
+; CHECK: xxspltd 34, 0, 0
 }
diff --git a/test/CodeGen/R600/128bit-kernel-args.ll b/test/CodeGen/R600/128bit-kernel-args.ll
deleted file mode 100644
index 557d86aa837..00000000000
--- a/test/CodeGen/R600/128bit-kernel-args.ll
+++ /dev/null
@@ -1,29 +0,0 @@
-; RUN: llc < %s -march=r600 -mcpu=redwood | FileCheck %s --check-prefix=R600
-; RUN: llc < %s -march=amdgcn -mcpu=SI -verify-machineinstrs | FileCheck %s --check-prefix=SI
-; RUN: llc < %s -march=amdgcn -mcpu=tonga -verify-machineinstrs | FileCheck %s --check-prefix=SI
-
-; R600: {{^}}v4i32_kernel_arg:
-; R600-DAG: MOV {{[* ]*}}T[[GPR:[0-9]]].X, KC0[3].Y
-; R600-DAG: MOV {{[* ]*}}T[[GPR]].Y, KC0[3].Z
-; R600-DAG: MOV {{[* ]*}}T[[GPR]].Z, KC0[3].W
-; R600-DAG: MOV {{[* ]*}}T[[GPR]].W, KC0[4].X
-; SI: {{^}}v4i32_kernel_arg:
-; SI: buffer_store_dwordx4
-define void @v4i32_kernel_arg(<4 x i32> addrspace(1)* %out, <4 x i32>  %in) {
-entry:
-  store <4 x i32> %in, <4 x i32> addrspace(1)* %out
-  ret void
-}
-
-; R600: {{^}}v4f32_kernel_arg:
-; R600-DAG: MOV {{[* ]*}}T[[GPR:[0-9]]].X, KC0[3].Y
-; R600-DAG: MOV {{[* ]*}}T[[GPR]].Y, KC0[3].Z
-; R600-DAG: MOV {{[* ]*}}T[[GPR]].Z, KC0[3].W
-; R600-DAG: MOV {{[* ]*}}T[[GPR]].W, KC0[4].X
-; SI: {{^}}v4f32_kernel_arg:
-; SI: buffer_store_dwordx4
-define void @v4f32_kernel_arg(<4 x float> addrspace(1)* %out, <4 x float>  %in) {
-entry:
-  store <4 x float> %in, <4 x float> addrspace(1)* %out
-  ret void
-}
diff --git a/test/CodeGen/R600/64bit-kernel-args.ll b/test/CodeGen/R600/64bit-kernel-args.ll
deleted file mode 100644
index 2e08901ac1e..00000000000
--- a/test/CodeGen/R600/64bit-kernel-args.ll
+++ /dev/null
@@ -1,14 +0,0 @@
-; RUN: llc < %s -march=amdgcn -mcpu=tahiti -verify-machineinstrs | FileCheck %s --check-prefix=GCN --check-prefix=SI
-; RUN: llc < %s -march=amdgcn -mcpu=tonga -verify-machineinstrs | FileCheck %s --check-prefix=GCN --check-prefix=VI
-
-; GCN: {{^}}f64_kernel_arg:
-; SI-DAG: s_load_dwordx2 s[{{[0-9]:[0-9]}}], s[0:1], 0x9
-; SI-DAG: s_load_dwordx2 s[{{[0-9]:[0-9]}}], s[0:1], 0xb
-; VI-DAG: s_load_dwordx2 s[{{[0-9]:[0-9]}}], s[0:1], 0x24
-; VI-DAG: s_load_dwordx2 s[{{[0-9]:[0-9]}}], s[0:1], 0x2c
-; GCN: buffer_store_dwordx2
-define void @f64_kernel_arg(double addrspace(1)* %out, double  %in) {
-entry:
-  store double %in, double addrspace(1)* %out
-  ret void
-}
diff --git a/test/CodeGen/R600/call.ll b/test/CodeGen/R600/call.ll
deleted file mode 100644
index eb716490bb8..00000000000
--- a/test/CodeGen/R600/call.ll
+++ /dev/null
@@ -1,34 +0,0 @@
-; RUN: not llc -march=amdgcn -mcpu=SI -verify-machineinstrs< %s 2>&1 | FileCheck %s
-; RUN: not llc -march=amdgcn -mcpu=tonga -verify-machineinstrs< %s 2>&1 | FileCheck %s
-; RUN: not llc -march=r600 -mcpu=cypress < %s 2>&1 | FileCheck %s
-
-; CHECK: error: unsupported call to function external_function in test_call_external
-
-
-declare i32 @external_function(i32) nounwind
-
-define i32 @defined_function(i32 %x) nounwind noinline {
-  %y = add i32 %x, 8
-  ret i32 %y
-}
-
-define void @test_call(i32 addrspace(1)* %out, i32 addrspace(1)* %in) {
-  %b_ptr = getelementptr i32, i32 addrspace(1)* %in, i32 1
-  %a = load i32, i32 addrspace(1)* %in
-  %b = load i32, i32 addrspace(1)* %b_ptr
-  %c = call i32 @defined_function(i32 %b) nounwind
-  %result = add i32 %a, %c
-  store i32 %result, i32 addrspace(1)* %out
-  ret void
-}
-
-define void @test_call_external(i32 addrspace(1)* %out, i32 addrspace(1)* %in) {
-  %b_ptr = getelementptr i32, i32 addrspace(1)* %in, i32 1
-  %a = load i32, i32 addrspace(1)* %in
-  %b = load i32, i32 addrspace(1)* %b_ptr
-  %c = call i32 @external_function(i32 %b) nounwind
-  %result = add i32 %a, %c
-  store i32 %result, i32 addrspace(1)* %out
-  ret void
-}
-
diff --git a/test/CodeGen/R600/codegen-prepare-addrmode-sext.ll b/test/CodeGen/R600/codegen-prepare-addrmode-sext.ll
deleted file mode 100644
index 0aecc189e0b..00000000000
--- a/test/CodeGen/R600/codegen-prepare-addrmode-sext.ll
+++ /dev/null
@@ -1,20 +0,0 @@
-; RUN: opt -codegenprepare -S -o - %s | FileCheck --check-prefix=OPT %s
-; RUN: llc < %s -march=amdgcn -mcpu=verde -verify-machineinstrs | FileCheck --check-prefix=SI-LLC %s
-
-target datalayout = "e-p:32:32-p1:64:64-p2:64:64-p3:32:32-p4:32:32-p5:64:64-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-v2048:2048-n32:64"
-target triple = "r600--"
-
-; OPT-LABEL: @test
-; OPT: mul nsw i32
-; OPT-NEXT: sext
-; SI-LLC-LABEL: {{^}}test:
-; SI-LLC: s_mul_i32
-; SI-LLC-NOT: mul
-define void @test(i8 addrspace(1)* nocapture readonly %in, i32 %a, i8 %b) {
-entry:
-  %0 = mul nsw i32 %a, 3
-  %1 = sext i32 %0 to i64
-  %2 = getelementptr i8, i8 addrspace(1)* %in, i64 %1
-  store i8 %b, i8 addrspace(1)* %2
-  ret void
-}
diff --git a/test/CodeGen/R600/elf.ll b/test/CodeGen/R600/elf.ll
deleted file mode 100644
index d0fd06a3437..00000000000
--- a/test/CodeGen/R600/elf.ll
+++ /dev/null
@@ -1,34 +0,0 @@
-; RUN: llc < %s -march=amdgcn -mcpu=SI -verify-machineinstrs -filetype=obj | llvm-readobj -s -symbols - | FileCheck --check-prefix=ELF %s
-; RUN: llc < %s -march=amdgcn -mcpu=SI -verify-machineinstrs -o - | FileCheck --check-prefix=CONFIG --check-prefix=TYPICAL %s
-; RUN: llc < %s -march=amdgcn -mcpu=tonga -verify-machineinstrs -filetype=obj | llvm-readobj -s -symbols - | FileCheck --check-prefix=ELF %s
-; RUN: llc < %s -march=amdgcn -mcpu=tonga -verify-machineinstrs -o - | FileCheck --check-prefix=CONFIG --check-prefix=TONGA %s
-; RUN: llc < %s -march=amdgcn -mcpu=carrizo -verify-machineinstrs -filetype=obj | llvm-readobj -s -symbols - | FileCheck --check-prefix=ELF %s
-; RUN: llc < %s -march=amdgcn -mcpu=carrizo -verify-machineinstrs -o - | FileCheck --check-prefix=CONFIG --check-prefix=TYPICAL %s
-
-; Test that we don't try to produce a COFF file on windows
-; RUN: llc < %s -mtriple=amdgcn-pc-mingw -mcpu=SI -verify-machineinstrs -filetype=obj | llvm-readobj -s -symbols - | FileCheck --check-prefix=ELF %s
-
-; ELF: Format: ELF32
-; ELF: Name: .AMDGPU.config
-; ELF: Type: SHT_PROGBITS
-
-; ELF: Symbol {
-; ELF: Name: test
-; ELF: Binding: Global
-
-; CONFIG: .section .AMDGPU.config
-; CONFIG-NEXT: .long   45096
-; TYPICAL-NEXT: .long   0
-; TONGA-NEXT: .long   576
-; CONFIG: .align 256
-; CONFIG: test:
-define void @test(i32 %p) #0 {
-   %i = add i32 %p, 2
-   %r = bitcast i32 %i to float
-   call void @llvm.SI.export(i32 15, i32 0, i32 1, i32 12, i32 0, float %r, float %r, float %r, float %r)
-   ret void
-}
-
-declare void @llvm.SI.export(i32, i32, i32, i32, i32, float, float, float, float)
-
-attributes #0 = { "ShaderType"="0" } ; Pixel Shader
diff --git a/test/CodeGen/R600/global_atomics.ll b/test/CodeGen/R600/global_atomics.ll
deleted file mode 100644
index 847950f6376..00000000000
--- a/test/CodeGen/R600/global_atomics.ll
+++ /dev/null
@@ -1,801 +0,0 @@
-; RUN: llc < %s -march=amdgcn -mcpu=SI -verify-machineinstrs | FileCheck --check-prefix=SI --check-prefix=FUNC %s
-
-; FUNC-LABEL: {{^}}atomic_add_i32_offset:
-; SI: buffer_atomic_add v{{[0-9]+}}, s[{{[0-9]+}}:{{[0-9]+}}], 0 offset:16{{$}}
-define void @atomic_add_i32_offset(i32 addrspace(1)* %out, i32 %in) {
-entry:
-  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
-  %0  = atomicrmw volatile add i32 addrspace(1)* %gep, i32 %in seq_cst
-  ret void
-}
-
-; FUNC-LABEL: {{^}}atomic_add_i32_ret_offset:
-; SI: buffer_atomic_add [[RET:v[0-9]+]], s[{{[0-9]+}}:{{[0-9]+}}], 0 offset:16 glc {{$}}
-; SI: buffer_store_dword [[RET]]
-define void @atomic_add_i32_ret_offset(i32 addrspace(1)* %out, i32 addrspace(1)* %out2, i32 %in) {
-entry:
-  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
-  %0  = atomicrmw volatile add i32 addrspace(1)* %gep, i32 %in seq_cst
-  store i32 %0, i32 addrspace(1)* %out2
-  ret void
-}
-
-; FUNC-LABEL: {{^}}atomic_add_i32_addr64_offset:
-; SI: buffer_atomic_add v{{[0-9]+}}, v[{{[0-9]+}}:{{[0-9]+}}], s[{{[0-9]+}}:{{[0-9]+}}], 0 addr64 offset:16{{$}}
-define void @atomic_add_i32_addr64_offset(i32 addrspace(1)* %out, i32 %in, i64 %index) {
-entry:
-  %ptr = getelementptr i32, i32 addrspace(1)* %out, i64 %index
-  %gep = getelementptr i32, i32 addrspace(1)* %ptr, i32 4
-  %0  = atomicrmw volatile add i32 addrspace(1)* %gep, i32 %in seq_cst
-  ret void
-}
-
-; FUNC-LABEL: {{^}}atomic_add_i32_ret_addr64_offset:
-; SI: buffer_atomic_add [[RET:v[0-9]+]], v[{{[0-9]+}}:{{[0-9]+}}], s[{{[0-9]+}}:{{[0-9]+}}], 0 addr64 offset:16 glc{{$}}
-; SI: buffer_store_dword [[RET]]
-define void @atomic_add_i32_ret_addr64_offset(i32 addrspace(1)* %out, i32 addrspace(1)* %out2, i32 %in, i64 %index) {
-entry:
-  %ptr = getelementptr i32, i32 addrspace(1)* %out, i64 %index
-  %gep = getelementptr i32, i32 addrspace(1)* %ptr, i32 4
-  %0  = atomicrmw volatile add i32 addrspace(1)* %gep, i32 %in seq_cst
-  store i32 %0, i32 addrspace(1)* %out2
-  ret void
-}
-
-; FUNC-LABEL: {{^}}atomic_add_i32:
-; SI: buffer_atomic_add v{{[0-9]+}}, s[{{[0-9]+}}:{{[0-9]+}}], 0{{$}}
-define void @atomic_add_i32(i32 addrspace(1)* %out, i32 %in) {
-entry:
-  %0  = atomicrmw volatile add i32 addrspace(1)* %out, i32 %in seq_cst
-  ret void
-}
-
-; FUNC-LABEL: {{^}}atomic_add_i32_ret:
-; SI: buffer_atomic_add [[RET:v[0-9]+]], s[{{[0-9]+}}:{{[0-9]+}}], 0 glc
-; SI: buffer_store_dword [[RET]]
-define void @atomic_add_i32_ret(i32 addrspace(1)* %out, i32 addrspace(1)* %out2, i32 %in) {
-entry:
-  %0  = atomicrmw volatile add i32 addrspace(1)* %out, i32 %in seq_cst
-  store i32 %0, i32 addrspace(1)* %out2
-  ret void
-}
-
-; FUNC-LABEL: {{^}}atomic_add_i32_addr64:
-; SI: buffer_atomic_add v{{[0-9]+}}, v[{{[0-9]+}}:{{[0-9]+}}], s[{{[0-9]+}}:{{[0-9]+}}], 0 addr64{{$}}
-define void @atomic_add_i32_addr64(i32 addrspace(1)* %out, i32 %in, i64 %index) {
-entry:
-  %ptr = getelementptr i32, i32 addrspace(1)* %out, i64 %index
-  %0  = atomicrmw volatile add i32 addrspace(1)* %ptr, i32 %in seq_cst
-  ret void
-}
-
-; FUNC-LABEL: {{^}}atomic_add_i32_ret_addr64:
-; SI: buffer_atomic_add [[RET:v[0-9]+]], v[{{[0-9]+}}:{{[0-9]+}}], s[{{[0-9]+}}:{{[0-9]+}}], 0 addr64 glc{{$}}
-; SI: buffer_store_dword [[RET]]
-define void @atomic_add_i32_ret_addr64(i32 addrspace(1)* %out, i32 addrspace(1)* %out2, i32 %in, i64 %index) {
-entry:
-  %ptr = getelementptr i32, i32 addrspace(1)* %out, i64 %index
-  %0  = atomicrmw volatile add i32 addrspace(1)* %ptr, i32 %in seq_cst
-  store i32 %0, i32 addrspace(1)* %out2
-  ret void
-}
-
-; FUNC-LABEL: {{^}}atomic_and_i32_offset:
-; SI: buffer_atomic_and v{{[0-9]+}}, s[{{[0-9]+}}:{{[0-9]+}}], 0 offset:16{{$}}
-define void @atomic_and_i32_offset(i32 addrspace(1)* %out, i32 %in) {
-entry:
-  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
-  %0  = atomicrmw volatile and i32 addrspace(1)* %gep, i32 %in seq_cst
-  ret void
-}
-
-; FUNC-LABEL: {{^}}atomic_and_i32_ret_offset:
-; SI: buffer_atomic_and [[RET:v[0-9]+]], s[{{[0-9]+}}:{{[0-9]+}}], 0 offset:16 glc {{$}}
-; SI: buffer_store_dword [[RET]]
-define void @atomic_and_i32_ret_offset(i32 addrspace(1)* %out, i32 addrspace(1)* %out2, i32 %in) {
-entry:
-  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
-  %0  = atomicrmw volatile and i32 addrspace(1)* %gep, i32 %in seq_cst
-  store i32 %0, i32 addrspace(1)* %out2
-  ret void
-}
-
-; FUNC-LABEL: {{^}}atomic_and_i32_addr64_offset:
-; SI: buffer_atomic_and v{{[0-9]+}}, v[{{[0-9]+}}:{{[0-9]+}}], s[{{[0-9]+}}:{{[0-9]+}}], 0 addr64 offset:16{{$}}
-define void @atomic_and_i32_addr64_offset(i32 addrspace(1)* %out, i32 %in, i64 %index) {
-entry:
-  %ptr = getelementptr i32, i32 addrspace(1)* %out, i64 %index
-  %gep = getelementptr i32, i32 addrspace(1)* %ptr, i32 4
-  %0  = atomicrmw volatile and i32 addrspace(1)* %gep, i32 %in seq_cst
-  ret void
-}
-
-; FUNC-LABEL: {{^}}atomic_and_i32_ret_addr64_offset:
-; SI: buffer_atomic_and [[RET:v[0-9]+]], v[{{[0-9]+}}:{{[0-9]+}}], s[{{[0-9]+}}:{{[0-9]+}}], 0 addr64 offset:16 glc{{$}}
-; SI: buffer_store_dword [[RET]]
-define void @atomic_and_i32_ret_addr64_offset(i32 addrspace(1)* %out, i32 addrspace(1)* %out2, i32 %in, i64 %index) {
-entry:
-  %ptr = getelementptr i32, i32 addrspace(1)* %out, i64 %index
-  %gep = getelementptr i32, i32 addrspace(1)* %ptr, i32 4
-  %0  = atomicrmw volatile and i32 addrspace(1)* %gep, i32 %in seq_cst
-  store i32 %0, i32 addrspace(1)* %out2
-  ret void
-}
-
-; FUNC-LABEL: {{^}}atomic_and_i32:
-; SI: buffer_atomic_and v{{[0-9]+}}, s[{{[0-9]+}}:{{[0-9]+}}], 0{{$}}
-define void @atomic_and_i32(i32 addrspace(1)* %out, i32 %in) {
-entry:
-  %0  = atomicrmw volatile and i32 addrspace(1)* %out, i32 %in seq_cst
-  ret void
-}
-
-; FUNC-LABEL: {{^}}atomic_and_i32_ret:
-; SI: buffer_atomic_and [[RET:v[0-9]+]], s[{{[0-9]+}}:{{[0-9]+}}], 0 glc
-; SI: buffer_store_dword [[RET]]
-define void @atomic_and_i32_ret(i32 addrspace(1)* %out, i32 addrspace(1)* %out2, i32 %in) {
-entry:
-  %0  = atomicrmw volatile and i32 addrspace(1)* %out, i32 %in seq_cst
-  store i32 %0, i32 addrspace(1)* %out2
-  ret void
-}
-
-; FUNC-LABEL: {{^}}atomic_and_i32_addr64:
-; SI: buffer_atomic_and v{{[0-9]+}}, v[{{[0-9]+}}:{{[0-9]+}}], s[{{[0-9]+}}:{{[0-9]+}}], 0 addr64{{$}}
-define void @atomic_and_i32_addr64(i32 addrspace(1)* %out, i32 %in, i64 %index) {
-entry:
-  %ptr = getelementptr i32, i32 addrspace(1)* %out, i64 %index
-  %0  = atomicrmw volatile and i32 addrspace(1)* %ptr, i32 %in seq_cst
-  ret void
-}
-
-; FUNC-LABEL: {{^}}atomic_and_i32_ret_addr64:
-; SI: buffer_atomic_and [[RET:v[0-9]+]], v[{{[0-9]+}}:{{[0-9]+}}], s[{{[0-9]+}}:{{[0-9]+}}], 0 addr64 glc{{$}}
-; SI: buffer_store_dword [[RET]]
-define void @atomic_and_i32_ret_addr64(i32 addrspace(1)* %out, i32 addrspace(1)* %out2, i32 %in, i64 %index) {
-entry:
-  %ptr = getelementptr i32, i32 addrspace(1)* %out, i64 %index
-  %0  = atomicrmw volatile and i32 addrspace(1)* %ptr, i32 %in seq_cst
-  store i32 %0, i32 addrspace(1)* %out2
-  ret void
-}
-
-; FUNC-LABEL: {{^}}atomic_sub_i32_offset:
-; SI: buffer_atomic_sub v{{[0-9]+}}, s[{{[0-9]+}}:{{[0-9]+}}], 0 offset:16{{$}}
-define void @atomic_sub_i32_offset(i32 addrspace(1)* %out, i32 %in) {
-entry:
-  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
-  %0  = atomicrmw volatile sub i32 addrspace(1)* %gep, i32 %in seq_cst
-  ret void
-}
-
-; FUNC-LABEL: {{^}}atomic_sub_i32_ret_offset:
-; SI: buffer_atomic_sub [[RET:v[0-9]+]], s[{{[0-9]+}}:{{[0-9]+}}], 0 offset:16 glc {{$}}
-; SI: buffer_store_dword [[RET]]
-define void @atomic_sub_i32_ret_offset(i32 addrspace(1)* %out, i32 addrspace(1)* %out2, i32 %in) {
-entry:
-  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
-  %0  = atomicrmw volatile sub i32 addrspace(1)* %gep, i32 %in seq_cst
-  store i32 %0, i32 addrspace(1)* %out2
-  ret void
-}
-
-; FUNC-LABEL: {{^}}atomic_sub_i32_addr64_offset:
-; SI: buffer_atomic_sub v{{[0-9]+}}, v[{{[0-9]+}}:{{[0-9]+}}], s[{{[0-9]+}}:{{[0-9]+}}], 0 addr64 offset:16{{$}}
-define void @atomic_sub_i32_addr64_offset(i32 addrspace(1)* %out, i32 %in, i64 %index) {
-entry:
-  %ptr = getelementptr i32, i32 addrspace(1)* %out, i64 %index
-  %gep = getelementptr i32, i32 addrspace(1)* %ptr, i32 4
-  %0  = atomicrmw volatile sub i32 addrspace(1)* %gep, i32 %in seq_cst
-  ret void
-}
-
-; FUNC-LABEL: {{^}}atomic_sub_i32_ret_addr64_offset:
-; SI: buffer_atomic_sub [[RET:v[0-9]+]], v[{{[0-9]+}}:{{[0-9]+}}], s[{{[0-9]+}}:{{[0-9]+}}], 0 addr64 offset:16 glc{{$}}
-; SI: buffer_store_dword [[RET]]
-define void @atomic_sub_i32_ret_addr64_offset(i32 addrspace(1)* %out, i32 addrspace(1)* %out2, i32 %in, i64 %index) {
-entry:
-  %ptr = getelementptr i32, i32 addrspace(1)* %out, i64 %index
-  %gep = getelementptr i32, i32 addrspace(1)* %ptr, i32 4
-  %0  = atomicrmw volatile sub i32 addrspace(1)* %gep, i32 %in seq_cst
-  store i32 %0, i32 addrspace(1)* %out2
-  ret void
-}
-
-; FUNC-LABEL: {{^}}atomic_sub_i32:
-; SI: buffer_atomic_sub v{{[0-9]+}}, s[{{[0-9]+}}:{{[0-9]+}}], 0{{$}}
-define void @atomic_sub_i32(i32 addrspace(1)* %out, i32 %in) {
-entry:
-  %0  = atomicrmw volatile sub i32 addrspace(1)* %out, i32 %in seq_cst
-  ret void
-}
-
-; FUNC-LABEL: {{^}}atomic_sub_i32_ret:
-; SI: buffer_atomic_sub [[RET:v[0-9]+]], s[{{[0-9]+}}:{{[0-9]+}}], 0 glc
-; SI: buffer_store_dword [[RET]]
-define void @atomic_sub_i32_ret(i32 addrspace(1)* %out, i32 addrspace(1)* %out2, i32 %in) {
-entry:
-  %0  = atomicrmw volatile sub i32 addrspace(1)* %out, i32 %in seq_cst
-  store i32 %0, i32 addrspace(1)* %out2
-  ret void
-}
-
-; FUNC-LABEL: {{^}}atomic_sub_i32_addr64:
-; SI: buffer_atomic_sub v{{[0-9]+}}, v[{{[0-9]+}}:{{[0-9]+}}], s[{{[0-9]+}}:{{[0-9]+}}], 0 addr64{{$}}
-define void @atomic_sub_i32_addr64(i32 addrspace(1)* %out, i32 %in, i64 %index) {
-entry:
-  %ptr = getelementptr i32, i32 addrspace(1)* %out, i64 %index
-  %0  = atomicrmw volatile sub i32 addrspace(1)* %ptr, i32 %in seq_cst
-  ret void
-}
-
-; FUNC-LABEL: {{^}}atomic_sub_i32_ret_addr64:
-; SI: buffer_atomic_sub [[RET:v[0-9]+]], v[{{[0-9]+}}:{{[0-9]+}}], s[{{[0-9]+}}:{{[0-9]+}}], 0 addr64 glc{{$}}
-; SI: buffer_store_dword [[RET]]
-define void @atomic_sub_i32_ret_addr64(i32 addrspace(1)* %out, i32 addrspace(1)* %out2, i32 %in, i64 %index) {
-entry:
-  %ptr = getelementptr i32, i32 addrspace(1)* %out, i64 %index
-  %0  = atomicrmw volatile sub i32 addrspace(1)* %ptr, i32 %in seq_cst
-  store i32 %0, i32 addrspace(1)* %out2
-  ret void
-}
-
-; FUNC-LABEL: {{^}}atomic_max_i32_offset:
-; SI: buffer_atomic_smax v{{[0-9]+}}, s[{{[0-9]+}}:{{[0-9]+}}], 0 offset:16{{$}}
-define void @atomic_max_i32_offset(i32 addrspace(1)* %out, i32 %in) {
-entry:
-  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
-  %0  = atomicrmw volatile max i32 addrspace(1)* %gep, i32 %in seq_cst
-  ret void
-}
-
-; FUNC-LABEL: {{^}}atomic_max_i32_ret_offset:
-; SI: buffer_atomic_smax [[RET:v[0-9]+]], s[{{[0-9]+}}:{{[0-9]+}}], 0 offset:16 glc {{$}}
-; SI: buffer_store_dword [[RET]]
-define void @atomic_max_i32_ret_offset(i32 addrspace(1)* %out, i32 addrspace(1)* %out2, i32 %in) {
-entry:
-  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
-  %0  = atomicrmw volatile max i32 addrspace(1)* %gep, i32 %in seq_cst
-  store i32 %0, i32 addrspace(1)* %out2
-  ret void
-}
-
-; FUNC-LABEL: {{^}}atomic_max_i32_addr64_offset:
-; SI: buffer_atomic_smax v{{[0-9]+}}, v[{{[0-9]+}}:{{[0-9]+}}], s[{{[0-9]+}}:{{[0-9]+}}], 0 addr64 offset:16{{$}}
-define void @atomic_max_i32_addr64_offset(i32 addrspace(1)* %out, i32 %in, i64 %index) {
-entry:
-  %ptr = getelementptr i32, i32 addrspace(1)* %out, i64 %index
-  %gep = getelementptr i32, i32 addrspace(1)* %ptr, i32 4
-  %0  = atomicrmw volatile max i32 addrspace(1)* %gep, i32 %in seq_cst
-  ret void
-}
-
-; FUNC-LABEL: {{^}}atomic_max_i32_ret_addr64_offset:
-; SI: buffer_atomic_smax [[RET:v[0-9]+]], v[{{[0-9]+}}:{{[0-9]+}}], s[{{[0-9]+}}:{{[0-9]+}}], 0 addr64 offset:16 glc{{$}}
-; SI: buffer_store_dword [[RET]]
-define void @atomic_max_i32_ret_addr64_offset(i32 addrspace(1)* %out, i32 addrspace(1)* %out2, i32 %in, i64 %index) {
-entry:
-  %ptr = getelementptr i32, i32 addrspace(1)* %out, i64 %index
-  %gep = getelementptr i32, i32 addrspace(1)* %ptr, i32 4
-  %0  = atomicrmw volatile max i32 addrspace(1)* %gep, i32 %in seq_cst
-  store i32 %0, i32 addrspace(1)* %out2
-  ret void
-}
-
-; FUNC-LABEL: {{^}}atomic_max_i32:
-; SI: buffer_atomic_smax v{{[0-9]+}}, s[{{[0-9]+}}:{{[0-9]+}}], 0{{$}}
-define void @atomic_max_i32(i32 addrspace(1)* %out, i32 %in) {
-entry:
-  %0  = atomicrmw volatile max i32 addrspace(1)* %out, i32 %in seq_cst
-  ret void
-}
-
-; FUNC-LABEL: {{^}}atomic_max_i32_ret:
-; SI: buffer_atomic_smax [[RET:v[0-9]+]], s[{{[0-9]+}}:{{[0-9]+}}], 0 glc
-; SI: buffer_store_dword [[RET]]
-define void @atomic_max_i32_ret(i32 addrspace(1)* %out, i32 addrspace(1)* %out2, i32 %in) {
-entry:
-  %0  = atomicrmw volatile max i32 addrspace(1)* %out, i32 %in seq_cst
-  store i32 %0, i32 addrspace(1)* %out2
-  ret void
-}
-
-; FUNC-LABEL: {{^}}atomic_max_i32_addr64:
-; SI: buffer_atomic_smax v{{[0-9]+}}, v[{{[0-9]+}}:{{[0-9]+}}], s[{{[0-9]+}}:{{[0-9]+}}], 0 addr64{{$}}
-define void @atomic_max_i32_addr64(i32 addrspace(1)* %out, i32 %in, i64 %index) {
-entry:
-  %ptr = getelementptr i32, i32 addrspace(1)* %out, i64 %index
-  %0  = atomicrmw volatile max i32 addrspace(1)* %ptr, i32 %in seq_cst
-  ret void
-}
-
-; FUNC-LABEL: {{^}}atomic_max_i32_ret_addr64:
-; SI: buffer_atomic_smax [[RET:v[0-9]+]], v[{{[0-9]+}}:{{[0-9]+}}], s[{{[0-9]+}}:{{[0-9]+}}], 0 addr64 glc{{$}}
-; SI: buffer_store_dword [[RET]]
-define void @atomic_max_i32_ret_addr64(i32 addrspace(1)* %out, i32 addrspace(1)* %out2, i32 %in, i64 %index) {
-entry:
-  %ptr = getelementptr i32, i32 addrspace(1)* %out, i64 %index
-  %0  = atomicrmw volatile max i32 addrspace(1)* %ptr, i32 %in seq_cst
-  store i32 %0, i32 addrspace(1)* %out2
-  ret void
-}
-
-; FUNC-LABEL: {{^}}atomic_umax_i32_offset:
-; SI: buffer_atomic_umax v{{[0-9]+}}, s[{{[0-9]+}}:{{[0-9]+}}], 0 offset:16{{$}}
-define void @atomic_umax_i32_offset(i32 addrspace(1)* %out, i32 %in) {
-entry:
-  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
-  %0  = atomicrmw volatile umax i32 addrspace(1)* %gep, i32 %in seq_cst
-  ret void
-}
-
-; FUNC-LABEL: {{^}}atomic_umax_i32_ret_offset:
-; SI: buffer_atomic_umax [[RET:v[0-9]+]], s[{{[0-9]+}}:{{[0-9]+}}], 0 offset:16 glc {{$}}
-; SI: buffer_store_dword [[RET]]
-define void @atomic_umax_i32_ret_offset(i32 addrspace(1)* %out, i32 addrspace(1)* %out2, i32 %in) {
-entry:
-  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
-  %0  = atomicrmw volatile umax i32 addrspace(1)* %gep, i32 %in seq_cst
-  store i32 %0, i32 addrspace(1)* %out2
-  ret void
-}
-
-; FUNC-LABEL: {{^}}atomic_umax_i32_addr64_offset:
-; SI: buffer_atomic_umax v{{[0-9]+}}, v[{{[0-9]+}}:{{[0-9]+}}], s[{{[0-9]+}}:{{[0-9]+}}], 0 addr64 offset:16{{$}}
-define void @atomic_umax_i32_addr64_offset(i32 addrspace(1)* %out, i32 %in, i64 %index) {
-entry:
-  %ptr = getelementptr i32, i32 addrspace(1)* %out, i64 %index
-  %gep = getelementptr i32, i32 addrspace(1)* %ptr, i32 4
-  %0  = atomicrmw volatile umax i32 addrspace(1)* %gep, i32 %in seq_cst
-  ret void
-}
-
-; FUNC-LABEL: {{^}}atomic_umax_i32_ret_addr64_offset:
-; SI: buffer_atomic_umax [[RET:v[0-9]+]], v[{{[0-9]+}}:{{[0-9]+}}], s[{{[0-9]+}}:{{[0-9]+}}], 0 addr64 offset:16 glc{{$}}
-; SI: buffer_store_dword [[RET]]
-define void @atomic_umax_i32_ret_addr64_offset(i32 addrspace(1)* %out, i32 addrspace(1)* %out2, i32 %in, i64 %index) {
-entry:
-  %ptr = getelementptr i32, i32 addrspace(1)* %out, i64 %index
-  %gep = getelementptr i32, i32 addrspace(1)* %ptr, i32 4
-  %0  = atomicrmw volatile umax i32 addrspace(1)* %gep, i32 %in seq_cst
-  store i32 %0, i32 addrspace(1)* %out2
-  ret void
-}
-
-; FUNC-LABEL: {{^}}atomic_umax_i32:
-; SI: buffer_atomic_umax v{{[0-9]+}}, s[{{[0-9]+}}:{{[0-9]+}}], 0{{$}}
-define void @atomic_umax_i32(i32 addrspace(1)* %out, i32 %in) {
-entry:
-  %0  = atomicrmw volatile umax i32 addrspace(1)* %out, i32 %in seq_cst
-  ret void
-}
-
-; FUNC-LABEL: {{^}}atomic_umax_i32_ret:
-; SI: buffer_atomic_umax [[RET:v[0-9]+]], s[{{[0-9]+}}:{{[0-9]+}}], 0 glc
-; SI: buffer_store_dword [[RET]]
-define void @atomic_umax_i32_ret(i32 addrspace(1)* %out, i32 addrspace(1)* %out2, i32 %in) {
-entry:
-  %0  = atomicrmw volatile umax i32 addrspace(1)* %out, i32 %in seq_cst
-  store i32 %0, i32 addrspace(1)* %out2
-  ret void
-}
-
-; FUNC-LABEL: {{^}}atomic_umax_i32_addr64:
-; SI: buffer_atomic_umax v{{[0-9]+}}, v[{{[0-9]+}}:{{[0-9]+}}], s[{{[0-9]+}}:{{[0-9]+}}], 0 addr64{{$}}
-define void @atomic_umax_i32_addr64(i32 addrspace(1)* %out, i32 %in, i64 %index) {
-entry:
-  %ptr = getelementptr i32, i32 addrspace(1)* %out, i64 %index
-  %0  = atomicrmw volatile umax i32 addrspace(1)* %ptr, i32 %in seq_cst
-  ret void
-}
-
-; FUNC-LABEL: {{^}}atomic_umax_i32_ret_addr64:
-; SI: buffer_atomic_umax [[RET:v[0-9]+]], v[{{[0-9]+}}:{{[0-9]+}}], s[{{[0-9]+}}:{{[0-9]+}}], 0 addr64 glc{{$}}
-; SI: buffer_store_dword [[RET]]
-define void @atomic_umax_i32_ret_addr64(i32 addrspace(1)* %out, i32 addrspace(1)* %out2, i32 %in, i64 %index) {
-entry:
-  %ptr = getelementptr i32, i32 addrspace(1)* %out, i64 %index
-  %0  = atomicrmw volatile umax i32 addrspace(1)* %ptr, i32 %in seq_cst
-  store i32 %0, i32 addrspace(1)* %out2
-  ret void
-}
-
-; FUNC-LABEL: {{^}}atomic_min_i32_offset:
-; SI: buffer_atomic_smin v{{[0-9]+}}, s[{{[0-9]+}}:{{[0-9]+}}], 0 offset:16{{$}}
-define void @atomic_min_i32_offset(i32 addrspace(1)* %out, i32 %in) {
-entry:
-  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
-  %0  = atomicrmw volatile min i32 addrspace(1)* %gep, i32 %in seq_cst
-  ret void
-}
-
-; FUNC-LABEL: {{^}}atomic_min_i32_ret_offset:
-; SI: buffer_atomic_smin [[RET:v[0-9]+]], s[{{[0-9]+}}:{{[0-9]+}}], 0 offset:16 glc {{$}}
-; SI: buffer_store_dword [[RET]]
-define void @atomic_min_i32_ret_offset(i32 addrspace(1)* %out, i32 addrspace(1)* %out2, i32 %in) {
-entry:
-  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
-  %0  = atomicrmw volatile min i32 addrspace(1)* %gep, i32 %in seq_cst
-  store i32 %0, i32 addrspace(1)* %out2
-  ret void
-}
-
-; FUNC-LABEL: {{^}}atomic_min_i32_addr64_offset:
-; SI: buffer_atomic_smin v{{[0-9]+}}, v[{{[0-9]+}}:{{[0-9]+}}], s[{{[0-9]+}}:{{[0-9]+}}], 0 addr64 offset:16{{$}}
-define void @atomic_min_i32_addr64_offset(i32 addrspace(1)* %out, i32 %in, i64 %index) {
-entry:
-  %ptr = getelementptr i32, i32 addrspace(1)* %out, i64 %index
-  %gep = getelementptr i32, i32 addrspace(1)* %ptr, i32 4
-  %0  = atomicrmw volatile min i32 addrspace(1)* %gep, i32 %in seq_cst
-  ret void
-}
-
-; FUNC-LABEL: {{^}}atomic_min_i32_ret_addr64_offset:
-; SI: buffer_atomic_smin [[RET:v[0-9]+]], v[{{[0-9]+}}:{{[0-9]+}}], s[{{[0-9]+}}:{{[0-9]+}}], 0 addr64 offset:16 glc{{$}}
-; SI: buffer_store_dword [[RET]]
-define void @atomic_min_i32_ret_addr64_offset(i32 addrspace(1)* %out, i32 addrspace(1)* %out2, i32 %in, i64 %index) {
-entry:
-  %ptr = getelementptr i32, i32 addrspace(1)* %out, i64 %index
-  %gep = getelementptr i32, i32 addrspace(1)* %ptr, i32 4
-  %0  = atomicrmw volatile min i32 addrspace(1)* %gep, i32 %in seq_cst
-  store i32 %0, i32 addrspace(1)* %out2
-  ret void
-}
-
-; FUNC-LABEL: {{^}}atomic_min_i32:
-; SI: buffer_atomic_smin v{{[0-9]+}}, s[{{[0-9]+}}:{{[0-9]+}}], 0{{$}}
-define void @atomic_min_i32(i32 addrspace(1)* %out, i32 %in) {
-entry:
-  %0  = atomicrmw volatile min i32 addrspace(1)* %out, i32 %in seq_cst
-  ret void
-}
-
-; FUNC-LABEL: {{^}}atomic_min_i32_ret:
-; SI: buffer_atomic_smin [[RET:v[0-9]+]], s[{{[0-9]+}}:{{[0-9]+}}], 0 glc
-; SI: buffer_store_dword [[RET]]
-define void @atomic_min_i32_ret(i32 addrspace(1)* %out, i32 addrspace(1)* %out2, i32 %in) {
-entry:
-  %0  = atomicrmw volatile min i32 addrspace(1)* %out, i32 %in seq_cst
-  store i32 %0, i32 addrspace(1)* %out2
-  ret void
-}
-
-; FUNC-LABEL: {{^}}atomic_min_i32_addr64:
-; SI: buffer_atomic_smin v{{[0-9]+}}, v[{{[0-9]+}}:{{[0-9]+}}], s[{{[0-9]+}}:{{[0-9]+}}], 0 addr64{{$}}
-define void @atomic_min_i32_addr64(i32 addrspace(1)* %out, i32 %in, i64 %index) {
-entry:
-  %ptr = getelementptr i32, i32 addrspace(1)* %out, i64 %index
-  %0  = atomicrmw volatile min i32 addrspace(1)* %ptr, i32 %in seq_cst
-  ret void
-}
-
-; FUNC-LABEL: {{^}}atomic_min_i32_ret_addr64:
-; SI: buffer_atomic_smin [[RET:v[0-9]+]], v[{{[0-9]+}}:{{[0-9]+}}], s[{{[0-9]+}}:{{[0-9]+}}], 0 addr64 glc{{$}}
-; SI: buffer_store_dword [[RET]]
-define void @atomic_min_i32_ret_addr64(i32 addrspace(1)* %out, i32 addrspace(1)* %out2, i32 %in, i64 %index) {
-entry:
-  %ptr = getelementptr i32, i32 addrspace(1)* %out, i64 %index
-  %0  = atomicrmw volatile min i32 addrspace(1)* %ptr, i32 %in seq_cst
-  store i32 %0, i32 addrspace(1)* %out2
-  ret void
-}
-
-; FUNC-LABEL: {{^}}atomic_umin_i32_offset:
-; SI: buffer_atomic_umin v{{[0-9]+}}, s[{{[0-9]+}}:{{[0-9]+}}], 0 offset:16{{$}}
-define void @atomic_umin_i32_offset(i32 addrspace(1)* %out, i32 %in) {
-entry:
-  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
-  %0  = atomicrmw volatile umin i32 addrspace(1)* %gep, i32 %in seq_cst
-  ret void
-}
-
-; FUNC-LABEL: {{^}}atomic_umin_i32_ret_offset:
-; SI: buffer_atomic_umin [[RET:v[0-9]+]], s[{{[0-9]+}}:{{[0-9]+}}], 0 offset:16 glc {{$}}
-; SI: buffer_store_dword [[RET]]
-define void @atomic_umin_i32_ret_offset(i32 addrspace(1)* %out, i32 addrspace(1)* %out2, i32 %in) {
-entry:
-  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
-  %0  = atomicrmw volatile umin i32 addrspace(1)* %gep, i32 %in seq_cst
-  store i32 %0, i32 addrspace(1)* %out2
-  ret void
-}
-
-; FUNC-LABEL: {{^}}atomic_umin_i32_addr64_offset:
-; SI: buffer_atomic_umin v{{[0-9]+}}, v[{{[0-9]+}}:{{[0-9]+}}], s[{{[0-9]+}}:{{[0-9]+}}], 0 addr64 offset:16{{$}}
-define void @atomic_umin_i32_addr64_offset(i32 addrspace(1)* %out, i32 %in, i64 %index) {
-entry:
-  %ptr = getelementptr i32, i32 addrspace(1)* %out, i64 %index
-  %gep = getelementptr i32, i32 addrspace(1)* %ptr, i32 4
-  %0  = atomicrmw volatile umin i32 addrspace(1)* %gep, i32 %in seq_cst
-  ret void
-}
-
-; FUNC-LABEL: {{^}}atomic_umin_i32_ret_addr64_offset:
-; SI: buffer_atomic_umin [[RET:v[0-9]+]], v[{{[0-9]+}}:{{[0-9]+}}], s[{{[0-9]+}}:{{[0-9]+}}], 0 addr64 offset:16 glc{{$}}
-; SI: buffer_store_dword [[RET]]
-define void @atomic_umin_i32_ret_addr64_offset(i32 addrspace(1)* %out, i32 addrspace(1)* %out2, i32 %in, i64 %index) {
-entry:
-  %ptr = getelementptr i32, i32 addrspace(1)* %out, i64 %index
-  %gep = getelementptr i32, i32 addrspace(1)* %ptr, i32 4
-  %0  = atomicrmw volatile umin i32 addrspace(1)* %gep, i32 %in seq_cst
-  store i32 %0, i32 addrspace(1)* %out2
-  ret void
-}
-
-; FUNC-LABEL: {{^}}atomic_umin_i32:
-; SI: buffer_atomic_umin v{{[0-9]+}}, s[{{[0-9]+}}:{{[0-9]+}}], 0{{$}}
-define void @atomic_umin_i32(i32 addrspace(1)* %out, i32 %in) {
-entry:
-  %0  = atomicrmw volatile umin i32 addrspace(1)* %out, i32 %in seq_cst
-  ret void
-}
-
-; FUNC-LABEL: {{^}}atomic_umin_i32_ret:
-; SI: buffer_atomic_umin [[RET:v[0-9]+]], s[{{[0-9]+}}:{{[0-9]+}}], 0 glc
-; SI: buffer_store_dword [[RET]]
-define void @atomic_umin_i32_ret(i32 addrspace(1)* %out, i32 addrspace(1)* %out2, i32 %in) {
-entry:
-  %0  = atomicrmw volatile umin i32 addrspace(1)* %out, i32 %in seq_cst
-  store i32 %0, i32 addrspace(1)* %out2
-  ret void
-}
-
-; FUNC-LABEL: {{^}}atomic_umin_i32_addr64:
-; SI: buffer_atomic_umin v{{[0-9]+}}, v[{{[0-9]+}}:{{[0-9]+}}], s[{{[0-9]+}}:{{[0-9]+}}], 0 addr64{{$}}
-define void @atomic_umin_i32_addr64(i32 addrspace(1)* %out, i32 %in, i64 %index) {
-entry:
-  %ptr = getelementptr i32, i32 addrspace(1)* %out, i64 %index
-  %0  = atomicrmw volatile umin i32 addrspace(1)* %ptr, i32 %in seq_cst
-  ret void
-}
-
-; FUNC-LABEL: {{^}}atomic_umin_i32_ret_addr64:
-; SI: buffer_atomic_umin [[RET:v[0-9]+]], v[{{[0-9]+}}:{{[0-9]+}}], s[{{[0-9]+}}:{{[0-9]+}}], 0 addr64 glc{{$}}
-; SI: buffer_store_dword [[RET]]
-define void @atomic_umin_i32_ret_addr64(i32 addrspace(1)* %out, i32 addrspace(1)* %out2, i32 %in, i64 %index) {
-entry:
-  %ptr = getelementptr i32, i32 addrspace(1)* %out, i64 %index
-  %0  = atomicrmw volatile umin i32 addrspace(1)* %ptr, i32 %in seq_cst
-  store i32 %0, i32 addrspace(1)* %out2
-  ret void
-}
-
-; FUNC-LABEL: {{^}}atomic_or_i32_offset:
-; SI: buffer_atomic_or v{{[0-9]+}}, s[{{[0-9]+}}:{{[0-9]+}}], 0 offset:16{{$}}
-define void @atomic_or_i32_offset(i32 addrspace(1)* %out, i32 %in) {
-entry:
-  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
-  %0  = atomicrmw volatile or i32 addrspace(1)* %gep, i32 %in seq_cst
-  ret void
-}
-
-; FUNC-LABEL: {{^}}atomic_or_i32_ret_offset:
-; SI: buffer_atomic_or [[RET:v[0-9]+]], s[{{[0-9]+}}:{{[0-9]+}}], 0 offset:16 glc {{$}}
-; SI: buffer_store_dword [[RET]]
-define void @atomic_or_i32_ret_offset(i32 addrspace(1)* %out, i32 addrspace(1)* %out2, i32 %in) {
-entry:
-  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
-  %0  = atomicrmw volatile or i32 addrspace(1)* %gep, i32 %in seq_cst
-  store i32 %0, i32 addrspace(1)* %out2
-  ret void
-}
-
-; FUNC-LABEL: {{^}}atomic_or_i32_addr64_offset:
-; SI: buffer_atomic_or v{{[0-9]+}}, v[{{[0-9]+}}:{{[0-9]+}}], s[{{[0-9]+}}:{{[0-9]+}}], 0 addr64 offset:16{{$}}
-define void @atomic_or_i32_addr64_offset(i32 addrspace(1)* %out, i32 %in, i64 %index) {
-entry:
-  %ptr = getelementptr i32, i32 addrspace(1)* %out, i64 %index
-  %gep = getelementptr i32, i32 addrspace(1)* %ptr, i32 4
-  %0  = atomicrmw volatile or i32 addrspace(1)* %gep, i32 %in seq_cst
-  ret void
-}
-
-; FUNC-LABEL: {{^}}atomic_or_i32_ret_addr64_offset:
-; SI: buffer_atomic_or [[RET:v[0-9]+]], v[{{[0-9]+}}:{{[0-9]+}}], s[{{[0-9]+}}:{{[0-9]+}}], 0 addr64 offset:16 glc{{$}}
-; SI: buffer_store_dword [[RET]]
-define void @atomic_or_i32_ret_addr64_offset(i32 addrspace(1)* %out, i32 addrspace(1)* %out2, i32 %in, i64 %index) {
-entry:
-  %ptr = getelementptr i32, i32 addrspace(1)* %out, i64 %index
-  %gep = getelementptr i32, i32 addrspace(1)* %ptr, i32 4
-  %0  = atomicrmw volatile or i32 addrspace(1)* %gep, i32 %in seq_cst
-  store i32 %0, i32 addrspace(1)* %out2
-  ret void
-}
-
-; FUNC-LABEL: {{^}}atomic_or_i32:
-; SI: buffer_atomic_or v{{[0-9]+}}, s[{{[0-9]+}}:{{[0-9]+}}], 0{{$}}
-define void @atomic_or_i32(i32 addrspace(1)* %out, i32 %in) {
-entry:
-  %0  = atomicrmw volatile or i32 addrspace(1)* %out, i32 %in seq_cst
-  ret void
-}
-
-; FUNC-LABEL: {{^}}atomic_or_i32_ret:
-; SI: buffer_atomic_or [[RET:v[0-9]+]], s[{{[0-9]+}}:{{[0-9]+}}], 0 glc
-; SI: buffer_store_dword [[RET]]
-define void @atomic_or_i32_ret(i32 addrspace(1)* %out, i32 addrspace(1)* %out2, i32 %in) {
-entry:
-  %0  = atomicrmw volatile or i32 addrspace(1)* %out, i32 %in seq_cst
-  store i32 %0, i32 addrspace(1)* %out2
-  ret void
-}
-
-; FUNC-LABEL: {{^}}atomic_or_i32_addr64:
-; SI: buffer_atomic_or v{{[0-9]+}}, v[{{[0-9]+}}:{{[0-9]+}}], s[{{[0-9]+}}:{{[0-9]+}}], 0 addr64{{$}}
-define void @atomic_or_i32_addr64(i32 addrspace(1)* %out, i32 %in, i64 %index) {
-entry:
-  %ptr = getelementptr i32, i32 addrspace(1)* %out, i64 %index
-  %0  = atomicrmw volatile or i32 addrspace(1)* %ptr, i32 %in seq_cst
-  ret void
-}
-
-; FUNC-LABEL: {{^}}atomic_or_i32_ret_addr64:
-; SI: buffer_atomic_or [[RET:v[0-9]+]], v[{{[0-9]+}}:{{[0-9]+}}], s[{{[0-9]+}}:{{[0-9]+}}], 0 addr64 glc{{$}}
-; SI: buffer_store_dword [[RET]]
-define void @atomic_or_i32_ret_addr64(i32 addrspace(1)* %out, i32 addrspace(1)* %out2, i32 %in, i64 %index) {
-entry:
-  %ptr = getelementptr i32, i32 addrspace(1)* %out, i64 %index
-  %0  = atomicrmw volatile or i32 addrspace(1)* %ptr, i32 %in seq_cst
-  store i32 %0, i32 addrspace(1)* %out2
-  ret void
-}
-
-; FUNC-LABEL: {{^}}atomic_xchg_i32_offset:
-; SI: buffer_atomic_swap v{{[0-9]+}}, s[{{[0-9]+}}:{{[0-9]+}}], 0 offset:16{{$}}
-define void @atomic_xchg_i32_offset(i32 addrspace(1)* %out, i32 %in) {
-entry:
-  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
-  %0  = atomicrmw volatile xchg i32 addrspace(1)* %gep, i32 %in seq_cst
-  ret void
-}
-
-; FUNC-LABEL: {{^}}atomic_xchg_i32_ret_offset:
-; SI: buffer_atomic_swap [[RET:v[0-9]+]], s[{{[0-9]+}}:{{[0-9]+}}], 0 offset:16 glc {{$}}
-; SI: buffer_store_dword [[RET]]
-define void @atomic_xchg_i32_ret_offset(i32 addrspace(1)* %out, i32 addrspace(1)* %out2, i32 %in) {
-entry:
-  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
-  %0  = atomicrmw volatile xchg i32 addrspace(1)* %gep, i32 %in seq_cst
-  store i32 %0, i32 addrspace(1)* %out2
-  ret void
-}
-
-; FUNC-LABEL: {{^}}atomic_xchg_i32_addr64_offset:
-; SI: buffer_atomic_swap v{{[0-9]+}}, v[{{[0-9]+}}:{{[0-9]+}}], s[{{[0-9]+}}:{{[0-9]+}}], 0 addr64 offset:16{{$}}
-define void @atomic_xchg_i32_addr64_offset(i32 addrspace(1)* %out, i32 %in, i64 %index) {
-entry:
-  %ptr = getelementptr i32, i32 addrspace(1)* %out, i64 %index
-  %gep = getelementptr i32, i32 addrspace(1)* %ptr, i32 4
-  %0  = atomicrmw volatile xchg i32 addrspace(1)* %gep, i32 %in seq_cst
-  ret void
-}
-
-; FUNC-LABEL: {{^}}atomic_xchg_i32_ret_addr64_offset:
-; SI: buffer_atomic_swap [[RET:v[0-9]+]], v[{{[0-9]+}}:{{[0-9]+}}], s[{{[0-9]+}}:{{[0-9]+}}], 0 addr64 offset:16 glc{{$}}
-; SI: buffer_store_dword [[RET]]
-define void @atomic_xchg_i32_ret_addr64_offset(i32 addrspace(1)* %out, i32 addrspace(1)* %out2, i32 %in, i64 %index) {
-entry:
-  %ptr = getelementptr i32, i32 addrspace(1)* %out, i64 %index
-  %gep = getelementptr i32, i32 addrspace(1)* %ptr, i32 4
-  %0  = atomicrmw volatile xchg i32 addrspace(1)* %gep, i32 %in seq_cst
-  store i32 %0, i32 addrspace(1)* %out2
-  ret void
-}
-
-; FUNC-LABEL: {{^}}atomic_xchg_i32:
-; SI: buffer_atomic_swap v{{[0-9]+}}, s[{{[0-9]+}}:{{[0-9]+}}], 0{{$}}
-define void @atomic_xchg_i32(i32 addrspace(1)* %out, i32 %in) {
-entry:
-  %0  = atomicrmw volatile xchg i32 addrspace(1)* %out, i32 %in seq_cst
-  ret void
-}
-
-; FUNC-LABEL: {{^}}atomic_xchg_i32_ret:
-; SI: buffer_atomic_swap [[RET:v[0-9]+]], s[{{[0-9]+}}:{{[0-9]+}}], 0 glc
-; SI: buffer_store_dword [[RET]]
-define void @atomic_xchg_i32_ret(i32 addrspace(1)* %out, i32 addrspace(1)* %out2, i32 %in) {
-entry:
-  %0  = atomicrmw volatile xchg i32 addrspace(1)* %out, i32 %in seq_cst
-  store i32 %0, i32 addrspace(1)* %out2
-  ret void
-}
-
-; FUNC-LABEL: {{^}}atomic_xchg_i32_addr64:
-; SI: buffer_atomic_swap v{{[0-9]+}}, v[{{[0-9]+}}:{{[0-9]+}}], s[{{[0-9]+}}:{{[0-9]+}}], 0 addr64{{$}}
-define void @atomic_xchg_i32_addr64(i32 addrspace(1)* %out, i32 %in, i64 %index) {
-entry:
-  %ptr = getelementptr i32, i32 addrspace(1)* %out, i64 %index
-  %0  = atomicrmw volatile xchg i32 addrspace(1)* %ptr, i32 %in seq_cst
-  ret void
-}
-
-; FUNC-LABEL: {{^}}atomic_xchg_i32_ret_addr64:
-; SI: buffer_atomic_swap [[RET:v[0-9]+]], v[{{[0-9]+}}:{{[0-9]+}}], s[{{[0-9]+}}:{{[0-9]+}}], 0 addr64 glc{{$}}
-; SI: buffer_store_dword [[RET]]
-define void @atomic_xchg_i32_ret_addr64(i32 addrspace(1)* %out, i32 addrspace(1)* %out2, i32 %in, i64 %index) {
-entry:
-  %ptr = getelementptr i32, i32 addrspace(1)* %out, i64 %index
-  %0  = atomicrmw volatile xchg i32 addrspace(1)* %ptr, i32 %in seq_cst
-  store i32 %0, i32 addrspace(1)* %out2
-  ret void
-}
-
-; FUNC-LABEL: {{^}}atomic_xor_i32_offset:
-; SI: buffer_atomic_xor v{{[0-9]+}}, s[{{[0-9]+}}:{{[0-9]+}}], 0 offset:16{{$}}
-define void @atomic_xor_i32_offset(i32 addrspace(1)* %out, i32 %in) {
-entry:
-  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
-  %0  = atomicrmw volatile xor i32 addrspace(1)* %gep, i32 %in seq_cst
-  ret void
-}
-
-; FUNC-LABEL: {{^}}atomic_xor_i32_ret_offset:
-; SI: buffer_atomic_xor [[RET:v[0-9]+]], s[{{[0-9]+}}:{{[0-9]+}}], 0 offset:16 glc {{$}}
-; SI: buffer_store_dword [[RET]]
-define void @atomic_xor_i32_ret_offset(i32 addrspace(1)* %out, i32 addrspace(1)* %out2, i32 %in) {
-entry:
-  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
-  %0  = atomicrmw volatile xor i32 addrspace(1)* %gep, i32 %in seq_cst
-  store i32 %0, i32 addrspace(1)* %out2
-  ret void
-}
-
-; FUNC-LABEL: {{^}}atomic_xor_i32_addr64_offset:
-; SI: buffer_atomic_xor v{{[0-9]+}}, v[{{[0-9]+}}:{{[0-9]+}}], s[{{[0-9]+}}:{{[0-9]+}}], 0 addr64 offset:16{{$}}
-define void @atomic_xor_i32_addr64_offset(i32 addrspace(1)* %out, i32 %in, i64 %index) {
-entry:
-  %ptr = getelementptr i32, i32 addrspace(1)* %out, i64 %index
-  %gep = getelementptr i32, i32 addrspace(1)* %ptr, i32 4
-  %0  = atomicrmw volatile xor i32 addrspace(1)* %gep, i32 %in seq_cst
-  ret void
-}
-
-; FUNC-LABEL: {{^}}atomic_xor_i32_ret_addr64_offset:
-; SI: buffer_atomic_xor [[RET:v[0-9]+]], v[{{[0-9]+}}:{{[0-9]+}}], s[{{[0-9]+}}:{{[0-9]+}}], 0 addr64 offset:16 glc{{$}}
-; SI: buffer_store_dword [[RET]]
-define void @atomic_xor_i32_ret_addr64_offset(i32 addrspace(1)* %out, i32 addrspace(1)* %out2, i32 %in, i64 %index) {
-entry:
-  %ptr = getelementptr i32, i32 addrspace(1)* %out, i64 %index
-  %gep = getelementptr i32, i32 addrspace(1)* %ptr, i32 4
-  %0  = atomicrmw volatile xor i32 addrspace(1)* %gep, i32 %in seq_cst
-  store i32 %0, i32 addrspace(1)* %out2
-  ret void
-}
-
-; FUNC-LABEL: {{^}}atomic_xor_i32:
-; SI: buffer_atomic_xor v{{[0-9]+}}, s[{{[0-9]+}}:{{[0-9]+}}], 0{{$}}
-define void @atomic_xor_i32(i32 addrspace(1)* %out, i32 %in) {
-entry:
-  %0  = atomicrmw volatile xor i32 addrspace(1)* %out, i32 %in seq_cst
-  ret void
-}
-
-; FUNC-LABEL: {{^}}atomic_xor_i32_ret:
-; SI: buffer_atomic_xor [[RET:v[0-9]+]], s[{{[0-9]+}}:{{[0-9]+}}], 0 glc
-; SI: buffer_store_dword [[RET]]
-define void @atomic_xor_i32_ret(i32 addrspace(1)* %out, i32 addrspace(1)* %out2, i32 %in) {
-entry:
-  %0  = atomicrmw volatile xor i32 addrspace(1)* %out, i32 %in seq_cst
-  store i32 %0, i32 addrspace(1)* %out2
-  ret void
-}
-
-; FUNC-LABEL: {{^}}atomic_xor_i32_addr64:
-; SI: buffer_atomic_xor v{{[0-9]+}}, v[{{[0-9]+}}:{{[0-9]+}}], s[{{[0-9]+}}:{{[0-9]+}}], 0 addr64{{$}}
-define void @atomic_xor_i32_addr64(i32 addrspace(1)* %out, i32 %in, i64 %index) {
-entry:
-  %ptr = getelementptr i32, i32 addrspace(1)* %out, i64 %index
-  %0  = atomicrmw volatile xor i32 addrspace(1)* %ptr, i32 %in seq_cst
-  ret void
-}
-
-; FUNC-LABEL: {{^}}atomic_xor_i32_ret_addr64:
-; SI: buffer_atomic_xor [[RET:v[0-9]+]], v[{{[0-9]+}}:{{[0-9]+}}], s[{{[0-9]+}}:{{[0-9]+}}], 0 addr64 glc{{$}}
-; SI: buffer_store_dword [[RET]]
-define void @atomic_xor_i32_ret_addr64(i32 addrspace(1)* %out, i32 addrspace(1)* %out2, i32 %in, i64 %index) {
-entry:
-  %ptr = getelementptr i32, i32 addrspace(1)* %out, i64 %index
-  %0  = atomicrmw volatile xor i32 addrspace(1)* %ptr, i32 %in seq_cst
-  store i32 %0, i32 addrspace(1)* %out2
-  ret void
-}
diff --git a/test/CodeGen/R600/half.ll b/test/CodeGen/R600/half.ll
deleted file mode 100644
index 42ee788e88d..00000000000
--- a/test/CodeGen/R600/half.ll
+++ /dev/null
@@ -1,62 +0,0 @@
-; RUN: llc < %s -march=amdgcn -mcpu=SI | FileCheck %s
-; RUN: llc < %s -march=amdgcn -mcpu=tonga | FileCheck %s
-
-define void @test_load_store(half addrspace(1)* %in, half addrspace(1)* %out) {
-; CHECK-LABEL: {{^}}test_load_store:
-; CHECK: buffer_load_ushort [[TMP:v[0-9]+]]
-; CHECK: buffer_store_short [[TMP]]
-  %val = load half, half addrspace(1)* %in
-  store half %val, half addrspace(1) * %out
-  ret void
-}
-
-define void @test_bitcast_from_half(half addrspace(1)* %in, i16 addrspace(1)* %out) {
-; CHECK-LABEL: {{^}}test_bitcast_from_half:
-; CHECK: buffer_load_ushort [[TMP:v[0-9]+]]
-; CHECK: buffer_store_short [[TMP]]
-  %val = load half, half addrspace(1) * %in
-  %val_int = bitcast half %val to i16
-  store i16 %val_int, i16 addrspace(1)* %out
-  ret void
-}
-
-define void @test_bitcast_to_half(half addrspace(1)* %out, i16 addrspace(1)* %in) {
-; CHECK-LABEL: {{^}}test_bitcast_to_half:
-; CHECK: buffer_load_ushort [[TMP:v[0-9]+]]
-; CHECK: buffer_store_short [[TMP]]
-  %val = load i16, i16 addrspace(1)* %in
-  %val_fp = bitcast i16 %val to half
-  store half %val_fp, half addrspace(1)* %out
-  ret void
-}
-
-define void @test_extend32(half addrspace(1)* %in, float addrspace(1)* %out) {
-; CHECK-LABEL: {{^}}test_extend32:
-; CHECK: v_cvt_f32_f16_e32
-
-  %val16 = load half, half addrspace(1)* %in
-  %val32 = fpext half %val16 to float
-  store float %val32, float addrspace(1)* %out
-  ret void
-}
-
-define void @test_extend64(half addrspace(1)* %in, double addrspace(1)* %out) {
-; CHECK-LABEL: {{^}}test_extend64:
-; CHECK: v_cvt_f32_f16_e32
-; CHECK: v_cvt_f64_f32_e32
-
-  %val16 = load half, half addrspace(1)* %in
-  %val64 = fpext half %val16 to double
-  store double %val64, double addrspace(1)* %out
-  ret void
-}
-
-define void @test_trunc32(float addrspace(1)* %in, half addrspace(1)* %out) {
-; CHECK-LABEL: {{^}}test_trunc32:
-; CHECK: v_cvt_f16_f32_e32
-
-  %val32 = load float, float addrspace(1)* %in
-  %val16 = fptrunc float %val32 to half
-  store half %val16, half addrspace(1)* %out
-  ret void
-}
diff --git a/test/CodeGen/R600/hsa.ll b/test/CodeGen/R600/hsa.ll
deleted file mode 100644
index f9113399afe..00000000000
--- a/test/CodeGen/R600/hsa.ll
+++ /dev/null
@@ -1,14 +0,0 @@
-; RUN: llc < %s -mtriple=amdgcn--amdhsa -mcpu=kaveri | FileCheck --check-prefix=HSA %s
-
-; HSA: .section        .hsa.version
-; HSA-NEXT: .ascii  "HSA Code Unit:0.0:AMD:0.1:GFX8.1:0"
-; HSA: {{^}}simple:
-; Make sure we are setting the ATC bit:
-; HSA: s_mov_b32 s[[HI:[0-9]]], 0x100f000
-; HSA: buffer_store_dword v{{[0-9]+}}, s[0:[[HI]]], 0
-
-define void @simple(i32 addrspace(1)* %out) {
-entry:
-  store i32 0, i32 addrspace(1)* %out
-  ret void
-}
diff --git a/test/CodeGen/R600/indirect-addressing-si.ll b/test/CodeGen/R600/indirect-addressing-si.ll
deleted file mode 100644
index 319910fcabf..00000000000
--- a/test/CodeGen/R600/indirect-addressing-si.ll
+++ /dev/null
@@ -1,49 +0,0 @@
-; RUN: llc < %s -march=amdgcn -mcpu=SI -verify-machineinstrs | FileCheck %s
-; RUN: llc < %s -march=amdgcn -mcpu=tonga -verify-machineinstrs | FileCheck %s
-
-; Tests for indirect addressing on SI, which is implemented using dynamic
-; indexing of vectors.
-
-; CHECK-LABEL: {{^}}extract_w_offset:
-; CHECK: s_mov_b32 m0
-; CHECK-NEXT: v_movrels_b32_e32
-define void @extract_w_offset(float addrspace(1)* %out, i32 %in) {
-entry:
-  %0 = add i32 %in, 1
-  %1 = extractelement <4 x float> <float 1.0, float 2.0, float 3.0, float 4.0>, i32 %0
-  store float %1, float addrspace(1)* %out
-  ret void
-}
-
-; CHECK-LABEL: {{^}}extract_wo_offset:
-; CHECK: s_mov_b32 m0
-; CHECK-NEXT: v_movrels_b32_e32
-define void @extract_wo_offset(float addrspace(1)* %out, i32 %in) {
-entry:
-  %0 = extractelement <4 x float> <float 1.0, float 2.0, float 3.0, float 4.0>, i32 %in
-  store float %0, float addrspace(1)* %out
-  ret void
-}
-
-; CHECK-LABEL: {{^}}insert_w_offset:
-; CHECK: s_mov_b32 m0
-; CHECK-NEXT: v_movreld_b32_e32
-define void @insert_w_offset(float addrspace(1)* %out, i32 %in) {
-entry:
-  %0 = add i32 %in, 1
-  %1 = insertelement <4 x float> <float 1.0, float 2.0, float 3.0, float 4.0>, float 5.0, i32 %0
-  %2 = extractelement <4 x float> %1, i32 2
-  store float %2, float addrspace(1)* %out
-  ret void
-}
-
-; CHECK-LABEL: {{^}}insert_wo_offset:
-; CHECK: s_mov_b32 m0
-; CHECK-NEXT: v_movreld_b32_e32
-define void @insert_wo_offset(float addrspace(1)* %out, i32 %in) {
-entry:
-  %0 = insertelement <4 x float> <float 1.0, float 2.0, float 3.0, float 4.0>, float 5.0, i32 %in
-  %1 = extractelement <4 x float> %0, i32 2
-  store float %1, float addrspace(1)* %out
-  ret void
-}
diff --git a/test/CodeGen/R600/lit.local.cfg b/test/CodeGen/R600/lit.local.cfg
deleted file mode 100644
index ad9ce2541ef..00000000000
--- a/test/CodeGen/R600/lit.local.cfg
+++ /dev/null
@@ -1,2 +0,0 @@
-if not 'R600' in config.root.targets:
-    config.unsupported = True
diff --git a/test/CodeGen/R600/llvm.AMDGPU.kill.ll b/test/CodeGen/R600/llvm.AMDGPU.kill.ll
deleted file mode 100644
index d1ff3b1b7d6..00000000000
--- a/test/CodeGen/R600/llvm.AMDGPU.kill.ll
+++ /dev/null
@@ -1,23 +0,0 @@
-; RUN: llc -march=amdgcn -mcpu=verde -verify-machineinstrs < %s | FileCheck -check-prefix=SI %s
-; RUN: llc -march=amdgcn -mcpu=tonga -verify-machineinstrs < %s | FileCheck -check-prefix=SI %s
-
-; SI-LABEL: {{^}}kill_gs_const:
-; SI-NOT: v_cmpx_le_f32
-; SI: s_mov_b64 exec, 0
-
-define void @kill_gs_const() #0 {
-main_body:
-  %0 = icmp ule i32 0, 3
-  %1 = select i1 %0, float 1.000000e+00, float -1.000000e+00
-  call void @llvm.AMDGPU.kill(float %1)
-  %2 = icmp ule i32 3, 0
-  %3 = select i1 %2, float 1.000000e+00, float -1.000000e+00
-  call void @llvm.AMDGPU.kill(float %3)
-  ret void
-}
-
-declare void @llvm.AMDGPU.kill(float)
-
-attributes #0 = { "ShaderType"="2" }
-
-!0 = !{!"const", null, i32 1}
diff --git a/test/CodeGen/R600/llvm.SI.fs.interp.ll b/test/CodeGen/R600/llvm.SI.fs.interp.ll
deleted file mode 100644
index 6b36140b06c..00000000000
--- a/test/CodeGen/R600/llvm.SI.fs.interp.ll
+++ /dev/null
@@ -1,30 +0,0 @@
-;RUN: llc < %s -march=amdgcn -mcpu=verde -verify-machineinstrs | FileCheck %s
-;RUN: llc < %s -march=amdgcn -mcpu=tonga -verify-machineinstrs | FileCheck %s
-
-;CHECK-NOT: s_wqm
-;CHECK: s_mov_b32
-;CHECK-NEXT: v_interp_mov_f32
-;CHECK: v_interp_p1_f32
-;CHECK: v_interp_p2_f32
-
-define void @main(<16 x i8> addrspace(2)* inreg, <16 x i8> addrspace(2)* inreg, <32 x i8> addrspace(2)* inreg, i32 inreg, <2 x i32>) #0 {
-main_body:
-  %5 = call float @llvm.SI.fs.constant(i32 0, i32 0, i32 %3)
-  %6 = call float @llvm.SI.fs.interp(i32 0, i32 0, i32 %3, <2 x i32> %4)
-  %7 = call float @llvm.SI.fs.interp(i32 1, i32 0, i32 %3, <2 x i32> %4)
-  call void @llvm.SI.export(i32 15, i32 1, i32 1, i32 0, i32 1, float %5, float %6, float %7, float %7)
-  ret void
-}
-
-declare void @llvm.AMDGPU.shader.type(i32)
-
-; Function Attrs: nounwind readnone
-declare float @llvm.SI.fs.constant(i32, i32, i32) #1
-
-; Function Attrs: nounwind readnone
-declare float @llvm.SI.fs.interp(i32, i32, i32, <2 x i32>) #1
-
-declare void @llvm.SI.export(i32, i32, i32, i32, i32, float, float, float, float)
-
-attributes #0 = { "ShaderType"="0" }
-attributes #1 = { nounwind readnone }
diff --git a/test/CodeGen/R600/llvm.sqrt.ll b/test/CodeGen/R600/llvm.sqrt.ll
deleted file mode 100644
index cc4717a944d..00000000000
--- a/test/CodeGen/R600/llvm.sqrt.ll
+++ /dev/null
@@ -1,55 +0,0 @@
-; RUN: llc < %s -march=r600 --mcpu=redwood | FileCheck %s --check-prefix=R600
-; RUN: llc < %s -march=amdgcn --mcpu=SI -verify-machineinstrs| FileCheck %s --check-prefix=SI
-; RUN: llc < %s -march=amdgcn --mcpu=tonga -verify-machineinstrs| FileCheck %s --check-prefix=SI
-
-; R600-LABEL: {{^}}sqrt_f32:
-; R600: RECIPSQRT_CLAMPED * T{{[0-9]\.[XYZW]}}, KC0[2].Z
-; R600: MUL NON-IEEE T{{[0-9]\.[XYZW]}}, KC0[2].Z, PS
-; SI-LABEL: {{^}}sqrt_f32:
-; SI: v_sqrt_f32_e32
-define void @sqrt_f32(float addrspace(1)* %out, float %in) {
-entry:
-  %0 = call float @llvm.sqrt.f32(float %in)
-  store float %0, float addrspace(1)* %out
-  ret void
-}
-
-; R600-LABEL: {{^}}sqrt_v2f32:
-; R600-DAG: RECIPSQRT_CLAMPED * T{{[0-9]\.[XYZW]}}, KC0[2].W
-; R600-DAG: MUL NON-IEEE T{{[0-9]\.[XYZW]}}, KC0[2].W, PS
-; R600-DAG: RECIPSQRT_CLAMPED * T{{[0-9]\.[XYZW]}}, KC0[3].X
-; R600-DAG: MUL NON-IEEE T{{[0-9]\.[XYZW]}}, KC0[3].X, PS
-; SI-LABEL: {{^}}sqrt_v2f32:
-; SI: v_sqrt_f32_e32
-; SI: v_sqrt_f32_e32
-define void @sqrt_v2f32(<2 x float> addrspace(1)* %out, <2 x float> %in) {
-entry:
-  %0 = call <2 x float> @llvm.sqrt.v2f32(<2 x float> %in)
-  store <2 x float> %0, <2 x float> addrspace(1)* %out
-  ret void
-}
-
-; R600-LABEL: {{^}}sqrt_v4f32:
-; R600-DAG: RECIPSQRT_CLAMPED * T{{[0-9]\.[XYZW]}}, KC0[3].Y
-; R600-DAG: MUL NON-IEEE T{{[0-9]\.[XYZW]}}, KC0[3].Y, PS
-; R600-DAG: RECIPSQRT_CLAMPED * T{{[0-9]\.[XYZW]}}, KC0[3].Z
-; R600-DAG: MUL NON-IEEE T{{[0-9]\.[XYZW]}}, KC0[3].Z, PS
-; R600-DAG: RECIPSQRT_CLAMPED * T{{[0-9]\.[XYZW]}}, KC0[3].W
-; R600-DAG: MUL NON-IEEE T{{[0-9]\.[XYZW]}}, KC0[3].W, PS
-; R600-DAG: RECIPSQRT_CLAMPED * T{{[0-9]\.[XYZW]}}, KC0[4].X
-; R600-DAG: MUL NON-IEEE T{{[0-9]\.[XYZW]}}, KC0[4].X, PS
-; SI-LABEL: {{^}}sqrt_v4f32:
-; SI: v_sqrt_f32_e32
-; SI: v_sqrt_f32_e32
-; SI: v_sqrt_f32_e32
-; SI: v_sqrt_f32_e32
-define void @sqrt_v4f32(<4 x float> addrspace(1)* %out, <4 x float> %in) {
-entry:
-  %0 = call <4 x float> @llvm.sqrt.v4f32(<4 x float> %in)
-  store <4 x float> %0, <4 x float> addrspace(1)* %out
-  ret void
-}
-
-declare float @llvm.sqrt.f32(float %in)
-declare <2 x float> @llvm.sqrt.v2f32(<2 x float> %in)
-declare <4 x float> @llvm.sqrt.v4f32(<4 x float> %in)
diff --git a/test/CodeGen/R600/load.ll b/test/CodeGen/R600/load.ll
deleted file mode 100644
index e285831b7dc..00000000000
--- a/test/CodeGen/R600/load.ll
+++ /dev/null
@@ -1,740 +0,0 @@
-; RUN: llc < %s -march=r600 -mcpu=redwood | FileCheck --check-prefix=R600 --check-prefix=FUNC %s
-; RUN: llc < %s -march=r600 -mcpu=cayman | FileCheck --check-prefix=R600 --check-prefix=FUNC %s
-; RUN: llc < %s -march=amdgcn -mcpu=SI -verify-machineinstrs | FileCheck --check-prefix=SI --check-prefix=FUNC %s
-; RUN: llc < %s -march=amdgcn -mcpu=tonga -verify-machineinstrs | FileCheck --check-prefix=SI --check-prefix=FUNC %s
-
-;===------------------------------------------------------------------------===;
-; GLOBAL ADDRESS SPACE
-;===------------------------------------------------------------------------===;
-
-; Load an i8 value from the global address space.
-; FUNC-LABEL: {{^}}load_i8:
-; R600: VTX_READ_8 T{{[0-9]+\.X, T[0-9]+\.X}}
-
-; SI: buffer_load_ubyte v{{[0-9]+}},
-define void @load_i8(i32 addrspace(1)* %out, i8 addrspace(1)* %in) {
-  %1 = load i8, i8 addrspace(1)* %in
-  %2 = zext i8 %1 to i32
-  store i32 %2, i32 addrspace(1)* %out
-  ret void
-}
-
-; FUNC-LABEL: {{^}}load_i8_sext:
-; R600: VTX_READ_8 [[DST:T[0-9]\.[XYZW]]], [[DST]]
-; R600: LSHL {{[* ]*}}T{{[0-9]}}.[[LSHL_CHAN:[XYZW]]], [[DST]]
-; R600: 24
-; R600: ASHR {{[* ]*}}T{{[0-9]\.[XYZW]}}, PV.[[LSHL_CHAN]]
-; R600: 24
-; SI: buffer_load_sbyte
-define void @load_i8_sext(i32 addrspace(1)* %out, i8 addrspace(1)* %in) {
-entry:
-  %0 = load i8, i8 addrspace(1)* %in
-  %1 = sext i8 %0 to i32
-  store i32 %1, i32 addrspace(1)* %out
-  ret void
-}
-
-; FUNC-LABEL: {{^}}load_v2i8:
-; R600: VTX_READ_8
-; R600: VTX_READ_8
-; SI: buffer_load_ubyte
-; SI: buffer_load_ubyte
-define void @load_v2i8(<2 x i32> addrspace(1)* %out, <2 x i8> addrspace(1)* %in) {
-entry:
-  %0 = load <2 x i8>, <2 x i8> addrspace(1)* %in
-  %1 = zext <2 x i8> %0 to <2 x i32>
-  store <2 x i32> %1, <2 x i32> addrspace(1)* %out
-  ret void
-}
-
-; FUNC-LABEL: {{^}}load_v2i8_sext:
-; R600-DAG: VTX_READ_8 [[DST_X:T[0-9]\.[XYZW]]], [[DST_X]]
-; R600-DAG: VTX_READ_8 [[DST_Y:T[0-9]\.[XYZW]]], [[DST_Y]]
-; R600-DAG: LSHL {{[* ]*}}T{{[0-9]}}.[[LSHL_X_CHAN:[XYZW]]], [[DST_X]]
-; R600-DAG: 24
-; R600-DAG: ASHR {{[* ]*}}T{{[0-9]\.[XYZW]}}, PV.[[LSHL_X_CHAN]]
-; R600-DAG: 24
-; R600-DAG: LSHL {{[* ]*}}T{{[0-9]}}.[[LSHL_Y_CHAN:[XYZW]]], [[DST_Y]]
-; R600-DAG: 24
-; R600-DAG: ASHR {{[* ]*}}T{{[0-9]\.[XYZW]}}, PV.[[LSHL_Y_CHAN]]
-; R600-DAG: 24
-; SI: buffer_load_sbyte
-; SI: buffer_load_sbyte
-define void @load_v2i8_sext(<2 x i32> addrspace(1)* %out, <2 x i8> addrspace(1)* %in) {
-entry:
-  %0 = load <2 x i8>, <2 x i8> addrspace(1)* %in
-  %1 = sext <2 x i8> %0 to <2 x i32>
-  store <2 x i32> %1, <2 x i32> addrspace(1)* %out
-  ret void
-}
-
-; FUNC-LABEL: {{^}}load_v4i8:
-; R600: VTX_READ_8
-; R600: VTX_READ_8
-; R600: VTX_READ_8
-; R600: VTX_READ_8
-; SI: buffer_load_ubyte
-; SI: buffer_load_ubyte
-; SI: buffer_load_ubyte
-; SI: buffer_load_ubyte
-define void @load_v4i8(<4 x i32> addrspace(1)* %out, <4 x i8> addrspace(1)* %in) {
-entry:
-  %0 = load <4 x i8>, <4 x i8> addrspace(1)* %in
-  %1 = zext <4 x i8> %0 to <4 x i32>
-  store <4 x i32> %1, <4 x i32> addrspace(1)* %out
-  ret void
-}
-
-; FUNC-LABEL: {{^}}load_v4i8_sext:
-; R600-DAG: VTX_READ_8 [[DST_X:T[0-9]\.[XYZW]]], [[DST_X]]
-; R600-DAG: VTX_READ_8 [[DST_Y:T[0-9]\.[XYZW]]], [[DST_Y]]
-; R600-DAG: VTX_READ_8 [[DST_Z:T[0-9]\.[XYZW]]], [[DST_Z]]
-; R600-DAG: VTX_READ_8 [[DST_W:T[0-9]\.[XYZW]]], [[DST_W]]
-; R600-DAG: LSHL {{[* ]*}}T{{[0-9]}}.[[LSHL_X_CHAN:[XYZW]]], [[DST_X]]
-; R600-DAG: 24
-; R600-DAG: ASHR {{[* ]*}}T{{[0-9]\.[XYZW]}}, PV.[[LSHL_X_CHAN]]
-; R600-DAG: 24
-; R600-DAG: LSHL {{[* ]*}}T{{[0-9]}}.[[LSHL_Y_CHAN:[XYZW]]], [[DST_Y]]
-; R600-DAG: 24
-; R600-DAG: ASHR {{[* ]*}}T{{[0-9]\.[XYZW]}}, PV.[[LSHL_Y_CHAN]]
-; R600-DAG: 24
-; R600-DAG: LSHL {{[* ]*}}T{{[0-9]}}.[[LSHL_Z_CHAN:[XYZW]]], [[DST_Z]]
-; R600-DAG: 24
-; R600-DAG: ASHR {{[* ]*}}T{{[0-9]\.[XYZW]}}, PV.[[LSHL_Z_CHAN]]
-; R600-DAG: 24
-; R600-DAG: LSHL {{[* ]*}}T{{[0-9]}}.[[LSHL_W_CHAN:[XYZW]]], [[DST_W]]
-; R600-DAG: 24
-; R600-DAG: ASHR {{[* ]*}}T{{[0-9]\.[XYZW]}}, PV.[[LSHL_W_CHAN]]
-; R600-DAG: 24
-; SI: buffer_load_sbyte
-; SI: buffer_load_sbyte
-; SI: buffer_load_sbyte
-; SI: buffer_load_sbyte
-define void @load_v4i8_sext(<4 x i32> addrspace(1)* %out, <4 x i8> addrspace(1)* %in) {
-entry:
-  %0 = load <4 x i8>, <4 x i8> addrspace(1)* %in
-  %1 = sext <4 x i8> %0 to <4 x i32>
-  store <4 x i32> %1, <4 x i32> addrspace(1)* %out
-  ret void
-}
-
-; Load an i16 value from the global address space.
-; FUNC-LABEL: {{^}}load_i16:
-; R600: VTX_READ_16 T{{[0-9]+\.X, T[0-9]+\.X}}
-; SI: buffer_load_ushort
-define void @load_i16(i32 addrspace(1)* %out, i16 addrspace(1)* %in) {
-entry:
-  %0 = load i16	, i16	 addrspace(1)* %in
-  %1 = zext i16 %0 to i32
-  store i32 %1, i32 addrspace(1)* %out
-  ret void
-}
-
-; FUNC-LABEL: {{^}}load_i16_sext:
-; R600: VTX_READ_16 [[DST:T[0-9]\.[XYZW]]], [[DST]]
-; R600: LSHL {{[* ]*}}T{{[0-9]}}.[[LSHL_CHAN:[XYZW]]], [[DST]]
-; R600: 16
-; R600: ASHR {{[* ]*}}T{{[0-9]\.[XYZW]}}, PV.[[LSHL_CHAN]]
-; R600: 16
-; SI: buffer_load_sshort
-define void @load_i16_sext(i32 addrspace(1)* %out, i16 addrspace(1)* %in) {
-entry:
-  %0 = load i16, i16 addrspace(1)* %in
-  %1 = sext i16 %0 to i32
-  store i32 %1, i32 addrspace(1)* %out
-  ret void
-}
-
-; FUNC-LABEL: {{^}}load_v2i16:
-; R600: VTX_READ_16
-; R600: VTX_READ_16
-; SI: buffer_load_ushort
-; SI: buffer_load_ushort
-define void @load_v2i16(<2 x i32> addrspace(1)* %out, <2 x i16> addrspace(1)* %in) {
-entry:
-  %0 = load <2 x i16>, <2 x i16> addrspace(1)* %in
-  %1 = zext <2 x i16> %0 to <2 x i32>
-  store <2 x i32> %1, <2 x i32> addrspace(1)* %out
-  ret void
-}
-
-; FUNC-LABEL: {{^}}load_v2i16_sext:
-; R600-DAG: VTX_READ_16 [[DST_X:T[0-9]\.[XYZW]]], [[DST_X]]
-; R600-DAG: VTX_READ_16 [[DST_Y:T[0-9]\.[XYZW]]], [[DST_Y]]
-; R600-DAG: LSHL {{[* ]*}}T{{[0-9]}}.[[LSHL_X_CHAN:[XYZW]]], [[DST_X]]
-; R600-DAG: 16
-; R600-DAG: ASHR {{[* ]*}}T{{[0-9]\.[XYZW]}}, PV.[[LSHL_X_CHAN]]
-; R600-DAG: 16
-; R600-DAG: LSHL {{[* ]*}}T{{[0-9]}}.[[LSHL_Y_CHAN:[XYZW]]], [[DST_Y]]
-; R600-DAG: 16
-; R600-DAG: ASHR {{[* ]*}}T{{[0-9]\.[XYZW]}}, PV.[[LSHL_Y_CHAN]]
-; R600-DAG: 16
-; SI: buffer_load_sshort
-; SI: buffer_load_sshort
-define void @load_v2i16_sext(<2 x i32> addrspace(1)* %out, <2 x i16> addrspace(1)* %in) {
-entry:
-  %0 = load <2 x i16>, <2 x i16> addrspace(1)* %in
-  %1 = sext <2 x i16> %0 to <2 x i32>
-  store <2 x i32> %1, <2 x i32> addrspace(1)* %out
-  ret void
-}
-
-; FUNC-LABEL: {{^}}load_v4i16:
-; R600: VTX_READ_16
-; R600: VTX_READ_16
-; R600: VTX_READ_16
-; R600: VTX_READ_16
-; SI: buffer_load_ushort
-; SI: buffer_load_ushort
-; SI: buffer_load_ushort
-; SI: buffer_load_ushort
-define void @load_v4i16(<4 x i32> addrspace(1)* %out, <4 x i16> addrspace(1)* %in) {
-entry:
-  %0 = load <4 x i16>, <4 x i16> addrspace(1)* %in
-  %1 = zext <4 x i16> %0 to <4 x i32>
-  store <4 x i32> %1, <4 x i32> addrspace(1)* %out
-  ret void
-}
-
-; FUNC-LABEL: {{^}}load_v4i16_sext:
-; R600-DAG: VTX_READ_16 [[DST_X:T[0-9]\.[XYZW]]], [[DST_X]]
-; R600-DAG: VTX_READ_16 [[DST_Y:T[0-9]\.[XYZW]]], [[DST_Y]]
-; R600-DAG: VTX_READ_16 [[DST_Z:T[0-9]\.[XYZW]]], [[DST_Z]]
-; R600-DAG: VTX_READ_16 [[DST_W:T[0-9]\.[XYZW]]], [[DST_W]]
-; R600-DAG: LSHL {{[* ]*}}T{{[0-9]}}.[[LSHL_X_CHAN:[XYZW]]], [[DST_X]]
-; R600-DAG: 16
-; R600-DAG: ASHR {{[* ]*}}T{{[0-9]\.[XYZW]}}, PV.[[LSHL_X_CHAN]]
-; R600-DAG: 16
-; R600-DAG: LSHL {{[* ]*}}T{{[0-9]}}.[[LSHL_Y_CHAN:[XYZW]]], [[DST_Y]]
-; R600-DAG: 16
-; R600-DAG: ASHR {{[* ]*}}T{{[0-9]\.[XYZW]}}, PV.[[LSHL_Y_CHAN]]
-; R600-DAG: 16
-; R600-DAG: LSHL {{[* ]*}}T{{[0-9]}}.[[LSHL_Z_CHAN:[XYZW]]], [[DST_Z]]
-; R600-DAG: 16
-; R600-DAG: ASHR {{[* ]*}}T{{[0-9]\.[XYZW]}}, PV.[[LSHL_Z_CHAN]]
-; R600-DAG: 16
-; R600-DAG: LSHL {{[* ]*}}T{{[0-9]}}.[[LSHL_W_CHAN:[XYZW]]], [[DST_W]]
-; R600-DAG: 16
-; R600-DAG: ASHR {{[* ]*}}T{{[0-9]\.[XYZW]}}, PV.[[LSHL_W_CHAN]]
-; R600-DAG: 16
-; SI: buffer_load_sshort
-; SI: buffer_load_sshort
-; SI: buffer_load_sshort
-; SI: buffer_load_sshort
-define void @load_v4i16_sext(<4 x i32> addrspace(1)* %out, <4 x i16> addrspace(1)* %in) {
-entry:
-  %0 = load <4 x i16>, <4 x i16> addrspace(1)* %in
-  %1 = sext <4 x i16> %0 to <4 x i32>
-  store <4 x i32> %1, <4 x i32> addrspace(1)* %out
-  ret void
-}
-
-; load an i32 value from the global address space.
-; FUNC-LABEL: {{^}}load_i32:
-; R600: VTX_READ_32 T{{[0-9]+}}.X, T{{[0-9]+}}.X, 0
-
-; SI: buffer_load_dword v{{[0-9]+}}
-define void @load_i32(i32 addrspace(1)* %out, i32 addrspace(1)* %in) {
-entry:
-  %0 = load i32, i32 addrspace(1)* %in
-  store i32 %0, i32 addrspace(1)* %out
-  ret void
-}
-
-; load a f32 value from the global address space.
-; FUNC-LABEL: {{^}}load_f32:
-; R600: VTX_READ_32 T{{[0-9]+}}.X, T{{[0-9]+}}.X, 0
-
-; SI: buffer_load_dword v{{[0-9]+}}
-define void @load_f32(float addrspace(1)* %out, float addrspace(1)* %in) {
-entry:
-  %0 = load float, float addrspace(1)* %in
-  store float %0, float addrspace(1)* %out
-  ret void
-}
-
-; load a v2f32 value from the global address space
-; FUNC-LABEL: {{^}}load_v2f32:
-; R600: MEM_RAT
-; R600: VTX_READ_64
-; SI: buffer_load_dwordx2
-define void @load_v2f32(<2 x float> addrspace(1)* %out, <2 x float> addrspace(1)* %in) {
-entry:
-  %0 = load <2 x float>, <2 x float> addrspace(1)* %in
-  store <2 x float> %0, <2 x float> addrspace(1)* %out
-  ret void
-}
-
-; FUNC-LABEL: {{^}}load_i64:
-; R600: VTX_READ_64
-; SI: buffer_load_dwordx2
-define void @load_i64(i64 addrspace(1)* %out, i64 addrspace(1)* %in) {
-entry:
-  %0 = load i64, i64 addrspace(1)* %in
-  store i64 %0, i64 addrspace(1)* %out
-  ret void
-}
-
-; FUNC-LABEL: {{^}}load_i64_sext:
-; R600: MEM_RAT
-; R600: MEM_RAT
-; R600: ASHR {{[* ]*}}T{{[0-9]\.[XYZW]}}, T{{[0-9]\.[XYZW]}},  literal.x
-; R600: 31
-; SI: buffer_load_dword
-
-define void @load_i64_sext(i64 addrspace(1)* %out, i32 addrspace(1)* %in) {
-entry:
-  %0 = load i32, i32 addrspace(1)* %in
-  %1 = sext i32 %0 to i64
-  store i64 %1, i64 addrspace(1)* %out
-  ret void
-}
-
-; FUNC-LABEL: {{^}}load_i64_zext:
-; R600: MEM_RAT
-; R600: MEM_RAT
-define void @load_i64_zext(i64 addrspace(1)* %out, i32 addrspace(1)* %in) {
-entry:
-  %0 = load i32, i32 addrspace(1)* %in
-  %1 = zext i32 %0 to i64
-  store i64 %1, i64 addrspace(1)* %out
-  ret void
-}
-
-; FUNC-LABEL: {{^}}load_v8i32:
-; R600: VTX_READ_128
-; R600: VTX_READ_128
-; XXX: We should be using DWORDX4 instructions on SI.
-; SI: buffer_load_dword
-; SI: buffer_load_dword
-; SI: buffer_load_dword
-; SI: buffer_load_dword
-; SI: buffer_load_dword
-; SI: buffer_load_dword
-; SI: buffer_load_dword
-; SI: buffer_load_dword
-define void @load_v8i32(<8 x i32> addrspace(1)* %out, <8 x i32> addrspace(1)* %in) {
-entry:
-  %0 = load <8 x i32>, <8 x i32> addrspace(1)* %in
-  store <8 x i32> %0, <8 x i32> addrspace(1)* %out
-  ret void
-}
-
-; FUNC-LABEL: {{^}}load_v16i32:
-; R600: VTX_READ_128
-; R600: VTX_READ_128
-; R600: VTX_READ_128
-; R600: VTX_READ_128
-; XXX: We should be using DWORDX4 instructions on SI.
-; SI: buffer_load_dword
-; SI: buffer_load_dword
-; SI: buffer_load_dword
-; SI: buffer_load_dword
-; SI: buffer_load_dword
-; SI: buffer_load_dword
-; SI: buffer_load_dword
-; SI: buffer_load_dword
-; SI: buffer_load_dword
-; SI: buffer_load_dword
-; SI: buffer_load_dword
-; SI: buffer_load_dword
-; SI: buffer_load_dword
-; SI: buffer_load_dword
-; SI: buffer_load_dword
-; SI: buffer_load_dword
-define void @load_v16i32(<16 x i32> addrspace(1)* %out, <16 x i32> addrspace(1)* %in) {
-entry:
-  %0 = load <16 x i32>, <16 x i32> addrspace(1)* %in
-  store <16 x i32> %0, <16 x i32> addrspace(1)* %out
-  ret void
-}
-
-;===------------------------------------------------------------------------===;
-; CONSTANT ADDRESS SPACE
-;===------------------------------------------------------------------------===;
-
-; Load a sign-extended i8 value
-; FUNC-LABEL: {{^}}load_const_i8_sext:
-; R600: VTX_READ_8 [[DST:T[0-9]\.[XYZW]]], [[DST]]
-; R600: LSHL {{[* ]*}}T{{[0-9]}}.[[LSHL_CHAN:[XYZW]]], [[DST]]
-; R600: 24
-; R600: ASHR {{[* ]*}}T{{[0-9]\.[XYZW]}}, PV.[[LSHL_CHAN]]
-; R600: 24
-; SI: buffer_load_sbyte v{{[0-9]+}},
-define void @load_const_i8_sext(i32 addrspace(1)* %out, i8 addrspace(2)* %in) {
-entry:
-  %0 = load i8, i8 addrspace(2)* %in
-  %1 = sext i8 %0 to i32
-  store i32 %1, i32 addrspace(1)* %out
-  ret void
-}
-
-; Load an aligned i8 value
-; FUNC-LABEL: {{^}}load_const_i8_aligned:
-; R600: VTX_READ_8 T{{[0-9]+\.X, T[0-9]+\.X}}
-; SI: buffer_load_ubyte v{{[0-9]+}},
-define void @load_const_i8_aligned(i32 addrspace(1)* %out, i8 addrspace(2)* %in) {
-entry:
-  %0 = load i8, i8 addrspace(2)* %in
-  %1 = zext i8 %0 to i32
-  store i32 %1, i32 addrspace(1)* %out
-  ret void
-}
-
-; Load an un-aligned i8 value
-; FUNC-LABEL: {{^}}load_const_i8_unaligned:
-; R600: VTX_READ_8 T{{[0-9]+\.X, T[0-9]+\.X}}
-; SI: buffer_load_ubyte v{{[0-9]+}},
-define void @load_const_i8_unaligned(i32 addrspace(1)* %out, i8 addrspace(2)* %in) {
-entry:
-  %0 = getelementptr i8, i8 addrspace(2)* %in, i32 1
-  %1 = load i8, i8 addrspace(2)* %0
-  %2 = zext i8 %1 to i32
-  store i32 %2, i32 addrspace(1)* %out
-  ret void
-}
-
-; Load a sign-extended i16 value
-; FUNC-LABEL: {{^}}load_const_i16_sext:
-; R600: VTX_READ_16 [[DST:T[0-9]\.[XYZW]]], [[DST]]
-; R600: LSHL {{[* ]*}}T{{[0-9]}}.[[LSHL_CHAN:[XYZW]]], [[DST]]
-; R600: 16
-; R600: ASHR {{[* ]*}}T{{[0-9]\.[XYZW]}}, PV.[[LSHL_CHAN]]
-; R600: 16
-; SI: buffer_load_sshort
-define void @load_const_i16_sext(i32 addrspace(1)* %out, i16 addrspace(2)* %in) {
-entry:
-  %0 = load i16, i16 addrspace(2)* %in
-  %1 = sext i16 %0 to i32
-  store i32 %1, i32 addrspace(1)* %out
-  ret void
-}
-
-; Load an aligned i16 value
-; FUNC-LABEL: {{^}}load_const_i16_aligned:
-; R600: VTX_READ_16 T{{[0-9]+\.X, T[0-9]+\.X}}
-; SI: buffer_load_ushort
-define void @load_const_i16_aligned(i32 addrspace(1)* %out, i16 addrspace(2)* %in) {
-entry:
-  %0 = load i16, i16 addrspace(2)* %in
-  %1 = zext i16 %0 to i32
-  store i32 %1, i32 addrspace(1)* %out
-  ret void
-}
-
-; Load an un-aligned i16 value
-; FUNC-LABEL: {{^}}load_const_i16_unaligned:
-; R600: VTX_READ_16 T{{[0-9]+\.X, T[0-9]+\.X}}
-; SI: buffer_load_ushort
-define void @load_const_i16_unaligned(i32 addrspace(1)* %out, i16 addrspace(2)* %in) {
-entry:
-  %0 = getelementptr i16, i16 addrspace(2)* %in, i32 1
-  %1 = load i16, i16 addrspace(2)* %0
-  %2 = zext i16 %1 to i32
-  store i32 %2, i32 addrspace(1)* %out
-  ret void
-}
-
-; Load an i32 value from the constant address space.
-; FUNC-LABEL: {{^}}load_const_addrspace_i32:
-; R600: VTX_READ_32 T{{[0-9]+}}.X, T{{[0-9]+}}.X, 0
-
-; SI: s_load_dword s{{[0-9]+}}
-define void @load_const_addrspace_i32(i32 addrspace(1)* %out, i32 addrspace(2)* %in) {
-entry:
-  %0 = load i32, i32 addrspace(2)* %in
-  store i32 %0, i32 addrspace(1)* %out
-  ret void
-}
-
-; Load a f32 value from the constant address space.
-; FUNC-LABEL: {{^}}load_const_addrspace_f32:
-; R600: VTX_READ_32 T{{[0-9]+}}.X, T{{[0-9]+}}.X, 0
-
-; SI: s_load_dword s{{[0-9]+}}
-define void @load_const_addrspace_f32(float addrspace(1)* %out, float addrspace(2)* %in) {
-  %1 = load float, float addrspace(2)* %in
-  store float %1, float addrspace(1)* %out
-  ret void
-}
-
-;===------------------------------------------------------------------------===;
-; LOCAL ADDRESS SPACE
-;===------------------------------------------------------------------------===;
-
-; Load an i8 value from the local address space.
-; FUNC-LABEL: {{^}}load_i8_local:
-; R600: LDS_UBYTE_READ_RET
-; SI-NOT: s_wqm_b64
-; SI: s_mov_b32 m0
-; SI: ds_read_u8
-define void @load_i8_local(i32 addrspace(1)* %out, i8 addrspace(3)* %in) {
-  %1 = load i8, i8 addrspace(3)* %in
-  %2 = zext i8 %1 to i32
-  store i32 %2, i32 addrspace(1)* %out
-  ret void
-}
-
-; FUNC-LABEL: {{^}}load_i8_sext_local:
-; R600: LDS_UBYTE_READ_RET
-; R600: ASHR
-; SI-NOT: s_wqm_b64
-; SI: s_mov_b32 m0
-; SI: ds_read_i8
-define void @load_i8_sext_local(i32 addrspace(1)* %out, i8 addrspace(3)* %in) {
-entry:
-  %0 = load i8, i8 addrspace(3)* %in
-  %1 = sext i8 %0 to i32
-  store i32 %1, i32 addrspace(1)* %out
-  ret void
-}
-
-; FUNC-LABEL: {{^}}load_v2i8_local:
-; R600: LDS_UBYTE_READ_RET
-; R600: LDS_UBYTE_READ_RET
-; SI-NOT: s_wqm_b64
-; SI: s_mov_b32 m0
-; SI: ds_read_u8
-; SI: ds_read_u8
-define void @load_v2i8_local(<2 x i32> addrspace(1)* %out, <2 x i8> addrspace(3)* %in) {
-entry:
-  %0 = load <2 x i8>, <2 x i8> addrspace(3)* %in
-  %1 = zext <2 x i8> %0 to <2 x i32>
-  store <2 x i32> %1, <2 x i32> addrspace(1)* %out
-  ret void
-}
-
-; FUNC-LABEL: {{^}}load_v2i8_sext_local:
-; R600-DAG: LDS_UBYTE_READ_RET
-; R600-DAG: LDS_UBYTE_READ_RET
-; R600-DAG: ASHR
-; R600-DAG: ASHR
-; SI-NOT: s_wqm_b64
-; SI: s_mov_b32 m0
-; SI: ds_read_i8
-; SI: ds_read_i8
-define void @load_v2i8_sext_local(<2 x i32> addrspace(1)* %out, <2 x i8> addrspace(3)* %in) {
-entry:
-  %0 = load <2 x i8>, <2 x i8> addrspace(3)* %in
-  %1 = sext <2 x i8> %0 to <2 x i32>
-  store <2 x i32> %1, <2 x i32> addrspace(1)* %out
-  ret void
-}
-
-; FUNC-LABEL: {{^}}load_v4i8_local:
-; R600: LDS_UBYTE_READ_RET
-; R600: LDS_UBYTE_READ_RET
-; R600: LDS_UBYTE_READ_RET
-; R600: LDS_UBYTE_READ_RET
-; SI-NOT: s_wqm_b64
-; SI: s_mov_b32 m0
-; SI: ds_read_u8
-; SI: ds_read_u8
-; SI: ds_read_u8
-; SI: ds_read_u8
-define void @load_v4i8_local(<4 x i32> addrspace(1)* %out, <4 x i8> addrspace(3)* %in) {
-entry:
-  %0 = load <4 x i8>, <4 x i8> addrspace(3)* %in
-  %1 = zext <4 x i8> %0 to <4 x i32>
-  store <4 x i32> %1, <4 x i32> addrspace(1)* %out
-  ret void
-}
-
-; FUNC-LABEL: {{^}}load_v4i8_sext_local:
-; R600-DAG: LDS_UBYTE_READ_RET
-; R600-DAG: LDS_UBYTE_READ_RET
-; R600-DAG: LDS_UBYTE_READ_RET
-; R600-DAG: LDS_UBYTE_READ_RET
-; R600-DAG: ASHR
-; R600-DAG: ASHR
-; R600-DAG: ASHR
-; R600-DAG: ASHR
-; SI-NOT: s_wqm_b64
-; SI: s_mov_b32 m0
-; SI: ds_read_i8
-; SI: ds_read_i8
-; SI: ds_read_i8
-; SI: ds_read_i8
-define void @load_v4i8_sext_local(<4 x i32> addrspace(1)* %out, <4 x i8> addrspace(3)* %in) {
-entry:
-  %0 = load <4 x i8>, <4 x i8> addrspace(3)* %in
-  %1 = sext <4 x i8> %0 to <4 x i32>
-  store <4 x i32> %1, <4 x i32> addrspace(1)* %out
-  ret void
-}
-
-; Load an i16 value from the local address space.
-; FUNC-LABEL: {{^}}load_i16_local:
-; R600: LDS_USHORT_READ_RET
-; SI-NOT: s_wqm_b64
-; SI: s_mov_b32 m0
-; SI: ds_read_u16
-define void @load_i16_local(i32 addrspace(1)* %out, i16 addrspace(3)* %in) {
-entry:
-  %0 = load i16	, i16	 addrspace(3)* %in
-  %1 = zext i16 %0 to i32
-  store i32 %1, i32 addrspace(1)* %out
-  ret void
-}
-
-; FUNC-LABEL: {{^}}load_i16_sext_local:
-; R600: LDS_USHORT_READ_RET
-; R600: ASHR
-; SI-NOT: s_wqm_b64
-; SI: s_mov_b32 m0
-; SI: ds_read_i16
-define void @load_i16_sext_local(i32 addrspace(1)* %out, i16 addrspace(3)* %in) {
-entry:
-  %0 = load i16, i16 addrspace(3)* %in
-  %1 = sext i16 %0 to i32
-  store i32 %1, i32 addrspace(1)* %out
-  ret void
-}
-
-; FUNC-LABEL: {{^}}load_v2i16_local:
-; R600: LDS_USHORT_READ_RET
-; R600: LDS_USHORT_READ_RET
-; SI-NOT: s_wqm_b64
-; SI: s_mov_b32 m0
-; SI: ds_read_u16
-; SI: ds_read_u16
-define void @load_v2i16_local(<2 x i32> addrspace(1)* %out, <2 x i16> addrspace(3)* %in) {
-entry:
-  %0 = load <2 x i16>, <2 x i16> addrspace(3)* %in
-  %1 = zext <2 x i16> %0 to <2 x i32>
-  store <2 x i32> %1, <2 x i32> addrspace(1)* %out
-  ret void
-}
-
-; FUNC-LABEL: {{^}}load_v2i16_sext_local:
-; R600-DAG: LDS_USHORT_READ_RET
-; R600-DAG: LDS_USHORT_READ_RET
-; R600-DAG: ASHR
-; R600-DAG: ASHR
-; SI-NOT: s_wqm_b64
-; SI: s_mov_b32 m0
-; SI: ds_read_i16
-; SI: ds_read_i16
-define void @load_v2i16_sext_local(<2 x i32> addrspace(1)* %out, <2 x i16> addrspace(3)* %in) {
-entry:
-  %0 = load <2 x i16>, <2 x i16> addrspace(3)* %in
-  %1 = sext <2 x i16> %0 to <2 x i32>
-  store <2 x i32> %1, <2 x i32> addrspace(1)* %out
-  ret void
-}
-
-; FUNC-LABEL: {{^}}load_v4i16_local:
-; R600: LDS_USHORT_READ_RET
-; R600: LDS_USHORT_READ_RET
-; R600: LDS_USHORT_READ_RET
-; R600: LDS_USHORT_READ_RET
-; SI-NOT: s_wqm_b64
-; SI: s_mov_b32 m0
-; SI: ds_read_u16
-; SI: ds_read_u16
-; SI: ds_read_u16
-; SI: ds_read_u16
-define void @load_v4i16_local(<4 x i32> addrspace(1)* %out, <4 x i16> addrspace(3)* %in) {
-entry:
-  %0 = load <4 x i16>, <4 x i16> addrspace(3)* %in
-  %1 = zext <4 x i16> %0 to <4 x i32>
-  store <4 x i32> %1, <4 x i32> addrspace(1)* %out
-  ret void
-}
-
-; FUNC-LABEL: {{^}}load_v4i16_sext_local:
-; R600-DAG: LDS_USHORT_READ_RET
-; R600-DAG: LDS_USHORT_READ_RET
-; R600-DAG: LDS_USHORT_READ_RET
-; R600-DAG: LDS_USHORT_READ_RET
-; R600-DAG: ASHR
-; R600-DAG: ASHR
-; R600-DAG: ASHR
-; R600-DAG: ASHR
-; SI-NOT: s_wqm_b64
-; SI: s_mov_b32 m0
-; SI: ds_read_i16
-; SI: ds_read_i16
-; SI: ds_read_i16
-; SI: ds_read_i16
-define void @load_v4i16_sext_local(<4 x i32> addrspace(1)* %out, <4 x i16> addrspace(3)* %in) {
-entry:
-  %0 = load <4 x i16>, <4 x i16> addrspace(3)* %in
-  %1 = sext <4 x i16> %0 to <4 x i32>
-  store <4 x i32> %1, <4 x i32> addrspace(1)* %out
-  ret void
-}
-
-; load an i32 value from the local address space.
-; FUNC-LABEL: {{^}}load_i32_local:
-; R600: LDS_READ_RET
-; SI-NOT: s_wqm_b64
-; SI: s_mov_b32 m0
-; SI: ds_read_b32
-define void @load_i32_local(i32 addrspace(1)* %out, i32 addrspace(3)* %in) {
-entry:
-  %0 = load i32, i32 addrspace(3)* %in
-  store i32 %0, i32 addrspace(1)* %out
-  ret void
-}
-
-; load a f32 value from the local address space.
-; FUNC-LABEL: {{^}}load_f32_local:
-; R600: LDS_READ_RET
-; SI: s_mov_b32 m0
-; SI: ds_read_b32
-define void @load_f32_local(float addrspace(1)* %out, float addrspace(3)* %in) {
-entry:
-  %0 = load float, float addrspace(3)* %in
-  store float %0, float addrspace(1)* %out
-  ret void
-}
-
-; load a v2f32 value from the local address space
-; FUNC-LABEL: {{^}}load_v2f32_local:
-; R600: LDS_READ_RET
-; R600: LDS_READ_RET
-; SI: s_mov_b32 m0
-; SI: ds_read_b64
-define void @load_v2f32_local(<2 x float> addrspace(1)* %out, <2 x float> addrspace(3)* %in) {
-entry:
-  %0 = load <2 x float>, <2 x float> addrspace(3)* %in
-  store <2 x float> %0, <2 x float> addrspace(1)* %out
-  ret void
-}
-
-; Test loading a i32 and v2i32 value from the same base pointer.
-; FUNC-LABEL: {{^}}load_i32_v2i32_local:
-; R600: LDS_READ_RET
-; R600: LDS_READ_RET
-; R600: LDS_READ_RET
-; SI-DAG: ds_read_b32
-; SI-DAG: ds_read2_b32
-define void @load_i32_v2i32_local(<2 x i32> addrspace(1)* %out, i32 addrspace(3)* %in) {
-  %scalar = load i32, i32 addrspace(3)* %in
-  %tmp0 = bitcast i32 addrspace(3)* %in to <2 x i32> addrspace(3)*
-  %vec_ptr = getelementptr <2 x i32>, <2 x i32> addrspace(3)* %tmp0, i32 2
-  %vec0 = load <2 x i32>, <2 x i32> addrspace(3)* %vec_ptr, align 4
-  %vec1 = insertelement <2 x i32> <i32 0, i32 0>, i32 %scalar, i32 0
-  %vec = add <2 x i32> %vec0, %vec1
-  store <2 x i32> %vec, <2 x i32> addrspace(1)* %out
-  ret void
-}
-
-
-@lds = addrspace(3) global [512 x i32] undef, align 4
-
-; On SI we need to make sure that the base offset is a register and not
-; an immediate.
-; FUNC-LABEL: {{^}}load_i32_local_const_ptr:
-; SI: v_mov_b32_e32 v[[ZERO:[0-9]+]], 0
-; SI: ds_read_b32 v0, v[[ZERO]] offset:4
-; R600: LDS_READ_RET
-define void @load_i32_local_const_ptr(i32 addrspace(1)* %out, i32 addrspace(3)* %in) {
-entry:
-  %tmp0 = getelementptr [512 x i32], [512 x i32] addrspace(3)* @lds, i32 0, i32 1
-  %tmp1 = load i32, i32 addrspace(3)* %tmp0
-  %tmp2 = getelementptr i32, i32 addrspace(1)* %out, i32 1
-  store i32 %tmp1, i32 addrspace(1)* %tmp2
-  ret void
-}
diff --git a/test/CodeGen/R600/max.ll b/test/CodeGen/R600/max.ll
deleted file mode 100644
index 1aa9e688301..00000000000
--- a/test/CodeGen/R600/max.ll
+++ /dev/null
@@ -1,117 +0,0 @@
-; RUN: llc -march=amdgcn -mcpu=SI < %s | FileCheck -check-prefix=SI -check-prefix=FUNC %s
-
-declare i32 @llvm.r600.read.tidig.x() nounwind readnone
-
-; FUNC-LABEL: @v_test_imax_sge_i32
-; SI: v_max_i32_e32
-define void @v_test_imax_sge_i32(i32 addrspace(1)* %out, i32 addrspace(1)* %aptr, i32 addrspace(1)* %bptr) nounwind {
-  %tid = call i32 @llvm.r600.read.tidig.x() nounwind readnone
-  %gep0 = getelementptr i32, i32 addrspace(1)* %aptr, i32 %tid
-  %gep1 = getelementptr i32, i32 addrspace(1)* %bptr, i32 %tid
-  %outgep = getelementptr i32, i32 addrspace(1)* %out, i32 %tid
-  %a = load i32, i32 addrspace(1)* %gep0, align 4
-  %b = load i32, i32 addrspace(1)* %gep1, align 4
-  %cmp = icmp sge i32 %a, %b
-  %val = select i1 %cmp, i32 %a, i32 %b
-  store i32 %val, i32 addrspace(1)* %outgep, align 4
-  ret void
-}
-
-; FUNC-LABEL: @s_test_imax_sge_i32
-; SI: s_max_i32
-define void @s_test_imax_sge_i32(i32 addrspace(1)* %out, i32 %a, i32 %b) nounwind {
-  %cmp = icmp sge i32 %a, %b
-  %val = select i1 %cmp, i32 %a, i32 %b
-  store i32 %val, i32 addrspace(1)* %out, align 4
-  ret void
-}
-
-; FUNC-LABEL: {{^}}s_test_imax_sge_imm_i32:
-; SI: s_max_i32 {{s[0-9]+}}, {{s[0-9]+}}, 9
-define void @s_test_imax_sge_imm_i32(i32 addrspace(1)* %out, i32 %a) nounwind {
-  %cmp = icmp sge i32 %a, 9
-  %val = select i1 %cmp, i32 %a, i32 9
-  store i32 %val, i32 addrspace(1)* %out, align 4
-  ret void
-}
-
-; FUNC-LABEL: {{^}}s_test_imax_sgt_imm_i32:
-; SI: s_max_i32 {{s[0-9]+}}, {{s[0-9]+}}, 9
-define void @s_test_imax_sgt_imm_i32(i32 addrspace(1)* %out, i32 %a) nounwind {
-  %cmp = icmp sgt i32 %a, 9
-  %val = select i1 %cmp, i32 %a, i32 9
-  store i32 %val, i32 addrspace(1)* %out, align 4
-  ret void
-}
-
-; FUNC-LABEL: @v_test_imax_sgt_i32
-; SI: v_max_i32_e32
-define void @v_test_imax_sgt_i32(i32 addrspace(1)* %out, i32 addrspace(1)* %aptr, i32 addrspace(1)* %bptr) nounwind {
-  %tid = call i32 @llvm.r600.read.tidig.x() nounwind readnone
-  %gep0 = getelementptr i32, i32 addrspace(1)* %aptr, i32 %tid
-  %gep1 = getelementptr i32, i32 addrspace(1)* %bptr, i32 %tid
-  %outgep = getelementptr i32, i32 addrspace(1)* %out, i32 %tid
-  %a = load i32, i32 addrspace(1)* %gep0, align 4
-  %b = load i32, i32 addrspace(1)* %gep1, align 4
-  %cmp = icmp sgt i32 %a, %b
-  %val = select i1 %cmp, i32 %a, i32 %b
-  store i32 %val, i32 addrspace(1)* %outgep, align 4
-  ret void
-}
-
-; FUNC-LABEL: @s_test_imax_sgt_i32
-; SI: s_max_i32
-define void @s_test_imax_sgt_i32(i32 addrspace(1)* %out, i32 %a, i32 %b) nounwind {
-  %cmp = icmp sgt i32 %a, %b
-  %val = select i1 %cmp, i32 %a, i32 %b
-  store i32 %val, i32 addrspace(1)* %out, align 4
-  ret void
-}
-
-; FUNC-LABEL: @v_test_umax_uge_i32
-; SI: v_max_u32_e32
-define void @v_test_umax_uge_i32(i32 addrspace(1)* %out, i32 addrspace(1)* %aptr, i32 addrspace(1)* %bptr) nounwind {
-  %tid = call i32 @llvm.r600.read.tidig.x() nounwind readnone
-  %gep0 = getelementptr i32, i32 addrspace(1)* %aptr, i32 %tid
-  %gep1 = getelementptr i32, i32 addrspace(1)* %bptr, i32 %tid
-  %outgep = getelementptr i32, i32 addrspace(1)* %out, i32 %tid
-  %a = load i32, i32 addrspace(1)* %gep0, align 4
-  %b = load i32, i32 addrspace(1)* %gep1, align 4
-  %cmp = icmp uge i32 %a, %b
-  %val = select i1 %cmp, i32 %a, i32 %b
-  store i32 %val, i32 addrspace(1)* %outgep, align 4
-  ret void
-}
-
-; FUNC-LABEL: @s_test_umax_uge_i32
-; SI: s_max_u32
-define void @s_test_umax_uge_i32(i32 addrspace(1)* %out, i32 %a, i32 %b) nounwind {
-  %cmp = icmp uge i32 %a, %b
-  %val = select i1 %cmp, i32 %a, i32 %b
-  store i32 %val, i32 addrspace(1)* %out, align 4
-  ret void
-}
-
-; FUNC-LABEL: @v_test_umax_ugt_i32
-; SI: v_max_u32_e32
-define void @v_test_umax_ugt_i32(i32 addrspace(1)* %out, i32 addrspace(1)* %aptr, i32 addrspace(1)* %bptr) nounwind {
-  %tid = call i32 @llvm.r600.read.tidig.x() nounwind readnone
-  %gep0 = getelementptr i32, i32 addrspace(1)* %aptr, i32 %tid
-  %gep1 = getelementptr i32, i32 addrspace(1)* %bptr, i32 %tid
-  %outgep = getelementptr i32, i32 addrspace(1)* %out, i32 %tid
-  %a = load i32, i32 addrspace(1)* %gep0, align 4
-  %b = load i32, i32 addrspace(1)* %gep1, align 4
-  %cmp = icmp ugt i32 %a, %b
-  %val = select i1 %cmp, i32 %a, i32 %b
-  store i32 %val, i32 addrspace(1)* %outgep, align 4
-  ret void
-}
-
-; FUNC-LABEL: @s_test_umax_ugt_i32
-; SI: s_max_u32
-define void @s_test_umax_ugt_i32(i32 addrspace(1)* %out, i32 %a, i32 %b) nounwind {
-  %cmp = icmp ugt i32 %a, %b
-  %val = select i1 %cmp, i32 %a, i32 %b
-  store i32 %val, i32 addrspace(1)* %out, align 4
-  ret void
-}
diff --git a/test/CodeGen/R600/min.ll b/test/CodeGen/R600/min.ll
deleted file mode 100644
index 275e9a7d899..00000000000
--- a/test/CodeGen/R600/min.ll
+++ /dev/null
@@ -1,138 +0,0 @@
-; RUN: llc -march=amdgcn -mcpu=SI < %s | FileCheck -check-prefix=SI -check-prefix=FUNC %s
-
-declare i32 @llvm.r600.read.tidig.x() nounwind readnone
-
-; FUNC-LABEL: @v_test_imin_sle_i32
-; SI: v_min_i32_e32
-define void @v_test_imin_sle_i32(i32 addrspace(1)* %out, i32 addrspace(1)* %aptr, i32 addrspace(1)* %bptr) nounwind {
-  %tid = call i32 @llvm.r600.read.tidig.x() nounwind readnone
-  %gep0 = getelementptr i32, i32 addrspace(1)* %aptr, i32 %tid
-  %gep1 = getelementptr i32, i32 addrspace(1)* %bptr, i32 %tid
-  %outgep = getelementptr i32, i32 addrspace(1)* %out, i32 %tid
-  %a = load i32, i32 addrspace(1)* %gep0, align 4
-  %b = load i32, i32 addrspace(1)* %gep1, align 4
-  %cmp = icmp sle i32 %a, %b
-  %val = select i1 %cmp, i32 %a, i32 %b
-  store i32 %val, i32 addrspace(1)* %outgep, align 4
-  ret void
-}
-
-; FUNC-LABEL: @s_test_imin_sle_i32
-; SI: s_min_i32
-define void @s_test_imin_sle_i32(i32 addrspace(1)* %out, i32 %a, i32 %b) nounwind {
-  %cmp = icmp sle i32 %a, %b
-  %val = select i1 %cmp, i32 %a, i32 %b
-  store i32 %val, i32 addrspace(1)* %out, align 4
-  ret void
-}
-
-; FUNC-LABEL: @v_test_imin_slt_i32
-; SI: v_min_i32_e32
-define void @v_test_imin_slt_i32(i32 addrspace(1)* %out, i32 addrspace(1)* %aptr, i32 addrspace(1)* %bptr) nounwind {
-  %tid = call i32 @llvm.r600.read.tidig.x() nounwind readnone
-  %gep0 = getelementptr i32, i32 addrspace(1)* %aptr, i32 %tid
-  %gep1 = getelementptr i32, i32 addrspace(1)* %bptr, i32 %tid
-  %outgep = getelementptr i32, i32 addrspace(1)* %out, i32 %tid
-  %a = load i32, i32 addrspace(1)* %gep0, align 4
-  %b = load i32, i32 addrspace(1)* %gep1, align 4
-  %cmp = icmp slt i32 %a, %b
-  %val = select i1 %cmp, i32 %a, i32 %b
-  store i32 %val, i32 addrspace(1)* %outgep, align 4
-  ret void
-}
-
-; FUNC-LABEL: @s_test_imin_slt_i32
-; SI: s_min_i32
-define void @s_test_imin_slt_i32(i32 addrspace(1)* %out, i32 %a, i32 %b) nounwind {
-  %cmp = icmp slt i32 %a, %b
-  %val = select i1 %cmp, i32 %a, i32 %b
-  store i32 %val, i32 addrspace(1)* %out, align 4
-  ret void
-}
-
-; FUNC-LABEL: {{^}}s_test_imin_slt_imm_i32:
-; SI: s_min_i32 {{s[0-9]+}}, {{s[0-9]+}}, 8
-define void @s_test_imin_slt_imm_i32(i32 addrspace(1)* %out, i32 %a) nounwind {
-  %cmp = icmp slt i32 %a, 8
-  %val = select i1 %cmp, i32 %a, i32 8
-  store i32 %val, i32 addrspace(1)* %out, align 4
-  ret void
-}
-
-; FUNC-LABEL: {{^}}s_test_imin_sle_imm_i32:
-; SI: s_min_i32 {{s[0-9]+}}, {{s[0-9]+}}, 8
-define void @s_test_imin_sle_imm_i32(i32 addrspace(1)* %out, i32 %a) nounwind {
-  %cmp = icmp sle i32 %a, 8
-  %val = select i1 %cmp, i32 %a, i32 8
-  store i32 %val, i32 addrspace(1)* %out, align 4
-  ret void
-}
-
-; FUNC-LABEL: @v_test_umin_ule_i32
-; SI: v_min_u32_e32
-define void @v_test_umin_ule_i32(i32 addrspace(1)* %out, i32 addrspace(1)* %aptr, i32 addrspace(1)* %bptr) nounwind {
-  %tid = call i32 @llvm.r600.read.tidig.x() nounwind readnone
-  %gep0 = getelementptr i32, i32 addrspace(1)* %aptr, i32 %tid
-  %gep1 = getelementptr i32, i32 addrspace(1)* %bptr, i32 %tid
-  %outgep = getelementptr i32, i32 addrspace(1)* %out, i32 %tid
-  %a = load i32, i32 addrspace(1)* %gep0, align 4
-  %b = load i32, i32 addrspace(1)* %gep1, align 4
-  %cmp = icmp ule i32 %a, %b
-  %val = select i1 %cmp, i32 %a, i32 %b
-  store i32 %val, i32 addrspace(1)* %outgep, align 4
-  ret void
-}
-
-; FUNC-LABEL: @s_test_umin_ule_i32
-; SI: s_min_u32
-define void @s_test_umin_ule_i32(i32 addrspace(1)* %out, i32 %a, i32 %b) nounwind {
-  %cmp = icmp ule i32 %a, %b
-  %val = select i1 %cmp, i32 %a, i32 %b
-  store i32 %val, i32 addrspace(1)* %out, align 4
-  ret void
-}
-
-; FUNC-LABEL: @v_test_umin_ult_i32
-; SI: v_min_u32_e32
-define void @v_test_umin_ult_i32(i32 addrspace(1)* %out, i32 addrspace(1)* %aptr, i32 addrspace(1)* %bptr) nounwind {
-  %tid = call i32 @llvm.r600.read.tidig.x() nounwind readnone
-  %gep0 = getelementptr i32, i32 addrspace(1)* %aptr, i32 %tid
-  %gep1 = getelementptr i32, i32 addrspace(1)* %bptr, i32 %tid
-  %outgep = getelementptr i32, i32 addrspace(1)* %out, i32 %tid
-  %a = load i32, i32 addrspace(1)* %gep0, align 4
-  %b = load i32, i32 addrspace(1)* %gep1, align 4
-  %cmp = icmp ult i32 %a, %b
-  %val = select i1 %cmp, i32 %a, i32 %b
-  store i32 %val, i32 addrspace(1)* %outgep, align 4
-  ret void
-}
-
-; FUNC-LABEL: @s_test_umin_ult_i32
-; SI: s_min_u32
-define void @s_test_umin_ult_i32(i32 addrspace(1)* %out, i32 %a, i32 %b) nounwind {
-  %cmp = icmp ult i32 %a, %b
-  %val = select i1 %cmp, i32 %a, i32 %b
-  store i32 %val, i32 addrspace(1)* %out, align 4
-  ret void
-}
-
-; FUNC-LABEL: @v_test_umin_ult_i32_multi_use
-; SI-NOT: v_min
-; SI: v_cmp_lt_u32
-; SI-NEXT: v_cndmask_b32
-; SI-NOT: v_min
-; SI: s_endpgm
-define void @v_test_umin_ult_i32_multi_use(i32 addrspace(1)* %out0, i1 addrspace(1)* %out1, i32 addrspace(1)* %aptr, i32 addrspace(1)* %bptr) nounwind {
-  %tid = call i32 @llvm.r600.read.tidig.x() nounwind readnone
-  %gep0 = getelementptr i32, i32 addrspace(1)* %aptr, i32 %tid
-  %gep1 = getelementptr i32, i32 addrspace(1)* %bptr, i32 %tid
-  %outgep0 = getelementptr i32, i32 addrspace(1)* %out0, i32 %tid
-  %outgep1 = getelementptr i1, i1 addrspace(1)* %out1, i32 %tid
-  %a = load i32, i32 addrspace(1)* %gep0, align 4
-  %b = load i32, i32 addrspace(1)* %gep1, align 4
-  %cmp = icmp ult i32 %a, %b
-  %val = select i1 %cmp, i32 %a, i32 %b
-  store i32 %val, i32 addrspace(1)* %outgep0, align 4
-  store i1 %cmp, i1 addrspace(1)* %outgep1
-  ret void
-}
diff --git a/test/CodeGen/R600/scratch-buffer.ll b/test/CodeGen/R600/scratch-buffer.ll
deleted file mode 100644
index 56088718ada..00000000000
--- a/test/CodeGen/R600/scratch-buffer.ll
+++ /dev/null
@@ -1,87 +0,0 @@
-; RUN: llc -verify-machineinstrs -march=amdgcn -mcpu=SI < %s | FileCheck %s
-; RUN: llc -verify-machineinstrs -march=amdgcn -mcpu=tonga < %s | FileCheck %s
-
-; When a frame index offset is more than 12-bits, make sure we don't store
-; it in mubuf's offset field.
-
-; Also, make sure we use the same register for storing the scratch buffer addresss
-; for both stores. This register is allocated by the register scavenger, so we
-; should be able to reuse the same regiser for each scratch buffer access.
-
-; CHECK-LABEL: {{^}}legal_offset_fi:
-; CHECK: v_mov_b32_e32 [[OFFSET:v[0-9]+]], 0{{$}}
-; CHECK: buffer_store_dword v{{[0-9]+}}, [[OFFSET]], s[{{[0-9]+}}:{{[0-9]+}}], s{{[0-9]+}} offen
-; CHECK: v_mov_b32_e32 [[OFFSET]], 0x8000
-; CHECK: buffer_store_dword v{{[0-9]+}}, [[OFFSET]], s[{{[0-9]+}}:{{[0-9]+}}], s{{[0-9]+}} offen{{$}}
-
-define void @legal_offset_fi(i32 addrspace(1)* %out, i32 %cond, i32 %if_offset, i32 %else_offset) {
-entry:
-  %scratch0 = alloca [8192 x i32]
-  %scratch1 = alloca [8192 x i32]
-
-  %scratchptr0 = getelementptr [8192 x i32], [8192 x i32]* %scratch0, i32 0, i32 0
-  store i32 1, i32* %scratchptr0
-
-  %scratchptr1 = getelementptr [8192 x i32], [8192 x i32]* %scratch1, i32 0, i32 0
-  store i32 2, i32* %scratchptr1
-
-  %cmp = icmp eq i32 %cond, 0
-  br i1 %cmp, label %if, label %else
-
-if:
-  %if_ptr = getelementptr [8192 x i32], [8192 x i32]* %scratch0, i32 0, i32 %if_offset
-  %if_value = load i32, i32* %if_ptr
-  br label %done
-
-else:
-  %else_ptr = getelementptr [8192 x i32], [8192 x i32]* %scratch1, i32 0, i32 %else_offset
-  %else_value = load i32, i32* %else_ptr
-  br label %done
-
-done:
-  %value = phi i32 [%if_value, %if], [%else_value, %else]
-  store i32 %value, i32 addrspace(1)* %out
-  ret void
-
-  ret void
-
-}
-
-; CHECK-LABEL: {{^}}legal_offset_fi_offset
-; CHECK: buffer_store_dword v{{[0-9]+}}, v{{[0-9]+}}, s[{{[0-9]+}}:{{[0-9]+}}], s{{[0-9]+}} offen
-; CHECK: v_add_i32_e32 [[OFFSET:v[0-9]+]], 0x8000
-; CHECK: buffer_store_dword v{{[0-9]+}}, [[OFFSET]], s[{{[0-9]+}}:{{[0-9]+}}], s{{[0-9]+}} offen{{$}}
-
-define void @legal_offset_fi_offset(i32 addrspace(1)* %out, i32 %cond, i32 addrspace(1)* %offsets, i32 %if_offset, i32 %else_offset) {
-entry:
-  %scratch0 = alloca [8192 x i32]
-  %scratch1 = alloca [8192 x i32]
-
-  %offset0 = load i32, i32 addrspace(1)* %offsets
-  %scratchptr0 = getelementptr [8192 x i32], [8192 x i32]* %scratch0, i32 0, i32 %offset0
-  store i32 %offset0, i32* %scratchptr0
-
-  %offsetptr1 = getelementptr i32, i32 addrspace(1)* %offsets, i32 1
-  %offset1 = load i32, i32 addrspace(1)* %offsetptr1
-  %scratchptr1 = getelementptr [8192 x i32], [8192 x i32]* %scratch1, i32 0, i32 %offset1
-  store i32 %offset1, i32* %scratchptr1
-
-  %cmp = icmp eq i32 %cond, 0
-  br i1 %cmp, label %if, label %else
-
-if:
-  %if_ptr = getelementptr [8192 x i32], [8192 x i32]* %scratch0, i32 0, i32 %if_offset
-  %if_value = load i32, i32* %if_ptr
-  br label %done
-
-else:
-  %else_ptr = getelementptr [8192 x i32], [8192 x i32]* %scratch1, i32 0, i32 %else_offset
-  %else_value = load i32, i32* %else_ptr
-  br label %done
-
-done:
-  %value = phi i32 [%if_value, %if], [%else_value, %else]
-  store i32 %value, i32 addrspace(1)* %out
-  ret void
-}
-
diff --git a/test/CodeGen/R600/select-vectors.ll b/test/CodeGen/R600/select-vectors.ll
deleted file mode 100644
index 59082c65cc8..00000000000
--- a/test/CodeGen/R600/select-vectors.ll
+++ /dev/null
@@ -1,156 +0,0 @@
-; RUN: llc -verify-machineinstrs -march=amdgcn -mcpu=SI < %s | FileCheck -check-prefix=SI -check-prefix=FUNC %s
-; RUN: llc -verify-machineinstrs -march=amdgcn -mcpu=tonga < %s | FileCheck -check-prefix=SI -check-prefix=FUNC %s
-
-; Test expansion of scalar selects on vectors.
-; Evergreen not enabled since it seems to be having problems with doubles.
-
-
-; FUNC-LABEL: {{^}}select_v4i8:
-; SI: v_cndmask_b32_e64
-; SI: v_cndmask_b32_e64
-; SI: v_cndmask_b32_e64
-; SI: v_cndmask_b32_e64
-define void @select_v4i8(<4 x i8> addrspace(1)* %out, <4 x i8> %a, <4 x i8> %b, i8 %c) nounwind {
-  %cmp = icmp eq i8 %c, 0
-  %select = select i1 %cmp, <4 x i8> %a, <4 x i8> %b
-  store <4 x i8> %select, <4 x i8> addrspace(1)* %out, align 4
-  ret void
-}
-
-; FUNC-LABEL: {{^}}select_v4i16:
-; SI: v_cndmask_b32_e64
-; SI: v_cndmask_b32_e64
-; SI: v_cndmask_b32_e64
-; SI: v_cndmask_b32_e64
-define void @select_v4i16(<4 x i16> addrspace(1)* %out, <4 x i16> %a, <4 x i16> %b, i32 %c) nounwind {
-  %cmp = icmp eq i32 %c, 0
-  %select = select i1 %cmp, <4 x i16> %a, <4 x i16> %b
-  store <4 x i16> %select, <4 x i16> addrspace(1)* %out, align 4
-  ret void
-}
-
-; FUNC-LABEL: {{^}}select_v2i32:
-; SI: v_cndmask_b32_e64
-; SI: v_cndmask_b32_e64
-; SI: buffer_store_dwordx2
-define void @select_v2i32(<2 x i32> addrspace(1)* %out, <2 x i32> %a, <2 x i32> %b, i32 %c) nounwind {
-  %cmp = icmp eq i32 %c, 0
-  %select = select i1 %cmp, <2 x i32> %a, <2 x i32> %b
-  store <2 x i32> %select, <2 x i32> addrspace(1)* %out, align 8
-  ret void
-}
-
-; FUNC-LABEL: {{^}}select_v4i32:
-; SI: v_cndmask_b32_e64
-; SI: v_cndmask_b32_e64
-; SI: v_cndmask_b32_e64
-; SI: v_cndmask_b32_e64
-; SI: buffer_store_dwordx4
-define void @select_v4i32(<4 x i32> addrspace(1)* %out, <4 x i32> %a, <4 x i32> %b, i32 %c) nounwind {
-  %cmp = icmp eq i32 %c, 0
-  %select = select i1 %cmp, <4 x i32> %a, <4 x i32> %b
-  store <4 x i32> %select, <4 x i32> addrspace(1)* %out, align 16
-  ret void
-}
-
-; FUNC-LABEL: {{^}}select_v8i32:
-; SI: v_cndmask_b32_e64
-; SI: v_cndmask_b32_e64
-; SI: v_cndmask_b32_e64
-; SI: v_cndmask_b32_e64
-; SI: v_cndmask_b32_e64
-; SI: v_cndmask_b32_e64
-; SI: v_cndmask_b32_e64
-; SI: v_cndmask_b32_e64
-define void @select_v8i32(<8 x i32> addrspace(1)* %out, <8 x i32> %a, <8 x i32> %b, i32 %c) nounwind {
-  %cmp = icmp eq i32 %c, 0
-  %select = select i1 %cmp, <8 x i32> %a, <8 x i32> %b
-  store <8 x i32> %select, <8 x i32> addrspace(1)* %out, align 16
-  ret void
-}
-
-; FUNC-LABEL: {{^}}select_v2f32:
-; SI: buffer_store_dwordx2
-define void @select_v2f32(<2 x float> addrspace(1)* %out, <2 x float> %a, <2 x float> %b, i32 %c) nounwind {
-  %cmp = icmp eq i32 %c, 0
-  %select = select i1 %cmp, <2 x float> %a, <2 x float> %b
-  store <2 x float> %select, <2 x float> addrspace(1)* %out, align 16
-  ret void
-}
-
-; FUNC-LABEL: {{^}}select_v4f32:
-; SI: buffer_store_dwordx4
-define void @select_v4f32(<4 x float> addrspace(1)* %out, <4 x float> %a, <4 x float> %b, i32 %c) nounwind {
-  %cmp = icmp eq i32 %c, 0
-  %select = select i1 %cmp, <4 x float> %a, <4 x float> %b
-  store <4 x float> %select, <4 x float> addrspace(1)* %out, align 16
-  ret void
-}
-
-; FUNC-LABEL: {{^}}select_v8f32:
-; SI: v_cndmask_b32_e64
-; SI: v_cndmask_b32_e64
-; SI: v_cndmask_b32_e64
-; SI: v_cndmask_b32_e64
-; SI: v_cndmask_b32_e64
-; SI: v_cndmask_b32_e64
-; SI: v_cndmask_b32_e64
-; SI: v_cndmask_b32_e64
-define void @select_v8f32(<8 x float> addrspace(1)* %out, <8 x float> %a, <8 x float> %b, i32 %c) nounwind {
-  %cmp = icmp eq i32 %c, 0
-  %select = select i1 %cmp, <8 x float> %a, <8 x float> %b
-  store <8 x float> %select, <8 x float> addrspace(1)* %out, align 16
-  ret void
-}
-
-; FUNC-LABEL: {{^}}select_v2f64:
-; SI: v_cndmask_b32_e64
-; SI: v_cndmask_b32_e64
-; SI: v_cndmask_b32_e64
-; SI: v_cndmask_b32_e64
-define void @select_v2f64(<2 x double> addrspace(1)* %out, <2 x double> %a, <2 x double> %b, i32 %c) nounwind {
-  %cmp = icmp eq i32 %c, 0
-  %select = select i1 %cmp, <2 x double> %a, <2 x double> %b
-  store <2 x double> %select, <2 x double> addrspace(1)* %out, align 16
-  ret void
-}
-
-; FUNC-LABEL: {{^}}select_v4f64:
-; SI: v_cndmask_b32_e64
-; SI: v_cndmask_b32_e64
-; SI: v_cndmask_b32_e64
-; SI: v_cndmask_b32_e64
-; SI: v_cndmask_b32_e64
-; SI: v_cndmask_b32_e64
-; SI: v_cndmask_b32_e64
-; SI: v_cndmask_b32_e64
-define void @select_v4f64(<4 x double> addrspace(1)* %out, <4 x double> %a, <4 x double> %b, i32 %c) nounwind {
-  %cmp = icmp eq i32 %c, 0
-  %select = select i1 %cmp, <4 x double> %a, <4 x double> %b
-  store <4 x double> %select, <4 x double> addrspace(1)* %out, align 16
-  ret void
-}
-
-; FUNC-LABEL: {{^}}select_v8f64:
-; SI: v_cndmask_b32_e64
-; SI: v_cndmask_b32_e64
-; SI: v_cndmask_b32_e64
-; SI: v_cndmask_b32_e64
-; SI: v_cndmask_b32_e64
-; SI: v_cndmask_b32_e64
-; SI: v_cndmask_b32_e64
-; SI: v_cndmask_b32_e64
-; SI: v_cndmask_b32_e64
-; SI: v_cndmask_b32_e64
-; SI: v_cndmask_b32_e64
-; SI: v_cndmask_b32_e64
-; SI: v_cndmask_b32_e64
-; SI: v_cndmask_b32_e64
-; SI: v_cndmask_b32_e64
-; SI: v_cndmask_b32_e64
-define void @select_v8f64(<8 x double> addrspace(1)* %out, <8 x double> %a, <8 x double> %b, i32 %c) nounwind {
-  %cmp = icmp eq i32 %c, 0
-  %select = select i1 %cmp, <8 x double> %a, <8 x double> %b
-  store <8 x double> %select, <8 x double> addrspace(1)* %out, align 16
-  ret void
-}
diff --git a/test/CodeGen/R600/shl.ll b/test/CodeGen/R600/shl.ll
deleted file mode 100644
index 53b63dc4b8a..00000000000
--- a/test/CodeGen/R600/shl.ll
+++ /dev/null
@@ -1,180 +0,0 @@
-;RUN: llc < %s -march=r600 -mcpu=redwood | FileCheck --check-prefix=EG %s
-;RUN: llc < %s -march=amdgcn -mcpu=verde -verify-machineinstrs | FileCheck --check-prefix=SI %s
-;RUN: llc < %s -march=amdgcn -mcpu=tonga -verify-machineinstrs | FileCheck --check-prefix=VI %s
-
-;EG: {{^}}shl_v2i32:
-;EG: LSHL {{\*? *}}T{{[0-9]+\.[XYZW], T[0-9]+\.[XYZW], T[0-9]+\.[XYZW]}}
-;EG: LSHL {{\*? *}}T{{[0-9]+\.[XYZW], T[0-9]+\.[XYZW], T[0-9]+\.[XYZW]}}
-
-;SI: {{^}}shl_v2i32:
-;SI: v_lshl_b32_e32 v{{[0-9]+, v[0-9]+, v[0-9]+}}
-;SI: v_lshl_b32_e32 v{{[0-9]+, v[0-9]+, v[0-9]+}}
-
-;VI: {{^}}shl_v2i32:
-;VI: v_lshlrev_b32_e32 v{{[0-9]+, v[0-9]+, v[0-9]+}}
-;VI: v_lshlrev_b32_e32 v{{[0-9]+, v[0-9]+, v[0-9]+}}
-
-define void @shl_v2i32(<2 x i32> addrspace(1)* %out, <2 x i32> addrspace(1)* %in) {
-  %b_ptr = getelementptr <2 x i32>, <2 x i32> addrspace(1)* %in, i32 1
-  %a = load <2 x i32>, <2 x i32> addrspace(1) * %in
-  %b = load <2 x i32>, <2 x i32> addrspace(1) * %b_ptr
-  %result = shl <2 x i32> %a, %b
-  store <2 x i32> %result, <2 x i32> addrspace(1)* %out
-  ret void
-}
-
-;EG: {{^}}shl_v4i32:
-;EG: LSHL {{\*? *}}T{{[0-9]+\.[XYZW], T[0-9]+\.[XYZW], T[0-9]+\.[XYZW]}}
-;EG: LSHL {{\*? *}}T{{[0-9]+\.[XYZW], T[0-9]+\.[XYZW], T[0-9]+\.[XYZW]}}
-;EG: LSHL {{\*? *}}T{{[0-9]+\.[XYZW], T[0-9]+\.[XYZW], T[0-9]+\.[XYZW]}}
-;EG: LSHL {{\*? *}}T{{[0-9]+\.[XYZW], T[0-9]+\.[XYZW], T[0-9]+\.[XYZW]}}
-
-;SI: {{^}}shl_v4i32:
-;SI: v_lshl_b32_e32 v{{[0-9]+, v[0-9]+, v[0-9]+}}
-;SI: v_lshl_b32_e32 v{{[0-9]+, v[0-9]+, v[0-9]+}}
-;SI: v_lshl_b32_e32 v{{[0-9]+, v[0-9]+, v[0-9]+}}
-;SI: v_lshl_b32_e32 v{{[0-9]+, v[0-9]+, v[0-9]+}}
-
-;VI: {{^}}shl_v4i32:
-;VI: v_lshlrev_b32_e32 v{{[0-9]+, v[0-9]+, v[0-9]+}}
-;VI: v_lshlrev_b32_e32 v{{[0-9]+, v[0-9]+, v[0-9]+}}
-;VI: v_lshlrev_b32_e32 v{{[0-9]+, v[0-9]+, v[0-9]+}}
-;VI: v_lshlrev_b32_e32 v{{[0-9]+, v[0-9]+, v[0-9]+}}
-
-define void @shl_v4i32(<4 x i32> addrspace(1)* %out, <4 x i32> addrspace(1)* %in) {
-  %b_ptr = getelementptr <4 x i32>, <4 x i32> addrspace(1)* %in, i32 1
-  %a = load <4 x i32>, <4 x i32> addrspace(1) * %in
-  %b = load <4 x i32>, <4 x i32> addrspace(1) * %b_ptr
-  %result = shl <4 x i32> %a, %b
-  store <4 x i32> %result, <4 x i32> addrspace(1)* %out
-  ret void
-}
-
-;EG: {{^}}shl_i64:
-;EG: SUB_INT {{\*? *}}[[COMPSH:T[0-9]+\.[XYZW]]], {{literal.[xy]}}, [[SHIFT:T[0-9]+\.[XYZW]]]
-;EG: LSHR {{\* *}}[[TEMP:T[0-9]+\.[XYZW]]], [[OPLO:T[0-9]+\.[XYZW]]], {{[[COMPSH]]|PV.[XYZW]}}
-;EG: LSHR {{\*? *}}[[OVERF:T[0-9]+\.[XYZW]]], {{[[TEMP]]|PV.[XYZW]}}, 1
-;EG_CHECK-DAG: ADD_INT {{\*? *}}[[BIGSH:T[0-9]+\.[XYZW]]], [[SHIFT]], literal
-;EG-DAG: LSHL {{\*? *}}[[HISMTMP:T[0-9]+\.[XYZW]]], [[OPHI:T[0-9]+\.[XYZW]]], [[SHIFT]]
-;EG-DAG: OR_INT {{\*? *}}[[HISM:T[0-9]+\.[XYZW]]], {{[[HISMTMP]]|PV.[XYZW]}}, {{[[OVERF]]|PV.[XYZW]}}
-;EG-DAG: LSHL {{\*? *}}[[LOSM:T[0-9]+\.[XYZW]]], [[OPLO]], {{PS|[[SHIFT]]}}
-;EG-DAG: SETGT_UINT {{\*? *}}[[RESC:T[0-9]+\.[XYZW]]], [[SHIFT]], literal
-;EG-DAG: CNDE_INT {{\*? *}}[[RESLO:T[0-9]+\.[XYZW]]], {{T[0-9]+\.[XYZW]}}
-;EG-DAG: CNDE_INT {{\*? *}}[[RESHI:T[0-9]+\.[XYZW]]], {{T[0-9]+\.[XYZW], .*}}, 0.0
-
-;SI: {{^}}shl_i64:
-;SI: v_lshl_b64 {{v\[[0-9]+:[0-9]+\], v\[[0-9]+:[0-9]+\], v[0-9]+}}
-
-;VI: {{^}}shl_i64:
-;VI: v_lshlrev_b64 {{v\[[0-9]+:[0-9]+\], v[0-9]+, v\[[0-9]+:[0-9]+\]}}
-
-define void @shl_i64(i64 addrspace(1)* %out, i64 addrspace(1)* %in) {
-  %b_ptr = getelementptr i64, i64 addrspace(1)* %in, i64 1
-  %a = load i64, i64 addrspace(1) * %in
-  %b = load i64, i64 addrspace(1) * %b_ptr
-  %result = shl i64 %a, %b
-  store i64 %result, i64 addrspace(1)* %out
-  ret void
-}
-
-;EG: {{^}}shl_v2i64:
-;EG-DAG: SUB_INT {{\*? *}}[[COMPSHA:T[0-9]+\.[XYZW]]], {{literal.[xy]}}, [[SHA:T[0-9]+\.[XYZW]]]
-;EG-DAG: SUB_INT {{\*? *}}[[COMPSHB:T[0-9]+\.[XYZW]]], {{literal.[xy]}}, [[SHB:T[0-9]+\.[XYZW]]]
-;EG-DAG: LSHR {{\*? *}}[[COMPSHA]]
-;EG-DAG: LSHR {{\*? *}}[[COMPSHB]]
-;EG-DAG: LSHR {{.*}}, 1
-;EG-DAG: LSHR {{.*}}, 1
-;EG-DAG: ADD_INT  {{\*? *}}[[BIGSHA:T[0-9]+\.[XYZW]]]{{.*}}, literal
-;EG-DAG: ADD_INT  {{\*? *}}[[BIGSHB:T[0-9]+\.[XYZW]]]{{.*}}, literal
-;EG-DAG: LSHL {{.*}}, [[SHA]]
-;EG-DAG: LSHL {{.*}}, [[SHB]]
-;EG-DAG: LSHL {{.*}}, [[SHA]]
-;EG-DAG: LSHL {{.*}}, [[SHB]]
-;EG-DAG: LSHL
-;EG-DAG: LSHL
-;EG-DAG: SETGT_UINT {{\*? *T[0-9]\.[XYZW]}}, [[SHA]], literal
-;EG-DAG: SETGT_UINT {{\*? *T[0-9]\.[XYZW]}}, [[SHB]], literal
-;EG-DAG: CNDE_INT {{.*}}, 0.0
-;EG-DAG: CNDE_INT {{.*}}, 0.0
-;EG-DAG: CNDE_INT
-;EG-DAG: CNDE_INT
-
-;SI: {{^}}shl_v2i64:
-;SI: v_lshl_b64 {{v\[[0-9]+:[0-9]+\], v\[[0-9]+:[0-9]+\], v[0-9]+}}
-;SI: v_lshl_b64 {{v\[[0-9]+:[0-9]+\], v\[[0-9]+:[0-9]+\], v[0-9]+}}
-
-;VI: {{^}}shl_v2i64:
-;VI: v_lshlrev_b64 {{v\[[0-9]+:[0-9]+\], v[0-9]+, v\[[0-9]+:[0-9]+\]}}
-;VI: v_lshlrev_b64 {{v\[[0-9]+:[0-9]+\], v[0-9]+, v\[[0-9]+:[0-9]+\]}}
-
-define void @shl_v2i64(<2 x i64> addrspace(1)* %out, <2 x i64> addrspace(1)* %in) {
-  %b_ptr = getelementptr <2 x i64>, <2 x i64> addrspace(1)* %in, i64 1
-  %a = load <2 x i64>, <2 x i64> addrspace(1) * %in
-  %b = load <2 x i64>, <2 x i64> addrspace(1) * %b_ptr
-  %result = shl <2 x i64> %a, %b
-  store <2 x i64> %result, <2 x i64> addrspace(1)* %out
-  ret void
-}
-
-;EG: {{^}}shl_v4i64:
-;EG-DAG: SUB_INT {{\*? *}}[[COMPSHA:T[0-9]+\.[XYZW]]], {{literal.[xy]}}, [[SHA:T[0-9]+\.[XYZW]]]
-;EG-DAG: SUB_INT {{\*? *}}[[COMPSHB:T[0-9]+\.[XYZW]]], {{literal.[xy]}}, [[SHB:T[0-9]+\.[XYZW]]]
-;EG-DAG: SUB_INT {{\*? *}}[[COMPSHC:T[0-9]+\.[XYZW]]], {{literal.[xy]}}, [[SHC:T[0-9]+\.[XYZW]]]
-;EG-DAG: SUB_INT {{\*? *}}[[COMPSHD:T[0-9]+\.[XYZW]]], {{literal.[xy]}}, [[SHD:T[0-9]+\.[XYZW]]]
-;EG-DAG: LSHR {{\*? *}}[[COMPSHA]]
-;EG-DAG: LSHR {{\*? *}}[[COMPSHB]]
-;EG-DAG: LSHR {{\*? *}}[[COMPSHC]]
-;EG-DAG: LSHR {{\*? *}}[[COMPSHD]]
-;EG-DAG: LSHR {{.*}}, 1
-;EG-DAG: LSHR {{.*}}, 1
-;EG-DAG: LSHR {{.*}}, 1
-;EG-DAG: LSHR {{.*}}, 1
-;EG-DAG: ADD_INT  {{\*? *}}[[BIGSHA:T[0-9]+\.[XYZW]]]{{.*}}, literal
-;EG-DAG: ADD_INT  {{\*? *}}[[BIGSHB:T[0-9]+\.[XYZW]]]{{.*}}, literal
-;EG-DAG: ADD_INT  {{\*? *}}[[BIGSHC:T[0-9]+\.[XYZW]]]{{.*}}, literal
-;EG-DAG: ADD_INT  {{\*? *}}[[BIGSHD:T[0-9]+\.[XYZW]]]{{.*}}, literal
-;EG-DAG: LSHL {{.*}}, [[SHA]]
-;EG-DAG: LSHL {{.*}}, [[SHB]]
-;EG-DAG: LSHL {{.*}}, [[SHC]]
-;EG-DAG: LSHL {{.*}}, [[SHD]]
-;EG-DAG: LSHL {{.*}}, [[SHA]]
-;EG-DAG: LSHL {{.*}}, [[SHB]]
-;EG-DAG: LSHL {{.*}}, [[SHC]]
-;EG-DAG: LSHL {{.*}}, [[SHD]]
-;EG-DAG: LSHL
-;EG-DAG: LSHL
-;EG-DAG: LSHL
-;EG-DAG: LSHL
-;EG-DAG: SETGT_UINT {{\*? *T[0-9]\.[XYZW]}}, [[SHA]], literal
-;EG-DAG: SETGT_UINT {{\*? *T[0-9]\.[XYZW]}}, [[SHB]], literal
-;EG-DAG: SETGT_UINT {{\*? *T[0-9]\.[XYZW]}}, [[SHC]], literal
-;EG-DAG: SETGT_UINT {{\*? *T[0-9]\.[XYZW]}}, [[SHD]], literal
-;EG-DAG: CNDE_INT {{.*}}, 0.0
-;EG-DAG: CNDE_INT {{.*}}, 0.0
-;EG-DAG: CNDE_INT {{.*}}, 0.0
-;EG-DAG: CNDE_INT {{.*}}, 0.0
-;EG-DAG: CNDE_INT
-;EG-DAG: CNDE_INT
-;EG-DAG: CNDE_INT
-;EG-DAG: CNDE_INT
-
-;SI: {{^}}shl_v4i64:
-;SI: v_lshl_b64 {{v\[[0-9]+:[0-9]+\], v\[[0-9]+:[0-9]+\], v[0-9]+}}
-;SI: v_lshl_b64 {{v\[[0-9]+:[0-9]+\], v\[[0-9]+:[0-9]+\], v[0-9]+}}
-;SI: v_lshl_b64 {{v\[[0-9]+:[0-9]+\], v\[[0-9]+:[0-9]+\], v[0-9]+}}
-;SI: v_lshl_b64 {{v\[[0-9]+:[0-9]+\], v\[[0-9]+:[0-9]+\], v[0-9]+}}
-
-;VI: {{^}}shl_v4i64:
-;VI: v_lshlrev_b64 {{v\[[0-9]+:[0-9]+\], v[0-9]+, v\[[0-9]+:[0-9]+\]}}
-;VI: v_lshlrev_b64 {{v\[[0-9]+:[0-9]+\], v[0-9]+, v\[[0-9]+:[0-9]+\]}}
-;VI: v_lshlrev_b64 {{v\[[0-9]+:[0-9]+\], v[0-9]+, v\[[0-9]+:[0-9]+\]}}
-;VI: v_lshlrev_b64 {{v\[[0-9]+:[0-9]+\], v[0-9]+, v\[[0-9]+:[0-9]+\]}}
-
-define void @shl_v4i64(<4 x i64> addrspace(1)* %out, <4 x i64> addrspace(1)* %in) {
-  %b_ptr = getelementptr <4 x i64>, <4 x i64> addrspace(1)* %in, i64 1
-  %a = load <4 x i64>, <4 x i64> addrspace(1) * %in
-  %b = load <4 x i64>, <4 x i64> addrspace(1) * %b_ptr
-  %result = shl <4 x i64> %a, %b
-  store <4 x i64> %result, <4 x i64> addrspace(1)* %out
-  ret void
-}
diff --git a/test/CodeGen/R600/si-annotate-cf.ll b/test/CodeGen/R600/si-annotate-cf.ll
deleted file mode 100644
index 3f3098862ef..00000000000
--- a/test/CodeGen/R600/si-annotate-cf.ll
+++ /dev/null
@@ -1,25 +0,0 @@
-; RUN: llc < %s -march=amdgcn -mcpu=verde -verify-machineinstrs | FileCheck --check-prefix=SI --check-prefix=FUNC %s
-; RUN: llc < %s -march=amdgcn -mcpu=tonga -verify-machineinstrs | FileCheck --check-prefix=SI --check-prefix=FUNC %s
-
-; FUNC-LABEL: {{^}}break_inserted_outside_of_loop:
-
-; SI: [[LOOP_LABEL:[A-Z0-9]+]]:
-; Lowered break instructin:
-; SI: s_or_b64
-; Lowered Loop instruction:
-; SI: s_andn2_b64
-; s_cbranch_execnz [[LOOP_LABEL]]
-; SI: s_endpgm
-define void @break_inserted_outside_of_loop(i32 addrspace(1)* %out, i32 %a, i32 %b) {
-main_body:
-  %0 = and i32 %a, %b
-  %1 = trunc i32 %0 to i1
-  br label %ENDIF
-
-ENDLOOP:
-  store i32 0, i32 addrspace(1)* %out
-  ret void
-
-ENDIF:
-  br i1 %1, label %ENDLOOP, label %ENDIF
-}
diff --git a/test/CodeGen/R600/store.ll b/test/CodeGen/R600/store.ll
deleted file mode 100644
index b51b9b71f70..00000000000
--- a/test/CodeGen/R600/store.ll
+++ /dev/null
@@ -1,370 +0,0 @@
-; RUN: llc -march=amdgcn -mcpu=verde -verify-machineinstrs < %s | FileCheck -check-prefix=SI -check-prefix=FUNC %s
-; RUN: llc -march=amdgcn -mcpu=tonga -verify-machineinstrs < %s | FileCheck -check-prefix=SI -check-prefix=FUNC %s
-; RUN: llc -march=r600 -mcpu=redwood < %s | FileCheck -check-prefix=EG -check-prefix=FUNC %s
-; RUN: llc -march=r600 -mcpu=cayman < %s | FileCheck -check-prefix=CM -check-prefix=FUNC %s
-
-;===------------------------------------------------------------------------===;
-; Global Address Space
-;===------------------------------------------------------------------------===;
-; FUNC-LABEL: {{^}}store_i1:
-; EG: MEM_RAT MSKOR
-; SI: buffer_store_byte
-define void @store_i1(i1 addrspace(1)* %out) {
-entry:
-  store i1 true, i1 addrspace(1)* %out
-  ret void
-}
-
-; i8 store
-; FUNC-LABEL: {{^}}store_i8:
-; EG: MEM_RAT MSKOR T[[RW_GPR:[0-9]]].XW, T{{[0-9]}}.X
-
-; IG 0: Get the byte index and truncate the value
-; EG: AND_INT * T{{[0-9]}}.[[BI_CHAN:[XYZW]]], KC0[2].Y, literal.x
-; EG: LSHL T{{[0-9]}}.[[SHIFT_CHAN:[XYZW]]], PV.[[BI_CHAN]], literal.x
-; EG: AND_INT * T{{[0-9]}}.[[TRUNC_CHAN:[XYZW]]], KC0[2].Z, literal.y
-; EG-NEXT: 3(4.203895e-45), 255(3.573311e-43)
-
-
-; IG 1: Truncate the calculated the shift amount for the mask
-
-; IG 2: Shift the value and the mask
-; EG: LSHL T[[RW_GPR]].X, PS, PV.[[SHIFT_CHAN]]
-; EG: LSHL * T[[RW_GPR]].W, literal.x, PV.[[SHIFT_CHAN]]
-; EG-NEXT: 255
-; IG 3: Initialize the Y and Z channels to zero
-;       XXX: An optimal scheduler should merge this into one of the prevous IGs.
-; EG: MOV T[[RW_GPR]].Y, 0.0
-; EG: MOV * T[[RW_GPR]].Z, 0.0
-
-; SI: buffer_store_byte
-
-define void @store_i8(i8 addrspace(1)* %out, i8 %in) {
-entry:
-  store i8 %in, i8 addrspace(1)* %out
-  ret void
-}
-
-; i16 store
-; FUNC-LABEL: {{^}}store_i16:
-; EG: MEM_RAT MSKOR T[[RW_GPR:[0-9]]].XW, T{{[0-9]}}.X
-
-; IG 0: Get the byte index and truncate the value
-
-
-; EG: AND_INT * T{{[0-9]}}.[[BI_CHAN:[XYZW]]], KC0[2].Y, literal.x
-; EG-NEXT: 3(4.203895e-45),
-
-; EG: LSHL T{{[0-9]}}.[[SHIFT_CHAN:[XYZW]]], PV.[[BI_CHAN]], literal.x
-; EG: AND_INT * T{{[0-9]}}.[[TRUNC_CHAN:[XYZW]]], KC0[2].Z, literal.y
-
-; EG-NEXT: 3(4.203895e-45), 65535(9.183409e-41)
-; IG 1: Truncate the calculated the shift amount for the mask
-
-; IG 2: Shift the value and the mask
-; EG: LSHL T[[RW_GPR]].X, PS, PV.[[SHIFT_CHAN]]
-; EG: LSHL * T[[RW_GPR]].W, literal.x, PV.[[SHIFT_CHAN]]
-; EG-NEXT: 65535
-; IG 3: Initialize the Y and Z channels to zero
-;       XXX: An optimal scheduler should merge this into one of the prevous IGs.
-; EG: MOV T[[RW_GPR]].Y, 0.0
-; EG: MOV * T[[RW_GPR]].Z, 0.0
-
-; SI: buffer_store_short
-define void @store_i16(i16 addrspace(1)* %out, i16 %in) {
-entry:
-  store i16 %in, i16 addrspace(1)* %out
-  ret void
-}
-
-; FUNC-LABEL: {{^}}store_v2i8:
-; EG: MEM_RAT MSKOR
-; EG-NOT: MEM_RAT MSKOR
-
-; SI: buffer_store_byte
-; SI: buffer_store_byte
-define void @store_v2i8(<2 x i8> addrspace(1)* %out, <2 x i32> %in) {
-entry:
-  %0 = trunc <2 x i32> %in to <2 x i8>
-  store <2 x i8> %0, <2 x i8> addrspace(1)* %out
-  ret void
-}
-
-
-; FUNC-LABEL: {{^}}store_v2i16:
-; EG: MEM_RAT_CACHELESS STORE_RAW
-
-; CM: MEM_RAT_CACHELESS STORE_DWORD
-
-; SI: buffer_store_short
-; SI: buffer_store_short
-define void @store_v2i16(<2 x i16> addrspace(1)* %out, <2 x i32> %in) {
-entry:
-  %0 = trunc <2 x i32> %in to <2 x i16>
-  store <2 x i16> %0, <2 x i16> addrspace(1)* %out
-  ret void
-}
-
-; FUNC-LABEL: {{^}}store_v4i8:
-; EG: MEM_RAT_CACHELESS STORE_RAW
-
-; CM: MEM_RAT_CACHELESS STORE_DWORD
-
-; SI: buffer_store_byte
-; SI: buffer_store_byte
-; SI: buffer_store_byte
-; SI: buffer_store_byte
-define void @store_v4i8(<4 x i8> addrspace(1)* %out, <4 x i32> %in) {
-entry:
-  %0 = trunc <4 x i32> %in to <4 x i8>
-  store <4 x i8> %0, <4 x i8> addrspace(1)* %out
-  ret void
-}
-
-; floating-point store
-; FUNC-LABEL: {{^}}store_f32:
-; EG: MEM_RAT_CACHELESS STORE_RAW T{{[0-9]+\.X, T[0-9]+\.X}}, 1
-
-; CM: MEM_RAT_CACHELESS STORE_DWORD T{{[0-9]+\.X, T[0-9]+\.X}}
-
-; SI: buffer_store_dword
-
-define void @store_f32(float addrspace(1)* %out, float %in) {
-  store float %in, float addrspace(1)* %out
-  ret void
-}
-
-; FUNC-LABEL: {{^}}store_v4i16:
-; EG: MEM_RAT MSKOR
-; EG: MEM_RAT MSKOR
-; EG: MEM_RAT MSKOR
-; EG: MEM_RAT MSKOR
-; EG-NOT: MEM_RAT MSKOR
-
-; SI: buffer_store_short
-; SI: buffer_store_short
-; SI: buffer_store_short
-; SI: buffer_store_short
-; SI-NOT: buffer_store_byte
-define void @store_v4i16(<4 x i16> addrspace(1)* %out, <4 x i32> %in) {
-entry:
-  %0 = trunc <4 x i32> %in to <4 x i16>
-  store <4 x i16> %0, <4 x i16> addrspace(1)* %out
-  ret void
-}
-
-; vec2 floating-point stores
-; FUNC-LABEL: {{^}}store_v2f32:
-; EG: MEM_RAT_CACHELESS STORE_RAW
-
-; CM: MEM_RAT_CACHELESS STORE_DWORD
-
-; SI: buffer_store_dwordx2
-
-define void @store_v2f32(<2 x float> addrspace(1)* %out, float %a, float %b) {
-entry:
-  %0 = insertelement <2 x float> <float 0.0, float 0.0>, float %a, i32 0
-  %1 = insertelement <2 x float> %0, float %b, i32 1
-  store <2 x float> %1, <2 x float> addrspace(1)* %out
-  ret void
-}
-
-; FUNC-LABEL: {{^}}store_v4i32:
-; EG: MEM_RAT_CACHELESS STORE_RAW
-; EG-NOT: MEM_RAT_CACHELESS STORE_RAW
-
-; CM: MEM_RAT_CACHELESS STORE_DWORD
-; CM-NOT: MEM_RAT_CACHELESS STORE_DWORD
-
-; SI: buffer_store_dwordx4
-define void @store_v4i32(<4 x i32> addrspace(1)* %out, <4 x i32> %in) {
-entry:
-  store <4 x i32> %in, <4 x i32> addrspace(1)* %out
-  ret void
-}
-
-; FUNC-LABEL: {{^}}store_i64_i8:
-; EG: MEM_RAT MSKOR
-; SI: buffer_store_byte
-define void @store_i64_i8(i8 addrspace(1)* %out, i64 %in) {
-entry:
-  %0 = trunc i64 %in to i8
-  store i8 %0, i8 addrspace(1)* %out
-  ret void
-}
-
-; FUNC-LABEL: {{^}}store_i64_i16:
-; EG: MEM_RAT MSKOR
-; SI: buffer_store_short
-define void @store_i64_i16(i16 addrspace(1)* %out, i64 %in) {
-entry:
-  %0 = trunc i64 %in to i16
-  store i16 %0, i16 addrspace(1)* %out
-  ret void
-}
-
-;===------------------------------------------------------------------------===;
-; Local Address Space
-;===------------------------------------------------------------------------===;
-
-; FUNC-LABEL: {{^}}store_local_i1:
-; EG: LDS_BYTE_WRITE
-; SI: ds_write_b8
-define void @store_local_i1(i1 addrspace(3)* %out) {
-entry:
-  store i1 true, i1 addrspace(3)* %out
-  ret void
-}
-
-; FUNC-LABEL: {{^}}store_local_i8:
-; EG: LDS_BYTE_WRITE
-
-; SI: ds_write_b8
-define void @store_local_i8(i8 addrspace(3)* %out, i8 %in) {
-  store i8 %in, i8 addrspace(3)* %out
-  ret void
-}
-
-; FUNC-LABEL: {{^}}store_local_i16:
-; EG: LDS_SHORT_WRITE
-
-; SI: ds_write_b16
-define void @store_local_i16(i16 addrspace(3)* %out, i16 %in) {
-  store i16 %in, i16 addrspace(3)* %out
-  ret void
-}
-
-; FUNC-LABEL: {{^}}store_local_v2i16:
-; EG: LDS_WRITE
-
-; CM: LDS_WRITE
-
-; SI: ds_write_b16
-; SI: ds_write_b16
-define void @store_local_v2i16(<2 x i16> addrspace(3)* %out, <2 x i16> %in) {
-entry:
-  store <2 x i16> %in, <2 x i16> addrspace(3)* %out
-  ret void
-}
-
-; FUNC-LABEL: {{^}}store_local_v4i8:
-; EG: LDS_WRITE
-
-; CM: LDS_WRITE
-
-; SI: ds_write_b8
-; SI: ds_write_b8
-; SI: ds_write_b8
-; SI: ds_write_b8
-define void @store_local_v4i8(<4 x i8> addrspace(3)* %out, <4 x i8> %in) {
-entry:
-  store <4 x i8> %in, <4 x i8> addrspace(3)* %out
-  ret void
-}
-
-; FUNC-LABEL: {{^}}store_local_v2i32:
-; EG: LDS_WRITE
-; EG: LDS_WRITE
-
-; CM: LDS_WRITE
-; CM: LDS_WRITE
-
-; SI: ds_write_b64
-define void @store_local_v2i32(<2 x i32> addrspace(3)* %out, <2 x i32> %in) {
-entry:
-  store <2 x i32> %in, <2 x i32> addrspace(3)* %out
-  ret void
-}
-
-; FUNC-LABEL: {{^}}store_local_v4i32:
-; EG: LDS_WRITE
-; EG: LDS_WRITE
-; EG: LDS_WRITE
-; EG: LDS_WRITE
-
-; CM: LDS_WRITE
-; CM: LDS_WRITE
-; CM: LDS_WRITE
-; CM: LDS_WRITE
-
-; SI: ds_write_b32
-; SI: ds_write_b32
-; SI: ds_write_b32
-; SI: ds_write_b32
-define void @store_local_v4i32(<4 x i32> addrspace(3)* %out, <4 x i32> %in) {
-entry:
-  store <4 x i32> %in, <4 x i32> addrspace(3)* %out
-  ret void
-}
-
-; FUNC-LABEL: {{^}}store_local_i64_i8:
-; EG: LDS_BYTE_WRITE
-; SI: ds_write_b8
-define void @store_local_i64_i8(i8 addrspace(3)* %out, i64 %in) {
-entry:
-  %0 = trunc i64 %in to i8
-  store i8 %0, i8 addrspace(3)* %out
-  ret void
-}
-
-; FUNC-LABEL: {{^}}store_local_i64_i16:
-; EG: LDS_SHORT_WRITE
-; SI: ds_write_b16
-define void @store_local_i64_i16(i16 addrspace(3)* %out, i64 %in) {
-entry:
-  %0 = trunc i64 %in to i16
-  store i16 %0, i16 addrspace(3)* %out
-  ret void
-}
-
-; The stores in this function are combined by the optimizer to create a
-; 64-bit store with 32-bit alignment.  This is legal for SI and the legalizer
-; should not try to split the 64-bit store back into 2 32-bit stores.
-;
-; Evergreen / Northern Islands don't support 64-bit stores yet, so there should
-; be two 32-bit stores.
-
-; FUNC-LABEL: {{^}}vecload2:
-; EG: MEM_RAT_CACHELESS STORE_RAW
-
-; CM: MEM_RAT_CACHELESS STORE_DWORD
-
-; SI: buffer_store_dwordx2
-define void @vecload2(i32 addrspace(1)* nocapture %out, i32 addrspace(2)* nocapture %mem) #0 {
-entry:
-  %0 = load i32, i32 addrspace(2)* %mem, align 4
-  %arrayidx1.i = getelementptr inbounds i32, i32 addrspace(2)* %mem, i64 1
-  %1 = load i32, i32 addrspace(2)* %arrayidx1.i, align 4
-  store i32 %0, i32 addrspace(1)* %out, align 4
-  %arrayidx1 = getelementptr inbounds i32, i32 addrspace(1)* %out, i64 1
-  store i32 %1, i32 addrspace(1)* %arrayidx1, align 4
-  ret void
-}
-
-attributes #0 = { nounwind "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "unsafe-fp-math"="false" "use-soft-float"="false" }
-
-; When i128 was a legal type this program generated cannot select errors:
-
-; FUNC-LABEL: {{^}}"i128-const-store":
-; FIXME: We should be able to to this with one store instruction
-; EG: STORE_RAW
-; EG: STORE_RAW
-; EG: STORE_RAW
-; EG: STORE_RAW
-; CM: STORE_DWORD
-; CM: STORE_DWORD
-; CM: STORE_DWORD
-; CM: STORE_DWORD
-; SI: buffer_store_dwordx2
-; SI: buffer_store_dwordx2
-define void @i128-const-store(i32 addrspace(1)* %out) {
-entry:
-  store i32 1, i32 addrspace(1)* %out, align 4
-  %arrayidx2 = getelementptr inbounds i32, i32 addrspace(1)* %out, i64 1
-  store i32 1, i32 addrspace(1)* %arrayidx2, align 4
-  %arrayidx4 = getelementptr inbounds i32, i32 addrspace(1)* %out, i64 2
-  store i32 2, i32 addrspace(1)* %arrayidx4, align 4
-  %arrayidx6 = getelementptr inbounds i32, i32 addrspace(1)* %out, i64 3
-  store i32 2, i32 addrspace(1)* %arrayidx6, align 4
-  ret void
-}
diff --git a/test/CodeGen/R600/unroll.ll b/test/CodeGen/R600/unroll.ll
deleted file mode 100644
index ca8d822ec7e..00000000000
--- a/test/CodeGen/R600/unroll.ll
+++ /dev/null
@@ -1,37 +0,0 @@
-; RUN: opt -loop-unroll -simplifycfg -sroa %s -S -o - | FileCheck %s
-
-target datalayout = "e-p:32:32:32-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-v16:16:16-v24:32:32-v32:32:32-v48:64:64-v64:64:64-v96:128:128-v128:128:128-v192:256:256-v256:256:256-v512:512:512-v1024:1024:1024-v2048:2048:2048-n32:64"
-target triple = "r600--"
-
-; This test contains a simple loop that initializes an array declared in
-; private memory.  We want to make sure these kinds of loops are always
-; unrolled, because private memory is slow.
-
-; CHECK-LABEL: @test
-; CHECK-NOT: alloca
-; CHECK: store i32 5, i32 addrspace(1)* %out
-define void @test(i32 addrspace(1)* %out) {
-entry:
-  %0 = alloca [32 x i32]
-  br label %loop.header
-
-loop.header:
-  %counter = phi i32 [0, %entry], [%inc, %loop.inc]
-  br label %loop.body
-
-loop.body:
-  %ptr = getelementptr [32 x i32], [32 x i32]* %0, i32 0, i32 %counter
-  store i32 %counter, i32* %ptr
-  br label %loop.inc
-
-loop.inc:
-  %inc = add i32 %counter, 1
-  %1 = icmp sge i32 %counter, 32
-  br i1 %1, label  %exit, label %loop.header
-
-exit:
-  %2 = getelementptr [32 x i32], [32 x i32]* %0, i32 0, i32 5
-  %3 = load i32, i32* %2
-  store i32 %3, i32 addrspace(1)* %out
-  ret void
-}
diff --git a/test/CodeGen/SPARC/basictest.ll b/test/CodeGen/SPARC/basictest.ll
index 68f7c367203..3792100b2e6 100644
--- a/test/CodeGen/SPARC/basictest.ll
+++ b/test/CodeGen/SPARC/basictest.ll
@@ -36,3 +36,51 @@ entry:
   ret i32 0
 }
 
+; CHECK-LABEL: signed_divide:
+; CHECK: sra %o0, 31, %o2
+; CHECK: wr %g0, %o2, %y
+; CHECK: sdiv %o0, %o1, %o0
+define i32 @signed_divide(i32 %a, i32 %b) {
+  %r = sdiv i32 %a, %b
+  ret i32 %r
+}
+
+; CHECK-LABEL: unsigned_divide:
+; CHECK: wr %g0, %g0, %y
+; CHECK: udiv %o0, %o1, %o0
+define i32 @unsigned_divide(i32 %a, i32 %b) {
+  %r = udiv i32 %a, %b
+  ret i32 %r
+}
+
+; CHECK-LABEL: multiply_32x32:
+; CHECK: smul %o0, %o1, %o0
+define i32 @multiply_32x32(i32 %a, i32 %b) {
+  %r = mul i32 %a, %b
+  ret i32 %r
+}
+
+; CHECK-LABEL: signed_multiply_32x32_64:
+; CHECK: smul %o0, %o1, %o1
+; CHECK: rd %y, %o0
+define i64 @signed_multiply_32x32_64(i32 %a, i32 %b) {
+  %xa = sext i32 %a to i64
+  %xb = sext i32 %b to i64
+  %r = mul i64 %xa, %xb
+  ret i64 %r
+}
+
+; CHECK-LABEL: unsigned_multiply_32x32_64:
+; CHECK: umul %o0, %o1, %o2
+; CHECK: rd %y, %o2
+;FIXME: the smul in the output is totally redundant and should not there.
+; CHECK: smul %o0, %o1, %o1
+; CHECK: retl
+; CHECK: mov      %o2, %o0
+define i64 @unsigned_multiply_32x32_64(i32 %a, i32 %b) {
+  %xa = zext i32 %a to i64
+  %xb = zext i32 %b to i64
+  %r = mul i64 %xa, %xb
+  ret i64 %r
+}
+
diff --git a/test/CodeGen/SPARC/exception.ll b/test/CodeGen/SPARC/exception.ll
index 0af48d0b64b..f112328346d 100644
--- a/test/CodeGen/SPARC/exception.ll
+++ b/test/CodeGen/SPARC/exception.ll
@@ -71,7 +71,7 @@
 ; V9PIC: .L_ZTIi.DW.stub:
 ; V9PIC-NEXT:   .xword _ZTIi
 
-define i32 @main(i32 %argc, i8** nocapture readnone %argv) unnamed_addr #0 {
+define i32 @main(i32 %argc, i8** nocapture readnone %argv) unnamed_addr #0 personality i32 (i32, i64, i8*, i8*)* @__gxx_personality_v0 {
 entry:
   %0 = icmp eq i32 %argc, 2
   %1 = tail call i8* @__cxa_allocate_exception(i32 4) #1
@@ -102,7 +102,7 @@ entry:
   ret i32 %6
 
 "8":                                              ; preds = %"4", %"3"
-  %exc = landingpad { i8*, i32 } personality i32 (i32, i64, i8*, i8*)* @__gxx_personality_v0
+  %exc = landingpad { i8*, i32 }
           catch %struct.__fundamental_type_info_pseudo* @_ZTIi
           catch %struct.__fundamental_type_info_pseudo* @_ZTIf
   %exc_ptr12 = extractvalue { i8*, i32 } %exc, 0
diff --git a/test/CodeGen/SPARC/multiple-div.ll b/test/CodeGen/SPARC/multiple-div.ll
new file mode 100644
index 00000000000..6934f69ac18
--- /dev/null
+++ b/test/CodeGen/SPARC/multiple-div.ll
@@ -0,0 +1,21 @@
+; RUN: llc < %s -march=sparc | FileCheck %s
+; RUN: llc -O0 < %s -march=sparc | FileCheck %s
+
+;; llc -O0 used to try to spill Y to the stack, which isn't possible,
+;; and then crashed. Additionally, in -O1, it would omit the second
+;; apparently-redundant wr to %y, which is not actually redundant
+;; because the spec says to treat %y as potentially-written by udiv.
+
+; CHECK-LABEL: two_divides:
+; CHECK: wr %g0, %g0, %y
+; CHECK: udiv
+; CHECK: wr %g0, %g0, %y
+; CHECK: udiv
+; CHECK: add
+
+define i32 @two_divides(i32 %a, i32 %b) {
+  %r = udiv i32 %a, %b
+  %r2 = udiv i32 %b, %a
+  %r3 = add i32 %r, %r2
+  ret i32 %r3
+}
diff --git a/test/CodeGen/SPARC/obj-relocs.ll b/test/CodeGen/SPARC/obj-relocs.ll
index 115263ac5d4..0e7e04032be 100644
--- a/test/CodeGen/SPARC/obj-relocs.ll
+++ b/test/CodeGen/SPARC/obj-relocs.ll
@@ -1,31 +1,37 @@
 ; RUN: llc < %s -march=sparcv9 -filetype=obj --relocation-model=static | llvm-readobj -r | FileCheck %s --check-prefix=CHECK-ABS
 ; RUN: llc < %s -march=sparcv9 -filetype=obj --relocation-model=pic    | llvm-readobj -r | FileCheck %s --check-prefix=CHECK-PIC
 
-;CHECK-ABS: Relocations [
-;CHECK-ABS:    0x{{[0-9,A-F]+}} R_SPARC_H44 AGlobalVar 0x0
-;CHECK-ABS:    0x{{[0-9,A-F]+}} R_SPARC_M44 AGlobalVar 0x0
-;CHECK-ABS:    0x{{[0-9,A-F]+}} R_SPARC_L44 AGlobalVar 0x0
-;CHECK-ABS:    0x{{[0-9,A-F]+}} R_SPARC_WDISP30 bar 0x0
-;CHECK-ABS:]
+;CHECK-ABS:      Relocations [
+;CHECK-ABS:         0x{{[0-9,A-F]+}} R_SPARC_H44 AGlobalVar 0x0
+;CHECK-ABS-NEXT:    0x{{[0-9,A-F]+}} R_SPARC_M44 AGlobalVar 0x0
+;CHECK-ABS-NEXT:    0x{{[0-9,A-F]+}} R_SPARC_L44 AGlobalVar 0x0
+;CHECK-ABS-NEXT:    0x{{[0-9,A-F]+}} R_SPARC_H44 .rodata.str1.1 0x0
+;CHECK-ABS-NEXT:    0x{{[0-9,A-F]+}} R_SPARC_M44 .rodata.str1.1 0x0
+;CHECK-ABS-NEXT:    0x{{[0-9,A-F]+}} R_SPARC_WDISP30 bar 0x0
+;CHECK-ABS-NEXT:    0x{{[0-9,A-F]+}} R_SPARC_L44 .rodata.str1.1 0x0
+;CHECK-ABS:      ]
 
-; CHECK-PIC: Relocations [
-; CHECK-PIC:    0x{{[0-9,A-F]+}} R_SPARC_PC22 _GLOBAL_OFFSET_TABLE_ 0x4
-; CHECK-PIC:    0x{{[0-9,A-F]+}} R_SPARC_PC10 _GLOBAL_OFFSET_TABLE_ 0x8
-; CHECK-PIC:    0x{{[0-9,A-F]+}} R_SPARC_GOT22 AGlobalVar 0x0
-; CHECK-PIC:    0x{{[0-9,A-F]+}} R_SPARC_GOT10 AGlobalVar 0x0
-; CHECK-PIC:    0x{{[0-9,A-F]+}} R_SPARC_WPLT30 bar 0x0
-; CHECK-PIC: ]
+; CHECK-PIC:      Relocations [
+; CHECK-PIC:         0x{{[0-9,A-F]+}} R_SPARC_PC22 _GLOBAL_OFFSET_TABLE_ 0x4
+; CHECK-PIC-NEXT:    0x{{[0-9,A-F]+}} R_SPARC_PC10 _GLOBAL_OFFSET_TABLE_ 0x8
+; CHECK-PIC-NEXT:    0x{{[0-9,A-F]+}} R_SPARC_GOT22 AGlobalVar 0x0
+; CHECK-PIC-NEXT:    0x{{[0-9,A-F]+}} R_SPARC_GOT10 AGlobalVar 0x0
+; CHECK-PIC-NEXT:    0x{{[0-9,A-F]+}} R_SPARC_GOT22 .L.mystr 0x0
+; CHECK-PIC-NEXT:    0x{{[0-9,A-F]+}} R_SPARC_GOT10 .L.mystr 0x0
+; CHECK-PIC-NEXT:    0x{{[0-9,A-F]+}} R_SPARC_WPLT30 bar 0x0
+; CHECK-PIC:      ]
 
 
 @AGlobalVar = global i64 0, align 8
+@.mystr = private unnamed_addr constant [6 x i8] c"hello\00", align 1
 
 define i64 @foo(i64 %a) {
 entry:
   %0 = load i64, i64* @AGlobalVar, align 4
   %1 = add i64 %a, %0
-  %2 = call i64 @bar(i64 %1)
+  %2 = call i64 @bar(i8* getelementptr inbounds ([6 x i8], [6 x i8]* @.mystr, i32 0, i32 0), i64 %1)
   ret i64 %2
 }
 
 
-declare i64 @bar(i64)
+declare i64 @bar(i8*, i64)
diff --git a/test/CodeGen/SystemZ/fp-abs-01.ll b/test/CodeGen/SystemZ/fp-abs-01.ll
index d14a92acae8..3b143d93315 100644
--- a/test/CodeGen/SystemZ/fp-abs-01.ll
+++ b/test/CodeGen/SystemZ/fp-abs-01.ll
@@ -1,6 +1,7 @@
 ; Test floating-point absolute.
 ;
-; RUN: llc < %s -mtriple=s390x-linux-gnu | FileCheck %s
+; RUN: llc < %s -mtriple=s390x-linux-gnu -mcpu=z10 | FileCheck %s
+; RUN: llc < %s -mtriple=s390x-linux-gnu -mcpu=z13 | FileCheck %s
 
 ; Test f32.
 declare float @llvm.fabs.f32(float %f)
diff --git a/test/CodeGen/SystemZ/fp-abs-02.ll b/test/CodeGen/SystemZ/fp-abs-02.ll
index deec8c32b4a..e831ddb86fe 100644
--- a/test/CodeGen/SystemZ/fp-abs-02.ll
+++ b/test/CodeGen/SystemZ/fp-abs-02.ll
@@ -1,6 +1,7 @@
 ; Test negated floating-point absolute.
 ;
-; RUN: llc < %s -mtriple=s390x-linux-gnu | FileCheck %s
+; RUN: llc < %s -mtriple=s390x-linux-gnu -mcpu=z10 | FileCheck %s
+; RUN: llc < %s -mtriple=s390x-linux-gnu -mcpu=z13 | FileCheck %s
 
 ; Test f32.
 declare float @llvm.fabs.f32(float %f)
diff --git a/test/CodeGen/SystemZ/fp-add-02.ll b/test/CodeGen/SystemZ/fp-add-02.ll
index 07c7462020f..5be1ad79d45 100644
--- a/test/CodeGen/SystemZ/fp-add-02.ll
+++ b/test/CodeGen/SystemZ/fp-add-02.ll
@@ -1,7 +1,8 @@
 ; Test 64-bit floating-point addition.
 ;
-; RUN: llc < %s -mtriple=s390x-linux-gnu | FileCheck %s
-
+; RUN: llc < %s -mtriple=s390x-linux-gnu -mcpu=z10 \
+; RUN:   | FileCheck -check-prefix=CHECK -check-prefix=CHECK-SCALAR %s
+; RUN: llc < %s -mtriple=s390x-linux-gnu -mcpu=z13 | FileCheck %s
 declare double @foo()
 
 ; Check register addition.
@@ -76,7 +77,7 @@ define double @f6(double %f1, double *%base, i64 %index) {
 define double @f7(double *%ptr0) {
 ; CHECK-LABEL: f7:
 ; CHECK: brasl %r14, foo@PLT
-; CHECK: adb %f0, 160(%r15)
+; CHECK-SCALAR: adb %f0, 160(%r15)
 ; CHECK: br %r14
   %ptr1 = getelementptr double, double *%ptr0, i64 2
   %ptr2 = getelementptr double, double *%ptr0, i64 4
diff --git a/test/CodeGen/SystemZ/fp-cmp-02.ll b/test/CodeGen/SystemZ/fp-cmp-02.ll
index 95af309e795..94a256777c7 100644
--- a/test/CodeGen/SystemZ/fp-cmp-02.ll
+++ b/test/CodeGen/SystemZ/fp-cmp-02.ll
@@ -1,7 +1,10 @@
 ; Test 64-bit floating-point comparison.  The tests assume a z10 implementation
 ; of select, using conditional branches rather than LOCGR.
 ;
-; RUN: llc < %s -mtriple=s390x-linux-gnu -mcpu=z10 | FileCheck %s
+; RUN: llc < %s -mtriple=s390x-linux-gnu -mcpu=z10 \
+; RUN:   | FileCheck -check-prefix=CHECK -check-prefix=CHECK-SCALAR %s
+; RUN: llc < %s -mtriple=s390x-linux-gnu -mcpu=z13 \
+; RUN:   | FileCheck -check-prefix=CHECK -check-prefix=CHECK-VECTOR %s
 
 declare double @foo()
 
@@ -9,8 +12,9 @@ declare double @foo()
 define i64 @f1(i64 %a, i64 %b, double %f1, double %f2) {
 ; CHECK-LABEL: f1:
 ; CHECK: cdbr %f0, %f2
-; CHECK-NEXT: je
-; CHECK: lgr %r2, %r3
+; CHECK-SCALAR-NEXT: je
+; CHECK-SCALAR: lgr %r2, %r3
+; CHECK-VECTOR-NEXT: locgrne %r2, %r3
 ; CHECK: br %r14
   %cond = fcmp oeq double %f1, %f2
   %res = select i1 %cond, i64 %a, i64 %b
@@ -21,8 +25,9 @@ define i64 @f1(i64 %a, i64 %b, double %f1, double %f2) {
 define i64 @f2(i64 %a, i64 %b, double %f1, double *%ptr) {
 ; CHECK-LABEL: f2:
 ; CHECK: cdb %f0, 0(%r4)
-; CHECK-NEXT: je
-; CHECK: lgr %r2, %r3
+; CHECK-SCALAR-NEXT: je
+; CHECK-SCALAR: lgr %r2, %r3
+; CHECK-VECTOR-NEXT: locgrne %r2, %r3
 ; CHECK: br %r14
   %f2 = load double , double *%ptr
   %cond = fcmp oeq double %f1, %f2
@@ -34,8 +39,9 @@ define i64 @f2(i64 %a, i64 %b, double %f1, double *%ptr) {
 define i64 @f3(i64 %a, i64 %b, double %f1, double *%base) {
 ; CHECK-LABEL: f3:
 ; CHECK: cdb %f0, 4088(%r4)
-; CHECK-NEXT: je
-; CHECK: lgr %r2, %r3
+; CHECK-SCALAR-NEXT: je
+; CHECK-SCALAR: lgr %r2, %r3
+; CHECK-VECTOR-NEXT: locgrne %r2, %r3
 ; CHECK: br %r14
   %ptr = getelementptr double, double *%base, i64 511
   %f2 = load double , double *%ptr
@@ -50,8 +56,9 @@ define i64 @f4(i64 %a, i64 %b, double %f1, double *%base) {
 ; CHECK-LABEL: f4:
 ; CHECK: aghi %r4, 4096
 ; CHECK: cdb %f0, 0(%r4)
-; CHECK-NEXT: je
-; CHECK: lgr %r2, %r3
+; CHECK-SCALAR-NEXT: je
+; CHECK-SCALAR: lgr %r2, %r3
+; CHECK-VECTOR-NEXT: locgrne %r2, %r3
 ; CHECK: br %r14
   %ptr = getelementptr double, double *%base, i64 512
   %f2 = load double , double *%ptr
@@ -65,8 +72,9 @@ define i64 @f5(i64 %a, i64 %b, double %f1, double *%base) {
 ; CHECK-LABEL: f5:
 ; CHECK: aghi %r4, -8
 ; CHECK: cdb %f0, 0(%r4)
-; CHECK-NEXT: je
-; CHECK: lgr %r2, %r3
+; CHECK-SCALAR-NEXT: je
+; CHECK-SCALAR: lgr %r2, %r3
+; CHECK-VECTOR-NEXT: locgrne %r2, %r3
 ; CHECK: br %r14
   %ptr = getelementptr double, double *%base, i64 -1
   %f2 = load double , double *%ptr
@@ -80,8 +88,9 @@ define i64 @f6(i64 %a, i64 %b, double %f1, double *%base, i64 %index) {
 ; CHECK-LABEL: f6:
 ; CHECK: sllg %r1, %r5, 3
 ; CHECK: cdb %f0, 800(%r1,%r4)
-; CHECK-NEXT: je
-; CHECK: lgr %r2, %r3
+; CHECK-SCALAR-NEXT: je
+; CHECK-SCALAR: lgr %r2, %r3
+; CHECK-VECTOR-NEXT: locgrne %r2, %r3
 ; CHECK: br %r14
   %ptr1 = getelementptr double, double *%base, i64 %index
   %ptr2 = getelementptr double, double *%ptr1, i64 100
@@ -95,7 +104,7 @@ define i64 @f6(i64 %a, i64 %b, double %f1, double *%base, i64 %index) {
 define double @f7(double *%ptr0) {
 ; CHECK-LABEL: f7:
 ; CHECK: brasl %r14, foo@PLT
-; CHECK: cdb {{%f[0-9]+}}, 160(%r15)
+; CHECK-SCALAR: cdb {{%f[0-9]+}}, 160(%r15)
 ; CHECK: br %r14
   %ptr1 = getelementptr double, double *%ptr0, i64 2
   %ptr2 = getelementptr double, double *%ptr0, i64 4
@@ -152,9 +161,12 @@ define double @f7(double *%ptr0) {
 ; Check comparison with zero.
 define i64 @f8(i64 %a, i64 %b, double %f) {
 ; CHECK-LABEL: f8:
-; CHECK: ltdbr %f0, %f0
-; CHECK-NEXT: je
-; CHECK: lgr %r2, %r3
+; CHECK-SCALAR: ltdbr %f0, %f0
+; CHECK-SCALAR-NEXT: je
+; CHECK-SCALAR: lgr %r2, %r3
+; CHECK-VECTOR: lzdr %f1
+; CHECK-VECTOR-NEXT: cdbr %f0, %f1
+; CHECK-VECTOR-NEXT: locgrne %r2, %r3
 ; CHECK: br %r14
   %cond = fcmp oeq double %f, 0.0
   %res = select i1 %cond, i64 %a, i64 %b
@@ -165,8 +177,9 @@ define i64 @f8(i64 %a, i64 %b, double %f) {
 define i64 @f9(i64 %a, i64 %b, double %f2, double *%ptr) {
 ; CHECK-LABEL: f9:
 ; CHECK: cdb %f0, 0(%r4)
-; CHECK-NEXT: jl {{\.L.*}}
-; CHECK: lgr %r2, %r3
+; CHECK-SCALAR-NEXT: jl
+; CHECK-SCALAR: lgr %r2, %r3
+; CHECK-VECTOR-NEXT: locgrnl %r2, %r3
 ; CHECK: br %r14
   %f1 = load double , double *%ptr
   %cond = fcmp ogt double %f1, %f2
diff --git a/test/CodeGen/SystemZ/fp-conv-01.ll b/test/CodeGen/SystemZ/fp-conv-01.ll
index ebc174afada..06740ed4b4a 100644
--- a/test/CodeGen/SystemZ/fp-conv-01.ll
+++ b/test/CodeGen/SystemZ/fp-conv-01.ll
@@ -1,11 +1,15 @@
 ; Test floating-point truncations.
 ;
-; RUN: llc < %s -mtriple=s390x-linux-gnu | FileCheck %s
+; RUN: llc < %s -mtriple=s390x-linux-gnu -mcpu=z10 \
+; RUN:   | FileCheck -check-prefix=CHECK -check-prefix=CHECK-SCALAR %s
+; RUN: llc < %s -mtriple=s390x-linux-gnu -mcpu=z13 \
+; RUN:   | FileCheck -check-prefix=CHECK -check-prefix=CHECK-VECTOR %s
 
 ; Test f64->f32.
 define float @f1(double %d1, double %d2) {
 ; CHECK-LABEL: f1:
-; CHECK: ledbr %f0, %f2
+; CHECK-SCALAR: ledbr %f0, %f2
+; CHECK-VECTOR: ledbra %f0, 0, %f2, 0
 ; CHECK: br %r14
   %res = fptrunc double %d2 to float
   ret float %res
@@ -50,8 +54,10 @@ define double @f4(fp128 *%ptr) {
 define void @f5(double *%dst, fp128 *%ptr, double %d1, double %d2) {
 ; CHECK-LABEL: f5:
 ; CHECK: ldxbr %f1, %f1
-; CHECK: adbr %f1, %f2
-; CHECK: std %f1, 0(%r2)
+; CHECK-SCALAR: adbr %f1, %f2
+; CHECK-SCALAR: std %f1, 0(%r2)
+; CHECK-VECTOR: wfadb [[REG:%f[0-9]+]], %f1, %f2
+; CHECK-VECTOR: std [[REG]], 0(%r2)
 ; CHECK: br %r14
   %val = load fp128 , fp128 *%ptr
   %conv = fptrunc fp128 %val to double
diff --git a/test/CodeGen/SystemZ/fp-conv-02.ll b/test/CodeGen/SystemZ/fp-conv-02.ll
index e9376ba6973..be32bfe7ba9 100644
--- a/test/CodeGen/SystemZ/fp-conv-02.ll
+++ b/test/CodeGen/SystemZ/fp-conv-02.ll
@@ -1,6 +1,8 @@
 ; Test extensions of f32 to f64.
 ;
-; RUN: llc < %s -mtriple=s390x-linux-gnu | FileCheck %s
+; RUN: llc < %s -mtriple=s390x-linux-gnu -mcpu=z10 \
+; RUN:   | FileCheck -check-prefix=CHECK -check-prefix=CHECK-SCALAR %s
+; RUN: llc < %s -mtriple=s390x-linux-gnu -mcpu=z13 | FileCheck %s
 
 ; Check register extension.
 define double @f1(float %val) {
@@ -74,7 +76,7 @@ define double @f6(float *%base, i64 %index) {
 ; to use LDEB if possible.
 define void @f7(double *%ptr1, float *%ptr2) {
 ; CHECK-LABEL: f7:
-; CHECK: ldeb {{%f[0-9]+}}, 16{{[04]}}(%r15)
+; CHECK-SCALAR: ldeb {{%f[0-9]+}}, 16{{[04]}}(%r15)
 ; CHECK: br %r14
   %val0 = load volatile float , float *%ptr2
   %val1 = load volatile float , float *%ptr2
diff --git a/test/CodeGen/SystemZ/fp-div-02.ll b/test/CodeGen/SystemZ/fp-div-02.ll
index 82eeb480602..f120e7c923d 100644
--- a/test/CodeGen/SystemZ/fp-div-02.ll
+++ b/test/CodeGen/SystemZ/fp-div-02.ll
@@ -1,6 +1,8 @@
 ; Test 64-bit floating-point division.
 ;
-; RUN: llc < %s -mtriple=s390x-linux-gnu | FileCheck %s
+; RUN: llc < %s -mtriple=s390x-linux-gnu -mcpu=z10 \
+; RUN:   | FileCheck -check-prefix=CHECK -check-prefix=CHECK-SCALAR %s
+; RUN: llc < %s -mtriple=s390x-linux-gnu -mcpu=z13 | FileCheck %s
 
 declare double @foo()
 
@@ -76,7 +78,7 @@ define double @f6(double %f1, double *%base, i64 %index) {
 define double @f7(double *%ptr0) {
 ; CHECK-LABEL: f7:
 ; CHECK: brasl %r14, foo@PLT
-; CHECK: ddb %f0, 160(%r15)
+; CHECK-SCALAR: ddb %f0, 160(%r15)
 ; CHECK: br %r14
   %ptr1 = getelementptr double, double *%ptr0, i64 2
   %ptr2 = getelementptr double, double *%ptr0, i64 4
diff --git a/test/CodeGen/SystemZ/fp-move-01.ll b/test/CodeGen/SystemZ/fp-move-01.ll
index 31a8fc55d77..843b1b6a6e6 100644
--- a/test/CodeGen/SystemZ/fp-move-01.ll
+++ b/test/CodeGen/SystemZ/fp-move-01.ll
@@ -1,11 +1,13 @@
 ; Test moves between FPRs.
 ;
-; RUN: llc < %s -mtriple=s390x-linux-gnu | FileCheck %s
+; RUN: llc < %s -mtriple=s390x-linux-gnu -mcpu=z10 | FileCheck %s
+; RUN: llc < %s -mtriple=s390x-linux-gnu -mcpu=z13 | FileCheck %s
 
 ; Test f32 moves.
 define float @f1(float %a, float %b) {
 ; CHECK-LABEL: f1:
 ; CHECK: ler %f0, %f2
+; CHECK: br %r14
   ret float %b
 }
 
@@ -13,6 +15,7 @@ define float @f1(float %a, float %b) {
 define double @f2(double %a, double %b) {
 ; CHECK-LABEL: f2:
 ; CHECK: ldr %f0, %f2
+; CHECK: br %r14
   ret double %b
 }
 
@@ -22,6 +25,7 @@ define void @f3(fp128 *%x) {
 ; CHECK-LABEL: f3:
 ; CHECK: lxr
 ; CHECK: axbr
+; CHECK: br %r14
   %val = load volatile fp128 , fp128 *%x
   %sum = fadd fp128 %val, %val
   store volatile fp128 %sum, fp128 *%x
diff --git a/test/CodeGen/SystemZ/fp-move-04.ll b/test/CodeGen/SystemZ/fp-move-04.ll
index d3728d0e585..6650419b2c3 100644
--- a/test/CodeGen/SystemZ/fp-move-04.ll
+++ b/test/CodeGen/SystemZ/fp-move-04.ll
@@ -1,6 +1,7 @@
 ; Test 64-bit floating-point loads.
 ;
-; RUN: llc < %s -mtriple=s390x-linux-gnu | FileCheck %s
+; RUN: llc < %s -mtriple=s390x-linux-gnu -mcpu=z10 | FileCheck %s
+; RUN: llc < %s -mtriple=s390x-linux-gnu -mcpu=z13 | FileCheck %s
 
 ; Test the low end of the LD range.
 define double @f1(double *%src) {
diff --git a/test/CodeGen/SystemZ/fp-move-07.ll b/test/CodeGen/SystemZ/fp-move-07.ll
index c3ad2a59f66..5361002a97e 100644
--- a/test/CodeGen/SystemZ/fp-move-07.ll
+++ b/test/CodeGen/SystemZ/fp-move-07.ll
@@ -1,6 +1,7 @@
 ; Test 64-bit floating-point stores.
 ;
-; RUN: llc < %s -mtriple=s390x-linux-gnu | FileCheck %s
+; RUN: llc < %s -mtriple=s390x-linux-gnu -mcpu=z10 | FileCheck %s
+; RUN: llc < %s -mtriple=s390x-linux-gnu -mcpu=z13 | FileCheck %s
 
 ; Test the low end of the STD range.
 define void @f1(double *%src, double %val) {
diff --git a/test/CodeGen/SystemZ/fp-move-09.ll b/test/CodeGen/SystemZ/fp-move-09.ll
index e4a3e9222d5..5e8dce272c2 100644
--- a/test/CodeGen/SystemZ/fp-move-09.ll
+++ b/test/CodeGen/SystemZ/fp-move-09.ll
@@ -1,4 +1,4 @@
-; Test moves between FPRs and GPRs for z196 and above.
+; Test moves between FPRs and GPRs for z196 and zEC12.
 ;
 ; RUN: llc < %s -mtriple=s390x-linux-gnu -mcpu=z196 | FileCheck %s
 
diff --git a/test/CodeGen/SystemZ/fp-move-10.ll b/test/CodeGen/SystemZ/fp-move-10.ll
new file mode 100644
index 00000000000..602397d58a8
--- /dev/null
+++ b/test/CodeGen/SystemZ/fp-move-10.ll
@@ -0,0 +1,61 @@
+; Test moves between FPRs and GPRs for z13 and above.
+;
+; RUN: llc < %s -mtriple=s390x-linux-gnu -mcpu=z13 | FileCheck %s
+
+; Check that moves from i32s to floats use a low GR32 and vector operation.
+define float @f1(i16 *%ptr) {
+; CHECK-LABEL: f1:
+; CHECK: llh [[REG:%r[0-5]]], 0(%r2)
+; CHECK: oilh [[REG]], 16256
+; CHECK: vlvgf %v0, [[REG]], 0
+; CHECK: br %r14
+  %base = load i16, i16 *%ptr
+  %ext = zext i16 %base to i32
+  %full = or i32 %ext, 1065353216
+  %res = bitcast i32 %full to float
+  ret float %res
+}
+
+; Check that moves from floats to i32s use a low GR32 and vector operation.
+define void @f2(float %val, i8 *%ptr) {
+; CHECK-LABEL: f2:
+; CHECK: vlgvf [[REG:%r[0-5]]], %v0, 0
+; CHECK: stc [[REG]], 0(%r2)
+; CHECK: br %r14
+  %res = bitcast float %val to i32
+  %trunc = trunc i32 %res to i8
+  store i8 %trunc, i8 *%ptr
+  ret void
+}
+
+; Like f2, but with a conditional store.
+define void @f3(float %val, i8 *%ptr, i32 %which) {
+; CHECK-LABEL: f3:
+; CHECK-DAG: cijlh %r3, 0,
+; CHECK-DAG: vlgvf [[REG:%r[0-5]]], %v0, 0
+; CHECK: stc [[REG]], 0(%r2)
+; CHECK: br %r14
+  %int = bitcast float %val to i32
+  %trunc = trunc i32 %int to i8
+  %old = load i8, i8 *%ptr
+  %cmp = icmp eq i32 %which, 0
+  %res = select i1 %cmp, i8 %trunc, i8 %old
+  store i8 %res, i8 *%ptr
+  ret void
+}
+
+; ...and again with 16-bit memory.
+define void @f4(float %val, i16 *%ptr, i32 %which) {
+; CHECK-LABEL: f4:
+; CHECK-DAG: cijlh %r3, 0,
+; CHECK-DAG: vlgvf [[REG:%r[0-5]]], %v0, 0
+; CHECK: sth [[REG]], 0(%r2)
+; CHECK: br %r14
+  %int = bitcast float %val to i32
+  %trunc = trunc i32 %int to i16
+  %old = load i16, i16 *%ptr
+  %cmp = icmp eq i32 %which, 0
+  %res = select i1 %cmp, i16 %trunc, i16 %old
+  store i16 %res, i16 *%ptr
+  ret void
+}
diff --git a/test/CodeGen/SystemZ/fp-move-11.ll b/test/CodeGen/SystemZ/fp-move-11.ll
new file mode 100644
index 00000000000..ce45019425c
--- /dev/null
+++ b/test/CodeGen/SystemZ/fp-move-11.ll
@@ -0,0 +1,110 @@
+; Test 32-bit floating-point loads for z13.
+;
+; RUN: llc < %s -mtriple=s390x-linux-gnu -mcpu=z13 | FileCheck %s
+
+; Test that we use LDE instead of LE - low end of the LE range.
+define float @f1(float *%src) {
+; CHECK-LABEL: f1:
+; CHECK: lde %f0, 0(%r2)
+; CHECK: br %r14
+  %val = load float, float *%src
+  ret float %val
+}
+
+; Test that we use LDE instead of LE - high end of the LE range.
+define float @f2(float *%src) {
+; CHECK-LABEL: f2:
+; CHECK: lde %f0, 4092(%r2)
+; CHECK: br %r14
+  %ptr = getelementptr float, float *%src, i64 1023
+  %val = load float, float *%ptr
+  ret float %val
+}
+
+; Check the next word up, which should use LEY instead of LDE.
+define float @f3(float *%src) {
+; CHECK-LABEL: f3:
+; CHECK: ley %f0, 4096(%r2)
+; CHECK: br %r14
+  %ptr = getelementptr float, float *%src, i64 1024
+  %val = load float, float *%ptr
+  ret float %val
+}
+
+; Check the high end of the aligned LEY range.
+define float @f4(float *%src) {
+; CHECK-LABEL: f4:
+; CHECK: ley %f0, 524284(%r2)
+; CHECK: br %r14
+  %ptr = getelementptr float, float *%src, i64 131071
+  %val = load float, float *%ptr
+  ret float %val
+}
+
+; Check the next word up, which needs separate address logic.
+; Other sequences besides this one would be OK.
+define float @f5(float *%src) {
+; CHECK-LABEL: f5:
+; CHECK: agfi %r2, 524288
+; CHECK: lde %f0, 0(%r2)
+; CHECK: br %r14
+  %ptr = getelementptr float, float *%src, i64 131072
+  %val = load float, float *%ptr
+  ret float %val
+}
+
+; Check the high end of the negative aligned LEY range.
+define float @f6(float *%src) {
+; CHECK-LABEL: f6:
+; CHECK: ley %f0, -4(%r2)
+; CHECK: br %r14
+  %ptr = getelementptr float, float *%src, i64 -1
+  %val = load float, float *%ptr
+  ret float %val
+}
+
+; Check the low end of the LEY range.
+define float @f7(float *%src) {
+; CHECK-LABEL: f7:
+; CHECK: ley %f0, -524288(%r2)
+; CHECK: br %r14
+  %ptr = getelementptr float, float *%src, i64 -131072
+  %val = load float, float *%ptr
+  ret float %val
+}
+
+; Check the next word down, which needs separate address logic.
+; Other sequences besides this one would be OK.
+define float @f8(float *%src) {
+; CHECK-LABEL: f8:
+; CHECK: agfi %r2, -524292
+; CHECK: lde %f0, 0(%r2)
+; CHECK: br %r14
+  %ptr = getelementptr float, float *%src, i64 -131073
+  %val = load float, float *%ptr
+  ret float %val
+}
+
+; Check that LDE allows an index.
+define float @f9(i64 %src, i64 %index) {
+; CHECK-LABEL: f9:
+; CHECK: lde %f0, 4092({{%r3,%r2|%r2,%r3}})
+; CHECK: br %r14
+  %add1 = add i64 %src, %index
+  %add2 = add i64 %add1, 4092
+  %ptr = inttoptr i64 %add2 to float *
+  %val = load float, float *%ptr
+  ret float %val
+}
+
+; Check that LEY allows an index.
+define float @f10(i64 %src, i64 %index) {
+; CHECK-LABEL: f10:
+; CHECK: ley %f0, 4096({{%r3,%r2|%r2,%r3}})
+; CHECK: br %r14
+  %add1 = add i64 %src, %index
+  %add2 = add i64 %add1, 4096
+  %ptr = inttoptr i64 %add2 to float *
+  %val = load float, float *%ptr
+  ret float %val
+}
diff --git a/test/CodeGen/SystemZ/fp-mul-03.ll b/test/CodeGen/SystemZ/fp-mul-03.ll
index 701304ef3ee..0d52121f41c 100644
--- a/test/CodeGen/SystemZ/fp-mul-03.ll
+++ b/test/CodeGen/SystemZ/fp-mul-03.ll
@@ -1,6 +1,8 @@
 ; Test multiplication of two f64s, producing an f64 result.
 ;
-; RUN: llc < %s -mtriple=s390x-linux-gnu | FileCheck %s
+; RUN: llc < %s -mtriple=s390x-linux-gnu -mcpu=z10 \
+; RUN:   | FileCheck -check-prefix=CHECK -check-prefix=CHECK-SCALAR %s
+; RUN: llc < %s -mtriple=s390x-linux-gnu -mcpu=z13 | FileCheck %s
 
 declare double @foo()
 
@@ -76,7 +78,7 @@ define double @f6(double %f1, double *%base, i64 %index) {
 define double @f7(double *%ptr0) {
 ; CHECK-LABEL: f7:
 ; CHECK: brasl %r14, foo@PLT
-; CHECK: mdb %f0, 160(%r15)
+; CHECK-SCALAR: mdb %f0, 160(%r15)
 ; CHECK: br %r14
   %ptr1 = getelementptr double, double *%ptr0, i64 2
   %ptr2 = getelementptr double, double *%ptr0, i64 4
diff --git a/test/CodeGen/SystemZ/fp-mul-07.ll b/test/CodeGen/SystemZ/fp-mul-07.ll
index b1d0ae3c520..e0b4a5c5d78 100644
--- a/test/CodeGen/SystemZ/fp-mul-07.ll
+++ b/test/CodeGen/SystemZ/fp-mul-07.ll
@@ -1,11 +1,15 @@
-; RUN: llc < %s -mtriple=s390x-linux-gnu | FileCheck %s
+; RUN: llc < %s -mtriple=s390x-linux-gnu -mcpu=z10 \
+; RUN:   | FileCheck -check-prefix=CHECK -check-prefix=CHECK-SCALAR %s
+; RUN: llc < %s -mtriple=s390x-linux-gnu -mcpu=z13 \
+; RUN:   | FileCheck -check-prefix=CHECK -check-prefix=CHECK-VECTOR %s
 
 declare double @llvm.fma.f64(double %f1, double %f2, double %f3)
 
 define double @f1(double %f1, double %f2, double %acc) {
 ; CHECK-LABEL: f1:
-; CHECK: madbr %f4, %f0, %f2
-; CHECK: ldr %f0, %f4
+; CHECK-SCALAR: madbr %f4, %f0, %f2
+; CHECK-SCALAR: ldr %f0, %f4
+; CHECK-VECTOR: wfmadb %f0, %f0, %f2, %f4
 ; CHECK: br %r14
   %res = call double @llvm.fma.f64 (double %f1, double %f2, double %acc)
   ret double %res
diff --git a/test/CodeGen/SystemZ/fp-mul-09.ll b/test/CodeGen/SystemZ/fp-mul-09.ll
index f2eadf55ff3..927a8064823 100644
--- a/test/CodeGen/SystemZ/fp-mul-09.ll
+++ b/test/CodeGen/SystemZ/fp-mul-09.ll
@@ -1,11 +1,15 @@
-; RUN: llc < %s -mtriple=s390x-linux-gnu | FileCheck %s
+; RUN: llc < %s -mtriple=s390x-linux-gnu -mcpu=z10 \
+; RUN:   | FileCheck -check-prefix=CHECK -check-prefix=CHECK-SCALAR %s
+; RUN: llc < %s -mtriple=s390x-linux-gnu -mcpu=z13 \
+; RUN:   | FileCheck -check-prefix=CHECK -check-prefix=CHECK-VECTOR %s
 
 declare double @llvm.fma.f64(double %f1, double %f2, double %f3)
 
 define double @f1(double %f1, double %f2, double %acc) {
 ; CHECK-LABEL: f1:
-; CHECK: msdbr %f4, %f0, %f2
-; CHECK: ldr %f0, %f4
+; CHECK-SCALAR: msdbr %f4, %f0, %f2
+; CHECK-SCALAR: ldr %f0, %f4
+; CHECK-VECTOR: wfmsdb %f0, %f0, %f2, %f4
 ; CHECK: br %r14
   %negacc = fsub double -0.0, %acc
   %res = call double @llvm.fma.f64 (double %f1, double %f2, double %negacc)
diff --git a/test/CodeGen/SystemZ/fp-neg-01.ll b/test/CodeGen/SystemZ/fp-neg-01.ll
index 927bcd44d02..fe2e5f67cf5 100644
--- a/test/CodeGen/SystemZ/fp-neg-01.ll
+++ b/test/CodeGen/SystemZ/fp-neg-01.ll
@@ -1,6 +1,7 @@
 ; Test floating-point negation.
 ;
-; RUN: llc < %s -mtriple=s390x-linux-gnu | FileCheck %s
+; RUN: llc < %s -mtriple=s390x-linux-gnu -mcpu=z10 | FileCheck %s
+; RUN: llc < %s -mtriple=s390x-linux-gnu -mcpu=z13 | FileCheck %s
 
 ; Test f32.
 define float @f1(float %f) {
diff --git a/test/CodeGen/SystemZ/fp-round-02.ll b/test/CodeGen/SystemZ/fp-round-02.ll
index bd5419dad1d..428261478dc 100644
--- a/test/CodeGen/SystemZ/fp-round-02.ll
+++ b/test/CodeGen/SystemZ/fp-round-02.ll
@@ -1,6 +1,9 @@
 ; Test rounding functions for z196 and above.
 ;
-; RUN: llc < %s -mtriple=s390x-linux-gnu -mcpu=z196 | FileCheck %s
+; RUN: llc < %s -mtriple=s390x-linux-gnu -mcpu=z196 \
+; RUN:   | FileCheck -check-prefix=CHECK -check-prefix=CHECK-SCALAR %s
+; RUN: llc < %s -mtriple=s390x-linux-gnu -mcpu=z13 \
+; RUN:   | FileCheck -check-prefix=CHECK -check-prefix=CHECK-VECTOR %s
 
 ; Test rint for f32.
 declare float @llvm.rint.f32(float %f)
@@ -16,7 +19,8 @@ define float @f1(float %f) {
 declare double @llvm.rint.f64(double %f)
 define double @f2(double %f) {
 ; CHECK-LABEL: f2:
-; CHECK: fidbr %f0, 0, %f0
+; CHECK-SCALAR: fidbr %f0, 0, %f0
+; CHECK-VECTOR: fidbra %f0, 0, %f0, 0
 ; CHECK: br %r14
   %res = call double @llvm.rint.f64(double %f)
   ret double %res
diff --git a/test/CodeGen/SystemZ/fp-sqrt-02.ll b/test/CodeGen/SystemZ/fp-sqrt-02.ll
index a6d987b0d76..a162466064e 100644
--- a/test/CodeGen/SystemZ/fp-sqrt-02.ll
+++ b/test/CodeGen/SystemZ/fp-sqrt-02.ll
@@ -1,6 +1,8 @@
 ; Test 64-bit square root.
 ;
-; RUN: llc < %s -mtriple=s390x-linux-gnu | FileCheck %s
+; RUN: llc < %s -mtriple=s390x-linux-gnu -mcpu=z10 \
+; RUN:   | FileCheck -check-prefix=CHECK -check-prefix=CHECK-SCALAR %s
+; RUN: llc < %s -mtriple=s390x-linux-gnu -mcpu=z13 | FileCheck %s
 
 declare double @llvm.sqrt.f64(double %f)
 declare double @sqrt(double)
@@ -77,7 +79,7 @@ define double @f6(double *%base, i64 %index) {
 ; to use SQDB if possible.
 define void @f7(double *%ptr) {
 ; CHECK-LABEL: f7:
-; CHECK: sqdb {{%f[0-9]+}}, 160(%r15)
+; CHECK-SCALAR: sqdb {{%f[0-9]+}}, 160(%r15)
 ; CHECK: br %r14
   %val0 = load volatile double , double *%ptr
   %val1 = load volatile double , double *%ptr
diff --git a/test/CodeGen/SystemZ/fp-sub-02.ll b/test/CodeGen/SystemZ/fp-sub-02.ll
index f59ec0a31d7..143baac23e1 100644
--- a/test/CodeGen/SystemZ/fp-sub-02.ll
+++ b/test/CodeGen/SystemZ/fp-sub-02.ll
@@ -1,6 +1,8 @@
 ; Test 64-bit floating-point subtraction.
 ;
-; RUN: llc < %s -mtriple=s390x-linux-gnu | FileCheck %s
+; RUN: llc < %s -mtriple=s390x-linux-gnu -mcpu=z10 \
+; RUN:   | FileCheck -check-prefix=CHECK -check-prefix=CHECK-SCALAR %s
+; RUN: llc < %s -mtriple=s390x-linux-gnu -mcpu=z13 | FileCheck %s
 
 declare double @foo()
 
@@ -76,7 +78,7 @@ define double @f6(double %f1, double *%base, i64 %index) {
 define double @f7(double *%ptr0) {
 ; CHECK-LABEL: f7:
 ; CHECK: brasl %r14, foo@PLT
-; CHECK: sdb %f0, 16{{[04]}}(%r15)
+; CHECK-SCALAR: sdb %f0, 16{{[04]}}(%r15)
 ; CHECK: br %r14
   %ptr1 = getelementptr double, double *%ptr0, i64 2
   %ptr2 = getelementptr double, double *%ptr0, i64 4
diff --git a/test/CodeGen/SystemZ/frame-03.ll b/test/CodeGen/SystemZ/frame-03.ll
index 029c6d6d37d..21b8fdb0d67 100644
--- a/test/CodeGen/SystemZ/frame-03.ll
+++ b/test/CodeGen/SystemZ/frame-03.ll
@@ -2,7 +2,7 @@
 ; uses a different register class, but the set of saved and restored
 ; registers should be the same.
 ;
-; RUN: llc < %s -mtriple=s390x-linux-gnu | FileCheck %s
+; RUN: llc < %s -mtriple=s390x-linux-gnu -mcpu=z10 | FileCheck %s
 
 ; This function should require all FPRs, but no other spill slots.
 ; We need to save and restore 8 of the 16 FPRs, so the frame size
diff --git a/test/CodeGen/SystemZ/frame-07.ll b/test/CodeGen/SystemZ/frame-07.ll
index 253bbc26c1f..dd810142962 100644
--- a/test/CodeGen/SystemZ/frame-07.ll
+++ b/test/CodeGen/SystemZ/frame-07.ll
@@ -1,7 +1,7 @@
 ; Test the saving and restoring of FPRs in large frames.
 ;
-; RUN: llc < %s -mtriple=s390x-linux-gnu | FileCheck -check-prefix=CHECK-NOFP %s
-; RUN: llc < %s -mtriple=s390x-linux-gnu -disable-fp-elim | FileCheck -check-prefix=CHECK-FP %s
+; RUN: llc < %s -mtriple=s390x-linux-gnu -mcpu=z10 | FileCheck -check-prefix=CHECK-NOFP %s
+; RUN: llc < %s -mtriple=s390x-linux-gnu -mcpu=z10 -disable-fp-elim | FileCheck -check-prefix=CHECK-FP %s
 
 ; Test a frame size that requires some FPRs to be saved and loaded using
 ; the 20-bit STDY and LDY while others can use the 12-bit STD and LD.
diff --git a/test/CodeGen/SystemZ/frame-17.ll b/test/CodeGen/SystemZ/frame-17.ll
index 485297a2b21..502e541bafc 100644
--- a/test/CodeGen/SystemZ/frame-17.ll
+++ b/test/CodeGen/SystemZ/frame-17.ll
@@ -1,6 +1,6 @@
 ; Test spilling of FPRs.
 ;
-; RUN: llc < %s -mtriple=s390x-linux-gnu | FileCheck %s
+; RUN: llc < %s -mtriple=s390x-linux-gnu -mcpu=z10 | FileCheck %s
 
 ; We need to save and restore 8 of the 16 FPRs and allocate an additional
 ; 4-byte spill slot, rounded to 8 bytes.  The frame size should be exactly
diff --git a/test/CodeGen/SystemZ/frame-19.ll b/test/CodeGen/SystemZ/frame-19.ll
new file mode 100644
index 00000000000..f6e327c3ae3
--- /dev/null
+++ b/test/CodeGen/SystemZ/frame-19.ll
@@ -0,0 +1,314 @@
+; Test spilling of vector registers.
+;
+; RUN: llc < %s -mtriple=s390x-linux-gnu -mcpu=z13 | FileCheck %s
+
+; We need to allocate a 16-byte spill slot and save the 8 call-saved FPRs.
+; The frame size should be exactly 160 + 16 + 8 * 8 = 240.
+define void @f1(<16 x i8> *%ptr) {
+; CHECK-LABEL: f1:
+; CHECK: aghi %r15, -240
+; CHECK-DAG: std %f8,
+; CHECK-DAG: std %f9,
+; CHECK-DAG: std %f10,
+; CHECK-DAG: std %f11,
+; CHECK-DAG: std %f12,
+; CHECK-DAG: std %f13,
+; CHECK-DAG: std %f14,
+; CHECK-DAG: std %f15,
+; CHECK: vst {{%v[0-9]+}}, 160(%r15)
+; CHECK: vl {{%v[0-9]+}}, 160(%r15)
+; CHECK-DAG: ld %f8,
+; CHECK-DAG: ld %f9,
+; CHECK-DAG: ld %f10,
+; CHECK-DAG: ld %f11,
+; CHECK-DAG: ld %f12,
+; CHECK-DAG: ld %f13,
+; CHECK-DAG: ld %f14,
+; CHECK-DAG: ld %f15,
+; CHECK: aghi %r15, 240
+; CHECK: br %r14
+  %v0 = load volatile <16 x i8>, <16 x i8> *%ptr
+  %v1 = load volatile <16 x i8>, <16 x i8> *%ptr
+  %v2 = load volatile <16 x i8>, <16 x i8> *%ptr
+  %v3 = load volatile <16 x i8>, <16 x i8> *%ptr
+  %v4 = load volatile <16 x i8>, <16 x i8> *%ptr
+  %v5 = load volatile <16 x i8>, <16 x i8> *%ptr
+  %v6 = load volatile <16 x i8>, <16 x i8> *%ptr
+  %v7 = load volatile <16 x i8>, <16 x i8> *%ptr
+  %v8 = load volatile <16 x i8>, <16 x i8> *%ptr
+  %v9 = load volatile <16 x i8>, <16 x i8> *%ptr
+  %v10 = load volatile <16 x i8>, <16 x i8> *%ptr
+  %v11 = load volatile <16 x i8>, <16 x i8> *%ptr
+  %v12 = load volatile <16 x i8>, <16 x i8> *%ptr
+  %v13 = load volatile <16 x i8>, <16 x i8> *%ptr
+  %v14 = load volatile <16 x i8>, <16 x i8> *%ptr
+  %v15 = load volatile <16 x i8>, <16 x i8> *%ptr
+  %v16 = load volatile <16 x i8>, <16 x i8> *%ptr
+  %v17 = load volatile <16 x i8>, <16 x i8> *%ptr
+  %v18 = load volatile <16 x i8>, <16 x i8> *%ptr
+  %v19 = load volatile <16 x i8>, <16 x i8> *%ptr
+  %v20 = load volatile <16 x i8>, <16 x i8> *%ptr
+  %v21 = load volatile <16 x i8>, <16 x i8> *%ptr
+  %v22 = load volatile <16 x i8>, <16 x i8> *%ptr
+  %v23 = load volatile <16 x i8>, <16 x i8> *%ptr
+  %v24 = load volatile <16 x i8>, <16 x i8> *%ptr
+  %v25 = load volatile <16 x i8>, <16 x i8> *%ptr
+  %v26 = load volatile <16 x i8>, <16 x i8> *%ptr
+  %v27 = load volatile <16 x i8>, <16 x i8> *%ptr
+  %v28 = load volatile <16 x i8>, <16 x i8> *%ptr
+  %v29 = load volatile <16 x i8>, <16 x i8> *%ptr
+  %v30 = load volatile <16 x i8>, <16 x i8> *%ptr
+  %v31 = load volatile <16 x i8>, <16 x i8> *%ptr
+  %vx = load volatile <16 x i8>, <16 x i8> *%ptr
+  store volatile <16 x i8> %vx, <16 x i8> *%ptr
+  store volatile <16 x i8> %v31, <16 x i8> *%ptr
+  store volatile <16 x i8> %v30, <16 x i8> *%ptr
+  store volatile <16 x i8> %v29, <16 x i8> *%ptr
+  store volatile <16 x i8> %v28, <16 x i8> *%ptr
+  store volatile <16 x i8> %v27, <16 x i8> *%ptr
+  store volatile <16 x i8> %v26, <16 x i8> *%ptr
+  store volatile <16 x i8> %v25, <16 x i8> *%ptr
+  store volatile <16 x i8> %v24, <16 x i8> *%ptr
+  store volatile <16 x i8> %v23, <16 x i8> *%ptr
+  store volatile <16 x i8> %v22, <16 x i8> *%ptr
+  store volatile <16 x i8> %v21, <16 x i8> *%ptr
+  store volatile <16 x i8> %v20, <16 x i8> *%ptr
+  store volatile <16 x i8> %v19, <16 x i8> *%ptr
+  store volatile <16 x i8> %v18, <16 x i8> *%ptr
+  store volatile <16 x i8> %v17, <16 x i8> *%ptr
+  store volatile <16 x i8> %v16, <16 x i8> *%ptr
+  store volatile <16 x i8> %v15, <16 x i8> *%ptr
+  store volatile <16 x i8> %v14, <16 x i8> *%ptr
+  store volatile <16 x i8> %v13, <16 x i8> *%ptr
+  store volatile <16 x i8> %v12, <16 x i8> *%ptr
+  store volatile <16 x i8> %v11, <16 x i8> *%ptr
+  store volatile <16 x i8> %v10, <16 x i8> *%ptr
+  store volatile <16 x i8> %v9, <16 x i8> *%ptr
+  store volatile <16 x i8> %v8, <16 x i8> *%ptr
+  store volatile <16 x i8> %v7, <16 x i8> *%ptr
+  store volatile <16 x i8> %v6, <16 x i8> *%ptr
+  store volatile <16 x i8> %v5, <16 x i8> *%ptr
+  store volatile <16 x i8> %v4, <16 x i8> *%ptr
+  store volatile <16 x i8> %v3, <16 x i8> *%ptr
+  store volatile <16 x i8> %v2, <16 x i8> *%ptr
+  store volatile <16 x i8> %v1, <16 x i8> *%ptr
+  store volatile <16 x i8> %v0, <16 x i8> *%ptr
+  ret void
+}
+
+; Like f1, but no 16-byte slot should be needed.
+define void @f2(<16 x i8> *%ptr) {
+; CHECK-LABEL: f2:
+; CHECK: aghi %r15, -224
+; CHECK-DAG: std %f8,
+; CHECK-DAG: std %f9,
+; CHECK-DAG: std %f10,
+; CHECK-DAG: std %f11,
+; CHECK-DAG: std %f12,
+; CHECK-DAG: std %f13,
+; CHECK-DAG: std %f14,
+; CHECK-DAG: std %f15,
+; CHECK-NOT: vst {{.*}}(%r15)
+; CHECK-NOT: vl {{.*}}(%r15)
+; CHECK-DAG: ld %f8,
+; CHECK-DAG: ld %f9,
+; CHECK-DAG: ld %f10,
+; CHECK-DAG: ld %f11,
+; CHECK-DAG: ld %f12,
+; CHECK-DAG: ld %f13,
+; CHECK-DAG: ld %f14,
+; CHECK-DAG: ld %f15,
+; CHECK: aghi %r15, 224
+; CHECK: br %r14
+  %v0 = load volatile <16 x i8>, <16 x i8> *%ptr
+  %v1 = load volatile <16 x i8>, <16 x i8> *%ptr
+  %v2 = load volatile <16 x i8>, <16 x i8> *%ptr
+  %v3 = load volatile <16 x i8>, <16 x i8> *%ptr
+  %v4 = load volatile <16 x i8>, <16 x i8> *%ptr
+  %v5 = load volatile <16 x i8>, <16 x i8> *%ptr
+  %v6 = load volatile <16 x i8>, <16 x i8> *%ptr
+  %v7 = load volatile <16 x i8>, <16 x i8> *%ptr
+  %v8 = load volatile <16 x i8>, <16 x i8> *%ptr
+  %v9 = load volatile <16 x i8>, <16 x i8> *%ptr
+  %v10 = load volatile <16 x i8>, <16 x i8> *%ptr
+  %v11 = load volatile <16 x i8>, <16 x i8> *%ptr
+  %v12 = load volatile <16 x i8>, <16 x i8> *%ptr
+  %v13 = load volatile <16 x i8>, <16 x i8> *%ptr
+  %v14 = load volatile <16 x i8>, <16 x i8> *%ptr
+  %v15 = load volatile <16 x i8>, <16 x i8> *%ptr
+  %v16 = load volatile <16 x i8>, <16 x i8> *%ptr
+  %v17 = load volatile <16 x i8>, <16 x i8> *%ptr
+  %v18 = load volatile <16 x i8>, <16 x i8> *%ptr
+  %v19 = load volatile <16 x i8>, <16 x i8> *%ptr
+  %v20 = load volatile <16 x i8>, <16 x i8> *%ptr
+  %v21 = load volatile <16 x i8>, <16 x i8> *%ptr
+  %v22 = load volatile <16 x i8>, <16 x i8> *%ptr
+  %v23 = load volatile <16 x i8>, <16 x i8> *%ptr
+  %v24 = load volatile <16 x i8>, <16 x i8> *%ptr
+  %v25 = load volatile <16 x i8>, <16 x i8> *%ptr
+  %v26 = load volatile <16 x i8>, <16 x i8> *%ptr
+  %v27 = load volatile <16 x i8>, <16 x i8> *%ptr
+  %v28 = load volatile <16 x i8>, <16 x i8> *%ptr
+  %v29 = load volatile <16 x i8>, <16 x i8> *%ptr
+  %v30 = load volatile <16 x i8>, <16 x i8> *%ptr
+  %v31 = load volatile <16 x i8>, <16 x i8> *%ptr
+  store volatile <16 x i8> %v31, <16 x i8> *%ptr
+  store volatile <16 x i8> %v30, <16 x i8> *%ptr
+  store volatile <16 x i8> %v29, <16 x i8> *%ptr
+  store volatile <16 x i8> %v28, <16 x i8> *%ptr
+  store volatile <16 x i8> %v27, <16 x i8> *%ptr
+  store volatile <16 x i8> %v26, <16 x i8> *%ptr
+  store volatile <16 x i8> %v25, <16 x i8> *%ptr
+  store volatile <16 x i8> %v24, <16 x i8> *%ptr
+  store volatile <16 x i8> %v23, <16 x i8> *%ptr
+  store volatile <16 x i8> %v22, <16 x i8> *%ptr
+  store volatile <16 x i8> %v21, <16 x i8> *%ptr
+  store volatile <16 x i8> %v20, <16 x i8> *%ptr
+  store volatile <16 x i8> %v19, <16 x i8> *%ptr
+  store volatile <16 x i8> %v18, <16 x i8> *%ptr
+  store volatile <16 x i8> %v17, <16 x i8> *%ptr
+  store volatile <16 x i8> %v16, <16 x i8> *%ptr
+  store volatile <16 x i8> %v15, <16 x i8> *%ptr
+  store volatile <16 x i8> %v14, <16 x i8> *%ptr
+  store volatile <16 x i8> %v13, <16 x i8> *%ptr
+  store volatile <16 x i8> %v12, <16 x i8> *%ptr
+  store volatile <16 x i8> %v11, <16 x i8> *%ptr
+  store volatile <16 x i8> %v10, <16 x i8> *%ptr
+  store volatile <16 x i8> %v9, <16 x i8> *%ptr
+  store volatile <16 x i8> %v8, <16 x i8> *%ptr
+  store volatile <16 x i8> %v7, <16 x i8> *%ptr
+  store volatile <16 x i8> %v6, <16 x i8> *%ptr
+  store volatile <16 x i8> %v5, <16 x i8> *%ptr
+  store volatile <16 x i8> %v4, <16 x i8> *%ptr
+  store volatile <16 x i8> %v3, <16 x i8> *%ptr
+  store volatile <16 x i8> %v2, <16 x i8> *%ptr
+  store volatile <16 x i8> %v1, <16 x i8> *%ptr
+  store volatile <16 x i8> %v0, <16 x i8> *%ptr
+  ret void
+}
+
+; Like f2, but only %f8 should be saved.
+define void @f3(<16 x i8> *%ptr) {
+; CHECK-LABEL: f3:
+; CHECK: aghi %r15, -168
+; CHECK-DAG: std %f8,
+; CHECK-NOT: vst {{.*}}(%r15)
+; CHECK-NOT: vl {{.*}}(%r15)
+; CHECK-NOT: %v9
+; CHECK-NOT: %v10
+; CHECK-NOT: %v11
+; CHECK-NOT: %v12
+; CHECK-NOT: %v13
+; CHECK-NOT: %v14
+; CHECK-NOT: %v15
+; CHECK-DAG: ld %f8,
+; CHECK: aghi %r15, 168
+; CHECK: br %r14
+  %v0 = load volatile <16 x i8>, <16 x i8> *%ptr
+  %v1 = load volatile <16 x i8>, <16 x i8> *%ptr
+  %v2 = load volatile <16 x i8>, <16 x i8> *%ptr
+  %v3 = load volatile <16 x i8>, <16 x i8> *%ptr
+  %v4 = load volatile <16 x i8>, <16 x i8> *%ptr
+  %v5 = load volatile <16 x i8>, <16 x i8> *%ptr
+  %v6 = load volatile <16 x i8>, <16 x i8> *%ptr
+  %v7 = load volatile <16 x i8>, <16 x i8> *%ptr
+  %v8 = load volatile <16 x i8>, <16 x i8> *%ptr
+  %v16 = load volatile <16 x i8>, <16 x i8> *%ptr
+  %v17 = load volatile <16 x i8>, <16 x i8> *%ptr
+  %v18 = load volatile <16 x i8>, <16 x i8> *%ptr
+  %v19 = load volatile <16 x i8>, <16 x i8> *%ptr
+  %v20 = load volatile <16 x i8>, <16 x i8> *%ptr
+  %v21 = load volatile <16 x i8>, <16 x i8> *%ptr
+  %v22 = load volatile <16 x i8>, <16 x i8> *%ptr
+  %v23 = load volatile <16 x i8>, <16 x i8> *%ptr
+  %v24 = load volatile <16 x i8>, <16 x i8> *%ptr
+  %v25 = load volatile <16 x i8>, <16 x i8> *%ptr
+  %v26 = load volatile <16 x i8>, <16 x i8> *%ptr
+  %v27 = load volatile <16 x i8>, <16 x i8> *%ptr
+  %v28 = load volatile <16 x i8>, <16 x i8> *%ptr
+  %v29 = load volatile <16 x i8>, <16 x i8> *%ptr
+  %v30 = load volatile <16 x i8>, <16 x i8> *%ptr
+  %v31 = load volatile <16 x i8>, <16 x i8> *%ptr
+  store volatile <16 x i8> %v31, <16 x i8> *%ptr
+  store volatile <16 x i8> %v30, <16 x i8> *%ptr
+  store volatile <16 x i8> %v29, <16 x i8> *%ptr
+  store volatile <16 x i8> %v28, <16 x i8> *%ptr
+  store volatile <16 x i8> %v27, <16 x i8> *%ptr
+  store volatile <16 x i8> %v26, <16 x i8> *%ptr
+  store volatile <16 x i8> %v25, <16 x i8> *%ptr
+  store volatile <16 x i8> %v24, <16 x i8> *%ptr
+  store volatile <16 x i8> %v23, <16 x i8> *%ptr
+  store volatile <16 x i8> %v22, <16 x i8> *%ptr
+  store volatile <16 x i8> %v21, <16 x i8> *%ptr
+  store volatile <16 x i8> %v20, <16 x i8> *%ptr
+  store volatile <16 x i8> %v19, <16 x i8> *%ptr
+  store volatile <16 x i8> %v18, <16 x i8> *%ptr
+  store volatile <16 x i8> %v17, <16 x i8> *%ptr
+  store volatile <16 x i8> %v16, <16 x i8> *%ptr
+  store volatile <16 x i8> %v8, <16 x i8> *%ptr
+  store volatile <16 x i8> %v7, <16 x i8> *%ptr
+  store volatile <16 x i8> %v6, <16 x i8> *%ptr
+  store volatile <16 x i8> %v5, <16 x i8> *%ptr
+  store volatile <16 x i8> %v4, <16 x i8> *%ptr
+  store volatile <16 x i8> %v3, <16 x i8> *%ptr
+  store volatile <16 x i8> %v2, <16 x i8> *%ptr
+  store volatile <16 x i8> %v1, <16 x i8> *%ptr
+  store volatile <16 x i8> %v0, <16 x i8> *%ptr
+  ret void
+}
+
+; Like f2, but no registers should be saved.
+define void @f4(<16 x i8> *%ptr) {
+; CHECK-LABEL: f4:
+; CHECK-NOT: %r15
+; CHECK: br %r14
+  %v0 = load volatile <16 x i8>, <16 x i8> *%ptr
+  %v1 = load volatile <16 x i8>, <16 x i8> *%ptr
+  %v2 = load volatile <16 x i8>, <16 x i8> *%ptr
+  %v3 = load volatile <16 x i8>, <16 x i8> *%ptr
+  %v4 = load volatile <16 x i8>, <16 x i8> *%ptr
+  %v5 = load volatile <16 x i8>, <16 x i8> *%ptr
+  %v6 = load volatile <16 x i8>, <16 x i8> *%ptr
+  %v7 = load volatile <16 x i8>, <16 x i8> *%ptr
+  %v16 = load volatile <16 x i8>, <16 x i8> *%ptr
+  %v17 = load volatile <16 x i8>, <16 x i8> *%ptr
+  %v18 = load volatile <16 x i8>, <16 x i8> *%ptr
+  %v19 = load volatile <16 x i8>, <16 x i8> *%ptr
+  %v20 = load volatile <16 x i8>, <16 x i8> *%ptr
+  %v21 = load volatile <16 x i8>, <16 x i8> *%ptr
+  %v22 = load volatile <16 x i8>, <16 x i8> *%ptr
+  %v23 = load volatile <16 x i8>, <16 x i8> *%ptr
+  %v24 = load volatile <16 x i8>, <16 x i8> *%ptr
+  %v25 = load volatile <16 x i8>, <16 x i8> *%ptr
+  %v26 = load volatile <16 x i8>, <16 x i8> *%ptr
+  %v27 = load volatile <16 x i8>, <16 x i8> *%ptr
+  %v28 = load volatile <16 x i8>, <16 x i8> *%ptr
+  %v29 = load volatile <16 x i8>, <16 x i8> *%ptr
+  %v30 = load volatile <16 x i8>, <16 x i8> *%ptr
+  %v31 = load volatile <16 x i8>, <16 x i8> *%ptr
+  store volatile <16 x i8> %v31, <16 x i8> *%ptr
+  store volatile <16 x i8> %v30, <16 x i8> *%ptr
+  store volatile <16 x i8> %v29, <16 x i8> *%ptr
+  store volatile <16 x i8> %v28, <16 x i8> *%ptr
+  store volatile <16 x i8> %v27, <16 x i8> *%ptr
+  store volatile <16 x i8> %v26, <16 x i8> *%ptr
+  store volatile <16 x i8> %v25, <16 x i8> *%ptr
+  store volatile <16 x i8> %v24, <16 x i8> *%ptr
+  store volatile <16 x i8> %v23, <16 x i8> *%ptr
+  store volatile <16 x i8> %v22, <16 x i8> *%ptr
+  store volatile <16 x i8> %v21, <16 x i8> *%ptr
+  store volatile <16 x i8> %v20, <16 x i8> *%ptr
+  store volatile <16 x i8> %v19, <16 x i8> *%ptr
+  store volatile <16 x i8> %v18, <16 x i8> *%ptr
+  store volatile <16 x i8> %v17, <16 x i8> *%ptr
+  store volatile <16 x i8> %v16, <16 x i8> *%ptr
+  store volatile <16 x i8> %v7, <16 x i8> *%ptr
+  store volatile <16 x i8> %v6, <16 x i8> *%ptr
+  store volatile <16 x i8> %v5, <16 x i8> *%ptr
+  store volatile <16 x i8> %v4, <16 x i8> *%ptr
+  store volatile <16 x i8> %v3, <16 x i8> *%ptr
+  store volatile <16 x i8> %v2, <16 x i8> *%ptr
+  store volatile <16 x i8> %v1, <16 x i8> *%ptr
+  store volatile <16 x i8> %v0, <16 x i8> *%ptr
+  ret void
+}
diff --git a/test/CodeGen/SystemZ/frame-20.ll b/test/CodeGen/SystemZ/frame-20.ll
new file mode 100644
index 00000000000..8d601c6f6d5
--- /dev/null
+++ b/test/CodeGen/SystemZ/frame-20.ll
@@ -0,0 +1,445 @@
+; Like frame-03.ll, but for z13.  In this case we have 16 more registers
+; available.
+;
+; RUN: llc < %s -mtriple=s390x-linux-gnu -mcpu=z13 | FileCheck %s
+
+; This function should require all FPRs, but no other spill slots.
+; We need to save and restore 8 of the 16 FPRs, so the frame size
+; should be exactly 160 + 8 * 8 = 224.  The CFA offset is 160
+; (the caller-allocated part of the frame) + 224.
+define void @f1(double *%ptr) {
+; CHECK-LABEL: f1:
+; CHECK: aghi %r15, -224
+; CHECK: .cfi_def_cfa_offset 384
+; CHECK: std %f8, 216(%r15)
+; CHECK: std %f9, 208(%r15)
+; CHECK: std %f10, 200(%r15)
+; CHECK: std %f11, 192(%r15)
+; CHECK: std %f12, 184(%r15)
+; CHECK: std %f13, 176(%r15)
+; CHECK: std %f14, 168(%r15)
+; CHECK: std %f15, 160(%r15)
+; CHECK: .cfi_offset %f8, -168
+; CHECK: .cfi_offset %f9, -176
+; CHECK: .cfi_offset %f10, -184
+; CHECK: .cfi_offset %f11, -192
+; CHECK: .cfi_offset %f12, -200
+; CHECK: .cfi_offset %f13, -208
+; CHECK: .cfi_offset %f14, -216
+; CHECK: .cfi_offset %f15, -224
+; CHECK-DAG: ld %f0, 0(%r2)
+; CHECK-DAG: ld %f7, 0(%r2)
+; CHECK-DAG: ld %f8, 0(%r2)
+; CHECK-DAG: ld %f15, 0(%r2)
+; CHECK-DAG: vlrepg %v16, 0(%r2)
+; CHECK-DAG: vlrepg %v23, 0(%r2)
+; CHECK-DAG: vlrepg %v24, 0(%r2)
+; CHECK-DAG: vlrepg %v31, 0(%r2)
+; CHECK: ld %f8, 216(%r15)
+; CHECK: ld %f9, 208(%r15)
+; CHECK: ld %f10, 200(%r15)
+; CHECK: ld %f11, 192(%r15)
+; CHECK: ld %f12, 184(%r15)
+; CHECK: ld %f13, 176(%r15)
+; CHECK: ld %f14, 168(%r15)
+; CHECK: ld %f15, 160(%r15)
+; CHECK: aghi %r15, 224
+; CHECK: br %r14
+  %l0 = load volatile double, double *%ptr
+  %l1 = load volatile double, double *%ptr
+  %l2 = load volatile double, double *%ptr
+  %l3 = load volatile double, double *%ptr
+  %l4 = load volatile double, double *%ptr
+  %l5 = load volatile double, double *%ptr
+  %l6 = load volatile double, double *%ptr
+  %l7 = load volatile double, double *%ptr
+  %l8 = load volatile double, double *%ptr
+  %l9 = load volatile double, double *%ptr
+  %l10 = load volatile double, double *%ptr
+  %l11 = load volatile double, double *%ptr
+  %l12 = load volatile double, double *%ptr
+  %l13 = load volatile double, double *%ptr
+  %l14 = load volatile double, double *%ptr
+  %l15 = load volatile double, double *%ptr
+  %l16 = load volatile double, double *%ptr
+  %l17 = load volatile double, double *%ptr
+  %l18 = load volatile double, double *%ptr
+  %l19 = load volatile double, double *%ptr
+  %l20 = load volatile double, double *%ptr
+  %l21 = load volatile double, double *%ptr
+  %l22 = load volatile double, double *%ptr
+  %l23 = load volatile double, double *%ptr
+  %l24 = load volatile double, double *%ptr
+  %l25 = load volatile double, double *%ptr
+  %l26 = load volatile double, double *%ptr
+  %l27 = load volatile double, double *%ptr
+  %l28 = load volatile double, double *%ptr
+  %l29 = load volatile double, double *%ptr
+  %l30 = load volatile double, double *%ptr
+  %l31 = load volatile double, double *%ptr
+  %acc0 = fsub double %l0, %l0
+  %acc1 = fsub double %l1, %acc0
+  %acc2 = fsub double %l2, %acc1
+  %acc3 = fsub double %l3, %acc2
+  %acc4 = fsub double %l4, %acc3
+  %acc5 = fsub double %l5, %acc4
+  %acc6 = fsub double %l6, %acc5
+  %acc7 = fsub double %l7, %acc6
+  %acc8 = fsub double %l8, %acc7
+  %acc9 = fsub double %l9, %acc8
+  %acc10 = fsub double %l10, %acc9
+  %acc11 = fsub double %l11, %acc10
+  %acc12 = fsub double %l12, %acc11
+  %acc13 = fsub double %l13, %acc12
+  %acc14 = fsub double %l14, %acc13
+  %acc15 = fsub double %l15, %acc14
+  %acc16 = fsub double %l16, %acc15
+  %acc17 = fsub double %l17, %acc16
+  %acc18 = fsub double %l18, %acc17
+  %acc19 = fsub double %l19, %acc18
+  %acc20 = fsub double %l20, %acc19
+  %acc21 = fsub double %l21, %acc20
+  %acc22 = fsub double %l22, %acc21
+  %acc23 = fsub double %l23, %acc22
+  %acc24 = fsub double %l24, %acc23
+  %acc25 = fsub double %l25, %acc24
+  %acc26 = fsub double %l26, %acc25
+  %acc27 = fsub double %l27, %acc26
+  %acc28 = fsub double %l28, %acc27
+  %acc29 = fsub double %l29, %acc28
+  %acc30 = fsub double %l30, %acc29
+  %acc31 = fsub double %l31, %acc30
+  store volatile double %acc0, double *%ptr
+  store volatile double %acc1, double *%ptr
+  store volatile double %acc2, double *%ptr
+  store volatile double %acc3, double *%ptr
+  store volatile double %acc4, double *%ptr
+  store volatile double %acc5, double *%ptr
+  store volatile double %acc6, double *%ptr
+  store volatile double %acc7, double *%ptr
+  store volatile double %acc8, double *%ptr
+  store volatile double %acc9, double *%ptr
+  store volatile double %acc10, double *%ptr
+  store volatile double %acc11, double *%ptr
+  store volatile double %acc12, double *%ptr
+  store volatile double %acc13, double *%ptr
+  store volatile double %acc14, double *%ptr
+  store volatile double %acc15, double *%ptr
+  store volatile double %acc16, double *%ptr
+  store volatile double %acc17, double *%ptr
+  store volatile double %acc18, double *%ptr
+  store volatile double %acc19, double *%ptr
+  store volatile double %acc20, double *%ptr
+  store volatile double %acc21, double *%ptr
+  store volatile double %acc22, double *%ptr
+  store volatile double %acc23, double *%ptr
+  store volatile double %acc24, double *%ptr
+  store volatile double %acc25, double *%ptr
+  store volatile double %acc26, double *%ptr
+  store volatile double %acc27, double *%ptr
+  store volatile double %acc28, double *%ptr
+  store volatile double %acc29, double *%ptr
+  store volatile double %acc30, double *%ptr
+  store volatile double %acc31, double *%ptr
+  ret void
+}
+
+; Like f1, but requires one fewer FPR.  We allocate in numerical order,
+; so %f15 is the one that gets dropped.
+define void @f2(double *%ptr) {
+; CHECK-LABEL: f2:
+; CHECK: aghi %r15, -216
+; CHECK: .cfi_def_cfa_offset 376
+; CHECK: std %f8, 208(%r15)
+; CHECK: std %f9, 200(%r15)
+; CHECK: std %f10, 192(%r15)
+; CHECK: std %f11, 184(%r15)
+; CHECK: std %f12, 176(%r15)
+; CHECK: std %f13, 168(%r15)
+; CHECK: std %f14, 160(%r15)
+; CHECK: .cfi_offset %f8, -168
+; CHECK: .cfi_offset %f9, -176
+; CHECK: .cfi_offset %f10, -184
+; CHECK: .cfi_offset %f11, -192
+; CHECK: .cfi_offset %f12, -200
+; CHECK: .cfi_offset %f13, -208
+; CHECK: .cfi_offset %f14, -216
+; CHECK-NOT: %v15
+; CHECK-NOT: %f15
+; CHECK: ld %f8, 208(%r15)
+; CHECK: ld %f9, 200(%r15)
+; CHECK: ld %f10, 192(%r15)
+; CHECK: ld %f11, 184(%r15)
+; CHECK: ld %f12, 176(%r15)
+; CHECK: ld %f13, 168(%r15)
+; CHECK: ld %f14, 160(%r15)
+; CHECK: aghi %r15, 216
+; CHECK: br %r14
+  %l0 = load volatile double, double *%ptr
+  %l1 = load volatile double, double *%ptr
+  %l2 = load volatile double, double *%ptr
+  %l3 = load volatile double, double *%ptr
+  %l4 = load volatile double, double *%ptr
+  %l5 = load volatile double, double *%ptr
+  %l6 = load volatile double, double *%ptr
+  %l7 = load volatile double, double *%ptr
+  %l8 = load volatile double, double *%ptr
+  %l9 = load volatile double, double *%ptr
+  %l10 = load volatile double, double *%ptr
+  %l11 = load volatile double, double *%ptr
+  %l12 = load volatile double, double *%ptr
+  %l13 = load volatile double, double *%ptr
+  %l14 = load volatile double, double *%ptr
+  %l16 = load volatile double, double *%ptr
+  %l17 = load volatile double, double *%ptr
+  %l18 = load volatile double, double *%ptr
+  %l19 = load volatile double, double *%ptr
+  %l20 = load volatile double, double *%ptr
+  %l21 = load volatile double, double *%ptr
+  %l22 = load volatile double, double *%ptr
+  %l23 = load volatile double, double *%ptr
+  %l24 = load volatile double, double *%ptr
+  %l25 = load volatile double, double *%ptr
+  %l26 = load volatile double, double *%ptr
+  %l27 = load volatile double, double *%ptr
+  %l28 = load volatile double, double *%ptr
+  %l29 = load volatile double, double *%ptr
+  %l30 = load volatile double, double *%ptr
+  %l31 = load volatile double, double *%ptr
+  %acc0 = fsub double %l0, %l0
+  %acc1 = fsub double %l1, %acc0
+  %acc2 = fsub double %l2, %acc1
+  %acc3 = fsub double %l3, %acc2
+  %acc4 = fsub double %l4, %acc3
+  %acc5 = fsub double %l5, %acc4
+  %acc6 = fsub double %l6, %acc5
+  %acc7 = fsub double %l7, %acc6
+  %acc8 = fsub double %l8, %acc7
+  %acc9 = fsub double %l9, %acc8
+  %acc10 = fsub double %l10, %acc9
+  %acc11 = fsub double %l11, %acc10
+  %acc12 = fsub double %l12, %acc11
+  %acc13 = fsub double %l13, %acc12
+  %acc14 = fsub double %l14, %acc13
+  %acc16 = fsub double %l16, %acc14
+  %acc17 = fsub double %l17, %acc16
+  %acc18 = fsub double %l18, %acc17
+  %acc19 = fsub double %l19, %acc18
+  %acc20 = fsub double %l20, %acc19
+  %acc21 = fsub double %l21, %acc20
+  %acc22 = fsub double %l22, %acc21
+  %acc23 = fsub double %l23, %acc22
+  %acc24 = fsub double %l24, %acc23
+  %acc25 = fsub double %l25, %acc24
+  %acc26 = fsub double %l26, %acc25
+  %acc27 = fsub double %l27, %acc26
+  %acc28 = fsub double %l28, %acc27
+  %acc29 = fsub double %l29, %acc28
+  %acc30 = fsub double %l30, %acc29
+  %acc31 = fsub double %l31, %acc30
+  store volatile double %acc0, double *%ptr
+  store volatile double %acc1, double *%ptr
+  store volatile double %acc2, double *%ptr
+  store volatile double %acc3, double *%ptr
+  store volatile double %acc4, double *%ptr
+  store volatile double %acc5, double *%ptr
+  store volatile double %acc6, double *%ptr
+  store volatile double %acc7, double *%ptr
+  store volatile double %acc8, double *%ptr
+  store volatile double %acc9, double *%ptr
+  store volatile double %acc10, double *%ptr
+  store volatile double %acc11, double *%ptr
+  store volatile double %acc12, double *%ptr
+  store volatile double %acc13, double *%ptr
+  store volatile double %acc14, double *%ptr
+  store volatile double %acc16, double *%ptr
+  store volatile double %acc17, double *%ptr
+  store volatile double %acc18, double *%ptr
+  store volatile double %acc19, double *%ptr
+  store volatile double %acc20, double *%ptr
+  store volatile double %acc21, double *%ptr
+  store volatile double %acc22, double *%ptr
+  store volatile double %acc23, double *%ptr
+  store volatile double %acc24, double *%ptr
+  store volatile double %acc25, double *%ptr
+  store volatile double %acc26, double *%ptr
+  store volatile double %acc27, double *%ptr
+  store volatile double %acc28, double *%ptr
+  store volatile double %acc29, double *%ptr
+  store volatile double %acc30, double *%ptr
+  store volatile double %acc31, double *%ptr
+  ret void
+}
+
+; Like f1, but should require only one call-saved FPR.
+define void @f3(double *%ptr) {
+; CHECK-LABEL: f3:
+; CHECK: aghi %r15, -168
+; CHECK: .cfi_def_cfa_offset 328
+; CHECK: std %f8, 160(%r15)
+; CHECK: .cfi_offset %f8, -168
+; CHECK-NOT: {{%[fv]9}}
+; CHECK-NOT: {{%[fv]1[0-5]}}
+; CHECK: ld %f8, 160(%r15)
+; CHECK: aghi %r15, 168
+; CHECK: br %r14
+  %l0 = load volatile double, double *%ptr
+  %l1 = load volatile double, double *%ptr
+  %l2 = load volatile double, double *%ptr
+  %l3 = load volatile double, double *%ptr
+  %l4 = load volatile double, double *%ptr
+  %l5 = load volatile double, double *%ptr
+  %l6 = load volatile double, double *%ptr
+  %l7 = load volatile double, double *%ptr
+  %l8 = load volatile double, double *%ptr
+  %l16 = load volatile double, double *%ptr
+  %l17 = load volatile double, double *%ptr
+  %l18 = load volatile double, double *%ptr
+  %l19 = load volatile double, double *%ptr
+  %l20 = load volatile double, double *%ptr
+  %l21 = load volatile double, double *%ptr
+  %l22 = load volatile double, double *%ptr
+  %l23 = load volatile double, double *%ptr
+  %l24 = load volatile double, double *%ptr
+  %l25 = load volatile double, double *%ptr
+  %l26 = load volatile double, double *%ptr
+  %l27 = load volatile double, double *%ptr
+  %l28 = load volatile double, double *%ptr
+  %l29 = load volatile double, double *%ptr
+  %l30 = load volatile double, double *%ptr
+  %l31 = load volatile double, double *%ptr
+  %acc0 = fsub double %l0, %l0
+  %acc1 = fsub double %l1, %acc0
+  %acc2 = fsub double %l2, %acc1
+  %acc3 = fsub double %l3, %acc2
+  %acc4 = fsub double %l4, %acc3
+  %acc5 = fsub double %l5, %acc4
+  %acc6 = fsub double %l6, %acc5
+  %acc7 = fsub double %l7, %acc6
+  %acc8 = fsub double %l8, %acc7
+  %acc16 = fsub double %l16, %acc8
+  %acc17 = fsub double %l17, %acc16
+  %acc18 = fsub double %l18, %acc17
+  %acc19 = fsub double %l19, %acc18
+  %acc20 = fsub double %l20, %acc19
+  %acc21 = fsub double %l21, %acc20
+  %acc22 = fsub double %l22, %acc21
+  %acc23 = fsub double %l23, %acc22
+  %acc24 = fsub double %l24, %acc23
+  %acc25 = fsub double %l25, %acc24
+  %acc26 = fsub double %l26, %acc25
+  %acc27 = fsub double %l27, %acc26
+  %acc28 = fsub double %l28, %acc27
+  %acc29 = fsub double %l29, %acc28
+  %acc30 = fsub double %l30, %acc29
+  %acc31 = fsub double %l31, %acc30
+  store volatile double %acc0, double *%ptr
+  store volatile double %acc1, double *%ptr
+  store volatile double %acc2, double *%ptr
+  store volatile double %acc3, double *%ptr
+  store volatile double %acc4, double *%ptr
+  store volatile double %acc5, double *%ptr
+  store volatile double %acc6, double *%ptr
+  store volatile double %acc7, double *%ptr
+  store volatile double %acc8, double *%ptr
+  store volatile double %acc16, double *%ptr
+  store volatile double %acc17, double *%ptr
+  store volatile double %acc18, double *%ptr
+  store volatile double %acc19, double *%ptr
+  store volatile double %acc20, double *%ptr
+  store volatile double %acc21, double *%ptr
+  store volatile double %acc22, double *%ptr
+  store volatile double %acc23, double *%ptr
+  store volatile double %acc24, double *%ptr
+  store volatile double %acc25, double *%ptr
+  store volatile double %acc26, double *%ptr
+  store volatile double %acc27, double *%ptr
+  store volatile double %acc28, double *%ptr
+  store volatile double %acc29, double *%ptr
+  store volatile double %acc30, double *%ptr
+  store volatile double %acc31, double *%ptr
+  ret void
+}
+
+; This function should use all call-clobbered FPRs and vector registers
+; but no call-saved ones.  It shouldn't need to create a frame.
+define void @f4(double *%ptr) {
+; CHECK-LABEL: f4:
+; CHECK-NOT: %r15
+; CHECK-NOT: {{%[fv][89]}}
+; CHECK-NOT: {{%[fv]1[0-5]}}
+; CHECK: br %r14
+  %l0 = load volatile double, double *%ptr
+  %l1 = load volatile double, double *%ptr
+  %l2 = load volatile double, double *%ptr
+  %l3 = load volatile double, double *%ptr
+  %l4 = load volatile double, double *%ptr
+  %l5 = load volatile double, double *%ptr
+  %l6 = load volatile double, double *%ptr
+  %l7 = load volatile double, double *%ptr
+  %l16 = load volatile double, double *%ptr
+  %l17 = load volatile double, double *%ptr
+  %l18 = load volatile double, double *%ptr
+  %l19 = load volatile double, double *%ptr
+  %l20 = load volatile double, double *%ptr
+  %l21 = load volatile double, double *%ptr
+  %l22 = load volatile double, double *%ptr
+  %l23 = load volatile double, double *%ptr
+  %l24 = load volatile double, double *%ptr
+  %l25 = load volatile double, double *%ptr
+  %l26 = load volatile double, double *%ptr
+  %l27 = load volatile double, double *%ptr
+  %l28 = load volatile double, double *%ptr
+  %l29 = load volatile double, double *%ptr
+  %l30 = load volatile double, double *%ptr
+  %l31 = load volatile double, double *%ptr
+  %acc0 = fsub double %l0, %l0
+  %acc1 = fsub double %l1, %acc0
+  %acc2 = fsub double %l2, %acc1
+  %acc3 = fsub double %l3, %acc2
+  %acc4 = fsub double %l4, %acc3
+  %acc5 = fsub double %l5, %acc4
+  %acc6 = fsub double %l6, %acc5
+  %acc7 = fsub double %l7, %acc6
+  %acc16 = fsub double %l16, %acc7
+  %acc17 = fsub double %l17, %acc16
+  %acc18 = fsub double %l18, %acc17
+  %acc19 = fsub double %l19, %acc18
+  %acc20 = fsub double %l20, %acc19
+  %acc21 = fsub double %l21, %acc20
+  %acc22 = fsub double %l22, %acc21
+  %acc23 = fsub double %l23, %acc22
+  %acc24 = fsub double %l24, %acc23
+  %acc25 = fsub double %l25, %acc24
+  %acc26 = fsub double %l26, %acc25
+  %acc27 = fsub double %l27, %acc26
+  %acc28 = fsub double %l28, %acc27
+  %acc29 = fsub double %l29, %acc28
+  %acc30 = fsub double %l30, %acc29
+  %acc31 = fsub double %l31, %acc30
+  store volatile double %acc0, double *%ptr
+  store volatile double %acc1, double *%ptr
+  store volatile double %acc2, double *%ptr
+  store volatile double %acc3, double *%ptr
+  store volatile double %acc4, double *%ptr
+  store volatile double %acc5, double *%ptr
+  store volatile double %acc6, double *%ptr
+  store volatile double %acc7, double *%ptr
+  store volatile double %acc16, double *%ptr
+  store volatile double %acc17, double *%ptr
+  store volatile double %acc18, double *%ptr
+  store volatile double %acc19, double *%ptr
+  store volatile double %acc20, double *%ptr
+  store volatile double %acc21, double *%ptr
+  store volatile double %acc22, double *%ptr
+  store volatile double %acc23, double *%ptr
+  store volatile double %acc24, double *%ptr
+  store volatile double %acc25, double *%ptr
+  store volatile double %acc26, double *%ptr
+  store volatile double %acc27, double *%ptr
+  store volatile double %acc28, double *%ptr
+  store volatile double %acc29, double *%ptr
+  store volatile double %acc30, double *%ptr
+  store volatile double %acc31, double *%ptr
+  ret void
+}
diff --git a/test/CodeGen/SystemZ/vec-abi-align.ll b/test/CodeGen/SystemZ/vec-abi-align.ll
new file mode 100644
index 00000000000..01b97a8583e
--- /dev/null
+++ b/test/CodeGen/SystemZ/vec-abi-align.ll
@@ -0,0 +1,49 @@
+; Verify that we use the vector ABI datalayout if and only if
+; the vector facility is present.
+;
+; RUN: llc < %s -mtriple=s390x-linux-gnu | \
+; RUN:   FileCheck -check-prefix=CHECK-NOVECTOR %s
+; RUN: llc < %s -mtriple=s390x-linux-gnu -mcpu=generic | \
+; RUN:   FileCheck -check-prefix=CHECK-NOVECTOR %s
+; RUN: llc < %s -mtriple=s390x-linux-gnu -mcpu=z10 | \
+; RUN:   FileCheck -check-prefix=CHECK-NOVECTOR %s
+; RUN: llc < %s -mtriple=s390x-linux-gnu -mcpu=z196 | \
+; RUN:   FileCheck -check-prefix=CHECK-NOVECTOR %s
+; RUN: llc < %s -mtriple=s390x-linux-gnu -mcpu=zEC12 | \
+; RUN:   FileCheck -check-prefix=CHECK-NOVECTOR %s
+; RUN: llc < %s -mtriple=s390x-linux-gnu -mcpu=z13 | \
+; RUN:   FileCheck -check-prefix=CHECK-VECTOR %s
+
+; RUN: llc < %s -mtriple=s390x-linux-gnu -mattr=vector | \
+; RUN:   FileCheck -check-prefix=CHECK-VECTOR %s
+; RUN: llc < %s -mtriple=s390x-linux-gnu -mattr=+vector | \
+; RUN:   FileCheck -check-prefix=CHECK-VECTOR %s
+; RUN: llc < %s -mtriple=s390x-linux-gnu -mattr=-vector,vector | \
+; RUN:   FileCheck -check-prefix=CHECK-VECTOR %s
+; RUN: llc < %s -mtriple=s390x-linux-gnu -mattr=-vector,+vector | \
+; RUN:   FileCheck -check-prefix=CHECK-VECTOR %s
+; RUN: llc < %s -mtriple=s390x-linux-gnu -mattr=-vector | \
+; RUN:   FileCheck -check-prefix=CHECK-NOVECTOR %s
+; RUN: llc < %s -mtriple=s390x-linux-gnu -mattr=vector,-vector | \
+; RUN:   FileCheck -check-prefix=CHECK-NOVECTOR %s
+; RUN: llc < %s -mtriple=s390x-linux-gnu -mattr=+vector,-vector | \
+; RUN:   FileCheck -check-prefix=CHECK-NOVECTOR %s
+
+; RUN: llc < %s -mtriple=s390x-linux-gnu -mcpu=z13 -mattr=-vector | \
+; RUN:   FileCheck -check-prefix=CHECK-NOVECTOR %s
+
+%struct.S = type { i8, <2 x i64> }
+
+define void @test(%struct.S* %s) nounwind {
+; CHECK-VECTOR-LABEL: @test
+; CHECK-VECTOR: vl %v0, 8(%r2)
+; CHECK-NOVECTOR-LABEL: @test
+; CHECK-NOVECTOR-DAG: agsi 16(%r2), 1
+; CHECK-NOVECTOR-DAG: agsi 24(%r2), 1
+  %ptr = getelementptr %struct.S, %struct.S* %s, i64 0, i32 1
+  %vec = load <2 x i64>, <2 x i64>* %ptr
+  %add = add <2 x i64> %vec, <i64 1, i64 1>
+  store <2 x i64> %add, <2 x i64>* %ptr
+  ret void
+}
+
diff --git a/test/CodeGen/SystemZ/vec-abs-01.ll b/test/CodeGen/SystemZ/vec-abs-01.ll
new file mode 100644
index 00000000000..aec3b9314f1
--- /dev/null
+++ b/test/CodeGen/SystemZ/vec-abs-01.ll
@@ -0,0 +1,146 @@
+; Test v16i8 absolute.
+;
+; RUN: llc < %s -mtriple=s390x-linux-gnu -mcpu=z13 | FileCheck %s
+
+; Test with slt.
+define <16 x i8> @f1(<16 x i8> %val) {
+; CHECK-LABEL: f1:
+; CHECK: vlpb %v24, %v24
+; CHECK: br %r14
+  %cmp = icmp slt <16 x i8> %val, zeroinitializer
+  %neg = sub <16 x i8> zeroinitializer, %val
+  %ret = select <16 x i1> %cmp, <16 x i8> %neg, <16 x i8> %val
+  ret <16 x i8> %ret
+}
+
+; Test with sle.
+define <16 x i8> @f2(<16 x i8> %val) {
+; CHECK-LABEL: f2:
+; CHECK: vlpb %v24, %v24
+; CHECK: br %r14
+  %cmp = icmp sle <16 x i8> %val, zeroinitializer
+  %neg = sub <16 x i8> zeroinitializer, %val
+  %ret = select <16 x i1> %cmp, <16 x i8> %neg, <16 x i8> %val
+  ret <16 x i8> %ret
+}
+
+; Test with sgt.
+define <16 x i8> @f3(<16 x i8> %val) {
+; CHECK-LABEL: f3:
+; CHECK: vlpb %v24, %v24
+; CHECK: br %r14
+  %cmp = icmp sgt <16 x i8> %val, zeroinitializer
+  %neg = sub <16 x i8> zeroinitializer, %val
+  %ret = select <16 x i1> %cmp, <16 x i8> %val, <16 x i8> %neg
+  ret <16 x i8> %ret
+}
+
+; Test with sge.
+define <16 x i8> @f4(<16 x i8> %val) {
+; CHECK-LABEL: f4:
+; CHECK: vlpb %v24, %v24
+; CHECK: br %r14
+  %cmp = icmp sge <16 x i8> %val, zeroinitializer
+  %neg = sub <16 x i8> zeroinitializer, %val
+  %ret = select <16 x i1> %cmp, <16 x i8> %val, <16 x i8> %neg
+  ret <16 x i8> %ret
+}
+
+; Test that negative absolute uses VLPB too.  There is no vector equivalent
+; of LOAD NEGATIVE.
+define <16 x i8> @f5(<16 x i8> %val) {
+; CHECK-LABEL: f5:
+; CHECK: vlpb [[REG:%v[0-9]+]], %v24
+; CHECK: vlcb %v24, [[REG]]
+; CHECK: br %r14
+  %cmp = icmp slt <16 x i8> %val, zeroinitializer
+  %neg = sub <16 x i8> zeroinitializer, %val
+  %abs = select <16 x i1> %cmp, <16 x i8> %neg, <16 x i8> %val
+  %ret = sub <16 x i8> zeroinitializer, %abs
+  ret <16 x i8> %ret
+}
+
+; Try another form of negative absolute (slt version).
+define <16 x i8> @f6(<16 x i8> %val) {
+; CHECK-LABEL: f6:
+; CHECK: vlpb [[REG:%v[0-9]+]], %v24
+; CHECK: vlcb %v24, [[REG]]
+; CHECK: br %r14
+  %cmp = icmp slt <16 x i8> %val, zeroinitializer
+  %neg = sub <16 x i8> zeroinitializer, %val
+  %ret = select <16 x i1> %cmp, <16 x i8> %val, <16 x i8> %neg
+  ret <16 x i8> %ret
+}
+
+; Test with sle.
+define <16 x i8> @f7(<16 x i8> %val) {
+; CHECK-LABEL: f7:
+; CHECK: vlpb [[REG:%v[0-9]+]], %v24
+; CHECK: vlcb %v24, [[REG]]
+; CHECK: br %r14
+  %cmp = icmp sle <16 x i8> %val, zeroinitializer
+  %neg = sub <16 x i8> zeroinitializer, %val
+  %ret = select <16 x i1> %cmp, <16 x i8> %val, <16 x i8> %neg
+  ret <16 x i8> %ret
+}
+
+; Test with sgt.
+define <16 x i8> @f8(<16 x i8> %val) {
+; CHECK-LABEL: f8:
+; CHECK: vlpb [[REG:%v[0-9]+]], %v24
+; CHECK: vlcb %v24, [[REG]]
+; CHECK: br %r14
+  %cmp = icmp sgt <16 x i8> %val, zeroinitializer
+  %neg = sub <16 x i8> zeroinitializer, %val
+  %ret = select <16 x i1> %cmp, <16 x i8> %neg, <16 x i8> %val
+  ret <16 x i8> %ret
+}
+
+; Test with sge.
+define <16 x i8> @f9(<16 x i8> %val) {
+; CHECK-LABEL: f9:
+; CHECK: vlpb [[REG:%v[0-9]+]], %v24
+; CHECK: vlcb %v24, [[REG]]
+; CHECK: br %r14
+  %cmp = icmp sge <16 x i8> %val, zeroinitializer
+  %neg = sub <16 x i8> zeroinitializer, %val
+  %ret = select <16 x i1> %cmp, <16 x i8> %neg, <16 x i8> %val
+  ret <16 x i8> %ret
+}
+
+; Test with an SRA-based boolean vector.
+define <16 x i8> @f10(<16 x i8> %val) {
+; CHECK-LABEL: f10:
+; CHECK: vlpb %v24, %v24
+; CHECK: br %r14
+  %shr = ashr <16 x i8> %val,
+              <i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7,
+               i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7>
+  %neg = sub <16 x i8> zeroinitializer, %val
+  %and1 = and <16 x i8> %shr, %neg
+  %not = xor <16 x i8> %shr,
+             <i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1,
+              i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1>
+  %and2 = and <16 x i8> %not, %val
+  %ret = or <16 x i8> %and1, %and2
+  ret <16 x i8> %ret
+}
+
+; ...and again in reverse
+define <16 x i8> @f11(<16 x i8> %val) {
+; CHECK-LABEL: f11:
+; CHECK: vlpb [[REG:%v[0-9]+]], %v24
+; CHECK: vlcb %v24, [[REG]]
+; CHECK: br %r14
+  %shr = ashr <16 x i8> %val,
+              <i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7,
+               i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7>
+  %and1 = and <16 x i8> %shr, %val
+  %not = xor <16 x i8> %shr,
+             <i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1,
+              i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1>
+  %neg = sub <16 x i8> zeroinitializer, %val
+  %and2 = and <16 x i8> %not, %neg
+  %ret = or <16 x i8> %and1, %and2
+  ret <16 x i8> %ret
+}
diff --git a/test/CodeGen/SystemZ/vec-abs-02.ll b/test/CodeGen/SystemZ/vec-abs-02.ll
new file mode 100644
index 00000000000..c5af619f0ba
--- /dev/null
+++ b/test/CodeGen/SystemZ/vec-abs-02.ll
@@ -0,0 +1,142 @@
+; Test v8i16 absolute.
+;
+; RUN: llc < %s -mtriple=s390x-linux-gnu -mcpu=z13 | FileCheck %s
+
+; Test with slt.
+define <8 x i16> @f1(<8 x i16> %val) {
+; CHECK-LABEL: f1:
+; CHECK: vlph %v24, %v24
+; CHECK: br %r14
+  %cmp = icmp slt <8 x i16> %val, zeroinitializer
+  %neg = sub <8 x i16> zeroinitializer, %val
+  %ret = select <8 x i1> %cmp, <8 x i16> %neg, <8 x i16> %val
+  ret <8 x i16> %ret
+}
+
+; Test with sle.
+define <8 x i16> @f2(<8 x i16> %val) {
+; CHECK-LABEL: f2:
+; CHECK: vlph %v24, %v24
+; CHECK: br %r14
+  %cmp = icmp sle <8 x i16> %val, zeroinitializer
+  %neg = sub <8 x i16> zeroinitializer, %val
+  %ret = select <8 x i1> %cmp, <8 x i16> %neg, <8 x i16> %val
+  ret <8 x i16> %ret
+}
+
+; Test with sgt.
+define <8 x i16> @f3(<8 x i16> %val) {
+; CHECK-LABEL: f3:
+; CHECK: vlph %v24, %v24
+; CHECK: br %r14
+  %cmp = icmp sgt <8 x i16> %val, zeroinitializer
+  %neg = sub <8 x i16> zeroinitializer, %val
+  %ret = select <8 x i1> %cmp, <8 x i16> %val, <8 x i16> %neg
+  ret <8 x i16> %ret
+}
+
+; Test with sge.
+define <8 x i16> @f4(<8 x i16> %val) {
+; CHECK-LABEL: f4:
+; CHECK: vlph %v24, %v24
+; CHECK: br %r14
+  %cmp = icmp sge <8 x i16> %val, zeroinitializer
+  %neg = sub <8 x i16> zeroinitializer, %val
+  %ret = select <8 x i1> %cmp, <8 x i16> %val, <8 x i16> %neg
+  ret <8 x i16> %ret
+}
+
+; Test that negative absolute uses VLPH too.  There is no vector equivalent
+; of LOAD NEGATIVE.
+define <8 x i16> @f5(<8 x i16> %val) {
+; CHECK-LABEL: f5:
+; CHECK: vlph [[REG:%v[0-9]+]], %v24
+; CHECK: vlch %v24, [[REG]]
+; CHECK: br %r14
+  %cmp = icmp slt <8 x i16> %val, zeroinitializer
+  %neg = sub <8 x i16> zeroinitializer, %val
+  %abs = select <8 x i1> %cmp, <8 x i16> %neg, <8 x i16> %val
+  %ret = sub <8 x i16> zeroinitializer, %abs
+  ret <8 x i16> %ret
+}
+
+; Try another form of negative absolute (slt version).
+define <8 x i16> @f6(<8 x i16> %val) {
+; CHECK-LABEL: f6:
+; CHECK: vlph [[REG:%v[0-9]+]], %v24
+; CHECK: vlch %v24, [[REG]]
+; CHECK: br %r14
+  %cmp = icmp slt <8 x i16> %val, zeroinitializer
+  %neg = sub <8 x i16> zeroinitializer, %val
+  %ret = select <8 x i1> %cmp, <8 x i16> %val, <8 x i16> %neg
+  ret <8 x i16> %ret
+}
+
+; Test with sle.
+define <8 x i16> @f7(<8 x i16> %val) {
+; CHECK-LABEL: f7:
+; CHECK: vlph [[REG:%v[0-9]+]], %v24
+; CHECK: vlch %v24, [[REG]]
+; CHECK: br %r14
+  %cmp = icmp sle <8 x i16> %val, zeroinitializer
+  %neg = sub <8 x i16> zeroinitializer, %val
+  %ret = select <8 x i1> %cmp, <8 x i16> %val, <8 x i16> %neg
+  ret <8 x i16> %ret
+}
+
+; Test with sgt.
+define <8 x i16> @f8(<8 x i16> %val) {
+; CHECK-LABEL: f8:
+; CHECK: vlph [[REG:%v[0-9]+]], %v24
+; CHECK: vlch %v24, [[REG]]
+; CHECK: br %r14
+  %cmp = icmp sgt <8 x i16> %val, zeroinitializer
+  %neg = sub <8 x i16> zeroinitializer, %val
+  %ret = select <8 x i1> %cmp, <8 x i16> %neg, <8 x i16> %val
+  ret <8 x i16> %ret
+}
+
+; Test with sge.
+define <8 x i16> @f9(<8 x i16> %val) {
+; CHECK-LABEL: f9:
+; CHECK: vlph [[REG:%v[0-9]+]], %v24
+; CHECK: vlch %v24, [[REG]]
+; CHECK: br %r14
+  %cmp = icmp sge <8 x i16> %val, zeroinitializer
+  %neg = sub <8 x i16> zeroinitializer, %val
+  %ret = select <8 x i1> %cmp, <8 x i16> %neg, <8 x i16> %val
+  ret <8 x i16> %ret
+}
+
+; Test with an SRA-based boolean vector.
+define <8 x i16> @f10(<8 x i16> %val) {
+; CHECK-LABEL: f10:
+; CHECK: vlph %v24, %v24
+; CHECK: br %r14
+  %shr = ashr <8 x i16> %val,
+              <i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15>
+  %neg = sub <8 x i16> zeroinitializer, %val
+  %and1 = and <8 x i16> %shr, %neg
+  %not = xor <8 x i16> %shr,
+             <i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1>
+  %and2 = and <8 x i16> %not, %val
+  %ret = or <8 x i16> %and1, %and2
+  ret <8 x i16> %ret
+}
+
+; ...and again in reverse
+define <8 x i16> @f11(<8 x i16> %val) {
+; CHECK-LABEL: f11:
+; CHECK: vlph [[REG:%v[0-9]+]], %v24
+; CHECK: vlch %v24, [[REG]]
+; CHECK: br %r14
+  %shr = ashr <8 x i16> %val,
+              <i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15>
+  %and1 = and <8 x i16> %shr, %val
+  %not = xor <8 x i16> %shr,
+             <i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1>
+  %neg = sub <8 x i16> zeroinitializer, %val
+  %and2 = and <8 x i16> %not, %neg
+  %ret = or <8 x i16> %and1, %and2
+  ret <8 x i16> %ret
+}
diff --git a/test/CodeGen/SystemZ/vec-abs-03.ll b/test/CodeGen/SystemZ/vec-abs-03.ll
new file mode 100644
index 00000000000..cb17a8895e1
--- /dev/null
+++ b/test/CodeGen/SystemZ/vec-abs-03.ll
@@ -0,0 +1,138 @@
+; Test v4i32 absolute.
+;
+; RUN: llc < %s -mtriple=s390x-linux-gnu -mcpu=z13 | FileCheck %s
+
+; Test with slt.
+define <4 x i32> @f1(<4 x i32> %val) {
+; CHECK-LABEL: f1:
+; CHECK: vlpf %v24, %v24
+; CHECK: br %r14
+  %cmp = icmp slt <4 x i32> %val, zeroinitializer
+  %neg = sub <4 x i32> zeroinitializer, %val
+  %ret = select <4 x i1> %cmp, <4 x i32> %neg, <4 x i32> %val
+  ret <4 x i32> %ret
+}
+
+; Test with sle.
+define <4 x i32> @f2(<4 x i32> %val) {
+; CHECK-LABEL: f2:
+; CHECK: vlpf %v24, %v24
+; CHECK: br %r14
+  %cmp = icmp sle <4 x i32> %val, zeroinitializer
+  %neg = sub <4 x i32> zeroinitializer, %val
+  %ret = select <4 x i1> %cmp, <4 x i32> %neg, <4 x i32> %val
+  ret <4 x i32> %ret
+}
+
+; Test with sgt.
+define <4 x i32> @f3(<4 x i32> %val) {
+; CHECK-LABEL: f3:
+; CHECK: vlpf %v24, %v24
+; CHECK: br %r14
+  %cmp = icmp sgt <4 x i32> %val, zeroinitializer
+  %neg = sub <4 x i32> zeroinitializer, %val
+  %ret = select <4 x i1> %cmp, <4 x i32> %val, <4 x i32> %neg
+  ret <4 x i32> %ret
+}
+
+; Test with sge.
+define <4 x i32> @f4(<4 x i32> %val) {
+; CHECK-LABEL: f4:
+; CHECK: vlpf %v24, %v24
+; CHECK: br %r14
+  %cmp = icmp sge <4 x i32> %val, zeroinitializer
+  %neg = sub <4 x i32> zeroinitializer, %val
+  %ret = select <4 x i1> %cmp, <4 x i32> %val, <4 x i32> %neg
+  ret <4 x i32> %ret
+}
+
+; Test that negative absolute uses VLPF too.  There is no vector equivalent
+; of LOAD NEGATIVE.
+define <4 x i32> @f5(<4 x i32> %val) {
+; CHECK-LABEL: f5:
+; CHECK: vlpf [[REG:%v[0-9]+]], %v24
+; CHECK: vlcf %v24, [[REG]]
+; CHECK: br %r14
+  %cmp = icmp slt <4 x i32> %val, zeroinitializer
+  %neg = sub <4 x i32> zeroinitializer, %val
+  %abs = select <4 x i1> %cmp, <4 x i32> %neg, <4 x i32> %val
+  %ret = sub <4 x i32> zeroinitializer, %abs
+  ret <4 x i32> %ret
+}
+
+; Try another form of negative absolute (slt version).
+define <4 x i32> @f6(<4 x i32> %val) {
+; CHECK-LABEL: f6:
+; CHECK: vlpf [[REG:%v[0-9]+]], %v24
+; CHECK: vlcf %v24, [[REG]]
+; CHECK: br %r14
+  %cmp = icmp slt <4 x i32> %val, zeroinitializer
+  %neg = sub <4 x i32> zeroinitializer, %val
+  %ret = select <4 x i1> %cmp, <4 x i32> %val, <4 x i32> %neg
+  ret <4 x i32> %ret
+}
+
+; Test with sle.
+define <4 x i32> @f7(<4 x i32> %val) {
+; CHECK-LABEL: f7:
+; CHECK: vlpf [[REG:%v[0-9]+]], %v24
+; CHECK: vlcf %v24, [[REG]]
+; CHECK: br %r14
+  %cmp = icmp sle <4 x i32> %val, zeroinitializer
+  %neg = sub <4 x i32> zeroinitializer, %val
+  %ret = select <4 x i1> %cmp, <4 x i32> %val, <4 x i32> %neg
+  ret <4 x i32> %ret
+}
+
+; Test with sgt.
+define <4 x i32> @f8(<4 x i32> %val) {
+; CHECK-LABEL: f8:
+; CHECK: vlpf [[REG:%v[0-9]+]], %v24
+; CHECK: vlcf %v24, [[REG]]
+; CHECK: br %r14
+  %cmp = icmp sgt <4 x i32> %val, zeroinitializer
+  %neg = sub <4 x i32> zeroinitializer, %val
+  %ret = select <4 x i1> %cmp, <4 x i32> %neg, <4 x i32> %val
+  ret <4 x i32> %ret
+}
+
+; Test with sge.
+define <4 x i32> @f9(<4 x i32> %val) {
+; CHECK-LABEL: f9:
+; CHECK: vlpf [[REG:%v[0-9]+]], %v24
+; CHECK: vlcf %v24, [[REG]]
+; CHECK: br %r14
+  %cmp = icmp sge <4 x i32> %val, zeroinitializer
+  %neg = sub <4 x i32> zeroinitializer, %val
+  %ret = select <4 x i1> %cmp, <4 x i32> %neg, <4 x i32> %val
+  ret <4 x i32> %ret
+}
+
+; Test with an SRA-based boolean vector.
+define <4 x i32> @f10(<4 x i32> %val) {
+; CHECK-LABEL: f10:
+; CHECK: vlpf %v24, %v24
+; CHECK: br %r14
+  %shr = ashr <4 x i32> %val, <i32 31, i32 31, i32 31, i32 31>
+  %neg = sub <4 x i32> zeroinitializer, %val
+  %and1 = and <4 x i32> %shr, %neg
+  %not = xor <4 x i32> %shr, <i32 -1, i32 -1, i32 -1, i32 -1>
+  %and2 = and <4 x i32> %not, %val
+  %ret = or <4 x i32> %and1, %and2
+  ret <4 x i32> %ret
+}
+
+; ...and again in reverse
+define <4 x i32> @f11(<4 x i32> %val) {
+; CHECK-LABEL: f11:
+; CHECK: vlpf [[REG:%v[0-9]+]], %v24
+; CHECK: vlcf %v24, [[REG]]
+; CHECK: br %r14
+  %shr = ashr <4 x i32> %val, <i32 31, i32 31, i32 31, i32 31>
+  %and1 = and <4 x i32> %shr, %val
+  %not = xor <4 x i32> %shr, <i32 -1, i32 -1, i32 -1, i32 -1>
+  %neg = sub <4 x i32> zeroinitializer, %val
+  %and2 = and <4 x i32> %not, %neg
+  %ret = or <4 x i32> %and1, %and2
+  ret <4 x i32> %ret
+}
diff --git a/test/CodeGen/SystemZ/vec-abs-04.ll b/test/CodeGen/SystemZ/vec-abs-04.ll
new file mode 100644
index 00000000000..31c489b00b3
--- /dev/null
+++ b/test/CodeGen/SystemZ/vec-abs-04.ll
@@ -0,0 +1,138 @@
+; Test v2i64 absolute.
+;
+; RUN: llc < %s -mtriple=s390x-linux-gnu -mcpu=z13 | FileCheck %s
+
+; Test with slt.
+define <2 x i64> @f1(<2 x i64> %val) {
+; CHECK-LABEL: f1:
+; CHECK: vlpg %v24, %v24
+; CHECK: br %r14
+  %cmp = icmp slt <2 x i64> %val, zeroinitializer
+  %neg = sub <2 x i64> zeroinitializer, %val
+  %ret = select <2 x i1> %cmp, <2 x i64> %neg, <2 x i64> %val
+  ret <2 x i64> %ret
+}
+
+; Test with sle.
+define <2 x i64> @f2(<2 x i64> %val) {
+; CHECK-LABEL: f2:
+; CHECK: vlpg %v24, %v24
+; CHECK: br %r14
+  %cmp = icmp sle <2 x i64> %val, zeroinitializer
+  %neg = sub <2 x i64> zeroinitializer, %val
+  %ret = select <2 x i1> %cmp, <2 x i64> %neg, <2 x i64> %val
+  ret <2 x i64> %ret
+}
+
+; Test with sgt.
+define <2 x i64> @f3(<2 x i64> %val) {
+; CHECK-LABEL: f3:
+; CHECK: vlpg %v24, %v24
+; CHECK: br %r14
+  %cmp = icmp sgt <2 x i64> %val, zeroinitializer
+  %neg = sub <2 x i64> zeroinitializer, %val
+  %ret = select <2 x i1> %cmp, <2 x i64> %val, <2 x i64> %neg
+  ret <2 x i64> %ret
+}
+
+; Test with sge.
+define <2 x i64> @f4(<2 x i64> %val) {
+; CHECK-LABEL: f4:
+; CHECK: vlpg %v24, %v24
+; CHECK: br %r14
+  %cmp = icmp sge <2 x i64> %val, zeroinitializer
+  %neg = sub <2 x i64> zeroinitializer, %val
+  %ret = select <2 x i1> %cmp, <2 x i64> %val, <2 x i64> %neg
+  ret <2 x i64> %ret
+}
+
+; Test that negative absolute uses VLPG too.  There is no vector equivalent
+; of LOAD NEGATIVE.
+define <2 x i64> @f5(<2 x i64> %val) {
+; CHECK-LABEL: f5:
+; CHECK: vlpg [[REG:%v[0-9]+]], %v24
+; CHECK: vlcg %v24, [[REG]]
+; CHECK: br %r14
+  %cmp = icmp slt <2 x i64> %val, zeroinitializer
+  %neg = sub <2 x i64> zeroinitializer, %val
+  %abs = select <2 x i1> %cmp, <2 x i64> %neg, <2 x i64> %val
+  %ret = sub <2 x i64> zeroinitializer, %abs
+  ret <2 x i64> %ret
+}
+
+; Try another form of negative absolute (slt version).
+define <2 x i64> @f6(<2 x i64> %val) {
+; CHECK-LABEL: f6:
+; CHECK: vlpg [[REG:%v[0-9]+]], %v24
+; CHECK: vlcg %v24, [[REG]]
+; CHECK: br %r14
+  %cmp = icmp slt <2 x i64> %val, zeroinitializer
+  %neg = sub <2 x i64> zeroinitializer, %val
+  %ret = select <2 x i1> %cmp, <2 x i64> %val, <2 x i64> %neg
+  ret <2 x i64> %ret
+}
+
+; Test with sle.
+define <2 x i64> @f7(<2 x i64> %val) {
+; CHECK-LABEL: f7:
+; CHECK: vlpg [[REG:%v[0-9]+]], %v24
+; CHECK: vlcg %v24, [[REG]]
+; CHECK: br %r14
+  %cmp = icmp sle <2 x i64> %val, zeroinitializer
+  %neg = sub <2 x i64> zeroinitializer, %val
+  %ret = select <2 x i1> %cmp, <2 x i64> %val, <2 x i64> %neg
+  ret <2 x i64> %ret
+}
+
+; Test with sgt.
+define <2 x i64> @f8(<2 x i64> %val) {
+; CHECK-LABEL: f8:
+; CHECK: vlpg [[REG:%v[0-9]+]], %v24
+; CHECK: vlcg %v24, [[REG]]
+; CHECK: br %r14
+  %cmp = icmp sgt <2 x i64> %val, zeroinitializer
+  %neg = sub <2 x i64> zeroinitializer, %val
+  %ret = select <2 x i1> %cmp, <2 x i64> %neg, <2 x i64> %val
+  ret <2 x i64> %ret
+}
+
+; Test with sge.
+define <2 x i64> @f9(<2 x i64> %val) {
+; CHECK-LABEL: f9:
+; CHECK: vlpg [[REG:%v[0-9]+]], %v24
+; CHECK: vlcg %v24, [[REG]]
+; CHECK: br %r14
+  %cmp = icmp sge <2 x i64> %val, zeroinitializer
+  %neg = sub <2 x i64> zeroinitializer, %val
+  %ret = select <2 x i1> %cmp, <2 x i64> %neg, <2 x i64> %val
+  ret <2 x i64> %ret
+}
+
+; Test with an SRA-based boolean vector.
+define <2 x i64> @f10(<2 x i64> %val) {
+; CHECK-LABEL: f10:
+; CHECK: vlpg %v24, %v24
+; CHECK: br %r14
+  %shr = ashr <2 x i64> %val, <i64 63, i64 63>
+  %neg = sub <2 x i64> zeroinitializer, %val
+  %and1 = and <2 x i64> %shr, %neg
+  %not = xor <2 x i64> %shr, <i64 -1, i64 -1>
+  %and2 = and <2 x i64> %not, %val
+  %ret = or <2 x i64> %and1, %and2
+  ret <2 x i64> %ret
+}
+
+; ...and again in reverse
+define <2 x i64> @f11(<2 x i64> %val) {
+; CHECK-LABEL: f11:
+; CHECK: vlpg [[REG:%v[0-9]+]], %v24
+; CHECK: vlcg %v24, [[REG]]
+; CHECK: br %r14
+  %shr = ashr <2 x i64> %val, <i64 63, i64 63>
+  %and1 = and <2 x i64> %shr, %val
+  %not = xor <2 x i64> %shr, <i64 -1, i64 -1>
+  %neg = sub <2 x i64> zeroinitializer, %val
+  %and2 = and <2 x i64> %not, %neg
+  %ret = or <2 x i64> %and1, %and2
+  ret <2 x i64> %ret
+}
diff --git a/test/CodeGen/SystemZ/vec-abs-05.ll b/test/CodeGen/SystemZ/vec-abs-05.ll
new file mode 100644
index 00000000000..63210f87b94
--- /dev/null
+++ b/test/CodeGen/SystemZ/vec-abs-05.ll
@@ -0,0 +1,46 @@
+; Test f64 and v2f64 absolute.
+;
+; RUN: llc < %s -mtriple=s390x-linux-gnu -mcpu=z13 | FileCheck %s
+
+declare double @llvm.fabs.f64(double)
+declare <2 x double> @llvm.fabs.v2f64(<2 x double>)
+
+; Test a plain absolute.
+define <2 x double> @f1(<2 x double> %val) {
+; CHECK-LABEL: f1:
+; CHECK: vflpdb %v24, %v24
+; CHECK: br %r14
+  %ret = call <2 x double> @llvm.fabs.v2f64(<2 x double> %val)
+  ret <2 x double> %ret
+}
+
+; Test a negative absolute.
+define <2 x double> @f2(<2 x double> %val) {
+; CHECK-LABEL: f2:
+; CHECK: vflndb %v24, %v24
+; CHECK: br %r14
+  %abs = call <2 x double> @llvm.fabs.v2f64(<2 x double> %val)
+  %ret = fsub <2 x double> <double -0.0, double -0.0>, %abs
+  ret <2 x double> %ret
+}
+
+; Test an f64 absolute that uses vector registers.
+define double @f3(<2 x double> %val) {
+; CHECK-LABEL: f3:
+; CHECK: wflpdb %f0, %v24
+; CHECK: br %r14
+  %scalar = extractelement <2 x double> %val, i32 0
+  %ret = call double @llvm.fabs.f64(double %scalar)
+  ret double %ret
+}
+
+; Test an f64 negative absolute that uses vector registers.
+define double @f4(<2 x double> %val) {
+; CHECK-LABEL: f4:
+; CHECK: wflndb %f0, %v24
+; CHECK: br %r14
+  %scalar = extractelement <2 x double> %val, i32 0
+  %abs = call double @llvm.fabs.f64(double %scalar)
+  %ret = fsub double -0.0, %abs
+  ret double %ret
+}
diff --git a/test/CodeGen/SystemZ/vec-add-01.ll b/test/CodeGen/SystemZ/vec-add-01.ll
new file mode 100644
index 00000000000..31703437767
--- /dev/null
+++ b/test/CodeGen/SystemZ/vec-add-01.ll
@@ -0,0 +1,60 @@
+; Test vector addition.
+;
+; RUN: llc < %s -mtriple=s390x-linux-gnu -mcpu=z13 | FileCheck %s
+
+; Test a v16i8 addition.
+define <16 x i8> @f1(<16 x i8> %dummy, <16 x i8> %val1, <16 x i8> %val2) {
+; CHECK-LABEL: f1:
+; CHECK: vab %v24, %v26, %v28
+; CHECK: br %r14
+  %ret = add <16 x i8> %val1, %val2
+  ret <16 x i8> %ret
+}
+
+; Test a v8i16 addition.
+define <8 x i16> @f2(<8 x i16> %dummy, <8 x i16> %val1, <8 x i16> %val2) {
+; CHECK-LABEL: f2:
+; CHECK: vah %v24, %v26, %v28
+; CHECK: br %r14
+  %ret = add <8 x i16> %val1, %val2
+  ret <8 x i16> %ret
+}
+
+; Test a v4i32 addition.
+define <4 x i32> @f3(<4 x i32> %dummy, <4 x i32> %val1, <4 x i32> %val2) {
+; CHECK-LABEL: f3:
+; CHECK: vaf %v24, %v26, %v28
+; CHECK: br %r14
+  %ret = add <4 x i32> %val1, %val2
+  ret <4 x i32> %ret
+}
+
+; Test a v2i64 addition.
+define <2 x i64> @f4(<2 x i64> %dummy, <2 x i64> %val1, <2 x i64> %val2) {
+; CHECK-LABEL: f4:
+; CHECK: vag %v24, %v26, %v28
+; CHECK: br %r14
+  %ret = add <2 x i64> %val1, %val2
+  ret <2 x i64> %ret
+}
+
+; Test a v2f64 addition.
+define <2 x double> @f5(<2 x double> %dummy, <2 x double> %val1,
+                        <2 x double> %val2) {
+; CHECK-LABEL: f5:
+; CHECK: vfadb %v24, %v26, %v28
+; CHECK: br %r14
+  %ret = fadd <2 x double> %val1, %val2
+  ret <2 x double> %ret
+}
+
+; Test an f64 addition that uses vector registers.
+define double @f6(<2 x double> %val1, <2 x double> %val2) {
+; CHECK-LABEL: f6:
+; CHECK: wfadb %f0, %v24, %v26
+; CHECK: br %r14
+  %scalar1 = extractelement <2 x double> %val1, i32 0
+  %scalar2 = extractelement <2 x double> %val2, i32 0
+  %ret = fadd double %scalar1, %scalar2
+  ret double %ret
+}
diff --git a/test/CodeGen/SystemZ/vec-and-01.ll b/test/CodeGen/SystemZ/vec-and-01.ll
new file mode 100644
index 00000000000..d467de69cea
--- /dev/null
+++ b/test/CodeGen/SystemZ/vec-and-01.ll
@@ -0,0 +1,39 @@
+; Test vector AND.
+;
+; RUN: llc < %s -mtriple=s390x-linux-gnu -mcpu=z13 | FileCheck %s
+
+; Test a v16i8 AND.
+define <16 x i8> @f1(<16 x i8> %dummy, <16 x i8> %val1, <16 x i8> %val2) {
+; CHECK-LABEL: f1:
+; CHECK: vn %v24, %v26, %v28
+; CHECK: br %r14
+  %ret = and <16 x i8> %val1, %val2
+  ret <16 x i8> %ret
+}
+
+; Test a v8i16 AND.
+define <8 x i16> @f2(<8 x i16> %dummy, <8 x i16> %val1, <8 x i16> %val2) {
+; CHECK-LABEL: f2:
+; CHECK: vn %v24, %v26, %v28
+; CHECK: br %r14
+  %ret = and <8 x i16> %val1, %val2
+  ret <8 x i16> %ret
+}
+
+; Test a v4i32 AND.
+define <4 x i32> @f3(<4 x i32> %dummy, <4 x i32> %val1, <4 x i32> %val2) {
+; CHECK-LABEL: f3:
+; CHECK: vn %v24, %v26, %v28
+; CHECK: br %r14
+  %ret = and <4 x i32> %val1, %val2
+  ret <4 x i32> %ret
+}
+
+; Test a v2i64 AND.
+define <2 x i64> @f4(<2 x i64> %dummy, <2 x i64> %val1, <2 x i64> %val2) {
+; CHECK-LABEL: f4:
+; CHECK: vn %v24, %v26, %v28
+; CHECK: br %r14
+  %ret = and <2 x i64> %val1, %val2
+  ret <2 x i64> %ret
+}
diff --git a/test/CodeGen/SystemZ/vec-and-02.ll b/test/CodeGen/SystemZ/vec-and-02.ll
new file mode 100644
index 00000000000..30bc9241689
--- /dev/null
+++ b/test/CodeGen/SystemZ/vec-and-02.ll
@@ -0,0 +1,91 @@
+; Test vector AND-NOT.
+;
+; RUN: llc < %s -mtriple=s390x-linux-gnu -mcpu=z13 | FileCheck %s
+
+; Test a v16i8 AND-NOT.
+define <16 x i8> @f1(<16 x i8> %dummy, <16 x i8> %val1, <16 x i8> %val2) {
+; CHECK-LABEL: f1:
+; CHECK: vnc %v24, %v26, %v28
+; CHECK: br %r14
+  %not = xor <16 x i8> %val2, <i8 -1, i8 -1, i8 -1, i8 -1,
+                               i8 -1, i8 -1, i8 -1, i8 -1,
+                               i8 -1, i8 -1, i8 -1, i8 -1,
+                               i8 -1, i8 -1, i8 -1, i8 -1>
+  %ret = and <16 x i8> %val1, %not
+  ret <16 x i8> %ret
+}
+
+; ...and again with the reverse.
+define <16 x i8> @f2(<16 x i8> %dummy, <16 x i8> %val1, <16 x i8> %val2) {
+; CHECK-LABEL: f2:
+; CHECK: vnc %v24, %v28, %v26
+; CHECK: br %r14
+  %not = xor <16 x i8> %val1, <i8 -1, i8 -1, i8 -1, i8 -1,
+                               i8 -1, i8 -1, i8 -1, i8 -1,
+                               i8 -1, i8 -1, i8 -1, i8 -1,
+                               i8 -1, i8 -1, i8 -1, i8 -1>
+  %ret = and <16 x i8> %not, %val2
+  ret <16 x i8> %ret
+}
+
+; Test a v8i16 AND-NOT.
+define <8 x i16> @f3(<8 x i16> %dummy, <8 x i16> %val1, <8 x i16> %val2) {
+; CHECK-LABEL: f3:
+; CHECK: vnc %v24, %v26, %v28
+; CHECK: br %r14
+  %not = xor <8 x i16> %val2, <i16 -1, i16 -1, i16 -1, i16 -1,
+                               i16 -1, i16 -1, i16 -1, i16 -1>
+  %ret = and <8 x i16> %val1, %not
+  ret <8 x i16> %ret
+}
+
+; ...and again with the reverse.
+define <8 x i16> @f4(<8 x i16> %dummy, <8 x i16> %val1, <8 x i16> %val2) {
+; CHECK-LABEL: f4:
+; CHECK: vnc %v24, %v28, %v26
+; CHECK: br %r14
+  %not = xor <8 x i16> %val1, <i16 -1, i16 -1, i16 -1, i16 -1,
+                               i16 -1, i16 -1, i16 -1, i16 -1>
+  %ret = and <8 x i16> %not, %val2
+  ret <8 x i16> %ret
+}
+
+; Test a v4i32 AND-NOT.
+define <4 x i32> @f5(<4 x i32> %dummy, <4 x i32> %val1, <4 x i32> %val2) {
+; CHECK-LABEL: f5:
+; CHECK: vnc %v24, %v26, %v28
+; CHECK: br %r14
+  %not = xor <4 x i32> %val2, <i32 -1, i32 -1, i32 -1, i32 -1>
+  %ret = and <4 x i32> %val1, %not
+  ret <4 x i32> %ret
+}
+
+; ...and again with the reverse.
+define <4 x i32> @f6(<4 x i32> %dummy, <4 x i32> %val1, <4 x i32> %val2) {
+; CHECK-LABEL: f6:
+; CHECK: vnc %v24, %v28, %v26
+; CHECK: br %r14
+  %not = xor <4 x i32> %val1, <i32 -1, i32 -1, i32 -1, i32 -1>
+  %ret = and <4 x i32> %not, %val2
+  ret <4 x i32> %ret
+}
+
+; Test a v2i64 AND-NOT.
+define <2 x i64> @f7(<2 x i64> %dummy, <2 x i64> %val1, <2 x i64> %val2) {
+; CHECK-LABEL: f7:
+; CHECK: vnc %v24, %v26, %v28
+; CHECK: br %r14
+  %not = xor <2 x i64> %val2, <i64 -1, i64 -1>
+  %ret = and <2 x i64> %val1, %not
+  ret <2 x i64> %ret
+}
+
+; ...and again with the reverse.
+define <2 x i64> @f8(<2 x i64> %dummy, <2 x i64> %val1, <2 x i64> %val2) {
+; CHECK-LABEL: f8:
+; CHECK: vnc %v24, %v28, %v26
+; CHECK: br %r14
+  %not = xor <2 x i64> %val1, <i64 -1, i64 -1>
+  %ret = and <2 x i64> %not, %val2
+  ret <2 x i64> %ret
+}
diff --git a/test/CodeGen/SystemZ/vec-and-03.ll b/test/CodeGen/SystemZ/vec-and-03.ll
new file mode 100644
index 00000000000..c73d570fb7b
--- /dev/null
+++ b/test/CodeGen/SystemZ/vec-and-03.ll
@@ -0,0 +1,113 @@
+; Test vector zero extensions, which need to be implemented as ANDs.
+;
+; RUN: llc < %s -mtriple=s390x-linux-gnu -mcpu=z13 | FileCheck %s
+
+; Test a v16i1->v16i8 extension.
+define <16 x i8> @f1(<16 x i8> %val) {
+; CHECK-LABEL: f1:
+; CHECK: vrepib [[REG:%v[0-9]+]], 1
+; CHECK: vn %v24, %v24, [[REG]]
+; CHECK: br %r14
+  %trunc = trunc <16 x i8> %val to <16 x i1>
+  %ret = zext <16 x i1> %trunc to <16 x i8>
+  ret <16 x i8> %ret
+}
+
+; Test a v8i1->v8i16 extension.
+define <8 x i16> @f2(<8 x i16> %val) {
+; CHECK-LABEL: f2:
+; CHECK: vrepih [[REG:%v[0-9]+]], 1
+; CHECK: vn %v24, %v24, [[REG]]
+; CHECK: br %r14
+  %trunc = trunc <8 x i16> %val to <8 x i1>
+  %ret = zext <8 x i1> %trunc to <8 x i16>
+  ret <8 x i16> %ret
+}
+
+; Test a v8i8->v8i16 extension.
+define <8 x i16> @f3(<8 x i16> %val) {
+; CHECK-LABEL: f3:
+; CHECK: vgbm [[REG:%v[0-9]+]], 21845
+; CHECK: vn %v24, %v24, [[REG]]
+; CHECK: br %r14
+  %trunc = trunc <8 x i16> %val to <8 x i8>
+  %ret = zext <8 x i8> %trunc to <8 x i16>
+  ret <8 x i16> %ret
+}
+
+; Test a v4i1->v4i32 extension.
+define <4 x i32> @f4(<4 x i32> %val) {
+; CHECK-LABEL: f4:
+; CHECK: vrepif [[REG:%v[0-9]+]], 1
+; CHECK: vn %v24, %v24, [[REG]]
+; CHECK: br %r14
+  %trunc = trunc <4 x i32> %val to <4 x i1>
+  %ret = zext <4 x i1> %trunc to <4 x i32>
+  ret <4 x i32> %ret
+}
+
+; Test a v4i8->v4i32 extension.
+define <4 x i32> @f5(<4 x i32> %val) {
+; CHECK-LABEL: f5:
+; CHECK: vgbm [[REG:%v[0-9]+]], 4369
+; CHECK: vn %v24, %v24, [[REG]]
+; CHECK: br %r14
+  %trunc = trunc <4 x i32> %val to <4 x i8>
+  %ret = zext <4 x i8> %trunc to <4 x i32>
+  ret <4 x i32> %ret
+}
+
+; Test a v4i16->v4i32 extension.
+define <4 x i32> @f6(<4 x i32> %val) {
+; CHECK-LABEL: f6:
+; CHECK: vgbm [[REG:%v[0-9]+]], 13107
+; CHECK: vn %v24, %v24, [[REG]]
+; CHECK: br %r14
+  %trunc = trunc <4 x i32> %val to <4 x i16>
+  %ret = zext <4 x i16> %trunc to <4 x i32>
+  ret <4 x i32> %ret
+}
+
+; Test a v2i1->v2i64 extension.
+define <2 x i64> @f7(<2 x i64> %val) {
+; CHECK-LABEL: f7:
+; CHECK: vrepig [[REG:%v[0-9]+]], 1
+; CHECK: vn %v24, %v24, [[REG]]
+; CHECK: br %r14
+  %trunc = trunc <2 x i64> %val to <2 x i1>
+  %ret = zext <2 x i1> %trunc to <2 x i64>
+  ret <2 x i64> %ret
+}
+
+; Test a v2i8->v2i64 extension.
+define <2 x i64> @f8(<2 x i64> %val) {
+; CHECK-LABEL: f8:
+; CHECK: vgbm [[REG:%v[0-9]+]], 257
+; CHECK: vn %v24, %v24, [[REG]]
+; CHECK: br %r14
+  %trunc = trunc <2 x i64> %val to <2 x i8>
+  %ret = zext <2 x i8> %trunc to <2 x i64>
+  ret <2 x i64> %ret
+}
+
+; Test a v2i16->v2i64 extension.
+define <2 x i64> @f9(<2 x i64> %val) {
+; CHECK-LABEL: f9:
+; CHECK: vgbm [[REG:%v[0-9]+]], 771
+; CHECK: vn %v24, %v24, [[REG]]
+; CHECK: br %r14
+  %trunc = trunc <2 x i64> %val to <2 x i16>
+  %ret = zext <2 x i16> %trunc to <2 x i64>
+  ret <2 x i64> %ret
+}
+
+; Test a v2i32->v2i64 extension.
+define <2 x i64> @f10(<2 x i64> %val) {
+; CHECK-LABEL: f10:
+; CHECK: vgbm [[REG:%v[0-9]+]], 3855
+; CHECK: vn %v24, %v24, [[REG]]
+; CHECK: br %r14
+  %trunc = trunc <2 x i64> %val to <2 x i32>
+  %ret = zext <2 x i32> %trunc to <2 x i64>
+  ret <2 x i64> %ret
+}
diff --git a/test/CodeGen/SystemZ/vec-args-01.ll b/test/CodeGen/SystemZ/vec-args-01.ll
new file mode 100644
index 00000000000..e07ab7447b2
--- /dev/null
+++ b/test/CodeGen/SystemZ/vec-args-01.ll
@@ -0,0 +1,48 @@
+; Test the handling of named vector arguments.
+;
+; RUN: llc < %s -mtriple=s390x-linux-gnu -mcpu=z13 | FileCheck %s -check-prefix=CHECK-VEC
+; RUN: llc < %s -mtriple=s390x-linux-gnu -mcpu=z13 | FileCheck %s -check-prefix=CHECK-STACK
+
+; This routine has 6 integer arguments, which fill up r2-r5 and
+; the stack slot at offset 160, and 10 vector arguments, which
+; fill up v24-v31 and the two double-wide stack slots at 168
+; and 184.
+declare void @bar(i64, i64, i64, i64, i64, i64,
+                  <4 x i32>, <4 x i32>, <4 x i32>, <4 x i32>,
+                  <4 x i32>, <4 x i32>, <4 x i32>, <4 x i32>,
+                  <4 x i32>, <4 x i32>)
+
+define void @foo() {
+; CHECK-VEC-LABEL: foo:
+; CHECK-VEC-DAG: vrepif %v24, 1
+; CHECK-VEC-DAG: vrepif %v26, 2
+; CHECK-VEC-DAG: vrepif %v28, 3
+; CHECK-VEC-DAG: vrepif %v30, 4
+; CHECK-VEC-DAG: vrepif %v25, 5
+; CHECK-VEC-DAG: vrepif %v27, 6
+; CHECK-VEC-DAG: vrepif %v29, 7
+; CHECK-VEC-DAG: vrepif %v31, 8
+; CHECK-VEC: brasl %r14, bar@PLT
+;
+; CHECK-STACK-LABEL: foo:
+; CHECK-STACK: aghi %r15, -200
+; CHECK-STACK-DAG: mvghi 160(%r15), 6
+; CHECK-STACK-DAG: vrepif [[REG1:%v[0-9]+]], 9
+; CHECK-STACK-DAG: vst [[REG1]], 168(%r15)
+; CHECK-STACK-DAG: vrepif [[REG2:%v[0-9]+]], 10
+; CHECK-STACK-DAG: vst [[REG2]], 184(%r15)
+; CHECK-STACK: brasl %r14, bar@PLT
+
+  call void @bar (i64 1, i64 2, i64 3, i64 4, i64 5, i64 6,
+                  <4 x i32> <i32 1, i32 1, i32 1, i32 1>,
+                  <4 x i32> <i32 2, i32 2, i32 2, i32 2>,
+                  <4 x i32> <i32 3, i32 3, i32 3, i32 3>,
+                  <4 x i32> <i32 4, i32 4, i32 4, i32 4>,
+                  <4 x i32> <i32 5, i32 5, i32 5, i32 5>,
+                  <4 x i32> <i32 6, i32 6, i32 6, i32 6>,
+                  <4 x i32> <i32 7, i32 7, i32 7, i32 7>,
+                  <4 x i32> <i32 8, i32 8, i32 8, i32 8>,
+                  <4 x i32> <i32 9, i32 9, i32 9, i32 9>,
+                  <4 x i32> <i32 10, i32 10, i32 10, i32 10>)
+  ret void
+}
diff --git a/test/CodeGen/SystemZ/vec-args-02.ll b/test/CodeGen/SystemZ/vec-args-02.ll
new file mode 100644
index 00000000000..b6081598326
--- /dev/null
+++ b/test/CodeGen/SystemZ/vec-args-02.ll
@@ -0,0 +1,31 @@
+; Test the handling of unnamed vector arguments.
+;
+; RUN: llc < %s -mtriple=s390x-linux-gnu -mcpu=z13 | FileCheck %s -check-prefix=CHECK-VEC
+; RUN: llc < %s -mtriple=s390x-linux-gnu -mcpu=z13 | FileCheck %s -check-prefix=CHECK-STACK
+
+; This routine is called with two named vector argument (passed
+; in %v24 and %v26) and two unnamed vector arguments (passed
+; in the double-wide stack slots at 160 and 176).
+declare void @bar(<4 x i32>, <4 x i32>, ...)
+
+define void @foo() {
+; CHECK-VEC-LABEL: foo:
+; CHECK-VEC-DAG: vrepif %v24, 1
+; CHECK-VEC-DAG: vrepif %v26, 2
+; CHECK-VEC: brasl %r14, bar@PLT
+;
+; CHECK-STACK-LABEL: foo:
+; CHECK-STACK: aghi %r15, -192
+; CHECK-STACK-DAG: vrepif [[REG1:%v[0-9]+]], 3
+; CHECK-STACK-DAG: vst [[REG1]], 160(%r15)
+; CHECK-STACK-DAG: vrepif [[REG2:%v[0-9]+]], 4
+; CHECK-STACK-DAG: vst [[REG2]], 176(%r15)
+; CHECK-STACK: brasl %r14, bar@PLT
+
+  call void (<4 x i32>, <4 x i32>, ...) @bar
+              (<4 x i32> <i32 1, i32 1, i32 1, i32 1>,
+               <4 x i32> <i32 2, i32 2, i32 2, i32 2>,
+               <4 x i32> <i32 3, i32 3, i32 3, i32 3>,
+               <4 x i32> <i32 4, i32 4, i32 4, i32 4>)
+  ret void
+}
diff --git a/test/CodeGen/SystemZ/vec-args-03.ll b/test/CodeGen/SystemZ/vec-args-03.ll
new file mode 100644
index 00000000000..c47d8461021
--- /dev/null
+++ b/test/CodeGen/SystemZ/vec-args-03.ll
@@ -0,0 +1,30 @@
+; Test the handling of incoming vector arguments.
+;
+; RUN: llc < %s -mtriple=s390x-linux-gnu -mcpu=z13 | FileCheck %s
+
+; This routine has 10 vector arguments, which fill up %v24-%v31 and
+; the two double-wide stack slots at 160 and 176.
+define <4 x i32> @foo(<4 x i32> %v1, <4 x i32> %v2, <4 x i32> %v3, <4 x i32> %v4,
+                      <4 x i32> %v5, <4 x i32> %v6, <4 x i32> %v7, <4 x i32> %v8,
+                      <4 x i32> %v9, <4 x i32> %v10) {
+; CHECK-LABEL: foo:
+; CHECK: vl [[REG1:%v[0-9]+]], 176(%r15)
+; CHECK: vsf %v24, %v26, [[REG1]]
+; CHECK: br %r14
+  %y = sub <4 x i32> %v2, %v10
+  ret <4 x i32> %y
+}
+
+; This routine has 10 vector arguments, which fill up %v24-%v31 and
+; the two single-wide stack slots at 160 and 168.
+define <4 x i8> @bar(<4 x i8> %v1, <4 x i8> %v2, <4 x i8> %v3, <4 x i8> %v4,
+                     <4 x i8> %v5, <4 x i8> %v6, <4 x i8> %v7, <4 x i8> %v8,
+                     <4 x i8> %v9, <4 x i8> %v10) {
+; CHECK-LABEL: bar:
+; CHECK: vlrepg [[REG1:%v[0-9]+]], 168(%r15)
+; CHECK: vsb %v24, %v26, [[REG1]]
+; CHECK: br %r14
+  %y = sub <4 x i8> %v2, %v10
+  ret <4 x i8> %y
+}
+
diff --git a/test/CodeGen/SystemZ/vec-args-04.ll b/test/CodeGen/SystemZ/vec-args-04.ll
new file mode 100644
index 00000000000..3a25404934e
--- /dev/null
+++ b/test/CodeGen/SystemZ/vec-args-04.ll
@@ -0,0 +1,50 @@
+; Test the handling of named short vector arguments.
+;
+; RUN: llc < %s -mtriple=s390x-linux-gnu -mcpu=z13 | FileCheck %s -check-prefix=CHECK-VEC
+; RUN: llc < %s -mtriple=s390x-linux-gnu -mcpu=z13 | FileCheck %s -check-prefix=CHECK-STACK
+
+; This routine has 12 vector arguments, which fill up %v24-%v31
+; and the four single-wide stack slots starting at 160.
+declare void @bar(<1 x i8>, <2 x i8>, <4 x i8>, <8 x i8>,
+                  <1 x i8>, <2 x i8>, <4 x i8>, <8 x i8>,
+                  <1 x i8>, <2 x i8>, <4 x i8>, <8 x i8>)
+
+define void @foo() {
+; CHECK-VEC-LABEL: foo:
+; CHECK-VEC-DAG: vrepib %v24, 1
+; CHECK-VEC-DAG: vrepib %v26, 2
+; CHECK-VEC-DAG: vrepib %v28, 3
+; CHECK-VEC-DAG: vrepib %v30, 4
+; CHECK-VEC-DAG: vrepib %v25, 5
+; CHECK-VEC-DAG: vrepib %v27, 6
+; CHECK-VEC-DAG: vrepib %v29, 7
+; CHECK-VEC-DAG: vrepib %v31, 8
+; CHECK-VEC: brasl %r14, bar@PLT
+;
+; CHECK-STACK-LABEL: foo:
+; CHECK-STACK: aghi %r15, -192
+; CHECK-STACK-DAG: llihh [[REG1:%r[0-9]+]], 2304
+; CHECK-STACK-DAG: stg [[REG1]], 160(%r15)
+; CHECK-STACK-DAG: llihh [[REG2:%r[0-9]+]], 2570
+; CHECK-STACK-DAG: stg [[REG2]], 168(%r15)
+; CHECK-STACK-DAG: llihf [[REG3:%r[0-9]+]], 185273099
+; CHECK-STACK-DAG: stg [[REG3]], 176(%r15)
+; CHECK-STACK-DAG: llihf [[REG4:%r[0-9]+]], 202116108
+; CHECK-STACK-DAG: oilf [[REG4]], 202116108
+; CHECK-STACK-DAG: stg [[REG4]], 176(%r15)
+; CHECK-STACK: brasl %r14, bar@PLT
+
+  call void @bar (<1 x i8> <i8 1>,
+                  <2 x i8> <i8 2, i8 2>,
+                  <4 x i8> <i8 3, i8 3, i8 3, i8 3>,
+                  <8 x i8> <i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4>,
+                  <1 x i8> <i8 5>,
+                  <2 x i8> <i8 6, i8 6>,
+                  <4 x i8> <i8 7, i8 7, i8 7, i8 7>,
+                  <8 x i8> <i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8>,
+                  <1 x i8> <i8 9>,
+                  <2 x i8> <i8 10, i8 10>,
+                  <4 x i8> <i8 11, i8 11, i8 11, i8 11>,
+                  <8 x i8> <i8 12, i8 12, i8 12, i8 12, i8 12, i8 12, i8 12, i8 12>)
+  ret void
+}
diff --git a/test/CodeGen/SystemZ/vec-args-05.ll b/test/CodeGen/SystemZ/vec-args-05.ll
new file mode 100644
index 00000000000..cd1448b8611
--- /dev/null
+++ b/test/CodeGen/SystemZ/vec-args-05.ll
@@ -0,0 +1,32 @@
+; Test the handling of unnamed short vector arguments.
+;
+; RUN: llc < %s -mtriple=s390x-linux-gnu -mcpu=z13 | FileCheck %s -check-prefix=CHECK-VEC
+; RUN: llc < %s -mtriple=s390x-linux-gnu -mcpu=z13 | FileCheck %s -check-prefix=CHECK-STACK
+
+; This routine is called with two named vector argument (passed
+; in %v24 and %v26) and two unnamed vector arguments (passed
+; in the single-wide stack slots at 160 and 168).
+declare void @bar(<4 x i8>, <4 x i8>, ...)
+
+define void @foo() {
+; CHECK-VEC-LABEL: foo:
+; CHECK-VEC-DAG: vrepib %v24, 1
+; CHECK-VEC-DAG: vrepib %v26, 2
+; CHECK-VEC: brasl %r14, bar@PLT
+;
+; CHECK-STACK-LABEL: foo:
+; CHECK-STACK: aghi %r15, -176
+; CHECK-STACK-DAG: llihf [[REG1:%r[0-9]+]], 50529027
+; CHECK-STACK-DAG: stg [[REG1]], 160(%r15)
+; CHECK-STACK-DAG: llihf [[REG2:%r[0-9]+]], 67372036
+; CHECK-STACK-DAG: stg [[REG2]], 168(%r15)
+; CHECK-STACK: brasl %r14, bar@PLT
+
+  call void (<4 x i8>, <4 x i8>, ...) @bar
+              (<4 x i8> <i8 1, i8 1, i8 1, i8 1>,
+               <4 x i8> <i8 2, i8 2, i8 2, i8 2>,
+               <4 x i8> <i8 3, i8 3, i8 3, i8 3>,
+               <4 x i8> <i8 4, i8 4, i8 4, i8 4>)
+  ret void
+}
+
diff --git a/test/CodeGen/SystemZ/vec-args-error-01.ll b/test/CodeGen/SystemZ/vec-args-error-01.ll
new file mode 100644
index 00000000000..e2f53794959
--- /dev/null
+++ b/test/CodeGen/SystemZ/vec-args-error-01.ll
@@ -0,0 +1,9 @@
+; Verify that we detect unsupported single-element vector types.
+
+; RUN: not llc < %s -mtriple=s390x-linux-gnu -mcpu=z13 2>&1 | FileCheck %s
+
+define void @foo(<1 x i128>) {
+  ret void
+}
+
+; CHECK: LLVM ERROR: Unsupported vector argument or return type
diff --git a/test/CodeGen/SystemZ/vec-args-error-02.ll b/test/CodeGen/SystemZ/vec-args-error-02.ll
new file mode 100644
index 00000000000..a5ae1102a74
--- /dev/null
+++ b/test/CodeGen/SystemZ/vec-args-error-02.ll
@@ -0,0 +1,9 @@
+; Verify that we detect unsupported single-element vector types.
+
+; RUN: not llc < %s -mtriple=s390x-linux-gnu -mcpu=z13 2>&1 | FileCheck %s
+
+define <1 x i128> @foo() {
+  ret <1 x i128><i128 0>
+}
+
+; CHECK: LLVM ERROR: Unsupported vector argument or return type
diff --git a/test/CodeGen/SystemZ/vec-args-error-03.ll b/test/CodeGen/SystemZ/vec-args-error-03.ll
new file mode 100644
index 00000000000..14698aae43b
--- /dev/null
+++ b/test/CodeGen/SystemZ/vec-args-error-03.ll
@@ -0,0 +1,12 @@
+; Verify that we detect unsupported single-element vector types.
+
+; RUN: not llc < %s -mtriple=s390x-linux-gnu -mcpu=z13 2>&1 | FileCheck %s
+
+declare void @bar(<1 x i128>)
+
+define void @foo() {
+  call void @bar (<1 x i128> <i128 0>)
+  ret void
+}
+
+; CHECK: LLVM ERROR: Unsupported vector argument or return type
diff --git a/test/CodeGen/SystemZ/vec-args-error-04.ll b/test/CodeGen/SystemZ/vec-args-error-04.ll
new file mode 100644
index 00000000000..a54ee90022c
--- /dev/null
+++ b/test/CodeGen/SystemZ/vec-args-error-04.ll
@@ -0,0 +1,12 @@
+; Verify that we detect unsupported single-element vector types.
+
+; RUN: not llc < %s -mtriple=s390x-linux-gnu -mcpu=z13 2>&1 | FileCheck %s
+
+declare <1 x i128> @bar()
+
+define void @foo() {
+  %res = call <1 x i128> @bar ()
+  ret void
+}
+
+; CHECK: LLVM ERROR: Unsupported vector argument or return type
diff --git a/test/CodeGen/SystemZ/vec-args-error-05.ll b/test/CodeGen/SystemZ/vec-args-error-05.ll
new file mode 100644
index 00000000000..067deb1c88b
--- /dev/null
+++ b/test/CodeGen/SystemZ/vec-args-error-05.ll
@@ -0,0 +1,9 @@
+; Verify that we detect unsupported single-element vector types.
+
+; RUN: not llc < %s -mtriple=s390x-linux-gnu -mcpu=z13 2>&1 | FileCheck %s
+
+define void @foo(<1 x fp128>) {
+  ret void
+}
+
+; CHECK: LLVM ERROR: Unsupported vector argument or return type
diff --git a/test/CodeGen/SystemZ/vec-args-error-06.ll b/test/CodeGen/SystemZ/vec-args-error-06.ll
new file mode 100644
index 00000000000..a9184d73575
--- /dev/null
+++ b/test/CodeGen/SystemZ/vec-args-error-06.ll
@@ -0,0 +1,9 @@
+; Verify that we detect unsupported single-element vector types.
+
+; RUN: not llc < %s -mtriple=s390x-linux-gnu -mcpu=z13 2>&1 | FileCheck %s
+
+define <1 x fp128> @foo() {
+  ret <1 x fp128><fp128 0xL00000000000000000000000000000000>
+}
+
+; CHECK: LLVM ERROR: Unsupported vector argument or return type
diff --git a/test/CodeGen/SystemZ/vec-args-error-07.ll b/test/CodeGen/SystemZ/vec-args-error-07.ll
new file mode 100644
index 00000000000..4e914009391
--- /dev/null
+++ b/test/CodeGen/SystemZ/vec-args-error-07.ll
@@ -0,0 +1,12 @@
+; Verify that we detect unsupported single-element vector types.
+
+; RUN: not llc < %s -mtriple=s390x-linux-gnu -mcpu=z13 2>&1 | FileCheck %s
+
+declare void @bar(<1 x fp128>)
+
+define void @foo() {
+  call void @bar (<1 x fp128> <fp128 0xL00000000000000000000000000000000>)
+  ret void
+}
+
+; CHECK: LLVM ERROR: Unsupported vector argument or return type
diff --git a/test/CodeGen/SystemZ/vec-args-error-08.ll b/test/CodeGen/SystemZ/vec-args-error-08.ll
new file mode 100644
index 00000000000..7b16b9f46e3
--- /dev/null
+++ b/test/CodeGen/SystemZ/vec-args-error-08.ll
@@ -0,0 +1,12 @@
+; Verify that we detect unsupported single-element vector types.
+
+; RUN: not llc < %s -mtriple=s390x-linux-gnu -mcpu=z13 2>&1 | FileCheck %s
+
+declare <1 x fp128> @bar()
+
+define void @foo() {
+  %res = call <1 x fp128> @bar ()
+  ret void
+}
+
+; CHECK: LLVM ERROR: Unsupported vector argument or return type
diff --git a/test/CodeGen/SystemZ/vec-cmp-01.ll b/test/CodeGen/SystemZ/vec-cmp-01.ll
new file mode 100644
index 00000000000..a7546db8d7f
--- /dev/null
+++ b/test/CodeGen/SystemZ/vec-cmp-01.ll
@@ -0,0 +1,228 @@
+; Test v16i8 comparisons.
+;
+; RUN: llc < %s -mtriple=s390x-linux-gnu -mcpu=z13 | FileCheck %s
+
+; Test eq.
+define <16 x i8> @f1(<16 x i8> %dummy, <16 x i8> %val1, <16 x i8> %val2) {
+; CHECK-LABEL: f1:
+; CHECK: vceqb %v24, %v26, %v28
+; CHECK-NEXT: br %r14
+  %cmp = icmp eq <16 x i8> %val1, %val2
+  %ret = sext <16 x i1> %cmp to <16 x i8>
+  ret <16 x i8> %ret
+}
+
+; Test ne.
+define <16 x i8> @f2(<16 x i8> %dummy, <16 x i8> %val1, <16 x i8> %val2) {
+; CHECK-LABEL: f2:
+; CHECK: vceqb [[REG:%v[0-9]+]], %v26, %v28
+; CHECK-NEXT: vno %v24, [[REG]], [[REG]]
+; CHECK-NEXT: br %r14
+  %cmp = icmp ne <16 x i8> %val1, %val2
+  %ret = sext <16 x i1> %cmp to <16 x i8>
+  ret <16 x i8> %ret
+}
+
+; Test sgt.
+define <16 x i8> @f3(<16 x i8> %dummy, <16 x i8> %val1, <16 x i8> %val2) {
+; CHECK-LABEL: f3:
+; CHECK: vchb %v24, %v26, %v28
+; CHECK-NEXT: br %r14
+  %cmp = icmp sgt <16 x i8> %val1, %val2
+  %ret = sext <16 x i1> %cmp to <16 x i8>
+  ret <16 x i8> %ret
+}
+
+; Test sge.
+define <16 x i8> @f4(<16 x i8> %dummy, <16 x i8> %val1, <16 x i8> %val2) {
+; CHECK-LABEL: f4:
+; CHECK: vchb [[REG:%v[0-9]+]], %v28, %v26
+; CHECK-NEXT: vno %v24, [[REG]], [[REG]]
+; CHECK-NEXT: br %r14
+  %cmp = icmp sge <16 x i8> %val1, %val2
+  %ret = sext <16 x i1> %cmp to <16 x i8>
+  ret <16 x i8> %ret
+}
+
+; Test sle.
+define <16 x i8> @f5(<16 x i8> %dummy, <16 x i8> %val1, <16 x i8> %val2) {
+; CHECK-LABEL: f5:
+; CHECK: vchb [[REG:%v[0-9]+]], %v26, %v28
+; CHECK-NEXT: vno %v24, [[REG]], [[REG]]
+; CHECK-NEXT: br %r14
+  %cmp = icmp sle <16 x i8> %val1, %val2
+  %ret = sext <16 x i1> %cmp to <16 x i8>
+  ret <16 x i8> %ret
+}
+
+; Test slt.
+define <16 x i8> @f6(<16 x i8> %dummy, <16 x i8> %val1, <16 x i8> %val2) {
+; CHECK-LABEL: f6:
+; CHECK: vchb %v24, %v28, %v26
+; CHECK-NEXT: br %r14
+  %cmp = icmp slt <16 x i8> %val1, %val2
+  %ret = sext <16 x i1> %cmp to <16 x i8>
+  ret <16 x i8> %ret
+}
+
+; Test ugt.
+define <16 x i8> @f7(<16 x i8> %dummy, <16 x i8> %val1, <16 x i8> %val2) {
+; CHECK-LABEL: f7:
+; CHECK: vchlb %v24, %v26, %v28
+; CHECK-NEXT: br %r14
+  %cmp = icmp ugt <16 x i8> %val1, %val2
+  %ret = sext <16 x i1> %cmp to <16 x i8>
+  ret <16 x i8> %ret
+}
+
+; Test uge.
+define <16 x i8> @f8(<16 x i8> %dummy, <16 x i8> %val1, <16 x i8> %val2) {
+; CHECK-LABEL: f8:
+; CHECK: vchlb [[REG:%v[0-9]+]], %v28, %v26
+; CHECK-NEXT: vno %v24, [[REG]], [[REG]]
+; CHECK-NEXT: br %r14
+  %cmp = icmp uge <16 x i8> %val1, %val2
+  %ret = sext <16 x i1> %cmp to <16 x i8>
+  ret <16 x i8> %ret
+}
+
+; Test ule.
+define <16 x i8> @f9(<16 x i8> %dummy, <16 x i8> %val1, <16 x i8> %val2) {
+; CHECK-LABEL: f9:
+; CHECK: vchlb [[REG:%v[0-9]+]], %v26, %v28
+; CHECK-NEXT: vno %v24, [[REG]], [[REG]]
+; CHECK-NEXT: br %r14
+  %cmp = icmp ule <16 x i8> %val1, %val2
+  %ret = sext <16 x i1> %cmp to <16 x i8>
+  ret <16 x i8> %ret
+}
+
+; Test ult.
+define <16 x i8> @f10(<16 x i8> %dummy, <16 x i8> %val1, <16 x i8> %val2) {
+; CHECK-LABEL: f10:
+; CHECK: vchlb %v24, %v28, %v26
+; CHECK-NEXT: br %r14
+  %cmp = icmp ult <16 x i8> %val1, %val2
+  %ret = sext <16 x i1> %cmp to <16 x i8>
+  ret <16 x i8> %ret
+}
+
+; Test eq selects.
+define <16 x i8> @f11(<16 x i8> %val1, <16 x i8> %val2,
+                      <16 x i8> %val3, <16 x i8> %val4) {
+; CHECK-LABEL: f11:
+; CHECK: vceqb [[REG:%v[0-9]+]], %v24, %v26
+; CHECK-NEXT: vsel %v24, %v28, %v30, [[REG]]
+; CHECK-NEXT: br %r14
+  %cmp = icmp eq <16 x i8> %val1, %val2
+  %ret = select <16 x i1> %cmp, <16 x i8> %val3, <16 x i8> %val4
+  ret <16 x i8> %ret
+}
+
+; Test ne selects.
+define <16 x i8> @f12(<16 x i8> %val1, <16 x i8> %val2,
+                      <16 x i8> %val3, <16 x i8> %val4) {
+; CHECK-LABEL: f12:
+; CHECK: vceqb [[REG:%v[0-9]+]], %v24, %v26
+; CHECK-NEXT: vsel %v24, %v30, %v28, [[REG]]
+; CHECK-NEXT: br %r14
+  %cmp = icmp ne <16 x i8> %val1, %val2
+  %ret = select <16 x i1> %cmp, <16 x i8> %val3, <16 x i8> %val4
+  ret <16 x i8> %ret
+}
+
+; Test sgt selects.
+define <16 x i8> @f13(<16 x i8> %val1, <16 x i8> %val2,
+                      <16 x i8> %val3, <16 x i8> %val4) {
+; CHECK-LABEL: f13:
+; CHECK: vchb [[REG:%v[0-9]+]], %v24, %v26
+; CHECK-NEXT: vsel %v24, %v28, %v30, [[REG]]
+; CHECK-NEXT: br %r14
+  %cmp = icmp sgt <16 x i8> %val1, %val2
+  %ret = select <16 x i1> %cmp, <16 x i8> %val3, <16 x i8> %val4
+  ret <16 x i8> %ret
+}
+
+; Test sge selects.
+define <16 x i8> @f14(<16 x i8> %val1, <16 x i8> %val2,
+                      <16 x i8> %val3, <16 x i8> %val4) {
+; CHECK-LABEL: f14:
+; CHECK: vchb [[REG:%v[0-9]+]], %v26, %v24
+; CHECK-NEXT: vsel %v24, %v30, %v28, [[REG]]
+; CHECK-NEXT: br %r14
+  %cmp = icmp sge <16 x i8> %val1, %val2
+  %ret = select <16 x i1> %cmp, <16 x i8> %val3, <16 x i8> %val4
+  ret <16 x i8> %ret
+}
+
+; Test sle selects.
+define <16 x i8> @f15(<16 x i8> %val1, <16 x i8> %val2,
+                      <16 x i8> %val3, <16 x i8> %val4) {
+; CHECK-LABEL: f15:
+; CHECK: vchb [[REG:%v[0-9]+]], %v24, %v26
+; CHECK-NEXT: vsel %v24, %v30, %v28, [[REG]]
+; CHECK-NEXT: br %r14
+  %cmp = icmp sle <16 x i8> %val1, %val2
+  %ret = select <16 x i1> %cmp, <16 x i8> %val3, <16 x i8> %val4
+  ret <16 x i8> %ret
+}
+
+; Test slt selects.
+define <16 x i8> @f16(<16 x i8> %val1, <16 x i8> %val2,
+                      <16 x i8> %val3, <16 x i8> %val4) {
+; CHECK-LABEL: f16:
+; CHECK: vchb [[REG:%v[0-9]+]], %v26, %v24
+; CHECK-NEXT: vsel %v24, %v28, %v30, [[REG]]
+; CHECK-NEXT: br %r14
+  %cmp = icmp slt <16 x i8> %val1, %val2
+  %ret = select <16 x i1> %cmp, <16 x i8> %val3, <16 x i8> %val4
+  ret <16 x i8> %ret
+}
+
+; Test ugt selects.
+define <16 x i8> @f17(<16 x i8> %val1, <16 x i8> %val2,
+                      <16 x i8> %val3, <16 x i8> %val4) {
+; CHECK-LABEL: f17:
+; CHECK: vchlb [[REG:%v[0-9]+]], %v24, %v26
+; CHECK-NEXT: vsel %v24, %v28, %v30, [[REG]]
+; CHECK-NEXT: br %r14
+  %cmp = icmp ugt <16 x i8> %val1, %val2
+  %ret = select <16 x i1> %cmp, <16 x i8> %val3, <16 x i8> %val4
+  ret <16 x i8> %ret
+}
+
+; Test uge selects.
+define <16 x i8> @f18(<16 x i8> %val1, <16 x i8> %val2,
+                      <16 x i8> %val3, <16 x i8> %val4) {
+; CHECK-LABEL: f18:
+; CHECK: vchlb [[REG:%v[0-9]+]], %v26, %v24
+; CHECK-NEXT: vsel %v24, %v30, %v28, [[REG]]
+; CHECK-NEXT: br %r14
+  %cmp = icmp uge <16 x i8> %val1, %val2
+  %ret = select <16 x i1> %cmp, <16 x i8> %val3, <16 x i8> %val4
+  ret <16 x i8> %ret
+}
+
+; Test ule selects.
+define <16 x i8> @f19(<16 x i8> %val1, <16 x i8> %val2,
+                      <16 x i8> %val3, <16 x i8> %val4) {
+; CHECK-LABEL: f19:
+; CHECK: vchlb [[REG:%v[0-9]+]], %v24, %v26
+; CHECK-NEXT: vsel %v24, %v30, %v28, [[REG]]
+; CHECK-NEXT: br %r14
+  %cmp = icmp ule <16 x i8> %val1, %val2
+  %ret = select <16 x i1> %cmp, <16 x i8> %val3, <16 x i8> %val4
+  ret <16 x i8> %ret
+}
+
+; Test ult selects.
+define <16 x i8> @f20(<16 x i8> %val1, <16 x i8> %val2,
+                      <16 x i8> %val3, <16 x i8> %val4) {
+; CHECK-LABEL: f20:
+; CHECK: vchlb [[REG:%v[0-9]+]], %v26, %v24
+; CHECK-NEXT: vsel %v24, %v28, %v30, [[REG]]
+; CHECK-NEXT: br %r14
+  %cmp = icmp ult <16 x i8> %val1, %val2
+  %ret = select <16 x i1> %cmp, <16 x i8> %val3, <16 x i8> %val4
+  ret <16 x i8> %ret
+}
diff --git a/test/CodeGen/SystemZ/vec-cmp-02.ll b/test/CodeGen/SystemZ/vec-cmp-02.ll
new file mode 100644
index 00000000000..78fb46c01c0
--- /dev/null
+++ b/test/CodeGen/SystemZ/vec-cmp-02.ll
@@ -0,0 +1,228 @@
+; Test v8i16 comparisons.
+;
+; RUN: llc < %s -mtriple=s390x-linux-gnu -mcpu=z13 | FileCheck %s
+
+; Test eq.
+define <8 x i16> @f1(<8 x i16> %dummy, <8 x i16> %val1, <8 x i16> %val2) {
+; CHECK-LABEL: f1:
+; CHECK: vceqh %v24, %v26, %v28
+; CHECK-NEXT: br %r14
+  %cmp = icmp eq <8 x i16> %val1, %val2
+  %ret = sext <8 x i1> %cmp to <8 x i16>
+  ret <8 x i16> %ret
+}
+
+; Test ne.
+define <8 x i16> @f2(<8 x i16> %dummy, <8 x i16> %val1, <8 x i16> %val2) {
+; CHECK-LABEL: f2:
+; CHECK: vceqh [[REG:%v[0-9]+]], %v26, %v28
+; CHECK-NEXT: vno %v24, [[REG]], [[REG]]
+; CHECK-NEXT: br %r14
+  %cmp = icmp ne <8 x i16> %val1, %val2
+  %ret = sext <8 x i1> %cmp to <8 x i16>
+  ret <8 x i16> %ret
+}
+
+; Test sgt.
+define <8 x i16> @f3(<8 x i16> %dummy, <8 x i16> %val1, <8 x i16> %val2) {
+; CHECK-LABEL: f3:
+; CHECK: vchh %v24, %v26, %v28
+; CHECK-NEXT: br %r14
+  %cmp = icmp sgt <8 x i16> %val1, %val2
+  %ret = sext <8 x i1> %cmp to <8 x i16>
+  ret <8 x i16> %ret
+}
+
+; Test sge.
+define <8 x i16> @f4(<8 x i16> %dummy, <8 x i16> %val1, <8 x i16> %val2) {
+; CHECK-LABEL: f4:
+; CHECK: vchh [[REG:%v[0-9]+]], %v28, %v26
+; CHECK-NEXT: vno %v24, [[REG]], [[REG]]
+; CHECK-NEXT: br %r14
+  %cmp = icmp sge <8 x i16> %val1, %val2
+  %ret = sext <8 x i1> %cmp to <8 x i16>
+  ret <8 x i16> %ret
+}
+
+; Test sle.
+define <8 x i16> @f5(<8 x i16> %dummy, <8 x i16> %val1, <8 x i16> %val2) {
+; CHECK-LABEL: f5:
+; CHECK: vchh [[REG:%v[0-9]+]], %v26, %v28
+; CHECK-NEXT: vno %v24, [[REG]], [[REG]]
+; CHECK-NEXT: br %r14
+  %cmp = icmp sle <8 x i16> %val1, %val2
+  %ret = sext <8 x i1> %cmp to <8 x i16>
+  ret <8 x i16> %ret
+}
+
+; Test slt.
+define <8 x i16> @f6(<8 x i16> %dummy, <8 x i16> %val1, <8 x i16> %val2) {
+; CHECK-LABEL: f6:
+; CHECK: vchh %v24, %v28, %v26
+; CHECK-NEXT: br %r14
+  %cmp = icmp slt <8 x i16> %val1, %val2
+  %ret = sext <8 x i1> %cmp to <8 x i16>
+  ret <8 x i16> %ret
+}
+
+; Test ugt.
+define <8 x i16> @f7(<8 x i16> %dummy, <8 x i16> %val1, <8 x i16> %val2) {
+; CHECK-LABEL: f7:
+; CHECK: vchlh %v24, %v26, %v28
+; CHECK-NEXT: br %r14
+  %cmp = icmp ugt <8 x i16> %val1, %val2
+  %ret = sext <8 x i1> %cmp to <8 x i16>
+  ret <8 x i16> %ret
+}
+
+; Test uge.
+define <8 x i16> @f8(<8 x i16> %dummy, <8 x i16> %val1, <8 x i16> %val2) {
+; CHECK-LABEL: f8:
+; CHECK: vchlh [[REG:%v[0-9]+]], %v28, %v26
+; CHECK-NEXT: vno %v24, [[REG]], [[REG]]
+; CHECK-NEXT: br %r14
+  %cmp = icmp uge <8 x i16> %val1, %val2
+  %ret = sext <8 x i1> %cmp to <8 x i16>
+  ret <8 x i16> %ret
+}
+
+; Test ule.
+define <8 x i16> @f9(<8 x i16> %dummy, <8 x i16> %val1, <8 x i16> %val2) {
+; CHECK-LABEL: f9:
+; CHECK: vchlh [[REG:%v[0-9]+]], %v26, %v28
+; CHECK-NEXT: vno %v24, [[REG]], [[REG]]
+; CHECK-NEXT: br %r14
+  %cmp = icmp ule <8 x i16> %val1, %val2
+  %ret = sext <8 x i1> %cmp to <8 x i16>
+  ret <8 x i16> %ret
+}
+
+; Test ult.
+define <8 x i16> @f10(<8 x i16> %dummy, <8 x i16> %val1, <8 x i16> %val2) {
+; CHECK-LABEL: f10:
+; CHECK: vchlh %v24, %v28, %v26
+; CHECK-NEXT: br %r14
+  %cmp = icmp ult <8 x i16> %val1, %val2
+  %ret = sext <8 x i1> %cmp to <8 x i16>
+  ret <8 x i16> %ret
+}
+
+; Test eq selects.
+define <8 x i16> @f11(<8 x i16> %val1, <8 x i16> %val2,
+                      <8 x i16> %val3, <8 x i16> %val4) {
+; CHECK-LABEL: f11:
+; CHECK: vceqh [[REG:%v[0-9]+]], %v24, %v26
+; CHECK-NEXT: vsel %v24, %v28, %v30, [[REG]]
+; CHECK-NEXT: br %r14
+  %cmp = icmp eq <8 x i16> %val1, %val2
+  %ret = select <8 x i1> %cmp, <8 x i16> %val3, <8 x i16> %val4
+  ret <8 x i16> %ret
+}
+
+; Test ne selects.
+define <8 x i16> @f12(<8 x i16> %val1, <8 x i16> %val2,
+                      <8 x i16> %val3, <8 x i16> %val4) {
+; CHECK-LABEL: f12:
+; CHECK: vceqh [[REG:%v[0-9]+]], %v24, %v26
+; CHECK-NEXT: vsel %v24, %v30, %v28, [[REG]]
+; CHECK-NEXT: br %r14
+  %cmp = icmp ne <8 x i16> %val1, %val2
+  %ret = select <8 x i1> %cmp, <8 x i16> %val3, <8 x i16> %val4
+  ret <8 x i16> %ret
+}
+
+; Test sgt selects.
+define <8 x i16> @f13(<8 x i16> %val1, <8 x i16> %val2,
+                      <8 x i16> %val3, <8 x i16> %val4) {
+; CHECK-LABEL: f13:
+; CHECK: vchh [[REG:%v[0-9]+]], %v24, %v26
+; CHECK-NEXT: vsel %v24, %v28, %v30, [[REG]]
+; CHECK-NEXT: br %r14
+  %cmp = icmp sgt <8 x i16> %val1, %val2
+  %ret = select <8 x i1> %cmp, <8 x i16> %val3, <8 x i16> %val4
+  ret <8 x i16> %ret
+}
+
+; Test sge selects.
+define <8 x i16> @f14(<8 x i16> %val1, <8 x i16> %val2,
+                      <8 x i16> %val3, <8 x i16> %val4) {
+; CHECK-LABEL: f14:
+; CHECK: vchh [[REG:%v[0-9]+]], %v26, %v24
+; CHECK-NEXT: vsel %v24, %v30, %v28, [[REG]]
+; CHECK-NEXT: br %r14
+  %cmp = icmp sge <8 x i16> %val1, %val2
+  %ret = select <8 x i1> %cmp, <8 x i16> %val3, <8 x i16> %val4
+  ret <8 x i16> %ret
+}
+
+; Test sle selects.
+define <8 x i16> @f15(<8 x i16> %val1, <8 x i16> %val2,
+                      <8 x i16> %val3, <8 x i16> %val4) {
+; CHECK-LABEL: f15:
+; CHECK: vchh [[REG:%v[0-9]+]], %v24, %v26
+; CHECK-NEXT: vsel %v24, %v30, %v28, [[REG]]
+; CHECK-NEXT: br %r14
+  %cmp = icmp sle <8 x i16> %val1, %val2
+  %ret = select <8 x i1> %cmp, <8 x i16> %val3, <8 x i16> %val4
+  ret <8 x i16> %ret
+}
+
+; Test slt selects.
+define <8 x i16> @f16(<8 x i16> %val1, <8 x i16> %val2,
+                      <8 x i16> %val3, <8 x i16> %val4) {
+; CHECK-LABEL: f16:
+; CHECK: vchh [[REG:%v[0-9]+]], %v26, %v24
+; CHECK-NEXT: vsel %v24, %v28, %v30, [[REG]]
+; CHECK-NEXT: br %r14
+  %cmp = icmp slt <8 x i16> %val1, %val2
+  %ret = select <8 x i1> %cmp, <8 x i16> %val3, <8 x i16> %val4
+  ret <8 x i16> %ret
+}
+
+; Test ugt selects.
+define <8 x i16> @f17(<8 x i16> %val1, <8 x i16> %val2,
+                      <8 x i16> %val3, <8 x i16> %val4) {
+; CHECK-LABEL: f17:
+; CHECK: vchlh [[REG:%v[0-9]+]], %v24, %v26
+; CHECK-NEXT: vsel %v24, %v28, %v30, [[REG]]
+; CHECK-NEXT: br %r14
+  %cmp = icmp ugt <8 x i16> %val1, %val2
+  %ret = select <8 x i1> %cmp, <8 x i16> %val3, <8 x i16> %val4
+  ret <8 x i16> %ret
+}
+
+; Test uge selects.
+define <8 x i16> @f18(<8 x i16> %val1, <8 x i16> %val2,
+                      <8 x i16> %val3, <8 x i16> %val4) {
+; CHECK-LABEL: f18:
+; CHECK: vchlh [[REG:%v[0-9]+]], %v26, %v24
+; CHECK-NEXT: vsel %v24, %v30, %v28, [[REG]]
+; CHECK-NEXT: br %r14
+  %cmp = icmp uge <8 x i16> %val1, %val2
+  %ret = select <8 x i1> %cmp, <8 x i16> %val3, <8 x i16> %val4
+  ret <8 x i16> %ret
+}
+
+; Test ule selects.
+define <8 x i16> @f19(<8 x i16> %val1, <8 x i16> %val2,
+                      <8 x i16> %val3, <8 x i16> %val4) {
+; CHECK-LABEL: f19:
+; CHECK: vchlh [[REG:%v[0-9]+]], %v24, %v26
+; CHECK-NEXT: vsel %v24, %v30, %v28, [[REG]]
+; CHECK-NEXT: br %r14
+  %cmp = icmp ule <8 x i16> %val1, %val2
+  %ret = select <8 x i1> %cmp, <8 x i16> %val3, <8 x i16> %val4
+  ret <8 x i16> %ret
+}
+
+; Test ult selects.
+define <8 x i16> @f20(<8 x i16> %val1, <8 x i16> %val2,
+                      <8 x i16> %val3, <8 x i16> %val4) {
+; CHECK-LABEL: f20:
+; CHECK: vchlh [[REG:%v[0-9]+]], %v26, %v24
+; CHECK-NEXT: vsel %v24, %v28, %v30, [[REG]]
+; CHECK-NEXT: br %r14
+  %cmp = icmp ult <8 x i16> %val1, %val2
+  %ret = select <8 x i1> %cmp, <8 x i16> %val3, <8 x i16> %val4
+  ret <8 x i16> %ret
+}
diff --git a/test/CodeGen/SystemZ/vec-cmp-03.ll b/test/CodeGen/SystemZ/vec-cmp-03.ll
new file mode 100644
index 00000000000..4b070acc935
--- /dev/null
+++ b/test/CodeGen/SystemZ/vec-cmp-03.ll
@@ -0,0 +1,228 @@
+; Test v4i32 comparisons.
+;
+; RUN: llc < %s -mtriple=s390x-linux-gnu -mcpu=z13 | FileCheck %s
+
+; Test eq.
+define <4 x i32> @f1(<4 x i32> %dummy, <4 x i32> %val1, <4 x i32> %val2) {
+; CHECK-LABEL: f1:
+; CHECK: vceqf %v24, %v26, %v28
+; CHECK-NEXT: br %r14
+  %cmp = icmp eq <4 x i32> %val1, %val2
+  %ret = sext <4 x i1> %cmp to <4 x i32>
+  ret <4 x i32> %ret
+}
+
+; Test ne.
+define <4 x i32> @f2(<4 x i32> %dummy, <4 x i32> %val1, <4 x i32> %val2) {
+; CHECK-LABEL: f2:
+; CHECK: vceqf [[REG:%v[0-9]+]], %v26, %v28
+; CHECK-NEXT: vno %v24, [[REG]], [[REG]]
+; CHECK-NEXT: br %r14
+  %cmp = icmp ne <4 x i32> %val1, %val2
+  %ret = sext <4 x i1> %cmp to <4 x i32>
+  ret <4 x i32> %ret
+}
+
+; Test sgt.
+define <4 x i32> @f3(<4 x i32> %dummy, <4 x i32> %val1, <4 x i32> %val2) {
+; CHECK-LABEL: f3:
+; CHECK: vchf %v24, %v26, %v28
+; CHECK-NEXT: br %r14
+  %cmp = icmp sgt <4 x i32> %val1, %val2
+  %ret = sext <4 x i1> %cmp to <4 x i32>
+  ret <4 x i32> %ret
+}
+
+; Test sge.
+define <4 x i32> @f4(<4 x i32> %dummy, <4 x i32> %val1, <4 x i32> %val2) {
+; CHECK-LABEL: f4:
+; CHECK: vchf [[REG:%v[0-9]+]], %v28, %v26
+; CHECK-NEXT: vno %v24, [[REG]], [[REG]]
+; CHECK-NEXT: br %r14
+  %cmp = icmp sge <4 x i32> %val1, %val2
+  %ret = sext <4 x i1> %cmp to <4 x i32>
+  ret <4 x i32> %ret
+}
+
+; Test sle.
+define <4 x i32> @f5(<4 x i32> %dummy, <4 x i32> %val1, <4 x i32> %val2) {
+; CHECK-LABEL: f5:
+; CHECK: vchf [[REG:%v[0-9]+]], %v26, %v28
+; CHECK-NEXT: vno %v24, [[REG]], [[REG]]
+; CHECK-NEXT: br %r14
+  %cmp = icmp sle <4 x i32> %val1, %val2
+  %ret = sext <4 x i1> %cmp to <4 x i32>
+  ret <4 x i32> %ret
+}
+
+; Test slt.
+define <4 x i32> @f6(<4 x i32> %dummy, <4 x i32> %val1, <4 x i32> %val2) {
+; CHECK-LABEL: f6:
+; CHECK: vchf %v24, %v28, %v26
+; CHECK-NEXT: br %r14
+  %cmp = icmp slt <4 x i32> %val1, %val2
+  %ret = sext <4 x i1> %cmp to <4 x i32>
+  ret <4 x i32> %ret
+}
+
+; Test ugt.
+define <4 x i32> @f7(<4 x i32> %dummy, <4 x i32> %val1, <4 x i32> %val2) {
+; CHECK-LABEL: f7:
+; CHECK: vchlf %v24, %v26, %v28
+; CHECK-NEXT: br %r14
+  %cmp = icmp ugt <4 x i32> %val1, %val2
+  %ret = sext <4 x i1> %cmp to <4 x i32>
+  ret <4 x i32> %ret
+}
+
+; Test uge.
+define <4 x i32> @f8(<4 x i32> %dummy, <4 x i32> %val1, <4 x i32> %val2) {
+; CHECK-LABEL: f8:
+; CHECK: vchlf [[REG:%v[0-9]+]], %v28, %v26
+; CHECK-NEXT: vno %v24, [[REG]], [[REG]]
+; CHECK-NEXT: br %r14
+  %cmp = icmp uge <4 x i32> %val1, %val2
+  %ret = sext <4 x i1> %cmp to <4 x i32>
+  ret <4 x i32> %ret
+}
+
+; Test ule.
+define <4 x i32> @f9(<4 x i32> %dummy, <4 x i32> %val1, <4 x i32> %val2) {
+; CHECK-LABEL: f9:
+; CHECK: vchlf [[REG:%v[0-9]+]], %v26, %v28
+; CHECK-NEXT: vno %v24, [[REG]], [[REG]]
+; CHECK-NEXT: br %r14
+  %cmp = icmp ule <4 x i32> %val1, %val2
+  %ret = sext <4 x i1> %cmp to <4 x i32>
+  ret <4 x i32> %ret
+}
+
+; Test ult.
+define <4 x i32> @f10(<4 x i32> %dummy, <4 x i32> %val1, <4 x i32> %val2) {
+; CHECK-LABEL: f10:
+; CHECK: vchlf %v24, %v28, %v26
+; CHECK-NEXT: br %r14
+  %cmp = icmp ult <4 x i32> %val1, %val2
+  %ret = sext <4 x i1> %cmp to <4 x i32>
+  ret <4 x i32> %ret
+}
+
+; Test eq selects.
+define <4 x i32> @f11(<4 x i32> %val1, <4 x i32> %val2,
+                      <4 x i32> %val3, <4 x i32> %val4) {
+; CHECK-LABEL: f11:
+; CHECK: vceqf [[REG:%v[0-9]+]], %v24, %v26
+; CHECK-NEXT: vsel %v24, %v28, %v30, [[REG]]
+; CHECK-NEXT: br %r14
+  %cmp = icmp eq <4 x i32> %val1, %val2
+  %ret = select <4 x i1> %cmp, <4 x i32> %val3, <4 x i32> %val4
+  ret <4 x i32> %ret
+}
+
+; Test ne selects.
+define <4 x i32> @f12(<4 x i32> %val1, <4 x i32> %val2,
+                      <4 x i32> %val3, <4 x i32> %val4) {
+; CHECK-LABEL: f12:
+; CHECK: vceqf [[REG:%v[0-9]+]], %v24, %v26
+; CHECK-NEXT: vsel %v24, %v30, %v28, [[REG]]
+; CHECK-NEXT: br %r14
+  %cmp = icmp ne <4 x i32> %val1, %val2
+  %ret = select <4 x i1> %cmp, <4 x i32> %val3, <4 x i32> %val4
+  ret <4 x i32> %ret
+}
+
+; Test sgt selects.
+define <4 x i32> @f13(<4 x i32> %val1, <4 x i32> %val2,
+                      <4 x i32> %val3, <4 x i32> %val4) {
+; CHECK-LABEL: f13:
+; CHECK: vchf [[REG:%v[0-9]+]], %v24, %v26
+; CHECK-NEXT: vsel %v24, %v28, %v30, [[REG]]
+; CHECK-NEXT: br %r14
+  %cmp = icmp sgt <4 x i32> %val1, %val2
+  %ret = select <4 x i1> %cmp, <4 x i32> %val3, <4 x i32> %val4
+  ret <4 x i32> %ret
+}
+
+; Test sge selects.
+define <4 x i32> @f14(<4 x i32> %val1, <4 x i32> %val2,
+                      <4 x i32> %val3, <4 x i32> %val4) {
+; CHECK-LABEL: f14:
+; CHECK: vchf [[REG:%v[0-9]+]], %v26, %v24
+; CHECK-NEXT: vsel %v24, %v30, %v28, [[REG]]
+; CHECK-NEXT: br %r14
+  %cmp = icmp sge <4 x i32> %val1, %val2
+  %ret = select <4 x i1> %cmp, <4 x i32> %val3, <4 x i32> %val4
+  ret <4 x i32> %ret
+}
+
+; Test sle selects.
+define <4 x i32> @f15(<4 x i32> %val1, <4 x i32> %val2,
+                      <4 x i32> %val3, <4 x i32> %val4) {
+; CHECK-LABEL: f15:
+; CHECK: vchf [[REG:%v[0-9]+]], %v24, %v26
+; CHECK-NEXT: vsel %v24, %v30, %v28, [[REG]]
+; CHECK-NEXT: br %r14
+  %cmp = icmp sle <4 x i32> %val1, %val2
+  %ret = select <4 x i1> %cmp, <4 x i32> %val3, <4 x i32> %val4
+  ret <4 x i32> %ret
+}
+
+; Test slt selects.
+define <4 x i32> @f16(<4 x i32> %val1, <4 x i32> %val2,
+                      <4 x i32> %val3, <4 x i32> %val4) {
+; CHECK-LABEL: f16:
+; CHECK: vchf [[REG:%v[0-9]+]], %v26, %v24
+; CHECK-NEXT: vsel %v24, %v28, %v30, [[REG]]
+; CHECK-NEXT: br %r14
+  %cmp = icmp slt <4 x i32> %val1, %val2
+  %ret = select <4 x i1> %cmp, <4 x i32> %val3, <4 x i32> %val4
+  ret <4 x i32> %ret
+}
+
+; Test ugt selects.
+define <4 x i32> @f17(<4 x i32> %val1, <4 x i32> %val2,
+                      <4 x i32> %val3, <4 x i32> %val4) {
+; CHECK-LABEL: f17:
+; CHECK: vchlf [[REG:%v[0-9]+]], %v24, %v26
+; CHECK-NEXT: vsel %v24, %v28, %v30, [[REG]]
+; CHECK-NEXT: br %r14
+  %cmp = icmp ugt <4 x i32> %val1, %val2
+  %ret = select <4 x i1> %cmp, <4 x i32> %val3, <4 x i32> %val4
+  ret <4 x i32> %ret
+}
+
+; Test uge selects.
+define <4 x i32> @f18(<4 x i32> %val1, <4 x i32> %val2,
+                      <4 x i32> %val3, <4 x i32> %val4) {
+; CHECK-LABEL: f18:
+; CHECK: vchlf [[REG:%v[0-9]+]], %v26, %v24
+; CHECK-NEXT: vsel %v24, %v30, %v28, [[REG]]
+; CHECK-NEXT: br %r14
+  %cmp = icmp uge <4 x i32> %val1, %val2
+  %ret = select <4 x i1> %cmp, <4 x i32> %val3, <4 x i32> %val4
+  ret <4 x i32> %ret
+}
+
+; Test ule selects.
+define <4 x i32> @f19(<4 x i32> %val1, <4 x i32> %val2,
+                      <4 x i32> %val3, <4 x i32> %val4) {
+; CHECK-LABEL: f19:
+; CHECK: vchlf [[REG:%v[0-9]+]], %v24, %v26
+; CHECK-NEXT: vsel %v24, %v30, %v28, [[REG]]
+; CHECK-NEXT: br %r14
+  %cmp = icmp ule <4 x i32> %val1, %val2
+  %ret = select <4 x i1> %cmp, <4 x i32> %val3, <4 x i32> %val4
+  ret <4 x i32> %ret
+}
+
+; Test ult selects.
+define <4 x i32> @f20(<4 x i32> %val1, <4 x i32> %val2,
+                      <4 x i32> %val3, <4 x i32> %val4) {
+; CHECK-LABEL: f20:
+; CHECK: vchlf [[REG:%v[0-9]+]], %v26, %v24
+; CHECK-NEXT: vsel %v24, %v28, %v30, [[REG]]
+; CHECK-NEXT: br %r14
+  %cmp = icmp ult <4 x i32> %val1, %val2
+  %ret = select <4 x i1> %cmp, <4 x i32> %val3, <4 x i32> %val4
+  ret <4 x i32> %ret
+}
diff --git a/test/CodeGen/SystemZ/vec-cmp-04.ll b/test/CodeGen/SystemZ/vec-cmp-04.ll
new file mode 100644
index 00000000000..5cecaa7251b
--- /dev/null
+++ b/test/CodeGen/SystemZ/vec-cmp-04.ll
@@ -0,0 +1,228 @@
+; Test v2i64 comparisons.
+;
+; RUN: llc < %s -mtriple=s390x-linux-gnu -mcpu=z13 | FileCheck %s
+
+; Test eq.
+define <2 x i64> @f1(<2 x i64> %dummy, <2 x i64> %val1, <2 x i64> %val2) {
+; CHECK-LABEL: f1:
+; CHECK: vceqg %v24, %v26, %v28
+; CHECK-NEXT: br %r14
+  %cmp = icmp eq <2 x i64> %val1, %val2
+  %ret = sext <2 x i1> %cmp to <2 x i64>
+  ret <2 x i64> %ret
+}
+
+; Test ne.
+define <2 x i64> @f2(<2 x i64> %dummy, <2 x i64> %val1, <2 x i64> %val2) {
+; CHECK-LABEL: f2:
+; CHECK: vceqg [[REG:%v[0-9]+]], %v26, %v28
+; CHECK-NEXT: vno %v24, [[REG]], [[REG]]
+; CHECK-NEXT: br %r14
+  %cmp = icmp ne <2 x i64> %val1, %val2
+  %ret = sext <2 x i1> %cmp to <2 x i64>
+  ret <2 x i64> %ret
+}
+
+; Test sgt.
+define <2 x i64> @f3(<2 x i64> %dummy, <2 x i64> %val1, <2 x i64> %val2) {
+; CHECK-LABEL: f3:
+; CHECK: vchg %v24, %v26, %v28
+; CHECK-NEXT: br %r14
+  %cmp = icmp sgt <2 x i64> %val1, %val2
+  %ret = sext <2 x i1> %cmp to <2 x i64>
+  ret <2 x i64> %ret
+}
+
+; Test sge.
+define <2 x i64> @f4(<2 x i64> %dummy, <2 x i64> %val1, <2 x i64> %val2) {
+; CHECK-LABEL: f4:
+; CHECK: vchg [[REG:%v[0-9]+]], %v28, %v26
+; CHECK-NEXT: vno %v24, [[REG]], [[REG]]
+; CHECK-NEXT: br %r14
+  %cmp = icmp sge <2 x i64> %val1, %val2
+  %ret = sext <2 x i1> %cmp to <2 x i64>
+  ret <2 x i64> %ret
+}
+
+; Test sle.
+define <2 x i64> @f5(<2 x i64> %dummy, <2 x i64> %val1, <2 x i64> %val2) {
+; CHECK-LABEL: f5:
+; CHECK: vchg [[REG:%v[0-9]+]], %v26, %v28
+; CHECK-NEXT: vno %v24, [[REG]], [[REG]]
+; CHECK-NEXT: br %r14
+  %cmp = icmp sle <2 x i64> %val1, %val2
+  %ret = sext <2 x i1> %cmp to <2 x i64>
+  ret <2 x i64> %ret
+}
+
+; Test slt.
+define <2 x i64> @f6(<2 x i64> %dummy, <2 x i64> %val1, <2 x i64> %val2) {
+; CHECK-LABEL: f6:
+; CHECK: vchg %v24, %v28, %v26
+; CHECK-NEXT: br %r14
+  %cmp = icmp slt <2 x i64> %val1, %val2
+  %ret = sext <2 x i1> %cmp to <2 x i64>
+  ret <2 x i64> %ret
+}
+
+; Test ugt.
+define <2 x i64> @f7(<2 x i64> %dummy, <2 x i64> %val1, <2 x i64> %val2) {
+; CHECK-LABEL: f7:
+; CHECK: vchlg %v24, %v26, %v28
+; CHECK-NEXT: br %r14
+  %cmp = icmp ugt <2 x i64> %val1, %val2
+  %ret = sext <2 x i1> %cmp to <2 x i64>
+  ret <2 x i64> %ret
+}
+
+; Test uge.
+define <2 x i64> @f8(<2 x i64> %dummy, <2 x i64> %val1, <2 x i64> %val2) {
+; CHECK-LABEL: f8:
+; CHECK: vchlg [[REG:%v[0-9]+]], %v28, %v26
+; CHECK-NEXT: vno %v24, [[REG]], [[REG]]
+; CHECK-NEXT: br %r14
+  %cmp = icmp uge <2 x i64> %val1, %val2
+  %ret = sext <2 x i1> %cmp to <2 x i64>
+  ret <2 x i64> %ret
+}
+
+; Test ule.
+define <2 x i64> @f9(<2 x i64> %dummy, <2 x i64> %val1, <2 x i64> %val2) {
+; CHECK-LABEL: f9:
+; CHECK: vchlg [[REG:%v[0-9]+]], %v26, %v28
+; CHECK-NEXT: vno %v24, [[REG]], [[REG]]
+; CHECK-NEXT: br %r14
+  %cmp = icmp ule <2 x i64> %val1, %val2
+  %ret = sext <2 x i1> %cmp to <2 x i64>
+  ret <2 x i64> %ret
+}
+
+; Test ult.
+define <2 x i64> @f10(<2 x i64> %dummy, <2 x i64> %val1, <2 x i64> %val2) {
+; CHECK-LABEL: f10:
+; CHECK: vchlg %v24, %v28, %v26
+; CHECK-NEXT: br %r14
+  %cmp = icmp ult <2 x i64> %val1, %val2
+  %ret = sext <2 x i1> %cmp to <2 x i64>
+  ret <2 x i64> %ret
+}
+
+; Test eq selects.
+define <2 x i64> @f11(<2 x i64> %val1, <2 x i64> %val2,
+                      <2 x i64> %val3, <2 x i64> %val4) {
+; CHECK-LABEL: f11:
+; CHECK: vceqg [[REG:%v[0-9]+]], %v24, %v26
+; CHECK-NEXT: vsel %v24, %v28, %v30, [[REG]]
+; CHECK-NEXT: br %r14
+  %cmp = icmp eq <2 x i64> %val1, %val2
+  %ret = select <2 x i1> %cmp, <2 x i64> %val3, <2 x i64> %val4
+  ret <2 x i64> %ret
+}
+
+; Test ne selects.
+define <2 x i64> @f12(<2 x i64> %val1, <2 x i64> %val2,
+                      <2 x i64> %val3, <2 x i64> %val4) {
+; CHECK-LABEL: f12:
+; CHECK: vceqg [[REG:%v[0-9]+]], %v24, %v26
+; CHECK-NEXT: vsel %v24, %v30, %v28, [[REG]]
+; CHECK-NEXT: br %r14
+  %cmp = icmp ne <2 x i64> %val1, %val2
+  %ret = select <2 x i1> %cmp, <2 x i64> %val3, <2 x i64> %val4
+  ret <2 x i64> %ret
+}
+
+; Test sgt selects.
+define <2 x i64> @f13(<2 x i64> %val1, <2 x i64> %val2,
+                      <2 x i64> %val3, <2 x i64> %val4) {
+; CHECK-LABEL: f13:
+; CHECK: vchg [[REG:%v[0-9]+]], %v24, %v26
+; CHECK-NEXT: vsel %v24, %v28, %v30, [[REG]]
+; CHECK-NEXT: br %r14
+  %cmp = icmp sgt <2 x i64> %val1, %val2
+  %ret = select <2 x i1> %cmp, <2 x i64> %val3, <2 x i64> %val4
+  ret <2 x i64> %ret
+}
+
+; Test sge selects.
+define <2 x i64> @f14(<2 x i64> %val1, <2 x i64> %val2,
+                      <2 x i64> %val3, <2 x i64> %val4) {
+; CHECK-LABEL: f14:
+; CHECK: vchg [[REG:%v[0-9]+]], %v26, %v24
+; CHECK-NEXT: vsel %v24, %v30, %v28, [[REG]]
+; CHECK-NEXT: br %r14
+  %cmp = icmp sge <2 x i64> %val1, %val2
+  %ret = select <2 x i1> %cmp, <2 x i64> %val3, <2 x i64> %val4
+  ret <2 x i64> %ret
+}
+
+; Test sle selects.
+define <2 x i64> @f15(<2 x i64> %val1, <2 x i64> %val2,
+                      <2 x i64> %val3, <2 x i64> %val4) {
+; CHECK-LABEL: f15:
+; CHECK: vchg [[REG:%v[0-9]+]], %v24, %v26
+; CHECK-NEXT: vsel %v24, %v30, %v28, [[REG]]
+; CHECK-NEXT: br %r14
+  %cmp = icmp sle <2 x i64> %val1, %val2
+  %ret = select <2 x i1> %cmp, <2 x i64> %val3, <2 x i64> %val4
+  ret <2 x i64> %ret
+}
+
+; Test slt selects.
+define <2 x i64> @f16(<2 x i64> %val1, <2 x i64> %val2,
+                      <2 x i64> %val3, <2 x i64> %val4) {
+; CHECK-LABEL: f16:
+; CHECK: vchg [[REG:%v[0-9]+]], %v26, %v24
+; CHECK-NEXT: vsel %v24, %v28, %v30, [[REG]]
+; CHECK-NEXT: br %r14
+  %cmp = icmp slt <2 x i64> %val1, %val2
+  %ret = select <2 x i1> %cmp, <2 x i64> %val3, <2 x i64> %val4
+  ret <2 x i64> %ret
+}
+
+; Test ugt selects.
+define <2 x i64> @f17(<2 x i64> %val1, <2 x i64> %val2,
+                      <2 x i64> %val3, <2 x i64> %val4) {
+; CHECK-LABEL: f17:
+; CHECK: vchlg [[REG:%v[0-9]+]], %v24, %v26
+; CHECK-NEXT: vsel %v24, %v28, %v30, [[REG]]
+; CHECK-NEXT: br %r14
+  %cmp = icmp ugt <2 x i64> %val1, %val2
+  %ret = select <2 x i1> %cmp, <2 x i64> %val3, <2 x i64> %val4
+  ret <2 x i64> %ret
+}
+
+; Test uge selects.
+define <2 x i64> @f18(<2 x i64> %val1, <2 x i64> %val2,
+                      <2 x i64> %val3, <2 x i64> %val4) {
+; CHECK-LABEL: f18:
+; CHECK: vchlg [[REG:%v[0-9]+]], %v26, %v24
+; CHECK-NEXT: vsel %v24, %v30, %v28, [[REG]]
+; CHECK-NEXT: br %r14
+  %cmp = icmp uge <2 x i64> %val1, %val2
+  %ret = select <2 x i1> %cmp, <2 x i64> %val3, <2 x i64> %val4
+  ret <2 x i64> %ret
+}
+
+; Test ule selects.
+define <2 x i64> @f19(<2 x i64> %val1, <2 x i64> %val2,
+                      <2 x i64> %val3, <2 x i64> %val4) {
+; CHECK-LABEL: f19:
+; CHECK: vchlg [[REG:%v[0-9]+]], %v24, %v26
+; CHECK-NEXT: vsel %v24, %v30, %v28, [[REG]]
+; CHECK-NEXT: br %r14
+  %cmp = icmp ule <2 x i64> %val1, %val2
+  %ret = select <2 x i1> %cmp, <2 x i64> %val3, <2 x i64> %val4
+  ret <2 x i64> %ret
+}
+
+; Test ult selects.
+define <2 x i64> @f20(<2 x i64> %val1, <2 x i64> %val2,
+                      <2 x i64> %val3, <2 x i64> %val4) {
+; CHECK-LABEL: f20:
+; CHECK: vchlg [[REG:%v[0-9]+]], %v26, %v24
+; CHECK-NEXT: vsel %v24, %v28, %v30, [[REG]]
+; CHECK-NEXT: br %r14
+  %cmp = icmp ult <2 x i64> %val1, %val2
+  %ret = select <2 x i1> %cmp, <2 x i64> %val3, <2 x i64> %val4
+  ret <2 x i64> %ret
+}
diff --git a/test/CodeGen/SystemZ/vec-cmp-05.ll b/test/CodeGen/SystemZ/vec-cmp-05.ll
new file mode 100644
index 00000000000..74e99096097
--- /dev/null
+++ b/test/CodeGen/SystemZ/vec-cmp-05.ll
@@ -0,0 +1,472 @@
+; Test v4f32 comparisons.
+;
+; RUN: llc < %s -mtriple=s390x-linux-gnu -mcpu=z13 | FileCheck %s
+
+; Test oeq.
+define <4 x i32> @f1(<4 x float> %val1, <4 x float> %val2) {
+; CHECK-LABEL: f1:
+; CHECK-DAG: vmrhf [[HIGH0E:%v[0-9]+]], %v24, %v24
+; CHECK-DAG: vmrlf [[LOW0E:%v[0-9]+]], %v24, %v24
+; CHECK-DAG: vmrhf [[HIGH1E:%v[0-9]+]], %v26, %v26
+; CHECK-DAG: vmrlf [[LOW1E:%v[0-9]+]], %v26, %v26
+; CHECK-DAG: vldeb [[HIGH0D:%v[0-9]+]], [[HIGH0E]]
+; CHECK-DAG: vldeb [[HIGH1D:%v[0-9]+]], [[HIGH1E]]
+; CHECK-DAG: vldeb [[LOW0D:%v[0-9]+]], [[LOW0E]]
+; CHECK-DAG: vldeb [[LOW1D:%v[0-9]+]], [[LOW1E]]
+; CHECK-DAG: vfcedb [[HIGHRES:%v[0-9]+]], [[HIGH0D]], [[HIGH1D]]
+; CHECK-DAG: vfcedb [[LOWRES:%v[0-9]+]], [[LOW0D]], [[LOW1D]]
+; CHECK: vpkg %v24, [[HIGHRES]], [[LOWRES]]
+; CHECK-NEXT: br %r14
+  %cmp = fcmp oeq <4 x float> %val1, %val2
+  %ret = sext <4 x i1> %cmp to <4 x i32>
+  ret <4 x i32> %ret
+}
+
+; Test one.
+define <4 x i32> @f2(<4 x float> %val1, <4 x float> %val2) {
+; CHECK-LABEL: f2:
+; CHECK-DAG: vmrhf [[HIGH0E:%v[0-9]+]], %v24, %v24
+; CHECK-DAG: vmrlf [[LOW0E:%v[0-9]+]], %v24, %v24
+; CHECK-DAG: vmrhf [[HIGH1E:%v[0-9]+]], %v26, %v26
+; CHECK-DAG: vmrlf [[LOW1E:%v[0-9]+]], %v26, %v26
+; CHECK-DAG: vldeb [[HIGH0D:%v[0-9]+]], [[HIGH0E]]
+; CHECK-DAG: vldeb [[HIGH1D:%v[0-9]+]], [[HIGH1E]]
+; CHECK-DAG: vldeb [[LOW0D:%v[0-9]+]], [[LOW0E]]
+; CHECK-DAG: vldeb [[LOW1D:%v[0-9]+]], [[LOW1E]]
+; CHECK-DAG: vfchdb [[HIGHRES0:%v[0-9]+]], [[HIGH0D]], [[HIGH1D]]
+; CHECK-DAG: vfchdb [[LOWRES0:%v[0-9]+]], [[LOW0D]], [[LOW1D]]
+; CHECK-DAG: vfchdb [[HIGHRES1:%v[0-9]+]], [[HIGH1D]], [[HIGH0D]]
+; CHECK-DAG: vfchdb [[LOWRES1:%v[0-9]+]], [[LOW1D]], [[LOW0D]]
+; CHECK-DAG: vpkg [[RES0:%v[0-9]+]], [[HIGHRES0]], [[LOWRES0]]
+; CHECK-DAG: vpkg [[RES1:%v[0-9]+]], [[HIGHRES1]], [[LOWRES1]]
+; CHECK: vo %v24, [[RES1]], [[RES0]]
+; CHECK-NEXT: br %r14
+  %cmp = fcmp one <4 x float> %val1, %val2
+  %ret = sext <4 x i1> %cmp to <4 x i32>
+  ret <4 x i32> %ret
+}
+
+; Test ogt.
+define <4 x i32> @f3(<4 x float> %val1, <4 x float> %val2) {
+; CHECK-LABEL: f3:
+; CHECK-DAG: vmrhf [[HIGH0E:%v[0-9]+]], %v24, %v24
+; CHECK-DAG: vmrlf [[LOW0E:%v[0-9]+]], %v24, %v24
+; CHECK-DAG: vmrhf [[HIGH1E:%v[0-9]+]], %v26, %v26
+; CHECK-DAG: vmrlf [[LOW1E:%v[0-9]+]], %v26, %v26
+; CHECK-DAG: vldeb [[HIGH0D:%v[0-9]+]], [[HIGH0E]]
+; CHECK-DAG: vldeb [[HIGH1D:%v[0-9]+]], [[HIGH1E]]
+; CHECK-DAG: vldeb [[LOW0D:%v[0-9]+]], [[LOW0E]]
+; CHECK-DAG: vldeb [[LOW1D:%v[0-9]+]], [[LOW1E]]
+; CHECK-DAG: vfchdb [[HIGHRES:%v[0-9]+]], [[HIGH0D]], [[HIGH1D]]
+; CHECK-DAG: vfchdb [[LOWRES:%v[0-9]+]], [[LOW0D]], [[LOW1D]]
+; CHECK: vpkg %v24, [[HIGHRES]], [[LOWRES]]
+; CHECK-NEXT: br %r14
+  %cmp = fcmp ogt <4 x float> %val1, %val2
+  %ret = sext <4 x i1> %cmp to <4 x i32>
+  ret <4 x i32> %ret
+}
+
+; Test oge.
+define <4 x i32> @f4(<4 x float> %val1, <4 x float> %val2) {
+; CHECK-LABEL: f4:
+; CHECK-DAG: vmrhf [[HIGH0E:%v[0-9]+]], %v24, %v24
+; CHECK-DAG: vmrlf [[LOW0E:%v[0-9]+]], %v24, %v24
+; CHECK-DAG: vmrhf [[HIGH1E:%v[0-9]+]], %v26, %v26
+; CHECK-DAG: vmrlf [[LOW1E:%v[0-9]+]], %v26, %v26
+; CHECK-DAG: vldeb [[HIGH0D:%v[0-9]+]], [[HIGH0E]]
+; CHECK-DAG: vldeb [[HIGH1D:%v[0-9]+]], [[HIGH1E]]
+; CHECK-DAG: vldeb [[LOW0D:%v[0-9]+]], [[LOW0E]]
+; CHECK-DAG: vldeb [[LOW1D:%v[0-9]+]], [[LOW1E]]
+; CHECK-DAG: vfchedb [[HIGHRES:%v[0-9]+]], [[HIGH0D]], [[HIGH1D]]
+; CHECK-DAG: vfchedb [[LOWRES:%v[0-9]+]], [[LOW0D]], [[LOW1D]]
+; CHECK: vpkg %v24, [[HIGHRES]], [[LOWRES]]
+; CHECK-NEXT: br %r14
+  %cmp = fcmp oge <4 x float> %val1, %val2
+  %ret = sext <4 x i1> %cmp to <4 x i32>
+  ret <4 x i32> %ret
+}
+
+; Test ole.
+define <4 x i32> @f5(<4 x float> %val1, <4 x float> %val2) {
+; CHECK-LABEL: f5:
+; CHECK-DAG: vmrhf [[HIGH0E:%v[0-9]+]], %v24, %v24
+; CHECK-DAG: vmrlf [[LOW0E:%v[0-9]+]], %v24, %v24
+; CHECK-DAG: vmrhf [[HIGH1E:%v[0-9]+]], %v26, %v26
+; CHECK-DAG: vmrlf [[LOW1E:%v[0-9]+]], %v26, %v26
+; CHECK-DAG: vldeb [[HIGH0D:%v[0-9]+]], [[HIGH0E]]
+; CHECK-DAG: vldeb [[HIGH1D:%v[0-9]+]], [[HIGH1E]]
+; CHECK-DAG: vldeb [[LOW0D:%v[0-9]+]], [[LOW0E]]
+; CHECK-DAG: vldeb [[LOW1D:%v[0-9]+]], [[LOW1E]]
+; CHECK-DAG: vfchedb [[HIGHRES:%v[0-9]+]], [[HIGH1D]], [[HIGH0D]]
+; CHECK-DAG: vfchedb [[LOWRES:%v[0-9]+]], [[LOW1D]], [[LOW0D]]
+; CHECK: vpkg %v24, [[HIGHRES]], [[LOWRES]]
+; CHECK-NEXT: br %r14
+  %cmp = fcmp ole <4 x float> %val1, %val2
+  %ret = sext <4 x i1> %cmp to <4 x i32>
+  ret <4 x i32> %ret
+}
+
+; Test olt.
+define <4 x i32> @f6(<4 x float> %val1, <4 x float> %val2) {
+; CHECK-LABEL: f6:
+; CHECK-DAG: vmrhf [[HIGH0E:%v[0-9]+]], %v24, %v24
+; CHECK-DAG: vmrlf [[LOW0E:%v[0-9]+]], %v24, %v24
+; CHECK-DAG: vmrhf [[HIGH1E:%v[0-9]+]], %v26, %v26
+; CHECK-DAG: vmrlf [[LOW1E:%v[0-9]+]], %v26, %v26
+; CHECK-DAG: vldeb [[HIGH0D:%v[0-9]+]], [[HIGH0E]]
+; CHECK-DAG: vldeb [[HIGH1D:%v[0-9]+]], [[HIGH1E]]
+; CHECK-DAG: vldeb [[LOW0D:%v[0-9]+]], [[LOW0E]]
+; CHECK-DAG: vldeb [[LOW1D:%v[0-9]+]], [[LOW1E]]
+; CHECK-DAG: vfchdb [[HIGHRES:%v[0-9]+]], [[HIGH1D]], [[HIGH0D]]
+; CHECK-DAG: vfchdb [[LOWRES:%v[0-9]+]], [[LOW1D]], [[LOW0D]]
+; CHECK: vpkg %v24, [[HIGHRES]], [[LOWRES]]
+; CHECK-NEXT: br %r14
+  %cmp = fcmp olt <4 x float> %val1, %val2
+  %ret = sext <4 x i1> %cmp to <4 x i32>
+  ret <4 x i32> %ret
+}
+
+; Test ueq.
+define <4 x i32> @f7(<4 x float> %val1, <4 x float> %val2) {
+; CHECK-LABEL: f7:
+; CHECK-DAG: vmrhf [[HIGH0E:%v[0-9]+]], %v24, %v24
+; CHECK-DAG: vmrlf [[LOW0E:%v[0-9]+]], %v24, %v24
+; CHECK-DAG: vmrhf [[HIGH1E:%v[0-9]+]], %v26, %v26
+; CHECK-DAG: vmrlf [[LOW1E:%v[0-9]+]], %v26, %v26
+; CHECK-DAG: vldeb [[HIGH0D:%v[0-9]+]], [[HIGH0E]]
+; CHECK-DAG: vldeb [[HIGH1D:%v[0-9]+]], [[HIGH1E]]
+; CHECK-DAG: vldeb [[LOW0D:%v[0-9]+]], [[LOW0E]]
+; CHECK-DAG: vldeb [[LOW1D:%v[0-9]+]], [[LOW1E]]
+; CHECK-DAG: vfchdb [[HIGHRES0:%v[0-9]+]], [[HIGH0D]], [[HIGH1D]]
+; CHECK-DAG: vfchdb [[LOWRES0:%v[0-9]+]], [[LOW0D]], [[LOW1D]]
+; CHECK-DAG: vfchdb [[HIGHRES1:%v[0-9]+]], [[HIGH1D]], [[HIGH0D]]
+; CHECK-DAG: vfchdb [[LOWRES1:%v[0-9]+]], [[LOW1D]], [[LOW0D]]
+; CHECK-DAG: vpkg [[RES0:%v[0-9]+]], [[HIGHRES0]], [[LOWRES0]]
+; CHECK-DAG: vpkg [[RES1:%v[0-9]+]], [[HIGHRES1]], [[LOWRES1]]
+; CHECK: vno %v24, [[RES1]], [[RES0]]
+; CHECK-NEXT: br %r14
+  %cmp = fcmp ueq <4 x float> %val1, %val2
+  %ret = sext <4 x i1> %cmp to <4 x i32>
+  ret <4 x i32> %ret
+}
+
+; Test une.
+define <4 x i32> @f8(<4 x float> %val1, <4 x float> %val2) {
+; CHECK-LABEL: f8:
+; CHECK-DAG: vmrhf [[HIGH0E:%v[0-9]+]], %v24, %v24
+; CHECK-DAG: vmrlf [[LOW0E:%v[0-9]+]], %v24, %v24
+; CHECK-DAG: vmrhf [[HIGH1E:%v[0-9]+]], %v26, %v26
+; CHECK-DAG: vmrlf [[LOW1E:%v[0-9]+]], %v26, %v26
+; CHECK-DAG: vldeb [[HIGH0D:%v[0-9]+]], [[HIGH0E]]
+; CHECK-DAG: vldeb [[HIGH1D:%v[0-9]+]], [[HIGH1E]]
+; CHECK-DAG: vldeb [[LOW0D:%v[0-9]+]], [[LOW0E]]
+; CHECK-DAG: vldeb [[LOW1D:%v[0-9]+]], [[LOW1E]]
+; CHECK-DAG: vfcedb [[HIGHRES:%v[0-9]+]], [[HIGH0D]], [[HIGH1D]]
+; CHECK-DAG: vfcedb [[LOWRES:%v[0-9]+]], [[LOW0D]], [[LOW1D]]
+; CHECK: vpkg [[RES:%v[0-9]+]], [[HIGHRES]], [[LOWRES]]
+; CHECK-NEXT: vno %v24, [[RES]], [[RES]]
+; CHECK-NEXT: br %r14
+  %cmp = fcmp une <4 x float> %val1, %val2
+  %ret = sext <4 x i1> %cmp to <4 x i32>
+  ret <4 x i32> %ret
+}
+
+; Test ugt.
+define <4 x i32> @f9(<4 x float> %val1, <4 x float> %val2) {
+; CHECK-LABEL: f9:
+; CHECK-DAG: vmrhf [[HIGH0E:%v[0-9]+]], %v24, %v24
+; CHECK-DAG: vmrlf [[LOW0E:%v[0-9]+]], %v24, %v24
+; CHECK-DAG: vmrhf [[HIGH1E:%v[0-9]+]], %v26, %v26
+; CHECK-DAG: vmrlf [[LOW1E:%v[0-9]+]], %v26, %v26
+; CHECK-DAG: vldeb [[HIGH0D:%v[0-9]+]], [[HIGH0E]]
+; CHECK-DAG: vldeb [[HIGH1D:%v[0-9]+]], [[HIGH1E]]
+; CHECK-DAG: vldeb [[LOW0D:%v[0-9]+]], [[LOW0E]]
+; CHECK-DAG: vldeb [[LOW1D:%v[0-9]+]], [[LOW1E]]
+; CHECK-DAG: vfchedb [[HIGHRES:%v[0-9]+]], [[HIGH1D]], [[HIGH0D]]
+; CHECK-DAG: vfchedb [[LOWRES:%v[0-9]+]], [[LOW1D]], [[LOW0D]]
+; CHECK: vpkg [[RES:%v[0-9]+]], [[HIGHRES]], [[LOWRES]]
+; CHECK-NEXT: vno %v24, [[RES]], [[RES]]
+; CHECK-NEXT: br %r14
+  %cmp = fcmp ugt <4 x float> %val1, %val2
+  %ret = sext <4 x i1> %cmp to <4 x i32>
+  ret <4 x i32> %ret
+}
+
+; Test uge.
+define <4 x i32> @f10(<4 x float> %val1, <4 x float> %val2) {
+; CHECK-LABEL: f10:
+; CHECK-DAG: vmrhf [[HIGH0E:%v[0-9]+]], %v24, %v24
+; CHECK-DAG: vmrlf [[LOW0E:%v[0-9]+]], %v24, %v24
+; CHECK-DAG: vmrhf [[HIGH1E:%v[0-9]+]], %v26, %v26
+; CHECK-DAG: vmrlf [[LOW1E:%v[0-9]+]], %v26, %v26
+; CHECK-DAG: vldeb [[HIGH0D:%v[0-9]+]], [[HIGH0E]]
+; CHECK-DAG: vldeb [[HIGH1D:%v[0-9]+]], [[HIGH1E]]
+; CHECK-DAG: vldeb [[LOW0D:%v[0-9]+]], [[LOW0E]]
+; CHECK-DAG: vldeb [[LOW1D:%v[0-9]+]], [[LOW1E]]
+; CHECK-DAG: vfchdb [[HIGHRES:%v[0-9]+]], [[HIGH1D]], [[HIGH0D]]
+; CHECK-DAG: vfchdb [[LOWRES:%v[0-9]+]], [[LOW1D]], [[LOW0D]]
+; CHECK: vpkg [[RES:%v[0-9]+]], [[HIGHRES]], [[LOWRES]]
+; CHECK-NEXT: vno %v24, [[RES]], [[RES]]
+; CHECK-NEXT: br %r14
+  %cmp = fcmp uge <4 x float> %val1, %val2
+  %ret = sext <4 x i1> %cmp to <4 x i32>
+  ret <4 x i32> %ret
+}
+
+; Test ule.
+define <4 x i32> @f11(<4 x float> %val1, <4 x float> %val2) {
+; CHECK-LABEL: f11:
+; CHECK-DAG: vmrhf [[HIGH0E:%v[0-9]+]], %v24, %v24
+; CHECK-DAG: vmrlf [[LOW0E:%v[0-9]+]], %v24, %v24
+; CHECK-DAG: vmrhf [[HIGH1E:%v[0-9]+]], %v26, %v26
+; CHECK-DAG: vmrlf [[LOW1E:%v[0-9]+]], %v26, %v26
+; CHECK-DAG: vldeb [[HIGH0D:%v[0-9]+]], [[HIGH0E]]
+; CHECK-DAG: vldeb [[HIGH1D:%v[0-9]+]], [[HIGH1E]]
+; CHECK-DAG: vldeb [[LOW0D:%v[0-9]+]], [[LOW0E]]
+; CHECK-DAG: vldeb [[LOW1D:%v[0-9]+]], [[LOW1E]]
+; CHECK-DAG: vfchdb [[HIGHRES:%v[0-9]+]], [[HIGH0D]], [[HIGH1D]]
+; CHECK-DAG: vfchdb [[LOWRES:%v[0-9]+]], [[LOW0D]], [[LOW1D]]
+; CHECK: vpkg [[RES:%v[0-9]+]], [[HIGHRES]], [[LOWRES]]
+; CHECK-NEXT: vno %v24, [[RES]], [[RES]]
+; CHECK-NEXT: br %r14
+  %cmp = fcmp ule <4 x float> %val1, %val2
+  %ret = sext <4 x i1> %cmp to <4 x i32>
+  ret <4 x i32> %ret
+}
+
+; Test ult.
+define <4 x i32> @f12(<4 x float> %val1, <4 x float> %val2) {
+; CHECK-LABEL: f12:
+; CHECK-DAG: vmrhf [[HIGH0E:%v[0-9]+]], %v24, %v24
+; CHECK-DAG: vmrlf [[LOW0E:%v[0-9]+]], %v24, %v24
+; CHECK-DAG: vmrhf [[HIGH1E:%v[0-9]+]], %v26, %v26
+; CHECK-DAG: vmrlf [[LOW1E:%v[0-9]+]], %v26, %v26
+; CHECK-DAG: vldeb [[HIGH0D:%v[0-9]+]], [[HIGH0E]]
+; CHECK-DAG: vldeb [[HIGH1D:%v[0-9]+]], [[HIGH1E]]
+; CHECK-DAG: vldeb [[LOW0D:%v[0-9]+]], [[LOW0E]]
+; CHECK-DAG: vldeb [[LOW1D:%v[0-9]+]], [[LOW1E]]
+; CHECK-DAG: vfchedb [[HIGHRES:%v[0-9]+]], [[HIGH0D]], [[HIGH1D]]
+; CHECK-DAG: vfchedb [[LOWRES:%v[0-9]+]], [[LOW0D]], [[LOW1D]]
+; CHECK: vpkg [[RES:%v[0-9]+]], [[HIGHRES]], [[LOWRES]]
+; CHECK-NEXT: vno %v24, [[RES]], [[RES]]
+; CHECK-NEXT: br %r14
+  %cmp = fcmp ult <4 x float> %val1, %val2
+  %ret = sext <4 x i1> %cmp to <4 x i32>
+  ret <4 x i32> %ret
+}
+
+; Test ord.
+define <4 x i32> @f13(<4 x float> %val1, <4 x float> %val2) {
+; CHECK-LABEL: f13:
+; CHECK-DAG: vmrhf [[HIGH0E:%v[0-9]+]], %v24, %v24
+; CHECK-DAG: vmrlf [[LOW0E:%v[0-9]+]], %v24, %v24
+; CHECK-DAG: vmrhf [[HIGH1E:%v[0-9]+]], %v26, %v26
+; CHECK-DAG: vmrlf [[LOW1E:%v[0-9]+]], %v26, %v26
+; CHECK-DAG: vldeb [[HIGH0D:%v[0-9]+]], [[HIGH0E]]
+; CHECK-DAG: vldeb [[HIGH1D:%v[0-9]+]], [[HIGH1E]]
+; CHECK-DAG: vldeb [[LOW0D:%v[0-9]+]], [[LOW0E]]
+; CHECK-DAG: vldeb [[LOW1D:%v[0-9]+]], [[LOW1E]]
+; CHECK-DAG: vfchedb [[HIGHRES0:%v[0-9]+]], [[HIGH0D]], [[HIGH1D]]
+; CHECK-DAG: vfchedb [[LOWRES0:%v[0-9]+]], [[LOW0D]], [[LOW1D]]
+; CHECK-DAG: vfchdb [[HIGHRES1:%v[0-9]+]], [[HIGH1D]], [[HIGH0D]]
+; CHECK-DAG: vfchdb [[LOWRES1:%v[0-9]+]], [[LOW1D]], [[LOW0D]]
+; CHECK-DAG: vpkg [[RES0:%v[0-9]+]], [[HIGHRES0]], [[LOWRES0]]
+; CHECK-DAG: vpkg [[RES1:%v[0-9]+]], [[HIGHRES1]], [[LOWRES1]]
+; CHECK: vo %v24, [[RES1]], [[RES0]]
+; CHECK-NEXT: br %r14
+  %cmp = fcmp ord <4 x float> %val1, %val2
+  %ret = sext <4 x i1> %cmp to <4 x i32>
+  ret <4 x i32> %ret
+}
+
+; Test uno.
+define <4 x i32> @f14(<4 x float> %val1, <4 x float> %val2) {
+; CHECK-LABEL: f14:
+; CHECK-DAG: vmrhf [[HIGH0E:%v[0-9]+]], %v24, %v24
+; CHECK-DAG: vmrlf [[LOW0E:%v[0-9]+]], %v24, %v24
+; CHECK-DAG: vmrhf [[HIGH1E:%v[0-9]+]], %v26, %v26
+; CHECK-DAG: vmrlf [[LOW1E:%v[0-9]+]], %v26, %v26
+; CHECK-DAG: vldeb [[HIGH0D:%v[0-9]+]], [[HIGH0E]]
+; CHECK-DAG: vldeb [[HIGH1D:%v[0-9]+]], [[HIGH1E]]
+; CHECK-DAG: vldeb [[LOW0D:%v[0-9]+]], [[LOW0E]]
+; CHECK-DAG: vldeb [[LOW1D:%v[0-9]+]], [[LOW1E]]
+; CHECK-DAG: vfchedb [[HIGHRES0:%v[0-9]+]], [[HIGH0D]], [[HIGH1D]]
+; CHECK-DAG: vfchedb [[LOWRES0:%v[0-9]+]], [[LOW0D]], [[LOW1D]]
+; CHECK-DAG: vfchdb [[HIGHRES1:%v[0-9]+]], [[HIGH1D]], [[HIGH0D]]
+; CHECK-DAG: vfchdb [[LOWRES1:%v[0-9]+]], [[LOW1D]], [[LOW0D]]
+; CHECK-DAG: vpkg [[RES0:%v[0-9]+]], [[HIGHRES0]], [[LOWRES0]]
+; CHECK-DAG: vpkg [[RES1:%v[0-9]+]], [[HIGHRES1]], [[LOWRES1]]
+; CHECK: vno %v24, [[RES1]], [[RES0]]
+; CHECK-NEXT: br %r14
+  %cmp = fcmp uno <4 x float> %val1, %val2
+  %ret = sext <4 x i1> %cmp to <4 x i32>
+  ret <4 x i32> %ret
+}
+
+; Test oeq selects.
+define <4 x float> @f15(<4 x float> %val1, <4 x float> %val2,
+                        <4 x float> %val3, <4 x float> %val4) {
+; CHECK-LABEL: f15:
+; CHECK: vpkg [[REG:%v[0-9]+]],
+; CHECK-NEXT: vsel %v24, %v28, %v30, [[REG]]
+; CHECK-NEXT: br %r14
+  %cmp = fcmp oeq <4 x float> %val1, %val2
+  %ret = select <4 x i1> %cmp, <4 x float> %val3, <4 x float> %val4
+  ret <4 x float> %ret
+}
+
+; Test one selects.
+define <4 x float> @f16(<4 x float> %val1, <4 x float> %val2,
+                        <4 x float> %val3, <4 x float> %val4) {
+; CHECK-LABEL: f16:
+; CHECK: vo [[REG:%v[0-9]+]],
+; CHECK-NEXT: vsel %v24, %v28, %v30, [[REG]]
+; CHECK-NEXT: br %r14
+  %cmp = fcmp one <4 x float> %val1, %val2
+  %ret = select <4 x i1> %cmp, <4 x float> %val3, <4 x float> %val4
+  ret <4 x float> %ret
+}
+
+; Test ogt selects.
+define <4 x float> @f17(<4 x float> %val1, <4 x float> %val2,
+                        <4 x float> %val3, <4 x float> %val4) {
+; CHECK-LABEL: f17:
+; CHECK: vpkg [[REG:%v[0-9]+]],
+; CHECK-NEXT: vsel %v24, %v28, %v30, [[REG]]
+; CHECK-NEXT: br %r14
+  %cmp = fcmp ogt <4 x float> %val1, %val2
+  %ret = select <4 x i1> %cmp, <4 x float> %val3, <4 x float> %val4
+  ret <4 x float> %ret
+}
+
+; Test oge selects.
+define <4 x float> @f18(<4 x float> %val1, <4 x float> %val2,
+                        <4 x float> %val3, <4 x float> %val4) {
+; CHECK-LABEL: f18:
+; CHECK: vpkg [[REG:%v[0-9]+]],
+; CHECK-NEXT: vsel %v24, %v28, %v30, [[REG]]
+; CHECK-NEXT: br %r14
+  %cmp = fcmp oge <4 x float> %val1, %val2
+  %ret = select <4 x i1> %cmp, <4 x float> %val3, <4 x float> %val4
+  ret <4 x float> %ret
+}
+
+; Test ole selects.
+define <4 x float> @f19(<4 x float> %val1, <4 x float> %val2,
+                        <4 x float> %val3, <4 x float> %val4) {
+; CHECK-LABEL: f19:
+; CHECK: vpkg [[REG:%v[0-9]+]],
+; CHECK-NEXT: vsel %v24, %v28, %v30, [[REG]]
+; CHECK-NEXT: br %r14
+  %cmp = fcmp ole <4 x float> %val1, %val2
+  %ret = select <4 x i1> %cmp, <4 x float> %val3, <4 x float> %val4
+  ret <4 x float> %ret
+}
+
+; Test olt selects.
+define <4 x float> @f20(<4 x float> %val1, <4 x float> %val2,
+                        <4 x float> %val3, <4 x float> %val4) {
+; CHECK-LABEL: f20:
+; CHECK: vpkg [[REG:%v[0-9]+]],
+; CHECK-NEXT: vsel %v24, %v28, %v30, [[REG]]
+; CHECK-NEXT: br %r14
+  %cmp = fcmp olt <4 x float> %val1, %val2
+  %ret = select <4 x i1> %cmp, <4 x float> %val3, <4 x float> %val4
+  ret <4 x float> %ret
+}
+
+; Test ueq selects.
+define <4 x float> @f21(<4 x float> %val1, <4 x float> %val2,
+                        <4 x float> %val3, <4 x float> %val4) {
+; CHECK-LABEL: f21:
+; CHECK: vo [[REG:%v[0-9]+]],
+; CHECK-NEXT: vsel %v24, %v30, %v28, [[REG]]
+; CHECK-NEXT: br %r14
+  %cmp = fcmp ueq <4 x float> %val1, %val2
+  %ret = select <4 x i1> %cmp, <4 x float> %val3, <4 x float> %val4
+  ret <4 x float> %ret
+}
+
+; Test une selects.
+define <4 x float> @f22(<4 x float> %val1, <4 x float> %val2,
+                        <4 x float> %val3, <4 x float> %val4) {
+; CHECK-LABEL: f22:
+; CHECK: vpkg [[REG:%v[0-9]+]],
+; CHECK-NEXT: vsel %v24, %v30, %v28, [[REG]]
+; CHECK-NEXT: br %r14
+  %cmp = fcmp une <4 x float> %val1, %val2
+  %ret = select <4 x i1> %cmp, <4 x float> %val3, <4 x float> %val4
+  ret <4 x float> %ret
+}
+
+; Test ugt selects.
+define <4 x float> @f23(<4 x float> %val1, <4 x float> %val2,
+                        <4 x float> %val3, <4 x float> %val4) {
+; CHECK-LABEL: f23:
+; CHECK: vpkg [[REG:%v[0-9]+]],
+; CHECK-NEXT: vsel %v24, %v30, %v28, [[REG]]
+; CHECK-NEXT: br %r14
+  %cmp = fcmp ugt <4 x float> %val1, %val2
+  %ret = select <4 x i1> %cmp, <4 x float> %val3, <4 x float> %val4
+  ret <4 x float> %ret
+}
+
+; Test uge selects.
+define <4 x float> @f24(<4 x float> %val1, <4 x float> %val2,
+                        <4 x float> %val3, <4 x float> %val4) {
+; CHECK-LABEL: f24:
+; CHECK: vpkg [[REG:%v[0-9]+]],
+; CHECK-NEXT: vsel %v24, %v30, %v28, [[REG]]
+; CHECK-NEXT: br %r14
+  %cmp = fcmp uge <4 x float> %val1, %val2
+  %ret = select <4 x i1> %cmp, <4 x float> %val3, <4 x float> %val4
+  ret <4 x float> %ret
+}
+
+; Test ule selects.
+define <4 x float> @f25(<4 x float> %val1, <4 x float> %val2,
+                        <4 x float> %val3, <4 x float> %val4) {
+; CHECK-LABEL: f25:
+; CHECK: vpkg [[REG:%v[0-9]+]],
+; CHECK-NEXT: vsel %v24, %v30, %v28, [[REG]]
+; CHECK-NEXT: br %r14
+  %cmp = fcmp ule <4 x float> %val1, %val2
+  %ret = select <4 x i1> %cmp, <4 x float> %val3, <4 x float> %val4
+  ret <4 x float> %ret
+}
+
+; Test ult selects.
+define <4 x float> @f26(<4 x float> %val1, <4 x float> %val2,
+                        <4 x float> %val3, <4 x float> %val4) {
+; CHECK-LABEL: f26:
+; CHECK: vpkg [[REG:%v[0-9]+]],
+; CHECK-NEXT: vsel %v24, %v30, %v28, [[REG]]
+; CHECK-NEXT: br %r14
+  %cmp = fcmp ult <4 x float> %val1, %val2
+  %ret = select <4 x i1> %cmp, <4 x float> %val3, <4 x float> %val4
+  ret <4 x float> %ret
+}
+
+; Test ord selects.
+define <4 x float> @f27(<4 x float> %val1, <4 x float> %val2,
+                        <4 x float> %val3, <4 x float> %val4) {
+; CHECK-LABEL: f27:
+; CHECK: vo [[REG:%v[0-9]+]],
+; CHECK-NEXT: vsel %v24, %v28, %v30, [[REG]]
+; CHECK-NEXT: br %r14
+  %cmp = fcmp ord <4 x float> %val1, %val2
+  %ret = select <4 x i1> %cmp, <4 x float> %val3, <4 x float> %val4
+  ret <4 x float> %ret
+}
+
+; Test uno selects.
+define <4 x float> @f28(<4 x float> %val1, <4 x float> %val2,
+                        <4 x float> %val3, <4 x float> %val4) {
+; CHECK-LABEL: f28:
+; CHECK: vo [[REG:%v[0-9]+]],
+; CHECK-NEXT: vsel %v24, %v30, %v28, [[REG]]
+; CHECK-NEXT: br %r14
+  %cmp = fcmp uno <4 x float> %val1, %val2
+  %ret = select <4 x i1> %cmp, <4 x float> %val3, <4 x float> %val4
+  ret <4 x float> %ret
+}
diff --git a/test/CodeGen/SystemZ/vec-cmp-06.ll b/test/CodeGen/SystemZ/vec-cmp-06.ll
new file mode 100644
index 00000000000..eef57555b48
--- /dev/null
+++ b/test/CodeGen/SystemZ/vec-cmp-06.ll
@@ -0,0 +1,349 @@
+; Test f64 and v2f64 comparisons.
+;
+; RUN: llc < %s -mtriple=s390x-linux-gnu -mcpu=z13 | FileCheck %s
+
+; Test oeq.
+define <2 x i64> @f1(<2 x i64> %dummy, <2 x double> %val1, <2 x double> %val2) {
+; CHECK-LABEL: f1:
+; CHECK: vfcedb %v24, %v26, %v28
+; CHECK-NEXT: br %r14
+  %cmp = fcmp oeq <2 x double> %val1, %val2
+  %ret = sext <2 x i1> %cmp to <2 x i64>
+  ret <2 x i64> %ret
+}
+
+; Test one.
+define <2 x i64> @f2(<2 x i64> %dummy, <2 x double> %val1, <2 x double> %val2) {
+; CHECK-LABEL: f2:
+; CHECK-DAG: vfchdb [[REG1:%v[0-9]+]], %v28, %v26
+; CHECK-DAG: vfchdb [[REG2:%v[0-9]+]], %v26, %v28
+; CHECK: vo %v24, [[REG1]], [[REG2]]
+; CHECK-NEXT: br %r14
+  %cmp = fcmp one <2 x double> %val1, %val2
+  %ret = sext <2 x i1> %cmp to <2 x i64>
+  ret <2 x i64> %ret
+}
+
+; Test ogt.
+define <2 x i64> @f3(<2 x i64> %dummy, <2 x double> %val1, <2 x double> %val2) {
+; CHECK-LABEL: f3:
+; CHECK: vfchdb %v24, %v26, %v28
+; CHECK-NEXT: br %r14
+  %cmp = fcmp ogt <2 x double> %val1, %val2
+  %ret = sext <2 x i1> %cmp to <2 x i64>
+  ret <2 x i64> %ret
+}
+
+; Test oge.
+define <2 x i64> @f4(<2 x i64> %dummy, <2 x double> %val1, <2 x double> %val2) {
+; CHECK-LABEL: f4:
+; CHECK: vfchedb %v24, %v26, %v28
+; CHECK-NEXT: br %r14
+  %cmp = fcmp oge <2 x double> %val1, %val2
+  %ret = sext <2 x i1> %cmp to <2 x i64>
+  ret <2 x i64> %ret
+}
+
+; Test ole.
+define <2 x i64> @f5(<2 x i64> %dummy, <2 x double> %val1, <2 x double> %val2) {
+; CHECK-LABEL: f5:
+; CHECK: vfchedb %v24, %v28, %v26
+; CHECK-NEXT: br %r14
+  %cmp = fcmp ole <2 x double> %val1, %val2
+  %ret = sext <2 x i1> %cmp to <2 x i64>
+  ret <2 x i64> %ret
+}
+
+; Test olt.
+define <2 x i64> @f6(<2 x i64> %dummy, <2 x double> %val1, <2 x double> %val2) {
+; CHECK-LABEL: f6:
+; CHECK: vfchdb %v24, %v28, %v26
+; CHECK-NEXT: br %r14
+  %cmp = fcmp olt <2 x double> %val1, %val2
+  %ret = sext <2 x i1> %cmp to <2 x i64>
+  ret <2 x i64> %ret
+}
+
+; Test ueq.
+define <2 x i64> @f7(<2 x i64> %dummy, <2 x double> %val1, <2 x double> %val2) {
+; CHECK-LABEL: f7:
+; CHECK-DAG: vfchdb [[REG1:%v[0-9]+]], %v28, %v26
+; CHECK-DAG: vfchdb [[REG2:%v[0-9]+]], %v26, %v28
+; CHECK: vno %v24, [[REG1]], [[REG2]]
+; CHECK-NEXT: br %r14
+  %cmp = fcmp ueq <2 x double> %val1, %val2
+  %ret = sext <2 x i1> %cmp to <2 x i64>
+  ret <2 x i64> %ret
+}
+
+; Test une.
+define <2 x i64> @f8(<2 x i64> %dummy, <2 x double> %val1, <2 x double> %val2) {
+; CHECK-LABEL: f8:
+; CHECK: vfcedb [[REG:%v[0-9]+]], %v26, %v28
+; CHECK-NEXT: vno %v24, [[REG]], [[REG]]
+; CHECK-NEXT: br %r14
+  %cmp = fcmp une <2 x double> %val1, %val2
+  %ret = sext <2 x i1> %cmp to <2 x i64>
+  ret <2 x i64> %ret
+}
+
+; Test ugt.
+define <2 x i64> @f9(<2 x i64> %dummy, <2 x double> %val1, <2 x double> %val2) {
+; CHECK-LABEL: f9:
+; CHECK: vfchedb [[REG:%v[0-9]+]], %v28, %v26
+; CHECK-NEXT: vno %v24, [[REG]], [[REG]]
+; CHECK-NEXT: br %r14
+  %cmp = fcmp ugt <2 x double> %val1, %val2
+  %ret = sext <2 x i1> %cmp to <2 x i64>
+  ret <2 x i64> %ret
+}
+
+; Test uge.
+define <2 x i64> @f10(<2 x i64> %dummy, <2 x double> %val1,
+                      <2 x double> %val2) {
+; CHECK-LABEL: f10:
+; CHECK: vfchdb [[REG:%v[0-9]+]], %v28, %v26
+; CHECK-NEXT: vno %v24, [[REG]], [[REG]]
+; CHECK-NEXT: br %r14
+  %cmp = fcmp uge <2 x double> %val1, %val2
+  %ret = sext <2 x i1> %cmp to <2 x i64>
+  ret <2 x i64> %ret
+}
+
+; Test ule.
+define <2 x i64> @f11(<2 x i64> %dummy, <2 x double> %val1,
+                      <2 x double> %val2) {
+; CHECK-LABEL: f11:
+; CHECK: vfchdb [[REG:%v[0-9]+]], %v26, %v28
+; CHECK-NEXT: vno %v24, [[REG]], [[REG]]
+; CHECK-NEXT: br %r14
+  %cmp = fcmp ule <2 x double> %val1, %val2
+  %ret = sext <2 x i1> %cmp to <2 x i64>
+  ret <2 x i64> %ret
+}
+
+; Test ult.
+define <2 x i64> @f12(<2 x i64> %dummy, <2 x double> %val1,
+                      <2 x double> %val2) {
+; CHECK-LABEL: f12:
+; CHECK: vfchedb [[REG:%v[0-9]+]], %v26, %v28
+; CHECK-NEXT: vno %v24, [[REG]], [[REG]]
+; CHECK-NEXT: br %r14
+  %cmp = fcmp ult <2 x double> %val1, %val2
+  %ret = sext <2 x i1> %cmp to <2 x i64>
+  ret <2 x i64> %ret
+}
+
+; Test ord.
+define <2 x i64> @f13(<2 x i64> %dummy, <2 x double> %val1,
+                      <2 x double> %val2) {
+; CHECK-LABEL: f13:
+; CHECK-DAG: vfchdb [[REG1:%v[0-9]+]], %v28, %v26
+; CHECK-DAG: vfchedb [[REG2:%v[0-9]+]], %v26, %v28
+; CHECK: vo %v24, [[REG1]], [[REG2]]
+; CHECK-NEXT: br %r14
+  %cmp = fcmp ord <2 x double> %val1, %val2
+  %ret = sext <2 x i1> %cmp to <2 x i64>
+  ret <2 x i64> %ret
+}
+
+; Test uno.
+define <2 x i64> @f14(<2 x i64> %dummy, <2 x double> %val1,
+                      <2 x double> %val2) {
+; CHECK-LABEL: f14:
+; CHECK-DAG: vfchdb [[REG1:%v[0-9]+]], %v28, %v26
+; CHECK-DAG: vfchedb [[REG2:%v[0-9]+]], %v26, %v28
+; CHECK: vno %v24, [[REG1]], [[REG2]]
+; CHECK-NEXT: br %r14
+  %cmp = fcmp uno <2 x double> %val1, %val2
+  %ret = sext <2 x i1> %cmp to <2 x i64>
+  ret <2 x i64> %ret
+}
+
+; Test oeq selects.
+define <2 x double> @f15(<2 x double> %val1, <2 x double> %val2,
+                         <2 x double> %val3, <2 x double> %val4) {
+; CHECK-LABEL: f15:
+; CHECK: vfcedb [[REG:%v[0-9]+]], %v24, %v26
+; CHECK-NEXT: vsel %v24, %v28, %v30, [[REG]]
+; CHECK-NEXT: br %r14
+  %cmp = fcmp oeq <2 x double> %val1, %val2
+  %ret = select <2 x i1> %cmp, <2 x double> %val3, <2 x double> %val4
+  ret <2 x double> %ret
+}
+
+; Test one selects.
+define <2 x double> @f16(<2 x double> %val1, <2 x double> %val2,
+                         <2 x double> %val3, <2 x double> %val4) {
+; CHECK-LABEL: f16:
+; CHECK-DAG: vfchdb [[REG1:%v[0-9]+]], %v26, %v24
+; CHECK-DAG: vfchdb [[REG2:%v[0-9]+]], %v24, %v26
+; CHECK: vo [[REG:%v[0-9]+]], [[REG1]], [[REG2]]
+; CHECK-NEXT: vsel %v24, %v28, %v30, [[REG]]
+; CHECK-NEXT: br %r14
+  %cmp = fcmp one <2 x double> %val1, %val2
+  %ret = select <2 x i1> %cmp, <2 x double> %val3, <2 x double> %val4
+  ret <2 x double> %ret
+}
+
+; Test ogt selects.
+define <2 x double> @f17(<2 x double> %val1, <2 x double> %val2,
+                         <2 x double> %val3, <2 x double> %val4) {
+; CHECK-LABEL: f17:
+; CHECK: vfchdb [[REG:%v[0-9]+]], %v24, %v26
+; CHECK-NEXT: vsel %v24, %v28, %v30, [[REG]]
+; CHECK-NEXT: br %r14
+  %cmp = fcmp ogt <2 x double> %val1, %val2
+  %ret = select <2 x i1> %cmp, <2 x double> %val3, <2 x double> %val4
+  ret <2 x double> %ret
+}
+
+; Test oge selects.
+define <2 x double> @f18(<2 x double> %val1, <2 x double> %val2,
+                         <2 x double> %val3, <2 x double> %val4) {
+; CHECK-LABEL: f18:
+; CHECK: vfchedb [[REG:%v[0-9]+]], %v24, %v26
+; CHECK-NEXT: vsel %v24, %v28, %v30, [[REG]]
+; CHECK-NEXT: br %r14
+  %cmp = fcmp oge <2 x double> %val1, %val2
+  %ret = select <2 x i1> %cmp, <2 x double> %val3, <2 x double> %val4
+  ret <2 x double> %ret
+}
+
+; Test ole selects.
+define <2 x double> @f19(<2 x double> %val1, <2 x double> %val2,
+                         <2 x double> %val3, <2 x double> %val4) {
+; CHECK-LABEL: f19:
+; CHECK: vfchedb [[REG:%v[0-9]+]], %v26, %v24
+; CHECK-NEXT: vsel %v24, %v28, %v30, [[REG]]
+; CHECK-NEXT: br %r14
+  %cmp = fcmp ole <2 x double> %val1, %val2
+  %ret = select <2 x i1> %cmp, <2 x double> %val3, <2 x double> %val4
+  ret <2 x double> %ret
+}
+
+; Test olt selects.
+define <2 x double> @f20(<2 x double> %val1, <2 x double> %val2,
+                         <2 x double> %val3, <2 x double> %val4) {
+; CHECK-LABEL: f20:
+; CHECK: vfchdb [[REG:%v[0-9]+]], %v26, %v24
+; CHECK-NEXT: vsel %v24, %v28, %v30, [[REG]]
+; CHECK-NEXT: br %r14
+  %cmp = fcmp olt <2 x double> %val1, %val2
+  %ret = select <2 x i1> %cmp, <2 x double> %val3, <2 x double> %val4
+  ret <2 x double> %ret
+}
+
+; Test ueq selects.
+define <2 x double> @f21(<2 x double> %val1, <2 x double> %val2,
+                         <2 x double> %val3, <2 x double> %val4) {
+; CHECK-LABEL: f21:
+; CHECK-DAG: vfchdb [[REG1:%v[0-9]+]], %v26, %v24
+; CHECK-DAG: vfchdb [[REG2:%v[0-9]+]], %v24, %v26
+; CHECK: vo [[REG:%v[0-9]+]], [[REG1]], [[REG2]]
+; CHECK-NEXT: vsel %v24, %v30, %v28, [[REG]]
+; CHECK-NEXT: br %r14
+  %cmp = fcmp ueq <2 x double> %val1, %val2
+  %ret = select <2 x i1> %cmp, <2 x double> %val3, <2 x double> %val4
+  ret <2 x double> %ret
+}
+
+; Test une selects.
+define <2 x double> @f22(<2 x double> %val1, <2 x double> %val2,
+                         <2 x double> %val3, <2 x double> %val4) {
+; CHECK-LABEL: f22:
+; CHECK: vfcedb [[REG:%v[0-9]+]], %v24, %v26
+; CHECK-NEXT: vsel %v24, %v30, %v28, [[REG]]
+; CHECK-NEXT: br %r14
+  %cmp = fcmp une <2 x double> %val1, %val2
+  %ret = select <2 x i1> %cmp, <2 x double> %val3, <2 x double> %val4
+  ret <2 x double> %ret
+}
+
+; Test ugt selects.
+define <2 x double> @f23(<2 x double> %val1, <2 x double> %val2,
+                         <2 x double> %val3, <2 x double> %val4) {
+; CHECK-LABEL: f23:
+; CHECK: vfchedb [[REG:%v[0-9]+]], %v26, %v24
+; CHECK-NEXT: vsel %v24, %v30, %v28, [[REG]]
+; CHECK-NEXT: br %r14
+  %cmp = fcmp ugt <2 x double> %val1, %val2
+  %ret = select <2 x i1> %cmp, <2 x double> %val3, <2 x double> %val4
+  ret <2 x double> %ret
+}
+
+; Test uge selects.
+define <2 x double> @f24(<2 x double> %val1, <2 x double> %val2,
+                         <2 x double> %val3, <2 x double> %val4) {
+; CHECK-LABEL: f24:
+; CHECK: vfchdb [[REG:%v[0-9]+]], %v26, %v24
+; CHECK-NEXT: vsel %v24, %v30, %v28, [[REG]]
+; CHECK-NEXT: br %r14
+  %cmp = fcmp uge <2 x double> %val1, %val2
+  %ret = select <2 x i1> %cmp, <2 x double> %val3, <2 x double> %val4
+  ret <2 x double> %ret
+}
+
+; Test ule selects.
+define <2 x double> @f25(<2 x double> %val1, <2 x double> %val2,
+                         <2 x double> %val3, <2 x double> %val4) {
+; CHECK-LABEL: f25:
+; CHECK: vfchdb [[REG:%v[0-9]+]], %v24, %v26
+; CHECK-NEXT: vsel %v24, %v30, %v28, [[REG]]
+; CHECK-NEXT: br %r14
+  %cmp = fcmp ule <2 x double> %val1, %val2
+  %ret = select <2 x i1> %cmp, <2 x double> %val3, <2 x double> %val4
+  ret <2 x double> %ret
+}
+
+; Test ult selects.
+define <2 x double> @f26(<2 x double> %val1, <2 x double> %val2,
+                         <2 x double> %val3, <2 x double> %val4) {
+; CHECK-LABEL: f26:
+; CHECK: vfchedb [[REG:%v[0-9]+]], %v24, %v26
+; CHECK-NEXT: vsel %v24, %v30, %v28, [[REG]]
+; CHECK-NEXT: br %r14
+  %cmp = fcmp ult <2 x double> %val1, %val2
+  %ret = select <2 x i1> %cmp, <2 x double> %val3, <2 x double> %val4
+  ret <2 x double> %ret
+}
+
+; Test ord selects.
+define <2 x double> @f27(<2 x double> %val1, <2 x double> %val2,
+                         <2 x double> %val3, <2 x double> %val4) {
+; CHECK-LABEL: f27:
+; CHECK-DAG: vfchdb [[REG1:%v[0-9]+]], %v26, %v24
+; CHECK-DAG: vfchedb [[REG2:%v[0-9]+]], %v24, %v26
+; CHECK: vo [[REG:%v[0-9]+]], [[REG1]], [[REG2]]
+; CHECK-NEXT: vsel %v24, %v28, %v30, [[REG]]
+; CHECK-NEXT: br %r14
+  %cmp = fcmp ord <2 x double> %val1, %val2
+  %ret = select <2 x i1> %cmp, <2 x double> %val3, <2 x double> %val4
+  ret <2 x double> %ret
+}
+
+; Test uno selects.
+define <2 x double> @f28(<2 x double> %val1, <2 x double> %val2,
+                         <2 x double> %val3, <2 x double> %val4) {
+; CHECK-LABEL: f28:
+; CHECK-DAG: vfchdb [[REG1:%v[0-9]+]], %v26, %v24
+; CHECK-DAG: vfchedb [[REG2:%v[0-9]+]], %v24, %v26
+; CHECK: vo [[REG:%v[0-9]+]], [[REG1]], [[REG2]]
+; CHECK-NEXT: vsel %v24, %v30, %v28, [[REG]]
+; CHECK-NEXT: br %r14
+  %cmp = fcmp uno <2 x double> %val1, %val2
+  %ret = select <2 x i1> %cmp, <2 x double> %val3, <2 x double> %val4
+  ret <2 x double> %ret
+}
+
+; Test an f64 comparison that uses vector registers.
+define i64 @f29(i64 %a, i64 %b, double %f1, <2 x double> %vec) {
+; CHECK-LABEL: f29:
+; CHECK: wfcdb %f0, %v24
+; CHECK-NEXT: locgrne %r2, %r3
+; CHECK: br %r14
+  %f2 = extractelement <2 x double> %vec, i32 0
+  %cond = fcmp oeq double %f1, %f2
+  %res = select i1 %cond, i64 %a, i64 %b
+  ret i64 %res
+}
diff --git a/test/CodeGen/SystemZ/vec-combine-01.ll b/test/CodeGen/SystemZ/vec-combine-01.ll
new file mode 100644
index 00000000000..a3593442172
--- /dev/null
+++ b/test/CodeGen/SystemZ/vec-combine-01.ll
@@ -0,0 +1,155 @@
+; Test various target-specific DAG combiner patterns.
+;
+; RUN: llc < %s -mtriple=s390x-linux-gnu -mcpu=z13 | FileCheck %s
+
+; Check that an extraction followed by a truncation is effectively treated
+; as a bitcast.
+define void @f1(<4 x i32> %v1, <4 x i32> %v2, i8 *%ptr1, i8 *%ptr2) {
+; CHECK-LABEL: f1:
+; CHECK: vaf [[REG:%v[0-9]+]], %v24, %v26
+; CHECK-DAG: vsteb [[REG]], 0(%r2), 3
+; CHECK-DAG: vsteb [[REG]], 0(%r3), 15
+; CHECK: br %r14
+  %add = add <4 x i32> %v1, %v2
+  %elem1 = extractelement <4 x i32> %add, i32 0
+  %elem2 = extractelement <4 x i32> %add, i32 3
+  %trunc1 = trunc i32 %elem1 to i8
+  %trunc2 = trunc i32 %elem2 to i8
+  store i8 %trunc1, i8 *%ptr1
+  store i8 %trunc2, i8 *%ptr2
+  ret void
+}
+
+; Test a case where a pack-type shuffle can be eliminated.
+define i16 @f2(<4 x i32> %v1, <4 x i32> %v2, <4 x i32> %v3) {
+; CHECK-LABEL: f2:
+; CHECK-NOT: vpk
+; CHECK-DAG: vaf [[REG1:%v[0-9]+]], %v24, %v26
+; CHECK-DAG: vaf [[REG2:%v[0-9]+]], %v26, %v28
+; CHECK-DAG: vlgvh {{%r[0-5]}}, [[REG1]], 3
+; CHECK-DAG: vlgvh {{%r[0-5]}}, [[REG2]], 7
+; CHECK: br %r14
+  %add1 = add <4 x i32> %v1, %v2
+  %add2 = add <4 x i32> %v2, %v3
+  %shuffle = shufflevector <4 x i32> %add1, <4 x i32> %add2,
+                           <4 x i32> <i32 1, i32 3, i32 5, i32 7>
+  %bitcast = bitcast <4 x i32> %shuffle to <8 x i16>
+  %elem1 = extractelement <8 x i16> %bitcast, i32 1
+  %elem2 = extractelement <8 x i16> %bitcast, i32 7
+  %res = add i16 %elem1, %elem2
+  ret i16 %res
+}
+
+; ...and again in a case where there's also a splat and a bitcast.
+define i16 @f3(<4 x i32> %v1, <4 x i32> %v2, <2 x i64> %v3) {
+; CHECK-LABEL: f3:
+; CHECK-NOT: vrepg
+; CHECK-NOT: vpk
+; CHECK-DAG: vaf [[REG:%v[0-9]+]], %v24, %v26
+; CHECK-DAG: vlgvh {{%r[0-5]}}, [[REG]], 6
+; CHECK-DAG: vlgvh {{%r[0-5]}}, %v28, 3
+; CHECK: br %r14
+  %add = add <4 x i32> %v1, %v2
+  %splat = shufflevector <2 x i64> %v3, <2 x i64> undef,
+                         <2 x i32> <i32 0, i32 0>
+  %splatcast = bitcast <2 x i64> %splat to <4 x i32>
+  %shuffle = shufflevector <4 x i32> %add, <4 x i32> %splatcast,
+                           <4 x i32> <i32 1, i32 3, i32 5, i32 7>
+  %bitcast = bitcast <4 x i32> %shuffle to <8 x i16>
+  %elem1 = extractelement <8 x i16> %bitcast, i32 2
+  %elem2 = extractelement <8 x i16> %bitcast, i32 7
+  %res = add i16 %elem1, %elem2
+  ret i16 %res
+}
+
+; ...and again with a merge low instead of a pack.
+define i16 @f4(<4 x i32> %v1, <4 x i32> %v2, <2 x i64> %v3) {
+; CHECK-LABEL: f4:
+; CHECK-NOT: vrepg
+; CHECK-NOT: vmr
+; CHECK-DAG: vaf [[REG:%v[0-9]+]], %v24, %v26
+; CHECK-DAG: vlgvh {{%r[0-5]}}, [[REG]], 6
+; CHECK-DAG: vlgvh {{%r[0-5]}}, %v28, 3
+; CHECK: br %r14
+  %add = add <4 x i32> %v1, %v2
+  %splat = shufflevector <2 x i64> %v3, <2 x i64> undef,
+                         <2 x i32> <i32 0, i32 0>
+  %splatcast = bitcast <2 x i64> %splat to <4 x i32>
+  %shuffle = shufflevector <4 x i32> %add, <4 x i32> %splatcast,
+                           <4 x i32> <i32 2, i32 6, i32 3, i32 7>
+  %bitcast = bitcast <4 x i32> %shuffle to <8 x i16>
+  %elem1 = extractelement <8 x i16> %bitcast, i32 4
+  %elem2 = extractelement <8 x i16> %bitcast, i32 7
+  %res = add i16 %elem1, %elem2
+  ret i16 %res
+}
+
+; ...and again with a merge high.
+define i16 @f5(<4 x i32> %v1, <4 x i32> %v2, <2 x i64> %v3) {
+; CHECK-LABEL: f5:
+; CHECK-NOT: vrepg
+; CHECK-NOT: vmr
+; CHECK-DAG: vaf [[REG:%v[0-9]+]], %v24, %v26
+; CHECK-DAG: vlgvh {{%r[0-5]}}, [[REG]], 2
+; CHECK-DAG: vlgvh {{%r[0-5]}}, %v28, 3
+; CHECK: br %r14
+  %add = add <4 x i32> %v1, %v2
+  %splat = shufflevector <2 x i64> %v3, <2 x i64> undef,
+                         <2 x i32> <i32 0, i32 0>
+  %splatcast = bitcast <2 x i64> %splat to <4 x i32>
+  %shuffle = shufflevector <4 x i32> %add, <4 x i32> %splatcast,
+                           <4 x i32> <i32 0, i32 4, i32 1, i32 5>
+  %bitcast = bitcast <4 x i32> %shuffle to <8 x i16>
+  %elem1 = extractelement <8 x i16> %bitcast, i32 4
+  %elem2 = extractelement <8 x i16> %bitcast, i32 7
+  %res = add i16 %elem1, %elem2
+  ret i16 %res
+}
+
+; Test a case where an unpack high can be eliminated from the usual
+; load-extend sequence.
+define void @f6(<8 x i8> *%ptr1, i8 *%ptr2, i8 *%ptr3, i8 *%ptr4) {
+; CHECK-LABEL: f6:
+; CHECK: vlrepg [[REG:%v[0-9]+]], 0(%r2)
+; CHECK-NOT: vup
+; CHECK-DAG: vsteb [[REG]], 0(%r3), 1
+; CHECK-DAG: vsteb [[REG]], 0(%r4), 2
+; CHECK-DAG: vsteb [[REG]], 0(%r5), 7
+; CHECK: br %r14
+  %vec = load <8 x i8>, <8 x i8> *%ptr1
+  %ext = sext <8 x i8> %vec to <8 x i16>
+  %elem1 = extractelement <8 x i16> %ext, i32 1
+  %elem2 = extractelement <8 x i16> %ext, i32 2
+  %elem3 = extractelement <8 x i16> %ext, i32 7
+  %trunc1 = trunc i16 %elem1 to i8
+  %trunc2 = trunc i16 %elem2 to i8
+  %trunc3 = trunc i16 %elem3 to i8
+  store i8 %trunc1, i8 *%ptr2
+  store i8 %trunc2, i8 *%ptr3
+  store i8 %trunc3, i8 *%ptr4
+  ret void
+}
+
+; ...and again with a bitcast inbetween.
+define void @f7(<4 x i8> *%ptr1, i8 *%ptr2, i8 *%ptr3, i8 *%ptr4) {
+; CHECK-LABEL: f7:
+; CHECK: vlrepf [[REG:%v[0-9]+]], 0(%r2)
+; CHECK-NOT: vup
+; CHECK-DAG: vsteb [[REG]], 0(%r3), 0
+; CHECK-DAG: vsteb [[REG]], 0(%r4), 1
+; CHECK-DAG: vsteb [[REG]], 0(%r5), 3
+; CHECK: br %r14
+  %vec = load <4 x i8>, <4 x i8> *%ptr1
+  %ext = sext <4 x i8> %vec to <4 x i32>
+  %bitcast = bitcast <4 x i32> %ext to <8 x i16>
+  %elem1 = extractelement <8 x i16> %bitcast, i32 1
+  %elem2 = extractelement <8 x i16> %bitcast, i32 3
+  %elem3 = extractelement <8 x i16> %bitcast, i32 7
+  %trunc1 = trunc i16 %elem1 to i8
+  %trunc2 = trunc i16 %elem2 to i8
+  %trunc3 = trunc i16 %elem3 to i8
+  store i8 %trunc1, i8 *%ptr2
+  store i8 %trunc2, i8 *%ptr3
+  store i8 %trunc3, i8 *%ptr4
+  ret void
+}
diff --git a/test/CodeGen/SystemZ/vec-combine-02.ll b/test/CodeGen/SystemZ/vec-combine-02.ll
new file mode 100644
index 00000000000..db0bf849017
--- /dev/null
+++ b/test/CodeGen/SystemZ/vec-combine-02.ll
@@ -0,0 +1,433 @@
+; Test various representations of pack-like operations.
+;
+; RUN: llc < %s -mtriple=s390x-linux-gnu -mcpu=z13 | FileCheck %s
+
+; One way of writing a <4 x i32> -> <8 x i16> pack.
+define <8 x i16> @f1(<4 x i32> %val0, <4 x i32> %val1) {
+; CHECK-LABEL: f1:
+; CHECK: vpkf %v24, %v24, %v26
+; CHECK: br %r14
+  %elem0 = extractelement <4 x i32> %val0, i32 0
+  %elem1 = extractelement <4 x i32> %val0, i32 1
+  %elem2 = extractelement <4 x i32> %val0, i32 2
+  %elem3 = extractelement <4 x i32> %val0, i32 3
+  %elem4 = extractelement <4 x i32> %val1, i32 0
+  %elem5 = extractelement <4 x i32> %val1, i32 1
+  %elem6 = extractelement <4 x i32> %val1, i32 2
+  %elem7 = extractelement <4 x i32> %val1, i32 3
+  %hboth0 = bitcast i32 %elem0 to <2 x i16>
+  %hboth1 = bitcast i32 %elem1 to <2 x i16>
+  %hboth2 = bitcast i32 %elem2 to <2 x i16>
+  %hboth3 = bitcast i32 %elem3 to <2 x i16>
+  %hboth4 = bitcast i32 %elem4 to <2 x i16>
+  %hboth5 = bitcast i32 %elem5 to <2 x i16>
+  %hboth6 = bitcast i32 %elem6 to <2 x i16>
+  %hboth7 = bitcast i32 %elem7 to <2 x i16>
+  %hlow0 = shufflevector <2 x i16> %hboth0, <2 x i16> %hboth1,
+                         <2 x i32> <i32 1, i32 3>
+  %hlow1 = shufflevector <2 x i16> %hboth2, <2 x i16> %hboth3,
+                         <2 x i32> <i32 1, i32 3>
+  %hlow2 = shufflevector <2 x i16> %hboth4, <2 x i16> %hboth5,
+                         <2 x i32> <i32 1, i32 3>
+  %hlow3 = shufflevector <2 x i16> %hboth6, <2 x i16> %hboth7,
+                         <2 x i32> <i32 1, i32 3>
+  %join0 = shufflevector <2 x i16> %hlow0, <2 x i16> %hlow1,
+                         <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %join1 = shufflevector <2 x i16> %hlow2, <2 x i16> %hlow3,
+                         <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %ret = shufflevector <4 x i16> %join0, <4 x i16> %join1,
+                       <8 x i32> <i32 0, i32 1, i32 2, i32 3,
+                                  i32 4, i32 5, i32 6, i32 7>
+  ret <8 x i16> %ret
+}
+
+; A different way of writing a <4 x i32> -> <8 x i16> pack.
+define <8 x i16> @f2(<4 x i32> %val0, <4 x i32> %val1) {
+; CHECK-LABEL: f2:
+; CHECK: vpkf %v24, %v24, %v26
+; CHECK: br %r14
+  %elem0 = extractelement <4 x i32> %val0, i32 0
+  %elem1 = extractelement <4 x i32> %val0, i32 1
+  %elem2 = extractelement <4 x i32> %val0, i32 2
+  %elem3 = extractelement <4 x i32> %val0, i32 3
+  %elem4 = extractelement <4 x i32> %val1, i32 0
+  %elem5 = extractelement <4 x i32> %val1, i32 1
+  %elem6 = extractelement <4 x i32> %val1, i32 2
+  %elem7 = extractelement <4 x i32> %val1, i32 3
+  %wvec0 = insertelement <4 x i32> undef, i32 %elem0, i32 0
+  %wvec1 = insertelement <4 x i32> undef, i32 %elem1, i32 0
+  %wvec2 = insertelement <4 x i32> undef, i32 %elem2, i32 0
+  %wvec3 = insertelement <4 x i32> undef, i32 %elem3, i32 0
+  %wvec4 = insertelement <4 x i32> undef, i32 %elem4, i32 0
+  %wvec5 = insertelement <4 x i32> undef, i32 %elem5, i32 0
+  %wvec6 = insertelement <4 x i32> undef, i32 %elem6, i32 0
+  %wvec7 = insertelement <4 x i32> undef, i32 %elem7, i32 0
+  %hvec0 = bitcast <4 x i32> %wvec0 to <8 x i16>
+  %hvec1 = bitcast <4 x i32> %wvec1 to <8 x i16>
+  %hvec2 = bitcast <4 x i32> %wvec2 to <8 x i16>
+  %hvec3 = bitcast <4 x i32> %wvec3 to <8 x i16>
+  %hvec4 = bitcast <4 x i32> %wvec4 to <8 x i16>
+  %hvec5 = bitcast <4 x i32> %wvec5 to <8 x i16>
+  %hvec6 = bitcast <4 x i32> %wvec6 to <8 x i16>
+  %hvec7 = bitcast <4 x i32> %wvec7 to <8 x i16>
+  %hlow0 = shufflevector <8 x i16> %hvec0, <8 x i16> %hvec1,
+                         <8 x i32> <i32 1, i32 9, i32 undef, i32 undef,
+                                    i32 undef, i32 undef, i32 undef, i32 undef>
+  %hlow1 = shufflevector <8 x i16> %hvec2, <8 x i16> %hvec3,
+                         <8 x i32> <i32 1, i32 9, i32 undef, i32 undef,
+                                    i32 undef, i32 undef, i32 undef, i32 undef>
+  %hlow2 = shufflevector <8 x i16> %hvec4, <8 x i16> %hvec5,
+                         <8 x i32> <i32 1, i32 9, i32 undef, i32 undef,
+                                    i32 undef, i32 undef, i32 undef, i32 undef>
+  %hlow3 = shufflevector <8 x i16> %hvec6, <8 x i16> %hvec7,
+                         <8 x i32> <i32 1, i32 9, i32 undef, i32 undef,
+                                    i32 undef, i32 undef, i32 undef, i32 undef>
+  %join0 = shufflevector <8 x i16> %hlow0, <8 x i16> %hlow1,
+                         <8 x i32> <i32 0, i32 1, i32 8, i32 9,
+                                    i32 undef, i32 undef, i32 undef, i32 undef>
+  %join1 = shufflevector <8 x i16> %hlow2, <8 x i16> %hlow3,
+                         <8 x i32> <i32 0, i32 1, i32 8, i32 9,
+                                    i32 undef, i32 undef, i32 undef, i32 undef>
+  %ret = shufflevector <8 x i16> %join0, <8 x i16> %join1,
+                       <8 x i32> <i32 0, i32 1, i32 2, i32 3,
+                                  i32 8, i32 9, i32 10, i32 11>
+  ret <8 x i16> %ret
+}
+
+; A direct pack operation.
+define <8 x i16> @f3(<4 x i32> %val0, <4 x i32> %val1) {
+; CHECK-LABEL: f3:
+; CHECK: vpkf %v24, %v24, %v26
+; CHECK: br %r14
+  %bitcast0 = bitcast <4 x i32> %val0 to <8 x i16>
+  %bitcast1 = bitcast <4 x i32> %val1 to <8 x i16>
+  %ret = shufflevector <8 x i16> %bitcast0, <8 x i16> %bitcast1,
+                       <8 x i32> <i32 1, i32 3, i32 5, i32 7,
+                                  i32 9, i32 11, i32 13, i32 15>
+  ret <8 x i16> %ret
+}
+
+; One way of writing a <4 x i32> -> <16 x i8> pack.  It doesn't matter
+; whether the first pack is VPKF or VPKH since the even bytes of the
+; result are discarded.
+define <16 x i8> @f4(<4 x i32> %val0, <4 x i32> %val1,
+                     <4 x i32> %val2, <4 x i32> %val3) {
+; CHECK-LABEL: f4:
+; CHECK-DAG: vpk{{[hf]}} [[REG1:%v[0-9]+]], %v24, %v26
+; CHECK-DAG: vpk{{[hf]}} [[REG2:%v[0-9]+]], %v28, %v30
+; CHECK: vpkh %v24, [[REG1]], [[REG2]]
+; CHECK: br %r14
+  %bitcast0 = bitcast <4 x i32> %val0 to <8 x i16>
+  %bitcast1 = bitcast <4 x i32> %val1 to <8 x i16>
+  %bitcast2 = bitcast <4 x i32> %val2 to <8 x i16>
+  %bitcast3 = bitcast <4 x i32> %val3 to <8 x i16>
+  %join0 = shufflevector <8 x i16> %bitcast0, <8 x i16> %bitcast1,
+                         <8 x i32> <i32 1, i32 3, i32 5, i32 7,
+                                    i32 9, i32 11, i32 13, i32 15>
+  %join1 = shufflevector <8 x i16> %bitcast2, <8 x i16> %bitcast3,
+                         <8 x i32> <i32 1, i32 3, i32 5, i32 7,
+                                    i32 9, i32 11, i32 13, i32 15>
+  %bitcast4 = bitcast <8 x i16> %join0 to <16 x i8>
+  %bitcast5 = bitcast <8 x i16> %join1 to <16 x i8>
+  %ret = shufflevector <16 x i8> %bitcast4, <16 x i8> %bitcast5,
+                       <16 x i32> <i32 1, i32 3, i32 5, i32 7,
+                                   i32 9, i32 11, i32 13, i32 15,
+                                   i32 17, i32 19, i32 21, i32 23,
+                                   i32 25, i32 27, i32 29, i32 31>
+  ret <16 x i8> %ret
+}
+
+; Check the same operation, but with elements being extracted from the result.
+define void @f5(<4 x i32> %val0, <4 x i32> %val1,
+                <4 x i32> %val2, <4 x i32> %val3,
+                i8 *%base) {
+; CHECK-LABEL: f5:
+; CHECK-DAG: vsteb %v24, 0(%r2), 11
+; CHECK-DAG: vsteb %v26, 1(%r2), 15
+; CHECK-DAG: vsteb %v28, 2(%r2), 3
+; CHECK-DAG: vsteb %v30, 3(%r2), 7
+; CHECK: br %r14
+  %bitcast0 = bitcast <4 x i32> %val0 to <8 x i16>
+  %bitcast1 = bitcast <4 x i32> %val1 to <8 x i16>
+  %bitcast2 = bitcast <4 x i32> %val2 to <8 x i16>
+  %bitcast3 = bitcast <4 x i32> %val3 to <8 x i16>
+  %join0 = shufflevector <8 x i16> %bitcast0, <8 x i16> %bitcast1,
+                         <8 x i32> <i32 1, i32 3, i32 5, i32 7,
+                                    i32 9, i32 11, i32 13, i32 15>
+  %join1 = shufflevector <8 x i16> %bitcast2, <8 x i16> %bitcast3,
+                         <8 x i32> <i32 1, i32 3, i32 5, i32 7,
+                                    i32 9, i32 11, i32 13, i32 15>
+  %bitcast4 = bitcast <8 x i16> %join0 to <16 x i8>
+  %bitcast5 = bitcast <8 x i16> %join1 to <16 x i8>
+  %vec = shufflevector <16 x i8> %bitcast4, <16 x i8> %bitcast5,
+                       <16 x i32> <i32 1, i32 3, i32 5, i32 7,
+                                   i32 9, i32 11, i32 13, i32 15,
+                                   i32 17, i32 19, i32 21, i32 23,
+                                   i32 25, i32 27, i32 29, i32 31>
+
+  %ptr0 = getelementptr i8, i8 *%base, i64 0
+  %ptr1 = getelementptr i8, i8 *%base, i64 1
+  %ptr2 = getelementptr i8, i8 *%base, i64 2
+  %ptr3 = getelementptr i8, i8 *%base, i64 3
+
+  %byte0 = extractelement <16 x i8> %vec, i32 2
+  %byte1 = extractelement <16 x i8> %vec, i32 7
+  %byte2 = extractelement <16 x i8> %vec, i32 8
+  %byte3 = extractelement <16 x i8> %vec, i32 13
+
+  store i8 %byte0, i8 *%ptr0
+  store i8 %byte1, i8 *%ptr1
+  store i8 %byte2, i8 *%ptr2
+  store i8 %byte3, i8 *%ptr3
+
+  ret void
+}
+
+; A different way of writing a <4 x i32> -> <16 x i8> pack.
+define <16 x i8> @f6(<4 x i32> %val0, <4 x i32> %val1,
+                     <4 x i32> %val2, <4 x i32> %val3) {
+; CHECK-LABEL: f6:
+; CHECK-DAG: vpk{{[hf]}} [[REG1:%v[0-9]+]], %v24, %v26
+; CHECK-DAG: vpk{{[hf]}} [[REG2:%v[0-9]+]], %v28, %v30
+; CHECK: vpkh %v24, [[REG1]], [[REG2]]
+; CHECK: br %r14
+  %elem0 = extractelement <4 x i32> %val0, i32 0
+  %elem1 = extractelement <4 x i32> %val0, i32 1
+  %elem2 = extractelement <4 x i32> %val0, i32 2
+  %elem3 = extractelement <4 x i32> %val0, i32 3
+  %elem4 = extractelement <4 x i32> %val1, i32 0
+  %elem5 = extractelement <4 x i32> %val1, i32 1
+  %elem6 = extractelement <4 x i32> %val1, i32 2
+  %elem7 = extractelement <4 x i32> %val1, i32 3
+  %elem8 = extractelement <4 x i32> %val2, i32 0
+  %elem9 = extractelement <4 x i32> %val2, i32 1
+  %elem10 = extractelement <4 x i32> %val2, i32 2
+  %elem11 = extractelement <4 x i32> %val2, i32 3
+  %elem12 = extractelement <4 x i32> %val3, i32 0
+  %elem13 = extractelement <4 x i32> %val3, i32 1
+  %elem14 = extractelement <4 x i32> %val3, i32 2
+  %elem15 = extractelement <4 x i32> %val3, i32 3
+  %bitcast0 = bitcast i32 %elem0 to <2 x i16>
+  %bitcast1 = bitcast i32 %elem1 to <2 x i16>
+  %bitcast2 = bitcast i32 %elem2 to <2 x i16>
+  %bitcast3 = bitcast i32 %elem3 to <2 x i16>
+  %bitcast4 = bitcast i32 %elem4 to <2 x i16>
+  %bitcast5 = bitcast i32 %elem5 to <2 x i16>
+  %bitcast6 = bitcast i32 %elem6 to <2 x i16>
+  %bitcast7 = bitcast i32 %elem7 to <2 x i16>
+  %bitcast8 = bitcast i32 %elem8 to <2 x i16>
+  %bitcast9 = bitcast i32 %elem9 to <2 x i16>
+  %bitcast10 = bitcast i32 %elem10 to <2 x i16>
+  %bitcast11 = bitcast i32 %elem11 to <2 x i16>
+  %bitcast12 = bitcast i32 %elem12 to <2 x i16>
+  %bitcast13 = bitcast i32 %elem13 to <2 x i16>
+  %bitcast14 = bitcast i32 %elem14 to <2 x i16>
+  %bitcast15 = bitcast i32 %elem15 to <2 x i16>
+  %low0 = shufflevector <2 x i16> %bitcast0, <2 x i16> %bitcast1,
+                        <2 x i32> <i32 1, i32 3>
+  %low1 = shufflevector <2 x i16> %bitcast2, <2 x i16> %bitcast3,
+                        <2 x i32> <i32 1, i32 3>
+  %low2 = shufflevector <2 x i16> %bitcast4, <2 x i16> %bitcast5,
+                        <2 x i32> <i32 1, i32 3>
+  %low3 = shufflevector <2 x i16> %bitcast6, <2 x i16> %bitcast7,
+                        <2 x i32> <i32 1, i32 3>
+  %low4 = shufflevector <2 x i16> %bitcast8, <2 x i16> %bitcast9,
+                        <2 x i32> <i32 1, i32 3>
+  %low5 = shufflevector <2 x i16> %bitcast10, <2 x i16> %bitcast11,
+                        <2 x i32> <i32 1, i32 3>
+  %low6 = shufflevector <2 x i16> %bitcast12, <2 x i16> %bitcast13,
+                        <2 x i32> <i32 1, i32 3>
+  %low7 = shufflevector <2 x i16> %bitcast14, <2 x i16> %bitcast15,
+                        <2 x i32> <i32 1, i32 3>
+  %bytes0 = bitcast <2 x i16> %low0 to <4 x i8>
+  %bytes1 = bitcast <2 x i16> %low1 to <4 x i8>
+  %bytes2 = bitcast <2 x i16> %low2 to <4 x i8>
+  %bytes3 = bitcast <2 x i16> %low3 to <4 x i8>
+  %bytes4 = bitcast <2 x i16> %low4 to <4 x i8>
+  %bytes5 = bitcast <2 x i16> %low5 to <4 x i8>
+  %bytes6 = bitcast <2 x i16> %low6 to <4 x i8>
+  %bytes7 = bitcast <2 x i16> %low7 to <4 x i8>
+  %blow0 = shufflevector <4 x i8> %bytes0, <4 x i8> %bytes1,
+                         <4 x i32> <i32 1, i32 3, i32 5, i32 7>
+  %blow1 = shufflevector <4 x i8> %bytes2, <4 x i8> %bytes3,
+                         <4 x i32> <i32 1, i32 3, i32 5, i32 7>
+  %blow2 = shufflevector <4 x i8> %bytes4, <4 x i8> %bytes5,
+                         <4 x i32> <i32 1, i32 3, i32 5, i32 7>
+  %blow3 = shufflevector <4 x i8> %bytes6, <4 x i8> %bytes7,
+                         <4 x i32> <i32 1, i32 3, i32 5, i32 7>
+  %join0 = shufflevector <4 x i8> %blow0, <4 x i8> %blow1,
+                         <8 x i32> <i32 0, i32 1, i32 2, i32 3,
+                                    i32 4, i32 5, i32 6, i32 7>
+  %join1 = shufflevector <4 x i8> %blow2, <4 x i8> %blow3,
+                         <8 x i32> <i32 0, i32 1, i32 2, i32 3,
+                                    i32 4, i32 5, i32 6, i32 7>
+  %ret = shufflevector <8 x i8> %join0, <8 x i8> %join1,
+                       <16 x i32> <i32 0, i32 1, i32 2, i32 3,
+                                   i32 4, i32 5, i32 6, i32 7,
+                                   i32 8, i32 9, i32 10, i32 11,
+                                   i32 12, i32 13, i32 14, i32 15>
+  ret <16 x i8> %ret
+}
+
+; One way of writing a <2 x i64> -> <16 x i8> pack.
+define <16 x i8> @f7(<2 x i64> %val0, <2 x i64> %val1,
+                     <2 x i64> %val2, <2 x i64> %val3,
+                     <2 x i64> %val4, <2 x i64> %val5,
+                     <2 x i64> %val6, <2 x i64> %val7) {
+; CHECK-LABEL: f7:
+; CHECK-DAG: vpk{{[hfg]}} [[REG1:%v[0-9]+]], %v24, %v26
+; CHECK-DAG: vpk{{[hfg]}} [[REG2:%v[0-9]+]], %v28, %v30
+; CHECK-DAG: vpk{{[hfg]}} [[REG3:%v[0-9]+]], %v25, %v27
+; CHECK-DAG: vpk{{[hfg]}} [[REG4:%v[0-9]+]], %v29, %v31
+; CHECK-DAG: vpk{{[hf]}} [[REG5:%v[0-9]+]], [[REG1]], [[REG2]]
+; CHECK-DAG: vpk{{[hf]}} [[REG6:%v[0-9]+]], [[REG3]], [[REG4]]
+; CHECK: vpkh %v24, [[REG5]], [[REG6]]
+; CHECK: br %r14
+  %elem0 = extractelement <2 x i64> %val0, i32 0
+  %elem1 = extractelement <2 x i64> %val0, i32 1
+  %elem2 = extractelement <2 x i64> %val1, i32 0
+  %elem3 = extractelement <2 x i64> %val1, i32 1
+  %elem4 = extractelement <2 x i64> %val2, i32 0
+  %elem5 = extractelement <2 x i64> %val2, i32 1
+  %elem6 = extractelement <2 x i64> %val3, i32 0
+  %elem7 = extractelement <2 x i64> %val3, i32 1
+  %elem8 = extractelement <2 x i64> %val4, i32 0
+  %elem9 = extractelement <2 x i64> %val4, i32 1
+  %elem10 = extractelement <2 x i64> %val5, i32 0
+  %elem11 = extractelement <2 x i64> %val5, i32 1
+  %elem12 = extractelement <2 x i64> %val6, i32 0
+  %elem13 = extractelement <2 x i64> %val6, i32 1
+  %elem14 = extractelement <2 x i64> %val7, i32 0
+  %elem15 = extractelement <2 x i64> %val7, i32 1
+  %bitcast0 = bitcast i64 %elem0 to <2 x i32>
+  %bitcast1 = bitcast i64 %elem1 to <2 x i32>
+  %bitcast2 = bitcast i64 %elem2 to <2 x i32>
+  %bitcast3 = bitcast i64 %elem3 to <2 x i32>
+  %bitcast4 = bitcast i64 %elem4 to <2 x i32>
+  %bitcast5 = bitcast i64 %elem5 to <2 x i32>
+  %bitcast6 = bitcast i64 %elem6 to <2 x i32>
+  %bitcast7 = bitcast i64 %elem7 to <2 x i32>
+  %bitcast8 = bitcast i64 %elem8 to <2 x i32>
+  %bitcast9 = bitcast i64 %elem9 to <2 x i32>
+  %bitcast10 = bitcast i64 %elem10 to <2 x i32>
+  %bitcast11 = bitcast i64 %elem11 to <2 x i32>
+  %bitcast12 = bitcast i64 %elem12 to <2 x i32>
+  %bitcast13 = bitcast i64 %elem13 to <2 x i32>
+  %bitcast14 = bitcast i64 %elem14 to <2 x i32>
+  %bitcast15 = bitcast i64 %elem15 to <2 x i32>
+  %low0 = shufflevector <2 x i32> %bitcast0, <2 x i32> %bitcast1,
+                        <2 x i32> <i32 1, i32 3>
+  %low1 = shufflevector <2 x i32> %bitcast2, <2 x i32> %bitcast3,
+                        <2 x i32> <i32 1, i32 3>
+  %low2 = shufflevector <2 x i32> %bitcast4, <2 x i32> %bitcast5,
+                        <2 x i32> <i32 1, i32 3>
+  %low3 = shufflevector <2 x i32> %bitcast6, <2 x i32> %bitcast7,
+                        <2 x i32> <i32 1, i32 3>
+  %low4 = shufflevector <2 x i32> %bitcast8, <2 x i32> %bitcast9,
+                        <2 x i32> <i32 1, i32 3>
+  %low5 = shufflevector <2 x i32> %bitcast10, <2 x i32> %bitcast11,
+                        <2 x i32> <i32 1, i32 3>
+  %low6 = shufflevector <2 x i32> %bitcast12, <2 x i32> %bitcast13,
+                        <2 x i32> <i32 1, i32 3>
+  %low7 = shufflevector <2 x i32> %bitcast14, <2 x i32> %bitcast15,
+                        <2 x i32> <i32 1, i32 3>
+  %half0 = bitcast <2 x i32> %low0 to <4 x i16>
+  %half1 = bitcast <2 x i32> %low1 to <4 x i16>
+  %half2 = bitcast <2 x i32> %low2 to <4 x i16>
+  %half3 = bitcast <2 x i32> %low3 to <4 x i16>
+  %half4 = bitcast <2 x i32> %low4 to <4 x i16>
+  %half5 = bitcast <2 x i32> %low5 to <4 x i16>
+  %half6 = bitcast <2 x i32> %low6 to <4 x i16>
+  %half7 = bitcast <2 x i32> %low7 to <4 x i16>
+  %hlow0 = shufflevector <4 x i16> %half0, <4 x i16> %half1,
+                         <4 x i32> <i32 1, i32 3, i32 5, i32 7>
+  %hlow1 = shufflevector <4 x i16> %half2, <4 x i16> %half3,
+                         <4 x i32> <i32 1, i32 3, i32 5, i32 7>
+  %hlow2 = shufflevector <4 x i16> %half4, <4 x i16> %half5,
+                         <4 x i32> <i32 1, i32 3, i32 5, i32 7>
+  %hlow3 = shufflevector <4 x i16> %half6, <4 x i16> %half7,
+                         <4 x i32> <i32 1, i32 3, i32 5, i32 7>
+  %bytes0 = bitcast <4 x i16> %hlow0 to <8 x i8>
+  %bytes1 = bitcast <4 x i16> %hlow1 to <8 x i8>
+  %bytes2 = bitcast <4 x i16> %hlow2 to <8 x i8>
+  %bytes3 = bitcast <4 x i16> %hlow3 to <8 x i8>
+  %join0 = shufflevector <8 x i8> %bytes0, <8 x i8> %bytes1,
+                         <8 x i32> <i32 1, i32 3, i32 5, i32 7,
+                                    i32 9, i32 11, i32 13, i32 15>
+  %join1 = shufflevector <8 x i8> %bytes2, <8 x i8> %bytes3,
+                         <8 x i32> <i32 1, i32 3, i32 5, i32 7,
+                                    i32 9, i32 11, i32 13, i32 15>
+  %ret = shufflevector <8 x i8> %join0, <8 x i8> %join1,
+                       <16 x i32> <i32 0, i32 1, i32 2, i32 3,
+                                   i32 4, i32 5, i32 6, i32 7,
+                                   i32 8, i32 9, i32 10, i32 11,
+                                   i32 12, i32 13, i32 14, i32 15>
+  ret <16 x i8> %ret
+}
+
+; Test a <2 x i64> -> <4 x f32> pack in which only individual elements are
+; needed.
+define float @f8(i64 %scalar0, i64 %scalar1, i64 %scalar2, i64 %scalar3) {
+; CHECK-LABEL: f8:
+; CHECK-NOT: vperm
+; CHECK-NOT: vpk
+; CHECK-NOT: vmrh
+; CHECK: aebr {{%f[0-7]}},
+; CHECK: aebr {{%f[0-7]}},
+; CHECK: meebr %f0,
+; CHECK: br %r14
+  %vec0 = insertelement <2 x i64> undef, i64 %scalar0, i32 0
+  %vec1 = insertelement <2 x i64> undef, i64 %scalar1, i32 0
+  %vec2 = insertelement <2 x i64> undef, i64 %scalar2, i32 0
+  %vec3 = insertelement <2 x i64> undef, i64 %scalar3, i32 0
+  %join0 = shufflevector <2 x i64> %vec0, <2 x i64> %vec1,
+                         <2 x i32> <i32 0, i32 2>
+  %join1 = shufflevector <2 x i64> %vec2, <2 x i64> %vec3,
+                         <2 x i32> <i32 0, i32 2>
+  %bitcast0 = bitcast <2 x i64> %join0 to <4 x float>
+  %bitcast1 = bitcast <2 x i64> %join1 to <4 x float>
+  %pack = shufflevector <4 x float> %bitcast0, <4 x float> %bitcast1,
+                        <4 x i32> <i32 1, i32 3, i32 5, i32 7>
+  %elt0 = extractelement <4 x float> %pack, i32 0
+  %elt1 = extractelement <4 x float> %pack, i32 1
+  %elt2 = extractelement <4 x float> %pack, i32 2
+  %elt3 = extractelement <4 x float> %pack, i32 3
+  %add0 = fadd float %elt0, %elt2
+  %add1 = fadd float %elt1, %elt3
+  %ret = fmul float %add0, %add1
+  ret float %ret
+}
+
+; Test a <2 x f64> -> <4 x i32> pack in which only individual elements are
+; needed.
+define i32 @f9(double %scalar0, double %scalar1, double %scalar2,
+               double %scalar3) {
+; CHECK-LABEL: f9:
+; CHECK-NOT: vperm
+; CHECK-NOT: vpk
+; CHECK-NOT: vmrh
+; CHECK: ar {{%r[0-5]}},
+; CHECK: ar {{%r[0-5]}},
+; CHECK: or %r2,
+; CHECK: br %r14
+  %vec0 = insertelement <2 x double> undef, double %scalar0, i32 0
+  %vec1 = insertelement <2 x double> undef, double %scalar1, i32 0
+  %vec2 = insertelement <2 x double> undef, double %scalar2, i32 0
+  %vec3 = insertelement <2 x double> undef, double %scalar3, i32 0
+  %join0 = shufflevector <2 x double> %vec0, <2 x double> %vec1,
+                         <2 x i32> <i32 0, i32 2>
+  %join1 = shufflevector <2 x double> %vec2, <2 x double> %vec3,
+                         <2 x i32> <i32 0, i32 2>
+  %bitcast0 = bitcast <2 x double> %join0 to <4 x i32>
+  %bitcast1 = bitcast <2 x double> %join1 to <4 x i32>
+  %pack = shufflevector <4 x i32> %bitcast0, <4 x i32> %bitcast1,
+                        <4 x i32> <i32 1, i32 3, i32 5, i32 7>
+  %elt0 = extractelement <4 x i32> %pack, i32 0
+  %elt1 = extractelement <4 x i32> %pack, i32 1
+  %elt2 = extractelement <4 x i32> %pack, i32 2
+  %elt3 = extractelement <4 x i32> %pack, i32 3
+  %add0 = add i32 %elt0, %elt2
+  %add1 = add i32 %elt1, %elt3
+  %ret = or i32 %add0, %add1
+  ret i32 %ret
+}
diff --git a/test/CodeGen/SystemZ/vec-const-01.ll b/test/CodeGen/SystemZ/vec-const-01.ll
new file mode 100644
index 00000000000..4cdcbf7c2dc
--- /dev/null
+++ b/test/CodeGen/SystemZ/vec-const-01.ll
@@ -0,0 +1,103 @@
+; Test vector byte masks, v16i8 version.
+;
+; RUN: llc < %s -mtriple=s390x-linux-gnu -mcpu=z13 | FileCheck %s
+
+; Test an all-zeros vector.
+define <16 x i8> @f1() {
+; CHECK-LABEL: f1:
+; CHECK: vgbm %v24, 0
+; CHECK: br %r14
+  ret <16 x i8> zeroinitializer
+}
+
+; Test an all-ones vector.
+define <16 x i8> @f2() {
+; CHECK-LABEL: f2:
+; CHECK: vgbm %v24, 65535
+; CHECK: br %r14
+  ret <16 x i8> <i8 -1, i8 -1, i8 -1, i8 -1,
+                 i8 -1, i8 -1, i8 -1, i8 -1,
+                 i8 -1, i8 -1, i8 -1, i8 -1,
+                 i8 -1, i8 -1, i8 -1, i8 -1>
+}
+
+; Test a mixed vector (mask 0x8c75).
+define <16 x i8> @f3() {
+; CHECK-LABEL: f3:
+; CHECK: vgbm %v24, 35957
+; CHECK: br %r14
+  ret <16 x i8> <i8 -1, i8 0, i8 0, i8 0,
+                 i8 -1, i8 -1, i8 0, i8 0,
+                 i8 0, i8 -1, i8 -1, i8 -1,
+                 i8 0, i8 -1, i8 0, i8 -1>
+}
+
+; Test that undefs are treated as zero.
+define <16 x i8> @f4() {
+; CHECK-LABEL: f4:
+; CHECK: vgbm %v24, 35957
+; CHECK: br %r14
+  ret <16 x i8> <i8 -1, i8 undef, i8 undef, i8 undef,
+                 i8 -1, i8 -1, i8 undef, i8 undef,
+                 i8 undef, i8 -1, i8 -1, i8 -1,
+                 i8 undef, i8 -1, i8 undef, i8 -1>
+}
+
+; Test that we don't use VGBM if one of the bytes is not 0 or 0xff.
+define <16 x i8> @f5() {
+; CHECK-LABEL: f5:
+; CHECK-NOT: vgbm
+; CHECK: br %r14
+  ret <16 x i8> <i8 -1, i8 0, i8 0, i8 0,
+                 i8 -1, i8 -1, i8 0, i8 1,
+                 i8 0, i8 -1, i8 -1, i8 -1,
+                 i8 0, i8 -1, i8 0, i8 -1>
+}
+
+; Test an all-zeros v2i8 that gets promoted to v16i8.
+define <2 x i8> @f6() {
+; CHECK-LABEL: f6:
+; CHECK: vgbm %v24, 0
+; CHECK: br %r14
+  ret <2 x i8> zeroinitializer
+}
+
+; Test a mixed v2i8 that gets promoted to v16i8 (mask 0x8000).
+define <2 x i8> @f7() {
+; CHECK-LABEL: f7:
+; CHECK: vgbm %v24, 32768
+; CHECK: br %r14
+  ret <2 x i8> <i8 255, i8 0>
+}
+
+; Test an all-zeros v4i8 that gets promoted to v16i8.
+define <4 x i8> @f8() {
+; CHECK-LABEL: f8:
+; CHECK: vgbm %v24, 0
+; CHECK: br %r14
+  ret <4 x i8> zeroinitializer
+}
+
+; Test a mixed v4i8 that gets promoted to v16i8 (mask 0x9000).
+define <4 x i8> @f9() {
+; CHECK-LABEL: f9:
+; CHECK: vgbm %v24, 36864
+; CHECK: br %r14
+  ret <4 x i8> <i8 255, i8 0, i8 0, i8 255>
+}
+
+; Test an all-zeros v8i8 that gets promoted to v16i8.
+define <8 x i8> @f10() {
+; CHECK-LABEL: f10:
+; CHECK: vgbm %v24, 0
+; CHECK: br %r14
+  ret <8 x i8> zeroinitializer
+}
+
+; Test a mixed v8i8 that gets promoted to v16i8 (mask 0xE500).
+define <8 x i8> @f11() {
+; CHECK-LABEL: f11:
+; CHECK: vgbm %v24, 58624
+; CHECK: br %r14
+  ret <8 x i8> <i8 255, i8 255, i8 255, i8 0, i8 0, i8 255, i8 0, i8 255>
+}
diff --git a/test/CodeGen/SystemZ/vec-const-02.ll b/test/CodeGen/SystemZ/vec-const-02.ll
new file mode 100644
index 00000000000..73a89d4a841
--- /dev/null
+++ b/test/CodeGen/SystemZ/vec-const-02.ll
@@ -0,0 +1,79 @@
+; Test vector byte masks, v8i16 version.
+;
+; RUN: llc < %s -mtriple=s390x-linux-gnu -mcpu=z13 | FileCheck %s
+
+; Test an all-zeros vector.
+define <8 x i16> @f1() {
+; CHECK-LABEL: f1:
+; CHECK: vgbm %v24, 0
+; CHECK: br %r14
+  ret <8 x i16> zeroinitializer
+}
+
+; Test an all-ones vector.
+define <8 x i16> @f2() {
+; CHECK-LABEL: f2:
+; CHECK: vgbm %v24, 65535
+; CHECK: br %r14
+  ret <8 x i16> <i16 -1, i16 -1, i16 -1, i16 -1,
+                 i16 -1, i16 -1, i16 -1, i16 -1>
+}
+
+; Test a mixed vector (mask 0x8c76).
+define <8 x i16> @f3() {
+; CHECK-LABEL: f3:
+; CHECK: vgbm %v24, 35958
+; CHECK: br %r14
+  ret <8 x i16> <i16 65280, i16 0, i16 65535, i16 0,
+                 i16 255, i16 65535, i16 255, i16 65280>
+}
+
+; Test that undefs are treated as zero.
+define <8 x i16> @f4() {
+; CHECK-LABEL: f4:
+; CHECK: vgbm %v24, 35958
+; CHECK: br %r14
+  ret <8 x i16> <i16 65280, i16 undef, i16 65535, i16 undef,
+                 i16 255, i16 65535, i16 255, i16 65280>
+}
+
+; Test that we don't use VGBM if one of the bytes is not 0 or 0xff.
+define <8 x i16> @f5() {
+; CHECK-LABEL: f5:
+; CHECK-NOT: vgbm
+; CHECK: br %r14
+  ret <8 x i16> <i16 65280, i16 0, i16 65535, i16 0,
+                 i16 255, i16 65535, i16 256, i16 65280>
+}
+
+; Test an all-zeros v2i16 that gets promoted to v8i16.
+define <2 x i16> @f6() {
+; CHECK-LABEL: f6:
+; CHECK: vgbm %v24, 0
+; CHECK: br %r14
+  ret <2 x i16> zeroinitializer
+}
+
+; Test a mixed v2i16 that gets promoted to v8i16 (mask 0xc000).
+define <2 x i16> @f7() {
+; CHECK-LABEL: f7:
+; CHECK: vgbm %v24, 49152
+; CHECK: br %r14
+  ret <2 x i16> <i16 65535, i16 0>
+}
+
+; Test an all-zeros v4i16 that gets promoted to v8i16.
+define <4 x i16> @f8() {
+; CHECK-LABEL: f8:
+; CHECK: vgbm %v24, 0
+; CHECK: br %r14
+  ret <4 x i16> zeroinitializer
+}
+
+; Test a mixed v4i16 that gets promoted to v8i16 (mask 0x7200).
+define <4 x i16> @f9() {
+; CHECK-LABEL: f9:
+; CHECK: vgbm %v24, 29184
+; CHECK: br %r14
+  ret <4 x i16> <i16 255, i16 65535, i16 0, i16 65280>
+}
diff --git a/test/CodeGen/SystemZ/vec-const-03.ll b/test/CodeGen/SystemZ/vec-const-03.ll
new file mode 100644
index 00000000000..adc1105229e
--- /dev/null
+++ b/test/CodeGen/SystemZ/vec-const-03.ll
@@ -0,0 +1,59 @@
+; Test vector byte masks, v4i32 version.
+;
+; RUN: llc < %s -mtriple=s390x-linux-gnu -mcpu=z13 | FileCheck %s
+
+; Test an all-zeros vector.
+define <4 x i32> @f1() {
+; CHECK-LABEL: f1:
+; CHECK: vgbm %v24, 0
+; CHECK: br %r14
+  ret <4 x i32> zeroinitializer
+}
+
+; Test an all-ones vector.
+define <4 x i32> @f2() {
+; CHECK-LABEL: f2:
+; CHECK: vgbm %v24, 65535
+; CHECK: br %r14
+  ret <4 x i32> <i32 -1, i32 -1, i32 -1, i32 -1>
+}
+
+; Test a mixed vector (mask 0x8c76).
+define <4 x i32> @f3() {
+; CHECK-LABEL: f3:
+; CHECK: vgbm %v24, 35958
+; CHECK: br %r14
+  ret <4 x i32> <i32 4278190080, i32 4294901760, i32 16777215, i32 16776960>
+}
+
+; Test that undefs are treated as zero (mask 0x8076).
+define <4 x i32> @f4() {
+; CHECK-LABEL: f4:
+; CHECK: vgbm %v24, 32886
+; CHECK: br %r14
+  ret <4 x i32> <i32 4278190080, i32 undef, i32 16777215, i32 16776960>
+}
+
+; Test that we don't use VGBM if one of the bytes is not 0 or 0xff.
+define <4 x i32> @f5() {
+; CHECK-LABEL: f5:
+; CHECK-NOT: vgbm
+; CHECK: br %r14
+  ret <4 x i32> <i32 4278190080, i32 1, i32 16777215, i32 16776960>
+}
+
+; Test an all-zeros v2i32 that gets promoted to v4i32.
+define <2 x i32> @f6() {
+; CHECK-LABEL: f6:
+; CHECK: vgbm %v24, 0
+; CHECK: br %r14
+  ret <2 x i32> zeroinitializer
+}
+
+; Test a mixed v2i32 that gets promoted to v4i32 (mask 0xae00).
+define <2 x i32> @f7() {
+; CHECK-LABEL: f7:
+; CHECK: vgbm %v24, 44544
+; CHECK: br %r14
+  ret <2 x i32> <i32 4278255360, i32 -256>
+}
diff --git a/test/CodeGen/SystemZ/vec-const-04.ll b/test/CodeGen/SystemZ/vec-const-04.ll
new file mode 100644
index 00000000000..1c2fb414d25
--- /dev/null
+++ b/test/CodeGen/SystemZ/vec-const-04.ll
@@ -0,0 +1,43 @@
+; Test vector byte masks, v2i64 version.
+;
+; RUN: llc < %s -mtriple=s390x-linux-gnu -mcpu=z13 | FileCheck %s
+
+; Test an all-zeros vector.
+define <2 x i64> @f1() {
+; CHECK-LABEL: f1:
+; CHECK: vgbm %v24, 0
+; CHECK: br %r14
+  ret <2 x i64> zeroinitializer
+}
+
+; Test an all-ones vector.
+define <2 x i64> @f2() {
+; CHECK-LABEL: f2:
+; CHECK: vgbm %v24, 65535
+; CHECK: br %r14
+  ret <2 x i64> <i64 -1, i64 -1>
+}
+
+; Test a mixed vector (mask 0x8c76).
+define <2 x i64> @f3() {
+; CHECK-LABEL: f3:
+; CHECK: vgbm %v24, 35958
+; CHECK: br %r14
+  ret <2 x i64> <i64 18374686483966525440, i64 72057589759737600>
+}
+
+; Test that undefs are treated as zero (mask 0x8c00).
+define <2 x i64> @f4() {
+; CHECK-LABEL: f4:
+; CHECK: vgbm %v24, 35840
+; CHECK: br %r14
+  ret <2 x i64> <i64 18374686483966525440, i64 undef>
+}
+
+; Test that we don't use VGBM if one of the bytes is not 0 or 0xff.
+define <2 x i64> @f5() {
+; CHECK-LABEL: f5:
+; CHECK-NOT: vgbm
+; CHECK: br %r14
+  ret <2 x i64> <i64 18374686483966525441, i64 72057589759737600>
+}
diff --git a/test/CodeGen/SystemZ/vec-const-05.ll b/test/CodeGen/SystemZ/vec-const-05.ll
new file mode 100644
index 00000000000..55f3cdd5902
--- /dev/null
+++ b/test/CodeGen/SystemZ/vec-const-05.ll
@@ -0,0 +1,63 @@
+; Test vector byte masks, v4f32 version.
+;
+; RUN: llc < %s -mtriple=s390x-linux-gnu -mcpu=z13 | FileCheck %s
+
+; Test an all-zeros vector.
+define <4 x float> @f1() {
+; CHECK-LABEL: f1:
+; CHECK: vgbm %v24, 0
+; CHECK: br %r14
+  ret <4 x float> zeroinitializer
+}
+
+; Test an all-ones vector.
+define <4 x float> @f2() {
+; CHECK-LABEL: f2:
+; CHECK: vgbm %v24, 65535
+; CHECK: br %r14
+  ret <4 x float> <float 0xffffffffe0000000, float 0xffffffffe0000000,
+                   float 0xffffffffe0000000, float 0xffffffffe0000000>
+}
+
+; Test a mixed vector (mask 0xc731).
+define <4 x float> @f3() {
+; CHECK-LABEL: f3:
+; CHECK: vgbm %v24, 50993
+; CHECK: br %r14
+  ret <4 x float> <float 0xffffe00000000000, float 0x381fffffe0000000,
+                   float 0x379fffe000000000, float 0x371fe00000000000>
+}
+
+; Test that undefs are treated as zero (mask 0xc031).
+define <4 x float> @f4() {
+; CHECK-LABEL: f4:
+; CHECK: vgbm %v24, 49201
+; CHECK: br %r14
+  ret <4 x float> <float 0xffffe00000000000, float undef,
+                   float 0x379fffe000000000, float 0x371fe00000000000>
+}
+
+; Test that we don't use VGBM if one of the bytes is not 0 or 0xff.
+define <4 x float> @f5() {
+; CHECK-LABEL: f5:
+; CHECK-NOT: vgbm
+; CHECK: br %r14
+  ret <4 x float> <float 0xffffe00000000000, float 0x381fffffc0000000,
+                   float 0x379fffe000000000, float 0x371fe00000000000>
+}
+
+; Test an all-zeros v2f32 that gets promoted to v4f32.
+define <2 x float> @f6() {
+; CHECK-LABEL: f6:
+; CHECK: vgbm %v24, 0
+; CHECK: br %r14
+  ret <2 x float> zeroinitializer
+}
+
+; Test a mixed v2f32 that gets promoted to v4f32 (mask 0xc700).
+define <2 x float> @f7() {
+; CHECK-LABEL: f7:
+; CHECK: vgbm %v24, 50944
+; CHECK: br %r14
+  ret <2 x float> <float 0xffffe00000000000, float 0x381fffffe0000000>
+}
diff --git a/test/CodeGen/SystemZ/vec-const-06.ll b/test/CodeGen/SystemZ/vec-const-06.ll
new file mode 100644
index 00000000000..be53a0581ec
--- /dev/null
+++ b/test/CodeGen/SystemZ/vec-const-06.ll
@@ -0,0 +1,43 @@
+; Test vector byte masks, v2f64 version.
+;
+; RUN: llc < %s -mtriple=s390x-linux-gnu -mcpu=z13 | FileCheck %s
+
+; Test an all-zeros vector.
+define <2 x double> @f1() {
+; CHECK-LABEL: f1:
+; CHECK: vgbm %v24, 0
+; CHECK: br %r14
+  ret <2 x double> zeroinitializer
+}
+
+; Test an all-ones vector.
+define <2 x double> @f2() {
+; CHECK-LABEL: f2:
+; CHECK: vgbm %v24, 65535
+; CHECK: br %r14
+  ret <2 x double> <double 0xffffffffffffffff, double 0xffffffffffffffff>
+}
+
+; Test a mixed vector (mask 0x8c76).
+define <2 x double> @f3() {
+; CHECK-LABEL: f3:
+; CHECK: vgbm %v24, 35958
+; CHECK: br %r14
+  ret <2 x double> <double 0xff000000ffff0000, double 0x00ffffff00ffff00>
+}
+
+; Test that undefs are treated as zero (mask 0x8c00).
+define <2 x double> @f4() {
+; CHECK-LABEL: f4:
+; CHECK: vgbm %v24, 35840
+; CHECK: br %r14
+  ret <2 x double> <double 0xff000000ffff0000, double undef>
+}
+
+; Test that we don't use VGBM if one of the bytes is not 0 or 0xff.
+define <2 x double> @f5() {
+; CHECK-LABEL: f5:
+; CHECK-NOT: vgbm
+; CHECK: br %r14
+  ret <2 x double> <double 0xfe000000ffff0000, double 0x00ffffff00ffff00>
+}
diff --git a/test/CodeGen/SystemZ/vec-const-07.ll b/test/CodeGen/SystemZ/vec-const-07.ll
new file mode 100644
index 00000000000..6fcf95b6921
--- /dev/null
+++ b/test/CodeGen/SystemZ/vec-const-07.ll
@@ -0,0 +1,229 @@
+; Test vector replicates, v16i8 version.
+;
+; RUN: llc < %s -mtriple=s390x-linux-gnu -mcpu=z13 | FileCheck %s
+
+; Test a byte-granularity replicate with the lowest useful value.
+define <16 x i8> @f1() {
+; CHECK-LABEL: f1:
+; CHECK: vrepib %v24, 1
+; CHECK: br %r14
+  ret <16 x i8> <i8 1, i8 1, i8 1, i8 1,
+                 i8 1, i8 1, i8 1, i8 1,
+                 i8 1, i8 1, i8 1, i8 1,
+                 i8 1, i8 1, i8 1, i8 1>
+}
+
+; Test a byte-granularity replicate with an arbitrary value.
+define <16 x i8> @f2() {
+; CHECK-LABEL: f2:
+; CHECK: vrepib %v24, -55
+; CHECK: br %r14
+  ret <16 x i8> <i8 201, i8 201, i8 201, i8 201,
+                 i8 201, i8 201, i8 201, i8 201,
+                 i8 201, i8 201, i8 201, i8 201,
+                 i8 201, i8 201, i8 201, i8 201>
+}
+
+; Test a byte-granularity replicate with the highest useful value.
+define <16 x i8> @f3() {
+; CHECK-LABEL: f3:
+; CHECK: vrepib %v24, -2
+; CHECK: br %r14
+  ret <16 x i8> <i8 254, i8 254, i8 254, i8 254,
+                 i8 254, i8 254, i8 254, i8 254,
+                 i8 254, i8 254, i8 254, i8 254,
+                 i8 254, i8 254, i8 254, i8 254>
+}
+
+; Test a halfword-granularity replicate with the lowest useful value.
+define <16 x i8> @f4() {
+; CHECK-LABEL: f4:
+; CHECK: vrepih %v24, 1
+; CHECK: br %r14
+  ret <16 x i8> <i8 0, i8 1, i8 0, i8 1,
+                 i8 0, i8 1, i8 0, i8 1,
+                 i8 0, i8 1, i8 0, i8 1,
+                 i8 0, i8 1, i8 0, i8 1>
+}
+
+; Test a halfword-granularity replicate with an arbitrary value.
+define <16 x i8> @f5() {
+; CHECK-LABEL: f5:
+; CHECK: vrepih %v24, 25650
+; CHECK: br %r14
+  ret <16 x i8> <i8 100, i8 50, i8 100, i8 50,
+                 i8 100, i8 50, i8 100, i8 50,
+                 i8 100, i8 50, i8 100, i8 50,
+                 i8 100, i8 50, i8 100, i8 50>
+}
+
+; Test a halfword-granularity replicate with the highest useful value.
+define <16 x i8> @f6() {
+; CHECK-LABEL: f6:
+; CHECK: vrepih %v24, -2
+; CHECK: br %r14
+  ret <16 x i8> <i8 255, i8 254, i8 255, i8 254,
+                 i8 255, i8 254, i8 255, i8 254,
+                 i8 255, i8 254, i8 255, i8 254,
+                 i8 255, i8 254, i8 255, i8 254>
+}
+
+; Test a word-granularity replicate with the lowest useful positive value.
+define <16 x i8> @f7() {
+; CHECK-LABEL: f7:
+; CHECK: vrepif %v24, 1
+; CHECK: br %r14
+  ret <16 x i8> <i8 0, i8 0, i8 0, i8 1,
+                 i8 0, i8 0, i8 0, i8 1,
+                 i8 0, i8 0, i8 0, i8 1,
+                 i8 0, i8 0, i8 0, i8 1>
+}
+
+; Test a word-granularity replicate with the highest in-range value.
+define <16 x i8> @f8() {
+; CHECK-LABEL: f8:
+; CHECK: vrepif %v24, 32767
+; CHECK: br %r14
+  ret <16 x i8> <i8 0, i8 0, i8 127, i8 255,
+                 i8 0, i8 0, i8 127, i8 255,
+                 i8 0, i8 0, i8 127, i8 255,
+                 i8 0, i8 0, i8 127, i8 255>
+}
+
+; Test a word-granularity replicate with the next highest value.
+; This cannot use VREPIF.
+define <16 x i8> @f9() {
+; CHECK-LABEL: f9:
+; CHECK-NOT: vrepif
+; CHECK: br %r14
+  ret <16 x i8> <i8 0, i8 0, i8 128, i8 0,
+                 i8 0, i8 0, i8 128, i8 0,
+                 i8 0, i8 0, i8 128, i8 0,
+                 i8 0, i8 0, i8 128, i8 0>
+}
+
+; Test a word-granularity replicate with the lowest in-range value.
+define <16 x i8> @f10() {
+; CHECK-LABEL: f10:
+; CHECK: vrepif %v24, -32768
+; CHECK: br %r14
+  ret <16 x i8> <i8 255, i8 255, i8 128, i8 0,
+                 i8 255, i8 255, i8 128, i8 0,
+                 i8 255, i8 255, i8 128, i8 0,
+                 i8 255, i8 255, i8 128, i8 0>
+}
+
+; Test a word-granularity replicate with the next lowest value.
+; This cannot use VREPIF.
+define <16 x i8> @f11() {
+; CHECK-LABEL: f11:
+; CHECK-NOT: vrepif
+; CHECK: br %r14
+  ret <16 x i8> <i8 255, i8 255, i8 127, i8 255,
+                 i8 255, i8 255, i8 127, i8 255,
+                 i8 255, i8 255, i8 127, i8 255,
+                 i8 255, i8 255, i8 127, i8 255>
+}
+
+; Test a word-granularity replicate with the highest useful negative value.
+define <16 x i8> @f12() {
+; CHECK-LABEL: f12:
+; CHECK: vrepif %v24, -2
+; CHECK: br %r14
+  ret <16 x i8> <i8 255, i8 255, i8 255, i8 254,
+                 i8 255, i8 255, i8 255, i8 254,
+                 i8 255, i8 255, i8 255, i8 254,
+                 i8 255, i8 255, i8 255, i8 254>
+}
+
+; Test a doubleword-granularity replicate with the lowest useful positive
+; value.
+define <16 x i8> @f13() {
+; CHECK-LABEL: f13:
+; CHECK: vrepig %v24, 1
+; CHECK: br %r14
+  ret <16 x i8> <i8 0, i8 0, i8 0, i8 0,
+                 i8 0, i8 0, i8 0, i8 1,
+                 i8 0, i8 0, i8 0, i8 0,
+                 i8 0, i8 0, i8 0, i8 1>
+}
+
+; Test a doubleword-granularity replicate with the highest in-range value.
+define <16 x i8> @f14() {
+; CHECK-LABEL: f14:
+; CHECK: vrepig %v24, 32767
+; CHECK: br %r14
+  ret <16 x i8> <i8 0, i8 0, i8 0, i8 0,
+                 i8 0, i8 0, i8 127, i8 255,
+                 i8 0, i8 0, i8 0, i8 0,
+                 i8 0, i8 0, i8 127, i8 255>
+}
+
+; Test a doubleword-granularity replicate with the next highest value.
+; This cannot use VREPIG.
+define <16 x i8> @f15() {
+; CHECK-LABEL: f15:
+; CHECK-NOT: vrepig
+; CHECK: br %r14
+  ret <16 x i8> <i8 0, i8 0, i8 0, i8 0,
+                 i8 0, i8 0, i8 128, i8 0,
+                 i8 0, i8 0, i8 0, i8 0,
+                 i8 0, i8 0, i8 128, i8 0>
+}
+
+; Test a doubleword-granularity replicate with the lowest in-range value.
+define <16 x i8> @f16() {
+; CHECK-LABEL: f16:
+; CHECK: vrepig %v24, -32768
+; CHECK: br %r14
+  ret <16 x i8> <i8 255, i8 255, i8 255, i8 255,
+                 i8 255, i8 255, i8 128, i8 0,
+                 i8 255, i8 255, i8 255, i8 255,
+                 i8 255, i8 255, i8 128, i8 0>
+}
+
+; Test a doubleword-granularity replicate with the next lowest value.
+; This cannot use VREPIG.
+define <16 x i8> @f17() {
+; CHECK-LABEL: f17:
+; CHECK-NOT: vrepig
+; CHECK: br %r14
+  ret <16 x i8> <i8 255, i8 255, i8 255, i8 255,
+                 i8 255, i8 255, i8 127, i8 255,
+                 i8 255, i8 255, i8 255, i8 255,
+                 i8 255, i8 255, i8 127, i8 255>
+}
+
+; Test a doubleword-granularity replicate with the highest useful negative
+; value.
+define <16 x i8> @f18() {
+; CHECK-LABEL: f18:
+; CHECK: vrepig %v24, -2
+; CHECK: br %r14
+  ret <16 x i8> <i8 255, i8 255, i8 255, i8 255,
+                 i8 255, i8 255, i8 255, i8 254,
+                 i8 255, i8 255, i8 255, i8 255,
+                 i8 255, i8 255, i8 255, i8 254>
+}
+
+; Repeat f14 with undefs optimistically treated as 0.
+define <16 x i8> @f19() {
+; CHECK-LABEL: f19:
+; CHECK: vrepig %v24, 32767
+; CHECK: br %r14
+  ret <16 x i8> <i8 0, i8 undef, i8 0, i8 0,
+                 i8 0, i8 0, i8 127, i8 255,
+                 i8 undef, i8 0, i8 undef, i8 0,
+                 i8 0, i8 0, i8 127, i8 255>
+}
+
+; Repeat f18 with undefs optimistically treated as -1.
+define <16 x i8> @f20() {
+; CHECK-LABEL: f20:
+; CHECK: vrepig %v24, -2
+; CHECK: br %r14
+  ret <16 x i8> <i8 undef, i8 255, i8 255, i8 255,
+                 i8 255, i8 255, i8 undef, i8 254,
+                 i8 255, i8 255, i8 255, i8 undef,
+                 i8 255, i8 undef, i8 255, i8 254>
+}
diff --git a/test/CodeGen/SystemZ/vec-const-08.ll b/test/CodeGen/SystemZ/vec-const-08.ll
new file mode 100644
index 00000000000..5ab6947e548
--- /dev/null
+++ b/test/CodeGen/SystemZ/vec-const-08.ll
@@ -0,0 +1,189 @@
+; Test vector replicates, v8i16 version.
+;
+; RUN: llc < %s -mtriple=s390x-linux-gnu -mcpu=z13 | FileCheck %s
+
+; Test a byte-granularity replicate with the lowest useful value.
+define <8 x i16> @f1() {
+; CHECK-LABEL: f1:
+; CHECK: vrepib %v24, 1
+; CHECK: br %r14
+  ret <8 x i16> <i16 257, i16 257, i16 257, i16 257,
+                 i16 257, i16 257, i16 257, i16 257>
+}
+
+; Test a byte-granularity replicate with an arbitrary value.
+define <8 x i16> @f2() {
+; CHECK-LABEL: f2:
+; CHECK: vrepib %v24, -55
+; CHECK: br %r14
+  ret <8 x i16> <i16 51657, i16 51657, i16 51657, i16 51657,
+                 i16 51657, i16 51657, i16 51657, i16 51657>
+}
+
+; Test a byte-granularity replicate with the highest useful value.
+define <8 x i16> @f3() {
+; CHECK-LABEL: f3:
+; CHECK: vrepib %v24, -2
+; CHECK: br %r14
+  ret <8 x i16> <i16 -258, i16 -258, i16 -258, i16 -258,
+                 i16 -258, i16 -258, i16 -258, i16 -258>
+}
+
+; Test a halfword-granularity replicate with the lowest useful value.
+define <8 x i16> @f4() {
+; CHECK-LABEL: f4:
+; CHECK: vrepih %v24, 1
+; CHECK: br %r14
+  ret <8 x i16> <i16 1, i16 1, i16 1, i16 1,
+                 i16 1, i16 1, i16 1, i16 1>
+}
+
+; Test a halfword-granularity replicate with an arbitrary value.
+define <8 x i16> @f5() {
+; CHECK-LABEL: f5:
+; CHECK: vrepih %v24, 25650
+; CHECK: br %r14
+  ret <8 x i16> <i16 25650, i16 25650, i16 25650, i16 25650,
+                 i16 25650, i16 25650, i16 25650, i16 25650>
+}
+
+; Test a halfword-granularity replicate with the highest useful value.
+define <8 x i16> @f6() {
+; CHECK-LABEL: f6:
+; CHECK: vrepih %v24, -2
+; CHECK: br %r14
+  ret <8 x i16> <i16 65534, i16 65534, i16 65534, i16 65534,
+                 i16 65534, i16 65534, i16 65534, i16 65534>
+}
+
+; Test a word-granularity replicate with the lowest useful positive value.
+define <8 x i16> @f7() {
+; CHECK-LABEL: f7:
+; CHECK: vrepif %v24, 1
+; CHECK: br %r14
+  ret <8 x i16> <i16 0, i16 1, i16 0, i16 1,
+                 i16 0, i16 1, i16 0, i16 1>
+}
+
+; Test a word-granularity replicate with the highest in-range value.
+define <8 x i16> @f8() {
+; CHECK-LABEL: f8:
+; CHECK: vrepif %v24, 32767
+; CHECK: br %r14
+  ret <8 x i16> <i16 0, i16 32767, i16 0, i16 32767,
+                 i16 0, i16 32767, i16 0, i16 32767>
+}
+
+; Test a word-granularity replicate with the next highest value.
+; This cannot use VREPIF.
+define <8 x i16> @f9() {
+; CHECK-LABEL: f9:
+; CHECK-NOT: vrepif
+; CHECK: br %r14
+  ret <8 x i16> <i16 0, i16 32768, i16 0, i16 32768,
+                 i16 0, i16 32768, i16 0, i16 32768>
+}
+
+; Test a word-granularity replicate with the lowest in-range value.
+define <8 x i16> @f10() {
+; CHECK-LABEL: f10:
+; CHECK: vrepif %v24, -32768
+; CHECK: br %r14
+  ret <8 x i16> <i16 -1, i16 -32768, i16 -1, i16 -32768,
+                 i16 -1, i16 -32768, i16 -1, i16 -32768>
+}
+
+; Test a word-granularity replicate with the next lowest value.
+; This cannot use VREPIF.
+define <8 x i16> @f11() {
+; CHECK-LABEL: f11:
+; CHECK-NOT: vrepif
+; CHECK: br %r14
+  ret <8 x i16> <i16 -1, i16 -32769, i16 -1, i16 -32769,
+                 i16 -1, i16 -32769, i16 -1, i16 -32769>
+}
+
+; Test a word-granularity replicate with the highest useful negative value.
+define <8 x i16> @f12() {
+; CHECK-LABEL: f12:
+; CHECK: vrepif %v24, -2
+; CHECK: br %r14
+  ret <8 x i16> <i16 -1, i16 -2, i16 -1, i16 -2,
+                 i16 -1, i16 -2, i16 -1, i16 -2>
+}
+
+; Test a doubleword-granularity replicate with the lowest useful positive
+; value.
+define <8 x i16> @f13() {
+; CHECK-LABEL: f13:
+; CHECK: vrepig %v24, 1
+; CHECK: br %r14
+  ret <8 x i16> <i16 0, i16 0, i16 0, i16 1,
+                 i16 0, i16 0, i16 0, i16 1>
+}
+
+; Test a doubleword-granularity replicate with the highest in-range value.
+define <8 x i16> @f14() {
+; CHECK-LABEL: f14:
+; CHECK: vrepig %v24, 32767
+; CHECK: br %r14
+  ret <8 x i16> <i16 0, i16 0, i16 0, i16 32767,
+                 i16 0, i16 0, i16 0, i16 32767>
+}
+
+; Test a doubleword-granularity replicate with the next highest value.
+; This cannot use VREPIG.
+define <8 x i16> @f15() {
+; CHECK-LABEL: f15:
+; CHECK-NOT: vrepig
+; CHECK: br %r14
+  ret <8 x i16> <i16 0, i16 0, i16 0, i16 32768,
+                 i16 0, i16 0, i16 0, i16 32768>
+}
+
+; Test a doubleword-granularity replicate with the lowest in-range value.
+define <8 x i16> @f16() {
+; CHECK-LABEL: f16:
+; CHECK: vrepig %v24, -32768
+; CHECK: br %r14
+  ret <8 x i16> <i16 -1, i16 -1, i16 -1, i16 -32768,
+                 i16 -1, i16 -1, i16 -1, i16 -32768>
+}
+
+; Test a doubleword-granularity replicate with the next lowest value.
+; This cannot use VREPIG.
+define <8 x i16> @f17() {
+; CHECK-LABEL: f17:
+; CHECK-NOT: vrepig
+; CHECK: br %r14
+  ret <8 x i16> <i16 -1, i16 -1, i16 -1, i16 -32769,
+                 i16 -1, i16 -1, i16 -1, i16 -32769>
+}
+
+; Test a doubleword-granularity replicate with the highest useful negative
+; value.
+define <8 x i16> @f18() {
+; CHECK-LABEL: f18:
+; CHECK: vrepig %v24, -2
+; CHECK: br %r14
+  ret <8 x i16> <i16 -1, i16 -1, i16 -1, i16 -2,
+                 i16 -1, i16 -1, i16 -1, i16 -2>
+}
+
+; Repeat f14 with undefs optimistically treated as 0.
+define <8 x i16> @f19() {
+; CHECK-LABEL: f19:
+; CHECK: vrepig %v24, 32767
+; CHECK: br %r14
+  ret <8 x i16> <i16 0, i16 undef, i16 0, i16 32767,
+                 i16 undef, i16 0, i16 undef, i16 32767>
+}
+
+; Repeat f18 with undefs optimistically treated as -1.
+define <8 x i16> @f20() {
+; CHECK-LABEL: f20:
+; CHECK: vrepig %v24, -2
+; CHECK: br %r14
+  ret <8 x i16> <i16 -1, i16 -1, i16 undef, i16 -2,
+                 i16 undef, i16 undef, i16 -1, i16 -2>
+}
diff --git a/test/CodeGen/SystemZ/vec-const-09.ll b/test/CodeGen/SystemZ/vec-const-09.ll
new file mode 100644
index 00000000000..2cbe9259452
--- /dev/null
+++ b/test/CodeGen/SystemZ/vec-const-09.ll
@@ -0,0 +1,169 @@
+; Test vector replicates, v4i32 version.
+;
+; RUN: llc < %s -mtriple=s390x-linux-gnu -mcpu=z13 | FileCheck %s
+
+; Test a byte-granularity replicate with the lowest useful value.
+define <4 x i32> @f1() {
+; CHECK-LABEL: f1:
+; CHECK: vrepib %v24, 1
+; CHECK: br %r14
+  ret <4 x i32> <i32 16843009, i32 16843009, i32 16843009, i32 16843009>
+}
+
+; Test a byte-granularity replicate with an arbitrary value.
+define <4 x i32> @f2() {
+; CHECK-LABEL: f2:
+; CHECK: vrepib %v24, -55
+; CHECK: br %r14
+  ret <4 x i32> <i32 3385444809, i32 3385444809, i32 3385444809, i32 3385444809>
+}
+
+; Test a byte-granularity replicate with the highest useful value.
+define <4 x i32> @f3() {
+; CHECK-LABEL: f3:
+; CHECK: vrepib %v24, -2
+; CHECK: br %r14
+  ret <4 x i32> <i32 4278124286, i32 4278124286, i32 4278124286, i32 4278124286>
+}
+
+; Test a halfword-granularity replicate with the lowest useful value.
+define <4 x i32> @f4() {
+; CHECK-LABEL: f4:
+; CHECK: vrepih %v24, 1
+; CHECK: br %r14
+  ret <4 x i32> <i32 65537, i32 65537, i32 65537, i32 65537>
+}
+
+; Test a halfword-granularity replicate with an arbitrary value.
+define <4 x i32> @f5() {
+; CHECK-LABEL: f5:
+; CHECK: vrepih %v24, 25650
+; CHECK: br %r14
+  ret <4 x i32> <i32 1681024050, i32 1681024050, i32 1681024050, i32 1681024050>
+}
+
+; Test a halfword-granularity replicate with the highest useful value.
+define <4 x i32> @f6() {
+; CHECK-LABEL: f6:
+; CHECK: vrepih %v24, -2
+; CHECK: br %r14
+  ret <4 x i32> <i32 -65538, i32 -65538, i32 -65538, i32 -65538>
+}
+
+; Test a word-granularity replicate with the lowest useful positive value.
+define <4 x i32> @f7() {
+; CHECK-LABEL: f7:
+; CHECK: vrepif %v24, 1
+; CHECK: br %r14
+  ret <4 x i32> <i32 1, i32 1, i32 1, i32 1>
+}
+
+; Test a word-granularity replicate with the highest in-range value.
+define <4 x i32> @f8() {
+; CHECK-LABEL: f8:
+; CHECK: vrepif %v24, 32767
+; CHECK: br %r14
+  ret <4 x i32> <i32 32767, i32 32767, i32 32767, i32 32767>
+}
+
+; Test a word-granularity replicate with the next highest value.
+; This cannot use VREPIF.
+define <4 x i32> @f9() {
+; CHECK-LABEL: f9:
+; CHECK-NOT: vrepif
+; CHECK: br %r14
+  ret <4 x i32> <i32 32768, i32 32768, i32 32768, i32 32768>
+}
+
+; Test a word-granularity replicate with the lowest in-range value.
+define <4 x i32> @f10() {
+; CHECK-LABEL: f10:
+; CHECK: vrepif %v24, -32768
+; CHECK: br %r14
+  ret <4 x i32> <i32 -32768, i32 -32768, i32 -32768, i32 -32768>
+}
+
+; Test a word-granularity replicate with the next lowest value.
+; This cannot use VREPIF.
+define <4 x i32> @f11() {
+; CHECK-LABEL: f11:
+; CHECK-NOT: vrepif
+; CHECK: br %r14
+  ret <4 x i32> <i32 -32769, i32 -32769, i32 -32769, i32 -32769>
+}
+
+; Test a word-granularity replicate with the highest useful negative value.
+define <4 x i32> @f12() {
+; CHECK-LABEL: f12:
+; CHECK: vrepif %v24, -2
+; CHECK: br %r14
+  ret <4 x i32> <i32 -2, i32 -2, i32 -2, i32 -2>
+}
+
+; Test a doubleword-granularity replicate with the lowest useful positive
+; value.
+define <4 x i32> @f13() {
+; CHECK-LABEL: f13:
+; CHECK: vrepig %v24, 1
+; CHECK: br %r14
+  ret <4 x i32> <i32 0, i32 1, i32 0, i32 1>
+}
+
+; Test a doubleword-granularity replicate with the highest in-range value.
+define <4 x i32> @f14() {
+; CHECK-LABEL: f14:
+; CHECK: vrepig %v24, 32767
+; CHECK: br %r14
+  ret <4 x i32> <i32 0, i32 32767, i32 0, i32 32767>
+}
+
+; Test a doubleword-granularity replicate with the next highest value.
+; This cannot use VREPIG.
+define <4 x i32> @f15() {
+; CHECK-LABEL: f15:
+; CHECK-NOT: vrepig
+; CHECK: br %r14
+  ret <4 x i32> <i32 0, i32 32768, i32 0, i32 32768>
+}
+
+; Test a doubleword-granularity replicate with the lowest in-range value.
+define <4 x i32> @f16() {
+; CHECK-LABEL: f16:
+; CHECK: vrepig %v24, -32768
+; CHECK: br %r14
+  ret <4 x i32> <i32 -1, i32 -32768, i32 -1, i32 -32768>
+}
+
+; Test a doubleword-granularity replicate with the next lowest value.
+; This cannot use VREPIG.
+define <4 x i32> @f17() {
+; CHECK-LABEL: f17:
+; CHECK-NOT: vrepig
+; CHECK: br %r14
+  ret <4 x i32> <i32 -1, i32 -32769, i32 -1, i32 -32769>
+}
+
+; Test a doubleword-granularity replicate with the highest useful negative
+; value.
+define <4 x i32> @f18() {
+; CHECK-LABEL: f18:
+; CHECK: vrepig %v24, -2
+; CHECK: br %r14
+  ret <4 x i32> <i32 -1, i32 -2, i32 -1, i32 -2>
+}
+
+; Repeat f14 with undefs optimistically treated as 0, 32767.
+define <4 x i32> @f19() {
+; CHECK-LABEL: f19:
+; CHECK: vrepig %v24, 32767
+; CHECK: br %r14
+  ret <4 x i32> <i32 undef, i32 undef, i32 0, i32 32767>
+}
+
+; Repeat f18 with undefs optimistically treated as -2, -1.
+define <4 x i32> @f20() {
+; CHECK-LABEL: f20:
+; CHECK: vrepig %v24, -2
+; CHECK: br %r14
+  ret <4 x i32> <i32 -1, i32 undef, i32 undef, i32 -2>
+}
diff --git a/test/CodeGen/SystemZ/vec-const-10.ll b/test/CodeGen/SystemZ/vec-const-10.ll
new file mode 100644
index 00000000000..0613b69a277
--- /dev/null
+++ b/test/CodeGen/SystemZ/vec-const-10.ll
@@ -0,0 +1,169 @@
+; Test vector replicates, v2i64 version.
+;
+; RUN: llc < %s -mtriple=s390x-linux-gnu -mcpu=z13 | FileCheck %s
+
+; Test a byte-granularity replicate with the lowest useful value.
+define <2 x i64> @f1() {
+; CHECK-LABEL: f1:
+; CHECK: vrepib %v24, 1
+; CHECK: br %r14
+  ret <2 x i64> <i64 72340172838076673, i64 72340172838076673>
+}
+
+; Test a byte-granularity replicate with an arbitrary value.
+define <2 x i64> @f2() {
+; CHECK-LABEL: f2:
+; CHECK: vrepib %v24, -55
+; CHECK: br %r14
+  ret <2 x i64> <i64 -3906369333256140343, i64 -3906369333256140343>
+}
+
+; Test a byte-granularity replicate with the highest useful value.
+define <2 x i64> @f3() {
+; CHECK-LABEL: f3:
+; CHECK: vrepib %v24, -2
+; CHECK: br %r14
+  ret <2 x i64> <i64 -72340172838076674, i64 -72340172838076674>
+}
+
+; Test a halfword-granularity replicate with the lowest useful value.
+define <2 x i64> @f4() {
+; CHECK-LABEL: f4:
+; CHECK: vrepih %v24, 1
+; CHECK: br %r14
+  ret <2 x i64> <i64 281479271743489, i64 281479271743489>
+}
+
+; Test a halfword-granularity replicate with an arbitrary value.
+define <2 x i64> @f5() {
+; CHECK-LABEL: f5:
+; CHECK: vrepih %v24, 25650
+; CHECK: br %r14
+  ret <2 x i64> <i64 7219943320220492850, i64 7219943320220492850>
+}
+
+; Test a halfword-granularity replicate with the highest useful value.
+define <2 x i64> @f6() {
+; CHECK-LABEL: f6:
+; CHECK: vrepih %v24, -2
+; CHECK: br %r14
+  ret <2 x i64> <i64 -281479271743490, i64 -281479271743490>
+}
+
+; Test a word-granularity replicate with the lowest useful positive value.
+define <2 x i64> @f7() {
+; CHECK-LABEL: f7:
+; CHECK: vrepif %v24, 1
+; CHECK: br %r14
+  ret <2 x i64> <i64 4294967297, i64 4294967297>
+}
+
+; Test a word-granularity replicate with the highest in-range value.
+define <2 x i64> @f8() {
+; CHECK-LABEL: f8:
+; CHECK: vrepif %v24, 32767
+; CHECK: br %r14
+  ret <2 x i64> <i64 140733193420799, i64 140733193420799>
+}
+
+; Test a word-granularity replicate with the next highest value.
+; This cannot use VREPIF.
+define <2 x i64> @f9() {
+; CHECK-LABEL: f9:
+; CHECK-NOT: vrepif
+; CHECK: br %r14
+  ret <2 x i64> <i64 140737488388096, i64 140737488388096>
+}
+
+; Test a word-granularity replicate with the lowest in-range value.
+define <2 x i64> @f10() {
+; CHECK-LABEL: f10:
+; CHECK: vrepif %v24, -32768
+; CHECK: br %r14
+  ret <2 x i64> <i64 -140733193420800, i64 -140733193420800>
+}
+
+; Test a word-granularity replicate with the next lowest value.
+; This cannot use VREPIF.
+define <2 x i64> @f11() {
+; CHECK-LABEL: f11:
+; CHECK-NOT: vrepif
+; CHECK: br %r14
+  ret <2 x i64> <i64 -140737488388097, i64 -140737488388097>
+}
+
+; Test a word-granularity replicate with the highest useful negative value.
+define <2 x i64> @f12() {
+; CHECK-LABEL: f12:
+; CHECK: vrepif %v24, -2
+; CHECK: br %r14
+  ret <2 x i64> <i64 -4294967298, i64 -4294967298>
+}
+
+; Test a doubleword-granularity replicate with the lowest useful positive
+; value.
+define <2 x i64> @f13() {
+; CHECK-LABEL: f13:
+; CHECK: vrepig %v24, 1
+; CHECK: br %r14
+  ret <2 x i64> <i64 1, i64 1>
+}
+
+; Test a doubleword-granularity replicate with the highest in-range value.
+define <2 x i64> @f14() {
+; CHECK-LABEL: f14:
+; CHECK: vrepig %v24, 32767
+; CHECK: br %r14
+  ret <2 x i64> <i64 32767, i64 32767>
+}
+
+; Test a doubleword-granularity replicate with the next highest value.
+; This cannot use VREPIG.
+define <2 x i64> @f15() {
+; CHECK-LABEL: f15:
+; CHECK-NOT: vrepig
+; CHECK: br %r14
+  ret <2 x i64> <i64 32768, i64 32768>
+}
+
+; Test a doubleword-granularity replicate with the lowest in-range value.
+define <2 x i64> @f16() {
+; CHECK-LABEL: f16:
+; CHECK: vrepig %v24, -32768
+; CHECK: br %r14
+  ret <2 x i64> <i64 -32768, i64 -32768>
+}
+
+; Test a doubleword-granularity replicate with the next lowest value.
+; This cannot use VREPIG.
+define <2 x i64> @f17() {
+; CHECK-LABEL: f17:
+; CHECK-NOT: vrepig
+; CHECK: br %r14
+  ret <2 x i64> <i64 -32769, i64 -32769>
+}
+
+; Test a doubleword-granularity replicate with the highest useful negative
+; value.
+define <2 x i64> @f18() {
+; CHECK-LABEL: f18:
+; CHECK: vrepig %v24, -2
+; CHECK: br %r14
+  ret <2 x i64> <i64 -2, i64 -2>
+}
+
+; Repeat f14 with undefs optimistically treated as 32767.
+define <2 x i64> @f19() {
+; CHECK-LABEL: f19:
+; CHECK: vrepig %v24, 32767
+; CHECK: br %r14
+  ret <2 x i64> <i64 undef, i64 32767>
+}
+
+; Repeat f18 with undefs optimistically treated as -2.
+define <2 x i64> @f20() {
+; CHECK-LABEL: f20:
+; CHECK: vrepig %v24, -2
+; CHECK: br %r14
+  ret <2 x i64> <i64 undef, i64 -2>
+}
diff --git a/test/CodeGen/SystemZ/vec-const-11.ll b/test/CodeGen/SystemZ/vec-const-11.ll
new file mode 100644
index 00000000000..0c69b8803b2
--- /dev/null
+++ b/test/CodeGen/SystemZ/vec-const-11.ll
@@ -0,0 +1,189 @@
+; Test vector replicates, v4f32 version.
+;
+; RUN: llc < %s -mtriple=s390x-linux-gnu -mcpu=z13 | FileCheck %s
+
+; Test a byte-granularity replicate with the lowest useful value.
+define <4 x float> @f1() {
+; CHECK-LABEL: f1:
+; CHECK: vrepib %v24, 1
+; CHECK: br %r14
+  ret <4 x float> <float 0x3820202020000000, float 0x3820202020000000,
+                   float 0x3820202020000000, float 0x3820202020000000>
+}
+
+; Test a byte-granularity replicate with an arbitrary value.
+define <4 x float> @f2() {
+; CHECK-LABEL: f2:
+; CHECK: vrepib %v24, -55
+; CHECK: br %r14
+  ret <4 x float> <float 0xc139393920000000, float 0xc139393920000000,
+                   float 0xc139393920000000, float 0xc139393920000000>
+}
+
+; Test a byte-granularity replicate with the highest useful value.
+define <4 x float> @f3() {
+; CHECK-LABEL: f3:
+; CHECK: vrepib %v24, -2
+; CHECK: br %r14
+  ret <4 x float> <float 0xc7dfdfdfc0000000, float 0xc7dfdfdfc0000000,
+                   float 0xc7dfdfdfc0000000, float 0xc7dfdfdfc0000000>
+}
+
+; Test a halfword-granularity replicate with the lowest useful value.
+define <4 x float> @f4() {
+; CHECK-LABEL: f4:
+; CHECK: vrepih %v24, 1
+; CHECK: br %r14
+  ret <4 x float> <float 0x37a0001000000000, float 0x37a0001000000000,
+                   float 0x37a0001000000000, float 0x37a0001000000000>
+}
+
+; Test a halfword-granularity replicate with an arbitrary value.
+define <4 x float> @f5() {
+; CHECK-LABEL: f5:
+; CHECK: vrepih %v24, 25650
+; CHECK: br %r14
+  ret <4 x float> <float 0x44864c8640000000, float 0x44864c8640000000,
+                   float 0x44864c8640000000, float 0x44864c8640000000>
+}
+
+; Test a halfword-granularity replicate with the highest useful value.
+define <4 x float> @f6() {
+; CHECK-LABEL: f6:
+; CHECK: vrepih %v24, -2
+; CHECK: br %r14
+  ret <4 x float> <float 0xffffdfffc0000000, float 0xffffdfffc0000000,
+                   float 0xffffdfffc0000000, float 0xffffdfffc0000000>
+}
+
+; Test a word-granularity replicate with the lowest useful positive value.
+define <4 x float> @f7() {
+; CHECK-LABEL: f7:
+; CHECK: vrepif %v24, 1
+; CHECK: br %r14
+  ret <4 x float> <float 0x36a0000000000000, float 0x36a0000000000000,
+                   float 0x36a0000000000000, float 0x36a0000000000000>
+}
+
+; Test a word-granularity replicate with the highest in-range value.
+define <4 x float> @f8() {
+; CHECK-LABEL: f8:
+; CHECK: vrepif %v24, 32767
+; CHECK: br %r14
+  ret <4 x float> <float 0x378fffc000000000, float 0x378fffc000000000,
+                   float 0x378fffc000000000, float 0x378fffc000000000>
+}
+
+; Test a word-granularity replicate with the next highest value.
+; This cannot use VREPIF.
+define <4 x float> @f9() {
+; CHECK-LABEL: f9:
+; CHECK-NOT: vrepif
+; CHECK: br %r14
+  ret <4 x float> <float 0x3790000000000000, float 0x3790000000000000,
+                   float 0x3790000000000000, float 0x3790000000000000>
+}
+
+; Test a word-granularity replicate with the lowest in-range value.
+define <4 x float> @f10() {
+; CHECK-LABEL: f10:
+; CHECK: vrepif %v24, -32768
+; CHECK: br %r14
+  ret <4 x float> <float 0xfffff00000000000, float 0xfffff00000000000,
+                   float 0xfffff00000000000, float 0xfffff00000000000>
+}
+
+; Test a word-granularity replicate with the next lowest value.
+; This cannot use VREPIF.
+define <4 x float> @f11() {
+; CHECK-LABEL: f11:
+; CHECK-NOT: vrepif
+; CHECK: br %r14
+  ret <4 x float> <float 0xffffefffe0000000, float 0xffffefffe0000000,
+                   float 0xffffefffe0000000, float 0xffffefffe0000000>
+}
+
+; Test a word-granularity replicate with the highest useful negative value.
+define <4 x float> @f12() {
+; CHECK-LABEL: f12:
+; CHECK: vrepif %v24, -2
+; CHECK: br %r14
+  ret <4 x float> <float 0xffffffffc0000000, float 0xffffffffc0000000,
+                   float 0xffffffffc0000000, float 0xffffffffc0000000>
+}
+
+; Test a doubleword-granularity replicate with the lowest useful positive
+; value.
+define <4 x float> @f13() {
+; CHECK-LABEL: f13:
+; CHECK: vrepig %v24, 1
+; CHECK: br %r14
+  ret <4 x float> <float 0.0, float 0x36a0000000000000,
+                   float 0.0, float 0x36a0000000000000>
+}
+
+; Test a doubleword-granularity replicate with the highest in-range value.
+define <4 x float> @f14() {
+; CHECK-LABEL: f14:
+; CHECK: vrepig %v24, 32767
+; CHECK: br %r14
+  ret <4 x float> <float 0.0, float 0x378fffc000000000,
+                   float 0.0, float 0x378fffc000000000>
+}
+
+; Test a doubleword-granularity replicate with the next highest value.
+; This cannot use VREPIG.
+define <4 x float> @f15() {
+; CHECK-LABEL: f15:
+; CHECK-NOT: vrepig
+; CHECK: br %r14
+  ret <4 x float> <float 0.0, float 0x3790000000000000,
+                   float 0.0, float 0x3790000000000000>
+}
+
+; Test a doubleword-granularity replicate with the lowest in-range value.
+define <4 x float> @f16() {
+; CHECK-LABEL: f16:
+; CHECK: vrepig %v24, -32768
+; CHECK: br %r14
+  ret <4 x float> <float 0xffffffffe0000000, float 0xfffff00000000000,
+                   float 0xffffffffe0000000, float 0xfffff00000000000>
+}
+
+; Test a doubleword-granularity replicate with the next lowest value.
+; This cannot use VREPIG.
+define <4 x float> @f17() {
+; CHECK-LABEL: f17:
+; CHECK-NOT: vrepig
+; CHECK: br %r14
+  ret <4 x float> <float 0xffffffffe0000000, float 0xffffefffe0000000,
+                   float 0xffffffffe0000000, float 0xffffefffe0000000>
+}
+
+; Test a doubleword-granularity replicate with the highest useful negative
+; value.
+define <4 x float> @f18() {
+; CHECK-LABEL: f18:
+; CHECK: vrepig %v24, -2
+; CHECK: br %r14
+  ret <4 x float> <float 0xffffffffe0000000, float 0xffffffffc0000000,
+                   float 0xffffffffe0000000, float 0xffffffffc0000000>
+}
+
+; Repeat f14 with undefs optimistically treated as 0, 32767.
+define <4 x float> @f19() {
+; CHECK-LABEL: f19:
+; CHECK: vrepig %v24, 32767
+; CHECK: br %r14
+  ret <4 x float> <float undef, float undef,
+                   float 0.0, float 0x378fffc000000000>
+}
+
+; Repeat f18 with undefs optimistically treated as -2, -1.
+define <4 x float> @f20() {
+; CHECK-LABEL: f20:
+; CHECK: vrepig %v24, -2
+; CHECK: br %r14
+  ret <4 x float> <float 0xffffffffe0000000, float undef,
+                   float undef, float 0xffffffffc0000000>
+}
diff --git a/test/CodeGen/SystemZ/vec-const-12.ll b/test/CodeGen/SystemZ/vec-const-12.ll
new file mode 100644
index 00000000000..ca66a3d173e
--- /dev/null
+++ b/test/CodeGen/SystemZ/vec-const-12.ll
@@ -0,0 +1,169 @@
+; Test vector replicates, v2f64 version.
+;
+; RUN: llc < %s -mtriple=s390x-linux-gnu -mcpu=z13 | FileCheck %s
+
+; Test a byte-granularity replicate with the lowest useful value.
+define <2 x double> @f1() {
+; CHECK-LABEL: f1:
+; CHECK: vrepib %v24, 1
+; CHECK: br %r14
+  ret <2 x double> <double 0x0101010101010101, double 0x0101010101010101>
+}
+
+; Test a byte-granularity replicate with an arbitrary value.
+define <2 x double> @f2() {
+; CHECK-LABEL: f2:
+; CHECK: vrepib %v24, -55
+; CHECK: br %r14
+  ret <2 x double> <double 0xc9c9c9c9c9c9c9c9, double 0xc9c9c9c9c9c9c9c9>
+}
+
+; Test a byte-granularity replicate with the highest useful value.
+define <2 x double> @f3() {
+; CHECK-LABEL: f3:
+; CHECK: vrepib %v24, -2
+; CHECK: br %r14
+  ret <2 x double> <double 0xfefefefefefefefe, double 0xfefefefefefefefe>
+}
+
+; Test a halfword-granularity replicate with the lowest useful value.
+define <2 x double> @f4() {
+; CHECK-LABEL: f4:
+; CHECK: vrepih %v24, 1
+; CHECK: br %r14
+  ret <2 x double> <double 0x0001000100010001, double 0x0001000100010001>
+}
+
+; Test a halfword-granularity replicate with an arbitrary value.
+define <2 x double> @f5() {
+; CHECK-LABEL: f5:
+; CHECK: vrepih %v24, 25650
+; CHECK: br %r14
+  ret <2 x double> <double 0x6432643264326432, double 0x6432643264326432>
+}
+
+; Test a halfword-granularity replicate with the highest useful value.
+define <2 x double> @f6() {
+; CHECK-LABEL: f6:
+; CHECK: vrepih %v24, -2
+; CHECK: br %r14
+  ret <2 x double> <double 0xfffefffefffefffe, double 0xfffefffefffefffe>
+}
+
+; Test a word-granularity replicate with the lowest useful positive value.
+define <2 x double> @f7() {
+; CHECK-LABEL: f7:
+; CHECK: vrepif %v24, 1
+; CHECK: br %r14
+  ret <2 x double> <double 0x0000000100000001, double 0x0000000100000001>
+}
+
+; Test a word-granularity replicate with the highest in-range value.
+define <2 x double> @f8() {
+; CHECK-LABEL: f8:
+; CHECK: vrepif %v24, 32767
+; CHECK: br %r14
+  ret <2 x double> <double 0x00007fff00007fff, double 0x00007fff00007fff>
+}
+
+; Test a word-granularity replicate with the next highest value.
+; This cannot use VREPIF.
+define <2 x double> @f9() {
+; CHECK-LABEL: f9:
+; CHECK-NOT: vrepif
+; CHECK: br %r14
+  ret <2 x double> <double 0x0000800000008000, double 0x0000800000008000>
+}
+
+; Test a word-granularity replicate with the lowest in-range value.
+define <2 x double> @f10() {
+; CHECK-LABEL: f10:
+; CHECK: vrepif %v24, -32768
+; CHECK: br %r14
+  ret <2 x double> <double 0xffff8000ffff8000, double 0xffff8000ffff8000>
+}
+
+; Test a word-granularity replicate with the next lowest value.
+; This cannot use VREPIF.
+define <2 x double> @f11() {
+; CHECK-LABEL: f11:
+; CHECK-NOT: vrepif
+; CHECK: br %r14
+  ret <2 x double> <double 0xffff7fffffff7fff, double 0xffff7fffffff7fff>
+}
+
+; Test a word-granularity replicate with the highest useful negative value.
+define <2 x double> @f12() {
+; CHECK-LABEL: f12:
+; CHECK: vrepif %v24, -2
+; CHECK: br %r14
+  ret <2 x double> <double 0xfffffffefffffffe, double 0xfffffffefffffffe>
+}
+
+; Test a doubleword-granularity replicate with the lowest useful positive
+; value.
+define <2 x double> @f13() {
+; CHECK-LABEL: f13:
+; CHECK: vrepig %v24, 1
+; CHECK: br %r14
+  ret <2 x double> <double 0x0000000000000001, double 0x0000000000000001>
+}
+
+; Test a doubleword-granularity replicate with the highest in-range value.
+define <2 x double> @f14() {
+; CHECK-LABEL: f14:
+; CHECK: vrepig %v24, 32767
+; CHECK: br %r14
+  ret <2 x double> <double 0x0000000000007fff, double 0x0000000000007fff>
+}
+
+; Test a doubleword-granularity replicate with the next highest value.
+; This cannot use VREPIG.
+define <2 x double> @f15() {
+; CHECK-LABEL: f15:
+; CHECK-NOT: vrepig
+; CHECK: br %r14
+  ret <2 x double> <double 0x0000000000008000, double 0x0000000000008000>
+}
+
+; Test a doubleword-granularity replicate with the lowest in-range value.
+define <2 x double> @f16() {
+; CHECK-LABEL: f16:
+; CHECK: vrepig %v24, -32768
+; CHECK: br %r14
+  ret <2 x double> <double 0xffffffffffff8000, double 0xffffffffffff8000>
+}
+
+; Test a doubleword-granularity replicate with the next lowest value.
+; This cannot use VREPIG.
+define <2 x double> @f17() {
+; CHECK-LABEL: f17:
+; CHECK-NOT: vrepig
+; CHECK: br %r14
+  ret <2 x double> <double 0xffffffffffff7fff, double 0xffffffffffff7fff>
+}
+
+; Test a doubleword-granularity replicate with the highest useful negative
+; value.
+define <2 x double> @f18() {
+; CHECK-LABEL: f18:
+; CHECK: vrepig %v24, -2
+; CHECK: br %r14
+  ret <2 x double> <double 0xfffffffffffffffe, double 0xfffffffffffffffe>
+}
+
+; Repeat f14 with undefs optimistically treated as 32767.
+define <2 x double> @f19() {
+; CHECK-LABEL: f19:
+; CHECK: vrepig %v24, 32767
+; CHECK: br %r14
+  ret <2 x double> <double undef, double 0x0000000000007fff>
+}
+
+; Repeat f18 with undefs optimistically treated as -2.
+define <2 x double> @f20() {
+; CHECK-LABEL: f20:
+; CHECK: vrepig %v24, -2
+; CHECK: br %r14
+  ret <2 x double> <double undef, double 0xfffffffffffffffe>
+}
diff --git a/test/CodeGen/SystemZ/vec-const-13.ll b/test/CodeGen/SystemZ/vec-const-13.ll
new file mode 100644
index 00000000000..2cc425252c2
--- /dev/null
+++ b/test/CodeGen/SystemZ/vec-const-13.ll
@@ -0,0 +1,193 @@
+; Test vector replicates that use VECTOR GENERATE MASK, v16i8 version.
+;
+; RUN: llc < %s -mtriple=s390x-linux-gnu -mcpu=z13 | FileCheck %s
+
+; Test a word-granularity replicate with the lowest value that cannot use
+; VREPIF.
+define <16 x i8> @f1() {
+; CHECK-LABEL: f1:
+; CHECK: vgmf %v24, 16, 16
+; CHECK: br %r14
+  ret <16 x i8> <i8 0, i8 0, i8 128, i8 0,
+                 i8 0, i8 0, i8 128, i8 0,
+                 i8 0, i8 0, i8 128, i8 0,
+                 i8 0, i8 0, i8 128, i8 0>
+}
+
+; Test a word-granularity replicate that has the lower 17 bits set.
+define <16 x i8> @f2() {
+; CHECK-LABEL: f2:
+; CHECK: vgmf %v24, 15, 31
+; CHECK: br %r14
+  ret <16 x i8> <i8 0, i8 1, i8 255, i8 255,
+                 i8 0, i8 1, i8 255, i8 255,
+                 i8 0, i8 1, i8 255, i8 255,
+                 i8 0, i8 1, i8 255, i8 255>
+}
+
+; Test a word-granularity replicate that has the upper 15 bits set.
+define <16 x i8> @f3() {
+; CHECK-LABEL: f3:
+; CHECK: vgmf %v24, 0, 14
+; CHECK: br %r14
+  ret <16 x i8> <i8 255, i8 254, i8 0, i8 0,
+                 i8 255, i8 254, i8 0, i8 0,
+                 i8 255, i8 254, i8 0, i8 0,
+                 i8 255, i8 254, i8 0, i8 0>
+}
+
+; Test a word-granularity replicate that has middle bits set.
+define <16 x i8> @f4() {
+; CHECK-LABEL: f4:
+; CHECK: vgmf %v24, 12, 17
+; CHECK: br %r14
+  ret <16 x i8> <i8 0, i8 15, i8 192, i8 0,
+                 i8 0, i8 15, i8 192, i8 0,
+                 i8 0, i8 15, i8 192, i8 0,
+                 i8 0, i8 15, i8 192, i8 0>
+}
+
+; Test a word-granularity replicate with a wrap-around mask.
+define <16 x i8> @f5() {
+; CHECK-LABEL: f5:
+; CHECK: vgmf %v24, 17, 15
+; CHECK: br %r14
+  ret <16 x i8> <i8 255, i8 255, i8 127, i8 255,
+                 i8 255, i8 255, i8 127, i8 255,
+                 i8 255, i8 255, i8 127, i8 255,
+                 i8 255, i8 255, i8 127, i8 255>
+}
+
+; Test a doubleword-granularity replicate with the lowest value that cannot
+; use VREPIG.
+define <16 x i8> @f6() {
+; CHECK-LABEL: f6:
+; CHECK: vgmg %v24, 48, 48
+; CHECK: br %r14
+  ret <16 x i8> <i8 0, i8 0, i8 0, i8 0,
+                 i8 0, i8 0, i8 128, i8 0,
+                 i8 0, i8 0, i8 0, i8 0,
+                 i8 0, i8 0, i8 128, i8 0>
+}
+
+; Test a doubleword-granularity replicate that has the lower 22 bits set.
+define <16 x i8> @f7() {
+; CHECK-LABEL: f7:
+; CHECK: vgmg %v24, 42, 63
+; CHECK: br %r14
+  ret <16 x i8> <i8 0, i8 0, i8 0, i8 0,
+                 i8 0, i8 63, i8 255, i8 255,
+                 i8 0, i8 0, i8 0, i8 0,
+                 i8 0, i8 63, i8 255, i8 255>
+}
+
+; Test a doubleword-granularity replicate that has the upper 45 bits set.
+define <16 x i8> @f8() {
+; CHECK-LABEL: f8:
+; CHECK: vgmg %v24, 0, 44
+; CHECK: br %r14
+  ret <16 x i8> <i8 255, i8 255, i8 255, i8 255,
+                 i8 255, i8 248, i8 0, i8 0,
+                 i8 255, i8 255, i8 255, i8 255,
+                 i8 255, i8 248, i8 0, i8 0>
+}
+
+; Test a doubleword-granularity replicate that has middle bits set.
+define <16 x i8> @f9() {
+; CHECK-LABEL: f9:
+; CHECK: vgmg %v24, 31, 42
+; CHECK: br %r14
+  ret <16 x i8> <i8 0, i8 0, i8 0, i8 1,
+                 i8 255, i8 224, i8 0, i8 0,
+                 i8 0, i8 0, i8 0, i8 1,
+                 i8 255, i8 224, i8 0, i8 0>
+}
+
+; Test a doubleword-granularity replicate with a wrap-around mask.
+define <16 x i8> @f10() {
+; CHECK-LABEL: f10:
+; CHECK: vgmg %v24, 18, 0
+; CHECK: br %r14
+  ret <16 x i8> <i8 128, i8 0, i8 63, i8 255,
+                 i8 255, i8 255, i8 255, i8 255,
+                 i8 128, i8 0, i8 63, i8 255,
+                 i8 255, i8 255, i8 255, i8 255>
+}
+
+; Retest f1 with arbitrary undefs instead of 0s.
+define <16 x i8> @f11() {
+; CHECK-LABEL: f11:
+; CHECK: vgmf %v24, 16, 16
+; CHECK: br %r14
+  ret <16 x i8> <i8 0, i8 undef, i8 128, i8 0,
+                 i8 0, i8 0, i8 128, i8 undef,
+                 i8 undef, i8 0, i8 128, i8 0,
+                 i8 undef, i8 undef, i8 128, i8 0>
+}
+
+; Try a case where we want consistent undefs to be treated as 0.
+define <16 x i8> @f12() {
+; CHECK-LABEL: f12:
+; CHECK: vgmf %v24, 15, 23
+; CHECK: br %r14
+  ret <16 x i8> <i8 undef, i8 1, i8 255, i8 0,
+                 i8 undef, i8 1, i8 255, i8 0,
+                 i8 undef, i8 1, i8 255, i8 0,
+                 i8 undef, i8 1, i8 255, i8 0>
+}
+
+; ...and again with the lower bits of the replicated constant.
+define <16 x i8> @f13() {
+; CHECK-LABEL: f13:
+; CHECK: vgmf %v24, 15, 22
+; CHECK: br %r14
+  ret <16 x i8> <i8 0, i8 1, i8 254, i8 undef,
+                 i8 0, i8 1, i8 254, i8 undef,
+                 i8 0, i8 1, i8 254, i8 undef,
+                 i8 0, i8 1, i8 254, i8 undef>
+}
+
+; Try a case where we want consistent undefs to be treated as -1.
+define <16 x i8> @f14() {
+; CHECK-LABEL: f14:
+; CHECK: vgmf %v24, 28, 8
+; CHECK: br %r14
+  ret <16 x i8> <i8 undef, i8 128, i8 0, i8 15,
+                 i8 undef, i8 128, i8 0, i8 15,
+                 i8 undef, i8 128, i8 0, i8 15,
+                 i8 undef, i8 128, i8 0, i8 15>
+}
+
+; ...and again with the lower bits of the replicated constant.
+define <16 x i8> @f15() {
+; CHECK-LABEL: f15:
+; CHECK: vgmf %v24, 18, 3
+; CHECK: br %r14
+  ret <16 x i8> <i8 240, i8 0, i8 63, i8 undef,
+                 i8 240, i8 0, i8 63, i8 undef,
+                 i8 240, i8 0, i8 63, i8 undef,
+                 i8 240, i8 0, i8 63, i8 undef>
+}
+
+; Repeat f9 with arbitrary undefs.
+define <16 x i8> @f16() {
+; CHECK-LABEL: f16:
+; CHECK: vgmg %v24, 31, 42
+; CHECK: br %r14
+  ret <16 x i8> <i8 undef, i8 0, i8 undef, i8 1,
+                 i8 255, i8 undef, i8 0, i8 0,
+                 i8 0, i8 0, i8 0, i8 1,
+                 i8 undef, i8 224, i8 undef, i8 undef>
+}
+
+; Try a case where we want some consistent undefs to be treated as 0
+; and some to be treated as 255.
+define <16 x i8> @f17() {
+; CHECK-LABEL: f17:
+; CHECK: vgmg %v24, 23, 35
+; CHECK: br %r14
+  ret <16 x i8> <i8 0, i8 undef, i8 1, i8 undef,
+                 i8 240, i8 undef, i8 0, i8 0,
+                 i8 0, i8 undef, i8 1, i8 undef,
+                 i8 240, i8 undef, i8 0, i8 0>
+}
diff --git a/test/CodeGen/SystemZ/vec-const-14.ll b/test/CodeGen/SystemZ/vec-const-14.ll
new file mode 100644
index 00000000000..0e3f124dbf6
--- /dev/null
+++ b/test/CodeGen/SystemZ/vec-const-14.ll
@@ -0,0 +1,113 @@
+; Test vector replicates that use VECTOR GENERATE MASK, v8i16 version.
+;
+; RUN: llc < %s -mtriple=s390x-linux-gnu -mcpu=z13 | FileCheck %s
+
+; Test a word-granularity replicate with the lowest value that cannot use
+; VREPIF.
+define <8 x i16> @f1() {
+; CHECK-LABEL: f1:
+; CHECK: vgmf %v24, 16, 16
+; CHECK: br %r14
+  ret <8 x i16> <i16 0, i16 32768, i16 0, i16 32768,
+                 i16 0, i16 32768, i16 0, i16 32768>
+}
+
+; Test a word-granularity replicate that has the lower 17 bits set.
+define <8 x i16> @f2() {
+; CHECK-LABEL: f2:
+; CHECK: vgmf %v24, 15, 31
+; CHECK: br %r14
+  ret <8 x i16> <i16 1, i16 -1, i16 1, i16 -1,
+                 i16 1, i16 -1, i16 1, i16 -1>
+}
+
+; Test a word-granularity replicate that has the upper 15 bits set.
+define <8 x i16> @f3() {
+; CHECK-LABEL: f3:
+; CHECK: vgmf %v24, 0, 14
+; CHECK: br %r14
+  ret <8 x i16> <i16 -2, i16 0, i16 -2, i16 0,
+                 i16 -2, i16 0, i16 -2, i16 0>
+}
+
+; Test a word-granularity replicate that has middle bits set.
+define <8 x i16> @f4() {
+; CHECK-LABEL: f4:
+; CHECK: vgmf %v24, 12, 17
+; CHECK: br %r14
+  ret <8 x i16> <i16 15, i16 49152, i16 15, i16 49152,
+                 i16 15, i16 49152, i16 15, i16 49152>
+}
+
+; Test a word-granularity replicate with a wrap-around mask.
+define <8 x i16> @f5() {
+; CHECK-LABEL: f5:
+; CHECK: vgmf %v24, 17, 15
+; CHECK: br %r14
+  ret <8 x i16> <i16 -1, i16 32767, i16 -1, i16 32767,
+                 i16 -1, i16 32767, i16 -1, i16 32767>
+}
+
+; Test a doubleword-granularity replicate with the lowest value that cannot
+; use VREPIG.
+define <8 x i16> @f6() {
+; CHECK-LABEL: f6:
+; CHECK: vgmg %v24, 48, 48
+; CHECK: br %r14
+  ret <8 x i16> <i16 0, i16 0, i16 0, i16 32768,
+                 i16 0, i16 0, i16 0, i16 32768>
+}
+
+; Test a doubleword-granularity replicate that has the lower 22 bits set.
+define <8 x i16> @f7() {
+; CHECK-LABEL: f7:
+; CHECK: vgmg %v24, 42, 63
+; CHECK: br %r14
+  ret <8 x i16> <i16 0, i16 0, i16 63, i16 -1,
+                 i16 0, i16 0, i16 63, i16 -1>
+}
+
+; Test a doubleword-granularity replicate that has the upper 45 bits set.
+define <8 x i16> @f8() {
+; CHECK-LABEL: f8:
+; CHECK: vgmg %v24, 0, 44
+; CHECK: br %r14
+  ret <8 x i16> <i16 -1, i16 -1, i16 -8, i16 0,
+                 i16 -1, i16 -1, i16 -8, i16 0>
+}
+
+; Test a doubleword-granularity replicate that has middle bits set.
+define <8 x i16> @f9() {
+; CHECK-LABEL: f9:
+; CHECK: vgmg %v24, 31, 42
+; CHECK: br %r14
+  ret <8 x i16> <i16 0, i16 1, i16 -32, i16 0,
+                 i16 0, i16 1, i16 -32, i16 0>
+}
+
+; Test a doubleword-granularity replicate with a wrap-around mask.
+define <8 x i16> @f10() {
+; CHECK-LABEL: f10:
+; CHECK: vgmg %v24, 18, 0
+; CHECK: br %r14
+  ret <8 x i16> <i16 32768, i16 16383, i16 -1, i16 -1,
+                 i16 32768, i16 16383, i16 -1, i16 -1>
+}
+
+; Retest f1 with arbitrary undefs instead of 0s.
+define <8 x i16> @f11() {
+; CHECK-LABEL: f11:
+; CHECK: vgmf %v24, 16, 16
+; CHECK: br %r14
+  ret <8 x i16> <i16 undef, i16 32768, i16 0, i16 32768,
+                 i16 0, i16 32768, i16 undef, i16 32768>
+}
+
+; ...likewise f9.
+define <8 x i16> @f12() {
+; CHECK-LABEL: f12:
+; CHECK: vgmg %v24, 31, 42
+; CHECK: br %r14
+  ret <8 x i16> <i16 undef, i16 1, i16 -32, i16 0,
+                 i16 0, i16 1, i16 -32, i16 undef>
+}
diff --git a/test/CodeGen/SystemZ/vec-const-15.ll b/test/CodeGen/SystemZ/vec-const-15.ll
new file mode 100644
index 00000000000..cec445efe89
--- /dev/null
+++ b/test/CodeGen/SystemZ/vec-const-15.ll
@@ -0,0 +1,85 @@
+; Test vector replicates that use VECTOR GENERATE MASK, v4i32 version.
+;
+; RUN: llc < %s -mtriple=s390x-linux-gnu -mcpu=z13 | FileCheck %s
+
+; Test a word-granularity replicate with the lowest value that cannot use
+; VREPIF.
+define <4 x i32> @f1() {
+; CHECK-LABEL: f1:
+; CHECK: vgmf %v24, 16, 16
+; CHECK: br %r14
+  ret <4 x i32> <i32 32768, i32 32768, i32 32768, i32 32768>
+}
+
+; Test a word-granularity replicate that has the lower 17 bits set.
+define <4 x i32> @f2() {
+; CHECK-LABEL: f2:
+; CHECK: vgmf %v24, 15, 31
+; CHECK: br %r14
+  ret <4 x i32> <i32 131071, i32 131071, i32 131071, i32 131071>
+}
+
+; Test a word-granularity replicate that has the upper 15 bits set.
+define <4 x i32> @f3() {
+; CHECK-LABEL: f3:
+; CHECK: vgmf %v24, 0, 14
+; CHECK: br %r14
+  ret <4 x i32> <i32 -131072, i32 -131072, i32 -131072, i32 -131072>
+}
+
+; Test a word-granularity replicate that has middle bits set.
+define <4 x i32> @f4() {
+; CHECK-LABEL: f4:
+; CHECK: vgmf %v24, 12, 17
+; CHECK: br %r14
+  ret <4 x i32> <i32 1032192, i32 1032192, i32 1032192, i32 1032192>
+}
+
+; Test a word-granularity replicate with a wrap-around mask.
+define <4 x i32> @f5() {
+; CHECK-LABEL: f5:
+; CHECK: vgmf %v24, 17, 15
+; CHECK: br %r14
+  ret <4 x i32> <i32 -32769, i32 -32769, i32 -32769, i32 -32769>
+}
+
+; Test a doubleword-granularity replicate with the lowest value that cannot
+; use VREPIG.
+define <4 x i32> @f6() {
+; CHECK-LABEL: f6:
+; CHECK: vgmg %v24, 48, 48
+; CHECK: br %r14
+  ret <4 x i32> <i32 0, i32 32768, i32 0, i32 32768>
+}
+
+; Test a doubleword-granularity replicate that has the lower 22 bits set.
+define <4 x i32> @f7() {
+; CHECK-LABEL: f7:
+; CHECK: vgmg %v24, 42, 63
+; CHECK: br %r14
+  ret <4 x i32> <i32 0, i32 4194303, i32 0, i32 4194303>
+}
+
+; Test a doubleword-granularity replicate that has the upper 45 bits set.
+define <4 x i32> @f8() {
+; CHECK-LABEL: f8:
+; CHECK: vgmg %v24, 0, 44
+; CHECK: br %r14
+  ret <4 x i32> <i32 -1, i32 -524288, i32 -1, i32 -524288>
+}
+
+; Test a doubleword-granularity replicate that has middle bits set.
+define <4 x i32> @f9() {
+; CHECK-LABEL: f9:
+; CHECK: vgmg %v24, 31, 42
+; CHECK: br %r14
+  ret <4 x i32> <i32 1, i32 -2097152, i32 1, i32 -2097152>
+}
+
+; Test a doubleword-granularity replicate with a wrap-around mask.
+define <4 x i32> @f10() {
+; CHECK-LABEL: f10:
+; CHECK: vgmg %v24, 18, 0
+; CHECK: br %r14
+  ret <4 x i32> <i32 -2147467265, i32 -1, i32 -2147467265, i32 -1>
+}
diff --git a/test/CodeGen/SystemZ/vec-const-16.ll b/test/CodeGen/SystemZ/vec-const-16.ll
new file mode 100644
index 00000000000..1ab7de2761c
--- /dev/null
+++ b/test/CodeGen/SystemZ/vec-const-16.ll
@@ -0,0 +1,85 @@
+; Test vector replicates that use VECTOR GENERATE MASK, v2i64 version.
+;
+; RUN: llc < %s -mtriple=s390x-linux-gnu -mcpu=z13 | FileCheck %s
+
+; Test a word-granularity replicate with the lowest value that cannot use
+; VREPIF.
+define <2 x i64> @f1() {
+; CHECK-LABEL: f1:
+; CHECK: vgmf %v24, 16, 16
+; CHECK: br %r14
+  ret <2 x i64> <i64 140737488388096, i64 140737488388096>
+}
+
+; Test a word-granularity replicate that has the lower 17 bits set.
+define <2 x i64> @f2() {
+; CHECK-LABEL: f2:
+; CHECK: vgmf %v24, 15, 31
+; CHECK: br %r14
+  ret <2 x i64> <i64 562945658585087, i64 562945658585087>
+}
+
+; Test a word-granularity replicate that has the upper 15 bits set.
+define <2 x i64> @f3() {
+; CHECK-LABEL: f3:
+; CHECK: vgmf %v24, 0, 14
+; CHECK: br %r14
+  ret <2 x i64> <i64 -562945658585088, i64 -562945658585088>
+}
+
+; Test a word-granularity replicate that has middle bits set.
+define <2 x i64> @f4() {
+; CHECK-LABEL: f4:
+; CHECK: vgmf %v24, 12, 17
+; CHECK: br %r14
+  ret <2 x i64> <i64 4433230884225024, i64 4433230884225024>
+}
+
+; Test a word-granularity replicate with a wrap-around mask.
+define <2 x i64> @f5() {
+; CHECK-LABEL: f5:
+; CHECK: vgmf %v24, 17, 15
+; CHECK: br %r14
+  ret <2 x i64> <i64 -140737488388097, i64 -140737488388097>
+}
+
+; Test a doubleword-granularity replicate with the lowest value that cannot
+; use VREPIG.
+define <2 x i64> @f6() {
+; CHECK-LABEL: f6:
+; CHECK: vgmg %v24, 48, 48
+; CHECK: br %r14
+  ret <2 x i64> <i64 32768, i64 32768>
+}
+
+; Test a doubleword-granularity replicate that has the lower 22 bits set.
+define <2 x i64> @f7() {
+; CHECK-LABEL: f7:
+; CHECK: vgmg %v24, 42, 63
+; CHECK: br %r14
+  ret <2 x i64> <i64 4194303, i64 4194303>
+}
+
+; Test a doubleword-granularity replicate that has the upper 45 bits set.
+define <2 x i64> @f8() {
+; CHECK-LABEL: f8:
+; CHECK: vgmg %v24, 0, 44
+; CHECK: br %r14
+  ret <2 x i64> <i64 -524288, i64 -524288>
+}
+
+; Test a doubleword-granularity replicate that has middle bits set.
+define <2 x i64> @f9() {
+; CHECK-LABEL: f9:
+; CHECK: vgmg %v24, 31, 42
+; CHECK: br %r14
+  ret <2 x i64> <i64 8587837440, i64 8587837440>
+}
+
+; Test a doubleword-granularity replicate with a wrap-around mask.
+define <2 x i64> @f10() {
+; CHECK-LABEL: f10:
+; CHECK: vgmg %v24, 18, 0
+; CHECK: br %r14
+  ret <2 x i64> <i64 -9223301668110598145, i64 -9223301668110598145>
+}
diff --git a/test/CodeGen/SystemZ/vec-const-17.ll b/test/CodeGen/SystemZ/vec-const-17.ll
new file mode 100644
index 00000000000..1306eab556e
--- /dev/null
+++ b/test/CodeGen/SystemZ/vec-const-17.ll
@@ -0,0 +1,95 @@
+; Test vector replicates that use VECTOR GENERATE MASK, v4f32 version.
+;
+; RUN: llc < %s -mtriple=s390x-linux-gnu -mcpu=z13 | FileCheck %s
+
+; Test a word-granularity replicate with the lowest value that cannot use
+; VREPIF.
+define <4 x float> @f1() {
+; CHECK-LABEL: f1:
+; CHECK: vgmf %v24, 16, 16
+; CHECK: br %r14
+  ret <4 x float> <float 0x3790000000000000, float 0x3790000000000000,
+                   float 0x3790000000000000, float 0x3790000000000000>
+}
+
+; Test a word-granularity replicate that has the lower 17 bits set.
+define <4 x float> @f2() {
+; CHECK-LABEL: f2:
+; CHECK: vgmf %v24, 15, 31
+; CHECK: br %r14
+  ret <4 x float> <float 0x37affff000000000, float 0x37affff000000000,
+                   float 0x37affff000000000, float 0x37affff000000000>
+}
+
+; Test a word-granularity replicate that has the upper 15 bits set.
+define <4 x float> @f3() {
+; CHECK-LABEL: f3:
+; CHECK: vgmf %v24, 0, 14
+; CHECK: br %r14
+  ret <4 x float> <float 0xffffc00000000000, float 0xffffc00000000000,
+                   float 0xffffc00000000000, float 0xffffc00000000000>
+}
+
+; Test a word-granularity replicate that has middle bits set.
+define <4 x float> @f4() {
+; CHECK-LABEL: f4:
+; CHECK: vgmf %v24, 2, 8
+; CHECK: br %r14
+  ret <4 x float> <float 0x3ff0000000000000, float 0x3ff0000000000000,
+                   float 0x3ff0000000000000, float 0x3ff0000000000000>
+}
+
+; Test a word-granularity replicate with a wrap-around mask.
+define <4 x float> @f5() {
+; CHECK-LABEL: f5:
+; CHECK: vgmf %v24, 9, 1
+; CHECK: br %r14
+  ret <4 x float> <float 0xc00fffffe0000000, float 0xc00fffffe0000000,
+                   float 0xc00fffffe0000000, float 0xc00fffffe0000000>
+}
+
+; Test a doubleword-granularity replicate with the lowest value that cannot
+; use VREPIG.
+define <4 x float> @f6() {
+; CHECK-LABEL: f6:
+; CHECK: vgmg %v24, 48, 48
+; CHECK: br %r14
+  ret <4 x float> <float 0.0, float 0x3790000000000000,
+                   float 0.0, float 0x3790000000000000>
+}
+
+; Test a doubleword-granularity replicate that has the lower 22 bits set.
+define <4 x float> @f7() {
+; CHECK-LABEL: f7:
+; CHECK: vgmg %v24, 42, 63
+; CHECK: br %r14
+  ret <4 x float> <float 0.0, float 0x37ffffff80000000,
+                   float 0.0, float 0x37ffffff80000000>
+}
+
+; Test a doubleword-granularity replicate that has the upper 45 bits set.
+define <4 x float> @f8() {
+; CHECK-LABEL: f8:
+; CHECK: vgmg %v24, 0, 44
+; CHECK: br %r14
+  ret <4 x float> <float 0xffffffffe0000000, float 0xffff000000000000,
+                   float 0xffffffffe0000000, float 0xffff000000000000>
+}
+
+; Test a doubleword-granularity replicate that has middle bits set.
+define <4 x float> @f9() {
+; CHECK-LABEL: f9:
+; CHECK: vgmg %v24, 34, 41
+; CHECK: br %r14
+  ret <4 x float> <float 0.0, float 0x3ff8000000000000,
+                   float 0.0, float 0x3ff8000000000000>
+}
+
+; Test a doubleword-granularity replicate with a wrap-around mask.
+define <4 x float> @f10() {
+; CHECK-LABEL: f10:
+; CHECK: vgmg %v24, 32, 0
+; CHECK: br %r14
+  ret <4 x float> <float 0x8000000000000000, float 0xffffffffe0000000,
+                   float 0x8000000000000000, float 0xffffffffe0000000>
+}
diff --git a/test/CodeGen/SystemZ/vec-const-18.ll b/test/CodeGen/SystemZ/vec-const-18.ll
new file mode 100644
index 00000000000..c6c20c2a003
--- /dev/null
+++ b/test/CodeGen/SystemZ/vec-const-18.ll
@@ -0,0 +1,85 @@
+; Test vector replicates that use VECTOR GENERATE MASK, v2f64 version.
+;
+; RUN: llc < %s -mtriple=s390x-linux-gnu -mcpu=z13 | FileCheck %s
+
+; Test a word-granularity replicate with the lowest value that cannot use
+; VREPIF.
+define <2 x double> @f1() {
+; CHECK-LABEL: f1:
+; CHECK: vgmf %v24, 16, 16
+; CHECK: br %r14
+  ret <2 x double> <double 0x0000800000008000, double 0x0000800000008000>
+}
+
+; Test a word-granularity replicate that has the lower 17 bits set.
+define <2 x double> @f2() {
+; CHECK-LABEL: f2:
+; CHECK: vgmf %v24, 15, 31
+; CHECK: br %r14
+  ret <2 x double> <double 0x0001ffff0001ffff, double 0x0001ffff0001ffff>
+}
+
+; Test a word-granularity replicate that has the upper 15 bits set.
+define <2 x double> @f3() {
+; CHECK-LABEL: f3:
+; CHECK: vgmf %v24, 0, 14
+; CHECK: br %r14
+  ret <2 x double> <double 0xfffe0000fffe0000, double 0xfffe0000fffe0000>
+}
+
+; Test a word-granularity replicate that has middle bits set.
+define <2 x double> @f4() {
+; CHECK-LABEL: f4:
+; CHECK: vgmf %v24, 2, 11
+; CHECK: br %r14
+  ret <2 x double> <double 0x3ff000003ff00000, double 0x3ff000003ff00000>
+}
+
+; Test a word-granularity replicate with a wrap-around mask.
+define <2 x double> @f5() {
+; CHECK-LABEL: f5:
+; CHECK: vgmf %v24, 17, 15
+; CHECK: br %r14
+  ret <2 x double> <double 0xffff7fffffff7fff, double 0xffff7fffffff7fff>
+}
+
+; Test a doubleword-granularity replicate with the lowest value that cannot
+; use VREPIG.
+define <2 x double> @f6() {
+; CHECK-LABEL: f6:
+; CHECK: vgmg %v24, 48, 48
+; CHECK: br %r14
+  ret <2 x double> <double 0x0000000000008000, double 0x0000000000008000>
+}
+
+; Test a doubleword-granularity replicate that has the lower 22 bits set.
+define <2 x double> @f7() {
+; CHECK-LABEL: f7:
+; CHECK: vgmg %v24, 42, 63
+; CHECK: br %r14
+  ret <2 x double> <double 0x000000000003fffff, double 0x000000000003fffff>
+}
+
+; Test a doubleword-granularity replicate that has the upper 45 bits set.
+define <2 x double> @f8() {
+; CHECK-LABEL: f8:
+; CHECK: vgmg %v24, 0, 44
+; CHECK: br %r14
+  ret <2 x double> <double 0xfffffffffff80000, double 0xfffffffffff80000>
+}
+
+; Test a doubleword-granularity replicate that has middle bits set.
+define <2 x double> @f9() {
+; CHECK-LABEL: f9:
+; CHECK: vgmg %v24, 2, 11
+; CHECK: br %r14
+  ret <2 x double> <double 0x3ff0000000000000, double 0x3ff0000000000000>
+}
+
+; Test a doubleword-granularity replicate with a wrap-around mask.
+define <2 x double> @f10() {
+; CHECK-LABEL: f10:
+; CHECK: vgmg %v24, 10, 0
+; CHECK: br %r14
+  ret <2 x double> <double 0x803fffffffffffff, double 0x803fffffffffffff>
+}
diff --git a/test/CodeGen/SystemZ/vec-conv-01.ll b/test/CodeGen/SystemZ/vec-conv-01.ll
new file mode 100644
index 00000000000..cbf42c0f533
--- /dev/null
+++ b/test/CodeGen/SystemZ/vec-conv-01.ll
@@ -0,0 +1,95 @@
+; Test conversions between integer and float elements.
+;
+; RUN: llc < %s -mtriple=s390x-linux-gnu -mcpu=z13 | FileCheck %s
+
+; Test conversion of f64s to signed i64s.
+define <2 x i64> @f1(<2 x double> %doubles) {
+; CHECK-LABEL: f1:
+; CHECK: vcgdb %v24, %v24, 0, 5
+; CHECK: br %r14
+  %dwords = fptosi <2 x double> %doubles to <2 x i64>
+  ret <2 x i64> %dwords
+}
+
+; Test conversion of f64s to unsigned i64s.
+define <2 x i64> @f2(<2 x double> %doubles) {
+; CHECK-LABEL: f2:
+; CHECK: vclgdb %v24, %v24, 0, 5
+; CHECK: br %r14
+  %dwords = fptoui <2 x double> %doubles to <2 x i64>
+  ret <2 x i64> %dwords
+}
+
+; Test conversion of signed i64s to f64s.
+define <2 x double> @f3(<2 x i64> %dwords) {
+; CHECK-LABEL: f3:
+; CHECK: vcdgb %v24, %v24, 0, 0
+; CHECK: br %r14
+  %doubles = sitofp <2 x i64> %dwords to <2 x double>
+  ret <2 x double> %doubles
+}
+
+; Test conversion of unsigned i64s to f64s.
+define <2 x double> @f4(<2 x i64> %dwords) {
+; CHECK-LABEL: f4:
+; CHECK: vcdlgb %v24, %v24, 0, 0
+; CHECK: br %r14
+  %doubles = uitofp <2 x i64> %dwords to <2 x double>
+  ret <2 x double> %doubles
+}
+
+; Test conversion of f64s to signed i32s, which must compile.
+define void @f5(<2 x double> %doubles, <2 x i32> *%ptr) {
+  %words = fptosi <2 x double> %doubles to <2 x i32>
+  store <2 x i32> %words, <2 x i32> *%ptr
+  ret void
+}
+
+; Test conversion of f64s to unsigned i32s, which must compile.
+define void @f6(<2 x double> %doubles, <2 x i32> *%ptr) {
+  %words = fptoui <2 x double> %doubles to <2 x i32>
+  store <2 x i32> %words, <2 x i32> *%ptr
+  ret void
+}
+
+; Test conversion of signed i32s to f64s, which must compile.
+define <2 x double> @f7(<2 x i32> *%ptr) {
+  %words = load <2 x i32>, <2 x i32> *%ptr
+  %doubles = sitofp <2 x i32> %words to <2 x double>
+  ret <2 x double> %doubles
+}
+
+; Test conversion of unsigned i32s to f64s, which must compile.
+define <2 x double> @f8(<2 x i32> *%ptr) {
+  %words = load <2 x i32>, <2 x i32> *%ptr
+  %doubles = uitofp <2 x i32> %words to <2 x double>
+  ret <2 x double> %doubles
+}
+
+; Test conversion of f32s to signed i64s, which must compile.
+define <2 x i64> @f9(<2 x float> *%ptr) {
+  %floats = load <2 x float>, <2 x float> *%ptr
+  %dwords = fptosi <2 x float> %floats to <2 x i64>
+  ret <2 x i64> %dwords
+}
+
+; Test conversion of f32s to unsigned i64s, which must compile.
+define <2 x i64> @f10(<2 x float> *%ptr) {
+  %floats = load <2 x float>, <2 x float> *%ptr
+  %dwords = fptoui <2 x float> %floats to <2 x i64>
+  ret <2 x i64> %dwords
+}
+
+; Test conversion of signed i64s to f32, which must compile.
+define void @f11(<2 x i64> %dwords, <2 x float> *%ptr) {
+  %floats = sitofp <2 x i64> %dwords to <2 x float>
+  store <2 x float> %floats, <2 x float> *%ptr
+  ret void
+}
+
+; Test conversion of unsigned i64s to f32, which must compile.
+define void @f12(<2 x i64> %dwords, <2 x float> *%ptr) {
+  %floats = uitofp <2 x i64> %dwords to <2 x float>
+  store <2 x float> %floats, <2 x float> *%ptr
+  ret void
+}
diff --git a/test/CodeGen/SystemZ/vec-conv-02.ll b/test/CodeGen/SystemZ/vec-conv-02.ll
new file mode 100644
index 00000000000..ab84389f3c8
--- /dev/null
+++ b/test/CodeGen/SystemZ/vec-conv-02.ll
@@ -0,0 +1,33 @@
+; Test conversions between different-sized float elements.
+;
+; RUN: llc < %s -mtriple=s390x-linux-gnu -mcpu=z13 | FileCheck %s
+
+; Test cases where both elements of a v2f64 are converted to f32s.
+define void @f1(<2 x double> %val, <2 x float> *%ptr) {
+; CHECK-LABEL: f1:
+; CHECK: vledb {{%v[0-9]+}}, %v24, 0, 0
+; CHECK: br %r14
+  %res = fptrunc <2 x double> %val to <2 x float>
+  store <2 x float> %res, <2 x float> *%ptr
+  ret void
+}
+
+; Test conversion of an f64 in a vector register to an f32.
+define float @f2(<2 x double> %vec) {
+; CHECK-LABEL: f2:
+; CHECK: wledb %f0, %v24
+; CHECK: br %r14
+  %scalar = extractelement <2 x double> %vec, i32 0
+  %ret = fptrunc double %scalar to float
+  ret float %ret
+}
+
+; Test conversion of an f32 in a vector register to an f64.
+define double @f3(<4 x float> %vec) {
+; CHECK-LABEL: f3:
+; CHECK: wldeb %f0, %v24
+; CHECK: br %r14
+  %scalar = extractelement <4 x float> %vec, i32 0
+  %ret = fpext float %scalar to double
+  ret double %ret
+}
diff --git a/test/CodeGen/SystemZ/vec-ctlz-01.ll b/test/CodeGen/SystemZ/vec-ctlz-01.ll
new file mode 100644
index 00000000000..f6502202ef5
--- /dev/null
+++ b/test/CodeGen/SystemZ/vec-ctlz-01.ll
@@ -0,0 +1,81 @@
+; Test vector count leading zeros
+;
+; RUN: llc < %s -mtriple=s390x-linux-gnu -mcpu=z13 | FileCheck %s
+
+declare <16 x i8> @llvm.ctlz.v16i8(<16 x i8> %src, i1 %is_zero_undef)
+declare <8 x i16> @llvm.ctlz.v8i16(<8 x i16> %src, i1 %is_zero_undef)
+declare <4 x i32> @llvm.ctlz.v4i32(<4 x i32> %src, i1 %is_zero_undef)
+declare <2 x i64> @llvm.ctlz.v2i64(<2 x i64> %src, i1 %is_zero_undef)
+
+define <16 x i8> @f1(<16 x i8> %a) {
+; CHECK-LABEL: f1:
+; CHECK: vclzb %v24, %v24
+; CHECK: br    %r14
+
+  %res = call <16 x i8> @llvm.ctlz.v16i8(<16 x i8> %a, i1 false)
+  ret <16 x i8> %res
+}
+
+define <16 x i8> @f2(<16 x i8> %a) {
+; CHECK-LABEL: f2:
+; CHECK: vclzb %v24, %v24
+; CHECK: br    %r14
+
+  %res = call <16 x i8> @llvm.ctlz.v16i8(<16 x i8> %a, i1 true)
+  ret <16 x i8> %res
+}
+
+define <8 x i16> @f3(<8 x i16> %a) {
+; CHECK-LABEL: f3:
+; CHECK: vclzh %v24, %v24
+; CHECK: br    %r14
+
+  %res = call <8 x i16> @llvm.ctlz.v8i16(<8 x i16> %a, i1 false)
+  ret <8 x i16> %res
+}
+
+define <8 x i16> @f4(<8 x i16> %a) {
+; CHECK-LABEL: f4:
+; CHECK: vclzh %v24, %v24
+; CHECK: br    %r14
+
+  %res = call <8 x i16> @llvm.ctlz.v8i16(<8 x i16> %a, i1 true)
+  ret <8 x i16> %res
+}
+
+define <4 x i32> @f5(<4 x i32> %a) {
+; CHECK-LABEL: f5:
+; CHECK: vclzf %v24, %v24
+; CHECK: br    %r14
+
+  %res = call <4 x i32> @llvm.ctlz.v4i32(<4 x i32> %a, i1 false)
+  ret <4 x i32> %res
+}
+
+define <4 x i32> @f6(<4 x i32> %a) {
+; CHECK-LABEL: f6:
+; CHECK: vclzf %v24, %v24
+; CHECK: br    %r14
+
+  %res = call <4 x i32> @llvm.ctlz.v4i32(<4 x i32> %a, i1 true)
+  ret <4 x i32> %res
+}
+
+define <2 x i64> @f7(<2 x i64> %a) {
+; CHECK-LABEL: f7:
+; CHECK: vclzg %v24, %v24
+; CHECK: br    %r14
+
+  %res = call <2 x i64> @llvm.ctlz.v2i64(<2 x i64> %a, i1 false)
+  ret <2 x i64> %res
+}
+
+define <2 x i64> @f8(<2 x i64> %a) {
+; CHECK-LABEL: f8:
+; CHECK: vclzg %v24, %v24
+; CHECK: br    %r14
+
+  %res = call <2 x i64> @llvm.ctlz.v2i64(<2 x i64> %a, i1 true)
+  ret <2 x i64> %res
+}
+
diff --git a/test/CodeGen/SystemZ/vec-ctpop-01.ll b/test/CodeGen/SystemZ/vec-ctpop-01.ll
new file mode 100644
index 00000000000..0056af73a2e
--- /dev/null
+++ b/test/CodeGen/SystemZ/vec-ctpop-01.ll
@@ -0,0 +1,53 @@
+; Test vector population-count instruction
+;
+; RUN: llc < %s -mtriple=s390x-linux-gnu -mcpu=z13 | FileCheck %s
+
+declare <16 x i8> @llvm.ctpop.v16i8(<16 x i8> %a)
+declare <8 x i16> @llvm.ctpop.v8i16(<8 x i16> %a)
+declare <4 x i32> @llvm.ctpop.v4i32(<4 x i32> %a)
+declare <2 x i64> @llvm.ctpop.v2i64(<2 x i64> %a)
+
+define <16 x i8> @f1(<16 x i8> %a) {
+; CHECK-LABEL: f1:
+; CHECK: vpopct  %v24, %v24, 0
+; CHECK: br      %r14
+
+  %popcnt = call <16 x i8> @llvm.ctpop.v16i8(<16 x i8> %a)
+  ret <16 x i8> %popcnt
+}
+
+define <8 x i16> @f2(<8 x i16> %a) {
+; CHECK-LABEL: f2:
+; CHECK: vpopct  [[T1:%v[0-9]+]], %v24, 0
+; CHECK: veslh   [[T2:%v[0-9]+]], [[T1]], 8
+; CHECK: vah     [[T3:%v[0-9]+]], [[T1]], [[T2]]
+; CHECK: vesrlh  %v24, [[T3]], 8
+; CHECK: br      %r14
+
+  %popcnt = call <8 x i16> @llvm.ctpop.v8i16(<8 x i16> %a)
+  ret <8 x i16> %popcnt
+}
+
+define <4 x i32> @f3(<4 x i32> %a) {
+; CHECK-LABEL: f3:
+; CHECK: vpopct  [[T1:%v[0-9]+]], %v24, 0
+; CHECK: vgbm    [[T2:%v[0-9]+]], 0
+; CHECK: vsumb   %v24, [[T1]], [[T2]]
+; CHECK: br      %r14
+
+  %popcnt = call <4 x i32> @llvm.ctpop.v4i32(<4 x i32> %a)
+  ret <4 x i32> %popcnt
+}
+
+define <2 x i64> @f4(<2 x i64> %a) {
+; CHECK-LABEL: f4:
+; CHECK: vpopct  [[T1:%v[0-9]+]], %v24, 0
+; CHECK: vgbm    [[T2:%v[0-9]+]], 0
+; CHECK: vsumb   [[T3:%v[0-9]+]], [[T1]], [[T2]]
+; CHECK: vsumgf  %v24, [[T3]], [[T2]]
+; CHECK: br      %r14
+
+  %popcnt = call <2 x i64> @llvm.ctpop.v2i64(<2 x i64> %a)
+  ret <2 x i64> %popcnt
+}
+
diff --git a/test/CodeGen/SystemZ/vec-cttz-01.ll b/test/CodeGen/SystemZ/vec-cttz-01.ll
new file mode 100644
index 00000000000..00a0d21b42f
--- /dev/null
+++ b/test/CodeGen/SystemZ/vec-cttz-01.ll
@@ -0,0 +1,81 @@
+; Test vector count trailing zeros
+;
+; RUN: llc < %s -mtriple=s390x-linux-gnu -mcpu=z13 | FileCheck %s
+
+declare <16 x i8> @llvm.cttz.v16i8(<16 x i8> %src, i1 %is_zero_undef)
+declare <8 x i16> @llvm.cttz.v8i16(<8 x i16> %src, i1 %is_zero_undef)
+declare <4 x i32> @llvm.cttz.v4i32(<4 x i32> %src, i1 %is_zero_undef)
+declare <2 x i64> @llvm.cttz.v2i64(<2 x i64> %src, i1 %is_zero_undef)
+
+define <16 x i8> @f1(<16 x i8> %a) {
+; CHECK-LABEL: f1:
+; CHECK: vctzb %v24, %v24
+; CHECK: br    %r14
+
+  %res = call <16 x i8> @llvm.cttz.v16i8(<16 x i8> %a, i1 false)
+  ret <16 x i8> %res
+}
+
+define <16 x i8> @f2(<16 x i8> %a) {
+; CHECK-LABEL: f2:
+; CHECK: vctzb %v24, %v24
+; CHECK: br    %r14
+
+  %res = call <16 x i8> @llvm.cttz.v16i8(<16 x i8> %a, i1 true)
+  ret <16 x i8> %res
+}
+
+define <8 x i16> @f3(<8 x i16> %a) {
+; CHECK-LABEL: f3:
+; CHECK: vctzh %v24, %v24
+; CHECK: br    %r14
+
+  %res = call <8 x i16> @llvm.cttz.v8i16(<8 x i16> %a, i1 false)
+  ret <8 x i16> %res
+}
+
+define <8 x i16> @f4(<8 x i16> %a) {
+; CHECK-LABEL: f4:
+; CHECK: vctzh %v24, %v24
+; CHECK: br    %r14
+
+  %res = call <8 x i16> @llvm.cttz.v8i16(<8 x i16> %a, i1 true)
+  ret <8 x i16> %res
+}
+
+define <4 x i32> @f5(<4 x i32> %a) {
+; CHECK-LABEL: f5:
+; CHECK: vctzf %v24, %v24
+; CHECK: br    %r14
+
+  %res = call <4 x i32> @llvm.cttz.v4i32(<4 x i32> %a, i1 false)
+  ret <4 x i32> %res
+}
+
+define <4 x i32> @f6(<4 x i32> %a) {
+; CHECK-LABEL: f6:
+; CHECK: vctzf %v24, %v24
+; CHECK: br    %r14
+
+  %res = call <4 x i32> @llvm.cttz.v4i32(<4 x i32> %a, i1 true)
+  ret <4 x i32> %res
+}
+
+define <2 x i64> @f7(<2 x i64> %a) {
+; CHECK-LABEL: f7:
+; CHECK: vctzg %v24, %v24
+; CHECK: br    %r14
+
+  %res = call <2 x i64> @llvm.cttz.v2i64(<2 x i64> %a, i1 false)
+  ret <2 x i64> %res
+}
+
+define <2 x i64> @f8(<2 x i64> %a) {
+; CHECK-LABEL: f8:
+; CHECK: vctzg %v24, %v24
+; CHECK: br    %r14
+
+  %res = call <2 x i64> @llvm.cttz.v2i64(<2 x i64> %a, i1 true)
+  ret <2 x i64> %res
+}
+
diff --git a/test/CodeGen/SystemZ/vec-div-01.ll b/test/CodeGen/SystemZ/vec-div-01.ll
new file mode 100644
index 00000000000..506d40861d3
--- /dev/null
+++ b/test/CodeGen/SystemZ/vec-div-01.ll
@@ -0,0 +1,83 @@
+; Test vector division.  There is no native integer support for this,
+; so the integer cases are really a test of the operation legalization code.
+;
+; RUN: llc < %s -mtriple=s390x-linux-gnu -mcpu=z13 | FileCheck %s
+
+; Test a v16i8 division.
+define <16 x i8> @f1(<16 x i8> %dummy, <16 x i8> %val1, <16 x i8> %val2) {
+; CHECK-LABEL: f1:
+; CHECK: vlvgp [[REG:%v[0-9]+]],
+; CHECK-DAG: vlvgb [[REG]], {{%r[0-5]}}, 0
+; CHECK-DAG: vlvgb [[REG]], {{%r[0-5]}}, 1
+; CHECK-DAG: vlvgb [[REG]], {{%r[0-5]}}, 2
+; CHECK-DAG: vlvgb [[REG]], {{%r[0-5]}}, 3
+; CHECK-DAG: vlvgb [[REG]], {{%r[0-5]}}, 4
+; CHECK-DAG: vlvgb [[REG]], {{%r[0-5]}}, 5
+; CHECK-DAG: vlvgb [[REG]], {{%r[0-5]}}, 6
+; CHECK-DAG: vlvgb [[REG]], {{%r[0-5]}}, 8
+; CHECK-DAG: vlvgb [[REG]], {{%r[0-5]}}, 9
+; CHECK-DAG: vlvgb [[REG]], {{%r[0-5]}}, 10
+; CHECK-DAG: vlvgb [[REG]], {{%r[0-5]}}, 11
+; CHECK-DAG: vlvgb [[REG]], {{%r[0-5]}}, 12
+; CHECK-DAG: vlvgb [[REG]], {{%r[0-5]}}, 13
+; CHECK-DAG: vlvgb [[REG]], {{%r[0-5]}}, 14
+; CHECK: br %r14
+  %ret = sdiv <16 x i8> %val1, %val2
+  ret <16 x i8> %ret
+}
+
+; Test a v8i16 division.
+define <8 x i16> @f2(<8 x i16> %dummy, <8 x i16> %val1, <8 x i16> %val2) {
+; CHECK-LABEL: f2:
+; CHECK: vlvgp [[REG:%v[0-9]+]],
+; CHECK-DAG: vlvgh [[REG]], {{%r[0-5]}}, 0
+; CHECK-DAG: vlvgh [[REG]], {{%r[0-5]}}, 1
+; CHECK-DAG: vlvgh [[REG]], {{%r[0-5]}}, 2
+; CHECK-DAG: vlvgh [[REG]], {{%r[0-5]}}, 4
+; CHECK-DAG: vlvgh [[REG]], {{%r[0-5]}}, 5
+; CHECK-DAG: vlvgh [[REG]], {{%r[0-5]}}, 6
+; CHECK: br %r14
+  %ret = sdiv <8 x i16> %val1, %val2
+  ret <8 x i16> %ret
+}
+
+; Test a v4i32 division.
+define <4 x i32> @f3(<4 x i32> %dummy, <4 x i32> %val1, <4 x i32> %val2) {
+; CHECK-LABEL: f3:
+; CHECK: vlvgp [[REG:%v[0-9]+]],
+; CHECK-DAG: vlvgf [[REG]], {{%r[0-5]}}, 0
+; CHECK-DAG: vlvgf [[REG]], {{%r[0-5]}}, 2
+; CHECK: br %r14
+  %ret = sdiv <4 x i32> %val1, %val2
+  ret <4 x i32> %ret
+}
+
+; Test a v2i64 division.
+define <2 x i64> @f4(<2 x i64> %dummy, <2 x i64> %val1, <2 x i64> %val2) {
+; CHECK-LABEL: f4:
+; CHECK: vlvgp %v24,
+; CHECK: br %r14
+  %ret = sdiv <2 x i64> %val1, %val2
+  ret <2 x i64> %ret
+}
+
+; Test a v2f64 division.
+define <2 x double> @f5(<2 x double> %dummy, <2 x double> %val1,
+                        <2 x double> %val2) {
+; CHECK-LABEL: f5:
+; CHECK: vfddb %v24, %v26, %v28
+; CHECK: br %r14
+  %ret = fdiv <2 x double> %val1, %val2
+  ret <2 x double> %ret
+}
+
+; Test an f64 division that uses vector registers.
+define double @f6(<2 x double> %val1, <2 x double> %val2) {
+; CHECK-LABEL: f6:
+; CHECK: wfddb %f0, %v24, %v26
+; CHECK: br %r14
+  %scalar1 = extractelement <2 x double> %val1, i32 0
+  %scalar2 = extractelement <2 x double> %val2, i32 0
+  %ret = fdiv double %scalar1, %scalar2
+  ret double %ret
+}
diff --git a/test/CodeGen/SystemZ/vec-extract-01.ll b/test/CodeGen/SystemZ/vec-extract-01.ll
new file mode 100644
index 00000000000..549392ffd64
--- /dev/null
+++ b/test/CodeGen/SystemZ/vec-extract-01.ll
@@ -0,0 +1,13 @@
+; Verify ReplaceExtractVectorEltOfLoadWithNarrowedLoad fixes
+;
+; RUN: llc < %s -mtriple=s390x-linux-gnu -mcpu=z13 | FileCheck %s
+
+; Test a memory copy of a v2i32 (via the constant pool).
+define void @f1(<2 x i32> *%dest) {
+; CHECK-LABEL: f1:
+; CHECK: lgrl [[REG:%r[0-5]]], {{[._A-Za-z0-9]}}
+; CHECK: stg [[REG]], 0(%r2)
+; CHECK: br %r14
+  store <2 x i32> <i32 1000000, i32 99999>, <2 x i32> *%dest
+  ret void
+}
diff --git a/test/CodeGen/SystemZ/vec-extract-02.ll b/test/CodeGen/SystemZ/vec-extract-02.ll
new file mode 100644
index 00000000000..c91e852fcf4
--- /dev/null
+++ b/test/CodeGen/SystemZ/vec-extract-02.ll
@@ -0,0 +1,15 @@
+; Verify ReplaceExtractVectorEltOfLoadWithNarrowedLoad fixes
+;
+; RUN: llc < %s -mtriple=s390x-linux-gnu -mcpu=z13 | FileCheck %s
+
+; Test a case where a vector extraction can be simplified to a scalar load.
+; The index must be extended from i32 to i64.
+define i32 @f1(<4 x i32> *%ptr, i32 %index) {
+; CHECK-LABEL: f1:
+; CHECK: risbg {{%r[0-5]}}, %r3, 30, 189, 2
+; CHECK: l %r2,
+; CHECK: br %r14
+  %vec = load <4 x i32>, <4 x i32> *%ptr
+  %res = extractelement <4 x i32> %vec, i32 %index
+  ret i32 %res
+}
diff --git a/test/CodeGen/SystemZ/vec-intrinsics.ll b/test/CodeGen/SystemZ/vec-intrinsics.ll
new file mode 100644
index 00000000000..55527787da4
--- /dev/null
+++ b/test/CodeGen/SystemZ/vec-intrinsics.ll
@@ -0,0 +1,3335 @@
+; Test vector intrinsics.
+;
+; RUN: llc < %s -mtriple=s390x-linux-gnu -mcpu=z13 | FileCheck %s
+
+declare i32 @llvm.s390.lcbb(i8 *, i32)
+declare <16 x i8> @llvm.s390.vlbb(i8 *, i32)
+declare <16 x i8> @llvm.s390.vll(i32, i8 *)
+declare <2 x i64> @llvm.s390.vpdi(<2 x i64>, <2 x i64>, i32)
+declare <16 x i8> @llvm.s390.vperm(<16 x i8>, <16 x i8>, <16 x i8>)
+declare <16 x i8> @llvm.s390.vpksh(<8 x i16>, <8 x i16>)
+declare <8 x i16> @llvm.s390.vpksf(<4 x i32>, <4 x i32>)
+declare <4 x i32> @llvm.s390.vpksg(<2 x i64>, <2 x i64>)
+declare {<16 x i8>, i32} @llvm.s390.vpkshs(<8 x i16>, <8 x i16>)
+declare {<8 x i16>, i32} @llvm.s390.vpksfs(<4 x i32>, <4 x i32>)
+declare {<4 x i32>, i32} @llvm.s390.vpksgs(<2 x i64>, <2 x i64>)
+declare <16 x i8> @llvm.s390.vpklsh(<8 x i16>, <8 x i16>)
+declare <8 x i16> @llvm.s390.vpklsf(<4 x i32>, <4 x i32>)
+declare <4 x i32> @llvm.s390.vpklsg(<2 x i64>, <2 x i64>)
+declare {<16 x i8>, i32} @llvm.s390.vpklshs(<8 x i16>, <8 x i16>)
+declare {<8 x i16>, i32} @llvm.s390.vpklsfs(<4 x i32>, <4 x i32>)
+declare {<4 x i32>, i32} @llvm.s390.vpklsgs(<2 x i64>, <2 x i64>)
+declare void @llvm.s390.vstl(<16 x i8>, i32, i8 *)
+declare <8 x i16> @llvm.s390.vuphb(<16 x i8>)
+declare <4 x i32> @llvm.s390.vuphh(<8 x i16>)
+declare <2 x i64> @llvm.s390.vuphf(<4 x i32>)
+declare <8 x i16> @llvm.s390.vuplhb(<16 x i8>)
+declare <4 x i32> @llvm.s390.vuplhh(<8 x i16>)
+declare <2 x i64> @llvm.s390.vuplhf(<4 x i32>)
+declare <8 x i16> @llvm.s390.vuplb(<16 x i8>)
+declare <4 x i32> @llvm.s390.vuplhw(<8 x i16>)
+declare <2 x i64> @llvm.s390.vuplf(<4 x i32>)
+declare <8 x i16> @llvm.s390.vupllb(<16 x i8>)
+declare <4 x i32> @llvm.s390.vupllh(<8 x i16>)
+declare <2 x i64> @llvm.s390.vupllf(<4 x i32>)
+declare <16 x i8> @llvm.s390.vaccb(<16 x i8>, <16 x i8>)
+declare <8 x i16> @llvm.s390.vacch(<8 x i16>, <8 x i16>)
+declare <4 x i32> @llvm.s390.vaccf(<4 x i32>, <4 x i32>)
+declare <2 x i64> @llvm.s390.vaccg(<2 x i64>, <2 x i64>)
+declare <16 x i8> @llvm.s390.vaq(<16 x i8>, <16 x i8>)
+declare <16 x i8> @llvm.s390.vacq(<16 x i8>, <16 x i8>, <16 x i8>)
+declare <16 x i8> @llvm.s390.vaccq(<16 x i8>, <16 x i8>)
+declare <16 x i8> @llvm.s390.vacccq(<16 x i8>, <16 x i8>, <16 x i8>)
+declare <16 x i8> @llvm.s390.vavgb(<16 x i8>, <16 x i8>)
+declare <8 x i16> @llvm.s390.vavgh(<8 x i16>, <8 x i16>)
+declare <4 x i32> @llvm.s390.vavgf(<4 x i32>, <4 x i32>)
+declare <2 x i64> @llvm.s390.vavgg(<2 x i64>, <2 x i64>)
+declare <16 x i8> @llvm.s390.vavglb(<16 x i8>, <16 x i8>)
+declare <8 x i16> @llvm.s390.vavglh(<8 x i16>, <8 x i16>)
+declare <4 x i32> @llvm.s390.vavglf(<4 x i32>, <4 x i32>)
+declare <2 x i64> @llvm.s390.vavglg(<2 x i64>, <2 x i64>)
+declare <4 x i32> @llvm.s390.vcksm(<4 x i32>, <4 x i32>)
+declare <8 x i16> @llvm.s390.vgfmb(<16 x i8>, <16 x i8>)
+declare <4 x i32> @llvm.s390.vgfmh(<8 x i16>, <8 x i16>)
+declare <2 x i64> @llvm.s390.vgfmf(<4 x i32>, <4 x i32>)
+declare <16 x i8> @llvm.s390.vgfmg(<2 x i64>, <2 x i64>)
+declare <8 x i16> @llvm.s390.vgfmab(<16 x i8>, <16 x i8>, <8 x i16>)
+declare <4 x i32> @llvm.s390.vgfmah(<8 x i16>, <8 x i16>, <4 x i32>)
+declare <2 x i64> @llvm.s390.vgfmaf(<4 x i32>, <4 x i32>, <2 x i64>)
+declare <16 x i8> @llvm.s390.vgfmag(<2 x i64>, <2 x i64>, <16 x i8>)
+declare <16 x i8> @llvm.s390.vmahb(<16 x i8>, <16 x i8>, <16 x i8>)
+declare <8 x i16> @llvm.s390.vmahh(<8 x i16>, <8 x i16>, <8 x i16>)
+declare <4 x i32> @llvm.s390.vmahf(<4 x i32>, <4 x i32>, <4 x i32>)
+declare <16 x i8> @llvm.s390.vmalhb(<16 x i8>, <16 x i8>, <16 x i8>)
+declare <8 x i16> @llvm.s390.vmalhh(<8 x i16>, <8 x i16>, <8 x i16>)
+declare <4 x i32> @llvm.s390.vmalhf(<4 x i32>, <4 x i32>, <4 x i32>)
+declare <8 x i16> @llvm.s390.vmaeb(<16 x i8>, <16 x i8>, <8 x i16>)
+declare <4 x i32> @llvm.s390.vmaeh(<8 x i16>, <8 x i16>, <4 x i32>)
+declare <2 x i64> @llvm.s390.vmaef(<4 x i32>, <4 x i32>, <2 x i64>)
+declare <8 x i16> @llvm.s390.vmaleb(<16 x i8>, <16 x i8>, <8 x i16>)
+declare <4 x i32> @llvm.s390.vmaleh(<8 x i16>, <8 x i16>, <4 x i32>)
+declare <2 x i64> @llvm.s390.vmalef(<4 x i32>, <4 x i32>, <2 x i64>)
+declare <8 x i16> @llvm.s390.vmaob(<16 x i8>, <16 x i8>, <8 x i16>)
+declare <4 x i32> @llvm.s390.vmaoh(<8 x i16>, <8 x i16>, <4 x i32>)
+declare <2 x i64> @llvm.s390.vmaof(<4 x i32>, <4 x i32>, <2 x i64>)
+declare <8 x i16> @llvm.s390.vmalob(<16 x i8>, <16 x i8>, <8 x i16>)
+declare <4 x i32> @llvm.s390.vmaloh(<8 x i16>, <8 x i16>, <4 x i32>)
+declare <2 x i64> @llvm.s390.vmalof(<4 x i32>, <4 x i32>, <2 x i64>)
+declare <16 x i8> @llvm.s390.vmhb(<16 x i8>, <16 x i8>)
+declare <8 x i16> @llvm.s390.vmhh(<8 x i16>, <8 x i16>)
+declare <4 x i32> @llvm.s390.vmhf(<4 x i32>, <4 x i32>)
+declare <16 x i8> @llvm.s390.vmlhb(<16 x i8>, <16 x i8>)
+declare <8 x i16> @llvm.s390.vmlhh(<8 x i16>, <8 x i16>)
+declare <4 x i32> @llvm.s390.vmlhf(<4 x i32>, <4 x i32>)
+declare <8 x i16> @llvm.s390.vmeb(<16 x i8>, <16 x i8>)
+declare <4 x i32> @llvm.s390.vmeh(<8 x i16>, <8 x i16>)
+declare <2 x i64> @llvm.s390.vmef(<4 x i32>, <4 x i32>)
+declare <8 x i16> @llvm.s390.vmleb(<16 x i8>, <16 x i8>)
+declare <4 x i32> @llvm.s390.vmleh(<8 x i16>, <8 x i16>)
+declare <2 x i64> @llvm.s390.vmlef(<4 x i32>, <4 x i32>)
+declare <8 x i16> @llvm.s390.vmob(<16 x i8>, <16 x i8>)
+declare <4 x i32> @llvm.s390.vmoh(<8 x i16>, <8 x i16>)
+declare <2 x i64> @llvm.s390.vmof(<4 x i32>, <4 x i32>)
+declare <8 x i16> @llvm.s390.vmlob(<16 x i8>, <16 x i8>)
+declare <4 x i32> @llvm.s390.vmloh(<8 x i16>, <8 x i16>)
+declare <2 x i64> @llvm.s390.vmlof(<4 x i32>, <4 x i32>)
+declare <16 x i8> @llvm.s390.verllvb(<16 x i8>, <16 x i8>)
+declare <8 x i16> @llvm.s390.verllvh(<8 x i16>, <8 x i16>)
+declare <4 x i32> @llvm.s390.verllvf(<4 x i32>, <4 x i32>)
+declare <2 x i64> @llvm.s390.verllvg(<2 x i64>, <2 x i64>)
+declare <16 x i8> @llvm.s390.verllb(<16 x i8>, i32)
+declare <8 x i16> @llvm.s390.verllh(<8 x i16>, i32)
+declare <4 x i32> @llvm.s390.verllf(<4 x i32>, i32)
+declare <2 x i64> @llvm.s390.verllg(<2 x i64>, i32)
+declare <16 x i8> @llvm.s390.verimb(<16 x i8>, <16 x i8>, <16 x i8>, i32)
+declare <8 x i16> @llvm.s390.verimh(<8 x i16>, <8 x i16>, <8 x i16>, i32)
+declare <4 x i32> @llvm.s390.verimf(<4 x i32>, <4 x i32>, <4 x i32>, i32)
+declare <2 x i64> @llvm.s390.verimg(<2 x i64>, <2 x i64>, <2 x i64>, i32)
+declare <16 x i8> @llvm.s390.vsl(<16 x i8>, <16 x i8>)
+declare <16 x i8> @llvm.s390.vslb(<16 x i8>, <16 x i8>)
+declare <16 x i8> @llvm.s390.vsra(<16 x i8>, <16 x i8>)
+declare <16 x i8> @llvm.s390.vsrab(<16 x i8>, <16 x i8>)
+declare <16 x i8> @llvm.s390.vsrl(<16 x i8>, <16 x i8>)
+declare <16 x i8> @llvm.s390.vsrlb(<16 x i8>, <16 x i8>)
+declare <16 x i8> @llvm.s390.vsldb(<16 x i8>, <16 x i8>, i32)
+declare <16 x i8> @llvm.s390.vscbib(<16 x i8>, <16 x i8>)
+declare <8 x i16> @llvm.s390.vscbih(<8 x i16>, <8 x i16>)
+declare <4 x i32> @llvm.s390.vscbif(<4 x i32>, <4 x i32>)
+declare <2 x i64> @llvm.s390.vscbig(<2 x i64>, <2 x i64>)
+declare <16 x i8> @llvm.s390.vsq(<16 x i8>, <16 x i8>)
+declare <16 x i8> @llvm.s390.vsbiq(<16 x i8>, <16 x i8>, <16 x i8>)
+declare <16 x i8> @llvm.s390.vscbiq(<16 x i8>, <16 x i8>)
+declare <16 x i8> @llvm.s390.vsbcbiq(<16 x i8>, <16 x i8>, <16 x i8>)
+declare <4 x i32> @llvm.s390.vsumb(<16 x i8>, <16 x i8>)
+declare <4 x i32> @llvm.s390.vsumh(<8 x i16>, <8 x i16>)
+declare <2 x i64> @llvm.s390.vsumgh(<8 x i16>, <8 x i16>)
+declare <2 x i64> @llvm.s390.vsumgf(<4 x i32>, <4 x i32>)
+declare <16 x i8> @llvm.s390.vsumqf(<4 x i32>, <4 x i32>)
+declare <16 x i8> @llvm.s390.vsumqg(<2 x i64>, <2 x i64>)
+declare i32 @llvm.s390.vtm(<16 x i8>, <16 x i8>)
+declare {<16 x i8>, i32} @llvm.s390.vceqbs(<16 x i8>, <16 x i8>)
+declare {<8 x i16>, i32} @llvm.s390.vceqhs(<8 x i16>, <8 x i16>)
+declare {<4 x i32>, i32} @llvm.s390.vceqfs(<4 x i32>, <4 x i32>)
+declare {<2 x i64>, i32} @llvm.s390.vceqgs(<2 x i64>, <2 x i64>)
+declare {<16 x i8>, i32} @llvm.s390.vchbs(<16 x i8>, <16 x i8>)
+declare {<8 x i16>, i32} @llvm.s390.vchhs(<8 x i16>, <8 x i16>)
+declare {<4 x i32>, i32} @llvm.s390.vchfs(<4 x i32>, <4 x i32>)
+declare {<2 x i64>, i32} @llvm.s390.vchgs(<2 x i64>, <2 x i64>)
+declare {<16 x i8>, i32} @llvm.s390.vchlbs(<16 x i8>, <16 x i8>)
+declare {<8 x i16>, i32} @llvm.s390.vchlhs(<8 x i16>, <8 x i16>)
+declare {<4 x i32>, i32} @llvm.s390.vchlfs(<4 x i32>, <4 x i32>)
+declare {<2 x i64>, i32} @llvm.s390.vchlgs(<2 x i64>, <2 x i64>)
+declare <16 x i8> @llvm.s390.vfaeb(<16 x i8>, <16 x i8>, i32)
+declare <8 x i16> @llvm.s390.vfaeh(<8 x i16>, <8 x i16>, i32)
+declare <4 x i32> @llvm.s390.vfaef(<4 x i32>, <4 x i32>, i32)
+declare {<16 x i8>, i32} @llvm.s390.vfaebs(<16 x i8>, <16 x i8>, i32)
+declare {<8 x i16>, i32} @llvm.s390.vfaehs(<8 x i16>, <8 x i16>, i32)
+declare {<4 x i32>, i32} @llvm.s390.vfaefs(<4 x i32>, <4 x i32>, i32)
+declare <16 x i8> @llvm.s390.vfaezb(<16 x i8>, <16 x i8>, i32)
+declare <8 x i16> @llvm.s390.vfaezh(<8 x i16>, <8 x i16>, i32)
+declare <4 x i32> @llvm.s390.vfaezf(<4 x i32>, <4 x i32>, i32)
+declare {<16 x i8>, i32} @llvm.s390.vfaezbs(<16 x i8>, <16 x i8>, i32)
+declare {<8 x i16>, i32} @llvm.s390.vfaezhs(<8 x i16>, <8 x i16>, i32)
+declare {<4 x i32>, i32} @llvm.s390.vfaezfs(<4 x i32>, <4 x i32>, i32)
+declare <16 x i8> @llvm.s390.vfeeb(<16 x i8>, <16 x i8>)
+declare <8 x i16> @llvm.s390.vfeeh(<8 x i16>, <8 x i16>)
+declare <4 x i32> @llvm.s390.vfeef(<4 x i32>, <4 x i32>)
+declare {<16 x i8>, i32} @llvm.s390.vfeebs(<16 x i8>, <16 x i8>)
+declare {<8 x i16>, i32} @llvm.s390.vfeehs(<8 x i16>, <8 x i16>)
+declare {<4 x i32>, i32} @llvm.s390.vfeefs(<4 x i32>, <4 x i32>)
+declare <16 x i8> @llvm.s390.vfeezb(<16 x i8>, <16 x i8>)
+declare <8 x i16> @llvm.s390.vfeezh(<8 x i16>, <8 x i16>)
+declare <4 x i32> @llvm.s390.vfeezf(<4 x i32>, <4 x i32>)
+declare {<16 x i8>, i32} @llvm.s390.vfeezbs(<16 x i8>, <16 x i8>)
+declare {<8 x i16>, i32} @llvm.s390.vfeezhs(<8 x i16>, <8 x i16>)
+declare {<4 x i32>, i32} @llvm.s390.vfeezfs(<4 x i32>, <4 x i32>)
+declare <16 x i8> @llvm.s390.vfeneb(<16 x i8>, <16 x i8>)
+declare <8 x i16> @llvm.s390.vfeneh(<8 x i16>, <8 x i16>)
+declare <4 x i32> @llvm.s390.vfenef(<4 x i32>, <4 x i32>)
+declare {<16 x i8>, i32} @llvm.s390.vfenebs(<16 x i8>, <16 x i8>)
+declare {<8 x i16>, i32} @llvm.s390.vfenehs(<8 x i16>, <8 x i16>)
+declare {<4 x i32>, i32} @llvm.s390.vfenefs(<4 x i32>, <4 x i32>)
+declare <16 x i8> @llvm.s390.vfenezb(<16 x i8>, <16 x i8>)
+declare <8 x i16> @llvm.s390.vfenezh(<8 x i16>, <8 x i16>)
+declare <4 x i32> @llvm.s390.vfenezf(<4 x i32>, <4 x i32>)
+declare {<16 x i8>, i32} @llvm.s390.vfenezbs(<16 x i8>, <16 x i8>)
+declare {<8 x i16>, i32} @llvm.s390.vfenezhs(<8 x i16>, <8 x i16>)
+declare {<4 x i32>, i32} @llvm.s390.vfenezfs(<4 x i32>, <4 x i32>)
+declare <16 x i8> @llvm.s390.vistrb(<16 x i8>)
+declare <8 x i16> @llvm.s390.vistrh(<8 x i16>)
+declare <4 x i32> @llvm.s390.vistrf(<4 x i32>)
+declare {<16 x i8>, i32} @llvm.s390.vistrbs(<16 x i8>)
+declare {<8 x i16>, i32} @llvm.s390.vistrhs(<8 x i16>)
+declare {<4 x i32>, i32} @llvm.s390.vistrfs(<4 x i32>)
+declare <16 x i8> @llvm.s390.vstrcb(<16 x i8>, <16 x i8>, <16 x i8>, i32)
+declare <8 x i16> @llvm.s390.vstrch(<8 x i16>, <8 x i16>, <8 x i16>, i32)
+declare <4 x i32> @llvm.s390.vstrcf(<4 x i32>, <4 x i32>, <4 x i32>, i32)
+declare {<16 x i8>, i32} @llvm.s390.vstrcbs(<16 x i8>, <16 x i8>, <16 x i8>,
+                                            i32)
+declare {<8 x i16>, i32} @llvm.s390.vstrchs(<8 x i16>, <8 x i16>, <8 x i16>,
+                                            i32)
+declare {<4 x i32>, i32} @llvm.s390.vstrcfs(<4 x i32>, <4 x i32>, <4 x i32>,
+                                            i32)
+declare <16 x i8> @llvm.s390.vstrczb(<16 x i8>, <16 x i8>, <16 x i8>, i32)
+declare <8 x i16> @llvm.s390.vstrczh(<8 x i16>, <8 x i16>, <8 x i16>, i32)
+declare <4 x i32> @llvm.s390.vstrczf(<4 x i32>, <4 x i32>, <4 x i32>, i32)
+declare {<16 x i8>, i32} @llvm.s390.vstrczbs(<16 x i8>, <16 x i8>, <16 x i8>,
+                                             i32)
+declare {<8 x i16>, i32} @llvm.s390.vstrczhs(<8 x i16>, <8 x i16>, <8 x i16>,
+                                             i32)
+declare {<4 x i32>, i32} @llvm.s390.vstrczfs(<4 x i32>, <4 x i32>, <4 x i32>,
+                                             i32)
+declare {<2 x i64>, i32} @llvm.s390.vfcedbs(<2 x double>, <2 x double>)
+declare {<2 x i64>, i32} @llvm.s390.vfchdbs(<2 x double>, <2 x double>)
+declare {<2 x i64>, i32} @llvm.s390.vfchedbs(<2 x double>, <2 x double>)
+declare {<2 x i64>, i32} @llvm.s390.vftcidb(<2 x double>, i32)
+declare <2 x double> @llvm.s390.vfidb(<2 x double>, i32, i32)
+
+; LCBB with the lowest M3 operand.
+define i32 @test_lcbb1(i8 *%ptr) {
+; CHECK-LABEL: test_lcbb1:
+; CHECK: lcbb %r2, 0(%r2), 0
+; CHECK: br %r14
+  %res = call i32 @llvm.s390.lcbb(i8 *%ptr, i32 0)
+  ret i32 %res
+}
+
+; LCBB with the highest M3 operand.
+define i32 @test_lcbb2(i8 *%ptr) {
+; CHECK-LABEL: test_lcbb2:
+; CHECK: lcbb %r2, 0(%r2), 15
+; CHECK: br %r14
+  %res = call i32 @llvm.s390.lcbb(i8 *%ptr, i32 15)
+  ret i32 %res
+}
+
+; LCBB with a displacement and index.
+define i32 @test_lcbb3(i8 *%base, i64 %index) {
+; CHECK-LABEL: test_lcbb3:
+; CHECK: lcbb %r2, 4095({{%r2,%r3|%r3,%r2}}), 4
+; CHECK: br %r14
+  %add = add i64 %index, 4095
+  %ptr = getelementptr i8, i8 *%base, i64 %add
+  %res = call i32 @llvm.s390.lcbb(i8 *%ptr, i32 4)
+  ret i32 %res
+}
+
+; LCBB with an out-of-range displacement.
+define i32 @test_lcbb4(i8 *%base) {
+; CHECK-LABEL: test_lcbb4:
+; CHECK: lcbb %r2, 0({{%r[1-5]}}), 5
+; CHECK: br %r14
+  %ptr = getelementptr i8, i8 *%base, i64 4096
+  %res = call i32 @llvm.s390.lcbb(i8 *%ptr, i32 5)
+  ret i32 %res
+}
+
+; VLBB with the lowest M3 operand.
+define <16 x i8> @test_vlbb1(i8 *%ptr) {
+; CHECK-LABEL: test_vlbb1:
+; CHECK: vlbb %v24, 0(%r2), 0
+; CHECK: br %r14
+  %res = call <16 x i8> @llvm.s390.vlbb(i8 *%ptr, i32 0)
+  ret <16 x i8> %res
+}
+
+; VLBB with the highest M3 operand.
+define <16 x i8> @test_vlbb2(i8 *%ptr) {
+; CHECK-LABEL: test_vlbb2:
+; CHECK: vlbb %v24, 0(%r2), 15
+; CHECK: br %r14
+  %res = call <16 x i8> @llvm.s390.vlbb(i8 *%ptr, i32 15)
+  ret <16 x i8> %res
+}
+
+; VLBB with a displacement and index.
+define <16 x i8> @test_vlbb3(i8 *%base, i64 %index) {
+; CHECK-LABEL: test_vlbb3:
+; CHECK: vlbb %v24, 4095({{%r2,%r3|%r3,%r2}}), 4
+; CHECK: br %r14
+  %add = add i64 %index, 4095
+  %ptr = getelementptr i8, i8 *%base, i64 %add
+  %res = call <16 x i8> @llvm.s390.vlbb(i8 *%ptr, i32 4)
+  ret <16 x i8> %res
+}
+
+; VLBB with an out-of-range displacement.
+define <16 x i8> @test_vlbb4(i8 *%base) {
+; CHECK-LABEL: test_vlbb4:
+; CHECK: vlbb %v24, 0({{%r[1-5]}}), 5
+; CHECK: br %r14
+  %ptr = getelementptr i8, i8 *%base, i64 4096
+  %res = call <16 x i8> @llvm.s390.vlbb(i8 *%ptr, i32 5)
+  ret <16 x i8> %res
+}
+
+; VLL with the lowest in-range displacement.
+define <16 x i8> @test_vll1(i8 *%ptr, i32 %length) {
+; CHECK-LABEL: test_vll1:
+; CHECK: vll %v24, %r3, 0(%r2)
+; CHECK: br %r14
+  %res = call <16 x i8> @llvm.s390.vll(i32 %length, i8 *%ptr)
+  ret <16 x i8> %res
+}
+
+; VLL with the highest in-range displacement.
+define <16 x i8> @test_vll2(i8 *%base, i32 %length) {
+; CHECK-LABEL: test_vll2:
+; CHECK: vll %v24, %r3, 4095(%r2)
+; CHECK: br %r14
+  %ptr = getelementptr i8, i8 *%base, i64 4095
+  %res = call <16 x i8> @llvm.s390.vll(i32 %length, i8 *%ptr)
+  ret <16 x i8> %res
+}
+
+; VLL with an out-of-range displacementa.
+define <16 x i8> @test_vll3(i8 *%base, i32 %length) {
+; CHECK-LABEL: test_vll3:
+; CHECK: vll %v24, %r3, 0({{%r[1-5]}})
+; CHECK: br %r14
+  %ptr = getelementptr i8, i8 *%base, i64 4096
+  %res = call <16 x i8> @llvm.s390.vll(i32 %length, i8 *%ptr)
+  ret <16 x i8> %res
+}
+
+; Check that VLL doesn't allow an index.
+define <16 x i8> @test_vll4(i8 *%base, i64 %index, i32 %length) {
+; CHECK-LABEL: test_vll4:
+; CHECK: vll %v24, %r4, 0({{%r[1-5]}})
+; CHECK: br %r14
+  %ptr = getelementptr i8, i8 *%base, i64 %index
+  %res = call <16 x i8> @llvm.s390.vll(i32 %length, i8 *%ptr)
+  ret <16 x i8> %res
+}
+
+; VPDI taking element 0 from each half.
+define <2 x i64> @test_vpdi1(<2 x i64> %a, <2 x i64> %b) {
+; CHECK-LABEL: test_vpdi1:
+; CHECK: vpdi %v24, %v24, %v26, 0
+; CHECK: br %r14
+  %res = call <2 x i64> @llvm.s390.vpdi(<2 x i64> %a, <2 x i64> %b, i32 0)
+  ret <2 x i64> %res
+}
+
+; VPDI taking element 1 from each half.
+define <2 x i64> @test_vpdi2(<2 x i64> %a, <2 x i64> %b) {
+; CHECK-LABEL: test_vpdi2:
+; CHECK: vpdi %v24, %v24, %v26, 10
+; CHECK: br %r14
+  %res = call <2 x i64> @llvm.s390.vpdi(<2 x i64> %a, <2 x i64> %b, i32 10)
+  ret <2 x i64> %res
+}
+
+; VPERM.
+define <16 x i8> @test_vperm(<16 x i8> %a, <16 x i8> %b, <16 x i8> %c) {
+; CHECK-LABEL: test_vperm:
+; CHECK: vperm %v24, %v24, %v26, %v28
+; CHECK: br %r14
+  %res = call <16 x i8> @llvm.s390.vperm(<16 x i8> %a, <16 x i8> %b,
+                                         <16 x i8> %c)
+  ret <16 x i8> %res
+}
+
+; VPKSH.
+define <16 x i8> @test_vpksh(<8 x i16> %a, <8 x i16> %b) {
+; CHECK-LABEL: test_vpksh:
+; CHECK: vpksh %v24, %v24, %v26
+; CHECK: br %r14
+  %res = call <16 x i8> @llvm.s390.vpksh(<8 x i16> %a, <8 x i16> %b)
+  ret <16 x i8> %res
+}
+
+; VPKSF.
+define <8 x i16> @test_vpksf(<4 x i32> %a, <4 x i32> %b) {
+; CHECK-LABEL: test_vpksf:
+; CHECK: vpksf %v24, %v24, %v26
+; CHECK: br %r14
+  %res = call <8 x i16> @llvm.s390.vpksf(<4 x i32> %a, <4 x i32> %b)
+  ret <8 x i16> %res
+}
+
+; VPKSG.
+define <4 x i32> @test_vpksg(<2 x i64> %a, <2 x i64> %b) {
+; CHECK-LABEL: test_vpksg:
+; CHECK: vpksg %v24, %v24, %v26
+; CHECK: br %r14
+  %res = call <4 x i32> @llvm.s390.vpksg(<2 x i64> %a, <2 x i64> %b)
+  ret <4 x i32> %res
+}
+
+; VPKSHS with no processing of the result.
+define <16 x i8> @test_vpkshs(<8 x i16> %a, <8 x i16> %b, i32 *%ccptr) {
+; CHECK-LABEL: test_vpkshs:
+; CHECK: vpkshs %v24, %v24, %v26
+; CHECK: ipm [[REG:%r[0-5]]]
+; CHECK: srl [[REG]], 28
+; CHECK: st [[REG]], 0(%r2)
+; CHECK: br %r14
+  %call = call {<16 x i8>, i32} @llvm.s390.vpkshs(<8 x i16> %a, <8 x i16> %b)
+  %res = extractvalue {<16 x i8>, i32} %call, 0
+  %cc = extractvalue {<16 x i8>, i32} %call, 1
+  store i32 %cc, i32 *%ccptr
+  ret <16 x i8> %res
+}
+
+; VPKSHS, storing to %ptr if all values were saturated.
+define <16 x i8> @test_vpkshs_all_store(<8 x i16> %a, <8 x i16> %b, i32 *%ptr) {
+; CHECK-LABEL: test_vpkshs_all_store:
+; CHECK: vpkshs %v24, %v24, %v26
+; CHECK-NEXT: {{jno|jle}} {{\.L*}}
+; CHECK: mvhi 0(%r2), 0
+; CHECK: br %r14
+  %call = call {<16 x i8>, i32} @llvm.s390.vpkshs(<8 x i16> %a, <8 x i16> %b)
+  %res = extractvalue {<16 x i8>, i32} %call, 0
+  %cc = extractvalue {<16 x i8>, i32} %call, 1
+  %cmp = icmp uge i32 %cc, 3
+  br i1 %cmp, label %store, label %exit
+
+store:
+  store i32 0, i32 *%ptr
+  br label %exit
+
+exit:
+  ret <16 x i8> %res
+}
+
+; VPKSFS with no processing of the result.
+define <8 x i16> @test_vpksfs(<4 x i32> %a, <4 x i32> %b, i32 *%ccptr) {
+; CHECK-LABEL: test_vpksfs:
+; CHECK: vpksfs %v24, %v24, %v26
+; CHECK: ipm [[REG:%r[0-5]]]
+; CHECK: srl [[REG]], 28
+; CHECK: st [[REG]], 0(%r2)
+; CHECK: br %r14
+  %call = call {<8 x i16>, i32} @llvm.s390.vpksfs(<4 x i32> %a, <4 x i32> %b)
+  %res = extractvalue {<8 x i16>, i32} %call, 0
+  %cc = extractvalue {<8 x i16>, i32} %call, 1
+  store i32 %cc, i32 *%ccptr
+  ret <8 x i16> %res
+}
+
+; VPKSFS, storing to %ptr if any values were saturated.
+define <8 x i16> @test_vpksfs_any_store(<4 x i32> %a, <4 x i32> %b, i32 *%ptr) {
+; CHECK-LABEL: test_vpksfs_any_store:
+; CHECK: vpksfs %v24, %v24, %v26
+; CHECK-NEXT: {{jhe|je}} {{\.L*}}
+; CHECK: mvhi 0(%r2), 0
+; CHECK: br %r14
+  %call = call {<8 x i16>, i32} @llvm.s390.vpksfs(<4 x i32> %a, <4 x i32> %b)
+  %res = extractvalue {<8 x i16>, i32} %call, 0
+  %cc = extractvalue {<8 x i16>, i32} %call, 1
+  %cmp = icmp ugt i32 %cc, 0
+  br i1 %cmp, label %store, label %exit
+
+store:
+  store i32 0, i32 *%ptr
+  br label %exit
+
+exit:
+  ret <8 x i16> %res
+}
+
+; VPKSGS with no processing of the result.
+define <4 x i32> @test_vpksgs(<2 x i64> %a, <2 x i64> %b, i32 *%ccptr) {
+; CHECK-LABEL: test_vpksgs:
+; CHECK: vpksgs %v24, %v24, %v26
+; CHECK: ipm [[REG:%r[0-5]]]
+; CHECK: srl [[REG]], 28
+; CHECK: st [[REG]], 0(%r2)
+; CHECK: br %r14
+  %call = call {<4 x i32>, i32} @llvm.s390.vpksgs(<2 x i64> %a, <2 x i64> %b)
+  %res = extractvalue {<4 x i32>, i32} %call, 0
+  %cc = extractvalue {<4 x i32>, i32} %call, 1
+  store i32 %cc, i32 *%ccptr
+  ret <4 x i32> %res
+}
+
+; VPKSGS, storing to %ptr if no elements were saturated
+define <4 x i32> @test_vpksgs_none_store(<2 x i64> %a, <2 x i64> %b,
+                                         i32 *%ptr) {
+; CHECK-LABEL: test_vpksgs_none_store:
+; CHECK: vpksgs %v24, %v24, %v26
+; CHECK-NEXT: {{jnhe|jne}} {{\.L*}}
+; CHECK: mvhi 0(%r2), 0
+; CHECK: br %r14
+  %call = call {<4 x i32>, i32} @llvm.s390.vpksgs(<2 x i64> %a, <2 x i64> %b)
+  %res = extractvalue {<4 x i32>, i32} %call, 0
+  %cc = extractvalue {<4 x i32>, i32} %call, 1
+  %cmp = icmp sle i32 %cc, 0
+  br i1 %cmp, label %store, label %exit
+
+store:
+  store i32 0, i32 *%ptr
+  br label %exit
+
+exit:
+  ret <4 x i32> %res
+}
+
+; VPKLSH.
+define <16 x i8> @test_vpklsh(<8 x i16> %a, <8 x i16> %b) {
+; CHECK-LABEL: test_vpklsh:
+; CHECK: vpklsh %v24, %v24, %v26
+; CHECK: br %r14
+  %res = call <16 x i8> @llvm.s390.vpklsh(<8 x i16> %a, <8 x i16> %b)
+  ret <16 x i8> %res
+}
+
+; VPKLSF.
+define <8 x i16> @test_vpklsf(<4 x i32> %a, <4 x i32> %b) {
+; CHECK-LABEL: test_vpklsf:
+; CHECK: vpklsf %v24, %v24, %v26
+; CHECK: br %r14
+  %res = call <8 x i16> @llvm.s390.vpklsf(<4 x i32> %a, <4 x i32> %b)
+  ret <8 x i16> %res
+}
+
+; VPKLSG.
+define <4 x i32> @test_vpklsg(<2 x i64> %a, <2 x i64> %b) {
+; CHECK-LABEL: test_vpklsg:
+; CHECK: vpklsg %v24, %v24, %v26
+; CHECK: br %r14
+  %res = call <4 x i32> @llvm.s390.vpklsg(<2 x i64> %a, <2 x i64> %b)
+  ret <4 x i32> %res
+}
+
+; VPKLSHS with no processing of the result.
+define <16 x i8> @test_vpklshs(<8 x i16> %a, <8 x i16> %b, i32 *%ccptr) {
+; CHECK-LABEL: test_vpklshs:
+; CHECK: vpklshs %v24, %v24, %v26
+; CHECK: ipm [[REG:%r[0-5]]]
+; CHECK: srl [[REG]], 28
+; CHECK: st [[REG]], 0(%r2)
+; CHECK: br %r14
+  %call = call {<16 x i8>, i32} @llvm.s390.vpklshs(<8 x i16> %a, <8 x i16> %b)
+  %res = extractvalue {<16 x i8>, i32} %call, 0
+  %cc = extractvalue {<16 x i8>, i32} %call, 1
+  store i32 %cc, i32 *%ccptr
+  ret <16 x i8> %res
+}
+
+; VPKLSHS, storing to %ptr if all values were saturated.
+define <16 x i8> @test_vpklshs_all_store(<8 x i16> %a, <8 x i16> %b,
+                                         i32 *%ptr) {
+; CHECK-LABEL: test_vpklshs_all_store:
+; CHECK: vpklshs %v24, %v24, %v26
+; CHECK-NEXT: {{jno|jle}} {{\.L*}}
+; CHECK: mvhi 0(%r2), 0
+; CHECK: br %r14
+  %call = call {<16 x i8>, i32} @llvm.s390.vpklshs(<8 x i16> %a, <8 x i16> %b)
+  %res = extractvalue {<16 x i8>, i32} %call, 0
+  %cc = extractvalue {<16 x i8>, i32} %call, 1
+  %cmp = icmp eq i32 %cc, 3
+  br i1 %cmp, label %store, label %exit
+
+store:
+  store i32 0, i32 *%ptr
+  br label %exit
+
+exit:
+  ret <16 x i8> %res
+}
+
+; VPKLSFS with no processing of the result.
+define <8 x i16> @test_vpklsfs(<4 x i32> %a, <4 x i32> %b, i32 *%ccptr) {
+; CHECK-LABEL: test_vpklsfs:
+; CHECK: vpklsfs %v24, %v24, %v26
+; CHECK: ipm [[REG:%r[0-5]]]
+; CHECK: srl [[REG]], 28
+; CHECK: st [[REG]], 0(%r2)
+; CHECK: br %r14
+  %call = call {<8 x i16>, i32} @llvm.s390.vpklsfs(<4 x i32> %a, <4 x i32> %b)
+  %res = extractvalue {<8 x i16>, i32} %call, 0
+  %cc = extractvalue {<8 x i16>, i32} %call, 1
+  store i32 %cc, i32 *%ccptr
+  ret <8 x i16> %res
+}
+
+; VPKLSFS, storing to %ptr if any values were saturated.
+define <8 x i16> @test_vpklsfs_any_store(<4 x i32> %a, <4 x i32> %b,
+                                         i32 *%ptr) {
+; CHECK-LABEL: test_vpklsfs_any_store:
+; CHECK: vpklsfs %v24, %v24, %v26
+; CHECK-NEXT: {{jhe|je}} {{\.L*}}
+; CHECK: mvhi 0(%r2), 0
+; CHECK: br %r14
+  %call = call {<8 x i16>, i32} @llvm.s390.vpklsfs(<4 x i32> %a, <4 x i32> %b)
+  %res = extractvalue {<8 x i16>, i32} %call, 0
+  %cc = extractvalue {<8 x i16>, i32} %call, 1
+  %cmp = icmp ne i32 %cc, 0
+  br i1 %cmp, label %store, label %exit
+
+store:
+  store i32 0, i32 *%ptr
+  br label %exit
+
+exit:
+  ret <8 x i16> %res
+}
+
+; VPKLSGS with no processing of the result.
+define <4 x i32> @test_vpklsgs(<2 x i64> %a, <2 x i64> %b, i32 *%ccptr) {
+; CHECK-LABEL: test_vpklsgs:
+; CHECK: vpklsgs %v24, %v24, %v26
+; CHECK: ipm [[REG:%r[0-5]]]
+; CHECK: srl [[REG]], 28
+; CHECK: st [[REG]], 0(%r2)
+; CHECK: br %r14
+  %call = call {<4 x i32>, i32} @llvm.s390.vpklsgs(<2 x i64> %a, <2 x i64> %b)
+  %res = extractvalue {<4 x i32>, i32} %call, 0
+  %cc = extractvalue {<4 x i32>, i32} %call, 1
+  store i32 %cc, i32 *%ccptr
+  ret <4 x i32> %res
+}
+
+; VPKLSGS, storing to %ptr if no elements were saturated
+define <4 x i32> @test_vpklsgs_none_store(<2 x i64> %a, <2 x i64> %b,
+                                          i32 *%ptr) {
+; CHECK-LABEL: test_vpklsgs_none_store:
+; CHECK: vpklsgs %v24, %v24, %v26
+; CHECK-NEXT: {{jnhe|jne}} {{\.L*}}
+; CHECK: mvhi 0(%r2), 0
+; CHECK: br %r14
+  %call = call {<4 x i32>, i32} @llvm.s390.vpklsgs(<2 x i64> %a, <2 x i64> %b)
+  %res = extractvalue {<4 x i32>, i32} %call, 0
+  %cc = extractvalue {<4 x i32>, i32} %call, 1
+  %cmp = icmp eq i32 %cc, 0
+  br i1 %cmp, label %store, label %exit
+
+store:
+  store i32 0, i32 *%ptr
+  br label %exit
+
+exit:
+  ret <4 x i32> %res
+}
+
+; VSTL with the lowest in-range displacement.
+define void @test_vstl1(<16 x i8> %vec, i8 *%ptr, i32 %length) {
+; CHECK-LABEL: test_vstl1:
+; CHECK: vstl %v24, %r3, 0(%r2)
+; CHECK: br %r14
+  call void @llvm.s390.vstl(<16 x i8> %vec, i32 %length, i8 *%ptr)
+  ret void
+}
+
+; VSTL with the highest in-range displacement.
+define void @test_vstl2(<16 x i8> %vec, i8 *%base, i32 %length) {
+; CHECK-LABEL: test_vstl2:
+; CHECK: vstl %v24, %r3, 4095(%r2)
+; CHECK: br %r14
+  %ptr = getelementptr i8, i8 *%base, i64 4095
+  call void @llvm.s390.vstl(<16 x i8> %vec, i32 %length, i8 *%ptr)
+  ret void
+}
+
+; VSTL with an out-of-range displacement.
+define void @test_vstl3(<16 x i8> %vec, i8 *%base, i32 %length) {
+; CHECK-LABEL: test_vstl3:
+; CHECK: vstl %v24, %r3, 0({{%r[1-5]}})
+; CHECK: br %r14
+  %ptr = getelementptr i8, i8 *%base, i64 4096
+  call void @llvm.s390.vstl(<16 x i8> %vec, i32 %length, i8 *%ptr)
+  ret void
+}
+
+; Check that VSTL doesn't allow an index.
+define void @test_vstl4(<16 x i8> %vec, i8 *%base, i64 %index, i32 %length) {
+; CHECK-LABEL: test_vstl4:
+; CHECK: vstl %v24, %r4, 0({{%r[1-5]}})
+; CHECK: br %r14
+  %ptr = getelementptr i8, i8 *%base, i64 %index
+  call void @llvm.s390.vstl(<16 x i8> %vec, i32 %length, i8 *%ptr)
+  ret void
+}
+
+; VUPHB.
+define <8 x i16> @test_vuphb(<16 x i8> %a) {
+; CHECK-LABEL: test_vuphb:
+; CHECK: vuphb %v24, %v24
+; CHECK: br %r14
+  %res = call <8 x i16> @llvm.s390.vuphb(<16 x i8> %a)
+  ret <8 x i16> %res
+}
+
+; VUPHH.
+define <4 x i32> @test_vuphh(<8 x i16> %a) {
+; CHECK-LABEL: test_vuphh:
+; CHECK: vuphh %v24, %v24
+; CHECK: br %r14
+  %res = call <4 x i32> @llvm.s390.vuphh(<8 x i16> %a)
+  ret <4 x i32> %res
+}
+
+; VUPHF.
+define <2 x i64> @test_vuphf(<4 x i32> %a) {
+; CHECK-LABEL: test_vuphf:
+; CHECK: vuphf %v24, %v24
+; CHECK: br %r14
+  %res = call <2 x i64> @llvm.s390.vuphf(<4 x i32> %a)
+  ret <2 x i64> %res
+}
+
+; VUPLHB.
+define <8 x i16> @test_vuplhb(<16 x i8> %a) {
+; CHECK-LABEL: test_vuplhb:
+; CHECK: vuplhb %v24, %v24
+; CHECK: br %r14
+  %res = call <8 x i16> @llvm.s390.vuplhb(<16 x i8> %a)
+  ret <8 x i16> %res
+}
+
+; VUPLHH.
+define <4 x i32> @test_vuplhh(<8 x i16> %a) {
+; CHECK-LABEL: test_vuplhh:
+; CHECK: vuplhh %v24, %v24
+; CHECK: br %r14
+  %res = call <4 x i32> @llvm.s390.vuplhh(<8 x i16> %a)
+  ret <4 x i32> %res
+}
+
+; VUPLHF.
+define <2 x i64> @test_vuplhf(<4 x i32> %a) {
+; CHECK-LABEL: test_vuplhf:
+; CHECK: vuplhf %v24, %v24
+; CHECK: br %r14
+  %res = call <2 x i64> @llvm.s390.vuplhf(<4 x i32> %a)
+  ret <2 x i64> %res
+}
+
+; VUPLB.
+define <8 x i16> @test_vuplb(<16 x i8> %a) {
+; CHECK-LABEL: test_vuplb:
+; CHECK: vuplb %v24, %v24
+; CHECK: br %r14
+  %res = call <8 x i16> @llvm.s390.vuplb(<16 x i8> %a)
+  ret <8 x i16> %res
+}
+
+; VUPLHW.
+define <4 x i32> @test_vuplhw(<8 x i16> %a) {
+; CHECK-LABEL: test_vuplhw:
+; CHECK: vuplhw %v24, %v24
+; CHECK: br %r14
+  %res = call <4 x i32> @llvm.s390.vuplhw(<8 x i16> %a)
+  ret <4 x i32> %res
+}
+
+; VUPLF.
+define <2 x i64> @test_vuplf(<4 x i32> %a) {
+; CHECK-LABEL: test_vuplf:
+; CHECK: vuplf %v24, %v24
+; CHECK: br %r14
+  %res = call <2 x i64> @llvm.s390.vuplf(<4 x i32> %a)
+  ret <2 x i64> %res
+}
+
+; VUPLLB.
+define <8 x i16> @test_vupllb(<16 x i8> %a) {
+; CHECK-LABEL: test_vupllb:
+; CHECK: vupllb %v24, %v24
+; CHECK: br %r14
+  %res = call <8 x i16> @llvm.s390.vupllb(<16 x i8> %a)
+  ret <8 x i16> %res
+}
+
+; VUPLLH.
+define <4 x i32> @test_vupllh(<8 x i16> %a) {
+; CHECK-LABEL: test_vupllh:
+; CHECK: vupllh %v24, %v24
+; CHECK: br %r14
+  %res = call <4 x i32> @llvm.s390.vupllh(<8 x i16> %a)
+  ret <4 x i32> %res
+}
+
+; VUPLLF.
+define <2 x i64> @test_vupllf(<4 x i32> %a) {
+; CHECK-LABEL: test_vupllf:
+; CHECK: vupllf %v24, %v24
+; CHECK: br %r14
+  %res = call <2 x i64> @llvm.s390.vupllf(<4 x i32> %a)
+  ret <2 x i64> %res
+}
+
+; VACCB.
+define <16 x i8> @test_vaccb(<16 x i8> %a, <16 x i8> %b) {
+; CHECK-LABEL: test_vaccb:
+; CHECK: vaccb %v24, %v24, %v26
+; CHECK: br %r14
+  %res = call <16 x i8> @llvm.s390.vaccb(<16 x i8> %a, <16 x i8> %b)
+  ret <16 x i8> %res
+}
+
+; VACCH.
+define <8 x i16> @test_vacch(<8 x i16> %a, <8 x i16> %b) {
+; CHECK-LABEL: test_vacch:
+; CHECK: vacch %v24, %v24, %v26
+; CHECK: br %r14
+  %res = call <8 x i16> @llvm.s390.vacch(<8 x i16> %a, <8 x i16> %b)
+  ret <8 x i16> %res
+}
+
+; VACCF.
+define <4 x i32> @test_vaccf(<4 x i32> %a, <4 x i32> %b) {
+; CHECK-LABEL: test_vaccf:
+; CHECK: vaccf %v24, %v24, %v26
+; CHECK: br %r14
+  %res = call <4 x i32> @llvm.s390.vaccf(<4 x i32> %a, <4 x i32> %b)
+  ret <4 x i32> %res
+}
+
+; VACCG.
+define <2 x i64> @test_vaccg(<2 x i64> %a, <2 x i64> %b) {
+; CHECK-LABEL: test_vaccg:
+; CHECK: vaccg %v24, %v24, %v26
+; CHECK: br %r14
+  %res = call <2 x i64> @llvm.s390.vaccg(<2 x i64> %a, <2 x i64> %b)
+  ret <2 x i64> %res
+}
+
+; VAQ.
+define <16 x i8> @test_vaq(<16 x i8> %a, <16 x i8> %b) {
+; CHECK-LABEL: test_vaq:
+; CHECK: vaq %v24, %v24, %v26
+; CHECK: br %r14
+  %res = call <16 x i8> @llvm.s390.vaq(<16 x i8> %a, <16 x i8> %b)
+  ret <16 x i8> %res
+}
+
+; VACQ.
+define <16 x i8> @test_vacq(<16 x i8> %a, <16 x i8> %b, <16 x i8> %c) {
+; CHECK-LABEL: test_vacq:
+; CHECK: vacq %v24, %v24, %v26, %v28
+; CHECK: br %r14
+  %res = call <16 x i8> @llvm.s390.vacq(<16 x i8> %a, <16 x i8> %b,
+                                        <16 x i8> %c)
+  ret <16 x i8> %res
+}
+
+; VACCQ.
+define <16 x i8> @test_vaccq(<16 x i8> %a, <16 x i8> %b) {
+; CHECK-LABEL: test_vaccq:
+; CHECK: vaccq %v24, %v24, %v26
+; CHECK: br %r14
+  %res = call <16 x i8> @llvm.s390.vaccq(<16 x i8> %a, <16 x i8> %b)
+  ret <16 x i8> %res
+}
+
+; VACCCQ.
+define <16 x i8> @test_vacccq(<16 x i8> %a, <16 x i8> %b, <16 x i8> %c) {
+; CHECK-LABEL: test_vacccq:
+; CHECK: vacccq %v24, %v24, %v26, %v28
+; CHECK: br %r14
+  %res = call <16 x i8> @llvm.s390.vacccq(<16 x i8> %a, <16 x i8> %b,
+                                          <16 x i8> %c)
+  ret <16 x i8> %res
+}
+
+; VAVGB.
+define <16 x i8> @test_vavgb(<16 x i8> %a, <16 x i8> %b) {
+; CHECK-LABEL: test_vavgb:
+; CHECK: vavgb %v24, %v24, %v26
+; CHECK: br %r14
+  %res = call <16 x i8> @llvm.s390.vavgb(<16 x i8> %a, <16 x i8> %b)
+  ret <16 x i8> %res
+}
+
+; VAVGH.
+define <8 x i16> @test_vavgh(<8 x i16> %a, <8 x i16> %b) {
+; CHECK-LABEL: test_vavgh:
+; CHECK: vavgh %v24, %v24, %v26
+; CHECK: br %r14
+  %res = call <8 x i16> @llvm.s390.vavgh(<8 x i16> %a, <8 x i16> %b)
+  ret <8 x i16> %res
+}
+
+; VAVGF.
+define <4 x i32> @test_vavgf(<4 x i32> %a, <4 x i32> %b) {
+; CHECK-LABEL: test_vavgf:
+; CHECK: vavgf %v24, %v24, %v26
+; CHECK: br %r14
+  %res = call <4 x i32> @llvm.s390.vavgf(<4 x i32> %a, <4 x i32> %b)
+  ret <4 x i32> %res
+}
+
+; VAVGG.
+define <2 x i64> @test_vavgg(<2 x i64> %a, <2 x i64> %b) {
+; CHECK-LABEL: test_vavgg:
+; CHECK: vavgg %v24, %v24, %v26
+; CHECK: br %r14
+  %res = call <2 x i64> @llvm.s390.vavgg(<2 x i64> %a, <2 x i64> %b)
+  ret <2 x i64> %res
+}
+
+; VAVGLB.
+define <16 x i8> @test_vavglb(<16 x i8> %a, <16 x i8> %b) {
+; CHECK-LABEL: test_vavglb:
+; CHECK: vavglb %v24, %v24, %v26
+; CHECK: br %r14
+  %res = call <16 x i8> @llvm.s390.vavglb(<16 x i8> %a, <16 x i8> %b)
+  ret <16 x i8> %res
+}
+
+; VAVGLH.
+define <8 x i16> @test_vavglh(<8 x i16> %a, <8 x i16> %b) {
+; CHECK-LABEL: test_vavglh:
+; CHECK: vavglh %v24, %v24, %v26
+; CHECK: br %r14
+  %res = call <8 x i16> @llvm.s390.vavglh(<8 x i16> %a, <8 x i16> %b)
+  ret <8 x i16> %res
+}
+
+; VAVGLF.
+define <4 x i32> @test_vavglf(<4 x i32> %a, <4 x i32> %b) {
+; CHECK-LABEL: test_vavglf:
+; CHECK: vavglf %v24, %v24, %v26
+; CHECK: br %r14
+  %res = call <4 x i32> @llvm.s390.vavglf(<4 x i32> %a, <4 x i32> %b)
+  ret <4 x i32> %res
+}
+
+; VAVGLG.
+define <2 x i64> @test_vavglg(<2 x i64> %a, <2 x i64> %b) {
+; CHECK-LABEL: test_vavglg:
+; CHECK: vavglg %v24, %v24, %v26
+; CHECK: br %r14
+  %res = call <2 x i64> @llvm.s390.vavglg(<2 x i64> %a, <2 x i64> %b)
+  ret <2 x i64> %res
+}
+
+; VCKSM.
+define <4 x i32> @test_vcksm(<4 x i32> %a, <4 x i32> %b) {
+; CHECK-LABEL: test_vcksm:
+; CHECK: vcksm %v24, %v24, %v26
+; CHECK: br %r14
+  %res = call <4 x i32> @llvm.s390.vcksm(<4 x i32> %a, <4 x i32> %b)
+  ret <4 x i32> %res
+}
+
+; VGFMB.
+define <8 x i16> @test_vgfmb(<16 x i8> %a, <16 x i8> %b) {
+; CHECK-LABEL: test_vgfmb:
+; CHECK: vgfmb %v24, %v24, %v26
+; CHECK: br %r14
+  %res = call <8 x i16> @llvm.s390.vgfmb(<16 x i8> %a, <16 x i8> %b)
+  ret <8 x i16> %res
+}
+
+; VGFMH.
+define <4 x i32> @test_vgfmh(<8 x i16> %a, <8 x i16> %b) {
+; CHECK-LABEL: test_vgfmh:
+; CHECK: vgfmh %v24, %v24, %v26
+; CHECK: br %r14
+  %res = call <4 x i32> @llvm.s390.vgfmh(<8 x i16> %a, <8 x i16> %b)
+  ret <4 x i32> %res
+}
+
+; VGFMF.
+define <2 x i64> @test_vgfmf(<4 x i32> %a, <4 x i32> %b) {
+; CHECK-LABEL: test_vgfmf:
+; CHECK: vgfmf %v24, %v24, %v26
+; CHECK: br %r14
+  %res = call <2 x i64> @llvm.s390.vgfmf(<4 x i32> %a, <4 x i32> %b)
+  ret <2 x i64> %res
+}
+
+; VGFMG.
+define <16 x i8> @test_vgfmg(<2 x i64> %a, <2 x i64> %b) {
+; CHECK-LABEL: test_vgfmg:
+; CHECK: vgfmg %v24, %v24, %v26
+; CHECK: br %r14
+  %res = call <16 x i8> @llvm.s390.vgfmg(<2 x i64> %a, <2 x i64> %b)
+  ret <16 x i8> %res
+}
+
+; VGFMAB.
+define <8 x i16> @test_vgfmab(<16 x i8> %a, <16 x i8> %b, <8 x i16> %c) {
+; CHECK-LABEL: test_vgfmab:
+; CHECK: vgfmab %v24, %v24, %v26, %v28
+; CHECK: br %r14
+  %res = call <8 x i16> @llvm.s390.vgfmab(<16 x i8> %a, <16 x i8> %b,
+                                          <8 x i16> %c)
+  ret <8 x i16> %res
+}
+
+; VGFMAH.
+define <4 x i32> @test_vgfmah(<8 x i16> %a, <8 x i16> %b, <4 x i32> %c) {
+; CHECK-LABEL: test_vgfmah:
+; CHECK: vgfmah %v24, %v24, %v26, %v28
+; CHECK: br %r14
+  %res = call <4 x i32> @llvm.s390.vgfmah(<8 x i16> %a, <8 x i16> %b,
+                                          <4 x i32> %c)
+  ret <4 x i32> %res
+}
+
+; VGFMAF.
+define <2 x i64> @test_vgfmaf(<4 x i32> %a, <4 x i32> %b, <2 x i64> %c) {
+; CHECK-LABEL: test_vgfmaf:
+; CHECK: vgfmaf %v24, %v24, %v26, %v28
+; CHECK: br %r14
+  %res = call <2 x i64> @llvm.s390.vgfmaf(<4 x i32> %a, <4 x i32> %b,
+                                          <2 x i64> %c)
+  ret <2 x i64> %res
+}
+
+; VGFMAG.
+define <16 x i8> @test_vgfmag(<2 x i64> %a, <2 x i64> %b, <16 x i8> %c) {
+; CHECK-LABEL: test_vgfmag:
+; CHECK: vgfmag %v24, %v24, %v26, %v28
+; CHECK: br %r14
+  %res = call <16 x i8> @llvm.s390.vgfmag(<2 x i64> %a, <2 x i64> %b,
+                                          <16 x i8> %c)
+  ret <16 x i8> %res
+}
+
+; VMAHB.
+define <16 x i8> @test_vmahb(<16 x i8> %a, <16 x i8> %b, <16 x i8> %c) {
+; CHECK-LABEL: test_vmahb:
+; CHECK: vmahb %v24, %v24, %v26, %v28
+; CHECK: br %r14
+  %res = call <16 x i8> @llvm.s390.vmahb(<16 x i8> %a, <16 x i8> %b,
+                                         <16 x i8> %c)
+  ret <16 x i8> %res
+}
+
+; VMAHH.
+define <8 x i16> @test_vmahh(<8 x i16> %a, <8 x i16> %b, <8 x i16> %c) {
+; CHECK-LABEL: test_vmahh:
+; CHECK: vmahh %v24, %v24, %v26, %v28
+; CHECK: br %r14
+  %res = call <8 x i16> @llvm.s390.vmahh(<8 x i16> %a, <8 x i16> %b,
+                                         <8 x i16> %c)
+  ret <8 x i16> %res
+}
+
+; VMAHF.
+define <4 x i32> @test_vmahf(<4 x i32> %a, <4 x i32> %b, <4 x i32> %c) {
+; CHECK-LABEL: test_vmahf:
+; CHECK: vmahf %v24, %v24, %v26, %v28
+; CHECK: br %r14
+  %res = call <4 x i32> @llvm.s390.vmahf(<4 x i32> %a, <4 x i32> %b,
+                                         <4 x i32> %c)
+  ret <4 x i32> %res
+}
+
+; VMALHB.
+define <16 x i8> @test_vmalhb(<16 x i8> %a, <16 x i8> %b, <16 x i8> %c) {
+; CHECK-LABEL: test_vmalhb:
+; CHECK: vmalhb %v24, %v24, %v26, %v28
+; CHECK: br %r14
+  %res = call <16 x i8> @llvm.s390.vmalhb(<16 x i8> %a, <16 x i8> %b,
+                                          <16 x i8> %c)
+  ret <16 x i8> %res
+}
+
+; VMALHH.
+define <8 x i16> @test_vmalhh(<8 x i16> %a, <8 x i16> %b, <8 x i16> %c) {
+; CHECK-LABEL: test_vmalhh:
+; CHECK: vmalhh %v24, %v24, %v26, %v28
+; CHECK: br %r14
+  %res = call <8 x i16> @llvm.s390.vmalhh(<8 x i16> %a, <8 x i16> %b,
+                                          <8 x i16> %c)
+  ret <8 x i16> %res
+}
+
+; VMALHF.
+define <4 x i32> @test_vmalhf(<4 x i32> %a, <4 x i32> %b, <4 x i32> %c) {
+; CHECK-LABEL: test_vmalhf:
+; CHECK: vmalhf %v24, %v24, %v26, %v28
+; CHECK: br %r14
+  %res = call <4 x i32> @llvm.s390.vmalhf(<4 x i32> %a, <4 x i32> %b,
+                                          <4 x i32> %c)
+  ret <4 x i32> %res
+}
+
+; VMAEB.
+define <8 x i16> @test_vmaeb(<16 x i8> %a, <16 x i8> %b, <8 x i16> %c) {
+; CHECK-LABEL: test_vmaeb:
+; CHECK: vmaeb %v24, %v24, %v26, %v28
+; CHECK: br %r14
+  %res = call <8 x i16> @llvm.s390.vmaeb(<16 x i8> %a, <16 x i8> %b,
+                                         <8 x i16> %c)
+  ret <8 x i16> %res
+}
+
+; VMAEH.
+define <4 x i32> @test_vmaeh(<8 x i16> %a, <8 x i16> %b, <4 x i32> %c) {
+; CHECK-LABEL: test_vmaeh:
+; CHECK: vmaeh %v24, %v24, %v26, %v28
+; CHECK: br %r14
+  %res = call <4 x i32> @llvm.s390.vmaeh(<8 x i16> %a, <8 x i16> %b,
+                                         <4 x i32> %c)
+  ret <4 x i32> %res
+}
+
+; VMAEF.
+define <2 x i64> @test_vmaef(<4 x i32> %a, <4 x i32> %b, <2 x i64> %c) {
+; CHECK-LABEL: test_vmaef:
+; CHECK: vmaef %v24, %v24, %v26, %v28
+; CHECK: br %r14
+  %res = call <2 x i64> @llvm.s390.vmaef(<4 x i32> %a, <4 x i32> %b,
+                                         <2 x i64> %c)
+  ret <2 x i64> %res
+}
+
+; VMALEB.
+define <8 x i16> @test_vmaleb(<16 x i8> %a, <16 x i8> %b, <8 x i16> %c) {
+; CHECK-LABEL: test_vmaleb:
+; CHECK: vmaleb %v24, %v24, %v26, %v28
+; CHECK: br %r14
+  %res = call <8 x i16> @llvm.s390.vmaleb(<16 x i8> %a, <16 x i8> %b,
+                                          <8 x i16> %c)
+  ret <8 x i16> %res
+}
+
+; VMALEH.
+define <4 x i32> @test_vmaleh(<8 x i16> %a, <8 x i16> %b, <4 x i32> %c) {
+; CHECK-LABEL: test_vmaleh:
+; CHECK: vmaleh %v24, %v24, %v26, %v28
+; CHECK: br %r14
+  %res = call <4 x i32> @llvm.s390.vmaleh(<8 x i16> %a, <8 x i16> %b,
+                                          <4 x i32> %c)
+  ret <4 x i32> %res
+}
+
+; VMALEF.
+define <2 x i64> @test_vmalef(<4 x i32> %a, <4 x i32> %b, <2 x i64> %c) {
+; CHECK-LABEL: test_vmalef:
+; CHECK: vmalef %v24, %v24, %v26, %v28
+; CHECK: br %r14
+  %res = call <2 x i64> @llvm.s390.vmalef(<4 x i32> %a, <4 x i32> %b,
+                                          <2 x i64> %c)
+  ret <2 x i64> %res
+}
+
+; VMAOB.
+define <8 x i16> @test_vmaob(<16 x i8> %a, <16 x i8> %b, <8 x i16> %c) {
+; CHECK-LABEL: test_vmaob:
+; CHECK: vmaob %v24, %v24, %v26, %v28
+; CHECK: br %r14
+  %res = call <8 x i16> @llvm.s390.vmaob(<16 x i8> %a, <16 x i8> %b,
+                                         <8 x i16> %c)
+  ret <8 x i16> %res
+}
+
+; VMAOH.
+define <4 x i32> @test_vmaoh(<8 x i16> %a, <8 x i16> %b, <4 x i32> %c) {
+; CHECK-LABEL: test_vmaoh:
+; CHECK: vmaoh %v24, %v24, %v26, %v28
+; CHECK: br %r14
+  %res = call <4 x i32> @llvm.s390.vmaoh(<8 x i16> %a, <8 x i16> %b,
+                                         <4 x i32> %c)
+  ret <4 x i32> %res
+}
+
+; VMAOF.
+define <2 x i64> @test_vmaof(<4 x i32> %a, <4 x i32> %b, <2 x i64> %c) {
+; CHECK-LABEL: test_vmaof:
+; CHECK: vmaof %v24, %v24, %v26, %v28
+; CHECK: br %r14
+  %res = call <2 x i64> @llvm.s390.vmaof(<4 x i32> %a, <4 x i32> %b,
+                                         <2 x i64> %c)
+  ret <2 x i64> %res
+}
+
+; VMALOB.
+define <8 x i16> @test_vmalob(<16 x i8> %a, <16 x i8> %b, <8 x i16> %c) {
+; CHECK-LABEL: test_vmalob:
+; CHECK: vmalob %v24, %v24, %v26, %v28
+; CHECK: br %r14
+  %res = call <8 x i16> @llvm.s390.vmalob(<16 x i8> %a, <16 x i8> %b,
+                                          <8 x i16> %c)
+  ret <8 x i16> %res
+}
+
+; VMALOH.
+define <4 x i32> @test_vmaloh(<8 x i16> %a, <8 x i16> %b, <4 x i32> %c) {
+; CHECK-LABEL: test_vmaloh:
+; CHECK: vmaloh %v24, %v24, %v26, %v28
+; CHECK: br %r14
+  %res = call <4 x i32> @llvm.s390.vmaloh(<8 x i16> %a, <8 x i16> %b,
+                                          <4 x i32> %c)
+  ret <4 x i32> %res
+}
+
+; VMALOF.
+define <2 x i64> @test_vmalof(<4 x i32> %a, <4 x i32> %b, <2 x i64> %c) {
+; CHECK-LABEL: test_vmalof:
+; CHECK: vmalof %v24, %v24, %v26, %v28
+; CHECK: br %r14
+  %res = call <2 x i64> @llvm.s390.vmalof(<4 x i32> %a, <4 x i32> %b,
+                                          <2 x i64> %c)
+  ret <2 x i64> %res
+}
+
+; VMHB.
+define <16 x i8> @test_vmhb(<16 x i8> %a, <16 x i8> %b) {
+; CHECK-LABEL: test_vmhb:
+; CHECK: vmhb %v24, %v24, %v26
+; CHECK: br %r14
+  %res = call <16 x i8> @llvm.s390.vmhb(<16 x i8> %a, <16 x i8> %b)
+  ret <16 x i8> %res
+}
+
+; VMHH.
+define <8 x i16> @test_vmhh(<8 x i16> %a, <8 x i16> %b) {
+; CHECK-LABEL: test_vmhh:
+; CHECK: vmhh %v24, %v24, %v26
+; CHECK: br %r14
+  %res = call <8 x i16> @llvm.s390.vmhh(<8 x i16> %a, <8 x i16> %b)
+  ret <8 x i16> %res
+}
+
+; VMHF.
+define <4 x i32> @test_vmhf(<4 x i32> %a, <4 x i32> %b) {
+; CHECK-LABEL: test_vmhf:
+; CHECK: vmhf %v24, %v24, %v26
+; CHECK: br %r14
+  %res = call <4 x i32> @llvm.s390.vmhf(<4 x i32> %a, <4 x i32> %b)
+  ret <4 x i32> %res
+}
+
+; VMLHB.
+define <16 x i8> @test_vmlhb(<16 x i8> %a, <16 x i8> %b) {
+; CHECK-LABEL: test_vmlhb:
+; CHECK: vmlhb %v24, %v24, %v26
+; CHECK: br %r14
+  %res = call <16 x i8> @llvm.s390.vmlhb(<16 x i8> %a, <16 x i8> %b)
+  ret <16 x i8> %res
+}
+
+; VMLHH.
+define <8 x i16> @test_vmlhh(<8 x i16> %a, <8 x i16> %b) {
+; CHECK-LABEL: test_vmlhh:
+; CHECK: vmlhh %v24, %v24, %v26
+; CHECK: br %r14
+  %res = call <8 x i16> @llvm.s390.vmlhh(<8 x i16> %a, <8 x i16> %b)
+  ret <8 x i16> %res
+}
+
+; VMLHF.
+define <4 x i32> @test_vmlhf(<4 x i32> %a, <4 x i32> %b) {
+; CHECK-LABEL: test_vmlhf:
+; CHECK: vmlhf %v24, %v24, %v26
+; CHECK: br %r14
+  %res = call <4 x i32> @llvm.s390.vmlhf(<4 x i32> %a, <4 x i32> %b)
+  ret <4 x i32> %res
+}
+
+; VMEB.
+define <8 x i16> @test_vmeb(<16 x i8> %a, <16 x i8> %b) {
+; CHECK-LABEL: test_vmeb:
+; CHECK: vmeb %v24, %v24, %v26
+; CHECK: br %r14
+  %res = call <8 x i16> @llvm.s390.vmeb(<16 x i8> %a, <16 x i8> %b)
+  ret <8 x i16> %res
+}
+
+; VMEH.
+define <4 x i32> @test_vmeh(<8 x i16> %a, <8 x i16> %b) {
+; CHECK-LABEL: test_vmeh:
+; CHECK: vmeh %v24, %v24, %v26
+; CHECK: br %r14
+  %res = call <4 x i32> @llvm.s390.vmeh(<8 x i16> %a, <8 x i16> %b)
+  ret <4 x i32> %res
+}
+
+; VMEF.
+define <2 x i64> @test_vmef(<4 x i32> %a, <4 x i32> %b) {
+; CHECK-LABEL: test_vmef:
+; CHECK: vmef %v24, %v24, %v26
+; CHECK: br %r14
+  %res = call <2 x i64> @llvm.s390.vmef(<4 x i32> %a, <4 x i32> %b)
+  ret <2 x i64> %res
+}
+
+; VMLEB.
+define <8 x i16> @test_vmleb(<16 x i8> %a, <16 x i8> %b) {
+; CHECK-LABEL: test_vmleb:
+; CHECK: vmleb %v24, %v24, %v26
+; CHECK: br %r14
+  %res = call <8 x i16> @llvm.s390.vmleb(<16 x i8> %a, <16 x i8> %b)
+  ret <8 x i16> %res
+}
+
+; VMLEH.
+define <4 x i32> @test_vmleh(<8 x i16> %a, <8 x i16> %b) {
+; CHECK-LABEL: test_vmleh:
+; CHECK: vmleh %v24, %v24, %v26
+; CHECK: br %r14
+  %res = call <4 x i32> @llvm.s390.vmleh(<8 x i16> %a, <8 x i16> %b)
+  ret <4 x i32> %res
+}
+
+; VMLEF.
+define <2 x i64> @test_vmlef(<4 x i32> %a, <4 x i32> %b) {
+; CHECK-LABEL: test_vmlef:
+; CHECK: vmlef %v24, %v24, %v26
+; CHECK: br %r14
+  %res = call <2 x i64> @llvm.s390.vmlef(<4 x i32> %a, <4 x i32> %b)
+  ret <2 x i64> %res
+}
+
+; VMOB.
+define <8 x i16> @test_vmob(<16 x i8> %a, <16 x i8> %b) {
+; CHECK-LABEL: test_vmob:
+; CHECK: vmob %v24, %v24, %v26
+; CHECK: br %r14
+  %res = call <8 x i16> @llvm.s390.vmob(<16 x i8> %a, <16 x i8> %b)
+  ret <8 x i16> %res
+}
+
+; VMOH.
+define <4 x i32> @test_vmoh(<8 x i16> %a, <8 x i16> %b) {
+; CHECK-LABEL: test_vmoh:
+; CHECK: vmoh %v24, %v24, %v26
+; CHECK: br %r14
+  %res = call <4 x i32> @llvm.s390.vmoh(<8 x i16> %a, <8 x i16> %b)
+  ret <4 x i32> %res
+}
+
+; VMOF.
+define <2 x i64> @test_vmof(<4 x i32> %a, <4 x i32> %b) {
+; CHECK-LABEL: test_vmof:
+; CHECK: vmof %v24, %v24, %v26
+; CHECK: br %r14
+  %res = call <2 x i64> @llvm.s390.vmof(<4 x i32> %a, <4 x i32> %b)
+  ret <2 x i64> %res
+}
+
+; VMLOB.
+define <8 x i16> @test_vmlob(<16 x i8> %a, <16 x i8> %b) {
+; CHECK-LABEL: test_vmlob:
+; CHECK: vmlob %v24, %v24, %v26
+; CHECK: br %r14
+  %res = call <8 x i16> @llvm.s390.vmlob(<16 x i8> %a, <16 x i8> %b)
+  ret <8 x i16> %res
+}
+
+; VMLOH.
+define <4 x i32> @test_vmloh(<8 x i16> %a, <8 x i16> %b) {
+; CHECK-LABEL: test_vmloh:
+; CHECK: vmloh %v24, %v24, %v26
+; CHECK: br %r14
+  %res = call <4 x i32> @llvm.s390.vmloh(<8 x i16> %a, <8 x i16> %b)
+  ret <4 x i32> %res
+}
+
+; VMLOF.
+define <2 x i64> @test_vmlof(<4 x i32> %a, <4 x i32> %b) {
+; CHECK-LABEL: test_vmlof:
+; CHECK: vmlof %v24, %v24, %v26
+; CHECK: br %r14
+  %res = call <2 x i64> @llvm.s390.vmlof(<4 x i32> %a, <4 x i32> %b)
+  ret <2 x i64> %res
+}
+
+; VERLLVB.
+define <16 x i8> @test_verllvb(<16 x i8> %a, <16 x i8> %b) {
+; CHECK-LABEL: test_verllvb:
+; CHECK: verllvb %v24, %v24, %v26
+; CHECK: br %r14
+  %res = call <16 x i8> @llvm.s390.verllvb(<16 x i8> %a, <16 x i8> %b)
+  ret <16 x i8> %res
+}
+
+; VERLLVH.
+define <8 x i16> @test_verllvh(<8 x i16> %a, <8 x i16> %b) {
+; CHECK-LABEL: test_verllvh:
+; CHECK: verllvh %v24, %v24, %v26
+; CHECK: br %r14
+  %res = call <8 x i16> @llvm.s390.verllvh(<8 x i16> %a, <8 x i16> %b)
+  ret <8 x i16> %res
+}
+
+; VERLLVF.
+define <4 x i32> @test_verllvf(<4 x i32> %a, <4 x i32> %b) {
+; CHECK-LABEL: test_verllvf:
+; CHECK: verllvf %v24, %v24, %v26
+; CHECK: br %r14
+  %res = call <4 x i32> @llvm.s390.verllvf(<4 x i32> %a, <4 x i32> %b)
+  ret <4 x i32> %res
+}
+
+; VERLLVG.
+define <2 x i64> @test_verllvg(<2 x i64> %a, <2 x i64> %b) {
+; CHECK-LABEL: test_verllvg:
+; CHECK: verllvg %v24, %v24, %v26
+; CHECK: br %r14
+  %res = call <2 x i64> @llvm.s390.verllvg(<2 x i64> %a, <2 x i64> %b)
+  ret <2 x i64> %res
+}
+
+; VERLLB.
+define <16 x i8> @test_verllb(<16 x i8> %a, i32 %b) {
+; CHECK-LABEL: test_verllb:
+; CHECK: verllb %v24, %v24, 0(%r2)
+; CHECK: br %r14
+  %res = call <16 x i8> @llvm.s390.verllb(<16 x i8> %a, i32 %b)
+  ret <16 x i8> %res
+}
+
+; VERLLH.
+define <8 x i16> @test_verllh(<8 x i16> %a, i32 %b) {
+; CHECK-LABEL: test_verllh:
+; CHECK: verllh %v24, %v24, 0(%r2)
+; CHECK: br %r14
+  %res = call <8 x i16> @llvm.s390.verllh(<8 x i16> %a, i32 %b)
+  ret <8 x i16> %res
+}
+
+; VERLLF.
+define <4 x i32> @test_verllf(<4 x i32> %a, i32 %b) {
+; CHECK-LABEL: test_verllf:
+; CHECK: verllf %v24, %v24, 0(%r2)
+; CHECK: br %r14
+  %res = call <4 x i32> @llvm.s390.verllf(<4 x i32> %a, i32 %b)
+  ret <4 x i32> %res
+}
+
+; VERLLG.
+define <2 x i64> @test_verllg(<2 x i64> %a, i32 %b) {
+; CHECK-LABEL: test_verllg:
+; CHECK: verllg %v24, %v24, 0(%r2)
+; CHECK: br %r14
+  %res = call <2 x i64> @llvm.s390.verllg(<2 x i64> %a, i32 %b)
+  ret <2 x i64> %res
+}
+
+; VERLLB with the smallest count.
+define <16 x i8> @test_verllb_1(<16 x i8> %a) {
+; CHECK-LABEL: test_verllb_1:
+; CHECK: verllb %v24, %v24, 1
+; CHECK: br %r14
+  %res = call <16 x i8> @llvm.s390.verllb(<16 x i8> %a, i32 1)
+  ret <16 x i8> %res
+}
+
+; VERLLB with the largest count.
+define <16 x i8> @test_verllb_4095(<16 x i8> %a) {
+; CHECK-LABEL: test_verllb_4095:
+; CHECK: verllb %v24, %v24, 4095
+; CHECK: br %r14
+  %res = call <16 x i8> @llvm.s390.verllb(<16 x i8> %a, i32 4095)
+  ret <16 x i8> %res
+}
+
+; VERLLB with the largest count + 1.
+define <16 x i8> @test_verllb_4096(<16 x i8> %a) {
+; CHECK-LABEL: test_verllb_4096:
+; CHECK: lhi [[REG:%r[1-5]]], 4096
+; CHECK: verllb %v24, %v24, 0([[REG]])
+; CHECK: br %r14
+  %res = call <16 x i8> @llvm.s390.verllb(<16 x i8> %a, i32 4096)
+  ret <16 x i8> %res
+}
+
+; VERIMB.
+define <16 x i8> @test_verimb(<16 x i8> %a, <16 x i8> %b, <16 x i8> %c) {
+; CHECK-LABEL: test_verimb:
+; CHECK: verimb %v24, %v26, %v28, 1
+; CHECK: br %r14
+  %res = call <16 x i8> @llvm.s390.verimb(<16 x i8> %a, <16 x i8> %b, <16 x i8> %c, i32 1)
+  ret <16 x i8> %res
+}
+
+; VERIMH.
+define <8 x i16> @test_verimh(<8 x i16> %a, <8 x i16> %b, <8 x i16> %c) {
+; CHECK-LABEL: test_verimh:
+; CHECK: verimh %v24, %v26, %v28, 1
+; CHECK: br %r14
+  %res = call <8 x i16> @llvm.s390.verimh(<8 x i16> %a, <8 x i16> %b, <8 x i16> %c, i32 1)
+  ret <8 x i16> %res
+}
+
+; VERIMF.
+define <4 x i32> @test_verimf(<4 x i32> %a, <4 x i32> %b, <4 x i32> %c) {
+; CHECK-LABEL: test_verimf:
+; CHECK: verimf %v24, %v26, %v28, 1
+; CHECK: br %r14
+  %res = call <4 x i32> @llvm.s390.verimf(<4 x i32> %a, <4 x i32> %b, <4 x i32> %c, i32 1)
+  ret <4 x i32> %res
+}
+
+; VERIMG.
+define <2 x i64> @test_verimg(<2 x i64> %a, <2 x i64> %b, <2 x i64> %c) {
+; CHECK-LABEL: test_verimg:
+; CHECK: verimg %v24, %v26, %v28, 1
+; CHECK: br %r14
+  %res = call <2 x i64> @llvm.s390.verimg(<2 x i64> %a, <2 x i64> %b, <2 x i64> %c, i32 1)
+  ret <2 x i64> %res
+}
+
+; VERIMB with a different mask.
+define <16 x i8> @test_verimb_254(<16 x i8> %a, <16 x i8> %b, <16 x i8> %c) {
+; CHECK-LABEL: test_verimb_254:
+; CHECK: verimb %v24, %v26, %v28, 254
+; CHECK: br %r14
+  %res = call <16 x i8> @llvm.s390.verimb(<16 x i8> %a, <16 x i8> %b, <16 x i8> %c, i32 254)
+  ret <16 x i8> %res
+}
+
+; VSL.
+define <16 x i8> @test_vsl(<16 x i8> %a, <16 x i8> %b) {
+; CHECK-LABEL: test_vsl:
+; CHECK: vsl %v24, %v24, %v26
+; CHECK: br %r14
+  %res = call <16 x i8> @llvm.s390.vsl(<16 x i8> %a, <16 x i8> %b)
+  ret <16 x i8> %res
+}
+
+; VSLB.
+define <16 x i8> @test_vslb(<16 x i8> %a, <16 x i8> %b) {
+; CHECK-LABEL: test_vslb:
+; CHECK: vslb %v24, %v24, %v26
+; CHECK: br %r14
+  %res = call <16 x i8> @llvm.s390.vslb(<16 x i8> %a, <16 x i8> %b)
+  ret <16 x i8> %res
+}
+
+; VSRA.
+define <16 x i8> @test_vsra(<16 x i8> %a, <16 x i8> %b) {
+; CHECK-LABEL: test_vsra:
+; CHECK: vsra %v24, %v24, %v26
+; CHECK: br %r14
+  %res = call <16 x i8> @llvm.s390.vsra(<16 x i8> %a, <16 x i8> %b)
+  ret <16 x i8> %res
+}
+
+; VSRAB.
+define <16 x i8> @test_vsrab(<16 x i8> %a, <16 x i8> %b) {
+; CHECK-LABEL: test_vsrab:
+; CHECK: vsrab %v24, %v24, %v26
+; CHECK: br %r14
+  %res = call <16 x i8> @llvm.s390.vsrab(<16 x i8> %a, <16 x i8> %b)
+  ret <16 x i8> %res
+}
+
+; VSRL.
+define <16 x i8> @test_vsrl(<16 x i8> %a, <16 x i8> %b) {
+; CHECK-LABEL: test_vsrl:
+; CHECK: vsrl %v24, %v24, %v26
+; CHECK: br %r14
+  %res = call <16 x i8> @llvm.s390.vsrl(<16 x i8> %a, <16 x i8> %b)
+  ret <16 x i8> %res
+}
+
+; VSRLB.
+define <16 x i8> @test_vsrlb(<16 x i8> %a, <16 x i8> %b) {
+; CHECK-LABEL: test_vsrlb:
+; CHECK: vsrlb %v24, %v24, %v26
+; CHECK: br %r14
+  %res = call <16 x i8> @llvm.s390.vsrlb(<16 x i8> %a, <16 x i8> %b)
+  ret <16 x i8> %res
+}
+
+; VSLDB with the minimum useful value.
+define <16 x i8> @test_vsldb_1(<16 x i8> %a, <16 x i8> %b) {
+; CHECK-LABEL: test_vsldb_1:
+; CHECK: vsldb %v24, %v24, %v26, 1
+; CHECK: br %r14
+  %res = call <16 x i8> @llvm.s390.vsldb(<16 x i8> %a, <16 x i8> %b, i32 1)
+  ret <16 x i8> %res
+}
+
+; VSLDB with the maximum value.
+define <16 x i8> @test_vsldb_15(<16 x i8> %a, <16 x i8> %b) {
+; CHECK-LABEL: test_vsldb_15:
+; CHECK: vsldb %v24, %v24, %v26, 15
+; CHECK: br %r14
+  %res = call <16 x i8> @llvm.s390.vsldb(<16 x i8> %a, <16 x i8> %b, i32 15)
+  ret <16 x i8> %res
+}
+
+; VSCBIB.
+define <16 x i8> @test_vscbib(<16 x i8> %a, <16 x i8> %b) {
+; CHECK-LABEL: test_vscbib:
+; CHECK: vscbib %v24, %v24, %v26
+; CHECK: br %r14
+  %res = call <16 x i8> @llvm.s390.vscbib(<16 x i8> %a, <16 x i8> %b)
+  ret <16 x i8> %res
+}
+
+; VSCBIH.
+define <8 x i16> @test_vscbih(<8 x i16> %a, <8 x i16> %b) {
+; CHECK-LABEL: test_vscbih:
+; CHECK: vscbih %v24, %v24, %v26
+; CHECK: br %r14
+  %res = call <8 x i16> @llvm.s390.vscbih(<8 x i16> %a, <8 x i16> %b)
+  ret <8 x i16> %res
+}
+
+; VSCBIF.
+define <4 x i32> @test_vscbif(<4 x i32> %a, <4 x i32> %b) {
+; CHECK-LABEL: test_vscbif:
+; CHECK: vscbif %v24, %v24, %v26
+; CHECK: br %r14
+  %res = call <4 x i32> @llvm.s390.vscbif(<4 x i32> %a, <4 x i32> %b)
+  ret <4 x i32> %res
+}
+
+; VSCBIG.
+define <2 x i64> @test_vscbig(<2 x i64> %a, <2 x i64> %b) {
+; CHECK-LABEL: test_vscbig:
+; CHECK: vscbig %v24, %v24, %v26
+; CHECK: br %r14
+  %res = call <2 x i64> @llvm.s390.vscbig(<2 x i64> %a, <2 x i64> %b)
+  ret <2 x i64> %res
+}
+
+; VSQ.
+define <16 x i8> @test_vsq(<16 x i8> %a, <16 x i8> %b) {
+; CHECK-LABEL: test_vsq:
+; CHECK: vsq %v24, %v24, %v26
+; CHECK: br %r14
+  %res = call <16 x i8> @llvm.s390.vsq(<16 x i8> %a, <16 x i8> %b)
+  ret <16 x i8> %res
+}
+
+; VSBIQ.
+define <16 x i8> @test_vsbiq(<16 x i8> %a, <16 x i8> %b, <16 x i8> %c) {
+; CHECK-LABEL: test_vsbiq:
+; CHECK: vsbiq %v24, %v24, %v26, %v28
+; CHECK: br %r14
+  %res = call <16 x i8> @llvm.s390.vsbiq(<16 x i8> %a, <16 x i8> %b,
+                                         <16 x i8> %c)
+  ret <16 x i8> %res
+}
+
+; VSCBIQ.
+define <16 x i8> @test_vscbiq(<16 x i8> %a, <16 x i8> %b) {
+; CHECK-LABEL: test_vscbiq:
+; CHECK: vscbiq %v24, %v24, %v26
+; CHECK: br %r14
+  %res = call <16 x i8> @llvm.s390.vscbiq(<16 x i8> %a, <16 x i8> %b)
+  ret <16 x i8> %res
+}
+
+; VSBCBIQ.
+define <16 x i8> @test_vsbcbiq(<16 x i8> %a, <16 x i8> %b, <16 x i8> %c) {
+; CHECK-LABEL: test_vsbcbiq:
+; CHECK: vsbcbiq %v24, %v24, %v26, %v28
+; CHECK: br %r14
+  %res = call <16 x i8> @llvm.s390.vsbcbiq(<16 x i8> %a, <16 x i8> %b,
+                                           <16 x i8> %c)
+  ret <16 x i8> %res
+}
+
+; VSUMB.
+define <4 x i32> @test_vsumb(<16 x i8> %a, <16 x i8> %b) {
+; CHECK-LABEL: test_vsumb:
+; CHECK: vsumb %v24, %v24, %v26
+; CHECK: br %r14
+  %res = call <4 x i32> @llvm.s390.vsumb(<16 x i8> %a, <16 x i8> %b)
+  ret <4 x i32> %res
+}
+
+; VSUMH.
+define <4 x i32> @test_vsumh(<8 x i16> %a, <8 x i16> %b) {
+; CHECK-LABEL: test_vsumh:
+; CHECK: vsumh %v24, %v24, %v26
+; CHECK: br %r14
+  %res = call <4 x i32> @llvm.s390.vsumh(<8 x i16> %a, <8 x i16> %b)
+  ret <4 x i32> %res
+}
+
+; VSUMGH.
+define <2 x i64> @test_vsumgh(<8 x i16> %a, <8 x i16> %b) {
+; CHECK-LABEL: test_vsumgh:
+; CHECK: vsumgh %v24, %v24, %v26
+; CHECK: br %r14
+  %res = call <2 x i64> @llvm.s390.vsumgh(<8 x i16> %a, <8 x i16> %b)
+  ret <2 x i64> %res
+}
+
+; VSUMGF.
+define <2 x i64> @test_vsumgf(<4 x i32> %a, <4 x i32> %b) {
+; CHECK-LABEL: test_vsumgf:
+; CHECK: vsumgf %v24, %v24, %v26
+; CHECK: br %r14
+  %res = call <2 x i64> @llvm.s390.vsumgf(<4 x i32> %a, <4 x i32> %b)
+  ret <2 x i64> %res
+}
+
+; VSUMQF.
+define <16 x i8> @test_vsumqf(<4 x i32> %a, <4 x i32> %b) {
+; CHECK-LABEL: test_vsumqf:
+; CHECK: vsumqf %v24, %v24, %v26
+; CHECK: br %r14
+  %res = call <16 x i8> @llvm.s390.vsumqf(<4 x i32> %a, <4 x i32> %b)
+  ret <16 x i8> %res
+}
+
+; VSUMQG.
+define <16 x i8> @test_vsumqg(<2 x i64> %a, <2 x i64> %b) {
+; CHECK-LABEL: test_vsumqg:
+; CHECK: vsumqg %v24, %v24, %v26
+; CHECK: br %r14
+  %res = call <16 x i8> @llvm.s390.vsumqg(<2 x i64> %a, <2 x i64> %b)
+  ret <16 x i8> %res
+}
+
+; VTM with no processing of the result.
+define i32 @test_vtm(<16 x i8> %a, <16 x i8> %b) {
+; CHECK-LABEL: test_vtm:
+; CHECK: vtm %v24, %v26
+; CHECK: ipm %r2
+; CHECK: srl %r2, 28
+; CHECK: br %r14
+  %res = call i32 @llvm.s390.vtm(<16 x i8> %a, <16 x i8> %b)
+  ret i32 %res
+}
+
+; VTM, storing to %ptr if all bits are set.
+define void @test_vtm_all_store(<16 x i8> %a, <16 x i8> %b, i32 *%ptr) {
+; CHECK-LABEL: test_vtm_all_store:
+; CHECK-NOT: %r
+; CHECK: vtm %v24, %v26
+; CHECK-NEXT: {{jno|jle}} {{\.L*}}
+; CHECK: mvhi 0(%r2), 0
+; CHECK: br %r14
+  %res = call i32 @llvm.s390.vtm(<16 x i8> %a, <16 x i8> %b)
+  %cmp = icmp sge i32 %res, 3
+  br i1 %cmp, label %store, label %exit
+
+store:
+  store i32 0, i32 *%ptr
+  br label %exit
+
+exit:
+  ret void
+}
+
+; VCEQBS with no processing of the result.
+define i32 @test_vceqbs(<16 x i8> %a, <16 x i8> %b) {
+; CHECK-LABEL: test_vceqbs:
+; CHECK: vceqbs {{%v[0-9]+}}, %v24, %v26
+; CHECK: ipm %r2
+; CHECK: srl %r2, 28
+; CHECK: br %r14
+  %call = call {<16 x i8>, i32} @llvm.s390.vceqbs(<16 x i8> %a, <16 x i8> %b)
+  %res = extractvalue {<16 x i8>, i32} %call, 1
+  ret i32 %res
+}
+
+; VCEQBS, returning 1 if any elements are equal (CC != 3).
+define i32 @test_vceqbs_any_bool(<16 x i8> %a, <16 x i8> %b) {
+; CHECK-LABEL: test_vceqbs_any_bool:
+; CHECK: vceqbs {{%v[0-9]+}}, %v24, %v26
+; CHECK: ipm %r2
+; CHECK: afi %r2, -536870912
+; CHECK: srl %r2, 31
+; CHECK: br %r14
+  %call = call {<16 x i8>, i32} @llvm.s390.vceqbs(<16 x i8> %a, <16 x i8> %b)
+  %res = extractvalue {<16 x i8>, i32} %call, 1
+  %cmp = icmp ne i32 %res, 3
+  %ext = zext i1 %cmp to i32
+  ret i32 %ext
+}
+
+; VCEQBS, storing to %ptr if any elements are equal.
+define <16 x i8> @test_vceqbs_any_store(<16 x i8> %a, <16 x i8> %b, i32 *%ptr) {
+; CHECK-LABEL: test_vceqbs_any_store:
+; CHECK-NOT: %r
+; CHECK: vceqbs %v24, %v24, %v26
+; CHECK-NEXT: {{jo|jnle}} {{\.L*}}
+; CHECK: mvhi 0(%r2), 0
+; CHECK: br %r14
+  %call = call {<16 x i8>, i32} @llvm.s390.vceqbs(<16 x i8> %a, <16 x i8> %b)
+  %res = extractvalue {<16 x i8>, i32} %call, 0
+  %cc = extractvalue {<16 x i8>, i32} %call, 1
+  %cmp = icmp ule i32 %cc, 2
+  br i1 %cmp, label %store, label %exit
+
+store:
+  store i32 0, i32 *%ptr
+  br label %exit
+
+exit:
+  ret <16 x i8> %res
+}
+
+; VCEQHS with no processing of the result.
+define i32 @test_vceqhs(<8 x i16> %a, <8 x i16> %b) {
+; CHECK-LABEL: test_vceqhs:
+; CHECK: vceqhs {{%v[0-9]+}}, %v24, %v26
+; CHECK: ipm %r2
+; CHECK: srl %r2, 28
+; CHECK: br %r14
+  %call = call {<8 x i16>, i32} @llvm.s390.vceqhs(<8 x i16> %a, <8 x i16> %b)
+  %res = extractvalue {<8 x i16>, i32} %call, 1
+  ret i32 %res
+}
+
+; VCEQHS, returning 1 if not all elements are equal.
+define i32 @test_vceqhs_notall_bool(<8 x i16> %a, <8 x i16> %b) {
+; CHECK-LABEL: test_vceqhs_notall_bool:
+; CHECK: vceqhs {{%v[0-9]+}}, %v24, %v26
+; CHECK: ipm [[REG:%r[0-5]]]
+; CHECK: risblg %r2, [[REG]], 31, 159, 36
+; CHECK: br %r14
+  %call = call {<8 x i16>, i32} @llvm.s390.vceqhs(<8 x i16> %a, <8 x i16> %b)
+  %res = extractvalue {<8 x i16>, i32} %call, 1
+  %cmp = icmp sge i32 %res, 1
+  %ext = zext i1 %cmp to i32
+  ret i32 %ext
+}
+
+; VCEQHS, storing to %ptr if not all elements are equal.
+define <8 x i16> @test_vceqhs_notall_store(<8 x i16> %a, <8 x i16> %b,
+                                           i32 *%ptr) {
+; CHECK-LABEL: test_vceqhs_notall_store:
+; CHECK-NOT: %r
+; CHECK: vceqhs %v24, %v24, %v26
+; CHECK-NEXT: {{jhe|je}} {{\.L*}}
+; CHECK: mvhi 0(%r2), 0
+; CHECK: br %r14
+  %call = call {<8 x i16>, i32} @llvm.s390.vceqhs(<8 x i16> %a, <8 x i16> %b)
+  %res = extractvalue {<8 x i16>, i32} %call, 0
+  %cc = extractvalue {<8 x i16>, i32} %call, 1
+  %cmp = icmp ugt i32 %cc, 0
+  br i1 %cmp, label %store, label %exit
+
+store:
+  store i32 0, i32 *%ptr
+  br label %exit
+
+exit:
+  ret <8 x i16> %res
+}
+
+; VCEQFS with no processing of the result.
+define i32 @test_vceqfs(<4 x i32> %a, <4 x i32> %b) {
+; CHECK-LABEL: test_vceqfs:
+; CHECK: vceqfs {{%v[0-9]+}}, %v24, %v26
+; CHECK: ipm %r2
+; CHECK: srl %r2, 28
+; CHECK: br %r14
+  %call = call {<4 x i32>, i32} @llvm.s390.vceqfs(<4 x i32> %a, <4 x i32> %b)
+  %res = extractvalue {<4 x i32>, i32} %call, 1
+  ret i32 %res
+}
+
+; VCEQFS, returning 1 if no elements are equal.
+define i32 @test_vceqfs_none_bool(<4 x i32> %a, <4 x i32> %b) {
+; CHECK-LABEL: test_vceqfs_none_bool:
+; CHECK: vceqfs {{%v[0-9]+}}, %v24, %v26
+; CHECK: ipm [[REG:%r[0-5]]]
+; CHECK: risblg %r2, [[REG]], 31, 159, 35
+; CHECK: br %r14
+  %call = call {<4 x i32>, i32} @llvm.s390.vceqfs(<4 x i32> %a, <4 x i32> %b)
+  %res = extractvalue {<4 x i32>, i32} %call, 1
+  %cmp = icmp eq i32 %res, 3
+  %ext = zext i1 %cmp to i32
+  ret i32 %ext
+}
+
+; VCEQFS, storing to %ptr if no elements are equal.
+define <4 x i32> @test_vceqfs_none_store(<4 x i32> %a, <4 x i32> %b,
+                                         i32 *%ptr) {
+; CHECK-LABEL: test_vceqfs_none_store:
+; CHECK-NOT: %r
+; CHECK: vceqfs %v24, %v24, %v26
+; CHECK-NEXT: {{jno|jle}} {{\.L*}}
+; CHECK: mvhi 0(%r2), 0
+; CHECK: br %r14
+  %call = call {<4 x i32>, i32} @llvm.s390.vceqfs(<4 x i32> %a, <4 x i32> %b)
+  %res = extractvalue {<4 x i32>, i32} %call, 0
+  %cc = extractvalue {<4 x i32>, i32} %call, 1
+  %cmp = icmp uge i32 %cc, 3
+  br i1 %cmp, label %store, label %exit
+
+store:
+  store i32 0, i32 *%ptr
+  br label %exit
+
+exit:
+  ret <4 x i32> %res
+}
+
+; VCEQGS with no processing of the result.
+define i32 @test_vceqgs(<2 x i64> %a, <2 x i64> %b) {
+; CHECK-LABEL: test_vceqgs:
+; CHECK: vceqgs {{%v[0-9]+}}, %v24, %v26
+; CHECK: ipm %r2
+; CHECK: srl %r2, 28
+; CHECK: br %r14
+  %call = call {<2 x i64>, i32} @llvm.s390.vceqgs(<2 x i64> %a, <2 x i64> %b)
+  %res = extractvalue {<2 x i64>, i32} %call, 1
+  ret i32 %res
+}
+
+; VCEQGS returning 1 if all elements are equal (CC == 0).
+define i32 @test_vceqgs_all_bool(<2 x i64> %a, <2 x i64> %b) {
+; CHECK-LABEL: test_vceqgs_all_bool:
+; CHECK: vceqgs {{%v[0-9]+}}, %v24, %v26
+; CHECK: ipm %r2
+; CHECK: afi %r2, -268435456
+; CHECK: srl %r2, 31
+; CHECK: br %r14
+  %call = call {<2 x i64>, i32} @llvm.s390.vceqgs(<2 x i64> %a, <2 x i64> %b)
+  %res = extractvalue {<2 x i64>, i32} %call, 1
+  %cmp = icmp ult i32 %res, 1
+  %ext = zext i1 %cmp to i32
+  ret i32 %ext
+}
+
+; VCEQGS, storing to %ptr if all elements are equal.
+define <2 x i64> @test_vceqgs_all_store(<2 x i64> %a, <2 x i64> %b, i32 *%ptr) {
+; CHECK-LABEL: test_vceqgs_all_store:
+; CHECK-NOT: %r
+; CHECK: vceqgs %v24, %v24, %v26
+; CHECK-NEXT: {{jnhe|jne}} {{\.L*}}
+; CHECK: mvhi 0(%r2), 0
+; CHECK: br %r14
+  %call = call {<2 x i64>, i32} @llvm.s390.vceqgs(<2 x i64> %a, <2 x i64> %b)
+  %res = extractvalue {<2 x i64>, i32} %call, 0
+  %cc = extractvalue {<2 x i64>, i32} %call, 1
+  %cmp = icmp sle i32 %cc, 0
+  br i1 %cmp, label %store, label %exit
+
+store:
+  store i32 0, i32 *%ptr
+  br label %exit
+
+exit:
+  ret <2 x i64> %res
+}
+
+; VCHBS with no processing of the result.
+define i32 @test_vchbs(<16 x i8> %a, <16 x i8> %b) {
+; CHECK-LABEL: test_vchbs:
+; CHECK: vchbs {{%v[0-9]+}}, %v24, %v26
+; CHECK: ipm %r2
+; CHECK: srl %r2, 28
+; CHECK: br %r14
+  %call = call {<16 x i8>, i32} @llvm.s390.vchbs(<16 x i8> %a, <16 x i8> %b)
+  %res = extractvalue {<16 x i8>, i32} %call, 1
+  ret i32 %res
+}
+
+; VCHBS, returning 1 if any elements are higher (CC != 3).
+define i32 @test_vchbs_any_bool(<16 x i8> %a, <16 x i8> %b) {
+; CHECK-LABEL: test_vchbs_any_bool:
+; CHECK: vchbs {{%v[0-9]+}}, %v24, %v26
+; CHECK: ipm %r2
+; CHECK: afi %r2, -536870912
+; CHECK: srl %r2, 31
+; CHECK: br %r14
+  %call = call {<16 x i8>, i32} @llvm.s390.vchbs(<16 x i8> %a, <16 x i8> %b)
+  %res = extractvalue {<16 x i8>, i32} %call, 1
+  %cmp = icmp ne i32 %res, 3
+  %ext = zext i1 %cmp to i32
+  ret i32 %ext
+}
+
+; VCHBS, storing to %ptr if any elements are higher.
+define <16 x i8> @test_vchbs_any_store(<16 x i8> %a, <16 x i8> %b, i32 *%ptr) {
+; CHECK-LABEL: test_vchbs_any_store:
+; CHECK-NOT: %r
+; CHECK: vchbs %v24, %v24, %v26
+; CHECK-NEXT: {{jo|jnle}} {{\.L*}}
+; CHECK: mvhi 0(%r2), 0
+; CHECK: br %r14
+  %call = call {<16 x i8>, i32} @llvm.s390.vchbs(<16 x i8> %a, <16 x i8> %b)
+  %res = extractvalue {<16 x i8>, i32} %call, 0
+  %cc = extractvalue {<16 x i8>, i32} %call, 1
+  %cmp = icmp ule i32 %cc, 2
+  br i1 %cmp, label %store, label %exit
+
+store:
+  store i32 0, i32 *%ptr
+  br label %exit
+
+exit:
+  ret <16 x i8> %res
+}
+
+; VCHHS with no processing of the result.
+define i32 @test_vchhs(<8 x i16> %a, <8 x i16> %b) {
+; CHECK-LABEL: test_vchhs:
+; CHECK: vchhs {{%v[0-9]+}}, %v24, %v26
+; CHECK: ipm %r2
+; CHECK: srl %r2, 28
+; CHECK: br %r14
+  %call = call {<8 x i16>, i32} @llvm.s390.vchhs(<8 x i16> %a, <8 x i16> %b)
+  %res = extractvalue {<8 x i16>, i32} %call, 1
+  ret i32 %res
+}
+
+; VCHHS, returning 1 if not all elements are higher.
+define i32 @test_vchhs_notall_bool(<8 x i16> %a, <8 x i16> %b) {
+; CHECK-LABEL: test_vchhs_notall_bool:
+; CHECK: vchhs {{%v[0-9]+}}, %v24, %v26
+; CHECK: ipm [[REG:%r[0-5]]]
+; CHECK: risblg %r2, [[REG]], 31, 159, 36
+; CHECK: br %r14
+  %call = call {<8 x i16>, i32} @llvm.s390.vchhs(<8 x i16> %a, <8 x i16> %b)
+  %res = extractvalue {<8 x i16>, i32} %call, 1
+  %cmp = icmp sge i32 %res, 1
+  %ext = zext i1 %cmp to i32
+  ret i32 %ext
+}
+
+; VCHHS, storing to %ptr if not all elements are higher.
+define <8 x i16> @test_vchhs_notall_store(<8 x i16> %a, <8 x i16> %b,
+                                          i32 *%ptr) {
+; CHECK-LABEL: test_vchhs_notall_store:
+; CHECK-NOT: %r
+; CHECK: vchhs %v24, %v24, %v26
+; CHECK-NEXT: {{jhe|je}} {{\.L*}}
+; CHECK: mvhi 0(%r2), 0
+; CHECK: br %r14
+  %call = call {<8 x i16>, i32} @llvm.s390.vchhs(<8 x i16> %a, <8 x i16> %b)
+  %res = extractvalue {<8 x i16>, i32} %call, 0
+  %cc = extractvalue {<8 x i16>, i32} %call, 1
+  %cmp = icmp ugt i32 %cc, 0
+  br i1 %cmp, label %store, label %exit
+
+store:
+  store i32 0, i32 *%ptr
+  br label %exit
+
+exit:
+  ret <8 x i16> %res
+}
+
+; VCHFS with no processing of the result.
+define i32 @test_vchfs(<4 x i32> %a, <4 x i32> %b) {
+; CHECK-LABEL: test_vchfs:
+; CHECK: vchfs {{%v[0-9]+}}, %v24, %v26
+; CHECK: ipm %r2
+; CHECK: srl %r2, 28
+; CHECK: br %r14
+  %call = call {<4 x i32>, i32} @llvm.s390.vchfs(<4 x i32> %a, <4 x i32> %b)
+  %res = extractvalue {<4 x i32>, i32} %call, 1
+  ret i32 %res
+}
+
+; VCHFS, returning 1 if no elements are higher.
+define i32 @test_vchfs_none_bool(<4 x i32> %a, <4 x i32> %b) {
+; CHECK-LABEL: test_vchfs_none_bool:
+; CHECK: vchfs {{%v[0-9]+}}, %v24, %v26
+; CHECK: ipm [[REG:%r[0-5]]]
+; CHECK: risblg %r2, [[REG]], 31, 159, 35
+; CHECK: br %r14
+  %call = call {<4 x i32>, i32} @llvm.s390.vchfs(<4 x i32> %a, <4 x i32> %b)
+  %res = extractvalue {<4 x i32>, i32} %call, 1
+  %cmp = icmp eq i32 %res, 3
+  %ext = zext i1 %cmp to i32
+  ret i32 %ext
+}
+
+; VCHFS, storing to %ptr if no elements are higher.
+define <4 x i32> @test_vchfs_none_store(<4 x i32> %a, <4 x i32> %b, i32 *%ptr) {
+; CHECK-LABEL: test_vchfs_none_store:
+; CHECK-NOT: %r
+; CHECK: vchfs %v24, %v24, %v26
+; CHECK-NEXT: {{jno|jle}} {{\.L*}}
+; CHECK: mvhi 0(%r2), 0
+; CHECK: br %r14
+  %call = call {<4 x i32>, i32} @llvm.s390.vchfs(<4 x i32> %a, <4 x i32> %b)
+  %res = extractvalue {<4 x i32>, i32} %call, 0
+  %cc = extractvalue {<4 x i32>, i32} %call, 1
+  %cmp = icmp uge i32 %cc, 3
+  br i1 %cmp, label %store, label %exit
+
+store:
+  store i32 0, i32 *%ptr
+  br label %exit
+
+exit:
+  ret <4 x i32> %res
+}
+
+; VCHGS with no processing of the result.
+define i32 @test_vchgs(<2 x i64> %a, <2 x i64> %b) {
+; CHECK-LABEL: test_vchgs:
+; CHECK: vchgs {{%v[0-9]+}}, %v24, %v26
+; CHECK: ipm %r2
+; CHECK: srl %r2, 28
+; CHECK: br %r14
+  %call = call {<2 x i64>, i32} @llvm.s390.vchgs(<2 x i64> %a, <2 x i64> %b)
+  %res = extractvalue {<2 x i64>, i32} %call, 1
+  ret i32 %res
+}
+
+; VCHGS returning 1 if all elements are higher (CC == 0).
+define i32 @test_vchgs_all_bool(<2 x i64> %a, <2 x i64> %b) {
+; CHECK-LABEL: test_vchgs_all_bool:
+; CHECK: vchgs {{%v[0-9]+}}, %v24, %v26
+; CHECK: ipm %r2
+; CHECK: afi %r2, -268435456
+; CHECK: srl %r2, 31
+; CHECK: br %r14
+  %call = call {<2 x i64>, i32} @llvm.s390.vchgs(<2 x i64> %a, <2 x i64> %b)
+  %res = extractvalue {<2 x i64>, i32} %call, 1
+  %cmp = icmp ult i32 %res, 1
+  %ext = zext i1 %cmp to i32
+  ret i32 %ext
+}
+
+; VCHGS, storing to %ptr if all elements are higher.
+define <2 x i64> @test_vchgs_all_store(<2 x i64> %a, <2 x i64> %b, i32 *%ptr) {
+; CHECK-LABEL: test_vchgs_all_store:
+; CHECK-NOT: %r
+; CHECK: vchgs %v24, %v24, %v26
+; CHECK-NEXT: {{jnhe|jne}} {{\.L*}}
+; CHECK: mvhi 0(%r2), 0
+; CHECK: br %r14
+  %call = call {<2 x i64>, i32} @llvm.s390.vchgs(<2 x i64> %a, <2 x i64> %b)
+  %res = extractvalue {<2 x i64>, i32} %call, 0
+  %cc = extractvalue {<2 x i64>, i32} %call, 1
+  %cmp = icmp sle i32 %cc, 0
+  br i1 %cmp, label %store, label %exit
+
+store:
+  store i32 0, i32 *%ptr
+  br label %exit
+
+exit:
+  ret <2 x i64> %res
+}
+
+; VCHLBS with no processing of the result.
+define i32 @test_vchlbs(<16 x i8> %a, <16 x i8> %b) {
+; CHECK-LABEL: test_vchlbs:
+; CHECK: vchlbs {{%v[0-9]+}}, %v24, %v26
+; CHECK: ipm %r2
+; CHECK: srl %r2, 28
+; CHECK: br %r14
+  %call = call {<16 x i8>, i32} @llvm.s390.vchlbs(<16 x i8> %a, <16 x i8> %b)
+  %res = extractvalue {<16 x i8>, i32} %call, 1
+  ret i32 %res
+}
+
+; VCHLBS, returning 1 if any elements are higher (CC != 3).
+define i32 @test_vchlbs_any_bool(<16 x i8> %a, <16 x i8> %b) {
+; CHECK-LABEL: test_vchlbs_any_bool:
+; CHECK: vchlbs {{%v[0-9]+}}, %v24, %v26
+; CHECK: ipm %r2
+; CHECK: afi %r2, -536870912
+; CHECK: srl %r2, 31
+; CHECK: br %r14
+  %call = call {<16 x i8>, i32} @llvm.s390.vchlbs(<16 x i8> %a, <16 x i8> %b)
+  %res = extractvalue {<16 x i8>, i32} %call, 1
+  %cmp = icmp ne i32 %res, 3
+  %ext = zext i1 %cmp to i32
+  ret i32 %ext
+}
+
+; VCHLBS, storing to %ptr if any elements are higher.
+define <16 x i8> @test_vchlbs_any_store(<16 x i8> %a, <16 x i8> %b, i32 *%ptr) {
+; CHECK-LABEL: test_vchlbs_any_store:
+; CHECK-NOT: %r
+; CHECK: vchlbs %v24, %v24, %v26
+; CHECK-NEXT: {{jo|jnle}} {{\.L*}}
+; CHECK: mvhi 0(%r2), 0
+; CHECK: br %r14
+  %call = call {<16 x i8>, i32} @llvm.s390.vchlbs(<16 x i8> %a, <16 x i8> %b)
+  %res = extractvalue {<16 x i8>, i32} %call, 0
+  %cc = extractvalue {<16 x i8>, i32} %call, 1
+  %cmp = icmp sle i32 %cc, 2
+  br i1 %cmp, label %store, label %exit
+
+store:
+  store i32 0, i32 *%ptr
+  br label %exit
+
+exit:
+  ret <16 x i8> %res
+}
+
+; VCHLHS with no processing of the result.
+define i32 @test_vchlhs(<8 x i16> %a, <8 x i16> %b) {
+; CHECK-LABEL: test_vchlhs:
+; CHECK: vchlhs {{%v[0-9]+}}, %v24, %v26
+; CHECK: ipm %r2
+; CHECK: srl %r2, 28
+; CHECK: br %r14
+  %call = call {<8 x i16>, i32} @llvm.s390.vchlhs(<8 x i16> %a, <8 x i16> %b)
+  %res = extractvalue {<8 x i16>, i32} %call, 1
+  ret i32 %res
+}
+
+; VCHLHS, returning 1 if not all elements are higher.
+define i32 @test_vchlhs_notall_bool(<8 x i16> %a, <8 x i16> %b) {
+; CHECK-LABEL: test_vchlhs_notall_bool:
+; CHECK: vchlhs {{%v[0-9]+}}, %v24, %v26
+; CHECK: ipm [[REG:%r[0-5]]]
+; CHECK: risblg %r2, [[REG]], 31, 159, 36
+; CHECK: br %r14
+  %call = call {<8 x i16>, i32} @llvm.s390.vchlhs(<8 x i16> %a, <8 x i16> %b)
+  %res = extractvalue {<8 x i16>, i32} %call, 1
+  %cmp = icmp uge i32 %res, 1
+  %ext = zext i1 %cmp to i32
+  ret i32 %ext
+}
+
+; VCHLHS, storing to %ptr if not all elements are higher.
+define <8 x i16> @test_vchlhs_notall_store(<8 x i16> %a, <8 x i16> %b,
+                                           i32 *%ptr) {
+; CHECK-LABEL: test_vchlhs_notall_store:
+; CHECK-NOT: %r
+; CHECK: vchlhs %v24, %v24, %v26
+; CHECK-NEXT: {{jhe|je}} {{\.L*}}
+; CHECK: mvhi 0(%r2), 0
+; CHECK: br %r14
+  %call = call {<8 x i16>, i32} @llvm.s390.vchlhs(<8 x i16> %a, <8 x i16> %b)
+  %res = extractvalue {<8 x i16>, i32} %call, 0
+  %cc = extractvalue {<8 x i16>, i32} %call, 1
+  %cmp = icmp sgt i32 %cc, 0
+  br i1 %cmp, label %store, label %exit
+
+store:
+  store i32 0, i32 *%ptr
+  br label %exit
+
+exit:
+  ret <8 x i16> %res
+}
+
+; VCHLFS with no processing of the result.
+define i32 @test_vchlfs(<4 x i32> %a, <4 x i32> %b) {
+; CHECK-LABEL: test_vchlfs:
+; CHECK: vchlfs {{%v[0-9]+}}, %v24, %v26
+; CHECK: ipm %r2
+; CHECK: srl %r2, 28
+; CHECK: br %r14
+  %call = call {<4 x i32>, i32} @llvm.s390.vchlfs(<4 x i32> %a, <4 x i32> %b)
+  %res = extractvalue {<4 x i32>, i32} %call, 1
+  ret i32 %res
+}
+
+; VCHLFS, returning 1 if no elements are higher.
+define i32 @test_vchlfs_none_bool(<4 x i32> %a, <4 x i32> %b) {
+; CHECK-LABEL: test_vchlfs_none_bool:
+; CHECK: vchlfs {{%v[0-9]+}}, %v24, %v26
+; CHECK: ipm [[REG:%r[0-5]]]
+; CHECK: risblg %r2, [[REG]], 31, 159, 35
+; CHECK: br %r14
+  %call = call {<4 x i32>, i32} @llvm.s390.vchlfs(<4 x i32> %a, <4 x i32> %b)
+  %res = extractvalue {<4 x i32>, i32} %call, 1
+  %cmp = icmp eq i32 %res, 3
+  %ext = zext i1 %cmp to i32
+  ret i32 %ext
+}
+
+; VCHLFS, storing to %ptr if no elements are higher.
+define <4 x i32> @test_vchlfs_none_store(<4 x i32> %a, <4 x i32> %b,
+                                         i32 *%ptr) {
+; CHECK-LABEL: test_vchlfs_none_store:
+; CHECK-NOT: %r
+; CHECK: vchlfs %v24, %v24, %v26
+; CHECK-NEXT: {{jno|jle}} {{\.L*}}
+; CHECK: mvhi 0(%r2), 0
+; CHECK: br %r14
+  %call = call {<4 x i32>, i32} @llvm.s390.vchlfs(<4 x i32> %a, <4 x i32> %b)
+  %res = extractvalue {<4 x i32>, i32} %call, 0
+  %cc = extractvalue {<4 x i32>, i32} %call, 1
+  %cmp = icmp sge i32 %cc, 3
+  br i1 %cmp, label %store, label %exit
+
+store:
+  store i32 0, i32 *%ptr
+  br label %exit
+
+exit:
+  ret <4 x i32> %res
+}
+
+; VCHLGS with no processing of the result.
+define i32 @test_vchlgs(<2 x i64> %a, <2 x i64> %b) {
+; CHECK-LABEL: test_vchlgs:
+; CHECK: vchlgs {{%v[0-9]+}}, %v24, %v26
+; CHECK: ipm %r2
+; CHECK: srl %r2, 28
+; CHECK: br %r14
+  %call = call {<2 x i64>, i32} @llvm.s390.vchlgs(<2 x i64> %a, <2 x i64> %b)
+  %res = extractvalue {<2 x i64>, i32} %call, 1
+  ret i32 %res
+}
+
+; VCHLGS returning 1 if all elements are higher (CC == 0).
+define i32 @test_vchlgs_all_bool(<2 x i64> %a, <2 x i64> %b) {
+; CHECK-LABEL: test_vchlgs_all_bool:
+; CHECK: vchlgs {{%v[0-9]+}}, %v24, %v26
+; CHECK: ipm %r2
+; CHECK: afi %r2, -268435456
+; CHECK: srl %r2, 31
+; CHECK: br %r14
+  %call = call {<2 x i64>, i32} @llvm.s390.vchlgs(<2 x i64> %a, <2 x i64> %b)
+  %res = extractvalue {<2 x i64>, i32} %call, 1
+  %cmp = icmp slt i32 %res, 1
+  %ext = zext i1 %cmp to i32
+  ret i32 %ext
+}
+
+; VCHLGS, storing to %ptr if all elements are higher.
+define <2 x i64> @test_vchlgs_all_store(<2 x i64> %a, <2 x i64> %b, i32 *%ptr) {
+; CHECK-LABEL: test_vchlgs_all_store:
+; CHECK-NOT: %r
+; CHECK: vchlgs %v24, %v24, %v26
+; CHECK-NEXT: {{jnhe|jne}} {{\.L*}}
+; CHECK: mvhi 0(%r2), 0
+; CHECK: br %r14
+  %call = call {<2 x i64>, i32} @llvm.s390.vchlgs(<2 x i64> %a, <2 x i64> %b)
+  %res = extractvalue {<2 x i64>, i32} %call, 0
+  %cc = extractvalue {<2 x i64>, i32} %call, 1
+  %cmp = icmp ule i32 %cc, 0
+  br i1 %cmp, label %store, label %exit
+
+store:
+  store i32 0, i32 *%ptr
+  br label %exit
+
+exit:
+  ret <2 x i64> %res
+}
+
+; VFAEB with !IN !RT.
+define <16 x i8> @test_vfaeb_0(<16 x i8> %a, <16 x i8> %b) {
+; CHECK-LABEL: test_vfaeb_0:
+; CHECK: vfaeb %v24, %v24, %v26, 0
+; CHECK: br %r14
+  %res = call <16 x i8> @llvm.s390.vfaeb(<16 x i8> %a, <16 x i8> %b, i32 0)
+  ret <16 x i8> %res
+}
+
+; VFAEB with !IN RT.
+define <16 x i8> @test_vfaeb_4(<16 x i8> %a, <16 x i8> %b) {
+; CHECK-LABEL: test_vfaeb_4:
+; CHECK: vfaeb %v24, %v24, %v26, 4
+; CHECK: br %r14
+  %res = call <16 x i8> @llvm.s390.vfaeb(<16 x i8> %a, <16 x i8> %b, i32 4)
+  ret <16 x i8> %res
+}
+
+; VFAEB with IN !RT.
+define <16 x i8> @test_vfaeb_8(<16 x i8> %a, <16 x i8> %b) {
+; CHECK-LABEL: test_vfaeb_8:
+; CHECK: vfaeb %v24, %v24, %v26, 8
+; CHECK: br %r14
+  %res = call <16 x i8> @llvm.s390.vfaeb(<16 x i8> %a, <16 x i8> %b, i32 8)
+  ret <16 x i8> %res
+}
+
+; VFAEB with IN RT.
+define <16 x i8> @test_vfaeb_12(<16 x i8> %a, <16 x i8> %b) {
+; CHECK-LABEL: test_vfaeb_12:
+; CHECK: vfaeb %v24, %v24, %v26, 12
+; CHECK: br %r14
+  %res = call <16 x i8> @llvm.s390.vfaeb(<16 x i8> %a, <16 x i8> %b, i32 12)
+  ret <16 x i8> %res
+}
+
+; VFAEB with CS -- should be ignored.
+define <16 x i8> @test_vfaeb_1(<16 x i8> %a, <16 x i8> %b) {
+; CHECK-LABEL: test_vfaeb_1:
+; CHECK: vfaeb %v24, %v24, %v26, 0
+; CHECK: br %r14
+  %res = call <16 x i8> @llvm.s390.vfaeb(<16 x i8> %a, <16 x i8> %b, i32 1)
+  ret <16 x i8> %res
+}
+
+; VFAEH.
+define <8 x i16> @test_vfaeh(<8 x i16> %a, <8 x i16> %b) {
+; CHECK-LABEL: test_vfaeh:
+; CHECK: vfaeh %v24, %v24, %v26, 4
+; CHECK: br %r14
+  %res = call <8 x i16> @llvm.s390.vfaeh(<8 x i16> %a, <8 x i16> %b, i32 4)
+  ret <8 x i16> %res
+}
+
+; VFAEF.
+define <4 x i32> @test_vfaef(<4 x i32> %a, <4 x i32> %b) {
+; CHECK-LABEL: test_vfaef:
+; CHECK: vfaef %v24, %v24, %v26, 8
+; CHECK: br %r14
+  %res = call <4 x i32> @llvm.s390.vfaef(<4 x i32> %a, <4 x i32> %b, i32 8)
+  ret <4 x i32> %res
+}
+
+; VFAEBS.
+define <16 x i8> @test_vfaebs(<16 x i8> %a, <16 x i8> %b, i32 *%ccptr) {
+; CHECK-LABEL: test_vfaebs:
+; CHECK: vfaebs %v24, %v24, %v26, 0
+; CHECK: ipm [[REG:%r[0-5]]]
+; CHECK: srl [[REG]], 28
+; CHECK: st [[REG]], 0(%r2)
+; CHECK: br %r14
+  %call = call {<16 x i8>, i32} @llvm.s390.vfaebs(<16 x i8> %a, <16 x i8> %b,
+                                                  i32 0)
+  %res = extractvalue {<16 x i8>, i32} %call, 0
+  %cc = extractvalue {<16 x i8>, i32} %call, 1
+  store i32 %cc, i32 *%ccptr
+  ret <16 x i8> %res
+}
+
+; VFAEHS.
+define <8 x i16> @test_vfaehs(<8 x i16> %a, <8 x i16> %b, i32 *%ccptr) {
+; CHECK-LABEL: test_vfaehs:
+; CHECK: vfaehs %v24, %v24, %v26, 4
+; CHECK: ipm [[REG:%r[0-5]]]
+; CHECK: srl [[REG]], 28
+; CHECK: st [[REG]], 0(%r2)
+; CHECK: br %r14
+  %call = call {<8 x i16>, i32} @llvm.s390.vfaehs(<8 x i16> %a, <8 x i16> %b,
+                                                  i32 4)
+  %res = extractvalue {<8 x i16>, i32} %call, 0
+  %cc = extractvalue {<8 x i16>, i32} %call, 1
+  store i32 %cc, i32 *%ccptr
+  ret <8 x i16> %res
+}
+
+; VFAEFS.
+define <4 x i32> @test_vfaefs(<4 x i32> %a, <4 x i32> %b, i32 *%ccptr) {
+; CHECK-LABEL: test_vfaefs:
+; CHECK: vfaefs %v24, %v24, %v26, 8
+; CHECK: ipm [[REG:%r[0-5]]]
+; CHECK: srl [[REG]], 28
+; CHECK: st [[REG]], 0(%r2)
+; CHECK: br %r14
+  %call = call {<4 x i32>, i32} @llvm.s390.vfaefs(<4 x i32> %a, <4 x i32> %b,
+                                                  i32 8)
+  %res = extractvalue {<4 x i32>, i32} %call, 0
+  %cc = extractvalue {<4 x i32>, i32} %call, 1
+  store i32 %cc, i32 *%ccptr
+  ret <4 x i32> %res
+}
+
+; VFAEZB with !IN !RT.
+define <16 x i8> @test_vfaezb_0(<16 x i8> %a, <16 x i8> %b) {
+; CHECK-LABEL: test_vfaezb_0:
+; CHECK: vfaezb %v24, %v24, %v26, 0
+; CHECK: br %r14
+  %res = call <16 x i8> @llvm.s390.vfaezb(<16 x i8> %a, <16 x i8> %b, i32 0)
+  ret <16 x i8> %res
+}
+
+; VFAEZB with !IN RT.
+define <16 x i8> @test_vfaezb_4(<16 x i8> %a, <16 x i8> %b) {
+; CHECK-LABEL: test_vfaezb_4:
+; CHECK: vfaezb %v24, %v24, %v26, 4
+; CHECK: br %r14
+  %res = call <16 x i8> @llvm.s390.vfaezb(<16 x i8> %a, <16 x i8> %b, i32 4)
+  ret <16 x i8> %res
+}
+
+; VFAEZB with IN !RT.
+define <16 x i8> @test_vfaezb_8(<16 x i8> %a, <16 x i8> %b) {
+; CHECK-LABEL: test_vfaezb_8:
+; CHECK: vfaezb %v24, %v24, %v26, 8
+; CHECK: br %r14
+  %res = call <16 x i8> @llvm.s390.vfaezb(<16 x i8> %a, <16 x i8> %b, i32 8)
+  ret <16 x i8> %res
+}
+
+; VFAEZB with IN RT.
+define <16 x i8> @test_vfaezb_12(<16 x i8> %a, <16 x i8> %b) {
+; CHECK-LABEL: test_vfaezb_12:
+; CHECK: vfaezb %v24, %v24, %v26, 12
+; CHECK: br %r14
+  %res = call <16 x i8> @llvm.s390.vfaezb(<16 x i8> %a, <16 x i8> %b, i32 12)
+  ret <16 x i8> %res
+}
+
+; VFAEZB with CS -- should be ignored.
+define <16 x i8> @test_vfaezb_1(<16 x i8> %a, <16 x i8> %b) {
+; CHECK-LABEL: test_vfaezb_1:
+; CHECK: vfaezb %v24, %v24, %v26, 0
+; CHECK: br %r14
+  %res = call <16 x i8> @llvm.s390.vfaezb(<16 x i8> %a, <16 x i8> %b, i32 1)
+  ret <16 x i8> %res
+}
+
+; VFAEZH.
+define <8 x i16> @test_vfaezh(<8 x i16> %a, <8 x i16> %b) {
+; CHECK-LABEL: test_vfaezh:
+; CHECK: vfaezh %v24, %v24, %v26, 4
+; CHECK: br %r14
+  %res = call <8 x i16> @llvm.s390.vfaezh(<8 x i16> %a, <8 x i16> %b, i32 4)
+  ret <8 x i16> %res
+}
+
+; VFAEZF.
+define <4 x i32> @test_vfaezf(<4 x i32> %a, <4 x i32> %b) {
+; CHECK-LABEL: test_vfaezf:
+; CHECK: vfaezf %v24, %v24, %v26, 8
+; CHECK: br %r14
+  %res = call <4 x i32> @llvm.s390.vfaezf(<4 x i32> %a, <4 x i32> %b, i32 8)
+  ret <4 x i32> %res
+}
+
+; VFAEZBS.
+define <16 x i8> @test_vfaezbs(<16 x i8> %a, <16 x i8> %b, i32 *%ccptr) {
+; CHECK-LABEL: test_vfaezbs:
+; CHECK: vfaezbs %v24, %v24, %v26, 0
+; CHECK: ipm [[REG:%r[0-5]]]
+; CHECK: srl [[REG]], 28
+; CHECK: st [[REG]], 0(%r2)
+; CHECK: br %r14
+  %call = call {<16 x i8>, i32} @llvm.s390.vfaezbs(<16 x i8> %a, <16 x i8> %b,
+                                                   i32 0)
+  %res = extractvalue {<16 x i8>, i32} %call, 0
+  %cc = extractvalue {<16 x i8>, i32} %call, 1
+  store i32 %cc, i32 *%ccptr
+  ret <16 x i8> %res
+}
+
+; VFAEZHS.
+define <8 x i16> @test_vfaezhs(<8 x i16> %a, <8 x i16> %b, i32 *%ccptr) {
+; CHECK-LABEL: test_vfaezhs:
+; CHECK: vfaezhs %v24, %v24, %v26, 4
+; CHECK: ipm [[REG:%r[0-5]]]
+; CHECK: srl [[REG]], 28
+; CHECK: st [[REG]], 0(%r2)
+; CHECK: br %r14
+  %call = call {<8 x i16>, i32} @llvm.s390.vfaezhs(<8 x i16> %a, <8 x i16> %b,
+                                                   i32 4)
+  %res = extractvalue {<8 x i16>, i32} %call, 0
+  %cc = extractvalue {<8 x i16>, i32} %call, 1
+  store i32 %cc, i32 *%ccptr
+  ret <8 x i16> %res
+}
+
+; VFAEZFS.
+define <4 x i32> @test_vfaezfs(<4 x i32> %a, <4 x i32> %b, i32 *%ccptr) {
+; CHECK-LABEL: test_vfaezfs:
+; CHECK: vfaezfs %v24, %v24, %v26, 8
+; CHECK: ipm [[REG:%r[0-5]]]
+; CHECK: srl [[REG]], 28
+; CHECK: st [[REG]], 0(%r2)
+; CHECK: br %r14
+  %call = call {<4 x i32>, i32} @llvm.s390.vfaezfs(<4 x i32> %a, <4 x i32> %b,
+                                                   i32 8)
+  %res = extractvalue {<4 x i32>, i32} %call, 0
+  %cc = extractvalue {<4 x i32>, i32} %call, 1
+  store i32 %cc, i32 *%ccptr
+  ret <4 x i32> %res
+}
+
+; VFEEB.
+define <16 x i8> @test_vfeeb_0(<16 x i8> %a, <16 x i8> %b) {
+; CHECK-LABEL: test_vfeeb_0:
+; CHECK: vfeeb %v24, %v24, %v26
+; CHECK: br %r14
+  %res = call <16 x i8> @llvm.s390.vfeeb(<16 x i8> %a, <16 x i8> %b)
+  ret <16 x i8> %res
+}
+
+; VFEEH.
+define <8 x i16> @test_vfeeh(<8 x i16> %a, <8 x i16> %b) {
+; CHECK-LABEL: test_vfeeh:
+; CHECK: vfeeh %v24, %v24, %v26
+; CHECK: br %r14
+  %res = call <8 x i16> @llvm.s390.vfeeh(<8 x i16> %a, <8 x i16> %b)
+  ret <8 x i16> %res
+}
+
+; VFEEF.
+define <4 x i32> @test_vfeef(<4 x i32> %a, <4 x i32> %b) {
+; CHECK-LABEL: test_vfeef:
+; CHECK: vfeef %v24, %v24, %v26
+; CHECK: br %r14
+  %res = call <4 x i32> @llvm.s390.vfeef(<4 x i32> %a, <4 x i32> %b)
+  ret <4 x i32> %res
+}
+
+; VFEEBS.
+define <16 x i8> @test_vfeebs(<16 x i8> %a, <16 x i8> %b, i32 *%ccptr) {
+; CHECK-LABEL: test_vfeebs:
+; CHECK: vfeebs %v24, %v24, %v26
+; CHECK: ipm [[REG:%r[0-5]]]
+; CHECK: srl [[REG]], 28
+; CHECK: st [[REG]], 0(%r2)
+; CHECK: br %r14
+  %call = call {<16 x i8>, i32} @llvm.s390.vfeebs(<16 x i8> %a, <16 x i8> %b)
+  %res = extractvalue {<16 x i8>, i32} %call, 0
+  %cc = extractvalue {<16 x i8>, i32} %call, 1
+  store i32 %cc, i32 *%ccptr
+  ret <16 x i8> %res
+}
+
+; VFEEHS.
+define <8 x i16> @test_vfeehs(<8 x i16> %a, <8 x i16> %b, i32 *%ccptr) {
+; CHECK-LABEL: test_vfeehs:
+; CHECK: vfeehs %v24, %v24, %v26
+; CHECK: ipm [[REG:%r[0-5]]]
+; CHECK: srl [[REG]], 28
+; CHECK: st [[REG]], 0(%r2)
+; CHECK: br %r14
+  %call = call {<8 x i16>, i32} @llvm.s390.vfeehs(<8 x i16> %a, <8 x i16> %b)
+  %res = extractvalue {<8 x i16>, i32} %call, 0
+  %cc = extractvalue {<8 x i16>, i32} %call, 1
+  store i32 %cc, i32 *%ccptr
+  ret <8 x i16> %res
+}
+
+; VFEEFS.
+define <4 x i32> @test_vfeefs(<4 x i32> %a, <4 x i32> %b, i32 *%ccptr) {
+; CHECK-LABEL: test_vfeefs:
+; CHECK: vfeefs %v24, %v24, %v26
+; CHECK: ipm [[REG:%r[0-5]]]
+; CHECK: srl [[REG]], 28
+; CHECK: st [[REG]], 0(%r2)
+; CHECK: br %r14
+  %call = call {<4 x i32>, i32} @llvm.s390.vfeefs(<4 x i32> %a, <4 x i32> %b)
+  %res = extractvalue {<4 x i32>, i32} %call, 0
+  %cc = extractvalue {<4 x i32>, i32} %call, 1
+  store i32 %cc, i32 *%ccptr
+  ret <4 x i32> %res
+}
+
+; VFEEZB.
+define <16 x i8> @test_vfeezb(<16 x i8> %a, <16 x i8> %b) {
+; CHECK-LABEL: test_vfeezb:
+; CHECK: vfeezb %v24, %v24, %v26
+; CHECK: br %r14
+  %res = call <16 x i8> @llvm.s390.vfeezb(<16 x i8> %a, <16 x i8> %b)
+  ret <16 x i8> %res
+}
+
+; VFEEZH.
+define <8 x i16> @test_vfeezh(<8 x i16> %a, <8 x i16> %b) {
+; CHECK-LABEL: test_vfeezh:
+; CHECK: vfeezh %v24, %v24, %v26
+; CHECK: br %r14
+  %res = call <8 x i16> @llvm.s390.vfeezh(<8 x i16> %a, <8 x i16> %b)
+  ret <8 x i16> %res
+}
+
+; VFEEZF.
+define <4 x i32> @test_vfeezf(<4 x i32> %a, <4 x i32> %b) {
+; CHECK-LABEL: test_vfeezf:
+; CHECK: vfeezf %v24, %v24, %v26
+; CHECK: br %r14
+  %res = call <4 x i32> @llvm.s390.vfeezf(<4 x i32> %a, <4 x i32> %b)
+  ret <4 x i32> %res
+}
+
+; VFEEZBS.
+define <16 x i8> @test_vfeezbs(<16 x i8> %a, <16 x i8> %b, i32 *%ccptr) {
+; CHECK-LABEL: test_vfeezbs:
+; CHECK: vfeezbs %v24, %v24, %v26
+; CHECK: ipm [[REG:%r[0-5]]]
+; CHECK: srl [[REG]], 28
+; CHECK: st [[REG]], 0(%r2)
+; CHECK: br %r14
+  %call = call {<16 x i8>, i32} @llvm.s390.vfeezbs(<16 x i8> %a, <16 x i8> %b)
+  %res = extractvalue {<16 x i8>, i32} %call, 0
+  %cc = extractvalue {<16 x i8>, i32} %call, 1
+  store i32 %cc, i32 *%ccptr
+  ret <16 x i8> %res
+}
+
+; VFEEZHS.
+define <8 x i16> @test_vfeezhs(<8 x i16> %a, <8 x i16> %b, i32 *%ccptr) {
+; CHECK-LABEL: test_vfeezhs:
+; CHECK: vfeezhs %v24, %v24, %v26
+; CHECK: ipm [[REG:%r[0-5]]]
+; CHECK: srl [[REG]], 28
+; CHECK: st [[REG]], 0(%r2)
+; CHECK: br %r14
+  %call = call {<8 x i16>, i32} @llvm.s390.vfeezhs(<8 x i16> %a, <8 x i16> %b)
+  %res = extractvalue {<8 x i16>, i32} %call, 0
+  %cc = extractvalue {<8 x i16>, i32} %call, 1
+  store i32 %cc, i32 *%ccptr
+  ret <8 x i16> %res
+}
+
+; VFEEZFS.
+define <4 x i32> @test_vfeezfs(<4 x i32> %a, <4 x i32> %b, i32 *%ccptr) {
+; CHECK-LABEL: test_vfeezfs:
+; CHECK: vfeezfs %v24, %v24, %v26
+; CHECK: ipm [[REG:%r[0-5]]]
+; CHECK: srl [[REG]], 28
+; CHECK: st [[REG]], 0(%r2)
+; CHECK: br %r14
+  %call = call {<4 x i32>, i32} @llvm.s390.vfeezfs(<4 x i32> %a, <4 x i32> %b)
+  %res = extractvalue {<4 x i32>, i32} %call, 0
+  %cc = extractvalue {<4 x i32>, i32} %call, 1
+  store i32 %cc, i32 *%ccptr
+  ret <4 x i32> %res
+}
+
+; VFENEB.
+define <16 x i8> @test_vfeneb_0(<16 x i8> %a, <16 x i8> %b) {
+; CHECK-LABEL: test_vfeneb_0:
+; CHECK: vfeneb %v24, %v24, %v26
+; CHECK: br %r14
+  %res = call <16 x i8> @llvm.s390.vfeneb(<16 x i8> %a, <16 x i8> %b)
+  ret <16 x i8> %res
+}
+
+; VFENEH.
+define <8 x i16> @test_vfeneh(<8 x i16> %a, <8 x i16> %b) {
+; CHECK-LABEL: test_vfeneh:
+; CHECK: vfeneh %v24, %v24, %v26
+; CHECK: br %r14
+  %res = call <8 x i16> @llvm.s390.vfeneh(<8 x i16> %a, <8 x i16> %b)
+  ret <8 x i16> %res
+}
+
+; VFENEF.
+define <4 x i32> @test_vfenef(<4 x i32> %a, <4 x i32> %b) {
+; CHECK-LABEL: test_vfenef:
+; CHECK: vfenef %v24, %v24, %v26
+; CHECK: br %r14
+  %res = call <4 x i32> @llvm.s390.vfenef(<4 x i32> %a, <4 x i32> %b)
+  ret <4 x i32> %res
+}
+
+; VFENEBS.
+define <16 x i8> @test_vfenebs(<16 x i8> %a, <16 x i8> %b, i32 *%ccptr) {
+; CHECK-LABEL: test_vfenebs:
+; CHECK: vfenebs %v24, %v24, %v26
+; CHECK: ipm [[REG:%r[0-5]]]
+; CHECK: srl [[REG]], 28
+; CHECK: st [[REG]], 0(%r2)
+; CHECK: br %r14
+  %call = call {<16 x i8>, i32} @llvm.s390.vfenebs(<16 x i8> %a, <16 x i8> %b)
+  %res = extractvalue {<16 x i8>, i32} %call, 0
+  %cc = extractvalue {<16 x i8>, i32} %call, 1
+  store i32 %cc, i32 *%ccptr
+  ret <16 x i8> %res
+}
+
+; VFENEHS.
+define <8 x i16> @test_vfenehs(<8 x i16> %a, <8 x i16> %b, i32 *%ccptr) {
+; CHECK-LABEL: test_vfenehs:
+; CHECK: vfenehs %v24, %v24, %v26
+; CHECK: ipm [[REG:%r[0-5]]]
+; CHECK: srl [[REG]], 28
+; CHECK: st [[REG]], 0(%r2)
+; CHECK: br %r14
+  %call = call {<8 x i16>, i32} @llvm.s390.vfenehs(<8 x i16> %a, <8 x i16> %b)
+  %res = extractvalue {<8 x i16>, i32} %call, 0
+  %cc = extractvalue {<8 x i16>, i32} %call, 1
+  store i32 %cc, i32 *%ccptr
+  ret <8 x i16> %res
+}
+
+; VFENEFS.
+define <4 x i32> @test_vfenefs(<4 x i32> %a, <4 x i32> %b, i32 *%ccptr) {
+; CHECK-LABEL: test_vfenefs:
+; CHECK: vfenefs %v24, %v24, %v26
+; CHECK: ipm [[REG:%r[0-5]]]
+; CHECK: srl [[REG]], 28
+; CHECK: st [[REG]], 0(%r2)
+; CHECK: br %r14
+  %call = call {<4 x i32>, i32} @llvm.s390.vfenefs(<4 x i32> %a, <4 x i32> %b)
+  %res = extractvalue {<4 x i32>, i32} %call, 0
+  %cc = extractvalue {<4 x i32>, i32} %call, 1
+  store i32 %cc, i32 *%ccptr
+  ret <4 x i32> %res
+}
+
+; VFENEZB.
+define <16 x i8> @test_vfenezb(<16 x i8> %a, <16 x i8> %b) {
+; CHECK-LABEL: test_vfenezb:
+; CHECK: vfenezb %v24, %v24, %v26
+; CHECK: br %r14
+  %res = call <16 x i8> @llvm.s390.vfenezb(<16 x i8> %a, <16 x i8> %b)
+  ret <16 x i8> %res
+}
+
+; VFENEZH.
+define <8 x i16> @test_vfenezh(<8 x i16> %a, <8 x i16> %b) {
+; CHECK-LABEL: test_vfenezh:
+; CHECK: vfenezh %v24, %v24, %v26
+; CHECK: br %r14
+  %res = call <8 x i16> @llvm.s390.vfenezh(<8 x i16> %a, <8 x i16> %b)
+  ret <8 x i16> %res
+}
+
+; VFENEZF.
+define <4 x i32> @test_vfenezf(<4 x i32> %a, <4 x i32> %b) {
+; CHECK-LABEL: test_vfenezf:
+; CHECK: vfenezf %v24, %v24, %v26
+; CHECK: br %r14
+  %res = call <4 x i32> @llvm.s390.vfenezf(<4 x i32> %a, <4 x i32> %b)
+  ret <4 x i32> %res
+}
+
+; VFENEZBS.
+define <16 x i8> @test_vfenezbs(<16 x i8> %a, <16 x i8> %b, i32 *%ccptr) {
+; CHECK-LABEL: test_vfenezbs:
+; CHECK: vfenezbs %v24, %v24, %v26
+; CHECK: ipm [[REG:%r[0-5]]]
+; CHECK: srl [[REG]], 28
+; CHECK: st [[REG]], 0(%r2)
+; CHECK: br %r14
+  %call = call {<16 x i8>, i32} @llvm.s390.vfenezbs(<16 x i8> %a, <16 x i8> %b)
+  %res = extractvalue {<16 x i8>, i32} %call, 0
+  %cc = extractvalue {<16 x i8>, i32} %call, 1
+  store i32 %cc, i32 *%ccptr
+  ret <16 x i8> %res
+}
+
+; VFENEZHS.
+define <8 x i16> @test_vfenezhs(<8 x i16> %a, <8 x i16> %b, i32 *%ccptr) {
+; CHECK-LABEL: test_vfenezhs:
+; CHECK: vfenezhs %v24, %v24, %v26
+; CHECK: ipm [[REG:%r[0-5]]]
+; CHECK: srl [[REG]], 28
+; CHECK: st [[REG]], 0(%r2)
+; CHECK: br %r14
+  %call = call {<8 x i16>, i32} @llvm.s390.vfenezhs(<8 x i16> %a, <8 x i16> %b)
+  %res = extractvalue {<8 x i16>, i32} %call, 0
+  %cc = extractvalue {<8 x i16>, i32} %call, 1
+  store i32 %cc, i32 *%ccptr
+  ret <8 x i16> %res
+}
+
+; VFENEZFS.
+define <4 x i32> @test_vfenezfs(<4 x i32> %a, <4 x i32> %b, i32 *%ccptr) {
+; CHECK-LABEL: test_vfenezfs:
+; CHECK: vfenezfs %v24, %v24, %v26
+; CHECK: ipm [[REG:%r[0-5]]]
+; CHECK: srl [[REG]], 28
+; CHECK: st [[REG]], 0(%r2)
+; CHECK: br %r14
+  %call = call {<4 x i32>, i32} @llvm.s390.vfenezfs(<4 x i32> %a, <4 x i32> %b)
+  %res = extractvalue {<4 x i32>, i32} %call, 0
+  %cc = extractvalue {<4 x i32>, i32} %call, 1
+  store i32 %cc, i32 *%ccptr
+  ret <4 x i32> %res
+}
+
+; VISTRB.
+define <16 x i8> @test_vistrb(<16 x i8> %a) {
+; CHECK-LABEL: test_vistrb:
+; CHECK: vistrb %v24, %v24
+; CHECK: br %r14
+  %res = call <16 x i8> @llvm.s390.vistrb(<16 x i8> %a)
+  ret <16 x i8> %res
+}
+
+; VISTRH.
+define <8 x i16> @test_vistrh(<8 x i16> %a) {
+; CHECK-LABEL: test_vistrh:
+; CHECK: vistrh %v24, %v24
+; CHECK: br %r14
+  %res = call <8 x i16> @llvm.s390.vistrh(<8 x i16> %a)
+  ret <8 x i16> %res
+}
+
+; VISTRF.
+define <4 x i32> @test_vistrf(<4 x i32> %a) {
+; CHECK-LABEL: test_vistrf:
+; CHECK: vistrf %v24, %v24
+; CHECK: br %r14
+  %res = call <4 x i32> @llvm.s390.vistrf(<4 x i32> %a)
+  ret <4 x i32> %res
+}
+
+; VISTRBS.
+define <16 x i8> @test_vistrbs(<16 x i8> %a, i32 *%ccptr) {
+; CHECK-LABEL: test_vistrbs:
+; CHECK: vistrbs %v24, %v24
+; CHECK: ipm [[REG:%r[0-5]]]
+; CHECK: srl [[REG]], 28
+; CHECK: st [[REG]], 0(%r2)
+; CHECK: br %r14
+  %call = call {<16 x i8>, i32} @llvm.s390.vistrbs(<16 x i8> %a)
+  %res = extractvalue {<16 x i8>, i32} %call, 0
+  %cc = extractvalue {<16 x i8>, i32} %call, 1
+  store i32 %cc, i32 *%ccptr
+  ret <16 x i8> %res
+}
+
+; VISTRHS.
+define <8 x i16> @test_vistrhs(<8 x i16> %a, i32 *%ccptr) {
+; CHECK-LABEL: test_vistrhs:
+; CHECK: vistrhs %v24, %v24
+; CHECK: ipm [[REG:%r[0-5]]]
+; CHECK: srl [[REG]], 28
+; CHECK: st [[REG]], 0(%r2)
+; CHECK: br %r14
+  %call = call {<8 x i16>, i32} @llvm.s390.vistrhs(<8 x i16> %a)
+  %res = extractvalue {<8 x i16>, i32} %call, 0
+  %cc = extractvalue {<8 x i16>, i32} %call, 1
+  store i32 %cc, i32 *%ccptr
+  ret <8 x i16> %res
+}
+
+; VISTRFS.
+define <4 x i32> @test_vistrfs(<4 x i32> %a, i32 *%ccptr) {
+; CHECK-LABEL: test_vistrfs:
+; CHECK: vistrfs %v24, %v24
+; CHECK: ipm [[REG:%r[0-5]]]
+; CHECK: srl [[REG]], 28
+; CHECK: st [[REG]], 0(%r2)
+; CHECK: br %r14
+  %call = call {<4 x i32>, i32} @llvm.s390.vistrfs(<4 x i32> %a)
+  %res = extractvalue {<4 x i32>, i32} %call, 0
+  %cc = extractvalue {<4 x i32>, i32} %call, 1
+  store i32 %cc, i32 *%ccptr
+  ret <4 x i32> %res
+}
+
+; VSTRCB with !IN !RT.
+define <16 x i8> @test_vstrcb_0(<16 x i8> %a, <16 x i8> %b, <16 x i8> %c) {
+; CHECK-LABEL: test_vstrcb_0:
+; CHECK: vstrcb %v24, %v24, %v26, %v28, 0
+; CHECK: br %r14
+  %res = call <16 x i8> @llvm.s390.vstrcb(<16 x i8> %a, <16 x i8> %b,
+                                          <16 x i8> %c, i32 0)
+  ret <16 x i8> %res
+}
+
+; VSTRCB with !IN RT.
+define <16 x i8> @test_vstrcb_4(<16 x i8> %a, <16 x i8> %b, <16 x i8> %c) {
+; CHECK-LABEL: test_vstrcb_4:
+; CHECK: vstrcb %v24, %v24, %v26, %v28, 4
+; CHECK: br %r14
+  %res = call <16 x i8> @llvm.s390.vstrcb(<16 x i8> %a, <16 x i8> %b,
+                                          <16 x i8> %c, i32 4)
+  ret <16 x i8> %res
+}
+
+; VSTRCB with IN !RT.
+define <16 x i8> @test_vstrcb_8(<16 x i8> %a, <16 x i8> %b, <16 x i8> %c) {
+; CHECK-LABEL: test_vstrcb_8:
+; CHECK: vstrcb %v24, %v24, %v26, %v28, 8
+; CHECK: br %r14
+  %res = call <16 x i8> @llvm.s390.vstrcb(<16 x i8> %a, <16 x i8> %b,
+                                          <16 x i8> %c, i32 8)
+  ret <16 x i8> %res
+}
+
+; VSTRCB with IN RT.
+define <16 x i8> @test_vstrcb_12(<16 x i8> %a, <16 x i8> %b, <16 x i8> %c) {
+; CHECK-LABEL: test_vstrcb_12:
+; CHECK: vstrcb %v24, %v24, %v26, %v28, 12
+; CHECK: br %r14
+  %res = call <16 x i8> @llvm.s390.vstrcb(<16 x i8> %a, <16 x i8> %b,
+                                          <16 x i8> %c, i32 12)
+  ret <16 x i8> %res
+}
+
+; VSTRCB with CS -- should be ignored.
+define <16 x i8> @test_vstrcb_1(<16 x i8> %a, <16 x i8> %b, <16 x i8> %c) {
+; CHECK-LABEL: test_vstrcb_1:
+; CHECK: vstrcb %v24, %v24, %v26, %v28, 0
+; CHECK: br %r14
+  %res = call <16 x i8> @llvm.s390.vstrcb(<16 x i8> %a, <16 x i8> %b,
+                                          <16 x i8> %c, i32 1)
+  ret <16 x i8> %res
+}
+
+; VSTRCH.
+define <8 x i16> @test_vstrch(<8 x i16> %a, <8 x i16> %b, <8 x i16> %c) {
+; CHECK-LABEL: test_vstrch:
+; CHECK: vstrch %v24, %v24, %v26, %v28, 4
+; CHECK: br %r14
+  %res = call <8 x i16> @llvm.s390.vstrch(<8 x i16> %a, <8 x i16> %b,
+                                          <8 x i16> %c, i32 4)
+  ret <8 x i16> %res
+}
+
+; VSTRCF.
+define <4 x i32> @test_vstrcf(<4 x i32> %a, <4 x i32> %b, <4 x i32> %c) {
+; CHECK-LABEL: test_vstrcf:
+; CHECK: vstrcf %v24, %v24, %v26, %v28, 8
+; CHECK: br %r14
+  %res = call <4 x i32> @llvm.s390.vstrcf(<4 x i32> %a, <4 x i32> %b,
+                                          <4 x i32> %c, i32 8)
+  ret <4 x i32> %res
+}
+
+; VSTRCBS.
+define <16 x i8> @test_vstrcbs(<16 x i8> %a, <16 x i8> %b, <16 x i8> %c,
+                               i32 *%ccptr) {
+; CHECK-LABEL: test_vstrcbs:
+; CHECK: vstrcbs %v24, %v24, %v26, %v28, 0
+; CHECK: ipm [[REG:%r[0-5]]]
+; CHECK: srl [[REG]], 28
+; CHECK: st [[REG]], 0(%r2)
+; CHECK: br %r14
+  %call = call {<16 x i8>, i32} @llvm.s390.vstrcbs(<16 x i8> %a, <16 x i8> %b,
+                                                   <16 x i8> %c, i32 0)
+  %res = extractvalue {<16 x i8>, i32} %call, 0
+  %cc = extractvalue {<16 x i8>, i32} %call, 1
+  store i32 %cc, i32 *%ccptr
+  ret <16 x i8> %res
+}
+
+; VSTRCHS.
+define <8 x i16> @test_vstrchs(<8 x i16> %a, <8 x i16> %b, <8 x i16> %c,
+                               i32 *%ccptr) {
+; CHECK-LABEL: test_vstrchs:
+; CHECK: vstrchs %v24, %v24, %v26, %v28, 4
+; CHECK: ipm [[REG:%r[0-5]]]
+; CHECK: srl [[REG]], 28
+; CHECK: st [[REG]], 0(%r2)
+; CHECK: br %r14
+  %call = call {<8 x i16>, i32} @llvm.s390.vstrchs(<8 x i16> %a, <8 x i16> %b,
+                                                   <8 x i16> %c, i32 4)
+  %res = extractvalue {<8 x i16>, i32} %call, 0
+  %cc = extractvalue {<8 x i16>, i32} %call, 1
+  store i32 %cc, i32 *%ccptr
+  ret <8 x i16> %res
+}
+
+; VSTRCFS.
+define <4 x i32> @test_vstrcfs(<4 x i32> %a, <4 x i32> %b, <4 x i32> %c,
+                               i32 *%ccptr) {
+; CHECK-LABEL: test_vstrcfs:
+; CHECK: vstrcfs %v24, %v24, %v26, %v28, 8
+; CHECK: ipm [[REG:%r[0-5]]]
+; CHECK: srl [[REG]], 28
+; CHECK: st [[REG]], 0(%r2)
+; CHECK: br %r14
+  %call = call {<4 x i32>, i32} @llvm.s390.vstrcfs(<4 x i32> %a, <4 x i32> %b,
+                                                   <4 x i32> %c, i32 8)
+  %res = extractvalue {<4 x i32>, i32} %call, 0
+  %cc = extractvalue {<4 x i32>, i32} %call, 1
+  store i32 %cc, i32 *%ccptr
+  ret <4 x i32> %res
+}
+
+; VSTRCZB with !IN !RT.
+define <16 x i8> @test_vstrczb_0(<16 x i8> %a, <16 x i8> %b, <16 x i8> %c) {
+; CHECK-LABEL: test_vstrczb_0:
+; CHECK: vstrczb %v24, %v24, %v26, %v28, 0
+; CHECK: br %r14
+  %res = call <16 x i8> @llvm.s390.vstrczb(<16 x i8> %a, <16 x i8> %b,
+                                           <16 x i8> %c, i32 0)
+  ret <16 x i8> %res
+}
+
+; VSTRCZB with !IN RT.
+define <16 x i8> @test_vstrczb_4(<16 x i8> %a, <16 x i8> %b, <16 x i8> %c) {
+; CHECK-LABEL: test_vstrczb_4:
+; CHECK: vstrczb %v24, %v24, %v26, %v28, 4
+; CHECK: br %r14
+  %res = call <16 x i8> @llvm.s390.vstrczb(<16 x i8> %a, <16 x i8> %b,
+                                           <16 x i8> %c, i32 4)
+  ret <16 x i8> %res
+}
+
+; VSTRCZB with IN !RT.
+define <16 x i8> @test_vstrczb_8(<16 x i8> %a, <16 x i8> %b, <16 x i8> %c) {
+; CHECK-LABEL: test_vstrczb_8:
+; CHECK: vstrczb %v24, %v24, %v26, %v28, 8
+; CHECK: br %r14
+  %res = call <16 x i8> @llvm.s390.vstrczb(<16 x i8> %a, <16 x i8> %b,
+                                           <16 x i8> %c, i32 8)
+  ret <16 x i8> %res
+}
+
+; VSTRCZB with IN RT.
+define <16 x i8> @test_vstrczb_12(<16 x i8> %a, <16 x i8> %b, <16 x i8> %c) {
+; CHECK-LABEL: test_vstrczb_12:
+; CHECK: vstrczb %v24, %v24, %v26, %v28, 12
+; CHECK: br %r14
+  %res = call <16 x i8> @llvm.s390.vstrczb(<16 x i8> %a, <16 x i8> %b,
+                                           <16 x i8> %c, i32 12)
+  ret <16 x i8> %res
+}
+
+; VSTRCZB with CS -- should be ignored.
+define <16 x i8> @test_vstrczb_1(<16 x i8> %a, <16 x i8> %b, <16 x i8> %c) {
+; CHECK-LABEL: test_vstrczb_1:
+; CHECK: vstrczb %v24, %v24, %v26, %v28, 0
+; CHECK: br %r14
+  %res = call <16 x i8> @llvm.s390.vstrczb(<16 x i8> %a, <16 x i8> %b,
+                                           <16 x i8> %c, i32 1)
+  ret <16 x i8> %res
+}
+
+; VSTRCZH.
+define <8 x i16> @test_vstrczh(<8 x i16> %a, <8 x i16> %b, <8 x i16> %c) {
+; CHECK-LABEL: test_vstrczh:
+; CHECK: vstrczh %v24, %v24, %v26, %v28, 4
+; CHECK: br %r14
+  %res = call <8 x i16> @llvm.s390.vstrczh(<8 x i16> %a, <8 x i16> %b,
+                                           <8 x i16> %c,  i32 4)
+  ret <8 x i16> %res
+}
+
+; VSTRCZF.
+define <4 x i32> @test_vstrczf(<4 x i32> %a, <4 x i32> %b, <4 x i32> %c) {
+; CHECK-LABEL: test_vstrczf:
+; CHECK: vstrczf %v24, %v24, %v26, %v28, 8
+; CHECK: br %r14
+  %res = call <4 x i32> @llvm.s390.vstrczf(<4 x i32> %a, <4 x i32> %b,
+                                           <4 x i32> %c, i32 8)
+  ret <4 x i32> %res
+}
+
+; VSTRCZBS.
+define <16 x i8> @test_vstrczbs(<16 x i8> %a, <16 x i8> %b, <16 x i8> %c,
+                                i32 *%ccptr) {
+; CHECK-LABEL: test_vstrczbs:
+; CHECK: vstrczbs %v24, %v24, %v26, %v28, 0
+; CHECK: ipm [[REG:%r[0-5]]]
+; CHECK: srl [[REG]], 28
+; CHECK: st [[REG]], 0(%r2)
+; CHECK: br %r14
+  %call = call {<16 x i8>, i32} @llvm.s390.vstrczbs(<16 x i8> %a, <16 x i8> %b,
+                                                    <16 x i8> %c, i32 0)
+  %res = extractvalue {<16 x i8>, i32} %call, 0
+  %cc = extractvalue {<16 x i8>, i32} %call, 1
+  store i32 %cc, i32 *%ccptr
+  ret <16 x i8> %res
+}
+
+; VSTRCZHS.
+define <8 x i16> @test_vstrczhs(<8 x i16> %a, <8 x i16> %b, <8 x i16> %c,
+                                i32 *%ccptr) {
+; CHECK-LABEL: test_vstrczhs:
+; CHECK: vstrczhs %v24, %v24, %v26, %v28, 4
+; CHECK: ipm [[REG:%r[0-5]]]
+; CHECK: srl [[REG]], 28
+; CHECK: st [[REG]], 0(%r2)
+; CHECK: br %r14
+  %call = call {<8 x i16>, i32} @llvm.s390.vstrczhs(<8 x i16> %a, <8 x i16> %b,
+                                                    <8 x i16> %c, i32 4)
+  %res = extractvalue {<8 x i16>, i32} %call, 0
+  %cc = extractvalue {<8 x i16>, i32} %call, 1
+  store i32 %cc, i32 *%ccptr
+  ret <8 x i16> %res
+}
+
+; VSTRCZFS.
+define <4 x i32> @test_vstrczfs(<4 x i32> %a, <4 x i32> %b, <4 x i32> %c,
+                                i32 *%ccptr) {
+; CHECK-LABEL: test_vstrczfs:
+; CHECK: vstrczfs %v24, %v24, %v26, %v28, 8
+; CHECK: ipm [[REG:%r[0-5]]]
+; CHECK: srl [[REG]], 28
+; CHECK: st [[REG]], 0(%r2)
+; CHECK: br %r14
+  %call = call {<4 x i32>, i32} @llvm.s390.vstrczfs(<4 x i32> %a, <4 x i32> %b,
+                                                    <4 x i32> %c, i32 8)
+  %res = extractvalue {<4 x i32>, i32} %call, 0
+  %cc = extractvalue {<4 x i32>, i32} %call, 1
+  store i32 %cc, i32 *%ccptr
+  ret <4 x i32> %res
+}
+
+; VFCEDBS with no processing of the result.
+define i32 @test_vfcedbs(<2 x double> %a, <2 x double> %b) {
+; CHECK-LABEL: test_vfcedbs:
+; CHECK: vfcedbs {{%v[0-9]+}}, %v24, %v26
+; CHECK: ipm %r2
+; CHECK: srl %r2, 28
+; CHECK: br %r14
+  %call = call {<2 x i64>, i32} @llvm.s390.vfcedbs(<2 x double> %a,
+                                                   <2 x double> %b)
+  %res = extractvalue {<2 x i64>, i32} %call, 1
+  ret i32 %res
+}
+
+; VFCEDBS, returning 1 if any elements are equal (CC != 3).
+define i32 @test_vfcedbs_any_bool(<2 x double> %a, <2 x double> %b) {
+; CHECK-LABEL: test_vfcedbs_any_bool:
+; CHECK: vfcedbs {{%v[0-9]+}}, %v24, %v26
+; CHECK: ipm %r2
+; CHECK: afi %r2, -536870912
+; CHECK: srl %r2, 31
+; CHECK: br %r14
+  %call = call {<2 x i64>, i32} @llvm.s390.vfcedbs(<2 x double> %a,
+                                                   <2 x double> %b)
+  %res = extractvalue {<2 x i64>, i32} %call, 1
+  %cmp = icmp ne i32 %res, 3
+  %ext = zext i1 %cmp to i32
+  ret i32 %ext
+}
+
+; VFCEDBS, storing to %ptr if any elements are equal.
+define <2 x i64> @test_vfcedbs_any_store(<2 x double> %a, <2 x double> %b,
+                                         i32 *%ptr) {
+; CHECK-LABEL: test_vfcedbs_any_store:
+; CHECK-NOT: %r
+; CHECK: vfcedbs %v24, %v24, %v26
+; CHECK-NEXT: {{jo|jnle}} {{\.L*}}
+; CHECK: mvhi 0(%r2), 0
+; CHECK: br %r14
+  %call = call {<2 x i64>, i32} @llvm.s390.vfcedbs(<2 x double> %a,
+                                                   <2 x double> %b)
+  %res = extractvalue {<2 x i64>, i32} %call, 0
+  %cc = extractvalue {<2 x i64>, i32} %call, 1
+  %cmp = icmp ule i32 %cc, 2
+  br i1 %cmp, label %store, label %exit
+
+store:
+  store i32 0, i32 *%ptr
+  br label %exit
+
+exit:
+  ret <2 x i64> %res
+}
+
+; VFCHDBS with no processing of the result.
+define i32 @test_vfchdbs(<2 x double> %a, <2 x double> %b) {
+; CHECK-LABEL: test_vfchdbs:
+; CHECK: vfchdbs {{%v[0-9]+}}, %v24, %v26
+; CHECK: ipm %r2
+; CHECK: srl %r2, 28
+; CHECK: br %r14
+  %call = call {<2 x i64>, i32} @llvm.s390.vfchdbs(<2 x double> %a,
+                                                   <2 x double> %b)
+  %res = extractvalue {<2 x i64>, i32} %call, 1
+  ret i32 %res
+}
+
+; VFCHDBS, returning 1 if not all elements are higher.
+define i32 @test_vfchdbs_notall_bool(<2 x double> %a, <2 x double> %b) {
+; CHECK-LABEL: test_vfchdbs_notall_bool:
+; CHECK: vfchdbs {{%v[0-9]+}}, %v24, %v26
+; CHECK: ipm [[REG:%r[0-5]]]
+; CHECK: risblg %r2, [[REG]], 31, 159, 36
+; CHECK: br %r14
+  %call = call {<2 x i64>, i32} @llvm.s390.vfchdbs(<2 x double> %a,
+                                                   <2 x double> %b)
+  %res = extractvalue {<2 x i64>, i32} %call, 1
+  %cmp = icmp sge i32 %res, 1
+  %ext = zext i1 %cmp to i32
+  ret i32 %ext
+}
+
+; VFCHDBS, storing to %ptr if not all elements are higher.
+define <2 x i64> @test_vfchdbs_notall_store(<2 x double> %a, <2 x double> %b,
+                                            i32 *%ptr) {
+; CHECK-LABEL: test_vfchdbs_notall_store:
+; CHECK-NOT: %r
+; CHECK: vfchdbs %v24, %v24, %v26
+; CHECK-NEXT: {{jhe|je}} {{\.L*}}
+; CHECK: mvhi 0(%r2), 0
+; CHECK: br %r14
+  %call = call {<2 x i64>, i32} @llvm.s390.vfchdbs(<2 x double> %a,
+                                                   <2 x double> %b)
+  %res = extractvalue {<2 x i64>, i32} %call, 0
+  %cc = extractvalue {<2 x i64>, i32} %call, 1
+  %cmp = icmp ugt i32 %cc, 0
+  br i1 %cmp, label %store, label %exit
+
+store:
+  store i32 0, i32 *%ptr
+  br label %exit
+
+exit:
+  ret <2 x i64> %res
+}
+
+; VFCHEDBS with no processing of the result.
+define i32 @test_vfchedbs(<2 x double> %a, <2 x double> %b) {
+; CHECK-LABEL: test_vfchedbs:
+; CHECK: vfchedbs {{%v[0-9]+}}, %v24, %v26
+; CHECK: ipm %r2
+; CHECK: srl %r2, 28
+; CHECK: br %r14
+  %call = call {<2 x i64>, i32} @llvm.s390.vfchedbs(<2 x double> %a,
+						    <2 x double> %b)
+  %res = extractvalue {<2 x i64>, i32} %call, 1
+  ret i32 %res
+}
+
+; VFCHEDBS, returning 1 if neither element is higher or equal.
+define i32 @test_vfchedbs_none_bool(<2 x double> %a, <2 x double> %b) {
+; CHECK-LABEL: test_vfchedbs_none_bool:
+; CHECK: vfchedbs {{%v[0-9]+}}, %v24, %v26
+; CHECK: ipm [[REG:%r[0-5]]]
+; CHECK: risblg %r2, [[REG]], 31, 159, 35
+; CHECK: br %r14
+  %call = call {<2 x i64>, i32} @llvm.s390.vfchedbs(<2 x double> %a,
+						    <2 x double> %b)
+  %res = extractvalue {<2 x i64>, i32} %call, 1
+  %cmp = icmp eq i32 %res, 3
+  %ext = zext i1 %cmp to i32
+  ret i32 %ext
+}
+
+; VFCHEDBS, storing to %ptr if neither element is higher or equal.
+define <2 x i64> @test_vfchedbs_none_store(<2 x double> %a, <2 x double> %b,
+                                           i32 *%ptr) {
+; CHECK-LABEL: test_vfchedbs_none_store:
+; CHECK-NOT: %r
+; CHECK: vfchedbs %v24, %v24, %v26
+; CHECK-NEXT: {{jno|jle}} {{\.L*}}
+; CHECK: mvhi 0(%r2), 0
+; CHECK: br %r14
+  %call = call {<2 x i64>, i32} @llvm.s390.vfchedbs(<2 x double> %a,
+						    <2 x double> %b)
+  %res = extractvalue {<2 x i64>, i32} %call, 0
+  %cc = extractvalue {<2 x i64>, i32} %call, 1
+  %cmp = icmp uge i32 %cc, 3
+  br i1 %cmp, label %store, label %exit
+
+store:
+  store i32 0, i32 *%ptr
+  br label %exit
+
+exit:
+  ret <2 x i64> %res
+}
+
+; VFTCIDB with the lowest useful class selector and no processing of the result.
+define i32 @test_vftcidb(<2 x double> %a) {
+; CHECK-LABEL: test_vftcidb:
+; CHECK: vftcidb {{%v[0-9]+}}, %v24, 1
+; CHECK: ipm %r2
+; CHECK: srl %r2, 28
+; CHECK: br %r14
+  %call = call {<2 x i64>, i32} @llvm.s390.vftcidb(<2 x double> %a, i32 1)
+  %res = extractvalue {<2 x i64>, i32} %call, 1
+  ret i32 %res
+}
+
+; VFTCIDB with the highest useful class selector, returning 1 if all elements
+; have the right class (CC == 0).
+define i32 @test_vftcidb_all_bool(<2 x double> %a) {
+; CHECK-LABEL: test_vftcidb_all_bool:
+; CHECK: vftcidb {{%v[0-9]+}}, %v24, 4094
+; CHECK: afi %r2, -268435456
+; CHECK: srl %r2, 31
+; CHECK: br %r14
+  %call = call {<2 x i64>, i32} @llvm.s390.vftcidb(<2 x double> %a, i32 4094)
+  %res = extractvalue {<2 x i64>, i32} %call, 1
+  %cmp = icmp eq i32 %res, 0
+  %ext = zext i1 %cmp to i32
+  ret i32 %ext
+}
+
+; VFIDB with a rounding mode not usable via standard intrinsics.
+define <2 x double> @test_vfidb_0_4(<2 x double> %a) {
+; CHECK-LABEL: test_vfidb_0_4:
+; CHECK: vfidb %v24, %v24, 0, 4
+; CHECK: br %r14
+  %res = call <2 x double> @llvm.s390.vfidb(<2 x double> %a, i32 0, i32 4)
+  ret <2 x double> %res
+}
+
+; VFIDB with IEEE-inexact exception suppressed.
+define <2 x double> @test_vfidb_4_0(<2 x double> %a) {
+; CHECK-LABEL: test_vfidb_4_0:
+; CHECK: vfidb %v24, %v24, 4, 0
+; CHECK: br %r14
+  %res = call <2 x double> @llvm.s390.vfidb(<2 x double> %a, i32 4, i32 0)
+  ret <2 x double> %res
+}
+
diff --git a/test/CodeGen/SystemZ/vec-log-01.ll b/test/CodeGen/SystemZ/vec-log-01.ll
new file mode 100644
index 00000000000..f9b7402f08e
--- /dev/null
+++ b/test/CodeGen/SystemZ/vec-log-01.ll
@@ -0,0 +1,15 @@
+; Test v2f64 logarithm.
+;
+; RUN: llc < %s -mtriple=s390x-linux-gnu -mcpu=z13 | FileCheck %s
+
+declare <2 x double> @llvm.log.v2f64(<2 x double>)
+
+define <2 x double> @f1(<2 x double> %val) {
+; CHECK-LABEL: f1:
+; CHECK: brasl %r14, log@PLT
+; CHECK: brasl %r14, log@PLT
+; CHECK: vmrhg %v24,
+; CHECK: br %r14
+  %ret = call <2 x double> @llvm.log.v2f64(<2 x double> %val)
+  ret <2 x double> %ret
+}
diff --git a/test/CodeGen/SystemZ/vec-max-01.ll b/test/CodeGen/SystemZ/vec-max-01.ll
new file mode 100644
index 00000000000..ca6f08aa493
--- /dev/null
+++ b/test/CodeGen/SystemZ/vec-max-01.ll
@@ -0,0 +1,83 @@
+; Test v16i8 maximum.
+;
+; RUN: llc < %s -mtriple=s390x-linux-gnu -mcpu=z13 | FileCheck %s
+
+; Test with slt.
+define <16 x i8> @f1(<16 x i8> %val1, <16 x i8> %val2) {
+; CHECK-LABEL: f1:
+; CHECK: vmxb %v24, {{%v24, %v26|%v26, %v24}}
+; CHECK: br %r14
+  %cmp = icmp slt <16 x i8> %val1, %val2
+  %ret = select <16 x i1> %cmp, <16 x i8> %val2, <16 x i8> %val1
+  ret <16 x i8> %ret
+}
+
+; Test with sle.
+define <16 x i8> @f2(<16 x i8> %val1, <16 x i8> %val2) {
+; CHECK-LABEL: f2:
+; CHECK: vmxb %v24, {{%v24, %v26|%v26, %v24}}
+; CHECK: br %r14
+  %cmp = icmp sle <16 x i8> %val1, %val2
+  %ret = select <16 x i1> %cmp, <16 x i8> %val2, <16 x i8> %val1
+  ret <16 x i8> %ret
+}
+
+; Test with sgt.
+define <16 x i8> @f3(<16 x i8> %val1, <16 x i8> %val2) {
+; CHECK-LABEL: f3:
+; CHECK: vmxb %v24, {{%v24, %v26|%v26, %v24}}
+; CHECK: br %r14
+  %cmp = icmp sgt <16 x i8> %val1, %val2
+  %ret = select <16 x i1> %cmp, <16 x i8> %val1, <16 x i8> %val2
+  ret <16 x i8> %ret
+}
+
+; Test with sge.
+define <16 x i8> @f4(<16 x i8> %val1, <16 x i8> %val2) {
+; CHECK-LABEL: f4:
+; CHECK: vmxb %v24, {{%v24, %v26|%v26, %v24}}
+; CHECK: br %r14
+  %cmp = icmp sge <16 x i8> %val1, %val2
+  %ret = select <16 x i1> %cmp, <16 x i8> %val1, <16 x i8> %val2
+  ret <16 x i8> %ret
+}
+
+; Test with ult.
+define <16 x i8> @f5(<16 x i8> %val1, <16 x i8> %val2) {
+; CHECK-LABEL: f5:
+; CHECK: vmxlb %v24, {{%v24, %v26|%v26, %v24}}
+; CHECK: br %r14
+  %cmp = icmp ult <16 x i8> %val1, %val2
+  %ret = select <16 x i1> %cmp, <16 x i8> %val2, <16 x i8> %val1
+  ret <16 x i8> %ret
+}
+
+; Test with ule.
+define <16 x i8> @f6(<16 x i8> %val1, <16 x i8> %val2) {
+; CHECK-LABEL: f6:
+; CHECK: vmxlb %v24, {{%v24, %v26|%v26, %v24}}
+; CHECK: br %r14
+  %cmp = icmp ule <16 x i8> %val1, %val2
+  %ret = select <16 x i1> %cmp, <16 x i8> %val2, <16 x i8> %val1
+  ret <16 x i8> %ret
+}
+
+; Test with ugt.
+define <16 x i8> @f7(<16 x i8> %val1, <16 x i8> %val2) {
+; CHECK-LABEL: f7:
+; CHECK: vmxlb %v24, {{%v24, %v26|%v26, %v24}}
+; CHECK: br %r14
+  %cmp = icmp ugt <16 x i8> %val1, %val2
+  %ret = select <16 x i1> %cmp, <16 x i8> %val1, <16 x i8> %val2
+  ret <16 x i8> %ret
+}
+
+; Test with uge.
+define <16 x i8> @f8(<16 x i8> %val1, <16 x i8> %val2) {
+; CHECK-LABEL: f8:
+; CHECK: vmxlb %v24, {{%v24, %v26|%v26, %v24}}
+; CHECK: br %r14
+  %cmp = icmp uge <16 x i8> %val1, %val2
+  %ret = select <16 x i1> %cmp, <16 x i8> %val1, <16 x i8> %val2
+  ret <16 x i8> %ret
+}
diff --git a/test/CodeGen/SystemZ/vec-max-02.ll b/test/CodeGen/SystemZ/vec-max-02.ll
new file mode 100644
index 00000000000..2c61603b6f3
--- /dev/null
+++ b/test/CodeGen/SystemZ/vec-max-02.ll
@@ -0,0 +1,83 @@
+; Test v8i16 maximum.
+;
+; RUN: llc < %s -mtriple=s390x-linux-gnu -mcpu=z13 | FileCheck %s
+
+; Test with slt.
+define <8 x i16> @f1(<8 x i16> %val1, <8 x i16> %val2) {
+; CHECK-LABEL: f1:
+; CHECK: vmxh %v24, {{%v24, %v26|%v26, %v24}}
+; CHECK: br %r14
+  %cmp = icmp slt <8 x i16> %val1, %val2
+  %ret = select <8 x i1> %cmp, <8 x i16> %val2, <8 x i16> %val1
+  ret <8 x i16> %ret
+}
+
+; Test with sle.
+define <8 x i16> @f2(<8 x i16> %val1, <8 x i16> %val2) {
+; CHECK-LABEL: f2:
+; CHECK: vmxh %v24, {{%v24, %v26|%v26, %v24}}
+; CHECK: br %r14
+  %cmp = icmp sle <8 x i16> %val1, %val2
+  %ret = select <8 x i1> %cmp, <8 x i16> %val2, <8 x i16> %val1
+  ret <8 x i16> %ret
+}
+
+; Test with sgt.
+define <8 x i16> @f3(<8 x i16> %val1, <8 x i16> %val2) {
+; CHECK-LABEL: f3:
+; CHECK: vmxh %v24, {{%v24, %v26|%v26, %v24}}
+; CHECK: br %r14
+  %cmp = icmp sgt <8 x i16> %val1, %val2
+  %ret = select <8 x i1> %cmp, <8 x i16> %val1, <8 x i16> %val2
+  ret <8 x i16> %ret
+}
+
+; Test with sge.
+define <8 x i16> @f4(<8 x i16> %val1, <8 x i16> %val2) {
+; CHECK-LABEL: f4:
+; CHECK: vmxh %v24, {{%v24, %v26|%v26, %v24}}
+; CHECK: br %r14
+  %cmp = icmp sge <8 x i16> %val1, %val2
+  %ret = select <8 x i1> %cmp, <8 x i16> %val1, <8 x i16> %val2
+  ret <8 x i16> %ret
+}
+
+; Test with ult.
+define <8 x i16> @f5(<8 x i16> %val1, <8 x i16> %val2) {
+; CHECK-LABEL: f5:
+; CHECK: vmxlh %v24, {{%v24, %v26|%v26, %v24}}
+; CHECK: br %r14
+  %cmp = icmp ult <8 x i16> %val1, %val2
+  %ret = select <8 x i1> %cmp, <8 x i16> %val2, <8 x i16> %val1
+  ret <8 x i16> %ret
+}
+
+; Test with ule.
+define <8 x i16> @f6(<8 x i16> %val1, <8 x i16> %val2) {
+; CHECK-LABEL: f6:
+; CHECK: vmxlh %v24, {{%v24, %v26|%v26, %v24}}
+; CHECK: br %r14
+  %cmp = icmp ule <8 x i16> %val1, %val2
+  %ret = select <8 x i1> %cmp, <8 x i16> %val2, <8 x i16> %val1
+  ret <8 x i16> %ret
+}
+
+; Test with ugt.
+define <8 x i16> @f7(<8 x i16> %val1, <8 x i16> %val2) {
+; CHECK-LABEL: f7:
+; CHECK: vmxlh %v24, {{%v24, %v26|%v26, %v24}}
+; CHECK: br %r14
+  %cmp = icmp ugt <8 x i16> %val1, %val2
+  %ret = select <8 x i1> %cmp, <8 x i16> %val1, <8 x i16> %val2
+  ret <8 x i16> %ret
+}
+
+; Test with uge.
+define <8 x i16> @f8(<8 x i16> %val1, <8 x i16> %val2) {
+; CHECK-LABEL: f8:
+; CHECK: vmxlh %v24, {{%v24, %v26|%v26, %v24}}
+; CHECK: br %r14
+  %cmp = icmp uge <8 x i16> %val1, %val2
+  %ret = select <8 x i1> %cmp, <8 x i16> %val1, <8 x i16> %val2
+  ret <8 x i16> %ret
+}
diff --git a/test/CodeGen/SystemZ/vec-max-03.ll b/test/CodeGen/SystemZ/vec-max-03.ll
new file mode 100644
index 00000000000..a4387948399
--- /dev/null
+++ b/test/CodeGen/SystemZ/vec-max-03.ll
@@ -0,0 +1,83 @@
+; Test v4i32 maximum.
+;
+; RUN: llc < %s -mtriple=s390x-linux-gnu -mcpu=z13 | FileCheck %s
+
+; Test with slt.
+define <4 x i32> @f1(<4 x i32> %val1, <4 x i32> %val2) {
+; CHECK-LABEL: f1:
+; CHECK: vmxf %v24, {{%v24, %v26|%v26, %v24}}
+; CHECK: br %r14
+  %cmp = icmp slt <4 x i32> %val1, %val2
+  %ret = select <4 x i1> %cmp, <4 x i32> %val2, <4 x i32> %val1
+  ret <4 x i32> %ret
+}
+
+; Test with sle.
+define <4 x i32> @f2(<4 x i32> %val1, <4 x i32> %val2) {
+; CHECK-LABEL: f2:
+; CHECK: vmxf %v24, {{%v24, %v26|%v26, %v24}}
+; CHECK: br %r14
+  %cmp = icmp sle <4 x i32> %val1, %val2
+  %ret = select <4 x i1> %cmp, <4 x i32> %val2, <4 x i32> %val1
+  ret <4 x i32> %ret
+}
+
+; Test with sgt.
+define <4 x i32> @f3(<4 x i32> %val1, <4 x i32> %val2) {
+; CHECK-LABEL: f3:
+; CHECK: vmxf %v24, {{%v24, %v26|%v26, %v24}}
+; CHECK: br %r14
+  %cmp = icmp sgt <4 x i32> %val1, %val2
+  %ret = select <4 x i1> %cmp, <4 x i32> %val1, <4 x i32> %val2
+  ret <4 x i32> %ret
+}
+
+; Test with sge.
+define <4 x i32> @f4(<4 x i32> %val1, <4 x i32> %val2) {
+; CHECK-LABEL: f4:
+; CHECK: vmxf %v24, {{%v24, %v26|%v26, %v24}}
+; CHECK: br %r14
+  %cmp = icmp sge <4 x i32> %val1, %val2
+  %ret = select <4 x i1> %cmp, <4 x i32> %val1, <4 x i32> %val2
+  ret <4 x i32> %ret
+}
+
+; Test with ult.
+define <4 x i32> @f5(<4 x i32> %val1, <4 x i32> %val2) {
+; CHECK-LABEL: f5:
+; CHECK: vmxlf %v24, {{%v24, %v26|%v26, %v24}}
+; CHECK: br %r14
+  %cmp = icmp ult <4 x i32> %val1, %val2
+  %ret = select <4 x i1> %cmp, <4 x i32> %val2, <4 x i32> %val1
+  ret <4 x i32> %ret
+}
+
+; Test with ule.
+define <4 x i32> @f6(<4 x i32> %val1, <4 x i32> %val2) {
+; CHECK-LABEL: f6:
+; CHECK: vmxlf %v24, {{%v24, %v26|%v26, %v24}}
+; CHECK: br %r14
+  %cmp = icmp ule <4 x i32> %val1, %val2
+  %ret = select <4 x i1> %cmp, <4 x i32> %val2, <4 x i32> %val1
+  ret <4 x i32> %ret
+}
+
+; Test with ugt.
+define <4 x i32> @f7(<4 x i32> %val1, <4 x i32> %val2) {
+; CHECK-LABEL: f7:
+; CHECK: vmxlf %v24, {{%v24, %v26|%v26, %v24}}
+; CHECK: br %r14
+  %cmp = icmp ugt <4 x i32> %val1, %val2
+  %ret = select <4 x i1> %cmp, <4 x i32> %val1, <4 x i32> %val2
+  ret <4 x i32> %ret
+}
+
+; Test with uge.
+define <4 x i32> @f8(<4 x i32> %val1, <4 x i32> %val2) {
+; CHECK-LABEL: f8:
+; CHECK: vmxlf %v24, {{%v24, %v26|%v26, %v24}}
+; CHECK: br %r14
+  %cmp = icmp uge <4 x i32> %val1, %val2
+  %ret = select <4 x i1> %cmp, <4 x i32> %val1, <4 x i32> %val2
+  ret <4 x i32> %ret
+}
diff --git a/test/CodeGen/SystemZ/vec-max-04.ll b/test/CodeGen/SystemZ/vec-max-04.ll
new file mode 100644
index 00000000000..ab7c6239127
--- /dev/null
+++ b/test/CodeGen/SystemZ/vec-max-04.ll
@@ -0,0 +1,83 @@
+; Test v2i64 maximum.
+;
+; RUN: llc < %s -mtriple=s390x-linux-gnu -mcpu=z13 | FileCheck %s
+
+; Test with slt.
+define <2 x i64> @f1(<2 x i64> %val1, <2 x i64> %val2) {
+; CHECK-LABEL: f1:
+; CHECK: vmxg %v24, {{%v24, %v26|%v26, %v24}}
+; CHECK: br %r14
+  %cmp = icmp slt <2 x i64> %val1, %val2
+  %ret = select <2 x i1> %cmp, <2 x i64> %val2, <2 x i64> %val1
+  ret <2 x i64> %ret
+}
+
+; Test with sle.
+define <2 x i64> @f2(<2 x i64> %val1, <2 x i64> %val2) {
+; CHECK-LABEL: f2:
+; CHECK: vmxg %v24, {{%v24, %v26|%v26, %v24}}
+; CHECK: br %r14
+  %cmp = icmp sle <2 x i64> %val1, %val2
+  %ret = select <2 x i1> %cmp, <2 x i64> %val2, <2 x i64> %val1
+  ret <2 x i64> %ret
+}
+
+; Test with sgt.
+define <2 x i64> @f3(<2 x i64> %val1, <2 x i64> %val2) {
+; CHECK-LABEL: f3:
+; CHECK: vmxg %v24, {{%v24, %v26|%v26, %v24}}
+; CHECK: br %r14
+  %cmp = icmp sgt <2 x i64> %val1, %val2
+  %ret = select <2 x i1> %cmp, <2 x i64> %val1, <2 x i64> %val2
+  ret <2 x i64> %ret
+}
+
+; Test with sge.
+define <2 x i64> @f4(<2 x i64> %val1, <2 x i64> %val2) {
+; CHECK-LABEL: f4:
+; CHECK: vmxg %v24, {{%v24, %v26|%v26, %v24}}
+; CHECK: br %r14
+  %cmp = icmp sge <2 x i64> %val1, %val2
+  %ret = select <2 x i1> %cmp, <2 x i64> %val1, <2 x i64> %val2
+  ret <2 x i64> %ret
+}
+
+; Test with ult.
+define <2 x i64> @f5(<2 x i64> %val1, <2 x i64> %val2) {
+; CHECK-LABEL: f5:
+; CHECK: vmxlg %v24, {{%v24, %v26|%v26, %v24}}
+; CHECK: br %r14
+  %cmp = icmp ult <2 x i64> %val1, %val2
+  %ret = select <2 x i1> %cmp, <2 x i64> %val2, <2 x i64> %val1
+  ret <2 x i64> %ret
+}
+
+; Test with ule.
+define <2 x i64> @f6(<2 x i64> %val1, <2 x i64> %val2) {
+; CHECK-LABEL: f6:
+; CHECK: vmxlg %v24, {{%v24, %v26|%v26, %v24}}
+; CHECK: br %r14
+  %cmp = icmp ule <2 x i64> %val1, %val2
+  %ret = select <2 x i1> %cmp, <2 x i64> %val2, <2 x i64> %val1
+  ret <2 x i64> %ret
+}
+
+; Test with ugt.
+define <2 x i64> @f7(<2 x i64> %val1, <2 x i64> %val2) {
+; CHECK-LABEL: f7:
+; CHECK: vmxlg %v24, {{%v24, %v26|%v26, %v24}}
+; CHECK: br %r14
+  %cmp = icmp ugt <2 x i64> %val1, %val2
+  %ret = select <2 x i1> %cmp, <2 x i64> %val1, <2 x i64> %val2
+  ret <2 x i64> %ret
+}
+
+; Test with uge.
+define <2 x i64> @f8(<2 x i64> %val1, <2 x i64> %val2) {
+; CHECK-LABEL: f8:
+; CHECK: vmxlg %v24, {{%v24, %v26|%v26, %v24}}
+; CHECK: br %r14
+  %cmp = icmp uge <2 x i64> %val1, %val2
+  %ret = select <2 x i1> %cmp, <2 x i64> %val1, <2 x i64> %val2
+  ret <2 x i64> %ret
+}
diff --git a/test/CodeGen/SystemZ/vec-min-01.ll b/test/CodeGen/SystemZ/vec-min-01.ll
new file mode 100644
index 00000000000..255dc57e113
--- /dev/null
+++ b/test/CodeGen/SystemZ/vec-min-01.ll
@@ -0,0 +1,83 @@
+; Test v16i8 minimum.
+;
+; RUN: llc < %s -mtriple=s390x-linux-gnu -mcpu=z13 | FileCheck %s
+
+; Test with slt.
+define <16 x i8> @f1(<16 x i8> %val1, <16 x i8> %val2) {
+; CHECK-LABEL: f1:
+; CHECK: vmnb %v24, {{%v24, %v26|%v26, %v24}}
+; CHECK: br %r14
+  %cmp = icmp slt <16 x i8> %val2, %val1
+  %ret = select <16 x i1> %cmp, <16 x i8> %val2, <16 x i8> %val1
+  ret <16 x i8> %ret
+}
+
+; Test with sle.
+define <16 x i8> @f2(<16 x i8> %val1, <16 x i8> %val2) {
+; CHECK-LABEL: f2:
+; CHECK: vmnb %v24, {{%v24, %v26|%v26, %v24}}
+; CHECK: br %r14
+  %cmp = icmp sle <16 x i8> %val2, %val1
+  %ret = select <16 x i1> %cmp, <16 x i8> %val2, <16 x i8> %val1
+  ret <16 x i8> %ret
+}
+
+; Test with sgt.
+define <16 x i8> @f3(<16 x i8> %val1, <16 x i8> %val2) {
+; CHECK-LABEL: f3:
+; CHECK: vmnb %v24, {{%v24, %v26|%v26, %v24}}
+; CHECK: br %r14
+  %cmp = icmp sgt <16 x i8> %val2, %val1
+  %ret = select <16 x i1> %cmp, <16 x i8> %val1, <16 x i8> %val2
+  ret <16 x i8> %ret
+}
+
+; Test with sge.
+define <16 x i8> @f4(<16 x i8> %val1, <16 x i8> %val2) {
+; CHECK-LABEL: f4:
+; CHECK: vmnb %v24, {{%v24, %v26|%v26, %v24}}
+; CHECK: br %r14
+  %cmp = icmp sge <16 x i8> %val2, %val1
+  %ret = select <16 x i1> %cmp, <16 x i8> %val1, <16 x i8> %val2
+  ret <16 x i8> %ret
+}
+
+; Test with ult.
+define <16 x i8> @f5(<16 x i8> %val1, <16 x i8> %val2) {
+; CHECK-LABEL: f5:
+; CHECK: vmnlb %v24, {{%v24, %v26|%v26, %v24}}
+; CHECK: br %r14
+  %cmp = icmp ult <16 x i8> %val2, %val1
+  %ret = select <16 x i1> %cmp, <16 x i8> %val2, <16 x i8> %val1
+  ret <16 x i8> %ret
+}
+
+; Test with ule.
+define <16 x i8> @f6(<16 x i8> %val1, <16 x i8> %val2) {
+; CHECK-LABEL: f6:
+; CHECK: vmnlb %v24, {{%v24, %v26|%v26, %v24}}
+; CHECK: br %r14
+  %cmp = icmp ule <16 x i8> %val2, %val1
+  %ret = select <16 x i1> %cmp, <16 x i8> %val2, <16 x i8> %val1
+  ret <16 x i8> %ret
+}
+
+; Test with ugt.
+define <16 x i8> @f7(<16 x i8> %val1, <16 x i8> %val2) {
+; CHECK-LABEL: f7:
+; CHECK: vmnlb %v24, {{%v24, %v26|%v26, %v24}}
+; CHECK: br %r14
+  %cmp = icmp ugt <16 x i8> %val2, %val1
+  %ret = select <16 x i1> %cmp, <16 x i8> %val1, <16 x i8> %val2
+  ret <16 x i8> %ret
+}
+
+; Test with uge.
+define <16 x i8> @f8(<16 x i8> %val1, <16 x i8> %val2) {
+; CHECK-LABEL: f8:
+; CHECK: vmnlb %v24, {{%v24, %v26|%v26, %v24}}
+; CHECK: br %r14
+  %cmp = icmp uge <16 x i8> %val2, %val1
+  %ret = select <16 x i1> %cmp, <16 x i8> %val1, <16 x i8> %val2
+  ret <16 x i8> %ret
+}
diff --git a/test/CodeGen/SystemZ/vec-min-02.ll b/test/CodeGen/SystemZ/vec-min-02.ll
new file mode 100644
index 00000000000..cad8a61506c
--- /dev/null
+++ b/test/CodeGen/SystemZ/vec-min-02.ll
@@ -0,0 +1,83 @@
+; Test v8i16 minimum.
+;
+; RUN: llc < %s -mtriple=s390x-linux-gnu -mcpu=z13 | FileCheck %s
+
+; Test with slt.
+define <8 x i16> @f1(<8 x i16> %val1, <8 x i16> %val2) {
+; CHECK-LABEL: f1:
+; CHECK: vmnh %v24, {{%v24, %v26|%v26, %v24}}
+; CHECK: br %r14
+  %cmp = icmp slt <8 x i16> %val2, %val1
+  %ret = select <8 x i1> %cmp, <8 x i16> %val2, <8 x i16> %val1
+  ret <8 x i16> %ret
+}
+
+; Test with sle.
+define <8 x i16> @f2(<8 x i16> %val1, <8 x i16> %val2) {
+; CHECK-LABEL: f2:
+; CHECK: vmnh %v24, {{%v24, %v26|%v26, %v24}}
+; CHECK: br %r14
+  %cmp = icmp sle <8 x i16> %val2, %val1
+  %ret = select <8 x i1> %cmp, <8 x i16> %val2, <8 x i16> %val1
+  ret <8 x i16> %ret
+}
+
+; Test with sgt.
+define <8 x i16> @f3(<8 x i16> %val1, <8 x i16> %val2) {
+; CHECK-LABEL: f3:
+; CHECK: vmnh %v24, {{%v24, %v26|%v26, %v24}}
+; CHECK: br %r14
+  %cmp = icmp sgt <8 x i16> %val2, %val1
+  %ret = select <8 x i1> %cmp, <8 x i16> %val1, <8 x i16> %val2
+  ret <8 x i16> %ret
+}
+
+; Test with sge.
+define <8 x i16> @f4(<8 x i16> %val1, <8 x i16> %val2) {
+; CHECK-LABEL: f4:
+; CHECK: vmnh %v24, {{%v24, %v26|%v26, %v24}}
+; CHECK: br %r14
+  %cmp = icmp sge <8 x i16> %val2, %val1
+  %ret = select <8 x i1> %cmp, <8 x i16> %val1, <8 x i16> %val2
+  ret <8 x i16> %ret
+}
+
+; Test with ult.
+define <8 x i16> @f5(<8 x i16> %val1, <8 x i16> %val2) {
+; CHECK-LABEL: f5:
+; CHECK: vmnlh %v24, {{%v24, %v26|%v26, %v24}}
+; CHECK: br %r14
+  %cmp = icmp ult <8 x i16> %val2, %val1
+  %ret = select <8 x i1> %cmp, <8 x i16> %val2, <8 x i16> %val1
+  ret <8 x i16> %ret
+}
+
+; Test with ule.
+define <8 x i16> @f6(<8 x i16> %val1, <8 x i16> %val2) {
+; CHECK-LABEL: f6:
+; CHECK: vmnlh %v24, {{%v24, %v26|%v26, %v24}}
+; CHECK: br %r14
+  %cmp = icmp ule <8 x i16> %val2, %val1
+  %ret = select <8 x i1> %cmp, <8 x i16> %val2, <8 x i16> %val1
+  ret <8 x i16> %ret
+}
+
+; Test with ugt.
+define <8 x i16> @f7(<8 x i16> %val1, <8 x i16> %val2) {
+; CHECK-LABEL: f7:
+; CHECK: vmnlh %v24, {{%v24, %v26|%v26, %v24}}
+; CHECK: br %r14
+  %cmp = icmp ugt <8 x i16> %val2, %val1
+  %ret = select <8 x i1> %cmp, <8 x i16> %val1, <8 x i16> %val2
+  ret <8 x i16> %ret
+}
+
+; Test with uge.
+define <8 x i16> @f8(<8 x i16> %val1, <8 x i16> %val2) {
+; CHECK-LABEL: f8:
+; CHECK: vmnlh %v24, {{%v24, %v26|%v26, %v24}}
+; CHECK: br %r14
+  %cmp = icmp uge <8 x i16> %val2, %val1
+  %ret = select <8 x i1> %cmp, <8 x i16> %val1, <8 x i16> %val2
+  ret <8 x i16> %ret
+}
diff --git a/test/CodeGen/SystemZ/vec-min-03.ll b/test/CodeGen/SystemZ/vec-min-03.ll
new file mode 100644
index 00000000000..febac50aa46
--- /dev/null
+++ b/test/CodeGen/SystemZ/vec-min-03.ll
@@ -0,0 +1,83 @@
+; Test v4i32 minimum.
+;
+; RUN: llc < %s -mtriple=s390x-linux-gnu -mcpu=z13 | FileCheck %s
+
+; Test with slt.
+define <4 x i32> @f1(<4 x i32> %val1, <4 x i32> %val2) {
+; CHECK-LABEL: f1:
+; CHECK: vmnf %v24, {{%v24, %v26|%v26, %v24}}
+; CHECK: br %r14
+  %cmp = icmp slt <4 x i32> %val2, %val1
+  %ret = select <4 x i1> %cmp, <4 x i32> %val2, <4 x i32> %val1
+  ret <4 x i32> %ret
+}
+
+; Test with sle.
+define <4 x i32> @f2(<4 x i32> %val1, <4 x i32> %val2) {
+; CHECK-LABEL: f2:
+; CHECK: vmnf %v24, {{%v24, %v26|%v26, %v24}}
+; CHECK: br %r14
+  %cmp = icmp sle <4 x i32> %val2, %val1
+  %ret = select <4 x i1> %cmp, <4 x i32> %val2, <4 x i32> %val1
+  ret <4 x i32> %ret
+}
+
+; Test with sgt.
+define <4 x i32> @f3(<4 x i32> %val1, <4 x i32> %val2) {
+; CHECK-LABEL: f3:
+; CHECK: vmnf %v24, {{%v24, %v26|%v26, %v24}}
+; CHECK: br %r14
+  %cmp = icmp sgt <4 x i32> %val2, %val1
+  %ret = select <4 x i1> %cmp, <4 x i32> %val1, <4 x i32> %val2
+  ret <4 x i32> %ret
+}
+
+; Test with sge.
+define <4 x i32> @f4(<4 x i32> %val1, <4 x i32> %val2) {
+; CHECK-LABEL: f4:
+; CHECK: vmnf %v24, {{%v24, %v26|%v26, %v24}}
+; CHECK: br %r14
+  %cmp = icmp sge <4 x i32> %val2, %val1
+  %ret = select <4 x i1> %cmp, <4 x i32> %val1, <4 x i32> %val2
+  ret <4 x i32> %ret
+}
+
+; Test with ult.
+define <4 x i32> @f5(<4 x i32> %val1, <4 x i32> %val2) {
+; CHECK-LABEL: f5:
+; CHECK: vmnlf %v24, {{%v24, %v26|%v26, %v24}}
+; CHECK: br %r14
+  %cmp = icmp ult <4 x i32> %val2, %val1
+  %ret = select <4 x i1> %cmp, <4 x i32> %val2, <4 x i32> %val1
+  ret <4 x i32> %ret
+}
+
+; Test with ule.
+define <4 x i32> @f6(<4 x i32> %val1, <4 x i32> %val2) {
+; CHECK-LABEL: f6:
+; CHECK: vmnlf %v24, {{%v24, %v26|%v26, %v24}}
+; CHECK: br %r14
+  %cmp = icmp ule <4 x i32> %val2, %val1
+  %ret = select <4 x i1> %cmp, <4 x i32> %val2, <4 x i32> %val1
+  ret <4 x i32> %ret
+}
+
+; Test with ugt.
+define <4 x i32> @f7(<4 x i32> %val1, <4 x i32> %val2) {
+; CHECK-LABEL: f7:
+; CHECK: vmnlf %v24, {{%v24, %v26|%v26, %v24}}
+; CHECK: br %r14
+  %cmp = icmp ugt <4 x i32> %val2, %val1
+  %ret = select <4 x i1> %cmp, <4 x i32> %val1, <4 x i32> %val2
+  ret <4 x i32> %ret
+}
+
+; Test with uge.
+define <4 x i32> @f8(<4 x i32> %val1, <4 x i32> %val2) {
+; CHECK-LABEL: f8:
+; CHECK: vmnlf %v24, {{%v24, %v26|%v26, %v24}}
+; CHECK: br %r14
+  %cmp = icmp uge <4 x i32> %val2, %val1
+  %ret = select <4 x i1> %cmp, <4 x i32> %val1, <4 x i32> %val2
+  ret <4 x i32> %ret
+}
diff --git a/test/CodeGen/SystemZ/vec-min-04.ll b/test/CodeGen/SystemZ/vec-min-04.ll
new file mode 100644
index 00000000000..765ce1956b5
--- /dev/null
+++ b/test/CodeGen/SystemZ/vec-min-04.ll
@@ -0,0 +1,83 @@
+; Test v2i64 minimum.
+;
+; RUN: llc < %s -mtriple=s390x-linux-gnu -mcpu=z13 | FileCheck %s
+
+; Test with slt.
+define <2 x i64> @f1(<2 x i64> %val1, <2 x i64> %val2) {
+; CHECK-LABEL: f1:
+; CHECK: vmng %v24, {{%v24, %v26|%v26, %v24}}
+; CHECK: br %r14
+  %cmp = icmp slt <2 x i64> %val2, %val1
+  %ret = select <2 x i1> %cmp, <2 x i64> %val2, <2 x i64> %val1
+  ret <2 x i64> %ret
+}
+
+; Test with sle.
+define <2 x i64> @f2(<2 x i64> %val1, <2 x i64> %val2) {
+; CHECK-LABEL: f2:
+; CHECK: vmng %v24, {{%v24, %v26|%v26, %v24}}
+; CHECK: br %r14
+  %cmp = icmp sle <2 x i64> %val2, %val1
+  %ret = select <2 x i1> %cmp, <2 x i64> %val2, <2 x i64> %val1
+  ret <2 x i64> %ret
+}
+
+; Test with sgt.
+define <2 x i64> @f3(<2 x i64> %val1, <2 x i64> %val2) {
+; CHECK-LABEL: f3:
+; CHECK: vmng %v24, {{%v24, %v26|%v26, %v24}}
+; CHECK: br %r14
+  %cmp = icmp sgt <2 x i64> %val2, %val1
+  %ret = select <2 x i1> %cmp, <2 x i64> %val1, <2 x i64> %val2
+  ret <2 x i64> %ret
+}
+
+; Test with sge.
+define <2 x i64> @f4(<2 x i64> %val1, <2 x i64> %val2) {
+; CHECK-LABEL: f4:
+; CHECK: vmng %v24, {{%v24, %v26|%v26, %v24}}
+; CHECK: br %r14
+  %cmp = icmp sge <2 x i64> %val2, %val1
+  %ret = select <2 x i1> %cmp, <2 x i64> %val1, <2 x i64> %val2
+  ret <2 x i64> %ret
+}
+
+; Test with ult.
+define <2 x i64> @f5(<2 x i64> %val1, <2 x i64> %val2) {
+; CHECK-LABEL: f5:
+; CHECK: vmnlg %v24, {{%v24, %v26|%v26, %v24}}
+; CHECK: br %r14
+  %cmp = icmp ult <2 x i64> %val2, %val1
+  %ret = select <2 x i1> %cmp, <2 x i64> %val2, <2 x i64> %val1
+  ret <2 x i64> %ret
+}
+
+; Test with ule.
+define <2 x i64> @f6(<2 x i64> %val1, <2 x i64> %val2) {
+; CHECK-LABEL: f6:
+; CHECK: vmnlg %v24, {{%v24, %v26|%v26, %v24}}
+; CHECK: br %r14
+  %cmp = icmp ule <2 x i64> %val2, %val1
+  %ret = select <2 x i1> %cmp, <2 x i64> %val2, <2 x i64> %val1
+  ret <2 x i64> %ret
+}
+
+; Test with ugt.
+define <2 x i64> @f7(<2 x i64> %val1, <2 x i64> %val2) {
+; CHECK-LABEL: f7:
+; CHECK: vmnlg %v24, {{%v24, %v26|%v26, %v24}}
+; CHECK: br %r14
+  %cmp = icmp ugt <2 x i64> %val2, %val1
+  %ret = select <2 x i1> %cmp, <2 x i64> %val1, <2 x i64> %val2
+  ret <2 x i64> %ret
+}
+
+; Test with uge.
+define <2 x i64> @f8(<2 x i64> %val1, <2 x i64> %val2) {
+; CHECK-LABEL: f8:
+; CHECK: vmnlg %v24, {{%v24, %v26|%v26, %v24}}
+; CHECK: br %r14
+  %cmp = icmp uge <2 x i64> %val2, %val1
+  %ret = select <2 x i1> %cmp, <2 x i64> %val1, <2 x i64> %val2
+  ret <2 x i64> %ret
+}
diff --git a/test/CodeGen/SystemZ/vec-move-01.ll b/test/CodeGen/SystemZ/vec-move-01.ll
new file mode 100644
index 00000000000..3ef98b7eda0
--- /dev/null
+++ b/test/CodeGen/SystemZ/vec-move-01.ll
@@ -0,0 +1,107 @@
+; Test vector register moves.
+;
+; RUN: llc < %s -mtriple=s390x-linux-gnu -mcpu=z13 | FileCheck %s
+
+; Test v16i8 moves.
+define <16 x i8> @f1(<16 x i8> %val1, <16 x i8> %val2) {
+; CHECK-LABEL: f1:
+; CHECK: vlr %v24, %v26
+; CHECK: br %r14
+  ret <16 x i8> %val2
+}
+
+; Test v8i16 moves.
+define <8 x i16> @f2(<8 x i16> %val1, <8 x i16> %val2) {
+; CHECK-LABEL: f2:
+; CHECK: vlr %v24, %v26
+; CHECK: br %r14
+  ret <8 x i16> %val2
+}
+
+; Test v4i32 moves.
+define <4 x i32> @f3(<4 x i32> %val1, <4 x i32> %val2) {
+; CHECK-LABEL: f3:
+; CHECK: vlr %v24, %v26
+; CHECK: br %r14
+  ret <4 x i32> %val2
+}
+
+; Test v2i64 moves.
+define <2 x i64> @f4(<2 x i64> %val1, <2 x i64> %val2) {
+; CHECK-LABEL: f4:
+; CHECK: vlr %v24, %v26
+; CHECK: br %r14
+  ret <2 x i64> %val2
+}
+
+; Test v4f32 moves.
+define <4 x float> @f5(<4 x float> %val1, <4 x float> %val2) {
+; CHECK-LABEL: f5:
+; CHECK: vlr %v24, %v26
+; CHECK: br %r14
+  ret <4 x float> %val2
+}
+
+; Test v2f64 moves.
+define <2 x double> @f6(<2 x double> %val1, <2 x double> %val2) {
+; CHECK-LABEL: f6:
+; CHECK: vlr %v24, %v26
+; CHECK: br %r14
+  ret <2 x double> %val2
+}
+
+; Test v2i8 moves.
+define <2 x i8> @f7(<2 x i8> %val1, <2 x i8> %val2) {
+; CHECK-LABEL: f7:
+; CHECK: vlr %v24, %v26
+; CHECK: br %r14
+  ret <2 x i8> %val2
+}
+
+; Test v4i8 moves.
+define <4 x i8> @f8(<4 x i8> %val1, <4 x i8> %val2) {
+; CHECK-LABEL: f8:
+; CHECK: vlr %v24, %v26
+; CHECK: br %r14
+  ret <4 x i8> %val2
+}
+
+; Test v8i8 moves.
+define <8 x i8> @f9(<8 x i8> %val1, <8 x i8> %val2) {
+; CHECK-LABEL: f9:
+; CHECK: vlr %v24, %v26
+; CHECK: br %r14
+  ret <8 x i8> %val2
+}
+
+; Test v2i16 moves.
+define <2 x i16> @f10(<2 x i16> %val1, <2 x i16> %val2) {
+; CHECK-LABEL: f10:
+; CHECK: vlr %v24, %v26
+; CHECK: br %r14
+  ret <2 x i16> %val2
+}
+
+; Test v4i16 moves.
+define <4 x i16> @f11(<4 x i16> %val1, <4 x i16> %val2) {
+; CHECK-LABEL: f11:
+; CHECK: vlr %v24, %v26
+; CHECK: br %r14
+  ret <4 x i16> %val2
+}
+
+; Test v2i32 moves.
+define <2 x i32> @f12(<2 x i32> %val1, <2 x i32> %val2) {
+; CHECK-LABEL: f12:
+; CHECK: vlr %v24, %v26
+; CHECK: br %r14
+  ret <2 x i32> %val2
+}
+
+; Test v2f32 moves.
+define <2 x float> @f13(<2 x float> %val1, <2 x float> %val2) {
+; CHECK-LABEL: f13:
+; CHECK: vlr %v24, %v26
+; CHECK: br %r14
+  ret <2 x float> %val2
+}
diff --git a/test/CodeGen/SystemZ/vec-move-02.ll b/test/CodeGen/SystemZ/vec-move-02.ll
new file mode 100644
index 00000000000..dcaf0acccb2
--- /dev/null
+++ b/test/CodeGen/SystemZ/vec-move-02.ll
@@ -0,0 +1,174 @@
+; Test vector loads.
+;
+; RUN: llc < %s -mtriple=s390x-linux-gnu -mcpu=z13 | FileCheck %s
+
+; Test v16i8 loads.
+define <16 x i8> @f1(<16 x i8> *%ptr) {
+; CHECK-LABEL: f1:
+; CHECK: vl %v24, 0(%r2)
+; CHECK: br %r14
+  %ret = load <16 x i8>, <16 x i8> *%ptr
+  ret <16 x i8> %ret
+}
+
+; Test v8i16 loads.
+define <8 x i16> @f2(<8 x i16> *%ptr) {
+; CHECK-LABEL: f2:
+; CHECK: vl %v24, 0(%r2)
+; CHECK: br %r14
+  %ret = load <8 x i16>, <8 x i16> *%ptr
+  ret <8 x i16> %ret
+}
+
+; Test v4i32 loads.
+define <4 x i32> @f3(<4 x i32> *%ptr) {
+; CHECK-LABEL: f3:
+; CHECK: vl %v24, 0(%r2)
+; CHECK: br %r14
+  %ret = load <4 x i32>, <4 x i32> *%ptr
+  ret <4 x i32> %ret
+}
+
+; Test v2i64 loads.
+define <2 x i64> @f4(<2 x i64> *%ptr) {
+; CHECK-LABEL: f4:
+; CHECK: vl %v24, 0(%r2)
+; CHECK: br %r14
+  %ret = load <2 x i64>, <2 x i64> *%ptr
+  ret <2 x i64> %ret
+}
+
+; Test v4f32 loads.
+define <4 x float> @f5(<4 x float> *%ptr) {
+; CHECK-LABEL: f5:
+; CHECK: vl %v24, 0(%r2)
+; CHECK: br %r14
+  %ret = load <4 x float>, <4 x float> *%ptr
+  ret <4 x float> %ret
+}
+
+; Test v2f64 loads.
+define <2 x double> @f6(<2 x double> *%ptr) {
+; CHECK-LABEL: f6:
+; CHECK: vl %v24, 0(%r2)
+; CHECK: br %r14
+  %ret = load <2 x double>, <2 x double> *%ptr
+  ret <2 x double> %ret
+}
+
+; Test the highest aligned in-range offset.
+define <16 x i8> @f7(<16 x i8> *%base) {
+; CHECK-LABEL: f7:
+; CHECK: vl %v24, 4080(%r2)
+; CHECK: br %r14
+  %ptr = getelementptr <16 x i8>, <16 x i8> *%base, i64 255
+  %ret = load <16 x i8>, <16 x i8> *%ptr
+  ret <16 x i8> %ret
+}
+
+; Test the highest unaligned in-range offset.
+define <16 x i8> @f8(i8 *%base) {
+; CHECK-LABEL: f8:
+; CHECK: vl %v24, 4095(%r2)
+; CHECK: br %r14
+  %addr = getelementptr i8, i8 *%base, i64 4095
+  %ptr = bitcast i8 *%addr to <16 x i8> *
+  %ret = load <16 x i8>, <16 x i8> *%ptr, align 1
+  ret <16 x i8> %ret
+}
+
+; Test the next offset up, which requires separate address logic,
+define <16 x i8> @f9(<16 x i8> *%base) {
+; CHECK-LABEL: f9:
+; CHECK: aghi %r2, 4096
+; CHECK: vl %v24, 0(%r2)
+; CHECK: br %r14
+  %ptr = getelementptr <16 x i8>, <16 x i8> *%base, i64 256
+  %ret = load <16 x i8>, <16 x i8> *%ptr
+  ret <16 x i8> %ret
+}
+
+; Test negative offsets, which also require separate address logic,
+define <16 x i8> @f10(<16 x i8> *%base) {
+; CHECK-LABEL: f10:
+; CHECK: aghi %r2, -16
+; CHECK: vl %v24, 0(%r2)
+; CHECK: br %r14
+  %ptr = getelementptr <16 x i8>, <16 x i8> *%base, i64 -1
+  %ret = load <16 x i8>, <16 x i8> *%ptr
+  ret <16 x i8> %ret
+}
+
+; Check that indexes are allowed.
+define <16 x i8> @f11(i8 *%base, i64 %index) {
+; CHECK-LABEL: f11:
+; CHECK: vl %v24, 0(%r3,%r2)
+; CHECK: br %r14
+  %addr = getelementptr i8, i8 *%base, i64 %index
+  %ptr = bitcast i8 *%addr to <16 x i8> *
+  %ret = load <16 x i8>, <16 x i8> *%ptr, align 1
+  ret <16 x i8> %ret
+}
+
+; Test v2i8 loads.
+define <2 x i8> @f12(<2 x i8> *%ptr) {
+; CHECK-LABEL: f12:
+; CHECK: vlreph %v24, 0(%r2)
+; CHECK: br %r14
+  %ret = load <2 x i8>, <2 x i8> *%ptr
+  ret <2 x i8> %ret
+}
+
+; Test v4i8 loads.
+define <4 x i8> @f13(<4 x i8> *%ptr) {
+; CHECK-LABEL: f13:
+; CHECK: vlrepf %v24, 0(%r2)
+; CHECK: br %r14
+  %ret = load <4 x i8>, <4 x i8> *%ptr
+  ret <4 x i8> %ret
+}
+
+; Test v8i8 loads.
+define <8 x i8> @f14(<8 x i8> *%ptr) {
+; CHECK-LABEL: f14:
+; CHECK: vlrepg %v24, 0(%r2)
+; CHECK: br %r14
+  %ret = load <8 x i8>, <8 x i8> *%ptr
+  ret <8 x i8> %ret
+}
+
+; Test v2i16 loads.
+define <2 x i16> @f15(<2 x i16> *%ptr) {
+; CHECK-LABEL: f15:
+; CHECK: vlrepf %v24, 0(%r2)
+; CHECK: br %r14
+  %ret = load <2 x i16>, <2 x i16> *%ptr
+  ret <2 x i16> %ret
+}
+
+; Test v4i16 loads.
+define <4 x i16> @f16(<4 x i16> *%ptr) {
+; CHECK-LABEL: f16:
+; CHECK: vlrepg %v24, 0(%r2)
+; CHECK: br %r14
+  %ret = load <4 x i16>, <4 x i16> *%ptr
+  ret <4 x i16> %ret
+}
+
+; Test v2i32 loads.
+define <2 x i32> @f17(<2 x i32> *%ptr) {
+; CHECK-LABEL: f17:
+; CHECK: vlrepg %v24, 0(%r2)
+; CHECK: br %r14
+  %ret = load <2 x i32>, <2 x i32> *%ptr
+  ret <2 x i32> %ret
+}
+
+; Test v2f32 loads.
+define <2 x float> @f18(<2 x float> *%ptr) {
+; CHECK-LABEL: f18:
+; CHECK: vlrepg %v24, 0(%r2)
+; CHECK: br %r14
+  %ret = load <2 x float>, <2 x float> *%ptr
+  ret <2 x float> %ret
+}
diff --git a/test/CodeGen/SystemZ/vec-move-03.ll b/test/CodeGen/SystemZ/vec-move-03.ll
new file mode 100644
index 00000000000..f40e2cb2bf2
--- /dev/null
+++ b/test/CodeGen/SystemZ/vec-move-03.ll
@@ -0,0 +1,174 @@
+; Test vector stores.
+;
+; RUN: llc < %s -mtriple=s390x-linux-gnu -mcpu=z13 | FileCheck %s
+
+; Test v16i8 stores.
+define void @f1(<16 x i8> %val, <16 x i8> *%ptr) {
+; CHECK-LABEL: f1:
+; CHECK: vst %v24, 0(%r2)
+; CHECK: br %r14
+  store <16 x i8> %val, <16 x i8> *%ptr
+  ret void
+}
+
+; Test v8i16 stores.
+define void @f2(<8 x i16> %val, <8 x i16> *%ptr) {
+; CHECK-LABEL: f2:
+; CHECK: vst %v24, 0(%r2)
+; CHECK: br %r14
+  store <8 x i16> %val, <8 x i16> *%ptr
+  ret void
+}
+
+; Test v4i32 stores.
+define void @f3(<4 x i32> %val, <4 x i32> *%ptr) {
+; CHECK-LABEL: f3:
+; CHECK: vst %v24, 0(%r2)
+; CHECK: br %r14
+  store <4 x i32> %val, <4 x i32> *%ptr
+  ret void
+}
+
+; Test v2i64 stores.
+define void @f4(<2 x i64> %val, <2 x i64> *%ptr) {
+; CHECK-LABEL: f4:
+; CHECK: vst %v24, 0(%r2)
+; CHECK: br %r14
+  store <2 x i64> %val, <2 x i64> *%ptr
+  ret void
+}
+
+; Test v4f32 stores.
+define void @f5(<4 x float> %val, <4 x float> *%ptr) {
+; CHECK-LABEL: f5:
+; CHECK: vst %v24, 0(%r2)
+; CHECK: br %r14
+  store <4 x float> %val, <4 x float> *%ptr
+  ret void
+}
+
+; Test v2f64 stores.
+define void @f6(<2 x double> %val, <2 x double> *%ptr) {
+; CHECK-LABEL: f6:
+; CHECK: vst %v24, 0(%r2)
+; CHECK: br %r14
+  store <2 x double> %val, <2 x double> *%ptr
+  ret void
+}
+
+; Test the highest aligned in-range offset.
+define void @f7(<16 x i8> %val, <16 x i8> *%base) {
+; CHECK-LABEL: f7:
+; CHECK: vst %v24, 4080(%r2)
+; CHECK: br %r14
+  %ptr = getelementptr <16 x i8>, <16 x i8> *%base, i64 255
+  store <16 x i8> %val, <16 x i8> *%ptr
+  ret void
+}
+
+; Test the highest unaligned in-range offset.
+define void @f8(<16 x i8> %val, i8 *%base) {
+; CHECK-LABEL: f8:
+; CHECK: vst %v24, 4095(%r2)
+; CHECK: br %r14
+  %addr = getelementptr i8, i8 *%base, i64 4095
+  %ptr = bitcast i8 *%addr to <16 x i8> *
+  store <16 x i8> %val, <16 x i8> *%ptr, align 1
+  ret void
+}
+
+; Test the next offset up, which requires separate address logic,
+define void @f9(<16 x i8> %val, <16 x i8> *%base) {
+; CHECK-LABEL: f9:
+; CHECK: aghi %r2, 4096
+; CHECK: vst %v24, 0(%r2)
+; CHECK: br %r14
+  %ptr = getelementptr <16 x i8>, <16 x i8> *%base, i64 256
+  store <16 x i8> %val, <16 x i8> *%ptr
+  ret void
+}
+
+; Test negative offsets, which also require separate address logic,
+define void @f10(<16 x i8> %val, <16 x i8> *%base) {
+; CHECK-LABEL: f10:
+; CHECK: aghi %r2, -16
+; CHECK: vst %v24, 0(%r2)
+; CHECK: br %r14
+  %ptr = getelementptr <16 x i8>, <16 x i8> *%base, i64 -1
+  store <16 x i8> %val, <16 x i8> *%ptr
+  ret void
+}
+
+; Check that indexes are allowed.
+define void @f11(<16 x i8> %val, i8 *%base, i64 %index) {
+; CHECK-LABEL: f11:
+; CHECK: vst %v24, 0(%r3,%r2)
+; CHECK: br %r14
+  %addr = getelementptr i8, i8 *%base, i64 %index
+  %ptr = bitcast i8 *%addr to <16 x i8> *
+  store <16 x i8> %val, <16 x i8> *%ptr, align 1
+  ret void
+}
+
+; Test v2i8 stores.
+define void @f12(<2 x i8> %val, <2 x i8> *%ptr) {
+; CHECK-LABEL: f12:
+; CHECK: vsteh %v24, 0(%r2), 0
+; CHECK: br %r14
+  store <2 x i8> %val, <2 x i8> *%ptr
+  ret void
+}
+
+; Test v4i8 stores.
+define void @f13(<4 x i8> %val, <4 x i8> *%ptr) {
+; CHECK-LABEL: f13:
+; CHECK: vstef %v24, 0(%r2)
+; CHECK: br %r14
+  store <4 x i8> %val, <4 x i8> *%ptr
+  ret void
+}
+
+; Test v8i8 stores.
+define void @f14(<8 x i8> %val, <8 x i8> *%ptr) {
+; CHECK-LABEL: f14:
+; CHECK: vsteg %v24, 0(%r2)
+; CHECK: br %r14
+  store <8 x i8> %val, <8 x i8> *%ptr
+  ret void
+}
+
+; Test v2i16 stores.
+define void @f15(<2 x i16> %val, <2 x i16> *%ptr) {
+; CHECK-LABEL: f15:
+; CHECK: vstef %v24, 0(%r2), 0
+; CHECK: br %r14
+  store <2 x i16> %val, <2 x i16> *%ptr
+  ret void
+}
+
+; Test v4i16 stores.
+define void @f16(<4 x i16> %val, <4 x i16> *%ptr) {
+; CHECK-LABEL: f16:
+; CHECK: vsteg %v24, 0(%r2)
+; CHECK: br %r14
+  store <4 x i16> %val, <4 x i16> *%ptr
+  ret void
+}
+
+; Test v2i32 stores.
+define void @f17(<2 x i32> %val, <2 x i32> *%ptr) {
+; CHECK-LABEL: f17:
+; CHECK: vsteg %v24, 0(%r2), 0
+; CHECK: br %r14
+  store <2 x i32> %val, <2 x i32> *%ptr
+  ret void
+}
+
+; Test v2f32 stores.
+define void @f18(<2 x float> %val, <2 x float> *%ptr) {
+; CHECK-LABEL: f18:
+; CHECK: vsteg %v24, 0(%r2), 0
+; CHECK: br %r14
+  store <2 x float> %val, <2 x float> *%ptr
+  ret void
+}
diff --git a/test/CodeGen/SystemZ/vec-move-04.ll b/test/CodeGen/SystemZ/vec-move-04.ll
new file mode 100644
index 00000000000..27c9e5f71f4
--- /dev/null
+++ b/test/CodeGen/SystemZ/vec-move-04.ll
@@ -0,0 +1,179 @@
+; Test vector insertion of register variables.
+;
+; RUN: llc < %s -mtriple=s390x-linux-gnu -mcpu=z13 | FileCheck %s
+
+; Test v16i8 insertion into the first element.
+define <16 x i8> @f1(<16 x i8> %val, i8 %element) {
+; CHECK-LABEL: f1:
+; CHECK: vlvgb %v24, %r2, 0
+; CHECK: br %r14
+  %ret = insertelement <16 x i8> %val, i8 %element, i32 0
+  ret <16 x i8> %ret
+}
+
+; Test v16i8 insertion into the last element.
+define <16 x i8> @f2(<16 x i8> %val, i8 %element) {
+; CHECK-LABEL: f2:
+; CHECK: vlvgb %v24, %r2, 15
+; CHECK: br %r14
+  %ret = insertelement <16 x i8> %val, i8 %element, i32 15
+  ret <16 x i8> %ret
+}
+
+; Test v16i8 insertion into a variable element.
+define <16 x i8> @f3(<16 x i8> %val, i8 %element, i32 %index) {
+; CHECK-LABEL: f3:
+; CHECK: vlvgb %v24, %r2, 0(%r3)
+; CHECK: br %r14
+  %ret = insertelement <16 x i8> %val, i8 %element, i32 %index
+  ret <16 x i8> %ret
+}
+
+; Test v8i16 insertion into the first element.
+define <8 x i16> @f4(<8 x i16> %val, i16 %element) {
+; CHECK-LABEL: f4:
+; CHECK: vlvgh %v24, %r2, 0
+; CHECK: br %r14
+  %ret = insertelement <8 x i16> %val, i16 %element, i32 0
+  ret <8 x i16> %ret
+}
+
+; Test v8i16 insertion into the last element.
+define <8 x i16> @f5(<8 x i16> %val, i16 %element) {
+; CHECK-LABEL: f5:
+; CHECK: vlvgh %v24, %r2, 7
+; CHECK: br %r14
+  %ret = insertelement <8 x i16> %val, i16 %element, i32 7
+  ret <8 x i16> %ret
+}
+
+; Test v8i16 insertion into a variable element.
+define <8 x i16> @f6(<8 x i16> %val, i16 %element, i32 %index) {
+; CHECK-LABEL: f6:
+; CHECK: vlvgh %v24, %r2, 0(%r3)
+; CHECK: br %r14
+  %ret = insertelement <8 x i16> %val, i16 %element, i32 %index
+  ret <8 x i16> %ret
+}
+
+; Test v4i32 insertion into the first element.
+define <4 x i32> @f7(<4 x i32> %val, i32 %element) {
+; CHECK-LABEL: f7:
+; CHECK: vlvgf %v24, %r2, 0
+; CHECK: br %r14
+  %ret = insertelement <4 x i32> %val, i32 %element, i32 0
+  ret <4 x i32> %ret
+}
+
+; Test v4i32 insertion into the last element.
+define <4 x i32> @f8(<4 x i32> %val, i32 %element) {
+; CHECK-LABEL: f8:
+; CHECK: vlvgf %v24, %r2, 3
+; CHECK: br %r14
+  %ret = insertelement <4 x i32> %val, i32 %element, i32 3
+  ret <4 x i32> %ret
+}
+
+; Test v4i32 insertion into a variable element.
+define <4 x i32> @f9(<4 x i32> %val, i32 %element, i32 %index) {
+; CHECK-LABEL: f9:
+; CHECK: vlvgf %v24, %r2, 0(%r3)
+; CHECK: br %r14
+  %ret = insertelement <4 x i32> %val, i32 %element, i32 %index
+  ret <4 x i32> %ret
+}
+
+; Test v2i64 insertion into the first element.
+define <2 x i64> @f10(<2 x i64> %val, i64 %element) {
+; CHECK-LABEL: f10:
+; CHECK: vlvgg %v24, %r2, 0
+; CHECK: br %r14
+  %ret = insertelement <2 x i64> %val, i64 %element, i32 0
+  ret <2 x i64> %ret
+}
+
+; Test v2i64 insertion into the last element.
+define <2 x i64> @f11(<2 x i64> %val, i64 %element) {
+; CHECK-LABEL: f11:
+; CHECK: vlvgg %v24, %r2, 1
+; CHECK: br %r14
+  %ret = insertelement <2 x i64> %val, i64 %element, i32 1
+  ret <2 x i64> %ret
+}
+
+; Test v2i64 insertion into a variable element.
+define <2 x i64> @f12(<2 x i64> %val, i64 %element, i32 %index) {
+; CHECK-LABEL: f12:
+; CHECK: vlvgg %v24, %r2, 0(%r3)
+; CHECK: br %r14
+  %ret = insertelement <2 x i64> %val, i64 %element, i32 %index
+  ret <2 x i64> %ret
+}
+
+; Test v4f32 insertion into the first element.
+define <4 x float> @f13(<4 x float> %val, float %element) {
+; CHECK-LABEL: f13:
+; CHECK: vlgvf [[REG:%r[0-5]]], %v0, 0
+; CHECK: vlvgf %v24, [[REG]], 0
+; CHECK: br %r14
+  %ret = insertelement <4 x float> %val, float %element, i32 0
+  ret <4 x float> %ret
+}
+
+; Test v4f32 insertion into the last element.
+define <4 x float> @f14(<4 x float> %val, float %element) {
+; CHECK-LABEL: f14:
+; CHECK: vlgvf [[REG:%r[0-5]]], %v0, 0
+; CHECK: vlvgf %v24, [[REG]], 3
+; CHECK: br %r14
+  %ret = insertelement <4 x float> %val, float %element, i32 3
+  ret <4 x float> %ret
+}
+
+; Test v4f32 insertion into a variable element.
+define <4 x float> @f15(<4 x float> %val, float %element, i32 %index) {
+; CHECK-LABEL: f15:
+; CHECK: vlgvf [[REG:%r[0-5]]], %v0, 0
+; CHECK: vlvgf %v24, [[REG]], 0(%r2)
+; CHECK: br %r14
+  %ret = insertelement <4 x float> %val, float %element, i32 %index
+  ret <4 x float> %ret
+}
+
+; Test v2f64 insertion into the first element.
+define <2 x double> @f16(<2 x double> %val, double %element) {
+; CHECK-LABEL: f16:
+; CHECK: vpdi %v24, %v0, %v24, 1
+; CHECK: br %r14
+  %ret = insertelement <2 x double> %val, double %element, i32 0
+  ret <2 x double> %ret
+}
+
+; Test v2f64 insertion into the last element.
+define <2 x double> @f17(<2 x double> %val, double %element) {
+; CHECK-LABEL: f17:
+; CHECK: vpdi %v24, %v24, %v0, 0
+; CHECK: br %r14
+  %ret = insertelement <2 x double> %val, double %element, i32 1
+  ret <2 x double> %ret
+}
+
+; Test v2f64 insertion into a variable element.
+define <2 x double> @f18(<2 x double> %val, double %element, i32 %index) {
+; CHECK-LABEL: f18:
+; CHECK: lgdr [[REG:%r[0-5]]], %f0
+; CHECK: vlvgg %v24, [[REG]], 0(%r2)
+; CHECK: br %r14
+  %ret = insertelement <2 x double> %val, double %element, i32 %index
+  ret <2 x double> %ret
+}
+
+; Test v16i8 insertion into a variable element plus one.
+define <16 x i8> @f19(<16 x i8> %val, i8 %element, i32 %index) {
+; CHECK-LABEL: f19:
+; CHECK: vlvgb %v24, %r2, 1(%r3)
+; CHECK: br %r14
+  %add = add i32 %index, 1
+  %ret = insertelement <16 x i8> %val, i8 %element, i32 %add
+  ret <16 x i8> %ret
+}
diff --git a/test/CodeGen/SystemZ/vec-move-05.ll b/test/CodeGen/SystemZ/vec-move-05.ll
new file mode 100644
index 00000000000..99871196d68
--- /dev/null
+++ b/test/CodeGen/SystemZ/vec-move-05.ll
@@ -0,0 +1,249 @@
+; Test vector extraction.
+;
+; RUN: llc < %s -mtriple=s390x-linux-gnu -mcpu=z13 | FileCheck %s
+
+; Test v16i8 extraction of the first element.
+define i8 @f1(<16 x i8> %val) {
+; CHECK-LABEL: f1:
+; CHECK: vlgvb %r2, %v24, 0
+; CHECK: br %r14
+  %ret = extractelement <16 x i8> %val, i32 0
+  ret i8 %ret
+}
+
+; Test v16i8 extraction of the last element.
+define i8 @f2(<16 x i8> %val) {
+; CHECK-LABEL: f2:
+; CHECK: vlgvb %r2, %v24, 15
+; CHECK: br %r14
+  %ret = extractelement <16 x i8> %val, i32 15
+  ret i8 %ret
+}
+
+; Test v16i8 extractions of an absurd element number.  This must compile
+; but we don't care what it does.
+define i8 @f3(<16 x i8> %val) {
+; CHECK-LABEL: f3:
+; CHECK-NOT: vlgvb %r2, %v24, 100000
+; CHECK: br %r14
+  %ret = extractelement <16 x i8> %val, i32 100000
+  ret i8 %ret
+}
+
+; Test v16i8 extraction of a variable element.
+define i8 @f4(<16 x i8> %val, i32 %index) {
+; CHECK-LABEL: f4:
+; CHECK: vlgvb %r2, %v24, 0(%r2)
+; CHECK: br %r14
+  %ret = extractelement <16 x i8> %val, i32 %index
+  ret i8 %ret
+}
+
+; Test v8i16 extraction of the first element.
+define i16 @f5(<8 x i16> %val) {
+; CHECK-LABEL: f5:
+; CHECK: vlgvh %r2, %v24, 0
+; CHECK: br %r14
+  %ret = extractelement <8 x i16> %val, i32 0
+  ret i16 %ret
+}
+
+; Test v8i16 extraction of the last element.
+define i16 @f6(<8 x i16> %val) {
+; CHECK-LABEL: f6:
+; CHECK: vlgvh %r2, %v24, 7
+; CHECK: br %r14
+  %ret = extractelement <8 x i16> %val, i32 7
+  ret i16 %ret
+}
+
+; Test v8i16 extractions of an absurd element number.  This must compile
+; but we don't care what it does.
+define i16 @f7(<8 x i16> %val) {
+; CHECK-LABEL: f7:
+; CHECK-NOT: vlgvh %r2, %v24, 100000
+; CHECK: br %r14
+  %ret = extractelement <8 x i16> %val, i32 100000
+  ret i16 %ret
+}
+
+; Test v8i16 extraction of a variable element.
+define i16 @f8(<8 x i16> %val, i32 %index) {
+; CHECK-LABEL: f8:
+; CHECK: vlgvh %r2, %v24, 0(%r2)
+; CHECK: br %r14
+  %ret = extractelement <8 x i16> %val, i32 %index
+  ret i16 %ret
+}
+
+; Test v4i32 extraction of the first element.
+define i32 @f9(<4 x i32> %val) {
+; CHECK-LABEL: f9:
+; CHECK: vlgvf %r2, %v24, 0
+; CHECK: br %r14
+  %ret = extractelement <4 x i32> %val, i32 0
+  ret i32 %ret
+}
+
+; Test v4i32 extraction of the last element.
+define i32 @f10(<4 x i32> %val) {
+; CHECK-LABEL: f10:
+; CHECK: vlgvf %r2, %v24, 3
+; CHECK: br %r14
+  %ret = extractelement <4 x i32> %val, i32 3
+  ret i32 %ret
+}
+
+; Test v4i32 extractions of an absurd element number.  This must compile
+; but we don't care what it does.
+define i32 @f11(<4 x i32> %val) {
+; CHECK-LABEL: f11:
+; CHECK-NOT: vlgvf %r2, %v24, 100000
+; CHECK: br %r14
+  %ret = extractelement <4 x i32> %val, i32 100000
+  ret i32 %ret
+}
+
+; Test v4i32 extraction of a variable element.
+define i32 @f12(<4 x i32> %val, i32 %index) {
+; CHECK-LABEL: f12:
+; CHECK: vlgvf %r2, %v24, 0(%r2)
+; CHECK: br %r14
+  %ret = extractelement <4 x i32> %val, i32 %index
+  ret i32 %ret
+}
+
+; Test v2i64 extraction of the first element.
+define i64 @f13(<2 x i64> %val) {
+; CHECK-LABEL: f13:
+; CHECK: vlgvg %r2, %v24, 0
+; CHECK: br %r14
+  %ret = extractelement <2 x i64> %val, i32 0
+  ret i64 %ret
+}
+
+; Test v2i64 extraction of the last element.
+define i64 @f14(<2 x i64> %val) {
+; CHECK-LABEL: f14:
+; CHECK: vlgvg %r2, %v24, 1
+; CHECK: br %r14
+  %ret = extractelement <2 x i64> %val, i32 1
+  ret i64 %ret
+}
+
+; Test v2i64 extractions of an absurd element number.  This must compile
+; but we don't care what it does.
+define i64 @f15(<2 x i64> %val) {
+; CHECK-LABEL: f15:
+; CHECK-NOT: vlgvg %r2, %v24, 100000
+; CHECK: br %r14
+  %ret = extractelement <2 x i64> %val, i32 100000
+  ret i64 %ret
+}
+
+; Test v2i64 extraction of a variable element.
+define i64 @f16(<2 x i64> %val, i32 %index) {
+; CHECK-LABEL: f16:
+; CHECK: vlgvg %r2, %v24, 0(%r2)
+; CHECK: br %r14
+  %ret = extractelement <2 x i64> %val, i32 %index
+  ret i64 %ret
+}
+
+; Test v4f32 extraction of element 0.
+define float @f17(<4 x float> %val) {
+; CHECK-LABEL: f17:
+; CHECK: vlr %v0, %v24
+; CHECK: br %r14
+  %ret = extractelement <4 x float> %val, i32 0
+  ret float %ret
+}
+
+; Test v4f32 extraction of element 1.
+define float @f18(<4 x float> %val) {
+; CHECK-LABEL: f18:
+; CHECK: vrepf %v0, %v24, 1
+; CHECK: br %r14
+  %ret = extractelement <4 x float> %val, i32 1
+  ret float %ret
+}
+
+; Test v4f32 extraction of element 2.
+define float @f19(<4 x float> %val) {
+; CHECK-LABEL: f19:
+; CHECK: vrepf %v0, %v24, 2
+; CHECK: br %r14
+  %ret = extractelement <4 x float> %val, i32 2
+  ret float %ret
+}
+
+; Test v4f32 extraction of element 3.
+define float @f20(<4 x float> %val) {
+; CHECK-LABEL: f20:
+; CHECK: vrepf %v0, %v24, 3
+; CHECK: br %r14
+  %ret = extractelement <4 x float> %val, i32 3
+  ret float %ret
+}
+
+; Test v4f32 extractions of an absurd element number.  This must compile
+; but we don't care what it does.
+define float @f21(<4 x float> %val) {
+  %ret = extractelement <4 x float> %val, i32 100000
+  ret float %ret
+}
+
+; Test v4f32 extraction of a variable element.
+define float @f22(<4 x float> %val, i32 %index) {
+; CHECK-LABEL: f22:
+; CHECK: vlgvf [[REG:%r[0-5]]], %v24, 0(%r2)
+; CHECK: vlvgf %v0, [[REG]], 0
+; CHECK: br %r14
+  %ret = extractelement <4 x float> %val, i32 %index
+  ret float %ret
+}
+
+; Test v2f64 extraction of the first element.
+define double @f23(<2 x double> %val) {
+; CHECK-LABEL: f23:
+; CHECK: vlr %v0, %v24
+; CHECK: br %r14
+  %ret = extractelement <2 x double> %val, i32 0
+  ret double %ret
+}
+
+; Test v2f64 extraction of the last element.
+define double @f24(<2 x double> %val) {
+; CHECK-LABEL: f24:
+; CHECK: vrepg %v0, %v24, 1
+; CHECK: br %r14
+  %ret = extractelement <2 x double> %val, i32 1
+  ret double %ret
+}
+
+; Test v2f64 extractions of an absurd element number.  This must compile
+; but we don't care what it does.
+define double @f25(<2 x double> %val) {
+  %ret = extractelement <2 x double> %val, i32 100000
+  ret double %ret
+}
+
+; Test v2f64 extraction of a variable element.
+define double @f26(<2 x double> %val, i32 %index) {
+; CHECK-LABEL: f26:
+; CHECK: vlgvg [[REG:%r[0-5]]], %v24, 0(%r2)
+; CHECK: ldgr %f0, [[REG]]
+; CHECK: br %r14
+  %ret = extractelement <2 x double> %val, i32 %index
+  ret double %ret
+}
+
+; Test v16i8 extraction of a variable element with an offset.
+define i8 @f27(<16 x i8> %val, i32 %index) {
+; CHECK-LABEL: f27:
+; CHECK: vlgvb %r2, %v24, 1(%r2)
+; CHECK: br %r14
+  %add = add i32 %index, 1
+  %ret = extractelement <16 x i8> %val, i32 %add
+  ret i8 %ret
+}
diff --git a/test/CodeGen/SystemZ/vec-move-06.ll b/test/CodeGen/SystemZ/vec-move-06.ll
new file mode 100644
index 00000000000..de3960cad95
--- /dev/null
+++ b/test/CodeGen/SystemZ/vec-move-06.ll
@@ -0,0 +1,13 @@
+; Test vector builds using VLVGP.
+;
+; RUN: llc < %s -mtriple=s390x-linux-gnu -mcpu=z13 | FileCheck %s
+
+; Test the basic v2i64 usage.
+define <2 x i64> @f1(i64 %a, i64 %b) {
+; CHECK-LABEL: f1:
+; CHECK: vlvgp %v24, %r2, %r3
+; CHECK: br %r14
+  %veca = insertelement <2 x i64> undef, i64 %a, i32 0
+  %vecb = insertelement <2 x i64> %veca, i64 %b, i32 1
+  ret <2 x i64> %vecb
+}
diff --git a/test/CodeGen/SystemZ/vec-move-07.ll b/test/CodeGen/SystemZ/vec-move-07.ll
new file mode 100644
index 00000000000..b0d06f782de
--- /dev/null
+++ b/test/CodeGen/SystemZ/vec-move-07.ll
@@ -0,0 +1,57 @@
+; Test scalar_to_vector expansion.
+;
+; RUN: llc < %s -mtriple=s390x-linux-gnu -mcpu=z13 | FileCheck %s
+
+; Test v16i8.
+define <16 x i8> @f1(i8 %val) {
+; CHECK-LABEL: f1:
+; CHECK: vlvgb %v24, %r2, 0
+; CHECK: br %r14
+  %ret = insertelement <16 x i8> undef, i8 %val, i32 0
+  ret <16 x i8> %ret
+}
+
+; Test v8i16.
+define <8 x i16> @f2(i16 %val) {
+; CHECK-LABEL: f2:
+; CHECK: vlvgh %v24, %r2, 0
+; CHECK: br %r14
+  %ret = insertelement <8 x i16> undef, i16 %val, i32 0
+  ret <8 x i16> %ret
+}
+
+; Test v4i32.
+define <4 x i32> @f3(i32 %val) {
+; CHECK-LABEL: f3:
+; CHECK: vlvgf %v24, %r2, 0
+; CHECK: br %r14
+  %ret = insertelement <4 x i32> undef, i32 %val, i32 0
+  ret <4 x i32> %ret
+}
+
+; Test v2i64.  Here we load %val into both halves.
+define <2 x i64> @f4(i64 %val) {
+; CHECK-LABEL: f4:
+; CHECK: vlvgp %v24, %r2, %r2
+; CHECK: br %r14
+  %ret = insertelement <2 x i64> undef, i64 %val, i32 0
+  ret <2 x i64> %ret
+}
+
+; Test v4f32, which is just a move.
+define <4 x float> @f5(float %val) {
+; CHECK-LABEL: f5:
+; CHECK: vlr %v24, %v0
+; CHECK: br %r14
+  %ret = insertelement <4 x float> undef, float %val, i32 0
+  ret <4 x float> %ret
+}
+
+; Likewise v2f64.
+define <2 x double> @f6(double %val) {
+; CHECK-LABEL: f6:
+; CHECK: vlr %v24, %v0
+; CHECK: br %r14
+  %ret = insertelement <2 x double> undef, double %val, i32 0
+  ret <2 x double> %ret
+}
diff --git a/test/CodeGen/SystemZ/vec-move-08.ll b/test/CodeGen/SystemZ/vec-move-08.ll
new file mode 100644
index 00000000000..5396a1edec6
--- /dev/null
+++ b/test/CodeGen/SystemZ/vec-move-08.ll
@@ -0,0 +1,444 @@
+; Test vector insertion of memory values.
+;
+; RUN: llc < %s -mtriple=s390x-linux-gnu -mcpu=z13 | FileCheck %s
+
+; Test v16i8 insertion into the first element.
+define <16 x i8> @f1(<16 x i8> %val, i8 *%ptr) {
+; CHECK-LABEL: f1:
+; CHECK: vleb %v24, 0(%r2), 0
+; CHECK: br %r14
+  %element = load i8, i8 *%ptr
+  %ret = insertelement <16 x i8> %val, i8 %element, i32 0
+  ret <16 x i8> %ret
+}
+
+; Test v16i8 insertion into the last element.
+define <16 x i8> @f2(<16 x i8> %val, i8 *%ptr) {
+; CHECK-LABEL: f2:
+; CHECK: vleb %v24, 0(%r2), 15
+; CHECK: br %r14
+  %element = load i8, i8 *%ptr
+  %ret = insertelement <16 x i8> %val, i8 %element, i32 15
+  ret <16 x i8> %ret
+}
+
+; Test v16i8 insertion with the highest in-range offset.
+define <16 x i8> @f3(<16 x i8> %val, i8 *%base) {
+; CHECK-LABEL: f3:
+; CHECK: vleb %v24, 4095(%r2), 10
+; CHECK: br %r14
+  %ptr = getelementptr i8, i8 *%base, i32 4095
+  %element = load i8, i8 *%ptr
+  %ret = insertelement <16 x i8> %val, i8 %element, i32 10
+  ret <16 x i8> %ret
+}
+
+; Test v16i8 insertion with the first ouf-of-range offset.
+define <16 x i8> @f4(<16 x i8> %val, i8 *%base) {
+; CHECK-LABEL: f4:
+; CHECK: aghi %r2, 4096
+; CHECK: vleb %v24, 0(%r2), 5
+; CHECK: br %r14
+  %ptr = getelementptr i8, i8 *%base, i32 4096
+  %element = load i8, i8 *%ptr
+  %ret = insertelement <16 x i8> %val, i8 %element, i32 5
+  ret <16 x i8> %ret
+}
+
+; Test v16i8 insertion into a variable element.
+define <16 x i8> @f5(<16 x i8> %val, i8 *%ptr, i32 %index) {
+; CHECK-LABEL: f5:
+; CHECK-NOT: vleb
+; CHECK: br %r14
+  %element = load i8, i8 *%ptr
+  %ret = insertelement <16 x i8> %val, i8 %element, i32 %index
+  ret <16 x i8> %ret
+}
+
+; Test v8i16 insertion into the first element.
+define <8 x i16> @f6(<8 x i16> %val, i16 *%ptr) {
+; CHECK-LABEL: f6:
+; CHECK: vleh %v24, 0(%r2), 0
+; CHECK: br %r14
+  %element = load i16, i16 *%ptr
+  %ret = insertelement <8 x i16> %val, i16 %element, i32 0
+  ret <8 x i16> %ret
+}
+
+; Test v8i16 insertion into the last element.
+define <8 x i16> @f7(<8 x i16> %val, i16 *%ptr) {
+; CHECK-LABEL: f7:
+; CHECK: vleh %v24, 0(%r2), 7
+; CHECK: br %r14
+  %element = load i16, i16 *%ptr
+  %ret = insertelement <8 x i16> %val, i16 %element, i32 7
+  ret <8 x i16> %ret
+}
+
+; Test v8i16 insertion with the highest in-range offset.
+define <8 x i16> @f8(<8 x i16> %val, i16 *%base) {
+; CHECK-LABEL: f8:
+; CHECK: vleh %v24, 4094(%r2), 5
+; CHECK: br %r14
+  %ptr = getelementptr i16, i16 *%base, i32 2047
+  %element = load i16, i16 *%ptr
+  %ret = insertelement <8 x i16> %val, i16 %element, i32 5
+  ret <8 x i16> %ret
+}
+
+; Test v8i16 insertion with the first ouf-of-range offset.
+define <8 x i16> @f9(<8 x i16> %val, i16 *%base) {
+; CHECK-LABEL: f9:
+; CHECK: aghi %r2, 4096
+; CHECK: vleh %v24, 0(%r2), 1
+; CHECK: br %r14
+  %ptr = getelementptr i16, i16 *%base, i32 2048
+  %element = load i16, i16 *%ptr
+  %ret = insertelement <8 x i16> %val, i16 %element, i32 1
+  ret <8 x i16> %ret
+}
+
+; Test v8i16 insertion into a variable element.
+define <8 x i16> @f10(<8 x i16> %val, i16 *%ptr, i32 %index) {
+; CHECK-LABEL: f10:
+; CHECK-NOT: vleh
+; CHECK: br %r14
+  %element = load i16, i16 *%ptr
+  %ret = insertelement <8 x i16> %val, i16 %element, i32 %index
+  ret <8 x i16> %ret
+}
+
+; Test v4i32 insertion into the first element.
+define <4 x i32> @f11(<4 x i32> %val, i32 *%ptr) {
+; CHECK-LABEL: f11:
+; CHECK: vlef %v24, 0(%r2), 0
+; CHECK: br %r14
+  %element = load i32, i32 *%ptr
+  %ret = insertelement <4 x i32> %val, i32 %element, i32 0
+  ret <4 x i32> %ret
+}
+
+; Test v4i32 insertion into the last element.
+define <4 x i32> @f12(<4 x i32> %val, i32 *%ptr) {
+; CHECK-LABEL: f12:
+; CHECK: vlef %v24, 0(%r2), 3
+; CHECK: br %r14
+  %element = load i32, i32 *%ptr
+  %ret = insertelement <4 x i32> %val, i32 %element, i32 3
+  ret <4 x i32> %ret
+}
+
+; Test v4i32 insertion with the highest in-range offset.
+define <4 x i32> @f13(<4 x i32> %val, i32 *%base) {
+; CHECK-LABEL: f13:
+; CHECK: vlef %v24, 4092(%r2), 2
+; CHECK: br %r14
+  %ptr = getelementptr i32, i32 *%base, i32 1023
+  %element = load i32, i32 *%ptr
+  %ret = insertelement <4 x i32> %val, i32 %element, i32 2
+  ret <4 x i32> %ret
+}
+
+; Test v4i32 insertion with the first ouf-of-range offset.
+define <4 x i32> @f14(<4 x i32> %val, i32 *%base) {
+; CHECK-LABEL: f14:
+; CHECK: aghi %r2, 4096
+; CHECK: vlef %v24, 0(%r2), 1
+; CHECK: br %r14
+  %ptr = getelementptr i32, i32 *%base, i32 1024
+  %element = load i32, i32 *%ptr
+  %ret = insertelement <4 x i32> %val, i32 %element, i32 1
+  ret <4 x i32> %ret
+}
+
+; Test v4i32 insertion into a variable element.
+define <4 x i32> @f15(<4 x i32> %val, i32 *%ptr, i32 %index) {
+; CHECK-LABEL: f15:
+; CHECK-NOT: vlef
+; CHECK: br %r14
+  %element = load i32, i32 *%ptr
+  %ret = insertelement <4 x i32> %val, i32 %element, i32 %index
+  ret <4 x i32> %ret
+}
+
+; Test v2i64 insertion into the first element.
+define <2 x i64> @f16(<2 x i64> %val, i64 *%ptr) {
+; CHECK-LABEL: f16:
+; CHECK: vleg %v24, 0(%r2), 0
+; CHECK: br %r14
+  %element = load i64, i64 *%ptr
+  %ret = insertelement <2 x i64> %val, i64 %element, i32 0
+  ret <2 x i64> %ret
+}
+
+; Test v2i64 insertion into the last element.
+define <2 x i64> @f17(<2 x i64> %val, i64 *%ptr) {
+; CHECK-LABEL: f17:
+; CHECK: vleg %v24, 0(%r2), 1
+; CHECK: br %r14
+  %element = load i64, i64 *%ptr
+  %ret = insertelement <2 x i64> %val, i64 %element, i32 1
+  ret <2 x i64> %ret
+}
+
+; Test v2i64 insertion with the highest in-range offset.
+define <2 x i64> @f18(<2 x i64> %val, i64 *%base) {
+; CHECK-LABEL: f18:
+; CHECK: vleg %v24, 4088(%r2), 1
+; CHECK: br %r14
+  %ptr = getelementptr i64, i64 *%base, i32 511
+  %element = load i64, i64 *%ptr
+  %ret = insertelement <2 x i64> %val, i64 %element, i32 1
+  ret <2 x i64> %ret
+}
+
+; Test v2i64 insertion with the first ouf-of-range offset.
+define <2 x i64> @f19(<2 x i64> %val, i64 *%base) {
+; CHECK-LABEL: f19:
+; CHECK: aghi %r2, 4096
+; CHECK: vleg %v24, 0(%r2), 0
+; CHECK: br %r14
+  %ptr = getelementptr i64, i64 *%base, i32 512
+  %element = load i64, i64 *%ptr
+  %ret = insertelement <2 x i64> %val, i64 %element, i32 0
+  ret <2 x i64> %ret
+}
+
+; Test v2i64 insertion into a variable element.
+define <2 x i64> @f20(<2 x i64> %val, i64 *%ptr, i32 %index) {
+; CHECK-LABEL: f20:
+; CHECK-NOT: vleg
+; CHECK: br %r14
+  %element = load i64, i64 *%ptr
+  %ret = insertelement <2 x i64> %val, i64 %element, i32 %index
+  ret <2 x i64> %ret
+}
+
+; Test v4f32 insertion into the first element.
+define <4 x float> @f21(<4 x float> %val, float *%ptr) {
+; CHECK-LABEL: f21:
+; CHECK: vlef %v24, 0(%r2), 0
+; CHECK: br %r14
+  %element = load float, float *%ptr
+  %ret = insertelement <4 x float> %val, float %element, i32 0
+  ret <4 x float> %ret
+}
+
+; Test v4f32 insertion into the last element.
+define <4 x float> @f22(<4 x float> %val, float *%ptr) {
+; CHECK-LABEL: f22:
+; CHECK: vlef %v24, 0(%r2), 3
+; CHECK: br %r14
+  %element = load float, float *%ptr
+  %ret = insertelement <4 x float> %val, float %element, i32 3
+  ret <4 x float> %ret
+}
+
+; Test v4f32 insertion with the highest in-range offset.
+define <4 x float> @f23(<4 x float> %val, float *%base) {
+; CHECK-LABEL: f23:
+; CHECK: vlef %v24, 4092(%r2), 2
+; CHECK: br %r14
+  %ptr = getelementptr float, float *%base, i32 1023
+  %element = load float, float *%ptr
+  %ret = insertelement <4 x float> %val, float %element, i32 2
+  ret <4 x float> %ret
+}
+
+; Test v4f32 insertion with the first ouf-of-range offset.
+define <4 x float> @f24(<4 x float> %val, float *%base) {
+; CHECK-LABEL: f24:
+; CHECK: aghi %r2, 4096
+; CHECK: vlef %v24, 0(%r2), 1
+; CHECK: br %r14
+  %ptr = getelementptr float, float *%base, i32 1024
+  %element = load float, float *%ptr
+  %ret = insertelement <4 x float> %val, float %element, i32 1
+  ret <4 x float> %ret
+}
+
+; Test v4f32 insertion into a variable element.
+define <4 x float> @f25(<4 x float> %val, float *%ptr, i32 %index) {
+; CHECK-LABEL: f25:
+; CHECK-NOT: vlef
+; CHECK: br %r14
+  %element = load float, float *%ptr
+  %ret = insertelement <4 x float> %val, float %element, i32 %index
+  ret <4 x float> %ret
+}
+
+; Test v2f64 insertion into the first element.
+define <2 x double> @f26(<2 x double> %val, double *%ptr) {
+; CHECK-LABEL: f26:
+; CHECK: vleg %v24, 0(%r2), 0
+; CHECK: br %r14
+  %element = load double, double *%ptr
+  %ret = insertelement <2 x double> %val, double %element, i32 0
+  ret <2 x double> %ret
+}
+
+; Test v2f64 insertion into the last element.
+define <2 x double> @f27(<2 x double> %val, double *%ptr) {
+; CHECK-LABEL: f27:
+; CHECK: vleg %v24, 0(%r2), 1
+; CHECK: br %r14
+  %element = load double, double *%ptr
+  %ret = insertelement <2 x double> %val, double %element, i32 1
+  ret <2 x double> %ret
+}
+
+; Test v2f64 insertion with the highest in-range offset.
+define <2 x double> @f28(<2 x double> %val, double *%base) {
+; CHECK-LABEL: f28:
+; CHECK: vleg %v24, 4088(%r2), 1
+; CHECK: br %r14
+  %ptr = getelementptr double, double *%base, i32 511
+  %element = load double, double *%ptr
+  %ret = insertelement <2 x double> %val, double %element, i32 1
+  ret <2 x double> %ret
+}
+
+; Test v2f64 insertion with the first ouf-of-range offset.
+define <2 x double> @f29(<2 x double> %val, double *%base) {
+; CHECK-LABEL: f29:
+; CHECK: aghi %r2, 4096
+; CHECK: vleg %v24, 0(%r2), 0
+; CHECK: br %r14
+  %ptr = getelementptr double, double *%base, i32 512
+  %element = load double, double *%ptr
+  %ret = insertelement <2 x double> %val, double %element, i32 0
+  ret <2 x double> %ret
+}
+
+; Test v2f64 insertion into a variable element.
+define <2 x double> @f30(<2 x double> %val, double *%ptr, i32 %index) {
+; CHECK-LABEL: f30:
+; CHECK-NOT: vleg
+; CHECK: br %r14
+  %element = load double, double *%ptr
+  %ret = insertelement <2 x double> %val, double %element, i32 %index
+  ret <2 x double> %ret
+}
+
+; Test a v4i32 gather of the first element.
+define <4 x i32> @f31(<4 x i32> %val, <4 x i32> %index, i64 %base) {
+; CHECK-LABEL: f31:
+; CHECK: vgef %v24, 0(%v26,%r2), 0
+; CHECK: br %r14
+  %elem = extractelement <4 x i32> %index, i32 0
+  %ext = zext i32 %elem to i64
+  %add = add i64 %base, %ext
+  %ptr = inttoptr i64 %add to i32 *
+  %element = load i32, i32 *%ptr
+  %ret = insertelement <4 x i32> %val, i32 %element, i32 0
+  ret <4 x i32> %ret
+}
+
+; Test a v4i32 gather of the last element.
+define <4 x i32> @f32(<4 x i32> %val, <4 x i32> %index, i64 %base) {
+; CHECK-LABEL: f32:
+; CHECK: vgef %v24, 0(%v26,%r2), 3
+; CHECK: br %r14
+  %elem = extractelement <4 x i32> %index, i32 3
+  %ext = zext i32 %elem to i64
+  %add = add i64 %base, %ext
+  %ptr = inttoptr i64 %add to i32 *
+  %element = load i32, i32 *%ptr
+  %ret = insertelement <4 x i32> %val, i32 %element, i32 3
+  ret <4 x i32> %ret
+}
+
+; Test a v4i32 gather with the highest in-range offset.
+define <4 x i32> @f33(<4 x i32> %val, <4 x i32> %index, i64 %base) {
+; CHECK-LABEL: f33:
+; CHECK: vgef %v24, 4095(%v26,%r2), 1
+; CHECK: br %r14
+  %elem = extractelement <4 x i32> %index, i32 1
+  %ext = zext i32 %elem to i64
+  %add1 = add i64 %base, %ext
+  %add2 = add i64 %add1, 4095
+  %ptr = inttoptr i64 %add2 to i32 *
+  %element = load i32, i32 *%ptr
+  %ret = insertelement <4 x i32> %val, i32 %element, i32 1
+  ret <4 x i32> %ret
+}
+
+; Test a v2i64 gather of the first element.
+define <2 x i64> @f34(<2 x i64> %val, <2 x i64> %index, i64 %base) {
+; CHECK-LABEL: f34:
+; CHECK: vgeg %v24, 0(%v26,%r2), 0
+; CHECK: br %r14
+  %elem = extractelement <2 x i64> %index, i32 0
+  %add = add i64 %base, %elem
+  %ptr = inttoptr i64 %add to i64 *
+  %element = load i64, i64 *%ptr
+  %ret = insertelement <2 x i64> %val, i64 %element, i32 0
+  ret <2 x i64> %ret
+}
+
+; Test a v2i64 gather of the last element.
+define <2 x i64> @f35(<2 x i64> %val, <2 x i64> %index, i64 %base) {
+; CHECK-LABEL: f35:
+; CHECK: vgeg %v24, 0(%v26,%r2), 1
+; CHECK: br %r14
+  %elem = extractelement <2 x i64> %index, i32 1
+  %add = add i64 %base, %elem
+  %ptr = inttoptr i64 %add to i64 *
+  %element = load i64, i64 *%ptr
+  %ret = insertelement <2 x i64> %val, i64 %element, i32 1
+  ret <2 x i64> %ret
+}
+
+; Test a v4f32 gather of the first element.
+define <4 x float> @f36(<4 x float> %val, <4 x i32> %index, i64 %base) {
+; CHECK-LABEL: f36:
+; CHECK: vgef %v24, 0(%v26,%r2), 0
+; CHECK: br %r14
+  %elem = extractelement <4 x i32> %index, i32 0
+  %ext = zext i32 %elem to i64
+  %add = add i64 %base, %ext
+  %ptr = inttoptr i64 %add to float *
+  %element = load float, float *%ptr
+  %ret = insertelement <4 x float> %val, float %element, i32 0
+  ret <4 x float> %ret
+}
+
+; Test a v4f32 gather of the last element.
+define <4 x float> @f37(<4 x float> %val, <4 x i32> %index, i64 %base) {
+; CHECK-LABEL: f37:
+; CHECK: vgef %v24, 0(%v26,%r2), 3
+; CHECK: br %r14
+  %elem = extractelement <4 x i32> %index, i32 3
+  %ext = zext i32 %elem to i64
+  %add = add i64 %base, %ext
+  %ptr = inttoptr i64 %add to float *
+  %element = load float, float *%ptr
+  %ret = insertelement <4 x float> %val, float %element, i32 3
+  ret <4 x float> %ret
+}
+
+; Test a v2f64 gather of the first element.
+define <2 x double> @f38(<2 x double> %val, <2 x i64> %index, i64 %base) {
+; CHECK-LABEL: f38:
+; CHECK: vgeg %v24, 0(%v26,%r2), 0
+; CHECK: br %r14
+  %elem = extractelement <2 x i64> %index, i32 0
+  %add = add i64 %base, %elem
+  %ptr = inttoptr i64 %add to double *
+  %element = load double, double *%ptr
+  %ret = insertelement <2 x double> %val, double %element, i32 0
+  ret <2 x double> %ret
+}
+
+; Test a v2f64 gather of the last element.
+define <2 x double> @f39(<2 x double> %val, <2 x i64> %index, i64 %base) {
+; CHECK-LABEL: f39:
+; CHECK: vgeg %v24, 0(%v26,%r2), 1
+; CHECK: br %r14
+  %elem = extractelement <2 x i64> %index, i32 1
+  %add = add i64 %base, %elem
+  %ptr = inttoptr i64 %add to double *
+  %element = load double, double *%ptr
+  %ret = insertelement <2 x double> %val, double %element, i32 1
+  ret <2 x double> %ret
+}
diff --git a/test/CodeGen/SystemZ/vec-move-09.ll b/test/CodeGen/SystemZ/vec-move-09.ll
new file mode 100644
index 00000000000..5a53a2d6a19
--- /dev/null
+++ b/test/CodeGen/SystemZ/vec-move-09.ll
@@ -0,0 +1,291 @@
+; Test vector insertion of constants.
+;
+; RUN: llc < %s -mtriple=s390x-linux-gnu -mcpu=z13 | FileCheck %s
+
+; Test v16i8 insertion into the first element.
+define <16 x i8> @f1(<16 x i8> %val) {
+; CHECK-LABEL: f1:
+; CHECK: vleib %v24, 0, 0
+; CHECK: br %r14
+  %ret = insertelement <16 x i8> %val, i8 0, i32 0
+  ret <16 x i8> %ret
+}
+
+; Test v16i8 insertion into the last element.
+define <16 x i8> @f2(<16 x i8> %val) {
+; CHECK-LABEL: f2:
+; CHECK: vleib %v24, 100, 15
+; CHECK: br %r14
+  %ret = insertelement <16 x i8> %val, i8 100, i32 15
+  ret <16 x i8> %ret
+}
+
+; Test v16i8 insertion with the maximum signed value.
+define <16 x i8> @f3(<16 x i8> %val) {
+; CHECK-LABEL: f3:
+; CHECK: vleib %v24, 127, 10
+; CHECK: br %r14
+  %ret = insertelement <16 x i8> %val, i8 127, i32 10
+  ret <16 x i8> %ret
+}
+
+; Test v16i8 insertion with the minimum signed value.
+define <16 x i8> @f4(<16 x i8> %val) {
+; CHECK-LABEL: f4:
+; CHECK: vleib %v24, -128, 11
+; CHECK: br %r14
+  %ret = insertelement <16 x i8> %val, i8 128, i32 11
+  ret <16 x i8> %ret
+}
+
+; Test v16i8 insertion with the maximum unsigned value.
+define <16 x i8> @f5(<16 x i8> %val) {
+; CHECK-LABEL: f5:
+; CHECK: vleib %v24, -1, 12
+; CHECK: br %r14
+  %ret = insertelement <16 x i8> %val, i8 255, i32 12
+  ret <16 x i8> %ret
+}
+
+; Test v16i8 insertion into a variable element.
+define <16 x i8> @f6(<16 x i8> %val, i32 %index) {
+; CHECK-LABEL: f6:
+; CHECK-NOT: vleib
+; CHECK: br %r14
+  %ret = insertelement <16 x i8> %val, i8 0, i32 %index
+  ret <16 x i8> %ret
+}
+
+; Test v8i16 insertion into the first element.
+define <8 x i16> @f7(<8 x i16> %val) {
+; CHECK-LABEL: f7:
+; CHECK: vleih %v24, 0, 0
+; CHECK: br %r14
+  %ret = insertelement <8 x i16> %val, i16 0, i32 0
+  ret <8 x i16> %ret
+}
+
+; Test v8i16 insertion into the last element.
+define <8 x i16> @f8(<8 x i16> %val) {
+; CHECK-LABEL: f8:
+; CHECK: vleih %v24, 0, 7
+; CHECK: br %r14
+  %ret = insertelement <8 x i16> %val, i16 0, i32 7
+  ret <8 x i16> %ret
+}
+
+; Test v8i16 insertion with the maximum signed value.
+define <8 x i16> @f9(<8 x i16> %val) {
+; CHECK-LABEL: f9:
+; CHECK: vleih %v24, 32767, 4
+; CHECK: br %r14
+  %ret = insertelement <8 x i16> %val, i16 32767, i32 4
+  ret <8 x i16> %ret
+}
+
+; Test v8i16 insertion with the minimum signed value.
+define <8 x i16> @f10(<8 x i16> %val) {
+; CHECK-LABEL: f10:
+; CHECK: vleih %v24, -32768, 5
+; CHECK: br %r14
+  %ret = insertelement <8 x i16> %val, i16 32768, i32 5
+  ret <8 x i16> %ret
+}
+
+; Test v8i16 insertion with the maximum unsigned value.
+define <8 x i16> @f11(<8 x i16> %val) {
+; CHECK-LABEL: f11:
+; CHECK: vleih %v24, -1, 6
+; CHECK: br %r14
+  %ret = insertelement <8 x i16> %val, i16 65535, i32 6
+  ret <8 x i16> %ret
+}
+
+; Test v8i16 insertion into a variable element.
+define <8 x i16> @f12(<8 x i16> %val, i32 %index) {
+; CHECK-LABEL: f12:
+; CHECK-NOT: vleih
+; CHECK: br %r14
+  %ret = insertelement <8 x i16> %val, i16 0, i32 %index
+  ret <8 x i16> %ret
+}
+
+; Test v4i32 insertion into the first element.
+define <4 x i32> @f13(<4 x i32> %val) {
+; CHECK-LABEL: f13:
+; CHECK: vleif %v24, 0, 0
+; CHECK: br %r14
+  %ret = insertelement <4 x i32> %val, i32 0, i32 0
+  ret <4 x i32> %ret
+}
+
+; Test v4i32 insertion into the last element.
+define <4 x i32> @f14(<4 x i32> %val) {
+; CHECK-LABEL: f14:
+; CHECK: vleif %v24, 0, 3
+; CHECK: br %r14
+  %ret = insertelement <4 x i32> %val, i32 0, i32 3
+  ret <4 x i32> %ret
+}
+
+; Test v4i32 insertion with the maximum value allowed by VLEIF.
+define <4 x i32> @f15(<4 x i32> %val) {
+; CHECK-LABEL: f15:
+; CHECK: vleif %v24, 32767, 1
+; CHECK: br %r14
+  %ret = insertelement <4 x i32> %val, i32 32767, i32 1
+  ret <4 x i32> %ret
+}
+
+; Test v4i32 insertion with the next value up.
+define <4 x i32> @f16(<4 x i32> %val) {
+; CHECK-LABEL: f16:
+; CHECK-NOT: vleif
+; CHECK: br %r14
+  %ret = insertelement <4 x i32> %val, i32 32768, i32 1
+  ret <4 x i32> %ret
+}
+
+; Test v4i32 insertion with the minimum value allowed by VLEIF.
+define <4 x i32> @f17(<4 x i32> %val) {
+; CHECK-LABEL: f17:
+; CHECK: vleif %v24, -32768, 2
+; CHECK: br %r14
+  %ret = insertelement <4 x i32> %val, i32 -32768, i32 2
+  ret <4 x i32> %ret
+}
+
+; Test v4i32 insertion with the next value down.
+define <4 x i32> @f18(<4 x i32> %val) {
+; CHECK-LABEL: f18:
+; CHECK-NOT: vleif
+; CHECK: br %r14
+  %ret = insertelement <4 x i32> %val, i32 -32769, i32 2
+  ret <4 x i32> %ret
+}
+
+; Test v4i32 insertion into a variable element.
+define <4 x i32> @f19(<4 x i32> %val, i32 %index) {
+; CHECK-LABEL: f19:
+; CHECK-NOT: vleif
+; CHECK: br %r14
+  %ret = insertelement <4 x i32> %val, i32 0, i32 %index
+  ret <4 x i32> %ret
+}
+
+; Test v2i64 insertion into the first element.
+define <2 x i64> @f20(<2 x i64> %val) {
+; CHECK-LABEL: f20:
+; CHECK: vleig %v24, 0, 0
+; CHECK: br %r14
+  %ret = insertelement <2 x i64> %val, i64 0, i32 0
+  ret <2 x i64> %ret
+}
+
+; Test v2i64 insertion into the last element.
+define <2 x i64> @f21(<2 x i64> %val) {
+; CHECK-LABEL: f21:
+; CHECK: vleig %v24, 0, 1
+; CHECK: br %r14
+  %ret = insertelement <2 x i64> %val, i64 0, i32 1
+  ret <2 x i64> %ret
+}
+
+; Test v2i64 insertion with the maximum value allowed by VLEIG.
+define <2 x i64> @f22(<2 x i64> %val) {
+; CHECK-LABEL: f22:
+; CHECK: vleig %v24, 32767, 1
+; CHECK: br %r14
+  %ret = insertelement <2 x i64> %val, i64 32767, i32 1
+  ret <2 x i64> %ret
+}
+
+; Test v2i64 insertion with the next value up.
+define <2 x i64> @f23(<2 x i64> %val) {
+; CHECK-LABEL: f23:
+; CHECK-NOT: vleig
+; CHECK: br %r14
+  %ret = insertelement <2 x i64> %val, i64 32768, i32 1
+  ret <2 x i64> %ret
+}
+
+; Test v2i64 insertion with the minimum value allowed by VLEIG.
+define <2 x i64> @f24(<2 x i64> %val) {
+; CHECK-LABEL: f24:
+; CHECK: vleig %v24, -32768, 0
+; CHECK: br %r14
+  %ret = insertelement <2 x i64> %val, i64 -32768, i32 0
+  ret <2 x i64> %ret
+}
+
+; Test v2i64 insertion with the next value down.
+define <2 x i64> @f25(<2 x i64> %val) {
+; CHECK-LABEL: f25:
+; CHECK-NOT: vleig
+; CHECK: br %r14
+  %ret = insertelement <2 x i64> %val, i64 -32769, i32 0
+  ret <2 x i64> %ret
+}
+
+; Test v2i64 insertion into a variable element.
+define <2 x i64> @f26(<2 x i64> %val, i32 %index) {
+; CHECK-LABEL: f26:
+; CHECK-NOT: vleig
+; CHECK: br %r14
+  %ret = insertelement <2 x i64> %val, i64 0, i32 %index
+  ret <2 x i64> %ret
+}
+
+; Test v4f32 insertion of 0 into the first element.
+define <4 x float> @f27(<4 x float> %val) {
+; CHECK-LABEL: f27:
+; CHECK: vleif %v24, 0, 0
+; CHECK: br %r14
+  %ret = insertelement <4 x float> %val, float 0.0, i32 0
+  ret <4 x float> %ret
+}
+
+; Test v4f32 insertion of 0 into the last element.
+define <4 x float> @f28(<4 x float> %val) {
+; CHECK-LABEL: f28:
+; CHECK: vleif %v24, 0, 3
+; CHECK: br %r14
+  %ret = insertelement <4 x float> %val, float 0.0, i32 3
+  ret <4 x float> %ret
+}
+
+; Test v4f32 insertion of a nonzero value.
+define <4 x float> @f29(<4 x float> %val) {
+; CHECK-LABEL: f29:
+; CHECK-NOT: vleif
+; CHECK: br %r14
+  %ret = insertelement <4 x float> %val, float 1.0, i32 1
+  ret <4 x float> %ret
+}
+
+; Test v2f64 insertion of 0 into the first element.
+define <2 x double> @f30(<2 x double> %val) {
+; CHECK-LABEL: f30:
+; CHECK: vleig %v24, 0, 0
+; CHECK: br %r14
+  %ret = insertelement <2 x double> %val, double 0.0, i32 0
+  ret <2 x double> %ret
+}
+
+; Test v2f64 insertion of 0 into the last element.
+define <2 x double> @f31(<2 x double> %val) {
+; CHECK-LABEL: f31:
+; CHECK: vleig %v24, 0, 1
+; CHECK: br %r14
+  %ret = insertelement <2 x double> %val, double 0.0, i32 1
+  ret <2 x double> %ret
+}
+
+; Test v2f64 insertion of a nonzero value.
+define <2 x double> @f32(<2 x double> %val) {
+; CHECK-LABEL: f32:
+; CHECK-NOT: vleig
+; CHECK: br %r14
+  %ret = insertelement <2 x double> %val, double 1.0, i32 1
+  ret <2 x double> %ret
+}
diff --git a/test/CodeGen/SystemZ/vec-move-10.ll b/test/CodeGen/SystemZ/vec-move-10.ll
new file mode 100644
index 00000000000..894d0c2b41f
--- /dev/null
+++ b/test/CodeGen/SystemZ/vec-move-10.ll
@@ -0,0 +1,499 @@
+; Test vector extraction to memory.
+;
+; RUN: llc < %s -mtriple=s390x-linux-gnu -mcpu=z13 | FileCheck %s
+
+; Test v16i8 extraction from the first element.
+define void @f1(<16 x i8> %val, i8 *%ptr) {
+; CHECK-LABEL: f1:
+; CHECK: vsteb %v24, 0(%r2), 0
+; CHECK: br %r14
+  %element = extractelement <16 x i8> %val, i32 0
+  store i8 %element, i8 *%ptr
+  ret void
+}
+
+; Test v16i8 extraction from the last element.
+define void @f2(<16 x i8> %val, i8 *%ptr) {
+; CHECK-LABEL: f2:
+; CHECK: vsteb %v24, 0(%r2), 15
+; CHECK: br %r14
+  %element = extractelement <16 x i8> %val, i32 15
+  store i8 %element, i8 *%ptr
+  ret void
+}
+
+; Test v16i8 extraction of an invalid element.  This must compile,
+; but we don't care what it does.
+define void @f3(<16 x i8> %val, i8 *%ptr) {
+; CHECK-LABEL: f3:
+; CHECK-NOT: vsteb %v24, 0(%r2), 16
+; CHECK: br %r14
+  %element = extractelement <16 x i8> %val, i32 16
+  store i8 %element, i8 *%ptr
+  ret void
+}
+
+; Test v16i8 extraction with the highest in-range offset.
+define void @f4(<16 x i8> %val, i8 *%base) {
+; CHECK-LABEL: f4:
+; CHECK: vsteb %v24, 4095(%r2), 10
+; CHECK: br %r14
+  %ptr = getelementptr i8, i8 *%base, i32 4095
+  %element = extractelement <16 x i8> %val, i32 10
+  store i8 %element, i8 *%ptr
+  ret void
+}
+
+; Test v16i8 extraction with the first ouf-of-range offset.
+define void @f5(<16 x i8> %val, i8 *%base) {
+; CHECK-LABEL: f5:
+; CHECK: aghi %r2, 4096
+; CHECK: vsteb %v24, 0(%r2), 5
+; CHECK: br %r14
+  %ptr = getelementptr i8, i8 *%base, i32 4096
+  %element = extractelement <16 x i8> %val, i32 5
+  store i8 %element, i8 *%ptr
+  ret void
+}
+
+; Test v16i8 extraction from a variable element.
+define void @f6(<16 x i8> %val, i8 *%ptr, i32 %index) {
+; CHECK-LABEL: f6:
+; CHECK-NOT: vsteb
+; CHECK: br %r14
+  %element = extractelement <16 x i8> %val, i32 %index
+  store i8 %element, i8 *%ptr
+  ret void
+}
+
+; Test v8i16 extraction from the first element.
+define void @f7(<8 x i16> %val, i16 *%ptr) {
+; CHECK-LABEL: f7:
+; CHECK: vsteh %v24, 0(%r2), 0
+; CHECK: br %r14
+  %element = extractelement <8 x i16> %val, i32 0
+  store i16 %element, i16 *%ptr
+  ret void
+}
+
+; Test v8i16 extraction from the last element.
+define void @f8(<8 x i16> %val, i16 *%ptr) {
+; CHECK-LABEL: f8:
+; CHECK: vsteh %v24, 0(%r2), 7
+; CHECK: br %r14
+  %element = extractelement <8 x i16> %val, i32 7
+  store i16 %element, i16 *%ptr
+  ret void
+}
+
+; Test v8i16 extraction of an invalid element.  This must compile,
+; but we don't care what it does.
+define void @f9(<8 x i16> %val, i16 *%ptr) {
+; CHECK-LABEL: f9:
+; CHECK-NOT: vsteh %v24, 0(%r2), 8
+; CHECK: br %r14
+  %element = extractelement <8 x i16> %val, i32 8
+  store i16 %element, i16 *%ptr
+  ret void
+}
+
+; Test v8i16 extraction with the highest in-range offset.
+define void @f10(<8 x i16> %val, i16 *%base) {
+; CHECK-LABEL: f10:
+; CHECK: vsteh %v24, 4094(%r2), 5
+; CHECK: br %r14
+  %ptr = getelementptr i16, i16 *%base, i32 2047
+  %element = extractelement <8 x i16> %val, i32 5
+  store i16 %element, i16 *%ptr
+  ret void
+}
+
+; Test v8i16 extraction with the first ouf-of-range offset.
+define void @f11(<8 x i16> %val, i16 *%base) {
+; CHECK-LABEL: f11:
+; CHECK: aghi %r2, 4096
+; CHECK: vsteh %v24, 0(%r2), 1
+; CHECK: br %r14
+  %ptr = getelementptr i16, i16 *%base, i32 2048
+  %element = extractelement <8 x i16> %val, i32 1
+  store i16 %element, i16 *%ptr
+  ret void
+}
+
+; Test v8i16 extraction from a variable element.
+define void @f12(<8 x i16> %val, i16 *%ptr, i32 %index) {
+; CHECK-LABEL: f12:
+; CHECK-NOT: vsteh
+; CHECK: br %r14
+  %element = extractelement <8 x i16> %val, i32 %index
+  store i16 %element, i16 *%ptr
+  ret void
+}
+
+; Test v4i32 extraction from the first element.
+define void @f13(<4 x i32> %val, i32 *%ptr) {
+; CHECK-LABEL: f13:
+; CHECK: vstef %v24, 0(%r2), 0
+; CHECK: br %r14
+  %element = extractelement <4 x i32> %val, i32 0
+  store i32 %element, i32 *%ptr
+  ret void
+}
+
+; Test v4i32 extraction from the last element.
+define void @f14(<4 x i32> %val, i32 *%ptr) {
+; CHECK-LABEL: f14:
+; CHECK: vstef %v24, 0(%r2), 3
+; CHECK: br %r14
+  %element = extractelement <4 x i32> %val, i32 3
+  store i32 %element, i32 *%ptr
+  ret void
+}
+
+; Test v4i32 extraction of an invalid element.  This must compile,
+; but we don't care what it does.
+define void @f15(<4 x i32> %val, i32 *%ptr) {
+; CHECK-LABEL: f15:
+; CHECK-NOT: vstef %v24, 0(%r2), 4
+; CHECK: br %r14
+  %element = extractelement <4 x i32> %val, i32 4
+  store i32 %element, i32 *%ptr
+  ret void
+}
+
+; Test v4i32 extraction with the highest in-range offset.
+define void @f16(<4 x i32> %val, i32 *%base) {
+; CHECK-LABEL: f16:
+; CHECK: vstef %v24, 4092(%r2), 2
+; CHECK: br %r14
+  %ptr = getelementptr i32, i32 *%base, i32 1023
+  %element = extractelement <4 x i32> %val, i32 2
+  store i32 %element, i32 *%ptr
+  ret void
+}
+
+; Test v4i32 extraction with the first ouf-of-range offset.
+define void @f17(<4 x i32> %val, i32 *%base) {
+; CHECK-LABEL: f17:
+; CHECK: aghi %r2, 4096
+; CHECK: vstef %v24, 0(%r2), 1
+; CHECK: br %r14
+  %ptr = getelementptr i32, i32 *%base, i32 1024
+  %element = extractelement <4 x i32> %val, i32 1
+  store i32 %element, i32 *%ptr
+  ret void
+}
+
+; Test v4i32 extraction from a variable element.
+define void @f18(<4 x i32> %val, i32 *%ptr, i32 %index) {
+; CHECK-LABEL: f18:
+; CHECK-NOT: vstef
+; CHECK: br %r14
+  %element = extractelement <4 x i32> %val, i32 %index
+  store i32 %element, i32 *%ptr
+  ret void
+}
+
+; Test v2i64 extraction from the first element.
+define void @f19(<2 x i64> %val, i64 *%ptr) {
+; CHECK-LABEL: f19:
+; CHECK: vsteg %v24, 0(%r2), 0
+; CHECK: br %r14
+  %element = extractelement <2 x i64> %val, i32 0
+  store i64 %element, i64 *%ptr
+  ret void
+}
+
+; Test v2i64 extraction from the last element.
+define void @f20(<2 x i64> %val, i64 *%ptr) {
+; CHECK-LABEL: f20:
+; CHECK: vsteg %v24, 0(%r2), 1
+; CHECK: br %r14
+  %element = extractelement <2 x i64> %val, i32 1
+  store i64 %element, i64 *%ptr
+  ret void
+}
+
+; Test v2i64 extraction of an invalid element.  This must compile,
+; but we don't care what it does.
+define void @f21(<2 x i64> %val, i64 *%ptr) {
+; CHECK-LABEL: f21:
+; CHECK-NOT: vsteg %v24, 0(%r2), 2
+; CHECK: br %r14
+  %element = extractelement <2 x i64> %val, i32 2
+  store i64 %element, i64 *%ptr
+  ret void
+}
+
+; Test v2i64 extraction with the highest in-range offset.
+define void @f22(<2 x i64> %val, i64 *%base) {
+; CHECK-LABEL: f22:
+; CHECK: vsteg %v24, 4088(%r2), 1
+; CHECK: br %r14
+  %ptr = getelementptr i64, i64 *%base, i32 511
+  %element = extractelement <2 x i64> %val, i32 1
+  store i64 %element, i64 *%ptr
+  ret void
+}
+
+; Test v2i64 extraction with the first ouf-of-range offset.
+define void @f23(<2 x i64> %val, i64 *%base) {
+; CHECK-LABEL: f23:
+; CHECK: aghi %r2, 4096
+; CHECK: vsteg %v24, 0(%r2), 0
+; CHECK: br %r14
+  %ptr = getelementptr i64, i64 *%base, i32 512
+  %element = extractelement <2 x i64> %val, i32 0
+  store i64 %element, i64 *%ptr
+  ret void
+}
+
+; Test v2i64 extraction from a variable element.
+define void @f24(<2 x i64> %val, i64 *%ptr, i32 %index) {
+; CHECK-LABEL: f24:
+; CHECK-NOT: vsteg
+; CHECK: br %r14
+  %element = extractelement <2 x i64> %val, i32 %index
+  store i64 %element, i64 *%ptr
+  ret void
+}
+
+; Test v4f32 extraction from the first element.
+define void @f25(<4 x float> %val, float *%ptr) {
+; CHECK-LABEL: f25:
+; CHECK: vstef %v24, 0(%r2), 0
+; CHECK: br %r14
+  %element = extractelement <4 x float> %val, i32 0
+  store float %element, float *%ptr
+  ret void
+}
+
+; Test v4f32 extraction from the last element.
+define void @f26(<4 x float> %val, float *%ptr) {
+; CHECK-LABEL: f26:
+; CHECK: vstef %v24, 0(%r2), 3
+; CHECK: br %r14
+  %element = extractelement <4 x float> %val, i32 3
+  store float %element, float *%ptr
+  ret void
+}
+
+; Test v4f32 extraction of an invalid element.  This must compile,
+; but we don't care what it does.
+define void @f27(<4 x float> %val, float *%ptr) {
+; CHECK-LABEL: f27:
+; CHECK-NOT: vstef %v24, 0(%r2), 4
+; CHECK: br %r14
+  %element = extractelement <4 x float> %val, i32 4
+  store float %element, float *%ptr
+  ret void
+}
+
+; Test v4f32 extraction with the highest in-range offset.
+define void @f28(<4 x float> %val, float *%base) {
+; CHECK-LABEL: f28:
+; CHECK: vstef %v24, 4092(%r2), 2
+; CHECK: br %r14
+  %ptr = getelementptr float, float *%base, i32 1023
+  %element = extractelement <4 x float> %val, i32 2
+  store float %element, float *%ptr
+  ret void
+}
+
+; Test v4f32 extraction with the first ouf-of-range offset.
+define void @f29(<4 x float> %val, float *%base) {
+; CHECK-LABEL: f29:
+; CHECK: aghi %r2, 4096
+; CHECK: vstef %v24, 0(%r2), 1
+; CHECK: br %r14
+  %ptr = getelementptr float, float *%base, i32 1024
+  %element = extractelement <4 x float> %val, i32 1
+  store float %element, float *%ptr
+  ret void
+}
+
+; Test v4f32 extraction from a variable element.
+define void @f30(<4 x float> %val, float *%ptr, i32 %index) {
+; CHECK-LABEL: f30:
+; CHECK-NOT: vstef
+; CHECK: br %r14
+  %element = extractelement <4 x float> %val, i32 %index
+  store float %element, float *%ptr
+  ret void
+}
+
+; Test v2f64 extraction from the first element.
+define void @f32(<2 x double> %val, double *%ptr) {
+; CHECK-LABEL: f32:
+; CHECK: vsteg %v24, 0(%r2), 0
+; CHECK: br %r14
+  %element = extractelement <2 x double> %val, i32 0
+  store double %element, double *%ptr
+  ret void
+}
+
+; Test v2f64 extraction from the last element.
+define void @f33(<2 x double> %val, double *%ptr) {
+; CHECK-LABEL: f33:
+; CHECK: vsteg %v24, 0(%r2), 1
+; CHECK: br %r14
+  %element = extractelement <2 x double> %val, i32 1
+  store double %element, double *%ptr
+  ret void
+}
+
+; Test v2f64 extraction with the highest in-range offset.
+define void @f34(<2 x double> %val, double *%base) {
+; CHECK-LABEL: f34:
+; CHECK: vsteg %v24, 4088(%r2), 1
+; CHECK: br %r14
+  %ptr = getelementptr double, double *%base, i32 511
+  %element = extractelement <2 x double> %val, i32 1
+  store double %element, double *%ptr
+  ret void
+}
+
+; Test v2f64 extraction with the first ouf-of-range offset.
+define void @f35(<2 x double> %val, double *%base) {
+; CHECK-LABEL: f35:
+; CHECK: aghi %r2, 4096
+; CHECK: vsteg %v24, 0(%r2), 0
+; CHECK: br %r14
+  %ptr = getelementptr double, double *%base, i32 512
+  %element = extractelement <2 x double> %val, i32 0
+  store double %element, double *%ptr
+  ret void
+}
+
+; Test v2f64 extraction from a variable element.
+define void @f36(<2 x double> %val, double *%ptr, i32 %index) {
+; CHECK-LABEL: f36:
+; CHECK-NOT: vsteg
+; CHECK: br %r14
+  %element = extractelement <2 x double> %val, i32 %index
+  store double %element, double *%ptr
+  ret void
+}
+
+; Test a v4i32 scatter of the first element.
+define void @f37(<4 x i32> %val, <4 x i32> %index, i64 %base) {
+; CHECK-LABEL: f37:
+; CHECK: vscef %v24, 0(%v26,%r2), 0
+; CHECK: br %r14
+  %elem = extractelement <4 x i32> %index, i32 0
+  %ext = zext i32 %elem to i64
+  %add = add i64 %base, %ext
+  %ptr = inttoptr i64 %add to i32 *
+  %element = extractelement <4 x i32> %val, i32 0
+  store i32 %element, i32 *%ptr
+  ret void
+}
+
+; Test a v4i32 scatter of the last element.
+define void @f38(<4 x i32> %val, <4 x i32> %index, i64 %base) {
+; CHECK-LABEL: f38:
+; CHECK: vscef %v24, 0(%v26,%r2), 3
+; CHECK: br %r14
+  %elem = extractelement <4 x i32> %index, i32 3
+  %ext = zext i32 %elem to i64
+  %add = add i64 %base, %ext
+  %ptr = inttoptr i64 %add to i32 *
+  %element = extractelement <4 x i32> %val, i32 3
+  store i32 %element, i32 *%ptr
+  ret void
+}
+
+; Test a v4i32 scatter with the highest in-range offset.
+define void @f39(<4 x i32> %val, <4 x i32> %index, i64 %base) {
+; CHECK-LABEL: f39:
+; CHECK: vscef %v24, 4095(%v26,%r2), 1
+; CHECK: br %r14
+  %elem = extractelement <4 x i32> %index, i32 1
+  %ext = zext i32 %elem to i64
+  %add1 = add i64 %base, %ext
+  %add2 = add i64 %add1, 4095
+  %ptr = inttoptr i64 %add2 to i32 *
+  %element = extractelement <4 x i32> %val, i32 1
+  store i32 %element, i32 *%ptr
+  ret void
+}
+
+; Test a v2i64 scatter of the first element.
+define void @f40(<2 x i64> %val, <2 x i64> %index, i64 %base) {
+; CHECK-LABEL: f40:
+; CHECK: vsceg %v24, 0(%v26,%r2), 0
+; CHECK: br %r14
+  %elem = extractelement <2 x i64> %index, i32 0
+  %add = add i64 %base, %elem
+  %ptr = inttoptr i64 %add to i64 *
+  %element = extractelement <2 x i64> %val, i32 0
+  store i64 %element, i64 *%ptr
+  ret void
+}
+
+; Test a v2i64 scatter of the last element.
+define void @f41(<2 x i64> %val, <2 x i64> %index, i64 %base) {
+; CHECK-LABEL: f41:
+; CHECK: vsceg %v24, 0(%v26,%r2), 1
+; CHECK: br %r14
+  %elem = extractelement <2 x i64> %index, i32 1
+  %add = add i64 %base, %elem
+  %ptr = inttoptr i64 %add to i64 *
+  %element = extractelement <2 x i64> %val, i32 1
+  store i64 %element, i64 *%ptr
+  ret void
+}
+
+; Test a v4f32 scatter of the first element.
+define void @f42(<4 x float> %val, <4 x i32> %index, i64 %base) {
+; CHECK-LABEL: f42:
+; CHECK: vscef %v24, 0(%v26,%r2), 0
+; CHECK: br %r14
+  %elem = extractelement <4 x i32> %index, i32 0
+  %ext = zext i32 %elem to i64
+  %add = add i64 %base, %ext
+  %ptr = inttoptr i64 %add to float *
+  %element = extractelement <4 x float> %val, i32 0
+  store float %element, float *%ptr
+  ret void
+}
+
+; Test a v4f32 scatter of the last element.
+define void @f43(<4 x float> %val, <4 x i32> %index, i64 %base) {
+; CHECK-LABEL: f43:
+; CHECK: vscef %v24, 0(%v26,%r2), 3
+; CHECK: br %r14
+  %elem = extractelement <4 x i32> %index, i32 3
+  %ext = zext i32 %elem to i64
+  %add = add i64 %base, %ext
+  %ptr = inttoptr i64 %add to float *
+  %element = extractelement <4 x float> %val, i32 3
+  store float %element, float *%ptr
+  ret void
+}
+
+; Test a v2f64 scatter of the first element.
+define void @f44(<2 x double> %val, <2 x i64> %index, i64 %base) {
+; CHECK-LABEL: f44:
+; CHECK: vsceg %v24, 0(%v26,%r2), 0
+; CHECK: br %r14
+  %elem = extractelement <2 x i64> %index, i32 0
+  %add = add i64 %base, %elem
+  %ptr = inttoptr i64 %add to double *
+  %element = extractelement <2 x double> %val, i32 0
+  store double %element, double *%ptr
+  ret void
+}
+
+; Test a v2f64 scatter of the last element.
+define void @f45(<2 x double> %val, <2 x i64> %index, i64 %base) {
+; CHECK-LABEL: f45:
+; CHECK: vsceg %v24, 0(%v26,%r2), 1
+; CHECK: br %r14
+  %elem = extractelement <2 x i64> %index, i32 1
+  %add = add i64 %base, %elem
+  %ptr = inttoptr i64 %add to double *
+  %element = extractelement <2 x double> %val, i32 1
+  store double %element, double *%ptr
+  ret void
+}
diff --git a/test/CodeGen/SystemZ/vec-move-11.ll b/test/CodeGen/SystemZ/vec-move-11.ll
new file mode 100644
index 00000000000..fd9c3d3559f
--- /dev/null
+++ b/test/CodeGen/SystemZ/vec-move-11.ll
@@ -0,0 +1,111 @@
+; Test insertions of register values into a nonzero index of an undef.
+;
+; RUN: llc < %s -mtriple=s390x-linux-gnu -mcpu=z13 | FileCheck %s
+
+; Test v16i8 insertion into an undef, with an arbitrary index.
+define <16 x i8> @f1(i8 %val) {
+; CHECK-LABEL: f1:
+; CHECK: vlvgb %v24, %r2, 12
+; CHECK-NEXT: br %r14
+  %ret = insertelement <16 x i8> undef, i8 %val, i32 12
+  ret <16 x i8> %ret
+}
+
+; Test v16i8 insertion into an undef, with the first good index for VLVGP.
+define <16 x i8> @f2(i8 %val) {
+; CHECK-LABEL: f2:
+; CHECK: vlvgp %v24, %r2, %r2
+; CHECK-NEXT: br %r14
+  %ret = insertelement <16 x i8> undef, i8 %val, i32 7
+  ret <16 x i8> %ret
+}
+
+; Test v16i8 insertion into an undef, with the second good index for VLVGP.
+define <16 x i8> @f3(i8 %val) {
+; CHECK-LABEL: f3:
+; CHECK: vlvgp %v24, %r2, %r2
+; CHECK-NEXT: br %r14
+  %ret = insertelement <16 x i8> undef, i8 %val, i32 15
+  ret <16 x i8> %ret
+}
+
+; Test v8i16 insertion into an undef, with an arbitrary index.
+define <8 x i16> @f4(i16 %val) {
+; CHECK-LABEL: f4:
+; CHECK: vlvgh %v24, %r2, 5
+; CHECK-NEXT: br %r14
+  %ret = insertelement <8 x i16> undef, i16 %val, i32 5
+  ret <8 x i16> %ret
+}
+
+; Test v8i16 insertion into an undef, with the first good index for VLVGP.
+define <8 x i16> @f5(i16 %val) {
+; CHECK-LABEL: f5:
+; CHECK: vlvgp %v24, %r2, %r2
+; CHECK-NEXT: br %r14
+  %ret = insertelement <8 x i16> undef, i16 %val, i32 3
+  ret <8 x i16> %ret
+}
+
+; Test v8i16 insertion into an undef, with the second good index for VLVGP.
+define <8 x i16> @f6(i16 %val) {
+; CHECK-LABEL: f6:
+; CHECK: vlvgp %v24, %r2, %r2
+; CHECK-NEXT: br %r14
+  %ret = insertelement <8 x i16> undef, i16 %val, i32 7
+  ret <8 x i16> %ret
+}
+
+; Test v4i32 insertion into an undef, with an arbitrary index.
+define <4 x i32> @f7(i32 %val) {
+; CHECK-LABEL: f7:
+; CHECK: vlvgf %v24, %r2, 2
+; CHECK-NEXT: br %r14
+  %ret = insertelement <4 x i32> undef, i32 %val, i32 2
+  ret <4 x i32> %ret
+}
+
+; Test v4i32 insertion into an undef, with the first good index for VLVGP.
+define <4 x i32> @f8(i32 %val) {
+; CHECK-LABEL: f8:
+; CHECK: vlvgp %v24, %r2, %r2
+; CHECK-NEXT: br %r14
+  %ret = insertelement <4 x i32> undef, i32 %val, i32 1
+  ret <4 x i32> %ret
+}
+
+; Test v4i32 insertion into an undef, with the second good index for VLVGP.
+define <4 x i32> @f9(i32 %val) {
+; CHECK-LABEL: f9:
+; CHECK: vlvgp %v24, %r2, %r2
+; CHECK-NEXT: br %r14
+  %ret = insertelement <4 x i32> undef, i32 %val, i32 3
+  ret <4 x i32> %ret
+}
+
+; Test v2i64 insertion into an undef.
+define <2 x i64> @f10(i64 %val) {
+; CHECK-LABEL: f10:
+; CHECK: vlvgp %v24, %r2, %r2
+; CHECK-NEXT: br %r14
+  %ret = insertelement <2 x i64> undef, i64 %val, i32 1
+  ret <2 x i64> %ret
+}
+
+; Test v4f32 insertion into an undef.
+define <4 x float> @f11(float %val) {
+; CHECK-LABEL: f11:
+; CHECK: vrepf %v24, %v0, 0
+; CHECK: br %r14
+  %ret = insertelement <4 x float> undef, float %val, i32 2
+  ret <4 x float> %ret
+}
+
+; Test v2f64 insertion into an undef.
+define <2 x double> @f12(double %val) {
+; CHECK-LABEL: f12:
+; CHECK: vrepg %v24, %v0, 0
+; CHECK: br %r14
+  %ret = insertelement <2 x double> undef, double %val, i32 1
+  ret <2 x double> %ret
+}
diff --git a/test/CodeGen/SystemZ/vec-move-12.ll b/test/CodeGen/SystemZ/vec-move-12.ll
new file mode 100644
index 00000000000..bc8ff97f805
--- /dev/null
+++ b/test/CodeGen/SystemZ/vec-move-12.ll
@@ -0,0 +1,123 @@
+; Test insertions of memory values into a nonzero index of an undef.
+;
+; RUN: llc < %s -mtriple=s390x-linux-gnu -mcpu=z13 | FileCheck %s
+
+; Test v16i8 insertion into an undef, with an arbitrary index.
+define <16 x i8> @f1(i8 *%ptr) {
+; CHECK-LABEL: f1:
+; CHECK: vlrepb %v24, 0(%r2)
+; CHECK-NEXT: br %r14
+  %val = load i8, i8 *%ptr
+  %ret = insertelement <16 x i8> undef, i8 %val, i32 12
+  ret <16 x i8> %ret
+}
+
+; Test v16i8 insertion into an undef, with the first good index for VLVGP.
+define <16 x i8> @f2(i8 *%ptr) {
+; CHECK-LABEL: f2:
+; CHECK: {{vlrepb|vllezb}} %v24, 0(%r2)
+; CHECK-NEXT: br %r14
+  %val = load i8, i8 *%ptr
+  %ret = insertelement <16 x i8> undef, i8 %val, i32 7
+  ret <16 x i8> %ret
+}
+
+; Test v16i8 insertion into an undef, with the second good index for VLVGP.
+define <16 x i8> @f3(i8 *%ptr) {
+; CHECK-LABEL: f3:
+; CHECK: vlrepb %v24, 0(%r2)
+; CHECK-NEXT: br %r14
+  %val = load i8, i8 *%ptr
+  %ret = insertelement <16 x i8> undef, i8 %val, i32 15
+  ret <16 x i8> %ret
+}
+
+; Test v8i16 insertion into an undef, with an arbitrary index.
+define <8 x i16> @f4(i16 *%ptr) {
+; CHECK-LABEL: f4:
+; CHECK: vlreph %v24, 0(%r2)
+; CHECK-NEXT: br %r14
+  %val = load i16, i16 *%ptr
+  %ret = insertelement <8 x i16> undef, i16 %val, i32 5
+  ret <8 x i16> %ret
+}
+
+; Test v8i16 insertion into an undef, with the first good index for VLVGP.
+define <8 x i16> @f5(i16 *%ptr) {
+; CHECK-LABEL: f5:
+; CHECK: {{vlreph|vllezh}} %v24, 0(%r2)
+; CHECK-NEXT: br %r14
+  %val = load i16, i16 *%ptr
+  %ret = insertelement <8 x i16> undef, i16 %val, i32 3
+  ret <8 x i16> %ret
+}
+
+; Test v8i16 insertion into an undef, with the second good index for VLVGP.
+define <8 x i16> @f6(i16 *%ptr) {
+; CHECK-LABEL: f6:
+; CHECK: vlreph %v24, 0(%r2)
+; CHECK-NEXT: br %r14
+  %val = load i16, i16 *%ptr
+  %ret = insertelement <8 x i16> undef, i16 %val, i32 7
+  ret <8 x i16> %ret
+}
+
+; Test v4i32 insertion into an undef, with an arbitrary index.
+define <4 x i32> @f7(i32 *%ptr) {
+; CHECK-LABEL: f7:
+; CHECK: vlrepf %v24, 0(%r2)
+; CHECK-NEXT: br %r14
+  %val = load i32, i32 *%ptr
+  %ret = insertelement <4 x i32> undef, i32 %val, i32 2
+  ret <4 x i32> %ret
+}
+
+; Test v4i32 insertion into an undef, with the first good index for VLVGP.
+define <4 x i32> @f8(i32 *%ptr) {
+; CHECK-LABEL: f8:
+; CHECK: {{vlrepf|vllezf}} %v24, 0(%r2)
+; CHECK-NEXT: br %r14
+  %val = load i32, i32 *%ptr
+  %ret = insertelement <4 x i32> undef, i32 %val, i32 1
+  ret <4 x i32> %ret
+}
+
+; Test v4i32 insertion into an undef, with the second good index for VLVGP.
+define <4 x i32> @f9(i32 *%ptr) {
+; CHECK-LABEL: f9:
+; CHECK: vlrepf %v24, 0(%r2)
+; CHECK-NEXT: br %r14
+  %val = load i32, i32 *%ptr
+  %ret = insertelement <4 x i32> undef, i32 %val, i32 3
+  ret <4 x i32> %ret
+}
+
+; Test v2i64 insertion into an undef.
+define <2 x i64> @f10(i64 *%ptr) {
+; CHECK-LABEL: f10:
+; CHECK: vlrepg %v24, 0(%r2)
+; CHECK-NEXT: br %r14
+  %val = load i64, i64 *%ptr
+  %ret = insertelement <2 x i64> undef, i64 %val, i32 1
+  ret <2 x i64> %ret
+}
+
+; Test v4f32 insertion into an undef.
+define <4 x float> @f11(float *%ptr) {
+; CHECK-LABEL: f11:
+; CHECK: vlrepf %v24, 0(%r2)
+; CHECK: br %r14
+  %val = load float, float *%ptr
+  %ret = insertelement <4 x float> undef, float %val, i32 2
+  ret <4 x float> %ret
+}
+
+; Test v2f64 insertion into an undef.
+define <2 x double> @f12(double *%ptr) {
+; CHECK-LABEL: f12:
+; CHECK: vlrepg %v24, 0(%r2)
+; CHECK: br %r14
+  %val = load double, double *%ptr
+  %ret = insertelement <2 x double> undef, double %val, i32 1
+  ret <2 x double> %ret
+}
diff --git a/test/CodeGen/SystemZ/vec-move-13.ll b/test/CodeGen/SystemZ/vec-move-13.ll
new file mode 100644
index 00000000000..165c3498702
--- /dev/null
+++ b/test/CodeGen/SystemZ/vec-move-13.ll
@@ -0,0 +1,69 @@
+; Test insertions of register values into 0.
+;
+; RUN: llc < %s -mtriple=s390x-linux-gnu -mcpu=z13 | FileCheck %s
+
+; Test v16i8 insertion into 0.
+define <16 x i8> @f1(i8 %val1, i8 %val2) {
+; CHECK-LABEL: f1:
+; CHECK: vgbm %v24, 0
+; CHECK-DAG: vlvgb %v24, %r2, 2
+; CHECK-DAG: vlvgb %v24, %r3, 12
+; CHECK: br %r14
+  %vec1 = insertelement <16 x i8> zeroinitializer, i8 %val1, i32 2
+  %vec2 = insertelement <16 x i8> %vec1, i8 %val2, i32 12
+  ret <16 x i8> %vec2
+}
+
+; Test v8i16 insertion into 0.
+define <8 x i16> @f2(i16 %val1, i16 %val2) {
+; CHECK-LABEL: f2:
+; CHECK: vgbm %v24, 0
+; CHECK-DAG: vlvgh %v24, %r2, 3
+; CHECK-DAG: vlvgh %v24, %r3, 5
+; CHECK: br %r14
+  %vec1 = insertelement <8 x i16> zeroinitializer, i16 %val1, i32 3
+  %vec2 = insertelement <8 x i16> %vec1, i16 %val2, i32 5
+  ret <8 x i16> %vec2
+}
+
+; Test v4i32 insertion into 0.
+define <4 x i32> @f3(i32 %val) {
+; CHECK-LABEL: f3:
+; CHECK: vgbm %v24, 0
+; CHECK: vlvgf %v24, %r2, 3
+; CHECK: br %r14
+  %ret = insertelement <4 x i32> zeroinitializer, i32 %val, i32 3
+  ret <4 x i32> %ret
+}
+
+; Test v2i64 insertion into 0.
+define <2 x i64> @f4(i64 %val) {
+; CHECK-LABEL: f4:
+; CHECK: lghi [[REG:%r[0-5]]], 0
+; CHECK: vlvgp %v24, [[REG]], %r2
+; CHECK: br %r14
+  %ret = insertelement <2 x i64> zeroinitializer, i64 %val, i32 1
+  ret <2 x i64> %ret
+}
+
+; Test v4f32 insertion into 0.
+define <4 x float> @f5(float %val) {
+; CHECK-LABEL: f5:
+; CHECK-DAG: vuplhf [[REG:%v[0-9]+]], %v0
+; CHECK-DAG: vgbm [[ZERO:%v[0-9]+]], 0
+; CHECK: vmrhg %v24, [[ZERO]], [[REG]]
+; CHECK: br %r14
+  %ret = insertelement <4 x float> zeroinitializer, float %val, i32 3
+  ret <4 x float> %ret
+}
+
+; Test v2f64 insertion into 0.
+define <2 x double> @f6(double %val) {
+; CHECK-LABEL: f6:
+; CHECK: vgbm [[REG:%v[0-9]+]], 0
+; CHECK: vmrhg %v24, [[REG]], %v0
+; CHECK: br %r14
+  %ret = insertelement <2 x double> zeroinitializer, double %val, i32 1
+  ret <2 x double> %ret
+}
+
diff --git a/test/CodeGen/SystemZ/vec-move-14.ll b/test/CodeGen/SystemZ/vec-move-14.ll
new file mode 100644
index 00000000000..e41eb9da034
--- /dev/null
+++ b/test/CodeGen/SystemZ/vec-move-14.ll
@@ -0,0 +1,96 @@
+; Test insertions of memory values into 0.
+;
+; RUN: llc < %s -mtriple=s390x-linux-gnu -mcpu=z13 | FileCheck %s
+
+; Test VLLEZB.
+define <16 x i8> @f1(i8 *%ptr) {
+; CHECK-LABEL: f1:
+; CHECK: vllezb %v24, 0(%r2)
+; CHECK: br %r14
+  %val = load i8, i8 *%ptr
+  %ret = insertelement <16 x i8> zeroinitializer, i8 %val, i32 7
+  ret <16 x i8> %ret
+}
+
+; Test VLLEZB with the highest in-range offset.
+define <16 x i8> @f2(i8 *%base) {
+; CHECK-LABEL: f2:
+; CHECK: vllezb %v24, 4095(%r2)
+; CHECK: br %r14
+  %ptr = getelementptr i8, i8 *%base, i64 4095
+  %val = load i8, i8 *%ptr
+  %ret = insertelement <16 x i8> zeroinitializer, i8 %val, i32 7
+  ret <16 x i8> %ret
+}
+
+; Test VLLEZB with the next highest offset.
+define <16 x i8> @f3(i8 *%base) {
+; CHECK-LABEL: f3:
+; CHECK-NOT: vllezb %v24, 4096(%r2)
+; CHECK: br %r14
+  %ptr = getelementptr i8, i8 *%base, i64 4096
+  %val = load i8, i8 *%ptr
+  %ret = insertelement <16 x i8> zeroinitializer, i8 %val, i32 7
+  ret <16 x i8> %ret
+}
+
+; Test that VLLEZB allows an index.
+define <16 x i8> @f4(i8 *%base, i64 %index) {
+; CHECK-LABEL: f4:
+; CHECK: vllezb %v24, 0({{%r2,%r3|%r3,%r2}})
+; CHECK: br %r14
+  %ptr = getelementptr i8, i8 *%base, i64 %index
+  %val = load i8, i8 *%ptr
+  %ret = insertelement <16 x i8> zeroinitializer, i8 %val, i32 7
+  ret <16 x i8> %ret
+}
+
+; Test VLLEZH.
+define <8 x i16> @f5(i16 *%ptr) {
+; CHECK-LABEL: f5:
+; CHECK: vllezh %v24, 0(%r2)
+; CHECK: br %r14
+  %val = load i16, i16 *%ptr
+  %ret = insertelement <8 x i16> zeroinitializer, i16 %val, i32 3
+  ret <8 x i16> %ret
+}
+
+; Test VLLEZF.
+define <4 x i32> @f6(i32 *%ptr) {
+; CHECK-LABEL: f6:
+; CHECK: vllezf %v24, 0(%r2)
+; CHECK: br %r14
+  %val = load i32, i32 *%ptr
+  %ret = insertelement <4 x i32> zeroinitializer, i32 %val, i32 1
+  ret <4 x i32> %ret
+}
+
+; Test VLLEZG.
+define <2 x i64> @f7(i64 *%ptr) {
+; CHECK-LABEL: f7:
+; CHECK: vllezg %v24, 0(%r2)
+; CHECK: br %r14
+  %val = load i64, i64 *%ptr
+  %ret = insertelement <2 x i64> zeroinitializer, i64 %val, i32 0
+  ret <2 x i64> %ret
+}
+
+; Test VLLEZF with a float.
+define <4 x float> @f8(float *%ptr) {
+; CHECK-LABEL: f8:
+; CHECK: vllezf %v24, 0(%r2)
+; CHECK: br %r14
+  %val = load float, float *%ptr
+  %ret = insertelement <4 x float> zeroinitializer, float %val, i32 1
+  ret <4 x float> %ret
+}
+
+; Test VLLEZG with a double.
+define <2 x double> @f9(double *%ptr) {
+; CHECK-LABEL: f9:
+; CHECK: vllezg %v24, 0(%r2)
+; CHECK: br %r14
+  %val = load double, double *%ptr
+  %ret = insertelement <2 x double> zeroinitializer, double %val, i32 0
+  ret <2 x double> %ret
+}
diff --git a/test/CodeGen/SystemZ/vec-move-15.ll b/test/CodeGen/SystemZ/vec-move-15.ll
new file mode 100644
index 00000000000..503627c163c
--- /dev/null
+++ b/test/CodeGen/SystemZ/vec-move-15.ll
@@ -0,0 +1,105 @@
+; Test vector sign-extending loads.
+;
+; RUN: llc < %s -mtriple=s390x-linux-gnu -mcpu=z13 | FileCheck %s
+
+; Test a v16i1->v16i8 extension.
+define <16 x i8> @f1(<16 x i1> *%ptr) {
+; No expected output, but must compile.
+  %val = load <16 x i1>, <16 x i1> *%ptr
+  %ret = sext <16 x i1> %val to <16 x i8>
+  ret <16 x i8> %ret
+}
+
+; Test a v8i1->v8i16 extension.
+define <8 x i16> @f2(<8 x i1> *%ptr) {
+; No expected output, but must compile.
+  %val = load <8 x i1>, <8 x i1> *%ptr
+  %ret = sext <8 x i1> %val to <8 x i16>
+  ret <8 x i16> %ret
+}
+
+; Test a v8i8->v8i16 extension.
+define <8 x i16> @f3(<8 x i8> *%ptr) {
+; CHECK-LABEL: f3:
+; CHECK: vlrepg [[REG1:%v[0-9]+]], 0(%r2)
+; CHECK: vuphb %v24, [[REG1]]
+; CHECK: br %r14
+  %val = load <8 x i8>, <8 x i8> *%ptr
+  %ret = sext <8 x i8> %val to <8 x i16>
+  ret <8 x i16> %ret
+}
+
+; Test a v4i1->v4i32 extension.
+define <4 x i32> @f4(<4 x i1> *%ptr) {
+; No expected output, but must compile.
+  %val = load <4 x i1>, <4 x i1> *%ptr
+  %ret = sext <4 x i1> %val to <4 x i32>
+  ret <4 x i32> %ret
+}
+
+; Test a v4i8->v4i32 extension.
+define <4 x i32> @f5(<4 x i8> *%ptr) {
+; CHECK-LABEL: f5:
+; CHECK: vlrepf [[REG1:%v[0-9]+]], 0(%r2)
+; CHECK: vuphb [[REG2:%v[0-9]+]], [[REG1]]
+; CHECK: vuphh %v24, [[REG2]]
+; CHECK: br %r14
+  %val = load <4 x i8>, <4 x i8> *%ptr
+  %ret = sext <4 x i8> %val to <4 x i32>
+  ret <4 x i32> %ret
+}
+
+; Test a v4i16->v4i32 extension.
+define <4 x i32> @f6(<4 x i16> *%ptr) {
+; CHECK-LABEL: f6:
+; CHECK: vlrepg [[REG1:%v[0-9]+]], 0(%r2)
+; CHECK: vuphh %v24, [[REG1]]
+; CHECK: br %r14
+  %val = load <4 x i16>, <4 x i16> *%ptr
+  %ret = sext <4 x i16> %val to <4 x i32>
+  ret <4 x i32> %ret
+}
+
+; Test a v2i1->v2i64 extension.
+define <2 x i64> @f7(<2 x i1> *%ptr) {
+; No expected output, but must compile.
+  %val = load <2 x i1>, <2 x i1> *%ptr
+  %ret = sext <2 x i1> %val to <2 x i64>
+  ret <2 x i64> %ret
+}
+
+; Test a v2i8->v2i64 extension.
+define <2 x i64> @f8(<2 x i8> *%ptr) {
+; CHECK-LABEL: f8:
+; CHECK: vlreph [[REG1:%v[0-9]+]], 0(%r2)
+; CHECK: vuphb [[REG2:%v[0-9]+]], [[REG1]]
+; CHECK: vuphh [[REG3:%v[0-9]+]], [[REG2]]
+; CHECK: vuphf %v24, [[REG3]]
+; CHECK: br %r14
+  %val = load <2 x i8>, <2 x i8> *%ptr
+  %ret = sext <2 x i8> %val to <2 x i64>
+  ret <2 x i64> %ret
+}
+
+; Test a v2i16->v2i64 extension.
+define <2 x i64> @f9(<2 x i16> *%ptr) {
+; CHECK-LABEL: f9:
+; CHECK: vlrepf [[REG1:%v[0-9]+]], 0(%r2)
+; CHECK: vuphh [[REG2:%v[0-9]+]], [[REG1]]
+; CHECK: vuphf %v24, [[REG2]]
+; CHECK: br %r14
+  %val = load <2 x i16>, <2 x i16> *%ptr
+  %ret = sext <2 x i16> %val to <2 x i64>
+  ret <2 x i64> %ret
+}
+
+; Test a v2i32->v2i64 extension.
+define <2 x i64> @f10(<2 x i32> *%ptr) {
+; CHECK-LABEL: f10:
+; CHECK: vlrepg [[REG1:%v[0-9]+]], 0(%r2)
+; CHECK: vuphf %v24, [[REG1]]
+; CHECK: br %r14
+  %val = load <2 x i32>, <2 x i32> *%ptr
+  %ret = sext <2 x i32> %val to <2 x i64>
+  ret <2 x i64> %ret
+}
diff --git a/test/CodeGen/SystemZ/vec-move-16.ll b/test/CodeGen/SystemZ/vec-move-16.ll
new file mode 100644
index 00000000000..cd257739680
--- /dev/null
+++ b/test/CodeGen/SystemZ/vec-move-16.ll
@@ -0,0 +1,105 @@
+; Test vector zero-extending loads.
+;
+; RUN: llc < %s -mtriple=s390x-linux-gnu -mcpu=z13 | FileCheck %s
+
+; Test a v16i1->v16i8 extension.
+define <16 x i8> @f1(<16 x i1> *%ptr) {
+; No expected output, but must compile.
+  %val = load <16 x i1>, <16 x i1> *%ptr
+  %ret = zext <16 x i1> %val to <16 x i8>
+  ret <16 x i8> %ret
+}
+
+; Test a v8i1->v8i16 extension.
+define <8 x i16> @f2(<8 x i1> *%ptr) {
+; No expected output, but must compile.
+  %val = load <8 x i1>, <8 x i1> *%ptr
+  %ret = zext <8 x i1> %val to <8 x i16>
+  ret <8 x i16> %ret
+}
+
+; Test a v8i8->v8i16 extension.
+define <8 x i16> @f3(<8 x i8> *%ptr) {
+; CHECK-LABEL: f3:
+; CHECK: vlrepg [[REG1:%v[0-9]+]], 0(%r2)
+; CHECK: vuplhb %v24, [[REG1]]
+; CHECK: br %r14
+  %val = load <8 x i8>, <8 x i8> *%ptr
+  %ret = zext <8 x i8> %val to <8 x i16>
+  ret <8 x i16> %ret
+}
+
+; Test a v4i1->v4i32 extension.
+define <4 x i32> @f4(<4 x i1> *%ptr) {
+; No expected output, but must compile.
+  %val = load <4 x i1>, <4 x i1> *%ptr
+  %ret = zext <4 x i1> %val to <4 x i32>
+  ret <4 x i32> %ret
+}
+
+; Test a v4i8->v4i32 extension.
+define <4 x i32> @f5(<4 x i8> *%ptr) {
+; CHECK-LABEL: f5:
+; CHECK: vlrepf [[REG1:%v[0-9]+]], 0(%r2)
+; CHECK: vuplhb [[REG2:%v[0-9]+]], [[REG1]]
+; CHECK: vuplhh %v24, [[REG2]]
+; CHECK: br %r14
+  %val = load <4 x i8>, <4 x i8> *%ptr
+  %ret = zext <4 x i8> %val to <4 x i32>
+  ret <4 x i32> %ret
+}
+
+; Test a v4i16->v4i32 extension.
+define <4 x i32> @f6(<4 x i16> *%ptr) {
+; CHECK-LABEL: f6:
+; CHECK: vlrepg [[REG1:%v[0-9]+]], 0(%r2)
+; CHECK: vuplhh %v24, [[REG1]]
+; CHECK: br %r14
+  %val = load <4 x i16>, <4 x i16> *%ptr
+  %ret = zext <4 x i16> %val to <4 x i32>
+  ret <4 x i32> %ret
+}
+
+; Test a v2i1->v2i64 extension.
+define <2 x i64> @f7(<2 x i1> *%ptr) {
+; No expected output, but must compile.
+  %val = load <2 x i1>, <2 x i1> *%ptr
+  %ret = zext <2 x i1> %val to <2 x i64>
+  ret <2 x i64> %ret
+}
+
+; Test a v2i8->v2i64 extension.
+define <2 x i64> @f8(<2 x i8> *%ptr) {
+; CHECK-LABEL: f8:
+; CHECK: vlreph [[REG1:%v[0-9]+]], 0(%r2)
+; CHECK: vuplhb [[REG2:%v[0-9]+]], [[REG1]]
+; CHECK: vuplhh [[REG3:%v[0-9]+]], [[REG2]]
+; CHECK: vuplhf %v24, [[REG3]]
+; CHECK: br %r14
+  %val = load <2 x i8>, <2 x i8> *%ptr
+  %ret = zext <2 x i8> %val to <2 x i64>
+  ret <2 x i64> %ret
+}
+
+; Test a v2i16->v2i64 extension.
+define <2 x i64> @f9(<2 x i16> *%ptr) {
+; CHECK-LABEL: f9:
+; CHECK: vlrepf [[REG1:%v[0-9]+]], 0(%r2)
+; CHECK: vuplhh [[REG2:%v[0-9]+]], [[REG1]]
+; CHECK: vuplhf %v24, [[REG2]]
+; CHECK: br %r14
+  %val = load <2 x i16>, <2 x i16> *%ptr
+  %ret = zext <2 x i16> %val to <2 x i64>
+  ret <2 x i64> %ret
+}
+
+; Test a v2i32->v2i64 extension.
+define <2 x i64> @f10(<2 x i32> *%ptr) {
+; CHECK-LABEL: f10:
+; CHECK: vlrepg [[REG1:%v[0-9]+]], 0(%r2)
+; CHECK: vuplhf %v24, [[REG1]]
+; CHECK: br %r14
+  %val = load <2 x i32>, <2 x i32> *%ptr
+  %ret = zext <2 x i32> %val to <2 x i64>
+  ret <2 x i64> %ret
+}
diff --git a/test/CodeGen/SystemZ/vec-move-17.ll b/test/CodeGen/SystemZ/vec-move-17.ll
new file mode 100644
index 00000000000..e7fc06c9260
--- /dev/null
+++ b/test/CodeGen/SystemZ/vec-move-17.ll
@@ -0,0 +1,104 @@
+; Test vector truncating stores.
+;
+; RUN: llc < %s -mtriple=s390x-linux-gnu -mcpu=z13 | FileCheck %s
+
+; Test a v16i8->v16i1 truncation.
+define void @f1(<16 x i8> %val, <16 x i1> *%ptr) {
+; No expected output, but must compile.
+  %trunc = trunc <16 x i8> %val to <16 x i1>
+  store <16 x i1> %trunc, <16 x i1> *%ptr
+  ret void
+}
+
+; Test a v8i16->v8i1 truncation.
+define void @f2(<8 x i16> %val, <8 x i1> *%ptr) {
+; No expected output, but must compile.
+  %trunc = trunc <8 x i16> %val to <8 x i1>
+  store <8 x i1> %trunc, <8 x i1> *%ptr
+  ret void
+}
+
+; Test a v8i16->v8i8 truncation.
+define void @f3(<8 x i16> %val, <8 x i8> *%ptr) {
+; CHECK-LABEL: f3:
+; CHECK: vpkh [[REG1:%v[0-9]+]], %v24, %v24
+; CHECK: vsteg [[REG1]], 0(%r2)
+; CHECK: br %r14
+  %trunc = trunc <8 x i16> %val to <8 x i8>
+  store <8 x i8> %trunc, <8 x i8> *%ptr
+  ret void
+}
+
+; Test a v4i32->v4i1 truncation.
+define void @f4(<4 x i32> %val, <4 x i1> *%ptr) {
+; No expected output, but must compile.
+  %trunc = trunc <4 x i32> %val to <4 x i1>
+  store <4 x i1> %trunc, <4 x i1> *%ptr
+  ret void
+}
+
+; Test a v4i32->v4i8 truncation.  At the moment we use a VPERM rather than
+; a chain of packs.
+define void @f5(<4 x i32> %val, <4 x i8> *%ptr) {
+; CHECK-LABEL: f5:
+; CHECK: vperm [[REG:%v[0-9]+]],
+; CHECK: vstef [[REG]], 0(%r2)
+; CHECK: br %r14
+  %trunc = trunc <4 x i32> %val to <4 x i8>
+  store <4 x i8> %trunc, <4 x i8> *%ptr
+  ret void
+}
+
+; Test a v4i32->v4i16 truncation.
+define void @f6(<4 x i32> %val, <4 x i16> *%ptr) {
+; CHECK-LABEL: f6:
+; CHECK: vpkf [[REG1:%v[0-9]+]], %v24, %v24
+; CHECK: vsteg [[REG1]], 0(%r2)
+; CHECK: br %r14
+  %trunc = trunc <4 x i32> %val to <4 x i16>
+  store <4 x i16> %trunc, <4 x i16> *%ptr
+  ret void
+}
+
+; Test a v2i64->v2i1 truncation.
+define void @f7(<2 x i64> %val, <2 x i1> *%ptr) {
+; No expected output, but must compile.
+  %trunc = trunc <2 x i64> %val to <2 x i1>
+  store <2 x i1> %trunc, <2 x i1> *%ptr
+  ret void
+}
+
+; Test a v2i64->v2i8 truncation.  At the moment we use a VPERM rather than
+; a chain of packs.
+define void @f8(<2 x i64> %val, <2 x i8> *%ptr) {
+; CHECK-LABEL: f8:
+; CHECK: vperm [[REG:%v[0-9]+]],
+; CHECK: vsteh [[REG]], 0(%r2)
+; CHECK: br %r14
+  %trunc = trunc <2 x i64> %val to <2 x i8>
+  store <2 x i8> %trunc, <2 x i8> *%ptr
+  ret void
+}
+
+; Test a v2i64->v2i16 truncation.  At the moment we use a VPERM rather than
+; a chain of packs.
+define void @f9(<2 x i64> %val, <2 x i16> *%ptr) {
+; CHECK-LABEL: f9:
+; CHECK: vperm [[REG:%v[0-9]+]],
+; CHECK: vstef [[REG]], 0(%r2)
+; CHECK: br %r14
+  %trunc = trunc <2 x i64> %val to <2 x i16>
+  store <2 x i16> %trunc, <2 x i16> *%ptr
+  ret void
+}
+
+; Test a v2i64->v2i32 truncation.
+define void @f10(<2 x i64> %val, <2 x i32> *%ptr) {
+; CHECK-LABEL: f10:
+; CHECK: vpkg [[REG1:%v[0-9]+]], %v24, %v24
+; CHECK: vsteg [[REG1]], 0(%r2)
+; CHECK: br %r14
+  %trunc = trunc <2 x i64> %val to <2 x i32>
+  store <2 x i32> %trunc, <2 x i32> *%ptr
+  ret void
+}
diff --git a/test/CodeGen/SystemZ/vec-mul-01.ll b/test/CodeGen/SystemZ/vec-mul-01.ll
new file mode 100644
index 00000000000..5ecc30d4427
--- /dev/null
+++ b/test/CodeGen/SystemZ/vec-mul-01.ll
@@ -0,0 +1,60 @@
+; Test vector multiplication.
+;
+; RUN: llc < %s -mtriple=s390x-linux-gnu -mcpu=z13 | FileCheck %s
+
+; Test a v16i8 multiplication.
+define <16 x i8> @f1(<16 x i8> %dummy, <16 x i8> %val1, <16 x i8> %val2) {
+; CHECK-LABEL: f1:
+; CHECK: vmlb %v24, %v26, %v28
+; CHECK: br %r14
+  %ret = mul <16 x i8> %val1, %val2
+  ret <16 x i8> %ret
+}
+
+; Test a v8i16 multiplication.
+define <8 x i16> @f2(<8 x i16> %dummy, <8 x i16> %val1, <8 x i16> %val2) {
+; CHECK-LABEL: f2:
+; CHECK: vmlhw %v24, %v26, %v28
+; CHECK: br %r14
+  %ret = mul <8 x i16> %val1, %val2
+  ret <8 x i16> %ret
+}
+
+; Test a v4i32 multiplication.
+define <4 x i32> @f3(<4 x i32> %dummy, <4 x i32> %val1, <4 x i32> %val2) {
+; CHECK-LABEL: f3:
+; CHECK: vmlf %v24, %v26, %v28
+; CHECK: br %r14
+  %ret = mul <4 x i32> %val1, %val2
+  ret <4 x i32> %ret
+}
+
+; Test a v2i64 multiplication.  There's no vector equivalent.
+define <2 x i64> @f4(<2 x i64> %dummy, <2 x i64> %val1, <2 x i64> %val2) {
+; CHECK-LABEL: f4:
+; CHECK-NOT: vmlg
+; CHECK: br %r14
+  %ret = mul <2 x i64> %val1, %val2
+  ret <2 x i64> %ret
+}
+
+; Test a v2f64 multiplication.
+define <2 x double> @f5(<2 x double> %dummy, <2 x double> %val1,
+                        <2 x double> %val2) {
+; CHECK-LABEL: f5:
+; CHECK: vfmdb %v24, %v26, %v28
+; CHECK: br %r14
+  %ret = fmul <2 x double> %val1, %val2
+  ret <2 x double> %ret
+}
+
+; Test an f64 multiplication that uses vector registers.
+define double @f6(<2 x double> %val1, <2 x double> %val2) {
+; CHECK-LABEL: f6:
+; CHECK: wfmdb %f0, %v24, %v26
+; CHECK: br %r14
+  %scalar1 = extractelement <2 x double> %val1, i32 0
+  %scalar2 = extractelement <2 x double> %val2, i32 0
+  %ret = fmul double %scalar1, %scalar2
+  ret double %ret
+}
diff --git a/test/CodeGen/SystemZ/vec-mul-02.ll b/test/CodeGen/SystemZ/vec-mul-02.ll
new file mode 100644
index 00000000000..11a651e4997
--- /dev/null
+++ b/test/CodeGen/SystemZ/vec-mul-02.ll
@@ -0,0 +1,63 @@
+; Test vector multiply-and-add.
+;
+; RUN: llc < %s -mtriple=s390x-linux-gnu -mcpu=z13 | FileCheck %s
+
+declare <2 x double> @llvm.fma.v2f64(<2 x double>, <2 x double>, <2 x double>)
+
+; Test a v16i8 multiply-and-add.
+define <16 x i8> @f1(<16 x i8> %dummy, <16 x i8> %val1, <16 x i8> %val2,
+                     <16 x i8> %val3) {
+; CHECK-LABEL: f1:
+; CHECK: vmalb %v24, %v26, %v28, %v30
+; CHECK: br %r14
+  %mul = mul <16 x i8> %val1, %val2
+  %ret = add <16 x i8> %mul, %val3
+  ret <16 x i8> %ret
+}
+
+; Test a v8i16 multiply-and-add.
+define <8 x i16> @f2(<8 x i16> %dummy, <8 x i16> %val1, <8 x i16> %val2,
+                     <8 x i16> %val3) {
+; CHECK-LABEL: f2:
+; CHECK: vmalhw %v24, %v26, %v28, %v30
+; CHECK: br %r14
+  %mul = mul <8 x i16> %val1, %val2
+  %ret = add <8 x i16> %mul, %val3
+  ret <8 x i16> %ret
+}
+
+; Test a v4i32 multiply-and-add.
+define <4 x i32> @f3(<4 x i32> %dummy, <4 x i32> %val1, <4 x i32> %val2,
+                     <4 x i32> %val3) {
+; CHECK-LABEL: f3:
+; CHECK: vmalf %v24, %v26, %v28, %v30
+; CHECK: br %r14
+  %mul = mul <4 x i32> %val1, %val2
+  %ret = add <4 x i32> %mul, %val3
+  ret <4 x i32> %ret
+}
+
+; Test a v2f64 multiply-and-add.
+define <2 x double> @f4(<2 x double> %dummy, <2 x double> %val1,
+                        <2 x double> %val2, <2 x double> %val3) {
+; CHECK-LABEL: f4:
+; CHECK: vfmadb %v24, %v26, %v28, %v30
+; CHECK: br %r14
+  %ret = call <2 x double> @llvm.fma.v2f64 (<2 x double> %val1,
+                                            <2 x double> %val2,
+                                            <2 x double> %val3)
+  ret <2 x double> %ret
+}
+
+; Test a v2f64 multiply-and-subtract.
+define <2 x double> @f5(<2 x double> %dummy, <2 x double> %val1,
+                        <2 x double> %val2, <2 x double> %val3) {
+; CHECK-LABEL: f5:
+; CHECK: vfmsdb %v24, %v26, %v28, %v30
+; CHECK: br %r14
+  %negval3 = fsub <2 x double> <double -0.0, double -0.0>, %val3
+  %ret = call <2 x double> @llvm.fma.v2f64 (<2 x double> %val1,
+                                            <2 x double> %val2,
+                                            <2 x double> %negval3)
+  ret <2 x double> %ret
+}
diff --git a/test/CodeGen/SystemZ/vec-neg-01.ll b/test/CodeGen/SystemZ/vec-neg-01.ll
new file mode 100644
index 00000000000..b1389ce4d6d
--- /dev/null
+++ b/test/CodeGen/SystemZ/vec-neg-01.ll
@@ -0,0 +1,58 @@
+; Test vector negation.
+;
+; RUN: llc < %s -mtriple=s390x-linux-gnu -mcpu=z13 | FileCheck %s
+
+; Test a v16i8 negation.
+define <16 x i8> @f1(<16 x i8> %dummy, <16 x i8> %val) {
+; CHECK-LABEL: f1:
+; CHECK: vlcb %v24, %v26
+; CHECK: br %r14
+  %ret = sub <16 x i8> zeroinitializer, %val
+  ret <16 x i8> %ret
+}
+
+; Test a v8i16 negation.
+define <8 x i16> @f2(<8 x i16> %dummy, <8 x i16> %val) {
+; CHECK-LABEL: f2:
+; CHECK: vlch %v24, %v26
+; CHECK: br %r14
+  %ret = sub <8 x i16> zeroinitializer, %val
+  ret <8 x i16> %ret
+}
+
+; Test a v4i32 negation.
+define <4 x i32> @f3(<4 x i32> %dummy, <4 x i32> %val) {
+; CHECK-LABEL: f3:
+; CHECK: vlcf %v24, %v26
+; CHECK: br %r14
+  %ret = sub <4 x i32> zeroinitializer, %val
+  ret <4 x i32> %ret
+}
+
+; Test a v2i64 negation.
+define <2 x i64> @f4(<2 x i64> %dummy, <2 x i64> %val) {
+; CHECK-LABEL: f4:
+; CHECK: vlcg %v24, %v26
+; CHECK: br %r14
+  %ret = sub <2 x i64> zeroinitializer, %val
+  ret <2 x i64> %ret
+}
+
+; Test a v2f64 negation.
+define <2 x double> @f5(<2 x double> %dummy, <2 x double> %val) {
+; CHECK-LABEL: f5:
+; CHECK: vflcdb %v24, %v26
+; CHECK: br %r14
+  %ret = fsub <2 x double> <double -0.0, double -0.0>, %val
+  ret <2 x double> %ret
+}
+
+; Test an f64 negation that uses vector registers.
+define double @f6(<2 x double> %val) {
+; CHECK-LABEL: f6:
+; CHECK: wflcdb %f0, %v24
+; CHECK: br %r14
+  %scalar = extractelement <2 x double> %val, i32 0
+  %ret = fsub double -0.0, %scalar
+  ret double %ret
+}
diff --git a/test/CodeGen/SystemZ/vec-or-01.ll b/test/CodeGen/SystemZ/vec-or-01.ll
new file mode 100644
index 00000000000..789150ad2d1
--- /dev/null
+++ b/test/CodeGen/SystemZ/vec-or-01.ll
@@ -0,0 +1,39 @@
+; Test vector OR.
+;
+; RUN: llc < %s -mtriple=s390x-linux-gnu -mcpu=z13 | FileCheck %s
+
+; Test a v16i8 OR.
+define <16 x i8> @f1(<16 x i8> %dummy, <16 x i8> %val1, <16 x i8> %val2) {
+; CHECK-LABEL: f1:
+; CHECK: vo %v24, %v26, %v28
+; CHECK: br %r14
+  %ret = or <16 x i8> %val1, %val2
+  ret <16 x i8> %ret
+}
+
+; Test a v8i16 OR.
+define <8 x i16> @f2(<8 x i16> %dummy, <8 x i16> %val1, <8 x i16> %val2) {
+; CHECK-LABEL: f2:
+; CHECK: vo %v24, %v26, %v28
+; CHECK: br %r14
+  %ret = or <8 x i16> %val1, %val2
+  ret <8 x i16> %ret
+}
+
+; Test a v4i32 OR.
+define <4 x i32> @f3(<4 x i32> %dummy, <4 x i32> %val1, <4 x i32> %val2) {
+; CHECK-LABEL: f3:
+; CHECK: vo %v24, %v26, %v28
+; CHECK: br %r14
+  %ret = or <4 x i32> %val1, %val2
+  ret <4 x i32> %ret
+}
+
+; Test a v2i64 OR.
+define <2 x i64> @f4(<2 x i64> %dummy, <2 x i64> %val1, <2 x i64> %val2) {
+; CHECK-LABEL: f4:
+; CHECK: vo %v24, %v26, %v28
+; CHECK: br %r14
+  %ret = or <2 x i64> %val1, %val2
+  ret <2 x i64> %ret
+}
diff --git a/test/CodeGen/SystemZ/vec-or-02.ll b/test/CodeGen/SystemZ/vec-or-02.ll
new file mode 100644
index 00000000000..eeb86e36ff0
--- /dev/null
+++ b/test/CodeGen/SystemZ/vec-or-02.ll
@@ -0,0 +1,107 @@
+; Test vector (or (and X, Z), (and Y, (not Z))) patterns.
+;
+; RUN: llc < %s -mtriple=s390x-linux-gnu -mcpu=z13 | FileCheck %s
+
+; Test v16i8.
+define <16 x i8> @f1(<16 x i8> %val1, <16 x i8> %val2, <16 x i8> %val3) {
+; CHECK-LABEL: f1:
+; CHECK: vsel %v24, %v24, %v26, %v28
+; CHECK: br %r14
+  %not = xor <16 x i8> %val3, <i8 -1, i8 -1, i8 -1, i8 -1,
+                               i8 -1, i8 -1, i8 -1, i8 -1,
+                               i8 -1, i8 -1, i8 -1, i8 -1,
+                               i8 -1, i8 -1, i8 -1, i8 -1>
+  %and1 = and <16 x i8> %val1, %val3
+  %and2 = and <16 x i8> %val2, %not
+  %ret = or <16 x i8> %and1, %and2
+  ret <16 x i8> %ret
+}
+
+; ...and again with the XOR applied to the other operand of the AND.
+define <16 x i8> @f2(<16 x i8> %val1, <16 x i8> %val2, <16 x i8> %val3) {
+; CHECK-LABEL: f2:
+; CHECK: vsel %v24, %v26, %v24, %v28
+; CHECK: br %r14
+  %not = xor <16 x i8> %val3, <i8 -1, i8 -1, i8 -1, i8 -1,
+                               i8 -1, i8 -1, i8 -1, i8 -1,
+                               i8 -1, i8 -1, i8 -1, i8 -1,
+                               i8 -1, i8 -1, i8 -1, i8 -1>
+  %and1 = and <16 x i8> %val1, %not
+  %and2 = and <16 x i8> %val2, %val3
+  %ret = or <16 x i8> %and1, %and2
+  ret <16 x i8> %ret
+}
+
+; Test v8i16.
+define <8 x i16> @f3(<8 x i16> %val1, <8 x i16> %val2, <8 x i16> %val3) {
+; CHECK-LABEL: f3:
+; CHECK: vsel %v24, %v24, %v26, %v28
+; CHECK: br %r14
+  %not = xor <8 x i16> %val3, <i16 -1, i16 -1, i16 -1, i16 -1,
+                               i16 -1, i16 -1, i16 -1, i16 -1>
+  %and1 = and <8 x i16> %val1, %val3
+  %and2 = and <8 x i16> %val2, %not
+  %ret = or <8 x i16> %and1, %and2
+  ret <8 x i16> %ret
+}
+
+; ...and again with the XOR applied to the other operand of the AND.
+define <8 x i16> @f4(<8 x i16> %val1, <8 x i16> %val2, <8 x i16> %val3) {
+; CHECK-LABEL: f4:
+; CHECK: vsel %v24, %v26, %v24, %v28
+; CHECK: br %r14
+  %not = xor <8 x i16> %val3, <i16 -1, i16 -1, i16 -1, i16 -1,
+                               i16 -1, i16 -1, i16 -1, i16 -1>
+  %and1 = and <8 x i16> %val1, %not
+  %and2 = and <8 x i16> %val2, %val3
+  %ret = or <8 x i16> %and1, %and2
+  ret <8 x i16> %ret
+}
+
+; Test v4i32.
+define <4 x i32> @f5(<4 x i32> %val1, <4 x i32> %val2, <4 x i32> %val3) {
+; CHECK-LABEL: f5:
+; CHECK: vsel %v24, %v24, %v26, %v28
+; CHECK: br %r14
+  %not = xor <4 x i32> %val3, <i32 -1, i32 -1, i32 -1, i32 -1>
+  %and1 = and <4 x i32> %val1, %val3
+  %and2 = and <4 x i32> %val2, %not
+  %ret = or <4 x i32> %and1, %and2
+  ret <4 x i32> %ret
+}
+
+; ...and again with the XOR applied to the other operand of the AND.
+define <4 x i32> @f6(<4 x i32> %val1, <4 x i32> %val2, <4 x i32> %val3) {
+; CHECK-LABEL: f6:
+; CHECK: vsel %v24, %v26, %v24, %v28
+; CHECK: br %r14
+  %not = xor <4 x i32> %val3, <i32 -1, i32 -1, i32 -1, i32 -1>
+  %and1 = and <4 x i32> %val1, %not
+  %and2 = and <4 x i32> %val2, %val3
+  %ret = or <4 x i32> %and1, %and2
+  ret <4 x i32> %ret
+}
+
+; Test v2i64.
+define <2 x i64> @f7(<2 x i64> %val1, <2 x i64> %val2, <2 x i64> %val3) {
+; CHECK-LABEL: f7:
+; CHECK: vsel %v24, %v24, %v26, %v28
+; CHECK: br %r14
+  %not = xor <2 x i64> %val3, <i64 -1, i64 -1>
+  %and1 = and <2 x i64> %val1, %val3
+  %and2 = and <2 x i64> %val2, %not
+  %ret = or <2 x i64> %and1, %and2
+  ret <2 x i64> %ret
+}
+
+; ...and again with the XOR applied to the other operand of the AND.
+define <2 x i64> @f8(<2 x i64> %val1, <2 x i64> %val2, <2 x i64> %val3) {
+; CHECK-LABEL: f8:
+; CHECK: vsel %v24, %v26, %v24, %v28
+; CHECK: br %r14
+  %not = xor <2 x i64> %val3, <i64 -1, i64 -1>
+  %and1 = and <2 x i64> %val1, %not
+  %and2 = and <2 x i64> %val2, %val3
+  %ret = or <2 x i64> %and1, %and2
+  ret <2 x i64> %ret
+}
diff --git a/test/CodeGen/SystemZ/vec-perm-01.ll b/test/CodeGen/SystemZ/vec-perm-01.ll
new file mode 100644
index 00000000000..4beec05eaec
--- /dev/null
+++ b/test/CodeGen/SystemZ/vec-perm-01.ll
@@ -0,0 +1,175 @@
+; Test vector splat.
+;
+; RUN: llc < %s -mtriple=s390x-linux-gnu -mcpu=z13 | FileCheck %s
+
+; Test v16i8 splat of the first element.
+define <16 x i8> @f1(<16 x i8> %val) {
+; CHECK-LABEL: f1:
+; CHECK: vrepb %v24, %v24, 0
+; CHECK: br %r14
+  %ret = shufflevector <16 x i8> %val, <16 x i8> undef,
+                       <16 x i32> zeroinitializer
+  ret <16 x i8> %ret
+}
+
+; Test v16i8 splat of the last element.
+define <16 x i8> @f2(<16 x i8> %val) {
+; CHECK-LABEL: f2:
+; CHECK: vrepb %v24, %v24, 15
+; CHECK: br %r14
+  %ret = shufflevector <16 x i8> %val, <16 x i8> undef,
+                       <16 x i32> <i32 15, i32 15, i32 15, i32 15,
+                                   i32 15, i32 15, i32 15, i32 15,
+                                   i32 15, i32 15, i32 15, i32 15,
+                                   i32 15, i32 15, i32 15, i32 15>
+  ret <16 x i8> %ret
+}
+
+; Test v16i8 splat of an arbitrary element, using the second operand of
+; the shufflevector.
+define <16 x i8> @f3(<16 x i8> %val) {
+; CHECK-LABEL: f3:
+; CHECK: vrepb %v24, %v24, 4
+; CHECK: br %r14
+  %ret = shufflevector <16 x i8> undef, <16 x i8> %val,
+                       <16 x i32> <i32 20, i32 20, i32 20, i32 20,
+                                   i32 20, i32 20, i32 20, i32 20,
+                                   i32 20, i32 20, i32 20, i32 20,
+                                   i32 20, i32 20, i32 20, i32 20>
+  ret <16 x i8> %ret
+}
+
+; Test v8i16 splat of the first element.
+define <8 x i16> @f4(<8 x i16> %val) {
+; CHECK-LABEL: f4:
+; CHECK: vreph %v24, %v24, 0
+; CHECK: br %r14
+  %ret = shufflevector <8 x i16> %val, <8 x i16> undef,
+                       <8 x i32> zeroinitializer
+  ret <8 x i16> %ret
+}
+
+; Test v8i16 splat of the last element.
+define <8 x i16> @f5(<8 x i16> %val) {
+; CHECK-LABEL: f5:
+; CHECK: vreph %v24, %v24, 7
+; CHECK: br %r14
+  %ret = shufflevector <8 x i16> %val, <8 x i16> undef,
+                       <8 x i32> <i32 7, i32 7, i32 7, i32 7,
+                                  i32 7, i32 7, i32 7, i32 7>
+  ret <8 x i16> %ret
+}
+
+; Test v8i16 splat of an arbitrary element, using the second operand of
+; the shufflevector.
+define <8 x i16> @f6(<8 x i16> %val) {
+; CHECK-LABEL: f6:
+; CHECK: vreph %v24, %v24, 2
+; CHECK: br %r14
+  %ret = shufflevector <8 x i16> undef, <8 x i16> %val,
+                       <8 x i32> <i32 10, i32 10, i32 10, i32 10,
+                                  i32 10, i32 10, i32 10, i32 10>
+  ret <8 x i16> %ret
+}
+
+; Test v4i32 splat of the first element.
+define <4 x i32> @f7(<4 x i32> %val) {
+; CHECK-LABEL: f7:
+; CHECK: vrepf %v24, %v24, 0
+; CHECK: br %r14
+  %ret = shufflevector <4 x i32> %val, <4 x i32> undef,
+                       <4 x i32> zeroinitializer
+  ret <4 x i32> %ret
+}
+
+; Test v4i32 splat of the last element.
+define <4 x i32> @f8(<4 x i32> %val) {
+; CHECK-LABEL: f8:
+; CHECK: vrepf %v24, %v24, 3
+; CHECK: br %r14
+  %ret = shufflevector <4 x i32> %val, <4 x i32> undef,
+                       <4 x i32> <i32 3, i32 3, i32 3, i32 3>
+  ret <4 x i32> %ret
+}
+
+; Test v4i32 splat of an arbitrary element, using the second operand of
+; the shufflevector.
+define <4 x i32> @f9(<4 x i32> %val) {
+; CHECK-LABEL: f9:
+; CHECK: vrepf %v24, %v24, 1
+; CHECK: br %r14
+  %ret = shufflevector <4 x i32> undef, <4 x i32> %val,
+                       <4 x i32> <i32 5, i32 5, i32 5, i32 5>
+  ret <4 x i32> %ret
+}
+
+; Test v2i64 splat of the first element.
+define <2 x i64> @f10(<2 x i64> %val) {
+; CHECK-LABEL: f10:
+; CHECK: vrepg %v24, %v24, 0
+; CHECK: br %r14
+  %ret = shufflevector <2 x i64> %val, <2 x i64> undef,
+                       <2 x i32> zeroinitializer
+  ret <2 x i64> %ret
+}
+
+; Test v2i64 splat of the last element.
+define <2 x i64> @f11(<2 x i64> %val) {
+; CHECK-LABEL: f11:
+; CHECK: vrepg %v24, %v24, 1
+; CHECK: br %r14
+  %ret = shufflevector <2 x i64> %val, <2 x i64> undef,
+                       <2 x i32> <i32 1, i32 1>
+  ret <2 x i64> %ret
+}
+
+; Test v4f32 splat of the first element.
+define <4 x float> @f12(<4 x float> %val) {
+; CHECK-LABEL: f12:
+; CHECK: vrepf %v24, %v24, 0
+; CHECK: br %r14
+  %ret = shufflevector <4 x float> %val, <4 x float> undef,
+                       <4 x i32> zeroinitializer
+  ret <4 x float> %ret
+}
+
+; Test v4f32 splat of the last element.
+define <4 x float> @f13(<4 x float> %val) {
+; CHECK-LABEL: f13:
+; CHECK: vrepf %v24, %v24, 3
+; CHECK: br %r14
+  %ret = shufflevector <4 x float> %val, <4 x float> undef,
+                       <4 x i32> <i32 3, i32 3, i32 3, i32 3>
+  ret <4 x float> %ret
+}
+
+; Test v4f32 splat of an arbitrary element, using the second operand of
+; the shufflevector.
+define <4 x float> @f14(<4 x float> %val) {
+; CHECK-LABEL: f14:
+; CHECK: vrepf %v24, %v24, 1
+; CHECK: br %r14
+  %ret = shufflevector <4 x float> undef, <4 x float> %val,
+                       <4 x i32> <i32 5, i32 5, i32 5, i32 5>
+  ret <4 x float> %ret
+}
+
+; Test v2f64 splat of the first element.
+define <2 x double> @f15(<2 x double> %val) {
+; CHECK-LABEL: f15:
+; CHECK: vrepg %v24, %v24, 0
+; CHECK: br %r14
+  %ret = shufflevector <2 x double> %val, <2 x double> undef,
+                       <2 x i32> zeroinitializer
+  ret <2 x double> %ret
+}
+
+; Test v2f64 splat of the last element.
+define <2 x double> @f16(<2 x double> %val) {
+; CHECK-LABEL: f16:
+; CHECK: vrepg %v24, %v24, 1
+; CHECK: br %r14
+  %ret = shufflevector <2 x double> %val, <2 x double> undef,
+                       <2 x i32> <i32 1, i32 1>
+  ret <2 x double> %ret
+}
diff --git a/test/CodeGen/SystemZ/vec-perm-02.ll b/test/CodeGen/SystemZ/vec-perm-02.ll
new file mode 100644
index 00000000000..e5c6df8e955
--- /dev/null
+++ b/test/CodeGen/SystemZ/vec-perm-02.ll
@@ -0,0 +1,200 @@
+; Test replications of a scalar register value, represented as splats.
+;
+; RUN: llc < %s -mtriple=s390x-linux-gnu -mcpu=z13 | FileCheck %s
+
+; Test v16i8 splat of the first element.
+define <16 x i8> @f1(i8 %scalar) {
+; CHECK-LABEL: f1:
+; CHECK: vlvgp [[REG:%v[0-9]+]], %r2, %r2
+; CHECK: vrepb %v24, [[REG]], 7
+; CHECK: br %r14
+  %val = insertelement <16 x i8> undef, i8 %scalar, i32 0
+  %ret = shufflevector <16 x i8> %val, <16 x i8> undef,
+                       <16 x i32> zeroinitializer
+  ret <16 x i8> %ret
+}
+
+; Test v16i8 splat of the last element.
+define <16 x i8> @f2(i8 %scalar) {
+; CHECK-LABEL: f2:
+; CHECK: vlvgp [[REG:%v[0-9]+]], %r2, %r2
+; CHECK: vrepb %v24, [[REG]], 7
+; CHECK: br %r14
+  %val = insertelement <16 x i8> undef, i8 %scalar, i32 15
+  %ret = shufflevector <16 x i8> %val, <16 x i8> undef,
+                       <16 x i32> <i32 15, i32 15, i32 15, i32 15,
+                                   i32 15, i32 15, i32 15, i32 15,
+                                   i32 15, i32 15, i32 15, i32 15,
+                                   i32 15, i32 15, i32 15, i32 15>
+  ret <16 x i8> %ret
+}
+
+; Test v16i8 splat of an arbitrary element, using the second operand of
+; the shufflevector.
+define <16 x i8> @f3(i8 %scalar) {
+; CHECK-LABEL: f3:
+; CHECK: vlvgp [[REG:%v[0-9]+]], %r2, %r2
+; CHECK: vrepb %v24, [[REG]], 7
+; CHECK: br %r14
+  %val = insertelement <16 x i8> undef, i8 %scalar, i32 4
+  %ret = shufflevector <16 x i8> undef, <16 x i8> %val,
+                       <16 x i32> <i32 20, i32 20, i32 20, i32 20,
+                                   i32 20, i32 20, i32 20, i32 20,
+                                   i32 20, i32 20, i32 20, i32 20,
+                                   i32 20, i32 20, i32 20, i32 20>
+  ret <16 x i8> %ret
+}
+
+; Test v8i16 splat of the first element.
+define <8 x i16> @f4(i16 %scalar) {
+; CHECK-LABEL: f4:
+; CHECK: vlvgp [[REG:%v[0-9]+]], %r2, %r2
+; CHECK: vreph %v24, [[REG]], 3
+; CHECK: br %r14
+  %val = insertelement <8 x i16> undef, i16 %scalar, i32 0
+  %ret = shufflevector <8 x i16> %val, <8 x i16> undef,
+                       <8 x i32> zeroinitializer
+  ret <8 x i16> %ret
+}
+
+; Test v8i16 splat of the last element.
+define <8 x i16> @f5(i16 %scalar) {
+; CHECK-LABEL: f5:
+; CHECK: vlvgp [[REG:%v[0-9]+]], %r2, %r2
+; CHECK: vreph %v24, [[REG]], 3
+; CHECK: br %r14
+  %val = insertelement <8 x i16> undef, i16 %scalar, i32 7
+  %ret = shufflevector <8 x i16> %val, <8 x i16> undef,
+                       <8 x i32> <i32 7, i32 7, i32 7, i32 7,
+                                  i32 7, i32 7, i32 7, i32 7>
+  ret <8 x i16> %ret
+}
+
+; Test v8i16 splat of an arbitrary element, using the second operand of
+; the shufflevector.
+define <8 x i16> @f6(i16 %scalar) {
+; CHECK-LABEL: f6:
+; CHECK: vlvgp [[REG:%v[0-9]+]], %r2, %r2
+; CHECK: vreph %v24, [[REG]], 3
+; CHECK: br %r14
+  %val = insertelement <8 x i16> undef, i16 %scalar, i32 2
+  %ret = shufflevector <8 x i16> undef, <8 x i16> %val,
+                       <8 x i32> <i32 10, i32 10, i32 10, i32 10,
+                                  i32 10, i32 10, i32 10, i32 10>
+  ret <8 x i16> %ret
+}
+
+; Test v4i32 splat of the first element.
+define <4 x i32> @f7(i32 %scalar) {
+; CHECK-LABEL: f7:
+; CHECK: vlvgp [[REG:%v[0-9]+]], %r2, %r2
+; CHECK: vrepf %v24, [[REG]], 1
+; CHECK: br %r14
+  %val = insertelement <4 x i32> undef, i32 %scalar, i32 0
+  %ret = shufflevector <4 x i32> %val, <4 x i32> undef,
+                       <4 x i32> zeroinitializer
+  ret <4 x i32> %ret
+}
+
+; Test v4i32 splat of the last element.
+define <4 x i32> @f8(i32 %scalar) {
+; CHECK-LABEL: f8:
+; CHECK: vlvgp [[REG:%v[0-9]+]], %r2, %r2
+; CHECK: vrepf %v24, [[REG]], 1
+; CHECK: br %r14
+  %val = insertelement <4 x i32> undef, i32 %scalar, i32 3
+  %ret = shufflevector <4 x i32> %val, <4 x i32> undef,
+                       <4 x i32> <i32 3, i32 3, i32 3, i32 3>
+  ret <4 x i32> %ret
+}
+
+; Test v4i32 splat of an arbitrary element, using the second operand of
+; the shufflevector.
+define <4 x i32> @f9(i32 %scalar) {
+; CHECK-LABEL: f9:
+; CHECK: vlvgp [[REG:%v[0-9]+]], %r2, %r2
+; CHECK: vrepf %v24, [[REG]], 1
+; CHECK: br %r14
+  %val = insertelement <4 x i32> undef, i32 %scalar, i32 1
+  %ret = shufflevector <4 x i32> undef, <4 x i32> %val,
+                       <4 x i32> <i32 5, i32 5, i32 5, i32 5>
+  ret <4 x i32> %ret
+}
+
+; Test v2i64 splat of the first element.
+define <2 x i64> @f10(i64 %scalar) {
+; CHECK-LABEL: f10:
+; CHECK: vlvgp %v24, %r2, %r2
+; CHECK: br %r14
+  %val = insertelement <2 x i64> undef, i64 %scalar, i32 0
+  %ret = shufflevector <2 x i64> %val, <2 x i64> undef,
+                       <2 x i32> zeroinitializer
+  ret <2 x i64> %ret
+}
+
+; Test v2i64 splat of the last element.
+define <2 x i64> @f11(i64 %scalar) {
+; CHECK-LABEL: f11:
+; CHECK: vlvgp %v24, %r2, %r2
+; CHECK: br %r14
+  %val = insertelement <2 x i64> undef, i64 %scalar, i32 1
+  %ret = shufflevector <2 x i64> %val, <2 x i64> undef,
+                       <2 x i32> <i32 1, i32 1>
+  ret <2 x i64> %ret
+}
+
+; Test v4f32 splat of the first element.
+define <4 x float> @f12(float %scalar) {
+; CHECK-LABEL: f12:
+; CHECK: vrepf %v24, %v0, 0
+; CHECK: br %r14
+  %val = insertelement <4 x float> undef, float %scalar, i32 0
+  %ret = shufflevector <4 x float> %val, <4 x float> undef,
+                       <4 x i32> zeroinitializer
+  ret <4 x float> %ret
+}
+
+; Test v4f32 splat of the last element.
+define <4 x float> @f13(float %scalar) {
+; CHECK-LABEL: f13:
+; CHECK: vrepf %v24, %v0, 0
+; CHECK: br %r14
+  %val = insertelement <4 x float> undef, float %scalar, i32 3
+  %ret = shufflevector <4 x float> %val, <4 x float> undef,
+                       <4 x i32> <i32 3, i32 3, i32 3, i32 3>
+  ret <4 x float> %ret
+}
+
+; Test v4f32 splat of an arbitrary element, using the second operand of
+; the shufflevector.
+define <4 x float> @f14(float %scalar) {
+; CHECK-LABEL: f14:
+; CHECK: vrepf %v24, %v0, 0
+; CHECK: br %r14
+  %val = insertelement <4 x float> undef, float %scalar, i32 1
+  %ret = shufflevector <4 x float> undef, <4 x float> %val,
+                       <4 x i32> <i32 5, i32 5, i32 5, i32 5>
+  ret <4 x float> %ret
+}
+
+; Test v2f64 splat of the first element.
+define <2 x double> @f15(double %scalar) {
+; CHECK-LABEL: f15:
+; CHECK: vrepg %v24, %v0, 0
+; CHECK: br %r14
+  %val = insertelement <2 x double> undef, double %scalar, i32 0
+  %ret = shufflevector <2 x double> %val, <2 x double> undef,
+                       <2 x i32> zeroinitializer
+  ret <2 x double> %ret
+}
+
+; Test v2f64 splat of the last element.
+define <2 x double> @f16(double %scalar) {
+; CHECK-LABEL: f16:
+; CHECK: vrepg %v24, %v0, 0
+; CHECK: br %r14
+  %val = insertelement <2 x double> undef, double %scalar, i32 1
+  %ret = shufflevector <2 x double> %val, <2 x double> undef,
+                       <2 x i32> <i32 1, i32 1>
+  ret <2 x double> %ret
+}
diff --git a/test/CodeGen/SystemZ/vec-perm-03.ll b/test/CodeGen/SystemZ/vec-perm-03.ll
new file mode 100644
index 00000000000..663815549c3
--- /dev/null
+++ b/test/CodeGen/SystemZ/vec-perm-03.ll
@@ -0,0 +1,251 @@
+; Test replications of a scalar memory value, represented as splats.
+;
+; RUN: llc < %s -mtriple=s390x-linux-gnu -mcpu=z13 | FileCheck %s
+
+; Test a v16i8 replicating load with no offset.
+define <16 x i8> @f1(i8 *%ptr) {
+; CHECK-LABEL: f1:
+; CHECK: vlrepb %v24, 0(%r2)
+; CHECK: br %r14
+  %scalar = load i8, i8 *%ptr
+  %val = insertelement <16 x i8> undef, i8 %scalar, i32 0
+  %ret = shufflevector <16 x i8> %val, <16 x i8> undef,
+                       <16 x i32> zeroinitializer
+  ret <16 x i8> %ret
+}
+
+; Test a v16i8 replicating load with the maximum in-range offset.
+define <16 x i8> @f2(i8 *%base) {
+; CHECK-LABEL: f2:
+; CHECK: vlrepb %v24, 4095(%r2)
+; CHECK: br %r14
+  %ptr = getelementptr i8, i8 *%base, i64 4095
+  %scalar = load i8, i8 *%ptr
+  %val = insertelement <16 x i8> undef, i8 %scalar, i32 0
+  %ret = shufflevector <16 x i8> %val, <16 x i8> undef,
+                       <16 x i32> zeroinitializer
+  ret <16 x i8> %ret
+}
+
+; Test a v16i8 replicating load with the first out-of-range offset.
+define <16 x i8> @f3(i8 *%base) {
+; CHECK-LABEL: f3:
+; CHECK: aghi %r2, 4096
+; CHECK: vlrepb %v24, 0(%r2)
+; CHECK: br %r14
+  %ptr = getelementptr i8, i8 *%base, i64 4096
+  %scalar = load i8, i8 *%ptr
+  %val = insertelement <16 x i8> undef, i8 %scalar, i32 0
+  %ret = shufflevector <16 x i8> %val, <16 x i8> undef,
+                       <16 x i32> zeroinitializer
+  ret <16 x i8> %ret
+}
+
+; Test a v8i16 replicating load with no offset.
+define <8 x i16> @f4(i16 *%ptr) {
+; CHECK-LABEL: f4:
+; CHECK: vlreph %v24, 0(%r2)
+; CHECK: br %r14
+  %scalar = load i16, i16 *%ptr
+  %val = insertelement <8 x i16> undef, i16 %scalar, i32 0
+  %ret = shufflevector <8 x i16> %val, <8 x i16> undef,
+                       <8 x i32> zeroinitializer
+  ret <8 x i16> %ret
+}
+
+; Test a v8i16 replicating load with the maximum in-range offset.
+define <8 x i16> @f5(i16 *%base) {
+; CHECK-LABEL: f5:
+; CHECK: vlreph %v24, 4094(%r2)
+; CHECK: br %r14
+  %ptr = getelementptr i16, i16 *%base, i64 2047
+  %scalar = load i16, i16 *%ptr
+  %val = insertelement <8 x i16> undef, i16 %scalar, i32 0
+  %ret = shufflevector <8 x i16> %val, <8 x i16> undef,
+                       <8 x i32> zeroinitializer
+  ret <8 x i16> %ret
+}
+
+; Test a v8i16 replicating load with the first out-of-range offset.
+define <8 x i16> @f6(i16 *%base) {
+; CHECK-LABEL: f6:
+; CHECK: aghi %r2, 4096
+; CHECK: vlreph %v24, 0(%r2)
+; CHECK: br %r14
+  %ptr = getelementptr i16, i16 *%base, i64 2048
+  %scalar = load i16, i16 *%ptr
+  %val = insertelement <8 x i16> undef, i16 %scalar, i32 0
+  %ret = shufflevector <8 x i16> %val, <8 x i16> undef,
+                       <8 x i32> zeroinitializer
+  ret <8 x i16> %ret
+}
+
+; Test a v4i32 replicating load with no offset.
+define <4 x i32> @f7(i32 *%ptr) {
+; CHECK-LABEL: f7:
+; CHECK: vlrepf %v24, 0(%r2)
+; CHECK: br %r14
+  %scalar = load i32, i32 *%ptr
+  %val = insertelement <4 x i32> undef, i32 %scalar, i32 0
+  %ret = shufflevector <4 x i32> %val, <4 x i32> undef,
+                       <4 x i32> zeroinitializer
+  ret <4 x i32> %ret
+}
+
+; Test a v4i32 replicating load with the maximum in-range offset.
+define <4 x i32> @f8(i32 *%base) {
+; CHECK-LABEL: f8:
+; CHECK: vlrepf %v24, 4092(%r2)
+; CHECK: br %r14
+  %ptr = getelementptr i32, i32 *%base, i64 1023
+  %scalar = load i32, i32 *%ptr
+  %val = insertelement <4 x i32> undef, i32 %scalar, i32 0
+  %ret = shufflevector <4 x i32> %val, <4 x i32> undef,
+                       <4 x i32> zeroinitializer
+  ret <4 x i32> %ret
+}
+
+; Test a v4i32 replicating load with the first out-of-range offset.
+define <4 x i32> @f9(i32 *%base) {
+; CHECK-LABEL: f9:
+; CHECK: aghi %r2, 4096
+; CHECK: vlrepf %v24, 0(%r2)
+; CHECK: br %r14
+  %ptr = getelementptr i32, i32 *%base, i64 1024
+  %scalar = load i32, i32 *%ptr
+  %val = insertelement <4 x i32> undef, i32 %scalar, i32 0
+  %ret = shufflevector <4 x i32> %val, <4 x i32> undef,
+                       <4 x i32> zeroinitializer
+  ret <4 x i32> %ret
+}
+
+; Test a v2i64 replicating load with no offset.
+define <2 x i64> @f10(i64 *%ptr) {
+; CHECK-LABEL: f10:
+; CHECK: vlrepg %v24, 0(%r2)
+; CHECK: br %r14
+  %scalar = load i64, i64 *%ptr
+  %val = insertelement <2 x i64> undef, i64 %scalar, i32 0
+  %ret = shufflevector <2 x i64> %val, <2 x i64> undef,
+                       <2 x i32> zeroinitializer
+  ret <2 x i64> %ret
+}
+
+; Test a v2i64 replicating load with the maximum in-range offset.
+define <2 x i64> @f11(i64 *%base) {
+; CHECK-LABEL: f11:
+; CHECK: vlrepg %v24, 4088(%r2)
+; CHECK: br %r14
+  %ptr = getelementptr i64, i64 *%base, i32 511
+  %scalar = load i64, i64 *%ptr
+  %val = insertelement <2 x i64> undef, i64 %scalar, i32 0
+  %ret = shufflevector <2 x i64> %val, <2 x i64> undef,
+                       <2 x i32> zeroinitializer
+  ret <2 x i64> %ret
+}
+
+; Test a v2i64 replicating load with the first out-of-range offset.
+define <2 x i64> @f12(i64 *%base) {
+; CHECK-LABEL: f12:
+; CHECK: aghi %r2, 4096
+; CHECK: vlrepg %v24, 0(%r2)
+; CHECK: br %r14
+  %ptr = getelementptr i64, i64 *%base, i32 512
+  %scalar = load i64, i64 *%ptr
+  %val = insertelement <2 x i64> undef, i64 %scalar, i32 0
+  %ret = shufflevector <2 x i64> %val, <2 x i64> undef,
+                       <2 x i32> zeroinitializer
+  ret <2 x i64> %ret
+}
+
+; Test a v4f32 replicating load with no offset.
+define <4 x float> @f13(float *%ptr) {
+; CHECK-LABEL: f13:
+; CHECK: vlrepf %v24, 0(%r2)
+; CHECK: br %r14
+  %scalar = load float, float *%ptr
+  %val = insertelement <4 x float> undef, float %scalar, i32 0
+  %ret = shufflevector <4 x float> %val, <4 x float> undef,
+                       <4 x i32> zeroinitializer
+  ret <4 x float> %ret
+}
+
+; Test a v4f32 replicating load with the maximum in-range offset.
+define <4 x float> @f14(float *%base) {
+; CHECK-LABEL: f14:
+; CHECK: vlrepf %v24, 4092(%r2)
+; CHECK: br %r14
+  %ptr = getelementptr float, float *%base, i64 1023
+  %scalar = load float, float *%ptr
+  %val = insertelement <4 x float> undef, float %scalar, i32 0
+  %ret = shufflevector <4 x float> %val, <4 x float> undef,
+                       <4 x i32> zeroinitializer
+  ret <4 x float> %ret
+}
+
+; Test a v4f32 replicating load with the first out-of-range offset.
+define <4 x float> @f15(float *%base) {
+; CHECK-LABEL: f15:
+; CHECK: aghi %r2, 4096
+; CHECK: vlrepf %v24, 0(%r2)
+; CHECK: br %r14
+  %ptr = getelementptr float, float *%base, i64 1024
+  %scalar = load float, float *%ptr
+  %val = insertelement <4 x float> undef, float %scalar, i32 0
+  %ret = shufflevector <4 x float> %val, <4 x float> undef,
+                       <4 x i32> zeroinitializer
+  ret <4 x float> %ret
+}
+
+; Test a v2f64 replicating load with no offset.
+define <2 x double> @f16(double *%ptr) {
+; CHECK-LABEL: f16:
+; CHECK: vlrepg %v24, 0(%r2)
+; CHECK: br %r14
+  %scalar = load double, double *%ptr
+  %val = insertelement <2 x double> undef, double %scalar, i32 0
+  %ret = shufflevector <2 x double> %val, <2 x double> undef,
+                       <2 x i32> zeroinitializer
+  ret <2 x double> %ret
+}
+
+; Test a v2f64 replicating load with the maximum in-range offset.
+define <2 x double> @f17(double *%base) {
+; CHECK-LABEL: f17:
+; CHECK: vlrepg %v24, 4088(%r2)
+; CHECK: br %r14
+  %ptr = getelementptr double, double *%base, i32 511
+  %scalar = load double, double *%ptr
+  %val = insertelement <2 x double> undef, double %scalar, i32 0
+  %ret = shufflevector <2 x double> %val, <2 x double> undef,
+                       <2 x i32> zeroinitializer
+  ret <2 x double> %ret
+}
+
+; Test a v2f64 replicating load with the first out-of-range offset.
+define <2 x double> @f18(double *%base) {
+; CHECK-LABEL: f18:
+; CHECK: aghi %r2, 4096
+; CHECK: vlrepg %v24, 0(%r2)
+; CHECK: br %r14
+  %ptr = getelementptr double, double *%base, i32 512
+  %scalar = load double, double *%ptr
+  %val = insertelement <2 x double> undef, double %scalar, i32 0
+  %ret = shufflevector <2 x double> %val, <2 x double> undef,
+                       <2 x i32> zeroinitializer
+  ret <2 x double> %ret
+}
+
+; Test a v16i8 replicating load with an index.
+define <16 x i8> @f19(i8 *%base, i64 %index) {
+; CHECK-LABEL: f19:
+; CHECK: vlrepb %v24, 1023(%r3,%r2)
+; CHECK: br %r14
+  %ptr1 = getelementptr i8, i8 *%base, i64 %index
+  %ptr = getelementptr i8, i8 *%ptr1, i64 1023
+  %scalar = load i8, i8 *%ptr
+  %val = insertelement <16 x i8> undef, i8 %scalar, i32 0
+  %ret = shufflevector <16 x i8> %val, <16 x i8> undef,
+                       <16 x i32> zeroinitializer
+  ret <16 x i8> %ret
+}
diff --git a/test/CodeGen/SystemZ/vec-perm-04.ll b/test/CodeGen/SystemZ/vec-perm-04.ll
new file mode 100644
index 00000000000..0df6f4fbb01
--- /dev/null
+++ b/test/CodeGen/SystemZ/vec-perm-04.ll
@@ -0,0 +1,200 @@
+; Test vector merge high.
+;
+; RUN: llc < %s -mtriple=s390x-linux-gnu -mcpu=z13 | FileCheck %s
+
+; Test a canonical v16i8 merge high.
+define <16 x i8> @f1(<16 x i8> %val1, <16 x i8> %val2) {
+; CHECK-LABEL: f1:
+; CHECK: vmrhb %v24, %v24, %v26
+; CHECK: br %r14
+  %ret = shufflevector <16 x i8> %val1, <16 x i8> %val2,
+                       <16 x i32> <i32 0, i32 16, i32 1, i32 17,
+                                   i32 2, i32 18, i32 3, i32 19,
+                                   i32 4, i32 20, i32 5, i32 21,
+                                   i32 6, i32 22, i32 7, i32 23>
+  ret <16 x i8> %ret
+}
+
+; Test a reversed v16i8 merge high.
+define <16 x i8> @f2(<16 x i8> %val1, <16 x i8> %val2) {
+; CHECK-LABEL: f2:
+; CHECK: vmrhb %v24, %v26, %v24
+; CHECK: br %r14
+  %ret = shufflevector <16 x i8> %val1, <16 x i8> %val2,
+                       <16 x i32> <i32 16, i32 0, i32 17, i32 1,
+                                   i32 18, i32 2, i32 19, i32 3,
+                                   i32 20, i32 4, i32 21, i32 5,
+                                   i32 22, i32 6, i32 23, i32 7>
+  ret <16 x i8> %ret
+}
+
+; Test a v16i8 merge high with only the first operand being used.
+define <16 x i8> @f3(<16 x i8> %val1, <16 x i8> %val2) {
+; CHECK-LABEL: f3:
+; CHECK: vmrhb %v24, %v24, %v24
+; CHECK: br %r14
+  %ret = shufflevector <16 x i8> %val1, <16 x i8> %val2,
+                       <16 x i32> <i32 0, i32 0, i32 1, i32 1,
+                                   i32 2, i32 2, i32 3, i32 3,
+                                   i32 4, i32 4, i32 5, i32 5,
+                                   i32 6, i32 6, i32 7, i32 7>
+  ret <16 x i8> %ret
+}
+
+; Test a v16i8 merge high with only the second operand being used.
+; This is converted into @f3 by target-independent code.
+define <16 x i8> @f4(<16 x i8> %val1, <16 x i8> %val2) {
+; CHECK-LABEL: f4:
+; CHECK: vmrhb %v24, %v26, %v26
+; CHECK: br %r14
+  %ret = shufflevector <16 x i8> %val1, <16 x i8> %val2,
+                       <16 x i32> <i32 16, i32 16, i32 17, i32 17,
+                                   i32 18, i32 18, i32 19, i32 19,
+                                   i32 20, i32 20, i32 21, i32 21,
+                                   i32 22, i32 22, i32 23, i32 23>
+  ret <16 x i8> %ret
+}
+
+; Test a v16i8 merge with both operands being the same.  This too is
+; converted into @f3 by target-independent code.
+define <16 x i8> @f5(<16 x i8> %val) {
+; CHECK-LABEL: f5:
+; CHECK: vmrhb %v24, %v24, %v24
+; CHECK: br %r14
+  %ret = shufflevector <16 x i8> %val, <16 x i8> %val,
+                       <16 x i32> <i32 0, i32 16, i32 17, i32 17,
+                                   i32 18, i32 2, i32 3, i32 3,
+                                   i32 20, i32 20, i32 5, i32 5,
+                                   i32 6, i32 22, i32 23, i32 7>
+  ret <16 x i8> %ret
+}
+
+; Test a v16i8 merge in which some of the indices are don't care.
+define <16 x i8> @f6(<16 x i8> %val1, <16 x i8> %val2) {
+; CHECK-LABEL: f6:
+; CHECK: vmrhb %v24, %v24, %v26
+; CHECK: br %r14
+  %ret = shufflevector <16 x i8> %val1, <16 x i8> %val2,
+                       <16 x i32> <i32 0, i32 undef, i32 1, i32 17,
+                                   i32 undef, i32 18, i32 undef, i32 undef,
+                                   i32 undef, i32 20, i32 5, i32 21,
+                                   i32 undef, i32 22, i32 7, i32 undef>
+  ret <16 x i8> %ret
+}
+
+; Test a v16i8 merge in which one of the operands is undefined and where
+; indices for that operand are "don't care".  Target-independent code
+; converts the indices themselves into "undef"s.
+define <16 x i8> @f7(<16 x i8> %val) {
+; CHECK-LABEL: f7:
+; CHECK: vmrhb %v24, %v24, %v24
+; CHECK: br %r14
+  %ret = shufflevector <16 x i8> undef, <16 x i8> %val,
+                       <16 x i32> <i32 11, i32 16, i32 17, i32 5,
+                                   i32 18, i32 10, i32 19, i32 19,
+                                   i32 20, i32 20, i32 21, i32 3,
+                                   i32 2, i32 22, i32 9, i32 23>
+  ret <16 x i8> %ret
+}
+
+; Test a canonical v8i16 merge high.
+define <8 x i16> @f8(<8 x i16> %val1, <8 x i16> %val2) {
+; CHECK-LABEL: f8:
+; CHECK: vmrhh %v24, %v24, %v26
+; CHECK: br %r14
+  %ret = shufflevector <8 x i16> %val1, <8 x i16> %val2,
+                       <8 x i32> <i32 0, i32 8, i32 1, i32 9,
+                                  i32 2, i32 10, i32 3, i32 11>
+  ret <8 x i16> %ret
+}
+
+; Test a reversed v8i16 merge high.
+define <8 x i16> @f9(<8 x i16> %val1, <8 x i16> %val2) {
+; CHECK-LABEL: f9:
+; CHECK: vmrhh %v24, %v26, %v24
+; CHECK: br %r14
+  %ret = shufflevector <8 x i16> %val1, <8 x i16> %val2,
+                       <8 x i32> <i32 8, i32 0, i32 9, i32 1,
+                                  i32 10, i32 2, i32 11, i32 3>
+  ret <8 x i16> %ret
+}
+
+; Test a canonical v4i32 merge high.
+define <4 x i32> @f10(<4 x i32> %val1, <4 x i32> %val2) {
+; CHECK-LABEL: f10:
+; CHECK: vmrhf %v24, %v24, %v26
+; CHECK: br %r14
+  %ret = shufflevector <4 x i32> %val1, <4 x i32> %val2,
+                       <4 x i32> <i32 0, i32 4, i32 1, i32 5>
+  ret <4 x i32> %ret
+}
+
+; Test a reversed v4i32 merge high.
+define <4 x i32> @f11(<4 x i32> %val1, <4 x i32> %val2) {
+; CHECK-LABEL: f11:
+; CHECK: vmrhf %v24, %v26, %v24
+; CHECK: br %r14
+  %ret = shufflevector <4 x i32> %val1, <4 x i32> %val2,
+                       <4 x i32> <i32 4, i32 0, i32 5, i32 1>
+  ret <4 x i32> %ret
+}
+
+; Test a canonical v2i64 merge high.
+define <2 x i64> @f12(<2 x i64> %val1, <2 x i64> %val2) {
+; CHECK-LABEL: f12:
+; CHECK: vmrhg %v24, %v24, %v26
+; CHECK: br %r14
+  %ret = shufflevector <2 x i64> %val1, <2 x i64> %val2,
+                       <2 x i32> <i32 0, i32 2>
+  ret <2 x i64> %ret
+}
+
+; Test a reversed v2i64 merge high.
+define <2 x i64> @f13(<2 x i64> %val1, <2 x i64> %val2) {
+; CHECK-LABEL: f13:
+; CHECK: vmrhg %v24, %v26, %v24
+; CHECK: br %r14
+  %ret = shufflevector <2 x i64> %val1, <2 x i64> %val2,
+                       <2 x i32> <i32 2, i32 0>
+  ret <2 x i64> %ret
+}
+
+; Test a canonical v4f32 merge high.
+define <4 x float> @f14(<4 x float> %val1, <4 x float> %val2) {
+; CHECK-LABEL: f14:
+; CHECK: vmrhf %v24, %v24, %v26
+; CHECK: br %r14
+  %ret = shufflevector <4 x float> %val1, <4 x float> %val2,
+                       <4 x i32> <i32 0, i32 4, i32 1, i32 5>
+  ret <4 x float> %ret
+}
+
+; Test a reversed v4f32 merge high.
+define <4 x float> @f15(<4 x float> %val1, <4 x float> %val2) {
+; CHECK-LABEL: f15:
+; CHECK: vmrhf %v24, %v26, %v24
+; CHECK: br %r14
+  %ret = shufflevector <4 x float> %val1, <4 x float> %val2,
+                       <4 x i32> <i32 4, i32 0, i32 5, i32 1>
+  ret <4 x float> %ret
+}
+
+; Test a canonical v2f64 merge high.
+define <2 x double> @f16(<2 x double> %val1, <2 x double> %val2) {
+; CHECK-LABEL: f16:
+; CHECK: vmrhg %v24, %v24, %v26
+; CHECK: br %r14
+  %ret = shufflevector <2 x double> %val1, <2 x double> %val2,
+                       <2 x i32> <i32 0, i32 2>
+  ret <2 x double> %ret
+}
+
+; Test a reversed v2f64 merge high.
+define <2 x double> @f17(<2 x double> %val1, <2 x double> %val2) {
+; CHECK-LABEL: f17:
+; CHECK: vmrhg %v24, %v26, %v24
+; CHECK: br %r14
+  %ret = shufflevector <2 x double> %val1, <2 x double> %val2,
+                       <2 x i32> <i32 2, i32 0>
+  ret <2 x double> %ret
+}
diff --git a/test/CodeGen/SystemZ/vec-perm-05.ll b/test/CodeGen/SystemZ/vec-perm-05.ll
new file mode 100644
index 00000000000..b585cefbf84
--- /dev/null
+++ b/test/CodeGen/SystemZ/vec-perm-05.ll
@@ -0,0 +1,200 @@
+; Test vector merge low.
+;
+; RUN: llc < %s -mtriple=s390x-linux-gnu -mcpu=z13 | FileCheck %s
+
+; Test a canonical v16i8 merge low.
+define <16 x i8> @f1(<16 x i8> %val1, <16 x i8> %val2) {
+; CHECK-LABEL: f1:
+; CHECK: vmrlb %v24, %v24, %v26
+; CHECK: br %r14
+  %ret = shufflevector <16 x i8> %val1, <16 x i8> %val2,
+                       <16 x i32> <i32 8, i32 24, i32 9, i32 25,
+                                   i32 10, i32 26, i32 11, i32 27,
+                                   i32 12, i32 28, i32 13, i32 29,
+                                   i32 14, i32 30, i32 15, i32 31>
+  ret <16 x i8> %ret
+}
+
+; Test a reversed v16i8 merge low.
+define <16 x i8> @f2(<16 x i8> %val1, <16 x i8> %val2) {
+; CHECK-LABEL: f2:
+; CHECK: vmrlb %v24, %v26, %v24
+; CHECK: br %r14
+  %ret = shufflevector <16 x i8> %val1, <16 x i8> %val2,
+                       <16 x i32> <i32 24, i32 8, i32 25, i32 9,
+                                   i32 26, i32 10, i32 27, i32 11,
+                                   i32 28, i32 12, i32 29, i32 13,
+                                   i32 30, i32 14, i32 31, i32 15>
+  ret <16 x i8> %ret
+}
+
+; Test a v16i8 merge low with only the first operand being used.
+define <16 x i8> @f3(<16 x i8> %val1, <16 x i8> %val2) {
+; CHECK-LABEL: f3:
+; CHECK: vmrlb %v24, %v24, %v24
+; CHECK: br %r14
+  %ret = shufflevector <16 x i8> %val1, <16 x i8> %val2,
+                       <16 x i32> <i32 8, i32 8, i32 9, i32 9,
+                                   i32 10, i32 10, i32 11, i32 11,
+                                   i32 12, i32 12, i32 13, i32 13,
+                                   i32 14, i32 14, i32 15, i32 15>
+  ret <16 x i8> %ret
+}
+
+; Test a v16i8 merge low with only the second operand being used.
+; This is converted into @f3 by target-independent code.
+define <16 x i8> @f4(<16 x i8> %val1, <16 x i8> %val2) {
+; CHECK-LABEL: f4:
+; CHECK: vmrlb %v24, %v26, %v26
+; CHECK: br %r14
+  %ret = shufflevector <16 x i8> %val1, <16 x i8> %val2,
+                       <16 x i32> <i32 24, i32 24, i32 25, i32 25,
+                                   i32 26, i32 26, i32 27, i32 27,
+                                   i32 28, i32 28, i32 29, i32 29,
+                                   i32 30, i32 30, i32 31, i32 31>
+  ret <16 x i8> %ret
+}
+
+; Test a v16i8 merge with both operands being the same.  This too is
+; converted into @f3 by target-independent code.
+define <16 x i8> @f5(<16 x i8> %val) {
+; CHECK-LABEL: f5:
+; CHECK: vmrlb %v24, %v24, %v24
+; CHECK: br %r14
+  %ret = shufflevector <16 x i8> %val, <16 x i8> %val,
+                       <16 x i32> <i32 8, i32 24, i32 25, i32 25,
+                                   i32 26, i32 10, i32 11, i32 11,
+                                   i32 28, i32 28, i32 13, i32 13,
+                                   i32 14, i32 30, i32 31, i32 15>
+  ret <16 x i8> %ret
+}
+
+; Test a v16i8 merge in which some of the indices are don't care.
+define <16 x i8> @f6(<16 x i8> %val1, <16 x i8> %val2) {
+; CHECK-LABEL: f6:
+; CHECK: vmrlb %v24, %v24, %v26
+; CHECK: br %r14
+  %ret = shufflevector <16 x i8> %val1, <16 x i8> %val2,
+                       <16 x i32> <i32 8, i32 undef, i32 9, i32 25,
+                                   i32 undef, i32 26, i32 undef, i32 undef,
+                                   i32 undef, i32 28, i32 13, i32 29,
+                                   i32 undef, i32 30, i32 15, i32 undef>
+  ret <16 x i8> %ret
+}
+
+; Test a v16i8 merge in which one of the operands is undefined and where
+; indices for that operand are "don't care".  Target-independent code
+; converts the indices themselves into "undef"s.
+define <16 x i8> @f7(<16 x i8> %val) {
+; CHECK-LABEL: f7:
+; CHECK: vmrlb %v24, %v24, %v24
+; CHECK: br %r14
+  %ret = shufflevector <16 x i8> undef, <16 x i8> %val,
+                       <16 x i32> <i32 11, i32 24, i32 25, i32 5,
+                                   i32 26, i32 10, i32 27, i32 27,
+                                   i32 28, i32 28, i32 29, i32 3,
+                                   i32 2, i32 30, i32 9, i32 31>
+  ret <16 x i8> %ret
+}
+
+; Test a canonical v8i16 merge low.
+define <8 x i16> @f8(<8 x i16> %val1, <8 x i16> %val2) {
+; CHECK-LABEL: f8:
+; CHECK: vmrlh %v24, %v24, %v26
+; CHECK: br %r14
+  %ret = shufflevector <8 x i16> %val1, <8 x i16> %val2,
+                       <8 x i32> <i32 4, i32 12, i32 5, i32 13,
+                                  i32 6, i32 14, i32 7, i32 15>
+  ret <8 x i16> %ret
+}
+
+; Test a reversed v8i16 merge low.
+define <8 x i16> @f9(<8 x i16> %val1, <8 x i16> %val2) {
+; CHECK-LABEL: f9:
+; CHECK: vmrlh %v24, %v26, %v24
+; CHECK: br %r14
+  %ret = shufflevector <8 x i16> %val1, <8 x i16> %val2,
+                       <8 x i32> <i32 12, i32 4, i32 13, i32 5,
+                                  i32 14, i32 6, i32 15, i32 7>
+  ret <8 x i16> %ret
+}
+
+; Test a canonical v4i32 merge low.
+define <4 x i32> @f10(<4 x i32> %val1, <4 x i32> %val2) {
+; CHECK-LABEL: f10:
+; CHECK: vmrlf %v24, %v24, %v26
+; CHECK: br %r14
+  %ret = shufflevector <4 x i32> %val1, <4 x i32> %val2,
+                       <4 x i32> <i32 2, i32 6, i32 3, i32 7>
+  ret <4 x i32> %ret
+}
+
+; Test a reversed v4i32 merge low.
+define <4 x i32> @f11(<4 x i32> %val1, <4 x i32> %val2) {
+; CHECK-LABEL: f11:
+; CHECK: vmrlf %v24, %v26, %v24
+; CHECK: br %r14
+  %ret = shufflevector <4 x i32> %val1, <4 x i32> %val2,
+                       <4 x i32> <i32 6, i32 2, i32 7, i32 3>
+  ret <4 x i32> %ret
+}
+
+; Test a canonical v2i64 merge low.
+define <2 x i64> @f12(<2 x i64> %val1, <2 x i64> %val2) {
+; CHECK-LABEL: f12:
+; CHECK: vmrlg %v24, %v24, %v26
+; CHECK: br %r14
+  %ret = shufflevector <2 x i64> %val1, <2 x i64> %val2,
+                       <2 x i32> <i32 1, i32 3>
+  ret <2 x i64> %ret
+}
+
+; Test a reversed v2i64 merge low.
+define <2 x i64> @f13(<2 x i64> %val1, <2 x i64> %val2) {
+; CHECK-LABEL: f13:
+; CHECK: vmrlg %v24, %v26, %v24
+; CHECK: br %r14
+  %ret = shufflevector <2 x i64> %val1, <2 x i64> %val2,
+                       <2 x i32> <i32 3, i32 1>
+  ret <2 x i64> %ret
+}
+
+; Test a canonical v4f32 merge low.
+define <4 x float> @f14(<4 x float> %val1, <4 x float> %val2) {
+; CHECK-LABEL: f14:
+; CHECK: vmrlf %v24, %v24, %v26
+; CHECK: br %r14
+  %ret = shufflevector <4 x float> %val1, <4 x float> %val2,
+                       <4 x i32> <i32 2, i32 6, i32 3, i32 7>
+  ret <4 x float> %ret
+}
+
+; Test a reversed v4f32 merge low.
+define <4 x float> @f15(<4 x float> %val1, <4 x float> %val2) {
+; CHECK-LABEL: f15:
+; CHECK: vmrlf %v24, %v26, %v24
+; CHECK: br %r14
+  %ret = shufflevector <4 x float> %val1, <4 x float> %val2,
+                       <4 x i32> <i32 6, i32 2, i32 7, i32 3>
+  ret <4 x float> %ret
+}
+
+; Test a canonical v2f64 merge low.
+define <2 x double> @f16(<2 x double> %val1, <2 x double> %val2) {
+; CHECK-LABEL: f16:
+; CHECK: vmrlg %v24, %v24, %v26
+; CHECK: br %r14
+  %ret = shufflevector <2 x double> %val1, <2 x double> %val2,
+                       <2 x i32> <i32 1, i32 3>
+  ret <2 x double> %ret
+}
+
+; Test a reversed v2f64 merge low.
+define <2 x double> @f17(<2 x double> %val1, <2 x double> %val2) {
+; CHECK-LABEL: f17:
+; CHECK: vmrlg %v24, %v26, %v24
+; CHECK: br %r14
+  %ret = shufflevector <2 x double> %val1, <2 x double> %val2,
+                       <2 x i32> <i32 3, i32 1>
+  ret <2 x double> %ret
+}
diff --git a/test/CodeGen/SystemZ/vec-perm-06.ll b/test/CodeGen/SystemZ/vec-perm-06.ll
new file mode 100644
index 00000000000..835276a3672
--- /dev/null
+++ b/test/CodeGen/SystemZ/vec-perm-06.ll
@@ -0,0 +1,160 @@
+; Test vector pack.
+;
+; RUN: llc < %s -mtriple=s390x-linux-gnu -mcpu=z13 | FileCheck %s
+
+; Test a canonical v16i8 pack.
+define <16 x i8> @f1(<16 x i8> %val1, <16 x i8> %val2) {
+; CHECK-LABEL: f1:
+; CHECK: vpkh %v24, %v24, %v26
+; CHECK: br %r14
+  %ret = shufflevector <16 x i8> %val1, <16 x i8> %val2,
+                       <16 x i32> <i32 1, i32 3, i32 5, i32 7,
+                                   i32 9, i32 11, i32 13, i32 15,
+                                   i32 17, i32 19, i32 21, i32 23,
+                                   i32 25, i32 27, i32 29, i32 31>
+  ret <16 x i8> %ret
+}
+
+; Test a reversed v16i8 pack.
+define <16 x i8> @f2(<16 x i8> %val1, <16 x i8> %val2) {
+; CHECK-LABEL: f2:
+; CHECK: vpkh %v24, %v26, %v24
+; CHECK: br %r14
+  %ret = shufflevector <16 x i8> %val1, <16 x i8> %val2,
+                       <16 x i32> <i32 17, i32 19, i32 21, i32 23,
+                                   i32 25, i32 27, i32 29, i32 31,
+                                   i32 1, i32 3, i32 5, i32 7,
+                                   i32 9, i32 11, i32 13, i32 15>
+  ret <16 x i8> %ret
+}
+
+; Test a v16i8 pack with only the first operand being used.
+define <16 x i8> @f3(<16 x i8> %val1, <16 x i8> %val2) {
+; CHECK-LABEL: f3:
+; CHECK: vpkh %v24, %v24, %v24
+; CHECK: br %r14
+  %ret = shufflevector <16 x i8> %val1, <16 x i8> %val2,
+                       <16 x i32> <i32 1, i32 3, i32 5, i32 7,
+                                   i32 9, i32 11, i32 13, i32 15,
+                                   i32 1, i32 3, i32 5, i32 7,
+                                   i32 9, i32 11, i32 13, i32 15>
+  ret <16 x i8> %ret
+}
+
+; Test a v16i8 pack with only the second operand being used.
+; This is converted into @f3 by target-independent code.
+define <16 x i8> @f4(<16 x i8> %val1, <16 x i8> %val2) {
+; CHECK-LABEL: f4:
+; CHECK: vpkh %v24, %v26, %v26
+; CHECK: br %r14
+  %ret = shufflevector <16 x i8> %val1, <16 x i8> %val2,
+                       <16 x i32> <i32 17, i32 19, i32 21, i32 23,
+                                   i32 25, i32 27, i32 29, i32 31,
+                                   i32 17, i32 19, i32 21, i32 23,
+                                   i32 25, i32 27, i32 29, i32 31>
+  ret <16 x i8> %ret
+}
+
+; Test a v16i8 pack with both operands being the same.  This too is
+; converted into @f3 by target-independent code.
+define <16 x i8> @f5(<16 x i8> %val) {
+; CHECK-LABEL: f5:
+; CHECK: vpkh %v24, %v24, %v24
+; CHECK: br %r14
+  %ret = shufflevector <16 x i8> %val, <16 x i8> %val,
+                       <16 x i32> <i32 1, i32 3, i32 5, i32 7,
+                                   i32 9, i32 11, i32 13, i32 15,
+                                   i32 17, i32 19, i32 21, i32 23,
+                                   i32 25, i32 27, i32 29, i32 31>
+  ret <16 x i8> %ret
+}
+
+; Test a v16i8 pack in which some of the indices are don't care.
+define <16 x i8> @f6(<16 x i8> %val1, <16 x i8> %val2) {
+; CHECK-LABEL: f6:
+; CHECK: vpkh %v24, %v24, %v26
+; CHECK: br %r14
+  %ret = shufflevector <16 x i8> %val1, <16 x i8> %val2,
+                       <16 x i32> <i32 1, i32 undef, i32 5, i32 7,
+                                   i32 undef, i32 11, i32 undef, i32 undef,
+                                   i32 undef, i32 19, i32 21, i32 23,
+                                   i32 undef, i32 27, i32 29, i32 undef>
+  ret <16 x i8> %ret
+}
+
+; Test a v16i8 pack in which one of the operands is undefined and where
+; indices for that operand are "don't care".  Target-independent code
+; converts the indices themselves into "undef"s.
+define <16 x i8> @f7(<16 x i8> %val) {
+; CHECK-LABEL: f7:
+; CHECK: vpkh %v24, %v24, %v24
+; CHECK: br %r14
+  %ret = shufflevector <16 x i8> undef, <16 x i8> %val,
+                       <16 x i32> <i32 7, i32 1, i32 9, i32 15,
+                                   i32 15, i32 3, i32 5, i32 1,
+                                   i32 17, i32 19, i32 21, i32 23,
+                                   i32 25, i32 27, i32 29, i32 31>
+  ret <16 x i8> %ret
+}
+
+; Test a canonical v8i16 pack.
+define <8 x i16> @f8(<8 x i16> %val1, <8 x i16> %val2) {
+; CHECK-LABEL: f8:
+; CHECK: vpkf %v24, %v24, %v26
+; CHECK: br %r14
+  %ret = shufflevector <8 x i16> %val1, <8 x i16> %val2,
+                       <8 x i32> <i32 1, i32 3, i32 5, i32 7,
+                                  i32 9, i32 11, i32 13, i32 15>
+  ret <8 x i16> %ret
+}
+
+; Test a reversed v8i16 pack.
+define <8 x i16> @f9(<8 x i16> %val1, <8 x i16> %val2) {
+; CHECK-LABEL: f9:
+; CHECK: vpkf %v24, %v26, %v24
+; CHECK: br %r14
+  %ret = shufflevector <8 x i16> %val1, <8 x i16> %val2,
+                       <8 x i32> <i32 9, i32 11, i32 13, i32 15,
+                                  i32 1, i32 3, i32 5, i32 7>
+  ret <8 x i16> %ret
+}
+
+; Test a canonical v4i32 pack.
+define <4 x i32> @f10(<4 x i32> %val1, <4 x i32> %val2) {
+; CHECK-LABEL: f10:
+; CHECK: vpkg %v24, %v24, %v26
+; CHECK: br %r14
+  %ret = shufflevector <4 x i32> %val1, <4 x i32> %val2,
+                       <4 x i32> <i32 1, i32 3, i32 5, i32 7>
+  ret <4 x i32> %ret
+}
+
+; Test a reversed v4i32 pack.
+define <4 x i32> @f11(<4 x i32> %val1, <4 x i32> %val2) {
+; CHECK-LABEL: f11:
+; CHECK: vpkg %v24, %v26, %v24
+; CHECK: br %r14
+  %ret = shufflevector <4 x i32> %val1, <4 x i32> %val2,
+                       <4 x i32> <i32 5, i32 7, i32 1, i32 3>
+  ret <4 x i32> %ret
+}
+
+; Test a canonical v4f32 pack.
+define <4 x float> @f12(<4 x float> %val1, <4 x float> %val2) {
+; CHECK-LABEL: f12:
+; CHECK: vpkg %v24, %v24, %v26
+; CHECK: br %r14
+  %ret = shufflevector <4 x float> %val1, <4 x float> %val2,
+                       <4 x i32> <i32 1, i32 3, i32 5, i32 7>
+  ret <4 x float> %ret
+}
+
+; Test a reversed v4f32 pack.
+define <4 x float> @f13(<4 x float> %val1, <4 x float> %val2) {
+; CHECK-LABEL: f13:
+; CHECK: vpkg %v24, %v26, %v24
+; CHECK: br %r14
+  %ret = shufflevector <4 x float> %val1, <4 x float> %val2,
+                       <4 x i32> <i32 5, i32 7, i32 1, i32 3>
+  ret <4 x float> %ret
+}
diff --git a/test/CodeGen/SystemZ/vec-perm-07.ll b/test/CodeGen/SystemZ/vec-perm-07.ll
new file mode 100644
index 00000000000..9a370af2c0e
--- /dev/null
+++ b/test/CodeGen/SystemZ/vec-perm-07.ll
@@ -0,0 +1,145 @@
+; Test vector shift left double immediate.
+;
+; RUN: llc < %s -mtriple=s390x-linux-gnu -mcpu=z13 | FileCheck %s
+
+; Test a v16i8 shift with the lowest useful shift amount.
+define <16 x i8> @f1(<16 x i8> %val1, <16 x i8> %val2) {
+; CHECK-LABEL: f1:
+; CHECK: vsldb %v24, %v24, %v26, 1
+; CHECK: br %r14
+  %ret = shufflevector <16 x i8> %val1, <16 x i8> %val2,
+                       <16 x i32> <i32 1, i32 2, i32 3, i32 4,
+                                   i32 5, i32 6, i32 7, i32 8,
+                                   i32 9, i32 10, i32 11, i32 12,
+                                   i32 13, i32 14, i32 15, i32 16>
+  ret <16 x i8> %ret
+}
+
+; Test a v16i8 shift with the highest shift amount.
+define <16 x i8> @f2(<16 x i8> %val1, <16 x i8> %val2) {
+; CHECK-LABEL: f2:
+; CHECK: vsldb %v24, %v24, %v26, 15
+; CHECK: br %r14
+  %ret = shufflevector <16 x i8> %val1, <16 x i8> %val2,
+                       <16 x i32> <i32 15, i32 16, i32 17, i32 18,
+                                   i32 19, i32 20, i32 21, i32 22,
+                                   i32 23, i32 24, i32 25, i32 26,
+                                   i32 27, i32 28, i32 29, i32 30>
+  ret <16 x i8> %ret
+}
+
+; Test a v16i8 shift in which the operands need to be reversed.
+define <16 x i8> @f3(<16 x i8> %val1, <16 x i8> %val2) {
+; CHECK-LABEL: f3:
+; CHECK: vsldb %v24, %v26, %v24, 4
+; CHECK: br %r14
+  %ret = shufflevector <16 x i8> %val1, <16 x i8> %val2,
+                       <16 x i32> <i32 20, i32 21, i32 22, i32 23,
+                                   i32 24, i32 25, i32 26, i32 27,
+                                   i32 28, i32 29, i32 30, i32 31,
+                                   i32 0, i32 1, i32 2, i32 3>
+  ret <16 x i8> %ret
+}
+
+; Test a v16i8 shift in which the operands need to be duplicated.
+define <16 x i8> @f4(<16 x i8> %val) {
+; CHECK-LABEL: f4:
+; CHECK: vsldb %v24, %v24, %v24, 7
+; CHECK: br %r14
+  %ret = shufflevector <16 x i8> %val, <16 x i8> undef,
+                       <16 x i32> <i32 7, i32 8, i32 9, i32 10,
+                                   i32 11, i32 12, i32 13, i32 14,
+                                   i32 15, i32 0, i32 1, i32 2,
+                                   i32 3, i32 4, i32 5, i32 6>
+  ret <16 x i8> %ret
+}
+
+; Test a v16i8 shift in which some of the indices are undefs.
+define <16 x i8> @f5(<16 x i8> %val1, <16 x i8> %val2) {
+; CHECK-LABEL: f5:
+; CHECK: vsldb %v24, %v24, %v26, 11
+; CHECK: br %r14
+  %ret = shufflevector <16 x i8> %val1, <16 x i8> %val2,
+                       <16 x i32> <i32 undef, i32 undef, i32 undef, i32 undef,
+                                   i32 15, i32 16, i32 undef, i32 18,
+                                   i32 19, i32 20, i32 21, i32 22,
+                                   i32 23, i32 24, i32 25, i32 26>
+  ret <16 x i8> %ret
+}
+
+; ...and again with reversed operands.
+define <16 x i8> @f6(<16 x i8> %val1, <16 x i8> %val2) {
+; CHECK-LABEL: f6:
+; CHECK: vsldb %v24, %v26, %v24, 13
+; CHECK: br %r14
+  %ret = shufflevector <16 x i8> %val1, <16 x i8> %val2,
+                       <16 x i32> <i32 undef, i32 undef, i32 31, i32 0,
+                                   i32 1, i32 2, i32 3, i32 4,
+                                   i32 5, i32 6, i32 7, i32 8,
+                                   i32 9, i32 10, i32 11, i32 12>
+  ret <16 x i8> %ret
+}
+
+; Test a v8i16 shift with the lowest useful shift amount.
+define <8 x i16> @f7(<8 x i16> %val1, <8 x i16> %val2) {
+; CHECK-LABEL: f7:
+; CHECK: vsldb %v24, %v24, %v26, 2
+; CHECK: br %r14
+  %ret = shufflevector <8 x i16> %val1, <8 x i16> %val2,
+                       <8 x i32> <i32 1, i32 2, i32 3, i32 4,
+                                  i32 5, i32 6, i32 7, i32 8>
+  ret <8 x i16> %ret
+}
+
+; Test a v8i16 shift with the highest useful shift amount.
+define <8 x i16> @f8(<8 x i16> %val1, <8 x i16> %val2) {
+; CHECK-LABEL: f8:
+; CHECK: vsldb %v24, %v24, %v26, 14
+; CHECK: br %r14
+  %ret = shufflevector <8 x i16> %val1, <8 x i16> %val2,
+                       <8 x i32> <i32 7, i32 8, i32 9, i32 10,
+                                  i32 11, i32 12, i32 13, i32 14>
+  ret <8 x i16> %ret
+}
+
+; Test a v4i32 shift with the lowest useful shift amount.
+define <4 x i32> @f9(<4 x i32> %val1, <4 x i32> %val2) {
+; CHECK-LABEL: f9:
+; CHECK: vsldb %v24, %v24, %v26, 4
+; CHECK: br %r14
+  %ret = shufflevector <4 x i32> %val1, <4 x i32> %val2,
+                       <4 x i32> <i32 1, i32 2, i32 3, i32 4>
+  ret <4 x i32> %ret
+}
+
+; Test a v4i32 shift with the highest useful shift amount.
+define <4 x i32> @f10(<4 x i32> %val1, <4 x i32> %val2) {
+; CHECK-LABEL: f10:
+; CHECK: vsldb %v24, %v24, %v26, 12
+; CHECK: br %r14
+  %ret = shufflevector <4 x i32> %val1, <4 x i32> %val2,
+                       <4 x i32> <i32 3, i32 4, i32 5, i32 6>
+  ret <4 x i32> %ret
+}
+
+; Test a v4f32 shift with the lowest useful shift amount.
+define <4 x float> @f12(<4 x float> %val1, <4 x float> %val2) {
+; CHECK-LABEL: f12:
+; CHECK: vsldb %v24, %v24, %v26, 4
+; CHECK: br %r14
+  %ret = shufflevector <4 x float> %val1, <4 x float> %val2,
+                       <4 x i32> <i32 1, i32 2, i32 3, i32 4>
+  ret <4 x float> %ret
+}
+
+; Test a v4f32 shift with the highest useful shift amount.
+define <4 x float> @f13(<4 x float> %val1, <4 x float> %val2) {
+; CHECK-LABEL: f13:
+; CHECK: vsldb %v24, %v24, %v26, 12
+; CHECK: br %r14
+  %ret = shufflevector <4 x float> %val1, <4 x float> %val2,
+                       <4 x i32> <i32 3, i32 4, i32 5, i32 6>
+  ret <4 x float> %ret
+}
+
+; We use VPDI for v2i64 shuffles.
diff --git a/test/CodeGen/SystemZ/vec-perm-08.ll b/test/CodeGen/SystemZ/vec-perm-08.ll
new file mode 100644
index 00000000000..a18ca7b7397
--- /dev/null
+++ b/test/CodeGen/SystemZ/vec-perm-08.ll
@@ -0,0 +1,170 @@
+; Test vector permutes using VPDI.
+;
+; RUN: llc < %s -mtriple=s390x-linux-gnu -mcpu=z13 | FileCheck %s
+
+; Test a high1/low2 permute for v16i8.
+define <16 x i8> @f1(<16 x i8> %val1, <16 x i8> %val2) {
+; CHECK-LABEL: f1:
+; CHECK: vpdi %v24, %v24, %v26, 1
+; CHECK: br %r14
+  %ret = shufflevector <16 x i8> %val1, <16 x i8> %val2,
+                       <16 x i32> <i32 0, i32 1, i32 2, i32 3,
+                                   i32 4, i32 5, i32 6, i32 7,
+                                   i32 24, i32 25, i32 26, i32 27,
+                                   i32 28, i32 29, i32 30, i32 31>
+  ret <16 x i8> %ret
+}
+
+; Test a low2/high1 permute for v16i8.
+define <16 x i8> @f2(<16 x i8> %val1, <16 x i8> %val2) {
+; CHECK-LABEL: f2:
+; CHECK: vpdi %v24, %v26, %v24, 4
+; CHECK: br %r14
+  %ret = shufflevector <16 x i8> %val1, <16 x i8> %val2,
+                       <16 x i32> <i32 24, i32 25, i32 26, i32 27,
+                                   i32 28, i32 29, i32 30, i32 31,
+                                   i32 0, i32 1, i32 2, i32 3,
+                                   i32 4, i32 5, i32 6, i32 7>
+  ret <16 x i8> %ret
+}
+
+; Test a low1/high2 permute for v16i8.
+define <16 x i8> @f3(<16 x i8> %val1, <16 x i8> %val2) {
+; CHECK-LABEL: f3:
+; CHECK: vpdi %v24, %v24, %v26, 4
+; CHECK: br %r14
+  %ret = shufflevector <16 x i8> %val1, <16 x i8> %val2,
+                       <16 x i32> <i32 8, i32 9, i32 10, i32 undef,
+                                   i32 12, i32 undef, i32 14, i32 15,
+                                   i32 16, i32 17, i32 undef, i32 19,
+                                   i32 20, i32 21, i32 22, i32 undef>
+  ret <16 x i8> %ret
+}
+
+; Test a high2/low1 permute for v16i8.
+define <16 x i8> @f4(<16 x i8> %val1, <16 x i8> %val2) {
+; CHECK-LABEL: f4:
+; CHECK: vpdi %v24, %v26, %v24, 1
+; CHECK: br %r14
+  %ret = shufflevector <16 x i8> %val1, <16 x i8> %val2,
+                       <16 x i32> <i32 16, i32 17, i32 18, i32 19,
+                                   i32 20, i32 21, i32 22, i32 23,
+                                   i32 8, i32 9, i32 10, i32 11,
+                                   i32 12, i32 13, i32 14, i32 15>
+  ret <16 x i8> %ret
+}
+
+; Test reversing two doublewords in a v16i8.
+define <16 x i8> @f5(<16 x i8> %val) {
+; CHECK-LABEL: f5:
+; CHECK: vpdi %v24, %v24, %v24, 4
+; CHECK: br %r14
+  %ret = shufflevector <16 x i8> %val, <16 x i8> undef,
+                       <16 x i32> <i32 8, i32 9, i32 10, i32 11,
+                                   i32 12, i32 13, i32 14, i32 15,
+                                   i32 0, i32 1, i32 2, i32 3,
+                                   i32 4, i32 5, i32 6, i32 7>
+  ret <16 x i8> %ret
+}
+
+; Test a high1/low2 permute for v8i16.
+define <8 x i16> @f6(<8 x i16> %val1, <8 x i16> %val2) {
+; CHECK-LABEL: f6:
+; CHECK: vpdi %v24, %v24, %v26, 1
+; CHECK: br %r14
+  %ret = shufflevector <8 x i16> %val1, <8 x i16> %val2,
+                       <8 x i32> <i32 0, i32 1, i32 2, i32 3,
+                                  i32 12, i32 13, i32 14, i32 15>
+  ret <8 x i16> %ret
+}
+
+; Test a low2/high1 permute for v8i16.
+define <8 x i16> @f7(<8 x i16> %val1, <8 x i16> %val2) {
+; CHECK-LABEL: f7:
+; CHECK: vpdi %v24, %v26, %v24, 4
+; CHECK: br %r14
+  %ret = shufflevector <8 x i16> %val1, <8 x i16> %val2,
+                       <8 x i32> <i32 12, i32 13, i32 14, i32 15,
+                                  i32 0, i32 1, i32 2, i32 3>
+  ret <8 x i16> %ret
+}
+
+; Test a high1/low2 permute for v4i32.
+define <4 x i32> @f8(<4 x i32> %val1, <4 x i32> %val2) {
+; CHECK-LABEL: f8:
+; CHECK: vpdi %v24, %v24, %v26, 1
+; CHECK: br %r14
+  %ret = shufflevector <4 x i32> %val1, <4 x i32> %val2,
+                       <4 x i32> <i32 0, i32 1, i32 6, i32 7>
+  ret <4 x i32> %ret
+}
+
+; Test a low2/high1 permute for v4i32.
+define <4 x i32> @f9(<4 x i32> %val1, <4 x i32> %val2) {
+; CHECK-LABEL: f9:
+; CHECK: vpdi %v24, %v26, %v24, 4
+; CHECK: br %r14
+  %ret = shufflevector <4 x i32> %val1, <4 x i32> %val2,
+                       <4 x i32> <i32 6, i32 7, i32 0, i32 1>
+  ret <4 x i32> %ret
+}
+
+; Test a high1/low2 permute for v2i64.
+define <2 x i64> @f10(<2 x i64> %val1, <2 x i64> %val2) {
+; CHECK-LABEL: f10:
+; CHECK: vpdi %v24, %v24, %v26, 1
+; CHECK: br %r14
+  %ret = shufflevector <2 x i64> %val1, <2 x i64> %val2,
+                       <2 x i32> <i32 0, i32 3>
+  ret <2 x i64> %ret
+}
+
+; Test low2/high1 permute for v2i64.
+define <2 x i64> @f11(<2 x i64> %val1, <2 x i64> %val2) {
+; CHECK-LABEL: f11:
+; CHECK: vpdi %v24, %v26, %v24, 4
+; CHECK: br %r14
+  %ret = shufflevector <2 x i64> %val1, <2 x i64> %val2,
+                       <2 x i32> <i32 3, i32 0>
+  ret <2 x i64> %ret
+}
+
+; Test a high1/low2 permute for v4f32.
+define <4 x float> @f12(<4 x float> %val1, <4 x float> %val2) {
+; CHECK-LABEL: f12:
+; CHECK: vpdi %v24, %v24, %v26, 1
+; CHECK: br %r14
+  %ret = shufflevector <4 x float> %val1, <4 x float> %val2,
+                       <4 x i32> <i32 0, i32 1, i32 6, i32 7>
+  ret <4 x float> %ret
+}
+
+; Test a low2/high1 permute for v4f32.
+define <4 x float> @f13(<4 x float> %val1, <4 x float> %val2) {
+; CHECK-LABEL: f13:
+; CHECK: vpdi %v24, %v26, %v24, 4
+; CHECK: br %r14
+  %ret = shufflevector <4 x float> %val1, <4 x float> %val2,
+                       <4 x i32> <i32 6, i32 7, i32 0, i32 1>
+  ret <4 x float> %ret
+}
+
+; Test a high1/low2 permute for v2f64.
+define <2 x double> @f14(<2 x double> %val1, <2 x double> %val2) {
+; CHECK-LABEL: f14:
+; CHECK: vpdi %v24, %v24, %v26, 1
+; CHECK: br %r14
+  %ret = shufflevector <2 x double> %val1, <2 x double> %val2,
+                       <2 x i32> <i32 0, i32 3>
+  ret <2 x double> %ret
+}
+
+; Test a low2/high1 permute for v2f64.
+define <2 x double> @f15(<2 x double> %val1, <2 x double> %val2) {
+; CHECK-LABEL: f15:
+; CHECK: vpdi %v24, %v26, %v24, 4
+; CHECK: br %r14
+  %ret = shufflevector <2 x double> %val1, <2 x double> %val2,
+                       <2 x i32> <i32 3, i32 0>
+  ret <2 x double> %ret
+}
diff --git a/test/CodeGen/SystemZ/vec-perm-09.ll b/test/CodeGen/SystemZ/vec-perm-09.ll
new file mode 100644
index 00000000000..9c9632cf030
--- /dev/null
+++ b/test/CodeGen/SystemZ/vec-perm-09.ll
@@ -0,0 +1,38 @@
+; Test general vector permute of a v16i8.
+;
+; RUN: llc < %s -mtriple=s390x-linux-gnu -mcpu=z13 | \
+; RUN:   FileCheck -check-prefix=CHECK-CODE %s
+; RUN: llc < %s -mtriple=s390x-linux-gnu -mcpu=z13 | \
+; RUN:   FileCheck -check-prefix=CHECK-VECTOR %s
+
+define <16 x i8> @f1(<16 x i8> %val1, <16 x i8> %val2) {
+; CHECK-CODE-LABEL: f1:
+; CHECK-CODE: larl [[REG:%r[0-5]]],
+; CHECK-CODE: vl [[MASK:%v[0-9]+]], 0([[REG]])
+; CHECK-CODE: vperm %v24, %v24, %v26, [[MASK]]
+; CHECK-CODE: br %r14
+;
+; CHECK-VECTOR: .byte 1
+; CHECK-VECTOR-NEXT: .byte 19
+; CHECK-VECTOR-NEXT: .byte 6
+; CHECK-VECTOR-NEXT: .byte 5
+; CHECK-VECTOR-NEXT: .byte 20
+; CHECK-VECTOR-NEXT: .byte 22
+; CHECK-VECTOR-NEXT: .byte 1
+; CHECK-VECTOR-NEXT: .byte 1
+; CHECK-VECTOR-NEXT: .byte 25
+; CHECK-VECTOR-NEXT: .byte 29
+; CHECK-VECTOR-NEXT: .byte 11
+; Any byte would be OK here
+; CHECK-VECTOR-NEXT: .space 1
+; CHECK-VECTOR-NEXT: .byte 31
+; CHECK-VECTOR-NEXT: .byte 4
+; CHECK-VECTOR-NEXT: .byte 15
+; CHECK-VECTOR-NEXT: .byte 19
+  %ret = shufflevector <16 x i8> %val1, <16 x i8> %val2,
+                       <16 x i32> <i32 1, i32 19, i32 6, i32 5,
+                                   i32 20, i32 22, i32 1, i32 1,
+                                   i32 25, i32 29, i32 11, i32 undef,
+                                   i32 31, i32 4, i32 15, i32 19>
+  ret <16 x i8> %ret
+}
diff --git a/test/CodeGen/SystemZ/vec-perm-10.ll b/test/CodeGen/SystemZ/vec-perm-10.ll
new file mode 100644
index 00000000000..382e6dc4c3f
--- /dev/null
+++ b/test/CodeGen/SystemZ/vec-perm-10.ll
@@ -0,0 +1,36 @@
+; Test general vector permute of a v8i16.
+;
+; RUN: llc < %s -mtriple=s390x-linux-gnu -mcpu=z13 | \
+; RUN:   FileCheck -check-prefix=CHECK-CODE %s
+; RUN: llc < %s -mtriple=s390x-linux-gnu -mcpu=z13 | \
+; RUN:   FileCheck -check-prefix=CHECK-VECTOR %s
+
+define <8 x i16> @f1(<8 x i16> %val1, <8 x i16> %val2) {
+; CHECK-CODE-LABEL: f1:
+; CHECK-CODE: larl [[REG:%r[0-5]]],
+; CHECK-CODE: vl [[MASK:%v[0-9]+]], 0([[REG]])
+; CHECK-CODE: vperm %v24, %v26, %v24, [[MASK]]
+; CHECK-CODE: br %r14
+;
+; CHECK-VECTOR: .byte 0
+; CHECK-VECTOR-NEXT: .byte 1
+; CHECK-VECTOR-NEXT: .byte 26
+; CHECK-VECTOR-NEXT: .byte 27
+; Any 2 bytes would be OK here
+; CHECK-VECTOR-NEXT: .space 1
+; CHECK-VECTOR-NEXT: .space 1
+; CHECK-VECTOR-NEXT: .byte 28
+; CHECK-VECTOR-NEXT: .byte 29
+; CHECK-VECTOR-NEXT: .byte 6
+; CHECK-VECTOR-NEXT: .byte 7
+; CHECK-VECTOR-NEXT: .byte 14
+; CHECK-VECTOR-NEXT: .byte 15
+; CHECK-VECTOR-NEXT: .byte 8
+; CHECK-VECTOR-NEXT: .byte 9
+; CHECK-VECTOR-NEXT: .byte 16
+; CHECK-VECTOR-NEXT: .byte 17
+  %ret = shufflevector <8 x i16> %val1, <8 x i16> %val2,
+                       <8 x i32> <i32 8, i32 5, i32 undef, i32 6,
+                                  i32 11, i32 15, i32 12, i32 0>
+  ret <8 x i16> %ret
+}
diff --git a/test/CodeGen/SystemZ/vec-perm-11.ll b/test/CodeGen/SystemZ/vec-perm-11.ll
new file mode 100644
index 00000000000..c9e29880fe0
--- /dev/null
+++ b/test/CodeGen/SystemZ/vec-perm-11.ll
@@ -0,0 +1,35 @@
+; Test general vector permute of a v4i32.
+;
+; RUN: llc < %s -mtriple=s390x-linux-gnu -mcpu=z13 | \
+; RUN:   FileCheck -check-prefix=CHECK-CODE %s
+; RUN: llc < %s -mtriple=s390x-linux-gnu -mcpu=z13 | \
+; RUN:   FileCheck -check-prefix=CHECK-VECTOR %s
+
+define <4 x i32> @f1(<4 x i32> %val1, <4 x i32> %val2) {
+; CHECK-CODE-LABEL: f1:
+; CHECK-CODE: larl [[REG:%r[0-5]]],
+; CHECK-CODE: vl [[MASK:%v[0-9]+]], 0([[REG]])
+; CHECK-CODE: vperm %v24, %v26, %v24, [[MASK]]
+; CHECK-CODE: br %r14
+;
+; CHECK-VECTOR: .byte 4
+; CHECK-VECTOR-NEXT: .byte 5
+; CHECK-VECTOR-NEXT: .byte 6
+; CHECK-VECTOR-NEXT: .byte 7
+; CHECK-VECTOR-NEXT: .byte 20
+; CHECK-VECTOR-NEXT: .byte 21
+; CHECK-VECTOR-NEXT: .byte 22
+; CHECK-VECTOR-NEXT: .byte 23
+; Any 4 bytes would be OK here
+; CHECK-VECTOR-NEXT: .space 1
+; CHECK-VECTOR-NEXT: .space 1
+; CHECK-VECTOR-NEXT: .space 1
+; CHECK-VECTOR-NEXT: .space 1
+; CHECK-VECTOR-NEXT: .byte 12
+; CHECK-VECTOR-NEXT: .byte 13
+; CHECK-VECTOR-NEXT: .byte 14
+; CHECK-VECTOR-NEXT: .byte 15
+  %ret = shufflevector <4 x i32> %val1, <4 x i32> %val2,
+                       <4 x i32> <i32 5, i32 1, i32 undef, i32 7>
+  ret <4 x i32> %ret
+}
diff --git a/test/CodeGen/SystemZ/vec-round-01.ll b/test/CodeGen/SystemZ/vec-round-01.ll
new file mode 100644
index 00000000000..82718276bb0
--- /dev/null
+++ b/test/CodeGen/SystemZ/vec-round-01.ll
@@ -0,0 +1,118 @@
+; Test v2f64 rounding.
+;
+; RUN: llc < %s -mtriple=s390x-linux-gnu -mcpu=z13 | FileCheck %s
+
+declare double @llvm.rint.f64(double)
+declare double @llvm.nearbyint.f64(double)
+declare double @llvm.floor.f64(double)
+declare double @llvm.ceil.f64(double)
+declare double @llvm.trunc.f64(double)
+declare double @llvm.round.f64(double)
+declare <2 x double> @llvm.rint.v2f64(<2 x double>)
+declare <2 x double> @llvm.nearbyint.v2f64(<2 x double>)
+declare <2 x double> @llvm.floor.v2f64(<2 x double>)
+declare <2 x double> @llvm.ceil.v2f64(<2 x double>)
+declare <2 x double> @llvm.trunc.v2f64(<2 x double>)
+declare <2 x double> @llvm.round.v2f64(<2 x double>)
+
+define <2 x double> @f1(<2 x double> %val) {
+; CHECK-LABEL: f1:
+; CHECK: vfidb %v24, %v24, 0, 0
+; CHECK: br %r14
+  %res = call <2 x double> @llvm.rint.v2f64(<2 x double> %val)
+  ret <2 x double> %res
+}
+
+define <2 x double> @f2(<2 x double> %val) {
+; CHECK-LABEL: f2:
+; CHECK: vfidb %v24, %v24, 4, 0
+; CHECK: br %r14
+  %res = call <2 x double> @llvm.nearbyint.v2f64(<2 x double> %val)
+  ret <2 x double> %res
+}
+
+define <2 x double> @f3(<2 x double> %val) {
+; CHECK-LABEL: f3:
+; CHECK: vfidb %v24, %v24, 4, 7
+; CHECK: br %r14
+  %res = call <2 x double> @llvm.floor.v2f64(<2 x double> %val)
+  ret <2 x double> %res
+}
+
+define <2 x double> @f4(<2 x double> %val) {
+; CHECK-LABEL: f4:
+; CHECK: vfidb %v24, %v24, 4, 6
+; CHECK: br %r14
+  %res = call <2 x double> @llvm.ceil.v2f64(<2 x double> %val)
+  ret <2 x double> %res
+}
+
+define <2 x double> @f5(<2 x double> %val) {
+; CHECK-LABEL: f5:
+; CHECK: vfidb %v24, %v24, 4, 5
+; CHECK: br %r14
+  %res = call <2 x double> @llvm.trunc.v2f64(<2 x double> %val)
+  ret <2 x double> %res
+}
+
+define <2 x double> @f6(<2 x double> %val) {
+; CHECK-LABEL: f6:
+; CHECK: vfidb %v24, %v24, 4, 1
+; CHECK: br %r14
+  %res = call <2 x double> @llvm.round.v2f64(<2 x double> %val)
+  ret <2 x double> %res
+}
+
+define double @f7(<2 x double> %val) {
+; CHECK-LABEL: f7:
+; CHECK: wfidb %f0, %v24, 0, 0
+; CHECK: br %r14
+  %scalar = extractelement <2 x double> %val, i32 0
+  %res = call double @llvm.rint.f64(double %scalar)
+  ret double %res
+}
+
+define double @f8(<2 x double> %val) {
+; CHECK-LABEL: f8:
+; CHECK: wfidb %f0, %v24, 4, 0
+; CHECK: br %r14
+  %scalar = extractelement <2 x double> %val, i32 0
+  %res = call double @llvm.nearbyint.f64(double %scalar)
+  ret double %res
+}
+
+define double @f9(<2 x double> %val) {
+; CHECK-LABEL: f9:
+; CHECK: wfidb %f0, %v24, 4, 7
+; CHECK: br %r14
+  %scalar = extractelement <2 x double> %val, i32 0
+  %res = call double @llvm.floor.f64(double %scalar)
+  ret double %res
+}
+
+define double @f10(<2 x double> %val) {
+; CHECK-LABEL: f10:
+; CHECK: wfidb %f0, %v24, 4, 6
+; CHECK: br %r14
+  %scalar = extractelement <2 x double> %val, i32 0
+  %res = call double @llvm.ceil.f64(double %scalar)
+  ret double %res
+}
+
+define double @f11(<2 x double> %val) {
+; CHECK-LABEL: f11:
+; CHECK: wfidb %f0, %v24, 4, 5
+; CHECK: br %r14
+  %scalar = extractelement <2 x double> %val, i32 0
+  %res = call double @llvm.trunc.f64(double %scalar)
+  ret double %res
+}
+
+define double @f12(<2 x double> %val) {
+; CHECK-LABEL: f12:
+; CHECK: wfidb %f0, %v24, 4, 1
+; CHECK: br %r14
+  %scalar = extractelement <2 x double> %val, i32 0
+  %res = call double @llvm.round.f64(double %scalar)
+  ret double %res
+}
diff --git a/test/CodeGen/SystemZ/vec-shift-01.ll b/test/CodeGen/SystemZ/vec-shift-01.ll
new file mode 100644
index 00000000000..be8605b182c
--- /dev/null
+++ b/test/CodeGen/SystemZ/vec-shift-01.ll
@@ -0,0 +1,39 @@
+; Test vector shift left with vector shift amount.
+;
+; RUN: llc < %s -mtriple=s390x-linux-gnu -mcpu=z13 | FileCheck %s
+
+; Test a v16i8 shift.
+define <16 x i8> @f1(<16 x i8> %dummy, <16 x i8> %val1, <16 x i8> %val2) {
+; CHECK-LABEL: f1:
+; CHECK: veslvb %v24, %v26, %v28
+; CHECK: br %r14
+  %ret = shl <16 x i8> %val1, %val2
+  ret <16 x i8> %ret
+}
+
+; Test a v8i16 shift.
+define <8 x i16> @f2(<8 x i16> %dummy, <8 x i16> %val1, <8 x i16> %val2) {
+; CHECK-LABEL: f2:
+; CHECK: veslvh %v24, %v26, %v28
+; CHECK: br %r14
+  %ret = shl <8 x i16> %val1, %val2
+  ret <8 x i16> %ret
+}
+
+; Test a v4i32 shift.
+define <4 x i32> @f3(<4 x i32> %dummy, <4 x i32> %val1, <4 x i32> %val2) {
+; CHECK-LABEL: f3:
+; CHECK: veslvf %v24, %v26, %v28
+; CHECK: br %r14
+  %ret = shl <4 x i32> %val1, %val2
+  ret <4 x i32> %ret
+}
+
+; Test a v2i64 shift.
+define <2 x i64> @f4(<2 x i64> %dummy, <2 x i64> %val1, <2 x i64> %val2) {
+; CHECK-LABEL: f4:
+; CHECK: veslvg %v24, %v26, %v28
+; CHECK: br %r14
+  %ret = shl <2 x i64> %val1, %val2
+  ret <2 x i64> %ret
+}
diff --git a/test/CodeGen/SystemZ/vec-shift-02.ll b/test/CodeGen/SystemZ/vec-shift-02.ll
new file mode 100644
index 00000000000..2825872e023
--- /dev/null
+++ b/test/CodeGen/SystemZ/vec-shift-02.ll
@@ -0,0 +1,39 @@
+; Test vector arithmetic shift right with vector shift amount.
+;
+; RUN: llc < %s -mtriple=s390x-linux-gnu -mcpu=z13 | FileCheck %s
+
+; Test a v16i8 shift.
+define <16 x i8> @f1(<16 x i8> %dummy, <16 x i8> %val1, <16 x i8> %val2) {
+; CHECK-LABEL: f1:
+; CHECK: vesravb %v24, %v26, %v28
+; CHECK: br %r14
+  %ret = ashr <16 x i8> %val1, %val2
+  ret <16 x i8> %ret
+}
+
+; Test a v8i16 shift.
+define <8 x i16> @f2(<8 x i16> %dummy, <8 x i16> %val1, <8 x i16> %val2) {
+; CHECK-LABEL: f2:
+; CHECK: vesravh %v24, %v26, %v28
+; CHECK: br %r14
+  %ret = ashr <8 x i16> %val1, %val2
+  ret <8 x i16> %ret
+}
+
+; Test a v4i32 shift.
+define <4 x i32> @f3(<4 x i32> %dummy, <4 x i32> %val1, <4 x i32> %val2) {
+; CHECK-LABEL: f3:
+; CHECK: vesravf %v24, %v26, %v28
+; CHECK: br %r14
+  %ret = ashr <4 x i32> %val1, %val2
+  ret <4 x i32> %ret
+}
+
+; Test a v2i64 shift.
+define <2 x i64> @f4(<2 x i64> %dummy, <2 x i64> %val1, <2 x i64> %val2) {
+; CHECK-LABEL: f4:
+; CHECK: vesravg %v24, %v26, %v28
+; CHECK: br %r14
+  %ret = ashr <2 x i64> %val1, %val2
+  ret <2 x i64> %ret
+}
diff --git a/test/CodeGen/SystemZ/vec-shift-03.ll b/test/CodeGen/SystemZ/vec-shift-03.ll
new file mode 100644
index 00000000000..c923d8b5d45
--- /dev/null
+++ b/test/CodeGen/SystemZ/vec-shift-03.ll
@@ -0,0 +1,39 @@
+; Test vector logical shift right with vector shift amount.
+;
+; RUN: llc < %s -mtriple=s390x-linux-gnu -mcpu=z13 | FileCheck %s
+
+; Test a v16i8 shift.
+define <16 x i8> @f1(<16 x i8> %dummy, <16 x i8> %val1, <16 x i8> %val2) {
+; CHECK-LABEL: f1:
+; CHECK: vesrlvb %v24, %v26, %v28
+; CHECK: br %r14
+  %ret = lshr <16 x i8> %val1, %val2
+  ret <16 x i8> %ret
+}
+
+; Test a v8i16 shift.
+define <8 x i16> @f2(<8 x i16> %dummy, <8 x i16> %val1, <8 x i16> %val2) {
+; CHECK-LABEL: f2:
+; CHECK: vesrlvh %v24, %v26, %v28
+; CHECK: br %r14
+  %ret = lshr <8 x i16> %val1, %val2
+  ret <8 x i16> %ret
+}
+
+; Test a v4i32 shift.
+define <4 x i32> @f3(<4 x i32> %dummy, <4 x i32> %val1, <4 x i32> %val2) {
+; CHECK-LABEL: f3:
+; CHECK: vesrlvf %v24, %v26, %v28
+; CHECK: br %r14
+  %ret = lshr <4 x i32> %val1, %val2
+  ret <4 x i32> %ret
+}
+
+; Test a v2i64 shift.
+define <2 x i64> @f4(<2 x i64> %dummy, <2 x i64> %val1, <2 x i64> %val2) {
+; CHECK-LABEL: f4:
+; CHECK: vesrlvg %v24, %v26, %v28
+; CHECK: br %r14
+  %ret = lshr <2 x i64> %val1, %val2
+  ret <2 x i64> %ret
+}
diff --git a/test/CodeGen/SystemZ/vec-shift-04.ll b/test/CodeGen/SystemZ/vec-shift-04.ll
new file mode 100644
index 00000000000..6fd12897bf5
--- /dev/null
+++ b/test/CodeGen/SystemZ/vec-shift-04.ll
@@ -0,0 +1,134 @@
+; Test vector shift left with scalar shift amount.
+;
+; RUN: llc < %s -mtriple=s390x-linux-gnu -mcpu=z13 | FileCheck %s
+
+; Test a v16i8 shift by a variable.
+define <16 x i8> @f1(<16 x i8> %dummy, <16 x i8> %val1, i32 %shift) {
+; CHECK-LABEL: f1:
+; CHECK: veslb %v24, %v26, 0(%r2)
+; CHECK: br %r14
+  %truncshift = trunc i32 %shift to i8
+  %shiftvec = insertelement <16 x i8> undef, i8 %truncshift, i32 0
+  %val2 = shufflevector <16 x i8> %shiftvec, <16 x i8> undef,
+                        <16 x i32> zeroinitializer
+  %ret = shl <16 x i8> %val1, %val2
+  ret <16 x i8> %ret
+}
+
+; Test a v16i8 shift by the lowest useful constant.
+define <16 x i8> @f2(<16 x i8> %dummy, <16 x i8> %val) {
+; CHECK-LABEL: f2:
+; CHECK: veslb %v24, %v26, 1
+; CHECK: br %r14
+  %ret = shl <16 x i8> %val, <i8 1, i8 1, i8 1, i8 1,
+                              i8 1, i8 1, i8 1, i8 1,
+                              i8 1, i8 1, i8 1, i8 1,
+                              i8 1, i8 1, i8 1, i8 1>
+  ret <16 x i8> %ret
+}
+
+; Test a v16i8 shift by the highest useful constant.
+define <16 x i8> @f3(<16 x i8> %dummy, <16 x i8> %val) {
+; CHECK-LABEL: f3:
+; CHECK: veslb %v24, %v26, 7
+; CHECK: br %r14
+  %ret = shl <16 x i8> %val, <i8 7, i8 7, i8 7, i8 7,
+                              i8 7, i8 7, i8 7, i8 7,
+                              i8 7, i8 7, i8 7, i8 7,
+                              i8 7, i8 7, i8 7, i8 7>
+  ret <16 x i8> %ret
+}
+
+; Test a v8i16 shift by a variable.
+define <8 x i16> @f4(<8 x i16> %dummy, <8 x i16> %val1, i32 %shift) {
+; CHECK-LABEL: f4:
+; CHECK: veslh %v24, %v26, 0(%r2)
+; CHECK: br %r14
+  %truncshift = trunc i32 %shift to i16
+  %shiftvec = insertelement <8 x i16> undef, i16 %truncshift, i32 0
+  %val2 = shufflevector <8 x i16> %shiftvec, <8 x i16> undef,
+                        <8 x i32> zeroinitializer
+  %ret = shl <8 x i16> %val1, %val2
+  ret <8 x i16> %ret
+}
+
+; Test a v8i16 shift by the lowest useful constant.
+define <8 x i16> @f5(<8 x i16> %dummy, <8 x i16> %val) {
+; CHECK-LABEL: f5:
+; CHECK: veslh %v24, %v26, 1
+; CHECK: br %r14
+  %ret = shl <8 x i16> %val, <i16 1, i16 1, i16 1, i16 1,
+                              i16 1, i16 1, i16 1, i16 1>
+  ret <8 x i16> %ret
+}
+
+; Test a v8i16 shift by the highest useful constant.
+define <8 x i16> @f6(<8 x i16> %dummy, <8 x i16> %val) {
+; CHECK-LABEL: f6:
+; CHECK: veslh %v24, %v26, 15
+; CHECK: br %r14
+  %ret = shl <8 x i16> %val, <i16 15, i16 15, i16 15, i16 15,
+                              i16 15, i16 15, i16 15, i16 15>
+  ret <8 x i16> %ret
+}
+
+; Test a v4i32 shift by a variable.
+define <4 x i32> @f7(<4 x i32> %dummy, <4 x i32> %val1, i32 %shift) {
+; CHECK-LABEL: f7:
+; CHECK: veslf %v24, %v26, 0(%r2)
+; CHECK: br %r14
+  %shiftvec = insertelement <4 x i32> undef, i32 %shift, i32 0
+  %val2 = shufflevector <4 x i32> %shiftvec, <4 x i32> undef,
+                        <4 x i32> zeroinitializer
+  %ret = shl <4 x i32> %val1, %val2
+  ret <4 x i32> %ret
+}
+
+; Test a v4i32 shift by the lowest useful constant.
+define <4 x i32> @f8(<4 x i32> %dummy, <4 x i32> %val) {
+; CHECK-LABEL: f8:
+; CHECK: veslf %v24, %v26, 1
+; CHECK: br %r14
+  %ret = shl <4 x i32> %val, <i32 1, i32 1, i32 1, i32 1>
+  ret <4 x i32> %ret
+}
+
+; Test a v4i32 shift by the highest useful constant.
+define <4 x i32> @f9(<4 x i32> %dummy, <4 x i32> %val) {
+; CHECK-LABEL: f9:
+; CHECK: veslf %v24, %v26, 31
+; CHECK: br %r14
+  %ret = shl <4 x i32> %val, <i32 31, i32 31, i32 31, i32 31>
+  ret <4 x i32> %ret
+}
+
+; Test a v2i64 shift by a variable.
+define <2 x i64> @f10(<2 x i64> %dummy, <2 x i64> %val1, i32 %shift) {
+; CHECK-LABEL: f10:
+; CHECK: veslg %v24, %v26, 0(%r2)
+; CHECK: br %r14
+  %extshift = sext i32 %shift to i64
+  %shiftvec = insertelement <2 x i64> undef, i64 %extshift, i32 0
+  %val2 = shufflevector <2 x i64> %shiftvec, <2 x i64> undef,
+                        <2 x i32> zeroinitializer
+  %ret = shl <2 x i64> %val1, %val2
+  ret <2 x i64> %ret
+}
+
+; Test a v2i64 shift by the lowest useful constant.
+define <2 x i64> @f11(<2 x i64> %dummy, <2 x i64> %val) {
+; CHECK-LABEL: f11:
+; CHECK: veslg %v24, %v26, 1
+; CHECK: br %r14
+  %ret = shl <2 x i64> %val, <i64 1, i64 1>
+  ret <2 x i64> %ret
+}
+
+; Test a v2i64 shift by the highest useful constant.
+define <2 x i64> @f12(<2 x i64> %dummy, <2 x i64> %val) {
+; CHECK-LABEL: f12:
+; CHECK: veslg %v24, %v26, 63
+; CHECK: br %r14
+  %ret = shl <2 x i64> %val, <i64 63, i64 63>
+  ret <2 x i64> %ret
+}
diff --git a/test/CodeGen/SystemZ/vec-shift-05.ll b/test/CodeGen/SystemZ/vec-shift-05.ll
new file mode 100644
index 00000000000..22ce46b2d0d
--- /dev/null
+++ b/test/CodeGen/SystemZ/vec-shift-05.ll
@@ -0,0 +1,134 @@
+; Test vector arithmetic shift right with scalar shift amount.
+;
+; RUN: llc < %s -mtriple=s390x-linux-gnu -mcpu=z13 | FileCheck %s
+
+; Test a v16i8 shift by a variable.
+define <16 x i8> @f1(<16 x i8> %dummy, <16 x i8> %val1, i32 %shift) {
+; CHECK-LABEL: f1:
+; CHECK: vesrab %v24, %v26, 0(%r2)
+; CHECK: br %r14
+  %truncshift = trunc i32 %shift to i8
+  %shiftvec = insertelement <16 x i8> undef, i8 %truncshift, i32 0
+  %val2 = shufflevector <16 x i8> %shiftvec, <16 x i8> undef,
+                        <16 x i32> zeroinitializer
+  %ret = ashr <16 x i8> %val1, %val2
+  ret <16 x i8> %ret
+}
+
+; Test a v16i8 shift by the lowest useful constant.
+define <16 x i8> @f2(<16 x i8> %dummy, <16 x i8> %val) {
+; CHECK-LABEL: f2:
+; CHECK: vesrab %v24, %v26, 1
+; CHECK: br %r14
+  %ret = ashr <16 x i8> %val, <i8 1, i8 1, i8 1, i8 1,
+                               i8 1, i8 1, i8 1, i8 1,
+                               i8 1, i8 1, i8 1, i8 1,
+                               i8 1, i8 1, i8 1, i8 1>
+  ret <16 x i8> %ret
+}
+
+; Test a v16i8 shift by the highest useful constant.
+define <16 x i8> @f3(<16 x i8> %dummy, <16 x i8> %val) {
+; CHECK-LABEL: f3:
+; CHECK: vesrab %v24, %v26, 7
+; CHECK: br %r14
+  %ret = ashr <16 x i8> %val, <i8 7, i8 7, i8 7, i8 7,
+                               i8 7, i8 7, i8 7, i8 7,
+                               i8 7, i8 7, i8 7, i8 7,
+                               i8 7, i8 7, i8 7, i8 7>
+  ret <16 x i8> %ret
+}
+
+; Test a v8i16 shift by a variable.
+define <8 x i16> @f4(<8 x i16> %dummy, <8 x i16> %val1, i32 %shift) {
+; CHECK-LABEL: f4:
+; CHECK: vesrah %v24, %v26, 0(%r2)
+; CHECK: br %r14
+  %truncshift = trunc i32 %shift to i16
+  %shiftvec = insertelement <8 x i16> undef, i16 %truncshift, i32 0
+  %val2 = shufflevector <8 x i16> %shiftvec, <8 x i16> undef,
+                        <8 x i32> zeroinitializer
+  %ret = ashr <8 x i16> %val1, %val2
+  ret <8 x i16> %ret
+}
+
+; Test a v8i16 shift by the lowest useful constant.
+define <8 x i16> @f5(<8 x i16> %dummy, <8 x i16> %val) {
+; CHECK-LABEL: f5:
+; CHECK: vesrah %v24, %v26, 1
+; CHECK: br %r14
+  %ret = ashr <8 x i16> %val, <i16 1, i16 1, i16 1, i16 1,
+                               i16 1, i16 1, i16 1, i16 1>
+  ret <8 x i16> %ret
+}
+
+; Test a v8i16 shift by the highest useful constant.
+define <8 x i16> @f6(<8 x i16> %dummy, <8 x i16> %val) {
+; CHECK-LABEL: f6:
+; CHECK: vesrah %v24, %v26, 15
+; CHECK: br %r14
+  %ret = ashr <8 x i16> %val, <i16 15, i16 15, i16 15, i16 15,
+                               i16 15, i16 15, i16 15, i16 15>
+  ret <8 x i16> %ret
+}
+
+; Test a v4i32 shift by a variable.
+define <4 x i32> @f7(<4 x i32> %dummy, <4 x i32> %val1, i32 %shift) {
+; CHECK-LABEL: f7:
+; CHECK: vesraf %v24, %v26, 0(%r2)
+; CHECK: br %r14
+  %shiftvec = insertelement <4 x i32> undef, i32 %shift, i32 0
+  %val2 = shufflevector <4 x i32> %shiftvec, <4 x i32> undef,
+                        <4 x i32> zeroinitializer
+  %ret = ashr <4 x i32> %val1, %val2
+  ret <4 x i32> %ret
+}
+
+; Test a v4i32 shift by the lowest useful constant.
+define <4 x i32> @f8(<4 x i32> %dummy, <4 x i32> %val) {
+; CHECK-LABEL: f8:
+; CHECK: vesraf %v24, %v26, 1
+; CHECK: br %r14
+  %ret = ashr <4 x i32> %val, <i32 1, i32 1, i32 1, i32 1>
+  ret <4 x i32> %ret
+}
+
+; Test a v4i32 shift by the highest useful constant.
+define <4 x i32> @f9(<4 x i32> %dummy, <4 x i32> %val) {
+; CHECK-LABEL: f9:
+; CHECK: vesraf %v24, %v26, 31
+; CHECK: br %r14
+  %ret = ashr <4 x i32> %val, <i32 31, i32 31, i32 31, i32 31>
+  ret <4 x i32> %ret
+}
+
+; Test a v2i64 shift by a variable.
+define <2 x i64> @f10(<2 x i64> %dummy, <2 x i64> %val1, i32 %shift) {
+; CHECK-LABEL: f10:
+; CHECK: vesrag %v24, %v26, 0(%r2)
+; CHECK: br %r14
+  %extshift = sext i32 %shift to i64
+  %shiftvec = insertelement <2 x i64> undef, i64 %extshift, i32 0
+  %val2 = shufflevector <2 x i64> %shiftvec, <2 x i64> undef,
+                        <2 x i32> zeroinitializer
+  %ret = ashr <2 x i64> %val1, %val2
+  ret <2 x i64> %ret
+}
+
+; Test a v2i64 shift by the lowest useful constant.
+define <2 x i64> @f11(<2 x i64> %dummy, <2 x i64> %val) {
+; CHECK-LABEL: f11:
+; CHECK: vesrag %v24, %v26, 1
+; CHECK: br %r14
+  %ret = ashr <2 x i64> %val, <i64 1, i64 1>
+  ret <2 x i64> %ret
+}
+
+; Test a v2i64 shift by the highest useful constant.
+define <2 x i64> @f12(<2 x i64> %dummy, <2 x i64> %val) {
+; CHECK-LABEL: f12:
+; CHECK: vesrag %v24, %v26, 63
+; CHECK: br %r14
+  %ret = ashr <2 x i64> %val, <i64 63, i64 63>
+  ret <2 x i64> %ret
+}
diff --git a/test/CodeGen/SystemZ/vec-shift-06.ll b/test/CodeGen/SystemZ/vec-shift-06.ll
new file mode 100644
index 00000000000..8a5bb0a9a55
--- /dev/null
+++ b/test/CodeGen/SystemZ/vec-shift-06.ll
@@ -0,0 +1,134 @@
+; Test vector logical shift right with scalar shift amount.
+;
+; RUN: llc < %s -mtriple=s390x-linux-gnu -mcpu=z13 | FileCheck %s
+
+; Test a v16i8 shift by a variable.
+define <16 x i8> @f1(<16 x i8> %dummy, <16 x i8> %val1, i32 %shift) {
+; CHECK-LABEL: f1:
+; CHECK: vesrlb %v24, %v26, 0(%r2)
+; CHECK: br %r14
+  %truncshift = trunc i32 %shift to i8
+  %shiftvec = insertelement <16 x i8> undef, i8 %truncshift, i32 0
+  %val2 = shufflevector <16 x i8> %shiftvec, <16 x i8> undef,
+                        <16 x i32> zeroinitializer
+  %ret = lshr <16 x i8> %val1, %val2
+  ret <16 x i8> %ret
+}
+
+; Test a v16i8 shift by the lowest useful constant.
+define <16 x i8> @f2(<16 x i8> %dummy, <16 x i8> %val) {
+; CHECK-LABEL: f2:
+; CHECK: vesrlb %v24, %v26, 1
+; CHECK: br %r14
+  %ret = lshr <16 x i8> %val, <i8 1, i8 1, i8 1, i8 1,
+                               i8 1, i8 1, i8 1, i8 1,
+                               i8 1, i8 1, i8 1, i8 1,
+                               i8 1, i8 1, i8 1, i8 1>
+  ret <16 x i8> %ret
+}
+
+; Test a v16i8 shift by the highest useful constant.
+define <16 x i8> @f3(<16 x i8> %dummy, <16 x i8> %val) {
+; CHECK-LABEL: f3:
+; CHECK: vesrlb %v24, %v26, 7
+; CHECK: br %r14
+  %ret = lshr <16 x i8> %val, <i8 7, i8 7, i8 7, i8 7,
+                               i8 7, i8 7, i8 7, i8 7,
+                               i8 7, i8 7, i8 7, i8 7,
+                               i8 7, i8 7, i8 7, i8 7>
+  ret <16 x i8> %ret
+}
+
+; Test a v8i16 shift by a variable.
+define <8 x i16> @f4(<8 x i16> %dummy, <8 x i16> %val1, i32 %shift) {
+; CHECK-LABEL: f4:
+; CHECK: vesrlh %v24, %v26, 0(%r2)
+; CHECK: br %r14
+  %truncshift = trunc i32 %shift to i16
+  %shiftvec = insertelement <8 x i16> undef, i16 %truncshift, i32 0
+  %val2 = shufflevector <8 x i16> %shiftvec, <8 x i16> undef,
+                        <8 x i32> zeroinitializer
+  %ret = lshr <8 x i16> %val1, %val2
+  ret <8 x i16> %ret
+}
+
+; Test a v8i16 shift by the lowest useful constant.
+define <8 x i16> @f5(<8 x i16> %dummy, <8 x i16> %val) {
+; CHECK-LABEL: f5:
+; CHECK: vesrlh %v24, %v26, 1
+; CHECK: br %r14
+  %ret = lshr <8 x i16> %val, <i16 1, i16 1, i16 1, i16 1,
+                               i16 1, i16 1, i16 1, i16 1>
+  ret <8 x i16> %ret
+}
+
+; Test a v8i16 shift by the highest useful constant.
+define <8 x i16> @f6(<8 x i16> %dummy, <8 x i16> %val) {
+; CHECK-LABEL: f6:
+; CHECK: vesrlh %v24, %v26, 15
+; CHECK: br %r14
+  %ret = lshr <8 x i16> %val, <i16 15, i16 15, i16 15, i16 15,
+                               i16 15, i16 15, i16 15, i16 15>
+  ret <8 x i16> %ret
+}
+
+; Test a v4i32 shift by a variable.
+define <4 x i32> @f7(<4 x i32> %dummy, <4 x i32> %val1, i32 %shift) {
+; CHECK-LABEL: f7:
+; CHECK: vesrlf %v24, %v26, 0(%r2)
+; CHECK: br %r14
+  %shiftvec = insertelement <4 x i32> undef, i32 %shift, i32 0
+  %val2 = shufflevector <4 x i32> %shiftvec, <4 x i32> undef,
+                        <4 x i32> zeroinitializer
+  %ret = lshr <4 x i32> %val1, %val2
+  ret <4 x i32> %ret
+}
+
+; Test a v4i32 shift by the lowest useful constant.
+define <4 x i32> @f8(<4 x i32> %dummy, <4 x i32> %val) {
+; CHECK-LABEL: f8:
+; CHECK: vesrlf %v24, %v26, 1
+; CHECK: br %r14
+  %ret = lshr <4 x i32> %val, <i32 1, i32 1, i32 1, i32 1>
+  ret <4 x i32> %ret
+}
+
+; Test a v4i32 shift by the highest useful constant.
+define <4 x i32> @f9(<4 x i32> %dummy, <4 x i32> %val) {
+; CHECK-LABEL: f9:
+; CHECK: vesrlf %v24, %v26, 31
+; CHECK: br %r14
+  %ret = lshr <4 x i32> %val, <i32 31, i32 31, i32 31, i32 31>
+  ret <4 x i32> %ret
+}
+
+; Test a v2i64 shift by a variable.
+define <2 x i64> @f10(<2 x i64> %dummy, <2 x i64> %val1, i32 %shift) {
+; CHECK-LABEL: f10:
+; CHECK: vesrlg %v24, %v26, 0(%r2)
+; CHECK: br %r14
+  %extshift = sext i32 %shift to i64
+  %shiftvec = insertelement <2 x i64> undef, i64 %extshift, i32 0
+  %val2 = shufflevector <2 x i64> %shiftvec, <2 x i64> undef,
+                        <2 x i32> zeroinitializer
+  %ret = lshr <2 x i64> %val1, %val2
+  ret <2 x i64> %ret
+}
+
+; Test a v2i64 shift by the lowest useful constant.
+define <2 x i64> @f11(<2 x i64> %dummy, <2 x i64> %val) {
+; CHECK-LABEL: f11:
+; CHECK: vesrlg %v24, %v26, 1
+; CHECK: br %r14
+  %ret = lshr <2 x i64> %val, <i64 1, i64 1>
+  ret <2 x i64> %ret
+}
+
+; Test a v2i64 shift by the highest useful constant.
+define <2 x i64> @f12(<2 x i64> %dummy, <2 x i64> %val) {
+; CHECK-LABEL: f12:
+; CHECK: vesrlg %v24, %v26, 63
+; CHECK: br %r14
+  %ret = lshr <2 x i64> %val, <i64 63, i64 63>
+  ret <2 x i64> %ret
+}
diff --git a/test/CodeGen/SystemZ/vec-shift-07.ll b/test/CodeGen/SystemZ/vec-shift-07.ll
new file mode 100644
index 00000000000..f229c5e25a4
--- /dev/null
+++ b/test/CodeGen/SystemZ/vec-shift-07.ll
@@ -0,0 +1,182 @@
+; Test vector sign extensions.
+;
+; RUN: llc < %s -mtriple=s390x-linux-gnu -mcpu=z13 | FileCheck %s
+
+; Test a v16i1->v16i8 extension.
+define <16 x i8> @f1(<16 x i8> %val) {
+; CHECK-LABEL: f1:
+; CHECK: veslb [[REG:%v[0-9]+]], %v24, 7
+; CHECK: vesrab %v24, [[REG]], 7
+; CHECK: br %r14
+  %trunc = trunc <16 x i8> %val to <16 x i1>
+  %ret = sext <16 x i1> %trunc to <16 x i8>
+  ret <16 x i8> %ret
+}
+
+; Test a v8i1->v8i16 extension.
+define <8 x i16> @f2(<8 x i16> %val) {
+; CHECK-LABEL: f2:
+; CHECK: veslh [[REG:%v[0-9]+]], %v24, 15
+; CHECK: vesrah %v24, [[REG]], 15
+; CHECK: br %r14
+  %trunc = trunc <8 x i16> %val to <8 x i1>
+  %ret = sext <8 x i1> %trunc to <8 x i16>
+  ret <8 x i16> %ret
+}
+
+; Test a v8i8->v8i16 extension.
+define <8 x i16> @f3(<8 x i16> %val) {
+; CHECK-LABEL: f3:
+; CHECK: veslh [[REG:%v[0-9]+]], %v24, 8
+; CHECK: vesrah %v24, [[REG]], 8
+; CHECK: br %r14
+  %trunc = trunc <8 x i16> %val to <8 x i8>
+  %ret = sext <8 x i8> %trunc to <8 x i16>
+  ret <8 x i16> %ret
+}
+
+; Test a v4i1->v4i32 extension.
+define <4 x i32> @f4(<4 x i32> %val) {
+; CHECK-LABEL: f4:
+; CHECK: veslf [[REG:%v[0-9]+]], %v24, 31
+; CHECK: vesraf %v24, [[REG]], 31
+; CHECK: br %r14
+  %trunc = trunc <4 x i32> %val to <4 x i1>
+  %ret = sext <4 x i1> %trunc to <4 x i32>
+  ret <4 x i32> %ret
+}
+
+; Test a v4i8->v4i32 extension.
+define <4 x i32> @f5(<4 x i32> %val) {
+; CHECK-LABEL: f5:
+; CHECK: veslf [[REG:%v[0-9]+]], %v24, 24
+; CHECK: vesraf %v24, [[REG]], 24
+; CHECK: br %r14
+  %trunc = trunc <4 x i32> %val to <4 x i8>
+  %ret = sext <4 x i8> %trunc to <4 x i32>
+  ret <4 x i32> %ret
+}
+
+; Test a v4i16->v4i32 extension.
+define <4 x i32> @f6(<4 x i32> %val) {
+; CHECK-LABEL: f6:
+; CHECK: veslf [[REG:%v[0-9]+]], %v24, 16
+; CHECK: vesraf %v24, [[REG]], 16
+; CHECK: br %r14
+  %trunc = trunc <4 x i32> %val to <4 x i16>
+  %ret = sext <4 x i16> %trunc to <4 x i32>
+  ret <4 x i32> %ret
+}
+
+; Test a v2i1->v2i64 extension.
+define <2 x i64> @f7(<2 x i64> %val) {
+; CHECK-LABEL: f7:
+; CHECK: veslg [[REG:%v[0-9]+]], %v24, 63
+; CHECK: vesrag %v24, [[REG]], 63
+; CHECK: br %r14
+  %trunc = trunc <2 x i64> %val to <2 x i1>
+  %ret = sext <2 x i1> %trunc to <2 x i64>
+  ret <2 x i64> %ret
+}
+
+; Test a v2i8->v2i64 extension.
+define <2 x i64> @f8(<2 x i64> %val) {
+; CHECK-LABEL: f8:
+; CHECK: vsegb %v24, %v24
+; CHECK: br %r14
+  %trunc = trunc <2 x i64> %val to <2 x i8>
+  %ret = sext <2 x i8> %trunc to <2 x i64>
+  ret <2 x i64> %ret
+}
+
+; Test a v2i16->v2i64 extension.
+define <2 x i64> @f9(<2 x i64> %val) {
+; CHECK-LABEL: f9:
+; CHECK: vsegh %v24, %v24
+; CHECK: br %r14
+  %trunc = trunc <2 x i64> %val to <2 x i16>
+  %ret = sext <2 x i16> %trunc to <2 x i64>
+  ret <2 x i64> %ret
+}
+
+; Test a v2i32->v2i64 extension.
+define <2 x i64> @f10(<2 x i64> %val) {
+; CHECK-LABEL: f10:
+; CHECK: vsegf %v24, %v24
+; CHECK: br %r14
+  %trunc = trunc <2 x i64> %val to <2 x i32>
+  %ret = sext <2 x i32> %trunc to <2 x i64>
+  ret <2 x i64> %ret
+}
+
+; Test an alternative v2i8->v2i64 extension.
+define <2 x i64> @f11(<2 x i64> %val) {
+; CHECK-LABEL: f11:
+; CHECK: vsegb %v24, %v24
+; CHECK: br %r14
+  %shl = shl <2 x i64> %val, <i64 56, i64 56>
+  %ret = ashr <2 x i64> %shl, <i64 56, i64 56>
+  ret <2 x i64> %ret
+}
+
+; Test an alternative v2i16->v2i64 extension.
+define <2 x i64> @f12(<2 x i64> %val) {
+; CHECK-LABEL: f12:
+; CHECK: vsegh %v24, %v24
+; CHECK: br %r14
+  %shl = shl <2 x i64> %val, <i64 48, i64 48>
+  %ret = ashr <2 x i64> %shl, <i64 48, i64 48>
+  ret <2 x i64> %ret
+}
+
+; Test an alternative v2i32->v2i64 extension.
+define <2 x i64> @f13(<2 x i64> %val) {
+; CHECK-LABEL: f13:
+; CHECK: vsegf %v24, %v24
+; CHECK: br %r14
+  %shl = shl <2 x i64> %val, <i64 32, i64 32>
+  %ret = ashr <2 x i64> %shl, <i64 32, i64 32>
+  ret <2 x i64> %ret
+}
+
+; Test an extraction-based v2i8->v2i64 extension.
+define <2 x i64> @f14(<16 x i8> %val) {
+; CHECK-LABEL: f14:
+; CHECK: vsegb %v24, %v24
+; CHECK: br %r14
+  %elt0 = extractelement <16 x i8> %val, i32 7
+  %elt1 = extractelement <16 x i8> %val, i32 15
+  %ext0 = sext i8 %elt0 to i64
+  %ext1 = sext i8 %elt1 to i64
+  %vec0 = insertelement <2 x i64> undef, i64 %ext0, i32 0
+  %vec1 = insertelement <2 x i64> %vec0, i64 %ext1, i32 1
+  ret <2 x i64> %vec1
+}
+
+; Test an extraction-based v2i16->v2i64 extension.
+define <2 x i64> @f15(<16 x i16> %val) {
+; CHECK-LABEL: f15:
+; CHECK: vsegh %v24, %v24
+; CHECK: br %r14
+  %elt0 = extractelement <16 x i16> %val, i32 3
+  %elt1 = extractelement <16 x i16> %val, i32 7
+  %ext0 = sext i16 %elt0 to i64
+  %ext1 = sext i16 %elt1 to i64
+  %vec0 = insertelement <2 x i64> undef, i64 %ext0, i32 0
+  %vec1 = insertelement <2 x i64> %vec0, i64 %ext1, i32 1
+  ret <2 x i64> %vec1
+}
+
+; Test an extraction-based v2i32->v2i64 extension.
+define <2 x i64> @f16(<16 x i32> %val) {
+; CHECK-LABEL: f16:
+; CHECK: vsegf %v24, %v24
+; CHECK: br %r14
+  %elt0 = extractelement <16 x i32> %val, i32 1
+  %elt1 = extractelement <16 x i32> %val, i32 3
+  %ext0 = sext i32 %elt0 to i64
+  %ext1 = sext i32 %elt1 to i64
+  %vec0 = insertelement <2 x i64> undef, i64 %ext0, i32 0
+  %vec1 = insertelement <2 x i64> %vec0, i64 %ext1, i32 1
+  ret <2 x i64> %vec1
+}
diff --git a/test/CodeGen/SystemZ/vec-sqrt-01.ll b/test/CodeGen/SystemZ/vec-sqrt-01.ll
new file mode 100644
index 00000000000..5c3ffb3b064
--- /dev/null
+++ b/test/CodeGen/SystemZ/vec-sqrt-01.ll
@@ -0,0 +1,23 @@
+; Test f64 and v2f64 square root.
+;
+; RUN: llc < %s -mtriple=s390x-linux-gnu -mcpu=z13 | FileCheck %s
+
+declare double @llvm.sqrt.f64(double)
+declare <2 x double> @llvm.sqrt.v2f64(<2 x double>)
+
+define <2 x double> @f1(<2 x double> %val) {
+; CHECK-LABEL: f1:
+; CHECK: vfsqdb %v24, %v24
+; CHECK: br %r14
+  %ret = call <2 x double> @llvm.sqrt.v2f64(<2 x double> %val)
+  ret <2 x double> %ret
+}
+
+define double @f2(<2 x double> %val) {
+; CHECK-LABEL: f2:
+; CHECK: wfsqdb %f0, %v24
+; CHECK: br %r14
+  %scalar = extractelement <2 x double> %val, i32 0
+  %ret = call double @llvm.sqrt.f64(double %scalar)
+  ret double %ret
+}
diff --git a/test/CodeGen/SystemZ/vec-sub-01.ll b/test/CodeGen/SystemZ/vec-sub-01.ll
new file mode 100644
index 00000000000..4afad8bef65
--- /dev/null
+++ b/test/CodeGen/SystemZ/vec-sub-01.ll
@@ -0,0 +1,148 @@
+; Test vector subtraction.
+;
+; RUN: llc < %s -mtriple=s390x-linux-gnu -mcpu=z13 | FileCheck %s
+
+; Test a v16i8 subtraction.
+define <16 x i8> @f1(<16 x i8> %dummy, <16 x i8> %val1, <16 x i8> %val2) {
+; CHECK-LABEL: f1:
+; CHECK: vsb %v24, %v26, %v28
+; CHECK: br %r14
+  %ret = sub <16 x i8> %val1, %val2
+  ret <16 x i8> %ret
+}
+
+; Test a v8i16 subtraction.
+define <8 x i16> @f2(<8 x i16> %dummy, <8 x i16> %val1, <8 x i16> %val2) {
+; CHECK-LABEL: f2:
+; CHECK: vsh %v24, %v26, %v28
+; CHECK: br %r14
+  %ret = sub <8 x i16> %val1, %val2
+  ret <8 x i16> %ret
+}
+
+; Test a v4i32 subtraction.
+define <4 x i32> @f3(<4 x i32> %dummy, <4 x i32> %val1, <4 x i32> %val2) {
+; CHECK-LABEL: f3:
+; CHECK: vsf %v24, %v26, %v28
+; CHECK: br %r14
+  %ret = sub <4 x i32> %val1, %val2
+  ret <4 x i32> %ret
+}
+
+; Test a v2i64 subtraction.
+define <2 x i64> @f4(<2 x i64> %dummy, <2 x i64> %val1, <2 x i64> %val2) {
+; CHECK-LABEL: f4:
+; CHECK: vsg %v24, %v26, %v28
+; CHECK: br %r14
+  %ret = sub <2 x i64> %val1, %val2
+  ret <2 x i64> %ret
+}
+
+; Test a v4f32 subtraction, as an example of an operation that needs to be
+; scalarized and reassembled.  At present there's an unnecessary move that
+; could be avoided with smarter ordering.  It also isn't important whether
+; the VSLDBs use the result of the VLRs or use %v24 and %v26 directly.
+define <4 x float> @f5(<4 x float> %val1, <4 x float> %val2) {
+; CHECK-LABEL: f5:
+; CHECK-DAG: vlr %v[[A1:[0-5]]], %v24
+; CHECK-DAG: vlr %v[[A2:[0-5]]], %v26
+; CHECK-DAG: vrepf %v[[B1:[0-5]]], %v[[A1]], 1
+; CHECK-DAG: vrepf %v[[B2:[0-5]]], %v[[A2]], 1
+; CHECK-DAG: vrepf %v[[C1:[0-5]]], %v[[A1]], 2
+; CHECK-DAG: vrepf %v[[C2:[0-5]]], %v[[A2]], 2
+; CHECK-DAG: vrepf %v[[D1:[0-5]]], %v[[A1]], 3
+; CHECK-DAG: vrepf %v[[D2:[0-5]]], %v[[A2]], 3
+; CHECK-DAG: ler %f[[A1copy:[0-5]]], %f[[A1]]
+; CHECK-DAG: sebr %f[[A1copy]], %f[[A2]]
+; CHECK-DAG: sebr %f[[B1]], %f[[B2]]
+; CHECK-DAG: sebr %f[[C1]], %f[[C2]]
+; CHECK-DAG: sebr %f[[D1]], %f[[D2]]
+; CHECK-DAG: vmrhf [[HIGH:%v[0-9]+]], %v[[A1copy]], %v[[B1]]
+; CHECK-DAG: vmrhf [[LOW:%v[0-9]+]], %v[[C1]], %v[[D1]]
+; CHECK: vmrhg %v24, [[HIGH]], [[LOW]]
+; CHECK: br %r14
+  %ret = fsub <4 x float> %val1, %val2
+  ret <4 x float> %ret
+}
+
+; Test a v2f64 subtraction.
+define <2 x double> @f6(<2 x double> %dummy, <2 x double> %val1,
+                        <2 x double> %val2) {
+; CHECK-LABEL: f6:
+; CHECK: vfsdb %v24, %v26, %v28
+; CHECK: br %r14
+  %ret = fsub <2 x double> %val1, %val2
+  ret <2 x double> %ret
+}
+
+; Test an f64 subtraction that uses vector registers.
+define double @f7(<2 x double> %val1, <2 x double> %val2) {
+; CHECK-LABEL: f7:
+; CHECK: wfsdb %f0, %v24, %v26
+; CHECK: br %r14
+  %scalar1 = extractelement <2 x double> %val1, i32 0
+  %scalar2 = extractelement <2 x double> %val2, i32 0
+  %ret = fsub double %scalar1, %scalar2
+  ret double %ret
+}
+
+; Test a v2i8 subtraction, which gets promoted to v16i8.
+define <2 x i8> @f8(<2 x i8> %dummy, <2 x i8> %val1, <2 x i8> %val2) {
+; CHECK-LABEL: f8:
+; CHECK: vsb %v24, %v26, %v28
+; CHECK: br %r14
+  %ret = sub <2 x i8> %val1, %val2
+  ret <2 x i8> %ret
+}
+
+; Test a v4i8 subtraction, which gets promoted to v16i8.
+define <4 x i8> @f9(<4 x i8> %dummy, <4 x i8> %val1, <4 x i8> %val2) {
+; CHECK-LABEL: f9:
+; CHECK: vsb %v24, %v26, %v28
+; CHECK: br %r14
+  %ret = sub <4 x i8> %val1, %val2
+  ret <4 x i8> %ret
+}
+
+; Test a v8i8 subtraction, which gets promoted to v16i8.
+define <8 x i8> @f10(<8 x i8> %dummy, <8 x i8> %val1, <8 x i8> %val2) {
+; CHECK-LABEL: f10:
+; CHECK: vsb %v24, %v26, %v28
+; CHECK: br %r14
+  %ret = sub <8 x i8> %val1, %val2
+  ret <8 x i8> %ret
+}
+
+; Test a v2i16 subtraction, which gets promoted to v8i16.
+define <2 x i16> @f11(<2 x i16> %dummy, <2 x i16> %val1, <2 x i16> %val2) {
+; CHECK-LABEL: f11:
+; CHECK: vsh %v24, %v26, %v28
+; CHECK: br %r14
+  %ret = sub <2 x i16> %val1, %val2
+  ret <2 x i16> %ret
+}
+
+; Test a v4i16 subtraction, which gets promoted to v8i16.
+define <4 x i16> @f12(<4 x i16> %dummy, <4 x i16> %val1, <4 x i16> %val2) {
+; CHECK-LABEL: f12:
+; CHECK: vsh %v24, %v26, %v28
+; CHECK: br %r14
+  %ret = sub <4 x i16> %val1, %val2
+  ret <4 x i16> %ret
+}
+
+; Test a v2i32 subtraction, which gets promoted to v4i32.
+define <2 x i32> @f13(<2 x i32> %dummy, <2 x i32> %val1, <2 x i32> %val2) {
+; CHECK-LABEL: f13:
+; CHECK: vsf %v24, %v26, %v28
+; CHECK: br %r14
+  %ret = sub <2 x i32> %val1, %val2
+  ret <2 x i32> %ret
+}
+
+; Test a v2f32 subtraction, which gets promoted to v4f32.
+define <2 x float> @f14(<2 x float> %val1, <2 x float> %val2) {
+; No particular output expected, but must compile.
+  %ret = fsub <2 x float> %val1, %val2
+  ret <2 x float> %ret
+}
diff --git a/test/CodeGen/SystemZ/vec-xor-01.ll b/test/CodeGen/SystemZ/vec-xor-01.ll
new file mode 100644
index 00000000000..063b768117c
--- /dev/null
+++ b/test/CodeGen/SystemZ/vec-xor-01.ll
@@ -0,0 +1,39 @@
+; Test vector XOR.
+;
+; RUN: llc < %s -mtriple=s390x-linux-gnu -mcpu=z13 | FileCheck %s
+
+; Test a v16i8 XOR.
+define <16 x i8> @f1(<16 x i8> %dummy, <16 x i8> %val1, <16 x i8> %val2) {
+; CHECK-LABEL: f1:
+; CHECK: vx %v24, %v26, %v28
+; CHECK: br %r14
+  %ret = xor <16 x i8> %val1, %val2
+  ret <16 x i8> %ret
+}
+
+; Test a v8i16 XOR.
+define <8 x i16> @f2(<8 x i16> %dummy, <8 x i16> %val1, <8 x i16> %val2) {
+; CHECK-LABEL: f2:
+; CHECK: vx %v24, %v26, %v28
+; CHECK: br %r14
+  %ret = xor <8 x i16> %val1, %val2
+  ret <8 x i16> %ret
+}
+
+; Test a v4i32 XOR.
+define <4 x i32> @f3(<4 x i32> %dummy, <4 x i32> %val1, <4 x i32> %val2) {
+; CHECK-LABEL: f3:
+; CHECK: vx %v24, %v26, %v28
+; CHECK: br %r14
+  %ret = xor <4 x i32> %val1, %val2
+  ret <4 x i32> %ret
+}
+
+; Test a v2i64 XOR.
+define <2 x i64> @f4(<2 x i64> %dummy, <2 x i64> %val1, <2 x i64> %val2) {
+; CHECK-LABEL: f4:
+; CHECK: vx %v24, %v26, %v28
+; CHECK: br %r14
+  %ret = xor <2 x i64> %val1, %val2
+  ret <2 x i64> %ret
+}
diff --git a/test/CodeGen/Thumb/2010-07-15-debugOrdering.ll b/test/CodeGen/Thumb/2010-07-15-debugOrdering.ll
index 92358308bbb..0fd1a9e1e23 100644
--- a/test/CodeGen/Thumb/2010-07-15-debugOrdering.ll
+++ b/test/CodeGen/Thumb/2010-07-15-debugOrdering.ll
@@ -25,7 +25,7 @@ define void @_Z19getClosestDiagonal3ii(%0* noalias sret, i32, i32) nounwind {
   %storemerge = phi double [ -1.000000e+00, %4 ], [ 1.000000e+00, %3 ], [ 1.000000e+00, %3 ] ; <double> [#uses=1]
   %v_6 = icmp slt i32 %1, 2                         ; <i1> [#uses=1]
   %storemerge1 = select i1 %v_6, double 1.000000e+00, double -1.000000e+00 ; <double> [#uses=3]
-  call void @llvm.dbg.value(metadata double %storemerge, i64 0, metadata !91, metadata !MDExpression()), !dbg !0
+  call void @llvm.dbg.value(metadata double %storemerge, i64 0, metadata !91, metadata !DIExpression()), !dbg !0
   %v_7 = icmp eq i32 %2, 1, !dbg !92                ; <i1> [#uses=1]
   %storemerge2 = select i1 %v_7, double 1.000000e+00, double -1.000000e+00 ; <double> [#uses=3]
   %v_8 = getelementptr inbounds %0, %0* %0, i32 0, i32 0, i32 0 ; <double*> [#uses=1]
@@ -48,108 +48,108 @@ declare void @llvm.dbg.value(metadata, i64, metadata, metadata) nounwind readnon
 
 !llvm.dbg.cu = !{!5}
 !llvm.module.flags = !{!104}
-!0 = !MDLocation(line: 46, scope: !1)
-!1 = distinct !MDLexicalBlock(line: 44, column: 0, file: !101, scope: !2)
-!2 = distinct !MDLexicalBlock(line: 44, column: 0, file: !101, scope: !3)
-!3 = !MDSubprogram(name: "getClosestDiagonal3", linkageName: "_Z19getClosestDiagonal3ii", line: 44, isLocal: false, isDefinition: true, virtualIndex: 6, isOptimized: false, file: !101, scope: null, type: !6)
-!4 = !MDFile(filename: "ggEdgeDiscrepancy.cc", directory: "/Volumes/Home/grosbaj/sources/llvm-externals/speccpu2000/benchspec/CINT2000/252.eon/src")
-!5 = !MDCompileUnit(language: DW_LANG_C_plus_plus, producer: "4.2.1 (Based on Apple Inc. build 5658) (LLVM build 00)", isOptimized: true, emissionKind: 0, file: !101, enums: !102, retainedTypes: !102, subprograms: !103)
-!6 = !MDSubroutineType(types: !7)
+!0 = !DILocation(line: 46, scope: !1)
+!1 = distinct !DILexicalBlock(line: 44, column: 0, file: !101, scope: !2)
+!2 = distinct !DILexicalBlock(line: 44, column: 0, file: !101, scope: !3)
+!3 = !DISubprogram(name: "getClosestDiagonal3", linkageName: "_Z19getClosestDiagonal3ii", line: 44, isLocal: false, isDefinition: true, virtualIndex: 6, isOptimized: false, file: !101, scope: null, type: !6)
+!4 = !DIFile(filename: "ggEdgeDiscrepancy.cc", directory: "/Volumes/Home/grosbaj/sources/llvm-externals/speccpu2000/benchspec/CINT2000/252.eon/src")
+!5 = !DICompileUnit(language: DW_LANG_C_plus_plus, producer: "4.2.1 (Based on Apple Inc. build 5658) (LLVM build 00)", isOptimized: true, emissionKind: 0, file: !101, enums: !102, retainedTypes: !102, subprograms: !103)
+!6 = !DISubroutineType(types: !7)
 !7 = !{!8, !22, !22}
-!8 = !MDCompositeType(tag: DW_TAG_structure_type, name: "ggVector3", line: 66, size: 192, align: 32, file: !99, elements: !10)
-!9 = !MDFile(filename: "ggVector3.h", directory: "/Volumes/Home/grosbaj/sources/llvm-externals/speccpu2000/benchspec/CINT2000/252.eon/src")
-!99 = !MDFile(filename: "ggVector3.h", directory: "/Volumes/Home/grosbaj/sources/llvm-externals/speccpu2000/benchspec/CINT2000/252.eon/src")
+!8 = !DICompositeType(tag: DW_TAG_structure_type, name: "ggVector3", line: 66, size: 192, align: 32, file: !99, elements: !10)
+!9 = !DIFile(filename: "ggVector3.h", directory: "/Volumes/Home/grosbaj/sources/llvm-externals/speccpu2000/benchspec/CINT2000/252.eon/src")
+!99 = !DIFile(filename: "ggVector3.h", directory: "/Volumes/Home/grosbaj/sources/llvm-externals/speccpu2000/benchspec/CINT2000/252.eon/src")
 !10 = !{!11, !16, !23, !26, !29, !30, !35, !36, !37, !41, !42, !43, !46, !47, !48, !52, !53, !54, !57, !60, !63, !66, !70, !71, !74, !75, !76, !77, !78, !81, !82, !83, !84, !85, !88, !89, !90}
-!11 = !MDDerivedType(tag: DW_TAG_member, name: "e", line: 160, size: 192, align: 32, file: !99, scope: !8, baseType: !12)
-!12 = !MDCompositeType(tag: DW_TAG_array_type, size: 192, align: 32, file: !101, scope: !4, baseType: !13, elements: !14)
-!13 = !MDBasicType(tag: DW_TAG_base_type, name: "double", size: 64, align: 32, encoding: DW_ATE_float)
+!11 = !DIDerivedType(tag: DW_TAG_member, name: "e", line: 160, size: 192, align: 32, file: !99, scope: !8, baseType: !12)
+!12 = !DICompositeType(tag: DW_TAG_array_type, size: 192, align: 32, file: !101, scope: !4, baseType: !13, elements: !14)
+!13 = !DIBasicType(tag: DW_TAG_base_type, name: "double", size: 64, align: 32, encoding: DW_ATE_float)
 !14 = !{!15}
-!15 = !MDSubrange(count: 3)
-!16 = !MDSubprogram(name: "ggVector3", line: 72, isLocal: false, isDefinition: false, virtualIndex: 6, isOptimized: false, file: !9, scope: !8, type: !17)
-!17 = !MDSubroutineType(types: !18)
+!15 = !DISubrange(count: 3)
+!16 = !DISubprogram(name: "ggVector3", line: 72, isLocal: false, isDefinition: false, virtualIndex: 6, isOptimized: false, file: !9, scope: !8, type: !17)
+!17 = !DISubroutineType(types: !18)
 !18 = !{null, !19, !20}
-!19 = !MDDerivedType(tag: DW_TAG_pointer_type, size: 32, align: 32, flags: DIFlagArtificial, file: !101, scope: !4, baseType: !8)
-!20 = !MDDerivedType(tag: DW_TAG_typedef, name: "ggBoolean", line: 478, file: !100, baseType: !22)
-!21 = !MDFile(filename: "math.h", directory: "/Developer/Platforms/iPhoneOS.platform/Developer/SDKs/iPhoneOS4.2.Internal.sdk/usr/include/architecture/arm")
-!100 = !MDFile(filename: "math.h", directory: "/Developer/Platforms/iPhoneOS.platform/Developer/SDKs/iPhoneOS4.2.Internal.sdk/usr/include/architecture/arm")
-!22 = !MDBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
-!23 = !MDSubprogram(name: "ggVector3", line: 73, isLocal: false, isDefinition: false, virtualIndex: 6, isOptimized: false, file: !9, scope: !8, type: !24)
-!24 = !MDSubroutineType(types: !25)
+!19 = !DIDerivedType(tag: DW_TAG_pointer_type, size: 32, align: 32, flags: DIFlagArtificial, file: !101, scope: !4, baseType: !8)
+!20 = !DIDerivedType(tag: DW_TAG_typedef, name: "ggBoolean", line: 478, file: !100, baseType: !22)
+!21 = !DIFile(filename: "math.h", directory: "/Developer/Platforms/iPhoneOS.platform/Developer/SDKs/iPhoneOS4.2.Internal.sdk/usr/include/architecture/arm")
+!100 = !DIFile(filename: "math.h", directory: "/Developer/Platforms/iPhoneOS.platform/Developer/SDKs/iPhoneOS4.2.Internal.sdk/usr/include/architecture/arm")
+!22 = !DIBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
+!23 = !DISubprogram(name: "ggVector3", line: 73, isLocal: false, isDefinition: false, virtualIndex: 6, isOptimized: false, file: !9, scope: !8, type: !24)
+!24 = !DISubroutineType(types: !25)
 !25 = !{null, !19}
-!26 = !MDSubprogram(name: "ggVector3", line: 74, isLocal: false, isDefinition: false, virtualIndex: 6, isOptimized: false, file: !9, scope: !8, type: !27)
-!27 = !MDSubroutineType(types: !28)
+!26 = !DISubprogram(name: "ggVector3", line: 74, isLocal: false, isDefinition: false, virtualIndex: 6, isOptimized: false, file: !9, scope: !8, type: !27)
+!27 = !DISubroutineType(types: !28)
 !28 = !{null, !19, !13, !13, !13}
-!29 = !MDSubprogram(name: "Set", linkageName: "_ZN9ggVector33SetEddd", line: 81, isLocal: false, isDefinition: false, virtualIndex: 6, isOptimized: false, file: !9, scope: !8, type: !27)
-!30 = !MDSubprogram(name: "x", linkageName: "_ZNK9ggVector31xEv", line: 82, isLocal: false, isDefinition: false, virtualIndex: 6, isOptimized: false, file: !9, scope: !8, type: !31)
-!31 = !MDSubroutineType(types: !32)
+!29 = !DISubprogram(name: "Set", linkageName: "_ZN9ggVector33SetEddd", line: 81, isLocal: false, isDefinition: false, virtualIndex: 6, isOptimized: false, file: !9, scope: !8, type: !27)
+!30 = !DISubprogram(name: "x", linkageName: "_ZNK9ggVector31xEv", line: 82, isLocal: false, isDefinition: false, virtualIndex: 6, isOptimized: false, file: !9, scope: !8, type: !31)
+!31 = !DISubroutineType(types: !32)
 !32 = !{!13, !33}
-!33 = !MDDerivedType(tag: DW_TAG_pointer_type, size: 32, align: 32, flags: DIFlagArtificial, file: !101, scope: !4, baseType: !34)
-!34 = !MDDerivedType(tag: DW_TAG_const_type, size: 192, align: 32, file: !101, scope: !4, baseType: !8)
-!35 = !MDSubprogram(name: "y", linkageName: "_ZNK9ggVector31yEv", line: 83, isLocal: false, isDefinition: false, virtualIndex: 6, isOptimized: false, file: !9, scope: !8, type: !31)
-!36 = !MDSubprogram(name: "z", linkageName: "_ZNK9ggVector31zEv", line: 84, isLocal: false, isDefinition: false, virtualIndex: 6, isOptimized: false, file: !9, scope: !8, type: !31)
-!37 = !MDSubprogram(name: "x", linkageName: "_ZN9ggVector31xEv", line: 85, isLocal: false, isDefinition: true, virtualIndex: 6, isOptimized: false, file: !9, scope: !8, type: !38)
-!38 = !MDSubroutineType(types: !39)
+!33 = !DIDerivedType(tag: DW_TAG_pointer_type, size: 32, align: 32, flags: DIFlagArtificial, file: !101, scope: !4, baseType: !34)
+!34 = !DIDerivedType(tag: DW_TAG_const_type, size: 192, align: 32, file: !101, scope: !4, baseType: !8)
+!35 = !DISubprogram(name: "y", linkageName: "_ZNK9ggVector31yEv", line: 83, isLocal: false, isDefinition: false, virtualIndex: 6, isOptimized: false, file: !9, scope: !8, type: !31)
+!36 = !DISubprogram(name: "z", linkageName: "_ZNK9ggVector31zEv", line: 84, isLocal: false, isDefinition: false, virtualIndex: 6, isOptimized: false, file: !9, scope: !8, type: !31)
+!37 = !DISubprogram(name: "x", linkageName: "_ZN9ggVector31xEv", line: 85, isLocal: false, isDefinition: true, virtualIndex: 6, isOptimized: false, file: !9, scope: !8, type: !38)
+!38 = !DISubroutineType(types: !39)
 !39 = !{!40, !19}
-!40 = !MDDerivedType(tag: DW_TAG_reference_type, name: "double", size: 32, align: 32, file: !101, scope: !4, baseType: !13)
-!41 = !MDSubprogram(name: "y", linkageName: "_ZN9ggVector31yEv", line: 86, isLocal: false, isDefinition: true, virtualIndex: 6, isOptimized: false, file: !9, scope: !8, type: !38)
-!42 = !MDSubprogram(name: "z", linkageName: "_ZN9ggVector31zEv", line: 87, isLocal: false, isDefinition: true, virtualIndex: 6, isOptimized: false, file: !9, scope: !8, type: !38)
-!43 = !MDSubprogram(name: "SetX", linkageName: "_ZN9ggVector34SetXEd", line: 88, isLocal: false, isDefinition: false, virtualIndex: 6, isOptimized: false, file: !9, scope: !8, type: !44)
-!44 = !MDSubroutineType(types: !45)
+!40 = !DIDerivedType(tag: DW_TAG_reference_type, name: "double", size: 32, align: 32, file: !101, scope: !4, baseType: !13)
+!41 = !DISubprogram(name: "y", linkageName: "_ZN9ggVector31yEv", line: 86, isLocal: false, isDefinition: true, virtualIndex: 6, isOptimized: false, file: !9, scope: !8, type: !38)
+!42 = !DISubprogram(name: "z", linkageName: "_ZN9ggVector31zEv", line: 87, isLocal: false, isDefinition: true, virtualIndex: 6, isOptimized: false, file: !9, scope: !8, type: !38)
+!43 = !DISubprogram(name: "SetX", linkageName: "_ZN9ggVector34SetXEd", line: 88, isLocal: false, isDefinition: false, virtualIndex: 6, isOptimized: false, file: !9, scope: !8, type: !44)
+!44 = !DISubroutineType(types: !45)
 !45 = !{null, !19, !13}
-!46 = !MDSubprogram(name: "SetY", linkageName: "_ZN9ggVector34SetYEd", line: 89, isLocal: false, isDefinition: false, virtualIndex: 6, isOptimized: false, file: !9, scope: !8, type: !44)
-!47 = !MDSubprogram(name: "SetZ", linkageName: "_ZN9ggVector34SetZEd", line: 90, isLocal: false, isDefinition: false, virtualIndex: 6, isOptimized: false, file: !9, scope: !8, type: !44)
-!48 = !MDSubprogram(name: "ggVector3", line: 92, isLocal: false, isDefinition: false, virtualIndex: 6, isOptimized: false, file: !9, scope: !8, type: !49)
-!49 = !MDSubroutineType(types: !50)
+!46 = !DISubprogram(name: "SetY", linkageName: "_ZN9ggVector34SetYEd", line: 89, isLocal: false, isDefinition: false, virtualIndex: 6, isOptimized: false, file: !9, scope: !8, type: !44)
+!47 = !DISubprogram(name: "SetZ", linkageName: "_ZN9ggVector34SetZEd", line: 90, isLocal: false, isDefinition: false, virtualIndex: 6, isOptimized: false, file: !9, scope: !8, type: !44)
+!48 = !DISubprogram(name: "ggVector3", line: 92, isLocal: false, isDefinition: false, virtualIndex: 6, isOptimized: false, file: !9, scope: !8, type: !49)
+!49 = !DISubroutineType(types: !50)
 !50 = !{null, !19, !51}
-!51 = !MDDerivedType(tag: DW_TAG_reference_type, size: 32, align: 32, file: !101, scope: !4, baseType: !34)
-!52 = !MDSubprogram(name: "tolerance", linkageName: "_ZNK9ggVector39toleranceEv", line: 100, isLocal: false, isDefinition: false, virtualIndex: 6, isOptimized: false, file: !9, scope: !8, type: !31)
-!53 = !MDSubprogram(name: "tolerance", linkageName: "_ZN9ggVector39toleranceEv", line: 101, isLocal: false, isDefinition: false, virtualIndex: 6, isOptimized: false, file: !9, scope: !8, type: !38)
-!54 = !MDSubprogram(name: "operator+", linkageName: "_ZNK9ggVector3psEv", line: 107, isLocal: false, isDefinition: false, virtualIndex: 6, isOptimized: false, file: !9, scope: !8, type: !55)
-!55 = !MDSubroutineType(types: !56)
+!51 = !DIDerivedType(tag: DW_TAG_reference_type, size: 32, align: 32, file: !101, scope: !4, baseType: !34)
+!52 = !DISubprogram(name: "tolerance", linkageName: "_ZNK9ggVector39toleranceEv", line: 100, isLocal: false, isDefinition: false, virtualIndex: 6, isOptimized: false, file: !9, scope: !8, type: !31)
+!53 = !DISubprogram(name: "tolerance", linkageName: "_ZN9ggVector39toleranceEv", line: 101, isLocal: false, isDefinition: false, virtualIndex: 6, isOptimized: false, file: !9, scope: !8, type: !38)
+!54 = !DISubprogram(name: "operator+", linkageName: "_ZNK9ggVector3psEv", line: 107, isLocal: false, isDefinition: false, virtualIndex: 6, isOptimized: false, file: !9, scope: !8, type: !55)
+!55 = !DISubroutineType(types: !56)
 !56 = !{!51, !33}
-!57 = !MDSubprogram(name: "operator-", linkageName: "_ZNK9ggVector3ngEv", line: 108, isLocal: false, isDefinition: false, virtualIndex: 6, isOptimized: false, file: !9, scope: !8, type: !58)
-!58 = !MDSubroutineType(types: !59)
+!57 = !DISubprogram(name: "operator-", linkageName: "_ZNK9ggVector3ngEv", line: 108, isLocal: false, isDefinition: false, virtualIndex: 6, isOptimized: false, file: !9, scope: !8, type: !58)
+!58 = !DISubroutineType(types: !59)
 !59 = !{!8, !33}
-!60 = !MDSubprogram(name: "operator[]", linkageName: "_ZNK9ggVector3ixEi", line: 290, isLocal: false, isDefinition: false, virtualIndex: 6, isOptimized: false, file: !9, scope: !8, type: !61)
-!61 = !MDSubroutineType(types: !62)
+!60 = !DISubprogram(name: "operator[]", linkageName: "_ZNK9ggVector3ixEi", line: 290, isLocal: false, isDefinition: false, virtualIndex: 6, isOptimized: false, file: !9, scope: !8, type: !61)
+!61 = !DISubroutineType(types: !62)
 !62 = !{!13, !33, !22}
-!63 = !MDSubprogram(name: "operator[]", linkageName: "_ZN9ggVector3ixEi", line: 278, isLocal: false, isDefinition: false, virtualIndex: 6, isOptimized: false, file: !9, scope: !8, type: !64)
-!64 = !MDSubroutineType(types: !65)
+!63 = !DISubprogram(name: "operator[]", linkageName: "_ZN9ggVector3ixEi", line: 278, isLocal: false, isDefinition: false, virtualIndex: 6, isOptimized: false, file: !9, scope: !8, type: !64)
+!64 = !DISubroutineType(types: !65)
 !65 = !{!40, !19, !22}
-!66 = !MDSubprogram(name: "operator+=", linkageName: "_ZN9ggVector3pLERKS_", line: 303, isLocal: false, isDefinition: false, virtualIndex: 6, isOptimized: false, file: !9, scope: !8, type: !67)
-!67 = !MDSubroutineType(types: !68)
+!66 = !DISubprogram(name: "operator+=", linkageName: "_ZN9ggVector3pLERKS_", line: 303, isLocal: false, isDefinition: false, virtualIndex: 6, isOptimized: false, file: !9, scope: !8, type: !67)
+!67 = !DISubroutineType(types: !68)
 !68 = !{!69, !19, !51}
-!69 = !MDDerivedType(tag: DW_TAG_reference_type, name: "ggVector3", size: 32, align: 32, file: !101, scope: !4, baseType: !8)
-!70 = !MDSubprogram(name: "operator-=", linkageName: "_ZN9ggVector3mIERKS_", line: 310, isLocal: false, isDefinition: false, virtualIndex: 6, isOptimized: false, file: !9, scope: !8, type: !67)
-!71 = !MDSubprogram(name: "operator*=", linkageName: "_ZN9ggVector3mLEd", line: 317, isLocal: false, isDefinition: false, virtualIndex: 6, isOptimized: false, file: !9, scope: !8, type: !72)
-!72 = !MDSubroutineType(types: !73)
+!69 = !DIDerivedType(tag: DW_TAG_reference_type, name: "ggVector3", size: 32, align: 32, file: !101, scope: !4, baseType: !8)
+!70 = !DISubprogram(name: "operator-=", linkageName: "_ZN9ggVector3mIERKS_", line: 310, isLocal: false, isDefinition: false, virtualIndex: 6, isOptimized: false, file: !9, scope: !8, type: !67)
+!71 = !DISubprogram(name: "operator*=", linkageName: "_ZN9ggVector3mLEd", line: 317, isLocal: false, isDefinition: false, virtualIndex: 6, isOptimized: false, file: !9, scope: !8, type: !72)
+!72 = !DISubroutineType(types: !73)
 !73 = !{!69, !19, !13}
-!74 = !MDSubprogram(name: "operator/=", linkageName: "_ZN9ggVector3dVEd", line: 324, isLocal: false, isDefinition: false, virtualIndex: 6, isOptimized: false, file: !9, scope: !8, type: !72)
-!75 = !MDSubprogram(name: "length", linkageName: "_ZNK9ggVector36lengthEv", line: 121, isLocal: false, isDefinition: false, virtualIndex: 6, isOptimized: false, file: !9, scope: !8, type: !31)
-!76 = !MDSubprogram(name: "squaredLength", linkageName: "_ZNK9ggVector313squaredLengthEv", line: 122, isLocal: false, isDefinition: false, virtualIndex: 6, isOptimized: false, file: !9, scope: !8, type: !31)
-!77 = !MDSubprogram(name: "MakeUnitVector", linkageName: "_ZN9ggVector314MakeUnitVectorEv", line: 217, isLocal: false, isDefinition: true, virtualIndex: 6, isOptimized: false, file: !9, scope: !8, type: !24)
-!78 = !MDSubprogram(name: "Perturb", linkageName: "_ZNK9ggVector37PerturbEdd", line: 126, isLocal: false, isDefinition: false, virtualIndex: 6, isOptimized: false, file: !9, scope: !8, type: !79)
-!79 = !MDSubroutineType(types: !80)
+!74 = !DISubprogram(name: "operator/=", linkageName: "_ZN9ggVector3dVEd", line: 324, isLocal: false, isDefinition: false, virtualIndex: 6, isOptimized: false, file: !9, scope: !8, type: !72)
+!75 = !DISubprogram(name: "length", linkageName: "_ZNK9ggVector36lengthEv", line: 121, isLocal: false, isDefinition: false, virtualIndex: 6, isOptimized: false, file: !9, scope: !8, type: !31)
+!76 = !DISubprogram(name: "squaredLength", linkageName: "_ZNK9ggVector313squaredLengthEv", line: 122, isLocal: false, isDefinition: false, virtualIndex: 6, isOptimized: false, file: !9, scope: !8, type: !31)
+!77 = !DISubprogram(name: "MakeUnitVector", linkageName: "_ZN9ggVector314MakeUnitVectorEv", line: 217, isLocal: false, isDefinition: true, virtualIndex: 6, isOptimized: false, file: !9, scope: !8, type: !24)
+!78 = !DISubprogram(name: "Perturb", linkageName: "_ZNK9ggVector37PerturbEdd", line: 126, isLocal: false, isDefinition: false, virtualIndex: 6, isOptimized: false, file: !9, scope: !8, type: !79)
+!79 = !DISubroutineType(types: !80)
 !80 = !{!8, !33, !13, !13}
-!81 = !MDSubprogram(name: "maxComponent", linkageName: "_ZNK9ggVector312maxComponentEv", line: 128, isLocal: false, isDefinition: false, virtualIndex: 6, isOptimized: false, file: !9, scope: !8, type: !31)
-!82 = !MDSubprogram(name: "minComponent", linkageName: "_ZNK9ggVector312minComponentEv", line: 129, isLocal: false, isDefinition: false, virtualIndex: 6, isOptimized: false, file: !9, scope: !8, type: !31)
-!83 = !MDSubprogram(name: "maxAbsComponent", linkageName: "_ZNK9ggVector315maxAbsComponentEv", line: 131, isLocal: false, isDefinition: false, virtualIndex: 6, isOptimized: false, file: !9, scope: !8, type: !31)
-!84 = !MDSubprogram(name: "minAbsComponent", linkageName: "_ZNK9ggVector315minAbsComponentEv", line: 132, isLocal: false, isDefinition: false, virtualIndex: 6, isOptimized: false, file: !9, scope: !8, type: !31)
-!85 = !MDSubprogram(name: "indexOfMinComponent", linkageName: "_ZNK9ggVector319indexOfMinComponentEv", line: 133, isLocal: false, isDefinition: false, virtualIndex: 6, isOptimized: false, file: !9, scope: !8, type: !86)
-!86 = !MDSubroutineType(types: !87)
+!81 = !DISubprogram(name: "maxComponent", linkageName: "_ZNK9ggVector312maxComponentEv", line: 128, isLocal: false, isDefinition: false, virtualIndex: 6, isOptimized: false, file: !9, scope: !8, type: !31)
+!82 = !DISubprogram(name: "minComponent", linkageName: "_ZNK9ggVector312minComponentEv", line: 129, isLocal: false, isDefinition: false, virtualIndex: 6, isOptimized: false, file: !9, scope: !8, type: !31)
+!83 = !DISubprogram(name: "maxAbsComponent", linkageName: "_ZNK9ggVector315maxAbsComponentEv", line: 131, isLocal: false, isDefinition: false, virtualIndex: 6, isOptimized: false, file: !9, scope: !8, type: !31)
+!84 = !DISubprogram(name: "minAbsComponent", linkageName: "_ZNK9ggVector315minAbsComponentEv", line: 132, isLocal: false, isDefinition: false, virtualIndex: 6, isOptimized: false, file: !9, scope: !8, type: !31)
+!85 = !DISubprogram(name: "indexOfMinComponent", linkageName: "_ZNK9ggVector319indexOfMinComponentEv", line: 133, isLocal: false, isDefinition: false, virtualIndex: 6, isOptimized: false, file: !9, scope: !8, type: !86)
+!86 = !DISubroutineType(types: !87)
 !87 = !{!22, !33}
-!88 = !MDSubprogram(name: "indexOfMinAbsComponent", linkageName: "_ZNK9ggVector322indexOfMinAbsComponentEv", line: 137, isLocal: false, isDefinition: false, virtualIndex: 6, isOptimized: false, file: !9, scope: !8, type: !86)
-!89 = !MDSubprogram(name: "indexOfMaxComponent", linkageName: "_ZNK9ggVector319indexOfMaxComponentEv", line: 146, isLocal: false, isDefinition: false, virtualIndex: 6, isOptimized: false, file: !9, scope: !8, type: !86)
-!90 = !MDSubprogram(name: "indexOfMaxAbsComponent", linkageName: "_ZNK9ggVector322indexOfMaxAbsComponentEv", line: 150, isLocal: false, isDefinition: false, virtualIndex: 6, isOptimized: false, file: !9, scope: !8, type: !86)
-!91 = !MDLocalVariable(tag: DW_TAG_auto_variable, name: "vx", line: 46, scope: !1, file: !4, type: !13)
-!92 = !MDLocation(line: 48, scope: !1)
-!93 = !MDLocation(line: 218, scope: !94, inlinedAt: !96)
-!94 = distinct !MDLexicalBlock(line: 217, column: 0, file: !101, scope: !95)
-!95 = distinct !MDLexicalBlock(line: 217, column: 0, file: !101, scope: !77)
-!96 = !MDLocation(line: 51, scope: !1)
-!97 = !MDLocation(line: 227, scope: !94, inlinedAt: !96)
-!98 = !MDLocation(line: 52, scope: !1)
-!101 = !MDFile(filename: "ggEdgeDiscrepancy.cc", directory: "/Volumes/Home/grosbaj/sources/llvm-externals/speccpu2000/benchspec/CINT2000/252.eon/src")
+!88 = !DISubprogram(name: "indexOfMinAbsComponent", linkageName: "_ZNK9ggVector322indexOfMinAbsComponentEv", line: 137, isLocal: false, isDefinition: false, virtualIndex: 6, isOptimized: false, file: !9, scope: !8, type: !86)
+!89 = !DISubprogram(name: "indexOfMaxComponent", linkageName: "_ZNK9ggVector319indexOfMaxComponentEv", line: 146, isLocal: false, isDefinition: false, virtualIndex: 6, isOptimized: false, file: !9, scope: !8, type: !86)
+!90 = !DISubprogram(name: "indexOfMaxAbsComponent", linkageName: "_ZNK9ggVector322indexOfMaxAbsComponentEv", line: 150, isLocal: false, isDefinition: false, virtualIndex: 6, isOptimized: false, file: !9, scope: !8, type: !86)
+!91 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "vx", line: 46, scope: !1, file: !4, type: !13)
+!92 = !DILocation(line: 48, scope: !1)
+!93 = !DILocation(line: 218, scope: !94, inlinedAt: !96)
+!94 = distinct !DILexicalBlock(line: 217, column: 0, file: !101, scope: !95)
+!95 = distinct !DILexicalBlock(line: 217, column: 0, file: !101, scope: !77)
+!96 = !DILocation(line: 51, scope: !1)
+!97 = !DILocation(line: 227, scope: !94, inlinedAt: !96)
+!98 = !DILocation(line: 52, scope: !1)
+!101 = !DIFile(filename: "ggEdgeDiscrepancy.cc", directory: "/Volumes/Home/grosbaj/sources/llvm-externals/speccpu2000/benchspec/CINT2000/252.eon/src")
 !102 = !{}
 !103 = !{!3, !77}
 !104 = !{i32 1, !"Debug Info Version", i32 3}
diff --git a/test/CodeGen/Thumb/cortex-m0-unaligned-access.ll b/test/CodeGen/Thumb/cortex-m0-unaligned-access.ll
index cba1ca68569..1ba7cb795d1 100644
--- a/test/CodeGen/Thumb/cortex-m0-unaligned-access.ll
+++ b/test/CodeGen/Thumb/cortex-m0-unaligned-access.ll
@@ -1,4 +1,4 @@
-; RUN: llc -mtriple=thumbv6m-apple-unknown-macho < %s | FileCheck --check-prefix=V6M %s
+; RUN: llc -mtriple=thumbv6m-apple-unknown-macho -mattr=+strict-align < %s | FileCheck --check-prefix=V6M %s
 ; RUN: llc -mtriple=thumbv7m-apple-unknown-macho < %s | FileCheck --check-prefix=V7M %s
 
 define i32 @split_load(i32* %p) nounwind {
diff --git a/test/CodeGen/Thumb/pop.ll b/test/CodeGen/Thumb/pop.ll
index 3c539c69017..851f793e5ee 100644
--- a/test/CodeGen/Thumb/pop.ll
+++ b/test/CodeGen/Thumb/pop.ll
@@ -3,9 +3,9 @@
 
 define void @t(i8* %a, ...) nounwind {
 ; CHECK-LABEL:      t:
-; CHECK:      pop {r3}
+; CHECK:      pop {[[POP_REG:r[0-3]]]}
 ; CHECK-NEXT: add sp, #12
-; CHECK-NEXT: bx r3
+; CHECK-NEXT: bx [[POP_REG]]
 entry:
   %a.addr = alloca i8, i32 4
   call void @llvm.va_start(i8* %a.addr)
diff --git a/test/CodeGen/Thumb/sjljehprepare-lower-vector.ll b/test/CodeGen/Thumb/sjljehprepare-lower-vector.ll
index ab082c79ba6..605fe4627c9 100644
--- a/test/CodeGen/Thumb/sjljehprepare-lower-vector.ll
+++ b/test/CodeGen/Thumb/sjljehprepare-lower-vector.ll
@@ -4,7 +4,7 @@
 target datalayout = "e-p:32:32:32-i1:8:32-i8:8:32-i16:16:32-i32:32:32-i64:32:64-f32:32:32-f64:32:64-v64:32:64-v128:32:128-a0:0:32-n32-S32"
 target triple = "thumbv7-apple-ios"
 
-define i8* @foo(<4 x i32> %c) {
+define i8* @foo(<4 x i32> %c) personality i8* bitcast (i32 (...)* @baz to i8*) {
 entry:
   invoke void @bar ()
     to label %unreachable unwind label %handler
@@ -13,7 +13,7 @@ unreachable:
   unreachable
 
 handler:
-  %tmp = landingpad { i8*, i32 } personality i8* bitcast (i32 (...)* @baz to i8*)
+  %tmp = landingpad { i8*, i32 }
   cleanup
   resume { i8*, i32 } undef
 }
diff --git a/test/CodeGen/Thumb/thumb-shrink-wrapping.ll b/test/CodeGen/Thumb/thumb-shrink-wrapping.ll
new file mode 100644
index 00000000000..62f78258e7d
--- /dev/null
+++ b/test/CodeGen/Thumb/thumb-shrink-wrapping.ll
@@ -0,0 +1,517 @@
+; RUN: llc %s -o - -enable-shrink-wrap=true -ifcvt-fn-start=1 -ifcvt-fn-stop=0 -mtriple=thumb-macho \
+; RUN:      | FileCheck %s --check-prefix=CHECK --check-prefix=ENABLE 
+; RUN: llc %s -o - -enable-shrink-wrap=false -ifcvt-fn-start=1 -ifcvt-fn-stop=0 -mtriple=thumb-macho \
+; RUN:      | FileCheck %s --check-prefix=CHECK --check-prefix=DISABLE
+;
+; Note: Lots of tests use inline asm instead of regular calls.
+; This allows to have a better control on what the allocation will do.
+; Otherwise, we may have spill right in the entry block, defeating
+; shrink-wrapping. Moreover, some of the inline asm statements (nop)
+; are here to ensure that the related paths do not end up as critical
+; edges.
+; Also disable the late if-converter as it makes harder to reason on
+; the diffs.
+
+; Initial motivating example: Simple diamond with a call just on one side.
+; CHECK-LABEL: foo:
+;
+; Compare the arguments and jump to exit.
+; No prologue needed.
+; ENABLE: cmp r0, r1
+; ENABLE-NEXT: bge [[EXIT_LABEL:LBB[0-9_]+]]
+;
+; Prologue code.
+; CHECK: push {r7, lr}
+; CHECK-NEXT: sub sp, #8
+;
+; Compare the arguments and jump to exit.
+; After the prologue is set.
+; DISABLE: cmp r0, r1
+; DISABLE-NEXT: bge [[EXIT_LABEL:LBB[0-9_]+]]
+;
+; Store %a in the alloca.
+; CHECK: str r0, [sp, #4]
+; Set the alloca address in the second argument.
+; Set the first argument to zero.
+; CHECK: movs r0, #0
+; CHECK-NEXT: add r1, sp, #4
+; CHECK-NEXT: bl
+;
+; With shrink-wrapping, epilogue is just after the call.
+; ENABLE-NEXT: add sp, #8
+; ENABLE-NEXT: pop {r7, lr}
+;
+; CHECK: [[EXIT_LABEL]]:
+;
+; Without shrink-wrapping, epilogue is in the exit block.
+; Epilogue code. (What we pop does not matter.)
+; DISABLE: add sp, #8
+; DISABLE-NEXT: pop {r7, pc}
+;
+; ENABLE-NEXT: bx lr
+define i32 @foo(i32 %a, i32 %b) {
+  %tmp = alloca i32, align 4
+  %tmp2 = icmp slt i32 %a, %b
+  br i1 %tmp2, label %true, label %false
+
+true:
+  store i32 %a, i32* %tmp, align 4
+  %tmp4 = call i32 @doSomething(i32 0, i32* %tmp)
+  br label %false
+
+false:
+  %tmp.0 = phi i32 [ %tmp4, %true ], [ %a, %0 ]
+  ret i32 %tmp.0
+}
+
+; Function Attrs: optsize
+declare i32 @doSomething(i32, i32*)
+
+
+; Check that we do not perform the restore inside the loop whereas the save
+; is outside.
+; CHECK-LABEL: freqSaveAndRestoreOutsideLoop:
+;
+; Shrink-wrapping allows to skip the prologue in the else case.
+; ENABLE: cmp r0, #0
+; ENABLE-NEXT: beq [[ELSE_LABEL:LBB[0-9_]+]]
+;
+; Prologue code.
+; Make sure we save the CSR used in the inline asm: r4.
+; CHECK: push {r4, lr}
+;
+; DISABLE: cmp r0, #0
+; DISABLE-NEXT: beq [[ELSE_LABEL:LBB[0-9_]+]]
+;
+; SUM is in r0 because it is coalesced with the second
+; argument on the else path.
+; CHECK: movs [[SUM:r0]], #0
+; CHECK-NEXT: movs [[IV:r[0-9]+]], #10
+;
+; Next BB.
+; CHECK: [[LOOP:LBB[0-9_]+]]: @ %for.body
+; CHECK: movs [[TMP:r[0-9]+]], #1
+; CHECK: adds [[SUM]], [[TMP]], [[SUM]]
+; CHECK-NEXT: subs [[IV]], [[IV]], #1
+; CHECK-NEXT: cmp [[IV]], #0
+; CHECK-NEXT: bne [[LOOP]]
+;
+; Next BB.
+; SUM << 3.
+; CHECK: lsls [[SUM]], [[SUM]], #3
+;
+; Duplicated epilogue.
+; DISABLE: pop {r4, pc}
+;
+; CHECK: [[ELSE_LABEL]]: @ %if.else
+; Shift second argument by one and store into returned register.
+; CHECK: lsls r0, r1, #1
+; DISABLE-NEXT: pop {r4, pc}
+;
+; ENABLE-NEXT: bx lr
+define i32 @freqSaveAndRestoreOutsideLoop(i32 %cond, i32 %N) {
+entry:
+  %tobool = icmp eq i32 %cond, 0
+  br i1 %tobool, label %if.else, label %for.preheader
+
+for.preheader:
+  tail call void asm "nop", ""()
+  br label %for.body
+
+for.body:                                         ; preds = %entry, %for.body
+  %i.05 = phi i32 [ %inc, %for.body ], [ 0, %for.preheader ]
+  %sum.04 = phi i32 [ %add, %for.body ], [ 0, %for.preheader ]
+  %call = tail call i32 asm sideeffect "movs $0, #1", "=r,~{r4}"()
+  %add = add nsw i32 %call, %sum.04
+  %inc = add nuw nsw i32 %i.05, 1
+  %exitcond = icmp eq i32 %inc, 10
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:                                          ; preds = %for.body
+  %shl = shl i32 %add, 3
+  br label %if.end
+
+if.else:                                          ; preds = %entry
+  %mul = shl nsw i32 %N, 1
+  br label %if.end
+
+if.end:                                           ; preds = %if.else, %for.end
+  %sum.1 = phi i32 [ %shl, %for.end ], [ %mul, %if.else ]
+  ret i32 %sum.1
+}
+
+declare i32 @something(...)
+
+; Check that we do not perform the shrink-wrapping inside the loop even
+; though that would be legal. The cost model must prevent that.
+; CHECK-LABEL: freqSaveAndRestoreOutsideLoop2:
+; Prologue code.
+; Make sure we save the CSR used in the inline asm: r4.
+; CHECK: push {r4
+; This is the nop.
+; CHECK: mov r8, r8
+; CHECK: movs [[SUM:r0]], #0
+; CHECK-NEXT: movs [[IV:r[0-9]+]], #10
+; Next BB.
+; CHECK: [[LOOP_LABEL:LBB[0-9_]+]]: @ %for.body
+; CHECK: movs [[TMP:r[0-9]+]], #1
+; CHECK: adds [[SUM]], [[TMP]], [[SUM]]
+; CHECK-NEXT: subs [[IV]], [[IV]], #1
+; CHECK-NEXT: cmp [[IV]], #0
+; CHECK-NEXT: bne [[LOOP_LABEL]]
+; Next BB.
+; CHECK: @ %for.exit
+; This is the nop.
+; CHECK: mov r8, r8
+; CHECK: pop {r4
+define i32 @freqSaveAndRestoreOutsideLoop2(i32 %cond) {
+entry:
+  br label %for.preheader
+
+for.preheader:
+  tail call void asm "nop", ""()
+  br label %for.body
+
+for.body:                                         ; preds = %for.body, %entry
+  %i.04 = phi i32 [ 0, %for.preheader ], [ %inc, %for.body ]
+  %sum.03 = phi i32 [ 0, %for.preheader ], [ %add, %for.body ]
+  %call = tail call i32 asm sideeffect "movs $0, #1", "=r,~{r4}"()
+  %add = add nsw i32 %call, %sum.03
+  %inc = add nuw nsw i32 %i.04, 1
+  %exitcond = icmp eq i32 %inc, 10
+  br i1 %exitcond, label %for.exit, label %for.body
+
+for.exit:
+  tail call void asm "nop", ""()
+  br label %for.end
+
+for.end:                                          ; preds = %for.body
+  ret i32 %add
+}
+
+; Check with a more complex case that we do not have save within the loop and
+; restore outside.
+; CHECK-LABEL: loopInfoSaveOutsideLoop:
+;
+; ENABLE: cmp r0, #0
+; ENABLE-NEXT: beq [[ELSE_LABEL:LBB[0-9_]+]]
+;
+; Prologue code.
+; Make sure we save the CSR used in the inline asm: r4.
+; CHECK: push {r4, lr}
+;
+; DISABLE: cmp r0, #0
+; DISABLE-NEXT: beq [[ELSE_LABEL:LBB[0-9_]+]]
+;
+; SUM is in r0 because it is coalesced with the second
+; argument on the else path.
+; CHECK: movs [[SUM:r0]], #0
+; CHECK-NEXT: movs [[IV:r[0-9]+]], #10
+;
+; Next BB.
+; CHECK: [[LOOP:LBB[0-9_]+]]: @ %for.body
+; CHECK: movs [[TMP:r[0-9]+]], #1
+; CHECK: adds [[SUM]], [[TMP]], [[SUM]]
+; CHECK-NEXT: subs [[IV]], [[IV]], #1
+; CHECK-NEXT: cmp [[IV]], #0
+; CHECK-NEXT: bne [[LOOP]]
+;
+; Next BB.
+; SUM << 3.
+; CHECK: lsls [[SUM]], [[SUM]], #3
+; ENABLE-NEXT: pop {r4, lr}
+;
+; Duplicated epilogue.
+; DISABLE: pop {r4, pc}
+;
+; CHECK: [[ELSE_LABEL]]: @ %if.else
+; Shift second argument by one and store into returned register.
+; CHECK: lsls r0, r1, #1
+; DISABLE-NEXT: pop {r4, pc}
+;
+; ENABLE-NEXT: bx lr
+define i32 @loopInfoSaveOutsideLoop(i32 %cond, i32 %N) {
+entry:
+  %tobool = icmp eq i32 %cond, 0
+  br i1 %tobool, label %if.else, label %for.preheader
+
+for.preheader:
+  tail call void asm "nop", ""()
+  br label %for.body
+
+for.body:                                         ; preds = %entry, %for.body
+  %i.05 = phi i32 [ %inc, %for.body ], [ 0, %for.preheader ]
+  %sum.04 = phi i32 [ %add, %for.body ], [ 0, %for.preheader ]
+  %call = tail call i32 asm sideeffect "movs $0, #1", "=r,~{r4}"()
+  %add = add nsw i32 %call, %sum.04
+  %inc = add nuw nsw i32 %i.05, 1
+  %exitcond = icmp eq i32 %inc, 10
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:                                          ; preds = %for.body
+  tail call void asm "nop", "~{r4}"()
+  %shl = shl i32 %add, 3
+  br label %if.end
+
+if.else:                                          ; preds = %entry
+  %mul = shl nsw i32 %N, 1
+  br label %if.end
+
+if.end:                                           ; preds = %if.else, %for.end
+  %sum.1 = phi i32 [ %shl, %for.end ], [ %mul, %if.else ]
+  ret i32 %sum.1
+}
+
+declare void @somethingElse(...)
+
+; Check with a more complex case that we do not have restore within the loop and
+; save outside.
+; CHECK-LABEL: loopInfoRestoreOutsideLoop:
+;
+; ENABLE: cmp r0, #0
+; ENABLE-NEXT: beq [[ELSE_LABEL:LBB[0-9_]+]]
+;
+; Prologue code.
+; Make sure we save the CSR used in the inline asm: r4.
+; CHECK: push {r4, lr}
+;
+; DISABLE-NEXT: cmp r0, #0
+; DISABLE-NEXT: beq [[ELSE_LABEL:LBB[0-9_]+]]
+;
+; SUM is in r0 because it is coalesced with the second
+; argument on the else path.
+; CHECK: movs [[SUM:r0]], #0
+; CHECK-NEXT: movs [[IV:r[0-9]+]], #10
+;
+; Next BB.
+; CHECK: [[LOOP:LBB[0-9_]+]]: @ %for.body
+; CHECK: movs [[TMP:r[0-9]+]], #1
+; CHECK: adds [[SUM]], [[TMP]], [[SUM]]
+; CHECK-NEXT: subs [[IV]], [[IV]], #1
+; CHECK-NEXT: cmp [[IV]], #0
+; CHECK-NEXT: bne [[LOOP]]
+;
+; Next BB.
+; SUM << 3.
+; CHECK: lsls [[SUM]], [[SUM]], #3
+; ENABLE: pop {r4, lr}
+;
+; Duplicated epilogue.
+; DISABLE: pop {r4, pc}
+;
+; CHECK: [[ELSE_LABEL]]: @ %if.else
+; Shift second argument by one and store into returned register.
+; CHECK: lsls r0, r1, #1
+; DISABLE-NEXT: pop {r4, pc}
+;
+; ENABLE-NEXT: bx lr
+define i32 @loopInfoRestoreOutsideLoop(i32 %cond, i32 %N) #0 {
+entry:
+  %tobool = icmp eq i32 %cond, 0
+  br i1 %tobool, label %if.else, label %if.then
+
+if.then:                                          ; preds = %entry
+  tail call void asm "nop", "~{r4}"()
+  br label %for.body
+
+for.body:                                         ; preds = %for.body, %if.then
+  %i.05 = phi i32 [ 0, %if.then ], [ %inc, %for.body ]
+  %sum.04 = phi i32 [ 0, %if.then ], [ %add, %for.body ]
+  %call = tail call i32 asm sideeffect "movs $0, #1", "=r,~{r4}"()
+  %add = add nsw i32 %call, %sum.04
+  %inc = add nuw nsw i32 %i.05, 1
+  %exitcond = icmp eq i32 %inc, 10
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:                                          ; preds = %for.body
+  %shl = shl i32 %add, 3
+  br label %if.end
+
+if.else:                                          ; preds = %entry
+  %mul = shl nsw i32 %N, 1
+  br label %if.end
+
+if.end:                                           ; preds = %if.else, %for.end
+  %sum.1 = phi i32 [ %shl, %for.end ], [ %mul, %if.else ]
+  ret i32 %sum.1
+}
+
+; Check that we handle function with no frame information correctly.
+; CHECK-LABEL: emptyFrame:
+; CHECK: @ %entry
+; CHECK-NEXT: movs r0, #0
+; CHECK-NEXT: bx lr
+define i32 @emptyFrame() {
+entry:
+  ret i32 0
+}
+
+; Check that we handle inline asm correctly.
+; CHECK-LABEL: inlineAsm:
+;
+; ENABLE: cmp r0, #0
+; ENABLE-NEXT: beq [[ELSE_LABEL:LBB[0-9_]+]]
+;
+; Prologue code.
+; Make sure we save the CSR used in the inline asm: r4.
+; CHECK: push {r4, lr}
+;
+; DISABLE: cmp r0, #0
+; DISABLE-NEXT: beq [[ELSE_LABEL:LBB[0-9_]+]]
+;
+; CHECK: movs [[IV:r[0-9]+]], #10
+;
+; Next BB.
+; CHECK: [[LOOP:LBB[0-9_]+]]: @ %for.body
+; CHECK: movs r4, #1
+; CHECK: subs [[IV]], [[IV]], #1
+; CHECK-NEXT: cmp [[IV]], #0
+; CHECK-NEXT: bne [[LOOP]]
+;
+; Next BB.
+; CHECK: movs r0, #0
+; ENABLE-NEXT: pop {r4, lr}
+;
+; Duplicated epilogue.
+; DISABLE-NEXT: pop {r4, pc}
+;
+; CHECK: [[ELSE_LABEL]]: @ %if.else
+; Shift second argument by one and store into returned register.
+; CHECK: lsls r0, r1, #1
+; DISABLE-NEXT: pop {r4, pc}
+;
+; ENABLE-NEXT: bx lr
+define i32 @inlineAsm(i32 %cond, i32 %N) {
+entry:
+  %tobool = icmp eq i32 %cond, 0
+  br i1 %tobool, label %if.else, label %for.preheader
+
+for.preheader:
+  tail call void asm "nop", ""()
+  br label %for.body
+
+for.body:                                         ; preds = %entry, %for.body
+  %i.03 = phi i32 [ %inc, %for.body ], [ 0, %for.preheader ]
+  tail call void asm sideeffect "movs r4, #1", "~{r4}"()
+  %inc = add nuw nsw i32 %i.03, 1
+  %exitcond = icmp eq i32 %inc, 10
+  br i1 %exitcond, label %for.exit, label %for.body
+
+for.exit:
+  tail call void asm "nop", ""()
+  br label %if.end
+
+if.else:                                          ; preds = %entry
+  %mul = shl nsw i32 %N, 1
+  br label %if.end
+
+if.end:                                           ; preds = %for.body, %if.else
+  %sum.0 = phi i32 [ %mul, %if.else ], [ 0, %for.exit ]
+  ret i32 %sum.0
+}
+
+; Check that we handle calls to variadic functions correctly.
+; CHECK-LABEL: callVariadicFunc:
+;
+; ENABLE: cmp r0, #0
+; ENABLE-NEXT: beq [[ELSE_LABEL:LBB[0-9_]+]]
+;
+; Prologue code.
+; CHECK: push {[[TMP:r[0-9]+]], lr}
+; CHECK-NEXT: sub sp, #16
+;
+; DISABLE: cmp r0, #0
+; DISABLE-NEXT: beq [[ELSE_LABEL:LBB[0-9_]+]]
+;
+; Setup of the varags.
+; CHECK: mov [[TMP_SP:r[0-9]+]], sp
+; CHECK-NEXT: str r1, {{\[}}[[TMP_SP]]]
+; CHECK-NEXT: str r1, {{\[}}[[TMP_SP]], #4]
+; CHECK-NEXT: str r1, {{\[}}[[TMP_SP]], #8]
+; Thumb has quite a strange way for moving stuff
+; in around. Oh well, match the current sequence.
+; CHECK: push {r1}
+; CHECK-NEXT: pop {r0}
+; CHECK: push {r1}
+; CHECK-NEXT: pop {r2}
+; CHECK: push {r1}
+; CHECK-NEXT: pop {r3}
+; CHECK-NEXT: bl
+; CHECK-NEXT: lsls r0, r0, #3
+; CHECK-NEXT: add sp, #16
+;
+; ENABLE-NEXT: pop {[[TMP]], lr}
+;
+; Duplicated epilogue.
+; DISABLE-NEXT: pop {[[TMP]], pc}
+;
+; CHECK: [[ELSE_LABEL]]: @ %if.else
+; Shift second argument by one and store into returned register.
+; CHECK: lsls r0, r1, #1
+;
+; Epilogue code.
+; ENABLE-NEXT: bx lr
+;
+; DISABLE-NEXT: add sp, #16
+; DISABLE-NEXT: pop {[[TMP]], pc}
+define i32 @callVariadicFunc(i32 %cond, i32 %N) {
+entry:
+  %tobool = icmp eq i32 %cond, 0
+  br i1 %tobool, label %if.else, label %if.then
+
+if.then:                                          ; preds = %entry
+  %call = tail call i32 (i32, ...) @someVariadicFunc(i32 %N, i32 %N, i32 %N, i32 %N, i32 %N, i32 %N, i32 %N)
+  %shl = shl i32 %call, 3
+  br label %if.end
+
+if.else:                                          ; preds = %entry
+  %mul = shl nsw i32 %N, 1
+  br label %if.end
+
+if.end:                                           ; preds = %if.else, %if.then
+  %sum.0 = phi i32 [ %shl, %if.then ], [ %mul, %if.else ]
+  ret i32 %sum.0
+}
+
+declare i32 @someVariadicFunc(i32, ...)
+
+; Make sure we do not insert unreachable code after noreturn function.
+; Although this is not incorrect to insert such code, it is useless
+; and it hurts the binary size.
+;
+; CHECK-LABEL: noreturn:
+; DISABLE: push
+;
+; CHECK: movs [[TMP:r[0-9]+]], #255
+; CHECK-NEXT: tst  r0, [[TMP]]
+; CHECK-NEXT: bne      [[ABORT:LBB[0-9_]+]]
+;
+; CHECK: movs r0, #42
+;
+; ENABLE-NEXT: bx lr
+;
+; DISABLE-NEXT: pop
+;;
+; CHECK: [[ABORT]]: @ %if.abort
+;
+; ENABLE: push
+;
+; CHECK: bl
+; ENABLE-NOT: pop
+define i32 @noreturn(i8 signext %bad_thing) {
+entry:
+  %tobool = icmp eq i8 %bad_thing, 0
+  br i1 %tobool, label %if.end, label %if.abort
+
+if.abort:
+  %call = tail call i32 asm sideeffect "movs $0, #1", "=r,~{r4}"()
+  tail call void @abort() #0
+  unreachable
+
+if.end:
+  ret i32 42
+}
+
+declare void @abort() #0
+
+attributes #0 = { noreturn nounwind }
diff --git a/test/CodeGen/Thumb/vargs.ll b/test/CodeGen/Thumb/vargs.ll
index 1c7b631741b..45b42125e16 100644
--- a/test/CodeGen/Thumb/vargs.ll
+++ b/test/CodeGen/Thumb/vargs.ll
@@ -32,12 +32,12 @@ bb7:            ; preds = %bb
         call void @llvm.va_end( i8* %va.upgrd.4 )
         ret void
 
-; The return sequence should pop the lr to r3, recover the stack space used to
+; The return sequence should pop the lr to r0-3, recover the stack space used to
 ; store variadic argument registers, then return via r3. Possibly there is a pop
 ; before this, but only if the function happened to use callee-saved registers.
-; CHECK: pop {r3}
+; CHECK: pop {[[POP_REG:r[0-3]]]}
 ; CHECK: add sp, #[[IMM]]
-; CHECK: bx r3
+; CHECK: bx [[POP_REG]]
 }
 
 declare void @llvm.va_start(i8*)
diff --git a/test/CodeGen/Thumb2/2009-08-06-SpDecBug.ll b/test/CodeGen/Thumb2/2009-08-06-SpDecBug.ll
index 59c23673211..a451321838e 100644
--- a/test/CodeGen/Thumb2/2009-08-06-SpDecBug.ll
+++ b/test/CodeGen/Thumb2/2009-08-06-SpDecBug.ll
@@ -7,8 +7,8 @@ entry:
 ; CHECK-LABEL: __gcov_execlp:
 ; CHECK: sub sp, #8
 ; CHECK: push
-; CHECK: add r7, sp, #4
-; CHECK: sub.w r4, r7, #4
+; CHECK: add r7, sp, #8
+; CHECK: sub.w r4, r7, #8
 ; CHECK: mov sp, r4
 ; CHECK-NOT: mov sp, r7
 ; CHECK: add sp, #8
diff --git a/test/CodeGen/Thumb2/2011-06-07-TwoAddrEarlyClobber.ll b/test/CodeGen/Thumb2/2011-06-07-TwoAddrEarlyClobber.ll
index fc7d0e1504a..b9bfdcbec4e 100644
--- a/test/CodeGen/Thumb2/2011-06-07-TwoAddrEarlyClobber.ll
+++ b/test/CodeGen/Thumb2/2011-06-07-TwoAddrEarlyClobber.ll
@@ -3,9 +3,9 @@
 %struct.op = type { %struct.op*, %struct.op*, %struct.op* ()*, i32, i16, i16, i8, i8 }
 
 ; CHECK: Perl_ck_sort
-; CHECK: ldreq
-; CHECK: moveq [[REGISTER:(r[0-9]+)|(lr)]]
-; CHECK: streq {{(r[0-9])|(lr)}}, {{\[}}[[REGISTER]]{{\]}}, #24
+; CHECK: ldr
+; CHECK: mov [[REGISTER:(r[0-9]+)|(lr)]]
+; CHECK: str {{(r[0-9])|(lr)}}, {{\[}}[[REGISTER]]{{\]}}, #24
 
 define void @Perl_ck_sort() nounwind optsize {
 entry:
diff --git a/test/CodeGen/Thumb2/aapcs.ll b/test/CodeGen/Thumb2/aapcs.ll
index 21af8c119b0..299562fe4c5 100644
--- a/test/CodeGen/Thumb2/aapcs.ll
+++ b/test/CodeGen/Thumb2/aapcs.ll
@@ -33,8 +33,7 @@ define float @float_on_stack(double %a, double %b, double %c, double %d, double
 
 define double @double_on_stack(double %a, double %b, double %c, double %d, double %e, double %f, double %g, double %h, double %i) {
 ; CHECK-LABEL: double_on_stack:
-; SOFT: ldr r0, [sp, #48]
-; SOFT: ldr r1, [sp, #52]
+; SOFT: ldrd r0, r1, [sp, #48]
 ; HARD: vldr d0, [sp]
 ; CHECK-NEXT: bx lr
   ret double %i
@@ -42,8 +41,7 @@ define double @double_on_stack(double %a, double %b, double %c, double %d, doubl
 
 define double @double_not_split(double %a, double %b, double %c, double %d, double %e, double %f, double %g, float %h, double %i) {
 ; CHECK-LABEL: double_not_split:
-; SOFT: ldr r0, [sp, #48]
-; SOFT: ldr r1, [sp, #52]
+; SOFT: ldrd r0, r1, [sp, #48]
 ; HARD: vldr d0, [sp]
 ; CHECK-NEXT: bx lr
   ret double %i
diff --git a/test/CodeGen/Thumb2/cbnz.ll b/test/CodeGen/Thumb2/cbnz.ll
new file mode 100644
index 00000000000..5c0bb5bfe1c
--- /dev/null
+++ b/test/CodeGen/Thumb2/cbnz.ll
@@ -0,0 +1,54 @@
+; RUN: llc -mtriple thumbv7-unknown-linux -o - %s | FileCheck %s
+
+declare void @x()
+declare void @y()
+
+define void @f(i32 %x, i32 %y) {
+  ; CHECK-LABEL: f:
+  ; CHECK: cbnz
+  %p = icmp eq i32 %x, 0
+  br i1 %p, label %t, label %f
+
+t:
+  call void @x()
+  call void @x()
+  call void @x()
+  call void @x()
+  call void @x()
+  call void @x()
+  call void @x()
+  call void @x()
+  call void @x()
+  call void @x()
+  call void @x()
+  call void @x()
+  call void @x()
+  call void @x()
+  call void @x()
+  call void @x()
+  ; CHECK: cbnz
+  %q = icmp eq i32 %y, 0
+  br i1 %q, label %t2, label %f
+
+t2:
+  call void @x()
+  call void @x()
+  call void @x()
+  call void @x()
+  call void @x()
+  call void @x()
+  call void @x()
+  call void @x()
+  call void @x()
+  call void @x()
+  call void @x()
+  call void @x()
+  call void @x()
+  call void @x()
+  call void @x()
+  br label %f
+
+f:
+  call void @y()
+  ret void
+}
diff --git a/test/CodeGen/Thumb2/constant-islands-jump-table.ll b/test/CodeGen/Thumb2/constant-islands-jump-table.ll
index 0dd7092291b..5ffe1f9b09f 100644
--- a/test/CodeGen/Thumb2/constant-islands-jump-table.ll
+++ b/test/CodeGen/Thumb2/constant-islands-jump-table.ll
@@ -1,7 +1,7 @@
 ; RUN: llc < %s -mtriple=thumbv7-linux-gnueabihf -O1 %s -o - | FileCheck %s
 
 ; CHECK-LABEL: test_jump_table:
-; CHECK: b .LBB
+; CHECK: b{{.*}} .LBB
 ; CHECK-NOT: tbh
 
 define i32 @test_jump_table(i32 %x, float %in) {
diff --git a/test/CodeGen/Thumb2/constant-islands.ll b/test/CodeGen/Thumb2/constant-islands.ll
index a64d72e86ef..583849195e6 100644
--- a/test/CodeGen/Thumb2/constant-islands.ll
+++ b/test/CodeGen/Thumb2/constant-islands.ll
@@ -76,7 +76,7 @@ declare %class.btCapsuleShape* @_ZN14btCapsuleShapeC1Eff(%class.btCapsuleShape*,
 
 declare %class.btMatrix3x3* @_ZN11btTransform8getBasisEv(%class.btTransform*) nounwind inlinehint ssp align 2
 
-define %class.RagDoll* @_ZN7RagDollC2EP15btDynamicsWorldRK9btVector3f(%class.RagDoll* %this, %class.btDynamicsWorld* %ownerWorld, %class.btVector3* %positionOffset, float %scale) unnamed_addr ssp align 2 {
+define %class.RagDoll* @_ZN7RagDollC2EP15btDynamicsWorldRK9btVector3f(%class.RagDoll* %this, %class.btDynamicsWorld* %ownerWorld, %class.btVector3* %positionOffset, float %scale) unnamed_addr ssp align 2 personality i8* bitcast (i32 (...)* @__gxx_personality_sj0 to i8*) {
 entry:
   %retval = alloca %class.RagDoll*, align 4
   %this.addr = alloca %class.RagDoll*, align 4
@@ -635,7 +635,7 @@ for.inc:                                          ; preds = %for.body
   br label %for.cond
 
 lpad:                                             ; preds = %entry
-  %67 = landingpad { i8*, i32 } personality i8* bitcast (i32 (...)* @__gxx_personality_sj0 to i8*)
+  %67 = landingpad { i8*, i32 }
           cleanup
   %68 = extractvalue { i8*, i32 } %67, 0
   store i8* %68, i8** %exn.slot
@@ -648,7 +648,7 @@ invoke.cont4:                                     ; preds = %lpad
   br label %eh.resume
 
 lpad8:                                            ; preds = %invoke.cont
-  %70 = landingpad { i8*, i32 } personality i8* bitcast (i32 (...)* @__gxx_personality_sj0 to i8*)
+  %70 = landingpad { i8*, i32 }
           cleanup
   %71 = extractvalue { i8*, i32 } %70, 0
   store i8* %71, i8** %exn.slot
@@ -661,7 +661,7 @@ invoke.cont11:                                    ; preds = %lpad8
   br label %eh.resume
 
 lpad17:                                           ; preds = %invoke.cont9
-  %73 = landingpad { i8*, i32 } personality i8* bitcast (i32 (...)* @__gxx_personality_sj0 to i8*)
+  %73 = landingpad { i8*, i32 }
           cleanup
   %74 = extractvalue { i8*, i32 } %73, 0
   store i8* %74, i8** %exn.slot
@@ -674,7 +674,7 @@ invoke.cont20:                                    ; preds = %lpad17
   br label %eh.resume
 
 lpad26:                                           ; preds = %invoke.cont18
-  %76 = landingpad { i8*, i32 } personality i8* bitcast (i32 (...)* @__gxx_personality_sj0 to i8*)
+  %76 = landingpad { i8*, i32 }
           cleanup
   %77 = extractvalue { i8*, i32 } %76, 0
   store i8* %77, i8** %exn.slot
@@ -687,7 +687,7 @@ invoke.cont29:                                    ; preds = %lpad26
   br label %eh.resume
 
 lpad35:                                           ; preds = %invoke.cont27
-  %79 = landingpad { i8*, i32 } personality i8* bitcast (i32 (...)* @__gxx_personality_sj0 to i8*)
+  %79 = landingpad { i8*, i32 }
           cleanup
   %80 = extractvalue { i8*, i32 } %79, 0
   store i8* %80, i8** %exn.slot
@@ -700,7 +700,7 @@ invoke.cont38:                                    ; preds = %lpad35
   br label %eh.resume
 
 lpad44:                                           ; preds = %invoke.cont36
-  %82 = landingpad { i8*, i32 } personality i8* bitcast (i32 (...)* @__gxx_personality_sj0 to i8*)
+  %82 = landingpad { i8*, i32 }
           cleanup
   %83 = extractvalue { i8*, i32 } %82, 0
   store i8* %83, i8** %exn.slot
@@ -713,7 +713,7 @@ invoke.cont47:                                    ; preds = %lpad44
   br label %eh.resume
 
 lpad53:                                           ; preds = %invoke.cont45
-  %85 = landingpad { i8*, i32 } personality i8* bitcast (i32 (...)* @__gxx_personality_sj0 to i8*)
+  %85 = landingpad { i8*, i32 }
           cleanup
   %86 = extractvalue { i8*, i32 } %85, 0
   store i8* %86, i8** %exn.slot
@@ -726,7 +726,7 @@ invoke.cont56:                                    ; preds = %lpad53
   br label %eh.resume
 
 lpad62:                                           ; preds = %invoke.cont54
-  %88 = landingpad { i8*, i32 } personality i8* bitcast (i32 (...)* @__gxx_personality_sj0 to i8*)
+  %88 = landingpad { i8*, i32 }
           cleanup
   %89 = extractvalue { i8*, i32 } %88, 0
   store i8* %89, i8** %exn.slot
@@ -739,7 +739,7 @@ invoke.cont65:                                    ; preds = %lpad62
   br label %eh.resume
 
 lpad71:                                           ; preds = %invoke.cont63
-  %91 = landingpad { i8*, i32 } personality i8* bitcast (i32 (...)* @__gxx_personality_sj0 to i8*)
+  %91 = landingpad { i8*, i32 }
           cleanup
   %92 = extractvalue { i8*, i32 } %91, 0
   store i8* %92, i8** %exn.slot
@@ -752,7 +752,7 @@ invoke.cont74:                                    ; preds = %lpad71
   br label %eh.resume
 
 lpad80:                                           ; preds = %invoke.cont72
-  %94 = landingpad { i8*, i32 } personality i8* bitcast (i32 (...)* @__gxx_personality_sj0 to i8*)
+  %94 = landingpad { i8*, i32 }
           cleanup
   %95 = extractvalue { i8*, i32 } %94, 0
   store i8* %95, i8** %exn.slot
@@ -765,7 +765,7 @@ invoke.cont83:                                    ; preds = %lpad80
   br label %eh.resume
 
 lpad89:                                           ; preds = %invoke.cont81
-  %97 = landingpad { i8*, i32 } personality i8* bitcast (i32 (...)* @__gxx_personality_sj0 to i8*)
+  %97 = landingpad { i8*, i32 }
           cleanup
   %98 = extractvalue { i8*, i32 } %97, 0
   store i8* %98, i8** %exn.slot
@@ -1264,7 +1264,7 @@ invoke.cont517:                                   ; preds = %invoke.cont488
   ret %class.RagDoll* %200
 
 lpad258:                                          ; preds = %for.end
-  %201 = landingpad { i8*, i32 } personality i8* bitcast (i32 (...)* @__gxx_personality_sj0 to i8*)
+  %201 = landingpad { i8*, i32 }
           cleanup
   %202 = extractvalue { i8*, i32 } %201, 0
   store i8* %202, i8** %exn.slot
@@ -1274,7 +1274,7 @@ lpad258:                                          ; preds = %for.end
   br label %eh.resume
 
 lpad284:                                          ; preds = %invoke.cont259
-  %204 = landingpad { i8*, i32 } personality i8* bitcast (i32 (...)* @__gxx_personality_sj0 to i8*)
+  %204 = landingpad { i8*, i32 }
           cleanup
   %205 = extractvalue { i8*, i32 } %204, 0
   store i8* %205, i8** %exn.slot
@@ -1284,7 +1284,7 @@ lpad284:                                          ; preds = %invoke.cont259
   br label %eh.resume
 
 lpad313:                                          ; preds = %invoke.cont285
-  %207 = landingpad { i8*, i32 } personality i8* bitcast (i32 (...)* @__gxx_personality_sj0 to i8*)
+  %207 = landingpad { i8*, i32 }
           cleanup
   %208 = extractvalue { i8*, i32 } %207, 0
   store i8* %208, i8** %exn.slot
@@ -1294,7 +1294,7 @@ lpad313:                                          ; preds = %invoke.cont285
   br label %eh.resume
 
 lpad342:                                          ; preds = %invoke.cont314
-  %210 = landingpad { i8*, i32 } personality i8* bitcast (i32 (...)* @__gxx_personality_sj0 to i8*)
+  %210 = landingpad { i8*, i32 }
           cleanup
   %211 = extractvalue { i8*, i32 } %210, 0
   store i8* %211, i8** %exn.slot
@@ -1304,7 +1304,7 @@ lpad342:                                          ; preds = %invoke.cont314
   br label %eh.resume
 
 lpad371:                                          ; preds = %invoke.cont343
-  %213 = landingpad { i8*, i32 } personality i8* bitcast (i32 (...)* @__gxx_personality_sj0 to i8*)
+  %213 = landingpad { i8*, i32 }
           cleanup
   %214 = extractvalue { i8*, i32 } %213, 0
   store i8* %214, i8** %exn.slot
@@ -1314,7 +1314,7 @@ lpad371:                                          ; preds = %invoke.cont343
   br label %eh.resume
 
 lpad400:                                          ; preds = %invoke.cont372
-  %216 = landingpad { i8*, i32 } personality i8* bitcast (i32 (...)* @__gxx_personality_sj0 to i8*)
+  %216 = landingpad { i8*, i32 }
           cleanup
   %217 = extractvalue { i8*, i32 } %216, 0
   store i8* %217, i8** %exn.slot
@@ -1324,7 +1324,7 @@ lpad400:                                          ; preds = %invoke.cont372
   br label %eh.resume
 
 lpad429:                                          ; preds = %invoke.cont401
-  %219 = landingpad { i8*, i32 } personality i8* bitcast (i32 (...)* @__gxx_personality_sj0 to i8*)
+  %219 = landingpad { i8*, i32 }
           cleanup
   %220 = extractvalue { i8*, i32 } %219, 0
   store i8* %220, i8** %exn.slot
@@ -1334,7 +1334,7 @@ lpad429:                                          ; preds = %invoke.cont401
   br label %eh.resume
 
 lpad458:                                          ; preds = %invoke.cont430
-  %222 = landingpad { i8*, i32 } personality i8* bitcast (i32 (...)* @__gxx_personality_sj0 to i8*)
+  %222 = landingpad { i8*, i32 }
           cleanup
   %223 = extractvalue { i8*, i32 } %222, 0
   store i8* %223, i8** %exn.slot
@@ -1344,7 +1344,7 @@ lpad458:                                          ; preds = %invoke.cont430
   br label %eh.resume
 
 lpad487:                                          ; preds = %invoke.cont459
-  %225 = landingpad { i8*, i32 } personality i8* bitcast (i32 (...)* @__gxx_personality_sj0 to i8*)
+  %225 = landingpad { i8*, i32 }
           cleanup
   %226 = extractvalue { i8*, i32 } %225, 0
   store i8* %226, i8** %exn.slot
@@ -1354,7 +1354,7 @@ lpad487:                                          ; preds = %invoke.cont459
   br label %eh.resume
 
 lpad516:                                          ; preds = %invoke.cont488
-  %228 = landingpad { i8*, i32 } personality i8* bitcast (i32 (...)* @__gxx_personality_sj0 to i8*)
+  %228 = landingpad { i8*, i32 }
           cleanup
   %229 = extractvalue { i8*, i32 } %228, 0
   store i8* %229, i8** %exn.slot
@@ -1371,7 +1371,7 @@ eh.resume:                                        ; preds = %lpad516, %lpad487,
   resume { i8*, i32 } %lpad.val526
 
 terminate.lpad:                                   ; preds = %lpad89, %lpad80, %lpad71, %lpad62, %lpad53, %lpad44, %lpad35, %lpad26, %lpad17, %lpad8, %lpad
-  %231 = landingpad { i8*, i32 } personality i8* bitcast (i32 (...)* @__gxx_personality_sj0 to i8*)
+  %231 = landingpad { i8*, i32 }
           catch i8* null
   call void @_ZSt9terminatev() noreturn nounwind
   unreachable
diff --git a/test/CodeGen/Thumb2/float-cmp.ll b/test/CodeGen/Thumb2/float-cmp.ll
index 88d6c3b0adb..77b0999337c 100644
--- a/test/CodeGen/Thumb2/float-cmp.ll
+++ b/test/CodeGen/Thumb2/float-cmp.ll
@@ -81,8 +81,9 @@ define i1 @cmp_f_ord(float %a, float %b) {
 }
 define i1 @cmp_f_ugt(float %a, float %b) {
 ; CHECK-LABEL: cmp_f_ugt:
-; NONE: bl __aeabi_fcmpgt
-; NONE: bl __aeabi_fcmpun
+; NONE: bl __aeabi_fcmple
+; NONE: cmp r0, #0
+; NONE-NEXT: it eq
 ; HARD: vcmpe.f32
 ; HARD: movhi r0, #1
   %1 = fcmp ugt float %a, %b
@@ -90,8 +91,9 @@ define i1 @cmp_f_ugt(float %a, float %b) {
 }
 define i1 @cmp_f_uge(float %a, float %b) {
 ; CHECK-LABEL: cmp_f_uge:
-; NONE: bl __aeabi_fcmpge
-; NONE: bl __aeabi_fcmpun
+; NONE: bl __aeabi_fcmplt
+; NONE: cmp r0, #0
+; NONE-NEXT: it eq
 ; HARD: vcmpe.f32
 ; HARD: movpl r0, #1
   %1 = fcmp uge float %a, %b
@@ -99,8 +101,9 @@ define i1 @cmp_f_uge(float %a, float %b) {
 }
 define i1 @cmp_f_ult(float %a, float %b) {
 ; CHECK-LABEL: cmp_f_ult:
-; NONE: bl __aeabi_fcmplt
-; NONE: bl __aeabi_fcmpun
+; NONE: bl __aeabi_fcmpge
+; NONE: cmp r0, #0
+; NONE-NEXT: it eq
 ; HARD: vcmpe.f32
 ; HARD: movlt r0, #1
   %1 = fcmp ult float %a, %b
@@ -108,8 +111,9 @@ define i1 @cmp_f_ult(float %a, float %b) {
 }
 define i1 @cmp_f_ule(float %a, float %b) {
 ; CHECK-LABEL: cmp_f_ule:
-; NONE: bl __aeabi_fcmple
-; NONE: bl __aeabi_fcmpun
+; NONE: bl __aeabi_fcmpgt
+; NONE: cmp r0, #0
+; NONE-NEXT: it eq
 ; HARD: vcmpe.f32
 ; HARD: movle r0, #1
   %1 = fcmp ule float %a, %b
@@ -214,10 +218,8 @@ define i1 @cmp_d_ord(double %a, double %b) {
 }
 define i1 @cmp_d_ugt(double %a, double %b) {
 ; CHECK-LABEL: cmp_d_ugt:
-; NONE: bl __aeabi_dcmpgt
-; NONE: bl __aeabi_dcmpun
-; SP: bl __aeabi_dcmpgt
-; SP: bl __aeabi_dcmpun
+; NONE: bl __aeabi_dcmple
+; SP: bl __aeabi_dcmple
 ; DP: vcmpe.f64
 ; DP: movhi r0, #1
   %1 = fcmp ugt double %a, %b
@@ -226,10 +228,8 @@ define i1 @cmp_d_ugt(double %a, double %b) {
 
 define i1 @cmp_d_ult(double %a, double %b) {
 ; CHECK-LABEL: cmp_d_ult:
-; NONE: bl __aeabi_dcmplt
-; NONE: bl __aeabi_dcmpun
-; SP: bl __aeabi_dcmplt
-; SP: bl __aeabi_dcmpun
+; NONE: bl __aeabi_dcmpge
+; SP: bl __aeabi_dcmpge
 ; DP: vcmpe.f64
 ; DP: movlt r0, #1
   %1 = fcmp ult double %a, %b
@@ -268,10 +268,8 @@ define i1 @cmp_d_ueq(double %a, double %b) {
 
 define i1 @cmp_d_uge(double %a, double %b) {
 ; CHECK-LABEL: cmp_d_uge:
-; NONE: bl __aeabi_dcmpge
-; NONE: bl __aeabi_dcmpun
-; SP: bl __aeabi_dcmpge
-; SP: bl __aeabi_dcmpun
+; NONE: bl __aeabi_dcmplt
+; SP: bl __aeabi_dcmplt
 ; DP: vcmpe.f64
 ; DP: movpl r0, #1
   %1 = fcmp uge double %a, %b
@@ -280,10 +278,8 @@ define i1 @cmp_d_uge(double %a, double %b) {
 
 define i1 @cmp_d_ule(double %a, double %b) {
 ; CHECK-LABEL: cmp_d_ule:
-; NONE: bl __aeabi_dcmple
-; NONE: bl __aeabi_dcmpun
-; SP: bl __aeabi_dcmple
-; SP: bl __aeabi_dcmpun
+; NONE: bl __aeabi_dcmpgt
+; SP: bl __aeabi_dcmpgt
 ; DP: vcmpe.f64
 ; DP: movle r0, #1
   %1 = fcmp ule double %a, %b
diff --git a/test/CodeGen/Thumb2/float-intrinsics-double.ll b/test/CodeGen/Thumb2/float-intrinsics-double.ll
index 01a23bd0fe6..38d04d30b87 100644
--- a/test/CodeGen/Thumb2/float-intrinsics-double.ll
+++ b/test/CodeGen/Thumb2/float-intrinsics-double.ll
@@ -109,9 +109,8 @@ declare double     @llvm.fabs.f64(double %Val)
 define double @abs_d(double %a) {
 ; CHECK-LABEL: abs_d:
 ; NONE: bic r1, r1, #-2147483648
-; SP: bl __aeabi_dcmpgt
-; SP: bl __aeabi_dcmpun
 ; SP: bl __aeabi_dsub
+; SP: bl __aeabi_dcmple
 ; DP: vabs.f64 d0, d0
   %1 = call double @llvm.fabs.f64(double %a)
   ret double %1
diff --git a/test/CodeGen/Thumb2/float-ops.ll b/test/CodeGen/Thumb2/float-ops.ll
index e0396e60df4..c9f93f2d613 100644
--- a/test/CodeGen/Thumb2/float-ops.ll
+++ b/test/CodeGen/Thumb2/float-ops.ll
@@ -109,7 +109,7 @@ entry:
 define double @load_d(double* %a) {
 entry:
 ; CHECK-LABEL: load_d:
-; NONE: ldm.w r0, {r0, r1}
+; NONE: ldm r0, {r0, r1}
 ; HARD: vldr d0, [r0]
   %0 = load double, double* %a, align 8
   ret double %0
@@ -127,9 +127,7 @@ entry:
 define void @store_d(double* %a, double %b) {
 entry:
 ; CHECK-LABEL: store_d:
-; NONE: mov r1, r3
-; NONE: str r2, [r0]
-; NONE: str r1, [r0, #4]
+; NONE: strd r2, r3, [r0]
 ; HARD: vstr d0, [r0]
   store double %b, double* %a, align 8
   ret void
diff --git a/test/CodeGen/Thumb2/ifcvt-compare.ll b/test/CodeGen/Thumb2/ifcvt-compare.ll
new file mode 100644
index 00000000000..8af139a5ef6
--- /dev/null
+++ b/test/CodeGen/Thumb2/ifcvt-compare.ll
@@ -0,0 +1,47 @@
+; RUN: llc -mtriple=thumbv7-unknown-linux %s -o - | FileCheck %s
+
+declare void @x()
+
+define void @f0(i32 %x) optsize {
+  ; CHECK-LABEL: f0:
+  ; CHECK: cbnz
+  %p = icmp eq i32 %x, 0
+  br i1 %p, label %t, label %f
+
+t:
+  call void @x()
+  br label %f
+
+f:
+  ret void
+}
+
+define void @f1(i32 %x) optsize {
+  ; CHECK-LABEL: f1:
+  ; CHECK: cmp r0, #1
+  ; CHECK: it eq
+  %p = icmp eq i32 %x, 1
+  br i1 %p, label %t, label %f
+
+t:
+  call void @x()
+  br label %f
+
+f:
+  ret void
+}
+
+define void @f2(i32 %x) {
+  ; CHECK-LABEL: f2:
+  ; CHECK: cmp r0, #0
+  ; CHECK: it eq
+  %p = icmp eq i32 %x, 0
+  br i1 %p, label %t, label %f
+
+t:
+  call void @x()
+  br label %f
+
+f:
+  ret void
+}
diff --git a/test/CodeGen/Thumb2/large-stack.ll b/test/CodeGen/Thumb2/large-stack.ll
index ff9e0501970..4fe49825fa3 100644
--- a/test/CodeGen/Thumb2/large-stack.ll
+++ b/test/CodeGen/Thumb2/large-stack.ll
@@ -29,7 +29,7 @@ define i32 @test3() {
 ; DARWIN: sub.w sp, sp, #805306368
 ; DARWIN: sub sp, #20
 ; LINUX-LABEL: test3:
-; LINUX: push.w {r4, r7, r11, lr}
+; LINUX: push {r4, r6, r7, lr}
 ; LINUX: sub.w sp, sp, #805306368
 ; LINUX: sub sp, #16
     %retval = alloca i32, align 4
diff --git a/test/CodeGen/Thumb2/pic-load.ll b/test/CodeGen/Thumb2/pic-load.ll
index 53d456c5345..cfdad03dcd5 100644
--- a/test/CodeGen/Thumb2/pic-load.ll
+++ b/test/CodeGen/Thumb2/pic-load.ll
@@ -1,4 +1,5 @@
-; RUN: llc < %s -mtriple=thumbv7-apple-darwin9 -relocation-model=pic | FileCheck %s
+; RUN: llc < %s -mtriple=thumbv7-apple-darwin9 -relocation-model=pic | FileCheck %s --check-prefix=CHECK --check-prefix=PIC
+; RUN: llc < %s -mtriple=thumbv7-apple-ios -relocation-model=pic -mcpu=swift -mattr=+no-movt | FileCheck %s --check-prefix=CHECK --check-prefix=PIC-NOMOVT
 
 	%struct.anon = type { void ()* }
 	%struct.one_atexit_routine = type { %struct.anon, i32, i8* }
@@ -8,7 +9,14 @@
 define hidden i32 @atexit(void ()* %func) nounwind {
 entry:
 ; CHECK-LABEL: atexit:
-; CHECK: add r0, pc
+; CHECK-PIC: add r0, pc
+; CHECK-NOMOVT: ldr r[[REGNUM:[0-9]+]], LCPI0_0
+; CHECK-NOMOVT: LPC0_0:
+; CHECK-NOMOVT: add r[[REGNUM]], pc
+; CHECK-NOMOVT: ldr r1, [r[[REGNUM]]
+; CHECK-NOMOVT: blx _atexit_common
+; CHECK-NOMOVT: LCPI0_0:
+; CHECK-NOMOVT: .long L___dso_handle$non_lazy_ptr-(LPC0_0+4)
 	%r = alloca %struct.one_atexit_routine, align 4		; <%struct.one_atexit_routine*> [#uses=3]
 	%0 = getelementptr %struct.one_atexit_routine, %struct.one_atexit_routine* %r, i32 0, i32 0, i32 0		; <void ()**> [#uses=1]
 	store void ()* %func, void ()** %0, align 4
diff --git a/test/CodeGen/Thumb2/setjmp_longjmp.ll b/test/CodeGen/Thumb2/setjmp_longjmp.ll
new file mode 100644
index 00000000000..9e0fad00c14
--- /dev/null
+++ b/test/CodeGen/Thumb2/setjmp_longjmp.ll
@@ -0,0 +1,89 @@
+; RUN: llc %s -o - | FileCheck %s
+target triple = "thumbv7-apple-ios"
+
+declare i32 @llvm.eh.sjlj.setjmp(i8*)
+declare void @llvm.eh.sjlj.longjmp(i8*)
+declare i8* @llvm.frameaddress(i32)
+declare i8* @llvm.stacksave()
+@g = external global i32
+
+; CHECK-LABEL: double_foobar
+;
+; setjmp sequence:
+; CHECK: mov [[PCREG:r[0-9]+]], pc
+; CHECK-NEXT: adds [[PCREG]], [[PCREG]], #7
+; CHECK-NEXT: str [[PCREG]], {{\[}}[[BUFREG:r[0-9]+]], #4]
+; CHECK-NEXT: movs r0, #0
+; CHECK-NEXT: b [[LABEL:L[a-zA-Z0-9]+]]
+; CHECK-NEXT: movs r0, #1
+; CHECK-NEXT: [[LABEL]]:
+;
+; setjmp sequence 2:
+; CHECK: mov [[PCREG:r[0-9]+]], pc
+; CHECK-NEXT: adds [[PCREG]], [[PCREG]], #7
+; CHECK-NEXT: str [[PCREG]], {{\[}}[[BUFREG:r[0-9]+]], #4]
+; CHECK-NEXT: movs r0, #0
+; CHECK-NEXT: b [[LABEL:L[a-zA-Z0-9]+]]
+; CHECK-NEXT: movs r0, #1
+; CHECK-NEXT: [[LABEL]]:
+
+; longjmp sequence:
+; CHECK: ldr [[TEMPREG:r[0-9]+]], [{{\s*}}[[BUFREG:r[0-9]+]], #8]
+; CHECK-NEXT: mov sp, [[TEMPREG]]
+; CHECK-NEXT: ldr [[DESTREG:r[0-9]+]], {{\[}}[[BUFREG]], #4]
+; CHECK-NEXT: ldr r7, {{\[}}[[BUFREG]]{{\]}}
+; CHECK-NEXT: bx [[DESTREG]]
+;
+; longjmp sequence2:
+; CHECK: ldr [[TEMPREG:r[0-9]+]], [{{\s*}}[[BUFREG:r[0-9]+]], #8]
+; CHECK-NEXT: mov sp, [[TEMPREG]]
+; CHECK-NEXT: ldr [[DESTREG:r[0-9]+]], {{\[}}[[BUFREG]], #4]
+; CHECK-NEXT: ldr r7, {{\[}}[[BUFREG]]{{\]}}
+; CHECK-NEXT: bx [[DESTREG]]
+define void @double_foobar() {
+entry:
+  %buf = alloca [5 x i8*], align 4
+  %bufptr = bitcast [5 x i8*]* %buf to i8*
+  %arraydecay = getelementptr inbounds [5 x i8*], [5 x i8*]* %buf, i32 0, i32 0
+
+  %fa = tail call i8* @llvm.frameaddress(i32 0)
+  store i8* %fa, i8** %arraydecay, align 4
+  %ss = tail call i8* @llvm.stacksave()
+  %ssgep = getelementptr [5 x i8*], [5 x i8*]* %buf, i32 0, i32 2
+  store i8* %ss, i8** %ssgep, align 4
+
+  %setjmpres = call i32 @llvm.eh.sjlj.setjmp(i8* %bufptr)
+  %tobool = icmp ne i32 %setjmpres, 0
+  br i1 %tobool, label %if.then, label %if.else
+
+if.then:
+  store volatile i32 1, i32* @g, align 4
+  br label %if.end
+
+if.else:
+  store volatile i32 0, i32* @g, align 4
+  call void @llvm.eh.sjlj.longjmp(i8* %bufptr)
+  unreachable
+
+if.end:
+  %fa2 = tail call i8* @llvm.frameaddress(i32 0)
+  store i8* %fa2, i8** %arraydecay, align 4
+  %ss2 = tail call i8* @llvm.stacksave()
+  store i8* %ss2, i8** %ssgep, align 4
+
+  %setjmpres2 = call i32 @llvm.eh.sjlj.setjmp(i8* %bufptr)
+  %tobool2 = icmp ne i32 %setjmpres2, 0
+  br i1 %tobool2, label %if2.then, label %if2.else
+
+if2.then:
+  store volatile i32 3, i32* @g, align 4
+  br label %if2.end
+
+if2.else:
+  store volatile i32 2, i32* @g, align 4
+  call void @llvm.eh.sjlj.longjmp(i8* %bufptr)
+  unreachable
+
+if2.end:
+  ret void
+}
diff --git a/test/CodeGen/Thumb2/thumb2-tbh.ll b/test/CodeGen/Thumb2/thumb2-tbh.ll
index a5a5ed0c8da..0761ed589a2 100644
--- a/test/CodeGen/Thumb2/thumb2-tbh.ll
+++ b/test/CodeGen/Thumb2/thumb2-tbh.ll
@@ -14,9 +14,19 @@ declare void @Z_fatal(i8*) noreturn nounwind
 
 declare noalias i8* @calloc(i32, i32) nounwind
 
+; Jump tables are not anchored next to the TBB/TBH any more. Make sure the
+; correct address is still calculated (i.e. via a PC-relative symbol *at* the
+; TBB/TBH).
 define i32 @main(i32 %argc, i8** nocapture %argv) nounwind {
 ; CHECK-LABEL: main:
-; CHECK: tbb
+; CHECK-NOT: adr {{r[0-9]+}}, LJTI
+; CHECK: [[PCREL_ANCHOR:LCPI[0-9]+_[0-9]+]]:
+; CHECK-NEXT:     tbb [pc, {{r[0-9]+}}]
+
+; CHECK: LJTI0_0:
+; CHECK-NEXT: .data_region jt8
+; CHECK-NEXT: .byte (LBB{{[0-9]+_[0-9]+}}-([[PCREL_ANCHOR]]+4))/2
+
 entry:
 	br label %bb42.i
 
diff --git a/test/CodeGen/Thumb2/tpsoft.ll b/test/CodeGen/Thumb2/tpsoft.ll
index 89757cad563..de358d98d49 100644
--- a/test/CodeGen/Thumb2/tpsoft.ll
+++ b/test/CodeGen/Thumb2/tpsoft.ll
@@ -32,13 +32,13 @@ bb:                                               ; preds = %entry
 ; ELFOBJ:        Section {
 ; ELFOBJ:          Name: .text
 ; ELFOBJ-LE:          SectionData (
-;;;                  BL __aeabi_read_tp is ---------+
-;;;                                                 V
-; ELFOBJ-LE-NEXT:     0000: 2DE90048 0E487844 0168FFF7 FEFF4058
+;;;                  BL __aeabi_read_tp is ---+
+;;;                                           V
+; ELFOBJ-LE-NEXT:     0000: 80B50E48 78440168 FFF7FEFF 40580D28
 ; ELFOBJ-BE:          SectionData (
-;;;                  BL __aeabi_read_tp is ---------+
-;;;                                                 V
-; ELFOBJ-BE-NEXT:     0000: E92D4800 480E4478 6801F7FF FFFE5840
+;;;                  BL __aeabi_read_tp is ---+
+;;;                                           V
+; ELFOBJ-BE-NEXT:     0000: B580480E 44786801 F7FFFFFE 5840280D
 
 
 bb1:                                              ; preds = %entry
diff --git a/test/CodeGen/Thumb2/v8_IT_3.ll b/test/CodeGen/Thumb2/v8_IT_3.ll
index 589a20c73d5..3ccee5fbb8c 100644
--- a/test/CodeGen/Thumb2/v8_IT_3.ll
+++ b/test/CodeGen/Thumb2/v8_IT_3.ll
@@ -24,14 +24,14 @@ entry:
   %tmp = load i32, i32* @G, align 4
   %tmp1 = call i32 @bar(i32 0, i32 0, i32 %tmp) nounwind
   switch i32 %tmp1, label %bb8 [
-    i32 0, label %bb
+    i32 1, label %bb
     i32 536870913, label %bb4
     i32 536870914, label %bb6
   ]
 
 bb:
   %tmp2 = load i32, i32* @G, align 4
-  %tmp4 = icmp eq i32 %tmp2, 0
+  %tmp4 = icmp eq i32 %tmp2, 1
   br i1 %tmp4, label %bb1, label %bb8
 
 bb1:
diff --git a/test/CodeGen/Thumb2/v8_IT_5.ll b/test/CodeGen/Thumb2/v8_IT_5.ll
index 2da75ad2143..78b80d7dcde 100644
--- a/test/CodeGen/Thumb2/v8_IT_5.ll
+++ b/test/CodeGen/Thumb2/v8_IT_5.ll
@@ -3,8 +3,7 @@
 ; CHECK: it	ne
 ; CHECK-NEXT: cmpne
 ; CHECK-NEXT: bne [[JUMPTARGET:.LBB[0-9]+_[0-9]+]]
-; CHECK: cmp
-; CHECK-NEXT: beq
+; CHECK: cbz
 ; CHECK-NEXT: %if.else163
 ; CHECK-NEXT: mov.w
 ; CHECK-NEXT: b
diff --git a/test/CodeGen/WebAssembly/cpus.ll b/test/CodeGen/WebAssembly/cpus.ll
new file mode 100644
index 00000000000..bbc9c8fe4f3
--- /dev/null
+++ b/test/CodeGen/WebAssembly/cpus.ll
@@ -0,0 +1,17 @@
+; This tests that llc accepts all valid WebAssembly CPUs.
+
+; RUN: llc < %s -mtriple=wasm32-unknown-unknown -mcpu=mvp 2>&1 | FileCheck %s
+; RUN: llc < %s -mtriple=wasm64-unknown-unknown -mcpu=mvp 2>&1 | FileCheck %s
+; RUN: llc < %s -mtriple=wasm32-unknown-unknown -mcpu=generic 2>&1 | FileCheck %s
+; RUN: llc < %s -mtriple=wasm64-unknown-unknown -mcpu=generic 2>&1 | FileCheck %s
+; RUN: llc < %s -mtriple=wasm32-unknown-unknown -mcpu=bleeding-edge 2>&1 | FileCheck %s
+; RUN: llc < %s -mtriple=wasm64-unknown-unknown -mcpu=bleeding-edge 2>&1 | FileCheck %s
+; RUN: llc < %s -mtriple=wasm32-unknown-unknown -mcpu=invalidcpu 2>&1 | FileCheck %s --check-prefix=INVALID
+; RUN: llc < %s -mtriple=wasm64-unknown-unknown -mcpu=invalidcpu 2>&1 | FileCheck %s --check-prefix=INVALID
+
+; CHECK-NOT: {{.*}}  is not a recognized processor for this target
+; INVALID: {{.*}}  is not a recognized processor for this target
+
+define i32 @f(i32 %i_like_the_web) {
+  ret i32 %i_like_the_web
+}
diff --git a/test/CodeGen/WebAssembly/integer.ll b/test/CodeGen/WebAssembly/integer.ll
new file mode 100644
index 00000000000..44a7151b331
--- /dev/null
+++ b/test/CodeGen/WebAssembly/integer.ll
@@ -0,0 +1,165 @@
+; RUN: llc < %s -asm-verbose=false | FileCheck %s
+
+target datalayout = "e-p:32:32-i64:64-v128:8:128-n32:64-S128"
+target triple = "wasm32-unknown-unknown"
+
+declare i32 @llvm.ctlz.i32(i32, i1)
+declare i32 @llvm.cttz.i32(i32, i1)
+declare i32 @llvm.ctpop.i32(i32)
+
+; CHECK-LABEL: add32:
+; CHECK-NEXT: (setlocal @0 (argument 1))
+; CHECK-NEXT: (setlocal @1 (argument 0))
+; CHECK-NEXT: (setlocal @2 (ADD_I32 @1 @0))
+; CHECK-NEXT: (RETURN @2)
+define i32 @add32(i32 %x, i32 %y) {
+  %a = add i32 %x, %y
+  ret i32 %a
+}
+
+; CHECK-LABEL: sub32:
+; CHECK-NEXT: (setlocal @0 (argument 1))
+; CHECK-NEXT: (setlocal @1 (argument 0))
+; CHECK-NEXT: (setlocal @2 (SUB_I32 @1 @0))
+; CHECK-NEXT: (RETURN @2)
+define i32 @sub32(i32 %x, i32 %y) {
+  %a = sub i32 %x, %y
+  ret i32 %a
+}
+
+; CHECK-LABEL: mul32:
+; CHECK-NEXT: (setlocal @0 (argument 1))
+; CHECK-NEXT: (setlocal @1 (argument 0))
+; CHECK-NEXT: (setlocal @2 (MUL_I32 @1 @0))
+; CHECK-NEXT: (RETURN @2)
+define i32 @mul32(i32 %x, i32 %y) {
+  %a = mul i32 %x, %y
+  ret i32 %a
+}
+
+; CHECK-LABEL: sdiv32:
+; CHECK-NEXT: (setlocal @0 (argument 1))
+; CHECK-NEXT: (setlocal @1 (argument 0))
+; CHECK-NEXT: (setlocal @2 (SDIV_I32 @1 @0))
+; CHECK-NEXT: (RETURN @2)
+define i32 @sdiv32(i32 %x, i32 %y) {
+  %a = sdiv i32 %x, %y
+  ret i32 %a
+}
+
+; CHECK-LABEL: udiv32:
+; CHECK-NEXT: (setlocal @0 (argument 1))
+; CHECK-NEXT: (setlocal @1 (argument 0))
+; CHECK-NEXT: (setlocal @2 (UDIV_I32 @1 @0))
+; CHECK-NEXT: (RETURN @2)
+define i32 @udiv32(i32 %x, i32 %y) {
+  %a = udiv i32 %x, %y
+  ret i32 %a
+}
+
+; CHECK-LABEL: srem32:
+; CHECK-NEXT: (setlocal @0 (argument 1))
+; CHECK-NEXT: (setlocal @1 (argument 0))
+; CHECK-NEXT: (setlocal @2 (SREM_I32 @1 @0))
+; CHECK-NEXT: (RETURN @2)
+define i32 @srem32(i32 %x, i32 %y) {
+  %a = srem i32 %x, %y
+  ret i32 %a
+}
+
+; CHECK-LABEL: urem32:
+; CHECK-NEXT: (setlocal @0 (argument 1))
+; CHECK-NEXT: (setlocal @1 (argument 0))
+; CHECK-NEXT: (setlocal @2 (UREM_I32 @1 @0))
+; CHECK-NEXT: (RETURN @2)
+define i32 @urem32(i32 %x, i32 %y) {
+  %a = urem i32 %x, %y
+  ret i32 %a
+}
+
+; CHECK-LABEL: and32:
+; CHECK-NEXT: (setlocal @0 (argument 1))
+; CHECK-NEXT: (setlocal @1 (argument 0))
+; CHECK-NEXT: (setlocal @2 (AND_I32 @1 @0))
+; CHECK-NEXT: (RETURN @2)
+define i32 @and32(i32 %x, i32 %y) {
+  %a = and i32 %x, %y
+  ret i32 %a
+}
+
+; CHECK-LABEL: ior32:
+; CHECK-NEXT: (setlocal @0 (argument 1))
+; CHECK-NEXT: (setlocal @1 (argument 0))
+; CHECK-NEXT: (setlocal @2 (IOR_I32 @1 @0))
+; CHECK-NEXT: (RETURN @2)
+define i32 @ior32(i32 %x, i32 %y) {
+  %a = or i32 %x, %y
+  ret i32 %a
+}
+
+; CHECK-LABEL: xor32:
+; CHECK-NEXT: (setlocal @0 (argument 1))
+; CHECK-NEXT: (setlocal @1 (argument 0))
+; CHECK-NEXT: (setlocal @2 (XOR_I32 @1 @0))
+; CHECK-NEXT: (RETURN @2)
+define i32 @xor32(i32 %x, i32 %y) {
+  %a = xor i32 %x, %y
+  ret i32 %a
+}
+
+; CHECK-LABEL: shl32:
+; CHECK-NEXT: (setlocal @0 (argument 1))
+; CHECK-NEXT: (setlocal @1 (argument 0))
+; CHECK-NEXT: (setlocal @2 (SHL_I32 @1 @0))
+; CHECK-NEXT: (RETURN @2)
+define i32 @shl32(i32 %x, i32 %y) {
+  %a = shl i32 %x, %y
+  ret i32 %a
+}
+
+; CHECK-LABEL: shr32:
+; CHECK-NEXT: (setlocal @0 (argument 1))
+; CHECK-NEXT: (setlocal @1 (argument 0))
+; CHECK-NEXT: (setlocal @2 (SHR_I32 @1 @0))
+; CHECK-NEXT: (RETURN @2)
+define i32 @shr32(i32 %x, i32 %y) {
+  %a = lshr i32 %x, %y
+  ret i32 %a
+}
+
+; CHECK-LABEL: sar32:
+; CHECK-NEXT: (setlocal @0 (argument 1))
+; CHECK-NEXT: (setlocal @1 (argument 0))
+; CHECK-NEXT: (setlocal @2 (SAR_I32 @1 @0))
+; CHECK-NEXT: (RETURN @2)
+define i32 @sar32(i32 %x, i32 %y) {
+  %a = ashr i32 %x, %y
+  ret i32 %a
+}
+
+; CHECK-LABEL: clz32:
+; CHECK-NEXT: (setlocal @0 (argument 0))
+; CHECK-NEXT: (setlocal @1 (CLZ_I32 @0))
+; CHECK-NEXT: (RETURN @1)
+define i32 @clz32(i32 %x) {
+  %a = call i32 @llvm.ctlz.i32(i32 %x, i1 false)
+  ret i32 %a
+}
+
+; CHECK-LABEL: ctz32:
+; CHECK-NEXT: (setlocal @0 (argument 0))
+; CHECK-NEXT: (setlocal @1 (CTZ_I32 @0))
+; CHECK-NEXT: (RETURN @1)
+define i32 @ctz32(i32 %x) {
+  %a = call i32 @llvm.cttz.i32(i32 %x, i1 false)
+  ret i32 %a
+}
+
+; CHECK-LABEL: popcnt32:
+; CHECK-NEXT: (setlocal @0 (argument 0))
+; CHECK-NEXT: (setlocal @1 (POPCNT_I32 @0))
+; CHECK-NEXT: (RETURN @1)
+define i32 @popcnt32(i32 %x) {
+  %a = call i32 @llvm.ctpop.i32(i32 %x)
+  ret i32 %a
+}
diff --git a/test/CodeGen/WebAssembly/lit.local.cfg b/test/CodeGen/WebAssembly/lit.local.cfg
new file mode 100644
index 00000000000..743473517cd
--- /dev/null
+++ b/test/CodeGen/WebAssembly/lit.local.cfg
@@ -0,0 +1,2 @@
+if not 'WebAssembly' in config.root.targets:
+    config.unsupported = True
diff --git a/test/CodeGen/WebAssembly/unused-argument.ll b/test/CodeGen/WebAssembly/unused-argument.ll
new file mode 100644
index 00000000000..5e546f7eddf
--- /dev/null
+++ b/test/CodeGen/WebAssembly/unused-argument.ll
@@ -0,0 +1,20 @@
+; RUN: llc < %s -asm-verbose=false | FileCheck %s
+
+; Make sure that argument offsets are correct even if some arguments are unused.
+
+target datalayout = "e-p:32:32-i64:64-v128:8:128-n32:64-S128"
+target triple = "wasm32-unknown-unknown"
+
+; CHECK-LABEL: unused_first:
+; CHECK-NEXT: (setlocal @0 (argument 1))
+; CHECK-NEXT: (RETURN @0)
+define i32 @unused_first(i32 %x, i32 %y) {
+  ret i32 %y
+}
+
+; CHECK-LABEL: unused_second:
+; CHECK-NEXT: (setlocal @0 (argument 0))
+; CHECK-NEXT: (RETURN @0)
+define i32 @unused_second(i32 %x, i32 %y) {
+  ret i32 %x
+}
diff --git a/test/CodeGen/WinEH/cppeh-alloca-sink.ll b/test/CodeGen/WinEH/cppeh-alloca-sink.ll
index d50237fa78a..f215dca2ddd 100644
--- a/test/CodeGen/WinEH/cppeh-alloca-sink.ll
+++ b/test/CodeGen/WinEH/cppeh-alloca-sink.ll
@@ -51,7 +51,7 @@ $"\01??_R0H@8" = comdat any
 @llvm.eh.handlertype.H.0 = private unnamed_addr constant %eh.CatchHandlerType { i32 0, i8* bitcast (%rtti.TypeDescriptor2* @"\01??_R0H@8" to i8*) }, section "llvm.metadata"
 
 ; Function Attrs: uwtable
-define void @sink_alloca_to_catch() #0 {
+define void @sink_alloca_to_catch() #0 personality i8* bitcast (i32 (...)* @__CxxFrameHandler3 to i8*) {
 entry:
   %0 = alloca i32
   %only_used_in_catch = alloca i32, align 4
@@ -59,7 +59,7 @@ entry:
           to label %try.cont unwind label %lpad
 
 lpad:                                             ; preds = %entry
-  %1 = landingpad { i8*, i32 } personality i8* bitcast (i32 (...)* @__CxxFrameHandler3 to i8*)
+  %1 = landingpad { i8*, i32 }
           catch %eh.CatchHandlerType* @llvm.eh.handlertype.H.0
   %2 = extractvalue { i8*, i32 } %1, 1
   %3 = tail call i32 @llvm.eh.typeid.for(i8* bitcast (%eh.CatchHandlerType* @llvm.eh.handlertype.H.0 to i8*)) #3
@@ -81,12 +81,12 @@ eh.resume:                                        ; preds = %lpad
 }
 
 ; CHECK-LABEL: define void @sink_alloca_to_catch()
-; CHECK: call void (...) @llvm.frameescape(i32* %only_used_in_catch)
+; CHECK: call void (...) @llvm.localescape(i32* %only_used_in_catch)
 
 declare void @use_catch_var(i32*) #1
 
 ; Function Attrs: uwtable
-define void @dont_sink_alloca_to_catch(i32 %n) #0 {
+define void @dont_sink_alloca_to_catch(i32 %n) #0 personality i8* bitcast (i32 (...)* @__CxxFrameHandler3 to i8*) {
 entry:
   %0 = alloca i32
   %n.addr = alloca i32, align 4
@@ -109,7 +109,7 @@ invoke.cont:                                      ; preds = %while.body
   br label %try.cont
 
 lpad:                                             ; preds = %while.body
-  %2 = landingpad { i8*, i32 } personality i8* bitcast (i32 (...)* @__CxxFrameHandler3 to i8*)
+  %2 = landingpad { i8*, i32 }
           catch i8* bitcast (%eh.CatchHandlerType* @llvm.eh.handlertype.H.0 to i8*)
   %3 = extractvalue { i8*, i32 } %2, 0
   store i8* %3, i8** %exn.slot
@@ -141,7 +141,7 @@ try.cont:                                         ; preds = %invoke.cont2, %invo
   br label %while.cond
 
 lpad1:                                            ; preds = %catch
-  %8 = landingpad { i8*, i32 } personality i8* bitcast (i32 (...)* @__CxxFrameHandler3 to i8*)
+  %8 = landingpad { i8*, i32 }
           cleanup
   %9 = extractvalue { i8*, i32 } %8, 0
   store i8* %9, i8** %exn.slot
@@ -162,14 +162,14 @@ eh.resume:                                        ; preds = %lpad1, %catch.dispa
 }
 
 ; CHECK-LABEL: define void @dont_sink_alloca_to_catch(i32 %n)
-; CHECK: call void (...) @llvm.frameescape(i32* %live_in_out_catch)
+; CHECK: call void (...) @llvm.localescape(i32* %live_in_out_catch)
 
 ; CHECK-LABEL: define internal i8* @sink_alloca_to_catch.catch(i8*, i8*)
-; CHECK: %only_used_in_catch.i8 = call i8* @llvm.framerecover({{.*}}, i32 0)
+; CHECK: %only_used_in_catch.i8 = call i8* @llvm.localrecover({{.*}}, i32 0)
 ; CHECK: %only_used_in_catch = bitcast
 
 ; CHECK-LABEL: define internal i8* @dont_sink_alloca_to_catch.catch(i8*, i8*)
-; CHECK: %live_in_out_catch.i8 = call i8* @llvm.framerecover({{.*}}, i32 0)
+; CHECK: %live_in_out_catch.i8 = call i8* @llvm.localrecover({{.*}}, i32 0)
 ; CHECK: %live_in_out_catch = bitcast
 
 
diff --git a/test/CodeGen/WinEH/cppeh-catch-all-win32.ll b/test/CodeGen/WinEH/cppeh-catch-all-win32.ll
new file mode 100644
index 00000000000..b2e84b90d69
--- /dev/null
+++ b/test/CodeGen/WinEH/cppeh-catch-all-win32.ll
@@ -0,0 +1,86 @@
+; RUN: opt -winehprepare -S -o - < %s | FileCheck %s
+
+; This test is based on the following code:
+;
+; extern "C" void may_throw();
+; extern "C" void handle_exception();
+; extern "C" void test() {
+;   try {
+;     may_throw();
+;   } catch (...) {
+;     handle_exception();
+;   }
+; }
+
+target datalayout = "e-m:x-p:32:32-i64:64-f80:32-n8:16:32-a:0:32-S32"
+target triple = "i686-pc-windows-msvc"
+
+; The function entry in this case remains unchanged.
+; CHECK: define void @test()
+; CHECK: entry:
+; CHECK:   invoke void @may_throw()
+; CHECK:           to label %invoke.cont unwind label %[[LPAD_LABEL:lpad[0-9]*]]
+
+define void @test() #0 personality i8* bitcast (i32 (...)* @__CxxFrameHandler3 to i8*) {
+entry:
+  %exn.slot = alloca i8*
+  %ehselector.slot = alloca i32
+  invoke void @may_throw()
+          to label %invoke.cont unwind label %lpad
+
+invoke.cont:                                      ; preds = %entry
+  br label %try.cont
+
+; CHECK: [[LPAD_LABEL]]:{{[ ]+}}; preds = %entry
+; CHECK:   landingpad { i8*, i32 }
+; CHECK-NEXT:           catch i8* null
+; CHECK-NEXT:   [[RECOVER:\%.+]] = call i8* (...) @llvm.eh.actions(i32 1, i8* null, i32 -1, i8* ()* @test.catch)
+; CHECK-NEXT:   indirectbr i8* [[RECOVER]], [label %try.cont]
+
+lpad:                                             ; preds = %entry
+  %0 = landingpad { i8*, i32 }
+          catch i8* null
+  %1 = extractvalue { i8*, i32 } %0, 0
+  store i8* %1, i8** %exn.slot
+  %2 = extractvalue { i8*, i32 } %0, 1
+  store i32 %2, i32* %ehselector.slot
+  br label %catch
+
+; CHECK-NOT: catch:
+; CHECK-NOT: @handle_exception()
+
+catch:                                            ; preds = %lpad
+  %exn = load i8*, i8** %exn.slot
+  call void @llvm.eh.begincatch(i8* %exn, i8* null) #1
+  call void @handle_exception()
+  call void @llvm.eh.endcatch() #1
+  br label %try.cont
+
+try.cont:                                         ; preds = %catch, %invoke.cont
+  ret void
+
+; CHECK: }
+}
+
+; CHECK: define internal i8* @test.catch()
+; CHECK:   call i8* @llvm.frameaddress(i32 1)
+; CHECK:   call i8* @llvm.x86.seh.recoverfp(i8* bitcast (void ()* @test to i8*), i8* %{{.*}})
+; CHECK:   call void @handle_exception()
+; CHECK:   ret i8* blockaddress(@test, %try.cont)
+; CHECK: }
+
+
+declare void @may_throw() #0
+
+declare i32 @__CxxFrameHandler3(...)
+
+; Function Attrs: nounwind
+declare void @llvm.eh.begincatch(i8* nocapture, i8* nocapture) #1
+
+declare void @handle_exception() #0
+
+; Function Attrs: nounwind
+declare void @llvm.eh.endcatch() #1
+
+attributes #0 = { "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "no-realign-stack" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #1 = { nounwind }
diff --git a/test/CodeGen/WinEH/cppeh-catch-all.ll b/test/CodeGen/WinEH/cppeh-catch-all.ll
index 4d017fde862..266dd3e305c 100644
--- a/test/CodeGen/WinEH/cppeh-catch-all.ll
+++ b/test/CodeGen/WinEH/cppeh-catch-all.ll
@@ -22,10 +22,10 @@ target triple = "x86_64-pc-windows-msvc"
 ; CHECK: define void @_Z4testv()
 ; CHECK: entry:
 ; CHECK:   invoke void @_Z9may_throwv()
-; CHECK:           to label %invoke.cont unwind label %[[LPAD_LABEL:lpad[0-9]+]]
+; CHECK:           to label %invoke.cont unwind label %[[LPAD_LABEL:lpad[0-9]*]]
 
 ; Function Attrs: uwtable
-define void @_Z4testv() #0 {
+define void @_Z4testv() #0 personality i8* bitcast (i32 (...)* @__CxxFrameHandler3 to i8*) {
 entry:
   %exn.slot = alloca i8*
   %ehselector.slot = alloca i32
@@ -36,13 +36,13 @@ invoke.cont:                                      ; preds = %entry
   br label %try.cont
 
 ; CHECK: [[LPAD_LABEL]]:{{[ ]+}}; preds = %entry
-; CHECK:   landingpad { i8*, i32 } personality i8* bitcast (i32 (...)* @__CxxFrameHandler3 to i8*)
+; CHECK:   landingpad { i8*, i32 }
 ; CHECK-NEXT:           catch i8* null
 ; CHECK-NEXT:   [[RECOVER:\%.+]] = call i8* (...) @llvm.eh.actions(i32 1, i8* null, i32 -1, i8* (i8*, i8*)* @_Z4testv.catch)
 ; CHECK-NEXT:   indirectbr i8* [[RECOVER]], [label %try.cont]
 
 lpad:                                             ; preds = %entry
-  %tmp = landingpad { i8*, i32 } personality i8* bitcast (i32 (...)* @__CxxFrameHandler3 to i8*)
+  %tmp = landingpad { i8*, i32 }
           catch i8* null
   %tmp1 = extractvalue { i8*, i32 } %tmp, 0
   store i8* %tmp1, i8** %exn.slot
diff --git a/test/CodeGen/WinEH/cppeh-catch-and-throw.ll b/test/CodeGen/WinEH/cppeh-catch-and-throw.ll
new file mode 100644
index 00000000000..d604b86deb3
--- /dev/null
+++ b/test/CodeGen/WinEH/cppeh-catch-and-throw.ll
@@ -0,0 +1,143 @@
+; RUN: opt -mtriple=x86_64-pc-windows-msvc -winehprepare -S -o - < %s | FileCheck %s
+
+; This test is based on the following code:
+;
+; class Obj {
+; public:
+;   ~Obj();
+; };
+;
+; void test(void)
+; {
+;   try {
+;     Obj o;
+;     throw 1;
+;   } catch (...) {
+;     throw;
+;   }
+; }
+
+; ModuleID = 'cppeh-catch-and-throw.cpp'
+target datalayout = "e-m:w-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-pc-windows-msvc"
+
+%rtti.TypeDescriptor2 = type { i8**, i8*, [3 x i8] }
+%eh.CatchableType = type { i32, i32, i32, i32, i32, i32, i32 }
+%eh.CatchableTypeArray.1 = type { i32, [1 x i32] }
+%eh.ThrowInfo = type { i32, i32, i32, i32 }
+%class.Obj = type { i8 }
+
+$"\01??_R0H@8" = comdat any
+
+$"_CT??_R0H@84" = comdat any
+
+$_CTA1H = comdat any
+
+$_TI1H = comdat any
+
+@"\01??_7type_info@@6B@" = external constant i8*
+@"\01??_R0H@8" = linkonce_odr global %rtti.TypeDescriptor2 { i8** @"\01??_7type_info@@6B@", i8* null, [3 x i8] c".H\00" }, comdat
+@__ImageBase = external constant i8
+@"_CT??_R0H@84" = linkonce_odr unnamed_addr constant %eh.CatchableType { i32 1, i32 trunc (i64 sub nuw nsw (i64 ptrtoint (%rtti.TypeDescriptor2* @"\01??_R0H@8" to i64), i64 ptrtoint (i8* @__ImageBase to i64)) to i32), i32 0, i32 -1, i32 0, i32 4, i32 0 }, section ".xdata", comdat
+@_CTA1H = linkonce_odr unnamed_addr constant %eh.CatchableTypeArray.1 { i32 1, [1 x i32] [i32 trunc (i64 sub nuw nsw (i64 ptrtoint (%eh.CatchableType* @"_CT??_R0H@84" to i64), i64 ptrtoint (i8* @__ImageBase to i64)) to i32)] }, section ".xdata", comdat
+@_TI1H = linkonce_odr unnamed_addr constant %eh.ThrowInfo { i32 0, i32 0, i32 0, i32 trunc (i64 sub nuw nsw (i64 ptrtoint (%eh.CatchableTypeArray.1* @_CTA1H to i64), i64 ptrtoint (i8* @__ImageBase to i64)) to i32) }, section ".xdata", comdat
+
+; This is just a minimal check to verify that main was handled by WinEHPrepare.
+; CHECK: define void @"\01?test@@YAXXZ"()
+; CHECK: entry:
+; CHECK:   call void (...) @llvm.localescape
+; CHECK:   invoke void @_CxxThrowException
+; CHECK: }
+
+; Function Attrs: uwtable
+define void @"\01?test@@YAXXZ"() #0 personality i8* bitcast (i32 (...)* @__CxxFrameHandler3 to i8*) {
+entry:
+  %o = alloca %class.Obj, align 1
+  %tmp = alloca i32, align 4
+  %exn.slot = alloca i8*
+  %ehselector.slot = alloca i32
+  store i32 1, i32* %tmp
+  %0 = bitcast i32* %tmp to i8*
+  invoke void @_CxxThrowException(i8* %0, %eh.ThrowInfo* @_TI1H) #3
+          to label %unreachable unwind label %lpad
+
+lpad:                                             ; preds = %entry
+  %1 = landingpad { i8*, i32 }
+          catch i8* null
+  %2 = extractvalue { i8*, i32 } %1, 0
+  store i8* %2, i8** %exn.slot
+  %3 = extractvalue { i8*, i32 } %1, 1
+  store i32 %3, i32* %ehselector.slot
+  call void @"\01??1Obj@@QEAA@XZ"(%class.Obj* %o) #2
+  br label %catch
+
+catch:                                            ; preds = %lpad
+  %exn = load i8*, i8** %exn.slot
+  call void @llvm.eh.begincatch(i8* %exn, i8* null) #2
+  invoke void @_CxxThrowException(i8* null, %eh.ThrowInfo* null) #3
+          to label %unreachable unwind label %lpad1
+
+lpad1:                                            ; preds = %catch
+  %4 = landingpad { i8*, i32 }
+          cleanup
+  %5 = extractvalue { i8*, i32 } %4, 0
+  store i8* %5, i8** %exn.slot
+  %6 = extractvalue { i8*, i32 } %4, 1
+  store i32 %6, i32* %ehselector.slot
+  call void @llvm.eh.endcatch() #2
+  br label %eh.resume
+
+try.cont:                                         ; No predecessors!
+  ret void
+
+eh.resume:                                        ; preds = %lpad1
+  %exn2 = load i8*, i8** %exn.slot
+  %sel = load i32, i32* %ehselector.slot
+  %lpad.val = insertvalue { i8*, i32 } undef, i8* %exn2, 0
+  %lpad.val3 = insertvalue { i8*, i32 } %lpad.val, i32 %sel, 1
+  resume { i8*, i32 } %lpad.val3
+
+unreachable:                                      ; preds = %catch, %entry
+  unreachable
+}
+
+; Verify that we inserted a stub invoke into the outlined cleanup handler.
+;
+; CHECK-LABEL: define internal void @"\01?test@@YAXXZ.cleanup"(i8*, i8*)
+; CHECK: entry:
+; CHECK:   call i8* @llvm.localrecover
+; CHECK:   call void @"\01??1Obj@@QEAA@XZ"
+; CHECK:   invoke void @llvm.donothing()
+; CHECK:           to label %[[SPLIT_LABEL:.+]] unwind label %[[LPAD_LABEL:.+]]
+;
+; CHECK: [[SPLIT_LABEL]]
+;
+; CHECK: [[LPAD_LABEL]]
+; CHECK:   landingpad { i8*, i32 }
+; CHECK:           cleanup
+; CHECK:   unreachable
+; CHECK: }
+
+declare void @_CxxThrowException(i8*, %eh.ThrowInfo*)
+
+declare i32 @__CxxFrameHandler3(...)
+
+; Function Attrs: nounwind
+declare void @"\01??1Obj@@QEAA@XZ"(%class.Obj*) #1
+
+; Function Attrs: nounwind
+declare void @llvm.eh.begincatch(i8* nocapture, i8* nocapture) #2
+
+; Function Attrs: nounwind
+declare void @llvm.eh.endcatch() #2
+
+attributes #0 = { uwtable "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #1 = { nounwind "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #2 = { nounwind }
+attributes #3 = { noreturn }
+
+!llvm.module.flags = !{!0}
+!llvm.ident = !{!1}
+
+!0 = !{i32 1, !"PIC Level", i32 2}
+!1 = !{!"clang version 3.7.0 (trunk 235214) (llvm/trunk 235213)"}
diff --git a/test/CodeGen/WinEH/cppeh-catch-scalar.ll b/test/CodeGen/WinEH/cppeh-catch-scalar.ll
index 4ea383857ae..3b5ab746d63 100644
--- a/test/CodeGen/WinEH/cppeh-catch-scalar.ll
+++ b/test/CodeGen/WinEH/cppeh-catch-scalar.ll
@@ -24,12 +24,12 @@ target triple = "x86_64-pc-windows-msvc"
 ; CHECK: define void @_Z4testv()
 ; CHECK: entry:
 ; CHECK:   [[I_PTR:\%.+]] = alloca i32, align 4
-; CHECK:   call void (...) @llvm.frameescape(i32* [[I_PTR]])
+; CHECK:   call void (...) @llvm.localescape(i32* [[I_PTR]])
 ; CHECK:   invoke void @_Z9may_throwv()
-; CHECK:           to label %invoke.cont unwind label %[[LPAD_LABEL:lpad[0-9]+]]
+; CHECK:           to label %invoke.cont unwind label %[[LPAD_LABEL:lpad[0-9]*]]
 
 ; Function Attrs: uwtable
-define void @_Z4testv() #0 {
+define void @_Z4testv() #0 personality i8* bitcast (i32 (...)* @__CxxFrameHandler3 to i8*) {
 entry:
   %exn.slot = alloca i8*
   %ehselector.slot = alloca i32
@@ -41,13 +41,13 @@ invoke.cont:                                      ; preds = %entry
   br label %try.cont
 
 ; CHECK: [[LPAD_LABEL]]:{{[ ]+}}; preds = %entry
-; CHECK:   landingpad { i8*, i32 } personality i8* bitcast (i32 (...)* @__CxxFrameHandler3 to i8*)
+; CHECK:   landingpad { i8*, i32 }
 ; CHECK-NEXT:           catch i8* bitcast (i8** @_ZTIi to i8*)
 ; CHECK-NEXT:   [[RECOVER:\%.+]] = call i8* (...) @llvm.eh.actions(i32 1, i8* bitcast (i8** @_ZTIi to i8*), i32 0, i8* (i8*, i8*)* @_Z4testv.catch)
 ; CHECK-NEXT:   indirectbr i8* [[RECOVER]], [label %try.cont]
 
 lpad:                                             ; preds = %entry
-  %tmp = landingpad { i8*, i32 } personality i8* bitcast (i32 (...)* @__CxxFrameHandler3 to i8*)
+  %tmp = landingpad { i8*, i32 }
           catch i8* bitcast (i8** @_ZTIi to i8*)
   %tmp1 = extractvalue { i8*, i32 } %tmp, 0
   store i8* %tmp1, i8** %exn.slot
@@ -96,7 +96,7 @@ eh.resume:                                        ; preds = %catch.dispatch
 
 ; CHECK: define internal i8* @_Z4testv.catch(i8*, i8*)
 ; CHECK: entry:
-; CHECK:   [[RECOVER_I:\%.+]] = call i8* @llvm.framerecover(i8* bitcast (void ()* @_Z4testv to i8*), i8* %1, i32 0)
+; CHECK:   [[RECOVER_I:\%.+]] = call i8* @llvm.localrecover(i8* bitcast (void ()* @_Z4testv to i8*), i8* %1, i32 0)
 ; CHECK:   [[I_PTR1:\%.+]] = bitcast i8* [[RECOVER_I]] to i32*
 ; CHECK:   [[TMP:\%.+]] = load i32, i32* [[I_PTR1]], align 4
 ; CHECK:   call void @_Z10handle_inti(i32 [[TMP]])
diff --git a/test/CodeGen/WinEH/cppeh-catch-unwind.ll b/test/CodeGen/WinEH/cppeh-catch-unwind.ll
index 37a542941eb..8fdda9bbc02 100644
--- a/test/CodeGen/WinEH/cppeh-catch-unwind.ll
+++ b/test/CodeGen/WinEH/cppeh-catch-unwind.ll
@@ -31,17 +31,17 @@ $"\01??_R0H@8" = comdat any
 @"\01??_R0H@8" = linkonce_odr global %rtti.TypeDescriptor2 { i8** @"\01??_7type_info@@6B@", i8* null, [3 x i8] c".H\00" }, comdat
 
 
-; CHECK-LABEL: define void @"\01?test@@YAXXZ"() #0 {
+; CHECK-LABEL: define void @"\01?test@@YAXXZ"() #0 personality i8* bitcast (i32 (...)* @__CxxFrameHandler3 to i8*) {
 ; CHECK: entry:
 ; CHECK:   [[OBJ_PTR:\%.+]] = alloca %class.SomeClass
 ; CHECK:   [[TMP0:\%.+]] = alloca i32, align 4
 ; CHECK:   [[TMP1:\%.+]] = alloca i32, align 4
-; CHECK:   call void (...) @llvm.frameescape(i32* [[TMP1]], %class.SomeClass* [[OBJ_PTR]], i32* [[TMP0]])
+; CHECK:   call void (...) @llvm.localescape(i32* [[TMP1]], %class.SomeClass* [[OBJ_PTR]], i32* [[TMP0]])
 ; CHECK:   %call = invoke %class.SomeClass* @"\01??0SomeClass@@QEAA@XZ"(%class.SomeClass* %obj)
-; CHECK:           to label %invoke.cont unwind label %[[LPAD_LABEL:lpad[0-9]+]]
+; CHECK:           to label %invoke.cont unwind label %[[LPAD_LABEL:lpad[0-9]*]]
 
 ; Function Attrs: uwtable
-define void @"\01?test@@YAXXZ"() #0 {
+define void @"\01?test@@YAXXZ"() #0 personality i8* bitcast (i32 (...)* @__CxxFrameHandler3 to i8*) {
 entry:
   %obj = alloca %class.SomeClass, align 1
   %0 = alloca i32, align 4
@@ -51,7 +51,7 @@ entry:
 
 ; CHECK: invoke.cont:
 ; CHECK:   invoke void @"\01?may_throw@@YAXXZ"()
-; CHECK:           to label %invoke.cont2 unwind label %[[LPAD1_LABEL:lpad[0-9]+]]
+; CHECK:           to label %invoke.cont2 unwind label %[[LPAD1_LABEL:lpad[0-9]*]]
 
 invoke.cont:                                      ; preds = %entry
   invoke void @"\01?may_throw@@YAXXZ"()
@@ -59,34 +59,34 @@ invoke.cont:                                      ; preds = %entry
 
 ; CHECK: invoke.cont2:
 ; CHECK:   invoke void @"\01?may_throw@@YAXXZ"()
-; CHECK:           to label %try.cont unwind label %[[LPAD3_LABEL:lpad[0-9]+]]
+; CHECK:           to label %try.cont unwind label %[[LPAD3_LABEL:lpad[0-9]*]]
 
 invoke.cont2:                                     ; preds = %invoke.cont
   invoke void @"\01?may_throw@@YAXXZ"()
           to label %try.cont unwind label %lpad3
 
 ; CHECK: [[LPAD_LABEL]]:{{[ ]+}}; preds = %entry
-; CHECK:   [[LPAD_VAL:\%.+]] = landingpad { i8*, i32 } personality i8* bitcast (i32 (...)* @__CxxFrameHandler3 to i8*)
+; CHECK:   [[LPAD_VAL:\%.+]] = landingpad { i8*, i32 }
 ; CHECK-NEXT:           catch i8* bitcast (%rtti.TypeDescriptor2* @"\01??_R0H@8" to i8*)
 ; CHECK-NEXT:   [[RECOVER:\%.+]] = call i8* (...) @llvm.eh.actions(i32 1, i8* bitcast (%rtti.TypeDescriptor2* @"\01??_R0H@8" to i8*), i32 0, i8* (i8*, i8*)* @"\01?test@@YAXXZ.catch")
 ; CHECK-NEXT:   indirectbr i8* [[RECOVER]], [label %try.cont15]
 
 lpad:                                             ; preds = %entry
-  %2 = landingpad { i8*, i32 } personality i8* bitcast (i32 (...)* @__CxxFrameHandler3 to i8*)
+  %2 = landingpad { i8*, i32 }
           catch i8* bitcast (%rtti.TypeDescriptor2* @"\01??_R0H@8" to i8*)
   %3 = extractvalue { i8*, i32 } %2, 0
   %4 = extractvalue { i8*, i32 } %2, 1
   br label %catch.dispatch7
 
 ; CHECK: [[LPAD1_LABEL]]:{{[ ]+}}; preds = %invoke.cont
-; CHECK:   [[LPAD1_VAL:\%.+]] = landingpad { i8*, i32 } personality i8* bitcast (i32 (...)* @__CxxFrameHandler3 to i8*)
+; CHECK:   [[LPAD1_VAL:\%.+]] = landingpad { i8*, i32 }
 ; CHECK-NEXT:           cleanup
 ; CHECK-NEXT:           catch i8* bitcast (%rtti.TypeDescriptor2* @"\01??_R0H@8" to i8*)
 ; CHECK-NEXT:   [[RECOVER1:\%.+]] = call i8* (...) @llvm.eh.actions(i32 0, void (i8*, i8*)* @"\01?test@@YAXXZ.cleanup", i32 1, i8* bitcast (%rtti.TypeDescriptor2* @"\01??_R0H@8" to i8*), i32 0, i8* (i8*, i8*)* @"\01?test@@YAXXZ.catch")
 ; CHECK-NEXT:   indirectbr i8* [[RECOVER1]], [label %try.cont15]
 
 lpad1:                                            ; preds = %invoke.cont
-  %5 = landingpad { i8*, i32 } personality i8* bitcast (i32 (...)* @__CxxFrameHandler3 to i8*)
+  %5 = landingpad { i8*, i32 }
           cleanup
           catch i8* bitcast (%rtti.TypeDescriptor2* @"\01??_R0H@8" to i8*)
   %6 = extractvalue { i8*, i32 } %5, 0
@@ -94,14 +94,14 @@ lpad1:                                            ; preds = %invoke.cont
   br label %ehcleanup
 
 ; CHECK: [[LPAD3_LABEL]]:{{[ ]+}}; preds = %invoke.cont2
-; CHECK:   [[LPAD3_VAL:\%.+]] = landingpad { i8*, i32 } personality i8* bitcast (i32 (...)* @__CxxFrameHandler3 to i8*)
+; CHECK:   [[LPAD3_VAL:\%.+]] = landingpad { i8*, i32 }
 ; CHECK-NEXT:           cleanup
 ; CHECK-NEXT:           catch i8* bitcast (%rtti.TypeDescriptor2* @"\01??_R0H@8" to i8*)
-; CHECK-NEXT:   [[RECOVER3:\%.+]] = call i8* (...) @llvm.eh.actions(i32 1, i8* bitcast (%rtti.TypeDescriptor2* @"\01??_R0H@8" to i8*), i32 2, i8* (i8*, i8*)* @"\01?test@@YAXXZ.catch1", i32 0, void (i8*, i8*)* @"\01?test@@YAXXZ.cleanup")
-; CHECK-NEXT:   indirectbr i8* [[RECOVER3]], [label %try.cont]
+; CHECK-NEXT:   [[RECOVER3:\%.+]] = call i8* (...) @llvm.eh.actions(i32 1, i8* bitcast (%rtti.TypeDescriptor2* @"\01??_R0H@8" to i8*), i32 2, i8* (i8*, i8*)* @"\01?test@@YAXXZ.catch.1", i32 0, void (i8*, i8*)* @"\01?test@@YAXXZ.cleanup")
+; CHECK-NEXT:   indirectbr i8* [[RECOVER3]], [label %try.cont, label %try.cont15]
 
 lpad3:                                            ; preds = %invoke.cont2
-  %8 = landingpad { i8*, i32 } personality i8* bitcast (i32 (...)* @__CxxFrameHandler3 to i8*)
+  %8 = landingpad { i8*, i32 }
           cleanup
           catch i8* bitcast (%rtti.TypeDescriptor2* @"\01??_R0H@8" to i8*)
   %9 = extractvalue { i8*, i32 } %8, 0
@@ -128,7 +128,7 @@ try.cont:                                         ; preds = %invoke.cont2, %invo
 
 ; CHECK-NOT: lpad5:
 lpad5:                                            ; preds = %catch
-  %13 = landingpad { i8*, i32 } personality i8* bitcast (i32 (...)* @__CxxFrameHandler3 to i8*)
+  %13 = landingpad { i8*, i32 }
           cleanup
           catch i8* bitcast (%rtti.TypeDescriptor2* @"\01??_R0H@8" to i8*)
   %14 = extractvalue { i8*, i32 } %13, 0
@@ -177,7 +177,7 @@ eh.resume:                                        ; preds = %catch.dispatch7
 
 ; CHECK-LABEL: define internal i8* @"\01?test@@YAXXZ.catch"(i8*, i8*)
 ; CHECK: entry:
-; CHECK:   [[RECOVER_TMP1:\%.+]] = call i8* @llvm.framerecover(i8* bitcast (void ()* @"\01?test@@YAXXZ" to i8*), i8* %1, i32 0)
+; CHECK:   [[RECOVER_TMP1:\%.+]] = call i8* @llvm.localrecover(i8* bitcast (void ()* @"\01?test@@YAXXZ" to i8*), i8* %1, i32 0)
 ; CHECK:   [[TMP1_PTR:\%.+]] = bitcast i8* [[RECOVER_TMP1]] to i32*
 ; CHECK:   call void @"\01?handle_exception@@YAXXZ"()
 ; CHECK:   ret i8* blockaddress(@"\01?test@@YAXXZ", %try.cont15)
@@ -185,15 +185,15 @@ eh.resume:                                        ; preds = %catch.dispatch7
 
 ; CHECK-LABEL: define internal void @"\01?test@@YAXXZ.cleanup"(i8*, i8*)
 ; CHECK: entry:
-; CHECK:   [[RECOVER_OBJ:\%.+]] = call i8* @llvm.framerecover(i8* bitcast (void ()* @"\01?test@@YAXXZ" to i8*), i8* %1, i32 1)
+; CHECK:   [[RECOVER_OBJ:\%.+]] = call i8* @llvm.localrecover(i8* bitcast (void ()* @"\01?test@@YAXXZ" to i8*), i8* %1, i32 1)
 ; CHECK:   [[OBJ_PTR:\%.+]] = bitcast i8* %obj.i8 to %class.SomeClass*
 ; CHECK:   call void @"\01??1SomeClass@@QEAA@XZ"(%class.SomeClass* [[OBJ_PTR]])
 ; CHECK:   ret void
 ; CHECK: }
 
-; CHECK-LABEL: define internal i8* @"\01?test@@YAXXZ.catch1"(i8*, i8*)
+; CHECK-LABEL: define internal i8* @"\01?test@@YAXXZ.catch.1"(i8*, i8*)
 ; CHECK: entry:
-; CHECK:   [[RECOVER_TMP0:\%.+]] = call i8* @llvm.framerecover(i8* bitcast (void ()* @"\01?test@@YAXXZ" to i8*), i8* %1, i32 2)
+; CHECK:   [[RECOVER_TMP0:\%.+]] = call i8* @llvm.localrecover(i8* bitcast (void ()* @"\01?test@@YAXXZ" to i8*), i8* %1, i32 2)
 ; CHECK:   [[TMP0_PTR:\%.+]] = bitcast i8* [[RECOVER_TMP0]] to i32*
 ; CHECK:   invoke void @"\01?handle_exception@@YAXXZ"()
 ; CHECK:           to label %invoke.cont6 unwind label %[[LPAD5_LABEL:lpad[0-9]+]]
@@ -202,7 +202,7 @@ eh.resume:                                        ; preds = %catch.dispatch7
 ; CHECK:   ret i8* blockaddress(@"\01?test@@YAXXZ", %try.cont)
 ;
 ; CHECK: [[LPAD5_LABEL]]:{{[ ]+}}; preds = %entry
-; CHECK:   [[LPAD5_VAL:\%.+]] = landingpad { i8*, i32 } personality i8* bitcast (i32 (...)* @__CxxFrameHandler3 to i8*)
+; CHECK:   [[LPAD5_VAL:\%.+]] = landingpad { i8*, i32 }
 ; CHECK:           cleanup
 ; CHECK:           catch i8* bitcast (%rtti.TypeDescriptor2* @"\01??_R0H@8" to i8*)
 ; CHECK: }
diff --git a/test/CodeGen/WinEH/cppeh-cleanup-invoke.ll b/test/CodeGen/WinEH/cppeh-cleanup-invoke.ll
index 5a570431510..7e5f659f2a4 100644
--- a/test/CodeGen/WinEH/cppeh-cleanup-invoke.ll
+++ b/test/CodeGen/WinEH/cppeh-cleanup-invoke.ll
@@ -26,7 +26,7 @@ $"\01??_R0H@8" = comdat any
 @"\01??_R0H@8" = linkonce_odr global %rtti.TypeDescriptor2 { i8** @"\01??_7type_info@@6B@", i8* null, [3 x i8] c".H\00" }, comdat
 @llvm.eh.handlertype.H.0 = private unnamed_addr constant %eh.CatchHandlerType { i32 0, i8* bitcast (%rtti.TypeDescriptor2* @"\01??_R0H@8" to i8*) }, section "llvm.metadata"
 
-define i32 @main() {
+define i32 @main() personality i8* bitcast (i32 (...)* @__CxxFrameHandler3 to i8*) {
 entry:
   %o = alloca %struct.HasDtor, align 1
   invoke void @may_throw()
@@ -37,14 +37,14 @@ invoke.cont2:                                     ; preds = %invoke.cont
   br label %try.cont
 
 lpad:                                             ; preds = %entry
-  %0 = landingpad { i8*, i32 } personality i8* bitcast (i32 (...)* @__CxxFrameHandler3 to i8*)
+  %0 = landingpad { i8*, i32 }
           catch %eh.CatchHandlerType* @llvm.eh.handlertype.H.0
   %1 = extractvalue { i8*, i32 } %0, 0
   %2 = extractvalue { i8*, i32 } %0, 1
   br label %catch.dispatch
 
 lpad1:                                            ; preds = %invoke.cont
-  %3 = landingpad { i8*, i32 } personality i8* bitcast (i32 (...)* @__CxxFrameHandler3 to i8*)
+  %3 = landingpad { i8*, i32 }
           cleanup
           catch %eh.CatchHandlerType* @llvm.eh.handlertype.H.0
   %4 = extractvalue { i8*, i32 } %3, 0
diff --git a/test/CodeGen/WinEH/cppeh-demote-liveout.ll b/test/CodeGen/WinEH/cppeh-demote-liveout.ll
new file mode 100644
index 00000000000..309952bfc94
--- /dev/null
+++ b/test/CodeGen/WinEH/cppeh-demote-liveout.ll
@@ -0,0 +1,72 @@
+; RUN: opt -mtriple=x86_64-pc-windows-msvc -winehprepare -S < %s | FileCheck %s
+
+; Notionally based on this C++ source:
+; int liveout_catch(int p) {
+;   int val = p + 1;
+;   try {
+;     might_throw();
+;   } catch (int) {
+;     val++;
+;   }
+;   return val;
+; }
+
+declare void @llvm.eh.begincatch(i8*, i8*)
+declare void @llvm.eh.endcatch()
+declare void @might_throw()
+declare i32 @__CxxFrameHandler3(...)
+declare i32 @llvm.eh.typeid.for(i8*)
+
+@typeinfo.int = external global i32
+
+define i32 @liveout_catch(i32 %p) personality i32 (...)* @__CxxFrameHandler3 {
+entry:
+  %val.entry = add i32 %p, 1
+  invoke void @might_throw()
+      to label %ret unwind label %lpad
+
+lpad:
+  %ehvals = landingpad { i8*, i32 }
+      cleanup
+      catch i32* @typeinfo.int
+  %ehptr = extractvalue { i8*, i32 } %ehvals, 0
+  %sel = extractvalue { i8*, i32 } %ehvals, 1
+  %int_sel = call i32 @llvm.eh.typeid.for(i8* bitcast (i32* @typeinfo.int to i8*))
+  %match = icmp eq i32 %sel, %int_sel
+  br i1 %match, label %catchit, label %resume
+
+catchit:
+  call void @llvm.eh.begincatch(i8* %ehptr, i8* null)
+  %val.lpad = add i32 %val.entry, 1
+  call void @llvm.eh.endcatch()
+  br label %ret
+
+ret:
+  %rv = phi i32 [%val.entry, %entry], [%val.lpad, %catchit]
+  ret i32 %rv
+
+resume:
+  resume {i8*, i32} %ehvals
+}
+
+; CHECK-LABEL: define i32 @liveout_catch(i32 %p)
+; CHECK: %val.entry = add i32 %p, 1
+; CHECK-NEXT: store i32 %val.entry, i32* %val.entry.reg2mem
+; CHECK: invoke void @might_throw()
+;
+; CHECK: landingpad
+; CHECK: indirectbr i8* {{.*}}, [label %catchit.split]
+;
+; CHECK: catchit.split:
+; CHECK: load i32, i32* %val.lpad.reg2mem
+; CHECK: br label %ret
+;
+; CHECK: ret:
+; CHECK: %rv = phi i32 [ {{.*}}, %entry ], [ {{.*}}, %catchit.split ]
+; CHECK: ret i32
+
+; CHECK-LABEL: define internal i8* @liveout_catch.catch(i8*, i8*)
+; CHECK: %[[val:[^ ]*]] = load i32, i32*
+; CHECK-NEXT: %[[val_lpad:[^ ]*]] = add i32 %[[val]], 1
+; CHECK-NEXT: store i32 %[[val_lpad]], i32*
+; CHECK: ret i8* blockaddress(@liveout_catch, %catchit.split)
diff --git a/test/CodeGen/WinEH/cppeh-frame-vars.ll b/test/CodeGen/WinEH/cppeh-frame-vars.ll
index 773dc942c19..c2dbd8ecab6 100644
--- a/test/CodeGen/WinEH/cppeh-frame-vars.ll
+++ b/test/CodeGen/WinEH/cppeh-frame-vars.ll
@@ -58,11 +58,11 @@ $"\01??_R0H@8" = comdat any
 ; CHECK:   [[TMP:\%.+]] = bitcast %struct.SomeData* [[DATA_PTR]] to i8*
 ; CHECK:   call void @llvm.memset(i8* [[TMP]], i8 0, i64 8, i32 4, i1 false)
 ; CHECK:   store i32 0, i32* [[I_PTR]], align 4
-; CHECK:   call void (...) @llvm.frameescape(i32* [[E_PTR]], i32* [[NUMEXCEPTIONS_PTR]], [10 x i32]* [[EXCEPTIONVAL_PTR]], i32* [[I_PTR]], %struct.SomeData* [[DATA_PTR]])
+; CHECK:   call void (...) @llvm.localescape(i32* [[E_PTR]], i32* [[NUMEXCEPTIONS_PTR]], [10 x i32]* [[EXCEPTIONVAL_PTR]], i32* [[I_PTR]], %struct.SomeData* [[DATA_PTR]])
 ; CHECK:   br label %for.cond
 
 ; Function Attrs: uwtable
-define void @"\01?test@@YAXXZ"() #0 {
+define void @"\01?test@@YAXXZ"() #0 personality i8* bitcast (i32 (...)* @__CxxFrameHandler3 to i8*) {
 entry:
   %NumExceptions = alloca i32, align 4
   %ExceptionVal = alloca [10 x i32], align 16
@@ -84,7 +84,7 @@ for.cond:                                         ; preds = %for.inc, %entry
 
 ; CHECK: for.body:
 ; CHECK:   invoke void @"\01?may_throw@@YAXXZ"()
-; CHECK:           to label %invoke.cont unwind label %[[LPAD_LABEL:lpad[0-9]+]]
+; CHECK:           to label %invoke.cont unwind label %[[LPAD_LABEL:lpad[0-9]*]]
 
 for.body:                                         ; preds = %for.cond
   invoke void @"\01?may_throw@@YAXXZ"()
@@ -99,13 +99,13 @@ invoke.cont:                                      ; preds = %for.body
   br label %try.cont
 
 ; CHECK: [[LPAD_LABEL]]:{{[ ]+}}; preds = %for.body
-; CHECK:   landingpad { i8*, i32 } personality i8* bitcast (i32 (...)* @__CxxFrameHandler3 to i8*)
+; CHECK:   landingpad { i8*, i32 }
 ; CHECK-NEXT:           catch i8* bitcast (%rtti.TypeDescriptor2* @"\01??_R0H@8" to i8*)
 ; CHECK-NEXT:   [[RECOVER:\%.+]] = call i8* (...) @llvm.eh.actions(i32 1, i8* bitcast (%rtti.TypeDescriptor2* @"\01??_R0H@8" to i8*), i32 0, i8* (i8*, i8*)* @"\01?test@@YAXXZ.catch")
 ; CHECK-NEXT:   indirectbr i8* [[RECOVER]], [label %try.cont]
 
 lpad:                                             ; preds = %for.body
-  %tmp4 = landingpad { i8*, i32 } personality i8* bitcast (i32 (...)* @__CxxFrameHandler3 to i8*)
+  %tmp4 = landingpad { i8*, i32 }
           catch i8* bitcast (%rtti.TypeDescriptor2* @"\01??_R0H@8" to i8*)
   %tmp5 = extractvalue { i8*, i32 } %tmp4, 0
   store i8* %tmp5, i8** %exn.slot
@@ -198,15 +198,15 @@ eh.resume:                                        ; preds = %catch.dispatch
 ; The following catch handler should be outlined.
 ; CHECK-LABEL: define internal i8* @"\01?test@@YAXXZ.catch"(i8*, i8*)
 ; CHECK: entry:
-; CHECK:   [[RECOVER_E:\%.+]] = call i8* @llvm.framerecover(i8* bitcast (void ()* @"\01?test@@YAXXZ" to i8*), i8* %1, i32 0)
+; CHECK:   [[RECOVER_E:\%.+]] = call i8* @llvm.localrecover(i8* bitcast (void ()* @"\01?test@@YAXXZ" to i8*), i8* %1, i32 0)
 ; CHECK:   [[E_PTR1:\%.+]] = bitcast i8* [[RECOVER_E]] to i32*
-; CHECK:   [[RECOVER_NUMEXCEPTIONS:\%.+]] = call i8* @llvm.framerecover(i8* bitcast (void ()* @"\01?test@@YAXXZ" to i8*), i8* %1, i32 1)
+; CHECK:   [[RECOVER_NUMEXCEPTIONS:\%.+]] = call i8* @llvm.localrecover(i8* bitcast (void ()* @"\01?test@@YAXXZ" to i8*), i8* %1, i32 1)
 ; CHECK:   [[NUMEXCEPTIONS_PTR1:\%.+]] = bitcast i8* [[RECOVER_NUMEXCEPTIONS]] to i32*
-; CHECK:   [[RECOVER_EXCEPTIONVAL:\%.+]] = call i8* @llvm.framerecover(i8* bitcast (void ()* @"\01?test@@YAXXZ" to i8*), i8* %1, i32 2)
+; CHECK:   [[RECOVER_EXCEPTIONVAL:\%.+]] = call i8* @llvm.localrecover(i8* bitcast (void ()* @"\01?test@@YAXXZ" to i8*), i8* %1, i32 2)
 ; CHECK:   [[EXCEPTIONVAL_PTR1:\%.+]] = bitcast i8* [[RECOVER_EXCEPTIONVAL]] to [10 x i32]*
-; CHECK:   [[RECOVER_I:\%.+]] = call i8* @llvm.framerecover(i8* bitcast (void ()* @"\01?test@@YAXXZ" to i8*), i8* %1, i32 3)
+; CHECK:   [[RECOVER_I:\%.+]] = call i8* @llvm.localrecover(i8* bitcast (void ()* @"\01?test@@YAXXZ" to i8*), i8* %1, i32 3)
 ; CHECK:   [[I_PTR1:\%.+]] = bitcast i8* [[RECOVER_I]] to i32*
-; CHECK:   [[RECOVER_DATA:\%.+]] = call i8* @llvm.framerecover(i8* bitcast (void ()* @"\01?test@@YAXXZ" to i8*), i8* %1, i32 4)
+; CHECK:   [[RECOVER_DATA:\%.+]] = call i8* @llvm.localrecover(i8* bitcast (void ()* @"\01?test@@YAXXZ" to i8*), i8* %1, i32 4)
 ; CHECK:   [[DATA_PTR1:\%.+]] = bitcast i8* [[RECOVER_DATA]] to %struct.SomeData*
 ; CHECK:   [[TMP:\%.+]] = load i32, i32* [[E_PTR1]], align 4
 ; CHECK:   [[TMP1:\%.+]] = load i32, i32* [[NUMEXCEPTIONS_PTR]], align 4
diff --git a/test/CodeGen/WinEH/cppeh-inalloca.ll b/test/CodeGen/WinEH/cppeh-inalloca.ll
index 93ec600ccd5..649c5e72e2d 100644
--- a/test/CodeGen/WinEH/cppeh-inalloca.ll
+++ b/test/CodeGen/WinEH/cppeh-inalloca.ll
@@ -1,4 +1,4 @@
-; RUN: opt -mtriple=i386-pc-windows-msvc -winehprepare -S -o - < %s | FileCheck %s
+; RUN: opt -mtriple=x86_64-pc-windows-msvc -winehprepare -S -o - < %s | FileCheck %s
 
 ; This test is built from the following code:
 ; struct A {
@@ -24,8 +24,6 @@
 ; the inalloca instruction was manually sunk into the landingpad.
 
 ; ModuleID = 'cppeh-inalloca.cpp'
-target datalayout = "e-m:w-p:32:32-i64:64-f80:32-n8:16:32-S32"
-target triple = "i386-pc-windows-msvc"
 
 %rtti.TypeDescriptor2 = type { i8**, i8*, [3 x i8] }
 %struct.A = type { i32 }
@@ -39,16 +37,15 @@ $"\01??_R0H@8" = comdat any
 ; CHECK: define i32 @"\01?test@@YAHUA@@@Z"(<{ %struct.A }>* inalloca)
 ; CHECK: entry:
 ; CHECK:   [[TMP_REGMEM:\%.+]] = alloca <{ %struct.A }>*
-; CHECK:   [[TMP:\%.+]] = select i1 true, <{ %struct.A }>* %0, <{ %struct.A }>* undef
-; CHECK:   store <{ %struct.A }>* [[TMP]], <{ %struct.A }>** [[TMP_REGMEM]]
+; CHECK:   store <{ %struct.A }>* %0, <{ %struct.A }>** [[TMP_REGMEM]]
 ; CHECK:   [[RETVAL:\%.+]] = alloca i32, align 4
 ; CHECK:   [[E_PTR:\%.+]] = alloca i32, align 4
 ; CHECK:   [[CLEANUP_SLOT:\%.+]] = alloca i32
-; CHECK:   call void (...) @llvm.frameescape(i32* %e, <{ %struct.A }>** [[TMP_REGMEM]], i32* [[RETVAL]], i32* [[CLEANUP_SLOT]])
+; CHECK:   call void (...) @llvm.localescape(i32* %e, <{ %struct.A }>** [[TMP_REGMEM]], i32* [[RETVAL]], i32* [[CLEANUP_SLOT]])
 ; CHECK:   invoke void @"\01?may_throw@@YAXXZ"()
 ; CHECK:           to label %invoke.cont unwind label %[[LPAD_LABEL:lpad[0-9]*]]
 
-define i32 @"\01?test@@YAHUA@@@Z"(<{ %struct.A }>* inalloca) #0 {
+define i32 @"\01?test@@YAHUA@@@Z"(<{ %struct.A }>* inalloca) #0 personality i8* bitcast (i32 (...)* @__CxxFrameHandler3 to i8*) {
 entry:
   %retval = alloca i32, align 4
   %exn.slot = alloca i8*
@@ -62,14 +59,14 @@ invoke.cont:                                      ; preds = %entry
   br label %try.cont
 
 ; CHECK: [[LPAD_LABEL]]:{{[ ]+}}; preds = %entry
-; CHECK:   landingpad { i8*, i32 } personality i8* bitcast (i32 (...)* @__CxxFrameHandler3 to i8*)
+; CHECK:   landingpad { i8*, i32 }
 ; CHECK-NEXT:           cleanup
 ; CHECK-NEXT:           catch i8* bitcast (%rtti.TypeDescriptor2* @"\01??_R0H@8" to i8*)
 ; CHECK-NEXT:   [[RECOVER:\%recover.*]] = call i8* (...) @llvm.eh.actions(i32 1, i8* bitcast (%rtti.TypeDescriptor2* @"\01??_R0H@8" to i8*), i32 0, i8* (i8*, i8*)* @"\01?test@@YAHUA@@@Z.catch", i32 0, void (i8*, i8*)* @"\01?test@@YAHUA@@@Z.cleanup")
 ; CHECK-NEXT:   indirectbr i8* [[RECOVER]], [label %cleanup]
 
 lpad:                                             ; preds = %entry
-  %1 = landingpad { i8*, i32 } personality i8* bitcast (i32 (...)* @__CxxFrameHandler3 to i8*)
+  %1 = landingpad { i8*, i32 }
           cleanup
           catch i8* bitcast (%rtti.TypeDescriptor2* @"\01??_R0H@8" to i8*)
   %2 = extractvalue { i8*, i32 } %1, 0
@@ -109,10 +106,8 @@ try.cont:                                         ; preds = %invoke.cont
 
 ; The cleanup block should be re-written like this.
 ; CHECK: cleanup:{{[ ]+}}; preds = %[[LPAD_LABEL]], %try.cont
-; CHECK-NOT:  %a2 = getelementptr inbounds <{ %struct.A }>, <{ %struct.A }>* %0, i32 0, i32 0
-; CHECK:   [[TMP_RELOAD:\%.+]] = load volatile <{ %struct.A }>*, <{ %struct.A }>** [[TMP_REGMEM]]
-; CHECK:   [[A2:\%.+]] = getelementptr inbounds <{ %struct.A }>, <{ %struct.A }>* [[TMP_RELOAD]], i32 0, i32 0
-; CHECK:   call x86_thiscallcc void @"\01??1A@@QAE@XZ"(%struct.A* [[A2]])
+; CHECK:   %a2 = getelementptr inbounds <{ %struct.A }>, <{ %struct.A }>* %0, i32 0, i32 0
+; CHECK:   call x86_thiscallcc void @"\01??1A@@QAE@XZ"(%struct.A* %a2)
 ; CHECK:   [[TMP1:\%.+]] = load i32, i32* [[RETVAL]]
 ; CHECK:   ret i32 [[TMP1]]
 
@@ -144,16 +139,16 @@ eh.resume:                                        ; preds = %ehcleanup
 ; The following catch handler should be outlined.
 ; CHECK: define internal i8* @"\01?test@@YAHUA@@@Z.catch"(i8*, i8*)
 ; CHECK: entry:
-; CHECK:   [[RECOVER_E:\%.+]] = call i8* @llvm.framerecover(i8* bitcast (i32 (<{ %struct.A }>*)* @"\01?test@@YAHUA@@@Z" to i8*), i8* %1, i32 0)
+; CHECK:   [[RECOVER_E:\%.+]] = call i8* @llvm.localrecover(i8* bitcast (i32 (<{ %struct.A }>*)* @"\01?test@@YAHUA@@@Z" to i8*), i8* %1, i32 0)
 ; CHECK:   [[E_PTR:\%.+]] = bitcast i8* [[RECOVER_E]] to i32*
-; CHECK:   [[RECOVER_EH_TEMP:\%.+]] = call i8* @llvm.framerecover(i8* bitcast (i32 (<{ %struct.A }>*)* @"\01?test@@YAHUA@@@Z" to i8*), i8* %1, i32 1)
+; CHECK:   [[RECOVER_EH_TEMP:\%.+]] = call i8* @llvm.localrecover(i8* bitcast (i32 (<{ %struct.A }>*)* @"\01?test@@YAHUA@@@Z" to i8*), i8* %1, i32 1)
 ; CHECK:   [[EH_TEMP:\%.+]] = bitcast i8* [[RECOVER_EH_TEMP]] to <{ %struct.A }>**
-; CHECK:   [[TMP_RELOAD:\%.+]] = load <{ %struct.A }>*, <{ %struct.A }>** [[EH_TEMP]]
-; CHECK:   [[RECOVER_RETVAL:\%.+]] = call i8* @llvm.framerecover(i8* bitcast (i32 (<{ %struct.A }>*)* @"\01?test@@YAHUA@@@Z" to i8*), i8* %1, i32 2)
+; CHECK:   [[RECOVER_RETVAL:\%.+]] = call i8* @llvm.localrecover(i8* bitcast (i32 (<{ %struct.A }>*)* @"\01?test@@YAHUA@@@Z" to i8*), i8* %1, i32 2)
 ; CHECK:   [[RETVAL1:\%.+]] = bitcast i8* [[RECOVER_RETVAL]] to i32*
-; CHECK:   [[RECOVER_CLEANUPSLOT:\%.+]] = call i8* @llvm.framerecover(i8* bitcast (i32 (<{ %struct.A }>*)* @"\01?test@@YAHUA@@@Z" to i8*), i8* %1, i32 3)
+; CHECK:   [[RECOVER_CLEANUPSLOT:\%.+]] = call i8* @llvm.localrecover(i8* bitcast (i32 (<{ %struct.A }>*)* @"\01?test@@YAHUA@@@Z" to i8*), i8* %1, i32 3)
 ; CHECK:   [[CLEANUPSLOT1:\%.+]] = bitcast i8* [[RECOVER_CLEANUPSLOT]] to i32*
 ; CHECK:   [[E_I8PTR:\%.+]] = bitcast i32* [[E_PTR]] to i8*
+; CHECK:   [[TMP_RELOAD:\%.+]] = load <{ %struct.A }>*, <{ %struct.A }>** [[EH_TEMP]]
 ; CHECK:   [[RECOVER_A:\%.+]] = getelementptr inbounds <{ %struct.A }>, <{ %struct.A }>* [[TMP_RELOAD]], i32 0, i32 0
 ; CHECK:   [[A1:\%.+]] = getelementptr inbounds %struct.A, %struct.A* [[RECOVER_A]], i32 0, i32 0
 ; CHECK:   [[TMP2:\%.+]] = load i32, i32* [[A1]], align 4
@@ -167,7 +162,7 @@ eh.resume:                                        ; preds = %ehcleanup
 ; The following cleanup handler should be outlined.
 ; CHECK: define internal void @"\01?test@@YAHUA@@@Z.cleanup"(i8*, i8*)
 ; CHECK: entry:
-; CHECK:   [[RECOVER_EH_TEMP1:\%.+]] = call i8* @llvm.framerecover(i8* bitcast (i32 (<{ %struct.A }>*)* @"\01?test@@YAHUA@@@Z" to i8*), i8* %1, i32 1)
+; CHECK:   [[RECOVER_EH_TEMP1:\%.+]] = call i8* @llvm.localrecover(i8* bitcast (i32 (<{ %struct.A }>*)* @"\01?test@@YAHUA@@@Z" to i8*), i8* %1, i32 1)
 ; CHECK:   [[EH_TEMP1:\%.+]] = bitcast i8* [[RECOVER_EH_TEMP]] to <{ %struct.A }>**
 ; CHECK:   [[TMP_RELOAD1:\%.+]] = load <{ %struct.A }>*, <{ %struct.A }>** [[EH_TEMP1]]
 ; CHECK:   [[A3:\%.+]] = getelementptr inbounds <{ %struct.A }>, <{ %struct.A }>* [[TMP_RELOAD1]], i32 0, i32 0
diff --git a/test/CodeGen/WinEH/cppeh-min-unwind.ll b/test/CodeGen/WinEH/cppeh-min-unwind.ll
index 7868b33f716..98d6d6fcacb 100644
--- a/test/CodeGen/WinEH/cppeh-min-unwind.ll
+++ b/test/CodeGen/WinEH/cppeh-min-unwind.ll
@@ -25,12 +25,12 @@ target triple = "x86_64-pc-windows-msvc"
 ; CHECK: entry:
 ; CHECK:   [[OBJ_PTR:\%.+]] = alloca %class.SomeClass, align 4
 ; CHECK:   call void @_ZN9SomeClassC1Ev(%class.SomeClass* [[OBJ_PTR]])
-; CHECK:   call void (...) @llvm.frameescape(%class.SomeClass* [[OBJ_PTR]])
+; CHECK:   call void (...) @llvm.localescape(%class.SomeClass* [[OBJ_PTR]])
 ; CHECK:   invoke void @_Z9may_throwv()
-; CHECK:           to label %invoke.cont unwind label %[[LPAD_LABEL:lpad[0-9]+]]
+; CHECK:           to label %invoke.cont unwind label %[[LPAD_LABEL:lpad[0-9]*]]
 
 ; Function Attrs: uwtable
-define void @_Z4testv() #0 {
+define void @_Z4testv() #0 personality i8* bitcast (i32 (...)* @__CxxFrameHandler3 to i8*) {
 entry:
   %obj = alloca %class.SomeClass, align 4
   %exn.slot = alloca i8*
@@ -44,13 +44,13 @@ invoke.cont:                                      ; preds = %entry
   ret void
 
 ; CHECK: [[LPAD_LABEL]]:{{[ ]+}}; preds = %entry
-; CHECK:   landingpad { i8*, i32 } personality i8* bitcast (i32 (...)* @__CxxFrameHandler3 to i8*)
+; CHECK:   landingpad { i8*, i32 }
 ; CHECK-NEXT:           cleanup
 ; CHECK-NEXT:   [[RECOVER:\%.+]] = call i8* (...) @llvm.eh.actions(i32 0, void (i8*, i8*)* @_Z4testv.cleanup)
 ; CHECK-NEXT:   indirectbr i8* [[RECOVER]], []
 
 lpad:                                             ; preds = %entry
-  %tmp = landingpad { i8*, i32 } personality i8* bitcast (i32 (...)* @__CxxFrameHandler3 to i8*)
+  %tmp = landingpad { i8*, i32 }
           cleanup
   %tmp1 = extractvalue { i8*, i32 } %tmp, 0
   store i8* %tmp1, i8** %exn.slot
@@ -74,7 +74,7 @@ eh.resume:                                        ; preds = %lpad
 ; This cleanup handler should be outlined.
 ; CHECK: define internal void @_Z4testv.cleanup(i8*, i8*)
 ; CHECK: entry:
-; CHECK:   [[RECOVER_OBJ:\%.+]] = call i8* @llvm.framerecover(i8* bitcast (void ()* @_Z4testv to i8*), i8* %1, i32 0)
+; CHECK:   [[RECOVER_OBJ:\%.+]] = call i8* @llvm.localrecover(i8* bitcast (void ()* @_Z4testv to i8*), i8* %1, i32 0)
 ; CHECK:   [[OBJ_PTR1:\%.+]] = bitcast i8* [[RECOVER_OBJ]] to %class.SomeClass*
 ; CHECK:   call void @_ZN9SomeClassD1Ev(%class.SomeClass* [[OBJ_PTR1]])
 ; CHECK:   ret void
diff --git a/test/CodeGen/WinEH/cppeh-mixed-catch-and-cleanup.ll b/test/CodeGen/WinEH/cppeh-mixed-catch-and-cleanup.ll
new file mode 100644
index 00000000000..c69633f17e2
--- /dev/null
+++ b/test/CodeGen/WinEH/cppeh-mixed-catch-and-cleanup.ll
@@ -0,0 +1,106 @@
+; RUN: opt -mtriple=x86_64-pc-windows-msvc -winehprepare -S -o - < %s | FileCheck %s
+
+; This test is based on the following code:
+;
+; void test()
+; {
+;   try {
+;     Obj o;
+;     may_throw();
+;   } catch (...) {
+;   }
+; }
+;
+; The purpose of this test is to verify that we create separate catch and
+; cleanup handlers.  When compiling for the C++ 11 standard, this isn't
+; strictly necessary, since calling the destructor from the catch handler
+; would be logically equivalent to calling it from a cleanup handler.
+; However, if the -std=c++98 option is used, an exception in the cleanup
+; code should terminate the process (the MSVCRT runtime will do that) but
+; if the destructor is called from the catch handler, it wouldn't terminate
+; the process
+
+
+; ModuleID = 'cppeh-mixed-catch-and-cleanup.cpp'
+target datalayout = "e-m:w-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-pc-windows-msvc"
+
+%class.Obj = type { i8 }
+
+; This just verifies that the function was processed by WinEHPrepare.
+;
+; CHECK-LABEL: define void @"\01?test@@YAXXZ"()
+; CHECK: entry:
+; CHECK:   call void (...) @llvm.localescape
+; CHECK: }
+
+; Function Attrs: nounwind uwtable
+define void @"\01?test@@YAXXZ"() #0 personality i8* bitcast (i32 (...)* @__CxxFrameHandler3 to i8*) {
+entry:
+  %o = alloca %class.Obj, align 1
+  %exn.slot = alloca i8*
+  %ehselector.slot = alloca i32
+  invoke void @"\01?may_throw@@YAXXZ"()
+          to label %invoke.cont unwind label %lpad
+
+invoke.cont:                                      ; preds = %entry
+  call void @"\01??1Obj@@QEAA@XZ"(%class.Obj* %o) #3
+  br label %try.cont
+
+lpad:                                             ; preds = %entry
+  %0 = landingpad { i8*, i32 }
+          catch i8* null
+  %1 = extractvalue { i8*, i32 } %0, 0
+  store i8* %1, i8** %exn.slot
+  %2 = extractvalue { i8*, i32 } %0, 1
+  store i32 %2, i32* %ehselector.slot
+  call void @"\01??1Obj@@QEAA@XZ"(%class.Obj* %o) #3
+  %exn = load i8*, i8** %exn.slot
+  call void @llvm.eh.begincatch(i8* %exn, i8* null) #3
+  call void @llvm.eh.endcatch() #3
+  br label %try.cont
+
+try.cont:                                         ; preds = %catch, %invoke.cont
+  ret void
+}
+
+; Verify that a cleanup handler was created and that it calls ~Obj().
+; CHECK-LABEL: define internal void @"\01?test@@YAXXZ.cleanup"(i8*, i8*)
+; CHECK: entry:
+; CHECK:   @llvm.localrecover(i8* bitcast (void ()* @"\01?test@@YAXXZ" to i8*), i8* %1, i32 0)
+; CHECK:   call void @"\01??1Obj@@QEAA@XZ"
+; CHECK:   ret void
+; CHECK: }
+
+; Verify that a catch handler was created and that it does not call ~Obj().
+; CHECK-LABEL: define internal i8* @"\01?test@@YAXXZ.catch"(i8*, i8*)
+; CHECK: entry:
+; CHECK-NOT:  call void @"\01??1Obj@@QEAA@XZ"
+; CHECK:   ret i8* blockaddress(@"\01?test@@YAXXZ", %try.cont)
+; CHECK: }
+
+
+
+declare void @"\01?may_throw@@YAXXZ"() #1
+
+declare i32 @__CxxFrameHandler3(...)
+
+; Function Attrs: nounwind
+declare void @"\01??1Obj@@QEAA@XZ"(%class.Obj*) #2
+
+; Function Attrs: nounwind
+declare void @llvm.eh.begincatch(i8* nocapture, i8* nocapture) #3
+
+; Function Attrs: nounwind
+declare void @llvm.eh.endcatch() #3
+
+attributes #0 = { nounwind uwtable "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #1 = { "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #2 = { nounwind "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #3 = { nounwind }
+
+!llvm.module.flags = !{!0}
+!llvm.ident = !{!1}
+
+!0 = !{i32 1, !"PIC Level", i32 2}
+!1 = !{!"clang version 3.7.0 (trunk 235779) (llvm/trunk 235769)"}
diff --git a/test/CodeGen/WinEH/cppeh-multi-catch.ll b/test/CodeGen/WinEH/cppeh-multi-catch.ll
index 6052629198c..266cdea20cd 100644
--- a/test/CodeGen/WinEH/cppeh-multi-catch.ll
+++ b/test/CodeGen/WinEH/cppeh-multi-catch.ll
@@ -45,17 +45,17 @@ $"\01??_R0?AVSomeClass@@@8" = comdat any
 @"llvm.eh.handlermapentry.reference.?AVSomeClass@@" = private unnamed_addr constant %eh.HandlerMapEntry { i32 8, i32 trunc (i64 sub nuw nsw (i64 ptrtoint (%rtti.TypeDescriptor15* @"\01??_R0?AVSomeClass@@@8" to i64), i64 ptrtoint (i8* @__ImageBase to i64)) to i32) }, section "llvm.metadata"
 
 
-; CHECK: define void @"\01?test@@YAXXZ"() #0 {
+; CHECK: define void @"\01?test@@YAXXZ"() #0 personality i8* bitcast (i32 (...)* @__CxxFrameHandler3 to i8*) {
 ; CHECK: entry:
 ; CHECK:   [[OBJ_PTR:\%.+]] = alloca %class.SomeClass*, align 8
 ; CHECK:   [[LL_PTR:\%.+]] = alloca i64, align 8
 ; CHECK:   [[I_PTR:\%.+]] = alloca i32, align 4
-; CHECK:   call void (...) @llvm.frameescape(i32* [[I_PTR]], i64* [[LL_PTR]], %class.SomeClass** [[OBJ_PTR]])
+; CHECK:   call void (...) @llvm.localescape(i32* [[I_PTR]], i64* [[LL_PTR]], %class.SomeClass** [[OBJ_PTR]])
 ; CHECK:   invoke void @"\01?may_throw@@YAXXZ"()
-; CHECK:           to label %invoke.cont unwind label %[[LPAD_LABEL:lpad[0-9]+]]
+; CHECK:           to label %invoke.cont unwind label %[[LPAD_LABEL:lpad[0-9]*]]
 
 ; Function Attrs: uwtable
-define void @"\01?test@@YAXXZ"() #0 {
+define void @"\01?test@@YAXXZ"() #0 personality i8* bitcast (i32 (...)* @__CxxFrameHandler3 to i8*) {
 entry:
   %exn.slot = alloca i8*
   %ehselector.slot = alloca i32
@@ -69,20 +69,20 @@ invoke.cont:                                      ; preds = %entry
   br label %try.cont
 
 ; CHECK: [[LPAD_LABEL]]:{{[ ]+}}; preds = %entry
-; CHECK:   landingpad { i8*, i32 } personality i8* bitcast (i32 (...)* @__CxxFrameHandler3 to i8*)
+; CHECK:   landingpad { i8*, i32 }
 ; CHECK-NEXT:           catch %eh.HandlerMapEntry* @llvm.eh.handlermapentry.H
 ; CHECK-NEXT:           catch %eh.HandlerMapEntry* @llvm.eh.handlermapentry._J
 ; CHECK-NEXT:           catch %eh.HandlerMapEntry* @"llvm.eh.handlermapentry.reference.?AVSomeClass@@"
 ; CHECK-NEXT:           catch i8* null
 ; CHECK-NEXT:   [[RECOVER:\%.+]] = call i8* (...) @llvm.eh.actions(
 ; CHECK-SAME:	     i32 1, i8* bitcast (%eh.HandlerMapEntry* @llvm.eh.handlermapentry.H to i8*), i32 0, i8* (i8*, i8*)* @"\01?test@@YAXXZ.catch",
-; CHECK-SAME:        i32 1, i8* bitcast (%eh.HandlerMapEntry* @llvm.eh.handlermapentry._J to i8*), i32 1, i8* (i8*, i8*)* @"\01?test@@YAXXZ.catch1",
-; CHECK-SAME:        i32 1, i8* bitcast (%eh.HandlerMapEntry* @"llvm.eh.handlermapentry.reference.?AVSomeClass@@" to i8*), i32 2, i8* (i8*, i8*)* @"\01?test@@YAXXZ.catch2",
-; CHECK-SAME:        i32 1, i8* null, i32 -1, i8* (i8*, i8*)* @"\01?test@@YAXXZ.catch3")
-; CHECK-NEXT:   indirectbr i8* [[RECOVER]], [label %catch14.split, label %catch10.split, label %catch6.split, label %catch.split]
+; CHECK-SAME:        i32 1, i8* bitcast (%eh.HandlerMapEntry* @llvm.eh.handlermapentry._J to i8*), i32 1, i8* (i8*, i8*)* @"\01?test@@YAXXZ.catch.1",
+; CHECK-SAME:        i32 1, i8* bitcast (%eh.HandlerMapEntry* @"llvm.eh.handlermapentry.reference.?AVSomeClass@@" to i8*), i32 2, i8* (i8*, i8*)* @"\01?test@@YAXXZ.catch.2",
+; CHECK-SAME:        i32 1, i8* null, i32 -1, i8* (i8*, i8*)* @"\01?test@@YAXXZ.catch.3")
+; CHECK-NEXT:   indirectbr i8* [[RECOVER]], [label %ret]
 
 lpad:                                             ; preds = %entry
-  %0 = landingpad { i8*, i32 } personality i8* bitcast (i32 (...)* @__CxxFrameHandler3 to i8*)
+  %0 = landingpad { i8*, i32 }
           catch %eh.HandlerMapEntry* @llvm.eh.handlermapentry.H
           catch %eh.HandlerMapEntry* @llvm.eh.handlermapentry._J
           catch %eh.HandlerMapEntry* @"llvm.eh.handlermapentry.reference.?AVSomeClass@@"
@@ -100,8 +100,11 @@ catch.dispatch:                                   ; preds = %lpad
   %matches = icmp eq i32 %sel, %3
   br i1 %matches, label %catch14, label %catch.fallthrough
 
+ret:
+  ret void
+
 ; CHECK-NOT: catch14:
-; CHECK: catch14.split:
+; CHECK: ret:
 ; CHECK-NEXT:   ret void
 catch14:                                          ; preds = %catch.dispatch
   %exn15 = load i8*, i8** %exn.slot
@@ -110,10 +113,10 @@ catch14:                                          ; preds = %catch.dispatch
   %5 = load i32, i32* %i, align 4
   call void @"\01?handle_int@@YAXH@Z"(i32 %5)
   call void @llvm.eh.endcatch() #3
-  ret void
+  br label %ret
 
 try.cont:                                         ; preds = %invoke.cont
-  ret void
+  br label %ret
 
 ; CHECK-NOT: catch.fallthrough:
 catch.fallthrough:                                ; preds = %catch.dispatch
@@ -122,8 +125,6 @@ catch.fallthrough:                                ; preds = %catch.dispatch
   br i1 %matches1, label %catch10, label %catch.fallthrough2
 
 ; CHECK-NOT: catch10:
-; CHECK: catch10.split:
-; CHECK-NEXT:   ret void
 catch10:                                          ; preds = %catch.fallthrough
   %exn11 = load i8*, i8** %exn.slot
   %7 = bitcast i64* %ll to i8*
@@ -131,7 +132,7 @@ catch10:                                          ; preds = %catch.fallthrough
   %8 = load i64, i64* %ll, align 8
   call void @"\01?handle_long_long@@YAX_J@Z"(i64 %8)
   call void @llvm.eh.endcatch() #3
-  ret void
+  br label %ret
 
 ; CHECK-NOT: catch.fallthrough2:
 catch.fallthrough2:                               ; preds = %catch.fallthrough
@@ -140,8 +141,6 @@ catch.fallthrough2:                               ; preds = %catch.fallthrough
   br i1 %matches3, label %catch6, label %catch
 
 ; CHECK-NOT: catch6:
-; CHECK: catch6.split:
-; CHECK-NEXT:   ret void
 catch6:                                           ; preds = %catch.fallthrough2
   %exn7 = load i8*, i8** %exn.slot
   %10 = bitcast %class.SomeClass** %obj to i8*
@@ -149,50 +148,48 @@ catch6:                                           ; preds = %catch.fallthrough2
   %11 = load %class.SomeClass*, %class.SomeClass** %obj, align 8
   call void @"\01?handle_obj@@YAXPEAVSomeClass@@@Z"(%class.SomeClass* %11)
   call void @llvm.eh.endcatch() #3
-  ret void
+  br label %ret
 
 ; CHECK-NOT: catch:
-; CHECK: catch.split:
-; CHECK-NEXT:   ret void
 catch:                                            ; preds = %catch.fallthrough2
   %exn = load i8*, i8** %exn.slot
   call void @llvm.eh.begincatch(i8* %exn, i8* null) #3
   call void @"\01?handle_exception@@YAXXZ"()  call void @llvm.eh.endcatch() #3
-  ret void
+  br label %ret
 ; CHECK: }
 }
 
 ; CHECK-LABEL: define internal i8* @"\01?test@@YAXXZ.catch"(i8*, i8*)
 ; CHECK: entry:
-; CHECK:   [[RECOVER_I:\%.+]] = call i8* @llvm.framerecover(i8* bitcast (void ()* @"\01?test@@YAXXZ" to i8*), i8* %1, i32 0)
+; CHECK:   [[RECOVER_I:\%.+]] = call i8* @llvm.localrecover(i8* bitcast (void ()* @"\01?test@@YAXXZ" to i8*), i8* %1, i32 0)
 ; CHECK:   [[I_PTR:\%.+]] = bitcast i8* [[RECOVER_I]] to i32*
 ; CHECK:   [[TMP1:\%.+]] = load i32, i32* [[I_PTR]], align 4
 ; CHECK:   call void @"\01?handle_int@@YAXH@Z"(i32 [[TMP1]])
-; CHECK:   ret i8* blockaddress(@"\01?test@@YAXXZ", %catch14.split)
+; CHECK:   ret i8* blockaddress(@"\01?test@@YAXXZ", %ret)
 ; CHECK: }
 
-; CHECK-LABEL: define internal i8* @"\01?test@@YAXXZ.catch1"(i8*, i8*)
+; CHECK-LABEL: define internal i8* @"\01?test@@YAXXZ.catch.1"(i8*, i8*)
 ; CHECK: entry:
-; CHECK:   [[RECOVER_LL:\%.+]] = call i8* @llvm.framerecover(i8* bitcast (void ()* @"\01?test@@YAXXZ" to i8*), i8* %1, i32 1)
+; CHECK:   [[RECOVER_LL:\%.+]] = call i8* @llvm.localrecover(i8* bitcast (void ()* @"\01?test@@YAXXZ" to i8*), i8* %1, i32 1)
 ; CHECK:   [[LL_PTR:\%.+]] = bitcast i8* [[RECOVER_LL]] to i64*
 ; CHECK:   [[TMP2:\%.+]] = load i64, i64* [[LL_PTR]], align 8
 ; CHECK:   call void @"\01?handle_long_long@@YAX_J@Z"(i64 [[TMP2]])
-; CHECK:   ret i8* blockaddress(@"\01?test@@YAXXZ", %catch10.split)
+; CHECK:   ret i8* blockaddress(@"\01?test@@YAXXZ", %ret)
 ; CHECK: }
 
-; CHECK-LABEL: define internal i8* @"\01?test@@YAXXZ.catch2"(i8*, i8*)
+; CHECK-LABEL: define internal i8* @"\01?test@@YAXXZ.catch.2"(i8*, i8*)
 ; CHECK: entry:
-; CHECK:   [[RECOVER_OBJ:\%.+]] = call i8* @llvm.framerecover(i8* bitcast (void ()* @"\01?test@@YAXXZ" to i8*), i8* %1, i32 2)
+; CHECK:   [[RECOVER_OBJ:\%.+]] = call i8* @llvm.localrecover(i8* bitcast (void ()* @"\01?test@@YAXXZ" to i8*), i8* %1, i32 2)
 ; CHECK:   [[OBJ_PTR:\%.+]] = bitcast i8* [[RECOVER_OBJ]] to %class.SomeClass**
 ; CHECK:   [[TMP3:\%.+]] = load %class.SomeClass*, %class.SomeClass** [[OBJ_PTR]], align 8
 ; CHECK:   call void @"\01?handle_obj@@YAXPEAVSomeClass@@@Z"(%class.SomeClass* [[TMP3]])
-; CHECK:   ret i8* blockaddress(@"\01?test@@YAXXZ", %catch6.split)
+; CHECK:   ret i8* blockaddress(@"\01?test@@YAXXZ", %ret)
 ; CHECK: }
 
-; CHECK-LABEL: define internal i8* @"\01?test@@YAXXZ.catch3"(i8*, i8*)
+; CHECK-LABEL: define internal i8* @"\01?test@@YAXXZ.catch.3"(i8*, i8*)
 ; CHECK: entry:
 ; CHECK:   call void @"\01?handle_exception@@YAXXZ"()
-; CHECK:   ret i8* blockaddress(@"\01?test@@YAXXZ", %catch.split)
+; CHECK:   ret i8* blockaddress(@"\01?test@@YAXXZ", %ret)
 ; CHECK: }
 
 
diff --git a/test/CodeGen/WinEH/cppeh-nested-1.ll b/test/CodeGen/WinEH/cppeh-nested-1.ll
index 5cd2b189036..d525d8a1a67 100644
--- a/test/CodeGen/WinEH/cppeh-nested-1.ll
+++ b/test/CodeGen/WinEH/cppeh-nested-1.ll
@@ -34,12 +34,12 @@ $"\01??_R0H@8" = comdat any
 ; CHECK: entry:
 ; CHECK:   %i = alloca i32, align 4
 ; CHECK:   %f = alloca float, align 4
-; CHECK:   call void (...) @llvm.frameescape(i32* %i, float* %f)
+; CHECK:   call void (...) @llvm.localescape(float* %f, i32* %i)
 ; CHECK:   invoke void @"\01?may_throw@@YAXXZ"()
 ; CHECK:           to label %invoke.cont unwind label %[[LPAD_LABEL:lpad[0-9]*]]
 
 ; Function Attrs: uwtable
-define void @"\01?test@@YAXXZ"() #0 {
+define void @"\01?test@@YAXXZ"() #0 personality i8* bitcast (i32 (...)* @__CxxFrameHandler3 to i8*) {
 entry:
   %exn.slot = alloca i8*
   %ehselector.slot = alloca i32
@@ -52,14 +52,14 @@ invoke.cont:                                      ; preds = %entry
   br label %try.cont
 
 ; CHECK: [[LPAD_LABEL]]:
-; CHECK:   landingpad { i8*, i32 } personality i8* bitcast (i32 (...)* @__CxxFrameHandler3 to i8*)
+; CHECK:   landingpad { i8*, i32 }
 ; CHECK:           catch i8* bitcast (%rtti.TypeDescriptor2* @"\01??_R0H@8" to i8*)
 ; CHECK:           catch i8* bitcast (%rtti.TypeDescriptor2* @"\01??_R0M@8" to i8*)
-; CHECK:   [[RECOVER:\%.+]] = call i8* (...) @llvm.eh.actions(i32 1, i8* bitcast (%rtti.TypeDescriptor2* @"\01??_R0H@8" to i8*), i32 0, i8* (i8*, i8*)* @"\01?test@@YAXXZ.catch", i32 1, i8* bitcast (%rtti.TypeDescriptor2* @"\01??_R0M@8" to i8*), i32 1, i8* (i8*, i8*)* @"\01?test@@YAXXZ.catch1")
+; CHECK:   [[RECOVER:\%.+]] = call i8* (...) @llvm.eh.actions(i32 1, i8* bitcast (%rtti.TypeDescriptor2* @"\01??_R0H@8" to i8*), i32 1, i8* (i8*, i8*)* @"\01?test@@YAXXZ.catch.1", i32 1, i8* bitcast (%rtti.TypeDescriptor2* @"\01??_R0M@8" to i8*), i32 0, i8* (i8*, i8*)* @"\01?test@@YAXXZ.catch")
 ; CHECK:   indirectbr i8* [[RECOVER]], [label %try.cont, label %try.cont10]
 
 lpad:                                             ; preds = %entry
-  %0 = landingpad { i8*, i32 } personality i8* bitcast (i32 (...)* @__CxxFrameHandler3 to i8*)
+  %0 = landingpad { i8*, i32 }
           catch i8* bitcast (%rtti.TypeDescriptor2* @"\01??_R0H@8" to i8*)
           catch i8* bitcast (%rtti.TypeDescriptor2* @"\01??_R0M@8" to i8*)
   %1 = extractvalue { i8*, i32 } %0, 0
@@ -94,7 +94,7 @@ try.cont:                                         ; preds = %invoke.cont2, %invo
 
 ; CHECK-NOT: lpad1:
 lpad1:                                            ; preds = %catch
-  %6 = landingpad { i8*, i32 } personality i8* bitcast (i32 (...)* @__CxxFrameHandler3 to i8*)
+  %6 = landingpad { i8*, i32 }
           catch i8* bitcast (%rtti.TypeDescriptor2* @"\01??_R0M@8" to i8*)
   %7 = extractvalue { i8*, i32 } %6, 0
   store i8* %7, i8** %exn.slot
@@ -136,7 +136,16 @@ eh.resume:                                        ; %catch.dispatch3
 
 ; CHECK: define internal i8* @"\01?test@@YAXXZ.catch"(i8*, i8*)
 ; CHECK: entry:
-; CHECK:   [[RECOVER_I:\%.+]] = call i8* @llvm.framerecover(i8* bitcast (void ()* @"\01?test@@YAXXZ" to i8*), i8* %1, i32 0)
+; CHECK:   [[RECOVER_F1:\%.+]] = call i8* @llvm.localrecover(i8* bitcast (void ()* @"\01?test@@YAXXZ" to i8*), i8* %1, i32 0)
+; CHECK:   [[F_PTR1:\%.+]] = bitcast i8* [[RECOVER_F1]] to float*
+; CHECK:   [[TMP2:\%.+]] = load float, float* [[F_PTR1]], align 4
+; CHECK:   call void @"\01?handle_float@@YAXM@Z"(float [[TMP2]])
+; CHECK:   ret i8* blockaddress(@"\01?test@@YAXXZ", %try.cont10)
+; CHECK: }
+
+; CHECK: define internal i8* @"\01?test@@YAXXZ.catch.1"(i8*, i8*)
+; CHECK: entry:
+; CHECK:   [[RECOVER_I:\%.+]] = call i8* @llvm.localrecover(i8* bitcast (void ()* @"\01?test@@YAXXZ" to i8*), i8* %1, i32 1)
 ; CHECK:   [[I_PTR:\%.+]] = bitcast i8* [[RECOVER_I]] to i32*
 ; CHECK:   [[TMP1:\%.+]] = load i32, i32* [[I_PTR]], align 4
 ; CHECK:   invoke void @"\01?handle_int@@YAXH@Z"(i32 [[TMP1]])
@@ -146,22 +155,13 @@ eh.resume:                                        ; %catch.dispatch3
 ; CHECK:   ret i8* blockaddress(@"\01?test@@YAXXZ", %try.cont)
 ;
 ; CHECK: [[LPAD1_LABEL]]:{{[ ]+}}; preds = %entry
-; CHECK:   [[LPAD1_VAL:\%.+]] = landingpad { i8*, i32 } personality i8* bitcast (i32 (...)* @__CxxFrameHandler3 to i8*)
+; CHECK:   [[LPAD1_VAL:\%.+]] = landingpad { i8*, i32 }
 ; CHECK:           catch i8* bitcast (%rtti.TypeDescriptor2* @"\01??_R0M@8" to i8*)
-; CHECK:   [[RECOVER1:\%.+]] = call i8* (...) @llvm.eh.actions(i32 1, i8* bitcast (%rtti.TypeDescriptor2* @"\01??_R0M@8" to i8*), i32 1, i8* (i8*, i8*)* @"\01?test@@YAXXZ.catch1")
+; CHECK:   [[RECOVER1:\%.+]] = call i8* (...) @llvm.eh.actions(i32 1, i8* bitcast (%rtti.TypeDescriptor2* @"\01??_R0M@8" to i8*), i32 0, i8* (i8*, i8*)* @"\01?test@@YAXXZ.catch")
 ; CHECK:   indirectbr i8* [[RECOVER1]], []
 ;
 ; CHECK: }
 
-; CHECK: define internal i8* @"\01?test@@YAXXZ.catch1"(i8*, i8*)
-; CHECK: entry:
-; CHECK:   [[RECOVER_F1:\%.+]] = call i8* @llvm.framerecover(i8* bitcast (void ()* @"\01?test@@YAXXZ" to i8*), i8* %1, i32 1)
-; CHECK:   [[F_PTR1:\%.+]] = bitcast i8* [[RECOVER_F1]] to float*
-; CHECK:   [[TMP2:\%.+]] = load float, float* [[F_PTR1]], align 4
-; CHECK:   call void @"\01?handle_float@@YAXM@Z"(float [[TMP2]])
-; CHECK:   ret i8* blockaddress(@"\01?test@@YAXXZ", %try.cont10)
-; CHECK: }
-
 
 declare void @"\01?may_throw@@YAXXZ"() #1
 
diff --git a/test/CodeGen/WinEH/cppeh-nested-2.ll b/test/CodeGen/WinEH/cppeh-nested-2.ll
index ebcd3c969eb..2764e7478c7 100644
--- a/test/CodeGen/WinEH/cppeh-nested-2.ll
+++ b/test/CodeGen/WinEH/cppeh-nested-2.ll
@@ -44,12 +44,12 @@ target triple = "x86_64-pc-windows-msvc"
 ; CHECK:   %inner = alloca %class.Inner, align 1
 ; CHECK:   %i = alloca i32, align 4
 ; CHECK:   %f = alloca float, align 4
-; CHECK:   call void (...) @llvm.frameescape(float* %f, i32* %i, %class.Outer* %outer, %class.Inner* %inner)
+; CHECK:   call void (...) @llvm.localescape(float* %f, i32* %i, %class.Outer* %outer, %class.Inner* %inner)
 ; CHECK:   invoke void @_ZN5OuterC1Ev(%class.Outer* %outer)
 ; CHECK:           to label %invoke.cont unwind label %[[LPAD_LABEL:lpad[0-9]*]]
 
 ; Function Attrs: uwtable
-define void @_Z4testv() #0 {
+define void @_Z4testv() #0 personality i8* bitcast (i32 (...)* @__CxxFrameHandler3 to i8*) {
 entry:
   %outer = alloca %class.Outer, align 1
   %exn.slot = alloca i8*
@@ -91,13 +91,13 @@ invoke.cont5:                                     ; preds = %invoke.cont4
   br label %try.cont
 
 ; CHECK: [[LPAD_LABEL]]:
-; CHECK:   landingpad { i8*, i32 } personality i8* bitcast (i32 (...)* @__CxxFrameHandler3 to i8*)
+; CHECK:   landingpad { i8*, i32 }
 ; CHECK-NEXT:           catch i8* bitcast (i8** @_ZTIf to i8*)
 ; CHECK-NEXT:   [[RECOVER:\%.+]] = call i8* (...) @llvm.eh.actions(i32 1, i8* bitcast (i8** @_ZTIf to i8*), i32 0, i8* (i8*, i8*)* @_Z4testv.catch)
 ; CHECK-NEXT:   indirectbr i8* [[RECOVER]], [label %try.cont19]
 
 lpad:                                             ; preds = %try.cont, %entry
-  %tmp = landingpad { i8*, i32 } personality i8* bitcast (i32 (...)* @__CxxFrameHandler3 to i8*)
+  %tmp = landingpad { i8*, i32 }
           catch i8* bitcast (i8** @_ZTIf to i8*)
   %tmp1 = extractvalue { i8*, i32 } %tmp, 0
   store i8* %tmp1, i8** %exn.slot
@@ -106,18 +106,18 @@ lpad:                                             ; preds = %try.cont, %entry
   br label %catch.dispatch11
 
 ; CHECK: [[LPAD1_LABEL]]:
-; CHECK:   landingpad { i8*, i32 } personality i8* bitcast (i32 (...)* @__CxxFrameHandler3 to i8*)
+; CHECK:   landingpad { i8*, i32 }
 ; CHECK-NEXT:           cleanup
 ; CHECK-NEXT:           catch i8* bitcast (i8** @_ZTIi to i8*)
 ; CHECK-NEXT:           catch i8* bitcast (i8** @_ZTIf to i8*)
 ; CHECK-NEXT:   [[RECOVER1:\%.+]] = call i8* (...) @llvm.eh.actions(
-; CHECK-SAME:       i32 1, i8* bitcast (i8** @_ZTIi to i8*), i32 1, i8* (i8*, i8*)* @_Z4testv.catch1,
+; CHECK-SAME:       i32 1, i8* bitcast (i8** @_ZTIi to i8*), i32 1, i8* (i8*, i8*)* @_Z4testv.catch.1,
 ; CHECK-SAME:       i32 0, void (i8*, i8*)* @_Z4testv.cleanup,
 ; CHECK-SAME:       i32 1, i8* bitcast (i8** @_ZTIf to i8*), i32 0, i8* (i8*, i8*)* @_Z4testv.catch)
 ; CHECK-NEXT:   indirectbr i8* [[RECOVER1]], [label %try.cont, label %try.cont19]
 
 lpad1:                                            ; preds = %invoke.cont4, %invoke.cont
-  %tmp3 = landingpad { i8*, i32 } personality i8* bitcast (i32 (...)* @__CxxFrameHandler3 to i8*)
+  %tmp3 = landingpad { i8*, i32 }
           cleanup
           catch i8* bitcast (i8** @_ZTIi to i8*)
           catch i8* bitcast (i8** @_ZTIf to i8*)
@@ -128,19 +128,19 @@ lpad1:                                            ; preds = %invoke.cont4, %invo
   br label %catch.dispatch
 
 ; CHECK: [[LPAD3_LABEL]]:
-; CHECK:   landingpad { i8*, i32 } personality i8* bitcast (i32 (...)* @__CxxFrameHandler3 to i8*)
+; CHECK:   landingpad { i8*, i32 }
 ; CHECK-NEXT:           cleanup
 ; CHECK-NEXT:           catch i8* bitcast (i8** @_ZTIi to i8*)
 ; CHECK-NEXT:           catch i8* bitcast (i8** @_ZTIf to i8*)
 ; CHECK-NEXT:   [[RECOVER3:\%.+]] = call i8* (...) @llvm.eh.actions(
-; CHECK-SAME:       i32 0, void (i8*, i8*)* @_Z4testv.cleanup2,
-; CHECK-SAME:       i32 1, i8* bitcast (i8** @_ZTIi to i8*), i32 1, i8* (i8*, i8*)* @_Z4testv.catch1,
+; CHECK-SAME:       i32 0, void (i8*, i8*)* @_Z4testv.cleanup.2,
+; CHECK-SAME:       i32 1, i8* bitcast (i8** @_ZTIi to i8*), i32 1, i8* (i8*, i8*)* @_Z4testv.catch.1,
 ; CHECK-SAME:       i32 0, void (i8*, i8*)* @_Z4testv.cleanup,
 ; CHECK-SAME:       i32 1, i8* bitcast (i8** @_ZTIf to i8*), i32 0, i8* (i8*, i8*)* @_Z4testv.catch)
 ; CHECK-NEXT:   indirectbr i8* [[RECOVER3]], [label %try.cont, label %try.cont19]
 
 lpad3:                                            ; preds = %invoke.cont2
-  %tmp6 = landingpad { i8*, i32 } personality i8* bitcast (i32 (...)* @__CxxFrameHandler3 to i8*)
+  %tmp6 = landingpad { i8*, i32 }
           cleanup
           catch i8* bitcast (i8** @_ZTIi to i8*)
           catch i8* bitcast (i8** @_ZTIf to i8*)
@@ -189,7 +189,7 @@ invoke.cont9:                                     ; preds = %try.cont
 ; CHECK-NOT: lpad7:
 
 lpad7:                                            ; preds = %catch
-  %tmp14 = landingpad { i8*, i32 } personality i8* bitcast (i32 (...)* @__CxxFrameHandler3 to i8*)
+  %tmp14 = landingpad { i8*, i32 }
           cleanup
           catch i8* bitcast (i8** @_ZTIf to i8*)
   %tmp15 = extractvalue { i8*, i32 } %tmp14, 0
@@ -243,7 +243,7 @@ eh.resume:                                        ; preds = %catch.dispatch11
 ; This catch handler should be outlined.
 ; CHECK: define internal i8* @_Z4testv.catch(i8*, i8*)
 ; CHECK: entry:
-; CHECK:   [[RECOVER_F:\%.+]] = call i8* @llvm.framerecover(i8* bitcast (void ()* @_Z4testv to i8*), i8* %1, i32 0)
+; CHECK:   [[RECOVER_F:\%.+]] = call i8* @llvm.localrecover(i8* bitcast (void ()* @_Z4testv to i8*), i8* %1, i32 0)
 ; CHECK:   [[F_PTR:\%.+]] = bitcast i8* [[RECOVER_F]] to float*
 ; CHECK:   [[TMP:\%.+]] = load float, float* [[F_PTR]], align 4
 ; CHECK:   call void @_Z12handle_floatf(float [[TMP]])
@@ -251,9 +251,9 @@ eh.resume:                                        ; preds = %catch.dispatch11
 ; CHECK: }
 
 ; This catch handler should be outlined.
-; CHECK: define internal i8* @_Z4testv.catch1(i8*, i8*)
+; CHECK: define internal i8* @_Z4testv.catch.1(i8*, i8*)
 ; CHECK: entry:
-; CHECK:   [[RECOVER_I:\%.+]] = call i8* @llvm.framerecover(i8* bitcast (void ()* @_Z4testv to i8*), i8* %1, i32 1)
+; CHECK:   [[RECOVER_I:\%.+]] = call i8* @llvm.localrecover(i8* bitcast (void ()* @_Z4testv to i8*), i8* %1, i32 1)
 ; CHECK:   [[I_PTR:\%.+]] = bitcast i8* [[RECOVER_I]] to i32*
 ; CHECK:   [[TMP1:\%.+]] = load i32, i32* [[I_PTR]], align 4
 ; CHECK:   invoke void @_Z10handle_inti(i32 [[TMP1]])
@@ -263,23 +263,23 @@ eh.resume:                                        ; preds = %catch.dispatch11
 ; CHECK:   ret i8* blockaddress(@_Z4testv, %try.cont)
 ;
 ; CHECK: [[LPAD7_LABEL]]:{{[ ]+}}; preds = %entry
-; CHECK:   [[LPAD7_VAL:\%.+]] = landingpad { i8*, i32 } personality i8* bitcast (i32 (...)* @__CxxFrameHandler3 to i8*)
+; CHECK:   [[LPAD7_VAL:\%.+]] = landingpad { i8*, i32 }
 ; (FIXME) The nested handler body isn't being populated yet.
 ; CHECK: }
 
 ; This cleanup handler should be outlined.
 ; CHECK: define internal void @_Z4testv.cleanup(i8*, i8*)
 ; CHECK: entry:
-; CHECK:   [[RECOVER_OUTER:\%.+]] = call i8* @llvm.framerecover(i8* bitcast (void ()* @_Z4testv to i8*), i8* %1, i32 2)
+; CHECK:   [[RECOVER_OUTER:\%.+]] = call i8* @llvm.localrecover(i8* bitcast (void ()* @_Z4testv to i8*), i8* %1, i32 2)
 ; CHECK:   [[OUTER_PTR:\%.+]] = bitcast i8* [[RECOVER_OUTER]] to %class.Outer*
 ; CHECK:   call void @_ZN5OuterD1Ev(%class.Outer* [[OUTER_PTR]])
 ; CHECK:   ret void
 ; CHECK: }
 
 ; This cleanup handler should be outlined.
-; CHECK: define internal void @_Z4testv.cleanup2(i8*, i8*)
+; CHECK: define internal void @_Z4testv.cleanup.2(i8*, i8*)
 ; CHECK: entry:
-; CHECK:   [[RECOVER_INNER:\%.+]] = call i8* @llvm.framerecover(i8* bitcast (void ()* @_Z4testv to i8*), i8* %1, i32 3)
+; CHECK:   [[RECOVER_INNER:\%.+]] = call i8* @llvm.localrecover(i8* bitcast (void ()* @_Z4testv to i8*), i8* %1, i32 3)
 ; CHECK:   [[INNER_PTR:\%.+]] = bitcast i8* [[RECOVER_INNER]] to %class.Inner*
 ; CHECK:   call void @_ZN5InnerD1Ev(%class.Inner* [[INNER_PTR]])
 ; CHECK:   ret void
diff --git a/test/CodeGen/WinEH/cppeh-nested-3.ll b/test/CodeGen/WinEH/cppeh-nested-3.ll
index 4e33c55af3e..88759f406fb 100644
--- a/test/CodeGen/WinEH/cppeh-nested-3.ll
+++ b/test/CodeGen/WinEH/cppeh-nested-3.ll
@@ -41,12 +41,12 @@ $"\01??_R0H@8" = comdat any
 ; CHECK:   %i = alloca i32, align 4
 ; CHECK:   %j = alloca i32, align 4
 ; CHECK:   %f = alloca float, align 4
-; CHECK:   call void (...) @llvm.frameescape(i32* %i, float* %f, i32* %j)
+; CHECK:   call void (...) @llvm.localescape(i32* %j, i32* %i, float* %f)
 ; CHECK:   invoke void @"\01?may_throw@@YAXXZ"()
 ; CHECK:           to label %invoke.cont unwind label %[[LPAD_LABEL:lpad[0-9]*]]
 
 ; Function Attrs: uwtable
-define void @"\01?test@@YAXXZ"() #0 {
+define void @"\01?test@@YAXXZ"() #0 personality i8* bitcast (i32 (...)* @__CxxFrameHandler3 to i8*) {
 entry:
   %exn.slot = alloca i8*
   %ehselector.slot = alloca i32
@@ -60,14 +60,14 @@ invoke.cont:                                      ; preds = %entry
   br label %try.cont10
 
 ; CHECK: [[LPAD_LABEL]]:
-; CHECK:   landingpad { i8*, i32 } personality i8* bitcast (i32 (...)* @__CxxFrameHandler3 to i8*)
+; CHECK:   landingpad { i8*, i32 }
 ; CHECK:           catch i8* bitcast (%rtti.TypeDescriptor2* @"\01??_R0H@8" to i8*)
 ; CHECK:           catch i8* bitcast (%rtti.TypeDescriptor2* @"\01??_R0M@8" to i8*)
-; CHECK:   [[RECOVER:\%.+]] = call i8* (...) @llvm.eh.actions(i32 1, i8* bitcast (%rtti.TypeDescriptor2* @"\01??_R0H@8" to i8*), i32 0, i8* (i8*, i8*)* @"\01?test@@YAXXZ.catch", i32 1, i8* bitcast (%rtti.TypeDescriptor2* @"\01??_R0M@8" to i8*), i32 1, i8* (i8*, i8*)* @"\01?test@@YAXXZ.catch1")
+; CHECK:   [[RECOVER:\%.+]] = call i8* (...) @llvm.eh.actions(i32 1, i8* bitcast (%rtti.TypeDescriptor2* @"\01??_R0H@8" to i8*), i32 1, i8* (i8*, i8*)* @"\01?test@@YAXXZ.catch.2", i32 1, i8* bitcast (%rtti.TypeDescriptor2* @"\01??_R0M@8" to i8*), i32 2, i8* (i8*, i8*)* @"\01?test@@YAXXZ.catch.1")
 ; CHECK:   indirectbr i8* [[RECOVER]], [label %try.cont10, label %try.cont19]
 
 lpad:                                             ; preds = %entry
-  %0 = landingpad { i8*, i32 } personality i8* bitcast (i32 (...)* @__CxxFrameHandler3 to i8*)
+  %0 = landingpad { i8*, i32 }
           catch i8* bitcast (%rtti.TypeDescriptor2* @"\01??_R0H@8" to i8*)
           catch i8* bitcast (%rtti.TypeDescriptor2* @"\01??_R0M@8" to i8*)
   %1 = extractvalue { i8*, i32 } %0, 0
@@ -97,7 +97,7 @@ invoke.cont2:                                     ; preds = %catch
 
 ; CHECK-NOT: lpad1:
 lpad1:                                            ; preds = %catch
-  %5 = landingpad { i8*, i32 } personality i8* bitcast (i32 (...)* @__CxxFrameHandler3 to i8*)
+  %5 = landingpad { i8*, i32 }
           catch i8* bitcast (%rtti.TypeDescriptor2* @"\01??_R0H@8" to i8*)
           catch i8* bitcast (%rtti.TypeDescriptor2* @"\01??_R0M@8" to i8*)
   %6 = extractvalue { i8*, i32 } %5, 0
@@ -139,7 +139,7 @@ try.cont10:                                       ; preds = %invoke.cont9, %invo
 
 ; CHECK-NOT: lpad8:
 lpad8:                                            ; preds = %try.cont
-  %12 = landingpad { i8*, i32 } personality i8* bitcast (i32 (...)* @__CxxFrameHandler3 to i8*)
+  %12 = landingpad { i8*, i32 }
           catch i8* bitcast (%rtti.TypeDescriptor2* @"\01??_R0M@8" to i8*)
   %13 = extractvalue { i8*, i32 } %12, 0
   store i8* %13, i8** %exn.slot
@@ -181,7 +181,27 @@ eh.resume:                                        ; preds = %lpad16, %catch.disp
 
 ; CHECK: define internal i8* @"\01?test@@YAXXZ.catch"(i8*, i8*)
 ; CHECK: entry:
-; CHECK:   [[RECOVER_I:\%.+]] = call i8* @llvm.framerecover(i8* bitcast (void ()* @"\01?test@@YAXXZ" to i8*), i8* %1, i32 0)
+; CHECK:   [[RECOVER_J:\%.+]] = call i8* @llvm.localrecover(i8* bitcast (void ()* @"\01?test@@YAXXZ" to i8*), i8* %1, i32 0)
+; CHECK:   [[J_PTR:\%.+]] = bitcast i8* [[RECOVER_J]] to i32*
+; CHECK:   [[RECOVER_I1:\%.+]] = call i8* @llvm.localrecover(i8* bitcast (void ()* @"\01?test@@YAXXZ" to i8*), i8* %1, i32 1)
+; CHECK:   [[I_PTR1:\%.+]] = bitcast i8* [[RECOVER_I1]] to i32*
+; CHECK:   [[TMP3:\%.+]] = load i32, i32* [[J_PTR]], align 4
+; CHECK:   store i32 [[TMP3]], i32* [[I_PTR1]]
+; CHECK:   ret i8* blockaddress(@"\01?test@@YAXXZ.catch.2", %invoke.cont2)
+; CHECK: }
+
+; CHECK: define internal i8* @"\01?test@@YAXXZ.catch.1"(i8*, i8*)
+; CHECK: entry:
+; CHECK:   [[RECOVER_F:\%.+]] = call i8* @llvm.localrecover(i8* bitcast (void ()* @"\01?test@@YAXXZ" to i8*), i8* %1, i32 2)
+; CHECK:   [[F_PTR:\%.+]] = bitcast i8* [[RECOVER_F]] to float*
+; CHECK:   [[TMP2:\%.+]] = load float, float* [[F_PTR]], align 4
+; CHECK:   call void @"\01?handle_float@@YAXM@Z"(float [[TMP2]])
+; CHECK:   ret i8* blockaddress(@"\01?test@@YAXXZ", %try.cont19)
+; CHECK: }
+
+; CHECK: define internal i8* @"\01?test@@YAXXZ.catch.2"(i8*, i8*)
+; CHECK: entry:
+; CHECK:   [[RECOVER_I:\%.+]] = call i8* @llvm.localrecover(i8* bitcast (void ()* @"\01?test@@YAXXZ" to i8*), i8* %1, i32 1)
 ; CHECK:   [[I_PTR:\%.+]] = bitcast i8* [[RECOVER_I]] to i32*
 ; CHECK:   invoke void @"\01?may_throw@@YAXXZ"()
 ; CHECK:           to label %invoke.cont2 unwind label %[[LPAD1_LABEL:lpad[0-9]*]]
@@ -192,44 +212,23 @@ eh.resume:                                        ; preds = %lpad16, %catch.disp
 ; CHECK:           to label %invoke.cont9 unwind label %[[LPAD8_LABEL:lpad[0-9]*]]
 ;
 ; CHECK: [[LPAD1_LABEL]]:{{[ ]+}}; preds = %entry
-; CHECK:   [[LPAD1_VAL:\%.+]] = landingpad { i8*, i32 } personality i8* bitcast (i32 (...)* @__CxxFrameHandler3 to i8*)
+; CHECK:   [[LPAD1_VAL:\%.+]] = landingpad { i8*, i32 }
 ; CHECK:           catch i8* bitcast (%rtti.TypeDescriptor2* @"\01??_R0H@8" to i8*)
 ; CHECK:           catch i8* bitcast (%rtti.TypeDescriptor2* @"\01??_R0M@8" to i8*)
-; CHECK:   [[RECOVER1:\%.+]] = call i8* (...) @llvm.eh.actions(i32 1, i8* bitcast (%rtti.TypeDescriptor2* @"\01??_R0H@8" to i8*), i32 2, i8* (i8*, i8*)* @"\01?test@@YAXXZ.catch2", i32 1, i8* bitcast (%rtti.TypeDescriptor2* @"\01??_R0M@8" to i8*), i32 1, i8* (i8*, i8*)* @"\01?test@@YAXXZ.catch1")
+; CHECK:   [[RECOVER1:\%.+]] = call i8* (...) @llvm.eh.actions(i32 1, i8* bitcast (%rtti.TypeDescriptor2* @"\01??_R0H@8" to i8*), i32 0, i8* (i8*, i8*)* @"\01?test@@YAXXZ.catch", i32 1, i8* bitcast (%rtti.TypeDescriptor2* @"\01??_R0M@8" to i8*), i32 2, i8* (i8*, i8*)* @"\01?test@@YAXXZ.catch.1")
 ; CHECK:   indirectbr i8* [[RECOVER1]], [label %invoke.cont2]
 ;
 ; CHECK: invoke.cont9:
 ; CHECK:   ret i8* blockaddress(@"\01?test@@YAXXZ", %try.cont10)
 ;
 ; CHECK: [[LPAD8_LABEL]]:{{[ ]+}}; preds = %invoke.cont2
-; CHECK:   [[LPAD8_VAL:\%.+]] = landingpad { i8*, i32 } personality i8* bitcast (i32 (...)* @__CxxFrameHandler3 to i8*)
+; CHECK:   [[LPAD8_VAL:\%.+]] = landingpad { i8*, i32 }
 ; CHECK:           catch i8* bitcast (%rtti.TypeDescriptor2* @"\01??_R0M@8" to i8*)
-; CHECK:   [[RECOVER2:\%.+]] = call i8* (...) @llvm.eh.actions(i32 1, i8* bitcast (%rtti.TypeDescriptor2* @"\01??_R0M@8" to i8*), i32 1, i8* (i8*, i8*)* @"\01?test@@YAXXZ.catch1")
+; CHECK:   [[RECOVER2:\%.+]] = call i8* (...) @llvm.eh.actions(i32 1, i8* bitcast (%rtti.TypeDescriptor2* @"\01??_R0M@8" to i8*), i32 2, i8* (i8*, i8*)* @"\01?test@@YAXXZ.catch.1")
 ; CHECK:   indirectbr i8* [[RECOVER2]], []
 ;
 ; CHECK: }
 
-; CHECK: define internal i8* @"\01?test@@YAXXZ.catch1"(i8*, i8*)
-; CHECK: entry:
-; CHECK:   [[RECOVER_F:\%.+]] = call i8* @llvm.framerecover(i8* bitcast (void ()* @"\01?test@@YAXXZ" to i8*), i8* %1, i32 1)
-; CHECK:   [[F_PTR:\%.+]] = bitcast i8* [[RECOVER_F]] to float*
-; CHECK:   [[TMP2:\%.+]] = load float, float* [[F_PTR]], align 4
-; CHECK:   call void @"\01?handle_float@@YAXM@Z"(float [[TMP2]])
-; CHECK:   ret i8* blockaddress(@"\01?test@@YAXXZ", %try.cont19)
-; CHECK: }
-
-; CHECK: define internal i8* @"\01?test@@YAXXZ.catch2"(i8*, i8*)
-; CHECK: entry:
-; CHECK:   [[RECOVER_J:\%.+]] = call i8* @llvm.framerecover(i8* bitcast (void ()* @"\01?test@@YAXXZ" to i8*), i8* %1, i32 2)
-; CHECK:   [[J_PTR:\%.+]] = bitcast i8* [[RECOVER_J]] to i32*
-; CHECK:   [[RECOVER_I1:\%.+]] = call i8* @llvm.framerecover(i8* bitcast (void ()* @"\01?test@@YAXXZ" to i8*), i8* %1, i32 0)
-; CHECK:   [[I_PTR1:\%.+]] = bitcast i8* [[RECOVER_I1]] to i32*
-; CHECK:   [[TMP3:\%.+]] = load i32, i32* [[J_PTR]], align 4
-; CHECK:   store i32 [[TMP3]], i32* [[I_PTR1]]
-; CHECK:   ret i8* blockaddress(@"\01?test@@YAXXZ.catch", %invoke.cont2)
-; CHECK: }
-
-
 declare void @"\01?may_throw@@YAXXZ"() #1
 
 declare i32 @__CxxFrameHandler3(...)
diff --git a/test/CodeGen/WinEH/cppeh-nested-rethrow.ll b/test/CodeGen/WinEH/cppeh-nested-rethrow.ll
new file mode 100644
index 00000000000..53f532c8eb1
--- /dev/null
+++ b/test/CodeGen/WinEH/cppeh-nested-rethrow.ll
@@ -0,0 +1,212 @@
+; RUN: opt -mtriple=x86_64-pc-windows-msvc -winehprepare -S -o - < %s | FileCheck %s
+
+; This test was generated from the following code.
+;
+; void test1() {
+;   try {
+;     try {
+;       throw 1;
+;     } catch(...) { throw; }
+;   } catch (...) { }
+; }
+; void test2() {
+;   try {
+;     throw 1;
+;   } catch(...) {
+;     try {
+;       throw; 
+;     } catch (...) {}
+;   }
+; }
+;
+; These two functions result in functionally equivalent code, but the last
+; catch block contains a call to llvm.eh.endcatch that tripped up processing
+; during development.
+;
+; The main purpose of this test is to verify that we can correctly
+; handle the case of nested landing pads that return directly to a block in
+; the parent function.
+
+; ModuleID = 'cppeh-nested-rethrow.cpp'
+target datalayout = "e-m:w-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-pc-windows-msvc"
+
+%rtti.TypeDescriptor2 = type { i8**, i8*, [3 x i8] }
+%eh.CatchableType = type { i32, i32, i32, i32, i32, i32, i32 }
+%eh.CatchableTypeArray.1 = type { i32, [1 x i32] }
+%eh.ThrowInfo = type { i32, i32, i32, i32 }
+
+$"\01??_R0H@8" = comdat any
+
+$"_CT??_R0H@84" = comdat any
+
+$_CTA1H = comdat any
+
+$_TI1H = comdat any
+
+@"\01??_7type_info@@6B@" = external constant i8*
+@"\01??_R0H@8" = linkonce_odr global %rtti.TypeDescriptor2 { i8** @"\01??_7type_info@@6B@", i8* null, [3 x i8] c".H\00" }, comdat
+@__ImageBase = external constant i8
+@"_CT??_R0H@84" = linkonce_odr unnamed_addr constant %eh.CatchableType { i32 1, i32 trunc (i64 sub nuw nsw (i64 ptrtoint (%rtti.TypeDescriptor2* @"\01??_R0H@8" to i64), i64 ptrtoint (i8* @__ImageBase to i64)) to i32), i32 0, i32 -1, i32 0, i32 4, i32 0 }, section ".xdata", comdat
+@_CTA1H = linkonce_odr unnamed_addr constant %eh.CatchableTypeArray.1 { i32 1, [1 x i32] [i32 trunc (i64 sub nuw nsw (i64 ptrtoint (%eh.CatchableType* @"_CT??_R0H@84" to i64), i64 ptrtoint (i8* @__ImageBase to i64)) to i32)] }, section ".xdata", comdat
+@_TI1H = linkonce_odr unnamed_addr constant %eh.ThrowInfo { i32 0, i32 0, i32 0, i32 trunc (i64 sub nuw nsw (i64 ptrtoint (%eh.CatchableTypeArray.1* @_CTA1H to i64), i64 ptrtoint (i8* @__ImageBase to i64)) to i32) }, section ".xdata", comdat
+
+; CHECK-LABEL: define void @"\01?test1@@YAXXZ"()
+; CHECK: entry:
+; CHECK:   call void (...) @llvm.localescape
+
+; Function Attrs: nounwind uwtable
+define void @"\01?test1@@YAXXZ"() #0 personality i8* bitcast (i32 (...)* @__CxxFrameHandler3 to i8*) {
+entry:
+  %tmp = alloca i32, align 4
+  %exn.slot = alloca i8*
+  %ehselector.slot = alloca i32
+  store i32 1, i32* %tmp
+  %0 = bitcast i32* %tmp to i8*
+  invoke void @_CxxThrowException(i8* %0, %eh.ThrowInfo* @_TI1H) #2
+          to label %unreachable unwind label %lpad
+
+lpad:                                             ; preds = %entry
+  %1 = landingpad { i8*, i32 }
+          catch i8* null
+  %2 = extractvalue { i8*, i32 } %1, 0
+  store i8* %2, i8** %exn.slot
+  %3 = extractvalue { i8*, i32 } %1, 1
+  store i32 %3, i32* %ehselector.slot
+  br label %catch
+
+catch:                                            ; preds = %lpad
+  %exn = load i8*, i8** %exn.slot
+  call void @llvm.eh.begincatch(i8* %exn, i8* null) #1
+  invoke void @_CxxThrowException(i8* null, %eh.ThrowInfo* null) #2
+          to label %unreachable unwind label %lpad1
+
+lpad1:                                            ; preds = %catch
+  %4 = landingpad { i8*, i32 }
+          catch i8* null
+  %5 = extractvalue { i8*, i32 } %4, 0
+  store i8* %5, i8** %exn.slot
+  %6 = extractvalue { i8*, i32 } %4, 1
+  store i32 %6, i32* %ehselector.slot
+  br label %catch2
+
+catch2:                                           ; preds = %lpad1
+  %exn3 = load i8*, i8** %exn.slot
+  call void @llvm.eh.begincatch(i8* %exn3, i8* null) #1
+  call void @llvm.eh.endcatch() #1
+  br label %try.cont.4
+
+; This block should not be eliminated.
+; CHECK: try.cont.4:
+try.cont.4:                                        ; preds = %catch2, %try.cont
+  ret void
+
+try.cont:                                         ; No predecessors!
+  br label %try.cont.4
+
+unreachable:                                      ; preds = %catch, %entry
+  unreachable
+; CHECK: }
+}
+
+declare void @_CxxThrowException(i8*, %eh.ThrowInfo*)
+
+declare i32 @__CxxFrameHandler3(...)
+
+; Function Attrs: nounwind
+declare void @llvm.eh.begincatch(i8* nocapture, i8* nocapture) #1
+
+; Function Attrs: nounwind
+declare void @llvm.eh.endcatch() #1
+
+; CHECK-LABEL: define void @"\01?test2@@YAXXZ"()
+; CHECK: entry:
+; CHECK:   call void (...) @llvm.localescape
+
+; Function Attrs: nounwind uwtable
+define void @"\01?test2@@YAXXZ"() #0 personality i8* bitcast (i32 (...)* @__CxxFrameHandler3 to i8*) {
+entry:
+  %tmp = alloca i32, align 4
+  %exn.slot = alloca i8*
+  %ehselector.slot = alloca i32
+  store i32 1, i32* %tmp
+  %0 = bitcast i32* %tmp to i8*
+  invoke void @_CxxThrowException(i8* %0, %eh.ThrowInfo* @_TI1H) #2
+          to label %unreachable unwind label %lpad
+
+lpad:                                             ; preds = %entry
+  %1 = landingpad { i8*, i32 }
+          catch i8* null
+  %2 = extractvalue { i8*, i32 } %1, 0
+  store i8* %2, i8** %exn.slot
+  %3 = extractvalue { i8*, i32 } %1, 1
+  store i32 %3, i32* %ehselector.slot
+  br label %catch
+
+catch:                                            ; preds = %lpad
+  %exn = load i8*, i8** %exn.slot
+  call void @llvm.eh.begincatch(i8* %exn, i8* null) #1
+  invoke void @_CxxThrowException(i8* null, %eh.ThrowInfo* null) #2
+          to label %unreachable unwind label %lpad1
+
+lpad1:                                            ; preds = %catch
+  %4 = landingpad { i8*, i32 }
+          catch i8* null
+  %5 = extractvalue { i8*, i32 } %4, 0
+  store i8* %5, i8** %exn.slot
+  %6 = extractvalue { i8*, i32 } %4, 1
+  store i32 %6, i32* %ehselector.slot
+  br label %catch2
+
+catch2:                                           ; preds = %lpad1
+  %exn3 = load i8*, i8** %exn.slot
+  call void @llvm.eh.begincatch(i8* %exn3, i8* null) #1
+  call void @llvm.eh.endcatch() #1
+  br label %try.cont
+
+; This block should not be eliminated.
+; CHECK: try.cont:
+; The endcatch call should be eliminated.
+; CHECK-NOT: call void @llvm.eh.endcatch()
+try.cont:                                         ; preds = %catch2
+  call void @llvm.eh.endcatch() #1
+  br label %try.cont.4
+
+try.cont.4:                                        ; preds = %try.cont
+  ret void
+
+unreachable:                                      ; preds = %catch, %entry
+  unreachable
+; CHECK: }
+}
+
+; The outlined test1.catch handler should return to a valid block address.
+; CHECK-LABEL: define internal i8* @"\01?test1@@YAXXZ.catch"(i8*, i8*)
+; CHECK-NOT:  ret i8* inttoptr (i32 1 to i8*)
+; CHECK: }
+
+; The outlined test1.catch1 handler should not contain a return instruction.
+; CHECK-LABEL: define internal i8* @"\01?test1@@YAXXZ.catch.1"(i8*, i8*)
+; CHECK-NOT: ret
+; CHECK: }
+
+; The outlined test2.catch handler should return to a valid block address.
+; CHECK-LABEL: define internal i8* @"\01?test2@@YAXXZ.catch"(i8*, i8*)
+; CHECK-NOT:  ret i8* inttoptr (i32 1 to i8*)
+; CHECK: }
+
+; The outlined test2.catch2 handler should not contain a return instruction.
+; CHECK-LABEL: define internal i8* @"\01?test2@@YAXXZ.catch.2"(i8*, i8*)
+; CHECK-NOT: ret
+; CHECK: }
+
+
+attributes #0 = { nounwind uwtable "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+sse,+sse2" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #1 = { nounwind }
+attributes #2 = { noreturn }
+
+!llvm.module.flags = !{!0}
+!llvm.ident = !{!1}
+
+!0 = !{i32 1, !"PIC Level", i32 2}
+!1 = !{!"clang version 3.7.0 (trunk 236059)"}
diff --git a/test/CodeGen/WinEH/cppeh-nonalloca-frame-values.ll b/test/CodeGen/WinEH/cppeh-nonalloca-frame-values.ll
index de873d1170f..7b474c9d38a 100644
--- a/test/CodeGen/WinEH/cppeh-nonalloca-frame-values.ll
+++ b/test/CodeGen/WinEH/cppeh-nonalloca-frame-values.ll
@@ -55,8 +55,8 @@ $"\01??_R0H@8" = comdat any
 ; CHECK: entry:
 ; CHECK:   [[NUMEXCEPTIONS_REGMEM:\%.+]] = alloca i32
 ; CHECK:   [[I_REGMEM:\%.+]] = alloca i32
-; CHECK:   [[A_REGMEM:\%.+]] = alloca i32*
 ; CHECK:   [[B_REGMEM:\%.+]] = alloca i32*
+; CHECK:   [[A_REGMEM:\%.+]] = alloca i32*
 ; CHECK:   [[E_PTR:\%.+]] = alloca i32, align 4
 ; CHECK:   [[EXCEPTIONVAL:\%.+]] = alloca [10 x i32], align 16
 ; CHECK:   [[DATA_PTR:\%.+]] = alloca i64, align 8
@@ -68,13 +68,11 @@ $"\01??_R0H@8" = comdat any
 ; CHECK:   store i32* [[A_PTR]], i32** [[A_REGMEM]]
 ; CHECK:   [[B_PTR:\%.+]] = getelementptr inbounds %struct.SomeData, %struct.SomeData* [[TMPCAST]], i64 0, i32 1
 ; CHECK:   store i32* [[B_PTR]], i32** [[B_REGMEM]]
-; CHECK:   store i32 0, i32* [[NUMEXCEPTIONS_REGMEM]]
-; CHECK:   store i32 0, i32* [[I_REGMEM]]
-; CHECK:   call void (...) @llvm.frameescape(i32* %e, i32* %NumExceptions.020.reg2mem, [10 x i32]* [[EXCEPTIONVAL]], i32* [[I_REGMEM]], i32** [[A_REGMEM]], i32** [[B_REGMEM]])
+; CHECK:   call void (...) @llvm.localescape(i32* %e, i32* %NumExceptions.020.reg2mem, [10 x i32]* [[EXCEPTIONVAL]], i32* %inc.reg2mem, i32* [[I_REGMEM]], i32** [[A_REGMEM]], i32** [[B_REGMEM]])
 ; CHECK:   br label %for.body
 
 ; Function Attrs: uwtable
-define void @"\01?test@@YAXXZ"() #0 {
+define void @"\01?test@@YAXXZ"() #0 personality i8* bitcast (i32 (...)* @__CxxFrameHandler3 to i8*) {
 entry:
   %e = alloca i32, align 4
   %ExceptionVal = alloca [10 x i32], align 16
@@ -88,10 +86,11 @@ entry:
   br label %for.body
 
 ; CHECK: for.body:
-; CHECK-NOT:  phi i32 [ 0, %entry ], [ {{\%NumExceptions.*}}, %try.cont ]
-; CHECK-NOT:  phi i32 [ 0, %entry ], [ {{\%inc.*}}, %try.cont ]
-; CHECK:  [[I_RELOAD:\%.+]] = load i32, i32* [[I_REGMEM]]
-; CHECK:  [[NUMEXCEPTIONS_RELOAD:\%.+]] = load i32, i32* [[NUMEXCEPTIONS_REGMEM]]
+; CHECK:   [[NUMEXCEPTIONS_PHI:\%.*]] = phi i32 [ 0, %entry ], [ {{\%NumExceptions.*}}, %try.cont ]
+; CHECK:   [[I_PHI:\%.*]] = phi i32 [ 0, %entry ], [ {{\%inc.*}}, %try.cont ]
+; CHECK:   store i32 [[I_PHI]], i32* [[I_REGMEM]]
+; CHECK:   store i32 [[NUMEXCEPTIONS_PHI]], i32* [[NUMEXCEPTIONS_REGMEM]]
+; CHECK:   invoke void @"\01?may_throw@@YAXXZ"()
 for.body:                                         ; preds = %entry, %try.cont
   %NumExceptions.020 = phi i32 [ 0, %entry ], [ %NumExceptions.1, %try.cont ]
   %i.019 = phi i32 [ 0, %entry ], [ %inc5, %try.cont ]
@@ -99,11 +98,12 @@ for.body:                                         ; preds = %entry, %try.cont
           to label %invoke.cont unwind label %lpad
 
 ; CHECK: invoke.cont:                                      ; preds = %for.body
-; CHECK:   [[A_RELOAD:\%.+]] = load volatile i32*, i32** [[A_REGMEM]]
+; CHECK:   [[A_RELOAD:\%.+]] = load i32*, i32** [[A_REGMEM]]
 ; CHECK:   [[TMP1:\%.+]] = load i32, i32* [[A_RELOAD]], align 8
+; CHECK:   [[I_RELOAD:\%.+]] = load i32, i32* [[I_REGMEM]]
 ; CHECK:   [[ADD:\%.+]] = add nsw i32 [[TMP1]], [[I_RELOAD]]
-; CHECK:   [[A_RELOAD1:\%.+]] = load volatile i32*, i32** [[A_REGMEM]]
-; CHECK:   store i32 [[ADD]], i32* [[A_RELOAD1]], align 8
+; CHECK:   [[A_RELOAD1:\%.+]] = load i32*, i32** [[A_REGMEM]]
+; CHECK:   [[NUMEXCEPTIONS_RELOAD:\%.+]] = load i32, i32* [[NUMEXCEPTIONS_REGMEM]]
 ; CHECK:   br label %try.cont
 invoke.cont:                                      ; preds = %for.body
   %1 = load i32, i32* %a, align 8, !tbaa !2
@@ -112,13 +112,13 @@ invoke.cont:                                      ; preds = %for.body
   br label %try.cont
 
 ; CHECK: [[LPAD_LABEL:lpad[0-9]*]]:{{[ ]+}}; preds = %for.body
-; CHECK:   landingpad { i8*, i32 } personality i8* bitcast (i32 (...)* @__CxxFrameHandler3 to i8*)
+; CHECK:   landingpad { i8*, i32 }
 ; CHECK-NEXT:           catch i8* bitcast (%rtti.TypeDescriptor2* @"\01??_R0H@8" to i8*)
 ; CHECK-NEXT:   [[RECOVER:\%.+]] = call i8* (...) @llvm.eh.actions(i32 1, i8* bitcast (%rtti.TypeDescriptor2* @"\01??_R0H@8" to i8*), i32 0, i8* (i8*, i8*)* @"\01?test@@YAXXZ.catch")
-; CHECK-NEXT:   indirectbr i8* [[RECOVER]], [label %try.cont]
+; CHECK-NEXT:   indirectbr i8* [[RECOVER]], [label %[[SPLIT_RECOVER_BB:.*]]]
 
 lpad:                                             ; preds = %for.body
-  %2 = landingpad { i8*, i32 } personality i8* bitcast (i32 (...)* @__CxxFrameHandler3 to i8*)
+  %2 = landingpad { i8*, i32 }
           catch i8* bitcast (%rtti.TypeDescriptor2* @"\01??_R0H@8" to i8*)
   %3 = extractvalue { i8*, i32 } %2, 1
   %4 = tail call i32 @llvm.eh.typeid.for(i8* bitcast (%rtti.TypeDescriptor2* @"\01??_R0H@8" to i8*)) #1
@@ -139,8 +139,6 @@ catch:                                            ; preds = %lpad
   %cmp1 = icmp eq i32 %tmp8, %i.019
   br i1 %cmp1, label %if.then, label %if.else
 
-; CHECK-NOT: if.then:
-
 if.then:                                          ; preds = %catch
   %tmp9 = load i32, i32* %b, align 4, !tbaa !8
   %add2 = add nsw i32 %tmp9, %i.019
@@ -156,18 +154,20 @@ if.else:                                          ; preds = %catch
   br label %if.end
 
 ; CHECK-NOT: if.end:
+; CHECK: [[SPLIT_RECOVER_BB]]:
+; CHECK: [[INC_RELOAD:\%.*]] = load i32, i32*
+; CHECK: br label %try.cont
 
 if.end:                                           ; preds = %if.else, %if.then
   tail call void @llvm.eh.endcatch() #1
   br label %try.cont
 
-; CHECK: try.cont:{{[ ]+}}; preds = %[[LPAD_LABEL]], %invoke.cont
-; CHECK-NOT:  phi i32
-; CHECK:   tail call void @"\01?does_not_throw@@YAXH@Z"(i32 [[NUMEXCEPTIONS_RELOAD]])
+; CHECK: try.cont:{{[ ]+}}; preds = %[[SPLIT_RECOVER_BB]], %invoke.cont
+; CHECK:   [[NUMEXCEPTIONS_PHI:\%.*]] = phi i32 [ [[NUMEXCEPTIONS_RELOAD]], %invoke.cont ], [ [[INC_RELOAD]], %[[SPLIT_RECOVER_BB]] ]
+; CHECK:   tail call void @"\01?does_not_throw@@YAXH@Z"(i32 [[NUMEXCEPTIONS_PHI]])
+; CHECK:   [[I_RELOAD:\%.+]] = load i32, i32* [[I_REGMEM]]
 ; CHECK:   [[INC:\%.+]] = add nuw nsw i32 [[I_RELOAD]], 1
 ; CHECK:   [[CMP:\%.+]] = icmp slt i32 [[INC]], 10
-; CHECK:   store i32 [[NUMEXCEPTIONS_RELOAD]], i32* [[NUMEXCEPTIONS_REGMEM]]
-; CHECK:   store i32 [[INC]], i32* [[I_REGMEM]]
 ; CHECK:   br i1 [[CMP]], label %for.body, label %for.end
 
 try.cont:                                         ; preds = %if.end, %invoke.cont
@@ -192,45 +192,50 @@ eh.resume:                                        ; preds = %lpad
 ; The following catch handler should be outlined.
 ; CHECK: define internal i8* @"\01?test@@YAXXZ.catch"(i8*, i8*)
 ; CHECK: entry:
-; CHECK:   [[RECOVER_E:\%.+]] = call i8* @llvm.framerecover(i8* bitcast (void ()* @"\01?test@@YAXXZ" to i8*), i8* %1, i32 0)
+; CHECK:   [[RECOVER_E:\%.+]] = call i8* @llvm.localrecover(i8* bitcast (void ()* @"\01?test@@YAXXZ" to i8*), i8* %1, i32 0)
 ; CHECK:   [[E_PTR:\%.+]] = bitcast i8* [[RECOVER_E]] to i32*
-; CHECK:   [[RECOVER_EH_TEMP:\%.+]] = call i8* @llvm.framerecover(i8* bitcast (void ()* @"\01?test@@YAXXZ" to i8*), i8* %1, i32 1)
-; CHECK:   [[EH_TEMP:\%.+]] = bitcast i8* [[RECOVER_EH_TEMP]] to i32*
-; CHECK:   [[NUMEXCEPTIONS_RELOAD:\%.+]] = load i32, i32* [[EH_TEMP]]
-; CHECK:   [[RECOVER_EXCEPTIONVAL:\%.+]] = call i8* @llvm.framerecover(i8* bitcast (void ()* @"\01?test@@YAXXZ" to i8*), i8* %1, i32 2)
+; CHECK:   [[RECOVER_NUMEXCEPTIONS:\%.+]] = call i8* @llvm.localrecover(i8* bitcast (void ()* @"\01?test@@YAXXZ" to i8*), i8* %1, i32 1)
+; CHECK:   [[NUMEXCEPTIONS_REGMEM:\%.+]] = bitcast i8* [[RECOVER_NUMEXCEPTIONS]] to i32*
+; CHECK:   [[RECOVER_EXCEPTIONVAL:\%.+]] = call i8* @llvm.localrecover(i8* bitcast (void ()* @"\01?test@@YAXXZ" to i8*), i8* %1, i32 2)
 ; CHECK:   [[EXCEPTIONVAL:\%.+]] = bitcast i8* [[RECOVER_EXCEPTIONVAL]] to [10 x i32]*
-; CHECK:   [[RECOVER_EH_TEMP1:\%.+]] = call i8* @llvm.framerecover(i8* bitcast (void ()* @"\01?test@@YAXXZ" to i8*), i8* %1, i32 3)
-; CHECK:   [[EH_TEMP1:\%.+]] = bitcast i8* [[RECOVER_EH_TEMP1]] to i32*
-; CHECK:   [[I_RELOAD:\%.+]] = load i32, i32* [[EH_TEMP1]]
-; CHECK:   [[RECOVER_EH_TEMP2:\%.+]] = call i8* @llvm.framerecover(i8* bitcast (void ()* @"\01?test@@YAXXZ" to i8*), i8* %1, i32 4)
-; CHECK:   [[EH_TEMP2:\%.+]] = bitcast i8* [[RECOVER_EH_TEMP2]] to i32**
-; CHECK:   [[A_RELOAD:\%.+]] = load i32*, i32** [[EH_TEMP2]]
-; CHECK:   [[RECOVER_EH_TEMP3:\%.+]] = call i8* @llvm.framerecover(i8* bitcast (void ()* @"\01?test@@YAXXZ" to i8*), i8* %1, i32 5)
-; CHECK:   [[EH_TEMP3:\%.+]] = bitcast i8* [[RECOVER_EH_TEMP3]] to i32**
-; CHECK:   [[B_RELOAD:\%.+]] = load i32*, i32** [[EH_TEMP3]]
+; CHECK:   [[RECOVER_INC:\%.+]] = call i8* @llvm.localrecover(i8* bitcast (void ()* @"\01?test@@YAXXZ" to i8*), i8* %1, i32 3)
+; CHECK:   [[INC_REGMEM:\%.+]] = bitcast i8* [[RECOVER_INC]] to i32*
+; CHECK:   [[RECOVER_I:\%.+]] = call i8* @llvm.localrecover(i8* bitcast (void ()* @"\01?test@@YAXXZ" to i8*), i8* %1, i32 4)
+; CHECK:   [[I_REGMEM:\%.+]] = bitcast i8* [[RECOVER_I]] to i32*
+; CHECK:   [[RECOVER_A:\%.+]] = call i8* @llvm.localrecover(i8* bitcast (void ()* @"\01?test@@YAXXZ" to i8*), i8* %1, i32 5)
+; CHECK:   [[A_REGMEM:\%.+]] = bitcast i8* [[RECOVER_A]] to i32**
+; CHECK:   [[RECOVER_B:\%.+]] = call i8* @llvm.localrecover(i8* bitcast (void ()* @"\01?test@@YAXXZ" to i8*), i8* %1, i32 6)
+; CHECK:   [[B_REGMEM:\%.+]] = bitcast i8* [[RECOVER_B]] to i32**
 ; CHECK:   [[E_I8PTR:\%.+]] = bitcast i32* [[E_PTR]] to i8*
 ; CHECK:   [[TMP:\%.+]] = load i32, i32* [[E_PTR]], align 4
+; CHECK:   [[NUMEXCEPTIONS_RELOAD:\%.+]] = load i32, i32* [[NUMEXCEPTIONS_REGMEM]]
 ; CHECK:   [[IDXPROM:\%.+]] = sext i32 [[NUMEXCEPTIONS_RELOAD]] to i64
 ; CHECK:   [[ARRAYIDX:\%.+]] = getelementptr inbounds [10 x i32], [10 x i32]* [[EXCEPTIONVAL]], i64 0, i64 [[IDXPROM]]
 ; CHECK:   store i32 [[TMP]], i32* [[ARRAYIDX]], align 4
+; CHECK:   [[NUMEXCEPTIONS_RELOAD:\%.+]] = load i32, i32* [[NUMEXCEPTIONS_REGMEM]]
 ; CHECK:   [[INC:\%.+]] = add nsw i32 [[NUMEXCEPTIONS_RELOAD]], 1
 ; CHECK:   [[CMP:\%.+]] = icmp eq i32 [[TMP]], [[I_RELOAD]]
 ; CHECK:   br i1 [[CMP]], label %if.then, label %if.else
 ;
 ; CHECK: if.then:{{[ ]+}}; preds = %entry
+; CHECK:   [[B_RELOAD:\%.+]] = load i32*, i32** [[B_REGMEM]]
 ; CHECK:   [[TMP1:\%.+]] = load i32, i32* [[B_RELOAD]], align 4
+; CHECK:   [[I_RELOAD:\%.+]] = load i32, i32* [[I_REGMEM]]
 ; CHECK:   [[ADD:\%.+]] = add nsw i32 [[TMP1]], [[I_RELOAD]]
+; CHECK:   [[B_RELOAD:\%.+]] = load i32*, i32** [[B_REGMEM]]
 ; CHECK:   store i32 [[ADD]], i32* [[B_RELOAD]], align 4
 ; CHECK:   br label %if.end
 ;
 ; CHECK: if.else:{{[ ]+}}; preds = %entry
+; CHECK:   [[A_RELOAD:\%.+]] = load i32*, i32** [[A_REGMEM]]
 ; CHECK:   [[TMP2:\%.+]] = load i32, i32* [[A_RELOAD]], align 8
 ; CHECK:   [[ADD2:\%.+]] = add nsw i32 [[TMP2]], [[TMP]]
+; CHECK:   [[A_RELOAD:\%.+]] = load i32*, i32** [[A_REGMEM]]
 ; CHECK:   store i32 [[ADD2]], i32* [[A_RELOAD]], align 8
 ; CHECK:   br label %if.end
 ;
 ; CHECK: if.end:{{[ ]+}}; preds = %if.else, %if.then
-; CHECK:   ret i8* blockaddress(@"\01?test@@YAXXZ", %try.cont)
+; CHECK:   ret i8* blockaddress(@"\01?test@@YAXXZ", %[[SPLIT_RECOVER_BB]])
 ; CHECK: }
 
 ; Function Attrs: nounwind
diff --git a/test/CodeGen/WinEH/cppeh-prepared-catch-all.ll b/test/CodeGen/WinEH/cppeh-prepared-catch-all.ll
index f395d64c7b5..31b5e58562b 100644
--- a/test/CodeGen/WinEH/cppeh-prepared-catch-all.ll
+++ b/test/CodeGen/WinEH/cppeh-prepared-catch-all.ll
@@ -18,13 +18,13 @@ declare void @llvm.eh.begincatch(i8* nocapture, i8* nocapture) #2
 declare void @llvm.eh.endcatch() #2
 
 ; Function Attrs: nounwind uwtable
-define void @test_catch_all() #0 {
+define void @test_catch_all() #0 personality i8* bitcast (i32 (...)* @__CxxFrameHandler3 to i8*) {
 entry:
   invoke void @may_throw()
           to label %try.cont unwind label %lpad
 
 lpad:                                             ; preds = %entry
-  %0 = landingpad { i8*, i32 } personality i8* bitcast (i32 (...)* @__CxxFrameHandler3 to i8*)
+  %0 = landingpad { i8*, i32 }
           catch i8* null
   %1 = extractvalue { i8*, i32 } %0, 0
   tail call void @llvm.eh.begincatch(i8* %1, i8* null) #2
diff --git a/test/CodeGen/WinEH/cppeh-prepared-catch-reordered.ll b/test/CodeGen/WinEH/cppeh-prepared-catch-reordered.ll
index 4946c6affc3..2d31a1d5cf4 100644
--- a/test/CodeGen/WinEH/cppeh-prepared-catch-reordered.ll
+++ b/test/CodeGen/WinEH/cppeh-prepared-catch-reordered.ll
@@ -43,13 +43,13 @@ $"\01??_C@_06PNOAJMHG@e?3?5?$CFd?6?$AA@" = comdat any
 declare void @_CxxThrowException(i8*, %eh.ThrowInfo*)
 
 ; Function Attrs: uwtable
-define i32 @main() #1 {
+define i32 @main() #1 personality i8* bitcast (i32 (...)* @__CxxFrameHandler3 to i8*) {
 entry:
   %tmp.i = alloca i32, align 4
   %e = alloca i32, align 4
   %0 = bitcast i32* %tmp.i to i8*
   store i32 42, i32* %tmp.i, align 4, !tbaa !2
-  call void (...) @llvm.frameescape(i32* %e)
+  call void (...) @llvm.localescape(i32* %e)
   invoke void @_CxxThrowException(i8* %0, %eh.ThrowInfo* @_TI1H) #6
           to label %.noexc unwind label %lpad1
 
@@ -57,7 +57,7 @@ entry:
   unreachable
 
 lpad1:                                            ; preds = %entry
-  %1 = landingpad { i8*, i32 } personality i8* bitcast (i32 (...)* @__CxxFrameHandler3 to i8*)
+  %1 = landingpad { i8*, i32 }
           catch %eh.CatchHandlerType* @llvm.eh.handlertype.H.0
   %recover = call i8* (...) @llvm.eh.actions(i32 1, i8* bitcast (%eh.CatchHandlerType* @llvm.eh.handlertype.H.0 to i8*), i32 0, i8* (i8*, i8*)* @main.catch)
   indirectbr i8* %recover, [label %try.cont.split]
@@ -90,9 +90,9 @@ declare void @llvm.lifetime.start(i64, i8* nocapture) #3
 ; Function Attrs: nounwind
 declare i8* @llvm.eh.actions(...) #3
 
-define internal i8* @main.catch(i8*, i8*) #5 {
+define internal i8* @main.catch(i8*, i8*) #5 personality i8* bitcast (i32 (...)* @__CxxFrameHandler3 to i8*) {
 entry:
-  %e.i8 = call i8* @llvm.framerecover(i8* bitcast (i32 ()* @main to i8*), i8* %1, i32 0)
+  %e.i8 = call i8* @llvm.localrecover(i8* bitcast (i32 ()* @main to i8*), i8* %1, i32 0)
   %e = bitcast i8* %e.i8 to i32*
   %2 = bitcast i32* %e to i8*
   %3 = load i32, i32* %e, align 4, !tbaa !2
@@ -104,8 +104,9 @@ entry.split:                                      ; preds = %entry
   ret i8* blockaddress(@main, %try.cont.split)
 
 stub:                                             ; preds = %entry
-  %4 = landingpad { i8*, i32 } personality i8* bitcast (i32 (...)* @__CxxFrameHandler3 to i8*)
+  %4 = landingpad { i8*, i32 }
           cleanup
+  %recover = call i8* (...) @llvm.eh.actions()
   unreachable
 }
 
@@ -113,13 +114,14 @@ stub:                                             ; preds = %entry
 ; CHECK:        .seh_handlerdata
 ; CHECK:        .long   ($cppxdata$main)@IMGREL
 
+; CHECK: .align 4
 ; CHECK-NEXT: $cppxdata$main:
 ; CHECK-NEXT:         .long   429065506
 ; CHECK-NEXT:         .long   2
 ; CHECK-NEXT:         .long   ($stateUnwindMap$main)@IMGREL
 ; CHECK-NEXT:         .long   1
 ; CHECK-NEXT:         .long   ($tryMap$main)@IMGREL
-; CHECK-NEXT:         .long   1
+; CHECK-NEXT:         .long   3
 ; CHECK-NEXT:         .long   ($ip2state$main)@IMGREL
 ; CHECK-NEXT:         .long   40
 ; CHECK-NEXT:         .long   0
@@ -138,10 +140,10 @@ stub:                                             ; preds = %entry
 declare void @llvm.donothing() #2
 
 ; Function Attrs: nounwind
-declare void @llvm.frameescape(...) #3
+declare void @llvm.localescape(...) #3
 
 ; Function Attrs: nounwind readnone
-declare i8* @llvm.framerecover(i8*, i8*, i32) #2
+declare i8* @llvm.localrecover(i8*, i8*, i32) #2
 
 attributes #0 = { noreturn uwtable "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "unsafe-fp-math"="false" "use-soft-float"="false" }
 attributes #1 = { uwtable "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "unsafe-fp-math"="false" "use-soft-float"="false" "wineh-parent"="main" }
diff --git a/test/CodeGen/WinEH/cppeh-prepared-catch.ll b/test/CodeGen/WinEH/cppeh-prepared-catch.ll
index 98b4afcd055..a5d86dceea9 100644
--- a/test/CodeGen/WinEH/cppeh-prepared-catch.ll
+++ b/test/CodeGen/WinEH/cppeh-prepared-catch.ll
@@ -1,7 +1,5 @@
-; RUN: llc < %s | FileCheck %s
-
-target datalayout = "e-m:w-i64:64-f80:128-n8:16:32:64-S128"
-target triple = "x86_64-pc-windows-msvc"
+; RUN: llc -mtriple=x86_64-windows-msvc < %s | FileCheck %s --check-prefix=CHECK --check-prefix=X64
+; RUN: llc -mtriple=i686-windows-msvc < %s | FileCheck %s --check-prefix=CHECK --check-prefix=X86
 
 ; This test case is equivalent to:
 ; void f() {
@@ -30,9 +28,9 @@ $"\01??_R0H@8" = comdat any
 @"\01??_R0H@8" = linkonce_odr global %rtti.TypeDescriptor2 { i8** @"\01??_7type_info@@6B@", i8* null, [3 x i8] c".H\00" }, comdat
 @llvm.eh.handlertype.H.8 = private unnamed_addr constant %eh.CatchHandlerType { i32 8, i8* bitcast (%rtti.TypeDescriptor2* @"\01??_R0H@8" to i8*) }, section "llvm.metadata"
 
-define internal i8* @"\01?f@@YAXXZ.catch"(i8*, i8*) #4 {
+define internal i8* @"\01?f@@YAXXZ.catch"(i8*, i8*) #4 personality i8* bitcast (i32 (...)* @__CxxFrameHandler3 to i8*) {
 entry:
-  %.i8 = call i8* @llvm.framerecover(i8* bitcast (void ()* @"\01?f@@YAXXZ" to i8*), i8* %1, i32 0)
+  %.i8 = call i8* @llvm.localrecover(i8* bitcast (void ()* @"\01?f@@YAXXZ" to i8*), i8* %1, i32 0)
   %bc2 = bitcast i8* %.i8 to i32**
   %bc3 = bitcast i32** %bc2 to i8*
   invoke void @"\01?may_throw@@YAXXZ"()
@@ -42,7 +40,7 @@ invoke.cont2:                                     ; preds = %entry
   ret i8* blockaddress(@"\01?f@@YAXXZ", %try.cont)
 
 lpad1:                                            ; preds = %entry
-  %lp4 = landingpad { i8*, i32 } personality i8* bitcast (i32 (...)* @__CxxFrameHandler3 to i8*)
+  %lp4 = landingpad { i8*, i32 }
           cleanup
           catch %eh.CatchHandlerType* @llvm.eh.handlertype.N.0
   %recover = call i8* (...) @llvm.eh.actions(i32 1, i8* bitcast (%eh.CatchHandlerType* @llvm.eh.handlertype.N.0 to i8*), i32 1, i8* (i8*, i8*)* @"\01?f@@YAXXZ.catch1")
@@ -51,43 +49,44 @@ lpad1:                                            ; preds = %entry
 
 ; CHECK-LABEL: "?f@@YAXXZ.catch":
 ; No code should be generated for the indirectbr.
-; CHECK-NOT: jmpq *
-; CHECK:        .seh_handlerdata
-; CHECK:        .long   ("$cppxdata$?f@@YAXXZ")@IMGREL
+; CHECK-NOT: jmp{{[ql]}} *
+; X64:        .seh_handlerdata
+; X64-NEXT:        .long   ("$cppxdata$?f@@YAXXZ")@IMGREL
 
 
-define internal i8* @"\01?f@@YAXXZ.catch1"(i8*, i8*) #4 {
+define internal i8* @"\01?f@@YAXXZ.catch1"(i8*, i8*) #4 personality i8* bitcast (i32 (...)* @__CxxFrameHandler3 to i8*) {
 entry:
-  %.i8 = call i8* @llvm.framerecover(i8* bitcast (void ()* @"\01?f@@YAXXZ" to i8*), i8* %1, i32 1)
+  %.i8 = call i8* @llvm.localrecover(i8* bitcast (void ()* @"\01?f@@YAXXZ" to i8*), i8* %1, i32 1)
   %2 = bitcast i8* %.i8 to double*
   %3 = bitcast double* %2 to i8*
-  invoke void (...)* @llvm.donothing()
+  invoke void () @llvm.donothing()
           to label %done unwind label %lpad
 
 done:
   ret i8* blockaddress(@"\01?f@@YAXXZ", %try.cont8)
 
 lpad:                                             ; preds = %entry
-  %4 = landingpad { i8*, i32 } personality i8* bitcast (i32 (...)* @__CxxFrameHandler3 to i8*)
+  %4 = landingpad { i8*, i32 }
           cleanup
+  %recover = call i8* (...) @llvm.eh.actions()
   unreachable
 }
 
 ; CHECK-LABEL: "?f@@YAXXZ.catch1":
 ; No code should be generated for the indirectbr.
-; CHECK-NOT: jmpq *
-; CHECK: ".L?f@@YAXXZ.catch1$parent_frame_offset" = 16
-; CHECK:         movq    %rdx, 16(%rsp)
-; CHECK:        .seh_handlerdata
-; CHECK:        .long   ("$cppxdata$?f@@YAXXZ")@IMGREL
+; CHECK-NOT: jmp{{[ql]}} *
+; X64: ".L?f@@YAXXZ.catch1$parent_frame_offset" = 16
+; X64:         movq    %rdx, 16(%rsp)
+; X64:        .seh_handlerdata
+; X64:        .long   ("$cppxdata$?f@@YAXXZ")@IMGREL
 
-define void @"\01?f@@YAXXZ"() #0 {
+define void @"\01?f@@YAXXZ"() #0 personality i8* bitcast (i32 (...)* @__CxxFrameHandler3 to i8*) {
 entry:
   %exn.slot = alloca i8*
   %ehselector.slot = alloca i32
   %0 = alloca i32*, align 8
   %1 = alloca double, align 8
-  call void (...) @llvm.frameescape(i32** %0, double* %1)
+  call void (...) @llvm.localescape(i32** %0, double* %1)
   invoke void @"\01?may_throw@@YAXXZ"()
           to label %invoke.cont unwind label %lpad2
 
@@ -95,7 +94,7 @@ invoke.cont:                                      ; preds = %entry
   br label %try.cont
 
 lpad2:                                            ; preds = %entry
-  %2 = landingpad { i8*, i32 } personality i8* bitcast (i32 (...)* @__CxxFrameHandler3 to i8*)
+  %2 = landingpad { i8*, i32 }
           catch %eh.CatchHandlerType* @llvm.eh.handlertype.H.8
           catch %eh.CatchHandlerType* @llvm.eh.handlertype.N.0
   %recover = call i8* (...) @llvm.eh.actions(i32 1, i8* bitcast (%eh.CatchHandlerType* @llvm.eh.handlertype.H.8 to i8*), i32 0, i8* (i8*, i8*)* @"\01?f@@YAXXZ.catch", i32 1, i8* bitcast (%eh.CatchHandlerType* @llvm.eh.handlertype.N.0 to i8*), i32 1, i8* (i8*, i8*)* @"\01?f@@YAXXZ.catch1")
@@ -106,7 +105,7 @@ try.cont:                                         ; preds = %lpad2, %invoke.cont
           to label %try.cont8 unwind label %lpad1
 
 lpad1:
-  %3 = landingpad { i8*, i32 } personality i8* bitcast (i32 (...)* @__CxxFrameHandler3 to i8*)
+  %3 = landingpad { i8*, i32 }
           catch %eh.CatchHandlerType* @llvm.eh.handlertype.N.0
   %recover2 = call i8* (...) @llvm.eh.actions(i32 1, i8* bitcast (%eh.CatchHandlerType* @llvm.eh.handlertype.N.0 to i8*), i32 1, i8* (i8*, i8*)* @"\01?f@@YAXXZ.catch1")
   indirectbr i8* %recover2, [label %try.cont8]
@@ -117,20 +116,38 @@ try.cont8:                                        ; preds = %lpad2, %try.cont
 
 ; CHECK-LABEL: "?f@@YAXXZ":
 ; No code should be generated for the indirectbr.
-; CHECK-NOT: jmpq *
-; CHECK:             .seh_handlerdata
-; CHECK-NEXT:        .long   ("$cppxdata$?f@@YAXXZ")@IMGREL
-; CHECK-NEXT:"$cppxdata$?f@@YAXXZ":
-; CHECK-NEXT:        .long   429065506
-; CHECK-NEXT:        .long   4
-; CHECK-NEXT:        .long   ("$stateUnwindMap$?f@@YAXXZ")@IMGREL
-; CHECK-NEXT:        .long   2
-; CHECK-NEXT:        .long   ("$tryMap$?f@@YAXXZ")@IMGREL
-; CHECK-NEXT:        .long   3
-; CHECK-NEXT:        .long   ("$ip2state$?f@@YAXXZ")@IMGREL
-; CHECK-NEXT:        .long   32
-; CHECK-NEXT:        .long   0
-; CHECK-NEXT:        .long   1
+; CHECK-NOT: jmp{{[ql]}} *
+
+; X64:             .seh_handlerdata
+; X64-NEXT:        .long   ("$cppxdata$?f@@YAXXZ")@IMGREL
+; X86:             .section .xdata,"dr"
+
+; CHECK: .align 4
+
+; X64: "$cppxdata$?f@@YAXXZ":
+; X64-NEXT:          .long   429065506
+; X64-NEXT:          .long   4
+; X64-NEXT:          .long   ("$stateUnwindMap$?f@@YAXXZ")@IMGREL
+; X64-NEXT:          .long   2
+; X64-NEXT:          .long   ("$tryMap$?f@@YAXXZ")@IMGREL
+; X64-NEXT:          .long   6
+; X64-NEXT:          .long   ("$ip2state$?f@@YAXXZ")@IMGREL
+; X64-NEXT:          .long   32
+; X64-NEXT:          .long   0
+; X64-NEXT:          .long   1
+
+; X86: "L__ehtable$?f@@YAXXZ":
+; X86-NEXT:          .long   429065506
+; X86-NEXT:          .long   4
+; X86-NEXT:          .long   ("$stateUnwindMap$?f@@YAXXZ")
+; X86-NEXT:          .long   2
+; X86-NEXT:          .long   ("$tryMap$?f@@YAXXZ")
+; X86-NEXT:          .long   0
+; X86-NEXT:          .long   0
+; X86-NEXT:          .long   0
+; X86-NEXT:          .long   1
+
+
 ; CHECK-NEXT:"$stateUnwindMap$?f@@YAXXZ":
 ; CHECK-NEXT:        .long   -1
 ; CHECK-NEXT:        .long   0
@@ -145,31 +162,43 @@ try.cont8:                                        ; preds = %lpad2, %try.cont
 ; CHECK-NEXT:        .long   1
 ; CHECK-NEXT:        .long   2
 ; CHECK-NEXT:        .long   1
-; CHECK-NEXT:        .long   ("$handlerMap$0$?f@@YAXXZ")@IMGREL
+; CHECK-NEXT:        .long   ("$handlerMap$0$?f@@YAXXZ")
 ; CHECK-NEXT:        .long   0
 ; CHECK-NEXT:        .long   2
 ; CHECK-NEXT:        .long   3
 ; CHECK-NEXT:        .long   1
-; CHECK-NEXT:        .long   ("$handlerMap$1$?f@@YAXXZ")@IMGREL
+; CHECK-NEXT:        .long   ("$handlerMap$1$?f@@YAXXZ")
 ; CHECK-NEXT:"$handlerMap$0$?f@@YAXXZ":
 ; CHECK-NEXT:        .long   8
-; CHECK-NEXT:        .long   "??_R0H@8"@IMGREL
-; CHECK-NEXT:        .long   ".L?f@@YAXXZ$frame_escape_0"
-; CHECK-NEXT:        .long   "?f@@YAXXZ.catch"@IMGREL
-; CHECK-NEXT:        .long   ".L?f@@YAXXZ.catch$parent_frame_offset"
+; CHECK-NEXT:        .long   "??_R0H@8"
+; CHECK-NEXT:        .long   "{{.?}}L?f@@YAXXZ$frame_escape_0"
+; CHECK-NEXT:        .long   "?f@@YAXXZ.catch"
+; X64-NEXT:          .long   ".L?f@@YAXXZ.catch$parent_frame_offset"
 ; CHECK-NEXT:"$handlerMap$1$?f@@YAXXZ":
 ; CHECK-NEXT:        .long   0
-; CHECK-NEXT:        .long   "??_R0N@8"@IMGREL
-; CHECK-NEXT:        .long   ".L?f@@YAXXZ$frame_escape_1"
-; CHECK-NEXT:        .long   "?f@@YAXXZ.catch1"@IMGREL
-; CHECK-NEXT:        .long   ".L?f@@YAXXZ.catch1$parent_frame_offset"
-; CHECK-NEXT:"$ip2state$?f@@YAXXZ":
-; CHECK-NEXT:        .long   .Ltmp0@IMGREL
-; CHECK-NEXT:        .long   0
-; CHECK-NEXT:        .long   .Ltmp13@IMGREL
-; CHECK-NEXT:        .long   1
-; CHECK-NEXT:        .long   .Ltmp16@IMGREL
-; CHECK-NEXT:        .long   0
+; CHECK-NEXT:        .long   "??_R0N@8"
+; CHECK-NEXT:        .long   "{{.?}}L?f@@YAXXZ$frame_escape_1"
+; CHECK-NEXT:        .long   "?f@@YAXXZ.catch1"
+; X64-NEXT:          .long   ".L?f@@YAXXZ.catch1$parent_frame_offset"
+
+; X64-NEXT:"$ip2state$?f@@YAXXZ":
+; X64-NEXT:        .long   .Lfunc_begin0
+; X64-NEXT:        .long   2
+; X64-NEXT:        .long   .Ltmp0
+; X64-NEXT:        .long   0
+; X64-NEXT:        .long   .Lfunc_begin1
+; X64-NEXT:        .long   3
+; X64-NEXT:        .long   .Lfunc_begin2
+; X64-NEXT:        .long   -1
+; X64-NEXT:        .long   .Ltmp13
+; X64-NEXT:        .long   1
+; X64-NEXT:        .long   .Ltmp16
+; X64-NEXT:        .long   0
+
+
+; X86: "___ehhandler$?f@@YAXXZ": # @"__ehhandler$?f@@YAXXZ"
+; X86: movl $"L__ehtable$?f@@YAXXZ", %eax
+; X86: jmp ___CxxFrameHandler3 # TAILCALL
 
 
 declare void @"\01?may_throw@@YAXXZ"() #1
@@ -189,12 +218,12 @@ declare void @llvm.eh.endcatch() #3
 declare i8* @llvm.eh.actions(...) #3
 
 ; Function Attrs: nounwind
-declare void @llvm.frameescape(...) #3
+declare void @llvm.localescape(...) #3
 
 ; Function Attrs: nounwind readnone
-declare i8* @llvm.framerecover(i8*, i8*, i32) #2
+declare i8* @llvm.localrecover(i8*, i8*, i32) #2
 
-declare void @llvm.donothing(...)
+declare void @llvm.donothing()
 
 attributes #0 = { "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "no-realign-stack" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" "wineh-parent"="
?f@@YAXXZ" }
 attributes #1 = { "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "no-realign-stack" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
diff --git a/test/CodeGen/WinEH/cppeh-prepared-cleanups.ll b/test/CodeGen/WinEH/cppeh-prepared-cleanups.ll
index b9585892135..b5cfd65030a 100644
--- a/test/CodeGen/WinEH/cppeh-prepared-cleanups.ll
+++ b/test/CodeGen/WinEH/cppeh-prepared-cleanups.ll
@@ -30,13 +30,14 @@ $_TI1H = comdat any
 ; CHECK-LABEL: "?test1@@YAXXZ":
 ; CHECK:             .seh_handlerdata
 ; CHECK-NEXT:        .long   ("$cppxdata$?test1@@YAXXZ")@IMGREL
+; CHECK-NEXT: .align 4
 ; CHECK-NEXT:"$cppxdata$?test1@@YAXXZ":
 ; CHECK-NEXT:        .long   429065506
 ; CHECK-NEXT:        .long   1
 ; CHECK-NEXT:        .long   ("$stateUnwindMap$?test1@@YAXXZ")@IMGREL
 ; CHECK-NEXT:        .long   0
 ; CHECK-NEXT:        .long   0
-; CHECK-NEXT:        .long   1
+; CHECK-NEXT:        .long   2
 ; CHECK-NEXT:        .long   ("$ip2state$?test1@@YAXXZ")@IMGREL
 ; CHECK-NEXT:        .long   32
 ; CHECK-NEXT:        .long   0
@@ -45,10 +46,12 @@ $_TI1H = comdat any
 ; CHECK-NEXT:        .long   -1
 ; CHECK-NEXT:        .long   "?test1@@YAXXZ.cleanup"@IMGREL
 ; CHECK-NEXT:"$ip2state$?test1@@YAXXZ":
+; CHECK-NEXT:        .long   .Lfunc_begin0@IMGREL
+; CHECK-NEXT:        .long   -1
 ; CHECK-NEXT:        .long   .Ltmp0@IMGREL
 ; CHECK-NEXT:        .long   0
 
-define void @"\01?test1@@YAXXZ"() #0 {
+define void @"\01?test1@@YAXXZ"() #0 personality i8* bitcast (i32 (...)* @__CxxFrameHandler3 to i8*) {
 entry:
   %unwindhelp = alloca i64
   %tmp = alloca i32, align 4
@@ -56,7 +59,7 @@ entry:
   %ehselector.slot = alloca i32
   store i32 0, i32* %tmp
   %0 = bitcast i32* %tmp to i8*
-  call void (...) @llvm.frameescape()
+  call void (...) @llvm.localescape()
   store volatile i64 -2, i64* %unwindhelp
   %1 = bitcast i64* %unwindhelp to i8*
   call void @llvm.eh.unwindhelp(i8* %1)
@@ -64,7 +67,7 @@ entry:
           to label %unreachable unwind label %lpad1
 
 lpad1:                                            ; preds = %entry
-  %2 = landingpad { i8*, i32 } personality i8* bitcast (i32 (...)* @__CxxFrameHandler3 to i8*)
+  %2 = landingpad { i8*, i32 }
           cleanup
   %recover = call i8* (...) @llvm.eh.actions(i32 0, void (i8*, i8*)* @"\01?test1@@YAXXZ.cleanup")
   indirectbr i8* %recover, []
@@ -90,6 +93,7 @@ entry:
 ; CHECK-LABEL: "?test2@@YAX_N@Z":
 ; CHECK:             .seh_handlerdata
 ; CHECK-NEXT:        .long   ("$cppxdata$?test2@@YAX_N@Z")@IMGREL
+; CHECK-NEXT: .align 4
 ; CHECK-NEXT:"$cppxdata$?test2@@YAX_N@Z":
 ; CHECK-NEXT:        .long   429065506
 ; CHECK-NEXT:        .long   2
@@ -116,7 +120,7 @@ entry:
 ; CHECK-NEXT:        .long   .Ltmp12@IMGREL
 ; CHECK-NEXT:        .long   0
 
-define void @"\01?test2@@YAX_N@Z"(i1 zeroext %b) #2 {
+define void @"\01?test2@@YAX_N@Z"(i1 zeroext %b) #2 personality i8* bitcast (i32 (...)* @__CxxFrameHandler3 to i8*) {
   %b.addr = alloca i8, align 1
   %s = alloca %struct.S, align 1
   %exn.slot = alloca i8*
@@ -124,7 +128,7 @@ define void @"\01?test2@@YAX_N@Z"(i1 zeroext %b) #2 {
   %s1 = alloca %struct.S, align 1
   %frombool = zext i1 %b to i8
   store i8 %frombool, i8* %b.addr, align 1
-  call void (...) @llvm.frameescape(%struct.S* %s, %struct.S* %s1)
+  call void (...) @llvm.localescape(%struct.S* %s, %struct.S* %s1)
   call void @"\01?may_throw@@YAXXZ"()
   invoke void @"\01?may_throw@@YAXXZ"()
           to label %invoke.cont unwind label %lpad1
@@ -143,13 +147,13 @@ invoke.cont3:                                     ; preds = %if.then
   br label %if.end
 
 lpad1:                                            ; preds = %entry, %if.end
-  %2 = landingpad { i8*, i32 } personality i8* bitcast (i32 (...)* @__CxxFrameHandler3 to i8*)
+  %2 = landingpad { i8*, i32 }
           cleanup
   %recover = call i8* (...) @llvm.eh.actions(i32 0, void (i8*, i8*)* @"\01?test2@@YAX_N@Z.cleanup")
   indirectbr i8* %recover, []
 
 lpad3:                                            ; preds = %if.then
-  %3 = landingpad { i8*, i32 } personality i8* bitcast (i32 (...)* @__CxxFrameHandler3 to i8*)
+  %3 = landingpad { i8*, i32 }
           cleanup
   %recover4 = call i8* (...) @llvm.eh.actions(i32 0, void (i8*, i8*)* @"\01?test2@@YAX_N@Z.cleanup1", i32 0, void (i8*, i8*)* @"\01?test2@@YAX_N@Z.cleanup")
   indirectbr i8* %recover4, []
@@ -186,17 +190,17 @@ entry:
 }
 
 ; Function Attrs: nounwind
-declare void @llvm.frameescape(...) #4
+declare void @llvm.localescape(...) #4
 
 ; Function Attrs: nounwind readnone
-declare i8* @llvm.framerecover(i8*, i8*, i32) #6
+declare i8* @llvm.localrecover(i8*, i8*, i32) #6
 
 ; Function Attrs: nounwind
 declare void @llvm.eh.unwindhelp(i8*) #4
 
-define internal void @"\01?test2@@YAX_N@Z.cleanup"(i8*, i8*) #7 {
+define internal void @"\01?test2@@YAX_N@Z.cleanup"(i8*, i8*) #7 personality i8* bitcast (i32 (...)* @__CxxFrameHandler3 to i8*) {
 entry:
-  %s.i8 = call i8* @llvm.framerecover(i8* bitcast (void (i1)* @"\01?test2@@YAX_N@Z" to i8*), i8* %1, i32 0)
+  %s.i8 = call i8* @llvm.localrecover(i8* bitcast (void (i1)* @"\01?test2@@YAX_N@Z" to i8*), i8* %1, i32 0)
   %s = bitcast i8* %s.i8 to %struct.S*
   call void @"\01??_DS@@QEAA@XZ"(%struct.S* %s) #4
   invoke void @llvm.donothing()
@@ -206,14 +210,14 @@ entry.split:                                      ; preds = %entry
   ret void
 
 stub:                                             ; preds = %entry
-  %2 = landingpad { i8*, i32 } personality i8* bitcast (i32 (...)* @__CxxFrameHandler3 to i8*)
+  %2 = landingpad { i8*, i32 }
           cleanup
   unreachable
 }
 
-define internal void @"\01?test2@@YAX_N@Z.cleanup1"(i8*, i8*) #7 {
+define internal void @"\01?test2@@YAX_N@Z.cleanup1"(i8*, i8*) #7 personality i8* bitcast (i32 (...)* @__CxxFrameHandler3 to i8*) {
 entry:
-  %s1.i8 = call i8* @llvm.framerecover(i8* bitcast (void (i1)* @"\01?test2@@YAX_N@Z" to i8*), i8* %1, i32 1)
+  %s1.i8 = call i8* @llvm.localrecover(i8* bitcast (void (i1)* @"\01?test2@@YAX_N@Z" to i8*), i8* %1, i32 1)
   %s1 = bitcast i8* %s1.i8 to %struct.S*
   call void @"\01??_DS@@QEAA@XZ"(%struct.S* %s1) #4
   invoke void @llvm.donothing()
@@ -223,7 +227,7 @@ entry.split:                                      ; preds = %entry
   ret void
 
 stub:                                             ; preds = %entry
-  %2 = landingpad { i8*, i32 } personality i8* bitcast (i32 (...)* @__CxxFrameHandler3 to i8*)
+  %2 = landingpad { i8*, i32 }
           cleanup
   unreachable
 }
diff --git a/test/CodeGen/WinEH/cppeh-shared-empty-catch.ll b/test/CodeGen/WinEH/cppeh-shared-empty-catch.ll
new file mode 100644
index 00000000000..87ccc9d9ded
--- /dev/null
+++ b/test/CodeGen/WinEH/cppeh-shared-empty-catch.ll
@@ -0,0 +1,110 @@
+; RUN: opt -mtriple=x86_64-pc-windows-msvc -winehprepare -S -o - < %s | FileCheck %s
+
+; This test is based on the following source, built with -O2
+;
+; void f() {
+;   try {
+;     g();
+;     try {
+;       throw;
+;     } catch (int) {
+;     }
+;   } catch (...) {
+;   }
+; }
+;
+
+; ModuleID = '<stdin>'
+target datalayout = "e-m:w-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-pc-windows-msvc"
+
+%rtti.TypeDescriptor2 = type { i8**, i8*, [3 x i8] }
+%eh.CatchHandlerType = type { i32, i8* }
+%eh.ThrowInfo = type { i32, i32, i32, i32 }
+
+$"\01??_R0H@8" = comdat any
+
+@"\01??_7type_info@@6B@" = external constant i8*
+@"\01??_R0H@8" = linkonce_odr global %rtti.TypeDescriptor2 { i8** @"\01??_7type_info@@6B@", i8* null, [3 x i8] c".H\00" }, comdat
+@llvm.eh.handlertype.H.0 = private unnamed_addr constant %eh.CatchHandlerType { i32 0, i8* bitcast (%rtti.TypeDescriptor2* @"\01??_R0H@8" to i8*) }, section "llvm.metadata"
+
+; CHECK-LABEL: define void @"\01?f@@YAXXZ"()
+; CHECK: entry:
+; CHECK:   call void (...) @llvm.localescape()
+; CHECK:   invoke void @"\01?g@@YAXXZ"()
+
+; Function Attrs: nounwind
+define void @"\01?f@@YAXXZ"() #0 personality i8* bitcast (i32 (...)* @__CxxFrameHandler3 to i8*) {
+entry:
+  invoke void @"\01?g@@YAXXZ"()
+          to label %invoke.cont unwind label %lpad
+
+; CHECK-LABEL: invoke.cont:
+; CHECK:   invoke void @_CxxThrowException(i8* null, %eh.ThrowInfo* null)
+; CHECK:           to label %unreachable unwind label %[[LPAD1_LABEL:lpad[0-9]+]]
+
+invoke.cont:                                      ; preds = %entry
+  invoke void @_CxxThrowException(i8* null, %eh.ThrowInfo* null) #4
+          to label %unreachable unwind label %lpad1
+
+lpad:                                             ; preds = %entry
+  %0 = landingpad { i8*, i32 }
+          catch i8* null
+  %1 = extractvalue { i8*, i32 } %0, 0
+  br label %catch2
+
+; Note: Even though this landing pad has two catch clauses, it only has one action because both
+;       handlers do the same thing.
+; CHECK: [[LPAD1_LABEL]]:
+; CHECK:   landingpad { i8*, i32 }
+; CHECK-NEXT:           catch %eh.CatchHandlerType* @llvm.eh.handlertype.H.0
+; CHECK-NEXT:           catch i8* null
+; CHECK-NEXT:   [[RECOVER:\%.+]] = call i8* (...) @llvm.eh.actions(i32 1, i8* null, i32 -1, i8* (i8*, i8*)* @"\01?f@@YAXXZ.catch")
+; CHECK-NEXT:   indirectbr i8* [[RECOVER]], [label %try.cont4]
+
+lpad1:                                            ; preds = %invoke.cont
+  %2 = landingpad { i8*, i32 }
+          catch %eh.CatchHandlerType* @llvm.eh.handlertype.H.0
+          catch i8* null
+  %3 = extractvalue { i8*, i32 } %2, 0
+  br label %catch2
+
+catch2:                                           ; preds = %lpad1, %lpad
+  %exn.slot.0 = phi i8* [ %3, %lpad1 ], [ %1, %lpad ]
+  tail call void @llvm.eh.begincatch(i8* %exn.slot.0, i8* null) #3
+  tail call void @llvm.eh.endcatch() #3
+  br label %try.cont4
+
+try.cont4:                                        ; preds = %catch, %catch2
+  ret void
+
+unreachable:                                      ; preds = %invoke.cont
+  unreachable
+
+; CHECK: }
+}
+
+declare void @"\01?g@@YAXXZ"() #1
+
+declare i32 @__CxxFrameHandler3(...)
+
+declare void @_CxxThrowException(i8*, %eh.ThrowInfo*)
+
+; Function Attrs: nounwind readnone
+declare i32 @llvm.eh.typeid.for(i8*) #2
+
+; Function Attrs: nounwind
+declare void @llvm.eh.begincatch(i8* nocapture, i8* nocapture) #3
+
+; Function Attrs: nounwind
+declare void @llvm.eh.endcatch() #3
+
+attributes #0 = { nounwind "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "no-realign-stack" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #1 = { "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "no-realign-stack" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #2 = { nounwind readnone }
+attributes #3 = { nounwind }
+attributes #4 = { noreturn }
+
+!llvm.ident = !{!0}
+
+!0 = !{!"clang version 3.7.0 (trunk 235112) (llvm/trunk 235121)"}
diff --git a/test/CodeGen/WinEH/cppeh-similar-catch-blocks.ll b/test/CodeGen/WinEH/cppeh-similar-catch-blocks.ll
new file mode 100644
index 00000000000..09213536815
--- /dev/null
+++ b/test/CodeGen/WinEH/cppeh-similar-catch-blocks.ll
@@ -0,0 +1,394 @@
+; RUN: opt -mtriple=x86_64-pc-windows-msvc -winehprepare -S -o - < %s | FileCheck %s
+
+; This test is based on the following code:
+;
+; int main(void) {
+;   try {
+;     try {
+;       throw 'a';
+;     } catch (char c) {
+;       printf("%c\n", c);
+;     }
+;     throw 1;
+;   } catch(int x) {
+;     printf("%d\n", x);
+;   } catch(...) {
+;     printf("...\n");
+;   }
+;   try {
+;     try {
+;       throw 'b';
+;     } catch (char c) {
+;       printf("%c\n", c);
+;     }
+;     throw 2;
+;   } catch(int x) {
+;     printf("%d\n", x);
+;   } catch (char c) {
+;     printf("%c\n", c);
+;   } catch(...) {
+;     printf("...\n");
+;   }
+;   return 0;
+; }
+
+; This test is just checking for failures in processing the IR.
+; Extensive handler matching is not required.
+
+; ModuleID = 'cppeh-similar-catch-blocks.cpp'
+target datalayout = "e-m:w-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-pc-windows-msvc"
+
+%rtti.TypeDescriptor2 = type { i8**, i8*, [3 x i8] }
+%eh.CatchHandlerType = type { i32, i8* }
+%eh.CatchableType = type { i32, i32, i32, i32, i32, i32, i32 }
+%eh.CatchableTypeArray.1 = type { i32, [1 x i32] }
+%eh.ThrowInfo = type { i32, i32, i32, i32 }
+
+$"\01??_R0H@8" = comdat any
+
+$"\01??_R0D@8" = comdat any
+
+$"_CT??_R0D@81" = comdat any
+
+$_CTA1D = comdat any
+
+$_TI1D = comdat any
+
+$"\01??_C@_03PJCJOCBM@?$CFc?6?$AA@" = comdat any
+
+$"_CT??_R0H@84" = comdat any
+
+$_CTA1H = comdat any
+
+$_TI1H = comdat any
+
+$"\01??_C@_04MPPNMCOK@?4?4?4?6?$AA@" = comdat any
+
+$"\01??_C@_03PMGGPEJJ@?$CFd?6?$AA@" = comdat any
+
+@"\01??_7type_info@@6B@" = external constant i8*
+@"\01??_R0H@8" = linkonce_odr global %rtti.TypeDescriptor2 { i8** @"\01??_7type_info@@6B@", i8* null, [3 x i8] c".H\00" }, comdat
+@llvm.eh.handlertype.H.0 = private unnamed_addr constant %eh.CatchHandlerType { i32 0, i8* bitcast (%rtti.TypeDescriptor2* @"\01??_R0H@8" to i8*) }, section "llvm.metadata"
+@"\01??_R0D@8" = linkonce_odr global %rtti.TypeDescriptor2 { i8** @"\01??_7type_info@@6B@", i8* null, [3 x i8] c".D\00" }, comdat
+@llvm.eh.handlertype.D.0 = private unnamed_addr constant %eh.CatchHandlerType { i32 0, i8* bitcast (%rtti.TypeDescriptor2* @"\01??_R0D@8" to i8*) }, section "llvm.metadata"
+@__ImageBase = external constant i8
+@"_CT??_R0D@81" = linkonce_odr unnamed_addr constant %eh.CatchableType { i32 1, i32 trunc (i64 sub nuw nsw (i64 ptrtoint (%rtti.TypeDescriptor2* @"\01??_R0D@8" to i64), i64 ptrtoint (i8* @__ImageBase to i64)) to i32), i32 0, i32 -1, i32 0, i32 1, i32 0 }, section ".xdata", comdat
+@_CTA1D = linkonce_odr unnamed_addr constant %eh.CatchableTypeArray.1 { i32 1, [1 x i32] [i32 trunc (i64 sub nuw nsw (i64 ptrtoint (%eh.CatchableType* @"_CT??_R0D@81" to i64), i64 ptrtoint (i8* @__ImageBase to i64)) to i32)] }, section ".xdata", comdat
+@_TI1D = linkonce_odr unnamed_addr constant %eh.ThrowInfo { i32 0, i32 0, i32 0, i32 trunc (i64 sub nuw nsw (i64 ptrtoint (%eh.CatchableTypeArray.1* @_CTA1D to i64), i64 ptrtoint (i8* @__ImageBase to i64)) to i32) }, section ".xdata", comdat
+@"\01??_C@_03PJCJOCBM@?$CFc?6?$AA@" = linkonce_odr unnamed_addr constant [4 x i8] c"%c\0A\00", comdat, align 1
+@"_CT??_R0H@84" = linkonce_odr unnamed_addr constant %eh.CatchableType { i32 1, i32 trunc (i64 sub nuw nsw (i64 ptrtoint (%rtti.TypeDescriptor2* @"\01??_R0H@8" to i64), i64 ptrtoint (i8* @__ImageBase to i64)) to i32), i32 0, i32 -1, i32 0, i32 4, i32 0 }, section ".xdata", comdat
+@_CTA1H = linkonce_odr unnamed_addr constant %eh.CatchableTypeArray.1 { i32 1, [1 x i32] [i32 trunc (i64 sub nuw nsw (i64 ptrtoint (%eh.CatchableType* @"_CT??_R0H@84" to i64), i64 ptrtoint (i8* @__ImageBase to i64)) to i32)] }, section ".xdata", comdat
+@_TI1H = linkonce_odr unnamed_addr constant %eh.ThrowInfo { i32 0, i32 0, i32 0, i32 trunc (i64 sub nuw nsw (i64 ptrtoint (%eh.CatchableTypeArray.1* @_CTA1H to i64), i64 ptrtoint (i8* @__ImageBase to i64)) to i32) }, section ".xdata", comdat
+@"\01??_C@_04MPPNMCOK@?4?4?4?6?$AA@" = linkonce_odr unnamed_addr constant [5 x i8] c"...\0A\00", comdat, align 1
+@"\01??_C@_03PMGGPEJJ@?$CFd?6?$AA@" = linkonce_odr unnamed_addr constant [4 x i8] c"%d\0A\00", comdat, align 1
+
+; This is just a minimal check to verify that main was handled by WinEHPrepare.
+; CHECK: define i32 @main()
+; CHECK: entry:
+; CHECK:   call void (...) @llvm.localescape(i32* [[X_PTR:\%.+]], i32* [[X2_PTR:\%.+]], i8* [[C2_PTR:\%.+]], i8* [[C3_PTR:\%.+]], i8* [[C_PTR:\%.+]])
+; CHECK:   invoke void @_CxxThrowException
+; CHECK: }
+
+; Function Attrs: uwtable
+define i32 @main() #0 personality i8* bitcast (i32 (...)* @__CxxFrameHandler3 to i8*) {
+entry:
+  %retval = alloca i32, align 4
+  %tmp = alloca i8, align 1
+  %exn.slot = alloca i8*
+  %ehselector.slot = alloca i32
+  %c = alloca i8, align 1
+  %tmp3 = alloca i32, align 4
+  %x = alloca i32, align 4
+  %tmp20 = alloca i8, align 1
+  %c28 = alloca i8, align 1
+  %tmp34 = alloca i32, align 4
+  %c48 = alloca i8, align 1
+  %x56 = alloca i32, align 4
+  store i32 0, i32* %retval
+  store i8 97, i8* %tmp
+  invoke void @_CxxThrowException(i8* %tmp, %eh.ThrowInfo* @_TI1D) #4
+          to label %unreachable unwind label %lpad
+
+lpad:                                             ; preds = %entry
+  %0 = landingpad { i8*, i32 }
+          catch %eh.CatchHandlerType* @llvm.eh.handlertype.D.0
+          catch %eh.CatchHandlerType* @llvm.eh.handlertype.H.0
+          catch i8* null
+  %1 = extractvalue { i8*, i32 } %0, 0
+  store i8* %1, i8** %exn.slot
+  %2 = extractvalue { i8*, i32 } %0, 1
+  store i32 %2, i32* %ehselector.slot
+  br label %catch.dispatch
+
+catch.dispatch:                                   ; preds = %lpad
+  %sel = load i32, i32* %ehselector.slot
+  %3 = call i32 @llvm.eh.typeid.for(i8* bitcast (%eh.CatchHandlerType* @llvm.eh.handlertype.D.0 to i8*)) #2
+  %matches = icmp eq i32 %sel, %3
+  br i1 %matches, label %catch, label %catch.dispatch5
+
+catch:                                            ; preds = %catch.dispatch
+  %exn = load i8*, i8** %exn.slot
+  call void @llvm.eh.begincatch(i8* %exn, i8* %c) #2
+  %4 = load i8, i8* %c, align 1
+  %conv = sext i8 %4 to i32
+  %call = invoke i32 (i8*, ...) @printf(i8* getelementptr inbounds ([4 x i8], [4 x i8]* @"\01??_C@_03PJCJOCBM@?$CFc?6?$AA@", i32 0, i32 0), i32 %conv)
+          to label %invoke.cont unwind label %lpad2
+
+invoke.cont:                                      ; preds = %catch
+  call void @llvm.eh.endcatch() #2
+  br label %try.cont
+
+try.cont:                                         ; preds = %invoke.cont
+  store i32 1, i32* %tmp3
+  %5 = bitcast i32* %tmp3 to i8*
+  invoke void @_CxxThrowException(i8* %5, %eh.ThrowInfo* @_TI1H) #4
+          to label %unreachable unwind label %lpad4
+
+lpad2:                                            ; preds = %catch
+  %6 = landingpad { i8*, i32 }
+          catch %eh.CatchHandlerType* @llvm.eh.handlertype.H.0
+          catch i8* null
+  %7 = extractvalue { i8*, i32 } %6, 0
+  store i8* %7, i8** %exn.slot
+  %8 = extractvalue { i8*, i32 } %6, 1
+  store i32 %8, i32* %ehselector.slot
+  call void @llvm.eh.endcatch() #2
+  br label %catch.dispatch5
+
+lpad4:                                            ; preds = %try.cont
+  %9 = landingpad { i8*, i32 }
+          catch %eh.CatchHandlerType* @llvm.eh.handlertype.H.0
+          catch i8* null
+  %10 = extractvalue { i8*, i32 } %9, 0
+  store i8* %10, i8** %exn.slot
+  %11 = extractvalue { i8*, i32 } %9, 1
+  store i32 %11, i32* %ehselector.slot
+  br label %catch.dispatch5
+
+catch.dispatch5:                                  ; preds = %lpad4, %lpad2, %catch.dispatch
+  %sel6 = load i32, i32* %ehselector.slot
+  %12 = call i32 @llvm.eh.typeid.for(i8* bitcast (%eh.CatchHandlerType* @llvm.eh.handlertype.H.0 to i8*)) #2
+  %matches7 = icmp eq i32 %sel6, %12
+  br i1 %matches7, label %catch13, label %catch8
+
+catch13:                                          ; preds = %catch.dispatch5
+  %exn14 = load i8*, i8** %exn.slot
+  %13 = bitcast i32* %x to i8*
+  call void @llvm.eh.begincatch(i8* %exn14, i8* %13) #2
+  %14 = load i32, i32* %x, align 4
+  %call18 = invoke i32 (i8*, ...) @printf(i8* getelementptr inbounds ([4 x i8], [4 x i8]* @"\01??_C@_03PMGGPEJJ@?$CFd?6?$AA@", i32 0, i32 0), i32 %14)
+          to label %invoke.cont17 unwind label %lpad16
+
+invoke.cont17:                                    ; preds = %catch13
+  call void @llvm.eh.endcatch() #2
+  br label %try.cont19
+
+try.cont19:                                       ; preds = %invoke.cont17, %invoke.cont11
+  store i8 98, i8* %tmp20
+  invoke void @_CxxThrowException(i8* %tmp20, %eh.ThrowInfo* @_TI1D) #4
+          to label %unreachable unwind label %lpad21
+
+catch8:                                           ; preds = %catch.dispatch5
+  %exn9 = load i8*, i8** %exn.slot
+  call void @llvm.eh.begincatch(i8* %exn9, i8* null) #2
+  %call12 = invoke i32 (i8*, ...) @printf(i8* getelementptr inbounds ([5 x i8], [5 x i8]* @"\01??_C@_04MPPNMCOK@?4?4?4?6?$AA@", i32 0, i32 0))
+          to label %invoke.cont11 unwind label %lpad10
+
+invoke.cont11:                                    ; preds = %catch8
+  call void @llvm.eh.endcatch() #2
+  br label %try.cont19
+
+lpad10:                                           ; preds = %catch8
+  %15 = landingpad { i8*, i32 }
+          cleanup
+  %16 = extractvalue { i8*, i32 } %15, 0
+  store i8* %16, i8** %exn.slot
+  %17 = extractvalue { i8*, i32 } %15, 1
+  store i32 %17, i32* %ehselector.slot
+  call void @llvm.eh.endcatch() #2
+  br label %eh.resume
+
+lpad16:                                           ; preds = %catch13
+  %18 = landingpad { i8*, i32 }
+          cleanup
+  %19 = extractvalue { i8*, i32 } %18, 0
+  store i8* %19, i8** %exn.slot
+  %20 = extractvalue { i8*, i32 } %18, 1
+  store i32 %20, i32* %ehselector.slot
+  call void @llvm.eh.endcatch() #2
+  br label %eh.resume
+
+lpad21:                                           ; preds = %try.cont19
+  %21 = landingpad { i8*, i32 }
+          catch i8* bitcast (%eh.CatchHandlerType* @llvm.eh.handlertype.D.0 to i8*)
+          catch i8* bitcast (%eh.CatchHandlerType* @llvm.eh.handlertype.H.0 to i8*)
+          catch i8* null
+  %22 = extractvalue { i8*, i32 } %21, 0
+  store i8* %22, i8** %exn.slot
+  %23 = extractvalue { i8*, i32 } %21, 1
+  store i32 %23, i32* %ehselector.slot
+  br label %catch.dispatch22
+
+catch.dispatch22:                                 ; preds = %lpad21
+  %sel23 = load i32, i32* %ehselector.slot
+  %24 = call i32 @llvm.eh.typeid.for(i8* bitcast (%eh.CatchHandlerType* @llvm.eh.handlertype.D.0 to i8*)) #2
+  %matches24 = icmp eq i32 %sel23, %24
+  br i1 %matches24, label %catch25, label %catch.dispatch36
+
+catch25:                                          ; preds = %catch.dispatch22
+  %exn26 = load i8*, i8** %exn.slot
+  call void @llvm.eh.begincatch(i8* %exn26, i8* %c28) #2
+  %25 = load i8, i8* %c28, align 1
+  %conv29 = sext i8 %25 to i32
+  %call32 = invoke i32 (i8*, ...) @printf(i8* getelementptr inbounds ([4 x i8], [4 x i8]* @"\01??_C@_03PJCJOCBM@?$CFc?6?$AA@", i32 0, i32 0), i32 %conv29)
+          to label %invoke.cont31 unwind label %lpad30
+
+invoke.cont31:                                    ; preds = %catch25
+  call void @llvm.eh.endcatch() #2
+  br label %try.cont33
+
+try.cont33:                                       ; preds = %invoke.cont31
+  store i32 2, i32* %tmp34
+  %26 = bitcast i32* %tmp34 to i8*
+  invoke void @_CxxThrowException(i8* %26, %eh.ThrowInfo* @_TI1H) #4
+          to label %unreachable unwind label %lpad35
+
+lpad30:                                           ; preds = %catch25
+  %27 = landingpad { i8*, i32 }
+          catch i8* bitcast (%eh.CatchHandlerType* @llvm.eh.handlertype.H.0 to i8*)
+          catch i8* bitcast (%eh.CatchHandlerType* @llvm.eh.handlertype.D.0 to i8*)
+          catch i8* null
+  %28 = extractvalue { i8*, i32 } %27, 0
+  store i8* %28, i8** %exn.slot
+  %29 = extractvalue { i8*, i32 } %27, 1
+  store i32 %29, i32* %ehselector.slot
+  call void @llvm.eh.endcatch() #2
+  br label %catch.dispatch36
+
+lpad35:                                           ; preds = %try.cont33
+  %30 = landingpad { i8*, i32 }
+          catch i8* bitcast (%eh.CatchHandlerType* @llvm.eh.handlertype.H.0 to i8*)
+          catch i8* bitcast (%eh.CatchHandlerType* @llvm.eh.handlertype.D.0 to i8*)
+          catch i8* null
+  %31 = extractvalue { i8*, i32 } %30, 0
+  store i8* %31, i8** %exn.slot
+  %32 = extractvalue { i8*, i32 } %30, 1
+  store i32 %32, i32* %ehselector.slot
+  br label %catch.dispatch36
+
+catch.dispatch36:                                 ; preds = %lpad35, %lpad30, %catch.dispatch22
+  %sel37 = load i32, i32* %ehselector.slot
+  %33 = call i32 @llvm.eh.typeid.for(i8* bitcast (%eh.CatchHandlerType* @llvm.eh.handlertype.H.0 to i8*)) #2
+  %matches38 = icmp eq i32 %sel37, %33
+  br i1 %matches38, label %catch53, label %catch.fallthrough
+
+catch53:                                          ; preds = %catch.dispatch36
+  %exn54 = load i8*, i8** %exn.slot
+  %34 = bitcast i32* %x56 to i8*
+  call void @llvm.eh.begincatch(i8* %exn54, i8* %34) #2
+  %35 = load i32, i32* %x56, align 4
+  %call59 = invoke i32 (i8*, ...) @printf(i8* getelementptr inbounds ([4 x i8], [4 x i8]* @"\01??_C@_03PMGGPEJJ@?$CFd?6?$AA@", i32 0, i32 0), i32 %35)
+          to label %invoke.cont58 unwind label %lpad57
+
+invoke.cont58:                                    ; preds = %catch53
+  call void @llvm.eh.endcatch() #2
+  br label %try.cont60
+
+try.cont60:                                       ; preds = %invoke.cont58, %invoke.cont51, %invoke.cont43
+  ret i32 0
+
+catch.fallthrough:                                ; preds = %catch.dispatch36
+  %36 = call i32 @llvm.eh.typeid.for(i8* bitcast (%eh.CatchHandlerType* @llvm.eh.handlertype.D.0 to i8*)) #2
+  %matches39 = icmp eq i32 %sel37, %36
+  br i1 %matches39, label %catch45, label %catch40
+
+catch45:                                          ; preds = %catch.fallthrough
+  %exn46 = load i8*, i8** %exn.slot
+  call void @llvm.eh.begincatch(i8* %exn46, i8* %c48) #2
+  %37 = load i8, i8* %c48, align 1
+  %conv49 = sext i8 %37 to i32
+  %call52 = invoke i32 (i8*, ...) @printf(i8* getelementptr inbounds ([4 x i8], [4 x i8]* @"\01??_C@_03PJCJOCBM@?$CFc?6?$AA@", i32 0, i32 0), i32 %conv49)
+          to label %invoke.cont51 unwind label %lpad50
+
+invoke.cont51:                                    ; preds = %catch45
+  call void @llvm.eh.endcatch() #2
+  br label %try.cont60
+
+catch40:                                          ; preds = %catch.fallthrough
+  %exn41 = load i8*, i8** %exn.slot
+  call void @llvm.eh.begincatch(i8* %exn41, i8* null) #2
+  %call44 = invoke i32 (i8*, ...) @printf(i8* getelementptr inbounds ([5 x i8], [5 x i8]* @"\01??_C@_04MPPNMCOK@?4?4?4?6?$AA@", i32 0, i32 0))
+          to label %invoke.cont43 unwind label %lpad42
+
+invoke.cont43:                                    ; preds = %catch40
+  call void @llvm.eh.endcatch() #2
+  br label %try.cont60
+
+lpad42:                                           ; preds = %catch40
+  %38 = landingpad { i8*, i32 }
+          cleanup
+  %39 = extractvalue { i8*, i32 } %38, 0
+  store i8* %39, i8** %exn.slot
+  %40 = extractvalue { i8*, i32 } %38, 1
+  store i32 %40, i32* %ehselector.slot
+  call void @llvm.eh.endcatch() #2
+  br label %eh.resume
+
+lpad50:                                           ; preds = %catch45
+  %41 = landingpad { i8*, i32 }
+          cleanup
+  %42 = extractvalue { i8*, i32 } %41, 0
+  store i8* %42, i8** %exn.slot
+  %43 = extractvalue { i8*, i32 } %41, 1
+  store i32 %43, i32* %ehselector.slot
+  call void @llvm.eh.endcatch() #2
+  br label %eh.resume
+
+lpad57:                                           ; preds = %catch53
+  %44 = landingpad { i8*, i32 }
+          cleanup
+  %45 = extractvalue { i8*, i32 } %44, 0
+  store i8* %45, i8** %exn.slot
+  %46 = extractvalue { i8*, i32 } %44, 1
+  store i32 %46, i32* %ehselector.slot
+  call void @llvm.eh.endcatch() #2
+  br label %eh.resume
+
+eh.resume:                                        ; preds = %lpad57, %lpad50, %lpad42, %lpad16, %lpad10
+  %exn61 = load i8*, i8** %exn.slot
+  %sel62 = load i32, i32* %ehselector.slot
+  %lpad.val = insertvalue { i8*, i32 } undef, i8* %exn61, 0
+  %lpad.val63 = insertvalue { i8*, i32 } %lpad.val, i32 %sel62, 1
+  resume { i8*, i32 } %lpad.val63
+
+unreachable:                                      ; preds = %try.cont33, %try.cont19, %try.cont, %entry
+  unreachable
+}
+
+declare void @_CxxThrowException(i8*, %eh.ThrowInfo*)
+
+declare i32 @__CxxFrameHandler3(...)
+
+; Function Attrs: nounwind readnone
+declare i32 @llvm.eh.typeid.for(i8*) #1
+
+; Function Attrs: nounwind
+declare void @llvm.eh.begincatch(i8* nocapture, i8* nocapture) #2
+
+declare i32 @printf(i8*, ...) #3
+
+; Function Attrs: nounwind
+declare void @llvm.eh.endcatch() #2
+
+attributes #0 = { uwtable "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #1 = { nounwind readnone }
+attributes #2 = { nounwind }
+attributes #3 = { "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #4 = { noreturn }
+
+!llvm.module.flags = !{!0}
+!llvm.ident = !{!1}
+
+!0 = !{i32 1, !"PIC Level", i32 2}
+!1 = !{!"clang version 3.7.0 (trunk 235214) (llvm/trunk 235213)"}
diff --git a/test/CodeGen/WinEH/cppeh-state-calc-1.ll b/test/CodeGen/WinEH/cppeh-state-calc-1.ll
new file mode 100644
index 00000000000..abc5d5292cf
--- /dev/null
+++ b/test/CodeGen/WinEH/cppeh-state-calc-1.ll
@@ -0,0 +1,289 @@
+; RUN: llc < %s | FileCheck %s
+
+; This test was generated from the following code.
+;
+; void test() {
+;   try {
+;     try {
+;       try {
+;         two();
+;         throw 2;
+;       } catch (int x) {
+;         catch_two();
+;       }
+;       a();
+;       throw 'a';
+;     } catch (char c) {
+;       catch_a();
+;     }
+;     one();
+;     throw 1;
+;   } catch(int x) { 
+;     catch_one();
+;   } catch(...) {
+;     catch_all();
+;   }
+; }
+;
+; The function calls before the throws were declared as 'noexcept' and are
+; just here to make blocks easier to identify in the IR.
+
+; ModuleID = '<stdin>'
+target datalayout = "e-m:w-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-pc-windows-msvc"
+
+%rtti.TypeDescriptor2 = type { i8**, i8*, [3 x i8] }
+%eh.CatchHandlerType = type { i32, i8* }
+%eh.CatchableType = type { i32, i32, i32, i32, i32, i32, i32 }
+%eh.CatchableTypeArray.1 = type { i32, [1 x i32] }
+%eh.ThrowInfo = type { i32, i32, i32, i32 }
+
+$"\01??_R0H@8" = comdat any
+
+$"\01??_R0D@8" = comdat any
+
+$"_CT??_R0H@84" = comdat any
+
+$_CTA1H = comdat any
+
+$_TI1H = comdat any
+
+$"_CT??_R0D@81" = comdat any
+
+$_CTA1D = comdat any
+
+$_TI1D = comdat any
+
+@"\01??_7type_info@@6B@" = external constant i8*
+@"\01??_R0H@8" = linkonce_odr global %rtti.TypeDescriptor2 { i8** @"\01??_7type_info@@6B@", i8* null, [3 x i8] c".H\00" }, comdat
+@llvm.eh.handlertype.H.0 = private unnamed_addr constant %eh.CatchHandlerType { i32 0, i8* bitcast (%rtti.TypeDescriptor2* @"\01??_R0H@8" to i8*) }, section "llvm.metadata"
+@"\01??_R0D@8" = linkonce_odr global %rtti.TypeDescriptor2 { i8** @"\01??_7type_info@@6B@", i8* null, [3 x i8] c".D\00" }, comdat
+@llvm.eh.handlertype.D.0 = private unnamed_addr constant %eh.CatchHandlerType { i32 0, i8* bitcast (%rtti.TypeDescriptor2* @"\01??_R0D@8" to i8*) }, section "llvm.metadata"
+@__ImageBase = external constant i8
+@"_CT??_R0H@84" = linkonce_odr unnamed_addr constant %eh.CatchableType { i32 1, i32 trunc (i64 sub nuw nsw (i64 ptrtoint (%rtti.TypeDescriptor2* @"\01??_R0H@8" to i64), i64 ptrtoint (i8* @__ImageBase to i64)) to i32), i32 0, i32 -1, i32 0, i32 4, i32 0 }, section ".xdata", comdat
+@_CTA1H = linkonce_odr unnamed_addr constant %eh.CatchableTypeArray.1 { i32 1, [1 x i32] [i32 trunc (i64 sub nuw nsw (i64 ptrtoint (%eh.CatchableType* @"_CT??_R0H@84" to i64), i64 ptrtoint (i8* @__ImageBase to i64)) to i32)] }, section ".xdata", comdat
+@_TI1H = linkonce_odr unnamed_addr constant %eh.ThrowInfo { i32 0, i32 0, i32 0, i32 trunc (i64 sub nuw nsw (i64 ptrtoint (%eh.CatchableTypeArray.1* @_CTA1H to i64), i64 ptrtoint (i8* @__ImageBase to i64)) to i32) }, section ".xdata", comdat
+@"_CT??_R0D@81" = linkonce_odr unnamed_addr constant %eh.CatchableType { i32 1, i32 trunc (i64 sub nuw nsw (i64 ptrtoint (%rtti.TypeDescriptor2* @"\01??_R0D@8" to i64), i64 ptrtoint (i8* @__ImageBase to i64)) to i32), i32 0, i32 -1, i32 0, i32 1, i32 0 }, section ".xdata", comdat
+@_CTA1D = linkonce_odr unnamed_addr constant %eh.CatchableTypeArray.1 { i32 1, [1 x i32] [i32 trunc (i64 sub nuw nsw (i64 ptrtoint (%eh.CatchableType* @"_CT??_R0D@81" to i64), i64 ptrtoint (i8* @__ImageBase to i64)) to i32)] }, section ".xdata", comdat
+@_TI1D = linkonce_odr unnamed_addr constant %eh.ThrowInfo { i32 0, i32 0, i32 0, i32 trunc (i64 sub nuw nsw (i64 ptrtoint (%eh.CatchableTypeArray.1* @_CTA1D to i64), i64 ptrtoint (i8* @__ImageBase to i64)) to i32) }, section ".xdata", comdat
+
+; Function Attrs: nounwind uwtable
+define void @"\01?test@@YAXXZ"() #0 personality i8* bitcast (i32 (...)* @__CxxFrameHandler3 to i8*) {
+entry:
+  %tmp = alloca i32, align 4
+  %x = alloca i32, align 4
+  %tmp2 = alloca i8, align 1
+  %c = alloca i8, align 1
+  %tmp11 = alloca i32, align 4
+  %x21 = alloca i32, align 4
+  call void @"\01?two@@YAXXZ"() #3
+  store i32 2, i32* %tmp
+  %0 = bitcast i32* %tmp to i8*
+  call void (...) @llvm.localescape(i32* %x, i8* %c, i32* %x21)
+  invoke void @_CxxThrowException(i8* %0, %eh.ThrowInfo* @_TI1H) #5
+          to label %unreachable unwind label %lpad
+
+lpad:                                             ; preds = %entry
+  %1 = landingpad { i8*, i32 }
+          catch i8* bitcast (%eh.CatchHandlerType* @llvm.eh.handlertype.H.0 to i8*)
+          catch %eh.CatchHandlerType* @llvm.eh.handlertype.D.0
+          catch %eh.CatchHandlerType* @llvm.eh.handlertype.H.0
+          catch i8* null
+  %recover = call i8* (...) @llvm.eh.actions(i32 1, i8* bitcast (%eh.CatchHandlerType* @llvm.eh.handlertype.H.0 to i8*), i32 0, i8* (i8*, i8*)* @"\01?test@@YAXXZ.catch", i32 1, i8* bitcast (%eh.CatchHandlerType* @llvm.eh.handlertype.D.0 to i8*), i32 1, i8* (i8*, i8*)* @"\01?test@@YAXXZ.catch1", i32 1, i8* bitcast (%eh.CatchHandlerType* @llvm.eh.handlertype.H.0 to i8*), i32 2, i8* (i8*, i8*)* @"\01?test@@YAXXZ.catch2", i32 1, i8* null, i32 -1, i8* (i8*, i8*)* @"\01?test@@YAXXZ.catch3")
+  indirectbr i8* %recover, [label %try.cont, label %try.cont10, label %try.cont22]
+
+try.cont:                                         ; preds = %lpad
+  call void @"\01?a@@YAXXZ"() #3
+  store i8 97, i8* %tmp2
+  invoke void @_CxxThrowException(i8* %tmp2, %eh.ThrowInfo* @_TI1D) #5
+          to label %unreachable unwind label %lpad3
+
+lpad3:                                            ; preds = %try.cont
+  %2 = landingpad { i8*, i32 }
+          catch %eh.CatchHandlerType* @llvm.eh.handlertype.D.0
+          catch %eh.CatchHandlerType* @llvm.eh.handlertype.H.0
+          catch i8* null
+  %recover1 = call i8* (...) @llvm.eh.actions(i32 1, i8* bitcast (%eh.CatchHandlerType* @llvm.eh.handlertype.D.0 to i8*), i32 1, i8* (i8*, i8*)* @"\01?test@@YAXXZ.catch1", i32 1, i8* bitcast (%eh.CatchHandlerType* @llvm.eh.handlertype.H.0 to i8*), i32 2, i8* (i8*, i8*)* @"\01?test@@YAXXZ.catch2", i32 1, i8* null, i32 -1, i8* (i8*, i8*)* @"\01?test@@YAXXZ.catch3")
+  indirectbr i8* %recover1, [label %try.cont10, label %try.cont22]
+
+try.cont10:                                       ; preds = %lpad3, %lpad
+  call void @"\01?one@@YAXXZ"() #3
+  store i32 1, i32* %tmp11
+  %3 = bitcast i32* %tmp11 to i8*
+  invoke void @_CxxThrowException(i8* %3, %eh.ThrowInfo* @_TI1H) #5
+          to label %unreachable unwind label %lpad12
+
+lpad12:                                           ; preds = %try.cont10
+  %4 = landingpad { i8*, i32 }
+          catch %eh.CatchHandlerType* @llvm.eh.handlertype.H.0
+          catch i8* null
+  %recover2 = call i8* (...) @llvm.eh.actions(i32 1, i8* bitcast (%eh.CatchHandlerType* @llvm.eh.handlertype.H.0 to i8*), i32 2, i8* (i8*, i8*)* @"\01?test@@YAXXZ.catch2", i32 1, i8* null, i32 -1, i8* (i8*, i8*)* @"\01?test@@YAXXZ.catch3")
+  indirectbr i8* %recover2, [label %try.cont22]
+
+try.cont22:                                       ; preds = %lpad12, %lpad3, %lpad
+  ret void
+
+unreachable:                                      ; preds = %try.cont10, %try.cont, %entry
+  unreachable
+}
+
+; Function Attrs: nounwind
+declare void @"\01?two@@YAXXZ"() #1
+
+declare void @_CxxThrowException(i8*, %eh.ThrowInfo*)
+
+declare i32 @__CxxFrameHandler3(...)
+
+; Function Attrs: nounwind readnone
+declare i32 @llvm.eh.typeid.for(i8*) #2
+
+; Function Attrs: nounwind
+declare void @llvm.eh.begincatch(i8* nocapture, i8* nocapture) #3
+
+; Function Attrs: nounwind
+declare void @"\01?catch_two@@YAXXZ"() #1
+
+; Function Attrs: nounwind
+declare void @llvm.eh.endcatch() #3
+
+; Function Attrs: nounwind
+declare void @"\01?a@@YAXXZ"() #1
+
+; Function Attrs: nounwind
+declare void @"\01?catch_a@@YAXXZ"() #1
+
+; Function Attrs: nounwind
+declare void @"\01?one@@YAXXZ"() #1
+
+; Function Attrs: nounwind
+declare void @"\01?catch_all@@YAXXZ"() #1
+
+; Function Attrs: nounwind
+declare void @"\01?catch_one@@YAXXZ"() #1
+
+; Function Attrs: nounwind
+declare i8* @llvm.eh.actions(...) #3
+
+define internal i8* @"\01?test@@YAXXZ.catch"(i8*, i8*) #4 personality i8* bitcast (i32 (...)* @__CxxFrameHandler3 to i8*) {
+entry:
+  %x.i8 = call i8* @llvm.localrecover(i8* bitcast (void ()* @"\01?test@@YAXXZ" to i8*), i8* %1, i32 0)
+  %x = bitcast i8* %x.i8 to i32*
+  %2 = bitcast i32* %x to i8*
+  call void @"\01?catch_two@@YAXXZ"() #3
+  invoke void @llvm.donothing()
+          to label %entry.split unwind label %stub
+
+entry.split:                                      ; preds = %entry
+  ret i8* blockaddress(@"\01?test@@YAXXZ", %try.cont)
+
+stub:                                             ; preds = %entry
+  %3 = landingpad { i8*, i32 }
+          cleanup
+  %recover = call i8* (...) @llvm.eh.actions()
+  unreachable
+}
+
+; Function Attrs: nounwind readnone
+declare void @llvm.donothing() #2
+
+define internal i8* @"\01?test@@YAXXZ.catch1"(i8*, i8*) #4 personality i8* bitcast (i32 (...)* @__CxxFrameHandler3 to i8*) {
+entry:
+  call void @"\01?catch_a@@YAXXZ"() #3
+  invoke void @llvm.donothing()
+          to label %entry.split unwind label %stub
+
+entry.split:                                      ; preds = %entry
+  ret i8* blockaddress(@"\01?test@@YAXXZ", %try.cont10)
+
+stub:                                             ; preds = %entry
+  %2 = landingpad { i8*, i32 }
+          cleanup
+  %recover = call i8* (...) @llvm.eh.actions()
+  unreachable
+}
+
+define internal i8* @"\01?test@@YAXXZ.catch2"(i8*, i8*) #4 personality i8* bitcast (i32 (...)* @__CxxFrameHandler3 to i8*) {
+entry:
+  %x21.i8 = call i8* @llvm.localrecover(i8* bitcast (void ()* @"\01?test@@YAXXZ" to i8*), i8* %1, i32 2)
+  %x21 = bitcast i8* %x21.i8 to i32*
+  %2 = bitcast i32* %x21 to i8*
+  call void @"\01?catch_one@@YAXXZ"() #3
+  invoke void @llvm.donothing()
+          to label %entry.split unwind label %stub
+
+entry.split:                                      ; preds = %entry
+  ret i8* blockaddress(@"\01?test@@YAXXZ", %try.cont22)
+
+stub:                                             ; preds = %entry
+  %3 = landingpad { i8*, i32 }
+          cleanup
+  %recover = call i8* (...) @llvm.eh.actions()
+  unreachable
+}
+
+define internal i8* @"\01?test@@YAXXZ.catch3"(i8*, i8*) #4 personality i8* bitcast (i32 (...)* @__CxxFrameHandler3 to i8*) {
+entry:
+  call void @"\01?catch_all@@YAXXZ"() #3
+  invoke void @llvm.donothing()
+          to label %entry.split unwind label %stub
+
+entry.split:                                      ; preds = %entry
+  ret i8* blockaddress(@"\01?test@@YAXXZ", %try.cont22)
+
+stub:                                             ; preds = %entry
+  %2 = landingpad { i8*, i32 }
+          cleanup
+  %recover = call i8* (...) @llvm.eh.actions()
+  unreachable
+}
+
+; Function Attrs: nounwind
+declare void @llvm.localescape(...) #3
+
+; Function Attrs: nounwind readnone
+declare i8* @llvm.localrecover(i8*, i8*, i32) #2
+
+attributes #0 = { nounwind uwtable "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+sse,+sse2" "unsafe-fp-math"="false" "use-soft-float"="false" "wineh-parent"="
?test@@YAXXZ" }
+attributes #1 = { nounwind "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+sse,+sse2" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #2 = { nounwind readnone }
+attributes #3 = { nounwind }
+attributes #4 = { "wineh-parent"="
?test@@YAXXZ" }
+attributes #5 = { noreturn }
+
+!llvm.module.flags = !{!0}
+!llvm.ident = !{!1}
+
+!0 = !{i32 1, !"PIC Level", i32 2}
+!1 = !{!"clang version 3.7.0 (trunk 236059)"}
+
+; CHECK-LABEL: "$cppxdata$?test@@YAXXZ":
+; CHECK-NEXT: 	.long	429065506
+; CHECK-NEXT: 	.long
+; CHECK-NEXT: 	.long	("$stateUnwindMap$?test@@YAXXZ")@IMGREL
+; CHECK-NEXT: 	.long
+; CHECK-NEXT: 	.long	("$tryMap$?test@@YAXXZ")@IMGREL
+; CHECK-NEXT: 	.long
+; CHECK-NEXT: 	.long	("$ip2state$?test@@YAXXZ")@IMGREL
+; CHECK-NEXT: 	.long	40
+; CHECK-NEXT: 	.long	0
+; CHECK-NEXT: 	.long	1
+; CHECK: "$stateUnwindMap$?test@@YAXXZ":
+; CHECK: "$tryMap$?test@@YAXXZ":
+; CHECK: "$handlerMap$0$?test@@YAXXZ":
+; CHECK: "$ip2state$?test@@YAXXZ":
+; CHECK-NEXT: 	.long	.Lfunc_begin0@IMGREL
+; CHECK-NEXT: 	.long	-1
+; CHECK-NEXT: 	.long	.Ltmp0@IMGREL
+; CHECK-NEXT: 	.long	2
+; CHECK-NEXT: 	.long	.Ltmp3@IMGREL
+; CHECK-NEXT: 	.long	1
+; CHECK-NEXT: 	.long	.Ltmp6@IMGREL
+; CHECK-NEXT: 	.long	0
+; CHECK-NEXT: 	.long	.Lfunc_begin1@IMGREL
+; CHECK-NEXT: 	.long	3
+; CHECK-NEXT: 	.long	.Lfunc_begin2@IMGREL
+; CHECK-NEXT: 	.long	4
+; CHECK-NEXT: 	.long	.Lfunc_begin3@IMGREL
+; CHECK-NEXT: 	.long	5
+; CHECK-NEXT: 	.long	.Lfunc_begin4@IMGREL
+; CHECK-NEXT: 	.long	6
diff --git a/test/CodeGen/WinEH/seh-catch-all.ll b/test/CodeGen/WinEH/seh-catch-all.ll
index fb2b9ba7cfd..5ac2295a5b4 100644
--- a/test/CodeGen/WinEH/seh-catch-all.ll
+++ b/test/CodeGen/WinEH/seh-catch-all.ll
@@ -1,4 +1,4 @@
-; RUN: opt -S -winehprepare -sehprepare < %s | FileCheck %s
+; RUN: opt -S -winehprepare < %s | FileCheck %s
 
 target datalayout = "e-m:w-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-pc-windows-msvc"
@@ -21,7 +21,7 @@ declare i32 @__C_specific_handler(...)
 declare i8* @llvm.frameaddress(i32)
 
 ; Function Attrs: uwtable
-define void @seh_catch_all() {
+define void @seh_catch_all() personality i8* bitcast (i32 (...)* @__C_specific_handler to i8*) {
 entry:
   %exn.slot = alloca i8*
   %ehselector.slot = alloca i32
@@ -32,7 +32,7 @@ invoke.cont:                                      ; preds = %entry
   br label %__try.cont
 
 lpad:                                             ; preds = %entry
-  %0 = landingpad { i8*, i32 } personality i8* bitcast (i32 (...)* @__C_specific_handler to i8*)
+  %0 = landingpad { i8*, i32 }
           catch i8* null
   %1 = extractvalue { i8*, i32 } %0, 0
   store i8* %1, i8** %exn.slot
@@ -51,9 +51,9 @@ __try.cont:                                       ; preds = %__except, %invoke.c
 ; CHECK-LABEL: define void @seh_catch_all()
 ; CHECK: landingpad
 ; CHECK-NEXT: catch i8* null
-; CHECK-NEXT: call i8* (...) @llvm.eh.actions(i32 1, i8* null, i32 -1, i8* blockaddress(@seh_catch_all, %catch.all))
+; CHECK-NEXT: call i8* (...) @llvm.eh.actions(i32 1, i8* null, i32 -1, i8* blockaddress(@seh_catch_all, %lpad.split))
 ; CHECK-NEXT: indirectbr
 ;
-; CHECK: catch.all:
+; CHECK: lpad.split:
 ; CHECK-NOT: extractvalue
 ; CHECK: call i32 @puts
diff --git a/test/CodeGen/WinEH/seh-exception-code.ll b/test/CodeGen/WinEH/seh-exception-code.ll
new file mode 100644
index 00000000000..2998e798213
--- /dev/null
+++ b/test/CodeGen/WinEH/seh-exception-code.ll
@@ -0,0 +1,66 @@
+; RUN: opt -winehprepare -S < %s | FileCheck %s
+
+; WinEHPrepare was crashing during phi demotion.
+
+target datalayout = "e-m:w-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-pc-windows-msvc18.0.0"
+
+declare i32 @__C_specific_handler(...)
+
+@str = linkonce_odr unnamed_addr constant [16 x i8] c"caught it! %lx\0A\00", align 1
+
+; Function Attrs: nounwind uwtable
+declare void @maycrash()
+
+; Function Attrs: nounwind
+declare i32 @printf(i8* nocapture readonly, ...)
+
+; Function Attrs: nounwind uwtable
+define void @doit() personality i8* bitcast (i32 (...)* @__C_specific_handler to i8*) {
+entry:
+  invoke void @maycrash()
+          to label %invoke.cont unwind label %lpad
+
+invoke.cont:                                      ; preds = %entry
+  invoke void @maycrash()
+          to label %__try.cont unwind label %lpad.1
+
+lpad:                                             ; preds = %entry
+  %0 = landingpad { i8*, i32 }
+          catch i8* null
+  %1 = extractvalue { i8*, i32 } %0, 0
+  br label %__except
+
+lpad.1:                                           ; preds = %invoke.cont, %lpad
+  %2 = landingpad { i8*, i32 }
+          catch i8* null
+  %3 = extractvalue { i8*, i32 } %2, 0
+  br label %__except
+
+__except:                                         ; preds = %lpad, %lpad.1
+  %exn.slot.0 = phi i8* [ %3, %lpad.1 ], [ %1, %lpad ]
+  %4 = ptrtoint i8* %exn.slot.0 to i64
+  %5 = trunc i64 %4 to i32
+  %call = tail call i32 (i8*, ...) @printf(i8* getelementptr inbounds ([16 x i8], [16 x i8]* @str, i64 0, i64 0), i32 %5)
+  br label %__try.cont
+
+__try.cont:                                       ; preds = %invoke.cont, %__except
+  ret void
+}
+
+; CHECK-LABEL: define void @doit()
+; CHECK: landingpad
+; CHECK: indirectbr i8* %{{[^,]*}}, [label %[[except_split1:.*]]]
+; CHECK: [[except_split1]]:
+; CHECK: call i32 @llvm.eh.exceptioncode()
+; CHECK: br label %__except
+;
+; CHECK: landingpad
+; CHECK: indirectbr i8* %{{[^,]*}}, [label %[[except_split2:.*]]]
+; CHECK: [[except_split2]]:
+; CHECK: call i32 @llvm.eh.exceptioncode()
+; CHECK: br label %__except
+;
+; CHECK: __except:
+; CHECK: phi
+; CHECK: call i32 (i8*, ...) @printf
diff --git a/test/CodeGen/WinEH/seh-exception-code2.ll b/test/CodeGen/WinEH/seh-exception-code2.ll
new file mode 100644
index 00000000000..0356956502c
--- /dev/null
+++ b/test/CodeGen/WinEH/seh-exception-code2.ll
@@ -0,0 +1,91 @@
+; RUN: opt -winehprepare -S < %s | FileCheck %s
+
+; WinEHPrepare was crashing during phi demotion.
+
+target datalayout = "e-m:w-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-pc-windows-msvc18.0.0"
+
+declare i32 @__C_specific_handler(...)
+
+@str = linkonce_odr unnamed_addr constant [16 x i8] c"caught it! %lx\0A\00", align 1
+
+declare void @maycrash()
+declare void @finally(i1 %abnormal)
+declare i32 @printf(i8* nocapture readonly, ...)
+declare i32 @llvm.eh.typeid.for(i8*)
+
+; Function Attrs: nounwind uwtable
+define void @doit() personality i8* bitcast (i32 (...)* @__C_specific_handler to i8*) {
+entry:
+  invoke void @maycrash()
+          to label %invoke.cont unwind label %lpad.1
+
+invoke.cont:                                      ; preds = %entry
+  invoke void @maycrash()
+          to label %__try.cont unwind label %lpad
+
+lpad:                                             ; preds = %entry
+  %lp0 = landingpad { i8*, i32 }
+          cleanup
+          catch i8* bitcast (i32 (i8*, i8*)* @"\01?filt$0@0@doit@@" to i8*)
+  %ehptr.0 = extractvalue { i8*, i32 } %lp0, 0
+  %ehsel.0 = extractvalue { i8*, i32 } %lp0, 1
+  call void @finally(i1 true)
+  br label %ehdispatch
+
+lpad.1:                                           ; preds = %invoke.cont, %lpad
+  %lp1 = landingpad { i8*, i32 }
+          catch i8* bitcast (i32 (i8*, i8*)* @"\01?filt$0@0@doit@@" to i8*)
+  %ehptr.1 = extractvalue { i8*, i32 } %lp1, 0
+  %ehsel.1 = extractvalue { i8*, i32 } %lp1, 1
+  br label %ehdispatch
+
+ehdispatch:
+  %ehptr.2 = phi i8* [ %ehptr.0, %lpad ], [ %ehptr.1, %lpad.1 ]
+  %ehsel.2 = phi i32 [ %ehsel.0, %lpad ], [ %ehsel.1, %lpad.1 ]
+  %mysel = call i32 @llvm.eh.typeid.for(i8* bitcast (i32 (i8*, i8*)* @"\01?filt$0@0@doit@@" to i8*))
+  %matches = icmp eq i32 %ehsel.2, %mysel
+  br i1 %matches, label %__except, label %eh.resume
+
+__except:                                         ; preds = %lpad, %lpad.1
+  %t4 = ptrtoint i8* %ehptr.2 to i64
+  %t5 = trunc i64 %t4 to i32
+  %call = tail call i32 (i8*, ...) @printf(i8* getelementptr inbounds ([16 x i8], [16 x i8]* @str, i64 0, i64 0), i32 %t5)
+  br label %__try.cont
+
+__try.cont:                                       ; preds = %invoke.cont, %__except
+  call void @finally(i1 false)
+  ret void
+
+eh.resume:
+  %ehvals0 = insertvalue { i8*, i32 } undef, i8* %ehptr.2, 0
+  %ehvals = insertvalue { i8*, i32 } %ehvals0, i32 %ehsel.2, 1
+  resume { i8*, i32 } %ehvals
+}
+
+define internal i32 @"\01?filt$0@0@doit@@"(i8* %exception_pointers, i8* %frame_pointer) #1 {
+entry:
+  %0 = bitcast i8* %exception_pointers to { i32*, i8* }*
+  %1 = getelementptr inbounds { i32*, i8* }, { i32*, i8* }* %0, i32 0, i32 0
+  %2 = load i32*, i32** %1
+  %3 = load i32, i32* %2
+  %cmp = icmp eq i32 %3, -1073741819
+  %4 = zext i1 %cmp to i32
+  ret i32 %4
+}
+
+; CHECK-LABEL: define void @doit()
+; CHECK: %lp0 = landingpad { i8*, i32 }
+; CHECK-NEXT: cleanup
+; CHECK-NEXT: catch i8*
+; CHECK-NEXT: call i8* (...) @llvm.eh.actions({{.*}})
+; CHECK-NEXT: indirectbr i8* %{{[^,]*}}, [label %__except]
+;
+; CHECK: %lp1 = landingpad { i8*, i32 }
+; CHECK-NEXT: catch i8*
+; CHECK-NEXT: call i8* (...) @llvm.eh.actions({{.*}})
+; CHECK-NEXT: indirectbr i8* %{{[^,]*}}, [label %__except]
+;
+; CHECK: __except:
+; CHECK: call i32 @llvm.eh.exceptioncode()
+; CHECK: call i32 (i8*, ...) @printf
diff --git a/test/CodeGen/WinEH/seh-inlined-finally.ll b/test/CodeGen/WinEH/seh-inlined-finally.ll
index 2e6171a8ced..157adf0c818 100644
--- a/test/CodeGen/WinEH/seh-inlined-finally.ll
+++ b/test/CodeGen/WinEH/seh-inlined-finally.ll
@@ -1,4 +1,4 @@
-; RUN: opt -S -winehprepare -sehprepare < %s | FileCheck %s
+; RUN: opt -S -winehprepare < %s | FileCheck %s
 
 ; Check that things work when the mid-level optimizer inlines the finally
 ; block.
@@ -6,11 +6,20 @@
 target datalayout = "e-m:w-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-pc-windows-msvc"
 
+%struct._RTL_CRITICAL_SECTION = type { %struct._RTL_CRITICAL_SECTION_DEBUG*, i32, i32, i8*, i8*, i64 }
+%struct._RTL_CRITICAL_SECTION_DEBUG = type { i16, i16, %struct._RTL_CRITICAL_SECTION*, %struct._LIST_ENTRY, i32, i32, i32, i16, i16 }
+%struct._LIST_ENTRY = type { %struct._LIST_ENTRY*, %struct._LIST_ENTRY* }
+
 declare i32 @puts(i8*)
 declare void @may_crash()
 declare i32 @__C_specific_handler(...)
+declare i8* @llvm.localrecover(i8*, i8*, i32) #1
+declare i8* @llvm.localaddress()
+declare void @llvm.localescape(...)
+declare dllimport void @EnterCriticalSection(%struct._RTL_CRITICAL_SECTION*)
+declare dllimport void @LeaveCriticalSection(%struct._RTL_CRITICAL_SECTION*)
 
-define void @use_finally() {
+define void @use_finally() personality i8* bitcast (i32 (...)* @__C_specific_handler to i8*) {
 entry:
   invoke void @may_crash()
           to label %invoke.cont unwind label %lpad
@@ -20,7 +29,7 @@ invoke.cont:                                      ; preds = %entry
   ret void
 
 lpad:                                             ; preds = %entry
-  %0 = landingpad { i8*, i32 } personality i8* bitcast (i32 (...)* @__C_specific_handler to i8*)
+  %0 = landingpad { i8*, i32 }
           cleanup
   %call.i2 = tail call i32 @puts(i8* null)
   resume { i8*, i32 } %0
@@ -33,3 +42,42 @@ lpad:                                             ; preds = %entry
 ; CHECK-NEXT: cleanup
 ; CHECK-NEXT: call i8* (...) @llvm.eh.actions(i32 0, void (i8*, i8*)* @use_finally.cleanup)
 ; CHECK-NEXT: indirectbr i8* %recover, []
+
+; Function Attrs: nounwind uwtable
+define i32 @call_may_crash_locked() personality i8* bitcast (i32 (...)* @__C_specific_handler to i8*) {
+entry:
+  %p = alloca %struct._RTL_CRITICAL_SECTION, align 8
+  call void (...) @llvm.localescape(%struct._RTL_CRITICAL_SECTION* %p)
+  call void @EnterCriticalSection(%struct._RTL_CRITICAL_SECTION* %p)
+  invoke void @may_crash()
+          to label %invoke.cont unwind label %lpad
+
+invoke.cont:                                      ; preds = %entry
+  %tmp2 = call i8* @llvm.localaddress()
+  %tmp3 = call i8* @llvm.localrecover(i8* bitcast (i32 ()* @call_may_crash_locked to i8*), i8* %tmp2, i32 0) #2
+  %tmp6 = bitcast i8* %tmp3 to %struct._RTL_CRITICAL_SECTION*
+  call void @LeaveCriticalSection(%struct._RTL_CRITICAL_SECTION* %tmp6)
+  ret i32 42
+
+lpad:                                             ; preds = %entry
+  %tmp7 = landingpad { i8*, i32 }
+            cleanup
+  %tmp8 = call i8* @llvm.localaddress()
+  %tmp9 = call i8* @llvm.localrecover(i8* bitcast (i32 ()* @call_may_crash_locked to i8*), i8* %tmp8, i32 0)
+  %tmp12 = bitcast i8* %tmp9 to %struct._RTL_CRITICAL_SECTION*
+  call void @LeaveCriticalSection(%struct._RTL_CRITICAL_SECTION* %tmp12)
+  resume { i8*, i32 } %tmp7
+}
+
+; CHECK-LABEL: define i32 @call_may_crash_locked()
+; CHECK: invoke void @may_crash()
+;
+; CHECK: landingpad
+; CHECK-NEXT: cleanup
+; CHECK-NEXT: call i8* (...) @llvm.eh.actions(i32 0, void (i8*, i8*)* @call_may_crash_locked.cleanup)
+; CHECK-NEXT: indirectbr i8* %recover, []
+
+; CHECK-LABEL: define internal void @call_may_crash_locked.cleanup(i8*, i8*)
+; CHECK: %tmp9 = call i8* @llvm.localrecover(i8* bitcast (i32 ()* @call_may_crash_locked to i8*), i8* %1, i32 0)
+; CHECK: %tmp12 = bitcast i8* %tmp9 to %struct._RTL_CRITICAL_SECTION*
+; CHECK: call void @LeaveCriticalSection(%struct._RTL_CRITICAL_SECTION* %tmp12)
diff --git a/test/CodeGen/WinEH/seh-outlined-finally-win32.ll b/test/CodeGen/WinEH/seh-outlined-finally-win32.ll
new file mode 100644
index 00000000000..3649433c4b6
--- /dev/null
+++ b/test/CodeGen/WinEH/seh-outlined-finally-win32.ll
@@ -0,0 +1,172 @@
+; RUN: opt -S -winehprepare < %s | FileCheck %s
+
+; Test case based on this code:
+;
+; extern "C" int _abnormal_termination();
+; #pragma intrinsic(_abnormal_termination)
+; extern "C" int printf(const char *, ...);
+; extern "C" void may_crash() {
+;   *(volatile int *)0 = 42;
+; }
+; int main() {
+;   int myres = 0;
+;   __try {
+;     __try {
+;       may_crash();
+;     } __finally {
+;       printf("inner finally %d\n", _abnormal_termination());
+;       may_crash();
+;     }
+;   } __finally {
+;     printf("outer finally %d\n", _abnormal_termination());
+;   }
+; }
+;
+; Note that if the inner finally crashes, the outer finally still runs. There
+; is nothing like a std::terminate call in this situation.
+
+target datalayout = "e-m:x-p:32:32-i64:64-f80:32-n8:16:32-a:0:32-S32"
+target triple = "i686-pc-windows-msvc"
+
+$"\01??_C@_0BC@LHHILCPN@outer?5finally?5?$CFd?6?$AA@" = comdat any
+
+$"\01??_C@_0BC@JELAHKN@inner?5finally?5?$CFd?6?$AA@" = comdat any
+
+@"\01??_C@_0BC@LHHILCPN@outer?5finally?5?$CFd?6?$AA@" = linkonce_odr unnamed_addr constant [18 x i8] c"outer finally %d\0A\00", comdat, align 1
+@"\01??_C@_0BC@JELAHKN@inner?5finally?5?$CFd?6?$AA@" = linkonce_odr unnamed_addr constant [18 x i8] c"inner finally %d\0A\00", comdat, align 1
+
+; Function Attrs: nounwind
+define void @may_crash() #0 {
+entry:
+  store volatile i32 42, i32* null, align 4
+  ret void
+}
+
+; Function Attrs: nounwind
+define i32 @main() #0 personality i8* bitcast (i32 (...)* @_except_handler3 to i8*) {
+entry:
+  %myres = alloca i32, align 4
+  %exn.slot = alloca i8*
+  %ehselector.slot = alloca i32
+  store i32 0, i32* %myres, align 4
+  invoke void @may_crash() #4
+          to label %invoke.cont unwind label %lpad
+
+invoke.cont:                                      ; preds = %entry
+  %0 = call i8* @llvm.frameaddress(i32 0)
+  invoke void @"\01?fin$1@0@main@@"(i8 zeroext 0, i8* %0) #4
+          to label %invoke.cont.2 unwind label %lpad.1
+
+invoke.cont.2:                                    ; preds = %invoke.cont
+  %1 = call i8* @llvm.frameaddress(i32 0)
+  call void @"\01?fin$0@0@main@@"(i8 zeroext 0, i8* %1)
+  ret i32 0
+
+lpad:                                             ; preds = %entry
+  %2 = landingpad { i8*, i32 }
+          cleanup
+  %3 = extractvalue { i8*, i32 } %2, 0
+  store i8* %3, i8** %exn.slot
+  %4 = extractvalue { i8*, i32 } %2, 1
+  store i32 %4, i32* %ehselector.slot
+  %5 = call i8* @llvm.frameaddress(i32 0)
+  invoke void @"\01?fin$1@0@main@@"(i8 zeroext 1, i8* %5) #4
+          to label %invoke.cont.3 unwind label %lpad.1
+
+lpad.1:                                           ; preds = %lpad, %invoke.cont
+  %6 = landingpad { i8*, i32 }
+          cleanup
+  %7 = extractvalue { i8*, i32 } %6, 0
+  store i8* %7, i8** %exn.slot
+  %8 = extractvalue { i8*, i32 } %6, 1
+  store i32 %8, i32* %ehselector.slot
+  br label %ehcleanup
+
+invoke.cont.3:                                    ; preds = %lpad
+  br label %ehcleanup
+
+ehcleanup:                                        ; preds = %invoke.cont.3, %lpad.1
+  %9 = call i8* @llvm.frameaddress(i32 0)
+  call void @"\01?fin$0@0@main@@"(i8 zeroext 1, i8* %9)
+  br label %eh.resume
+
+eh.resume:                                        ; preds = %ehcleanup
+  %exn = load i8*, i8** %exn.slot
+  %sel = load i32, i32* %ehselector.slot
+  %lpad.val = insertvalue { i8*, i32 } undef, i8* %exn, 0
+  %lpad.val.4 = insertvalue { i8*, i32 } %lpad.val, i32 %sel, 1
+  resume { i8*, i32 } %lpad.val.4
+}
+
+; CHECK-LABEL: define i32 @main()
+; CHECK: invoke void @may_crash()
+;
+; CHECK: landingpad { i8*, i32 }
+; CHECK-NEXT: cleanup
+; CHECK-NEXT: call i8* (...) @llvm.eh.actions(i32 0, void ()* @main.cleanup)
+; CHECK-NEXT: indirectbr
+;
+; CHECK: landingpad { i8*, i32 }
+; CHECK-NEXT: cleanup
+; CHECK-NEXT: call i8* (...) @llvm.eh.actions(i32 0, void ()* @main.cleanup.1)
+; CHECK-NEXT: indirectbr
+
+; CHECK-LABEL: define internal void @main.cleanup()
+; CHECK: call i8* @llvm.frameaddress(i32 1)
+; CHECK: call i8* @llvm.x86.seh.recoverfp(i8* bitcast (i32 ()* @main to i8*), i8* %{{.*}})
+; CHECK: call void @"\01?fin$1@0@main@@"(i8 zeroext 1, i8* %{{.*}})
+; CHECK: call void @"\01?fin$0@0@main@@"(i8 zeroext 1, i8* %{{.*}})
+
+; CHECK-LABEL: define internal void @main.cleanup.1()
+; CHECK: call i8* @llvm.frameaddress(i32 1)
+; CHECK: call i8* @llvm.x86.seh.recoverfp(i8* bitcast (i32 ()* @main to i8*), i8* %{{.*}})
+; CHECK: call void @"\01?fin$0@0@main@@"(i8 zeroext 1, i8* %{{.*}})
+
+; Function Attrs: noinline nounwind
+define internal void @"\01?fin$0@0@main@@"(i8 zeroext %abnormal_termination, i8* %frame_pointer) #1 {
+entry:
+  %frame_pointer.addr = alloca i8*, align 4
+  %abnormal_termination.addr = alloca i8, align 1
+  %0 = call i8* @llvm.frameaddress(i32 1)
+  %1 = call i8* @llvm.x86.seh.recoverfp(i8* bitcast (i32 ()* @main to i8*), i8* %0)
+  store i8* %frame_pointer, i8** %frame_pointer.addr, align 4
+  store i8 %abnormal_termination, i8* %abnormal_termination.addr, align 1
+  %2 = zext i8 %abnormal_termination to i32
+  %call = call i32 (i8*, ...) @printf(i8* getelementptr inbounds ([18 x i8], [18 x i8]* @"\01??_C@_0BC@LHHILCPN@outer?5finally?5?$CFd?6?$AA@", i32 0, i32 0), i32 %2)
+  ret void
+}
+
+; Function Attrs: nounwind readnone
+declare i8* @llvm.frameaddress(i32) #2
+
+; Function Attrs: nounwind readnone
+declare i8* @llvm.x86.seh.recoverfp(i8*, i8*) #2
+
+declare i32 @printf(i8*, ...) #3
+
+; Function Attrs: noinline nounwind
+define internal void @"\01?fin$1@0@main@@"(i8 zeroext %abnormal_termination, i8* %frame_pointer) #1 {
+entry:
+  %frame_pointer.addr = alloca i8*, align 4
+  %abnormal_termination.addr = alloca i8, align 1
+  %0 = call i8* @llvm.frameaddress(i32 1)
+  %1 = call i8* @llvm.x86.seh.recoverfp(i8* bitcast (i32 ()* @main to i8*), i8* %0)
+  store i8* %frame_pointer, i8** %frame_pointer.addr, align 4
+  store i8 %abnormal_termination, i8* %abnormal_termination.addr, align 1
+  %2 = zext i8 %abnormal_termination to i32
+  %call = call i32 (i8*, ...) @printf(i8* getelementptr inbounds ([18 x i8], [18 x i8]* @"\01??_C@_0BC@JELAHKN@inner?5finally?5?$CFd?6?$AA@", i32 0, i32 0), i32 %2)
+  call void @may_crash()
+  ret void
+}
+
+declare i32 @_except_handler3(...)
+
+attributes #0 = { nounwind "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "no-realign-stack" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #1 = { noinline nounwind "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "no-realign-stack" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #2 = { nounwind readnone }
+attributes #3 = { "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "no-realign-stack" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #4 = { noinline }
+
+!llvm.ident = !{!0}
+
+!0 = !{!"clang version 3.7.0 "}
diff --git a/test/CodeGen/WinEH/seh-outlined-finally.ll b/test/CodeGen/WinEH/seh-outlined-finally.ll
index bc9d3221ad4..529f85b9602 100644
--- a/test/CodeGen/WinEH/seh-outlined-finally.ll
+++ b/test/CodeGen/WinEH/seh-outlined-finally.ll
@@ -1,4 +1,4 @@
-; RUN: opt -S -winehprepare -sehprepare -mtriple=x86_64-windows-msvc < %s | FileCheck %s
+; RUN: opt -S -winehprepare -mtriple=x86_64-windows-msvc < %s | FileCheck %s
 
 ; Test case based on this code:
 ;
@@ -39,7 +39,7 @@ entry:
 }
 
 ; Function Attrs: uwtable
-define i32 @main() #1 {
+define i32 @main() #1 personality i8* bitcast (i32 (...)* @__C_specific_handler to i8*) {
 entry:
   %myres = alloca i32, align 4
   %exn.slot = alloca i8*
@@ -49,28 +49,28 @@ entry:
           to label %invoke.cont unwind label %lpad
 
 invoke.cont:                                      ; preds = %entry
-  %0 = call i8* @llvm.frameaddress(i32 0)
+  %0 = call i8* @llvm.localaddress()
   invoke void @"\01?fin$1@0@main@@"(i1 zeroext false, i8* %0) #4
           to label %invoke.cont2 unwind label %lpad1
 
 invoke.cont2:                                     ; preds = %invoke.cont
-  %1 = call i8* @llvm.frameaddress(i32 0)
+  %1 = call i8* @llvm.localaddress()
   call void @"\01?fin$0@0@main@@"(i1 zeroext false, i8* %1)
   ret i32 0
 
 lpad:                                             ; preds = %entry
-  %2 = landingpad { i8*, i32 } personality i8* bitcast (i32 (...)* @__C_specific_handler to i8*)
+  %2 = landingpad { i8*, i32 }
           cleanup
   %3 = extractvalue { i8*, i32 } %2, 0
   store i8* %3, i8** %exn.slot
   %4 = extractvalue { i8*, i32 } %2, 1
   store i32 %4, i32* %ehselector.slot
-  %5 = call i8* @llvm.frameaddress(i32 0)
+  %5 = call i8* @llvm.localaddress()
   invoke void @"\01?fin$1@0@main@@"(i1 zeroext true, i8* %5) #4
           to label %invoke.cont3 unwind label %lpad1
 
 lpad1:                                            ; preds = %lpad, %invoke.cont
-  %6 = landingpad { i8*, i32 } personality i8* bitcast (i32 (...)* @__C_specific_handler to i8*)
+  %6 = landingpad { i8*, i32 }
           cleanup
   %7 = extractvalue { i8*, i32 } %6, 0
   store i8* %7, i8** %exn.slot
@@ -82,7 +82,7 @@ invoke.cont3:                                     ; preds = %lpad
   br label %ehcleanup
 
 ehcleanup:                                        ; preds = %invoke.cont3, %lpad1
-  %9 = call i8* @llvm.frameaddress(i32 0)
+  %9 = call i8* @llvm.localaddress()
   call void @"\01?fin$0@0@main@@"(i1 zeroext true, i8* %9)
   br label %eh.resume
 
@@ -146,7 +146,7 @@ entry:
 declare i32 @__C_specific_handler(...)
 
 ; Function Attrs: nounwind readnone
-declare i8* @llvm.frameaddress(i32) #3
+declare i8* @llvm.localaddress() #3
 
 attributes #0 = { nounwind uwtable "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "unsafe-fp-math"="false" "use-soft-float"="false" }
 attributes #1 = { uwtable "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "unsafe-fp-math"="false" "use-soft-float"="false" }
diff --git a/test/CodeGen/WinEH/seh-prepared-basic.ll b/test/CodeGen/WinEH/seh-prepared-basic.ll
new file mode 100644
index 00000000000..b6a30309f1c
--- /dev/null
+++ b/test/CodeGen/WinEH/seh-prepared-basic.ll
@@ -0,0 +1,83 @@
+; RUN: llc < %s | FileCheck %s
+
+; Test case based on this code:
+; extern "C" unsigned long _exception_code();
+; extern "C" int filt(unsigned long);
+; extern "C" void g();
+; extern "C" void do_except() {
+;   __try {
+;     g();
+;   } __except(filt(_exception_code())) {
+;   }
+; }
+
+target datalayout = "e-m:w-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-pc-windows-msvc"
+
+; Function Attrs: uwtable
+define void @do_except() #0 personality i8* bitcast (i32 (...)* @__C_specific_handler to i8*) {
+entry:
+  call void (...) @llvm.localescape()
+  invoke void @g() #5
+          to label %__try.cont unwind label %lpad1
+
+lpad1:                                            ; preds = %entry
+  %ehvals = landingpad { i8*, i32 }
+          catch i8* bitcast (i32 (i8*, i8*)* @"\01?filt$0@0@do_except@@" to i8*)
+  %recover = call i8* (...) @llvm.eh.actions(i32 1, i8* bitcast (i32 (i8*, i8*)* @"\01?filt$0@0@do_except@@" to i8*), i32 -1, i8* blockaddress(@do_except, %__try.cont))
+  indirectbr i8* %recover, [label %__try.cont]
+
+__try.cont:                                       ; preds = %lpad1, %entry
+  ret void
+}
+
+; CHECK-LABEL: do_except:
+; CHECK: .seh_handler __C_specific_handler
+; CHECK-NOT: jmpq *
+; CHECK: .seh_handlerdata
+; CHECK-NEXT: .long 1
+; CHECK-NEXT: .long .Ltmp{{.*}}
+; CHECK-NEXT: .long .Ltmp{{.*}}
+; CHECK-NEXT: .long "?filt$0@0@do_except@@"@IMGREL
+; CHECK-NEXT: .long .Ltmp{{.*}}@IMGREL
+
+; Function Attrs: noinline nounwind
+define internal i32 @"\01?filt$0@0@do_except@@"(i8* nocapture readonly %exception_pointers, i8* nocapture readnone %frame_pointer) #1 {
+entry:
+  %0 = bitcast i8* %exception_pointers to i32**
+  %1 = load i32*, i32** %0, align 8
+  %2 = load i32, i32* %1, align 4
+  %call = tail call i32 @filt(i32 %2) #4
+  ret i32 %call
+}
+
+declare i32 @filt(i32) #2
+
+declare void @g() #2
+
+declare i32 @__C_specific_handler(...)
+
+; Function Attrs: nounwind readnone
+declare i32 @llvm.eh.typeid.for(i8*) #3
+
+; Function Attrs: nounwind
+declare i8* @llvm.eh.actions(...) #4
+
+; Function Attrs: nounwind
+declare void @llvm.localescape(...) #4
+
+; Function Attrs: nounwind readnone
+declare i8* @llvm.localrecover(i8*, i8*, i32) #3
+
+attributes #0 = { uwtable "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "unsafe-fp-math"="false" "use-soft-float"="false" "wineh-parent"="do_except" }
+attributes #1 = { noinline nounwind "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #2 = { "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #3 = { nounwind readnone }
+attributes #4 = { nounwind }
+attributes #5 = { noinline }
+
+!llvm.module.flags = !{!0}
+!llvm.ident = !{!1}
+
+!0 = !{i32 1, !"PIC Level", i32 2}
+!1 = !{!"clang version 3.7.0 "}
diff --git a/test/CodeGen/WinEH/seh-resume-phi.ll b/test/CodeGen/WinEH/seh-resume-phi.ll
new file mode 100644
index 00000000000..d2bd64167d2
--- /dev/null
+++ b/test/CodeGen/WinEH/seh-resume-phi.ll
@@ -0,0 +1,66 @@
+; RUN: opt -S -winehprepare < %s | FileCheck %s
+
+target datalayout = "e-m:w-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-pc-windows-msvc"
+
+declare void @might_crash(i8* %ehptr)
+declare i32 @filt()
+declare void @cleanup()
+declare i32 @__C_specific_handler(...)
+declare i32 @llvm.eh.typeid.for(i8*)
+
+define void @resume_phi() personality i32 (...)* @__C_specific_handler {
+entry:
+  invoke void @might_crash(i8* null)
+          to label %return unwind label %lpad1
+
+lpad1:
+  %ehvals1 = landingpad { i8*, i32 }
+          catch i32 ()* @filt
+  %ehptr1 = extractvalue { i8*, i32 } %ehvals1, 0
+  %ehsel1 = extractvalue { i8*, i32 } %ehvals1, 1
+  %filt_sel = tail call i32 @llvm.eh.typeid.for(i8* bitcast (i32 ()* @filt to i8*))
+  %matches = icmp eq i32 %ehsel1, %filt_sel
+  br i1 %matches, label %__except, label %eh.resume
+
+__except:
+  invoke void @might_crash(i8* %ehptr1)
+          to label %return unwind label %lpad2
+
+lpad2:
+  %ehvals2 = landingpad { i8*, i32 }
+          cleanup
+  %ehptr2 = extractvalue { i8*, i32 } %ehvals2, 0
+  %ehsel2 = extractvalue { i8*, i32 } %ehvals2, 1
+  call void @cleanup()
+  br label %eh.resume
+
+return:
+  ret void
+
+eh.resume:
+  %ehptr.phi = phi i8* [ %ehptr1, %lpad1 ], [ %ehptr2, %lpad2 ]
+  %ehsel.phi = phi i32 [ %ehsel1, %lpad1 ], [ %ehsel2, %lpad2 ]
+  %ehval.phi1 = insertvalue { i8*, i32 } undef, i8* %ehptr.phi, 0
+  %ehval.phi2 = insertvalue { i8*, i32 } %ehval.phi1, i32 %ehsel.phi, 1
+  resume { i8*, i32 } %ehval.phi2
+}
+
+; CHECK-LABEL: define void @resume_phi()
+; CHECK: invoke void @might_crash(i8* null)
+; CHECK: landingpad { i8*, i32 }
+; CHECK-NEXT: catch i32 ()* @filt
+; CHECK-NEXT: call i8* (...) @llvm.eh.actions(
+; CHECK-SAME: i32 1, i8* bitcast (i32 ()* @filt to i8*), i32 -1, i8* blockaddress(@resume_phi, %__except))
+; CHECK-NEXT: indirectbr {{.*}} [label %__except]
+;
+; CHECK: __except:
+; CHECK: call i32 @llvm.eh.exceptioncode()
+; CHECK: invoke void @might_crash(i8* %{{.*}})
+; CHECK: landingpad { i8*, i32 }
+; CHECK-NEXT: cleanup
+; CHECK-NEXT: call i8* (...) @llvm.eh.actions(i32 0, void (i8*, i8*)* @resume_phi.cleanup)
+; CHECK-NEXT: indirectbr {{.*}} []
+
+; CHECK-LABEL: define internal void @resume_phi.cleanup(i8*, i8*)
+; CHECK: call void @cleanup()
diff --git a/test/CodeGen/WinEH/seh-simple.ll b/test/CodeGen/WinEH/seh-simple.ll
index 344a0c87215..060186484ae 100644
--- a/test/CodeGen/WinEH/seh-simple.ll
+++ b/test/CodeGen/WinEH/seh-simple.ll
@@ -1,7 +1,10 @@
-; RUN: opt -S -winehprepare -sehprepare -mtriple=x86_64-windows-msvc < %s | FileCheck %s
+; RUN: opt -S -winehprepare -mtriple=x86_64-windows-msvc < %s \
+; RUN: 		| FileCheck %s --check-prefix=CHECK --check-prefix=X64
 
-target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
-target triple = "x86_64-pc-windows-msvc"
+; This test should also pass in 32-bit using _except_handler3.
+; RUN: sed -e 's/__C_specific_handler/_except_handler3/' %s \
+; RUN: 		| opt -S -winehprepare -mtriple=i686-windows-msvc \
+; RUN: 		| FileCheck %s --check-prefix=CHECK --check-prefix=X86
 
 declare void @cleanup()
 declare i32 @filt()
@@ -9,7 +12,7 @@ declare void @might_crash()
 declare i32 @__C_specific_handler(...)
 declare i32 @llvm.eh.typeid.for(i8*)
 
-define i32 @simple_except_store() {
+define i32 @simple_except_store() personality i32 (...)* @__C_specific_handler {
 entry:
   %retval = alloca i32
   store i32 0, i32* %retval
@@ -17,7 +20,7 @@ entry:
           to label %return unwind label %lpad
 
 lpad:
-  %ehvals = landingpad { i8*, i32 } personality i32 (...)* @__C_specific_handler
+  %ehvals = landingpad { i8*, i32 }
           catch i32 ()* @filt
   %sel = extractvalue { i8*, i32 } %ehvals, 1
   %filt_sel = tail call i32 @llvm.eh.typeid.for(i8* bitcast (i32 ()* @filt to i8*))
@@ -42,7 +45,7 @@ eh.resume:
 ; CHECK-NEXT: call i8* (...) @llvm.eh.actions(i32 1, i8* bitcast (i32 ()* @filt to i8*), i32 -1, i8* blockaddress(@simple_except_store, %__except))
 ; CHECK-NEXT: indirectbr {{.*}} [label %__except]
 
-define i32 @catch_all() {
+define i32 @catch_all() personality i32 (...)* @__C_specific_handler {
 entry:
   %retval = alloca i32
   store i32 0, i32* %retval
@@ -50,7 +53,7 @@ entry:
           to label %return unwind label %lpad
 
 lpad:
-  %ehvals = landingpad { i8*, i32 } personality i32 (...)* @__C_specific_handler
+  %ehvals = landingpad { i8*, i32 }
           catch i8* null
   store i32 1, i32* %retval
   br label %return
@@ -63,20 +66,20 @@ return:
 ; CHECK-LABEL: define i32 @catch_all()
 ; CHECK: landingpad { i8*, i32 }
 ; CHECK-NEXT: catch i8* null
-; CHECK-NEXT: call i8* (...) @llvm.eh.actions(i32 1, i8* null, i32 -1, i8* blockaddress(@catch_all, %catch.all))
-; CHECK-NEXT: indirectbr {{.*}} [label %catch.all]
+; CHECK-NEXT: call i8* (...) @llvm.eh.actions(i32 1, i8* null, i32 -1, i8* blockaddress(@catch_all, %lpad.split))
+; CHECK-NEXT: indirectbr {{.*}} [label %lpad.split]
 ;
-; CHECK: catch.all:
+; CHECK: lpad.split:
 ; CHECK: store i32 1, i32* %retval
 
 
-define i32 @except_phi() {
+define i32 @except_phi() personality i32 (...)* @__C_specific_handler {
 entry:
   invoke void @might_crash()
           to label %return unwind label %lpad
 
 lpad:
-  %ehvals = landingpad { i8*, i32 } personality i32 (...)* @__C_specific_handler
+  %ehvals = landingpad { i8*, i32 }
           catch i32 ()* @filt
   %sel = extractvalue { i8*, i32 } %ehvals, 1
   %filt_sel = tail call i32 @llvm.eh.typeid.for(i8* bitcast (i32 ()* @filt to i8*))
@@ -94,20 +97,101 @@ eh.resume:
 ; CHECK-LABEL: define i32 @except_phi()
 ; CHECK: landingpad { i8*, i32 }
 ; CHECK-NEXT: catch i32 ()* @filt
-; CHECK-NEXT: call i8* (...) @llvm.eh.actions(i32 1, i8* bitcast (i32 ()* @filt to i8*), i32 -1, i8* blockaddress(@except_phi, %return))
-; CHECK-NEXT: indirectbr {{.*}} [label %return]
+; CHECK-NEXT: call i8* (...) @llvm.eh.actions(i32 1, i8* bitcast (i32 ()* @filt to i8*), i32 -1, i8* blockaddress(@except_phi, %lpad.return_crit_edge))
+; CHECK-NEXT: indirectbr {{.*}} [label %lpad.return_crit_edge]
+;
+; CHECK: lpad.return_crit_edge:
+; CHECK: br label %return
+;
+; CHECK: return:
+; CHECK-NEXT: %r = phi i32 [ 0, %entry ], [ 1, %lpad.return_crit_edge ]
+; CHECK-NEXT: ret i32 %r
+
+define i32 @except_join() personality i32 (...)* @__C_specific_handler {
+entry:
+  invoke void @might_crash()
+          to label %return unwind label %lpad
+
+lpad:
+  %ehvals = landingpad { i8*, i32 }
+          catch i32 ()* @filt
+  %sel = extractvalue { i8*, i32 } %ehvals, 1
+  %filt_sel = tail call i32 @llvm.eh.typeid.for(i8* bitcast (i32 ()* @filt to i8*))
+  %matches = icmp eq i32 %sel, %filt_sel
+  br i1 %matches, label %return, label %eh.resume
+
+return:
+  ret i32 0
+
+eh.resume:
+  resume { i8*, i32 } %ehvals
+}
+
+; CHECK-LABEL: define i32 @except_join()
+; CHECK: landingpad { i8*, i32 }
+; CHECK-NEXT: catch i32 ()* @filt
+; CHECK-NEXT: call i8* (...) @llvm.eh.actions(i32 1, i8* bitcast (i32 ()* @filt to i8*), i32 -1, i8* blockaddress(@except_join, %lpad.return_crit_edge))
+; CHECK-NEXT: indirectbr {{.*}} [label %lpad.return_crit_edge]
+;
+; CHECK: lpad.return_crit_edge:
+; CHECK: br label %return
+;
+; CHECK: return:
+; CHECK-NEXT: ret i32 0
+
+define i32 @lpad_phi() personality i32 (...)* @__C_specific_handler {
+entry:
+  invoke void @might_crash()
+          to label %cont unwind label %lpad
+
+cont:
+  invoke void @might_crash()
+          to label %return unwind label %lpad
+
+lpad:
+  %ncalls.1 = phi i32 [ 0, %entry ], [ 1, %cont ]
+  %ehvals = landingpad { i8*, i32 }
+          catch i32 ()* @filt
+  %sel = extractvalue { i8*, i32 } %ehvals, 1
+  %filt_sel = tail call i32 @llvm.eh.typeid.for(i8* bitcast (i32 ()* @filt to i8*))
+  %matches = icmp eq i32 %sel, %filt_sel
+  br i1 %matches, label %return, label %eh.resume
+
+return:
+  %r = phi i32 [2, %cont], [%ncalls.1, %lpad]
+  ret i32 %r
+
+eh.resume:
+  resume { i8*, i32 } %ehvals
+}
+
+; CHECK-LABEL: define i32 @lpad_phi()
+; CHECK: alloca i32
+; CHECK: store i32 0, i32*
+; CHECK: invoke void @might_crash()
+; CHECK: store i32 1, i32*
+; CHECK: invoke void @might_crash()
+; CHECK: landingpad { i8*, i32 }
+; CHECK-NEXT: cleanup
+; CHECK-NEXT: catch i32 ()* @filt
+; CHECK-NEXT: call i8* (...) @llvm.eh.actions(i32 0, void ({{.*}})* @lpad_phi.cleanup, i32 1, i8* bitcast (i32 ()* @filt to i8*), i32 -1, i8* blockaddress(@lpad_phi, %lpad.return_crit_edge))
+; CHECK-NEXT: indirectbr {{.*}} [label %lpad.return_crit_edge]
+;
+; CHECK: lpad.return_crit_edge:
+; CHECK: load i32, i32*
+; CHECK: br label %return
 ;
 ; CHECK: return:
-; CHECK-NEXT: %r = phi i32 [ 0, %entry ], [ 1, %lpad1 ]
+; CHECK-NEXT: %r = phi i32 [ 2, %cont ], [ %{{.*}}, %lpad.return_crit_edge ]
 ; CHECK-NEXT: ret i32 %r
 
-define i32 @cleanup_and_except() {
+define i32 @cleanup_and_except() personality i32 (...)* @__C_specific_handler {
 entry:
   invoke void @might_crash()
           to label %return unwind label %lpad
 
 lpad:
-  %ehvals = landingpad { i8*, i32 } personality i32 (...)* @__C_specific_handler
+  %ehvals = landingpad { i8*, i32 }
           cleanup
           catch i32 ()* @filt
   call void @cleanup()
@@ -129,10 +213,21 @@ eh.resume:
 ; CHECK-NEXT: cleanup
 ; CHECK-NEXT: catch i32 ()* @filt
 ; CHECK-NEXT: call i8* (...) @llvm.eh.actions(
-; CHECK: i32 0, void (i8*, i8*)* @cleanup_and_except.cleanup,
-; CHECK: i32 1, i8* bitcast (i32 ()* @filt to i8*), i32 -1, i8* blockaddress(@cleanup_and_except, %return))
-; CHECK-NEXT: indirectbr {{.*}} [label %return]
+; CHECK: i32 0, void ({{.*}})* @cleanup_and_except.cleanup,
+; CHECK: i32 1, i8* bitcast (i32 ()* @filt to i8*), i32 -1, i8* blockaddress(@cleanup_and_except, %lpad.return_crit_edge))
+; CHECK-NEXT: indirectbr {{.*}} [label %lpad.return_crit_edge]
+;
+; CHECK: lpad.return_crit_edge:
+; CHECK: br label %return
 ;
 ; CHECK: return:
-; CHECK-NEXT: %r = phi i32 [ 0, %entry ], [ 1, %lpad1 ]
+; CHECK-NEXT: %r = phi i32 [ 0, %entry ], [ 1, %lpad.return_crit_edge ]
 ; CHECK-NEXT: ret i32 %r
+
+; FIXME: This cleanup is an artifact of bad demotion.
+; X64-LABEL: define internal void @lpad_phi.cleanup(i8*, i8*)
+; X86-LABEL: define internal void @lpad_phi.cleanup()
+; X86: call i8* @llvm.frameaddress(i32 1)
+; CHECK: call i8* @llvm.localrecover({{.*}})
+; CHECK: load i32
+; CHECK: store i32 %{{.*}}, i32*
diff --git a/test/CodeGen/X86/2007-05-05-Personality.ll b/test/CodeGen/X86/2007-05-05-Personality.ll
index b99c58c6e4a..f177a35273a 100644
--- a/test/CodeGen/X86/2007-05-05-Personality.ll
+++ b/test/CodeGen/X86/2007-05-05-Personality.ll
@@ -12,13 +12,13 @@
 
 @error = external global i8
 
-define void @_ada_x() {
+define void @_ada_x() personality i8* bitcast (i32 (...)* @__gnat_eh_personality to i8*) {
 entry:
   invoke void @raise()
           to label %eh_then unwind label %unwind
 
 unwind:                                           ; preds = %entry
-  %eh_ptr = landingpad { i8*, i32 } personality i8* bitcast (i32 (...)* @__gnat_eh_personality to i8*)
+  %eh_ptr = landingpad { i8*, i32 }
               catch i8* @error
   %eh_select = extractvalue { i8*, i32 } %eh_ptr, 1
   %eh_typeid = tail call i32 @llvm.eh.typeid.for(i8* @error)
diff --git a/test/CodeGen/X86/2008-04-17-CoalescerBug.ll b/test/CodeGen/X86/2008-04-17-CoalescerBug.ll
index 2e76a3356cf..3d3851cbd4c 100644
--- a/test/CodeGen/X86/2008-04-17-CoalescerBug.ll
+++ b/test/CodeGen/X86/2008-04-17-CoalescerBug.ll
@@ -13,7 +13,7 @@
 @.str33 = external constant [29 x i32]		; <[29 x i32]*> [#uses=1]
 @.str89 = external constant [5 x i32]		; <[5 x i32]*> [#uses=1]
 
-define void @_ZNK10wxDateTime6FormatEPKwRKNS_8TimeZoneE(%struct.wxString* noalias sret  %agg.result, %struct.wxDateTime* %this, i32* %format, %"struct.wxDateTime::TimeZone"* %tz, i1 %foo) {
+define void @_ZNK10wxDateTime6FormatEPKwRKNS_8TimeZoneE(%struct.wxString* noalias sret  %agg.result, %struct.wxDateTime* %this, i32* %format, %"struct.wxDateTime::TimeZone"* %tz, i1 %foo) personality i32 (...)* @__gxx_personality_v0 {
 entry:
 	br i1 %foo, label %bb116.i, label %bb115.critedge.i
 bb115.critedge.i:		; preds = %entry
@@ -117,13 +117,13 @@ bb448.i8694:		; preds = %bb440.i8663, %bb278.i8617
 invcont5814:		; preds = %bb448.i8694, %bb265.i8606
 	%tmp812.0.0 = phi i16 [ %tmp477478.i8670, %bb448.i8694 ], [ %tmp273274.i8595, %bb265.i8606 ]		; <i16> [#uses=1]
 	%tmp58165817 = zext i16 %tmp812.0.0 to i32		; <i32> [#uses=1]
-	invoke void (%struct.wxString*, i32*, ...)* @_ZN8wxString6FormatEPKwz( %struct.wxString* noalias sret  null, i32* null, i32 %tmp58165817 )
+	invoke void (%struct.wxString*, i32*, ...) @_ZN8wxString6FormatEPKwz( %struct.wxString* noalias sret  null, i32* null, i32 %tmp58165817 )
 			to label %invcont5831 unwind label %lpad
 invcont5831:		; preds = %invcont5814
 	%tmp5862 = invoke zeroext  i8 @_ZN12wxStringBase10ConcatSelfEmPKwm( %struct.wxStringBase* null, i32 0, i32* null, i32 0 ) 
 			to label %bb7834 unwind label %lpad8185		; <i8> [#uses=0]
 bb5968:		; preds = %bb3314
-	invoke void (%struct.wxString*, i32*, ...)* @_ZN8wxString6FormatEPKwz( %struct.wxString* noalias sret  null, i32* null, i32 0 )
+	invoke void (%struct.wxString*, i32*, ...) @_ZN8wxString6FormatEPKwz( %struct.wxString* noalias sret  null, i32* null, i32 0 )
 			to label %invcont5981 unwind label %lpad
 invcont5981:		; preds = %bb5968
 	ret void
@@ -151,11 +151,11 @@ bb7819:		; preds = %bb3314
 bb7834:		; preds = %bb7806, %invcont5831
 	br label %bb3261
 lpad:		; preds = %bb7806, %bb5968, %invcont5814, %bb440.i8663, %bb155.i8541, %bb5657, %bb3306
-        %exn = landingpad {i8*, i32} personality i32 (...)* @__gxx_personality_v0
+        %exn = landingpad {i8*, i32}
                  cleanup
 	ret void
 lpad8185:		; preds = %invcont5831
-        %exn8185 = landingpad {i8*, i32} personality i32 (...)* @__gxx_personality_v0
+        %exn8185 = landingpad {i8*, i32}
                  cleanup
 	ret void
 }
diff --git a/test/CodeGen/X86/2008-05-28-LocalRegAllocBug.ll b/test/CodeGen/X86/2008-05-28-LocalRegAllocBug.ll
index fc7ddf0bc67..7ddedacbabd 100644
--- a/test/CodeGen/X86/2008-05-28-LocalRegAllocBug.ll
+++ b/test/CodeGen/X86/2008-05-28-LocalRegAllocBug.ll
@@ -6,7 +6,7 @@ declare i8* @_Znwm(i32)
 
 declare i8* @__cxa_begin_catch(i8*) nounwind 
 
-define i32 @main(i32 %argc, i8** %argv) {
+define i32 @main(i32 %argc, i8** %argv) personality i32 (...)* @__gxx_personality_v0 {
 entry:
 	br i1 false, label %bb37, label %bb34
 
@@ -21,7 +21,7 @@ tmp12.i.i.i.i.i.noexc65:		; preds = %bb37
 	unreachable
 
 lpad243:		; preds = %bb37
-        %exn = landingpad {i8*, i32} personality i32 (...)* @__gxx_personality_v0
+        %exn = landingpad {i8*, i32}
                  cleanup
 	%eh_ptr244 = extractvalue { i8*, i32 } %exn, 0
 	store i32 (...)** getelementptr ([5 x i32 (...)*], [5 x i32 (...)*]* @_ZTVN10Evaluation10GridOutputILi3EEE, i32 0, i32 2), i32 (...)*** null, align 8
diff --git a/test/CodeGen/X86/2009-02-12-DebugInfoVLA.ll b/test/CodeGen/X86/2009-02-12-DebugInfoVLA.ll
index 6c07473e8a3..29086d2cce7 100644
--- a/test/CodeGen/X86/2009-02-12-DebugInfoVLA.ll
+++ b/test/CodeGen/X86/2009-02-12-DebugInfoVLA.ll
@@ -24,9 +24,9 @@ entry:
   %2 = alloca i64                                 ; <i64*> [#uses=1]
   %3 = alloca i64                                 ; <i64*> [#uses=6]
   %"alloca point" = bitcast i32 0 to i32          ; <i32> [#uses=0]
-  call void @llvm.dbg.declare(metadata i8** %s1_addr, metadata !0, metadata !MDExpression()), !dbg !7
+  call void @llvm.dbg.declare(metadata i8** %s1_addr, metadata !0, metadata !DIExpression()), !dbg !7
   store i8* %s1, i8** %s1_addr
-  call void @llvm.dbg.declare(metadata [0 x i8]** %str.0, metadata !8, metadata !MDExpression()), !dbg !7
+  call void @llvm.dbg.declare(metadata [0 x i8]** %str.0, metadata !8, metadata !DIExpression()), !dbg !7
   %4 = call i8* @llvm.stacksave(), !dbg !7        ; <i8*> [#uses=1]
   store i8* %4, i8** %saved_stack.1, align 8, !dbg !7
   %5 = load i8*, i8** %s1_addr, align 8, !dbg !13      ; <i8*> [#uses=1]
@@ -76,22 +76,22 @@ declare i64 @strlen(i8*) nounwind readonly
 
 declare void @llvm.stackrestore(i8*) nounwind
 
-!0 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "s1", line: 2, arg: 0, scope: !1, file: !2, type: !6)
-!1 = !MDSubprogram(name: "foo", linkageName: "foo", line: 2, isLocal: false, isDefinition: true, virtualIndex: 6, isOptimized: false, scope: !2, type: !3)
-!2 = !MDCompileUnit(language: DW_LANG_C89, producer: "4.2.1 (Based on Apple Inc. build 5658) (LLVM build)", isOptimized: true, emissionKind: 0, file: !17, enums: !18, retainedTypes: !18)
-!3 = !MDSubroutineType(types: !4)
+!0 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "s1", line: 2, arg: 1, scope: !1, file: !2, type: !6)
+!1 = !DISubprogram(name: "foo", linkageName: "foo", line: 2, isLocal: false, isDefinition: true, virtualIndex: 6, isOptimized: false, scope: !2, type: !3)
+!2 = !DICompileUnit(language: DW_LANG_C89, producer: "4.2.1 (Based on Apple Inc. build 5658) (LLVM build)", isOptimized: true, emissionKind: 0, file: !17, enums: !18, retainedTypes: !18)
+!3 = !DISubroutineType(types: !4)
 !4 = !{!5, !6}
-!5 = !MDBasicType(tag: DW_TAG_base_type, name: "char", size: 8, align: 8, encoding: DW_ATE_signed_char)
-!6 = !MDDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, scope: !2, baseType: !5)
-!7 = !MDLocation(line: 2, scope: !1)
-!8 = !MDLocalVariable(tag: DW_TAG_auto_variable, name: "str.0", line: 3, scope: !1, file: !2, type: !9)
-!9 = !MDDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, flags: DIFlagArtificial, scope: !2, baseType: !10)
-!10 = !MDCompositeType(tag: DW_TAG_array_type, size: 8, align: 8, scope: !2, baseType: !5, elements: !11)
+!5 = !DIBasicType(tag: DW_TAG_base_type, name: "char", size: 8, align: 8, encoding: DW_ATE_signed_char)
+!6 = !DIDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, scope: !2, baseType: !5)
+!7 = !DILocation(line: 2, scope: !1)
+!8 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "str.0", line: 3, scope: !1, file: !2, type: !9)
+!9 = !DIDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, flags: DIFlagArtificial, scope: !2, baseType: !10)
+!10 = !DICompositeType(tag: DW_TAG_array_type, size: 8, align: 8, scope: !2, baseType: !5, elements: !11)
 !11 = !{!12}
-!12 = !MDSubrange(count: 1)
-!13 = !MDLocation(line: 3, scope: !14)
-!14 = distinct !MDLexicalBlock(line: 0, column: 0, file: !17, scope: !1)
-!15 = !MDLocation(line: 4, scope: !14)
-!16 = !MDLocation(line: 5, scope: !14)
-!17 = !MDFile(filename: "vla.c", directory: "/tmp/")
+!12 = !DISubrange(count: 1)
+!13 = !DILocation(line: 3, scope: !14)
+!14 = distinct !DILexicalBlock(line: 0, column: 0, file: !17, scope: !1)
+!15 = !DILocation(line: 4, scope: !14)
+!16 = !DILocation(line: 5, scope: !14)
+!17 = !DIFile(filename: "vla.c", directory: "/tmp/")
 !18 = !{i32 0}
diff --git a/test/CodeGen/X86/2009-03-13-PHIElimBug.ll b/test/CodeGen/X86/2009-03-13-PHIElimBug.ll
index e14c30a2744..91f29c4f24c 100644
--- a/test/CodeGen/X86/2009-03-13-PHIElimBug.ll
+++ b/test/CodeGen/X86/2009-03-13-PHIElimBug.ll
@@ -6,7 +6,7 @@ declare i32 @f()
 
 declare i32 @g()
 
-define i32 @phi() {
+define i32 @phi() personality i32 (...)* @__gxx_personality_v0 {
 entry:
 	%a = call i32 @f()		; <i32> [#uses=1]
 	%b = invoke i32 @g()
@@ -24,7 +24,7 @@ cont2:		; preds = %cont
 
 lpad:		; preds = %cont, %entry
 	%y = phi i32 [ %a, %entry ], [ %aa, %cont ]		; <i32> [#uses=1]
-        %exn = landingpad {i8*, i32} personality i32 (...)* @__gxx_personality_v0
+        %exn = landingpad {i8*, i32}
                  cleanup
 	ret i32 %y
 }
diff --git a/test/CodeGen/X86/2009-03-16-PHIElimInLPad.ll b/test/CodeGen/X86/2009-03-16-PHIElimInLPad.ll
index f8c7a151b2c..6814ed1d894 100644
--- a/test/CodeGen/X86/2009-03-16-PHIElimInLPad.ll
+++ b/test/CodeGen/X86/2009-03-16-PHIElimInLPad.ll
@@ -3,7 +3,7 @@
 
 declare i32 @f()
 
-define i32 @phi(i32 %x) {
+define i32 @phi(i32 %x) personality i32 (...)* @__gxx_personality_v0 {
 entry:
 	%a = invoke i32 @f()
 			to label %cont unwind label %lpad		; <i32> [#uses=1]
@@ -17,7 +17,7 @@ cont2:		; preds = %cont
 
 lpad:		; preds = %cont, %entry
 	%v = phi i32 [ %x, %entry ], [ %a, %cont ]		; <i32> [#uses=1]
-        %exn = landingpad {i8*, i32} personality i32 (...)* @__gxx_personality_v0
+        %exn = landingpad {i8*, i32}
                  cleanup
 	ret i32 %v
 }
diff --git a/test/CodeGen/X86/2009-09-10-LoadFoldingBug.ll b/test/CodeGen/X86/2009-09-10-LoadFoldingBug.ll
index 2ec49f486c9..aa88576c148 100644
--- a/test/CodeGen/X86/2009-09-10-LoadFoldingBug.ll
+++ b/test/CodeGen/X86/2009-09-10-LoadFoldingBug.ll
@@ -9,7 +9,7 @@
 
 %struct.ComplexType = type { i32 }
 
-define i32 @t(i32 %clientPort, i32 %pluginID, i32 %requestID, i32 %objectID, i64 %serverIdentifier, i64 %argumentsData, i32 %argumentsLength) ssp {
+define i32 @t(i32 %clientPort, i32 %pluginID, i32 %requestID, i32 %objectID, i64 %serverIdentifier, i64 %argumentsData, i32 %argumentsLength) ssp personality i32 (...)* @__gxx_personality_v0 {
 entry:
 ; CHECK: _t:
 ; CHECK: movl 16(%rbp),
@@ -34,7 +34,7 @@ invcont2:                                         ; preds = %invcont1
   ret i32 0
 
 lpad:                                             ; preds = %invcont1, %invcont, %entry
-  %exn = landingpad {i8*, i32} personality i32 (...)* @__gxx_personality_v0
+  %exn = landingpad {i8*, i32}
             cleanup
   %8 = call i32 @vm_deallocate(i32 undef, i64 0, i64 %0) ; <i32> [#uses=0]
   unreachable
diff --git a/test/CodeGen/X86/2009-10-16-Scope.ll b/test/CodeGen/X86/2009-10-16-Scope.ll
index 374c6966652..bda7340b364 100644
--- a/test/CodeGen/X86/2009-10-16-Scope.ll
+++ b/test/CodeGen/X86/2009-10-16-Scope.ll
@@ -9,7 +9,7 @@ entry:
   br label %do.body, !dbg !0
 
 do.body:                                          ; preds = %entry
-  call void @llvm.dbg.declare(metadata i32* %count_, metadata !4, metadata !MDExpression()), !dbg !MDLocation(scope: !5)
+  call void @llvm.dbg.declare(metadata i32* %count_, metadata !4, metadata !DIExpression()), !dbg !DILocation(scope: !5)
   %conv = ptrtoint i32* %count_ to i32, !dbg !0   ; <i32> [#uses=1]
   %call = call i32 @foo(i32 %conv) ssp, !dbg !0   ; <i32> [#uses=0]
   br label %do.end, !dbg !0
@@ -22,13 +22,13 @@ declare void @llvm.dbg.declare(metadata, metadata, metadata) nounwind readnone
 
 declare i32 @foo(i32) ssp
 
-!0 = !MDLocation(line: 5, column: 2, scope: !1)
-!1 = distinct !MDLexicalBlock(line: 1, column: 1, file: null, scope: !2)
-!2 = !MDSubprogram(name: "bar", linkageName: "bar", line: 4, isLocal: false, isDefinition: true, virtualIndex: 6, isOptimized: false, scope: !3)
-!3 = !MDCompileUnit(language: DW_LANG_C99, producer: "clang 1.1", isOptimized: true, emissionKind: 0, file: !8, retainedTypes: !9)
-!4 = !MDLocalVariable(tag: DW_TAG_auto_variable, name: "count_", line: 5, scope: !5, file: !3, type: !6)
-!5 = distinct !MDLexicalBlock(line: 1, column: 1, file: null, scope: !1)
-!6 = !MDBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
-!7 = !MDLocation(line: 6, column: 1, scope: !2)
-!8 = !MDFile(filename: "genmodes.i", directory: "/Users/yash/Downloads")
+!0 = !DILocation(line: 5, column: 2, scope: !1)
+!1 = distinct !DILexicalBlock(line: 1, column: 1, file: null, scope: !2)
+!2 = !DISubprogram(name: "bar", linkageName: "bar", line: 4, isLocal: false, isDefinition: true, virtualIndex: 6, isOptimized: false, scope: !3)
+!3 = !DICompileUnit(language: DW_LANG_C99, producer: "clang 1.1", isOptimized: true, emissionKind: 0, file: !8, retainedTypes: !9)
+!4 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "count_", line: 5, scope: !5, file: !3, type: !6)
+!5 = distinct !DILexicalBlock(line: 1, column: 1, file: null, scope: !1)
+!6 = !DIBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
+!7 = !DILocation(line: 6, column: 1, scope: !2)
+!8 = !DIFile(filename: "genmodes.i", directory: "/Users/yash/Downloads")
 !9 = !{i32 0}
diff --git a/test/CodeGen/X86/2009-10-19-atomic-cmp-eflags.ll b/test/CodeGen/X86/2009-10-19-atomic-cmp-eflags.ll
index 386b028086b..e1ca3fd2a97 100644
--- a/test/CodeGen/X86/2009-10-19-atomic-cmp-eflags.ll
+++ b/test/CodeGen/X86/2009-10-19-atomic-cmp-eflags.ll
@@ -37,8 +37,7 @@ lt_init.exit:                                     ; preds = %if.end.i, %if.then.
   %4 = call i64 @llvm.readcyclecounter() nounwind ; <i64> [#uses=1]
   %5 = sub i64 %4, %2                             ; <i64> [#uses=1]
   %6 = atomicrmw add i64* getelementptr inbounds ([1216 x i64], [1216 x i64]* @__profiling_callsite_timestamps_live, i32 0, i32 51), i64 %5 monotonic
-;CHECK: lock
-;CHECK-NEXT: {{xadd|addq}} %rdx, __profiling_callsite_timestamps_live
+;CHECK: lock {{xadd|addq}} %rdx, __profiling_callsite_timestamps_live
 ;CHECK-NEXT: cmpl $0,
 ;CHECK-NEXT: jne
   %cmp = icmp eq i32 %3, 0                        ; <i1> [#uses=1]
diff --git a/test/CodeGen/X86/2009-11-25-ImpDefBug.ll b/test/CodeGen/X86/2009-11-25-ImpDefBug.ll
index 0bf13de6127..2f4e11e54e3 100644
--- a/test/CodeGen/X86/2009-11-25-ImpDefBug.ll
+++ b/test/CodeGen/X86/2009-11-25-ImpDefBug.ll
@@ -20,7 +20,7 @@ declare void @_ZNSt6vectorIP10ASN1ObjectSaIS1_EE13_M_insert_auxEN9__gnu_cxx17__n
 
 declare i32 @_Z17LoadObjectFromBERR8xmstreamPP10ASN1ObjectPPF10ASN1StatusP13ASN1ObjHeaderS3_E(%struct.xmstream*, %struct.ASN1Object**, i32 (%struct.ASN1ObjHeader*, %struct.ASN1Object**)**)
 
-define i32 @_ZN8ASN1Unit4loadER8xmstreamjm18ASN1LengthEncoding(%struct.ASN1Unit* %this, %struct.xmstream* nocapture %stream, i32 %numObjects, i64 %size, i32 %lEncoding) {
+define i32 @_ZN8ASN1Unit4loadER8xmstreamjm18ASN1LengthEncoding(%struct.ASN1Unit* %this, %struct.xmstream* nocapture %stream, i32 %numObjects, i64 %size, i32 %lEncoding) personality i32 (...)* @__gxx_personality_v0 {
 entry:
   br label %meshBB85
 
@@ -46,7 +46,7 @@ bb1.i5:                                           ; preds = %bb.i1
 
 lpad:                                             ; preds = %bb1.i.fragment.cl, %bb1.i.fragment, %bb5
   %.SV10.phi807 = phi i8* [ undef, %bb1.i.fragment.cl ], [ undef, %bb1.i.fragment ], [ undef, %bb5 ] ; <i8*> [#uses=1]
-  %exn = landingpad {i8*, i32} personality i32 (...)* @__gxx_personality_v0
+  %exn = landingpad {i8*, i32}
             cleanup
   %1 = load i8, i8* %.SV10.phi807, align 8            ; <i8> [#uses=0]
   br i1 undef, label %meshBB81.bbcl.disp, label %bb13.fragment.bbcl.disp
diff --git a/test/CodeGen/X86/2010-01-08-Atomic64Bug.ll b/test/CodeGen/X86/2010-01-08-Atomic64Bug.ll
index 850f678c9c2..3391f1f5ec4 100644
--- a/test/CodeGen/X86/2010-01-08-Atomic64Bug.ll
+++ b/test/CodeGen/X86/2010-01-08-Atomic64Bug.ll
@@ -14,8 +14,7 @@ entry:
 ; CHECK: addl $1, %ebx
 ; CHECK: movl %edx, %ecx
 ; CHECK: adcl $0, %ecx
-; CHECK: lock
-; CHECK-NEXT: cmpxchg8b ([[REG]])
+; CHECK: lock cmpxchg8b ([[REG]])
 ; CHECK-NEXT: jne
   %0 = atomicrmw add i64* %p, i64 1 seq_cst
   ret void
diff --git a/test/CodeGen/X86/2010-01-18-DbgValue.ll b/test/CodeGen/X86/2010-01-18-DbgValue.ll
index b03556a0d73..a1779433607 100644
--- a/test/CodeGen/X86/2010-01-18-DbgValue.ll
+++ b/test/CodeGen/X86/2010-01-18-DbgValue.ll
@@ -12,7 +12,7 @@ entry:
   %retval = alloca double                         ; <double*> [#uses=2]
   %0 = alloca double                              ; <double*> [#uses=2]
   %"alloca point" = bitcast i32 0 to i32          ; <i32> [#uses=0]
-  call void @llvm.dbg.declare(metadata %struct.Rect* %my_r0, metadata !0, metadata !MDExpression()), !dbg !15
+  call void @llvm.dbg.declare(metadata %struct.Rect* %my_r0, metadata !0, metadata !DIExpression()), !dbg !15
   %1 = getelementptr inbounds %struct.Rect, %struct.Rect* %my_r0, i32 0, i32 0, !dbg !16 ; <%struct.Pt*> [#uses=1]
   %2 = getelementptr inbounds %struct.Pt, %struct.Pt* %1, i32 0, i32 0, !dbg !16 ; <double*> [#uses=1]
   %3 = load double, double* %2, align 8, !dbg !16         ; <double> [#uses=1]
@@ -31,25 +31,25 @@ declare void @llvm.dbg.declare(metadata, metadata, metadata) nounwind readnone
 !llvm.dbg.cu = !{!3}
 !llvm.module.flags = !{!21}
 
-!0 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "my_r0", line: 11, arg: 0, scope: !1, file: !2, type: !7)
-!1 = !MDSubprogram(name: "foo", linkageName: "foo", line: 11, isLocal: false, isDefinition: true, virtualIndex: 6, isOptimized: false, scopeLine: 11, file: !19, scope: !2, type: !4, function: double (%struct.Rect*)* @foo)
-!2 = !MDFile(filename: "b2.c", directory: "/tmp/")
-!3 = !MDCompileUnit(language: DW_LANG_C89, producer: "4.2.1 (Based on Apple Inc. build 5658) (LLVM build)", isOptimized: false, emissionKind: 0, file: !19, enums: !20, retainedTypes: !20, subprograms: !18)
-!4 = !MDSubroutineType(types: !5)
+!0 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "my_r0", line: 11, arg: 1, scope: !1, file: !2, type: !7)
+!1 = !DISubprogram(name: "foo", linkageName: "foo", line: 11, isLocal: false, isDefinition: true, virtualIndex: 6, isOptimized: false, scopeLine: 11, file: !19, scope: !2, type: !4, function: double (%struct.Rect*)* @foo)
+!2 = !DIFile(filename: "b2.c", directory: "/tmp/")
+!3 = !DICompileUnit(language: DW_LANG_C89, producer: "4.2.1 (Based on Apple Inc. build 5658) (LLVM build)", isOptimized: false, emissionKind: 0, file: !19, enums: !20, retainedTypes: !20, subprograms: !18)
+!4 = !DISubroutineType(types: !5)
 !5 = !{!6, !7}
-!6 = !MDBasicType(tag: DW_TAG_base_type, name: "double", size: 64, align: 64, encoding: DW_ATE_float)
-!7 = !MDCompositeType(tag: DW_TAG_structure_type, name: "Rect", line: 6, size: 256, align: 64, file: !19, scope: !2, elements: !8)
+!6 = !DIBasicType(tag: DW_TAG_base_type, name: "double", size: 64, align: 64, encoding: DW_ATE_float)
+!7 = !DICompositeType(tag: DW_TAG_structure_type, name: "Rect", line: 6, size: 256, align: 64, file: !19, scope: !2, elements: !8)
 !8 = !{!9, !14}
-!9 = !MDDerivedType(tag: DW_TAG_member, name: "P1", line: 7, size: 128, align: 64, file: !19, scope: !7, baseType: !10)
-!10 = !MDCompositeType(tag: DW_TAG_structure_type, name: "Pt", line: 1, size: 128, align: 64, file: !19, scope: !2, elements: !11)
+!9 = !DIDerivedType(tag: DW_TAG_member, name: "P1", line: 7, size: 128, align: 64, file: !19, scope: !7, baseType: !10)
+!10 = !DICompositeType(tag: DW_TAG_structure_type, name: "Pt", line: 1, size: 128, align: 64, file: !19, scope: !2, elements: !11)
 !11 = !{!12, !13}
-!12 = !MDDerivedType(tag: DW_TAG_member, name: "x", line: 2, size: 64, align: 64, file: !19, scope: !10, baseType: !6)
-!13 = !MDDerivedType(tag: DW_TAG_member, name: "y", line: 3, size: 64, align: 64, offset: 64, file: !19, scope: !10, baseType: !6)
-!14 = !MDDerivedType(tag: DW_TAG_member, name: "P2", line: 8, size: 128, align: 64, offset: 128, file: !19, scope: !7, baseType: !10)
-!15 = !MDLocation(line: 11, scope: !1)
-!16 = !MDLocation(line: 12, scope: !17)
-!17 = distinct !MDLexicalBlock(line: 11, column: 0, file: !19, scope: !1)
+!12 = !DIDerivedType(tag: DW_TAG_member, name: "x", line: 2, size: 64, align: 64, file: !19, scope: !10, baseType: !6)
+!13 = !DIDerivedType(tag: DW_TAG_member, name: "y", line: 3, size: 64, align: 64, offset: 64, file: !19, scope: !10, baseType: !6)
+!14 = !DIDerivedType(tag: DW_TAG_member, name: "P2", line: 8, size: 128, align: 64, offset: 128, file: !19, scope: !7, baseType: !10)
+!15 = !DILocation(line: 11, scope: !1)
+!16 = !DILocation(line: 12, scope: !17)
+!17 = distinct !DILexicalBlock(line: 11, column: 0, file: !19, scope: !1)
 !18 = !{!1}
-!19 = !MDFile(filename: "b2.c", directory: "/tmp/")
+!19 = !DIFile(filename: "b2.c", directory: "/tmp/")
 !20 = !{}
 !21 = !{i32 1, !"Debug Info Version", i32 3}
diff --git a/test/CodeGen/X86/2010-02-01-DbgValueCrash.ll b/test/CodeGen/X86/2010-02-01-DbgValueCrash.ll
index 7c0fd686bc5..e65edac86ec 100644
--- a/test/CodeGen/X86/2010-02-01-DbgValueCrash.ll
+++ b/test/CodeGen/X86/2010-02-01-DbgValueCrash.ll
@@ -8,7 +8,7 @@
 
 define i32 @"main(tart.core.String[])->int32"(i32 %args) {
 entry:
-  tail call void @llvm.dbg.value(metadata %tart.reflect.ComplexType* @.type.SwitchStmtTest, i64 0, metadata !8, metadata !MDExpression()), !dbg !MDLocation(scope: !9)
+  tail call void @llvm.dbg.value(metadata %tart.reflect.ComplexType* @.type.SwitchStmtTest, i64 0, metadata !8, metadata !DIExpression()), !dbg !DILocation(scope: !9)
   tail call void @"tart.reflect.ComplexType.create->tart.core.Object"(%tart.reflect.ComplexType* @.type.SwitchStmtTest) ; <%tart.core.Object*> [#uses=2]
   ret i32 3
 }
@@ -16,20 +16,20 @@ entry:
 declare void @llvm.dbg.value(metadata, i64, metadata, metadata) nounwind readnone
 declare void @"tart.reflect.ComplexType.create->tart.core.Object"(%tart.reflect.ComplexType*) nounwind readnone
 
-!0 = !MDCompileUnit(language: DW_LANG_C89, producer: "4.2.1 (Based on Apple Inc. build 5658) (LLVM build)", isOptimized: true, emissionKind: 0, file: !15, enums: !16, retainedTypes: !16)
-!1 = !MDDerivedType(tag: DW_TAG_const_type, size: 192, align: 64, file: !15, scope: !0, baseType: !2)
-!2 = !MDCompositeType(tag: DW_TAG_structure_type, name: "C", line: 1, size: 192, align: 64, file: !15, scope: !0, elements: !3)
+!0 = !DICompileUnit(language: DW_LANG_C89, producer: "4.2.1 (Based on Apple Inc. build 5658) (LLVM build)", isOptimized: true, emissionKind: 0, file: !15, enums: !16, retainedTypes: !16)
+!1 = !DIDerivedType(tag: DW_TAG_const_type, size: 192, align: 64, file: !15, scope: !0, baseType: !2)
+!2 = !DICompositeType(tag: DW_TAG_structure_type, name: "C", line: 1, size: 192, align: 64, file: !15, scope: !0, elements: !3)
 !3 = !{!4, !6, !7}
-!4 = !MDDerivedType(tag: DW_TAG_member, name: "x", line: 1, size: 64, align: 64, file: !15, scope: !2, baseType: !5)
-!5 = !MDBasicType(tag: DW_TAG_base_type, name: "double", size: 64, align: 64, encoding: DW_ATE_float)
-!6 = !MDDerivedType(tag: DW_TAG_member, name: "y", line: 1, size: 64, align: 64, offset: 64, file: !15, scope: !2, baseType: !5)
-!7 = !MDDerivedType(tag: DW_TAG_member, name: "z", line: 1, size: 64, align: 64, offset: 128, file: !15, scope: !2, baseType: !5)
-!8 = !MDLocalVariable(tag: DW_TAG_auto_variable, name: "t", line: 5, scope: !9, file: !0, type: !2)
-!9 = distinct !MDLexicalBlock(line: 0, column: 0, file: null, scope: !10)
-!10 = !MDSubprogram(name: "foo", linkageName: "foo", line: 4, isLocal: false, isDefinition: true, virtualIndex: 6, isOptimized: false, scope: !0, type: !11)
-!11 = !MDSubroutineType(types: !12)
+!4 = !DIDerivedType(tag: DW_TAG_member, name: "x", line: 1, size: 64, align: 64, file: !15, scope: !2, baseType: !5)
+!5 = !DIBasicType(tag: DW_TAG_base_type, name: "double", size: 64, align: 64, encoding: DW_ATE_float)
+!6 = !DIDerivedType(tag: DW_TAG_member, name: "y", line: 1, size: 64, align: 64, offset: 64, file: !15, scope: !2, baseType: !5)
+!7 = !DIDerivedType(tag: DW_TAG_member, name: "z", line: 1, size: 64, align: 64, offset: 128, file: !15, scope: !2, baseType: !5)
+!8 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "t", line: 5, scope: !9, file: !0, type: !2)
+!9 = distinct !DILexicalBlock(line: 0, column: 0, file: null, scope: !10)
+!10 = !DISubprogram(name: "foo", linkageName: "foo", line: 4, isLocal: false, isDefinition: true, virtualIndex: 6, isOptimized: false, scope: !0, type: !11)
+!11 = !DISubroutineType(types: !12)
 !12 = !{!13}
-!13 = !MDBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
+!13 = !DIBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
 !14 = !{%tart.reflect.ComplexType* @.type.SwitchStmtTest}
-!15 = !MDFile(filename: "sm.c", directory: "")
+!15 = !DIFile(filename: "sm.c", directory: "")
 !16 = !{i32 0}
diff --git a/test/CodeGen/X86/2010-04-06-SSEDomainFixCrash.ll b/test/CodeGen/X86/2010-04-06-SSEDomainFixCrash.ll
index 2ba4d9aaded..41c318b62ea 100644
--- a/test/CodeGen/X86/2010-04-06-SSEDomainFixCrash.ll
+++ b/test/CodeGen/X86/2010-04-06-SSEDomainFixCrash.ll
@@ -7,7 +7,7 @@ target triple = "i386-apple-darwin10.0"
 
 declare i32 @_ZN11HullLibrary16CreateConvexHullERK8HullDescR10HullResult(i8*, i8* nocapture, i8* nocapture) ssp align 2
 
-define void @_ZN17btSoftBodyHelpers4DrawEP10btSoftBodyP12btIDebugDrawi(i8* %psb, i8* %idraw, i32 %drawflags) ssp align 2 {
+define void @_ZN17btSoftBodyHelpers4DrawEP10btSoftBodyP12btIDebugDrawi(i8* %psb, i8* %idraw, i32 %drawflags) ssp align 2 personality i32 (...)* @__gxx_personality_v0 {
 entry:
   br i1 undef, label %bb92, label %bb58
 
@@ -60,7 +60,7 @@ bb92:                                             ; preds = %entry
   unreachable
 
 lpad159:                                          ; preds = %bb58
-  %exn = landingpad {i8*, i32} personality i32 (...)* @__gxx_personality_v0
+  %exn = landingpad {i8*, i32}
             cleanup
   unreachable
 }
diff --git a/test/CodeGen/X86/2010-04-30-LocalAlloc-LandingPad.ll b/test/CodeGen/X86/2010-04-30-LocalAlloc-LandingPad.ll
index a46b8d9148e..fc5520e12ac 100644
--- a/test/CodeGen/X86/2010-04-30-LocalAlloc-LandingPad.ll
+++ b/test/CodeGen/X86/2010-04-30-LocalAlloc-LandingPad.ll
@@ -13,7 +13,7 @@ target triple = "i386-apple-darwin10.0.0"
 ; CHECK: movl %esi,{{.*}}(%ebp) 
 ; CHECK: calll __Z6throwsv
 
-define i8* @_Z4test1SiS_(%struct.S* byval %s1, i32 %n, %struct.S* byval %s2) ssp {
+define i8* @_Z4test1SiS_(%struct.S* byval %s1, i32 %n, %struct.S* byval %s2) ssp personality i8* bitcast (i32 (...)* @__gxx_personality_v0 to i8*) {
 entry:
   %retval = alloca i8*, align 4                   ; <i8**> [#uses=2]
   %n.addr = alloca i32, align 4                   ; <i32*> [#uses=1]
@@ -30,13 +30,13 @@ invoke.cont:                                      ; preds = %entry
   br label %finally
 
 terminate.handler:                                ; preds = %match.end
-  %1 = landingpad { i8*, i32 } personality i8* bitcast (i32 (...)* @__gxx_personality_v0 to i8*)
+  %1 = landingpad { i8*, i32 }
            cleanup
   call void @_ZSt9terminatev() noreturn nounwind
   unreachable
 
 try.handler:                                      ; preds = %entry
-  %exc1.ptr = landingpad { i8*, i32 } personality i8* bitcast (i32 (...)* @__gxx_personality_v0 to i8*)
+  %exc1.ptr = landingpad { i8*, i32 }
            catch i8* null
   %exc1 = extractvalue { i8*, i32 } %exc1.ptr, 0
   %selector = extractvalue { i8*, i32 } %exc1.ptr, 1
@@ -49,7 +49,7 @@ match:                                            ; preds = %try.handler
   %5 = bitcast i8* %4 to i32*                     ; <i32*> [#uses=1]
   %6 = load i32, i32* %5                               ; <i32> [#uses=1]
   store i32 %6, i32* %0
-  %call = invoke i32 (i8*, ...)* @printf(i8* getelementptr inbounds ([4 x i8], [4 x i8]* @.str, i32 0, i32 0), %struct.S* %s2)
+  %call = invoke i32 (i8*, ...) @printf(i8* getelementptr inbounds ([4 x i8], [4 x i8]* @.str, i32 0, i32 0), %struct.S* %s2)
           to label %invoke.cont2 unwind label %match.handler ; <i32> [#uses=0]
 
 invoke.cont2:                                     ; preds = %match
@@ -57,7 +57,7 @@ invoke.cont2:                                     ; preds = %match
   br label %match.end
 
 match.handler:                                    ; preds = %match
-  %exc3 = landingpad { i8*, i32 } personality i8* bitcast (i32 (...)* @__gxx_personality_v0 to i8*)
+  %exc3 = landingpad { i8*, i32 }
            cleanup
   %7 = extractvalue { i8*, i32 } %exc3, 0
   store i8* %7, i8** %_rethrow
diff --git a/test/CodeGen/X86/2010-05-25-DotDebugLoc.ll b/test/CodeGen/X86/2010-05-25-DotDebugLoc.ll
index 29df2910645..4e5592f8900 100644
--- a/test/CodeGen/X86/2010-05-25-DotDebugLoc.ll
+++ b/test/CodeGen/X86/2010-05-25-DotDebugLoc.ll
@@ -2,18 +2,17 @@
 ; RUN: llc -mtriple=x86_64-pc-linux -O2 -regalloc=basic < %s | FileCheck %s
 ; Test to check .debug_loc support. This test case emits many debug_loc entries.
 
-; CHECK: .short {{.*}} # Loc expr size
-; CHECK-NEXT: .Ltmp
+; CHECK: .short 1 # Loc expr size
 ; CHECK-NEXT: DW_OP_reg
 
 %0 = type { double }
 
 define hidden %0 @__divsc3(float %a, float %b, float %c, float %d) nounwind readnone {
 entry:
-  tail call void @llvm.dbg.value(metadata float %a, i64 0, metadata !0, metadata !MDExpression()), !dbg !MDLocation(scope: !1)
-  tail call void @llvm.dbg.value(metadata float %b, i64 0, metadata !11, metadata !MDExpression()), !dbg !MDLocation(scope: !1)
-  tail call void @llvm.dbg.value(metadata float %c, i64 0, metadata !12, metadata !MDExpression()), !dbg !MDLocation(scope: !1)
-  tail call void @llvm.dbg.value(metadata float %d, i64 0, metadata !13, metadata !MDExpression()), !dbg !MDLocation(scope: !1)
+  tail call void @llvm.dbg.value(metadata float %a, i64 0, metadata !0, metadata !DIExpression()), !dbg !DILocation(scope: !1)
+  tail call void @llvm.dbg.value(metadata float %b, i64 0, metadata !11, metadata !DIExpression()), !dbg !DILocation(scope: !1)
+  tail call void @llvm.dbg.value(metadata float %c, i64 0, metadata !12, metadata !DIExpression()), !dbg !DILocation(scope: !1)
+  tail call void @llvm.dbg.value(metadata float %d, i64 0, metadata !13, metadata !DIExpression()), !dbg !DILocation(scope: !1)
   %0 = tail call float @fabsf(float %c) nounwind readnone, !dbg !19 ; <float> [#uses=1]
   %1 = tail call float @fabsf(float %d) nounwind readnone, !dbg !19 ; <float> [#uses=1]
   %2 = fcmp olt float %0, %1, !dbg !19            ; <i1> [#uses=1]
@@ -21,34 +20,34 @@ entry:
 
 bb:                                               ; preds = %entry
   %3 = fdiv float %c, %d, !dbg !20                ; <float> [#uses=3]
-  tail call void @llvm.dbg.value(metadata float %3, i64 0, metadata !16, metadata !MDExpression()), !dbg !20
+  tail call void @llvm.dbg.value(metadata float %3, i64 0, metadata !16, metadata !DIExpression()), !dbg !20
   %4 = fmul float %3, %c, !dbg !21                ; <float> [#uses=1]
   %5 = fadd float %4, %d, !dbg !21                ; <float> [#uses=2]
-  tail call void @llvm.dbg.value(metadata float %5, i64 0, metadata !14, metadata !MDExpression()), !dbg !21
+  tail call void @llvm.dbg.value(metadata float %5, i64 0, metadata !14, metadata !DIExpression()), !dbg !21
   %6 = fmul float %3, %a, !dbg !22                ; <float> [#uses=1]
   %7 = fadd float %6, %b, !dbg !22                ; <float> [#uses=1]
   %8 = fdiv float %7, %5, !dbg !22                ; <float> [#uses=1]
-  tail call void @llvm.dbg.value(metadata float %8, i64 0, metadata !17, metadata !MDExpression()), !dbg !22
+  tail call void @llvm.dbg.value(metadata float %8, i64 0, metadata !17, metadata !DIExpression()), !dbg !22
   %9 = fmul float %3, %b, !dbg !23                ; <float> [#uses=1]
   %10 = fsub float %9, %a, !dbg !23               ; <float> [#uses=1]
   %11 = fdiv float %10, %5, !dbg !23              ; <float> [#uses=1]
-  tail call void @llvm.dbg.value(metadata float %11, i64 0, metadata !18, metadata !MDExpression()), !dbg !23
+  tail call void @llvm.dbg.value(metadata float %11, i64 0, metadata !18, metadata !DIExpression()), !dbg !23
   br label %bb2, !dbg !23
 
 bb1:                                              ; preds = %entry
   %12 = fdiv float %d, %c, !dbg !24               ; <float> [#uses=3]
-  tail call void @llvm.dbg.value(metadata float %12, i64 0, metadata !16, metadata !MDExpression()), !dbg !24
+  tail call void @llvm.dbg.value(metadata float %12, i64 0, metadata !16, metadata !DIExpression()), !dbg !24
   %13 = fmul float %12, %d, !dbg !25              ; <float> [#uses=1]
   %14 = fadd float %13, %c, !dbg !25              ; <float> [#uses=2]
-  tail call void @llvm.dbg.value(metadata float %14, i64 0, metadata !14, metadata !MDExpression()), !dbg !25
+  tail call void @llvm.dbg.value(metadata float %14, i64 0, metadata !14, metadata !DIExpression()), !dbg !25
   %15 = fmul float %12, %b, !dbg !26              ; <float> [#uses=1]
   %16 = fadd float %15, %a, !dbg !26              ; <float> [#uses=1]
   %17 = fdiv float %16, %14, !dbg !26             ; <float> [#uses=1]
-  tail call void @llvm.dbg.value(metadata float %17, i64 0, metadata !17, metadata !MDExpression()), !dbg !26
+  tail call void @llvm.dbg.value(metadata float %17, i64 0, metadata !17, metadata !DIExpression()), !dbg !26
   %18 = fmul float %12, %a, !dbg !27              ; <float> [#uses=1]
   %19 = fsub float %b, %18, !dbg !27              ; <float> [#uses=1]
   %20 = fdiv float %19, %14, !dbg !27             ; <float> [#uses=1]
-  tail call void @llvm.dbg.value(metadata float %20, i64 0, metadata !18, metadata !MDExpression()), !dbg !27
+  tail call void @llvm.dbg.value(metadata float %20, i64 0, metadata !18, metadata !DIExpression()), !dbg !27
   br label %bb2, !dbg !27
 
 bb2:                                              ; preds = %bb1, %bb
@@ -74,9 +73,9 @@ bb6:                                              ; preds = %bb4
 bb8:                                              ; preds = %bb6
   %27 = tail call float @copysignf(float 0x7FF0000000000000, float %c) nounwind readnone, !dbg !30 ; <float> [#uses=2]
   %28 = fmul float %27, %a, !dbg !30              ; <float> [#uses=1]
-  tail call void @llvm.dbg.value(metadata float %28, i64 0, metadata !17, metadata !MDExpression()), !dbg !30
+  tail call void @llvm.dbg.value(metadata float %28, i64 0, metadata !17, metadata !DIExpression()), !dbg !30
   %29 = fmul float %27, %b, !dbg !31              ; <float> [#uses=1]
-  tail call void @llvm.dbg.value(metadata float %29, i64 0, metadata !18, metadata !MDExpression()), !dbg !31
+  tail call void @llvm.dbg.value(metadata float %29, i64 0, metadata !18, metadata !DIExpression()), !dbg !31
   br label %bb46, !dbg !31
 
 bb9:                                              ; preds = %bb6, %bb4
@@ -106,24 +105,24 @@ bb15:                                             ; preds = %bb14
 bb16:                                             ; preds = %bb15
   %iftmp.0.0 = select i1 %33, float 1.000000e+00, float 0.000000e+00 ; <float> [#uses=1]
   %42 = tail call float @copysignf(float %iftmp.0.0, float %a) nounwind readnone, !dbg !33 ; <float> [#uses=2]
-  tail call void @llvm.dbg.value(metadata float %42, i64 0, metadata !0, metadata !MDExpression()), !dbg !33
+  tail call void @llvm.dbg.value(metadata float %42, i64 0, metadata !0, metadata !DIExpression()), !dbg !33
   %43 = fcmp ord float %b, 0.000000e+00           ; <i1> [#uses=1]
   %44 = fsub float %b, %b, !dbg !34               ; <float> [#uses=1]
   %45 = fcmp uno float %44, 0.000000e+00          ; <i1> [#uses=1]
   %46 = and i1 %43, %45, !dbg !34                 ; <i1> [#uses=1]
   %iftmp.1.0 = select i1 %46, float 1.000000e+00, float 0.000000e+00 ; <float> [#uses=1]
   %47 = tail call float @copysignf(float %iftmp.1.0, float %b) nounwind readnone, !dbg !34 ; <float> [#uses=2]
-  tail call void @llvm.dbg.value(metadata float %47, i64 0, metadata !11, metadata !MDExpression()), !dbg !34
+  tail call void @llvm.dbg.value(metadata float %47, i64 0, metadata !11, metadata !DIExpression()), !dbg !34
   %48 = fmul float %42, %c, !dbg !35              ; <float> [#uses=1]
   %49 = fmul float %47, %d, !dbg !35              ; <float> [#uses=1]
   %50 = fadd float %48, %49, !dbg !35             ; <float> [#uses=1]
   %51 = fmul float %50, 0x7FF0000000000000, !dbg !35 ; <float> [#uses=1]
-  tail call void @llvm.dbg.value(metadata float %51, i64 0, metadata !17, metadata !MDExpression()), !dbg !35
+  tail call void @llvm.dbg.value(metadata float %51, i64 0, metadata !17, metadata !DIExpression()), !dbg !35
   %52 = fmul float %47, %c, !dbg !36              ; <float> [#uses=1]
   %53 = fmul float %42, %d, !dbg !36              ; <float> [#uses=1]
   %54 = fsub float %52, %53, !dbg !36             ; <float> [#uses=1]
   %55 = fmul float %54, 0x7FF0000000000000, !dbg !36 ; <float> [#uses=1]
-  tail call void @llvm.dbg.value(metadata float %55, i64 0, metadata !18, metadata !MDExpression()), !dbg !36
+  tail call void @llvm.dbg.value(metadata float %55, i64 0, metadata !18, metadata !DIExpression()), !dbg !36
   br label %bb46, !dbg !36
 
 bb27:                                             ; preds = %bb15, %bb14, %bb11
@@ -154,24 +153,24 @@ bb34:                                             ; preds = %bb33, %bb30
 bb35:                                             ; preds = %bb34
   %iftmp.2.0 = select i1 %59, float 1.000000e+00, float 0.000000e+00 ; <float> [#uses=1]
   %67 = tail call float @copysignf(float %iftmp.2.0, float %c) nounwind readnone, !dbg !38 ; <float> [#uses=2]
-  tail call void @llvm.dbg.value(metadata float %67, i64 0, metadata !12, metadata !MDExpression()), !dbg !38
+  tail call void @llvm.dbg.value(metadata float %67, i64 0, metadata !12, metadata !DIExpression()), !dbg !38
   %68 = fcmp ord float %d, 0.000000e+00           ; <i1> [#uses=1]
   %69 = fsub float %d, %d, !dbg !39               ; <float> [#uses=1]
   %70 = fcmp uno float %69, 0.000000e+00          ; <i1> [#uses=1]
   %71 = and i1 %68, %70, !dbg !39                 ; <i1> [#uses=1]
   %iftmp.3.0 = select i1 %71, float 1.000000e+00, float 0.000000e+00 ; <float> [#uses=1]
   %72 = tail call float @copysignf(float %iftmp.3.0, float %d) nounwind readnone, !dbg !39 ; <float> [#uses=2]
-  tail call void @llvm.dbg.value(metadata float %72, i64 0, metadata !13, metadata !MDExpression()), !dbg !39
+  tail call void @llvm.dbg.value(metadata float %72, i64 0, metadata !13, metadata !DIExpression()), !dbg !39
   %73 = fmul float %67, %a, !dbg !40              ; <float> [#uses=1]
   %74 = fmul float %72, %b, !dbg !40              ; <float> [#uses=1]
   %75 = fadd float %73, %74, !dbg !40             ; <float> [#uses=1]
   %76 = fmul float %75, 0.000000e+00, !dbg !40    ; <float> [#uses=1]
-  tail call void @llvm.dbg.value(metadata float %76, i64 0, metadata !17, metadata !MDExpression()), !dbg !40
+  tail call void @llvm.dbg.value(metadata float %76, i64 0, metadata !17, metadata !DIExpression()), !dbg !40
   %77 = fmul float %67, %b, !dbg !41              ; <float> [#uses=1]
   %78 = fmul float %72, %a, !dbg !41              ; <float> [#uses=1]
   %79 = fsub float %77, %78, !dbg !41             ; <float> [#uses=1]
   %80 = fmul float %79, 0.000000e+00, !dbg !41    ; <float> [#uses=1]
-  tail call void @llvm.dbg.value(metadata float %80, i64 0, metadata !18, metadata !MDExpression()), !dbg !41
+  tail call void @llvm.dbg.value(metadata float %80, i64 0, metadata !18, metadata !DIExpression()), !dbg !41
   br label %bb46, !dbg !41
 
 bb46:                                             ; preds = %bb35, %bb34, %bb33, %bb30, %bb16, %bb8, %bb2
@@ -200,52 +199,52 @@ declare void @llvm.dbg.value(metadata, i64, metadata, metadata) nounwind readnon
 !llvm.dbg.cu = !{!3}
 !llvm.module.flags = !{!48}
 
-!0 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "a", line: 1921, arg: 0, scope: !1, file: !2, type: !9)
-!1 = !MDSubprogram(name: "__divsc3", linkageName: "__divsc3", line: 1922, isLocal: false, isDefinition: true, virtualIndex: 6, isOptimized: true, scopeLine: 1922, file: !45, scope: !2, type: !4, function: %0 (float, float, float, float)* @__divsc3, variables: !43)
-!2 = !MDFile(filename: "libgcc2.c", directory: "/Users/yash/clean/LG.D/gcc/../../llvmgcc/gcc")
-!3 = !MDCompileUnit(language: DW_LANG_C89, producer: "4.2.1 (Based on Apple Inc. build 5658) (LLVM build)", isOptimized: true, emissionKind: 1, file: !45, enums: !47, retainedTypes: !47, subprograms: !44, imports:  null)
-!4 = !MDSubroutineType(types: !5)
+!0 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "a", line: 1921, arg: 1, scope: !1, file: !2, type: !9)
+!1 = !DISubprogram(name: "__divsc3", linkageName: "__divsc3", line: 1922, isLocal: false, isDefinition: true, virtualIndex: 6, isOptimized: true, scopeLine: 1922, file: !45, scope: !2, type: !4, function: %0 (float, float, float, float)* @__divsc3, variables: !43)
+!2 = !DIFile(filename: "libgcc2.c", directory: "/Users/yash/clean/LG.D/gcc/../../llvmgcc/gcc")
+!3 = !DICompileUnit(language: DW_LANG_C89, producer: "4.2.1 (Based on Apple Inc. build 5658) (LLVM build)", isOptimized: true, emissionKind: 1, file: !45, enums: !47, retainedTypes: !47, subprograms: !44, imports:  null)
+!4 = !DISubroutineType(types: !5)
 !5 = !{!6, !9, !9, !9, !9}
-!6 = !MDDerivedType(tag: DW_TAG_typedef, name: "SCtype", line: 170, file: !46, scope: !7, baseType: !8)
-!7 = !MDFile(filename: "libgcc2.h", directory: "/Users/yash/clean/LG.D/gcc/../../llvmgcc/gcc")
-!8 = !MDBasicType(tag: DW_TAG_base_type, name: "complex float", size: 64, align: 32, encoding: DW_ATE_complex_float)
-!9 = !MDDerivedType(tag: DW_TAG_typedef, name: "SFtype", line: 167, file: !46, scope: !7, baseType: !10)
-!10 = !MDBasicType(tag: DW_TAG_base_type, name: "float", size: 32, align: 32, encoding: DW_ATE_float)
-!11 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "b", line: 1921, arg: 0, scope: !1, file: !2, type: !9)
-!12 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "c", line: 1921, arg: 0, scope: !1, file: !2, type: !9)
-!13 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "d", line: 1921, arg: 0, scope: !1, file: !2, type: !9)
-!14 = !MDLocalVariable(tag: DW_TAG_auto_variable, name: "denom", line: 1923, scope: !15, file: !2, type: !9)
-!15 = distinct !MDLexicalBlock(line: 1922, column: 0, file: !45, scope: !1)
-!16 = !MDLocalVariable(tag: DW_TAG_auto_variable, name: "ratio", line: 1923, scope: !15, file: !2, type: !9)
-!17 = !MDLocalVariable(tag: DW_TAG_auto_variable, name: "x", line: 1923, scope: !15, file: !2, type: !9)
-!18 = !MDLocalVariable(tag: DW_TAG_auto_variable, name: "y", line: 1923, scope: !15, file: !2, type: !9)
-!19 = !MDLocation(line: 1929, scope: !15)
-!20 = !MDLocation(line: 1931, scope: !15)
-!21 = !MDLocation(line: 1932, scope: !15)
-!22 = !MDLocation(line: 1933, scope: !15)
-!23 = !MDLocation(line: 1934, scope: !15)
-!24 = !MDLocation(line: 1938, scope: !15)
-!25 = !MDLocation(line: 1939, scope: !15)
-!26 = !MDLocation(line: 1940, scope: !15)
-!27 = !MDLocation(line: 1941, scope: !15)
-!28 = !MDLocation(line: 1946, scope: !15)
-!29 = !MDLocation(line: 1948, scope: !15)
-!30 = !MDLocation(line: 1950, scope: !15)
-!31 = !MDLocation(line: 1951, scope: !15)
-!32 = !MDLocation(line: 1953, scope: !15)
-!33 = !MDLocation(line: 1955, scope: !15)
-!34 = !MDLocation(line: 1956, scope: !15)
-!35 = !MDLocation(line: 1957, scope: !15)
-!36 = !MDLocation(line: 1958, scope: !15)
-!37 = !MDLocation(line: 1960, scope: !15)
-!38 = !MDLocation(line: 1962, scope: !15)
-!39 = !MDLocation(line: 1963, scope: !15)
-!40 = !MDLocation(line: 1964, scope: !15)
-!41 = !MDLocation(line: 1965, scope: !15)
-!42 = !MDLocation(line: 1969, scope: !15)
+!6 = !DIDerivedType(tag: DW_TAG_typedef, name: "SCtype", line: 170, file: !46, scope: !7, baseType: !8)
+!7 = !DIFile(filename: "libgcc2.h", directory: "/Users/yash/clean/LG.D/gcc/../../llvmgcc/gcc")
+!8 = !DIBasicType(tag: DW_TAG_base_type, name: "complex float", size: 64, align: 32, encoding: DW_ATE_complex_float)
+!9 = !DIDerivedType(tag: DW_TAG_typedef, name: "SFtype", line: 167, file: !46, scope: !7, baseType: !10)
+!10 = !DIBasicType(tag: DW_TAG_base_type, name: "float", size: 32, align: 32, encoding: DW_ATE_float)
+!11 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "b", line: 1921, arg: 2, scope: !1, file: !2, type: !9)
+!12 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "c", line: 1921, arg: 3, scope: !1, file: !2, type: !9)
+!13 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "d", line: 1921, arg: 4, scope: !1, file: !2, type: !9)
+!14 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "denom", line: 1923, scope: !15, file: !2, type: !9)
+!15 = distinct !DILexicalBlock(line: 1922, column: 0, file: !45, scope: !1)
+!16 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "ratio", line: 1923, scope: !15, file: !2, type: !9)
+!17 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "x", line: 1923, scope: !15, file: !2, type: !9)
+!18 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "y", line: 1923, scope: !15, file: !2, type: !9)
+!19 = !DILocation(line: 1929, scope: !15)
+!20 = !DILocation(line: 1931, scope: !15)
+!21 = !DILocation(line: 1932, scope: !15)
+!22 = !DILocation(line: 1933, scope: !15)
+!23 = !DILocation(line: 1934, scope: !15)
+!24 = !DILocation(line: 1938, scope: !15)
+!25 = !DILocation(line: 1939, scope: !15)
+!26 = !DILocation(line: 1940, scope: !15)
+!27 = !DILocation(line: 1941, scope: !15)
+!28 = !DILocation(line: 1946, scope: !15)
+!29 = !DILocation(line: 1948, scope: !15)
+!30 = !DILocation(line: 1950, scope: !15)
+!31 = !DILocation(line: 1951, scope: !15)
+!32 = !DILocation(line: 1953, scope: !15)
+!33 = !DILocation(line: 1955, scope: !15)
+!34 = !DILocation(line: 1956, scope: !15)
+!35 = !DILocation(line: 1957, scope: !15)
+!36 = !DILocation(line: 1958, scope: !15)
+!37 = !DILocation(line: 1960, scope: !15)
+!38 = !DILocation(line: 1962, scope: !15)
+!39 = !DILocation(line: 1963, scope: !15)
+!40 = !DILocation(line: 1964, scope: !15)
+!41 = !DILocation(line: 1965, scope: !15)
+!42 = !DILocation(line: 1969, scope: !15)
 !43 = !{!0, !11, !12, !13, !14, !16, !17, !18}
 !44 = !{!1}
-!45 = !MDFile(filename: "libgcc2.c", directory: "/Users/yash/clean/LG.D/gcc/../../llvmgcc/gcc")
-!46 = !MDFile(filename: "libgcc2.h", directory: "/Users/yash/clean/LG.D/gcc/../../llvmgcc/gcc")
+!45 = !DIFile(filename: "libgcc2.c", directory: "/Users/yash/clean/LG.D/gcc/../../llvmgcc/gcc")
+!46 = !DIFile(filename: "libgcc2.h", directory: "/Users/yash/clean/LG.D/gcc/../../llvmgcc/gcc")
 !47 = !{}
 !48 = !{i32 1, !"Debug Info Version", i32 3}
diff --git a/test/CodeGen/X86/2010-05-26-DotDebugLoc.ll b/test/CodeGen/X86/2010-05-26-DotDebugLoc.ll
index fe68711e67d..a4b94d4b4e2 100644
--- a/test/CodeGen/X86/2010-05-26-DotDebugLoc.ll
+++ b/test/CodeGen/X86/2010-05-26-DotDebugLoc.ll
@@ -9,7 +9,7 @@ target triple = "x86_64-apple-darwin10"
 
 define i8* @bar(%struct.a* %myvar) nounwind optsize noinline ssp {
 entry:
-  tail call void @llvm.dbg.value(metadata %struct.a* %myvar, i64 0, metadata !8, metadata !MDExpression()), !dbg !MDLocation(scope: !9)
+  tail call void @llvm.dbg.value(metadata %struct.a* %myvar, i64 0, metadata !8, metadata !DIExpression()), !dbg !DILocation(scope: !9)
   %0 = getelementptr inbounds %struct.a, %struct.a* %myvar, i64 0, i32 0, !dbg !28 ; <i32*> [#uses=1]
   %1 = load i32, i32* %0, align 8, !dbg !28            ; <i32> [#uses=1]
   tail call void @foo(i32 %1) nounwind optsize noinline ssp, !dbg !28
@@ -24,43 +24,43 @@ declare void @llvm.dbg.value(metadata, i64, metadata, metadata) nounwind readnon
 !llvm.dbg.cu = !{!2}
 !llvm.module.flags = !{!38}
 
-!0 = !MDGlobalVariable(name: "ret", line: 7, isLocal: false, isDefinition: true, scope: !1, file: !1, type: !3)
-!1 = !MDFile(filename: "foo.c", directory: "/tmp/")
-!2 = !MDCompileUnit(language: DW_LANG_C89, producer: "4.2.1 (Based on Apple Inc. build 5658) (LLVM build)", isOptimized: true, emissionKind: 1, file: !36, enums: !37, retainedTypes: !37, subprograms: !32, globals: !31, imports:  !37)
-!3 = !MDBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
-!4 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "x", line: 12, arg: 0, scope: !5, file: !1, type: !3)
-!5 = !MDSubprogram(name: "foo", linkageName: "foo", line: 13, isLocal: false, isDefinition: true, virtualIndex: 6, isOptimized: true, scopeLine: 13, file: !36, scope: !1, type: !6, function: void (i32)* @foo, variables: !33)
-!6 = !MDSubroutineType(types: !7)
+!0 = !DIGlobalVariable(name: "ret", line: 7, isLocal: false, isDefinition: true, scope: !1, file: !1, type: !3)
+!1 = !DIFile(filename: "foo.c", directory: "/tmp/")
+!2 = !DICompileUnit(language: DW_LANG_C89, producer: "4.2.1 (Based on Apple Inc. build 5658) (LLVM build)", isOptimized: true, emissionKind: 1, file: !36, enums: !37, retainedTypes: !37, subprograms: !32, globals: !31, imports:  !37)
+!3 = !DIBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
+!4 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "x", line: 12, arg: 1, scope: !5, file: !1, type: !3)
+!5 = !DISubprogram(name: "foo", linkageName: "foo", line: 13, isLocal: false, isDefinition: true, virtualIndex: 6, isOptimized: true, scopeLine: 13, file: !36, scope: !1, type: !6, function: void (i32)* @foo, variables: !33)
+!6 = !DISubroutineType(types: !7)
 !7 = !{null, !3}
-!8 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "myvar", line: 17, arg: 0, scope: !9, file: !1, type: !13)
-!9 = !MDSubprogram(name: "bar", linkageName: "bar", line: 17, isLocal: false, isDefinition: true, virtualIndex: 6, isOptimized: true, scopeLine: 17, file: !36, scope: !1, type: !10, function: i8* (%struct.a*)* @bar, variables: !34)
-!10 = !MDSubroutineType(types: !11)
+!8 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "myvar", line: 17, arg: 1, scope: !9, file: !1, type: !13)
+!9 = !DISubprogram(name: "bar", linkageName: "bar", line: 17, isLocal: false, isDefinition: true, virtualIndex: 6, isOptimized: true, scopeLine: 17, file: !36, scope: !1, type: !10, function: i8* (%struct.a*)* @bar, variables: !34)
+!10 = !DISubroutineType(types: !11)
 !11 = !{!12, !13}
-!12 = !MDDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, file: !36, scope: !1, baseType: null)
-!13 = !MDDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, file: !36, scope: !1, baseType: !14)
-!14 = !MDCompositeType(tag: DW_TAG_structure_type, name: "a", line: 2, size: 128, align: 64, file: !36, scope: !1, elements: !15)
+!12 = !DIDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, file: !36, scope: !1, baseType: null)
+!13 = !DIDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, file: !36, scope: !1, baseType: !14)
+!14 = !DICompositeType(tag: DW_TAG_structure_type, name: "a", line: 2, size: 128, align: 64, file: !36, scope: !1, elements: !15)
 !15 = !{!16, !17}
-!16 = !MDDerivedType(tag: DW_TAG_member, name: "c", line: 3, size: 32, align: 32, file: !36, scope: !14, baseType: !3)
-!17 = !MDDerivedType(tag: DW_TAG_member, name: "d", line: 4, size: 64, align: 64, offset: 64, file: !36, scope: !14, baseType: !13)
-!18 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "argc", line: 22, arg: 0, scope: !19, file: !1, type: !3)
-!19 = !MDSubprogram(name: "main", linkageName: "main", line: 22, isLocal: false, isDefinition: true, virtualIndex: 6, isOptimized: true, scopeLine: 22, file: !36, scope: !1, type: !20, variables: !35)
-!20 = !MDSubroutineType(types: !21)
+!16 = !DIDerivedType(tag: DW_TAG_member, name: "c", line: 3, size: 32, align: 32, file: !36, scope: !14, baseType: !3)
+!17 = !DIDerivedType(tag: DW_TAG_member, name: "d", line: 4, size: 64, align: 64, offset: 64, file: !36, scope: !14, baseType: !13)
+!18 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "argc", line: 22, arg: 1, scope: !19, file: !1, type: !3)
+!19 = !DISubprogram(name: "main", linkageName: "main", line: 22, isLocal: false, isDefinition: true, virtualIndex: 6, isOptimized: true, scopeLine: 22, file: !36, scope: !1, type: !20, variables: !35)
+!20 = !DISubroutineType(types: !21)
 !21 = !{!3, !3, !22}
-!22 = !MDDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, file: !36, scope: !1, baseType: !23)
-!23 = !MDDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, file: !36, scope: !1, baseType: !24)
-!24 = !MDBasicType(tag: DW_TAG_base_type, name: "char", size: 8, align: 8, encoding: DW_ATE_signed_char)
-!25 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "argv", line: 22, arg: 0, scope: !19, file: !1, type: !22)
-!26 = !MDLocalVariable(tag: DW_TAG_auto_variable, name: "e", line: 23, scope: !27, file: !1, type: !14)
-!27 = distinct !MDLexicalBlock(line: 22, column: 0, file: !36, scope: !19)
-!28 = !MDLocation(line: 18, scope: !29)
-!29 = distinct !MDLexicalBlock(line: 17, column: 0, file: !36, scope: !9)
-!30 = !MDLocation(line: 19, scope: !29)
+!22 = !DIDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, file: !36, scope: !1, baseType: !23)
+!23 = !DIDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, file: !36, scope: !1, baseType: !24)
+!24 = !DIBasicType(tag: DW_TAG_base_type, name: "char", size: 8, align: 8, encoding: DW_ATE_signed_char)
+!25 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "argv", line: 22, arg: 2, scope: !19, file: !1, type: !22)
+!26 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "e", line: 23, scope: !27, file: !1, type: !14)
+!27 = distinct !DILexicalBlock(line: 22, column: 0, file: !36, scope: !19)
+!28 = !DILocation(line: 18, scope: !29)
+!29 = distinct !DILexicalBlock(line: 17, column: 0, file: !36, scope: !9)
+!30 = !DILocation(line: 19, scope: !29)
 !31 = !{!0}
 !32 = !{!5, !9, !19}
 !33 = !{!4}
 !34 = !{!8}
 !35 = !{!18, !25, !26}
-!36 = !MDFile(filename: "foo.c", directory: "/tmp/")
+!36 = !DIFile(filename: "foo.c", directory: "/tmp/")
 !37 = !{}
 
 ; The variable bar:myvar changes registers after the first movq.
@@ -77,18 +77,12 @@ declare void @llvm.dbg.value(metadata, i64, metadata, metadata) nounwind readnon
 ; CHECK-NEXT: .quad   [[SET1]]
 ; CHECK-NEXT: [[SET2:.*]] = [[LABEL]]-Lfunc_begin0
 ; CHECK-NEXT: .quad   [[SET2]]
-; CHECK-NEXT: Lset{{.*}} = Ltmp{{.*}}-Ltmp{{.*}}               ## Loc expr size
-; CHECK-NEXT: .short  Lset{{.*}}
-; CHECK-NEXT: Ltmp{{.*}}:
+; CHECK-NEXT: .short  1     ## Loc expr size
 ; CHECK-NEXT: .byte   85
-; CHECK-NEXT: Ltmp{{.*}}:
 ; CHECK-NEXT: [[SET3:.*]] = [[LABEL]]-Lfunc_begin0
 ; CHECK-NEXT: .quad   [[SET3]]
 ; CHECK-NEXT: [[SET4:.*]] = [[CLOBBER]]-Lfunc_begin0
 ; CHECK-NEXT: .quad   [[SET4]]
-; CHECK-NEXT: Lset{{.*}} = Ltmp{{.*}}-Ltmp{{.*}}               ## Loc expr size
-; CHECK-NEXT: .short  Lset{{.*}}
-; CHECK-NEXT: Ltmp{{.*}}:
+; CHECK-NEXT: .short  1     ## Loc expr size
 ; CHECK-NEXT: .byte   83
-; CHECK-NEXT: Ltmp{{.*}}:
 !38 = !{i32 1, !"Debug Info Version", i32 3}
diff --git a/test/CodeGen/X86/2010-05-28-Crash.ll b/test/CodeGen/X86/2010-05-28-Crash.ll
index 097cd24bbfc..6a6ee6a19aa 100644
--- a/test/CodeGen/X86/2010-05-28-Crash.ll
+++ b/test/CodeGen/X86/2010-05-28-Crash.ll
@@ -4,7 +4,7 @@
 
 define i32 @foo(i32 %y) nounwind optsize ssp {
 entry:
-  tail call void @llvm.dbg.value(metadata i32 %y, i64 0, metadata !0, metadata !MDExpression()), !dbg !MDLocation(scope: !1)
+  tail call void @llvm.dbg.value(metadata i32 %y, i64 0, metadata !0, metadata !DIExpression()), !dbg !DILocation(scope: !1)
   %0 = tail call i32 (...) @zoo(i32 %y) nounwind, !dbg !9 ; <i32> [#uses=1]
   ret i32 %0, !dbg !9
 }
@@ -15,8 +15,8 @@ declare void @llvm.dbg.value(metadata, i64, metadata, metadata) nounwind readnon
 
 define i32 @bar(i32 %x) nounwind optsize ssp {
 entry:
-  tail call void @llvm.dbg.value(metadata i32 %x, i64 0, metadata !7, metadata !MDExpression()), !dbg !MDLocation(scope: !8)
-  tail call void @llvm.dbg.value(metadata i32 1, i64 0, metadata !0, metadata !MDExpression()) nounwind, !dbg !MDLocation(scope: !1)
+  tail call void @llvm.dbg.value(metadata i32 %x, i64 0, metadata !7, metadata !DIExpression()), !dbg !DILocation(scope: !8)
+  tail call void @llvm.dbg.value(metadata i32 1, i64 0, metadata !0, metadata !DIExpression()) nounwind, !dbg !DILocation(scope: !1)
   %0 = tail call i32 (...) @zoo(i32 1) nounwind, !dbg !12 ; <i32> [#uses=1]
   %1 = add nsw i32 %0, %x, !dbg !13               ; <i32> [#uses=1]
   ret i32 %1, !dbg !13
@@ -25,25 +25,25 @@ entry:
 !llvm.dbg.cu = !{!3}
 !llvm.module.flags = !{!20}
 
-!0 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "y", line: 2, arg: 0, scope: !1, file: !2, type: !6)
-!1 = !MDSubprogram(name: "foo", linkageName: "foo", line: 2, isLocal: false, isDefinition: true, virtualIndex: 6, isOptimized: true, scopeLine: 2, file: !18, scope: !2, type: !4, function: i32 (i32)* @foo, variables: !15)
-!2 = !MDFile(filename: "f.c", directory: "/tmp")
-!3 = !MDCompileUnit(language: DW_LANG_C89, producer: "4.2.1 (Based on Apple Inc. build 5658) (LLVM build)", isOptimized: true, emissionKind: 1, file: !18, enums: !19, retainedTypes: !19, subprograms: !17, imports:  null)
-!4 = !MDSubroutineType(types: !5)
+!0 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "y", line: 2, arg: 1, scope: !1, file: !2, type: !6)
+!1 = !DISubprogram(name: "foo", linkageName: "foo", line: 2, isLocal: false, isDefinition: true, virtualIndex: 6, isOptimized: true, scopeLine: 2, file: !18, scope: !2, type: !4, function: i32 (i32)* @foo, variables: !15)
+!2 = !DIFile(filename: "f.c", directory: "/tmp")
+!3 = !DICompileUnit(language: DW_LANG_C89, producer: "4.2.1 (Based on Apple Inc. build 5658) (LLVM build)", isOptimized: true, emissionKind: 1, file: !18, enums: !19, retainedTypes: !19, subprograms: !17, imports:  null)
+!4 = !DISubroutineType(types: !5)
 !5 = !{!6, !6}
-!6 = !MDBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
-!7 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "x", line: 6, arg: 0, scope: !8, file: !2, type: !6)
-!8 = !MDSubprogram(name: "bar", linkageName: "bar", line: 6, isLocal: false, isDefinition: true, virtualIndex: 6, isOptimized: true, scopeLine: 6, file: !18, scope: !2, type: !4, function: i32 (i32)* @bar, variables: !16)
-!9 = !MDLocation(line: 3, scope: !10)
-!10 = distinct !MDLexicalBlock(line: 2, column: 0, file: !18, scope: !1)
+!6 = !DIBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
+!7 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "x", line: 6, arg: 1, scope: !8, file: !2, type: !6)
+!8 = !DISubprogram(name: "bar", linkageName: "bar", line: 6, isLocal: false, isDefinition: true, virtualIndex: 6, isOptimized: true, scopeLine: 6, file: !18, scope: !2, type: !4, function: i32 (i32)* @bar, variables: !16)
+!9 = !DILocation(line: 3, scope: !10)
+!10 = distinct !DILexicalBlock(line: 2, column: 0, file: !18, scope: !1)
 !11 = !{i32 1}
-!12 = !MDLocation(line: 3, scope: !10, inlinedAt: !13)
-!13 = !MDLocation(line: 7, scope: !14)
-!14 = distinct !MDLexicalBlock(line: 6, column: 0, file: !18, scope: !8)
+!12 = !DILocation(line: 3, scope: !10, inlinedAt: !13)
+!13 = !DILocation(line: 7, scope: !14)
+!14 = distinct !DILexicalBlock(line: 6, column: 0, file: !18, scope: !8)
 !15 = !{!0}
 !16 = !{!7}
 !17 = !{!1, !8}
-!18 = !MDFile(filename: "f.c", directory: "/tmp")
+!18 = !DIFile(filename: "f.c", directory: "/tmp")
 !19 = !{}
 
 ;CHECK: DEBUG_VALUE: bar:x <- E
diff --git a/test/CodeGen/X86/2010-06-01-DeadArg-DbgInfo.ll b/test/CodeGen/X86/2010-06-01-DeadArg-DbgInfo.ll
index 942faf44abb..be985b8779e 100644
--- a/test/CodeGen/X86/2010-06-01-DeadArg-DbgInfo.ll
+++ b/test/CodeGen/X86/2010-06-01-DeadArg-DbgInfo.ll
@@ -10,8 +10,8 @@ target triple = "x86_64-apple-darwin10.2"
 define i32 @_ZN3foo3bazEi(%struct.foo* nocapture %this, i32 %x) nounwind readnone optsize noinline ssp align 2 {
 ;CHECK: DEBUG_VALUE: baz:this <- RDI{{$}}
 entry:
-  tail call void @llvm.dbg.value(metadata %struct.foo* %this, i64 0, metadata !15, metadata !MDExpression()), !dbg !MDLocation(scope: !8)
-  tail call void @llvm.dbg.value(metadata i32 %x, i64 0, metadata !16, metadata !MDExpression()), !dbg !MDLocation(scope: !8)
+  tail call void @llvm.dbg.value(metadata %struct.foo* %this, i64 0, metadata !15, metadata !DIExpression()), !dbg !DILocation(scope: !8)
+  tail call void @llvm.dbg.value(metadata i32 %x, i64 0, metadata !16, metadata !DIExpression()), !dbg !DILocation(scope: !8)
   %0 = mul nsw i32 %x, 7, !dbg !29                ; <i32> [#uses=1]
   %1 = add nsw i32 %0, 1, !dbg !29                ; <i32> [#uses=1]
   ret i32 %1, !dbg !29
@@ -23,38 +23,38 @@ declare void @llvm.dbg.value(metadata, i64, metadata, metadata) nounwind readnon
 !llvm.module.flags = !{!34}
 !llvm.dbg.lv = !{!0, !14, !15, !16, !17, !24, !25, !28}
 
-!0 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "this", line: 11, arg: 0, scope: !1, file: !3, type: !12)
-!1 = !MDSubprogram(name: "bar", linkageName: "_ZN3foo3barEi", line: 11, isLocal: false, isDefinition: true, virtualIndex: 6, isOptimized: true, scopeLine: 11, file: !31, scope: !2, type: !9, function: i32 (%struct.foo*, i32)* null)
-!2 = !MDCompositeType(tag: DW_TAG_structure_type, name: "foo", line: 3, size: 32, align: 32, file: !31, scope: !3, elements: !5)
-!3 = !MDFile(filename: "foo.cp", directory: "/tmp/")
-!4 = !MDCompileUnit(language: DW_LANG_C_plus_plus, producer: "4.2.1 LLVM build", isOptimized: true, emissionKind: 0, file: !31, enums: !32, retainedTypes: !32, subprograms: !33)
+!0 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "this", line: 11, arg: 1, scope: !1, file: !3, type: !12)
+!1 = !DISubprogram(name: "bar", linkageName: "_ZN3foo3barEi", line: 11, isLocal: false, isDefinition: true, virtualIndex: 6, isOptimized: true, scopeLine: 11, file: !31, scope: !2, type: !9, function: i32 (%struct.foo*, i32)* null)
+!2 = !DICompositeType(tag: DW_TAG_structure_type, name: "foo", line: 3, size: 32, align: 32, file: !31, scope: !3, elements: !5)
+!3 = !DIFile(filename: "foo.cp", directory: "/tmp/")
+!4 = !DICompileUnit(language: DW_LANG_C_plus_plus, producer: "4.2.1 LLVM build", isOptimized: true, emissionKind: 0, file: !31, enums: !32, retainedTypes: !32, subprograms: !33)
 !5 = !{!6, !1, !8}
-!6 = !MDDerivedType(tag: DW_TAG_member, name: "y", line: 8, size: 32, align: 32, file: !31, scope: !2, baseType: !7)
-!7 = !MDBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
-!8 = !MDSubprogram(name: "baz", linkageName: "_ZN3foo3bazEi", line: 15, isLocal: false, isDefinition: true, virtualIndex: 6, isOptimized: true, scopeLine: 15, file: !31, scope: !2, type: !9, function: i32 (%struct.foo*, i32)* @_ZN3foo3bazEi)
-!9 = !MDSubroutineType(types: !10)
+!6 = !DIDerivedType(tag: DW_TAG_member, name: "y", line: 8, size: 32, align: 32, file: !31, scope: !2, baseType: !7)
+!7 = !DIBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
+!8 = !DISubprogram(name: "baz", linkageName: "_ZN3foo3bazEi", line: 15, isLocal: false, isDefinition: true, virtualIndex: 6, isOptimized: true, scopeLine: 15, file: !31, scope: !2, type: !9, function: i32 (%struct.foo*, i32)* @_ZN3foo3bazEi)
+!9 = !DISubroutineType(types: !10)
 !10 = !{!7, !11, !7}
-!11 = !MDDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, flags: DIFlagArtificial, file: !31, scope: !3, baseType: !2)
-!12 = !MDDerivedType(tag: DW_TAG_const_type, size: 64, align: 64, flags: DIFlagArtificial, file: !31, scope: !3, baseType: !13)
-!13 = !MDDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, file: !31, scope: !3, baseType: !2)
-!14 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "x", line: 11, arg: 0, scope: !1, file: !3, type: !7)
-!15 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "this", line: 15, arg: 0, scope: !8, file: !3, type: !12)
-!16 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "x", line: 15, arg: 0, scope: !8, file: !3, type: !7)
-!17 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "argc", line: 19, arg: 0, scope: !18, file: !3, type: !7)
-!18 = !MDSubprogram(name: "main", linkageName: "main", line: 19, isLocal: false, isDefinition: true, virtualIndex: 6, isOptimized: true, scopeLine: 19, file: !31, scope: !3, type: !19)
-!19 = !MDSubroutineType(types: !20)
+!11 = !DIDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, flags: DIFlagArtificial, file: !31, scope: !3, baseType: !2)
+!12 = !DIDerivedType(tag: DW_TAG_const_type, size: 64, align: 64, flags: DIFlagArtificial, file: !31, scope: !3, baseType: !13)
+!13 = !DIDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, file: !31, scope: !3, baseType: !2)
+!14 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "x", line: 11, arg: 2, scope: !1, file: !3, type: !7)
+!15 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "this", line: 15, arg: 1, scope: !8, file: !3, type: !12)
+!16 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "x", line: 15, arg: 2, scope: !8, file: !3, type: !7)
+!17 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "argc", line: 19, arg: 1, scope: !18, file: !3, type: !7)
+!18 = !DISubprogram(name: "main", linkageName: "main", line: 19, isLocal: false, isDefinition: true, virtualIndex: 6, isOptimized: true, scopeLine: 19, file: !31, scope: !3, type: !19)
+!19 = !DISubroutineType(types: !20)
 !20 = !{!7, !7, !21}
-!21 = !MDDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, file: !31, scope: !3, baseType: !22)
-!22 = !MDDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, file: !31, scope: !3, baseType: !23)
-!23 = !MDBasicType(tag: DW_TAG_base_type, name: "char", size: 8, align: 8, encoding: DW_ATE_signed_char)
-!24 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "argv", line: 19, arg: 0, scope: !18, file: !3, type: !21)
-!25 = !MDLocalVariable(tag: DW_TAG_auto_variable, name: "a", line: 20, scope: !26, file: !3, type: !2)
-!26 = distinct !MDLexicalBlock(line: 19, column: 0, file: !31, scope: !27)
-!27 = distinct !MDLexicalBlock(line: 19, column: 0, file: !31, scope: !18)
-!28 = !MDLocalVariable(tag: DW_TAG_auto_variable, name: "b", line: 21, scope: !26, file: !3, type: !7)
-!29 = !MDLocation(line: 16, scope: !30)
-!30 = distinct !MDLexicalBlock(line: 15, column: 0, file: !31, scope: !8)
-!31 = !MDFile(filename: "foo.cp", directory: "/tmp/")
+!21 = !DIDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, file: !31, scope: !3, baseType: !22)
+!22 = !DIDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, file: !31, scope: !3, baseType: !23)
+!23 = !DIBasicType(tag: DW_TAG_base_type, name: "char", size: 8, align: 8, encoding: DW_ATE_signed_char)
+!24 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "argv", line: 19, arg: 2, scope: !18, file: !3, type: !21)
+!25 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "a", line: 20, scope: !26, file: !3, type: !2)
+!26 = distinct !DILexicalBlock(line: 19, column: 0, file: !31, scope: !27)
+!27 = distinct !DILexicalBlock(line: 19, column: 0, file: !31, scope: !18)
+!28 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "b", line: 21, scope: !26, file: !3, type: !7)
+!29 = !DILocation(line: 16, scope: !30)
+!30 = distinct !DILexicalBlock(line: 15, column: 0, file: !31, scope: !8)
+!31 = !DIFile(filename: "foo.cp", directory: "/tmp/")
 !32 = !{}
 !33 = !{!1, !8, !18}
 !34 = !{i32 1, !"Debug Info Version", i32 3}
diff --git a/test/CodeGen/X86/2010-07-06-DbgCrash.ll b/test/CodeGen/X86/2010-07-06-DbgCrash.ll
index fabd8260662..3ce36eec400 100644
--- a/test/CodeGen/X86/2010-07-06-DbgCrash.ll
+++ b/test/CodeGen/X86/2010-07-06-DbgCrash.ll
@@ -3,27 +3,27 @@
 @.str = private constant [4 x i8] c"one\00", align 1 ; <[4 x i8]*> [#uses=1]
 @.str1 = private constant [4 x i8] c"two\00", align 1 ; <[5 x i8]*> [#uses=1]
 @C.9.2167 = internal constant [2 x i8*] [i8* getelementptr inbounds ([4 x i8], [4 x i8]* @.str, i64 0, i64 0), i8* getelementptr inbounds ([4 x i8], [4 x i8]* @.str1, i64 0, i64 0)]
-!38 = !MDFile(filename: "pbmsrch.c", directory: "/Users/grawp/LLVM/test-suite/MultiSource/Benchmarks/MiBench/office-stringsearch")
-!39 = !MDCompileUnit(language: DW_LANG_C89, producer: "4.2.1 (Based on Apple Inc. build 5658) (LLVM build 9999)", isOptimized: true, emissionKind: 0, file: !109, enums: !108, retainedTypes: !108)
-!46 = !MDDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, file: !109, baseType: !47)
-!47 = !MDBasicType(tag: DW_TAG_base_type, name: "char", size: 8, align: 8, encoding: DW_ATE_signed_char)
-!97 = !MDSubprogram(name: "main", linkageName: "main", line: 73, isLocal: false, isDefinition: true, virtualIndex: 6, isOptimized: false, scope: !39, type: !98)
-!98 = !MDSubroutineType(types: !99)
+!38 = !DIFile(filename: "pbmsrch.c", directory: "/Users/grawp/LLVM/test-suite/MultiSource/Benchmarks/MiBench/office-stringsearch")
+!39 = !DICompileUnit(language: DW_LANG_C89, producer: "4.2.1 (Based on Apple Inc. build 5658) (LLVM build 9999)", isOptimized: true, emissionKind: 0, file: !109, enums: !108, retainedTypes: !108)
+!46 = !DIDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, file: !109, baseType: !47)
+!47 = !DIBasicType(tag: DW_TAG_base_type, name: "char", size: 8, align: 8, encoding: DW_ATE_signed_char)
+!97 = !DISubprogram(name: "main", linkageName: "main", line: 73, isLocal: false, isDefinition: true, virtualIndex: 6, isOptimized: false, scope: !39, type: !98)
+!98 = !DISubroutineType(types: !99)
 !99 = !{!100}
-!100 = !MDBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
+!100 = !DIBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
 !101 = !{[2 x i8*]* @C.9.2167}
-!102 = !MDLocalVariable(tag: DW_TAG_auto_variable, name: "find_strings", line: 75, scope: !103, file: !38, type: !104)
-!103 = distinct !MDLexicalBlock(line: 73, column: 0, file: null, scope: !97)
-!104 = !MDCompositeType(tag: DW_TAG_array_type, size: 85312, align: 64, file: !109, baseType: !46, elements: !105)
+!102 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "find_strings", line: 75, scope: !103, file: !38, type: !104)
+!103 = distinct !DILexicalBlock(line: 73, column: 0, file: null, scope: !97)
+!104 = !DICompositeType(tag: DW_TAG_array_type, size: 85312, align: 64, file: !109, baseType: !46, elements: !105)
 !105 = !{!106}
-!106 = !MDSubrange(count: 1333)
-!107 = !MDLocation(line: 73, scope: !103)
+!106 = !DISubrange(count: 1333)
+!107 = !DILocation(line: 73, scope: !103)
 !108 = !{i32 0}
-!109 = !MDFile(filename: "pbmsrch.c", directory: "/Users/grawp/LLVM/test-suite/MultiSource/Benchmarks/MiBench/office-stringsearch")
+!109 = !DIFile(filename: "pbmsrch.c", directory: "/Users/grawp/LLVM/test-suite/MultiSource/Benchmarks/MiBench/office-stringsearch")
 
 define i32 @main() nounwind ssp {
 bb.nph:
-  tail call void @llvm.dbg.declare(metadata [2 x i8*]* @C.9.2167, metadata !102, metadata !MDExpression()), !dbg !107
+  tail call void @llvm.dbg.declare(metadata [2 x i8*]* @C.9.2167, metadata !102, metadata !DIExpression()), !dbg !107
   ret i32 0, !dbg !107
 }
 
diff --git a/test/CodeGen/X86/2010-08-04-MingWCrash.ll b/test/CodeGen/X86/2010-08-04-MingWCrash.ll
index 61f527b0470..e97615a417a 100644
--- a/test/CodeGen/X86/2010-08-04-MingWCrash.ll
+++ b/test/CodeGen/X86/2010-08-04-MingWCrash.ll
@@ -1,6 +1,6 @@
 ; RUN: llc < %s -mtriple=i386-pc-mingw32
 
-define void @func() nounwind {
+define void @func() nounwind personality i8* bitcast (i32 (...)* @__gxx_personality_v0 to i8*) {
 invoke.cont:
   %call = tail call i8* @malloc()
   %a = invoke i32 @bar()
@@ -10,7 +10,7 @@ bb1:
   ret void
 
 lpad:
-  %exn.ptr = landingpad { i8*, i32 } personality i8* bitcast (i32 (...)* @__gxx_personality_v0 to i8*)
+  %exn.ptr = landingpad { i8*, i32 }
            catch i8* null
   %exn = extractvalue { i8*, i32 } %exn.ptr, 0
   %eh.selector = extractvalue { i8*, i32 } %exn.ptr, 1
diff --git a/test/CodeGen/X86/2010-08-04-StackVariable.ll b/test/CodeGen/X86/2010-08-04-StackVariable.ll
index d3ad8608a8b..e2a2884e842 100644
--- a/test/CodeGen/X86/2010-08-04-StackVariable.ll
+++ b/test/CodeGen/X86/2010-08-04-StackVariable.ll
@@ -6,8 +6,8 @@
 define i32 @_Z3fooi4SVal(i32 %i, %struct.SVal* noalias %location) nounwind ssp {
 entry:
   %"alloca point" = bitcast i32 0 to i32          ; <i32> [#uses=0]
-  call void @llvm.dbg.value(metadata i32 %i, i64 0, metadata !23, metadata !MDExpression()), !dbg !24
-  call void @llvm.dbg.value(metadata %struct.SVal* %location, i64 0, metadata !25, metadata !MDExpression()), !dbg !24
+  call void @llvm.dbg.value(metadata i32 %i, i64 0, metadata !23, metadata !DIExpression()), !dbg !24
+  call void @llvm.dbg.value(metadata %struct.SVal* %location, i64 0, metadata !25, metadata !DIExpression()), !dbg !24
   %0 = icmp ne i32 %i, 0, !dbg !27                ; <i1> [#uses=1]
   br i1 %0, label %bb, label %bb1, !dbg !27
 
@@ -34,7 +34,7 @@ return:                                           ; preds = %bb2
 define linkonce_odr void @_ZN4SValC1Ev(%struct.SVal* %this) nounwind ssp align 2 {
 entry:
   %"alloca point" = bitcast i32 0 to i32          ; <i32> [#uses=0]
-  call void @llvm.dbg.value(metadata %struct.SVal* %this, i64 0, metadata !31, metadata !MDExpression()), !dbg !34
+  call void @llvm.dbg.value(metadata %struct.SVal* %this, i64 0, metadata !31, metadata !DIExpression()), !dbg !34
   %0 = getelementptr inbounds %struct.SVal, %struct.SVal* %this, i32 0, i32 0, !dbg !34 ; <i8**> [#uses=1]
   store i8* null, i8** %0, align 8, !dbg !34
   %1 = getelementptr inbounds %struct.SVal, %struct.SVal* %this, i32 0, i32 1, !dbg !34 ; <i32*> [#uses=1]
@@ -52,7 +52,7 @@ entry:
   %0 = alloca %struct.SVal                        ; <%struct.SVal*> [#uses=3]
   %v = alloca %struct.SVal                        ; <%struct.SVal*> [#uses=4]
   %"alloca point" = bitcast i32 0 to i32          ; <i32> [#uses=0]
-  call void @llvm.dbg.declare(metadata %struct.SVal* %v, metadata !38, metadata !MDExpression()), !dbg !41
+  call void @llvm.dbg.declare(metadata %struct.SVal* %v, metadata !38, metadata !DIExpression()), !dbg !41
   call void @_ZN4SValC1Ev(%struct.SVal* %v) nounwind, !dbg !41
   %1 = getelementptr inbounds %struct.SVal, %struct.SVal* %v, i32 0, i32 1, !dbg !42 ; <i32*> [#uses=1]
   store i32 1, i32* %1, align 8, !dbg !42
@@ -65,7 +65,7 @@ entry:
   %7 = load i32, i32* %6, align 8, !dbg !43            ; <i32> [#uses=1]
   store i32 %7, i32* %5, align 8, !dbg !43
   %8 = call i32 @_Z3fooi4SVal(i32 2, %struct.SVal* noalias %0) nounwind, !dbg !43 ; <i32> [#uses=0]
-  call void @llvm.dbg.value(metadata i32 %8, i64 0, metadata !44, metadata !MDExpression()), !dbg !43
+  call void @llvm.dbg.value(metadata i32 %8, i64 0, metadata !44, metadata !DIExpression()), !dbg !43
   br label %return, !dbg !45
 
 return:                                           ; preds = %entry
@@ -78,52 +78,52 @@ declare void @llvm.dbg.value(metadata, i64, metadata, metadata) nounwind readnon
 !llvm.module.flags = !{!49}
 !46 = !{!16, !17, !20}
 
-!0 = !MDSubprogram(name: "SVal", line: 11, isLocal: false, isDefinition: false, virtualIndex: 6, isOptimized: false, scopeLine: 11, file: !47, scope: !1, type: !14)
-!1 = !MDCompositeType(tag: DW_TAG_structure_type, name: "SVal", line: 1, size: 128, align: 64, file: !47, scope: !2, elements: !4)
-!2 = !MDFile(filename: "small.cc", directory: "/Users/manav/R8248330")
-!3 = !MDCompileUnit(language: DW_LANG_C_plus_plus, producer: "4.2.1 (Based on Apple Inc. build 5658) (LLVM build)", isOptimized: false, emissionKind: 1, file: !47, enums: !48, retainedTypes: !48, subprograms: !46, imports:  null)
+!0 = !DISubprogram(name: "SVal", line: 11, isLocal: false, isDefinition: false, virtualIndex: 6, isOptimized: false, scopeLine: 11, file: !47, scope: !1, type: !14)
+!1 = !DICompositeType(tag: DW_TAG_structure_type, name: "SVal", line: 1, size: 128, align: 64, file: !47, scope: !2, elements: !4)
+!2 = !DIFile(filename: "small.cc", directory: "/Users/manav/R8248330")
+!3 = !DICompileUnit(language: DW_LANG_C_plus_plus, producer: "4.2.1 (Based on Apple Inc. build 5658) (LLVM build)", isOptimized: false, emissionKind: 1, file: !47, enums: !48, retainedTypes: !48, subprograms: !46, imports:  null)
 !4 = !{!5, !7, !0, !9}
-!5 = !MDDerivedType(tag: DW_TAG_member, name: "Data", line: 7, size: 64, align: 64, file: !47, scope: !1, baseType: !6)
-!6 = !MDDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, file: !47, scope: !2, baseType: null)
-!7 = !MDDerivedType(tag: DW_TAG_member, name: "Kind", line: 8, size: 32, align: 32, offset: 64, file: !47, scope: !1, baseType: !8)
-!8 = !MDBasicType(tag: DW_TAG_base_type, name: "unsigned int", size: 32, align: 32, encoding: DW_ATE_unsigned)
-!9 = !MDSubprogram(name: "~SVal", line: 12, isLocal: false, isDefinition: false, virtualIndex: 6, isOptimized: false, scopeLine: 12, file: !47, scope: !1, type: !10)
-!10 = !MDSubroutineType(types: !11)
+!5 = !DIDerivedType(tag: DW_TAG_member, name: "Data", line: 7, size: 64, align: 64, file: !47, scope: !1, baseType: !6)
+!6 = !DIDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, file: !47, scope: !2, baseType: null)
+!7 = !DIDerivedType(tag: DW_TAG_member, name: "Kind", line: 8, size: 32, align: 32, offset: 64, file: !47, scope: !1, baseType: !8)
+!8 = !DIBasicType(tag: DW_TAG_base_type, name: "unsigned int", size: 32, align: 32, encoding: DW_ATE_unsigned)
+!9 = !DISubprogram(name: "~SVal", line: 12, isLocal: false, isDefinition: false, virtualIndex: 6, isOptimized: false, scopeLine: 12, file: !47, scope: !1, type: !10)
+!10 = !DISubroutineType(types: !11)
 !11 = !{null, !12, !13}
-!12 = !MDDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, flags: DIFlagArtificial, file: !47, scope: !2, baseType: !1)
-!13 = !MDBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
-!14 = !MDSubroutineType(types: !15)
+!12 = !DIDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, flags: DIFlagArtificial, file: !47, scope: !2, baseType: !1)
+!13 = !DIBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
+!14 = !DISubroutineType(types: !15)
 !15 = !{null, !12}
-!16 = !MDSubprogram(name: "SVal", linkageName: "_ZN4SValC1Ev", line: 11, isLocal: false, isDefinition: true, virtualIndex: 6, isOptimized: false, scopeLine: 11, file: !47, scope: !1, type: !14, function: void (%struct.SVal*)* @_ZN4SValC1Ev)
-!17 = !MDSubprogram(name: "foo", linkageName: "_Z3fooi4SVal", line: 16, isLocal: false, isDefinition: true, virtualIndex: 6, isOptimized: false, scopeLine: 16, file: !47, scope: !2, type: !18, function: i32 (i32, %struct.SVal*)* @_Z3fooi4SVal)
-!18 = !MDSubroutineType(types: !19)
+!16 = !DISubprogram(name: "SVal", linkageName: "_ZN4SValC1Ev", line: 11, isLocal: false, isDefinition: true, virtualIndex: 6, isOptimized: false, scopeLine: 11, file: !47, scope: !1, type: !14, function: void (%struct.SVal*)* @_ZN4SValC1Ev)
+!17 = !DISubprogram(name: "foo", linkageName: "_Z3fooi4SVal", line: 16, isLocal: false, isDefinition: true, virtualIndex: 6, isOptimized: false, scopeLine: 16, file: !47, scope: !2, type: !18, function: i32 (i32, %struct.SVal*)* @_Z3fooi4SVal)
+!18 = !DISubroutineType(types: !19)
 !19 = !{!13, !13, !1}
-!20 = !MDSubprogram(name: "main", linkageName: "main", line: 23, isLocal: false, isDefinition: true, virtualIndex: 6, isOptimized: false, scopeLine: 23, file: !47, scope: !2, type: !21, function: i32 ()* @main)
-!21 = !MDSubroutineType(types: !22)
+!20 = !DISubprogram(name: "main", linkageName: "main", line: 23, isLocal: false, isDefinition: true, virtualIndex: 6, isOptimized: false, scopeLine: 23, file: !47, scope: !2, type: !21, function: i32 ()* @main)
+!21 = !DISubroutineType(types: !22)
 !22 = !{!13}
-!23 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "i", line: 16, arg: 0, scope: !17, file: !2, type: !13)
-!24 = !MDLocation(line: 16, scope: !17)
-!25 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "location", line: 16, arg: 0, scope: !17, file: !2, type: !26)
-!26 = !MDDerivedType(tag: DW_TAG_reference_type, name: "SVal", size: 64, align: 64, file: !47, scope: !2, baseType: !1)
-!27 = !MDLocation(line: 17, scope: !28)
-!28 = distinct !MDLexicalBlock(line: 16, column: 0, file: !47, scope: !17)
-!29 = !MDLocation(line: 18, scope: !28)
-!30 = !MDLocation(line: 20, scope: !28)
-!31 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "this", line: 11, arg: 0, scope: !16, file: !2, type: !32)
-!32 = !MDDerivedType(tag: DW_TAG_const_type, size: 64, align: 64, flags: DIFlagArtificial, file: !47, scope: !2, baseType: !33)
-!33 = !MDDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, file: !47, scope: !2, baseType: !1)
-!34 = !MDLocation(line: 11, scope: !16)
-!35 = !MDLocation(line: 11, scope: !36)
-!36 = distinct !MDLexicalBlock(line: 11, column: 0, file: !47, scope: !37)
-!37 = distinct !MDLexicalBlock(line: 11, column: 0, file: !47, scope: !16)
-!38 = !MDLocalVariable(tag: DW_TAG_auto_variable, name: "v", line: 24, scope: !39, file: !2, type: !1)
-!39 = distinct !MDLexicalBlock(line: 23, column: 0, file: !47, scope: !40)
-!40 = distinct !MDLexicalBlock(line: 23, column: 0, file: !47, scope: !20)
-!41 = !MDLocation(line: 24, scope: !39)
-!42 = !MDLocation(line: 25, scope: !39)
-!43 = !MDLocation(line: 26, scope: !39)
-!44 = !MDLocalVariable(tag: DW_TAG_auto_variable, name: "k", line: 26, scope: !39, file: !2, type: !13)
-!45 = !MDLocation(line: 27, scope: !39)
-!47 = !MDFile(filename: "small.cc", directory: "/Users/manav/R8248330")
+!23 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "i", line: 16, arg: 1, scope: !17, file: !2, type: !13)
+!24 = !DILocation(line: 16, scope: !17)
+!25 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "location", line: 16, arg: 2, scope: !17, file: !2, type: !26)
+!26 = !DIDerivedType(tag: DW_TAG_reference_type, name: "SVal", size: 64, align: 64, file: !47, scope: !2, baseType: !1)
+!27 = !DILocation(line: 17, scope: !28)
+!28 = distinct !DILexicalBlock(line: 16, column: 0, file: !47, scope: !17)
+!29 = !DILocation(line: 18, scope: !28)
+!30 = !DILocation(line: 20, scope: !28)
+!31 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "this", line: 11, arg: 1, scope: !16, file: !2, type: !32)
+!32 = !DIDerivedType(tag: DW_TAG_const_type, size: 64, align: 64, flags: DIFlagArtificial, file: !47, scope: !2, baseType: !33)
+!33 = !DIDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, file: !47, scope: !2, baseType: !1)
+!34 = !DILocation(line: 11, scope: !16)
+!35 = !DILocation(line: 11, scope: !36)
+!36 = distinct !DILexicalBlock(line: 11, column: 0, file: !47, scope: !37)
+!37 = distinct !DILexicalBlock(line: 11, column: 0, file: !47, scope: !16)
+!38 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "v", line: 24, scope: !39, file: !2, type: !1)
+!39 = distinct !DILexicalBlock(line: 23, column: 0, file: !47, scope: !40)
+!40 = distinct !DILexicalBlock(line: 23, column: 0, file: !47, scope: !20)
+!41 = !DILocation(line: 24, scope: !39)
+!42 = !DILocation(line: 25, scope: !39)
+!43 = !DILocation(line: 26, scope: !39)
+!44 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "k", line: 26, scope: !39, file: !2, type: !13)
+!45 = !DILocation(line: 27, scope: !39)
+!47 = !DIFile(filename: "small.cc", directory: "/Users/manav/R8248330")
 !48 = !{}
 !49 = !{i32 1, !"Debug Info Version", i32 3}
diff --git a/test/CodeGen/X86/2010-09-16-EmptyFilename.ll b/test/CodeGen/X86/2010-09-16-EmptyFilename.ll
index 30abdc57124..d94bd1c79f9 100644
--- a/test/CodeGen/X86/2010-09-16-EmptyFilename.ll
+++ b/test/CodeGen/X86/2010-09-16-EmptyFilename.ll
@@ -15,21 +15,21 @@ entry:
 !llvm.dbg.cu = !{!2}
 !llvm.module.flags = !{!17}
 
-!0 = !MDSubprogram(name: "foo", linkageName: "foo", line: 53, isLocal: false, isDefinition: true, virtualIndex: 6, isOptimized: false, file: !14, scope: !1, type: !3, function: i32 ()* @foo)
-!1 = !MDFile(filename: "", directory: "/private/tmp")
-!2 = !MDCompileUnit(language: DW_LANG_C99, producer: "clang version 2.9 (trunk 114084)", isOptimized: false, emissionKind: 0, file: !15, enums: !16, retainedTypes: !16, subprograms: !13)
-!3 = !MDSubroutineType(types: !4)
+!0 = !DISubprogram(name: "foo", linkageName: "foo", line: 53, isLocal: false, isDefinition: true, virtualIndex: 6, isOptimized: false, file: !14, scope: !1, type: !3, function: i32 ()* @foo)
+!1 = !DIFile(filename: "", directory: "/private/tmp")
+!2 = !DICompileUnit(language: DW_LANG_C99, producer: "clang version 2.9 (trunk 114084)", isOptimized: false, emissionKind: 0, file: !15, enums: !16, retainedTypes: !16, subprograms: !13)
+!3 = !DISubroutineType(types: !4)
 !4 = !{!5}
-!5 = !MDBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
-!6 = !MDSubprogram(name: "bar", linkageName: "bar", line: 4, isLocal: false, isDefinition: true, virtualIndex: 6, isOptimized: false, file: !15, scope: !7, type: !3, function: i32 ()* @bar)
-!7 = !MDFile(filename: "bug.c", directory: "/private/tmp")
-!8 = !MDLocation(line: 53, column: 13, scope: !9)
-!9 = distinct !MDLexicalBlock(line: 53, column: 11, file: !14, scope: !0)
-!10 = !MDLocation(line: 4, column: 13, scope: !11)
-!11 = distinct !MDLexicalBlock(line: 4, column: 13, file: !15, scope: !12)
-!12 = distinct !MDLexicalBlock(line: 4, column: 11, file: !15, scope: !6)
+!5 = !DIBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
+!6 = !DISubprogram(name: "bar", linkageName: "bar", line: 4, isLocal: false, isDefinition: true, virtualIndex: 6, isOptimized: false, file: !15, scope: !7, type: !3, function: i32 ()* @bar)
+!7 = !DIFile(filename: "bug.c", directory: "/private/tmp")
+!8 = !DILocation(line: 53, column: 13, scope: !9)
+!9 = distinct !DILexicalBlock(line: 53, column: 11, file: !14, scope: !0)
+!10 = !DILocation(line: 4, column: 13, scope: !11)
+!11 = distinct !DILexicalBlock(line: 4, column: 13, file: !15, scope: !12)
+!12 = distinct !DILexicalBlock(line: 4, column: 11, file: !15, scope: !6)
 !13 = !{!0, !6}
-!14 = !MDFile(filename: "", directory: "/private/tmp")
-!15 = !MDFile(filename: "bug.c", directory: "/private/tmp")
+!14 = !DIFile(filename: "", directory: "/private/tmp")
+!15 = !DIFile(filename: "bug.c", directory: "/private/tmp")
 !16 = !{}
 !17 = !{i32 1, !"Debug Info Version", i32 3}
diff --git a/test/CodeGen/X86/2010-10-08-cmpxchg8b.ll b/test/CodeGen/X86/2010-10-08-cmpxchg8b.ll
index ebf51a5d660..ee50cb13e63 100644
--- a/test/CodeGen/X86/2010-10-08-cmpxchg8b.ll
+++ b/test/CodeGen/X86/2010-10-08-cmpxchg8b.ll
@@ -16,8 +16,7 @@ define void @foo(i64* %ptr) nounwind inlinehint {
 entry:
   br label %loop
 loop:
-; CHECK: lock
-; CHECK-NEXT: cmpxchg8b
+; CHECK: lock cmpxchg8b
   %pair = cmpxchg i64* %ptr, i64 0, i64 1 monotonic monotonic
   %r = extractvalue { i64, i1 } %pair, 0
   %stored1  = icmp eq i64 %r, 0
diff --git a/test/CodeGen/X86/2010-11-02-DbgParameter.ll b/test/CodeGen/X86/2010-11-02-DbgParameter.ll
index d920513581e..cc292ccae04 100644
--- a/test/CodeGen/X86/2010-11-02-DbgParameter.ll
+++ b/test/CodeGen/X86/2010-11-02-DbgParameter.ll
@@ -9,7 +9,7 @@ target triple = "i386-apple-darwin11.0.0"
 define i32 @foo(%struct.bar* nocapture %i) nounwind readnone optsize noinline ssp {
 ; CHECK: TAG_formal_parameter
 entry:
-  tail call void @llvm.dbg.value(metadata %struct.bar* %i, i64 0, metadata !6, metadata !MDExpression()), !dbg !12
+  tail call void @llvm.dbg.value(metadata %struct.bar* %i, i64 0, metadata !6, metadata !DIExpression()), !dbg !12
   ret i32 1, !dbg !13
 }
 
@@ -18,23 +18,23 @@ declare void @llvm.dbg.value(metadata, i64, metadata, metadata) nounwind readnon
 !llvm.dbg.cu = !{!2}
 !llvm.module.flags = !{!19}
 
-!0 = !MDSubprogram(name: "foo", line: 3, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: true, scopeLine: 3, file: !17, scope: !1, type: !3, function: i32 (%struct.bar*)* @foo, variables: !16)
-!1 = !MDFile(filename: "one.c", directory: "/private/tmp")
-!2 = !MDCompileUnit(language: DW_LANG_C99, producer: "clang version 2.9 (trunk 117922)", isOptimized: true, emissionKind: 0, file: !17, enums: !18, retainedTypes: !18, subprograms: !15, imports:  null)
-!3 = !MDSubroutineType(types: !4)
+!0 = !DISubprogram(name: "foo", line: 3, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: true, scopeLine: 3, file: !17, scope: !1, type: !3, function: i32 (%struct.bar*)* @foo, variables: !16)
+!1 = !DIFile(filename: "one.c", directory: "/private/tmp")
+!2 = !DICompileUnit(language: DW_LANG_C99, producer: "clang version 2.9 (trunk 117922)", isOptimized: true, emissionKind: 0, file: !17, enums: !18, retainedTypes: !18, subprograms: !15, imports:  null)
+!3 = !DISubroutineType(types: !4)
 !4 = !{!5}
-!5 = !MDBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
-!6 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "i", line: 3, arg: 0, scope: !0, file: !1, type: !7)
-!7 = !MDDerivedType(tag: DW_TAG_pointer_type, size: 32, align: 32, file: !17, scope: !1, baseType: !8)
-!8 = !MDCompositeType(tag: DW_TAG_structure_type, name: "bar", line: 2, size: 64, align: 32, file: !17, scope: !1, elements: !9)
+!5 = !DIBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
+!6 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "i", line: 3, arg: 1, scope: !0, file: !1, type: !7)
+!7 = !DIDerivedType(tag: DW_TAG_pointer_type, size: 32, align: 32, file: !17, scope: !1, baseType: !8)
+!8 = !DICompositeType(tag: DW_TAG_structure_type, name: "bar", line: 2, size: 64, align: 32, file: !17, scope: !1, elements: !9)
 !9 = !{!10, !11}
-!10 = !MDDerivedType(tag: DW_TAG_member, name: "x", line: 2, size: 32, align: 32, file: !17, scope:  !1, baseType: !5)
-!11 = !MDDerivedType(tag: DW_TAG_member, name: "y", line: 2, size: 32, align: 32, offset: 32, file: !17, scope: !1, baseType: !5)
-!12 = !MDLocation(line: 3, column: 47, scope: !0)
-!13 = !MDLocation(line: 4, column: 2, scope: !14)
-!14 = distinct !MDLexicalBlock(line: 3, column: 50, file: !17, scope: !0)
+!10 = !DIDerivedType(tag: DW_TAG_member, name: "x", line: 2, size: 32, align: 32, file: !17, scope:  !1, baseType: !5)
+!11 = !DIDerivedType(tag: DW_TAG_member, name: "y", line: 2, size: 32, align: 32, offset: 32, file: !17, scope: !1, baseType: !5)
+!12 = !DILocation(line: 3, column: 47, scope: !0)
+!13 = !DILocation(line: 4, column: 2, scope: !14)
+!14 = distinct !DILexicalBlock(line: 3, column: 50, file: !17, scope: !0)
 !15 = !{!0}
 !16 = !{!6}
-!17 = !MDFile(filename: "one.c", directory: "/private/tmp")
+!17 = !DIFile(filename: "one.c", directory: "/private/tmp")
 !18 = !{}
 !19 = !{i32 1, !"Debug Info Version", i32 3}
diff --git a/test/CodeGen/X86/2011-01-24-DbgValue-Before-Use.ll b/test/CodeGen/X86/2011-01-24-DbgValue-Before-Use.ll
index c02bd2d9734..c873915da5a 100644
--- a/test/CodeGen/X86/2011-01-24-DbgValue-Before-Use.ll
+++ b/test/CodeGen/X86/2011-01-24-DbgValue-Before-Use.ll
@@ -22,8 +22,8 @@ target triple = "x86_64-apple-darwin10.0.0"
 
 define i64 @gcd(i64 %a, i64 %b) nounwind readnone optsize noinline ssp {
 entry:
-  tail call void @llvm.dbg.value(metadata i64 %a, i64 0, metadata !10, metadata !MDExpression()), !dbg !18
-  tail call void @llvm.dbg.value(metadata i64 %b, i64 0, metadata !11, metadata !MDExpression()), !dbg !19
+  tail call void @llvm.dbg.value(metadata i64 %a, i64 0, metadata !10, metadata !DIExpression()), !dbg !18
+  tail call void @llvm.dbg.value(metadata i64 %b, i64 0, metadata !11, metadata !DIExpression()), !dbg !19
   br label %while.body, !dbg !20
 
 while.body:                                       ; preds = %while.body, %entry
@@ -34,14 +34,14 @@ while.body:                                       ; preds = %while.body, %entry
   br i1 %cmp, label %if.then, label %while.body, !dbg !23
 
 if.then:                                          ; preds = %while.body
-  tail call void @llvm.dbg.value(metadata i64 %rem, i64 0, metadata !12, metadata !MDExpression()), !dbg !21
+  tail call void @llvm.dbg.value(metadata i64 %rem, i64 0, metadata !12, metadata !DIExpression()), !dbg !21
   ret i64 %b.addr.0, !dbg !23
 }
 
 define i32 @main() nounwind optsize ssp {
 entry:
   %call = tail call i32 @rand() nounwind optsize, !dbg !24
-  tail call void @llvm.dbg.value(metadata i32 %call, i64 0, metadata !14, metadata !MDExpression()), !dbg !24
+  tail call void @llvm.dbg.value(metadata i32 %call, i64 0, metadata !14, metadata !DIExpression()), !dbg !24
   %cmp = icmp ugt i32 %call, 21, !dbg !25
   br i1 %cmp, label %cond.true, label %cond.end, !dbg !25
 
@@ -51,7 +51,7 @@ cond.true:                                        ; preds = %entry
 
 cond.end:                                         ; preds = %entry, %cond.true
   %cond = phi i32 [ %call1, %cond.true ], [ %call, %entry ], !dbg !25
-  tail call void @llvm.dbg.value(metadata i32 %cond, i64 0, metadata !17, metadata !MDExpression()), !dbg !25
+  tail call void @llvm.dbg.value(metadata i32 %cond, i64 0, metadata !17, metadata !DIExpression()), !dbg !25
   %conv = sext i32 %cond to i64, !dbg !26
   %conv5 = zext i32 %call to i64, !dbg !26
   %call6 = tail call i64 @gcd(i64 %conv, i64 %conv5) optsize, !dbg !26
@@ -78,37 +78,37 @@ declare i32 @puts(i8* nocapture) nounwind
 !llvm.dbg.cu = !{!2}
 !llvm.module.flags = !{!33}
 
-!0 = !MDSubprogram(name: "gcd", line: 5, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: true, file: !31, scope: !1, type: !3, function: i64 (i64, i64)* @gcd, variables: !29)
-!1 = !MDFile(filename: "rem_small.c", directory: "/private/tmp")
-!2 = !MDCompileUnit(language: DW_LANG_C99, producer: "clang version 2.9 (trunk 124117)", isOptimized: true, emissionKind: 1, file: !31, enums: !32, retainedTypes: !32, subprograms: !28, imports:  null)
-!3 = !MDSubroutineType(types: !4)
+!0 = !DISubprogram(name: "gcd", line: 5, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: true, file: !31, scope: !1, type: !3, function: i64 (i64, i64)* @gcd, variables: !29)
+!1 = !DIFile(filename: "rem_small.c", directory: "/private/tmp")
+!2 = !DICompileUnit(language: DW_LANG_C99, producer: "clang version 2.9 (trunk 124117)", isOptimized: true, emissionKind: 1, file: !31, enums: !32, retainedTypes: !32, subprograms: !28, imports:  null)
+!3 = !DISubroutineType(types: !4)
 !4 = !{!5}
-!5 = !MDBasicType(tag: DW_TAG_base_type, name: "long int", size: 64, align: 64, encoding: DW_ATE_signed)
-!6 = !MDSubprogram(name: "main", line: 25, isLocal: false, isDefinition: true, virtualIndex: 6, isOptimized: true, file: !31, scope: !1, type: !7, function: i32 ()* @main, variables: !30)
-!7 = !MDSubroutineType(types: !8)
+!5 = !DIBasicType(tag: DW_TAG_base_type, name: "long int", size: 64, align: 64, encoding: DW_ATE_signed)
+!6 = !DISubprogram(name: "main", line: 25, isLocal: false, isDefinition: true, virtualIndex: 6, isOptimized: true, file: !31, scope: !1, type: !7, function: i32 ()* @main, variables: !30)
+!7 = !DISubroutineType(types: !8)
 !8 = !{!9}
-!9 = !MDBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
-!10 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "a", line: 5, arg: 0, scope: !0, file: !1, type: !5)
-!11 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "b", line: 5, arg: 0, scope: !0, file: !1, type: !5)
-!12 = !MDLocalVariable(tag: DW_TAG_auto_variable, name: "c", line: 6, scope: !13, file: !1, type: !5)
-!13 = distinct !MDLexicalBlock(line: 5, column: 52, file: !31, scope: !0)
-!14 = !MDLocalVariable(tag: DW_TAG_auto_variable, name: "m", line: 26, scope: !15, file: !1, type: !16)
-!15 = distinct !MDLexicalBlock(line: 25, column: 12, file: !31, scope: !6)
-!16 = !MDBasicType(tag: DW_TAG_base_type, name: "unsigned int", size: 32, align: 32, encoding: DW_ATE_unsigned)
-!17 = !MDLocalVariable(tag: DW_TAG_auto_variable, name: "z_s", line: 27, scope: !15, file: !1, type: !9)
-!18 = !MDLocation(line: 5, column: 41, scope: !0)
-!19 = !MDLocation(line: 5, column: 49, scope: !0)
-!20 = !MDLocation(line: 7, column: 5, scope: !13)
-!21 = !MDLocation(line: 8, column: 9, scope: !22)
-!22 = distinct !MDLexicalBlock(line: 7, column: 14, file: !31, scope: !13)
-!23 = !MDLocation(line: 9, column: 9, scope: !22)
-!24 = !MDLocation(line: 26, column: 38, scope: !15)
-!25 = !MDLocation(line: 27, column: 38, scope: !15)
-!26 = !MDLocation(line: 28, column: 9, scope: !15)
-!27 = !MDLocation(line: 30, column: 1, scope: !15)
+!9 = !DIBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
+!10 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "a", line: 5, arg: 1, scope: !0, file: !1, type: !5)
+!11 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "b", line: 5, arg: 2, scope: !0, file: !1, type: !5)
+!12 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "c", line: 6, scope: !13, file: !1, type: !5)
+!13 = distinct !DILexicalBlock(line: 5, column: 52, file: !31, scope: !0)
+!14 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "m", line: 26, scope: !15, file: !1, type: !16)
+!15 = distinct !DILexicalBlock(line: 25, column: 12, file: !31, scope: !6)
+!16 = !DIBasicType(tag: DW_TAG_base_type, name: "unsigned int", size: 32, align: 32, encoding: DW_ATE_unsigned)
+!17 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "z_s", line: 27, scope: !15, file: !1, type: !9)
+!18 = !DILocation(line: 5, column: 41, scope: !0)
+!19 = !DILocation(line: 5, column: 49, scope: !0)
+!20 = !DILocation(line: 7, column: 5, scope: !13)
+!21 = !DILocation(line: 8, column: 9, scope: !22)
+!22 = distinct !DILexicalBlock(line: 7, column: 14, file: !31, scope: !13)
+!23 = !DILocation(line: 9, column: 9, scope: !22)
+!24 = !DILocation(line: 26, column: 38, scope: !15)
+!25 = !DILocation(line: 27, column: 38, scope: !15)
+!26 = !DILocation(line: 28, column: 9, scope: !15)
+!27 = !DILocation(line: 30, column: 1, scope: !15)
 !28 = !{!0, !6}
 !29 = !{!10, !11, !12}
 !30 = !{!14, !17}
-!31 = !MDFile(filename: "rem_small.c", directory: "/private/tmp")
+!31 = !DIFile(filename: "rem_small.c", directory: "/private/tmp")
 !32 = !{}
 !33 = !{i32 1, !"Debug Info Version", i32 3}
diff --git a/test/CodeGen/X86/2011-12-15-vec_shift.ll b/test/CodeGen/X86/2011-12-15-vec_shift.ll
index 0183e107460..4d49b3af88e 100644
--- a/test/CodeGen/X86/2011-12-15-vec_shift.ll
+++ b/test/CodeGen/X86/2011-12-15-vec_shift.ll
@@ -12,8 +12,8 @@ define <16 x i8> @shift(<16 x i8> %a, <16 x i8> %b) nounwind {
 
   ; Make sure we're masking and pcmp'ing the VSELECT conditon vector.
   ; CHECK-WO-SSE4: psllw $5, [[REG1:%xmm.]]
-  ; CHECK-WO-SSE4: pand [[REG1]], [[REG2:%xmm.]]
-  ; CHECK-WO-SSE4: pcmpeqb {{%xmm., }}[[REG2]]
+  ; CHECK-WO-SSE4: pxor [[REG2:%xmm.]], [[REG2:%xmm.]]
+  ; CHECK-WO-SSE4: pcmpgtb {{%xmm., }}[[REG2]]
   %1 = shl <16 x i8> %a, %b
   ret <16 x i8> %1
 }
diff --git a/test/CodeGen/X86/2012-01-10-UndefExceptionEdge.ll b/test/CodeGen/X86/2012-01-10-UndefExceptionEdge.ll
index 21443441c9f..20615afdfa1 100644
--- a/test/CodeGen/X86/2012-01-10-UndefExceptionEdge.ll
+++ b/test/CodeGen/X86/2012-01-10-UndefExceptionEdge.ll
@@ -16,7 +16,7 @@ target triple = "i386-apple-macosx10.7"
 
 declare void @llvm.memset.p0i8.i32(i8* nocapture, i8, i32, i32, i1) nounwind
 
-define void @f(i32* nocapture %arg, i32* nocapture %arg1, i32* nocapture %arg2, i32* nocapture %arg3, i32 %arg4, i32 %arg5) optsize ssp {
+define void @f(i32* nocapture %arg, i32* nocapture %arg1, i32* nocapture %arg2, i32* nocapture %arg3, i32 %arg4, i32 %arg5) optsize ssp personality i8* bitcast (i32 (...)* @__gxx_personality_v0 to i8*) {
 bb:
   br i1 undef, label %bb6, label %bb7
 
@@ -43,7 +43,7 @@ bb11:                                             ; preds = %bb7
 
 bb20:                                             ; preds = %bb43, %bb41, %bb29, %bb7
   %tmp21 = phi i32 [ undef, %bb7 ], [ %tmp12, %bb43 ], [ %tmp12, %bb29 ], [ %tmp12, %bb41 ]
-  %tmp22 = landingpad { i8*, i32 } personality i8* bitcast (i32 (...)* @__gxx_personality_v0 to i8*)
+  %tmp22 = landingpad { i8*, i32 }
           catch i8* bitcast ({ i8*, i8* }* @Exception to i8*)
   br i1 undef, label %bb23, label %bb69
 
diff --git a/test/CodeGen/X86/2012-01-16-mfence-nosse-flags.ll b/test/CodeGen/X86/2012-01-16-mfence-nosse-flags.ll
index 16eef0a28aa..a6c34b8fffa 100644
--- a/test/CodeGen/X86/2012-01-16-mfence-nosse-flags.ll
+++ b/test/CodeGen/X86/2012-01-16-mfence-nosse-flags.ll
@@ -13,8 +13,7 @@ entry:
 ; Make sure the fence comes before the comparison, since it
 ; clobbers EFLAGS.
 
-; CHECK: lock
-; CHECK-NEXT: orl {{.*}}, (%esp)
+; CHECK: lock orl {{.*}}, (%esp)
 ; CHECK-NEXT: testl [[REG:%e[a-z]+]], [[REG]]
 
 if.then:                                          ; preds = %entry
diff --git a/test/CodeGen/X86/2012-05-19-CoalescerCrash.ll b/test/CodeGen/X86/2012-05-19-CoalescerCrash.ll
index 837fbc0777f..a3f68fa4c22 100644
--- a/test/CodeGen/X86/2012-05-19-CoalescerCrash.ll
+++ b/test/CodeGen/X86/2012-05-19-CoalescerCrash.ll
@@ -7,7 +7,7 @@
 
 target triple = "i386-pc-linux-gnu"
 
-define void @_ZN4llvm17AsmMatcherEmitter3runERNS_11raw_ostreamE() align 2 {
+define void @_ZN4llvm17AsmMatcherEmitter3runERNS_11raw_ostreamE() align 2 personality i8* bitcast (i32 (...)* @__gxx_personality_v0 to i8*) {
   invoke void @_ZNK4llvm13CodeGenTarget12getAsmParserEv()
           to label %1 unwind label %5
 
@@ -16,7 +16,7 @@ define void @_ZN4llvm17AsmMatcherEmitter3runERNS_11raw_ostreamE() align 2 {
           to label %4 unwind label %2
 
 ; <label>:2                                       ; preds = %1
-  %3 = landingpad { i8*, i32 } personality i8* bitcast (i32 (...)* @__gxx_personality_v0 to i8*)
+  %3 = landingpad { i8*, i32 }
           cleanup
   unreachable
 
@@ -25,12 +25,12 @@ define void @_ZN4llvm17AsmMatcherEmitter3runERNS_11raw_ostreamE() align 2 {
           to label %12 unwind label %7
 
 ; <label>:5                                       ; preds = %0
-  %6 = landingpad { i8*, i32 } personality i8* bitcast (i32 (...)* @__gxx_personality_v0 to i8*)
+  %6 = landingpad { i8*, i32 }
           cleanup
   br label %33
 
 ; <label>:7                                       ; preds = %4
-  %8 = landingpad { i8*, i32 } personality i8* bitcast (i32 (...)* @__gxx_personality_v0 to i8*)
+  %8 = landingpad { i8*, i32 }
           cleanup
   br label %9
 
@@ -52,7 +52,7 @@ define void @_ZN4llvm17AsmMatcherEmitter3runERNS_11raw_ostreamE() align 2 {
   br i1 %15, label %20, label %18
 
 ; <label>:16                                      ; preds = %12
-  %17 = landingpad { i8*, i32 } personality i8* bitcast (i32 (...)* @__gxx_personality_v0 to i8*)
+  %17 = landingpad { i8*, i32 }
           cleanup
   br label %26
 
@@ -67,7 +67,7 @@ define void @_ZN4llvm17AsmMatcherEmitter3runERNS_11raw_ostreamE() align 2 {
   br label %14
 
 ; <label>:21                                      ; preds = %18
-  %22 = landingpad { i8*, i32 } personality i8* bitcast (i32 (...)* @__gxx_personality_v0 to i8*)
+  %22 = landingpad { i8*, i32 }
           cleanup
   %23 = extractvalue { i8*, i32 } %22, 1
   br i1 undef, label %26, label %24
@@ -88,7 +88,7 @@ define void @_ZN4llvm17AsmMatcherEmitter3runERNS_11raw_ostreamE() align 2 {
   br label %9
 
 ; <label>:30                                      ; preds = %26
-  %31 = landingpad { i8*, i32 } personality i8* bitcast (i32 (...)* @__gxx_personality_v0 to i8*)
+  %31 = landingpad { i8*, i32 }
           catch i8* null
   unreachable
 
@@ -100,7 +100,7 @@ define void @_ZN4llvm17AsmMatcherEmitter3runERNS_11raw_ostreamE() align 2 {
   unreachable
 
 ; <label>:35                                      ; preds = %9
-  %36 = landingpad { i8*, i32 } personality i8* bitcast (i32 (...)* @__gxx_personality_v0 to i8*)
+  %36 = landingpad { i8*, i32 }
           catch i8* null
   unreachable
 }
diff --git a/test/CodeGen/X86/2012-11-28-merge-store-alias.ll b/test/CodeGen/X86/2012-11-28-merge-store-alias.ll
index ed1daadf629..c16deeff3d9 100644
--- a/test/CodeGen/X86/2012-11-28-merge-store-alias.ll
+++ b/test/CodeGen/X86/2012-11-28-merge-store-alias.ll
@@ -3,6 +3,7 @@
 ; CHECK: merge_stores_can
 ; CHECK: callq foo
 ; CHECK: xorps %xmm0, %xmm0
+; CHECK-NEXT: movl 36(%rsp), %ebp
 ; CHECK-NEXT: movups  %xmm0
 ; CHECK: callq foo
 ; CHECK: ret
diff --git a/test/CodeGen/X86/2012-11-30-handlemove-dbg.ll b/test/CodeGen/X86/2012-11-30-handlemove-dbg.ll
index d175fab706c..a27db95ba12 100644
--- a/test/CodeGen/X86/2012-11-30-handlemove-dbg.ll
+++ b/test/CodeGen/X86/2012-11-30-handlemove-dbg.ll
@@ -16,7 +16,7 @@ declare void @llvm.dbg.declare(metadata, metadata, metadata) nounwind readnone
 
 define signext i16 @subdivp(%struct.node.0.27* nocapture %p, double %dsq, double %tolsq, %struct.hgstruct.2.29* nocapture byval align 8 %hg) nounwind uwtable readonly ssp {
 entry:
-  call void @llvm.dbg.declare(metadata %struct.hgstruct.2.29* %hg, metadata !4, metadata !MDExpression()), !dbg !MDLocation(scope: !14)
+  call void @llvm.dbg.declare(metadata %struct.hgstruct.2.29* %hg, metadata !4, metadata !DIExpression()), !dbg !DILocation(scope: !14)
   %type = getelementptr inbounds %struct.node.0.27, %struct.node.0.27* %p, i64 0, i32 0
   %0 = load i16, i16* %type, align 2
   %cmp = icmp eq i16 %0, 1
@@ -38,15 +38,15 @@ declare void @llvm.dbg.value(metadata, i64, metadata, metadata) nounwind readnon
 !llvm.dbg.cu = !{!0}
 !llvm.module.flags = !{!12}
 
-!0 = !MDCompileUnit(language: DW_LANG_C99, producer: "clang version 3.3 (trunk 168918) (llvm/trunk 168920)", isOptimized: true, emissionKind: 0, file: !11, enums: !2, retainedTypes: !2, subprograms: !13, globals: !2)
+!0 = !DICompileUnit(language: DW_LANG_C99, producer: "clang version 3.3 (trunk 168918) (llvm/trunk 168920)", isOptimized: true, emissionKind: 0, file: !11, enums: !2, retainedTypes: !2, subprograms: !13, globals: !2)
 !2 = !{}
-!4 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "hg", line: 725, arg: 4, scope: !14, file: !5, type: !6)
-!5 = !MDFile(filename: "MultiSource/Benchmarks/Olden/bh/newbh.c", directory: "MultiSource/Benchmarks/Olden/bh")
-!6 = !MDDerivedType(tag: DW_TAG_typedef, name: "hgstruct", line: 492, file: !11, baseType: !7)
-!7 = !MDCompositeType(tag: DW_TAG_structure_type, line: 487, size: 512, align: 64, file: !11)
-!11 = !MDFile(filename: "MultiSource/Benchmarks/Olden/bh/newbh.c", directory: "MultiSource/Benchmarks/Olden/bh")
+!4 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "hg", line: 725, arg: 4, scope: !14, file: !5, type: !6)
+!5 = !DIFile(filename: "MultiSource/Benchmarks/Olden/bh/newbh.c", directory: "MultiSource/Benchmarks/Olden/bh")
+!6 = !DIDerivedType(tag: DW_TAG_typedef, name: "hgstruct", line: 492, file: !11, baseType: !7)
+!7 = !DICompositeType(tag: DW_TAG_structure_type, line: 487, size: 512, align: 64, file: !11)
+!11 = !DIFile(filename: "MultiSource/Benchmarks/Olden/bh/newbh.c", directory: "MultiSource/Benchmarks/Olden/bh")
 !12 = !{i32 1, !"Debug Info Version", i32 3}
 !13 = !{!14}
-!14 = !MDSubprogram(name: "subdivp", isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: true, scopeLine: 1, file: !11, scope: !5, type: !15, function: i16 (%struct.node.0.27*, double, double, %struct.hgstruct.2.29* )* @subdivp)
-!15 = !MDSubroutineType(types: !16)
+!14 = !DISubprogram(name: "subdivp", isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: true, scopeLine: 1, file: !11, scope: !5, type: !15, function: i16 (%struct.node.0.27*, double, double, %struct.hgstruct.2.29* )* @subdivp)
+!15 = !DISubroutineType(types: !16)
 !16 = !{null}
diff --git a/test/CodeGen/X86/2012-11-30-misched-dbg.ll b/test/CodeGen/X86/2012-11-30-misched-dbg.ll
index 08ade9c85b9..22227faab94 100644
--- a/test/CodeGen/X86/2012-11-30-misched-dbg.ll
+++ b/test/CodeGen/X86/2012-11-30-misched-dbg.ll
@@ -43,7 +43,7 @@ if.then3344:
   br label %if.then4073
 
 if.then4073:                                      ; preds = %if.then3344
-  call void @llvm.dbg.declare(metadata [20 x i8]* %num14075, metadata !4, metadata !MDExpression()), !dbg !MDLocation(scope: !5)
+  call void @llvm.dbg.declare(metadata [20 x i8]* %num14075, metadata !4, metadata !DIExpression()), !dbg !DILocation(scope: !5)
   %arraydecay4078 = getelementptr inbounds [20 x i8], [20 x i8]* %num14075, i64 0, i64 0
   %0 = load i32, i32* undef, align 4
   %add4093 = add nsw i32 %0, 0
@@ -65,29 +65,29 @@ declare i32 @__sprintf_chk(i8*, i32, i64, i8*, ...)
 !llvm.dbg.cu = !{!0}
 !llvm.module.flags = !{!35}
 
-!0 = !MDCompileUnit(language: DW_LANG_C99, producer: "clang version 3.3 (trunk 168918) (llvm/trunk 168920)", isOptimized: true, emissionKind: 0, file: !19, enums: !2, retainedTypes: !2, subprograms: !20, globals: !2)
+!0 = !DICompileUnit(language: DW_LANG_C99, producer: "clang version 3.3 (trunk 168918) (llvm/trunk 168920)", isOptimized: true, emissionKind: 0, file: !19, enums: !2, retainedTypes: !2, subprograms: !20, globals: !2)
 !1 = !{!2}
 !2 = !{}
-!4 = !MDLocalVariable(tag: DW_TAG_auto_variable, name: "num1", line: 815, scope: !5, file: !14, type: !15)
-!5 = distinct !MDLexicalBlock(line: 815, column: 0, file: !14, scope: !6)
-!6 = distinct !MDLexicalBlock(line: 812, column: 0, file: !14, scope: !7)
-!7 = distinct !MDLexicalBlock(line: 807, column: 0, file: !14, scope: !8)
-!8 = distinct !MDLexicalBlock(line: 440, column: 0, file: !14, scope: !9)
-!9 = distinct !MDLexicalBlock(line: 435, column: 0, file: !14, scope: !10)
-!10 = distinct !MDLexicalBlock(line: 434, column: 0, file: !14, scope: !11)
-!11 = distinct !MDLexicalBlock(line: 250, column: 0, file: !14, scope: !12)
-!12 = distinct !MDLexicalBlock(line: 249, column: 0, file: !14, scope: !13)
-!13 = distinct !MDLexicalBlock(line: 221, column: 0, file: !14, scope: !21)
-!14 = !MDFile(filename: "MultiSource/Benchmarks/MiBench/consumer-typeset/z19.c", directory: "MultiSource/Benchmarks/MiBench/consumer-typeset")
-!15 = !MDCompositeType(tag: DW_TAG_array_type, size: 160, align: 8, baseType: !16, elements: !17)
-!16 = !MDBasicType(tag: DW_TAG_base_type, name: "char", size: 8, align: 8, encoding: DW_ATE_signed_char)
+!4 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "num1", line: 815, scope: !5, file: !14, type: !15)
+!5 = distinct !DILexicalBlock(line: 815, column: 0, file: !14, scope: !6)
+!6 = distinct !DILexicalBlock(line: 812, column: 0, file: !14, scope: !7)
+!7 = distinct !DILexicalBlock(line: 807, column: 0, file: !14, scope: !8)
+!8 = distinct !DILexicalBlock(line: 440, column: 0, file: !14, scope: !9)
+!9 = distinct !DILexicalBlock(line: 435, column: 0, file: !14, scope: !10)
+!10 = distinct !DILexicalBlock(line: 434, column: 0, file: !14, scope: !11)
+!11 = distinct !DILexicalBlock(line: 250, column: 0, file: !14, scope: !12)
+!12 = distinct !DILexicalBlock(line: 249, column: 0, file: !14, scope: !13)
+!13 = distinct !DILexicalBlock(line: 221, column: 0, file: !14, scope: !21)
+!14 = !DIFile(filename: "MultiSource/Benchmarks/MiBench/consumer-typeset/z19.c", directory: "MultiSource/Benchmarks/MiBench/consumer-typeset")
+!15 = !DICompositeType(tag: DW_TAG_array_type, size: 160, align: 8, baseType: !16, elements: !17)
+!16 = !DIBasicType(tag: DW_TAG_base_type, name: "char", size: 8, align: 8, encoding: DW_ATE_signed_char)
 !17 = !{!18}
-!18 = !MDSubrange(count: 20)
-!19 = !MDFile(filename: "MultiSource/Benchmarks/MiBench/consumer-typeset/z19.c", directory: "MultiSource/Benchmarks/MiBench/consumer-typeset")
+!18 = !DISubrange(count: 20)
+!19 = !DIFile(filename: "MultiSource/Benchmarks/MiBench/consumer-typeset/z19.c", directory: "MultiSource/Benchmarks/MiBench/consumer-typeset")
 
 !20 = !{!21}
-!21 = !MDSubprogram(name: "AttachGalley", isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: true, scopeLine: 1, file: !19, scope: !14, type: !22, function: i32 (%union.rec**)* @AttachGalley)
-!22 = !MDSubroutineType(types: !23)
+!21 = !DISubprogram(name: "AttachGalley", isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: true, scopeLine: 1, file: !19, scope: !14, type: !22, function: i32 (%union.rec**)* @AttachGalley)
+!22 = !DISubroutineType(types: !23)
 !23 = !{null}
 
 ; Test DebugValue uses visited by RegisterPressureTracker findUseBetween().
@@ -99,7 +99,7 @@ declare i32 @__sprintf_chk(i8*, i32, i64, i8*, ...)
 %"class.__gnu_cxx::hash_map" = type { %"class.__gnu_cxx::hashtable" }
 %"class.__gnu_cxx::hashtable" = type { i64, i64, i64, i64, i64, i64 }
 
-define void @main() uwtable ssp {
+define void @main() uwtable ssp personality i8* bitcast (i32 (...)* @__gxx_personality_v0 to i8*) {
 entry:
   %X = alloca %"class.__gnu_cxx::hash_map", align 8
   br i1 undef, label %cond.true, label %cond.end
@@ -108,7 +108,7 @@ cond.true:                                        ; preds = %entry
   unreachable
 
 cond.end:                                         ; preds = %entry
-  call void @llvm.dbg.declare(metadata %"class.__gnu_cxx::hash_map"* %X, metadata !31, metadata !MDExpression()), !dbg !MDLocation(scope: !37)
+  call void @llvm.dbg.declare(metadata %"class.__gnu_cxx::hash_map"* %X, metadata !31, metadata !DIExpression()), !dbg !DILocation(scope: !37)
   %_M_num_elements.i.i.i.i = getelementptr inbounds %"class.__gnu_cxx::hash_map", %"class.__gnu_cxx::hash_map"* %X, i64 0, i32 0, i32 5
   invoke void @_Znwm()
           to label %exit.i unwind label %lpad2.i.i.i.i
@@ -117,7 +117,7 @@ exit.i:                                           ; preds = %cond.end
   unreachable
 
 lpad2.i.i.i.i:                                    ; preds = %cond.end
-  %0 = landingpad { i8*, i32 } personality i8* bitcast (i32 (...)* @__gxx_personality_v0 to i8*)
+  %0 = landingpad { i8*, i32 }
           cleanup
   br i1 undef, label %lpad.body.i.i, label %if.then.i.i.i.i.i.i.i.i
 
@@ -134,11 +134,11 @@ declare void @_Znwm()
 
 !llvm.dbg.cu = !{!30}
 
-!30 = !MDCompileUnit(language: DW_LANG_C_plus_plus, producer: "clang version 3.3 (trunk 169129) (llvm/trunk 169135)", isOptimized: true, emissionKind: 0, file: !34, enums: !2, retainedTypes: !2, subprograms: !36)
-!31 = !MDLocalVariable(tag: DW_TAG_auto_variable, name: "X", line: 29, scope: !37, type: !32)
-!32 = !MDDerivedType(tag: DW_TAG_typedef, name: "HM", line: 28, file: !34, baseType: null)
-!33 = !MDFile(filename: "SingleSource/Benchmarks/Shootout-C++/hash.cpp", directory: "SingleSource/Benchmarks/Shootout-C++")
-!34 = !MDFile(filename: "SingleSource/Benchmarks/Shootout-C++/hash.cpp", directory: "SingleSource/Benchmarks/Shootout-C++")
+!30 = !DICompileUnit(language: DW_LANG_C_plus_plus, producer: "clang version 3.3 (trunk 169129) (llvm/trunk 169135)", isOptimized: true, emissionKind: 0, file: !34, enums: !2, retainedTypes: !2, subprograms: !36)
+!31 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "X", line: 29, scope: !37, type: !32)
+!32 = !DIDerivedType(tag: DW_TAG_typedef, name: "HM", line: 28, file: !34, baseType: null)
+!33 = !DIFile(filename: "SingleSource/Benchmarks/Shootout-C++/hash.cpp", directory: "SingleSource/Benchmarks/Shootout-C++")
+!34 = !DIFile(filename: "SingleSource/Benchmarks/Shootout-C++/hash.cpp", directory: "SingleSource/Benchmarks/Shootout-C++")
 !35 = !{i32 1, !"Debug Info Version", i32 3}
 !36 = !{!37}
-!37 = !MDSubprogram(name: "main", isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: true, scopeLine: 1, file: !19, scope: !14, type: !22, function: void ()* @main)
+!37 = !DISubprogram(name: "main", isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: true, scopeLine: 1, file: !19, scope: !14, type: !22, function: void ()* @main)
diff --git a/test/CodeGen/X86/2012-11-30-regpres-dbg.ll b/test/CodeGen/X86/2012-11-30-regpres-dbg.ll
index 871c68f740c..77c017eb0e3 100644
--- a/test/CodeGen/X86/2012-11-30-regpres-dbg.ll
+++ b/test/CodeGen/X86/2012-11-30-regpres-dbg.ll
@@ -20,7 +20,7 @@ if.then:                                          ; preds = %entry
   unreachable
 
 if.end:                                           ; preds = %entry
-  call void @llvm.dbg.declare(metadata %struct.btCompoundLeafCallback* %callback, metadata !3, metadata !MDExpression()), !dbg !MDLocation(scope: !2)
+  call void @llvm.dbg.declare(metadata %struct.btCompoundLeafCallback* %callback, metadata !3, metadata !DIExpression()), !dbg !DILocation(scope: !2)
   %m = getelementptr inbounds %struct.btCompoundLeafCallback, %struct.btCompoundLeafCallback* %callback, i64 0, i32 1
   store i32 0, i32* undef, align 8
   %cmp12447 = icmp sgt i32 undef, 0
@@ -36,13 +36,13 @@ invoke.cont44:                                    ; preds = %if.end
 !llvm.dbg.cu = !{!0}
 !llvm.module.flags = !{!8}
 
-!0 = !MDCompileUnit(language: DW_LANG_C_plus_plus, producer: "clang version 3.3 (trunk 168984) (llvm/trunk 168983)", isOptimized: true, emissionKind: 0, file: !6, subprograms: !1)
+!0 = !DICompileUnit(language: DW_LANG_C_plus_plus, producer: "clang version 3.3 (trunk 168984) (llvm/trunk 168983)", isOptimized: true, emissionKind: 0, file: !6, subprograms: !1)
 !1 = !{!2}
-!2 = !MDSubprogram(name: "test", isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: true, scopeLine: 1, file: !6, scope: !5, type: !7, function: void ()* @test)
-!3 = !MDLocalVariable(tag: DW_TAG_auto_variable, name: "callback", line: 214, scope: !2, type: !4)
-!4 = !MDCompositeType(tag: DW_TAG_structure_type, name: "btCompoundLeafCallback", line: 90, size: 512, align: 64, file: !6)
-!5 = !MDFile(filename: "MultiSource/Benchmarks/Bullet/btCompoundCollisionAlgorithm.cpp", directory: "MultiSource/Benchmarks/Bullet")
-!6 = !MDFile(filename: "MultiSource/Benchmarks/Bullet/btCompoundCollisionAlgorithm.cpp", directory: "MultiSource/Benchmarks/Bullet")
-!7 = !MDSubroutineType(types: !9)
+!2 = !DISubprogram(name: "test", isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: true, scopeLine: 1, file: !6, scope: !5, type: !7, function: void ()* @test)
+!3 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "callback", line: 214, scope: !2, type: !4)
+!4 = !DICompositeType(tag: DW_TAG_structure_type, name: "btCompoundLeafCallback", line: 90, size: 512, align: 64, file: !6)
+!5 = !DIFile(filename: "MultiSource/Benchmarks/Bullet/btCompoundCollisionAlgorithm.cpp", directory: "MultiSource/Benchmarks/Bullet")
+!6 = !DIFile(filename: "MultiSource/Benchmarks/Bullet/btCompoundCollisionAlgorithm.cpp", directory: "MultiSource/Benchmarks/Bullet")
+!7 = !DISubroutineType(types: !9)
 !8 = !{i32 1, !"Debug Info Version", i32 3}
 !9 = !{null}
diff --git a/test/CodeGen/X86/2014-08-29-CompactUnwind.ll b/test/CodeGen/X86/2014-08-29-CompactUnwind.ll
index 120eba7a6dd..e7e8bb724fc 100644
--- a/test/CodeGen/X86/2014-08-29-CompactUnwind.ll
+++ b/test/CodeGen/X86/2014-08-29-CompactUnwind.ll
@@ -1,4 +1,4 @@
-; RUN: llc < %s -disable-fp-elim -mtriple x86_64-apple-darwin11 -mcpu corei7 -filetype=obj -o - | llvm-objdump -d -unwind-info -s - | FileCheck %s
+; RUN: llc < %s -mtriple x86_64-apple-darwin11 -mcpu corei7 -filetype=obj -o - | llvm-objdump -d -unwind-info -s - | FileCheck %s
 ; Regression test for http://llvm.org/bugs/show_bug.cgi?id=20800.
 
 ; ModuleID = 'asan_report.ii'
diff --git a/test/CodeGen/X86/MachineBranchProb.ll b/test/CodeGen/X86/MachineBranchProb.ll
index 9b4e73716f4..408c6b9151c 100644
--- a/test/CodeGen/X86/MachineBranchProb.ll
+++ b/test/CodeGen/X86/MachineBranchProb.ll
@@ -18,9 +18,9 @@ for.cond2:                                        ; preds = %for.inc, %for.cond
   %or.cond = or i1 %tobool, %cmp4
   br i1 %or.cond, label %for.inc20, label %for.inc, !prof !0
 ; CHECK: BB#1: derived from LLVM BB %for.cond2
-; CHECK: Successors according to CFG: BB#3(56008718) BB#4(2203492365)
+; CHECK: Successors according to CFG: BB#3(56008718) BB#4(3615818718)
 ; CHECK: BB#4: derived from LLVM BB %for.cond2
-; CHECK: Successors according to CFG: BB#3(112017436) BB#2(4294967294)
+; CHECK: Successors according to CFG: BB#3(56008718) BB#2(3559810000)
 
 for.inc:                                          ; preds = %for.cond2
   %shl = shl i32 %bit.0, 1
diff --git a/test/CodeGen/X86/MachineSink-DbgValue.ll b/test/CodeGen/X86/MachineSink-DbgValue.ll
index 79db4454edf..6f057c5f18e 100644
--- a/test/CodeGen/X86/MachineSink-DbgValue.ll
+++ b/test/CodeGen/X86/MachineSink-DbgValue.ll
@@ -4,10 +4,10 @@ target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f3
 target triple = "x86_64-apple-macosx10.7.0"
 
 define i32 @foo(i32 %i, i32* nocapture %c) nounwind uwtable readonly ssp {
-  tail call void @llvm.dbg.value(metadata i32 %i, i64 0, metadata !6, metadata !MDExpression()), !dbg !12
+  tail call void @llvm.dbg.value(metadata i32 %i, i64 0, metadata !6, metadata !DIExpression()), !dbg !12
   %ab = load i32, i32* %c, align 1, !dbg !14
-  tail call void @llvm.dbg.value(metadata i32* %c, i64 0, metadata !7, metadata !MDExpression()), !dbg !13
-  tail call void @llvm.dbg.value(metadata i32 %ab, i64 0, metadata !10, metadata !MDExpression()), !dbg !14
+  tail call void @llvm.dbg.value(metadata i32* %c, i64 0, metadata !7, metadata !DIExpression()), !dbg !13
+  tail call void @llvm.dbg.value(metadata i32 %ab, i64 0, metadata !10, metadata !DIExpression()), !dbg !14
   %cd = icmp eq i32 %i, 42, !dbg !15
   br i1 %cd, label %bb1, label %bb2, !dbg !15
 
@@ -28,26 +28,26 @@ declare void @llvm.dbg.value(metadata, i64, metadata, metadata) nounwind readnon
 !llvm.dbg.cu = !{!0}
 !llvm.module.flags = !{!22}
 
-!0 = !MDCompileUnit(language: DW_LANG_C99, producer: "Apple clang version 3.0 (tags/Apple/clang-211.10.1) (based on LLVM 3.0svn)", isOptimized: true, emissionKind: 1, file: !20, enums: !21, retainedTypes: !21, subprograms: !18, imports:  null)
-!1 = !MDSubprogram(name: "foo", line: 2, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: true, file: !20, scope: !2, type: !3, function: i32 (i32, i32*)* @foo, variables: !19)
-!2 = !MDFile(filename: "a.c", directory: "/private/tmp")
-!3 = !MDSubroutineType(types: !4)
+!0 = !DICompileUnit(language: DW_LANG_C99, producer: "Apple clang version 3.0 (tags/Apple/clang-211.10.1) (based on LLVM 3.0svn)", isOptimized: true, emissionKind: 1, file: !20, enums: !21, retainedTypes: !21, subprograms: !18, imports:  null)
+!1 = !DISubprogram(name: "foo", line: 2, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: true, file: !20, scope: !2, type: !3, function: i32 (i32, i32*)* @foo, variables: !19)
+!2 = !DIFile(filename: "a.c", directory: "/private/tmp")
+!3 = !DISubroutineType(types: !4)
 !4 = !{!5}
-!5 = !MDBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
-!6 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "i", line: 2, arg: 1, scope: !1, file: !2, type: !5)
-!7 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "c", line: 2, arg: 2, scope: !1, file: !2, type: !8)
-!8 = !MDDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, scope: !0, baseType: !9)
-!9 = !MDBasicType(tag: DW_TAG_base_type, name: "char", size: 8, align: 8, encoding: DW_ATE_signed_char)
-!10 = !MDLocalVariable(tag: DW_TAG_auto_variable, name: "a", line: 3, scope: !11, file: !2, type: !9)
-!11 = distinct !MDLexicalBlock(line: 2, column: 25, file: !20, scope: !1)
-!12 = !MDLocation(line: 2, column: 13, scope: !1)
-!13 = !MDLocation(line: 2, column: 22, scope: !1)
-!14 = !MDLocation(line: 3, column: 14, scope: !11)
-!15 = !MDLocation(line: 4, column: 3, scope: !11)
-!16 = !MDLocation(line: 5, column: 5, scope: !11)
-!17 = !MDLocation(line: 7, column: 1, scope: !11)
+!5 = !DIBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
+!6 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "i", line: 2, arg: 1, scope: !1, file: !2, type: !5)
+!7 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "c", line: 2, arg: 2, scope: !1, file: !2, type: !8)
+!8 = !DIDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, scope: !0, baseType: !9)
+!9 = !DIBasicType(tag: DW_TAG_base_type, name: "char", size: 8, align: 8, encoding: DW_ATE_signed_char)
+!10 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "a", line: 3, scope: !11, file: !2, type: !9)
+!11 = distinct !DILexicalBlock(line: 2, column: 25, file: !20, scope: !1)
+!12 = !DILocation(line: 2, column: 13, scope: !1)
+!13 = !DILocation(line: 2, column: 22, scope: !1)
+!14 = !DILocation(line: 3, column: 14, scope: !11)
+!15 = !DILocation(line: 4, column: 3, scope: !11)
+!16 = !DILocation(line: 5, column: 5, scope: !11)
+!17 = !DILocation(line: 7, column: 1, scope: !11)
 !18 = !{!1}
 !19 = !{!6, !7, !10}
-!20 = !MDFile(filename: "a.c", directory: "/private/tmp")
+!20 = !DIFile(filename: "a.c", directory: "/private/tmp")
 !21 = !{}
 !22 = !{i32 1, !"Debug Info Version", i32 3}
diff --git a/test/CodeGen/X86/MergeConsecutiveStores.ll b/test/CodeGen/X86/MergeConsecutiveStores.ll
index aff6fbc254b..c8f249b7529 100644
--- a/test/CodeGen/X86/MergeConsecutiveStores.ll
+++ b/test/CodeGen/X86/MergeConsecutiveStores.ll
@@ -291,17 +291,12 @@ block4:                                       ; preds = %4, %.lr.ph
   ret void
 }
 
+;; On x86, even unaligned copies can be merged to vector ops.
 ; CHECK-LABEL: merge_loads_no_align:
 ;  load:
-; CHECK: movl
-; CHECK: movl
-; CHECK: movl
-; CHECK: movl
+; CHECK: vmovups
 ;  store:
-; CHECK: movl
-; CHECK: movl
-; CHECK: movl
-; CHECK: movl
+; CHECK: vmovups
 ; CHECK: ret
 define void @merge_loads_no_align(i32 %count, %struct.B* noalias nocapture %q, %struct.B* noalias nocapture %p) nounwind uwtable noinline ssp {
   %a1 = icmp sgt i32 %count, 0
@@ -468,6 +463,67 @@ define void @merge_vec_element_store(<8 x float> %v, float* %ptr) {
 ; CHECK-NEXT: retq
 }
 
+; PR21711 - Merge vector stores into wider vector stores.
+; These should be merged into 32-byte stores.
+define void @merge_vec_extract_stores(<8 x float> %v1, <8 x float> %v2, <4 x float>* %ptr) {
+  %idx0 = getelementptr inbounds <4 x float>, <4 x float>* %ptr, i64 3
+  %idx1 = getelementptr inbounds <4 x float>, <4 x float>* %ptr, i64 4
+  %idx2 = getelementptr inbounds <4 x float>, <4 x float>* %ptr, i64 5
+  %idx3 = getelementptr inbounds <4 x float>, <4 x float>* %ptr, i64 6
+  %shuffle0 = shufflevector <8 x float> %v1, <8 x float> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %shuffle1 = shufflevector <8 x float> %v1, <8 x float> undef, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
+  %shuffle2 = shufflevector <8 x float> %v2, <8 x float> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %shuffle3 = shufflevector <8 x float> %v2, <8 x float> undef, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
+  store <4 x float> %shuffle0, <4 x float>* %idx0, align 16
+  store <4 x float> %shuffle1, <4 x float>* %idx1, align 16
+  store <4 x float> %shuffle2, <4 x float>* %idx2, align 16
+  store <4 x float> %shuffle3, <4 x float>* %idx3, align 16
+  ret void
+
+; CHECK-LABEL: merge_vec_extract_stores
+; CHECK:      vmovaps %xmm0, 48(%rdi)
+; CHECK-NEXT: vextractf128 $1, %ymm0, 64(%rdi)
+; CHECK-NEXT: vmovaps %xmm1, 80(%rdi)
+; CHECK-NEXT: vextractf128 $1, %ymm1, 96(%rdi)
+; CHECK-NEXT: vzeroupper
+; CHECK-NEXT: retq
+}
+
+; Merging vector stores when sourced from vector loads is not currently handled.
+define void @merge_vec_stores_from_loads(<4 x float>* %v, <4 x float>* %ptr) {
+  %load_idx0 = getelementptr inbounds <4 x float>, <4 x float>* %v, i64 0
+  %load_idx1 = getelementptr inbounds <4 x float>, <4 x float>* %v, i64 1
+  %v0 = load <4 x float>, <4 x float>* %load_idx0
+  %v1 = load <4 x float>, <4 x float>* %load_idx1
+  %store_idx0 = getelementptr inbounds <4 x float>, <4 x float>* %ptr, i64 0
+  %store_idx1 = getelementptr inbounds <4 x float>, <4 x float>* %ptr, i64 1
+  store <4 x float> %v0, <4 x float>* %store_idx0, align 16
+  store <4 x float> %v1, <4 x float>* %store_idx1, align 16
+  ret void
+
+; CHECK-LABEL: merge_vec_stores_from_loads
+; CHECK:      vmovaps
+; CHECK-NEXT: vmovaps
+; CHECK-NEXT: vmovaps
+; CHECK-NEXT: vmovaps
+; CHECK-NEXT: retq
+}
+
+; Merging vector stores when sourced from a constant vector is not currently handled. 
+define void @merge_vec_stores_of_constants(<4 x i32>* %ptr) {
+  %idx0 = getelementptr inbounds <4 x i32>, <4 x i32>* %ptr, i64 3
+  %idx1 = getelementptr inbounds <4 x i32>, <4 x i32>* %ptr, i64 4
+  store <4 x i32> <i32 0, i32 0, i32 0, i32 0>, <4 x i32>* %idx0, align 16
+  store <4 x i32> <i32 0, i32 0, i32 0, i32 0>, <4 x i32>* %idx1, align 16
+  ret void
+
+; CHECK-LABEL: merge_vec_stores_of_constants
+; CHECK:      vxorps
+; CHECK-NEXT: vmovaps
+; CHECK-NEXT: vmovaps
+; CHECK-NEXT: retq
+}
+
 ; This is a minimized test based on real code that was failing.
 ; We could merge stores (and loads) like this...
 
diff --git a/test/CodeGen/X86/StackColoring-dbg.ll b/test/CodeGen/X86/StackColoring-dbg.ll
index 7ac08d12e3b..98c27f44fab 100644
--- a/test/CodeGen/X86/StackColoring-dbg.ll
+++ b/test/CodeGen/X86/StackColoring-dbg.ll
@@ -17,7 +17,7 @@ entry:
 for.body:
   call void @llvm.lifetime.end(i64 -1, i8* %0) nounwind
   call void @llvm.lifetime.start(i64 -1, i8* %x.i) nounwind
-  call void @llvm.dbg.declare(metadata i8* %x.i, metadata !22, metadata !MDExpression()) nounwind, !dbg !MDLocation(scope: !2)
+  call void @llvm.dbg.declare(metadata i8* %x.i, metadata !22, metadata !DIExpression()) nounwind, !dbg !DILocation(scope: !2)
   br label %for.body
 }
 
@@ -27,9 +27,9 @@ declare void @llvm.lifetime.end(i64, i8* nocapture) nounwind
 
 !llvm.dbg.cu = !{!0}
 !llvm.module.flags = !{!23}
-!0 = !MDCompileUnit(language: DW_LANG_C89, producer: "clang", isOptimized: true, emissionKind: 0, file: !1, enums: !{}, retainedTypes: !{})
-!1 = !MDFile(filename: "t.c", directory: "")
-!16 = !MDBasicType(tag: DW_TAG_base_type, name: "char", size: 8, align: 8, encoding: DW_ATE_signed_char)
-!2 = !MDSubprogram()
-!22 = !MDLocalVariable(tag: DW_TAG_auto_variable, name: "x", line: 16, scope: !2, file: !1, type: !16)
+!0 = !DICompileUnit(language: DW_LANG_C89, producer: "clang", isOptimized: true, emissionKind: 0, file: !1, enums: !{}, retainedTypes: !{})
+!1 = !DIFile(filename: "t.c", directory: "")
+!16 = !DIBasicType(tag: DW_TAG_base_type, name: "char", size: 8, align: 8, encoding: DW_ATE_signed_char)
+!2 = !DISubprogram()
+!22 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "x", line: 16, scope: !2, file: !1, type: !16)
 !23 = !{i32 1, !"Debug Info Version", i32 3}
diff --git a/test/CodeGen/X86/StackColoring.ll b/test/CodeGen/X86/StackColoring.ll
index 414ccf469eb..634f66ad52d 100644
--- a/test/CodeGen/X86/StackColoring.ll
+++ b/test/CodeGen/X86/StackColoring.ll
@@ -1,9 +1,10 @@
-; RUN: llc -mcpu=corei7 -no-stack-coloring=false < %s | FileCheck %s --check-prefix=YESCOLOR
-; RUN: llc -mcpu=corei7 -no-stack-coloring=true  < %s | FileCheck %s --check-prefix=NOCOLOR
+; RUN: llc -mcpu=corei7 -no-stack-coloring=false < %s | FileCheck %s --check-prefix=YESCOLOR --check-prefix=CHECK
+; RUN: llc -mcpu=corei7 -no-stack-coloring=true  < %s | FileCheck %s --check-prefix=NOCOLOR --check-prefix=CHECK
 
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
 target triple = "x86_64-apple-macosx10.8.0"
 
+;CHECK-LABEL: myCall_w2:
 ;YESCOLOR: subq  $144, %rsp
 ;NOCOLOR: subq  $272, %rsp
 
@@ -28,6 +29,7 @@ entry:
 }
 
 
+;CHECK-LABEL: myCall2_no_merge
 ;YESCOLOR: subq  $272, %rsp
 ;NOCOLOR: subq  $272, %rsp
 
@@ -56,6 +58,7 @@ bb3:
   ret i32 0
 }
 
+;CHECK-LABEL: myCall2_w2
 ;YESCOLOR: subq  $144, %rsp
 ;NOCOLOR: subq  $272, %rsp
 
@@ -82,12 +85,11 @@ bb2:
 bb3:
   ret i32 0
 }
+
+;CHECK-LABEL: myCall_w4:
 ;YESCOLOR: subq  $200, %rsp
 ;NOCOLOR: subq  $408, %rsp
 
-
-
-
 define i32 @myCall_w4(i32 %in) {
 entry:
   %a1 = alloca [14 x i8*], align 8
@@ -119,6 +121,7 @@ entry:
   ret i32 %t7
 }
 
+;CHECK-LABEL: myCall2_w4:
 ;YESCOLOR: subq  $112, %rsp
 ;NOCOLOR: subq  $400, %rsp
 
@@ -158,6 +161,7 @@ bb3:
 }
 
 
+;CHECK-LABEL: myCall2_noend:
 ;YESCOLOR: subq  $144, %rsp
 ;NOCOLOR: subq  $272, %rsp
 
@@ -185,6 +189,7 @@ bb3:
   ret i32 0
 }
 
+;CHECK-LABEL: myCall2_noend2:
 ;YESCOLOR: subq  $144, %rsp
 ;NOCOLOR: subq  $272, %rsp
 define i32 @myCall2_noend2(i32 %in, i1 %d) {
@@ -211,6 +216,7 @@ bb3:
 }
 
 
+;CHECK-LABEL: myCall2_nostart:
 ;YESCOLOR: subq  $144, %rsp
 ;NOCOLOR: subq  $272, %rsp
 define i32 @myCall2_nostart(i32 %in, i1 %d) {
@@ -236,6 +242,7 @@ bb3:
 }
 
 ; Adopt the test from Transforms/Inline/array_merge.ll'
+;CHECK-LABEL: array_merge:
 ;YESCOLOR: subq  $816, %rsp
 ;NOCOLOR: subq  $1616, %rsp
 define void @array_merge() nounwind ssp {
@@ -261,6 +268,7 @@ entry:
   ret void
 }
 
+;CHECK-LABEL: func_phi_lifetime:
 ;YESCOLOR: subq  $272, %rsp
 ;NOCOLOR: subq  $272, %rsp
 define i32 @func_phi_lifetime(i32 %in, i1 %d) {
@@ -297,8 +305,7 @@ bb3:
 }
 
 
-;YESCOLOR-LABEL: multi_region_bb:
-;NOCOLOR-LABEL: multi_region_bb:
+;CHECK-LABEL: multi_region_bb:
 define void @multi_region_bb() nounwind ssp {
 entry:
   %A.i1 = alloca [100 x i32], align 4
@@ -323,10 +330,9 @@ entry:
   call void @llvm.lifetime.end(i64 -1, i8* %3) nounwind
   ret void
 }
-
-
 ;YESCOLOR: subq  $272, %rsp
 ;NOCOLOR: subq  $272, %rsp
+
 define i32 @myCall_end_before_begin(i32 %in, i1 %d) {
 entry:
   %a = alloca [17 x i8*], align 8
@@ -353,9 +359,8 @@ bb3:
 
 ; Regression test for PR15707.  %buf1 and %buf2 should not be merged
 ; in this test case.
-;YESCOLOR-LABEL: myCall_pr15707:
+;CHECK-LABEL: myCall_pr15707:
 ;YESCOLOR: subq $200008, %rsp
-;NOCOLOR-LABEL: myCall_pr15707:
 ;NOCOLOR: subq $200008, %rsp
 define void @myCall_pr15707() {
   %buf1 = alloca i8, i32 100000, align 16
@@ -374,8 +379,7 @@ define void @myCall_pr15707() {
 
 ; Check that we don't assert and crash even when there are allocas
 ; outside the declared lifetime regions.
-;YESCOLOR-LABEL: bad_range:
-;NOCOLOR-LABEL:  bad_range:
+;CHECK-LABEL: bad_range:
 define void @bad_range() nounwind ssp {
 entry:
   %A.i1 = alloca [100 x i32], align 4
@@ -400,8 +404,7 @@ block2:
 
 ; Check that we don't assert and crash even when there are usages
 ; of allocas which do not read or write outside the declared lifetime regions.
-;YESCOLOR-LABEL: shady_range:
-;NOCOLOR-LABEL:  shady_range:
+;CHECK-LABEL: shady_range:
 
 %struct.Klass = type { i32, i32 }
 
diff --git a/test/CodeGen/X86/absdiff_expand.ll b/test/CodeGen/X86/absdiff_expand.ll
new file mode 100644
index 00000000000..8ba87274d9b
--- /dev/null
+++ b/test/CodeGen/X86/absdiff_expand.ll
@@ -0,0 +1,242 @@
+; RUN: llc -mtriple=x86_64-unknown-linux-gnu  < %s | FileCheck %s -check-prefix=CHECK
+
+declare <4 x i8> @llvm.uabsdiff.v4i8(<4 x i8>, <4 x i8>)
+
+define <4 x i8> @test_uabsdiff_v4i8_expand(<4 x i8> %a1, <4 x i8> %a2) {
+; CHECK-LABEL: test_uabsdiff_v4i8_expand
+; CHECK:             psubd  %xmm1, %xmm0
+; CHECK-NEXT:        pxor   %xmm1, %xmm1
+; CHECK-NEXT:        psubd  %xmm0, %xmm1
+; CHECK-NEXT:        movdqa  .LCPI{{[0-9_]*}}
+; CHECK-NEXT:        movdqa  %xmm1, %xmm3
+; CHECK-NEXT:        pxor   %xmm2, %xmm3
+; CHECK-NEXT:        pcmpgtd        %xmm3, %xmm2
+; CHECK-NEXT:        pand    %xmm2, %xmm0
+; CHECK-NEXT:        pandn   %xmm1, %xmm2
+; CHECK-NEXT:        por     %xmm2, %xmm0
+; CHECK-NEXT:        retq
+
+  %1 = call <4 x i8> @llvm.uabsdiff.v4i8(<4 x i8> %a1, <4 x i8> %a2)
+  ret <4 x i8> %1
+}
+
+declare <4 x i8> @llvm.sabsdiff.v4i8(<4 x i8>, <4 x i8>)
+
+define <4 x i8> @test_sabsdiff_v4i8_expand(<4 x i8> %a1, <4 x i8> %a2) {
+; CHECK-LABEL: test_sabsdiff_v4i8_expand
+; CHECK:      psubd  %xmm1, %xmm0
+; CHECK-NEXT: pxor   %xmm1, %xmm1
+; CHECK-NEXT: pxor    %xmm2, %xmm2
+; CHECK-NEXT: psubd  %xmm0, %xmm2
+; CHECK-NEXT: pcmpgtd  %xmm2, %xmm1
+; CHECK-NEXT: pand    %xmm1, %xmm0
+; CHECK-NEXT: pandn   %xmm2, %xmm1
+; CHECK-NEXT: por     %xmm1, %xmm0
+; CHECK-NEXT: retq
+
+  %1 = call <4 x i8> @llvm.sabsdiff.v4i8(<4 x i8> %a1, <4 x i8> %a2)
+  ret <4 x i8> %1
+}
+
+
+declare <8 x i8> @llvm.sabsdiff.v8i8(<8 x i8>, <8 x i8>)
+
+define <8 x i8> @test_sabsdiff_v8i8_expand(<8 x i8> %a1, <8 x i8> %a2) {
+; CHECK-LABEL: test_sabsdiff_v8i8_expand
+; CHECK:      psubw  %xmm1, %xmm0
+; CHECK-NEXT: pxor   %xmm1, %xmm1
+; CHECK-NEXT: pxor   %xmm2, %xmm2
+; CHECK-NEXT: psubw  %xmm0, %xmm2
+; CHECK-NEXT: pcmpgtw        %xmm2, %xmm1
+; CHECK-NEXT: pand  %xmm1, %xmm0
+; CHECK-NEXT: pandn %xmm2, %xmm1
+; CHECK-NEXT: por  %xmm1, %xmm0
+; CHECK-NEXT: retq
+  %1 = call <8 x i8> @llvm.sabsdiff.v8i8(<8 x i8> %a1, <8 x i8> %a2)
+  ret <8 x i8> %1
+}
+
+declare <16 x i8> @llvm.uabsdiff.v16i8(<16 x i8>, <16 x i8>)
+
+define <16 x i8> @test_uabsdiff_v16i8_expand(<16 x i8> %a1, <16 x i8> %a2) {
+; CHECK-LABEL: test_uabsdiff_v16i8_expand
+; CHECK:             psubb  %xmm1, %xmm0
+; CHECK-NEXT:        pxor   %xmm1, %xmm1
+; CHECK-NEXT:        psubb  %xmm0, %xmm1
+; CHECK-NEXT:        movdqa  .LCPI{{[0-9_]*}}
+; CHECK-NEXT:        movdqa  %xmm1, %xmm3
+; CHECK-NEXT:        pxor   %xmm2, %xmm3
+; CHECK-NEXT:        pcmpgtb        %xmm3, %xmm2
+; CHECK-NEXT:        pand    %xmm2, %xmm0
+; CHECK-NEXT:        pandn   %xmm1, %xmm2
+; CHECK-NEXT:        por     %xmm2, %xmm0
+; CHECK-NEXT:        retq
+  %1 = call <16 x i8> @llvm.uabsdiff.v16i8(<16 x i8> %a1, <16 x i8> %a2)
+  ret <16 x i8> %1
+}
+
+declare <8 x i16> @llvm.uabsdiff.v8i16(<8 x i16>, <8 x i16>)
+
+define <8 x i16> @test_uabsdiff_v8i16_expand(<8 x i16> %a1, <8 x i16> %a2) {
+; CHECK-LABEL: test_uabsdiff_v8i16_expand
+; CHECK:             psubw  %xmm1, %xmm0
+; CHECK-NEXT:        pxor   %xmm1, %xmm1
+; CHECK-NEXT:        psubw  %xmm0, %xmm1
+; CHECK-NEXT:        movdqa  .LCPI{{[0-9_]*}}
+; CHECK-NEXT:        movdqa  %xmm1, %xmm3
+; CHECK-NEXT:        pxor   %xmm2, %xmm3
+; CHECK-NEXT:        pcmpgtw        %xmm3, %xmm2
+; CHECK-NEXT:        pand    %xmm2, %xmm0
+; CHECK-NEXT:        pandn   %xmm1, %xmm2
+; CHECK-NEXT:        por     %xmm2, %xmm0
+; CHECK-NEXT:        retq
+  %1 = call <8 x i16> @llvm.uabsdiff.v8i16(<8 x i16> %a1, <8 x i16> %a2)
+  ret <8 x i16> %1
+}
+
+declare <8 x i16> @llvm.sabsdiff.v8i16(<8 x i16>, <8 x i16>)
+
+define <8 x i16> @test_sabsdiff_v8i16_expand(<8 x i16> %a1, <8 x i16> %a2) {
+; CHECK-LABEL: test_sabsdiff_v8i16_expand
+; CHECK:      psubw  %xmm1, %xmm0
+; CHECK-NEXT: pxor   %xmm1, %xmm1
+; CHECK-NEXT: pxor   %xmm2, %xmm2
+; CHECK-NEXT: psubw  %xmm0, %xmm2
+; CHECK-NEXT: pcmpgtw        %xmm2, %xmm1
+; CHECK-NEXT: pand  %xmm1, %xmm0
+; CHECK-NEXT: pandn %xmm2, %xmm1
+; CHECK-NEXT: por  %xmm1, %xmm0
+; CHECK-NEXT: retq
+  %1 = call <8 x i16> @llvm.sabsdiff.v8i16(<8 x i16> %a1, <8 x i16> %a2)
+  ret <8 x i16> %1
+}
+
+declare <4 x i32> @llvm.sabsdiff.v4i32(<4 x i32>, <4 x i32>)
+
+define <4 x i32> @test_sabsdiff_v4i32_expand(<4 x i32> %a1, <4 x i32> %a2) {
+; CHECK-LABEL: test_sabsdiff_v4i32_expand
+; CHECK:             psubd  %xmm1, %xmm0
+; CHECK-NEXT:        pxor  %xmm1, %xmm1
+; CHECK-NEXT:        pxor  %xmm2, %xmm2
+; CHECK-NEXT:        psubd  %xmm0, %xmm2
+; CHECK-NEXT:        pcmpgtd        %xmm2, %xmm1
+; CHECK-NEXT:        pand    %xmm1, %xmm0
+; CHECK-NEXT:        pandn   %xmm2, %xmm1
+; CHECK-NEXT:        por    %xmm1, %xmm0
+; CHECK-NEXT:        retq
+  %1 = call <4 x i32> @llvm.sabsdiff.v4i32(<4 x i32> %a1, <4 x i32> %a2)
+  ret <4 x i32> %1
+}
+
+declare <4 x i32> @llvm.uabsdiff.v4i32(<4 x i32>, <4 x i32>)
+
+define <4 x i32> @test_uabsdiff_v4i32_expand(<4 x i32> %a1, <4 x i32> %a2) {
+; CHECK-LABEL: test_uabsdiff_v4i32_expand
+; CHECK:             psubd  %xmm1, %xmm0
+; CHECK-NEXT:        pxor   %xmm1, %xmm1
+; CHECK-NEXT:        psubd  %xmm0, %xmm1
+; CHECK-NEXT:        movdqa  .LCPI{{[0-9_]*}}
+; CHECK-NEXT:        movdqa  %xmm1, %xmm3
+; CHECK-NEXT:        pxor   %xmm2, %xmm3
+; CHECK-NEXT:        pcmpgtd        %xmm3, %xmm2
+; CHECK-NEXT:        pand    %xmm2, %xmm0
+; CHECK-NEXT:        pandn   %xmm1, %xmm2
+; CHECK-NEXT:        por     %xmm2, %xmm0
+; CHECK-NEXT:        retq
+  %1 = call <4 x i32> @llvm.uabsdiff.v4i32(<4 x i32> %a1, <4 x i32> %a2)
+  ret <4 x i32> %1
+}
+
+declare <2 x i32> @llvm.sabsdiff.v2i32(<2 x i32>, <2 x i32>)
+
+define <2 x i32> @test_sabsdiff_v2i32_expand(<2 x i32> %a1, <2 x i32> %a2) {
+; CHECK-LABEL: test_sabsdiff_v2i32_expand
+; CHECK:        psubq   %xmm1, %xmm0
+; CHECK-NEXT:   pxor    %xmm1, %xmm1
+; CHECK-NEXT:   psubq   %xmm0, %xmm1
+; CHECK-NEXT:   movdqa  .LCPI{{[0-9_]*}}
+; CHECK-NEXT:   movdqa  %xmm1, %xmm3
+; CHECK-NEXT:   pxor    %xmm2, %xmm3
+; CHECK-NEXT:   movdqa  %xmm2, %xmm4
+; CHECK-NEXT:   pcmpgtd %xmm3, %xmm4
+; CHECK-NEXT:   pshufd  $160, %xmm4, %xmm5      # xmm5 = xmm4[0,0,2,2]
+; CHECK-NEXT:   pcmpeqd %xmm2, %xmm3
+; CHECK-NEXT:   pshufd  $245, %xmm3, %xmm2      # xmm2 = xmm3[1,1,3,3]
+; CHECK-NEXT:   pand    %xmm5, %xmm2
+; CHECK-NEXT:   pshufd  $245, %xmm4, %xmm3      # xmm3 = xmm4[1,1,3,3]
+; CHECK-NEXT:   por     %xmm2, %xmm3
+; CHECK-NEXT:   pand    %xmm3, %xmm0
+; CHECK-NEXT:   pandn   %xmm1, %xmm3
+; CHECK-NEXT:   por     %xmm3, %xmm0
+; CHECK-NEXT:   retq
+  %1 = call <2 x i32> @llvm.sabsdiff.v2i32(<2 x i32> %a1, <2 x i32> %a2)
+  ret <2 x i32> %1
+}
+
+declare <2 x i64> @llvm.sabsdiff.v2i64(<2 x i64>, <2 x i64>)
+
+define <2 x i64> @test_sabsdiff_v2i64_expand(<2 x i64> %a1, <2 x i64> %a2) {
+; CHECK-LABEL: test_sabsdiff_v2i64_expand
+; CHECK:        psubq   %xmm1, %xmm0
+; CHECK-NEXT:   pxor    %xmm1, %xmm1
+; CHECK-NEXT:   psubq   %xmm0, %xmm1
+; CHECK-NEXT:   movdqa  .LCPI{{[0-9_]*}}
+; CHECK-NEXT:   movdqa  %xmm1, %xmm3
+; CHECK-NEXT:   pxor    %xmm2, %xmm3
+; CHECK-NEXT:   movdqa  %xmm2, %xmm4
+; CHECK-NEXT:   pcmpgtd %xmm3, %xmm4
+; CHECK-NEXT:   pshufd  $160, %xmm4, %xmm5      # xmm5 = xmm4[0,0,2,2]
+; CHECK-NEXT:   pcmpeqd %xmm2, %xmm3
+; CHECK-NEXT:   pshufd  $245, %xmm3, %xmm2      # xmm2 = xmm3[1,1,3,3]
+; CHECK-NEXT:   pand    %xmm5, %xmm2
+; CHECK-NEXT:   pshufd  $245, %xmm4, %xmm3      # xmm3 = xmm4[1,1,3,3]
+; CHECK-NEXT:   por     %xmm2, %xmm3
+; CHECK-NEXT:   pand    %xmm3, %xmm0
+; CHECK-NEXT:   pandn   %xmm1, %xmm3
+; CHECK-NEXT:   por     %xmm3, %xmm0
+; CHECK-NEXT:   retq
+  %1 = call <2 x i64> @llvm.sabsdiff.v2i64(<2 x i64> %a1, <2 x i64> %a2)
+  ret <2 x i64> %1
+}
+
+declare <16 x i32> @llvm.sabsdiff.v16i32(<16 x i32>, <16 x i32>)
+
+define <16 x i32> @test_sabsdiff_v16i32_expand(<16 x i32> %a1, <16 x i32> %a2) {
+; CHECK-LABEL: test_sabsdiff_v16i32_expand
+; CHECK:             psubd  %xmm4, %xmm0
+; CHECK-NEXT:        pxor    %xmm8, %xmm8
+; CHECK-NEXT:        pxor    %xmm9, %xmm9
+; CHECK-NEXT:        psubd   %xmm0, %xmm9
+; CHECK-NEXT:        pxor    %xmm4, %xmm4
+; CHECK-NEXT:        pcmpgtd %xmm9, %xmm4
+; CHECK-NEXT:        pand    %xmm4, %xmm0
+; CHECK-NEXT:        pandn   %xmm9, %xmm4
+; CHECK-NEXT:        por     %xmm4, %xmm0
+; CHECK-NEXT:        psubd   %xmm5, %xmm1
+; CHECK-NEXT:        pxor    %xmm4, %xmm4
+; CHECK-NEXT:        psubd   %xmm1, %xmm4
+; CHECK-NEXT:        pxor    %xmm5, %xmm5
+; CHECK-NEXT:        pcmpgtd %xmm4, %xmm5
+; CHECK-NEXT:        pand    %xmm5, %xmm1
+; CHECK-NEXT:        pandn   %xmm4, %xmm5
+; CHECK-NEXT:        por     %xmm5, %xmm1
+; CHECK-NEXT:        psubd   %xmm6, %xmm2
+; CHECK-NEXT:        pxor    %xmm4, %xmm4
+; CHECK-NEXT:        psubd   %xmm2, %xmm4
+; CHECK-NEXT:        pxor    %xmm5, %xmm5
+; CHECK-NEXT:        pcmpgtd %xmm4, %xmm5
+; CHECK-NEXT:        pand    %xmm5, %xmm2
+; CHECK-NEXT:        pandn   %xmm4, %xmm5
+; CHECK-NEXT:        por     %xmm5, %xmm2
+; CHECK-NEXT:        psubd   %xmm7, %xmm3
+; CHECK-NEXT:        pxor    %xmm4, %xmm4
+; CHECK-NEXT:        psubd   %xmm3, %xmm4
+; CHECK-NEXT:        pcmpgtd %xmm4, %xmm8
+; CHECK-NEXT:        pand    %xmm8, %xmm3
+; CHECK-NEXT:        pandn   %xmm4, %xmm8
+; CHECK-NEXT:        por     %xmm8, %xmm3
+; CHECK-NEXT:        retq
+  %1 = call <16 x i32> @llvm.sabsdiff.v16i32(<16 x i32> %a1, <16 x i32> %a2)
+  ret <16 x i32> %1
+}
+
diff --git a/test/CodeGen/X86/addr-mode-matcher.ll b/test/CodeGen/X86/addr-mode-matcher.ll
index 51e1408d985..83d6858bda1 100644
--- a/test/CodeGen/X86/addr-mode-matcher.ll
+++ b/test/CodeGen/X86/addr-mode-matcher.ll
@@ -1,6 +1,4 @@
 ; RUN: llc < %s | FileCheck %s
-; @LOCALMOD
-; RUN: llc -mtriple=x86_64-nacl < %s | FileCheck %s --check-prefix=NACL
 
 ; This testcase used to hit an assert during ISel.  For details, see the big
 ; comment inside the function.
@@ -12,17 +10,6 @@
 ; CHECK-NOT: leal
 ; CHECK: movl {{.*}},4),
 
-; NACL-LABEL: foo:
-; The AND should be turned into a subreg access.
-; NACL-NOT: and
-; The shift should not be folded into the scale of the address in the load.
-; It could in principle be:
-; leal (%rXX, 4), %eXX
-; movl nacl:%(%r15,%rXX), %eXX
-; But it is actually:
-; NACL: shll $2,
-; NACL: movl %nacl:(%r15,%{{...}}),
-
 target datalayout = "e-m:o-p:32:32-f64:32:64-f80:128-n8:16:32-S128"
 target triple = "i386-apple-macosx10.6.0"
 
diff --git a/test/CodeGen/X86/and-encoding.ll b/test/CodeGen/X86/and-encoding.ll
new file mode 100644
index 00000000000..f7bbac2a4bd
--- /dev/null
+++ b/test/CodeGen/X86/and-encoding.ll
@@ -0,0 +1,41 @@
+; RUN: llc -show-mc-encoding < %s | FileCheck %s
+
+; Test that the direct object emission selects the and variant with 8 bit
+; immediate.
+; We used to get this wrong when using direct object emission, but not when
+; reading assembly.
+
+
+target triple = "x86_64-pc-linux"
+
+define void @f1() {
+; CHECK-LABEL: f1:
+; CHECK: andq    $-32, %rsp              # encoding: [0x48,0x83,0xe4,0xe0]
+  %foo = alloca i8, align 32
+  ret void
+}
+
+define void @f2(i1 *%x, i16 *%y) {
+; CHECK-LABEL: f2:
+; CHECK: andl	$1, %eax                # encoding: [0x83,0xe0,0x01]
+  %a = load i1, i1* %x
+  %b = zext i1 %a to i16
+  store i16 %b, i16* %y
+  ret void
+}
+
+define i32 @f3(i1 *%x) {
+; CHECK-LABEL: f3:
+; CHECK: andl	$1, %eax                # encoding: [0x83,0xe0,0x01]
+  %a = load i1, i1* %x
+  %b = zext i1 %a to i32
+  ret i32 %b
+}
+
+define i64 @f4(i1 *%x) {
+; CHECK-LABEL: f4:
+; CHECK: andl	$1, %eax                # encoding: [0x83,0xe0,0x01]
+  %a = load i1, i1* %x
+  %b = zext i1 %a to i64
+  ret i64 %b
+}
diff --git a/test/CodeGen/X86/asm-label2.ll b/test/CodeGen/X86/asm-label2.ll
index 8715aa98ba5..031bd3852e6 100644
--- a/test/CodeGen/X86/asm-label2.ll
+++ b/test/CodeGen/X86/asm-label2.ll
@@ -7,7 +7,7 @@
 ; CHECK:        jmp     LBB0_1
 ; CHECK: LBB0_1:
 
-define void @foobar()  {
+define void @foobar() personality i32 (...)* @__gxx_personality_v0 {
 entry:
   invoke void @_zed()
           to label %invoke.cont unwind label %lpad
@@ -16,7 +16,7 @@ invoke.cont:                                      ; preds = %entry
   ret void
 
 lpad:                                             ; preds = %entry
-  %exn = landingpad {i8*, i32} personality i32 (...)* @__gxx_personality_v0
+  %exn = landingpad {i8*, i32}
             cleanup
   unreachable
 }
diff --git a/test/CodeGen/X86/asm-mismatched-types.ll b/test/CodeGen/X86/asm-mismatched-types.ll
new file mode 100644
index 00000000000..97f9c0872f8
--- /dev/null
+++ b/test/CodeGen/X86/asm-mismatched-types.ll
@@ -0,0 +1,135 @@
+; RUN: llc -o - %s -no-integrated-as | FileCheck %s
+target triple = "x86_64--"
+
+; Allow to specify any of the 8/16/32/64 register names interchangeably in
+; constraints
+
+; Produced by C-programs like this:
+; void foo(int p) { register int reg __asm__("r8") = p;
+; __asm__ __volatile__("# REG: %0" : : "r" (reg)); }
+
+; CHECK-LABEL: reg64_as_32:
+; CHECK: # REG: %r8d
+define void @reg64_as_32(i32 %p) {
+  call void asm sideeffect "# REG: $0", "{r8}"(i32 %p)
+  ret void
+}
+
+; CHECK-LABEL: reg64_as_32_float:
+; CHECK: # REG: %r8d
+define void @reg64_as_32_float(float %p) {
+  call void asm sideeffect "# REG: $0", "{r8}"(float %p)
+  ret void
+}
+
+; CHECK-LABEL: reg64_as_16:
+; CHECK: # REG: %r9w
+define void @reg64_as_16(i16 %p) {
+  call void asm sideeffect "# REG: $0", "{r9}"(i16 %p)
+  ret void
+}
+
+; CHECK-LABEL: reg64_as_8:
+; CHECK: # REG: %bpl
+define void @reg64_as_8(i8 %p) {
+  call void asm sideeffect "# REG: $0", "{rbp}"(i8 %p)
+  ret void
+}
+
+; CHECK-LABEL: reg32_as_16:
+; CHECK: # REG: %r15w
+define void @reg32_as_16(i16 %p) {
+  call void asm sideeffect "# REG: $0", "{r15d}"(i16 %p)
+  ret void
+}
+
+; CHECK-LABEL: reg32_as_8:
+; CHECK: # REG: %r12b
+define void @reg32_as_8(i8 %p) {
+  call void asm sideeffect "# REG: $0", "{r12d}"(i8 %p)
+  ret void
+}
+
+; CHECK-LABEL: reg16_as_8:
+; CHECK: # REG: %cl
+define void @reg16_as_8(i8 %p) {
+  call void asm sideeffect "# REG: $0", "{cx}"(i8 %p)
+  ret void
+}
+
+; CHECK-LABEL: reg32_as_64:
+; CHECK: # REG: %rbp
+define void @reg32_as_64(i64 %p) {
+  call void asm sideeffect "# REG: $0", "{ebp}"(i64 %p)
+  ret void
+}
+
+; CHECK-LABEL: reg32_as_64_float:
+; CHECK: # REG: %rbp
+define void @reg32_as_64_float(double %p) {
+  call void asm sideeffect "# REG: $0", "{ebp}"(double %p)
+  ret void
+}
+
+; CHECK-LABEL: reg16_as_64:
+; CHECK: # REG: %r13
+define void @reg16_as_64(i64 %p) {
+  call void asm sideeffect "# REG: $0", "{r13w}"(i64 %p)
+  ret void
+}
+
+; CHECK-LABEL: reg16_as_64_float:
+; CHECK: # REG: %r13
+define void @reg16_as_64_float(double %p) {
+  call void asm sideeffect "# REG: $0", "{r13w}"(double %p)
+  ret void
+}
+
+; CHECK-LABEL: reg8_as_64:
+; CHECK: # REG: %rax
+define void @reg8_as_64(i64 %p) {
+  call void asm sideeffect "# REG: $0", "{al}"(i64 %p)
+  ret void
+}
+
+; CHECK-LABEL: reg8_as_64_float:
+; CHECK: # REG: %rax
+define void @reg8_as_64_float(double %p) {
+  call void asm sideeffect "# REG: $0", "{al}"(double %p)
+  ret void
+}
+
+; CHECK-LABEL: reg16_as_32:
+; CHECK: # REG: %r11d
+define void @reg16_as_32(i32 %p) {
+  call void asm sideeffect "# REG: $0", "{r11w}"(i32 %p)
+  ret void
+}
+
+; CHECK-LABEL: reg16_as_32_float:
+; CHECK: # REG: %r11d
+define void @reg16_as_32_float(float %p) {
+  call void asm sideeffect "# REG: $0", "{r11w}"(float %p)
+  ret void
+}
+
+; CHECK-LABEL: reg8_as_32:
+; CHECK: # REG: %r9d
+define void @reg8_as_32(i32 %p) {
+  call void asm sideeffect "# REG: $0", "{r9b}"(i32 %p)
+  ret void
+}
+
+; CHECK-LABEL: reg8_as_32_float:
+; CHECK: # REG: %r9d
+define void @reg8_as_32_float(float %p) {
+  call void asm sideeffect "# REG: $0", "{r9b}"(float %p)
+  ret void
+}
+
+; CHECK-LABEL: reg8_as_16:
+; CHECK: # REG: %di
+define void @reg8_as_16(i16 %p) {
+  call void asm sideeffect "# REG: $0", "{dil}"(i16 %p)
+  ret void
+}
diff --git a/test/CodeGen/X86/asm-reject-reg-type-mismatch.ll b/test/CodeGen/X86/asm-reject-reg-type-mismatch.ll
new file mode 100644
index 00000000000..c7e86f565ee
--- /dev/null
+++ b/test/CodeGen/X86/asm-reject-reg-type-mismatch.ll
@@ -0,0 +1,8 @@
+; RUN: not llc -o /dev/null %s 2>&1 | FileCheck %s
+target triple = "x86_64--"
+
+; CHECK: error: couldn't allocate output register for constraint '{ax}'
+define i128 @blup() {
+  %v = tail call i128 asm "", "={ax},0"(i128 0)
+  ret i128 %v
+}
diff --git a/test/CodeGen/X86/atomic-load-store-wide.ll b/test/CodeGen/X86/atomic-load-store-wide.ll
index 5c463978ae5..df0af5f7f27 100644
--- a/test/CodeGen/X86/atomic-load-store-wide.ll
+++ b/test/CodeGen/X86/atomic-load-store-wide.ll
@@ -5,8 +5,7 @@
 
 define void @test1(i64* %ptr, i64 %val1) {
 ; CHECK-LABEL: test1
-; CHECK: lock
-; CHECK-NEXT: cmpxchg8b
+; CHECK: lock cmpxchg8b
 ; CHECK-NEXT: jne
   store atomic i64 %val1, i64* %ptr seq_cst, align 8
   ret void
@@ -14,8 +13,7 @@ define void @test1(i64* %ptr, i64 %val1) {
 
 define i64 @test2(i64* %ptr) {
 ; CHECK-LABEL: test2
-; CHECK: lock
-; CHECK-NEXT: cmpxchg8b
+; CHECK: lock cmpxchg8b
   %val = load atomic i64, i64* %ptr seq_cst, align 8
   ret i64 %val
 }
diff --git a/test/CodeGen/X86/atomic-minmax-i6432.ll b/test/CodeGen/X86/atomic-minmax-i6432.ll
index ffb7a3fd6f6..4989bc14ef8 100644
--- a/test/CodeGen/X86/atomic-minmax-i6432.ll
+++ b/test/CodeGen/X86/atomic-minmax-i6432.ll
@@ -11,8 +11,7 @@ define void @atomic_maxmin_i6432() {
 ; LINUX: seta
 ; LINUX: cmovne
 ; LINUX: cmovne
-; LINUX: lock
-; LINUX-NEXT: cmpxchg8b
+; LINUX: lock cmpxchg8b
 ; LINUX: jne [[LABEL]]
   %2 = atomicrmw min  i64* @sc64, i64 6 acquire
 ; LINUX: [[LABEL:.LBB[0-9]+_[0-9]+]]
@@ -20,8 +19,7 @@ define void @atomic_maxmin_i6432() {
 ; LINUX: setb
 ; LINUX: cmovne
 ; LINUX: cmovne
-; LINUX: lock
-; LINUX-NEXT: cmpxchg8b
+; LINUX: lock cmpxchg8b
 ; LINUX: jne [[LABEL]]
   %3 = atomicrmw umax i64* @sc64, i64 7 acquire
 ; LINUX: [[LABEL:.LBB[0-9]+_[0-9]+]]
@@ -29,8 +27,7 @@ define void @atomic_maxmin_i6432() {
 ; LINUX: seta
 ; LINUX: cmovne
 ; LINUX: cmovne
-; LINUX: lock
-; LINUX-NEXT: cmpxchg8b
+; LINUX: lock cmpxchg8b
 ; LINUX: jne [[LABEL]]
   %4 = atomicrmw umin i64* @sc64, i64 8 acquire
 ; LINUX: [[LABEL:.LBB[0-9]+_[0-9]+]]
@@ -38,8 +35,7 @@ define void @atomic_maxmin_i6432() {
 ; LINUX: setb
 ; LINUX: cmovne
 ; LINUX: cmovne
-; LINUX: lock
-; LINUX-NEXT: cmpxchg8b
+; LINUX: lock cmpxchg8b
 ; LINUX: jne [[LABEL]]
   ret void
 }
diff --git a/test/CodeGen/X86/atomic-or.ll b/test/CodeGen/X86/atomic-or.ll
index 07838468ade..60e9968bdc7 100644
--- a/test/CodeGen/X86/atomic-or.ll
+++ b/test/CodeGen/X86/atomic-or.ll
@@ -9,8 +9,7 @@ entry:
   %tmp = load i64*, i64** %p.addr, align 8
 ; CHECK-LABEL: t1:
 ; CHECK: movl    $2147483648, %eax
-; CHECK: lock
-; CHECK-NEXT: orq %r{{.*}}, (%r{{.*}})
+; CHECK: lock orq %r{{.*}}, (%r{{.*}})
   %0 = atomicrmw or i64* %tmp, i64 2147483648 seq_cst
   ret void
 }
@@ -21,8 +20,7 @@ entry:
   store i64* %p, i64** %p.addr, align 8
   %tmp = load i64*, i64** %p.addr, align 8
 ; CHECK-LABEL: t2:
-; CHECK: lock
-; CHECK-NEXT: orq $2147483644, (%r{{.*}})
+; CHECK: lock orq $2147483644, (%r{{.*}})
   %0 = atomicrmw or i64* %tmp, i64 2147483644 seq_cst
   ret void
 }
diff --git a/test/CodeGen/X86/avx-intrinsics-x86.ll b/test/CodeGen/X86/avx-intrinsics-x86.ll
index 9b6c2ac97bd..28a0272ecf0 100644
--- a/test/CodeGen/X86/avx-intrinsics-x86.ll
+++ b/test/CodeGen/X86/avx-intrinsics-x86.ll
@@ -856,7 +856,7 @@ define void @test_x86_sse2_storel_dq(i8* %a0, <4 x i32> %a1) {
 ; CHECK-LABEL: test_x86_sse2_storel_dq:
 ; CHECK:       # BB#0:
 ; CHECK-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; CHECK-NEXT:    vmovq %xmm0, (%eax)
+; CHECK-NEXT:    vmovlps %xmm0, (%eax)
 ; CHECK-NEXT:    retl
   call void @llvm.x86.sse2.storel.dq(i8* %a0, <4 x i32> %a1)
   ret void
diff --git a/test/CodeGen/X86/avx-isa-check.ll b/test/CodeGen/X86/avx-isa-check.ll
new file mode 100644
index 00000000000..1fd092f7056
--- /dev/null
+++ b/test/CodeGen/X86/avx-isa-check.ll
@@ -0,0 +1,216 @@
+; check AVX2 instructions that are disabled in case avx512VL/avx512BW present
+   
+; RUN: llc < %s -mtriple=x86_64-apple-darwin -show-mc-encoding -mcpu=core-avx2 -mattr=+avx2                 -o /dev/null
+; RUN: llc < %s -mtriple=x86_64-apple-darwin -show-mc-encoding -mcpu=knl                                    -o /dev/null
+; RUN: llc < %s -mtriple=x86_64-apple-darwin -show-mc-encoding -mcpu=knl  -mattr=+avx512vl                  -o /dev/null
+; RUN: llc < %s -mtriple=x86_64-apple-darwin -show-mc-encoding -mcpu=knl  -mattr=+avx512bw                  -o /dev/null
+; RUN: llc < %s -mtriple=x86_64-apple-darwin -show-mc-encoding -mcpu=knl  -mattr=+avx512vl -mattr=+avx512bw -o /dev/null
+; RUN: llc < %s -mtriple=x86_64-apple-darwin -show-mc-encoding -mcpu=skx                                    -o /dev/null
+
+define <4 x i64> @vpand_256(<4 x i64> %a, <4 x i64> %b) nounwind uwtable readnone ssp {
+  ; Force the execution domain with an add.
+  %a2 = add <4 x i64> %a, <i64 1, i64 1, i64 1, i64 1>
+  %x = and <4 x i64> %a2, %b
+  ret <4 x i64> %x
+}
+
+define <2 x i64> @vpand_128(<2 x i64> %a, <2 x i64> %b) nounwind uwtable readnone ssp {
+  ; Force the execution domain with an add.
+  %a2 = add <2 x i64> %a, <i64 1, i64 1>
+  %x = and <2 x i64> %a2, %b
+  ret <2 x i64> %x
+}
+
+define <4 x i64> @vpandn_256(<4 x i64> %a, <4 x i64> %b) nounwind uwtable readnone ssp {
+  ; Force the execution domain with an add.
+  %a2 = add <4 x i64> %a, <i64 1, i64 1, i64 1, i64 1>
+  %y = xor <4 x i64> %a2, <i64 -1, i64 -1, i64 -1, i64 -1>
+  %x = and <4 x i64> %a, %y
+  ret <4 x i64> %x
+}
+
+define <2 x i64> @vpandn_128(<2 x i64> %a, <2 x i64> %b) nounwind uwtable readnone ssp {
+  ; Force the execution domain with an add.
+  %a2 = add <2 x i64> %a, <i64 1, i64 1>
+  %y = xor <2 x i64> %a2, <i64 -1, i64 -1>
+  %x = and <2 x i64> %a, %y
+  ret <2 x i64> %x
+}
+
+define <4 x i64> @vpor_256(<4 x i64> %a, <4 x i64> %b) nounwind uwtable readnone ssp {
+  ; Force the execution domain with an add.
+  %a2 = add <4 x i64> %a, <i64 1, i64 1, i64 1, i64 1>
+  %x = or <4 x i64> %a2, %b
+  ret <4 x i64> %x
+}
+
+define <4 x i64> @vpxor_256(<4 x i64> %a, <4 x i64> %b) nounwind uwtable readnone ssp {
+  ; Force the execution domain with an add.
+  %a2 = add <4 x i64> %a, <i64 1, i64 1, i64 1, i64 1>
+  %x = xor <4 x i64> %a2, %b
+  ret <4 x i64> %x
+}
+
+define <2 x i64> @vpor_128(<2 x i64> %a, <2 x i64> %b) nounwind uwtable readnone ssp {
+  ; Force the execution domain with an add.
+  %a2 = add <2 x i64> %a, <i64 1, i64 1>
+  %x = or <2 x i64> %a2, %b
+  ret <2 x i64> %x
+}
+
+define <2 x i64> @vpxor_128(<2 x i64> %a, <2 x i64> %b) nounwind uwtable readnone ssp {
+  ; Force the execution domain with an add.
+  %a2 = add <2 x i64> %a, <i64 1, i64 1>
+  %x = xor <2 x i64> %a2, %b
+  ret <2 x i64> %x
+}
+
+define <4 x i64> @test_vpaddq_256(<4 x i64> %i, <4 x i64> %j) nounwind readnone {
+  %x = add <4 x i64> %i, %j
+  ret <4 x i64> %x
+}
+
+define <8 x i32> @test_vpaddd_256(<8 x i32> %i, <8 x i32> %j) nounwind readnone {
+  %x = add <8 x i32> %i, %j
+  ret <8 x i32> %x
+}
+
+define <16 x i16> @test_vpaddw_256(<16 x i16> %i, <16 x i16> %j) nounwind readnone {
+  %x = add <16 x i16> %i, %j
+  ret <16 x i16> %x
+}
+
+define <32 x i8> @test_vpaddb_256(<32 x i8> %i, <32 x i8> %j) nounwind readnone {
+  %x = add <32 x i8> %i, %j
+  ret <32 x i8> %x
+}
+
+define <4 x i64> @test_vpsubq_256(<4 x i64> %i, <4 x i64> %j) nounwind readnone {
+  %x = sub <4 x i64> %i, %j
+  ret <4 x i64> %x
+}
+
+define <8 x i32> @test_vpsubd_256(<8 x i32> %i, <8 x i32> %j) nounwind readnone {
+  %x = sub <8 x i32> %i, %j
+  ret <8 x i32> %x
+}
+
+define <16 x i16> @test_vpsubw_256(<16 x i16> %i, <16 x i16> %j) nounwind readnone {
+  %x = sub <16 x i16> %i, %j
+  ret <16 x i16> %x
+}
+
+define <32 x i8> @test_vpsubb_256(<32 x i8> %i, <32 x i8> %j) nounwind readnone {
+  %x = sub <32 x i8> %i, %j
+  ret <32 x i8> %x
+}
+
+define <16 x i16> @test_vpmullw_256(<16 x i16> %i, <16 x i16> %j) nounwind readnone {
+  %x = mul <16 x i16> %i, %j
+  ret <16 x i16> %x
+}
+
+define <8 x i32> @test_vpcmpgtd_256(<8 x i32> %i, <8 x i32> %j) nounwind readnone {
+  %bincmp = icmp slt <8 x i32> %i, %j
+  %x = sext <8 x i1> %bincmp to <8 x i32>
+  ret <8 x i32> %x
+}
+
+define <32 x i8> @test_vpcmpeqb_256(<32 x i8> %i, <32 x i8> %j) nounwind readnone {
+  %bincmp = icmp eq <32 x i8> %i, %j
+  %x = sext <32 x i1> %bincmp to <32 x i8>
+  ret <32 x i8> %x
+}
+
+define <16 x i16> @test_vpcmpeqw_256(<16 x i16> %i, <16 x i16> %j) nounwind readnone {
+  %bincmp = icmp eq <16 x i16> %i, %j
+  %x = sext <16 x i1> %bincmp to <16 x i16>
+  ret <16 x i16> %x
+}
+
+define <32 x i8> @test_vpcmpgtb_256(<32 x i8> %i, <32 x i8> %j) nounwind readnone {
+  %bincmp = icmp slt <32 x i8> %i, %j
+  %x = sext <32 x i1> %bincmp to <32 x i8>
+  ret <32 x i8> %x
+}
+
+define <16 x i16> @test_vpcmpgtw_256(<16 x i16> %i, <16 x i16> %j) nounwind readnone {
+  %bincmp = icmp slt <16 x i16> %i, %j
+  %x = sext <16 x i1> %bincmp to <16 x i16>
+  ret <16 x i16> %x
+}
+
+define <8 x i32> @test_vpcmpeqd_256(<8 x i32> %i, <8 x i32> %j) nounwind readnone {
+  %bincmp = icmp eq <8 x i32> %i, %j
+  %x = sext <8 x i1> %bincmp to <8 x i32>
+  ret <8 x i32> %x
+}
+
+define <2 x i64> @test_vpaddq_128(<2 x i64> %i, <2 x i64> %j) nounwind readnone {
+  %x = add <2 x i64> %i, %j
+  ret <2 x i64> %x
+}
+
+define <4 x i32> @test_vpaddd_128(<4 x i32> %i, <4 x i32> %j) nounwind readnone {
+  %x = add <4 x i32> %i, %j
+  ret <4 x i32> %x
+}
+
+define <8 x i16> @test_vpaddw_128(<8 x i16> %i, <8 x i16> %j) nounwind readnone {
+  %x = add <8 x i16> %i, %j
+  ret <8 x i16> %x
+}
+
+define <16 x i8> @test_vpaddb_128(<16 x i8> %i, <16 x i8> %j) nounwind readnone {
+  %x = add <16 x i8> %i, %j
+  ret <16 x i8> %x
+}
+
+define <2 x i64> @test_vpsubq_128(<2 x i64> %i, <2 x i64> %j) nounwind readnone {
+  %x = sub <2 x i64> %i, %j
+  ret <2 x i64> %x
+}
+
+define <4 x i32> @test_vpsubd_128(<4 x i32> %i, <4 x i32> %j) nounwind readnone {
+  %x = sub <4 x i32> %i, %j
+  ret <4 x i32> %x
+}
+
+define <8 x i16> @test_vpsubw_128(<8 x i16> %i, <8 x i16> %j) nounwind readnone {
+  %x = sub <8 x i16> %i, %j
+  ret <8 x i16> %x
+}
+
+define <16 x i8> @test_vpsubb_128(<16 x i8> %i, <16 x i8> %j) nounwind readnone {
+  %x = sub <16 x i8> %i, %j
+  ret <16 x i8> %x
+}
+
+define <8 x i16> @test_vpmullw_128(<8 x i16> %i, <8 x i16> %j) nounwind readnone {
+  %x = mul <8 x i16> %i, %j
+  ret <8 x i16> %x
+}
+
+define <8 x i16> @test_vpcmpgtw_128(<8 x i16> %i, <8 x i16> %j) nounwind readnone {
+  %bincmp = icmp slt <8 x i16> %i, %j
+  %x = sext <8 x i1> %bincmp to <8 x i16>
+  ret <8 x i16> %x
+}
+
+define <16 x i8> @test_vpcmpgtb_128(<16 x i8> %i, <16 x i8> %j) nounwind readnone {
+  %bincmp = icmp slt <16 x i8> %i, %j
+  %x = sext <16 x i1> %bincmp to <16 x i8>
+  ret <16 x i8> %x
+}
+
+define <8 x i16> @test_vpcmpeqw_128(<8 x i16> %i, <8 x i16> %j) nounwind readnone {
+  %bincmp = icmp eq <8 x i16> %i, %j
+  %x = sext <8 x i1> %bincmp to <8 x i16>
+  ret <8 x i16> %x
+}
+
+define <16 x i8> @test_vpcmpeqb_128(<16 x i8> %i, <16 x i8> %j) nounwind readnone {
+  %bincmp = icmp eq <16 x i8> %i, %j
+  %x = sext <16 x i1> %bincmp to <16 x i8>
+  ret <16 x i8> %x
+}
diff --git a/test/CodeGen/X86/avx-shuffle-x86_32.ll b/test/CodeGen/X86/avx-shuffle-x86_32.ll
index 78b4888cfa1..4bdba37c711 100644
--- a/test/CodeGen/X86/avx-shuffle-x86_32.ll
+++ b/test/CodeGen/X86/avx-shuffle-x86_32.ll
@@ -6,3 +6,14 @@ define <4 x i64> @test1(<4 x i64> %a) nounwind {
  ; CHECK-LABEL: test1:
  ; CHECK-NOT: vinsertf128
  }
+
+define <8 x i16> @test2(<4 x i16>* %v) nounwind {
+; CHECK-LABEL: test2
+; CHECK: vmovsd
+; CHECK: vmovq
+  %v9 = load <4 x i16>, <4 x i16> * %v, align 8
+  %v10 = shufflevector <4 x i16> %v9, <4 x i16> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef>
+  %v11 = shufflevector <8 x i16> <i16 undef, i16 undef, i16 undef, i16 undef, i16 0, i16 0, i16 0, i16 0>, <8 x i16> %v10, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 4, i32 5, i32 6, i32 7>
+  ret <8 x i16> %v11
+}
+
diff --git a/test/CodeGen/X86/avx-vperm2x128.ll b/test/CodeGen/X86/avx-vperm2x128.ll
index 10ed079a264..4e43f6f5192 100644
--- a/test/CodeGen/X86/avx-vperm2x128.ll
+++ b/test/CodeGen/X86/avx-vperm2x128.ll
@@ -147,8 +147,8 @@ define <16 x i16> @E5i(<16 x i16>* %a, <16 x i16>* %b) nounwind uwtable readnone
 ; AVX1-LABEL: E5i:
 ; AVX1:       ## BB#0: ## %entry
 ; AVX1-NEXT:    vmovdqa (%rdi), %ymm0
-; AVX1-NEXT:    vpaddw {{.*}}(%rip), %xmm0, %xmm0
 ; AVX1-NEXT:    vmovaps (%rsi), %ymm1
+; AVX1-NEXT:    vpaddw {{.*}}(%rip), %xmm0, %xmm0
 ; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
 ; AVX1-NEXT:    retq
 ;
@@ -269,7 +269,7 @@ entry:
 define <4 x double> @vperm2z_0x08(<4 x double> %a) {
 ; ALL-LABEL: vperm2z_0x08:
 ; ALL:       # BB#0:
-; ALL-NEXT:    vperm2f128 $40, %ymm0, %ymm0, %ymm0
+; ALL-NEXT:    vperm2f128 {{.*#+}} ymm0 = zero,zero,ymm0[0,1]
 ; ALL-NEXT:    retq
   %s = shufflevector <4 x double> %a, <4 x double> <double 0.0, double 0.0, double undef, double undef>, <4 x i32> <i32 4, i32 5, i32 0, i32 1>
   ret <4 x double> %s
@@ -279,7 +279,7 @@ define <4 x double> @vperm2z_0x18(<4 x double> %a) {
 ; ALL-LABEL: vperm2z_0x18:
 ; ALL:       # BB#0:
 ; ALL-NEXT:    vxorpd %ymm1, %ymm1, %ymm1
-; ALL-NEXT:    vblendpd $12, %ymm0, %ymm1, %ymm0
+; ALL-NEXT:    vblendpd {{.*#+}} ymm0 = ymm1[0,1],ymm0[2,3]
 ; ALL-NEXT:    retq
   %s = shufflevector <4 x double> %a, <4 x double> <double 0.0, double 0.0, double undef, double undef>, <4 x i32> <i32 4, i32 5, i32 2, i32 3>
   ret <4 x double> %s
@@ -288,7 +288,7 @@ define <4 x double> @vperm2z_0x18(<4 x double> %a) {
 define <4 x double> @vperm2z_0x28(<4 x double> %a) {
 ; ALL-LABEL: vperm2z_0x28:
 ; ALL:       # BB#0:
-; ALL-NEXT:    vperm2f128 $40, %ymm0, %ymm0, %ymm0
+; ALL-NEXT:    vperm2f128 {{.*#+}} ymm0 = zero,zero,ymm0[0,1]
 ; ALL-NEXT:    retq
   %s = shufflevector <4 x double> <double 0.0, double 0.0, double undef, double undef>, <4 x double> %a, <4 x i32> <i32 0, i32 1, i32 4, i32 5>
   ret <4 x double> %s
@@ -298,7 +298,7 @@ define <4 x double> @vperm2z_0x38(<4 x double> %a) {
 ; ALL-LABEL: vperm2z_0x38:
 ; ALL:       # BB#0:
 ; ALL-NEXT:    vxorpd %ymm1, %ymm1, %ymm1
-; ALL-NEXT:    vblendpd $12, %ymm0, %ymm1, %ymm0
+; ALL-NEXT:    vblendpd {{.*#+}} ymm0 = ymm1[0,1],ymm0[2,3]
 ; ALL-NEXT:    retq
   %s = shufflevector <4 x double> <double 0.0, double 0.0, double undef, double undef>, <4 x double> %a, <4 x i32> <i32 0, i32 1, i32 6, i32 7>
   ret <4 x double> %s
@@ -307,7 +307,7 @@ define <4 x double> @vperm2z_0x38(<4 x double> %a) {
 define <4 x double> @vperm2z_0x80(<4 x double> %a) {
 ; ALL-LABEL: vperm2z_0x80:
 ; ALL:       # BB#0:
-; ALL-NEXT:    vperm2f128 $128, %ymm0, %ymm0, %ymm0
+; ALL-NEXT:    vperm2f128 {{.*#+}} ymm0 = ymm0[0,1],zero,zero
 ; ALL-NEXT:    retq
   %s = shufflevector <4 x double> %a, <4 x double> <double 0.0, double 0.0, double undef, double undef>, <4 x i32> <i32 0, i32 1, i32 4, i32 5>
   ret <4 x double> %s
@@ -316,7 +316,7 @@ define <4 x double> @vperm2z_0x80(<4 x double> %a) {
 define <4 x double> @vperm2z_0x81(<4 x double> %a) {
 ; ALL-LABEL: vperm2z_0x81:
 ; ALL:       # BB#0:
-; ALL-NEXT:    vperm2f128 $129, %ymm0, %ymm0, %ymm0
+; ALL-NEXT:    vperm2f128 {{.*#+}} ymm0 = ymm0[2,3],zero,zero
 ; ALL-NEXT:    retq
   %s = shufflevector <4 x double> %a, <4 x double> <double 0.0, double 0.0, double undef, double undef>, <4 x i32> <i32 2, i32 3, i32 4, i32 5>
   ret <4 x double> %s
@@ -325,7 +325,7 @@ define <4 x double> @vperm2z_0x81(<4 x double> %a) {
 define <4 x double> @vperm2z_0x82(<4 x double> %a) {
 ; ALL-LABEL: vperm2z_0x82:
 ; ALL:       # BB#0:
-; ALL-NEXT:    vperm2f128 $128, %ymm0, %ymm0, %ymm0
+; ALL-NEXT:    vperm2f128 {{.*#+}} ymm0 = ymm0[0,1],zero,zero
 ; ALL-NEXT:    retq
   %s = shufflevector <4 x double> <double 0.0, double 0.0, double undef, double undef>, <4 x double> %a, <4 x i32> <i32 4, i32 5, i32 0, i32 1>
   ret <4 x double> %s
@@ -334,7 +334,7 @@ define <4 x double> @vperm2z_0x82(<4 x double> %a) {
 define <4 x double> @vperm2z_0x83(<4 x double> %a) {
 ; ALL-LABEL: vperm2z_0x83:
 ; ALL:       # BB#0:
-; ALL-NEXT:    vperm2f128 $129, %ymm0, %ymm0, %ymm0
+; ALL-NEXT:    vperm2f128 {{.*#+}} ymm0 = ymm0[2,3],zero,zero
 ; ALL-NEXT:    retq
   %s = shufflevector <4 x double> <double 0.0, double 0.0, double undef, double undef>, <4 x double> %a, <4 x i32> <i32 6, i32 7, i32 0, i32 1>
   ret <4 x double> %s
@@ -345,8 +345,8 @@ define <4 x double> @vperm2z_0x83(<4 x double> %a) {
 define <4 x i64> @vperm2z_int_0x83(<4 x i64> %a, <4 x i64> %b) {
 ; ALL-LABEL: vperm2z_int_0x83:
 ; ALL:       # BB#0:
-; AVX1:    vperm2f128 $129, %ymm0, %ymm0, %ymm0
-; AVX2:    vperm2i128 $129, %ymm0, %ymm0, %ymm0
+; AVX1:    vperm2f128 {{.*#+}} ymm0 = ymm0[2,3],zero,zero
+; AVX2:    vperm2i128 {{.*#+}} ymm0 = ymm0[2,3],zero,zero
   %s = shufflevector <4 x i64> <i64 0, i64 0, i64 undef, i64 undef>, <4 x i64> %a, <4 x i32> <i32 6, i32 7, i32 0, i32 1>
   %c = add <4 x i64> %b, %s
   ret <4 x i64> %c
diff --git a/test/CodeGen/X86/avx2-arith.ll b/test/CodeGen/X86/avx2-arith.ll
index 72bdd9d0472..a205be1c0cd 100644
--- a/test/CodeGen/X86/avx2-arith.ll
+++ b/test/CodeGen/X86/avx2-arith.ll
@@ -60,6 +60,49 @@ define <16 x i16> @test_vpmullw(<16 x i16> %i, <16 x i16> %j) nounwind readnone
   ret <16 x i16> %x
 }
 
+; CHECK: mul-v16i8
+; CHECK:       # BB#0:
+; CHECK-NEXT:  vpmovsxbw %xmm1, %ymm1
+; CHECK-NEXT:  vpmovsxbw %xmm0, %ymm0
+; CHECK-NEXT:  vpmullw %ymm1, %ymm0, %ymm0
+; CHECK-NEXT:  vextracti128 $1, %ymm0, %xmm1
+; CHECK-NEXT:  vmovdqa {{.*#+}} xmm2 = <0,2,4,6,8,10,12,14,u,u,u,u,u,u,u,u>
+; CHECK-NEXT:  vpshufb %xmm2, %xmm1, %xmm1
+; CHECK-NEXT:  vpshufb %xmm2, %xmm0, %xmm0
+; CHECK-NEXT:  vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
+; CHECK-NEXT:  vzeroupper
+; CHECK-NEXT:  retq
+define <16 x i8> @mul-v16i8(<16 x i8> %i, <16 x i8> %j) nounwind readnone {
+  %x = mul <16 x i8> %i, %j
+  ret <16 x i8> %x
+}
+
+; CHECK: mul-v32i8
+; CHECK:       # BB#0:
+; CHECK-NEXT:  vextracti128 $1, %ymm1, %xmm2
+; CHECK-NEXT:  vpmovsxbw %xmm2, %ymm2
+; CHECK-NEXT:  vextracti128 $1, %ymm0, %xmm3
+; CHECK-NEXT:  vpmovsxbw %xmm3, %ymm3
+; CHECK-NEXT:  vpmullw %ymm2, %ymm3, %ymm2
+; CHECK-NEXT:  vextracti128 $1, %ymm2, %xmm3
+; CHECK-NEXT:  vmovdqa {{.*#+}} xmm4 = <0,2,4,6,8,10,12,14,u,u,u,u,u,u,u,u>
+; CHECK-NEXT:  vpshufb %xmm4, %xmm3, %xmm3
+; CHECK-NEXT:  vpshufb %xmm4, %xmm2, %xmm2
+; CHECK-NEXT:  vpunpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm3[0]
+; CHECK-NEXT:  vpmovsxbw %xmm1, %ymm1
+; CHECK-NEXT:  vpmovsxbw %xmm0, %ymm0
+; CHECK-NEXT:  vpmullw %ymm1, %ymm0, %ymm0
+; CHECK-NEXT:  vextracti128 $1, %ymm0, %xmm1
+; CHECK-NEXT:  vpshufb %xmm4, %xmm1, %xmm1
+; CHECK-NEXT:  vpshufb %xmm4, %xmm0, %xmm0
+; CHECK-NEXT:  vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
+; CHECK-NEXT:  vinserti128 $1, %xmm2, %ymm0, %ymm0
+; CHECK-NEXT:  retq
+define <32 x i8> @mul-v32i8(<32 x i8> %i, <32 x i8> %j) nounwind readnone {
+  %x = mul <32 x i8> %i, %j
+  ret <32 x i8> %x
+}
+
 ; CHECK: mul-v4i64
 ; CHECK: vpmuludq %ymm
 ; CHECK-NEXT: vpsrlq $32, %ymm
diff --git a/test/CodeGen/X86/avx2-conversions.ll b/test/CodeGen/X86/avx2-conversions.ll
index 9b6d5aa5eea..3c8a482f26b 100644
--- a/test/CodeGen/X86/avx2-conversions.ll
+++ b/test/CodeGen/X86/avx2-conversions.ll
@@ -1,136 +1,151 @@
 ; RUN: llc < %s -mtriple=x86_64-apple-darwin -mcpu=core-avx2 -mattr=+avx2 | FileCheck %s
 
-; CHECK: trunc4
-; CHECK: vpermd
-; CHECK-NOT: vinsert
-; CHECK: ret
 define <4 x i32> @trunc4(<4 x i64> %A) nounwind {
+; CHECK-LABEL: trunc4:
+; CHECK:       ## BB#0:
+; CHECK-NEXT:    vmovdqa {{.*#+}} ymm1 = <0,2,4,6,u,u,u,u>
+; CHECK-NEXT:    vpermd %ymm0, %ymm1, %ymm0
+; CHECK-NEXT:    vzeroupper
+; CHECK-NEXT:    retq
   %B = trunc <4 x i64> %A to <4 x i32>
   ret <4 x i32>%B
 }
 
-; CHECK: trunc8
-; CHECK: vpshufb
-; CHECK-NOT: vinsert
-; CHECK: ret
-
 define <8 x i16> @trunc8(<8 x i32> %A) nounwind {
+; CHECK-LABEL: trunc8:
+; CHECK:       ## BB#0:
+; CHECK-NEXT:    vpshufb {{.*#+}} ymm0 = ymm0[0,1,4,5,8,9,12,13],zero,zero,zero,zero,zero,zero,zero,zero,ymm0[16,17,20,21,24,25,28,29],zero,zero,zero,zero,zero,zero,zero,zero
+; CHECK-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,2,2,3]
+; CHECK-NEXT:    vzeroupper
+; CHECK-NEXT:    retq
   %B = trunc <8 x i32> %A to <8 x i16>
   ret <8 x i16>%B
 }
 
-; CHECK: sext4
-; CHECK: vpmovsxdq
-; CHECK-NOT: vinsert
-; CHECK: ret
 define <4 x i64> @sext4(<4 x i32> %A) nounwind {
+; CHECK-LABEL: sext4:
+; CHECK:       ## BB#0:
+; CHECK-NEXT:    vpmovsxdq %xmm0, %ymm0
+; CHECK-NEXT:    retq
   %B = sext <4 x i32> %A to <4 x i64>
   ret <4 x i64>%B
 }
 
-; CHECK: sext8
-; CHECK: vpmovsxwd
-; CHECK-NOT: vinsert
-; CHECK: ret
 define <8 x i32> @sext8(<8 x i16> %A) nounwind {
+; CHECK-LABEL: sext8:
+; CHECK:       ## BB#0:
+; CHECK-NEXT:    vpmovsxwd %xmm0, %ymm0
+; CHECK-NEXT:    retq
   %B = sext <8 x i16> %A to <8 x i32>
   ret <8 x i32>%B
 }
 
-; CHECK: zext4
-; CHECK: vpmovzxdq
-; CHECK-NOT: vinsert
-; CHECK: ret
 define <4 x i64> @zext4(<4 x i32> %A) nounwind {
+; CHECK-LABEL: zext4:
+; CHECK:       ## BB#0:
+; CHECK-NEXT:    vpmovzxdq {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero
+; CHECK-NEXT:    retq
   %B = zext <4 x i32> %A to <4 x i64>
   ret <4 x i64>%B
 }
 
-; CHECK: zext8
-; CHECK: vpmovzxwd
-; CHECK-NOT: vinsert
-; CHECK: ret
 define <8 x i32> @zext8(<8 x i16> %A) nounwind {
+; CHECK-LABEL: zext8:
+; CHECK:       ## BB#0:
+; CHECK-NEXT:    vpmovzxwd {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
+; CHECK-NEXT:    retq
   %B = zext <8 x i16> %A to <8 x i32>
   ret <8 x i32>%B
 }
-; CHECK: zext_8i8_8i32
-; CHECK: vpmovzxwd
-; CHECK: vpand
-; CHECK: ret
+
 define <8 x i32> @zext_8i8_8i32(<8 x i8> %A) nounwind {
-  %B = zext <8 x i8> %A to <8 x i32>  
+; CHECK-LABEL: zext_8i8_8i32:
+; CHECK:       ## BB#0:
+; CHECK-NEXT:    vpmovzxwd {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
+; CHECK-NEXT:    vpbroadcastd {{.*}}(%rip), %ymm1
+; CHECK-NEXT:    vpand %ymm1, %ymm0, %ymm0
+; CHECK-NEXT:    retq
+  %B = zext <8 x i8> %A to <8 x i32>
   ret <8 x i32>%B
 }
 
-; CHECK-LABEL: zext_16i8_16i16:
-; CHECK: vpmovzxbw
-; CHECK-NOT: vinsert
-; CHECK: ret
 define <16 x i16> @zext_16i8_16i16(<16 x i8> %z) {
+; CHECK-LABEL: zext_16i8_16i16:
+; CHECK:       ## BB#0:
+; CHECK-NEXT:    vpmovzxbw {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero,xmm0[8],zero,xmm0[9],zero,xmm0[10],zero,xmm0[11],zero,xmm0[12],zero,xmm0[13],zero,xmm0[14],zero,xmm0[15],zero
+; CHECK-NEXT:    retq
   %t = zext <16 x i8> %z to <16 x i16>
   ret <16 x i16> %t
 }
 
-; CHECK-LABEL: sext_16i8_16i16:
-; CHECK: vpmovsxbw
-; CHECK-NOT: vinsert
-; CHECK: ret
 define <16 x i16> @sext_16i8_16i16(<16 x i8> %z) {
+; CHECK-LABEL: sext_16i8_16i16:
+; CHECK:       ## BB#0:
+; CHECK-NEXT:    vpmovsxbw %xmm0, %ymm0
+; CHECK-NEXT:    retq
   %t = sext <16 x i8> %z to <16 x i16>
   ret <16 x i16> %t
 }
 
-; CHECK-LABEL: trunc_16i16_16i8:
-; CHECK: vpshufb
-; CHECK: vpshufb
-; CHECK: vpunpcklqdq
-; CHECK: ret
 define <16 x i8> @trunc_16i16_16i8(<16 x i16> %z) {
+; CHECK-LABEL: trunc_16i16_16i8:
+; CHECK:       ## BB#0:
+; CHECK-NEXT:    vextracti128 $1, %ymm0, %xmm1
+; CHECK-NEXT:    vmovdqa {{.*#+}} xmm2 = <0,2,4,6,8,10,12,14,u,u,u,u,u,u,u,u>
+; CHECK-NEXT:    vpshufb %xmm2, %xmm1, %xmm1
+; CHECK-NEXT:    vpshufb %xmm2, %xmm0, %xmm0
+; CHECK-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
+; CHECK-NEXT:    vzeroupper
+; CHECK-NEXT:    retq
   %t = trunc <16 x i16> %z to <16 x i8>
   ret <16 x i8> %t
 }
 
-; CHECK: load_sext_test1
-; CHECK: vpmovsxdq (%r{{[^,]*}}), %ymm{{.*}}
-; CHECK: ret 
 define <4 x i64> @load_sext_test1(<4 x i32> *%ptr) {
+; CHECK-LABEL: load_sext_test1:
+; CHECK:       ## BB#0:
+; CHECK-NEXT:    vpmovsxdq (%rdi), %ymm0
+; CHECK-NEXT:    retq
  %X = load <4 x i32>, <4 x i32>* %ptr
  %Y = sext <4 x i32> %X to <4 x i64>
  ret <4 x i64>%Y
 }
 
-; CHECK: load_sext_test2
-; CHECK: vpmovsxbq (%r{{[^,]*}}), %ymm{{.*}}
-; CHECK: ret 
 define <4 x i64> @load_sext_test2(<4 x i8> *%ptr) {
+; CHECK-LABEL: load_sext_test2:
+; CHECK:       ## BB#0:
+; CHECK-NEXT:    vpmovsxbq (%rdi), %ymm0
+; CHECK-NEXT:    retq
  %X = load <4 x i8>, <4 x i8>* %ptr
  %Y = sext <4 x i8> %X to <4 x i64>
  ret <4 x i64>%Y
 }
 
-; CHECK: load_sext_test3
-; CHECK: vpmovsxwq (%r{{[^,]*}}), %ymm{{.*}}
-; CHECK: ret 
 define <4 x i64> @load_sext_test3(<4 x i16> *%ptr) {
+; CHECK-LABEL: load_sext_test3:
+; CHECK:       ## BB#0:
+; CHECK-NEXT:    vpmovsxwq (%rdi), %ymm0
+; CHECK-NEXT:    retq
  %X = load <4 x i16>, <4 x i16>* %ptr
  %Y = sext <4 x i16> %X to <4 x i64>
  ret <4 x i64>%Y
 }
 
-; CHECK: load_sext_test4
-; CHECK: vpmovsxwd (%r{{[^,]*}}), %ymm{{.*}}
-; CHECK: ret 
 define <8 x i32> @load_sext_test4(<8 x i16> *%ptr) {
+; CHECK-LABEL: load_sext_test4:
+; CHECK:       ## BB#0:
+; CHECK-NEXT:    vpmovsxwd (%rdi), %ymm0
+; CHECK-NEXT:    retq
  %X = load <8 x i16>, <8 x i16>* %ptr
  %Y = sext <8 x i16> %X to <8 x i32>
  ret <8 x i32>%Y
 }
 
-; CHECK: load_sext_test5
-; CHECK: vpmovsxbd (%r{{[^,]*}}), %ymm{{.*}}
-; CHECK: ret 
 define <8 x i32> @load_sext_test5(<8 x i8> *%ptr) {
+; CHECK-LABEL: load_sext_test5:
+; CHECK:       ## BB#0:
+; CHECK-NEXT:    vpmovsxbd (%rdi), %ymm0
+; CHECK-NEXT:    retq
  %X = load <8 x i8>, <8 x i8>* %ptr
  %Y = sext <8 x i8> %X to <8 x i32>
  ret <8 x i32>%Y
diff --git a/test/CodeGen/X86/avx2-vector-shifts.ll b/test/CodeGen/X86/avx2-vector-shifts.ll
index e355301dd05..b92b7803500 100644
--- a/test/CodeGen/X86/avx2-vector-shifts.ll
+++ b/test/CodeGen/X86/avx2-vector-shifts.ll
@@ -266,3 +266,162 @@ define <4 x i32> @srl_trunc_and_v4i64(<4 x i32> %x, <4 x i64> %y) nounwind {
   %sra = lshr <4 x i32> %x, %trunc
   ret <4 x i32> %sra
 }
+
+;
+; Vectorized byte shifts
+;
+
+define <8 x i16> @shl_8i16(<8 x i16> %r, <8 x i16> %a) nounwind {
+; CHECK-LABEL:  shl_8i16
+; CHECK:        vpmovzxwd {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
+; CHECK-NEXT:   vpmovzxwd {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
+; CHECK-NEXT:   vpsllvd %ymm1, %ymm0, %ymm0
+; CHECK-NEXT:   vpshufb {{.*#+}} ymm0 = ymm0[0,1,4,5,8,9,12,13],zero,zero,zero,zero,zero,zero,zero,zero,ymm0[16,17,20,21,24,25,28,29],zero,zero,zero,zero,zero,zero,zero,zero
+; CHECK-NEXT:   vpermq {{.*#+}} ymm0 = ymm0[0,2,2,3]
+; CHECK:        retq
+  %shl = shl <8 x i16> %r, %a
+  ret <8 x i16> %shl
+}
+
+define <16 x i16> @shl_16i16(<16 x i16> %r, <16 x i16> %a) nounwind {
+; CHECK-LABEL:  shl_16i16
+; CHECK:        vpxor %ymm2, %ymm2, %ymm2
+; CHECK-NEXT:   vpunpckhwd {{.*#+}} ymm3 = ymm1[4],ymm2[4],ymm1[5],ymm2[5],ymm1[6],ymm2[6],ymm1[7],ymm2[7],ymm1[12],ymm2[12],ymm1[13],ymm2[13],ymm1[14],ymm2[14],ymm1[15],ymm2[15]
+; CHECK-NEXT:   vpunpckhwd {{.*#+}} ymm4 = ymm0[4,4,5,5,6,6,7,7,12,12,13,13,14,14,15,15]
+; CHECK-NEXT:   vpsllvd %ymm3, %ymm4, %ymm3
+; CHECK-NEXT:   vpsrld $16, %ymm3, %ymm3
+; CHECK-NEXT:   vpunpcklwd {{.*#+}} ymm1 = ymm1[0],ymm2[0],ymm1[1],ymm2[1],ymm1[2],ymm2[2],ymm1[3],ymm2[3],ymm1[8],ymm2[8],ymm1[9],ymm2[9],ymm1[10],ymm2[10],ymm1[11],ymm2[11]
+; CHECK-NEXT:   vpunpcklwd {{.*#+}} ymm0 = ymm0[0,0,1,1,2,2,3,3,8,8,9,9,10,10,11,11]
+; CHECK-NEXT:   vpsllvd %ymm1, %ymm0, %ymm0
+; CHECK-NEXT:   vpsrld $16, %ymm0, %ymm0
+; CHECK-NEXT:   vpackusdw %ymm3, %ymm0, %ymm0
+; CHECK-NEXT:   retq
+  %shl = shl <16 x i16> %r, %a
+  ret <16 x i16> %shl
+}
+
+define <32 x i8> @shl_32i8(<32 x i8> %r, <32 x i8> %a) nounwind {
+; CHECK-LABEL:  shl_32i8
+; CHECK:        vpsllw    $5, %ymm1, %ymm1
+; CHECK-NEXT:   vpsllw    $4, %ymm0, %ymm2
+; CHECK-NEXT:   vpand     {{.*}}(%rip), %ymm2, %ymm2
+; CHECK-NEXT:   vpblendvb %ymm1, %ymm2, %ymm0, %ymm0
+; CHECK-NEXT:   vpsllw    $2, %ymm0, %ymm2
+; CHECK-NEXT:   vpand     {{.*}}(%rip), %ymm2, %ymm2
+; CHECK-NEXT:   vpaddb    %ymm1, %ymm1, %ymm1
+; CHECK-NEXT:   vpblendvb %ymm1, %ymm2, %ymm0, %ymm0
+; CHECK-NEXT:   vpaddb    %ymm0, %ymm0, %ymm2
+; CHECK-NEXT:   vpaddb    %ymm1, %ymm1, %ymm1
+; CHECK-NEXT:   vpblendvb %ymm1, %ymm2, %ymm0, %ymm0
+; CHECK-NEXT:   retq
+  %shl = shl <32 x i8> %r, %a
+  ret <32 x i8> %shl
+}
+
+define <8 x i16> @ashr_8i16(<8 x i16> %r, <8 x i16> %a) nounwind {
+; CHECK-LABEL:  ashr_8i16
+; CHECK:        vpmovzxwd {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
+; CHECK-NEXT:   vpmovsxwd %xmm0, %ymm0
+; CHECK-NEXT:   vpsravd %ymm1, %ymm0, %ymm0
+; CHECK-NEXT:   vpshufb {{.*#+}} ymm0 = ymm0[0,1,4,5,8,9,12,13],zero,zero,zero,zero,zero,zero,zero,zero,ymm0[16,17,20,21,24,25,28,29],zero,zero,zero,zero,zero,zero,zero,zero
+; CHECK-NEXT:   vpermq {{.*#+}} ymm0 = ymm0[0,2,2,3]
+; CHECK:        retq
+  %ashr = ashr <8 x i16> %r, %a
+  ret <8 x i16> %ashr
+}
+
+define <16 x i16> @ashr_16i16(<16 x i16> %r, <16 x i16> %a) nounwind {
+; CHECK-LABEL:  ashr_16i16
+; CHECK:        vpxor %ymm2, %ymm2, %ymm2
+; CHECK-NEXT:   vpunpckhwd {{.*#+}} ymm3 = ymm1[4],ymm2[4],ymm1[5],ymm2[5],ymm1[6],ymm2[6],ymm1[7],ymm2[7],ymm1[12],ymm2[12],ymm1[13],ymm2[13],ymm1[14],ymm2[14],ymm1[15],ymm2[15]
+; CHECK-NEXT:   vpunpckhwd {{.*#+}} ymm4 = ymm0[4,4,5,5,6,6,7,7,12,12,13,13,14,14,15,15]
+; CHECK-NEXT:   vpsravd %ymm3, %ymm4, %ymm3
+; CHECK-NEXT:   vpsrld $16, %ymm3, %ymm3
+; CHECK-NEXT:   vpunpcklwd {{.*#+}} ymm1 = ymm1[0],ymm2[0],ymm1[1],ymm2[1],ymm1[2],ymm2[2],ymm1[3],ymm2[3],ymm1[8],ymm2[8],ymm1[9],ymm2[9],ymm1[10],ymm2[10],ymm1[11],ymm2[11]
+; CHECK-NEXT:   vpunpcklwd {{.*#+}} ymm0 = ymm0[0,0,1,1,2,2,3,3,8,8,9,9,10,10,11,11]
+; CHECK-NEXT:   vpsravd %ymm1, %ymm0, %ymm0
+; CHECK-NEXT:   vpsrld $16, %ymm0, %ymm0
+; CHECK-NEXT:   vpackusdw %ymm3, %ymm0, %ymm0
+; CHECK-NEXT:   retq
+  %ashr = ashr <16 x i16> %r, %a
+  ret <16 x i16> %ashr
+}
+
+define <32 x i8> @ashr_32i8(<32 x i8> %r, <32 x i8> %a) nounwind {
+; CHECK-LABEL:  ashr_32i8
+; CHECK:        vpsllw     $5, %ymm1, %ymm1
+; CHECK-NEXT:   vpunpckhbw {{.*#+}} ymm2 = ymm0[8],ymm1[8],ymm0[9],ymm1[9],ymm0[10],ymm1[10],ymm0[11],ymm1[11],ymm0[12],ymm1[12],ymm0[13],ymm1[13],ymm0[14],ymm1[14],ymm0[15],ymm1[15],ymm0[24],ymm1[24],ymm0[25],ymm1[25],ymm0[26],ymm1[26],ymm0[27],ymm1[27],ymm0[28],ymm1[28],ymm0[29],ymm1[29],ymm0[30],ymm1[30],ymm0[31],ymm1[31]
+; CHECK-NEXT:   vpunpckhbw {{.*#+}} ymm3 = ymm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31]
+; CHECK-NEXT:   vpsraw     $4, %ymm3, %ymm4
+; CHECK-NEXT:   vpblendvb  %ymm2, %ymm4, %ymm3, %ymm3
+; CHECK-NEXT:   vpsraw     $2, %ymm3, %ymm4
+; CHECK-NEXT:   vpaddw     %ymm2, %ymm2, %ymm2
+; CHECK-NEXT:   vpblendvb  %ymm2, %ymm4, %ymm3, %ymm3
+; CHECK-NEXT:   vpsraw     $1, %ymm3, %ymm4
+; CHECK-NEXT:   vpaddw     %ymm2, %ymm2, %ymm2
+; CHECK-NEXT:   vpblendvb  %ymm2, %ymm4, %ymm3, %ymm2
+; CHECK-NEXT:   vpsrlw     $8, %ymm2, %ymm2
+; CHECK-NEXT:   vpunpcklbw {{.*#+}} ymm1 = ymm0[0],ymm1[0],ymm0[1],ymm1[1],ymm0[2],ymm1[2],ymm0[3],ymm1[3],ymm0[4],ymm1[4],ymm0[5],ymm1[5],ymm0[6],ymm1[6],ymm0[7],ymm1[7],ymm0[16],ymm1[16],ymm0[17],ymm1[17],ymm0[18],ymm1[18],ymm0[19],ymm1[19],ymm0[20],ymm1[20],ymm0[21],ymm1[21],ymm0[22],ymm1[22],ymm0[23],ymm1[23]
+; CHECK-NEXT:   vpunpcklbw {{.*#+}} ymm0 = ymm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]
+; CHECK-NEXT:   vpsraw     $4, %ymm0, %ymm3
+; CHECK-NEXT:   vpblendvb  %ymm1, %ymm3, %ymm0, %ymm0
+; CHECK-NEXT:   vpsraw     $2, %ymm0, %ymm3
+; CHECK-NEXT:   vpaddw     %ymm1, %ymm1, %ymm1
+; CHECK-NEXT:   vpblendvb  %ymm1, %ymm3, %ymm0, %ymm0
+; CHECK-NEXT:   vpsraw     $1, %ymm0, %ymm3
+; CHECK-NEXT:   vpaddw     %ymm1, %ymm1, %ymm1
+; CHECK-NEXT:   vpblendvb  %ymm1, %ymm3, %ymm0, %ymm0
+; CHECK-NEXT:   vpsrlw     $8, %ymm0, %ymm0
+; CHECK-NEXT:   vpackuswb  %ymm2, %ymm0, %ymm0
+; CHECK-NEXT:   retq
+  %ashr = ashr <32 x i8> %r, %a
+  ret <32 x i8> %ashr
+}
+
+define <8 x i16> @lshr_8i16(<8 x i16> %r, <8 x i16> %a) nounwind {
+; CHECK-LABEL:  lshr_8i16
+; CHECK:        vpmovzxwd {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
+; CHECK-NEXT:   vpmovzxwd {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
+; CHECK-NEXT:   vpsrlvd %ymm1, %ymm0, %ymm0
+; CHECK-NEXT:   vpshufb {{.*#+}} ymm0 = ymm0[0,1,4,5,8,9,12,13],zero,zero,zero,zero,zero,zero,zero,zero,ymm0[16,17,20,21,24,25,28,29],zero,zero,zero,zero,zero,zero,zero,zero
+; CHECK-NEXT:   vpermq {{.*#+}} ymm0 = ymm0[0,2,2,3]
+; CHECK:        retq
+  %lshr = lshr <8 x i16> %r, %a
+  ret <8 x i16> %lshr
+}
+
+define <16 x i16> @lshr_16i16(<16 x i16> %r, <16 x i16> %a) nounwind {
+; CHECK-LABEL:  lshr_16i16
+; CHECK:        vpxor %ymm2, %ymm2, %ymm2
+; CHECK-NEXT:   vpunpckhwd {{.*#+}} ymm3 = ymm1[4],ymm2[4],ymm1[5],ymm2[5],ymm1[6],ymm2[6],ymm1[7],ymm2[7],ymm1[12],ymm2[12],ymm1[13],ymm2[13],ymm1[14],ymm2[14],ymm1[15],ymm2[15]
+; CHECK-NEXT:   vpunpckhwd {{.*#+}} ymm4 = ymm0[4,4,5,5,6,6,7,7,12,12,13,13,14,14,15,15]
+; CHECK-NEXT:   vpsrlvd %ymm3, %ymm4, %ymm3
+; CHECK-NEXT:   vpsrld $16, %ymm3, %ymm3
+; CHECK-NEXT:   vpunpcklwd {{.*#+}} ymm1 = ymm1[0],ymm2[0],ymm1[1],ymm2[1],ymm1[2],ymm2[2],ymm1[3],ymm2[3],ymm1[8],ymm2[8],ymm1[9],ymm2[9],ymm1[10],ymm2[10],ymm1[11],ymm2[11]
+; CHECK-NEXT:   vpunpcklwd {{.*#+}} ymm0 = ymm0[0,0,1,1,2,2,3,3,8,8,9,9,10,10,11,11]
+; CHECK-NEXT:   vpsrlvd %ymm1, %ymm0, %ymm0
+; CHECK-NEXT:   vpsrld $16, %ymm0, %ymm0
+; CHECK-NEXT:   vpackusdw %ymm3, %ymm0, %ymm0
+; CHECK-NEXT:   retq
+  %lshr = lshr <16 x i16> %r, %a
+  ret <16 x i16> %lshr
+}
+
+define <32 x i8> @lshr_32i8(<32 x i8> %r, <32 x i8> %a) nounwind {
+; CHECK-LABEL:  lshr_32i8
+; CHECK:        vpsllw    $5, %ymm1, %ymm1
+; CHECK-NEXT:   vpsrlw    $4, %ymm0, %ymm2
+; CHECK-NEXT:   vpand     {{.*}}(%rip), %ymm2, %ymm2
+; CHECK-NEXT:   vpblendvb %ymm1, %ymm2, %ymm0, %ymm0
+; CHECK-NEXT:   vpsrlw    $2, %ymm0, %ymm2
+; CHECK-NEXT:   vpand     {{.*}}(%rip), %ymm2, %ymm2
+; CHECK-NEXT:   vpaddb    %ymm1, %ymm1, %ymm1
+; CHECK-NEXT:   vpblendvb %ymm1, %ymm2, %ymm0, %ymm0
+; CHECK-NEXT:   vpsrlw    $1, %ymm0, %ymm2
+; CHECK-NEXT:   vpand     {{.*}}(%rip), %ymm2, %ymm2
+; CHECK-NEXT:   vpaddb    %ymm1, %ymm1, %ymm1
+; CHECK-NEXT:   vpblendvb %ymm1, %ymm2, %ymm0, %ymm0
+; CHECK-NEXT:   retq
+  %lshr = lshr <32 x i8> %r, %a
+  ret <32 x i8> %lshr
+}
diff --git a/test/CodeGen/X86/avx512-bugfix-23634.ll b/test/CodeGen/X86/avx512-bugfix-23634.ll
new file mode 100644
index 00000000000..c31a13ad311
--- /dev/null
+++ b/test/CodeGen/X86/avx512-bugfix-23634.ll
@@ -0,0 +1,35 @@
+; RUN: llc < %s -march=x86-64 -mtriple=x86_64-apple-darwin -mcpu=knl | FileCheck %s
+
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+; CHECK-LABEL: f_fu
+; CHECK-NOT: vpblend
+; CHECK: vmovdqa32 {{.*}} {%k1}
+
+define void @f_fu(float* %ret, float*  %aa, float %b) {
+allocas:
+  %ptr_cast_for_load = bitcast float* %aa to <16 x float>*
+  %ptr_masked_load.39 = load <16 x float>, <16 x float>* %ptr_cast_for_load, align 4
+  %b_load_to_int32 = fptosi float %b to i32
+  %b_load_to_int32_broadcast_init = insertelement <16 x i32> undef, i32 %b_load_to_int32, i32 0
+  %b_load_to_int32_broadcast = shufflevector <16 x i32> %b_load_to_int32_broadcast_init, <16 x i32> undef, <16 x i32> zeroinitializer
+  %b_to_int32 = fptosi float %b to i32
+  %b_to_int32_broadcast_init = insertelement <16 x i32> undef, i32 %b_to_int32, i32 0
+  %b_to_int32_broadcast = shufflevector <16 x i32> %b_to_int32_broadcast_init, <16 x i32> undef, <16 x i32> zeroinitializer
+
+  %a_load_to_int32 = fptosi <16 x float> %ptr_masked_load.39 to <16 x i32>
+  %div_v019_load_ = sdiv <16 x i32> %b_to_int32_broadcast, <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>
+
+  %v1.i = select <16 x i1> <i1 false, i1 true, i1 false, i1 true, i1 false, i1 true, i1 false, i1 true, i1 false, i1 true, i1 false, i1 true, i1 false, i1 true, i1 false, i1 true>, <16 x i32> <i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17>, <16 x i32> %a_load_to_int32
+
+  %foo_test = add <16 x i32> %div_v019_load_, %b_load_to_int32_broadcast 
+
+
+  %add_struct_offset_y_struct_offset33_x = add <16 x i32> %foo_test, %v1.i 
+
+  %val = sitofp <16 x i32> %add_struct_offset_y_struct_offset33_x to <16 x float>
+  %ptrcast = bitcast float* %ret to <16 x float>*
+  store <16 x float> %val, <16 x float>* %ptrcast, align 4
+  ret void
+}
\ No newline at end of file
diff --git a/test/CodeGen/X86/avx512-build-vector.ll b/test/CodeGen/X86/avx512-build-vector.ll
index 8373c6da261..e5373c575c1 100644
--- a/test/CodeGen/X86/avx512-build-vector.ll
+++ b/test/CodeGen/X86/avx512-build-vector.ll
@@ -1,19 +1,5 @@
 ; RUN: llc < %s -march=x86-64 -mtriple=x86_64-apple-darwin -mcpu=knl | FileCheck %s
 
-define <16 x i32> @test1(i32* %x) {
-; CHECK-LABEL: test1:
-; CHECK:       ## BB#0:
-; CHECK-NEXT:    vmovd (%rdi), %xmm0
-; CHECK-NEXT:    vinserti128 $1, %xmm0, %ymm0, %ymm0
-; CHECK-NEXT:    vpxor %ymm1, %ymm1, %ymm1
-; CHECK-NEXT:    vpblendd {{.*#+}} ymm0 = ymm1[0,1,2,3],ymm0[4],ymm1[5,6,7]
-; CHECK-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm0
-; CHECK-NEXT:    retq
-   %y = load i32, i32* %x, align 4
-   %res = insertelement <16 x i32>zeroinitializer, i32 %y, i32 4
-   ret <16 x i32>%res
-}
-
 define <16 x i32> @test2(<16 x i32> %x) {
 ; CHECK-LABEL: test2:
 ; CHECK:       ## BB#0:
diff --git a/test/CodeGen/X86/avx512-calling-conv.ll b/test/CodeGen/X86/avx512-calling-conv.ll
new file mode 100644
index 00000000000..edb6bef1a4a
--- /dev/null
+++ b/test/CodeGen/X86/avx512-calling-conv.ll
@@ -0,0 +1,154 @@
+; RUN: llc < %s -mtriple=x86_64-apple-darwin -mcpu=knl | FileCheck %s --check-prefix=KNL
+; RUN: llc < %s -mtriple=x86_64-apple-darwin -mcpu=skx | FileCheck %s --check-prefix=SKX
+; RUN: llc < %s -mtriple=i686-apple-darwin -mcpu=knl | FileCheck %s --check-prefix=KNL_X32
+
+; KNL-LABEL: test1
+; KNL: vxorps
+define <16 x i1> @test1() {
+  ret <16 x i1> zeroinitializer
+}
+
+; SKX-LABEL: test2
+; SKX: vpmovb2m
+; SKX: vpmovb2m
+; SKX: kandw
+; SKX: vpmovm2b
+; KNL-LABEL: test2
+; KNL: vpmovsxbd
+; KNL: vpmovsxbd
+; KNL: vpandd
+; KNL: vpmovdb
+define <16 x i1> @test2(<16 x i1>%a, <16 x i1>%b) {
+  %c = and <16 x i1>%a, %b
+  ret <16 x i1> %c
+}
+
+; SKX-LABEL: test3
+; SKX: vpmovw2m
+; SKX: vpmovw2m
+; SKX: kandb
+; SKX: vpmovm2w
+define <8 x i1> @test3(<8 x i1>%a, <8 x i1>%b) {
+  %c = and <8 x i1>%a, %b
+  ret <8 x i1> %c
+}
+
+; SKX-LABEL: test4
+; SKX: vpmovd2m
+; SKX: vpmovd2m
+; SKX: kandw
+; SKX: vpmovm2d
+define <4 x i1> @test4(<4 x i1>%a, <4 x i1>%b) {
+  %c = and <4 x i1>%a, %b
+  ret <4 x i1> %c
+}
+
+; SKX-LABEL: test5
+; SKX: vpcmpgtd
+; SKX: vpmovm2w
+; SKX: call
+; SKX: vpmovzxwd
+declare <8 x i1> @func8xi1(<8 x i1> %a)
+define <8 x i32> @test5(<8 x i32>%a, <8 x i32>%b) {
+  %cmpRes = icmp sgt <8 x i32>%a, %b
+  %resi = call <8 x i1> @func8xi1(<8 x i1> %cmpRes)
+  %res = sext <8 x i1>%resi to <8 x i32>
+  ret <8 x i32> %res
+}
+
+declare <16 x i1> @func16xi1(<16 x i1> %a)
+
+; KNL-LABEL: test6
+; KNL: vpbroadcastd
+; KNL: vpmovdb
+; KNL: call
+; KNL: vpmovzxbd
+; KNL: vpslld  $31, %zmm
+; KNL: vpsrad  $31, %zmm
+define <16 x i32> @test6(<16 x i32>%a, <16 x i32>%b) {
+  %cmpRes = icmp sgt <16 x i32>%a, %b
+  %resi = call <16 x i1> @func16xi1(<16 x i1> %cmpRes)
+  %res = sext <16 x i1>%resi to <16 x i32>
+  ret <16 x i32> %res
+}
+
+declare <4 x i1> @func4xi1(<4 x i1> %a)
+; SKX-LABEL: test7
+; SKX: vpmovm2d
+; SKX: call
+; SKX: vpslld  $31, %xmm
+; SKX: vpsrad  $31, %xmm
+
+define <4 x i32> @test7(<4 x i32>%a, <4 x i32>%b) {
+  %cmpRes = icmp sgt <4 x i32>%a, %b
+  %resi = call <4 x i1> @func4xi1(<4 x i1> %cmpRes)
+  %res = sext <4 x i1>%resi to <4 x i32>
+  ret <4 x i32> %res
+}
+
+; SKX-LABEL: test7a
+; SKX: call
+; SKX: vpmovw2m  %xmm0, %k0
+; SKX: kandb
+define <8 x i1> @test7a(<8 x i32>%a, <8 x i32>%b) {
+  %cmpRes = icmp sgt <8 x i32>%a, %b
+  %resi = call <8 x i1> @func8xi1(<8 x i1> %cmpRes)
+  %res = and <8 x i1>%resi,  <i1 true, i1 false, i1 true, i1 false, i1 true, i1 false, i1 true, i1 false>
+  ret <8 x i1> %res
+}
+
+
+; KNL_X32-LABEL: test8
+; KNL_X32: testb $1, 4(%esp)
+; KNL_X32:jne
+
+; KNL-LABEL: test8
+; KNL: testb   $1, %dil
+; KNL:jne
+
+define <16 x i8> @test8(<16 x i8> %a1, <16 x i8> %a2, i1 %cond) {
+  %res = select i1 %cond, <16 x i8> %a1, <16 x i8> %a2
+  ret <16 x i8> %res
+}
+
+; KNL-LABEL: test9
+; KNL: vucomisd
+; KNL: setb
+define i1 @test9(double %a, double %b) {
+  %c = fcmp ugt double %a, %b
+  ret i1 %c
+}
+
+; KNL_X32-LABEL: test10
+; KNL_X32: testb $1, 12(%esp)
+; KNL_X32: cmovnel
+
+; KNL-LABEL: test10
+; KNL: testb   $1, %dl
+; KNL: cmovel
+define i32 @test10(i32 %a, i32 %b, i1 %cond) {
+  %c = select i1 %cond, i32 %a, i32 %b
+  ret i32 %c
+}
+
+; KNL-LABEL: test11
+; KNL: cmp
+; KNL: setg
+define i1 @test11(i32 %a, i32 %b) {
+  %c = icmp sgt i32 %a, %b
+  ret i1 %c
+}
+
+; KNL-LABEL: test12
+; KNL: callq _test11
+;; return value in %al
+; KNL: movzbl	%al, %ebx
+; KNL: callq _test10
+; KNL: testb   $1, %bl
+
+define i32 @test12(i32 %a1, i32 %a2, i32 %b1) {
+  %cond = call i1 @test11(i32 %a1, i32 %b1)
+  %res = call i32 @test10(i32 %a1, i32 %a2, i1 %cond)
+  %res1 = select i1 %cond, i32 %res, i32 0
+  ret i32 %res1
+}
\ No newline at end of file
diff --git a/test/CodeGen/X86/avx512-cvt.ll b/test/CodeGen/X86/avx512-cvt.ll
index 842b9f8494b..aa34076c910 100644
--- a/test/CodeGen/X86/avx512-cvt.ll
+++ b/test/CodeGen/X86/avx512-cvt.ll
@@ -1,4 +1,4 @@
-; RUN: llc < %s -mtriple=x86_64-apple-darwin -mcpu=knl --show-mc-encoding | FileCheck %s
+; RUN: llc < %s -mtriple=x86_64-apple-darwin -mcpu=skx --show-mc-encoding | FileCheck %s
 
 ; CHECK-LABEL: sitof32
 ; CHECK: vcvtdq2ps %zmm
@@ -8,6 +8,70 @@ define <16 x float> @sitof32(<16 x i32> %a) nounwind {
   ret <16 x float> %b
 }
 
+; CHECK-LABEL: sltof864
+; CHECK: vcvtqq2pd
+define <8 x double> @sltof864(<8 x i64> %a) {
+  %b = sitofp <8 x i64> %a to <8 x double>
+  ret <8 x double> %b
+}
+
+; CHECK-LABEL: sltof464
+; CHECK: vcvtqq2pd
+define <4 x double> @sltof464(<4 x i64> %a) {
+  %b = sitofp <4 x i64> %a to <4 x double>
+  ret <4 x double> %b
+}
+
+; CHECK-LABEL: sltof2f32
+; CHECK: vcvtqq2ps
+define <2 x float> @sltof2f32(<2 x i64> %a) {
+  %b = sitofp <2 x i64> %a to <2 x float>
+  ret <2 x float>%b
+}
+
+; CHECK-LABEL: sltof4f32_mem
+; CHECK: vcvtqq2psy (%rdi)
+define <4 x float> @sltof4f32_mem(<4 x i64>* %a) {
+  %a1 = load <4 x i64>, <4 x i64>* %a, align 8
+  %b = sitofp <4 x i64> %a1 to <4 x float>
+  ret <4 x float>%b
+}
+
+; CHECK-LABEL: f64tosl
+; CHECK: vcvttpd2qq
+define <4 x i64> @f64tosl(<4 x double> %a) {
+  %b = fptosi <4 x double> %a to <4 x i64>
+  ret <4 x i64> %b
+}
+
+; CHECK-LABEL: f32tosl
+; CHECK: vcvttps2qq
+define <4 x i64> @f32tosl(<4 x float> %a) {
+  %b = fptosi <4 x float> %a to <4 x i64>
+  ret <4 x i64> %b
+}
+
+; CHECK-LABEL: sltof432
+; CHECK: vcvtqq2ps 
+define <4 x float> @sltof432(<4 x i64> %a) {
+  %b = sitofp <4 x i64> %a to <4 x float>
+  ret <4 x float> %b
+}
+
+; CHECK-LABEL: ultof432
+; CHECK: vcvtuqq2ps 
+define <4 x float> @ultof432(<4 x i64> %a) {
+  %b = uitofp <4 x i64> %a to <4 x float>
+  ret <4 x float> %b
+}
+
+; CHECK-LABEL: ultof64
+; CHECK: vcvtuqq2pd
+define <8 x double> @ultof64(<8 x i64> %a) {
+  %b = uitofp <8 x i64> %a to <8 x double>
+  ret <8 x double> %b
+}
+
 ; CHECK-LABEL: fptosi00
 ; CHECK: vcvttps2dq %zmm
 ; CHECK: ret
@@ -64,16 +128,39 @@ define <8 x i32> @fptosi01(<8 x double> %a) {
   ret <8 x i32> %b
 }
 
+; CHECK-LABEL: fptosi03
+; CHECK: vcvttpd2dq %ymm
+; CHECK: ret
+define <4 x i32> @fptosi03(<4 x double> %a) {
+  %b = fptosi <4 x double> %a to <4 x i32>
+  ret <4 x i32> %b
+}
+
 ; CHECK-LABEL: fptrunc00
 ; CHECK: vcvtpd2ps %zmm
 ; CHECK-NEXT: vcvtpd2ps %zmm
-; CHECK-NEXT: vinsertf64x4    $1
+; CHECK-NEXT: vinsertf
 ; CHECK: ret
 define <16 x float> @fptrunc00(<16 x double> %b) nounwind {
   %a = fptrunc <16 x double> %b to <16 x float>
   ret <16 x float> %a
 }
 
+; CHECK-LABEL: fptrunc01
+; CHECK: vcvtpd2ps %ymm
+define <4 x float> @fptrunc01(<4 x double> %b) {
+  %a = fptrunc <4 x double> %b to <4 x float>
+  ret <4 x float> %a
+}
+
+; CHECK-LABEL: fptrunc02
+; CHECK: vcvtpd2ps %ymm0, %xmm0 {%k1} {z}
+define <4 x float> @fptrunc02(<4 x double> %b, <4 x i1> %mask) {
+  %a = fptrunc <4 x double> %b to <4 x float>
+  %c = select <4 x i1>%mask, <4 x float>%a, <4 x float> zeroinitializer
+  ret <4 x float> %c
+}
+
 ; CHECK-LABEL: fpext00
 ; CHECK: vcvtps2pd %ymm0, %zmm0
 ; CHECK: ret
@@ -82,6 +169,16 @@ define <8 x double> @fpext00(<8 x float> %b) nounwind {
   ret <8 x double> %a
 }
 
+; CHECK-LABEL: fpext01
+; CHECK: vcvtps2pd %xmm0, %ymm0 {%k1} {z}
+; CHECK: ret
+define <4 x double> @fpext01(<4 x float> %b, <4 x double>%b1, <4 x double>%a1) {
+  %a = fpext <4 x float> %b to <4 x double>
+  %mask = fcmp ogt <4 x double>%a1, %b1
+  %c = select <4 x i1>%mask,  <4 x double>%a, <4 x double>zeroinitializer
+  ret <4 x double> %c
+}
+
 ; CHECK-LABEL: funcA
 ; CHECK: vcvtsi2sdq (%rdi){{.*}} encoding: [0x62
 ; CHECK: ret
@@ -257,7 +354,7 @@ define double @uitofp03(i32 %a) nounwind {
 }
 
 ; CHECK-LABEL: @sitofp_16i1_float
-; CHECK: vpbroadcastd
+; CHECK: vpmovm2d
 ; CHECK: vcvtdq2ps
 define <16 x float> @sitofp_16i1_float(<16 x i32> %a) {
   %mask = icmp slt <16 x i32> %a, zeroinitializer
@@ -301,10 +398,27 @@ define <8 x double> @sitofp_8i8_double(<8 x i8> %a) {
 
 
 ; CHECK-LABEL: @sitofp_8i1_double
-; CHECK: vpbroadcastq
+; CHECK: vpmovm2d
 ; CHECK: vcvtdq2pd
 define <8 x double> @sitofp_8i1_double(<8 x double> %a) {
   %cmpres = fcmp ogt <8 x double> %a, zeroinitializer
   %1 = sitofp <8 x i1> %cmpres to <8 x double>
   ret <8 x double> %1
 }
+
+; CHECK-LABEL: @uitofp_16i8
+; CHECK:  vpmovzxbd  
+; CHECK: vcvtudq2ps
+define <16 x float> @uitofp_16i8(<16 x i8>%a) {
+  %b = uitofp <16 x i8> %a to <16 x float>
+  ret <16 x float>%b
+}
+
+; CHECK-LABEL: @uitofp_16i16
+; CHECK: vpmovzxwd
+; CHECK: vcvtudq2ps
+define <16 x float> @uitofp_16i16(<16 x i16>%a) {
+  %b = uitofp <16 x i16> %a to <16 x float>
+  ret <16 x float>%b
+}
+
diff --git a/test/CodeGen/X86/avx512-ext.ll b/test/CodeGen/X86/avx512-ext.ll
new file mode 100644
index 00000000000..aa1dd4928c3
--- /dev/null
+++ b/test/CodeGen/X86/avx512-ext.ll
@@ -0,0 +1,937 @@
+; RUN: llc < %s -mtriple=x86_64-apple-darwin -mcpu=knl | FileCheck %s --check-prefix=KNL
+; RUN: llc < %s -mtriple=x86_64-apple-darwin -mcpu=skx | FileCheck %s --check-prefix=SKX 
+ 
+ ;SKX-LABEL: zext_8x8mem_to_8x16:                  
+;SKX:       ## BB#0:
+;SKX-NEXT:  vpmovw2m  %xmm0, %k1     
+;SKX-NEXT:  vpmovzxbw (%rdi), %xmm0 {%k1} {z} 
+;SKX-NEXT:  retq                            
+define <8 x i16> @zext_8x8mem_to_8x16(<8 x i8> *%i , <8 x i1> %mask) nounwind readnone {
+  %a   = load <8 x i8>,<8 x i8> *%i,align 1
+  %x   = zext <8 x i8> %a to <8 x i16>  
+  %ret = select <8 x i1> %mask, <8 x i16> %x, <8 x i16> zeroinitializer  
+  ret <8 x i16> %ret
+}
+
+;SKX-LABEL: sext_8x8mem_to_8x16:                  
+;SKX:       ## BB#0:
+;SKX-NEXT:  vpmovw2m  %xmm0, %k1     
+;SKX-NEXT:  vpmovsxbw (%rdi), %xmm0 {%k1} {z} 
+;SKX-NEXT:  retq                       
+define <8 x i16> @sext_8x8mem_to_8x16(<8 x i8> *%i , <8 x i1> %mask) nounwind readnone {
+  %a   = load <8 x i8>,<8 x i8> *%i,align 1
+  %x   = sext <8 x i8> %a to <8 x i16>  
+  %ret = select <8 x i1> %mask, <8 x i16> %x, <8 x i16> zeroinitializer  
+  ret <8 x i16> %ret
+}
+
+;SKX-LABEL: zext_16x8mem_to_16x16:                
+;SKX:       ## BB#0:
+;SKX-NEXT:  vpmovb2m  %xmm0, %k1     
+;SKX-NEXT:  vpmovzxbw (%rdi), %ymm0 {%k1} {z} 
+;SKX-NEXT:  retq            
+define <16 x i16> @zext_16x8mem_to_16x16(<16 x i8> *%i , <16 x i1> %mask) nounwind readnone {
+  %a   = load <16 x i8>,<16 x i8> *%i,align 1
+  %x   = zext <16 x i8> %a to <16 x i16>  
+  %ret = select <16 x i1> %mask, <16 x i16> %x, <16 x i16> zeroinitializer  
+  ret <16 x i16> %ret
+}
+
+;SKX-LABEL: sext_16x8mem_to_16x16:                
+;SKX:       ## BB#0:
+;SKX-NEXT:  vpmovb2m  %xmm0, %k1     
+;SKX-NEXT:  vpmovsxbw (%rdi), %ymm0 {%k1} {z} 
+;SKX-NEXT:  retq  
+define <16 x i16> @sext_16x8mem_to_16x16(<16 x i8> *%i , <16 x i1> %mask) nounwind readnone {
+  %a   = load <16 x i8>,<16 x i8> *%i,align 1
+  %x   = sext <16 x i8> %a to <16 x i16>  
+  %ret = select <16 x i1> %mask, <16 x i16> %x, <16 x i16> zeroinitializer  
+  ret <16 x i16> %ret
+}
+
+;SKX-LABEL: zext_16x8_to_16x16:                   
+;SKX:       ## BB#0:
+;SKX-NEXT:  vpmovzxbw %xmm0, %ymm0    
+;SKX-NEXT:  retq  
+define <16 x i16> @zext_16x8_to_16x16(<16 x i8> %a ) nounwind readnone {  
+  %x   = zext <16 x i8> %a to <16 x i16>  
+  ret <16 x i16> %x
+}
+
+;SKX-LABEL: zext_16x8_to_16x16_mask:              
+;SKX:       ## BB#0:
+;SKX-NEXT:  vpmovb2m  %xmm1, %k1     
+;SKX-NEXT:  vpmovzxbw %xmm0, %ymm0 {%k1} {z} 
+;SKX-NEXT:  retq 
+define <16 x i16> @zext_16x8_to_16x16_mask(<16 x i8> %a ,<16 x i1> %mask) nounwind readnone {  
+  %x   = zext <16 x i8> %a to <16 x i16> 
+  %ret = select <16 x i1> %mask, <16 x i16> %x, <16 x i16> zeroinitializer   
+  ret <16 x i16> %ret
+}
+
+;SKX-LABEL: sext_16x8_to_16x16:                   
+;SKX:       ## BB#0:
+;SKX-NEXT:  vpmovsxbw %xmm0, %ymm0    
+;SKX-NEXT:  retq
+define <16 x i16> @sext_16x8_to_16x16(<16 x i8> %a ) nounwind readnone {  
+  %x   = sext <16 x i8> %a to <16 x i16>  
+  ret <16 x i16> %x
+}
+
+;SKX-LABEL: sext_16x8_to_16x16_mask:              
+;SKX:       ## BB#0:
+;SKX-NEXT:  vpmovb2m  %xmm1, %k1     
+;SKX-NEXT:  vpmovsxbw %xmm0, %ymm0 {%k1} {z} 
+;SKX-NEXT:  retq 
+define <16 x i16> @sext_16x8_to_16x16_mask(<16 x i8> %a ,<16 x i1> %mask) nounwind readnone {  
+  %x   = sext <16 x i8> %a to <16 x i16> 
+  %ret = select <16 x i1> %mask, <16 x i16> %x, <16 x i16> zeroinitializer   
+  ret <16 x i16> %ret
+}
+
+;SKX-LABEL: zext_32x8mem_to_32x16:                
+;SKX:       ## BB#0:
+;SKX-NEXT:  vpmovb2m  %ymm0, %k1     
+;SKX-NEXT:  vpmovzxbw (%rdi), %zmm0 {%k1} {z} 
+;SKX-NEXT:  retq                       
+define <32 x i16> @zext_32x8mem_to_32x16(<32 x i8> *%i , <32 x i1> %mask) nounwind readnone {
+  %a   = load <32 x i8>,<32 x i8> *%i,align 1
+  %x   = zext <32 x i8> %a to <32 x i16>  
+  %ret = select <32 x i1> %mask, <32 x i16> %x, <32 x i16> zeroinitializer  
+  ret <32 x i16> %ret
+}
+
+;SKX-LABEL: sext_32x8mem_to_32x16:                
+;SKX:       ## BB#0:
+;SKX-NEXT:  vpmovb2m  %ymm0, %k1     
+;SKX-NEXT:  vpmovsxbw (%rdi), %zmm0 {%k1} {z} 
+;SKX-NEXT:  retq
+define <32 x i16> @sext_32x8mem_to_32x16(<32 x i8> *%i , <32 x i1> %mask) nounwind readnone {
+  %a   = load <32 x i8>,<32 x i8> *%i,align 1
+  %x   = sext <32 x i8> %a to <32 x i16>  
+  %ret = select <32 x i1> %mask, <32 x i16> %x, <32 x i16> zeroinitializer  
+  ret <32 x i16> %ret
+}
+
+;SKX-LABEL: zext_32x8_to_32x16:                   
+;SKX:       ## BB#0:
+;SKX-NEXT:  vpmovzxbw %ymm0, %zmm0    
+;SKX-NEXT:  retq 
+define <32 x i16> @zext_32x8_to_32x16(<32 x i8> %a ) nounwind readnone {  
+  %x   = zext <32 x i8> %a to <32 x i16>  
+  ret <32 x i16> %x
+}
+
+;SKX-LABEL: zext_32x8_to_32x16_mask:              
+;SKX:       ## BB#0:
+;SKX-NEXT:  vpmovb2m  %ymm1, %k1    
+;SKX-NEXT:  vpmovzxbw %ymm0, %zmm0 {%k1} {z} 
+;SKX-NEXT:  retq
+define <32 x i16> @zext_32x8_to_32x16_mask(<32 x i8> %a ,<32 x i1> %mask) nounwind readnone {  
+  %x   = zext <32 x i8> %a to <32 x i16>
+  %ret = select <32 x i1> %mask, <32 x i16> %x, <32 x i16> zeroinitializer  
+  ret <32 x i16> %ret
+}
+
+;SKX-LABEL: sext_32x8_to_32x16:                   
+;SKX:       ## BB#0:
+;SKX-NEXT:  vpmovsxbw %ymm0, %zmm0    
+;SKX-NEXT:  retq
+define <32 x i16> @sext_32x8_to_32x16(<32 x i8> %a ) nounwind readnone {  
+  %x   = sext <32 x i8> %a to <32 x i16>  
+  ret <32 x i16> %x
+}
+
+;SKX-LABEL: sext_32x8_to_32x16_mask:              
+;SKX:       ## BB#0:
+;SKX-NEXT:  vpmovb2m  %ymm1, %k1     
+;SKX-NEXT:  vpmovsxbw %ymm0, %zmm0 {%k1} {z} 
+;SKX-NEXT:  retq
+define <32 x i16> @sext_32x8_to_32x16_mask(<32 x i8> %a ,<32 x i1> %mask) nounwind readnone {  
+  %x   = sext <32 x i8> %a to <32 x i16>
+  %ret = select <32 x i1> %mask, <32 x i16> %x, <32 x i16> zeroinitializer  
+  ret <32 x i16> %ret
+}
+
+;SKX-LABEL: zext_4x8mem_to_4x32:
+;SKX:       ## BB#0:
+;SKX-NEXT:  vpmovd2m    %xmm0, %k1      
+;SKX-NEXT:  vpmovzxbd    (%rdi), %xmm0 {%k1} {z} 
+;SKX-NEXT:  retq                            
+define <4 x i32> @zext_4x8mem_to_4x32(<4 x i8> *%i , <4 x i1> %mask) nounwind readnone {
+  %a   = load <4 x i8>,<4 x i8> *%i,align 1
+  %x   = zext <4 x i8> %a to <4 x i32>  
+  %ret = select <4 x i1> %mask, <4 x i32> %x, <4 x i32> zeroinitializer  
+  ret <4 x i32> %ret
+}
+
+;SKX-LABEL: sext_4x8mem_to_4x32:
+;SKX:       ## BB#0:
+;SKX-NEXT:  vpmovd2m    %xmm0, %k1      
+;SKX-NEXT:  vpmovsxbd    (%rdi), %xmm0 {%k1} {z} 
+;SKX-NEXT:  retq       
+define <4 x i32> @sext_4x8mem_to_4x32(<4 x i8> *%i , <4 x i1> %mask) nounwind readnone {
+  %a   = load <4 x i8>,<4 x i8> *%i,align 1
+  %x   = sext <4 x i8> %a to <4 x i32>  
+  %ret = select <4 x i1> %mask, <4 x i32> %x, <4 x i32> zeroinitializer  
+  ret <4 x i32> %ret
+}
+
+;SKX-LABEL: zext_8x8mem_to_8x32:
+;SKX:       ## BB#0:
+;SKX-NEXT:  vpmovw2m    %xmm0, %k1      
+;SKX-NEXT:  vpmovzxbd    (%rdi), %ymm0 {%k1} {z} 
+;SKX-NEXT:  retq    
+define <8 x i32> @zext_8x8mem_to_8x32(<8 x i8> *%i , <8 x i1> %mask) nounwind readnone {
+  %a   = load <8 x i8>,<8 x i8> *%i,align 1
+  %x   = zext <8 x i8> %a to <8 x i32>  
+  %ret = select <8 x i1> %mask, <8 x i32> %x, <8 x i32> zeroinitializer  
+  ret <8 x i32> %ret
+}
+
+;SKX-LABEL: sext_8x8mem_to_8x32:
+;SKX:       ## BB#0:
+;SKX-NEXT:  vpmovw2m    %xmm0, %k1      
+;SKX-NEXT:  vpmovsxbd    (%rdi), %ymm0 {%k1} {z} 
+;SKX-NEXT:  retq         
+define <8 x i32> @sext_8x8mem_to_8x32(<8 x i8> *%i , <8 x i1> %mask) nounwind readnone {
+  %a   = load <8 x i8>,<8 x i8> *%i,align 1
+  %x   = sext <8 x i8> %a to <8 x i32>  
+  %ret = select <8 x i1> %mask, <8 x i32> %x, <8 x i32> zeroinitializer  
+  ret <8 x i32> %ret
+}
+
+;KNL-LABEL: zext_16x8mem_to_16x32:   
+;KNL:       vpmovzxbd    (%rdi), %zmm0 {%k1} {z} 
+;KNL-NEXT:  retq 
+define <16 x i32> @zext_16x8mem_to_16x32(<16 x i8> *%i , <16 x i1> %mask) nounwind readnone {
+  %a   = load <16 x i8>,<16 x i8> *%i,align 1
+  %x   = zext <16 x i8> %a to <16 x i32>
+  %ret = select <16 x i1> %mask, <16 x i32> %x, <16 x i32> zeroinitializer
+  ret <16 x i32> %ret
+}
+
+;KNL-LABEL: sext_16x8mem_to_16x32:   
+;KNL:       vpmovsxbd    (%rdi), %zmm0 {%k1} {z} 
+;KNL-NEXT:  retq  
+define <16 x i32> @sext_16x8mem_to_16x32(<16 x i8> *%i , <16 x i1> %mask) nounwind readnone {
+  %a   = load <16 x i8>,<16 x i8> *%i,align 1
+  %x   = sext <16 x i8> %a to <16 x i32>
+  %ret = select <16 x i1> %mask, <16 x i32> %x, <16 x i32> zeroinitializer
+  ret <16 x i32> %ret
+}
+
+;KNL-LABEL: zext_16x8_to_16x32_mask:                    
+;KNL:       vpmovzxbd %xmm0, %zmm0 {%k1} {z} 
+;KNL-NEXT:  retq                
+define <16 x i32> @zext_16x8_to_16x32_mask(<16 x i8> %a , <16 x i1> %mask) nounwind readnone {
+  %x   = zext <16 x i8> %a to <16 x i32>
+  %ret = select <16 x i1> %mask, <16 x i32> %x, <16 x i32> zeroinitializer
+  ret <16 x i32> %ret
+}
+
+;KNL-LABEL: sext_16x8_to_16x32_mask:                    
+;KNL:       vpmovsxbd %xmm0, %zmm0 {%k1} {z} 
+;KNL-NEXT:  retq
+define <16 x i32> @sext_16x8_to_16x32_mask(<16 x i8> %a , <16 x i1> %mask) nounwind readnone {
+  %x   = sext <16 x i8> %a to <16 x i32>
+  %ret = select <16 x i1> %mask, <16 x i32> %x, <16 x i32> zeroinitializer
+  ret <16 x i32> %ret
+}
+
+; KNL-LABEL: zext_16x8_to_16x32
+; KNL: vpmovzxbd {{.*}}%zmm
+; KNL: ret
+define <16 x i32> @zext_16x8_to_16x32(<16 x i8> %i) nounwind readnone {
+  %x = zext <16 x i8> %i to <16 x i32>
+  ret <16 x i32> %x
+}
+
+; KNL-LABEL: sext_16x8_to_16x32
+; KNL: vpmovsxbd {{.*}}%zmm
+; KNL: ret
+define <16 x i32> @sext_16x8_to_16x32(<16 x i8> %i) nounwind readnone {
+  %x = sext <16 x i8> %i to <16 x i32>
+  ret <16 x i32> %x
+}
+
+;SKX-LABEL: zext_2x8mem_to_2x64:
+;SKX:       ## BB#0:
+;SKX-NEXT:  vpmovq2m  %xmm0, %k1
+;SKX-NEXT:  vpmovzxbq (%rdi), %xmm0 {%k1} {z} 
+;SKX-NEXT:  retq
+define <2 x i64> @zext_2x8mem_to_2x64(<2 x i8> *%i , <2 x i1> %mask) nounwind readnone {
+  %a   = load <2 x i8>,<2 x i8> *%i,align 1
+  %x   = zext <2 x i8> %a to <2 x i64>
+  %ret = select <2 x  i1> %mask, <2 x i64> %x, <2 x i64> zeroinitializer
+  ret <2 x i64> %ret
+}
+;SKX-LABEL: sext_2x8mem_to_2x64mask:
+;SKX:       ## BB#0:
+;SKX-NEXT:  vpmovq2m  %xmm0, %k1
+;SKX-NEXT:  vpmovsxbq (%rdi), %xmm0 {%k1} {z} 
+;SKX-NEXT:  retq
+define <2 x i64> @sext_2x8mem_to_2x64mask(<2 x i8> *%i , <2 x i1> %mask) nounwind readnone {
+  %a   = load <2 x i8>,<2 x i8> *%i,align 1
+  %x   = sext <2 x i8> %a to <2 x i64>
+  %ret = select <2 x i1> %mask, <2 x i64> %x, <2 x i64> zeroinitializer
+  ret <2 x i64> %ret
+}
+;SKX-LABEL: sext_2x8mem_to_2x64:
+;SKX:       ## BB#0:
+;SKX-NEXT:  vpmovsxbq (%rdi), %xmm0   
+;SKX-NEXT:  retq
+define <2 x i64> @sext_2x8mem_to_2x64(<2 x i8> *%i) nounwind readnone {
+  %a   = load <2 x i8>,<2 x i8> *%i,align 1
+  %x   = sext <2 x i8> %a to <2 x i64>
+  ret <2 x i64> %x
+}
+
+;SKX-LABEL: zext_4x8mem_to_4x64:
+;SKX:       ## BB#0:
+;SKX-NEXT:  vpmovd2m  %xmm0, %k1
+;SKX-NEXT:  vpmovzxbq (%rdi), %ymm0 {%k1} {z} 
+;SKX-NEXT:  retq
+define <4 x i64> @zext_4x8mem_to_4x64(<4 x i8> *%i , <4 x i1> %mask) nounwind readnone {
+  %a   = load <4 x i8>,<4 x i8> *%i,align 1
+  %x   = zext <4 x i8> %a to <4 x i64>
+  %ret = select <4 x  i1> %mask, <4 x i64> %x, <4 x i64> zeroinitializer
+  ret <4 x i64> %ret
+}
+
+;SKX-LABEL: sext_4x8mem_to_4x64mask:
+;SKX:       ## BB#0:
+;SKX-NEXT:  vpmovd2m  %xmm0, %k1
+;SKX-NEXT:  vpmovsxbq (%rdi), %ymm0 {%k1} {z} 
+;SKX-NEXT:  retq
+define <4 x i64> @sext_4x8mem_to_4x64mask(<4 x i8> *%i , <4 x i1> %mask) nounwind readnone {
+  %a   = load <4 x i8>,<4 x i8> *%i,align 1
+  %x   = sext <4 x i8> %a to <4 x i64>
+  %ret = select <4 x i1> %mask, <4 x i64> %x, <4 x i64> zeroinitializer
+  ret <4 x i64> %ret
+}
+
+;SKX-LABEL: sext_4x8mem_to_4x64:
+;SKX:       ## BB#0:
+;SKX-NEXT:  vpmovsxbq (%rdi), %ymm0   
+;SKX-NEXT:  retq
+define <4 x i64> @sext_4x8mem_to_4x64(<4 x i8> *%i) nounwind readnone {
+  %a   = load <4 x i8>,<4 x i8> *%i,align 1
+  %x   = sext <4 x i8> %a to <4 x i64>
+  ret <4 x i64> %x
+}
+
+;KNL-LABEL: zext_8x8mem_to_8x64:
+;KNL:       vpmovzxbq (%rdi), %zmm0 {%k1} {z} 
+;KNL-NEXT:  retq
+define <8 x i64> @zext_8x8mem_to_8x64(<8 x i8> *%i , <8 x i1> %mask) nounwind readnone {
+  %a   = load <8 x i8>,<8 x i8> *%i,align 1
+  %x   = zext <8 x i8> %a to <8 x i64>
+  %ret = select <8 x  i1> %mask, <8 x i64> %x, <8 x i64> zeroinitializer
+  ret <8 x i64> %ret
+}
+
+;KNL-LABEL: sext_8x8mem_to_8x64mask:
+;KNL:       vpmovsxbq (%rdi), %zmm0 {%k1} {z} 
+;KNL-NEXT:  retq
+define <8 x i64> @sext_8x8mem_to_8x64mask(<8 x i8> *%i , <8 x i1> %mask) nounwind readnone {
+  %a   = load <8 x i8>,<8 x i8> *%i,align 1
+  %x   = sext <8 x i8> %a to <8 x i64>
+  %ret = select <8 x i1> %mask, <8 x i64> %x, <8 x i64> zeroinitializer
+  ret <8 x i64> %ret
+}
+
+;KNL-LABEL: sext_8x8mem_to_8x64:
+;KNL:       vpmovsxbq (%rdi), %zmm0   
+;KNL-NEXT:  retq
+define <8 x i64> @sext_8x8mem_to_8x64(<8 x i8> *%i) nounwind readnone {
+  %a   = load <8 x i8>,<8 x i8> *%i,align 1
+  %x   = sext <8 x i8> %a to <8 x i64>
+  ret <8 x i64> %x
+}
+
+;SKX-LABEL: zext_4x16mem_to_4x32:
+;SKX:       ## BB#0:
+;SKX-NEXT:  vpmovd2m  %xmm0, %k1
+;SKX-NEXT:  vpmovzxwd (%rdi), %xmm0 {%k1} {z} 
+;SKX-NEXT:  retq
+define <4 x i32> @zext_4x16mem_to_4x32(<4 x i16> *%i , <4 x i1> %mask) nounwind readnone {
+  %a   = load <4 x i16>,<4 x i16> *%i,align 1
+  %x   = zext <4 x i16> %a to <4 x i32>
+  %ret = select <4 x i1> %mask, <4 x i32> %x, <4 x i32> zeroinitializer
+  ret <4 x i32> %ret
+}
+
+;SKX-LABEL: sext_4x16mem_to_4x32mask:
+;SKX:       ## BB#0:
+;SKX-NEXT:  vpmovd2m  %xmm0, %k1
+;SKX-NEXT:  vpmovsxwd (%rdi), %xmm0 {%k1} {z} 
+;SKX-NEXT:  retq
+define <4 x i32> @sext_4x16mem_to_4x32mask(<4 x i16> *%i , <4 x i1> %mask) nounwind readnone {
+  %a   = load <4 x i16>,<4 x i16> *%i,align 1
+  %x   = sext <4 x i16> %a to <4 x i32>
+  %ret = select <4 x i1> %mask, <4 x i32> %x, <4 x i32> zeroinitializer
+  ret <4 x i32> %ret
+}
+
+;SKX-LABEL: sext_4x16mem_to_4x32:
+;SKX:       ## BB#0:
+;SKX-NEXT:  vpmovsxwd (%rdi), %xmm0   
+;SKX-NEXT:  retq
+define <4 x i32> @sext_4x16mem_to_4x32(<4 x i16> *%i) nounwind readnone {
+  %a   = load <4 x i16>,<4 x i16> *%i,align 1
+  %x   = sext <4 x i16> %a to <4 x i32>
+  ret <4 x i32> %x
+}
+
+
+;SKX-LABEL: zext_8x16mem_to_8x32:
+;SKX:       ## BB#0:
+;SKX-NEXT:  vpmovw2m  %xmm0, %k1
+;SKX-NEXT:  vpmovzxwd (%rdi), %ymm0 {%k1} {z} 
+;SKX-NEXT:  retq
+define <8 x i32> @zext_8x16mem_to_8x32(<8 x i16> *%i , <8 x i1> %mask) nounwind readnone {
+  %a   = load <8 x i16>,<8 x i16> *%i,align 1
+  %x   = zext <8 x i16> %a to <8 x i32>
+  %ret = select <8 x i1> %mask, <8 x i32> %x, <8 x i32> zeroinitializer
+  ret <8 x i32> %ret
+}
+
+;SKX-LABEL: sext_8x16mem_to_8x32mask:
+;SKX:       ## BB#0:
+;SKX-NEXT:  vpmovw2m  %xmm0, %k1
+;SKX-NEXT:  vpmovsxwd (%rdi), %ymm0 {%k1} {z} 
+;SKX-NEXT:  retq
+define <8 x i32> @sext_8x16mem_to_8x32mask(<8 x i16> *%i , <8 x i1> %mask) nounwind readnone {
+  %a   = load <8 x i16>,<8 x i16> *%i,align 1
+  %x   = sext <8 x i16> %a to <8 x i32>
+  %ret = select <8 x i1> %mask, <8 x i32> %x, <8 x i32> zeroinitializer
+  ret <8 x i32> %ret
+}
+
+;SKX-LABEL: sext_8x16mem_to_8x32:
+;SKX:       ## BB#0:
+;SKX-NEXT:  vpmovsxwd (%rdi), %ymm0   
+;SKX-NEXT:  retq
+define <8 x i32> @sext_8x16mem_to_8x32(<8 x i16> *%i) nounwind readnone {
+  %a   = load <8 x i16>,<8 x i16> *%i,align 1
+  %x   = sext <8 x i16> %a to <8 x i32>
+  ret <8 x i32> %x
+}
+
+;SKX-LABEL: zext_8x16_to_8x32mask:
+;SKX:       ## BB#0:
+;SKX-NEXT:  vpmovw2m  %xmm1, %k1
+;SKX-NEXT:  vpmovzxwd %xmm0, %ymm0 {%k1} {z} 
+;SKX-NEXT:  retq
+define <8 x i32> @zext_8x16_to_8x32mask(<8 x i16> %a , <8 x i1> %mask) nounwind readnone {
+  %x   = zext <8 x i16> %a to <8 x i32>
+  %ret = select <8 x i1> %mask, <8 x i32> %x, <8 x i32> zeroinitializer
+  ret <8 x i32> %ret
+}
+
+;SKX-LABEL: zext_8x16_to_8x32:
+;SKX:       ## BB#0:
+;SKX-NEXT:  vpmovzxwd %xmm0, %ymm0    
+;SKX-NEXT:  retq
+define <8 x i32> @zext_8x16_to_8x32(<8 x i16> %a ) nounwind readnone {
+  %x   = zext <8 x i16> %a to <8 x i32>
+  ret <8 x i32> %x
+}
+
+;SKX-LABEL: zext_16x16mem_to_16x32:
+;KNL-LABEL: zext_16x16mem_to_16x32:
+;SKX:       ## BB#0:
+;SKX-NEXT:  vpmovb2m  %xmm0, %k1
+;SKX-NEXT:  vpmovzxwd (%rdi), %zmm0 {%k1} {z} 
+;KNL:       vpmovzxwd (%rdi), %zmm0 {%k1} {z} 
+;SKX-NEXT:  retq
+define <16 x i32> @zext_16x16mem_to_16x32(<16 x i16> *%i , <16 x i1> %mask) nounwind readnone {
+  %a   = load <16 x i16>,<16 x i16> *%i,align 1
+  %x   = zext <16 x i16> %a to <16 x i32>
+  %ret = select <16 x i1> %mask, <16 x i32> %x, <16 x i32> zeroinitializer
+  ret <16 x i32> %ret
+}
+
+;SKX-LABEL: sext_16x16mem_to_16x32mask:
+;KNL-LABEL: sext_16x16mem_to_16x32mask:
+;SKX:       ## BB#0:
+;SKX-NEXT:  vpmovb2m  %xmm0, %k1
+;SKX-NEXT:  vpmovsxwd (%rdi), %zmm0 {%k1} {z} 
+;KNL:       vpmovsxwd (%rdi), %zmm0 {%k1} {z} 
+;SKX-NEXT:  retq
+define <16 x i32> @sext_16x16mem_to_16x32mask(<16 x i16> *%i , <16 x i1> %mask) nounwind readnone {
+  %a   = load <16 x i16>,<16 x i16> *%i,align 1
+  %x   = sext <16 x i16> %a to <16 x i32>
+  %ret = select <16 x i1> %mask, <16 x i32> %x, <16 x i32> zeroinitializer
+  ret <16 x i32> %ret
+}
+
+;SKX-LABEL: sext_16x16mem_to_16x32:
+;KNL-LABEL: sext_16x16mem_to_16x32:
+;SKX:       ## BB#0:
+;SKX-NEXT:  vpmovsxwd (%rdi), %zmm0   
+;KNL:       vpmovsxwd (%rdi), %zmm0   
+;SKX-NEXT:  retq
+define <16 x i32> @sext_16x16mem_to_16x32(<16 x i16> *%i) nounwind readnone {
+  %a   = load <16 x i16>,<16 x i16> *%i,align 1
+  %x   = sext <16 x i16> %a to <16 x i32>
+  ret <16 x i32> %x
+}
+;SKX-LABEL: zext_16x16_to_16x32mask:
+;KNL-LABEL: zext_16x16_to_16x32mask:
+;SKX:       ## BB#0:
+;SKX-NEXT:  vpmovb2m  %xmm1, %k1
+;SKX-NEXT:  vpmovzxwd %ymm0, %zmm0 {%k1} {z} 
+;KNL:       vpmovzxwd %ymm0, %zmm0 {%k1} {z} 
+;SKX-NEXT:  retq
+define <16 x i32> @zext_16x16_to_16x32mask(<16 x i16> %a , <16 x i1> %mask) nounwind readnone {
+  %x   = zext <16 x i16> %a to <16 x i32>
+  %ret = select <16 x i1> %mask, <16 x i32> %x, <16 x i32> zeroinitializer
+  ret <16 x i32> %ret
+}
+
+;SKX-LABEL: zext_16x16_to_16x32:
+;KNL-LABEL: zext_16x16_to_16x32:
+;SKX:       ## BB#0:
+;SKX-NEXT:  vpmovzxwd %ymm0, %zmm0    
+;KNL:       vpmovzxwd %ymm0, %zmm0    
+;SKX-NEXT:  retq
+define <16 x i32> @zext_16x16_to_16x32(<16 x i16> %a ) nounwind readnone {
+  %x   = zext <16 x i16> %a to <16 x i32>
+  ret <16 x i32> %x
+}
+
+;SKX-LABEL: zext_2x16mem_to_2x64:
+;SKX:       ## BB#0:
+;SKX-NEXT:  vpmovq2m  %xmm0, %k1
+;SKX-NEXT:  vpmovzxwq (%rdi), %xmm0 {%k1} {z} 
+;SKX-NEXT:  retq
+define <2 x i64> @zext_2x16mem_to_2x64(<2 x i16> *%i , <2 x i1> %mask) nounwind readnone {
+  %a   = load <2 x i16>,<2 x i16> *%i,align 1
+  %x   = zext <2 x i16> %a to <2 x i64>
+  %ret = select <2 x  i1> %mask, <2 x i64> %x, <2 x i64> zeroinitializer
+  ret <2 x i64> %ret
+}
+
+;SKX-LABEL: sext_2x16mem_to_2x64mask:
+;SKX:       ## BB#0:
+;SKX-NEXT:  vpmovq2m  %xmm0, %k1
+;SKX-NEXT:  vpmovsxwq (%rdi), %xmm0 {%k1} {z} 
+;SKX-NEXT:  retq
+define <2 x i64> @sext_2x16mem_to_2x64mask(<2 x i16> *%i , <2 x i1> %mask) nounwind readnone {
+  %a   = load <2 x i16>,<2 x i16> *%i,align 1
+  %x   = sext <2 x i16> %a to <2 x i64>
+  %ret = select <2 x i1> %mask, <2 x i64> %x, <2 x i64> zeroinitializer
+  ret <2 x i64> %ret
+}
+
+;SKX-LABEL: sext_2x16mem_to_2x64:
+;SKX:       ## BB#0:
+;SKX-NEXT:  vpmovsxwq (%rdi), %xmm0   
+;SKX-NEXT:  retq
+define <2 x i64> @sext_2x16mem_to_2x64(<2 x i16> *%i) nounwind readnone {
+  %a   = load <2 x i16>,<2 x i16> *%i,align 1
+  %x   = sext <2 x i16> %a to <2 x i64>
+  ret <2 x i64> %x
+}
+
+;SKX-LABEL: zext_4x16mem_to_4x64:
+;SKX:       ## BB#0:
+;SKX-NEXT:  vpmovd2m  %xmm0, %k1
+;SKX-NEXT:  vpmovzxwq (%rdi), %ymm0 {%k1} {z} 
+;SKX-NEXT:  retq
+define <4 x i64> @zext_4x16mem_to_4x64(<4 x i16> *%i , <4 x i1> %mask) nounwind readnone {
+  %a   = load <4 x i16>,<4 x i16> *%i,align 1
+  %x   = zext <4 x i16> %a to <4 x i64>
+  %ret = select <4 x  i1> %mask, <4 x i64> %x, <4 x i64> zeroinitializer
+  ret <4 x i64> %ret
+}
+
+;SKX-LABEL: sext_4x16mem_to_4x64mask:
+;SKX:       ## BB#0:
+;SKX-NEXT:  vpmovd2m  %xmm0, %k1
+;SKX-NEXT:  vpmovsxwq (%rdi), %ymm0 {%k1} {z} 
+;SKX-NEXT:  retq
+define <4 x i64> @sext_4x16mem_to_4x64mask(<4 x i16> *%i , <4 x i1> %mask) nounwind readnone {
+  %a   = load <4 x i16>,<4 x i16> *%i,align 1
+  %x   = sext <4 x i16> %a to <4 x i64>
+  %ret = select <4 x i1> %mask, <4 x i64> %x, <4 x i64> zeroinitializer
+  ret <4 x i64> %ret
+}
+
+;SKX-LABEL: sext_4x16mem_to_4x64:
+;SKX:       ## BB#0:
+;SKX-NEXT:  vpmovsxwq (%rdi), %ymm0   
+;SKX-NEXT:  retq
+define <4 x i64> @sext_4x16mem_to_4x64(<4 x i16> *%i) nounwind readnone {
+  %a   = load <4 x i16>,<4 x i16> *%i,align 1
+  %x   = sext <4 x i16> %a to <4 x i64>
+  ret <4 x i64> %x
+}
+
+;SKX-LABEL: zext_8x16mem_to_8x64:
+;KNL-LABEL: zext_8x16mem_to_8x64:
+;SKX:       ## BB#0:
+;SKX-NEXT:  vpmovw2m  %xmm0, %k1
+;SKX-NEXT:  vpmovzxwq (%rdi), %zmm0 {%k1} {z} 
+;KNL:       vpmovzxwq (%rdi), %zmm0 {%k1} {z} 
+;SKX-NEXT:  retq
+define <8 x i64> @zext_8x16mem_to_8x64(<8 x i16> *%i , <8 x i1> %mask) nounwind readnone {
+  %a   = load <8 x i16>,<8 x i16> *%i,align 1
+  %x   = zext <8 x i16> %a to <8 x i64>
+  %ret = select <8 x  i1> %mask, <8 x i64> %x, <8 x i64> zeroinitializer
+  ret <8 x i64> %ret
+}
+
+;SKX-LABEL: sext_8x16mem_to_8x64mask:
+;KNL-LABEL: sext_8x16mem_to_8x64mask:
+;SKX:       ## BB#0:
+;SKX-NEXT:  vpmovw2m  %xmm0, %k1
+;SKX-NEXT:  vpmovsxwq (%rdi), %zmm0 {%k1} {z} 
+;KNL:       vpmovsxwq (%rdi), %zmm0 {%k1} {z} 
+;SKX-NEXT:  retq
+define <8 x i64> @sext_8x16mem_to_8x64mask(<8 x i16> *%i , <8 x i1> %mask) nounwind readnone {
+  %a   = load <8 x i16>,<8 x i16> *%i,align 1
+  %x   = sext <8 x i16> %a to <8 x i64>
+  %ret = select <8 x i1> %mask, <8 x i64> %x, <8 x i64> zeroinitializer
+  ret <8 x i64> %ret
+}
+
+;SKX-LABEL: sext_8x16mem_to_8x64:
+;KNL-LABEL: sext_8x16mem_to_8x64:
+;SKX:       ## BB#0:
+;SKX-NEXT:  vpmovsxwq (%rdi), %zmm0   
+;KNL:       vpmovsxwq (%rdi), %zmm0   
+;SKX-NEXT:  retq
+define <8 x i64> @sext_8x16mem_to_8x64(<8 x i16> *%i) nounwind readnone {
+  %a   = load <8 x i16>,<8 x i16> *%i,align 1
+  %x   = sext <8 x i16> %a to <8 x i64>
+  ret <8 x i64> %x
+}
+
+;SKX-LABEL: zext_8x16_to_8x64mask:
+;KNL-LABEL: zext_8x16_to_8x64mask:
+;SKX:       ## BB#0:
+;SKX-NEXT:  vpmovw2m  %xmm1, %k1
+;SKX-NEXT:  vpmovzxwq %xmm0, %zmm0 {%k1} {z} 
+;KNL:       vpmovzxwq %xmm0, %zmm0 {%k1} {z} 
+;SKX-NEXT:  retq
+define <8 x i64> @zext_8x16_to_8x64mask(<8 x i16> %a , <8 x i1> %mask) nounwind readnone {
+  %x   = zext <8 x i16> %a to <8 x i64>
+  %ret = select <8 x  i1> %mask, <8 x i64> %x, <8 x i64> zeroinitializer
+  ret <8 x i64> %ret
+}
+
+;SKX-LABEL: zext_8x16_to_8x64:
+;KNL-LABEL: zext_8x16_to_8x64:
+;SKX:       ## BB#0:
+;SKX-NEXT:  vpmovzxwq %xmm0, %zmm0    
+;KNL:       vpmovzxwq %xmm0, %zmm0    
+;SKX-NEXT:  retq
+; KNL: ret
+define <8 x i64> @zext_8x16_to_8x64(<8 x i16> %a) nounwind readnone {
+  %ret   = zext <8 x i16> %a to <8 x i64>
+  ret <8 x i64> %ret
+}
+
+;SKX-LABEL: zext_2x32mem_to_2x64:
+;SKX:       ## BB#0:
+;SKX-NEXT:  vpmovq2m  %xmm0, %k1
+;SKX-NEXT:  vpmovzxdq (%rdi), %xmm0 {%k1} {z} 
+;SKX-NEXT:  retq
+define <2 x i64> @zext_2x32mem_to_2x64(<2 x i32> *%i , <2 x i1> %mask) nounwind readnone {
+  %a   = load <2 x i32>,<2 x i32> *%i,align 1
+  %x   = zext <2 x i32> %a to <2 x i64>
+  %ret = select <2 x  i1> %mask, <2 x i64> %x, <2 x i64> zeroinitializer
+  ret <2 x i64> %ret
+}
+
+;SKX-LABEL: sext_2x32mem_to_2x64mask:
+;SKX:       ## BB#0:
+;SKX-NEXT:  vpmovq2m  %xmm0, %k1
+;SKX-NEXT:  vpmovsxdq (%rdi), %xmm0 {%k1} {z} 
+;SKX-NEXT:  retq
+define <2 x i64> @sext_2x32mem_to_2x64mask(<2 x i32> *%i , <2 x i1> %mask) nounwind readnone {
+  %a   = load <2 x i32>,<2 x i32> *%i,align 1
+  %x   = sext <2 x i32> %a to <2 x i64>
+  %ret = select <2 x i1> %mask, <2 x i64> %x, <2 x i64> zeroinitializer
+  ret <2 x i64> %ret
+}
+
+;SKX-LABEL: sext_2x32mem_to_2x64:
+;SKX:       ## BB#0:
+;SKX-NEXT:  vpmovsxdq (%rdi), %xmm0   
+;SKX-NEXT:  retq
+define <2 x i64> @sext_2x32mem_to_2x64(<2 x i32> *%i) nounwind readnone {
+  %a   = load <2 x i32>,<2 x i32> *%i,align 1
+  %x   = sext <2 x i32> %a to <2 x i64>
+  ret <2 x i64> %x
+}
+
+;SKX-LABEL: zext_4x32mem_to_4x64:
+;SKX:       ## BB#0:
+;SKX-NEXT:  vpmovd2m  %xmm0, %k1
+;SKX-NEXT:  vpmovzxdq (%rdi), %ymm0 {%k1} {z} 
+;SKX-NEXT:  retq
+define <4 x i64> @zext_4x32mem_to_4x64(<4 x i32> *%i , <4 x i1> %mask) nounwind readnone {
+  %a   = load <4 x i32>,<4 x i32> *%i,align 1
+  %x   = zext <4 x i32> %a to <4 x i64>
+  %ret = select <4 x  i1> %mask, <4 x i64> %x, <4 x i64> zeroinitializer
+  ret <4 x i64> %ret
+}
+
+;SKX-LABEL: sext_4x32mem_to_4x64mask:
+;SKX:       ## BB#0:
+;SKX-NEXT:  vpmovd2m  %xmm0, %k1
+;SKX-NEXT:  vpmovsxdq (%rdi), %ymm0 {%k1} {z} 
+;SKX-NEXT:  retq
+define <4 x i64> @sext_4x32mem_to_4x64mask(<4 x i32> *%i , <4 x i1> %mask) nounwind readnone {
+  %a   = load <4 x i32>,<4 x i32> *%i,align 1
+  %x   = sext <4 x i32> %a to <4 x i64>
+  %ret = select <4 x i1> %mask, <4 x i64> %x, <4 x i64> zeroinitializer
+  ret <4 x i64> %ret
+}
+
+;SKX-LABEL: sext_4x32mem_to_4x64:
+;SKX:       ## BB#0:
+;SKX-NEXT:  vpmovsxdq (%rdi), %ymm0   
+;SKX-NEXT:  retq
+define <4 x i64> @sext_4x32mem_to_4x64(<4 x i32> *%i) nounwind readnone {
+  %a   = load <4 x i32>,<4 x i32> *%i,align 1
+  %x   = sext <4 x i32> %a to <4 x i64>
+  ret <4 x i64> %x
+}
+
+;SKX-LABEL: sext_4x32_to_4x64:
+;SKX:       ## BB#0:
+;SKX-NEXT:  vpmovsxdq %xmm0, %ymm0    
+;SKX-NEXT:  retq
+define <4 x i64> @sext_4x32_to_4x64(<4 x i32> %a) nounwind readnone {
+  %x   = sext <4 x i32> %a to <4 x i64>
+  ret <4 x i64> %x
+}
+
+;SKX-LABEL: zext_4x32_to_4x64mask:
+;SKX:       ## BB#0:
+;SKX-NEXT:  vpmovd2m  %xmm1, %k1
+;SKX-NEXT:  vpmovzxdq %xmm0, %ymm0 {%k1} {z} 
+;SKX-NEXT:  retq
+define <4 x i64> @zext_4x32_to_4x64mask(<4 x i32> %a , <4 x i1> %mask) nounwind readnone {
+  %x   = zext <4 x i32> %a to <4 x i64>
+  %ret = select <4 x  i1> %mask, <4 x i64> %x, <4 x i64> zeroinitializer
+  ret <4 x i64> %ret
+}
+
+;SKX-LABEL: zext_8x32mem_to_8x64:
+;SKX:       ## BB#0:
+;SKX-NEXT:  vpmovw2m  %xmm0, %k1
+;SKX-NEXT:  vpmovzxdq (%rdi), %zmm0 {%k1} {z} 
+;SKX-NEXT:  retq
+define <8 x i64> @zext_8x32mem_to_8x64(<8 x i32> *%i , <8 x i1> %mask) nounwind readnone {
+  %a   = load <8 x i32>,<8 x i32> *%i,align 1
+  %x   = zext <8 x i32> %a to <8 x i64>
+  %ret = select <8 x  i1> %mask, <8 x i64> %x, <8 x i64> zeroinitializer
+  ret <8 x i64> %ret
+}
+
+;SKX-LABEL: sext_8x32mem_to_8x64mask:
+;SKX:       ## BB#0:
+;SKX-NEXT:  vpmovw2m  %xmm0, %k1
+;SKX-NEXT:  vpmovsxdq (%rdi), %zmm0 {%k1} {z} 
+;SKX-NEXT:  retq
+define <8 x i64> @sext_8x32mem_to_8x64mask(<8 x i32> *%i , <8 x i1> %mask) nounwind readnone {
+  %a   = load <8 x i32>,<8 x i32> *%i,align 1
+  %x   = sext <8 x i32> %a to <8 x i64>
+  %ret = select <8 x i1> %mask, <8 x i64> %x, <8 x i64> zeroinitializer
+  ret <8 x i64> %ret
+}
+
+;SKX-LABEL: sext_8x32mem_to_8x64:
+;KNL-LABEL: sext_8x32mem_to_8x64:
+;SKX:       ## BB#0:
+;SKX-NEXT:  vpmovsxdq (%rdi), %zmm0   
+;KNL:       vpmovsxdq (%rdi), %zmm0   
+;SKX-NEXT:  retq
+define <8 x i64> @sext_8x32mem_to_8x64(<8 x i32> *%i) nounwind readnone {
+  %a   = load <8 x i32>,<8 x i32> *%i,align 1
+  %x   = sext <8 x i32> %a to <8 x i64>
+  ret <8 x i64> %x
+}
+
+;SKX-LABEL: sext_8x32_to_8x64:
+;KNL-LABEL: sext_8x32_to_8x64:
+;SKX:       ## BB#0:
+;SKX-NEXT:  vpmovsxdq %ymm0, %zmm0    
+;KNL:       vpmovsxdq %ymm0, %zmm0    
+;SKX-NEXT:  retq
+define <8 x i64> @sext_8x32_to_8x64(<8 x i32> %a) nounwind readnone {
+  %x   = sext <8 x i32> %a to <8 x i64>
+  ret <8 x i64> %x
+}
+
+;SKX-LABEL: zext_8x32_to_8x64mask:
+;KNL-LABEL: zext_8x32_to_8x64mask:
+;SKX:       ## BB#0:
+;SKX-NEXT:  vpmovw2m  %xmm1, %k1
+;SKX-NEXT:  vpmovzxdq %ymm0, %zmm0 {%k1} {z} 
+;KNL:       vpmovzxdq %ymm0, %zmm0 {%k1} {z} 
+;SKX-NEXT:  retq
+define <8 x i64> @zext_8x32_to_8x64mask(<8 x i32> %a , <8 x i1> %mask) nounwind readnone {
+  %x   = zext <8 x i32> %a to <8 x i64>
+  %ret = select <8 x  i1> %mask, <8 x i64> %x, <8 x i64> zeroinitializer
+  ret <8 x i64> %ret
+}
+;KNL-LABEL: fptrunc_test
+;KNL: vcvtpd2ps {{.*}}%zmm
+;KNL: ret
+define <8 x float> @fptrunc_test(<8 x double> %a) nounwind readnone {
+  %b = fptrunc <8 x double> %a to <8 x float>
+  ret <8 x float> %b
+}
+
+;KNL-LABEL: fpext_test
+;KNL: vcvtps2pd {{.*}}%zmm
+;KNL: ret
+define <8 x double> @fpext_test(<8 x float> %a) nounwind readnone {
+  %b = fpext <8 x float> %a to <8 x double>
+  ret <8 x double> %b
+}
+
+; KNL-LABEL: zext_16i1_to_16xi32
+; KNL: vpbroadcastd LCP{{.*}}(%rip), %zmm0 {%k1} {z}
+; KNL: ret
+define   <16 x i32> @zext_16i1_to_16xi32(i16 %b) {
+  %a = bitcast i16 %b to <16 x i1>
+  %c = zext <16 x i1> %a to <16 x i32>
+  ret <16 x i32> %c
+}
+
+; KNL-LABEL: zext_8i1_to_8xi64
+; KNL: vpbroadcastq LCP{{.*}}(%rip), %zmm0 {%k1} {z}
+; KNL: ret
+define   <8 x i64> @zext_8i1_to_8xi64(i8 %b) {
+  %a = bitcast i8 %b to <8 x i1>
+  %c = zext <8 x i1> %a to <8 x i64>
+  ret <8 x i64> %c
+}
+
+; KNL-LABEL: trunc_16i8_to_16i1
+; KNL: vpmovsxbd
+; KNL: vpandd
+; KNL: vptestmd
+; KNL: ret
+; SKX-LABEL: trunc_16i8_to_16i1
+; SKX: vpmovb2m %xmm
+define i16 @trunc_16i8_to_16i1(<16 x i8> %a) {
+  %mask_b = trunc <16 x i8>%a to <16 x i1>
+  %mask = bitcast <16 x i1> %mask_b to i16
+  ret i16 %mask
+}
+
+; KNL-LABEL: trunc_16i32_to_16i1
+; KNL: vpandd
+; KNL: vptestmd
+; KNL: ret
+; SKX-LABEL: trunc_16i32_to_16i1
+; SKX: vpmovd2m %zmm
+define i16 @trunc_16i32_to_16i1(<16 x i32> %a) {
+  %mask_b = trunc <16 x i32>%a to <16 x i1>
+  %mask = bitcast <16 x i1> %mask_b to i16
+  ret i16 %mask
+}
+
+; SKX-LABEL: trunc_4i32_to_4i1
+; SKX: vpmovd2m        %xmm
+; SKX: kandw
+; SKX: vpmovm2d
+define <4 x i32> @trunc_4i32_to_4i1(<4 x i32> %a, <4 x i32> %b) {
+  %mask_a = trunc <4 x i32>%a to <4 x i1>
+  %mask_b = trunc <4 x i32>%b to <4 x i1>
+  %a_and_b = and <4 x i1>%mask_a, %mask_b
+  %res = sext <4 x i1>%a_and_b to <4 x i32>
+  ret <4 x i32>%res
+}
+
+; KNL-LABEL: trunc_8i16_to_8i1
+; KNL: vpmovsxwq
+; KNL: vpandq LCP{{.*}}(%rip){1to8}
+; KNL: vptestmq
+; KNL: ret
+
+; SKX-LABEL: trunc_8i16_to_8i1
+; SKX: vpmovw2m %xmm
+define i8 @trunc_8i16_to_8i1(<8 x i16> %a) {
+  %mask_b = trunc <8 x i16>%a to <8 x i1>
+  %mask = bitcast <8 x i1> %mask_b to i8
+  ret i8 %mask
+}
+
+; KNL-LABEL: sext_8i1_8i32
+; KNL: vpbroadcastq  LCP{{.*}}(%rip), %zmm0 {%k1} {z}
+; SKX: vpmovm2d
+; KNL: ret
+define <8 x i32> @sext_8i1_8i32(<8 x i32> %a1, <8 x i32> %a2) nounwind {
+  %x = icmp slt <8 x i32> %a1, %a2
+  %x1 = xor <8 x i1>%x, <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>
+  %y = sext <8 x i1> %x1 to <8 x i32>
+  ret <8 x i32> %y
+}
+
+
+; KNL-LABEL: trunc_i32_to_i1
+; KNL: movw    $-4, %ax
+; KNL: kmovw   %eax, %k1
+; KNL: korw
+define i16 @trunc_i32_to_i1(i32 %a) {
+  %a_i = trunc i32 %a to i1
+  %maskv = insertelement <16 x i1> <i1 true, i1 false, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>, i1 %a_i, i32 0
+  %res = bitcast <16 x i1> %maskv to i16
+  ret i16 %res
+}
+
+; KNL-LABEL: sext_8i1_8i16
+; SKX: vpmovm2w
+; KNL: ret
+define <8 x i16> @sext_8i1_8i16(<8 x i32> %a1, <8 x i32> %a2) nounwind {
+  %x = icmp slt <8 x i32> %a1, %a2
+  %y = sext <8 x i1> %x to <8 x i16>
+  ret <8 x i16> %y
+}
+
+; KNL-LABEL: sext_16i1_16i32
+; SKX: vpmovm2d
+; KNL: ret
+define <16 x i32> @sext_16i1_16i32(<16 x i32> %a1, <16 x i32> %a2) nounwind {
+  %x = icmp slt <16 x i32> %a1, %a2
+  %y = sext <16 x i1> %x to <16 x i32>
+  ret <16 x i32> %y
+}
+
+; KNL-LABEL: sext_8i1_8i64
+; SKX: vpmovm2q
+; KNL: ret
+define <8 x i64> @sext_8i1_8i64(<8 x i32> %a1, <8 x i32> %a2) nounwind {
+  %x = icmp slt <8 x i32> %a1, %a2
+  %y = sext <8 x i1> %x to <8 x i64>
+  ret <8 x i64> %y
+}
+
+; KNL-LABEL: @extload_v8i64
+; KNL: vpmovsxbq
+define void @extload_v8i64(<8 x i8>* %a, <8 x i64>* %res) {
+  %sign_load = load <8 x i8>, <8 x i8>* %a
+  %c = sext <8 x i8> %sign_load to <8 x i64>
+  store <8 x i64> %c, <8 x i64>* %res
+  ret void
+}
+
+;SKX-LABEL: test21:
+;SKX:       vmovdqu16 %zmm0, %zmm3 {%k1}
+;SKX-NEXT:  kshiftrq  $32, %k1, %k1
+;SKX-NEXT:  vmovdqu16 %zmm1, %zmm2 {%k1}
+define <64 x i16> @test21(<64 x i16> %x , <64 x i1> %mask) nounwind readnone {
+  %ret = select <64 x i1> %mask, <64 x i16> %x, <64 x i16> zeroinitializer
+  ret <64 x i16> %ret
+}  
+
diff --git a/test/CodeGen/X86/avx512-fma-intrinsics.ll b/test/CodeGen/X86/avx512-fma-intrinsics.ll
index 9814a610827..c30fc909f09 100644
--- a/test/CodeGen/X86/avx512-fma-intrinsics.ll
+++ b/test/CodeGen/X86/avx512-fma-intrinsics.ll
@@ -1,422 +1,675 @@
-; RUN: llc < %s -mtriple=x86_64-apple-darwin -mcpu=knl --show-mc-encoding | FileCheck %s
+; RUN: llc < %s -mtriple=x86_64-apple-darwin -mattr=+avx512f --show-mc-encoding | FileCheck %s
 
-declare <16 x float> @llvm.x86.fma.mask.vfmadd.ps.512(<16 x float>, <16 x float>, <16 x float>, i16, i32)
-declare <8 x double> @llvm.x86.fma.mask.vfmadd.pd.512(<8 x double>, <8 x double>, <8 x double>, i8, i32)
-declare <16 x float> @llvm.x86.fma.mask.vfmsub.ps.512(<16 x float>, <16 x float>, <16 x float>, i16, i32)
-
-define <8 x double> @test_x86_vfmsubpd_z(<8 x double> %a0, <8 x double> %a1, <8 x double> %a2) {
-  ; CHECK-LABEL: test_x86_vfmsubpd_z
-  ; CHECK: vfmsub213pd %zmm
-  %res = call <8 x double> @llvm.x86.fma.mask.vfmsub.pd.512(<8 x double> %a0, <8 x double> %a1, <8 x double> %a2, i8 -1, i32 4) nounwind
-  ret <8 x double> %res
-}
-declare <8 x double> @llvm.x86.fma.mask.vfmsub.pd.512(<8 x double>, <8 x double>, <8 x double>, i8, i32) nounwind readnone
-
-define <8 x double> @test_mask_vfmsub_pd(<8 x double> %a0, <8 x double> %a1, <8 x double> %a2, i8 %mask) {
-  ; CHECK-LABEL: test_mask_vfmsub_pd
-  ; CHECK: vfmsub213pd %zmm
-  %res = call <8 x double> @llvm.x86.fma.mask.vfmsub.pd.512(<8 x double> %a0, <8 x double> %a1, <8 x double> %a2, i8 %mask, i32 4) nounwind
-  ret <8 x double> %res
-}
+declare <16 x float> @llvm.x86.avx512.mask.vfmadd.ps.512(<16 x float>, <16 x float>, <16 x float>, i16, i32)
+declare <8 x double> @llvm.x86.avx512.mask.vfmadd.pd.512(<8 x double>, <8 x double>, <8 x double>, i8, i32)
 
 define <16 x float> @test_x86_vfnmadd_ps_z(<16 x float> %a0, <16 x float> %a1, <16 x float> %a2) {
   ; CHECK-LABEL: test_x86_vfnmadd_ps_z
   ; CHECK: vfnmadd213ps %zmm
-  %res = call <16 x float> @llvm.x86.fma.mask.vfnmadd.ps.512(<16 x float> %a0, <16 x float> %a1, <16 x float> %a2, i16 -1, i32 4) nounwind
+  %res = call <16 x float> @llvm.x86.avx512.mask.vfnmadd.ps.512(<16 x float> %a0, <16 x float> %a1, <16 x float> %a2, i16 -1, i32 4) nounwind
   ret <16 x float> %res
 }
-declare <16 x float> @llvm.x86.fma.mask.vfnmadd.ps.512(<16 x float>, <16 x float>, <16 x float>, i16, i32) nounwind readnone
+declare <16 x float> @llvm.x86.avx512.mask.vfnmadd.ps.512(<16 x float>, <16 x float>, <16 x float>, i16, i32) nounwind readnone
 
 define <16 x float> @test_mask_vfnmadd_ps(<16 x float> %a0, <16 x float> %a1, <16 x float> %a2, i16 %mask) {
   ; CHECK-LABEL: test_mask_vfnmadd_ps
   ; CHECK: vfnmadd213ps %zmm
-  %res = call <16 x float> @llvm.x86.fma.mask.vfnmadd.ps.512(<16 x float> %a0, <16 x float> %a1, <16 x float> %a2, i16 %mask, i32 4) nounwind
+  %res = call <16 x float> @llvm.x86.avx512.mask.vfnmadd.ps.512(<16 x float> %a0, <16 x float> %a1, <16 x float> %a2, i16 %mask, i32 4) nounwind
   ret <16 x float> %res
 }
 
 define <8 x double> @test_x86_vfnmadd_pd_z(<8 x double> %a0, <8 x double> %a1, <8 x double> %a2) {
   ; CHECK-LABEL: test_x86_vfnmadd_pd_z
   ; CHECK: vfnmadd213pd %zmm
-  %res = call <8 x double> @llvm.x86.fma.mask.vfnmadd.pd.512(<8 x double> %a0, <8 x double> %a1, <8 x double> %a2, i8 -1, i32 4) nounwind
+  %res = call <8 x double> @llvm.x86.avx512.mask.vfnmadd.pd.512(<8 x double> %a0, <8 x double> %a1, <8 x double> %a2, i8 -1, i32 4) nounwind
   ret <8 x double> %res
 }
-declare <8 x double> @llvm.x86.fma.mask.vfnmadd.pd.512(<8 x double>, <8 x double>, <8 x double>, i8, i32) nounwind readnone
+declare <8 x double> @llvm.x86.avx512.mask.vfnmadd.pd.512(<8 x double>, <8 x double>, <8 x double>, i8, i32) nounwind readnone
 
 define <8 x double> @test_mask_vfnmadd_pd(<8 x double> %a0, <8 x double> %a1, <8 x double> %a2, i8 %mask) {
   ; CHECK-LABEL: test_mask_vfnmadd_pd
   ; CHECK: vfnmadd213pd %zmm
-  %res = call <8 x double> @llvm.x86.fma.mask.vfnmadd.pd.512(<8 x double> %a0, <8 x double> %a1, <8 x double> %a2, i8 %mask, i32 4) nounwind
+  %res = call <8 x double> @llvm.x86.avx512.mask.vfnmadd.pd.512(<8 x double> %a0, <8 x double> %a1, <8 x double> %a2, i8 %mask, i32 4) nounwind
   ret <8 x double> %res
 }
 
 define <16 x float> @test_x86_vfnmsubps_z(<16 x float> %a0, <16 x float> %a1, <16 x float> %a2) {
   ; CHECK-LABEL: test_x86_vfnmsubps_z
   ; CHECK: vfnmsub213ps %zmm
-  %res = call <16 x float> @llvm.x86.fma.mask.vfnmsub.ps.512(<16 x float> %a0, <16 x float> %a1, <16 x float> %a2, i16 -1, i32 4) nounwind
+  %res = call <16 x float> @llvm.x86.avx512.mask.vfnmsub.ps.512(<16 x float> %a0, <16 x float> %a1, <16 x float> %a2, i16 -1, i32 4) nounwind
   ret <16 x float> %res
 }
-declare <16 x float> @llvm.x86.fma.mask.vfnmsub.ps.512(<16 x float>, <16 x float>, <16 x float>, i16, i32) nounwind readnone
+declare <16 x float> @llvm.x86.avx512.mask.vfnmsub.ps.512(<16 x float>, <16 x float>, <16 x float>, i16, i32) nounwind readnone
 
 define <16 x float> @test_mask_vfnmsub_ps(<16 x float> %a0, <16 x float> %a1, <16 x float> %a2, i16 %mask) {
   ; CHECK-LABEL: test_mask_vfnmsub_ps
   ; CHECK: vfnmsub213ps %zmm
-  %res = call <16 x float> @llvm.x86.fma.mask.vfnmsub.ps.512(<16 x float> %a0, <16 x float> %a1, <16 x float> %a2, i16 %mask, i32 4) nounwind
+  %res = call <16 x float> @llvm.x86.avx512.mask.vfnmsub.ps.512(<16 x float> %a0, <16 x float> %a1, <16 x float> %a2, i16 %mask, i32 4) nounwind
   ret <16 x float> %res
 }
 
 define <8 x double> @test_x86_vfnmsubpd_z(<8 x double> %a0, <8 x double> %a1, <8 x double> %a2) {
   ; CHECK-LABEL: test_x86_vfnmsubpd_z
   ; CHECK: vfnmsub213pd %zmm
-  %res = call <8 x double> @llvm.x86.fma.mask.vfnmsub.pd.512(<8 x double> %a0, <8 x double> %a1, <8 x double> %a2, i8 -1, i32 4) nounwind
+  %res = call <8 x double> @llvm.x86.avx512.mask.vfnmsub.pd.512(<8 x double> %a0, <8 x double> %a1, <8 x double> %a2, i8 -1, i32 4) nounwind
   ret <8 x double> %res
 }
-declare <8 x double> @llvm.x86.fma.mask.vfnmsub.pd.512(<8 x double>, <8 x double>, <8 x double>, i8, i32) nounwind readnone
+declare <8 x double> @llvm.x86.avx512.mask.vfnmsub.pd.512(<8 x double>, <8 x double>, <8 x double>, i8, i32) nounwind readnone
 
 define <8 x double> @test_mask_vfnmsub_pd(<8 x double> %a0, <8 x double> %a1, <8 x double> %a2, i8 %mask) {
   ; CHECK-LABEL: test_mask_vfnmsub_pd
   ; CHECK: vfnmsub213pd %zmm
-  %res = call <8 x double> @llvm.x86.fma.mask.vfnmsub.pd.512(<8 x double> %a0, <8 x double> %a1, <8 x double> %a2, i8 %mask, i32 4) nounwind
+  %res = call <8 x double> @llvm.x86.avx512.mask.vfnmsub.pd.512(<8 x double> %a0, <8 x double> %a1, <8 x double> %a2, i8 %mask, i32 4) nounwind
   ret <8 x double> %res
 }
 
 define <16 x float> @test_x86_vfmaddsubps_z(<16 x float> %a0, <16 x float> %a1, <16 x float> %a2) {
   ; CHECK-LABEL: test_x86_vfmaddsubps_z
   ; CHECK: vfmaddsub213ps %zmm
-  %res = call <16 x float> @llvm.x86.fma.mask.vfmaddsub.ps.512(<16 x float> %a0, <16 x float> %a1, <16 x float> %a2, i16 -1, i32 4) nounwind
+  %res = call <16 x float> @llvm.x86.avx512.mask.vfmaddsub.ps.512(<16 x float> %a0, <16 x float> %a1, <16 x float> %a2, i16 -1, i32 4) nounwind
   ret <16 x float> %res
 }
 
 define <16 x float> @test_mask_fmaddsub_ps(<16 x float> %a, <16 x float> %b, <16 x float> %c, i16 %mask) {
 ; CHECK-LABEL: test_mask_fmaddsub_ps:
 ; CHECK: vfmaddsub213ps %zmm2, %zmm1, %zmm0 {%k1} ## encoding: [0x62,0xf2,0x75,0x49,0xa6,0xc2]
-  %res = call <16 x float> @llvm.x86.fma.mask.vfmaddsub.ps.512(<16 x float> %a, <16 x float> %b, <16 x float> %c, i16 %mask, i32 4)
+  %res = call <16 x float> @llvm.x86.avx512.mask.vfmaddsub.ps.512(<16 x float> %a, <16 x float> %b, <16 x float> %c, i16 %mask, i32 4)
   ret <16 x float> %res
 }
 
-declare <16 x float> @llvm.x86.fma.mask.vfmaddsub.ps.512(<16 x float>, <16 x float>, <16 x float>, i16, i32) nounwind readnone
+declare <16 x float> @llvm.x86.avx512.mask.vfmaddsub.ps.512(<16 x float>, <16 x float>, <16 x float>, i16, i32) nounwind readnone
 
 define <8 x double> @test_x86_vfmaddsubpd_z(<8 x double> %a0, <8 x double> %a1, <8 x double> %a2) {
   ; CHECK-LABEL: test_x86_vfmaddsubpd_z
   ; CHECK: vfmaddsub213pd %zmm
-  %res = call <8 x double> @llvm.x86.fma.mask.vfmaddsub.pd.512(<8 x double> %a0, <8 x double> %a1, <8 x double> %a2, i8 -1, i32 4) nounwind
+  %res = call <8 x double> @llvm.x86.avx512.mask.vfmaddsub.pd.512(<8 x double> %a0, <8 x double> %a1, <8 x double> %a2, i8 -1, i32 4) nounwind
   ret <8 x double> %res
 }
-declare <8 x double> @llvm.x86.fma.mask.vfmaddsub.pd.512(<8 x double>, <8 x double>, <8 x double>, i8, i32) nounwind readnone
+declare <8 x double> @llvm.x86.avx512.mask.vfmaddsub.pd.512(<8 x double>, <8 x double>, <8 x double>, i8, i32) nounwind readnone
 
 define <8 x double> @test_mask_vfmaddsub_pd(<8 x double> %a0, <8 x double> %a1, <8 x double> %a2, i8 %mask) {
   ; CHECK-LABEL: test_mask_vfmaddsub_pd
   ; CHECK: vfmaddsub213pd %zmm
-  %res = call <8 x double> @llvm.x86.fma.mask.vfmaddsub.pd.512(<8 x double> %a0, <8 x double> %a1, <8 x double> %a2, i8 %mask, i32 4) nounwind
-  ret <8 x double> %res
-}
-
-define <16 x float> @test_x86_vfmsubaddps_z(<16 x float> %a0, <16 x float> %a1, <16 x float> %a2) {
-  ; CHECK-LABEL: test_x86_vfmsubaddps_z
-  ; CHECK: vfmsubadd213ps %zmm
-  %res = call <16 x float> @llvm.x86.fma.mask.vfmsubadd.ps.512(<16 x float> %a0, <16 x float> %a1, <16 x float> %a2, i16 -1, i32 4) nounwind
-  ret <16 x float> %res
-}
-declare <16 x float> @llvm.x86.fma.mask.vfmsubadd.ps.512(<16 x float>, <16 x float>, <16 x float>, i16, i32) nounwind readnone
-
-define <16 x float> @test_mask_vfmsubadd_ps(<16 x float> %a0, <16 x float> %a1, <16 x float> %a2, i16 %mask) {
-  ; CHECK-LABEL: test_mask_vfmsubadd_ps
-  ; CHECK: vfmsubadd213ps %zmm
-  %res = call <16 x float> @llvm.x86.fma.mask.vfmsubadd.ps.512(<16 x float> %a0, <16 x float> %a1, <16 x float> %a2, i16 %mask, i32 4) nounwind
-  ret <16 x float> %res
-}
-
-define <8 x double> @test_x86_vfmsubaddpd_z(<8 x double> %a0, <8 x double> %a1, <8 x double> %a2) {
-  ; CHECK-LABEL: test_x86_vfmsubaddpd_z
-  ; CHECK: vfmsubadd213pd %zmm
-  %res = call <8 x double> @llvm.x86.fma.mask.vfmsubadd.pd.512(<8 x double> %a0, <8 x double> %a1, <8 x double> %a2, i8 -1, i32 4) nounwind
+  %res = call <8 x double> @llvm.x86.avx512.mask.vfmaddsub.pd.512(<8 x double> %a0, <8 x double> %a1, <8 x double> %a2, i8 %mask, i32 4) nounwind
   ret <8 x double> %res
 }
-declare <8 x double> @llvm.x86.fma.mask.vfmsubadd.pd.512(<8 x double>, <8 x double>, <8 x double>, i8, i32) nounwind readnone
 
-define <8 x double> @test_mask_vfmsubadd_pd(<8 x double> %a0, <8 x double> %a1, <8 x double> %a2, i8 %mask) {
-  ; CHECK-LABEL: test_mask_vfmsubadd_pd
-  ; CHECK: vfmsubadd213pd %zmm
-  %res = call <8 x double> @llvm.x86.fma.mask.vfmsubadd.pd.512(<8 x double> %a0, <8 x double> %a1, <8 x double> %a2, i8 %mask, i32 4) nounwind
-  ret <8 x double> %res
+define <8 x double>@test_int_x86_avx512_mask_vfmaddsub_pd_512(<8 x double> %x0, <8 x double> %x1, <8 x double> %x2, i8 %x3){
+; CHECK-LABEL: test_int_x86_avx512_mask_vfmaddsub_pd_512:
+; CHECK:       ## BB#0:
+; CHECK-NEXT:    movzbl %dil, %eax
+; CHECK-NEXT:    kmovw %eax, %k1
+; CHECK-NEXT:    vmovaps %zmm0, %zmm3
+; CHECK-NEXT:    vfmaddsub213pd %zmm2, %zmm1, %zmm3 {%k1}
+; CHECK-NEXT:    vfmaddsub213pd {rn-sae}, %zmm2, %zmm1, %zmm0
+; CHECK-NEXT:    vaddpd %zmm0, %zmm3, %zmm0
+; CHECK-NEXT:    retq
+  %res = call <8 x double> @llvm.x86.avx512.mask.vfmaddsub.pd.512(<8 x double> %x0, <8 x double> %x1, <8 x double> %x2, i8 %x3, i32 4)
+  %res1 = call <8 x double> @llvm.x86.avx512.mask.vfmaddsub.pd.512(<8 x double> %x0, <8 x double> %x1, <8 x double> %x2, i8 -1, i32 0)
+  %res2 = fadd <8 x double> %res, %res1
+  ret <8 x double> %res2
+}
+
+declare <8 x double> @llvm.x86.avx512.mask3.vfmaddsub.pd.512(<8 x double>, <8 x double>, <8 x double>, i8, i32)
+
+define <8 x double>@test_int_x86_avx512_mask3_vfmaddsub_pd_512(<8 x double> %x0, <8 x double> %x1, <8 x double> %x2, i8 %x3){
+; CHECK-LABEL: test_int_x86_avx512_mask3_vfmaddsub_pd_512:
+; CHECK:       ## BB#0:
+; CHECK-NEXT:    movzbl %dil, %eax
+; CHECK-NEXT:    kmovw %eax, %k1
+; CHECK-NEXT:    vmovaps %zmm2, %zmm3
+; CHECK-NEXT:    vfmaddsub231pd %zmm1, %zmm0, %zmm3 {%k1}
+; CHECK-NEXT:    vfmaddsub213pd {rn-sae}, %zmm2, %zmm1, %zmm0
+; CHECK-NEXT:    vaddpd %zmm0, %zmm3, %zmm0
+; CHECK-NEXT:    retq
+  %res = call <8 x double> @llvm.x86.avx512.mask3.vfmaddsub.pd.512(<8 x double> %x0, <8 x double> %x1, <8 x double> %x2, i8 %x3, i32 4)
+  %res1 = call <8 x double> @llvm.x86.avx512.mask3.vfmaddsub.pd.512(<8 x double> %x0, <8 x double> %x1, <8 x double> %x2, i8 -1, i32 0)
+  %res2 = fadd <8 x double> %res, %res1
+  ret <8 x double> %res2
+}
+
+declare <8 x double> @llvm.x86.avx512.maskz.vfmaddsub.pd.512(<8 x double>, <8 x double>, <8 x double>, i8, i32)
+
+define <8 x double>@test_int_x86_avx512_maskz_vfmaddsub_pd_512(<8 x double> %x0, <8 x double> %x1, <8 x double> %x2, i8 %x3){
+; CHECK-LABEL: test_int_x86_avx512_maskz_vfmaddsub_pd_512:
+; CHECK:       ## BB#0:
+; CHECK-NEXT:    movzbl %dil, %eax
+; CHECK-NEXT:    kmovw %eax, %k1
+; CHECK-NEXT:    vmovaps %zmm0, %zmm3
+; CHECK-NEXT:    vfmaddsub213pd %zmm2, %zmm1, %zmm3 {%k1} {z}
+; CHECK-NEXT:    vfmaddsub213pd {rn-sae}, %zmm2, %zmm1, %zmm0
+; CHECK-NEXT:    vaddpd %zmm0, %zmm3, %zmm0
+; CHECK-NEXT:    retq
+  %res = call <8 x double> @llvm.x86.avx512.maskz.vfmaddsub.pd.512(<8 x double> %x0, <8 x double> %x1, <8 x double> %x2, i8 %x3, i32 4)
+  %res1 = call <8 x double> @llvm.x86.avx512.maskz.vfmaddsub.pd.512(<8 x double> %x0, <8 x double> %x1, <8 x double> %x2, i8 -1, i32 0)
+  %res2 = fadd <8 x double> %res, %res1
+  ret <8 x double> %res2
+}
+
+define <16 x float>@test_int_x86_avx512_mask_vfmaddsub_ps_512(<16 x float> %x0, <16 x float> %x1, <16 x float> %x2, i16 %x3){
+; CHECK-LABEL: test_int_x86_avx512_mask_vfmaddsub_ps_512:
+; CHECK:       ## BB#0:
+; CHECK-NEXT:    kmovw %edi, %k1
+; CHECK-NEXT:    vmovaps %zmm0, %zmm3
+; CHECK-NEXT:    vfmaddsub213ps %zmm2, %zmm1, %zmm3 {%k1}
+; CHECK-NEXT:    vfmaddsub213ps {rn-sae}, %zmm2, %zmm1, %zmm0
+; CHECK-NEXT:    vaddps %zmm0, %zmm3, %zmm0
+; CHECK-NEXT:    retq
+  %res = call <16 x float> @llvm.x86.avx512.mask.vfmaddsub.ps.512(<16 x float> %x0, <16 x float> %x1, <16 x float> %x2, i16 %x3, i32 4)
+  %res1 = call <16 x float> @llvm.x86.avx512.mask.vfmaddsub.ps.512(<16 x float> %x0, <16 x float> %x1, <16 x float> %x2, i16 -1, i32 0)
+  %res2 = fadd <16 x float> %res, %res1
+  ret <16 x float> %res2
+}
+
+declare <16 x float> @llvm.x86.avx512.mask3.vfmaddsub.ps.512(<16 x float>, <16 x float>, <16 x float>, i16, i32)
+
+define <16 x float>@test_int_x86_avx512_mask3_vfmaddsub_ps_512(<16 x float> %x0, <16 x float> %x1, <16 x float> %x2, i16 %x3){
+; CHECK-LABEL: test_int_x86_avx512_mask3_vfmaddsub_ps_512:
+; CHECK:       ## BB#0:
+; CHECK-NEXT:    kmovw %edi, %k1
+; CHECK-NEXT:    vmovaps %zmm2, %zmm3
+; CHECK-NEXT:    vfmaddsub231ps %zmm1, %zmm0, %zmm3 {%k1}
+; CHECK-NEXT:    vfmaddsub213ps {rn-sae}, %zmm2, %zmm1, %zmm0
+; CHECK-NEXT:    vaddps %zmm0, %zmm3, %zmm0
+; CHECK-NEXT:    retq
+  %res = call <16 x float> @llvm.x86.avx512.mask3.vfmaddsub.ps.512(<16 x float> %x0, <16 x float> %x1, <16 x float> %x2, i16 %x3, i32 4)
+  %res1 = call <16 x float> @llvm.x86.avx512.mask3.vfmaddsub.ps.512(<16 x float> %x0, <16 x float> %x1, <16 x float> %x2, i16 -1, i32 0)
+  %res2 = fadd <16 x float> %res, %res1
+  ret <16 x float> %res2
+}
+
+declare <16 x float> @llvm.x86.avx512.maskz.vfmaddsub.ps.512(<16 x float>, <16 x float>, <16 x float>, i16, i32)
+
+define <16 x float>@test_int_x86_avx512_maskz_vfmaddsub_ps_512(<16 x float> %x0, <16 x float> %x1, <16 x float> %x2, i16 %x3){
+; CHECK-LABEL: test_int_x86_avx512_maskz_vfmaddsub_ps_512:
+; CHECK:       ## BB#0:
+; CHECK-NEXT:    kmovw %edi, %k1
+; CHECK-NEXT:    vmovaps %zmm0, %zmm3
+; CHECK-NEXT:    vfmaddsub213ps %zmm2, %zmm1, %zmm3 {%k1} {z}
+; CHECK-NEXT:    vfmaddsub213ps {rn-sae}, %zmm2, %zmm1, %zmm0
+; CHECK-NEXT:    vaddps %zmm0, %zmm3, %zmm0
+; CHECK-NEXT:    retq
+  %res = call <16 x float> @llvm.x86.avx512.maskz.vfmaddsub.ps.512(<16 x float> %x0, <16 x float> %x1, <16 x float> %x2, i16 %x3, i32 4)
+  %res1 = call <16 x float> @llvm.x86.avx512.maskz.vfmaddsub.ps.512(<16 x float> %x0, <16 x float> %x1, <16 x float> %x2, i16 -1, i32 0)
+  %res2 = fadd <16 x float> %res, %res1
+  ret <16 x float> %res2
+}
+
+declare <8 x double> @llvm.x86.avx512.mask3.vfmsubadd.pd.512(<8 x double>, <8 x double>, <8 x double>, i8, i32)
+
+define <8 x double>@test_int_x86_avx512_mask3_vfmsubadd_pd_512(<8 x double> %x0, <8 x double> %x1, <8 x double> %x2, i8 %x3){
+; CHECK-LABEL: test_int_x86_avx512_mask3_vfmsubadd_pd_512:
+; CHECK:       ## BB#0:
+; CHECK-NEXT:    movzbl %dil, %eax
+; CHECK-NEXT:    kmovw %eax, %k1
+; CHECK-NEXT:    vmovaps %zmm2, %zmm3
+; CHECK-NEXT:    vfmsubadd231pd %zmm1, %zmm0, %zmm3 {%k1}
+; CHECK-NEXT:    vfmsubadd213pd {rn-sae}, %zmm2, %zmm1, %zmm0
+; CHECK-NEXT:    vaddpd %zmm0, %zmm3, %zmm0
+; CHECK-NEXT:    retq
+  %res = call <8 x double> @llvm.x86.avx512.mask3.vfmsubadd.pd.512(<8 x double> %x0, <8 x double> %x1, <8 x double> %x2, i8 %x3, i32 4)
+  %res1 = call <8 x double> @llvm.x86.avx512.mask3.vfmsubadd.pd.512(<8 x double> %x0, <8 x double> %x1, <8 x double> %x2, i8 -1, i32 0)
+  %res2 = fadd <8 x double> %res, %res1
+  ret <8 x double> %res2
+}
+
+declare <16 x float> @llvm.x86.avx512.mask3.vfmsubadd.ps.512(<16 x float>, <16 x float>, <16 x float>, i16, i32)
+
+define <16 x float>@test_int_x86_avx512_mask3_vfmsubadd_ps_512(<16 x float> %x0, <16 x float> %x1, <16 x float> %x2, i16 %x3){
+; CHECK-LABEL: test_int_x86_avx512_mask3_vfmsubadd_ps_512:
+; CHECK:       ## BB#0:
+; CHECK-NEXT:    kmovw %edi, %k1
+; CHECK-NEXT:    vmovaps %zmm2, %zmm3
+; CHECK-NEXT:    vfmsubadd231ps %zmm1, %zmm0, %zmm3 {%k1}
+; CHECK-NEXT:    vfmsubadd213ps {rn-sae}, %zmm2, %zmm1, %zmm0
+; CHECK-NEXT:    vaddps %zmm0, %zmm3, %zmm0
+; CHECK-NEXT:    retq
+  %res = call <16 x float> @llvm.x86.avx512.mask3.vfmsubadd.ps.512(<16 x float> %x0, <16 x float> %x1, <16 x float> %x2, i16 %x3, i32 4)
+  %res1 = call <16 x float> @llvm.x86.avx512.mask3.vfmsubadd.ps.512(<16 x float> %x0, <16 x float> %x1, <16 x float> %x2, i16 -1, i32 0)
+  %res2 = fadd <16 x float> %res, %res1
+  ret <16 x float> %res2
 }
 
 define <16 x float> @test_mask_round_vfmadd512_ps_rrb_rne(<16 x float> %a0, <16 x float> %a1, <16 x float> %a2, i16 %mask) {
   ; CHECK-LABEL: test_mask_round_vfmadd512_ps_rrb_rne
   ; CHECK: vfmadd213ps  {rn-sae}, %zmm2, %zmm1, %zmm0 {%k1} ## encoding: [0x62,0xf2,0x75,0x19,0xa8,0xc2]
-  %res = call <16 x float> @llvm.x86.fma.mask.vfmadd.ps.512(<16 x float> %a0, <16 x float> %a1, <16 x float> %a2, i16 %mask, i32 0) nounwind
+  %res = call <16 x float> @llvm.x86.avx512.mask.vfmadd.ps.512(<16 x float> %a0, <16 x float> %a1, <16 x float> %a2, i16 %mask, i32 0) nounwind
   ret <16 x float> %res
 }
 
 define <16 x float> @test_mask_round_vfmadd512_ps_rrb_rtn(<16 x float> %a0, <16 x float> %a1, <16 x float> %a2, i16 %mask) {
   ; CHECK-LABEL: test_mask_round_vfmadd512_ps_rrb_rtn
   ; CHECK: vfmadd213ps  {rd-sae}, %zmm2, %zmm1, %zmm0 {%k1} ## encoding: [0x62,0xf2,0x75,0x39,0xa8,0xc2]
-  %res = call <16 x float> @llvm.x86.fma.mask.vfmadd.ps.512(<16 x float> %a0, <16 x float> %a1, <16 x float> %a2, i16 %mask, i32 1) nounwind
+  %res = call <16 x float> @llvm.x86.avx512.mask.vfmadd.ps.512(<16 x float> %a0, <16 x float> %a1, <16 x float> %a2, i16 %mask, i32 1) nounwind
   ret <16 x float> %res
 }
 
 define <16 x float> @test_mask_round_vfmadd512_ps_rrb_rtp(<16 x float> %a0, <16 x float> %a1, <16 x float> %a2, i16 %mask) {
   ; CHECK-LABEL: test_mask_round_vfmadd512_ps_rrb_rtp
   ; CHECK: vfmadd213ps  {ru-sae}, %zmm2, %zmm1, %zmm0 {%k1} ## encoding: [0x62,0xf2,0x75,0x59,0xa8,0xc2]
-  %res = call <16 x float> @llvm.x86.fma.mask.vfmadd.ps.512(<16 x float> %a0, <16 x float> %a1, <16 x float> %a2, i16 %mask, i32 2) nounwind
+  %res = call <16 x float> @llvm.x86.avx512.mask.vfmadd.ps.512(<16 x float> %a0, <16 x float> %a1, <16 x float> %a2, i16 %mask, i32 2) nounwind
   ret <16 x float> %res
 }
 
 define <16 x float> @test_mask_round_vfmadd512_ps_rrb_rtz(<16 x float> %a0, <16 x float> %a1, <16 x float> %a2, i16 %mask) {
   ; CHECK-LABEL: test_mask_round_vfmadd512_ps_rrb_rtz
   ; CHECK: vfmadd213ps  {rz-sae}, %zmm2, %zmm1, %zmm0 {%k1} ## encoding: [0x62,0xf2,0x75,0x79,0xa8,0xc2]
-  %res = call <16 x float> @llvm.x86.fma.mask.vfmadd.ps.512(<16 x float> %a0, <16 x float> %a1, <16 x float> %a2, i16 %mask, i32 3) nounwind
+  %res = call <16 x float> @llvm.x86.avx512.mask.vfmadd.ps.512(<16 x float> %a0, <16 x float> %a1, <16 x float> %a2, i16 %mask, i32 3) nounwind
   ret <16 x float> %res
 }
 
 define <16 x float> @test_mask_round_vfmadd512_ps_rrb_current(<16 x float> %a0, <16 x float> %a1, <16 x float> %a2, i16 %mask) {
   ; CHECK-LABEL: test_mask_round_vfmadd512_ps_rrb_current
   ; CHECK: vfmadd213ps  %zmm2, %zmm1, %zmm0 {%k1} ## encoding: [0x62,0xf2,0x75,0x49,0xa8,0xc2]
-  %res = call <16 x float> @llvm.x86.fma.mask.vfmadd.ps.512(<16 x float> %a0, <16 x float> %a1, <16 x float> %a2, i16 %mask, i32 4) nounwind
+  %res = call <16 x float> @llvm.x86.avx512.mask.vfmadd.ps.512(<16 x float> %a0, <16 x float> %a1, <16 x float> %a2, i16 %mask, i32 4) nounwind
   ret <16 x float> %res
 }
 
 define <16 x float> @test_mask_round_vfmadd512_ps_rrbz_rne(<16 x float> %a0, <16 x float> %a1, <16 x float> %a2) {
   ; CHECK-LABEL: test_mask_round_vfmadd512_ps_rrbz_rne
   ; CHECK: vfmadd213ps  {rn-sae}, %zmm2, %zmm1, %zmm0 ## encoding: [0x62,0xf2,0x75,0x18,0xa8,0xc2]
-  %res = call <16 x float> @llvm.x86.fma.mask.vfmadd.ps.512(<16 x float> %a0, <16 x float> %a1, <16 x float> %a2, i16 -1, i32 0) nounwind
+  %res = call <16 x float> @llvm.x86.avx512.mask.vfmadd.ps.512(<16 x float> %a0, <16 x float> %a1, <16 x float> %a2, i16 -1, i32 0) nounwind
   ret <16 x float> %res
 }
 
 define <16 x float> @test_mask_round_vfmadd512_ps_rrbz_rtn(<16 x float> %a0, <16 x float> %a1, <16 x float> %a2) {
   ; CHECK-LABEL: test_mask_round_vfmadd512_ps_rrbz_rtn
   ; CHECK: vfmadd213ps  {rd-sae}, %zmm2, %zmm1, %zmm0 ## encoding: [0x62,0xf2,0x75,0x38,0xa8,0xc2]
-  %res = call <16 x float> @llvm.x86.fma.mask.vfmadd.ps.512(<16 x float> %a0, <16 x float> %a1, <16 x float> %a2, i16 -1, i32 1) nounwind
+  %res = call <16 x float> @llvm.x86.avx512.mask.vfmadd.ps.512(<16 x float> %a0, <16 x float> %a1, <16 x float> %a2, i16 -1, i32 1) nounwind
   ret <16 x float> %res
 }
 
 define <16 x float> @test_mask_round_vfmadd512_ps_rrbz_rtp(<16 x float> %a0, <16 x float> %a1, <16 x float> %a2) {
   ; CHECK-LABEL: test_mask_round_vfmadd512_ps_rrbz_rtp
   ; CHECK: vfmadd213ps  {ru-sae}, %zmm2, %zmm1, %zmm0 ## encoding: [0x62,0xf2,0x75,0x58,0xa8,0xc2]
-  %res = call <16 x float> @llvm.x86.fma.mask.vfmadd.ps.512(<16 x float> %a0, <16 x float> %a1, <16 x float> %a2, i16 -1, i32 2) nounwind
+  %res = call <16 x float> @llvm.x86.avx512.mask.vfmadd.ps.512(<16 x float> %a0, <16 x float> %a1, <16 x float> %a2, i16 -1, i32 2) nounwind
   ret <16 x float> %res
 }
 
 define <16 x float> @test_mask_round_vfmadd512_ps_rrbz_rtz(<16 x float> %a0, <16 x float> %a1, <16 x float> %a2) {
   ; CHECK-LABEL: test_mask_round_vfmadd512_ps_rrbz_rtz
   ; CHECK: vfmadd213ps  {rz-sae}, %zmm2, %zmm1, %zmm0 ## encoding: [0x62,0xf2,0x75,0x78,0xa8,0xc2]
-  %res = call <16 x float> @llvm.x86.fma.mask.vfmadd.ps.512(<16 x float> %a0, <16 x float> %a1, <16 x float> %a2, i16 -1, i32 3) nounwind
+  %res = call <16 x float> @llvm.x86.avx512.mask.vfmadd.ps.512(<16 x float> %a0, <16 x float> %a1, <16 x float> %a2, i16 -1, i32 3) nounwind
   ret <16 x float> %res
 }
 
 define <16 x float> @test_mask_round_vfmadd512_ps_rrbz_current(<16 x float> %a0, <16 x float> %a1, <16 x float> %a2) {
   ; CHECK-LABEL: test_mask_round_vfmadd512_ps_rrbz_current
   ; CHECK: vfmadd213ps  %zmm2, %zmm1, %zmm0 ## encoding: [0x62,0xf2,0x75,0x48,0xa8,0xc2]
-  %res = call <16 x float> @llvm.x86.fma.mask.vfmadd.ps.512(<16 x float> %a0, <16 x float> %a1, <16 x float> %a2, i16 -1, i32 4) nounwind
-  ret <16 x float> %res
-}
-
-define <16 x float> @test_mask_round_vfmsub512_ps_rrb_rne(<16 x float> %a0, <16 x float> %a1, <16 x float> %a2, i16 %mask) {
-  ; CHECK-LABEL: test_mask_round_vfmsub512_ps_rrb_rne
-  ; CHECK: vfmsub213ps  {rn-sae}, %zmm2, %zmm1, %zmm0 {%k1} ## encoding: [0x62,0xf2,0x75,0x19,0xaa,0xc2]
-  %res = call <16 x float> @llvm.x86.fma.mask.vfmsub.ps.512(<16 x float> %a0, <16 x float> %a1, <16 x float> %a2, i16 %mask, i32 0) nounwind
-  ret <16 x float> %res
-}
-
-define <16 x float> @test_mask_round_vfmsub512_ps_rrb_rtn(<16 x float> %a0, <16 x float> %a1, <16 x float> %a2, i16 %mask) {
-  ; CHECK-LABEL: test_mask_round_vfmsub512_ps_rrb_rtn
-  ; CHECK: vfmsub213ps  {rd-sae}, %zmm2, %zmm1, %zmm0 {%k1} ## encoding: [0x62,0xf2,0x75,0x39,0xaa,0xc2]
-  %res = call <16 x float> @llvm.x86.fma.mask.vfmsub.ps.512(<16 x float> %a0, <16 x float> %a1, <16 x float> %a2, i16 %mask, i32 1) nounwind
-  ret <16 x float> %res
-}
-
-define <16 x float> @test_mask_round_vfmsub512_ps_rrb_rtp(<16 x float> %a0, <16 x float> %a1, <16 x float> %a2, i16 %mask) {
-  ; CHECK-LABEL: test_mask_round_vfmsub512_ps_rrb_rtp
-  ; CHECK: vfmsub213ps  {ru-sae}, %zmm2, %zmm1, %zmm0 {%k1} ## encoding: [0x62,0xf2,0x75,0x59,0xaa,0xc2]
-  %res = call <16 x float> @llvm.x86.fma.mask.vfmsub.ps.512(<16 x float> %a0, <16 x float> %a1, <16 x float> %a2, i16 %mask, i32 2) nounwind
-  ret <16 x float> %res
-}
-
-define <16 x float> @test_mask_round_vfmsub512_ps_rrb_rtz(<16 x float> %a0, <16 x float> %a1, <16 x float> %a2, i16 %mask) {
-  ; CHECK-LABEL: test_mask_round_vfmsub512_ps_rrb_rtz
-  ; CHECK: vfmsub213ps  {rz-sae}, %zmm2, %zmm1, %zmm0 {%k1} ## encoding: [0x62,0xf2,0x75,0x79,0xaa,0xc2]
-  %res = call <16 x float> @llvm.x86.fma.mask.vfmsub.ps.512(<16 x float> %a0, <16 x float> %a1, <16 x float> %a2, i16 %mask, i32 3) nounwind
-  ret <16 x float> %res
-}
-
-define <16 x float> @test_mask_round_vfmsub512_ps_rrb_current(<16 x float> %a0, <16 x float> %a1, <16 x float> %a2, i16 %mask) {
-  ; CHECK-LABEL: test_mask_round_vfmsub512_ps_rrb_current
-  ; CHECK: vfmsub213ps  %zmm2, %zmm1, %zmm0 {%k1} ## encoding: [0x62,0xf2,0x75,0x49,0xaa,0xc2]
-  %res = call <16 x float> @llvm.x86.fma.mask.vfmsub.ps.512(<16 x float> %a0, <16 x float> %a1, <16 x float> %a2, i16 %mask, i32 4) nounwind
-  ret <16 x float> %res
-}
-
-define <16 x float> @test_mask_round_vfmsub512_ps_rrbz_rne(<16 x float> %a0, <16 x float> %a1, <16 x float> %a2) {
-  ; CHECK-LABEL: test_mask_round_vfmsub512_ps_rrbz_rne
-  ; CHECK: vfmsub213ps  {rn-sae}, %zmm2, %zmm1, %zmm0 ## encoding: [0x62,0xf2,0x75,0x18,0xaa,0xc2]
-  %res = call <16 x float> @llvm.x86.fma.mask.vfmsub.ps.512(<16 x float> %a0, <16 x float> %a1, <16 x float> %a2, i16 -1, i32 0) nounwind
-  ret <16 x float> %res
-}
-
-define <16 x float> @test_mask_round_vfmsub512_ps_rrbz_rtn(<16 x float> %a0, <16 x float> %a1, <16 x float> %a2) {
-  ; CHECK-LABEL: test_mask_round_vfmsub512_ps_rrbz_rtn
-  ; CHECK: vfmsub213ps  {rd-sae}, %zmm2, %zmm1, %zmm0 ## encoding: [0x62,0xf2,0x75,0x38,0xaa,0xc2]
-  %res = call <16 x float> @llvm.x86.fma.mask.vfmsub.ps.512(<16 x float> %a0, <16 x float> %a1, <16 x float> %a2, i16 -1, i32 1) nounwind
-  ret <16 x float> %res
-}
-
-define <16 x float> @test_mask_round_vfmsub512_ps_rrbz_rtp(<16 x float> %a0, <16 x float> %a1, <16 x float> %a2) {
-  ; CHECK-LABEL: test_mask_round_vfmsub512_ps_rrbz_rtp
-  ; CHECK: vfmsub213ps  {ru-sae}, %zmm2, %zmm1, %zmm0 ## encoding: [0x62,0xf2,0x75,0x58,0xaa,0xc2]
-  %res = call <16 x float> @llvm.x86.fma.mask.vfmsub.ps.512(<16 x float> %a0, <16 x float> %a1, <16 x float> %a2, i16 -1, i32 2) nounwind
-  ret <16 x float> %res
-}
-
-define <16 x float> @test_mask_round_vfmsub512_ps_rrbz_rtz(<16 x float> %a0, <16 x float> %a1, <16 x float> %a2) {
-  ; CHECK-LABEL: test_mask_round_vfmsub512_ps_rrbz_rtz
-  ; CHECK: vfmsub213ps  {rz-sae}, %zmm2, %zmm1, %zmm0 ## encoding: [0x62,0xf2,0x75,0x78,0xaa,0xc2]
-  %res = call <16 x float> @llvm.x86.fma.mask.vfmsub.ps.512(<16 x float> %a0, <16 x float> %a1, <16 x float> %a2, i16 -1, i32 3) nounwind
+  %res = call <16 x float> @llvm.x86.avx512.mask.vfmadd.ps.512(<16 x float> %a0, <16 x float> %a1, <16 x float> %a2, i16 -1, i32 4) nounwind
   ret <16 x float> %res
 }
 
-define <16 x float> @test_mask_round_vfmsub512_ps_rrbz_current(<16 x float> %a0, <16 x float> %a1, <16 x float> %a2) {
-  ; CHECK-LABEL: test_mask_round_vfmsub512_ps_rrbz_current
-  ; CHECK: vfmsub213ps  %zmm2, %zmm1, %zmm0 ## encoding: [0x62,0xf2,0x75,0x48,0xaa,0xc2]
-  %res = call <16 x float> @llvm.x86.fma.mask.vfmsub.ps.512(<16 x float> %a0, <16 x float> %a1, <16 x float> %a2, i16 -1, i32 4) nounwind
-  ret <16 x float> %res
+declare <8 x double> @llvm.x86.avx512.mask3.vfmsub.pd.512(<8 x double>, <8 x double>, <8 x double>, i8, i32)
+
+define <8 x double>@test_int_x86_avx512_mask3_vfmsub_pd_512(<8 x double> %x0, <8 x double> %x1, <8 x double> %x2, i8 %x3){
+; CHECK-LABEL: test_int_x86_avx512_mask3_vfmsub_pd_512:
+; CHECK:       ## BB#0:
+; CHECK-NEXT:    movzbl %dil, %eax
+; CHECK-NEXT:    kmovw %eax, %k1
+; CHECK-NEXT:    vmovaps %zmm2, %zmm3
+; CHECK-NEXT:    vfmsub231pd %zmm1, %zmm0, %zmm3 {%k1}
+; CHECK-NEXT:    vfmsub213pd {rn-sae}, %zmm2, %zmm1, %zmm0
+; CHECK-NEXT:    vaddpd %zmm0, %zmm3, %zmm0
+; CHECK-NEXT:    retq
+  %res = call <8 x double> @llvm.x86.avx512.mask3.vfmsub.pd.512(<8 x double> %x0, <8 x double> %x1, <8 x double> %x2, i8 %x3, i32 4)
+  %res1 = call <8 x double> @llvm.x86.avx512.mask3.vfmsub.pd.512(<8 x double> %x0, <8 x double> %x1, <8 x double> %x2, i8 -1, i32 0)
+  %res2 = fadd <8 x double> %res, %res1
+  ret <8 x double> %res2
+}
+
+declare <16 x float> @llvm.x86.avx512.mask3.vfmsub.ps.512(<16 x float>, <16 x float>, <16 x float>, i16, i32)
+
+define <16 x float>@test_int_x86_avx512_mask3_vfmsub_ps_512(<16 x float> %x0, <16 x float> %x1, <16 x float> %x2, i16 %x3){
+; CHECK-LABEL: test_int_x86_avx512_mask3_vfmsub_ps_512:
+; CHECK:       ## BB#0:
+; CHECK-NEXT:    kmovw %edi, %k1
+; CHECK-NEXT:    vmovaps %zmm2, %zmm3
+; CHECK-NEXT:    vfmsub231ps %zmm1, %zmm0, %zmm3 {%k1}
+; CHECK-NEXT:    vfmsub213ps {rn-sae}, %zmm2, %zmm1, %zmm0
+; CHECK-NEXT:    vaddps %zmm0, %zmm3, %zmm0
+; CHECK-NEXT:    retq
+  %res = call <16 x float> @llvm.x86.avx512.mask3.vfmsub.ps.512(<16 x float> %x0, <16 x float> %x1, <16 x float> %x2, i16 %x3, i32 4)
+  %res1 = call <16 x float> @llvm.x86.avx512.mask3.vfmsub.ps.512(<16 x float> %x0, <16 x float> %x1, <16 x float> %x2, i16 -1, i32 0)
+  %res2 = fadd <16 x float> %res, %res1
+  ret <16 x float> %res2
 }
 
 define <8 x double> @test_mask_round_vfmadd512_pd_rrb_rne(<8 x double> %a0, <8 x double> %a1, <8 x double> %a2, i8 %mask) {
   ; CHECK-LABEL: test_mask_round_vfmadd512_pd_rrb_rne
   ; CHECK: vfmadd213pd  {rn-sae}, %zmm2, %zmm1, %zmm0 {%k1} ## encoding: [0x62,0xf2,0xf5,0x19,0xa8,0xc2]
-  %res = call <8 x double> @llvm.x86.fma.mask.vfmadd.pd.512(<8 x double> %a0, <8 x double> %a1, <8 x double> %a2, i8 %mask, i32 0) nounwind
+  %res = call <8 x double> @llvm.x86.avx512.mask.vfmadd.pd.512(<8 x double> %a0, <8 x double> %a1, <8 x double> %a2, i8 %mask, i32 0) nounwind
   ret <8 x double> %res
 }
 
 define <8 x double> @test_mask_round_vfmadd512_pd_rrb_rtn(<8 x double> %a0, <8 x double> %a1, <8 x double> %a2, i8 %mask) {
   ; CHECK-LABEL: test_mask_round_vfmadd512_pd_rrb_rtn
   ; CHECK: vfmadd213pd  {rd-sae}, %zmm2, %zmm1, %zmm0 {%k1} ## encoding: [0x62,0xf2,0xf5,0x39,0xa8,0xc2]
-  %res = call <8 x double> @llvm.x86.fma.mask.vfmadd.pd.512(<8 x double> %a0, <8 x double> %a1, <8 x double> %a2, i8 %mask, i32 1) nounwind
+  %res = call <8 x double> @llvm.x86.avx512.mask.vfmadd.pd.512(<8 x double> %a0, <8 x double> %a1, <8 x double> %a2, i8 %mask, i32 1) nounwind
   ret <8 x double> %res
 }
 
 define <8 x double> @test_mask_round_vfmadd512_pd_rrb_rtp(<8 x double> %a0, <8 x double> %a1, <8 x double> %a2, i8 %mask) {
   ; CHECK-LABEL: test_mask_round_vfmadd512_pd_rrb_rtp
   ; CHECK: vfmadd213pd  {ru-sae}, %zmm2, %zmm1, %zmm0 {%k1} ## encoding: [0x62,0xf2,0xf5,0x59,0xa8,0xc2]
-  %res = call <8 x double> @llvm.x86.fma.mask.vfmadd.pd.512(<8 x double> %a0, <8 x double> %a1, <8 x double> %a2, i8 %mask, i32 2) nounwind
+  %res = call <8 x double> @llvm.x86.avx512.mask.vfmadd.pd.512(<8 x double> %a0, <8 x double> %a1, <8 x double> %a2, i8 %mask, i32 2) nounwind
   ret <8 x double> %res
 }
 
 define <8 x double> @test_mask_round_vfmadd512_pd_rrb_rtz(<8 x double> %a0, <8 x double> %a1, <8 x double> %a2, i8 %mask) {
   ; CHECK-LABEL: test_mask_round_vfmadd512_pd_rrb_rtz
   ; CHECK: vfmadd213pd  {rz-sae}, %zmm2, %zmm1, %zmm0 {%k1} ## encoding: [0x62,0xf2,0xf5,0x79,0xa8,0xc2]
-  %res = call <8 x double> @llvm.x86.fma.mask.vfmadd.pd.512(<8 x double> %a0, <8 x double> %a1, <8 x double> %a2, i8 %mask, i32 3) nounwind
+  %res = call <8 x double> @llvm.x86.avx512.mask.vfmadd.pd.512(<8 x double> %a0, <8 x double> %a1, <8 x double> %a2, i8 %mask, i32 3) nounwind
   ret <8 x double> %res
 }
 
 define <8 x double> @test_mask_round_vfmadd512_pd_rrb_current(<8 x double> %a0, <8 x double> %a1, <8 x double> %a2, i8 %mask) {
   ; CHECK-LABEL: test_mask_round_vfmadd512_pd_rrb_current
   ; CHECK: vfmadd213pd  %zmm2, %zmm1, %zmm0 {%k1} ## encoding: [0x62,0xf2,0xf5,0x49,0xa8,0xc2]
-  %res = call <8 x double> @llvm.x86.fma.mask.vfmadd.pd.512(<8 x double> %a0, <8 x double> %a1, <8 x double> %a2, i8 %mask, i32 4) nounwind
+  %res = call <8 x double> @llvm.x86.avx512.mask.vfmadd.pd.512(<8 x double> %a0, <8 x double> %a1, <8 x double> %a2, i8 %mask, i32 4) nounwind
   ret <8 x double> %res
 }
 
 define <8 x double> @test_mask_round_vfmadd512_pd_rrbz_rne(<8 x double> %a0, <8 x double> %a1, <8 x double> %a2) {
   ; CHECK-LABEL: test_mask_round_vfmadd512_pd_rrbz_rne
   ; CHECK: vfmadd213pd  {rn-sae}, %zmm2, %zmm1, %zmm0 ## encoding: [0x62,0xf2,0xf5,0x18,0xa8,0xc2]
-  %res = call <8 x double> @llvm.x86.fma.mask.vfmadd.pd.512(<8 x double> %a0, <8 x double> %a1, <8 x double> %a2, i8 -1, i32 0) nounwind
+  %res = call <8 x double> @llvm.x86.avx512.mask.vfmadd.pd.512(<8 x double> %a0, <8 x double> %a1, <8 x double> %a2, i8 -1, i32 0) nounwind
   ret <8 x double> %res
 }
 
 define <8 x double> @test_mask_round_vfmadd512_pd_rrbz_rtn(<8 x double> %a0, <8 x double> %a1, <8 x double> %a2) {
   ; CHECK-LABEL: test_mask_round_vfmadd512_pd_rrbz_rtn
   ; CHECK: vfmadd213pd  {rd-sae}, %zmm2, %zmm1, %zmm0 ## encoding: [0x62,0xf2,0xf5,0x38,0xa8,0xc2]
-  %res = call <8 x double> @llvm.x86.fma.mask.vfmadd.pd.512(<8 x double> %a0, <8 x double> %a1, <8 x double> %a2, i8 -1, i32 1) nounwind
+  %res = call <8 x double> @llvm.x86.avx512.mask.vfmadd.pd.512(<8 x double> %a0, <8 x double> %a1, <8 x double> %a2, i8 -1, i32 1) nounwind
   ret <8 x double> %res
 }
 
 define <8 x double> @test_mask_round_vfmadd512_pd_rrbz_rtp(<8 x double> %a0, <8 x double> %a1, <8 x double> %a2) {
   ; CHECK-LABEL: test_mask_round_vfmadd512_pd_rrbz_rtp
   ; CHECK: vfmadd213pd  {ru-sae}, %zmm2, %zmm1, %zmm0 ## encoding: [0x62,0xf2,0xf5,0x58,0xa8,0xc2]
-  %res = call <8 x double> @llvm.x86.fma.mask.vfmadd.pd.512(<8 x double> %a0, <8 x double> %a1, <8 x double> %a2, i8 -1, i32 2) nounwind
+  %res = call <8 x double> @llvm.x86.avx512.mask.vfmadd.pd.512(<8 x double> %a0, <8 x double> %a1, <8 x double> %a2, i8 -1, i32 2) nounwind
   ret <8 x double> %res
 }
 
 define <8 x double> @test_mask_round_vfmadd512_pd_rrbz_rtz(<8 x double> %a0, <8 x double> %a1, <8 x double> %a2) {
   ; CHECK-LABEL: test_mask_round_vfmadd512_pd_rrbz_rtz
   ; CHECK: vfmadd213pd  {rz-sae}, %zmm2, %zmm1, %zmm0 ## encoding: [0x62,0xf2,0xf5,0x78,0xa8,0xc2]
-  %res = call <8 x double> @llvm.x86.fma.mask.vfmadd.pd.512(<8 x double> %a0, <8 x double> %a1, <8 x double> %a2, i8 -1, i32 3) nounwind
+  %res = call <8 x double> @llvm.x86.avx512.mask.vfmadd.pd.512(<8 x double> %a0, <8 x double> %a1, <8 x double> %a2, i8 -1, i32 3) nounwind
   ret <8 x double> %res
 }
 
 define <8 x double> @test_mask_round_vfmadd512_pd_rrbz_current(<8 x double> %a0, <8 x double> %a1, <8 x double> %a2) {
   ; CHECK-LABEL: test_mask_round_vfmadd512_pd_rrbz_current
   ; CHECK: vfmadd213pd  %zmm2, %zmm1, %zmm0 ## encoding: [0x62,0xf2,0xf5,0x48,0xa8,0xc2]
-  %res = call <8 x double> @llvm.x86.fma.mask.vfmadd.pd.512(<8 x double> %a0, <8 x double> %a1, <8 x double> %a2, i8 -1, i32 4) nounwind
+  %res = call <8 x double> @llvm.x86.avx512.mask.vfmadd.pd.512(<8 x double> %a0, <8 x double> %a1, <8 x double> %a2, i8 -1, i32 4) nounwind
   ret <8 x double> %res
 }
 
+define <8 x double>@test_int_x86_avx512_mask_vfmadd_pd_512(<8 x double> %x0, <8 x double> %x1, <8 x double> %x2, i8 %x3){
+; CHECK-LABEL: test_int_x86_avx512_mask_vfmadd_pd_512:
+; CHECK:       ## BB#0:
+; CHECK-NEXT:    movzbl %dil, %eax
+; CHECK-NEXT:    kmovw %eax, %k1
+; CHECK-NEXT:    vmovaps %zmm0, %zmm3
+; CHECK-NEXT:    vfmadd213pd %zmm2, %zmm1, %zmm3 {%k1}
+; CHECK-NEXT:    vfmadd213pd {rn-sae}, %zmm2, %zmm1, %zmm0
+; CHECK-NEXT:    vaddpd %zmm0, %zmm3, %zmm0
+; CHECK-NEXT:    retq
+  %res = call <8 x double> @llvm.x86.avx512.mask.vfmadd.pd.512(<8 x double> %x0, <8 x double> %x1, <8 x double> %x2, i8 %x3, i32 4)
+  %res1 = call <8 x double> @llvm.x86.avx512.mask.vfmadd.pd.512(<8 x double> %x0, <8 x double> %x1, <8 x double> %x2, i8 -1, i32 0)
+  %res2 = fadd <8 x double> %res, %res1
+  ret <8 x double> %res2
+}
+
+declare <8 x double> @llvm.x86.avx512.mask3.vfmadd.pd.512(<8 x double>, <8 x double>, <8 x double>, i8, i32)
+
+define <8 x double>@test_int_x86_avx512_mask3_vfmadd_pd_512(<8 x double> %x0, <8 x double> %x1, <8 x double> %x2, i8 %x3){
+; CHECK-LABEL: test_int_x86_avx512_mask3_vfmadd_pd_512:
+; CHECK:       ## BB#0:
+; CHECK-NEXT:    movzbl %dil, %eax
+; CHECK-NEXT:    kmovw %eax, %k1
+; CHECK-NEXT:    vmovaps %zmm2, %zmm3
+; CHECK-NEXT:    vfmadd231pd %zmm1, %zmm0, %zmm3 {%k1}
+; CHECK-NEXT:    vfmadd213pd {rn-sae}, %zmm2, %zmm1, %zmm0
+; CHECK-NEXT:    vaddpd %zmm0, %zmm3, %zmm0
+; CHECK-NEXT:    retq
+  %res = call <8 x double> @llvm.x86.avx512.mask3.vfmadd.pd.512(<8 x double> %x0, <8 x double> %x1, <8 x double> %x2, i8 %x3, i32 4)
+  %res1 = call <8 x double> @llvm.x86.avx512.mask3.vfmadd.pd.512(<8 x double> %x0, <8 x double> %x1, <8 x double> %x2, i8 -1, i32 0)
+  %res2 = fadd <8 x double> %res, %res1
+  ret <8 x double> %res2
+}
+
+declare <8 x double> @llvm.x86.avx512.maskz.vfmadd.pd.512(<8 x double>, <8 x double>, <8 x double>, i8, i32)
+
+define <8 x double>@test_int_x86_avx512_maskz_vfmadd_pd_512(<8 x double> %x0, <8 x double> %x1, <8 x double> %x2, i8 %x3){
+; CHECK-LABEL: test_int_x86_avx512_maskz_vfmadd_pd_512:
+; CHECK:       ## BB#0:
+; CHECK-NEXT:    movzbl %dil, %eax
+; CHECK-NEXT:    kmovw %eax, %k1
+; CHECK-NEXT:    vmovaps %zmm0, %zmm3
+; CHECK-NEXT:    vfmadd213pd %zmm2, %zmm1, %zmm3 {%k1} {z}
+; CHECK-NEXT:    vfmadd213pd {rn-sae}, %zmm2, %zmm1, %zmm0
+; CHECK-NEXT:    vaddpd %zmm0, %zmm3, %zmm0
+; CHECK-NEXT:    retq
+  %res = call <8 x double> @llvm.x86.avx512.maskz.vfmadd.pd.512(<8 x double> %x0, <8 x double> %x1, <8 x double> %x2, i8 %x3, i32 4)
+  %res1 = call <8 x double> @llvm.x86.avx512.maskz.vfmadd.pd.512(<8 x double> %x0, <8 x double> %x1, <8 x double> %x2, i8 -1, i32 0)
+  %res2 = fadd <8 x double> %res, %res1
+  ret <8 x double> %res2
+}
+
+define <16 x float>@test_int_x86_avx512_mask_vfmadd_ps_512(<16 x float> %x0, <16 x float> %x1, <16 x float> %x2, i16 %x3){
+; CHECK-LABEL: test_int_x86_avx512_mask_vfmadd_ps_512:
+; CHECK:       ## BB#0:
+; CHECK-NEXT:    kmovw %edi, %k1
+; CHECK-NEXT:    vmovaps %zmm0, %zmm3
+; CHECK-NEXT:    vfmadd213ps %zmm2, %zmm1, %zmm3 {%k1}
+; CHECK-NEXT:    vfmadd213ps {rn-sae}, %zmm2, %zmm1, %zmm0
+; CHECK-NEXT:    vaddps %zmm0, %zmm3, %zmm0
+; CHECK-NEXT:    retq
+  %res = call <16 x float> @llvm.x86.avx512.mask.vfmadd.ps.512(<16 x float> %x0, <16 x float> %x1, <16 x float> %x2, i16 %x3, i32 4)
+  %res1 = call <16 x float> @llvm.x86.avx512.mask.vfmadd.ps.512(<16 x float> %x0, <16 x float> %x1, <16 x float> %x2, i16 -1, i32 0)
+  %res2 = fadd <16 x float> %res, %res1
+  ret <16 x float> %res2
+}
+
+declare <16 x float> @llvm.x86.avx512.mask3.vfmadd.ps.512(<16 x float>, <16 x float>, <16 x float>, i16, i32)
+
+define <16 x float>@test_int_x86_avx512_mask3_vfmadd_ps_512(<16 x float> %x0, <16 x float> %x1, <16 x float> %x2, i16 %x3){
+; CHECK-LABEL: test_int_x86_avx512_mask3_vfmadd_ps_512:
+; CHECK:       ## BB#0:
+; CHECK-NEXT:    kmovw %edi, %k1
+; CHECK-NEXT:    vmovaps %zmm2, %zmm3
+; CHECK-NEXT:    vfmadd231ps %zmm1, %zmm0, %zmm3 {%k1}
+; CHECK-NEXT:    vfmadd213ps {rn-sae}, %zmm2, %zmm1, %zmm0
+; CHECK-NEXT:    vaddps %zmm0, %zmm3, %zmm0
+; CHECK-NEXT:    retq
+  %res = call <16 x float> @llvm.x86.avx512.mask3.vfmadd.ps.512(<16 x float> %x0, <16 x float> %x1, <16 x float> %x2, i16 %x3, i32 4)
+  %res1 = call <16 x float> @llvm.x86.avx512.mask3.vfmadd.ps.512(<16 x float> %x0, <16 x float> %x1, <16 x float> %x2, i16 -1, i32 0)
+  %res2 = fadd <16 x float> %res, %res1
+  ret <16 x float> %res2
+}
+
+declare <16 x float> @llvm.x86.avx512.maskz.vfmadd.ps.512(<16 x float>, <16 x float>, <16 x float>, i16, i32)
+
+define <16 x float>@test_int_x86_avx512_maskz_vfmadd_ps_512(<16 x float> %x0, <16 x float> %x1, <16 x float> %x2, i16 %x3){
+; CHECK-LABEL: test_int_x86_avx512_maskz_vfmadd_ps_512:
+; CHECK:       ## BB#0:
+; CHECK-NEXT:    kmovw %edi, %k1
+; CHECK-NEXT:    vmovaps %zmm0, %zmm3
+; CHECK-NEXT:    vfmadd213ps %zmm2, %zmm1, %zmm3 {%k1} {z}
+; CHECK-NEXT:    vfmadd213ps {rn-sae}, %zmm2, %zmm1, %zmm0
+; CHECK-NEXT:    vaddps %zmm0, %zmm3, %zmm0
+; CHECK-NEXT:    retq
+  %res = call <16 x float> @llvm.x86.avx512.maskz.vfmadd.ps.512(<16 x float> %x0, <16 x float> %x1, <16 x float> %x2, i16 %x3, i32 4)
+  %res1 = call <16 x float> @llvm.x86.avx512.maskz.vfmadd.ps.512(<16 x float> %x0, <16 x float> %x1, <16 x float> %x2, i16 -1, i32 0)
+  %res2 = fadd <16 x float> %res, %res1
+  ret <16 x float> %res2
+}
+
 
 define <8 x double> @test_mask_round_vfnmsub512_pd_rrb_rne(<8 x double> %a0, <8 x double> %a1, <8 x double> %a2, i8 %mask) {
   ; CHECK-LABEL: test_mask_round_vfnmsub512_pd_rrb_rne
   ; CHECK: vfnmsub213pd  {rn-sae}, %zmm2, %zmm1, %zmm0 {%k1} ## encoding: [0x62,0xf2,0xf5,0x19,0xae,0xc2]
-  %res = call <8 x double> @llvm.x86.fma.mask.vfnmsub.pd.512(<8 x double> %a0, <8 x double> %a1, <8 x double> %a2, i8 %mask, i32 0) nounwind
+  %res = call <8 x double> @llvm.x86.avx512.mask.vfnmsub.pd.512(<8 x double> %a0, <8 x double> %a1, <8 x double> %a2, i8 %mask, i32 0) nounwind
   ret <8 x double> %res
 }
 
 define <8 x double> @test_mask_round_vfnmsub512_pd_rrb_rtn(<8 x double> %a0, <8 x double> %a1, <8 x double> %a2, i8 %mask) {
   ; CHECK-LABEL: test_mask_round_vfnmsub512_pd_rrb_rtn
   ; CHECK: vfnmsub213pd  {rd-sae}, %zmm2, %zmm1, %zmm0 {%k1} ## encoding: [0x62,0xf2,0xf5,0x39,0xae,0xc2]
-  %res = call <8 x double> @llvm.x86.fma.mask.vfnmsub.pd.512(<8 x double> %a0, <8 x double> %a1, <8 x double> %a2, i8 %mask, i32 1) nounwind
+  %res = call <8 x double> @llvm.x86.avx512.mask.vfnmsub.pd.512(<8 x double> %a0, <8 x double> %a1, <8 x double> %a2, i8 %mask, i32 1) nounwind
   ret <8 x double> %res
 }
 
 define <8 x double> @test_mask_round_vfnmsub512_pd_rrb_rtp(<8 x double> %a0, <8 x double> %a1, <8 x double> %a2, i8 %mask) {
   ; CHECK-LABEL: test_mask_round_vfnmsub512_pd_rrb_rtp
   ; CHECK: vfnmsub213pd  {ru-sae}, %zmm2, %zmm1, %zmm0 {%k1} ## encoding: [0x62,0xf2,0xf5,0x59,0xae,0xc2]
-  %res = call <8 x double> @llvm.x86.fma.mask.vfnmsub.pd.512(<8 x double> %a0, <8 x double> %a1, <8 x double> %a2, i8 %mask, i32 2) nounwind
+  %res = call <8 x double> @llvm.x86.avx512.mask.vfnmsub.pd.512(<8 x double> %a0, <8 x double> %a1, <8 x double> %a2, i8 %mask, i32 2) nounwind
   ret <8 x double> %res
 }
 
 define <8 x double> @test_mask_round_vfnmsub512_pd_rrb_rtz(<8 x double> %a0, <8 x double> %a1, <8 x double> %a2, i8 %mask) {
   ; CHECK-LABEL: test_mask_round_vfnmsub512_pd_rrb_rtz
   ; CHECK: vfnmsub213pd  {rz-sae}, %zmm2, %zmm1, %zmm0 {%k1} ## encoding: [0x62,0xf2,0xf5,0x79,0xae,0xc2]
-  %res = call <8 x double> @llvm.x86.fma.mask.vfnmsub.pd.512(<8 x double> %a0, <8 x double> %a1, <8 x double> %a2, i8 %mask, i32 3) nounwind
+  %res = call <8 x double> @llvm.x86.avx512.mask.vfnmsub.pd.512(<8 x double> %a0, <8 x double> %a1, <8 x double> %a2, i8 %mask, i32 3) nounwind
   ret <8 x double> %res
 }
 
 define <8 x double> @test_mask_round_vfnmsub512_pd_rrb_current(<8 x double> %a0, <8 x double> %a1, <8 x double> %a2, i8 %mask) {
   ; CHECK-LABEL: test_mask_round_vfnmsub512_pd_rrb_current
   ; CHECK: vfnmsub213pd  %zmm2, %zmm1, %zmm0 {%k1} ## encoding: [0x62,0xf2,0xf5,0x49,0xae,0xc2]
-  %res = call <8 x double> @llvm.x86.fma.mask.vfnmsub.pd.512(<8 x double> %a0, <8 x double> %a1, <8 x double> %a2, i8 %mask, i32 4) nounwind
+  %res = call <8 x double> @llvm.x86.avx512.mask.vfnmsub.pd.512(<8 x double> %a0, <8 x double> %a1, <8 x double> %a2, i8 %mask, i32 4) nounwind
   ret <8 x double> %res
 }
 
 define <8 x double> @test_mask_round_vfnmsub512_pd_rrbz_rne(<8 x double> %a0, <8 x double> %a1, <8 x double> %a2) {
   ; CHECK-LABEL: test_mask_round_vfnmsub512_pd_rrbz_rne
   ; CHECK: vfnmsub213pd  {rn-sae}, %zmm2, %zmm1, %zmm0 ## encoding: [0x62,0xf2,0xf5,0x18,0xae,0xc2]
-  %res = call <8 x double> @llvm.x86.fma.mask.vfnmsub.pd.512(<8 x double> %a0, <8 x double> %a1, <8 x double> %a2, i8 -1, i32 0) nounwind
+  %res = call <8 x double> @llvm.x86.avx512.mask.vfnmsub.pd.512(<8 x double> %a0, <8 x double> %a1, <8 x double> %a2, i8 -1, i32 0) nounwind
   ret <8 x double> %res
 }
 
 define <8 x double> @test_mask_round_vfnmsub512_pd_rrbz_rtn(<8 x double> %a0, <8 x double> %a1, <8 x double> %a2) {
   ; CHECK-LABEL: test_mask_round_vfnmsub512_pd_rrbz_rtn
   ; CHECK: vfnmsub213pd  {rd-sae}, %zmm2, %zmm1, %zmm0 ## encoding: [0x62,0xf2,0xf5,0x38,0xae,0xc2]
-  %res = call <8 x double> @llvm.x86.fma.mask.vfnmsub.pd.512(<8 x double> %a0, <8 x double> %a1, <8 x double> %a2, i8 -1, i32 1) nounwind
+  %res = call <8 x double> @llvm.x86.avx512.mask.vfnmsub.pd.512(<8 x double> %a0, <8 x double> %a1, <8 x double> %a2, i8 -1, i32 1) nounwind
   ret <8 x double> %res
 }
 
 define <8 x double> @test_mask_round_vfnmsub512_pd_rrbz_rtp(<8 x double> %a0, <8 x double> %a1, <8 x double> %a2) {
   ; CHECK-LABEL: test_mask_round_vfnmsub512_pd_rrbz_rtp
   ; CHECK: vfnmsub213pd  {ru-sae}, %zmm2, %zmm1, %zmm0 ## encoding: [0x62,0xf2,0xf5,0x58,0xae,0xc2]
-  %res = call <8 x double> @llvm.x86.fma.mask.vfnmsub.pd.512(<8 x double> %a0, <8 x double> %a1, <8 x double> %a2, i8 -1, i32 2) nounwind
+  %res = call <8 x double> @llvm.x86.avx512.mask.vfnmsub.pd.512(<8 x double> %a0, <8 x double> %a1, <8 x double> %a2, i8 -1, i32 2) nounwind
   ret <8 x double> %res
 }
 
 define <8 x double> @test_mask_round_vfnmsub512_pd_rrbz_rtz(<8 x double> %a0, <8 x double> %a1, <8 x double> %a2) {
   ; CHECK-LABEL: test_mask_round_vfnmsub512_pd_rrbz_rtz
   ; CHECK: vfnmsub213pd  {rz-sae}, %zmm2, %zmm1, %zmm0 ## encoding: [0x62,0xf2,0xf5,0x78,0xae,0xc2]
-  %res = call <8 x double> @llvm.x86.fma.mask.vfnmsub.pd.512(<8 x double> %a0, <8 x double> %a1, <8 x double> %a2, i8 -1, i32 3) nounwind
+  %res = call <8 x double> @llvm.x86.avx512.mask.vfnmsub.pd.512(<8 x double> %a0, <8 x double> %a1, <8 x double> %a2, i8 -1, i32 3) nounwind
   ret <8 x double> %res
 }
 
 define <8 x double> @test_mask_round_vfnmsub512_pd_rrbz_current(<8 x double> %a0, <8 x double> %a1, <8 x double> %a2) {
   ; CHECK-LABEL: test_mask_round_vfnmsub512_pd_rrbz_current
   ; CHECK: vfnmsub213pd  %zmm2, %zmm1, %zmm0 ## encoding: [0x62,0xf2,0xf5,0x48,0xae,0xc2]
-  %res = call <8 x double> @llvm.x86.fma.mask.vfnmsub.pd.512(<8 x double> %a0, <8 x double> %a1, <8 x double> %a2, i8 -1, i32 4) nounwind
+  %res = call <8 x double> @llvm.x86.avx512.mask.vfnmsub.pd.512(<8 x double> %a0, <8 x double> %a1, <8 x double> %a2, i8 -1, i32 4) nounwind
   ret <8 x double> %res
 }
+
+define <8 x double>@test_int_x86_avx512_mask_vfnmsub_pd_512(<8 x double> %x0, <8 x double> %x1, <8 x double> %x2, i8 %x3){
+; CHECK-LABEL: test_int_x86_avx512_mask_vfnmsub_pd_512:
+; CHECK:       ## BB#0:
+; CHECK-NEXT:    movzbl %dil, %eax
+; CHECK-NEXT:    kmovw %eax, %k1
+; CHECK-NEXT:    vmovaps %zmm0, %zmm3
+; CHECK-NEXT:    vfnmsub213pd %zmm2, %zmm1, %zmm3 {%k1}
+; CHECK-NEXT:    vfnmsub213pd {rn-sae}, %zmm2, %zmm1, %zmm0
+; CHECK-NEXT:    vaddpd %zmm0, %zmm3, %zmm0
+; CHECK-NEXT:    retq
+  %res = call <8 x double> @llvm.x86.avx512.mask.vfnmsub.pd.512(<8 x double> %x0, <8 x double> %x1, <8 x double> %x2, i8 %x3, i32 4)
+  %res1 = call <8 x double> @llvm.x86.avx512.mask.vfnmsub.pd.512(<8 x double> %x0, <8 x double> %x1, <8 x double> %x2, i8 -1, i32 0)
+  %res2 = fadd <8 x double> %res, %res1
+  ret <8 x double> %res2
+}
+
+declare <8 x double> @llvm.x86.avx512.mask3.vfnmsub.pd.512(<8 x double>, <8 x double>, <8 x double>, i8, i32)
+
+define <8 x double>@test_int_x86_avx512_mask3_vfnmsub_pd_512(<8 x double> %x0, <8 x double> %x1, <8 x double> %x2, i8 %x3){
+; CHECK-LABEL: test_int_x86_avx512_mask3_vfnmsub_pd_512:
+; CHECK:       ## BB#0:
+; CHECK-NEXT:    movzbl %dil, %eax
+; CHECK-NEXT:    kmovw %eax, %k1
+; CHECK-NEXT:    vmovaps %zmm2, %zmm3
+; CHECK-NEXT:    vfnmsub231pd %zmm1, %zmm0, %zmm3 {%k1}
+; CHECK-NEXT:    vfnmsub213pd {rn-sae}, %zmm2, %zmm1, %zmm0
+; CHECK-NEXT:    vaddpd %zmm0, %zmm3, %zmm0
+; CHECK-NEXT:    retq
+  %res = call <8 x double> @llvm.x86.avx512.mask3.vfnmsub.pd.512(<8 x double> %x0, <8 x double> %x1, <8 x double> %x2, i8 %x3, i32 4)
+  %res1 = call <8 x double> @llvm.x86.avx512.mask3.vfnmsub.pd.512(<8 x double> %x0, <8 x double> %x1, <8 x double> %x2, i8 -1, i32 0)
+  %res2 = fadd <8 x double> %res, %res1
+  ret <8 x double> %res2
+}
+
+define <16 x float>@test_int_x86_avx512_mask_vfnmsub_ps_512(<16 x float> %x0, <16 x float> %x1, <16 x float> %x2, i16 %x3){
+; CHECK-LABEL: test_int_x86_avx512_mask_vfnmsub_ps_512:
+; CHECK:       ## BB#0:
+; CHECK-NEXT:    kmovw %edi, %k1
+; CHECK-NEXT:    vmovaps %zmm0, %zmm3
+; CHECK-NEXT:    vfnmsub213ps %zmm2, %zmm1, %zmm3 {%k1}
+; CHECK-NEXT:    vfnmsub213ps {rn-sae}, %zmm2, %zmm1, %zmm0
+; CHECK-NEXT:    vaddps %zmm0, %zmm3, %zmm0
+; CHECK-NEXT:    retq
+  %res = call <16 x float> @llvm.x86.avx512.mask.vfnmsub.ps.512(<16 x float> %x0, <16 x float> %x1, <16 x float> %x2, i16 %x3, i32 4)
+  %res1 = call <16 x float> @llvm.x86.avx512.mask.vfnmsub.ps.512(<16 x float> %x0, <16 x float> %x1, <16 x float> %x2, i16 -1, i32 0)
+  %res2 = fadd <16 x float> %res, %res1
+  ret <16 x float> %res2
+}
+
+declare <16 x float> @llvm.x86.avx512.mask3.vfnmsub.ps.512(<16 x float>, <16 x float>, <16 x float>, i16, i32)
+
+define <16 x float>@test_int_x86_avx512_mask3_vfnmsub_ps_512(<16 x float> %x0, <16 x float> %x1, <16 x float> %x2, i16 %x3){
+; CHECK-LABEL: test_int_x86_avx512_mask3_vfnmsub_ps_512:
+; CHECK:       ## BB#0:
+; CHECK-NEXT:    kmovw %edi, %k1
+; CHECK-NEXT:    vmovaps %zmm2, %zmm3
+; CHECK-NEXT:    vfnmsub231ps %zmm1, %zmm0, %zmm3 {%k1}
+; CHECK-NEXT:    vfnmsub213ps {rn-sae}, %zmm2, %zmm1, %zmm0
+; CHECK-NEXT:    vaddps %zmm0, %zmm3, %zmm0
+; CHECK-NEXT:    retq
+  %res = call <16 x float> @llvm.x86.avx512.mask3.vfnmsub.ps.512(<16 x float> %x0, <16 x float> %x1, <16 x float> %x2, i16 %x3, i32 4)
+  %res1 = call <16 x float> @llvm.x86.avx512.mask3.vfnmsub.ps.512(<16 x float> %x0, <16 x float> %x1, <16 x float> %x2, i16 -1, i32 0)
+  %res2 = fadd <16 x float> %res, %res1
+  ret <16 x float> %res2
+}
+
+define <8 x double>@test_int_x86_avx512_mask_vfnmadd_pd_512(<8 x double> %x0, <8 x double> %x1, <8 x double> %x2, i8 %x3){
+; CHECK-LABEL: test_int_x86_avx512_mask_vfnmadd_pd_512:
+; CHECK:       ## BB#0:
+; CHECK-NEXT:    movzbl %dil, %eax
+; CHECK-NEXT:    kmovw %eax, %k1
+; CHECK-NEXT:    vmovaps %zmm0, %zmm3
+; CHECK-NEXT:    vfnmadd213pd %zmm2, %zmm1, %zmm3 {%k1}
+; CHECK-NEXT:    vfnmadd213pd {rn-sae}, %zmm2, %zmm1, %zmm0
+; CHECK-NEXT:    vaddpd %zmm0, %zmm3, %zmm0
+; CHECK-NEXT:    retq
+  %res = call <8 x double> @llvm.x86.avx512.mask.vfnmadd.pd.512(<8 x double> %x0, <8 x double> %x1, <8 x double> %x2, i8 %x3, i32 4)
+  %res1 = call <8 x double> @llvm.x86.avx512.mask.vfnmadd.pd.512(<8 x double> %x0, <8 x double> %x1, <8 x double> %x2, i8 -1, i32 0)
+  %res2 = fadd <8 x double> %res, %res1
+  ret <8 x double> %res2
+}
+
+define <16 x float>@test_int_x86_avx512_mask_vfnmadd_ps_512(<16 x float> %x0, <16 x float> %x1, <16 x float> %x2, i16 %x3){
+; CHECK-LABEL: test_int_x86_avx512_mask_vfnmadd_ps_512:
+; CHECK:       ## BB#0:
+; CHECK-NEXT:    kmovw %edi, %k1
+; CHECK-NEXT:    vmovaps %zmm0, %zmm3
+; CHECK-NEXT:    vfnmadd213ps %zmm2, %zmm1, %zmm3 {%k1}
+; CHECK-NEXT:    vfnmadd213ps {rn-sae}, %zmm2, %zmm1, %zmm0
+; CHECK-NEXT:    vaddps %zmm0, %zmm3, %zmm0
+; CHECK-NEXT:    retq
+  %res = call <16 x float> @llvm.x86.avx512.mask.vfnmadd.ps.512(<16 x float> %x0, <16 x float> %x1, <16 x float> %x2, i16 %x3, i32 4)
+  %res1 = call <16 x float> @llvm.x86.avx512.mask.vfnmadd.ps.512(<16 x float> %x0, <16 x float> %x1, <16 x float> %x2, i16 -1, i32 0)
+  %res2 = fadd <16 x float> %res, %res1
+  ret <16 x float> %res2
+}
diff --git a/test/CodeGen/X86/avx512-fma.ll b/test/CodeGen/X86/avx512-fma.ll
index d6926e2571a..ed046de005c 100644
--- a/test/CodeGen/X86/avx512-fma.ll
+++ b/test/CodeGen/X86/avx512-fma.ll
@@ -1,4 +1,5 @@
-; RUN: llc < %s -mtriple=x86_64-apple-darwin -mcpu=knl -fp-contract=fast | FileCheck %s
+; RUN: llc < %s -mtriple=x86_64-apple-darwin -mattr=+avx512f -fp-contract=fast | FileCheck %s
+; RUN: llc < %s -mtriple=x86_64-apple-darwin -mcpu=skx -fp-contract=fast | FileCheck %s --check-prefix=SKX
 
 ; CHECK-LABEL: test_x86_fmadd_ps_z
 ; CHECK: vfmadd213ps     %zmm2, %zmm1, %zmm0
@@ -58,26 +59,129 @@ define <8 x double> @test_x86_fmsub_pd_z(<8 x double> %a0, <8 x double> %a1, <8
   ret <8 x double> %res
 }
 
-define double @test_x86_fmsub_sd_z(double %a0, double %a1, double %a2) {
+define double @test_x86_fmsub_213(double %a0, double %a1, double %a2) {
+; CHECK-LABEL: test_x86_fmsub_213:
+; CHECK:       ## BB#0:
+; CHECK-NEXT:    vfmsub213sd %xmm2, %xmm0, %xmm1
+; CHECK-NEXT:    vmovaps %zmm1, %zmm0
+; CHECK-NEXT:    retq
   %x = fmul double %a0, %a1
   %res = fsub double %x, %a2
   ret double %res
 }
 
-;CHECK-LABEL: test132_br
-;CHECK: vfmadd132ps  LCP{{.*}}(%rip){1to16}
-;CHECK: ret
-define <16 x float> @test132_br(<16 x float> %a1, <16 x float> %a2) nounwind {
+define double @test_x86_fmsub_213_m(double %a0, double %a1, double * %a2_ptr) {
+; CHECK-LABEL: test_x86_fmsub_213_m:
+; CHECK:       ## BB#0:
+; CHECK-NEXT:    vfmsub213sd (%rdi), %xmm0, %xmm1
+; CHECK-NEXT:    vmovaps %zmm1, %zmm0
+; CHECK-NEXT:    retq
+  %a2 = load double , double *%a2_ptr
+  %x = fmul double %a0, %a1
+  %res = fsub double %x, %a2
+  ret double %res
+}
+
+define double @test_x86_fmsub_231_m(double %a0, double %a1, double * %a2_ptr) {
+; CHECK-LABEL: test_x86_fmsub_231_m:
+; CHECK:       ## BB#0:
+; CHECK-NEXT:    vfmsub231sd (%rdi), %xmm0, %xmm1
+; CHECK-NEXT:    vmovaps %zmm1, %zmm0
+; CHECK-NEXT:    retq
+  %a2 = load double , double *%a2_ptr
+  %x = fmul double %a0, %a2
+  %res = fsub double %x, %a1
+  ret double %res
+}
+
+define <16 x float> @test231_br(<16 x float> %a1, <16 x float> %a2) nounwind {
+; CHECK-LABEL: test231_br:
+; CHECK:       ## BB#0:
+; CHECK-NEXT:    vfmadd231ps {{.*}}(%rip){1to16}, %zmm0, %zmm1
+; CHECK-NEXT:    vmovaps %zmm1, %zmm0
+; CHECK-NEXT:    retq
   %b1 = fmul <16 x float> %a1, <float 0x3FB99999A0000000, float 0x3FB99999A0000000, float 0x3FB99999A0000000, float 0x3FB99999A0000000, float 0x3FB99999A0000000, float 0x3FB99999A0000000, float 0x3FB99999A0000000, float 0x3FB99999A0000000, float 0x3FB99999A0000000, float 0x3FB99999A0000000, float 0x3FB99999A0000000, float 0x3FB99999A0000000, float 0x3FB99999A0000000, float 0x3FB99999A0000000, float 0x3FB99999A0000000, float 0x3FB99999A0000000>
   %b2 = fadd <16 x float> %b1, %a2
   ret <16 x float> %b2
 }
 
-;CHECK-LABEL: test213_br
-;CHECK: vfmadd213ps  LCP{{.*}}(%rip){1to16}
-;CHECK: ret
 define <16 x float> @test213_br(<16 x float> %a1, <16 x float> %a2) nounwind {
+; CHECK-LABEL: test213_br:
+; CHECK:       ## BB#0:
+; CHECK-NEXT:    vfmadd213ps {{.*}}(%rip){1to16}, %zmm1, %zmm0
+; CHECK-NEXT:    retq
   %b1 = fmul <16 x float> %a1, %a2
   %b2 = fadd <16 x float> %b1, <float 0x3FB99999A0000000, float 0x3FB99999A0000000, float 0x3FB99999A0000000, float 0x3FB99999A0000000, float 0x3FB99999A0000000, float 0x3FB99999A0000000, float 0x3FB99999A0000000, float 0x3FB99999A0000000, float 0x3FB99999A0000000, float 0x3FB99999A0000000, float 0x3FB99999A0000000, float 0x3FB99999A0000000, float 0x3FB99999A0000000, float 0x3FB99999A0000000, float 0x3FB99999A0000000, float 0x3FB99999A0000000>
   ret <16 x float> %b2
 }
+
+;mask (a*c+b , a)
+define <16 x float> @test_x86_fmadd132_ps(<16 x float> %a0, <16 x float> %a1, <16 x float> *%a2_ptrt, <16 x i1> %mask) {
+; CHECK-LABEL: test_x86_fmadd132_ps:
+; CHECK:       ## BB#0:
+; CHECK-NEXT:    vpmovsxbd %xmm2, %zmm2
+; CHECK-NEXT:    vpandd {{.*}}(%rip){1to16}, %zmm2, %zmm2
+; CHECK-NEXT:    vptestmd %zmm2, %zmm2, %k1
+; CHECK-NEXT:    vfmadd132ps (%rdi), %zmm1, %zmm0 {%k1}
+; CHECK-NEXT:    retq
+;
+; SKX-LABEL: test_x86_fmadd132_ps:
+; SKX:       ## BB#0:
+; SKX-NEXT:    vpmovb2m %xmm2, %k1
+; SKX-NEXT:    vfmadd132ps (%rdi), %zmm1, %zmm0 {%k1}
+; SKX-NEXT:    retq
+  %a2   = load <16 x float>,<16 x float> *%a2_ptrt,align 1
+  %x = fmul <16 x float> %a0, %a2
+  %y = fadd <16 x float> %x, %a1
+  %res = select <16 x i1> %mask, <16 x float> %y, <16 x float> %a0
+  ret <16 x float> %res
+}
+
+;mask (a*c+b , b)
+define <16 x float> @test_x86_fmadd231_ps(<16 x float> %a0, <16 x float> %a1, <16 x float> *%a2_ptrt, <16 x i1> %mask) {
+; CHECK-LABEL: test_x86_fmadd231_ps:
+; CHECK:       ## BB#0:
+; CHECK-NEXT:    vpmovsxbd %xmm2, %zmm2
+; CHECK-NEXT:    vpandd {{.*}}(%rip){1to16}, %zmm2, %zmm2
+; CHECK-NEXT:    vptestmd %zmm2, %zmm2, %k1
+; CHECK-NEXT:    vfmadd231ps (%rdi), %zmm0, %zmm1 {%k1}
+; CHECK-NEXT:    vmovaps %zmm1, %zmm0
+; CHECK-NEXT:    retq
+;
+; SKX-LABEL: test_x86_fmadd231_ps:
+; SKX:       ## BB#0:
+; SKX-NEXT:    vpmovb2m %xmm2, %k1
+; SKX-NEXT:    vfmadd231ps (%rdi), %zmm0, %zmm1 {%k1}
+; SKX-NEXT:    vmovaps %zmm1, %zmm0
+; SKX-NEXT:    retq
+  %a2   = load <16 x float>,<16 x float> *%a2_ptrt,align 1
+  %x = fmul <16 x float> %a0, %a2
+  %y = fadd <16 x float> %x, %a1
+  %res = select <16 x i1> %mask, <16 x float> %y, <16 x float> %a1
+  ret <16 x float> %res
+}
+
+;mask (b*a+c , b)
+define <16 x float> @test_x86_fmadd213_ps(<16 x float> %a0, <16 x float> %a1, <16 x float> *%a2_ptrt, <16 x i1> %mask) {
+; CHECK-LABEL: test_x86_fmadd213_ps:
+; CHECK:       ## BB#0:
+; CHECK-NEXT:    vpmovsxbd %xmm2, %zmm2
+; CHECK-NEXT:    vpandd {{.*}}(%rip){1to16}, %zmm2, %zmm2
+; CHECK-NEXT:    vptestmd %zmm2, %zmm2, %k1
+; CHECK-NEXT:    vfmadd213ps (%rdi), %zmm0, %zmm1 {%k1}
+; CHECK-NEXT:    vmovaps %zmm1, %zmm0
+; CHECK-NEXT:    retq
+;
+; SKX-LABEL: test_x86_fmadd213_ps:
+; SKX:       ## BB#0:
+; SKX-NEXT:    vpmovb2m %xmm2, %k1
+; SKX-NEXT:    vfmadd213ps (%rdi), %zmm0, %zmm1 {%k1}
+; SKX-NEXT:    vmovaps %zmm1, %zmm0
+; SKX-NEXT:    retq
+  %a2   = load <16 x float>,<16 x float> *%a2_ptrt,align 1
+  %x = fmul <16 x float> %a1, %a0
+  %y = fadd <16 x float> %x, %a2
+  %res = select <16 x i1> %mask, <16 x float> %y, <16 x float> %a1
+  ret <16 x float> %res
+}
+
diff --git a/test/CodeGen/X86/avx512-gather-scatter-intrin.ll b/test/CodeGen/X86/avx512-gather-scatter-intrin.ll
index 0e32a1c2806..3fca5a89a6a 100644
--- a/test/CodeGen/X86/avx512-gather-scatter-intrin.ll
+++ b/test/CodeGen/X86/avx512-gather-scatter-intrin.ll
@@ -1,4 +1,4 @@
-; RUN: llc < %s -mtriple=x86_64-apple-darwin -mcpu=knl | FileCheck %s
+; RUN: llc < %s -mtriple=x86_64-apple-darwin -mcpu=skx | FileCheck %s
 
 declare <16 x float> @llvm.x86.avx512.gather.dps.512 (<16 x float>, i8*, <16 x i32>, i16, i32)
 declare void @llvm.x86.avx512.scatter.dps.512 (i8*, i16, <16 x i32>, <16 x float>, i32)
@@ -10,52 +10,60 @@ declare void @llvm.x86.avx512.scatter.qps.512 (i8*, i8, <8 x i64>, <8 x float>,
 declare <8 x double> @llvm.x86.avx512.gather.qpd.512 (<8 x double>, i8*, <8 x i64>, i8, i32)
 declare void @llvm.x86.avx512.scatter.qpd.512 (i8*, i8, <8 x i64>, <8 x double>, i32)
 
-;CHECK-LABEL: gather_mask_dps
-;CHECK: kmovw
-;CHECK: vgatherdps
-;CHECK: vpadd
-;CHECK: vscatterdps
-;CHECK: ret
 define void @gather_mask_dps(<16 x i32> %ind, <16 x float> %src, i16 %mask, i8* %base, i8* %stbuf)  {
+; CHECK-LABEL: gather_mask_dps:
+; CHECK:       ## BB#0:
+; CHECK-NEXT:    kmovw %edi, %k1
+; CHECK-NEXT:    kmovw %k1, %k2
+; CHECK-NEXT:    vgatherdps (%rsi,%zmm0,4), %zmm1 {%k2}
+; CHECK-NEXT:    vpaddd {{.*}}(%rip), %zmm0, %zmm0
+; CHECK-NEXT:    vscatterdps %zmm1, (%rdx,%zmm0,4) {%k1}
+; CHECK-NEXT:    retq
   %x = call <16 x float> @llvm.x86.avx512.gather.dps.512 (<16 x float> %src, i8* %base, <16 x i32>%ind, i16 %mask, i32 4)
   %ind2 = add <16 x i32> %ind, <i32 0, i32 1, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3>
   call void @llvm.x86.avx512.scatter.dps.512 (i8* %stbuf, i16 %mask, <16 x i32>%ind2, <16 x float> %x, i32 4)
   ret void
 }
 
-;CHECK-LABEL: gather_mask_dpd
-;CHECK: kmovw
-;CHECK: vgatherdpd
-;CHECK: vpadd
-;CHECK: vscatterdpd
-;CHECK: ret
 define void @gather_mask_dpd(<8 x i32> %ind, <8 x double> %src, i8 %mask, i8* %base, i8* %stbuf)  {
+; CHECK-LABEL: gather_mask_dpd:
+; CHECK:       ## BB#0:
+; CHECK-NEXT:    kmovb %edi, %k1
+; CHECK-NEXT:    kmovw %k1, %k2
+; CHECK-NEXT:    vgatherdpd (%rsi,%ymm0,4), %zmm1 {%k2}
+; CHECK-NEXT:    vpaddd {{.*}}(%rip), %ymm0, %ymm0
+; CHECK-NEXT:    vscatterdpd %zmm1, (%rdx,%ymm0,4) {%k1}
+; CHECK-NEXT:    retq
   %x = call <8 x double> @llvm.x86.avx512.gather.dpd.512 (<8 x double> %src, i8* %base, <8 x i32>%ind, i8 %mask, i32 4)
   %ind2 = add <8 x i32> %ind, <i32 0, i32 1, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3>
   call void @llvm.x86.avx512.scatter.dpd.512 (i8* %stbuf, i8 %mask, <8 x i32>%ind2, <8 x double> %x, i32 4)
   ret void
 }
 
-;CHECK-LABEL: gather_mask_qps
-;CHECK: kmovw
-;CHECK: vgatherqps
-;CHECK: vpadd
-;CHECK: vscatterqps
-;CHECK: ret
 define void @gather_mask_qps(<8 x i64> %ind, <8 x float> %src, i8 %mask, i8* %base, i8* %stbuf)  {
+; CHECK-LABEL: gather_mask_qps:
+; CHECK:       ## BB#0:
+; CHECK-NEXT:    kmovb %edi, %k1
+; CHECK-NEXT:    kmovw %k1, %k2
+; CHECK-NEXT:    vgatherqps (%rsi,%zmm0,4), %ymm1 {%k2}
+; CHECK-NEXT:    vpaddq {{.*}}(%rip), %zmm0, %zmm0
+; CHECK-NEXT:    vscatterqps %ymm1, (%rdx,%zmm0,4) {%k1}
+; CHECK-NEXT:    retq
   %x = call <8 x float> @llvm.x86.avx512.gather.qps.512 (<8 x float> %src, i8* %base, <8 x i64>%ind, i8 %mask, i32 4)
   %ind2 = add <8 x i64> %ind, <i64 0, i64 1, i64 2, i64 3, i64 0, i64 1, i64 2, i64 3>
   call void @llvm.x86.avx512.scatter.qps.512 (i8* %stbuf, i8 %mask, <8 x i64>%ind2, <8 x float> %x, i32 4)
   ret void
 }
 
-;CHECK-LABEL: gather_mask_qpd
-;CHECK: kmovw
-;CHECK: vgatherqpd
-;CHECK: vpadd
-;CHECK: vscatterqpd
-;CHECK: ret
 define void @gather_mask_qpd(<8 x i64> %ind, <8 x double> %src, i8 %mask, i8* %base, i8* %stbuf)  {
+; CHECK-LABEL: gather_mask_qpd:
+; CHECK:       ## BB#0:
+; CHECK-NEXT:    kmovb %edi, %k1
+; CHECK-NEXT:    kmovw %k1, %k2
+; CHECK-NEXT:    vgatherqpd (%rsi,%zmm0,4), %zmm1 {%k2}
+; CHECK-NEXT:    vpaddq {{.*}}(%rip), %zmm0, %zmm0
+; CHECK-NEXT:    vscatterqpd %zmm1, (%rdx,%zmm0,4) {%k1}
+; CHECK-NEXT:    retq
   %x = call <8 x double> @llvm.x86.avx512.gather.qpd.512 (<8 x double> %src, i8* %base, <8 x i64>%ind, i8 %mask, i32 4)
   %ind2 = add <8 x i64> %ind, <i64 0, i64 1, i64 2, i64 3, i64 0, i64 1, i64 2, i64 3>
   call void @llvm.x86.avx512.scatter.qpd.512 (i8* %stbuf, i8 %mask, <8 x i64>%ind2, <8 x double> %x, i32 4)
@@ -74,162 +82,710 @@ declare void @llvm.x86.avx512.scatter.qpi.512 (i8*, i8, <8 x i64>, <8 x i32>, i3
 declare <8 x i64> @llvm.x86.avx512.gather.qpq.512 (<8 x i64>, i8*, <8 x i64>, i8, i32)
 declare void @llvm.x86.avx512.scatter.qpq.512 (i8*, i8, <8 x i64>, <8 x i64>, i32)
 
-;CHECK-LABEL: gather_mask_dd
-;CHECK: kmovw
-;CHECK: vpgatherdd
-;CHECK: vpadd
-;CHECK: vpscatterdd
-;CHECK: ret
 define void @gather_mask_dd(<16 x i32> %ind, <16 x i32> %src, i16 %mask, i8* %base, i8* %stbuf)  {
+; CHECK-LABEL: gather_mask_dd:
+; CHECK:       ## BB#0:
+; CHECK-NEXT:    kmovw %edi, %k1
+; CHECK-NEXT:    kmovw %k1, %k2
+; CHECK-NEXT:    vpgatherdd (%rsi,%zmm0,4), %zmm1 {%k2}
+; CHECK-NEXT:    vpaddd {{.*}}(%rip), %zmm0, %zmm0
+; CHECK-NEXT:    vpscatterdd %zmm1, (%rdx,%zmm0,4) {%k1}
+; CHECK-NEXT:    retq
   %x = call <16 x i32> @llvm.x86.avx512.gather.dpi.512 (<16 x i32> %src, i8* %base, <16 x i32>%ind, i16 %mask, i32 4)
   %ind2 = add <16 x i32> %ind, <i32 0, i32 1, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3>
   call void @llvm.x86.avx512.scatter.dpi.512 (i8* %stbuf, i16 %mask, <16 x i32>%ind2, <16 x i32> %x, i32 4)
   ret void
 }
 
-;CHECK-LABEL: gather_mask_qd
-;CHECK: kmovw
-;CHECK: vpgatherqd
-;CHECK: vpadd
-;CHECK: vpscatterqd
-;CHECK: ret
 define void @gather_mask_qd(<8 x i64> %ind, <8 x i32> %src, i8 %mask, i8* %base, i8* %stbuf)  {
+; CHECK-LABEL: gather_mask_qd:
+; CHECK:       ## BB#0:
+; CHECK-NEXT:    kmovb %edi, %k1
+; CHECK-NEXT:    kmovw %k1, %k2
+; CHECK-NEXT:    vpgatherqd (%rsi,%zmm0,4), %ymm1 {%k2}
+; CHECK-NEXT:    vpaddq {{.*}}(%rip), %zmm0, %zmm0
+; CHECK-NEXT:    vpscatterqd %ymm1, (%rdx,%zmm0,4) {%k1}
+; CHECK-NEXT:    retq
   %x = call <8 x i32> @llvm.x86.avx512.gather.qpi.512 (<8 x i32> %src, i8* %base, <8 x i64>%ind, i8 %mask, i32 4)
   %ind2 = add <8 x i64> %ind, <i64 0, i64 1, i64 2, i64 3, i64 0, i64 1, i64 2, i64 3>
   call void @llvm.x86.avx512.scatter.qpi.512 (i8* %stbuf, i8 %mask, <8 x i64>%ind2, <8 x i32> %x, i32 4)
   ret void
 }
 
-;CHECK-LABEL: gather_mask_qq
-;CHECK: kmovw
-;CHECK: vpgatherqq
-;CHECK: vpadd
-;CHECK: vpscatterqq
-;CHECK: ret
 define void @gather_mask_qq(<8 x i64> %ind, <8 x i64> %src, i8 %mask, i8* %base, i8* %stbuf)  {
+; CHECK-LABEL: gather_mask_qq:
+; CHECK:       ## BB#0:
+; CHECK-NEXT:    kmovb %edi, %k1
+; CHECK-NEXT:    kmovw %k1, %k2
+; CHECK-NEXT:    vpgatherqq (%rsi,%zmm0,4), %zmm1 {%k2}
+; CHECK-NEXT:    vpaddq {{.*}}(%rip), %zmm0, %zmm0
+; CHECK-NEXT:    vpscatterqq %zmm1, (%rdx,%zmm0,4) {%k1}
+; CHECK-NEXT:    retq
   %x = call <8 x i64> @llvm.x86.avx512.gather.qpq.512 (<8 x i64> %src, i8* %base, <8 x i64>%ind, i8 %mask, i32 4)
   %ind2 = add <8 x i64> %ind, <i64 0, i64 1, i64 2, i64 3, i64 0, i64 1, i64 2, i64 3>
   call void @llvm.x86.avx512.scatter.qpq.512 (i8* %stbuf, i8 %mask, <8 x i64>%ind2, <8 x i64> %x, i32 4)
   ret void
 }
 
-;CHECK-LABEL: gather_mask_dq
-;CHECK: kmovw
-;CHECK: vpgatherdq
-;CHECK: vpadd
-;CHECK: vpscatterdq
-;CHECK: ret
 define void @gather_mask_dq(<8 x i32> %ind, <8 x i64> %src, i8 %mask, i8* %base, i8* %stbuf)  {
+; CHECK-LABEL: gather_mask_dq:
+; CHECK:       ## BB#0:
+; CHECK-NEXT:    kmovb %edi, %k1
+; CHECK-NEXT:    kmovw %k1, %k2
+; CHECK-NEXT:    vpgatherdq (%rsi,%ymm0,4), %zmm1 {%k2}
+; CHECK-NEXT:    vpaddd {{.*}}(%rip), %ymm0, %ymm0
+; CHECK-NEXT:    vpscatterdq %zmm1, (%rdx,%ymm0,4) {%k1}
+; CHECK-NEXT:    retq
   %x = call <8 x i64> @llvm.x86.avx512.gather.dpq.512 (<8 x i64> %src, i8* %base, <8 x i32>%ind, i8 %mask, i32 4)
   %ind2 = add <8 x i32> %ind, <i32 0, i32 1, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3>
   call void @llvm.x86.avx512.scatter.dpq.512 (i8* %stbuf, i8 %mask, <8 x i32>%ind2, <8 x i64> %x, i32 4)
   ret void
 }
 
-
-;CHECK-LABEL: gather_mask_dpd_execdomain
-;CHECK: vgatherdpd
-;CHECK: vmovapd
-;CHECK: ret
 define void @gather_mask_dpd_execdomain(<8 x i32> %ind, <8 x double> %src, i8 %mask, i8* %base, <8 x double>* %stbuf)  {
+; CHECK-LABEL: gather_mask_dpd_execdomain:
+; CHECK:       ## BB#0:
+; CHECK-NEXT:    kmovb %edi, %k1
+; CHECK-NEXT:    vgatherdpd (%rsi,%ymm0,4), %zmm1 {%k1}
+; CHECK-NEXT:    vmovapd %zmm1, (%rdx)
+; CHECK-NEXT:    retq
   %x = call <8 x double> @llvm.x86.avx512.gather.dpd.512 (<8 x double> %src, i8* %base, <8 x i32>%ind, i8 %mask, i32 4)
   store <8 x double> %x, <8 x double>* %stbuf
   ret void
 }
 
-;CHECK-LABEL: gather_mask_qpd_execdomain
-;CHECK: vgatherqpd
-;CHECK: vmovapd
-;CHECK: ret
 define void @gather_mask_qpd_execdomain(<8 x i64> %ind, <8 x double> %src, i8 %mask, i8* %base, <8 x double>* %stbuf)  {
+; CHECK-LABEL: gather_mask_qpd_execdomain:
+; CHECK:       ## BB#0:
+; CHECK-NEXT:    kmovb %edi, %k1
+; CHECK-NEXT:    vgatherqpd (%rsi,%zmm0,4), %zmm1 {%k1}
+; CHECK-NEXT:    vmovapd %zmm1, (%rdx)
+; CHECK-NEXT:    retq
   %x = call <8 x double> @llvm.x86.avx512.gather.qpd.512 (<8 x double> %src, i8* %base, <8 x i64>%ind, i8 %mask, i32 4)
   store <8 x double> %x, <8 x double>* %stbuf
   ret void
 }
 
-;CHECK-LABEL: gather_mask_dps_execdomain
-;CHECK: vgatherdps
-;CHECK: vmovaps 
-;CHECK: ret
 define <16 x float> @gather_mask_dps_execdomain(<16 x i32> %ind, <16 x float> %src, i16 %mask, i8* %base)  {
+; CHECK-LABEL: gather_mask_dps_execdomain:
+; CHECK:       ## BB#0:
+; CHECK-NEXT:    kmovw %edi, %k1
+; CHECK-NEXT:    vgatherdps (%rsi,%zmm0,4), %zmm1 {%k1}
+; CHECK-NEXT:    vmovaps %zmm1, %zmm0
+; CHECK-NEXT:    retq
   %res = call <16 x float> @llvm.x86.avx512.gather.dps.512 (<16 x float> %src, i8* %base, <16 x i32>%ind, i16 %mask, i32 4)
   ret <16 x float> %res;
 }
 
-;CHECK-LABEL: gather_mask_qps_execdomain
-;CHECK: vgatherqps
-;CHECK: vmovaps
-;CHECK: ret
 define <8 x float> @gather_mask_qps_execdomain(<8 x i64> %ind, <8 x float> %src, i8 %mask, i8* %base)  {
+; CHECK-LABEL: gather_mask_qps_execdomain:
+; CHECK:       ## BB#0:
+; CHECK-NEXT:    kmovb %edi, %k1
+; CHECK-NEXT:    vgatherqps (%rsi,%zmm0,4), %ymm1 {%k1}
+; CHECK-NEXT:    vmovaps %zmm1, %zmm0
+; CHECK-NEXT:    retq
   %res = call <8 x float> @llvm.x86.avx512.gather.qps.512 (<8 x float> %src, i8* %base, <8 x i64>%ind, i8 %mask, i32 4)
   ret <8 x float> %res;
 }
 
-;CHECK-LABEL: scatter_mask_dpd_execdomain
-;CHECK: vmovapd
-;CHECK: vscatterdpd
-;CHECK: ret
 define void @scatter_mask_dpd_execdomain(<8 x i32> %ind, <8 x double>* %src, i8 %mask, i8* %base, i8* %stbuf)  {
-  %x = load <8 x double>, <8 x double>* %src, align 64 
+; CHECK-LABEL: scatter_mask_dpd_execdomain:
+; CHECK:       ## BB#0:
+; CHECK-NEXT:    kmovb %esi, %k1
+; CHECK-NEXT:    vmovapd (%rdi), %zmm1
+; CHECK-NEXT:    vscatterdpd %zmm1, (%rcx,%ymm0,4) {%k1}
+; CHECK-NEXT:    retq
+  %x = load <8 x double>, <8 x double>* %src, align 64
   call void @llvm.x86.avx512.scatter.dpd.512 (i8* %stbuf, i8 %mask, <8 x i32>%ind, <8 x double> %x, i32 4)
   ret void
 }
 
-;CHECK-LABEL: scatter_mask_qpd_execdomain
-;CHECK: vmovapd
-;CHECK: vscatterqpd
-;CHECK: ret
 define void @scatter_mask_qpd_execdomain(<8 x i64> %ind, <8 x double>* %src, i8 %mask, i8* %base, i8* %stbuf)  {
+; CHECK-LABEL: scatter_mask_qpd_execdomain:
+; CHECK:       ## BB#0:
+; CHECK-NEXT:    kmovb %esi, %k1
+; CHECK-NEXT:    vmovapd (%rdi), %zmm1
+; CHECK-NEXT:    vscatterqpd %zmm1, (%rcx,%zmm0,4) {%k1}
+; CHECK-NEXT:    retq
   %x = load <8 x double>, <8 x double>* %src, align 64
   call void @llvm.x86.avx512.scatter.qpd.512 (i8* %stbuf, i8 %mask, <8 x i64>%ind, <8 x double> %x, i32 4)
   ret void
 }
 
-;CHECK-LABEL: scatter_mask_dps_execdomain
-;CHECK: vmovaps
-;CHECK: vscatterdps
-;CHECK: ret
 define void @scatter_mask_dps_execdomain(<16 x i32> %ind, <16 x float>* %src, i16 %mask, i8* %base, i8* %stbuf)  {
+; CHECK-LABEL: scatter_mask_dps_execdomain:
+; CHECK:       ## BB#0:
+; CHECK-NEXT:    kmovw %esi, %k1
+; CHECK-NEXT:    vmovaps (%rdi), %zmm1
+; CHECK-NEXT:    vscatterdps %zmm1, (%rcx,%zmm0,4) {%k1}
+; CHECK-NEXT:    retq
   %x = load <16 x float>, <16 x float>* %src, align 64
   call void @llvm.x86.avx512.scatter.dps.512 (i8* %stbuf, i16 %mask, <16 x i32>%ind, <16 x float> %x, i32 4)
   ret void
 }
 
-;CHECK-LABEL: scatter_mask_qps_execdomain
-;CHECK: vmovaps
-;CHECK: vscatterqps
-;CHECK: ret
 define void @scatter_mask_qps_execdomain(<8 x i64> %ind, <8 x float>* %src, i8 %mask, i8* %base, i8* %stbuf)  {
-  %x = load <8 x float>, <8 x float>* %src, align 32 
+; CHECK-LABEL: scatter_mask_qps_execdomain:
+; CHECK:       ## BB#0:
+; CHECK-NEXT:    kmovb %esi, %k1
+; CHECK-NEXT:    vmovaps (%rdi), %ymm1
+; CHECK-NEXT:    vscatterqps %ymm1, (%rcx,%zmm0,4) {%k1}
+; CHECK-NEXT:    retq
+  %x = load <8 x float>, <8 x float>* %src, align 32
   call void @llvm.x86.avx512.scatter.qps.512 (i8* %stbuf, i8 %mask, <8 x i64>%ind, <8 x float> %x, i32 4)
   ret void
 }
 
-;CHECK-LABEL: gather_qps
-;CHECK: kxnorw
-;CHECK: vgatherqps
-;CHECK: vpadd
-;CHECK: vscatterqps
-;CHECK: ret
 define void @gather_qps(<8 x i64> %ind, <8 x float> %src, i8* %base, i8* %stbuf)  {
+; CHECK-LABEL: gather_qps:
+; CHECK:       ## BB#0:
+; CHECK-NEXT:    kxnorw %k1, %k1, %k1
+; CHECK-NEXT:    kxnorw %k2, %k2, %k2
+; CHECK-NEXT:    vgatherqps (%rdi,%zmm0,4), %ymm1 {%k2}
+; CHECK-NEXT:    vpaddq {{.*}}(%rip), %zmm0, %zmm0
+; CHECK-NEXT:    vscatterqps %ymm1, (%rsi,%zmm0,4) {%k1}
+; CHECK-NEXT:    retq
   %x = call <8 x float> @llvm.x86.avx512.gather.qps.512 (<8 x float> %src, i8* %base, <8 x i64>%ind, i8 -1, i32 4)
   %ind2 = add <8 x i64> %ind, <i64 0, i64 1, i64 2, i64 3, i64 0, i64 1, i64 2, i64 3>
   call void @llvm.x86.avx512.scatter.qps.512 (i8* %stbuf, i8 -1, <8 x i64>%ind2, <8 x float> %x, i32 4)
   ret void
 }
 
-;CHECK-LABEL: prefetch
-;CHECK: gatherpf0
-;CHECK: gatherpf1
-;CHECK: scatterpf0
-;CHECK: scatterpf1
-;CHECK: ret
 declare  void @llvm.x86.avx512.gatherpf.qps.512(i8, <8 x i64>, i8* , i32, i32);
 declare  void @llvm.x86.avx512.scatterpf.qps.512(i8, <8 x i64>, i8* , i32, i32);
 define void @prefetch(<8 x i64> %ind, i8* %base) {
+; CHECK-LABEL: prefetch:
+; CHECK:       ## BB#0:
+; CHECK-NEXT:    kxnorw %k1, %k1, %k1
+; CHECK-NEXT:    vgatherpf0qps (%rdi,%zmm0,4) {%k1}
+; CHECK-NEXT:    vgatherpf1qps (%rdi,%zmm0,4) {%k1}
+; CHECK-NEXT:    vscatterpf0qps (%rdi,%zmm0,2) {%k1}
+; CHECK-NEXT:    vscatterpf1qps (%rdi,%zmm0,2) {%k1}
+; CHECK-NEXT:    retq
   call void @llvm.x86.avx512.gatherpf.qps.512(i8 -1, <8 x i64> %ind, i8* %base, i32 4, i32 0)
   call void @llvm.x86.avx512.gatherpf.qps.512(i8 -1, <8 x i64> %ind, i8* %base, i32 4, i32 1)
   call void @llvm.x86.avx512.scatterpf.qps.512(i8 -1, <8 x i64> %ind, i8* %base, i32 2, i32 0)
   call void @llvm.x86.avx512.scatterpf.qps.512(i8 -1, <8 x i64> %ind, i8* %base, i32 2, i32 1)
   ret void
 }
+
+
+declare <2 x double> @llvm.x86.avx512.gather3div2.df(<2 x double>, i8*, <2 x i64>, i8, i32)
+
+define <2 x double>@test_int_x86_avx512_gather3div2_df(<2 x double> %x0, i8* %x1, <2 x i64> %x2, i8 %x3) {
+; CHECK-LABEL: test_int_x86_avx512_gather3div2_df:
+; CHECK:       ## BB#0:
+; CHECK-NEXT:    kmovb %esi, %k1
+; CHECK-NEXT:    vmovaps %zmm0, %zmm2
+; CHECK-NEXT:    vgatherqpd (%rdi,%xmm1,4), %xmm2 {%k1}
+; CHECK-NEXT:    kxnorw %k1, %k1, %k1
+; CHECK-NEXT:    vgatherqpd (%rdi,%xmm1,0), %xmm0 {%k1}
+; CHECK-NEXT:    vaddpd %xmm0, %xmm2, %xmm0
+; CHECK-NEXT:    retq
+  %res = call <2 x double> @llvm.x86.avx512.gather3div2.df(<2 x double> %x0, i8* %x1, <2 x i64> %x2, i8 %x3, i32 4)
+  %res1 = call <2 x double> @llvm.x86.avx512.gather3div2.df(<2 x double> %x0, i8* %x1, <2 x i64> %x2, i8 -1, i32 0)
+  %res2 = fadd <2 x double> %res, %res1
+  ret <2 x double> %res2
+}
+
+declare <4 x i32> @llvm.x86.avx512.gather3div2.di(<2 x i64>, i8*, <2 x i64>, i8, i32)
+
+define <4 x i32>@test_int_x86_avx512_gather3div2_di(<2 x i64> %x0, i8* %x1, <2 x i64> %x2, i8 %x3) {
+; CHECK-LABEL: test_int_x86_avx512_gather3div2_di:
+; CHECK:       ## BB#0:
+; CHECK-NEXT:    kmovb %esi, %k1
+; CHECK-NEXT:    vpgatherqq (%rdi,%xmm1,8), %xmm0 {%k1}
+; CHECK-NEXT:    vpaddd %xmm0, %xmm0, %xmm0
+; CHECK-NEXT:    retq
+  %res = call <4 x i32> @llvm.x86.avx512.gather3div2.di(<2 x i64> %x0, i8* %x1, <2 x i64> %x2, i8 %x3, i32 8)
+  %res1 = call <4 x i32> @llvm.x86.avx512.gather3div2.di(<2 x i64> %x0, i8* %x1, <2 x i64> %x2, i8 %x3, i32 8)
+  %res2 = add <4 x i32> %res, %res1
+  ret <4 x i32> %res2
+}
+
+declare <4 x double> @llvm.x86.avx512.gather3div4.df(<4 x double>, i8*, <4 x i64>, i8, i32)
+
+define <4 x double>@test_int_x86_avx512_gather3div4_df(<4 x double> %x0, i8* %x1, <4 x i64> %x2, i8 %x3) {
+; CHECK-LABEL: test_int_x86_avx512_gather3div4_df:
+; CHECK:       ## BB#0:
+; CHECK-NEXT:    kmovb %esi, %k1
+; CHECK-NEXT:    vmovaps %zmm0, %zmm2
+; CHECK-NEXT:    vgatherqpd (%rdi,%ymm1,4), %ymm2 {%k1}
+; CHECK-NEXT:    kxnorw %k1, %k1, %k1
+; CHECK-NEXT:    vgatherqpd (%rdi,%ymm1,0), %ymm0 {%k1}
+; CHECK-NEXT:    vaddpd %ymm0, %ymm2, %ymm0
+; CHECK-NEXT:    retq
+  %res = call <4 x double> @llvm.x86.avx512.gather3div4.df(<4 x double> %x0, i8* %x1, <4 x i64> %x2, i8 %x3, i32 4)
+  %res1 = call <4 x double> @llvm.x86.avx512.gather3div4.df(<4 x double> %x0, i8* %x1, <4 x i64> %x2, i8 -1, i32 0)
+  %res2 = fadd <4 x double> %res, %res1
+  ret <4 x double> %res2
+}
+
+declare <8 x i32> @llvm.x86.avx512.gather3div4.di(<4 x i64>, i8*, <4 x i64>, i8, i32)
+
+define <8 x i32>@test_int_x86_avx512_gather3div4_di(<4 x i64> %x0, i8* %x1, <4 x i64> %x2, i8 %x3) {
+; CHECK-LABEL: test_int_x86_avx512_gather3div4_di:
+; CHECK:       ## BB#0:
+; CHECK-NEXT:    kmovb %esi, %k1
+; CHECK-NEXT:    vmovaps %zmm0, %zmm2
+; CHECK-NEXT:    vpgatherqq (%rdi,%ymm1,8), %ymm2 {%k1}
+; CHECK-NEXT:    kxnorw %k1, %k1, %k1
+; CHECK-NEXT:    vpgatherqq (%rdi,%ymm1,8), %ymm0 {%k1}
+; CHECK-NEXT:    vpaddd %ymm0, %ymm2, %ymm0
+; CHECK-NEXT:    retq
+  %res = call <8 x i32> @llvm.x86.avx512.gather3div4.di(<4 x i64> %x0, i8* %x1, <4 x i64> %x2, i8 %x3, i32 8)
+  %res1 = call <8 x i32> @llvm.x86.avx512.gather3div4.di(<4 x i64> %x0, i8* %x1, <4 x i64> %x2, i8 -1, i32 8)
+  %res2 = add <8 x i32> %res, %res1
+  ret <8 x i32> %res2
+}
+
+declare <4 x float> @llvm.x86.avx512.gather3div4.sf(<4 x float>, i8*, <2 x i64>, i8, i32)
+
+define <4 x float>@test_int_x86_avx512_gather3div4_sf(<4 x float> %x0, i8* %x1, <2 x i64> %x2, i8 %x3) {
+; CHECK-LABEL: test_int_x86_avx512_gather3div4_sf:
+; CHECK:       ## BB#0:
+; CHECK-NEXT:    kmovb %esi, %k1
+; CHECK-NEXT:    vmovaps %zmm0, %zmm2
+; CHECK-NEXT:    vgatherqps (%rdi,%xmm1,4), %xmm2 {%k1}
+; CHECK-NEXT:    kxnorw %k1, %k1, %k1
+; CHECK-NEXT:    vgatherqps (%rdi,%xmm1,0), %xmm0 {%k1}
+; CHECK-NEXT:    vaddps %xmm0, %xmm2, %xmm0
+; CHECK-NEXT:    retq
+  %res = call <4 x float> @llvm.x86.avx512.gather3div4.sf(<4 x float> %x0, i8* %x1, <2 x i64> %x2, i8 %x3, i32 4)
+  %res1 = call <4 x float> @llvm.x86.avx512.gather3div4.sf(<4 x float> %x0, i8* %x1, <2 x i64> %x2, i8 -1, i32 0)
+  %res2 = fadd <4 x float> %res, %res1
+  ret <4 x float> %res2
+}
+
+declare <4 x i32> @llvm.x86.avx512.gather3div4.si(<4 x i32>, i8*, <2 x i64>, i8, i32)
+
+define <4 x i32>@test_int_x86_avx512_gather3div4_si(<4 x i32> %x0, i8* %x1, <2 x i64> %x2, i8 %x3) {
+; CHECK-LABEL: test_int_x86_avx512_gather3div4_si:
+; CHECK:       ## BB#0:
+; CHECK-NEXT:    kmovb %esi, %k1
+; CHECK-NEXT:    kxnorw %k2, %k2, %k2
+; CHECK-NEXT:    vmovaps %zmm0, %zmm2
+; CHECK-NEXT:    vpgatherqd (%rdi,%xmm1,4), %xmm2 {%k2}
+; CHECK-NEXT:    vpgatherqd (%rdi,%xmm1,4), %xmm0 {%k1}
+; CHECK-NEXT:    vpaddd %xmm0, %xmm2, %xmm0
+; CHECK-NEXT:    retq
+  %res = call <4 x i32> @llvm.x86.avx512.gather3div4.si(<4 x i32> %x0, i8* %x1, <2 x i64> %x2, i8 -1, i32 4)
+  %res1 = call <4 x i32> @llvm.x86.avx512.gather3div4.si(<4 x i32> %x0, i8* %x1, <2 x i64> %x2, i8 %x3, i32 4)
+  %res2 = add <4 x i32> %res, %res1
+  ret <4 x i32> %res2
+}
+
+declare <4 x float> @llvm.x86.avx512.gather3div8.sf(<4 x float>, i8*, <4 x i64>, i8, i32)
+
+define <4 x float>@test_int_x86_avx512_gather3div8_sf(<4 x float> %x0, i8* %x1, <4 x i64> %x2, i8 %x3) {
+; CHECK-LABEL: test_int_x86_avx512_gather3div8_sf:
+; CHECK:       ## BB#0:
+; CHECK-NEXT:    kmovb %esi, %k1
+; CHECK-NEXT:    vmovaps %zmm0, %zmm2
+; CHECK-NEXT:    vgatherqps (%rdi,%ymm1,4), %xmm2 {%k1}
+; CHECK-NEXT:    kxnorw %k1, %k1, %k1
+; CHECK-NEXT:    vgatherqps (%rdi,%ymm1,0), %xmm0 {%k1}
+; CHECK-NEXT:    vaddps %xmm0, %xmm2, %xmm0
+; CHECK-NEXT:    retq
+  %res = call <4 x float> @llvm.x86.avx512.gather3div8.sf(<4 x float> %x0, i8* %x1, <4 x i64> %x2, i8 %x3, i32 4)
+  %res1 = call <4 x float> @llvm.x86.avx512.gather3div8.sf(<4 x float> %x0, i8* %x1, <4 x i64> %x2, i8 -1, i32 0)
+  %res2 = fadd <4 x float> %res, %res1
+  ret <4 x float> %res2
+}
+
+declare <4 x i32> @llvm.x86.avx512.gather3div8.si(<4 x i32>, i8*, <4 x i64>, i8, i32)
+
+define <4 x i32>@test_int_x86_avx512_gather3div8_si(<4 x i32> %x0, i8* %x1, <4 x i64> %x2, i8 %x3) {
+; CHECK-LABEL: test_int_x86_avx512_gather3div8_si:
+; CHECK:       ## BB#0:
+; CHECK-NEXT:    kmovb %esi, %k1
+; CHECK-NEXT:    vmovaps %zmm0, %zmm2
+; CHECK-NEXT:    kmovw %k1, %k2
+; CHECK-NEXT:    vpgatherqd (%rdi,%ymm1,4), %xmm2 {%k2}
+; CHECK-NEXT:    vpgatherqd (%rdi,%ymm1,2), %xmm0 {%k1}
+; CHECK-NEXT:    vpaddd %xmm0, %xmm2, %xmm0
+; CHECK-NEXT:    retq
+  %res = call <4 x i32> @llvm.x86.avx512.gather3div8.si(<4 x i32> %x0, i8* %x1, <4 x i64> %x2, i8 %x3, i32 4)
+  %res1 = call <4 x i32> @llvm.x86.avx512.gather3div8.si(<4 x i32> %x0, i8* %x1, <4 x i64> %x2, i8 %x3, i32 2)
+  %res2 = add <4 x i32> %res, %res1
+  ret <4 x i32> %res2
+}
+
+declare <2 x double> @llvm.x86.avx512.gather3siv2.df(<2 x double>, i8*, <4 x i32>, i8, i32)
+
+define <2 x double>@test_int_x86_avx512_gather3siv2_df(<2 x double> %x0, i8* %x1, <4 x i32> %x2, i8 %x3) {
+; CHECK-LABEL: test_int_x86_avx512_gather3siv2_df:
+; CHECK:       ## BB#0:
+; CHECK-NEXT:    kmovb %esi, %k1
+; CHECK-NEXT:    vmovaps %zmm0, %zmm2
+; CHECK-NEXT:    vgatherdpd (%rdi,%xmm1,4), %xmm2 {%k1}
+; CHECK-NEXT:    kxnorw %k1, %k1, %k1
+; CHECK-NEXT:    vgatherdpd (%rdi,%xmm1,0), %xmm0 {%k1}
+; CHECK-NEXT:    vaddpd %xmm0, %xmm2, %xmm0
+; CHECK-NEXT:    retq
+  %res = call <2 x double> @llvm.x86.avx512.gather3siv2.df(<2 x double> %x0, i8* %x1, <4 x i32> %x2, i8 %x3, i32 4)
+  %res1 = call <2 x double> @llvm.x86.avx512.gather3siv2.df(<2 x double> %x0, i8* %x1, <4 x i32> %x2, i8 -1, i32 0)
+  %res2 = fadd <2 x double> %res, %res1
+  ret <2 x double> %res2
+}
+
+declare <4 x i32> @llvm.x86.avx512.gather3siv2.di(<2 x i64>, i8*, <4 x i32>, i8, i32)
+
+define <4 x i32>@test_int_x86_avx512_gather3siv2_di(<2 x i64> %x0, i8* %x1, <4 x i32> %x2, i8 %x3) {
+; CHECK-LABEL: test_int_x86_avx512_gather3siv2_di:
+; CHECK:       ## BB#0:
+; CHECK-NEXT:    kmovb %esi, %k1
+; CHECK-NEXT:    vpgatherdq (%rdi,%xmm1,8), %xmm0 {%k1}
+; CHECK-NEXT:    vpaddd %xmm0, %xmm0, %xmm0
+; CHECK-NEXT:    retq
+  %res = call <4 x i32> @llvm.x86.avx512.gather3siv2.di(<2 x i64> %x0, i8* %x1, <4 x i32> %x2, i8 %x3, i32 8)
+  %res1 = call <4 x i32> @llvm.x86.avx512.gather3siv2.di(<2 x i64> %x0, i8* %x1, <4 x i32> %x2, i8 %x3, i32 8)
+  %res2 = add <4 x i32> %res, %res1
+  ret <4 x i32> %res2
+}
+
+declare <4 x double> @llvm.x86.avx512.gather3siv4.df(<4 x double>, i8*, <4 x i32>, i8, i32)
+
+define <4 x double>@test_int_x86_avx512_gather3siv4_df(<4 x double> %x0, i8* %x1, <4 x i32> %x2, i8 %x3) {
+; CHECK-LABEL: test_int_x86_avx512_gather3siv4_df:
+; CHECK:       ## BB#0:
+; CHECK-NEXT:    kmovb %esi, %k1
+; CHECK-NEXT:    vmovaps %zmm0, %zmm2
+; CHECK-NEXT:    vgatherdpd (%rdi,%xmm1,4), %ymm2 {%k1}
+; CHECK-NEXT:    kxnorw %k1, %k1, %k1
+; CHECK-NEXT:    vgatherdpd (%rdi,%xmm1,0), %ymm0 {%k1}
+; CHECK-NEXT:    vaddpd %ymm0, %ymm2, %ymm0
+; CHECK-NEXT:    retq
+  %res = call <4 x double> @llvm.x86.avx512.gather3siv4.df(<4 x double> %x0, i8* %x1, <4 x i32> %x2, i8 %x3, i32 4)
+  %res1 = call <4 x double> @llvm.x86.avx512.gather3siv4.df(<4 x double> %x0, i8* %x1, <4 x i32> %x2, i8 -1, i32 0)
+  %res2 = fadd <4 x double> %res, %res1
+  ret <4 x double> %res2
+}
+
+declare <8 x i32> @llvm.x86.avx512.gather3siv4.di(<4 x i64>, i8*, <4 x i32>, i8, i32)
+
+define <8 x i32>@test_int_x86_avx512_gather3siv4_di(<4 x i64> %x0, i8* %x1, <4 x i32> %x2, i8 %x3) {
+; CHECK-LABEL: test_int_x86_avx512_gather3siv4_di:
+; CHECK:       ## BB#0:
+; CHECK-NEXT:    kmovb %esi, %k1
+; CHECK-NEXT:    vpgatherdq (%rdi,%xmm1,8), %ymm0 {%k1}
+; CHECK-NEXT:    vpaddd %ymm0, %ymm0, %ymm0
+; CHECK-NEXT:    retq
+  %res = call <8 x i32> @llvm.x86.avx512.gather3siv4.di(<4 x i64> %x0, i8* %x1, <4 x i32> %x2, i8 %x3, i32 8)
+  %res1 = call <8 x i32> @llvm.x86.avx512.gather3siv4.di(<4 x i64> %x0, i8* %x1, <4 x i32> %x2, i8 %x3, i32 8)
+  %res2 = add <8 x i32> %res, %res1
+  ret <8 x i32> %res2
+}
+
+declare <4 x float> @llvm.x86.avx512.gather3siv4.sf(<4 x float>, i8*, <4 x i32>, i8, i32)
+
+define <4 x float>@test_int_x86_avx512_gather3siv4_sf(<4 x float> %x0, i8* %x1, <4 x i32> %x2, i8 %x3) {
+; CHECK-LABEL: test_int_x86_avx512_gather3siv4_sf:
+; CHECK:       ## BB#0:
+; CHECK-NEXT:    kmovb %esi, %k1
+; CHECK-NEXT:    vmovaps %zmm0, %zmm2
+; CHECK-NEXT:    vgatherdps (%rdi,%xmm1,4), %xmm2 {%k1}
+; CHECK-NEXT:    kxnorw %k1, %k1, %k1
+; CHECK-NEXT:    vgatherdps (%rdi,%xmm1,0), %xmm0 {%k1}
+; CHECK-NEXT:    vaddps %xmm0, %xmm2, %xmm0
+; CHECK-NEXT:    retq
+  %res = call <4 x float> @llvm.x86.avx512.gather3siv4.sf(<4 x float> %x0, i8* %x1, <4 x i32> %x2, i8 %x3, i32 4)
+  %res1 = call <4 x float> @llvm.x86.avx512.gather3siv4.sf(<4 x float> %x0, i8* %x1, <4 x i32> %x2, i8 -1, i32 0)
+  %res2 = fadd <4 x float> %res, %res1
+  ret <4 x float> %res2
+}
+
+declare <4 x i32> @llvm.x86.avx512.gather3siv4.si(<4 x i32>, i8*, <4 x i32>, i8, i32)
+
+define <4 x i32>@test_int_x86_avx512_gather3siv4_si(<4 x i32> %x0, i8* %x1, <4 x i32> %x2, i8 %x3) {
+; CHECK-LABEL: test_int_x86_avx512_gather3siv4_si:
+; CHECK:       ## BB#0:
+; CHECK-NEXT:    kmovb %esi, %k1
+; CHECK-NEXT:    kxnorw %k2, %k2, %k2
+; CHECK-NEXT:    vmovaps %zmm0, %zmm2
+; CHECK-NEXT:    vpgatherdd (%rdi,%xmm1,4), %xmm2 {%k2}
+; CHECK-NEXT:    vpgatherdd (%rdi,%xmm1,0), %xmm0 {%k1}
+; CHECK-NEXT:    vpaddd %xmm0, %xmm2, %xmm0
+; CHECK-NEXT:    retq
+  %res = call <4 x i32> @llvm.x86.avx512.gather3siv4.si(<4 x i32> %x0, i8* %x1, <4 x i32> %x2, i8 -1, i32 4)
+  %res1 = call <4 x i32> @llvm.x86.avx512.gather3siv4.si(<4 x i32> %x0, i8* %x1, <4 x i32> %x2, i8 %x3, i32 0)
+  %res2 = add <4 x i32> %res, %res1
+  ret <4 x i32> %res2
+}
+
+declare <8 x float> @llvm.x86.avx512.gather3siv8.sf(<8 x float>, i8*, <8 x i32>, i8, i32)
+
+define <8 x float>@test_int_x86_avx512_gather3siv8_sf(<8 x float> %x0, i8* %x1, <8 x i32> %x2, i8 %x3) {
+; CHECK-LABEL: test_int_x86_avx512_gather3siv8_sf:
+; CHECK:       ## BB#0:
+; CHECK-NEXT:    kmovb %esi, %k1
+; CHECK-NEXT:    vmovaps %zmm0, %zmm2
+; CHECK-NEXT:    vgatherdps (%rdi,%ymm1,4), %ymm2 {%k1}
+; CHECK-NEXT:    kxnorw %k1, %k1, %k1
+; CHECK-NEXT:    vgatherdps (%rdi,%ymm1,0), %ymm0 {%k1}
+; CHECK-NEXT:    vaddps %ymm0, %ymm2, %ymm0
+; CHECK-NEXT:    retq
+  %res = call <8 x float> @llvm.x86.avx512.gather3siv8.sf(<8 x float> %x0, i8* %x1, <8 x i32> %x2, i8 %x3, i32 4)
+  %res1 = call <8 x float> @llvm.x86.avx512.gather3siv8.sf(<8 x float> %x0, i8* %x1, <8 x i32> %x2, i8 -1, i32 0)
+  %res2 = fadd <8 x float> %res, %res1
+  ret <8 x float> %res2
+}
+
+declare <8 x i32> @llvm.x86.avx512.gather3siv8.si(<8 x i32>, i8*, <8 x i32>, i8, i32)
+
+define <8 x i32>@test_int_x86_avx512_gather3siv8_si(<8 x i32> %x0, i8* %x1, <8 x i32> %x2, i8 %x3) {
+; CHECK-LABEL: test_int_x86_avx512_gather3siv8_si:
+; CHECK:       ## BB#0:
+; CHECK-NEXT:    kmovb %esi, %k1
+; CHECK-NEXT:    vmovaps %zmm0, %zmm2
+; CHECK-NEXT:    kmovw %k1, %k2
+; CHECK-NEXT:    vpgatherdd (%rdi,%ymm1,4), %ymm2 {%k2}
+; CHECK-NEXT:    vpgatherdd (%rdi,%ymm1,0), %ymm0 {%k1}
+; CHECK-NEXT:    vpaddd %ymm0, %ymm2, %ymm0
+; CHECK-NEXT:    retq
+  %res = call <8 x i32> @llvm.x86.avx512.gather3siv8.si(<8 x i32> %x0, i8* %x1, <8 x i32> %x2, i8 %x3, i32 4)
+  %res1 = call <8 x i32> @llvm.x86.avx512.gather3siv8.si(<8 x i32> %x0, i8* %x1, <8 x i32> %x2, i8 %x3, i32 0)
+  %res2 = add <8 x i32> %res, %res1
+  ret <8 x i32> %res2
+}
+
+declare void @llvm.x86.avx512.scatterdiv2.df(i8*, i8, <2 x i64>, <2 x double>, i32)
+
+define void@test_int_x86_avx512_scatterdiv2_df(i8* %x0, i8 %x1, <2 x i64> %x2, <2 x double> %x3) {
+; CHECK-LABEL: test_int_x86_avx512_scatterdiv2_df:
+; CHECK:       ## BB#0:
+; CHECK-NEXT:    kmovb %esi, %k1
+; CHECK-NEXT:    kxnorw %k2, %k2, %k2
+; CHECK-NEXT:    vscatterqpd %xmm1, (%rdi,%xmm0,0) {%k2}
+; CHECK-NEXT:    vscatterqpd %xmm1, (%rdi,%xmm0,4) {%k1}
+; CHECK-NEXT:    retq
+  call void @llvm.x86.avx512.scatterdiv2.df(i8* %x0, i8 -1, <2 x i64> %x2, <2 x double> %x3, i32 0)
+  call void @llvm.x86.avx512.scatterdiv2.df(i8* %x0, i8 %x1, <2 x i64> %x2, <2 x double> %x3, i32 4)
+  ret void
+}
+
+declare void @llvm.x86.avx512.scatterdiv2.di(i8*, i8, <2 x i64>, <2 x i64>, i32)
+
+define void@test_int_x86_avx512_scatterdiv2_di(i8* %x0, i8 %x1, <2 x i64> %x2, <2 x i64> %x3) {
+; CHECK-LABEL: test_int_x86_avx512_scatterdiv2_di:
+; CHECK:       ## BB#0:
+; CHECK-NEXT:    kmovb %esi, %k1
+; CHECK-NEXT:    vpscatterqq %xmm1, (%rdi,%xmm0,0) {%k1}
+; CHECK-NEXT:    kxnorw %k1, %k1, %k1
+; CHECK-NEXT:    vpscatterqq %xmm1, (%rdi,%xmm0,4) {%k1}
+; CHECK-NEXT:    retq
+  call void @llvm.x86.avx512.scatterdiv2.di(i8* %x0, i8 %x1, <2 x i64> %x2, <2 x i64> %x3, i32 0)
+  call void @llvm.x86.avx512.scatterdiv2.di(i8* %x0, i8 -1, <2 x i64> %x2, <2 x i64> %x3, i32 4)
+  ret void
+}
+
+declare void @llvm.x86.avx512.scatterdiv4.df(i8*, i8, <4 x i64>, <4 x double>, i32)
+
+define void@test_int_x86_avx512_scatterdiv4_df(i8* %x0, i8 %x1, <4 x i64> %x2, <4 x double> %x3) {
+; CHECK-LABEL: test_int_x86_avx512_scatterdiv4_df:
+; CHECK:       ## BB#0:
+; CHECK-NEXT:    kmovb %esi, %k1
+; CHECK-NEXT:    vscatterqpd %ymm1, (%rdi,%ymm0,0) {%k1}
+; CHECK-NEXT:    kxnorw %k1, %k1, %k1
+; CHECK-NEXT:    vscatterqpd %ymm1, (%rdi,%ymm0,4) {%k1}
+; CHECK-NEXT:    retq
+  call void @llvm.x86.avx512.scatterdiv4.df(i8* %x0, i8 %x1, <4 x i64> %x2, <4 x double> %x3, i32 0)
+  call void @llvm.x86.avx512.scatterdiv4.df(i8* %x0, i8 -1, <4 x i64> %x2, <4 x double> %x3, i32 4)
+  ret void
+}
+
+declare void @llvm.x86.avx512.scatterdiv4.di(i8*, i8, <4 x i64>, <4 x i64>, i32)
+
+define void@test_int_x86_avx512_scatterdiv4_di(i8* %x0, i8 %x1, <4 x i64> %x2, <4 x i64> %x3) {
+; CHECK-LABEL: test_int_x86_avx512_scatterdiv4_di:
+; CHECK:       ## BB#0:
+; CHECK-NEXT:    kmovb %esi, %k1
+; CHECK-NEXT:    vpscatterqq %ymm1, (%rdi,%ymm0,0) {%k1}
+; CHECK-NEXT:    kxnorw %k1, %k1, %k1
+; CHECK-NEXT:    vpscatterqq %ymm1, (%rdi,%ymm0,4) {%k1}
+; CHECK-NEXT:    retq
+  call void @llvm.x86.avx512.scatterdiv4.di(i8* %x0, i8 %x1, <4 x i64> %x2, <4 x i64> %x3, i32 0)
+  call void @llvm.x86.avx512.scatterdiv4.di(i8* %x0, i8 -1, <4 x i64> %x2, <4 x i64> %x3, i32 4)
+  ret void
+}
+
+declare void @llvm.x86.avx512.scatterdiv4.sf(i8*, i8, <2 x i64>, <4 x float>, i32)
+
+define void@test_int_x86_avx512_scatterdiv4_sf(i8* %x0, i8 %x1, <2 x i64> %x2, <4 x float> %x3) {
+; CHECK-LABEL: test_int_x86_avx512_scatterdiv4_sf:
+; CHECK:       ## BB#0:
+; CHECK-NEXT:    kmovb %esi, %k1
+; CHECK-NEXT:    vscatterqps %xmm1, (%rdi,%xmm0,0) {%k1}
+; CHECK-NEXT:    kxnorw %k1, %k1, %k1
+; CHECK-NEXT:    vscatterqps %xmm1, (%rdi,%xmm0,4) {%k1}
+; CHECK-NEXT:    retq
+  call void @llvm.x86.avx512.scatterdiv4.sf(i8* %x0, i8 %x1, <2 x i64> %x2, <4 x float> %x3, i32 0)
+  call void @llvm.x86.avx512.scatterdiv4.sf(i8* %x0, i8 -1, <2 x i64> %x2, <4 x float> %x3, i32 4)
+  ret void
+}
+
+declare void @llvm.x86.avx512.scatterdiv4.si(i8*, i8, <2 x i64>, <4 x i32>, i32)
+
+define void@test_int_x86_avx512_scatterdiv4_si(i8* %x0, i8 %x1, <2 x i64> %x2, <4 x i32> %x3) {
+; CHECK-LABEL: test_int_x86_avx512_scatterdiv4_si:
+; CHECK:       ## BB#0:
+; CHECK-NEXT:    kmovb %esi, %k1
+; CHECK-NEXT:    kxnorw %k2, %k2, %k2
+; CHECK-NEXT:    vpscatterqd %xmm1, (%rdi,%xmm0,0) {%k2}
+; CHECK-NEXT:    vpscatterqd %xmm1, (%rdi,%xmm0,4) {%k1}
+; CHECK-NEXT:    retq
+  call void @llvm.x86.avx512.scatterdiv4.si(i8* %x0, i8 -1, <2 x i64> %x2, <4 x i32> %x3, i32 0)
+  call void @llvm.x86.avx512.scatterdiv4.si(i8* %x0, i8 %x1, <2 x i64> %x2, <4 x i32> %x3, i32 4)
+  ret void
+}
+
+declare void @llvm.x86.avx512.scatterdiv8.sf(i8*, i8, <4 x i64>, <4 x float>, i32)
+
+define void@test_int_x86_avx512_scatterdiv8_sf(i8* %x0, i8 %x1, <4 x i64> %x2, <4 x float> %x3) {
+; CHECK-LABEL: test_int_x86_avx512_scatterdiv8_sf:
+; CHECK:       ## BB#0:
+; CHECK-NEXT:    kmovb %esi, %k1
+; CHECK-NEXT:    vscatterqps %xmm1, (%rdi,%ymm0,0) {%k1}
+; CHECK-NEXT:    kxnorw %k1, %k1, %k1
+; CHECK-NEXT:    vscatterqps %xmm1, (%rdi,%ymm0,4) {%k1}
+; CHECK-NEXT:    retq
+  call void @llvm.x86.avx512.scatterdiv8.sf(i8* %x0, i8 %x1, <4 x i64> %x2, <4 x float> %x3, i32 0)
+  call void @llvm.x86.avx512.scatterdiv8.sf(i8* %x0, i8 -1, <4 x i64> %x2, <4 x float> %x3, i32 4)
+  ret void
+}
+
+declare void @llvm.x86.avx512.scatterdiv8.si(i8*, i8, <4 x i64>, <4 x i32>, i32)
+
+define void@test_int_x86_avx512_scatterdiv8_si(i8* %x0, i8 %x1, <4 x i64> %x2, <4 x i32> %x3) {
+; CHECK-LABEL: test_int_x86_avx512_scatterdiv8_si:
+; CHECK:       ## BB#0:
+; CHECK-NEXT:    kmovb %esi, %k1
+; CHECK-NEXT:    vpscatterqd %xmm1, (%rdi,%ymm0,0) {%k1}
+; CHECK-NEXT:    kxnorw %k1, %k1, %k1
+; CHECK-NEXT:    vpscatterqd %xmm1, (%rdi,%ymm0,4) {%k1}
+; CHECK-NEXT:    retq
+  call void @llvm.x86.avx512.scatterdiv8.si(i8* %x0, i8 %x1, <4 x i64> %x2, <4 x i32> %x3, i32 0)
+  call void @llvm.x86.avx512.scatterdiv8.si(i8* %x0, i8 -1, <4 x i64> %x2, <4 x i32> %x3, i32 4)
+  ret void
+}
+
+declare void @llvm.x86.avx512.scattersiv2.df(i8*, i8, <4 x i32>, <2 x double>, i32)
+
+define void@test_int_x86_avx512_scattersiv2_df(i8* %x0, i8 %x1, <4 x i32> %x2, <2 x double> %x3) {
+; CHECK-LABEL: test_int_x86_avx512_scattersiv2_df:
+; CHECK:       ## BB#0:
+; CHECK-NEXT:    kmovb %esi, %k1
+; CHECK-NEXT:    kxnorw %k2, %k2, %k2
+; CHECK-NEXT:    vscatterdpd %xmm1, (%rdi,%xmm0,0) {%k2}
+; CHECK-NEXT:    vscatterdpd %xmm1, (%rdi,%xmm0,4) {%k1}
+; CHECK-NEXT:    retq
+  call void @llvm.x86.avx512.scattersiv2.df(i8* %x0, i8 -1, <4 x i32> %x2, <2 x double> %x3, i32 0)
+  call void @llvm.x86.avx512.scattersiv2.df(i8* %x0, i8 %x1, <4 x i32> %x2, <2 x double> %x3, i32 4)
+  ret void
+}
+
+declare void @llvm.x86.avx512.scattersiv2.di(i8*, i8, <4 x i32>, <2 x i64>, i32)
+
+define void@test_int_x86_avx512_scattersiv2_di(i8* %x0, i8 %x1, <4 x i32> %x2, <2 x i64> %x3) {
+; CHECK-LABEL: test_int_x86_avx512_scattersiv2_di:
+; CHECK:       ## BB#0:
+; CHECK-NEXT:    kmovb %esi, %k1
+; CHECK-NEXT:    kxnorw %k2, %k2, %k2
+; CHECK-NEXT:    vpscatterdq %xmm1, (%rdi,%xmm0,0) {%k2}
+; CHECK-NEXT:    vpscatterdq %xmm1, (%rdi,%xmm0,4) {%k1}
+; CHECK-NEXT:    retq
+  call void @llvm.x86.avx512.scattersiv2.di(i8* %x0, i8 -1, <4 x i32> %x2, <2 x i64> %x3, i32 0)
+  call void @llvm.x86.avx512.scattersiv2.di(i8* %x0, i8 %x1, <4 x i32> %x2, <2 x i64> %x3, i32 4)
+  ret void
+}
+
+declare void @llvm.x86.avx512.scattersiv4.df(i8*, i8, <4 x i32>, <4 x double>, i32)
+
+define void@test_int_x86_avx512_scattersiv4_df(i8* %x0, i8 %x1, <4 x i32> %x2, <4 x double> %x3) {
+; CHECK-LABEL: test_int_x86_avx512_scattersiv4_df:
+; CHECK:       ## BB#0:
+; CHECK-NEXT:    kmovb %esi, %k1
+; CHECK-NEXT:    vscatterdpd %ymm1, (%rdi,%xmm0,0) {%k1}
+; CHECK-NEXT:    kxnorw %k1, %k1, %k1
+; CHECK-NEXT:    vscatterdpd %ymm1, (%rdi,%xmm0,4) {%k1}
+; CHECK-NEXT:    retq
+  call void @llvm.x86.avx512.scattersiv4.df(i8* %x0, i8 %x1, <4 x i32> %x2, <4 x double> %x3, i32 0)
+  call void @llvm.x86.avx512.scattersiv4.df(i8* %x0, i8 -1, <4 x i32> %x2, <4 x double> %x3, i32 4)
+  ret void
+}
+
+declare void @llvm.x86.avx512.scattersiv4.di(i8*, i8, <4 x i32>, <4 x i64>, i32)
+
+define void@test_int_x86_avx512_scattersiv4_di(i8* %x0, i8 %x1, <4 x i32> %x2, <4 x i64> %x3) {
+; CHECK-LABEL: test_int_x86_avx512_scattersiv4_di:
+; CHECK:       ## BB#0:
+; CHECK-NEXT:    kmovb %esi, %k1
+; CHECK-NEXT:    kxnorw %k2, %k2, %k2
+; CHECK-NEXT:    vpscatterdq %ymm1, (%rdi,%xmm0,0) {%k2}
+; CHECK-NEXT:    vpscatterdq %ymm1, (%rdi,%xmm0,4) {%k1}
+; CHECK-NEXT:    retq
+  call void @llvm.x86.avx512.scattersiv4.di(i8* %x0, i8 -1, <4 x i32> %x2, <4 x i64> %x3, i32 0)
+  call void @llvm.x86.avx512.scattersiv4.di(i8* %x0, i8 %x1, <4 x i32> %x2, <4 x i64> %x3, i32 4)
+  ret void
+}
+
+declare void @llvm.x86.avx512.scattersiv4.sf(i8*, i8, <4 x i32>, <4 x float>, i32)
+
+define void@test_int_x86_avx512_scattersiv4_sf(i8* %x0, i8 %x1, <4 x i32> %x2, <4 x float> %x3) {
+; CHECK-LABEL: test_int_x86_avx512_scattersiv4_sf:
+; CHECK:       ## BB#0:
+; CHECK-NEXT:    kmovb %esi, %k1
+; CHECK-NEXT:    vscatterdps %xmm1, (%rdi,%xmm0,0) {%k1}
+; CHECK-NEXT:    kxnorw %k1, %k1, %k1
+; CHECK-NEXT:    vscatterdps %xmm1, (%rdi,%xmm0,4) {%k1}
+; CHECK-NEXT:    retq
+  call void @llvm.x86.avx512.scattersiv4.sf(i8* %x0, i8 %x1, <4 x i32> %x2, <4 x float> %x3, i32 0)
+  call void @llvm.x86.avx512.scattersiv4.sf(i8* %x0, i8 -1, <4 x i32> %x2, <4 x float> %x3, i32 4)
+  ret void
+}
+
+declare void @llvm.x86.avx512.scattersiv4.si(i8*, i8, <4 x i32>, <4 x i32>, i32)
+
+define void@test_int_x86_avx512_scattersiv4_si(i8* %x0, i8 %x1, <4 x i32> %x2, <4 x i32> %x3) {
+; CHECK-LABEL: test_int_x86_avx512_scattersiv4_si:
+; CHECK:       ## BB#0:
+; CHECK-NEXT:    kmovb %esi, %k1
+; CHECK-NEXT:    vpscatterdd %xmm1, (%rdi,%xmm0,0) {%k1}
+; CHECK-NEXT:    kxnorw %k1, %k1, %k1
+; CHECK-NEXT:    vpscatterdd %xmm1, (%rdi,%xmm0,4) {%k1}
+; CHECK-NEXT:    retq
+  call void @llvm.x86.avx512.scattersiv4.si(i8* %x0, i8 %x1, <4 x i32> %x2, <4 x i32> %x3, i32 0)
+  call void @llvm.x86.avx512.scattersiv4.si(i8* %x0, i8 -1, <4 x i32> %x2, <4 x i32> %x3, i32 4)
+  ret void
+}
+
+declare void @llvm.x86.avx512.scattersiv8.sf(i8*, i8, <8 x i32>, <8 x float>, i32)
+
+define void@test_int_x86_avx512_scattersiv8_sf(i8* %x0, i8 %x1, <8 x i32> %x2, <8 x float> %x3) {
+; CHECK-LABEL: test_int_x86_avx512_scattersiv8_sf:
+; CHECK:       ## BB#0:
+; CHECK-NEXT:    kmovb %esi, %k1
+; CHECK-NEXT:    vscatterdps %ymm1, (%rdi,%ymm0,0) {%k1}
+; CHECK-NEXT:    kxnorw %k1, %k1, %k1
+; CHECK-NEXT:    vscatterdps %ymm1, (%rdi,%ymm0,4) {%k1}
+; CHECK-NEXT:    retq
+  call void @llvm.x86.avx512.scattersiv8.sf(i8* %x0, i8 %x1, <8 x i32> %x2, <8 x float> %x3, i32 0)
+  call void @llvm.x86.avx512.scattersiv8.sf(i8* %x0, i8 -1, <8 x i32> %x2, <8 x float> %x3, i32 4)
+  ret void
+}
+
+declare void @llvm.x86.avx512.scattersiv8.si(i8*, i8, <8 x i32>, <8 x i32>, i32)
+
+define void@test_int_x86_avx512_scattersiv8_si(i8* %x0, i8 %x1, <8 x i32> %x2, <8 x i32> %x3) {
+; CHECK-LABEL: test_int_x86_avx512_scattersiv8_si:
+; CHECK:       ## BB#0:
+; CHECK-NEXT:    kmovb %esi, %k1
+; CHECK-NEXT:    vpscatterdd %ymm1, (%rdi,%ymm0,0) {%k1}
+; CHECK-NEXT:    kxnorw %k1, %k1, %k1
+; CHECK-NEXT:    vpscatterdd %ymm1, (%rdi,%ymm0,4) {%k1}
+; CHECK-NEXT:    retq
+  call void @llvm.x86.avx512.scattersiv8.si(i8* %x0, i8 %x1, <8 x i32> %x2, <8 x i32> %x3, i32 0)
+  call void @llvm.x86.avx512.scattersiv8.si(i8* %x0, i8 -1, <8 x i32> %x2, <8 x i32> %x3, i32 4)
+  ret void
+}
+
diff --git a/test/CodeGen/X86/avx512-insert-extract.ll b/test/CodeGen/X86/avx512-insert-extract.ll
index 6498b20d803..6f985f0bf3a 100644
--- a/test/CodeGen/X86/avx512-insert-extract.ll
+++ b/test/CodeGen/X86/avx512-insert-extract.ll
@@ -137,10 +137,12 @@ define i64 @test12(<16 x i64>%a, <16 x i64>%b, i64 %a1, i64 %b1) {
 }
 
 ;CHECK-LABEL: test13
-;CHECK: cmpl
-;CHECK: sbbl
-;CHECK: orl $65532
-;CHECK: ret
+;CHECK: cmpl    %esi, %edi
+;CHECK: setb    %al
+;CHECK: andl    $1, %eax
+;CHECK: kmovw   %eax, %k0
+;CHECK: movw    $-4
+;CHECK: korw    
 define i16 @test13(i32 %a, i32 %b) {
   %cmp_res = icmp ult i32 %a, %b
   %maskv = insertelement <16 x i1> <i1 true, i1 false, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>, i1 %cmp_res, i32 0
@@ -167,19 +169,22 @@ define i64 @test14(<8 x i64>%a, <8 x i64>%b, i64 %a1, i64 %b1) {
 }
 
 ;CHECK-LABEL: test15
-;CHECK: kshiftlw
-;CHECK: kmovw
-;CHECK: ret
+;CHECK: movb (%rdi), %al
+;CHECK: andb $1, %al
+;CHECK: movw    $-1, %ax
+;CHECK: cmovew
 define i16 @test15(i1 *%addr) {
-  %x = load i1 , i1 * %addr, align 128
+  %x = load i1 , i1 * %addr, align 1
   %x1 = insertelement <16 x i1> undef, i1 %x, i32 10
   %x2 = bitcast <16 x i1>%x1 to i16
   ret i16 %x2
 }
 
 ;CHECK-LABEL: test16
-;CHECK: kshiftlw
-;CHECK: kshiftrw
+;CHECK: movb (%rdi), %al
+;CHECK: andw $1, %ax
+;CHECK: kmovw
+;CHECK: kshiftlw        $10
 ;CHECK: korw
 ;CHECK: ret
 define i16 @test16(i1 *%addr, i16 %a) {
@@ -191,11 +196,11 @@ define i16 @test16(i1 *%addr, i16 %a) {
 }
 
 ;CHECK-LABEL: test17
-;KNL: kshiftlw
-;KNL: kshiftrw
+;KNL: movb (%rdi), %al
+;KNL: andw $1, %ax
+;KNL: kshiftlw $4
 ;KNL: korw
-;SKX: kshiftlb
-;SKX: kshiftrb
+;SKX: kshiftlb $4
 ;SKX: korb
 ;CHECK: ret
 define i8 @test17(i1 *%addr, i8 %a) {
diff --git a/test/CodeGen/X86/avx512-intrinsics.ll b/test/CodeGen/X86/avx512-intrinsics.ll
index 07d984a0ee9..9a494ea1c3b 100644
--- a/test/CodeGen/X86/avx512-intrinsics.ll
+++ b/test/CodeGen/X86/avx512-intrinsics.ll
@@ -98,18 +98,55 @@ define <4 x float> @test_rcp14_ss(<4 x float> %a0) {
 declare <4 x float> @llvm.x86.avx512.rcp14.ss(<4 x float>, <4 x float>, <4 x float>, i8) nounwind readnone
 
 define <8 x double> @test_sqrt_pd_512(<8 x double> %a0) {
+  ; CHECK-LABEL: test_sqrt_pd_512
   ; CHECK: vsqrtpd
-  %res = call <8 x double> @llvm.x86.avx512.sqrt.pd.512(<8 x double> %a0,  <8 x double> zeroinitializer, i8 -1, i32 4) ; <<8 x double>> [#uses=1]
+  %res = call <8 x double> @llvm.x86.avx512.mask.sqrt.pd.512(<8 x double> %a0,  <8 x double> zeroinitializer, i8 -1, i32 4) 
   ret <8 x double> %res
 }
-declare <8 x double> @llvm.x86.avx512.sqrt.pd.512(<8 x double>, <8 x double>, i8, i32) nounwind readnone
+declare <8 x double> @llvm.x86.avx512.mask.sqrt.pd.512(<8 x double>, <8 x double>, i8, i32) nounwind readnone
 
 define <16 x float> @test_sqrt_ps_512(<16 x float> %a0) {
+  ; CHECK-LABEL: test_sqrt_ps_512
   ; CHECK: vsqrtps
-  %res = call <16 x float> @llvm.x86.avx512.sqrt.ps.512(<16 x float> %a0, <16 x float> zeroinitializer, i16 -1, i32 4) ; <<16 x float>> [#uses=1]
+  %res = call <16 x float> @llvm.x86.avx512.mask.sqrt.ps.512(<16 x float> %a0, <16 x float> zeroinitializer, i16 -1, i32 4) 
   ret <16 x float> %res
 }
-declare <16 x float> @llvm.x86.avx512.sqrt.ps.512(<16 x float>, <16 x float>, i16, i32) nounwind readnone
+define <16 x float> @test_sqrt_round_ps_512(<16 x float> %a0) {
+  ; CHECK-LABEL: test_sqrt_round_ps_512
+  ; CHECK: vsqrtps {rz-sae}
+  %res = call <16 x float> @llvm.x86.avx512.mask.sqrt.ps.512(<16 x float> %a0, <16 x float> zeroinitializer, i16 -1, i32 3) 
+  ret <16 x float> %res
+}
+declare <16 x float> @llvm.x86.avx512.mask.sqrt.ps.512(<16 x float>, <16 x float>, i16, i32) nounwind readnone
+
+define <8 x double> @test_getexp_pd_512(<8 x double> %a0) {
+  ; CHECK-LABEL: test_getexp_pd_512
+  ; CHECK: vgetexppd
+  %res = call <8 x double> @llvm.x86.avx512.mask.getexp.pd.512(<8 x double> %a0,  <8 x double> zeroinitializer, i8 -1, i32 4) 
+  ret <8 x double> %res
+}
+define <8 x double> @test_getexp_round_pd_512(<8 x double> %a0) {
+  ; CHECK-LABEL: test_getexp_round_pd_512
+  ; CHECK: vgetexppd {sae}
+  %res = call <8 x double> @llvm.x86.avx512.mask.getexp.pd.512(<8 x double> %a0,  <8 x double> zeroinitializer, i8 -1, i32 8) 
+  ret <8 x double> %res
+}
+declare <8 x double> @llvm.x86.avx512.mask.getexp.pd.512(<8 x double>, <8 x double>, i8, i32) nounwind readnone
+
+define <16 x float> @test_getexp_ps_512(<16 x float> %a0) {
+  ; CHECK-LABEL: test_getexp_ps_512
+  ; CHECK: vgetexpps
+  %res = call <16 x float> @llvm.x86.avx512.mask.getexp.ps.512(<16 x float> %a0, <16 x float> zeroinitializer, i16 -1, i32 4) 
+  ret <16 x float> %res
+}
+
+define <16 x float> @test_getexp_round_ps_512(<16 x float> %a0) {
+  ; CHECK-LABEL: test_getexp_round_ps_512
+  ; CHECK: vgetexpps {sae}
+  %res = call <16 x float> @llvm.x86.avx512.mask.getexp.ps.512(<16 x float> %a0, <16 x float> zeroinitializer, i16 -1, i32 8) 
+  ret <16 x float> %res
+}
+declare <16 x float> @llvm.x86.avx512.mask.getexp.ps.512(<16 x float>, <16 x float>, i16, i32) nounwind readnone
 
 define <4 x float> @test_sqrt_ss(<4 x float> %a0, <4 x float> %a1) {
   ; CHECK: vsqrtss {{.*}}encoding: [0x62
@@ -139,13 +176,6 @@ define <2 x double> @test_x86_sse2_cvtsi642sd(<2 x double> %a0, i64 %a1) {
 }
 declare <2 x double> @llvm.x86.sse2.cvtsi642sd(<2 x double>, i64) nounwind readnone
 
-define <2 x double> @test_x86_avx512_cvtusi642sd(<2 x double> %a0, i64 %a1) {
-  ; CHECK: vcvtusi2sdq {{.*}}encoding: [0x62
-  %res = call <2 x double> @llvm.x86.avx512.cvtusi642sd(<2 x double> %a0, i64 %a1) ; <<2 x double>> [#uses=1]
-  ret <2 x double> %res
-}
-declare <2 x double> @llvm.x86.avx512.cvtusi642sd(<2 x double>, i64) nounwind readnone
-
 define i64 @test_x86_sse2_cvttsd2si64(<2 x double> %a0) {
   ; CHECK: vcvttsd2si {{.*}}encoding: [0x62
   %res = call i64 @llvm.x86.sse2.cvttsd2si64(<2 x double> %a0) ; <i64> [#uses=1]
@@ -376,73 +406,21 @@ define <8 x i64> @test_x86_mask_blend_q_512(i8 %a0, <8 x i64> %a1, <8 x i64> %a2
 }
 declare <8 x i64> @llvm.x86.avx512.mask.blend.q.512(<8 x i64>, <8 x i64>, i8) nounwind readonly
 
- define <8 x i32> @test_cvtpd2udq(<8 x double> %a) {
- ;CHECK: vcvtpd2udq {ru-sae}{{.*}}encoding: [0x62,0xf1,0xfc,0x58,0x79,0xc0]
-  %res = call <8 x i32> @llvm.x86.avx512.mask.cvtpd2udq.512(<8 x double> %a, <8 x i32>zeroinitializer, i8 -1, i32 2)
-  ret <8 x i32>%res
- }
- declare <8 x i32> @llvm.x86.avx512.mask.cvtpd2udq.512(<8 x double>, <8 x i32>, i8, i32)
-
- define <16 x i32> @test_cvtps2udq(<16 x float> %a) {
- ;CHECK: vcvtps2udq {rd-sae}{{.*}}encoding: [0x62,0xf1,0x7c,0x38,0x79,0xc0]
-  %res = call <16 x i32> @llvm.x86.avx512.mask.cvtps2udq.512(<16 x float> %a, <16 x i32>zeroinitializer, i16 -1, i32 1)
-  ret <16 x i32>%res
- }
- declare <16 x i32> @llvm.x86.avx512.mask.cvtps2udq.512(<16 x float>, <16 x i32>, i16, i32)
-
  define i16 @test_cmpps(<16 x float> %a, <16 x float> %b) {
  ;CHECK: vcmpleps {sae}{{.*}}encoding: [0x62,0xf1,0x7c,0x18,0xc2,0xc1,0x02]
-   %res = call i16 @llvm.x86.avx512.mask.cmp.ps.512(<16 x float> %a, <16 x float> %b, i8 2, i16 -1, i32 8)
+   %res = call i16 @llvm.x86.avx512.mask.cmp.ps.512(<16 x float> %a, <16 x float> %b, i32 2, i16 -1, i32 8)
    ret i16 %res
  }
- declare i16 @llvm.x86.avx512.mask.cmp.ps.512(<16 x float> , <16 x float> , i8, i16, i32)
+ declare i16 @llvm.x86.avx512.mask.cmp.ps.512(<16 x float> , <16 x float> , i32, i16, i32)
 
  define i8 @test_cmppd(<8 x double> %a, <8 x double> %b) {
  ;CHECK: vcmpneqpd %zmm{{.*}}encoding: [0x62,0xf1,0xfd,0x48,0xc2,0xc1,0x04]
-   %res = call i8 @llvm.x86.avx512.mask.cmp.pd.512(<8 x double> %a, <8 x double> %b, i8 4, i8 -1, i32 4)
+   %res = call i8 @llvm.x86.avx512.mask.cmp.pd.512(<8 x double> %a, <8 x double> %b, i32 4, i8 -1, i32 4)
    ret i8 %res
  }
- declare i8 @llvm.x86.avx512.mask.cmp.pd.512(<8 x double> , <8 x double> , i8, i8, i32)
-
- ; cvt intrinsics
- define <16 x float> @test_cvtdq2ps(<16 x i32> %a) {
- ;CHECK: vcvtdq2ps {rd-sae}{{.*}}encoding: [0x62,0xf1,0x7c,0x38,0x5b,0xc0]
-  %res = call <16 x float> @llvm.x86.avx512.mask.cvtdq2ps.512(<16 x i32> %a, <16 x float>zeroinitializer, i16 -1, i32 1)
-  ret <16 x float>%res
- }
- declare <16 x float> @llvm.x86.avx512.mask.cvtdq2ps.512(<16 x i32>, <16 x float>, i16, i32)
-
- define <16 x float> @test_cvtudq2ps(<16 x i32> %a) {
- ;CHECK: vcvtudq2ps {rd-sae}{{.*}}encoding: [0x62,0xf1,0x7f,0x38,0x7a,0xc0]
-  %res = call <16 x float> @llvm.x86.avx512.mask.cvtudq2ps.512(<16 x i32> %a, <16 x float>zeroinitializer, i16 -1, i32 1)
-  ret <16 x float>%res
- }
- declare <16 x float> @llvm.x86.avx512.mask.cvtudq2ps.512(<16 x i32>, <16 x float>, i16, i32)
-
- define <8 x double> @test_cvtdq2pd(<8 x i32> %a) {
- ;CHECK: vcvtdq2pd {{.*}}encoding: [0x62,0xf1,0x7e,0x48,0xe6,0xc0]
-  %res = call <8 x double> @llvm.x86.avx512.mask.cvtdq2pd.512(<8 x i32> %a, <8 x double>zeroinitializer, i8 -1)
-  ret <8 x double>%res
- }
- declare <8 x double> @llvm.x86.avx512.mask.cvtdq2pd.512(<8 x i32>, <8 x double>, i8)
-
- define <8 x double> @test_cvtudq2pd(<8 x i32> %a) {
- ;CHECK: vcvtudq2pd {{.*}}encoding: [0x62,0xf1,0x7e,0x48,0x7a,0xc0]
-  %res = call <8 x double> @llvm.x86.avx512.mask.cvtudq2pd.512(<8 x i32> %a, <8 x double>zeroinitializer, i8 -1)
-  ret <8 x double>%res
- }
- declare <8 x double> @llvm.x86.avx512.mask.cvtudq2pd.512(<8 x i32>, <8 x double>, i8)
+ declare i8 @llvm.x86.avx512.mask.cmp.pd.512(<8 x double> , <8 x double> , i32, i8, i32)
 
  ; fp min - max
-define <16 x float> @test_vmaxps(<16 x float> %a0, <16 x float> %a1) {
-  ; CHECK: vmaxps
-  %res = call <16 x float> @llvm.x86.avx512.mask.max.ps.512(<16 x float> %a0, <16 x float> %a1,
-                    <16 x float>zeroinitializer, i16 -1, i32 4)
-  ret <16 x float> %res
-}
-declare <16 x float> @llvm.x86.avx512.mask.max.ps.512(<16 x float>, <16 x float>,
-                    <16 x float>, i16, i32)
-
 define <8 x double> @test_vmaxpd(<8 x double> %a0, <8 x double> %a1) {
   ; CHECK: vmaxpd
   %res = call <8 x double> @llvm.x86.avx512.mask.max.pd.512(<8 x double> %a0, <8 x double> %a1,
@@ -452,15 +430,6 @@ define <8 x double> @test_vmaxpd(<8 x double> %a0, <8 x double> %a1) {
 declare <8 x double> @llvm.x86.avx512.mask.max.pd.512(<8 x double>, <8 x double>,
                     <8 x double>, i8, i32)
 
-define <16 x float> @test_vminps(<16 x float> %a0, <16 x float> %a1) {
-  ; CHECK: vminps
-  %res = call <16 x float> @llvm.x86.avx512.mask.min.ps.512(<16 x float> %a0, <16 x float> %a1,
-                    <16 x float>zeroinitializer, i16 -1, i32 4)
-  ret <16 x float> %res
-}
-declare <16 x float> @llvm.x86.avx512.mask.min.ps.512(<16 x float>, <16 x float>,
-                    <16 x float>, i16, i32)
-
 define <8 x double> @test_vminpd(<8 x double> %a0, <8 x double> %a1) {
   ; CHECK: vminpd
   %res = call <8 x double> @llvm.x86.avx512.mask.min.pd.512(<8 x double> %a0, <8 x double> %a1,
@@ -470,50 +439,31 @@ define <8 x double> @test_vminpd(<8 x double> %a0, <8 x double> %a1) {
 declare <8 x double> @llvm.x86.avx512.mask.min.pd.512(<8 x double>, <8 x double>,
                     <8 x double>, i8, i32)
 
- define <8 x float> @test_cvtpd2ps(<8 x double> %a) {
- ;CHECK: vcvtpd2ps {rd-sae}{{.*}}encoding: [0x62,0xf1,0xfd,0x38,0x5a,0xc0]
-  %res = call <8 x float> @llvm.x86.avx512.mask.cvtpd2ps.512(<8 x double> %a, <8 x float>zeroinitializer, i8 -1, i32 1)
-  ret <8 x float>%res
- }
- declare <8 x float> @llvm.x86.avx512.mask.cvtpd2ps.512(<8 x double>, <8 x float>, i8, i32)
-
- define <16 x i32> @test_pabsd(<16 x i32> %a) {
- ;CHECK: vpabsd {{.*}}encoding: [0x62,0xf2,0x7d,0x48,0x1e,0xc0]
- %res = call <16 x i32> @llvm.x86.avx512.mask.pabs.d.512(<16 x i32> %a, <16 x i32>zeroinitializer, i16 -1)
- ret < 16 x i32> %res
- }
  declare <16 x i32> @llvm.x86.avx512.mask.pabs.d.512(<16 x i32>, <16 x i32>, i16)
 
- define <8 x i64> @test_pabsq(<8 x i64> %a) {
- ;CHECK: vpabsq {{.*}}encoding: [0x62,0xf2,0xfd,0x48,0x1f,0xc0]
- %res = call <8 x i64> @llvm.x86.avx512.mask.pabs.q.512(<8 x i64> %a, <8 x i64>zeroinitializer, i8 -1)
- ret <8 x i64> %res
- }
- declare <8 x i64> @llvm.x86.avx512.mask.pabs.q.512(<8 x i64>, <8 x i64>, i8)
-
-define <8 x i64> @test_vpmaxq(<8 x i64> %a0, <8 x i64> %a1) {
-  ; CHECK: vpmaxsq {{.*}}encoding: [0x62,0xf2,0xfd,0x48,0x3d,0xc1]
-  %res = call <8 x i64> @llvm.x86.avx512.mask.pmaxs.q.512(<8 x i64> %a0, <8 x i64> %a1,
-                    <8 x i64>zeroinitializer, i8 -1)
-  ret <8 x i64> %res
+; CHECK-LABEL: @test_int_x86_avx512_mask_pabs_d_512
+; CHECK-NOT: call 
+; CHECK: kmov 
+; CHECK: vpabsd{{.*}}{%k1} 
+define <16 x i32>@test_int_x86_avx512_mask_pabs_d_512(<16 x i32> %x0, <16 x i32> %x1, i16 %x2) {
+  %res = call <16 x i32> @llvm.x86.avx512.mask.pabs.d.512(<16 x i32> %x0, <16 x i32> %x1, i16 %x2)
+  %res1 = call <16 x i32> @llvm.x86.avx512.mask.pabs.d.512(<16 x i32> %x0, <16 x i32> %x1, i16 -1)
+  %res2 = add <16 x i32> %res, %res1
+  ret <16 x i32> %res2
 }
-declare <8 x i64> @llvm.x86.avx512.mask.pmaxs.q.512(<8 x i64>, <8 x i64>, <8 x i64>, i8)
 
-define <16 x i32> @test_vpminud(<16 x i32> %a0, <16 x i32> %a1) {
-  ; CHECK: vpminud {{.*}}encoding: [0x62,0xf2,0x7d,0x48,0x3b,0xc1]
-  %res = call <16 x i32> @llvm.x86.avx512.mask.pminu.d.512(<16 x i32> %a0, <16 x i32> %a1,
-                    <16 x i32>zeroinitializer, i16 -1)
-  ret <16 x i32> %res
-}
-declare <16 x i32> @llvm.x86.avx512.mask.pminu.d.512(<16 x i32>, <16 x i32>, <16 x i32>, i16)
+declare <8 x i64> @llvm.x86.avx512.mask.pabs.q.512(<8 x i64>, <8 x i64>, i8)
 
-define <16 x i32> @test_vpmaxsd(<16 x i32> %a0, <16 x i32> %a1) {
-  ; CHECK: vpmaxsd {{.*}}encoding: [0x62,0xf2,0x7d,0x48,0x3d,0xc1]
-  %res = call <16 x i32> @llvm.x86.avx512.mask.pmaxs.d.512(<16 x i32> %a0, <16 x i32> %a1,
-                    <16 x i32>zeroinitializer, i16 -1)
-  ret <16 x i32> %res
+; CHECK-LABEL: @test_int_x86_avx512_mask_pabs_q_512
+; CHECK-NOT: call 
+; CHECK: kmov 
+; CHECK: vpabsq{{.*}}{%k1} 
+define <8 x i64>@test_int_x86_avx512_mask_pabs_q_512(<8 x i64> %x0, <8 x i64> %x1, i8 %x2) {
+  %res = call <8 x i64> @llvm.x86.avx512.mask.pabs.q.512(<8 x i64> %x0, <8 x i64> %x1, i8 %x2)
+  %res1 = call <8 x i64> @llvm.x86.avx512.mask.pabs.q.512(<8 x i64> %x0, <8 x i64> %x1, i8 -1)
+  %res2 = add <8 x i64> %res, %res1
+  ret <8 x i64> %res2
 }
-declare <16 x i32> @llvm.x86.avx512.mask.pmaxs.d.512(<16 x i32>, <16 x i32>, <16 x i32>, i16)
 
 define i8 @test_vptestmq(<8 x i64> %a0, <8 x i64> %a1) {
   ; CHECK: vptestmq {{.*}}encoding: [0x62,0xf2,0xfd,0x48,0x27,0xc1]
@@ -611,28 +561,6 @@ define <8 x double> @test_load_aligned_pd(<8 x double> %data, i8* %ptr, i8 %mask
   ret <8 x double> %res
 }
 
-define <16 x float> @test_vpermt2ps(<16 x float>%x, <16 x float>%y, <16 x i32>%perm) {
-; CHECK: vpermt2ps {{.*}}encoding: [0x62,0xf2,0x6d,0x48,0x7f,0xc1]
-  %res = call <16 x float> @llvm.x86.avx512.mask.vpermt.ps.512(<16 x i32>%perm, <16 x float>%x, <16 x float>%y, i16 -1)
-  ret <16 x float> %res
-}
-
-define <16 x float> @test_vpermt2ps_mask(<16 x float>%x, <16 x float>%y, <16 x i32>%perm, i16 %mask) {
-; CHECK-LABEL: test_vpermt2ps_mask:
-; CHECK: vpermt2ps %zmm1, %zmm2, %zmm0 {%k1} ## encoding: [0x62,0xf2,0x6d,0x49,0x7f,0xc1]
-  %res = call <16 x float> @llvm.x86.avx512.mask.vpermt.ps.512(<16 x i32>%perm, <16 x float>%x, <16 x float>%y, i16 %mask)
-  ret <16 x float> %res
-}
-
-declare <16 x float> @llvm.x86.avx512.mask.vpermt.ps.512(<16 x i32>, <16 x float>, <16 x float>, i16)
-
-define <8 x i64> @test_vmovntdqa(i8 *%x) {
-; CHECK-LABEL: test_vmovntdqa:
-; CHECK: vmovntdqa (%rdi), %zmm0 ## encoding: [0x62,0xf2,0x7d,0x48,0x2a,0x07]
-  %res = call <8 x i64> @llvm.x86.avx512.movntdqa(i8* %x)
-  ret <8 x i64> %res
-}
-
 declare <8 x i64> @llvm.x86.avx512.movntdqa(i8*)
 
 define <8 x i64> @test_valign_q(<8 x i64> %a, <8 x i64> %b) {
@@ -736,28 +664,28 @@ declare i8 @llvm.x86.avx512.mask.pcmpgt.q.512(<8 x i64>, <8 x i64>, i8)
 define <8 x i16> @test_cmp_d_512(<16 x i32> %a0, <16 x i32> %a1) {
 ; CHECK_LABEL: test_cmp_d_512
 ; CHECK: vpcmpeqd %zmm1, %zmm0, %k0 ##
-  %res0 = call i16 @llvm.x86.avx512.mask.cmp.d.512(<16 x i32> %a0, <16 x i32> %a1, i8 0, i16 -1)
+  %res0 = call i16 @llvm.x86.avx512.mask.cmp.d.512(<16 x i32> %a0, <16 x i32> %a1, i32 0, i16 -1)
   %vec0 = insertelement <8 x i16> undef, i16 %res0, i32 0
 ; CHECK: vpcmpltd %zmm1, %zmm0, %k0 ##
-  %res1 = call i16 @llvm.x86.avx512.mask.cmp.d.512(<16 x i32> %a0, <16 x i32> %a1, i8 1, i16 -1)
+  %res1 = call i16 @llvm.x86.avx512.mask.cmp.d.512(<16 x i32> %a0, <16 x i32> %a1, i32 1, i16 -1)
   %vec1 = insertelement <8 x i16> %vec0, i16 %res1, i32 1
 ; CHECK: vpcmpled %zmm1, %zmm0, %k0 ##
-  %res2 = call i16 @llvm.x86.avx512.mask.cmp.d.512(<16 x i32> %a0, <16 x i32> %a1, i8 2, i16 -1)
+  %res2 = call i16 @llvm.x86.avx512.mask.cmp.d.512(<16 x i32> %a0, <16 x i32> %a1, i32 2, i16 -1)
   %vec2 = insertelement <8 x i16> %vec1, i16 %res2, i32 2
 ; CHECK: vpcmpunordd %zmm1, %zmm0, %k0 ##
-  %res3 = call i16 @llvm.x86.avx512.mask.cmp.d.512(<16 x i32> %a0, <16 x i32> %a1, i8 3, i16 -1)
+  %res3 = call i16 @llvm.x86.avx512.mask.cmp.d.512(<16 x i32> %a0, <16 x i32> %a1, i32 3, i16 -1)
   %vec3 = insertelement <8 x i16> %vec2, i16 %res3, i32 3
 ; CHECK: vpcmpneqd %zmm1, %zmm0, %k0 ##
-  %res4 = call i16 @llvm.x86.avx512.mask.cmp.d.512(<16 x i32> %a0, <16 x i32> %a1, i8 4, i16 -1)
+  %res4 = call i16 @llvm.x86.avx512.mask.cmp.d.512(<16 x i32> %a0, <16 x i32> %a1, i32 4, i16 -1)
   %vec4 = insertelement <8 x i16> %vec3, i16 %res4, i32 4
 ; CHECK: vpcmpnltd %zmm1, %zmm0, %k0 ##
-  %res5 = call i16 @llvm.x86.avx512.mask.cmp.d.512(<16 x i32> %a0, <16 x i32> %a1, i8 5, i16 -1)
+  %res5 = call i16 @llvm.x86.avx512.mask.cmp.d.512(<16 x i32> %a0, <16 x i32> %a1, i32 5, i16 -1)
   %vec5 = insertelement <8 x i16> %vec4, i16 %res5, i32 5
 ; CHECK: vpcmpnled %zmm1, %zmm0, %k0 ##
-  %res6 = call i16 @llvm.x86.avx512.mask.cmp.d.512(<16 x i32> %a0, <16 x i32> %a1, i8 6, i16 -1)
+  %res6 = call i16 @llvm.x86.avx512.mask.cmp.d.512(<16 x i32> %a0, <16 x i32> %a1, i32 6, i16 -1)
   %vec6 = insertelement <8 x i16> %vec5, i16 %res6, i32 6
 ; CHECK: vpcmpordd %zmm1, %zmm0, %k0 ##
-  %res7 = call i16 @llvm.x86.avx512.mask.cmp.d.512(<16 x i32> %a0, <16 x i32> %a1, i8 7, i16 -1)
+  %res7 = call i16 @llvm.x86.avx512.mask.cmp.d.512(<16 x i32> %a0, <16 x i32> %a1, i32 7, i16 -1)
   %vec7 = insertelement <8 x i16> %vec6, i16 %res7, i32 7
   ret <8 x i16> %vec7
 }
@@ -765,59 +693,59 @@ define <8 x i16> @test_cmp_d_512(<16 x i32> %a0, <16 x i32> %a1) {
 define <8 x i16> @test_mask_cmp_d_512(<16 x i32> %a0, <16 x i32> %a1, i16 %mask) {
 ; CHECK_LABEL: test_mask_cmp_d_512
 ; CHECK: vpcmpeqd %zmm1, %zmm0, %k0 {%k1} ##
-  %res0 = call i16 @llvm.x86.avx512.mask.cmp.d.512(<16 x i32> %a0, <16 x i32> %a1, i8 0, i16 %mask)
+  %res0 = call i16 @llvm.x86.avx512.mask.cmp.d.512(<16 x i32> %a0, <16 x i32> %a1, i32 0, i16 %mask)
   %vec0 = insertelement <8 x i16> undef, i16 %res0, i32 0
 ; CHECK: vpcmpltd %zmm1, %zmm0, %k0 {%k1} ##
-  %res1 = call i16 @llvm.x86.avx512.mask.cmp.d.512(<16 x i32> %a0, <16 x i32> %a1, i8 1, i16 %mask)
+  %res1 = call i16 @llvm.x86.avx512.mask.cmp.d.512(<16 x i32> %a0, <16 x i32> %a1, i32 1, i16 %mask)
   %vec1 = insertelement <8 x i16> %vec0, i16 %res1, i32 1
 ; CHECK: vpcmpled %zmm1, %zmm0, %k0 {%k1} ##
-  %res2 = call i16 @llvm.x86.avx512.mask.cmp.d.512(<16 x i32> %a0, <16 x i32> %a1, i8 2, i16 %mask)
+  %res2 = call i16 @llvm.x86.avx512.mask.cmp.d.512(<16 x i32> %a0, <16 x i32> %a1, i32 2, i16 %mask)
   %vec2 = insertelement <8 x i16> %vec1, i16 %res2, i32 2
 ; CHECK: vpcmpunordd %zmm1, %zmm0, %k0 {%k1} ##
-  %res3 = call i16 @llvm.x86.avx512.mask.cmp.d.512(<16 x i32> %a0, <16 x i32> %a1, i8 3, i16 %mask)
+  %res3 = call i16 @llvm.x86.avx512.mask.cmp.d.512(<16 x i32> %a0, <16 x i32> %a1, i32 3, i16 %mask)
   %vec3 = insertelement <8 x i16> %vec2, i16 %res3, i32 3
 ; CHECK: vpcmpneqd %zmm1, %zmm0, %k0 {%k1} ##
-  %res4 = call i16 @llvm.x86.avx512.mask.cmp.d.512(<16 x i32> %a0, <16 x i32> %a1, i8 4, i16 %mask)
+  %res4 = call i16 @llvm.x86.avx512.mask.cmp.d.512(<16 x i32> %a0, <16 x i32> %a1, i32 4, i16 %mask)
   %vec4 = insertelement <8 x i16> %vec3, i16 %res4, i32 4
 ; CHECK: vpcmpnltd %zmm1, %zmm0, %k0 {%k1} ##
-  %res5 = call i16 @llvm.x86.avx512.mask.cmp.d.512(<16 x i32> %a0, <16 x i32> %a1, i8 5, i16 %mask)
+  %res5 = call i16 @llvm.x86.avx512.mask.cmp.d.512(<16 x i32> %a0, <16 x i32> %a1, i32 5, i16 %mask)
   %vec5 = insertelement <8 x i16> %vec4, i16 %res5, i32 5
 ; CHECK: vpcmpnled %zmm1, %zmm0, %k0 {%k1} ##
-  %res6 = call i16 @llvm.x86.avx512.mask.cmp.d.512(<16 x i32> %a0, <16 x i32> %a1, i8 6, i16 %mask)
+  %res6 = call i16 @llvm.x86.avx512.mask.cmp.d.512(<16 x i32> %a0, <16 x i32> %a1, i32 6, i16 %mask)
   %vec6 = insertelement <8 x i16> %vec5, i16 %res6, i32 6
 ; CHECK: vpcmpordd %zmm1, %zmm0, %k0 {%k1} ##
-  %res7 = call i16 @llvm.x86.avx512.mask.cmp.d.512(<16 x i32> %a0, <16 x i32> %a1, i8 7, i16 %mask)
+  %res7 = call i16 @llvm.x86.avx512.mask.cmp.d.512(<16 x i32> %a0, <16 x i32> %a1, i32 7, i16 %mask)
   %vec7 = insertelement <8 x i16> %vec6, i16 %res7, i32 7
   ret <8 x i16> %vec7
 }
 
-declare i16 @llvm.x86.avx512.mask.cmp.d.512(<16 x i32>, <16 x i32>, i8, i16) nounwind readnone
+declare i16 @llvm.x86.avx512.mask.cmp.d.512(<16 x i32>, <16 x i32>, i32, i16) nounwind readnone
 
 define <8 x i16> @test_ucmp_d_512(<16 x i32> %a0, <16 x i32> %a1) {
 ; CHECK_LABEL: test_ucmp_d_512
 ; CHECK: vpcmpequd %zmm1, %zmm0, %k0 ##
-  %res0 = call i16 @llvm.x86.avx512.mask.ucmp.d.512(<16 x i32> %a0, <16 x i32> %a1, i8 0, i16 -1)
+  %res0 = call i16 @llvm.x86.avx512.mask.ucmp.d.512(<16 x i32> %a0, <16 x i32> %a1, i32 0, i16 -1)
   %vec0 = insertelement <8 x i16> undef, i16 %res0, i32 0
 ; CHECK: vpcmpltud %zmm1, %zmm0, %k0 ##
-  %res1 = call i16 @llvm.x86.avx512.mask.ucmp.d.512(<16 x i32> %a0, <16 x i32> %a1, i8 1, i16 -1)
+  %res1 = call i16 @llvm.x86.avx512.mask.ucmp.d.512(<16 x i32> %a0, <16 x i32> %a1, i32 1, i16 -1)
   %vec1 = insertelement <8 x i16> %vec0, i16 %res1, i32 1
 ; CHECK: vpcmpleud %zmm1, %zmm0, %k0 ##
-  %res2 = call i16 @llvm.x86.avx512.mask.ucmp.d.512(<16 x i32> %a0, <16 x i32> %a1, i8 2, i16 -1)
+  %res2 = call i16 @llvm.x86.avx512.mask.ucmp.d.512(<16 x i32> %a0, <16 x i32> %a1, i32 2, i16 -1)
   %vec2 = insertelement <8 x i16> %vec1, i16 %res2, i32 2
 ; CHECK: vpcmpunordud %zmm1, %zmm0, %k0 ##
-  %res3 = call i16 @llvm.x86.avx512.mask.ucmp.d.512(<16 x i32> %a0, <16 x i32> %a1, i8 3, i16 -1)
+  %res3 = call i16 @llvm.x86.avx512.mask.ucmp.d.512(<16 x i32> %a0, <16 x i32> %a1, i32 3, i16 -1)
   %vec3 = insertelement <8 x i16> %vec2, i16 %res3, i32 3
 ; CHECK: vpcmpnequd %zmm1, %zmm0, %k0 ##
-  %res4 = call i16 @llvm.x86.avx512.mask.ucmp.d.512(<16 x i32> %a0, <16 x i32> %a1, i8 4, i16 -1)
+  %res4 = call i16 @llvm.x86.avx512.mask.ucmp.d.512(<16 x i32> %a0, <16 x i32> %a1, i32 4, i16 -1)
   %vec4 = insertelement <8 x i16> %vec3, i16 %res4, i32 4
 ; CHECK: vpcmpnltud %zmm1, %zmm0, %k0 ##
-  %res5 = call i16 @llvm.x86.avx512.mask.ucmp.d.512(<16 x i32> %a0, <16 x i32> %a1, i8 5, i16 -1)
+  %res5 = call i16 @llvm.x86.avx512.mask.ucmp.d.512(<16 x i32> %a0, <16 x i32> %a1, i32 5, i16 -1)
   %vec5 = insertelement <8 x i16> %vec4, i16 %res5, i32 5
 ; CHECK: vpcmpnleud %zmm1, %zmm0, %k0 ##
-  %res6 = call i16 @llvm.x86.avx512.mask.ucmp.d.512(<16 x i32> %a0, <16 x i32> %a1, i8 6, i16 -1)
+  %res6 = call i16 @llvm.x86.avx512.mask.ucmp.d.512(<16 x i32> %a0, <16 x i32> %a1, i32 6, i16 -1)
   %vec6 = insertelement <8 x i16> %vec5, i16 %res6, i32 6
 ; CHECK: vpcmpordud %zmm1, %zmm0, %k0 ##
-  %res7 = call i16 @llvm.x86.avx512.mask.ucmp.d.512(<16 x i32> %a0, <16 x i32> %a1, i8 7, i16 -1)
+  %res7 = call i16 @llvm.x86.avx512.mask.ucmp.d.512(<16 x i32> %a0, <16 x i32> %a1, i32 7, i16 -1)
   %vec7 = insertelement <8 x i16> %vec6, i16 %res7, i32 7
   ret <8 x i16> %vec7
 }
@@ -825,59 +753,59 @@ define <8 x i16> @test_ucmp_d_512(<16 x i32> %a0, <16 x i32> %a1) {
 define <8 x i16> @test_mask_ucmp_d_512(<16 x i32> %a0, <16 x i32> %a1, i16 %mask) {
 ; CHECK_LABEL: test_mask_ucmp_d_512
 ; CHECK: vpcmpequd %zmm1, %zmm0, %k0 {%k1} ##
-  %res0 = call i16 @llvm.x86.avx512.mask.ucmp.d.512(<16 x i32> %a0, <16 x i32> %a1, i8 0, i16 %mask)
+  %res0 = call i16 @llvm.x86.avx512.mask.ucmp.d.512(<16 x i32> %a0, <16 x i32> %a1, i32 0, i16 %mask)
   %vec0 = insertelement <8 x i16> undef, i16 %res0, i32 0
 ; CHECK: vpcmpltud %zmm1, %zmm0, %k0 {%k1} ##
-  %res1 = call i16 @llvm.x86.avx512.mask.ucmp.d.512(<16 x i32> %a0, <16 x i32> %a1, i8 1, i16 %mask)
+  %res1 = call i16 @llvm.x86.avx512.mask.ucmp.d.512(<16 x i32> %a0, <16 x i32> %a1, i32 1, i16 %mask)
   %vec1 = insertelement <8 x i16> %vec0, i16 %res1, i32 1
 ; CHECK: vpcmpleud %zmm1, %zmm0, %k0 {%k1} ##
-  %res2 = call i16 @llvm.x86.avx512.mask.ucmp.d.512(<16 x i32> %a0, <16 x i32> %a1, i8 2, i16 %mask)
+  %res2 = call i16 @llvm.x86.avx512.mask.ucmp.d.512(<16 x i32> %a0, <16 x i32> %a1, i32 2, i16 %mask)
   %vec2 = insertelement <8 x i16> %vec1, i16 %res2, i32 2
 ; CHECK: vpcmpunordud %zmm1, %zmm0, %k0 {%k1} ##
-  %res3 = call i16 @llvm.x86.avx512.mask.ucmp.d.512(<16 x i32> %a0, <16 x i32> %a1, i8 3, i16 %mask)
+  %res3 = call i16 @llvm.x86.avx512.mask.ucmp.d.512(<16 x i32> %a0, <16 x i32> %a1, i32 3, i16 %mask)
   %vec3 = insertelement <8 x i16> %vec2, i16 %res3, i32 3
 ; CHECK: vpcmpnequd %zmm1, %zmm0, %k0 {%k1} ##
-  %res4 = call i16 @llvm.x86.avx512.mask.ucmp.d.512(<16 x i32> %a0, <16 x i32> %a1, i8 4, i16 %mask)
+  %res4 = call i16 @llvm.x86.avx512.mask.ucmp.d.512(<16 x i32> %a0, <16 x i32> %a1, i32 4, i16 %mask)
   %vec4 = insertelement <8 x i16> %vec3, i16 %res4, i32 4
 ; CHECK: vpcmpnltud %zmm1, %zmm0, %k0 {%k1} ##
-  %res5 = call i16 @llvm.x86.avx512.mask.ucmp.d.512(<16 x i32> %a0, <16 x i32> %a1, i8 5, i16 %mask)
+  %res5 = call i16 @llvm.x86.avx512.mask.ucmp.d.512(<16 x i32> %a0, <16 x i32> %a1, i32 5, i16 %mask)
   %vec5 = insertelement <8 x i16> %vec4, i16 %res5, i32 5
 ; CHECK: vpcmpnleud %zmm1, %zmm0, %k0 {%k1} ##
-  %res6 = call i16 @llvm.x86.avx512.mask.ucmp.d.512(<16 x i32> %a0, <16 x i32> %a1, i8 6, i16 %mask)
+  %res6 = call i16 @llvm.x86.avx512.mask.ucmp.d.512(<16 x i32> %a0, <16 x i32> %a1, i32 6, i16 %mask)
   %vec6 = insertelement <8 x i16> %vec5, i16 %res6, i32 6
 ; CHECK: vpcmpordud %zmm1, %zmm0, %k0 {%k1} ##
-  %res7 = call i16 @llvm.x86.avx512.mask.ucmp.d.512(<16 x i32> %a0, <16 x i32> %a1, i8 7, i16 %mask)
+  %res7 = call i16 @llvm.x86.avx512.mask.ucmp.d.512(<16 x i32> %a0, <16 x i32> %a1, i32 7, i16 %mask)
   %vec7 = insertelement <8 x i16> %vec6, i16 %res7, i32 7
   ret <8 x i16> %vec7
 }
 
-declare i16 @llvm.x86.avx512.mask.ucmp.d.512(<16 x i32>, <16 x i32>, i8, i16) nounwind readnone
+declare i16 @llvm.x86.avx512.mask.ucmp.d.512(<16 x i32>, <16 x i32>, i32, i16) nounwind readnone
 
 define <8 x i8> @test_cmp_q_512(<8 x i64> %a0, <8 x i64> %a1) {
 ; CHECK_LABEL: test_cmp_q_512
 ; CHECK: vpcmpeqq %zmm1, %zmm0, %k0 ##
-  %res0 = call i8 @llvm.x86.avx512.mask.cmp.q.512(<8 x i64> %a0, <8 x i64> %a1, i8 0, i8 -1)
+  %res0 = call i8 @llvm.x86.avx512.mask.cmp.q.512(<8 x i64> %a0, <8 x i64> %a1, i32 0, i8 -1)
   %vec0 = insertelement <8 x i8> undef, i8 %res0, i32 0
 ; CHECK: vpcmpltq %zmm1, %zmm0, %k0 ##
-  %res1 = call i8 @llvm.x86.avx512.mask.cmp.q.512(<8 x i64> %a0, <8 x i64> %a1, i8 1, i8 -1)
+  %res1 = call i8 @llvm.x86.avx512.mask.cmp.q.512(<8 x i64> %a0, <8 x i64> %a1, i32 1, i8 -1)
   %vec1 = insertelement <8 x i8> %vec0, i8 %res1, i32 1
 ; CHECK: vpcmpleq %zmm1, %zmm0, %k0 ##
-  %res2 = call i8 @llvm.x86.avx512.mask.cmp.q.512(<8 x i64> %a0, <8 x i64> %a1, i8 2, i8 -1)
+  %res2 = call i8 @llvm.x86.avx512.mask.cmp.q.512(<8 x i64> %a0, <8 x i64> %a1, i32 2, i8 -1)
   %vec2 = insertelement <8 x i8> %vec1, i8 %res2, i32 2
 ; CHECK: vpcmpunordq %zmm1, %zmm0, %k0 ##
-  %res3 = call i8 @llvm.x86.avx512.mask.cmp.q.512(<8 x i64> %a0, <8 x i64> %a1, i8 3, i8 -1)
+  %res3 = call i8 @llvm.x86.avx512.mask.cmp.q.512(<8 x i64> %a0, <8 x i64> %a1, i32 3, i8 -1)
   %vec3 = insertelement <8 x i8> %vec2, i8 %res3, i32 3
 ; CHECK: vpcmpneqq %zmm1, %zmm0, %k0 ##
-  %res4 = call i8 @llvm.x86.avx512.mask.cmp.q.512(<8 x i64> %a0, <8 x i64> %a1, i8 4, i8 -1)
+  %res4 = call i8 @llvm.x86.avx512.mask.cmp.q.512(<8 x i64> %a0, <8 x i64> %a1, i32 4, i8 -1)
   %vec4 = insertelement <8 x i8> %vec3, i8 %res4, i32 4
 ; CHECK: vpcmpnltq %zmm1, %zmm0, %k0 ##
-  %res5 = call i8 @llvm.x86.avx512.mask.cmp.q.512(<8 x i64> %a0, <8 x i64> %a1, i8 5, i8 -1)
+  %res5 = call i8 @llvm.x86.avx512.mask.cmp.q.512(<8 x i64> %a0, <8 x i64> %a1, i32 5, i8 -1)
   %vec5 = insertelement <8 x i8> %vec4, i8 %res5, i32 5
 ; CHECK: vpcmpnleq %zmm1, %zmm0, %k0 ##
-  %res6 = call i8 @llvm.x86.avx512.mask.cmp.q.512(<8 x i64> %a0, <8 x i64> %a1, i8 6, i8 -1)
+  %res6 = call i8 @llvm.x86.avx512.mask.cmp.q.512(<8 x i64> %a0, <8 x i64> %a1, i32 6, i8 -1)
   %vec6 = insertelement <8 x i8> %vec5, i8 %res6, i32 6
 ; CHECK: vpcmpordq %zmm1, %zmm0, %k0 ##
-  %res7 = call i8 @llvm.x86.avx512.mask.cmp.q.512(<8 x i64> %a0, <8 x i64> %a1, i8 7, i8 -1)
+  %res7 = call i8 @llvm.x86.avx512.mask.cmp.q.512(<8 x i64> %a0, <8 x i64> %a1, i32 7, i8 -1)
   %vec7 = insertelement <8 x i8> %vec6, i8 %res7, i32 7
   ret <8 x i8> %vec7
 }
@@ -885,59 +813,59 @@ define <8 x i8> @test_cmp_q_512(<8 x i64> %a0, <8 x i64> %a1) {
 define <8 x i8> @test_mask_cmp_q_512(<8 x i64> %a0, <8 x i64> %a1, i8 %mask) {
 ; CHECK_LABEL: test_mask_cmp_q_512
 ; CHECK: vpcmpeqq %zmm1, %zmm0, %k0 {%k1} ##
-  %res0 = call i8 @llvm.x86.avx512.mask.cmp.q.512(<8 x i64> %a0, <8 x i64> %a1, i8 0, i8 %mask)
+  %res0 = call i8 @llvm.x86.avx512.mask.cmp.q.512(<8 x i64> %a0, <8 x i64> %a1, i32 0, i8 %mask)
   %vec0 = insertelement <8 x i8> undef, i8 %res0, i32 0
 ; CHECK: vpcmpltq %zmm1, %zmm0, %k0 {%k1} ##
-  %res1 = call i8 @llvm.x86.avx512.mask.cmp.q.512(<8 x i64> %a0, <8 x i64> %a1, i8 1, i8 %mask)
+  %res1 = call i8 @llvm.x86.avx512.mask.cmp.q.512(<8 x i64> %a0, <8 x i64> %a1, i32 1, i8 %mask)
   %vec1 = insertelement <8 x i8> %vec0, i8 %res1, i32 1
 ; CHECK: vpcmpleq %zmm1, %zmm0, %k0 {%k1} ##
-  %res2 = call i8 @llvm.x86.avx512.mask.cmp.q.512(<8 x i64> %a0, <8 x i64> %a1, i8 2, i8 %mask)
+  %res2 = call i8 @llvm.x86.avx512.mask.cmp.q.512(<8 x i64> %a0, <8 x i64> %a1, i32 2, i8 %mask)
   %vec2 = insertelement <8 x i8> %vec1, i8 %res2, i32 2
 ; CHECK: vpcmpunordq %zmm1, %zmm0, %k0 {%k1} ##
-  %res3 = call i8 @llvm.x86.avx512.mask.cmp.q.512(<8 x i64> %a0, <8 x i64> %a1, i8 3, i8 %mask)
+  %res3 = call i8 @llvm.x86.avx512.mask.cmp.q.512(<8 x i64> %a0, <8 x i64> %a1, i32 3, i8 %mask)
   %vec3 = insertelement <8 x i8> %vec2, i8 %res3, i32 3
 ; CHECK: vpcmpneqq %zmm1, %zmm0, %k0 {%k1} ##
-  %res4 = call i8 @llvm.x86.avx512.mask.cmp.q.512(<8 x i64> %a0, <8 x i64> %a1, i8 4, i8 %mask)
+  %res4 = call i8 @llvm.x86.avx512.mask.cmp.q.512(<8 x i64> %a0, <8 x i64> %a1, i32 4, i8 %mask)
   %vec4 = insertelement <8 x i8> %vec3, i8 %res4, i32 4
 ; CHECK: vpcmpnltq %zmm1, %zmm0, %k0 {%k1} ##
-  %res5 = call i8 @llvm.x86.avx512.mask.cmp.q.512(<8 x i64> %a0, <8 x i64> %a1, i8 5, i8 %mask)
+  %res5 = call i8 @llvm.x86.avx512.mask.cmp.q.512(<8 x i64> %a0, <8 x i64> %a1, i32 5, i8 %mask)
   %vec5 = insertelement <8 x i8> %vec4, i8 %res5, i32 5
 ; CHECK: vpcmpnleq %zmm1, %zmm0, %k0 {%k1} ##
-  %res6 = call i8 @llvm.x86.avx512.mask.cmp.q.512(<8 x i64> %a0, <8 x i64> %a1, i8 6, i8 %mask)
+  %res6 = call i8 @llvm.x86.avx512.mask.cmp.q.512(<8 x i64> %a0, <8 x i64> %a1, i32 6, i8 %mask)
   %vec6 = insertelement <8 x i8> %vec5, i8 %res6, i32 6
 ; CHECK: vpcmpordq %zmm1, %zmm0, %k0 {%k1} ##
-  %res7 = call i8 @llvm.x86.avx512.mask.cmp.q.512(<8 x i64> %a0, <8 x i64> %a1, i8 7, i8 %mask)
+  %res7 = call i8 @llvm.x86.avx512.mask.cmp.q.512(<8 x i64> %a0, <8 x i64> %a1, i32 7, i8 %mask)
   %vec7 = insertelement <8 x i8> %vec6, i8 %res7, i32 7
   ret <8 x i8> %vec7
 }
 
-declare i8 @llvm.x86.avx512.mask.cmp.q.512(<8 x i64>, <8 x i64>, i8, i8) nounwind readnone
+declare i8 @llvm.x86.avx512.mask.cmp.q.512(<8 x i64>, <8 x i64>, i32, i8) nounwind readnone
 
 define <8 x i8> @test_ucmp_q_512(<8 x i64> %a0, <8 x i64> %a1) {
 ; CHECK_LABEL: test_ucmp_q_512
 ; CHECK: vpcmpequq %zmm1, %zmm0, %k0 ##
-  %res0 = call i8 @llvm.x86.avx512.mask.ucmp.q.512(<8 x i64> %a0, <8 x i64> %a1, i8 0, i8 -1)
+  %res0 = call i8 @llvm.x86.avx512.mask.ucmp.q.512(<8 x i64> %a0, <8 x i64> %a1, i32 0, i8 -1)
   %vec0 = insertelement <8 x i8> undef, i8 %res0, i32 0
 ; CHECK: vpcmpltuq %zmm1, %zmm0, %k0 ##
-  %res1 = call i8 @llvm.x86.avx512.mask.ucmp.q.512(<8 x i64> %a0, <8 x i64> %a1, i8 1, i8 -1)
+  %res1 = call i8 @llvm.x86.avx512.mask.ucmp.q.512(<8 x i64> %a0, <8 x i64> %a1, i32 1, i8 -1)
   %vec1 = insertelement <8 x i8> %vec0, i8 %res1, i32 1
 ; CHECK: vpcmpleuq %zmm1, %zmm0, %k0 ##
-  %res2 = call i8 @llvm.x86.avx512.mask.ucmp.q.512(<8 x i64> %a0, <8 x i64> %a1, i8 2, i8 -1)
+  %res2 = call i8 @llvm.x86.avx512.mask.ucmp.q.512(<8 x i64> %a0, <8 x i64> %a1, i32 2, i8 -1)
   %vec2 = insertelement <8 x i8> %vec1, i8 %res2, i32 2
 ; CHECK: vpcmpunorduq %zmm1, %zmm0, %k0 ##
-  %res3 = call i8 @llvm.x86.avx512.mask.ucmp.q.512(<8 x i64> %a0, <8 x i64> %a1, i8 3, i8 -1)
+  %res3 = call i8 @llvm.x86.avx512.mask.ucmp.q.512(<8 x i64> %a0, <8 x i64> %a1, i32 3, i8 -1)
   %vec3 = insertelement <8 x i8> %vec2, i8 %res3, i32 3
 ; CHECK: vpcmpnequq %zmm1, %zmm0, %k0 ##
-  %res4 = call i8 @llvm.x86.avx512.mask.ucmp.q.512(<8 x i64> %a0, <8 x i64> %a1, i8 4, i8 -1)
+  %res4 = call i8 @llvm.x86.avx512.mask.ucmp.q.512(<8 x i64> %a0, <8 x i64> %a1, i32 4, i8 -1)
   %vec4 = insertelement <8 x i8> %vec3, i8 %res4, i32 4
 ; CHECK: vpcmpnltuq %zmm1, %zmm0, %k0 ##
-  %res5 = call i8 @llvm.x86.avx512.mask.ucmp.q.512(<8 x i64> %a0, <8 x i64> %a1, i8 5, i8 -1)
+  %res5 = call i8 @llvm.x86.avx512.mask.ucmp.q.512(<8 x i64> %a0, <8 x i64> %a1, i32 5, i8 -1)
   %vec5 = insertelement <8 x i8> %vec4, i8 %res5, i32 5
 ; CHECK: vpcmpnleuq %zmm1, %zmm0, %k0 ##
-  %res6 = call i8 @llvm.x86.avx512.mask.ucmp.q.512(<8 x i64> %a0, <8 x i64> %a1, i8 6, i8 -1)
+  %res6 = call i8 @llvm.x86.avx512.mask.ucmp.q.512(<8 x i64> %a0, <8 x i64> %a1, i32 6, i8 -1)
   %vec6 = insertelement <8 x i8> %vec5, i8 %res6, i32 6
 ; CHECK: vpcmporduq %zmm1, %zmm0, %k0 ##
-  %res7 = call i8 @llvm.x86.avx512.mask.ucmp.q.512(<8 x i64> %a0, <8 x i64> %a1, i8 7, i8 -1)
+  %res7 = call i8 @llvm.x86.avx512.mask.ucmp.q.512(<8 x i64> %a0, <8 x i64> %a1, i32 7, i8 -1)
   %vec7 = insertelement <8 x i8> %vec6, i8 %res7, i32 7
   ret <8 x i8> %vec7
 }
@@ -945,33 +873,33 @@ define <8 x i8> @test_ucmp_q_512(<8 x i64> %a0, <8 x i64> %a1) {
 define <8 x i8> @test_mask_ucmp_q_512(<8 x i64> %a0, <8 x i64> %a1, i8 %mask) {
 ; CHECK_LABEL: test_mask_ucmp_q_512
 ; CHECK: vpcmpequq %zmm1, %zmm0, %k0 {%k1} ##
-  %res0 = call i8 @llvm.x86.avx512.mask.ucmp.q.512(<8 x i64> %a0, <8 x i64> %a1, i8 0, i8 %mask)
+  %res0 = call i8 @llvm.x86.avx512.mask.ucmp.q.512(<8 x i64> %a0, <8 x i64> %a1, i32 0, i8 %mask)
   %vec0 = insertelement <8 x i8> undef, i8 %res0, i32 0
 ; CHECK: vpcmpltuq %zmm1, %zmm0, %k0 {%k1} ##
-  %res1 = call i8 @llvm.x86.avx512.mask.ucmp.q.512(<8 x i64> %a0, <8 x i64> %a1, i8 1, i8 %mask)
+  %res1 = call i8 @llvm.x86.avx512.mask.ucmp.q.512(<8 x i64> %a0, <8 x i64> %a1, i32 1, i8 %mask)
   %vec1 = insertelement <8 x i8> %vec0, i8 %res1, i32 1
 ; CHECK: vpcmpleuq %zmm1, %zmm0, %k0 {%k1} ##
-  %res2 = call i8 @llvm.x86.avx512.mask.ucmp.q.512(<8 x i64> %a0, <8 x i64> %a1, i8 2, i8 %mask)
+  %res2 = call i8 @llvm.x86.avx512.mask.ucmp.q.512(<8 x i64> %a0, <8 x i64> %a1, i32 2, i8 %mask)
   %vec2 = insertelement <8 x i8> %vec1, i8 %res2, i32 2
 ; CHECK: vpcmpunorduq %zmm1, %zmm0, %k0 {%k1} ##
-  %res3 = call i8 @llvm.x86.avx512.mask.ucmp.q.512(<8 x i64> %a0, <8 x i64> %a1, i8 3, i8 %mask)
+  %res3 = call i8 @llvm.x86.avx512.mask.ucmp.q.512(<8 x i64> %a0, <8 x i64> %a1, i32 3, i8 %mask)
   %vec3 = insertelement <8 x i8> %vec2, i8 %res3, i32 3
 ; CHECK: vpcmpnequq %zmm1, %zmm0, %k0 {%k1} ##
-  %res4 = call i8 @llvm.x86.avx512.mask.ucmp.q.512(<8 x i64> %a0, <8 x i64> %a1, i8 4, i8 %mask)
+  %res4 = call i8 @llvm.x86.avx512.mask.ucmp.q.512(<8 x i64> %a0, <8 x i64> %a1, i32 4, i8 %mask)
   %vec4 = insertelement <8 x i8> %vec3, i8 %res4, i32 4
 ; CHECK: vpcmpnltuq %zmm1, %zmm0, %k0 {%k1} ##
-  %res5 = call i8 @llvm.x86.avx512.mask.ucmp.q.512(<8 x i64> %a0, <8 x i64> %a1, i8 5, i8 %mask)
+  %res5 = call i8 @llvm.x86.avx512.mask.ucmp.q.512(<8 x i64> %a0, <8 x i64> %a1, i32 5, i8 %mask)
   %vec5 = insertelement <8 x i8> %vec4, i8 %res5, i32 5
 ; CHECK: vpcmpnleuq %zmm1, %zmm0, %k0 {%k1} ##
-  %res6 = call i8 @llvm.x86.avx512.mask.ucmp.q.512(<8 x i64> %a0, <8 x i64> %a1, i8 6, i8 %mask)
+  %res6 = call i8 @llvm.x86.avx512.mask.ucmp.q.512(<8 x i64> %a0, <8 x i64> %a1, i32 6, i8 %mask)
   %vec6 = insertelement <8 x i8> %vec5, i8 %res6, i32 6
 ; CHECK: vpcmporduq %zmm1, %zmm0, %k0 {%k1} ##
-  %res7 = call i8 @llvm.x86.avx512.mask.ucmp.q.512(<8 x i64> %a0, <8 x i64> %a1, i8 7, i8 %mask)
+  %res7 = call i8 @llvm.x86.avx512.mask.ucmp.q.512(<8 x i64> %a0, <8 x i64> %a1, i32 7, i8 %mask)
   %vec7 = insertelement <8 x i8> %vec6, i8 %res7, i32 7
   ret <8 x i8> %vec7
 }
 
-declare i8 @llvm.x86.avx512.mask.ucmp.q.512(<8 x i64>, <8 x i64>, i8, i8) nounwind readnone
+declare i8 @llvm.x86.avx512.mask.ucmp.q.512(<8 x i64>, <8 x i64>, i32, i8) nounwind readnone
 
 define <4 x float> @test_mask_vextractf32x4(<4 x float> %b, <16 x float> %a, i8 %mask) {
 ; CHECK-LABEL: test_mask_vextractf32x4:
@@ -2163,3 +2091,1863 @@ define <8 x i64> @test_mask_mul_epu32_rmbkz(<16 x i32> %a, i64* %ptr_b, i8 %mask
 }
 
 declare <8 x i64> @llvm.x86.avx512.mask.pmulu.dq.512(<16 x i32>, <16 x i32>, <8 x i64>, i8)
+
+define <16 x i32> @test_mask_mullo_epi32_rr_512(<16 x i32> %a, <16 x i32> %b) {
+  ;CHECK-LABEL: test_mask_mullo_epi32_rr_512
+  ;CHECK: vpmulld %zmm1, %zmm0, %zmm0 ## encoding: [0x62,0xf2,0x7d,0x48,0x40,0xc1]
+  %res = call <16 x i32> @llvm.x86.avx512.mask.pmull.d.512(<16 x i32> %a, <16 x i32> %b, <16 x i32> zeroinitializer, i16 -1)
+  ret <16 x i32> %res
+}
+
+define <16 x i32> @test_mask_mullo_epi32_rrk_512(<16 x i32> %a, <16 x i32> %b, <16 x i32> %passThru, i16 %mask) {
+  ;CHECK-LABEL: test_mask_mullo_epi32_rrk_512
+  ;CHECK: vpmulld %zmm1, %zmm0, %zmm2 {%k1} ## encoding: [0x62,0xf2,0x7d,0x49,0x40,0xd1]
+  %res = call <16 x i32> @llvm.x86.avx512.mask.pmull.d.512(<16 x i32> %a, <16 x i32> %b, <16 x i32> %passThru, i16 %mask)
+  ret < 16 x i32> %res
+}
+
+define <16 x i32> @test_mask_mullo_epi32_rrkz_512(<16 x i32> %a, <16 x i32> %b, i16 %mask) {
+  ;CHECK-LABEL: test_mask_mullo_epi32_rrkz_512
+  ;CHECK: vpmulld %zmm1, %zmm0, %zmm0 {%k1} {z} ## encoding: [0x62,0xf2,0x7d,0xc9,0x40,0xc1]
+  %res = call <16 x i32> @llvm.x86.avx512.mask.pmull.d.512(<16 x i32> %a, <16 x i32> %b, <16 x i32> zeroinitializer, i16 %mask)
+  ret < 16 x i32> %res
+}
+
+define <16 x i32> @test_mask_mullo_epi32_rm_512(<16 x i32> %a, <16 x i32>* %ptr_b) {
+  ;CHECK-LABEL: test_mask_mullo_epi32_rm_512
+  ;CHECK: vpmulld (%rdi), %zmm0, %zmm0 ## encoding: [0x62,0xf2,0x7d,0x48,0x40,0x07]
+  %b = load <16 x i32>, <16 x i32>* %ptr_b
+  %res = call <16 x i32> @llvm.x86.avx512.mask.pmull.d.512(<16 x i32> %a, <16 x i32> %b, <16 x i32> zeroinitializer, i16 -1)
+  ret < 16 x i32> %res
+}
+
+define <16 x i32> @test_mask_mullo_epi32_rmk_512(<16 x i32> %a, <16 x i32>* %ptr_b, <16 x i32> %passThru, i16 %mask) {
+  ;CHECK-LABEL: test_mask_mullo_epi32_rmk_512
+  ;CHECK: vpmulld (%rdi), %zmm0, %zmm1 {%k1} ## encoding: [0x62,0xf2,0x7d,0x49,0x40,0x0f]
+  %b = load <16 x i32>, <16 x i32>* %ptr_b
+  %res = call <16 x i32> @llvm.x86.avx512.mask.pmull.d.512(<16 x i32> %a, <16 x i32> %b, <16 x i32> %passThru, i16 %mask)
+  ret < 16 x i32> %res
+}
+
+define <16 x i32> @test_mask_mullo_epi32_rmkz_512(<16 x i32> %a, <16 x i32>* %ptr_b, i16 %mask) {
+  ;CHECK-LABEL: test_mask_mullo_epi32_rmkz_512
+  ;CHECK: vpmulld (%rdi), %zmm0, %zmm0 {%k1} {z} ## encoding: [0x62,0xf2,0x7d,0xc9,0x40,0x07]
+  %b = load <16 x i32>, <16 x i32>* %ptr_b
+  %res = call <16 x i32> @llvm.x86.avx512.mask.pmull.d.512(<16 x i32> %a, <16 x i32> %b, <16 x i32> zeroinitializer, i16 %mask)
+  ret < 16 x i32> %res
+}
+
+define <16 x i32> @test_mask_mullo_epi32_rmb_512(<16 x i32> %a, i32* %ptr_b) {
+  ;CHECK-LABEL: test_mask_mullo_epi32_rmb_512
+  ;CHECK: vpmulld (%rdi){1to16}, %zmm0, %zmm0 ## encoding: [0x62,0xf2,0x7d,0x58,0x40,0x07]
+  %q = load i32, i32* %ptr_b
+  %vecinit.i = insertelement <16 x i32> undef, i32 %q, i32 0
+  %b = shufflevector <16 x i32> %vecinit.i, <16 x i32> undef, <16 x i32> zeroinitializer
+  %res = call <16 x i32> @llvm.x86.avx512.mask.pmull.d.512(<16 x i32> %a, <16 x i32> %b, <16 x i32> zeroinitializer, i16 -1)
+  ret < 16 x i32> %res
+}
+
+define <16 x i32> @test_mask_mullo_epi32_rmbk_512(<16 x i32> %a, i32* %ptr_b, <16 x i32> %passThru, i16 %mask) {
+  ;CHECK-LABEL: test_mask_mullo_epi32_rmbk_512
+  ;CHECK: vpmulld (%rdi){1to16}, %zmm0, %zmm1 {%k1} ## encoding: [0x62,0xf2,0x7d,0x59,0x40,0x0f]
+  %q = load i32, i32* %ptr_b
+  %vecinit.i = insertelement <16 x i32> undef, i32 %q, i32 0
+  %b = shufflevector <16 x i32> %vecinit.i, <16 x i32> undef, <16 x i32> zeroinitializer
+  %res = call <16 x i32> @llvm.x86.avx512.mask.pmull.d.512(<16 x i32> %a, <16 x i32> %b, <16 x i32> %passThru, i16 %mask)
+  ret < 16 x i32> %res
+}
+
+define <16 x i32> @test_mask_mullo_epi32_rmbkz_512(<16 x i32> %a, i32* %ptr_b, i16 %mask) {
+  ;CHECK-LABEL: test_mask_mullo_epi32_rmbkz_512
+  ;CHECK: vpmulld (%rdi){1to16}, %zmm0, %zmm0 {%k1} {z} ## encoding: [0x62,0xf2,0x7d,0xd9,0x40,0x07]
+  %q = load i32, i32* %ptr_b
+  %vecinit.i = insertelement <16 x i32> undef, i32 %q, i32 0
+  %b = shufflevector <16 x i32> %vecinit.i, <16 x i32> undef, <16 x i32> zeroinitializer
+  %res = call <16 x i32> @llvm.x86.avx512.mask.pmull.d.512(<16 x i32> %a, <16 x i32> %b, <16 x i32> zeroinitializer, i16 %mask)
+  ret < 16 x i32> %res
+}
+
+declare <16 x i32> @llvm.x86.avx512.mask.pmull.d.512(<16 x i32>, <16 x i32>, <16 x i32>, i16)
+
+define <16 x float> @test_mm512_maskz_add_round_ps_rn_sae(<16 x float> %a0, <16 x float> %a1, i16 %mask) {
+  ;CHECK-LABEL: test_mm512_maskz_add_round_ps_rn_sae
+  ;CHECK: vaddps {rn-sae}, %zmm1, %zmm0, %zmm0 {%k1} {z}
+  %res = call <16 x float> @llvm.x86.avx512.mask.add.ps.512(<16 x float> %a0, <16 x float> %a1, <16 x float>zeroinitializer, i16 %mask, i32 0)
+  ret <16 x float> %res
+}
+define <16 x float> @test_mm512_maskz_add_round_ps_rd_sae(<16 x float> %a0, <16 x float> %a1, i16 %mask) {
+  ;CHECK-LABEL: test_mm512_maskz_add_round_ps_rd_sae
+  ;CHECK: vaddps {rd-sae}, %zmm1, %zmm0, %zmm0
+  %res = call <16 x float> @llvm.x86.avx512.mask.add.ps.512(<16 x float> %a0, <16 x float> %a1, <16 x float>zeroinitializer, i16 %mask, i32 1)
+  ret <16 x float> %res
+}
+define <16 x float> @test_mm512_maskz_add_round_ps_ru_sae(<16 x float> %a0, <16 x float> %a1, i16 %mask) {
+  ;CHECK-LABEL: test_mm512_maskz_add_round_ps_ru_sae
+  ;CHECK: vaddps  {ru-sae}, %zmm1, %zmm0, %zmm0  {%k1} {z}
+  %res = call <16 x float> @llvm.x86.avx512.mask.add.ps.512(<16 x float> %a0, <16 x float> %a1, <16 x float>zeroinitializer, i16 %mask, i32 2)
+  ret <16 x float> %res
+}
+
+define <16 x float> @test_mm512_maskz_add_round_ps_rz_sae(<16 x float> %a0, <16 x float> %a1, i16 %mask) {
+  ;CHECK-LABEL: test_mm512_maskz_add_round_ps_rz_sae
+  ;CHECK: vaddps  {rz-sae}, %zmm1, %zmm0, %zmm0 {%k1} {z}
+  %res = call <16 x float> @llvm.x86.avx512.mask.add.ps.512(<16 x float> %a0, <16 x float> %a1, <16 x float>zeroinitializer, i16 %mask, i32 3)
+  ret <16 x float> %res
+}
+
+
+define <16 x float> @test_mm512_maskz_add_round_ps_current(<16 x float> %a0, <16 x float> %a1, i16 %mask) {
+  ;CHECK-LABEL: test_mm512_maskz_add_round_ps_current
+  ;CHECK: vaddps %zmm1, %zmm0, %zmm0 {%k1} {z}
+  %res = call <16 x float> @llvm.x86.avx512.mask.add.ps.512(<16 x float> %a0, <16 x float> %a1, <16 x float>zeroinitializer, i16 %mask, i32 4)
+  ret <16 x float> %res
+}
+
+define <16 x float> @test_mm512_mask_add_round_ps_rn_sae(<16 x float> %a0, <16 x float> %a1, <16 x float> %src, i16 %mask) {
+  ;CHECK-LABEL: test_mm512_mask_add_round_ps_rn_sae
+  ;CHECK: vaddps {rn-sae}, %zmm1, %zmm0, %zmm2 {%k1} 
+  %res = call <16 x float> @llvm.x86.avx512.mask.add.ps.512(<16 x float> %a0, <16 x float> %a1, <16 x float> %src, i16 %mask, i32 0)
+  ret <16 x float> %res
+}
+define <16 x float> @test_mm512_mask_add_round_ps_rd_sae(<16 x float> %a0, <16 x float> %a1, <16 x float> %src, i16 %mask) {
+  ;CHECK-LABEL: test_mm512_mask_add_round_ps_rd_sae
+  ;CHECK: vaddps {rd-sae}, %zmm1, %zmm0, %zmm2 {%k1} 
+  %res = call <16 x float> @llvm.x86.avx512.mask.add.ps.512(<16 x float> %a0, <16 x float> %a1, <16 x float> %src, i16 %mask, i32 1)
+  ret <16 x float> %res
+}
+define <16 x float> @test_mm512_mask_add_round_ps_ru_sae(<16 x float> %a0, <16 x float> %a1, <16 x float> %src, i16 %mask) {
+  ;CHECK-LABEL: test_mm512_mask_add_round_ps_ru_sae
+  ;CHECK: vaddps  {ru-sae}, %zmm1, %zmm0, %zmm2 {%k1} 
+  %res = call <16 x float> @llvm.x86.avx512.mask.add.ps.512(<16 x float> %a0, <16 x float> %a1, <16 x float> %src, i16 %mask, i32 2)
+  ret <16 x float> %res
+}
+
+define <16 x float> @test_mm512_mask_add_round_ps_rz_sae(<16 x float> %a0, <16 x float> %a1, <16 x float> %src, i16 %mask) {
+  ;CHECK-LABEL: test_mm512_mask_add_round_ps_rz_sae
+  ;CHECK: vaddps  {rz-sae}, %zmm1, %zmm0, %zmm2 {%k1} 
+  %res = call <16 x float> @llvm.x86.avx512.mask.add.ps.512(<16 x float> %a0, <16 x float> %a1, <16 x float> %src, i16 %mask, i32 3)
+  ret <16 x float> %res
+}
+
+
+define <16 x float> @test_mm512_mask_add_round_ps_current(<16 x float> %a0, <16 x float> %a1, <16 x float> %src, i16 %mask) {
+  ;CHECK-LABEL: test_mm512_mask_add_round_ps_current
+  ;CHECK: vaddps %zmm1, %zmm0, %zmm2 {%k1} 
+  %res = call <16 x float> @llvm.x86.avx512.mask.add.ps.512(<16 x float> %a0, <16 x float> %a1, <16 x float> %src, i16 %mask, i32 4)
+  ret <16 x float> %res
+}
+
+
+define <16 x float> @test_mm512_add_round_ps_rn_sae(<16 x float> %a0, <16 x float> %a1, i16 %mask) {
+  ;CHECK-LABEL: test_mm512_add_round_ps_rn_sae
+  ;CHECK: vaddps {rn-sae}, %zmm1, %zmm0, %zmm0
+  %res = call <16 x float> @llvm.x86.avx512.mask.add.ps.512(<16 x float> %a0, <16 x float> %a1, <16 x float>zeroinitializer, i16 -1, i32 0)
+  ret <16 x float> %res
+}
+define <16 x float> @test_mm512_add_round_ps_rd_sae(<16 x float> %a0, <16 x float> %a1, i16 %mask) {
+  ;CHECK-LABEL: test_mm512_add_round_ps_rd_sae
+  ;CHECK: vaddps {rd-sae}, %zmm1, %zmm0, %zmm0
+  %res = call <16 x float> @llvm.x86.avx512.mask.add.ps.512(<16 x float> %a0, <16 x float> %a1, <16 x float>zeroinitializer, i16 -1, i32 1)
+  ret <16 x float> %res
+}
+define <16 x float> @test_mm512_add_round_ps_ru_sae(<16 x float> %a0, <16 x float> %a1, i16 %mask) {
+  ;CHECK-LABEL: test_mm512_add_round_ps_ru_sae
+  ;CHECK: vaddps  {ru-sae}, %zmm1, %zmm0, %zmm0
+  %res = call <16 x float> @llvm.x86.avx512.mask.add.ps.512(<16 x float> %a0, <16 x float> %a1, <16 x float>zeroinitializer, i16 -1, i32 2)
+  ret <16 x float> %res
+}
+
+define <16 x float> @test_mm512_add_round_ps_rz_sae(<16 x float> %a0, <16 x float> %a1, i16 %mask) {
+  ;CHECK-LABEL: test_mm512_add_round_ps_rz_sae
+  ;CHECK: vaddps  {rz-sae}, %zmm1, %zmm0, %zmm0
+  %res = call <16 x float> @llvm.x86.avx512.mask.add.ps.512(<16 x float> %a0, <16 x float> %a1, <16 x float>zeroinitializer, i16 -1, i32 3)
+  ret <16 x float> %res
+}
+
+define <16 x float> @test_mm512_add_round_ps_current(<16 x float> %a0, <16 x float> %a1, i16 %mask) {
+  ;CHECK-LABEL: test_mm512_add_round_ps_current
+  ;CHECK: vaddps %zmm1, %zmm0, %zmm0
+  %res = call <16 x float> @llvm.x86.avx512.mask.add.ps.512(<16 x float> %a0, <16 x float> %a1, <16 x float>zeroinitializer, i16 -1, i32 4)
+  ret <16 x float> %res
+}
+declare <16 x float> @llvm.x86.avx512.mask.add.ps.512(<16 x float>, <16 x float>, <16 x float>, i16, i32)
+
+define <16 x float> @test_mm512_mask_sub_round_ps_rn_sae(<16 x float> %a0, <16 x float> %a1, <16 x float> %src, i16 %mask) {
+  ;CHECK-LABEL: test_mm512_mask_sub_round_ps_rn_sae
+  ;CHECK: vsubps {rn-sae}, %zmm1, %zmm0, %zmm2 {%k1} 
+  %res = call <16 x float> @llvm.x86.avx512.mask.sub.ps.512(<16 x float> %a0, <16 x float> %a1, <16 x float> %src, i16 %mask, i32 0)
+  ret <16 x float> %res
+}
+define <16 x float> @test_mm512_mask_sub_round_ps_rd_sae(<16 x float> %a0, <16 x float> %a1, <16 x float> %src, i16 %mask) {
+  ;CHECK-LABEL: test_mm512_mask_sub_round_ps_rd_sae
+  ;CHECK: vsubps {rd-sae}, %zmm1, %zmm0, %zmm2 {%k1} 
+  %res = call <16 x float> @llvm.x86.avx512.mask.sub.ps.512(<16 x float> %a0, <16 x float> %a1, <16 x float> %src, i16 %mask, i32 1)
+  ret <16 x float> %res
+}
+define <16 x float> @test_mm512_mask_sub_round_ps_ru_sae(<16 x float> %a0, <16 x float> %a1, <16 x float> %src, i16 %mask) {
+  ;CHECK-LABEL: test_mm512_mask_sub_round_ps_ru_sae
+  ;CHECK: vsubps  {ru-sae}, %zmm1, %zmm0, %zmm2 {%k1} 
+  %res = call <16 x float> @llvm.x86.avx512.mask.sub.ps.512(<16 x float> %a0, <16 x float> %a1, <16 x float> %src, i16 %mask, i32 2)
+  ret <16 x float> %res
+}
+
+define <16 x float> @test_mm512_mask_sub_round_ps_rz_sae(<16 x float> %a0, <16 x float> %a1, <16 x float> %src, i16 %mask) {
+  ;CHECK-LABEL: test_mm512_mask_sub_round_ps_rz_sae
+  ;CHECK: vsubps  {rz-sae}, %zmm1, %zmm0, %zmm2 {%k1} 
+  %res = call <16 x float> @llvm.x86.avx512.mask.sub.ps.512(<16 x float> %a0, <16 x float> %a1, <16 x float> %src, i16 %mask, i32 3)
+  ret <16 x float> %res
+}
+
+
+define <16 x float> @test_mm512_mask_sub_round_ps_current(<16 x float> %a0, <16 x float> %a1, <16 x float> %src, i16 %mask) {
+  ;CHECK-LABEL: test_mm512_mask_sub_round_ps_current
+  ;CHECK: vsubps %zmm1, %zmm0, %zmm2 {%k1} 
+  %res = call <16 x float> @llvm.x86.avx512.mask.sub.ps.512(<16 x float> %a0, <16 x float> %a1, <16 x float> %src, i16 %mask, i32 4)
+  ret <16 x float> %res
+}
+
+define <16 x float> @test_mm512_sub_round_ps_rn_sae(<16 x float> %a0, <16 x float> %a1, i16 %mask) {
+  ;CHECK-LABEL: test_mm512_sub_round_ps_rn_sae
+  ;CHECK: vsubps {rn-sae}, %zmm1, %zmm0, %zmm0
+  %res = call <16 x float> @llvm.x86.avx512.mask.sub.ps.512(<16 x float> %a0, <16 x float> %a1, <16 x float>zeroinitializer, i16 -1, i32 0)
+  ret <16 x float> %res
+}
+define <16 x float> @test_mm512_sub_round_ps_rd_sae(<16 x float> %a0, <16 x float> %a1, i16 %mask) {
+  ;CHECK-LABEL: test_mm512_sub_round_ps_rd_sae
+  ;CHECK: vsubps {rd-sae}, %zmm1, %zmm0, %zmm0
+  %res = call <16 x float> @llvm.x86.avx512.mask.sub.ps.512(<16 x float> %a0, <16 x float> %a1, <16 x float>zeroinitializer, i16 -1, i32 1)
+  ret <16 x float> %res
+}
+define <16 x float> @test_mm512_sub_round_ps_ru_sae(<16 x float> %a0, <16 x float> %a1, i16 %mask) {
+  ;CHECK-LABEL: test_mm512_sub_round_ps_ru_sae
+  ;CHECK: vsubps  {ru-sae}, %zmm1, %zmm0, %zmm0
+  %res = call <16 x float> @llvm.x86.avx512.mask.sub.ps.512(<16 x float> %a0, <16 x float> %a1, <16 x float>zeroinitializer, i16 -1, i32 2)
+  ret <16 x float> %res
+}
+
+define <16 x float> @test_mm512_sub_round_ps_rz_sae(<16 x float> %a0, <16 x float> %a1, i16 %mask) {
+  ;CHECK-LABEL: test_mm512_sub_round_ps_rz_sae
+  ;CHECK: vsubps  {rz-sae}, %zmm1, %zmm0, %zmm0
+  %res = call <16 x float> @llvm.x86.avx512.mask.sub.ps.512(<16 x float> %a0, <16 x float> %a1, <16 x float>zeroinitializer, i16 -1, i32 3)
+  ret <16 x float> %res
+}
+
+define <16 x float> @test_mm512_sub_round_ps_current(<16 x float> %a0, <16 x float> %a1, i16 %mask) {
+  ;CHECK-LABEL: test_mm512_sub_round_ps_current
+  ;CHECK: vsubps %zmm1, %zmm0, %zmm0
+  %res = call <16 x float> @llvm.x86.avx512.mask.sub.ps.512(<16 x float> %a0, <16 x float> %a1, <16 x float>zeroinitializer, i16 -1, i32 4)
+  ret <16 x float> %res
+}
+
+define <16 x float> @test_mm512_maskz_div_round_ps_rn_sae(<16 x float> %a0, <16 x float> %a1, i16 %mask) {
+  ;CHECK-LABEL: test_mm512_maskz_div_round_ps_rn_sae
+  ;CHECK: vdivps {rn-sae}, %zmm1, %zmm0, %zmm0 {%k1} {z}
+  %res = call <16 x float> @llvm.x86.avx512.mask.div.ps.512(<16 x float> %a0, <16 x float> %a1, <16 x float>zeroinitializer, i16 %mask, i32 0)
+  ret <16 x float> %res
+}
+define <16 x float> @test_mm512_maskz_div_round_ps_rd_sae(<16 x float> %a0, <16 x float> %a1, i16 %mask) {
+  ;CHECK-LABEL: test_mm512_maskz_div_round_ps_rd_sae
+  ;CHECK: vdivps {rd-sae}, %zmm1, %zmm0, %zmm0
+  %res = call <16 x float> @llvm.x86.avx512.mask.div.ps.512(<16 x float> %a0, <16 x float> %a1, <16 x float>zeroinitializer, i16 %mask, i32 1)
+  ret <16 x float> %res
+}
+define <16 x float> @test_mm512_maskz_div_round_ps_ru_sae(<16 x float> %a0, <16 x float> %a1, i16 %mask) {
+  ;CHECK-LABEL: test_mm512_maskz_div_round_ps_ru_sae
+  ;CHECK: vdivps  {ru-sae}, %zmm1, %zmm0, %zmm0  {%k1} {z}
+  %res = call <16 x float> @llvm.x86.avx512.mask.div.ps.512(<16 x float> %a0, <16 x float> %a1, <16 x float>zeroinitializer, i16 %mask, i32 2)
+  ret <16 x float> %res
+}
+
+define <16 x float> @test_mm512_maskz_div_round_ps_rz_sae(<16 x float> %a0, <16 x float> %a1, i16 %mask) {
+  ;CHECK-LABEL: test_mm512_maskz_div_round_ps_rz_sae
+  ;CHECK: vdivps  {rz-sae}, %zmm1, %zmm0, %zmm0 {%k1} {z}
+  %res = call <16 x float> @llvm.x86.avx512.mask.div.ps.512(<16 x float> %a0, <16 x float> %a1, <16 x float>zeroinitializer, i16 %mask, i32 3)
+  ret <16 x float> %res
+}
+
+
+define <16 x float> @test_mm512_maskz_div_round_ps_current(<16 x float> %a0, <16 x float> %a1, i16 %mask) {
+  ;CHECK-LABEL: test_mm512_maskz_div_round_ps_current
+  ;CHECK: vdivps %zmm1, %zmm0, %zmm0 {%k1} {z}
+  %res = call <16 x float> @llvm.x86.avx512.mask.div.ps.512(<16 x float> %a0, <16 x float> %a1, <16 x float>zeroinitializer, i16 %mask, i32 4)
+  ret <16 x float> %res
+}
+
+define <16 x float> @test_mm512_mask_div_round_ps_rn_sae(<16 x float> %a0, <16 x float> %a1, <16 x float> %src, i16 %mask) {
+  ;CHECK-LABEL: test_mm512_mask_div_round_ps_rn_sae
+  ;CHECK: vdivps {rn-sae}, %zmm1, %zmm0, %zmm2 {%k1} 
+  %res = call <16 x float> @llvm.x86.avx512.mask.div.ps.512(<16 x float> %a0, <16 x float> %a1, <16 x float> %src, i16 %mask, i32 0)
+  ret <16 x float> %res
+}
+define <16 x float> @test_mm512_mask_div_round_ps_rd_sae(<16 x float> %a0, <16 x float> %a1, <16 x float> %src, i16 %mask) {
+  ;CHECK-LABEL: test_mm512_mask_div_round_ps_rd_sae
+  ;CHECK: vdivps {rd-sae}, %zmm1, %zmm0, %zmm2 {%k1} 
+  %res = call <16 x float> @llvm.x86.avx512.mask.div.ps.512(<16 x float> %a0, <16 x float> %a1, <16 x float> %src, i16 %mask, i32 1)
+  ret <16 x float> %res
+}
+define <16 x float> @test_mm512_mask_div_round_ps_ru_sae(<16 x float> %a0, <16 x float> %a1, <16 x float> %src, i16 %mask) {
+  ;CHECK-LABEL: test_mm512_mask_div_round_ps_ru_sae
+  ;CHECK: vdivps  {ru-sae}, %zmm1, %zmm0, %zmm2 {%k1} 
+  %res = call <16 x float> @llvm.x86.avx512.mask.div.ps.512(<16 x float> %a0, <16 x float> %a1, <16 x float> %src, i16 %mask, i32 2)
+  ret <16 x float> %res
+}
+
+define <16 x float> @test_mm512_mask_div_round_ps_rz_sae(<16 x float> %a0, <16 x float> %a1, <16 x float> %src, i16 %mask) {
+  ;CHECK-LABEL: test_mm512_mask_div_round_ps_rz_sae
+  ;CHECK: vdivps  {rz-sae}, %zmm1, %zmm0, %zmm2 {%k1} 
+  %res = call <16 x float> @llvm.x86.avx512.mask.div.ps.512(<16 x float> %a0, <16 x float> %a1, <16 x float> %src, i16 %mask, i32 3)
+  ret <16 x float> %res
+}
+
+
+define <16 x float> @test_mm512_mask_div_round_ps_current(<16 x float> %a0, <16 x float> %a1, <16 x float> %src, i16 %mask) {
+  ;CHECK-LABEL: test_mm512_mask_div_round_ps_current
+  ;CHECK: vdivps %zmm1, %zmm0, %zmm2 {%k1} 
+  %res = call <16 x float> @llvm.x86.avx512.mask.div.ps.512(<16 x float> %a0, <16 x float> %a1, <16 x float> %src, i16 %mask, i32 4)
+  ret <16 x float> %res
+}
+
+
+define <16 x float> @test_mm512_div_round_ps_rn_sae(<16 x float> %a0, <16 x float> %a1, i16 %mask) {
+  ;CHECK-LABEL: test_mm512_div_round_ps_rn_sae
+  ;CHECK: vdivps {rn-sae}, %zmm1, %zmm0, %zmm0
+  %res = call <16 x float> @llvm.x86.avx512.mask.div.ps.512(<16 x float> %a0, <16 x float> %a1, <16 x float>zeroinitializer, i16 -1, i32 0)
+  ret <16 x float> %res
+}
+define <16 x float> @test_mm512_div_round_ps_rd_sae(<16 x float> %a0, <16 x float> %a1, i16 %mask) {
+  ;CHECK-LABEL: test_mm512_div_round_ps_rd_sae
+  ;CHECK: vdivps {rd-sae}, %zmm1, %zmm0, %zmm0
+  %res = call <16 x float> @llvm.x86.avx512.mask.div.ps.512(<16 x float> %a0, <16 x float> %a1, <16 x float>zeroinitializer, i16 -1, i32 1)
+  ret <16 x float> %res
+}
+define <16 x float> @test_mm512_div_round_ps_ru_sae(<16 x float> %a0, <16 x float> %a1, i16 %mask) {
+  ;CHECK-LABEL: test_mm512_div_round_ps_ru_sae
+  ;CHECK: vdivps  {ru-sae}, %zmm1, %zmm0, %zmm0
+  %res = call <16 x float> @llvm.x86.avx512.mask.div.ps.512(<16 x float> %a0, <16 x float> %a1, <16 x float>zeroinitializer, i16 -1, i32 2)
+  ret <16 x float> %res
+}
+
+define <16 x float> @test_mm512_div_round_ps_rz_sae(<16 x float> %a0, <16 x float> %a1, i16 %mask) {
+  ;CHECK-LABEL: test_mm512_div_round_ps_rz_sae
+  ;CHECK: vdivps  {rz-sae}, %zmm1, %zmm0, %zmm0
+  %res = call <16 x float> @llvm.x86.avx512.mask.div.ps.512(<16 x float> %a0, <16 x float> %a1, <16 x float>zeroinitializer, i16 -1, i32 3)
+  ret <16 x float> %res
+}
+
+define <16 x float> @test_mm512_div_round_ps_current(<16 x float> %a0, <16 x float> %a1, i16 %mask) {
+  ;CHECK-LABEL: test_mm512_div_round_ps_current
+  ;CHECK: vdivps %zmm1, %zmm0, %zmm0
+  %res = call <16 x float> @llvm.x86.avx512.mask.div.ps.512(<16 x float> %a0, <16 x float> %a1, <16 x float>zeroinitializer, i16 -1, i32 4)
+  ret <16 x float> %res
+}
+declare <16 x float> @llvm.x86.avx512.mask.div.ps.512(<16 x float>, <16 x float>, <16 x float>, i16, i32)
+
+define <16 x float> @test_mm512_maskz_min_round_ps_sae(<16 x float> %a0, <16 x float> %a1, i16 %mask) {
+  ;CHECK-LABEL: test_mm512_maskz_min_round_ps_sae
+  ;CHECK: vminps {sae}, %zmm1, %zmm0, %zmm0 {%k1} {z}
+  %res = call <16 x float> @llvm.x86.avx512.mask.min.ps.512(<16 x float> %a0, <16 x float> %a1, <16 x float>zeroinitializer, i16 %mask, i32 8)
+  ret <16 x float> %res
+}
+
+define <16 x float> @test_mm512_maskz_min_round_ps_current(<16 x float> %a0, <16 x float> %a1, i16 %mask) {
+  ;CHECK-LABEL: test_mm512_maskz_min_round_ps_current
+  ;CHECK: vminps %zmm1, %zmm0, %zmm0 {%k1} {z}
+  %res = call <16 x float> @llvm.x86.avx512.mask.min.ps.512(<16 x float> %a0, <16 x float> %a1, <16 x float>zeroinitializer, i16 %mask, i32 4)
+  ret <16 x float> %res
+}
+
+define <16 x float> @test_mm512_mask_min_round_ps_sae(<16 x float> %a0, <16 x float> %a1, <16 x float> %src, i16 %mask) {
+  ;CHECK-LABEL: test_mm512_mask_min_round_ps_sae
+  ;CHECK: vminps {sae}, %zmm1, %zmm0, %zmm2 {%k1}
+  %res = call <16 x float> @llvm.x86.avx512.mask.min.ps.512(<16 x float> %a0, <16 x float> %a1, <16 x float> %src, i16 %mask, i32 8)
+  ret <16 x float> %res
+}
+
+define <16 x float> @test_mm512_mask_min_round_ps_current(<16 x float> %a0, <16 x float> %a1, <16 x float> %src, i16 %mask) {
+  ;CHECK-LABEL: test_mm512_mask_min_round_ps_current
+  ;CHECK: vminps %zmm1, %zmm0, %zmm2 {%k1} 
+  %res = call <16 x float> @llvm.x86.avx512.mask.min.ps.512(<16 x float> %a0, <16 x float> %a1, <16 x float> %src, i16 %mask, i32 4)
+  ret <16 x float> %res
+}
+
+define <16 x float> @test_mm512_min_round_ps_sae(<16 x float> %a0, <16 x float> %a1, i16 %mask) {
+  ;CHECK-LABEL: test_mm512_min_round_ps_sae
+  ;CHECK: vminps {sae}, %zmm1, %zmm0, %zmm0
+  %res = call <16 x float> @llvm.x86.avx512.mask.min.ps.512(<16 x float> %a0, <16 x float> %a1, <16 x float>zeroinitializer, i16 -1, i32 8)
+  ret <16 x float> %res
+}
+
+define <16 x float> @test_mm512_min_round_ps_current(<16 x float> %a0, <16 x float> %a1, i16 %mask) {
+  ;CHECK-LABEL: test_mm512_min_round_ps_current
+  ;CHECK: vminps %zmm1, %zmm0, %zmm0
+  %res = call <16 x float> @llvm.x86.avx512.mask.min.ps.512(<16 x float> %a0, <16 x float> %a1, <16 x float>zeroinitializer, i16 -1, i32 4)
+  ret <16 x float> %res
+}
+declare <16 x float> @llvm.x86.avx512.mask.min.ps.512(<16 x float>, <16 x float>, <16 x float>, i16, i32)
+
+define <16 x float> @test_mm512_maskz_max_round_ps_sae(<16 x float> %a0, <16 x float> %a1, i16 %mask) {
+  ;CHECK-LABEL: test_mm512_maskz_max_round_ps_sae
+  ;CHECK: vmaxps {sae}, %zmm1, %zmm0, %zmm0 {%k1} {z}
+  %res = call <16 x float> @llvm.x86.avx512.mask.max.ps.512(<16 x float> %a0, <16 x float> %a1, <16 x float>zeroinitializer, i16 %mask, i32 8)
+  ret <16 x float> %res
+}
+
+define <16 x float> @test_mm512_maskz_max_round_ps_current(<16 x float> %a0, <16 x float> %a1, i16 %mask) {
+  ;CHECK-LABEL: test_mm512_maskz_max_round_ps_current
+  ;CHECK: vmaxps %zmm1, %zmm0, %zmm0 {%k1} {z}
+  %res = call <16 x float> @llvm.x86.avx512.mask.max.ps.512(<16 x float> %a0, <16 x float> %a1, <16 x float>zeroinitializer, i16 %mask, i32 4)
+  ret <16 x float> %res
+}
+
+define <16 x float> @test_mm512_mask_max_round_ps_sae(<16 x float> %a0, <16 x float> %a1, <16 x float> %src, i16 %mask) {
+  ;CHECK-LABEL: test_mm512_mask_max_round_ps_sae
+  ;CHECK: vmaxps {sae}, %zmm1, %zmm0, %zmm2 {%k1}
+  %res = call <16 x float> @llvm.x86.avx512.mask.max.ps.512(<16 x float> %a0, <16 x float> %a1, <16 x float> %src, i16 %mask, i32 8)
+  ret <16 x float> %res
+}
+
+define <16 x float> @test_mm512_mask_max_round_ps_current(<16 x float> %a0, <16 x float> %a1, <16 x float> %src, i16 %mask) {
+  ;CHECK-LABEL: test_mm512_mask_max_round_ps_current
+  ;CHECK: vmaxps %zmm1, %zmm0, %zmm2 {%k1} 
+  %res = call <16 x float> @llvm.x86.avx512.mask.max.ps.512(<16 x float> %a0, <16 x float> %a1, <16 x float> %src, i16 %mask, i32 4)
+  ret <16 x float> %res
+}
+
+define <16 x float> @test_mm512_max_round_ps_sae(<16 x float> %a0, <16 x float> %a1, i16 %mask) {
+  ;CHECK-LABEL: test_mm512_max_round_ps_sae
+  ;CHECK: vmaxps {sae}, %zmm1, %zmm0, %zmm0 
+  %res = call <16 x float> @llvm.x86.avx512.mask.max.ps.512(<16 x float> %a0, <16 x float> %a1, <16 x float>zeroinitializer, i16 -1, i32 8)
+  ret <16 x float> %res
+}
+
+define <16 x float> @test_mm512_max_round_ps_current(<16 x float> %a0, <16 x float> %a1, i16 %mask) {
+  ;CHECK-LABEL: test_mm512_max_round_ps_current
+  ;CHECK: vmaxps %zmm1, %zmm0, %zmm0
+  %res = call <16 x float> @llvm.x86.avx512.mask.max.ps.512(<16 x float> %a0, <16 x float> %a1, <16 x float>zeroinitializer, i16 -1, i32 4)
+  ret <16 x float> %res
+}
+declare <16 x float> @llvm.x86.avx512.mask.max.ps.512(<16 x float>, <16 x float>, <16 x float>, i16, i32)
+
+declare <4 x float> @llvm.x86.avx512.mask.add.ss.round(<4 x float>, <4 x float>, <4 x float>, i8, i32) nounwind readnone
+
+define <4 x float> @test_mask_add_ss_rn(<4 x float> %a0, <4 x float> %a1, <4 x float> %a2, i8 %mask) {
+; CHECK-LABEL: test_mask_add_ss_rn
+; CHECK: vaddss  {rn-sae}, %xmm1, %xmm0, %xmm2 {%k1}
+  %res = call <4 x float> @llvm.x86.avx512.mask.add.ss.round(<4 x float>%a0, <4 x float> %a1, <4 x float> %a2, i8 %mask, i32 0)
+  ret <4 x float> %res
+}
+
+define <4 x float> @test_mask_add_ss_rd(<4 x float> %a0, <4 x float> %a1, <4 x float> %a2, i8 %mask) {
+; CHECK-LABEL: test_mask_add_ss_rd
+; CHECK: vaddss  {rd-sae}, %xmm1, %xmm0, %xmm2 {%k1}
+  %res = call <4 x float> @llvm.x86.avx512.mask.add.ss.round(<4 x float>%a0, <4 x float> %a1, <4 x float> %a2, i8 %mask, i32 1)
+  ret <4 x float> %res
+}
+
+define <4 x float> @test_mask_add_ss_ru(<4 x float> %a0, <4 x float> %a1, <4 x float> %a2, i8 %mask) {
+; CHECK-LABEL: test_mask_add_ss_ru
+; CHECK: vaddss  {ru-sae}, %xmm1, %xmm0, %xmm2 {%k1}
+  %res = call <4 x float> @llvm.x86.avx512.mask.add.ss.round(<4 x float>%a0, <4 x float> %a1, <4 x float> %a2, i8 %mask, i32 2)
+  ret <4 x float> %res
+}
+
+define <4 x float> @test_mask_add_ss_rz(<4 x float> %a0, <4 x float> %a1, <4 x float> %a2, i8 %mask) {
+; CHECK-LABEL: test_mask_add_ss_rz
+; CHECK: vaddss  {rz-sae}, %xmm1, %xmm0, %xmm2 {%k1}
+  %res = call <4 x float> @llvm.x86.avx512.mask.add.ss.round(<4 x float>%a0, <4 x float> %a1, <4 x float> %a2, i8 %mask, i32 3)
+  ret <4 x float> %res
+}
+
+define <4 x float> @test_mask_add_ss_current(<4 x float> %a0, <4 x float> %a1, <4 x float> %a2, i8 %mask) {
+; CHECK-LABEL: test_mask_add_ss_current
+; CHECK: vaddss %xmm1, %xmm0, %xmm2 {%k1}
+  %res = call <4 x float> @llvm.x86.avx512.mask.add.ss.round(<4 x float>%a0, <4 x float> %a1, <4 x float> %a2, i8 %mask, i32 4)
+  ret <4 x float> %res
+}
+
+define <4 x float> @test_maskz_add_ss_rn(<4 x float> %a0, <4 x float> %a1, i8 %mask) {
+; CHECK-LABEL: test_maskz_add_ss_rn
+; CHECK: vaddss  {rn-sae}, %xmm1, %xmm0, %xmm0 {%k1} {z}
+  %res = call <4 x float> @llvm.x86.avx512.mask.add.ss.round(<4 x float>%a0, <4 x float> %a1, <4 x float> zeroinitializer, i8 %mask, i32 0)
+  ret <4 x float> %res
+}
+
+define <4 x float> @test_add_ss_rn(<4 x float> %a0, <4 x float> %a1) {
+; CHECK-LABEL: test_add_ss_rn
+; CHECK: vaddss  {rn-sae}, %xmm1, %xmm0, %xmm0
+  %res = call <4 x float> @llvm.x86.avx512.mask.add.ss.round(<4 x float>%a0, <4 x float> %a1, <4 x float> zeroinitializer, i8 -1, i32 0)
+  ret <4 x float> %res
+}
+
+declare <2 x double> @llvm.x86.avx512.mask.add.sd.round(<2 x double>, <2 x double>, <2 x double>, i8, i32) nounwind readnone
+
+define <2 x double> @test_mask_add_sd_rn(<2 x double> %a0, <2 x double> %a1, <2 x double> %a2, i8 %mask) {
+; CHECK-LABEL: test_mask_add_sd_rn
+; CHECK: vaddsd  {rn-sae}, %xmm1, %xmm0, %xmm2 {%k1}
+  %res = call <2 x double> @llvm.x86.avx512.mask.add.sd.round(<2 x double>%a0, <2 x double> %a1, <2 x double> %a2, i8 %mask, i32 0)
+  ret <2 x double> %res
+}
+
+define <2 x double> @test_mask_add_sd_rd(<2 x double> %a0, <2 x double> %a1, <2 x double> %a2, i8 %mask) {
+; CHECK-LABEL: test_mask_add_sd_rd
+; CHECK: vaddsd  {rd-sae}, %xmm1, %xmm0, %xmm2 {%k1}
+  %res = call <2 x double> @llvm.x86.avx512.mask.add.sd.round(<2 x double>%a0, <2 x double> %a1, <2 x double> %a2, i8 %mask, i32 1)
+  ret <2 x double> %res
+}
+
+define <2 x double> @test_mask_add_sd_ru(<2 x double> %a0, <2 x double> %a1, <2 x double> %a2, i8 %mask) {
+; CHECK-LABEL: test_mask_add_sd_ru
+; CHECK: vaddsd  {ru-sae}, %xmm1, %xmm0, %xmm2 {%k1}
+  %res = call <2 x double> @llvm.x86.avx512.mask.add.sd.round(<2 x double>%a0, <2 x double> %a1, <2 x double> %a2, i8 %mask, i32 2)
+  ret <2 x double> %res
+}
+
+define <2 x double> @test_mask_add_sd_rz(<2 x double> %a0, <2 x double> %a1, <2 x double> %a2, i8 %mask) {
+; CHECK-LABEL: test_mask_add_sd_rz
+; CHECK: vaddsd  {rz-sae}, %xmm1, %xmm0, %xmm2 {%k1}
+  %res = call <2 x double> @llvm.x86.avx512.mask.add.sd.round(<2 x double>%a0, <2 x double> %a1, <2 x double> %a2, i8 %mask, i32 3)
+  ret <2 x double> %res
+}
+
+define <2 x double> @test_mask_add_sd_current(<2 x double> %a0, <2 x double> %a1, <2 x double> %a2, i8 %mask) {
+; CHECK-LABEL: test_mask_add_sd_current
+; CHECK: vaddsd %xmm1, %xmm0, %xmm2 {%k1}
+  %res = call <2 x double> @llvm.x86.avx512.mask.add.sd.round(<2 x double>%a0, <2 x double> %a1, <2 x double> %a2, i8 %mask, i32 4)
+  ret <2 x double> %res
+}
+
+define <2 x double> @test_maskz_add_sd_rn(<2 x double> %a0, <2 x double> %a1, i8 %mask) {
+; CHECK-LABEL: test_maskz_add_sd_rn
+; CHECK: vaddsd  {rn-sae}, %xmm1, %xmm0, %xmm0 {%k1} {z}
+  %res = call <2 x double> @llvm.x86.avx512.mask.add.sd.round(<2 x double>%a0, <2 x double> %a1, <2 x double> zeroinitializer, i8 %mask, i32 0)
+  ret <2 x double> %res
+}
+
+define <2 x double> @test_add_sd_rn(<2 x double> %a0, <2 x double> %a1) {
+; CHECK-LABEL: test_add_sd_rn
+; CHECK: vaddsd  {rn-sae}, %xmm1, %xmm0, %xmm0
+  %res = call <2 x double> @llvm.x86.avx512.mask.add.sd.round(<2 x double>%a0, <2 x double> %a1, <2 x double> zeroinitializer, i8 -1, i32 0)
+  ret <2 x double> %res
+}
+
+declare <4 x float> @llvm.x86.avx512.mask.max.ss.round(<4 x float>, <4 x float>, <4 x float>, i8, i32) nounwind readnone
+
+define <4 x float> @test_mask_max_ss_sae(<4 x float> %a0, <4 x float> %a1, <4 x float> %a2, i8 %mask) {
+; CHECK-LABEL: test_mask_max_ss_sae
+; CHECK: vmaxss  {sae}, %xmm1, %xmm0, %xmm2 {%k1}
+  %res = call <4 x float> @llvm.x86.avx512.mask.max.ss.round(<4 x float>%a0, <4 x float> %a1, <4 x float> %a2, i8 %mask, i32 8)
+  ret <4 x float> %res
+}
+
+define <4 x float> @test_maskz_max_ss_sae(<4 x float> %a0, <4 x float> %a1, i8 %mask) {
+; CHECK-LABEL: test_maskz_max_ss_sae
+; CHECK: vmaxss  {sae}, %xmm1, %xmm0, %xmm0 {%k1} {z}
+  %res = call <4 x float> @llvm.x86.avx512.mask.max.ss.round(<4 x float>%a0, <4 x float> %a1, <4 x float> zeroinitializer, i8 %mask, i32 8)
+  ret <4 x float> %res
+}
+
+define <4 x float> @test_max_ss_sae(<4 x float> %a0, <4 x float> %a1) {
+; CHECK-LABEL: test_max_ss_sae
+; CHECK: vmaxss  {sae}, %xmm1, %xmm0, %xmm0
+  %res = call <4 x float> @llvm.x86.avx512.mask.max.ss.round(<4 x float>%a0, <4 x float> %a1, <4 x float> zeroinitializer, i8 -1, i32 8)
+  ret <4 x float> %res
+}
+
+define <4 x float> @test_mask_max_ss(<4 x float> %a0, <4 x float> %a1, <4 x float> %a2, i8 %mask) {
+; CHECK-LABEL: test_mask_max_ss
+; CHECK: vmaxss  %xmm1, %xmm0, %xmm2 {%k1}
+  %res = call <4 x float> @llvm.x86.avx512.mask.max.ss.round(<4 x float>%a0, <4 x float> %a1, <4 x float> %a2, i8 %mask, i32 4)
+  ret <4 x float> %res
+}
+
+define <4 x float> @test_maskz_max_ss(<4 x float> %a0, <4 x float> %a1, i8 %mask) {
+; CHECK-LABEL: test_maskz_max_ss
+; CHECK: vmaxss  %xmm1, %xmm0, %xmm0 {%k1} {z}
+  %res = call <4 x float> @llvm.x86.avx512.mask.max.ss.round(<4 x float>%a0, <4 x float> %a1, <4 x float> zeroinitializer, i8 %mask, i32 4)
+  ret <4 x float> %res
+}
+
+define <4 x float> @test_max_ss(<4 x float> %a0, <4 x float> %a1) {
+; CHECK-LABEL: test_max_ss
+; CHECK: vmaxss  %xmm1, %xmm0, %xmm0
+  %res = call <4 x float> @llvm.x86.avx512.mask.max.ss.round(<4 x float>%a0, <4 x float> %a1, <4 x float> zeroinitializer, i8 -1, i32 4)
+  ret <4 x float> %res
+}
+declare <2 x double> @llvm.x86.avx512.mask.max.sd.round(<2 x double>, <2 x double>, <2 x double>, i8, i32) nounwind readnone
+
+define <2 x double> @test_mask_max_sd_sae(<2 x double> %a0, <2 x double> %a1, <2 x double> %a2, i8 %mask) {
+; CHECK-LABEL: test_mask_max_sd_sae
+; CHECK: vmaxsd  {sae}, %xmm1, %xmm0, %xmm2 {%k1}
+  %res = call <2 x double> @llvm.x86.avx512.mask.max.sd.round(<2 x double>%a0, <2 x double> %a1, <2 x double> %a2, i8 %mask, i32 8)
+  ret <2 x double> %res
+}
+
+define <2 x double> @test_maskz_max_sd_sae(<2 x double> %a0, <2 x double> %a1, i8 %mask) {
+; CHECK-LABEL: test_maskz_max_sd_sae
+; CHECK: vmaxsd  {sae}, %xmm1, %xmm0, %xmm0 {%k1} {z}
+  %res = call <2 x double> @llvm.x86.avx512.mask.max.sd.round(<2 x double>%a0, <2 x double> %a1, <2 x double> zeroinitializer, i8 %mask, i32 8)
+  ret <2 x double> %res
+}
+
+define <2 x double> @test_max_sd_sae(<2 x double> %a0, <2 x double> %a1) {
+; CHECK-LABEL: test_max_sd_sae
+; CHECK: vmaxsd  {sae}, %xmm1, %xmm0, %xmm0
+  %res = call <2 x double> @llvm.x86.avx512.mask.max.sd.round(<2 x double>%a0, <2 x double> %a1, <2 x double> zeroinitializer, i8 -1, i32 8)
+  ret <2 x double> %res
+}
+
+define <2 x double> @test_mask_max_sd(<2 x double> %a0, <2 x double> %a1, <2 x double> %a2, i8 %mask) {
+; CHECK-LABEL: test_mask_max_sd
+; CHECK: vmaxsd  %xmm1, %xmm0, %xmm2 {%k1}
+  %res = call <2 x double> @llvm.x86.avx512.mask.max.sd.round(<2 x double>%a0, <2 x double> %a1, <2 x double> %a2, i8 %mask, i32 4)
+  ret <2 x double> %res
+}
+
+define <2 x double> @test_maskz_max_sd(<2 x double> %a0, <2 x double> %a1, i8 %mask) {
+; CHECK-LABEL: test_maskz_max_sd
+; CHECK: vmaxsd  %xmm1, %xmm0, %xmm0 {%k1} {z}
+  %res = call <2 x double> @llvm.x86.avx512.mask.max.sd.round(<2 x double>%a0, <2 x double> %a1, <2 x double> zeroinitializer, i8 %mask, i32 4)
+  ret <2 x double> %res
+}
+
+define <2 x double> @test_max_sd(<2 x double> %a0, <2 x double> %a1) {
+; CHECK-LABEL: test_max_sd
+; CHECK: vmaxsd  %xmm1, %xmm0, %xmm0
+  %res = call <2 x double> @llvm.x86.avx512.mask.max.sd.round(<2 x double>%a0, <2 x double> %a1, <2 x double> zeroinitializer, i8 -1, i32 4)
+  ret <2 x double> %res
+}
+
+define <2 x double> @test_x86_avx512_cvtsi2sd32(<2 x double> %a, i32 %b) {
+; CHECK-LABEL: test_x86_avx512_cvtsi2sd32:
+; CHECK:       ## BB#0:
+; CHECK-NEXT:    vcvtsi2sdl %edi, {rz-sae}, %xmm0, %xmm0 
+; CHECK-NEXT:    retq 
+  %res = call <2 x double> @llvm.x86.avx512.cvtsi2sd32(<2 x double> %a, i32 %b, i32 3) ; <<<2 x double>> [#uses=1]
+  ret <2 x double> %res
+}
+declare <2 x double> @llvm.x86.avx512.cvtsi2sd32(<2 x double>, i32, i32) nounwind readnone
+
+define <2 x double> @test_x86_avx512_cvtsi2sd64(<2 x double> %a, i64 %b) {
+; CHECK-LABEL: test_x86_avx512_cvtsi2sd64:
+; CHECK:       ## BB#0:
+; CHECK-NEXT:    vcvtsi2sdq %rdi, {rz-sae}, %xmm0, %xmm0 
+; CHECK-NEXT:    retq 
+  %res = call <2 x double> @llvm.x86.avx512.cvtsi2sd64(<2 x double> %a, i64 %b, i32 3) ; <<<2 x double>> [#uses=1]
+  ret <2 x double> %res
+}
+declare <2 x double> @llvm.x86.avx512.cvtsi2sd64(<2 x double>, i64, i32) nounwind readnone
+
+define <4 x float> @test_x86_avx512_cvtsi2ss32(<4 x float> %a, i32 %b) {
+; CHECK-LABEL: test_x86_avx512_cvtsi2ss32:
+; CHECK:       ## BB#0:
+; CHECK-NEXT:    vcvtsi2ssl %edi, {rz-sae}, %xmm0, %xmm0 
+; CHECK-NEXT:    retq 
+  %res = call <4 x float> @llvm.x86.avx512.cvtsi2ss32(<4 x float> %a, i32 %b, i32 3) ; <<<4 x float>> [#uses=1]
+  ret <4 x float> %res
+}
+declare <4 x float> @llvm.x86.avx512.cvtsi2ss32(<4 x float>, i32, i32) nounwind readnone
+
+define <4 x float> @test_x86_avx512_cvtsi2ss64(<4 x float> %a, i64 %b) {
+; CHECK-LABEL: test_x86_avx512_cvtsi2ss64:
+; CHECK:       ## BB#0:
+; CHECK-NEXT:    vcvtsi2ssq %rdi, {rz-sae}, %xmm0, %xmm0 
+; CHECK-NEXT:    retq 
+  %res = call <4 x float> @llvm.x86.avx512.cvtsi2ss64(<4 x float> %a, i64 %b, i32 3) ; <<<4 x float>> [#uses=1]
+  ret <4 x float> %res
+}
+declare <4 x float> @llvm.x86.avx512.cvtsi2ss64(<4 x float>, i64, i32) nounwind readnone
+
+define <4 x float> @test_x86_avx512__mm_cvt_roundu32_ss (<4 x float> %a, i32 %b)
+; CHECK-LABEL: test_x86_avx512__mm_cvt_roundu32_ss:
+; CHECK:       ## BB#0:
+; CHECK-NEXT:    vcvtusi2ssl %edi, {rd-sae}, %xmm0, %xmm0 
+; CHECK-NEXT:    retq 
+{
+  %res = call <4 x float> @llvm.x86.avx512.cvtusi2ss(<4 x float> %a, i32 %b, i32 1) ; <<<4 x float>> [#uses=1]
+  ret <4 x float> %res
+}
+
+define <4 x float> @test_x86_avx512__mm_cvt_roundu32_ss_mem(<4 x float> %a, i32* %ptr)
+; CHECK-LABEL: test_x86_avx512__mm_cvt_roundu32_ss_mem:
+; CHECK:       ## BB#0:
+; CHECK-NEXT:    movl (%rdi), %eax 
+; CHECK-NEXT:    vcvtusi2ssl %eax, {rd-sae}, %xmm0, %xmm0 
+; CHECK-NEXT:    retq 
+{
+  %b = load i32, i32* %ptr
+  %res = call <4 x float> @llvm.x86.avx512.cvtusi2ss(<4 x float> %a, i32 %b, i32 1) ; <<<4 x float>> [#uses=1]
+  ret <4 x float> %res
+}
+
+define <4 x float> @test_x86_avx512__mm_cvtu32_ss(<4 x float> %a, i32 %b)
+; CHECK-LABEL: test_x86_avx512__mm_cvtu32_ss:
+; CHECK:       ## BB#0:
+; CHECK-NEXT:    vcvtusi2ssl %edi, %xmm0, %xmm0 
+; CHECK-NEXT:    retq 
+{
+  %res = call <4 x float> @llvm.x86.avx512.cvtusi2ss(<4 x float> %a, i32 %b, i32 4) ; <<<4 x float>> [#uses=1]
+  ret <4 x float> %res
+}
+
+define <4 x float> @test_x86_avx512__mm_cvtu32_ss_mem(<4 x float> %a, i32* %ptr)
+; CHECK-LABEL: test_x86_avx512__mm_cvtu32_ss_mem:
+; CHECK:       ## BB#0:
+; CHECK-NEXT:    vcvtusi2ssl (%rdi), %xmm0, %xmm0
+; CHECK-NEXT:    retq 
+{
+  %b = load i32, i32* %ptr
+  %res = call <4 x float> @llvm.x86.avx512.cvtusi2ss(<4 x float> %a, i32 %b, i32 4) ; <<<4 x float>> [#uses=1]
+  ret <4 x float> %res
+}
+declare <4 x float> @llvm.x86.avx512.cvtusi2ss(<4 x float>, i32, i32) nounwind readnone
+
+define <4 x float> @_mm_cvt_roundu64_ss (<4 x float> %a, i64 %b)
+; CHECK-LABEL: _mm_cvt_roundu64_ss:
+; CHECK:       ## BB#0:
+; CHECK-NEXT:    vcvtusi2ssq %rdi, {rd-sae}, %xmm0, %xmm0 
+; CHECK-NEXT:    retq 
+{
+  %res = call <4 x float> @llvm.x86.avx512.cvtusi642ss(<4 x float> %a, i64 %b, i32 1) ; <<<4 x float>> [#uses=1]
+  ret <4 x float> %res
+}
+
+define <4 x float> @_mm_cvtu64_ss(<4 x float> %a, i64 %b)
+; CHECK-LABEL: _mm_cvtu64_ss:
+; CHECK:       ## BB#0:
+; CHECK-NEXT:    vcvtusi2ssq %rdi, %xmm0, %xmm0 
+; CHECK-NEXT:    retq 
+{
+  %res = call <4 x float> @llvm.x86.avx512.cvtusi642ss(<4 x float> %a, i64 %b, i32 4) ; <<<4 x float>> [#uses=1]
+  ret <4 x float> %res
+}
+declare <4 x float> @llvm.x86.avx512.cvtusi642ss(<4 x float>, i64, i32) nounwind readnone
+
+define <2 x double> @test_x86_avx512_mm_cvtu32_sd(<2 x double> %a, i32 %b)
+; CHECK-LABEL: test_x86_avx512_mm_cvtu32_sd:
+; CHECK:       ## BB#0:
+; CHECK-NEXT:    vcvtusi2sdl %edi, %xmm0, %xmm0 
+; CHECK-NEXT:    retq 
+{
+  %res = call <2 x double> @llvm.x86.avx512.cvtusi2sd(<2 x double> %a, i32 %b) ; <<<2 x double>> [#uses=1]
+  ret <2 x double> %res
+}
+declare <2 x double> @llvm.x86.avx512.cvtusi2sd(<2 x double>, i32) nounwind readnone
+
+define <2 x double> @test_x86_avx512_mm_cvtu64_sd(<2 x double> %a, i64 %b)
+; CHECK-LABEL: test_x86_avx512_mm_cvtu64_sd:
+; CHECK:       ## BB#0:
+; CHECK-NEXT:    vcvtusi2sdq %rdi, {rd-sae}, %xmm0, %xmm0 
+; CHECK-NEXT:    retq 
+{
+  %res = call <2 x double> @llvm.x86.avx512.cvtusi642sd(<2 x double> %a, i64 %b, i32 1) ; <<<2 x double>> [#uses=1]
+  ret <2 x double> %res
+}
+
+define <2 x double> @test_x86_avx512__mm_cvt_roundu64_sd(<2 x double> %a, i64 %b)
+; CHECK-LABEL: test_x86_avx512__mm_cvt_roundu64_sd:
+; CHECK:       ## BB#0:
+; CHECK-NEXT:    vcvtusi2sdq %rdi, %xmm0, %xmm0 
+; CHECK-NEXT:    retq 
+{
+  %res = call <2 x double> @llvm.x86.avx512.cvtusi642sd(<2 x double> %a, i64 %b, i32 4) ; <<<2 x double>> [#uses=1]
+  ret <2 x double> %res
+}
+declare <2 x double> @llvm.x86.avx512.cvtusi642sd(<2 x double>, i64, i32) nounwind readnone
+
+define <8 x i64> @test_vpmaxq(<8 x i64> %a0, <8 x i64> %a1) {
+  ; CHECK: vpmaxsq {{.*}}encoding: [0x62,0xf2,0xfd,0x48,0x3d,0xc1]
+  %res = call <8 x i64> @llvm.x86.avx512.mask.pmaxs.q.512(<8 x i64> %a0, <8 x i64> %a1,
+                    <8 x i64>zeroinitializer, i8 -1)
+  ret <8 x i64> %res
+}
+declare <8 x i64> @llvm.x86.avx512.mask.pmaxs.q.512(<8 x i64>, <8 x i64>, <8 x i64>, i8)
+
+define <16 x i32> @test_vpminud(<16 x i32> %a0, <16 x i32> %a1) {
+  ; CHECK: vpminud {{.*}}encoding: [0x62,0xf2,0x7d,0x48,0x3b,0xc1]
+  %res = call <16 x i32> @llvm.x86.avx512.mask.pminu.d.512(<16 x i32> %a0, <16 x i32> %a1,
+                    <16 x i32>zeroinitializer, i16 -1)
+  ret <16 x i32> %res
+}
+declare <16 x i32> @llvm.x86.avx512.mask.pminu.d.512(<16 x i32>, <16 x i32>, <16 x i32>, i16)
+
+define <16 x i32> @test_vpmaxsd(<16 x i32> %a0, <16 x i32> %a1) {
+  ; CHECK: vpmaxsd {{.*}}encoding: [0x62,0xf2,0x7d,0x48,0x3d,0xc1]
+  %res = call <16 x i32> @llvm.x86.avx512.mask.pmaxs.d.512(<16 x i32> %a0, <16 x i32> %a1,
+                    <16 x i32>zeroinitializer, i16 -1)
+  ret <16 x i32> %res
+}
+declare <16 x i32> @llvm.x86.avx512.mask.pmaxs.d.512(<16 x i32>, <16 x i32>, <16 x i32>, i16)
+
+; CHECK-LABEL: @test_int_x86_avx512_mask_pmaxs_d_512
+; CHECK-NOT: call 
+; CHECK: vpmaxsd %zmm
+; CHECK: {%k1} 
+define <16 x i32>@test_int_x86_avx512_mask_pmaxs_d_512(<16 x i32> %x0, <16 x i32> %x1, <16 x i32> %x2, i16 %x3) {
+  %res = call <16 x i32> @llvm.x86.avx512.mask.pmaxs.d.512(<16 x i32> %x0, <16 x i32> %x1, <16 x i32> %x2, i16 %x3)
+  %res1 = call <16 x i32> @llvm.x86.avx512.mask.pmaxs.d.512(<16 x i32> %x0, <16 x i32> %x1, <16 x i32> %x2, i16 -1)
+  %res2 = add <16 x i32> %res, %res1
+  ret <16 x i32> %res2
+}
+
+; CHECK-LABEL: @test_int_x86_avx512_mask_pmaxs_q_512
+; CHECK-NOT: call 
+; CHECK: vpmaxsq %zmm
+; CHECK: {%k1} 
+define <8 x i64>@test_int_x86_avx512_mask_pmaxs_q_512(<8 x i64> %x0, <8 x i64> %x1, <8 x i64> %x2, i8 %x3) {
+  %res = call <8 x i64> @llvm.x86.avx512.mask.pmaxs.q.512(<8 x i64> %x0, <8 x i64> %x1, <8 x i64> %x2, i8 %x3)
+  %res1 = call <8 x i64> @llvm.x86.avx512.mask.pmaxs.q.512(<8 x i64> %x0, <8 x i64> %x1, <8 x i64> %x2, i8 -1)
+  %res2 = add <8 x i64> %res, %res1
+  ret <8 x i64> %res2
+}
+
+declare <16 x i32> @llvm.x86.avx512.mask.pmaxu.d.512(<16 x i32>, <16 x i32>, <16 x i32>, i16)
+
+; CHECK-LABEL: @test_int_x86_avx512_mask_pmaxu_d_512
+; CHECK-NOT: call 
+; CHECK: vpmaxud %zmm
+; CHECK: {%k1} 
+define <16 x i32>@test_int_x86_avx512_mask_pmaxu_d_512(<16 x i32> %x0, <16 x i32> %x1, <16 x i32> %x2, i16 %x3) {
+  %res = call <16 x i32> @llvm.x86.avx512.mask.pmaxu.d.512(<16 x i32> %x0, <16 x i32> %x1, <16 x i32> %x2, i16 %x3)
+  %res1 = call <16 x i32> @llvm.x86.avx512.mask.pmaxu.d.512(<16 x i32> %x0, <16 x i32> %x1, <16 x i32> %x2, i16 -1)
+  %res2 = add <16 x i32> %res, %res1
+  ret <16 x i32> %res2
+}
+
+declare <8 x i64> @llvm.x86.avx512.mask.pmaxu.q.512(<8 x i64>, <8 x i64>, <8 x i64>, i8)
+
+; CHECK-LABEL: @test_int_x86_avx512_mask_pmaxu_q_512
+; CHECK-NOT: call 
+; CHECK: vpmaxuq %zmm
+; CHECK: {%k1} 
+define <8 x i64>@test_int_x86_avx512_mask_pmaxu_q_512(<8 x i64> %x0, <8 x i64> %x1, <8 x i64> %x2, i8 %x3) {
+  %res = call <8 x i64> @llvm.x86.avx512.mask.pmaxu.q.512(<8 x i64> %x0, <8 x i64> %x1, <8 x i64> %x2, i8 %x3)
+  %res1 = call <8 x i64> @llvm.x86.avx512.mask.pmaxu.q.512(<8 x i64> %x0, <8 x i64> %x1, <8 x i64> %x2, i8 -1)
+  %res2 = add <8 x i64> %res, %res1
+  ret <8 x i64> %res2
+}
+
+declare <16 x i32> @llvm.x86.avx512.mask.pmins.d.512(<16 x i32>, <16 x i32>, <16 x i32>, i16)
+
+; CHECK-LABEL: @test_int_x86_avx512_mask_pmins_d_512
+; CHECK-NOT: call 
+; CHECK: vpminsd %zmm
+; CHECK: {%k1} 
+define <16 x i32>@test_int_x86_avx512_mask_pmins_d_512(<16 x i32> %x0, <16 x i32> %x1, <16 x i32> %x2, i16 %x3) {
+  %res = call <16 x i32> @llvm.x86.avx512.mask.pmins.d.512(<16 x i32> %x0, <16 x i32> %x1, <16 x i32> %x2, i16 %x3)
+  %res1 = call <16 x i32> @llvm.x86.avx512.mask.pmins.d.512(<16 x i32> %x0, <16 x i32> %x1, <16 x i32> %x2, i16 -1)
+  %res2 = add <16 x i32> %res, %res1
+  ret <16 x i32> %res2
+}
+
+declare <8 x i64> @llvm.x86.avx512.mask.pmins.q.512(<8 x i64>, <8 x i64>, <8 x i64>, i8)
+
+; CHECK-LABEL: @test_int_x86_avx512_mask_pmins_q_512
+; CHECK-NOT: call 
+; CHECK: vpminsq %zmm
+; CHECK: {%k1} 
+define <8 x i64>@test_int_x86_avx512_mask_pmins_q_512(<8 x i64> %x0, <8 x i64> %x1, <8 x i64> %x2, i8 %x3) {
+  %res = call <8 x i64> @llvm.x86.avx512.mask.pmins.q.512(<8 x i64> %x0, <8 x i64> %x1, <8 x i64> %x2, i8 %x3)
+  %res1 = call <8 x i64> @llvm.x86.avx512.mask.pmins.q.512(<8 x i64> %x0, <8 x i64> %x1, <8 x i64> %x2, i8 -1)
+  %res2 = add <8 x i64> %res, %res1
+  ret <8 x i64> %res2
+}
+
+; CHECK-LABEL: @test_int_x86_avx512_mask_pminu_d_512
+; CHECK-NOT: call 
+; CHECK: vpminud %zmm
+; CHECK: {%k1} 
+define <16 x i32>@test_int_x86_avx512_mask_pminu_d_512(<16 x i32> %x0, <16 x i32> %x1, <16 x i32> %x2, i16 %x3) {
+  %res = call <16 x i32> @llvm.x86.avx512.mask.pminu.d.512(<16 x i32> %x0, <16 x i32> %x1, <16 x i32> %x2, i16 %x3)
+  %res1 = call <16 x i32> @llvm.x86.avx512.mask.pminu.d.512(<16 x i32> %x0, <16 x i32> %x1, <16 x i32> %x2, i16 -1)
+  %res2 = add <16 x i32> %res, %res1
+  ret <16 x i32> %res2
+}
+
+declare <8 x i64> @llvm.x86.avx512.mask.pminu.q.512(<8 x i64>, <8 x i64>, <8 x i64>, i8)
+
+; CHECK-LABEL: @test_int_x86_avx512_mask_pminu_q_512
+; CHECK-NOT: call 
+; CHECK: vpminuq %zmm
+; CHECK: {%k1} 
+define <8 x i64>@test_int_x86_avx512_mask_pminu_q_512(<8 x i64> %x0, <8 x i64> %x1, <8 x i64> %x2, i8 %x3) {
+  %res = call <8 x i64> @llvm.x86.avx512.mask.pminu.q.512(<8 x i64> %x0, <8 x i64> %x1, <8 x i64> %x2, i8 %x3)
+  %res1 = call <8 x i64> @llvm.x86.avx512.mask.pminu.q.512(<8 x i64> %x0, <8 x i64> %x1, <8 x i64> %x2, i8 -1)
+  %res2 = add <8 x i64> %res, %res1
+  ret <8 x i64> %res2
+}
+
+declare <16 x i32> @llvm.x86.avx512.mask.vpermi2var.d.512(<16 x i32>, <16 x i32>, <16 x i32>, i16)
+
+; CHECK-LABEL: @test_int_x86_avx512_mask_vpermi2var_d_512
+; CHECK-NOT: call 
+; CHECK: kmov 
+; CHECK: vpermi2d {{.*}}{%k1} 
+define <16 x i32>@test_int_x86_avx512_mask_vpermi2var_d_512(<16 x i32> %x0, <16 x i32> %x1, <16 x i32> %x2, i16 %x3) {
+  %res = call <16 x i32> @llvm.x86.avx512.mask.vpermi2var.d.512(<16 x i32> %x0, <16 x i32> %x1, <16 x i32> %x2, i16 %x3)
+  %res1 = call <16 x i32> @llvm.x86.avx512.mask.vpermi2var.d.512(<16 x i32> %x0, <16 x i32> %x1, <16 x i32> %x2, i16 -1)
+  %res2 = add <16 x i32> %res, %res1
+  ret <16 x i32> %res2
+}
+
+declare <8 x double> @llvm.x86.avx512.mask.vpermi2var.pd.512(<8 x double>, <8 x i64>, <8 x double>, i8)
+
+; CHECK-LABEL: @test_int_x86_avx512_mask_vpermi2var_pd_512
+; CHECK-NOT: call 
+; CHECK: kmov 
+; CHECK:  vpermi2pd {{.*}}{%k1} 
+define <8 x double>@test_int_x86_avx512_mask_vpermi2var_pd_512(<8 x double> %x0, <8 x i64> %x1, <8 x double> %x2, i8 %x3) {
+  %res = call <8 x double> @llvm.x86.avx512.mask.vpermi2var.pd.512(<8 x double> %x0, <8 x i64> %x1, <8 x double> %x2, i8 %x3)
+  %res1 = call <8 x double> @llvm.x86.avx512.mask.vpermi2var.pd.512(<8 x double> %x0, <8 x i64> %x1, <8 x double> %x2, i8 -1)
+  %res2 = fadd <8 x double> %res, %res1
+  ret <8 x double> %res2
+}
+
+declare <16 x float> @llvm.x86.avx512.mask.vpermi2var.ps.512(<16 x float>, <16 x i32>, <16 x float>, i16)
+
+; CHECK-LABEL: @test_int_x86_avx512_mask_vpermi2var_ps_512
+; CHECK-NOT: call 
+; CHECK: kmov 
+; CHECK: vpermi2ps {{.*}}{%k1} 
+define <16 x float>@test_int_x86_avx512_mask_vpermi2var_ps_512(<16 x float> %x0, <16 x i32> %x1, <16 x float> %x2, i16 %x3) {
+  %res = call <16 x float> @llvm.x86.avx512.mask.vpermi2var.ps.512(<16 x float> %x0, <16 x i32> %x1, <16 x float> %x2, i16 %x3)
+  %res1 = call <16 x float> @llvm.x86.avx512.mask.vpermi2var.ps.512(<16 x float> %x0, <16 x i32> %x1, <16 x float> %x2, i16 -1)
+  %res2 = fadd <16 x float> %res, %res1
+  ret <16 x float> %res2
+}
+
+declare <8 x i64> @llvm.x86.avx512.mask.vpermi2var.q.512(<8 x i64>, <8 x i64>, <8 x i64>, i8)
+
+; CHECK-LABEL: @test_int_x86_avx512_mask_vpermi2var_q_512
+; CHECK-NOT: call 
+; CHECK: kmov 
+; CHECK: vpermi2q {{.*}}{%k1} 
+define <8 x i64>@test_int_x86_avx512_mask_vpermi2var_q_512(<8 x i64> %x0, <8 x i64> %x1, <8 x i64> %x2, i8 %x3) {
+  %res = call <8 x i64> @llvm.x86.avx512.mask.vpermi2var.q.512(<8 x i64> %x0, <8 x i64> %x1, <8 x i64> %x2, i8 %x3)
+  %res1 = call <8 x i64> @llvm.x86.avx512.mask.vpermi2var.q.512(<8 x i64> %x0, <8 x i64> %x1, <8 x i64> %x2, i8 -1)
+  %res2 = add <8 x i64> %res, %res1
+  ret <8 x i64> %res2
+}
+
+declare <16 x i32> @llvm.x86.avx512.maskz.vpermt2var.d.512(<16 x i32>, <16 x i32>, <16 x i32>, i16)
+
+; CHECK-LABEL: @test_int_x86_avx512_maskz_vpermt2var_d_512
+; CHECK-NOT: call 
+; CHECK: kmov 
+; CHECK: vpermt2d {{.*}}{%k1} {z}
+define <16 x i32>@test_int_x86_avx512_maskz_vpermt2var_d_512(<16 x i32> %x0, <16 x i32> %x1, <16 x i32> %x2, i16 %x3) {
+  %res = call <16 x i32> @llvm.x86.avx512.maskz.vpermt2var.d.512(<16 x i32> %x0, <16 x i32> %x1, <16 x i32> %x2, i16 %x3)
+  %res1 = call <16 x i32> @llvm.x86.avx512.maskz.vpermt2var.d.512(<16 x i32> %x0, <16 x i32> %x1, <16 x i32> %x2, i16 -1)
+  %res2 = add <16 x i32> %res, %res1
+  ret <16 x i32> %res2
+}
+
+declare <8 x double> @llvm.x86.avx512.maskz.vpermt2var.pd.512(<8 x i64>, <8 x double>, <8 x double>, i8)
+
+; CHECK-LABEL: @test_int_x86_avx512_maskz_vpermt2var_pd_512
+; CHECK-NOT: call 
+; CHECK: kmov 
+; CHECK: vpermt2pd {{.*}}{%k1} {z}
+define <8 x double>@test_int_x86_avx512_maskz_vpermt2var_pd_512(<8 x i64> %x0, <8 x double> %x1, <8 x double> %x2, i8 %x3) {
+  %res = call <8 x double> @llvm.x86.avx512.maskz.vpermt2var.pd.512(<8 x i64> %x0, <8 x double> %x1, <8 x double> %x2, i8 %x3)
+  %res1 = call <8 x double> @llvm.x86.avx512.maskz.vpermt2var.pd.512(<8 x i64> %x0, <8 x double> %x1, <8 x double> %x2, i8 -1)
+  %res2 = fadd <8 x double> %res, %res1
+  ret <8 x double> %res2
+}
+
+declare <16 x float> @llvm.x86.avx512.maskz.vpermt2var.ps.512(<16 x i32>, <16 x float>, <16 x float>, i16)
+
+; CHECK-LABEL: @test_int_x86_avx512_maskz_vpermt2var_ps_512
+; CHECK-NOT: call 
+; CHECK: kmov 
+; CHECK: vpermt2ps {{.*}}{%k1} {z}
+define <16 x float>@test_int_x86_avx512_maskz_vpermt2var_ps_512(<16 x i32> %x0, <16 x float> %x1, <16 x float> %x2, i16 %x3) {
+  %res = call <16 x float> @llvm.x86.avx512.maskz.vpermt2var.ps.512(<16 x i32> %x0, <16 x float> %x1, <16 x float> %x2, i16 %x3)
+  %res1 = call <16 x float> @llvm.x86.avx512.maskz.vpermt2var.ps.512(<16 x i32> %x0, <16 x float> %x1, <16 x float> %x2, i16 -1)
+  %res2 = fadd <16 x float> %res, %res1
+  ret <16 x float> %res2
+}
+
+
+declare <8 x i64> @llvm.x86.avx512.maskz.vpermt2var.q.512(<8 x i64>, <8 x i64>, <8 x i64>, i8)
+
+; CHECK-LABEL: @test_int_x86_avx512_maskz_vpermt2var_q_512
+; CHECK-NOT: call 
+; CHECK: kmov 
+; CHECK: vpermt2q {{.*}}{%k1} {z}
+define <8 x i64>@test_int_x86_avx512_maskz_vpermt2var_q_512(<8 x i64> %x0, <8 x i64> %x1, <8 x i64> %x2, i8 %x3) {
+  %res = call <8 x i64> @llvm.x86.avx512.maskz.vpermt2var.q.512(<8 x i64> %x0, <8 x i64> %x1, <8 x i64> %x2, i8 %x3)
+  %res1 = call <8 x i64> @llvm.x86.avx512.maskz.vpermt2var.q.512(<8 x i64> %x0, <8 x i64> %x1, <8 x i64> %x2, i8 -1)
+  %res2 = add <8 x i64> %res, %res1
+  ret <8 x i64> %res2
+}
+
+declare <16 x i32> @llvm.x86.avx512.mask.vpermt2var.d.512(<16 x i32>, <16 x i32>, <16 x i32>, i16)
+
+; CHECK-LABEL: @test_int_x86_avx512_mask_vpermt2var_d_512
+; CHECK-NOT: call 
+; CHECK: kmov 
+; CHECK: vpermt2d {{.*}}{%k1}
+; CHECK-NOT: {z}
+define <16 x i32>@test_int_x86_avx512_mask_vpermt2var_d_512(<16 x i32> %x0, <16 x i32> %x1, <16 x i32> %x2, i16 %x3) {
+  %res = call <16 x i32> @llvm.x86.avx512.mask.vpermt2var.d.512(<16 x i32> %x0, <16 x i32> %x1, <16 x i32> %x2, i16 %x3)
+  %res1 = call <16 x i32> @llvm.x86.avx512.mask.vpermt2var.d.512(<16 x i32> %x0, <16 x i32> %x1, <16 x i32> %x2, i16 -1)
+  %res2 = add <16 x i32> %res, %res1
+  ret <16 x i32> %res2
+}
+
+declare <8 x double> @llvm.x86.avx512.mask.scalef.pd.512(<8 x double>, <8 x double>, <8 x double>, i8, i32)
+; CHECK-LABEL: @test_int_x86_avx512_mask_scalef_pd_512
+; CHECK-NOT: call 
+; CHECK: kmov 
+; CHECK: vscalefpd{{.*}}{%k1} 
+define <8 x double>@test_int_x86_avx512_mask_scalef_pd_512(<8 x double> %x0, <8 x double> %x1, <8 x double> %x2, i8 %x3) {
+  %res = call <8 x double> @llvm.x86.avx512.mask.scalef.pd.512(<8 x double> %x0, <8 x double> %x1, <8 x double> %x2, i8 %x3, i32 3)
+  %res1 = call <8 x double> @llvm.x86.avx512.mask.scalef.pd.512(<8 x double> %x0, <8 x double> %x1, <8 x double> %x2, i8 -1, i32 0)
+  %res2 = fadd <8 x double> %res, %res1
+  ret <8 x double> %res2
+}
+
+declare <16 x float> @llvm.x86.avx512.mask.scalef.ps.512(<16 x float>, <16 x float>, <16 x float>, i16, i32)
+; CHECK-LABEL: @test_int_x86_avx512_mask_scalef_ps_512
+; CHECK-NOT: call 
+; CHECK: kmov 
+; CHECK: vscalefps{{.*}}{%k1} 
+define <16 x float>@test_int_x86_avx512_mask_scalef_ps_512(<16 x float> %x0, <16 x float> %x1, <16 x float> %x2, i16 %x3) {
+  %res = call <16 x float> @llvm.x86.avx512.mask.scalef.ps.512(<16 x float> %x0, <16 x float> %x1, <16 x float> %x2, i16 %x3, i32 2)
+  %res1 = call <16 x float> @llvm.x86.avx512.mask.scalef.ps.512(<16 x float> %x0, <16 x float> %x1, <16 x float> %x2, i16 -1, i32 0)
+  %res2 = fadd <16 x float> %res, %res1
+  ret <16 x float> %res2
+}
+
+declare <8 x double> @llvm.x86.avx512.mask.unpckh.pd.512(<8 x double>, <8 x double>, <8 x double>, i8)
+
+define <8 x double>@test_int_x86_avx512_mask_unpckh_pd_512(<8 x double> %x0, <8 x double> %x1, <8 x double> %x2, i8 %x3) {
+; CHECK-LABEL: test_int_x86_avx512_mask_unpckh_pd_512:
+; CHECK:       ## BB#0:
+; CHECK-NEXT:    movzbl %dil, %eax
+; CHECK-NEXT:    kmovw %eax, %k1
+; CHECK-NEXT:    vunpckhpd %zmm1, %zmm0, %zmm2 {%k1}
+; CHECK-NEXT:    vunpckhpd %zmm1, %zmm0, %zmm0
+  %res = call <8 x double> @llvm.x86.avx512.mask.unpckh.pd.512(<8 x double> %x0, <8 x double> %x1, <8 x double> %x2, i8 %x3)
+  %res1 = call <8 x double> @llvm.x86.avx512.mask.unpckh.pd.512(<8 x double> %x0, <8 x double> %x1, <8 x double> %x2, i8 -1)
+  %res2 = fadd <8 x double> %res, %res1
+  ret <8 x double> %res2
+}
+
+declare <16 x float> @llvm.x86.avx512.mask.unpckh.ps.512(<16 x float>, <16 x float>, <16 x float>, i16)
+
+define <16 x float>@test_int_x86_avx512_mask_unpckh_ps_512(<16 x float> %x0, <16 x float> %x1, <16 x float> %x2, i16 %x3) {
+; CHECK-LABEL: test_int_x86_avx512_mask_unpckh_ps_512:
+; CHECK:       ## BB#0:
+; CHECK-NEXT:    kmovw %edi, %k1
+; CHECK-NEXT:    vunpckhps %zmm1, %zmm0, %zmm2 {%k1}
+; CHECK-NEXT:    vunpckhps %zmm1, %zmm0, %zmm0
+  %res = call <16 x float> @llvm.x86.avx512.mask.unpckh.ps.512(<16 x float> %x0, <16 x float> %x1, <16 x float> %x2, i16 %x3)
+  %res1 = call <16 x float> @llvm.x86.avx512.mask.unpckh.ps.512(<16 x float> %x0, <16 x float> %x1, <16 x float> %x2, i16 -1)
+  %res2 = fadd <16 x float> %res, %res1
+  ret <16 x float> %res2
+}
+
+declare <8 x double> @llvm.x86.avx512.mask.unpckl.pd.512(<8 x double>, <8 x double>, <8 x double>, i8)
+
+define <8 x double>@test_int_x86_avx512_mask_unpckl_pd_512(<8 x double> %x0, <8 x double> %x1, <8 x double> %x2, i8 %x3) {
+; CHECK-LABEL: test_int_x86_avx512_mask_unpckl_pd_512:
+; CHECK:       ## BB#0:
+; CHECK-NEXT:    movzbl %dil, %eax
+; CHECK-NEXT:    kmovw %eax, %k1
+; CHECK-NEXT:    vunpcklpd %zmm1, %zmm0, %zmm2 {%k1}
+; CHECK-NEXT:    vunpcklpd %zmm1, %zmm0, %zmm0
+  %res = call <8 x double> @llvm.x86.avx512.mask.unpckl.pd.512(<8 x double> %x0, <8 x double> %x1, <8 x double> %x2, i8 %x3)
+  %res1 = call <8 x double> @llvm.x86.avx512.mask.unpckl.pd.512(<8 x double> %x0, <8 x double> %x1, <8 x double> %x2, i8 -1)
+  %res2 = fadd <8 x double> %res, %res1
+  ret <8 x double> %res2
+}
+
+declare <16 x float> @llvm.x86.avx512.mask.unpckl.ps.512(<16 x float>, <16 x float>, <16 x float>, i16)
+
+define <16 x float>@test_int_x86_avx512_mask_unpckl_ps_512(<16 x float> %x0, <16 x float> %x1, <16 x float> %x2, i16 %x3) {
+; CHECK-LABEL: test_int_x86_avx512_mask_unpckl_ps_512:
+; CHECK:       ## BB#0:
+; CHECK-NEXT:    kmovw %edi, %k1
+; CHECK-NEXT:    vunpcklps %zmm1, %zmm0, %zmm2 {%k1}
+; CHECK-NEXT:    vunpcklps %zmm1, %zmm0, %zmm0
+  %res = call <16 x float> @llvm.x86.avx512.mask.unpckl.ps.512(<16 x float> %x0, <16 x float> %x1, <16 x float> %x2, i16 %x3)
+  %res1 = call <16 x float> @llvm.x86.avx512.mask.unpckl.ps.512(<16 x float> %x0, <16 x float> %x1, <16 x float> %x2, i16 -1)
+  %res2 = fadd <16 x float> %res, %res1
+  ret <16 x float> %res2
+}
+
+declare <8 x i64> @llvm.x86.avx512.mask.punpcklqd.q.512(<8 x i64>, <8 x i64>, <8 x i64>, i8)
+
+define <8 x i64>@test_int_x86_avx512_mask_punpcklqd_q_512(<8 x i64> %x0, <8 x i64> %x1, <8 x i64> %x2, i8 %x3) {
+; CHECK-LABEL: test_int_x86_avx512_mask_punpcklqd_q_512:
+; CHECK:       ## BB#0:
+; CHECK-NEXT:    movzbl %dil, %eax
+; CHECK-NEXT:    kmovw %eax, %k1
+; CHECK-NEXT:    vpunpcklqdq %zmm1, %zmm0, %zmm2 {%k1}
+; CHECK-NEXT:    vpunpcklqdq %zmm1, %zmm0, %zmm3 {%k1} {z}
+; CHECK-NEXT:    vpunpcklqdq {{.*#+}} 
+; CHECK:         vpaddq %zmm0, %zmm2, %zmm0
+; CHECK-NEXT:    vpaddq %zmm0, %zmm3, %zmm0
+; CHECK-NEXT:    retq
+  %res = call <8 x i64> @llvm.x86.avx512.mask.punpcklqd.q.512(<8 x i64> %x0, <8 x i64> %x1, <8 x i64> %x2, i8 %x3)
+  %res1 = call <8 x i64> @llvm.x86.avx512.mask.punpcklqd.q.512(<8 x i64> %x0, <8 x i64> %x1, <8 x i64> %x2, i8 -1)
+  %res2 = call <8 x i64> @llvm.x86.avx512.mask.punpcklqd.q.512(<8 x i64> %x0, <8 x i64> %x1, <8 x i64> zeroinitializer,i8 %x3)
+  %res3 = add <8 x i64> %res, %res1
+  %res4 = add <8 x i64> %res2, %res3
+  ret <8 x i64> %res4
+}
+
+declare <8 x i64> @llvm.x86.avx512.mask.punpckhqd.q.512(<8 x i64>, <8 x i64>, <8 x i64>, i8)
+
+define <8 x i64>@test_int_x86_avx512_mask_punpckhqd_q_512(<8 x i64> %x0, <8 x i64> %x1, <8 x i64> %x2, i8 %x3) {
+; CHECK-LABEL: test_int_x86_avx512_mask_punpckhqd_q_512:
+; CHECK:       ## BB#0:
+; CHECK-NEXT:    movzbl %dil, %eax
+; CHECK-NEXT:    kmovw %eax, %k1
+; CHECK-NEXT:    vpunpckhqdq %zmm1, %zmm0, %zmm2 {%k1}
+; CHECK-NEXT:    vpunpckhqdq {{.*#+}}
+; CHECK:         vpaddq %zmm0, %zmm2, %zmm0
+; CHECK-NEXT:    retq
+  %res = call <8 x i64> @llvm.x86.avx512.mask.punpckhqd.q.512(<8 x i64> %x0, <8 x i64> %x1, <8 x i64> %x2, i8 %x3)
+  %res1 = call <8 x i64> @llvm.x86.avx512.mask.punpckhqd.q.512(<8 x i64> %x0, <8 x i64> %x1, <8 x i64> %x2, i8 -1)
+  %res2 = add <8 x i64> %res, %res1
+  ret <8 x i64> %res2
+}
+
+declare <16 x i32> @llvm.x86.avx512.mask.punpckhd.q.512(<16 x i32>, <16 x i32>, <16 x i32>, i16)
+
+define <16 x i32>@test_int_x86_avx512_mask_punpckhd_q_512(<16 x i32> %x0, <16 x i32> %x1, <16 x i32> %x2, i16 %x3) {
+; CHECK-LABEL: test_int_x86_avx512_mask_punpckhd_q_512:
+; CHECK:       ## BB#0:
+; CHECK-NEXT:    kmovw %edi, %k1
+; CHECK-NEXT:    vpunpckhdq %zmm1, %zmm0, %zmm2 {%k1}
+; CHECK-NEXT:    vpunpckhdq {{.*#+}}
+; CHECK:         vpaddd %zmm0, %zmm2, %zmm0
+; CHECK-NEXT:    retq
+  %res = call <16 x i32> @llvm.x86.avx512.mask.punpckhd.q.512(<16 x i32> %x0, <16 x i32> %x1, <16 x i32> %x2, i16 %x3)
+  %res1 = call <16 x i32> @llvm.x86.avx512.mask.punpckhd.q.512(<16 x i32> %x0, <16 x i32> %x1, <16 x i32> %x2, i16 -1)
+  %res2 = add <16 x i32> %res, %res1
+  ret <16 x i32> %res2
+}
+
+declare <16 x i32> @llvm.x86.avx512.mask.punpckld.q.512(<16 x i32>, <16 x i32>, <16 x i32>, i16)
+
+define <16 x i32>@test_int_x86_avx512_mask_punpckld_q_512(<16 x i32> %x0, <16 x i32> %x1, <16 x i32> %x2, i16 %x3) {
+; CHECK-LABEL: test_int_x86_avx512_mask_punpckld_q_512:
+; CHECK:       ## BB#0:
+; CHECK-NEXT:    kmovw %edi, %k1
+; CHECK-NEXT:    vpunpckldq %zmm1, %zmm0, %zmm2 {%k1}
+; CHECK-NEXT:    vpunpckldq {{.*#+}} 
+; CHECK:         vpaddd %zmm0, %zmm2, %zmm0
+; CHECK-NEXT:    retq
+  %res = call <16 x i32> @llvm.x86.avx512.mask.punpckld.q.512(<16 x i32> %x0, <16 x i32> %x1, <16 x i32> %x2, i16 %x3)
+  %res1 = call <16 x i32> @llvm.x86.avx512.mask.punpckld.q.512(<16 x i32> %x0, <16 x i32> %x1, <16 x i32> %x2, i16 -1)
+  %res2 = add <16 x i32> %res, %res1
+  ret <16 x i32> %res2
+}
+
+declare <16 x i8> @llvm.x86.avx512.mask.pmov.qb.512(<8 x i64>, <16 x i8>, i8)
+
+define <16 x i8>@test_int_x86_avx512_mask_pmov_qb_512(<8 x i64> %x0, <16 x i8> %x1, i8 %x2) {
+; CHECK-LABEL: test_int_x86_avx512_mask_pmov_qb_512:
+; CHECK:       vpmovqb %zmm0, %xmm1 {%k1}
+; CHECK-NEXT:  vpmovqb %zmm0, %xmm2 {%k1} {z}
+; CHECK-NEXT:  vpmovqb %zmm0, %xmm0
+    %res0 = call <16 x i8> @llvm.x86.avx512.mask.pmov.qb.512(<8 x i64> %x0, <16 x i8> %x1, i8 -1)
+    %res1 = call <16 x i8> @llvm.x86.avx512.mask.pmov.qb.512(<8 x i64> %x0, <16 x i8> %x1, i8 %x2)
+    %res2 = call <16 x i8> @llvm.x86.avx512.mask.pmov.qb.512(<8 x i64> %x0, <16 x i8> zeroinitializer, i8 %x2)
+    %res3 = add <16 x i8> %res0, %res1
+    %res4 = add <16 x i8> %res3, %res2
+    ret <16 x i8> %res4
+}
+
+declare void @llvm.x86.avx512.mask.pmov.qb.mem.512(i8* %ptr, <8 x i64>, i8)
+
+define void @test_int_x86_avx512_mask_pmov_qb_mem_512(i8* %ptr, <8 x i64> %x1, i8 %x2) {
+; CHECK-LABEL: test_int_x86_avx512_mask_pmov_qb_mem_512:
+; CHECK:  vpmovqb %zmm0, (%rdi)
+; CHECK:  vpmovqb %zmm0, (%rdi) {%k1}
+    call void @llvm.x86.avx512.mask.pmov.qb.mem.512(i8* %ptr, <8 x i64> %x1, i8 -1)
+    call void @llvm.x86.avx512.mask.pmov.qb.mem.512(i8* %ptr, <8 x i64> %x1, i8 %x2)
+    ret void
+}
+
+declare <16 x i8> @llvm.x86.avx512.mask.pmovs.qb.512(<8 x i64>, <16 x i8>, i8)
+
+define <16 x i8>@test_int_x86_avx512_mask_pmovs_qb_512(<8 x i64> %x0, <16 x i8> %x1, i8 %x2) {
+; CHECK-LABEL: test_int_x86_avx512_mask_pmovs_qb_512:
+; CHECK:       vpmovsqb %zmm0, %xmm1 {%k1}
+; CHECK-NEXT:  vpmovsqb %zmm0, %xmm2 {%k1} {z}
+; CHECK-NEXT:  vpmovsqb %zmm0, %xmm0
+    %res0 = call <16 x i8> @llvm.x86.avx512.mask.pmovs.qb.512(<8 x i64> %x0, <16 x i8> %x1, i8 -1)
+    %res1 = call <16 x i8> @llvm.x86.avx512.mask.pmovs.qb.512(<8 x i64> %x0, <16 x i8> %x1, i8 %x2)
+    %res2 = call <16 x i8> @llvm.x86.avx512.mask.pmovs.qb.512(<8 x i64> %x0, <16 x i8> zeroinitializer, i8 %x2)
+    %res3 = add <16 x i8> %res0, %res1
+    %res4 = add <16 x i8> %res3, %res2
+    ret <16 x i8> %res4
+}
+
+declare void @llvm.x86.avx512.mask.pmovs.qb.mem.512(i8* %ptr, <8 x i64>, i8)
+
+define void @test_int_x86_avx512_mask_pmovs_qb_mem_512(i8* %ptr, <8 x i64> %x1, i8 %x2) {
+; CHECK-LABEL: test_int_x86_avx512_mask_pmovs_qb_mem_512:
+; CHECK:  vpmovsqb %zmm0, (%rdi)
+; CHECK:  vpmovsqb %zmm0, (%rdi) {%k1}
+    call void @llvm.x86.avx512.mask.pmovs.qb.mem.512(i8* %ptr, <8 x i64> %x1, i8 -1)
+    call void @llvm.x86.avx512.mask.pmovs.qb.mem.512(i8* %ptr, <8 x i64> %x1, i8 %x2)
+    ret void
+}
+
+declare <16 x i8> @llvm.x86.avx512.mask.pmovus.qb.512(<8 x i64>, <16 x i8>, i8)
+
+define <16 x i8>@test_int_x86_avx512_mask_pmovus_qb_512(<8 x i64> %x0, <16 x i8> %x1, i8 %x2) {
+; CHECK-LABEL: test_int_x86_avx512_mask_pmovus_qb_512:
+; CHECK:       vpmovusqb %zmm0, %xmm1 {%k1}
+; CHECK-NEXT:  vpmovusqb %zmm0, %xmm2 {%k1} {z}
+; CHECK-NEXT:  vpmovusqb %zmm0, %xmm0
+    %res0 = call <16 x i8> @llvm.x86.avx512.mask.pmovus.qb.512(<8 x i64> %x0, <16 x i8> %x1, i8 -1)
+    %res1 = call <16 x i8> @llvm.x86.avx512.mask.pmovus.qb.512(<8 x i64> %x0, <16 x i8> %x1, i8 %x2)
+    %res2 = call <16 x i8> @llvm.x86.avx512.mask.pmovus.qb.512(<8 x i64> %x0, <16 x i8> zeroinitializer, i8 %x2)
+    %res3 = add <16 x i8> %res0, %res1
+    %res4 = add <16 x i8> %res3, %res2
+    ret <16 x i8> %res4
+}
+
+declare void @llvm.x86.avx512.mask.pmovus.qb.mem.512(i8* %ptr, <8 x i64>, i8)
+
+define void @test_int_x86_avx512_mask_pmovus_qb_mem_512(i8* %ptr, <8 x i64> %x1, i8 %x2) {
+; CHECK-LABEL: test_int_x86_avx512_mask_pmovus_qb_mem_512:
+; CHECK:  vpmovusqb %zmm0, (%rdi)
+; CHECK:  vpmovusqb %zmm0, (%rdi) {%k1}
+    call void @llvm.x86.avx512.mask.pmovus.qb.mem.512(i8* %ptr, <8 x i64> %x1, i8 -1)
+    call void @llvm.x86.avx512.mask.pmovus.qb.mem.512(i8* %ptr, <8 x i64> %x1, i8 %x2)
+    ret void
+}
+
+declare <8 x i16> @llvm.x86.avx512.mask.pmov.qw.512(<8 x i64>, <8 x i16>, i8)
+
+define <8 x i16>@test_int_x86_avx512_mask_pmov_qw_512(<8 x i64> %x0, <8 x i16> %x1, i8 %x2) {
+; CHECK-LABEL: test_int_x86_avx512_mask_pmov_qw_512:
+; CHECK:       vpmovqw %zmm0, %xmm1 {%k1}
+; CHECK-NEXT:  vpmovqw %zmm0, %xmm2 {%k1} {z}
+; CHECK-NEXT:  vpmovqw %zmm0, %xmm0
+    %res0 = call <8 x i16> @llvm.x86.avx512.mask.pmov.qw.512(<8 x i64> %x0, <8 x i16> %x1, i8 -1)
+    %res1 = call <8 x i16> @llvm.x86.avx512.mask.pmov.qw.512(<8 x i64> %x0, <8 x i16> %x1, i8 %x2)
+    %res2 = call <8 x i16> @llvm.x86.avx512.mask.pmov.qw.512(<8 x i64> %x0, <8 x i16> zeroinitializer, i8 %x2)
+    %res3 = add <8 x i16> %res0, %res1
+    %res4 = add <8 x i16> %res3, %res2
+    ret <8 x i16> %res4
+}
+
+declare void @llvm.x86.avx512.mask.pmov.qw.mem.512(i8* %ptr, <8 x i64>, i8)
+
+define void @test_int_x86_avx512_mask_pmov_qw_mem_512(i8* %ptr, <8 x i64> %x1, i8 %x2) {
+; CHECK-LABEL: test_int_x86_avx512_mask_pmov_qw_mem_512:
+; CHECK:  vpmovqw %zmm0, (%rdi)
+; CHECK:  vpmovqw %zmm0, (%rdi) {%k1}
+    call void @llvm.x86.avx512.mask.pmov.qw.mem.512(i8* %ptr, <8 x i64> %x1, i8 -1)
+    call void @llvm.x86.avx512.mask.pmov.qw.mem.512(i8* %ptr, <8 x i64> %x1, i8 %x2)
+    ret void
+}
+
+declare <8 x i16> @llvm.x86.avx512.mask.pmovs.qw.512(<8 x i64>, <8 x i16>, i8)
+
+define <8 x i16>@test_int_x86_avx512_mask_pmovs_qw_512(<8 x i64> %x0, <8 x i16> %x1, i8 %x2) {
+; CHECK-LABEL: test_int_x86_avx512_mask_pmovs_qw_512:
+; CHECK:       vpmovsqw %zmm0, %xmm1 {%k1}
+; CHECK-NEXT:  vpmovsqw %zmm0, %xmm2 {%k1} {z}
+; CHECK-NEXT:  vpmovsqw %zmm0, %xmm0
+    %res0 = call <8 x i16> @llvm.x86.avx512.mask.pmovs.qw.512(<8 x i64> %x0, <8 x i16> %x1, i8 -1)
+    %res1 = call <8 x i16> @llvm.x86.avx512.mask.pmovs.qw.512(<8 x i64> %x0, <8 x i16> %x1, i8 %x2)
+    %res2 = call <8 x i16> @llvm.x86.avx512.mask.pmovs.qw.512(<8 x i64> %x0, <8 x i16> zeroinitializer, i8 %x2)
+    %res3 = add <8 x i16> %res0, %res1
+    %res4 = add <8 x i16> %res3, %res2
+    ret <8 x i16> %res4
+}
+
+declare void @llvm.x86.avx512.mask.pmovs.qw.mem.512(i8* %ptr, <8 x i64>, i8)
+
+define void @test_int_x86_avx512_mask_pmovs_qw_mem_512(i8* %ptr, <8 x i64> %x1, i8 %x2) {
+; CHECK-LABEL: test_int_x86_avx512_mask_pmovs_qw_mem_512:
+; CHECK:  vpmovsqw %zmm0, (%rdi)
+; CHECK:  vpmovsqw %zmm0, (%rdi) {%k1}
+    call void @llvm.x86.avx512.mask.pmovs.qw.mem.512(i8* %ptr, <8 x i64> %x1, i8 -1)
+    call void @llvm.x86.avx512.mask.pmovs.qw.mem.512(i8* %ptr, <8 x i64> %x1, i8 %x2)
+    ret void
+}
+
+declare <8 x i16> @llvm.x86.avx512.mask.pmovus.qw.512(<8 x i64>, <8 x i16>, i8)
+
+define <8 x i16>@test_int_x86_avx512_mask_pmovus_qw_512(<8 x i64> %x0, <8 x i16> %x1, i8 %x2) {
+; CHECK-LABEL: test_int_x86_avx512_mask_pmovus_qw_512:
+; CHECK:       vpmovusqw %zmm0, %xmm1 {%k1}
+; CHECK-NEXT:  vpmovusqw %zmm0, %xmm2 {%k1} {z}
+; CHECK-NEXT:  vpmovusqw %zmm0, %xmm0
+    %res0 = call <8 x i16> @llvm.x86.avx512.mask.pmovus.qw.512(<8 x i64> %x0, <8 x i16> %x1, i8 -1)
+    %res1 = call <8 x i16> @llvm.x86.avx512.mask.pmovus.qw.512(<8 x i64> %x0, <8 x i16> %x1, i8 %x2)
+    %res2 = call <8 x i16> @llvm.x86.avx512.mask.pmovus.qw.512(<8 x i64> %x0, <8 x i16> zeroinitializer, i8 %x2)
+    %res3 = add <8 x i16> %res0, %res1
+    %res4 = add <8 x i16> %res3, %res2
+    ret <8 x i16> %res4
+}
+
+declare void @llvm.x86.avx512.mask.pmovus.qw.mem.512(i8* %ptr, <8 x i64>, i8)
+
+define void @test_int_x86_avx512_mask_pmovus_qw_mem_512(i8* %ptr, <8 x i64> %x1, i8 %x2) {
+; CHECK-LABEL: test_int_x86_avx512_mask_pmovus_qw_mem_512:
+; CHECK:  vpmovusqw %zmm0, (%rdi)
+; CHECK:  vpmovusqw %zmm0, (%rdi) {%k1}
+    call void @llvm.x86.avx512.mask.pmovus.qw.mem.512(i8* %ptr, <8 x i64> %x1, i8 -1)
+    call void @llvm.x86.avx512.mask.pmovus.qw.mem.512(i8* %ptr, <8 x i64> %x1, i8 %x2)
+    ret void
+}
+
+declare <8 x i32> @llvm.x86.avx512.mask.pmov.qd.512(<8 x i64>, <8 x i32>, i8)
+
+define <8 x i32>@test_int_x86_avx512_mask_pmov_qd_512(<8 x i64> %x0, <8 x i32> %x1, i8 %x2) {
+; CHECK-LABEL: test_int_x86_avx512_mask_pmov_qd_512:
+; CHECK:       vpmovqd %zmm0, %ymm1 {%k1}
+; CHECK-NEXT:  vpmovqd %zmm0, %ymm2 {%k1} {z}
+; CHECK-NEXT:  vpmovqd %zmm0, %ymm0
+    %res0 = call <8 x i32> @llvm.x86.avx512.mask.pmov.qd.512(<8 x i64> %x0, <8 x i32> %x1, i8 -1)
+    %res1 = call <8 x i32> @llvm.x86.avx512.mask.pmov.qd.512(<8 x i64> %x0, <8 x i32> %x1, i8 %x2)
+    %res2 = call <8 x i32> @llvm.x86.avx512.mask.pmov.qd.512(<8 x i64> %x0, <8 x i32> zeroinitializer, i8 %x2)
+    %res3 = add <8 x i32> %res0, %res1
+    %res4 = add <8 x i32> %res3, %res2
+    ret <8 x i32> %res4
+}
+
+declare void @llvm.x86.avx512.mask.pmov.qd.mem.512(i8* %ptr, <8 x i64>, i8)
+
+define void @test_int_x86_avx512_mask_pmov_qd_mem_512(i8* %ptr, <8 x i64> %x1, i8 %x2) {
+; CHECK-LABEL: test_int_x86_avx512_mask_pmov_qd_mem_512:
+; CHECK:  vpmovqd %zmm0, (%rdi)
+; CHECK:  vpmovqd %zmm0, (%rdi) {%k1}
+    call void @llvm.x86.avx512.mask.pmov.qd.mem.512(i8* %ptr, <8 x i64> %x1, i8 -1)
+    call void @llvm.x86.avx512.mask.pmov.qd.mem.512(i8* %ptr, <8 x i64> %x1, i8 %x2)
+    ret void
+}
+
+declare <8 x i32> @llvm.x86.avx512.mask.pmovs.qd.512(<8 x i64>, <8 x i32>, i8)
+
+define <8 x i32>@test_int_x86_avx512_mask_pmovs_qd_512(<8 x i64> %x0, <8 x i32> %x1, i8 %x2) {
+; CHECK-LABEL: test_int_x86_avx512_mask_pmovs_qd_512:
+; CHECK:       vpmovsqd %zmm0, %ymm1 {%k1}
+; CHECK-NEXT:  vpmovsqd %zmm0, %ymm2 {%k1} {z}
+; CHECK-NEXT:  vpmovsqd %zmm0, %ymm0
+    %res0 = call <8 x i32> @llvm.x86.avx512.mask.pmovs.qd.512(<8 x i64> %x0, <8 x i32> %x1, i8 -1)
+    %res1 = call <8 x i32> @llvm.x86.avx512.mask.pmovs.qd.512(<8 x i64> %x0, <8 x i32> %x1, i8 %x2)
+    %res2 = call <8 x i32> @llvm.x86.avx512.mask.pmovs.qd.512(<8 x i64> %x0, <8 x i32> zeroinitializer, i8 %x2)
+    %res3 = add <8 x i32> %res0, %res1
+    %res4 = add <8 x i32> %res3, %res2
+    ret <8 x i32> %res4
+}
+
+declare void @llvm.x86.avx512.mask.pmovs.qd.mem.512(i8* %ptr, <8 x i64>, i8)
+
+define void @test_int_x86_avx512_mask_pmovs_qd_mem_512(i8* %ptr, <8 x i64> %x1, i8 %x2) {
+; CHECK-LABEL: test_int_x86_avx512_mask_pmovs_qd_mem_512:
+; CHECK:  vpmovsqd %zmm0, (%rdi)
+; CHECK:  vpmovsqd %zmm0, (%rdi) {%k1}
+    call void @llvm.x86.avx512.mask.pmovs.qd.mem.512(i8* %ptr, <8 x i64> %x1, i8 -1)
+    call void @llvm.x86.avx512.mask.pmovs.qd.mem.512(i8* %ptr, <8 x i64> %x1, i8 %x2)
+    ret void
+}
+
+declare <8 x i32> @llvm.x86.avx512.mask.pmovus.qd.512(<8 x i64>, <8 x i32>, i8)
+
+define <8 x i32>@test_int_x86_avx512_mask_pmovus_qd_512(<8 x i64> %x0, <8 x i32> %x1, i8 %x2) {
+; CHECK-LABEL: test_int_x86_avx512_mask_pmovus_qd_512:
+; CHECK:       vpmovusqd %zmm0, %ymm1 {%k1}
+; CHECK-NEXT:  vpmovusqd %zmm0, %ymm2 {%k1} {z}
+; CHECK-NEXT:  vpmovusqd %zmm0, %ymm0
+    %res0 = call <8 x i32> @llvm.x86.avx512.mask.pmovus.qd.512(<8 x i64> %x0, <8 x i32> %x1, i8 -1)
+    %res1 = call <8 x i32> @llvm.x86.avx512.mask.pmovus.qd.512(<8 x i64> %x0, <8 x i32> %x1, i8 %x2)
+    %res2 = call <8 x i32> @llvm.x86.avx512.mask.pmovus.qd.512(<8 x i64> %x0, <8 x i32> zeroinitializer, i8 %x2)
+    %res3 = add <8 x i32> %res0, %res1
+    %res4 = add <8 x i32> %res3, %res2
+    ret <8 x i32> %res4
+}
+
+declare void @llvm.x86.avx512.mask.pmovus.qd.mem.512(i8* %ptr, <8 x i64>, i8)
+
+define void @test_int_x86_avx512_mask_pmovus_qd_mem_512(i8* %ptr, <8 x i64> %x1, i8 %x2) {
+; CHECK-LABEL: test_int_x86_avx512_mask_pmovus_qd_mem_512:
+; CHECK:  vpmovusqd %zmm0, (%rdi)
+; CHECK:  vpmovusqd %zmm0, (%rdi) {%k1}
+    call void @llvm.x86.avx512.mask.pmovus.qd.mem.512(i8* %ptr, <8 x i64> %x1, i8 -1)
+    call void @llvm.x86.avx512.mask.pmovus.qd.mem.512(i8* %ptr, <8 x i64> %x1, i8 %x2)
+    ret void
+}
+
+declare <16 x i8> @llvm.x86.avx512.mask.pmov.db.512(<16 x i32>, <16 x i8>, i16)
+
+define <16 x i8>@test_int_x86_avx512_mask_pmov_db_512(<16 x i32> %x0, <16 x i8> %x1, i16 %x2) {
+; CHECK-LABEL: test_int_x86_avx512_mask_pmov_db_512:
+; CHECK:       vpmovdb %zmm0, %xmm1 {%k1}
+; CHECK-NEXT:  vpmovdb %zmm0, %xmm2 {%k1} {z}
+; CHECK-NEXT:  vpmovdb %zmm0, %xmm0
+    %res0 = call <16 x i8> @llvm.x86.avx512.mask.pmov.db.512(<16 x i32> %x0, <16 x i8> %x1, i16 -1)
+    %res1 = call <16 x i8> @llvm.x86.avx512.mask.pmov.db.512(<16 x i32> %x0, <16 x i8> %x1, i16 %x2)
+    %res2 = call <16 x i8> @llvm.x86.avx512.mask.pmov.db.512(<16 x i32> %x0, <16 x i8> zeroinitializer, i16 %x2)
+    %res3 = add <16 x i8> %res0, %res1
+    %res4 = add <16 x i8> %res3, %res2
+    ret <16 x i8> %res4
+}
+
+declare void @llvm.x86.avx512.mask.pmov.db.mem.512(i8* %ptr, <16 x i32>, i16)
+
+define void @test_int_x86_avx512_mask_pmov_db_mem_512(i8* %ptr, <16 x i32> %x1, i16 %x2) {
+; CHECK-LABEL: test_int_x86_avx512_mask_pmov_db_mem_512:
+; CHECK:  vpmovdb %zmm0, (%rdi)
+; CHECK:  vpmovdb %zmm0, (%rdi) {%k1}
+    call void @llvm.x86.avx512.mask.pmov.db.mem.512(i8* %ptr, <16 x i32> %x1, i16 -1)
+    call void @llvm.x86.avx512.mask.pmov.db.mem.512(i8* %ptr, <16 x i32> %x1, i16 %x2)
+    ret void
+}
+
+declare <16 x i8> @llvm.x86.avx512.mask.pmovs.db.512(<16 x i32>, <16 x i8>, i16)
+
+define <16 x i8>@test_int_x86_avx512_mask_pmovs_db_512(<16 x i32> %x0, <16 x i8> %x1, i16 %x2) {
+; CHECK-LABEL: test_int_x86_avx512_mask_pmovs_db_512:
+; CHECK:       vpmovsdb %zmm0, %xmm1 {%k1}
+; CHECK-NEXT:  vpmovsdb %zmm0, %xmm2 {%k1} {z}
+; CHECK-NEXT:  vpmovsdb %zmm0, %xmm0
+    %res0 = call <16 x i8> @llvm.x86.avx512.mask.pmovs.db.512(<16 x i32> %x0, <16 x i8> %x1, i16 -1)
+    %res1 = call <16 x i8> @llvm.x86.avx512.mask.pmovs.db.512(<16 x i32> %x0, <16 x i8> %x1, i16 %x2)
+    %res2 = call <16 x i8> @llvm.x86.avx512.mask.pmovs.db.512(<16 x i32> %x0, <16 x i8> zeroinitializer, i16 %x2)
+    %res3 = add <16 x i8> %res0, %res1
+    %res4 = add <16 x i8> %res3, %res2
+    ret <16 x i8> %res4
+}
+
+declare void @llvm.x86.avx512.mask.pmovs.db.mem.512(i8* %ptr, <16 x i32>, i16)
+
+define void @test_int_x86_avx512_mask_pmovs_db_mem_512(i8* %ptr, <16 x i32> %x1, i16 %x2) {
+; CHECK-LABEL: test_int_x86_avx512_mask_pmovs_db_mem_512:
+; CHECK:  vpmovsdb %zmm0, (%rdi)
+; CHECK:  vpmovsdb %zmm0, (%rdi) {%k1}
+    call void @llvm.x86.avx512.mask.pmovs.db.mem.512(i8* %ptr, <16 x i32> %x1, i16 -1)
+    call void @llvm.x86.avx512.mask.pmovs.db.mem.512(i8* %ptr, <16 x i32> %x1, i16 %x2)
+    ret void
+}
+
+declare <16 x i8> @llvm.x86.avx512.mask.pmovus.db.512(<16 x i32>, <16 x i8>, i16)
+
+define <16 x i8>@test_int_x86_avx512_mask_pmovus_db_512(<16 x i32> %x0, <16 x i8> %x1, i16 %x2) {
+; CHECK-LABEL: test_int_x86_avx512_mask_pmovus_db_512:
+; CHECK:       vpmovusdb %zmm0, %xmm1 {%k1}
+; CHECK-NEXT:  vpmovusdb %zmm0, %xmm2 {%k1} {z}
+; CHECK-NEXT:  vpmovusdb %zmm0, %xmm0
+    %res0 = call <16 x i8> @llvm.x86.avx512.mask.pmovus.db.512(<16 x i32> %x0, <16 x i8> %x1, i16 -1)
+    %res1 = call <16 x i8> @llvm.x86.avx512.mask.pmovus.db.512(<16 x i32> %x0, <16 x i8> %x1, i16 %x2)
+    %res2 = call <16 x i8> @llvm.x86.avx512.mask.pmovus.db.512(<16 x i32> %x0, <16 x i8> zeroinitializer, i16 %x2)
+    %res3 = add <16 x i8> %res0, %res1
+    %res4 = add <16 x i8> %res3, %res2
+    ret <16 x i8> %res4
+}
+
+declare void @llvm.x86.avx512.mask.pmovus.db.mem.512(i8* %ptr, <16 x i32>, i16)
+
+define void @test_int_x86_avx512_mask_pmovus_db_mem_512(i8* %ptr, <16 x i32> %x1, i16 %x2) {
+; CHECK-LABEL: test_int_x86_avx512_mask_pmovus_db_mem_512:
+; CHECK:  vpmovusdb %zmm0, (%rdi)
+; CHECK:  vpmovusdb %zmm0, (%rdi) {%k1}
+    call void @llvm.x86.avx512.mask.pmovus.db.mem.512(i8* %ptr, <16 x i32> %x1, i16 -1)
+    call void @llvm.x86.avx512.mask.pmovus.db.mem.512(i8* %ptr, <16 x i32> %x1, i16 %x2)
+    ret void
+}
+
+declare <16 x i16> @llvm.x86.avx512.mask.pmov.dw.512(<16 x i32>, <16 x i16>, i16)
+
+define <16 x i16>@test_int_x86_avx512_mask_pmov_dw_512(<16 x i32> %x0, <16 x i16> %x1, i16 %x2) {
+; CHECK-LABEL: test_int_x86_avx512_mask_pmov_dw_512:
+; CHECK:       vpmovdw %zmm0, %ymm1 {%k1}
+; CHECK-NEXT:  vpmovdw %zmm0, %ymm2 {%k1} {z}
+; CHECK-NEXT:  vpmovdw %zmm0, %ymm0
+    %res0 = call <16 x i16> @llvm.x86.avx512.mask.pmov.dw.512(<16 x i32> %x0, <16 x i16> %x1, i16 -1)
+    %res1 = call <16 x i16> @llvm.x86.avx512.mask.pmov.dw.512(<16 x i32> %x0, <16 x i16> %x1, i16 %x2)
+    %res2 = call <16 x i16> @llvm.x86.avx512.mask.pmov.dw.512(<16 x i32> %x0, <16 x i16> zeroinitializer, i16 %x2)
+    %res3 = add <16 x i16> %res0, %res1
+    %res4 = add <16 x i16> %res3, %res2
+    ret <16 x i16> %res4
+}
+
+declare void @llvm.x86.avx512.mask.pmov.dw.mem.512(i8* %ptr, <16 x i32>, i16)
+
+define void @test_int_x86_avx512_mask_pmov_dw_mem_512(i8* %ptr, <16 x i32> %x1, i16 %x2) {
+; CHECK-LABEL: test_int_x86_avx512_mask_pmov_dw_mem_512:
+; CHECK:  vpmovdw %zmm0, (%rdi)
+; CHECK:  vpmovdw %zmm0, (%rdi) {%k1}
+    call void @llvm.x86.avx512.mask.pmov.dw.mem.512(i8* %ptr, <16 x i32> %x1, i16 -1)
+    call void @llvm.x86.avx512.mask.pmov.dw.mem.512(i8* %ptr, <16 x i32> %x1, i16 %x2)
+    ret void
+}
+
+declare <16 x i16> @llvm.x86.avx512.mask.pmovs.dw.512(<16 x i32>, <16 x i16>, i16)
+
+define <16 x i16>@test_int_x86_avx512_mask_pmovs_dw_512(<16 x i32> %x0, <16 x i16> %x1, i16 %x2) {
+; CHECK-LABEL: test_int_x86_avx512_mask_pmovs_dw_512:
+; CHECK:       vpmovsdw %zmm0, %ymm1 {%k1}
+; CHECK-NEXT:  vpmovsdw %zmm0, %ymm2 {%k1} {z}
+; CHECK-NEXT:  vpmovsdw %zmm0, %ymm0
+    %res0 = call <16 x i16> @llvm.x86.avx512.mask.pmovs.dw.512(<16 x i32> %x0, <16 x i16> %x1, i16 -1)
+    %res1 = call <16 x i16> @llvm.x86.avx512.mask.pmovs.dw.512(<16 x i32> %x0, <16 x i16> %x1, i16 %x2)
+    %res2 = call <16 x i16> @llvm.x86.avx512.mask.pmovs.dw.512(<16 x i32> %x0, <16 x i16> zeroinitializer, i16 %x2)
+    %res3 = add <16 x i16> %res0, %res1
+    %res4 = add <16 x i16> %res3, %res2
+    ret <16 x i16> %res4
+}
+
+declare void @llvm.x86.avx512.mask.pmovs.dw.mem.512(i8* %ptr, <16 x i32>, i16)
+
+define void @test_int_x86_avx512_mask_pmovs_dw_mem_512(i8* %ptr, <16 x i32> %x1, i16 %x2) {
+; CHECK-LABEL: test_int_x86_avx512_mask_pmovs_dw_mem_512:
+; CHECK:  vpmovsdw %zmm0, (%rdi)
+; CHECK:  vpmovsdw %zmm0, (%rdi) {%k1}
+    call void @llvm.x86.avx512.mask.pmovs.dw.mem.512(i8* %ptr, <16 x i32> %x1, i16 -1)
+    call void @llvm.x86.avx512.mask.pmovs.dw.mem.512(i8* %ptr, <16 x i32> %x1, i16 %x2)
+    ret void
+}
+
+declare <16 x i16> @llvm.x86.avx512.mask.pmovus.dw.512(<16 x i32>, <16 x i16>, i16)
+
+define <16 x i16>@test_int_x86_avx512_mask_pmovus_dw_512(<16 x i32> %x0, <16 x i16> %x1, i16 %x2) {
+; CHECK-LABEL: test_int_x86_avx512_mask_pmovus_dw_512:
+; CHECK:       vpmovusdw %zmm0, %ymm1 {%k1}
+; CHECK-NEXT:  vpmovusdw %zmm0, %ymm2 {%k1} {z}
+; CHECK-NEXT:  vpmovusdw %zmm0, %ymm0
+    %res0 = call <16 x i16> @llvm.x86.avx512.mask.pmovus.dw.512(<16 x i32> %x0, <16 x i16> %x1, i16 -1)
+    %res1 = call <16 x i16> @llvm.x86.avx512.mask.pmovus.dw.512(<16 x i32> %x0, <16 x i16> %x1, i16 %x2)
+    %res2 = call <16 x i16> @llvm.x86.avx512.mask.pmovus.dw.512(<16 x i32> %x0, <16 x i16> zeroinitializer, i16 %x2)
+    %res3 = add <16 x i16> %res0, %res1
+    %res4 = add <16 x i16> %res3, %res2
+    ret <16 x i16> %res4
+}
+
+declare void @llvm.x86.avx512.mask.pmovus.dw.mem.512(i8* %ptr, <16 x i32>, i16)
+
+define void @test_int_x86_avx512_mask_pmovus_dw_mem_512(i8* %ptr, <16 x i32> %x1, i16 %x2) {
+; CHECK-LABEL: test_int_x86_avx512_mask_pmovus_dw_mem_512:
+; CHECK:  vpmovusdw %zmm0, (%rdi)
+; CHECK:  vpmovusdw %zmm0, (%rdi) {%k1}
+    call void @llvm.x86.avx512.mask.pmovus.dw.mem.512(i8* %ptr, <16 x i32> %x1, i16 -1)
+    call void @llvm.x86.avx512.mask.pmovus.dw.mem.512(i8* %ptr, <16 x i32> %x1, i16 %x2)
+    ret void
+}
+
+declare <8 x double> @llvm.x86.avx512.mask.cvtdq2pd.512(<8 x i32>, <8 x double>, i8)
+
+define <8 x double>@test_int_x86_avx512_mask_cvt_dq2pd_512(<8 x i32> %x0, <8 x double> %x1, i8 %x2) {
+; CHECK-LABEL: test_int_x86_avx512_mask_cvt_dq2pd_512:
+; CHECK:       ## BB#0:
+; CHECK-NEXT:    movzbl %dil, %eax
+; CHECK-NEXT:    kmovw %eax, %k1
+; CHECK-NEXT:    vcvtdq2pd %ymm0, %zmm1 {%k1}
+; CHECK-NEXT:    vcvtdq2pd %ymm0, %zmm0
+; CHECK-NEXT:    vaddpd %zmm0, %zmm1, %zmm0
+; CHECK-NEXT:    retq
+  %res = call <8 x double> @llvm.x86.avx512.mask.cvtdq2pd.512(<8 x i32> %x0, <8 x double> %x1, i8 %x2)
+  %res1 = call <8 x double> @llvm.x86.avx512.mask.cvtdq2pd.512(<8 x i32> %x0, <8 x double> %x1, i8 -1)
+  %res2 = fadd <8 x double> %res, %res1
+  ret <8 x double> %res2
+}
+
+declare <16 x float> @llvm.x86.avx512.mask.cvtdq2ps.512(<16 x i32>, <16 x float>, i16, i32)
+
+define <16 x float>@test_int_x86_avx512_mask_cvt_dq2ps_512(<16 x i32> %x0, <16 x float> %x1, i16 %x2) {
+; CHECK-LABEL: test_int_x86_avx512_mask_cvt_dq2ps_512:
+; CHECK:       ## BB#0:
+; CHECK-NEXT:    kmovw %edi, %k1
+; CHECK-NEXT:    vcvtdq2ps %zmm0, %zmm1 {%k1}
+; CHECK-NEXT:    vcvtdq2ps {rn-sae}, %zmm0, %zmm0
+; CHECK-NEXT:    vaddps %zmm0, %zmm1, %zmm0
+; CHECK-NEXT:    retq
+  %res = call <16 x float> @llvm.x86.avx512.mask.cvtdq2ps.512(<16 x i32> %x0, <16 x float> %x1, i16 %x2, i32 4)
+  %res1 = call <16 x float> @llvm.x86.avx512.mask.cvtdq2ps.512(<16 x i32> %x0, <16 x float> %x1, i16 -1, i32 0)
+  %res2 = fadd <16 x float> %res, %res1
+  ret <16 x float> %res2
+}
+
+declare <8 x i32> @llvm.x86.avx512.mask.cvtpd2dq.512(<8 x double>, <8 x i32>, i8, i32)
+
+define <8 x i32>@test_int_x86_avx512_mask_cvt_pd2dq_512(<8 x double> %x0, <8 x i32> %x1, i8 %x2) {
+; CHECK-LABEL: test_int_x86_avx512_mask_cvt_pd2dq_512:
+; CHECK:       ## BB#0:
+; CHECK-NEXT:    movzbl %dil, %eax
+; CHECK-NEXT:    kmovw %eax, %k1
+; CHECK-NEXT:    vcvtpd2dq %zmm0, %ymm1 {%k1}
+; CHECK-NEXT:    vcvtpd2dq {rn-sae}, %zmm0, %ymm0
+; CHECK-NEXT:    vpaddd %ymm0, %ymm1, %ymm0
+; CHECK-NEXT:    retq
+  %res = call <8 x i32> @llvm.x86.avx512.mask.cvtpd2dq.512(<8 x double> %x0, <8 x i32> %x1, i8 %x2, i32 4)
+  %res1 = call <8 x i32> @llvm.x86.avx512.mask.cvtpd2dq.512(<8 x double> %x0, <8 x i32> %x1, i8 -1, i32 0)
+  %res2 = add <8 x i32> %res, %res1
+  ret <8 x i32> %res2
+}
+
+declare <8 x float> @llvm.x86.avx512.mask.cvtpd2ps.512(<8 x double>, <8 x float>, i8, i32)
+
+define <8 x float>@test_int_x86_avx512_mask_cvt_pd2ps_512(<8 x double> %x0, <8 x float> %x1, i8 %x2) {
+; CHECK-LABEL: test_int_x86_avx512_mask_cvt_pd2ps_512:
+; CHECK:       ## BB#0:
+; CHECK-NEXT:    movzbl %dil, %eax
+; CHECK-NEXT:    kmovw %eax, %k1
+; CHECK-NEXT:    vcvtpd2ps %zmm0, %ymm1 {%k1}
+; CHECK-NEXT:    vcvtpd2ps {ru-sae}, %zmm0, %ymm0
+; CHECK-NEXT:    vaddps %ymm0, %ymm1, %ymm0
+; CHECK-NEXT:    retq
+  %res = call <8 x float> @llvm.x86.avx512.mask.cvtpd2ps.512(<8 x double> %x0, <8 x float> %x1, i8 %x2, i32 4)
+  %res1 = call <8 x float> @llvm.x86.avx512.mask.cvtpd2ps.512(<8 x double> %x0, <8 x float> %x1, i8 -1, i32 2)
+  %res2 = fadd <8 x float> %res, %res1
+  ret <8 x float> %res2
+}
+
+declare <8 x i32> @llvm.x86.avx512.mask.cvtpd2udq.512(<8 x double>, <8 x i32>, i8, i32)
+
+define <8 x i32>@test_int_x86_avx512_mask_cvt_pd2udq_512(<8 x double> %x0, <8 x i32> %x1, i8 %x2) {
+; CHECK-LABEL: test_int_x86_avx512_mask_cvt_pd2udq_512:
+; CHECK:       ## BB#0:
+; CHECK-NEXT:    movzbl %dil, %eax
+; CHECK-NEXT:    kmovw %eax, %k1
+; CHECK-NEXT:    vcvtpd2udq {ru-sae}, %zmm0, %ymm1 {%k1}
+; CHECK-NEXT:    vcvtpd2udq {rn-sae}, %zmm0, %ymm0
+; CHECK-NEXT:    vpaddd %ymm0, %ymm1, %ymm0
+; CHECK-NEXT:    retq
+  %res = call <8 x i32> @llvm.x86.avx512.mask.cvtpd2udq.512(<8 x double> %x0, <8 x i32> %x1, i8 %x2, i32 2)
+  %res1 = call <8 x i32> @llvm.x86.avx512.mask.cvtpd2udq.512(<8 x double> %x0, <8 x i32> %x1, i8 -1, i32 0)
+  %res2 = add <8 x i32> %res, %res1
+  ret <8 x i32> %res2
+}
+
+declare <16 x i32> @llvm.x86.avx512.mask.cvtps2dq.512(<16 x float>, <16 x i32>, i16, i32)
+
+define <16 x i32>@test_int_x86_avx512_mask_cvt_ps2dq_512(<16 x float> %x0, <16 x i32> %x1, i16 %x2) {
+; CHECK-LABEL: test_int_x86_avx512_mask_cvt_ps2dq_512:
+; CHECK:       ## BB#0:
+; CHECK-NEXT:    kmovw %edi, %k1
+; CHECK-NEXT:    vcvtps2dq {ru-sae}, %zmm0, %zmm1 {%k1}
+; CHECK-NEXT:    vcvtps2dq {rn-sae}, %zmm0, %zmm0
+; CHECK-NEXT:    vpaddd %zmm0, %zmm1, %zmm0
+; CHECK-NEXT:    retq
+  %res = call <16 x i32> @llvm.x86.avx512.mask.cvtps2dq.512(<16 x float> %x0, <16 x i32> %x1, i16 %x2, i32 2)
+  %res1 = call <16 x i32> @llvm.x86.avx512.mask.cvtps2dq.512(<16 x float> %x0, <16 x i32> %x1, i16 -1, i32 0)
+  %res2 = add <16 x i32> %res, %res1
+  ret <16 x i32> %res2
+}
+
+declare <8 x double> @llvm.x86.avx512.mask.cvtps2pd.512(<8 x float>, <8 x double>, i8, i32)
+
+define <8 x double>@test_int_x86_avx512_mask_cvt_ps2pd_512(<8 x float> %x0, <8 x double> %x1, i8 %x2) {
+; CHECK-LABEL: test_int_x86_avx512_mask_cvt_ps2pd_512:
+; CHECK:       ## BB#0:
+; CHECK-NEXT:    movzbl %dil, %eax
+; CHECK-NEXT:    kmovw %eax, %k1
+; CHECK-NEXT:    vcvtps2pd %ymm0, %zmm1 {%k1}
+; CHECK-NEXT:    vcvtps2pd {sae}, %ymm0, %zmm0
+; CHECK-NEXT:    vaddpd %zmm0, %zmm1, %zmm0
+; CHECK-NEXT:    retq
+  %res = call <8 x double> @llvm.x86.avx512.mask.cvtps2pd.512(<8 x float> %x0, <8 x double> %x1, i8 %x2, i32 4)
+  %res1 = call <8 x double> @llvm.x86.avx512.mask.cvtps2pd.512(<8 x float> %x0, <8 x double> %x1, i8 -1, i32 8)
+  %res2 = fadd <8 x double> %res, %res1
+  ret <8 x double> %res2
+}
+
+declare <16 x i32> @llvm.x86.avx512.mask.cvtps2udq.512(<16 x float>, <16 x i32>, i16, i32)
+
+define <16 x i32>@test_int_x86_avx512_mask_cvt_ps2udq_512(<16 x float> %x0, <16 x i32> %x1, i16 %x2) {
+; CHECK-LABEL: test_int_x86_avx512_mask_cvt_ps2udq_512:
+; CHECK:       ## BB#0:
+; CHECK-NEXT:    kmovw %edi, %k1
+; CHECK-NEXT:    vcvtps2udq {ru-sae}, %zmm0, %zmm1 {%k1}
+; CHECK-NEXT:    vcvtps2udq {rn-sae}, %zmm0, %zmm0
+; CHECK-NEXT:    vpaddd %zmm0, %zmm1, %zmm0
+; CHECK-NEXT:    retq
+  %res = call <16 x i32> @llvm.x86.avx512.mask.cvtps2udq.512(<16 x float> %x0, <16 x i32> %x1, i16 %x2, i32 2)
+  %res1 = call <16 x i32> @llvm.x86.avx512.mask.cvtps2udq.512(<16 x float> %x0, <16 x i32> %x1, i16 -1, i32 0)
+  %res2 = add <16 x i32> %res, %res1
+  ret <16 x i32> %res2
+}
+
+declare <8 x i32> @llvm.x86.avx512.mask.cvttpd2dq.512(<8 x double>, <8 x i32>, i8, i32)
+
+define <8 x i32>@test_int_x86_avx512_mask_cvtt_pd2dq_512(<8 x double> %x0, <8 x i32> %x1, i8 %x2) {
+; CHECK-LABEL: test_int_x86_avx512_mask_cvtt_pd2dq_512:
+; CHECK:       ## BB#0:
+; CHECK-NEXT:    movzbl %dil, %eax
+; CHECK-NEXT:    kmovw %eax, %k1
+; CHECK-NEXT:    vcvttpd2dq %zmm0, %ymm1 {%k1}
+; CHECK-NEXT:    vcvttpd2dq {sae}, %zmm0, %ymm0
+; CHECK-NEXT:    vpaddd %ymm0, %ymm1, %ymm0
+; CHECK-NEXT:    retq
+  %res = call <8 x i32> @llvm.x86.avx512.mask.cvttpd2dq.512(<8 x double> %x0, <8 x i32> %x1, i8 %x2, i32 4)
+  %res1 = call <8 x i32> @llvm.x86.avx512.mask.cvttpd2dq.512(<8 x double> %x0, <8 x i32> %x1, i8 -1, i32 8)
+  %res2 = add <8 x i32> %res, %res1
+  ret <8 x i32> %res2
+}
+
+declare <8 x double> @llvm.x86.avx512.mask.cvtudq2pd.512(<8 x i32>, <8 x double>, i8)
+
+define <8 x double>@test_int_x86_avx512_mask_cvt_udq2pd_512(<8 x i32> %x0, <8 x double> %x1, i8 %x2) {
+; CHECK-LABEL: test_int_x86_avx512_mask_cvt_udq2pd_512:
+; CHECK:       ## BB#0:
+; CHECK-NEXT:    movzbl %dil, %eax
+; CHECK-NEXT:    kmovw %eax, %k1
+; CHECK-NEXT:    vcvtudq2pd %ymm0, %zmm1 {%k1}
+; CHECK-NEXT:    vcvtudq2pd %ymm0, %zmm0
+; CHECK-NEXT:    vaddpd %zmm0, %zmm1, %zmm0
+; CHECK-NEXT:    retq
+  %res = call <8 x double> @llvm.x86.avx512.mask.cvtudq2pd.512(<8 x i32> %x0, <8 x double> %x1, i8 %x2)
+  %res1 = call <8 x double> @llvm.x86.avx512.mask.cvtudq2pd.512(<8 x i32> %x0, <8 x double> %x1, i8 -1)
+  %res2 = fadd <8 x double> %res, %res1
+  ret <8 x double> %res2
+}
+
+
+declare <16 x float> @llvm.x86.avx512.mask.cvtudq2ps.512(<16 x i32>, <16 x float>, i16, i32)
+
+define <16 x float>@test_int_x86_avx512_mask_cvt_udq2ps_512(<16 x i32> %x0, <16 x float> %x1, i16 %x2) {
+; CHECK-LABEL: test_int_x86_avx512_mask_cvt_udq2ps_512:
+; CHECK:       ## BB#0:
+; CHECK-NEXT:    kmovw %edi, %k1
+; CHECK-NEXT:    vcvtudq2ps %zmm0, %zmm1 {%k1}
+; CHECK-NEXT:    vcvtudq2ps {rn-sae}, %zmm0, %zmm0
+; CHECK-NEXT:    vaddps %zmm0, %zmm1, %zmm0
+; CHECK-NEXT:    retq
+  %res = call <16 x float> @llvm.x86.avx512.mask.cvtudq2ps.512(<16 x i32> %x0, <16 x float> %x1, i16 %x2, i32 4)
+  %res1 = call <16 x float> @llvm.x86.avx512.mask.cvtudq2ps.512(<16 x i32> %x0, <16 x float> %x1, i16 -1, i32 0)
+  %res2 = fadd <16 x float> %res, %res1
+  ret <16 x float> %res2
+}
+
+declare <8 x i32> @llvm.x86.avx512.mask.cvttpd2udq.512(<8 x double>, <8 x i32>, i8, i32)
+
+define <8 x i32>@test_int_x86_avx512_mask_cvtt_pd2udq_512(<8 x double> %x0, <8 x i32> %x1, i8 %x2) {
+; CHECK-LABEL: test_int_x86_avx512_mask_cvtt_pd2udq_512:
+; CHECK:       ## BB#0:
+; CHECK-NEXT:    movzbl %dil, %eax
+; CHECK-NEXT:    kmovw %eax, %k1
+; CHECK-NEXT:    vcvttpd2udq %zmm0, %ymm1 {%k1}
+; CHECK-NEXT:    vcvttpd2udq {sae}, %zmm0, %ymm0
+; CHECK-NEXT:    vpaddd %ymm0, %ymm1, %ymm0
+; CHECK-NEXT:    retq
+  %res = call <8 x i32> @llvm.x86.avx512.mask.cvttpd2udq.512(<8 x double> %x0, <8 x i32> %x1, i8 %x2, i32 4)
+  %res1 = call <8 x i32> @llvm.x86.avx512.mask.cvttpd2udq.512(<8 x double> %x0, <8 x i32> %x1, i8 -1, i32 8)
+  %res2 = add <8 x i32> %res, %res1
+  ret <8 x i32> %res2
+}
+
+declare <16 x i32> @llvm.x86.avx512.mask.cvttps2dq.512(<16 x float>, <16 x i32>, i16, i32)
+
+define <16 x i32>@test_int_x86_avx512_mask_cvtt_ps2dq_512(<16 x float> %x0, <16 x i32> %x1, i16 %x2) {
+; CHECK-LABEL: test_int_x86_avx512_mask_cvtt_ps2dq_512:
+; CHECK:       ## BB#0:
+; CHECK-NEXT:    kmovw %edi, %k1
+; CHECK-NEXT:    vcvttps2dq %zmm0, %zmm1 {%k1}
+; CHECK-NEXT:    vcvttps2dq {sae}, %zmm0, %zmm0
+; CHECK-NEXT:    vpaddd %zmm0, %zmm1, %zmm0
+; CHECK-NEXT:    retq
+  %res = call <16 x i32> @llvm.x86.avx512.mask.cvttps2dq.512(<16 x float> %x0, <16 x i32> %x1, i16 %x2, i32 4)
+  %res1 = call <16 x i32> @llvm.x86.avx512.mask.cvttps2dq.512(<16 x float> %x0, <16 x i32> %x1, i16 -1, i32 8)
+  %res2 = add <16 x i32> %res, %res1
+  ret <16 x i32> %res2
+}
+
+declare <16 x i32> @llvm.x86.avx512.mask.cvttps2udq.512(<16 x float>, <16 x i32>, i16, i32)
+
+define <16 x i32>@test_int_x86_avx512_mask_cvtt_ps2udq_512(<16 x float> %x0, <16 x i32> %x1, i16 %x2) {
+; CHECK-LABEL: test_int_x86_avx512_mask_cvtt_ps2udq_512:
+; CHECK:       ## BB#0:
+; CHECK-NEXT:    kmovw %edi, %k1
+; CHECK-NEXT:    vcvttps2udq %zmm0, %zmm1 {%k1}
+; CHECK-NEXT:    vcvttps2udq {sae}, %zmm0, %zmm0
+; CHECK-NEXT:    vpaddd %zmm0, %zmm1, %zmm0
+; CHECK-NEXT:    retq
+  %res = call <16 x i32> @llvm.x86.avx512.mask.cvttps2udq.512(<16 x float> %x0, <16 x i32> %x1, i16 %x2, i32 4)
+  %res1 = call <16 x i32> @llvm.x86.avx512.mask.cvttps2udq.512(<16 x float> %x0, <16 x i32> %x1, i16 -1, i32 8)
+  %res2 = add <16 x i32> %res, %res1
+  ret <16 x i32> %res2
+}
+
+
+declare <4 x float> @llvm.x86.avx512.mask.scalef.ss(<4 x float>, <4 x float>,<4 x float>, i8, i32)
+; CHECK-LABEL: @test_int_x86_avx512_mask_scalef_ss
+; CHECK-NOT: call 
+; CHECK: kmov 
+; CHECK: vscalefss {{.*}}{%k1} 
+; CHECK: vscalefss	{rn-sae}
+define <4 x float>@test_int_x86_avx512_mask_scalef_ss(<4 x float> %x0, <4 x float> %x1, <4 x float> %x3, i8 %x4) {
+  %res = call <4 x float> @llvm.x86.avx512.mask.scalef.ss(<4 x float> %x0, <4 x float> %x1, <4 x float> %x3, i8 %x4, i32 4)
+  %res1 = call <4 x float> @llvm.x86.avx512.mask.scalef.ss(<4 x float> %x0, <4 x float> %x1, <4 x float> %x3, i8 -1, i32 8)
+  %res2 = fadd <4 x float> %res, %res1
+  ret <4 x float> %res2
+}
+
+declare <2 x double> @llvm.x86.avx512.mask.scalef.sd(<2 x double>, <2 x double>,<2 x double>, i8, i32)
+; CHECK-LABEL: @test_int_x86_avx512_mask_scalef_sd
+; CHECK-NOT: call 
+; CHECK: kmov 
+; CHECK: vscalefsd {{.*}}{%k1} 
+; CHECK: vscalefsd	{rn-sae}
+define <2 x double>@test_int_x86_avx512_mask_scalef_sd(<2 x double> %x0, <2 x double> %x1, <2 x double> %x3, i8 %x4) {
+  %res = call <2 x double> @llvm.x86.avx512.mask.scalef.sd(<2 x double> %x0, <2 x double> %x1, <2 x double> %x3, i8 %x4, i32 4)
+  %res1 = call <2 x double> @llvm.x86.avx512.mask.scalef.sd(<2 x double> %x0, <2 x double> %x1, <2 x double> %x3, i8 -1, i32 8)
+  %res2 = fadd <2 x double> %res, %res1
+  ret <2 x double> %res2
+}
+
+declare <4 x float> @llvm.x86.avx512.mask.getexp.ss(<4 x float>, <4 x float>, <4 x float>, i8, i32) nounwind readnone
+
+define <4 x float> @test_getexp_ss(<4 x float> %a0, <4 x float> %a1, <4 x float> %a2, i8 %mask) {
+; CHECK-LABEL: test_getexp_ss:
+; CHECK:       ## BB#0:
+; CHECK-NEXT:    andl $1, %edi
+; CHECK-NEXT:    kmovw %edi, %k1
+; CHECK-NEXT:    vmovaps %zmm2, %zmm3
+; CHECK-NEXT:    vgetexpss %xmm1, %xmm0, %xmm3 {%k1}
+; CHECK-NEXT:    vgetexpss {sae}, %xmm1, %xmm0, %xmm2 {%k1}
+; CHECK-NEXT:    vgetexpss {sae}, %xmm1, %xmm0, %xmm4 {%k1} {z}
+; CHECK-NEXT:    vgetexpss {sae}, %xmm1, %xmm0, %xmm0
+; CHECK-NEXT:    vaddps %xmm2, %xmm3, %xmm1
+; CHECK-NEXT:    vaddps %xmm0, %xmm4, %xmm0
+; CHECK-NEXT:    vaddps %xmm0, %xmm1, %xmm0
+; CHECK-NEXT:    retq
+  %res0 = call <4 x float> @llvm.x86.avx512.mask.getexp.ss(<4 x float>%a0, <4 x float> %a1, <4 x float> %a2, i8 %mask, i32 4)
+  %res1 = call <4 x float> @llvm.x86.avx512.mask.getexp.ss(<4 x float>%a0, <4 x float> %a1, <4 x float> %a2, i8 %mask, i32 8)
+  %res2 = call <4 x float> @llvm.x86.avx512.mask.getexp.ss(<4 x float>%a0, <4 x float> %a1, <4 x float> zeroinitializer, i8 %mask, i32 8)
+  %res3 = call <4 x float> @llvm.x86.avx512.mask.getexp.ss(<4 x float>%a0, <4 x float> %a1, <4 x float> zeroinitializer, i8 -1, i32 8)
+
+  %res.1 = fadd <4 x float> %res0, %res1
+  %res.2 = fadd <4 x float> %res2, %res3
+  %res   = fadd <4 x float> %res.1, %res.2
+  ret <4 x float> %res
+}
+
+declare <2 x double> @llvm.x86.avx512.mask.getexp.sd(<2 x double>, <2 x double>, <2 x double>, i8, i32) nounwind readnone
+
+define <2 x double> @test_getexp_sd(<2 x double> %a0, <2 x double> %a1, <2 x double> %a2, i8 %mask) {
+; CHECK-LABEL: test_getexp_sd:
+; CHECK:       ## BB#0:
+; CHECK-NEXT:    andl $1, %edi
+; CHECK-NEXT:    kmovw %edi, %k1
+; CHECK-NEXT:    vmovaps %zmm2, %zmm3
+; CHECK-NEXT:    vgetexpsd %xmm1, %xmm0, %xmm3 {%k1}
+; CHECK-NEXT:    vgetexpsd %xmm1, %xmm0, %xmm4
+; CHECK-NEXT:    vgetexpsd {sae}, %xmm1, %xmm0, %xmm2 {%k1}
+; CHECK-NEXT:    vgetexpsd {sae}, %xmm1, %xmm0, %xmm0 {%k1} {z}
+; CHECK-NEXT:    vaddpd %xmm2, %xmm3, %xmm1
+; CHECK-NEXT:    vaddpd %xmm4, %xmm0, %xmm0
+; CHECK-NEXT:    vaddpd %xmm0, %xmm1, %xmm0
+; CHECK-NEXT:    retq
+  %res0 = call <2 x double> @llvm.x86.avx512.mask.getexp.sd(<2 x double>%a0, <2 x double> %a1, <2 x double> %a2, i8 %mask, i32 4)
+  %res1 = call <2 x double> @llvm.x86.avx512.mask.getexp.sd(<2 x double>%a0, <2 x double> %a1, <2 x double> %a2, i8 %mask, i32 8)
+  %res2 = call <2 x double> @llvm.x86.avx512.mask.getexp.sd(<2 x double>%a0, <2 x double> %a1, <2 x double> zeroinitializer, i8 %mask, i32 8)
+  %res3 = call <2 x double> @llvm.x86.avx512.mask.getexp.sd(<2 x double>%a0, <2 x double> %a1, <2 x double> zeroinitializer, i8 -1, i32 4)
+
+  %res.1 = fadd <2 x double> %res0, %res1
+  %res.2 = fadd <2 x double> %res2, %res3
+  %res   = fadd <2 x double> %res.1, %res.2
+  ret <2 x double> %res
+}
+
diff --git a/test/CodeGen/X86/avx512-mask-bugfix.ll b/test/CodeGen/X86/avx512-mask-bugfix.ll
new file mode 100755
index 00000000000..1940680f1c1
--- /dev/null
+++ b/test/CodeGen/X86/avx512-mask-bugfix.ll
@@ -0,0 +1,57 @@
+; RUN: llc < %s -mtriple=x86_64-apple-darwin -mcpu=knl  | FileCheck %s
+
+; ModuleID = 'foo.ll'
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+; Function Attrs: nounwind readnone
+declare i32 @llvm.x86.avx.movmsk.ps.256(<8 x float>) #0
+
+; Function Attrs: nounwind readnone
+declare i64 @llvm.cttz.i64(i64, i1) #0
+
+; Function Attrs: nounwind
+define void @foo(float* noalias %aFOO, float %b, i32 %a) {
+allocas:
+  %full_mask_memory.i57 = alloca <8 x float>
+  %return_value_memory.i60 = alloca i1
+  %cmp.i = icmp eq i32 %a, 65535
+  br i1 %cmp.i, label %all_on, label %some_on
+
+all_on:
+  %mask0 = load <8 x float>, <8 x float>* %full_mask_memory.i57
+  %v0.i.i.i70 = call i32 @llvm.x86.avx.movmsk.ps.256(<8 x float> %mask0) #0
+  %allon.i.i76 = icmp eq i32 %v0.i.i.i70, 65535
+  br i1 %allon.i.i76, label %check_neighbors.i.i121, label %domixed.i.i100
+
+domixed.i.i100: 
+  br label %check_neighbors.i.i121
+
+check_neighbors.i.i121: 
+  %v1.i5.i.i116 = call i32 @llvm.x86.avx.movmsk.ps.256(<8 x float> %mask0) #0
+  %alleq.i.i120 = icmp eq i32 %v1.i5.i.i116, 65535
+  br i1 %alleq.i.i120, label %all_equal.i.i123, label %not_all_equal.i.i124
+
+; CHECK: kxnorw  %k0, %k0, %k0
+; CHECK: kshiftrw        $15, %k0, %k0
+; CHECK: jmp
+; CHECK: kxorw   %k0, %k0, %k0
+
+all_equal.i.i123:
+  br label %reduce_equal___vyi.exit128
+
+not_all_equal.i.i124:        
+  br label %reduce_equal___vyi.exit128
+
+reduce_equal___vyi.exit128:
+  %calltmp2.i125 = phi i1 [ true, %all_equal.i.i123 ], [ false, %not_all_equal.i.i124 ]
+  store i1 %calltmp2.i125, i1* %return_value_memory.i60
+  %return_value.i126 = load i1, i1* %return_value_memory.i60
+  %. = select i1 %return_value.i126, i32 1, i32 0
+  %select_to_float = sitofp i32 %. to float
+  ret void
+
+some_on:
+  ret void
+}
+
diff --git a/test/CodeGen/X86/avx512-mask-op.ll b/test/CodeGen/X86/avx512-mask-op.ll
index c4e62517eb0..d2efd7d6db6 100644
--- a/test/CodeGen/X86/avx512-mask-op.ll
+++ b/test/CodeGen/X86/avx512-mask-op.ll
@@ -131,4 +131,279 @@ entry:
   %mask = load <8 x i1>, <8 x i1>* %maskPtr
   %mask_convert = bitcast <8 x i1> %mask to i8
   ret i8 %mask_convert
-}
\ No newline at end of file
+}
+
+; SKX-LABEL: test4
+; SKX: vpcmpgt
+; SKX: knot
+; SKX: vpcmpgt
+; SKX: vpmovm2d
+define <4 x i32> @test4(<4 x i64> %x, <4 x i64> %y, <4 x i64> %x1, <4 x i64> %y1) {
+  %x_gt_y = icmp sgt <4 x i64> %x, %y
+  %x1_gt_y1 = icmp sgt <4 x i64> %x1, %y1
+  %res = icmp sgt <4 x i1>%x_gt_y, %x1_gt_y1
+  %resse = sext <4 x i1>%res to <4 x i32>
+  ret <4 x i32> %resse
+}
+
+; SKX-LABEL: test5
+; SKX: vpcmpgt
+; SKX: knot
+; SKX: vpcmpgt
+; SKX: vpmovm2q
+define <2 x i64> @test5(<2 x i64> %x, <2 x i64> %y, <2 x i64> %x1, <2 x i64> %y1) {
+  %x_gt_y = icmp slt <2 x i64> %x, %y
+  %x1_gt_y1 = icmp sgt <2 x i64> %x1, %y1
+  %res = icmp slt <2 x i1>%x_gt_y, %x1_gt_y1
+  %resse = sext <2 x i1>%res to <2 x i64>
+  ret <2 x i64> %resse
+}
+
+; KNL-LABEL: test6
+; KNL: vpmovsxbd
+; KNL: vpandd
+; KNL: kmovw   %eax, %k1
+; KNL vptestmd {{.*}}, %k0 {%k1}
+
+; SKX-LABEL: test6
+; SKX: vpmovb2m
+; SKX: kmovw   %eax, %k1
+; SKX: kandw
+define void @test6(<16 x i1> %mask)  {
+allocas:
+  %a= and <16 x i1> %mask, <i1 true, i1 false, i1 true, i1 false, i1 true, i1 false, i1 true, i1 false, i1 true, i1 false, i1 true, i1 false, i1 true, i1 false, i1 true, i1 false>
+  %b = bitcast <16 x i1> %a to i16
+  %c = icmp eq i16 %b, 0
+  br i1 %c, label %true, label %false
+
+true:
+  ret void
+
+false:
+  ret void
+}
+
+; KNL-LABEL: test7
+; KNL: vpmovsxwq
+; KNL: vpandq
+; KNL: vptestmq {{.*}}, %k0
+; KNL: korw
+
+; SKX-LABEL: test7
+; SKX: vpmovw2m
+; SKX: kmovb   %eax, %k1
+; SKX: korb
+
+define void @test7(<8 x i1> %mask)  {
+allocas:
+  %a= or <8 x i1> %mask, <i1 true, i1 false, i1 true, i1 false, i1 true, i1 false, i1 true, i1 false>
+  %b = bitcast <8 x i1> %a to i8
+  %c = icmp eq i8 %b, 0
+  br i1 %c, label %true, label %false
+
+true:
+  ret void
+
+false:
+  ret void
+}
+
+; KNL-LABEL: test8
+; KNL: vpxord  %zmm2, %zmm2, %zmm2
+; KNL: jg
+; KNL: vpcmpltud       %zmm2, %zmm1, %k1
+; KNL: jmp
+; KNL: vpcmpgtd        %zmm2, %zmm0, %k1
+
+; SKX-LABEL: test8
+; SKX: jg
+; SKX: vpcmpltud {{.*}}, %k0
+; SKX: vpmovm2b
+; SKX: vpcmpgtd {{.*}}, %k0
+; SKX: vpmovm2b
+
+define <16 x i8> @test8(<16 x i32>%a, <16 x i32>%b, i32 %a1, i32 %b1) {
+  %cond = icmp sgt i32 %a1, %b1
+  %cmp1 = icmp sgt <16 x i32> %a, zeroinitializer
+  %cmp2 = icmp ult <16 x i32> %b, zeroinitializer
+  %mix = select i1 %cond, <16 x i1> %cmp1, <16 x i1> %cmp2
+  %res = sext <16 x i1> %mix to <16 x i8>
+  ret <16 x i8> %res
+}
+
+; KNL-LABEL: test9
+; KNL: jg
+; KNL: vpmovsxbd       %xmm1, %zmm0
+; KNL: jmp
+; KNL: vpmovsxbd       %xmm0, %zmm0
+
+; SKX-LABEL: test9
+; SKX: vpmovb2m        %xmm1, %k0
+; SKX: vpmovm2b        %k0, %xmm0
+; SKX: retq
+; SKX: vpmovb2m        %xmm0, %k0
+; SKX: vpmovm2b        %k0, %xmm0
+
+define <16 x i1> @test9(<16 x i1>%a, <16 x i1>%b, i32 %a1, i32 %b1) {
+  %mask = icmp sgt i32 %a1, %b1
+  %c = select i1 %mask, <16 x i1>%a, <16 x i1>%b
+  ret <16 x i1>%c
+}
+
+; KNL-LABEL: test10
+; KNL: jg
+; KNL: vpmovsxwq       %xmm1, %zmm0
+; KNL: jmp
+; KNL: vpmovsxwq       %xmm0, %zmm0
+
+; SKX-LABEL: test10
+; SKX: jg
+; SKX: vpmovw2m        %xmm1, %k0
+; SKX: vpmovm2w        %k0, %xmm0
+; SKX: retq
+; SKX: vpmovw2m        %xmm0, %k0
+; SKX: vpmovm2w        %k0, %xmm0
+define <8 x i1> @test10(<8 x i1>%a, <8 x i1>%b, i32 %a1, i32 %b1) {
+  %mask = icmp sgt i32 %a1, %b1
+  %c = select i1 %mask, <8 x i1>%a, <8 x i1>%b
+  ret <8 x i1>%c
+}
+
+; SKX-LABEL: test11
+; SKX: jg
+; SKX: vpmovd2m        %xmm1, %k0
+; SKX: vpmovm2d        %k0, %xmm0
+; SKX: retq
+; SKX: vpmovd2m        %xmm0, %k0
+; SKX: vpmovm2d        %k0, %xmm0
+define <4 x i1> @test11(<4 x i1>%a, <4 x i1>%b, i32 %a1, i32 %b1) {
+  %mask = icmp sgt i32 %a1, %b1
+  %c = select i1 %mask, <4 x i1>%a, <4 x i1>%b
+  ret <4 x i1>%c
+}
+
+; KNL-LABEL: test12
+; KNL: movl    %edi, %eax
+define i32 @test12(i32 %x, i32 %y)  {
+  %a = bitcast i16 21845 to <16 x i1>
+  %b = extractelement <16 x i1> %a, i32 0
+  %c = select i1 %b, i32 %x, i32 %y
+  ret i32 %c
+}
+
+; KNL-LABEL: test13
+; KNL: movl    %esi, %eax
+define i32 @test13(i32 %x, i32 %y)  {
+  %a = bitcast i16 21845 to <16 x i1>
+  %b = extractelement <16 x i1> %a, i32 3
+  %c = select i1 %b, i32 %x, i32 %y
+  ret i32 %c
+}
+
+; SKX-LABEL: test14
+; SKX: movb     $11, %al
+; SKX: kmovb    %eax, %k0
+; SKX: vpmovm2d %k0, %xmm0
+
+define <4 x i1> @test14()  {
+  %a = bitcast i16 21845 to <16 x i1>
+  %b = extractelement <16 x i1> %a, i32 2
+  %c = insertelement <4 x i1> <i1 true, i1 false, i1 false, i1 true>, i1 %b, i32 1
+  ret <4 x i1> %c
+}
+
+; KNL-LABEL: test15
+; KNL: cmovgw
+define <16 x i1> @test15(i32 %x, i32 %y)  {
+  %a = bitcast i16 21845 to <16 x i1>
+  %b = bitcast i16 1 to <16 x i1>
+  %mask = icmp sgt i32 %x, %y
+  %c = select i1 %mask, <16 x i1> %a, <16 x i1> %b
+  ret <16 x i1> %c
+}
+
+; SKX-LABEL: test16
+; SKX: kxnorw  %k1, %k1, %k1
+; SKX: kshiftrw        $15, %k1, %k1
+; SKX: kshiftlq        $5, %k1, %k1
+; SKX: korq    %k1, %k0, %k0
+; SKX: vpmovm2b        %k0, %zmm0
+define <64 x i8> @test16(i64 %x) {
+  %a = bitcast i64 %x to <64 x i1>
+  %b = insertelement <64 x i1>%a, i1 true, i32 5
+  %c = sext <64 x i1>%b to <64 x i8>
+  ret <64 x i8>%c
+}
+
+; SKX-LABEL: test17
+; SKX: setg    %al
+; SKX: andl    $1, %eax
+; SKX: kmovw   %eax, %k1
+; SKX: kshiftlq        $5, %k1, %k1
+; SKX: korq    %k1, %k0, %k0
+; SKX: vpmovm2b        %k0, %zmm0
+define <64 x i8> @test17(i64 %x, i32 %y, i32 %z) {
+  %a = bitcast i64 %x to <64 x i1>
+  %b = icmp sgt i32 %y, %z
+  %c = insertelement <64 x i1>%a, i1 %b, i32 5
+  %d = sext <64 x i1>%c to <64 x i8>
+  ret <64 x i8>%d
+}
+
+; KNL-LABEL: test18
+define <8 x i1> @test18(i8 %a, i16 %y) {
+  %b = bitcast i8 %a to <8 x i1>
+  %b1 = bitcast i16 %y to <16 x i1>
+  %el1 = extractelement <16 x i1>%b1, i32 8
+  %el2 = extractelement <16 x i1>%b1, i32 9
+  %c = insertelement <8 x i1>%b, i1 %el1, i32 7
+  %d = insertelement <8 x i1>%c, i1 %el2, i32 6
+  ret <8 x i1>%d
+}
+
+; KNL-LABEL: test19
+; KNL: movzbl  %dil, %eax
+; KNL: kmovw   %eax, %k0
+; KNL: kshiftlw        $13, %k0, %k0
+; KNL: kshiftrw        $15, %k0, %k0
+; KNL: kmovw   %k0, %eax
+; KNL: andl    $1, %eax
+; KNL: testb   %al, %al
+
+define <8 x i1> @test19(i8 %a) {
+  %b = bitcast i8 %a to <8 x i1>
+  %c = shufflevector < 8 x i1>%b, <8 x i1>undef, <8 x i32> <i32 undef, i32 2, i32 undef, i32 undef, i32 2, i32 undef, i32 2, i32 undef>
+  ret <8 x i1> %c
+}
+
+; KNL-LABEL: test20
+; KNL: movzbl  %dil, %eax
+; KNL: kmovw   %eax, %k0
+; KNL: kshiftlw        $13, %k0, %k1
+; KNL: kshiftrw        $15, %k1, %k1
+; KNL: kshiftlw        $12, %k0, %k0
+; KNL: kshiftrw        $15, %k0, %k0
+; KNL: kshiftlw        $4, %k0, %k0
+; KNL: kshiftlw        $1, %k1, %k2
+; KNL: korw    %k0, %k2, %k0
+; KNL: kshiftlw        $6, %k1, %k1
+; KNL: korw    %k1, %k0, %k1
+define <8 x i1> @test20(i8 %a, i16 %y) {
+  %b = bitcast i8 %a to <8 x i1>
+  %c = shufflevector < 8 x i1>%b, <8 x i1>undef, <8 x i32> <i32 undef, i32 2, i32 undef, i32 undef, i32 3, i32 undef, i32 2, i32 undef>
+  ret <8 x i1> %c
+}
+
+; KNL-LABEL: test21
+; KNL: vpand %ymm
+; KNL: vextracti128    $1, %ymm2
+; KNL: vpand %ymm
+
+; SKX-LABEL: test21
+; SKX: vpmovb2m
+; SKX: vmovdqu16 {{.*}}%k1
+
+define <32 x i16> @test21(<32 x i16> %x , <32 x i1> %mask) nounwind readnone {
+  %ret = select <32 x i1> %mask, <32 x i16> %x, <32 x i16> zeroinitializer
+  ret <32 x i16> %ret
+}
diff --git a/test/CodeGen/X86/avx512-select.ll b/test/CodeGen/X86/avx512-select.ll
index 0dbf286d3c5..b92e6f62813 100644
--- a/test/CodeGen/X86/avx512-select.ll
+++ b/test/CodeGen/X86/avx512-select.ll
@@ -50,8 +50,10 @@ define <16 x double> @select04(<16 x double> %a, <16 x double> %b) {
 }
 
 ; CHECK-LABEL: select05
-; CHECK: kmovw   %esi, %k0
-; CHECK-NEXT: kmovw   %edi, %k1
+; CHECK: movzbl  %sil, %eax
+; CHECK: kmovw   %eax, %k0
+; CHECK: movzbl  %dil, %eax
+; CHECK: kmovw   %eax, %k1
 ; CHECK-NEXT: korw    %k1, %k0, %k0
 ; CHECK-NEXT: kmovw   %k0, %eax
 define i8 @select05(i8 %a.0, i8 %m) {
@@ -63,8 +65,10 @@ define i8 @select05(i8 %a.0, i8 %m) {
 }
 
 ; CHECK-LABEL: select06
-; CHECK: kmovw   %esi, %k0
-; CHECK-NEXT: kmovw   %edi, %k1
+; CHECK: movzbl  %sil, %eax
+; CHECK: kmovw   %eax, %k0
+; CHECK: movzbl  %dil, %eax
+; CHECK: kmovw   %eax, %k1
 ; CHECK-NEXT: kandw    %k1, %k0, %k0
 ; CHECK-NEXT: kmovw   %k0, %eax
 define i8 @select06(i8 %a.0, i8 %m) {
@@ -76,9 +80,12 @@ define i8 @select06(i8 %a.0, i8 %m) {
 }
 
 ; CHECK-LABEL: select07
-; CHECK-DAG:  kmovw   %edx, %k0
-; CHECK-DAG:  kmovw   %edi, %k1
-; CHECK-DAG:  kmovw   %esi, %k2
+; CHECK-DAG: movzbl  %dl, %eax
+; CHECK-DAG: kmovw   %eax, %k0
+; CHECK-DAG: movzbl  %dil, %eax
+; CHECK-DAG: kmovw   %eax, %k1
+; CHECK-DAG: movzbl  %sil, %eax
+; CHECK-DAG: kmovw   %eax, %k2
 ; CHECK: kandw %k0, %k1, %k1
 ; CHECK-NEXT: knotw    %k0, %k0
 ; CHECK-NEXT: kandw    %k0, %k2, %k0
diff --git a/test/CodeGen/X86/avx512-shift.ll b/test/CodeGen/X86/avx512-shift.ll
index 0636cd242f8..10883a5a9a6 100644
--- a/test/CodeGen/X86/avx512-shift.ll
+++ b/test/CodeGen/X86/avx512-shift.ll
@@ -1,4 +1,5 @@
 ;RUN: llc < %s -mtriple=x86_64-apple-darwin -mcpu=knl | FileCheck %s
+;RUN: llc < %s -mtriple=x86_64-apple-darwin -mcpu=skx | FileCheck --check-prefix=SKX %s
 
 ;CHECK-LABEL: shift_16_i32
 ;CHECK: vpsrld
@@ -24,6 +25,18 @@ define <8 x i64> @shift_8_i64(<8 x i64> %a) {
    ret <8 x i64> %d;
 }
 
+;SKX-LABEL: shift_4_i64
+;SKX: vpsrlq
+;SKX: vpsllq
+;SKX: vpsraq
+;SKX: ret
+define <4 x i64> @shift_4_i64(<4 x i64> %a) {
+   %b = lshr <4 x i64> %a, <i64 1, i64 1, i64 1, i64 1>
+   %c = shl <4 x i64> %b,  <i64 12, i64 12, i64 12, i64 12>
+   %d = ashr <4 x i64> %c, <i64 12, i64 12, i64 12, i64 12>
+   ret <4 x i64> %d;
+}
+
 ; CHECK-LABEL: variable_shl4
 ; CHECK: vpsllvq %zmm
 ; CHECK: ret
@@ -72,6 +85,22 @@ define <8 x i64> @variable_sra2(<8 x i64> %x, <8 x i64> %y) {
   ret <8 x i64> %k
 }
 
+; SKX-LABEL: variable_sra3
+; SKX: vpsravq %ymm
+; SKX: ret
+define <4 x i64> @variable_sra3(<4 x i64> %x, <4 x i64> %y) {
+  %k = ashr <4 x i64> %x, %y
+  ret <4 x i64> %k
+}
+
+; SKX-LABEL: variable_sra4
+; SKX: vpsravw %xmm
+; SKX: ret
+define <8 x i16> @variable_sra4(<8 x i16> %x, <8 x i16> %y) {
+  %k = ashr <8 x i16> %x, %y
+  ret <8 x i16> %k
+}
+
 ; CHECK-LABEL: variable_sra01_load
 ; CHECK: vpsravd (%
 ; CHECK: ret
diff --git a/test/CodeGen/X86/avx512-trunc-ext.ll b/test/CodeGen/X86/avx512-trunc-ext.ll
deleted file mode 100644
index 91ef5d58f43..00000000000
--- a/test/CodeGen/X86/avx512-trunc-ext.ll
+++ /dev/null
@@ -1,176 +0,0 @@
-; RUN: llc < %s -mtriple=x86_64-apple-darwin -mcpu=knl | FileCheck %s
-; RUN: llc < %s -mtriple=x86_64-apple-darwin -mcpu=skx | FileCheck --check-prefix=SKX %s
-
-; CHECK-LABEL: trunc_16x32_to_16x8
-; CHECK: vpmovdb
-; CHECK: ret
-define <16 x i8> @trunc_16x32_to_16x8(<16 x i32> %i) nounwind readnone {
-  %x = trunc <16 x i32> %i to <16 x i8>
-  ret <16 x i8> %x
-}
-
-; CHECK-LABEL: trunc_8x64_to_8x16
-; CHECK: vpmovqw
-; CHECK: ret
-define <8 x i16> @trunc_8x64_to_8x16(<8 x i64> %i) nounwind readnone {
-  %x = trunc <8 x i64> %i to <8 x i16>
-  ret <8 x i16> %x
-}
-
-
-; CHECK-LABEL: zext_16x8_to_16x32
-; CHECK: vpmovzxbd {{.*}}%zmm
-; CHECK: ret
-define <16 x i32> @zext_16x8_to_16x32(<16 x i8> %i) nounwind readnone {
-  %x = zext <16 x i8> %i to <16 x i32>
-  ret <16 x i32> %x
-}
-
-; CHECK-LABEL: sext_16x8_to_16x32
-; CHECK: vpmovsxbd {{.*}}%zmm
-; CHECK: ret
-define <16 x i32> @sext_16x8_to_16x32(<16 x i8> %i) nounwind readnone {
-  %x = sext <16 x i8> %i to <16 x i32>
-  ret <16 x i32> %x
-}
-
-
-; CHECK-LABEL: zext_16x16_to_16x32
-; CHECK: vpmovzxwd {{.*}}%zmm
-; CHECK: ret
-define <16 x i32> @zext_16x16_to_16x32(<16 x i16> %i) nounwind readnone {
-  %x = zext <16 x i16> %i to <16 x i32>
-  ret <16 x i32> %x
-}
-
-; CHECK-LABEL: zext_8x16_to_8x64
-; CHECK: vpmovzxwq
-; CHECK: ret
-define <8 x i64> @zext_8x16_to_8x64(<8 x i16> %i) nounwind readnone {
-  %x = zext <8 x i16> %i to <8 x i64>
-  ret <8 x i64> %x
-}
-
-;CHECK-LABEL: fptrunc_test
-;CHECK: vcvtpd2ps {{.*}}%zmm
-;CHECK: ret
-define <8 x float> @fptrunc_test(<8 x double> %a) nounwind readnone {
-  %b = fptrunc <8 x double> %a to <8 x float>
-  ret <8 x float> %b
-}
-
-;CHECK-LABEL: fpext_test
-;CHECK: vcvtps2pd {{.*}}%zmm
-;CHECK: ret
-define <8 x double> @fpext_test(<8 x float> %a) nounwind readnone {
-  %b = fpext <8 x float> %a to <8 x double>
-  ret <8 x double> %b
-}
-
-; CHECK-LABEL: zext_16i1_to_16xi32
-; CHECK: vpbroadcastd LCP{{.*}}(%rip), %zmm0 {%k1} {z}
-; CHECK: ret
-define   <16 x i32> @zext_16i1_to_16xi32(i16 %b) {
-  %a = bitcast i16 %b to <16 x i1>
-  %c = zext <16 x i1> %a to <16 x i32>
-  ret <16 x i32> %c
-}
-
-; CHECK-LABEL: zext_8i1_to_8xi64
-; CHECK: vpbroadcastq LCP{{.*}}(%rip), %zmm0 {%k1} {z}
-; CHECK: ret
-define   <8 x i64> @zext_8i1_to_8xi64(i8 %b) {
-  %a = bitcast i8 %b to <8 x i1>
-  %c = zext <8 x i1> %a to <8 x i64>
-  ret <8 x i64> %c
-}
-
-; CHECK-LABEL: trunc_16i8_to_16i1
-; CHECK: vpmovsxbd
-; CHECK: vpandd
-; CHECK: vptestmd
-; CHECK: ret
-define i16 @trunc_16i8_to_16i1(<16 x i8> %a) {
-  %mask_b = trunc <16 x i8>%a to <16 x i1>
-  %mask = bitcast <16 x i1> %mask_b to i16
-  ret i16 %mask
-}
-
-; CHECK-LABEL: trunc_16i32_to_16i1
-; CHECK: vpandd
-; CHECK: vptestmd
-; CHECK: ret
-define i16 @trunc_16i32_to_16i1(<16 x i32> %a) {
-  %mask_b = trunc <16 x i32>%a to <16 x i1>
-  %mask = bitcast <16 x i1> %mask_b to i16
-  ret i16 %mask
-}
-
-; CHECK-LABEL: trunc_8i16_to_8i1
-; CHECK: vpmovsxwq
-; CHECK: vpandq LCP{{.*}}(%rip){1to8}
-; CHECK: vptestmq
-; CHECK: ret
-define i8 @trunc_8i16_to_8i1(<8 x i16> %a) {
-  %mask_b = trunc <8 x i16>%a to <8 x i1>
-  %mask = bitcast <8 x i1> %mask_b to i8
-  ret i8 %mask
-}
-
-; CHECK-LABEL: sext_8i1_8i32
-; CHECK: vpbroadcastq  LCP{{.*}}(%rip), %zmm0 {%k1} {z}
-; SKX: vpmovm2d
-; CHECK: ret
-define <8 x i32> @sext_8i1_8i32(<8 x i32> %a1, <8 x i32> %a2) nounwind {
-  %x = icmp slt <8 x i32> %a1, %a2
-  %x1 = xor <8 x i1>%x, <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>
-  %y = sext <8 x i1> %x1 to <8 x i32>
-  ret <8 x i32> %y
-}
-
-; CHECK-LABEL: trunc_v16i32_to_v16i16
-; CHECK: vpmovdw
-; CHECK: ret
-define <16 x i16> @trunc_v16i32_to_v16i16(<16 x i32> %x) {
-  %1 = trunc <16 x i32> %x to <16 x i16>
-  ret <16 x i16> %1
-}
-
-; CHECK-LABEL: trunc_i32_to_i1
-; CHECK: testb
-; CHECK: setne
-; CKECK: orl
-; CHECK: ret
-define i16 @trunc_i32_to_i1(i32 %a) {
-  %a_i = trunc i32 %a to i1
-  %maskv = insertelement <16 x i1> <i1 true, i1 false, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>, i1 %a_i, i32 0
-  %res = bitcast <16 x i1> %maskv to i16
-  ret i16 %res
-}
-
-; CHECK-LABEL: sext_8i1_8i16
-; SKX: vpmovm2w
-; CHECK: ret
-define <8 x i16> @sext_8i1_8i16(<8 x i32> %a1, <8 x i32> %a2) nounwind {
-  %x = icmp slt <8 x i32> %a1, %a2
-  %y = sext <8 x i1> %x to <8 x i16>
-  ret <8 x i16> %y
-}
-
-; CHECK-LABEL: sext_16i1_16i32
-; SKX: vpmovm2d
-; CHECK: ret
-define <16 x i32> @sext_16i1_16i32(<16 x i32> %a1, <16 x i32> %a2) nounwind {
-  %x = icmp slt <16 x i32> %a1, %a2
-  %y = sext <16 x i1> %x to <16 x i32>
-  ret <16 x i32> %y
-}
-
-; CHECK-LABEL: sext_8i1_8i64
-; SKX: vpmovm2q
-; CHECK: ret
-define <8 x i64> @sext_8i1_8i64(<8 x i32> %a1, <8 x i32> %a2) nounwind {
-  %x = icmp slt <8 x i32> %a1, %a2
-  %y = sext <8 x i1> %x to <8 x i64>
-  ret <8 x i64> %y
-}
diff --git a/test/CodeGen/X86/avx512-trunc.ll b/test/CodeGen/X86/avx512-trunc.ll
new file mode 100644
index 00000000000..9205feda7eb
--- /dev/null
+++ b/test/CodeGen/X86/avx512-trunc.ll
@@ -0,0 +1,364 @@
+; RUN: llc < %s -mtriple=x86_64-apple-darwin -mcpu=knl | FileCheck %s --check-prefix=KNL
+; RUN: llc < %s -mtriple=x86_64-apple-darwin -mcpu=skx | FileCheck %s --check-prefix=SKX 
+ 
+ attributes #0 = { nounwind }
+ 
+; KNL-LABEL: trunc_16x32_to_16x8
+; KNL: vpmovdb
+; KNL: ret
+define <16 x i8> @trunc_16x32_to_16x8(<16 x i32> %i) #0 {
+  %x = trunc <16 x i32> %i to <16 x i8>
+  ret <16 x i8> %x
+}
+
+; KNL-LABEL: trunc_8x64_to_8x16
+; KNL: vpmovqw
+; KNL: ret
+define <8 x i16> @trunc_8x64_to_8x16(<8 x i64> %i) #0 {
+  %x = trunc <8 x i64> %i to <8 x i16>
+  ret <8 x i16> %x
+}
+
+; KNL-LABEL: trunc_v16i32_to_v16i16
+; KNL: vpmovdw
+; KNL: ret
+define <16 x i16> @trunc_v16i32_to_v16i16(<16 x i32> %x) #0 {
+  %1 = trunc <16 x i32> %x to <16 x i16>
+  ret <16 x i16> %1
+}
+
+define <8 x i8> @trunc_qb_512(<8 x i64> %i) #0 {
+; SKX-LABEL: trunc_qb_512:
+; SKX:       ## BB#0:
+; SKX-NEXT:    vpmovqw %zmm0, %xmm0
+; SKX-NEXT:    retq
+  %x = trunc <8 x i64> %i to <8 x i8>
+  ret <8 x i8> %x
+}
+
+define void @trunc_qb_512_mem(<8 x i64> %i, <8 x i8>* %res) #0 {
+; SKX-LABEL: trunc_qb_512_mem:
+; SKX:       ## BB#0:
+; SKX-NEXT:    vpmovqb %zmm0, (%rdi)
+; SKX-NEXT:    retq
+    %x = trunc <8 x i64> %i to <8 x i8>
+    store <8 x i8> %x, <8 x i8>* %res
+    ret void
+}
+
+define <4 x i8> @trunc_qb_256(<4 x i64> %i) #0 {
+; SKX-LABEL: trunc_qb_256:
+; SKX:       ## BB#0:
+; SKX-NEXT:    vpmovqd %ymm0, %xmm0
+; SKX-NEXT:    retq
+  %x = trunc <4 x i64> %i to <4 x i8>
+  ret <4 x i8> %x
+}
+
+define void @trunc_qb_256_mem(<4 x i64> %i, <4 x i8>* %res) #0 {
+; SKX-LABEL: trunc_qb_256_mem:
+; SKX:       ## BB#0:
+; SKX-NEXT:    vpmovqb %ymm0, (%rdi)
+; SKX-NEXT:    retq
+    %x = trunc <4 x i64> %i to <4 x i8>
+    store <4 x i8> %x, <4 x i8>* %res
+    ret void
+}
+
+define <2 x i8> @trunc_qb_128(<2 x i64> %i) #0 {
+; SKX-LABEL: trunc_qb_128:
+; SKX:       ## BB#0:
+; SKX-NEXT:    retq
+  %x = trunc <2 x i64> %i to <2 x i8>
+  ret <2 x i8> %x
+}
+
+define void @trunc_qb_128_mem(<2 x i64> %i, <2 x i8>* %res) #0 {
+; SKX-LABEL: trunc_qb_128_mem:
+; SKX:       ## BB#0:
+; SKX-NEXT:    vpmovqb %xmm0, (%rdi)
+; SKX-NEXT:    retq
+    %x = trunc <2 x i64> %i to <2 x i8>
+    store <2 x i8> %x, <2 x i8>* %res
+    ret void
+}
+
+define <8 x i16> @trunc_qw_512(<8 x i64> %i) #0 {
+; SKX-LABEL: trunc_qw_512:
+; SKX:       ## BB#0:
+; SKX-NEXT:    vpmovqw %zmm0, %xmm0
+; SKX-NEXT:    retq
+  %x = trunc <8 x i64> %i to <8 x i16>
+  ret <8 x i16> %x
+}
+
+define void @trunc_qw_512_mem(<8 x i64> %i, <8 x i16>* %res) #0 {
+; SKX-LABEL: trunc_qw_512_mem:
+; SKX:       ## BB#0:
+; SKX-NEXT:    vpmovqw %zmm0, (%rdi)
+; SKX-NEXT:    retq
+    %x = trunc <8 x i64> %i to <8 x i16>
+    store <8 x i16> %x, <8 x i16>* %res
+    ret void
+}
+
+define <4 x i16> @trunc_qw_256(<4 x i64> %i) #0 {
+; SKX-LABEL: trunc_qw_256:
+; SKX:       ## BB#0:
+; SKX-NEXT:    vpmovqd %ymm0, %xmm0
+; SKX-NEXT:    retq
+  %x = trunc <4 x i64> %i to <4 x i16>
+  ret <4 x i16> %x
+}
+
+define void @trunc_qw_256_mem(<4 x i64> %i, <4 x i16>* %res) #0 {
+; SKX-LABEL: trunc_qw_256_mem:
+; SKX:       ## BB#0:
+; SKX-NEXT:    vpmovqw %ymm0, (%rdi)
+; SKX-NEXT:    retq
+    %x = trunc <4 x i64> %i to <4 x i16>
+    store <4 x i16> %x, <4 x i16>* %res
+    ret void
+}
+
+define <2 x i16> @trunc_qw_128(<2 x i64> %i) #0 {
+; SKX-LABEL: trunc_qw_128:
+; SKX:       ## BB#0:
+; SKX-NEXT:    retq
+  %x = trunc <2 x i64> %i to <2 x i16>
+  ret <2 x i16> %x
+}
+
+define void @trunc_qw_128_mem(<2 x i64> %i, <2 x i16>* %res) #0 {
+; SKX-LABEL: trunc_qw_128_mem:
+; SKX:       ## BB#0:
+; SKX-NEXT:    vpmovqw %xmm0, (%rdi)
+; SKX-NEXT:    retq
+    %x = trunc <2 x i64> %i to <2 x i16>
+    store <2 x i16> %x, <2 x i16>* %res
+    ret void
+}
+
+define <8 x i32> @trunc_qd_512(<8 x i64> %i) #0 {
+; SKX-LABEL: trunc_qd_512:
+; SKX:       ## BB#0:
+; SKX-NEXT:    vpmovqd %zmm0, %ymm0
+; SKX-NEXT:    retq
+  %x = trunc <8 x i64> %i to <8 x i32>
+  ret <8 x i32> %x
+}
+
+define void @trunc_qd_512_mem(<8 x i64> %i, <8 x i32>* %res) #0 {
+; SKX-LABEL: trunc_qd_512_mem:
+; SKX:       ## BB#0:
+; SKX-NEXT:    vpmovqd %zmm0, (%rdi)
+; SKX-NEXT:    retq
+    %x = trunc <8 x i64> %i to <8 x i32>
+    store <8 x i32> %x, <8 x i32>* %res
+    ret void
+}
+
+define <4 x i32> @trunc_qd_256(<4 x i64> %i) #0 {
+; SKX-LABEL: trunc_qd_256:
+; SKX:       ## BB#0:
+; SKX-NEXT:    vpmovqd %ymm0, %xmm0
+; SKX-NEXT:    retq
+  %x = trunc <4 x i64> %i to <4 x i32>
+  ret <4 x i32> %x
+}
+
+define void @trunc_qd_256_mem(<4 x i64> %i, <4 x i32>* %res) #0 {
+; SKX-LABEL: trunc_qd_256_mem:
+; SKX:       ## BB#0:
+; SKX-NEXT:    vpmovqd %ymm0, (%rdi)
+; SKX-NEXT:    retq
+    %x = trunc <4 x i64> %i to <4 x i32>
+    store <4 x i32> %x, <4 x i32>* %res
+    ret void
+}
+
+define <2 x i32> @trunc_qd_128(<2 x i64> %i) #0 {
+; SKX-LABEL: trunc_qd_128:
+; SKX:       ## BB#0:
+; SKX-NEXT:    retq
+  %x = trunc <2 x i64> %i to <2 x i32>
+  ret <2 x i32> %x
+}
+
+define void @trunc_qd_128_mem(<2 x i64> %i, <2 x i32>* %res) #0 {
+; SKX-LABEL: trunc_qd_128_mem:
+; SKX:       ## BB#0:
+; SKX-NEXT:    vpmovqd %xmm0, (%rdi)
+; SKX-NEXT:    retq
+    %x = trunc <2 x i64> %i to <2 x i32>
+    store <2 x i32> %x, <2 x i32>* %res
+    ret void
+}
+
+define <16 x i8> @trunc_db_512(<16 x i32> %i) #0 {
+; SKX-LABEL: trunc_db_512:
+; SKX:       ## BB#0:
+; SKX-NEXT:    vpmovdb %zmm0, %xmm0
+; SKX-NEXT:    retq
+  %x = trunc <16 x i32> %i to <16 x i8>
+  ret <16 x i8> %x
+}
+
+define void @trunc_db_512_mem(<16 x i32> %i, <16 x i8>* %res) #0 {
+; SKX-LABEL: trunc_db_512_mem:
+; SKX:       ## BB#0:
+; SKX-NEXT:    vpmovdb %zmm0, (%rdi)
+; SKX-NEXT:    retq
+    %x = trunc <16 x i32> %i to <16 x i8>
+    store <16 x i8> %x, <16 x i8>* %res
+    ret void
+}
+
+define <8 x i8> @trunc_db_256(<8 x i32> %i) #0 {
+; SKX-LABEL: trunc_db_256:
+; SKX:       ## BB#0:
+; SKX-NEXT:    vpmovdw %ymm0, %xmm0
+; SKX-NEXT:    retq
+  %x = trunc <8 x i32> %i to <8 x i8>
+  ret <8 x i8> %x
+}
+
+define void @trunc_db_256_mem(<8 x i32> %i, <8 x i8>* %res) #0 {
+; SKX-LABEL: trunc_db_256_mem:
+; SKX:       ## BB#0:
+; SKX-NEXT:    vpmovdb %ymm0, (%rdi)
+; SKX-NEXT:    retq
+    %x = trunc <8 x i32> %i to <8 x i8>
+    store <8 x i8> %x, <8 x i8>* %res
+    ret void
+}
+
+define <4 x i8> @trunc_db_128(<4 x i32> %i) #0 {
+; SKX-LABEL: trunc_db_128:
+; SKX:       ## BB#0:
+; SKX-NEXT:    retq
+  %x = trunc <4 x i32> %i to <4 x i8>
+  ret <4 x i8> %x
+}
+
+define void @trunc_db_128_mem(<4 x i32> %i, <4 x i8>* %res) #0 {
+; SKX-LABEL: trunc_db_128_mem:
+; SKX:       ## BB#0:
+; SKX-NEXT:    vpmovdb %xmm0, (%rdi)
+; SKX-NEXT:    retq
+    %x = trunc <4 x i32> %i to <4 x i8>
+    store <4 x i8> %x, <4 x i8>* %res
+    ret void
+}
+
+define <16 x i16> @trunc_dw_512(<16 x i32> %i) #0 {
+; SKX-LABEL: trunc_dw_512:
+; SKX:       ## BB#0:
+; SKX-NEXT:    vpmovdw %zmm0, %ymm0
+; SKX-NEXT:    retq
+  %x = trunc <16 x i32> %i to <16 x i16>
+  ret <16 x i16> %x
+}
+
+define void @trunc_dw_512_mem(<16 x i32> %i, <16 x i16>* %res) #0 {
+; SKX-LABEL: trunc_dw_512_mem:
+; SKX:       ## BB#0:
+; SKX-NEXT:    vpmovdw %zmm0, (%rdi)
+; SKX-NEXT:    retq
+    %x = trunc <16 x i32> %i to <16 x i16>
+    store <16 x i16> %x, <16 x i16>* %res
+    ret void
+}
+
+define <8 x i16> @trunc_dw_256(<8 x i32> %i) #0 {
+; SKX-LABEL: trunc_dw_256:
+; SKX:       ## BB#0:
+; SKX-NEXT:    vpmovdw %ymm0, %xmm0
+; SKX-NEXT:    retq
+  %x = trunc <8 x i32> %i to <8 x i16>
+  ret <8 x i16> %x
+}
+
+define void @trunc_dw_256_mem(<8 x i32> %i, <8 x i16>* %res) #0 {
+; SKX-LABEL: trunc_dw_256_mem:
+; SKX:       ## BB#0:
+; SKX-NEXT:    vpmovdw %ymm0, (%rdi)
+; SKX-NEXT:    retq
+    %x = trunc <8 x i32> %i to <8 x i16>
+    store <8 x i16> %x, <8 x i16>* %res
+    ret void
+}
+
+define <4 x i16> @trunc_dw_128(<4 x i32> %i) #0 {
+; SKX-LABEL: trunc_dw_128:
+; SKX:       ## BB#0:
+; SKX-NEXT:    retq
+  %x = trunc <4 x i32> %i to <4 x i16>
+  ret <4 x i16> %x
+}
+
+define void @trunc_dw_128_mem(<4 x i32> %i, <4 x i16>* %res) #0 {
+; SKX-LABEL: trunc_dw_128_mem:
+; SKX:       ## BB#0:
+; SKX-NEXT:    vpmovdw %xmm0, (%rdi)
+; SKX-NEXT:    retq
+    %x = trunc <4 x i32> %i to <4 x i16>
+    store <4 x i16> %x, <4 x i16>* %res
+    ret void
+}
+
+define <32 x i8> @trunc_wb_512(<32 x i16> %i) #0 {
+; SKX-LABEL: trunc_wb_512:
+; SKX:       ## BB#0:
+; SKX-NEXT:    vpmovwb %zmm0, %ymm0
+; SKX-NEXT:    retq
+  %x = trunc <32 x i16> %i to <32 x i8>
+  ret <32 x i8> %x
+}
+
+define void @trunc_wb_512_mem(<32 x i16> %i, <32 x i8>* %res) #0 {
+; SKX-LABEL: trunc_wb_512_mem:
+; SKX:       ## BB#0:
+; SKX-NEXT:    vpmovwb %zmm0, (%rdi)
+; SKX-NEXT:    retq
+    %x = trunc <32 x i16> %i to <32 x i8>
+    store <32 x i8> %x, <32 x i8>* %res
+    ret void
+}
+
+define <16 x i8> @trunc_wb_256(<16 x i16> %i) #0 {
+; SKX-LABEL: trunc_wb_256:
+; SKX:       ## BB#0:
+; SKX-NEXT:    vpmovwb %ymm0, %xmm0
+; SKX-NEXT:    retq
+  %x = trunc <16 x i16> %i to <16 x i8>
+  ret <16 x i8> %x
+}
+
+define void @trunc_wb_256_mem(<16 x i16> %i, <16 x i8>* %res) #0 {
+; SKX-LABEL: trunc_wb_256_mem:
+; SKX:       ## BB#0:
+; SKX-NEXT:    vpmovwb %ymm0, (%rdi)
+; SKX-NEXT:    retq
+    %x = trunc <16 x i16> %i to <16 x i8>
+    store <16 x i8> %x, <16 x i8>* %res
+    ret void
+}
+
+define <8 x i8> @trunc_wb_128(<8 x i16> %i) #0 {
+; SKX-LABEL: trunc_wb_128:
+; SKX:       ## BB#0:
+; SKX-NEXT:    retq
+  %x = trunc <8 x i16> %i to <8 x i8>
+  ret <8 x i8> %x
+}
+
+define void @trunc_wb_128_mem(<8 x i16> %i, <8 x i8>* %res) #0 {
+; SKX-LABEL: trunc_wb_128_mem:
+; SKX:       ## BB#0:
+; SKX-NEXT:    vpmovwb %xmm0, (%rdi)
+; SKX-NEXT:    retq
+    %x = trunc <8 x i16> %i to <8 x i8>
+    store <8 x i8> %x, <8 x i8>* %res
+    ret void
+}
diff --git a/test/CodeGen/X86/avx512-vbroadcast.ll b/test/CodeGen/X86/avx512-vbroadcast.ll
index cc81d682079..854f1019f0f 100644
--- a/test/CodeGen/X86/avx512-vbroadcast.ll
+++ b/test/CodeGen/X86/avx512-vbroadcast.ll
@@ -300,3 +300,31 @@ entry:
   %vecinit7.i = insertelement <8 x double> %vecinit6.i, double %0, i32 7
   ret <8 x double> %vecinit7.i
 }
+
+; CHECK-LABEL: test1
+; CHECK: vbroadcastss
+define <16 x float> @test1(<8 x float>%a)  {
+  %res = shufflevector <8 x float> %a, <8 x float> undef, <16 x i32> zeroinitializer
+  ret <16 x float>%res
+}
+
+; CHECK-LABEL: test2
+; CHECK: vbroadcastsd
+define <8 x double> @test2(<4 x double>%a)  {
+  %res = shufflevector <4 x double> %a, <4 x double> undef, <8 x i32> zeroinitializer
+  ret <8 x double>%res
+}
+
+; CHECK-LABEL: test3
+; CHECK: vpbroadcastd
+define <16 x i32> @test3(<8 x i32>%a)  {
+  %res = shufflevector <8 x i32> %a, <8 x i32> undef, <16 x i32> zeroinitializer
+  ret <16 x i32>%res
+}
+
+; CHECK-LABEL: test4
+; CHECK: vpbroadcastq
+define <8 x i64> @test4(<4 x i64>%a)  {
+  %res = shufflevector <4 x i64> %a, <4 x i64> undef, <8 x i32> zeroinitializer
+  ret <8 x i64>%res
+}
diff --git a/test/CodeGen/X86/avx512-vec-cmp.ll b/test/CodeGen/X86/avx512-vec-cmp.ll
index 26e2c776b25..6a4a3aa7e37 100644
--- a/test/CodeGen/X86/avx512-vec-cmp.ll
+++ b/test/CodeGen/X86/avx512-vec-cmp.ll
@@ -1,36 +1,37 @@
-; RUN: llc < %s -mtriple=x86_64-apple-darwin -mcpu=knl | FileCheck %s
+; RUN: llc < %s -mtriple=x86_64-apple-darwin -mcpu=knl | FileCheck %s --check-prefix=KNL
+; RUN: llc < %s -mtriple=x86_64-apple-darwin -mcpu=skx | FileCheck %s --check-prefix=SKX
 
 define <16 x float> @test1(<16 x float> %x, <16 x float> %y) nounwind {
-; CHECK-LABEL: test1:
-; CHECK:       ## BB#0:
-; CHECK-NEXT:    vcmpleps %zmm1, %zmm0, %k1
-; CHECK-NEXT:    vmovaps %zmm0, %zmm1 {%k1}
-; CHECK-NEXT:    vmovaps %zmm1, %zmm0
-; CHECK-NEXT:    retq
+; KNL-LABEL: test1:
+; KNL:       ## BB#0:
+; KNL-NEXT:    vcmpleps %zmm1, %zmm0, %k1
+; KNL-NEXT:    vmovaps %zmm0, %zmm1 {%k1}
+; KNL-NEXT:    vmovaps %zmm1, %zmm0
+; KNL-NEXT:    retq
   %mask = fcmp ole <16 x float> %x, %y
   %max = select <16 x i1> %mask, <16 x float> %x, <16 x float> %y
   ret <16 x float> %max
 }
 
 define <8 x double> @test2(<8 x double> %x, <8 x double> %y) nounwind {
-; CHECK-LABEL: test2:
-; CHECK:       ## BB#0:
-; CHECK-NEXT:    vcmplepd %zmm1, %zmm0, %k1
-; CHECK-NEXT:    vmovapd %zmm0, %zmm1 {%k1}
-; CHECK-NEXT:    vmovaps %zmm1, %zmm0
-; CHECK-NEXT:    retq
+; KNL-LABEL: test2:
+; KNL:       ## BB#0:
+; KNL-NEXT:    vcmplepd %zmm1, %zmm0, %k1
+; KNL-NEXT:    vmovapd %zmm0, %zmm1 {%k1}
+; KNL-NEXT:    vmovaps %zmm1, %zmm0
+; KNL-NEXT:    retq
   %mask = fcmp ole <8 x double> %x, %y
   %max = select <8 x i1> %mask, <8 x double> %x, <8 x double> %y
   ret <8 x double> %max
 }
 
 define <16 x i32> @test3(<16 x i32> %x, <16 x i32> %x1, <16 x i32>* %yp) nounwind {
-; CHECK-LABEL: test3:
-; CHECK:       ## BB#0:
-; CHECK-NEXT:    vpcmpeqd (%rdi), %zmm0, %k1
-; CHECK-NEXT:    vmovdqa32 %zmm0, %zmm1 {%k1}
-; CHECK-NEXT:    vmovaps %zmm1, %zmm0
-; CHECK-NEXT:    retq
+; KNL-LABEL: test3:
+; KNL:       ## BB#0:
+; KNL-NEXT:    vpcmpeqd (%rdi), %zmm0, %k1
+; KNL-NEXT:    vmovdqa32 %zmm0, %zmm1 {%k1}
+; KNL-NEXT:    vmovaps %zmm1, %zmm0
+; KNL-NEXT:    retq
   %y = load <16 x i32>, <16 x i32>* %yp, align 4
   %mask = icmp eq <16 x i32> %x, %y
   %max = select <16 x i1> %mask, <16 x i32> %x, <16 x i32> %x1
@@ -38,98 +39,114 @@ define <16 x i32> @test3(<16 x i32> %x, <16 x i32> %x1, <16 x i32>* %yp) nounwin
 }
 
 define <16 x i32> @test4_unsigned(<16 x i32> %x, <16 x i32> %y, <16 x i32> %x1) nounwind {
-; CHECK-LABEL: test4_unsigned:
-; CHECK:       ## BB#0:
-; CHECK-NEXT:    vpcmpnltud %zmm1, %zmm0, %k1
-; CHECK-NEXT:    vmovdqa32 %zmm2, %zmm1 {%k1}
-; CHECK-NEXT:    vmovaps %zmm1, %zmm0
-; CHECK-NEXT:    retq
+; KNL-LABEL: test4_unsigned:
+; KNL:       ## BB#0:
+; KNL-NEXT:    vpcmpnltud %zmm1, %zmm0, %k1
+; KNL-NEXT:    vmovdqa32 %zmm2, %zmm1 {%k1}
+; KNL-NEXT:    vmovaps %zmm1, %zmm0
+; KNL-NEXT:    retq
   %mask = icmp uge <16 x i32> %x, %y
   %max = select <16 x i1> %mask, <16 x i32> %x1, <16 x i32> %y
   ret <16 x i32> %max
 }
 
 define <8 x i64> @test5(<8 x i64> %x, <8 x i64> %y) nounwind {
-; CHECK-LABEL: test5:
-; CHECK:       ## BB#0:
-; CHECK-NEXT:    vpcmpeqq %zmm1, %zmm0, %k1
-; CHECK-NEXT:    vmovdqa64 %zmm0, %zmm1 {%k1}
-; CHECK-NEXT:    vmovaps %zmm1, %zmm0
-; CHECK-NEXT:    retq
+; KNL-LABEL: test5:
+; KNL:       ## BB#0:
+; KNL-NEXT:    vpcmpeqq %zmm1, %zmm0, %k1
+; KNL-NEXT:    vmovdqa64 %zmm0, %zmm1 {%k1}
+; KNL-NEXT:    vmovaps %zmm1, %zmm0
+; KNL-NEXT:    retq
   %mask = icmp eq <8 x i64> %x, %y
   %max = select <8 x i1> %mask, <8 x i64> %x, <8 x i64> %y
   ret <8 x i64> %max
 }
 
 define <8 x i64> @test6_unsigned(<8 x i64> %x, <8 x i64> %y, <8 x i64> %x1) nounwind {
-; CHECK-LABEL: test6_unsigned:
-; CHECK:       ## BB#0:
-; CHECK-NEXT:    vpcmpnleuq %zmm1, %zmm0, %k1
-; CHECK-NEXT:    vmovdqa64 %zmm2, %zmm1 {%k1}
-; CHECK-NEXT:    vmovaps %zmm1, %zmm0
-; CHECK-NEXT:    retq
+; KNL-LABEL: test6_unsigned:
+; KNL:       ## BB#0:
+; KNL-NEXT:    vpcmpnleuq %zmm1, %zmm0, %k1
+; KNL-NEXT:    vmovdqa64 %zmm2, %zmm1 {%k1}
+; KNL-NEXT:    vmovaps %zmm1, %zmm0
+; KNL-NEXT:    retq
   %mask = icmp ugt <8 x i64> %x, %y
   %max = select <8 x i1> %mask, <8 x i64> %x1, <8 x i64> %y
   ret <8 x i64> %max
 }
 
 define <4 x float> @test7(<4 x float> %a, <4 x float> %b) {
-; CHECK-LABEL: test7:
-; CHECK:       ## BB#0:
-; CHECK-NEXT:    vxorps %xmm2, %xmm2, %xmm2
-; CHECK-NEXT:    vcmpltps %xmm2, %xmm0, %xmm2
-; CHECK-NEXT:    vblendvps %xmm2, %xmm0, %xmm1, %xmm0
-; CHECK-NEXT:    retq
+; KNL-LABEL: test7:
+; KNL:       ## BB#0:
+; KNL-NEXT:    vxorps %xmm2, %xmm2, %xmm2
+; KNL-NEXT:    vcmpltps %xmm2, %xmm0, %xmm2
+; KNL-NEXT:    vblendvps %xmm2, %xmm0, %xmm1, %xmm0
+; KNL-NEXT:    retq
+; SKX-LABEL: test7:
+; SKX:       ## BB#0:
+; SKX:    vxorps   %xmm2, %xmm2, %xmm2
+; SKX:    vcmpltps %xmm2, %xmm0, %k1 
+; SKX:    vmovaps  %xmm0, %xmm1 {%k1}
+; SKX:    vmovaps  %zmm1, %zmm0
+; SKX:    retq
+
   %mask = fcmp olt <4 x float> %a, zeroinitializer
   %c = select <4 x i1>%mask, <4 x float>%a, <4 x float>%b
   ret <4 x float>%c
 }
 
 define <2 x double> @test8(<2 x double> %a, <2 x double> %b) {
-; CHECK-LABEL: test8:
-; CHECK:       ## BB#0:
-; CHECK-NEXT:    vxorpd %xmm2, %xmm2, %xmm2
-; CHECK-NEXT:    vcmpltpd %xmm2, %xmm0, %xmm2
-; CHECK-NEXT:    vblendvpd %xmm2, %xmm0, %xmm1, %xmm0
-; CHECK-NEXT:    retq
+; KNL-LABEL: test8:
+; KNL:       ## BB#0:
+; KNL-NEXT:    vxorpd %xmm2, %xmm2, %xmm2
+; KNL-NEXT:    vcmpltpd %xmm2, %xmm0, %xmm2
+; KNL-NEXT:    vblendvpd %xmm2, %xmm0, %xmm1, %xmm0
+; KNL-NEXT:    retq
+; SKX-LABEL: test8:
+; SKX:       ## BB#0:
+; SKX: vxorpd  %xmm2, %xmm2, %xmm2
+; SKX: vcmpltpd    %xmm2, %xmm0, %k1 
+; SKX: vmovapd %xmm0, %xmm1 {%k1}
+; SKX: vmovaps %zmm1, %zmm0
+; SKX: retq
   %mask = fcmp olt <2 x double> %a, zeroinitializer
   %c = select <2 x i1>%mask, <2 x double>%a, <2 x double>%b
   ret <2 x double>%c
 }
 
 define <8 x i32> @test9(<8 x i32> %x, <8 x i32> %y) nounwind {
-; CHECK-LABEL: test9:
-; CHECK:       ## BB#0:
-; CHECK-NEXT:      ## kill: YMM1<def> YMM1<kill> ZMM1<def>
-; CHECK-NEXT:      ## kill: YMM0<def> YMM0<kill> ZMM0<def>
-; CHECK-NEXT:    vpcmpeqd %zmm1, %zmm0, %k1
-; CHECK-NEXT:    vpblendmd %zmm0, %zmm1, %zmm0 {%k1}
-; CHECK-NEXT:      ## kill: YMM0<def> YMM0<kill> ZMM0<kill>
-; CHECK-NEXT:    retq
+; KNL-LABEL: test9:
+; KNL:       ## BB#0:
+; KNL-NEXT:    vpcmpeqd %zmm1, %zmm0, %k1
+; KNL-NEXT:    vpblendmd %zmm0, %zmm1, %zmm0 {%k1}
+; KNL-NEXT:    retq
   %mask = icmp eq <8 x i32> %x, %y
   %max = select <8 x i1> %mask, <8 x i32> %x, <8 x i32> %y
   ret <8 x i32> %max
 }
 
 define <8 x float> @test10(<8 x float> %x, <8 x float> %y) nounwind {
-; CHECK-LABEL: test10:
-; CHECK:       ## BB#0:
-; CHECK-NEXT:      ## kill: YMM1<def> YMM1<kill> ZMM1<def>
-; CHECK-NEXT:      ## kill: YMM0<def> YMM0<kill> ZMM0<def>
-; CHECK-NEXT:    vcmpeqps %zmm1, %zmm0, %k1
-; CHECK-NEXT:    vblendmps %zmm0, %zmm1, %zmm0 {%k1}
-; CHECK-NEXT:      ## kill: YMM0<def> YMM0<kill> ZMM0<kill>
-; CHECK-NEXT:    retq
+; KNL-LABEL: test10:
+; KNL:       ## BB#0:
+; KNL-NEXT:    vcmpeqps %zmm1, %zmm0, %k1
+; KNL-NEXT:    vblendmps %zmm0, %zmm1, %zmm0 {%k1}
+; KNL-NEXT:    retq
+; SKX-LABEL: test10:
+; SKX:       ## BB#0:
+; SKX: vcmpeqps    %ymm1, %ymm0, %k1 
+; SKX: vmovaps %ymm0, %ymm1 {%k1}
+; SKX: vmovaps %zmm1, %zmm0
+; SKX: retq
+
   %mask = fcmp oeq <8 x float> %x, %y
   %max = select <8 x i1> %mask, <8 x float> %x, <8 x float> %y
   ret <8 x float> %max
 }
 
 define <8 x i32> @test11_unsigned(<8 x i32> %x, <8 x i32> %y) nounwind {
-; CHECK-LABEL: test11_unsigned:
-; CHECK:       ## BB#0:
-; CHECK-NEXT:    vpmaxud %ymm1, %ymm0, %ymm0
-; CHECK-NEXT:    retq
+; KNL-LABEL: test11_unsigned:
+; KNL:       ## BB#0:
+; KNL-NEXT:    vpmaxud %ymm1, %ymm0, %ymm0
+; KNL-NEXT:    retq
   %mask = icmp ugt <8 x i32> %x, %y
   %max = select <8 x i1> %mask, <8 x i32> %x, <8 x i32> %y
   ret <8 x i32> %max
@@ -137,25 +154,24 @@ define <8 x i32> @test11_unsigned(<8 x i32> %x, <8 x i32> %y) nounwind {
 
 
 define i16 @test12(<16 x i64> %a, <16 x i64> %b) nounwind {
-; CHECK-LABEL: test12:
-; CHECK:       ## BB#0:
-; CHECK-NEXT:    vpcmpeqq %zmm2, %zmm0, %k0
-; CHECK-NEXT:    vpcmpeqq %zmm3, %zmm1, %k1
-; CHECK-NEXT:    kunpckbw %k0, %k1, %k0
-; CHECK-NEXT:    kmovw %k0, %eax
-; CHECK-NEXT:      ## kill: AX<def> AX<kill> EAX<kill>
-; CHECK-NEXT:    retq
+; KNL-LABEL: test12:
+; KNL:       ## BB#0:
+; KNL-NEXT:    vpcmpeqq %zmm2, %zmm0, %k0
+; KNL-NEXT:    vpcmpeqq %zmm3, %zmm1, %k1
+; KNL-NEXT:    kunpckbw %k0, %k1, %k0
+; KNL-NEXT:    kmovw %k0, %eax
+; KNL-NEXT:    retq
   %res = icmp eq <16 x i64> %a, %b
   %res1 = bitcast <16 x i1> %res to i16
   ret i16 %res1
 }
 
 define <16 x i32> @test13(<16 x float>%a, <16 x float>%b)
-; CHECK-LABEL: test13:
-; CHECK:       ## BB#0:
-; CHECK-NEXT:    vcmpeqps %zmm1, %zmm0, %k1
-; CHECK-NEXT:    vpbroadcastd {{.*}}(%rip), %zmm0 {%k1} {z}
-; CHECK-NEXT:    retq
+; KNL-LABEL: test13:
+; KNL:       ## BB#0:
+; KNL-NEXT:    vcmpeqps %zmm1, %zmm0, %k1
+; KNL-NEXT:    vpbroadcastd {{.*}}(%rip), %zmm0 {%k1} {z}
+; KNL-NEXT:    retq
 {
   %cmpvector_i = fcmp oeq <16 x float> %a, %b
   %conv = zext <16 x i1> %cmpvector_i to <16 x i32>
@@ -163,14 +179,14 @@ define <16 x i32> @test13(<16 x float>%a, <16 x float>%b)
 }
 
 define <16 x i32> @test14(<16 x i32>%a, <16 x i32>%b) {
-; CHECK-LABEL: test14:
-; CHECK:       ## BB#0:
-; CHECK-NEXT:    vpsubd %zmm1, %zmm0, %zmm1
-; CHECK-NEXT:    vpcmpgtd %zmm0, %zmm1, %k0
-; CHECK-NEXT:    knotw %k0, %k0
-; CHECK-NEXT:    knotw %k0, %k1
-; CHECK-NEXT:    vmovdqu32 %zmm1, %zmm0 {%k1} {z}
-; CHECK-NEXT:    retq
+; KNL-LABEL: test14:
+; KNL:       ## BB#0:
+; KNL-NEXT:    vpsubd %zmm1, %zmm0, %zmm1
+; KNL-NEXT:    vpcmpgtd %zmm0, %zmm1, %k0
+; KNL-NEXT:    knotw %k0, %k0
+; KNL-NEXT:    knotw %k0, %k1
+; KNL-NEXT:    vmovdqu32 %zmm1, %zmm0 {%k1} {z}
+; KNL-NEXT:    retq
   %sub_r = sub <16 x i32> %a, %b
   %cmp.i2.i = icmp sgt <16 x i32> %sub_r, %a
   %sext.i3.i = sext <16 x i1> %cmp.i2.i to <16 x i32>
@@ -180,14 +196,14 @@ define <16 x i32> @test14(<16 x i32>%a, <16 x i32>%b) {
 }
 
 define <8 x i64> @test15(<8 x i64>%a, <8 x i64>%b) {
-; CHECK-LABEL: test15:
-; CHECK:       ## BB#0:
-; CHECK-NEXT:    vpsubq %zmm1, %zmm0, %zmm1
-; CHECK-NEXT:    vpcmpgtq %zmm0, %zmm1, %k0
-; CHECK-NEXT:    knotw %k0, %k0
-; CHECK-NEXT:    knotw %k0, %k1
-; CHECK-NEXT:    vmovdqu64 %zmm1, %zmm0 {%k1} {z}
-; CHECK-NEXT:    retq
+; KNL-LABEL: test15:
+; KNL:       ## BB#0:
+; KNL-NEXT:    vpsubq %zmm1, %zmm0, %zmm1
+; KNL-NEXT:    vpcmpgtq %zmm0, %zmm1, %k0
+; KNL-NEXT:    knotw %k0, %k0
+; KNL-NEXT:    knotw %k0, %k1
+; KNL-NEXT:    vmovdqu64 %zmm1, %zmm0 {%k1} {z}
+; KNL-NEXT:    retq
   %sub_r = sub <8 x i64> %a, %b
   %cmp.i2.i = icmp sgt <8 x i64> %sub_r, %a
   %sext.i3.i = sext <8 x i1> %cmp.i2.i to <8 x i64>
@@ -197,24 +213,24 @@ define <8 x i64> @test15(<8 x i64>%a, <8 x i64>%b) {
 }
 
 define <16 x i32> @test16(<16 x i32> %x, <16 x i32> %y, <16 x i32> %x1) nounwind {
-; CHECK-LABEL: test16:
-; CHECK:       ## BB#0:
-; CHECK-NEXT:    vpcmpled %zmm0, %zmm1, %k1
-; CHECK-NEXT:    vmovdqa32 %zmm2, %zmm1 {%k1}
-; CHECK-NEXT:    vmovaps %zmm1, %zmm0
-; CHECK-NEXT:    retq
+; KNL-LABEL: test16:
+; KNL:       ## BB#0:
+; KNL-NEXT:    vpcmpled %zmm0, %zmm1, %k1
+; KNL-NEXT:    vmovdqa32 %zmm2, %zmm1 {%k1}
+; KNL-NEXT:    vmovaps %zmm1, %zmm0
+; KNL-NEXT:    retq
   %mask = icmp sge <16 x i32> %x, %y
   %max = select <16 x i1> %mask, <16 x i32> %x1, <16 x i32> %y
   ret <16 x i32> %max
 }
 
 define <16 x i32> @test17(<16 x i32> %x, <16 x i32> %x1, <16 x i32>* %y.ptr) nounwind {
-; CHECK-LABEL: test17:
-; CHECK:       ## BB#0:
-; CHECK-NEXT:    vpcmpgtd (%rdi), %zmm0, %k1
-; CHECK-NEXT:    vmovdqa32 %zmm0, %zmm1 {%k1}
-; CHECK-NEXT:    vmovaps %zmm1, %zmm0
-; CHECK-NEXT:    retq
+; KNL-LABEL: test17:
+; KNL:       ## BB#0:
+; KNL-NEXT:    vpcmpgtd (%rdi), %zmm0, %k1
+; KNL-NEXT:    vmovdqa32 %zmm0, %zmm1 {%k1}
+; KNL-NEXT:    vmovaps %zmm1, %zmm0
+; KNL-NEXT:    retq
   %y = load <16 x i32>, <16 x i32>* %y.ptr, align 4
   %mask = icmp sgt <16 x i32> %x, %y
   %max = select <16 x i1> %mask, <16 x i32> %x, <16 x i32> %x1
@@ -222,12 +238,12 @@ define <16 x i32> @test17(<16 x i32> %x, <16 x i32> %x1, <16 x i32>* %y.ptr) nou
 }
 
 define <16 x i32> @test18(<16 x i32> %x, <16 x i32> %x1, <16 x i32>* %y.ptr) nounwind {
-; CHECK-LABEL: test18:
-; CHECK:       ## BB#0:
-; CHECK-NEXT:    vpcmpled (%rdi), %zmm0, %k1
-; CHECK-NEXT:    vmovdqa32 %zmm0, %zmm1 {%k1}
-; CHECK-NEXT:    vmovaps %zmm1, %zmm0
-; CHECK-NEXT:    retq
+; KNL-LABEL: test18:
+; KNL:       ## BB#0:
+; KNL-NEXT:    vpcmpled (%rdi), %zmm0, %k1
+; KNL-NEXT:    vmovdqa32 %zmm0, %zmm1 {%k1}
+; KNL-NEXT:    vmovaps %zmm1, %zmm0
+; KNL-NEXT:    retq
   %y = load <16 x i32>, <16 x i32>* %y.ptr, align 4
   %mask = icmp sle <16 x i32> %x, %y
   %max = select <16 x i1> %mask, <16 x i32> %x, <16 x i32> %x1
@@ -235,12 +251,12 @@ define <16 x i32> @test18(<16 x i32> %x, <16 x i32> %x1, <16 x i32>* %y.ptr) nou
 }
 
 define <16 x i32> @test19(<16 x i32> %x, <16 x i32> %x1, <16 x i32>* %y.ptr) nounwind {
-; CHECK-LABEL: test19:
-; CHECK:       ## BB#0:
-; CHECK-NEXT:    vpcmpleud (%rdi), %zmm0, %k1
-; CHECK-NEXT:    vmovdqa32 %zmm0, %zmm1 {%k1}
-; CHECK-NEXT:    vmovaps %zmm1, %zmm0
-; CHECK-NEXT:    retq
+; KNL-LABEL: test19:
+; KNL:       ## BB#0:
+; KNL-NEXT:    vpcmpleud (%rdi), %zmm0, %k1
+; KNL-NEXT:    vmovdqa32 %zmm0, %zmm1 {%k1}
+; KNL-NEXT:    vmovaps %zmm1, %zmm0
+; KNL-NEXT:    retq
   %y = load <16 x i32>, <16 x i32>* %y.ptr, align 4
   %mask = icmp ule <16 x i32> %x, %y
   %max = select <16 x i1> %mask, <16 x i32> %x, <16 x i32> %x1
@@ -248,13 +264,13 @@ define <16 x i32> @test19(<16 x i32> %x, <16 x i32> %x1, <16 x i32>* %y.ptr) nou
 }
 
 define <16 x i32> @test20(<16 x i32> %x, <16 x i32> %y, <16 x i32> %x1, <16 x i32> %y1) nounwind {
-; CHECK-LABEL: test20:
-; CHECK:       ## BB#0:
-; CHECK-NEXT:    vpcmpeqd %zmm1, %zmm0, %k1
-; CHECK-NEXT:    vpcmpeqd %zmm3, %zmm2, %k1 {%k1}
-; CHECK-NEXT:    vmovdqa32 %zmm0, %zmm1 {%k1}
-; CHECK-NEXT:    vmovaps %zmm1, %zmm0
-; CHECK-NEXT:    retq
+; KNL-LABEL: test20:
+; KNL:       ## BB#0:
+; KNL-NEXT:    vpcmpeqd %zmm1, %zmm0, %k1
+; KNL-NEXT:    vpcmpeqd %zmm3, %zmm2, %k1 {%k1}
+; KNL-NEXT:    vmovdqa32 %zmm0, %zmm1 {%k1}
+; KNL-NEXT:    vmovaps %zmm1, %zmm0
+; KNL-NEXT:    retq
   %mask1 = icmp eq <16 x i32> %x1, %y1
   %mask0 = icmp eq <16 x i32> %x, %y
   %mask = select <16 x i1> %mask0, <16 x i1> %mask1, <16 x i1> zeroinitializer
@@ -263,13 +279,13 @@ define <16 x i32> @test20(<16 x i32> %x, <16 x i32> %y, <16 x i32> %x1, <16 x i3
 }
 
 define <8 x i64> @test21(<8 x i64> %x, <8 x i64> %y, <8 x i64> %x1, <8 x i64> %y1) nounwind {
-; CHECK-LABEL: test21:
-; CHECK:       ## BB#0:
-; CHECK-NEXT:    vpcmpleq %zmm1, %zmm0, %k1
-; CHECK-NEXT:    vpcmpleq %zmm2, %zmm3, %k1 {%k1}
-; CHECK-NEXT:    vmovdqa64 %zmm0, %zmm2 {%k1}
-; CHECK-NEXT:    vmovaps %zmm2, %zmm0
-; CHECK-NEXT:    retq
+; KNL-LABEL: test21:
+; KNL:       ## BB#0:
+; KNL-NEXT:    vpcmpleq %zmm1, %zmm0, %k1
+; KNL-NEXT:    vpcmpleq %zmm2, %zmm3, %k1 {%k1}
+; KNL-NEXT:    vmovdqa64 %zmm0, %zmm2 {%k1}
+; KNL-NEXT:    vmovaps %zmm2, %zmm0
+; KNL-NEXT:    retq
   %mask1 = icmp sge <8 x i64> %x1, %y1
   %mask0 = icmp sle <8 x i64> %x, %y
   %mask = select <8 x i1> %mask0, <8 x i1> %mask1, <8 x i1> zeroinitializer
@@ -278,13 +294,13 @@ define <8 x i64> @test21(<8 x i64> %x, <8 x i64> %y, <8 x i64> %x1, <8 x i64> %y
 }
 
 define <8 x i64> @test22(<8 x i64> %x, <8 x i64>* %y.ptr, <8 x i64> %x1, <8 x i64> %y1) nounwind {
-; CHECK-LABEL: test22:
-; CHECK:       ## BB#0:
-; CHECK-NEXT:    vpcmpgtq %zmm2, %zmm1, %k1
-; CHECK-NEXT:    vpcmpgtq (%rdi), %zmm0, %k1 {%k1}
-; CHECK-NEXT:    vmovdqa64 %zmm0, %zmm1 {%k1}
-; CHECK-NEXT:    vmovaps %zmm1, %zmm0
-; CHECK-NEXT:    retq
+; KNL-LABEL: test22:
+; KNL:       ## BB#0:
+; KNL-NEXT:    vpcmpgtq %zmm2, %zmm1, %k1
+; KNL-NEXT:    vpcmpgtq (%rdi), %zmm0, %k1 {%k1}
+; KNL-NEXT:    vmovdqa64 %zmm0, %zmm1 {%k1}
+; KNL-NEXT:    vmovaps %zmm1, %zmm0
+; KNL-NEXT:    retq
   %mask1 = icmp sgt <8 x i64> %x1, %y1
   %y = load <8 x i64>, <8 x i64>* %y.ptr, align 4
   %mask0 = icmp sgt <8 x i64> %x, %y
@@ -294,13 +310,13 @@ define <8 x i64> @test22(<8 x i64> %x, <8 x i64>* %y.ptr, <8 x i64> %x1, <8 x i6
 }
 
 define <16 x i32> @test23(<16 x i32> %x, <16 x i32>* %y.ptr, <16 x i32> %x1, <16 x i32> %y1) nounwind {
-; CHECK-LABEL: test23:
-; CHECK:       ## BB#0:
-; CHECK-NEXT:    vpcmpled %zmm1, %zmm2, %k1
-; CHECK-NEXT:    vpcmpleud (%rdi), %zmm0, %k1 {%k1}
-; CHECK-NEXT:    vmovdqa32 %zmm0, %zmm1 {%k1}
-; CHECK-NEXT:    vmovaps %zmm1, %zmm0
-; CHECK-NEXT:    retq
+; KNL-LABEL: test23:
+; KNL:       ## BB#0:
+; KNL-NEXT:    vpcmpled %zmm1, %zmm2, %k1
+; KNL-NEXT:    vpcmpleud (%rdi), %zmm0, %k1 {%k1}
+; KNL-NEXT:    vmovdqa32 %zmm0, %zmm1 {%k1}
+; KNL-NEXT:    vmovaps %zmm1, %zmm0
+; KNL-NEXT:    retq
   %mask1 = icmp sge <16 x i32> %x1, %y1
   %y = load <16 x i32>, <16 x i32>* %y.ptr, align 4
   %mask0 = icmp ule <16 x i32> %x, %y
@@ -310,12 +326,12 @@ define <16 x i32> @test23(<16 x i32> %x, <16 x i32>* %y.ptr, <16 x i32> %x1, <16
 }
 
 define <8 x i64> @test24(<8 x i64> %x, <8 x i64> %x1, i64* %yb.ptr) nounwind {
-; CHECK-LABEL: test24:
-; CHECK:       ## BB#0:
-; CHECK-NEXT:    vpcmpeqq (%rdi){1to8}, %zmm0, %k1
-; CHECK-NEXT:    vmovdqa64 %zmm0, %zmm1 {%k1}
-; CHECK-NEXT:    vmovaps %zmm1, %zmm0
-; CHECK-NEXT:    retq
+; KNL-LABEL: test24:
+; KNL:       ## BB#0:
+; KNL-NEXT:    vpcmpeqq (%rdi){1to8}, %zmm0, %k1
+; KNL-NEXT:    vmovdqa64 %zmm0, %zmm1 {%k1}
+; KNL-NEXT:    vmovaps %zmm1, %zmm0
+; KNL-NEXT:    retq
   %yb = load i64, i64* %yb.ptr, align 4
   %y.0 = insertelement <8 x i64> undef, i64 %yb, i32 0
   %y = shufflevector <8 x i64> %y.0, <8 x i64> undef, <8 x i32> zeroinitializer
@@ -325,12 +341,12 @@ define <8 x i64> @test24(<8 x i64> %x, <8 x i64> %x1, i64* %yb.ptr) nounwind {
 }
 
 define <16 x i32> @test25(<16 x i32> %x, i32* %yb.ptr, <16 x i32> %x1) nounwind {
-; CHECK-LABEL: test25:
-; CHECK:       ## BB#0:
-; CHECK-NEXT:    vpcmpled (%rdi){1to16}, %zmm0, %k1
-; CHECK-NEXT:    vmovdqa32 %zmm0, %zmm1 {%k1}
-; CHECK-NEXT:    vmovaps %zmm1, %zmm0
-; CHECK-NEXT:    retq
+; KNL-LABEL: test25:
+; KNL:       ## BB#0:
+; KNL-NEXT:    vpcmpled (%rdi){1to16}, %zmm0, %k1
+; KNL-NEXT:    vmovdqa32 %zmm0, %zmm1 {%k1}
+; KNL-NEXT:    vmovaps %zmm1, %zmm0
+; KNL-NEXT:    retq
   %yb = load i32, i32* %yb.ptr, align 4
   %y.0 = insertelement <16 x i32> undef, i32 %yb, i32 0
   %y = shufflevector <16 x i32> %y.0, <16 x i32> undef, <16 x i32> zeroinitializer
@@ -340,13 +356,13 @@ define <16 x i32> @test25(<16 x i32> %x, i32* %yb.ptr, <16 x i32> %x1) nounwind
 }
 
 define <16 x i32> @test26(<16 x i32> %x, i32* %yb.ptr, <16 x i32> %x1, <16 x i32> %y1) nounwind {
-; CHECK-LABEL: test26:
-; CHECK:       ## BB#0:
-; CHECK-NEXT:    vpcmpled %zmm1, %zmm2, %k1
-; CHECK-NEXT:    vpcmpgtd (%rdi){1to16}, %zmm0, %k1 {%k1}
-; CHECK-NEXT:    vmovdqa32 %zmm0, %zmm1 {%k1}
-; CHECK-NEXT:    vmovaps %zmm1, %zmm0
-; CHECK-NEXT:    retq
+; KNL-LABEL: test26:
+; KNL:       ## BB#0:
+; KNL-NEXT:    vpcmpled %zmm1, %zmm2, %k1
+; KNL-NEXT:    vpcmpgtd (%rdi){1to16}, %zmm0, %k1 {%k1}
+; KNL-NEXT:    vmovdqa32 %zmm0, %zmm1 {%k1}
+; KNL-NEXT:    vmovaps %zmm1, %zmm0
+; KNL-NEXT:    retq
   %mask1 = icmp sge <16 x i32> %x1, %y1
   %yb = load i32, i32* %yb.ptr, align 4
   %y.0 = insertelement <16 x i32> undef, i32 %yb, i32 0
@@ -358,13 +374,13 @@ define <16 x i32> @test26(<16 x i32> %x, i32* %yb.ptr, <16 x i32> %x1, <16 x i32
 }
 
 define <8 x i64> @test27(<8 x i64> %x, i64* %yb.ptr, <8 x i64> %x1, <8 x i64> %y1) nounwind {
-; CHECK-LABEL: test27:
-; CHECK:       ## BB#0:
-; CHECK-NEXT:    vpcmpleq        %zmm1, %zmm2, %k1
-; CHECK-NEXT:    vpcmpleq        (%rdi){1to8}, %zmm0, %k1 {%k1}
-; CHECK-NEXT:    vmovdqa64 %zmm0, %zmm1 {%k1}
-; CHECK-NEXT:    vmovaps %zmm1, %zmm0
-; CHECK-NEXT:    retq
+; KNL-LABEL: test27:
+; KNL:       ## BB#0:
+; KNL-NEXT:    vpcmpleq        %zmm1, %zmm2, %k1
+; KNL-NEXT:    vpcmpleq        (%rdi){1to8}, %zmm0, %k1 {%k1}
+; KNL-NEXT:    vmovdqa64 %zmm0, %zmm1 {%k1}
+; KNL-NEXT:    vmovaps %zmm1, %zmm0
+; KNL-NEXT:    retq
   %mask1 = icmp sge <8 x i64> %x1, %y1
   %yb = load i64, i64* %yb.ptr, align 4
   %y.0 = insertelement <8 x i64> undef, i64 %yb, i32 0
@@ -374,3 +390,201 @@ define <8 x i64> @test27(<8 x i64> %x, i64* %yb.ptr, <8 x i64> %x1, <8 x i64> %y
   %max = select <8 x i1> %mask, <8 x i64> %x, <8 x i64> %x1
   ret <8 x i64> %max
 }
+
+; KNL-LABEL: test28
+; KNL: vpcmpgtq
+; KNL: vpcmpgtq
+; KNL: kxnorw
+define <8 x i32>@test28(<8 x i64> %x, <8 x i64> %y, <8 x i64> %x1, <8 x i64> %y1) {
+  %x_gt_y = icmp sgt <8 x i64> %x, %y
+  %x1_gt_y1 = icmp sgt <8 x i64> %x1, %y1
+  %res = icmp eq <8 x i1>%x_gt_y, %x1_gt_y1
+  %resse = sext <8 x i1>%res to <8 x i32>
+  ret <8 x i32> %resse
+}
+
+; KNL-LABEL: test29
+; KNL: vpcmpgtd
+; KNL: vpcmpgtd
+; KNL: kxorw
+define <16 x i8>@test29(<16 x i32> %x, <16 x i32> %y, <16 x i32> %x1, <16 x i32> %y1) {
+  %x_gt_y = icmp sgt <16 x i32> %x, %y
+  %x1_gt_y1 = icmp sgt <16 x i32> %x1, %y1
+  %res = icmp ne <16 x i1>%x_gt_y, %x1_gt_y1
+  %resse = sext <16 x i1>%res to <16 x i8>
+  ret <16 x i8> %resse
+}
+
+define <4 x double> @test30(<4 x double> %x, <4 x double> %y) nounwind {
+; SKX-LABEL: test30:
+; SKX: vcmpeqpd   %ymm1, %ymm0, %k1 
+; SKX: vmovapd    %ymm0, %ymm1 {%k1}
+
+  %mask = fcmp oeq <4 x double> %x, %y
+  %max = select <4 x i1> %mask, <4 x double> %x, <4 x double> %y
+  ret <4 x double> %max
+}
+
+define <2 x double> @test31(<2 x double> %x, <2 x double> %x1, <2 x double>* %yp) nounwind {
+; SKX-LABEL: test31:     
+; SKX: vcmpltpd        (%rdi), %xmm0, %k1 
+; SKX: vmovapd %xmm0, %xmm1 {%k1}
+
+  %y = load <2 x double>, <2 x double>* %yp, align 4
+  %mask = fcmp olt <2 x double> %x, %y
+  %max = select <2 x i1> %mask, <2 x double> %x, <2 x double> %x1
+  ret <2 x double> %max
+}
+
+define <4 x double> @test32(<4 x double> %x, <4 x double> %x1, <4 x double>* %yp) nounwind {
+; SKX-LABEL: test32:
+; SKX: vcmpltpd        (%rdi), %ymm0, %k1 
+; SKX: vmovapd %ymm0, %ymm1 {%k1}
+
+  %y = load <4 x double>, <4 x double>* %yp, align 4
+  %mask = fcmp ogt <4 x double> %y, %x
+  %max = select <4 x i1> %mask, <4 x double> %x, <4 x double> %x1
+  ret <4 x double> %max
+}
+
+define <8 x double> @test33(<8 x double> %x, <8 x double> %x1, <8 x double>* %yp) nounwind {
+; SKX-LABEL: test33:     
+; SKX: vcmpltpd        (%rdi), %zmm0, %k1 
+; SKX: vmovapd %zmm0, %zmm1 {%k1}
+  %y = load <8 x double>, <8 x double>* %yp, align 4
+  %mask = fcmp olt <8 x double> %x, %y
+  %max = select <8 x i1> %mask, <8 x double> %x, <8 x double> %x1
+  ret <8 x double> %max
+}
+
+define <4 x float> @test34(<4 x float> %x, <4 x float> %x1, <4 x float>* %yp) nounwind {
+; SKX-LABEL: test34:     
+; SKX: vcmpltps        (%rdi), %xmm0, %k1 
+; SKX: vmovaps %xmm0, %xmm1 {%k1}
+  %y = load <4 x float>, <4 x float>* %yp, align 4
+  %mask = fcmp olt <4 x float> %x, %y
+  %max = select <4 x i1> %mask, <4 x float> %x, <4 x float> %x1
+  ret <4 x float> %max
+}
+
+define <8 x float> @test35(<8 x float> %x, <8 x float> %x1, <8 x float>* %yp) nounwind {
+; SKX-LABEL: test35:
+; SKX: vcmpltps        (%rdi), %ymm0, %k1 
+; SKX: vmovaps %ymm0, %ymm1 {%k1}
+
+  %y = load <8 x float>, <8 x float>* %yp, align 4
+  %mask = fcmp ogt <8 x float> %y, %x
+  %max = select <8 x i1> %mask, <8 x float> %x, <8 x float> %x1
+  ret <8 x float> %max
+}
+
+define <16 x float> @test36(<16 x float> %x, <16 x float> %x1, <16 x float>* %yp) nounwind {
+; SKX-LABEL: test36:     
+; SKX: vcmpltps        (%rdi), %zmm0, %k1 
+; SKX: vmovaps %zmm0, %zmm1 {%k1}
+  %y = load <16 x float>, <16 x float>* %yp, align 4
+  %mask = fcmp olt <16 x float> %x, %y
+  %max = select <16 x i1> %mask, <16 x float> %x, <16 x float> %x1
+  ret <16 x float> %max
+}
+
+define <8 x double> @test37(<8 x double> %x, <8 x double> %x1, double* %ptr) nounwind {
+; SKX-LABEL: test37:                                
+; SKX: vcmpltpd  (%rdi){1to8}, %zmm0, %k1 
+; SKX: vmovapd %zmm0, %zmm1 {%k1}
+
+  %a = load double, double* %ptr
+  %v = insertelement <8 x double> undef, double %a, i32 0
+  %shuffle = shufflevector <8 x double> %v, <8 x double> undef, <8 x i32> zeroinitializer
+
+  %mask = fcmp ogt <8 x double> %shuffle, %x
+  %max = select <8 x i1> %mask, <8 x double> %x, <8 x double> %x1
+  ret <8 x double> %max
+}
+
+define <4 x double> @test38(<4 x double> %x, <4 x double> %x1, double* %ptr) nounwind {
+; SKX-LABEL: test38:                                
+; SKX: vcmpltpd  (%rdi){1to4}, %ymm0, %k1 
+; SKX: vmovapd %ymm0, %ymm1 {%k1}
+
+  %a = load double, double* %ptr
+  %v = insertelement <4 x double> undef, double %a, i32 0
+  %shuffle = shufflevector <4 x double> %v, <4 x double> undef, <4 x i32> zeroinitializer
+  
+  %mask = fcmp ogt <4 x double> %shuffle, %x
+  %max = select <4 x i1> %mask, <4 x double> %x, <4 x double> %x1
+  ret <4 x double> %max
+}
+
+define <2 x double> @test39(<2 x double> %x, <2 x double> %x1, double* %ptr) nounwind {
+; SKX-LABEL: test39:                                
+; SKX: vcmpltpd  (%rdi){1to2}, %xmm0, %k1 
+; SKX: vmovapd %xmm0, %xmm1 {%k1}
+
+  %a = load double, double* %ptr
+  %v = insertelement <2 x double> undef, double %a, i32 0
+  %shuffle = shufflevector <2 x double> %v, <2 x double> undef, <2 x i32> <i32 0, i32 0>
+  
+  %mask = fcmp ogt <2 x double> %shuffle, %x
+  %max = select <2 x i1> %mask, <2 x double> %x, <2 x double> %x1
+  ret <2 x double> %max
+}
+
+
+define <16  x float> @test40(<16  x float> %x, <16  x float> %x1, float* %ptr) nounwind {
+; SKX-LABEL: test40:                                
+; SKX: vcmpltps  (%rdi){1to16}, %zmm0, %k1 
+; SKX: vmovaps %zmm0, %zmm1 {%k1}
+
+  %a = load float, float* %ptr
+  %v = insertelement <16  x float> undef, float %a, i32 0
+  %shuffle = shufflevector <16  x float> %v, <16  x float> undef, <16 x i32> <i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0>
+  
+  %mask = fcmp ogt <16  x float> %shuffle, %x
+  %max = select <16 x i1> %mask, <16  x float> %x, <16  x float> %x1
+  ret <16  x float> %max
+}
+
+define <8  x float> @test41(<8  x float> %x, <8  x float> %x1, float* %ptr) nounwind {
+; SKX-LABEL: test41:                                
+; SKX: vcmpltps  (%rdi){1to8}, %ymm0, %k1 
+; SKX: vmovaps %ymm0, %ymm1 {%k1}
+
+  %a = load float, float* %ptr
+  %v = insertelement <8  x float> undef, float %a, i32 0
+  %shuffle = shufflevector <8  x float> %v, <8  x float> undef, <8 x i32> <i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0>
+  
+  %mask = fcmp ogt <8  x float> %shuffle, %x
+  %max = select <8 x i1> %mask, <8  x float> %x, <8  x float> %x1
+  ret <8  x float> %max
+}
+
+define <4  x float> @test42(<4  x float> %x, <4  x float> %x1, float* %ptr) nounwind {
+; SKX-LABEL: test42:                                
+; SKX: vcmpltps  (%rdi){1to4}, %xmm0, %k1 
+; SKX: vmovaps %xmm0, %xmm1 {%k1}
+  
+  %a = load float, float* %ptr
+  %v = insertelement <4  x float> undef, float %a, i32 0
+  %shuffle = shufflevector <4  x float> %v, <4  x float> undef, <4 x i32> <i32 0, i32 0, i32 0, i32 0>
+  
+  %mask = fcmp ogt <4  x float> %shuffle, %x
+  %max = select <4 x i1> %mask, <4  x float> %x, <4  x float> %x1
+  ret <4  x float> %max
+}
+
+define <8 x double> @test43(<8 x double> %x, <8 x double> %x1, double* %ptr,<8 x i1> %mask_in) nounwind {
+; SKX-LABEL: test43:                                
+; SKX: vpmovw2m  %xmm2, %k1
+; SKX: vcmpltpd  (%rdi){1to8}, %zmm0, %k1 {%k1}
+; SKX: vmovapd %zmm0, %zmm1 {%k1}
+
+  %a = load double, double* %ptr
+  %v = insertelement <8 x double> undef, double %a, i32 0
+  %shuffle = shufflevector <8 x double> %v, <8 x double> undef, <8 x i32> zeroinitializer
+  
+  %mask_cmp = fcmp ogt <8 x double> %shuffle, %x
+  %mask = and <8 x i1> %mask_cmp, %mask_in
+  %max = select <8 x i1> %mask, <8 x double> %x, <8 x double> %x1
+  ret <8 x double> %max
+}
diff --git a/test/CodeGen/X86/avx512bw-intrinsics.ll b/test/CodeGen/X86/avx512bw-intrinsics.ll
index 308de16aefd..9d4c8beafd3 100644
--- a/test/CodeGen/X86/avx512bw-intrinsics.ll
+++ b/test/CodeGen/X86/avx512bw-intrinsics.ll
@@ -1,4 +1,4 @@
-; RUN: llc < %s -mtriple=x86_64-apple-darwin -mcpu=skx --show-mc-encoding| FileCheck %s
+; RUN: llc < %s -mtriple=x86_64-apple-darwin -mcpu=knl -mattr=+avx512bw --show-mc-encoding| FileCheck %s
 
 define i64 @test_pcmpeq_b(<64 x i8> %a, <64 x i8> %b) {
 ; CHECK-LABEL: test_pcmpeq_b
@@ -67,28 +67,28 @@ declare i32 @llvm.x86.avx512.mask.pcmpgt.w.512(<32 x i16>, <32 x i16>, i32)
 define <8 x i64> @test_cmp_b_512(<64 x i8> %a0, <64 x i8> %a1) {
 ; CHECK_LABEL: test_cmp_b_512
 ; CHECK: vpcmpeqb %zmm1, %zmm0, %k0 ##
-  %res0 = call i64 @llvm.x86.avx512.mask.cmp.b.512(<64 x i8> %a0, <64 x i8> %a1, i8 0, i64 -1)
+  %res0 = call i64 @llvm.x86.avx512.mask.cmp.b.512(<64 x i8> %a0, <64 x i8> %a1, i32 0, i64 -1)
   %vec0 = insertelement <8 x i64> undef, i64 %res0, i32 0
 ; CHECK: vpcmpltb %zmm1, %zmm0, %k0 ##
-  %res1 = call i64 @llvm.x86.avx512.mask.cmp.b.512(<64 x i8> %a0, <64 x i8> %a1, i8 1, i64 -1)
+  %res1 = call i64 @llvm.x86.avx512.mask.cmp.b.512(<64 x i8> %a0, <64 x i8> %a1, i32 1, i64 -1)
   %vec1 = insertelement <8 x i64> %vec0, i64 %res1, i32 1
 ; CHECK: vpcmpleb %zmm1, %zmm0, %k0 ##
-  %res2 = call i64 @llvm.x86.avx512.mask.cmp.b.512(<64 x i8> %a0, <64 x i8> %a1, i8 2, i64 -1)
+  %res2 = call i64 @llvm.x86.avx512.mask.cmp.b.512(<64 x i8> %a0, <64 x i8> %a1, i32 2, i64 -1)
   %vec2 = insertelement <8 x i64> %vec1, i64 %res2, i32 2
 ; CHECK: vpcmpunordb %zmm1, %zmm0, %k0 ##
-  %res3 = call i64 @llvm.x86.avx512.mask.cmp.b.512(<64 x i8> %a0, <64 x i8> %a1, i8 3, i64 -1)
+  %res3 = call i64 @llvm.x86.avx512.mask.cmp.b.512(<64 x i8> %a0, <64 x i8> %a1, i32 3, i64 -1)
   %vec3 = insertelement <8 x i64> %vec2, i64 %res3, i32 3
 ; CHECK: vpcmpneqb %zmm1, %zmm0, %k0 ##
-  %res4 = call i64 @llvm.x86.avx512.mask.cmp.b.512(<64 x i8> %a0, <64 x i8> %a1, i8 4, i64 -1)
+  %res4 = call i64 @llvm.x86.avx512.mask.cmp.b.512(<64 x i8> %a0, <64 x i8> %a1, i32 4, i64 -1)
   %vec4 = insertelement <8 x i64> %vec3, i64 %res4, i32 4
 ; CHECK: vpcmpnltb %zmm1, %zmm0, %k0 ##
-  %res5 = call i64 @llvm.x86.avx512.mask.cmp.b.512(<64 x i8> %a0, <64 x i8> %a1, i8 5, i64 -1)
+  %res5 = call i64 @llvm.x86.avx512.mask.cmp.b.512(<64 x i8> %a0, <64 x i8> %a1, i32 5, i64 -1)
   %vec5 = insertelement <8 x i64> %vec4, i64 %res5, i32 5
 ; CHECK: vpcmpnleb %zmm1, %zmm0, %k0 ##
-  %res6 = call i64 @llvm.x86.avx512.mask.cmp.b.512(<64 x i8> %a0, <64 x i8> %a1, i8 6, i64 -1)
+  %res6 = call i64 @llvm.x86.avx512.mask.cmp.b.512(<64 x i8> %a0, <64 x i8> %a1, i32 6, i64 -1)
   %vec6 = insertelement <8 x i64> %vec5, i64 %res6, i32 6
 ; CHECK: vpcmpordb %zmm1, %zmm0, %k0 ##
-  %res7 = call i64 @llvm.x86.avx512.mask.cmp.b.512(<64 x i8> %a0, <64 x i8> %a1, i8 7, i64 -1)
+  %res7 = call i64 @llvm.x86.avx512.mask.cmp.b.512(<64 x i8> %a0, <64 x i8> %a1, i32 7, i64 -1)
   %vec7 = insertelement <8 x i64> %vec6, i64 %res7, i32 7
   ret <8 x i64> %vec7
 }
@@ -96,59 +96,59 @@ define <8 x i64> @test_cmp_b_512(<64 x i8> %a0, <64 x i8> %a1) {
 define <8 x i64> @test_mask_cmp_b_512(<64 x i8> %a0, <64 x i8> %a1, i64 %mask) {
 ; CHECK_LABEL: test_mask_cmp_b_512
 ; CHECK: vpcmpeqb %zmm1, %zmm0, %k0 {%k1} ##
-  %res0 = call i64 @llvm.x86.avx512.mask.cmp.b.512(<64 x i8> %a0, <64 x i8> %a1, i8 0, i64 %mask)
+  %res0 = call i64 @llvm.x86.avx512.mask.cmp.b.512(<64 x i8> %a0, <64 x i8> %a1, i32 0, i64 %mask)
   %vec0 = insertelement <8 x i64> undef, i64 %res0, i32 0
 ; CHECK: vpcmpltb %zmm1, %zmm0, %k0 {%k1} ##
-  %res1 = call i64 @llvm.x86.avx512.mask.cmp.b.512(<64 x i8> %a0, <64 x i8> %a1, i8 1, i64 %mask)
+  %res1 = call i64 @llvm.x86.avx512.mask.cmp.b.512(<64 x i8> %a0, <64 x i8> %a1, i32 1, i64 %mask)
   %vec1 = insertelement <8 x i64> %vec0, i64 %res1, i32 1
 ; CHECK: vpcmpleb %zmm1, %zmm0, %k0 {%k1} ##
-  %res2 = call i64 @llvm.x86.avx512.mask.cmp.b.512(<64 x i8> %a0, <64 x i8> %a1, i8 2, i64 %mask)
+  %res2 = call i64 @llvm.x86.avx512.mask.cmp.b.512(<64 x i8> %a0, <64 x i8> %a1, i32 2, i64 %mask)
   %vec2 = insertelement <8 x i64> %vec1, i64 %res2, i32 2
 ; CHECK: vpcmpunordb %zmm1, %zmm0, %k0 {%k1} ##
-  %res3 = call i64 @llvm.x86.avx512.mask.cmp.b.512(<64 x i8> %a0, <64 x i8> %a1, i8 3, i64 %mask)
+  %res3 = call i64 @llvm.x86.avx512.mask.cmp.b.512(<64 x i8> %a0, <64 x i8> %a1, i32 3, i64 %mask)
   %vec3 = insertelement <8 x i64> %vec2, i64 %res3, i32 3
 ; CHECK: vpcmpneqb %zmm1, %zmm0, %k0 {%k1} ##
-  %res4 = call i64 @llvm.x86.avx512.mask.cmp.b.512(<64 x i8> %a0, <64 x i8> %a1, i8 4, i64 %mask)
+  %res4 = call i64 @llvm.x86.avx512.mask.cmp.b.512(<64 x i8> %a0, <64 x i8> %a1, i32 4, i64 %mask)
   %vec4 = insertelement <8 x i64> %vec3, i64 %res4, i32 4
 ; CHECK: vpcmpnltb %zmm1, %zmm0, %k0 {%k1} ##
-  %res5 = call i64 @llvm.x86.avx512.mask.cmp.b.512(<64 x i8> %a0, <64 x i8> %a1, i8 5, i64 %mask)
+  %res5 = call i64 @llvm.x86.avx512.mask.cmp.b.512(<64 x i8> %a0, <64 x i8> %a1, i32 5, i64 %mask)
   %vec5 = insertelement <8 x i64> %vec4, i64 %res5, i32 5
 ; CHECK: vpcmpnleb %zmm1, %zmm0, %k0 {%k1} ##
-  %res6 = call i64 @llvm.x86.avx512.mask.cmp.b.512(<64 x i8> %a0, <64 x i8> %a1, i8 6, i64 %mask)
+  %res6 = call i64 @llvm.x86.avx512.mask.cmp.b.512(<64 x i8> %a0, <64 x i8> %a1, i32 6, i64 %mask)
   %vec6 = insertelement <8 x i64> %vec5, i64 %res6, i32 6
 ; CHECK: vpcmpordb %zmm1, %zmm0, %k0 {%k1} ##
-  %res7 = call i64 @llvm.x86.avx512.mask.cmp.b.512(<64 x i8> %a0, <64 x i8> %a1, i8 7, i64 %mask)
+  %res7 = call i64 @llvm.x86.avx512.mask.cmp.b.512(<64 x i8> %a0, <64 x i8> %a1, i32 7, i64 %mask)
   %vec7 = insertelement <8 x i64> %vec6, i64 %res7, i32 7
   ret <8 x i64> %vec7
 }
 
-declare i64 @llvm.x86.avx512.mask.cmp.b.512(<64 x i8>, <64 x i8>, i8, i64) nounwind readnone
+declare i64 @llvm.x86.avx512.mask.cmp.b.512(<64 x i8>, <64 x i8>, i32, i64) nounwind readnone
 
 define <8 x i64> @test_ucmp_b_512(<64 x i8> %a0, <64 x i8> %a1) {
 ; CHECK_LABEL: test_ucmp_b_512
 ; CHECK: vpcmpequb %zmm1, %zmm0, %k0 ##
-  %res0 = call i64 @llvm.x86.avx512.mask.ucmp.b.512(<64 x i8> %a0, <64 x i8> %a1, i8 0, i64 -1)
+  %res0 = call i64 @llvm.x86.avx512.mask.ucmp.b.512(<64 x i8> %a0, <64 x i8> %a1, i32 0, i64 -1)
   %vec0 = insertelement <8 x i64> undef, i64 %res0, i32 0
 ; CHECK: vpcmpltub %zmm1, %zmm0, %k0 ##
-  %res1 = call i64 @llvm.x86.avx512.mask.ucmp.b.512(<64 x i8> %a0, <64 x i8> %a1, i8 1, i64 -1)
+  %res1 = call i64 @llvm.x86.avx512.mask.ucmp.b.512(<64 x i8> %a0, <64 x i8> %a1, i32 1, i64 -1)
   %vec1 = insertelement <8 x i64> %vec0, i64 %res1, i32 1
 ; CHECK: vpcmpleub %zmm1, %zmm0, %k0 ##
-  %res2 = call i64 @llvm.x86.avx512.mask.ucmp.b.512(<64 x i8> %a0, <64 x i8> %a1, i8 2, i64 -1)
+  %res2 = call i64 @llvm.x86.avx512.mask.ucmp.b.512(<64 x i8> %a0, <64 x i8> %a1, i32 2, i64 -1)
   %vec2 = insertelement <8 x i64> %vec1, i64 %res2, i32 2
 ; CHECK: vpcmpunordub %zmm1, %zmm0, %k0 ##
-  %res3 = call i64 @llvm.x86.avx512.mask.ucmp.b.512(<64 x i8> %a0, <64 x i8> %a1, i8 3, i64 -1)
+  %res3 = call i64 @llvm.x86.avx512.mask.ucmp.b.512(<64 x i8> %a0, <64 x i8> %a1, i32 3, i64 -1)
   %vec3 = insertelement <8 x i64> %vec2, i64 %res3, i32 3
 ; CHECK: vpcmpnequb %zmm1, %zmm0, %k0 ##
-  %res4 = call i64 @llvm.x86.avx512.mask.ucmp.b.512(<64 x i8> %a0, <64 x i8> %a1, i8 4, i64 -1)
+  %res4 = call i64 @llvm.x86.avx512.mask.ucmp.b.512(<64 x i8> %a0, <64 x i8> %a1, i32 4, i64 -1)
   %vec4 = insertelement <8 x i64> %vec3, i64 %res4, i32 4
 ; CHECK: vpcmpnltub %zmm1, %zmm0, %k0 ##
-  %res5 = call i64 @llvm.x86.avx512.mask.ucmp.b.512(<64 x i8> %a0, <64 x i8> %a1, i8 5, i64 -1)
+  %res5 = call i64 @llvm.x86.avx512.mask.ucmp.b.512(<64 x i8> %a0, <64 x i8> %a1, i32 5, i64 -1)
   %vec5 = insertelement <8 x i64> %vec4, i64 %res5, i32 5
 ; CHECK: vpcmpnleub %zmm1, %zmm0, %k0 ##
-  %res6 = call i64 @llvm.x86.avx512.mask.ucmp.b.512(<64 x i8> %a0, <64 x i8> %a1, i8 6, i64 -1)
+  %res6 = call i64 @llvm.x86.avx512.mask.ucmp.b.512(<64 x i8> %a0, <64 x i8> %a1, i32 6, i64 -1)
   %vec6 = insertelement <8 x i64> %vec5, i64 %res6, i32 6
 ; CHECK: vpcmpordub %zmm1, %zmm0, %k0 ##
-  %res7 = call i64 @llvm.x86.avx512.mask.ucmp.b.512(<64 x i8> %a0, <64 x i8> %a1, i8 7, i64 -1)
+  %res7 = call i64 @llvm.x86.avx512.mask.ucmp.b.512(<64 x i8> %a0, <64 x i8> %a1, i32 7, i64 -1)
   %vec7 = insertelement <8 x i64> %vec6, i64 %res7, i32 7
   ret <8 x i64> %vec7
 }
@@ -156,59 +156,59 @@ define <8 x i64> @test_ucmp_b_512(<64 x i8> %a0, <64 x i8> %a1) {
 define <8 x i64> @test_mask_x86_avx512_ucmp_b_512(<64 x i8> %a0, <64 x i8> %a1, i64 %mask) {
 ; CHECK_LABEL: test_mask_ucmp_b_512
 ; CHECK: vpcmpequb %zmm1, %zmm0, %k0 {%k1} ##
-  %res0 = call i64 @llvm.x86.avx512.mask.ucmp.b.512(<64 x i8> %a0, <64 x i8> %a1, i8 0, i64 %mask)
+  %res0 = call i64 @llvm.x86.avx512.mask.ucmp.b.512(<64 x i8> %a0, <64 x i8> %a1, i32 0, i64 %mask)
   %vec0 = insertelement <8 x i64> undef, i64 %res0, i32 0
 ; CHECK: vpcmpltub %zmm1, %zmm0, %k0 {%k1} ##
-  %res1 = call i64 @llvm.x86.avx512.mask.ucmp.b.512(<64 x i8> %a0, <64 x i8> %a1, i8 1, i64 %mask)
+  %res1 = call i64 @llvm.x86.avx512.mask.ucmp.b.512(<64 x i8> %a0, <64 x i8> %a1, i32 1, i64 %mask)
   %vec1 = insertelement <8 x i64> %vec0, i64 %res1, i32 1
 ; CHECK: vpcmpleub %zmm1, %zmm0, %k0 {%k1} ##
-  %res2 = call i64 @llvm.x86.avx512.mask.ucmp.b.512(<64 x i8> %a0, <64 x i8> %a1, i8 2, i64 %mask)
+  %res2 = call i64 @llvm.x86.avx512.mask.ucmp.b.512(<64 x i8> %a0, <64 x i8> %a1, i32 2, i64 %mask)
   %vec2 = insertelement <8 x i64> %vec1, i64 %res2, i32 2
 ; CHECK: vpcmpunordub %zmm1, %zmm0, %k0 {%k1} ##
-  %res3 = call i64 @llvm.x86.avx512.mask.ucmp.b.512(<64 x i8> %a0, <64 x i8> %a1, i8 3, i64 %mask)
+  %res3 = call i64 @llvm.x86.avx512.mask.ucmp.b.512(<64 x i8> %a0, <64 x i8> %a1, i32 3, i64 %mask)
   %vec3 = insertelement <8 x i64> %vec2, i64 %res3, i32 3
 ; CHECK: vpcmpnequb %zmm1, %zmm0, %k0 {%k1} ##
-  %res4 = call i64 @llvm.x86.avx512.mask.ucmp.b.512(<64 x i8> %a0, <64 x i8> %a1, i8 4, i64 %mask)
+  %res4 = call i64 @llvm.x86.avx512.mask.ucmp.b.512(<64 x i8> %a0, <64 x i8> %a1, i32 4, i64 %mask)
   %vec4 = insertelement <8 x i64> %vec3, i64 %res4, i32 4
 ; CHECK: vpcmpnltub %zmm1, %zmm0, %k0 {%k1} ##
-  %res5 = call i64 @llvm.x86.avx512.mask.ucmp.b.512(<64 x i8> %a0, <64 x i8> %a1, i8 5, i64 %mask)
+  %res5 = call i64 @llvm.x86.avx512.mask.ucmp.b.512(<64 x i8> %a0, <64 x i8> %a1, i32 5, i64 %mask)
   %vec5 = insertelement <8 x i64> %vec4, i64 %res5, i32 5
 ; CHECK: vpcmpnleub %zmm1, %zmm0, %k0 {%k1} ##
-  %res6 = call i64 @llvm.x86.avx512.mask.ucmp.b.512(<64 x i8> %a0, <64 x i8> %a1, i8 6, i64 %mask)
+  %res6 = call i64 @llvm.x86.avx512.mask.ucmp.b.512(<64 x i8> %a0, <64 x i8> %a1, i32 6, i64 %mask)
   %vec6 = insertelement <8 x i64> %vec5, i64 %res6, i32 6
 ; CHECK: vpcmpordub %zmm1, %zmm0, %k0 {%k1} ##
-  %res7 = call i64 @llvm.x86.avx512.mask.ucmp.b.512(<64 x i8> %a0, <64 x i8> %a1, i8 7, i64 %mask)
+  %res7 = call i64 @llvm.x86.avx512.mask.ucmp.b.512(<64 x i8> %a0, <64 x i8> %a1, i32 7, i64 %mask)
   %vec7 = insertelement <8 x i64> %vec6, i64 %res7, i32 7
   ret <8 x i64> %vec7
 }
 
-declare i64 @llvm.x86.avx512.mask.ucmp.b.512(<64 x i8>, <64 x i8>, i8, i64) nounwind readnone
+declare i64 @llvm.x86.avx512.mask.ucmp.b.512(<64 x i8>, <64 x i8>, i32, i64) nounwind readnone
 
 define <8 x i32> @test_cmp_w_512(<32 x i16> %a0, <32 x i16> %a1) {
 ; CHECK_LABEL: test_cmp_w_512
 ; CHECK: vpcmpeqw %zmm1, %zmm0, %k0 ##
-  %res0 = call i32 @llvm.x86.avx512.mask.cmp.w.512(<32 x i16> %a0, <32 x i16> %a1, i8 0, i32 -1)
+  %res0 = call i32 @llvm.x86.avx512.mask.cmp.w.512(<32 x i16> %a0, <32 x i16> %a1, i32 0, i32 -1)
   %vec0 = insertelement <8 x i32> undef, i32 %res0, i32 0
 ; CHECK: vpcmpltw %zmm1, %zmm0, %k0 ##
-  %res1 = call i32 @llvm.x86.avx512.mask.cmp.w.512(<32 x i16> %a0, <32 x i16> %a1, i8 1, i32 -1)
+  %res1 = call i32 @llvm.x86.avx512.mask.cmp.w.512(<32 x i16> %a0, <32 x i16> %a1, i32 1, i32 -1)
   %vec1 = insertelement <8 x i32> %vec0, i32 %res1, i32 1
 ; CHECK: vpcmplew %zmm1, %zmm0, %k0 ##
-  %res2 = call i32 @llvm.x86.avx512.mask.cmp.w.512(<32 x i16> %a0, <32 x i16> %a1, i8 2, i32 -1)
+  %res2 = call i32 @llvm.x86.avx512.mask.cmp.w.512(<32 x i16> %a0, <32 x i16> %a1, i32 2, i32 -1)
   %vec2 = insertelement <8 x i32> %vec1, i32 %res2, i32 2
 ; CHECK: vpcmpunordw %zmm1, %zmm0, %k0 ##
-  %res3 = call i32 @llvm.x86.avx512.mask.cmp.w.512(<32 x i16> %a0, <32 x i16> %a1, i8 3, i32 -1)
+  %res3 = call i32 @llvm.x86.avx512.mask.cmp.w.512(<32 x i16> %a0, <32 x i16> %a1, i32 3, i32 -1)
   %vec3 = insertelement <8 x i32> %vec2, i32 %res3, i32 3
 ; CHECK: vpcmpneqw %zmm1, %zmm0, %k0 ##
-  %res4 = call i32 @llvm.x86.avx512.mask.cmp.w.512(<32 x i16> %a0, <32 x i16> %a1, i8 4, i32 -1)
+  %res4 = call i32 @llvm.x86.avx512.mask.cmp.w.512(<32 x i16> %a0, <32 x i16> %a1, i32 4, i32 -1)
   %vec4 = insertelement <8 x i32> %vec3, i32 %res4, i32 4
 ; CHECK: vpcmpnltw %zmm1, %zmm0, %k0 ##
-  %res5 = call i32 @llvm.x86.avx512.mask.cmp.w.512(<32 x i16> %a0, <32 x i16> %a1, i8 5, i32 -1)
+  %res5 = call i32 @llvm.x86.avx512.mask.cmp.w.512(<32 x i16> %a0, <32 x i16> %a1, i32 5, i32 -1)
   %vec5 = insertelement <8 x i32> %vec4, i32 %res5, i32 5
 ; CHECK: vpcmpnlew %zmm1, %zmm0, %k0 ##
-  %res6 = call i32 @llvm.x86.avx512.mask.cmp.w.512(<32 x i16> %a0, <32 x i16> %a1, i8 6, i32 -1)
+  %res6 = call i32 @llvm.x86.avx512.mask.cmp.w.512(<32 x i16> %a0, <32 x i16> %a1, i32 6, i32 -1)
   %vec6 = insertelement <8 x i32> %vec5, i32 %res6, i32 6
 ; CHECK: vpcmpordw %zmm1, %zmm0, %k0 ##
-  %res7 = call i32 @llvm.x86.avx512.mask.cmp.w.512(<32 x i16> %a0, <32 x i16> %a1, i8 7, i32 -1)
+  %res7 = call i32 @llvm.x86.avx512.mask.cmp.w.512(<32 x i16> %a0, <32 x i16> %a1, i32 7, i32 -1)
   %vec7 = insertelement <8 x i32> %vec6, i32 %res7, i32 7
   ret <8 x i32> %vec7
 }
@@ -216,59 +216,59 @@ define <8 x i32> @test_cmp_w_512(<32 x i16> %a0, <32 x i16> %a1) {
 define <8 x i32> @test_mask_cmp_w_512(<32 x i16> %a0, <32 x i16> %a1, i32 %mask) {
 ; CHECK_LABEL: test_mask_cmp_w_512
 ; CHECK: vpcmpeqw %zmm1, %zmm0, %k0 {%k1} ##
-  %res0 = call i32 @llvm.x86.avx512.mask.cmp.w.512(<32 x i16> %a0, <32 x i16> %a1, i8 0, i32 %mask)
+  %res0 = call i32 @llvm.x86.avx512.mask.cmp.w.512(<32 x i16> %a0, <32 x i16> %a1, i32 0, i32 %mask)
   %vec0 = insertelement <8 x i32> undef, i32 %res0, i32 0
 ; CHECK: vpcmpltw %zmm1, %zmm0, %k0 {%k1} ##
-  %res1 = call i32 @llvm.x86.avx512.mask.cmp.w.512(<32 x i16> %a0, <32 x i16> %a1, i8 1, i32 %mask)
+  %res1 = call i32 @llvm.x86.avx512.mask.cmp.w.512(<32 x i16> %a0, <32 x i16> %a1, i32 1, i32 %mask)
   %vec1 = insertelement <8 x i32> %vec0, i32 %res1, i32 1
 ; CHECK: vpcmplew %zmm1, %zmm0, %k0 {%k1} ##
-  %res2 = call i32 @llvm.x86.avx512.mask.cmp.w.512(<32 x i16> %a0, <32 x i16> %a1, i8 2, i32 %mask)
+  %res2 = call i32 @llvm.x86.avx512.mask.cmp.w.512(<32 x i16> %a0, <32 x i16> %a1, i32 2, i32 %mask)
   %vec2 = insertelement <8 x i32> %vec1, i32 %res2, i32 2
 ; CHECK: vpcmpunordw %zmm1, %zmm0, %k0 {%k1} ##
-  %res3 = call i32 @llvm.x86.avx512.mask.cmp.w.512(<32 x i16> %a0, <32 x i16> %a1, i8 3, i32 %mask)
+  %res3 = call i32 @llvm.x86.avx512.mask.cmp.w.512(<32 x i16> %a0, <32 x i16> %a1, i32 3, i32 %mask)
   %vec3 = insertelement <8 x i32> %vec2, i32 %res3, i32 3
 ; CHECK: vpcmpneqw %zmm1, %zmm0, %k0 {%k1} ##
-  %res4 = call i32 @llvm.x86.avx512.mask.cmp.w.512(<32 x i16> %a0, <32 x i16> %a1, i8 4, i32 %mask)
+  %res4 = call i32 @llvm.x86.avx512.mask.cmp.w.512(<32 x i16> %a0, <32 x i16> %a1, i32 4, i32 %mask)
   %vec4 = insertelement <8 x i32> %vec3, i32 %res4, i32 4
 ; CHECK: vpcmpnltw %zmm1, %zmm0, %k0 {%k1} ##
-  %res5 = call i32 @llvm.x86.avx512.mask.cmp.w.512(<32 x i16> %a0, <32 x i16> %a1, i8 5, i32 %mask)
+  %res5 = call i32 @llvm.x86.avx512.mask.cmp.w.512(<32 x i16> %a0, <32 x i16> %a1, i32 5, i32 %mask)
   %vec5 = insertelement <8 x i32> %vec4, i32 %res5, i32 5
 ; CHECK: vpcmpnlew %zmm1, %zmm0, %k0 {%k1} ##
-  %res6 = call i32 @llvm.x86.avx512.mask.cmp.w.512(<32 x i16> %a0, <32 x i16> %a1, i8 6, i32 %mask)
+  %res6 = call i32 @llvm.x86.avx512.mask.cmp.w.512(<32 x i16> %a0, <32 x i16> %a1, i32 6, i32 %mask)
   %vec6 = insertelement <8 x i32> %vec5, i32 %res6, i32 6
 ; CHECK: vpcmpordw %zmm1, %zmm0, %k0 {%k1} ##
-  %res7 = call i32 @llvm.x86.avx512.mask.cmp.w.512(<32 x i16> %a0, <32 x i16> %a1, i8 7, i32 %mask)
+  %res7 = call i32 @llvm.x86.avx512.mask.cmp.w.512(<32 x i16> %a0, <32 x i16> %a1, i32 7, i32 %mask)
   %vec7 = insertelement <8 x i32> %vec6, i32 %res7, i32 7
   ret <8 x i32> %vec7
 }
 
-declare i32 @llvm.x86.avx512.mask.cmp.w.512(<32 x i16>, <32 x i16>, i8, i32) nounwind readnone
+declare i32 @llvm.x86.avx512.mask.cmp.w.512(<32 x i16>, <32 x i16>, i32, i32) nounwind readnone
 
 define <8 x i32> @test_ucmp_w_512(<32 x i16> %a0, <32 x i16> %a1) {
 ; CHECK_LABEL: test_ucmp_w_512
 ; CHECK: vpcmpequw %zmm1, %zmm0, %k0 ##
-  %res0 = call i32 @llvm.x86.avx512.mask.ucmp.w.512(<32 x i16> %a0, <32 x i16> %a1, i8 0, i32 -1)
+  %res0 = call i32 @llvm.x86.avx512.mask.ucmp.w.512(<32 x i16> %a0, <32 x i16> %a1, i32 0, i32 -1)
   %vec0 = insertelement <8 x i32> undef, i32 %res0, i32 0
 ; CHECK: vpcmpltuw %zmm1, %zmm0, %k0 ##
-  %res1 = call i32 @llvm.x86.avx512.mask.ucmp.w.512(<32 x i16> %a0, <32 x i16> %a1, i8 1, i32 -1)
+  %res1 = call i32 @llvm.x86.avx512.mask.ucmp.w.512(<32 x i16> %a0, <32 x i16> %a1, i32 1, i32 -1)
   %vec1 = insertelement <8 x i32> %vec0, i32 %res1, i32 1
 ; CHECK: vpcmpleuw %zmm1, %zmm0, %k0 ##
-  %res2 = call i32 @llvm.x86.avx512.mask.ucmp.w.512(<32 x i16> %a0, <32 x i16> %a1, i8 2, i32 -1)
+  %res2 = call i32 @llvm.x86.avx512.mask.ucmp.w.512(<32 x i16> %a0, <32 x i16> %a1, i32 2, i32 -1)
   %vec2 = insertelement <8 x i32> %vec1, i32 %res2, i32 2
 ; CHECK: vpcmpunorduw %zmm1, %zmm0, %k0 ##
-  %res3 = call i32 @llvm.x86.avx512.mask.ucmp.w.512(<32 x i16> %a0, <32 x i16> %a1, i8 3, i32 -1)
+  %res3 = call i32 @llvm.x86.avx512.mask.ucmp.w.512(<32 x i16> %a0, <32 x i16> %a1, i32 3, i32 -1)
   %vec3 = insertelement <8 x i32> %vec2, i32 %res3, i32 3
 ; CHECK: vpcmpnequw %zmm1, %zmm0, %k0 ##
-  %res4 = call i32 @llvm.x86.avx512.mask.ucmp.w.512(<32 x i16> %a0, <32 x i16> %a1, i8 4, i32 -1)
+  %res4 = call i32 @llvm.x86.avx512.mask.ucmp.w.512(<32 x i16> %a0, <32 x i16> %a1, i32 4, i32 -1)
   %vec4 = insertelement <8 x i32> %vec3, i32 %res4, i32 4
 ; CHECK: vpcmpnltuw %zmm1, %zmm0, %k0 ##
-  %res5 = call i32 @llvm.x86.avx512.mask.ucmp.w.512(<32 x i16> %a0, <32 x i16> %a1, i8 5, i32 -1)
+  %res5 = call i32 @llvm.x86.avx512.mask.ucmp.w.512(<32 x i16> %a0, <32 x i16> %a1, i32 5, i32 -1)
   %vec5 = insertelement <8 x i32> %vec4, i32 %res5, i32 5
 ; CHECK: vpcmpnleuw %zmm1, %zmm0, %k0 ##
-  %res6 = call i32 @llvm.x86.avx512.mask.ucmp.w.512(<32 x i16> %a0, <32 x i16> %a1, i8 6, i32 -1)
+  %res6 = call i32 @llvm.x86.avx512.mask.ucmp.w.512(<32 x i16> %a0, <32 x i16> %a1, i32 6, i32 -1)
   %vec6 = insertelement <8 x i32> %vec5, i32 %res6, i32 6
 ; CHECK: vpcmporduw %zmm1, %zmm0, %k0 ##
-  %res7 = call i32 @llvm.x86.avx512.mask.ucmp.w.512(<32 x i16> %a0, <32 x i16> %a1, i8 7, i32 -1)
+  %res7 = call i32 @llvm.x86.avx512.mask.ucmp.w.512(<32 x i16> %a0, <32 x i16> %a1, i32 7, i32 -1)
   %vec7 = insertelement <8 x i32> %vec6, i32 %res7, i32 7
   ret <8 x i32> %vec7
 }
@@ -276,49 +276,43 @@ define <8 x i32> @test_ucmp_w_512(<32 x i16> %a0, <32 x i16> %a1) {
 define <8 x i32> @test_mask_ucmp_w_512(<32 x i16> %a0, <32 x i16> %a1, i32 %mask) {
 ; CHECK_LABEL: test_mask_ucmp_w_512
 ; CHECK: vpcmpequw %zmm1, %zmm0, %k0 {%k1} ##
-  %res0 = call i32 @llvm.x86.avx512.mask.ucmp.w.512(<32 x i16> %a0, <32 x i16> %a1, i8 0, i32 %mask)
+  %res0 = call i32 @llvm.x86.avx512.mask.ucmp.w.512(<32 x i16> %a0, <32 x i16> %a1, i32 0, i32 %mask)
   %vec0 = insertelement <8 x i32> undef, i32 %res0, i32 0
 ; CHECK: vpcmpltuw %zmm1, %zmm0, %k0 {%k1} ##
-  %res1 = call i32 @llvm.x86.avx512.mask.ucmp.w.512(<32 x i16> %a0, <32 x i16> %a1, i8 1, i32 %mask)
+  %res1 = call i32 @llvm.x86.avx512.mask.ucmp.w.512(<32 x i16> %a0, <32 x i16> %a1, i32 1, i32 %mask)
   %vec1 = insertelement <8 x i32> %vec0, i32 %res1, i32 1
 ; CHECK: vpcmpleuw %zmm1, %zmm0, %k0 {%k1} ##
-  %res2 = call i32 @llvm.x86.avx512.mask.ucmp.w.512(<32 x i16> %a0, <32 x i16> %a1, i8 2, i32 %mask)
+  %res2 = call i32 @llvm.x86.avx512.mask.ucmp.w.512(<32 x i16> %a0, <32 x i16> %a1, i32 2, i32 %mask)
   %vec2 = insertelement <8 x i32> %vec1, i32 %res2, i32 2
 ; CHECK: vpcmpunorduw %zmm1, %zmm0, %k0 {%k1} ##
-  %res3 = call i32 @llvm.x86.avx512.mask.ucmp.w.512(<32 x i16> %a0, <32 x i16> %a1, i8 3, i32 %mask)
+  %res3 = call i32 @llvm.x86.avx512.mask.ucmp.w.512(<32 x i16> %a0, <32 x i16> %a1, i32 3, i32 %mask)
   %vec3 = insertelement <8 x i32> %vec2, i32 %res3, i32 3
 ; CHECK: vpcmpnequw %zmm1, %zmm0, %k0 {%k1} ##
-  %res4 = call i32 @llvm.x86.avx512.mask.ucmp.w.512(<32 x i16> %a0, <32 x i16> %a1, i8 4, i32 %mask)
+  %res4 = call i32 @llvm.x86.avx512.mask.ucmp.w.512(<32 x i16> %a0, <32 x i16> %a1, i32 4, i32 %mask)
   %vec4 = insertelement <8 x i32> %vec3, i32 %res4, i32 4
 ; CHECK: vpcmpnltuw %zmm1, %zmm0, %k0 {%k1} ##
-  %res5 = call i32 @llvm.x86.avx512.mask.ucmp.w.512(<32 x i16> %a0, <32 x i16> %a1, i8 5, i32 %mask)
+  %res5 = call i32 @llvm.x86.avx512.mask.ucmp.w.512(<32 x i16> %a0, <32 x i16> %a1, i32 5, i32 %mask)
   %vec5 = insertelement <8 x i32> %vec4, i32 %res5, i32 5
 ; CHECK: vpcmpnleuw %zmm1, %zmm0, %k0 {%k1} ##
-  %res6 = call i32 @llvm.x86.avx512.mask.ucmp.w.512(<32 x i16> %a0, <32 x i16> %a1, i8 6, i32 %mask)
+  %res6 = call i32 @llvm.x86.avx512.mask.ucmp.w.512(<32 x i16> %a0, <32 x i16> %a1, i32 6, i32 %mask)
   %vec6 = insertelement <8 x i32> %vec5, i32 %res6, i32 6
 ; CHECK: vpcmporduw %zmm1, %zmm0, %k0 {%k1} ##
-  %res7 = call i32 @llvm.x86.avx512.mask.ucmp.w.512(<32 x i16> %a0, <32 x i16> %a1, i8 7, i32 %mask)
+  %res7 = call i32 @llvm.x86.avx512.mask.ucmp.w.512(<32 x i16> %a0, <32 x i16> %a1, i32 7, i32 %mask)
   %vec7 = insertelement <8 x i32> %vec6, i32 %res7, i32 7
   ret <8 x i32> %vec7
 }
 
-declare i32 @llvm.x86.avx512.mask.ucmp.w.512(<32 x i16>, <32 x i16>, i8, i32) nounwind readnone
+declare i32 @llvm.x86.avx512.mask.ucmp.w.512(<32 x i16>, <32 x i16>, i32, i32) nounwind readnone
 
-; CHECK-LABEL: test_x86_mask_blend_b_256
-; CHECK: vpblendmb
-define <32 x i8> @test_x86_mask_blend_b_256(i32 %a0, <32 x i8> %a1, <32 x i8> %a2) {
-  %res = call <32 x i8> @llvm.x86.avx512.mask.blend.b.256(<32 x i8> %a1, <32 x i8> %a2, i32 %a0) ; <<32 x i8>> [#uses=1]
-  ret <32 x i8> %res
-}
-declare <32 x i8> @llvm.x86.avx512.mask.blend.b.256(<32 x i8>, <32 x i8>, i32) nounwind readonly
+declare <64 x i8> @llvm.x86.avx512.mask.blend.b.512(<64 x i8>, <64 x i8>, i64) nounwind readonly
 
-; CHECK-LABEL: test_x86_mask_blend_w_256
-define <16 x i16> @test_x86_mask_blend_w_256(i16 %mask, <16 x i16> %a1, <16 x i16> %a2) {
+; CHECK-LABEL: test_x86_mask_blend_w_512
+define <32 x i16> @test_x86_mask_blend_w_512(i32 %mask, <32 x i16> %a1, <32 x i16> %a2) {
   ; CHECK: vpblendmw
-  %res = call <16 x i16> @llvm.x86.avx512.mask.blend.w.256(<16 x i16> %a1, <16 x i16> %a2, i16 %mask) ; <<16 x i16>> [#uses=1]
-  ret <16 x i16> %res
+  %res = call <32 x i16> @llvm.x86.avx512.mask.blend.w.512(<32 x i16> %a1, <32 x i16> %a2, i32 %mask) ; <<32 x i16>> [#uses=1]
+  ret <32 x i16> %res
 }
-declare <16 x i16> @llvm.x86.avx512.mask.blend.w.256(<16 x i16>, <16 x i16>, i16) nounwind readonly
+declare <32 x i16> @llvm.x86.avx512.mask.blend.w.512(<32 x i16>, <32 x i16>, i32) nounwind readonly
 
 ; CHECK-LABEL: test_x86_mask_blend_b_512
 ; CHECK: vpblendmb
@@ -326,28 +320,864 @@ define <64 x i8> @test_x86_mask_blend_b_512(i64 %a0, <64 x i8> %a1, <64 x i8> %a
   %res = call <64 x i8> @llvm.x86.avx512.mask.blend.b.512(<64 x i8> %a1, <64 x i8> %a2, i64 %a0) ; <<64 x i8>> [#uses=1]
   ret <64 x i8> %res
 }
-declare <64 x i8> @llvm.x86.avx512.mask.blend.b.512(<64 x i8>, <64 x i8>, i64) nounwind readonly
 
-; CHECK-LABEL: test_x86_mask_blend_w_512
-define <32 x i16> @test_x86_mask_blend_w_512(i32 %mask, <32 x i16> %a1, <32 x i16> %a2) {
-  ; CHECK: vpblendmw
-  %res = call <32 x i16> @llvm.x86.avx512.mask.blend.w.512(<32 x i16> %a1, <32 x i16> %a2, i32 %mask) ; <<32 x i16>> [#uses=1]
+define <32 x i16> @test_mask_packs_epi32_rr_512(<16 x i32> %a, <16 x i32> %b) {
+  ;CHECK-LABEL: test_mask_packs_epi32_rr_512
+  ;CHECK: vpackssdw       %zmm1, %zmm0, %zmm0  ## encoding: [0x62,0xf1,0x7d,0x48,0x6b,0xc1]
+  %res = call <32 x i16> @llvm.x86.avx512.mask.packssdw.512(<16 x i32> %a, <16 x i32> %b, <32 x i16> zeroinitializer, i32 -1)
   ret <32 x i16> %res
 }
-declare <32 x i16> @llvm.x86.avx512.mask.blend.w.512(<32 x i16>, <32 x i16>, i32) nounwind readonly
 
-; CHECK-LABEL: test_x86_mask_blend_b_128
-; CHECK: vpblendmb
-define <16 x i8> @test_x86_mask_blend_b_128(i16 %a0, <16 x i8> %a1, <16 x i8> %a2) {
-  %res = call <16 x i8> @llvm.x86.avx512.mask.blend.b.128(<16 x i8> %a1, <16 x i8> %a2, i16 %a0) ; <<16 x i8>> [#uses=1]
-  ret <16 x i8> %res
+define <32 x i16> @test_mask_packs_epi32_rrk_512(<16 x i32> %a, <16 x i32> %b, <32 x i16> %passThru, i32 %mask) {
+  ;CHECK-LABEL: test_mask_packs_epi32_rrk_512
+  ;CHECK: vpackssdw       %zmm1, %zmm0, %zmm2 {%k1} ## encoding: [0x62,0xf1,0x7d,0x49,0x6b,0xd1]
+  %res = call <32 x i16> @llvm.x86.avx512.mask.packssdw.512(<16 x i32> %a, <16 x i32> %b, <32 x i16> %passThru, i32 %mask)
+  ret <32 x i16> %res
 }
-declare <16 x i8> @llvm.x86.avx512.mask.blend.b.128(<16 x i8>, <16 x i8>, i16) nounwind readonly
 
-; CHECK-LABEL: test_x86_mask_blend_w_128
-define <8 x i16> @test_x86_mask_blend_w_128(i8 %mask, <8 x i16> %a1, <8 x i16> %a2) {
-  ; CHECK: vpblendmw
-  %res = call <8 x i16> @llvm.x86.avx512.mask.blend.w.128(<8 x i16> %a1, <8 x i16> %a2, i8 %mask) ; <<8 x i16>> [#uses=1]
-  ret <8 x i16> %res
+define <32 x i16> @test_mask_packs_epi32_rrkz_512(<16 x i32> %a, <16 x i32> %b, i32 %mask) {
+  ;CHECK-LABEL: test_mask_packs_epi32_rrkz_512
+  ;CHECK: vpackssdw       %zmm1, %zmm0, %zmm0 {%k1} {z} ## encoding: [0x62,0xf1,0x7d,0xc9,0x6b,0xc1]
+  %res = call <32 x i16> @llvm.x86.avx512.mask.packssdw.512(<16 x i32> %a, <16 x i32> %b, <32 x i16> zeroinitializer, i32 %mask)
+  ret <32 x i16> %res
+}
+
+define <32 x i16> @test_mask_packs_epi32_rm_512(<16 x i32> %a, <16 x i32>* %ptr_b) {
+  ;CHECK-LABEL: test_mask_packs_epi32_rm_512
+  ;CHECK: vpackssdw       (%rdi), %zmm0, %zmm0  ## encoding: [0x62,0xf1,0x7d,0x48,0x6b,0x07]
+  %b = load <16 x i32>, <16 x i32>* %ptr_b
+  %res = call <32 x i16> @llvm.x86.avx512.mask.packssdw.512(<16 x i32> %a, <16 x i32> %b, <32 x i16> zeroinitializer, i32 -1)
+  ret <32 x i16> %res
+}
+
+define <32 x i16> @test_mask_packs_epi32_rmk_512(<16 x i32> %a, <16 x i32>* %ptr_b, <32 x i16> %passThru, i32 %mask) {
+  ;CHECK-LABEL: test_mask_packs_epi32_rmk_512
+  ;CHECK: vpackssdw       (%rdi), %zmm0, %zmm1 {%k1} ## encoding: [0x62,0xf1,0x7d,0x49,0x6b,0x0f]
+  %b = load <16 x i32>, <16 x i32>* %ptr_b
+  %res = call <32 x i16> @llvm.x86.avx512.mask.packssdw.512(<16 x i32> %a, <16 x i32> %b, <32 x i16> %passThru, i32 %mask)
+  ret <32 x i16> %res
+}
+
+define <32 x i16> @test_mask_packs_epi32_rmkz_512(<16 x i32> %a, <16 x i32>* %ptr_b, i32 %mask) {
+  ;CHECK-LABEL: test_mask_packs_epi32_rmkz_512
+  ;CHECK: vpackssdw       (%rdi), %zmm0, %zmm0 {%k1} {z} ## encoding: [0x62,0xf1,0x7d,0xc9,0x6b,0x07]
+  %b = load <16 x i32>, <16 x i32>* %ptr_b
+  %res = call <32 x i16> @llvm.x86.avx512.mask.packssdw.512(<16 x i32> %a, <16 x i32> %b, <32 x i16> zeroinitializer, i32 %mask)
+  ret <32 x i16> %res
+}
+
+define <32 x i16> @test_mask_packs_epi32_rmb_512(<16 x i32> %a, i32* %ptr_b) {
+  ;CHECK-LABEL: test_mask_packs_epi32_rmb_512
+  ;CHECK: vpackssdw       (%rdi){1to16}, %zmm0, %zmm0  ## encoding: [0x62,0xf1,0x7d,0x58,0x6b,0x07]
+  %q = load i32, i32* %ptr_b
+  %vecinit.i = insertelement <16 x i32> undef, i32 %q, i32 0
+  %b = shufflevector <16 x i32> %vecinit.i, <16 x i32> undef, <16 x i32> zeroinitializer
+  %res = call <32 x i16> @llvm.x86.avx512.mask.packssdw.512(<16 x i32> %a, <16 x i32> %b, <32 x i16> zeroinitializer, i32 -1)
+  ret <32 x i16> %res
+}
+
+define <32 x i16> @test_mask_packs_epi32_rmbk_512(<16 x i32> %a, i32* %ptr_b, <32 x i16> %passThru, i32 %mask) {
+  ;CHECK-LABEL: test_mask_packs_epi32_rmbk_512
+  ;CHECK: vpackssdw       (%rdi){1to16}, %zmm0, %zmm1 {%k1} ## encoding: [0x62,0xf1,0x7d,0x59,0x6b,0x0f]
+  %q = load i32, i32* %ptr_b
+  %vecinit.i = insertelement <16 x i32> undef, i32 %q, i32 0
+  %b = shufflevector <16 x i32> %vecinit.i, <16 x i32> undef, <16 x i32> zeroinitializer
+  %res = call <32 x i16> @llvm.x86.avx512.mask.packssdw.512(<16 x i32> %a, <16 x i32> %b, <32 x i16> %passThru, i32 %mask)
+  ret <32 x i16> %res
+}
+
+define <32 x i16> @test_mask_packs_epi32_rmbkz_512(<16 x i32> %a, i32* %ptr_b, i32 %mask) {
+  ;CHECK-LABEL: test_mask_packs_epi32_rmbkz_512
+  ;CHECK: vpackssdw       (%rdi){1to16}, %zmm0, %zmm0 {%k1} {z} ## encoding: [0x62,0xf1,0x7d,0xd9,0x6b,0x07]
+  %q = load i32, i32* %ptr_b
+  %vecinit.i = insertelement <16 x i32> undef, i32 %q, i32 0
+  %b = shufflevector <16 x i32> %vecinit.i, <16 x i32> undef, <16 x i32> zeroinitializer
+  %res = call <32 x i16> @llvm.x86.avx512.mask.packssdw.512(<16 x i32> %a, <16 x i32> %b, <32 x i16> zeroinitializer, i32 %mask)
+  ret <32 x i16> %res
+}
+
+declare <32 x i16> @llvm.x86.avx512.mask.packssdw.512(<16 x i32>, <16 x i32>, <32 x i16>, i32)
+
+define <64 x i8> @test_mask_packs_epi16_rr_512(<32 x i16> %a, <32 x i16> %b) {
+  ;CHECK-LABEL: test_mask_packs_epi16_rr_512
+  ;CHECK: vpacksswb       %zmm1, %zmm0, %zmm0  ## encoding: [0x62,0xf1,0xfd,0x48,0x63,0xc1]
+  %res = call <64 x i8> @llvm.x86.avx512.mask.packsswb.512(<32 x i16> %a, <32 x i16> %b, <64 x i8> zeroinitializer, i64 -1)
+  ret <64 x i8> %res
+}
+
+define <64 x i8> @test_mask_packs_epi16_rrk_512(<32 x i16> %a, <32 x i16> %b, <64 x i8> %passThru, i64 %mask) {
+  ;CHECK-LABEL: test_mask_packs_epi16_rrk_512
+  ;CHECK: vpacksswb       %zmm1, %zmm0, %zmm2 {%k1} ## encoding: [0x62,0xf1,0xfd,0x49,0x63,0xd1]
+  %res = call <64 x i8> @llvm.x86.avx512.mask.packsswb.512(<32 x i16> %a, <32 x i16> %b, <64 x i8> %passThru, i64 %mask)
+  ret <64 x i8> %res
+}
+
+define <64 x i8> @test_mask_packs_epi16_rrkz_512(<32 x i16> %a, <32 x i16> %b, i64 %mask) {
+  ;CHECK-LABEL: test_mask_packs_epi16_rrkz_512
+  ;CHECK: vpacksswb       %zmm1, %zmm0, %zmm0 {%k1} {z} ## encoding: [0x62,0xf1,0xfd,0xc9,0x63,0xc1]
+  %res = call <64 x i8> @llvm.x86.avx512.mask.packsswb.512(<32 x i16> %a, <32 x i16> %b, <64 x i8> zeroinitializer, i64 %mask)
+  ret <64 x i8> %res
+}
+
+define <64 x i8> @test_mask_packs_epi16_rm_512(<32 x i16> %a, <32 x i16>* %ptr_b) {
+  ;CHECK-LABEL: test_mask_packs_epi16_rm_512
+  ;CHECK: vpacksswb       (%rdi), %zmm0, %zmm0  ## encoding: [0x62,0xf1,0xfd,0x48,0x63,0x07]
+  %b = load <32 x i16>, <32 x i16>* %ptr_b
+  %res = call <64 x i8> @llvm.x86.avx512.mask.packsswb.512(<32 x i16> %a, <32 x i16> %b, <64 x i8> zeroinitializer, i64 -1)
+  ret <64 x i8> %res
+}
+
+define <64 x i8> @test_mask_packs_epi16_rmk_512(<32 x i16> %a, <32 x i16>* %ptr_b, <64 x i8> %passThru, i64 %mask) {
+  ;CHECK-LABEL: test_mask_packs_epi16_rmk_512
+  ;CHECK: vpacksswb       (%rdi), %zmm0, %zmm1 {%k1} ## encoding: [0x62,0xf1,0xfd,0x49,0x63,0x0f]
+  %b = load <32 x i16>, <32 x i16>* %ptr_b
+  %res = call <64 x i8> @llvm.x86.avx512.mask.packsswb.512(<32 x i16> %a, <32 x i16> %b, <64 x i8> %passThru, i64 %mask)
+  ret <64 x i8> %res
+}
+
+define <64 x i8> @test_mask_packs_epi16_rmkz_512(<32 x i16> %a, <32 x i16>* %ptr_b, i64 %mask) {
+  ;CHECK-LABEL: test_mask_packs_epi16_rmkz_512
+  ;CHECK: vpacksswb       (%rdi), %zmm0, %zmm0 {%k1} {z} ## encoding: [0x62,0xf1,0xfd,0xc9,0x63,0x07]
+  %b = load <32 x i16>, <32 x i16>* %ptr_b
+  %res = call <64 x i8> @llvm.x86.avx512.mask.packsswb.512(<32 x i16> %a, <32 x i16> %b, <64 x i8> zeroinitializer, i64 %mask)
+  ret <64 x i8> %res
+}
+
+declare <64 x i8> @llvm.x86.avx512.mask.packsswb.512(<32 x i16>, <32 x i16>, <64 x i8>, i64)
+
+
+define <32 x i16> @test_mask_packus_epi32_rr_512(<16 x i32> %a, <16 x i32> %b) {
+  ;CHECK-LABEL: test_mask_packus_epi32_rr_512
+  ;CHECK: vpackusdw       %zmm1, %zmm0, %zmm0  
+  %res = call <32 x i16> @llvm.x86.avx512.mask.packusdw.512(<16 x i32> %a, <16 x i32> %b, <32 x i16> zeroinitializer, i32 -1)
+  ret <32 x i16> %res
+}
+
+define <32 x i16> @test_mask_packus_epi32_rrk_512(<16 x i32> %a, <16 x i32> %b, <32 x i16> %passThru, i32 %mask) {
+  ;CHECK-LABEL: test_mask_packus_epi32_rrk_512
+  ;CHECK: vpackusdw       %zmm1, %zmm0, %zmm2 {%k1} 
+  %res = call <32 x i16> @llvm.x86.avx512.mask.packusdw.512(<16 x i32> %a, <16 x i32> %b, <32 x i16> %passThru, i32 %mask)
+  ret <32 x i16> %res
+}
+
+define <32 x i16> @test_mask_packus_epi32_rrkz_512(<16 x i32> %a, <16 x i32> %b, i32 %mask) {
+  ;CHECK-LABEL: test_mask_packus_epi32_rrkz_512
+  ;CHECK: vpackusdw       %zmm1, %zmm0, %zmm0 {%k1} {z} 
+  %res = call <32 x i16> @llvm.x86.avx512.mask.packusdw.512(<16 x i32> %a, <16 x i32> %b, <32 x i16> zeroinitializer, i32 %mask)
+  ret <32 x i16> %res
+}
+
+define <32 x i16> @test_mask_packus_epi32_rm_512(<16 x i32> %a, <16 x i32>* %ptr_b) {
+  ;CHECK-LABEL: test_mask_packus_epi32_rm_512
+  ;CHECK: vpackusdw       (%rdi), %zmm0, %zmm0  
+  %b = load <16 x i32>, <16 x i32>* %ptr_b
+  %res = call <32 x i16> @llvm.x86.avx512.mask.packusdw.512(<16 x i32> %a, <16 x i32> %b, <32 x i16> zeroinitializer, i32 -1)
+  ret <32 x i16> %res
+}
+
+define <32 x i16> @test_mask_packus_epi32_rmk_512(<16 x i32> %a, <16 x i32>* %ptr_b, <32 x i16> %passThru, i32 %mask) {
+  ;CHECK-LABEL: test_mask_packus_epi32_rmk_512
+  ;CHECK: vpackusdw       (%rdi), %zmm0, %zmm1 {%k1} 
+  %b = load <16 x i32>, <16 x i32>* %ptr_b
+  %res = call <32 x i16> @llvm.x86.avx512.mask.packusdw.512(<16 x i32> %a, <16 x i32> %b, <32 x i16> %passThru, i32 %mask)
+  ret <32 x i16> %res
+}
+
+define <32 x i16> @test_mask_packus_epi32_rmkz_512(<16 x i32> %a, <16 x i32>* %ptr_b, i32 %mask) {
+  ;CHECK-LABEL: test_mask_packus_epi32_rmkz_512
+  ;CHECK: vpackusdw       (%rdi), %zmm0, %zmm0 {%k1} {z} 
+  %b = load <16 x i32>, <16 x i32>* %ptr_b
+  %res = call <32 x i16> @llvm.x86.avx512.mask.packusdw.512(<16 x i32> %a, <16 x i32> %b, <32 x i16> zeroinitializer, i32 %mask)
+  ret <32 x i16> %res
+}
+
+define <32 x i16> @test_mask_packus_epi32_rmb_512(<16 x i32> %a, i32* %ptr_b) {
+  ;CHECK-LABEL: test_mask_packus_epi32_rmb_512
+  ;CHECK: vpackusdw       (%rdi){1to16}, %zmm0, %zmm0  
+  %q = load i32, i32* %ptr_b
+  %vecinit.i = insertelement <16 x i32> undef, i32 %q, i32 0
+  %b = shufflevector <16 x i32> %vecinit.i, <16 x i32> undef, <16 x i32> zeroinitializer
+  %res = call <32 x i16> @llvm.x86.avx512.mask.packusdw.512(<16 x i32> %a, <16 x i32> %b, <32 x i16> zeroinitializer, i32 -1)
+  ret <32 x i16> %res
+}
+
+define <32 x i16> @test_mask_packus_epi32_rmbk_512(<16 x i32> %a, i32* %ptr_b, <32 x i16> %passThru, i32 %mask) {
+  ;CHECK-LABEL: test_mask_packus_epi32_rmbk_512
+  ;CHECK: vpackusdw       (%rdi){1to16}, %zmm0, %zmm1 {%k1} 
+  %q = load i32, i32* %ptr_b
+  %vecinit.i = insertelement <16 x i32> undef, i32 %q, i32 0
+  %b = shufflevector <16 x i32> %vecinit.i, <16 x i32> undef, <16 x i32> zeroinitializer
+  %res = call <32 x i16> @llvm.x86.avx512.mask.packusdw.512(<16 x i32> %a, <16 x i32> %b, <32 x i16> %passThru, i32 %mask)
+  ret <32 x i16> %res
+}
+
+define <32 x i16> @test_mask_packus_epi32_rmbkz_512(<16 x i32> %a, i32* %ptr_b, i32 %mask) {
+  ;CHECK-LABEL: test_mask_packus_epi32_rmbkz_512
+  ;CHECK: vpackusdw       (%rdi){1to16}, %zmm0, %zmm0 {%k1} {z} 
+  %q = load i32, i32* %ptr_b
+  %vecinit.i = insertelement <16 x i32> undef, i32 %q, i32 0
+  %b = shufflevector <16 x i32> %vecinit.i, <16 x i32> undef, <16 x i32> zeroinitializer
+  %res = call <32 x i16> @llvm.x86.avx512.mask.packusdw.512(<16 x i32> %a, <16 x i32> %b, <32 x i16> zeroinitializer, i32 %mask)
+  ret <32 x i16> %res
+}
+
+declare <32 x i16> @llvm.x86.avx512.mask.packusdw.512(<16 x i32>, <16 x i32>, <32 x i16>, i32)
+
+define <64 x i8> @test_mask_packus_epi16_rr_512(<32 x i16> %a, <32 x i16> %b) {
+  ;CHECK-LABEL: test_mask_packus_epi16_rr_512
+  ;CHECK: vpackuswb       %zmm1, %zmm0, %zmm0  
+  %res = call <64 x i8> @llvm.x86.avx512.mask.packuswb.512(<32 x i16> %a, <32 x i16> %b, <64 x i8> zeroinitializer, i64 -1)
+  ret <64 x i8> %res
+}
+
+define <64 x i8> @test_mask_packus_epi16_rrk_512(<32 x i16> %a, <32 x i16> %b, <64 x i8> %passThru, i64 %mask) {
+  ;CHECK-LABEL: test_mask_packus_epi16_rrk_512
+  ;CHECK: vpackuswb       %zmm1, %zmm0, %zmm2 {%k1} 
+  %res = call <64 x i8> @llvm.x86.avx512.mask.packuswb.512(<32 x i16> %a, <32 x i16> %b, <64 x i8> %passThru, i64 %mask)
+  ret <64 x i8> %res
+}
+
+define <64 x i8> @test_mask_packus_epi16_rrkz_512(<32 x i16> %a, <32 x i16> %b, i64 %mask) {
+  ;CHECK-LABEL: test_mask_packus_epi16_rrkz_512
+  ;CHECK: vpackuswb       %zmm1, %zmm0, %zmm0 {%k1} {z} 
+  %res = call <64 x i8> @llvm.x86.avx512.mask.packuswb.512(<32 x i16> %a, <32 x i16> %b, <64 x i8> zeroinitializer, i64 %mask)
+  ret <64 x i8> %res
+}
+
+define <64 x i8> @test_mask_packus_epi16_rm_512(<32 x i16> %a, <32 x i16>* %ptr_b) {
+  ;CHECK-LABEL: test_mask_packus_epi16_rm_512
+  ;CHECK: vpackuswb       (%rdi), %zmm0, %zmm0  
+  %b = load <32 x i16>, <32 x i16>* %ptr_b
+  %res = call <64 x i8> @llvm.x86.avx512.mask.packuswb.512(<32 x i16> %a, <32 x i16> %b, <64 x i8> zeroinitializer, i64 -1)
+  ret <64 x i8> %res
+}
+
+define <64 x i8> @test_mask_packus_epi16_rmk_512(<32 x i16> %a, <32 x i16>* %ptr_b, <64 x i8> %passThru, i64 %mask) {
+  ;CHECK-LABEL: test_mask_packus_epi16_rmk_512
+  ;CHECK: vpackuswb       (%rdi), %zmm0, %zmm1 {%k1} 
+  %b = load <32 x i16>, <32 x i16>* %ptr_b
+  %res = call <64 x i8> @llvm.x86.avx512.mask.packuswb.512(<32 x i16> %a, <32 x i16> %b, <64 x i8> %passThru, i64 %mask)
+  ret <64 x i8> %res
+}
+
+define <64 x i8> @test_mask_packus_epi16_rmkz_512(<32 x i16> %a, <32 x i16>* %ptr_b, i64 %mask) {
+  ;CHECK-LABEL: test_mask_packus_epi16_rmkz_512
+  ;CHECK: vpackuswb       (%rdi), %zmm0, %zmm0 {%k1} {z} 
+  %b = load <32 x i16>, <32 x i16>* %ptr_b
+  %res = call <64 x i8> @llvm.x86.avx512.mask.packuswb.512(<32 x i16> %a, <32 x i16> %b, <64 x i8> zeroinitializer, i64 %mask)
+  ret <64 x i8> %res
+}
+
+declare <64 x i8> @llvm.x86.avx512.mask.packuswb.512(<32 x i16>, <32 x i16>, <64 x i8>, i64)
+
+define <32 x i16> @test_mask_adds_epi16_rr_512(<32 x i16> %a, <32 x i16> %b) {
+  ;CHECK-LABEL: test_mask_adds_epi16_rr_512
+  ;CHECK: vpaddsw %zmm1, %zmm0, %zmm0     
+  %res = call <32 x i16> @llvm.x86.avx512.mask.padds.w.512(<32 x i16> %a, <32 x i16> %b, <32 x i16> zeroinitializer, i32 -1)
+  ret <32 x i16> %res
+}
+
+define <32 x i16> @test_mask_adds_epi16_rrk_512(<32 x i16> %a, <32 x i16> %b, <32 x i16> %passThru, i32 %mask) {
+  ;CHECK-LABEL: test_mask_adds_epi16_rrk_512
+  ;CHECK: vpaddsw %zmm1, %zmm0, %zmm2 {%k1} 
+  %res = call <32 x i16> @llvm.x86.avx512.mask.padds.w.512(<32 x i16> %a, <32 x i16> %b, <32 x i16> %passThru, i32 %mask)
+  ret <32 x i16> %res
+}
+
+define <32 x i16> @test_mask_adds_epi16_rrkz_512(<32 x i16> %a, <32 x i16> %b, i32 %mask) {
+  ;CHECK-LABEL: test_mask_adds_epi16_rrkz_512
+  ;CHECK: vpaddsw %zmm1, %zmm0, %zmm0 {%k1} {z} 
+  %res = call <32 x i16> @llvm.x86.avx512.mask.padds.w.512(<32 x i16> %a, <32 x i16> %b, <32 x i16> zeroinitializer, i32 %mask)
+  ret <32 x i16> %res
+}
+
+define <32 x i16> @test_mask_adds_epi16_rm_512(<32 x i16> %a, <32 x i16>* %ptr_b) {
+  ;CHECK-LABEL: test_mask_adds_epi16_rm_512
+  ;CHECK: vpaddsw (%rdi), %zmm0, %zmm0    
+  %b = load <32 x i16>, <32 x i16>* %ptr_b
+  %res = call <32 x i16> @llvm.x86.avx512.mask.padds.w.512(<32 x i16> %a, <32 x i16> %b, <32 x i16> zeroinitializer, i32 -1)
+  ret <32 x i16> %res
+}
+
+define <32 x i16> @test_mask_adds_epi16_rmk_512(<32 x i16> %a, <32 x i16>* %ptr_b, <32 x i16> %passThru, i32 %mask) {
+  ;CHECK-LABEL: test_mask_adds_epi16_rmk_512
+  ;CHECK: vpaddsw (%rdi), %zmm0, %zmm1 {%k1} 
+  %b = load <32 x i16>, <32 x i16>* %ptr_b
+  %res = call <32 x i16> @llvm.x86.avx512.mask.padds.w.512(<32 x i16> %a, <32 x i16> %b, <32 x i16> %passThru, i32 %mask)
+  ret <32 x i16> %res
+}
+
+define <32 x i16> @test_mask_adds_epi16_rmkz_512(<32 x i16> %a, <32 x i16>* %ptr_b, i32 %mask) {
+  ;CHECK-LABEL: test_mask_adds_epi16_rmkz_512
+  ;CHECK: vpaddsw (%rdi), %zmm0, %zmm0 {%k1} {z} 
+  %b = load <32 x i16>, <32 x i16>* %ptr_b
+  %res = call <32 x i16> @llvm.x86.avx512.mask.padds.w.512(<32 x i16> %a, <32 x i16> %b, <32 x i16> zeroinitializer, i32 %mask)
+  ret <32 x i16> %res
+}
+
+declare <32 x i16> @llvm.x86.avx512.mask.padds.w.512(<32 x i16>, <32 x i16>, <32 x i16>, i32)
+
+define <32 x i16> @test_mask_subs_epi16_rr_512(<32 x i16> %a, <32 x i16> %b) {
+  ;CHECK-LABEL: test_mask_subs_epi16_rr_512
+  ;CHECK: vpsubsw %zmm1, %zmm0, %zmm0     
+  %res = call <32 x i16> @llvm.x86.avx512.mask.psubs.w.512(<32 x i16> %a, <32 x i16> %b, <32 x i16> zeroinitializer, i32 -1)
+  ret <32 x i16> %res
+}
+
+define <32 x i16> @test_mask_subs_epi16_rrk_512(<32 x i16> %a, <32 x i16> %b, <32 x i16> %passThru, i32 %mask) {
+  ;CHECK-LABEL: test_mask_subs_epi16_rrk_512
+  ;CHECK: vpsubsw %zmm1, %zmm0, %zmm2 {%k1} 
+  %res = call <32 x i16> @llvm.x86.avx512.mask.psubs.w.512(<32 x i16> %a, <32 x i16> %b, <32 x i16> %passThru, i32 %mask)
+  ret <32 x i16> %res
+}
+
+define <32 x i16> @test_mask_subs_epi16_rrkz_512(<32 x i16> %a, <32 x i16> %b, i32 %mask) {
+  ;CHECK-LABEL: test_mask_subs_epi16_rrkz_512
+  ;CHECK: vpsubsw %zmm1, %zmm0, %zmm0 {%k1} {z} 
+  %res = call <32 x i16> @llvm.x86.avx512.mask.psubs.w.512(<32 x i16> %a, <32 x i16> %b, <32 x i16> zeroinitializer, i32 %mask)
+  ret <32 x i16> %res
+}
+
+define <32 x i16> @test_mask_subs_epi16_rm_512(<32 x i16> %a, <32 x i16>* %ptr_b) {
+  ;CHECK-LABEL: test_mask_subs_epi16_rm_512
+  ;CHECK: vpsubsw (%rdi), %zmm0, %zmm0    
+  %b = load <32 x i16>, <32 x i16>* %ptr_b
+  %res = call <32 x i16> @llvm.x86.avx512.mask.psubs.w.512(<32 x i16> %a, <32 x i16> %b, <32 x i16> zeroinitializer, i32 -1)
+  ret <32 x i16> %res
+}
+
+define <32 x i16> @test_mask_subs_epi16_rmk_512(<32 x i16> %a, <32 x i16>* %ptr_b, <32 x i16> %passThru, i32 %mask) {
+  ;CHECK-LABEL: test_mask_subs_epi16_rmk_512
+  ;CHECK: vpsubsw (%rdi), %zmm0, %zmm1 {%k1} 
+  %b = load <32 x i16>, <32 x i16>* %ptr_b
+  %res = call <32 x i16> @llvm.x86.avx512.mask.psubs.w.512(<32 x i16> %a, <32 x i16> %b, <32 x i16> %passThru, i32 %mask)
+  ret <32 x i16> %res
+}
+
+define <32 x i16> @test_mask_subs_epi16_rmkz_512(<32 x i16> %a, <32 x i16>* %ptr_b, i32 %mask) {
+  ;CHECK-LABEL: test_mask_subs_epi16_rmkz_512
+  ;CHECK: vpsubsw (%rdi), %zmm0, %zmm0 {%k1} {z} 
+  %b = load <32 x i16>, <32 x i16>* %ptr_b
+  %res = call <32 x i16> @llvm.x86.avx512.mask.psubs.w.512(<32 x i16> %a, <32 x i16> %b, <32 x i16> zeroinitializer, i32 %mask)
+  ret <32 x i16> %res
+}
+
+declare <32 x i16> @llvm.x86.avx512.mask.psubs.w.512(<32 x i16>, <32 x i16>, <32 x i16>, i32)
+
+define <32 x i16> @test_mask_adds_epu16_rr_512(<32 x i16> %a, <32 x i16> %b) {
+  ;CHECK-LABEL: test_mask_adds_epu16_rr_512
+  ;CHECK: vpaddusw %zmm1, %zmm0, %zmm0     
+  %res = call <32 x i16> @llvm.x86.avx512.mask.paddus.w.512(<32 x i16> %a, <32 x i16> %b, <32 x i16> zeroinitializer, i32 -1)
+  ret <32 x i16> %res
+}
+
+define <32 x i16> @test_mask_adds_epu16_rrk_512(<32 x i16> %a, <32 x i16> %b, <32 x i16> %passThru, i32 %mask) {
+  ;CHECK-LABEL: test_mask_adds_epu16_rrk_512
+  ;CHECK: vpaddusw %zmm1, %zmm0, %zmm2 {%k1} 
+  %res = call <32 x i16> @llvm.x86.avx512.mask.paddus.w.512(<32 x i16> %a, <32 x i16> %b, <32 x i16> %passThru, i32 %mask)
+  ret <32 x i16> %res
+}
+
+define <32 x i16> @test_mask_adds_epu16_rrkz_512(<32 x i16> %a, <32 x i16> %b, i32 %mask) {
+  ;CHECK-LABEL: test_mask_adds_epu16_rrkz_512
+  ;CHECK: vpaddusw %zmm1, %zmm0, %zmm0 {%k1} {z} 
+  %res = call <32 x i16> @llvm.x86.avx512.mask.paddus.w.512(<32 x i16> %a, <32 x i16> %b, <32 x i16> zeroinitializer, i32 %mask)
+  ret <32 x i16> %res
+}
+
+define <32 x i16> @test_mask_adds_epu16_rm_512(<32 x i16> %a, <32 x i16>* %ptr_b) {
+  ;CHECK-LABEL: test_mask_adds_epu16_rm_512
+  ;CHECK: vpaddusw (%rdi), %zmm0, %zmm0    
+  %b = load <32 x i16>, <32 x i16>* %ptr_b
+  %res = call <32 x i16> @llvm.x86.avx512.mask.paddus.w.512(<32 x i16> %a, <32 x i16> %b, <32 x i16> zeroinitializer, i32 -1)
+  ret <32 x i16> %res
+}
+
+define <32 x i16> @test_mask_adds_epu16_rmk_512(<32 x i16> %a, <32 x i16>* %ptr_b, <32 x i16> %passThru, i32 %mask) {
+  ;CHECK-LABEL: test_mask_adds_epu16_rmk_512
+  ;CHECK: vpaddusw (%rdi), %zmm0, %zmm1 {%k1} 
+  %b = load <32 x i16>, <32 x i16>* %ptr_b
+  %res = call <32 x i16> @llvm.x86.avx512.mask.paddus.w.512(<32 x i16> %a, <32 x i16> %b, <32 x i16> %passThru, i32 %mask)
+  ret <32 x i16> %res
+}
+
+define <32 x i16> @test_mask_adds_epu16_rmkz_512(<32 x i16> %a, <32 x i16>* %ptr_b, i32 %mask) {
+  ;CHECK-LABEL: test_mask_adds_epu16_rmkz_512
+  ;CHECK: vpaddusw (%rdi), %zmm0, %zmm0 {%k1} {z} 
+  %b = load <32 x i16>, <32 x i16>* %ptr_b
+  %res = call <32 x i16> @llvm.x86.avx512.mask.paddus.w.512(<32 x i16> %a, <32 x i16> %b, <32 x i16> zeroinitializer, i32 %mask)
+  ret <32 x i16> %res
+}
+
+declare <32 x i16> @llvm.x86.avx512.mask.paddus.w.512(<32 x i16>, <32 x i16>, <32 x i16>, i32)
+
+define <32 x i16> @test_mask_subs_epu16_rr_512(<32 x i16> %a, <32 x i16> %b) {
+  ;CHECK-LABEL: test_mask_subs_epu16_rr_512
+  ;CHECK: vpsubusw %zmm1, %zmm0, %zmm0     
+  %res = call <32 x i16> @llvm.x86.avx512.mask.psubus.w.512(<32 x i16> %a, <32 x i16> %b, <32 x i16> zeroinitializer, i32 -1)
+  ret <32 x i16> %res
+}
+
+define <32 x i16> @test_mask_subs_epu16_rrk_512(<32 x i16> %a, <32 x i16> %b, <32 x i16> %passThru, i32 %mask) {
+  ;CHECK-LABEL: test_mask_subs_epu16_rrk_512
+  ;CHECK: vpsubusw %zmm1, %zmm0, %zmm2 {%k1} 
+  %res = call <32 x i16> @llvm.x86.avx512.mask.psubus.w.512(<32 x i16> %a, <32 x i16> %b, <32 x i16> %passThru, i32 %mask)
+  ret <32 x i16> %res
+}
+
+define <32 x i16> @test_mask_subs_epu16_rrkz_512(<32 x i16> %a, <32 x i16> %b, i32 %mask) {
+  ;CHECK-LABEL: test_mask_subs_epu16_rrkz_512
+  ;CHECK: vpsubusw %zmm1, %zmm0, %zmm0 {%k1} {z} 
+  %res = call <32 x i16> @llvm.x86.avx512.mask.psubus.w.512(<32 x i16> %a, <32 x i16> %b, <32 x i16> zeroinitializer, i32 %mask)
+  ret <32 x i16> %res
+}
+
+define <32 x i16> @test_mask_subs_epu16_rm_512(<32 x i16> %a, <32 x i16>* %ptr_b) {
+  ;CHECK-LABEL: test_mask_subs_epu16_rm_512
+  ;CHECK: vpsubusw (%rdi), %zmm0, %zmm0    
+  %b = load <32 x i16>, <32 x i16>* %ptr_b
+  %res = call <32 x i16> @llvm.x86.avx512.mask.psubus.w.512(<32 x i16> %a, <32 x i16> %b, <32 x i16> zeroinitializer, i32 -1)
+  ret <32 x i16> %res
+}
+
+define <32 x i16> @test_mask_subs_epu16_rmk_512(<32 x i16> %a, <32 x i16>* %ptr_b, <32 x i16> %passThru, i32 %mask) {
+  ;CHECK-LABEL: test_mask_subs_epu16_rmk_512
+  ;CHECK: vpsubusw (%rdi), %zmm0, %zmm1 {%k1} 
+  %b = load <32 x i16>, <32 x i16>* %ptr_b
+  %res = call <32 x i16> @llvm.x86.avx512.mask.psubus.w.512(<32 x i16> %a, <32 x i16> %b, <32 x i16> %passThru, i32 %mask)
+  ret <32 x i16> %res
+}
+
+define <32 x i16> @test_mask_subs_epu16_rmkz_512(<32 x i16> %a, <32 x i16>* %ptr_b, i32 %mask) {
+  ;CHECK-LABEL: test_mask_subs_epu16_rmkz_512
+  ;CHECK: vpsubusw (%rdi), %zmm0, %zmm0 {%k1} {z} 
+  %b = load <32 x i16>, <32 x i16>* %ptr_b
+  %res = call <32 x i16> @llvm.x86.avx512.mask.psubus.w.512(<32 x i16> %a, <32 x i16> %b, <32 x i16> zeroinitializer, i32 %mask)
+  ret <32 x i16> %res
+}
+
+declare <32 x i16> @llvm.x86.avx512.mask.psubus.w.512(<32 x i16>, <32 x i16>, <32 x i16>, i32)
+
+declare <64 x i8> @llvm.x86.avx512.mask.pmaxs.b.512(<64 x i8>, <64 x i8>, <64 x i8>, i64)
+
+; CHECK-LABEL: @test_int_x86_avx512_mask_pmaxs_b_512
+; CHECK-NOT: call 
+; CHECK: vpmaxsb %zmm
+; CHECK: {%k1} 
+define <64 x i8>@test_int_x86_avx512_mask_pmaxs_b_512(<64 x i8> %x0, <64 x i8> %x1, <64 x i8> %x2, i64 %x3) {
+  %res = call <64 x i8> @llvm.x86.avx512.mask.pmaxs.b.512(<64 x i8> %x0, <64 x i8> %x1, <64 x i8> %x2, i64 %x3)
+  %res1 = call <64 x i8> @llvm.x86.avx512.mask.pmaxs.b.512(<64 x i8> %x0, <64 x i8> %x1, <64 x i8> %x2, i64 -1)
+  %res2 = add <64 x i8> %res, %res1
+  ret <64 x i8> %res2
+}
+
+declare <32 x i16> @llvm.x86.avx512.mask.pmaxs.w.512(<32 x i16>, <32 x i16>, <32 x i16>, i32)
+
+; CHECK-LABEL: @test_int_x86_avx512_mask_pmaxs_w_512
+; CHECK-NOT: call 
+; CHECK: vpmaxsw %zmm
+; CHECK: {%k1} 
+define <32 x i16>@test_int_x86_avx512_mask_pmaxs_w_512(<32 x i16> %x0, <32 x i16> %x1, <32 x i16> %x2, i32 %x3) {
+  %res = call <32 x i16> @llvm.x86.avx512.mask.pmaxs.w.512(<32 x i16> %x0, <32 x i16> %x1, <32 x i16> %x2, i32 %x3)
+  %res1 = call <32 x i16> @llvm.x86.avx512.mask.pmaxs.w.512(<32 x i16> %x0, <32 x i16> %x1, <32 x i16> %x2, i32 -1)
+  %res2 = add <32 x i16> %res, %res1
+  ret <32 x i16> %res2
+}
+
+declare <64 x i8> @llvm.x86.avx512.mask.pmaxu.b.512(<64 x i8>, <64 x i8>, <64 x i8>, i64)
+
+; CHECK-LABEL: @test_int_x86_avx512_mask_pmaxu_b_512
+; CHECK-NOT: call 
+; CHECK: vpmaxub %zmm
+; CHECK: {%k1} 
+define <64 x i8>@test_int_x86_avx512_mask_pmaxu_b_512(<64 x i8> %x0, <64 x i8> %x1, <64 x i8> %x2, i64 %x3) {
+  %res = call <64 x i8> @llvm.x86.avx512.mask.pmaxu.b.512(<64 x i8> %x0, <64 x i8> %x1, <64 x i8> %x2, i64 %x3)
+  %res1 = call <64 x i8> @llvm.x86.avx512.mask.pmaxu.b.512(<64 x i8> %x0, <64 x i8> %x1, <64 x i8> %x2, i64 -1)
+  %res2 = add <64 x i8> %res, %res1
+  ret <64 x i8> %res2
+}
+
+declare <32 x i16> @llvm.x86.avx512.mask.pmaxu.w.512(<32 x i16>, <32 x i16>, <32 x i16>, i32)
+
+; CHECK-LABEL: @test_int_x86_avx512_mask_pmaxu_w_512
+; CHECK-NOT: call 
+; CHECK: vpmaxuw %zmm
+; CHECK: {%k1} 
+define <32 x i16>@test_int_x86_avx512_mask_pmaxu_w_512(<32 x i16> %x0, <32 x i16> %x1, <32 x i16> %x2, i32 %x3) {
+  %res = call <32 x i16> @llvm.x86.avx512.mask.pmaxu.w.512(<32 x i16> %x0, <32 x i16> %x1, <32 x i16> %x2, i32 %x3)
+  %res1 = call <32 x i16> @llvm.x86.avx512.mask.pmaxu.w.512(<32 x i16> %x0, <32 x i16> %x1, <32 x i16> %x2, i32 -1)
+  %res2 = add <32 x i16> %res, %res1
+  ret <32 x i16> %res2
+}
+
+declare <64 x i8> @llvm.x86.avx512.mask.pmins.b.512(<64 x i8>, <64 x i8>, <64 x i8>, i64)
+
+; CHECK-LABEL: @test_int_x86_avx512_mask_pmins_b_512
+; CHECK-NOT: call 
+; CHECK: vpminsb %zmm
+; CHECK: {%k1} 
+define <64 x i8>@test_int_x86_avx512_mask_pmins_b_512(<64 x i8> %x0, <64 x i8> %x1, <64 x i8> %x2, i64 %x3) {
+  %res = call <64 x i8> @llvm.x86.avx512.mask.pmins.b.512(<64 x i8> %x0, <64 x i8> %x1, <64 x i8> %x2, i64 %x3)
+  %res1 = call <64 x i8> @llvm.x86.avx512.mask.pmins.b.512(<64 x i8> %x0, <64 x i8> %x1, <64 x i8> %x2, i64 -1)
+  %res2 = add <64 x i8> %res, %res1
+  ret <64 x i8> %res2
+}
+
+declare <32 x i16> @llvm.x86.avx512.mask.pmins.w.512(<32 x i16>, <32 x i16>, <32 x i16>, i32)
+
+; CHECK-LABEL: @test_int_x86_avx512_mask_pmins_w_512
+; CHECK-NOT: call 
+; CHECK: vpminsw %zmm
+; CHECK: {%k1} 
+define <32 x i16>@test_int_x86_avx512_mask_pmins_w_512(<32 x i16> %x0, <32 x i16> %x1, <32 x i16> %x2, i32 %x3) {
+  %res = call <32 x i16> @llvm.x86.avx512.mask.pmins.w.512(<32 x i16> %x0, <32 x i16> %x1, <32 x i16> %x2, i32 %x3)
+  %res1 = call <32 x i16> @llvm.x86.avx512.mask.pmins.w.512(<32 x i16> %x0, <32 x i16> %x1, <32 x i16> %x2, i32 -1)
+  %res2 = add <32 x i16> %res, %res1
+  ret <32 x i16> %res2
+}
+
+declare <64 x i8> @llvm.x86.avx512.mask.pminu.b.512(<64 x i8>, <64 x i8>, <64 x i8>, i64)
+
+; CHECK-LABEL: @test_int_x86_avx512_mask_pminu_b_512
+; CHECK-NOT: call 
+; CHECK: vpminub %zmm
+; CHECK: {%k1} 
+define <64 x i8>@test_int_x86_avx512_mask_pminu_b_512(<64 x i8> %x0, <64 x i8> %x1, <64 x i8> %x2, i64 %x3) {
+  %res = call <64 x i8> @llvm.x86.avx512.mask.pminu.b.512(<64 x i8> %x0, <64 x i8> %x1, <64 x i8> %x2, i64 %x3)
+  %res1 = call <64 x i8> @llvm.x86.avx512.mask.pminu.b.512(<64 x i8> %x0, <64 x i8> %x1, <64 x i8> %x2, i64 -1)
+  %res2 = add <64 x i8> %res, %res1
+  ret <64 x i8> %res2
+}
+
+declare <32 x i16> @llvm.x86.avx512.mask.pminu.w.512(<32 x i16>, <32 x i16>, <32 x i16>, i32)
+
+; CHECK-LABEL: @test_int_x86_avx512_mask_pminu_w_512
+; CHECK-NOT: call 
+; CHECK: vpminuw %zmm
+; CHECK: {%k1} 
+define <32 x i16>@test_int_x86_avx512_mask_pminu_w_512(<32 x i16> %x0, <32 x i16> %x1, <32 x i16> %x2, i32 %x3) {
+  %res = call <32 x i16> @llvm.x86.avx512.mask.pminu.w.512(<32 x i16> %x0, <32 x i16> %x1, <32 x i16> %x2, i32 %x3)
+  %res1 = call <32 x i16> @llvm.x86.avx512.mask.pminu.w.512(<32 x i16> %x0, <32 x i16> %x1, <32 x i16> %x2, i32 -1)
+  %res2 = add <32 x i16> %res, %res1
+  ret <32 x i16> %res2
 }
-declare <8 x i16> @llvm.x86.avx512.mask.blend.w.128(<8 x i16>, <8 x i16>, i8) nounwind readonly
+
+declare <32 x i16> @llvm.x86.avx512.mask.vpermt2var.hi.512(<32 x i16>, <32 x i16>, <32 x i16>, i32)
+
+; CHECK-LABEL: @test_int_x86_avx512_mask_vpermt2var_hi_512
+; CHECK-NOT: call 
+; CHECK: kmov 
+; CHECK: vpermt2w %zmm{{.*}}{%k1} 
+define <32 x i16>@test_int_x86_avx512_mask_vpermt2var_hi_512(<32 x i16> %x0, <32 x i16> %x1, <32 x i16> %x2, i32 %x3) {
+  %res = call <32 x i16> @llvm.x86.avx512.mask.vpermt2var.hi.512(<32 x i16> %x0, <32 x i16> %x1, <32 x i16> %x2, i32 %x3)
+  %res1 = call <32 x i16> @llvm.x86.avx512.mask.vpermt2var.hi.512(<32 x i16> %x0, <32 x i16> %x1, <32 x i16> %x2, i32 -1)
+  %res2 = add <32 x i16> %res, %res1
+  ret <32 x i16> %res2
+}
+
+declare <32 x i16> @llvm.x86.avx512.maskz.vpermt2var.hi.512(<32 x i16>, <32 x i16>, <32 x i16>, i32)
+
+; CHECK-LABEL: @test_int_x86_avx512_maskz_vpermt2var_hi_512
+; CHECK-NOT: call 
+; CHECK: kmov 
+; CHECK: vpermt2w %zmm{{.*}}{%k1} {z}
+define <32 x i16>@test_int_x86_avx512_maskz_vpermt2var_hi_512(<32 x i16> %x0, <32 x i16> %x1, <32 x i16> %x2, i32 %x3) {
+  %res = call <32 x i16> @llvm.x86.avx512.maskz.vpermt2var.hi.512(<32 x i16> %x0, <32 x i16> %x1, <32 x i16> %x2, i32 %x3)
+  %res1 = call <32 x i16> @llvm.x86.avx512.maskz.vpermt2var.hi.512(<32 x i16> %x0, <32 x i16> %x1, <32 x i16> %x2, i32 -1)
+  %res2 = add <32 x i16> %res, %res1
+  ret <32 x i16> %res2
+}
+
+declare <32 x i16> @llvm.x86.avx512.mask.vpermi2var.hi.512(<32 x i16>, <32 x i16>, <32 x i16>, i32)
+
+; CHECK-LABEL: @test_int_x86_avx512_mask_vpermi2var_hi_512
+; CHECK-NOT: call 
+; CHECK: kmov 
+; CHECK: vpermi2w %zmm{{.*}}{%k1} 
+define <32 x i16>@test_int_x86_avx512_mask_vpermi2var_hi_512(<32 x i16> %x0, <32 x i16> %x1, <32 x i16> %x2, i32 %x3) {
+  %res = call <32 x i16> @llvm.x86.avx512.mask.vpermi2var.hi.512(<32 x i16> %x0, <32 x i16> %x1, <32 x i16> %x2, i32 %x3)
+  %res1 = call <32 x i16> @llvm.x86.avx512.mask.vpermi2var.hi.512(<32 x i16> %x0, <32 x i16> %x1, <32 x i16> %x2, i32 -1)
+  %res2 = add <32 x i16> %res, %res1
+  ret <32 x i16> %res2
+}
+
+declare <64 x i8> @llvm.x86.avx512.mask.pavg.b.512(<64 x i8>, <64 x i8>, <64 x i8>, i64)
+
+; CHECK-LABEL: @test_int_x86_avx512_mask_pavg_b_512
+; CHECK-NOT: call 
+; CHECK: vpavgb %zmm
+; CHECK: {%k1} 
+define <64 x i8>@test_int_x86_avx512_mask_pavg_b_512(<64 x i8> %x0, <64 x i8> %x1, <64 x i8> %x2, i64 %x3) {
+  %res = call <64 x i8> @llvm.x86.avx512.mask.pavg.b.512(<64 x i8> %x0, <64 x i8> %x1, <64 x i8> %x2, i64 %x3)
+  %res1 = call <64 x i8> @llvm.x86.avx512.mask.pavg.b.512(<64 x i8> %x0, <64 x i8> %x1, <64 x i8> %x2, i64 -1)
+  %res2 = add <64 x i8> %res, %res1
+  ret <64 x i8> %res2
+}
+
+declare <32 x i16> @llvm.x86.avx512.mask.pavg.w.512(<32 x i16>, <32 x i16>, <32 x i16>, i32)
+
+; CHECK-LABEL: @test_int_x86_avx512_mask_pavg_w_512
+; CHECK-NOT: call 
+; CHECK: vpavgw %zmm
+; CHECK: {%k1} 
+define <32 x i16>@test_int_x86_avx512_mask_pavg_w_512(<32 x i16> %x0, <32 x i16> %x1, <32 x i16> %x2, i32 %x3) {
+  %res = call <32 x i16> @llvm.x86.avx512.mask.pavg.w.512(<32 x i16> %x0, <32 x i16> %x1, <32 x i16> %x2, i32 %x3)
+  %res1 = call <32 x i16> @llvm.x86.avx512.mask.pavg.w.512(<32 x i16> %x0, <32 x i16> %x1, <32 x i16> %x2, i32 -1)
+  %res2 = add <32 x i16> %res, %res1
+  ret <32 x i16> %res2
+}
+
+declare <64 x i8> @llvm.x86.avx512.mask.pshuf.b.512(<64 x i8>, <64 x i8>, <64 x i8>, i64)
+
+; CHECK-LABEL: @test_int_x86_avx512_mask_pshuf_b_512
+; CHECK-NOT: call 
+; CHECK: kmov 
+; CHECK: vpshufb %zmm{{.*}}{%k1} 
+define <64 x i8>@test_int_x86_avx512_mask_pshuf_b_512(<64 x i8> %x0, <64 x i8> %x1, <64 x i8> %x2, i64 %x3) {
+  %res = call <64 x i8> @llvm.x86.avx512.mask.pshuf.b.512(<64 x i8> %x0, <64 x i8> %x1, <64 x i8> %x2, i64 %x3)
+  %res1 = call <64 x i8> @llvm.x86.avx512.mask.pshuf.b.512(<64 x i8> %x0, <64 x i8> %x1, <64 x i8> %x2, i64 -1)
+  %res2 = add <64 x i8> %res, %res1
+  ret <64 x i8> %res2
+}
+
+declare <32 x i16> @llvm.x86.avx512.mask.pabs.w.512(<32 x i16>, <32 x i16>, i32)
+
+; CHECK-LABEL: @test_int_x86_avx512_mask_pabs_w_512
+; CHECK-NOT: call 
+; CHECK: kmov 
+; CHECK: vpabsw{{.*}}{%k1} 
+define <32 x i16>@test_int_x86_avx512_mask_pabs_w_512(<32 x i16> %x0, <32 x i16> %x1, i32 %x2) {
+  %res = call <32 x i16> @llvm.x86.avx512.mask.pabs.w.512(<32 x i16> %x0, <32 x i16> %x1, i32 %x2)
+  %res1 = call <32 x i16> @llvm.x86.avx512.mask.pabs.w.512(<32 x i16> %x0, <32 x i16> %x1, i32 -1)
+  %res2 = add <32 x i16> %res, %res1
+  ret <32 x i16> %res2
+}
+
+declare <64 x i8> @llvm.x86.avx512.mask.pabs.b.512(<64 x i8>, <64 x i8>, i64)
+
+; CHECK-LABEL: @test_int_x86_avx512_mask_pabs_b_512
+; CHECK-NOT: call 
+; CHECK: kmov 
+; CHECK: vpabsb{{.*}}{%k1} 
+define <64 x i8>@test_int_x86_avx512_mask_pabs_b_512(<64 x i8> %x0, <64 x i8> %x1, i64 %x2) {
+  %res = call <64 x i8> @llvm.x86.avx512.mask.pabs.b.512(<64 x i8> %x0, <64 x i8> %x1, i64 %x2)
+  %res1 = call <64 x i8> @llvm.x86.avx512.mask.pabs.b.512(<64 x i8> %x0, <64 x i8> %x1, i64 -1)
+  %res2 = add <64 x i8> %res, %res1
+  ret <64 x i8> %res2
+}
+
+declare <32 x i16> @llvm.x86.avx512.mask.pmulhu.w.512(<32 x i16>, <32 x i16>, <32 x i16>, i32)
+
+; CHECK-LABEL: @test_int_x86_avx512_mask_pmulhu_w_512
+; CHECK-NOT: call 
+; CHECK: kmov 
+; CHECK: {%k1} 
+; CHECK: vpmulhuw {{.*}}encoding: [0x62
+define <32 x i16>@test_int_x86_avx512_mask_pmulhu_w_512(<32 x i16> %x0, <32 x i16> %x1, <32 x i16> %x2, i32 %x3) {
+  %res = call <32 x i16> @llvm.x86.avx512.mask.pmulhu.w.512(<32 x i16> %x0, <32 x i16> %x1, <32 x i16> %x2, i32 %x3)
+  %res1 = call <32 x i16> @llvm.x86.avx512.mask.pmulhu.w.512(<32 x i16> %x0, <32 x i16> %x1, <32 x i16> %x2, i32 -1)
+  %res2 = add <32 x i16> %res, %res1
+  ret <32 x i16> %res2
+}
+
+declare <32 x i16> @llvm.x86.avx512.mask.pmulh.w.512(<32 x i16>, <32 x i16>, <32 x i16>, i32)
+
+; CHECK-LABEL: @test_int_x86_avx512_mask_pmulh_w_512
+; CHECK-NOT: call 
+; CHECK: kmov 
+; CHECK: {%k1} 
+; CHECK: vpmulhw {{.*}}encoding: [0x62
+define <32 x i16>@test_int_x86_avx512_mask_pmulh_w_512(<32 x i16> %x0, <32 x i16> %x1, <32 x i16> %x2, i32 %x3) {
+  %res = call <32 x i16> @llvm.x86.avx512.mask.pmulh.w.512(<32 x i16> %x0, <32 x i16> %x1, <32 x i16> %x2, i32 %x3)
+  %res1 = call <32 x i16> @llvm.x86.avx512.mask.pmulh.w.512(<32 x i16> %x0, <32 x i16> %x1, <32 x i16> %x2, i32 -1)
+  %res2 = add <32 x i16> %res, %res1
+  ret <32 x i16> %res2
+}
+
+declare <32 x i16> @llvm.x86.avx512.mask.pmul.hr.sw.512(<32 x i16>, <32 x i16>, <32 x i16>, i32)
+
+; CHECK-LABEL: @test_int_x86_avx512_mask_pmulhr_sw_512
+; CHECK-NOT: call 
+; CHECK: kmov 
+; CHECK: {%k1} 
+; CHECK: vpmulhrsw {{.*}}encoding: [0x62
+define <32 x i16>@test_int_x86_avx512_mask_pmulhr_sw_512(<32 x i16> %x0, <32 x i16> %x1, <32 x i16> %x2, i32 %x3) {
+  %res = call <32 x i16> @llvm.x86.avx512.mask.pmul.hr.sw.512(<32 x i16> %x0, <32 x i16> %x1, <32 x i16> %x2, i32 %x3)
+  %res1 = call <32 x i16> @llvm.x86.avx512.mask.pmul.hr.sw.512(<32 x i16> %x0, <32 x i16> %x1, <32 x i16> %x2, i32 -1)
+  %res2 = add <32 x i16> %res, %res1
+  ret <32 x i16> %res2
+}
+
+declare <32 x i8> @llvm.x86.avx512.mask.pmov.wb.512(<32 x i16>, <32 x i8>, i32)
+
+define <32 x i8>@test_int_x86_avx512_mask_pmov_wb_512(<32 x i16> %x0, <32 x i8> %x1, i32 %x2) {
+; CHECK-LABEL: test_int_x86_avx512_mask_pmov_wb_512:
+; CHECK:       vpmovwb %zmm0, %ymm1 {%k1}
+; CHECK-NEXT:  vpmovwb %zmm0, %ymm2 {%k1} {z}
+; CHECK-NEXT:  vpmovwb %zmm0, %ymm0
+    %res0 = call <32 x i8> @llvm.x86.avx512.mask.pmov.wb.512(<32 x i16> %x0, <32 x i8> %x1, i32 -1)
+    %res1 = call <32 x i8> @llvm.x86.avx512.mask.pmov.wb.512(<32 x i16> %x0, <32 x i8> %x1, i32 %x2)
+    %res2 = call <32 x i8> @llvm.x86.avx512.mask.pmov.wb.512(<32 x i16> %x0, <32 x i8> zeroinitializer, i32 %x2)
+    %res3 = add <32 x i8> %res0, %res1
+    %res4 = add <32 x i8> %res3, %res2
+    ret <32 x i8> %res4
+}
+
+declare void @llvm.x86.avx512.mask.pmov.wb.mem.512(i8* %ptr, <32 x i16>, i32)
+
+define void @test_int_x86_avx512_mask_pmov_wb_mem_512(i8* %ptr, <32 x i16> %x1, i32 %x2) {
+; CHECK-LABEL: test_int_x86_avx512_mask_pmov_wb_mem_512:
+; CHECK:  vpmovwb %zmm0, (%rdi)
+; CHECK:  vpmovwb %zmm0, (%rdi) {%k1}
+    call void @llvm.x86.avx512.mask.pmov.wb.mem.512(i8* %ptr, <32 x i16> %x1, i32 -1)
+    call void @llvm.x86.avx512.mask.pmov.wb.mem.512(i8* %ptr, <32 x i16> %x1, i32 %x2)
+    ret void
+}
+
+declare <32 x i8> @llvm.x86.avx512.mask.pmovs.wb.512(<32 x i16>, <32 x i8>, i32)
+
+define <32 x i8>@test_int_x86_avx512_mask_pmovs_wb_512(<32 x i16> %x0, <32 x i8> %x1, i32 %x2) {
+; CHECK-LABEL: test_int_x86_avx512_mask_pmovs_wb_512:
+; CHECK:       vpmovswb %zmm0, %ymm1 {%k1}
+; CHECK-NEXT:  vpmovswb %zmm0, %ymm2 {%k1} {z}
+; CHECK-NEXT:  vpmovswb %zmm0, %ymm0
+    %res0 = call <32 x i8> @llvm.x86.avx512.mask.pmovs.wb.512(<32 x i16> %x0, <32 x i8> %x1, i32 -1)
+    %res1 = call <32 x i8> @llvm.x86.avx512.mask.pmovs.wb.512(<32 x i16> %x0, <32 x i8> %x1, i32 %x2)
+    %res2 = call <32 x i8> @llvm.x86.avx512.mask.pmovs.wb.512(<32 x i16> %x0, <32 x i8> zeroinitializer, i32 %x2)
+    %res3 = add <32 x i8> %res0, %res1
+    %res4 = add <32 x i8> %res3, %res2
+    ret <32 x i8> %res4
+}
+
+declare void @llvm.x86.avx512.mask.pmovs.wb.mem.512(i8* %ptr, <32 x i16>, i32)
+
+define void @test_int_x86_avx512_mask_pmovs_wb_mem_512(i8* %ptr, <32 x i16> %x1, i32 %x2) {
+; CHECK-LABEL: test_int_x86_avx512_mask_pmovs_wb_mem_512:
+; CHECK:  vpmovswb %zmm0, (%rdi)
+; CHECK:  vpmovswb %zmm0, (%rdi) {%k1}
+    call void @llvm.x86.avx512.mask.pmovs.wb.mem.512(i8* %ptr, <32 x i16> %x1, i32 -1)
+    call void @llvm.x86.avx512.mask.pmovs.wb.mem.512(i8* %ptr, <32 x i16> %x1, i32 %x2)
+    ret void
+}
+
+declare <32 x i8> @llvm.x86.avx512.mask.pmovus.wb.512(<32 x i16>, <32 x i8>, i32)
+
+define <32 x i8>@test_int_x86_avx512_mask_pmovus_wb_512(<32 x i16> %x0, <32 x i8> %x1, i32 %x2) {
+; CHECK-LABEL: test_int_x86_avx512_mask_pmovus_wb_512:
+; CHECK:       vpmovuswb %zmm0, %ymm1 {%k1}
+; CHECK-NEXT:  vpmovuswb %zmm0, %ymm2 {%k1} {z}
+; CHECK-NEXT:  vpmovuswb %zmm0, %ymm0
+    %res0 = call <32 x i8> @llvm.x86.avx512.mask.pmovus.wb.512(<32 x i16> %x0, <32 x i8> %x1, i32 -1)
+    %res1 = call <32 x i8> @llvm.x86.avx512.mask.pmovus.wb.512(<32 x i16> %x0, <32 x i8> %x1, i32 %x2)
+    %res2 = call <32 x i8> @llvm.x86.avx512.mask.pmovus.wb.512(<32 x i16> %x0, <32 x i8> zeroinitializer, i32 %x2)
+    %res3 = add <32 x i8> %res0, %res1
+    %res4 = add <32 x i8> %res3, %res2
+    ret <32 x i8> %res4
+}
+
+declare void @llvm.x86.avx512.mask.pmovus.wb.mem.512(i8* %ptr, <32 x i16>, i32)
+
+define void @test_int_x86_avx512_mask_pmovus_wb_mem_512(i8* %ptr, <32 x i16> %x1, i32 %x2) {
+; CHECK-LABEL: test_int_x86_avx512_mask_pmovus_wb_mem_512:
+; CHECK:  vpmovuswb %zmm0, (%rdi)
+; CHECK:  vpmovuswb %zmm0, (%rdi) {%k1}
+    call void @llvm.x86.avx512.mask.pmovus.wb.mem.512(i8* %ptr, <32 x i16> %x1, i32 -1)
+    call void @llvm.x86.avx512.mask.pmovus.wb.mem.512(i8* %ptr, <32 x i16> %x1, i32 %x2)
+    ret void
+}
+
+declare <32 x i16> @llvm.x86.avx512.mask.pmaddubs.w.512(<64 x i8>, <64 x i8>, <32 x i16>, i32)
+
+define <32 x i16>@test_int_x86_avx512_mask_pmaddubs_w_512(<64 x i8> %x0, <64 x i8> %x1, <32 x i16> %x2, i32 %x3) {
+; CHECK-LABEL: test_int_x86_avx512_mask_pmaddubs_w_512:
+; CHECK:       ## BB#0:
+; CHECK-NEXT:    kmovd %edi, %k1 ## encoding: [0xc5,0xfb,0x92,0xcf]
+; CHECK-NEXT:    vpmaddubsw %zmm1, %zmm0, %zmm2 {%k1}
+; CHECK-NEXT:    vpmaddubsw %zmm1, %zmm0, %zmm0
+; CHECK-NEXT:    vpaddw %zmm0, %zmm2, %zmm0 
+; CHECK-NEXT:    retq
+  %res = call <32 x i16> @llvm.x86.avx512.mask.pmaddubs.w.512(<64 x i8> %x0, <64 x i8> %x1, <32 x i16> %x2, i32 %x3)
+  %res1 = call <32 x i16> @llvm.x86.avx512.mask.pmaddubs.w.512(<64 x i8> %x0, <64 x i8> %x1, <32 x i16> %x2, i32 -1)
+  %res2 = add <32 x i16> %res, %res1
+  ret <32 x i16> %res2
+}
+
+declare <16 x i32> @llvm.x86.avx512.mask.pmaddw.d.512(<32 x i16>, <32 x i16>, <16 x i32>, i16)
+
+define <16 x i32>@test_int_x86_avx512_mask_pmaddw_d_512(<32 x i16> %x0, <32 x i16> %x1, <16 x i32> %x2, i16 %x3) {
+; CHECK-LABEL: test_int_x86_avx512_mask_pmaddw_d_512:
+; CHECK:       ## BB#0:
+; CHECK-NEXT:    kmovw %edi, %k1
+; CHECK-NEXT:    vpmaddwd %zmm1, %zmm0, %zmm2 {%k1} 
+; CHECK-NEXT:    vpmaddwd %zmm1, %zmm0, %zmm0
+; CHECK-NEXT:    vpaddd %zmm0, %zmm2, %zmm0 
+; CHECK-NEXT:    retq
+  %res = call <16 x i32> @llvm.x86.avx512.mask.pmaddw.d.512(<32 x i16> %x0, <32 x i16> %x1, <16 x i32> %x2, i16 %x3)
+  %res1 = call <16 x i32> @llvm.x86.avx512.mask.pmaddw.d.512(<32 x i16> %x0, <32 x i16> %x1, <16 x i32> %x2, i16 -1)
+  %res2 = add <16 x i32> %res, %res1
+  ret <16 x i32> %res2
+}
+
+declare <64 x i8> @llvm.x86.avx512.mask.punpckhb.w.512(<64 x i8>, <64 x i8>, <64 x i8>, i64)
+
+define <64 x i8>@test_int_x86_avx512_mask_punpckhb_w_512(<64 x i8> %x0, <64 x i8> %x1, <64 x i8> %x2, i64 %x3) {
+; CHECK-LABEL: test_int_x86_avx512_mask_punpckhb_w_512:
+; CHECK:       ## BB#0:
+; CHECK-NEXT:    kmovq %rdi, %k1
+; CHECK-NEXT:    vpunpckhbw %zmm1, %zmm0, %zmm2 {%k1}
+; CHECK-NEXT:    vpunpckhbw %zmm1, %zmm0, %zmm0
+; CHECK-NEXT:    vpaddb %zmm0, %zmm2, %zmm0
+; CHECK-NEXT:    retq
+  %res = call <64 x i8> @llvm.x86.avx512.mask.punpckhb.w.512(<64 x i8> %x0, <64 x i8> %x1, <64 x i8> %x2, i64 %x3)
+  %res1 = call <64 x i8> @llvm.x86.avx512.mask.punpckhb.w.512(<64 x i8> %x0, <64 x i8> %x1, <64 x i8> %x2, i64 -1)
+  %res2 = add <64 x i8> %res, %res1
+  ret <64 x i8> %res2
+}
+
+declare <64 x i8> @llvm.x86.avx512.mask.punpcklb.w.512(<64 x i8>, <64 x i8>, <64 x i8>, i64)
+
+define <64 x i8>@test_int_x86_avx512_mask_punpcklb_w_512(<64 x i8> %x0, <64 x i8> %x1, <64 x i8> %x2, i64 %x3) {
+; CHECK-LABEL: test_int_x86_avx512_mask_punpcklb_w_512:
+; CHECK:       ## BB#0:
+; CHECK-NEXT:    kmovq %rdi, %k1
+; CHECK-NEXT:    vpunpcklbw %zmm1, %zmm0, %zmm2 {%k1}
+; CHECK-NEXT:    vpunpcklbw %zmm1, %zmm0, %zmm0
+; CHECK-NEXT:    vpaddb %zmm0, %zmm2, %zmm0
+; CHECK-NEXT:    retq
+  %res = call <64 x i8> @llvm.x86.avx512.mask.punpcklb.w.512(<64 x i8> %x0, <64 x i8> %x1, <64 x i8> %x2, i64 %x3)
+  %res1 = call <64 x i8> @llvm.x86.avx512.mask.punpcklb.w.512(<64 x i8> %x0, <64 x i8> %x1, <64 x i8> %x2, i64 -1)
+  %res2 = add <64 x i8> %res, %res1
+  ret <64 x i8> %res2
+}
+
+declare <32 x i16> @llvm.x86.avx512.mask.punpckhw.d.512(<32 x i16>, <32 x i16>, <32 x i16>, i32)
+
+define <32 x i16>@test_int_x86_avx512_mask_punpckhw_d_512(<32 x i16> %x0, <32 x i16> %x1, <32 x i16> %x2, i32 %x3) {
+; CHECK-LABEL: test_int_x86_avx512_mask_punpckhw_d_512:
+; CHECK:       ## BB#0:
+; CHECK-NEXT:    kmovd %edi, %k1
+; CHECK-NEXT:    vpunpckhwd %zmm1, %zmm0, %zmm2 {%k1}
+; CHECK-NEXT:    vpunpckhwd %zmm1, %zmm0, %zmm0
+; CHECK-NEXT:    vpaddw %zmm0, %zmm2, %zmm0
+; CHECK-NEXT:    retq
+  %res = call <32 x i16> @llvm.x86.avx512.mask.punpckhw.d.512(<32 x i16> %x0, <32 x i16> %x1, <32 x i16> %x2, i32 %x3)
+  %res1 = call <32 x i16> @llvm.x86.avx512.mask.punpckhw.d.512(<32 x i16> %x0, <32 x i16> %x1, <32 x i16> %x2, i32 -1)
+  %res2 = add <32 x i16> %res, %res1
+  ret <32 x i16> %res2
+}
+
+declare <32 x i16> @llvm.x86.avx512.mask.punpcklw.d.512(<32 x i16>, <32 x i16>, <32 x i16>, i32)
+
+define <32 x i16>@test_int_x86_avx512_mask_punpcklw_d_512(<32 x i16> %x0, <32 x i16> %x1, <32 x i16> %x2, i32 %x3) {
+; CHECK-LABEL: test_int_x86_avx512_mask_punpcklw_d_512:
+; CHECK:       ## BB#0:
+; CHECK-NEXT:    kmovd %edi, %k1
+; CHECK-NEXT:    vpunpcklwd %zmm1, %zmm0, %zmm2 {%k1}
+; CHECK-NEXT:    vpunpcklwd %zmm1, %zmm0, %zmm0
+; CHECK-NEXT:    vpaddw %zmm0, %zmm2, %zmm0
+; CHECK-NEXT:    retq
+  %res = call <32 x i16> @llvm.x86.avx512.mask.punpcklw.d.512(<32 x i16> %x0, <32 x i16> %x1, <32 x i16> %x2, i32 %x3)
+  %res1 = call <32 x i16> @llvm.x86.avx512.mask.punpcklw.d.512(<32 x i16> %x0, <32 x i16> %x1, <32 x i16> %x2, i32 -1)
+  %res2 = add <32 x i16> %res, %res1
+  ret <32 x i16> %res2
+}
\ No newline at end of file
diff --git a/test/CodeGen/X86/avx512bwvl-intrinsics.ll b/test/CodeGen/X86/avx512bwvl-intrinsics.ll
index cffa0a5e9b7..21f6eb545bd 100644
--- a/test/CodeGen/X86/avx512bwvl-intrinsics.ll
+++ b/test/CodeGen/X86/avx512bwvl-intrinsics.ll
@@ -69,28 +69,28 @@ declare i16 @llvm.x86.avx512.mask.pcmpgt.w.256(<16 x i16>, <16 x i16>, i16)
 define <8 x i32> @test_cmp_b_256(<32 x i8> %a0, <32 x i8> %a1) {
 ; CHECK_LABEL: test_cmp_b_256
 ; CHECK: vpcmpeqb %ymm1, %ymm0, %k0 ##
-  %res0 = call i32 @llvm.x86.avx512.mask.cmp.b.256(<32 x i8> %a0, <32 x i8> %a1, i8 0, i32 -1)
+  %res0 = call i32 @llvm.x86.avx512.mask.cmp.b.256(<32 x i8> %a0, <32 x i8> %a1, i32 0, i32 -1)
   %vec0 = insertelement <8 x i32> undef, i32 %res0, i32 0
 ; CHECK: vpcmpltb %ymm1, %ymm0, %k0 ##
-  %res1 = call i32 @llvm.x86.avx512.mask.cmp.b.256(<32 x i8> %a0, <32 x i8> %a1, i8 1, i32 -1)
+  %res1 = call i32 @llvm.x86.avx512.mask.cmp.b.256(<32 x i8> %a0, <32 x i8> %a1, i32 1, i32 -1)
   %vec1 = insertelement <8 x i32> %vec0, i32 %res1, i32 1
 ; CHECK: vpcmpleb %ymm1, %ymm0, %k0 ##
-  %res2 = call i32 @llvm.x86.avx512.mask.cmp.b.256(<32 x i8> %a0, <32 x i8> %a1, i8 2, i32 -1)
+  %res2 = call i32 @llvm.x86.avx512.mask.cmp.b.256(<32 x i8> %a0, <32 x i8> %a1, i32 2, i32 -1)
   %vec2 = insertelement <8 x i32> %vec1, i32 %res2, i32 2
 ; CHECK: vpcmpunordb %ymm1, %ymm0, %k0 ##
-  %res3 = call i32 @llvm.x86.avx512.mask.cmp.b.256(<32 x i8> %a0, <32 x i8> %a1, i8 3, i32 -1)
+  %res3 = call i32 @llvm.x86.avx512.mask.cmp.b.256(<32 x i8> %a0, <32 x i8> %a1, i32 3, i32 -1)
   %vec3 = insertelement <8 x i32> %vec2, i32 %res3, i32 3
 ; CHECK: vpcmpneqb %ymm1, %ymm0, %k0 ##
-  %res4 = call i32 @llvm.x86.avx512.mask.cmp.b.256(<32 x i8> %a0, <32 x i8> %a1, i8 4, i32 -1)
+  %res4 = call i32 @llvm.x86.avx512.mask.cmp.b.256(<32 x i8> %a0, <32 x i8> %a1, i32 4, i32 -1)
   %vec4 = insertelement <8 x i32> %vec3, i32 %res4, i32 4
 ; CHECK: vpcmpnltb %ymm1, %ymm0, %k0 ##
-  %res5 = call i32 @llvm.x86.avx512.mask.cmp.b.256(<32 x i8> %a0, <32 x i8> %a1, i8 5, i32 -1)
+  %res5 = call i32 @llvm.x86.avx512.mask.cmp.b.256(<32 x i8> %a0, <32 x i8> %a1, i32 5, i32 -1)
   %vec5 = insertelement <8 x i32> %vec4, i32 %res5, i32 5
 ; CHECK: vpcmpnleb %ymm1, %ymm0, %k0 ##
-  %res6 = call i32 @llvm.x86.avx512.mask.cmp.b.256(<32 x i8> %a0, <32 x i8> %a1, i8 6, i32 -1)
+  %res6 = call i32 @llvm.x86.avx512.mask.cmp.b.256(<32 x i8> %a0, <32 x i8> %a1, i32 6, i32 -1)
   %vec6 = insertelement <8 x i32> %vec5, i32 %res6, i32 6
 ; CHECK: vpcmpordb %ymm1, %ymm0, %k0 ##
-  %res7 = call i32 @llvm.x86.avx512.mask.cmp.b.256(<32 x i8> %a0, <32 x i8> %a1, i8 7, i32 -1)
+  %res7 = call i32 @llvm.x86.avx512.mask.cmp.b.256(<32 x i8> %a0, <32 x i8> %a1, i32 7, i32 -1)
   %vec7 = insertelement <8 x i32> %vec6, i32 %res7, i32 7
   ret <8 x i32> %vec7
 }
@@ -98,59 +98,59 @@ define <8 x i32> @test_cmp_b_256(<32 x i8> %a0, <32 x i8> %a1) {
 define <8 x i32> @test_mask_cmp_b_256(<32 x i8> %a0, <32 x i8> %a1, i32 %mask) {
 ; CHECK_LABEL: test_mask_cmp_b_256
 ; CHECK: vpcmpeqb %ymm1, %ymm0, %k0 {%k1} ##
-  %res0 = call i32 @llvm.x86.avx512.mask.cmp.b.256(<32 x i8> %a0, <32 x i8> %a1, i8 0, i32 %mask)
+  %res0 = call i32 @llvm.x86.avx512.mask.cmp.b.256(<32 x i8> %a0, <32 x i8> %a1, i32 0, i32 %mask)
   %vec0 = insertelement <8 x i32> undef, i32 %res0, i32 0
 ; CHECK: vpcmpltb %ymm1, %ymm0, %k0 {%k1} ##
-  %res1 = call i32 @llvm.x86.avx512.mask.cmp.b.256(<32 x i8> %a0, <32 x i8> %a1, i8 1, i32 %mask)
+  %res1 = call i32 @llvm.x86.avx512.mask.cmp.b.256(<32 x i8> %a0, <32 x i8> %a1, i32 1, i32 %mask)
   %vec1 = insertelement <8 x i32> %vec0, i32 %res1, i32 1
 ; CHECK: vpcmpleb %ymm1, %ymm0, %k0 {%k1} ##
-  %res2 = call i32 @llvm.x86.avx512.mask.cmp.b.256(<32 x i8> %a0, <32 x i8> %a1, i8 2, i32 %mask)
+  %res2 = call i32 @llvm.x86.avx512.mask.cmp.b.256(<32 x i8> %a0, <32 x i8> %a1, i32 2, i32 %mask)
   %vec2 = insertelement <8 x i32> %vec1, i32 %res2, i32 2
 ; CHECK: vpcmpunordb %ymm1, %ymm0, %k0 {%k1} ##
-  %res3 = call i32 @llvm.x86.avx512.mask.cmp.b.256(<32 x i8> %a0, <32 x i8> %a1, i8 3, i32 %mask)
+  %res3 = call i32 @llvm.x86.avx512.mask.cmp.b.256(<32 x i8> %a0, <32 x i8> %a1, i32 3, i32 %mask)
   %vec3 = insertelement <8 x i32> %vec2, i32 %res3, i32 3
 ; CHECK: vpcmpneqb %ymm1, %ymm0, %k0 {%k1} ##
-  %res4 = call i32 @llvm.x86.avx512.mask.cmp.b.256(<32 x i8> %a0, <32 x i8> %a1, i8 4, i32 %mask)
+  %res4 = call i32 @llvm.x86.avx512.mask.cmp.b.256(<32 x i8> %a0, <32 x i8> %a1, i32 4, i32 %mask)
   %vec4 = insertelement <8 x i32> %vec3, i32 %res4, i32 4
 ; CHECK: vpcmpnltb %ymm1, %ymm0, %k0 {%k1} ##
-  %res5 = call i32 @llvm.x86.avx512.mask.cmp.b.256(<32 x i8> %a0, <32 x i8> %a1, i8 5, i32 %mask)
+  %res5 = call i32 @llvm.x86.avx512.mask.cmp.b.256(<32 x i8> %a0, <32 x i8> %a1, i32 5, i32 %mask)
   %vec5 = insertelement <8 x i32> %vec4, i32 %res5, i32 5
 ; CHECK: vpcmpnleb %ymm1, %ymm0, %k0 {%k1} ##
-  %res6 = call i32 @llvm.x86.avx512.mask.cmp.b.256(<32 x i8> %a0, <32 x i8> %a1, i8 6, i32 %mask)
+  %res6 = call i32 @llvm.x86.avx512.mask.cmp.b.256(<32 x i8> %a0, <32 x i8> %a1, i32 6, i32 %mask)
   %vec6 = insertelement <8 x i32> %vec5, i32 %res6, i32 6
 ; CHECK: vpcmpordb %ymm1, %ymm0, %k0 {%k1} ##
-  %res7 = call i32 @llvm.x86.avx512.mask.cmp.b.256(<32 x i8> %a0, <32 x i8> %a1, i8 7, i32 %mask)
+  %res7 = call i32 @llvm.x86.avx512.mask.cmp.b.256(<32 x i8> %a0, <32 x i8> %a1, i32 7, i32 %mask)
   %vec7 = insertelement <8 x i32> %vec6, i32 %res7, i32 7
   ret <8 x i32> %vec7
 }
 
-declare i32 @llvm.x86.avx512.mask.cmp.b.256(<32 x i8>, <32 x i8>, i8, i32) nounwind readnone
+declare i32 @llvm.x86.avx512.mask.cmp.b.256(<32 x i8>, <32 x i8>, i32, i32) nounwind readnone
 
 define <8 x i32> @test_ucmp_b_256(<32 x i8> %a0, <32 x i8> %a1) {
 ; CHECK_LABEL: test_ucmp_b_256
 ; CHECK: vpcmpequb %ymm1, %ymm0, %k0 ##
-  %res0 = call i32 @llvm.x86.avx512.mask.ucmp.b.256(<32 x i8> %a0, <32 x i8> %a1, i8 0, i32 -1)
+  %res0 = call i32 @llvm.x86.avx512.mask.ucmp.b.256(<32 x i8> %a0, <32 x i8> %a1, i32 0, i32 -1)
   %vec0 = insertelement <8 x i32> undef, i32 %res0, i32 0
 ; CHECK: vpcmpltub %ymm1, %ymm0, %k0 ##
-  %res1 = call i32 @llvm.x86.avx512.mask.ucmp.b.256(<32 x i8> %a0, <32 x i8> %a1, i8 1, i32 -1)
+  %res1 = call i32 @llvm.x86.avx512.mask.ucmp.b.256(<32 x i8> %a0, <32 x i8> %a1, i32 1, i32 -1)
   %vec1 = insertelement <8 x i32> %vec0, i32 %res1, i32 1
 ; CHECK: vpcmpleub %ymm1, %ymm0, %k0 ##
-  %res2 = call i32 @llvm.x86.avx512.mask.ucmp.b.256(<32 x i8> %a0, <32 x i8> %a1, i8 2, i32 -1)
+  %res2 = call i32 @llvm.x86.avx512.mask.ucmp.b.256(<32 x i8> %a0, <32 x i8> %a1, i32 2, i32 -1)
   %vec2 = insertelement <8 x i32> %vec1, i32 %res2, i32 2
 ; CHECK: vpcmpunordub %ymm1, %ymm0, %k0 ##
-  %res3 = call i32 @llvm.x86.avx512.mask.ucmp.b.256(<32 x i8> %a0, <32 x i8> %a1, i8 3, i32 -1)
+  %res3 = call i32 @llvm.x86.avx512.mask.ucmp.b.256(<32 x i8> %a0, <32 x i8> %a1, i32 3, i32 -1)
   %vec3 = insertelement <8 x i32> %vec2, i32 %res3, i32 3
 ; CHECK: vpcmpnequb %ymm1, %ymm0, %k0 ##
-  %res4 = call i32 @llvm.x86.avx512.mask.ucmp.b.256(<32 x i8> %a0, <32 x i8> %a1, i8 4, i32 -1)
+  %res4 = call i32 @llvm.x86.avx512.mask.ucmp.b.256(<32 x i8> %a0, <32 x i8> %a1, i32 4, i32 -1)
   %vec4 = insertelement <8 x i32> %vec3, i32 %res4, i32 4
 ; CHECK: vpcmpnltub %ymm1, %ymm0, %k0 ##
-  %res5 = call i32 @llvm.x86.avx512.mask.ucmp.b.256(<32 x i8> %a0, <32 x i8> %a1, i8 5, i32 -1)
+  %res5 = call i32 @llvm.x86.avx512.mask.ucmp.b.256(<32 x i8> %a0, <32 x i8> %a1, i32 5, i32 -1)
   %vec5 = insertelement <8 x i32> %vec4, i32 %res5, i32 5
 ; CHECK: vpcmpnleub %ymm1, %ymm0, %k0 ##
-  %res6 = call i32 @llvm.x86.avx512.mask.ucmp.b.256(<32 x i8> %a0, <32 x i8> %a1, i8 6, i32 -1)
+  %res6 = call i32 @llvm.x86.avx512.mask.ucmp.b.256(<32 x i8> %a0, <32 x i8> %a1, i32 6, i32 -1)
   %vec6 = insertelement <8 x i32> %vec5, i32 %res6, i32 6
 ; CHECK: vpcmpordub %ymm1, %ymm0, %k0 ##
-  %res7 = call i32 @llvm.x86.avx512.mask.ucmp.b.256(<32 x i8> %a0, <32 x i8> %a1, i8 7, i32 -1)
+  %res7 = call i32 @llvm.x86.avx512.mask.ucmp.b.256(<32 x i8> %a0, <32 x i8> %a1, i32 7, i32 -1)
   %vec7 = insertelement <8 x i32> %vec6, i32 %res7, i32 7
   ret <8 x i32> %vec7
 }
@@ -158,59 +158,59 @@ define <8 x i32> @test_ucmp_b_256(<32 x i8> %a0, <32 x i8> %a1) {
 define <8 x i32> @test_mask_ucmp_b_256(<32 x i8> %a0, <32 x i8> %a1, i32 %mask) {
 ; CHECK_LABEL: test_mask_ucmp_b_256
 ; CHECK: vpcmpequb %ymm1, %ymm0, %k0 {%k1} ##
-  %res0 = call i32 @llvm.x86.avx512.mask.ucmp.b.256(<32 x i8> %a0, <32 x i8> %a1, i8 0, i32 %mask)
+  %res0 = call i32 @llvm.x86.avx512.mask.ucmp.b.256(<32 x i8> %a0, <32 x i8> %a1, i32 0, i32 %mask)
   %vec0 = insertelement <8 x i32> undef, i32 %res0, i32 0
 ; CHECK: vpcmpltub %ymm1, %ymm0, %k0 {%k1} ##
-  %res1 = call i32 @llvm.x86.avx512.mask.ucmp.b.256(<32 x i8> %a0, <32 x i8> %a1, i8 1, i32 %mask)
+  %res1 = call i32 @llvm.x86.avx512.mask.ucmp.b.256(<32 x i8> %a0, <32 x i8> %a1, i32 1, i32 %mask)
   %vec1 = insertelement <8 x i32> %vec0, i32 %res1, i32 1
 ; CHECK: vpcmpleub %ymm1, %ymm0, %k0 {%k1} ##
-  %res2 = call i32 @llvm.x86.avx512.mask.ucmp.b.256(<32 x i8> %a0, <32 x i8> %a1, i8 2, i32 %mask)
+  %res2 = call i32 @llvm.x86.avx512.mask.ucmp.b.256(<32 x i8> %a0, <32 x i8> %a1, i32 2, i32 %mask)
   %vec2 = insertelement <8 x i32> %vec1, i32 %res2, i32 2
 ; CHECK: vpcmpunordub %ymm1, %ymm0, %k0 {%k1} ##
-  %res3 = call i32 @llvm.x86.avx512.mask.ucmp.b.256(<32 x i8> %a0, <32 x i8> %a1, i8 3, i32 %mask)
+  %res3 = call i32 @llvm.x86.avx512.mask.ucmp.b.256(<32 x i8> %a0, <32 x i8> %a1, i32 3, i32 %mask)
   %vec3 = insertelement <8 x i32> %vec2, i32 %res3, i32 3
 ; CHECK: vpcmpnequb %ymm1, %ymm0, %k0 {%k1} ##
-  %res4 = call i32 @llvm.x86.avx512.mask.ucmp.b.256(<32 x i8> %a0, <32 x i8> %a1, i8 4, i32 %mask)
+  %res4 = call i32 @llvm.x86.avx512.mask.ucmp.b.256(<32 x i8> %a0, <32 x i8> %a1, i32 4, i32 %mask)
   %vec4 = insertelement <8 x i32> %vec3, i32 %res4, i32 4
 ; CHECK: vpcmpnltub %ymm1, %ymm0, %k0 {%k1} ##
-  %res5 = call i32 @llvm.x86.avx512.mask.ucmp.b.256(<32 x i8> %a0, <32 x i8> %a1, i8 5, i32 %mask)
+  %res5 = call i32 @llvm.x86.avx512.mask.ucmp.b.256(<32 x i8> %a0, <32 x i8> %a1, i32 5, i32 %mask)
   %vec5 = insertelement <8 x i32> %vec4, i32 %res5, i32 5
 ; CHECK: vpcmpnleub %ymm1, %ymm0, %k0 {%k1} ##
-  %res6 = call i32 @llvm.x86.avx512.mask.ucmp.b.256(<32 x i8> %a0, <32 x i8> %a1, i8 6, i32 %mask)
+  %res6 = call i32 @llvm.x86.avx512.mask.ucmp.b.256(<32 x i8> %a0, <32 x i8> %a1, i32 6, i32 %mask)
   %vec6 = insertelement <8 x i32> %vec5, i32 %res6, i32 6
 ; CHECK: vpcmpordub %ymm1, %ymm0, %k0 {%k1} ##
-  %res7 = call i32 @llvm.x86.avx512.mask.ucmp.b.256(<32 x i8> %a0, <32 x i8> %a1, i8 7, i32 %mask)
+  %res7 = call i32 @llvm.x86.avx512.mask.ucmp.b.256(<32 x i8> %a0, <32 x i8> %a1, i32 7, i32 %mask)
   %vec7 = insertelement <8 x i32> %vec6, i32 %res7, i32 7
   ret <8 x i32> %vec7
 }
 
-declare i32 @llvm.x86.avx512.mask.ucmp.b.256(<32 x i8>, <32 x i8>, i8, i32) nounwind readnone
+declare i32 @llvm.x86.avx512.mask.ucmp.b.256(<32 x i8>, <32 x i8>, i32, i32) nounwind readnone
 
 define <8 x i16> @test_cmp_w_256(<16 x i16> %a0, <16 x i16> %a1) {
 ; CHECK_LABEL: test_cmp_w_256
 ; CHECK: vpcmpeqw %ymm1, %ymm0, %k0 ##
-  %res0 = call i16 @llvm.x86.avx512.mask.cmp.w.256(<16 x i16> %a0, <16 x i16> %a1, i8 0, i16 -1)
+  %res0 = call i16 @llvm.x86.avx512.mask.cmp.w.256(<16 x i16> %a0, <16 x i16> %a1, i32 0, i16 -1)
   %vec0 = insertelement <8 x i16> undef, i16 %res0, i32 0
 ; CHECK: vpcmpltw %ymm1, %ymm0, %k0 ##
-  %res1 = call i16 @llvm.x86.avx512.mask.cmp.w.256(<16 x i16> %a0, <16 x i16> %a1, i8 1, i16 -1)
+  %res1 = call i16 @llvm.x86.avx512.mask.cmp.w.256(<16 x i16> %a0, <16 x i16> %a1, i32 1, i16 -1)
   %vec1 = insertelement <8 x i16> %vec0, i16 %res1, i32 1
 ; CHECK: vpcmplew %ymm1, %ymm0, %k0 ##
-  %res2 = call i16 @llvm.x86.avx512.mask.cmp.w.256(<16 x i16> %a0, <16 x i16> %a1, i8 2, i16 -1)
+  %res2 = call i16 @llvm.x86.avx512.mask.cmp.w.256(<16 x i16> %a0, <16 x i16> %a1, i32 2, i16 -1)
   %vec2 = insertelement <8 x i16> %vec1, i16 %res2, i32 2
 ; CHECK: vpcmpunordw %ymm1, %ymm0, %k0 ##
-  %res3 = call i16 @llvm.x86.avx512.mask.cmp.w.256(<16 x i16> %a0, <16 x i16> %a1, i8 3, i16 -1)
+  %res3 = call i16 @llvm.x86.avx512.mask.cmp.w.256(<16 x i16> %a0, <16 x i16> %a1, i32 3, i16 -1)
   %vec3 = insertelement <8 x i16> %vec2, i16 %res3, i32 3
 ; CHECK: vpcmpneqw %ymm1, %ymm0, %k0 ##
-  %res4 = call i16 @llvm.x86.avx512.mask.cmp.w.256(<16 x i16> %a0, <16 x i16> %a1, i8 4, i16 -1)
+  %res4 = call i16 @llvm.x86.avx512.mask.cmp.w.256(<16 x i16> %a0, <16 x i16> %a1, i32 4, i16 -1)
   %vec4 = insertelement <8 x i16> %vec3, i16 %res4, i32 4
 ; CHECK: vpcmpnltw %ymm1, %ymm0, %k0 ##
-  %res5 = call i16 @llvm.x86.avx512.mask.cmp.w.256(<16 x i16> %a0, <16 x i16> %a1, i8 5, i16 -1)
+  %res5 = call i16 @llvm.x86.avx512.mask.cmp.w.256(<16 x i16> %a0, <16 x i16> %a1, i32 5, i16 -1)
   %vec5 = insertelement <8 x i16> %vec4, i16 %res5, i32 5
 ; CHECK: vpcmpnlew %ymm1, %ymm0, %k0 ##
-  %res6 = call i16 @llvm.x86.avx512.mask.cmp.w.256(<16 x i16> %a0, <16 x i16> %a1, i8 6, i16 -1)
+  %res6 = call i16 @llvm.x86.avx512.mask.cmp.w.256(<16 x i16> %a0, <16 x i16> %a1, i32 6, i16 -1)
   %vec6 = insertelement <8 x i16> %vec5, i16 %res6, i32 6
 ; CHECK: vpcmpordw %ymm1, %ymm0, %k0 ##
-  %res7 = call i16 @llvm.x86.avx512.mask.cmp.w.256(<16 x i16> %a0, <16 x i16> %a1, i8 7, i16 -1)
+  %res7 = call i16 @llvm.x86.avx512.mask.cmp.w.256(<16 x i16> %a0, <16 x i16> %a1, i32 7, i16 -1)
   %vec7 = insertelement <8 x i16> %vec6, i16 %res7, i32 7
   ret <8 x i16> %vec7
 }
@@ -218,59 +218,59 @@ define <8 x i16> @test_cmp_w_256(<16 x i16> %a0, <16 x i16> %a1) {
 define <8 x i16> @test_mask_cmp_w_256(<16 x i16> %a0, <16 x i16> %a1, i16 %mask) {
 ; CHECK_LABEL: test_mask_cmp_w_256
 ; CHECK: vpcmpeqw %ymm1, %ymm0, %k0 {%k1} ##
-  %res0 = call i16 @llvm.x86.avx512.mask.cmp.w.256(<16 x i16> %a0, <16 x i16> %a1, i8 0, i16 %mask)
+  %res0 = call i16 @llvm.x86.avx512.mask.cmp.w.256(<16 x i16> %a0, <16 x i16> %a1, i32 0, i16 %mask)
   %vec0 = insertelement <8 x i16> undef, i16 %res0, i32 0
 ; CHECK: vpcmpltw %ymm1, %ymm0, %k0 {%k1} ##
-  %res1 = call i16 @llvm.x86.avx512.mask.cmp.w.256(<16 x i16> %a0, <16 x i16> %a1, i8 1, i16 %mask)
+  %res1 = call i16 @llvm.x86.avx512.mask.cmp.w.256(<16 x i16> %a0, <16 x i16> %a1, i32 1, i16 %mask)
   %vec1 = insertelement <8 x i16> %vec0, i16 %res1, i32 1
 ; CHECK: vpcmplew %ymm1, %ymm0, %k0 {%k1} ##
-  %res2 = call i16 @llvm.x86.avx512.mask.cmp.w.256(<16 x i16> %a0, <16 x i16> %a1, i8 2, i16 %mask)
+  %res2 = call i16 @llvm.x86.avx512.mask.cmp.w.256(<16 x i16> %a0, <16 x i16> %a1, i32 2, i16 %mask)
   %vec2 = insertelement <8 x i16> %vec1, i16 %res2, i32 2
 ; CHECK: vpcmpunordw %ymm1, %ymm0, %k0 {%k1} ##
-  %res3 = call i16 @llvm.x86.avx512.mask.cmp.w.256(<16 x i16> %a0, <16 x i16> %a1, i8 3, i16 %mask)
+  %res3 = call i16 @llvm.x86.avx512.mask.cmp.w.256(<16 x i16> %a0, <16 x i16> %a1, i32 3, i16 %mask)
   %vec3 = insertelement <8 x i16> %vec2, i16 %res3, i32 3
 ; CHECK: vpcmpneqw %ymm1, %ymm0, %k0 {%k1} ##
-  %res4 = call i16 @llvm.x86.avx512.mask.cmp.w.256(<16 x i16> %a0, <16 x i16> %a1, i8 4, i16 %mask)
+  %res4 = call i16 @llvm.x86.avx512.mask.cmp.w.256(<16 x i16> %a0, <16 x i16> %a1, i32 4, i16 %mask)
   %vec4 = insertelement <8 x i16> %vec3, i16 %res4, i32 4
 ; CHECK: vpcmpnltw %ymm1, %ymm0, %k0 {%k1} ##
-  %res5 = call i16 @llvm.x86.avx512.mask.cmp.w.256(<16 x i16> %a0, <16 x i16> %a1, i8 5, i16 %mask)
+  %res5 = call i16 @llvm.x86.avx512.mask.cmp.w.256(<16 x i16> %a0, <16 x i16> %a1, i32 5, i16 %mask)
   %vec5 = insertelement <8 x i16> %vec4, i16 %res5, i32 5
 ; CHECK: vpcmpnlew %ymm1, %ymm0, %k0 {%k1} ##
-  %res6 = call i16 @llvm.x86.avx512.mask.cmp.w.256(<16 x i16> %a0, <16 x i16> %a1, i8 6, i16 %mask)
+  %res6 = call i16 @llvm.x86.avx512.mask.cmp.w.256(<16 x i16> %a0, <16 x i16> %a1, i32 6, i16 %mask)
   %vec6 = insertelement <8 x i16> %vec5, i16 %res6, i32 6
 ; CHECK: vpcmpordw %ymm1, %ymm0, %k0 {%k1} ##
-  %res7 = call i16 @llvm.x86.avx512.mask.cmp.w.256(<16 x i16> %a0, <16 x i16> %a1, i8 7, i16 %mask)
+  %res7 = call i16 @llvm.x86.avx512.mask.cmp.w.256(<16 x i16> %a0, <16 x i16> %a1, i32 7, i16 %mask)
   %vec7 = insertelement <8 x i16> %vec6, i16 %res7, i32 7
   ret <8 x i16> %vec7
 }
 
-declare i16 @llvm.x86.avx512.mask.cmp.w.256(<16 x i16>, <16 x i16>, i8, i16) nounwind readnone
+declare i16 @llvm.x86.avx512.mask.cmp.w.256(<16 x i16>, <16 x i16>, i32, i16) nounwind readnone
 
 define <8 x i16> @test_ucmp_w_256(<16 x i16> %a0, <16 x i16> %a1) {
 ; CHECK_LABEL: test_ucmp_w_256
 ; CHECK: vpcmpequw %ymm1, %ymm0, %k0 ##
-  %res0 = call i16 @llvm.x86.avx512.mask.ucmp.w.256(<16 x i16> %a0, <16 x i16> %a1, i8 0, i16 -1)
+  %res0 = call i16 @llvm.x86.avx512.mask.ucmp.w.256(<16 x i16> %a0, <16 x i16> %a1, i32 0, i16 -1)
   %vec0 = insertelement <8 x i16> undef, i16 %res0, i32 0
 ; CHECK: vpcmpltuw %ymm1, %ymm0, %k0 ##
-  %res1 = call i16 @llvm.x86.avx512.mask.ucmp.w.256(<16 x i16> %a0, <16 x i16> %a1, i8 1, i16 -1)
+  %res1 = call i16 @llvm.x86.avx512.mask.ucmp.w.256(<16 x i16> %a0, <16 x i16> %a1, i32 1, i16 -1)
   %vec1 = insertelement <8 x i16> %vec0, i16 %res1, i32 1
 ; CHECK: vpcmpleuw %ymm1, %ymm0, %k0 ##
-  %res2 = call i16 @llvm.x86.avx512.mask.ucmp.w.256(<16 x i16> %a0, <16 x i16> %a1, i8 2, i16 -1)
+  %res2 = call i16 @llvm.x86.avx512.mask.ucmp.w.256(<16 x i16> %a0, <16 x i16> %a1, i32 2, i16 -1)
   %vec2 = insertelement <8 x i16> %vec1, i16 %res2, i32 2
 ; CHECK: vpcmpunorduw %ymm1, %ymm0, %k0 ##
-  %res3 = call i16 @llvm.x86.avx512.mask.ucmp.w.256(<16 x i16> %a0, <16 x i16> %a1, i8 3, i16 -1)
+  %res3 = call i16 @llvm.x86.avx512.mask.ucmp.w.256(<16 x i16> %a0, <16 x i16> %a1, i32 3, i16 -1)
   %vec3 = insertelement <8 x i16> %vec2, i16 %res3, i32 3
 ; CHECK: vpcmpnequw %ymm1, %ymm0, %k0 ##
-  %res4 = call i16 @llvm.x86.avx512.mask.ucmp.w.256(<16 x i16> %a0, <16 x i16> %a1, i8 4, i16 -1)
+  %res4 = call i16 @llvm.x86.avx512.mask.ucmp.w.256(<16 x i16> %a0, <16 x i16> %a1, i32 4, i16 -1)
   %vec4 = insertelement <8 x i16> %vec3, i16 %res4, i32 4
 ; CHECK: vpcmpnltuw %ymm1, %ymm0, %k0 ##
-  %res5 = call i16 @llvm.x86.avx512.mask.ucmp.w.256(<16 x i16> %a0, <16 x i16> %a1, i8 5, i16 -1)
+  %res5 = call i16 @llvm.x86.avx512.mask.ucmp.w.256(<16 x i16> %a0, <16 x i16> %a1, i32 5, i16 -1)
   %vec5 = insertelement <8 x i16> %vec4, i16 %res5, i32 5
 ; CHECK: vpcmpnleuw %ymm1, %ymm0, %k0 ##
-  %res6 = call i16 @llvm.x86.avx512.mask.ucmp.w.256(<16 x i16> %a0, <16 x i16> %a1, i8 6, i16 -1)
+  %res6 = call i16 @llvm.x86.avx512.mask.ucmp.w.256(<16 x i16> %a0, <16 x i16> %a1, i32 6, i16 -1)
   %vec6 = insertelement <8 x i16> %vec5, i16 %res6, i32 6
 ; CHECK: vpcmporduw %ymm1, %ymm0, %k0 ##
-  %res7 = call i16 @llvm.x86.avx512.mask.ucmp.w.256(<16 x i16> %a0, <16 x i16> %a1, i8 7, i16 -1)
+  %res7 = call i16 @llvm.x86.avx512.mask.ucmp.w.256(<16 x i16> %a0, <16 x i16> %a1, i32 7, i16 -1)
   %vec7 = insertelement <8 x i16> %vec6, i16 %res7, i32 7
   ret <8 x i16> %vec7
 }
@@ -278,33 +278,33 @@ define <8 x i16> @test_ucmp_w_256(<16 x i16> %a0, <16 x i16> %a1) {
 define <8 x i16> @test_mask_ucmp_w_256(<16 x i16> %a0, <16 x i16> %a1, i16 %mask) {
 ; CHECK_LABEL: test_mask_ucmp_w_256
 ; CHECK: vpcmpequw %ymm1, %ymm0, %k0 {%k1} ##
-  %res0 = call i16 @llvm.x86.avx512.mask.ucmp.w.256(<16 x i16> %a0, <16 x i16> %a1, i8 0, i16 %mask)
+  %res0 = call i16 @llvm.x86.avx512.mask.ucmp.w.256(<16 x i16> %a0, <16 x i16> %a1, i32 0, i16 %mask)
   %vec0 = insertelement <8 x i16> undef, i16 %res0, i32 0
 ; CHECK: vpcmpltuw %ymm1, %ymm0, %k0 {%k1} ##
-  %res1 = call i16 @llvm.x86.avx512.mask.ucmp.w.256(<16 x i16> %a0, <16 x i16> %a1, i8 1, i16 %mask)
+  %res1 = call i16 @llvm.x86.avx512.mask.ucmp.w.256(<16 x i16> %a0, <16 x i16> %a1, i32 1, i16 %mask)
   %vec1 = insertelement <8 x i16> %vec0, i16 %res1, i32 1
 ; CHECK: vpcmpleuw %ymm1, %ymm0, %k0 {%k1} ##
-  %res2 = call i16 @llvm.x86.avx512.mask.ucmp.w.256(<16 x i16> %a0, <16 x i16> %a1, i8 2, i16 %mask)
+  %res2 = call i16 @llvm.x86.avx512.mask.ucmp.w.256(<16 x i16> %a0, <16 x i16> %a1, i32 2, i16 %mask)
   %vec2 = insertelement <8 x i16> %vec1, i16 %res2, i32 2
 ; CHECK: vpcmpunorduw %ymm1, %ymm0, %k0 {%k1} ##
-  %res3 = call i16 @llvm.x86.avx512.mask.ucmp.w.256(<16 x i16> %a0, <16 x i16> %a1, i8 3, i16 %mask)
+  %res3 = call i16 @llvm.x86.avx512.mask.ucmp.w.256(<16 x i16> %a0, <16 x i16> %a1, i32 3, i16 %mask)
   %vec3 = insertelement <8 x i16> %vec2, i16 %res3, i32 3
 ; CHECK: vpcmpnequw %ymm1, %ymm0, %k0 {%k1} ##
-  %res4 = call i16 @llvm.x86.avx512.mask.ucmp.w.256(<16 x i16> %a0, <16 x i16> %a1, i8 4, i16 %mask)
+  %res4 = call i16 @llvm.x86.avx512.mask.ucmp.w.256(<16 x i16> %a0, <16 x i16> %a1, i32 4, i16 %mask)
   %vec4 = insertelement <8 x i16> %vec3, i16 %res4, i32 4
 ; CHECK: vpcmpnltuw %ymm1, %ymm0, %k0 {%k1} ##
-  %res5 = call i16 @llvm.x86.avx512.mask.ucmp.w.256(<16 x i16> %a0, <16 x i16> %a1, i8 5, i16 %mask)
+  %res5 = call i16 @llvm.x86.avx512.mask.ucmp.w.256(<16 x i16> %a0, <16 x i16> %a1, i32 5, i16 %mask)
   %vec5 = insertelement <8 x i16> %vec4, i16 %res5, i32 5
 ; CHECK: vpcmpnleuw %ymm1, %ymm0, %k0 {%k1} ##
-  %res6 = call i16 @llvm.x86.avx512.mask.ucmp.w.256(<16 x i16> %a0, <16 x i16> %a1, i8 6, i16 %mask)
+  %res6 = call i16 @llvm.x86.avx512.mask.ucmp.w.256(<16 x i16> %a0, <16 x i16> %a1, i32 6, i16 %mask)
   %vec6 = insertelement <8 x i16> %vec5, i16 %res6, i32 6
 ; CHECK: vpcmporduw %ymm1, %ymm0, %k0 {%k1} ##
-  %res7 = call i16 @llvm.x86.avx512.mask.ucmp.w.256(<16 x i16> %a0, <16 x i16> %a1, i8 7, i16 %mask)
+  %res7 = call i16 @llvm.x86.avx512.mask.ucmp.w.256(<16 x i16> %a0, <16 x i16> %a1, i32 7, i16 %mask)
   %vec7 = insertelement <8 x i16> %vec6, i16 %res7, i32 7
   ret <8 x i16> %vec7
 }
 
-declare i16 @llvm.x86.avx512.mask.ucmp.w.256(<16 x i16>, <16 x i16>, i8, i16) nounwind readnone
+declare i16 @llvm.x86.avx512.mask.ucmp.w.256(<16 x i16>, <16 x i16>, i32, i16) nounwind readnone
 
 ; 128-bit
 
@@ -375,28 +375,28 @@ declare i8 @llvm.x86.avx512.mask.pcmpgt.w.128(<8 x i16>, <8 x i16>, i8)
 define <8 x i16> @test_cmp_b_128(<16 x i8> %a0, <16 x i8> %a1) {
 ; CHECK_LABEL: test_cmp_b_128
 ; CHECK: vpcmpeqb %xmm1, %xmm0, %k0 ##
-  %res0 = call i16 @llvm.x86.avx512.mask.cmp.b.128(<16 x i8> %a0, <16 x i8> %a1, i8 0, i16 -1)
+  %res0 = call i16 @llvm.x86.avx512.mask.cmp.b.128(<16 x i8> %a0, <16 x i8> %a1, i32 0, i16 -1)
   %vec0 = insertelement <8 x i16> undef, i16 %res0, i32 0
 ; CHECK: vpcmpltb %xmm1, %xmm0, %k0 ##
-  %res1 = call i16 @llvm.x86.avx512.mask.cmp.b.128(<16 x i8> %a0, <16 x i8> %a1, i8 1, i16 -1)
+  %res1 = call i16 @llvm.x86.avx512.mask.cmp.b.128(<16 x i8> %a0, <16 x i8> %a1, i32 1, i16 -1)
   %vec1 = insertelement <8 x i16> %vec0, i16 %res1, i32 1
 ; CHECK: vpcmpleb %xmm1, %xmm0, %k0 ##
-  %res2 = call i16 @llvm.x86.avx512.mask.cmp.b.128(<16 x i8> %a0, <16 x i8> %a1, i8 2, i16 -1)
+  %res2 = call i16 @llvm.x86.avx512.mask.cmp.b.128(<16 x i8> %a0, <16 x i8> %a1, i32 2, i16 -1)
   %vec2 = insertelement <8 x i16> %vec1, i16 %res2, i32 2
 ; CHECK: vpcmpunordb %xmm1, %xmm0, %k0 ##
-  %res3 = call i16 @llvm.x86.avx512.mask.cmp.b.128(<16 x i8> %a0, <16 x i8> %a1, i8 3, i16 -1)
+  %res3 = call i16 @llvm.x86.avx512.mask.cmp.b.128(<16 x i8> %a0, <16 x i8> %a1, i32 3, i16 -1)
   %vec3 = insertelement <8 x i16> %vec2, i16 %res3, i32 3
 ; CHECK: vpcmpneqb %xmm1, %xmm0, %k0 ##
-  %res4 = call i16 @llvm.x86.avx512.mask.cmp.b.128(<16 x i8> %a0, <16 x i8> %a1, i8 4, i16 -1)
+  %res4 = call i16 @llvm.x86.avx512.mask.cmp.b.128(<16 x i8> %a0, <16 x i8> %a1, i32 4, i16 -1)
   %vec4 = insertelement <8 x i16> %vec3, i16 %res4, i32 4
 ; CHECK: vpcmpnltb %xmm1, %xmm0, %k0 ##
-  %res5 = call i16 @llvm.x86.avx512.mask.cmp.b.128(<16 x i8> %a0, <16 x i8> %a1, i8 5, i16 -1)
+  %res5 = call i16 @llvm.x86.avx512.mask.cmp.b.128(<16 x i8> %a0, <16 x i8> %a1, i32 5, i16 -1)
   %vec5 = insertelement <8 x i16> %vec4, i16 %res5, i32 5
 ; CHECK: vpcmpnleb %xmm1, %xmm0, %k0 ##
-  %res6 = call i16 @llvm.x86.avx512.mask.cmp.b.128(<16 x i8> %a0, <16 x i8> %a1, i8 6, i16 -1)
+  %res6 = call i16 @llvm.x86.avx512.mask.cmp.b.128(<16 x i8> %a0, <16 x i8> %a1, i32 6, i16 -1)
   %vec6 = insertelement <8 x i16> %vec5, i16 %res6, i32 6
 ; CHECK: vpcmpordb %xmm1, %xmm0, %k0 ##
-  %res7 = call i16 @llvm.x86.avx512.mask.cmp.b.128(<16 x i8> %a0, <16 x i8> %a1, i8 7, i16 -1)
+  %res7 = call i16 @llvm.x86.avx512.mask.cmp.b.128(<16 x i8> %a0, <16 x i8> %a1, i32 7, i16 -1)
   %vec7 = insertelement <8 x i16> %vec6, i16 %res7, i32 7
   ret <8 x i16> %vec7
 }
@@ -404,59 +404,59 @@ define <8 x i16> @test_cmp_b_128(<16 x i8> %a0, <16 x i8> %a1) {
 define <8 x i16> @test_mask_cmp_b_128(<16 x i8> %a0, <16 x i8> %a1, i16 %mask) {
 ; CHECK_LABEL: test_mask_cmp_b_128
 ; CHECK: vpcmpeqb %xmm1, %xmm0, %k0 {%k1} ##
-  %res0 = call i16 @llvm.x86.avx512.mask.cmp.b.128(<16 x i8> %a0, <16 x i8> %a1, i8 0, i16 %mask)
+  %res0 = call i16 @llvm.x86.avx512.mask.cmp.b.128(<16 x i8> %a0, <16 x i8> %a1, i32 0, i16 %mask)
   %vec0 = insertelement <8 x i16> undef, i16 %res0, i32 0
 ; CHECK: vpcmpltb %xmm1, %xmm0, %k0 {%k1} ##
-  %res1 = call i16 @llvm.x86.avx512.mask.cmp.b.128(<16 x i8> %a0, <16 x i8> %a1, i8 1, i16 %mask)
+  %res1 = call i16 @llvm.x86.avx512.mask.cmp.b.128(<16 x i8> %a0, <16 x i8> %a1, i32 1, i16 %mask)
   %vec1 = insertelement <8 x i16> %vec0, i16 %res1, i32 1
 ; CHECK: vpcmpleb %xmm1, %xmm0, %k0 {%k1} ##
-  %res2 = call i16 @llvm.x86.avx512.mask.cmp.b.128(<16 x i8> %a0, <16 x i8> %a1, i8 2, i16 %mask)
+  %res2 = call i16 @llvm.x86.avx512.mask.cmp.b.128(<16 x i8> %a0, <16 x i8> %a1, i32 2, i16 %mask)
   %vec2 = insertelement <8 x i16> %vec1, i16 %res2, i32 2
 ; CHECK: vpcmpunordb %xmm1, %xmm0, %k0 {%k1} ##
-  %res3 = call i16 @llvm.x86.avx512.mask.cmp.b.128(<16 x i8> %a0, <16 x i8> %a1, i8 3, i16 %mask)
+  %res3 = call i16 @llvm.x86.avx512.mask.cmp.b.128(<16 x i8> %a0, <16 x i8> %a1, i32 3, i16 %mask)
   %vec3 = insertelement <8 x i16> %vec2, i16 %res3, i32 3
 ; CHECK: vpcmpneqb %xmm1, %xmm0, %k0 {%k1} ##
-  %res4 = call i16 @llvm.x86.avx512.mask.cmp.b.128(<16 x i8> %a0, <16 x i8> %a1, i8 4, i16 %mask)
+  %res4 = call i16 @llvm.x86.avx512.mask.cmp.b.128(<16 x i8> %a0, <16 x i8> %a1, i32 4, i16 %mask)
   %vec4 = insertelement <8 x i16> %vec3, i16 %res4, i32 4
 ; CHECK: vpcmpnltb %xmm1, %xmm0, %k0 {%k1} ##
-  %res5 = call i16 @llvm.x86.avx512.mask.cmp.b.128(<16 x i8> %a0, <16 x i8> %a1, i8 5, i16 %mask)
+  %res5 = call i16 @llvm.x86.avx512.mask.cmp.b.128(<16 x i8> %a0, <16 x i8> %a1, i32 5, i16 %mask)
   %vec5 = insertelement <8 x i16> %vec4, i16 %res5, i32 5
 ; CHECK: vpcmpnleb %xmm1, %xmm0, %k0 {%k1} ##
-  %res6 = call i16 @llvm.x86.avx512.mask.cmp.b.128(<16 x i8> %a0, <16 x i8> %a1, i8 6, i16 %mask)
+  %res6 = call i16 @llvm.x86.avx512.mask.cmp.b.128(<16 x i8> %a0, <16 x i8> %a1, i32 6, i16 %mask)
   %vec6 = insertelement <8 x i16> %vec5, i16 %res6, i32 6
 ; CHECK: vpcmpordb %xmm1, %xmm0, %k0 {%k1} ##
-  %res7 = call i16 @llvm.x86.avx512.mask.cmp.b.128(<16 x i8> %a0, <16 x i8> %a1, i8 7, i16 %mask)
+  %res7 = call i16 @llvm.x86.avx512.mask.cmp.b.128(<16 x i8> %a0, <16 x i8> %a1, i32 7, i16 %mask)
   %vec7 = insertelement <8 x i16> %vec6, i16 %res7, i32 7
   ret <8 x i16> %vec7
 }
 
-declare i16 @llvm.x86.avx512.mask.cmp.b.128(<16 x i8>, <16 x i8>, i8, i16) nounwind readnone
+declare i16 @llvm.x86.avx512.mask.cmp.b.128(<16 x i8>, <16 x i8>, i32, i16) nounwind readnone
 
 define <8 x i16> @test_ucmp_b_128(<16 x i8> %a0, <16 x i8> %a1) {
 ; CHECK_LABEL: test_ucmp_b_128
 ; CHECK: vpcmpequb %xmm1, %xmm0, %k0 ##
-  %res0 = call i16 @llvm.x86.avx512.mask.ucmp.b.128(<16 x i8> %a0, <16 x i8> %a1, i8 0, i16 -1)
+  %res0 = call i16 @llvm.x86.avx512.mask.ucmp.b.128(<16 x i8> %a0, <16 x i8> %a1, i32 0, i16 -1)
   %vec0 = insertelement <8 x i16> undef, i16 %res0, i32 0
 ; CHECK: vpcmpltub %xmm1, %xmm0, %k0 ##
-  %res1 = call i16 @llvm.x86.avx512.mask.ucmp.b.128(<16 x i8> %a0, <16 x i8> %a1, i8 1, i16 -1)
+  %res1 = call i16 @llvm.x86.avx512.mask.ucmp.b.128(<16 x i8> %a0, <16 x i8> %a1, i32 1, i16 -1)
   %vec1 = insertelement <8 x i16> %vec0, i16 %res1, i32 1
 ; CHECK: vpcmpleub %xmm1, %xmm0, %k0 ##
-  %res2 = call i16 @llvm.x86.avx512.mask.ucmp.b.128(<16 x i8> %a0, <16 x i8> %a1, i8 2, i16 -1)
+  %res2 = call i16 @llvm.x86.avx512.mask.ucmp.b.128(<16 x i8> %a0, <16 x i8> %a1, i32 2, i16 -1)
   %vec2 = insertelement <8 x i16> %vec1, i16 %res2, i32 2
 ; CHECK: vpcmpunordub %xmm1, %xmm0, %k0 ##
-  %res3 = call i16 @llvm.x86.avx512.mask.ucmp.b.128(<16 x i8> %a0, <16 x i8> %a1, i8 3, i16 -1)
+  %res3 = call i16 @llvm.x86.avx512.mask.ucmp.b.128(<16 x i8> %a0, <16 x i8> %a1, i32 3, i16 -1)
   %vec3 = insertelement <8 x i16> %vec2, i16 %res3, i32 3
 ; CHECK: vpcmpnequb %xmm1, %xmm0, %k0 ##
-  %res4 = call i16 @llvm.x86.avx512.mask.ucmp.b.128(<16 x i8> %a0, <16 x i8> %a1, i8 4, i16 -1)
+  %res4 = call i16 @llvm.x86.avx512.mask.ucmp.b.128(<16 x i8> %a0, <16 x i8> %a1, i32 4, i16 -1)
   %vec4 = insertelement <8 x i16> %vec3, i16 %res4, i32 4
 ; CHECK: vpcmpnltub %xmm1, %xmm0, %k0 ##
-  %res5 = call i16 @llvm.x86.avx512.mask.ucmp.b.128(<16 x i8> %a0, <16 x i8> %a1, i8 5, i16 -1)
+  %res5 = call i16 @llvm.x86.avx512.mask.ucmp.b.128(<16 x i8> %a0, <16 x i8> %a1, i32 5, i16 -1)
   %vec5 = insertelement <8 x i16> %vec4, i16 %res5, i32 5
 ; CHECK: vpcmpnleub %xmm1, %xmm0, %k0 ##
-  %res6 = call i16 @llvm.x86.avx512.mask.ucmp.b.128(<16 x i8> %a0, <16 x i8> %a1, i8 6, i16 -1)
+  %res6 = call i16 @llvm.x86.avx512.mask.ucmp.b.128(<16 x i8> %a0, <16 x i8> %a1, i32 6, i16 -1)
   %vec6 = insertelement <8 x i16> %vec5, i16 %res6, i32 6
 ; CHECK: vpcmpordub %xmm1, %xmm0, %k0 ##
-  %res7 = call i16 @llvm.x86.avx512.mask.ucmp.b.128(<16 x i8> %a0, <16 x i8> %a1, i8 7, i16 -1)
+  %res7 = call i16 @llvm.x86.avx512.mask.ucmp.b.128(<16 x i8> %a0, <16 x i8> %a1, i32 7, i16 -1)
   %vec7 = insertelement <8 x i16> %vec6, i16 %res7, i32 7
   ret <8 x i16> %vec7
 }
@@ -464,59 +464,59 @@ define <8 x i16> @test_ucmp_b_128(<16 x i8> %a0, <16 x i8> %a1) {
 define <8 x i16> @test_mask_ucmp_b_128(<16 x i8> %a0, <16 x i8> %a1, i16 %mask) {
 ; CHECK_LABEL: test_mask_ucmp_b_128
 ; CHECK: vpcmpequb %xmm1, %xmm0, %k0 {%k1} ##
-  %res0 = call i16 @llvm.x86.avx512.mask.ucmp.b.128(<16 x i8> %a0, <16 x i8> %a1, i8 0, i16 %mask)
+  %res0 = call i16 @llvm.x86.avx512.mask.ucmp.b.128(<16 x i8> %a0, <16 x i8> %a1, i32 0, i16 %mask)
   %vec0 = insertelement <8 x i16> undef, i16 %res0, i32 0
 ; CHECK: vpcmpltub %xmm1, %xmm0, %k0 {%k1} ##
-  %res1 = call i16 @llvm.x86.avx512.mask.ucmp.b.128(<16 x i8> %a0, <16 x i8> %a1, i8 1, i16 %mask)
+  %res1 = call i16 @llvm.x86.avx512.mask.ucmp.b.128(<16 x i8> %a0, <16 x i8> %a1, i32 1, i16 %mask)
   %vec1 = insertelement <8 x i16> %vec0, i16 %res1, i32 1
 ; CHECK: vpcmpleub %xmm1, %xmm0, %k0 {%k1} ##
-  %res2 = call i16 @llvm.x86.avx512.mask.ucmp.b.128(<16 x i8> %a0, <16 x i8> %a1, i8 2, i16 %mask)
+  %res2 = call i16 @llvm.x86.avx512.mask.ucmp.b.128(<16 x i8> %a0, <16 x i8> %a1, i32 2, i16 %mask)
   %vec2 = insertelement <8 x i16> %vec1, i16 %res2, i32 2
 ; CHECK: vpcmpunordub %xmm1, %xmm0, %k0 {%k1} ##
-  %res3 = call i16 @llvm.x86.avx512.mask.ucmp.b.128(<16 x i8> %a0, <16 x i8> %a1, i8 3, i16 %mask)
+  %res3 = call i16 @llvm.x86.avx512.mask.ucmp.b.128(<16 x i8> %a0, <16 x i8> %a1, i32 3, i16 %mask)
   %vec3 = insertelement <8 x i16> %vec2, i16 %res3, i32 3
 ; CHECK: vpcmpnequb %xmm1, %xmm0, %k0 {%k1} ##
-  %res4 = call i16 @llvm.x86.avx512.mask.ucmp.b.128(<16 x i8> %a0, <16 x i8> %a1, i8 4, i16 %mask)
+  %res4 = call i16 @llvm.x86.avx512.mask.ucmp.b.128(<16 x i8> %a0, <16 x i8> %a1, i32 4, i16 %mask)
   %vec4 = insertelement <8 x i16> %vec3, i16 %res4, i32 4
 ; CHECK: vpcmpnltub %xmm1, %xmm0, %k0 {%k1} ##
-  %res5 = call i16 @llvm.x86.avx512.mask.ucmp.b.128(<16 x i8> %a0, <16 x i8> %a1, i8 5, i16 %mask)
+  %res5 = call i16 @llvm.x86.avx512.mask.ucmp.b.128(<16 x i8> %a0, <16 x i8> %a1, i32 5, i16 %mask)
   %vec5 = insertelement <8 x i16> %vec4, i16 %res5, i32 5
 ; CHECK: vpcmpnleub %xmm1, %xmm0, %k0 {%k1} ##
-  %res6 = call i16 @llvm.x86.avx512.mask.ucmp.b.128(<16 x i8> %a0, <16 x i8> %a1, i8 6, i16 %mask)
+  %res6 = call i16 @llvm.x86.avx512.mask.ucmp.b.128(<16 x i8> %a0, <16 x i8> %a1, i32 6, i16 %mask)
   %vec6 = insertelement <8 x i16> %vec5, i16 %res6, i32 6
 ; CHECK: vpcmpordub %xmm1, %xmm0, %k0 {%k1} ##
-  %res7 = call i16 @llvm.x86.avx512.mask.ucmp.b.128(<16 x i8> %a0, <16 x i8> %a1, i8 7, i16 %mask)
+  %res7 = call i16 @llvm.x86.avx512.mask.ucmp.b.128(<16 x i8> %a0, <16 x i8> %a1, i32 7, i16 %mask)
   %vec7 = insertelement <8 x i16> %vec6, i16 %res7, i32 7
   ret <8 x i16> %vec7
 }
 
-declare i16 @llvm.x86.avx512.mask.ucmp.b.128(<16 x i8>, <16 x i8>, i8, i16) nounwind readnone
+declare i16 @llvm.x86.avx512.mask.ucmp.b.128(<16 x i8>, <16 x i8>, i32, i16) nounwind readnone
 
 define <8 x i8> @test_cmp_w_128(<8 x i16> %a0, <8 x i16> %a1) {
 ; CHECK_LABEL: test_cmp_w_128
 ; CHECK: vpcmpeqw %xmm1, %xmm0, %k0 ##
-  %res0 = call i8 @llvm.x86.avx512.mask.cmp.w.128(<8 x i16> %a0, <8 x i16> %a1, i8 0, i8 -1)
+  %res0 = call i8 @llvm.x86.avx512.mask.cmp.w.128(<8 x i16> %a0, <8 x i16> %a1, i32 0, i8 -1)
   %vec0 = insertelement <8 x i8> undef, i8 %res0, i32 0
 ; CHECK: vpcmpltw %xmm1, %xmm0, %k0 ##
-  %res1 = call i8 @llvm.x86.avx512.mask.cmp.w.128(<8 x i16> %a0, <8 x i16> %a1, i8 1, i8 -1)
+  %res1 = call i8 @llvm.x86.avx512.mask.cmp.w.128(<8 x i16> %a0, <8 x i16> %a1, i32 1, i8 -1)
   %vec1 = insertelement <8 x i8> %vec0, i8 %res1, i32 1
 ; CHECK: vpcmplew %xmm1, %xmm0, %k0 ##
-  %res2 = call i8 @llvm.x86.avx512.mask.cmp.w.128(<8 x i16> %a0, <8 x i16> %a1, i8 2, i8 -1)
+  %res2 = call i8 @llvm.x86.avx512.mask.cmp.w.128(<8 x i16> %a0, <8 x i16> %a1, i32 2, i8 -1)
   %vec2 = insertelement <8 x i8> %vec1, i8 %res2, i32 2
 ; CHECK: vpcmpunordw %xmm1, %xmm0, %k0 ##
-  %res3 = call i8 @llvm.x86.avx512.mask.cmp.w.128(<8 x i16> %a0, <8 x i16> %a1, i8 3, i8 -1)
+  %res3 = call i8 @llvm.x86.avx512.mask.cmp.w.128(<8 x i16> %a0, <8 x i16> %a1, i32 3, i8 -1)
   %vec3 = insertelement <8 x i8> %vec2, i8 %res3, i32 3
 ; CHECK: vpcmpneqw %xmm1, %xmm0, %k0 ##
-  %res4 = call i8 @llvm.x86.avx512.mask.cmp.w.128(<8 x i16> %a0, <8 x i16> %a1, i8 4, i8 -1)
+  %res4 = call i8 @llvm.x86.avx512.mask.cmp.w.128(<8 x i16> %a0, <8 x i16> %a1, i32 4, i8 -1)
   %vec4 = insertelement <8 x i8> %vec3, i8 %res4, i32 4
 ; CHECK: vpcmpnltw %xmm1, %xmm0, %k0 ##
-  %res5 = call i8 @llvm.x86.avx512.mask.cmp.w.128(<8 x i16> %a0, <8 x i16> %a1, i8 5, i8 -1)
+  %res5 = call i8 @llvm.x86.avx512.mask.cmp.w.128(<8 x i16> %a0, <8 x i16> %a1, i32 5, i8 -1)
   %vec5 = insertelement <8 x i8> %vec4, i8 %res5, i32 5
 ; CHECK: vpcmpnlew %xmm1, %xmm0, %k0 ##
-  %res6 = call i8 @llvm.x86.avx512.mask.cmp.w.128(<8 x i16> %a0, <8 x i16> %a1, i8 6, i8 -1)
+  %res6 = call i8 @llvm.x86.avx512.mask.cmp.w.128(<8 x i16> %a0, <8 x i16> %a1, i32 6, i8 -1)
   %vec6 = insertelement <8 x i8> %vec5, i8 %res6, i32 6
 ; CHECK: vpcmpordw %xmm1, %xmm0, %k0 ##
-  %res7 = call i8 @llvm.x86.avx512.mask.cmp.w.128(<8 x i16> %a0, <8 x i16> %a1, i8 7, i8 -1)
+  %res7 = call i8 @llvm.x86.avx512.mask.cmp.w.128(<8 x i16> %a0, <8 x i16> %a1, i32 7, i8 -1)
   %vec7 = insertelement <8 x i8> %vec6, i8 %res7, i32 7
   ret <8 x i8> %vec7
 }
@@ -524,59 +524,59 @@ define <8 x i8> @test_cmp_w_128(<8 x i16> %a0, <8 x i16> %a1) {
 define <8 x i8> @test_mask_cmp_w_128(<8 x i16> %a0, <8 x i16> %a1, i8 %mask) {
 ; CHECK_LABEL: test_mask_cmp_w_128
 ; CHECK: vpcmpeqw %xmm1, %xmm0, %k0 {%k1} ##
-  %res0 = call i8 @llvm.x86.avx512.mask.cmp.w.128(<8 x i16> %a0, <8 x i16> %a1, i8 0, i8 %mask)
+  %res0 = call i8 @llvm.x86.avx512.mask.cmp.w.128(<8 x i16> %a0, <8 x i16> %a1, i32 0, i8 %mask)
   %vec0 = insertelement <8 x i8> undef, i8 %res0, i32 0
 ; CHECK: vpcmpltw %xmm1, %xmm0, %k0 {%k1} ##
-  %res1 = call i8 @llvm.x86.avx512.mask.cmp.w.128(<8 x i16> %a0, <8 x i16> %a1, i8 1, i8 %mask)
+  %res1 = call i8 @llvm.x86.avx512.mask.cmp.w.128(<8 x i16> %a0, <8 x i16> %a1, i32 1, i8 %mask)
   %vec1 = insertelement <8 x i8> %vec0, i8 %res1, i32 1
 ; CHECK: vpcmplew %xmm1, %xmm0, %k0 {%k1} ##
-  %res2 = call i8 @llvm.x86.avx512.mask.cmp.w.128(<8 x i16> %a0, <8 x i16> %a1, i8 2, i8 %mask)
+  %res2 = call i8 @llvm.x86.avx512.mask.cmp.w.128(<8 x i16> %a0, <8 x i16> %a1, i32 2, i8 %mask)
   %vec2 = insertelement <8 x i8> %vec1, i8 %res2, i32 2
 ; CHECK: vpcmpunordw %xmm1, %xmm0, %k0 {%k1} ##
-  %res3 = call i8 @llvm.x86.avx512.mask.cmp.w.128(<8 x i16> %a0, <8 x i16> %a1, i8 3, i8 %mask)
+  %res3 = call i8 @llvm.x86.avx512.mask.cmp.w.128(<8 x i16> %a0, <8 x i16> %a1, i32 3, i8 %mask)
   %vec3 = insertelement <8 x i8> %vec2, i8 %res3, i32 3
 ; CHECK: vpcmpneqw %xmm1, %xmm0, %k0 {%k1} ##
-  %res4 = call i8 @llvm.x86.avx512.mask.cmp.w.128(<8 x i16> %a0, <8 x i16> %a1, i8 4, i8 %mask)
+  %res4 = call i8 @llvm.x86.avx512.mask.cmp.w.128(<8 x i16> %a0, <8 x i16> %a1, i32 4, i8 %mask)
   %vec4 = insertelement <8 x i8> %vec3, i8 %res4, i32 4
 ; CHECK: vpcmpnltw %xmm1, %xmm0, %k0 {%k1} ##
-  %res5 = call i8 @llvm.x86.avx512.mask.cmp.w.128(<8 x i16> %a0, <8 x i16> %a1, i8 5, i8 %mask)
+  %res5 = call i8 @llvm.x86.avx512.mask.cmp.w.128(<8 x i16> %a0, <8 x i16> %a1, i32 5, i8 %mask)
   %vec5 = insertelement <8 x i8> %vec4, i8 %res5, i32 5
 ; CHECK: vpcmpnlew %xmm1, %xmm0, %k0 {%k1} ##
-  %res6 = call i8 @llvm.x86.avx512.mask.cmp.w.128(<8 x i16> %a0, <8 x i16> %a1, i8 6, i8 %mask)
+  %res6 = call i8 @llvm.x86.avx512.mask.cmp.w.128(<8 x i16> %a0, <8 x i16> %a1, i32 6, i8 %mask)
   %vec6 = insertelement <8 x i8> %vec5, i8 %res6, i32 6
 ; CHECK: vpcmpordw %xmm1, %xmm0, %k0 {%k1} ##
-  %res7 = call i8 @llvm.x86.avx512.mask.cmp.w.128(<8 x i16> %a0, <8 x i16> %a1, i8 7, i8 %mask)
+  %res7 = call i8 @llvm.x86.avx512.mask.cmp.w.128(<8 x i16> %a0, <8 x i16> %a1, i32 7, i8 %mask)
   %vec7 = insertelement <8 x i8> %vec6, i8 %res7, i32 7
   ret <8 x i8> %vec7
 }
 
-declare i8 @llvm.x86.avx512.mask.cmp.w.128(<8 x i16>, <8 x i16>, i8, i8) nounwind readnone
+declare i8 @llvm.x86.avx512.mask.cmp.w.128(<8 x i16>, <8 x i16>, i32, i8) nounwind readnone
 
 define <8 x i8> @test_ucmp_w_128(<8 x i16> %a0, <8 x i16> %a1) {
 ; CHECK_LABEL: test_ucmp_w_128
 ; CHECK: vpcmpequw %xmm1, %xmm0, %k0 ##
-  %res0 = call i8 @llvm.x86.avx512.mask.ucmp.w.128(<8 x i16> %a0, <8 x i16> %a1, i8 0, i8 -1)
+  %res0 = call i8 @llvm.x86.avx512.mask.ucmp.w.128(<8 x i16> %a0, <8 x i16> %a1, i32 0, i8 -1)
   %vec0 = insertelement <8 x i8> undef, i8 %res0, i32 0
 ; CHECK: vpcmpltuw %xmm1, %xmm0, %k0 ##
-  %res1 = call i8 @llvm.x86.avx512.mask.ucmp.w.128(<8 x i16> %a0, <8 x i16> %a1, i8 1, i8 -1)
+  %res1 = call i8 @llvm.x86.avx512.mask.ucmp.w.128(<8 x i16> %a0, <8 x i16> %a1, i32 1, i8 -1)
   %vec1 = insertelement <8 x i8> %vec0, i8 %res1, i32 1
 ; CHECK: vpcmpleuw %xmm1, %xmm0, %k0 ##
-  %res2 = call i8 @llvm.x86.avx512.mask.ucmp.w.128(<8 x i16> %a0, <8 x i16> %a1, i8 2, i8 -1)
+  %res2 = call i8 @llvm.x86.avx512.mask.ucmp.w.128(<8 x i16> %a0, <8 x i16> %a1, i32 2, i8 -1)
   %vec2 = insertelement <8 x i8> %vec1, i8 %res2, i32 2
 ; CHECK: vpcmpunorduw %xmm1, %xmm0, %k0 ##
-  %res3 = call i8 @llvm.x86.avx512.mask.ucmp.w.128(<8 x i16> %a0, <8 x i16> %a1, i8 3, i8 -1)
+  %res3 = call i8 @llvm.x86.avx512.mask.ucmp.w.128(<8 x i16> %a0, <8 x i16> %a1, i32 3, i8 -1)
   %vec3 = insertelement <8 x i8> %vec2, i8 %res3, i32 3
 ; CHECK: vpcmpnequw %xmm1, %xmm0, %k0 ##
-  %res4 = call i8 @llvm.x86.avx512.mask.ucmp.w.128(<8 x i16> %a0, <8 x i16> %a1, i8 4, i8 -1)
+  %res4 = call i8 @llvm.x86.avx512.mask.ucmp.w.128(<8 x i16> %a0, <8 x i16> %a1, i32 4, i8 -1)
   %vec4 = insertelement <8 x i8> %vec3, i8 %res4, i32 4
 ; CHECK: vpcmpnltuw %xmm1, %xmm0, %k0 ##
-  %res5 = call i8 @llvm.x86.avx512.mask.ucmp.w.128(<8 x i16> %a0, <8 x i16> %a1, i8 5, i8 -1)
+  %res5 = call i8 @llvm.x86.avx512.mask.ucmp.w.128(<8 x i16> %a0, <8 x i16> %a1, i32 5, i8 -1)
   %vec5 = insertelement <8 x i8> %vec4, i8 %res5, i32 5
 ; CHECK: vpcmpnleuw %xmm1, %xmm0, %k0 ##
-  %res6 = call i8 @llvm.x86.avx512.mask.ucmp.w.128(<8 x i16> %a0, <8 x i16> %a1, i8 6, i8 -1)
+  %res6 = call i8 @llvm.x86.avx512.mask.ucmp.w.128(<8 x i16> %a0, <8 x i16> %a1, i32 6, i8 -1)
   %vec6 = insertelement <8 x i8> %vec5, i8 %res6, i32 6
 ; CHECK: vpcmporduw %xmm1, %xmm0, %k0 ##
-  %res7 = call i8 @llvm.x86.avx512.mask.ucmp.w.128(<8 x i16> %a0, <8 x i16> %a1, i8 7, i8 -1)
+  %res7 = call i8 @llvm.x86.avx512.mask.ucmp.w.128(<8 x i16> %a0, <8 x i16> %a1, i32 7, i8 -1)
   %vec7 = insertelement <8 x i8> %vec6, i8 %res7, i32 7
   ret <8 x i8> %vec7
 }
@@ -584,276 +584,953 @@ define <8 x i8> @test_ucmp_w_128(<8 x i16> %a0, <8 x i16> %a1) {
 define <8 x i8> @test_mask_ucmp_w_128(<8 x i16> %a0, <8 x i16> %a1, i8 %mask) {
 ; CHECK_LABEL: test_mask_ucmp_w_128
 ; CHECK: vpcmpequw %xmm1, %xmm0, %k0 {%k1} ##
-  %res0 = call i8 @llvm.x86.avx512.mask.ucmp.w.128(<8 x i16> %a0, <8 x i16> %a1, i8 0, i8 %mask)
+  %res0 = call i8 @llvm.x86.avx512.mask.ucmp.w.128(<8 x i16> %a0, <8 x i16> %a1, i32 0, i8 %mask)
   %vec0 = insertelement <8 x i8> undef, i8 %res0, i32 0
 ; CHECK: vpcmpltuw %xmm1, %xmm0, %k0 {%k1} ##
-  %res1 = call i8 @llvm.x86.avx512.mask.ucmp.w.128(<8 x i16> %a0, <8 x i16> %a1, i8 1, i8 %mask)
+  %res1 = call i8 @llvm.x86.avx512.mask.ucmp.w.128(<8 x i16> %a0, <8 x i16> %a1, i32 1, i8 %mask)
   %vec1 = insertelement <8 x i8> %vec0, i8 %res1, i32 1
 ; CHECK: vpcmpleuw %xmm1, %xmm0, %k0 {%k1} ##
-  %res2 = call i8 @llvm.x86.avx512.mask.ucmp.w.128(<8 x i16> %a0, <8 x i16> %a1, i8 2, i8 %mask)
+  %res2 = call i8 @llvm.x86.avx512.mask.ucmp.w.128(<8 x i16> %a0, <8 x i16> %a1, i32 2, i8 %mask)
   %vec2 = insertelement <8 x i8> %vec1, i8 %res2, i32 2
 ; CHECK: vpcmpunorduw %xmm1, %xmm0, %k0 {%k1} ##
-  %res3 = call i8 @llvm.x86.avx512.mask.ucmp.w.128(<8 x i16> %a0, <8 x i16> %a1, i8 3, i8 %mask)
+  %res3 = call i8 @llvm.x86.avx512.mask.ucmp.w.128(<8 x i16> %a0, <8 x i16> %a1, i32 3, i8 %mask)
   %vec3 = insertelement <8 x i8> %vec2, i8 %res3, i32 3
 ; CHECK: vpcmpnequw %xmm1, %xmm0, %k0 {%k1} ##
-  %res4 = call i8 @llvm.x86.avx512.mask.ucmp.w.128(<8 x i16> %a0, <8 x i16> %a1, i8 4, i8 %mask)
+  %res4 = call i8 @llvm.x86.avx512.mask.ucmp.w.128(<8 x i16> %a0, <8 x i16> %a1, i32 4, i8 %mask)
   %vec4 = insertelement <8 x i8> %vec3, i8 %res4, i32 4
 ; CHECK: vpcmpnltuw %xmm1, %xmm0, %k0 {%k1} ##
-  %res5 = call i8 @llvm.x86.avx512.mask.ucmp.w.128(<8 x i16> %a0, <8 x i16> %a1, i8 5, i8 %mask)
+  %res5 = call i8 @llvm.x86.avx512.mask.ucmp.w.128(<8 x i16> %a0, <8 x i16> %a1, i32 5, i8 %mask)
   %vec5 = insertelement <8 x i8> %vec4, i8 %res5, i32 5
 ; CHECK: vpcmpnleuw %xmm1, %xmm0, %k0 {%k1} ##
-  %res6 = call i8 @llvm.x86.avx512.mask.ucmp.w.128(<8 x i16> %a0, <8 x i16> %a1, i8 6, i8 %mask)
+  %res6 = call i8 @llvm.x86.avx512.mask.ucmp.w.128(<8 x i16> %a0, <8 x i16> %a1, i32 6, i8 %mask)
   %vec6 = insertelement <8 x i8> %vec5, i8 %res6, i32 6
 ; CHECK: vpcmporduw %xmm1, %xmm0, %k0 {%k1} ##
-  %res7 = call i8 @llvm.x86.avx512.mask.ucmp.w.128(<8 x i16> %a0, <8 x i16> %a1, i8 7, i8 %mask)
+  %res7 = call i8 @llvm.x86.avx512.mask.ucmp.w.128(<8 x i16> %a0, <8 x i16> %a1, i32 7, i8 %mask)
   %vec7 = insertelement <8 x i8> %vec6, i8 %res7, i32 7
   ret <8 x i8> %vec7
 }
 
-declare i8 @llvm.x86.avx512.mask.ucmp.w.128(<8 x i16>, <8 x i16>, i8, i8) nounwind readnone
+declare i8 @llvm.x86.avx512.mask.ucmp.w.128(<8 x i16>, <8 x i16>, i32, i8) nounwind readnone
 
-declare <8 x float> @llvm.x86.fma.mask.vfmadd.ps.256(<8 x float>, <8 x float>, <8 x float>, i8) nounwind readnone
+declare <8 x float> @llvm.x86.avx512.mask.vfmadd.ps.256(<8 x float>, <8 x float>, <8 x float>, i8) nounwind readnone
 
 define <8 x float> @test_mask_vfmadd256_ps(<8 x float> %a0, <8 x float> %a1, <8 x float> %a2, i8 %mask) {
   ; CHECK-LABEL: test_mask_vfmadd256_ps
   ; CHECK: vfmadd213ps %ymm2, %ymm1, %ymm0 {%k1} ## encoding: [0x62,0xf2,0x75,0x29,0xa8,0xc2]
-  %res = call <8 x float> @llvm.x86.fma.mask.vfmadd.ps.256(<8 x float> %a0, <8 x float> %a1, <8 x float> %a2, i8 %mask) nounwind
+  %res = call <8 x float> @llvm.x86.avx512.mask.vfmadd.ps.256(<8 x float> %a0, <8 x float> %a1, <8 x float> %a2, i8 %mask) nounwind
   ret <8 x float> %res
 }
 
-declare <4 x float> @llvm.x86.fma.mask.vfmadd.ps.128(<4 x float>, <4 x float>, <4 x float>, i8) nounwind readnone
+declare <4 x float> @llvm.x86.avx512.mask.vfmadd.ps.128(<4 x float>, <4 x float>, <4 x float>, i8) nounwind readnone
 
 define <4 x float> @test_mask_vfmadd128_ps(<4 x float> %a0, <4 x float> %a1, <4 x float> %a2, i8 %mask) {
   ; CHECK-LABEL: test_mask_vfmadd128_ps
   ; CHECK: vfmadd213ps %xmm2, %xmm1, %xmm0 {%k1} ## encoding: [0x62,0xf2,0x75,0x09,0xa8,0xc2]
-  %res = call <4 x float> @llvm.x86.fma.mask.vfmadd.ps.128(<4 x float> %a0, <4 x float> %a1, <4 x float> %a2, i8 %mask) nounwind
+  %res = call <4 x float> @llvm.x86.avx512.mask.vfmadd.ps.128(<4 x float> %a0, <4 x float> %a1, <4 x float> %a2, i8 %mask) nounwind
   ret <4 x float> %res
 }
 
-declare <4 x double> @llvm.x86.fma.mask.vfmadd.pd.256(<4 x double>, <4 x double>, <4 x double>, i8)
+declare <4 x double> @llvm.x86.avx512.mask.vfmadd.pd.256(<4 x double>, <4 x double>, <4 x double>, i8)
 
 define <4 x double> @test_mask_fmadd256_pd(<4 x double> %a, <4 x double> %b, <4 x double> %c, i8 %mask) {
 ; CHECK-LABEL: test_mask_fmadd256_pd:
 ; CHECK: vfmadd213pd %ymm2, %ymm1, %ymm0 {%k1} ## encoding: [0x62,0xf2,0xf5,0x29,0xa8,0xc2]
-  %res = call <4 x double> @llvm.x86.fma.mask.vfmadd.pd.256(<4 x double> %a, <4 x double> %b, <4 x double> %c, i8 %mask)
+  %res = call <4 x double> @llvm.x86.avx512.mask.vfmadd.pd.256(<4 x double> %a, <4 x double> %b, <4 x double> %c, i8 %mask)
   ret <4 x double> %res
 }
 
-declare <2 x double> @llvm.x86.fma.mask.vfmadd.pd.128(<2 x double>, <2 x double>, <2 x double>, i8)
+declare <2 x double> @llvm.x86.avx512.mask.vfmadd.pd.128(<2 x double>, <2 x double>, <2 x double>, i8)
 
 define <2 x double> @test_mask_fmadd128_pd(<2 x double> %a, <2 x double> %b, <2 x double> %c, i8 %mask) {
 ; CHECK-LABEL: test_mask_fmadd128_pd:
 ; CHECK: vfmadd213pd %xmm2, %xmm1, %xmm0 {%k1} ## encoding: [0x62,0xf2,0xf5,0x09,0xa8,0xc2]
-  %res = call <2 x double> @llvm.x86.fma.mask.vfmadd.pd.128(<2 x double> %a, <2 x double> %b, <2 x double> %c, i8 %mask)
+  %res = call <2 x double> @llvm.x86.avx512.mask.vfmadd.pd.128(<2 x double> %a, <2 x double> %b, <2 x double> %c, i8 %mask)
   ret <2 x double> %res
 }
 
-declare <8 x float> @llvm.x86.fma.mask.vfmsub.ps.256(<8 x float>, <8 x float>, <8 x float>, i8) nounwind readnone
+define <2 x double>@test_int_x86_avx512_mask_vfmadd_pd_128(<2 x double> %x0, <2 x double> %x1, <2 x double> %x2, i8 %x3) {
+; CHECK-LABEL: test_int_x86_avx512_mask_vfmadd_pd_128:
+; CHECK:       ## BB#0:
+; CHECK-NEXT:    movzbl %dil, %eax
+; CHECK-NEXT:    kmovw %eax, %k1
+; CHECK-NEXT:    vmovaps %zmm0, %zmm3
+; CHECK-NEXT:    vfmadd213pd %xmm2, %xmm1, %xmm3 {%k1}
+; CHECK-NEXT:    vfmadd213pd %xmm2, %xmm1, %xmm0
+; CHECK-NEXT:    vaddpd %xmm0, %xmm3, %xmm0
+; CHECK-NEXT:    retq
+  %res = call <2 x double> @llvm.x86.avx512.mask.vfmadd.pd.128(<2 x double> %x0, <2 x double> %x1, <2 x double> %x2, i8 %x3)
+  %res1 = call <2 x double> @llvm.x86.avx512.mask.vfmadd.pd.128(<2 x double> %x0, <2 x double> %x1, <2 x double> %x2, i8 -1)
+  %res2 = fadd <2 x double> %res, %res1
+  ret <2 x double> %res2
+}
 
-define <8 x float> @test_mask_vfmsub256_ps(<8 x float> %a0, <8 x float> %a1, <8 x float> %a2, i8 %mask) {
-  ; CHECK-LABEL: test_mask_vfmsub256_ps
-  ; CHECK: vfmsub213ps %ymm2, %ymm1, %ymm0 {%k1} ## encoding: [0x62,0xf2,0x75,0x29,0xaa,0xc2]
-  %res = call <8 x float> @llvm.x86.fma.mask.vfmsub.ps.256(<8 x float> %a0, <8 x float> %a1, <8 x float> %a2, i8 %mask) nounwind
-  ret <8 x float> %res
+declare <2 x double> @llvm.x86.avx512.mask3.vfmadd.pd.128(<2 x double>, <2 x double>, <2 x double>, i8)
+
+define <2 x double>@test_int_x86_avx512_mask3_vfmadd_pd_128(<2 x double> %x0, <2 x double> %x1, <2 x double> %x2, i8 %x3) {
+; CHECK-LABEL: test_int_x86_avx512_mask3_vfmadd_pd_128:
+; CHECK:       ## BB#0:
+; CHECK-NEXT:    movzbl %dil, %eax
+; CHECK-NEXT:    kmovw %eax, %k1
+; CHECK-NEXT:    vmovaps %zmm2, %zmm3
+; CHECK-NEXT:    vfmadd231pd %xmm1, %xmm0, %xmm3 {%k1}
+; CHECK-NEXT:    vfmadd213pd %xmm2, %xmm1, %xmm0
+; CHECK-NEXT:    vaddpd %xmm0, %xmm3, %xmm0
+; CHECK-NEXT:    retq
+  %res = call <2 x double> @llvm.x86.avx512.mask3.vfmadd.pd.128(<2 x double> %x0, <2 x double> %x1, <2 x double> %x2, i8 %x3)
+  %res1 = call <2 x double> @llvm.x86.avx512.mask3.vfmadd.pd.128(<2 x double> %x0, <2 x double> %x1, <2 x double> %x2, i8 -1)
+  %res2 = fadd <2 x double> %res, %res1
+  ret <2 x double> %res2
 }
 
-declare <4 x float> @llvm.x86.fma.mask.vfmsub.ps.128(<4 x float>, <4 x float>, <4 x float>, i8) nounwind readnone
+declare <2 x double> @llvm.x86.avx512.maskz.vfmadd.pd.128(<2 x double>, <2 x double>, <2 x double>, i8)
 
-define <4 x float> @test_mask_vfmsub128_ps(<4 x float> %a0, <4 x float> %a1, <4 x float> %a2, i8 %mask) {
-  ; CHECK-LABEL: test_mask_vfmsub128_ps
-  ; CHECK: vfmsub213ps %xmm2, %xmm1, %xmm0 {%k1} ## encoding: [0x62,0xf2,0x75,0x09,0xaa,0xc2]
-  %res = call <4 x float> @llvm.x86.fma.mask.vfmsub.ps.128(<4 x float> %a0, <4 x float> %a1, <4 x float> %a2, i8 %mask) nounwind
-  ret <4 x float> %res
+define <2 x double>@test_int_x86_avx512_maskz_vfmadd_pd_128(<2 x double> %x0, <2 x double> %x1, <2 x double> %x2, i8 %x3) {
+; CHECK-LABEL: test_int_x86_avx512_maskz_vfmadd_pd_128:
+; CHECK:       ## BB#0:
+; CHECK-NEXT:    movzbl %dil, %eax
+; CHECK-NEXT:    kmovw %eax, %k1
+; CHECK-NEXT:    vmovaps %zmm0, %zmm3
+; CHECK-NEXT:    vfmadd213pd %xmm2, %xmm1, %xmm3 {%k1} {z}
+; CHECK-NEXT:    vfmadd213pd %xmm2, %xmm1, %xmm0
+; CHECK-NEXT:    vaddpd %xmm0, %xmm3, %xmm0
+; CHECK-NEXT:    retq
+  %res = call <2 x double> @llvm.x86.avx512.maskz.vfmadd.pd.128(<2 x double> %x0, <2 x double> %x1, <2 x double> %x2, i8 %x3)
+  %res1 = call <2 x double> @llvm.x86.avx512.maskz.vfmadd.pd.128(<2 x double> %x0, <2 x double> %x1, <2 x double> %x2, i8 -1)
+  %res2 = fadd <2 x double> %res, %res1
+  ret <2 x double> %res2
 }
 
-declare <4 x double> @llvm.x86.fma.mask.vfmsub.pd.256(<4 x double>, <4 x double>, <4 x double>, i8) nounwind readnone
+define <4 x double>@test_int_x86_avx512_mask_vfmadd_pd_256(<4 x double> %x0, <4 x double> %x1, <4 x double> %x2, i8 %x3) {
+; CHECK-LABEL: test_int_x86_avx512_mask_vfmadd_pd_256:
+; CHECK:       ## BB#0:
+; CHECK-NEXT:    movzbl %dil, %eax
+; CHECK-NEXT:    kmovw %eax, %k1
+; CHECK-NEXT:    vmovaps %zmm0, %zmm3
+; CHECK-NEXT:    vfmadd213pd %ymm2, %ymm1, %ymm3 {%k1}
+; CHECK-NEXT:    vfmadd213pd %ymm2, %ymm1, %ymm0
+; CHECK-NEXT:    vaddpd %ymm0, %ymm3, %ymm0
+; CHECK-NEXT:    retq
+  %res = call <4 x double> @llvm.x86.avx512.mask.vfmadd.pd.256(<4 x double> %x0, <4 x double> %x1, <4 x double> %x2, i8 %x3)
+  %res1 = call <4 x double> @llvm.x86.avx512.mask.vfmadd.pd.256(<4 x double> %x0, <4 x double> %x1, <4 x double> %x2, i8 -1)
+  %res2 = fadd <4 x double> %res, %res1
+  ret <4 x double> %res2
+}
 
-define <4 x double> @test_mask_vfmsub256_pd(<4 x double> %a0, <4 x double> %a1, <4 x double> %a2, i8 %mask) {
-  ; CHECK-LABEL: test_mask_vfmsub256_pd
-  ; CHECK: vfmsub213pd %ymm2, %ymm1, %ymm0 {%k1} ## encoding: [0x62,0xf2,0xf5,0x29,0xaa,0xc2]
-  %res = call <4 x double> @llvm.x86.fma.mask.vfmsub.pd.256(<4 x double> %a0, <4 x double> %a1, <4 x double> %a2, i8 %mask) nounwind
-  ret <4 x double> %res
+declare <4 x double> @llvm.x86.avx512.mask3.vfmadd.pd.256(<4 x double>, <4 x double>, <4 x double>, i8)
+
+define <4 x double>@test_int_x86_avx512_mask3_vfmadd_pd_256(<4 x double> %x0, <4 x double> %x1, <4 x double> %x2, i8 %x3) {
+; CHECK-LABEL: test_int_x86_avx512_mask3_vfmadd_pd_256:
+; CHECK:       ## BB#0:
+; CHECK-NEXT:    movzbl %dil, %eax
+; CHECK-NEXT:    kmovw %eax, %k1
+; CHECK-NEXT:    vmovaps %zmm2, %zmm3
+; CHECK-NEXT:    vfmadd231pd %ymm1, %ymm0, %ymm3 {%k1}
+; CHECK-NEXT:    vfmadd213pd %ymm2, %ymm1, %ymm0
+; CHECK-NEXT:    vaddpd %ymm0, %ymm3, %ymm0
+; CHECK-NEXT:    retq
+  %res = call <4 x double> @llvm.x86.avx512.mask3.vfmadd.pd.256(<4 x double> %x0, <4 x double> %x1, <4 x double> %x2, i8 %x3)
+  %res1 = call <4 x double> @llvm.x86.avx512.mask3.vfmadd.pd.256(<4 x double> %x0, <4 x double> %x1, <4 x double> %x2, i8 -1)
+  %res2 = fadd <4 x double> %res, %res1
+  ret <4 x double> %res2
 }
 
-declare <2 x double> @llvm.x86.fma.mask.vfmsub.pd.128(<2 x double>, <2 x double>, <2 x double>, i8) nounwind readnone
+declare <4 x double> @llvm.x86.avx512.maskz.vfmadd.pd.256(<4 x double>, <4 x double>, <4 x double>, i8)
 
-define <2 x double> @test_mask_vfmsub128_pd(<2 x double> %a0, <2 x double> %a1, <2 x double> %a2, i8 %mask) {
-  ; CHECK-LABEL: test_mask_vfmsub128_pd
-  ; CHECK: vfmsub213pd %xmm2, %xmm1, %xmm0 {%k1} ## encoding: [0x62,0xf2,0xf5,0x09,0xaa,0xc2]
-  %res = call <2 x double> @llvm.x86.fma.mask.vfmsub.pd.128(<2 x double> %a0, <2 x double> %a1, <2 x double> %a2, i8 %mask) nounwind
-  ret <2 x double> %res
+define <4 x double>@test_int_x86_avx512_maskz_vfmadd_pd_256(<4 x double> %x0, <4 x double> %x1, <4 x double> %x2, i8 %x3) {
+; CHECK-LABEL: test_int_x86_avx512_maskz_vfmadd_pd_256:
+; CHECK:       ## BB#0:
+; CHECK-NEXT:    movzbl %dil, %eax
+; CHECK-NEXT:    kmovw %eax, %k1
+; CHECK-NEXT:    vmovaps %zmm0, %zmm3
+; CHECK-NEXT:    vfmadd213pd %ymm2, %ymm1, %ymm3 {%k1} {z}
+; CHECK-NEXT:    vfmadd213pd %ymm2, %ymm1, %ymm0
+; CHECK-NEXT:    vaddpd %ymm0, %ymm3, %ymm0
+; CHECK-NEXT:    retq
+  %res = call <4 x double> @llvm.x86.avx512.maskz.vfmadd.pd.256(<4 x double> %x0, <4 x double> %x1, <4 x double> %x2, i8 %x3)
+  %res1 = call <4 x double> @llvm.x86.avx512.maskz.vfmadd.pd.256(<4 x double> %x0, <4 x double> %x1, <4 x double> %x2, i8 -1)
+  %res2 = fadd <4 x double> %res, %res1
+  ret <4 x double> %res2
+}
+
+define <4 x float>@test_int_x86_avx512_mask_vfmadd_ps_128(<4 x float> %x0, <4 x float> %x1, <4 x float> %x2, i8 %x3) {
+; CHECK-LABEL: test_int_x86_avx512_mask_vfmadd_ps_128:
+; CHECK:       ## BB#0:
+; CHECK-NEXT:    movzbl %dil, %eax
+; CHECK-NEXT:    kmovw %eax, %k1
+; CHECK-NEXT:    vmovaps %zmm0, %zmm3
+; CHECK-NEXT:    vfmadd213ps %xmm2, %xmm1, %xmm3 {%k1}
+; CHECK-NEXT:    vfmadd213ps %xmm2, %xmm1, %xmm0
+; CHECK-NEXT:    vaddps %xmm0, %xmm3, %xmm0
+; CHECK-NEXT:    retq
+  %res = call <4 x float> @llvm.x86.avx512.mask.vfmadd.ps.128(<4 x float> %x0, <4 x float> %x1, <4 x float> %x2, i8 %x3)
+  %res1 = call <4 x float> @llvm.x86.avx512.mask.vfmadd.ps.128(<4 x float> %x0, <4 x float> %x1, <4 x float> %x2, i8 -1)
+  %res2 = fadd <4 x float> %res, %res1
+  ret <4 x float> %res2
+}
+
+declare <4 x float> @llvm.x86.avx512.mask3.vfmadd.ps.128(<4 x float>, <4 x float>, <4 x float>, i8)
+
+define <4 x float>@test_int_x86_avx512_mask3_vfmadd_ps_128(<4 x float> %x0, <4 x float> %x1, <4 x float> %x2, i8 %x3) {
+; CHECK-LABEL: test_int_x86_avx512_mask3_vfmadd_ps_128:
+; CHECK:       ## BB#0:
+; CHECK-NEXT:    movzbl %dil, %eax
+; CHECK-NEXT:    kmovw %eax, %k1
+; CHECK-NEXT:    vmovaps %zmm2, %zmm3
+; CHECK-NEXT:    vfmadd231ps %xmm1, %xmm0, %xmm3 {%k1}
+; CHECK-NEXT:    vfmadd213ps %xmm2, %xmm1, %xmm0
+; CHECK-NEXT:    vaddps %xmm0, %xmm3, %xmm0
+; CHECK-NEXT:    retq
+  %res = call <4 x float> @llvm.x86.avx512.mask3.vfmadd.ps.128(<4 x float> %x0, <4 x float> %x1, <4 x float> %x2, i8 %x3)
+  %res1 = call <4 x float> @llvm.x86.avx512.mask3.vfmadd.ps.128(<4 x float> %x0, <4 x float> %x1, <4 x float> %x2, i8 -1)
+  %res2 = fadd <4 x float> %res, %res1
+  ret <4 x float> %res2
+}
+
+declare <4 x float> @llvm.x86.avx512.maskz.vfmadd.ps.128(<4 x float>, <4 x float>, <4 x float>, i8)
+
+define <4 x float>@test_int_x86_avx512_maskz_vfmadd_ps_128(<4 x float> %x0, <4 x float> %x1, <4 x float> %x2, i8 %x3) {
+; CHECK-LABEL: test_int_x86_avx512_maskz_vfmadd_ps_128:
+; CHECK:       ## BB#0:
+; CHECK-NEXT:    movzbl %dil, %eax
+; CHECK-NEXT:    kmovw %eax, %k1
+; CHECK-NEXT:    vmovaps %zmm0, %zmm3
+; CHECK-NEXT:    vfmadd213ps %xmm2, %xmm1, %xmm3 {%k1} {z}
+; CHECK-NEXT:    vfmadd213ps %xmm2, %xmm1, %xmm0
+; CHECK-NEXT:    vaddps %xmm0, %xmm3, %xmm0
+; CHECK-NEXT:    retq
+  %res = call <4 x float> @llvm.x86.avx512.maskz.vfmadd.ps.128(<4 x float> %x0, <4 x float> %x1, <4 x float> %x2, i8 %x3)
+  %res1 = call <4 x float> @llvm.x86.avx512.maskz.vfmadd.ps.128(<4 x float> %x0, <4 x float> %x1, <4 x float> %x2, i8 -1)
+  %res2 = fadd <4 x float> %res, %res1
+  ret <4 x float> %res2
+}
+
+define <8 x float>@test_int_x86_avx512_mask_vfmadd_ps_256(<8 x float> %x0, <8 x float> %x1, <8 x float> %x2, i8 %x3) {
+; CHECK-LABEL: test_int_x86_avx512_mask_vfmadd_ps_256:
+; CHECK:       ## BB#0:
+; CHECK-NEXT:    movzbl %dil, %eax
+; CHECK-NEXT:    kmovw %eax, %k1
+; CHECK-NEXT:    vmovaps %zmm0, %zmm3
+; CHECK-NEXT:    vfmadd213ps %ymm2, %ymm1, %ymm3 {%k1}
+; CHECK-NEXT:    vfmadd213ps %ymm2, %ymm1, %ymm0
+; CHECK-NEXT:    vaddps %ymm0, %ymm3, %ymm0
+; CHECK-NEXT:    retq
+  %res = call <8 x float> @llvm.x86.avx512.mask.vfmadd.ps.256(<8 x float> %x0, <8 x float> %x1, <8 x float> %x2, i8 %x3)
+  %res1 = call <8 x float> @llvm.x86.avx512.mask.vfmadd.ps.256(<8 x float> %x0, <8 x float> %x1, <8 x float> %x2, i8 -1)
+  %res2 = fadd <8 x float> %res, %res1
+  ret <8 x float> %res2
+}
+
+declare <8 x float> @llvm.x86.avx512.mask3.vfmadd.ps.256(<8 x float>, <8 x float>, <8 x float>, i8)
+
+define <8 x float>@test_int_x86_avx512_mask3_vfmadd_ps_256(<8 x float> %x0, <8 x float> %x1, <8 x float> %x2, i8 %x3) {
+; CHECK-LABEL: test_int_x86_avx512_mask3_vfmadd_ps_256:
+; CHECK:       ## BB#0:
+; CHECK-NEXT:    movzbl %dil, %eax
+; CHECK-NEXT:    kmovw %eax, %k1
+; CHECK-NEXT:    vmovaps %zmm2, %zmm3
+; CHECK-NEXT:    vfmadd231ps %ymm1, %ymm0, %ymm3 {%k1}
+; CHECK-NEXT:    vfmadd213ps %ymm2, %ymm1, %ymm0
+; CHECK-NEXT:    vaddps %ymm0, %ymm3, %ymm0
+; CHECK-NEXT:    retq
+  %res = call <8 x float> @llvm.x86.avx512.mask3.vfmadd.ps.256(<8 x float> %x0, <8 x float> %x1, <8 x float> %x2, i8 %x3)
+  %res1 = call <8 x float> @llvm.x86.avx512.mask3.vfmadd.ps.256(<8 x float> %x0, <8 x float> %x1, <8 x float> %x2, i8 -1)
+  %res2 = fadd <8 x float> %res, %res1
+  ret <8 x float> %res2
+}
+
+declare <8 x float> @llvm.x86.avx512.maskz.vfmadd.ps.256(<8 x float>, <8 x float>, <8 x float>, i8)
+
+define <8 x float>@test_int_x86_avx512_maskz_vfmadd_ps_256(<8 x float> %x0, <8 x float> %x1, <8 x float> %x2, i8 %x3) {
+; CHECK-LABEL: test_int_x86_avx512_maskz_vfmadd_ps_256:
+; CHECK:       ## BB#0:
+; CHECK-NEXT:    movzbl %dil, %eax
+; CHECK-NEXT:    kmovw %eax, %k1
+; CHECK-NEXT:    vmovaps %zmm0, %zmm3
+; CHECK-NEXT:    vfmadd213ps %ymm2, %ymm1, %ymm3 {%k1} {z}
+; CHECK-NEXT:    vfmadd213ps %ymm2, %ymm1, %ymm0
+; CHECK-NEXT:    vaddps %ymm0, %ymm3, %ymm0
+; CHECK-NEXT:    retq
+  %res = call <8 x float> @llvm.x86.avx512.maskz.vfmadd.ps.256(<8 x float> %x0, <8 x float> %x1, <8 x float> %x2, i8 %x3)
+  %res1 = call <8 x float> @llvm.x86.avx512.maskz.vfmadd.ps.256(<8 x float> %x0, <8 x float> %x1, <8 x float> %x2, i8 -1)
+  %res2 = fadd <8 x float> %res, %res1
+  ret <8 x float> %res2
+}
+
+
+declare <2 x double> @llvm.x86.avx512.mask3.vfmsub.pd.128(<2 x double>, <2 x double>, <2 x double>, i8)
+
+define <2 x double>@test_int_x86_avx512_mask3_vfmsub_pd_128(<2 x double> %x0, <2 x double> %x1, <2 x double> %x2, i8 %x3) {
+; CHECK-LABEL: test_int_x86_avx512_mask3_vfmsub_pd_128:
+; CHECK:       ## BB#0:
+; CHECK-NEXT:    movzbl %dil, %eax
+; CHECK-NEXT:    kmovw %eax, %k1
+; CHECK-NEXT:    vmovaps %zmm2, %zmm3
+; CHECK-NEXT:    vfmsub231pd %xmm1, %xmm0, %xmm3 {%k1}
+; CHECK-NEXT:    vfmsub213pd %xmm2, %xmm1, %xmm0
+; CHECK-NEXT:    vaddpd %xmm0, %xmm3, %xmm0
+; CHECK-NEXT:    retq
+  %res = call <2 x double> @llvm.x86.avx512.mask3.vfmsub.pd.128(<2 x double> %x0, <2 x double> %x1, <2 x double> %x2, i8 %x3)
+  %res1 = call <2 x double> @llvm.x86.avx512.mask3.vfmsub.pd.128(<2 x double> %x0, <2 x double> %x1, <2 x double> %x2, i8 -1)
+  %res2 = fadd <2 x double> %res, %res1
+  ret <2 x double> %res2
+}
+
+
+declare <4 x double> @llvm.x86.avx512.mask3.vfmsub.pd.256(<4 x double>, <4 x double>, <4 x double>, i8)
+
+define <4 x double>@test_int_x86_avx512_mask3_vfmsub_pd_256(<4 x double> %x0, <4 x double> %x1, <4 x double> %x2, i8 %x3) {
+; CHECK-LABEL: test_int_x86_avx512_mask3_vfmsub_pd_256:
+; CHECK:       ## BB#0:
+; CHECK-NEXT:    movzbl %dil, %eax
+; CHECK-NEXT:    kmovw %eax, %k1
+; CHECK-NEXT:    vmovaps %zmm2, %zmm3
+; CHECK-NEXT:    vfmsub231pd %ymm1, %ymm0, %ymm3 {%k1}
+; CHECK-NEXT:    vfmsub213pd %ymm2, %ymm1, %ymm0
+; CHECK-NEXT:    vaddpd %ymm0, %ymm3, %ymm0
+; CHECK-NEXT:    retq
+  %res = call <4 x double> @llvm.x86.avx512.mask3.vfmsub.pd.256(<4 x double> %x0, <4 x double> %x1, <4 x double> %x2, i8 %x3)
+  %res1 = call <4 x double> @llvm.x86.avx512.mask3.vfmsub.pd.256(<4 x double> %x0, <4 x double> %x1, <4 x double> %x2, i8 -1)
+  %res2 = fadd <4 x double> %res, %res1
+  ret <4 x double> %res2
 }
 
-declare <8 x float> @llvm.x86.fma.mask.vfnmadd.ps.256(<8 x float>, <8 x float>, <8 x float>, i8) nounwind readnone
+declare <4 x float> @llvm.x86.avx512.mask3.vfmsub.ps.128(<4 x float>, <4 x float>, <4 x float>, i8)
+
+define <4 x float>@test_int_x86_avx512_mask3_vfmsub_ps_128(<4 x float> %x0, <4 x float> %x1, <4 x float> %x2, i8 %x3) {
+; CHECK-LABEL: test_int_x86_avx512_mask3_vfmsub_ps_128:
+; CHECK:       ## BB#0:
+; CHECK-NEXT:    movzbl %dil, %eax
+; CHECK-NEXT:    kmovw %eax, %k1
+; CHECK-NEXT:    vmovaps %zmm2, %zmm3
+; CHECK-NEXT:    vfmsub231ps %xmm1, %xmm0, %xmm3 {%k1}
+; CHECK-NEXT:    vfmsub213ps %xmm2, %xmm1, %xmm0
+; CHECK-NEXT:    vaddps %xmm0, %xmm3, %xmm0
+; CHECK-NEXT:    retq
+  %res = call <4 x float> @llvm.x86.avx512.mask3.vfmsub.ps.128(<4 x float> %x0, <4 x float> %x1, <4 x float> %x2, i8 %x3)
+  %res1 = call <4 x float> @llvm.x86.avx512.mask3.vfmsub.ps.128(<4 x float> %x0, <4 x float> %x1, <4 x float> %x2, i8 -1)
+  %res2 = fadd <4 x float> %res, %res1
+  ret <4 x float> %res2
+}
+
+declare <8 x float> @llvm.x86.avx512.mask3.vfmsub.ps.256(<8 x float>, <8 x float>, <8 x float>, i8)
+
+define <8 x float>@test_int_x86_avx512_mask3_vfmsub_ps_256(<8 x float> %x0, <8 x float> %x1, <8 x float> %x2, i8 %x3) {
+; CHECK-LABEL: test_int_x86_avx512_mask3_vfmsub_ps_256:
+; CHECK:       ## BB#0:
+; CHECK-NEXT:    movzbl %dil, %eax
+; CHECK-NEXT:    kmovw %eax, %k1
+; CHECK-NEXT:    vmovaps %zmm2, %zmm3
+; CHECK-NEXT:    vfmsub231ps %ymm1, %ymm0, %ymm3 {%k1}
+; CHECK-NEXT:    vfmsub213ps %ymm2, %ymm1, %ymm0
+; CHECK-NEXT:    vaddps %ymm0, %ymm3, %ymm0
+; CHECK-NEXT:    retq
+  %res = call <8 x float> @llvm.x86.avx512.mask3.vfmsub.ps.256(<8 x float> %x0, <8 x float> %x1, <8 x float> %x2, i8 %x3)
+  %res1 = call <8 x float> @llvm.x86.avx512.mask3.vfmsub.ps.256(<8 x float> %x0, <8 x float> %x1, <8 x float> %x2, i8 -1)
+  %res2 = fadd <8 x float> %res, %res1
+  ret <8 x float> %res2
+}
+
+declare <8 x float> @llvm.x86.avx512.mask.vfnmadd.ps.256(<8 x float>, <8 x float>, <8 x float>, i8) nounwind readnone
 
 define <8 x float> @test_mask_vfnmadd256_ps(<8 x float> %a0, <8 x float> %a1, <8 x float> %a2, i8 %mask) {
   ; CHECK-LABEL: test_mask_vfnmadd256_ps
   ; CHECK: vfnmadd213ps %ymm2, %ymm1, %ymm0 {%k1} ## encoding: [0x62,0xf2,0x75,0x29,0xac,0xc2]
-  %res = call <8 x float> @llvm.x86.fma.mask.vfnmadd.ps.256(<8 x float> %a0, <8 x float> %a1, <8 x float> %a2, i8 %mask) nounwind
+  %res = call <8 x float> @llvm.x86.avx512.mask.vfnmadd.ps.256(<8 x float> %a0, <8 x float> %a1, <8 x float> %a2, i8 %mask) nounwind
   ret <8 x float> %res
 }
 
-declare <4 x float> @llvm.x86.fma.mask.vfnmadd.ps.128(<4 x float>, <4 x float>, <4 x float>, i8) nounwind readnone
+declare <4 x float> @llvm.x86.avx512.mask.vfnmadd.ps.128(<4 x float>, <4 x float>, <4 x float>, i8) nounwind readnone
 
 define <4 x float> @test_mask_vfnmadd128_ps(<4 x float> %a0, <4 x float> %a1, <4 x float> %a2, i8 %mask) {
   ; CHECK-LABEL: test_mask_vfnmadd128_ps
   ; CHECK: vfnmadd213ps %xmm2, %xmm1, %xmm0 {%k1} ## encoding: [0x62,0xf2,0x75,0x09,0xac,0xc2]
-  %res = call <4 x float> @llvm.x86.fma.mask.vfnmadd.ps.128(<4 x float> %a0, <4 x float> %a1, <4 x float> %a2, i8 %mask) nounwind
+  %res = call <4 x float> @llvm.x86.avx512.mask.vfnmadd.ps.128(<4 x float> %a0, <4 x float> %a1, <4 x float> %a2, i8 %mask) nounwind
   ret <4 x float> %res
 }
 
-declare <4 x double> @llvm.x86.fma.mask.vfnmadd.pd.256(<4 x double>, <4 x double>, <4 x double>, i8) nounwind readnone
+declare <4 x double> @llvm.x86.avx512.mask.vfnmadd.pd.256(<4 x double>, <4 x double>, <4 x double>, i8) nounwind readnone
 
 define <4 x double> @test_mask_vfnmadd256_pd(<4 x double> %a0, <4 x double> %a1, <4 x double> %a2, i8 %mask) {
   ; CHECK-LABEL: test_mask_vfnmadd256_pd
   ; CHECK: vfnmadd213pd %ymm2, %ymm1, %ymm0 {%k1} ## encoding: [0x62,0xf2,0xf5,0x29,0xac,0xc2]
-  %res = call <4 x double> @llvm.x86.fma.mask.vfnmadd.pd.256(<4 x double> %a0, <4 x double> %a1, <4 x double> %a2, i8 %mask) nounwind
+  %res = call <4 x double> @llvm.x86.avx512.mask.vfnmadd.pd.256(<4 x double> %a0, <4 x double> %a1, <4 x double> %a2, i8 %mask) nounwind
   ret <4 x double> %res
 }
 
-declare <2 x double> @llvm.x86.fma.mask.vfnmadd.pd.128(<2 x double>, <2 x double>, <2 x double>, i8) nounwind readnone
+declare <2 x double> @llvm.x86.avx512.mask.vfnmadd.pd.128(<2 x double>, <2 x double>, <2 x double>, i8) nounwind readnone
 
 define <2 x double> @test_mask_vfnmadd128_pd(<2 x double> %a0, <2 x double> %a1, <2 x double> %a2, i8 %mask) {
   ; CHECK-LABEL: test_mask_vfnmadd128_pd
   ; CHECK: vfnmadd213pd %xmm2, %xmm1, %xmm0 {%k1} ## encoding: [0x62,0xf2,0xf5,0x09,0xac,0xc2]
-  %res = call <2 x double> @llvm.x86.fma.mask.vfnmadd.pd.128(<2 x double> %a0, <2 x double> %a1, <2 x double> %a2, i8 %mask) nounwind
+  %res = call <2 x double> @llvm.x86.avx512.mask.vfnmadd.pd.128(<2 x double> %a0, <2 x double> %a1, <2 x double> %a2, i8 %mask) nounwind
   ret <2 x double> %res
 }
 
-declare <8 x float> @llvm.x86.fma.mask.vfnmsub.ps.256(<8 x float>, <8 x float>, <8 x float>, i8) nounwind readnone
+declare <8 x float> @llvm.x86.avx512.mask.vfnmsub.ps.256(<8 x float>, <8 x float>, <8 x float>, i8) nounwind readnone
 
 define <8 x float> @test_mask_vfnmsub256_ps(<8 x float> %a0, <8 x float> %a1, <8 x float> %a2, i8 %mask) {
   ; CHECK-LABEL: test_mask_vfnmsub256_ps
   ; CHECK: vfnmsub213ps %ymm2, %ymm1, %ymm0 {%k1} ## encoding: [0x62,0xf2,0x75,0x29,0xae,0xc2]
-  %res = call <8 x float> @llvm.x86.fma.mask.vfnmsub.ps.256(<8 x float> %a0, <8 x float> %a1, <8 x float> %a2, i8 %mask) nounwind
+  %res = call <8 x float> @llvm.x86.avx512.mask.vfnmsub.ps.256(<8 x float> %a0, <8 x float> %a1, <8 x float> %a2, i8 %mask) nounwind
   ret <8 x float> %res
 }
 
-declare <4 x float> @llvm.x86.fma.mask.vfnmsub.ps.128(<4 x float>, <4 x float>, <4 x float>, i8) nounwind readnone
+declare <4 x float> @llvm.x86.avx512.mask.vfnmsub.ps.128(<4 x float>, <4 x float>, <4 x float>, i8) nounwind readnone
 
 define <4 x float> @test_mask_vfnmsub128_ps(<4 x float> %a0, <4 x float> %a1, <4 x float> %a2, i8 %mask) {
   ; CHECK-LABEL: test_mask_vfnmsub128_ps
   ; CHECK: vfnmsub213ps %xmm2, %xmm1, %xmm0 {%k1} ## encoding: [0x62,0xf2,0x75,0x09,0xae,0xc2]
-  %res = call <4 x float> @llvm.x86.fma.mask.vfnmsub.ps.128(<4 x float> %a0, <4 x float> %a1, <4 x float> %a2, i8 %mask) nounwind
+  %res = call <4 x float> @llvm.x86.avx512.mask.vfnmsub.ps.128(<4 x float> %a0, <4 x float> %a1, <4 x float> %a2, i8 %mask) nounwind
   ret <4 x float> %res
 }
 
-declare <4 x double> @llvm.x86.fma.mask.vfnmsub.pd.256(<4 x double>, <4 x double>, <4 x double>, i8) nounwind readnone
+declare <4 x double> @llvm.x86.avx512.mask.vfnmsub.pd.256(<4 x double>, <4 x double>, <4 x double>, i8) nounwind readnone
 
 define <4 x double> @test_mask_vfnmsub256_pd(<4 x double> %a0, <4 x double> %a1, <4 x double> %a2, i8 %mask) {
   ; CHECK-LABEL: test_mask_vfnmsub256_pd
   ; CHECK: vfnmsub213pd %ymm2, %ymm1, %ymm0 {%k1} ## encoding: [0x62,0xf2,0xf5,0x29,0xae,0xc2]
-  %res = call <4 x double> @llvm.x86.fma.mask.vfnmsub.pd.256(<4 x double> %a0, <4 x double> %a1, <4 x double> %a2, i8 %mask) nounwind
+  %res = call <4 x double> @llvm.x86.avx512.mask.vfnmsub.pd.256(<4 x double> %a0, <4 x double> %a1, <4 x double> %a2, i8 %mask) nounwind
   ret <4 x double> %res
 }
 
-declare <2 x double> @llvm.x86.fma.mask.vfnmsub.pd.128(<2 x double>, <2 x double>, <2 x double>, i8) nounwind readnone
+declare <2 x double> @llvm.x86.avx512.mask.vfnmsub.pd.128(<2 x double>, <2 x double>, <2 x double>, i8) nounwind readnone
 
 define <2 x double> @test_mask_vfnmsub128_pd(<2 x double> %a0, <2 x double> %a1, <2 x double> %a2, i8 %mask) {
   ; CHECK-LABEL: test_mask_vfnmsub128_pd
   ; CHECK: vfnmsub213pd %xmm2, %xmm1, %xmm0 {%k1} ## encoding: [0x62,0xf2,0xf5,0x09,0xae,0xc2]
-  %res = call <2 x double> @llvm.x86.fma.mask.vfnmsub.pd.128(<2 x double> %a0, <2 x double> %a1, <2 x double> %a2, i8 %mask) nounwind
+  %res = call <2 x double> @llvm.x86.avx512.mask.vfnmsub.pd.128(<2 x double> %a0, <2 x double> %a1, <2 x double> %a2, i8 %mask) nounwind
   ret <2 x double> %res
 }
 
-declare <8 x float> @llvm.x86.fma.mask.vfmaddsub.ps.256(<8 x float>, <8 x float>, <8 x float>, i8) nounwind readnone
+
+define <2 x double>@test_int_x86_avx512_mask_vfnmsub_pd_128(<2 x double> %x0, <2 x double> %x1, <2 x double> %x2, i8 %x3) {
+; CHECK-LABEL: test_int_x86_avx512_mask_vfnmsub_pd_128:
+; CHECK:       ## BB#0:
+; CHECK-NEXT:    movzbl %dil, %eax
+; CHECK-NEXT:    kmovw %eax, %k1
+; CHECK-NEXT:    vmovaps %zmm0, %zmm3
+; CHECK-NEXT:    vfnmsub213pd %xmm2, %xmm1, %xmm3 {%k1}
+; CHECK-NEXT:    vfnmsub213pd %xmm2, %xmm1, %xmm0
+; CHECK-NEXT:    vaddpd %xmm0, %xmm3, %xmm0
+; CHECK-NEXT:    retq
+  %res = call <2 x double> @llvm.x86.avx512.mask.vfnmsub.pd.128(<2 x double> %x0, <2 x double> %x1, <2 x double> %x2, i8 %x3)
+  %res1 = call <2 x double> @llvm.x86.avx512.mask.vfnmsub.pd.128(<2 x double> %x0, <2 x double> %x1, <2 x double> %x2, i8 -1)
+  %res2 = fadd <2 x double> %res, %res1
+  ret <2 x double> %res2
+}
+
+declare <2 x double> @llvm.x86.avx512.mask3.vfnmsub.pd.128(<2 x double>, <2 x double>, <2 x double>, i8)
+
+define <2 x double>@test_int_x86_avx512_mask3_vfnmsub_pd_128(<2 x double> %x0, <2 x double> %x1, <2 x double> %x2, i8 %x3) {
+; CHECK-LABEL: test_int_x86_avx512_mask3_vfnmsub_pd_128:
+; CHECK:       ## BB#0:
+; CHECK-NEXT:    movzbl %dil, %eax
+; CHECK-NEXT:    kmovw %eax, %k1
+; CHECK-NEXT:    vmovaps %zmm2, %zmm3
+; CHECK-NEXT:    vfnmsub231pd %xmm1, %xmm0, %xmm3 {%k1}
+; CHECK-NEXT:    vfnmsub213pd %xmm2, %xmm1, %xmm0
+; CHECK-NEXT:    vaddpd %xmm0, %xmm3, %xmm0
+; CHECK-NEXT:    retq
+  %res = call <2 x double> @llvm.x86.avx512.mask3.vfnmsub.pd.128(<2 x double> %x0, <2 x double> %x1, <2 x double> %x2, i8 %x3)
+  %res1 = call <2 x double> @llvm.x86.avx512.mask3.vfnmsub.pd.128(<2 x double> %x0, <2 x double> %x1, <2 x double> %x2, i8 -1)
+  %res2 = fadd <2 x double> %res, %res1
+  ret <2 x double> %res2
+}
+
+define <4 x double>@test_int_x86_avx512_mask_vfnmsub_pd_256(<4 x double> %x0, <4 x double> %x1, <4 x double> %x2, i8 %x3) {
+; CHECK-LABEL: test_int_x86_avx512_mask_vfnmsub_pd_256:
+; CHECK:       ## BB#0:
+; CHECK-NEXT:    movzbl %dil, %eax
+; CHECK-NEXT:    kmovw %eax, %k1
+; CHECK-NEXT:    vmovaps %zmm0, %zmm3
+; CHECK-NEXT:    vfnmsub213pd %ymm2, %ymm1, %ymm3 {%k1}
+; CHECK-NEXT:    vfnmsub213pd %ymm2, %ymm1, %ymm0
+; CHECK-NEXT:    vaddpd %ymm0, %ymm3, %ymm0
+; CHECK-NEXT:    retq
+  %res = call <4 x double> @llvm.x86.avx512.mask.vfnmsub.pd.256(<4 x double> %x0, <4 x double> %x1, <4 x double> %x2, i8 %x3)
+  %res1 = call <4 x double> @llvm.x86.avx512.mask.vfnmsub.pd.256(<4 x double> %x0, <4 x double> %x1, <4 x double> %x2, i8 -1)
+  %res2 = fadd <4 x double> %res, %res1
+  ret <4 x double> %res2
+}
+
+declare <4 x double> @llvm.x86.avx512.mask3.vfnmsub.pd.256(<4 x double>, <4 x double>, <4 x double>, i8)
+
+define <4 x double>@test_int_x86_avx512_mask3_vfnmsub_pd_256(<4 x double> %x0, <4 x double> %x1, <4 x double> %x2, i8 %x3) {
+; CHECK-LABEL: test_int_x86_avx512_mask3_vfnmsub_pd_256:
+; CHECK:       ## BB#0:
+; CHECK-NEXT:    movzbl %dil, %eax
+; CHECK-NEXT:    kmovw %eax, %k1
+; CHECK-NEXT:    vmovaps %zmm2, %zmm3
+; CHECK-NEXT:    vfnmsub231pd %ymm1, %ymm0, %ymm3 {%k1}
+; CHECK-NEXT:    vfnmsub213pd %ymm2, %ymm1, %ymm0
+; CHECK-NEXT:    vaddpd %ymm0, %ymm3, %ymm0
+; CHECK-NEXT:    retq
+  %res = call <4 x double> @llvm.x86.avx512.mask3.vfnmsub.pd.256(<4 x double> %x0, <4 x double> %x1, <4 x double> %x2, i8 %x3)
+  %res1 = call <4 x double> @llvm.x86.avx512.mask3.vfnmsub.pd.256(<4 x double> %x0, <4 x double> %x1, <4 x double> %x2, i8 -1)
+  %res2 = fadd <4 x double> %res, %res1
+  ret <4 x double> %res2
+}
+
+define <4 x float>@test_int_x86_avx512_mask_vfnmsub_ps_128(<4 x float> %x0, <4 x float> %x1, <4 x float> %x2, i8 %x3) {
+; CHECK-LABEL: test_int_x86_avx512_mask_vfnmsub_ps_128:
+; CHECK:       ## BB#0:
+; CHECK-NEXT:    movzbl %dil, %eax
+; CHECK-NEXT:    kmovw %eax, %k1
+; CHECK-NEXT:    vmovaps %zmm0, %zmm3
+; CHECK-NEXT:    vfnmsub213ps %xmm2, %xmm1, %xmm3 {%k1}
+; CHECK-NEXT:    vfnmsub213ps %xmm2, %xmm1, %xmm0
+; CHECK-NEXT:    vaddps %xmm0, %xmm3, %xmm0
+; CHECK-NEXT:    retq
+  %res = call <4 x float> @llvm.x86.avx512.mask.vfnmsub.ps.128(<4 x float> %x0, <4 x float> %x1, <4 x float> %x2, i8 %x3)
+  %res1 = call <4 x float> @llvm.x86.avx512.mask.vfnmsub.ps.128(<4 x float> %x0, <4 x float> %x1, <4 x float> %x2, i8 -1)
+  %res2 = fadd <4 x float> %res, %res1
+  ret <4 x float> %res2
+}
+
+declare <4 x float> @llvm.x86.avx512.mask3.vfnmsub.ps.128(<4 x float>, <4 x float>, <4 x float>, i8)
+
+define <4 x float>@test_int_x86_avx512_mask3_vfnmsub_ps_128(<4 x float> %x0, <4 x float> %x1, <4 x float> %x2, i8 %x3) {
+; CHECK-LABEL: test_int_x86_avx512_mask3_vfnmsub_ps_128:
+; CHECK:       ## BB#0:
+; CHECK-NEXT:    movzbl %dil, %eax
+; CHECK-NEXT:    kmovw %eax, %k1
+; CHECK-NEXT:    vmovaps %zmm2, %zmm3
+; CHECK-NEXT:    vfnmsub231ps %xmm1, %xmm0, %xmm3 {%k1}
+; CHECK-NEXT:    vfnmsub213ps %xmm2, %xmm1, %xmm0
+; CHECK-NEXT:    vaddps %xmm0, %xmm3, %xmm0
+; CHECK-NEXT:    retq
+  %res = call <4 x float> @llvm.x86.avx512.mask3.vfnmsub.ps.128(<4 x float> %x0, <4 x float> %x1, <4 x float> %x2, i8 %x3)
+  %res1 = call <4 x float> @llvm.x86.avx512.mask3.vfnmsub.ps.128(<4 x float> %x0, <4 x float> %x1, <4 x float> %x2, i8 -1)
+  %res2 = fadd <4 x float> %res, %res1
+  ret <4 x float> %res2
+}
+
+define <8 x float>@test_int_x86_avx512_mask_vfnmsub_ps_256(<8 x float> %x0, <8 x float> %x1, <8 x float> %x2, i8 %x3) {
+; CHECK-LABEL: test_int_x86_avx512_mask_vfnmsub_ps_256:
+; CHECK:       ## BB#0:
+; CHECK-NEXT:    movzbl %dil, %eax
+; CHECK-NEXT:    kmovw %eax, %k1
+; CHECK-NEXT:    vmovaps %zmm0, %zmm3
+; CHECK-NEXT:    vfnmsub213ps %ymm2, %ymm1, %ymm3 {%k1}
+; CHECK-NEXT:    vfnmsub213ps %ymm2, %ymm1, %ymm0
+; CHECK-NEXT:    vaddps %ymm0, %ymm3, %ymm0
+; CHECK-NEXT:    retq
+  %res = call <8 x float> @llvm.x86.avx512.mask.vfnmsub.ps.256(<8 x float> %x0, <8 x float> %x1, <8 x float> %x2, i8 %x3)
+  %res1 = call <8 x float> @llvm.x86.avx512.mask.vfnmsub.ps.256(<8 x float> %x0, <8 x float> %x1, <8 x float> %x2, i8 -1)
+  %res2 = fadd <8 x float> %res, %res1
+  ret <8 x float> %res2
+}
+
+declare <8 x float> @llvm.x86.avx512.mask3.vfnmsub.ps.256(<8 x float>, <8 x float>, <8 x float>, i8)
+
+define <8 x float>@test_int_x86_avx512_mask3_vfnmsub_ps_256(<8 x float> %x0, <8 x float> %x1, <8 x float> %x2, i8 %x3) {
+; CHECK-LABEL: test_int_x86_avx512_mask3_vfnmsub_ps_256:
+; CHECK:       ## BB#0:
+; CHECK-NEXT:    movzbl %dil, %eax
+; CHECK-NEXT:    kmovw %eax, %k1
+; CHECK-NEXT:    vmovaps %zmm2, %zmm3
+; CHECK-NEXT:    vfnmsub231ps %ymm1, %ymm0, %ymm3 {%k1}
+; CHECK-NEXT:    vfnmsub213ps %ymm2, %ymm1, %ymm0
+; CHECK-NEXT:    vaddps %ymm0, %ymm3, %ymm0
+; CHECK-NEXT:    retq
+  %res = call <8 x float> @llvm.x86.avx512.mask3.vfnmsub.ps.256(<8 x float> %x0, <8 x float> %x1, <8 x float> %x2, i8 %x3)
+  %res1 = call <8 x float> @llvm.x86.avx512.mask3.vfnmsub.ps.256(<8 x float> %x0, <8 x float> %x1, <8 x float> %x2, i8 -1)
+  %res2 = fadd <8 x float> %res, %res1
+  ret <8 x float> %res2
+}
+
+define <2 x double>@test_int_x86_avx512_mask_vfnmadd_pd_128(<2 x double> %x0, <2 x double> %x1, <2 x double> %x2, i8 %x3) {
+; CHECK-LABEL: test_int_x86_avx512_mask_vfnmadd_pd_128:
+; CHECK:       ## BB#0:
+; CHECK-NEXT:    movzbl %dil, %eax
+; CHECK-NEXT:    kmovw %eax, %k1
+; CHECK-NEXT:    vmovaps %zmm0, %zmm3
+; CHECK-NEXT:    vfnmadd213pd %xmm2, %xmm1, %xmm3 {%k1}
+; CHECK-NEXT:    vfnmadd213pd %xmm2, %xmm1, %xmm0
+; CHECK-NEXT:    vaddpd %xmm0, %xmm3, %xmm0
+; CHECK-NEXT:    retq
+  %res = call <2 x double> @llvm.x86.avx512.mask.vfnmadd.pd.128(<2 x double> %x0, <2 x double> %x1, <2 x double> %x2, i8 %x3)
+  %res1 = call <2 x double> @llvm.x86.avx512.mask.vfnmadd.pd.128(<2 x double> %x0, <2 x double> %x1, <2 x double> %x2, i8 -1)
+  %res2 = fadd <2 x double> %res, %res1
+  ret <2 x double> %res2
+}
+
+define <4 x double>@test_int_x86_avx512_mask_vfnmadd_pd_256(<4 x double> %x0, <4 x double> %x1, <4 x double> %x2, i8 %x3) {
+; CHECK-LABEL: test_int_x86_avx512_mask_vfnmadd_pd_256:
+; CHECK:       ## BB#0:
+; CHECK-NEXT:    movzbl %dil, %eax
+; CHECK-NEXT:    kmovw %eax, %k1
+; CHECK-NEXT:    vmovaps %zmm0, %zmm3
+; CHECK-NEXT:    vfnmadd213pd %ymm2, %ymm1, %ymm3 {%k1}
+; CHECK-NEXT:    vfnmadd213pd %ymm2, %ymm1, %ymm0
+; CHECK-NEXT:    vaddpd %ymm0, %ymm3, %ymm0
+; CHECK-NEXT:    retq
+  %res = call <4 x double> @llvm.x86.avx512.mask.vfnmadd.pd.256(<4 x double> %x0, <4 x double> %x1, <4 x double> %x2, i8 %x3)
+  %res1 = call <4 x double> @llvm.x86.avx512.mask.vfnmadd.pd.256(<4 x double> %x0, <4 x double> %x1, <4 x double> %x2, i8 -1)
+  %res2 = fadd <4 x double> %res, %res1
+  ret <4 x double> %res2
+}
+
+define <4 x float>@test_int_x86_avx512_mask_vfnmadd_ps_128(<4 x float> %x0, <4 x float> %x1, <4 x float> %x2, i8 %x3) {
+; CHECK-LABEL: test_int_x86_avx512_mask_vfnmadd_ps_128:
+; CHECK:       ## BB#0:
+; CHECK-NEXT:    movzbl %dil, %eax
+; CHECK-NEXT:    kmovw %eax, %k1
+; CHECK-NEXT:    vmovaps %zmm0, %zmm3
+; CHECK-NEXT:    vfnmadd213ps %xmm2, %xmm1, %xmm3 {%k1}
+; CHECK-NEXT:    vfnmadd213ps %xmm2, %xmm1, %xmm0
+; CHECK-NEXT:    vaddps %xmm0, %xmm3, %xmm0
+; CHECK-NEXT:    retq
+  %res = call <4 x float> @llvm.x86.avx512.mask.vfnmadd.ps.128(<4 x float> %x0, <4 x float> %x1, <4 x float> %x2, i8 %x3)
+  %res1 = call <4 x float> @llvm.x86.avx512.mask.vfnmadd.ps.128(<4 x float> %x0, <4 x float> %x1, <4 x float> %x2, i8 -1)
+  %res2 = fadd <4 x float> %res, %res1
+  ret <4 x float> %res2
+}
+
+define <8 x float>@test_int_x86_avx512_mask_vfnmadd_ps_256(<8 x float> %x0, <8 x float> %x1, <8 x float> %x2, i8 %x3) {
+; CHECK-LABEL: test_int_x86_avx512_mask_vfnmadd_ps_256:
+; CHECK:       ## BB#0:
+; CHECK-NEXT:    movzbl %dil, %eax
+; CHECK-NEXT:    kmovw %eax, %k1
+; CHECK-NEXT:    vmovaps %zmm0, %zmm3
+; CHECK-NEXT:    vfnmadd213ps %ymm2, %ymm1, %ymm3 {%k1}
+; CHECK-NEXT:    vfnmadd213ps %ymm2, %ymm1, %ymm0
+; CHECK-NEXT:    vaddps %ymm0, %ymm3, %ymm0
+; CHECK-NEXT:    retq
+  %res = call <8 x float> @llvm.x86.avx512.mask.vfnmadd.ps.256(<8 x float> %x0, <8 x float> %x1, <8 x float> %x2, i8 %x3)
+  %res1 = call <8 x float> @llvm.x86.avx512.mask.vfnmadd.ps.256(<8 x float> %x0, <8 x float> %x1, <8 x float> %x2, i8 -1)
+  %res2 = fadd <8 x float> %res, %res1
+  ret <8 x float> %res2
+}
+
+declare <8 x float> @llvm.x86.avx512.mask.vfmaddsub.ps.256(<8 x float>, <8 x float>, <8 x float>, i8) nounwind readnone
 
 define <8 x float> @test_mask_fmaddsub256_ps(<8 x float> %a, <8 x float> %b, <8 x float> %c, i8 %mask) {
 ; CHECK-LABEL: test_mask_fmaddsub256_ps:
 ; CHECK: vfmaddsub213ps %ymm2, %ymm1, %ymm0 {%k1} ## encoding: [0x62,0xf2,0x75,0x29,0xa6,0xc2]
-  %res = call <8 x float> @llvm.x86.fma.mask.vfmaddsub.ps.256(<8 x float> %a, <8 x float> %b, <8 x float> %c, i8 %mask)
+  %res = call <8 x float> @llvm.x86.avx512.mask.vfmaddsub.ps.256(<8 x float> %a, <8 x float> %b, <8 x float> %c, i8 %mask)
   ret <8 x float> %res
 }
 
-declare <4 x float> @llvm.x86.fma.mask.vfmaddsub.ps.128(<4 x float>, <4 x float>, <4 x float>, i8) nounwind readnone
+declare <4 x float> @llvm.x86.avx512.mask.vfmaddsub.ps.128(<4 x float>, <4 x float>, <4 x float>, i8) nounwind readnone
 
 define <4 x float> @test_mask_fmaddsub128_ps(<4 x float> %a, <4 x float> %b, <4 x float> %c, i8 %mask) {
 ; CHECK-LABEL: test_mask_fmaddsub128_ps:
 ; CHECK: vfmaddsub213ps %xmm2, %xmm1, %xmm0 {%k1} ## encoding: [0x62,0xf2,0x75,0x09,0xa6,0xc2]
-  %res = call <4 x float> @llvm.x86.fma.mask.vfmaddsub.ps.128(<4 x float> %a, <4 x float> %b, <4 x float> %c, i8 %mask)
+  %res = call <4 x float> @llvm.x86.avx512.mask.vfmaddsub.ps.128(<4 x float> %a, <4 x float> %b, <4 x float> %c, i8 %mask)
   ret <4 x float> %res
 }
 
-declare <4 x double> @llvm.x86.fma.mask.vfmaddsub.pd.256(<4 x double>, <4 x double>, <4 x double>, i8) nounwind readnone
+declare <4 x double> @llvm.x86.avx512.mask.vfmaddsub.pd.256(<4 x double>, <4 x double>, <4 x double>, i8) nounwind readnone
 
 define <4 x double> @test_mask_vfmaddsub256_pd(<4 x double> %a0, <4 x double> %a1, <4 x double> %a2, i8 %mask) {
   ; CHECK-LABEL: test_mask_vfmaddsub256_pd
   ; CHECK: vfmaddsub213pd %ymm2, %ymm1, %ymm0 {%k1} ## encoding: [0x62,0xf2,0xf5,0x29,0xa6,0xc2]
-  %res = call <4 x double> @llvm.x86.fma.mask.vfmaddsub.pd.256(<4 x double> %a0, <4 x double> %a1, <4 x double> %a2, i8 %mask) nounwind
+  %res = call <4 x double> @llvm.x86.avx512.mask.vfmaddsub.pd.256(<4 x double> %a0, <4 x double> %a1, <4 x double> %a2, i8 %mask) nounwind
   ret <4 x double> %res
 }
 
-declare <2 x double> @llvm.x86.fma.mask.vfmaddsub.pd.128(<2 x double>, <2 x double>, <2 x double>, i8) nounwind readnone
+declare <2 x double> @llvm.x86.avx512.mask.vfmaddsub.pd.128(<2 x double>, <2 x double>, <2 x double>, i8) nounwind readnone
 
 define <2 x double> @test_mask_vfmaddsub128_pd(<2 x double> %a0, <2 x double> %a1, <2 x double> %a2, i8 %mask) {
   ; CHECK-LABEL: test_mask_vfmaddsub128_pd
   ; CHECK: vfmaddsub213pd %xmm2, %xmm1, %xmm0 {%k1} ## encoding: [0x62,0xf2,0xf5,0x09,0xa6,0xc2]
-  %res = call <2 x double> @llvm.x86.fma.mask.vfmaddsub.pd.128(<2 x double> %a0, <2 x double> %a1, <2 x double> %a2, i8 %mask) nounwind
+  %res = call <2 x double> @llvm.x86.avx512.mask.vfmaddsub.pd.128(<2 x double> %a0, <2 x double> %a1, <2 x double> %a2, i8 %mask) nounwind
   ret <2 x double> %res
 }
 
-declare <8 x float> @llvm.x86.fma.mask.vfmsubadd.ps.256(<8 x float>, <8 x float>, <8 x float>, i8) nounwind readnone
+define <2 x double>@test_int_x86_avx512_mask_vfmaddsub_pd_128(<2 x double> %x0, <2 x double> %x1, <2 x double> %x2, i8 %x3) {
+; CHECK-LABEL: test_int_x86_avx512_mask_vfmaddsub_pd_128:
+; CHECK:       ## BB#0:
+; CHECK-NEXT:    movzbl %dil, %eax
+; CHECK-NEXT:    kmovw %eax, %k1
+; CHECK-NEXT:    vmovaps %zmm0, %zmm3
+; CHECK-NEXT:    vfmaddsub213pd %xmm2, %xmm1, %xmm3 {%k1}
+; CHECK-NEXT:    vfmaddsub213pd %xmm2, %xmm1, %xmm0
+; CHECK-NEXT:    vaddpd %xmm0, %xmm3, %xmm0
+; CHECK-NEXT:    retq
+  %res = call <2 x double> @llvm.x86.avx512.mask.vfmaddsub.pd.128(<2 x double> %x0, <2 x double> %x1, <2 x double> %x2, i8 %x3)
+  %res1 = call <2 x double> @llvm.x86.avx512.mask.vfmaddsub.pd.128(<2 x double> %x0, <2 x double> %x1, <2 x double> %x2, i8 -1)
+  %res2 = fadd <2 x double> %res, %res1
+  ret <2 x double> %res2
+}
+
+declare <2 x double> @llvm.x86.avx512.mask3.vfmaddsub.pd.128(<2 x double>, <2 x double>, <2 x double>, i8)
 
-define <8 x float> @test_mask_vfmsubadd256_ps(<8 x float> %a0, <8 x float> %a1, <8 x float> %a2, i8 %mask) {
-  ; CHECK-LABEL: test_mask_vfmsubadd256_ps
-  ; CHECK: vfmsubadd213ps %ymm2, %ymm1, %ymm0 {%k1} ## encoding: [0x62,0xf2,0x75,0x29,0xa7,0xc2]
-  %res = call <8 x float> @llvm.x86.fma.mask.vfmsubadd.ps.256(<8 x float> %a0, <8 x float> %a1, <8 x float> %a2, i8 %mask) nounwind
-  ret <8 x float> %res
+define <2 x double>@test_int_x86_avx512_mask3_vfmaddsub_pd_128(<2 x double> %x0, <2 x double> %x1, <2 x double> %x2, i8 %x3) {
+; CHECK-LABEL: test_int_x86_avx512_mask3_vfmaddsub_pd_128:
+; CHECK:       ## BB#0:
+; CHECK-NEXT:    movzbl %dil, %eax
+; CHECK-NEXT:    kmovw %eax, %k1
+; CHECK-NEXT:    vmovaps %zmm2, %zmm3
+; CHECK-NEXT:    vfmaddsub231pd %xmm1, %xmm0, %xmm3 {%k1}
+; CHECK-NEXT:    vfmaddsub213pd %xmm2, %xmm1, %xmm0
+; CHECK-NEXT:    vaddpd %xmm0, %xmm3, %xmm0
+; CHECK-NEXT:    retq
+  %res = call <2 x double> @llvm.x86.avx512.mask3.vfmaddsub.pd.128(<2 x double> %x0, <2 x double> %x1, <2 x double> %x2, i8 %x3)
+  %res1 = call <2 x double> @llvm.x86.avx512.mask3.vfmaddsub.pd.128(<2 x double> %x0, <2 x double> %x1, <2 x double> %x2, i8 -1)
+  %res2 = fadd <2 x double> %res, %res1
+  ret <2 x double> %res2
 }
 
-declare <4 x float> @llvm.x86.fma.mask.vfmsubadd.ps.128(<4 x float>, <4 x float>, <4 x float>, i8) nounwind readnone
+declare <2 x double> @llvm.x86.avx512.maskz.vfmaddsub.pd.128(<2 x double>, <2 x double>, <2 x double>, i8)
 
-define <4 x float> @test_mask_vfmsubadd128_ps(<4 x float> %a0, <4 x float> %a1, <4 x float> %a2, i8 %mask) {
-  ; CHECK-LABEL: test_mask_vfmsubadd128_ps
-  ; CHECK: vfmsubadd213ps %xmm2, %xmm1, %xmm0 {%k1} ## encoding: [0x62,0xf2,0x75,0x09,0xa7,0xc2]
-  %res = call <4 x float> @llvm.x86.fma.mask.vfmsubadd.ps.128(<4 x float> %a0, <4 x float> %a1, <4 x float> %a2, i8 %mask) nounwind
-  ret <4 x float> %res
+define <2 x double>@test_int_x86_avx512_maskz_vfmaddsub_pd_128(<2 x double> %x0, <2 x double> %x1, <2 x double> %x2, i8 %x3) {
+; CHECK-LABEL: test_int_x86_avx512_maskz_vfmaddsub_pd_128:
+; CHECK:       ## BB#0:
+; CHECK-NEXT:    movzbl %dil, %eax
+; CHECK-NEXT:    kmovw %eax, %k1
+; CHECK-NEXT:    vmovaps %zmm0, %zmm3
+; CHECK-NEXT:    vfmaddsub213pd %xmm2, %xmm1, %xmm3 {%k1} {z}
+; CHECK-NEXT:    vfmaddsub213pd %xmm2, %xmm1, %xmm0
+; CHECK-NEXT:    vaddpd %xmm0, %xmm3, %xmm0
+; CHECK-NEXT:    retq
+  %res = call <2 x double> @llvm.x86.avx512.maskz.vfmaddsub.pd.128(<2 x double> %x0, <2 x double> %x1, <2 x double> %x2, i8 %x3)
+  %res1 = call <2 x double> @llvm.x86.avx512.maskz.vfmaddsub.pd.128(<2 x double> %x0, <2 x double> %x1, <2 x double> %x2, i8 -1)
+  %res2 = fadd <2 x double> %res, %res1
+  ret <2 x double> %res2
+}
+
+define <4 x double>@test_int_x86_avx512_mask_vfmaddsub_pd_256(<4 x double> %x0, <4 x double> %x1, <4 x double> %x2, i8 %x3) {
+; CHECK-LABEL: test_int_x86_avx512_mask_vfmaddsub_pd_256:
+; CHECK:       ## BB#0:
+; CHECK-NEXT:    movzbl %dil, %eax
+; CHECK-NEXT:    kmovw %eax, %k1
+; CHECK-NEXT:    vmovaps %zmm0, %zmm3
+; CHECK-NEXT:    vfmaddsub213pd %ymm2, %ymm1, %ymm3 {%k1}
+; CHECK-NEXT:    vfmaddsub213pd %ymm2, %ymm1, %ymm0
+; CHECK-NEXT:    vaddpd %ymm0, %ymm3, %ymm0
+; CHECK-NEXT:    retq
+  %res = call <4 x double> @llvm.x86.avx512.mask.vfmaddsub.pd.256(<4 x double> %x0, <4 x double> %x1, <4 x double> %x2, i8 %x3)
+  %res1 = call <4 x double> @llvm.x86.avx512.mask.vfmaddsub.pd.256(<4 x double> %x0, <4 x double> %x1, <4 x double> %x2, i8 -1)
+  %res2 = fadd <4 x double> %res, %res1
+  ret <4 x double> %res2
 }
 
-declare <4 x double> @llvm.x86.fma.mask.vfmsubadd.pd.256(<4 x double>, <4 x double>, <4 x double>, i8) nounwind readnone
+declare <4 x double> @llvm.x86.avx512.mask3.vfmaddsub.pd.256(<4 x double>, <4 x double>, <4 x double>, i8)
 
-define <4 x double> @test_mask_vfmsubadd256_pd(<4 x double> %a0, <4 x double> %a1, <4 x double> %a2, i8 %mask) {
-  ; CHECK-LABEL: test_mask_vfmsubadd256_pd
-  ; CHECK: vfmsubadd213pd %ymm2, %ymm1, %ymm0 {%k1} ## encoding: [0x62,0xf2,0xf5,0x29,0xa7,0xc2]
-  %res = call <4 x double> @llvm.x86.fma.mask.vfmsubadd.pd.256(<4 x double> %a0, <4 x double> %a1, <4 x double> %a2, i8 %mask) nounwind
-  ret <4 x double> %res
+define <4 x double>@test_int_x86_avx512_mask3_vfmaddsub_pd_256(<4 x double> %x0, <4 x double> %x1, <4 x double> %x2, i8 %x3) {
+; CHECK-LABEL: test_int_x86_avx512_mask3_vfmaddsub_pd_256:
+; CHECK:       ## BB#0:
+; CHECK-NEXT:    movzbl %dil, %eax
+; CHECK-NEXT:    kmovw %eax, %k1
+; CHECK-NEXT:    vmovaps %zmm2, %zmm3
+; CHECK-NEXT:    vfmaddsub231pd %ymm1, %ymm0, %ymm3 {%k1}
+; CHECK-NEXT:    vfmaddsub213pd %ymm2, %ymm1, %ymm0
+; CHECK-NEXT:    vaddpd %ymm0, %ymm3, %ymm0
+; CHECK-NEXT:    retq
+  %res = call <4 x double> @llvm.x86.avx512.mask3.vfmaddsub.pd.256(<4 x double> %x0, <4 x double> %x1, <4 x double> %x2, i8 %x3)
+  %res1 = call <4 x double> @llvm.x86.avx512.mask3.vfmaddsub.pd.256(<4 x double> %x0, <4 x double> %x1, <4 x double> %x2, i8 -1)
+  %res2 = fadd <4 x double> %res, %res1
+  ret <4 x double> %res2
 }
-declare <2 x double> @llvm.x86.fma.mask.vfmsubadd.pd.128(<2 x double>, <2 x double>, <2 x double>, i8) nounwind readnone
 
-define <2 x double> @test_mask_vfmsubadd128_pd(<2 x double> %a0, <2 x double> %a1, <2 x double> %a2, i8 %mask) {
-  ; CHECK-LABEL: test_mask_vfmsubadd128_pd
-  ; CHECK: vfmsubadd213pd %xmm2, %xmm1, %xmm0 {%k1} ## encoding: [0x62,0xf2,0xf5,0x09,0xa7,0xc2]
-  %res = call <2 x double> @llvm.x86.fma.mask.vfmsubadd.pd.128(<2 x double> %a0, <2 x double> %a1, <2 x double> %a2, i8 %mask) nounwind
-  ret <2 x double> %res
+declare <4 x double> @llvm.x86.avx512.maskz.vfmaddsub.pd.256(<4 x double>, <4 x double>, <4 x double>, i8)
+
+define <4 x double>@test_int_x86_avx512_maskz_vfmaddsub_pd_256(<4 x double> %x0, <4 x double> %x1, <4 x double> %x2, i8 %x3) {
+; CHECK-LABEL: test_int_x86_avx512_maskz_vfmaddsub_pd_256:
+; CHECK:       ## BB#0:
+; CHECK-NEXT:    movzbl %dil, %eax
+; CHECK-NEXT:    kmovw %eax, %k1
+; CHECK-NEXT:    vmovaps %zmm0, %zmm3
+; CHECK-NEXT:    vfmaddsub213pd %ymm2, %ymm1, %ymm3 {%k1} {z}
+; CHECK-NEXT:    vfmaddsub213pd %ymm2, %ymm1, %ymm0
+; CHECK-NEXT:    vaddpd %ymm0, %ymm3, %ymm0
+; CHECK-NEXT:    retq
+  %res = call <4 x double> @llvm.x86.avx512.maskz.vfmaddsub.pd.256(<4 x double> %x0, <4 x double> %x1, <4 x double> %x2, i8 %x3)
+  %res1 = call <4 x double> @llvm.x86.avx512.maskz.vfmaddsub.pd.256(<4 x double> %x0, <4 x double> %x1, <4 x double> %x2, i8 -1)
+  %res2 = fadd <4 x double> %res, %res1
+  ret <4 x double> %res2
 }
 
-define <2 x double> @test_mask_vfmsubadd128rm_pd(<2 x double> %a0, <2 x double> %a1, <2 x double>* %ptr_a2, i8 %mask) {
-  ; CHECK-LABEL: test_mask_vfmsubadd128rm_pd
-  ; CHECK: vfmsubadd213pd (%rdi), %xmm1, %xmm0 {%k1} ## encoding: [0x62,0xf2,0xf5,0x09,0xa7,0x07]
-  %a2 = load <2 x double>, <2 x double>* %ptr_a2
-  %res = call <2 x double> @llvm.x86.fma.mask.vfmsubadd.pd.128(<2 x double> %a0, <2 x double> %a1, <2 x double> %a2, i8 %mask) nounwind
-  ret <2 x double> %res
+define <4 x float>@test_int_x86_avx512_mask_vfmaddsub_ps_128(<4 x float> %x0, <4 x float> %x1, <4 x float> %x2, i8 %x3) {
+; CHECK-LABEL: test_int_x86_avx512_mask_vfmaddsub_ps_128:
+; CHECK:       ## BB#0:
+; CHECK-NEXT:    movzbl %dil, %eax
+; CHECK-NEXT:    kmovw %eax, %k1
+; CHECK-NEXT:    vmovaps %zmm0, %zmm3
+; CHECK-NEXT:    vfmaddsub213ps %xmm2, %xmm1, %xmm3 {%k1}
+; CHECK-NEXT:    vfmaddsub213ps %xmm2, %xmm1, %xmm0
+; CHECK-NEXT:    vaddps %xmm0, %xmm3, %xmm0
+; CHECK-NEXT:    retq
+  %res = call <4 x float> @llvm.x86.avx512.mask.vfmaddsub.ps.128(<4 x float> %x0, <4 x float> %x1, <4 x float> %x2, i8 %x3)
+  %res1 = call <4 x float> @llvm.x86.avx512.mask.vfmaddsub.ps.128(<4 x float> %x0, <4 x float> %x1, <4 x float> %x2, i8 -1)
+  %res2 = fadd <4 x float> %res, %res1
+  ret <4 x float> %res2
+}
+
+declare <4 x float> @llvm.x86.avx512.mask3.vfmaddsub.ps.128(<4 x float>, <4 x float>, <4 x float>, i8)
+
+define <4 x float>@test_int_x86_avx512_mask3_vfmaddsub_ps_128(<4 x float> %x0, <4 x float> %x1, <4 x float> %x2, i8 %x3) {
+; CHECK-LABEL: test_int_x86_avx512_mask3_vfmaddsub_ps_128:
+; CHECK:       ## BB#0:
+; CHECK-NEXT:    movzbl %dil, %eax
+; CHECK-NEXT:    kmovw %eax, %k1
+; CHECK-NEXT:    vmovaps %zmm2, %zmm3
+; CHECK-NEXT:    vfmaddsub231ps %xmm1, %xmm0, %xmm3 {%k1}
+; CHECK-NEXT:    vfmaddsub213ps %xmm2, %xmm1, %xmm0
+; CHECK-NEXT:    vaddps %xmm0, %xmm3, %xmm0
+; CHECK-NEXT:    retq
+  %res = call <4 x float> @llvm.x86.avx512.mask3.vfmaddsub.ps.128(<4 x float> %x0, <4 x float> %x1, <4 x float> %x2, i8 %x3)
+  %res1 = call <4 x float> @llvm.x86.avx512.mask3.vfmaddsub.ps.128(<4 x float> %x0, <4 x float> %x1, <4 x float> %x2, i8 -1)
+  %res2 = fadd <4 x float> %res, %res1
+  ret <4 x float> %res2
+}
+
+declare <4 x float> @llvm.x86.avx512.maskz.vfmaddsub.ps.128(<4 x float>, <4 x float>, <4 x float>, i8)
+
+define <4 x float>@test_int_x86_avx512_maskz_vfmaddsub_ps_128(<4 x float> %x0, <4 x float> %x1, <4 x float> %x2, i8 %x3) {
+; CHECK-LABEL: test_int_x86_avx512_maskz_vfmaddsub_ps_128:
+; CHECK:       ## BB#0:
+; CHECK-NEXT:    movzbl %dil, %eax
+; CHECK-NEXT:    kmovw %eax, %k1
+; CHECK-NEXT:    vmovaps %zmm0, %zmm3
+; CHECK-NEXT:    vfmaddsub213ps %xmm2, %xmm1, %xmm3 {%k1} {z}
+; CHECK-NEXT:    vfmaddsub213ps %xmm2, %xmm1, %xmm0
+; CHECK-NEXT:    vaddps %xmm0, %xmm3, %xmm0
+; CHECK-NEXT:    retq
+  %res = call <4 x float> @llvm.x86.avx512.maskz.vfmaddsub.ps.128(<4 x float> %x0, <4 x float> %x1, <4 x float> %x2, i8 %x3)
+  %res1 = call <4 x float> @llvm.x86.avx512.maskz.vfmaddsub.ps.128(<4 x float> %x0, <4 x float> %x1, <4 x float> %x2, i8 -1)
+  %res2 = fadd <4 x float> %res, %res1
+  ret <4 x float> %res2
+}
+
+define <8 x float>@test_int_x86_avx512_mask_vfmaddsub_ps_256(<8 x float> %x0, <8 x float> %x1, <8 x float> %x2, i8 %x3) {
+; CHECK-LABEL: test_int_x86_avx512_mask_vfmaddsub_ps_256:
+; CHECK:       ## BB#0:
+; CHECK-NEXT:    movzbl %dil, %eax
+; CHECK-NEXT:    kmovw %eax, %k1
+; CHECK-NEXT:    vmovaps %zmm0, %zmm3
+; CHECK-NEXT:    vfmaddsub213ps %ymm2, %ymm1, %ymm3 {%k1}
+; CHECK-NEXT:    vfmaddsub213ps %ymm2, %ymm1, %ymm0
+; CHECK-NEXT:    vaddps %ymm0, %ymm3, %ymm0
+; CHECK-NEXT:    retq
+  %res = call <8 x float> @llvm.x86.avx512.mask.vfmaddsub.ps.256(<8 x float> %x0, <8 x float> %x1, <8 x float> %x2, i8 %x3)
+  %res1 = call <8 x float> @llvm.x86.avx512.mask.vfmaddsub.ps.256(<8 x float> %x0, <8 x float> %x1, <8 x float> %x2, i8 -1)
+  %res2 = fadd <8 x float> %res, %res1
+  ret <8 x float> %res2
+}
+
+declare <8 x float> @llvm.x86.avx512.mask3.vfmaddsub.ps.256(<8 x float>, <8 x float>, <8 x float>, i8)
+
+define <8 x float>@test_int_x86_avx512_mask3_vfmaddsub_ps_256(<8 x float> %x0, <8 x float> %x1, <8 x float> %x2, i8 %x3) {
+; CHECK-LABEL: test_int_x86_avx512_mask3_vfmaddsub_ps_256:
+; CHECK:       ## BB#0:
+; CHECK-NEXT:    movzbl %dil, %eax
+; CHECK-NEXT:    kmovw %eax, %k1
+; CHECK-NEXT:    vmovaps %zmm2, %zmm3
+; CHECK-NEXT:    vfmaddsub231ps %ymm1, %ymm0, %ymm3 {%k1}
+; CHECK-NEXT:    vfmaddsub213ps %ymm2, %ymm1, %ymm0
+; CHECK-NEXT:    vaddps %ymm0, %ymm3, %ymm0
+; CHECK-NEXT:    retq
+  %res = call <8 x float> @llvm.x86.avx512.mask3.vfmaddsub.ps.256(<8 x float> %x0, <8 x float> %x1, <8 x float> %x2, i8 %x3)
+  %res1 = call <8 x float> @llvm.x86.avx512.mask3.vfmaddsub.ps.256(<8 x float> %x0, <8 x float> %x1, <8 x float> %x2, i8 -1)
+  %res2 = fadd <8 x float> %res, %res1
+  ret <8 x float> %res2
+}
+
+declare <8 x float> @llvm.x86.avx512.maskz.vfmaddsub.ps.256(<8 x float>, <8 x float>, <8 x float>, i8)
+
+define <8 x float>@test_int_x86_avx512_maskz_vfmaddsub_ps_256(<8 x float> %x0, <8 x float> %x1, <8 x float> %x2, i8 %x3) {
+; CHECK-LABEL: test_int_x86_avx512_maskz_vfmaddsub_ps_256:
+; CHECK:       ## BB#0:
+; CHECK-NEXT:    movzbl %dil, %eax
+; CHECK-NEXT:    kmovw %eax, %k1
+; CHECK-NEXT:    vmovaps %zmm0, %zmm3
+; CHECK-NEXT:    vfmaddsub213ps %ymm2, %ymm1, %ymm3 {%k1} {z}
+; CHECK-NEXT:    vfmaddsub213ps %ymm2, %ymm1, %ymm0
+; CHECK-NEXT:    vaddps %ymm0, %ymm3, %ymm0
+; CHECK-NEXT:    retq
+  %res = call <8 x float> @llvm.x86.avx512.maskz.vfmaddsub.ps.256(<8 x float> %x0, <8 x float> %x1, <8 x float> %x2, i8 %x3)
+  %res1 = call <8 x float> @llvm.x86.avx512.maskz.vfmaddsub.ps.256(<8 x float> %x0, <8 x float> %x1, <8 x float> %x2, i8 -1)
+  %res2 = fadd <8 x float> %res, %res1
+  ret <8 x float> %res2
+}
+
+declare <2 x double> @llvm.x86.avx512.mask3.vfmsubadd.pd.128(<2 x double>, <2 x double>, <2 x double>, i8)
+
+define <2 x double>@test_int_x86_avx512_mask3_vfmsubadd_pd_128(<2 x double> %x0, <2 x double> %x1, <2 x double> %x2, i8 %x3) {
+; CHECK-LABEL: test_int_x86_avx512_mask3_vfmsubadd_pd_128:
+; CHECK:       ## BB#0:
+; CHECK-NEXT:    movzbl %dil, %eax
+; CHECK-NEXT:    kmovw %eax, %k1
+; CHECK-NEXT:    vmovaps %zmm2, %zmm3
+; CHECK-NEXT:    vfmsubadd231pd %xmm1, %xmm0, %xmm3 {%k1}
+; CHECK-NEXT:    vfmsubadd213pd %xmm2, %xmm1, %xmm0
+; CHECK-NEXT:    vaddpd %xmm0, %xmm3, %xmm0
+; CHECK-NEXT:    retq
+  %res = call <2 x double> @llvm.x86.avx512.mask3.vfmsubadd.pd.128(<2 x double> %x0, <2 x double> %x1, <2 x double> %x2, i8 %x3)
+  %res1 = call <2 x double> @llvm.x86.avx512.mask3.vfmsubadd.pd.128(<2 x double> %x0, <2 x double> %x1, <2 x double> %x2, i8 -1)
+  %res2=fadd <2 x double> %res, %res1
+  ret <2 x double> %res2
+}
+
+declare <4 x double> @llvm.x86.avx512.mask3.vfmsubadd.pd.256(<4 x double>, <4 x double>, <4 x double>, i8)
+
+define <4 x double>@test_int_x86_avx512_mask3_vfmsubadd_pd_256(<4 x double> %x0, <4 x double> %x1, <4 x double> %x2, i8 %x3) {
+; CHECK-LABEL: test_int_x86_avx512_mask3_vfmsubadd_pd_256:
+; CHECK:       ## BB#0:
+; CHECK-NEXT:    movzbl %dil, %eax
+; CHECK-NEXT:    kmovw %eax, %k1
+; CHECK-NEXT:    vmovaps %zmm2, %zmm3
+; CHECK-NEXT:    vfmsubadd231pd %ymm1, %ymm0, %ymm3 {%k1}
+; CHECK-NEXT:    vfmsubadd213pd %ymm2, %ymm1, %ymm0
+; CHECK-NEXT:    vaddpd %ymm0, %ymm3, %ymm0
+; CHECK-NEXT:    retq
+  %res = call <4 x double> @llvm.x86.avx512.mask3.vfmsubadd.pd.256(<4 x double> %x0, <4 x double> %x1, <4 x double> %x2, i8 %x3)
+  %res1 = call <4 x double> @llvm.x86.avx512.mask3.vfmsubadd.pd.256(<4 x double> %x0, <4 x double> %x1, <4 x double> %x2, i8 -1)
+  %res2=fadd <4 x double> %res, %res1
+  ret <4 x double> %res2
+}
+
+declare <4 x float> @llvm.x86.avx512.mask3.vfmsubadd.ps.128(<4 x float>, <4 x float>, <4 x float>, i8)
+
+define <4 x float>@test_int_x86_avx512_mask3_vfmsubadd_ps_128(<4 x float> %x0, <4 x float> %x1, <4 x float> %x2, i8 %x3) {
+; CHECK-LABEL: test_int_x86_avx512_mask3_vfmsubadd_ps_128:
+; CHECK:       ## BB#0:
+; CHECK-NEXT:    movzbl %dil, %eax
+; CHECK-NEXT:    kmovw %eax, %k1
+; CHECK-NEXT:    vmovaps %zmm2, %zmm3
+; CHECK-NEXT:    vfmsubadd231ps %xmm1, %xmm0, %xmm3 {%k1}
+; CHECK-NEXT:    vfmsubadd213ps %xmm2, %xmm1, %xmm0
+; CHECK-NEXT:    vaddps %xmm0, %xmm3, %xmm0
+; CHECK-NEXT:    retq
+  %res = call <4 x float> @llvm.x86.avx512.mask3.vfmsubadd.ps.128(<4 x float> %x0, <4 x float> %x1, <4 x float> %x2, i8 %x3)
+  %res1 = call <4 x float> @llvm.x86.avx512.mask3.vfmsubadd.ps.128(<4 x float> %x0, <4 x float> %x1, <4 x float> %x2, i8 -1)
+  %res2=fadd <4 x float> %res, %res1
+  ret <4 x float> %res2
 }
-declare <8 x double> @llvm.x86.fma.mask.vfmsubadd.pd.512(<8 x double>, <8 x double>, <8 x double>, i8, i32) nounwind readnone
-define <8 x double> @test_mask_vfmsubaddrm_pd(<8 x double> %a0, <8 x double> %a1, <8 x double>* %ptr_a2, i8 %mask) {
-  ; CHECK-LABEL: test_mask_vfmsubaddrm_pd
-  ; CHECK: vfmsubadd213pd  (%rdi), %zmm1, %zmm0 {%k1} ## encoding: [0x62,0xf2,0xf5,0x49,0xa7,0x07]
-  %a2 = load <8 x double>, <8 x double>* %ptr_a2, align 8
-  %res = call <8 x double> @llvm.x86.fma.mask.vfmsubadd.pd.512(<8 x double> %a0, <8 x double> %a1, <8 x double> %a2, i8 %mask, i32 4) nounwind
-  ret <8 x double> %res
+
+declare <8 x float> @llvm.x86.avx512.mask3.vfmsubadd.ps.256(<8 x float>, <8 x float>, <8 x float>, i8)
+
+define <8 x float>@test_int_x86_avx512_mask3_vfmsubadd_ps_256(<8 x float> %x0, <8 x float> %x1, <8 x float> %x2, i8 %x3) {
+; CHECK-LABEL: test_int_x86_avx512_mask3_vfmsubadd_ps_256:
+; CHECK:       ## BB#0:
+; CHECK-NEXT:    movzbl %dil, %eax
+; CHECK-NEXT:    kmovw %eax, %k1
+; CHECK-NEXT:    vmovaps %zmm2, %zmm3
+; CHECK-NEXT:    vfmsubadd231ps %ymm1, %ymm0, %ymm3 {%k1}
+; CHECK-NEXT:    vfmsubadd213ps %ymm2, %ymm1, %ymm0
+; CHECK-NEXT:    vaddps %ymm0, %ymm3, %ymm0
+; CHECK-NEXT:    retq
+  %res = call <8 x float> @llvm.x86.avx512.mask3.vfmsubadd.ps.256(<8 x float> %x0, <8 x float> %x1, <8 x float> %x2, i8 %x3)
+  %res1 = call <8 x float> @llvm.x86.avx512.mask3.vfmsubadd.ps.256(<8 x float> %x0, <8 x float> %x1, <8 x float> %x2, i8 -1)
+  %res2=fadd <8 x float> %res, %res1
+  ret <8 x float> %res2
 }
 
+
 define <4 x float> @test_mask_vfmadd128_ps_r(<4 x float> %a0, <4 x float> %a1, <4 x float> %a2, i8 %mask) {
   ; CHECK-LABEL: test_mask_vfmadd128_ps_r
   ; CHECK: vfmadd213ps %xmm2, %xmm1, %xmm0 {%k1} ## encoding: [0x62,0xf2,0x75,0x09,0xa8,0xc2]
-  %res = call <4 x float> @llvm.x86.fma.mask.vfmadd.ps.128(<4 x float> %a0, <4 x float> %a1, <4 x float> %a2, i8 %mask) nounwind
+  %res = call <4 x float> @llvm.x86.avx512.mask.vfmadd.ps.128(<4 x float> %a0, <4 x float> %a1, <4 x float> %a2, i8 %mask) nounwind
   ret <4 x float> %res
 }
 
 define <4 x float> @test_mask_vfmadd128_ps_rz(<4 x float> %a0, <4 x float> %a1, <4 x float> %a2) {
   ; CHECK-LABEL: test_mask_vfmadd128_ps_rz
   ; CHECK: vfmadd213ps %xmm2, %xmm1, %xmm0 ## encoding: [0x62,0xf2,0x75,0x08,0xa8,0xc2]
-  %res = call <4 x float> @llvm.x86.fma.mask.vfmadd.ps.128(<4 x float> %a0, <4 x float> %a1, <4 x float> %a2, i8 -1) nounwind
+  %res = call <4 x float> @llvm.x86.avx512.mask.vfmadd.ps.128(<4 x float> %a0, <4 x float> %a1, <4 x float> %a2, i8 -1) nounwind
   ret <4 x float> %res
 }
 
@@ -861,7 +1538,7 @@ define <4 x float> @test_mask_vfmadd128_ps_rmk(<4 x float> %a0, <4 x float> %a1,
   ; CHECK-LABEL: test_mask_vfmadd128_ps_rmk
   ; CHECK: vfmadd213ps	(%rdi), %xmm1, %xmm0 {%k1} ## encoding: [0x62,0xf2,0x75,0x09,0xa8,0x07]
   %a2 = load <4 x float>, <4 x float>* %ptr_a2
-  %res = call <4 x float> @llvm.x86.fma.mask.vfmadd.ps.128(<4 x float> %a0, <4 x float> %a1, <4 x float> %a2, i8 %mask) nounwind
+  %res = call <4 x float> @llvm.x86.avx512.mask.vfmadd.ps.128(<4 x float> %a0, <4 x float> %a1, <4 x float> %a2, i8 %mask) nounwind
   ret <4 x float> %res
 }
 
@@ -869,7 +1546,7 @@ define <4 x float> @test_mask_vfmadd128_ps_rmka(<4 x float> %a0, <4 x float> %a1
   ; CHECK-LABEL: test_mask_vfmadd128_ps_rmka
   ; CHECK: vfmadd213ps     (%rdi), %xmm1, %xmm0 {%k1} ## encoding: [0x62,0xf2,0x75,0x09,0xa8,0x07]
   %a2 = load <4 x float>, <4 x float>* %ptr_a2, align 8
-  %res = call <4 x float> @llvm.x86.fma.mask.vfmadd.ps.128(<4 x float> %a0, <4 x float> %a1, <4 x float> %a2, i8 %mask) nounwind
+  %res = call <4 x float> @llvm.x86.avx512.mask.vfmadd.ps.128(<4 x float> %a0, <4 x float> %a1, <4 x float> %a2, i8 %mask) nounwind
   ret <4 x float> %res
 }
 
@@ -877,7 +1554,7 @@ define <4 x float> @test_mask_vfmadd128_ps_rmkz(<4 x float> %a0, <4 x float> %a1
   ; CHECK-LABEL: test_mask_vfmadd128_ps_rmkz
   ; CHECK: vfmadd213ps	(%rdi), %xmm1, %xmm0 ## encoding: [0xc4,0xe2,0x71,0xa8,0x07]
   %a2 = load <4 x float>, <4 x float>* %ptr_a2
-  %res = call <4 x float> @llvm.x86.fma.mask.vfmadd.ps.128(<4 x float> %a0, <4 x float> %a1, <4 x float> %a2, i8 -1) nounwind
+  %res = call <4 x float> @llvm.x86.avx512.mask.vfmadd.ps.128(<4 x float> %a0, <4 x float> %a1, <4 x float> %a2, i8 -1) nounwind
   ret <4 x float> %res
 }
 
@@ -885,7 +1562,7 @@ define <4 x float> @test_mask_vfmadd128_ps_rmkza(<4 x float> %a0, <4 x float> %a
   ; CHECK-LABEL: test_mask_vfmadd128_ps_rmkza
   ; CHECK: vfmadd213ps	(%rdi), %xmm1, %xmm0 ## encoding: [0xc4,0xe2,0x71,0xa8,0x07]
   %a2 = load <4 x float>, <4 x float>* %ptr_a2, align 4
-  %res = call <4 x float> @llvm.x86.fma.mask.vfmadd.ps.128(<4 x float> %a0, <4 x float> %a1, <4 x float> %a2, i8 -1) nounwind
+  %res = call <4 x float> @llvm.x86.avx512.mask.vfmadd.ps.128(<4 x float> %a0, <4 x float> %a1, <4 x float> %a2, i8 -1) nounwind
   ret <4 x float> %res
 }
 
@@ -897,7 +1574,7 @@ define <4 x float> @test_mask_vfmadd128_ps_rmb(<4 x float> %a0, <4 x float> %a1,
   %vecinit2.i = insertelement <4 x float> %vecinit.i, float %q, i32 1
   %vecinit4.i = insertelement <4 x float> %vecinit2.i, float %q, i32 2
   %vecinit6.i = insertelement <4 x float> %vecinit4.i, float %q, i32 3
-  %res = call <4 x float> @llvm.x86.fma.mask.vfmadd.ps.128(<4 x float> %a0, <4 x float> %a1, <4 x float> %vecinit6.i, i8 %mask) nounwind
+  %res = call <4 x float> @llvm.x86.avx512.mask.vfmadd.ps.128(<4 x float> %a0, <4 x float> %a1, <4 x float> %vecinit6.i, i8 %mask) nounwind
   ret <4 x float> %res
 }
 
@@ -909,7 +1586,7 @@ define <4 x float> @test_mask_vfmadd128_ps_rmba(<4 x float> %a0, <4 x float> %a1
   %vecinit2.i = insertelement <4 x float> %vecinit.i, float %q, i32 1
   %vecinit4.i = insertelement <4 x float> %vecinit2.i, float %q, i32 2
   %vecinit6.i = insertelement <4 x float> %vecinit4.i, float %q, i32 3
-  %res = call <4 x float> @llvm.x86.fma.mask.vfmadd.ps.128(<4 x float> %a0, <4 x float> %a1, <4 x float> %vecinit6.i, i8 %mask) nounwind
+  %res = call <4 x float> @llvm.x86.avx512.mask.vfmadd.ps.128(<4 x float> %a0, <4 x float> %a1, <4 x float> %vecinit6.i, i8 %mask) nounwind
   ret <4 x float> %res
 }
 
@@ -921,7 +1598,7 @@ define <4 x float> @test_mask_vfmadd128_ps_rmbz(<4 x float> %a0, <4 x float> %a1
   %vecinit2.i = insertelement <4 x float> %vecinit.i, float %q, i32 1
   %vecinit4.i = insertelement <4 x float> %vecinit2.i, float %q, i32 2
   %vecinit6.i = insertelement <4 x float> %vecinit4.i, float %q, i32 3
-  %res = call <4 x float> @llvm.x86.fma.mask.vfmadd.ps.128(<4 x float> %a0, <4 x float> %a1, <4 x float> %vecinit6.i, i8 -1) nounwind
+  %res = call <4 x float> @llvm.x86.avx512.mask.vfmadd.ps.128(<4 x float> %a0, <4 x float> %a1, <4 x float> %vecinit6.i, i8 -1) nounwind
   ret <4 x float> %res
 }
 
@@ -933,21 +1610,21 @@ define <4 x float> @test_mask_vfmadd128_ps_rmbza(<4 x float> %a0, <4 x float> %a
   %vecinit2.i = insertelement <4 x float> %vecinit.i, float %q, i32 1
   %vecinit4.i = insertelement <4 x float> %vecinit2.i, float %q, i32 2
   %vecinit6.i = insertelement <4 x float> %vecinit4.i, float %q, i32 3
-  %res = call <4 x float> @llvm.x86.fma.mask.vfmadd.ps.128(<4 x float> %a0, <4 x float> %a1, <4 x float> %vecinit6.i, i8 -1) nounwind
+  %res = call <4 x float> @llvm.x86.avx512.mask.vfmadd.ps.128(<4 x float> %a0, <4 x float> %a1, <4 x float> %vecinit6.i, i8 -1) nounwind
   ret <4 x float> %res
 }
 
 define <2 x double> @test_mask_vfmadd128_pd_r(<2 x double> %a0, <2 x double> %a1, <2 x double> %a2, i8 %mask) {
   ; CHECK-LABEL: test_mask_vfmadd128_pd_r
   ; CHECK: vfmadd213pd %xmm2, %xmm1, %xmm0 {%k1} ## encoding: [0x62,0xf2,0xf5,0x09,0xa8,0xc2]
-  %res = call <2 x double> @llvm.x86.fma.mask.vfmadd.pd.128(<2 x double> %a0, <2 x double> %a1, <2 x double> %a2, i8 %mask) nounwind
+  %res = call <2 x double> @llvm.x86.avx512.mask.vfmadd.pd.128(<2 x double> %a0, <2 x double> %a1, <2 x double> %a2, i8 %mask) nounwind
   ret <2 x double> %res
 }
 
 define <2 x double> @test_mask_vfmadd128_pd_rz(<2 x double> %a0, <2 x double> %a1, <2 x double> %a2) {
   ; CHECK-LABEL: test_mask_vfmadd128_pd_rz
   ; CHECK: vfmadd213pd %xmm2, %xmm1, %xmm0 ## encoding: [0x62,0xf2,0xf5,0x08,0xa8,0xc2]
-  %res = call <2 x double> @llvm.x86.fma.mask.vfmadd.pd.128(<2 x double> %a0, <2 x double> %a1, <2 x double> %a2, i8 -1) nounwind
+  %res = call <2 x double> @llvm.x86.avx512.mask.vfmadd.pd.128(<2 x double> %a0, <2 x double> %a1, <2 x double> %a2, i8 -1) nounwind
   ret <2 x double> %res
 }
 
@@ -955,7 +1632,7 @@ define <2 x double> @test_mask_vfmadd128_pd_rmk(<2 x double> %a0, <2 x double> %
   ; CHECK-LABEL: test_mask_vfmadd128_pd_rmk
   ; CHECK: vfmadd213pd	(%rdi), %xmm1, %xmm0 {%k1} ## encoding: [0x62,0xf2,0xf5,0x09,0xa8,0x07]
   %a2 = load <2 x double>, <2 x double>* %ptr_a2
-  %res = call <2 x double> @llvm.x86.fma.mask.vfmadd.pd.128(<2 x double> %a0, <2 x double> %a1, <2 x double> %a2, i8 %mask) nounwind
+  %res = call <2 x double> @llvm.x86.avx512.mask.vfmadd.pd.128(<2 x double> %a0, <2 x double> %a1, <2 x double> %a2, i8 %mask) nounwind
   ret <2 x double> %res
 }
 
@@ -963,21 +1640,21 @@ define <2 x double> @test_mask_vfmadd128_pd_rmkz(<2 x double> %a0, <2 x double>
   ; CHECK-LABEL: test_mask_vfmadd128_pd_rmkz
   ; CHECK: vfmadd213pd	(%rdi), %xmm1, %xmm0 ## encoding: [0xc4,0xe2,0xf1,0xa8,0x07]
   %a2 = load <2 x double>, <2 x double>* %ptr_a2
-  %res = call <2 x double> @llvm.x86.fma.mask.vfmadd.pd.128(<2 x double> %a0, <2 x double> %a1, <2 x double> %a2, i8 -1) nounwind
+  %res = call <2 x double> @llvm.x86.avx512.mask.vfmadd.pd.128(<2 x double> %a0, <2 x double> %a1, <2 x double> %a2, i8 -1) nounwind
   ret <2 x double> %res
 }
 
 define <4 x double> @test_mask_vfmadd256_pd_r(<4 x double> %a0, <4 x double> %a1, <4 x double> %a2, i8 %mask) {
   ; CHECK-LABEL: test_mask_vfmadd256_pd_r
   ; CHECK: vfmadd213pd %ymm2, %ymm1, %ymm0 {%k1} ## encoding: [0x62,0xf2,0xf5,0x29,0xa8,0xc2]
-  %res = call <4 x double> @llvm.x86.fma.mask.vfmadd.pd.256(<4 x double> %a0, <4 x double> %a1, <4 x double> %a2, i8 %mask) nounwind
+  %res = call <4 x double> @llvm.x86.avx512.mask.vfmadd.pd.256(<4 x double> %a0, <4 x double> %a1, <4 x double> %a2, i8 %mask) nounwind
   ret <4 x double> %res
 }
 
 define <4 x double> @test_mask_vfmadd256_pd_rz(<4 x double> %a0, <4 x double> %a1, <4 x double> %a2) {
   ; CHECK-LABEL: test_mask_vfmadd256_pd_rz
   ; CHECK: vfmadd213pd %ymm2, %ymm1, %ymm0 ## encoding: [0x62,0xf2,0xf5,0x28,0xa8,0xc2]
-  %res = call <4 x double> @llvm.x86.fma.mask.vfmadd.pd.256(<4 x double> %a0, <4 x double> %a1, <4 x double> %a2, i8 -1) nounwind
+  %res = call <4 x double> @llvm.x86.avx512.mask.vfmadd.pd.256(<4 x double> %a0, <4 x double> %a1, <4 x double> %a2, i8 -1) nounwind
   ret <4 x double> %res
 }
 
@@ -985,7 +1662,7 @@ define <4 x double> @test_mask_vfmadd256_pd_rmk(<4 x double> %a0, <4 x double> %
   ; CHECK-LABEL: test_mask_vfmadd256_pd_rmk
   ; CHECK: vfmadd213pd	(%rdi), %ymm1, %ymm0 {%k1} ## encoding: [0x62,0xf2,0xf5,0x29,0xa8,0x07]
   %a2 = load <4 x double>, <4 x double>* %ptr_a2
-  %res = call <4 x double> @llvm.x86.fma.mask.vfmadd.pd.256(<4 x double> %a0, <4 x double> %a1, <4 x double> %a2, i8 %mask) nounwind
+  %res = call <4 x double> @llvm.x86.avx512.mask.vfmadd.pd.256(<4 x double> %a0, <4 x double> %a1, <4 x double> %a2, i8 %mask) nounwind
   ret <4 x double> %res
 }
 
@@ -993,6 +1670,2527 @@ define <4 x double> @test_mask_vfmadd256_pd_rmkz(<4 x double> %a0, <4 x double>
   ; CHECK-LABEL: test_mask_vfmadd256_pd_rmkz
   ; CHECK: vfmadd213pd	(%rdi), %ymm1, %ymm0 ## encoding: [0xc4,0xe2,0xf5,0xa8,0x07]
   %a2 = load <4 x double>, <4 x double>* %ptr_a2
-  %res = call <4 x double> @llvm.x86.fma.mask.vfmadd.pd.256(<4 x double> %a0, <4 x double> %a1, <4 x double> %a2, i8 -1) nounwind
+  %res = call <4 x double> @llvm.x86.avx512.mask.vfmadd.pd.256(<4 x double> %a0, <4 x double> %a1, <4 x double> %a2, i8 -1) nounwind
   ret <4 x double> %res
 }
+define <8 x i16> @test_mask_add_epi16_rr_128(<8 x i16> %a, <8 x i16> %b) {
+  ;CHECK-LABEL: test_mask_add_epi16_rr_128
+  ;CHECK: vpaddw %xmm1, %xmm0, %xmm0     ## encoding: [0x62,0xf1,0x7d,0x08,0xfd,0xc1]
+  %res = call <8 x i16> @llvm.x86.avx512.mask.padd.w.128(<8 x i16> %a, <8 x i16> %b, <8 x i16> zeroinitializer, i8 -1)
+  ret <8 x i16> %res
+}
+
+define <8 x i16> @test_mask_add_epi16_rrk_128(<8 x i16> %a, <8 x i16> %b, <8 x i16> %passThru, i8 %mask) {
+  ;CHECK-LABEL: test_mask_add_epi16_rrk_128
+  ;CHECK: vpaddw %xmm1, %xmm0, %xmm2 {%k1} ## encoding: [0x62,0xf1,0x7d,0x09,0xfd,0xd1]
+  %res = call <8 x i16> @llvm.x86.avx512.mask.padd.w.128(<8 x i16> %a, <8 x i16> %b, <8 x i16> %passThru, i8 %mask)
+  ret <8 x i16> %res
+}
+
+define <8 x i16> @test_mask_add_epi16_rrkz_128(<8 x i16> %a, <8 x i16> %b, i8 %mask) {
+  ;CHECK-LABEL: test_mask_add_epi16_rrkz_128
+  ;CHECK: vpaddw %xmm1, %xmm0, %xmm0 {%k1} {z} ## encoding: [0x62,0xf1,0x7d,0x89,0xfd,0xc1]
+  %res = call <8 x i16> @llvm.x86.avx512.mask.padd.w.128(<8 x i16> %a, <8 x i16> %b, <8 x i16> zeroinitializer, i8 %mask)
+  ret <8 x i16> %res
+}
+
+define <8 x i16> @test_mask_add_epi16_rm_128(<8 x i16> %a, <8 x i16>* %ptr_b) {
+  ;CHECK-LABEL: test_mask_add_epi16_rm_128
+  ;CHECK: vpaddw (%rdi), %xmm0, %xmm0    ## encoding: [0x62,0xf1,0x7d,0x08,0xfd,0x07]
+  %b = load <8 x i16>, <8 x i16>* %ptr_b
+  %res = call <8 x i16> @llvm.x86.avx512.mask.padd.w.128(<8 x i16> %a, <8 x i16> %b, <8 x i16> zeroinitializer, i8 -1)
+  ret <8 x i16> %res
+}
+
+define <8 x i16> @test_mask_add_epi16_rmk_128(<8 x i16> %a, <8 x i16>* %ptr_b, <8 x i16> %passThru, i8 %mask) {
+  ;CHECK-LABEL: test_mask_add_epi16_rmk_128
+  ;CHECK: vpaddw (%rdi), %xmm0, %xmm1 {%k1} ## encoding: [0x62,0xf1,0x7d,0x09,0xfd,0x0f]
+  %b = load <8 x i16>, <8 x i16>* %ptr_b
+  %res = call <8 x i16> @llvm.x86.avx512.mask.padd.w.128(<8 x i16> %a, <8 x i16> %b, <8 x i16> %passThru, i8 %mask)
+  ret <8 x i16> %res
+}
+
+define <8 x i16> @test_mask_add_epi16_rmkz_128(<8 x i16> %a, <8 x i16>* %ptr_b, i8 %mask) {
+  ;CHECK-LABEL: test_mask_add_epi16_rmkz_128
+  ;CHECK: vpaddw (%rdi), %xmm0, %xmm0 {%k1} {z} ## encoding: [0x62,0xf1,0x7d,0x89,0xfd,0x07]
+  %b = load <8 x i16>, <8 x i16>* %ptr_b
+  %res = call <8 x i16> @llvm.x86.avx512.mask.padd.w.128(<8 x i16> %a, <8 x i16> %b, <8 x i16> zeroinitializer, i8 %mask)
+  ret <8 x i16> %res
+}
+
+declare <8 x i16> @llvm.x86.avx512.mask.padd.w.128(<8 x i16>, <8 x i16>, <8 x i16>, i8)
+
+define <16 x i16> @test_mask_add_epi16_rr_256(<16 x i16> %a, <16 x i16> %b) {
+  ;CHECK-LABEL: test_mask_add_epi16_rr_256
+  ;CHECK: vpaddw %ymm1, %ymm0, %ymm0     ## encoding: [0x62,0xf1,0x7d,0x28,0xfd,0xc1]
+  %res = call <16 x i16> @llvm.x86.avx512.mask.padd.w.256(<16 x i16> %a, <16 x i16> %b, <16 x i16> zeroinitializer, i16 -1)
+  ret <16 x i16> %res
+}
+
+define <16 x i16> @test_mask_add_epi16_rrk_256(<16 x i16> %a, <16 x i16> %b, <16 x i16> %passThru, i16 %mask) {
+  ;CHECK-LABEL: test_mask_add_epi16_rrk_256
+  ;CHECK: vpaddw %ymm1, %ymm0, %ymm2 {%k1} ## encoding: [0x62,0xf1,0x7d,0x29,0xfd,0xd1]
+  %res = call <16 x i16> @llvm.x86.avx512.mask.padd.w.256(<16 x i16> %a, <16 x i16> %b, <16 x i16> %passThru, i16 %mask)
+  ret <16 x i16> %res
+}
+
+define <16 x i16> @test_mask_add_epi16_rrkz_256(<16 x i16> %a, <16 x i16> %b, i16 %mask) {
+  ;CHECK-LABEL: test_mask_add_epi16_rrkz_256
+  ;CHECK: vpaddw %ymm1, %ymm0, %ymm0 {%k1} {z} ## encoding: [0x62,0xf1,0x7d,0xa9,0xfd,0xc1]
+  %res = call <16 x i16> @llvm.x86.avx512.mask.padd.w.256(<16 x i16> %a, <16 x i16> %b, <16 x i16> zeroinitializer, i16 %mask)
+  ret <16 x i16> %res
+}
+
+define <16 x i16> @test_mask_add_epi16_rm_256(<16 x i16> %a, <16 x i16>* %ptr_b) {
+  ;CHECK-LABEL: test_mask_add_epi16_rm_256
+  ;CHECK: vpaddw (%rdi), %ymm0, %ymm0    ## encoding: [0x62,0xf1,0x7d,0x28,0xfd,0x07]
+  %b = load <16 x i16>, <16 x i16>* %ptr_b
+  %res = call <16 x i16> @llvm.x86.avx512.mask.padd.w.256(<16 x i16> %a, <16 x i16> %b, <16 x i16> zeroinitializer, i16 -1)
+  ret <16 x i16> %res
+}
+
+define <16 x i16> @test_mask_add_epi16_rmk_256(<16 x i16> %a, <16 x i16>* %ptr_b, <16 x i16> %passThru, i16 %mask) {
+  ;CHECK-LABEL: test_mask_add_epi16_rmk_256
+  ;CHECK: vpaddw (%rdi), %ymm0, %ymm1 {%k1} ## encoding: [0x62,0xf1,0x7d,0x29,0xfd,0x0f]
+  %b = load <16 x i16>, <16 x i16>* %ptr_b
+  %res = call <16 x i16> @llvm.x86.avx512.mask.padd.w.256(<16 x i16> %a, <16 x i16> %b, <16 x i16> %passThru, i16 %mask)
+  ret <16 x i16> %res
+}
+
+define <16 x i16> @test_mask_add_epi16_rmkz_256(<16 x i16> %a, <16 x i16>* %ptr_b, i16 %mask) {
+  ;CHECK-LABEL: test_mask_add_epi16_rmkz_256
+  ;CHECK: vpaddw (%rdi), %ymm0, %ymm0 {%k1} {z} ## encoding: [0x62,0xf1,0x7d,0xa9,0xfd,0x07]
+  %b = load <16 x i16>, <16 x i16>* %ptr_b
+  %res = call <16 x i16> @llvm.x86.avx512.mask.padd.w.256(<16 x i16> %a, <16 x i16> %b, <16 x i16> zeroinitializer, i16 %mask)
+  ret <16 x i16> %res
+}
+
+declare <16 x i16> @llvm.x86.avx512.mask.padd.w.256(<16 x i16>, <16 x i16>, <16 x i16>, i16)
+
+define <8 x i16> @test_mask_sub_epi16_rr_128(<8 x i16> %a, <8 x i16> %b) {
+  ;CHECK-LABEL: test_mask_sub_epi16_rr_128
+  ;CHECK: vpsubw %xmm1, %xmm0, %xmm0     ## encoding: [0x62,0xf1,0x7d,0x08,0xf9,0xc1]
+  %res = call <8 x i16> @llvm.x86.avx512.mask.psub.w.128(<8 x i16> %a, <8 x i16> %b, <8 x i16> zeroinitializer, i8 -1)
+  ret <8 x i16> %res
+}
+
+define <8 x i16> @test_mask_sub_epi16_rrk_128(<8 x i16> %a, <8 x i16> %b, <8 x i16> %passThru, i8 %mask) {
+  ;CHECK-LABEL: test_mask_sub_epi16_rrk_128
+  ;CHECK: vpsubw %xmm1, %xmm0, %xmm2 {%k1} ## encoding: [0x62,0xf1,0x7d,0x09,0xf9,0xd1]
+  %res = call <8 x i16> @llvm.x86.avx512.mask.psub.w.128(<8 x i16> %a, <8 x i16> %b, <8 x i16> %passThru, i8 %mask)
+  ret <8 x i16> %res
+}
+
+define <8 x i16> @test_mask_sub_epi16_rrkz_128(<8 x i16> %a, <8 x i16> %b, i8 %mask) {
+  ;CHECK-LABEL: test_mask_sub_epi16_rrkz_128
+  ;CHECK: vpsubw %xmm1, %xmm0, %xmm0 {%k1} {z} ## encoding: [0x62,0xf1,0x7d,0x89,0xf9,0xc1]
+  %res = call <8 x i16> @llvm.x86.avx512.mask.psub.w.128(<8 x i16> %a, <8 x i16> %b, <8 x i16> zeroinitializer, i8 %mask)
+  ret <8 x i16> %res
+}
+
+define <8 x i16> @test_mask_sub_epi16_rm_128(<8 x i16> %a, <8 x i16>* %ptr_b) {
+  ;CHECK-LABEL: test_mask_sub_epi16_rm_128
+  ;CHECK: vpsubw (%rdi), %xmm0, %xmm0    ## encoding: [0x62,0xf1,0x7d,0x08,0xf9,0x07]
+  %b = load <8 x i16>, <8 x i16>* %ptr_b
+  %res = call <8 x i16> @llvm.x86.avx512.mask.psub.w.128(<8 x i16> %a, <8 x i16> %b, <8 x i16> zeroinitializer, i8 -1)
+  ret <8 x i16> %res
+}
+
+define <8 x i16> @test_mask_sub_epi16_rmk_128(<8 x i16> %a, <8 x i16>* %ptr_b, <8 x i16> %passThru, i8 %mask) {
+  ;CHECK-LABEL: test_mask_sub_epi16_rmk_128
+  ;CHECK: vpsubw (%rdi), %xmm0, %xmm1 {%k1} ## encoding: [0x62,0xf1,0x7d,0x09,0xf9,0x0f]
+  %b = load <8 x i16>, <8 x i16>* %ptr_b
+  %res = call <8 x i16> @llvm.x86.avx512.mask.psub.w.128(<8 x i16> %a, <8 x i16> %b, <8 x i16> %passThru, i8 %mask)
+  ret <8 x i16> %res
+}
+
+define <8 x i16> @test_mask_sub_epi16_rmkz_128(<8 x i16> %a, <8 x i16>* %ptr_b, i8 %mask) {
+  ;CHECK-LABEL: test_mask_sub_epi16_rmkz_128
+  ;CHECK: vpsubw (%rdi), %xmm0, %xmm0 {%k1} {z} ## encoding: [0x62,0xf1,0x7d,0x89,0xf9,0x07]
+  %b = load <8 x i16>, <8 x i16>* %ptr_b
+  %res = call <8 x i16> @llvm.x86.avx512.mask.psub.w.128(<8 x i16> %a, <8 x i16> %b, <8 x i16> zeroinitializer, i8 %mask)
+  ret <8 x i16> %res
+}
+
+declare <8 x i16> @llvm.x86.avx512.mask.psub.w.128(<8 x i16>, <8 x i16>, <8 x i16>, i8)
+
+define <16 x i16> @test_mask_sub_epi16_rr_256(<16 x i16> %a, <16 x i16> %b) {
+  ;CHECK-LABEL: test_mask_sub_epi16_rr_256
+  ;CHECK: vpsubw %ymm1, %ymm0, %ymm0     ## encoding: [0x62,0xf1,0x7d,0x28,0xf9,0xc1]
+  %res = call <16 x i16> @llvm.x86.avx512.mask.psub.w.256(<16 x i16> %a, <16 x i16> %b, <16 x i16> zeroinitializer, i16 -1)
+  ret <16 x i16> %res
+}
+
+define <16 x i16> @test_mask_sub_epi16_rrk_256(<16 x i16> %a, <16 x i16> %b, <16 x i16> %passThru, i16 %mask) {
+  ;CHECK-LABEL: test_mask_sub_epi16_rrk_256
+  ;CHECK: vpsubw %ymm1, %ymm0, %ymm2 {%k1} ## encoding: [0x62,0xf1,0x7d,0x29,0xf9,0xd1]
+  %res = call <16 x i16> @llvm.x86.avx512.mask.psub.w.256(<16 x i16> %a, <16 x i16> %b, <16 x i16> %passThru, i16 %mask)
+  ret <16 x i16> %res
+}
+
+define <16 x i16> @test_mask_sub_epi16_rrkz_256(<16 x i16> %a, <16 x i16> %b, i16 %mask) {
+  ;CHECK-LABEL: test_mask_sub_epi16_rrkz_256
+  ;CHECK: vpsubw %ymm1, %ymm0, %ymm0 {%k1} {z} ## encoding: [0x62,0xf1,0x7d,0xa9,0xf9,0xc1]
+  %res = call <16 x i16> @llvm.x86.avx512.mask.psub.w.256(<16 x i16> %a, <16 x i16> %b, <16 x i16> zeroinitializer, i16 %mask)
+  ret <16 x i16> %res
+}
+
+define <16 x i16> @test_mask_sub_epi16_rm_256(<16 x i16> %a, <16 x i16>* %ptr_b) {
+  ;CHECK-LABEL: test_mask_sub_epi16_rm_256
+  ;CHECK: vpsubw (%rdi), %ymm0, %ymm0    ## encoding: [0x62,0xf1,0x7d,0x28,0xf9,0x07]
+  %b = load <16 x i16>, <16 x i16>* %ptr_b
+  %res = call <16 x i16> @llvm.x86.avx512.mask.psub.w.256(<16 x i16> %a, <16 x i16> %b, <16 x i16> zeroinitializer, i16 -1)
+  ret <16 x i16> %res
+}
+
+define <16 x i16> @test_mask_sub_epi16_rmk_256(<16 x i16> %a, <16 x i16>* %ptr_b, <16 x i16> %passThru, i16 %mask) {
+  ;CHECK-LABEL: test_mask_sub_epi16_rmk_256
+  ;CHECK: vpsubw (%rdi), %ymm0, %ymm1 {%k1} ## encoding: [0x62,0xf1,0x7d,0x29,0xf9,0x0f]
+  %b = load <16 x i16>, <16 x i16>* %ptr_b
+  %res = call <16 x i16> @llvm.x86.avx512.mask.psub.w.256(<16 x i16> %a, <16 x i16> %b, <16 x i16> %passThru, i16 %mask)
+  ret <16 x i16> %res
+}
+
+define <16 x i16> @test_mask_sub_epi16_rmkz_256(<16 x i16> %a, <16 x i16>* %ptr_b, i16 %mask) {
+  ;CHECK-LABEL: test_mask_sub_epi16_rmkz_256
+  ;CHECK: vpsubw (%rdi), %ymm0, %ymm0 {%k1} {z} ## encoding: [0x62,0xf1,0x7d,0xa9,0xf9,0x07]
+  %b = load <16 x i16>, <16 x i16>* %ptr_b
+  %res = call <16 x i16> @llvm.x86.avx512.mask.psub.w.256(<16 x i16> %a, <16 x i16> %b, <16 x i16> zeroinitializer, i16 %mask)
+  ret <16 x i16> %res
+}
+
+declare <16 x i16> @llvm.x86.avx512.mask.psub.w.256(<16 x i16>, <16 x i16>, <16 x i16>, i16)
+
+define <32 x i16> @test_mask_add_epi16_rr_512(<32 x i16> %a, <32 x i16> %b) {
+  ;CHECK-LABEL: test_mask_add_epi16_rr_512
+  ;CHECK: vpaddw %zmm1, %zmm0, %zmm0     ## encoding: [0x62,0xf1,0x7d,0x48,0xfd,0xc1]
+  %res = call <32 x i16> @llvm.x86.avx512.mask.padd.w.512(<32 x i16> %a, <32 x i16> %b, <32 x i16> zeroinitializer, i32 -1)
+  ret <32 x i16> %res
+}
+
+define <32 x i16> @test_mask_add_epi16_rrk_512(<32 x i16> %a, <32 x i16> %b, <32 x i16> %passThru, i32 %mask) {
+  ;CHECK-LABEL: test_mask_add_epi16_rrk_512
+  ;CHECK: vpaddw %zmm1, %zmm0, %zmm2 {%k1} ## encoding: [0x62,0xf1,0x7d,0x49,0xfd,0xd1]
+  %res = call <32 x i16> @llvm.x86.avx512.mask.padd.w.512(<32 x i16> %a, <32 x i16> %b, <32 x i16> %passThru, i32 %mask)
+  ret <32 x i16> %res
+}
+
+define <32 x i16> @test_mask_add_epi16_rrkz_512(<32 x i16> %a, <32 x i16> %b, i32 %mask) {
+  ;CHECK-LABEL: test_mask_add_epi16_rrkz_512
+  ;CHECK: vpaddw %zmm1, %zmm0, %zmm0 {%k1} {z} ## encoding: [0x62,0xf1,0x7d,0xc9,0xfd,0xc1]
+  %res = call <32 x i16> @llvm.x86.avx512.mask.padd.w.512(<32 x i16> %a, <32 x i16> %b, <32 x i16> zeroinitializer, i32 %mask)
+  ret <32 x i16> %res
+}
+
+define <32 x i16> @test_mask_add_epi16_rm_512(<32 x i16> %a, <32 x i16>* %ptr_b) {
+  ;CHECK-LABEL: test_mask_add_epi16_rm_512
+  ;CHECK: vpaddw (%rdi), %zmm0, %zmm0    ## encoding: [0x62,0xf1,0x7d,0x48,0xfd,0x07]
+  %b = load <32 x i16>, <32 x i16>* %ptr_b
+  %res = call <32 x i16> @llvm.x86.avx512.mask.padd.w.512(<32 x i16> %a, <32 x i16> %b, <32 x i16> zeroinitializer, i32 -1)
+  ret <32 x i16> %res
+}
+
+define <32 x i16> @test_mask_add_epi16_rmk_512(<32 x i16> %a, <32 x i16>* %ptr_b, <32 x i16> %passThru, i32 %mask) {
+  ;CHECK-LABEL: test_mask_add_epi16_rmk_512
+  ;CHECK: vpaddw (%rdi), %zmm0, %zmm1 {%k1} ## encoding: [0x62,0xf1,0x7d,0x49,0xfd,0x0f]
+  %b = load <32 x i16>, <32 x i16>* %ptr_b
+  %res = call <32 x i16> @llvm.x86.avx512.mask.padd.w.512(<32 x i16> %a, <32 x i16> %b, <32 x i16> %passThru, i32 %mask)
+  ret <32 x i16> %res
+}
+
+define <32 x i16> @test_mask_add_epi16_rmkz_512(<32 x i16> %a, <32 x i16>* %ptr_b, i32 %mask) {
+  ;CHECK-LABEL: test_mask_add_epi16_rmkz_512
+  ;CHECK: vpaddw (%rdi), %zmm0, %zmm0 {%k1} {z} ## encoding: [0x62,0xf1,0x7d,0xc9,0xfd,0x07]
+  %b = load <32 x i16>, <32 x i16>* %ptr_b
+  %res = call <32 x i16> @llvm.x86.avx512.mask.padd.w.512(<32 x i16> %a, <32 x i16> %b, <32 x i16> zeroinitializer, i32 %mask)
+  ret <32 x i16> %res
+}
+
+declare <32 x i16> @llvm.x86.avx512.mask.padd.w.512(<32 x i16>, <32 x i16>, <32 x i16>, i32)
+
+define <32 x i16> @test_mask_sub_epi16_rr_512(<32 x i16> %a, <32 x i16> %b) {
+  ;CHECK-LABEL: test_mask_sub_epi16_rr_512
+  ;CHECK: vpsubw %zmm1, %zmm0, %zmm0     ## encoding: [0x62,0xf1,0x7d,0x48,0xf9,0xc1]
+  %res = call <32 x i16> @llvm.x86.avx512.mask.psub.w.512(<32 x i16> %a, <32 x i16> %b, <32 x i16> zeroinitializer, i32 -1)
+  ret <32 x i16> %res
+}
+
+define <32 x i16> @test_mask_sub_epi16_rrk_512(<32 x i16> %a, <32 x i16> %b, <32 x i16> %passThru, i32 %mask) {
+  ;CHECK-LABEL: test_mask_sub_epi16_rrk_512
+  ;CHECK: vpsubw %zmm1, %zmm0, %zmm2 {%k1} ## encoding: [0x62,0xf1,0x7d,0x49,0xf9,0xd1]
+  %res = call <32 x i16> @llvm.x86.avx512.mask.psub.w.512(<32 x i16> %a, <32 x i16> %b, <32 x i16> %passThru, i32 %mask)
+  ret <32 x i16> %res
+}
+
+define <32 x i16> @test_mask_sub_epi16_rrkz_512(<32 x i16> %a, <32 x i16> %b, i32 %mask) {
+  ;CHECK-LABEL: test_mask_sub_epi16_rrkz_512
+  ;CHECK: vpsubw %zmm1, %zmm0, %zmm0 {%k1} {z} ## encoding: [0x62,0xf1,0x7d,0xc9,0xf9,0xc1]
+  %res = call <32 x i16> @llvm.x86.avx512.mask.psub.w.512(<32 x i16> %a, <32 x i16> %b, <32 x i16> zeroinitializer, i32 %mask)
+  ret <32 x i16> %res
+}
+
+define <32 x i16> @test_mask_sub_epi16_rm_512(<32 x i16> %a, <32 x i16>* %ptr_b) {
+  ;CHECK-LABEL: test_mask_sub_epi16_rm_512
+  ;CHECK: vpsubw (%rdi), %zmm0, %zmm0    ## encoding: [0x62,0xf1,0x7d,0x48,0xf9,0x07]
+  %b = load <32 x i16>, <32 x i16>* %ptr_b
+  %res = call <32 x i16> @llvm.x86.avx512.mask.psub.w.512(<32 x i16> %a, <32 x i16> %b, <32 x i16> zeroinitializer, i32 -1)
+  ret <32 x i16> %res
+}
+
+define <32 x i16> @test_mask_sub_epi16_rmk_512(<32 x i16> %a, <32 x i16>* %ptr_b, <32 x i16> %passThru, i32 %mask) {
+  ;CHECK-LABEL: test_mask_sub_epi16_rmk_512
+  ;CHECK: vpsubw (%rdi), %zmm0, %zmm1 {%k1} ## encoding: [0x62,0xf1,0x7d,0x49,0xf9,0x0f]
+  %b = load <32 x i16>, <32 x i16>* %ptr_b
+  %res = call <32 x i16> @llvm.x86.avx512.mask.psub.w.512(<32 x i16> %a, <32 x i16> %b, <32 x i16> %passThru, i32 %mask)
+  ret <32 x i16> %res
+}
+
+define <32 x i16> @test_mask_sub_epi16_rmkz_512(<32 x i16> %a, <32 x i16>* %ptr_b, i32 %mask) {
+  ;CHECK-LABEL: test_mask_sub_epi16_rmkz_512
+  ;CHECK: vpsubw (%rdi), %zmm0, %zmm0 {%k1} {z} ## encoding: [0x62,0xf1,0x7d,0xc9,0xf9,0x07]
+  %b = load <32 x i16>, <32 x i16>* %ptr_b
+  %res = call <32 x i16> @llvm.x86.avx512.mask.psub.w.512(<32 x i16> %a, <32 x i16> %b, <32 x i16> zeroinitializer, i32 %mask)
+  ret <32 x i16> %res
+}
+
+declare <32 x i16> @llvm.x86.avx512.mask.psub.w.512(<32 x i16>, <32 x i16>, <32 x i16>, i32)
+
+define <32 x i16> @test_mask_mullo_epi16_rr_512(<32 x i16> %a, <32 x i16> %b) {
+  ;CHECK-LABEL: test_mask_mullo_epi16_rr_512
+  ;CHECK: vpmullw %zmm1, %zmm0, %zmm0     ## encoding: [0x62,0xf1,0x7d,0x48,0xd5,0xc1]
+  %res = call <32 x i16> @llvm.x86.avx512.mask.pmull.w.512(<32 x i16> %a, <32 x i16> %b, <32 x i16> zeroinitializer, i32 -1)
+  ret <32 x i16> %res
+}
+
+define <32 x i16> @test_mask_mullo_epi16_rrk_512(<32 x i16> %a, <32 x i16> %b, <32 x i16> %passThru, i32 %mask) {
+  ;CHECK-LABEL: test_mask_mullo_epi16_rrk_512
+  ;CHECK: vpmullw %zmm1, %zmm0, %zmm2 {%k1} ## encoding: [0x62,0xf1,0x7d,0x49,0xd5,0xd1]
+  %res = call <32 x i16> @llvm.x86.avx512.mask.pmull.w.512(<32 x i16> %a, <32 x i16> %b, <32 x i16> %passThru, i32 %mask)
+  ret <32 x i16> %res
+}
+
+define <32 x i16> @test_mask_mullo_epi16_rrkz_512(<32 x i16> %a, <32 x i16> %b, i32 %mask) {
+  ;CHECK-LABEL: test_mask_mullo_epi16_rrkz_512
+  ;CHECK: vpmullw %zmm1, %zmm0, %zmm0 {%k1} {z} ## encoding: [0x62,0xf1,0x7d,0xc9,0xd5,0xc1]
+  %res = call <32 x i16> @llvm.x86.avx512.mask.pmull.w.512(<32 x i16> %a, <32 x i16> %b, <32 x i16> zeroinitializer, i32 %mask)
+  ret <32 x i16> %res
+}
+
+define <32 x i16> @test_mask_mullo_epi16_rm_512(<32 x i16> %a, <32 x i16>* %ptr_b) {
+  ;CHECK-LABEL: test_mask_mullo_epi16_rm_512
+  ;CHECK: vpmullw (%rdi), %zmm0, %zmm0    ## encoding: [0x62,0xf1,0x7d,0x48,0xd5,0x07]
+  %b = load <32 x i16>, <32 x i16>* %ptr_b
+  %res = call <32 x i16> @llvm.x86.avx512.mask.pmull.w.512(<32 x i16> %a, <32 x i16> %b, <32 x i16> zeroinitializer, i32 -1)
+  ret <32 x i16> %res
+}
+
+define <32 x i16> @test_mask_mullo_epi16_rmk_512(<32 x i16> %a, <32 x i16>* %ptr_b, <32 x i16> %passThru, i32 %mask) {
+  ;CHECK-LABEL: test_mask_mullo_epi16_rmk_512
+  ;CHECK: vpmullw (%rdi), %zmm0, %zmm1 {%k1} ## encoding: [0x62,0xf1,0x7d,0x49,0xd5,0x0f]
+  %b = load <32 x i16>, <32 x i16>* %ptr_b
+  %res = call <32 x i16> @llvm.x86.avx512.mask.pmull.w.512(<32 x i16> %a, <32 x i16> %b, <32 x i16> %passThru, i32 %mask)
+  ret <32 x i16> %res
+}
+
+define <32 x i16> @test_mask_mullo_epi16_rmkz_512(<32 x i16> %a, <32 x i16>* %ptr_b, i32 %mask) {
+  ;CHECK-LABEL: test_mask_mullo_epi16_rmkz_512
+  ;CHECK: vpmullw (%rdi), %zmm0, %zmm0 {%k1} {z} ## encoding: [0x62,0xf1,0x7d,0xc9,0xd5,0x07]
+  %b = load <32 x i16>, <32 x i16>* %ptr_b
+  %res = call <32 x i16> @llvm.x86.avx512.mask.pmull.w.512(<32 x i16> %a, <32 x i16> %b, <32 x i16> zeroinitializer, i32 %mask)
+  ret <32 x i16> %res
+}
+
+declare <32 x i16> @llvm.x86.avx512.mask.pmull.w.512(<32 x i16>, <32 x i16>, <32 x i16>, i32)
+
+define <8 x i16> @test_mask_mullo_epi16_rr_128(<8 x i16> %a, <8 x i16> %b) {
+  ;CHECK-LABEL: test_mask_mullo_epi16_rr_128
+  ;CHECK: vpmullw %xmm1, %xmm0, %xmm0     ## encoding: [0x62,0xf1,0x7d,0x08,0xd5,0xc1]
+  %res = call <8 x i16> @llvm.x86.avx512.mask.pmull.w.128(<8 x i16> %a, <8 x i16> %b, <8 x i16> zeroinitializer, i8 -1)
+  ret <8 x i16> %res
+}
+
+define <8 x i16> @test_mask_mullo_epi16_rrk_128(<8 x i16> %a, <8 x i16> %b, <8 x i16> %passThru, i8 %mask) {
+  ;CHECK-LABEL: test_mask_mullo_epi16_rrk_128
+  ;CHECK: vpmullw %xmm1, %xmm0, %xmm2 {%k1} ## encoding: [0x62,0xf1,0x7d,0x09,0xd5,0xd1]
+  %res = call <8 x i16> @llvm.x86.avx512.mask.pmull.w.128(<8 x i16> %a, <8 x i16> %b, <8 x i16> %passThru, i8 %mask)
+  ret <8 x i16> %res
+}
+
+define <8 x i16> @test_mask_mullo_epi16_rrkz_128(<8 x i16> %a, <8 x i16> %b, i8 %mask) {
+  ;CHECK-LABEL: test_mask_mullo_epi16_rrkz_128
+  ;CHECK: vpmullw %xmm1, %xmm0, %xmm0 {%k1} {z} ## encoding: [0x62,0xf1,0x7d,0x89,0xd5,0xc1]
+  %res = call <8 x i16> @llvm.x86.avx512.mask.pmull.w.128(<8 x i16> %a, <8 x i16> %b, <8 x i16> zeroinitializer, i8 %mask)
+  ret <8 x i16> %res
+}
+
+define <8 x i16> @test_mask_mullo_epi16_rm_128(<8 x i16> %a, <8 x i16>* %ptr_b) {
+  ;CHECK-LABEL: test_mask_mullo_epi16_rm_128
+  ;CHECK: vpmullw (%rdi), %xmm0, %xmm0    ## encoding: [0x62,0xf1,0x7d,0x08,0xd5,0x07]
+  %b = load <8 x i16>, <8 x i16>* %ptr_b
+  %res = call <8 x i16> @llvm.x86.avx512.mask.pmull.w.128(<8 x i16> %a, <8 x i16> %b, <8 x i16> zeroinitializer, i8 -1)
+  ret <8 x i16> %res
+}
+
+define <8 x i16> @test_mask_mullo_epi16_rmk_128(<8 x i16> %a, <8 x i16>* %ptr_b, <8 x i16> %passThru, i8 %mask) {
+  ;CHECK-LABEL: test_mask_mullo_epi16_rmk_128
+  ;CHECK: vpmullw (%rdi), %xmm0, %xmm1 {%k1} ## encoding: [0x62,0xf1,0x7d,0x09,0xd5,0x0f]
+  %b = load <8 x i16>, <8 x i16>* %ptr_b
+  %res = call <8 x i16> @llvm.x86.avx512.mask.pmull.w.128(<8 x i16> %a, <8 x i16> %b, <8 x i16> %passThru, i8 %mask)
+  ret <8 x i16> %res
+}
+
+define <8 x i16> @test_mask_mullo_epi16_rmkz_128(<8 x i16> %a, <8 x i16>* %ptr_b, i8 %mask) {
+  ;CHECK-LABEL: test_mask_mullo_epi16_rmkz_128
+  ;CHECK: vpmullw (%rdi), %xmm0, %xmm0 {%k1} {z} ## encoding: [0x62,0xf1,0x7d,0x89,0xd5,0x07]
+  %b = load <8 x i16>, <8 x i16>* %ptr_b
+  %res = call <8 x i16> @llvm.x86.avx512.mask.pmull.w.128(<8 x i16> %a, <8 x i16> %b, <8 x i16> zeroinitializer, i8 %mask)
+  ret <8 x i16> %res
+}
+
+declare <8 x i16> @llvm.x86.avx512.mask.pmull.w.128(<8 x i16>, <8 x i16>, <8 x i16>, i8)
+
+define <16 x i16> @test_mask_mullo_epi16_rr_256(<16 x i16> %a, <16 x i16> %b) {
+  ;CHECK-LABEL: test_mask_mullo_epi16_rr_256
+  ;CHECK: vpmullw %ymm1, %ymm0, %ymm0     ## encoding: [0x62,0xf1,0x7d,0x28,0xd5,0xc1]
+  %res = call <16 x i16> @llvm.x86.avx512.mask.pmull.w.256(<16 x i16> %a, <16 x i16> %b, <16 x i16> zeroinitializer, i16 -1)
+  ret <16 x i16> %res
+}
+
+define <16 x i16> @test_mask_mullo_epi16_rrk_256(<16 x i16> %a, <16 x i16> %b, <16 x i16> %passThru, i16 %mask) {
+  ;CHECK-LABEL: test_mask_mullo_epi16_rrk_256
+  ;CHECK: vpmullw %ymm1, %ymm0, %ymm2 {%k1} ## encoding: [0x62,0xf1,0x7d,0x29,0xd5,0xd1]
+  %res = call <16 x i16> @llvm.x86.avx512.mask.pmull.w.256(<16 x i16> %a, <16 x i16> %b, <16 x i16> %passThru, i16 %mask)
+  ret <16 x i16> %res
+}
+
+define <16 x i16> @test_mask_mullo_epi16_rrkz_256(<16 x i16> %a, <16 x i16> %b, i16 %mask) {
+  ;CHECK-LABEL: test_mask_mullo_epi16_rrkz_256
+  ;CHECK: vpmullw %ymm1, %ymm0, %ymm0 {%k1} {z} ## encoding: [0x62,0xf1,0x7d,0xa9,0xd5,0xc1]
+  %res = call <16 x i16> @llvm.x86.avx512.mask.pmull.w.256(<16 x i16> %a, <16 x i16> %b, <16 x i16> zeroinitializer, i16 %mask)
+  ret <16 x i16> %res
+}
+
+define <16 x i16> @test_mask_mullo_epi16_rm_256(<16 x i16> %a, <16 x i16>* %ptr_b) {
+  ;CHECK-LABEL: test_mask_mullo_epi16_rm_256
+  ;CHECK: vpmullw (%rdi), %ymm0, %ymm0    ## encoding: [0x62,0xf1,0x7d,0x28,0xd5,0x07]
+  %b = load <16 x i16>, <16 x i16>* %ptr_b
+  %res = call <16 x i16> @llvm.x86.avx512.mask.pmull.w.256(<16 x i16> %a, <16 x i16> %b, <16 x i16> zeroinitializer, i16 -1)
+  ret <16 x i16> %res
+}
+
+define <16 x i16> @test_mask_mullo_epi16_rmk_256(<16 x i16> %a, <16 x i16>* %ptr_b, <16 x i16> %passThru, i16 %mask) {
+  ;CHECK-LABEL: test_mask_mullo_epi16_rmk_256
+  ;CHECK: vpmullw (%rdi), %ymm0, %ymm1 {%k1} ## encoding: [0x62,0xf1,0x7d,0x29,0xd5,0x0f]
+  %b = load <16 x i16>, <16 x i16>* %ptr_b
+  %res = call <16 x i16> @llvm.x86.avx512.mask.pmull.w.256(<16 x i16> %a, <16 x i16> %b, <16 x i16> %passThru, i16 %mask)
+  ret <16 x i16> %res
+}
+
+define <16 x i16> @test_mask_mullo_epi16_rmkz_256(<16 x i16> %a, <16 x i16>* %ptr_b, i16 %mask) {
+  ;CHECK-LABEL: test_mask_mullo_epi16_rmkz_256
+  ;CHECK: vpmullw (%rdi), %ymm0, %ymm0 {%k1} {z} ## encoding: [0x62,0xf1,0x7d,0xa9,0xd5,0x07]
+  %b = load <16 x i16>, <16 x i16>* %ptr_b
+  %res = call <16 x i16> @llvm.x86.avx512.mask.pmull.w.256(<16 x i16> %a, <16 x i16> %b, <16 x i16> zeroinitializer, i16 %mask)
+  ret <16 x i16> %res
+}
+
+declare <16 x i16> @llvm.x86.avx512.mask.pmull.w.256(<16 x i16>, <16 x i16>, <16 x i16>, i16)
+
+
+define <8 x i16> @test_mask_packs_epi32_rr_128(<4 x i32> %a, <4 x i32> %b) {
+  ;CHECK-LABEL: test_mask_packs_epi32_rr_128
+  ;CHECK: vpackssdw       %xmm1, %xmm0, %xmm0 ## encoding: [0xc5,0xf9,0x6b,0xc1]
+  %res = call <8 x i16> @llvm.x86.avx512.mask.packssdw.128(<4 x i32> %a, <4 x i32> %b, <8 x i16> zeroinitializer, i8 -1)
+  ret <8 x i16> %res
+}
+
+define <8 x i16> @test_mask_packs_epi32_rrk_128(<4 x i32> %a, <4 x i32> %b, <8 x i16> %passThru, i8 %mask) {
+  ;CHECK-LABEL: test_mask_packs_epi32_rrk_128
+  ;CHECK: vpackssdw       %xmm1, %xmm0, %xmm2 {%k1} ## encoding: [0x62,0xf1,0x7d,0x09,0x6b,0xd1]
+  %res = call <8 x i16> @llvm.x86.avx512.mask.packssdw.128(<4 x i32> %a, <4 x i32> %b, <8 x i16> %passThru, i8 %mask)
+  ret <8 x i16> %res
+}
+
+define <8 x i16> @test_mask_packs_epi32_rrkz_128(<4 x i32> %a, <4 x i32> %b, i8 %mask) {
+  ;CHECK-LABEL: test_mask_packs_epi32_rrkz_128
+  ;CHECK: vpackssdw       %xmm1, %xmm0, %xmm0 {%k1} {z} ## encoding: [0x62,0xf1,0x7d,0x89,0x6b,0xc1]
+  %res = call <8 x i16> @llvm.x86.avx512.mask.packssdw.128(<4 x i32> %a, <4 x i32> %b, <8 x i16> zeroinitializer, i8 %mask)
+  ret <8 x i16> %res
+}
+
+define <8 x i16> @test_mask_packs_epi32_rm_128(<4 x i32> %a, <4 x i32>* %ptr_b) {
+  ;CHECK-LABEL: test_mask_packs_epi32_rm_128
+  ;CHECK: vpackssdw       (%rdi), %xmm0, %xmm0 ## encoding: [0xc5,0xf9,0x6b,0x07]
+  %b = load <4 x i32>, <4 x i32>* %ptr_b
+  %res = call <8 x i16> @llvm.x86.avx512.mask.packssdw.128(<4 x i32> %a, <4 x i32> %b, <8 x i16> zeroinitializer, i8 -1)
+  ret <8 x i16> %res
+}
+
+define <8 x i16> @test_mask_packs_epi32_rmk_128(<4 x i32> %a, <4 x i32>* %ptr_b, <8 x i16> %passThru, i8 %mask) {
+  ;CHECK-LABEL: test_mask_packs_epi32_rmk_128
+  ;CHECK: vpackssdw       (%rdi), %xmm0, %xmm1 {%k1} ## encoding: [0x62,0xf1,0x7d,0x09,0x6b,0x0f]
+  %b = load <4 x i32>, <4 x i32>* %ptr_b
+  %res = call <8 x i16> @llvm.x86.avx512.mask.packssdw.128(<4 x i32> %a, <4 x i32> %b, <8 x i16> %passThru, i8 %mask)
+  ret <8 x i16> %res
+}
+
+define <8 x i16> @test_mask_packs_epi32_rmkz_128(<4 x i32> %a, <4 x i32>* %ptr_b, i8 %mask) {
+  ;CHECK-LABEL: test_mask_packs_epi32_rmkz_128
+  ;CHECK: vpackssdw       (%rdi), %xmm0, %xmm0 {%k1} {z} ## encoding: [0x62,0xf1,0x7d,0x89,0x6b,0x07]
+  %b = load <4 x i32>, <4 x i32>* %ptr_b
+  %res = call <8 x i16> @llvm.x86.avx512.mask.packssdw.128(<4 x i32> %a, <4 x i32> %b, <8 x i16> zeroinitializer, i8 %mask)
+  ret <8 x i16> %res
+}
+
+define <8 x i16> @test_mask_packs_epi32_rmb_128(<4 x i32> %a, i32* %ptr_b) {
+  ;CHECK-LABEL: test_mask_packs_epi32_rmb_128
+  ;CHECK: vpackssdw       (%rdi){1to4}, %xmm0, %xmm0  ## encoding: [0x62,0xf1,0x7d,0x18,0x6b,0x07]
+  %q = load i32, i32* %ptr_b
+  %vecinit.i = insertelement <4 x i32> undef, i32 %q, i32 0
+  %b = shufflevector <4 x i32> %vecinit.i, <4 x i32> undef, <4 x i32> zeroinitializer
+  %res = call <8 x i16> @llvm.x86.avx512.mask.packssdw.128(<4 x i32> %a, <4 x i32> %b, <8 x i16> zeroinitializer, i8 -1)
+  ret <8 x i16> %res
+}
+
+define <8 x i16> @test_mask_packs_epi32_rmbk_128(<4 x i32> %a, i32* %ptr_b, <8 x i16> %passThru, i8 %mask) {
+  ;CHECK-LABEL: test_mask_packs_epi32_rmbk_128
+  ;CHECK: vpackssdw       (%rdi){1to4}, %xmm0, %xmm1 {%k1} ## encoding: [0x62,0xf1,0x7d,0x19,0x6b,0x0f]
+  %q = load i32, i32* %ptr_b
+  %vecinit.i = insertelement <4 x i32> undef, i32 %q, i32 0
+  %b = shufflevector <4 x i32> %vecinit.i, <4 x i32> undef, <4 x i32> zeroinitializer
+  %res = call <8 x i16> @llvm.x86.avx512.mask.packssdw.128(<4 x i32> %a, <4 x i32> %b, <8 x i16> %passThru, i8 %mask)
+  ret <8 x i16> %res
+}
+
+define <8 x i16> @test_mask_packs_epi32_rmbkz_128(<4 x i32> %a, i32* %ptr_b, i8 %mask) {
+  ;CHECK-LABEL: test_mask_packs_epi32_rmbkz_128
+  ;CHECK: vpackssdw       (%rdi){1to4}, %xmm0, %xmm0 {%k1} {z} ## encoding: [0x62,0xf1,0x7d,0x99,0x6b,0x07]
+  %q = load i32, i32* %ptr_b
+  %vecinit.i = insertelement <4 x i32> undef, i32 %q, i32 0
+  %b = shufflevector <4 x i32> %vecinit.i, <4 x i32> undef, <4 x i32> zeroinitializer
+  %res = call <8 x i16> @llvm.x86.avx512.mask.packssdw.128(<4 x i32> %a, <4 x i32> %b, <8 x i16> zeroinitializer, i8 %mask)
+  ret <8 x i16> %res
+}
+
+declare <8 x i16> @llvm.x86.avx512.mask.packssdw.128(<4 x i32>, <4 x i32>, <8 x i16>, i8)
+
+define <16 x i16> @test_mask_packs_epi32_rr_256(<8 x i32> %a, <8 x i32> %b) {
+  ;CHECK-LABEL: test_mask_packs_epi32_rr_256
+  ;CHECK: vpackssdw       %ymm1, %ymm0, %ymm0 ## encoding: [0xc5,0xfd,0x6b,0xc1]
+  %res = call <16 x i16> @llvm.x86.avx512.mask.packssdw.256(<8 x i32> %a, <8 x i32> %b, <16 x i16> zeroinitializer, i16 -1)
+  ret <16 x i16> %res
+}
+
+define <16 x i16> @test_mask_packs_epi32_rrk_256(<8 x i32> %a, <8 x i32> %b, <16 x i16> %passThru, i16 %mask) {
+  ;CHECK-LABEL: test_mask_packs_epi32_rrk_256
+  ;CHECK: vpackssdw       %ymm1, %ymm0, %ymm2 {%k1} ## encoding: [0x62,0xf1,0x7d,0x29,0x6b,0xd1]
+  %res = call <16 x i16> @llvm.x86.avx512.mask.packssdw.256(<8 x i32> %a, <8 x i32> %b, <16 x i16> %passThru, i16 %mask)
+  ret <16 x i16> %res
+}
+
+define <16 x i16> @test_mask_packs_epi32_rrkz_256(<8 x i32> %a, <8 x i32> %b, i16 %mask) {
+  ;CHECK-LABEL: test_mask_packs_epi32_rrkz_256
+  ;CHECK: vpackssdw       %ymm1, %ymm0, %ymm0 {%k1} {z} ## encoding: [0x62,0xf1,0x7d,0xa9,0x6b,0xc1]
+  %res = call <16 x i16> @llvm.x86.avx512.mask.packssdw.256(<8 x i32> %a, <8 x i32> %b, <16 x i16> zeroinitializer, i16 %mask)
+  ret <16 x i16> %res
+}
+
+define <16 x i16> @test_mask_packs_epi32_rm_256(<8 x i32> %a, <8 x i32>* %ptr_b) {
+  ;CHECK-LABEL: test_mask_packs_epi32_rm_256
+  ;CHECK: vpackssdw       (%rdi), %ymm0, %ymm0 ## encoding: [0xc5,0xfd,0x6b,0x07]
+  %b = load <8 x i32>, <8 x i32>* %ptr_b
+  %res = call <16 x i16> @llvm.x86.avx512.mask.packssdw.256(<8 x i32> %a, <8 x i32> %b, <16 x i16> zeroinitializer, i16 -1)
+  ret <16 x i16> %res
+}
+
+define <16 x i16> @test_mask_packs_epi32_rmk_256(<8 x i32> %a, <8 x i32>* %ptr_b, <16 x i16> %passThru, i16 %mask) {
+  ;CHECK-LABEL: test_mask_packs_epi32_rmk_256
+  ;CHECK: vpackssdw       (%rdi), %ymm0, %ymm1 {%k1} ## encoding: [0x62,0xf1,0x7d,0x29,0x6b,0x0f]
+  %b = load <8 x i32>, <8 x i32>* %ptr_b
+  %res = call <16 x i16> @llvm.x86.avx512.mask.packssdw.256(<8 x i32> %a, <8 x i32> %b, <16 x i16> %passThru, i16 %mask)
+  ret <16 x i16> %res
+}
+
+define <16 x i16> @test_mask_packs_epi32_rmkz_256(<8 x i32> %a, <8 x i32>* %ptr_b, i16 %mask) {
+  ;CHECK-LABEL: test_mask_packs_epi32_rmkz_256
+  ;CHECK: vpackssdw       (%rdi), %ymm0, %ymm0 {%k1} {z} ## encoding: [0x62,0xf1,0x7d,0xa9,0x6b,0x07]
+  %b = load <8 x i32>, <8 x i32>* %ptr_b
+  %res = call <16 x i16> @llvm.x86.avx512.mask.packssdw.256(<8 x i32> %a, <8 x i32> %b, <16 x i16> zeroinitializer, i16 %mask)
+  ret <16 x i16> %res
+}
+
+define <16 x i16> @test_mask_packs_epi32_rmb_256(<8 x i32> %a, i32* %ptr_b) {
+  ;CHECK-LABEL: test_mask_packs_epi32_rmb_256
+  ;CHECK: vpackssdw       (%rdi){1to8}, %ymm0, %ymm0  ## encoding: [0x62,0xf1,0x7d,0x38,0x6b,0x07]
+  %q = load i32, i32* %ptr_b
+  %vecinit.i = insertelement <8 x i32> undef, i32 %q, i32 0
+  %b = shufflevector <8 x i32> %vecinit.i, <8 x i32> undef, <8 x i32> zeroinitializer
+  %res = call <16 x i16> @llvm.x86.avx512.mask.packssdw.256(<8 x i32> %a, <8 x i32> %b, <16 x i16> zeroinitializer, i16 -1)
+  ret <16 x i16> %res
+}
+
+define <16 x i16> @test_mask_packs_epi32_rmbk_256(<8 x i32> %a, i32* %ptr_b, <16 x i16> %passThru, i16 %mask) {
+  ;CHECK-LABEL: test_mask_packs_epi32_rmbk_256
+  ;CHECK: vpackssdw       (%rdi){1to8}, %ymm0, %ymm1 {%k1} ## encoding: [0x62,0xf1,0x7d,0x39,0x6b,0x0f]
+  %q = load i32, i32* %ptr_b
+  %vecinit.i = insertelement <8 x i32> undef, i32 %q, i32 0
+  %b = shufflevector <8 x i32> %vecinit.i, <8 x i32> undef, <8 x i32> zeroinitializer
+  %res = call <16 x i16> @llvm.x86.avx512.mask.packssdw.256(<8 x i32> %a, <8 x i32> %b, <16 x i16> %passThru, i16 %mask)
+  ret <16 x i16> %res
+}
+
+define <16 x i16> @test_mask_packs_epi32_rmbkz_256(<8 x i32> %a, i32* %ptr_b, i16 %mask) {
+  ;CHECK-LABEL: test_mask_packs_epi32_rmbkz_256
+  ;CHECK: vpackssdw       (%rdi){1to8}, %ymm0, %ymm0 {%k1} {z} ## encoding: [0x62,0xf1,0x7d,0xb9,0x6b,0x07]  
+  %q = load i32, i32* %ptr_b
+  %vecinit.i = insertelement <8 x i32> undef, i32 %q, i32 0
+  %b = shufflevector <8 x i32> %vecinit.i, <8 x i32> undef, <8 x i32> zeroinitializer
+  %res = call <16 x i16> @llvm.x86.avx512.mask.packssdw.256(<8 x i32> %a, <8 x i32> %b, <16 x i16> zeroinitializer, i16 %mask)
+  ret <16 x i16> %res
+}
+
+declare <16 x i16> @llvm.x86.avx512.mask.packssdw.256(<8 x i32>, <8 x i32>, <16 x i16>, i16)
+
+define <16 x i8> @test_mask_packs_epi16_rr_128(<8 x i16> %a, <8 x i16> %b) {
+  ;CHECK-LABEL: test_mask_packs_epi16_rr_128
+  ;CHECK: vpacksswb       %xmm1, %xmm0, %xmm0 ## encoding: [0xc5,0xf9,0x63,0xc1]
+  %res = call <16 x i8> @llvm.x86.avx512.mask.packsswb.128(<8 x i16> %a, <8 x i16> %b, <16 x i8> zeroinitializer, i16 -1)
+  ret <16 x i8> %res
+}
+
+define <16 x i8> @test_mask_packs_epi16_rrk_128(<8 x i16> %a, <8 x i16> %b, <16 x i8> %passThru, i16 %mask) {
+  ;CHECK-LABEL: test_mask_packs_epi16_rrk_128
+  ;CHECK: vpacksswb       %xmm1, %xmm0, %xmm2 {%k1} ## encoding: [0x62,0xf1,0xfd,0x09,0x63,0xd1]
+  %res = call <16 x i8> @llvm.x86.avx512.mask.packsswb.128(<8 x i16> %a, <8 x i16> %b, <16 x i8> %passThru, i16 %mask)
+  ret <16 x i8> %res
+}
+
+define <16 x i8> @test_mask_packs_epi16_rrkz_128(<8 x i16> %a, <8 x i16> %b, i16 %mask) {
+  ;CHECK-LABEL: test_mask_packs_epi16_rrkz_128
+  ;CHECK: vpacksswb       %xmm1, %xmm0, %xmm0 {%k1} {z} ## encoding: [0x62,0xf1,0xfd,0x89,0x63,0xc1]
+  %res = call <16 x i8> @llvm.x86.avx512.mask.packsswb.128(<8 x i16> %a, <8 x i16> %b, <16 x i8> zeroinitializer, i16 %mask)
+  ret <16 x i8> %res
+}
+
+define <16 x i8> @test_mask_packs_epi16_rm_128(<8 x i16> %a, <8 x i16>* %ptr_b) {
+  ;CHECK-LABEL: test_mask_packs_epi16_rm_128
+  ;CHECK: vpacksswb       (%rdi), %xmm0, %xmm0 ## encoding: [0xc5,0xf9,0x63,0x07]
+  %b = load <8 x i16>, <8 x i16>* %ptr_b
+  %res = call <16 x i8> @llvm.x86.avx512.mask.packsswb.128(<8 x i16> %a, <8 x i16> %b, <16 x i8> zeroinitializer, i16 -1)
+  ret <16 x i8> %res
+}
+
+define <16 x i8> @test_mask_packs_epi16_rmk_128(<8 x i16> %a, <8 x i16>* %ptr_b, <16 x i8> %passThru, i16 %mask) {
+  ;CHECK-LABEL: test_mask_packs_epi16_rmk_128
+  ;CHECK: vpacksswb       (%rdi), %xmm0, %xmm1 {%k1} ## encoding: [0x62,0xf1,0xfd,0x09,0x63,0x0f]
+  %b = load <8 x i16>, <8 x i16>* %ptr_b
+  %res = call <16 x i8> @llvm.x86.avx512.mask.packsswb.128(<8 x i16> %a, <8 x i16> %b, <16 x i8> %passThru, i16 %mask)
+  ret <16 x i8> %res
+}
+
+define <16 x i8> @test_mask_packs_epi16_rmkz_128(<8 x i16> %a, <8 x i16>* %ptr_b, i16 %mask) {
+  ;CHECK-LABEL: test_mask_packs_epi16_rmkz_128
+  ;CHECK: vpacksswb       (%rdi), %xmm0, %xmm0 {%k1} {z} ## encoding: [0x62,0xf1,0xfd,0x89,0x63,0x07]
+  %b = load <8 x i16>, <8 x i16>* %ptr_b
+  %res = call <16 x i8> @llvm.x86.avx512.mask.packsswb.128(<8 x i16> %a, <8 x i16> %b, <16 x i8> zeroinitializer, i16 %mask)
+  ret <16 x i8> %res
+}
+
+declare <16 x i8> @llvm.x86.avx512.mask.packsswb.128(<8 x i16>, <8 x i16>, <16 x i8>, i16)
+
+define <32 x i8> @test_mask_packs_epi16_rr_256(<16 x i16> %a, <16 x i16> %b) {
+  ;CHECK-LABEL: test_mask_packs_epi16_rr_256
+  ;CHECK: vpacksswb       %ymm1, %ymm0, %ymm0 ## encoding: [0xc5,0xfd,0x63,0xc1]
+  %res = call <32 x i8> @llvm.x86.avx512.mask.packsswb.256(<16 x i16> %a, <16 x i16> %b, <32 x i8> zeroinitializer, i32 -1)
+  ret <32 x i8> %res
+}
+
+define <32 x i8> @test_mask_packs_epi16_rrk_256(<16 x i16> %a, <16 x i16> %b, <32 x i8> %passThru, i32 %mask) {
+  ;CHECK-LABEL: test_mask_packs_epi16_rrk_256
+  ;CHECK: vpacksswb       %ymm1, %ymm0, %ymm2 {%k1} ## encoding: [0x62,0xf1,0xfd,0x29,0x63,0xd1]
+  %res = call <32 x i8> @llvm.x86.avx512.mask.packsswb.256(<16 x i16> %a, <16 x i16> %b, <32 x i8> %passThru, i32 %mask)
+  ret <32 x i8> %res
+}
+
+define <32 x i8> @test_mask_packs_epi16_rrkz_256(<16 x i16> %a, <16 x i16> %b, i32 %mask) {
+  ;CHECK-LABEL: test_mask_packs_epi16_rrkz_256
+  ;CHECK: vpacksswb       %ymm1, %ymm0, %ymm0 {%k1} {z} ## encoding: [0x62,0xf1,0xfd,0xa9,0x63,0xc1]
+  %res = call <32 x i8> @llvm.x86.avx512.mask.packsswb.256(<16 x i16> %a, <16 x i16> %b, <32 x i8> zeroinitializer, i32 %mask)
+  ret <32 x i8> %res
+}
+
+define <32 x i8> @test_mask_packs_epi16_rm_256(<16 x i16> %a, <16 x i16>* %ptr_b) {
+  ;CHECK-LABEL: test_mask_packs_epi16_rm_256
+  ;CHECK: vpacksswb       (%rdi), %ymm0, %ymm0 ## encoding: [0xc5,0xfd,0x63,0x07]
+  %b = load <16 x i16>, <16 x i16>* %ptr_b
+  %res = call <32 x i8> @llvm.x86.avx512.mask.packsswb.256(<16 x i16> %a, <16 x i16> %b, <32 x i8> zeroinitializer, i32 -1)
+  ret <32 x i8> %res
+}
+
+define <32 x i8> @test_mask_packs_epi16_rmk_256(<16 x i16> %a, <16 x i16>* %ptr_b, <32 x i8> %passThru, i32 %mask) {
+  ;CHECK-LABEL: test_mask_packs_epi16_rmk_256
+  ;CHECK: vpacksswb       (%rdi), %ymm0, %ymm1 {%k1} ## encoding: [0x62,0xf1,0xfd,0x29,0x63,0x0f]
+  %b = load <16 x i16>, <16 x i16>* %ptr_b
+  %res = call <32 x i8> @llvm.x86.avx512.mask.packsswb.256(<16 x i16> %a, <16 x i16> %b, <32 x i8> %passThru, i32 %mask)
+  ret <32 x i8> %res
+}
+
+define <32 x i8> @test_mask_packs_epi16_rmkz_256(<16 x i16> %a, <16 x i16>* %ptr_b, i32 %mask) {
+  ;CHECK-LABEL: test_mask_packs_epi16_rmkz_256
+  ;CHECK: vpacksswb       (%rdi), %ymm0, %ymm0 {%k1} {z} ## encoding: [0x62,0xf1,0xfd,0xa9,0x63,0x07]
+  %b = load <16 x i16>, <16 x i16>* %ptr_b
+  %res = call <32 x i8> @llvm.x86.avx512.mask.packsswb.256(<16 x i16> %a, <16 x i16> %b, <32 x i8> zeroinitializer, i32 %mask)
+  ret <32 x i8> %res
+}
+
+declare <32 x i8> @llvm.x86.avx512.mask.packsswb.256(<16 x i16>, <16 x i16>, <32 x i8>, i32)
+
+
+define <8 x i16> @test_mask_packus_epi32_rr_128(<4 x i32> %a, <4 x i32> %b) {
+  ;CHECK-LABEL: test_mask_packus_epi32_rr_128
+  ;CHECK: vpackusdw       %xmm1, %xmm0, %xmm0 
+  %res = call <8 x i16> @llvm.x86.avx512.mask.packusdw.128(<4 x i32> %a, <4 x i32> %b, <8 x i16> zeroinitializer, i8 -1)
+  ret <8 x i16> %res
+}
+
+define <8 x i16> @test_mask_packus_epi32_rrk_128(<4 x i32> %a, <4 x i32> %b, <8 x i16> %passThru, i8 %mask) {
+  ;CHECK-LABEL: test_mask_packus_epi32_rrk_128
+  ;CHECK: vpackusdw       %xmm1, %xmm0, %xmm2 {%k1} 
+  %res = call <8 x i16> @llvm.x86.avx512.mask.packusdw.128(<4 x i32> %a, <4 x i32> %b, <8 x i16> %passThru, i8 %mask)
+  ret <8 x i16> %res
+}
+
+define <8 x i16> @test_mask_packus_epi32_rrkz_128(<4 x i32> %a, <4 x i32> %b, i8 %mask) {
+  ;CHECK-LABEL: test_mask_packus_epi32_rrkz_128
+  ;CHECK: vpackusdw       %xmm1, %xmm0, %xmm0 {%k1} {z} 
+  %res = call <8 x i16> @llvm.x86.avx512.mask.packusdw.128(<4 x i32> %a, <4 x i32> %b, <8 x i16> zeroinitializer, i8 %mask)
+  ret <8 x i16> %res
+}
+
+define <8 x i16> @test_mask_packus_epi32_rm_128(<4 x i32> %a, <4 x i32>* %ptr_b) {
+  ;CHECK-LABEL: test_mask_packus_epi32_rm_128
+  ;CHECK: vpackusdw       (%rdi), %xmm0, %xmm0 
+  %b = load <4 x i32>, <4 x i32>* %ptr_b
+  %res = call <8 x i16> @llvm.x86.avx512.mask.packusdw.128(<4 x i32> %a, <4 x i32> %b, <8 x i16> zeroinitializer, i8 -1)
+  ret <8 x i16> %res
+}
+
+define <8 x i16> @test_mask_packus_epi32_rmk_128(<4 x i32> %a, <4 x i32>* %ptr_b, <8 x i16> %passThru, i8 %mask) {
+  ;CHECK-LABEL: test_mask_packus_epi32_rmk_128
+  ;CHECK: vpackusdw       (%rdi), %xmm0, %xmm1 {%k1} 
+  %b = load <4 x i32>, <4 x i32>* %ptr_b
+  %res = call <8 x i16> @llvm.x86.avx512.mask.packusdw.128(<4 x i32> %a, <4 x i32> %b, <8 x i16> %passThru, i8 %mask)
+  ret <8 x i16> %res
+}
+
+define <8 x i16> @test_mask_packus_epi32_rmkz_128(<4 x i32> %a, <4 x i32>* %ptr_b, i8 %mask) {
+  ;CHECK-LABEL: test_mask_packus_epi32_rmkz_128
+  ;CHECK: vpackusdw       (%rdi), %xmm0, %xmm0 {%k1} {z} 
+  %b = load <4 x i32>, <4 x i32>* %ptr_b
+  %res = call <8 x i16> @llvm.x86.avx512.mask.packusdw.128(<4 x i32> %a, <4 x i32> %b, <8 x i16> zeroinitializer, i8 %mask)
+  ret <8 x i16> %res
+}
+
+define <8 x i16> @test_mask_packus_epi32_rmb_128(<4 x i32> %a, i32* %ptr_b) {
+  ;CHECK-LABEL: test_mask_packus_epi32_rmb_128
+  ;CHECK: vpackusdw       (%rdi){1to4}, %xmm0, %xmm0  
+  %q = load i32, i32* %ptr_b
+  %vecinit.i = insertelement <4 x i32> undef, i32 %q, i32 0
+  %b = shufflevector <4 x i32> %vecinit.i, <4 x i32> undef, <4 x i32> zeroinitializer
+  %res = call <8 x i16> @llvm.x86.avx512.mask.packusdw.128(<4 x i32> %a, <4 x i32> %b, <8 x i16> zeroinitializer, i8 -1)
+  ret <8 x i16> %res
+}
+
+define <8 x i16> @test_mask_packus_epi32_rmbk_128(<4 x i32> %a, i32* %ptr_b, <8 x i16> %passThru, i8 %mask) {
+  ;CHECK-LABEL: test_mask_packus_epi32_rmbk_128
+  ;CHECK: vpackusdw       (%rdi){1to4}, %xmm0, %xmm1 {%k1} 
+  %q = load i32, i32* %ptr_b
+  %vecinit.i = insertelement <4 x i32> undef, i32 %q, i32 0
+  %b = shufflevector <4 x i32> %vecinit.i, <4 x i32> undef, <4 x i32> zeroinitializer
+  %res = call <8 x i16> @llvm.x86.avx512.mask.packusdw.128(<4 x i32> %a, <4 x i32> %b, <8 x i16> %passThru, i8 %mask)
+  ret <8 x i16> %res
+}
+
+define <8 x i16> @test_mask_packus_epi32_rmbkz_128(<4 x i32> %a, i32* %ptr_b, i8 %mask) {
+  ;CHECK-LABEL: test_mask_packus_epi32_rmbkz_128
+  ;CHECK: vpackusdw       (%rdi){1to4}, %xmm0, %xmm0 {%k1} {z} 
+  %q = load i32, i32* %ptr_b
+  %vecinit.i = insertelement <4 x i32> undef, i32 %q, i32 0
+  %b = shufflevector <4 x i32> %vecinit.i, <4 x i32> undef, <4 x i32> zeroinitializer
+  %res = call <8 x i16> @llvm.x86.avx512.mask.packusdw.128(<4 x i32> %a, <4 x i32> %b, <8 x i16> zeroinitializer, i8 %mask)
+  ret <8 x i16> %res
+}
+
+declare <8 x i16> @llvm.x86.avx512.mask.packusdw.128(<4 x i32>, <4 x i32>, <8 x i16>, i8)
+
+define <16 x i16> @test_mask_packus_epi32_rr_256(<8 x i32> %a, <8 x i32> %b) {
+  ;CHECK-LABEL: test_mask_packus_epi32_rr_256
+  ;CHECK: vpackusdw       %ymm1, %ymm0, %ymm0 
+  %res = call <16 x i16> @llvm.x86.avx512.mask.packusdw.256(<8 x i32> %a, <8 x i32> %b, <16 x i16> zeroinitializer, i16 -1)
+  ret <16 x i16> %res
+}
+
+define <16 x i16> @test_mask_packus_epi32_rrk_256(<8 x i32> %a, <8 x i32> %b, <16 x i16> %passThru, i16 %mask) {
+  ;CHECK-LABEL: test_mask_packus_epi32_rrk_256
+  ;CHECK: vpackusdw       %ymm1, %ymm0, %ymm2 {%k1} 
+  %res = call <16 x i16> @llvm.x86.avx512.mask.packusdw.256(<8 x i32> %a, <8 x i32> %b, <16 x i16> %passThru, i16 %mask)
+  ret <16 x i16> %res
+}
+
+define <16 x i16> @test_mask_packus_epi32_rrkz_256(<8 x i32> %a, <8 x i32> %b, i16 %mask) {
+  ;CHECK-LABEL: test_mask_packus_epi32_rrkz_256
+  ;CHECK: vpackusdw       %ymm1, %ymm0, %ymm0 {%k1} {z} 
+  %res = call <16 x i16> @llvm.x86.avx512.mask.packusdw.256(<8 x i32> %a, <8 x i32> %b, <16 x i16> zeroinitializer, i16 %mask)
+  ret <16 x i16> %res
+}
+
+define <16 x i16> @test_mask_packus_epi32_rm_256(<8 x i32> %a, <8 x i32>* %ptr_b) {
+  ;CHECK-LABEL: test_mask_packus_epi32_rm_256
+  ;CHECK: vpackusdw       (%rdi), %ymm0, %ymm0 
+  %b = load <8 x i32>, <8 x i32>* %ptr_b
+  %res = call <16 x i16> @llvm.x86.avx512.mask.packusdw.256(<8 x i32> %a, <8 x i32> %b, <16 x i16> zeroinitializer, i16 -1)
+  ret <16 x i16> %res
+}
+
+define <16 x i16> @test_mask_packus_epi32_rmk_256(<8 x i32> %a, <8 x i32>* %ptr_b, <16 x i16> %passThru, i16 %mask) {
+  ;CHECK-LABEL: test_mask_packus_epi32_rmk_256
+  ;CHECK: vpackusdw       (%rdi), %ymm0, %ymm1 {%k1} 
+  %b = load <8 x i32>, <8 x i32>* %ptr_b
+  %res = call <16 x i16> @llvm.x86.avx512.mask.packusdw.256(<8 x i32> %a, <8 x i32> %b, <16 x i16> %passThru, i16 %mask)
+  ret <16 x i16> %res
+}
+
+define <16 x i16> @test_mask_packus_epi32_rmkz_256(<8 x i32> %a, <8 x i32>* %ptr_b, i16 %mask) {
+  ;CHECK-LABEL: test_mask_packus_epi32_rmkz_256
+  ;CHECK: vpackusdw       (%rdi), %ymm0, %ymm0 {%k1} {z} 
+  %b = load <8 x i32>, <8 x i32>* %ptr_b
+  %res = call <16 x i16> @llvm.x86.avx512.mask.packusdw.256(<8 x i32> %a, <8 x i32> %b, <16 x i16> zeroinitializer, i16 %mask)
+  ret <16 x i16> %res
+}
+
+define <16 x i16> @test_mask_packus_epi32_rmb_256(<8 x i32> %a, i32* %ptr_b) {
+  ;CHECK-LABEL: test_mask_packus_epi32_rmb_256
+  ;CHECK: vpackusdw       (%rdi){1to8}, %ymm0, %ymm0  
+  %q = load i32, i32* %ptr_b
+  %vecinit.i = insertelement <8 x i32> undef, i32 %q, i32 0
+  %b = shufflevector <8 x i32> %vecinit.i, <8 x i32> undef, <8 x i32> zeroinitializer
+  %res = call <16 x i16> @llvm.x86.avx512.mask.packusdw.256(<8 x i32> %a, <8 x i32> %b, <16 x i16> zeroinitializer, i16 -1)
+  ret <16 x i16> %res
+}
+
+define <16 x i16> @test_mask_packus_epi32_rmbk_256(<8 x i32> %a, i32* %ptr_b, <16 x i16> %passThru, i16 %mask) {
+  ;CHECK-LABEL: test_mask_packus_epi32_rmbk_256
+  ;CHECK: vpackusdw       (%rdi){1to8}, %ymm0, %ymm1 {%k1} 
+  %q = load i32, i32* %ptr_b
+  %vecinit.i = insertelement <8 x i32> undef, i32 %q, i32 0
+  %b = shufflevector <8 x i32> %vecinit.i, <8 x i32> undef, <8 x i32> zeroinitializer
+  %res = call <16 x i16> @llvm.x86.avx512.mask.packusdw.256(<8 x i32> %a, <8 x i32> %b, <16 x i16> %passThru, i16 %mask)
+  ret <16 x i16> %res
+}
+
+define <16 x i16> @test_mask_packus_epi32_rmbkz_256(<8 x i32> %a, i32* %ptr_b, i16 %mask) {
+  ;CHECK-LABEL: test_mask_packus_epi32_rmbkz_256
+  ;CHECK: vpackusdw       (%rdi){1to8}, %ymm0, %ymm0 {%k1} {z} 
+  %q = load i32, i32* %ptr_b
+  %vecinit.i = insertelement <8 x i32> undef, i32 %q, i32 0
+  %b = shufflevector <8 x i32> %vecinit.i, <8 x i32> undef, <8 x i32> zeroinitializer
+  %res = call <16 x i16> @llvm.x86.avx512.mask.packusdw.256(<8 x i32> %a, <8 x i32> %b, <16 x i16> zeroinitializer, i16 %mask)
+  ret <16 x i16> %res
+}
+
+declare <16 x i16> @llvm.x86.avx512.mask.packusdw.256(<8 x i32>, <8 x i32>, <16 x i16>, i16)
+
+define <16 x i8> @test_mask_packus_epi16_rr_128(<8 x i16> %a, <8 x i16> %b) {
+  ;CHECK-LABEL: test_mask_packus_epi16_rr_128
+  ;CHECK: vpackuswb       %xmm1, %xmm0, %xmm0 
+  %res = call <16 x i8> @llvm.x86.avx512.mask.packuswb.128(<8 x i16> %a, <8 x i16> %b, <16 x i8> zeroinitializer, i16 -1)
+  ret <16 x i8> %res
+}
+
+define <16 x i8> @test_mask_packus_epi16_rrk_128(<8 x i16> %a, <8 x i16> %b, <16 x i8> %passThru, i16 %mask) {
+  ;CHECK-LABEL: test_mask_packus_epi16_rrk_128
+  ;CHECK: vpackuswb       %xmm1, %xmm0, %xmm2 {%k1} 
+  %res = call <16 x i8> @llvm.x86.avx512.mask.packuswb.128(<8 x i16> %a, <8 x i16> %b, <16 x i8> %passThru, i16 %mask)
+  ret <16 x i8> %res
+}
+
+define <16 x i8> @test_mask_packus_epi16_rrkz_128(<8 x i16> %a, <8 x i16> %b, i16 %mask) {
+  ;CHECK-LABEL: test_mask_packus_epi16_rrkz_128
+  ;CHECK: vpackuswb       %xmm1, %xmm0, %xmm0 {%k1} {z} 
+  %res = call <16 x i8> @llvm.x86.avx512.mask.packuswb.128(<8 x i16> %a, <8 x i16> %b, <16 x i8> zeroinitializer, i16 %mask)
+  ret <16 x i8> %res
+}
+
+define <16 x i8> @test_mask_packus_epi16_rm_128(<8 x i16> %a, <8 x i16>* %ptr_b) {
+  ;CHECK-LABEL: test_mask_packus_epi16_rm_128
+  ;CHECK: vpackuswb       (%rdi), %xmm0, %xmm0 
+  %b = load <8 x i16>, <8 x i16>* %ptr_b
+  %res = call <16 x i8> @llvm.x86.avx512.mask.packuswb.128(<8 x i16> %a, <8 x i16> %b, <16 x i8> zeroinitializer, i16 -1)
+  ret <16 x i8> %res
+}
+
+define <16 x i8> @test_mask_packus_epi16_rmk_128(<8 x i16> %a, <8 x i16>* %ptr_b, <16 x i8> %passThru, i16 %mask) {
+  ;CHECK-LABEL: test_mask_packus_epi16_rmk_128
+  ;CHECK: vpackuswb       (%rdi), %xmm0, %xmm1 {%k1} 
+  %b = load <8 x i16>, <8 x i16>* %ptr_b
+  %res = call <16 x i8> @llvm.x86.avx512.mask.packuswb.128(<8 x i16> %a, <8 x i16> %b, <16 x i8> %passThru, i16 %mask)
+  ret <16 x i8> %res
+}
+
+define <16 x i8> @test_mask_packus_epi16_rmkz_128(<8 x i16> %a, <8 x i16>* %ptr_b, i16 %mask) {
+  ;CHECK-LABEL: test_mask_packus_epi16_rmkz_128
+  ;CHECK: vpackuswb       (%rdi), %xmm0, %xmm0 {%k1} {z} 
+  %b = load <8 x i16>, <8 x i16>* %ptr_b
+  %res = call <16 x i8> @llvm.x86.avx512.mask.packuswb.128(<8 x i16> %a, <8 x i16> %b, <16 x i8> zeroinitializer, i16 %mask)
+  ret <16 x i8> %res
+}
+
+declare <16 x i8> @llvm.x86.avx512.mask.packuswb.128(<8 x i16>, <8 x i16>, <16 x i8>, i16)
+
+define <32 x i8> @test_mask_packus_epi16_rr_256(<16 x i16> %a, <16 x i16> %b) {
+  ;CHECK-LABEL: test_mask_packus_epi16_rr_256
+  ;CHECK: vpackuswb       %ymm1, %ymm0, %ymm0 
+  %res = call <32 x i8> @llvm.x86.avx512.mask.packuswb.256(<16 x i16> %a, <16 x i16> %b, <32 x i8> zeroinitializer, i32 -1)
+  ret <32 x i8> %res
+}
+
+define <32 x i8> @test_mask_packus_epi16_rrk_256(<16 x i16> %a, <16 x i16> %b, <32 x i8> %passThru, i32 %mask) {
+  ;CHECK-LABEL: test_mask_packus_epi16_rrk_256
+  ;CHECK: vpackuswb       %ymm1, %ymm0, %ymm2 {%k1} 
+  %res = call <32 x i8> @llvm.x86.avx512.mask.packuswb.256(<16 x i16> %a, <16 x i16> %b, <32 x i8> %passThru, i32 %mask)
+  ret <32 x i8> %res
+}
+
+define <32 x i8> @test_mask_packus_epi16_rrkz_256(<16 x i16> %a, <16 x i16> %b, i32 %mask) {
+  ;CHECK-LABEL: test_mask_packus_epi16_rrkz_256
+  ;CHECK: vpackuswb       %ymm1, %ymm0, %ymm0 {%k1} {z} 
+  %res = call <32 x i8> @llvm.x86.avx512.mask.packuswb.256(<16 x i16> %a, <16 x i16> %b, <32 x i8> zeroinitializer, i32 %mask)
+  ret <32 x i8> %res
+}
+
+define <32 x i8> @test_mask_packus_epi16_rm_256(<16 x i16> %a, <16 x i16>* %ptr_b) {
+  ;CHECK-LABEL: test_mask_packus_epi16_rm_256
+  ;CHECK: vpackuswb       (%rdi), %ymm0, %ymm0 
+  %b = load <16 x i16>, <16 x i16>* %ptr_b
+  %res = call <32 x i8> @llvm.x86.avx512.mask.packuswb.256(<16 x i16> %a, <16 x i16> %b, <32 x i8> zeroinitializer, i32 -1)
+  ret <32 x i8> %res
+}
+
+define <32 x i8> @test_mask_packus_epi16_rmk_256(<16 x i16> %a, <16 x i16>* %ptr_b, <32 x i8> %passThru, i32 %mask) {
+  ;CHECK-LABEL: test_mask_packus_epi16_rmk_256
+  ;CHECK: vpackuswb       (%rdi), %ymm0, %ymm1 {%k1} 
+  %b = load <16 x i16>, <16 x i16>* %ptr_b
+  %res = call <32 x i8> @llvm.x86.avx512.mask.packuswb.256(<16 x i16> %a, <16 x i16> %b, <32 x i8> %passThru, i32 %mask)
+  ret <32 x i8> %res
+}
+
+define <32 x i8> @test_mask_packus_epi16_rmkz_256(<16 x i16> %a, <16 x i16>* %ptr_b, i32 %mask) {
+  ;CHECK-LABEL: test_mask_packus_epi16_rmkz_256
+  ;CHECK: vpackuswb       (%rdi), %ymm0, %ymm0 {%k1} {z} 
+  %b = load <16 x i16>, <16 x i16>* %ptr_b
+  %res = call <32 x i8> @llvm.x86.avx512.mask.packuswb.256(<16 x i16> %a, <16 x i16> %b, <32 x i8> zeroinitializer, i32 %mask)
+  ret <32 x i8> %res
+}
+
+declare <32 x i8> @llvm.x86.avx512.mask.packuswb.256(<16 x i16>, <16 x i16>, <32 x i8>, i32)
+
+define <8 x i16> @test_mask_adds_epi16_rr_128(<8 x i16> %a, <8 x i16> %b) {
+  ;CHECK-LABEL: test_mask_adds_epi16_rr_128
+  ;CHECK: vpaddsw %xmm1, %xmm0, %xmm0 
+  %res = call <8 x i16> @llvm.x86.avx512.mask.padds.w.128(<8 x i16> %a, <8 x i16> %b, <8 x i16> zeroinitializer, i8 -1)
+  ret <8 x i16> %res
+}
+
+define <8 x i16> @test_mask_adds_epi16_rrk_128(<8 x i16> %a, <8 x i16> %b, <8 x i16> %passThru, i8 %mask) {
+  ;CHECK-LABEL: test_mask_adds_epi16_rrk_128
+  ;CHECK: vpaddsw %xmm1, %xmm0, %xmm2 {%k1} 
+  %res = call <8 x i16> @llvm.x86.avx512.mask.padds.w.128(<8 x i16> %a, <8 x i16> %b, <8 x i16> %passThru, i8 %mask)
+  ret <8 x i16> %res
+}
+
+define <8 x i16> @test_mask_adds_epi16_rrkz_128(<8 x i16> %a, <8 x i16> %b, i8 %mask) {
+  ;CHECK-LABEL: test_mask_adds_epi16_rrkz_128
+  ;CHECK: vpaddsw %xmm1, %xmm0, %xmm0 {%k1} {z} 
+  %res = call <8 x i16> @llvm.x86.avx512.mask.padds.w.128(<8 x i16> %a, <8 x i16> %b, <8 x i16> zeroinitializer, i8 %mask)
+  ret <8 x i16> %res
+}
+
+define <8 x i16> @test_mask_adds_epi16_rm_128(<8 x i16> %a, <8 x i16>* %ptr_b) {
+  ;CHECK-LABEL: test_mask_adds_epi16_rm_128
+  ;CHECK: vpaddsw (%rdi), %xmm0, %xmm0 
+  %b = load <8 x i16>, <8 x i16>* %ptr_b
+  %res = call <8 x i16> @llvm.x86.avx512.mask.padds.w.128(<8 x i16> %a, <8 x i16> %b, <8 x i16> zeroinitializer, i8 -1)
+  ret <8 x i16> %res
+}
+
+define <8 x i16> @test_mask_adds_epi16_rmk_128(<8 x i16> %a, <8 x i16>* %ptr_b, <8 x i16> %passThru, i8 %mask) {
+  ;CHECK-LABEL: test_mask_adds_epi16_rmk_128
+  ;CHECK: vpaddsw (%rdi), %xmm0, %xmm1 {%k1} 
+  %b = load <8 x i16>, <8 x i16>* %ptr_b
+  %res = call <8 x i16> @llvm.x86.avx512.mask.padds.w.128(<8 x i16> %a, <8 x i16> %b, <8 x i16> %passThru, i8 %mask)
+  ret <8 x i16> %res
+}
+
+define <8 x i16> @test_mask_adds_epi16_rmkz_128(<8 x i16> %a, <8 x i16>* %ptr_b, i8 %mask) {
+  ;CHECK-LABEL: test_mask_adds_epi16_rmkz_128
+  ;CHECK: vpaddsw (%rdi), %xmm0, %xmm0 {%k1} {z} 
+  %b = load <8 x i16>, <8 x i16>* %ptr_b
+  %res = call <8 x i16> @llvm.x86.avx512.mask.padds.w.128(<8 x i16> %a, <8 x i16> %b, <8 x i16> zeroinitializer, i8 %mask)
+  ret <8 x i16> %res
+}
+
+declare <8 x i16> @llvm.x86.avx512.mask.padds.w.128(<8 x i16>, <8 x i16>, <8 x i16>, i8)
+
+define <16 x i16> @test_mask_adds_epi16_rr_256(<16 x i16> %a, <16 x i16> %b) {
+  ;CHECK-LABEL: test_mask_adds_epi16_rr_256
+  ;CHECK: vpaddsw %ymm1, %ymm0, %ymm0 
+  %res = call <16 x i16> @llvm.x86.avx512.mask.padds.w.256(<16 x i16> %a, <16 x i16> %b, <16 x i16> zeroinitializer, i16 -1)
+  ret <16 x i16> %res
+}
+
+define <16 x i16> @test_mask_adds_epi16_rrk_256(<16 x i16> %a, <16 x i16> %b, <16 x i16> %passThru, i16 %mask) {
+  ;CHECK-LABEL: test_mask_adds_epi16_rrk_256
+  ;CHECK: vpaddsw %ymm1, %ymm0, %ymm2 {%k1} 
+  %res = call <16 x i16> @llvm.x86.avx512.mask.padds.w.256(<16 x i16> %a, <16 x i16> %b, <16 x i16> %passThru, i16 %mask)
+  ret <16 x i16> %res
+}
+
+define <16 x i16> @test_mask_adds_epi16_rrkz_256(<16 x i16> %a, <16 x i16> %b, i16 %mask) {
+  ;CHECK-LABEL: test_mask_adds_epi16_rrkz_256
+  ;CHECK: vpaddsw %ymm1, %ymm0, %ymm0 {%k1} {z} 
+  %res = call <16 x i16> @llvm.x86.avx512.mask.padds.w.256(<16 x i16> %a, <16 x i16> %b, <16 x i16> zeroinitializer, i16 %mask)
+  ret <16 x i16> %res
+}
+
+define <16 x i16> @test_mask_adds_epi16_rm_256(<16 x i16> %a, <16 x i16>* %ptr_b) {
+  ;CHECK-LABEL: test_mask_adds_epi16_rm_256
+  ;CHECK: vpaddsw (%rdi), %ymm0, %ymm0    
+  %b = load <16 x i16>, <16 x i16>* %ptr_b
+  %res = call <16 x i16> @llvm.x86.avx512.mask.padds.w.256(<16 x i16> %a, <16 x i16> %b, <16 x i16> zeroinitializer, i16 -1)
+  ret <16 x i16> %res
+}
+
+define <16 x i16> @test_mask_adds_epi16_rmk_256(<16 x i16> %a, <16 x i16>* %ptr_b, <16 x i16> %passThru, i16 %mask) {
+  ;CHECK-LABEL: test_mask_adds_epi16_rmk_256
+  ;CHECK: vpaddsw (%rdi), %ymm0, %ymm1 {%k1} 
+  %b = load <16 x i16>, <16 x i16>* %ptr_b
+  %res = call <16 x i16> @llvm.x86.avx512.mask.padds.w.256(<16 x i16> %a, <16 x i16> %b, <16 x i16> %passThru, i16 %mask)
+  ret <16 x i16> %res
+}
+
+define <16 x i16> @test_mask_adds_epi16_rmkz_256(<16 x i16> %a, <16 x i16>* %ptr_b, i16 %mask) {
+  ;CHECK-LABEL: test_mask_adds_epi16_rmkz_256
+  ;CHECK: vpaddsw (%rdi), %ymm0, %ymm0 {%k1} {z} 
+  %b = load <16 x i16>, <16 x i16>* %ptr_b
+  %res = call <16 x i16> @llvm.x86.avx512.mask.padds.w.256(<16 x i16> %a, <16 x i16> %b, <16 x i16> zeroinitializer, i16 %mask)
+  ret <16 x i16> %res
+}
+
+declare <16 x i16> @llvm.x86.avx512.mask.padds.w.256(<16 x i16>, <16 x i16>, <16 x i16>, i16)
+
+define <8 x i16> @test_mask_subs_epi16_rr_128(<8 x i16> %a, <8 x i16> %b) {
+  ;CHECK-LABEL: test_mask_subs_epi16_rr_128
+  ;CHECK: vpsubsw %xmm1, %xmm0, %xmm0     
+  %res = call <8 x i16> @llvm.x86.avx512.mask.psubs.w.128(<8 x i16> %a, <8 x i16> %b, <8 x i16> zeroinitializer, i8 -1)
+  ret <8 x i16> %res
+}
+
+define <8 x i16> @test_mask_subs_epi16_rrk_128(<8 x i16> %a, <8 x i16> %b, <8 x i16> %passThru, i8 %mask) {
+  ;CHECK-LABEL: test_mask_subs_epi16_rrk_128
+  ;CHECK: vpsubsw %xmm1, %xmm0, %xmm2 {%k1} 
+  %res = call <8 x i16> @llvm.x86.avx512.mask.psubs.w.128(<8 x i16> %a, <8 x i16> %b, <8 x i16> %passThru, i8 %mask)
+  ret <8 x i16> %res
+}
+
+define <8 x i16> @test_mask_subs_epi16_rrkz_128(<8 x i16> %a, <8 x i16> %b, i8 %mask) {
+  ;CHECK-LABEL: test_mask_subs_epi16_rrkz_128
+  ;CHECK: vpsubsw %xmm1, %xmm0, %xmm0 {%k1} {z} 
+  %res = call <8 x i16> @llvm.x86.avx512.mask.psubs.w.128(<8 x i16> %a, <8 x i16> %b, <8 x i16> zeroinitializer, i8 %mask)
+  ret <8 x i16> %res
+}
+
+define <8 x i16> @test_mask_subs_epi16_rm_128(<8 x i16> %a, <8 x i16>* %ptr_b) {
+  ;CHECK-LABEL: test_mask_subs_epi16_rm_128
+  ;CHECK: vpsubsw (%rdi), %xmm0, %xmm0
+  %b = load <8 x i16>, <8 x i16>* %ptr_b
+  %res = call <8 x i16> @llvm.x86.avx512.mask.psubs.w.128(<8 x i16> %a, <8 x i16> %b, <8 x i16> zeroinitializer, i8 -1)
+  ret <8 x i16> %res
+}
+
+define <8 x i16> @test_mask_subs_epi16_rmk_128(<8 x i16> %a, <8 x i16>* %ptr_b, <8 x i16> %passThru, i8 %mask) {
+  ;CHECK-LABEL: test_mask_subs_epi16_rmk_128
+  ;CHECK: vpsubsw (%rdi), %xmm0, %xmm1 {%k1} 
+  %b = load <8 x i16>, <8 x i16>* %ptr_b
+  %res = call <8 x i16> @llvm.x86.avx512.mask.psubs.w.128(<8 x i16> %a, <8 x i16> %b, <8 x i16> %passThru, i8 %mask)
+  ret <8 x i16> %res
+}
+
+define <8 x i16> @test_mask_subs_epi16_rmkz_128(<8 x i16> %a, <8 x i16>* %ptr_b, i8 %mask) {
+  ;CHECK-LABEL: test_mask_subs_epi16_rmkz_128
+  ;CHECK: vpsubsw (%rdi), %xmm0, %xmm0 {%k1} {z} 
+  %b = load <8 x i16>, <8 x i16>* %ptr_b
+  %res = call <8 x i16> @llvm.x86.avx512.mask.psubs.w.128(<8 x i16> %a, <8 x i16> %b, <8 x i16> zeroinitializer, i8 %mask)
+  ret <8 x i16> %res
+}
+
+declare <8 x i16> @llvm.x86.avx512.mask.psubs.w.128(<8 x i16>, <8 x i16>, <8 x i16>, i8)
+
+define <16 x i16> @test_mask_subs_epi16_rr_256(<16 x i16> %a, <16 x i16> %b) {
+  ;CHECK-LABEL: test_mask_subs_epi16_rr_256
+  ;CHECK: vpsubsw %ymm1, %ymm0, %ymm0     
+  %res = call <16 x i16> @llvm.x86.avx512.mask.psubs.w.256(<16 x i16> %a, <16 x i16> %b, <16 x i16> zeroinitializer, i16 -1)
+  ret <16 x i16> %res
+}
+
+define <16 x i16> @test_mask_subs_epi16_rrk_256(<16 x i16> %a, <16 x i16> %b, <16 x i16> %passThru, i16 %mask) {
+  ;CHECK-LABEL: test_mask_subs_epi16_rrk_256
+  ;CHECK: vpsubsw %ymm1, %ymm0, %ymm2 {%k1} 
+  %res = call <16 x i16> @llvm.x86.avx512.mask.psubs.w.256(<16 x i16> %a, <16 x i16> %b, <16 x i16> %passThru, i16 %mask)
+  ret <16 x i16> %res
+}
+
+define <16 x i16> @test_mask_subs_epi16_rrkz_256(<16 x i16> %a, <16 x i16> %b, i16 %mask) {
+  ;CHECK-LABEL: test_mask_subs_epi16_rrkz_256
+  ;CHECK: vpsubsw %ymm1, %ymm0, %ymm0 {%k1} {z} 
+  %res = call <16 x i16> @llvm.x86.avx512.mask.psubs.w.256(<16 x i16> %a, <16 x i16> %b, <16 x i16> zeroinitializer, i16 %mask)
+  ret <16 x i16> %res
+}
+
+define <16 x i16> @test_mask_subs_epi16_rm_256(<16 x i16> %a, <16 x i16>* %ptr_b) {
+  ;CHECK-LABEL: test_mask_subs_epi16_rm_256
+  ;CHECK: vpsubsw (%rdi), %ymm0, %ymm0    
+  %b = load <16 x i16>, <16 x i16>* %ptr_b
+  %res = call <16 x i16> @llvm.x86.avx512.mask.psubs.w.256(<16 x i16> %a, <16 x i16> %b, <16 x i16> zeroinitializer, i16 -1)
+  ret <16 x i16> %res
+}
+
+define <16 x i16> @test_mask_subs_epi16_rmk_256(<16 x i16> %a, <16 x i16>* %ptr_b, <16 x i16> %passThru, i16 %mask) {
+  ;CHECK-LABEL: test_mask_subs_epi16_rmk_256
+  ;CHECK: vpsubsw (%rdi), %ymm0, %ymm1 {%k1} 
+  %b = load <16 x i16>, <16 x i16>* %ptr_b
+  %res = call <16 x i16> @llvm.x86.avx512.mask.psubs.w.256(<16 x i16> %a, <16 x i16> %b, <16 x i16> %passThru, i16 %mask)
+  ret <16 x i16> %res
+}
+
+define <16 x i16> @test_mask_subs_epi16_rmkz_256(<16 x i16> %a, <16 x i16>* %ptr_b, i16 %mask) {
+  ;CHECK-LABEL: test_mask_subs_epi16_rmkz_256
+  ;CHECK: vpsubsw (%rdi), %ymm0, %ymm0 {%k1} {z} 
+  %b = load <16 x i16>, <16 x i16>* %ptr_b
+  %res = call <16 x i16> @llvm.x86.avx512.mask.psubs.w.256(<16 x i16> %a, <16 x i16> %b, <16 x i16> zeroinitializer, i16 %mask)
+  ret <16 x i16> %res
+}
+
+declare <16 x i16> @llvm.x86.avx512.mask.psubs.w.256(<16 x i16>, <16 x i16>, <16 x i16>, i16)
+
+define <8 x i16> @test_mask_adds_epu16_rr_128(<8 x i16> %a, <8 x i16> %b) {
+  ;CHECK-LABEL: test_mask_adds_epu16_rr_128
+  ;CHECK: vpaddusw %xmm1, %xmm0, %xmm0 
+  %res = call <8 x i16> @llvm.x86.avx512.mask.paddus.w.128(<8 x i16> %a, <8 x i16> %b, <8 x i16> zeroinitializer, i8 -1)
+  ret <8 x i16> %res
+}
+
+define <8 x i16> @test_mask_adds_epu16_rrk_128(<8 x i16> %a, <8 x i16> %b, <8 x i16> %passThru, i8 %mask) {
+  ;CHECK-LABEL: test_mask_adds_epu16_rrk_128
+  ;CHECK: vpaddusw %xmm1, %xmm0, %xmm2 {%k1} 
+  %res = call <8 x i16> @llvm.x86.avx512.mask.paddus.w.128(<8 x i16> %a, <8 x i16> %b, <8 x i16> %passThru, i8 %mask)
+  ret <8 x i16> %res
+}
+
+define <8 x i16> @test_mask_adds_epu16_rrkz_128(<8 x i16> %a, <8 x i16> %b, i8 %mask) {
+  ;CHECK-LABEL: test_mask_adds_epu16_rrkz_128
+  ;CHECK: vpaddusw %xmm1, %xmm0, %xmm0 {%k1} {z} 
+  %res = call <8 x i16> @llvm.x86.avx512.mask.paddus.w.128(<8 x i16> %a, <8 x i16> %b, <8 x i16> zeroinitializer, i8 %mask)
+  ret <8 x i16> %res
+}
+
+define <8 x i16> @test_mask_adds_epu16_rm_128(<8 x i16> %a, <8 x i16>* %ptr_b) {
+  ;CHECK-LABEL: test_mask_adds_epu16_rm_128
+  ;CHECK: vpaddusw (%rdi), %xmm0, %xmm0 
+  %b = load <8 x i16>, <8 x i16>* %ptr_b
+  %res = call <8 x i16> @llvm.x86.avx512.mask.paddus.w.128(<8 x i16> %a, <8 x i16> %b, <8 x i16> zeroinitializer, i8 -1)
+  ret <8 x i16> %res
+}
+
+define <8 x i16> @test_mask_adds_epu16_rmk_128(<8 x i16> %a, <8 x i16>* %ptr_b, <8 x i16> %passThru, i8 %mask) {
+  ;CHECK-LABEL: test_mask_adds_epu16_rmk_128
+  ;CHECK: vpaddusw (%rdi), %xmm0, %xmm1 {%k1} 
+  %b = load <8 x i16>, <8 x i16>* %ptr_b
+  %res = call <8 x i16> @llvm.x86.avx512.mask.paddus.w.128(<8 x i16> %a, <8 x i16> %b, <8 x i16> %passThru, i8 %mask)
+  ret <8 x i16> %res
+}
+
+define <8 x i16> @test_mask_adds_epu16_rmkz_128(<8 x i16> %a, <8 x i16>* %ptr_b, i8 %mask) {
+  ;CHECK-LABEL: test_mask_adds_epu16_rmkz_128
+  ;CHECK: vpaddusw (%rdi), %xmm0, %xmm0 {%k1} {z} 
+  %b = load <8 x i16>, <8 x i16>* %ptr_b
+  %res = call <8 x i16> @llvm.x86.avx512.mask.paddus.w.128(<8 x i16> %a, <8 x i16> %b, <8 x i16> zeroinitializer, i8 %mask)
+  ret <8 x i16> %res
+}
+
+declare <8 x i16> @llvm.x86.avx512.mask.paddus.w.128(<8 x i16>, <8 x i16>, <8 x i16>, i8)
+
+define <16 x i16> @test_mask_adds_epu16_rr_256(<16 x i16> %a, <16 x i16> %b) {
+  ;CHECK-LABEL: test_mask_adds_epu16_rr_256
+  ;CHECK: vpaddusw %ymm1, %ymm0, %ymm0 
+  %res = call <16 x i16> @llvm.x86.avx512.mask.paddus.w.256(<16 x i16> %a, <16 x i16> %b, <16 x i16> zeroinitializer, i16 -1)
+  ret <16 x i16> %res
+}
+
+define <16 x i16> @test_mask_adds_epu16_rrk_256(<16 x i16> %a, <16 x i16> %b, <16 x i16> %passThru, i16 %mask) {
+  ;CHECK-LABEL: test_mask_adds_epu16_rrk_256
+  ;CHECK: vpaddusw %ymm1, %ymm0, %ymm2 {%k1} 
+  %res = call <16 x i16> @llvm.x86.avx512.mask.paddus.w.256(<16 x i16> %a, <16 x i16> %b, <16 x i16> %passThru, i16 %mask)
+  ret <16 x i16> %res
+}
+
+define <16 x i16> @test_mask_adds_epu16_rrkz_256(<16 x i16> %a, <16 x i16> %b, i16 %mask) {
+  ;CHECK-LABEL: test_mask_adds_epu16_rrkz_256
+  ;CHECK: vpaddusw %ymm1, %ymm0, %ymm0 {%k1} {z} 
+  %res = call <16 x i16> @llvm.x86.avx512.mask.paddus.w.256(<16 x i16> %a, <16 x i16> %b, <16 x i16> zeroinitializer, i16 %mask)
+  ret <16 x i16> %res
+}
+
+define <16 x i16> @test_mask_adds_epu16_rm_256(<16 x i16> %a, <16 x i16>* %ptr_b) {
+  ;CHECK-LABEL: test_mask_adds_epu16_rm_256
+  ;CHECK: vpaddusw (%rdi), %ymm0, %ymm0    
+  %b = load <16 x i16>, <16 x i16>* %ptr_b
+  %res = call <16 x i16> @llvm.x86.avx512.mask.paddus.w.256(<16 x i16> %a, <16 x i16> %b, <16 x i16> zeroinitializer, i16 -1)
+  ret <16 x i16> %res
+}
+
+define <16 x i16> @test_mask_adds_epu16_rmk_256(<16 x i16> %a, <16 x i16>* %ptr_b, <16 x i16> %passThru, i16 %mask) {
+  ;CHECK-LABEL: test_mask_adds_epu16_rmk_256
+  ;CHECK: vpaddusw (%rdi), %ymm0, %ymm1 {%k1} 
+  %b = load <16 x i16>, <16 x i16>* %ptr_b
+  %res = call <16 x i16> @llvm.x86.avx512.mask.paddus.w.256(<16 x i16> %a, <16 x i16> %b, <16 x i16> %passThru, i16 %mask)
+  ret <16 x i16> %res
+}
+
+define <16 x i16> @test_mask_adds_epu16_rmkz_256(<16 x i16> %a, <16 x i16>* %ptr_b, i16 %mask) {
+  ;CHECK-LABEL: test_mask_adds_epu16_rmkz_256
+  ;CHECK: vpaddusw (%rdi), %ymm0, %ymm0 {%k1} {z} 
+  %b = load <16 x i16>, <16 x i16>* %ptr_b
+  %res = call <16 x i16> @llvm.x86.avx512.mask.paddus.w.256(<16 x i16> %a, <16 x i16> %b, <16 x i16> zeroinitializer, i16 %mask)
+  ret <16 x i16> %res
+}
+
+declare <16 x i16> @llvm.x86.avx512.mask.paddus.w.256(<16 x i16>, <16 x i16>, <16 x i16>, i16)
+
+define <8 x i16> @test_mask_subs_epu16_rr_128(<8 x i16> %a, <8 x i16> %b) {
+  ;CHECK-LABEL: test_mask_subs_epu16_rr_128
+  ;CHECK: vpsubusw %xmm1, %xmm0, %xmm0     
+  %res = call <8 x i16> @llvm.x86.avx512.mask.psubus.w.128(<8 x i16> %a, <8 x i16> %b, <8 x i16> zeroinitializer, i8 -1)
+  ret <8 x i16> %res
+}
+
+define <8 x i16> @test_mask_subs_epu16_rrk_128(<8 x i16> %a, <8 x i16> %b, <8 x i16> %passThru, i8 %mask) {
+  ;CHECK-LABEL: test_mask_subs_epu16_rrk_128
+  ;CHECK: vpsubusw %xmm1, %xmm0, %xmm2 {%k1} 
+  %res = call <8 x i16> @llvm.x86.avx512.mask.psubus.w.128(<8 x i16> %a, <8 x i16> %b, <8 x i16> %passThru, i8 %mask)
+  ret <8 x i16> %res
+}
+
+define <8 x i16> @test_mask_subs_epu16_rrkz_128(<8 x i16> %a, <8 x i16> %b, i8 %mask) {
+  ;CHECK-LABEL: test_mask_subs_epu16_rrkz_128
+  ;CHECK: vpsubusw %xmm1, %xmm0, %xmm0 {%k1} {z} 
+  %res = call <8 x i16> @llvm.x86.avx512.mask.psubus.w.128(<8 x i16> %a, <8 x i16> %b, <8 x i16> zeroinitializer, i8 %mask)
+  ret <8 x i16> %res
+}
+
+define <8 x i16> @test_mask_subs_epu16_rm_128(<8 x i16> %a, <8 x i16>* %ptr_b) {
+  ;CHECK-LABEL: test_mask_subs_epu16_rm_128
+  ;CHECK: vpsubusw (%rdi), %xmm0, %xmm0
+  %b = load <8 x i16>, <8 x i16>* %ptr_b
+  %res = call <8 x i16> @llvm.x86.avx512.mask.psubus.w.128(<8 x i16> %a, <8 x i16> %b, <8 x i16> zeroinitializer, i8 -1)
+  ret <8 x i16> %res
+}
+
+define <8 x i16> @test_mask_subs_epu16_rmk_128(<8 x i16> %a, <8 x i16>* %ptr_b, <8 x i16> %passThru, i8 %mask) {
+  ;CHECK-LABEL: test_mask_subs_epu16_rmk_128
+  ;CHECK: vpsubusw (%rdi), %xmm0, %xmm1 {%k1} 
+  %b = load <8 x i16>, <8 x i16>* %ptr_b
+  %res = call <8 x i16> @llvm.x86.avx512.mask.psubus.w.128(<8 x i16> %a, <8 x i16> %b, <8 x i16> %passThru, i8 %mask)
+  ret <8 x i16> %res
+}
+
+define <8 x i16> @test_mask_subs_epu16_rmkz_128(<8 x i16> %a, <8 x i16>* %ptr_b, i8 %mask) {
+  ;CHECK-LABEL: test_mask_subs_epu16_rmkz_128
+  ;CHECK: vpsubusw (%rdi), %xmm0, %xmm0 {%k1} {z} 
+  %b = load <8 x i16>, <8 x i16>* %ptr_b
+  %res = call <8 x i16> @llvm.x86.avx512.mask.psubus.w.128(<8 x i16> %a, <8 x i16> %b, <8 x i16> zeroinitializer, i8 %mask)
+  ret <8 x i16> %res
+}
+
+declare <8 x i16> @llvm.x86.avx512.mask.psubus.w.128(<8 x i16>, <8 x i16>, <8 x i16>, i8)
+
+define <16 x i16> @test_mask_subs_epu16_rr_256(<16 x i16> %a, <16 x i16> %b) {
+  ;CHECK-LABEL: test_mask_subs_epu16_rr_256
+  ;CHECK: vpsubusw %ymm1, %ymm0, %ymm0     
+  %res = call <16 x i16> @llvm.x86.avx512.mask.psubus.w.256(<16 x i16> %a, <16 x i16> %b, <16 x i16> zeroinitializer, i16 -1)
+  ret <16 x i16> %res
+}
+
+define <16 x i16> @test_mask_subs_epu16_rrk_256(<16 x i16> %a, <16 x i16> %b, <16 x i16> %passThru, i16 %mask) {
+  ;CHECK-LABEL: test_mask_subs_epu16_rrk_256
+  ;CHECK: vpsubusw %ymm1, %ymm0, %ymm2 {%k1} 
+  %res = call <16 x i16> @llvm.x86.avx512.mask.psubus.w.256(<16 x i16> %a, <16 x i16> %b, <16 x i16> %passThru, i16 %mask)
+  ret <16 x i16> %res
+}
+
+define <16 x i16> @test_mask_subs_epu16_rrkz_256(<16 x i16> %a, <16 x i16> %b, i16 %mask) {
+  ;CHECK-LABEL: test_mask_subs_epu16_rrkz_256
+  ;CHECK: vpsubusw %ymm1, %ymm0, %ymm0 {%k1} {z} 
+  %res = call <16 x i16> @llvm.x86.avx512.mask.psubus.w.256(<16 x i16> %a, <16 x i16> %b, <16 x i16> zeroinitializer, i16 %mask)
+  ret <16 x i16> %res
+}
+
+define <16 x i16> @test_mask_subs_epu16_rm_256(<16 x i16> %a, <16 x i16>* %ptr_b) {
+  ;CHECK-LABEL: test_mask_subs_epu16_rm_256
+  ;CHECK: vpsubusw (%rdi), %ymm0, %ymm0    
+  %b = load <16 x i16>, <16 x i16>* %ptr_b
+  %res = call <16 x i16> @llvm.x86.avx512.mask.psubus.w.256(<16 x i16> %a, <16 x i16> %b, <16 x i16> zeroinitializer, i16 -1)
+  ret <16 x i16> %res
+}
+
+define <16 x i16> @test_mask_subs_epu16_rmk_256(<16 x i16> %a, <16 x i16>* %ptr_b, <16 x i16> %passThru, i16 %mask) {
+  ;CHECK-LABEL: test_mask_subs_epu16_rmk_256
+  ;CHECK: vpsubusw (%rdi), %ymm0, %ymm1 {%k1} 
+  %b = load <16 x i16>, <16 x i16>* %ptr_b
+  %res = call <16 x i16> @llvm.x86.avx512.mask.psubus.w.256(<16 x i16> %a, <16 x i16> %b, <16 x i16> %passThru, i16 %mask)
+  ret <16 x i16> %res
+}
+
+define <16 x i16> @test_mask_subs_epu16_rmkz_256(<16 x i16> %a, <16 x i16>* %ptr_b, i16 %mask) {
+  ;CHECK-LABEL: test_mask_subs_epu16_rmkz_256
+  ;CHECK: vpsubusw (%rdi), %ymm0, %ymm0 {%k1} {z} 
+  %b = load <16 x i16>, <16 x i16>* %ptr_b
+  %res = call <16 x i16> @llvm.x86.avx512.mask.psubus.w.256(<16 x i16> %a, <16 x i16> %b, <16 x i16> zeroinitializer, i16 %mask)
+  ret <16 x i16> %res
+}
+
+declare <16 x i16> @llvm.x86.avx512.mask.psubus.w.256(<16 x i16>, <16 x i16>, <16 x i16>, i16)
+
+define <16 x i8> @test_mask_adds_epi8_rr_128(<16 x i8> %a, <16 x i8> %b) {
+  ;CHECK-LABEL: test_mask_adds_epi8_rr_128
+  ;CHECK: vpaddsb %xmm1, %xmm0, %xmm0 
+  %res = call <16 x i8> @llvm.x86.avx512.mask.padds.b.128(<16 x i8> %a, <16 x i8> %b, <16 x i8> zeroinitializer, i16 -1)
+  ret <16 x i8> %res
+}
+
+define <16 x i8> @test_mask_adds_epi8_rrk_128(<16 x i8> %a, <16 x i8> %b, <16 x i8> %passThru, i16 %mask) {
+  ;CHECK-LABEL: test_mask_adds_epi8_rrk_128
+  ;CHECK: vpaddsb %xmm1, %xmm0, %xmm2 {%k1} 
+  %res = call <16 x i8> @llvm.x86.avx512.mask.padds.b.128(<16 x i8> %a, <16 x i8> %b, <16 x i8> %passThru, i16 %mask)
+  ret <16 x i8> %res
+}
+
+define <16 x i8> @test_mask_adds_epi8_rrkz_128(<16 x i8> %a, <16 x i8> %b, i16 %mask) {
+  ;CHECK-LABEL: test_mask_adds_epi8_rrkz_128
+  ;CHECK: vpaddsb %xmm1, %xmm0, %xmm0 {%k1} {z} 
+  %res = call <16 x i8> @llvm.x86.avx512.mask.padds.b.128(<16 x i8> %a, <16 x i8> %b, <16 x i8> zeroinitializer, i16 %mask)
+  ret <16 x i8> %res
+}
+
+define <16 x i8> @test_mask_adds_epi8_rm_128(<16 x i8> %a, <16 x i8>* %ptr_b) {
+  ;CHECK-LABEL: test_mask_adds_epi8_rm_128
+  ;CHECK: vpaddsb (%rdi), %xmm0, %xmm0 
+  %b = load <16 x i8>, <16 x i8>* %ptr_b
+  %res = call <16 x i8> @llvm.x86.avx512.mask.padds.b.128(<16 x i8> %a, <16 x i8> %b, <16 x i8> zeroinitializer, i16 -1)
+  ret <16 x i8> %res
+}
+
+define <16 x i8> @test_mask_adds_epi8_rmk_128(<16 x i8> %a, <16 x i8>* %ptr_b, <16 x i8> %passThru, i16 %mask) {
+  ;CHECK-LABEL: test_mask_adds_epi8_rmk_128
+  ;CHECK: vpaddsb (%rdi), %xmm0, %xmm1 {%k1} 
+  %b = load <16 x i8>, <16 x i8>* %ptr_b
+  %res = call <16 x i8> @llvm.x86.avx512.mask.padds.b.128(<16 x i8> %a, <16 x i8> %b, <16 x i8> %passThru, i16 %mask)
+  ret <16 x i8> %res
+}
+
+define <16 x i8> @test_mask_adds_epi8_rmkz_128(<16 x i8> %a, <16 x i8>* %ptr_b, i16 %mask) {
+  ;CHECK-LABEL: test_mask_adds_epi8_rmkz_128
+  ;CHECK: vpaddsb (%rdi), %xmm0, %xmm0 {%k1} {z} 
+  %b = load <16 x i8>, <16 x i8>* %ptr_b
+  %res = call <16 x i8> @llvm.x86.avx512.mask.padds.b.128(<16 x i8> %a, <16 x i8> %b, <16 x i8> zeroinitializer, i16 %mask)
+  ret <16 x i8> %res
+}
+
+declare <16 x i8> @llvm.x86.avx512.mask.padds.b.128(<16 x i8>, <16 x i8>, <16 x i8>, i16)
+
+define <32 x i8> @test_mask_adds_epi8_rr_256(<32 x i8> %a, <32 x i8> %b) {
+  ;CHECK-LABEL: test_mask_adds_epi8_rr_256
+  ;CHECK: vpaddsb %ymm1, %ymm0, %ymm0 
+  %res = call <32 x i8> @llvm.x86.avx512.mask.padds.b.256(<32 x i8> %a, <32 x i8> %b, <32 x i8> zeroinitializer, i32 -1)
+  ret <32 x i8> %res
+}
+
+define <32 x i8> @test_mask_adds_epi8_rrk_256(<32 x i8> %a, <32 x i8> %b, <32 x i8> %passThru, i32 %mask) {
+  ;CHECK-LABEL: test_mask_adds_epi8_rrk_256
+  ;CHECK: vpaddsb %ymm1, %ymm0, %ymm2 {%k1} 
+  %res = call <32 x i8> @llvm.x86.avx512.mask.padds.b.256(<32 x i8> %a, <32 x i8> %b, <32 x i8> %passThru, i32 %mask)
+  ret <32 x i8> %res
+}
+
+define <32 x i8> @test_mask_adds_epi8_rrkz_256(<32 x i8> %a, <32 x i8> %b, i32 %mask) {
+  ;CHECK-LABEL: test_mask_adds_epi8_rrkz_256
+  ;CHECK: vpaddsb %ymm1, %ymm0, %ymm0 {%k1} {z} 
+  %res = call <32 x i8> @llvm.x86.avx512.mask.padds.b.256(<32 x i8> %a, <32 x i8> %b, <32 x i8> zeroinitializer, i32 %mask)
+  ret <32 x i8> %res
+}
+
+define <32 x i8> @test_mask_adds_epi8_rm_256(<32 x i8> %a, <32 x i8>* %ptr_b) {
+  ;CHECK-LABEL: test_mask_adds_epi8_rm_256
+  ;CHECK: vpaddsb (%rdi), %ymm0, %ymm0    
+  %b = load <32 x i8>, <32 x i8>* %ptr_b
+  %res = call <32 x i8> @llvm.x86.avx512.mask.padds.b.256(<32 x i8> %a, <32 x i8> %b, <32 x i8> zeroinitializer, i32 -1)
+  ret <32 x i8> %res
+}
+
+define <32 x i8> @test_mask_adds_epi8_rmk_256(<32 x i8> %a, <32 x i8>* %ptr_b, <32 x i8> %passThru, i32 %mask) {
+  ;CHECK-LABEL: test_mask_adds_epi8_rmk_256
+  ;CHECK: vpaddsb (%rdi), %ymm0, %ymm1 {%k1} 
+  %b = load <32 x i8>, <32 x i8>* %ptr_b
+  %res = call <32 x i8> @llvm.x86.avx512.mask.padds.b.256(<32 x i8> %a, <32 x i8> %b, <32 x i8> %passThru, i32 %mask)
+  ret <32 x i8> %res
+}
+
+define <32 x i8> @test_mask_adds_epi8_rmkz_256(<32 x i8> %a, <32 x i8>* %ptr_b, i32 %mask) {
+  ;CHECK-LABEL: test_mask_adds_epi8_rmkz_256
+  ;CHECK: vpaddsb (%rdi), %ymm0, %ymm0 {%k1} {z} 
+  %b = load <32 x i8>, <32 x i8>* %ptr_b
+  %res = call <32 x i8> @llvm.x86.avx512.mask.padds.b.256(<32 x i8> %a, <32 x i8> %b, <32 x i8> zeroinitializer, i32 %mask)
+  ret <32 x i8> %res
+}
+
+declare <32 x i8> @llvm.x86.avx512.mask.padds.b.256(<32 x i8>, <32 x i8>, <32 x i8>, i32)
+
+define <16 x i8> @test_mask_subs_epi8_rr_128(<16 x i8> %a, <16 x i8> %b) {
+  ;CHECK-LABEL: test_mask_subs_epi8_rr_128
+  ;CHECK: vpsubsb %xmm1, %xmm0, %xmm0     
+  %res = call <16 x i8> @llvm.x86.avx512.mask.psubs.b.128(<16 x i8> %a, <16 x i8> %b, <16 x i8> zeroinitializer, i16 -1)
+  ret <16 x i8> %res
+}
+
+define <16 x i8> @test_mask_subs_epi8_rrk_128(<16 x i8> %a, <16 x i8> %b, <16 x i8> %passThru, i16 %mask) {
+  ;CHECK-LABEL: test_mask_subs_epi8_rrk_128
+  ;CHECK: vpsubsb %xmm1, %xmm0, %xmm2 {%k1} 
+  %res = call <16 x i8> @llvm.x86.avx512.mask.psubs.b.128(<16 x i8> %a, <16 x i8> %b, <16 x i8> %passThru, i16 %mask)
+  ret <16 x i8> %res
+}
+
+define <16 x i8> @test_mask_subs_epi8_rrkz_128(<16 x i8> %a, <16 x i8> %b, i16 %mask) {
+  ;CHECK-LABEL: test_mask_subs_epi8_rrkz_128
+  ;CHECK: vpsubsb %xmm1, %xmm0, %xmm0 {%k1} {z} 
+  %res = call <16 x i8> @llvm.x86.avx512.mask.psubs.b.128(<16 x i8> %a, <16 x i8> %b, <16 x i8> zeroinitializer, i16 %mask)
+  ret <16 x i8> %res
+}
+
+define <16 x i8> @test_mask_subs_epi8_rm_128(<16 x i8> %a, <16 x i8>* %ptr_b) {
+  ;CHECK-LABEL: test_mask_subs_epi8_rm_128
+  ;CHECK: vpsubsb (%rdi), %xmm0, %xmm0
+  %b = load <16 x i8>, <16 x i8>* %ptr_b
+  %res = call <16 x i8> @llvm.x86.avx512.mask.psubs.b.128(<16 x i8> %a, <16 x i8> %b, <16 x i8> zeroinitializer, i16 -1)
+  ret <16 x i8> %res
+}
+
+define <16 x i8> @test_mask_subs_epi8_rmk_128(<16 x i8> %a, <16 x i8>* %ptr_b, <16 x i8> %passThru, i16 %mask) {
+  ;CHECK-LABEL: test_mask_subs_epi8_rmk_128
+  ;CHECK: vpsubsb (%rdi), %xmm0, %xmm1 {%k1} 
+  %b = load <16 x i8>, <16 x i8>* %ptr_b
+  %res = call <16 x i8> @llvm.x86.avx512.mask.psubs.b.128(<16 x i8> %a, <16 x i8> %b, <16 x i8> %passThru, i16 %mask)
+  ret <16 x i8> %res
+}
+
+define <16 x i8> @test_mask_subs_epi8_rmkz_128(<16 x i8> %a, <16 x i8>* %ptr_b, i16 %mask) {
+  ;CHECK-LABEL: test_mask_subs_epi8_rmkz_128
+  ;CHECK: vpsubsb (%rdi), %xmm0, %xmm0 {%k1} {z} 
+  %b = load <16 x i8>, <16 x i8>* %ptr_b
+  %res = call <16 x i8> @llvm.x86.avx512.mask.psubs.b.128(<16 x i8> %a, <16 x i8> %b, <16 x i8> zeroinitializer, i16 %mask)
+  ret <16 x i8> %res
+}
+
+declare <16 x i8> @llvm.x86.avx512.mask.psubs.b.128(<16 x i8>, <16 x i8>, <16 x i8>, i16)
+
+define <32 x i8> @test_mask_subs_epi8_rr_256(<32 x i8> %a, <32 x i8> %b) {
+  ;CHECK-LABEL: test_mask_subs_epi8_rr_256
+  ;CHECK: vpsubsb %ymm1, %ymm0, %ymm0     
+  %res = call <32 x i8> @llvm.x86.avx512.mask.psubs.b.256(<32 x i8> %a, <32 x i8> %b, <32 x i8> zeroinitializer, i32 -1)
+  ret <32 x i8> %res
+}
+
+define <32 x i8> @test_mask_subs_epi8_rrk_256(<32 x i8> %a, <32 x i8> %b, <32 x i8> %passThru, i32 %mask) {
+  ;CHECK-LABEL: test_mask_subs_epi8_rrk_256
+  ;CHECK: vpsubsb %ymm1, %ymm0, %ymm2 {%k1} 
+  %res = call <32 x i8> @llvm.x86.avx512.mask.psubs.b.256(<32 x i8> %a, <32 x i8> %b, <32 x i8> %passThru, i32 %mask)
+  ret <32 x i8> %res
+}
+
+define <32 x i8> @test_mask_subs_epi8_rrkz_256(<32 x i8> %a, <32 x i8> %b, i32 %mask) {
+  ;CHECK-LABEL: test_mask_subs_epi8_rrkz_256
+  ;CHECK: vpsubsb %ymm1, %ymm0, %ymm0 {%k1} {z} 
+  %res = call <32 x i8> @llvm.x86.avx512.mask.psubs.b.256(<32 x i8> %a, <32 x i8> %b, <32 x i8> zeroinitializer, i32 %mask)
+  ret <32 x i8> %res
+}
+
+define <32 x i8> @test_mask_subs_epi8_rm_256(<32 x i8> %a, <32 x i8>* %ptr_b) {
+  ;CHECK-LABEL: test_mask_subs_epi8_rm_256
+  ;CHECK: vpsubsb (%rdi), %ymm0, %ymm0    
+  %b = load <32 x i8>, <32 x i8>* %ptr_b
+  %res = call <32 x i8> @llvm.x86.avx512.mask.psubs.b.256(<32 x i8> %a, <32 x i8> %b, <32 x i8> zeroinitializer, i32 -1)
+  ret <32 x i8> %res
+}
+
+define <32 x i8> @test_mask_subs_epi8_rmk_256(<32 x i8> %a, <32 x i8>* %ptr_b, <32 x i8> %passThru, i32 %mask) {
+  ;CHECK-LABEL: test_mask_subs_epi8_rmk_256
+  ;CHECK: vpsubsb (%rdi), %ymm0, %ymm1 {%k1} 
+  %b = load <32 x i8>, <32 x i8>* %ptr_b
+  %res = call <32 x i8> @llvm.x86.avx512.mask.psubs.b.256(<32 x i8> %a, <32 x i8> %b, <32 x i8> %passThru, i32 %mask)
+  ret <32 x i8> %res
+}
+
+define <32 x i8> @test_mask_subs_epi8_rmkz_256(<32 x i8> %a, <32 x i8>* %ptr_b, i32 %mask) {
+  ;CHECK-LABEL: test_mask_subs_epi8_rmkz_256
+  ;CHECK: vpsubsb (%rdi), %ymm0, %ymm0 {%k1} {z} 
+  %b = load <32 x i8>, <32 x i8>* %ptr_b
+  %res = call <32 x i8> @llvm.x86.avx512.mask.psubs.b.256(<32 x i8> %a, <32 x i8> %b, <32 x i8> zeroinitializer, i32 %mask)
+  ret <32 x i8> %res
+}
+
+declare <32 x i8> @llvm.x86.avx512.mask.psubs.b.256(<32 x i8>, <32 x i8>, <32 x i8>, i32)
+
+define <16 x i8> @test_mask_adds_epu8_rr_128(<16 x i8> %a, <16 x i8> %b) {
+  ;CHECK-LABEL: test_mask_adds_epu8_rr_128
+  ;CHECK: vpaddusb %xmm1, %xmm0, %xmm0 
+  %res = call <16 x i8> @llvm.x86.avx512.mask.paddus.b.128(<16 x i8> %a, <16 x i8> %b, <16 x i8> zeroinitializer, i16 -1)
+  ret <16 x i8> %res
+}
+
+define <16 x i8> @test_mask_adds_epu8_rrk_128(<16 x i8> %a, <16 x i8> %b, <16 x i8> %passThru, i16 %mask) {
+  ;CHECK-LABEL: test_mask_adds_epu8_rrk_128
+  ;CHECK: vpaddusb %xmm1, %xmm0, %xmm2 {%k1} 
+  %res = call <16 x i8> @llvm.x86.avx512.mask.paddus.b.128(<16 x i8> %a, <16 x i8> %b, <16 x i8> %passThru, i16 %mask)
+  ret <16 x i8> %res
+}
+
+define <16 x i8> @test_mask_adds_epu8_rrkz_128(<16 x i8> %a, <16 x i8> %b, i16 %mask) {
+  ;CHECK-LABEL: test_mask_adds_epu8_rrkz_128
+  ;CHECK: vpaddusb %xmm1, %xmm0, %xmm0 {%k1} {z} 
+  %res = call <16 x i8> @llvm.x86.avx512.mask.paddus.b.128(<16 x i8> %a, <16 x i8> %b, <16 x i8> zeroinitializer, i16 %mask)
+  ret <16 x i8> %res
+}
+
+define <16 x i8> @test_mask_adds_epu8_rm_128(<16 x i8> %a, <16 x i8>* %ptr_b) {
+  ;CHECK-LABEL: test_mask_adds_epu8_rm_128
+  ;CHECK: vpaddusb (%rdi), %xmm0, %xmm0 
+  %b = load <16 x i8>, <16 x i8>* %ptr_b
+  %res = call <16 x i8> @llvm.x86.avx512.mask.paddus.b.128(<16 x i8> %a, <16 x i8> %b, <16 x i8> zeroinitializer, i16 -1)
+  ret <16 x i8> %res
+}
+
+define <16 x i8> @test_mask_adds_epu8_rmk_128(<16 x i8> %a, <16 x i8>* %ptr_b, <16 x i8> %passThru, i16 %mask) {
+  ;CHECK-LABEL: test_mask_adds_epu8_rmk_128
+  ;CHECK: vpaddusb (%rdi), %xmm0, %xmm1 {%k1} 
+  %b = load <16 x i8>, <16 x i8>* %ptr_b
+  %res = call <16 x i8> @llvm.x86.avx512.mask.paddus.b.128(<16 x i8> %a, <16 x i8> %b, <16 x i8> %passThru, i16 %mask)
+  ret <16 x i8> %res
+}
+
+define <16 x i8> @test_mask_adds_epu8_rmkz_128(<16 x i8> %a, <16 x i8>* %ptr_b, i16 %mask) {
+  ;CHECK-LABEL: test_mask_adds_epu8_rmkz_128
+  ;CHECK: vpaddusb (%rdi), %xmm0, %xmm0 {%k1} {z} 
+  %b = load <16 x i8>, <16 x i8>* %ptr_b
+  %res = call <16 x i8> @llvm.x86.avx512.mask.paddus.b.128(<16 x i8> %a, <16 x i8> %b, <16 x i8> zeroinitializer, i16 %mask)
+  ret <16 x i8> %res
+}
+
+declare <16 x i8> @llvm.x86.avx512.mask.paddus.b.128(<16 x i8>, <16 x i8>, <16 x i8>, i16)
+
+define <32 x i8> @test_mask_adds_epu8_rr_256(<32 x i8> %a, <32 x i8> %b) {
+  ;CHECK-LABEL: test_mask_adds_epu8_rr_256
+  ;CHECK: vpaddusb %ymm1, %ymm0, %ymm0 
+  %res = call <32 x i8> @llvm.x86.avx512.mask.paddus.b.256(<32 x i8> %a, <32 x i8> %b, <32 x i8> zeroinitializer, i32 -1)
+  ret <32 x i8> %res
+}
+
+define <32 x i8> @test_mask_adds_epu8_rrk_256(<32 x i8> %a, <32 x i8> %b, <32 x i8> %passThru, i32 %mask) {
+  ;CHECK-LABEL: test_mask_adds_epu8_rrk_256
+  ;CHECK: vpaddusb %ymm1, %ymm0, %ymm2 {%k1} 
+  %res = call <32 x i8> @llvm.x86.avx512.mask.paddus.b.256(<32 x i8> %a, <32 x i8> %b, <32 x i8> %passThru, i32 %mask)
+  ret <32 x i8> %res
+}
+
+define <32 x i8> @test_mask_adds_epu8_rrkz_256(<32 x i8> %a, <32 x i8> %b, i32 %mask) {
+  ;CHECK-LABEL: test_mask_adds_epu8_rrkz_256
+  ;CHECK: vpaddusb %ymm1, %ymm0, %ymm0 {%k1} {z} 
+  %res = call <32 x i8> @llvm.x86.avx512.mask.paddus.b.256(<32 x i8> %a, <32 x i8> %b, <32 x i8> zeroinitializer, i32 %mask)
+  ret <32 x i8> %res
+}
+
+define <32 x i8> @test_mask_adds_epu8_rm_256(<32 x i8> %a, <32 x i8>* %ptr_b) {
+  ;CHECK-LABEL: test_mask_adds_epu8_rm_256
+  ;CHECK: vpaddusb (%rdi), %ymm0, %ymm0    
+  %b = load <32 x i8>, <32 x i8>* %ptr_b
+  %res = call <32 x i8> @llvm.x86.avx512.mask.paddus.b.256(<32 x i8> %a, <32 x i8> %b, <32 x i8> zeroinitializer, i32 -1)
+  ret <32 x i8> %res
+}
+
+define <32 x i8> @test_mask_adds_epu8_rmk_256(<32 x i8> %a, <32 x i8>* %ptr_b, <32 x i8> %passThru, i32 %mask) {
+  ;CHECK-LABEL: test_mask_adds_epu8_rmk_256
+  ;CHECK: vpaddusb (%rdi), %ymm0, %ymm1 {%k1} 
+  %b = load <32 x i8>, <32 x i8>* %ptr_b
+  %res = call <32 x i8> @llvm.x86.avx512.mask.paddus.b.256(<32 x i8> %a, <32 x i8> %b, <32 x i8> %passThru, i32 %mask)
+  ret <32 x i8> %res
+}
+
+define <32 x i8> @test_mask_adds_epu8_rmkz_256(<32 x i8> %a, <32 x i8>* %ptr_b, i32 %mask) {
+  ;CHECK-LABEL: test_mask_adds_epu8_rmkz_256
+  ;CHECK: vpaddusb (%rdi), %ymm0, %ymm0 {%k1} {z} 
+  %b = load <32 x i8>, <32 x i8>* %ptr_b
+  %res = call <32 x i8> @llvm.x86.avx512.mask.paddus.b.256(<32 x i8> %a, <32 x i8> %b, <32 x i8> zeroinitializer, i32 %mask)
+  ret <32 x i8> %res
+}
+
+declare <32 x i8> @llvm.x86.avx512.mask.paddus.b.256(<32 x i8>, <32 x i8>, <32 x i8>, i32)
+
+define <16 x i8> @test_mask_subs_epu8_rr_128(<16 x i8> %a, <16 x i8> %b) {
+  ;CHECK-LABEL: test_mask_subs_epu8_rr_128
+  ;CHECK: vpsubusb %xmm1, %xmm0, %xmm0     
+  %res = call <16 x i8> @llvm.x86.avx512.mask.psubus.b.128(<16 x i8> %a, <16 x i8> %b, <16 x i8> zeroinitializer, i16 -1)
+  ret <16 x i8> %res
+}
+
+define <16 x i8> @test_mask_subs_epu8_rrk_128(<16 x i8> %a, <16 x i8> %b, <16 x i8> %passThru, i16 %mask) {
+  ;CHECK-LABEL: test_mask_subs_epu8_rrk_128
+  ;CHECK: vpsubusb %xmm1, %xmm0, %xmm2 {%k1} 
+  %res = call <16 x i8> @llvm.x86.avx512.mask.psubus.b.128(<16 x i8> %a, <16 x i8> %b, <16 x i8> %passThru, i16 %mask)
+  ret <16 x i8> %res
+}
+
+define <16 x i8> @test_mask_subs_epu8_rrkz_128(<16 x i8> %a, <16 x i8> %b, i16 %mask) {
+  ;CHECK-LABEL: test_mask_subs_epu8_rrkz_128
+  ;CHECK: vpsubusb %xmm1, %xmm0, %xmm0 {%k1} {z} 
+  %res = call <16 x i8> @llvm.x86.avx512.mask.psubus.b.128(<16 x i8> %a, <16 x i8> %b, <16 x i8> zeroinitializer, i16 %mask)
+  ret <16 x i8> %res
+}
+
+define <16 x i8> @test_mask_subs_epu8_rm_128(<16 x i8> %a, <16 x i8>* %ptr_b) {
+  ;CHECK-LABEL: test_mask_subs_epu8_rm_128
+  ;CHECK: vpsubusb (%rdi), %xmm0, %xmm0
+  %b = load <16 x i8>, <16 x i8>* %ptr_b
+  %res = call <16 x i8> @llvm.x86.avx512.mask.psubus.b.128(<16 x i8> %a, <16 x i8> %b, <16 x i8> zeroinitializer, i16 -1)
+  ret <16 x i8> %res
+}
+
+define <16 x i8> @test_mask_subs_epu8_rmk_128(<16 x i8> %a, <16 x i8>* %ptr_b, <16 x i8> %passThru, i16 %mask) {
+  ;CHECK-LABEL: test_mask_subs_epu8_rmk_128
+  ;CHECK: vpsubusb (%rdi), %xmm0, %xmm1 {%k1} 
+  %b = load <16 x i8>, <16 x i8>* %ptr_b
+  %res = call <16 x i8> @llvm.x86.avx512.mask.psubus.b.128(<16 x i8> %a, <16 x i8> %b, <16 x i8> %passThru, i16 %mask)
+  ret <16 x i8> %res
+}
+
+define <16 x i8> @test_mask_subs_epu8_rmkz_128(<16 x i8> %a, <16 x i8>* %ptr_b, i16 %mask) {
+  ;CHECK-LABEL: test_mask_subs_epu8_rmkz_128
+  ;CHECK: vpsubusb (%rdi), %xmm0, %xmm0 {%k1} {z} 
+  %b = load <16 x i8>, <16 x i8>* %ptr_b
+  %res = call <16 x i8> @llvm.x86.avx512.mask.psubus.b.128(<16 x i8> %a, <16 x i8> %b, <16 x i8> zeroinitializer, i16 %mask)
+  ret <16 x i8> %res
+}
+
+declare <16 x i8> @llvm.x86.avx512.mask.psubus.b.128(<16 x i8>, <16 x i8>, <16 x i8>, i16)
+
+define <32 x i8> @test_mask_subs_epu8_rr_256(<32 x i8> %a, <32 x i8> %b) {
+  ;CHECK-LABEL: test_mask_subs_epu8_rr_256
+  ;CHECK: vpsubusb %ymm1, %ymm0, %ymm0     
+  %res = call <32 x i8> @llvm.x86.avx512.mask.psubus.b.256(<32 x i8> %a, <32 x i8> %b, <32 x i8> zeroinitializer, i32 -1)
+  ret <32 x i8> %res
+}
+
+define <32 x i8> @test_mask_subs_epu8_rrk_256(<32 x i8> %a, <32 x i8> %b, <32 x i8> %passThru, i32 %mask) {
+  ;CHECK-LABEL: test_mask_subs_epu8_rrk_256
+  ;CHECK: vpsubusb %ymm1, %ymm0, %ymm2 {%k1} 
+  %res = call <32 x i8> @llvm.x86.avx512.mask.psubus.b.256(<32 x i8> %a, <32 x i8> %b, <32 x i8> %passThru, i32 %mask)
+  ret <32 x i8> %res
+}
+
+define <32 x i8> @test_mask_subs_epu8_rrkz_256(<32 x i8> %a, <32 x i8> %b, i32 %mask) {
+  ;CHECK-LABEL: test_mask_subs_epu8_rrkz_256
+  ;CHECK: vpsubusb %ymm1, %ymm0, %ymm0 {%k1} {z} 
+  %res = call <32 x i8> @llvm.x86.avx512.mask.psubus.b.256(<32 x i8> %a, <32 x i8> %b, <32 x i8> zeroinitializer, i32 %mask)
+  ret <32 x i8> %res
+}
+
+define <32 x i8> @test_mask_subs_epu8_rm_256(<32 x i8> %a, <32 x i8>* %ptr_b) {
+  ;CHECK-LABEL: test_mask_subs_epu8_rm_256
+  ;CHECK: vpsubusb (%rdi), %ymm0, %ymm0    
+  %b = load <32 x i8>, <32 x i8>* %ptr_b
+  %res = call <32 x i8> @llvm.x86.avx512.mask.psubus.b.256(<32 x i8> %a, <32 x i8> %b, <32 x i8> zeroinitializer, i32 -1)
+  ret <32 x i8> %res
+}
+
+define <32 x i8> @test_mask_subs_epu8_rmk_256(<32 x i8> %a, <32 x i8>* %ptr_b, <32 x i8> %passThru, i32 %mask) {
+  ;CHECK-LABEL: test_mask_subs_epu8_rmk_256
+  ;CHECK: vpsubusb (%rdi), %ymm0, %ymm1 {%k1} 
+  %b = load <32 x i8>, <32 x i8>* %ptr_b
+  %res = call <32 x i8> @llvm.x86.avx512.mask.psubus.b.256(<32 x i8> %a, <32 x i8> %b, <32 x i8> %passThru, i32 %mask)
+  ret <32 x i8> %res
+}
+
+define <32 x i8> @test_mask_subs_epu8_rmkz_256(<32 x i8> %a, <32 x i8>* %ptr_b, i32 %mask) {
+  ;CHECK-LABEL: test_mask_subs_epu8_rmkz_256
+  ;CHECK: vpsubusb (%rdi), %ymm0, %ymm0 {%k1} {z} 
+  %b = load <32 x i8>, <32 x i8>* %ptr_b
+  %res = call <32 x i8> @llvm.x86.avx512.mask.psubus.b.256(<32 x i8> %a, <32 x i8> %b, <32 x i8> zeroinitializer, i32 %mask)
+  ret <32 x i8> %res
+}
+
+declare <32 x i8> @llvm.x86.avx512.mask.psubus.b.256(<32 x i8>, <32 x i8>, <32 x i8>, i32)
+
+declare <16 x i8> @llvm.x86.avx512.mask.pmaxs.b.128(<16 x i8>, <16 x i8>, <16 x i8>, i16)
+
+; CHECK-LABEL: @test_int_x86_avx512_mask_pmaxs_b_128
+; CHECK-NOT: call 
+; CHECK: vpmaxsb %xmm
+; CHECK: {%k1} 
+define <16 x i8>@test_int_x86_avx512_mask_pmaxs_b_128(<16 x i8> %x0, <16 x i8> %x1, <16 x i8> %x2, i16 %mask) {
+  %res = call <16 x i8> @llvm.x86.avx512.mask.pmaxs.b.128(<16 x i8> %x0, <16 x i8> %x1, <16 x i8> %x2 ,i16 %mask)
+  %res1 = call <16 x i8> @llvm.x86.avx512.mask.pmaxs.b.128(<16 x i8> %x0, <16 x i8> %x1, <16 x i8> zeroinitializer, i16 %mask)
+  %res2 = add <16 x i8> %res, %res1
+  ret <16 x i8> %res2
+}
+
+declare <32 x i8> @llvm.x86.avx512.mask.pmaxs.b.256(<32 x i8>, <32 x i8>, <32 x i8>, i32)
+
+; CHECK-LABEL: @test_int_x86_avx512_mask_pmaxs_b_256
+; CHECK-NOT: call 
+; CHECK: vpmaxsb %ymm
+; CHECK: {%k1} 
+define <32 x i8>@test_int_x86_avx512_mask_pmaxs_b_256(<32 x i8> %x0, <32 x i8> %x1, <32 x i8> %x2, i32 %x3) {
+  %res = call <32 x i8> @llvm.x86.avx512.mask.pmaxs.b.256(<32 x i8> %x0, <32 x i8> %x1, <32 x i8> %x2, i32 %x3)
+  %res1 = call <32 x i8> @llvm.x86.avx512.mask.pmaxs.b.256(<32 x i8> %x0, <32 x i8> %x1, <32 x i8> %x2, i32 -1)
+  %res2 = add <32 x i8> %res, %res1
+  ret <32 x i8> %res2
+}
+
+declare <8 x i16> @llvm.x86.avx512.mask.pmaxs.w.128(<8 x i16>, <8 x i16>, <8 x i16>, i8)
+
+; CHECK-LABEL: @test_int_x86_avx512_mask_pmaxs_w_128
+; CHECK-NOT: call 
+; CHECK: vpmaxsw %xmm
+; CHECK: {%k1} 
+define <8 x i16>@test_int_x86_avx512_mask_pmaxs_w_128(<8 x i16> %x0, <8 x i16> %x1, <8 x i16> %x2, i8 %x3) {
+  %res = call <8 x i16> @llvm.x86.avx512.mask.pmaxs.w.128(<8 x i16> %x0, <8 x i16> %x1, <8 x i16> %x2, i8 %x3)
+  %res1 = call <8 x i16> @llvm.x86.avx512.mask.pmaxs.w.128(<8 x i16> %x0, <8 x i16> %x1, <8 x i16> %x2, i8 -1)
+  %res2 = add <8 x i16> %res, %res1
+  ret <8 x i16> %res2
+}
+
+declare <16 x i16> @llvm.x86.avx512.mask.pmaxs.w.256(<16 x i16>, <16 x i16>, <16 x i16>, i16)
+
+; CHECK-LABEL: @test_int_x86_avx512_mask_pmaxs_w_256
+; CHECK-NOT: call 
+; CHECK: vpmaxsw %ymm
+; CHECK: {%k1} 
+define <16 x i16>@test_int_x86_avx512_mask_pmaxs_w_256(<16 x i16> %x0, <16 x i16> %x1, <16 x i16> %x2, i16 %mask) {
+  %res = call <16 x i16> @llvm.x86.avx512.mask.pmaxs.w.256(<16 x i16> %x0, <16 x i16> %x1, <16 x i16> %x2, i16 %mask)
+  %res1 = call <16 x i16> @llvm.x86.avx512.mask.pmaxs.w.256(<16 x i16> %x0, <16 x i16> %x1, <16 x i16> zeroinitializer, i16 %mask)
+  %res2 = add <16 x i16> %res, %res1
+  ret <16 x i16> %res2
+}
+
+declare <16 x i8> @llvm.x86.avx512.mask.pmaxu.b.128(<16 x i8>, <16 x i8>, <16 x i8>, i16)
+
+; CHECK-LABEL: @test_int_x86_avx512_mask_pmaxu_b_128
+; CHECK-NOT: call 
+; CHECK: vpmaxub %xmm
+; CHECK: {%k1} 
+define <16 x i8>@test_int_x86_avx512_mask_pmaxu_b_128(<16 x i8> %x0, <16 x i8> %x1, <16 x i8> %x2,i16 %mask) {
+  %res = call <16 x i8> @llvm.x86.avx512.mask.pmaxu.b.128(<16 x i8> %x0, <16 x i8> %x1, <16 x i8> %x2, i16 %mask)
+  %res1 = call <16 x i8> @llvm.x86.avx512.mask.pmaxu.b.128(<16 x i8> %x0, <16 x i8> %x1, <16 x i8> zeroinitializer, i16 %mask)
+  %res2 = add <16 x i8> %res, %res1
+  ret <16 x i8> %res2
+}
+
+declare <32 x i8> @llvm.x86.avx512.mask.pmaxu.b.256(<32 x i8>, <32 x i8>, <32 x i8>, i32)
+
+; CHECK-LABEL: @test_int_x86_avx512_mask_pmaxu_b_256
+; CHECK-NOT: call 
+; CHECK: vpmaxub %ymm
+; CHECK: {%k1} 
+define <32 x i8>@test_int_x86_avx512_mask_pmaxu_b_256(<32 x i8> %x0, <32 x i8> %x1, <32 x i8> %x2, i32 %x3) {
+  %res = call <32 x i8> @llvm.x86.avx512.mask.pmaxu.b.256(<32 x i8> %x0, <32 x i8> %x1, <32 x i8> %x2, i32 %x3)
+  %res1 = call <32 x i8> @llvm.x86.avx512.mask.pmaxu.b.256(<32 x i8> %x0, <32 x i8> %x1, <32 x i8> %x2, i32 -1)
+  %res2 = add <32 x i8> %res, %res1
+  ret <32 x i8> %res2
+}
+
+declare <8 x i16> @llvm.x86.avx512.mask.pmaxu.w.128(<8 x i16>, <8 x i16>, <8 x i16>, i8)
+
+; CHECK-LABEL: @test_int_x86_avx512_mask_pmaxu_w_128
+; CHECK-NOT: call 
+; CHECK: vpmaxuw %xmm
+; CHECK: {%k1} 
+define <8 x i16>@test_int_x86_avx512_mask_pmaxu_w_128(<8 x i16> %x0, <8 x i16> %x1, <8 x i16> %x2, i8 %x3) {
+  %res = call <8 x i16> @llvm.x86.avx512.mask.pmaxu.w.128(<8 x i16> %x0, <8 x i16> %x1, <8 x i16> %x2, i8 %x3)
+  %res1 = call <8 x i16> @llvm.x86.avx512.mask.pmaxu.w.128(<8 x i16> %x0, <8 x i16> %x1, <8 x i16> %x2, i8 -1)
+  %res2 = add <8 x i16> %res, %res1
+  ret <8 x i16> %res2
+}
+
+declare <16 x i16> @llvm.x86.avx512.mask.pmaxu.w.256(<16 x i16>, <16 x i16>, <16 x i16>, i16)
+
+; CHECK-LABEL: @test_int_x86_avx512_mask_pmaxu_w_256
+; CHECK-NOT: call 
+; CHECK: vpmaxuw %ymm
+; CHECK: {%k1} 
+define <16 x i16>@test_int_x86_avx512_mask_pmaxu_w_256(<16 x i16> %x0, <16 x i16> %x1, <16 x i16> %x2, i16 %mask) {
+  %res = call <16 x i16> @llvm.x86.avx512.mask.pmaxu.w.256(<16 x i16> %x0, <16 x i16> %x1, <16 x i16> %x2, i16 %mask)
+  %res1 = call <16 x i16> @llvm.x86.avx512.mask.pmaxu.w.256(<16 x i16> %x0, <16 x i16> %x1, <16 x i16> zeroinitializer, i16 %mask)
+  %res2 = add <16 x i16> %res, %res1
+  ret <16 x i16> %res2
+}
+
+declare <16 x i8> @llvm.x86.avx512.mask.pmins.b.128(<16 x i8>, <16 x i8>, <16 x i8>, i16)
+
+; CHECK-LABEL: @test_int_x86_avx512_mask_pmins_b_128
+; CHECK-NOT: call 
+; CHECK: vpminsb %xmm
+; CHECK: {%k1} 
+define <16 x i8>@test_int_x86_avx512_mask_pmins_b_128(<16 x i8> %x0, <16 x i8> %x1, <16 x i8> %x2, i16 %mask) {
+  %res = call <16 x i8> @llvm.x86.avx512.mask.pmins.b.128(<16 x i8> %x0, <16 x i8> %x1, <16 x i8> %x2, i16 %mask)
+  %res1 = call <16 x i8> @llvm.x86.avx512.mask.pmins.b.128(<16 x i8> %x0, <16 x i8> %x1, <16 x i8> zeroinitializer, i16 %mask)
+  %res2 = add <16 x i8> %res, %res1
+  ret <16 x i8> %res2
+}
+
+declare <32 x i8> @llvm.x86.avx512.mask.pmins.b.256(<32 x i8>, <32 x i8>, <32 x i8>, i32)
+
+; CHECK-LABEL: @test_int_x86_avx512_mask_pmins_b_256
+; CHECK-NOT: call 
+; CHECK: vpminsb %ymm
+; CHECK: {%k1} 
+define <32 x i8>@test_int_x86_avx512_mask_pmins_b_256(<32 x i8> %x0, <32 x i8> %x1, <32 x i8> %x2, i32 %x3) {
+  %res = call <32 x i8> @llvm.x86.avx512.mask.pmins.b.256(<32 x i8> %x0, <32 x i8> %x1, <32 x i8> %x2, i32 %x3)
+  %res1 = call <32 x i8> @llvm.x86.avx512.mask.pmins.b.256(<32 x i8> %x0, <32 x i8> %x1, <32 x i8> %x2, i32 -1)
+  %res2 = add <32 x i8> %res, %res1
+  ret <32 x i8> %res2
+}
+
+declare <8 x i16> @llvm.x86.avx512.mask.pmins.w.128(<8 x i16>, <8 x i16>, <8 x i16>, i8)
+
+; CHECK-LABEL: @test_int_x86_avx512_mask_pmins_w_128
+; CHECK-NOT: call 
+; CHECK: vpminsw %xmm
+; CHECK: {%k1} 
+define <8 x i16>@test_int_x86_avx512_mask_pmins_w_128(<8 x i16> %x0, <8 x i16> %x1, <8 x i16> %x2, i8 %x3) {
+  %res = call <8 x i16> @llvm.x86.avx512.mask.pmins.w.128(<8 x i16> %x0, <8 x i16> %x1, <8 x i16> %x2, i8 %x3)
+  %res1 = call <8 x i16> @llvm.x86.avx512.mask.pmins.w.128(<8 x i16> %x0, <8 x i16> %x1, <8 x i16> %x2, i8 -1)
+  %res2 = add <8 x i16> %res, %res1
+  ret <8 x i16> %res2
+}
+
+declare <16 x i16> @llvm.x86.avx512.mask.pmins.w.256(<16 x i16>, <16 x i16>, <16 x i16>, i16)
+
+; CHECK-LABEL: @test_int_x86_avx512_mask_pmins_w_256
+; CHECK-NOT: call 
+; CHECK: vpminsw %ymm
+; CHECK: {%k1} 
+define <16 x i16>@test_int_x86_avx512_mask_pmins_w_256(<16 x i16> %x0, <16 x i16> %x1, <16 x i16> %x2, i16 %mask) {
+  %res = call <16 x i16> @llvm.x86.avx512.mask.pmins.w.256(<16 x i16> %x0, <16 x i16> %x1, <16 x i16> %x2, i16 %mask)
+  %res1 = call <16 x i16> @llvm.x86.avx512.mask.pmins.w.256(<16 x i16> %x0, <16 x i16> %x1, <16 x i16> zeroinitializer, i16 %mask)
+  %res2 = add <16 x i16> %res, %res1
+  ret <16 x i16> %res2
+}
+
+declare <16 x i8> @llvm.x86.avx512.mask.pminu.b.128(<16 x i8>, <16 x i8>, <16 x i8>, i16)
+
+; CHECK-LABEL: @test_int_x86_avx512_mask_pminu_b_128
+; CHECK-NOT: call 
+; CHECK: vpminub %xmm
+; CHECK: {%k1} 
+define <16 x i8>@test_int_x86_avx512_mask_pminu_b_128(<16 x i8> %x0, <16 x i8> %x1, <16 x i8> %x2, i16 %mask) {
+  %res = call <16 x i8> @llvm.x86.avx512.mask.pminu.b.128(<16 x i8> %x0, <16 x i8> %x1, <16 x i8> %x2, i16 %mask)
+  %res1 = call <16 x i8> @llvm.x86.avx512.mask.pminu.b.128(<16 x i8> %x0, <16 x i8> %x1, <16 x i8> zeroinitializer, i16 %mask)
+  %res2 = add <16 x i8> %res, %res1
+  ret <16 x i8> %res2
+}
+
+declare <32 x i8> @llvm.x86.avx512.mask.pminu.b.256(<32 x i8>, <32 x i8>, <32 x i8>, i32)
+
+; CHECK-LABEL: @test_int_x86_avx512_mask_pminu_b_256
+; CHECK-NOT: call 
+; CHECK: vpminub %ymm
+; CHECK: {%k1} 
+define <32 x i8>@test_int_x86_avx512_mask_pminu_b_256(<32 x i8> %x0, <32 x i8> %x1, <32 x i8> %x2, i32 %x3) {
+  %res = call <32 x i8> @llvm.x86.avx512.mask.pminu.b.256(<32 x i8> %x0, <32 x i8> %x1, <32 x i8> %x2, i32 %x3)
+  %res1 = call <32 x i8> @llvm.x86.avx512.mask.pminu.b.256(<32 x i8> %x0, <32 x i8> %x1, <32 x i8> %x2, i32 -1)
+  %res2 = add <32 x i8> %res, %res1
+  ret <32 x i8> %res2
+}
+
+declare <8 x i16> @llvm.x86.avx512.mask.pminu.w.128(<8 x i16>, <8 x i16>, <8 x i16>, i8)
+
+; CHECK-LABEL: @test_int_x86_avx512_mask_pminu_w_128
+; CHECK-NOT: call 
+; CHECK: vpminuw %xmm
+; CHECK: {%k1} 
+define <8 x i16>@test_int_x86_avx512_mask_pminu_w_128(<8 x i16> %x0, <8 x i16> %x1, <8 x i16> %x2, i8 %x3) {
+  %res = call <8 x i16> @llvm.x86.avx512.mask.pminu.w.128(<8 x i16> %x0, <8 x i16> %x1, <8 x i16> %x2, i8 %x3)
+  %res1 = call <8 x i16> @llvm.x86.avx512.mask.pminu.w.128(<8 x i16> %x0, <8 x i16> %x1, <8 x i16> %x2, i8 -1)
+  %res2 = add <8 x i16> %res, %res1
+  ret <8 x i16> %res2
+}
+
+declare <16 x i16> @llvm.x86.avx512.mask.pminu.w.256(<16 x i16>, <16 x i16>, <16 x i16>, i16)
+
+; CHECK-LABEL: @test_int_x86_avx512_mask_pminu_w_256
+; CHECK-NOT: call 
+; CHECK: vpminuw %ymm
+; CHECK: {%k1} 
+define <16 x i16>@test_int_x86_avx512_mask_pminu_w_256(<16 x i16> %x0, <16 x i16> %x1, <16 x i16> %x2, i16 %mask) {
+  %res = call <16 x i16> @llvm.x86.avx512.mask.pminu.w.256(<16 x i16> %x0, <16 x i16> %x1, <16 x i16> %x2, i16 %mask)
+  %res1 = call <16 x i16> @llvm.x86.avx512.mask.pminu.w.256(<16 x i16> %x0, <16 x i16> %x1, <16 x i16> zeroinitializer, i16 %mask)
+  %res2 = add <16 x i16> %res, %res1
+  ret <16 x i16> %res2
+}
+
+declare <8 x i16> @llvm.x86.avx512.mask.vpermt2var.hi.128(<8 x i16>, <8 x i16>, <8 x i16>, i8)
+
+; CHECK-LABEL: @test_int_x86_avx512_mask_vpermt2var_hi_128
+; CHECK-NOT: call 
+; CHECK: kmov 
+; CHECK: vpermt2w %xmm{{.*}}{%k1} 
+; CHECK-NOT: {z}
+define <8 x i16>@test_int_x86_avx512_mask_vpermt2var_hi_128(<8 x i16> %x0, <8 x i16> %x1, <8 x i16> %x2, i8 %x3) {
+  %res = call <8 x i16> @llvm.x86.avx512.mask.vpermt2var.hi.128(<8 x i16> %x0, <8 x i16> %x1, <8 x i16> %x2, i8 %x3)
+  %res1 = call <8 x i16> @llvm.x86.avx512.mask.vpermt2var.hi.128(<8 x i16> %x0, <8 x i16> %x1, <8 x i16> %x2, i8 -1)
+  %res2 = add <8 x i16> %res, %res1
+  ret <8 x i16> %res2
+}
+
+declare <8 x i16> @llvm.x86.avx512.maskz.vpermt2var.hi.128(<8 x i16>, <8 x i16>, <8 x i16>, i8)
+
+; CHECK-LABEL: @test_int_x86_avx512_maskz_vpermt2var_hi_128
+; CHECK-NOT: call 
+; CHECK: kmov 
+; CHECK: vpermt2w %xmm{{.*}}{%k1} {z}
+define <8 x i16>@test_int_x86_avx512_maskz_vpermt2var_hi_128(<8 x i16> %x0, <8 x i16> %x1, <8 x i16> %x2, i8 %x3) {
+  %res = call <8 x i16> @llvm.x86.avx512.maskz.vpermt2var.hi.128(<8 x i16> %x0, <8 x i16> %x1, <8 x i16> %x2, i8 %x3)
+  %res1 = call <8 x i16> @llvm.x86.avx512.maskz.vpermt2var.hi.128(<8 x i16> %x0, <8 x i16> %x1, <8 x i16> %x2, i8 -1)
+  %res2 = add <8 x i16> %res, %res1
+  ret <8 x i16> %res2
+}
+
+declare <16 x i16> @llvm.x86.avx512.mask.vpermt2var.hi.256(<16 x i16>, <16 x i16>, <16 x i16>, i16)
+
+; CHECK-LABEL: @test_int_x86_avx512_mask_vpermt2var_hi_256
+; CHECK-NOT: call 
+; CHECK: kmov 
+; CHECK: vpermt2w %ymm{{.*}}{%k1} 
+define <16 x i16>@test_int_x86_avx512_mask_vpermt2var_hi_256(<16 x i16> %x0, <16 x i16> %x1, <16 x i16> %x2, i16 %x3) {
+  %res = call <16 x i16> @llvm.x86.avx512.mask.vpermt2var.hi.256(<16 x i16> %x0, <16 x i16> %x1, <16 x i16> %x2, i16 %x3)
+  %res1 = call <16 x i16> @llvm.x86.avx512.mask.vpermt2var.hi.256(<16 x i16> %x0, <16 x i16> %x1, <16 x i16> %x2, i16 -1)
+  %res2 = add <16 x i16> %res, %res1
+  ret <16 x i16> %res2
+}
+
+declare <16 x i16> @llvm.x86.avx512.maskz.vpermt2var.hi.256(<16 x i16>, <16 x i16>, <16 x i16>, i16)
+
+; CHECK-LABEL: @test_int_x86_avx512_maskz_vpermt2var_hi_256
+; CHECK-NOT: call 
+; CHECK: kmov 
+; CHECK: vpermt2w %ymm{{.*}}{%k1} {z}
+define <16 x i16>@test_int_x86_avx512_maskz_vpermt2var_hi_256(<16 x i16> %x0, <16 x i16> %x1, <16 x i16> %x2, i16 %x3) {
+  %res = call <16 x i16> @llvm.x86.avx512.maskz.vpermt2var.hi.256(<16 x i16> %x0, <16 x i16> %x1, <16 x i16> %x2, i16 %x3)
+  %res1 = call <16 x i16> @llvm.x86.avx512.maskz.vpermt2var.hi.256(<16 x i16> %x0, <16 x i16> %x1, <16 x i16> %x2, i16 -1)
+  %res2 = add <16 x i16> %res, %res1
+  ret <16 x i16> %res2
+}
+
+declare <8 x i16> @llvm.x86.avx512.mask.vpermi2var.hi.128(<8 x i16>, <8 x i16>, <8 x i16>, i8)
+
+; CHECK-LABEL: @test_int_x86_avx512_mask_vpermi2var_hi_128
+; CHECK-NOT: call 
+; CHECK: kmov 
+; CHECK: vpermi2w %xmm{{.*}}{%k1} 
+define <8 x i16>@test_int_x86_avx512_mask_vpermi2var_hi_128(<8 x i16> %x0, <8 x i16> %x1, <8 x i16> %x2, i8 %x3) {
+  %res = call <8 x i16> @llvm.x86.avx512.mask.vpermi2var.hi.128(<8 x i16> %x0, <8 x i16> %x1, <8 x i16> %x2, i8 %x3)
+  %res1 = call <8 x i16> @llvm.x86.avx512.mask.vpermi2var.hi.128(<8 x i16> %x0, <8 x i16> %x1, <8 x i16> %x2, i8 -1)
+  %res2 = add <8 x i16> %res, %res1
+  ret <8 x i16> %res2
+}
+
+declare <16 x i16> @llvm.x86.avx512.mask.vpermi2var.hi.256(<16 x i16>, <16 x i16>, <16 x i16>, i16)
+
+; CHECK-LABEL: @test_int_x86_avx512_mask_vpermi2var_hi_256
+; CHECK-NOT: call 
+; CHECK: kmov 
+; CHECK: vpermi2w %ymm{{.*}}{%k1} 
+define <16 x i16>@test_int_x86_avx512_mask_vpermi2var_hi_256(<16 x i16> %x0, <16 x i16> %x1, <16 x i16> %x2, i16 %x3) {
+  %res = call <16 x i16> @llvm.x86.avx512.mask.vpermi2var.hi.256(<16 x i16> %x0, <16 x i16> %x1, <16 x i16> %x2, i16 %x3)
+  %res1 = call <16 x i16> @llvm.x86.avx512.mask.vpermi2var.hi.256(<16 x i16> %x0, <16 x i16> %x1, <16 x i16> %x2, i16 -1)
+  %res2 = add <16 x i16> %res, %res1
+  ret <16 x i16> %res2
+}
+
+declare <16 x i8> @llvm.x86.avx512.mask.pavg.b.128(<16 x i8>, <16 x i8>, <16 x i8>, i16)
+
+; CHECK-LABEL: @test_int_x86_avx512_mask_pavg_b_128
+; CHECK-NOT: call 
+; CHECK: vpavgb %xmm
+; CHECK: {%k1} 
+define <16 x i8>@test_int_x86_avx512_mask_pavg_b_128(<16 x i8> %x0, <16 x i8> %x1, <16 x i8> %x2, i16 %x3) {
+  %res = call <16 x i8> @llvm.x86.avx512.mask.pavg.b.128(<16 x i8> %x0, <16 x i8> %x1, <16 x i8> %x2, i16 %x3)
+  %res1 = call <16 x i8> @llvm.x86.avx512.mask.pavg.b.128(<16 x i8> %x0, <16 x i8> %x1, <16 x i8> %x2, i16 -1)
+  %res2 = add <16 x i8> %res, %res1
+  ret <16 x i8> %res2
+}
+
+declare <32 x i8> @llvm.x86.avx512.mask.pavg.b.256(<32 x i8>, <32 x i8>, <32 x i8>, i32)
+
+; CHECK-LABEL: @test_int_x86_avx512_mask_pavg_b_256
+; CHECK-NOT: call 
+; CHECK: vpavgb %ymm
+; CHECK: {%k1} 
+define <32 x i8>@test_int_x86_avx512_mask_pavg_b_256(<32 x i8> %x0, <32 x i8> %x1, <32 x i8> %x2, i32 %x3) {
+  %res = call <32 x i8> @llvm.x86.avx512.mask.pavg.b.256(<32 x i8> %x0, <32 x i8> %x1, <32 x i8> %x2, i32 %x3)
+  %res1 = call <32 x i8> @llvm.x86.avx512.mask.pavg.b.256(<32 x i8> %x0, <32 x i8> %x1, <32 x i8> %x2, i32 -1)
+  %res2 = add <32 x i8> %res, %res1
+  ret <32 x i8> %res2
+}
+
+declare <8 x i16> @llvm.x86.avx512.mask.pavg.w.128(<8 x i16>, <8 x i16>, <8 x i16>, i8)
+
+; CHECK-LABEL: @test_int_x86_avx512_mask_pavg_w_128
+; CHECK-NOT: call 
+; CHECK: vpavgw %xmm
+; CHECK: {%k1} 
+define <8 x i16>@test_int_x86_avx512_mask_pavg_w_128(<8 x i16> %x0, <8 x i16> %x1, <8 x i16> %x2, i8 %x3) {
+  %res = call <8 x i16> @llvm.x86.avx512.mask.pavg.w.128(<8 x i16> %x0, <8 x i16> %x1, <8 x i16> %x2, i8 %x3)
+  %res1 = call <8 x i16> @llvm.x86.avx512.mask.pavg.w.128(<8 x i16> %x0, <8 x i16> %x1, <8 x i16> %x2, i8 -1)
+  %res2 = add <8 x i16> %res, %res1
+  ret <8 x i16> %res2
+}
+
+declare <16 x i16> @llvm.x86.avx512.mask.pavg.w.256(<16 x i16>, <16 x i16>, <16 x i16>, i16)
+
+; CHECK-LABEL: @test_int_x86_avx512_mask_pavg_w_256
+; CHECK-NOT: call 
+; CHECK: vpavgw %ymm
+; CHECK: {%k1} 
+define <16 x i16>@test_int_x86_avx512_mask_pavg_w_256(<16 x i16> %x0, <16 x i16> %x1, <16 x i16> %x2, i16 %x3) {
+  %res = call <16 x i16> @llvm.x86.avx512.mask.pavg.w.256(<16 x i16> %x0, <16 x i16> %x1, <16 x i16> %x2, i16 %x3)
+  %res1 = call <16 x i16> @llvm.x86.avx512.mask.pavg.w.256(<16 x i16> %x0, <16 x i16> %x1, <16 x i16> %x2, i16 -1)
+  %res2 = add <16 x i16> %res, %res1
+  ret <16 x i16> %res2
+}
+
+declare <16 x i8> @llvm.x86.avx512.mask.pshuf.b.128(<16 x i8>, <16 x i8>, <16 x i8>, i16)
+
+; CHECK-LABEL: @test_int_x86_avx512_mask_pshuf_b_128
+; CHECK-NOT: call 
+; CHECK: kmov 
+; CHECK: vpshufb %xmm{{.*}}{%k1} 
+define <16 x i8>@test_int_x86_avx512_mask_pshuf_b_128(<16 x i8> %x0, <16 x i8> %x1, <16 x i8> %x2, i16 %x3) {
+  %res = call <16 x i8> @llvm.x86.avx512.mask.pshuf.b.128(<16 x i8> %x0, <16 x i8> %x1, <16 x i8> %x2, i16 %x3)
+  %res1 = call <16 x i8> @llvm.x86.avx512.mask.pshuf.b.128(<16 x i8> %x0, <16 x i8> %x1, <16 x i8> %x2, i16 -1)
+  %res2 = add <16 x i8> %res, %res1
+  ret <16 x i8> %res2
+}
+
+declare <32 x i8> @llvm.x86.avx512.mask.pshuf.b.256(<32 x i8>, <32 x i8>, <32 x i8>, i32)
+
+; CHECK-LABEL: @test_int_x86_avx512_mask_pshuf_b_256
+; CHECK-NOT: call 
+; CHECK: kmov 
+; CHECK: vpshufb %ymm{{.*}}{%k1} 
+define <32 x i8>@test_int_x86_avx512_mask_pshuf_b_256(<32 x i8> %x0, <32 x i8> %x1, <32 x i8> %x2, i32 %x3) {
+  %res = call <32 x i8> @llvm.x86.avx512.mask.pshuf.b.256(<32 x i8> %x0, <32 x i8> %x1, <32 x i8> %x2, i32 %x3)
+  %res1 = call <32 x i8> @llvm.x86.avx512.mask.pshuf.b.256(<32 x i8> %x0, <32 x i8> %x1, <32 x i8> %x2, i32 -1)
+  %res2 = add <32 x i8> %res, %res1
+  ret <32 x i8> %res2
+}
+
+declare <16 x i8> @llvm.x86.avx512.mask.pabs.b.128(<16 x i8>, <16 x i8>, i16)
+
+; CHECK-LABEL: @test_int_x86_avx512_mask_pabs_b_128
+; CHECK-NOT: call 
+; CHECK: kmov 
+; CHECK: vpabsb{{.*}}{%k1} 
+define <16 x i8>@test_int_x86_avx512_mask_pabs_b_128(<16 x i8> %x0, <16 x i8> %x1, i16 %x2) {
+  %res = call <16 x i8> @llvm.x86.avx512.mask.pabs.b.128(<16 x i8> %x0, <16 x i8> %x1, i16 %x2)
+  %res1 = call <16 x i8> @llvm.x86.avx512.mask.pabs.b.128(<16 x i8> %x0, <16 x i8> %x1, i16 -1)
+  %res2 = add <16 x i8> %res, %res1
+  ret <16 x i8> %res2
+}
+
+declare <32 x i8> @llvm.x86.avx512.mask.pabs.b.256(<32 x i8>, <32 x i8>, i32)
+
+; CHECK-LABEL: @test_int_x86_avx512_mask_pabs_b_256
+; CHECK-NOT: call 
+; CHECK: kmov 
+; CHECK: vpabsb{{.*}}{%k1} 
+define <32 x i8>@test_int_x86_avx512_mask_pabs_b_256(<32 x i8> %x0, <32 x i8> %x1, i32 %x2) {
+  %res = call <32 x i8> @llvm.x86.avx512.mask.pabs.b.256(<32 x i8> %x0, <32 x i8> %x1, i32 %x2)
+  %res1 = call <32 x i8> @llvm.x86.avx512.mask.pabs.b.256(<32 x i8> %x0, <32 x i8> %x1, i32 -1)
+  %res2 = add <32 x i8> %res, %res1
+  ret <32 x i8> %res2
+}
+
+declare <8 x i16> @llvm.x86.avx512.mask.pabs.w.128(<8 x i16>, <8 x i16>, i8)
+
+; CHECK-LABEL: @test_int_x86_avx512_mask_pabs_w_128
+; CHECK-NOT: call 
+; CHECK: kmov 
+; CHECK: vpabsw{{.*}}{%k1} 
+define <8 x i16>@test_int_x86_avx512_mask_pabs_w_128(<8 x i16> %x0, <8 x i16> %x1, i8 %x2) {
+  %res = call <8 x i16> @llvm.x86.avx512.mask.pabs.w.128(<8 x i16> %x0, <8 x i16> %x1, i8 %x2)
+  %res1 = call <8 x i16> @llvm.x86.avx512.mask.pabs.w.128(<8 x i16> %x0, <8 x i16> %x1, i8 -1)
+  %res2 = add <8 x i16> %res, %res1
+  ret <8 x i16> %res2
+}
+
+declare <16 x i16> @llvm.x86.avx512.mask.pabs.w.256(<16 x i16>, <16 x i16>, i16)
+
+; CHECK-LABEL: @test_int_x86_avx512_mask_pabs_w_256
+; CHECK-NOT: call 
+; CHECK: kmov 
+; CHECK: vpabsw{{.*}}{%k1} 
+define <16 x i16>@test_int_x86_avx512_mask_pabs_w_256(<16 x i16> %x0, <16 x i16> %x1, i16 %x2) {
+  %res = call <16 x i16> @llvm.x86.avx512.mask.pabs.w.256(<16 x i16> %x0, <16 x i16> %x1, i16 %x2)
+  %res1 = call <16 x i16> @llvm.x86.avx512.mask.pabs.w.256(<16 x i16> %x0, <16 x i16> %x1, i16 -1)
+  %res2 = add <16 x i16> %res, %res1
+  ret <16 x i16> %res2
+}
+
+; CHECK-LABEL: test_x86_mask_blend_b_256
+; CHECK: vpblendmb
+define <32 x i8> @test_x86_mask_blend_b_256(i32 %a0, <32 x i8> %a1, <32 x i8> %a2) {
+  %res = call <32 x i8> @llvm.x86.avx512.mask.blend.b.256(<32 x i8> %a1, <32 x i8> %a2, i32 %a0) ; <<32 x i8>> [#uses=1]
+  ret <32 x i8> %res
+}
+declare <32 x i8> @llvm.x86.avx512.mask.blend.b.256(<32 x i8>, <32 x i8>, i32) nounwind readonly
+
+; CHECK-LABEL: test_x86_mask_blend_w_256
+define <16 x i16> @test_x86_mask_blend_w_256(i16 %mask, <16 x i16> %a1, <16 x i16> %a2) {
+  ; CHECK: vpblendmw
+  %res = call <16 x i16> @llvm.x86.avx512.mask.blend.w.256(<16 x i16> %a1, <16 x i16> %a2, i16 %mask) ; <<16 x i16>> [#uses=1]
+  ret <16 x i16> %res
+}
+declare <16 x i16> @llvm.x86.avx512.mask.blend.w.256(<16 x i16>, <16 x i16>, i16) nounwind readonly
+
+; CHECK-LABEL: test_x86_mask_blend_b_128
+; CHECK: vpblendmb
+define <16 x i8> @test_x86_mask_blend_b_128(i16 %a0, <16 x i8> %a1, <16 x i8> %a2) {
+  %res = call <16 x i8> @llvm.x86.avx512.mask.blend.b.128(<16 x i8> %a1, <16 x i8> %a2, i16 %a0) ; <<16 x i8>> [#uses=1]
+  ret <16 x i8> %res
+}
+declare <16 x i8> @llvm.x86.avx512.mask.blend.b.128(<16 x i8>, <16 x i8>, i16) nounwind readonly
+
+; CHECK-LABEL: test_x86_mask_blend_w_128
+define <8 x i16> @test_x86_mask_blend_w_128(i8 %mask, <8 x i16> %a1, <8 x i16> %a2) {
+  ; CHECK: vpblendmw
+  %res = call <8 x i16> @llvm.x86.avx512.mask.blend.w.128(<8 x i16> %a1, <8 x i16> %a2, i8 %mask) ; <<8 x i16>> [#uses=1]
+  ret <8 x i16> %res
+}
+declare <8 x i16> @llvm.x86.avx512.mask.blend.w.128(<8 x i16>, <8 x i16>, i8) nounwind readonly
+
+declare <8 x i16> @llvm.x86.avx512.mask.pmulhu.w.128(<8 x i16>, <8 x i16>, <8 x i16>, i8)
+
+; CHECK-LABEL: @test_int_x86_avx512_mask_pmulhu_w_128
+; CHECK-NOT: call 
+; CHECK: kmov 
+; CHECK: {%k1} 
+; CHECK: vpmulhuw {{.*}}encoding: [0x62
+define <8 x i16>@test_int_x86_avx512_mask_pmulhu_w_128(<8 x i16> %x0, <8 x i16> %x1, <8 x i16> %x2, i8 %x3) {
+  %res = call <8 x i16> @llvm.x86.avx512.mask.pmulhu.w.128(<8 x i16> %x0, <8 x i16> %x1, <8 x i16> %x2, i8 %x3)
+  %res1 = call <8 x i16> @llvm.x86.avx512.mask.pmulhu.w.128(<8 x i16> %x0, <8 x i16> %x1, <8 x i16> %x2, i8 -1)
+  %res2 = add <8 x i16> %res, %res1
+  ret <8 x i16> %res2
+}
+
+declare <16 x i16> @llvm.x86.avx512.mask.pmulhu.w.256(<16 x i16>, <16 x i16>, <16 x i16>, i16)
+
+; CHECK-LABEL: @test_int_x86_avx512_mask_pmulhu_w_256
+; CHECK-NOT: call 
+; CHECK: kmov 
+; CHECK: {%k1} 
+; CHECK: vpmulhuw {{.*}}encoding: [0x62
+define <16 x i16>@test_int_x86_avx512_mask_pmulhu_w_256(<16 x i16> %x0, <16 x i16> %x1, <16 x i16> %x2, i16 %x3) {
+  %res = call <16 x i16> @llvm.x86.avx512.mask.pmulhu.w.256(<16 x i16> %x0, <16 x i16> %x1, <16 x i16> %x2, i16 %x3)
+  %res1 = call <16 x i16> @llvm.x86.avx512.mask.pmulhu.w.256(<16 x i16> %x0, <16 x i16> %x1, <16 x i16> %x2, i16 -1)
+  %res2 = add <16 x i16> %res, %res1
+  ret <16 x i16> %res2
+}
+
+declare <8 x i16> @llvm.x86.avx512.mask.pmulh.w.128(<8 x i16>, <8 x i16>, <8 x i16>, i8)
+
+; CHECK-LABEL: @test_int_x86_avx512_mask_pmulh_w_128
+; CHECK-NOT: call 
+; CHECK: kmov 
+; CHECK: {%k1} 
+; CHECK: vpmulhw {{.*}}encoding: [0x62
+define <8 x i16>@test_int_x86_avx512_mask_pmulh_w_128(<8 x i16> %x0, <8 x i16> %x1, <8 x i16> %x2, i8 %x3) {
+  %res = call <8 x i16> @llvm.x86.avx512.mask.pmulh.w.128(<8 x i16> %x0, <8 x i16> %x1, <8 x i16> %x2, i8 %x3)
+  %res1 = call <8 x i16> @llvm.x86.avx512.mask.pmulh.w.128(<8 x i16> %x0, <8 x i16> %x1, <8 x i16> %x2, i8 -1)
+  %res2 = add <8 x i16> %res, %res1
+  ret <8 x i16> %res2
+}
+
+declare <16 x i16> @llvm.x86.avx512.mask.pmulh.w.256(<16 x i16>, <16 x i16>, <16 x i16>, i16)
+; CHECK-LABEL: @test_int_x86_avx512_mask_pmulh_w_256
+; CHECK-NOT: call 
+; CHECK: kmov 
+; CHECK: {%k1} 
+; CHECK: vpmulhw {{.*}}encoding: [0x62
+define <16 x i16>@test_int_x86_avx512_mask_pmulh_w_256(<16 x i16> %x0, <16 x i16> %x1, <16 x i16> %x2, i16 %x3) {
+  %res = call <16 x i16> @llvm.x86.avx512.mask.pmulh.w.256(<16 x i16> %x0, <16 x i16> %x1, <16 x i16> %x2, i16 %x3)
+  %res1 = call <16 x i16> @llvm.x86.avx512.mask.pmulh.w.256(<16 x i16> %x0, <16 x i16> %x1, <16 x i16> %x2, i16 -1)
+  %res2 = add <16 x i16> %res, %res1
+  ret <16 x i16> %res2
+}
+
+declare <8 x i16> @llvm.x86.avx512.mask.pmul.hr.sw.128(<8 x i16>, <8 x i16>, <8 x i16>, i8)
+; CHECK-LABEL: @test_int_x86_avx512_mask_pmulhr_sw_128
+; CHECK-NOT: call 
+; CHECK: kmov 
+; CHECK: {%k1} 
+; CHECK: vpmulhrsw {{.*}}encoding: [0x62
+define <8 x i16>@test_int_x86_avx512_mask_pmulhr_sw_128(<8 x i16> %x0, <8 x i16> %x1, <8 x i16> %x2, i8 %x3) {
+  %res = call <8 x i16> @llvm.x86.avx512.mask.pmul.hr.sw.128(<8 x i16> %x0, <8 x i16> %x1, <8 x i16> %x2, i8 %x3)
+  %res1 = call <8 x i16> @llvm.x86.avx512.mask.pmul.hr.sw.128(<8 x i16> %x0, <8 x i16> %x1, <8 x i16> %x2, i8 -1)
+  %res2 = add <8 x i16> %res, %res1
+  ret <8 x i16> %res2
+}
+
+declare <16 x i16> @llvm.x86.avx512.mask.pmul.hr.sw.256(<16 x i16>, <16 x i16>, <16 x i16>, i16)
+; CHECK-LABEL: @test_int_x86_avx512_mask_pmulhr_sw_256
+; CHECK-NOT: call 
+; CHECK: kmov 
+; CHECK: {%k1} 
+; CHECK: vpmulhrsw {{.*}}encoding: [0x62
+define <16 x i16>@test_int_x86_avx512_mask_pmulhr_sw_256(<16 x i16> %x0, <16 x i16> %x1, <16 x i16> %x2, i16 %x3) {
+  %res = call <16 x i16> @llvm.x86.avx512.mask.pmul.hr.sw.256(<16 x i16> %x0, <16 x i16> %x1, <16 x i16> %x2, i16 %x3)
+  %res1 = call <16 x i16> @llvm.x86.avx512.mask.pmul.hr.sw.256(<16 x i16> %x0, <16 x i16> %x1, <16 x i16> %x2, i16 -1)
+  %res2 = add <16 x i16> %res, %res1
+  ret <16 x i16> %res2
+}
+
+declare <16 x i8> @llvm.x86.avx512.mask.pmov.wb.128(<8 x i16>, <16 x i8>, i8)
+
+define <16 x i8>@test_int_x86_avx512_mask_pmov_wb_128(<8 x i16> %x0, <16 x i8> %x1, i8 %x2) {
+; CHECK-LABEL: test_int_x86_avx512_mask_pmov_wb_128:
+; CHECK:       vpmovwb %xmm0, %xmm1 {%k1}
+; CHECK-NEXT:  vpmovwb %xmm0, %xmm2 {%k1} {z}
+; CHECK-NEXT:  vpmovwb %xmm0, %xmm0
+    %res0 = call <16 x i8> @llvm.x86.avx512.mask.pmov.wb.128(<8 x i16> %x0, <16 x i8> %x1, i8 -1)
+    %res1 = call <16 x i8> @llvm.x86.avx512.mask.pmov.wb.128(<8 x i16> %x0, <16 x i8> %x1, i8 %x2)
+    %res2 = call <16 x i8> @llvm.x86.avx512.mask.pmov.wb.128(<8 x i16> %x0, <16 x i8> zeroinitializer, i8 %x2)
+    %res3 = add <16 x i8> %res0, %res1
+    %res4 = add <16 x i8> %res3, %res2
+    ret <16 x i8> %res4
+}
+
+declare void @llvm.x86.avx512.mask.pmov.wb.mem.128(i8* %ptr, <8 x i16>, i8)
+
+define void @test_int_x86_avx512_mask_pmov_wb_mem_128(i8* %ptr, <8 x i16> %x1, i8 %x2) {
+; CHECK-LABEL: test_int_x86_avx512_mask_pmov_wb_mem_128:
+; CHECK:  vpmovwb %xmm0, (%rdi)
+; CHECK:  vpmovwb %xmm0, (%rdi) {%k1}
+    call void @llvm.x86.avx512.mask.pmov.wb.mem.128(i8* %ptr, <8 x i16> %x1, i8 -1)
+    call void @llvm.x86.avx512.mask.pmov.wb.mem.128(i8* %ptr, <8 x i16> %x1, i8 %x2)
+    ret void
+}
+
+declare <16 x i8> @llvm.x86.avx512.mask.pmovs.wb.128(<8 x i16>, <16 x i8>, i8)
+
+define <16 x i8>@test_int_x86_avx512_mask_pmovs_wb_128(<8 x i16> %x0, <16 x i8> %x1, i8 %x2) {
+; CHECK-LABEL: test_int_x86_avx512_mask_pmovs_wb_128:
+; CHECK:       vpmovswb %xmm0, %xmm1 {%k1}
+; CHECK-NEXT:  vpmovswb %xmm0, %xmm2 {%k1} {z}
+; CHECK-NEXT:  vpmovswb %xmm0, %xmm0
+    %res0 = call <16 x i8> @llvm.x86.avx512.mask.pmovs.wb.128(<8 x i16> %x0, <16 x i8> %x1, i8 -1)
+    %res1 = call <16 x i8> @llvm.x86.avx512.mask.pmovs.wb.128(<8 x i16> %x0, <16 x i8> %x1, i8 %x2)
+    %res2 = call <16 x i8> @llvm.x86.avx512.mask.pmovs.wb.128(<8 x i16> %x0, <16 x i8> zeroinitializer, i8 %x2)
+    %res3 = add <16 x i8> %res0, %res1
+    %res4 = add <16 x i8> %res3, %res2
+    ret <16 x i8> %res4
+}
+
+declare void @llvm.x86.avx512.mask.pmovs.wb.mem.128(i8* %ptr, <8 x i16>, i8)
+
+define void @test_int_x86_avx512_mask_pmovs_wb_mem_128(i8* %ptr, <8 x i16> %x1, i8 %x2) {
+; CHECK-LABEL: test_int_x86_avx512_mask_pmovs_wb_mem_128:
+; CHECK:  vpmovswb %xmm0, (%rdi)
+; CHECK:  vpmovswb %xmm0, (%rdi) {%k1}
+    call void @llvm.x86.avx512.mask.pmovs.wb.mem.128(i8* %ptr, <8 x i16> %x1, i8 -1)
+    call void @llvm.x86.avx512.mask.pmovs.wb.mem.128(i8* %ptr, <8 x i16> %x1, i8 %x2)
+    ret void
+}
+
+declare <16 x i8> @llvm.x86.avx512.mask.pmovus.wb.128(<8 x i16>, <16 x i8>, i8)
+
+define <16 x i8>@test_int_x86_avx512_mask_pmovus_wb_128(<8 x i16> %x0, <16 x i8> %x1, i8 %x2) {
+; CHECK-LABEL: test_int_x86_avx512_mask_pmovus_wb_128:
+; CHECK:       vpmovuswb %xmm0, %xmm1 {%k1}
+; CHECK-NEXT:  vpmovuswb %xmm0, %xmm2 {%k1} {z}
+; CHECK-NEXT:  vpmovuswb %xmm0, %xmm0
+    %res0 = call <16 x i8> @llvm.x86.avx512.mask.pmovus.wb.128(<8 x i16> %x0, <16 x i8> %x1, i8 -1)
+    %res1 = call <16 x i8> @llvm.x86.avx512.mask.pmovus.wb.128(<8 x i16> %x0, <16 x i8> %x1, i8 %x2)
+    %res2 = call <16 x i8> @llvm.x86.avx512.mask.pmovus.wb.128(<8 x i16> %x0, <16 x i8> zeroinitializer, i8 %x2)
+    %res3 = add <16 x i8> %res0, %res1
+    %res4 = add <16 x i8> %res3, %res2
+    ret <16 x i8> %res4
+}
+
+declare void @llvm.x86.avx512.mask.pmovus.wb.mem.128(i8* %ptr, <8 x i16>, i8)
+
+define void @test_int_x86_avx512_mask_pmovus_wb_mem_128(i8* %ptr, <8 x i16> %x1, i8 %x2) {
+; CHECK-LABEL: test_int_x86_avx512_mask_pmovus_wb_mem_128:
+; CHECK:  vpmovuswb %xmm0, (%rdi)
+; CHECK:  vpmovuswb %xmm0, (%rdi) {%k1}
+    call void @llvm.x86.avx512.mask.pmovus.wb.mem.128(i8* %ptr, <8 x i16> %x1, i8 -1)
+    call void @llvm.x86.avx512.mask.pmovus.wb.mem.128(i8* %ptr, <8 x i16> %x1, i8 %x2)
+    ret void
+}
+
+declare <16 x i8> @llvm.x86.avx512.mask.pmov.wb.256(<16 x i16>, <16 x i8>, i16)
+
+define <16 x i8>@test_int_x86_avx512_mask_pmov_wb_256(<16 x i16> %x0, <16 x i8> %x1, i16 %x2) {
+; CHECK-LABEL: test_int_x86_avx512_mask_pmov_wb_256:
+; CHECK:       vpmovwb %ymm0, %xmm1 {%k1}
+; CHECK-NEXT:  vpmovwb %ymm0, %xmm2 {%k1} {z}
+; CHECK-NEXT:  vpmovwb %ymm0, %xmm0
+    %res0 = call <16 x i8> @llvm.x86.avx512.mask.pmov.wb.256(<16 x i16> %x0, <16 x i8> %x1, i16 -1)
+    %res1 = call <16 x i8> @llvm.x86.avx512.mask.pmov.wb.256(<16 x i16> %x0, <16 x i8> %x1, i16 %x2)
+    %res2 = call <16 x i8> @llvm.x86.avx512.mask.pmov.wb.256(<16 x i16> %x0, <16 x i8> zeroinitializer, i16 %x2)
+    %res3 = add <16 x i8> %res0, %res1
+    %res4 = add <16 x i8> %res3, %res2
+    ret <16 x i8> %res4
+}
+
+declare void @llvm.x86.avx512.mask.pmov.wb.mem.256(i8* %ptr, <16 x i16>, i16)
+
+define void @test_int_x86_avx512_mask_pmov_wb_mem_256(i8* %ptr, <16 x i16> %x1, i16 %x2) {
+; CHECK-LABEL: test_int_x86_avx512_mask_pmov_wb_mem_256:
+; CHECK:  vpmovwb %ymm0, (%rdi)
+; CHECK:  vpmovwb %ymm0, (%rdi) {%k1}
+    call void @llvm.x86.avx512.mask.pmov.wb.mem.256(i8* %ptr, <16 x i16> %x1, i16 -1)
+    call void @llvm.x86.avx512.mask.pmov.wb.mem.256(i8* %ptr, <16 x i16> %x1, i16 %x2)
+    ret void
+}
+
+declare <16 x i8> @llvm.x86.avx512.mask.pmovs.wb.256(<16 x i16>, <16 x i8>, i16)
+
+define <16 x i8>@test_int_x86_avx512_mask_pmovs_wb_256(<16 x i16> %x0, <16 x i8> %x1, i16 %x2) {
+; CHECK-LABEL: test_int_x86_avx512_mask_pmovs_wb_256:
+; CHECK:       vpmovswb %ymm0, %xmm1 {%k1}
+; CHECK-NEXT:  vpmovswb %ymm0, %xmm2 {%k1} {z}
+; CHECK-NEXT:  vpmovswb %ymm0, %xmm0
+    %res0 = call <16 x i8> @llvm.x86.avx512.mask.pmovs.wb.256(<16 x i16> %x0, <16 x i8> %x1, i16 -1)
+    %res1 = call <16 x i8> @llvm.x86.avx512.mask.pmovs.wb.256(<16 x i16> %x0, <16 x i8> %x1, i16 %x2)
+    %res2 = call <16 x i8> @llvm.x86.avx512.mask.pmovs.wb.256(<16 x i16> %x0, <16 x i8> zeroinitializer, i16 %x2)
+    %res3 = add <16 x i8> %res0, %res1
+    %res4 = add <16 x i8> %res3, %res2
+    ret <16 x i8> %res4
+}
+
+declare void @llvm.x86.avx512.mask.pmovs.wb.mem.256(i8* %ptr, <16 x i16>, i16)
+
+define void @test_int_x86_avx512_mask_pmovs_wb_mem_256(i8* %ptr, <16 x i16> %x1, i16 %x2) {
+; CHECK-LABEL: test_int_x86_avx512_mask_pmovs_wb_mem_256:
+; CHECK:  vpmovswb %ymm0, (%rdi)
+; CHECK:  vpmovswb %ymm0, (%rdi) {%k1}
+    call void @llvm.x86.avx512.mask.pmovs.wb.mem.256(i8* %ptr, <16 x i16> %x1, i16 -1)
+    call void @llvm.x86.avx512.mask.pmovs.wb.mem.256(i8* %ptr, <16 x i16> %x1, i16 %x2)
+    ret void
+}
+
+declare <16 x i8> @llvm.x86.avx512.mask.pmovus.wb.256(<16 x i16>, <16 x i8>, i16)
+
+define <16 x i8>@test_int_x86_avx512_mask_pmovus_wb_256(<16 x i16> %x0, <16 x i8> %x1, i16 %x2) {
+; CHECK-LABEL: test_int_x86_avx512_mask_pmovus_wb_256:
+; CHECK:       vpmovuswb %ymm0, %xmm1 {%k1}
+; CHECK-NEXT:  vpmovuswb %ymm0, %xmm2 {%k1} {z}
+; CHECK-NEXT:  vpmovuswb %ymm0, %xmm0
+    %res0 = call <16 x i8> @llvm.x86.avx512.mask.pmovus.wb.256(<16 x i16> %x0, <16 x i8> %x1, i16 -1)
+    %res1 = call <16 x i8> @llvm.x86.avx512.mask.pmovus.wb.256(<16 x i16> %x0, <16 x i8> %x1, i16 %x2)
+    %res2 = call <16 x i8> @llvm.x86.avx512.mask.pmovus.wb.256(<16 x i16> %x0, <16 x i8> zeroinitializer, i16 %x2)
+    %res3 = add <16 x i8> %res0, %res1
+    %res4 = add <16 x i8> %res3, %res2
+    ret <16 x i8> %res4
+}
+
+declare void @llvm.x86.avx512.mask.pmovus.wb.mem.256(i8* %ptr, <16 x i16>, i16)
+
+define void @test_int_x86_avx512_mask_pmovus_wb_mem_256(i8* %ptr, <16 x i16> %x1, i16 %x2) {
+; CHECK-LABEL: test_int_x86_avx512_mask_pmovus_wb_mem_256:
+; CHECK:  vpmovuswb %ymm0, (%rdi)
+; CHECK:  vpmovuswb %ymm0, (%rdi) {%k1}
+    call void @llvm.x86.avx512.mask.pmovus.wb.mem.256(i8* %ptr, <16 x i16> %x1, i16 -1)
+    call void @llvm.x86.avx512.mask.pmovus.wb.mem.256(i8* %ptr, <16 x i16> %x1, i16 %x2)
+    ret void
+}
+
+declare <4 x i32> @llvm.x86.avx512.mask.pmaddw.d.128(<8 x i16>, <8 x i16>, <4 x i32>, i8)
+
+define <4 x i32>@test_int_x86_avx512_mask_pmaddw_d_128(<8 x i16> %x0, <8 x i16> %x1, <4 x i32> %x2, i8 %x3) {
+; CHECK-LABEL: test_int_x86_avx512_mask_pmaddw_d_128:
+; CHECK:       ## BB#0:
+; CHECK-NEXT:    movzbl %dil, %eax
+; CHECK-NEXT:    kmovw %eax, %k1
+; CHECK-NEXT:    vpmaddwd %xmm1, %xmm0, %xmm2 {%k1}
+; CHECK-NEXT:    vpmaddwd %xmm1, %xmm0, %xmm0
+; CHECK-NEXT:    vpaddd %xmm0, %xmm2, %xmm0
+; CHECK-NEXT:    retq
+  %res = call <4 x i32> @llvm.x86.avx512.mask.pmaddw.d.128(<8 x i16> %x0, <8 x i16> %x1, <4 x i32> %x2, i8 %x3)
+  %res1 = call <4 x i32> @llvm.x86.avx512.mask.pmaddw.d.128(<8 x i16> %x0, <8 x i16> %x1, <4 x i32> %x2, i8 -1)
+  %res2 = add <4 x i32> %res, %res1
+  ret <4 x i32> %res2
+}
+
+declare <8 x i32> @llvm.x86.avx512.mask.pmaddw.d.256(<16 x i16>, <16 x i16>, <8 x i32>, i8)
+
+define <8 x i32>@test_int_x86_avx512_mask_pmaddw_d_256(<16 x i16> %x0, <16 x i16> %x1, <8 x i32> %x2, i8 %x3) {
+; CHECK-LABEL: test_int_x86_avx512_mask_pmaddw_d_256:
+; CHECK:       ## BB#0:
+; CHECK-NEXT:    movzbl %dil, %eax
+; CHECK-NEXT:    kmovw %eax, %k1
+; CHECK-NEXT:    vpmaddwd %ymm1, %ymm0, %ymm2 {%k1}
+; CHECK-NEXT:    vpmaddwd %ymm1, %ymm0, %ymm0
+; CHECK-NEXT:    vpaddd %ymm0, %ymm2, %ymm0
+; CHECK-NEXT:    retq
+  %res = call <8 x i32> @llvm.x86.avx512.mask.pmaddw.d.256(<16 x i16> %x0, <16 x i16> %x1, <8 x i32> %x2, i8 %x3)
+  %res1 = call <8 x i32> @llvm.x86.avx512.mask.pmaddw.d.256(<16 x i16> %x0, <16 x i16> %x1, <8 x i32> %x2, i8 -1)
+  %res2 = add <8 x i32> %res, %res1
+  ret <8 x i32> %res2
+}
+
+declare <8 x i16> @llvm.x86.avx512.mask.pmaddubs.w.128(<16 x i8>, <16 x i8>, <8 x i16>, i8)
+
+define <8 x i16>@test_int_x86_avx512_mask_pmaddubs_w_128(<16 x i8> %x0, <16 x i8> %x1, <8 x i16> %x2, i8 %x3) {
+; CHECK-LABEL: test_int_x86_avx512_mask_pmaddubs_w_128:
+; CHECK:       ## BB#0:
+; CHECK-NEXT:    movzbl %dil, %eax
+; CHECK-NEXT:    kmovw %eax, %k1
+; CHECK-NEXT:    vpmaddubsw %xmm1, %xmm0, %xmm2 {%k1}
+; CHECK-NEXT:    vpmaddubsw %xmm1, %xmm0, %xmm0
+; CHECK-NEXT:    vpaddw %xmm0, %xmm2, %xmm0
+; CHECK-NEXT:    retq
+  %res = call <8 x i16> @llvm.x86.avx512.mask.pmaddubs.w.128(<16 x i8> %x0, <16 x i8> %x1, <8 x i16> %x2, i8 %x3)
+  %res1 = call <8 x i16> @llvm.x86.avx512.mask.pmaddubs.w.128(<16 x i8> %x0, <16 x i8> %x1, <8 x i16> %x2, i8 -1)
+  %res2 = add <8 x i16> %res, %res1
+  ret <8 x i16> %res2
+}
+
+declare <16 x i16> @llvm.x86.avx512.mask.pmaddubs.w.256(<32 x i8>, <32 x i8>, <16 x i16>, i16)
+
+define <16 x i16>@test_int_x86_avx512_mask_pmaddubs_w_256(<32 x i8> %x0, <32 x i8> %x1, <16 x i16> %x2, i16 %x3) {
+; CHECK-LABEL: test_int_x86_avx512_mask_pmaddubs_w_256:
+; CHECK:       ## BB#0:
+; CHECK-NEXT:    kmovw %edi, %k1
+; CHECK-NEXT:    vpmaddubsw %ymm1, %ymm0, %ymm2 {%k1}
+; CHECK-NEXT:    vpmaddubsw %ymm1, %ymm0, %ymm0
+; CHECK-NEXT:    vpaddw %ymm0, %ymm2, %ymm0
+; CHECK-NEXT:    retq
+  %res = call <16 x i16> @llvm.x86.avx512.mask.pmaddubs.w.256(<32 x i8> %x0, <32 x i8> %x1, <16 x i16> %x2, i16 %x3)
+  %res1 = call <16 x i16> @llvm.x86.avx512.mask.pmaddubs.w.256(<32 x i8> %x0, <32 x i8> %x1, <16 x i16> %x2, i16 -1)
+  %res2 = add <16 x i16> %res, %res1
+  ret <16 x i16> %res2
+}
+
+declare <16 x i8> @llvm.x86.avx512.mask.punpckhb.w.128(<16 x i8>, <16 x i8>, <16 x i8>, i16)
+
+define <16 x i8>@test_int_x86_avx512_mask_punpckhb_w_128(<16 x i8> %x0, <16 x i8> %x1, <16 x i8> %x2, i16 %x3) {
+; CHECK-LABEL: test_int_x86_avx512_mask_punpckhb_w_128:
+; CHECK:         vpunpckhbw %xmm1, %xmm0, %xmm2 {%k1}
+; CHECK-NEXT:    vpunpckhbw %xmm1, %xmm0, %xmm0 ## encoding: [0x62,0xf1,0x7d,0x08,0x68,0xc1]
+  %res = call <16 x i8> @llvm.x86.avx512.mask.punpckhb.w.128(<16 x i8> %x0, <16 x i8> %x1, <16 x i8> %x2, i16 %x3)
+  %res1 = call <16 x i8> @llvm.x86.avx512.mask.punpckhb.w.128(<16 x i8> %x0, <16 x i8> %x1, <16 x i8> %x2, i16 -1)
+  %res2 = add <16 x i8> %res, %res1
+  ret <16 x i8> %res2
+}
+
+declare <16 x i8> @llvm.x86.avx512.mask.punpcklb.w.128(<16 x i8>, <16 x i8>, <16 x i8>, i16)
+
+define <16 x i8>@test_int_x86_avx512_mask_punpcklb_w_128(<16 x i8> %x0, <16 x i8> %x1, <16 x i8> %x2, i16 %x3) {
+; CHECK-LABEL: test_int_x86_avx512_mask_punpcklb_w_128:
+; CHECK:         vpunpcklbw %xmm1, %xmm0, %xmm2 {%k1}
+; CHECK-NEXT:    vpunpcklbw %xmm1, %xmm0, %xmm0 ## encoding: [0x62,0xf1,0x7d,0x08,0x60,0xc1]
+  %res = call <16 x i8> @llvm.x86.avx512.mask.punpcklb.w.128(<16 x i8> %x0, <16 x i8> %x1, <16 x i8> %x2, i16 %x3)
+  %res1 = call <16 x i8> @llvm.x86.avx512.mask.punpcklb.w.128(<16 x i8> %x0, <16 x i8> %x1, <16 x i8> %x2, i16 -1)
+  %res2 = add <16 x i8> %res, %res1
+  ret <16 x i8> %res2
+}
+
+declare <32 x i8> @llvm.x86.avx512.mask.punpckhb.w.256(<32 x i8>, <32 x i8>, <32 x i8>, i32)
+
+define <32 x i8>@test_int_x86_avx512_mask_punpckhb_w_256(<32 x i8> %x0, <32 x i8> %x1, <32 x i8> %x2, i32 %x3) {
+; CHECK-LABEL: test_int_x86_avx512_mask_punpckhb_w_256:
+; CHECK:         vpunpckhbw %ymm1, %ymm0, %ymm2 {%k1}
+; CHECK-NEXT:    vpunpckhbw %ymm1, %ymm0, %ymm0 ## encoding: [0x62,0xf1,0x7d,0x28,0x68,0xc1]
+  %res = call <32 x i8> @llvm.x86.avx512.mask.punpckhb.w.256(<32 x i8> %x0, <32 x i8> %x1, <32 x i8> %x2, i32 %x3)
+  %res1 = call <32 x i8> @llvm.x86.avx512.mask.punpckhb.w.256(<32 x i8> %x0, <32 x i8> %x1, <32 x i8> %x2, i32 -1)
+  %res2 = add <32 x i8> %res, %res1
+  ret <32 x i8> %res2
+}
+
+declare <32 x i8> @llvm.x86.avx512.mask.punpcklb.w.256(<32 x i8>, <32 x i8>, <32 x i8>, i32)
+
+define <32 x i8>@test_int_x86_avx512_mask_punpcklb_w_256(<32 x i8> %x0, <32 x i8> %x1, <32 x i8> %x2, i32 %x3) {
+; CHECK-LABEL: test_int_x86_avx512_mask_punpcklb_w_256:
+; CHECK:         vpunpcklbw %ymm1, %ymm0, %ymm2 {%k1}
+; CHECK-NEXT:    vpunpcklbw %ymm1, %ymm0, %ymm0 ## encoding: [0x62,0xf1,0x7d,0x28,0x60,0xc1]
+  %res = call <32 x i8> @llvm.x86.avx512.mask.punpcklb.w.256(<32 x i8> %x0, <32 x i8> %x1, <32 x i8> %x2, i32 %x3)
+  %res1 = call <32 x i8> @llvm.x86.avx512.mask.punpcklb.w.256(<32 x i8> %x0, <32 x i8> %x1, <32 x i8> %x2, i32 -1)
+  %res2 = add <32 x i8> %res, %res1
+  ret <32 x i8> %res2
+}
+
+declare <8 x i16> @llvm.x86.avx512.mask.punpcklw.d.128(<8 x i16>, <8 x i16>, <8 x i16>, i8)
+
+define <8 x i16>@test_int_x86_avx512_mask_punpcklw_d_128(<8 x i16> %x0, <8 x i16> %x1, <8 x i16> %x2, i8 %x3) {
+; CHECK-LABEL: test_int_x86_avx512_mask_punpcklw_d_128:
+; CHECK:         vpunpcklwd %xmm1, %xmm0, %xmm2 {%k1}
+; CHECK-NEXT:    vpunpcklwd %xmm1, %xmm0, %xmm0 ## encoding: [0x62,0xf1,0x7d,0x08,0x61,0xc1]
+  %res = call <8 x i16> @llvm.x86.avx512.mask.punpcklw.d.128(<8 x i16> %x0, <8 x i16> %x1, <8 x i16> %x2, i8 %x3)
+  %res1 = call <8 x i16> @llvm.x86.avx512.mask.punpcklw.d.128(<8 x i16> %x0, <8 x i16> %x1, <8 x i16> %x2, i8 -1)
+  %res2 = add <8 x i16> %res, %res1
+  ret <8 x i16> %res2
+}
+
+declare <8 x i16> @llvm.x86.avx512.mask.punpckhw.d.128(<8 x i16>, <8 x i16>, <8 x i16>, i8)
+
+define <8 x i16>@test_int_x86_avx512_mask_punpckhw_d_128(<8 x i16> %x0, <8 x i16> %x1, <8 x i16> %x2, i8 %x3) {
+; CHECK-LABEL: test_int_x86_avx512_mask_punpckhw_d_128:
+; CHECK:         vpunpckhwd %xmm1, %xmm0, %xmm2 {%k1}
+; CHECK-NEXT:    vpunpckhwd %xmm1, %xmm0, %xmm0 ## encoding: [0x62,0xf1,0x7d,0x08,0x69,0xc1]
+  %res = call <8 x i16> @llvm.x86.avx512.mask.punpckhw.d.128(<8 x i16> %x0, <8 x i16> %x1, <8 x i16> %x2, i8 %x3)
+  %res1 = call <8 x i16> @llvm.x86.avx512.mask.punpckhw.d.128(<8 x i16> %x0, <8 x i16> %x1, <8 x i16> %x2, i8 -1)
+  %res2 = add <8 x i16> %res, %res1
+  ret <8 x i16> %res2
+}
+
+declare <16 x i16> @llvm.x86.avx512.mask.punpcklw.d.256(<16 x i16>, <16 x i16>, <16 x i16>, i16)
+
+define <16 x i16>@test_int_x86_avx512_mask_punpcklw_d_256(<16 x i16> %x0, <16 x i16> %x1, <16 x i16> %x2, i16 %x3) {
+; CHECK-LABEL: test_int_x86_avx512_mask_punpcklw_d_256:
+; CHECK:         vpunpcklwd %ymm1, %ymm0, %ymm2 {%k1}
+; CHECK-NEXT:    vpunpcklwd %ymm1, %ymm0, %ymm0 ## encoding: [0x62,0xf1,0x7d,0x28,0x61,0xc1]
+  %res = call <16 x i16> @llvm.x86.avx512.mask.punpcklw.d.256(<16 x i16> %x0, <16 x i16> %x1, <16 x i16> %x2, i16 %x3)
+  %res1 = call <16 x i16> @llvm.x86.avx512.mask.punpcklw.d.256(<16 x i16> %x0, <16 x i16> %x1, <16 x i16> %x2, i16 -1)
+  %res2 = add <16 x i16> %res, %res1
+  ret <16 x i16> %res2
+}
+
+declare <16 x i16> @llvm.x86.avx512.mask.punpckhw.d.256(<16 x i16>, <16 x i16>, <16 x i16>, i16)
+
+define <16 x i16>@test_int_x86_avx512_mask_punpckhw_d_256(<16 x i16> %x0, <16 x i16> %x1, <16 x i16> %x2, i16 %x3) {
+; CHECK-LABEL: test_int_x86_avx512_mask_punpckhw_d_256:
+; CHECK:         vpunpckhwd %ymm1, %ymm0, %ymm2 {%k1}
+; CHECK-NEXT:    vpunpckhwd %ymm1, %ymm0, %ymm0 ## encoding: [0x62,0xf1,0x7d,0x28,0x69,0xc1]
+  %res = call <16 x i16> @llvm.x86.avx512.mask.punpckhw.d.256(<16 x i16> %x0, <16 x i16> %x1, <16 x i16> %x2, i16 %x3)
+  %res1 = call <16 x i16> @llvm.x86.avx512.mask.punpckhw.d.256(<16 x i16> %x0, <16 x i16> %x1, <16 x i16> %x2, i16 -1)
+  %res2 = add <16 x i16> %res, %res1
+  ret <16 x i16> %res2
+}
diff --git a/test/CodeGen/X86/avx512dq-intrinsics.ll b/test/CodeGen/X86/avx512dq-intrinsics.ll
new file mode 100644
index 00000000000..67a88e15570
--- /dev/null
+++ b/test/CodeGen/X86/avx512dq-intrinsics.ll
@@ -0,0 +1,317 @@
+
+; RUN: llc < %s -mtriple=x86_64-apple-darwin -mcpu=knl -mattr=+avx512dq | FileCheck %s
+
+declare <8 x i64> @llvm.x86.avx512.mask.cvtpd2qq.512(<8 x double>, <8 x i64>, i8, i32)
+
+define <8 x i64>@test_int_x86_avx512_mask_cvt_pd2qq_512(<8 x double> %x0, <8 x i64> %x1, i8 %x2) {
+; CHECK-LABEL: test_int_x86_avx512_mask_cvt_pd2qq_512:
+; CHECK:       ## BB#0:
+; CHECK-NEXT:    kmovb %edi, %k1
+; CHECK-NEXT:    vcvtpd2qq {ru-sae}, %zmm0, %zmm1 {%k1}
+; CHECK-NEXT:    vcvtpd2qq {rn-sae}, %zmm0, %zmm0
+; CHECK-NEXT:    vpaddq %zmm0, %zmm1, %zmm0
+; CHECK-NEXT:    retq
+  %res = call <8 x i64> @llvm.x86.avx512.mask.cvtpd2qq.512(<8 x double> %x0, <8 x i64> %x1, i8 %x2, i32 2)
+  %res1 = call <8 x i64> @llvm.x86.avx512.mask.cvtpd2qq.512(<8 x double> %x0, <8 x i64> %x1, i8 -1, i32 0)
+  %res2 = add <8 x i64> %res, %res1
+  ret <8 x i64> %res2
+}
+
+declare <8 x i64> @llvm.x86.avx512.mask.cvtpd2uqq.512(<8 x double>, <8 x i64>, i8, i32)
+
+define <8 x i64>@test_int_x86_avx512_mask_cvt_pd2uqq_512(<8 x double> %x0, <8 x i64> %x1, i8 %x2) {
+; CHECK-LABEL: test_int_x86_avx512_mask_cvt_pd2uqq_512:
+; CHECK:       ## BB#0:
+; CHECK-NEXT:    kmovb %edi, %k1
+; CHECK-NEXT:    vcvtpd2uqq {ru-sae}, %zmm0, %zmm1 {%k1}
+; CHECK-NEXT:    vcvtpd2uqq {rn-sae}, %zmm0, %zmm0
+; CHECK-NEXT:    vpaddq %zmm0, %zmm1, %zmm0
+; CHECK-NEXT:    retq
+  %res = call <8 x i64> @llvm.x86.avx512.mask.cvtpd2uqq.512(<8 x double> %x0, <8 x i64> %x1, i8 %x2, i32 2)
+  %res1 = call <8 x i64> @llvm.x86.avx512.mask.cvtpd2uqq.512(<8 x double> %x0, <8 x i64> %x1, i8 -1, i32 0)
+  %res2 = add <8 x i64> %res, %res1
+  ret <8 x i64> %res2
+}
+
+declare <8 x i64> @llvm.x86.avx512.mask.cvtps2qq.512(<8 x float>, <8 x i64>, i8, i32)
+
+define <8 x i64>@test_int_x86_avx512_mask_cvt_ps2qq_512(<8 x float> %x0, <8 x i64> %x1, i8 %x2) {
+; CHECK-LABEL: test_int_x86_avx512_mask_cvt_ps2qq_512:
+; CHECK:       ## BB#0:
+; CHECK-NEXT:    kmovb %edi, %k1
+; CHECK-NEXT:    vcvtps2qq {ru-sae}, %ymm0, %zmm1 {%k1}
+; CHECK-NEXT:    vcvtps2qq {rn-sae}, %ymm0, %zmm0
+; CHECK-NEXT:    vpaddq %zmm0, %zmm1, %zmm0
+; CHECK-NEXT:    retq
+  %res = call <8 x i64> @llvm.x86.avx512.mask.cvtps2qq.512(<8 x float> %x0, <8 x i64> %x1, i8 %x2, i32 2)
+  %res1 = call <8 x i64> @llvm.x86.avx512.mask.cvtps2qq.512(<8 x float> %x0, <8 x i64> %x1, i8 -1, i32 0)
+  %res2 = add <8 x i64> %res, %res1
+  ret <8 x i64> %res2
+}
+
+declare <8 x i64> @llvm.x86.avx512.mask.cvtps2uqq.512(<8 x float>, <8 x i64>, i8, i32)
+
+define <8 x i64>@test_int_x86_avx512_mask_cvt_ps2uqq_512(<8 x float> %x0, <8 x i64> %x1, i8 %x2) {
+; CHECK-LABEL: test_int_x86_avx512_mask_cvt_ps2uqq_512:
+; CHECK:       ## BB#0:
+; CHECK-NEXT:    kmovb %edi, %k1
+; CHECK-NEXT:    vcvtps2uqq {ru-sae}, %ymm0, %zmm1 {%k1}
+; CHECK-NEXT:    vcvtps2uqq {rn-sae}, %ymm0, %zmm0
+; CHECK-NEXT:    vpaddq %zmm0, %zmm1, %zmm0
+; CHECK-NEXT:    retq
+  %res = call <8 x i64> @llvm.x86.avx512.mask.cvtps2uqq.512(<8 x float> %x0, <8 x i64> %x1, i8 %x2, i32 2)
+  %res1 = call <8 x i64> @llvm.x86.avx512.mask.cvtps2uqq.512(<8 x float> %x0, <8 x i64> %x1, i8 -1, i32 0)
+  %res2 = add <8 x i64> %res, %res1
+  ret <8 x i64> %res2
+}
+
+declare <8 x double> @llvm.x86.avx512.mask.cvtqq2pd.512(<8 x i64>, <8 x double>, i8, i32)
+
+define <8 x double>@test_int_x86_avx512_mask_cvt_qq2pd_512(<8 x i64> %x0, <8 x double> %x1, i8 %x2) {
+; CHECK-LABEL: test_int_x86_avx512_mask_cvt_qq2pd_512:
+; CHECK:       ## BB#0:
+; CHECK-NEXT:    kmovb %edi, %k1
+; CHECK-NEXT:    vcvtqq2pd %zmm0, %zmm1 {%k1}
+; CHECK-NEXT:    vcvtqq2pd {rn-sae}, %zmm0, %zmm0
+; CHECK-NEXT:    vaddpd %zmm0, %zmm1, %zmm0
+; CHECK-NEXT:    retq
+  %res = call <8 x double> @llvm.x86.avx512.mask.cvtqq2pd.512(<8 x i64> %x0, <8 x double> %x1, i8 %x2, i32 4)
+  %res1 = call <8 x double> @llvm.x86.avx512.mask.cvtqq2pd.512(<8 x i64> %x0, <8 x double> %x1, i8 -1, i32 0)
+  %res2 = fadd <8 x double> %res, %res1
+  ret <8 x double> %res2
+}
+
+declare <8 x float> @llvm.x86.avx512.mask.cvtqq2ps.512(<8 x i64>, <8 x float>, i8, i32)
+
+define <8 x float>@test_int_x86_avx512_mask_cvt_qq2ps_512(<8 x i64> %x0, <8 x float> %x1, i8 %x2) {
+; CHECK-LABEL: test_int_x86_avx512_mask_cvt_qq2ps_512:
+; CHECK:       ## BB#0:
+; CHECK-NEXT:    kmovb %edi, %k1
+; CHECK-NEXT:    vcvtqq2ps %zmm0, %ymm1 {%k1}
+; CHECK-NEXT:    vcvtqq2ps {rn-sae}, %zmm0, %ymm0
+; CHECK-NEXT:    vaddps %ymm0, %ymm1, %ymm0
+; CHECK-NEXT:    retq
+  %res = call <8 x float> @llvm.x86.avx512.mask.cvtqq2ps.512(<8 x i64> %x0, <8 x float> %x1, i8 %x2, i32 4)
+  %res1 = call <8 x float> @llvm.x86.avx512.mask.cvtqq2ps.512(<8 x i64> %x0, <8 x float> %x1, i8 -1, i32 0)
+  %res2 = fadd <8 x float> %res, %res1
+  ret <8 x float> %res2
+}
+
+declare <8 x i64> @llvm.x86.avx512.mask.cvttpd2qq.512(<8 x double>, <8 x i64>, i8, i32)
+
+define <8 x i64>@test_int_x86_avx512_mask_cvtt_pd2qq_512(<8 x double> %x0, <8 x i64> %x1, i8 %x2) {
+; CHECK-LABEL: test_int_x86_avx512_mask_cvtt_pd2qq_512:
+; CHECK:       ## BB#0:
+; CHECK-NEXT:    kmovb %edi, %k1
+; CHECK-NEXT:    vcvttpd2qq %zmm0, %zmm1 {%k1}
+; CHECK-NEXT:    vcvttpd2qq {sae}, %zmm0, %zmm0
+; CHECK-NEXT:    vpaddq %zmm0, %zmm1, %zmm0
+; CHECK-NEXT:    retq
+  %res = call <8 x i64> @llvm.x86.avx512.mask.cvttpd2qq.512(<8 x double> %x0, <8 x i64> %x1, i8 %x2, i32 4)
+  %res1 = call <8 x i64> @llvm.x86.avx512.mask.cvttpd2qq.512(<8 x double> %x0, <8 x i64> %x1, i8 -1, i32 8)
+  %res2 = add <8 x i64> %res, %res1
+  ret <8 x i64> %res2
+}
+
+declare <8 x i64> @llvm.x86.avx512.mask.cvttpd2uqq.512(<8 x double>, <8 x i64>, i8, i32)
+
+define <8 x i64>@test_int_x86_avx512_mask_cvtt_pd2uqq_512(<8 x double> %x0, <8 x i64> %x1, i8 %x2) {
+; CHECK-LABEL: test_int_x86_avx512_mask_cvtt_pd2uqq_512:
+; CHECK:       ## BB#0:
+; CHECK-NEXT:    kmovb %edi, %k1
+; CHECK-NEXT:    vcvttpd2uqq %zmm0, %zmm1 {%k1}
+; CHECK-NEXT:    vcvttpd2uqq {sae}, %zmm0, %zmm0
+; CHECK-NEXT:    vpaddq %zmm0, %zmm1, %zmm0
+; CHECK-NEXT:    retq
+  %res = call <8 x i64> @llvm.x86.avx512.mask.cvttpd2uqq.512(<8 x double> %x0, <8 x i64> %x1, i8 %x2, i32 4)
+  %res1 = call <8 x i64> @llvm.x86.avx512.mask.cvttpd2uqq.512(<8 x double> %x0, <8 x i64> %x1, i8 -1, i32 8)
+  %res2 = add <8 x i64> %res, %res1
+  ret <8 x i64> %res2
+}
+
+declare <8 x i64> @llvm.x86.avx512.mask.cvttps2qq.512(<8 x float>, <8 x i64>, i8, i32)
+
+define <8 x i64>@test_int_x86_avx512_mask_cvtt_ps2qq_512(<8 x float> %x0, <8 x i64> %x1, i8 %x2) {
+; CHECK-LABEL: test_int_x86_avx512_mask_cvtt_ps2qq_512:
+; CHECK:       ## BB#0:
+; CHECK-NEXT:    kmovb %edi, %k1
+; CHECK-NEXT:    vcvttps2qq %ymm0, %zmm1 {%k1}
+; CHECK-NEXT:    vcvttps2qq {sae}, %ymm0, %zmm0
+; CHECK-NEXT:    vpaddq %zmm0, %zmm1, %zmm0
+; CHECK-NEXT:    retq
+  %res = call <8 x i64> @llvm.x86.avx512.mask.cvttps2qq.512(<8 x float> %x0, <8 x i64> %x1, i8 %x2, i32 4)
+  %res1 = call <8 x i64> @llvm.x86.avx512.mask.cvttps2qq.512(<8 x float> %x0, <8 x i64> %x1, i8 -1, i32 8)
+  %res2 = add <8 x i64> %res, %res1
+  ret <8 x i64> %res2
+}
+
+declare <8 x i64> @llvm.x86.avx512.mask.cvttps2uqq.512(<8 x float>, <8 x i64>, i8, i32)
+
+define <8 x i64>@test_int_x86_avx512_mask_cvtt_ps2uqq_512(<8 x float> %x0, <8 x i64> %x1, i8 %x2) {
+; CHECK-LABEL: test_int_x86_avx512_mask_cvtt_ps2uqq_512:
+; CHECK:       ## BB#0:
+; CHECK-NEXT:    kmovb %edi, %k1
+; CHECK-NEXT:    vcvttps2uqq %ymm0, %zmm1 {%k1}
+; CHECK-NEXT:    vcvttps2uqq {sae}, %ymm0, %zmm0
+; CHECK-NEXT:    vpaddq %zmm0, %zmm1, %zmm0
+; CHECK-NEXT:    retq
+  %res = call <8 x i64> @llvm.x86.avx512.mask.cvttps2uqq.512(<8 x float> %x0, <8 x i64> %x1, i8 %x2, i32 4)
+  %res1 = call <8 x i64> @llvm.x86.avx512.mask.cvttps2uqq.512(<8 x float> %x0, <8 x i64> %x1, i8 -1, i32 8)
+  %res2 = add <8 x i64> %res, %res1
+  ret <8 x i64> %res2
+}
+
+declare <8 x double> @llvm.x86.avx512.mask.cvtuqq2pd.512(<8 x i64>, <8 x double>, i8, i32)
+
+define <8 x double>@test_int_x86_avx512_mask_cvt_uqq2pd_512(<8 x i64> %x0, <8 x double> %x1, i8 %x2) {
+; CHECK-LABEL: test_int_x86_avx512_mask_cvt_uqq2pd_512:
+; CHECK:       ## BB#0:
+; CHECK-NEXT:    kmovb %edi, %k1
+; CHECK-NEXT:    vcvtuqq2pd %zmm0, %zmm1 {%k1}
+; CHECK-NEXT:    vcvtuqq2pd {rn-sae}, %zmm0, %zmm0
+; CHECK-NEXT:    vaddpd %zmm0, %zmm1, %zmm0
+; CHECK-NEXT:    retq
+  %res = call <8 x double> @llvm.x86.avx512.mask.cvtuqq2pd.512(<8 x i64> %x0, <8 x double> %x1, i8 %x2, i32 4)
+  %res1 = call <8 x double> @llvm.x86.avx512.mask.cvtuqq2pd.512(<8 x i64> %x0, <8 x double> %x1, i8 -1, i32 0)
+  %res2 = fadd <8 x double> %res, %res1
+  ret <8 x double> %res2
+}
+
+declare <8 x float> @llvm.x86.avx512.mask.cvtuqq2ps.512(<8 x i64>, <8 x float>, i8, i32)
+
+define <8 x float>@test_int_x86_avx512_mask_cvt_uqq2ps_512(<8 x i64> %x0, <8 x float> %x1, i8 %x2) {
+; CHECK-LABEL: test_int_x86_avx512_mask_cvt_uqq2ps_512:
+; CHECK:       ## BB#0:
+; CHECK-NEXT:    kmovb %edi, %k1
+; CHECK-NEXT:    vcvtuqq2ps %zmm0, %ymm1 {%k1}
+; CHECK-NEXT:    vcvtuqq2ps {rn-sae}, %zmm0, %ymm0
+; CHECK-NEXT:    vaddps %ymm0, %ymm1, %ymm0
+; CHECK-NEXT:    retq
+  %res = call <8 x float> @llvm.x86.avx512.mask.cvtuqq2ps.512(<8 x i64> %x0, <8 x float> %x1, i8 %x2, i32 4)
+  %res1 = call <8 x float> @llvm.x86.avx512.mask.cvtuqq2ps.512(<8 x i64> %x0, <8 x float> %x1, i8 -1, i32 0)
+  %res2 = fadd <8 x float> %res, %res1
+  ret <8 x float> %res2
+}
+
+declare <8 x double> @llvm.x86.avx512.mask.reduce.pd.512(<8 x double>, i32, <8 x double>, i8, i32)
+; CHECK-LABEL: @test_int_x86_avx512_mask_reduce_pd_512
+; CHECK-NOT: call
+; CHECK: kmov
+; CHECK: vreducepd {{.*}}{%k1}
+; CHECK: vreducepd
+; CHECK: {sae}
+define <8 x double>@test_int_x86_avx512_mask_reduce_pd_512(<8 x double> %x0, <8 x double> %x2, i8 %x3) {
+  %res = call <8 x double> @llvm.x86.avx512.mask.reduce.pd.512(<8 x double> %x0, i32 8, <8 x double> %x2, i8 %x3, i32 4)
+  %res1 = call <8 x double> @llvm.x86.avx512.mask.reduce.pd.512(<8 x double> %x0, i32 4, <8 x double> %x2, i8 -1, i32 8)
+  %res2 = fadd <8 x double> %res, %res1
+  ret <8 x double> %res2
+}
+
+declare <16 x float> @llvm.x86.avx512.mask.reduce.ps.512(<16 x float>, i32, <16 x float>, i16, i32)
+; CHECK-LABEL: @test_int_x86_avx512_mask_reduce_ps_512
+; CHECK-NOT: call 
+; CHECK: kmov 
+; CHECK: vreduceps
+; CHECK: {sae}
+; CKECK: {%k1}
+; CHECK: vreduceps
+define <16 x float>@test_int_x86_avx512_mask_reduce_ps_512(<16 x float> %x0, <16 x float> %x2, i16 %x3) {
+  %res = call <16 x float> @llvm.x86.avx512.mask.reduce.ps.512(<16 x float> %x0, i32 44, <16 x float> %x2, i16 %x3, i32 8)
+  %res1 = call <16 x float> @llvm.x86.avx512.mask.reduce.ps.512(<16 x float> %x0, i32 11, <16 x float> %x2, i16 -1, i32 4)
+  %res2 = fadd <16 x float> %res, %res1
+  ret <16 x float> %res2
+}
+
+declare <8 x double> @llvm.x86.avx512.mask.range.pd.512(<8 x double>, <8 x double>, i32, <8 x double>, i8, i32)
+; CHECK-LABEL: @test_int_x86_avx512_mask_range_pd_512
+; CHECK-NOT: call 
+; CHECK: kmov 
+; CHECK: vrangepd
+; CKECK: {%k1}
+; CHECK: vrangepd
+; CHECK: {sae}
+define <8 x double>@test_int_x86_avx512_mask_range_pd_512(<8 x double> %x0, <8 x double> %x1, <8 x double> %x3, i8 %x4) {
+  %res = call <8 x double> @llvm.x86.avx512.mask.range.pd.512(<8 x double> %x0, <8 x double> %x1, i32 8, <8 x double> %x3, i8 %x4, i32 4)
+  %res1 = call <8 x double> @llvm.x86.avx512.mask.range.pd.512(<8 x double> %x0, <8 x double> %x1, i32 4, <8 x double> %x3, i8 -1, i32 8)
+  %res2 = fadd <8 x double> %res, %res1
+  ret <8 x double> %res2
+}
+
+declare <16 x float> @llvm.x86.avx512.mask.range.ps.512(<16 x float>, <16 x float>, i32, <16 x float>, i16, i32)
+
+; CHECK-LABEL: @test_int_x86_avx512_mask_range_ps_512
+; CHECK-NOT: call 
+; CHECK: kmov 
+; CHECK: vrangeps
+; CKECK: {%k1}
+; CHECK: vrangeps
+; CHECK: {sae}
+define <16 x float>@test_int_x86_avx512_mask_range_ps_512(<16 x float> %x0, <16 x float> %x1, <16 x float> %x3, i16 %x4) {
+  %res = call <16 x float> @llvm.x86.avx512.mask.range.ps.512(<16 x float> %x0, <16 x float> %x1, i32 88, <16 x float> %x3, i16 %x4, i32 4)
+  %res1 = call <16 x float> @llvm.x86.avx512.mask.range.ps.512(<16 x float> %x0, <16 x float> %x1, i32 4, <16 x float> %x3, i16 -1, i32 8)
+  %res2 = fadd <16 x float> %res, %res1
+  ret <16 x float> %res2
+}
+
+declare <4 x float> @llvm.x86.avx512.mask.reduce.ss(<4 x float>, <4 x float>,<4 x float>, i8, i32, i32)
+
+; CHECK-LABEL: @test_int_x86_avx512_mask_reduce_ss
+; CHECK-NOT: call 
+; CHECK: kmov 
+; CHECK: vreducess
+; CKECK: {%k1}
+; CHECK: vreducess
+; CHECK: {sae}
+define <4 x float>@test_int_x86_avx512_mask_reduce_ss(<4 x float> %x0, <4 x float> %x1, <4 x float> %x3, i8 %x4) {
+  %res = call <4 x float> @llvm.x86.avx512.mask.reduce.ss(<4 x float> %x0, <4 x float> %x1, <4 x float> %x3, i8 %x4, i32 4, i32 4)
+  %res1 = call <4 x float> @llvm.x86.avx512.mask.reduce.ss(<4 x float> %x0, <4 x float> %x1, <4 x float> %x3, i8 -1, i32 4, i32 8)
+  %res2 = fadd <4 x float> %res, %res1
+  ret <4 x float> %res2
+}
+
+declare <4 x float> @llvm.x86.avx512.mask.range.ss(<4 x float>, <4 x float>,<4 x float>, i8, i32, i32)
+; CHECK-LABEL: @test_int_x86_avx512_mask_range_ss
+; CHECK-NOT: call 
+; CHECK: kmov 
+; CHECK: vrangess
+; CHECK: {sae}
+; CKECK: {%k1}
+; CHECK: vrangess
+; CHECK: {sae} 
+define <4 x float>@test_int_x86_avx512_mask_range_ss(<4 x float> %x0, <4 x float> %x1, <4 x float> %x3, i8 %x4) {
+  %res = call <4 x float> @llvm.x86.avx512.mask.range.ss(<4 x float> %x0, <4 x float> %x1, <4 x float> %x3, i8 %x4, i32 4, i32 8)
+  %res1 = call <4 x float> @llvm.x86.avx512.mask.range.ss(<4 x float> %x0, <4 x float> %x1, <4 x float> %x3, i8 -1, i32 4, i32 8)
+  %res2 = fadd <4 x float> %res, %res1
+  ret <4 x float> %res2
+}
+
+declare <2 x double> @llvm.x86.avx512.mask.reduce.sd(<2 x double>, <2 x double>,<2 x double>, i8, i32, i32)
+
+; CHECK-LABEL: @test_int_x86_avx512_mask_reduce_sd
+; CHECK-NOT: call 
+; CHECK: kmov 
+; CHECK: vreducesd
+; CKECK: {%k1}
+; CHECK: vreducesd
+; CHECK: {sae} 
+define <2 x double>@test_int_x86_avx512_mask_reduce_sd(<2 x double> %x0, <2 x double> %x1, <2 x double> %x3, i8 %x4) {
+  %res = call <2 x double> @llvm.x86.avx512.mask.reduce.sd(<2 x double> %x0, <2 x double> %x1, <2 x double> %x3, i8 %x4, i32 4, i32 4)
+  %res1 = call <2 x double> @llvm.x86.avx512.mask.reduce.sd(<2 x double> %x0, <2 x double> %x1, <2 x double> %x3, i8 -1, i32 4, i32 8)
+  %res2 = fadd <2 x double> %res, %res1
+  ret <2 x double> %res2
+}
+
+declare <2 x double> @llvm.x86.avx512.mask.range.sd(<2 x double>, <2 x double>,<2 x double>, i8, i32, i32)
+; CHECK-LABEL: @test_int_x86_avx512_mask_range_sd
+; CHECK-NOT: call 
+; CHECK: kmov 
+; CHECK: vrangesd
+; CKECK: {%k1}
+; CHECK: vrangesd
+; CHECK: {sae} 
+define <2 x double>@test_int_x86_avx512_mask_range_sd(<2 x double> %x0, <2 x double> %x1, <2 x double> %x3, i8 %x4) {
+  %res = call <2 x double> @llvm.x86.avx512.mask.range.sd(<2 x double> %x0, <2 x double> %x1, <2 x double> %x3, i8 %x4, i32 4, i32 4)
+  %res1 = call <2 x double> @llvm.x86.avx512.mask.range.sd(<2 x double> %x0, <2 x double> %x1, <2 x double> %x3, i8 -1, i32 4, i32 8)
+  %res2 = fadd <2 x double> %res, %res1
+  ret <2 x double> %res2
+}
diff --git a/test/CodeGen/X86/avx512dqvl-intrinsics.ll b/test/CodeGen/X86/avx512dqvl-intrinsics.ll
new file mode 100644
index 00000000000..2fcfac0f1bb
--- /dev/null
+++ b/test/CodeGen/X86/avx512dqvl-intrinsics.ll
@@ -0,0 +1,1650 @@
+; RUN: llc < %s -mtriple=x86_64-apple-darwin -mcpu=knl -mattr=+avx512dq -mattr=+avx512vl --show-mc-encoding| FileCheck %s
+
+define <8 x i64> @test_mask_mullo_epi64_rr_512(<8 x i64> %a, <8 x i64> %b) {
+  ;CHECK-LABEL: test_mask_mullo_epi64_rr_512
+  ;CHECK: vpmullq %zmm1, %zmm0, %zmm0     ## encoding: [0x62,0xf2,0xfd,0x48,0x40,0xc1]
+  %res = call <8 x i64> @llvm.x86.avx512.mask.pmull.q.512(<8 x i64> %a, <8 x i64> %b, <8 x i64> zeroinitializer, i8 -1)
+  ret <8 x i64> %res
+}
+
+define <8 x i64> @test_mask_mullo_epi64_rrk_512(<8 x i64> %a, <8 x i64> %b, <8 x i64> %passThru, i8 %mask) {
+  ;CHECK-LABEL: test_mask_mullo_epi64_rrk_512
+  ;CHECK: vpmullq %zmm1, %zmm0, %zmm2 {%k1} ## encoding: [0x62,0xf2,0xfd,0x49,0x40,0xd1]
+  %res = call <8 x i64> @llvm.x86.avx512.mask.pmull.q.512(<8 x i64> %a, <8 x i64> %b, <8 x i64> %passThru, i8 %mask)
+  ret <8 x i64> %res
+}
+
+define <8 x i64> @test_mask_mullo_epi64_rrkz_512(<8 x i64> %a, <8 x i64> %b, i8 %mask) {
+  ;CHECK-LABEL: test_mask_mullo_epi64_rrkz_512
+  ;CHECK: vpmullq %zmm1, %zmm0, %zmm0 {%k1} {z} ## encoding: [0x62,0xf2,0xfd,0xc9,0x40,0xc1]
+  %res = call <8 x i64> @llvm.x86.avx512.mask.pmull.q.512(<8 x i64> %a, <8 x i64> %b, <8 x i64> zeroinitializer, i8 %mask)
+  ret <8 x i64> %res
+}
+
+define <8 x i64> @test_mask_mullo_epi64_rm_512(<8 x i64> %a, <8 x i64>* %ptr_b) {
+  ;CHECK-LABEL: test_mask_mullo_epi64_rm_512
+  ;CHECK: vpmullq (%rdi), %zmm0, %zmm0    ## encoding: [0x62,0xf2,0xfd,0x48,0x40,0x07]
+  %b = load <8 x i64>, <8 x i64>* %ptr_b
+  %res = call <8 x i64> @llvm.x86.avx512.mask.pmull.q.512(<8 x i64> %a, <8 x i64> %b, <8 x i64> zeroinitializer, i8 -1)
+  ret <8 x i64> %res
+}
+
+define <8 x i64> @test_mask_mullo_epi64_rmk_512(<8 x i64> %a, <8 x i64>* %ptr_b, <8 x i64> %passThru, i8 %mask) {
+  ;CHECK-LABEL: test_mask_mullo_epi64_rmk_512
+  ;CHECK: vpmullq (%rdi), %zmm0, %zmm1 {%k1} ## encoding: [0x62,0xf2,0xfd,0x49,0x40,0x0f]
+  %b = load <8 x i64>, <8 x i64>* %ptr_b
+  %res = call <8 x i64> @llvm.x86.avx512.mask.pmull.q.512(<8 x i64> %a, <8 x i64> %b, <8 x i64> %passThru, i8 %mask)
+  ret <8 x i64> %res
+}
+
+define <8 x i64> @test_mask_mullo_epi64_rmkz_512(<8 x i64> %a, <8 x i64>* %ptr_b, i8 %mask) {
+  ;CHECK-LABEL: test_mask_mullo_epi64_rmkz_512
+  ;CHECK: vpmullq (%rdi), %zmm0, %zmm0 {%k1} {z} ## encoding: [0x62,0xf2,0xfd,0xc9,0x40,0x07]
+  %b = load <8 x i64>, <8 x i64>* %ptr_b
+  %res = call <8 x i64> @llvm.x86.avx512.mask.pmull.q.512(<8 x i64> %a, <8 x i64> %b, <8 x i64> zeroinitializer, i8 %mask)
+  ret <8 x i64> %res
+}
+
+define <8 x i64> @test_mask_mullo_epi64_rmb_512(<8 x i64> %a, i64* %ptr_b) {
+  ;CHECK-LABEL: test_mask_mullo_epi64_rmb_512
+  ;CHECK: vpmullq (%rdi){1to8}, %zmm0, %zmm0  ## encoding: [0x62,0xf2,0xfd,0x58,0x40,0x07]
+  %q = load i64, i64* %ptr_b
+  %vecinit.i = insertelement <8 x i64> undef, i64 %q, i32 0
+  %b = shufflevector <8 x i64> %vecinit.i, <8 x i64> undef, <8 x i32> zeroinitializer
+  %res = call <8 x i64> @llvm.x86.avx512.mask.pmull.q.512(<8 x i64> %a, <8 x i64> %b, <8 x i64> zeroinitializer, i8 -1)
+  ret <8 x i64> %res
+}
+
+define <8 x i64> @test_mask_mullo_epi64_rmbk_512(<8 x i64> %a, i64* %ptr_b, <8 x i64> %passThru, i8 %mask) {
+  ;CHECK-LABEL: test_mask_mullo_epi64_rmbk_512
+  ;CHECK: vpmullq (%rdi){1to8}, %zmm0, %zmm1 {%k1} ## encoding: [0x62,0xf2,0xfd,0x59,0x40,0x0f]
+  %q = load i64, i64* %ptr_b
+  %vecinit.i = insertelement <8 x i64> undef, i64 %q, i32 0
+  %b = shufflevector <8 x i64> %vecinit.i, <8 x i64> undef, <8 x i32> zeroinitializer
+  %res = call <8 x i64> @llvm.x86.avx512.mask.pmull.q.512(<8 x i64> %a, <8 x i64> %b, <8 x i64> %passThru, i8 %mask)
+  ret <8 x i64> %res
+}
+
+define <8 x i64> @test_mask_mullo_epi64_rmbkz_512(<8 x i64> %a, i64* %ptr_b, i8 %mask) {
+  ;CHECK-LABEL: test_mask_mullo_epi64_rmbkz_512
+  ;CHECK: vpmullq (%rdi){1to8}, %zmm0, %zmm0 {%k1} {z} ## encoding: [0x62,0xf2,0xfd,0xd9,0x40,0x07]
+  %q = load i64, i64* %ptr_b
+  %vecinit.i = insertelement <8 x i64> undef, i64 %q, i32 0
+  %b = shufflevector <8 x i64> %vecinit.i, <8 x i64> undef, <8 x i32> zeroinitializer
+  %res = call <8 x i64> @llvm.x86.avx512.mask.pmull.q.512(<8 x i64> %a, <8 x i64> %b, <8 x i64> zeroinitializer, i8 %mask)
+  ret <8 x i64> %res
+}
+declare <8 x i64> @llvm.x86.avx512.mask.pmull.q.512(<8 x i64>, <8 x i64>, <8 x i64>, i8)
+
+define <4 x i64> @test_mask_mullo_epi64_rr_256(<4 x i64> %a, <4 x i64> %b) {
+  ;CHECK-LABEL: test_mask_mullo_epi64_rr_256
+  ;CHECK: vpmullq %ymm1, %ymm0, %ymm0     ## encoding: [0x62,0xf2,0xfd,0x28,0x40,0xc1]
+  %res = call <4 x i64> @llvm.x86.avx512.mask.pmull.q.256(<4 x i64> %a, <4 x i64> %b, <4 x i64> zeroinitializer, i8 -1)
+  ret <4 x i64> %res
+}
+
+define <4 x i64> @test_mask_mullo_epi64_rrk_256(<4 x i64> %a, <4 x i64> %b, <4 x i64> %passThru, i8 %mask) {
+  ;CHECK-LABEL: test_mask_mullo_epi64_rrk_256
+  ;CHECK: vpmullq %ymm1, %ymm0, %ymm2 {%k1} ## encoding: [0x62,0xf2,0xfd,0x29,0x40,0xd1]
+  %res = call <4 x i64> @llvm.x86.avx512.mask.pmull.q.256(<4 x i64> %a, <4 x i64> %b, <4 x i64> %passThru, i8 %mask)
+  ret <4 x i64> %res
+}
+
+define <4 x i64> @test_mask_mullo_epi64_rrkz_256(<4 x i64> %a, <4 x i64> %b, i8 %mask) {
+  ;CHECK-LABEL: test_mask_mullo_epi64_rrkz_256
+  ;CHECK: vpmullq %ymm1, %ymm0, %ymm0 {%k1} {z} ## encoding: [0x62,0xf2,0xfd,0xa9,0x40,0xc1]
+  %res = call <4 x i64> @llvm.x86.avx512.mask.pmull.q.256(<4 x i64> %a, <4 x i64> %b, <4 x i64> zeroinitializer, i8 %mask)
+  ret <4 x i64> %res
+}
+
+define <4 x i64> @test_mask_mullo_epi64_rm_256(<4 x i64> %a, <4 x i64>* %ptr_b) {
+  ;CHECK-LABEL: test_mask_mullo_epi64_rm_256
+  ;CHECK: vpmullq (%rdi), %ymm0, %ymm0    ## encoding: [0x62,0xf2,0xfd,0x28,0x40,0x07]
+  %b = load <4 x i64>, <4 x i64>* %ptr_b
+  %res = call <4 x i64> @llvm.x86.avx512.mask.pmull.q.256(<4 x i64> %a, <4 x i64> %b, <4 x i64> zeroinitializer, i8 -1)
+  ret <4 x i64> %res
+}
+
+define <4 x i64> @test_mask_mullo_epi64_rmk_256(<4 x i64> %a, <4 x i64>* %ptr_b, <4 x i64> %passThru, i8 %mask) {
+  ;CHECK-LABEL: test_mask_mullo_epi64_rmk_256
+  ;CHECK: vpmullq (%rdi), %ymm0, %ymm1 {%k1} ## encoding: [0x62,0xf2,0xfd,0x29,0x40,0x0f]
+  %b = load <4 x i64>, <4 x i64>* %ptr_b
+  %res = call <4 x i64> @llvm.x86.avx512.mask.pmull.q.256(<4 x i64> %a, <4 x i64> %b, <4 x i64> %passThru, i8 %mask)
+  ret <4 x i64> %res
+}
+
+define <4 x i64> @test_mask_mullo_epi64_rmkz_256(<4 x i64> %a, <4 x i64>* %ptr_b, i8 %mask) {
+  ;CHECK-LABEL: test_mask_mullo_epi64_rmkz_256
+  ;CHECK: vpmullq (%rdi), %ymm0, %ymm0 {%k1} {z} ## encoding: [0x62,0xf2,0xfd,0xa9,0x40,0x07]
+  %b = load <4 x i64>, <4 x i64>* %ptr_b
+  %res = call <4 x i64> @llvm.x86.avx512.mask.pmull.q.256(<4 x i64> %a, <4 x i64> %b, <4 x i64> zeroinitializer, i8 %mask)
+  ret <4 x i64> %res
+}
+
+define <4 x i64> @test_mask_mullo_epi64_rmb_256(<4 x i64> %a, i64* %ptr_b) {
+  ;CHECK-LABEL: test_mask_mullo_epi64_rmb_256
+  ;CHECK: vpmullq (%rdi){1to4}, %ymm0, %ymm0  ## encoding: [0x62,0xf2,0xfd,0x38,0x40,0x07]
+  %q = load i64, i64* %ptr_b
+  %vecinit.i = insertelement <4 x i64> undef, i64 %q, i32 0
+  %b = shufflevector <4 x i64> %vecinit.i, <4 x i64> undef, <4 x i32> zeroinitializer
+  %res = call <4 x i64> @llvm.x86.avx512.mask.pmull.q.256(<4 x i64> %a, <4 x i64> %b, <4 x i64> zeroinitializer, i8 -1)
+  ret <4 x i64> %res
+}
+
+define <4 x i64> @test_mask_mullo_epi64_rmbk_256(<4 x i64> %a, i64* %ptr_b, <4 x i64> %passThru, i8 %mask) {
+  ;CHECK-LABEL: test_mask_mullo_epi64_rmbk_256
+  ;CHECK: vpmullq (%rdi){1to4}, %ymm0, %ymm1 {%k1} ## encoding: [0x62,0xf2,0xfd,0x39,0x40,0x0f]
+  %q = load i64, i64* %ptr_b
+  %vecinit.i = insertelement <4 x i64> undef, i64 %q, i32 0
+  %b = shufflevector <4 x i64> %vecinit.i, <4 x i64> undef, <4 x i32> zeroinitializer
+  %res = call <4 x i64> @llvm.x86.avx512.mask.pmull.q.256(<4 x i64> %a, <4 x i64> %b, <4 x i64> %passThru, i8 %mask)
+  ret <4 x i64> %res
+}
+
+define <4 x i64> @test_mask_mullo_epi64_rmbkz_256(<4 x i64> %a, i64* %ptr_b, i8 %mask) {
+  ;CHECK-LABEL: test_mask_mullo_epi64_rmbkz_256
+  ;CHECK: vpmullq (%rdi){1to4}, %ymm0, %ymm0 {%k1} {z} ## encoding: [0x62,0xf2,0xfd,0xb9,0x40,0x07]
+  %q = load i64, i64* %ptr_b
+  %vecinit.i = insertelement <4 x i64> undef, i64 %q, i32 0
+  %b = shufflevector <4 x i64> %vecinit.i, <4 x i64> undef, <4 x i32> zeroinitializer
+  %res = call <4 x i64> @llvm.x86.avx512.mask.pmull.q.256(<4 x i64> %a, <4 x i64> %b, <4 x i64> zeroinitializer, i8 %mask)
+  ret <4 x i64> %res
+}
+
+declare <4 x i64> @llvm.x86.avx512.mask.pmull.q.256(<4 x i64>, <4 x i64>, <4 x i64>, i8)
+
+define <2 x i64> @test_mask_mullo_epi64_rr_128(<2 x i64> %a, <2 x i64> %b) {
+  ;CHECK-LABEL: test_mask_mullo_epi64_rr_128
+  ;CHECK: vpmullq %xmm1, %xmm0, %xmm0     ## encoding: [0x62,0xf2,0xfd,0x08,0x40,0xc1]
+  %res = call <2 x i64> @llvm.x86.avx512.mask.pmull.q.128(<2 x i64> %a, <2 x i64> %b, <2 x i64> zeroinitializer, i8 -1)
+  ret <2 x i64> %res
+}
+
+define <2 x i64> @test_mask_mullo_epi64_rrk_128(<2 x i64> %a, <2 x i64> %b, <2 x i64> %passThru, i8 %mask) {
+  ;CHECK-LABEL: test_mask_mullo_epi64_rrk_128
+  ;CHECK: vpmullq %xmm1, %xmm0, %xmm2 {%k1} ## encoding: [0x62,0xf2,0xfd,0x09,0x40,0xd1]
+  %res = call <2 x i64> @llvm.x86.avx512.mask.pmull.q.128(<2 x i64> %a, <2 x i64> %b, <2 x i64> %passThru, i8 %mask)
+  ret <2 x i64> %res
+}
+
+define <2 x i64> @test_mask_mullo_epi64_rrkz_128(<2 x i64> %a, <2 x i64> %b, i8 %mask) {
+  ;CHECK-LABEL: test_mask_mullo_epi64_rrkz_128
+  ;CHECK: vpmullq %xmm1, %xmm0, %xmm0 {%k1} {z} ## encoding: [0x62,0xf2,0xfd,0x89,0x40,0xc1]
+  %res = call <2 x i64> @llvm.x86.avx512.mask.pmull.q.128(<2 x i64> %a, <2 x i64> %b, <2 x i64> zeroinitializer, i8 %mask)
+  ret <2 x i64> %res
+}
+
+define <2 x i64> @test_mask_mullo_epi64_rm_128(<2 x i64> %a, <2 x i64>* %ptr_b) {
+  ;CHECK-LABEL: test_mask_mullo_epi64_rm_128
+  ;CHECK: vpmullq (%rdi), %xmm0, %xmm0    ## encoding: [0x62,0xf2,0xfd,0x08,0x40,0x07]
+  %b = load <2 x i64>, <2 x i64>* %ptr_b
+  %res = call <2 x i64> @llvm.x86.avx512.mask.pmull.q.128(<2 x i64> %a, <2 x i64> %b, <2 x i64> zeroinitializer, i8 -1)
+  ret <2 x i64> %res
+}
+
+define <2 x i64> @test_mask_mullo_epi64_rmk_128(<2 x i64> %a, <2 x i64>* %ptr_b, <2 x i64> %passThru, i8 %mask) {
+  ;CHECK-LABEL: test_mask_mullo_epi64_rmk_128
+  ;CHECK: vpmullq (%rdi), %xmm0, %xmm1 {%k1} ## encoding: [0x62,0xf2,0xfd,0x09,0x40,0x0f]
+  %b = load <2 x i64>, <2 x i64>* %ptr_b
+  %res = call <2 x i64> @llvm.x86.avx512.mask.pmull.q.128(<2 x i64> %a, <2 x i64> %b, <2 x i64> %passThru, i8 %mask)
+  ret <2 x i64> %res
+}
+
+define <2 x i64> @test_mask_mullo_epi64_rmkz_128(<2 x i64> %a, <2 x i64>* %ptr_b, i8 %mask) {
+  ;CHECK-LABEL: test_mask_mullo_epi64_rmkz_128
+  ;CHECK: vpmullq (%rdi), %xmm0, %xmm0 {%k1} {z} ## encoding: [0x62,0xf2,0xfd,0x89,0x40,0x07]
+  %b = load <2 x i64>, <2 x i64>* %ptr_b
+  %res = call <2 x i64> @llvm.x86.avx512.mask.pmull.q.128(<2 x i64> %a, <2 x i64> %b, <2 x i64> zeroinitializer, i8 %mask)
+  ret <2 x i64> %res
+}
+
+define <2 x i64> @test_mask_mullo_epi64_rmb_128(<2 x i64> %a, i64* %ptr_b) {
+  ;CHECK-LABEL: test_mask_mullo_epi64_rmb_128
+  ;CHECK: vpmullq (%rdi){1to2}, %xmm0, %xmm0  ## encoding: [0x62,0xf2,0xfd,0x18,0x40,0x07]
+  %q = load i64, i64* %ptr_b
+  %vecinit.i = insertelement <2 x i64> undef, i64 %q, i32 0
+  %b = shufflevector <2 x i64> %vecinit.i, <2 x i64> undef, <2 x i32> zeroinitializer
+  %res = call <2 x i64> @llvm.x86.avx512.mask.pmull.q.128(<2 x i64> %a, <2 x i64> %b, <2 x i64> zeroinitializer, i8 -1)
+  ret <2 x i64> %res
+}
+
+define <2 x i64> @test_mask_mullo_epi64_rmbk_128(<2 x i64> %a, i64* %ptr_b, <2 x i64> %passThru, i8 %mask) {
+  ;CHECK-LABEL: test_mask_mullo_epi64_rmbk_128
+  ;CHECK: vpmullq (%rdi){1to2}, %xmm0, %xmm1 {%k1} ## encoding: [0x62,0xf2,0xfd,0x19,0x40,0x0f]
+  %q = load i64, i64* %ptr_b
+  %vecinit.i = insertelement <2 x i64> undef, i64 %q, i32 0
+  %b = shufflevector <2 x i64> %vecinit.i, <2 x i64> undef, <2 x i32> zeroinitializer
+  %res = call <2 x i64> @llvm.x86.avx512.mask.pmull.q.128(<2 x i64> %a, <2 x i64> %b, <2 x i64> %passThru, i8 %mask)
+  ret <2 x i64> %res
+}
+
+define <2 x i64> @test_mask_mullo_epi64_rmbkz_128(<2 x i64> %a, i64* %ptr_b, i8 %mask) {
+  ;CHECK-LABEL: test_mask_mullo_epi64_rmbkz_128
+  ;CHECK: vpmullq (%rdi){1to2}, %xmm0, %xmm0 {%k1} {z} ## encoding: [0x62,0xf2,0xfd,0x99,0x40,0x07]
+  %q = load i64, i64* %ptr_b
+  %vecinit.i = insertelement <2 x i64> undef, i64 %q, i32 0
+  %b = shufflevector <2 x i64> %vecinit.i, <2 x i64> undef, <2 x i32> zeroinitializer
+  %res = call <2 x i64> @llvm.x86.avx512.mask.pmull.q.128(<2 x i64> %a, <2 x i64> %b, <2 x i64> zeroinitializer, i8 %mask)
+  ret <2 x i64> %res
+}
+
+declare <2 x i64> @llvm.x86.avx512.mask.pmull.q.128(<2 x i64>, <2 x i64>, <2 x i64>, i8)
+
+define <4 x float> @test_mask_andnot_ps_rr_128(<4 x float> %a, <4 x float> %b) {
+  ;CHECK-LABEL: test_mask_andnot_ps_rr_128
+  ;CHECK: vandnps %xmm1, %xmm0, %xmm0     ## encoding: [0x62,0xf1,0x7c,0x08,0x55,0xc1]
+  %res = call <4 x float> @llvm.x86.avx512.mask.andn.ps.128(<4 x float> %a, <4 x float> %b, <4 x float> zeroinitializer, i8 -1)
+  ret <4 x float> %res
+}
+
+define <4 x float> @test_mask_andnot_ps_rrk_128(<4 x float> %a, <4 x float> %b, <4 x float> %passThru, i8 %mask) {
+  ;CHECK-LABEL: test_mask_andnot_ps_rrk_128
+  ;CHECK: vandnps %xmm1, %xmm0, %xmm2 {%k1} ## encoding: [0x62,0xf1,0x7c,0x09,0x55,0xd1]
+  %res = call <4 x float> @llvm.x86.avx512.mask.andn.ps.128(<4 x float> %a, <4 x float> %b, <4 x float> %passThru, i8 %mask)
+  ret <4 x float> %res
+}
+
+define <4 x float> @test_mask_andnot_ps_rrkz_128(<4 x float> %a, <4 x float> %b, i8 %mask) {
+  ;CHECK-LABEL: test_mask_andnot_ps_rrkz_128
+  ;CHECK: vandnps %xmm1, %xmm0, %xmm0 {%k1} {z} ## encoding: [0x62,0xf1,0x7c,0x89,0x55,0xc1]
+  %res = call <4 x float> @llvm.x86.avx512.mask.andn.ps.128(<4 x float> %a, <4 x float> %b, <4 x float> zeroinitializer, i8 %mask)
+  ret <4 x float> %res
+}
+
+define <4 x float> @test_mask_andnot_ps_rm_128(<4 x float> %a, <4 x float>* %ptr_b) {
+  ;CHECK-LABEL: test_mask_andnot_ps_rm_128
+  ;CHECK: vandnps (%rdi), %xmm0, %xmm0    ## encoding: [0x62,0xf1,0x7c,0x08,0x55,0x07]
+  %b = load <4 x float>, <4 x float>* %ptr_b
+  %res = call <4 x float> @llvm.x86.avx512.mask.andn.ps.128(<4 x float> %a, <4 x float> %b, <4 x float> zeroinitializer, i8 -1)
+  ret <4 x float> %res
+}
+
+define <4 x float> @test_mask_andnot_ps_rmk_128(<4 x float> %a, <4 x float>* %ptr_b, <4 x float> %passThru, i8 %mask) {
+  ;CHECK-LABEL: test_mask_andnot_ps_rmk_128
+  ;CHECK: vandnps (%rdi), %xmm0, %xmm1 {%k1} ## encoding: [0x62,0xf1,0x7c,0x09,0x55,0x0f]
+  %b = load <4 x float>, <4 x float>* %ptr_b
+  %res = call <4 x float> @llvm.x86.avx512.mask.andn.ps.128(<4 x float> %a, <4 x float> %b, <4 x float> %passThru, i8 %mask)
+  ret <4 x float> %res
+}
+
+define <4 x float> @test_mask_andnot_ps_rmkz_128(<4 x float> %a, <4 x float>* %ptr_b, i8 %mask) {
+  ;CHECK-LABEL: test_mask_andnot_ps_rmkz_128
+  ;CHECK: vandnps (%rdi), %xmm0, %xmm0 {%k1} {z} ## encoding: [0x62,0xf1,0x7c,0x89,0x55,0x07]
+  %b = load <4 x float>, <4 x float>* %ptr_b
+  %res = call <4 x float> @llvm.x86.avx512.mask.andn.ps.128(<4 x float> %a, <4 x float> %b, <4 x float> zeroinitializer, i8 %mask)
+  ret <4 x float> %res
+}
+
+define <4 x float> @test_mask_andnot_ps_rmb_128(<4 x float> %a, float* %ptr_b) {
+  ;CHECK-LABEL: test_mask_andnot_ps_rmb_128
+  ;CHECK: vandnps (%rdi){1to4}, %xmm0, %xmm0  ## encoding: [0x62,0xf1,0x7c,0x18,0x55,0x07]
+  %q = load float, float* %ptr_b
+  %vecinit.i = insertelement <4 x float> undef, float %q, i32 0
+  %b = shufflevector <4 x float> %vecinit.i, <4 x float> undef, <4 x i32> zeroinitializer
+  %res = call <4 x float> @llvm.x86.avx512.mask.andn.ps.128(<4 x float> %a, <4 x float> %b, <4 x float> zeroinitializer, i8 -1)
+  ret <4 x float> %res
+}
+
+define <4 x float> @test_mask_andnot_ps_rmbk_128(<4 x float> %a, float* %ptr_b, <4 x float> %passThru, i8 %mask) {
+  ;CHECK-LABEL: test_mask_andnot_ps_rmbk_128
+  ;CHECK: vandnps (%rdi){1to4}, %xmm0, %xmm1 {%k1} ## encoding: [0x62,0xf1,0x7c,0x19,0x55,0x0f]
+  %q = load float, float* %ptr_b
+  %vecinit.i = insertelement <4 x float> undef, float %q, i32 0
+  %b = shufflevector <4 x float> %vecinit.i, <4 x float> undef, <4 x i32> zeroinitializer
+  %res = call <4 x float> @llvm.x86.avx512.mask.andn.ps.128(<4 x float> %a, <4 x float> %b, <4 x float> %passThru, i8 %mask)
+  ret <4 x float> %res
+}
+
+define <4 x float> @test_mask_andnot_ps_rmbkz_128(<4 x float> %a, float* %ptr_b, i8 %mask) {
+  ;CHECK-LABEL: test_mask_andnot_ps_rmbkz_128
+  ;CHECK: vandnps (%rdi){1to4}, %xmm0, %xmm0 {%k1} {z} ## encoding: [0x62,0xf1,0x7c,0x99,0x55,0x07]
+  %q = load float, float* %ptr_b
+  %vecinit.i = insertelement <4 x float> undef, float %q, i32 0
+  %b = shufflevector <4 x float> %vecinit.i, <4 x float> undef, <4 x i32> zeroinitializer
+  %res = call <4 x float> @llvm.x86.avx512.mask.andn.ps.128(<4 x float> %a, <4 x float> %b, <4 x float> zeroinitializer, i8 %mask)
+  ret <4 x float> %res
+}
+
+declare <4 x float> @llvm.x86.avx512.mask.andn.ps.128(<4 x float>, <4 x float>, <4 x float>, i8)
+
+define <8 x float> @test_mask_andnot_ps_rr_256(<8 x float> %a, <8 x float> %b) {
+  ;CHECK-LABEL: test_mask_andnot_ps_rr_256
+  ;CHECK: vandnps %ymm1, %ymm0, %ymm0     ## encoding: [0x62,0xf1,0x7c,0x28,0x55,0xc1]
+  %res = call <8 x float> @llvm.x86.avx512.mask.andn.ps.256(<8 x float> %a, <8 x float> %b, <8 x float> zeroinitializer, i8 -1)
+  ret <8 x float> %res
+}
+
+define <8 x float> @test_mask_andnot_ps_rrk_256(<8 x float> %a, <8 x float> %b, <8 x float> %passThru, i8 %mask) {
+  ;CHECK-LABEL: test_mask_andnot_ps_rrk_256
+  ;CHECK: vandnps %ymm1, %ymm0, %ymm2 {%k1} ## encoding: [0x62,0xf1,0x7c,0x29,0x55,0xd1]
+  %res = call <8 x float> @llvm.x86.avx512.mask.andn.ps.256(<8 x float> %a, <8 x float> %b, <8 x float> %passThru, i8 %mask)
+  ret <8 x float> %res
+}
+
+define <8 x float> @test_mask_andnot_ps_rrkz_256(<8 x float> %a, <8 x float> %b, i8 %mask) {
+  ;CHECK-LABEL: test_mask_andnot_ps_rrkz_256
+  ;CHECK: vandnps %ymm1, %ymm0, %ymm0 {%k1} {z} ## encoding: [0x62,0xf1,0x7c,0xa9,0x55,0xc1]
+  %res = call <8 x float> @llvm.x86.avx512.mask.andn.ps.256(<8 x float> %a, <8 x float> %b, <8 x float> zeroinitializer, i8 %mask)
+  ret <8 x float> %res
+}
+
+define <8 x float> @test_mask_andnot_ps_rm_256(<8 x float> %a, <8 x float>* %ptr_b) {
+  ;CHECK-LABEL: test_mask_andnot_ps_rm_256
+  ;CHECK: vandnps (%rdi), %ymm0, %ymm0    ## encoding: [0x62,0xf1,0x7c,0x28,0x55,0x07]
+  %b = load <8 x float>, <8 x float>* %ptr_b
+  %res = call <8 x float> @llvm.x86.avx512.mask.andn.ps.256(<8 x float> %a, <8 x float> %b, <8 x float> zeroinitializer, i8 -1)
+  ret <8 x float> %res
+}
+
+define <8 x float> @test_mask_andnot_ps_rmk_256(<8 x float> %a, <8 x float>* %ptr_b, <8 x float> %passThru, i8 %mask) {
+  ;CHECK-LABEL: test_mask_andnot_ps_rmk_256
+  ;CHECK: vandnps (%rdi), %ymm0, %ymm1 {%k1} ## encoding: [0x62,0xf1,0x7c,0x29,0x55,0x0f]
+  %b = load <8 x float>, <8 x float>* %ptr_b
+  %res = call <8 x float> @llvm.x86.avx512.mask.andn.ps.256(<8 x float> %a, <8 x float> %b, <8 x float> %passThru, i8 %mask)
+  ret <8 x float> %res
+}
+
+define <8 x float> @test_mask_andnot_ps_rmkz_256(<8 x float> %a, <8 x float>* %ptr_b, i8 %mask) {
+  ;CHECK-LABEL: test_mask_andnot_ps_rmkz_256
+  ;CHECK: vandnps (%rdi), %ymm0, %ymm0 {%k1} {z} ## encoding: [0x62,0xf1,0x7c,0xa9,0x55,0x07]
+  %b = load <8 x float>, <8 x float>* %ptr_b
+  %res = call <8 x float> @llvm.x86.avx512.mask.andn.ps.256(<8 x float> %a, <8 x float> %b, <8 x float> zeroinitializer, i8 %mask)
+  ret <8 x float> %res
+}
+
+define <8 x float> @test_mask_andnot_ps_rmb_256(<8 x float> %a, float* %ptr_b) {
+  ;CHECK-LABEL: test_mask_andnot_ps_rmb_256
+  ;CHECK: vandnps (%rdi){1to8}, %ymm0, %ymm0  ## encoding: [0x62,0xf1,0x7c,0x38,0x55,0x07]
+  %q = load float, float* %ptr_b
+  %vecinit.i = insertelement <8 x float> undef, float %q, i32 0
+  %b = shufflevector <8 x float> %vecinit.i, <8 x float> undef, <8 x i32> zeroinitializer
+  %res = call <8 x float> @llvm.x86.avx512.mask.andn.ps.256(<8 x float> %a, <8 x float> %b, <8 x float> zeroinitializer, i8 -1)
+  ret <8 x float> %res
+}
+
+define <8 x float> @test_mask_andnot_ps_rmbk_256(<8 x float> %a, float* %ptr_b, <8 x float> %passThru, i8 %mask) {
+  ;CHECK-LABEL: test_mask_andnot_ps_rmbk_256
+  ;CHECK: vandnps (%rdi){1to8}, %ymm0, %ymm1 {%k1} ## encoding: [0x62,0xf1,0x7c,0x39,0x55,0x0f]
+  %q = load float, float* %ptr_b
+  %vecinit.i = insertelement <8 x float> undef, float %q, i32 0
+  %b = shufflevector <8 x float> %vecinit.i, <8 x float> undef, <8 x i32> zeroinitializer
+  %res = call <8 x float> @llvm.x86.avx512.mask.andn.ps.256(<8 x float> %a, <8 x float> %b, <8 x float> %passThru, i8 %mask)
+  ret <8 x float> %res
+}
+
+define <8 x float> @test_mask_andnot_ps_rmbkz_256(<8 x float> %a, float* %ptr_b, i8 %mask) {
+  ;CHECK-LABEL: test_mask_andnot_ps_rmbkz_256
+  ;CHECK: vandnps (%rdi){1to8}, %ymm0, %ymm0 {%k1} {z} ## encoding: [0x62,0xf1,0x7c,0xb9,0x55,0x07]
+  %q = load float, float* %ptr_b
+  %vecinit.i = insertelement <8 x float> undef, float %q, i32 0
+  %b = shufflevector <8 x float> %vecinit.i, <8 x float> undef, <8 x i32> zeroinitializer
+  %res = call <8 x float> @llvm.x86.avx512.mask.andn.ps.256(<8 x float> %a, <8 x float> %b, <8 x float> zeroinitializer, i8 %mask)
+  ret <8 x float> %res
+}
+
+declare <8 x float> @llvm.x86.avx512.mask.andn.ps.256(<8 x float>, <8 x float>, <8 x float>, i8)
+
+define <16 x float> @test_mask_andnot_ps_rr_512(<16 x float> %a, <16 x float> %b) {
+  ;CHECK-LABEL: test_mask_andnot_ps_rr_512
+  ;CHECK: vandnps %zmm1, %zmm0, %zmm0     ## encoding: [0x62,0xf1,0x7c,0x48,0x55,0xc1]
+  %res = call <16 x float> @llvm.x86.avx512.mask.andn.ps.512(<16 x float> %a, <16 x float> %b, <16 x float> zeroinitializer, i16 -1)
+  ret <16 x float> %res
+}
+
+define <16 x float> @test_mask_andnot_ps_rrk_512(<16 x float> %a, <16 x float> %b, <16 x float> %passThru, i16 %mask) {
+  ;CHECK-LABEL: test_mask_andnot_ps_rrk_512
+  ;CHECK: vandnps %zmm1, %zmm0, %zmm2 {%k1} ## encoding: [0x62,0xf1,0x7c,0x49,0x55,0xd1]
+  %res = call <16 x float> @llvm.x86.avx512.mask.andn.ps.512(<16 x float> %a, <16 x float> %b, <16 x float> %passThru, i16 %mask)
+  ret <16 x float> %res
+}
+
+define <16 x float> @test_mask_andnot_ps_rrkz_512(<16 x float> %a, <16 x float> %b, i16 %mask) {
+  ;CHECK-LABEL: test_mask_andnot_ps_rrkz_512
+  ;CHECK: vandnps %zmm1, %zmm0, %zmm0 {%k1} {z} ## encoding: [0x62,0xf1,0x7c,0xc9,0x55,0xc1]
+  %res = call <16 x float> @llvm.x86.avx512.mask.andn.ps.512(<16 x float> %a, <16 x float> %b, <16 x float> zeroinitializer, i16 %mask)
+  ret <16 x float> %res
+}
+
+define <16 x float> @test_mask_andnot_ps_rm_512(<16 x float> %a, <16 x float>* %ptr_b) {
+  ;CHECK-LABEL: test_mask_andnot_ps_rm_512
+  ;CHECK: vandnps (%rdi), %zmm0, %zmm0    ## encoding: [0x62,0xf1,0x7c,0x48,0x55,0x07]
+  %b = load <16 x float>, <16 x float>* %ptr_b
+  %res = call <16 x float> @llvm.x86.avx512.mask.andn.ps.512(<16 x float> %a, <16 x float> %b, <16 x float> zeroinitializer, i16 -1)
+  ret <16 x float> %res
+}
+
+define <16 x float> @test_mask_andnot_ps_rmk_512(<16 x float> %a, <16 x float>* %ptr_b, <16 x float> %passThru, i16 %mask) {
+  ;CHECK-LABEL: test_mask_andnot_ps_rmk_512
+  ;CHECK: vandnps (%rdi), %zmm0, %zmm1 {%k1} ## encoding: [0x62,0xf1,0x7c,0x49,0x55,0x0f]
+  %b = load <16 x float>, <16 x float>* %ptr_b
+  %res = call <16 x float> @llvm.x86.avx512.mask.andn.ps.512(<16 x float> %a, <16 x float> %b, <16 x float> %passThru, i16 %mask)
+  ret <16 x float> %res
+}
+
+define <16 x float> @test_mask_andnot_ps_rmkz_512(<16 x float> %a, <16 x float>* %ptr_b, i16 %mask) {
+  ;CHECK-LABEL: test_mask_andnot_ps_rmkz_512
+  ;CHECK: vandnps (%rdi), %zmm0, %zmm0 {%k1} {z} ## encoding: [0x62,0xf1,0x7c,0xc9,0x55,0x07]
+  %b = load <16 x float>, <16 x float>* %ptr_b
+  %res = call <16 x float> @llvm.x86.avx512.mask.andn.ps.512(<16 x float> %a, <16 x float> %b, <16 x float> zeroinitializer, i16 %mask)
+  ret <16 x float> %res
+}
+
+define <16 x float> @test_mask_andnot_ps_rmb_512(<16 x float> %a, float* %ptr_b) {
+  ;CHECK-LABEL: test_mask_andnot_ps_rmb_512
+  ;CHECK: vandnps (%rdi){1to16}, %zmm0, %zmm0  ## encoding: [0x62,0xf1,0x7c,0x58,0x55,0x07]
+  %q = load float, float* %ptr_b
+  %vecinit.i = insertelement <16 x float> undef, float %q, i32 0
+  %b = shufflevector <16 x float> %vecinit.i, <16 x float> undef, <16 x i32> zeroinitializer
+  %res = call <16 x float> @llvm.x86.avx512.mask.andn.ps.512(<16 x float> %a, <16 x float> %b, <16 x float> zeroinitializer, i16 -1)
+  ret <16 x float> %res
+}
+
+define <16 x float> @test_mask_andnot_ps_rmbk_512(<16 x float> %a, float* %ptr_b, <16 x float> %passThru, i16 %mask) {
+  ;CHECK-LABEL: test_mask_andnot_ps_rmbk_512
+  ;CHECK: vandnps (%rdi){1to16}, %zmm0, %zmm1 {%k1} ## encoding: [0x62,0xf1,0x7c,0x59,0x55,0x0f]
+  %q = load float, float* %ptr_b
+  %vecinit.i = insertelement <16 x float> undef, float %q, i32 0
+  %b = shufflevector <16 x float> %vecinit.i, <16 x float> undef, <16 x i32> zeroinitializer
+  %res = call <16 x float> @llvm.x86.avx512.mask.andn.ps.512(<16 x float> %a, <16 x float> %b, <16 x float> %passThru, i16 %mask)
+  ret <16 x float> %res
+}
+
+define <16 x float> @test_mask_andnot_ps_rmbkz_512(<16 x float> %a, float* %ptr_b, i16 %mask) {
+  ;CHECK-LABEL: test_mask_andnot_ps_rmbkz_512
+  ;CHECK: vandnps (%rdi){1to16}, %zmm0, %zmm0 {%k1} {z} ## encoding: [0x62,0xf1,0x7c,0xd9,0x55,0x07]
+  %q = load float, float* %ptr_b
+  %vecinit.i = insertelement <16 x float> undef, float %q, i32 0
+  %b = shufflevector <16 x float> %vecinit.i, <16 x float> undef, <16 x i32> zeroinitializer
+  %res = call <16 x float> @llvm.x86.avx512.mask.andn.ps.512(<16 x float> %a, <16 x float> %b, <16 x float> zeroinitializer, i16 %mask)
+  ret <16 x float> %res
+}
+
+declare <16 x float> @llvm.x86.avx512.mask.andn.ps.512(<16 x float>, <16 x float>, <16 x float>, i16)
+
+define <4 x float> @test_mask_and_ps_rr_128(<4 x float> %a, <4 x float> %b) {
+  ;CHECK-LABEL: test_mask_and_ps_rr_128
+  ;CHECK: vandps  %xmm1, %xmm0, %xmm0     ## encoding: [0x62,0xf1,0x7c,0x08,0x54,0xc1]
+  %res = call <4 x float> @llvm.x86.avx512.mask.and.ps.128(<4 x float> %a, <4 x float> %b, <4 x float> zeroinitializer, i8 -1)
+  ret <4 x float> %res
+}
+
+define <4 x float> @test_mask_and_ps_rrk_128(<4 x float> %a, <4 x float> %b, <4 x float> %passThru, i8 %mask) {
+  ;CHECK-LABEL: test_mask_and_ps_rrk_128
+  ;CHECK: vandps %xmm1, %xmm0, %xmm2 {%k1} ## encoding: [0x62,0xf1,0x7c,0x09,0x54,0xd1]
+  %res = call <4 x float> @llvm.x86.avx512.mask.and.ps.128(<4 x float> %a, <4 x float> %b, <4 x float> %passThru, i8 %mask)
+  ret <4 x float> %res
+}
+
+define <4 x float> @test_mask_and_ps_rrkz_128(<4 x float> %a, <4 x float> %b, i8 %mask) {
+  ;CHECK-LABEL: test_mask_and_ps_rrkz_128
+  ;CHECK: vandps %xmm1, %xmm0, %xmm0 {%k1} {z} ## encoding: [0x62,0xf1,0x7c,0x89,0x54,0xc1]
+  %res = call <4 x float> @llvm.x86.avx512.mask.and.ps.128(<4 x float> %a, <4 x float> %b, <4 x float> zeroinitializer, i8 %mask)
+  ret <4 x float> %res
+}
+
+define <4 x float> @test_mask_and_ps_rm_128(<4 x float> %a, <4 x float>* %ptr_b) {
+  ;CHECK-LABEL: test_mask_and_ps_rm_128
+  ;CHECK: vandps (%rdi), %xmm0, %xmm0    ## encoding: [0x62,0xf1,0x7c,0x08,0x54,0x07]
+  %b = load <4 x float>, <4 x float>* %ptr_b
+  %res = call <4 x float> @llvm.x86.avx512.mask.and.ps.128(<4 x float> %a, <4 x float> %b, <4 x float> zeroinitializer, i8 -1)
+  ret <4 x float> %res
+}
+
+define <4 x float> @test_mask_and_ps_rmk_128(<4 x float> %a, <4 x float>* %ptr_b, <4 x float> %passThru, i8 %mask) {
+  ;CHECK-LABEL: test_mask_and_ps_rmk_128
+  ;CHECK: vandps (%rdi), %xmm0, %xmm1 {%k1} ## encoding: [0x62,0xf1,0x7c,0x09,0x54,0x0f]
+  %b = load <4 x float>, <4 x float>* %ptr_b
+  %res = call <4 x float> @llvm.x86.avx512.mask.and.ps.128(<4 x float> %a, <4 x float> %b, <4 x float> %passThru, i8 %mask)
+  ret <4 x float> %res
+}
+
+define <4 x float> @test_mask_and_ps_rmkz_128(<4 x float> %a, <4 x float>* %ptr_b, i8 %mask) {
+  ;CHECK-LABEL: test_mask_and_ps_rmkz_128
+  ;CHECK: vandps (%rdi), %xmm0, %xmm0 {%k1} {z} ## encoding: [0x62,0xf1,0x7c,0x89,0x54,0x07]
+  %b = load <4 x float>, <4 x float>* %ptr_b
+  %res = call <4 x float> @llvm.x86.avx512.mask.and.ps.128(<4 x float> %a, <4 x float> %b, <4 x float> zeroinitializer, i8 %mask)
+  ret <4 x float> %res
+}
+
+define <4 x float> @test_mask_and_ps_rmb_128(<4 x float> %a, float* %ptr_b) {
+  ;CHECK-LABEL: test_mask_and_ps_rmb_128
+  ;CHECK: vandps (%rdi){1to4}, %xmm0, %xmm0  ## encoding: [0x62,0xf1,0x7c,0x18,0x54,0x07]
+  %q = load float, float* %ptr_b
+  %vecinit.i = insertelement <4 x float> undef, float %q, i32 0
+  %b = shufflevector <4 x float> %vecinit.i, <4 x float> undef, <4 x i32> zeroinitializer
+  %res = call <4 x float> @llvm.x86.avx512.mask.and.ps.128(<4 x float> %a, <4 x float> %b, <4 x float> zeroinitializer, i8 -1)
+  ret <4 x float> %res
+}
+
+define <4 x float> @test_mask_and_ps_rmbk_128(<4 x float> %a, float* %ptr_b, <4 x float> %passThru, i8 %mask) {
+  ;CHECK-LABEL: test_mask_and_ps_rmbk_128
+  ;CHECK: vandps (%rdi){1to4}, %xmm0, %xmm1 {%k1} ## encoding: [0x62,0xf1,0x7c,0x19,0x54,0x0f]
+  %q = load float, float* %ptr_b
+  %vecinit.i = insertelement <4 x float> undef, float %q, i32 0
+  %b = shufflevector <4 x float> %vecinit.i, <4 x float> undef, <4 x i32> zeroinitializer
+  %res = call <4 x float> @llvm.x86.avx512.mask.and.ps.128(<4 x float> %a, <4 x float> %b, <4 x float> %passThru, i8 %mask)
+  ret <4 x float> %res
+}
+
+define <4 x float> @test_mask_and_ps_rmbkz_128(<4 x float> %a, float* %ptr_b, i8 %mask) {
+  ;CHECK-LABEL: test_mask_and_ps_rmbkz_128
+  ;CHECK: vandps (%rdi){1to4}, %xmm0, %xmm0 {%k1} {z} ## encoding: [0x62,0xf1,0x7c,0x99,0x54,0x07]
+  %q = load float, float* %ptr_b
+  %vecinit.i = insertelement <4 x float> undef, float %q, i32 0
+  %b = shufflevector <4 x float> %vecinit.i, <4 x float> undef, <4 x i32> zeroinitializer
+  %res = call <4 x float> @llvm.x86.avx512.mask.and.ps.128(<4 x float> %a, <4 x float> %b, <4 x float> zeroinitializer, i8 %mask)
+  ret <4 x float> %res
+}
+
+declare <4 x float> @llvm.x86.avx512.mask.and.ps.128(<4 x float>, <4 x float>, <4 x float>, i8)
+
+define <8 x float> @test_mask_and_ps_rr_256(<8 x float> %a, <8 x float> %b) {
+  ;CHECK-LABEL: test_mask_and_ps_rr_256
+  ;CHECK: vandps %ymm1, %ymm0, %ymm0     ## encoding: [0x62,0xf1,0x7c,0x28,0x54,0xc1]
+  %res = call <8 x float> @llvm.x86.avx512.mask.and.ps.256(<8 x float> %a, <8 x float> %b, <8 x float> zeroinitializer, i8 -1)
+  ret <8 x float> %res
+}
+
+define <8 x float> @test_mask_and_ps_rrk_256(<8 x float> %a, <8 x float> %b, <8 x float> %passThru, i8 %mask) {
+  ;CHECK-LABEL: test_mask_and_ps_rrk_256
+  ;CHECK: vandps %ymm1, %ymm0, %ymm2 {%k1} ## encoding: [0x62,0xf1,0x7c,0x29,0x54,0xd1]
+  %res = call <8 x float> @llvm.x86.avx512.mask.and.ps.256(<8 x float> %a, <8 x float> %b, <8 x float> %passThru, i8 %mask)
+  ret <8 x float> %res
+}
+
+define <8 x float> @test_mask_and_ps_rrkz_256(<8 x float> %a, <8 x float> %b, i8 %mask) {
+  ;CHECK-LABEL: test_mask_and_ps_rrkz_256
+  ;CHECK: vandps %ymm1, %ymm0, %ymm0 {%k1} {z} ## encoding: [0x62,0xf1,0x7c,0xa9,0x54,0xc1]
+  %res = call <8 x float> @llvm.x86.avx512.mask.and.ps.256(<8 x float> %a, <8 x float> %b, <8 x float> zeroinitializer, i8 %mask)
+  ret <8 x float> %res
+}
+
+define <8 x float> @test_mask_and_ps_rm_256(<8 x float> %a, <8 x float>* %ptr_b) {
+  ;CHECK-LABEL: test_mask_and_ps_rm_256
+  ;CHECK: vandps (%rdi), %ymm0, %ymm0    ## encoding: [0x62,0xf1,0x7c,0x28,0x54,0x07]
+  %b = load <8 x float>, <8 x float>* %ptr_b
+  %res = call <8 x float> @llvm.x86.avx512.mask.and.ps.256(<8 x float> %a, <8 x float> %b, <8 x float> zeroinitializer, i8 -1)
+  ret <8 x float> %res
+}
+
+define <8 x float> @test_mask_and_ps_rmk_256(<8 x float> %a, <8 x float>* %ptr_b, <8 x float> %passThru, i8 %mask) {
+  ;CHECK-LABEL: test_mask_and_ps_rmk_256
+  ;CHECK: vandps (%rdi), %ymm0, %ymm1 {%k1} ## encoding: [0x62,0xf1,0x7c,0x29,0x54,0x0f]
+  %b = load <8 x float>, <8 x float>* %ptr_b
+  %res = call <8 x float> @llvm.x86.avx512.mask.and.ps.256(<8 x float> %a, <8 x float> %b, <8 x float> %passThru, i8 %mask)
+  ret <8 x float> %res
+}
+
+define <8 x float> @test_mask_and_ps_rmkz_256(<8 x float> %a, <8 x float>* %ptr_b, i8 %mask) {
+  ;CHECK-LABEL: test_mask_and_ps_rmkz_256
+  ;CHECK: vandps (%rdi), %ymm0, %ymm0 {%k1} {z} ## encoding: [0x62,0xf1,0x7c,0xa9,0x54,0x07]
+  %b = load <8 x float>, <8 x float>* %ptr_b
+  %res = call <8 x float> @llvm.x86.avx512.mask.and.ps.256(<8 x float> %a, <8 x float> %b, <8 x float> zeroinitializer, i8 %mask)
+  ret <8 x float> %res
+}
+
+define <8 x float> @test_mask_and_ps_rmb_256(<8 x float> %a, float* %ptr_b) {
+  ;CHECK-LABEL: test_mask_and_ps_rmb_256
+  ;CHECK: vandps (%rdi){1to8}, %ymm0, %ymm0  ## encoding: [0x62,0xf1,0x7c,0x38,0x54,0x07]
+  %q = load float, float* %ptr_b
+  %vecinit.i = insertelement <8 x float> undef, float %q, i32 0
+  %b = shufflevector <8 x float> %vecinit.i, <8 x float> undef, <8 x i32> zeroinitializer
+  %res = call <8 x float> @llvm.x86.avx512.mask.and.ps.256(<8 x float> %a, <8 x float> %b, <8 x float> zeroinitializer, i8 -1)
+  ret <8 x float> %res
+}
+
+define <8 x float> @test_mask_and_ps_rmbk_256(<8 x float> %a, float* %ptr_b, <8 x float> %passThru, i8 %mask) {
+  ;CHECK-LABEL: test_mask_and_ps_rmbk_256
+  ;CHECK: vandps (%rdi){1to8}, %ymm0, %ymm1 {%k1} ## encoding: [0x62,0xf1,0x7c,0x39,0x54,0x0f]
+  %q = load float, float* %ptr_b
+  %vecinit.i = insertelement <8 x float> undef, float %q, i32 0
+  %b = shufflevector <8 x float> %vecinit.i, <8 x float> undef, <8 x i32> zeroinitializer
+  %res = call <8 x float> @llvm.x86.avx512.mask.and.ps.256(<8 x float> %a, <8 x float> %b, <8 x float> %passThru, i8 %mask)
+  ret <8 x float> %res
+}
+
+define <8 x float> @test_mask_and_ps_rmbkz_256(<8 x float> %a, float* %ptr_b, i8 %mask) {
+  ;CHECK-LABEL: test_mask_and_ps_rmbkz_256
+  ;CHECK: vandps (%rdi){1to8}, %ymm0, %ymm0 {%k1} {z} ## encoding: [0x62,0xf1,0x7c,0xb9,0x54,0x07]
+  %q = load float, float* %ptr_b
+  %vecinit.i = insertelement <8 x float> undef, float %q, i32 0
+  %b = shufflevector <8 x float> %vecinit.i, <8 x float> undef, <8 x i32> zeroinitializer
+  %res = call <8 x float> @llvm.x86.avx512.mask.and.ps.256(<8 x float> %a, <8 x float> %b, <8 x float> zeroinitializer, i8 %mask)
+  ret <8 x float> %res
+}
+
+declare <8 x float> @llvm.x86.avx512.mask.and.ps.256(<8 x float>, <8 x float>, <8 x float>, i8)
+
+define <16 x float> @test_mask_and_ps_rr_512(<16 x float> %a, <16 x float> %b) {
+  ;CHECK-LABEL: test_mask_and_ps_rr_512
+  ;CHECK: vandps %zmm1, %zmm0, %zmm0     ## encoding: [0x62,0xf1,0x7c,0x48,0x54,0xc1]
+  %res = call <16 x float> @llvm.x86.avx512.mask.and.ps.512(<16 x float> %a, <16 x float> %b, <16 x float> zeroinitializer, i16 -1)
+  ret <16 x float> %res
+}
+
+define <16 x float> @test_mask_and_ps_rrk_512(<16 x float> %a, <16 x float> %b, <16 x float> %passThru, i16 %mask) {
+  ;CHECK-LABEL: test_mask_and_ps_rrk_512
+  ;CHECK: vandps %zmm1, %zmm0, %zmm2 {%k1} ## encoding: [0x62,0xf1,0x7c,0x49,0x54,0xd1]
+  %res = call <16 x float> @llvm.x86.avx512.mask.and.ps.512(<16 x float> %a, <16 x float> %b, <16 x float> %passThru, i16 %mask)
+  ret <16 x float> %res
+}
+
+define <16 x float> @test_mask_and_ps_rrkz_512(<16 x float> %a, <16 x float> %b, i16 %mask) {
+  ;CHECK-LABEL: test_mask_and_ps_rrkz_512
+  ;CHECK: vandps %zmm1, %zmm0, %zmm0 {%k1} {z} ## encoding: [0x62,0xf1,0x7c,0xc9,0x54,0xc1]
+  %res = call <16 x float> @llvm.x86.avx512.mask.and.ps.512(<16 x float> %a, <16 x float> %b, <16 x float> zeroinitializer, i16 %mask)
+  ret <16 x float> %res
+}
+
+define <16 x float> @test_mask_and_ps_rm_512(<16 x float> %a, <16 x float>* %ptr_b) {
+  ;CHECK-LABEL: test_mask_and_ps_rm_512
+  ;CHECK: vandps (%rdi), %zmm0, %zmm0    ## encoding: [0x62,0xf1,0x7c,0x48,0x54,0x07]
+  %b = load <16 x float>, <16 x float>* %ptr_b
+  %res = call <16 x float> @llvm.x86.avx512.mask.and.ps.512(<16 x float> %a, <16 x float> %b, <16 x float> zeroinitializer, i16 -1)
+  ret <16 x float> %res
+}
+
+define <16 x float> @test_mask_and_ps_rmk_512(<16 x float> %a, <16 x float>* %ptr_b, <16 x float> %passThru, i16 %mask) {
+  ;CHECK-LABEL: test_mask_and_ps_rmk_512
+  ;CHECK: vandps (%rdi), %zmm0, %zmm1 {%k1} ## encoding: [0x62,0xf1,0x7c,0x49,0x54,0x0f]
+  %b = load <16 x float>, <16 x float>* %ptr_b
+  %res = call <16 x float> @llvm.x86.avx512.mask.and.ps.512(<16 x float> %a, <16 x float> %b, <16 x float> %passThru, i16 %mask)
+  ret <16 x float> %res
+}
+
+define <16 x float> @test_mask_and_ps_rmkz_512(<16 x float> %a, <16 x float>* %ptr_b, i16 %mask) {
+  ;CHECK-LABEL: test_mask_and_ps_rmkz_512
+  ;CHECK: vandps (%rdi), %zmm0, %zmm0 {%k1} {z} ## encoding: [0x62,0xf1,0x7c,0xc9,0x54,0x07]
+  %b = load <16 x float>, <16 x float>* %ptr_b
+  %res = call <16 x float> @llvm.x86.avx512.mask.and.ps.512(<16 x float> %a, <16 x float> %b, <16 x float> zeroinitializer, i16 %mask)
+  ret <16 x float> %res
+}
+
+define <16 x float> @test_mask_and_ps_rmb_512(<16 x float> %a, float* %ptr_b) {
+  ;CHECK-LABEL: test_mask_and_ps_rmb_512
+  ;CHECK: vandps (%rdi){1to16}, %zmm0, %zmm0  ## encoding: [0x62,0xf1,0x7c,0x58,0x54,0x07]
+  %q = load float, float* %ptr_b
+  %vecinit.i = insertelement <16 x float> undef, float %q, i32 0
+  %b = shufflevector <16 x float> %vecinit.i, <16 x float> undef, <16 x i32> zeroinitializer
+  %res = call <16 x float> @llvm.x86.avx512.mask.and.ps.512(<16 x float> %a, <16 x float> %b, <16 x float> zeroinitializer, i16 -1)
+  ret <16 x float> %res
+}
+
+define <16 x float> @test_mask_and_ps_rmbk_512(<16 x float> %a, float* %ptr_b, <16 x float> %passThru, i16 %mask) {
+  ;CHECK-LABEL: test_mask_and_ps_rmbk_512
+  ;CHECK: vandps (%rdi){1to16}, %zmm0, %zmm1 {%k1} ## encoding: [0x62,0xf1,0x7c,0x59,0x54,0x0f]
+  %q = load float, float* %ptr_b
+  %vecinit.i = insertelement <16 x float> undef, float %q, i32 0
+  %b = shufflevector <16 x float> %vecinit.i, <16 x float> undef, <16 x i32> zeroinitializer
+  %res = call <16 x float> @llvm.x86.avx512.mask.and.ps.512(<16 x float> %a, <16 x float> %b, <16 x float> %passThru, i16 %mask)
+  ret <16 x float> %res
+}
+
+define <16 x float> @test_mask_and_ps_rmbkz_512(<16 x float> %a, float* %ptr_b, i16 %mask) {
+  ;CHECK-LABEL: test_mask_and_ps_rmbkz_512
+  ;CHECK: vandps (%rdi){1to16}, %zmm0, %zmm0 {%k1} {z} ## encoding: [0x62,0xf1,0x7c,0xd9,0x54,0x07]
+  %q = load float, float* %ptr_b
+  %vecinit.i = insertelement <16 x float> undef, float %q, i32 0
+  %b = shufflevector <16 x float> %vecinit.i, <16 x float> undef, <16 x i32> zeroinitializer
+  %res = call <16 x float> @llvm.x86.avx512.mask.and.ps.512(<16 x float> %a, <16 x float> %b, <16 x float> zeroinitializer, i16 %mask)
+  ret <16 x float> %res
+}
+
+declare <16 x float> @llvm.x86.avx512.mask.and.ps.512(<16 x float>, <16 x float>, <16 x float>, i16)
+
+define <4 x float> @test_mask_or_ps_rr_128(<4 x float> %a, <4 x float> %b) {
+  ;CHECK-LABEL: test_mask_or_ps_rr_128
+  ;CHECK: vorps  %xmm1, %xmm0, %xmm0     ## encoding: [0x62,0xf1,0x7c,0x08,0x56,0xc1]
+  %res = call <4 x float> @llvm.x86.avx512.mask.or.ps.128(<4 x float> %a, <4 x float> %b, <4 x float> zeroinitializer, i8 -1)
+  ret <4 x float> %res
+}
+
+define <4 x float> @test_mask_or_ps_rrk_128(<4 x float> %a, <4 x float> %b, <4 x float> %passThru, i8 %mask) {
+  ;CHECK-LABEL: test_mask_or_ps_rrk_128
+  ;CHECK: vorps %xmm1, %xmm0, %xmm2 {%k1} ## encoding: [0x62,0xf1,0x7c,0x09,0x56,0xd1]
+  %res = call <4 x float> @llvm.x86.avx512.mask.or.ps.128(<4 x float> %a, <4 x float> %b, <4 x float> %passThru, i8 %mask)
+  ret <4 x float> %res
+}
+
+define <4 x float> @test_mask_or_ps_rrkz_128(<4 x float> %a, <4 x float> %b, i8 %mask) {
+  ;CHECK-LABEL: test_mask_or_ps_rrkz_128
+  ;CHECK: vorps %xmm1, %xmm0, %xmm0 {%k1} {z} ## encoding: [0x62,0xf1,0x7c,0x89,0x56,0xc1]
+  %res = call <4 x float> @llvm.x86.avx512.mask.or.ps.128(<4 x float> %a, <4 x float> %b, <4 x float> zeroinitializer, i8 %mask)
+  ret <4 x float> %res
+}
+
+define <4 x float> @test_mask_or_ps_rm_128(<4 x float> %a, <4 x float>* %ptr_b) {
+  ;CHECK-LABEL: test_mask_or_ps_rm_128
+  ;CHECK: vorps (%rdi), %xmm0, %xmm0    ## encoding: [0x62,0xf1,0x7c,0x08,0x56,0x07]
+  %b = load <4 x float>, <4 x float>* %ptr_b
+  %res = call <4 x float> @llvm.x86.avx512.mask.or.ps.128(<4 x float> %a, <4 x float> %b, <4 x float> zeroinitializer, i8 -1)
+  ret <4 x float> %res
+}
+
+define <4 x float> @test_mask_or_ps_rmk_128(<4 x float> %a, <4 x float>* %ptr_b, <4 x float> %passThru, i8 %mask) {
+  ;CHECK-LABEL: test_mask_or_ps_rmk_128
+  ;CHECK: vorps (%rdi), %xmm0, %xmm1 {%k1} ## encoding: [0x62,0xf1,0x7c,0x09,0x56,0x0f]
+  %b = load <4 x float>, <4 x float>* %ptr_b
+  %res = call <4 x float> @llvm.x86.avx512.mask.or.ps.128(<4 x float> %a, <4 x float> %b, <4 x float> %passThru, i8 %mask)
+  ret <4 x float> %res
+}
+
+define <4 x float> @test_mask_or_ps_rmkz_128(<4 x float> %a, <4 x float>* %ptr_b, i8 %mask) {
+  ;CHECK-LABEL: test_mask_or_ps_rmkz_128
+  ;CHECK: vorps (%rdi), %xmm0, %xmm0 {%k1} {z} ## encoding: [0x62,0xf1,0x7c,0x89,0x56,0x07]
+  %b = load <4 x float>, <4 x float>* %ptr_b
+  %res = call <4 x float> @llvm.x86.avx512.mask.or.ps.128(<4 x float> %a, <4 x float> %b, <4 x float> zeroinitializer, i8 %mask)
+  ret <4 x float> %res
+}
+
+define <4 x float> @test_mask_or_ps_rmb_128(<4 x float> %a, float* %ptr_b) {
+  ;CHECK-LABEL: test_mask_or_ps_rmb_128
+  ;CHECK: vorps (%rdi){1to4}, %xmm0, %xmm0  ## encoding: [0x62,0xf1,0x7c,0x18,0x56,0x07]
+  %q = load float, float* %ptr_b
+  %vecinit.i = insertelement <4 x float> undef, float %q, i32 0
+  %b = shufflevector <4 x float> %vecinit.i, <4 x float> undef, <4 x i32> zeroinitializer
+  %res = call <4 x float> @llvm.x86.avx512.mask.or.ps.128(<4 x float> %a, <4 x float> %b, <4 x float> zeroinitializer, i8 -1)
+  ret <4 x float> %res
+}
+
+define <4 x float> @test_mask_or_ps_rmbk_128(<4 x float> %a, float* %ptr_b, <4 x float> %passThru, i8 %mask) {
+  ;CHECK-LABEL: test_mask_or_ps_rmbk_128
+  ;CHECK: vorps (%rdi){1to4}, %xmm0, %xmm1 {%k1} ## encoding: [0x62,0xf1,0x7c,0x19,0x56,0x0f]
+  %q = load float, float* %ptr_b
+  %vecinit.i = insertelement <4 x float> undef, float %q, i32 0
+  %b = shufflevector <4 x float> %vecinit.i, <4 x float> undef, <4 x i32> zeroinitializer
+  %res = call <4 x float> @llvm.x86.avx512.mask.or.ps.128(<4 x float> %a, <4 x float> %b, <4 x float> %passThru, i8 %mask)
+  ret <4 x float> %res
+}
+
+define <4 x float> @test_mask_or_ps_rmbkz_128(<4 x float> %a, float* %ptr_b, i8 %mask) {
+  ;CHECK-LABEL: test_mask_or_ps_rmbkz_128
+  ;CHECK: vorps (%rdi){1to4}, %xmm0, %xmm0 {%k1} {z} ## encoding: [0x62,0xf1,0x7c,0x99,0x56,0x07]
+  %q = load float, float* %ptr_b
+  %vecinit.i = insertelement <4 x float> undef, float %q, i32 0
+  %b = shufflevector <4 x float> %vecinit.i, <4 x float> undef, <4 x i32> zeroinitializer
+  %res = call <4 x float> @llvm.x86.avx512.mask.or.ps.128(<4 x float> %a, <4 x float> %b, <4 x float> zeroinitializer, i8 %mask)
+  ret <4 x float> %res
+}
+
+declare <4 x float> @llvm.x86.avx512.mask.or.ps.128(<4 x float>, <4 x float>, <4 x float>, i8)
+
+define <8 x float> @test_mask_or_ps_rr_256(<8 x float> %a, <8 x float> %b) {
+  ;CHECK-LABEL: test_mask_or_ps_rr_256
+  ;CHECK: vorps %ymm1, %ymm0, %ymm0     ## encoding: [0x62,0xf1,0x7c,0x28,0x56,0xc1]
+  %res = call <8 x float> @llvm.x86.avx512.mask.or.ps.256(<8 x float> %a, <8 x float> %b, <8 x float> zeroinitializer, i8 -1)
+  ret <8 x float> %res
+}
+
+define <8 x float> @test_mask_or_ps_rrk_256(<8 x float> %a, <8 x float> %b, <8 x float> %passThru, i8 %mask) {
+  ;CHECK-LABEL: test_mask_or_ps_rrk_256
+  ;CHECK: vorps %ymm1, %ymm0, %ymm2 {%k1} ## encoding: [0x62,0xf1,0x7c,0x29,0x56,0xd1]
+  %res = call <8 x float> @llvm.x86.avx512.mask.or.ps.256(<8 x float> %a, <8 x float> %b, <8 x float> %passThru, i8 %mask)
+  ret <8 x float> %res
+}
+
+define <8 x float> @test_mask_or_ps_rrkz_256(<8 x float> %a, <8 x float> %b, i8 %mask) {
+  ;CHECK-LABEL: test_mask_or_ps_rrkz_256
+  ;CHECK: vorps %ymm1, %ymm0, %ymm0 {%k1} {z} ## encoding: [0x62,0xf1,0x7c,0xa9,0x56,0xc1]
+  %res = call <8 x float> @llvm.x86.avx512.mask.or.ps.256(<8 x float> %a, <8 x float> %b, <8 x float> zeroinitializer, i8 %mask)
+  ret <8 x float> %res
+}
+
+define <8 x float> @test_mask_or_ps_rm_256(<8 x float> %a, <8 x float>* %ptr_b) {
+  ;CHECK-LABEL: test_mask_or_ps_rm_256
+  ;CHECK: vorps (%rdi), %ymm0, %ymm0    ## encoding: [0x62,0xf1,0x7c,0x28,0x56,0x07]
+  %b = load <8 x float>, <8 x float>* %ptr_b
+  %res = call <8 x float> @llvm.x86.avx512.mask.or.ps.256(<8 x float> %a, <8 x float> %b, <8 x float> zeroinitializer, i8 -1)
+  ret <8 x float> %res
+}
+
+define <8 x float> @test_mask_or_ps_rmk_256(<8 x float> %a, <8 x float>* %ptr_b, <8 x float> %passThru, i8 %mask) {
+  ;CHECK-LABEL: test_mask_or_ps_rmk_256
+  ;CHECK: vorps (%rdi), %ymm0, %ymm1 {%k1} ## encoding: [0x62,0xf1,0x7c,0x29,0x56,0x0f]
+  %b = load <8 x float>, <8 x float>* %ptr_b
+  %res = call <8 x float> @llvm.x86.avx512.mask.or.ps.256(<8 x float> %a, <8 x float> %b, <8 x float> %passThru, i8 %mask)
+  ret <8 x float> %res
+}
+
+define <8 x float> @test_mask_or_ps_rmkz_256(<8 x float> %a, <8 x float>* %ptr_b, i8 %mask) {
+  ;CHECK-LABEL: test_mask_or_ps_rmkz_256
+  ;CHECK: vorps (%rdi), %ymm0, %ymm0 {%k1} {z} ## encoding: [0x62,0xf1,0x7c,0xa9,0x56,0x07]
+  %b = load <8 x float>, <8 x float>* %ptr_b
+  %res = call <8 x float> @llvm.x86.avx512.mask.or.ps.256(<8 x float> %a, <8 x float> %b, <8 x float> zeroinitializer, i8 %mask)
+  ret <8 x float> %res
+}
+
+define <8 x float> @test_mask_or_ps_rmb_256(<8 x float> %a, float* %ptr_b) {
+  ;CHECK-LABEL: test_mask_or_ps_rmb_256
+  ;CHECK: vorps (%rdi){1to8}, %ymm0, %ymm0  ## encoding: [0x62,0xf1,0x7c,0x38,0x56,0x07]
+  %q = load float, float* %ptr_b
+  %vecinit.i = insertelement <8 x float> undef, float %q, i32 0
+  %b = shufflevector <8 x float> %vecinit.i, <8 x float> undef, <8 x i32> zeroinitializer
+  %res = call <8 x float> @llvm.x86.avx512.mask.or.ps.256(<8 x float> %a, <8 x float> %b, <8 x float> zeroinitializer, i8 -1)
+  ret <8 x float> %res
+}
+
+define <8 x float> @test_mask_or_ps_rmbk_256(<8 x float> %a, float* %ptr_b, <8 x float> %passThru, i8 %mask) {
+  ;CHECK-LABEL: test_mask_or_ps_rmbk_256
+  ;CHECK: vorps (%rdi){1to8}, %ymm0, %ymm1 {%k1} ## encoding: [0x62,0xf1,0x7c,0x39,0x56,0x0f]
+  %q = load float, float* %ptr_b
+  %vecinit.i = insertelement <8 x float> undef, float %q, i32 0
+  %b = shufflevector <8 x float> %vecinit.i, <8 x float> undef, <8 x i32> zeroinitializer
+  %res = call <8 x float> @llvm.x86.avx512.mask.or.ps.256(<8 x float> %a, <8 x float> %b, <8 x float> %passThru, i8 %mask)
+  ret <8 x float> %res
+}
+
+define <8 x float> @test_mask_or_ps_rmbkz_256(<8 x float> %a, float* %ptr_b, i8 %mask) {
+  ;CHECK-LABEL: test_mask_or_ps_rmbkz_256
+  ;CHECK: vorps (%rdi){1to8}, %ymm0, %ymm0 {%k1} {z} ## encoding: [0x62,0xf1,0x7c,0xb9,0x56,0x07]
+  %q = load float, float* %ptr_b
+  %vecinit.i = insertelement <8 x float> undef, float %q, i32 0
+  %b = shufflevector <8 x float> %vecinit.i, <8 x float> undef, <8 x i32> zeroinitializer
+  %res = call <8 x float> @llvm.x86.avx512.mask.or.ps.256(<8 x float> %a, <8 x float> %b, <8 x float> zeroinitializer, i8 %mask)
+  ret <8 x float> %res
+}
+
+declare <8 x float> @llvm.x86.avx512.mask.or.ps.256(<8 x float>, <8 x float>, <8 x float>, i8)
+
+define <16 x float> @test_mask_or_ps_rr_512(<16 x float> %a, <16 x float> %b) {
+  ;CHECK-LABEL: test_mask_or_ps_rr_512
+  ;CHECK: vorps %zmm1, %zmm0, %zmm0     ## encoding: [0x62,0xf1,0x7c,0x48,0x56,0xc1]
+  %res = call <16 x float> @llvm.x86.avx512.mask.or.ps.512(<16 x float> %a, <16 x float> %b, <16 x float> zeroinitializer, i16 -1)
+  ret <16 x float> %res
+}
+
+define <16 x float> @test_mask_or_ps_rrk_512(<16 x float> %a, <16 x float> %b, <16 x float> %passThru, i16 %mask) {
+  ;CHECK-LABEL: test_mask_or_ps_rrk_512
+  ;CHECK: vorps %zmm1, %zmm0, %zmm2 {%k1} ## encoding: [0x62,0xf1,0x7c,0x49,0x56,0xd1]
+  %res = call <16 x float> @llvm.x86.avx512.mask.or.ps.512(<16 x float> %a, <16 x float> %b, <16 x float> %passThru, i16 %mask)
+  ret <16 x float> %res
+}
+
+define <16 x float> @test_mask_or_ps_rrkz_512(<16 x float> %a, <16 x float> %b, i16 %mask) {
+  ;CHECK-LABEL: test_mask_or_ps_rrkz_512
+  ;CHECK: vorps %zmm1, %zmm0, %zmm0 {%k1} {z} ## encoding: [0x62,0xf1,0x7c,0xc9,0x56,0xc1]
+  %res = call <16 x float> @llvm.x86.avx512.mask.or.ps.512(<16 x float> %a, <16 x float> %b, <16 x float> zeroinitializer, i16 %mask)
+  ret <16 x float> %res
+}
+
+define <16 x float> @test_mask_or_ps_rm_512(<16 x float> %a, <16 x float>* %ptr_b) {
+  ;CHECK-LABEL: test_mask_or_ps_rm_512
+  ;CHECK: vorps (%rdi), %zmm0, %zmm0    ## encoding: [0x62,0xf1,0x7c,0x48,0x56,0x07]
+  %b = load <16 x float>, <16 x float>* %ptr_b
+  %res = call <16 x float> @llvm.x86.avx512.mask.or.ps.512(<16 x float> %a, <16 x float> %b, <16 x float> zeroinitializer, i16 -1)
+  ret <16 x float> %res
+}
+
+define <16 x float> @test_mask_or_ps_rmk_512(<16 x float> %a, <16 x float>* %ptr_b, <16 x float> %passThru, i16 %mask) {
+  ;CHECK-LABEL: test_mask_or_ps_rmk_512
+  ;CHECK: vorps (%rdi), %zmm0, %zmm1 {%k1} ## encoding: [0x62,0xf1,0x7c,0x49,0x56,0x0f]
+  %b = load <16 x float>, <16 x float>* %ptr_b
+  %res = call <16 x float> @llvm.x86.avx512.mask.or.ps.512(<16 x float> %a, <16 x float> %b, <16 x float> %passThru, i16 %mask)
+  ret <16 x float> %res
+}
+
+define <16 x float> @test_mask_or_ps_rmkz_512(<16 x float> %a, <16 x float>* %ptr_b, i16 %mask) {
+  ;CHECK-LABEL: test_mask_or_ps_rmkz_512
+  ;CHECK: vorps (%rdi), %zmm0, %zmm0 {%k1} {z} ## encoding: [0x62,0xf1,0x7c,0xc9,0x56,0x07]
+  %b = load <16 x float>, <16 x float>* %ptr_b
+  %res = call <16 x float> @llvm.x86.avx512.mask.or.ps.512(<16 x float> %a, <16 x float> %b, <16 x float> zeroinitializer, i16 %mask)
+  ret <16 x float> %res
+}
+
+define <16 x float> @test_mask_or_ps_rmb_512(<16 x float> %a, float* %ptr_b) {
+  ;CHECK-LABEL: test_mask_or_ps_rmb_512
+  ;CHECK: vorps (%rdi){1to16}, %zmm0, %zmm0  ## encoding: [0x62,0xf1,0x7c,0x58,0x56,0x07]
+  %q = load float, float* %ptr_b
+  %vecinit.i = insertelement <16 x float> undef, float %q, i32 0
+  %b = shufflevector <16 x float> %vecinit.i, <16 x float> undef, <16 x i32> zeroinitializer
+  %res = call <16 x float> @llvm.x86.avx512.mask.or.ps.512(<16 x float> %a, <16 x float> %b, <16 x float> zeroinitializer, i16 -1)
+  ret <16 x float> %res
+}
+
+define <16 x float> @test_mask_or_ps_rmbk_512(<16 x float> %a, float* %ptr_b, <16 x float> %passThru, i16 %mask) {
+  ;CHECK-LABEL: test_mask_or_ps_rmbk_512
+  ;CHECK: vorps (%rdi){1to16}, %zmm0, %zmm1 {%k1} ## encoding: [0x62,0xf1,0x7c,0x59,0x56,0x0f]
+  %q = load float, float* %ptr_b
+  %vecinit.i = insertelement <16 x float> undef, float %q, i32 0
+  %b = shufflevector <16 x float> %vecinit.i, <16 x float> undef, <16 x i32> zeroinitializer
+  %res = call <16 x float> @llvm.x86.avx512.mask.or.ps.512(<16 x float> %a, <16 x float> %b, <16 x float> %passThru, i16 %mask)
+  ret <16 x float> %res
+}
+
+define <16 x float> @test_mask_or_ps_rmbkz_512(<16 x float> %a, float* %ptr_b, i16 %mask) {
+  ;CHECK-LABEL: test_mask_or_ps_rmbkz_512
+  ;CHECK: vorps (%rdi){1to16}, %zmm0, %zmm0 {%k1} {z} ## encoding: [0x62,0xf1,0x7c,0xd9,0x56,0x07]
+  %q = load float, float* %ptr_b
+  %vecinit.i = insertelement <16 x float> undef, float %q, i32 0
+  %b = shufflevector <16 x float> %vecinit.i, <16 x float> undef, <16 x i32> zeroinitializer
+  %res = call <16 x float> @llvm.x86.avx512.mask.or.ps.512(<16 x float> %a, <16 x float> %b, <16 x float> zeroinitializer, i16 %mask)
+  ret <16 x float> %res
+}
+
+declare <16 x float> @llvm.x86.avx512.mask.or.ps.512(<16 x float>, <16 x float>, <16 x float>, i16)
+
+define <4 x float> @test_mask_xor_ps_rr_128(<4 x float> %a, <4 x float> %b) {
+  ;CHECK-LABEL: test_mask_xor_ps_rr_128
+  ;CHECK: vxorps  %xmm1, %xmm0, %xmm0     ## encoding: [0x62,0xf1,0x7c,0x08,0x57,0xc1]
+  %res = call <4 x float> @llvm.x86.avx512.mask.xor.ps.128(<4 x float> %a, <4 x float> %b, <4 x float> zeroinitializer, i8 -1)
+  ret <4 x float> %res
+}
+
+define <4 x float> @test_mask_xor_ps_rrk_128(<4 x float> %a, <4 x float> %b, <4 x float> %passThru, i8 %mask) {
+  ;CHECK-LABEL: test_mask_xor_ps_rrk_128
+  ;CHECK: vxorps %xmm1, %xmm0, %xmm2 {%k1} ## encoding: [0x62,0xf1,0x7c,0x09,0x57,0xd1]
+  %res = call <4 x float> @llvm.x86.avx512.mask.xor.ps.128(<4 x float> %a, <4 x float> %b, <4 x float> %passThru, i8 %mask)
+  ret <4 x float> %res
+}
+
+define <4 x float> @test_mask_xor_ps_rrkz_128(<4 x float> %a, <4 x float> %b, i8 %mask) {
+  ;CHECK-LABEL: test_mask_xor_ps_rrkz_128
+  ;CHECK: vxorps %xmm1, %xmm0, %xmm0 {%k1} {z} ## encoding: [0x62,0xf1,0x7c,0x89,0x57,0xc1]
+  %res = call <4 x float> @llvm.x86.avx512.mask.xor.ps.128(<4 x float> %a, <4 x float> %b, <4 x float> zeroinitializer, i8 %mask)
+  ret <4 x float> %res
+}
+
+define <4 x float> @test_mask_xor_ps_rm_128(<4 x float> %a, <4 x float>* %ptr_b) {
+  ;CHECK-LABEL: test_mask_xor_ps_rm_128
+  ;CHECK: vxorps (%rdi), %xmm0, %xmm0    ## encoding: [0x62,0xf1,0x7c,0x08,0x57,0x07]
+  %b = load <4 x float>, <4 x float>* %ptr_b
+  %res = call <4 x float> @llvm.x86.avx512.mask.xor.ps.128(<4 x float> %a, <4 x float> %b, <4 x float> zeroinitializer, i8 -1)
+  ret <4 x float> %res
+}
+
+define <4 x float> @test_mask_xor_ps_rmk_128(<4 x float> %a, <4 x float>* %ptr_b, <4 x float> %passThru, i8 %mask) {
+  ;CHECK-LABEL: test_mask_xor_ps_rmk_128
+  ;CHECK: vxorps (%rdi), %xmm0, %xmm1 {%k1} ## encoding: [0x62,0xf1,0x7c,0x09,0x57,0x0f]
+  %b = load <4 x float>, <4 x float>* %ptr_b
+  %res = call <4 x float> @llvm.x86.avx512.mask.xor.ps.128(<4 x float> %a, <4 x float> %b, <4 x float> %passThru, i8 %mask)
+  ret <4 x float> %res
+}
+
+define <4 x float> @test_mask_xor_ps_rmkz_128(<4 x float> %a, <4 x float>* %ptr_b, i8 %mask) {
+  ;CHECK-LABEL: test_mask_xor_ps_rmkz_128
+  ;CHECK: vxorps (%rdi), %xmm0, %xmm0 {%k1} {z} ## encoding: [0x62,0xf1,0x7c,0x89,0x57,0x07]
+  %b = load <4 x float>, <4 x float>* %ptr_b
+  %res = call <4 x float> @llvm.x86.avx512.mask.xor.ps.128(<4 x float> %a, <4 x float> %b, <4 x float> zeroinitializer, i8 %mask)
+  ret <4 x float> %res
+}
+
+define <4 x float> @test_mask_xor_ps_rmb_128(<4 x float> %a, float* %ptr_b) {
+  ;CHECK-LABEL: test_mask_xor_ps_rmb_128
+  ;CHECK: vxorps (%rdi){1to4}, %xmm0, %xmm0  ## encoding: [0x62,0xf1,0x7c,0x18,0x57,0x07]
+  %q = load float, float* %ptr_b
+  %vecinit.i = insertelement <4 x float> undef, float %q, i32 0
+  %b = shufflevector <4 x float> %vecinit.i, <4 x float> undef, <4 x i32> zeroinitializer
+  %res = call <4 x float> @llvm.x86.avx512.mask.xor.ps.128(<4 x float> %a, <4 x float> %b, <4 x float> zeroinitializer, i8 -1)
+  ret <4 x float> %res
+}
+
+define <4 x float> @test_mask_xor_ps_rmbk_128(<4 x float> %a, float* %ptr_b, <4 x float> %passThru, i8 %mask) {
+  ;CHECK-LABEL: test_mask_xor_ps_rmbk_128
+  ;CHECK: vxorps (%rdi){1to4}, %xmm0, %xmm1 {%k1} ## encoding: [0x62,0xf1,0x7c,0x19,0x57,0x0f]
+  %q = load float, float* %ptr_b
+  %vecinit.i = insertelement <4 x float> undef, float %q, i32 0
+  %b = shufflevector <4 x float> %vecinit.i, <4 x float> undef, <4 x i32> zeroinitializer
+  %res = call <4 x float> @llvm.x86.avx512.mask.xor.ps.128(<4 x float> %a, <4 x float> %b, <4 x float> %passThru, i8 %mask)
+  ret <4 x float> %res
+}
+
+define <4 x float> @test_mask_xor_ps_rmbkz_128(<4 x float> %a, float* %ptr_b, i8 %mask) {
+  ;CHECK-LABEL: test_mask_xor_ps_rmbkz_128
+  ;CHECK: vxorps (%rdi){1to4}, %xmm0, %xmm0 {%k1} {z} ## encoding: [0x62,0xf1,0x7c,0x99,0x57,0x07]
+  %q = load float, float* %ptr_b
+  %vecinit.i = insertelement <4 x float> undef, float %q, i32 0
+  %b = shufflevector <4 x float> %vecinit.i, <4 x float> undef, <4 x i32> zeroinitializer
+  %res = call <4 x float> @llvm.x86.avx512.mask.xor.ps.128(<4 x float> %a, <4 x float> %b, <4 x float> zeroinitializer, i8 %mask)
+  ret <4 x float> %res
+}
+
+declare <4 x float> @llvm.x86.avx512.mask.xor.ps.128(<4 x float>, <4 x float>, <4 x float>, i8)
+
+define <8 x float> @test_mask_xor_ps_rr_256(<8 x float> %a, <8 x float> %b) {
+  ;CHECK-LABEL: test_mask_xor_ps_rr_256
+  ;CHECK: vxorps %ymm1, %ymm0, %ymm0     ## encoding: [0x62,0xf1,0x7c,0x28,0x57,0xc1]
+  %res = call <8 x float> @llvm.x86.avx512.mask.xor.ps.256(<8 x float> %a, <8 x float> %b, <8 x float> zeroinitializer, i8 -1)
+  ret <8 x float> %res
+}
+
+define <8 x float> @test_mask_xor_ps_rrk_256(<8 x float> %a, <8 x float> %b, <8 x float> %passThru, i8 %mask) {
+  ;CHECK-LABEL: test_mask_xor_ps_rrk_256
+  ;CHECK: vxorps %ymm1, %ymm0, %ymm2 {%k1} ## encoding: [0x62,0xf1,0x7c,0x29,0x57,0xd1]
+  %res = call <8 x float> @llvm.x86.avx512.mask.xor.ps.256(<8 x float> %a, <8 x float> %b, <8 x float> %passThru, i8 %mask)
+  ret <8 x float> %res
+}
+
+define <8 x float> @test_mask_xor_ps_rrkz_256(<8 x float> %a, <8 x float> %b, i8 %mask) {
+  ;CHECK-LABEL: test_mask_xor_ps_rrkz_256
+  ;CHECK: vxorps %ymm1, %ymm0, %ymm0 {%k1} {z} ## encoding: [0x62,0xf1,0x7c,0xa9,0x57,0xc1]
+  %res = call <8 x float> @llvm.x86.avx512.mask.xor.ps.256(<8 x float> %a, <8 x float> %b, <8 x float> zeroinitializer, i8 %mask)
+  ret <8 x float> %res
+}
+
+define <8 x float> @test_mask_xor_ps_rm_256(<8 x float> %a, <8 x float>* %ptr_b) {
+  ;CHECK-LABEL: test_mask_xor_ps_rm_256
+  ;CHECK: vxorps (%rdi), %ymm0, %ymm0    ## encoding: [0x62,0xf1,0x7c,0x28,0x57,0x07]
+  %b = load <8 x float>, <8 x float>* %ptr_b
+  %res = call <8 x float> @llvm.x86.avx512.mask.xor.ps.256(<8 x float> %a, <8 x float> %b, <8 x float> zeroinitializer, i8 -1)
+  ret <8 x float> %res
+}
+
+define <8 x float> @test_mask_xor_ps_rmk_256(<8 x float> %a, <8 x float>* %ptr_b, <8 x float> %passThru, i8 %mask) {
+  ;CHECK-LABEL: test_mask_xor_ps_rmk_256
+  ;CHECK: vxorps (%rdi), %ymm0, %ymm1 {%k1} ## encoding: [0x62,0xf1,0x7c,0x29,0x57,0x0f]
+  %b = load <8 x float>, <8 x float>* %ptr_b
+  %res = call <8 x float> @llvm.x86.avx512.mask.xor.ps.256(<8 x float> %a, <8 x float> %b, <8 x float> %passThru, i8 %mask)
+  ret <8 x float> %res
+}
+
+define <8 x float> @test_mask_xor_ps_rmkz_256(<8 x float> %a, <8 x float>* %ptr_b, i8 %mask) {
+  ;CHECK-LABEL: test_mask_xor_ps_rmkz_256
+  ;CHECK: vxorps (%rdi), %ymm0, %ymm0 {%k1} {z} ## encoding: [0x62,0xf1,0x7c,0xa9,0x57,0x07]
+  %b = load <8 x float>, <8 x float>* %ptr_b
+  %res = call <8 x float> @llvm.x86.avx512.mask.xor.ps.256(<8 x float> %a, <8 x float> %b, <8 x float> zeroinitializer, i8 %mask)
+  ret <8 x float> %res
+}
+
+define <8 x float> @test_mask_xor_ps_rmb_256(<8 x float> %a, float* %ptr_b) {
+  ;CHECK-LABEL: test_mask_xor_ps_rmb_256
+  ;CHECK: vxorps (%rdi){1to8}, %ymm0, %ymm0  ## encoding: [0x62,0xf1,0x7c,0x38,0x57,0x07]
+  %q = load float, float* %ptr_b
+  %vecinit.i = insertelement <8 x float> undef, float %q, i32 0
+  %b = shufflevector <8 x float> %vecinit.i, <8 x float> undef, <8 x i32> zeroinitializer
+  %res = call <8 x float> @llvm.x86.avx512.mask.xor.ps.256(<8 x float> %a, <8 x float> %b, <8 x float> zeroinitializer, i8 -1)
+  ret <8 x float> %res
+}
+
+define <8 x float> @test_mask_xor_ps_rmbk_256(<8 x float> %a, float* %ptr_b, <8 x float> %passThru, i8 %mask) {
+  ;CHECK-LABEL: test_mask_xor_ps_rmbk_256
+  ;CHECK: vxorps (%rdi){1to8}, %ymm0, %ymm1 {%k1} ## encoding: [0x62,0xf1,0x7c,0x39,0x57,0x0f]
+  %q = load float, float* %ptr_b
+  %vecinit.i = insertelement <8 x float> undef, float %q, i32 0
+  %b = shufflevector <8 x float> %vecinit.i, <8 x float> undef, <8 x i32> zeroinitializer
+  %res = call <8 x float> @llvm.x86.avx512.mask.xor.ps.256(<8 x float> %a, <8 x float> %b, <8 x float> %passThru, i8 %mask)
+  ret <8 x float> %res
+}
+
+define <8 x float> @test_mask_xor_ps_rmbkz_256(<8 x float> %a, float* %ptr_b, i8 %mask) {
+  ;CHECK-LABEL: test_mask_xor_ps_rmbkz_256
+  ;CHECK: vxorps (%rdi){1to8}, %ymm0, %ymm0 {%k1} {z} ## encoding: [0x62,0xf1,0x7c,0xb9,0x57,0x07]
+  %q = load float, float* %ptr_b
+  %vecinit.i = insertelement <8 x float> undef, float %q, i32 0
+  %b = shufflevector <8 x float> %vecinit.i, <8 x float> undef, <8 x i32> zeroinitializer
+  %res = call <8 x float> @llvm.x86.avx512.mask.xor.ps.256(<8 x float> %a, <8 x float> %b, <8 x float> zeroinitializer, i8 %mask)
+  ret <8 x float> %res
+}
+
+declare <8 x float> @llvm.x86.avx512.mask.xor.ps.256(<8 x float>, <8 x float>, <8 x float>, i8)
+
+define <16 x float> @test_mask_xor_ps_rr_512(<16 x float> %a, <16 x float> %b) {
+  ;CHECK-LABEL: test_mask_xor_ps_rr_512
+  ;CHECK: vxorps %zmm1, %zmm0, %zmm0     ## encoding: [0x62,0xf1,0x7c,0x48,0x57,0xc1]
+  %res = call <16 x float> @llvm.x86.avx512.mask.xor.ps.512(<16 x float> %a, <16 x float> %b, <16 x float> zeroinitializer, i16 -1)
+  ret <16 x float> %res
+}
+
+define <16 x float> @test_mask_xor_ps_rrk_512(<16 x float> %a, <16 x float> %b, <16 x float> %passThru, i16 %mask) {
+  ;CHECK-LABEL: test_mask_xor_ps_rrk_512
+  ;CHECK: vxorps %zmm1, %zmm0, %zmm2 {%k1} ## encoding: [0x62,0xf1,0x7c,0x49,0x57,0xd1]
+  %res = call <16 x float> @llvm.x86.avx512.mask.xor.ps.512(<16 x float> %a, <16 x float> %b, <16 x float> %passThru, i16 %mask)
+  ret <16 x float> %res
+}
+
+define <16 x float> @test_mask_xor_ps_rrkz_512(<16 x float> %a, <16 x float> %b, i16 %mask) {
+  ;CHECK-LABEL: test_mask_xor_ps_rrkz_512
+  ;CHECK: vxorps %zmm1, %zmm0, %zmm0 {%k1} {z} ## encoding: [0x62,0xf1,0x7c,0xc9,0x57,0xc1]
+  %res = call <16 x float> @llvm.x86.avx512.mask.xor.ps.512(<16 x float> %a, <16 x float> %b, <16 x float> zeroinitializer, i16 %mask)
+  ret <16 x float> %res
+}
+
+define <16 x float> @test_mask_xor_ps_rm_512(<16 x float> %a, <16 x float>* %ptr_b) {
+  ;CHECK-LABEL: test_mask_xor_ps_rm_512
+  ;CHECK: vxorps (%rdi), %zmm0, %zmm0    ## encoding: [0x62,0xf1,0x7c,0x48,0x57,0x07]
+  %b = load <16 x float>, <16 x float>* %ptr_b
+  %res = call <16 x float> @llvm.x86.avx512.mask.xor.ps.512(<16 x float> %a, <16 x float> %b, <16 x float> zeroinitializer, i16 -1)
+  ret <16 x float> %res
+}
+
+define <16 x float> @test_mask_xor_ps_rmk_512(<16 x float> %a, <16 x float>* %ptr_b, <16 x float> %passThru, i16 %mask) {
+  ;CHECK-LABEL: test_mask_xor_ps_rmk_512
+  ;CHECK: vxorps (%rdi), %zmm0, %zmm1 {%k1} ## encoding: [0x62,0xf1,0x7c,0x49,0x57,0x0f]
+  %b = load <16 x float>, <16 x float>* %ptr_b
+  %res = call <16 x float> @llvm.x86.avx512.mask.xor.ps.512(<16 x float> %a, <16 x float> %b, <16 x float> %passThru, i16 %mask)
+  ret <16 x float> %res
+}
+
+define <16 x float> @test_mask_xor_ps_rmkz_512(<16 x float> %a, <16 x float>* %ptr_b, i16 %mask) {
+  ;CHECK-LABEL: test_mask_xor_ps_rmkz_512
+  ;CHECK: vxorps (%rdi), %zmm0, %zmm0 {%k1} {z} ## encoding: [0x62,0xf1,0x7c,0xc9,0x57,0x07]
+  %b = load <16 x float>, <16 x float>* %ptr_b
+  %res = call <16 x float> @llvm.x86.avx512.mask.xor.ps.512(<16 x float> %a, <16 x float> %b, <16 x float> zeroinitializer, i16 %mask)
+  ret <16 x float> %res
+}
+
+define <16 x float> @test_mask_xor_ps_rmb_512(<16 x float> %a, float* %ptr_b) {
+  ;CHECK-LABEL: test_mask_xor_ps_rmb_512
+  ;CHECK: vxorps (%rdi){1to16}, %zmm0, %zmm0  ## encoding: [0x62,0xf1,0x7c,0x58,0x57,0x07]
+  %q = load float, float* %ptr_b
+  %vecinit.i = insertelement <16 x float> undef, float %q, i32 0
+  %b = shufflevector <16 x float> %vecinit.i, <16 x float> undef, <16 x i32> zeroinitializer
+  %res = call <16 x float> @llvm.x86.avx512.mask.xor.ps.512(<16 x float> %a, <16 x float> %b, <16 x float> zeroinitializer, i16 -1)
+  ret <16 x float> %res
+}
+
+define <16 x float> @test_mask_xor_ps_rmbk_512(<16 x float> %a, float* %ptr_b, <16 x float> %passThru, i16 %mask) {
+  ;CHECK-LABEL: test_mask_xor_ps_rmbk_512
+  ;CHECK: vxorps (%rdi){1to16}, %zmm0, %zmm1 {%k1} 
+  %q = load float, float* %ptr_b
+  %vecinit.i = insertelement <16 x float> undef, float %q, i32 0
+  %b = shufflevector <16 x float> %vecinit.i, <16 x float> undef, <16 x i32> zeroinitializer
+  %res = call <16 x float> @llvm.x86.avx512.mask.xor.ps.512(<16 x float> %a, <16 x float> %b, <16 x float> %passThru, i16 %mask)
+  ret <16 x float> %res
+}
+
+define <16 x float> @test_mask_xor_ps_rmbkz_512(<16 x float> %a, float* %ptr_b, i16 %mask) {
+  ;CHECK-LABEL: test_mask_xor_ps_rmbkz_512
+  ;CHECK: vxorps (%rdi){1to16}, %zmm0, %zmm0 {%k1} {z} 
+  %q = load float, float* %ptr_b
+  %vecinit.i = insertelement <16 x float> undef, float %q, i32 0
+  %b = shufflevector <16 x float> %vecinit.i, <16 x float> undef, <16 x i32> zeroinitializer
+  %res = call <16 x float> @llvm.x86.avx512.mask.xor.ps.512(<16 x float> %a, <16 x float> %b, <16 x float> zeroinitializer, i16 %mask)
+  ret <16 x float> %res
+}
+
+declare <16 x float> @llvm.x86.avx512.mask.xor.ps.512(<16 x float>, <16 x float>, <16 x float>, i16)
+
+declare <2 x i64> @llvm.x86.avx512.mask.cvtpd2qq.128(<2 x double>, <2 x i64>, i8)
+
+define <2 x i64>@test_int_x86_avx512_mask_cvt_pd2qq_128(<2 x double> %x0, <2 x i64> %x1, i8 %x2) {
+; CHECK-LABEL: test_int_x86_avx512_mask_cvt_pd2qq_128:
+; CHECK:       ## BB#0:
+; CHECK-NEXT:    kmovb %edi, %k1
+; CHECK-NEXT:    vcvtpd2qq %xmm0, %xmm1 {%k1}
+; CHECK-NEXT:    vcvtpd2qq %xmm0, %xmm0
+; CHECK-NEXT:    vpaddq %xmm0, %xmm1, %xmm0
+; CHECK-NEXT:    retq
+  %res = call <2 x i64> @llvm.x86.avx512.mask.cvtpd2qq.128(<2 x double> %x0, <2 x i64> %x1, i8 %x2)
+  %res1 = call <2 x i64> @llvm.x86.avx512.mask.cvtpd2qq.128(<2 x double> %x0, <2 x i64> %x1, i8 -1)
+  %res2 = add <2 x i64> %res, %res1
+  ret <2 x i64> %res2
+}
+
+declare <4 x i64> @llvm.x86.avx512.mask.cvtpd2qq.256(<4 x double>, <4 x i64>, i8)
+
+define <4 x i64>@test_int_x86_avx512_mask_cvt_pd2qq_256(<4 x double> %x0, <4 x i64> %x1, i8 %x2) {
+; CHECK-LABEL: test_int_x86_avx512_mask_cvt_pd2qq_256:
+; CHECK:       ## BB#0:
+; CHECK-NEXT:    kmovb %edi, %k1
+; CHECK-NEXT:    vcvtpd2qq %ymm0, %ymm1 {%k1}
+; CHECK-NEXT:    vcvtpd2qq %ymm0, %ymm0
+; CHECK-NEXT:    vpaddq %ymm0, %ymm1, %ymm0
+; CHECK-NEXT:    retq
+  %res = call <4 x i64> @llvm.x86.avx512.mask.cvtpd2qq.256(<4 x double> %x0, <4 x i64> %x1, i8 %x2)
+  %res1 = call <4 x i64> @llvm.x86.avx512.mask.cvtpd2qq.256(<4 x double> %x0, <4 x i64> %x1, i8 -1)
+  %res2 = add <4 x i64> %res, %res1
+  ret <4 x i64> %res2
+}
+
+declare <2 x i64> @llvm.x86.avx512.mask.cvtpd2uqq.128(<2 x double>, <2 x i64>, i8)
+
+define <2 x i64>@test_int_x86_avx512_mask_cvt_pd2uqq_128(<2 x double> %x0, <2 x i64> %x1, i8 %x2) {
+; CHECK-LABEL: test_int_x86_avx512_mask_cvt_pd2uqq_128:
+; CHECK:       ## BB#0:
+; CHECK-NEXT:    kmovb %edi, %k1
+; CHECK-NEXT:    vcvtpd2uqq %xmm0, %xmm1 {%k1}
+; CHECK-NEXT:    vcvtpd2uqq %xmm0, %xmm0
+; CHECK-NEXT:    vpaddq %xmm0, %xmm1, %xmm0
+; CHECK-NEXT:    retq
+  %res = call <2 x i64> @llvm.x86.avx512.mask.cvtpd2uqq.128(<2 x double> %x0, <2 x i64> %x1, i8 %x2)
+  %res1 = call <2 x i64> @llvm.x86.avx512.mask.cvtpd2uqq.128(<2 x double> %x0, <2 x i64> %x1, i8 -1)
+  %res2 = add <2 x i64> %res, %res1
+  ret <2 x i64> %res2
+}
+
+declare <4 x i64> @llvm.x86.avx512.mask.cvtpd2uqq.256(<4 x double>, <4 x i64>, i8)
+
+define <4 x i64>@test_int_x86_avx512_mask_cvt_pd2uqq_256(<4 x double> %x0, <4 x i64> %x1, i8 %x2) {
+; CHECK-LABEL: test_int_x86_avx512_mask_cvt_pd2uqq_256:
+; CHECK:       ## BB#0:
+; CHECK-NEXT:    kmovb %edi, %k1
+; CHECK-NEXT:    vcvtpd2uqq %ymm0, %ymm1 {%k1}
+; CHECK-NEXT:    vcvtpd2uqq %ymm0, %ymm0
+; CHECK-NEXT:    vpaddq %ymm0, %ymm1, %ymm0
+; CHECK-NEXT:    retq
+  %res = call <4 x i64> @llvm.x86.avx512.mask.cvtpd2uqq.256(<4 x double> %x0, <4 x i64> %x1, i8 %x2)
+  %res1 = call <4 x i64> @llvm.x86.avx512.mask.cvtpd2uqq.256(<4 x double> %x0, <4 x i64> %x1, i8 -1)
+  %res2 = add <4 x i64> %res, %res1
+  ret <4 x i64> %res2
+}
+
+declare <2 x i64> @llvm.x86.avx512.mask.cvtps2qq.128(<4 x float>, <2 x i64>, i8)
+
+define <2 x i64>@test_int_x86_avx512_mask_cvt_ps2qq_128(<4 x float> %x0, <2 x i64> %x1, i8 %x2) {
+; CHECK-LABEL: test_int_x86_avx512_mask_cvt_ps2qq_128:
+; CHECK:       ## BB#0:
+; CHECK-NEXT:    kmovb %edi, %k1
+; CHECK-NEXT:    vcvtps2qq %xmm0, %xmm1 {%k1}
+; CHECK-NEXT:    vcvtps2qq %xmm0, %xmm0
+; CHECK-NEXT:    vpaddq %xmm0, %xmm1, %xmm0
+; CHECK-NEXT:    retq
+  %res = call <2 x i64> @llvm.x86.avx512.mask.cvtps2qq.128(<4 x float> %x0, <2 x i64> %x1, i8 %x2)
+  %res1 = call <2 x i64> @llvm.x86.avx512.mask.cvtps2qq.128(<4 x float> %x0, <2 x i64> %x1, i8 -1)
+  %res2 = add <2 x i64> %res, %res1
+  ret <2 x i64> %res2
+}
+
+declare <4 x i64> @llvm.x86.avx512.mask.cvtps2qq.256(<4 x float>, <4 x i64>, i8)
+
+define <4 x i64>@test_int_x86_avx512_mask_cvt_ps2qq_256(<4 x float> %x0, <4 x i64> %x1, i8 %x2) {
+; CHECK-LABEL: test_int_x86_avx512_mask_cvt_ps2qq_256:
+; CHECK:       ## BB#0:
+; CHECK-NEXT:    kmovb %edi, %k1
+; CHECK-NEXT:    vcvtps2qq %xmm0, %ymm1 {%k1}
+; CHECK-NEXT:    vcvtps2qq %xmm0, %ymm0
+; CHECK-NEXT:    vpaddq %ymm0, %ymm1, %ymm0
+; CHECK-NEXT:    retq
+  %res = call <4 x i64> @llvm.x86.avx512.mask.cvtps2qq.256(<4 x float> %x0, <4 x i64> %x1, i8 %x2)
+  %res1 = call <4 x i64> @llvm.x86.avx512.mask.cvtps2qq.256(<4 x float> %x0, <4 x i64> %x1, i8 -1)
+  %res2 = add <4 x i64> %res, %res1
+  ret <4 x i64> %res2
+}
+
+declare <2 x i64> @llvm.x86.avx512.mask.cvtps2uqq.128(<4 x float>, <2 x i64>, i8)
+
+define <2 x i64>@test_int_x86_avx512_mask_cvt_ps2uqq_128(<4 x float> %x0, <2 x i64> %x1, i8 %x2) {
+; CHECK-LABEL: test_int_x86_avx512_mask_cvt_ps2uqq_128:
+; CHECK:       ## BB#0:
+; CHECK-NEXT:    kmovb %edi, %k1
+; CHECK-NEXT:    vcvtps2uqq %xmm0, %xmm1 {%k1}
+; CHECK-NEXT:    vcvtps2uqq %xmm0, %xmm0
+; CHECK-NEXT:    vpaddq %xmm0, %xmm1, %xmm0
+; CHECK-NEXT:    retq
+  %res = call <2 x i64> @llvm.x86.avx512.mask.cvtps2uqq.128(<4 x float> %x0, <2 x i64> %x1, i8 %x2)
+  %res1 = call <2 x i64> @llvm.x86.avx512.mask.cvtps2uqq.128(<4 x float> %x0, <2 x i64> %x1, i8 -1)
+  %res2 = add <2 x i64> %res, %res1
+  ret <2 x i64> %res2
+}
+
+declare <4 x i64> @llvm.x86.avx512.mask.cvtps2uqq.256(<4 x float>, <4 x i64>, i8)
+
+define <4 x i64>@test_int_x86_avx512_mask_cvt_ps2uqq_256(<4 x float> %x0, <4 x i64> %x1, i8 %x2) {
+; CHECK-LABEL: test_int_x86_avx512_mask_cvt_ps2uqq_256:
+; CHECK:       ## BB#0:
+; CHECK-NEXT:    kmovb %edi, %k1
+; CHECK-NEXT:    vcvtps2uqq %xmm0, %ymm1 {%k1}
+; CHECK-NEXT:    vcvtps2uqq %xmm0, %ymm0
+; CHECK-NEXT:    vpaddq %ymm0, %ymm1, %ymm0
+; CHECK-NEXT:    retq
+  %res = call <4 x i64> @llvm.x86.avx512.mask.cvtps2uqq.256(<4 x float> %x0, <4 x i64> %x1, i8 %x2)
+  %res1 = call <4 x i64> @llvm.x86.avx512.mask.cvtps2uqq.256(<4 x float> %x0, <4 x i64> %x1, i8 -1)
+  %res2 = add <4 x i64> %res, %res1
+  ret <4 x i64> %res2
+}
+
+declare <2 x double> @llvm.x86.avx512.mask.cvtqq2pd.128(<2 x i64>, <2 x double>, i8)
+
+define <2 x double>@test_int_x86_avx512_mask_cvt_qq2pd_128(<2 x i64> %x0, <2 x double> %x1, i8 %x2) {
+; CHECK-LABEL: test_int_x86_avx512_mask_cvt_qq2pd_128:
+; CHECK:       ## BB#0:
+; CHECK-NEXT:    kmovb %edi, %k1
+; CHECK-NEXT:    vcvtqq2pd %xmm0, %xmm1 {%k1}
+; CHECK-NEXT:    vcvtqq2pd %xmm0, %xmm0
+; CHECK-NEXT:    vaddpd %xmm0, %xmm1, %xmm0
+; CHECK-NEXT:    retq
+  %res = call <2 x double> @llvm.x86.avx512.mask.cvtqq2pd.128(<2 x i64> %x0, <2 x double> %x1, i8 %x2)
+  %res1 = call <2 x double> @llvm.x86.avx512.mask.cvtqq2pd.128(<2 x i64> %x0, <2 x double> %x1, i8 -1)
+  %res2 = fadd <2 x double> %res, %res1
+  ret <2 x double> %res2
+}
+
+declare <4 x double> @llvm.x86.avx512.mask.cvtqq2pd.256(<4 x i64>, <4 x double>, i8)
+
+define <4 x double>@test_int_x86_avx512_mask_cvt_qq2pd_256(<4 x i64> %x0, <4 x double> %x1, i8 %x2) {
+; CHECK-LABEL: test_int_x86_avx512_mask_cvt_qq2pd_256:
+; CHECK:       ## BB#0:
+; CHECK-NEXT:    kmovb %edi, %k1
+; CHECK-NEXT:    vcvtqq2pd %ymm0, %ymm1 {%k1}
+; CHECK-NEXT:    vcvtqq2pd %ymm0, %ymm0
+; CHECK-NEXT:    vaddpd %ymm0, %ymm1, %ymm0
+; CHECK-NEXT:    retq
+  %res = call <4 x double> @llvm.x86.avx512.mask.cvtqq2pd.256(<4 x i64> %x0, <4 x double> %x1, i8 %x2)
+  %res1 = call <4 x double> @llvm.x86.avx512.mask.cvtqq2pd.256(<4 x i64> %x0, <4 x double> %x1, i8 -1)
+  %res2 = fadd <4 x double> %res, %res1
+  ret <4 x double> %res2
+}
+
+declare <4 x float> @llvm.x86.avx512.mask.cvtqq2ps.128(<2 x i64>, <4 x float>, i8)
+
+define <4 x float>@test_int_x86_avx512_mask_cvt_qq2ps_128(<2 x i64> %x0, <4 x float> %x1, i8 %x2) {
+; CHECK-LABEL: test_int_x86_avx512_mask_cvt_qq2ps_128:
+; CHECK:       ## BB#0:
+; CHECK-NEXT:    kmovb %edi, %k1
+; CHECK-NEXT:    vcvtqq2ps %xmm0, %xmm1 {%k1}
+; CHECK-NEXT:    vcvtqq2ps %xmm0, %xmm0
+; CHECK-NEXT:    vaddps %xmm0, %xmm1, %xmm0
+; CHECK-NEXT:    retq
+  %res = call <4 x float> @llvm.x86.avx512.mask.cvtqq2ps.128(<2 x i64> %x0, <4 x float> %x1, i8 %x2)
+  %res1 = call <4 x float> @llvm.x86.avx512.mask.cvtqq2ps.128(<2 x i64> %x0, <4 x float> %x1, i8 -1)
+  %res2 = fadd <4 x float> %res, %res1
+  ret <4 x float> %res2
+}
+
+declare <4 x float> @llvm.x86.avx512.mask.cvtqq2ps.256(<4 x i64>, <4 x float>, i8)
+
+define <4 x float>@test_int_x86_avx512_mask_cvt_qq2ps_256(<4 x i64> %x0, <4 x float> %x1, i8 %x2) {
+; CHECK-LABEL: test_int_x86_avx512_mask_cvt_qq2ps_256:
+; CHECK:       ## BB#0:
+; CHECK-NEXT:    kmovb %edi, %k1
+; CHECK-NEXT:    vcvtqq2ps %ymm0, %xmm1 {%k1}
+; CHECK-NEXT:    vcvtqq2ps %ymm0, %xmm0
+; CHECK-NEXT:    vaddps %xmm0, %xmm1, %xmm0
+; CHECK-NEXT:    retq
+  %res = call <4 x float> @llvm.x86.avx512.mask.cvtqq2ps.256(<4 x i64> %x0, <4 x float> %x1, i8 %x2)
+  %res1 = call <4 x float> @llvm.x86.avx512.mask.cvtqq2ps.256(<4 x i64> %x0, <4 x float> %x1, i8 -1)
+  %res2 = fadd <4 x float> %res, %res1
+  ret <4 x float> %res2
+}
+
+declare <2 x i64> @llvm.x86.avx512.mask.cvttpd2qq.128(<2 x double>, <2 x i64>, i8)
+
+define <2 x i64>@test_int_x86_avx512_mask_cvtt_pd2qq_128(<2 x double> %x0, <2 x i64> %x1, i8 %x2) {
+; CHECK-LABEL: test_int_x86_avx512_mask_cvtt_pd2qq_128:
+; CHECK:       ## BB#0:
+; CHECK-NEXT:    kmovb %edi, %k1
+; CHECK-NEXT:    vcvttpd2qq %xmm0, %xmm1 {%k1}
+; CHECK-NEXT:    vcvttpd2qq %xmm0, %xmm0
+; CHECK-NEXT:    vpaddq %xmm0, %xmm1, %xmm0
+; CHECK-NEXT:    retq
+  %res = call <2 x i64> @llvm.x86.avx512.mask.cvttpd2qq.128(<2 x double> %x0, <2 x i64> %x1, i8 %x2)
+  %res1 = call <2 x i64> @llvm.x86.avx512.mask.cvttpd2qq.128(<2 x double> %x0, <2 x i64> %x1, i8 -1)
+  %res2 = add <2 x i64> %res, %res1
+  ret <2 x i64> %res2
+}
+
+declare <4 x i64> @llvm.x86.avx512.mask.cvttpd2qq.256(<4 x double>, <4 x i64>, i8)
+
+define <4 x i64>@test_int_x86_avx512_mask_cvtt_pd2qq_256(<4 x double> %x0, <4 x i64> %x1, i8 %x2) {
+; CHECK-LABEL: test_int_x86_avx512_mask_cvtt_pd2qq_256:
+; CHECK:       ## BB#0:
+; CHECK-NEXT:    kmovb %edi, %k1
+; CHECK-NEXT:    vcvttpd2qq %ymm0, %ymm1 {%k1}
+; CHECK-NEXT:    vcvttpd2qq %ymm0, %ymm0
+; CHECK-NEXT:    vpaddq %ymm0, %ymm1, %ymm0
+; CHECK-NEXT:    retq
+  %res = call <4 x i64> @llvm.x86.avx512.mask.cvttpd2qq.256(<4 x double> %x0, <4 x i64> %x1, i8 %x2)
+  %res1 = call <4 x i64> @llvm.x86.avx512.mask.cvttpd2qq.256(<4 x double> %x0, <4 x i64> %x1, i8 -1)
+  %res2 = add <4 x i64> %res, %res1
+  ret <4 x i64> %res2
+}
+
+declare <2 x i64> @llvm.x86.avx512.mask.cvttpd2uqq.128(<2 x double>, <2 x i64>, i8)
+
+define <2 x i64>@test_int_x86_avx512_mask_cvtt_pd2uqq_128(<2 x double> %x0, <2 x i64> %x1, i8 %x2) {
+; CHECK-LABEL: test_int_x86_avx512_mask_cvtt_pd2uqq_128:
+; CHECK:       ## BB#0:
+; CHECK-NEXT:    kmovb %edi, %k1
+; CHECK-NEXT:    vcvttpd2uqq %xmm0, %xmm1 {%k1}
+; CHECK-NEXT:    vcvttpd2uqq %xmm0, %xmm0
+; CHECK-NEXT:    vpaddq %xmm0, %xmm1, %xmm0
+; CHECK-NEXT:    retq
+  %res = call <2 x i64> @llvm.x86.avx512.mask.cvttpd2uqq.128(<2 x double> %x0, <2 x i64> %x1, i8 %x2)
+  %res1 = call <2 x i64> @llvm.x86.avx512.mask.cvttpd2uqq.128(<2 x double> %x0, <2 x i64> %x1, i8 -1)
+  %res2 = add <2 x i64> %res, %res1
+  ret <2 x i64> %res2
+}
+
+declare <4 x i64> @llvm.x86.avx512.mask.cvttpd2uqq.256(<4 x double>, <4 x i64>, i8)
+
+define <4 x i64>@test_int_x86_avx512_mask_cvtt_pd2uqq_256(<4 x double> %x0, <4 x i64> %x1, i8 %x2) {
+; CHECK-LABEL: test_int_x86_avx512_mask_cvtt_pd2uqq_256:
+; CHECK:       ## BB#0:
+; CHECK-NEXT:    kmovb %edi, %k1
+; CHECK-NEXT:    vcvttpd2uqq %ymm0, %ymm1 {%k1}
+; CHECK-NEXT:    vcvttpd2uqq %ymm0, %ymm0
+; CHECK-NEXT:    vpaddq %ymm0, %ymm1, %ymm0
+; CHECK-NEXT:    retq
+  %res = call <4 x i64> @llvm.x86.avx512.mask.cvttpd2uqq.256(<4 x double> %x0, <4 x i64> %x1, i8 %x2)
+  %res1 = call <4 x i64> @llvm.x86.avx512.mask.cvttpd2uqq.256(<4 x double> %x0, <4 x i64> %x1, i8 -1)
+  %res2 = add <4 x i64> %res, %res1
+  ret <4 x i64> %res2
+}
+
+declare <2 x i64> @llvm.x86.avx512.mask.cvttps2qq.128(<4 x float>, <2 x i64>, i8)
+
+define <2 x i64>@test_int_x86_avx512_mask_cvtt_ps2qq_128(<4 x float> %x0, <2 x i64> %x1, i8 %x2) {
+; CHECK-LABEL: test_int_x86_avx512_mask_cvtt_ps2qq_128:
+; CHECK:       ## BB#0:
+; CHECK-NEXT:    kmovb %edi, %k1
+; CHECK-NEXT:    vcvttps2qq %xmm0, %xmm1 {%k1}
+; CHECK-NEXT:    vcvttps2qq %xmm0, %xmm0
+; CHECK-NEXT:    vpaddq %xmm0, %xmm1, %xmm0
+; CHECK-NEXT:    retq
+  %res = call <2 x i64> @llvm.x86.avx512.mask.cvttps2qq.128(<4 x float> %x0, <2 x i64> %x1, i8 %x2)
+  %res1 = call <2 x i64> @llvm.x86.avx512.mask.cvttps2qq.128(<4 x float> %x0, <2 x i64> %x1, i8 -1)
+  %res2 = add <2 x i64> %res, %res1
+  ret <2 x i64> %res2
+}
+
+declare <4 x i64> @llvm.x86.avx512.mask.cvttps2qq.256(<4 x float>, <4 x i64>, i8)
+
+define <4 x i64>@test_int_x86_avx512_mask_cvtt_ps2qq_256(<4 x float> %x0, <4 x i64> %x1, i8 %x2) {
+; CHECK-LABEL: test_int_x86_avx512_mask_cvtt_ps2qq_256:
+; CHECK:       ## BB#0:
+; CHECK-NEXT:    kmovb %edi, %k1
+; CHECK-NEXT:    vcvttps2qq %xmm0, %ymm1 {%k1}
+; CHECK-NEXT:    vcvttps2qq %xmm0, %ymm0
+; CHECK-NEXT:    vpaddq %ymm0, %ymm1, %ymm0
+; CHECK-NEXT:    retq
+  %res = call <4 x i64> @llvm.x86.avx512.mask.cvttps2qq.256(<4 x float> %x0, <4 x i64> %x1, i8 %x2)
+  %res1 = call <4 x i64> @llvm.x86.avx512.mask.cvttps2qq.256(<4 x float> %x0, <4 x i64> %x1, i8 -1)
+  %res2 = add <4 x i64> %res, %res1
+  ret <4 x i64> %res2
+}
+
+declare <2 x double> @llvm.x86.avx512.mask.cvtuqq2pd.128(<2 x i64>, <2 x double>, i8)
+
+define <2 x double>@test_int_x86_avx512_mask_cvt_uqq2pd_128(<2 x i64> %x0, <2 x double> %x1, i8 %x2) {
+; CHECK-LABEL: test_int_x86_avx512_mask_cvt_uqq2pd_128:
+; CHECK:       ## BB#0:
+; CHECK-NEXT:    kmovb %edi, %k1
+; CHECK-NEXT:    vcvtuqq2pd %xmm0, %xmm1 {%k1}
+; CHECK-NEXT:    vcvtuqq2pd %xmm0, %xmm0
+; CHECK-NEXT:    vaddpd %xmm0, %xmm1, %xmm0
+; CHECK-NEXT:    retq
+  %res = call <2 x double> @llvm.x86.avx512.mask.cvtuqq2pd.128(<2 x i64> %x0, <2 x double> %x1, i8 %x2)
+  %res1 = call <2 x double> @llvm.x86.avx512.mask.cvtuqq2pd.128(<2 x i64> %x0, <2 x double> %x1, i8 -1)
+  %res2 = fadd <2 x double> %res, %res1
+  ret <2 x double> %res2
+}
+
+declare <4 x double> @llvm.x86.avx512.mask.cvtuqq2pd.256(<4 x i64>, <4 x double>, i8)
+
+define <4 x double>@test_int_x86_avx512_mask_cvt_uqq2pd_256(<4 x i64> %x0, <4 x double> %x1, i8 %x2) {
+; CHECK-LABEL: test_int_x86_avx512_mask_cvt_uqq2pd_256:
+; CHECK:       ## BB#0:
+; CHECK-NEXT:    kmovb %edi, %k1
+; CHECK-NEXT:    vcvtuqq2pd %ymm0, %ymm1 {%k1}
+; CHECK-NEXT:    vcvtuqq2pd %ymm0, %ymm0
+; CHECK-NEXT:    vaddpd %ymm0, %ymm1, %ymm0
+; CHECK-NEXT:    retq
+  %res = call <4 x double> @llvm.x86.avx512.mask.cvtuqq2pd.256(<4 x i64> %x0, <4 x double> %x1, i8 %x2)
+  %res1 = call <4 x double> @llvm.x86.avx512.mask.cvtuqq2pd.256(<4 x i64> %x0, <4 x double> %x1, i8 -1)
+  %res2 = fadd <4 x double> %res, %res1
+  ret <4 x double> %res2
+}
+
+declare <4 x float> @llvm.x86.avx512.mask.cvtuqq2ps.128(<2 x i64>, <4 x float>, i8)
+
+define <4 x float>@test_int_x86_avx512_mask_cvt_uqq2ps_128(<2 x i64> %x0, <4 x float> %x1, i8 %x2) {
+; CHECK-LABEL: test_int_x86_avx512_mask_cvt_uqq2ps_128:
+; CHECK:       ## BB#0:
+; CHECK-NEXT:    kmovb %edi, %k1
+; CHECK-NEXT:    vcvtuqq2ps %xmm0, %xmm1 {%k1}
+; CHECK-NEXT:    vcvtuqq2ps %xmm0, %xmm0
+; CHECK-NEXT:    vaddps %xmm0, %xmm1, %xmm0
+; CHECK-NEXT:    retq
+  %res = call <4 x float> @llvm.x86.avx512.mask.cvtuqq2ps.128(<2 x i64> %x0, <4 x float> %x1, i8 %x2)
+  %res1 = call <4 x float> @llvm.x86.avx512.mask.cvtuqq2ps.128(<2 x i64> %x0, <4 x float> %x1, i8 -1)
+  %res2 = fadd <4 x float> %res, %res1
+  ret <4 x float> %res2
+}
+
+declare <4 x float> @llvm.x86.avx512.mask.cvtuqq2ps.256(<4 x i64>, <4 x float>, i8)
+
+define <4 x float>@test_int_x86_avx512_mask_cvt_uqq2ps_256(<4 x i64> %x0, <4 x float> %x1, i8 %x2) {
+; CHECK-LABEL: test_int_x86_avx512_mask_cvt_uqq2ps_256:
+; CHECK:       ## BB#0:
+; CHECK-NEXT:    kmovb %edi, %k1
+; CHECK-NEXT:    vcvtuqq2ps %ymm0, %xmm1 {%k1}
+; CHECK-NEXT:    vcvtuqq2ps %ymm0, %xmm0
+; CHECK-NEXT:    vaddps %xmm0, %xmm1, %xmm0
+; CHECK-NEXT:    retq
+  %res = call <4 x float> @llvm.x86.avx512.mask.cvtuqq2ps.256(<4 x i64> %x0, <4 x float> %x1, i8 %x2)
+  %res1 = call <4 x float> @llvm.x86.avx512.mask.cvtuqq2ps.256(<4 x i64> %x0, <4 x float> %x1, i8 -1)
+  %res2 = fadd <4 x float> %res, %res1
+  ret <4 x float> %res2
+}
+
+declare <2 x i64> @llvm.x86.avx512.mask.cvttps2uqq.128(<4 x float>, <2 x i64>, i8)
+
+define <2 x i64>@test_int_x86_avx512_mask_cvtt_ps2uqq_128(<4 x float> %x0, <2 x i64> %x1, i8 %x2) {
+; CHECK-LABEL: test_int_x86_avx512_mask_cvtt_ps2uqq_128:
+; CHECK:       ## BB#0:
+; CHECK-NEXT:    kmovb %edi, %k1
+; CHECK-NEXT:    vcvttps2uqq %xmm0, %xmm1 {%k1}
+; CHECK-NEXT:    vcvttps2uqq %xmm0, %xmm0
+; CHECK-NEXT:    vpaddq %xmm0, %xmm1, %xmm0
+; CHECK-NEXT:    retq
+  %res = call <2 x i64> @llvm.x86.avx512.mask.cvttps2uqq.128(<4 x float> %x0, <2 x i64> %x1, i8 %x2)
+  %res1 = call <2 x i64> @llvm.x86.avx512.mask.cvttps2uqq.128(<4 x float> %x0, <2 x i64> %x1, i8 -1)
+  %res2 = add <2 x i64> %res, %res1
+  ret <2 x i64> %res2
+}
+
+declare <4 x i64> @llvm.x86.avx512.mask.cvttps2uqq.256(<4 x float>, <4 x i64>, i8)
+
+define <4 x i64>@test_int_x86_avx512_mask_cvtt_ps2uqq_256(<4 x float> %x0, <4 x i64> %x1, i8 %x2) {
+; CHECK-LABEL: test_int_x86_avx512_mask_cvtt_ps2uqq_256:
+; CHECK:       ## BB#0:
+; CHECK-NEXT:    kmovb %edi, %k1
+; CHECK-NEXT:    vcvttps2uqq %xmm0, %ymm1 {%k1}
+; CHECK-NEXT:    vcvttps2uqq %xmm0, %ymm0
+; CHECK-NEXT:    vpaddq %ymm0, %ymm1, %ymm0
+; CHECK-NEXT:    retq
+  %res = call <4 x i64> @llvm.x86.avx512.mask.cvttps2uqq.256(<4 x float> %x0, <4 x i64> %x1, i8 %x2)
+  %res1 = call <4 x i64> @llvm.x86.avx512.mask.cvttps2uqq.256(<4 x float> %x0, <4 x i64> %x1, i8 -1)
+  %res2 = add <4 x i64> %res, %res1
+  ret <4 x i64> %res2
+}
+
+declare <2 x double> @llvm.x86.avx512.mask.reduce.pd.128(<2 x double>, i32, <2 x double>, i8)
+
+; CHECK-LABEL: @test_int_x86_avx512_mask_reduce_pd_128
+; CHECK-NOT: call 
+; CHECK: kmov 
+; CHECK: vreducepd {{.*}}{%k1} 
+; CHECK: vreducepd
+define <2 x double>@test_int_x86_avx512_mask_reduce_pd_128(<2 x double> %x0, <2 x double> %x2, i8 %x3) {
+  %res = call <2 x double> @llvm.x86.avx512.mask.reduce.pd.128(<2 x double> %x0, i32 4, <2 x double> %x2, i8 %x3)
+  %res1 = call <2 x double> @llvm.x86.avx512.mask.reduce.pd.128(<2 x double> %x0, i32 8, <2 x double> %x2, i8 -1)
+  %res2 = fadd <2 x double> %res, %res1
+  ret <2 x double> %res2
+}
+
+declare <4 x double> @llvm.x86.avx512.mask.reduce.pd.256(<4 x double>, i32, <4 x double>, i8)
+
+; CHECK-LABEL: @test_int_x86_avx512_mask_reduce_pd_256
+; CHECK-NOT: call 
+; CHECK: kmov 
+; CHECK: vreducepd {{.*}}{%k1} 
+; CHECK: vreducepd
+define <4 x double>@test_int_x86_avx512_mask_reduce_pd_256(<4 x double> %x0, <4 x double> %x2, i8 %x3) {
+  %res = call <4 x double> @llvm.x86.avx512.mask.reduce.pd.256(<4 x double> %x0, i32 4, <4 x double> %x2, i8 %x3)
+  %res1 = call <4 x double> @llvm.x86.avx512.mask.reduce.pd.256(<4 x double> %x0, i32 0, <4 x double> %x2, i8 -1)
+  %res2 = fadd <4 x double> %res, %res1
+  ret <4 x double> %res2
+}
+
+declare <4 x float> @llvm.x86.avx512.mask.reduce.ps.128(<4 x float>, i32, <4 x float>, i8)
+; CHECK-LABEL: @test_int_x86_avx512_mask_reduce_ps_128
+; CHECK-NOT: call 
+; CHECK: kmov 
+; CHECK: vreduceps {{.*}}{%k1} 
+; CHECK: vreduceps
+define <4 x float>@test_int_x86_avx512_mask_reduce_ps_128(<4 x float> %x0, <4 x float> %x2, i8 %x3) {
+  %res = call <4 x float> @llvm.x86.avx512.mask.reduce.ps.128(<4 x float> %x0, i32 4, <4 x float> %x2, i8 %x3)
+  %res1 = call <4 x float> @llvm.x86.avx512.mask.reduce.ps.128(<4 x float> %x0, i32 88, <4 x float> %x2, i8 -1)
+  %res2 = fadd <4 x float> %res, %res1
+  ret <4 x float> %res2
+}
+
+declare <8 x float> @llvm.x86.avx512.mask.reduce.ps.256(<8 x float>, i32, <8 x float>, i8)
+
+; CHECK-LABEL: @test_int_x86_avx512_mask_reduce_ps_256
+; CHECK-NOT: call 
+; CHECK: kmov 
+; CHECK: vreduceps {{.*}}{%k1} 
+; CHECK: vreduceps
+define <8 x float>@test_int_x86_avx512_mask_reduce_ps_256(<8 x float> %x0, <8 x float> %x2, i8 %x3) {
+  %res = call <8 x float> @llvm.x86.avx512.mask.reduce.ps.256(<8 x float> %x0, i32 11, <8 x float> %x2, i8 %x3)
+  %res1 = call <8 x float> @llvm.x86.avx512.mask.reduce.ps.256(<8 x float> %x0, i32 11, <8 x float> %x2, i8 -1)
+  %res2 = fadd <8 x float> %res, %res1
+  ret <8 x float> %res2
+}
+
+declare <2 x double> @llvm.x86.avx512.mask.range.pd.128(<2 x double>, <2 x double>, i32, <2 x double>, i8)
+
+; CHECK-LABEL: @test_int_x86_avx512_mask_range_pd_128
+; CHECK-NOT: call 
+; CHECK: kmov 
+; CHECK: vrangepd {{.*}}{%k1} 
+; CHECK: vrangepd
+define <2 x double>@test_int_x86_avx512_mask_range_pd_128(<2 x double> %x0, <2 x double> %x1, <2 x double> %x3, i8 %x4) {
+  %res = call <2 x double> @llvm.x86.avx512.mask.range.pd.128(<2 x double> %x0, <2 x double> %x1, i32 4, <2 x double> %x3, i8 %x4)
+  %res1 = call <2 x double> @llvm.x86.avx512.mask.range.pd.128(<2 x double> %x0, <2 x double> %x1, i32 8, <2 x double> %x3, i8 -1)
+  %res2 = fadd <2 x double> %res, %res1
+  ret <2 x double> %res2
+}
+
+declare <4 x double> @llvm.x86.avx512.mask.range.pd.256(<4 x double>, <4 x double>, i32, <4 x double>, i8)
+
+; CHECK-LABEL: @test_int_x86_avx512_mask_range_pd_256
+; CHECK-NOT: call 
+; CHECK: kmov 
+; CHECK: vrangepd {{.*}}{%k1} 
+; CHECK: vrangepd
+define <4 x double>@test_int_x86_avx512_mask_range_pd_256(<4 x double> %x0, <4 x double> %x1, <4 x double> %x3, i8 %x4) {
+  %res = call <4 x double> @llvm.x86.avx512.mask.range.pd.256(<4 x double> %x0, <4 x double> %x1, i32 4, <4 x double> %x3, i8 %x4)
+  %res1 = call <4 x double> @llvm.x86.avx512.mask.range.pd.256(<4 x double> %x0, <4 x double> %x1, i32 88, <4 x double> %x3, i8 -1)
+  %res2 = fadd <4 x double> %res, %res1
+  ret <4 x double> %res2
+}
+
+declare <4 x float> @llvm.x86.avx512.mask.range.ps.128(<4 x float>, <4 x float>, i32, <4 x float>, i8)
+
+; CHECK-LABEL: @test_int_x86_avx512_mask_range_ps_128
+; CHECK-NOT: call 
+; CHECK: kmov 
+; CHECK: vrangeps {{.*}}{%k1} 
+; CHECK: vrangeps
+define <4 x float>@test_int_x86_avx512_mask_range_ps_128(<4 x float> %x0, <4 x float> %x1, <4 x float> %x3, i8 %x4) {
+  %res = call <4 x float> @llvm.x86.avx512.mask.range.ps.128(<4 x float> %x0, <4 x float> %x1, i32 4, <4 x float> %x3, i8 %x4)
+  %res1 = call <4 x float> @llvm.x86.avx512.mask.range.ps.128(<4 x float> %x0, <4 x float> %x1, i32 88, <4 x float> %x3, i8 -1)
+  %res2 = fadd <4 x float> %res, %res1
+  ret <4 x float> %res2
+}
+
+declare <8 x float> @llvm.x86.avx512.mask.range.ps.256(<8 x float>, <8 x float>, i32, <8 x float>, i8)
+
+; CHECK-LABEL: @test_int_x86_avx512_mask_range_ps_256
+; CHECK-NOT: call 
+; CHECK: kmov 
+; CHECK: vrangeps {{.*}}{%k1} 
+; CHECK: vrangeps
+define <8 x float>@test_int_x86_avx512_mask_range_ps_256(<8 x float> %x0, <8 x float> %x1, <8 x float> %x3, i8 %x4) {
+  %res = call <8 x float> @llvm.x86.avx512.mask.range.ps.256(<8 x float> %x0, <8 x float> %x1, i32 4, <8 x float> %x3, i8 %x4)
+  %res1 = call <8 x float> @llvm.x86.avx512.mask.range.ps.256(<8 x float> %x0, <8 x float> %x1, i32 88, <8 x float> %x3, i8 -1)
+  %res2 = fadd <8 x float> %res, %res1
+  ret <8 x float> %res2
+}
diff --git a/test/CodeGen/X86/avx512er-intrinsics.ll b/test/CodeGen/X86/avx512er-intrinsics.ll
index dcde9c4153e..827a56d76ae 100644
--- a/test/CodeGen/X86/avx512er-intrinsics.ll
+++ b/test/CodeGen/X86/avx512er-intrinsics.ll
@@ -1,14 +1,14 @@
 ; RUN: llc < %s -mtriple=x86_64-pc-linux -mcpu=knl --show-mc-encoding| FileCheck %s
 
 define <16 x float> @test_rsqrt28_ps(<16 x float> %a0) {
-  ; CHECK: vrsqrt28ps %zmm0, %zmm0 {sae} # encoding: [0x62,0xf2,0x7d,0x18,0xcc,0xc0]
+  ; CHECK: vrsqrt28ps {sae}, %zmm0, %zmm0 # encoding: [0x62,0xf2,0x7d,0x18,0xcc,0xc0]
   %res = call <16 x float> @llvm.x86.avx512.rsqrt28.ps(<16 x float> %a0, <16 x float> zeroinitializer, i16 -1, i32 8)
   ret <16 x float> %res
 }
 
 define <16 x float> @test1_rsqrt28_ps(<16 x float> %a0, <16 x float> %a1) {
   ; CHECK: kmovw
-  ; CHECK: vrsqrt28ps %zmm0, %zmm1 {%k1}{sae} # encoding: [0x62,0xf2,0x7d,0x19,0xcc,0xc8]
+  ; CHECK: vrsqrt28ps {sae}, %zmm0, %zmm1 {%k1} # encoding: [0x62,0xf2,0x7d,0x19,0xcc,0xc8]
   %res = call <16 x float> @llvm.x86.avx512.rsqrt28.ps(<16 x float> %a0, <16 x float> %a1, i16 6, i32 8)
   ret <16 x float> %res
 }
@@ -27,7 +27,7 @@ define <16 x float> @test3_rsqrt28_ps(<16 x float> %a0) {
 }
 
 define <16 x float> @test4_rsqrt28_ps(<16 x float> %a0) {
-  ; CHECK: vrsqrt28ps %zmm0, %zmm0 {%k1} {z}{sae} # encoding: [0x62,0xf2,0x7d,0x99,0xcc,0xc0]
+  ; CHECK: vrsqrt28ps {sae}, %zmm0, %zmm0 {%k1} {z} # encoding: [0x62,0xf2,0x7d,0x99,0xcc,0xc0]
   %res = call <16 x float> @llvm.x86.avx512.rsqrt28.ps(<16 x float> %a0, <16 x float> undef, i16 6, i32 8) 
   ret <16 x float> %res
 }
@@ -36,61 +36,61 @@ define <16 x float> @test4_rsqrt28_ps(<16 x float> %a0) {
 declare <16 x float> @llvm.x86.avx512.rsqrt28.ps(<16 x float>, <16 x float>, i16, i32) nounwind readnone
 
 define <16 x float> @test_rcp28_ps_512(<16 x float> %a0) {
-  ; CHECK: vrcp28ps %zmm0, %zmm0 {sae} # encoding: [0x62,0xf2,0x7d,0x18,0xca,0xc0]
+  ; CHECK: vrcp28ps {sae}, %zmm0, %zmm0 # encoding: [0x62,0xf2,0x7d,0x18,0xca,0xc0]
   %res = call <16 x float> @llvm.x86.avx512.rcp28.ps(<16 x float> %a0, <16 x float> zeroinitializer, i16 -1, i32 8)
   ret <16 x float> %res
 }
 declare <16 x float> @llvm.x86.avx512.rcp28.ps(<16 x float>, <16 x float>, i16, i32) nounwind readnone
 
 define <8 x double> @test_rcp28_pd_512(<8 x double> %a0) {
-  ; CHECK: vrcp28pd %zmm0, %zmm0 {sae} # encoding: [0x62,0xf2,0xfd,0x18,0xca,0xc0]
+  ; CHECK: vrcp28pd {sae}, %zmm0, %zmm0 # encoding: [0x62,0xf2,0xfd,0x18,0xca,0xc0]
   %res = call <8 x double> @llvm.x86.avx512.rcp28.pd(<8 x double> %a0, <8 x double> zeroinitializer, i8 -1, i32 8) 
   ret <8 x double> %res
 }
 declare <8 x double> @llvm.x86.avx512.rcp28.pd(<8 x double>, <8 x double>, i8, i32) nounwind readnone
 
 define <16 x float> @test_exp2_ps_512(<16 x float> %a0) {
-  ; CHECK: vexp2ps %zmm0, %zmm0 {sae}      # encoding: [0x62,0xf2,0x7d,0x18,0xc8,0xc0]
+  ; CHECK: vexp2ps {sae}, %zmm0, %zmm0     # encoding: [0x62,0xf2,0x7d,0x18,0xc8,0xc0]
   %res = call <16 x float> @llvm.x86.avx512.exp2.ps(<16 x float> %a0, <16 x float> zeroinitializer, i16 -1, i32 8)
   ret <16 x float> %res
 }
 declare <16 x float> @llvm.x86.avx512.exp2.ps(<16 x float>, <16 x float>, i16, i32) nounwind readnone
 
 define <8 x double> @test_exp2_pd_512(<8 x double> %a0) {
-  ; CHECK: vexp2pd %zmm0, %zmm0 {sae}      # encoding: [0x62,0xf2,0xfd,0x18,0xc8,0xc0]
+  ; CHECK: vexp2pd {sae}, %zmm0, %zmm0      # encoding: [0x62,0xf2,0xfd,0x18,0xc8,0xc0]
   %res = call <8 x double> @llvm.x86.avx512.exp2.pd(<8 x double> %a0, <8 x double> zeroinitializer, i8 -1, i32 8)
   ret <8 x double> %res
 }
 declare <8 x double> @llvm.x86.avx512.exp2.pd(<8 x double>, <8 x double>, i8, i32) nounwind readnone
 
 define <4 x float> @test_rsqrt28_ss(<4 x float> %a0) {
-  ; CHECK: vrsqrt28ss %xmm0, %xmm0, %xmm0 {sae} # encoding: [0x62,0xf2,0x7d,0x18,0xcd,0xc0]
+  ; CHECK: vrsqrt28ss {sae}, %xmm0, %xmm0, %xmm0 # encoding: [0x62,0xf2,0x7d,0x18,0xcd,0xc0]
   %res = call <4 x float> @llvm.x86.avx512.rsqrt28.ss(<4 x float> %a0, <4 x float> %a0, <4 x float> zeroinitializer, i8 -1, i32 8) ; <<4 x float>> [#uses=1]
   ret <4 x float> %res
 }
 declare <4 x float> @llvm.x86.avx512.rsqrt28.ss(<4 x float>, <4 x float>, <4 x float>, i8, i32) nounwind readnone
 
 define <4 x float> @test_rcp28_ss(<4 x float> %a0) {
-  ; CHECK: vrcp28ss %xmm0, %xmm0, %xmm0 {sae} # encoding: [0x62,0xf2,0x7d,0x18,0xcb,0xc0]
+  ; CHECK: vrcp28ss {sae}, %xmm0, %xmm0, %xmm0 # encoding: [0x62,0xf2,0x7d,0x18,0xcb,0xc0]
   %res = call <4 x float> @llvm.x86.avx512.rcp28.ss(<4 x float> %a0, <4 x float> %a0, <4 x float> zeroinitializer, i8 -1, i32 8) ; <<4 x float>> [#uses=1]
   ret <4 x float> %res
 }
 declare <4 x float> @llvm.x86.avx512.rcp28.ss(<4 x float>, <4 x float>, <4 x float>, i8, i32) nounwind readnone
 
 define <4 x float> @test_rsqrt28_ss_maskz(<4 x float> %a0) {
-  ; CHECK: vrsqrt28ss %xmm0, %xmm0, %xmm0 {%k1} {z}{sae} # encoding: [0x62,0xf2,0x7d,0x99,0xcd,0xc0]
+  ; CHECK: vrsqrt28ss {sae}, %xmm0, %xmm0, %xmm0 {%k1} {z} # encoding: [0x62,0xf2,0x7d,0x99,0xcd,0xc0]
   %res = call <4 x float> @llvm.x86.avx512.rsqrt28.ss(<4 x float> %a0, <4 x float> %a0, <4 x float> zeroinitializer, i8 7, i32 8) ; 
   ret <4 x float> %res
 }
 
 define <4 x float> @test_rsqrt28_ss_mask(<4 x float> %a0, <4 x float> %b0, <4 x float> %c0) {
-  ; CHECK: vrsqrt28ss %xmm1, %xmm0, %xmm2 {%k1}{sae} # encoding: [0x62,0xf2,0x7d,0x19,0xcd,0xd1]
+  ; CHECK: vrsqrt28ss {sae}, %xmm1, %xmm0, %xmm2 {%k1} # encoding: [0x62,0xf2,0x7d,0x19,0xcd,0xd1]
   %res = call <4 x float> @llvm.x86.avx512.rsqrt28.ss(<4 x float> %a0, <4 x float> %b0, <4 x float> %c0, i8 7, i32 8) ;
   ret <4 x float> %res
 }
 
 define <2 x double> @test_rsqrt28_sd_maskz(<2 x double> %a0) {
-  ; CHECK: vrsqrt28sd %xmm0, %xmm0, %xmm0 {%k1} {z}{sae} # encoding: [0x62,0xf2,0xfd,0x99,0xcd,0xc0]
+  ; CHECK: vrsqrt28sd {sae}, %xmm0, %xmm0, %xmm0 {%k1} {z} # encoding: [0x62,0xf2,0xfd,0x99,0xcd,0xc0]
   %res = call <2 x double> @llvm.x86.avx512.rsqrt28.sd(<2 x double> %a0, <2 x double> %a0, <2 x double> zeroinitializer, i8 7, i32 8) ; 
   ret <2 x double> %res
 }
diff --git a/test/CodeGen/X86/avx512vl-intrinsics.ll b/test/CodeGen/X86/avx512vl-intrinsics.ll
index 38a7e7afa5c..396b47784a8 100644
--- a/test/CodeGen/X86/avx512vl-intrinsics.ll
+++ b/test/CodeGen/X86/avx512vl-intrinsics.ll
@@ -69,28 +69,28 @@ declare i8 @llvm.x86.avx512.mask.pcmpgt.q.256(<4 x i64>, <4 x i64>, i8)
 define <8 x i8> @test_cmp_d_256(<8 x i32> %a0, <8 x i32> %a1) {
 ; CHECK-LABEL: test_cmp_d_256
 ; CHECK: vpcmpeqd %ymm1, %ymm0, %k0 ##
-  %res0 = call i8 @llvm.x86.avx512.mask.cmp.d.256(<8 x i32> %a0, <8 x i32> %a1, i8 0, i8 -1)
+  %res0 = call i8 @llvm.x86.avx512.mask.cmp.d.256(<8 x i32> %a0, <8 x i32> %a1, i32 0, i8 -1)
   %vec0 = insertelement <8 x i8> undef, i8 %res0, i32 0
 ; CHECK: vpcmpltd %ymm1, %ymm0, %k0 ##
-  %res1 = call i8 @llvm.x86.avx512.mask.cmp.d.256(<8 x i32> %a0, <8 x i32> %a1, i8 1, i8 -1)
+  %res1 = call i8 @llvm.x86.avx512.mask.cmp.d.256(<8 x i32> %a0, <8 x i32> %a1, i32 1, i8 -1)
   %vec1 = insertelement <8 x i8> %vec0, i8 %res1, i32 1
 ; CHECK: vpcmpled %ymm1, %ymm0, %k0 ##
-  %res2 = call i8 @llvm.x86.avx512.mask.cmp.d.256(<8 x i32> %a0, <8 x i32> %a1, i8 2, i8 -1)
+  %res2 = call i8 @llvm.x86.avx512.mask.cmp.d.256(<8 x i32> %a0, <8 x i32> %a1, i32 2, i8 -1)
   %vec2 = insertelement <8 x i8> %vec1, i8 %res2, i32 2
 ; CHECK: vpcmpunordd %ymm1, %ymm0, %k0 ##
-  %res3 = call i8 @llvm.x86.avx512.mask.cmp.d.256(<8 x i32> %a0, <8 x i32> %a1, i8 3, i8 -1)
+  %res3 = call i8 @llvm.x86.avx512.mask.cmp.d.256(<8 x i32> %a0, <8 x i32> %a1, i32 3, i8 -1)
   %vec3 = insertelement <8 x i8> %vec2, i8 %res3, i32 3
 ; CHECK: vpcmpneqd %ymm1, %ymm0, %k0 ##
-  %res4 = call i8 @llvm.x86.avx512.mask.cmp.d.256(<8 x i32> %a0, <8 x i32> %a1, i8 4, i8 -1)
+  %res4 = call i8 @llvm.x86.avx512.mask.cmp.d.256(<8 x i32> %a0, <8 x i32> %a1, i32 4, i8 -1)
   %vec4 = insertelement <8 x i8> %vec3, i8 %res4, i32 4
 ; CHECK: vpcmpnltd %ymm1, %ymm0, %k0 ##
-  %res5 = call i8 @llvm.x86.avx512.mask.cmp.d.256(<8 x i32> %a0, <8 x i32> %a1, i8 5, i8 -1)
+  %res5 = call i8 @llvm.x86.avx512.mask.cmp.d.256(<8 x i32> %a0, <8 x i32> %a1, i32 5, i8 -1)
   %vec5 = insertelement <8 x i8> %vec4, i8 %res5, i32 5
 ; CHECK: vpcmpnled %ymm1, %ymm0, %k0 ##
-  %res6 = call i8 @llvm.x86.avx512.mask.cmp.d.256(<8 x i32> %a0, <8 x i32> %a1, i8 6, i8 -1)
+  %res6 = call i8 @llvm.x86.avx512.mask.cmp.d.256(<8 x i32> %a0, <8 x i32> %a1, i32 6, i8 -1)
   %vec6 = insertelement <8 x i8> %vec5, i8 %res6, i32 6
 ; CHECK: vpcmpordd %ymm1, %ymm0, %k0 ##
-  %res7 = call i8 @llvm.x86.avx512.mask.cmp.d.256(<8 x i32> %a0, <8 x i32> %a1, i8 7, i8 -1)
+  %res7 = call i8 @llvm.x86.avx512.mask.cmp.d.256(<8 x i32> %a0, <8 x i32> %a1, i32 7, i8 -1)
   %vec7 = insertelement <8 x i8> %vec6, i8 %res7, i32 7
   ret <8 x i8> %vec7
 }
@@ -98,59 +98,59 @@ define <8 x i8> @test_cmp_d_256(<8 x i32> %a0, <8 x i32> %a1) {
 define <8 x i8> @test_mask_cmp_d_256(<8 x i32> %a0, <8 x i32> %a1, i8 %mask) {
 ; CHECK-LABEL: test_mask_cmp_d_256
 ; CHECK: vpcmpeqd %ymm1, %ymm0, %k0 {%k1} ##
-  %res0 = call i8 @llvm.x86.avx512.mask.cmp.d.256(<8 x i32> %a0, <8 x i32> %a1, i8 0, i8 %mask)
+  %res0 = call i8 @llvm.x86.avx512.mask.cmp.d.256(<8 x i32> %a0, <8 x i32> %a1, i32 0, i8 %mask)
   %vec0 = insertelement <8 x i8> undef, i8 %res0, i32 0
 ; CHECK: vpcmpltd %ymm1, %ymm0, %k0 {%k1} ##
-  %res1 = call i8 @llvm.x86.avx512.mask.cmp.d.256(<8 x i32> %a0, <8 x i32> %a1, i8 1, i8 %mask)
+  %res1 = call i8 @llvm.x86.avx512.mask.cmp.d.256(<8 x i32> %a0, <8 x i32> %a1, i32 1, i8 %mask)
   %vec1 = insertelement <8 x i8> %vec0, i8 %res1, i32 1
 ; CHECK: vpcmpled %ymm1, %ymm0, %k0 {%k1} ##
-  %res2 = call i8 @llvm.x86.avx512.mask.cmp.d.256(<8 x i32> %a0, <8 x i32> %a1, i8 2, i8 %mask)
+  %res2 = call i8 @llvm.x86.avx512.mask.cmp.d.256(<8 x i32> %a0, <8 x i32> %a1, i32 2, i8 %mask)
   %vec2 = insertelement <8 x i8> %vec1, i8 %res2, i32 2
 ; CHECK: vpcmpunordd %ymm1, %ymm0, %k0 {%k1} ##
-  %res3 = call i8 @llvm.x86.avx512.mask.cmp.d.256(<8 x i32> %a0, <8 x i32> %a1, i8 3, i8 %mask)
+  %res3 = call i8 @llvm.x86.avx512.mask.cmp.d.256(<8 x i32> %a0, <8 x i32> %a1, i32 3, i8 %mask)
   %vec3 = insertelement <8 x i8> %vec2, i8 %res3, i32 3
 ; CHECK: vpcmpneqd %ymm1, %ymm0, %k0 {%k1} ##
-  %res4 = call i8 @llvm.x86.avx512.mask.cmp.d.256(<8 x i32> %a0, <8 x i32> %a1, i8 4, i8 %mask)
+  %res4 = call i8 @llvm.x86.avx512.mask.cmp.d.256(<8 x i32> %a0, <8 x i32> %a1, i32 4, i8 %mask)
   %vec4 = insertelement <8 x i8> %vec3, i8 %res4, i32 4
 ; CHECK: vpcmpnltd %ymm1, %ymm0, %k0 {%k1} ##
-  %res5 = call i8 @llvm.x86.avx512.mask.cmp.d.256(<8 x i32> %a0, <8 x i32> %a1, i8 5, i8 %mask)
+  %res5 = call i8 @llvm.x86.avx512.mask.cmp.d.256(<8 x i32> %a0, <8 x i32> %a1, i32 5, i8 %mask)
   %vec5 = insertelement <8 x i8> %vec4, i8 %res5, i32 5
 ; CHECK: vpcmpnled %ymm1, %ymm0, %k0 {%k1} ##
-  %res6 = call i8 @llvm.x86.avx512.mask.cmp.d.256(<8 x i32> %a0, <8 x i32> %a1, i8 6, i8 %mask)
+  %res6 = call i8 @llvm.x86.avx512.mask.cmp.d.256(<8 x i32> %a0, <8 x i32> %a1, i32 6, i8 %mask)
   %vec6 = insertelement <8 x i8> %vec5, i8 %res6, i32 6
 ; CHECK: vpcmpordd %ymm1, %ymm0, %k0 {%k1} ##
-  %res7 = call i8 @llvm.x86.avx512.mask.cmp.d.256(<8 x i32> %a0, <8 x i32> %a1, i8 7, i8 %mask)
+  %res7 = call i8 @llvm.x86.avx512.mask.cmp.d.256(<8 x i32> %a0, <8 x i32> %a1, i32 7, i8 %mask)
   %vec7 = insertelement <8 x i8> %vec6, i8 %res7, i32 7
   ret <8 x i8> %vec7
 }
 
-declare i8 @llvm.x86.avx512.mask.cmp.d.256(<8 x i32>, <8 x i32>, i8, i8) nounwind readnone
+declare i8 @llvm.x86.avx512.mask.cmp.d.256(<8 x i32>, <8 x i32>, i32, i8) nounwind readnone
 
 define <8 x i8> @test_ucmp_d_256(<8 x i32> %a0, <8 x i32> %a1) {
 ; CHECK-LABEL: test_ucmp_d_256
 ; CHECK: vpcmpequd %ymm1, %ymm0, %k0 ##
-  %res0 = call i8 @llvm.x86.avx512.mask.ucmp.d.256(<8 x i32> %a0, <8 x i32> %a1, i8 0, i8 -1)
+  %res0 = call i8 @llvm.x86.avx512.mask.ucmp.d.256(<8 x i32> %a0, <8 x i32> %a1, i32 0, i8 -1)
   %vec0 = insertelement <8 x i8> undef, i8 %res0, i32 0
 ; CHECK: vpcmpltud %ymm1, %ymm0, %k0 ##
-  %res1 = call i8 @llvm.x86.avx512.mask.ucmp.d.256(<8 x i32> %a0, <8 x i32> %a1, i8 1, i8 -1)
+  %res1 = call i8 @llvm.x86.avx512.mask.ucmp.d.256(<8 x i32> %a0, <8 x i32> %a1, i32 1, i8 -1)
   %vec1 = insertelement <8 x i8> %vec0, i8 %res1, i32 1
 ; CHECK: vpcmpleud %ymm1, %ymm0, %k0 ##
-  %res2 = call i8 @llvm.x86.avx512.mask.ucmp.d.256(<8 x i32> %a0, <8 x i32> %a1, i8 2, i8 -1)
+  %res2 = call i8 @llvm.x86.avx512.mask.ucmp.d.256(<8 x i32> %a0, <8 x i32> %a1, i32 2, i8 -1)
   %vec2 = insertelement <8 x i8> %vec1, i8 %res2, i32 2
 ; CHECK: vpcmpunordud %ymm1, %ymm0, %k0 ##
-  %res3 = call i8 @llvm.x86.avx512.mask.ucmp.d.256(<8 x i32> %a0, <8 x i32> %a1, i8 3, i8 -1)
+  %res3 = call i8 @llvm.x86.avx512.mask.ucmp.d.256(<8 x i32> %a0, <8 x i32> %a1, i32 3, i8 -1)
   %vec3 = insertelement <8 x i8> %vec2, i8 %res3, i32 3
 ; CHECK: vpcmpnequd %ymm1, %ymm0, %k0 ##
-  %res4 = call i8 @llvm.x86.avx512.mask.ucmp.d.256(<8 x i32> %a0, <8 x i32> %a1, i8 4, i8 -1)
+  %res4 = call i8 @llvm.x86.avx512.mask.ucmp.d.256(<8 x i32> %a0, <8 x i32> %a1, i32 4, i8 -1)
   %vec4 = insertelement <8 x i8> %vec3, i8 %res4, i32 4
 ; CHECK: vpcmpnltud %ymm1, %ymm0, %k0 ##
-  %res5 = call i8 @llvm.x86.avx512.mask.ucmp.d.256(<8 x i32> %a0, <8 x i32> %a1, i8 5, i8 -1)
+  %res5 = call i8 @llvm.x86.avx512.mask.ucmp.d.256(<8 x i32> %a0, <8 x i32> %a1, i32 5, i8 -1)
   %vec5 = insertelement <8 x i8> %vec4, i8 %res5, i32 5
 ; CHECK: vpcmpnleud %ymm1, %ymm0, %k0 ##
-  %res6 = call i8 @llvm.x86.avx512.mask.ucmp.d.256(<8 x i32> %a0, <8 x i32> %a1, i8 6, i8 -1)
+  %res6 = call i8 @llvm.x86.avx512.mask.ucmp.d.256(<8 x i32> %a0, <8 x i32> %a1, i32 6, i8 -1)
   %vec6 = insertelement <8 x i8> %vec5, i8 %res6, i32 6
 ; CHECK: vpcmpordud %ymm1, %ymm0, %k0 ##
-  %res7 = call i8 @llvm.x86.avx512.mask.ucmp.d.256(<8 x i32> %a0, <8 x i32> %a1, i8 7, i8 -1)
+  %res7 = call i8 @llvm.x86.avx512.mask.ucmp.d.256(<8 x i32> %a0, <8 x i32> %a1, i32 7, i8 -1)
   %vec7 = insertelement <8 x i8> %vec6, i8 %res7, i32 7
   ret <8 x i8> %vec7
 }
@@ -158,59 +158,59 @@ define <8 x i8> @test_ucmp_d_256(<8 x i32> %a0, <8 x i32> %a1) {
 define <8 x i8> @test_mask_ucmp_d_256(<8 x i32> %a0, <8 x i32> %a1, i8 %mask) {
 ; CHECK-LABEL: test_mask_ucmp_d_256
 ; CHECK: vpcmpequd %ymm1, %ymm0, %k0 {%k1} ##
-  %res0 = call i8 @llvm.x86.avx512.mask.ucmp.d.256(<8 x i32> %a0, <8 x i32> %a1, i8 0, i8 %mask)
+  %res0 = call i8 @llvm.x86.avx512.mask.ucmp.d.256(<8 x i32> %a0, <8 x i32> %a1, i32 0, i8 %mask)
   %vec0 = insertelement <8 x i8> undef, i8 %res0, i32 0
 ; CHECK: vpcmpltud %ymm1, %ymm0, %k0 {%k1} ##
-  %res1 = call i8 @llvm.x86.avx512.mask.ucmp.d.256(<8 x i32> %a0, <8 x i32> %a1, i8 1, i8 %mask)
+  %res1 = call i8 @llvm.x86.avx512.mask.ucmp.d.256(<8 x i32> %a0, <8 x i32> %a1, i32 1, i8 %mask)
   %vec1 = insertelement <8 x i8> %vec0, i8 %res1, i32 1
 ; CHECK: vpcmpleud %ymm1, %ymm0, %k0 {%k1} ##
-  %res2 = call i8 @llvm.x86.avx512.mask.ucmp.d.256(<8 x i32> %a0, <8 x i32> %a1, i8 2, i8 %mask)
+  %res2 = call i8 @llvm.x86.avx512.mask.ucmp.d.256(<8 x i32> %a0, <8 x i32> %a1, i32 2, i8 %mask)
   %vec2 = insertelement <8 x i8> %vec1, i8 %res2, i32 2
 ; CHECK: vpcmpunordud %ymm1, %ymm0, %k0 {%k1} ##
-  %res3 = call i8 @llvm.x86.avx512.mask.ucmp.d.256(<8 x i32> %a0, <8 x i32> %a1, i8 3, i8 %mask)
+  %res3 = call i8 @llvm.x86.avx512.mask.ucmp.d.256(<8 x i32> %a0, <8 x i32> %a1, i32 3, i8 %mask)
   %vec3 = insertelement <8 x i8> %vec2, i8 %res3, i32 3
 ; CHECK: vpcmpnequd %ymm1, %ymm0, %k0 {%k1} ##
-  %res4 = call i8 @llvm.x86.avx512.mask.ucmp.d.256(<8 x i32> %a0, <8 x i32> %a1, i8 4, i8 %mask)
+  %res4 = call i8 @llvm.x86.avx512.mask.ucmp.d.256(<8 x i32> %a0, <8 x i32> %a1, i32 4, i8 %mask)
   %vec4 = insertelement <8 x i8> %vec3, i8 %res4, i32 4
 ; CHECK: vpcmpnltud %ymm1, %ymm0, %k0 {%k1} ##
-  %res5 = call i8 @llvm.x86.avx512.mask.ucmp.d.256(<8 x i32> %a0, <8 x i32> %a1, i8 5, i8 %mask)
+  %res5 = call i8 @llvm.x86.avx512.mask.ucmp.d.256(<8 x i32> %a0, <8 x i32> %a1, i32 5, i8 %mask)
   %vec5 = insertelement <8 x i8> %vec4, i8 %res5, i32 5
 ; CHECK: vpcmpnleud %ymm1, %ymm0, %k0 {%k1} ##
-  %res6 = call i8 @llvm.x86.avx512.mask.ucmp.d.256(<8 x i32> %a0, <8 x i32> %a1, i8 6, i8 %mask)
+  %res6 = call i8 @llvm.x86.avx512.mask.ucmp.d.256(<8 x i32> %a0, <8 x i32> %a1, i32 6, i8 %mask)
   %vec6 = insertelement <8 x i8> %vec5, i8 %res6, i32 6
 ; CHECK: vpcmpordud %ymm1, %ymm0, %k0 {%k1} ##
-  %res7 = call i8 @llvm.x86.avx512.mask.ucmp.d.256(<8 x i32> %a0, <8 x i32> %a1, i8 7, i8 %mask)
+  %res7 = call i8 @llvm.x86.avx512.mask.ucmp.d.256(<8 x i32> %a0, <8 x i32> %a1, i32 7, i8 %mask)
   %vec7 = insertelement <8 x i8> %vec6, i8 %res7, i32 7
   ret <8 x i8> %vec7
 }
 
-declare i8 @llvm.x86.avx512.mask.ucmp.d.256(<8 x i32>, <8 x i32>, i8, i8) nounwind readnone
+declare i8 @llvm.x86.avx512.mask.ucmp.d.256(<8 x i32>, <8 x i32>, i32, i8) nounwind readnone
 
 define <8 x i8> @test_cmp_q_256(<4 x i64> %a0, <4 x i64> %a1) {
 ; CHECK-LABEL: test_cmp_q_256
 ; CHECK: vpcmpeqq %ymm1, %ymm0, %k0 ##
-  %res0 = call i8 @llvm.x86.avx512.mask.cmp.q.256(<4 x i64> %a0, <4 x i64> %a1, i8 0, i8 -1)
+  %res0 = call i8 @llvm.x86.avx512.mask.cmp.q.256(<4 x i64> %a0, <4 x i64> %a1, i32 0, i8 -1)
   %vec0 = insertelement <8 x i8> undef, i8 %res0, i32 0
 ; CHECK: vpcmpltq %ymm1, %ymm0, %k0 ##
-  %res1 = call i8 @llvm.x86.avx512.mask.cmp.q.256(<4 x i64> %a0, <4 x i64> %a1, i8 1, i8 -1)
+  %res1 = call i8 @llvm.x86.avx512.mask.cmp.q.256(<4 x i64> %a0, <4 x i64> %a1, i32 1, i8 -1)
   %vec1 = insertelement <8 x i8> %vec0, i8 %res1, i32 1
 ; CHECK: vpcmpleq %ymm1, %ymm0, %k0 ##
-  %res2 = call i8 @llvm.x86.avx512.mask.cmp.q.256(<4 x i64> %a0, <4 x i64> %a1, i8 2, i8 -1)
+  %res2 = call i8 @llvm.x86.avx512.mask.cmp.q.256(<4 x i64> %a0, <4 x i64> %a1, i32 2, i8 -1)
   %vec2 = insertelement <8 x i8> %vec1, i8 %res2, i32 2
 ; CHECK: vpcmpunordq %ymm1, %ymm0, %k0 ##
-  %res3 = call i8 @llvm.x86.avx512.mask.cmp.q.256(<4 x i64> %a0, <4 x i64> %a1, i8 3, i8 -1)
+  %res3 = call i8 @llvm.x86.avx512.mask.cmp.q.256(<4 x i64> %a0, <4 x i64> %a1, i32 3, i8 -1)
   %vec3 = insertelement <8 x i8> %vec2, i8 %res3, i32 3
 ; CHECK: vpcmpneqq %ymm1, %ymm0, %k0 ##
-  %res4 = call i8 @llvm.x86.avx512.mask.cmp.q.256(<4 x i64> %a0, <4 x i64> %a1, i8 4, i8 -1)
+  %res4 = call i8 @llvm.x86.avx512.mask.cmp.q.256(<4 x i64> %a0, <4 x i64> %a1, i32 4, i8 -1)
   %vec4 = insertelement <8 x i8> %vec3, i8 %res4, i32 4
 ; CHECK: vpcmpnltq %ymm1, %ymm0, %k0 ##
-  %res5 = call i8 @llvm.x86.avx512.mask.cmp.q.256(<4 x i64> %a0, <4 x i64> %a1, i8 5, i8 -1)
+  %res5 = call i8 @llvm.x86.avx512.mask.cmp.q.256(<4 x i64> %a0, <4 x i64> %a1, i32 5, i8 -1)
   %vec5 = insertelement <8 x i8> %vec4, i8 %res5, i32 5
 ; CHECK: vpcmpnleq %ymm1, %ymm0, %k0 ##
-  %res6 = call i8 @llvm.x86.avx512.mask.cmp.q.256(<4 x i64> %a0, <4 x i64> %a1, i8 6, i8 -1)
+  %res6 = call i8 @llvm.x86.avx512.mask.cmp.q.256(<4 x i64> %a0, <4 x i64> %a1, i32 6, i8 -1)
   %vec6 = insertelement <8 x i8> %vec5, i8 %res6, i32 6
 ; CHECK: vpcmpordq %ymm1, %ymm0, %k0 ##
-  %res7 = call i8 @llvm.x86.avx512.mask.cmp.q.256(<4 x i64> %a0, <4 x i64> %a1, i8 7, i8 -1)
+  %res7 = call i8 @llvm.x86.avx512.mask.cmp.q.256(<4 x i64> %a0, <4 x i64> %a1, i32 7, i8 -1)
   %vec7 = insertelement <8 x i8> %vec6, i8 %res7, i32 7
   ret <8 x i8> %vec7
 }
@@ -218,59 +218,59 @@ define <8 x i8> @test_cmp_q_256(<4 x i64> %a0, <4 x i64> %a1) {
 define <8 x i8> @test_mask_cmp_q_256(<4 x i64> %a0, <4 x i64> %a1, i8 %mask) {
 ; CHECK-LABEL: test_mask_cmp_q_256
 ; CHECK: vpcmpeqq %ymm1, %ymm0, %k0 {%k1} ##
-  %res0 = call i8 @llvm.x86.avx512.mask.cmp.q.256(<4 x i64> %a0, <4 x i64> %a1, i8 0, i8 %mask)
+  %res0 = call i8 @llvm.x86.avx512.mask.cmp.q.256(<4 x i64> %a0, <4 x i64> %a1, i32 0, i8 %mask)
   %vec0 = insertelement <8 x i8> undef, i8 %res0, i32 0
 ; CHECK: vpcmpltq %ymm1, %ymm0, %k0 {%k1} ##
-  %res1 = call i8 @llvm.x86.avx512.mask.cmp.q.256(<4 x i64> %a0, <4 x i64> %a1, i8 1, i8 %mask)
+  %res1 = call i8 @llvm.x86.avx512.mask.cmp.q.256(<4 x i64> %a0, <4 x i64> %a1, i32 1, i8 %mask)
   %vec1 = insertelement <8 x i8> %vec0, i8 %res1, i32 1
 ; CHECK: vpcmpleq %ymm1, %ymm0, %k0 {%k1} ##
-  %res2 = call i8 @llvm.x86.avx512.mask.cmp.q.256(<4 x i64> %a0, <4 x i64> %a1, i8 2, i8 %mask)
+  %res2 = call i8 @llvm.x86.avx512.mask.cmp.q.256(<4 x i64> %a0, <4 x i64> %a1, i32 2, i8 %mask)
   %vec2 = insertelement <8 x i8> %vec1, i8 %res2, i32 2
 ; CHECK: vpcmpunordq %ymm1, %ymm0, %k0 {%k1} ##
-  %res3 = call i8 @llvm.x86.avx512.mask.cmp.q.256(<4 x i64> %a0, <4 x i64> %a1, i8 3, i8 %mask)
+  %res3 = call i8 @llvm.x86.avx512.mask.cmp.q.256(<4 x i64> %a0, <4 x i64> %a1, i32 3, i8 %mask)
   %vec3 = insertelement <8 x i8> %vec2, i8 %res3, i32 3
 ; CHECK: vpcmpneqq %ymm1, %ymm0, %k0 {%k1} ##
-  %res4 = call i8 @llvm.x86.avx512.mask.cmp.q.256(<4 x i64> %a0, <4 x i64> %a1, i8 4, i8 %mask)
+  %res4 = call i8 @llvm.x86.avx512.mask.cmp.q.256(<4 x i64> %a0, <4 x i64> %a1, i32 4, i8 %mask)
   %vec4 = insertelement <8 x i8> %vec3, i8 %res4, i32 4
 ; CHECK: vpcmpnltq %ymm1, %ymm0, %k0 {%k1} ##
-  %res5 = call i8 @llvm.x86.avx512.mask.cmp.q.256(<4 x i64> %a0, <4 x i64> %a1, i8 5, i8 %mask)
+  %res5 = call i8 @llvm.x86.avx512.mask.cmp.q.256(<4 x i64> %a0, <4 x i64> %a1, i32 5, i8 %mask)
   %vec5 = insertelement <8 x i8> %vec4, i8 %res5, i32 5
 ; CHECK: vpcmpnleq %ymm1, %ymm0, %k0 {%k1} ##
-  %res6 = call i8 @llvm.x86.avx512.mask.cmp.q.256(<4 x i64> %a0, <4 x i64> %a1, i8 6, i8 %mask)
+  %res6 = call i8 @llvm.x86.avx512.mask.cmp.q.256(<4 x i64> %a0, <4 x i64> %a1, i32 6, i8 %mask)
   %vec6 = insertelement <8 x i8> %vec5, i8 %res6, i32 6
 ; CHECK: vpcmpordq %ymm1, %ymm0, %k0 {%k1} ##
-  %res7 = call i8 @llvm.x86.avx512.mask.cmp.q.256(<4 x i64> %a0, <4 x i64> %a1, i8 7, i8 %mask)
+  %res7 = call i8 @llvm.x86.avx512.mask.cmp.q.256(<4 x i64> %a0, <4 x i64> %a1, i32 7, i8 %mask)
   %vec7 = insertelement <8 x i8> %vec6, i8 %res7, i32 7
   ret <8 x i8> %vec7
 }
 
-declare i8 @llvm.x86.avx512.mask.cmp.q.256(<4 x i64>, <4 x i64>, i8, i8) nounwind readnone
+declare i8 @llvm.x86.avx512.mask.cmp.q.256(<4 x i64>, <4 x i64>, i32, i8) nounwind readnone
 
 define <8 x i8> @test_ucmp_q_256(<4 x i64> %a0, <4 x i64> %a1) {
 ; CHECK-LABEL: test_ucmp_q_256
 ; CHECK: vpcmpequq %ymm1, %ymm0, %k0 ##
-  %res0 = call i8 @llvm.x86.avx512.mask.ucmp.q.256(<4 x i64> %a0, <4 x i64> %a1, i8 0, i8 -1)
+  %res0 = call i8 @llvm.x86.avx512.mask.ucmp.q.256(<4 x i64> %a0, <4 x i64> %a1, i32 0, i8 -1)
   %vec0 = insertelement <8 x i8> undef, i8 %res0, i32 0
 ; CHECK: vpcmpltuq %ymm1, %ymm0, %k0 ##
-  %res1 = call i8 @llvm.x86.avx512.mask.ucmp.q.256(<4 x i64> %a0, <4 x i64> %a1, i8 1, i8 -1)
+  %res1 = call i8 @llvm.x86.avx512.mask.ucmp.q.256(<4 x i64> %a0, <4 x i64> %a1, i32 1, i8 -1)
   %vec1 = insertelement <8 x i8> %vec0, i8 %res1, i32 1
 ; CHECK: vpcmpleuq %ymm1, %ymm0, %k0 ##
-  %res2 = call i8 @llvm.x86.avx512.mask.ucmp.q.256(<4 x i64> %a0, <4 x i64> %a1, i8 2, i8 -1)
+  %res2 = call i8 @llvm.x86.avx512.mask.ucmp.q.256(<4 x i64> %a0, <4 x i64> %a1, i32 2, i8 -1)
   %vec2 = insertelement <8 x i8> %vec1, i8 %res2, i32 2
 ; CHECK: vpcmpunorduq %ymm1, %ymm0, %k0 ##
-  %res3 = call i8 @llvm.x86.avx512.mask.ucmp.q.256(<4 x i64> %a0, <4 x i64> %a1, i8 3, i8 -1)
+  %res3 = call i8 @llvm.x86.avx512.mask.ucmp.q.256(<4 x i64> %a0, <4 x i64> %a1, i32 3, i8 -1)
   %vec3 = insertelement <8 x i8> %vec2, i8 %res3, i32 3
 ; CHECK: vpcmpnequq %ymm1, %ymm0, %k0 ##
-  %res4 = call i8 @llvm.x86.avx512.mask.ucmp.q.256(<4 x i64> %a0, <4 x i64> %a1, i8 4, i8 -1)
+  %res4 = call i8 @llvm.x86.avx512.mask.ucmp.q.256(<4 x i64> %a0, <4 x i64> %a1, i32 4, i8 -1)
   %vec4 = insertelement <8 x i8> %vec3, i8 %res4, i32 4
 ; CHECK: vpcmpnltuq %ymm1, %ymm0, %k0 ##
-  %res5 = call i8 @llvm.x86.avx512.mask.ucmp.q.256(<4 x i64> %a0, <4 x i64> %a1, i8 5, i8 -1)
+  %res5 = call i8 @llvm.x86.avx512.mask.ucmp.q.256(<4 x i64> %a0, <4 x i64> %a1, i32 5, i8 -1)
   %vec5 = insertelement <8 x i8> %vec4, i8 %res5, i32 5
 ; CHECK: vpcmpnleuq %ymm1, %ymm0, %k0 ##
-  %res6 = call i8 @llvm.x86.avx512.mask.ucmp.q.256(<4 x i64> %a0, <4 x i64> %a1, i8 6, i8 -1)
+  %res6 = call i8 @llvm.x86.avx512.mask.ucmp.q.256(<4 x i64> %a0, <4 x i64> %a1, i32 6, i8 -1)
   %vec6 = insertelement <8 x i8> %vec5, i8 %res6, i32 6
 ; CHECK: vpcmporduq %ymm1, %ymm0, %k0 ##
-  %res7 = call i8 @llvm.x86.avx512.mask.ucmp.q.256(<4 x i64> %a0, <4 x i64> %a1, i8 7, i8 -1)
+  %res7 = call i8 @llvm.x86.avx512.mask.ucmp.q.256(<4 x i64> %a0, <4 x i64> %a1, i32 7, i8 -1)
   %vec7 = insertelement <8 x i8> %vec6, i8 %res7, i32 7
   ret <8 x i8> %vec7
 }
@@ -278,33 +278,33 @@ define <8 x i8> @test_ucmp_q_256(<4 x i64> %a0, <4 x i64> %a1) {
 define <8 x i8> @test_mask_ucmp_q_256(<4 x i64> %a0, <4 x i64> %a1, i8 %mask) {
 ; CHECK-LABEL: test_mask_ucmp_q_256
 ; CHECK: vpcmpequq %ymm1, %ymm0, %k0 {%k1} ##
-  %res0 = call i8 @llvm.x86.avx512.mask.ucmp.q.256(<4 x i64> %a0, <4 x i64> %a1, i8 0, i8 %mask)
+  %res0 = call i8 @llvm.x86.avx512.mask.ucmp.q.256(<4 x i64> %a0, <4 x i64> %a1, i32 0, i8 %mask)
   %vec0 = insertelement <8 x i8> undef, i8 %res0, i32 0
 ; CHECK: vpcmpltuq %ymm1, %ymm0, %k0 {%k1} ##
-  %res1 = call i8 @llvm.x86.avx512.mask.ucmp.q.256(<4 x i64> %a0, <4 x i64> %a1, i8 1, i8 %mask)
+  %res1 = call i8 @llvm.x86.avx512.mask.ucmp.q.256(<4 x i64> %a0, <4 x i64> %a1, i32 1, i8 %mask)
   %vec1 = insertelement <8 x i8> %vec0, i8 %res1, i32 1
 ; CHECK: vpcmpleuq %ymm1, %ymm0, %k0 {%k1} ##
-  %res2 = call i8 @llvm.x86.avx512.mask.ucmp.q.256(<4 x i64> %a0, <4 x i64> %a1, i8 2, i8 %mask)
+  %res2 = call i8 @llvm.x86.avx512.mask.ucmp.q.256(<4 x i64> %a0, <4 x i64> %a1, i32 2, i8 %mask)
   %vec2 = insertelement <8 x i8> %vec1, i8 %res2, i32 2
 ; CHECK: vpcmpunorduq %ymm1, %ymm0, %k0 {%k1} ##
-  %res3 = call i8 @llvm.x86.avx512.mask.ucmp.q.256(<4 x i64> %a0, <4 x i64> %a1, i8 3, i8 %mask)
+  %res3 = call i8 @llvm.x86.avx512.mask.ucmp.q.256(<4 x i64> %a0, <4 x i64> %a1, i32 3, i8 %mask)
   %vec3 = insertelement <8 x i8> %vec2, i8 %res3, i32 3
 ; CHECK: vpcmpnequq %ymm1, %ymm0, %k0 {%k1} ##
-  %res4 = call i8 @llvm.x86.avx512.mask.ucmp.q.256(<4 x i64> %a0, <4 x i64> %a1, i8 4, i8 %mask)
+  %res4 = call i8 @llvm.x86.avx512.mask.ucmp.q.256(<4 x i64> %a0, <4 x i64> %a1, i32 4, i8 %mask)
   %vec4 = insertelement <8 x i8> %vec3, i8 %res4, i32 4
 ; CHECK: vpcmpnltuq %ymm1, %ymm0, %k0 {%k1} ##
-  %res5 = call i8 @llvm.x86.avx512.mask.ucmp.q.256(<4 x i64> %a0, <4 x i64> %a1, i8 5, i8 %mask)
+  %res5 = call i8 @llvm.x86.avx512.mask.ucmp.q.256(<4 x i64> %a0, <4 x i64> %a1, i32 5, i8 %mask)
   %vec5 = insertelement <8 x i8> %vec4, i8 %res5, i32 5
 ; CHECK: vpcmpnleuq %ymm1, %ymm0, %k0 {%k1} ##
-  %res6 = call i8 @llvm.x86.avx512.mask.ucmp.q.256(<4 x i64> %a0, <4 x i64> %a1, i8 6, i8 %mask)
+  %res6 = call i8 @llvm.x86.avx512.mask.ucmp.q.256(<4 x i64> %a0, <4 x i64> %a1, i32 6, i8 %mask)
   %vec6 = insertelement <8 x i8> %vec5, i8 %res6, i32 6
 ; CHECK: vpcmporduq %ymm1, %ymm0, %k0 {%k1} ##
-  %res7 = call i8 @llvm.x86.avx512.mask.ucmp.q.256(<4 x i64> %a0, <4 x i64> %a1, i8 7, i8 %mask)
+  %res7 = call i8 @llvm.x86.avx512.mask.ucmp.q.256(<4 x i64> %a0, <4 x i64> %a1, i32 7, i8 %mask)
   %vec7 = insertelement <8 x i8> %vec6, i8 %res7, i32 7
   ret <8 x i8> %vec7
 }
 
-declare i8 @llvm.x86.avx512.mask.ucmp.q.256(<4 x i64>, <4 x i64>, i8, i8) nounwind readnone
+declare i8 @llvm.x86.avx512.mask.ucmp.q.256(<4 x i64>, <4 x i64>, i32, i8) nounwind readnone
 
 ; 128-bit
 
@@ -375,28 +375,28 @@ declare i8 @llvm.x86.avx512.mask.pcmpgt.q.128(<2 x i64>, <2 x i64>, i8)
 define <8 x i8> @test_cmp_d_128(<4 x i32> %a0, <4 x i32> %a1) {
 ; CHECK-LABEL: test_cmp_d_128
 ; CHECK: vpcmpeqd %xmm1, %xmm0, %k0 ##
-  %res0 = call i8 @llvm.x86.avx512.mask.cmp.d.128(<4 x i32> %a0, <4 x i32> %a1, i8 0, i8 -1)
+  %res0 = call i8 @llvm.x86.avx512.mask.cmp.d.128(<4 x i32> %a0, <4 x i32> %a1, i32 0, i8 -1)
   %vec0 = insertelement <8 x i8> undef, i8 %res0, i32 0
 ; CHECK: vpcmpltd %xmm1, %xmm0, %k0 ##
-  %res1 = call i8 @llvm.x86.avx512.mask.cmp.d.128(<4 x i32> %a0, <4 x i32> %a1, i8 1, i8 -1)
+  %res1 = call i8 @llvm.x86.avx512.mask.cmp.d.128(<4 x i32> %a0, <4 x i32> %a1, i32 1, i8 -1)
   %vec1 = insertelement <8 x i8> %vec0, i8 %res1, i32 1
 ; CHECK: vpcmpled %xmm1, %xmm0, %k0 ##
-  %res2 = call i8 @llvm.x86.avx512.mask.cmp.d.128(<4 x i32> %a0, <4 x i32> %a1, i8 2, i8 -1)
+  %res2 = call i8 @llvm.x86.avx512.mask.cmp.d.128(<4 x i32> %a0, <4 x i32> %a1, i32 2, i8 -1)
   %vec2 = insertelement <8 x i8> %vec1, i8 %res2, i32 2
 ; CHECK: vpcmpunordd %xmm1, %xmm0, %k0 ##
-  %res3 = call i8 @llvm.x86.avx512.mask.cmp.d.128(<4 x i32> %a0, <4 x i32> %a1, i8 3, i8 -1)
+  %res3 = call i8 @llvm.x86.avx512.mask.cmp.d.128(<4 x i32> %a0, <4 x i32> %a1, i32 3, i8 -1)
   %vec3 = insertelement <8 x i8> %vec2, i8 %res3, i32 3
 ; CHECK: vpcmpneqd %xmm1, %xmm0, %k0 ##
-  %res4 = call i8 @llvm.x86.avx512.mask.cmp.d.128(<4 x i32> %a0, <4 x i32> %a1, i8 4, i8 -1)
+  %res4 = call i8 @llvm.x86.avx512.mask.cmp.d.128(<4 x i32> %a0, <4 x i32> %a1, i32 4, i8 -1)
   %vec4 = insertelement <8 x i8> %vec3, i8 %res4, i32 4
 ; CHECK: vpcmpnltd %xmm1, %xmm0, %k0 ##
-  %res5 = call i8 @llvm.x86.avx512.mask.cmp.d.128(<4 x i32> %a0, <4 x i32> %a1, i8 5, i8 -1)
+  %res5 = call i8 @llvm.x86.avx512.mask.cmp.d.128(<4 x i32> %a0, <4 x i32> %a1, i32 5, i8 -1)
   %vec5 = insertelement <8 x i8> %vec4, i8 %res5, i32 5
 ; CHECK: vpcmpnled %xmm1, %xmm0, %k0 ##
-  %res6 = call i8 @llvm.x86.avx512.mask.cmp.d.128(<4 x i32> %a0, <4 x i32> %a1, i8 6, i8 -1)
+  %res6 = call i8 @llvm.x86.avx512.mask.cmp.d.128(<4 x i32> %a0, <4 x i32> %a1, i32 6, i8 -1)
   %vec6 = insertelement <8 x i8> %vec5, i8 %res6, i32 6
 ; CHECK: vpcmpordd %xmm1, %xmm0, %k0 ##
-  %res7 = call i8 @llvm.x86.avx512.mask.cmp.d.128(<4 x i32> %a0, <4 x i32> %a1, i8 7, i8 -1)
+  %res7 = call i8 @llvm.x86.avx512.mask.cmp.d.128(<4 x i32> %a0, <4 x i32> %a1, i32 7, i8 -1)
   %vec7 = insertelement <8 x i8> %vec6, i8 %res7, i32 7
   ret <8 x i8> %vec7
 }
@@ -404,59 +404,59 @@ define <8 x i8> @test_cmp_d_128(<4 x i32> %a0, <4 x i32> %a1) {
 define <8 x i8> @test_mask_cmp_d_128(<4 x i32> %a0, <4 x i32> %a1, i8 %mask) {
 ; CHECK-LABEL: test_mask_cmp_d_128
 ; CHECK: vpcmpeqd %xmm1, %xmm0, %k0 {%k1} ##
-  %res0 = call i8 @llvm.x86.avx512.mask.cmp.d.128(<4 x i32> %a0, <4 x i32> %a1, i8 0, i8 %mask)
+  %res0 = call i8 @llvm.x86.avx512.mask.cmp.d.128(<4 x i32> %a0, <4 x i32> %a1, i32 0, i8 %mask)
   %vec0 = insertelement <8 x i8> undef, i8 %res0, i32 0
 ; CHECK: vpcmpltd %xmm1, %xmm0, %k0 {%k1} ##
-  %res1 = call i8 @llvm.x86.avx512.mask.cmp.d.128(<4 x i32> %a0, <4 x i32> %a1, i8 1, i8 %mask)
+  %res1 = call i8 @llvm.x86.avx512.mask.cmp.d.128(<4 x i32> %a0, <4 x i32> %a1, i32 1, i8 %mask)
   %vec1 = insertelement <8 x i8> %vec0, i8 %res1, i32 1
 ; CHECK: vpcmpled %xmm1, %xmm0, %k0 {%k1} ##
-  %res2 = call i8 @llvm.x86.avx512.mask.cmp.d.128(<4 x i32> %a0, <4 x i32> %a1, i8 2, i8 %mask)
+  %res2 = call i8 @llvm.x86.avx512.mask.cmp.d.128(<4 x i32> %a0, <4 x i32> %a1, i32 2, i8 %mask)
   %vec2 = insertelement <8 x i8> %vec1, i8 %res2, i32 2
 ; CHECK: vpcmpunordd %xmm1, %xmm0, %k0 {%k1} ##
-  %res3 = call i8 @llvm.x86.avx512.mask.cmp.d.128(<4 x i32> %a0, <4 x i32> %a1, i8 3, i8 %mask)
+  %res3 = call i8 @llvm.x86.avx512.mask.cmp.d.128(<4 x i32> %a0, <4 x i32> %a1, i32 3, i8 %mask)
   %vec3 = insertelement <8 x i8> %vec2, i8 %res3, i32 3
 ; CHECK: vpcmpneqd %xmm1, %xmm0, %k0 {%k1} ##
-  %res4 = call i8 @llvm.x86.avx512.mask.cmp.d.128(<4 x i32> %a0, <4 x i32> %a1, i8 4, i8 %mask)
+  %res4 = call i8 @llvm.x86.avx512.mask.cmp.d.128(<4 x i32> %a0, <4 x i32> %a1, i32 4, i8 %mask)
   %vec4 = insertelement <8 x i8> %vec3, i8 %res4, i32 4
 ; CHECK: vpcmpnltd %xmm1, %xmm0, %k0 {%k1} ##
-  %res5 = call i8 @llvm.x86.avx512.mask.cmp.d.128(<4 x i32> %a0, <4 x i32> %a1, i8 5, i8 %mask)
+  %res5 = call i8 @llvm.x86.avx512.mask.cmp.d.128(<4 x i32> %a0, <4 x i32> %a1, i32 5, i8 %mask)
   %vec5 = insertelement <8 x i8> %vec4, i8 %res5, i32 5
 ; CHECK: vpcmpnled %xmm1, %xmm0, %k0 {%k1} ##
-  %res6 = call i8 @llvm.x86.avx512.mask.cmp.d.128(<4 x i32> %a0, <4 x i32> %a1, i8 6, i8 %mask)
+  %res6 = call i8 @llvm.x86.avx512.mask.cmp.d.128(<4 x i32> %a0, <4 x i32> %a1, i32 6, i8 %mask)
   %vec6 = insertelement <8 x i8> %vec5, i8 %res6, i32 6
 ; CHECK: vpcmpordd %xmm1, %xmm0, %k0 {%k1} ##
-  %res7 = call i8 @llvm.x86.avx512.mask.cmp.d.128(<4 x i32> %a0, <4 x i32> %a1, i8 7, i8 %mask)
+  %res7 = call i8 @llvm.x86.avx512.mask.cmp.d.128(<4 x i32> %a0, <4 x i32> %a1, i32 7, i8 %mask)
   %vec7 = insertelement <8 x i8> %vec6, i8 %res7, i32 7
   ret <8 x i8> %vec7
 }
 
-declare i8 @llvm.x86.avx512.mask.cmp.d.128(<4 x i32>, <4 x i32>, i8, i8) nounwind readnone
+declare i8 @llvm.x86.avx512.mask.cmp.d.128(<4 x i32>, <4 x i32>, i32, i8) nounwind readnone
 
 define <8 x i8> @test_ucmp_d_128(<4 x i32> %a0, <4 x i32> %a1) {
 ; CHECK-LABEL: test_ucmp_d_128
 ; CHECK: vpcmpequd %xmm1, %xmm0, %k0 ##
-  %res0 = call i8 @llvm.x86.avx512.mask.ucmp.d.128(<4 x i32> %a0, <4 x i32> %a1, i8 0, i8 -1)
+  %res0 = call i8 @llvm.x86.avx512.mask.ucmp.d.128(<4 x i32> %a0, <4 x i32> %a1, i32 0, i8 -1)
   %vec0 = insertelement <8 x i8> undef, i8 %res0, i32 0
 ; CHECK: vpcmpltud %xmm1, %xmm0, %k0 ##
-  %res1 = call i8 @llvm.x86.avx512.mask.ucmp.d.128(<4 x i32> %a0, <4 x i32> %a1, i8 1, i8 -1)
+  %res1 = call i8 @llvm.x86.avx512.mask.ucmp.d.128(<4 x i32> %a0, <4 x i32> %a1, i32 1, i8 -1)
   %vec1 = insertelement <8 x i8> %vec0, i8 %res1, i32 1
 ; CHECK: vpcmpleud %xmm1, %xmm0, %k0 ##
-  %res2 = call i8 @llvm.x86.avx512.mask.ucmp.d.128(<4 x i32> %a0, <4 x i32> %a1, i8 2, i8 -1)
+  %res2 = call i8 @llvm.x86.avx512.mask.ucmp.d.128(<4 x i32> %a0, <4 x i32> %a1, i32 2, i8 -1)
   %vec2 = insertelement <8 x i8> %vec1, i8 %res2, i32 2
 ; CHECK: vpcmpunordud %xmm1, %xmm0, %k0 ##
-  %res3 = call i8 @llvm.x86.avx512.mask.ucmp.d.128(<4 x i32> %a0, <4 x i32> %a1, i8 3, i8 -1)
+  %res3 = call i8 @llvm.x86.avx512.mask.ucmp.d.128(<4 x i32> %a0, <4 x i32> %a1, i32 3, i8 -1)
   %vec3 = insertelement <8 x i8> %vec2, i8 %res3, i32 3
 ; CHECK: vpcmpnequd %xmm1, %xmm0, %k0 ##
-  %res4 = call i8 @llvm.x86.avx512.mask.ucmp.d.128(<4 x i32> %a0, <4 x i32> %a1, i8 4, i8 -1)
+  %res4 = call i8 @llvm.x86.avx512.mask.ucmp.d.128(<4 x i32> %a0, <4 x i32> %a1, i32 4, i8 -1)
   %vec4 = insertelement <8 x i8> %vec3, i8 %res4, i32 4
 ; CHECK: vpcmpnltud %xmm1, %xmm0, %k0 ##
-  %res5 = call i8 @llvm.x86.avx512.mask.ucmp.d.128(<4 x i32> %a0, <4 x i32> %a1, i8 5, i8 -1)
+  %res5 = call i8 @llvm.x86.avx512.mask.ucmp.d.128(<4 x i32> %a0, <4 x i32> %a1, i32 5, i8 -1)
   %vec5 = insertelement <8 x i8> %vec4, i8 %res5, i32 5
 ; CHECK: vpcmpnleud %xmm1, %xmm0, %k0 ##
-  %res6 = call i8 @llvm.x86.avx512.mask.ucmp.d.128(<4 x i32> %a0, <4 x i32> %a1, i8 6, i8 -1)
+  %res6 = call i8 @llvm.x86.avx512.mask.ucmp.d.128(<4 x i32> %a0, <4 x i32> %a1, i32 6, i8 -1)
   %vec6 = insertelement <8 x i8> %vec5, i8 %res6, i32 6
 ; CHECK: vpcmpordud %xmm1, %xmm0, %k0 ##
-  %res7 = call i8 @llvm.x86.avx512.mask.ucmp.d.128(<4 x i32> %a0, <4 x i32> %a1, i8 7, i8 -1)
+  %res7 = call i8 @llvm.x86.avx512.mask.ucmp.d.128(<4 x i32> %a0, <4 x i32> %a1, i32 7, i8 -1)
   %vec7 = insertelement <8 x i8> %vec6, i8 %res7, i32 7
   ret <8 x i8> %vec7
 }
@@ -464,59 +464,59 @@ define <8 x i8> @test_ucmp_d_128(<4 x i32> %a0, <4 x i32> %a1) {
 define <8 x i8> @test_mask_ucmp_d_128(<4 x i32> %a0, <4 x i32> %a1, i8 %mask) {
 ; CHECK-LABEL: test_mask_ucmp_d_128
 ; CHECK: vpcmpequd %xmm1, %xmm0, %k0 {%k1} ##
-  %res0 = call i8 @llvm.x86.avx512.mask.ucmp.d.128(<4 x i32> %a0, <4 x i32> %a1, i8 0, i8 %mask)
+  %res0 = call i8 @llvm.x86.avx512.mask.ucmp.d.128(<4 x i32> %a0, <4 x i32> %a1, i32 0, i8 %mask)
   %vec0 = insertelement <8 x i8> undef, i8 %res0, i32 0
 ; CHECK: vpcmpltud %xmm1, %xmm0, %k0 {%k1} ##
-  %res1 = call i8 @llvm.x86.avx512.mask.ucmp.d.128(<4 x i32> %a0, <4 x i32> %a1, i8 1, i8 %mask)
+  %res1 = call i8 @llvm.x86.avx512.mask.ucmp.d.128(<4 x i32> %a0, <4 x i32> %a1, i32 1, i8 %mask)
   %vec1 = insertelement <8 x i8> %vec0, i8 %res1, i32 1
 ; CHECK: vpcmpleud %xmm1, %xmm0, %k0 {%k1} ##
-  %res2 = call i8 @llvm.x86.avx512.mask.ucmp.d.128(<4 x i32> %a0, <4 x i32> %a1, i8 2, i8 %mask)
+  %res2 = call i8 @llvm.x86.avx512.mask.ucmp.d.128(<4 x i32> %a0, <4 x i32> %a1, i32 2, i8 %mask)
   %vec2 = insertelement <8 x i8> %vec1, i8 %res2, i32 2
 ; CHECK: vpcmpunordud %xmm1, %xmm0, %k0 {%k1} ##
-  %res3 = call i8 @llvm.x86.avx512.mask.ucmp.d.128(<4 x i32> %a0, <4 x i32> %a1, i8 3, i8 %mask)
+  %res3 = call i8 @llvm.x86.avx512.mask.ucmp.d.128(<4 x i32> %a0, <4 x i32> %a1, i32 3, i8 %mask)
   %vec3 = insertelement <8 x i8> %vec2, i8 %res3, i32 3
 ; CHECK: vpcmpnequd %xmm1, %xmm0, %k0 {%k1} ##
-  %res4 = call i8 @llvm.x86.avx512.mask.ucmp.d.128(<4 x i32> %a0, <4 x i32> %a1, i8 4, i8 %mask)
+  %res4 = call i8 @llvm.x86.avx512.mask.ucmp.d.128(<4 x i32> %a0, <4 x i32> %a1, i32 4, i8 %mask)
   %vec4 = insertelement <8 x i8> %vec3, i8 %res4, i32 4
 ; CHECK: vpcmpnltud %xmm1, %xmm0, %k0 {%k1} ##
-  %res5 = call i8 @llvm.x86.avx512.mask.ucmp.d.128(<4 x i32> %a0, <4 x i32> %a1, i8 5, i8 %mask)
+  %res5 = call i8 @llvm.x86.avx512.mask.ucmp.d.128(<4 x i32> %a0, <4 x i32> %a1, i32 5, i8 %mask)
   %vec5 = insertelement <8 x i8> %vec4, i8 %res5, i32 5
 ; CHECK: vpcmpnleud %xmm1, %xmm0, %k0 {%k1} ##
-  %res6 = call i8 @llvm.x86.avx512.mask.ucmp.d.128(<4 x i32> %a0, <4 x i32> %a1, i8 6, i8 %mask)
+  %res6 = call i8 @llvm.x86.avx512.mask.ucmp.d.128(<4 x i32> %a0, <4 x i32> %a1, i32 6, i8 %mask)
   %vec6 = insertelement <8 x i8> %vec5, i8 %res6, i32 6
 ; CHECK: vpcmpordud %xmm1, %xmm0, %k0 {%k1} ##
-  %res7 = call i8 @llvm.x86.avx512.mask.ucmp.d.128(<4 x i32> %a0, <4 x i32> %a1, i8 7, i8 %mask)
+  %res7 = call i8 @llvm.x86.avx512.mask.ucmp.d.128(<4 x i32> %a0, <4 x i32> %a1, i32 7, i8 %mask)
   %vec7 = insertelement <8 x i8> %vec6, i8 %res7, i32 7
   ret <8 x i8> %vec7
 }
 
-declare i8 @llvm.x86.avx512.mask.ucmp.d.128(<4 x i32>, <4 x i32>, i8, i8) nounwind readnone
+declare i8 @llvm.x86.avx512.mask.ucmp.d.128(<4 x i32>, <4 x i32>, i32, i8) nounwind readnone
 
 define <8 x i8> @test_cmp_q_128(<2 x i64> %a0, <2 x i64> %a1) {
 ; CHECK-LABEL: test_cmp_q_128
 ; CHECK: vpcmpeqq %xmm1, %xmm0, %k0 ##
-  %res0 = call i8 @llvm.x86.avx512.mask.cmp.q.128(<2 x i64> %a0, <2 x i64> %a1, i8 0, i8 -1)
+  %res0 = call i8 @llvm.x86.avx512.mask.cmp.q.128(<2 x i64> %a0, <2 x i64> %a1, i32 0, i8 -1)
   %vec0 = insertelement <8 x i8> undef, i8 %res0, i32 0
 ; CHECK: vpcmpltq %xmm1, %xmm0, %k0 ##
-  %res1 = call i8 @llvm.x86.avx512.mask.cmp.q.128(<2 x i64> %a0, <2 x i64> %a1, i8 1, i8 -1)
+  %res1 = call i8 @llvm.x86.avx512.mask.cmp.q.128(<2 x i64> %a0, <2 x i64> %a1, i32 1, i8 -1)
   %vec1 = insertelement <8 x i8> %vec0, i8 %res1, i32 1
 ; CHECK: vpcmpleq %xmm1, %xmm0, %k0 ##
-  %res2 = call i8 @llvm.x86.avx512.mask.cmp.q.128(<2 x i64> %a0, <2 x i64> %a1, i8 2, i8 -1)
+  %res2 = call i8 @llvm.x86.avx512.mask.cmp.q.128(<2 x i64> %a0, <2 x i64> %a1, i32 2, i8 -1)
   %vec2 = insertelement <8 x i8> %vec1, i8 %res2, i32 2
 ; CHECK: vpcmpunordq %xmm1, %xmm0, %k0 ##
-  %res3 = call i8 @llvm.x86.avx512.mask.cmp.q.128(<2 x i64> %a0, <2 x i64> %a1, i8 3, i8 -1)
+  %res3 = call i8 @llvm.x86.avx512.mask.cmp.q.128(<2 x i64> %a0, <2 x i64> %a1, i32 3, i8 -1)
   %vec3 = insertelement <8 x i8> %vec2, i8 %res3, i32 3
 ; CHECK: vpcmpneqq %xmm1, %xmm0, %k0 ##
-  %res4 = call i8 @llvm.x86.avx512.mask.cmp.q.128(<2 x i64> %a0, <2 x i64> %a1, i8 4, i8 -1)
+  %res4 = call i8 @llvm.x86.avx512.mask.cmp.q.128(<2 x i64> %a0, <2 x i64> %a1, i32 4, i8 -1)
   %vec4 = insertelement <8 x i8> %vec3, i8 %res4, i32 4
 ; CHECK: vpcmpnltq %xmm1, %xmm0, %k0 ##
-  %res5 = call i8 @llvm.x86.avx512.mask.cmp.q.128(<2 x i64> %a0, <2 x i64> %a1, i8 5, i8 -1)
+  %res5 = call i8 @llvm.x86.avx512.mask.cmp.q.128(<2 x i64> %a0, <2 x i64> %a1, i32 5, i8 -1)
   %vec5 = insertelement <8 x i8> %vec4, i8 %res5, i32 5
 ; CHECK: vpcmpnleq %xmm1, %xmm0, %k0 ##
-  %res6 = call i8 @llvm.x86.avx512.mask.cmp.q.128(<2 x i64> %a0, <2 x i64> %a1, i8 6, i8 -1)
+  %res6 = call i8 @llvm.x86.avx512.mask.cmp.q.128(<2 x i64> %a0, <2 x i64> %a1, i32 6, i8 -1)
   %vec6 = insertelement <8 x i8> %vec5, i8 %res6, i32 6
 ; CHECK: vpcmpordq %xmm1, %xmm0, %k0 ##
-  %res7 = call i8 @llvm.x86.avx512.mask.cmp.q.128(<2 x i64> %a0, <2 x i64> %a1, i8 7, i8 -1)
+  %res7 = call i8 @llvm.x86.avx512.mask.cmp.q.128(<2 x i64> %a0, <2 x i64> %a1, i32 7, i8 -1)
   %vec7 = insertelement <8 x i8> %vec6, i8 %res7, i32 7
   ret <8 x i8> %vec7
 }
@@ -524,59 +524,59 @@ define <8 x i8> @test_cmp_q_128(<2 x i64> %a0, <2 x i64> %a1) {
 define <8 x i8> @test_mask_cmp_q_128(<2 x i64> %a0, <2 x i64> %a1, i8 %mask) {
 ; CHECK-LABEL: test_mask_cmp_q_128
 ; CHECK: vpcmpeqq %xmm1, %xmm0, %k0 {%k1} ##
-  %res0 = call i8 @llvm.x86.avx512.mask.cmp.q.128(<2 x i64> %a0, <2 x i64> %a1, i8 0, i8 %mask)
+  %res0 = call i8 @llvm.x86.avx512.mask.cmp.q.128(<2 x i64> %a0, <2 x i64> %a1, i32 0, i8 %mask)
   %vec0 = insertelement <8 x i8> undef, i8 %res0, i32 0
 ; CHECK: vpcmpltq %xmm1, %xmm0, %k0 {%k1} ##
-  %res1 = call i8 @llvm.x86.avx512.mask.cmp.q.128(<2 x i64> %a0, <2 x i64> %a1, i8 1, i8 %mask)
+  %res1 = call i8 @llvm.x86.avx512.mask.cmp.q.128(<2 x i64> %a0, <2 x i64> %a1, i32 1, i8 %mask)
   %vec1 = insertelement <8 x i8> %vec0, i8 %res1, i32 1
 ; CHECK: vpcmpleq %xmm1, %xmm0, %k0 {%k1} ##
-  %res2 = call i8 @llvm.x86.avx512.mask.cmp.q.128(<2 x i64> %a0, <2 x i64> %a1, i8 2, i8 %mask)
+  %res2 = call i8 @llvm.x86.avx512.mask.cmp.q.128(<2 x i64> %a0, <2 x i64> %a1, i32 2, i8 %mask)
   %vec2 = insertelement <8 x i8> %vec1, i8 %res2, i32 2
 ; CHECK: vpcmpunordq %xmm1, %xmm0, %k0 {%k1} ##
-  %res3 = call i8 @llvm.x86.avx512.mask.cmp.q.128(<2 x i64> %a0, <2 x i64> %a1, i8 3, i8 %mask)
+  %res3 = call i8 @llvm.x86.avx512.mask.cmp.q.128(<2 x i64> %a0, <2 x i64> %a1, i32 3, i8 %mask)
   %vec3 = insertelement <8 x i8> %vec2, i8 %res3, i32 3
 ; CHECK: vpcmpneqq %xmm1, %xmm0, %k0 {%k1} ##
-  %res4 = call i8 @llvm.x86.avx512.mask.cmp.q.128(<2 x i64> %a0, <2 x i64> %a1, i8 4, i8 %mask)
+  %res4 = call i8 @llvm.x86.avx512.mask.cmp.q.128(<2 x i64> %a0, <2 x i64> %a1, i32 4, i8 %mask)
   %vec4 = insertelement <8 x i8> %vec3, i8 %res4, i32 4
 ; CHECK: vpcmpnltq %xmm1, %xmm0, %k0 {%k1} ##
-  %res5 = call i8 @llvm.x86.avx512.mask.cmp.q.128(<2 x i64> %a0, <2 x i64> %a1, i8 5, i8 %mask)
+  %res5 = call i8 @llvm.x86.avx512.mask.cmp.q.128(<2 x i64> %a0, <2 x i64> %a1, i32 5, i8 %mask)
   %vec5 = insertelement <8 x i8> %vec4, i8 %res5, i32 5
 ; CHECK: vpcmpnleq %xmm1, %xmm0, %k0 {%k1} ##
-  %res6 = call i8 @llvm.x86.avx512.mask.cmp.q.128(<2 x i64> %a0, <2 x i64> %a1, i8 6, i8 %mask)
+  %res6 = call i8 @llvm.x86.avx512.mask.cmp.q.128(<2 x i64> %a0, <2 x i64> %a1, i32 6, i8 %mask)
   %vec6 = insertelement <8 x i8> %vec5, i8 %res6, i32 6
 ; CHECK: vpcmpordq %xmm1, %xmm0, %k0 {%k1} ##
-  %res7 = call i8 @llvm.x86.avx512.mask.cmp.q.128(<2 x i64> %a0, <2 x i64> %a1, i8 7, i8 %mask)
+  %res7 = call i8 @llvm.x86.avx512.mask.cmp.q.128(<2 x i64> %a0, <2 x i64> %a1, i32 7, i8 %mask)
   %vec7 = insertelement <8 x i8> %vec6, i8 %res7, i32 7
   ret <8 x i8> %vec7
 }
 
-declare i8 @llvm.x86.avx512.mask.cmp.q.128(<2 x i64>, <2 x i64>, i8, i8) nounwind readnone
+declare i8 @llvm.x86.avx512.mask.cmp.q.128(<2 x i64>, <2 x i64>, i32, i8) nounwind readnone
 
 define <8 x i8> @test_ucmp_q_128(<2 x i64> %a0, <2 x i64> %a1) {
 ; CHECK-LABEL: test_ucmp_q_128
 ; CHECK: vpcmpequq %xmm1, %xmm0, %k0 ##
-  %res0 = call i8 @llvm.x86.avx512.mask.ucmp.q.128(<2 x i64> %a0, <2 x i64> %a1, i8 0, i8 -1)
+  %res0 = call i8 @llvm.x86.avx512.mask.ucmp.q.128(<2 x i64> %a0, <2 x i64> %a1, i32 0, i8 -1)
   %vec0 = insertelement <8 x i8> undef, i8 %res0, i32 0
 ; CHECK: vpcmpltuq %xmm1, %xmm0, %k0 ##
-  %res1 = call i8 @llvm.x86.avx512.mask.ucmp.q.128(<2 x i64> %a0, <2 x i64> %a1, i8 1, i8 -1)
+  %res1 = call i8 @llvm.x86.avx512.mask.ucmp.q.128(<2 x i64> %a0, <2 x i64> %a1, i32 1, i8 -1)
   %vec1 = insertelement <8 x i8> %vec0, i8 %res1, i32 1
 ; CHECK: vpcmpleuq %xmm1, %xmm0, %k0 ##
-  %res2 = call i8 @llvm.x86.avx512.mask.ucmp.q.128(<2 x i64> %a0, <2 x i64> %a1, i8 2, i8 -1)
+  %res2 = call i8 @llvm.x86.avx512.mask.ucmp.q.128(<2 x i64> %a0, <2 x i64> %a1, i32 2, i8 -1)
   %vec2 = insertelement <8 x i8> %vec1, i8 %res2, i32 2
 ; CHECK: vpcmpunorduq %xmm1, %xmm0, %k0 ##
-  %res3 = call i8 @llvm.x86.avx512.mask.ucmp.q.128(<2 x i64> %a0, <2 x i64> %a1, i8 3, i8 -1)
+  %res3 = call i8 @llvm.x86.avx512.mask.ucmp.q.128(<2 x i64> %a0, <2 x i64> %a1, i32 3, i8 -1)
   %vec3 = insertelement <8 x i8> %vec2, i8 %res3, i32 3
 ; CHECK: vpcmpnequq %xmm1, %xmm0, %k0 ##
-  %res4 = call i8 @llvm.x86.avx512.mask.ucmp.q.128(<2 x i64> %a0, <2 x i64> %a1, i8 4, i8 -1)
+  %res4 = call i8 @llvm.x86.avx512.mask.ucmp.q.128(<2 x i64> %a0, <2 x i64> %a1, i32 4, i8 -1)
   %vec4 = insertelement <8 x i8> %vec3, i8 %res4, i32 4
 ; CHECK: vpcmpnltuq %xmm1, %xmm0, %k0 ##
-  %res5 = call i8 @llvm.x86.avx512.mask.ucmp.q.128(<2 x i64> %a0, <2 x i64> %a1, i8 5, i8 -1)
+  %res5 = call i8 @llvm.x86.avx512.mask.ucmp.q.128(<2 x i64> %a0, <2 x i64> %a1, i32 5, i8 -1)
   %vec5 = insertelement <8 x i8> %vec4, i8 %res5, i32 5
 ; CHECK: vpcmpnleuq %xmm1, %xmm0, %k0 ##
-  %res6 = call i8 @llvm.x86.avx512.mask.ucmp.q.128(<2 x i64> %a0, <2 x i64> %a1, i8 6, i8 -1)
+  %res6 = call i8 @llvm.x86.avx512.mask.ucmp.q.128(<2 x i64> %a0, <2 x i64> %a1, i32 6, i8 -1)
   %vec6 = insertelement <8 x i8> %vec5, i8 %res6, i32 6
 ; CHECK: vpcmporduq %xmm1, %xmm0, %k0 ##
-  %res7 = call i8 @llvm.x86.avx512.mask.ucmp.q.128(<2 x i64> %a0, <2 x i64> %a1, i8 7, i8 -1)
+  %res7 = call i8 @llvm.x86.avx512.mask.ucmp.q.128(<2 x i64> %a0, <2 x i64> %a1, i32 7, i8 -1)
   %vec7 = insertelement <8 x i8> %vec6, i8 %res7, i32 7
   ret <8 x i8> %vec7
 }
@@ -584,33 +584,33 @@ define <8 x i8> @test_ucmp_q_128(<2 x i64> %a0, <2 x i64> %a1) {
 define <8 x i8> @test_mask_ucmp_q_128(<2 x i64> %a0, <2 x i64> %a1, i8 %mask) {
 ; CHECK-LABEL: test_mask_ucmp_q_128
 ; CHECK: vpcmpequq %xmm1, %xmm0, %k0 {%k1} ##
-  %res0 = call i8 @llvm.x86.avx512.mask.ucmp.q.128(<2 x i64> %a0, <2 x i64> %a1, i8 0, i8 %mask)
+  %res0 = call i8 @llvm.x86.avx512.mask.ucmp.q.128(<2 x i64> %a0, <2 x i64> %a1, i32 0, i8 %mask)
   %vec0 = insertelement <8 x i8> undef, i8 %res0, i32 0
 ; CHECK: vpcmpltuq %xmm1, %xmm0, %k0 {%k1} ##
-  %res1 = call i8 @llvm.x86.avx512.mask.ucmp.q.128(<2 x i64> %a0, <2 x i64> %a1, i8 1, i8 %mask)
+  %res1 = call i8 @llvm.x86.avx512.mask.ucmp.q.128(<2 x i64> %a0, <2 x i64> %a1, i32 1, i8 %mask)
   %vec1 = insertelement <8 x i8> %vec0, i8 %res1, i32 1
 ; CHECK: vpcmpleuq %xmm1, %xmm0, %k0 {%k1} ##
-  %res2 = call i8 @llvm.x86.avx512.mask.ucmp.q.128(<2 x i64> %a0, <2 x i64> %a1, i8 2, i8 %mask)
+  %res2 = call i8 @llvm.x86.avx512.mask.ucmp.q.128(<2 x i64> %a0, <2 x i64> %a1, i32 2, i8 %mask)
   %vec2 = insertelement <8 x i8> %vec1, i8 %res2, i32 2
 ; CHECK: vpcmpunorduq %xmm1, %xmm0, %k0 {%k1} ##
-  %res3 = call i8 @llvm.x86.avx512.mask.ucmp.q.128(<2 x i64> %a0, <2 x i64> %a1, i8 3, i8 %mask)
+  %res3 = call i8 @llvm.x86.avx512.mask.ucmp.q.128(<2 x i64> %a0, <2 x i64> %a1, i32 3, i8 %mask)
   %vec3 = insertelement <8 x i8> %vec2, i8 %res3, i32 3
 ; CHECK: vpcmpnequq %xmm1, %xmm0, %k0 {%k1} ##
-  %res4 = call i8 @llvm.x86.avx512.mask.ucmp.q.128(<2 x i64> %a0, <2 x i64> %a1, i8 4, i8 %mask)
+  %res4 = call i8 @llvm.x86.avx512.mask.ucmp.q.128(<2 x i64> %a0, <2 x i64> %a1, i32 4, i8 %mask)
   %vec4 = insertelement <8 x i8> %vec3, i8 %res4, i32 4
 ; CHECK: vpcmpnltuq %xmm1, %xmm0, %k0 {%k1} ##
-  %res5 = call i8 @llvm.x86.avx512.mask.ucmp.q.128(<2 x i64> %a0, <2 x i64> %a1, i8 5, i8 %mask)
+  %res5 = call i8 @llvm.x86.avx512.mask.ucmp.q.128(<2 x i64> %a0, <2 x i64> %a1, i32 5, i8 %mask)
   %vec5 = insertelement <8 x i8> %vec4, i8 %res5, i32 5
 ; CHECK: vpcmpnleuq %xmm1, %xmm0, %k0 {%k1} ##
-  %res6 = call i8 @llvm.x86.avx512.mask.ucmp.q.128(<2 x i64> %a0, <2 x i64> %a1, i8 6, i8 %mask)
+  %res6 = call i8 @llvm.x86.avx512.mask.ucmp.q.128(<2 x i64> %a0, <2 x i64> %a1, i32 6, i8 %mask)
   %vec6 = insertelement <8 x i8> %vec5, i8 %res6, i32 6
 ; CHECK: vpcmporduq %xmm1, %xmm0, %k0 {%k1} ##
-  %res7 = call i8 @llvm.x86.avx512.mask.ucmp.q.128(<2 x i64> %a0, <2 x i64> %a1, i8 7, i8 %mask)
+  %res7 = call i8 @llvm.x86.avx512.mask.ucmp.q.128(<2 x i64> %a0, <2 x i64> %a1, i32 7, i8 %mask)
   %vec7 = insertelement <8 x i8> %vec6, i8 %res7, i32 7
   ret <8 x i8> %vec7
 }
 
-declare i8 @llvm.x86.avx512.mask.ucmp.q.128(<2 x i64>, <2 x i64>, i8, i8) nounwind readnone
+declare i8 @llvm.x86.avx512.mask.ucmp.q.128(<2 x i64>, <2 x i64>, i32, i8) nounwind readnone
 
 ; CHECK-LABEL: compr1
 ; CHECK: vcompresspd %zmm0
@@ -668,7 +668,7 @@ declare <4 x float> @llvm.x86.avx512.mask.compress.ps.128(<4 x float> %data, <4
 
 ; CHECK-LABEL: compr7
 ; CHECK-NOT: vcompress
-; CHECK: vmovapd
+; CHECK: vmovupd
 define void @compr7(i8* %addr, <8 x double> %data) {
   call void @llvm.x86.avx512.mask.compress.store.pd.512(i8* %addr, <8 x double> %data, i8 -1)
   ret void
@@ -757,7 +757,7 @@ declare <4 x float> @llvm.x86.avx512.mask.expand.ps.128(<4 x float> %data, <4 x
 
 ; CHECK-LABEL: expand7
 ; CHECK-NOT: vexpand
-; CHECK: vmovapd
+; CHECK: vmovupd
 define <8 x double> @expand7(i8* %addr, <8 x double> %data) {
   %res = call <8 x double> @llvm.x86.avx512.mask.expand.load.pd.512(i8* %addr, <8 x double> %data, i8 -1)
   ret <8 x double> %res
@@ -862,3 +862,3648 @@ define <2 x i64> @test_x86_mask_blend_q_128(i8 %a0, <2 x i64> %a1, <2 x i64> %a2
   ret <2 x i64> %res
 }
 declare <2 x i64> @llvm.x86.avx512.mask.blend.q.128(<2 x i64>, <2 x i64>, i8) nounwind readonly
+
+
+define < 2 x i64> @test_mask_mul_epi32_rr_128(< 4 x i32> %a, < 4 x i32> %b) {
+  ;CHECK-LABEL: test_mask_mul_epi32_rr_128
+  ;CHECK: vpmuldq %xmm1, %xmm0, %xmm0     ## encoding: [0x62,0xf2,0xfd,0x08,0x28,0xc1]
+  %res = call < 2 x i64> @llvm.x86.avx512.mask.pmul.dq.128(< 4 x i32> %a, < 4 x i32> %b, < 2 x i64> zeroinitializer, i8 -1)
+  ret < 2 x i64> %res
+}
+
+define < 2 x i64> @test_mask_mul_epi32_rrk_128(< 4 x i32> %a, < 4 x i32> %b, < 2 x i64> %passThru, i8 %mask) {
+  ;CHECK-LABEL: test_mask_mul_epi32_rrk_128
+  ;CHECK: vpmuldq %xmm1, %xmm0, %xmm2 {%k1} ## encoding: [0x62,0xf2,0xfd,0x09,0x28,0xd1]
+  %res = call < 2 x i64> @llvm.x86.avx512.mask.pmul.dq.128(< 4 x i32> %a, < 4 x i32> %b, < 2 x i64> %passThru, i8 %mask)
+  ret < 2 x i64> %res
+}
+
+define < 2 x i64> @test_mask_mul_epi32_rrkz_128(< 4 x i32> %a, < 4 x i32> %b, i8 %mask) {
+  ;CHECK-LABEL: test_mask_mul_epi32_rrkz_128
+  ;CHECK: vpmuldq %xmm1, %xmm0, %xmm0 {%k1} {z} ## encoding: [0x62,0xf2,0xfd,0x89,0x28,0xc1]
+  %res = call < 2 x i64> @llvm.x86.avx512.mask.pmul.dq.128(< 4 x i32> %a, < 4 x i32> %b, < 2 x i64> zeroinitializer, i8 %mask)
+  ret < 2 x i64> %res
+}
+
+define < 2 x i64> @test_mask_mul_epi32_rm_128(< 4 x i32> %a, < 4 x i32>* %ptr_b) {
+  ;CHECK-LABEL: test_mask_mul_epi32_rm_128
+  ;CHECK: vpmuldq (%rdi), %xmm0, %xmm0    ## encoding: [0x62,0xf2,0xfd,0x08,0x28,0x07]
+  %b = load < 4 x i32>, < 4 x i32>* %ptr_b
+  %res = call < 2 x i64> @llvm.x86.avx512.mask.pmul.dq.128(< 4 x i32> %a, < 4 x i32> %b, < 2 x i64> zeroinitializer, i8 -1)
+  ret < 2 x i64> %res
+}
+
+define < 2 x i64> @test_mask_mul_epi32_rmk_128(< 4 x i32> %a, < 4 x i32>* %ptr_b, < 2 x i64> %passThru, i8 %mask) {
+  ;CHECK-LABEL: test_mask_mul_epi32_rmk_128
+  ;CHECK: vpmuldq (%rdi), %xmm0, %xmm1 {%k1} ## encoding: [0x62,0xf2,0xfd,0x09,0x28,0x0f]
+  %b = load < 4 x i32>, < 4 x i32>* %ptr_b
+  %res = call < 2 x i64> @llvm.x86.avx512.mask.pmul.dq.128(< 4 x i32> %a, < 4 x i32> %b, < 2 x i64> %passThru, i8 %mask)
+  ret < 2 x i64> %res
+}
+
+define < 2 x i64> @test_mask_mul_epi32_rmkz_128(< 4 x i32> %a, < 4 x i32>* %ptr_b, i8 %mask) {
+  ;CHECK-LABEL: test_mask_mul_epi32_rmkz_128
+  ;CHECK: vpmuldq (%rdi), %xmm0, %xmm0 {%k1} {z} ## encoding: [0x62,0xf2,0xfd,0x89,0x28,0x07]
+  %b = load < 4 x i32>, < 4 x i32>* %ptr_b
+  %res = call < 2 x i64> @llvm.x86.avx512.mask.pmul.dq.128(< 4 x i32> %a, < 4 x i32> %b, < 2 x i64> zeroinitializer, i8 %mask)
+  ret < 2 x i64> %res
+}
+
+define < 2 x i64> @test_mask_mul_epi32_rmb_128(< 4 x i32> %a, i64* %ptr_b) {
+  ;CHECK-LABEL: test_mask_mul_epi32_rmb_128
+  ;CHECK: vpmuldq (%rdi){1to2}, %xmm0, %xmm0  ## encoding: [0x62,0xf2,0xfd,0x18,0x28,0x07]
+  %q = load i64, i64* %ptr_b
+  %vecinit.i = insertelement < 2 x i64> undef, i64 %q, i32 0
+  %b64 = shufflevector < 2 x i64> %vecinit.i, < 2 x i64> undef, <2 x i32> zeroinitializer
+  %b = bitcast < 2 x i64> %b64 to < 4 x i32>
+  %res = call < 2 x i64> @llvm.x86.avx512.mask.pmul.dq.128(< 4 x i32> %a, < 4 x i32> %b, < 2 x i64> zeroinitializer, i8 -1)
+  ret < 2 x i64> %res
+}
+
+define < 2 x i64> @test_mask_mul_epi32_rmbk_128(< 4 x i32> %a, i64* %ptr_b, < 2 x i64> %passThru, i8 %mask) {
+  ;CHECK-LABEL: test_mask_mul_epi32_rmbk_128
+  ;CHECK: vpmuldq (%rdi){1to2}, %xmm0, %xmm1 {%k1} ## encoding: [0x62,0xf2,0xfd,0x19,0x28,0x0f]
+  %q = load i64, i64* %ptr_b
+  %vecinit.i = insertelement < 2 x i64> undef, i64 %q, i32 0
+  %b64 = shufflevector < 2 x i64> %vecinit.i, < 2 x i64> undef, <2 x i32> zeroinitializer
+  %b = bitcast < 2 x i64> %b64 to < 4 x i32>
+  %res = call < 2 x i64> @llvm.x86.avx512.mask.pmul.dq.128(< 4 x i32> %a, < 4 x i32> %b, < 2 x i64> %passThru, i8 %mask)
+  ret < 2 x i64> %res
+}
+
+define < 2 x i64> @test_mask_mul_epi32_rmbkz_128(< 4 x i32> %a, i64* %ptr_b, i8 %mask) {
+  ;CHECK-LABEL: test_mask_mul_epi32_rmbkz_128
+  ;CHECK: vpmuldq (%rdi){1to2}, %xmm0, %xmm0 {%k1} {z} ## encoding: [0x62,0xf2,0xfd,0x99,0x28,0x07]
+  %q = load i64, i64* %ptr_b
+  %vecinit.i = insertelement < 2 x i64> undef, i64 %q, i32 0
+  %b64 = shufflevector < 2 x i64> %vecinit.i, < 2 x i64> undef, < 2 x i32> zeroinitializer
+  %b = bitcast < 2 x i64> %b64 to < 4 x i32>
+  %res = call < 2 x i64> @llvm.x86.avx512.mask.pmul.dq.128(< 4 x i32> %a, < 4 x i32> %b, < 2 x i64> zeroinitializer, i8 %mask)
+  ret < 2 x i64> %res
+}
+
+declare < 2 x i64> @llvm.x86.avx512.mask.pmul.dq.128(< 4 x i32>, < 4 x i32>, < 2 x i64>, i8)
+
+define < 4 x i64> @test_mask_mul_epi32_rr_256(< 8 x i32> %a, < 8 x i32> %b) {
+  ;CHECK-LABEL: test_mask_mul_epi32_rr_256
+  ;CHECK: vpmuldq %ymm1, %ymm0, %ymm0     ## encoding: [0x62,0xf2,0xfd,0x28,0x28,0xc1]
+  %res = call < 4 x i64> @llvm.x86.avx512.mask.pmul.dq.256(< 8 x i32> %a, < 8 x i32> %b, < 4 x i64> zeroinitializer, i8 -1)
+  ret < 4 x i64> %res
+}
+
+define < 4 x i64> @test_mask_mul_epi32_rrk_256(< 8 x i32> %a, < 8 x i32> %b, < 4 x i64> %passThru, i8 %mask) {
+  ;CHECK-LABEL: test_mask_mul_epi32_rrk_256
+  ;CHECK: vpmuldq %ymm1, %ymm0, %ymm2 {%k1} ## encoding: [0x62,0xf2,0xfd,0x29,0x28,0xd1]
+  %res = call < 4 x i64> @llvm.x86.avx512.mask.pmul.dq.256(< 8 x i32> %a, < 8 x i32> %b, < 4 x i64> %passThru, i8 %mask)
+  ret < 4 x i64> %res
+}
+
+define < 4 x i64> @test_mask_mul_epi32_rrkz_256(< 8 x i32> %a, < 8 x i32> %b, i8 %mask) {
+  ;CHECK-LABEL: test_mask_mul_epi32_rrkz_256
+  ;CHECK: vpmuldq %ymm1, %ymm0, %ymm0 {%k1} {z} ## encoding: [0x62,0xf2,0xfd,0xa9,0x28,0xc1]
+  %res = call < 4 x i64> @llvm.x86.avx512.mask.pmul.dq.256(< 8 x i32> %a, < 8 x i32> %b, < 4 x i64> zeroinitializer, i8 %mask)
+  ret < 4 x i64> %res
+}
+
+define < 4 x i64> @test_mask_mul_epi32_rm_256(< 8 x i32> %a, < 8 x i32>* %ptr_b) {
+  ;CHECK-LABEL: test_mask_mul_epi32_rm_256
+  ;CHECK: vpmuldq (%rdi), %ymm0, %ymm0    ## encoding: [0x62,0xf2,0xfd,0x28,0x28,0x07]
+  %b = load < 8 x i32>, < 8 x i32>* %ptr_b
+  %res = call < 4 x i64> @llvm.x86.avx512.mask.pmul.dq.256(< 8 x i32> %a, < 8 x i32> %b, < 4 x i64> zeroinitializer, i8 -1)
+  ret < 4 x i64> %res
+}
+
+define < 4 x i64> @test_mask_mul_epi32_rmk_256(< 8 x i32> %a, < 8 x i32>* %ptr_b, < 4 x i64> %passThru, i8 %mask) {
+  ;CHECK-LABEL: test_mask_mul_epi32_rmk_256
+  ;CHECK: vpmuldq (%rdi), %ymm0, %ymm1 {%k1} ## encoding: [0x62,0xf2,0xfd,0x29,0x28,0x0f]
+  %b = load < 8 x i32>, < 8 x i32>* %ptr_b
+  %res = call < 4 x i64> @llvm.x86.avx512.mask.pmul.dq.256(< 8 x i32> %a, < 8 x i32> %b, < 4 x i64> %passThru, i8 %mask)
+  ret < 4 x i64> %res
+}
+
+define < 4 x i64> @test_mask_mul_epi32_rmkz_256(< 8 x i32> %a, < 8 x i32>* %ptr_b, i8 %mask) {
+  ;CHECK-LABEL: test_mask_mul_epi32_rmkz_256
+  ;CHECK: vpmuldq (%rdi), %ymm0, %ymm0 {%k1} {z} ## encoding: [0x62,0xf2,0xfd,0xa9,0x28,0x07]
+  %b = load < 8 x i32>, < 8 x i32>* %ptr_b
+  %res = call < 4 x i64> @llvm.x86.avx512.mask.pmul.dq.256(< 8 x i32> %a, < 8 x i32> %b, < 4 x i64> zeroinitializer, i8 %mask)
+  ret < 4 x i64> %res
+}
+
+define < 4 x i64> @test_mask_mul_epi32_rmb_256(< 8 x i32> %a, i64* %ptr_b) {
+  ;CHECK-LABEL: test_mask_mul_epi32_rmb_256
+  ;CHECK: vpmuldq (%rdi){1to4}, %ymm0, %ymm0  ## encoding: [0x62,0xf2,0xfd,0x38,0x28,0x07]
+  %q = load i64, i64* %ptr_b
+  %vecinit.i = insertelement < 4 x i64> undef, i64 %q, i32 0
+  %b64 = shufflevector < 4 x i64> %vecinit.i, < 4 x i64> undef, < 4 x i32> zeroinitializer
+  %b = bitcast < 4 x i64> %b64 to < 8 x i32>
+  %res = call < 4 x i64> @llvm.x86.avx512.mask.pmul.dq.256(< 8 x i32> %a, < 8 x i32> %b, < 4 x i64> zeroinitializer, i8 -1)
+  ret < 4 x i64> %res
+}
+
+define < 4 x i64> @test_mask_mul_epi32_rmbk_256(< 8 x i32> %a, i64* %ptr_b, < 4 x i64> %passThru, i8 %mask) {
+  ;CHECK-LABEL: test_mask_mul_epi32_rmbk_256
+  ;CHECK: vpmuldq (%rdi){1to4}, %ymm0, %ymm1 {%k1} ## encoding: [0x62,0xf2,0xfd,0x39,0x28,0x0f]
+  %q = load i64, i64* %ptr_b
+  %vecinit.i = insertelement < 4 x i64> undef, i64 %q, i32 0
+  %b64 = shufflevector < 4 x i64> %vecinit.i, < 4 x i64> undef, < 4 x i32> zeroinitializer
+  %b = bitcast < 4 x i64> %b64 to < 8 x i32>
+  %res = call < 4 x i64> @llvm.x86.avx512.mask.pmul.dq.256(< 8 x i32> %a, < 8 x i32> %b, < 4 x i64> %passThru, i8 %mask)
+  ret < 4 x i64> %res
+}
+
+define < 4 x i64> @test_mask_mul_epi32_rmbkz_256(< 8 x i32> %a, i64* %ptr_b, i8 %mask) {
+  ;CHECK-LABEL: test_mask_mul_epi32_rmbkz_256
+  ;CHECK: vpmuldq (%rdi){1to4}, %ymm0, %ymm0 {%k1} {z} ## encoding: [0x62,0xf2,0xfd,0xb9,0x28,0x07]
+  %q = load i64, i64* %ptr_b
+  %vecinit.i = insertelement < 4 x i64> undef, i64 %q, i32 0
+  %b64 = shufflevector < 4 x i64> %vecinit.i, < 4 x i64> undef, < 4 x i32> zeroinitializer
+  %b = bitcast < 4 x i64> %b64 to < 8 x i32>
+  %res = call < 4 x i64> @llvm.x86.avx512.mask.pmul.dq.256(< 8 x i32> %a, < 8 x i32> %b, < 4 x i64> zeroinitializer, i8 %mask)
+  ret < 4 x i64> %res
+}
+
+declare < 4 x i64> @llvm.x86.avx512.mask.pmul.dq.256(< 8 x i32>, < 8 x i32>, < 4 x i64>, i8)
+
+define < 2 x i64> @test_mask_mul_epu32_rr_128(< 4 x i32> %a, < 4 x i32> %b) {
+  ;CHECK-LABEL: test_mask_mul_epu32_rr_128
+  ;CHECK: vpmuludq %xmm1, %xmm0, %xmm0 ## encoding: [0xc5,0xf9,0xf4,0xc1]
+  %res = call < 2 x i64> @llvm.x86.avx512.mask.pmulu.dq.128(< 4 x i32> %a, < 4 x i32> %b, < 2 x i64> zeroinitializer, i8 -1)
+  ret < 2 x i64> %res
+}
+
+define < 2 x i64> @test_mask_mul_epu32_rrk_128(< 4 x i32> %a, < 4 x i32> %b, < 2 x i64> %passThru, i8 %mask) {
+  ;CHECK-LABEL: test_mask_mul_epu32_rrk_128
+  ;CHECK: vpmuludq %xmm1, %xmm0, %xmm2 {%k1} ## encoding: [0x62,0xf1,0xfd,0x09,0xf4,0xd1]
+  %res = call < 2 x i64> @llvm.x86.avx512.mask.pmulu.dq.128(< 4 x i32> %a, < 4 x i32> %b, < 2 x i64> %passThru, i8 %mask)
+  ret < 2 x i64> %res
+}
+
+define < 2 x i64> @test_mask_mul_epu32_rrkz_128(< 4 x i32> %a, < 4 x i32> %b, i8 %mask) {
+  ;CHECK-LABEL: test_mask_mul_epu32_rrkz_128
+  ;CHECK: vpmuludq %xmm1, %xmm0, %xmm0 {%k1} {z} ## encoding: [0x62,0xf1,0xfd,0x89,0xf4,0xc1]
+  %res = call < 2 x i64> @llvm.x86.avx512.mask.pmulu.dq.128(< 4 x i32> %a, < 4 x i32> %b, < 2 x i64> zeroinitializer, i8 %mask)
+  ret < 2 x i64> %res
+}
+
+define < 2 x i64> @test_mask_mul_epu32_rm_128(< 4 x i32> %a, < 4 x i32>* %ptr_b) {
+  ;CHECK-LABEL: test_mask_mul_epu32_rm_128
+  ;CHECK: vpmuludq (%rdi), %xmm0, %xmm0 ## encoding: [0xc5,0xf9,0xf4,0x07]
+  %b = load < 4 x i32>, < 4 x i32>* %ptr_b
+  %res = call < 2 x i64> @llvm.x86.avx512.mask.pmulu.dq.128(< 4 x i32> %a, < 4 x i32> %b, < 2 x i64> zeroinitializer, i8 -1)
+  ret < 2 x i64> %res
+}
+
+define < 2 x i64> @test_mask_mul_epu32_rmk_128(< 4 x i32> %a, < 4 x i32>* %ptr_b, < 2 x i64> %passThru, i8 %mask) {
+  ;CHECK-LABEL: test_mask_mul_epu32_rmk_128
+  ;CHECK: vpmuludq (%rdi), %xmm0, %xmm1 {%k1} ## encoding: [0x62,0xf1,0xfd,0x09,0xf4,0x0f]
+  %b = load < 4 x i32>, < 4 x i32>* %ptr_b
+  %res = call < 2 x i64> @llvm.x86.avx512.mask.pmulu.dq.128(< 4 x i32> %a, < 4 x i32> %b, < 2 x i64> %passThru, i8 %mask)
+  ret < 2 x i64> %res
+}
+
+define < 2 x i64> @test_mask_mul_epu32_rmkz_128(< 4 x i32> %a, < 4 x i32>* %ptr_b, i8 %mask) {
+  ;CHECK-LABEL: test_mask_mul_epu32_rmkz_128
+  ;CHECK: vpmuludq (%rdi), %xmm0, %xmm0 {%k1} {z} ## encoding: [0x62,0xf1,0xfd,0x89,0xf4,0x07]
+  %b = load < 4 x i32>, < 4 x i32>* %ptr_b
+  %res = call < 2 x i64> @llvm.x86.avx512.mask.pmulu.dq.128(< 4 x i32> %a, < 4 x i32> %b, < 2 x i64> zeroinitializer, i8 %mask)
+  ret < 2 x i64> %res
+}
+
+define < 2 x i64> @test_mask_mul_epu32_rmb_128(< 4 x i32> %a, i64* %ptr_b) {
+  ;CHECK-LABEL: test_mask_mul_epu32_rmb_128
+  ;CHECK: vpmuludq (%rdi){1to2}, %xmm0, %xmm0  ## encoding: [0x62,0xf1,0xfd,0x18,0xf4,0x07]
+  %q = load i64, i64* %ptr_b
+  %vecinit.i = insertelement < 2 x i64> undef, i64 %q, i32 0
+  %b64 = shufflevector < 2 x i64> %vecinit.i, < 2 x i64> undef, <2 x i32> zeroinitializer
+  %b = bitcast < 2 x i64> %b64 to < 4 x i32>
+  %res = call < 2 x i64> @llvm.x86.avx512.mask.pmulu.dq.128(< 4 x i32> %a, < 4 x i32> %b, < 2 x i64> zeroinitializer, i8 -1)
+  ret < 2 x i64> %res
+}
+
+define < 2 x i64> @test_mask_mul_epu32_rmbk_128(< 4 x i32> %a, i64* %ptr_b, < 2 x i64> %passThru, i8 %mask) {
+  ;CHECK-LABEL: test_mask_mul_epu32_rmbk_128
+  ;CHECK: vpmuludq (%rdi){1to2}, %xmm0, %xmm1 {%k1} ## encoding: [0x62,0xf1,0xfd,0x19,0xf4,0x0f]
+  %q = load i64, i64* %ptr_b
+  %vecinit.i = insertelement < 2 x i64> undef, i64 %q, i32 0
+  %b64 = shufflevector < 2 x i64> %vecinit.i, < 2 x i64> undef, <2 x i32> zeroinitializer
+  %b = bitcast < 2 x i64> %b64 to < 4 x i32>
+  %res = call < 2 x i64> @llvm.x86.avx512.mask.pmulu.dq.128(< 4 x i32> %a, < 4 x i32> %b, < 2 x i64> %passThru, i8 %mask)
+  ret < 2 x i64> %res
+}
+
+define < 2 x i64> @test_mask_mul_epu32_rmbkz_128(< 4 x i32> %a, i64* %ptr_b, i8 %mask) {
+  ;CHECK-LABEL: test_mask_mul_epu32_rmbkz_128
+  ;CHECK: vpmuludq (%rdi){1to2}, %xmm0, %xmm0 {%k1} {z} ## encoding: [0x62,0xf1,0xfd,0x99,0xf4,0x07]
+  %q = load i64, i64* %ptr_b
+  %vecinit.i = insertelement < 2 x i64> undef, i64 %q, i32 0
+  %b64 = shufflevector < 2 x i64> %vecinit.i, < 2 x i64> undef, < 2 x i32> zeroinitializer
+  %b = bitcast < 2 x i64> %b64 to < 4 x i32>
+  %res = call < 2 x i64> @llvm.x86.avx512.mask.pmulu.dq.128(< 4 x i32> %a, < 4 x i32> %b, < 2 x i64> zeroinitializer, i8 %mask)
+  ret < 2 x i64> %res
+}
+
+declare < 2 x i64> @llvm.x86.avx512.mask.pmulu.dq.128(< 4 x i32>, < 4 x i32>, < 2 x i64>, i8)
+
+define < 4 x i64> @test_mask_mul_epu32_rr_256(< 8 x i32> %a, < 8 x i32> %b) {
+  ;CHECK-LABEL: test_mask_mul_epu32_rr_256
+  ;CHECK: vpmuludq %ymm1, %ymm0, %ymm0 ## encoding: [0xc5,0xfd,0xf4,0xc1]
+  %res = call < 4 x i64> @llvm.x86.avx512.mask.pmulu.dq.256(< 8 x i32> %a, < 8 x i32> %b, < 4 x i64> zeroinitializer, i8 -1)
+  ret < 4 x i64> %res
+}
+
+define < 4 x i64> @test_mask_mul_epu32_rrk_256(< 8 x i32> %a, < 8 x i32> %b, < 4 x i64> %passThru, i8 %mask) {
+  ;CHECK-LABEL: test_mask_mul_epu32_rrk_256
+  ;CHECK: vpmuludq %ymm1, %ymm0, %ymm2 {%k1} ## encoding: [0x62,0xf1,0xfd,0x29,0xf4,0xd1]
+  %res = call < 4 x i64> @llvm.x86.avx512.mask.pmulu.dq.256(< 8 x i32> %a, < 8 x i32> %b, < 4 x i64> %passThru, i8 %mask)
+  ret < 4 x i64> %res
+}
+
+define < 4 x i64> @test_mask_mul_epu32_rrkz_256(< 8 x i32> %a, < 8 x i32> %b, i8 %mask) {
+  ;CHECK-LABEL: test_mask_mul_epu32_rrkz_256
+  ;CHECK: vpmuludq %ymm1, %ymm0, %ymm0 {%k1} {z} ## encoding: [0x62,0xf1,0xfd,0xa9,0xf4,0xc1]
+  %res = call < 4 x i64> @llvm.x86.avx512.mask.pmulu.dq.256(< 8 x i32> %a, < 8 x i32> %b, < 4 x i64> zeroinitializer, i8 %mask)
+  ret < 4 x i64> %res
+}
+
+define < 4 x i64> @test_mask_mul_epu32_rm_256(< 8 x i32> %a, < 8 x i32>* %ptr_b) {
+  ;CHECK-LABEL: test_mask_mul_epu32_rm_256
+  ;CHECK: vpmuludq (%rdi), %ymm0, %ymm0 ## encoding: [0xc5,0xfd,0xf4,0x07]
+  %b = load < 8 x i32>, < 8 x i32>* %ptr_b
+  %res = call < 4 x i64> @llvm.x86.avx512.mask.pmulu.dq.256(< 8 x i32> %a, < 8 x i32> %b, < 4 x i64> zeroinitializer, i8 -1)
+  ret < 4 x i64> %res
+}
+
+define < 4 x i64> @test_mask_mul_epu32_rmk_256(< 8 x i32> %a, < 8 x i32>* %ptr_b, < 4 x i64> %passThru, i8 %mask) {
+  ;CHECK-LABEL: test_mask_mul_epu32_rmk_256
+  ;CHECK: vpmuludq (%rdi), %ymm0, %ymm1 {%k1} ## encoding: [0x62,0xf1,0xfd,0x29,0xf4,0x0f]
+  %b = load < 8 x i32>, < 8 x i32>* %ptr_b
+  %res = call < 4 x i64> @llvm.x86.avx512.mask.pmulu.dq.256(< 8 x i32> %a, < 8 x i32> %b, < 4 x i64> %passThru, i8 %mask)
+  ret < 4 x i64> %res
+}
+
+define < 4 x i64> @test_mask_mul_epu32_rmkz_256(< 8 x i32> %a, < 8 x i32>* %ptr_b, i8 %mask) {
+  ;CHECK-LABEL: test_mask_mul_epu32_rmkz_256
+  ;CHECK: vpmuludq (%rdi), %ymm0, %ymm0 {%k1} {z} ## encoding: [0x62,0xf1,0xfd,0xa9,0xf4,0x07]
+  %b = load < 8 x i32>, < 8 x i32>* %ptr_b
+  %res = call < 4 x i64> @llvm.x86.avx512.mask.pmulu.dq.256(< 8 x i32> %a, < 8 x i32> %b, < 4 x i64> zeroinitializer, i8 %mask)
+  ret < 4 x i64> %res
+}
+
+define < 4 x i64> @test_mask_mul_epu32_rmb_256(< 8 x i32> %a, i64* %ptr_b) {
+  ;CHECK-LABEL: test_mask_mul_epu32_rmb_256
+  ;CHECK: vpmuludq (%rdi){1to4}, %ymm0, %ymm0  ## encoding: [0x62,0xf1,0xfd,0x38,0xf4,0x07]
+  %q = load i64, i64* %ptr_b
+  %vecinit.i = insertelement < 4 x i64> undef, i64 %q, i32 0
+  %b64 = shufflevector < 4 x i64> %vecinit.i, < 4 x i64> undef, < 4 x i32> zeroinitializer
+  %b = bitcast < 4 x i64> %b64 to < 8 x i32>
+  %res = call < 4 x i64> @llvm.x86.avx512.mask.pmulu.dq.256(< 8 x i32> %a, < 8 x i32> %b, < 4 x i64> zeroinitializer, i8 -1)
+  ret < 4 x i64> %res
+}
+
+define < 4 x i64> @test_mask_mul_epu32_rmbk_256(< 8 x i32> %a, i64* %ptr_b, < 4 x i64> %passThru, i8 %mask) {
+  ;CHECK-LABEL: test_mask_mul_epu32_rmbk_256
+  ;CHECK: vpmuludq (%rdi){1to4}, %ymm0, %ymm1 {%k1} ## encoding: [0x62,0xf1,0xfd,0x39,0xf4,0x0f]
+  %q = load i64, i64* %ptr_b
+  %vecinit.i = insertelement < 4 x i64> undef, i64 %q, i32 0
+  %b64 = shufflevector < 4 x i64> %vecinit.i, < 4 x i64> undef, < 4 x i32> zeroinitializer
+  %b = bitcast < 4 x i64> %b64 to < 8 x i32>
+  %res = call < 4 x i64> @llvm.x86.avx512.mask.pmulu.dq.256(< 8 x i32> %a, < 8 x i32> %b, < 4 x i64> %passThru, i8 %mask)
+  ret < 4 x i64> %res
+}
+
+define < 4 x i64> @test_mask_mul_epu32_rmbkz_256(< 8 x i32> %a, i64* %ptr_b, i8 %mask) {
+  ;CHECK-LABEL: test_mask_mul_epu32_rmbkz_256
+  ;CHECK: vpmuludq (%rdi){1to4}, %ymm0, %ymm0 {%k1} {z} ## encoding: [0x62,0xf1,0xfd,0xb9,0xf4,0x07]
+  %q = load i64, i64* %ptr_b
+  %vecinit.i = insertelement < 4 x i64> undef, i64 %q, i32 0
+  %b64 = shufflevector < 4 x i64> %vecinit.i, < 4 x i64> undef, < 4 x i32> zeroinitializer
+  %b = bitcast < 4 x i64> %b64 to < 8 x i32>
+  %res = call < 4 x i64> @llvm.x86.avx512.mask.pmulu.dq.256(< 8 x i32> %a, < 8 x i32> %b, < 4 x i64> zeroinitializer, i8 %mask)
+  ret < 4 x i64> %res
+}
+
+declare < 4 x i64> @llvm.x86.avx512.mask.pmulu.dq.256(< 8 x i32>, < 8 x i32>, < 4 x i64>, i8)
+
+define <4 x i32> @test_mask_add_epi32_rr_128(<4 x i32> %a, <4 x i32> %b) {
+  ;CHECK-LABEL: test_mask_add_epi32_rr_128
+  ;CHECK: vpaddd %xmm1, %xmm0, %xmm0     ## encoding: [0x62,0xf1,0x7d,0x08,0xfe,0xc1]
+  %res = call <4 x i32> @llvm.x86.avx512.mask.padd.d.128(<4 x i32> %a, <4 x i32> %b, <4 x i32> zeroinitializer, i8 -1)
+  ret <4 x i32> %res
+}
+
+define <4 x i32> @test_mask_add_epi32_rrk_128(<4 x i32> %a, <4 x i32> %b, <4 x i32> %passThru, i8 %mask) {
+  ;CHECK-LABEL: test_mask_add_epi32_rrk_128
+  ;CHECK: vpaddd %xmm1, %xmm0, %xmm2 {%k1} ## encoding: [0x62,0xf1,0x7d,0x09,0xfe,0xd1]
+  %res = call <4 x i32> @llvm.x86.avx512.mask.padd.d.128(<4 x i32> %a, <4 x i32> %b, <4 x i32> %passThru, i8 %mask)
+  ret <4 x i32> %res
+}
+
+define <4 x i32> @test_mask_add_epi32_rrkz_128(<4 x i32> %a, <4 x i32> %b, i8 %mask) {
+  ;CHECK-LABEL: test_mask_add_epi32_rrkz_128
+  ;CHECK: vpaddd %xmm1, %xmm0, %xmm0 {%k1} {z} ## encoding: [0x62,0xf1,0x7d,0x89,0xfe,0xc1]
+  %res = call <4 x i32> @llvm.x86.avx512.mask.padd.d.128(<4 x i32> %a, <4 x i32> %b, <4 x i32> zeroinitializer, i8 %mask)
+  ret <4 x i32> %res
+}
+
+define <4 x i32> @test_mask_add_epi32_rm_128(<4 x i32> %a, <4 x i32>* %ptr_b) {
+  ;CHECK-LABEL: test_mask_add_epi32_rm_128
+  ;CHECK: vpaddd (%rdi), %xmm0, %xmm0    ## encoding: [0x62,0xf1,0x7d,0x08,0xfe,0x07]
+  %b = load <4 x i32>, <4 x i32>* %ptr_b
+  %res = call <4 x i32> @llvm.x86.avx512.mask.padd.d.128(<4 x i32> %a, <4 x i32> %b, <4 x i32> zeroinitializer, i8 -1)
+  ret <4 x i32> %res
+}
+
+define <4 x i32> @test_mask_add_epi32_rmk_128(<4 x i32> %a, <4 x i32>* %ptr_b, <4 x i32> %passThru, i8 %mask) {
+  ;CHECK-LABEL: test_mask_add_epi32_rmk_128
+  ;CHECK: vpaddd (%rdi), %xmm0, %xmm1 {%k1} ## encoding: [0x62,0xf1,0x7d,0x09,0xfe,0x0f]
+  %b = load <4 x i32>, <4 x i32>* %ptr_b
+  %res = call <4 x i32> @llvm.x86.avx512.mask.padd.d.128(<4 x i32> %a, <4 x i32> %b, <4 x i32> %passThru, i8 %mask)
+  ret <4 x i32> %res
+}
+
+define <4 x i32> @test_mask_add_epi32_rmkz_128(<4 x i32> %a, <4 x i32>* %ptr_b, i8 %mask) {
+  ;CHECK-LABEL: test_mask_add_epi32_rmkz_128
+  ;CHECK: vpaddd (%rdi), %xmm0, %xmm0 {%k1} {z} ## encoding: [0x62,0xf1,0x7d,0x89,0xfe,0x07]
+  %b = load <4 x i32>, <4 x i32>* %ptr_b
+  %res = call <4 x i32> @llvm.x86.avx512.mask.padd.d.128(<4 x i32> %a, <4 x i32> %b, <4 x i32> zeroinitializer, i8 %mask)
+  ret <4 x i32> %res
+}
+
+define <4 x i32> @test_mask_add_epi32_rmb_128(<4 x i32> %a, i32* %ptr_b) {
+  ;CHECK-LABEL: test_mask_add_epi32_rmb_128
+  ;CHECK: vpaddd (%rdi){1to4}, %xmm0, %xmm0  ## encoding: [0x62,0xf1,0x7d,0x18,0xfe,0x07]
+  %q = load i32, i32* %ptr_b
+  %vecinit.i = insertelement <4 x i32> undef, i32 %q, i32 0
+  %b = shufflevector <4 x i32> %vecinit.i, <4 x i32> undef, <4 x i32> zeroinitializer
+  %res = call <4 x i32> @llvm.x86.avx512.mask.padd.d.128(<4 x i32> %a, <4 x i32> %b, <4 x i32> zeroinitializer, i8 -1)
+  ret <4 x i32> %res
+}
+
+define <4 x i32> @test_mask_add_epi32_rmbk_128(<4 x i32> %a, i32* %ptr_b, <4 x i32> %passThru, i8 %mask) {
+  ;CHECK-LABEL: test_mask_add_epi32_rmbk_128
+  ;CHECK: vpaddd (%rdi){1to4}, %xmm0, %xmm1 {%k1} ## encoding: [0x62,0xf1,0x7d,0x19,0xfe,0x0f]
+  %q = load i32, i32* %ptr_b
+  %vecinit.i = insertelement <4 x i32> undef, i32 %q, i32 0
+  %b = shufflevector <4 x i32> %vecinit.i, <4 x i32> undef, <4 x i32> zeroinitializer
+  %res = call <4 x i32> @llvm.x86.avx512.mask.padd.d.128(<4 x i32> %a, <4 x i32> %b, <4 x i32> %passThru, i8 %mask)
+  ret <4 x i32> %res
+}
+
+define <4 x i32> @test_mask_add_epi32_rmbkz_128(<4 x i32> %a, i32* %ptr_b, i8 %mask) {
+  ;CHECK-LABEL: test_mask_add_epi32_rmbkz_128
+  ;CHECK: vpaddd (%rdi){1to4}, %xmm0, %xmm0 {%k1} {z} ## encoding: [0x62,0xf1,0x7d,0x99,0xfe,0x07]
+  %q = load i32, i32* %ptr_b
+  %vecinit.i = insertelement <4 x i32> undef, i32 %q, i32 0
+  %b = shufflevector <4 x i32> %vecinit.i, <4 x i32> undef, <4 x i32> zeroinitializer
+  %res = call <4 x i32> @llvm.x86.avx512.mask.padd.d.128(<4 x i32> %a, <4 x i32> %b, <4 x i32> zeroinitializer, i8 %mask)
+  ret <4 x i32> %res
+}
+
+declare <4 x i32> @llvm.x86.avx512.mask.padd.d.128(<4 x i32>, <4 x i32>, <4 x i32>, i8)
+
+define <4 x i32> @test_mask_sub_epi32_rr_128(<4 x i32> %a, <4 x i32> %b) {
+  ;CHECK-LABEL: test_mask_sub_epi32_rr_128
+  ;CHECK: vpsubd %xmm1, %xmm0, %xmm0     ## encoding: [0x62,0xf1,0x7d,0x08,0xfa,0xc1]
+  %res = call <4 x i32> @llvm.x86.avx512.mask.psub.d.128(<4 x i32> %a, <4 x i32> %b, <4 x i32> zeroinitializer, i8 -1)
+  ret <4 x i32> %res
+}
+
+define <4 x i32> @test_mask_sub_epi32_rrk_128(<4 x i32> %a, <4 x i32> %b, <4 x i32> %passThru, i8 %mask) {
+  ;CHECK-LABEL: test_mask_sub_epi32_rrk_128
+  ;CHECK: vpsubd %xmm1, %xmm0, %xmm2 {%k1} ## encoding: [0x62,0xf1,0x7d,0x09,0xfa,0xd1]
+  %res = call <4 x i32> @llvm.x86.avx512.mask.psub.d.128(<4 x i32> %a, <4 x i32> %b, <4 x i32> %passThru, i8 %mask)
+  ret <4 x i32> %res
+}
+
+define <4 x i32> @test_mask_sub_epi32_rrkz_128(<4 x i32> %a, <4 x i32> %b, i8 %mask) {
+  ;CHECK-LABEL: test_mask_sub_epi32_rrkz_128
+  ;CHECK: vpsubd %xmm1, %xmm0, %xmm0 {%k1} {z} ## encoding: [0x62,0xf1,0x7d,0x89,0xfa,0xc1]
+  %res = call <4 x i32> @llvm.x86.avx512.mask.psub.d.128(<4 x i32> %a, <4 x i32> %b, <4 x i32> zeroinitializer, i8 %mask)
+  ret <4 x i32> %res
+}
+
+define <4 x i32> @test_mask_sub_epi32_rm_128(<4 x i32> %a, <4 x i32>* %ptr_b) {
+  ;CHECK-LABEL: test_mask_sub_epi32_rm_128
+  ;CHECK: (%rdi), %xmm0, %xmm0    ## encoding: [0x62,0xf1,0x7d,0x08,0xfa,0x07]
+  %b = load <4 x i32>, <4 x i32>* %ptr_b
+  %res = call <4 x i32> @llvm.x86.avx512.mask.psub.d.128(<4 x i32> %a, <4 x i32> %b, <4 x i32> zeroinitializer, i8 -1)
+  ret <4 x i32> %res
+}
+
+define <4 x i32> @test_mask_sub_epi32_rmk_128(<4 x i32> %a, <4 x i32>* %ptr_b, <4 x i32> %passThru, i8 %mask) {
+  ;CHECK-LABEL: test_mask_sub_epi32_rmk_128
+  ;CHECK: vpsubd (%rdi), %xmm0, %xmm1 {%k1} ## encoding: [0x62,0xf1,0x7d,0x09,0xfa,0x0f]
+  %b = load <4 x i32>, <4 x i32>* %ptr_b
+  %res = call <4 x i32> @llvm.x86.avx512.mask.psub.d.128(<4 x i32> %a, <4 x i32> %b, <4 x i32> %passThru, i8 %mask)
+  ret <4 x i32> %res
+}
+
+define <4 x i32> @test_mask_sub_epi32_rmkz_128(<4 x i32> %a, <4 x i32>* %ptr_b, i8 %mask) {
+  ;CHECK-LABEL: test_mask_sub_epi32_rmkz_128
+  ;CHECK: vpsubd (%rdi), %xmm0, %xmm0 {%k1} {z} ## encoding: [0x62,0xf1,0x7d,0x89,0xfa,0x07]
+  %b = load <4 x i32>, <4 x i32>* %ptr_b
+  %res = call <4 x i32> @llvm.x86.avx512.mask.psub.d.128(<4 x i32> %a, <4 x i32> %b, <4 x i32> zeroinitializer, i8 %mask)
+  ret <4 x i32> %res
+}
+
+define <4 x i32> @test_mask_sub_epi32_rmb_128(<4 x i32> %a, i32* %ptr_b) {
+  ;CHECK-LABEL: test_mask_sub_epi32_rmb_128
+  ;CHECK: vpsubd (%rdi){1to4}, %xmm0, %xmm0  ## encoding: [0x62,0xf1,0x7d,0x18,0xfa,0x07]
+  %q = load i32, i32* %ptr_b
+  %vecinit.i = insertelement <4 x i32> undef, i32 %q, i32 0
+  %b = shufflevector <4 x i32> %vecinit.i, <4 x i32> undef, <4 x i32> zeroinitializer
+  %res = call <4 x i32> @llvm.x86.avx512.mask.psub.d.128(<4 x i32> %a, <4 x i32> %b, <4 x i32> zeroinitializer, i8 -1)
+  ret <4 x i32> %res
+}
+
+define <4 x i32> @test_mask_sub_epi32_rmbk_128(<4 x i32> %a, i32* %ptr_b, <4 x i32> %passThru, i8 %mask) {
+  ;CHECK-LABEL: test_mask_sub_epi32_rmbk_128
+  ;CHECK: vpsubd (%rdi){1to4}, %xmm0, %xmm1 {%k1} ## encoding: [0x62,0xf1,0x7d,0x19,0xfa,0x0f]
+  %q = load i32, i32* %ptr_b
+  %vecinit.i = insertelement <4 x i32> undef, i32 %q, i32 0
+  %b = shufflevector <4 x i32> %vecinit.i, <4 x i32> undef, <4 x i32> zeroinitializer
+  %res = call <4 x i32> @llvm.x86.avx512.mask.psub.d.128(<4 x i32> %a, <4 x i32> %b, <4 x i32> %passThru, i8 %mask)
+  ret <4 x i32> %res
+}
+
+define <4 x i32> @test_mask_sub_epi32_rmbkz_128(<4 x i32> %a, i32* %ptr_b, i8 %mask) {
+  ;CHECK-LABEL: test_mask_sub_epi32_rmbkz_128
+  ;CHECK: vpsubd (%rdi){1to4}, %xmm0, %xmm0 {%k1} {z} ## encoding: [0x62,0xf1,0x7d,0x99,0xfa,0x07]
+  %q = load i32, i32* %ptr_b
+  %vecinit.i = insertelement <4 x i32> undef, i32 %q, i32 0
+  %b = shufflevector <4 x i32> %vecinit.i, <4 x i32> undef, <4 x i32> zeroinitializer
+  %res = call <4 x i32> @llvm.x86.avx512.mask.psub.d.128(<4 x i32> %a, <4 x i32> %b, <4 x i32> zeroinitializer, i8 %mask)
+  ret <4 x i32> %res
+}
+
+declare <4 x i32> @llvm.x86.avx512.mask.psub.d.128(<4 x i32>, <4 x i32>, <4 x i32>, i8)
+
+define <8 x i32> @test_mask_sub_epi32_rr_256(<8 x i32> %a, <8 x i32> %b) {
+  ;CHECK-LABEL: test_mask_sub_epi32_rr_256
+  ;CHECK: vpsubd %ymm1, %ymm0, %ymm0     ## encoding: [0x62,0xf1,0x7d,0x28,0xfa,0xc1]
+  %res = call <8 x i32> @llvm.x86.avx512.mask.psub.d.256(<8 x i32> %a, <8 x i32> %b, <8 x i32> zeroinitializer, i8 -1)
+  ret <8 x i32> %res
+}
+
+define <8 x i32> @test_mask_sub_epi32_rrk_256(<8 x i32> %a, <8 x i32> %b, <8 x i32> %passThru, i8 %mask) {
+  ;CHECK-LABEL: test_mask_sub_epi32_rrk_256
+  ;CHECK: vpsubd %ymm1, %ymm0, %ymm2 {%k1} ## encoding: [0x62,0xf1,0x7d,0x29,0xfa,0xd1]
+  %res = call <8 x i32> @llvm.x86.avx512.mask.psub.d.256(<8 x i32> %a, <8 x i32> %b, <8 x i32> %passThru, i8 %mask)
+  ret <8 x i32> %res
+}
+
+define <8 x i32> @test_mask_sub_epi32_rrkz_256(<8 x i32> %a, <8 x i32> %b, i8 %mask) {
+  ;CHECK-LABEL: test_mask_sub_epi32_rrkz_256
+  ;CHECK: vpsubd %ymm1, %ymm0, %ymm0 {%k1} {z} ## encoding: [0x62,0xf1,0x7d,0xa9,0xfa,0xc1]
+  %res = call <8 x i32> @llvm.x86.avx512.mask.psub.d.256(<8 x i32> %a, <8 x i32> %b, <8 x i32> zeroinitializer, i8 %mask)
+  ret <8 x i32> %res
+}
+
+define <8 x i32> @test_mask_sub_epi32_rm_256(<8 x i32> %a, <8 x i32>* %ptr_b) {
+  ;CHECK-LABEL: test_mask_sub_epi32_rm_256
+  ;CHECK: vpsubd (%rdi), %ymm0, %ymm0    ## encoding: [0x62,0xf1,0x7d,0x28,0xfa,0x07]
+  %b = load <8 x i32>, <8 x i32>* %ptr_b
+  %res = call <8 x i32> @llvm.x86.avx512.mask.psub.d.256(<8 x i32> %a, <8 x i32> %b, <8 x i32> zeroinitializer, i8 -1)
+  ret <8 x i32> %res
+}
+
+define <8 x i32> @test_mask_sub_epi32_rmk_256(<8 x i32> %a, <8 x i32>* %ptr_b, <8 x i32> %passThru, i8 %mask) {
+  ;CHECK-LABEL: test_mask_sub_epi32_rmk_256
+  ;CHECK: vpsubd (%rdi), %ymm0, %ymm1 {%k1} ## encoding: [0x62,0xf1,0x7d,0x29,0xfa,0x0f]
+  %b = load <8 x i32>, <8 x i32>* %ptr_b
+  %res = call <8 x i32> @llvm.x86.avx512.mask.psub.d.256(<8 x i32> %a, <8 x i32> %b, <8 x i32> %passThru, i8 %mask)
+  ret <8 x i32> %res
+}
+
+define <8 x i32> @test_mask_sub_epi32_rmkz_256(<8 x i32> %a, <8 x i32>* %ptr_b, i8 %mask) {
+  ;CHECK-LABEL: test_mask_sub_epi32_rmkz_256
+  ;CHECK: vpsubd (%rdi), %ymm0, %ymm0 {%k1} {z} ## encoding: [0x62,0xf1,0x7d,0xa9,0xfa,0x07]
+  %b = load <8 x i32>, <8 x i32>* %ptr_b
+  %res = call <8 x i32> @llvm.x86.avx512.mask.psub.d.256(<8 x i32> %a, <8 x i32> %b, <8 x i32> zeroinitializer, i8 %mask)
+  ret <8 x i32> %res
+}
+
+define <8 x i32> @test_mask_sub_epi32_rmb_256(<8 x i32> %a, i32* %ptr_b) {
+  ;CHECK-LABEL: test_mask_sub_epi32_rmb_256
+  ;CHECK: vpsubd (%rdi){1to8}, %ymm0, %ymm0  ## encoding: [0x62,0xf1,0x7d,0x38,0xfa,0x07]
+  %q = load i32, i32* %ptr_b
+  %vecinit.i = insertelement <8 x i32> undef, i32 %q, i32 0
+  %b = shufflevector <8 x i32> %vecinit.i, <8 x i32> undef, <8 x i32> zeroinitializer
+  %res = call <8 x i32> @llvm.x86.avx512.mask.psub.d.256(<8 x i32> %a, <8 x i32> %b, <8 x i32> zeroinitializer, i8 -1)
+  ret <8 x i32> %res
+}
+
+define <8 x i32> @test_mask_sub_epi32_rmbk_256(<8 x i32> %a, i32* %ptr_b, <8 x i32> %passThru, i8 %mask) {
+  ;CHECK-LABEL: test_mask_sub_epi32_rmbk_256
+  ;CHECK: vpsubd (%rdi){1to8}, %ymm0, %ymm1 {%k1} ## encoding: [0x62,0xf1,0x7d,0x39,0xfa,0x0f]
+  %q = load i32, i32* %ptr_b
+  %vecinit.i = insertelement <8 x i32> undef, i32 %q, i32 0
+  %b = shufflevector <8 x i32> %vecinit.i, <8 x i32> undef, <8 x i32> zeroinitializer
+  %res = call <8 x i32> @llvm.x86.avx512.mask.psub.d.256(<8 x i32> %a, <8 x i32> %b, <8 x i32> %passThru, i8 %mask)
+  ret <8 x i32> %res
+}
+
+define <8 x i32> @test_mask_sub_epi32_rmbkz_256(<8 x i32> %a, i32* %ptr_b, i8 %mask) {
+  ;CHECK-LABEL: test_mask_sub_epi32_rmbkz_256
+  ;CHECK: vpsubd (%rdi){1to8}, %ymm0, %ymm0 {%k1} {z} ## encoding: [0x62,0xf1,0x7d,0xb9,0xfa,0x07]
+  %q = load i32, i32* %ptr_b
+  %vecinit.i = insertelement <8 x i32> undef, i32 %q, i32 0
+  %b = shufflevector <8 x i32> %vecinit.i, <8 x i32> undef, <8 x i32> zeroinitializer
+  %res = call <8 x i32> @llvm.x86.avx512.mask.psub.d.256(<8 x i32> %a, <8 x i32> %b, <8 x i32> zeroinitializer, i8 %mask)
+  ret <8 x i32> %res
+}
+
+declare <8 x i32> @llvm.x86.avx512.mask.psub.d.256(<8 x i32>, <8 x i32>, <8 x i32>, i8)
+
+define <8 x i32> @test_mask_add_epi32_rr_256(<8 x i32> %a, <8 x i32> %b) {
+  ;CHECK-LABEL: test_mask_add_epi32_rr_256
+  ;CHECK: vpaddd %ymm1, %ymm0, %ymm0     ## encoding: [0x62,0xf1,0x7d,0x28,0xfe,0xc1]
+  %res = call <8 x i32> @llvm.x86.avx512.mask.padd.d.256(<8 x i32> %a, <8 x i32> %b, <8 x i32> zeroinitializer, i8 -1)
+  ret <8 x i32> %res
+}
+
+define <8 x i32> @test_mask_add_epi32_rrk_256(<8 x i32> %a, <8 x i32> %b, <8 x i32> %passThru, i8 %mask) {
+  ;CHECK-LABEL: test_mask_add_epi32_rrk_256
+  ;CHECK: vpaddd %ymm1, %ymm0, %ymm2 {%k1} ## encoding: [0x62,0xf1,0x7d,0x29,0xfe,0xd1]
+  %res = call <8 x i32> @llvm.x86.avx512.mask.padd.d.256(<8 x i32> %a, <8 x i32> %b, <8 x i32> %passThru, i8 %mask)
+  ret <8 x i32> %res
+}
+
+define <8 x i32> @test_mask_add_epi32_rrkz_256(<8 x i32> %a, <8 x i32> %b, i8 %mask) {
+  ;CHECK-LABEL: test_mask_add_epi32_rrkz_256
+  ;CHECK: vpaddd %ymm1, %ymm0, %ymm0 {%k1} {z} ## encoding: [0x62,0xf1,0x7d,0xa9,0xfe,0xc1]
+  %res = call <8 x i32> @llvm.x86.avx512.mask.padd.d.256(<8 x i32> %a, <8 x i32> %b, <8 x i32> zeroinitializer, i8 %mask)
+  ret <8 x i32> %res
+}
+
+define <8 x i32> @test_mask_add_epi32_rm_256(<8 x i32> %a, <8 x i32>* %ptr_b) {
+  ;CHECK-LABEL: test_mask_add_epi32_rm_256
+  ;CHECK: vpaddd (%rdi), %ymm0, %ymm0    ## encoding: [0x62,0xf1,0x7d,0x28,0xfe,0x07]
+  %b = load <8 x i32>, <8 x i32>* %ptr_b
+  %res = call <8 x i32> @llvm.x86.avx512.mask.padd.d.256(<8 x i32> %a, <8 x i32> %b, <8 x i32> zeroinitializer, i8 -1)
+  ret <8 x i32> %res
+}
+
+define <8 x i32> @test_mask_add_epi32_rmk_256(<8 x i32> %a, <8 x i32>* %ptr_b, <8 x i32> %passThru, i8 %mask) {
+  ;CHECK-LABEL: test_mask_add_epi32_rmk_256
+  ;CHECK: vpaddd (%rdi), %ymm0, %ymm1 {%k1} ## encoding: [0x62,0xf1,0x7d,0x29,0xfe,0x0f]
+  %b = load <8 x i32>, <8 x i32>* %ptr_b
+  %res = call <8 x i32> @llvm.x86.avx512.mask.padd.d.256(<8 x i32> %a, <8 x i32> %b, <8 x i32> %passThru, i8 %mask)
+  ret <8 x i32> %res
+}
+
+define <8 x i32> @test_mask_add_epi32_rmkz_256(<8 x i32> %a, <8 x i32>* %ptr_b, i8 %mask) {
+  ;CHECK-LABEL: test_mask_add_epi32_rmkz_256
+  ;CHECK: vpaddd (%rdi), %ymm0, %ymm0 {%k1} {z} ## encoding: [0x62,0xf1,0x7d,0xa9,0xfe,0x07]
+  %b = load <8 x i32>, <8 x i32>* %ptr_b
+  %res = call <8 x i32> @llvm.x86.avx512.mask.padd.d.256(<8 x i32> %a, <8 x i32> %b, <8 x i32> zeroinitializer, i8 %mask)
+  ret <8 x i32> %res
+}
+
+define <8 x i32> @test_mask_add_epi32_rmb_256(<8 x i32> %a, i32* %ptr_b) {
+  ;CHECK-LABEL: test_mask_add_epi32_rmb_256
+  ;CHECK: vpaddd (%rdi){1to8}, %ymm0, %ymm0  ## encoding: [0x62,0xf1,0x7d,0x38,0xfe,0x07]
+  %q = load i32, i32* %ptr_b
+  %vecinit.i = insertelement <8 x i32> undef, i32 %q, i32 0
+  %b = shufflevector <8 x i32> %vecinit.i, <8 x i32> undef, <8 x i32> zeroinitializer
+  %res = call <8 x i32> @llvm.x86.avx512.mask.padd.d.256(<8 x i32> %a, <8 x i32> %b, <8 x i32> zeroinitializer, i8 -1)
+  ret <8 x i32> %res
+}
+
+define <8 x i32> @test_mask_add_epi32_rmbk_256(<8 x i32> %a, i32* %ptr_b, <8 x i32> %passThru, i8 %mask) {
+  ;CHECK-LABEL: test_mask_add_epi32_rmbk_256
+  ;CHECK: vpaddd (%rdi){1to8}, %ymm0, %ymm1 {%k1} ## encoding: [0x62,0xf1,0x7d,0x39,0xfe,0x0f]
+  %q = load i32, i32* %ptr_b
+  %vecinit.i = insertelement <8 x i32> undef, i32 %q, i32 0
+  %b = shufflevector <8 x i32> %vecinit.i, <8 x i32> undef, <8 x i32> zeroinitializer
+  %res = call <8 x i32> @llvm.x86.avx512.mask.padd.d.256(<8 x i32> %a, <8 x i32> %b, <8 x i32> %passThru, i8 %mask)
+  ret <8 x i32> %res
+}
+
+define <8 x i32> @test_mask_add_epi32_rmbkz_256(<8 x i32> %a, i32* %ptr_b, i8 %mask) {
+  ;CHECK-LABEL: test_mask_add_epi32_rmbkz_256
+  ;CHECK: vpaddd (%rdi){1to8}, %ymm0, %ymm0 {%k1} {z} ## encoding: [0x62,0xf1,0x7d,0xb9,0xfe,0x07]
+  %q = load i32, i32* %ptr_b
+  %vecinit.i = insertelement <8 x i32> undef, i32 %q, i32 0
+  %b = shufflevector <8 x i32> %vecinit.i, <8 x i32> undef, <8 x i32> zeroinitializer
+  %res = call <8 x i32> @llvm.x86.avx512.mask.padd.d.256(<8 x i32> %a, <8 x i32> %b, <8 x i32> zeroinitializer, i8 %mask)
+  ret <8 x i32> %res
+}
+
+declare <8 x i32> @llvm.x86.avx512.mask.padd.d.256(<8 x i32>, <8 x i32>, <8 x i32>, i8)
+
+define <4 x i32> @test_mask_and_epi32_rr_128(<4 x i32> %a, <4 x i32> %b) {
+  ;CHECK-LABEL: test_mask_and_epi32_rr_128
+  ;CHECK: vpandd  %xmm1, %xmm0, %xmm0     ## encoding: [0x62,0xf1,0x7d,0x08,0xdb,0xc1]
+  %res = call <4 x i32> @llvm.x86.avx512.mask.pand.d.128(<4 x i32> %a, <4 x i32> %b, <4 x i32> zeroinitializer, i8 -1)
+  ret <4 x i32> %res
+}
+
+define <4 x i32> @test_mask_and_epi32_rrk_128(<4 x i32> %a, <4 x i32> %b, <4 x i32> %passThru, i8 %mask) {
+  ;CHECK-LABEL: test_mask_and_epi32_rrk_128
+  ;CHECK: vpandd  %xmm1, %xmm0, %xmm2 {%k1} ## encoding: [0x62,0xf1,0x7d,0x09,0xdb,0xd1]
+  %res = call <4 x i32> @llvm.x86.avx512.mask.pand.d.128(<4 x i32> %a, <4 x i32> %b, <4 x i32> %passThru, i8 %mask)
+  ret <4 x i32> %res
+}
+
+define <4 x i32> @test_mask_and_epi32_rrkz_128(<4 x i32> %a, <4 x i32> %b, i8 %mask) {
+  ;CHECK-LABEL: test_mask_and_epi32_rrkz_128
+  ;CHECK: vpandd  %xmm1, %xmm0, %xmm0 {%k1} {z} ## encoding: [0x62,0xf1,0x7d,0x89,0xdb,0xc1]
+  %res = call <4 x i32> @llvm.x86.avx512.mask.pand.d.128(<4 x i32> %a, <4 x i32> %b, <4 x i32> zeroinitializer, i8 %mask)
+  ret <4 x i32> %res
+}
+
+define <4 x i32> @test_mask_and_epi32_rm_128(<4 x i32> %a, <4 x i32>* %ptr_b) {
+  ;CHECK-LABEL: test_mask_and_epi32_rm_128
+  ;CHECK: vpandd  (%rdi), %xmm0, %xmm0    ## encoding: [0x62,0xf1,0x7d,0x08,0xdb,0x07]
+  %b = load <4 x i32>, <4 x i32>* %ptr_b
+  %res = call <4 x i32> @llvm.x86.avx512.mask.pand.d.128(<4 x i32> %a, <4 x i32> %b, <4 x i32> zeroinitializer, i8 -1)
+  ret <4 x i32> %res
+}
+
+define <4 x i32> @test_mask_and_epi32_rmk_128(<4 x i32> %a, <4 x i32>* %ptr_b, <4 x i32> %passThru, i8 %mask) {
+  ;CHECK-LABEL: test_mask_and_epi32_rmk_128
+  ;CHECK: vpandd  (%rdi), %xmm0, %xmm1 {%k1} ## encoding: [0x62,0xf1,0x7d,0x09,0xdb,0x0f]
+  %b = load <4 x i32>, <4 x i32>* %ptr_b
+  %res = call <4 x i32> @llvm.x86.avx512.mask.pand.d.128(<4 x i32> %a, <4 x i32> %b, <4 x i32> %passThru, i8 %mask)
+  ret <4 x i32> %res
+}
+
+define <4 x i32> @test_mask_and_epi32_rmkz_128(<4 x i32> %a, <4 x i32>* %ptr_b, i8 %mask) {
+  ;CHECK-LABEL: test_mask_and_epi32_rmkz_128
+  ;CHECK: vpandd  (%rdi), %xmm0, %xmm0 {%k1} {z} ## encoding: [0x62,0xf1,0x7d,0x89,0xdb,0x07]
+  %b = load <4 x i32>, <4 x i32>* %ptr_b
+  %res = call <4 x i32> @llvm.x86.avx512.mask.pand.d.128(<4 x i32> %a, <4 x i32> %b, <4 x i32> zeroinitializer, i8 %mask)
+  ret <4 x i32> %res
+}
+
+define <4 x i32> @test_mask_and_epi32_rmb_128(<4 x i32> %a, i32* %ptr_b) {
+  ;CHECK-LABEL: test_mask_and_epi32_rmb_128
+  ;CHECK: vpandd  (%rdi){1to4}, %xmm0, %xmm0  ## encoding: [0x62,0xf1,0x7d,0x18,0xdb,0x07]
+  %q = load i32, i32* %ptr_b
+  %vecinit.i = insertelement <4 x i32> undef, i32 %q, i32 0
+  %b = shufflevector <4 x i32> %vecinit.i, <4 x i32> undef, <4 x i32> zeroinitializer
+  %res = call <4 x i32> @llvm.x86.avx512.mask.pand.d.128(<4 x i32> %a, <4 x i32> %b, <4 x i32> zeroinitializer, i8 -1)
+  ret <4 x i32> %res
+}
+
+define <4 x i32> @test_mask_and_epi32_rmbk_128(<4 x i32> %a, i32* %ptr_b, <4 x i32> %passThru, i8 %mask) {
+  ;CHECK-LABEL: test_mask_and_epi32_rmbk_128
+  ;CHECK: vpandd  (%rdi){1to4}, %xmm0, %xmm1 {%k1} ## encoding: [0x62,0xf1,0x7d,0x19,0xdb,0x0f]
+  %q = load i32, i32* %ptr_b
+  %vecinit.i = insertelement <4 x i32> undef, i32 %q, i32 0
+  %b = shufflevector <4 x i32> %vecinit.i, <4 x i32> undef, <4 x i32> zeroinitializer
+  %res = call <4 x i32> @llvm.x86.avx512.mask.pand.d.128(<4 x i32> %a, <4 x i32> %b, <4 x i32> %passThru, i8 %mask)
+  ret <4 x i32> %res
+}
+
+define <4 x i32> @test_mask_and_epi32_rmbkz_128(<4 x i32> %a, i32* %ptr_b, i8 %mask) {
+  ;CHECK-LABEL: test_mask_and_epi32_rmbkz_128
+  ;CHECK: vpandd  (%rdi){1to4}, %xmm0, %xmm0 {%k1} {z} ## encoding: [0x62,0xf1,0x7d,0x99,0xdb,0x07]
+  %q = load i32, i32* %ptr_b
+  %vecinit.i = insertelement <4 x i32> undef, i32 %q, i32 0
+  %b = shufflevector <4 x i32> %vecinit.i, <4 x i32> undef, <4 x i32> zeroinitializer
+  %res = call <4 x i32> @llvm.x86.avx512.mask.pand.d.128(<4 x i32> %a, <4 x i32> %b, <4 x i32> zeroinitializer, i8 %mask)
+  ret <4 x i32> %res
+}
+
+declare <4 x i32> @llvm.x86.avx512.mask.pand.d.128(<4 x i32>, <4 x i32>, <4 x i32>, i8)
+
+define <8 x i32> @test_mask_and_epi32_rr_256(<8 x i32> %a, <8 x i32> %b) {
+  ;CHECK-LABEL: test_mask_and_epi32_rr_256
+  ;CHECK: vpandd  %ymm1, %ymm0, %ymm0     ## encoding: [0x62,0xf1,0x7d,0x28,0xdb,0xc1]
+  %res = call <8 x i32> @llvm.x86.avx512.mask.pand.d.256(<8 x i32> %a, <8 x i32> %b, <8 x i32> zeroinitializer, i8 -1)
+  ret <8 x i32> %res
+}
+
+define <8 x i32> @test_mask_and_epi32_rrk_256(<8 x i32> %a, <8 x i32> %b, <8 x i32> %passThru, i8 %mask) {
+  ;CHECK-LABEL: test_mask_and_epi32_rrk_256
+  ;CHECK: vpandd  %ymm1, %ymm0, %ymm2 {%k1} ## encoding: [0x62,0xf1,0x7d,0x29,0xdb,0xd1]
+  %res = call <8 x i32> @llvm.x86.avx512.mask.pand.d.256(<8 x i32> %a, <8 x i32> %b, <8 x i32> %passThru, i8 %mask)
+  ret <8 x i32> %res
+}
+
+define <8 x i32> @test_mask_and_epi32_rrkz_256(<8 x i32> %a, <8 x i32> %b, i8 %mask) {
+  ;CHECK-LABEL: test_mask_and_epi32_rrkz_256
+  ;CHECK: vpandd  %ymm1, %ymm0, %ymm0 {%k1} {z} ## encoding: [0x62,0xf1,0x7d,0xa9,0xdb,0xc1]
+  %res = call <8 x i32> @llvm.x86.avx512.mask.pand.d.256(<8 x i32> %a, <8 x i32> %b, <8 x i32> zeroinitializer, i8 %mask)
+  ret <8 x i32> %res
+}
+
+define <8 x i32> @test_mask_and_epi32_rm_256(<8 x i32> %a, <8 x i32>* %ptr_b) {
+  ;CHECK-LABEL: test_mask_and_epi32_rm_256
+  ;CHECK: vpandd  (%rdi), %ymm0, %ymm0    ## encoding: [0x62,0xf1,0x7d,0x28,0xdb,0x07]
+  %b = load <8 x i32>, <8 x i32>* %ptr_b
+  %res = call <8 x i32> @llvm.x86.avx512.mask.pand.d.256(<8 x i32> %a, <8 x i32> %b, <8 x i32> zeroinitializer, i8 -1)
+  ret <8 x i32> %res
+}
+
+define <8 x i32> @test_mask_and_epi32_rmk_256(<8 x i32> %a, <8 x i32>* %ptr_b, <8 x i32> %passThru, i8 %mask) {
+  ;CHECK-LABEL: test_mask_and_epi32_rmk_256
+  ;CHECK: vpandd  (%rdi), %ymm0, %ymm1 {%k1} ## encoding: [0x62,0xf1,0x7d,0x29,0xdb,0x0f]
+  %b = load <8 x i32>, <8 x i32>* %ptr_b
+  %res = call <8 x i32> @llvm.x86.avx512.mask.pand.d.256(<8 x i32> %a, <8 x i32> %b, <8 x i32> %passThru, i8 %mask)
+  ret <8 x i32> %res
+}
+
+define <8 x i32> @test_mask_and_epi32_rmkz_256(<8 x i32> %a, <8 x i32>* %ptr_b, i8 %mask) {
+  ;CHECK-LABEL: test_mask_and_epi32_rmkz_256
+  ;CHECK: vpandd  (%rdi), %ymm0, %ymm0 {%k1} {z} ## encoding: [0x62,0xf1,0x7d,0xa9,0xdb,0x07]
+  %b = load <8 x i32>, <8 x i32>* %ptr_b
+  %res = call <8 x i32> @llvm.x86.avx512.mask.pand.d.256(<8 x i32> %a, <8 x i32> %b, <8 x i32> zeroinitializer, i8 %mask)
+  ret <8 x i32> %res
+}
+
+define <8 x i32> @test_mask_and_epi32_rmb_256(<8 x i32> %a, i32* %ptr_b) {
+  ;CHECK-LABEL: test_mask_and_epi32_rmb_256
+  ;CHECK: vpandd  (%rdi){1to8}, %ymm0, %ymm0  ## encoding: [0x62,0xf1,0x7d,0x38,0xdb,0x07]
+  %q = load i32, i32* %ptr_b
+  %vecinit.i = insertelement <8 x i32> undef, i32 %q, i32 0
+  %b = shufflevector <8 x i32> %vecinit.i, <8 x i32> undef, <8 x i32> zeroinitializer
+  %res = call <8 x i32> @llvm.x86.avx512.mask.pand.d.256(<8 x i32> %a, <8 x i32> %b, <8 x i32> zeroinitializer, i8 -1)
+  ret <8 x i32> %res
+}
+
+define <8 x i32> @test_mask_and_epi32_rmbk_256(<8 x i32> %a, i32* %ptr_b, <8 x i32> %passThru, i8 %mask) {
+  ;CHECK-LABEL: test_mask_and_epi32_rmbk_256
+  ;CHECK: vpandd  (%rdi){1to8}, %ymm0, %ymm1 {%k1} ## encoding: [0x62,0xf1,0x7d,0x39,0xdb,0x0f]
+  %q = load i32, i32* %ptr_b
+  %vecinit.i = insertelement <8 x i32> undef, i32 %q, i32 0
+  %b = shufflevector <8 x i32> %vecinit.i, <8 x i32> undef, <8 x i32> zeroinitializer
+  %res = call <8 x i32> @llvm.x86.avx512.mask.pand.d.256(<8 x i32> %a, <8 x i32> %b, <8 x i32> %passThru, i8 %mask)
+  ret <8 x i32> %res
+}
+
+define <8 x i32> @test_mask_and_epi32_rmbkz_256(<8 x i32> %a, i32* %ptr_b, i8 %mask) {
+  ;CHECK-LABEL: test_mask_and_epi32_rmbkz_256
+  ;CHECK: vpandd  (%rdi){1to8}, %ymm0, %ymm0 {%k1} {z} ## encoding: [0x62,0xf1,0x7d,0xb9,0xdb,0x07]
+  %q = load i32, i32* %ptr_b
+  %vecinit.i = insertelement <8 x i32> undef, i32 %q, i32 0
+  %b = shufflevector <8 x i32> %vecinit.i, <8 x i32> undef, <8 x i32> zeroinitializer
+  %res = call <8 x i32> @llvm.x86.avx512.mask.pand.d.256(<8 x i32> %a, <8 x i32> %b, <8 x i32> zeroinitializer, i8 %mask)
+  ret <8 x i32> %res
+}
+
+declare <8 x i32> @llvm.x86.avx512.mask.pand.d.256(<8 x i32>, <8 x i32>, <8 x i32>, i8)
+
+define <4 x i32> @test_mask_or_epi32_rr_128(<4 x i32> %a, <4 x i32> %b) {
+  ;CHECK-LABEL: test_mask_or_epi32_rr_128
+  ;CHECK: vpord   %xmm1, %xmm0, %xmm0     ## encoding: [0x62,0xf1,0x7d,0x08,0xeb,0xc1]
+  %res = call <4 x i32> @llvm.x86.avx512.mask.por.d.128(<4 x i32> %a, <4 x i32> %b, <4 x i32> zeroinitializer, i8 -1)
+  ret <4 x i32> %res
+}
+
+define <4 x i32> @test_mask_or_epi32_rrk_128(<4 x i32> %a, <4 x i32> %b, <4 x i32> %passThru, i8 %mask) {
+  ;CHECK-LABEL: test_mask_or_epi32_rrk_128
+  ;CHECK: vpord   %xmm1, %xmm0, %xmm2 {%k1} ## encoding: [0x62,0xf1,0x7d,0x09,0xeb,0xd1]
+  %res = call <4 x i32> @llvm.x86.avx512.mask.por.d.128(<4 x i32> %a, <4 x i32> %b, <4 x i32> %passThru, i8 %mask)
+  ret <4 x i32> %res
+}
+
+define <4 x i32> @test_mask_or_epi32_rrkz_128(<4 x i32> %a, <4 x i32> %b, i8 %mask) {
+  ;CHECK-LABEL: test_mask_or_epi32_rrkz_128
+  ;CHECK: vpord   %xmm1, %xmm0, %xmm0 {%k1} {z} ## encoding: [0x62,0xf1,0x7d,0x89,0xeb,0xc1]
+  %res = call <4 x i32> @llvm.x86.avx512.mask.por.d.128(<4 x i32> %a, <4 x i32> %b, <4 x i32> zeroinitializer, i8 %mask)
+  ret <4 x i32> %res
+}
+
+define <4 x i32> @test_mask_or_epi32_rm_128(<4 x i32> %a, <4 x i32>* %ptr_b) {
+  ;CHECK-LABEL: test_mask_or_epi32_rm_128
+  ;CHECK: vpord   (%rdi), %xmm0, %xmm0    ## encoding: [0x62,0xf1,0x7d,0x08,0xeb,0x07]
+  %b = load <4 x i32>, <4 x i32>* %ptr_b
+  %res = call <4 x i32> @llvm.x86.avx512.mask.por.d.128(<4 x i32> %a, <4 x i32> %b, <4 x i32> zeroinitializer, i8 -1)
+  ret <4 x i32> %res
+}
+
+define <4 x i32> @test_mask_or_epi32_rmk_128(<4 x i32> %a, <4 x i32>* %ptr_b, <4 x i32> %passThru, i8 %mask) {
+  ;CHECK-LABEL: test_mask_or_epi32_rmk_128
+  ;CHECK: vpord   (%rdi), %xmm0, %xmm1 {%k1} ## encoding: [0x62,0xf1,0x7d,0x09,0xeb,0x0f]
+  %b = load <4 x i32>, <4 x i32>* %ptr_b
+  %res = call <4 x i32> @llvm.x86.avx512.mask.por.d.128(<4 x i32> %a, <4 x i32> %b, <4 x i32> %passThru, i8 %mask)
+  ret <4 x i32> %res
+}
+
+define <4 x i32> @test_mask_or_epi32_rmkz_128(<4 x i32> %a, <4 x i32>* %ptr_b, i8 %mask) {
+  ;CHECK-LABEL: test_mask_or_epi32_rmkz_128
+  ;CHECK: vpord   (%rdi), %xmm0, %xmm0 {%k1} {z} ## encoding: [0x62,0xf1,0x7d,0x89,0xeb,0x07]
+  %b = load <4 x i32>, <4 x i32>* %ptr_b
+  %res = call <4 x i32> @llvm.x86.avx512.mask.por.d.128(<4 x i32> %a, <4 x i32> %b, <4 x i32> zeroinitializer, i8 %mask)
+  ret <4 x i32> %res
+}
+
+define <4 x i32> @test_mask_or_epi32_rmb_128(<4 x i32> %a, i32* %ptr_b) {
+  ;CHECK-LABEL: test_mask_or_epi32_rmb_128
+  ;CHECK: vpord   (%rdi){1to4}, %xmm0, %xmm0  ## encoding: [0x62,0xf1,0x7d,0x18,0xeb,0x07]
+  %q = load i32, i32* %ptr_b
+  %vecinit.i = insertelement <4 x i32> undef, i32 %q, i32 0
+  %b = shufflevector <4 x i32> %vecinit.i, <4 x i32> undef, <4 x i32> zeroinitializer
+  %res = call <4 x i32> @llvm.x86.avx512.mask.por.d.128(<4 x i32> %a, <4 x i32> %b, <4 x i32> zeroinitializer, i8 -1)
+  ret <4 x i32> %res
+}
+
+define <4 x i32> @test_mask_or_epi32_rmbk_128(<4 x i32> %a, i32* %ptr_b, <4 x i32> %passThru, i8 %mask) {
+  ;CHECK-LABEL: test_mask_or_epi32_rmbk_128
+  ;CHECK: vpord   (%rdi){1to4}, %xmm0, %xmm1 {%k1} ## encoding: [0x62,0xf1,0x7d,0x19,0xeb,0x0f]
+  %q = load i32, i32* %ptr_b
+  %vecinit.i = insertelement <4 x i32> undef, i32 %q, i32 0
+  %b = shufflevector <4 x i32> %vecinit.i, <4 x i32> undef, <4 x i32> zeroinitializer
+  %res = call <4 x i32> @llvm.x86.avx512.mask.por.d.128(<4 x i32> %a, <4 x i32> %b, <4 x i32> %passThru, i8 %mask)
+  ret <4 x i32> %res
+}
+
+define <4 x i32> @test_mask_or_epi32_rmbkz_128(<4 x i32> %a, i32* %ptr_b, i8 %mask) {
+  ;CHECK-LABEL: test_mask_or_epi32_rmbkz_128
+  ;CHECK: vpord   (%rdi){1to4}, %xmm0, %xmm0 {%k1} {z} ## encoding: [0x62,0xf1,0x7d,0x99,0xeb,0x07]
+  %q = load i32, i32* %ptr_b
+  %vecinit.i = insertelement <4 x i32> undef, i32 %q, i32 0
+  %b = shufflevector <4 x i32> %vecinit.i, <4 x i32> undef, <4 x i32> zeroinitializer
+  %res = call <4 x i32> @llvm.x86.avx512.mask.por.d.128(<4 x i32> %a, <4 x i32> %b, <4 x i32> zeroinitializer, i8 %mask)
+  ret <4 x i32> %res
+}
+
+declare <4 x i32> @llvm.x86.avx512.mask.por.d.128(<4 x i32>, <4 x i32>, <4 x i32>, i8)
+
+define <8 x i32> @test_mask_or_epi32_rr_256(<8 x i32> %a, <8 x i32> %b) {
+  ;CHECK-LABEL: test_mask_or_epi32_rr_256
+  ;CHECK: vpord   %ymm1, %ymm0, %ymm0     ## encoding: [0x62,0xf1,0x7d,0x28,0xeb,0xc1]
+  %res = call <8 x i32> @llvm.x86.avx512.mask.por.d.256(<8 x i32> %a, <8 x i32> %b, <8 x i32> zeroinitializer, i8 -1)
+  ret <8 x i32> %res
+}
+
+define <8 x i32> @test_mask_or_epi32_rrk_256(<8 x i32> %a, <8 x i32> %b, <8 x i32> %passThru, i8 %mask) {
+  ;CHECK-LABEL: test_mask_or_epi32_rrk_256
+  ;CHECK: vpord   %ymm1, %ymm0, %ymm2 {%k1} ## encoding: [0x62,0xf1,0x7d,0x29,0xeb,0xd1]
+  %res = call <8 x i32> @llvm.x86.avx512.mask.por.d.256(<8 x i32> %a, <8 x i32> %b, <8 x i32> %passThru, i8 %mask)
+  ret <8 x i32> %res
+}
+
+define <8 x i32> @test_mask_or_epi32_rrkz_256(<8 x i32> %a, <8 x i32> %b, i8 %mask) {
+  ;CHECK-LABEL: test_mask_or_epi32_rrkz_256
+  ;CHECK: vpord   %ymm1, %ymm0, %ymm0 {%k1} {z} ## encoding: [0x62,0xf1,0x7d,0xa9,0xeb,0xc1]
+  %res = call <8 x i32> @llvm.x86.avx512.mask.por.d.256(<8 x i32> %a, <8 x i32> %b, <8 x i32> zeroinitializer, i8 %mask)
+  ret <8 x i32> %res
+}
+
+define <8 x i32> @test_mask_or_epi32_rm_256(<8 x i32> %a, <8 x i32>* %ptr_b) {
+  ;CHECK-LABEL: test_mask_or_epi32_rm_256
+  ;CHECK: vpord   (%rdi), %ymm0, %ymm0    ## encoding: [0x62,0xf1,0x7d,0x28,0xeb,0x07]
+  %b = load <8 x i32>, <8 x i32>* %ptr_b
+  %res = call <8 x i32> @llvm.x86.avx512.mask.por.d.256(<8 x i32> %a, <8 x i32> %b, <8 x i32> zeroinitializer, i8 -1)
+  ret <8 x i32> %res
+}
+
+define <8 x i32> @test_mask_or_epi32_rmk_256(<8 x i32> %a, <8 x i32>* %ptr_b, <8 x i32> %passThru, i8 %mask) {
+  ;CHECK-LABEL: test_mask_or_epi32_rmk_256
+  ;CHECK: vpord   (%rdi), %ymm0, %ymm1 {%k1} ## encoding: [0x62,0xf1,0x7d,0x29,0xeb,0x0f]
+  %b = load <8 x i32>, <8 x i32>* %ptr_b
+  %res = call <8 x i32> @llvm.x86.avx512.mask.por.d.256(<8 x i32> %a, <8 x i32> %b, <8 x i32> %passThru, i8 %mask)
+  ret <8 x i32> %res
+}
+
+define <8 x i32> @test_mask_or_epi32_rmkz_256(<8 x i32> %a, <8 x i32>* %ptr_b, i8 %mask) {
+  ;CHECK-LABEL: test_mask_or_epi32_rmkz_256
+  ;CHECK: vpord   (%rdi), %ymm0, %ymm0 {%k1} {z} ## encoding: [0x62,0xf1,0x7d,0xa9,0xeb,0x07]
+  %b = load <8 x i32>, <8 x i32>* %ptr_b
+  %res = call <8 x i32> @llvm.x86.avx512.mask.por.d.256(<8 x i32> %a, <8 x i32> %b, <8 x i32> zeroinitializer, i8 %mask)
+  ret <8 x i32> %res
+}
+
+define <8 x i32> @test_mask_or_epi32_rmb_256(<8 x i32> %a, i32* %ptr_b) {
+  ;CHECK-LABEL: test_mask_or_epi32_rmb_256
+  ;CHECK: vpord   (%rdi){1to8}, %ymm0, %ymm0  ## encoding: [0x62,0xf1,0x7d,0x38,0xeb,0x07]
+  %q = load i32, i32* %ptr_b
+  %vecinit.i = insertelement <8 x i32> undef, i32 %q, i32 0
+  %b = shufflevector <8 x i32> %vecinit.i, <8 x i32> undef, <8 x i32> zeroinitializer
+  %res = call <8 x i32> @llvm.x86.avx512.mask.por.d.256(<8 x i32> %a, <8 x i32> %b, <8 x i32> zeroinitializer, i8 -1)
+  ret <8 x i32> %res
+}
+
+define <8 x i32> @test_mask_or_epi32_rmbk_256(<8 x i32> %a, i32* %ptr_b, <8 x i32> %passThru, i8 %mask) {
+  ;CHECK-LABEL: test_mask_or_epi32_rmbk_256
+  ;CHECK: vpord   (%rdi){1to8}, %ymm0, %ymm1 {%k1} ## encoding: [0x62,0xf1,0x7d,0x39,0xeb,0x0f]
+  %q = load i32, i32* %ptr_b
+  %vecinit.i = insertelement <8 x i32> undef, i32 %q, i32 0
+  %b = shufflevector <8 x i32> %vecinit.i, <8 x i32> undef, <8 x i32> zeroinitializer
+  %res = call <8 x i32> @llvm.x86.avx512.mask.por.d.256(<8 x i32> %a, <8 x i32> %b, <8 x i32> %passThru, i8 %mask)
+  ret <8 x i32> %res
+}
+
+define <8 x i32> @test_mask_or_epi32_rmbkz_256(<8 x i32> %a, i32* %ptr_b, i8 %mask) {
+  ;CHECK-LABEL: test_mask_or_epi32_rmbkz_256
+  ;CHECK: vpord   (%rdi){1to8}, %ymm0, %ymm0 {%k1} {z} ## encoding: [0x62,0xf1,0x7d,0xb9,0xeb,0x07]
+  %q = load i32, i32* %ptr_b
+  %vecinit.i = insertelement <8 x i32> undef, i32 %q, i32 0
+  %b = shufflevector <8 x i32> %vecinit.i, <8 x i32> undef, <8 x i32> zeroinitializer
+  %res = call <8 x i32> @llvm.x86.avx512.mask.por.d.256(<8 x i32> %a, <8 x i32> %b, <8 x i32> zeroinitializer, i8 %mask)
+  ret <8 x i32> %res
+}
+
+declare <8 x i32> @llvm.x86.avx512.mask.por.d.256(<8 x i32>, <8 x i32>, <8 x i32>, i8)
+
+define <4 x i32> @test_mask_xor_epi32_rr_128(<4 x i32> %a, <4 x i32> %b) {
+  ;CHECK-LABEL: test_mask_xor_epi32_rr_128
+  ;CHECK: vpxord  %xmm1, %xmm0, %xmm0     ## encoding: [0x62,0xf1,0x7d,0x08,0xef,0xc1]
+  %res = call <4 x i32> @llvm.x86.avx512.mask.pxor.d.128(<4 x i32> %a, <4 x i32> %b, <4 x i32> zeroinitializer, i8 -1)
+  ret <4 x i32> %res
+}
+
+define <4 x i32> @test_mask_xor_epi32_rrk_128(<4 x i32> %a, <4 x i32> %b, <4 x i32> %passThru, i8 %mask) {
+  ;CHECK-LABEL: test_mask_xor_epi32_rrk_128
+  ;CHECK: vpxord  %xmm1, %xmm0, %xmm2 {%k1} ## encoding: [0x62,0xf1,0x7d,0x09,0xef,0xd1]
+  %res = call <4 x i32> @llvm.x86.avx512.mask.pxor.d.128(<4 x i32> %a, <4 x i32> %b, <4 x i32> %passThru, i8 %mask)
+  ret <4 x i32> %res
+}
+
+define <4 x i32> @test_mask_xor_epi32_rrkz_128(<4 x i32> %a, <4 x i32> %b, i8 %mask) {
+  ;CHECK-LABEL: test_mask_xor_epi32_rrkz_128
+  ;CHECK: vpxord  %xmm1, %xmm0, %xmm0 {%k1} {z} ## encoding: [0x62,0xf1,0x7d,0x89,0xef,0xc1]
+  %res = call <4 x i32> @llvm.x86.avx512.mask.pxor.d.128(<4 x i32> %a, <4 x i32> %b, <4 x i32> zeroinitializer, i8 %mask)
+  ret <4 x i32> %res
+}
+
+define <4 x i32> @test_mask_xor_epi32_rm_128(<4 x i32> %a, <4 x i32>* %ptr_b) {
+  ;CHECK-LABEL: test_mask_xor_epi32_rm_128
+  ;CHECK: vpxord  (%rdi), %xmm0, %xmm0    ## encoding: [0x62,0xf1,0x7d,0x08,0xef,0x07]
+  %b = load <4 x i32>, <4 x i32>* %ptr_b
+  %res = call <4 x i32> @llvm.x86.avx512.mask.pxor.d.128(<4 x i32> %a, <4 x i32> %b, <4 x i32> zeroinitializer, i8 -1)
+  ret <4 x i32> %res
+}
+
+define <4 x i32> @test_mask_xor_epi32_rmk_128(<4 x i32> %a, <4 x i32>* %ptr_b, <4 x i32> %passThru, i8 %mask) {
+  ;CHECK-LABEL: test_mask_xor_epi32_rmk_128
+  ;CHECK: vpxord  (%rdi), %xmm0, %xmm1 {%k1} ## encoding: [0x62,0xf1,0x7d,0x09,0xef,0x0f]
+  %b = load <4 x i32>, <4 x i32>* %ptr_b
+  %res = call <4 x i32> @llvm.x86.avx512.mask.pxor.d.128(<4 x i32> %a, <4 x i32> %b, <4 x i32> %passThru, i8 %mask)
+  ret <4 x i32> %res
+}
+
+define <4 x i32> @test_mask_xor_epi32_rmkz_128(<4 x i32> %a, <4 x i32>* %ptr_b, i8 %mask) {
+  ;CHECK-LABEL: test_mask_xor_epi32_rmkz_128
+  ;CHECK: vpxord  (%rdi), %xmm0, %xmm0 {%k1} {z} ## encoding: [0x62,0xf1,0x7d,0x89,0xef,0x07]
+  %b = load <4 x i32>, <4 x i32>* %ptr_b
+  %res = call <4 x i32> @llvm.x86.avx512.mask.pxor.d.128(<4 x i32> %a, <4 x i32> %b, <4 x i32> zeroinitializer, i8 %mask)
+  ret <4 x i32> %res
+}
+
+define <4 x i32> @test_mask_xor_epi32_rmb_128(<4 x i32> %a, i32* %ptr_b) {
+  ;CHECK-LABEL: test_mask_xor_epi32_rmb_128
+  ;CHECK: vpxord  (%rdi){1to4}, %xmm0, %xmm0  ## encoding: [0x62,0xf1,0x7d,0x18,0xef,0x07]
+  %q = load i32, i32* %ptr_b
+  %vecinit.i = insertelement <4 x i32> undef, i32 %q, i32 0
+  %b = shufflevector <4 x i32> %vecinit.i, <4 x i32> undef, <4 x i32> zeroinitializer
+  %res = call <4 x i32> @llvm.x86.avx512.mask.pxor.d.128(<4 x i32> %a, <4 x i32> %b, <4 x i32> zeroinitializer, i8 -1)
+  ret <4 x i32> %res
+}
+
+define <4 x i32> @test_mask_xor_epi32_rmbk_128(<4 x i32> %a, i32* %ptr_b, <4 x i32> %passThru, i8 %mask) {
+  ;CHECK-LABEL: test_mask_xor_epi32_rmbk_128
+  ;CHECK: vpxord  (%rdi){1to4}, %xmm0, %xmm1 {%k1} ## encoding: [0x62,0xf1,0x7d,0x19,0xef,0x0f]
+  %q = load i32, i32* %ptr_b
+  %vecinit.i = insertelement <4 x i32> undef, i32 %q, i32 0
+  %b = shufflevector <4 x i32> %vecinit.i, <4 x i32> undef, <4 x i32> zeroinitializer
+  %res = call <4 x i32> @llvm.x86.avx512.mask.pxor.d.128(<4 x i32> %a, <4 x i32> %b, <4 x i32> %passThru, i8 %mask)
+  ret <4 x i32> %res
+}
+
+define <4 x i32> @test_mask_xor_epi32_rmbkz_128(<4 x i32> %a, i32* %ptr_b, i8 %mask) {
+  ;CHECK-LABEL: test_mask_xor_epi32_rmbkz_128
+  ;CHECK: vpxord  (%rdi){1to4}, %xmm0, %xmm0 {%k1} {z} ## encoding: [0x62,0xf1,0x7d,0x99,0xef,0x07]  
+  %q = load i32, i32* %ptr_b
+  %vecinit.i = insertelement <4 x i32> undef, i32 %q, i32 0
+  %b = shufflevector <4 x i32> %vecinit.i, <4 x i32> undef, <4 x i32> zeroinitializer
+  %res = call <4 x i32> @llvm.x86.avx512.mask.pxor.d.128(<4 x i32> %a, <4 x i32> %b, <4 x i32> zeroinitializer, i8 %mask)
+  ret <4 x i32> %res
+}
+
+declare <4 x i32> @llvm.x86.avx512.mask.pxor.d.128(<4 x i32>, <4 x i32>, <4 x i32>, i8)
+
+define <8 x i32> @test_mask_xor_epi32_rr_256(<8 x i32> %a, <8 x i32> %b) {
+  ;CHECK-LABEL: test_mask_xor_epi32_rr_256
+  ;CHECK: vpxord  %ymm1, %ymm0, %ymm0     ## encoding: [0x62,0xf1,0x7d,0x28,0xef,0xc1]
+  %res = call <8 x i32> @llvm.x86.avx512.mask.pxor.d.256(<8 x i32> %a, <8 x i32> %b, <8 x i32> zeroinitializer, i8 -1)
+  ret <8 x i32> %res
+}
+
+define <8 x i32> @test_mask_xor_epi32_rrk_256(<8 x i32> %a, <8 x i32> %b, <8 x i32> %passThru, i8 %mask) {
+  ;CHECK-LABEL: test_mask_xor_epi32_rrk_256
+  ;CHECK: vpxord  %ymm1, %ymm0, %ymm2 {%k1} ## encoding: [0x62,0xf1,0x7d,0x29,0xef,0xd1]
+  %res = call <8 x i32> @llvm.x86.avx512.mask.pxor.d.256(<8 x i32> %a, <8 x i32> %b, <8 x i32> %passThru, i8 %mask)
+  ret <8 x i32> %res
+}
+
+define <8 x i32> @test_mask_xor_epi32_rrkz_256(<8 x i32> %a, <8 x i32> %b, i8 %mask) {
+  ;CHECK-LABEL: test_mask_xor_epi32_rrkz_256
+  ;CHECK: vpxord  %ymm1, %ymm0, %ymm0 {%k1} {z} ## encoding: [0x62,0xf1,0x7d,0xa9,0xef,0xc1]
+  %res = call <8 x i32> @llvm.x86.avx512.mask.pxor.d.256(<8 x i32> %a, <8 x i32> %b, <8 x i32> zeroinitializer, i8 %mask)
+  ret <8 x i32> %res
+}
+
+define <8 x i32> @test_mask_xor_epi32_rm_256(<8 x i32> %a, <8 x i32>* %ptr_b) {
+  ;CHECK-LABEL: test_mask_xor_epi32_rm_256
+  ;CHECK: vpxord  (%rdi), %ymm0, %ymm0    ## encoding: [0x62,0xf1,0x7d,0x28,0xef,0x07]
+  %b = load <8 x i32>, <8 x i32>* %ptr_b
+  %res = call <8 x i32> @llvm.x86.avx512.mask.pxor.d.256(<8 x i32> %a, <8 x i32> %b, <8 x i32> zeroinitializer, i8 -1)
+  ret <8 x i32> %res
+}
+
+define <8 x i32> @test_mask_xor_epi32_rmk_256(<8 x i32> %a, <8 x i32>* %ptr_b, <8 x i32> %passThru, i8 %mask) {
+  ;CHECK-LABEL: test_mask_xor_epi32_rmk_256
+  ;CHECK: vpxord  (%rdi), %ymm0, %ymm1 {%k1} ## encoding: [0x62,0xf1,0x7d,0x29,0xef,0x0f]
+  %b = load <8 x i32>, <8 x i32>* %ptr_b
+  %res = call <8 x i32> @llvm.x86.avx512.mask.pxor.d.256(<8 x i32> %a, <8 x i32> %b, <8 x i32> %passThru, i8 %mask)
+  ret <8 x i32> %res
+}
+
+define <8 x i32> @test_mask_xor_epi32_rmkz_256(<8 x i32> %a, <8 x i32>* %ptr_b, i8 %mask) {
+  ;CHECK-LABEL: test_mask_xor_epi32_rmkz_256
+  ;CHECK: vpxord  (%rdi), %ymm0, %ymm0 {%k1} {z} ## encoding: [0x62,0xf1,0x7d,0xa9,0xef,0x07]
+  %b = load <8 x i32>, <8 x i32>* %ptr_b
+  %res = call <8 x i32> @llvm.x86.avx512.mask.pxor.d.256(<8 x i32> %a, <8 x i32> %b, <8 x i32> zeroinitializer, i8 %mask)
+  ret <8 x i32> %res
+}
+
+define <8 x i32> @test_mask_xor_epi32_rmb_256(<8 x i32> %a, i32* %ptr_b) {
+  ;CHECK-LABEL: test_mask_xor_epi32_rmb_256
+  ;CHECK: vpxord  (%rdi){1to8}, %ymm0, %ymm0  ## encoding: [0x62,0xf1,0x7d,0x38,0xef,0x07]
+  %q = load i32, i32* %ptr_b
+  %vecinit.i = insertelement <8 x i32> undef, i32 %q, i32 0
+  %b = shufflevector <8 x i32> %vecinit.i, <8 x i32> undef, <8 x i32> zeroinitializer
+  %res = call <8 x i32> @llvm.x86.avx512.mask.pxor.d.256(<8 x i32> %a, <8 x i32> %b, <8 x i32> zeroinitializer, i8 -1)
+  ret <8 x i32> %res
+}
+
+define <8 x i32> @test_mask_xor_epi32_rmbk_256(<8 x i32> %a, i32* %ptr_b, <8 x i32> %passThru, i8 %mask) {
+  ;CHECK-LABEL: test_mask_xor_epi32_rmbk_256
+  ;CHECK: vpxord  (%rdi){1to8}, %ymm0, %ymm1 {%k1} ## encoding: [0x62,0xf1,0x7d,0x39,0xef,0x0f]
+  %q = load i32, i32* %ptr_b
+  %vecinit.i = insertelement <8 x i32> undef, i32 %q, i32 0
+  %b = shufflevector <8 x i32> %vecinit.i, <8 x i32> undef, <8 x i32> zeroinitializer
+  %res = call <8 x i32> @llvm.x86.avx512.mask.pxor.d.256(<8 x i32> %a, <8 x i32> %b, <8 x i32> %passThru, i8 %mask)
+  ret <8 x i32> %res
+}
+
+define <8 x i32> @test_mask_xor_epi32_rmbkz_256(<8 x i32> %a, i32* %ptr_b, i8 %mask) {
+  ;CHECK-LABEL: test_mask_xor_epi32_rmbkz_256
+  ;CHECK: vpxord  (%rdi){1to8}, %ymm0, %ymm0 {%k1} {z} ## encoding: [0x62,0xf1,0x7d,0xb9,0xef,0x07]
+  %q = load i32, i32* %ptr_b
+  %vecinit.i = insertelement <8 x i32> undef, i32 %q, i32 0
+  %b = shufflevector <8 x i32> %vecinit.i, <8 x i32> undef, <8 x i32> zeroinitializer
+  %res = call <8 x i32> @llvm.x86.avx512.mask.pxor.d.256(<8 x i32> %a, <8 x i32> %b, <8 x i32> zeroinitializer, i8 %mask)
+  ret <8 x i32> %res
+}
+
+declare <8 x i32> @llvm.x86.avx512.mask.pxor.d.256(<8 x i32>, <8 x i32>, <8 x i32>, i8)
+
+define <4 x i32> @test_mask_andnot_epi32_rr_128(<4 x i32> %a, <4 x i32> %b) {
+  ;CHECK-LABEL: test_mask_andnot_epi32_rr_128
+  ;CHECK: vpandnd  %xmm1, %xmm0, %xmm0     ## encoding: [0x62,0xf1,0x7d,0x08,0xdf,0xc1]
+  %res = call <4 x i32> @llvm.x86.avx512.mask.pandn.d.128(<4 x i32> %a, <4 x i32> %b, <4 x i32> zeroinitializer, i8 -1)
+  ret <4 x i32> %res
+}
+
+define <4 x i32> @test_mask_andnot_epi32_rrk_128(<4 x i32> %a, <4 x i32> %b, <4 x i32> %passThru, i8 %mask) {
+  ;CHECK-LABEL: test_mask_andnot_epi32_rrk_128
+  ;CHECK: vpandnd  %xmm1, %xmm0, %xmm2 {%k1} ## encoding: [0x62,0xf1,0x7d,0x09,0xdf,0xd1]
+  %res = call <4 x i32> @llvm.x86.avx512.mask.pandn.d.128(<4 x i32> %a, <4 x i32> %b, <4 x i32> %passThru, i8 %mask)
+  ret <4 x i32> %res
+}
+
+define <4 x i32> @test_mask_andnot_epi32_rrkz_128(<4 x i32> %a, <4 x i32> %b, i8 %mask) {
+  ;CHECK-LABEL: test_mask_andnot_epi32_rrkz_128
+  ;CHECK: vpandnd  %xmm1, %xmm0, %xmm0 {%k1} {z} ## encoding: [0x62,0xf1,0x7d,0x89,0xdf,0xc1]
+  %res = call <4 x i32> @llvm.x86.avx512.mask.pandn.d.128(<4 x i32> %a, <4 x i32> %b, <4 x i32> zeroinitializer, i8 %mask)
+  ret <4 x i32> %res
+}
+
+define <4 x i32> @test_mask_andnot_epi32_rm_128(<4 x i32> %a, <4 x i32>* %ptr_b) {
+  ;CHECK-LABEL: test_mask_andnot_epi32_rm_128
+  ;CHECK: vpandnd  (%rdi), %xmm0, %xmm0    ## encoding: [0x62,0xf1,0x7d,0x08,0xdf,0x07]
+  %b = load <4 x i32>, <4 x i32>* %ptr_b
+  %res = call <4 x i32> @llvm.x86.avx512.mask.pandn.d.128(<4 x i32> %a, <4 x i32> %b, <4 x i32> zeroinitializer, i8 -1)
+  ret <4 x i32> %res
+}
+
+define <4 x i32> @test_mask_andnot_epi32_rmk_128(<4 x i32> %a, <4 x i32>* %ptr_b, <4 x i32> %passThru, i8 %mask) {
+  ;CHECK-LABEL: test_mask_andnot_epi32_rmk_128
+  ;CHECK: vpandnd  (%rdi), %xmm0, %xmm1 {%k1} ## encoding: [0x62,0xf1,0x7d,0x09,0xdf,0x0f]
+  %b = load <4 x i32>, <4 x i32>* %ptr_b
+  %res = call <4 x i32> @llvm.x86.avx512.mask.pandn.d.128(<4 x i32> %a, <4 x i32> %b, <4 x i32> %passThru, i8 %mask)
+  ret <4 x i32> %res
+}
+
+define <4 x i32> @test_mask_andnot_epi32_rmkz_128(<4 x i32> %a, <4 x i32>* %ptr_b, i8 %mask) {
+  ;CHECK-LABEL: test_mask_andnot_epi32_rmkz_128
+  ;CHECK: vpandnd  (%rdi), %xmm0, %xmm0 {%k1} {z} ## encoding: [0x62,0xf1,0x7d,0x89,0xdf,0x07]
+  %b = load <4 x i32>, <4 x i32>* %ptr_b
+  %res = call <4 x i32> @llvm.x86.avx512.mask.pandn.d.128(<4 x i32> %a, <4 x i32> %b, <4 x i32> zeroinitializer, i8 %mask)
+  ret <4 x i32> %res
+}
+
+define <4 x i32> @test_mask_andnot_epi32_rmb_128(<4 x i32> %a, i32* %ptr_b) {
+  ;CHECK-LABEL: test_mask_andnot_epi32_rmb_128
+  ;CHECK: vpandnd  (%rdi){1to4}, %xmm0, %xmm0  ## encoding: [0x62,0xf1,0x7d,0x18,0xdf,0x07]
+  %q = load i32, i32* %ptr_b
+  %vecinit.i = insertelement <4 x i32> undef, i32 %q, i32 0
+  %b = shufflevector <4 x i32> %vecinit.i, <4 x i32> undef, <4 x i32> zeroinitializer
+  %res = call <4 x i32> @llvm.x86.avx512.mask.pandn.d.128(<4 x i32> %a, <4 x i32> %b, <4 x i32> zeroinitializer, i8 -1)
+  ret <4 x i32> %res
+}
+
+define <4 x i32> @test_mask_andnot_epi32_rmbk_128(<4 x i32> %a, i32* %ptr_b, <4 x i32> %passThru, i8 %mask) {
+  ;CHECK-LABEL: test_mask_andnot_epi32_rmbk_128
+  ;CHECK: vpandnd  (%rdi){1to4}, %xmm0, %xmm1 {%k1} ## encoding: [0x62,0xf1,0x7d,0x19,0xdf,0x0f]
+  %q = load i32, i32* %ptr_b
+  %vecinit.i = insertelement <4 x i32> undef, i32 %q, i32 0
+  %b = shufflevector <4 x i32> %vecinit.i, <4 x i32> undef, <4 x i32> zeroinitializer
+  %res = call <4 x i32> @llvm.x86.avx512.mask.pandn.d.128(<4 x i32> %a, <4 x i32> %b, <4 x i32> %passThru, i8 %mask)
+  ret <4 x i32> %res
+}
+
+define <4 x i32> @test_mask_andnot_epi32_rmbkz_128(<4 x i32> %a, i32* %ptr_b, i8 %mask) {
+  ;CHECK-LABEL: test_mask_andnot_epi32_rmbkz_128
+  ;CHECK: vpandnd  (%rdi){1to4}, %xmm0, %xmm0 {%k1} {z} ## encoding: [0x62,0xf1,0x7d,0x99,0xdf,0x07]
+  %q = load i32, i32* %ptr_b
+  %vecinit.i = insertelement <4 x i32> undef, i32 %q, i32 0
+  %b = shufflevector <4 x i32> %vecinit.i, <4 x i32> undef, <4 x i32> zeroinitializer
+  %res = call <4 x i32> @llvm.x86.avx512.mask.pandn.d.128(<4 x i32> %a, <4 x i32> %b, <4 x i32> zeroinitializer, i8 %mask)
+  ret <4 x i32> %res
+}
+
+declare <4 x i32> @llvm.x86.avx512.mask.pandn.d.128(<4 x i32>, <4 x i32>, <4 x i32>, i8)
+
+define <8 x i32> @test_mask_andnot_epi32_rr_256(<8 x i32> %a, <8 x i32> %b) {
+  ;CHECK-LABEL: test_mask_andnot_epi32_rr_256
+  ;CHECK: vpandnd  %ymm1, %ymm0, %ymm0     ## encoding: [0x62,0xf1,0x7d,0x28,0xdf,0xc1]
+  %res = call <8 x i32> @llvm.x86.avx512.mask.pandn.d.256(<8 x i32> %a, <8 x i32> %b, <8 x i32> zeroinitializer, i8 -1)
+  ret <8 x i32> %res
+}
+
+define <8 x i32> @test_mask_andnot_epi32_rrk_256(<8 x i32> %a, <8 x i32> %b, <8 x i32> %passThru, i8 %mask) {
+  ;CHECK-LABEL: test_mask_andnot_epi32_rrk_256
+  ;CHECK: vpandnd  %ymm1, %ymm0, %ymm2 {%k1} ## encoding: [0x62,0xf1,0x7d,0x29,0xdf,0xd1]
+  %res = call <8 x i32> @llvm.x86.avx512.mask.pandn.d.256(<8 x i32> %a, <8 x i32> %b, <8 x i32> %passThru, i8 %mask)
+  ret <8 x i32> %res
+}
+
+define <8 x i32> @test_mask_andnot_epi32_rrkz_256(<8 x i32> %a, <8 x i32> %b, i8 %mask) {
+  ;CHECK-LABEL: test_mask_andnot_epi32_rrkz_256
+  ;CHECK: vpandnd  %ymm1, %ymm0, %ymm0 {%k1} {z} ## encoding: [0x62,0xf1,0x7d,0xa9,0xdf,0xc1]
+  %res = call <8 x i32> @llvm.x86.avx512.mask.pandn.d.256(<8 x i32> %a, <8 x i32> %b, <8 x i32> zeroinitializer, i8 %mask)
+  ret <8 x i32> %res
+}
+
+define <8 x i32> @test_mask_andnot_epi32_rm_256(<8 x i32> %a, <8 x i32>* %ptr_b) {
+  ;CHECK-LABEL: test_mask_andnot_epi32_rm_256
+  ;CHECK: vpandnd  (%rdi), %ymm0, %ymm0    ## encoding: [0x62,0xf1,0x7d,0x28,0xdf,0x07]
+  %b = load <8 x i32>, <8 x i32>* %ptr_b
+  %res = call <8 x i32> @llvm.x86.avx512.mask.pandn.d.256(<8 x i32> %a, <8 x i32> %b, <8 x i32> zeroinitializer, i8 -1)
+  ret <8 x i32> %res
+}
+
+define <8 x i32> @test_mask_andnot_epi32_rmk_256(<8 x i32> %a, <8 x i32>* %ptr_b, <8 x i32> %passThru, i8 %mask) {
+  ;CHECK-LABEL: test_mask_andnot_epi32_rmk_256
+  ;CHECK: vpandnd  (%rdi), %ymm0, %ymm1 {%k1} ## encoding: [0x62,0xf1,0x7d,0x29,0xdf,0x0f]
+  %b = load <8 x i32>, <8 x i32>* %ptr_b
+  %res = call <8 x i32> @llvm.x86.avx512.mask.pandn.d.256(<8 x i32> %a, <8 x i32> %b, <8 x i32> %passThru, i8 %mask)
+  ret <8 x i32> %res
+}
+
+define <8 x i32> @test_mask_andnot_epi32_rmkz_256(<8 x i32> %a, <8 x i32>* %ptr_b, i8 %mask) {
+  ;CHECK-LABEL: test_mask_andnot_epi32_rmkz_256
+  ;CHECK: vpandnd  (%rdi), %ymm0, %ymm0 {%k1} {z} ## encoding: [0x62,0xf1,0x7d,0xa9,0xdf,0x07]
+  %b = load <8 x i32>, <8 x i32>* %ptr_b
+  %res = call <8 x i32> @llvm.x86.avx512.mask.pandn.d.256(<8 x i32> %a, <8 x i32> %b, <8 x i32> zeroinitializer, i8 %mask)
+  ret <8 x i32> %res
+}
+
+define <8 x i32> @test_mask_andnot_epi32_rmb_256(<8 x i32> %a, i32* %ptr_b) {
+  ;CHECK-LABEL: test_mask_andnot_epi32_rmb_256
+  ;CHECK: vpandnd  (%rdi){1to8}, %ymm0, %ymm0  ## encoding: [0x62,0xf1,0x7d,0x38,0xdf,0x07]
+  %q = load i32, i32* %ptr_b
+  %vecinit.i = insertelement <8 x i32> undef, i32 %q, i32 0
+  %b = shufflevector <8 x i32> %vecinit.i, <8 x i32> undef, <8 x i32> zeroinitializer
+  %res = call <8 x i32> @llvm.x86.avx512.mask.pandn.d.256(<8 x i32> %a, <8 x i32> %b, <8 x i32> zeroinitializer, i8 -1)
+  ret <8 x i32> %res
+}
+
+define <8 x i32> @test_mask_andnot_epi32_rmbk_256(<8 x i32> %a, i32* %ptr_b, <8 x i32> %passThru, i8 %mask) {
+  ;CHECK-LABEL: test_mask_andnot_epi32_rmbk_256
+  ;CHECK: vpandnd  (%rdi){1to8}, %ymm0, %ymm1 {%k1} ## encoding: [0x62,0xf1,0x7d,0x39,0xdf,0x0f]
+  %q = load i32, i32* %ptr_b
+  %vecinit.i = insertelement <8 x i32> undef, i32 %q, i32 0
+  %b = shufflevector <8 x i32> %vecinit.i, <8 x i32> undef, <8 x i32> zeroinitializer
+  %res = call <8 x i32> @llvm.x86.avx512.mask.pandn.d.256(<8 x i32> %a, <8 x i32> %b, <8 x i32> %passThru, i8 %mask)
+  ret <8 x i32> %res
+}
+
+define <8 x i32> @test_mask_andnot_epi32_rmbkz_256(<8 x i32> %a, i32* %ptr_b, i8 %mask) {
+  ;CHECK-LABEL: test_mask_andnot_epi32_rmbkz_256
+  ;CHECK: vpandnd  (%rdi){1to8}, %ymm0, %ymm0 {%k1} {z} ## encoding: [0x62,0xf1,0x7d,0xb9,0xdf,0x07]
+  %q = load i32, i32* %ptr_b
+  %vecinit.i = insertelement <8 x i32> undef, i32 %q, i32 0
+  %b = shufflevector <8 x i32> %vecinit.i, <8 x i32> undef, <8 x i32> zeroinitializer
+  %res = call <8 x i32> @llvm.x86.avx512.mask.pandn.d.256(<8 x i32> %a, <8 x i32> %b, <8 x i32> zeroinitializer, i8 %mask)
+  ret <8 x i32> %res
+}
+
+declare <8 x i32> @llvm.x86.avx512.mask.pandn.d.256(<8 x i32>, <8 x i32>, <8 x i32>, i8)
+
+define <2 x i64> @test_mask_andnot_epi64_rr_128(<2 x i64> %a, <2 x i64> %b) {
+  ;CHECK-LABEL: test_mask_andnot_epi64_rr_128
+  ;CHECK: vpandnq  %xmm1, %xmm0, %xmm0     ## encoding: [0x62,0xf1,0xfd,0x08,0xdf,0xc1]
+  %res = call <2 x i64> @llvm.x86.avx512.mask.pandn.q.128(<2 x i64> %a, <2 x i64> %b, <2 x i64> zeroinitializer, i8 -1)
+  ret <2 x i64> %res
+}
+
+define <2 x i64> @test_mask_andnot_epi64_rrk_128(<2 x i64> %a, <2 x i64> %b, <2 x i64> %passThru, i8 %mask) {
+  ;CHECK-LABEL: test_mask_andnot_epi64_rrk_128
+  ;CHECK: vpandnq  %xmm1, %xmm0, %xmm2 {%k1} ## encoding: [0x62,0xf1,0xfd,0x09,0xdf,0xd1]
+  %res = call <2 x i64> @llvm.x86.avx512.mask.pandn.q.128(<2 x i64> %a, <2 x i64> %b, <2 x i64> %passThru, i8 %mask)
+  ret <2 x i64> %res
+}
+
+define <2 x i64> @test_mask_andnot_epi64_rrkz_128(<2 x i64> %a, <2 x i64> %b, i8 %mask) {
+  ;CHECK-LABEL: test_mask_andnot_epi64_rrkz_128
+  ;CHECK: vpandnq  %xmm1, %xmm0, %xmm0 {%k1} {z} ## encoding: [0x62,0xf1,0xfd,0x89,0xdf,0xc1]
+  %res = call <2 x i64> @llvm.x86.avx512.mask.pandn.q.128(<2 x i64> %a, <2 x i64> %b, <2 x i64> zeroinitializer, i8 %mask)
+  ret <2 x i64> %res
+}
+
+define <2 x i64> @test_mask_andnot_epi64_rm_128(<2 x i64> %a, <2 x i64>* %ptr_b) {
+  ;CHECK-LABEL: test_mask_andnot_epi64_rm_128
+  ;CHECK: vpandnq  (%rdi), %xmm0, %xmm0    ## encoding: [0x62,0xf1,0xfd,0x08,0xdf,0x07]
+  %b = load <2 x i64>, <2 x i64>* %ptr_b
+  %res = call <2 x i64> @llvm.x86.avx512.mask.pandn.q.128(<2 x i64> %a, <2 x i64> %b, <2 x i64> zeroinitializer, i8 -1)
+  ret <2 x i64> %res
+}
+
+define <2 x i64> @test_mask_andnot_epi64_rmk_128(<2 x i64> %a, <2 x i64>* %ptr_b, <2 x i64> %passThru, i8 %mask) {
+  ;CHECK-LABEL: test_mask_andnot_epi64_rmk_128
+  ;CHECK: vpandnq  (%rdi), %xmm0, %xmm1 {%k1} ## encoding: [0x62,0xf1,0xfd,0x09,0xdf,0x0f]
+  %b = load <2 x i64>, <2 x i64>* %ptr_b
+  %res = call <2 x i64> @llvm.x86.avx512.mask.pandn.q.128(<2 x i64> %a, <2 x i64> %b, <2 x i64> %passThru, i8 %mask)
+  ret <2 x i64> %res
+}
+
+define <2 x i64> @test_mask_andnot_epi64_rmkz_128(<2 x i64> %a, <2 x i64>* %ptr_b, i8 %mask) {
+  ;CHECK-LABEL: test_mask_andnot_epi64_rmkz_128
+  ;CHECK: vpandnq  (%rdi), %xmm0, %xmm0 {%k1} {z} ## encoding: [0x62,0xf1,0xfd,0x89,0xdf,0x07]
+  %b = load <2 x i64>, <2 x i64>* %ptr_b
+  %res = call <2 x i64> @llvm.x86.avx512.mask.pandn.q.128(<2 x i64> %a, <2 x i64> %b, <2 x i64> zeroinitializer, i8 %mask)
+  ret <2 x i64> %res
+}
+
+define <2 x i64> @test_mask_andnot_epi64_rmb_128(<2 x i64> %a, i64* %ptr_b) {
+  ;CHECK-LABEL: test_mask_andnot_epi64_rmb_128
+  ;CHECK: vpandnq  (%rdi){1to2}, %xmm0, %xmm0  ## encoding: [0x62,0xf1,0xfd,0x18,0xdf,0x07]
+  %q = load i64, i64* %ptr_b
+  %vecinit.i = insertelement <2 x i64> undef, i64 %q, i32 0
+  %b = shufflevector <2 x i64> %vecinit.i, <2 x i64> undef, <2 x i32> zeroinitializer
+  %res = call <2 x i64> @llvm.x86.avx512.mask.pandn.q.128(<2 x i64> %a, <2 x i64> %b, <2 x i64> zeroinitializer, i8 -1)
+  ret <2 x i64> %res
+}
+
+define <2 x i64> @test_mask_andnot_epi64_rmbk_128(<2 x i64> %a, i64* %ptr_b, <2 x i64> %passThru, i8 %mask) {
+  ;CHECK-LABEL: test_mask_andnot_epi64_rmbk_128
+  ;CHECK: vpandnq  (%rdi){1to2}, %xmm0, %xmm1 {%k1} ## encoding: [0x62,0xf1,0xfd,0x19,0xdf,0x0f]
+  %q = load i64, i64* %ptr_b
+  %vecinit.i = insertelement <2 x i64> undef, i64 %q, i32 0
+  %b = shufflevector <2 x i64> %vecinit.i, <2 x i64> undef, <2 x i32> zeroinitializer
+  %res = call <2 x i64> @llvm.x86.avx512.mask.pandn.q.128(<2 x i64> %a, <2 x i64> %b, <2 x i64> %passThru, i8 %mask)
+  ret <2 x i64> %res
+}
+
+define <2 x i64> @test_mask_andnot_epi64_rmbkz_128(<2 x i64> %a, i64* %ptr_b, i8 %mask) {
+  ;CHECK-LABEL: test_mask_andnot_epi64_rmbkz_128
+  ;CHECK: vpandnq  (%rdi){1to2}, %xmm0, %xmm0 {%k1} {z} ## encoding: [0x62,0xf1,0xfd,0x99,0xdf,0x07]
+  %q = load i64, i64* %ptr_b
+  %vecinit.i = insertelement <2 x i64> undef, i64 %q, i32 0
+  %b = shufflevector <2 x i64> %vecinit.i, <2 x i64> undef, <2 x i32> zeroinitializer
+  %res = call <2 x i64> @llvm.x86.avx512.mask.pandn.q.128(<2 x i64> %a, <2 x i64> %b, <2 x i64> zeroinitializer, i8 %mask)
+  ret <2 x i64> %res
+}
+
+declare <2 x i64> @llvm.x86.avx512.mask.pandn.q.128(<2 x i64>, <2 x i64>, <2 x i64>, i8)
+
+define <4 x i64> @test_mask_andnot_epi64_rr_256(<4 x i64> %a, <4 x i64> %b) {
+  ;CHECK-LABEL: test_mask_andnot_epi64_rr_256
+  ;CHECK: vpandnq  %ymm1, %ymm0, %ymm0     ## encoding: [0x62,0xf1,0xfd,0x28,0xdf,0xc1]
+  %res = call <4 x i64> @llvm.x86.avx512.mask.pandn.q.256(<4 x i64> %a, <4 x i64> %b, <4 x i64> zeroinitializer, i8 -1)
+  ret <4 x i64> %res
+}
+
+define <4 x i64> @test_mask_andnot_epi64_rrk_256(<4 x i64> %a, <4 x i64> %b, <4 x i64> %passThru, i8 %mask) {
+  ;CHECK-LABEL: test_mask_andnot_epi64_rrk_256
+  ;CHECK: vpandnq  %ymm1, %ymm0, %ymm2 {%k1} ## encoding: [0x62,0xf1,0xfd,0x29,0xdf,0xd1]
+  %res = call <4 x i64> @llvm.x86.avx512.mask.pandn.q.256(<4 x i64> %a, <4 x i64> %b, <4 x i64> %passThru, i8 %mask)
+  ret <4 x i64> %res
+}
+
+define <4 x i64> @test_mask_andnot_epi64_rrkz_256(<4 x i64> %a, <4 x i64> %b, i8 %mask) {
+  ;CHECK-LABEL: test_mask_andnot_epi64_rrkz_256
+  ;CHECK: vpandnq  %ymm1, %ymm0, %ymm0 {%k1} {z} ## encoding: [0x62,0xf1,0xfd,0xa9,0xdf,0xc1]
+  %res = call <4 x i64> @llvm.x86.avx512.mask.pandn.q.256(<4 x i64> %a, <4 x i64> %b, <4 x i64> zeroinitializer, i8 %mask)
+  ret <4 x i64> %res
+}
+
+define <4 x i64> @test_mask_andnot_epi64_rm_256(<4 x i64> %a, <4 x i64>* %ptr_b) {
+  ;CHECK-LABEL: test_mask_andnot_epi64_rm_256
+  ;CHECK: vpandnq  (%rdi), %ymm0, %ymm0    ## encoding: [0x62,0xf1,0xfd,0x28,0xdf,0x07]
+  %b = load <4 x i64>, <4 x i64>* %ptr_b
+  %res = call <4 x i64> @llvm.x86.avx512.mask.pandn.q.256(<4 x i64> %a, <4 x i64> %b, <4 x i64> zeroinitializer, i8 -1)
+  ret <4 x i64> %res
+}
+
+define <4 x i64> @test_mask_andnot_epi64_rmk_256(<4 x i64> %a, <4 x i64>* %ptr_b, <4 x i64> %passThru, i8 %mask) {
+  ;CHECK-LABEL: test_mask_andnot_epi64_rmk_256
+  ;CHECK: vpandnq  (%rdi), %ymm0, %ymm1 {%k1} ## encoding: [0x62,0xf1,0xfd,0x29,0xdf,0x0f]
+  %b = load <4 x i64>, <4 x i64>* %ptr_b
+  %res = call <4 x i64> @llvm.x86.avx512.mask.pandn.q.256(<4 x i64> %a, <4 x i64> %b, <4 x i64> %passThru, i8 %mask)
+  ret <4 x i64> %res
+}
+
+define <4 x i64> @test_mask_andnot_epi64_rmkz_256(<4 x i64> %a, <4 x i64>* %ptr_b, i8 %mask) {
+  ;CHECK-LABEL: test_mask_andnot_epi64_rmkz_256
+  ;CHECK: vpandnq  (%rdi), %ymm0, %ymm0 {%k1} {z} ## encoding: [0x62,0xf1,0xfd,0xa9,0xdf,0x07]
+  %b = load <4 x i64>, <4 x i64>* %ptr_b
+  %res = call <4 x i64> @llvm.x86.avx512.mask.pandn.q.256(<4 x i64> %a, <4 x i64> %b, <4 x i64> zeroinitializer, i8 %mask)
+  ret <4 x i64> %res
+}
+
+define <4 x i64> @test_mask_andnot_epi64_rmb_256(<4 x i64> %a, i64* %ptr_b) {
+  ;CHECK-LABEL: test_mask_andnot_epi64_rmb_256
+  ;CHECK: vpandnq  (%rdi){1to4}, %ymm0, %ymm0  ## encoding: [0x62,0xf1,0xfd,0x38,0xdf,0x07]
+  %q = load i64, i64* %ptr_b
+  %vecinit.i = insertelement <4 x i64> undef, i64 %q, i32 0
+  %b = shufflevector <4 x i64> %vecinit.i, <4 x i64> undef, <4 x i32> zeroinitializer
+  %res = call <4 x i64> @llvm.x86.avx512.mask.pandn.q.256(<4 x i64> %a, <4 x i64> %b, <4 x i64> zeroinitializer, i8 -1)
+  ret <4 x i64> %res
+}
+
+define <4 x i64> @test_mask_andnot_epi64_rmbk_256(<4 x i64> %a, i64* %ptr_b, <4 x i64> %passThru, i8 %mask) {
+  ;CHECK-LABEL: test_mask_andnot_epi64_rmbk_256
+  ;CHECK: vpandnq  (%rdi){1to4}, %ymm0, %ymm1 {%k1} ## encoding: [0x62,0xf1,0xfd,0x39,0xdf,0x0f]
+  %q = load i64, i64* %ptr_b
+  %vecinit.i = insertelement <4 x i64> undef, i64 %q, i32 0
+  %b = shufflevector <4 x i64> %vecinit.i, <4 x i64> undef, <4 x i32> zeroinitializer
+  %res = call <4 x i64> @llvm.x86.avx512.mask.pandn.q.256(<4 x i64> %a, <4 x i64> %b, <4 x i64> %passThru, i8 %mask)
+  ret <4 x i64> %res
+}
+
+define <4 x i64> @test_mask_andnot_epi64_rmbkz_256(<4 x i64> %a, i64* %ptr_b, i8 %mask) {
+  ;CHECK-LABEL: test_mask_andnot_epi64_rmbkz_256
+  ;CHECK: vpandnq  (%rdi){1to4}, %ymm0, %ymm0 {%k1} {z} ## encoding: [0x62,0xf1,0xfd,0xb9,0xdf,0x07]
+  %q = load i64, i64* %ptr_b
+  %vecinit.i = insertelement <4 x i64> undef, i64 %q, i32 0
+  %b = shufflevector <4 x i64> %vecinit.i, <4 x i64> undef, <4 x i32> zeroinitializer
+  %res = call <4 x i64> @llvm.x86.avx512.mask.pandn.q.256(<4 x i64> %a, <4 x i64> %b, <4 x i64> zeroinitializer, i8 %mask)
+  ret <4 x i64> %res
+}
+
+declare <4 x i64> @llvm.x86.avx512.mask.pandn.q.256(<4 x i64>, <4 x i64>, <4 x i64>, i8)
+
+define i8 @test_cmpps_256(<8 x float> %a, <8 x float> %b) {
+ ;CHECK: vcmpleps  %ymm1, %ymm0, %k0  ## encoding: [0x62,0xf1,0x7c,0x28,0xc2,0xc1,0x02]
+   %res = call i8 @llvm.x86.avx512.mask.cmp.ps.256(<8 x float> %a, <8 x float> %b, i32 2, i8 -1)
+   ret i8 %res
+ }
+ declare i8 @llvm.x86.avx512.mask.cmp.ps.256(<8 x float> , <8 x float> , i32, i8)
+
+define i8 @test_cmpps_128(<4 x float> %a, <4 x float> %b) {
+ ;CHECK: vcmpleps  %xmm1, %xmm0, %k0  ## encoding: [0x62,0xf1,0x7c,0x08,0xc2,0xc1,0x02]
+   %res = call i8 @llvm.x86.avx512.mask.cmp.ps.128(<4 x float> %a, <4 x float> %b, i32 2, i8 -1)
+   ret i8 %res
+ }
+ declare i8 @llvm.x86.avx512.mask.cmp.ps.128(<4 x float> , <4 x float> , i32, i8)
+
+define i8 @test_cmppd_256(<4 x double> %a, <4 x double> %b) {
+ ;CHECK: vcmplepd  %ymm1, %ymm0, %k0  ## encoding: [0x62,0xf1,0xfd,0x28,0xc2,0xc1,0x02]
+   %res = call i8 @llvm.x86.avx512.mask.cmp.pd.256(<4 x double> %a, <4 x double> %b, i32 2, i8 -1)
+   ret i8 %res
+ }
+ declare i8 @llvm.x86.avx512.mask.cmp.pd.256(<4 x double> , <4 x double> , i32, i8)
+
+define i8 @test_cmppd_128(<2 x double> %a, <2 x double> %b) {
+ ;CHECK: vcmplepd  %xmm1, %xmm0, %k0  ## encoding: [0x62,0xf1,0xfd,0x08,0xc2,0xc1,0x02]
+   %res = call i8 @llvm.x86.avx512.mask.cmp.pd.128(<2 x double> %a, <2 x double> %b, i32 2, i8 -1)
+   ret i8 %res
+ }
+ declare i8 @llvm.x86.avx512.mask.cmp.pd.128(<2 x double> , <2 x double> , i32, i8)
+
+define <8 x float> @test_mm512_maskz_add_ps_256(<8 x float> %a0, <8 x float> %a1, i8 %mask) {
+  ;CHECK-LABEL: test_mm512_maskz_add_ps_256
+  ;CHECK: vaddps %ymm1, %ymm0, %ymm0 {%k1} {z}
+  %res = call <8 x float> @llvm.x86.avx512.mask.add.ps.256(<8 x float> %a0, <8 x float> %a1, <8 x float>zeroinitializer, i8 %mask)
+  ret <8 x float> %res
+}
+
+define <8 x float> @test_mm512_mask_add_ps_256(<8 x float> %a0, <8 x float> %a1, <8 x float> %src, i8 %mask) {
+  ;CHECK-LABEL: test_mm512_mask_add_ps_256
+  ;CHECK: vaddps %ymm1, %ymm0, %ymm2 {%k1} 
+  %res = call <8 x float> @llvm.x86.avx512.mask.add.ps.256(<8 x float> %a0, <8 x float> %a1, <8 x float> %src, i8 %mask)
+  ret <8 x float> %res
+}
+
+define <8 x float> @test_mm512_add_ps_256(<8 x float> %a0, <8 x float> %a1, i8 %mask) {
+  ;CHECK-LABEL: test_mm512_add_ps_256
+  ;CHECK: vaddps %ymm1, %ymm0, %ymm0
+  %res = call <8 x float> @llvm.x86.avx512.mask.add.ps.256(<8 x float> %a0, <8 x float> %a1, <8 x float>zeroinitializer, i8 -1)
+  ret <8 x float> %res
+}
+declare <8 x float> @llvm.x86.avx512.mask.add.ps.256(<8 x float>, <8 x float>, <8 x float>, i8)
+
+define <4 x float> @test_mm512_maskz_add_ps_128(<4 x float> %a0, <4 x float> %a1, i8 %mask) {
+  ;CHECK-LABEL: test_mm512_maskz_add_ps_128
+  ;CHECK: vaddps %xmm1, %xmm0, %xmm0 {%k1} {z}
+  %res = call <4 x float> @llvm.x86.avx512.mask.add.ps.128(<4 x float> %a0, <4 x float> %a1, <4 x float>zeroinitializer, i8 %mask)
+  ret <4 x float> %res
+}
+
+define <4 x float> @test_mm512_mask_add_ps_128(<4 x float> %a0, <4 x float> %a1, <4 x float> %src, i8 %mask) {
+  ;CHECK-LABEL: test_mm512_mask_add_ps_128
+  ;CHECK: vaddps %xmm1, %xmm0, %xmm2 {%k1} 
+  %res = call <4 x float> @llvm.x86.avx512.mask.add.ps.128(<4 x float> %a0, <4 x float> %a1, <4 x float> %src, i8 %mask)
+  ret <4 x float> %res
+}
+
+define <4 x float> @test_mm512_add_ps_128(<4 x float> %a0, <4 x float> %a1, i8 %mask) {
+  ;CHECK-LABEL: test_mm512_add_ps_128
+  ;CHECK: vaddps %xmm1, %xmm0, %xmm0
+  %res = call <4 x float> @llvm.x86.avx512.mask.add.ps.128(<4 x float> %a0, <4 x float> %a1, <4 x float>zeroinitializer, i8 -1)
+  ret <4 x float> %res
+}
+declare <4 x float> @llvm.x86.avx512.mask.add.ps.128(<4 x float>, <4 x float>, <4 x float>, i8)
+
+define <8 x float> @test_mm512_maskz_sub_ps_256(<8 x float> %a0, <8 x float> %a1, i8 %mask) {
+  ;CHECK-LABEL: test_mm512_maskz_sub_ps_256
+  ;CHECK: vsubps %ymm1, %ymm0, %ymm0 {%k1} {z}
+  %res = call <8 x float> @llvm.x86.avx512.mask.sub.ps.256(<8 x float> %a0, <8 x float> %a1, <8 x float>zeroinitializer, i8 %mask)
+  ret <8 x float> %res
+}
+
+define <8 x float> @test_mm512_mask_sub_ps_256(<8 x float> %a0, <8 x float> %a1, <8 x float> %src, i8 %mask) {
+  ;CHECK-LABEL: test_mm512_mask_sub_ps_256
+  ;CHECK: vsubps %ymm1, %ymm0, %ymm2 {%k1} 
+  %res = call <8 x float> @llvm.x86.avx512.mask.sub.ps.256(<8 x float> %a0, <8 x float> %a1, <8 x float> %src, i8 %mask)
+  ret <8 x float> %res
+}
+
+define <8 x float> @test_mm512_sub_ps_256(<8 x float> %a0, <8 x float> %a1, i8 %mask) {
+  ;CHECK-LABEL: test_mm512_sub_ps_256
+  ;CHECK: vsubps %ymm1, %ymm0, %ymm0
+  %res = call <8 x float> @llvm.x86.avx512.mask.sub.ps.256(<8 x float> %a0, <8 x float> %a1, <8 x float>zeroinitializer, i8 -1)
+  ret <8 x float> %res
+}
+declare <8 x float> @llvm.x86.avx512.mask.sub.ps.256(<8 x float>, <8 x float>, <8 x float>, i8)
+
+define <4 x float> @test_mm512_maskz_sub_ps_128(<4 x float> %a0, <4 x float> %a1, i8 %mask) {
+  ;CHECK-LABEL: test_mm512_maskz_sub_ps_128
+  ;CHECK: vsubps %xmm1, %xmm0, %xmm0 {%k1} {z}
+  %res = call <4 x float> @llvm.x86.avx512.mask.sub.ps.128(<4 x float> %a0, <4 x float> %a1, <4 x float>zeroinitializer, i8 %mask)
+  ret <4 x float> %res
+}
+
+define <4 x float> @test_mm512_mask_sub_ps_128(<4 x float> %a0, <4 x float> %a1, <4 x float> %src, i8 %mask) {
+  ;CHECK-LABEL: test_mm512_mask_sub_ps_128
+  ;CHECK: vsubps %xmm1, %xmm0, %xmm2 {%k1} 
+  %res = call <4 x float> @llvm.x86.avx512.mask.sub.ps.128(<4 x float> %a0, <4 x float> %a1, <4 x float> %src, i8 %mask)
+  ret <4 x float> %res
+}
+
+define <4 x float> @test_mm512_sub_ps_128(<4 x float> %a0, <4 x float> %a1, i8 %mask) {
+  ;CHECK-LABEL: test_mm512_sub_ps_128
+  ;CHECK: vsubps %xmm1, %xmm0, %xmm0
+  %res = call <4 x float> @llvm.x86.avx512.mask.sub.ps.128(<4 x float> %a0, <4 x float> %a1, <4 x float>zeroinitializer, i8 -1)
+  ret <4 x float> %res
+}
+declare <4 x float> @llvm.x86.avx512.mask.sub.ps.128(<4 x float>, <4 x float>, <4 x float>, i8)
+
+define <8 x float> @test_mm512_maskz_mul_ps_256(<8 x float> %a0, <8 x float> %a1, i8 %mask) {
+  ;CHECK-LABEL: test_mm512_maskz_mul_ps_256
+  ;CHECK: vmulps %ymm1, %ymm0, %ymm0 {%k1} {z}
+  %res = call <8 x float> @llvm.x86.avx512.mask.mul.ps.256(<8 x float> %a0, <8 x float> %a1, <8 x float>zeroinitializer, i8 %mask)
+  ret <8 x float> %res
+}
+
+define <8 x float> @test_mm512_mask_mul_ps_256(<8 x float> %a0, <8 x float> %a1, <8 x float> %src, i8 %mask) {
+  ;CHECK-LABEL: test_mm512_mask_mul_ps_256
+  ;CHECK: vmulps %ymm1, %ymm0, %ymm2 {%k1} 
+  %res = call <8 x float> @llvm.x86.avx512.mask.mul.ps.256(<8 x float> %a0, <8 x float> %a1, <8 x float> %src, i8 %mask)
+  ret <8 x float> %res
+}
+
+define <8 x float> @test_mm512_mul_ps_256(<8 x float> %a0, <8 x float> %a1, i8 %mask) {
+  ;CHECK-LABEL: test_mm512_mul_ps_256
+  ;CHECK: vmulps %ymm1, %ymm0, %ymm0
+  %res = call <8 x float> @llvm.x86.avx512.mask.mul.ps.256(<8 x float> %a0, <8 x float> %a1, <8 x float>zeroinitializer, i8 -1)
+  ret <8 x float> %res
+}
+declare <8 x float> @llvm.x86.avx512.mask.mul.ps.256(<8 x float>, <8 x float>, <8 x float>, i8)
+
+define <4 x float> @test_mm512_maskz_mul_ps_128(<4 x float> %a0, <4 x float> %a1, i8 %mask) {
+  ;CHECK-LABEL: test_mm512_maskz_mul_ps_128
+  ;CHECK: vmulps %xmm1, %xmm0, %xmm0 {%k1} {z}
+  %res = call <4 x float> @llvm.x86.avx512.mask.mul.ps.128(<4 x float> %a0, <4 x float> %a1, <4 x float>zeroinitializer, i8 %mask)
+  ret <4 x float> %res
+}
+
+define <4 x float> @test_mm512_mask_mul_ps_128(<4 x float> %a0, <4 x float> %a1, <4 x float> %src, i8 %mask) {
+  ;CHECK-LABEL: test_mm512_mask_mul_ps_128
+  ;CHECK: vmulps %xmm1, %xmm0, %xmm2 {%k1} 
+  %res = call <4 x float> @llvm.x86.avx512.mask.mul.ps.128(<4 x float> %a0, <4 x float> %a1, <4 x float> %src, i8 %mask)
+  ret <4 x float> %res
+}
+
+define <4 x float> @test_mm512_mul_ps_128(<4 x float> %a0, <4 x float> %a1, i8 %mask) {
+  ;CHECK-LABEL: test_mm512_mul_ps_128
+  ;CHECK: vmulps %xmm1, %xmm0, %xmm0
+  %res = call <4 x float> @llvm.x86.avx512.mask.mul.ps.128(<4 x float> %a0, <4 x float> %a1, <4 x float>zeroinitializer, i8 -1)
+  ret <4 x float> %res
+}
+declare <4 x float> @llvm.x86.avx512.mask.mul.ps.128(<4 x float>, <4 x float>, <4 x float>, i8)
+
+define <8 x float> @test_mm512_maskz_div_ps_256(<8 x float> %a0, <8 x float> %a1, i8 %mask) {
+  ;CHECK-LABEL: test_mm512_maskz_div_ps_256
+  ;CHECK: vdivps %ymm1, %ymm0, %ymm0 {%k1} {z}
+  %res = call <8 x float> @llvm.x86.avx512.mask.div.ps.256(<8 x float> %a0, <8 x float> %a1, <8 x float>zeroinitializer, i8 %mask)
+  ret <8 x float> %res
+}
+
+define <8 x float> @test_mm512_mask_div_ps_256(<8 x float> %a0, <8 x float> %a1, <8 x float> %src, i8 %mask) {
+  ;CHECK-LABEL: test_mm512_mask_div_ps_256
+  ;CHECK: vdivps %ymm1, %ymm0, %ymm2 {%k1} 
+  %res = call <8 x float> @llvm.x86.avx512.mask.div.ps.256(<8 x float> %a0, <8 x float> %a1, <8 x float> %src, i8 %mask)
+  ret <8 x float> %res
+}
+
+define <8 x float> @test_mm512_div_ps_256(<8 x float> %a0, <8 x float> %a1, i8 %mask) {
+  ;CHECK-LABEL: test_mm512_div_ps_256
+  ;CHECK: vdivps %ymm1, %ymm0, %ymm0
+  %res = call <8 x float> @llvm.x86.avx512.mask.div.ps.256(<8 x float> %a0, <8 x float> %a1, <8 x float>zeroinitializer, i8 -1)
+  ret <8 x float> %res
+}
+declare <8 x float> @llvm.x86.avx512.mask.div.ps.256(<8 x float>, <8 x float>, <8 x float>, i8)
+
+define <4 x float> @test_mm512_maskz_div_ps_128(<4 x float> %a0, <4 x float> %a1, i8 %mask) {
+  ;CHECK-LABEL: test_mm512_maskz_div_ps_128
+  ;CHECK: vdivps %xmm1, %xmm0, %xmm0 {%k1} {z}
+  %res = call <4 x float> @llvm.x86.avx512.mask.div.ps.128(<4 x float> %a0, <4 x float> %a1, <4 x float>zeroinitializer, i8 %mask)
+  ret <4 x float> %res
+}
+
+define <4 x float> @test_mm512_mask_div_ps_128(<4 x float> %a0, <4 x float> %a1, <4 x float> %src, i8 %mask) {
+  ;CHECK-LABEL: test_mm512_mask_div_ps_128
+  ;CHECK: vdivps %xmm1, %xmm0, %xmm2 {%k1} 
+  %res = call <4 x float> @llvm.x86.avx512.mask.div.ps.128(<4 x float> %a0, <4 x float> %a1, <4 x float> %src, i8 %mask)
+  ret <4 x float> %res
+}
+
+define <4 x float> @test_mm512_div_ps_128(<4 x float> %a0, <4 x float> %a1, i8 %mask) {
+  ;CHECK-LABEL: test_mm512_div_ps_128
+  ;CHECK: vdivps %xmm1, %xmm0, %xmm0
+  %res = call <4 x float> @llvm.x86.avx512.mask.div.ps.128(<4 x float> %a0, <4 x float> %a1, <4 x float>zeroinitializer, i8 -1)
+  ret <4 x float> %res
+}
+declare <4 x float> @llvm.x86.avx512.mask.div.ps.128(<4 x float>, <4 x float>, <4 x float>, i8)
+
+define <8 x float> @test_mm512_maskz_max_ps_256(<8 x float> %a0, <8 x float> %a1, i8 %mask) {
+  ;CHECK-LABEL: test_mm512_maskz_max_ps_256
+  ;CHECK: vmaxps %ymm1, %ymm0, %ymm0 {%k1} {z}
+  %res = call <8 x float> @llvm.x86.avx512.mask.max.ps.256(<8 x float> %a0, <8 x float> %a1, <8 x float>zeroinitializer, i8 %mask)
+  ret <8 x float> %res
+}
+
+define <8 x float> @test_mm512_mask_max_ps_256(<8 x float> %a0, <8 x float> %a1, <8 x float> %src, i8 %mask) {
+  ;CHECK-LABEL: test_mm512_mask_max_ps_256
+  ;CHECK: vmaxps %ymm1, %ymm0, %ymm2 {%k1} 
+  %res = call <8 x float> @llvm.x86.avx512.mask.max.ps.256(<8 x float> %a0, <8 x float> %a1, <8 x float> %src, i8 %mask)
+  ret <8 x float> %res
+}
+
+define <8 x float> @test_mm512_max_ps_256(<8 x float> %a0, <8 x float> %a1, i8 %mask) {
+  ;CHECK-LABEL: test_mm512_max_ps_256
+  ;CHECK: vmaxps %ymm1, %ymm0, %ymm0
+  %res = call <8 x float> @llvm.x86.avx512.mask.max.ps.256(<8 x float> %a0, <8 x float> %a1, <8 x float>zeroinitializer, i8 -1)
+  ret <8 x float> %res
+}
+declare <8 x float> @llvm.x86.avx512.mask.max.ps.256(<8 x float>, <8 x float>, <8 x float>, i8)
+
+define <4 x float> @test_mm512_maskz_max_ps_128(<4 x float> %a0, <4 x float> %a1, i8 %mask) {
+  ;CHECK-LABEL: test_mm512_maskz_max_ps_128
+  ;CHECK: vmaxps %xmm1, %xmm0, %xmm0 {%k1} {z}
+  %res = call <4 x float> @llvm.x86.avx512.mask.max.ps.128(<4 x float> %a0, <4 x float> %a1, <4 x float>zeroinitializer, i8 %mask)
+  ret <4 x float> %res
+}
+
+define <4 x float> @test_mm512_mask_max_ps_128(<4 x float> %a0, <4 x float> %a1, <4 x float> %src, i8 %mask) {
+  ;CHECK-LABEL: test_mm512_mask_max_ps_128
+  ;CHECK: vmaxps %xmm1, %xmm0, %xmm2 {%k1} 
+  %res = call <4 x float> @llvm.x86.avx512.mask.max.ps.128(<4 x float> %a0, <4 x float> %a1, <4 x float> %src, i8 %mask)
+  ret <4 x float> %res
+}
+
+define <4 x float> @test_mm512_max_ps_128(<4 x float> %a0, <4 x float> %a1, i8 %mask) {
+  ;CHECK-LABEL: test_mm512_max_ps_128
+  ;CHECK: vmaxps %xmm1, %xmm0, %xmm0
+  %res = call <4 x float> @llvm.x86.avx512.mask.max.ps.128(<4 x float> %a0, <4 x float> %a1, <4 x float>zeroinitializer, i8 -1)
+  ret <4 x float> %res
+}
+declare <4 x float> @llvm.x86.avx512.mask.max.ps.128(<4 x float>, <4 x float>, <4 x float>, i8)
+
+define <8 x float> @test_mm512_maskz_min_ps_256(<8 x float> %a0, <8 x float> %a1, i8 %mask) {
+  ;CHECK-LABEL: test_mm512_maskz_min_ps_256
+  ;CHECK: vminps %ymm1, %ymm0, %ymm0 {%k1} {z}
+  %res = call <8 x float> @llvm.x86.avx512.mask.min.ps.256(<8 x float> %a0, <8 x float> %a1, <8 x float>zeroinitializer, i8 %mask)
+  ret <8 x float> %res
+}
+
+define <8 x float> @test_mm512_mask_min_ps_256(<8 x float> %a0, <8 x float> %a1, <8 x float> %src, i8 %mask) {
+  ;CHECK-LABEL: test_mm512_mask_min_ps_256
+  ;CHECK: vminps %ymm1, %ymm0, %ymm2 {%k1} 
+  %res = call <8 x float> @llvm.x86.avx512.mask.min.ps.256(<8 x float> %a0, <8 x float> %a1, <8 x float> %src, i8 %mask)
+  ret <8 x float> %res
+}
+
+define <8 x float> @test_mm512_min_ps_256(<8 x float> %a0, <8 x float> %a1, i8 %mask) {
+  ;CHECK-LABEL: test_mm512_min_ps_256
+  ;CHECK: vminps %ymm1, %ymm0, %ymm0
+  %res = call <8 x float> @llvm.x86.avx512.mask.min.ps.256(<8 x float> %a0, <8 x float> %a1, <8 x float>zeroinitializer, i8 -1)
+  ret <8 x float> %res
+}
+declare <8 x float> @llvm.x86.avx512.mask.min.ps.256(<8 x float>, <8 x float>, <8 x float>, i8)
+
+define <4 x float> @test_mm512_maskz_min_ps_128(<4 x float> %a0, <4 x float> %a1, i8 %mask) {
+  ;CHECK-LABEL: test_mm512_maskz_min_ps_128
+  ;CHECK: vminps %xmm1, %xmm0, %xmm0 {%k1} {z}
+  %res = call <4 x float> @llvm.x86.avx512.mask.min.ps.128(<4 x float> %a0, <4 x float> %a1, <4 x float>zeroinitializer, i8 %mask)
+  ret <4 x float> %res
+}
+
+define <4 x float> @test_mm512_mask_min_ps_128(<4 x float> %a0, <4 x float> %a1, <4 x float> %src, i8 %mask) {
+  ;CHECK-LABEL: test_mm512_mask_min_ps_128
+  ;CHECK: vminps %xmm1, %xmm0, %xmm2 {%k1} 
+  %res = call <4 x float> @llvm.x86.avx512.mask.min.ps.128(<4 x float> %a0, <4 x float> %a1, <4 x float> %src, i8 %mask)
+  ret <4 x float> %res
+}
+
+define <4 x float> @test_mm512_min_ps_128(<4 x float> %a0, <4 x float> %a1, i8 %mask) {
+  ;CHECK-LABEL: test_mm512_min_ps_128
+  ;CHECK: vminps %xmm1, %xmm0, %xmm0
+  %res = call <4 x float> @llvm.x86.avx512.mask.min.ps.128(<4 x float> %a0, <4 x float> %a1, <4 x float>zeroinitializer, i8 -1)
+  ret <4 x float> %res
+}
+declare <4 x float> @llvm.x86.avx512.mask.min.ps.128(<4 x float>, <4 x float>, <4 x float>, i8)
+
+define <4 x double> @test_sqrt_pd_256(<4 x double> %a0, i8 %mask) {
+  ; CHECK-LABEL: test_sqrt_pd_256
+  ; CHECK: vsqrtpd
+  %res = call <4 x double> @llvm.x86.avx512.mask.sqrt.pd.256(<4 x double> %a0,  <4 x double> zeroinitializer, i8 %mask)
+  ret <4 x double> %res
+}
+declare <4 x double> @llvm.x86.avx512.mask.sqrt.pd.256(<4 x double>, <4 x double>, i8) nounwind readnone
+
+define <8 x float> @test_sqrt_ps_256(<8 x float> %a0, i8 %mask) {
+  ; CHECK-LABEL: test_sqrt_ps_256
+  ; CHECK: vsqrtps
+  %res = call <8 x float> @llvm.x86.avx512.mask.sqrt.ps.256(<8 x float> %a0, <8 x float> zeroinitializer, i8 %mask)
+  ret <8 x float> %res
+}
+
+declare <8 x float> @llvm.x86.avx512.mask.sqrt.ps.256(<8 x float>, <8 x float>, i8) nounwind readnone
+
+define <4 x double> @test_getexp_pd_256(<4 x double> %a0) {
+  ; CHECK-LABEL: test_getexp_pd_256
+  ; CHECK: vgetexppd
+  %res = call <4 x double> @llvm.x86.avx512.mask.getexp.pd.256(<4 x double> %a0,  <4 x double> zeroinitializer, i8 -1)
+  ret <4 x double> %res
+}
+
+declare <4 x double> @llvm.x86.avx512.mask.getexp.pd.256(<4 x double>, <4 x double>, i8) nounwind readnone
+
+define <8 x float> @test_getexp_ps_256(<8 x float> %a0) {
+  ; CHECK-LABEL: test_getexp_ps_256
+  ; CHECK: vgetexpps
+  %res = call <8 x float> @llvm.x86.avx512.mask.getexp.ps.256(<8 x float> %a0, <8 x float> zeroinitializer, i8 -1)
+  ret <8 x float> %res
+}
+declare <8 x float> @llvm.x86.avx512.mask.getexp.ps.256(<8 x float>, <8 x float>, i8) nounwind readnone
+
+declare <4 x i32> @llvm.x86.avx512.mask.pmaxs.d.128(<4 x i32>, <4 x i32>, <4 x i32>, i8)
+
+; CHECK-LABEL: @test_int_x86_avx512_mask_pmaxs_d_128
+; CHECK-NOT: call 
+; CHECK: vpmaxsd %xmm
+; CHECK: {%k1} 
+define <4 x i32>@test_int_x86_avx512_mask_pmaxs_d_128(<4 x i32> %x0, <4 x i32> %x1, <4 x i32> %x2, i8 %mask) {
+  %res = call <4 x i32> @llvm.x86.avx512.mask.pmaxs.d.128(<4 x i32> %x0, <4 x i32> %x1, <4 x i32> %x2 ,i8 %mask)
+  %res1 = call <4 x i32> @llvm.x86.avx512.mask.pmaxs.d.128(<4 x i32> %x0, <4 x i32> %x1, <4 x i32> zeroinitializer, i8 %mask)
+  %res2 = add <4 x i32> %res, %res1
+  ret <4 x i32> %res2
+}
+
+declare <8 x i32> @llvm.x86.avx512.mask.pmaxs.d.256(<8 x i32>, <8 x i32>, <8 x i32>, i8)
+
+; CHECK-LABEL: @test_int_x86_avx512_mask_pmaxs_d_256
+; CHECK-NOT: call 
+; CHECK: vpmaxsd %ymm
+; CHECK: {%k1} 
+define <8 x i32>@test_int_x86_avx512_mask_pmaxs_d_256(<8 x i32> %x0, <8 x i32> %x1, <8 x i32> %x2, i8 %x3) {
+  %res = call <8 x i32> @llvm.x86.avx512.mask.pmaxs.d.256(<8 x i32> %x0, <8 x i32> %x1, <8 x i32> %x2, i8 %x3)
+  %res1 = call <8 x i32> @llvm.x86.avx512.mask.pmaxs.d.256(<8 x i32> %x0, <8 x i32> %x1, <8 x i32> %x2, i8 -1)
+  %res2 = add <8 x i32> %res, %res1
+  ret <8 x i32> %res2
+}
+
+declare <2 x i64> @llvm.x86.avx512.mask.pmaxs.q.128(<2 x i64>, <2 x i64>, <2 x i64>, i8)
+
+; CHECK-LABEL: @test_int_x86_avx512_mask_pmaxs_q_128
+; CHECK-NOT: call 
+; CHECK: vpmaxsq %xmm
+; CHECK: {%k1} 
+define <2 x i64>@test_int_x86_avx512_mask_pmaxs_q_128(<2 x i64> %x0, <2 x i64> %x1, <2 x i64> %x2, i8 %x3) {
+  %res = call <2 x i64> @llvm.x86.avx512.mask.pmaxs.q.128(<2 x i64> %x0, <2 x i64> %x1, <2 x i64> %x2, i8 %x3)
+  %res1 = call <2 x i64> @llvm.x86.avx512.mask.pmaxs.q.128(<2 x i64> %x0, <2 x i64> %x1, <2 x i64> %x2, i8 -1)
+  %res2 = add <2 x i64> %res, %res1
+  ret <2 x i64> %res2
+}
+
+declare <4 x i64> @llvm.x86.avx512.mask.pmaxs.q.256(<4 x i64>, <4 x i64>, <4 x i64>, i8)
+
+; CHECK-LABEL: @test_int_x86_avx512_mask_pmaxs_q_256
+; CHECK-NOT: call 
+; CHECK: vpmaxsq %ymm
+; CHECK: {%k1} 
+define <4 x i64>@test_int_x86_avx512_mask_pmaxs_q_256(<4 x i64> %x0, <4 x i64> %x1, <4 x i64> %x2, i8 %mask) {
+  %res = call <4 x i64> @llvm.x86.avx512.mask.pmaxs.q.256(<4 x i64> %x0, <4 x i64> %x1, <4 x i64> %x2, i8 %mask)
+  %res1 = call <4 x i64> @llvm.x86.avx512.mask.pmaxs.q.256(<4 x i64> %x0, <4 x i64> %x1, <4 x i64> zeroinitializer, i8 %mask)
+  %res2 = add <4 x i64> %res, %res1
+  ret <4 x i64> %res2
+}
+
+declare <4 x i32> @llvm.x86.avx512.mask.pmaxu.d.128(<4 x i32>, <4 x i32>, <4 x i32>, i8)
+
+; CHECK-LABEL: @test_int_x86_avx512_mask_pmaxu_d_128
+; CHECK-NOT: call 
+; CHECK: vpmaxud %xmm
+; CHECK: {%k1} 
+define <4 x i32>@test_int_x86_avx512_mask_pmaxu_d_128(<4 x i32> %x0, <4 x i32> %x1, <4 x i32> %x2,i8 %mask) {
+  %res = call <4 x i32> @llvm.x86.avx512.mask.pmaxu.d.128(<4 x i32> %x0, <4 x i32> %x1, <4 x i32> %x2, i8 %mask)
+  %res1 = call <4 x i32> @llvm.x86.avx512.mask.pmaxu.d.128(<4 x i32> %x0, <4 x i32> %x1, <4 x i32> zeroinitializer, i8 %mask)
+  %res2 = add <4 x i32> %res, %res1
+  ret <4 x i32> %res2
+}
+
+declare <8 x i32> @llvm.x86.avx512.mask.pmaxu.d.256(<8 x i32>, <8 x i32>, <8 x i32>, i8)
+
+; CHECK-LABEL: @test_int_x86_avx512_mask_pmaxu_d_256
+; CHECK-NOT: call 
+; CHECK: vpmaxud %ymm
+; CHECK: {%k1} 
+define <8 x i32>@test_int_x86_avx512_mask_pmaxu_d_256(<8 x i32> %x0, <8 x i32> %x1, <8 x i32> %x2, i8 %x3) {
+  %res = call <8 x i32> @llvm.x86.avx512.mask.pmaxu.d.256(<8 x i32> %x0, <8 x i32> %x1, <8 x i32> %x2, i8 %x3)
+  %res1 = call <8 x i32> @llvm.x86.avx512.mask.pmaxu.d.256(<8 x i32> %x0, <8 x i32> %x1, <8 x i32> %x2, i8 -1)
+  %res2 = add <8 x i32> %res, %res1
+  ret <8 x i32> %res2
+}
+
+declare <2 x i64> @llvm.x86.avx512.mask.pmaxu.q.128(<2 x i64>, <2 x i64>, <2 x i64>, i8)
+
+; CHECK-LABEL: @test_int_x86_avx512_mask_pmaxu_q_128
+; CHECK-NOT: call 
+; CHECK: vpmaxuq %xmm
+; CHECK: {%k1} 
+define <2 x i64>@test_int_x86_avx512_mask_pmaxu_q_128(<2 x i64> %x0, <2 x i64> %x1, <2 x i64> %x2, i8 %x3) {
+  %res = call <2 x i64> @llvm.x86.avx512.mask.pmaxu.q.128(<2 x i64> %x0, <2 x i64> %x1, <2 x i64> %x2, i8 %x3)
+  %res1 = call <2 x i64> @llvm.x86.avx512.mask.pmaxu.q.128(<2 x i64> %x0, <2 x i64> %x1, <2 x i64> %x2, i8 -1)
+  %res2 = add <2 x i64> %res, %res1
+  ret <2 x i64> %res2
+}
+
+declare <4 x i64> @llvm.x86.avx512.mask.pmaxu.q.256(<4 x i64>, <4 x i64>, <4 x i64>, i8)
+
+; CHECK-LABEL: @test_int_x86_avx512_mask_pmaxu_q_256
+; CHECK-NOT: call 
+; CHECK: vpmaxuq %ymm
+; CHECK: {%k1} 
+define <4 x i64>@test_int_x86_avx512_mask_pmaxu_q_256(<4 x i64> %x0, <4 x i64> %x1, <4 x i64> %x2, i8 %mask) {
+  %res = call <4 x i64> @llvm.x86.avx512.mask.pmaxu.q.256(<4 x i64> %x0, <4 x i64> %x1, <4 x i64> %x2, i8 %mask)
+  %res1 = call <4 x i64> @llvm.x86.avx512.mask.pmaxu.q.256(<4 x i64> %x0, <4 x i64> %x1, <4 x i64> zeroinitializer, i8 %mask)
+  %res2 = add <4 x i64> %res, %res1
+  ret <4 x i64> %res2
+}
+
+declare <4 x i32> @llvm.x86.avx512.mask.pmins.d.128(<4 x i32>, <4 x i32>, <4 x i32>, i8)
+
+; CHECK-LABEL: @test_int_x86_avx512_mask_pmins_d_128
+; CHECK-NOT: call 
+; CHECK: vpminsd %xmm
+; CHECK: {%k1} 
+define <4 x i32>@test_int_x86_avx512_mask_pmins_d_128(<4 x i32> %x0, <4 x i32> %x1, <4 x i32> %x2, i8 %mask) {
+  %res = call <4 x i32> @llvm.x86.avx512.mask.pmins.d.128(<4 x i32> %x0, <4 x i32> %x1, <4 x i32> %x2, i8 %mask)
+  %res1 = call <4 x i32> @llvm.x86.avx512.mask.pmins.d.128(<4 x i32> %x0, <4 x i32> %x1, <4 x i32> zeroinitializer, i8 %mask)
+  %res2 = add <4 x i32> %res, %res1
+  ret <4 x i32> %res2
+}
+
+declare <8 x i32> @llvm.x86.avx512.mask.pmins.d.256(<8 x i32>, <8 x i32>, <8 x i32>, i8)
+
+; CHECK-LABEL: @test_int_x86_avx512_mask_pmins_d_256
+; CHECK-NOT: call 
+; CHECK: vpminsd %ymm
+; CHECK: {%k1} 
+define <8 x i32>@test_int_x86_avx512_mask_pmins_d_256(<8 x i32> %x0, <8 x i32> %x1, <8 x i32> %x2, i8 %x3) {
+  %res = call <8 x i32> @llvm.x86.avx512.mask.pmins.d.256(<8 x i32> %x0, <8 x i32> %x1, <8 x i32> %x2, i8 %x3)
+  %res1 = call <8 x i32> @llvm.x86.avx512.mask.pmins.d.256(<8 x i32> %x0, <8 x i32> %x1, <8 x i32> %x2, i8 -1)
+  %res2 = add <8 x i32> %res, %res1
+  ret <8 x i32> %res2
+}
+
+declare <2 x i64> @llvm.x86.avx512.mask.pmins.q.128(<2 x i64>, <2 x i64>, <2 x i64>, i8)
+
+; CHECK-LABEL: @test_int_x86_avx512_mask_pmins_q_128
+; CHECK-NOT: call 
+; CHECK: vpminsq %xmm
+; CHECK: {%k1} 
+define <2 x i64>@test_int_x86_avx512_mask_pmins_q_128(<2 x i64> %x0, <2 x i64> %x1, <2 x i64> %x2, i8 %x3) {
+  %res = call <2 x i64> @llvm.x86.avx512.mask.pmins.q.128(<2 x i64> %x0, <2 x i64> %x1, <2 x i64> %x2, i8 %x3)
+  %res1 = call <2 x i64> @llvm.x86.avx512.mask.pmins.q.128(<2 x i64> %x0, <2 x i64> %x1, <2 x i64> %x2, i8 -1)
+  %res2 = add <2 x i64> %res, %res1
+  ret <2 x i64> %res2
+}
+
+declare <4 x i64> @llvm.x86.avx512.mask.pmins.q.256(<4 x i64>, <4 x i64>, <4 x i64>, i8)
+
+; CHECK-LABEL: @test_int_x86_avx512_mask_pmins_q_256
+; CHECK-NOT: call 
+; CHECK: vpminsq %ymm
+; CHECK: {%k1} 
+define <4 x i64>@test_int_x86_avx512_mask_pmins_q_256(<4 x i64> %x0, <4 x i64> %x1, <4 x i64> %x2, i8 %mask) {
+  %res = call <4 x i64> @llvm.x86.avx512.mask.pmins.q.256(<4 x i64> %x0, <4 x i64> %x1, <4 x i64> %x2, i8 %mask)
+  %res1 = call <4 x i64> @llvm.x86.avx512.mask.pmins.q.256(<4 x i64> %x0, <4 x i64> %x1, <4 x i64> zeroinitializer, i8 %mask)
+  %res2 = add <4 x i64> %res, %res1
+  ret <4 x i64> %res2
+}
+
+declare <4 x i32> @llvm.x86.avx512.mask.pminu.d.128(<4 x i32>, <4 x i32>, <4 x i32>, i8)
+
+; CHECK-LABEL: @test_int_x86_avx512_mask_pminu_d_128
+; CHECK-NOT: call 
+; CHECK: vpminud %xmm
+; CHECK: {%k1} 
+define <4 x i32>@test_int_x86_avx512_mask_pminu_d_128(<4 x i32> %x0, <4 x i32> %x1, <4 x i32> %x2, i8 %mask) {
+  %res = call <4 x i32> @llvm.x86.avx512.mask.pminu.d.128(<4 x i32> %x0, <4 x i32> %x1, <4 x i32> %x2, i8 %mask)
+  %res1 = call <4 x i32> @llvm.x86.avx512.mask.pminu.d.128(<4 x i32> %x0, <4 x i32> %x1, <4 x i32> zeroinitializer, i8 %mask)
+  %res2 = add <4 x i32> %res, %res1
+  ret <4 x i32> %res2
+}
+
+declare <8 x i32> @llvm.x86.avx512.mask.pminu.d.256(<8 x i32>, <8 x i32>, <8 x i32>, i8)
+
+; CHECK-LABEL: @test_int_x86_avx512_mask_pminu_d_256
+; CHECK-NOT: call 
+; CHECK: vpminud %ymm
+; CHECK: {%k1} 
+define <8 x i32>@test_int_x86_avx512_mask_pminu_d_256(<8 x i32> %x0, <8 x i32> %x1, <8 x i32> %x2, i8 %x3) {
+  %res = call <8 x i32> @llvm.x86.avx512.mask.pminu.d.256(<8 x i32> %x0, <8 x i32> %x1, <8 x i32> %x2, i8 %x3)
+  %res1 = call <8 x i32> @llvm.x86.avx512.mask.pminu.d.256(<8 x i32> %x0, <8 x i32> %x1, <8 x i32> %x2, i8 -1)
+  %res2 = add <8 x i32> %res, %res1
+  ret <8 x i32> %res2
+}
+
+declare <2 x i64> @llvm.x86.avx512.mask.pminu.q.128(<2 x i64>, <2 x i64>, <2 x i64>, i8)
+
+; CHECK-LABEL: @test_int_x86_avx512_mask_pminu_q_128
+; CHECK-NOT: call 
+; CHECK: vpminuq %xmm
+; CHECK: {%k1} 
+define <2 x i64>@test_int_x86_avx512_mask_pminu_q_128(<2 x i64> %x0, <2 x i64> %x1, <2 x i64> %x2, i8 %x3) {
+  %res = call <2 x i64> @llvm.x86.avx512.mask.pminu.q.128(<2 x i64> %x0, <2 x i64> %x1, <2 x i64> %x2, i8 %x3)
+  %res1 = call <2 x i64> @llvm.x86.avx512.mask.pminu.q.128(<2 x i64> %x0, <2 x i64> %x1, <2 x i64> %x2, i8 -1)
+  %res2 = add <2 x i64> %res, %res1
+  ret <2 x i64> %res2
+}
+
+declare <4 x i64> @llvm.x86.avx512.mask.pminu.q.256(<4 x i64>, <4 x i64>, <4 x i64>, i8)
+
+; CHECK-LABEL: @test_int_x86_avx512_mask_pminu_q_256
+; CHECK-NOT: call 
+; CHECK: vpminuq %ymm
+; CHECK: {%k1} 
+define <4 x i64>@test_int_x86_avx512_mask_pminu_q_256(<4 x i64> %x0, <4 x i64> %x1, <4 x i64> %x2, i8 %mask) {
+  %res = call <4 x i64> @llvm.x86.avx512.mask.pminu.q.256(<4 x i64> %x0, <4 x i64> %x1, <4 x i64> %x2, i8 %mask)
+  %res1 = call <4 x i64> @llvm.x86.avx512.mask.pminu.q.256(<4 x i64> %x0, <4 x i64> %x1, <4 x i64> zeroinitializer, i8 %mask)
+  %res2 = add <4 x i64> %res, %res1
+  ret <4 x i64> %res2
+}
+
+declare <4 x i32> @llvm.x86.avx512.mask.vpermt2var.d.128(<4 x i32>, <4 x i32>, <4 x i32>, i8)
+
+; CHECK-LABEL: @test_int_x86_avx512_mask_vpermt2var_d_128
+; CHECK-NOT: call 
+; CHECK: kmov 
+; CHECK: vpermt2d %xmm{{.*}}{%k1}
+; CHECK-NOT: {z}
+define <4 x i32>@test_int_x86_avx512_mask_vpermt2var_d_128(<4 x i32> %x0, <4 x i32> %x1, <4 x i32> %x2, i8 %x3) {
+  %res = call <4 x i32> @llvm.x86.avx512.mask.vpermt2var.d.128(<4 x i32> %x0, <4 x i32> %x1, <4 x i32> %x2, i8 %x3)
+  %res1 = call <4 x i32> @llvm.x86.avx512.mask.vpermt2var.d.128(<4 x i32> %x0, <4 x i32> %x1, <4 x i32> %x2, i8 -1)
+  %res2 = add <4 x i32> %res, %res1
+  ret <4 x i32> %res2
+}
+
+declare <4 x i32> @llvm.x86.avx512.maskz.vpermt2var.d.128(<4 x i32>, <4 x i32>, <4 x i32>, i8)
+
+; CHECK-LABEL: @test_int_x86_avx512_maskz_vpermt2var_d_128
+; CHECK-NOT: call 
+; CHECK: kmov 
+; CHECK: vpermt2d %xmm{{.*}}{%k1} {z}
+define <4 x i32>@test_int_x86_avx512_maskz_vpermt2var_d_128(<4 x i32> %x0, <4 x i32> %x1, <4 x i32> %x2, i8 %x3) {
+  %res = call <4 x i32> @llvm.x86.avx512.maskz.vpermt2var.d.128(<4 x i32> %x0, <4 x i32> %x1, <4 x i32> %x2, i8 %x3)
+  %res1 = call <4 x i32> @llvm.x86.avx512.maskz.vpermt2var.d.128(<4 x i32> %x0, <4 x i32> %x1, <4 x i32> %x2, i8 -1)
+  %res2 = add <4 x i32> %res, %res1
+  ret <4 x i32> %res2
+}
+
+declare <8 x i32> @llvm.x86.avx512.mask.vpermt2var.d.256(<8 x i32>, <8 x i32>, <8 x i32>, i8)
+
+; CHECK-LABEL: @test_int_x86_avx512_mask_vpermt2var_d_256
+; CHECK-NOT: call 
+; CHECK: kmov 
+; CHECK: vpermt2d %ymm{{.*}}{%k1}
+; CHECK-NOT: {z}
+define <8 x i32>@test_int_x86_avx512_mask_vpermt2var_d_256(<8 x i32> %x0, <8 x i32> %x1, <8 x i32> %x2, i8 %x3) {
+  %res = call <8 x i32> @llvm.x86.avx512.mask.vpermt2var.d.256(<8 x i32> %x0, <8 x i32> %x1, <8 x i32> %x2, i8 %x3)
+  %res1 = call <8 x i32> @llvm.x86.avx512.mask.vpermt2var.d.256(<8 x i32> %x0, <8 x i32> %x1, <8 x i32> %x2, i8 -1)
+  %res2 = add <8 x i32> %res, %res1
+  ret <8 x i32> %res2
+}
+
+declare <8 x i32> @llvm.x86.avx512.maskz.vpermt2var.d.256(<8 x i32>, <8 x i32>, <8 x i32>, i8)
+
+; CHECK-LABEL: @test_int_x86_avx512_maskz_vpermt2var_d_256
+; CHECK-NOT: call 
+; CHECK: kmov 
+; CHECK: vpermt2d {{.*}}{%k1} {z}
+define <8 x i32>@test_int_x86_avx512_maskz_vpermt2var_d_256(<8 x i32> %x0, <8 x i32> %x1, <8 x i32> %x2, i8 %x3) {
+  %res = call <8 x i32> @llvm.x86.avx512.maskz.vpermt2var.d.256(<8 x i32> %x0, <8 x i32> %x1, <8 x i32> %x2, i8 %x3)
+  %res1 = call <8 x i32> @llvm.x86.avx512.maskz.vpermt2var.d.256(<8 x i32> %x0, <8 x i32> %x1, <8 x i32> %x2, i8 -1)
+  %res2 = add <8 x i32> %res, %res1
+  ret <8 x i32> %res2
+}
+
+declare <2 x double> @llvm.x86.avx512.mask.vpermi2var.pd.128(<2 x double>, <2 x i64>, <2 x double>, i8)
+
+; CHECK-LABEL: @test_int_x86_avx512_mask_vpermi2var_pd_128
+; CHECK-NOT: call 
+; CHECK: kmov 
+; CHECK: vpermi2pd %xmm{{.*}}{%k1} 
+define <2 x double>@test_int_x86_avx512_mask_vpermi2var_pd_128(<2 x double> %x0, <2 x i64> %x1, <2 x double> %x2, i8 %x3) {
+  %res = call <2 x double> @llvm.x86.avx512.mask.vpermi2var.pd.128(<2 x double> %x0, <2 x i64> %x1, <2 x double> %x2, i8 %x3)
+  %res1 = call <2 x double> @llvm.x86.avx512.mask.vpermi2var.pd.128(<2 x double> %x0, <2 x i64> %x1, <2 x double> %x2, i8 -1)
+  %res2 = fadd <2 x double> %res, %res1
+  ret <2 x double> %res2
+}
+
+declare <4 x double> @llvm.x86.avx512.mask.vpermi2var.pd.256(<4 x double>, <4 x i64>, <4 x double>, i8)
+
+; CHECK-LABEL: @test_int_x86_avx512_mask_vpermi2var_pd_256
+; CHECK-NOT: call 
+; CHECK: kmov 
+; CHECK: vpermi2pd %ymm{{.*}}{%k1} 
+define <4 x double>@test_int_x86_avx512_mask_vpermi2var_pd_256(<4 x double> %x0, <4 x i64> %x1, <4 x double> %x2, i8 %x3) {
+  %res = call <4 x double> @llvm.x86.avx512.mask.vpermi2var.pd.256(<4 x double> %x0, <4 x i64> %x1, <4 x double> %x2, i8 %x3)
+  %res1 = call <4 x double> @llvm.x86.avx512.mask.vpermi2var.pd.256(<4 x double> %x0, <4 x i64> %x1, <4 x double> %x2, i8 -1)
+  %res2 = fadd <4 x double> %res, %res1
+  ret <4 x double> %res2
+}
+
+declare <4 x float> @llvm.x86.avx512.mask.vpermi2var.ps.128(<4 x float>, <4 x i32>, <4 x float>, i8)
+
+; CHECK-LABEL: @test_int_x86_avx512_mask_vpermi2var_ps_128
+; CHECK-NOT: call 
+; CHECK: kmov 
+; CHECK: vpermi2ps %xmm{{.*}}{%k1} 
+define <4 x float>@test_int_x86_avx512_mask_vpermi2var_ps_128(<4 x float> %x0, <4 x i32> %x1, <4 x float> %x2, i8 %x3) {
+  %res = call <4 x float> @llvm.x86.avx512.mask.vpermi2var.ps.128(<4 x float> %x0, <4 x i32> %x1, <4 x float> %x2, i8 %x3)
+  %res1 = call <4 x float> @llvm.x86.avx512.mask.vpermi2var.ps.128(<4 x float> %x0, <4 x i32> %x1, <4 x float> %x2, i8 -1)
+  %res2 = fadd <4 x float> %res, %res1
+  ret <4 x float> %res2
+}
+
+declare <8 x float> @llvm.x86.avx512.mask.vpermi2var.ps.256(<8 x float>, <8 x i32>, <8 x float>, i8)
+
+; CHECK-LABEL: @test_int_x86_avx512_mask_vpermi2var_ps_256
+; CHECK-NOT: call 
+; CHECK: kmov 
+; CHECK: vpermi2ps %ymm{{.*}}{%k1} 
+define <8 x float>@test_int_x86_avx512_mask_vpermi2var_ps_256(<8 x float> %x0, <8 x i32> %x1, <8 x float> %x2, i8 %x3) {
+  %res = call <8 x float> @llvm.x86.avx512.mask.vpermi2var.ps.256(<8 x float> %x0, <8 x i32> %x1, <8 x float> %x2, i8 %x3)
+  %res1 = call <8 x float> @llvm.x86.avx512.mask.vpermi2var.ps.256(<8 x float> %x0, <8 x i32> %x1, <8 x float> %x2, i8 -1)
+  %res2 = fadd <8 x float> %res, %res1
+  ret <8 x float> %res2
+}
+
+declare <2 x i64> @llvm.x86.avx512.mask.pabs.q.128(<2 x i64>, <2 x i64>, i8)
+
+; CHECK-LABEL: @test_int_x86_avx512_mask_pabs_q_128
+; CHECK-NOT: call 
+; CHECK: kmov 
+; CHECK: vpabsq{{.*}}{%k1} 
+define <2 x i64>@test_int_x86_avx512_mask_pabs_q_128(<2 x i64> %x0, <2 x i64> %x1, i8 %x2) {
+  %res = call <2 x i64> @llvm.x86.avx512.mask.pabs.q.128(<2 x i64> %x0, <2 x i64> %x1, i8 %x2)
+  %res1 = call <2 x i64> @llvm.x86.avx512.mask.pabs.q.128(<2 x i64> %x0, <2 x i64> %x1, i8 -1)
+  %res2 = add <2 x i64> %res, %res1
+  ret <2 x i64> %res2
+}
+
+declare <4 x i64> @llvm.x86.avx512.mask.pabs.q.256(<4 x i64>, <4 x i64>, i8)
+
+; CHECK-LABEL: @test_int_x86_avx512_mask_pabs_q_256
+; CHECK-NOT: call 
+; CHECK: kmov 
+; CHECK: vpabsq{{.*}}{%k1} 
+define <4 x i64>@test_int_x86_avx512_mask_pabs_q_256(<4 x i64> %x0, <4 x i64> %x1, i8 %x2) {
+  %res = call <4 x i64> @llvm.x86.avx512.mask.pabs.q.256(<4 x i64> %x0, <4 x i64> %x1, i8 %x2)
+  %res1 = call <4 x i64> @llvm.x86.avx512.mask.pabs.q.256(<4 x i64> %x0, <4 x i64> %x1, i8 -1)
+  %res2 = add <4 x i64> %res, %res1
+  ret <4 x i64> %res2
+}
+
+declare <4 x i32> @llvm.x86.avx512.mask.pabs.d.128(<4 x i32>, <4 x i32>, i8)
+
+; CHECK-LABEL: @test_int_x86_avx512_mask_pabs_d_128
+; CHECK-NOT: call 
+; CHECK: kmov 
+; CHECK: vpabsd{{.*}}{%k1} 
+define <4 x i32>@test_int_x86_avx512_mask_pabs_d_128(<4 x i32> %x0, <4 x i32> %x1, i8 %x2) {
+  %res = call <4 x i32> @llvm.x86.avx512.mask.pabs.d.128(<4 x i32> %x0, <4 x i32> %x1, i8 %x2)
+  %res1 = call <4 x i32> @llvm.x86.avx512.mask.pabs.d.128(<4 x i32> %x0, <4 x i32> %x1, i8 -1)
+  %res2 = add <4 x i32> %res, %res1
+  ret <4 x i32> %res2
+}
+
+declare <8 x i32> @llvm.x86.avx512.mask.pabs.d.256(<8 x i32>, <8 x i32>, i8)
+
+; CHECK-LABEL: @test_int_x86_avx512_mask_pabs_d_256
+; CHECK-NOT: call 
+; CHECK: kmov 
+; CHECK: vpabsd{{.*}}{%k1} 
+define <8 x i32>@test_int_x86_avx512_mask_pabs_d_256(<8 x i32> %x0, <8 x i32> %x1, i8 %x2) {
+  %res = call <8 x i32> @llvm.x86.avx512.mask.pabs.d.256(<8 x i32> %x0, <8 x i32> %x1, i8 %x2)
+  %res1 = call <8 x i32> @llvm.x86.avx512.mask.pabs.d.256(<8 x i32> %x0, <8 x i32> %x1, i8 -1)
+  %res2 = add <8 x i32> %res, %res1
+  ret <8 x i32> %res2
+}
+
+
+declare <2 x double> @llvm.x86.avx512.mask.scalef.pd.128(<2 x double>, <2 x double>, <2 x double>, i8)
+
+; CHECK-LABEL: @test_int_x86_avx512_mask_scalef_pd_128
+; CHECK-NOT: call 
+; CHECK: kmov 
+; CHECK: vscalefpd{{.*}}{%k1} 
+define <2 x double>@test_int_x86_avx512_mask_scalef_pd_128(<2 x double> %x0, <2 x double> %x1, <2 x double> %x2, i8 %x3) {
+  %res = call <2 x double> @llvm.x86.avx512.mask.scalef.pd.128(<2 x double> %x0, <2 x double> %x1, <2 x double> %x2, i8 %x3)
+  %res1 = call <2 x double> @llvm.x86.avx512.mask.scalef.pd.128(<2 x double> %x0, <2 x double> %x1, <2 x double> %x2, i8 -1)
+  %res2 = fadd <2 x double> %res, %res1
+  ret <2 x double> %res2
+}
+
+declare <4 x double> @llvm.x86.avx512.mask.scalef.pd.256(<4 x double>, <4 x double>, <4 x double>, i8)
+
+; CHECK-LABEL: @test_int_x86_avx512_mask_scalef_pd_256
+; CHECK-NOT: call 
+; CHECK: kmov 
+; CHECK: vscalefpd{{.*}}{%k1} 
+define <4 x double>@test_int_x86_avx512_mask_scalef_pd_256(<4 x double> %x0, <4 x double> %x1, <4 x double> %x2, i8 %x3) {
+  %res = call <4 x double> @llvm.x86.avx512.mask.scalef.pd.256(<4 x double> %x0, <4 x double> %x1, <4 x double> %x2, i8 %x3)
+  %res1 = call <4 x double> @llvm.x86.avx512.mask.scalef.pd.256(<4 x double> %x0, <4 x double> %x1, <4 x double> %x2, i8 -1)
+  %res2 = fadd <4 x double> %res, %res1
+  ret <4 x double> %res2
+}
+
+declare <4 x float> @llvm.x86.avx512.mask.scalef.ps.128(<4 x float>, <4 x float>, <4 x float>, i8)
+; CHECK-LABEL: @test_int_x86_avx512_mask_scalef_ps_128
+; CHECK-NOT: call 
+; CHECK: kmov 
+; CHECK: vscalefps{{.*}}{%k1} 
+define <4 x float>@test_int_x86_avx512_mask_scalef_ps_128(<4 x float> %x0, <4 x float> %x1, <4 x float> %x2, i8 %x3) {
+  %res = call <4 x float> @llvm.x86.avx512.mask.scalef.ps.128(<4 x float> %x0, <4 x float> %x1, <4 x float> %x2, i8 %x3)
+  %res1 = call <4 x float> @llvm.x86.avx512.mask.scalef.ps.128(<4 x float> %x0, <4 x float> %x1, <4 x float> %x2, i8 -1)
+  %res2 = fadd <4 x float> %res, %res1
+  ret <4 x float> %res2
+}
+
+declare <8 x float> @llvm.x86.avx512.mask.scalef.ps.256(<8 x float>, <8 x float>, <8 x float>, i8)
+; CHECK-LABEL: @test_int_x86_avx512_mask_scalef_ps_256
+; CHECK-NOT: call 
+; CHECK: kmov 
+; CHECK: vscalefps{{.*}}{%k1} 
+define <8 x float>@test_int_x86_avx512_mask_scalef_ps_256(<8 x float> %x0, <8 x float> %x1, <8 x float> %x2, i8 %x3) {
+  %res = call <8 x float> @llvm.x86.avx512.mask.scalef.ps.256(<8 x float> %x0, <8 x float> %x1, <8 x float> %x2, i8 %x3)
+  %res1 = call <8 x float> @llvm.x86.avx512.mask.scalef.ps.256(<8 x float> %x0, <8 x float> %x1, <8 x float> %x2, i8 -1)
+  %res2 = fadd <8 x float> %res, %res1
+  ret <8 x float> %res2
+}
+
+declare <2 x double> @llvm.x86.avx512.mask.unpckh.pd.128(<2 x double>, <2 x double>, <2 x double>, i8)
+
+define <2 x double>@test_int_x86_avx512_mask_unpckh_pd_128(<2 x double> %x0, <2 x double> %x1, <2 x double> %x2, i8 %x3) {
+; CHECK-LABEL: test_int_x86_avx512_mask_unpckh_pd_128:
+; CHECK:         vunpckhpd %xmm1, %xmm0, %xmm2 {%k1} 
+; CHECK-NEXT:    vunpckhpd %xmm1, %xmm0, %xmm0 ## encoding: [0x62,0xf1,0xfd,0x08,0x15,0xc1]
+  %res = call <2 x double> @llvm.x86.avx512.mask.unpckh.pd.128(<2 x double> %x0, <2 x double> %x1, <2 x double> %x2, i8 %x3)
+  %res1 = call <2 x double> @llvm.x86.avx512.mask.unpckh.pd.128(<2 x double> %x0, <2 x double> %x1, <2 x double> %x2, i8 -1)
+  %res2 = fadd <2 x double> %res, %res1
+  ret <2 x double> %res2
+}
+
+declare <4 x double> @llvm.x86.avx512.mask.unpckh.pd.256(<4 x double>, <4 x double>, <4 x double>, i8)
+
+define <4 x double>@test_int_x86_avx512_mask_unpckh_pd_256(<4 x double> %x0, <4 x double> %x1, <4 x double> %x2, i8 %x3) {
+; CHECK-LABEL: test_int_x86_avx512_mask_unpckh_pd_256:
+; CHECK:         vunpckhpd %ymm1, %ymm0, %ymm2 {%k1}
+; CHECK-NEXT:    vunpckhpd %ymm1, %ymm0, %ymm0 ## encoding: [0x62,0xf1,0xfd,0x28,0x15,0xc1]
+  %res = call <4 x double> @llvm.x86.avx512.mask.unpckh.pd.256(<4 x double> %x0, <4 x double> %x1, <4 x double> %x2, i8 %x3)
+  %res1 = call <4 x double> @llvm.x86.avx512.mask.unpckh.pd.256(<4 x double> %x0, <4 x double> %x1, <4 x double> %x2, i8 -1)
+  %res2 = fadd <4 x double> %res, %res1
+  ret <4 x double> %res2
+}
+
+declare <4 x float> @llvm.x86.avx512.mask.unpckh.ps.128(<4 x float>, <4 x float>, <4 x float>, i8)
+
+define <4 x float>@test_int_x86_avx512_mask_unpckh_ps_128(<4 x float> %x0, <4 x float> %x1, <4 x float> %x2, i8 %x3) {
+; CHECK-LABEL: test_int_x86_avx512_mask_unpckh_ps_128:
+; CHECK:         vunpckhps %xmm1, %xmm0, %xmm2 {%k1}
+; CHECK-NEXT:    vunpckhps %xmm1, %xmm0, %xmm0 ## encoding: [0x62,0xf1,0x7c,0x08,0x15,0xc1]
+  %res = call <4 x float> @llvm.x86.avx512.mask.unpckh.ps.128(<4 x float> %x0, <4 x float> %x1, <4 x float> %x2, i8 %x3)
+  %res1 = call <4 x float> @llvm.x86.avx512.mask.unpckh.ps.128(<4 x float> %x0, <4 x float> %x1, <4 x float> %x2, i8 -1)
+  %res2 = fadd <4 x float> %res, %res1
+  ret <4 x float> %res2
+}
+
+declare <8 x float> @llvm.x86.avx512.mask.unpckh.ps.256(<8 x float>, <8 x float>, <8 x float>, i8)
+
+define <8 x float>@test_int_x86_avx512_mask_unpckh_ps_256(<8 x float> %x0, <8 x float> %x1, <8 x float> %x2, i8 %x3) {
+; CHECK-LABEL: test_int_x86_avx512_mask_unpckh_ps_256:
+; CHECK:       ## BB#0:
+; CHECK:         vunpckhps %ymm1, %ymm0, %ymm2 {%k1} 
+; CHECK-NEXT:    vunpckhps %ymm1, %ymm0, %ymm0 ## encoding: [0x62,0xf1,0x7c,0x28,0x15,0xc1]
+  %res = call <8 x float> @llvm.x86.avx512.mask.unpckh.ps.256(<8 x float> %x0, <8 x float> %x1, <8 x float> %x2, i8 %x3)
+  %res1 = call <8 x float> @llvm.x86.avx512.mask.unpckh.ps.256(<8 x float> %x0, <8 x float> %x1, <8 x float> %x2, i8 -1)
+  %res2 = fadd <8 x float> %res, %res1
+  ret <8 x float> %res2
+}
+
+declare <2 x double> @llvm.x86.avx512.mask.unpckl.pd.128(<2 x double>, <2 x double>, <2 x double>, i8)
+
+define <2 x double>@test_int_x86_avx512_mask_unpckl_pd_128(<2 x double> %x0, <2 x double> %x1, <2 x double> %x2, i8 %x3) {
+; CHECK-LABEL: test_int_x86_avx512_mask_unpckl_pd_128:
+; CHECK:         vunpcklpd %xmm1, %xmm0, %xmm2 {%k1}
+; CHECK-NEXT:    vunpcklpd %xmm1, %xmm0, %xmm0 ## encoding: [0x62,0xf1,0xfd,0x08,0x14,0xc1]
+  %res = call <2 x double> @llvm.x86.avx512.mask.unpckl.pd.128(<2 x double> %x0, <2 x double> %x1, <2 x double> %x2, i8 %x3)
+  %res1 = call <2 x double> @llvm.x86.avx512.mask.unpckl.pd.128(<2 x double> %x0, <2 x double> %x1, <2 x double> %x2, i8 -1)
+  %res2 = fadd <2 x double> %res, %res1
+  ret <2 x double> %res2
+}
+
+declare <4 x double> @llvm.x86.avx512.mask.unpckl.pd.256(<4 x double>, <4 x double>, <4 x double>, i8)
+
+define <4 x double>@test_int_x86_avx512_mask_unpckl_pd_256(<4 x double> %x0, <4 x double> %x1, <4 x double> %x2, i8 %x3) {
+; CHECK-LABEL: test_int_x86_avx512_mask_unpckl_pd_256:
+; CHECK:         vunpcklpd %ymm1, %ymm0, %ymm2 {%k1}
+; CHECK-NEXT:    vunpcklpd %ymm1, %ymm0, %ymm0 ## encoding: [0x62,0xf1,0xfd,0x28,0x14,0xc1]
+  %res = call <4 x double> @llvm.x86.avx512.mask.unpckl.pd.256(<4 x double> %x0, <4 x double> %x1, <4 x double> %x2, i8 %x3)
+  %res1 = call <4 x double> @llvm.x86.avx512.mask.unpckl.pd.256(<4 x double> %x0, <4 x double> %x1, <4 x double> %x2, i8 -1)
+  %res2 = fadd <4 x double> %res, %res1
+  ret <4 x double> %res2
+}
+
+declare <4 x float> @llvm.x86.avx512.mask.unpckl.ps.128(<4 x float>, <4 x float>, <4 x float>, i8)
+
+define <4 x float>@test_int_x86_avx512_mask_unpckl_ps_128(<4 x float> %x0, <4 x float> %x1, <4 x float> %x2, i8 %x3) {
+; CHECK-LABEL: test_int_x86_avx512_mask_unpckl_ps_128:
+; CHECK:         vunpcklps %xmm1, %xmm0, %xmm2 {%k1}
+; CHECK-NEXT:    vunpcklps %xmm1, %xmm0, %xmm0 ## encoding: [0x62,0xf1,0x7c,0x08,0x14,0xc1]
+  %res = call <4 x float> @llvm.x86.avx512.mask.unpckl.ps.128(<4 x float> %x0, <4 x float> %x1, <4 x float> %x2, i8 %x3)
+  %res1 = call <4 x float> @llvm.x86.avx512.mask.unpckl.ps.128(<4 x float> %x0, <4 x float> %x1, <4 x float> %x2, i8 -1)
+  %res2 = fadd <4 x float> %res, %res1
+  ret <4 x float> %res2
+}
+
+declare <8 x float> @llvm.x86.avx512.mask.unpckl.ps.256(<8 x float>, <8 x float>, <8 x float>, i8)
+
+define <8 x float>@test_int_x86_avx512_mask_unpckl_ps_256(<8 x float> %x0, <8 x float> %x1, <8 x float> %x2, i8 %x3) {
+; CHECK-LABEL: test_int_x86_avx512_mask_unpckl_ps_256:
+; CHECK:         vunpcklps %ymm1, %ymm0, %ymm2 {%k1}
+; CHECK-NEXT:    vunpcklps %ymm1, %ymm0, %ymm0 ## encoding: [0x62,0xf1,0x7c,0x28,0x14,0xc1]
+  %res = call <8 x float> @llvm.x86.avx512.mask.unpckl.ps.256(<8 x float> %x0, <8 x float> %x1, <8 x float> %x2, i8 %x3)
+  %res1 = call <8 x float> @llvm.x86.avx512.mask.unpckl.ps.256(<8 x float> %x0, <8 x float> %x1, <8 x float> %x2, i8 -1)
+  %res2 = fadd <8 x float> %res, %res1
+  ret <8 x float> %res2
+}
+
+declare <4 x i32> @llvm.x86.avx512.mask.punpckhd.q.128(<4 x i32>, <4 x i32>, <4 x i32>, i8)
+
+define <4 x i32>@test_int_x86_avx512_mask_punpckhd_q_128(<4 x i32> %x0, <4 x i32> %x1, <4 x i32> %x2, i8 %x3) {
+; CHECK-LABEL: test_int_x86_avx512_mask_punpckhd_q_128:
+; CHECK:         vpunpckhdq %xmm1, %xmm0, %xmm2 {%k1}
+; CHECK-NEXT:    vpunpckhdq %xmm1, %xmm0, %xmm0 ## encoding: [0x62,0xf1,0x7d,0x08,0x6a,0xc1]
+  %res = call <4 x i32> @llvm.x86.avx512.mask.punpckhd.q.128(<4 x i32> %x0, <4 x i32> %x1, <4 x i32> %x2, i8 %x3)
+  %res1 = call <4 x i32> @llvm.x86.avx512.mask.punpckhd.q.128(<4 x i32> %x0, <4 x i32> %x1, <4 x i32> %x2, i8 -1)
+  %res2 = add <4 x i32> %res, %res1
+  ret <4 x i32> %res2
+}
+
+declare <4 x i32> @llvm.x86.avx512.mask.punpckld.q.128(<4 x i32>, <4 x i32>, <4 x i32>, i8)
+
+define <4 x i32>@test_int_x86_avx512_mask_punpckld_q_128(<4 x i32> %x0, <4 x i32> %x1, <4 x i32> %x2, i8 %x3) {
+; CHECK-LABEL: test_int_x86_avx512_mask_punpckld_q_128:
+; CHECK:         vpunpckldq %xmm1, %xmm0, %xmm2 {%k1}
+; CHECK-NEXT:    vpunpckldq %xmm1, %xmm0, %xmm0 ## encoding: [0x62,0xf1,0x7d,0x08,0x62,0xc1]
+  %res = call <4 x i32> @llvm.x86.avx512.mask.punpckld.q.128(<4 x i32> %x0, <4 x i32> %x1, <4 x i32> %x2, i8 %x3)
+  %res1 = call <4 x i32> @llvm.x86.avx512.mask.punpckld.q.128(<4 x i32> %x0, <4 x i32> %x1, <4 x i32> %x2, i8 -1)
+  %res2 = add <4 x i32> %res, %res1
+  ret <4 x i32> %res2
+}
+
+declare <8 x i32> @llvm.x86.avx512.mask.punpckhd.q.256(<8 x i32>, <8 x i32>, <8 x i32>, i8)
+
+define <8 x i32>@test_int_x86_avx512_mask_punpckhd_q_256(<8 x i32> %x0, <8 x i32> %x1, <8 x i32> %x2, i8 %x3) {
+; CHECK-LABEL: test_int_x86_avx512_mask_punpckhd_q_256:
+; CHECK:       ## BB#0:
+; CHECK:         vpunpckhdq %ymm1, %ymm0, %ymm2 {%k1}
+; CHECK-NEXT:    vpunpckhdq %ymm1, %ymm0, %ymm0 ## encoding: [0x62,0xf1,0x7d,0x28,0x6a,0xc1]
+  %res = call <8 x i32> @llvm.x86.avx512.mask.punpckhd.q.256(<8 x i32> %x0, <8 x i32> %x1, <8 x i32> %x2, i8 %x3)
+  %res1 = call <8 x i32> @llvm.x86.avx512.mask.punpckhd.q.256(<8 x i32> %x0, <8 x i32> %x1, <8 x i32> %x2, i8 -1)
+  %res2 = add <8 x i32> %res, %res1
+  ret <8 x i32> %res2
+}
+
+declare <8 x i32> @llvm.x86.avx512.mask.punpckld.q.256(<8 x i32>, <8 x i32>, <8 x i32>, i8)
+
+define <8 x i32>@test_int_x86_avx512_mask_punpckld_q_256(<8 x i32> %x0, <8 x i32> %x1, <8 x i32> %x2, i8 %x3) {
+; CHECK-LABEL: test_int_x86_avx512_mask_punpckld_q_256:
+; CHECK:         vpunpckldq %ymm1, %ymm0, %ymm2 {%k1}
+; CHECK-NEXT:    vpunpckldq %ymm1, %ymm0, %ymm0 ## encoding: [0x62,0xf1,0x7d,0x28,0x62,0xc1]
+  %res = call <8 x i32> @llvm.x86.avx512.mask.punpckld.q.256(<8 x i32> %x0, <8 x i32> %x1, <8 x i32> %x2, i8 %x3)
+  %res1 = call <8 x i32> @llvm.x86.avx512.mask.punpckld.q.256(<8 x i32> %x0, <8 x i32> %x1, <8 x i32> %x2, i8 -1)
+  %res2 = add <8 x i32> %res, %res1
+  ret <8 x i32> %res2
+}
+
+declare <2 x i64> @llvm.x86.avx512.mask.punpckhqd.q.128(<2 x i64>, <2 x i64>, <2 x i64>, i8)
+
+define <2 x i64>@test_int_x86_avx512_mask_punpckhqd_q_128(<2 x i64> %x0, <2 x i64> %x1, <2 x i64> %x2, i8 %x3) {
+; CHECK-LABEL: test_int_x86_avx512_mask_punpckhqd_q_128:
+; CHECK:         vpunpckhqdq %xmm1, %xmm0, %xmm2 {%k1}
+; CHECK-NEXT:    vpunpckhqdq %xmm1, %xmm0, %xmm0 ## encoding: [0x62,0xf1,0xfd,0x08,0x6d,0xc1]
+  %res = call <2 x i64> @llvm.x86.avx512.mask.punpckhqd.q.128(<2 x i64> %x0, <2 x i64> %x1, <2 x i64> %x2, i8 %x3)
+  %res1 = call <2 x i64> @llvm.x86.avx512.mask.punpckhqd.q.128(<2 x i64> %x0, <2 x i64> %x1, <2 x i64> %x2, i8 -1)
+  %res2 = add <2 x i64> %res, %res1
+  ret <2 x i64> %res2
+}
+
+declare <2 x i64> @llvm.x86.avx512.mask.punpcklqd.q.128(<2 x i64>, <2 x i64>, <2 x i64>, i8)
+
+define <2 x i64>@test_int_x86_avx512_mask_punpcklqd_q_128(<2 x i64> %x0, <2 x i64> %x1, <2 x i64> %x2, i8 %x3) {
+; CHECK-LABEL: test_int_x86_avx512_mask_punpcklqd_q_128:
+; CHECK:         vpunpcklqdq %xmm1, %xmm0, %xmm2 {%k1}
+; CHECK-NEXT:    vpunpcklqdq %xmm1, %xmm0, %xmm0 ## encoding: [0x62,0xf1,0xfd,0x08,0x6c,0xc1]
+  %res = call <2 x i64> @llvm.x86.avx512.mask.punpcklqd.q.128(<2 x i64> %x0, <2 x i64> %x1, <2 x i64> %x2, i8 %x3)
+  %res1 = call <2 x i64> @llvm.x86.avx512.mask.punpcklqd.q.128(<2 x i64> %x0, <2 x i64> %x1, <2 x i64> %x2, i8 -1)
+  %res2 = add <2 x i64> %res, %res1
+  ret <2 x i64> %res2
+}
+
+declare <4 x i64> @llvm.x86.avx512.mask.punpcklqd.q.256(<4 x i64>, <4 x i64>, <4 x i64>, i8)
+
+define <4 x i64>@test_int_x86_avx512_mask_punpcklqd_q_256(<4 x i64> %x0, <4 x i64> %x1, <4 x i64> %x2, i8 %x3) {
+; CHECK-LABEL: test_int_x86_avx512_mask_punpcklqd_q_256:
+; CHECK:         vpunpcklqdq %ymm1, %ymm0, %ymm2 {%k1}
+; CHECK-NEXT:    vpunpcklqdq %ymm1, %ymm0, %ymm0 ## encoding: [0x62,0xf1,0xfd,0x28,0x6c,0xc1]
+  %res = call <4 x i64> @llvm.x86.avx512.mask.punpcklqd.q.256(<4 x i64> %x0, <4 x i64> %x1, <4 x i64> %x2, i8 %x3)
+  %res1 = call <4 x i64> @llvm.x86.avx512.mask.punpcklqd.q.256(<4 x i64> %x0, <4 x i64> %x1, <4 x i64> %x2, i8 -1)
+  %res2 = add <4 x i64> %res, %res1
+  ret <4 x i64> %res2
+}
+
+declare <4 x i64> @llvm.x86.avx512.mask.punpckhqd.q.256(<4 x i64>, <4 x i64>, <4 x i64>, i8)
+
+define <4 x i64>@test_int_x86_avx512_mask_punpckhqd_q_256(<4 x i64> %x0, <4 x i64> %x1, <4 x i64> %x2, i8 %x3) {
+; CHECK-LABEL: test_int_x86_avx512_mask_punpckhqd_q_256:
+; CHECK:         vpunpckhqdq %ymm1, %ymm0, %ymm2 {%k1}
+; CHECK-NEXT:    vpunpckhqdq %ymm1, %ymm0, %ymm0 ## encoding: [0x62,0xf1,0xfd,0x28,0x6d,0xc1]
+  %res = call <4 x i64> @llvm.x86.avx512.mask.punpckhqd.q.256(<4 x i64> %x0, <4 x i64> %x1, <4 x i64> %x2, i8 %x3)
+  %res1 = call <4 x i64> @llvm.x86.avx512.mask.punpckhqd.q.256(<4 x i64> %x0, <4 x i64> %x1, <4 x i64> %x2, i8 -1)
+  %res2 = add <4 x i64> %res, %res1
+  ret <4 x i64> %res2
+}
+
+declare <16 x i8> @llvm.x86.avx512.mask.pmov.qb.128(<2 x i64>, <16 x i8>, i8)
+
+define <16 x i8>@test_int_x86_avx512_mask_pmov_qb_128(<2 x i64> %x0, <16 x i8> %x1, i8 %x2) {
+; CHECK-LABEL: test_int_x86_avx512_mask_pmov_qb_128:
+; CHECK:       vpmovqb %xmm0, %xmm1 {%k1}
+; CHECK-NEXT:  vpmovqb %xmm0, %xmm2 {%k1} {z}
+; CHECK-NEXT:  vpmovqb %xmm0, %xmm0
+    %res0 = call <16 x i8> @llvm.x86.avx512.mask.pmov.qb.128(<2 x i64> %x0, <16 x i8> %x1, i8 -1)
+    %res1 = call <16 x i8> @llvm.x86.avx512.mask.pmov.qb.128(<2 x i64> %x0, <16 x i8> %x1, i8 %x2)
+    %res2 = call <16 x i8> @llvm.x86.avx512.mask.pmov.qb.128(<2 x i64> %x0, <16 x i8> zeroinitializer, i8 %x2)
+    %res3 = add <16 x i8> %res0, %res1
+    %res4 = add <16 x i8> %res3, %res2
+    ret <16 x i8> %res4
+}
+
+declare void @llvm.x86.avx512.mask.pmov.qb.mem.128(i8* %ptr, <2 x i64>, i8)
+
+define void @test_int_x86_avx512_mask_pmov_qb_mem_128(i8* %ptr, <2 x i64> %x1, i8 %x2) {
+; CHECK-LABEL: test_int_x86_avx512_mask_pmov_qb_mem_128:
+; CHECK:  vpmovqb %xmm0, (%rdi)
+; CHECK:  vpmovqb %xmm0, (%rdi) {%k1}
+    call void @llvm.x86.avx512.mask.pmov.qb.mem.128(i8* %ptr, <2 x i64> %x1, i8 -1)
+    call void @llvm.x86.avx512.mask.pmov.qb.mem.128(i8* %ptr, <2 x i64> %x1, i8 %x2)
+    ret void
+}
+
+declare <16 x i8> @llvm.x86.avx512.mask.pmovs.qb.128(<2 x i64>, <16 x i8>, i8)
+
+define <16 x i8>@test_int_x86_avx512_mask_pmovs_qb_128(<2 x i64> %x0, <16 x i8> %x1, i8 %x2) {
+; CHECK-LABEL: test_int_x86_avx512_mask_pmovs_qb_128:
+; CHECK:       vpmovsqb %xmm0, %xmm1 {%k1}
+; CHECK-NEXT:  vpmovsqb %xmm0, %xmm2 {%k1} {z}
+; CHECK-NEXT:  vpmovsqb %xmm0, %xmm0
+    %res0 = call <16 x i8> @llvm.x86.avx512.mask.pmovs.qb.128(<2 x i64> %x0, <16 x i8> %x1, i8 -1)
+    %res1 = call <16 x i8> @llvm.x86.avx512.mask.pmovs.qb.128(<2 x i64> %x0, <16 x i8> %x1, i8 %x2)
+    %res2 = call <16 x i8> @llvm.x86.avx512.mask.pmovs.qb.128(<2 x i64> %x0, <16 x i8> zeroinitializer, i8 %x2)
+    %res3 = add <16 x i8> %res0, %res1
+    %res4 = add <16 x i8> %res3, %res2
+    ret <16 x i8> %res4
+}
+
+declare void @llvm.x86.avx512.mask.pmovs.qb.mem.128(i8* %ptr, <2 x i64>, i8)
+
+define void @test_int_x86_avx512_mask_pmovs_qb_mem_128(i8* %ptr, <2 x i64> %x1, i8 %x2) {
+; CHECK-LABEL: test_int_x86_avx512_mask_pmovs_qb_mem_128:
+; CHECK:  vpmovsqb %xmm0, (%rdi)
+; CHECK:  vpmovsqb %xmm0, (%rdi) {%k1}
+    call void @llvm.x86.avx512.mask.pmovs.qb.mem.128(i8* %ptr, <2 x i64> %x1, i8 -1)
+    call void @llvm.x86.avx512.mask.pmovs.qb.mem.128(i8* %ptr, <2 x i64> %x1, i8 %x2)
+    ret void
+}
+
+declare <16 x i8> @llvm.x86.avx512.mask.pmovus.qb.128(<2 x i64>, <16 x i8>, i8)
+
+define <16 x i8>@test_int_x86_avx512_mask_pmovus_qb_128(<2 x i64> %x0, <16 x i8> %x1, i8 %x2) {
+; CHECK-LABEL: test_int_x86_avx512_mask_pmovus_qb_128:
+; CHECK:       vpmovusqb %xmm0, %xmm1 {%k1}
+; CHECK-NEXT:  vpmovusqb %xmm0, %xmm2 {%k1} {z}
+; CHECK-NEXT:  vpmovusqb %xmm0, %xmm0
+    %res0 = call <16 x i8> @llvm.x86.avx512.mask.pmovus.qb.128(<2 x i64> %x0, <16 x i8> %x1, i8 -1)
+    %res1 = call <16 x i8> @llvm.x86.avx512.mask.pmovus.qb.128(<2 x i64> %x0, <16 x i8> %x1, i8 %x2)
+    %res2 = call <16 x i8> @llvm.x86.avx512.mask.pmovus.qb.128(<2 x i64> %x0, <16 x i8> zeroinitializer, i8 %x2)
+    %res3 = add <16 x i8> %res0, %res1
+    %res4 = add <16 x i8> %res3, %res2
+    ret <16 x i8> %res4
+}
+
+declare void @llvm.x86.avx512.mask.pmovus.qb.mem.128(i8* %ptr, <2 x i64>, i8)
+
+define void @test_int_x86_avx512_mask_pmovus_qb_mem_128(i8* %ptr, <2 x i64> %x1, i8 %x2) {
+; CHECK-LABEL: test_int_x86_avx512_mask_pmovus_qb_mem_128:
+; CHECK:  vpmovusqb %xmm0, (%rdi)
+; CHECK:  vpmovusqb %xmm0, (%rdi) {%k1}
+    call void @llvm.x86.avx512.mask.pmovus.qb.mem.128(i8* %ptr, <2 x i64> %x1, i8 -1)
+    call void @llvm.x86.avx512.mask.pmovus.qb.mem.128(i8* %ptr, <2 x i64> %x1, i8 %x2)
+    ret void
+}
+
+declare <16 x i8> @llvm.x86.avx512.mask.pmov.qb.256(<4 x i64>, <16 x i8>, i8)
+
+define <16 x i8>@test_int_x86_avx512_mask_pmov_qb_256(<4 x i64> %x0, <16 x i8> %x1, i8 %x2) {
+; CHECK-LABEL: test_int_x86_avx512_mask_pmov_qb_256:
+; CHECK:       vpmovqb %ymm0, %xmm1 {%k1}
+; CHECK-NEXT:  vpmovqb %ymm0, %xmm2 {%k1} {z}
+; CHECK-NEXT:  vpmovqb %ymm0, %xmm0
+    %res0 = call <16 x i8> @llvm.x86.avx512.mask.pmov.qb.256(<4 x i64> %x0, <16 x i8> %x1, i8 -1)
+    %res1 = call <16 x i8> @llvm.x86.avx512.mask.pmov.qb.256(<4 x i64> %x0, <16 x i8> %x1, i8 %x2)
+    %res2 = call <16 x i8> @llvm.x86.avx512.mask.pmov.qb.256(<4 x i64> %x0, <16 x i8> zeroinitializer, i8 %x2)
+    %res3 = add <16 x i8> %res0, %res1
+    %res4 = add <16 x i8> %res3, %res2
+    ret <16 x i8> %res4
+}
+
+declare void @llvm.x86.avx512.mask.pmov.qb.mem.256(i8* %ptr, <4 x i64>, i8)
+
+define void @test_int_x86_avx512_mask_pmov_qb_mem_256(i8* %ptr, <4 x i64> %x1, i8 %x2) {
+; CHECK-LABEL: test_int_x86_avx512_mask_pmov_qb_mem_256:
+; CHECK:  vpmovqb %ymm0, (%rdi)
+; CHECK:  vpmovqb %ymm0, (%rdi) {%k1}
+    call void @llvm.x86.avx512.mask.pmov.qb.mem.256(i8* %ptr, <4 x i64> %x1, i8 -1)
+    call void @llvm.x86.avx512.mask.pmov.qb.mem.256(i8* %ptr, <4 x i64> %x1, i8 %x2)
+    ret void
+}
+
+declare <16 x i8> @llvm.x86.avx512.mask.pmovs.qb.256(<4 x i64>, <16 x i8>, i8)
+
+define <16 x i8>@test_int_x86_avx512_mask_pmovs_qb_256(<4 x i64> %x0, <16 x i8> %x1, i8 %x2) {
+; CHECK-LABEL: test_int_x86_avx512_mask_pmovs_qb_256:
+; CHECK:       vpmovsqb %ymm0, %xmm1 {%k1}
+; CHECK-NEXT:  vpmovsqb %ymm0, %xmm2 {%k1} {z}
+; CHECK-NEXT:  vpmovsqb %ymm0, %xmm0
+    %res0 = call <16 x i8> @llvm.x86.avx512.mask.pmovs.qb.256(<4 x i64> %x0, <16 x i8> %x1, i8 -1)
+    %res1 = call <16 x i8> @llvm.x86.avx512.mask.pmovs.qb.256(<4 x i64> %x0, <16 x i8> %x1, i8 %x2)
+    %res2 = call <16 x i8> @llvm.x86.avx512.mask.pmovs.qb.256(<4 x i64> %x0, <16 x i8> zeroinitializer, i8 %x2)
+    %res3 = add <16 x i8> %res0, %res1
+    %res4 = add <16 x i8> %res3, %res2
+    ret <16 x i8> %res4
+}
+
+declare void @llvm.x86.avx512.mask.pmovs.qb.mem.256(i8* %ptr, <4 x i64>, i8)
+
+define void @test_int_x86_avx512_mask_pmovs_qb_mem_256(i8* %ptr, <4 x i64> %x1, i8 %x2) {
+; CHECK-LABEL: test_int_x86_avx512_mask_pmovs_qb_mem_256:
+; CHECK:  vpmovsqb %ymm0, (%rdi)
+; CHECK:  vpmovsqb %ymm0, (%rdi) {%k1}
+    call void @llvm.x86.avx512.mask.pmovs.qb.mem.256(i8* %ptr, <4 x i64> %x1, i8 -1)
+    call void @llvm.x86.avx512.mask.pmovs.qb.mem.256(i8* %ptr, <4 x i64> %x1, i8 %x2)
+    ret void
+}
+
+declare <16 x i8> @llvm.x86.avx512.mask.pmovus.qb.256(<4 x i64>, <16 x i8>, i8)
+
+define <16 x i8>@test_int_x86_avx512_mask_pmovus_qb_256(<4 x i64> %x0, <16 x i8> %x1, i8 %x2) {
+; CHECK-LABEL: test_int_x86_avx512_mask_pmovus_qb_256:
+; CHECK:       vpmovusqb %ymm0, %xmm1 {%k1}
+; CHECK-NEXT:  vpmovusqb %ymm0, %xmm2 {%k1} {z}
+; CHECK-NEXT:  vpmovusqb %ymm0, %xmm0
+    %res0 = call <16 x i8> @llvm.x86.avx512.mask.pmovus.qb.256(<4 x i64> %x0, <16 x i8> %x1, i8 -1)
+    %res1 = call <16 x i8> @llvm.x86.avx512.mask.pmovus.qb.256(<4 x i64> %x0, <16 x i8> %x1, i8 %x2)
+    %res2 = call <16 x i8> @llvm.x86.avx512.mask.pmovus.qb.256(<4 x i64> %x0, <16 x i8> zeroinitializer, i8 %x2)
+    %res3 = add <16 x i8> %res0, %res1
+    %res4 = add <16 x i8> %res3, %res2
+    ret <16 x i8> %res4
+}
+
+declare void @llvm.x86.avx512.mask.pmovus.qb.mem.256(i8* %ptr, <4 x i64>, i8)
+
+define void @test_int_x86_avx512_mask_pmovus_qb_mem_256(i8* %ptr, <4 x i64> %x1, i8 %x2) {
+; CHECK-LABEL: test_int_x86_avx512_mask_pmovus_qb_mem_256:
+; CHECK:  vpmovusqb %ymm0, (%rdi)
+; CHECK:  vpmovusqb %ymm0, (%rdi) {%k1}
+    call void @llvm.x86.avx512.mask.pmovus.qb.mem.256(i8* %ptr, <4 x i64> %x1, i8 -1)
+    call void @llvm.x86.avx512.mask.pmovus.qb.mem.256(i8* %ptr, <4 x i64> %x1, i8 %x2)
+    ret void
+}
+
+declare <8 x i16> @llvm.x86.avx512.mask.pmov.qw.128(<2 x i64>, <8 x i16>, i8)
+
+define <8 x i16>@test_int_x86_avx512_mask_pmov_qw_128(<2 x i64> %x0, <8 x i16> %x1, i8 %x2) {
+; CHECK-LABEL: test_int_x86_avx512_mask_pmov_qw_128:
+; CHECK:       vpmovqw %xmm0, %xmm1 {%k1}
+; CHECK-NEXT:  vpmovqw %xmm0, %xmm2 {%k1} {z}
+; CHECK-NEXT:  vpmovqw %xmm0, %xmm0
+    %res0 = call <8 x i16> @llvm.x86.avx512.mask.pmov.qw.128(<2 x i64> %x0, <8 x i16> %x1, i8 -1)
+    %res1 = call <8 x i16> @llvm.x86.avx512.mask.pmov.qw.128(<2 x i64> %x0, <8 x i16> %x1, i8 %x2)
+    %res2 = call <8 x i16> @llvm.x86.avx512.mask.pmov.qw.128(<2 x i64> %x0, <8 x i16> zeroinitializer, i8 %x2)
+    %res3 = add <8 x i16> %res0, %res1
+    %res4 = add <8 x i16> %res3, %res2
+    ret <8 x i16> %res4
+}
+
+declare void @llvm.x86.avx512.mask.pmov.qw.mem.128(i8* %ptr, <2 x i64>, i8)
+
+define void @test_int_x86_avx512_mask_pmov_qw_mem_128(i8* %ptr, <2 x i64> %x1, i8 %x2) {
+; CHECK-LABEL: test_int_x86_avx512_mask_pmov_qw_mem_128:
+; CHECK:  vpmovqw %xmm0, (%rdi)
+; CHECK:  vpmovqw %xmm0, (%rdi) {%k1}
+    call void @llvm.x86.avx512.mask.pmov.qw.mem.128(i8* %ptr, <2 x i64> %x1, i8 -1)
+    call void @llvm.x86.avx512.mask.pmov.qw.mem.128(i8* %ptr, <2 x i64> %x1, i8 %x2)
+    ret void
+}
+
+declare <8 x i16> @llvm.x86.avx512.mask.pmovs.qw.128(<2 x i64>, <8 x i16>, i8)
+
+define <8 x i16>@test_int_x86_avx512_mask_pmovs_qw_128(<2 x i64> %x0, <8 x i16> %x1, i8 %x2) {
+; CHECK-LABEL: test_int_x86_avx512_mask_pmovs_qw_128:
+; CHECK:       vpmovsqw %xmm0, %xmm1 {%k1}
+; CHECK-NEXT:  vpmovsqw %xmm0, %xmm2 {%k1} {z}
+; CHECK-NEXT:  vpmovsqw %xmm0, %xmm0
+    %res0 = call <8 x i16> @llvm.x86.avx512.mask.pmovs.qw.128(<2 x i64> %x0, <8 x i16> %x1, i8 -1)
+    %res1 = call <8 x i16> @llvm.x86.avx512.mask.pmovs.qw.128(<2 x i64> %x0, <8 x i16> %x1, i8 %x2)
+    %res2 = call <8 x i16> @llvm.x86.avx512.mask.pmovs.qw.128(<2 x i64> %x0, <8 x i16> zeroinitializer, i8 %x2)
+    %res3 = add <8 x i16> %res0, %res1
+    %res4 = add <8 x i16> %res3, %res2
+    ret <8 x i16> %res4
+}
+
+declare void @llvm.x86.avx512.mask.pmovs.qw.mem.128(i8* %ptr, <2 x i64>, i8)
+
+define void @test_int_x86_avx512_mask_pmovs_qw_mem_128(i8* %ptr, <2 x i64> %x1, i8 %x2) {
+; CHECK-LABEL: test_int_x86_avx512_mask_pmovs_qw_mem_128:
+; CHECK:  vpmovsqw %xmm0, (%rdi)
+; CHECK:  vpmovsqw %xmm0, (%rdi) {%k1}
+    call void @llvm.x86.avx512.mask.pmovs.qw.mem.128(i8* %ptr, <2 x i64> %x1, i8 -1)
+    call void @llvm.x86.avx512.mask.pmovs.qw.mem.128(i8* %ptr, <2 x i64> %x1, i8 %x2)
+    ret void
+}
+
+declare <8 x i16> @llvm.x86.avx512.mask.pmovus.qw.128(<2 x i64>, <8 x i16>, i8)
+
+define <8 x i16>@test_int_x86_avx512_mask_pmovus_qw_128(<2 x i64> %x0, <8 x i16> %x1, i8 %x2) {
+; CHECK-LABEL: test_int_x86_avx512_mask_pmovus_qw_128:
+; CHECK:       vpmovusqw %xmm0, %xmm1 {%k1}
+; CHECK-NEXT:  vpmovusqw %xmm0, %xmm2 {%k1} {z}
+; CHECK-NEXT:  vpmovusqw %xmm0, %xmm0
+    %res0 = call <8 x i16> @llvm.x86.avx512.mask.pmovus.qw.128(<2 x i64> %x0, <8 x i16> %x1, i8 -1)
+    %res1 = call <8 x i16> @llvm.x86.avx512.mask.pmovus.qw.128(<2 x i64> %x0, <8 x i16> %x1, i8 %x2)
+    %res2 = call <8 x i16> @llvm.x86.avx512.mask.pmovus.qw.128(<2 x i64> %x0, <8 x i16> zeroinitializer, i8 %x2)
+    %res3 = add <8 x i16> %res0, %res1
+    %res4 = add <8 x i16> %res3, %res2
+    ret <8 x i16> %res4
+}
+
+declare void @llvm.x86.avx512.mask.pmovus.qw.mem.128(i8* %ptr, <2 x i64>, i8)
+
+define void @test_int_x86_avx512_mask_pmovus_qw_mem_128(i8* %ptr, <2 x i64> %x1, i8 %x2) {
+; CHECK-LABEL: test_int_x86_avx512_mask_pmovus_qw_mem_128:
+; CHECK:  vpmovusqw %xmm0, (%rdi)
+; CHECK:  vpmovusqw %xmm0, (%rdi) {%k1}
+    call void @llvm.x86.avx512.mask.pmovus.qw.mem.128(i8* %ptr, <2 x i64> %x1, i8 -1)
+    call void @llvm.x86.avx512.mask.pmovus.qw.mem.128(i8* %ptr, <2 x i64> %x1, i8 %x2)
+    ret void
+}
+
+declare <8 x i16> @llvm.x86.avx512.mask.pmov.qw.256(<4 x i64>, <8 x i16>, i8)
+
+define <8 x i16>@test_int_x86_avx512_mask_pmov_qw_256(<4 x i64> %x0, <8 x i16> %x1, i8 %x2) {
+; CHECK-LABEL: test_int_x86_avx512_mask_pmov_qw_256:
+; CHECK:       vpmovqw %ymm0, %xmm1 {%k1}
+; CHECK-NEXT:  vpmovqw %ymm0, %xmm2 {%k1} {z}
+; CHECK-NEXT:  vpmovqw %ymm0, %xmm0
+    %res0 = call <8 x i16> @llvm.x86.avx512.mask.pmov.qw.256(<4 x i64> %x0, <8 x i16> %x1, i8 -1)
+    %res1 = call <8 x i16> @llvm.x86.avx512.mask.pmov.qw.256(<4 x i64> %x0, <8 x i16> %x1, i8 %x2)
+    %res2 = call <8 x i16> @llvm.x86.avx512.mask.pmov.qw.256(<4 x i64> %x0, <8 x i16> zeroinitializer, i8 %x2)
+    %res3 = add <8 x i16> %res0, %res1
+    %res4 = add <8 x i16> %res3, %res2
+    ret <8 x i16> %res4
+}
+
+declare void @llvm.x86.avx512.mask.pmov.qw.mem.256(i8* %ptr, <4 x i64>, i8)
+
+define void @test_int_x86_avx512_mask_pmov_qw_mem_256(i8* %ptr, <4 x i64> %x1, i8 %x2) {
+; CHECK-LABEL: test_int_x86_avx512_mask_pmov_qw_mem_256:
+; CHECK:  vpmovqw %ymm0, (%rdi)
+; CHECK:  vpmovqw %ymm0, (%rdi) {%k1}
+    call void @llvm.x86.avx512.mask.pmov.qw.mem.256(i8* %ptr, <4 x i64> %x1, i8 -1)
+    call void @llvm.x86.avx512.mask.pmov.qw.mem.256(i8* %ptr, <4 x i64> %x1, i8 %x2)
+    ret void
+}
+
+declare <8 x i16> @llvm.x86.avx512.mask.pmovs.qw.256(<4 x i64>, <8 x i16>, i8)
+
+define <8 x i16>@test_int_x86_avx512_mask_pmovs_qw_256(<4 x i64> %x0, <8 x i16> %x1, i8 %x2) {
+; CHECK-LABEL: test_int_x86_avx512_mask_pmovs_qw_256:
+; CHECK:       vpmovsqw %ymm0, %xmm1 {%k1}
+; CHECK-NEXT:  vpmovsqw %ymm0, %xmm2 {%k1} {z}
+; CHECK-NEXT:  vpmovsqw %ymm0, %xmm0
+    %res0 = call <8 x i16> @llvm.x86.avx512.mask.pmovs.qw.256(<4 x i64> %x0, <8 x i16> %x1, i8 -1)
+    %res1 = call <8 x i16> @llvm.x86.avx512.mask.pmovs.qw.256(<4 x i64> %x0, <8 x i16> %x1, i8 %x2)
+    %res2 = call <8 x i16> @llvm.x86.avx512.mask.pmovs.qw.256(<4 x i64> %x0, <8 x i16> zeroinitializer, i8 %x2)
+    %res3 = add <8 x i16> %res0, %res1
+    %res4 = add <8 x i16> %res3, %res2
+    ret <8 x i16> %res4
+}
+
+declare void @llvm.x86.avx512.mask.pmovs.qw.mem.256(i8* %ptr, <4 x i64>, i8)
+
+define void @test_int_x86_avx512_mask_pmovs_qw_mem_256(i8* %ptr, <4 x i64> %x1, i8 %x2) {
+; CHECK-LABEL: test_int_x86_avx512_mask_pmovs_qw_mem_256:
+; CHECK:  vpmovsqw %ymm0, (%rdi)
+; CHECK:  vpmovsqw %ymm0, (%rdi) {%k1}
+    call void @llvm.x86.avx512.mask.pmovs.qw.mem.256(i8* %ptr, <4 x i64> %x1, i8 -1)
+    call void @llvm.x86.avx512.mask.pmovs.qw.mem.256(i8* %ptr, <4 x i64> %x1, i8 %x2)
+    ret void
+}
+
+declare <8 x i16> @llvm.x86.avx512.mask.pmovus.qw.256(<4 x i64>, <8 x i16>, i8)
+
+define <8 x i16>@test_int_x86_avx512_mask_pmovus_qw_256(<4 x i64> %x0, <8 x i16> %x1, i8 %x2) {
+; CHECK-LABEL: test_int_x86_avx512_mask_pmovus_qw_256:
+; CHECK:       vpmovusqw %ymm0, %xmm1 {%k1}
+; CHECK-NEXT:  vpmovusqw %ymm0, %xmm2 {%k1} {z}
+; CHECK-NEXT:  vpmovusqw %ymm0, %xmm0
+    %res0 = call <8 x i16> @llvm.x86.avx512.mask.pmovus.qw.256(<4 x i64> %x0, <8 x i16> %x1, i8 -1)
+    %res1 = call <8 x i16> @llvm.x86.avx512.mask.pmovus.qw.256(<4 x i64> %x0, <8 x i16> %x1, i8 %x2)
+    %res2 = call <8 x i16> @llvm.x86.avx512.mask.pmovus.qw.256(<4 x i64> %x0, <8 x i16> zeroinitializer, i8 %x2)
+    %res3 = add <8 x i16> %res0, %res1
+    %res4 = add <8 x i16> %res3, %res2
+    ret <8 x i16> %res4
+}
+
+declare void @llvm.x86.avx512.mask.pmovus.qw.mem.256(i8* %ptr, <4 x i64>, i8)
+
+define void @test_int_x86_avx512_mask_pmovus_qw_mem_256(i8* %ptr, <4 x i64> %x1, i8 %x2) {
+; CHECK-LABEL: test_int_x86_avx512_mask_pmovus_qw_mem_256:
+; CHECK:  vpmovusqw %ymm0, (%rdi)
+; CHECK:  vpmovusqw %ymm0, (%rdi) {%k1}
+    call void @llvm.x86.avx512.mask.pmovus.qw.mem.256(i8* %ptr, <4 x i64> %x1, i8 -1)
+    call void @llvm.x86.avx512.mask.pmovus.qw.mem.256(i8* %ptr, <4 x i64> %x1, i8 %x2)
+    ret void
+}
+
+declare <4 x i32> @llvm.x86.avx512.mask.pmov.qd.128(<2 x i64>, <4 x i32>, i8)
+
+define <4 x i32>@test_int_x86_avx512_mask_pmov_qd_128(<2 x i64> %x0, <4 x i32> %x1, i8 %x2) {
+; CHECK-LABEL: test_int_x86_avx512_mask_pmov_qd_128:
+; CHECK:       vpmovqd %xmm0, %xmm1 {%k1}
+; CHECK-NEXT:  vpmovqd %xmm0, %xmm2 {%k1} {z}
+; CHECK-NEXT:  vpmovqd %xmm0, %xmm0
+    %res0 = call <4 x i32> @llvm.x86.avx512.mask.pmov.qd.128(<2 x i64> %x0, <4 x i32> %x1, i8 -1)
+    %res1 = call <4 x i32> @llvm.x86.avx512.mask.pmov.qd.128(<2 x i64> %x0, <4 x i32> %x1, i8 %x2)
+    %res2 = call <4 x i32> @llvm.x86.avx512.mask.pmov.qd.128(<2 x i64> %x0, <4 x i32> zeroinitializer, i8 %x2)
+    %res3 = add <4 x i32> %res0, %res1
+    %res4 = add <4 x i32> %res3, %res2
+    ret <4 x i32> %res4
+}
+
+declare void @llvm.x86.avx512.mask.pmov.qd.mem.128(i8* %ptr, <2 x i64>, i8)
+
+define void @test_int_x86_avx512_mask_pmov_qd_mem_128(i8* %ptr, <2 x i64> %x1, i8 %x2) {
+; CHECK-LABEL: test_int_x86_avx512_mask_pmov_qd_mem_128:
+; CHECK:  vpmovqd %xmm0, (%rdi)
+; CHECK:  vpmovqd %xmm0, (%rdi) {%k1}
+    call void @llvm.x86.avx512.mask.pmov.qd.mem.128(i8* %ptr, <2 x i64> %x1, i8 -1)
+    call void @llvm.x86.avx512.mask.pmov.qd.mem.128(i8* %ptr, <2 x i64> %x1, i8 %x2)
+    ret void
+}
+
+declare <4 x i32> @llvm.x86.avx512.mask.pmovs.qd.128(<2 x i64>, <4 x i32>, i8)
+
+define <4 x i32>@test_int_x86_avx512_mask_pmovs_qd_128(<2 x i64> %x0, <4 x i32> %x1, i8 %x2) {
+; CHECK-LABEL: test_int_x86_avx512_mask_pmovs_qd_128:
+; CHECK:       vpmovsqd %xmm0, %xmm1 {%k1}
+; CHECK-NEXT:  vpmovsqd %xmm0, %xmm2 {%k1} {z}
+; CHECK-NEXT:  vpmovsqd %xmm0, %xmm0
+    %res0 = call <4 x i32> @llvm.x86.avx512.mask.pmovs.qd.128(<2 x i64> %x0, <4 x i32> %x1, i8 -1)
+    %res1 = call <4 x i32> @llvm.x86.avx512.mask.pmovs.qd.128(<2 x i64> %x0, <4 x i32> %x1, i8 %x2)
+    %res2 = call <4 x i32> @llvm.x86.avx512.mask.pmovs.qd.128(<2 x i64> %x0, <4 x i32> zeroinitializer, i8 %x2)
+    %res3 = add <4 x i32> %res0, %res1
+    %res4 = add <4 x i32> %res3, %res2
+    ret <4 x i32> %res4
+}
+
+declare void @llvm.x86.avx512.mask.pmovs.qd.mem.128(i8* %ptr, <2 x i64>, i8)
+
+define void @test_int_x86_avx512_mask_pmovs_qd_mem_128(i8* %ptr, <2 x i64> %x1, i8 %x2) {
+; CHECK-LABEL: test_int_x86_avx512_mask_pmovs_qd_mem_128:
+; CHECK:  vpmovsqd %xmm0, (%rdi)
+; CHECK:  vpmovsqd %xmm0, (%rdi) {%k1}
+    call void @llvm.x86.avx512.mask.pmovs.qd.mem.128(i8* %ptr, <2 x i64> %x1, i8 -1)
+    call void @llvm.x86.avx512.mask.pmovs.qd.mem.128(i8* %ptr, <2 x i64> %x1, i8 %x2)
+    ret void
+}
+
+declare <4 x i32> @llvm.x86.avx512.mask.pmovus.qd.128(<2 x i64>, <4 x i32>, i8)
+
+define <4 x i32>@test_int_x86_avx512_mask_pmovus_qd_128(<2 x i64> %x0, <4 x i32> %x1, i8 %x2) {
+; CHECK-LABEL: test_int_x86_avx512_mask_pmovus_qd_128:
+; CHECK:       vpmovusqd %xmm0, %xmm1 {%k1}
+; CHECK-NEXT:  vpmovusqd %xmm0, %xmm2 {%k1} {z}
+; CHECK-NEXT:  vpmovusqd %xmm0, %xmm0
+    %res0 = call <4 x i32> @llvm.x86.avx512.mask.pmovus.qd.128(<2 x i64> %x0, <4 x i32> %x1, i8 -1)
+    %res1 = call <4 x i32> @llvm.x86.avx512.mask.pmovus.qd.128(<2 x i64> %x0, <4 x i32> %x1, i8 %x2)
+    %res2 = call <4 x i32> @llvm.x86.avx512.mask.pmovus.qd.128(<2 x i64> %x0, <4 x i32> zeroinitializer, i8 %x2)
+    %res3 = add <4 x i32> %res0, %res1
+    %res4 = add <4 x i32> %res3, %res2
+    ret <4 x i32> %res4
+}
+
+declare void @llvm.x86.avx512.mask.pmovus.qd.mem.128(i8* %ptr, <2 x i64>, i8)
+
+define void @test_int_x86_avx512_mask_pmovus_qd_mem_128(i8* %ptr, <2 x i64> %x1, i8 %x2) {
+; CHECK-LABEL: test_int_x86_avx512_mask_pmovus_qd_mem_128:
+; CHECK:  vpmovusqd %xmm0, (%rdi)
+; CHECK:  vpmovusqd %xmm0, (%rdi) {%k1}
+    call void @llvm.x86.avx512.mask.pmovus.qd.mem.128(i8* %ptr, <2 x i64> %x1, i8 -1)
+    call void @llvm.x86.avx512.mask.pmovus.qd.mem.128(i8* %ptr, <2 x i64> %x1, i8 %x2)
+    ret void
+}
+
+declare <4 x i32> @llvm.x86.avx512.mask.pmov.qd.256(<4 x i64>, <4 x i32>, i8)
+
+define <4 x i32>@test_int_x86_avx512_mask_pmov_qd_256(<4 x i64> %x0, <4 x i32> %x1, i8 %x2) {
+; CHECK-LABEL: test_int_x86_avx512_mask_pmov_qd_256:
+; CHECK:       vpmovqd %ymm0, %xmm1 {%k1}
+; CHECK-NEXT:  vpmovqd %ymm0, %xmm2 {%k1} {z}
+; CHECK-NEXT:  vpmovqd %ymm0, %xmm0
+    %res0 = call <4 x i32> @llvm.x86.avx512.mask.pmov.qd.256(<4 x i64> %x0, <4 x i32> %x1, i8 -1)
+    %res1 = call <4 x i32> @llvm.x86.avx512.mask.pmov.qd.256(<4 x i64> %x0, <4 x i32> %x1, i8 %x2)
+    %res2 = call <4 x i32> @llvm.x86.avx512.mask.pmov.qd.256(<4 x i64> %x0, <4 x i32> zeroinitializer, i8 %x2)
+    %res3 = add <4 x i32> %res0, %res1
+    %res4 = add <4 x i32> %res3, %res2
+    ret <4 x i32> %res4
+}
+
+declare void @llvm.x86.avx512.mask.pmov.qd.mem.256(i8* %ptr, <4 x i64>, i8)
+
+define void @test_int_x86_avx512_mask_pmov_qd_mem_256(i8* %ptr, <4 x i64> %x1, i8 %x2) {
+; CHECK-LABEL: test_int_x86_avx512_mask_pmov_qd_mem_256:
+; CHECK:  vpmovqd %ymm0, (%rdi)
+; CHECK:  vpmovqd %ymm0, (%rdi) {%k1}
+    call void @llvm.x86.avx512.mask.pmov.qd.mem.256(i8* %ptr, <4 x i64> %x1, i8 -1)
+    call void @llvm.x86.avx512.mask.pmov.qd.mem.256(i8* %ptr, <4 x i64> %x1, i8 %x2)
+    ret void
+}
+
+declare <4 x i32> @llvm.x86.avx512.mask.pmovs.qd.256(<4 x i64>, <4 x i32>, i8)
+
+define <4 x i32>@test_int_x86_avx512_mask_pmovs_qd_256(<4 x i64> %x0, <4 x i32> %x1, i8 %x2) {
+; CHECK-LABEL: test_int_x86_avx512_mask_pmovs_qd_256:
+; CHECK:       vpmovsqd %ymm0, %xmm1 {%k1}
+; CHECK-NEXT:  vpmovsqd %ymm0, %xmm2 {%k1} {z}
+; CHECK-NEXT:  vpmovsqd %ymm0, %xmm0
+    %res0 = call <4 x i32> @llvm.x86.avx512.mask.pmovs.qd.256(<4 x i64> %x0, <4 x i32> %x1, i8 -1)
+    %res1 = call <4 x i32> @llvm.x86.avx512.mask.pmovs.qd.256(<4 x i64> %x0, <4 x i32> %x1, i8 %x2)
+    %res2 = call <4 x i32> @llvm.x86.avx512.mask.pmovs.qd.256(<4 x i64> %x0, <4 x i32> zeroinitializer, i8 %x2)
+    %res3 = add <4 x i32> %res0, %res1
+    %res4 = add <4 x i32> %res3, %res2
+    ret <4 x i32> %res4
+}
+
+declare void @llvm.x86.avx512.mask.pmovs.qd.mem.256(i8* %ptr, <4 x i64>, i8)
+
+define void @test_int_x86_avx512_mask_pmovs_qd_mem_256(i8* %ptr, <4 x i64> %x1, i8 %x2) {
+; CHECK-LABEL: test_int_x86_avx512_mask_pmovs_qd_mem_256:
+; CHECK:  vpmovsqd %ymm0, (%rdi)
+; CHECK:  vpmovsqd %ymm0, (%rdi) {%k1}
+    call void @llvm.x86.avx512.mask.pmovs.qd.mem.256(i8* %ptr, <4 x i64> %x1, i8 -1)
+    call void @llvm.x86.avx512.mask.pmovs.qd.mem.256(i8* %ptr, <4 x i64> %x1, i8 %x2)
+    ret void
+}
+
+declare <4 x i32> @llvm.x86.avx512.mask.pmovus.qd.256(<4 x i64>, <4 x i32>, i8)
+
+define <4 x i32>@test_int_x86_avx512_mask_pmovus_qd_256(<4 x i64> %x0, <4 x i32> %x1, i8 %x2) {
+; CHECK-LABEL: test_int_x86_avx512_mask_pmovus_qd_256:
+; CHECK:       vpmovusqd %ymm0, %xmm1 {%k1}
+; CHECK-NEXT:  vpmovusqd %ymm0, %xmm2 {%k1} {z}
+; CHECK-NEXT:  vpmovusqd %ymm0, %xmm0
+    %res0 = call <4 x i32> @llvm.x86.avx512.mask.pmovus.qd.256(<4 x i64> %x0, <4 x i32> %x1, i8 -1)
+    %res1 = call <4 x i32> @llvm.x86.avx512.mask.pmovus.qd.256(<4 x i64> %x0, <4 x i32> %x1, i8 %x2)
+    %res2 = call <4 x i32> @llvm.x86.avx512.mask.pmovus.qd.256(<4 x i64> %x0, <4 x i32> zeroinitializer, i8 %x2)
+    %res3 = add <4 x i32> %res0, %res1
+    %res4 = add <4 x i32> %res3, %res2
+    ret <4 x i32> %res4
+}
+
+declare void @llvm.x86.avx512.mask.pmovus.qd.mem.256(i8* %ptr, <4 x i64>, i8)
+
+define void @test_int_x86_avx512_mask_pmovus_qd_mem_256(i8* %ptr, <4 x i64> %x1, i8 %x2) {
+; CHECK-LABEL: test_int_x86_avx512_mask_pmovus_qd_mem_256:
+; CHECK:  vpmovusqd %ymm0, (%rdi)
+; CHECK:  vpmovusqd %ymm0, (%rdi) {%k1}
+    call void @llvm.x86.avx512.mask.pmovus.qd.mem.256(i8* %ptr, <4 x i64> %x1, i8 -1)
+    call void @llvm.x86.avx512.mask.pmovus.qd.mem.256(i8* %ptr, <4 x i64> %x1, i8 %x2)
+    ret void
+}
+
+declare <16 x i8> @llvm.x86.avx512.mask.pmov.db.128(<4 x i32>, <16 x i8>, i8)
+
+define <16 x i8>@test_int_x86_avx512_mask_pmov_db_128(<4 x i32> %x0, <16 x i8> %x1, i8 %x2) {
+; CHECK-LABEL: test_int_x86_avx512_mask_pmov_db_128:
+; CHECK:       vpmovdb %xmm0, %xmm1 {%k1}
+; CHECK-NEXT:  vpmovdb %xmm0, %xmm2 {%k1} {z}
+; CHECK-NEXT:  vpmovdb %xmm0, %xmm0
+    %res0 = call <16 x i8> @llvm.x86.avx512.mask.pmov.db.128(<4 x i32> %x0, <16 x i8> %x1, i8 -1)
+    %res1 = call <16 x i8> @llvm.x86.avx512.mask.pmov.db.128(<4 x i32> %x0, <16 x i8> %x1, i8 %x2)
+    %res2 = call <16 x i8> @llvm.x86.avx512.mask.pmov.db.128(<4 x i32> %x0, <16 x i8> zeroinitializer, i8 %x2)
+    %res3 = add <16 x i8> %res0, %res1
+    %res4 = add <16 x i8> %res3, %res2
+    ret <16 x i8> %res4
+}
+
+declare void @llvm.x86.avx512.mask.pmov.db.mem.128(i8* %ptr, <4 x i32>, i8)
+
+define void @test_int_x86_avx512_mask_pmov_db_mem_128(i8* %ptr, <4 x i32> %x1, i8 %x2) {
+; CHECK-LABEL: test_int_x86_avx512_mask_pmov_db_mem_128:
+; CHECK:  vpmovdb %xmm0, (%rdi)
+; CHECK:  vpmovdb %xmm0, (%rdi) {%k1}
+    call void @llvm.x86.avx512.mask.pmov.db.mem.128(i8* %ptr, <4 x i32> %x1, i8 -1)
+    call void @llvm.x86.avx512.mask.pmov.db.mem.128(i8* %ptr, <4 x i32> %x1, i8 %x2)
+    ret void
+}
+
+declare <16 x i8> @llvm.x86.avx512.mask.pmovs.db.128(<4 x i32>, <16 x i8>, i8)
+
+define <16 x i8>@test_int_x86_avx512_mask_pmovs_db_128(<4 x i32> %x0, <16 x i8> %x1, i8 %x2) {
+; CHECK-LABEL: test_int_x86_avx512_mask_pmovs_db_128:
+; CHECK:       vpmovsdb %xmm0, %xmm1 {%k1}
+; CHECK-NEXT:  vpmovsdb %xmm0, %xmm2 {%k1} {z}
+; CHECK-NEXT:  vpmovsdb %xmm0, %xmm0
+    %res0 = call <16 x i8> @llvm.x86.avx512.mask.pmovs.db.128(<4 x i32> %x0, <16 x i8> %x1, i8 -1)
+    %res1 = call <16 x i8> @llvm.x86.avx512.mask.pmovs.db.128(<4 x i32> %x0, <16 x i8> %x1, i8 %x2)
+    %res2 = call <16 x i8> @llvm.x86.avx512.mask.pmovs.db.128(<4 x i32> %x0, <16 x i8> zeroinitializer, i8 %x2)
+    %res3 = add <16 x i8> %res0, %res1
+    %res4 = add <16 x i8> %res3, %res2
+    ret <16 x i8> %res4
+}
+
+declare void @llvm.x86.avx512.mask.pmovs.db.mem.128(i8* %ptr, <4 x i32>, i8)
+
+define void @test_int_x86_avx512_mask_pmovs_db_mem_128(i8* %ptr, <4 x i32> %x1, i8 %x2) {
+; CHECK-LABEL: test_int_x86_avx512_mask_pmovs_db_mem_128:
+; CHECK:  vpmovsdb %xmm0, (%rdi)
+; CHECK:  vpmovsdb %xmm0, (%rdi) {%k1}
+    call void @llvm.x86.avx512.mask.pmovs.db.mem.128(i8* %ptr, <4 x i32> %x1, i8 -1)
+    call void @llvm.x86.avx512.mask.pmovs.db.mem.128(i8* %ptr, <4 x i32> %x1, i8 %x2)
+    ret void
+}
+
+declare <16 x i8> @llvm.x86.avx512.mask.pmovus.db.128(<4 x i32>, <16 x i8>, i8)
+
+define <16 x i8>@test_int_x86_avx512_mask_pmovus_db_128(<4 x i32> %x0, <16 x i8> %x1, i8 %x2) {
+; CHECK-LABEL: test_int_x86_avx512_mask_pmovus_db_128:
+; CHECK:       vpmovusdb %xmm0, %xmm1 {%k1}
+; CHECK-NEXT:  vpmovusdb %xmm0, %xmm2 {%k1} {z}
+; CHECK-NEXT:  vpmovusdb %xmm0, %xmm0
+    %res0 = call <16 x i8> @llvm.x86.avx512.mask.pmovus.db.128(<4 x i32> %x0, <16 x i8> %x1, i8 -1)
+    %res1 = call <16 x i8> @llvm.x86.avx512.mask.pmovus.db.128(<4 x i32> %x0, <16 x i8> %x1, i8 %x2)
+    %res2 = call <16 x i8> @llvm.x86.avx512.mask.pmovus.db.128(<4 x i32> %x0, <16 x i8> zeroinitializer, i8 %x2)
+    %res3 = add <16 x i8> %res0, %res1
+    %res4 = add <16 x i8> %res3, %res2
+    ret <16 x i8> %res4
+}
+
+declare void @llvm.x86.avx512.mask.pmovus.db.mem.128(i8* %ptr, <4 x i32>, i8)
+
+define void @test_int_x86_avx512_mask_pmovus_db_mem_128(i8* %ptr, <4 x i32> %x1, i8 %x2) {
+; CHECK-LABEL: test_int_x86_avx512_mask_pmovus_db_mem_128:
+; CHECK:  vpmovusdb %xmm0, (%rdi)
+; CHECK:  vpmovusdb %xmm0, (%rdi) {%k1}
+    call void @llvm.x86.avx512.mask.pmovus.db.mem.128(i8* %ptr, <4 x i32> %x1, i8 -1)
+    call void @llvm.x86.avx512.mask.pmovus.db.mem.128(i8* %ptr, <4 x i32> %x1, i8 %x2)
+    ret void
+}
+
+declare <16 x i8> @llvm.x86.avx512.mask.pmov.db.256(<8 x i32>, <16 x i8>, i8)
+
+define <16 x i8>@test_int_x86_avx512_mask_pmov_db_256(<8 x i32> %x0, <16 x i8> %x1, i8 %x2) {
+; CHECK-LABEL: test_int_x86_avx512_mask_pmov_db_256:
+; CHECK:       vpmovdb %ymm0, %xmm1 {%k1}
+; CHECK-NEXT:  vpmovdb %ymm0, %xmm2 {%k1} {z}
+; CHECK-NEXT:  vpmovdb %ymm0, %xmm0
+    %res0 = call <16 x i8> @llvm.x86.avx512.mask.pmov.db.256(<8 x i32> %x0, <16 x i8> %x1, i8 -1)
+    %res1 = call <16 x i8> @llvm.x86.avx512.mask.pmov.db.256(<8 x i32> %x0, <16 x i8> %x1, i8 %x2)
+    %res2 = call <16 x i8> @llvm.x86.avx512.mask.pmov.db.256(<8 x i32> %x0, <16 x i8> zeroinitializer, i8 %x2)
+    %res3 = add <16 x i8> %res0, %res1
+    %res4 = add <16 x i8> %res3, %res2
+    ret <16 x i8> %res4
+}
+
+declare void @llvm.x86.avx512.mask.pmov.db.mem.256(i8* %ptr, <8 x i32>, i8)
+
+define void @test_int_x86_avx512_mask_pmov_db_mem_256(i8* %ptr, <8 x i32> %x1, i8 %x2) {
+; CHECK-LABEL: test_int_x86_avx512_mask_pmov_db_mem_256:
+; CHECK:  vpmovdb %ymm0, (%rdi)
+; CHECK:  vpmovdb %ymm0, (%rdi) {%k1}
+    call void @llvm.x86.avx512.mask.pmov.db.mem.256(i8* %ptr, <8 x i32> %x1, i8 -1)
+    call void @llvm.x86.avx512.mask.pmov.db.mem.256(i8* %ptr, <8 x i32> %x1, i8 %x2)
+    ret void
+}
+
+declare <16 x i8> @llvm.x86.avx512.mask.pmovs.db.256(<8 x i32>, <16 x i8>, i8)
+
+define <16 x i8>@test_int_x86_avx512_mask_pmovs_db_256(<8 x i32> %x0, <16 x i8> %x1, i8 %x2) {
+; CHECK-LABEL: test_int_x86_avx512_mask_pmovs_db_256:
+; CHECK:       vpmovsdb %ymm0, %xmm1 {%k1}
+; CHECK-NEXT:  vpmovsdb %ymm0, %xmm2 {%k1} {z}
+; CHECK-NEXT:  vpmovsdb %ymm0, %xmm0
+    %res0 = call <16 x i8> @llvm.x86.avx512.mask.pmovs.db.256(<8 x i32> %x0, <16 x i8> %x1, i8 -1)
+    %res1 = call <16 x i8> @llvm.x86.avx512.mask.pmovs.db.256(<8 x i32> %x0, <16 x i8> %x1, i8 %x2)
+    %res2 = call <16 x i8> @llvm.x86.avx512.mask.pmovs.db.256(<8 x i32> %x0, <16 x i8> zeroinitializer, i8 %x2)
+    %res3 = add <16 x i8> %res0, %res1
+    %res4 = add <16 x i8> %res3, %res2
+    ret <16 x i8> %res4
+}
+
+declare void @llvm.x86.avx512.mask.pmovs.db.mem.256(i8* %ptr, <8 x i32>, i8)
+
+define void @test_int_x86_avx512_mask_pmovs_db_mem_256(i8* %ptr, <8 x i32> %x1, i8 %x2) {
+; CHECK-LABEL: test_int_x86_avx512_mask_pmovs_db_mem_256:
+; CHECK:  vpmovsdb %ymm0, (%rdi)
+; CHECK:  vpmovsdb %ymm0, (%rdi) {%k1}
+    call void @llvm.x86.avx512.mask.pmovs.db.mem.256(i8* %ptr, <8 x i32> %x1, i8 -1)
+    call void @llvm.x86.avx512.mask.pmovs.db.mem.256(i8* %ptr, <8 x i32> %x1, i8 %x2)
+    ret void
+}
+
+declare <16 x i8> @llvm.x86.avx512.mask.pmovus.db.256(<8 x i32>, <16 x i8>, i8)
+
+define <16 x i8>@test_int_x86_avx512_mask_pmovus_db_256(<8 x i32> %x0, <16 x i8> %x1, i8 %x2) {
+; CHECK-LABEL: test_int_x86_avx512_mask_pmovus_db_256:
+; CHECK:       vpmovusdb %ymm0, %xmm1 {%k1}
+; CHECK-NEXT:  vpmovusdb %ymm0, %xmm2 {%k1} {z}
+; CHECK-NEXT:  vpmovusdb %ymm0, %xmm0
+    %res0 = call <16 x i8> @llvm.x86.avx512.mask.pmovus.db.256(<8 x i32> %x0, <16 x i8> %x1, i8 -1)
+    %res1 = call <16 x i8> @llvm.x86.avx512.mask.pmovus.db.256(<8 x i32> %x0, <16 x i8> %x1, i8 %x2)
+    %res2 = call <16 x i8> @llvm.x86.avx512.mask.pmovus.db.256(<8 x i32> %x0, <16 x i8> zeroinitializer, i8 %x2)
+    %res3 = add <16 x i8> %res0, %res1
+    %res4 = add <16 x i8> %res3, %res2
+    ret <16 x i8> %res4
+}
+
+declare void @llvm.x86.avx512.mask.pmovus.db.mem.256(i8* %ptr, <8 x i32>, i8)
+
+define void @test_int_x86_avx512_mask_pmovus_db_mem_256(i8* %ptr, <8 x i32> %x1, i8 %x2) {
+; CHECK-LABEL: test_int_x86_avx512_mask_pmovus_db_mem_256:
+; CHECK:  vpmovusdb %ymm0, (%rdi)
+; CHECK:  vpmovusdb %ymm0, (%rdi) {%k1}
+    call void @llvm.x86.avx512.mask.pmovus.db.mem.256(i8* %ptr, <8 x i32> %x1, i8 -1)
+    call void @llvm.x86.avx512.mask.pmovus.db.mem.256(i8* %ptr, <8 x i32> %x1, i8 %x2)
+    ret void
+}
+
+declare <8 x i16> @llvm.x86.avx512.mask.pmov.dw.128(<4 x i32>, <8 x i16>, i8)
+
+define <8 x i16>@test_int_x86_avx512_mask_pmov_dw_128(<4 x i32> %x0, <8 x i16> %x1, i8 %x2) {
+; CHECK-LABEL: test_int_x86_avx512_mask_pmov_dw_128:
+; CHECK:       vpmovdw %xmm0, %xmm1 {%k1}
+; CHECK-NEXT:  vpmovdw %xmm0, %xmm2 {%k1} {z}
+; CHECK-NEXT:  vpmovdw %xmm0, %xmm0
+    %res0 = call <8 x i16> @llvm.x86.avx512.mask.pmov.dw.128(<4 x i32> %x0, <8 x i16> %x1, i8 -1)
+    %res1 = call <8 x i16> @llvm.x86.avx512.mask.pmov.dw.128(<4 x i32> %x0, <8 x i16> %x1, i8 %x2)
+    %res2 = call <8 x i16> @llvm.x86.avx512.mask.pmov.dw.128(<4 x i32> %x0, <8 x i16> zeroinitializer, i8 %x2)
+    %res3 = add <8 x i16> %res0, %res1
+    %res4 = add <8 x i16> %res3, %res2
+    ret <8 x i16> %res4
+}
+
+declare void @llvm.x86.avx512.mask.pmov.dw.mem.128(i8* %ptr, <4 x i32>, i8)
+
+define void @test_int_x86_avx512_mask_pmov_dw_mem_128(i8* %ptr, <4 x i32> %x1, i8 %x2) {
+; CHECK-LABEL: test_int_x86_avx512_mask_pmov_dw_mem_128:
+; CHECK:  vpmovdw %xmm0, (%rdi)
+; CHECK:  vpmovdw %xmm0, (%rdi) {%k1}
+    call void @llvm.x86.avx512.mask.pmov.dw.mem.128(i8* %ptr, <4 x i32> %x1, i8 -1)
+    call void @llvm.x86.avx512.mask.pmov.dw.mem.128(i8* %ptr, <4 x i32> %x1, i8 %x2)
+    ret void
+}
+
+declare <8 x i16> @llvm.x86.avx512.mask.pmovs.dw.128(<4 x i32>, <8 x i16>, i8)
+
+define <8 x i16>@test_int_x86_avx512_mask_pmovs_dw_128(<4 x i32> %x0, <8 x i16> %x1, i8 %x2) {
+; CHECK-LABEL: test_int_x86_avx512_mask_pmovs_dw_128:
+; CHECK:       vpmovsdw %xmm0, %xmm1 {%k1}
+; CHECK-NEXT:  vpmovsdw %xmm0, %xmm2 {%k1} {z}
+; CHECK-NEXT:  vpmovsdw %xmm0, %xmm0
+    %res0 = call <8 x i16> @llvm.x86.avx512.mask.pmovs.dw.128(<4 x i32> %x0, <8 x i16> %x1, i8 -1)
+    %res1 = call <8 x i16> @llvm.x86.avx512.mask.pmovs.dw.128(<4 x i32> %x0, <8 x i16> %x1, i8 %x2)
+    %res2 = call <8 x i16> @llvm.x86.avx512.mask.pmovs.dw.128(<4 x i32> %x0, <8 x i16> zeroinitializer, i8 %x2)
+    %res3 = add <8 x i16> %res0, %res1
+    %res4 = add <8 x i16> %res3, %res2
+    ret <8 x i16> %res4
+}
+
+declare void @llvm.x86.avx512.mask.pmovs.dw.mem.128(i8* %ptr, <4 x i32>, i8)
+
+define void @test_int_x86_avx512_mask_pmovs_dw_mem_128(i8* %ptr, <4 x i32> %x1, i8 %x2) {
+; CHECK-LABEL: test_int_x86_avx512_mask_pmovs_dw_mem_128:
+; CHECK:  vpmovsdw %xmm0, (%rdi)
+; CHECK:  vpmovsdw %xmm0, (%rdi) {%k1}
+    call void @llvm.x86.avx512.mask.pmovs.dw.mem.128(i8* %ptr, <4 x i32> %x1, i8 -1)
+    call void @llvm.x86.avx512.mask.pmovs.dw.mem.128(i8* %ptr, <4 x i32> %x1, i8 %x2)
+    ret void
+}
+
+declare <8 x i16> @llvm.x86.avx512.mask.pmovus.dw.128(<4 x i32>, <8 x i16>, i8)
+
+define <8 x i16>@test_int_x86_avx512_mask_pmovus_dw_128(<4 x i32> %x0, <8 x i16> %x1, i8 %x2) {
+; CHECK-LABEL: test_int_x86_avx512_mask_pmovus_dw_128:
+; CHECK:       vpmovusdw %xmm0, %xmm1 {%k1}
+; CHECK-NEXT:  vpmovusdw %xmm0, %xmm2 {%k1} {z}
+; CHECK-NEXT:  vpmovusdw %xmm0, %xmm0
+    %res0 = call <8 x i16> @llvm.x86.avx512.mask.pmovus.dw.128(<4 x i32> %x0, <8 x i16> %x1, i8 -1)
+    %res1 = call <8 x i16> @llvm.x86.avx512.mask.pmovus.dw.128(<4 x i32> %x0, <8 x i16> %x1, i8 %x2)
+    %res2 = call <8 x i16> @llvm.x86.avx512.mask.pmovus.dw.128(<4 x i32> %x0, <8 x i16> zeroinitializer, i8 %x2)
+    %res3 = add <8 x i16> %res0, %res1
+    %res4 = add <8 x i16> %res3, %res2
+    ret <8 x i16> %res4
+}
+
+declare void @llvm.x86.avx512.mask.pmovus.dw.mem.128(i8* %ptr, <4 x i32>, i8)
+
+define void @test_int_x86_avx512_mask_pmovus_dw_mem_128(i8* %ptr, <4 x i32> %x1, i8 %x2) {
+; CHECK-LABEL: test_int_x86_avx512_mask_pmovus_dw_mem_128:
+; CHECK:  vpmovusdw %xmm0, (%rdi)
+; CHECK:  vpmovusdw %xmm0, (%rdi) {%k1}
+    call void @llvm.x86.avx512.mask.pmovus.dw.mem.128(i8* %ptr, <4 x i32> %x1, i8 -1)
+    call void @llvm.x86.avx512.mask.pmovus.dw.mem.128(i8* %ptr, <4 x i32> %x1, i8 %x2)
+    ret void
+}
+
+declare <8 x i16> @llvm.x86.avx512.mask.pmov.dw.256(<8 x i32>, <8 x i16>, i8)
+
+define <8 x i16>@test_int_x86_avx512_mask_pmov_dw_256(<8 x i32> %x0, <8 x i16> %x1, i8 %x2) {
+; CHECK-LABEL: test_int_x86_avx512_mask_pmov_dw_256:
+; CHECK:       vpmovdw %ymm0, %xmm1 {%k1}
+; CHECK-NEXT:  vpmovdw %ymm0, %xmm2 {%k1} {z}
+; CHECK-NEXT:  vpmovdw %ymm0, %xmm0
+    %res0 = call <8 x i16> @llvm.x86.avx512.mask.pmov.dw.256(<8 x i32> %x0, <8 x i16> %x1, i8 -1)
+    %res1 = call <8 x i16> @llvm.x86.avx512.mask.pmov.dw.256(<8 x i32> %x0, <8 x i16> %x1, i8 %x2)
+    %res2 = call <8 x i16> @llvm.x86.avx512.mask.pmov.dw.256(<8 x i32> %x0, <8 x i16> zeroinitializer, i8 %x2)
+    %res3 = add <8 x i16> %res0, %res1
+    %res4 = add <8 x i16> %res3, %res2
+    ret <8 x i16> %res4
+}
+
+declare void @llvm.x86.avx512.mask.pmov.dw.mem.256(i8* %ptr, <8 x i32>, i8)
+
+define void @test_int_x86_avx512_mask_pmov_dw_mem_256(i8* %ptr, <8 x i32> %x1, i8 %x2) {
+; CHECK-LABEL: test_int_x86_avx512_mask_pmov_dw_mem_256:
+; CHECK:  vpmovdw %ymm0, (%rdi)
+; CHECK:  vpmovdw %ymm0, (%rdi) {%k1}
+    call void @llvm.x86.avx512.mask.pmov.dw.mem.256(i8* %ptr, <8 x i32> %x1, i8 -1)
+    call void @llvm.x86.avx512.mask.pmov.dw.mem.256(i8* %ptr, <8 x i32> %x1, i8 %x2)
+    ret void
+}
+
+declare <8 x i16> @llvm.x86.avx512.mask.pmovs.dw.256(<8 x i32>, <8 x i16>, i8)
+
+define <8 x i16>@test_int_x86_avx512_mask_pmovs_dw_256(<8 x i32> %x0, <8 x i16> %x1, i8 %x2) {
+; CHECK-LABEL: test_int_x86_avx512_mask_pmovs_dw_256:
+; CHECK:       vpmovsdw %ymm0, %xmm1 {%k1}
+; CHECK-NEXT:  vpmovsdw %ymm0, %xmm2 {%k1} {z}
+; CHECK-NEXT:  vpmovsdw %ymm0, %xmm0
+    %res0 = call <8 x i16> @llvm.x86.avx512.mask.pmovs.dw.256(<8 x i32> %x0, <8 x i16> %x1, i8 -1)
+    %res1 = call <8 x i16> @llvm.x86.avx512.mask.pmovs.dw.256(<8 x i32> %x0, <8 x i16> %x1, i8 %x2)
+    %res2 = call <8 x i16> @llvm.x86.avx512.mask.pmovs.dw.256(<8 x i32> %x0, <8 x i16> zeroinitializer, i8 %x2)
+    %res3 = add <8 x i16> %res0, %res1
+    %res4 = add <8 x i16> %res3, %res2
+    ret <8 x i16> %res4
+}
+
+declare void @llvm.x86.avx512.mask.pmovs.dw.mem.256(i8* %ptr, <8 x i32>, i8)
+
+define void @test_int_x86_avx512_mask_pmovs_dw_mem_256(i8* %ptr, <8 x i32> %x1, i8 %x2) {
+; CHECK-LABEL: test_int_x86_avx512_mask_pmovs_dw_mem_256:
+; CHECK:  vpmovsdw %ymm0, (%rdi)
+; CHECK:  vpmovsdw %ymm0, (%rdi) {%k1}
+    call void @llvm.x86.avx512.mask.pmovs.dw.mem.256(i8* %ptr, <8 x i32> %x1, i8 -1)
+    call void @llvm.x86.avx512.mask.pmovs.dw.mem.256(i8* %ptr, <8 x i32> %x1, i8 %x2)
+    ret void
+}
+
+declare <8 x i16> @llvm.x86.avx512.mask.pmovus.dw.256(<8 x i32>, <8 x i16>, i8)
+
+define <8 x i16>@test_int_x86_avx512_mask_pmovus_dw_256(<8 x i32> %x0, <8 x i16> %x1, i8 %x2) {
+; CHECK-LABEL: test_int_x86_avx512_mask_pmovus_dw_256:
+; CHECK:       vpmovusdw %ymm0, %xmm1 {%k1}
+; CHECK-NEXT:  vpmovusdw %ymm0, %xmm2 {%k1} {z}
+; CHECK-NEXT:  vpmovusdw %ymm0, %xmm0
+    %res0 = call <8 x i16> @llvm.x86.avx512.mask.pmovus.dw.256(<8 x i32> %x0, <8 x i16> %x1, i8 -1)
+    %res1 = call <8 x i16> @llvm.x86.avx512.mask.pmovus.dw.256(<8 x i32> %x0, <8 x i16> %x1, i8 %x2)
+    %res2 = call <8 x i16> @llvm.x86.avx512.mask.pmovus.dw.256(<8 x i32> %x0, <8 x i16> zeroinitializer, i8 %x2)
+    %res3 = add <8 x i16> %res0, %res1
+    %res4 = add <8 x i16> %res3, %res2
+    ret <8 x i16> %res4
+}
+
+declare void @llvm.x86.avx512.mask.pmovus.dw.mem.256(i8* %ptr, <8 x i32>, i8)
+
+define void @test_int_x86_avx512_mask_pmovus_dw_mem_256(i8* %ptr, <8 x i32> %x1, i8 %x2) {
+; CHECK-LABEL: test_int_x86_avx512_mask_pmovus_dw_mem_256:
+; CHECK:  vpmovusdw %ymm0, (%rdi)
+; CHECK:  vpmovusdw %ymm0, (%rdi) {%k1}
+    call void @llvm.x86.avx512.mask.pmovus.dw.mem.256(i8* %ptr, <8 x i32> %x1, i8 -1)
+    call void @llvm.x86.avx512.mask.pmovus.dw.mem.256(i8* %ptr, <8 x i32> %x1, i8 %x2)
+    ret void
+}
+
+declare <2 x double> @llvm.x86.avx512.mask.cvtdq2pd.128(<4 x i32>, <2 x double>, i8)
+
+define <2 x double>@test_int_x86_avx512_mask_cvt_dq2pd_128(<4 x i32> %x0, <2 x double> %x1, i8 %x2) {
+; CHECK-LABEL: test_int_x86_avx512_mask_cvt_dq2pd_128:
+; CHECK:       ## BB#0:
+; CHECK-NEXT:    movzbl %dil, %eax
+; CHECK-NEXT:    kmovw %eax, %k1
+; CHECK-NEXT:    vcvtdq2pd %xmm0, %xmm1 {%k1}
+; CHECK-NEXT:    vcvtdq2pd %xmm0, %xmm0
+; CHECK-NEXT:    vaddpd %xmm0, %xmm1, %xmm0
+; CHECK-NEXT:    retq
+  %res = call <2 x double> @llvm.x86.avx512.mask.cvtdq2pd.128(<4 x i32> %x0, <2 x double> %x1, i8 %x2)
+  %res1 = call <2 x double> @llvm.x86.avx512.mask.cvtdq2pd.128(<4 x i32> %x0, <2 x double> %x1, i8 -1)
+  %res2 = fadd <2 x double> %res, %res1
+  ret <2 x double> %res2
+}
+
+declare <4 x double> @llvm.x86.avx512.mask.cvtdq2pd.256(<4 x i32>, <4 x double>, i8)
+
+define <4 x double>@test_int_x86_avx512_mask_cvt_dq2pd_256(<4 x i32> %x0, <4 x double> %x1, i8 %x2) {
+; CHECK-LABEL: test_int_x86_avx512_mask_cvt_dq2pd_256:
+; CHECK:       ## BB#0:
+; CHECK-NEXT:    movzbl %dil, %eax
+; CHECK-NEXT:    kmovw %eax, %k1
+; CHECK-NEXT:    vcvtdq2pd %xmm0, %ymm1 {%k1}
+; CHECK-NEXT:    vcvtdq2pd %xmm0, %ymm0
+; CHECK-NEXT:    vaddpd %ymm0, %ymm1, %ymm0
+; CHECK-NEXT:    retq
+  %res = call <4 x double> @llvm.x86.avx512.mask.cvtdq2pd.256(<4 x i32> %x0, <4 x double> %x1, i8 %x2)
+  %res1 = call <4 x double> @llvm.x86.avx512.mask.cvtdq2pd.256(<4 x i32> %x0, <4 x double> %x1, i8 -1)
+  %res2 = fadd <4 x double> %res, %res1
+  ret <4 x double> %res2
+}
+
+declare <4 x float> @llvm.x86.avx512.mask.cvtdq2ps.128(<4 x i32>, <4 x float>, i8)
+
+define <4 x float>@test_int_x86_avx512_mask_cvt_dq2ps_128(<4 x i32> %x0, <4 x float> %x1, i8 %x2) {
+; CHECK-LABEL: test_int_x86_avx512_mask_cvt_dq2ps_128:
+; CHECK:       ## BB#0:
+; CHECK-NEXT:    movzbl %dil, %eax
+; CHECK-NEXT:    kmovw %eax, %k1
+; CHECK-NEXT:    vcvtdq2ps %xmm0, %xmm1 {%k1}
+; CHECK-NEXT:    vcvtdq2ps %xmm0, %xmm0
+; CHECK-NEXT:    vaddps %xmm0, %xmm1, %xmm0
+; CHECK-NEXT:    retq
+  %res = call <4 x float> @llvm.x86.avx512.mask.cvtdq2ps.128(<4 x i32> %x0, <4 x float> %x1, i8 %x2)
+  %res1 = call <4 x float> @llvm.x86.avx512.mask.cvtdq2ps.128(<4 x i32> %x0, <4 x float> %x1, i8 -1)
+  %res2 = fadd <4 x float> %res, %res1
+  ret <4 x float> %res2
+}
+
+declare <8 x float> @llvm.x86.avx512.mask.cvtdq2ps.256(<8 x i32>, <8 x float>, i8)
+
+define <8 x float>@test_int_x86_avx512_mask_cvt_dq2ps_256(<8 x i32> %x0, <8 x float> %x1, i8 %x2) {
+; CHECK-LABEL: test_int_x86_avx512_mask_cvt_dq2ps_256:
+; CHECK:       ## BB#0:
+; CHECK-NEXT:    movzbl %dil, %eax
+; CHECK-NEXT:    kmovw %eax, %k1
+; CHECK-NEXT:    vcvtdq2ps %ymm0, %ymm1 {%k1}
+; CHECK-NEXT:    vcvtdq2ps %ymm0, %ymm0
+; CHECK-NEXT:    vaddps %ymm0, %ymm1, %ymm0
+; CHECK-NEXT:    retq
+  %res = call <8 x float> @llvm.x86.avx512.mask.cvtdq2ps.256(<8 x i32> %x0, <8 x float> %x1, i8 %x2)
+  %res1 = call <8 x float> @llvm.x86.avx512.mask.cvtdq2ps.256(<8 x i32> %x0, <8 x float> %x1, i8 -1)
+  %res2 = fadd <8 x float> %res, %res1
+  ret <8 x float> %res2
+}
+
+declare <4 x i32> @llvm.x86.avx512.mask.cvtpd2dq.128(<2 x double>, <4 x i32>, i8)
+
+define <4 x i32>@test_int_x86_avx512_mask_cvt_pd2dq_128(<2 x double> %x0, <4 x i32> %x1, i8 %x2) {
+; CHECK-LABEL: test_int_x86_avx512_mask_cvt_pd2dq_128:
+; CHECK:       ## BB#0:
+; CHECK-NEXT:    movzbl %dil, %eax
+; CHECK-NEXT:    kmovw %eax, %k1
+; CHECK-NEXT:    vcvtpd2dq %xmm0, %xmm1 {%k1}
+; CHECK-NEXT:    vcvtpd2dq %xmm0, %xmm0
+; CHECK-NEXT:    vpaddd %xmm0, %xmm1, %xmm0
+; CHECK-NEXT:    retq
+  %res = call <4 x i32> @llvm.x86.avx512.mask.cvtpd2dq.128(<2 x double> %x0, <4 x i32> %x1, i8 %x2)
+  %res1 = call <4 x i32> @llvm.x86.avx512.mask.cvtpd2dq.128(<2 x double> %x0, <4 x i32> %x1, i8 -1)
+  %res2 = add <4 x i32> %res, %res1
+  ret <4 x i32> %res2
+}
+
+declare <4 x i32> @llvm.x86.avx512.mask.cvtpd2dq.256(<4 x double>, <4 x i32>, i8)
+
+define <4 x i32>@test_int_x86_avx512_mask_cvt_pd2dq_256(<4 x double> %x0, <4 x i32> %x1, i8 %x2) {
+; CHECK-LABEL: test_int_x86_avx512_mask_cvt_pd2dq_256:
+; CHECK:       ## BB#0:
+; CHECK-NEXT:    movzbl %dil, %eax
+; CHECK-NEXT:    kmovw %eax, %k1
+; CHECK-NEXT:    vcvtpd2dq %ymm0, %xmm1 {%k1}
+; CHECK-NEXT:    vcvtpd2dq %ymm0, %xmm0
+; CHECK-NEXT:    vpaddd %xmm0, %xmm1, %xmm0
+; CHECK-NEXT:    retq
+  %res = call <4 x i32> @llvm.x86.avx512.mask.cvtpd2dq.256(<4 x double> %x0, <4 x i32> %x1, i8 %x2)
+  %res1 = call <4 x i32> @llvm.x86.avx512.mask.cvtpd2dq.256(<4 x double> %x0, <4 x i32> %x1, i8 -1)
+  %res2 = add <4 x i32> %res, %res1
+  ret <4 x i32> %res2
+}
+
+declare <4 x float> @llvm.x86.avx512.mask.cvtpd2ps.256(<4 x double>, <4 x float>, i8)
+
+define <4 x float>@test_int_x86_avx512_mask_cvt_pd2ps_256(<4 x double> %x0, <4 x float> %x1, i8 %x2) {
+; CHECK-LABEL: test_int_x86_avx512_mask_cvt_pd2ps_256:
+; CHECK:       ## BB#0:
+; CHECK-NEXT:    movzbl %dil, %eax
+; CHECK-NEXT:    kmovw %eax, %k1
+; CHECK-NEXT:    vcvtpd2ps %ymm0, %xmm1 {%k1}
+; CHECK-NEXT:    vcvtpd2ps %ymm0, %xmm0
+; CHECK-NEXT:    vaddps %xmm0, %xmm1, %xmm0
+; CHECK-NEXT:    retq
+  %res = call <4 x float> @llvm.x86.avx512.mask.cvtpd2ps.256(<4 x double> %x0, <4 x float> %x1, i8 %x2)
+  %res1 = call <4 x float> @llvm.x86.avx512.mask.cvtpd2ps.256(<4 x double> %x0, <4 x float> %x1, i8 -1)
+  %res2 = fadd <4 x float> %res, %res1
+  ret <4 x float> %res2
+}
+
+declare <4 x float> @llvm.x86.avx512.mask.cvtpd2ps(<2 x double>, <4 x float>, i8)
+
+define <4 x float>@test_int_x86_avx512_mask_cvt_pd2ps(<2 x double> %x0, <4 x float> %x1, i8 %x2) {
+; CHECK-LABEL: test_int_x86_avx512_mask_cvt_pd2ps:
+; CHECK:       ## BB#0:
+; CHECK-NEXT:    movzbl %dil, %eax
+; CHECK-NEXT:    kmovw %eax, %k1
+; CHECK-NEXT:    vcvtpd2ps %xmm0, %xmm1 {%k1}
+; CHECK-NEXT:    vcvtpd2ps %xmm0, %xmm0
+; CHECK-NEXT:    vaddps %xmm0, %xmm1, %xmm0
+; CHECK-NEXT:    retq
+  %res = call <4 x float> @llvm.x86.avx512.mask.cvtpd2ps(<2 x double> %x0, <4 x float> %x1, i8 %x2)
+  %res1 = call <4 x float> @llvm.x86.avx512.mask.cvtpd2ps(<2 x double> %x0, <4 x float> %x1, i8 -1)
+  %res2 = fadd <4 x float> %res, %res1
+  ret <4 x float> %res2
+}
+
+declare <4 x i32> @llvm.x86.avx512.mask.cvtpd2udq.128(<2 x double>, <4 x i32>, i8)
+
+define <4 x i32>@test_int_x86_avx512_mask_cvt_pd2udq_128(<2 x double> %x0, <4 x i32> %x1, i8 %x2) {
+; CHECK-LABEL: test_int_x86_avx512_mask_cvt_pd2udq_128:
+; CHECK:       ## BB#0:
+; CHECK-NEXT:    movzbl %dil, %eax
+; CHECK-NEXT:    kmovw %eax, %k1
+; CHECK-NEXT:    vcvtpd2udq %xmm0, %xmm1 {%k1}
+; CHECK-NEXT:    vcvtpd2udq %xmm0, %xmm0
+; CHECK-NEXT:    vpaddd %xmm0, %xmm1, %xmm0
+; CHECK-NEXT:    retq
+  %res = call <4 x i32> @llvm.x86.avx512.mask.cvtpd2udq.128(<2 x double> %x0, <4 x i32> %x1, i8 %x2)
+  %res1 = call <4 x i32> @llvm.x86.avx512.mask.cvtpd2udq.128(<2 x double> %x0, <4 x i32> %x1, i8 -1)
+  %res2 = add <4 x i32> %res, %res1
+  ret <4 x i32> %res2
+}
+
+declare <4 x i32> @llvm.x86.avx512.mask.cvtpd2udq.256(<4 x double>, <4 x i32>, i8)
+
+define <4 x i32>@test_int_x86_avx512_mask_cvt_pd2udq_256(<4 x double> %x0, <4 x i32> %x1, i8 %x2) {
+; CHECK-LABEL: test_int_x86_avx512_mask_cvt_pd2udq_256:
+; CHECK:       ## BB#0:
+; CHECK-NEXT:    movzbl %dil, %eax
+; CHECK-NEXT:    kmovw %eax, %k1
+; CHECK-NEXT:    vcvtpd2udq %ymm0, %xmm1 {%k1}
+; CHECK-NEXT:    vcvtpd2udq %ymm0, %xmm0
+; CHECK-NEXT:    vpaddd %xmm0, %xmm1, %xmm0
+; CHECK-NEXT:    retq
+  %res = call <4 x i32> @llvm.x86.avx512.mask.cvtpd2udq.256(<4 x double> %x0, <4 x i32> %x1, i8 %x2)
+  %res1 = call <4 x i32> @llvm.x86.avx512.mask.cvtpd2udq.256(<4 x double> %x0, <4 x i32> %x1, i8 -1)
+  %res2 = add <4 x i32> %res, %res1
+  ret <4 x i32> %res2
+}
+
+declare <4 x i32> @llvm.x86.avx512.mask.cvtps2dq.128(<4 x float>, <4 x i32>, i8)
+
+define <4 x i32>@test_int_x86_avx512_mask_cvt_ps2dq_128(<4 x float> %x0, <4 x i32> %x1, i8 %x2) {
+; CHECK-LABEL: test_int_x86_avx512_mask_cvt_ps2dq_128:
+; CHECK:       ## BB#0:
+; CHECK-NEXT:    movzbl %dil, %eax
+; CHECK-NEXT:    kmovw %eax, %k1
+; CHECK-NEXT:    vcvtps2dq %xmm0, %xmm1 {%k1}
+; CHECK-NEXT:    vcvtps2dq %xmm0, %xmm0
+; CHECK-NEXT:    vpaddd %xmm0, %xmm1, %xmm0
+; CHECK-NEXT:    retq
+  %res = call <4 x i32> @llvm.x86.avx512.mask.cvtps2dq.128(<4 x float> %x0, <4 x i32> %x1, i8 %x2)
+  %res1 = call <4 x i32> @llvm.x86.avx512.mask.cvtps2dq.128(<4 x float> %x0, <4 x i32> %x1, i8 -1)
+  %res2 = add <4 x i32> %res, %res1
+  ret <4 x i32> %res2
+}
+
+declare <8 x i32> @llvm.x86.avx512.mask.cvtps2dq.256(<8 x float>, <8 x i32>, i8)
+
+define <8 x i32>@test_int_x86_avx512_mask_cvt_ps2dq_256(<8 x float> %x0, <8 x i32> %x1, i8 %x2) {
+; CHECK-LABEL: test_int_x86_avx512_mask_cvt_ps2dq_256:
+; CHECK:       ## BB#0:
+; CHECK-NEXT:    movzbl %dil, %eax
+; CHECK-NEXT:    kmovw %eax, %k1
+; CHECK-NEXT:    vcvtps2dq %ymm0, %ymm1 {%k1}
+; CHECK-NEXT:    vcvtps2dq %ymm0, %ymm0
+; CHECK-NEXT:    vpaddd %ymm0, %ymm1, %ymm0
+; CHECK-NEXT:    retq
+  %res = call <8 x i32> @llvm.x86.avx512.mask.cvtps2dq.256(<8 x float> %x0, <8 x i32> %x1, i8 %x2)
+  %res1 = call <8 x i32> @llvm.x86.avx512.mask.cvtps2dq.256(<8 x float> %x0, <8 x i32> %x1, i8 -1)
+  %res2 = add <8 x i32> %res, %res1
+  ret <8 x i32> %res2
+}
+
+declare <2 x double> @llvm.x86.avx512.mask.cvtps2pd.128(<4 x float>, <2 x double>, i8)
+
+define <2 x double>@test_int_x86_avx512_mask_cvt_ps2pd_128(<4 x float> %x0, <2 x double> %x1, i8 %x2) {
+; CHECK-LABEL: test_int_x86_avx512_mask_cvt_ps2pd_128:
+; CHECK:       ## BB#0:
+; CHECK-NEXT:    movzbl %dil, %eax
+; CHECK-NEXT:    kmovw %eax, %k1
+; CHECK-NEXT:    vcvtps2pd %xmm0, %xmm1 {%k1}
+; CHECK-NEXT:    vcvtps2pd %xmm0, %xmm0
+; CHECK-NEXT:    vaddpd %xmm0, %xmm1, %xmm0
+; CHECK-NEXT:    retq
+  %res = call <2 x double> @llvm.x86.avx512.mask.cvtps2pd.128(<4 x float> %x0, <2 x double> %x1, i8 %x2)
+  %res1 = call <2 x double> @llvm.x86.avx512.mask.cvtps2pd.128(<4 x float> %x0, <2 x double> %x1, i8 -1)
+  %res2 = fadd <2 x double> %res, %res1
+  ret <2 x double> %res2
+}
+
+declare <4 x double> @llvm.x86.avx512.mask.cvtps2pd.256(<4 x float>, <4 x double>, i8)
+
+define <4 x double>@test_int_x86_avx512_mask_cvt_ps2pd_256(<4 x float> %x0, <4 x double> %x1, i8 %x2) {
+; CHECK-LABEL: test_int_x86_avx512_mask_cvt_ps2pd_256:
+; CHECK:       ## BB#0:
+; CHECK-NEXT:    movzbl %dil, %eax
+; CHECK-NEXT:    kmovw %eax, %k1
+; CHECK-NEXT:    vcvtps2pd %xmm0, %ymm1 {%k1}
+; CHECK-NEXT:    vcvtps2pd %xmm0, %ymm0
+; CHECK-NEXT:    vaddpd %ymm0, %ymm1, %ymm0
+; CHECK-NEXT:    retq
+  %res = call <4 x double> @llvm.x86.avx512.mask.cvtps2pd.256(<4 x float> %x0, <4 x double> %x1, i8 %x2)
+  %res1 = call <4 x double> @llvm.x86.avx512.mask.cvtps2pd.256(<4 x float> %x0, <4 x double> %x1, i8 -1)
+  %res2 = fadd <4 x double> %res, %res1
+  ret <4 x double> %res2
+}
+
+declare <4 x i32> @llvm.x86.avx512.mask.cvtps2udq.128(<4 x float>, <4 x i32>, i8)
+
+define <4 x i32>@test_int_x86_avx512_mask_cvt_ps2udq_128(<4 x float> %x0, <4 x i32> %x1, i8 %x2) {
+; CHECK-LABEL: test_int_x86_avx512_mask_cvt_ps2udq_128:
+; CHECK:       ## BB#0:
+; CHECK-NEXT:    movzbl %dil, %eax
+; CHECK-NEXT:    kmovw %eax, %k1
+; CHECK-NEXT:    vcvtps2udq %xmm0, %xmm1 {%k1}
+; CHECK-NEXT:    vcvtps2udq %xmm0, %xmm0
+; CHECK-NEXT:    vpaddd %xmm0, %xmm1, %xmm0
+; CHECK-NEXT:    retq
+  %res = call <4 x i32> @llvm.x86.avx512.mask.cvtps2udq.128(<4 x float> %x0, <4 x i32> %x1, i8 %x2)
+  %res1 = call <4 x i32> @llvm.x86.avx512.mask.cvtps2udq.128(<4 x float> %x0, <4 x i32> %x1, i8 -1)
+  %res2 = add <4 x i32> %res, %res1
+  ret <4 x i32> %res2
+}
+
+declare <8 x i32> @llvm.x86.avx512.mask.cvtps2udq.256(<8 x float>, <8 x i32>, i8)
+
+define <8 x i32>@test_int_x86_avx512_mask_cvt_ps2udq_256(<8 x float> %x0, <8 x i32> %x1, i8 %x2) {
+; CHECK-LABEL: test_int_x86_avx512_mask_cvt_ps2udq_256:
+; CHECK:       ## BB#0:
+; CHECK-NEXT:    movzbl %dil, %eax
+; CHECK-NEXT:    kmovw %eax, %k1
+; CHECK-NEXT:    vcvtps2udq %ymm0, %ymm1 {%k1}
+; CHECK-NEXT:    vcvtps2udq %ymm0, %ymm0
+; CHECK-NEXT:    vpaddd %ymm0, %ymm1, %ymm0
+; CHECK-NEXT:    retq
+  %res = call <8 x i32> @llvm.x86.avx512.mask.cvtps2udq.256(<8 x float> %x0, <8 x i32> %x1, i8 %x2)
+  %res1 = call <8 x i32> @llvm.x86.avx512.mask.cvtps2udq.256(<8 x float> %x0, <8 x i32> %x1, i8 -1)
+  %res2 = add <8 x i32> %res, %res1
+  ret <8 x i32> %res2
+}
+
+declare <4 x i32> @llvm.x86.avx512.mask.cvttpd2dq.128(<2 x double>, <4 x i32>, i8)
+
+define <4 x i32>@test_int_x86_avx512_mask_cvtt_pd2dq_128(<2 x double> %x0, <4 x i32> %x1, i8 %x2) {
+; CHECK-LABEL: test_int_x86_avx512_mask_cvtt_pd2dq_128:
+; CHECK:       ## BB#0:
+; CHECK-NEXT:    movzbl %dil, %eax
+; CHECK-NEXT:    kmovw %eax, %k1
+; CHECK-NEXT:    vcvttpd2dq %xmm0, %xmm1 {%k1}
+; CHECK-NEXT:    vcvttpd2dq %xmm0, %xmm0
+; CHECK-NEXT:    vpaddd %xmm0, %xmm1, %xmm0
+; CHECK-NEXT:    retq
+  %res = call <4 x i32> @llvm.x86.avx512.mask.cvttpd2dq.128(<2 x double> %x0, <4 x i32> %x1, i8 %x2)
+  %res1 = call <4 x i32> @llvm.x86.avx512.mask.cvttpd2dq.128(<2 x double> %x0, <4 x i32> %x1, i8 -1)
+  %res2 = add <4 x i32> %res, %res1
+  ret <4 x i32> %res2
+}
+
+declare <4 x i32> @llvm.x86.avx512.mask.cvttpd2dq.256(<4 x double>, <4 x i32>, i8)
+
+define <4 x i32>@test_int_x86_avx512_mask_cvtt_pd2dq_256(<4 x double> %x0, <4 x i32> %x1, i8 %x2) {
+; CHECK-LABEL: test_int_x86_avx512_mask_cvtt_pd2dq_256:
+; CHECK:       ## BB#0:
+; CHECK-NEXT:    movzbl %dil, %eax
+; CHECK-NEXT:    kmovw %eax, %k1
+; CHECK-NEXT:    vcvttpd2dq %ymm0, %xmm1 {%k1}
+; CHECK-NEXT:    vcvttpd2dq %ymm0, %xmm0
+; CHECK-NEXT:    vpaddd %xmm0, %xmm1, %xmm0
+; CHECK-NEXT:    retq
+  %res = call <4 x i32> @llvm.x86.avx512.mask.cvttpd2dq.256(<4 x double> %x0, <4 x i32> %x1, i8 %x2)
+  %res1 = call <4 x i32> @llvm.x86.avx512.mask.cvttpd2dq.256(<4 x double> %x0, <4 x i32> %x1, i8 -1)
+  %res2 = add <4 x i32> %res, %res1
+  ret <4 x i32> %res2
+}
+
+declare <4 x i32> @llvm.x86.avx512.mask.cvttpd2udq.128(<2 x double>, <4 x i32>, i8)
+
+define <4 x i32>@test_int_x86_avx512_mask_cvtt_pd2udq_128(<2 x double> %x0, <4 x i32> %x1, i8 %x2) {
+; CHECK-LABEL: test_int_x86_avx512_mask_cvtt_pd2udq_128:
+; CHECK:       ## BB#0:
+; CHECK-NEXT:    movzbl %dil, %eax
+; CHECK-NEXT:    kmovw %eax, %k1
+; CHECK-NEXT:    vcvttpd2udq %xmm0, %xmm1 {%k1}
+; CHECK-NEXT:    vcvttpd2udq %xmm0, %xmm0
+; CHECK-NEXT:    vpaddd %xmm0, %xmm1, %xmm0
+; CHECK-NEXT:    retq
+  %res = call <4 x i32> @llvm.x86.avx512.mask.cvttpd2udq.128(<2 x double> %x0, <4 x i32> %x1, i8 %x2)
+  %res1 = call <4 x i32> @llvm.x86.avx512.mask.cvttpd2udq.128(<2 x double> %x0, <4 x i32> %x1, i8 -1)
+  %res2 = add <4 x i32> %res, %res1
+  ret <4 x i32> %res2
+}
+
+declare <4 x i32> @llvm.x86.avx512.mask.cvttpd2udq.256(<4 x double>, <4 x i32>, i8)
+
+define <4 x i32>@test_int_x86_avx512_mask_cvtt_pd2udq_256(<4 x double> %x0, <4 x i32> %x1, i8 %x2) {
+; CHECK-LABEL: test_int_x86_avx512_mask_cvtt_pd2udq_256:
+; CHECK:       ## BB#0:
+; CHECK-NEXT:    movzbl %dil, %eax
+; CHECK-NEXT:    kmovw %eax, %k1
+; CHECK-NEXT:    vcvttpd2udq %ymm0, %xmm1 {%k1}
+; CHECK-NEXT:    vcvttpd2udq %ymm0, %xmm0
+; CHECK-NEXT:    vpaddd %xmm0, %xmm1, %xmm0
+; CHECK-NEXT:    retq
+  %res = call <4 x i32> @llvm.x86.avx512.mask.cvttpd2udq.256(<4 x double> %x0, <4 x i32> %x1, i8 %x2)
+  %res1 = call <4 x i32> @llvm.x86.avx512.mask.cvttpd2udq.256(<4 x double> %x0, <4 x i32> %x1, i8 -1)
+  %res2 = add <4 x i32> %res, %res1
+  ret <4 x i32> %res2
+}
+
+declare <4 x i32> @llvm.x86.avx512.mask.cvttps2dq.128(<4 x float>, <4 x i32>, i8)
+
+define <4 x i32>@test_int_x86_avx512_mask_cvtt_ps2dq_128(<4 x float> %x0, <4 x i32> %x1, i8 %x2) {
+; CHECK-LABEL: test_int_x86_avx512_mask_cvtt_ps2dq_128:
+; CHECK:       ## BB#0:
+; CHECK-NEXT:    movzbl %dil, %eax
+; CHECK-NEXT:    kmovw %eax, %k1
+; CHECK-NEXT:    vcvttps2dq %xmm0, %xmm1 {%k1}
+; CHECK-NEXT:    vcvttps2dq %xmm0, %xmm0
+; CHECK-NEXT:    vpaddd %xmm0, %xmm1, %xmm0
+; CHECK-NEXT:    retq
+  %res = call <4 x i32> @llvm.x86.avx512.mask.cvttps2dq.128(<4 x float> %x0, <4 x i32> %x1, i8 %x2)
+  %res1 = call <4 x i32> @llvm.x86.avx512.mask.cvttps2dq.128(<4 x float> %x0, <4 x i32> %x1, i8 -1)
+  %res2 = add <4 x i32> %res, %res1
+  ret <4 x i32> %res2
+}
+
+declare <8 x i32> @llvm.x86.avx512.mask.cvttps2dq.256(<8 x float>, <8 x i32>, i8)
+
+define <8 x i32>@test_int_x86_avx512_mask_cvtt_ps2dq_256(<8 x float> %x0, <8 x i32> %x1, i8 %x2) {
+; CHECK-LABEL: test_int_x86_avx512_mask_cvtt_ps2dq_256:
+; CHECK:       ## BB#0:
+; CHECK-NEXT:    movzbl %dil, %eax
+; CHECK-NEXT:    kmovw %eax, %k1
+; CHECK-NEXT:    vcvttps2dq %ymm0, %ymm1 {%k1}
+; CHECK-NEXT:    vcvttps2dq %ymm0, %ymm0
+; CHECK-NEXT:    vpaddd %ymm0, %ymm1, %ymm0
+; CHECK-NEXT:    retq
+  %res = call <8 x i32> @llvm.x86.avx512.mask.cvttps2dq.256(<8 x float> %x0, <8 x i32> %x1, i8 %x2)
+  %res1 = call <8 x i32> @llvm.x86.avx512.mask.cvttps2dq.256(<8 x float> %x0, <8 x i32> %x1, i8 -1)
+  %res2 = add <8 x i32> %res, %res1
+  ret <8 x i32> %res2
+}
+
+declare <4 x i32> @llvm.x86.avx512.mask.cvttps2udq.128(<4 x float>, <4 x i32>, i8)
+
+define <4 x i32>@test_int_x86_avx512_mask_cvtt_ps2udq_128(<4 x float> %x0, <4 x i32> %x1, i8 %x2) {
+; CHECK-LABEL: test_int_x86_avx512_mask_cvtt_ps2udq_128:
+; CHECK:       ## BB#0:
+; CHECK-NEXT:    movzbl %dil, %eax
+; CHECK-NEXT:    kmovw %eax, %k1
+; CHECK-NEXT:    vcvttps2udq %xmm0, %xmm1 {%k1}
+; CHECK-NEXT:    vcvttps2udq %xmm0, %xmm0
+; CHECK-NEXT:    vpaddd %xmm0, %xmm1, %xmm0
+; CHECK-NEXT:    retq
+  %res = call <4 x i32> @llvm.x86.avx512.mask.cvttps2udq.128(<4 x float> %x0, <4 x i32> %x1, i8 %x2)
+  %res1 = call <4 x i32> @llvm.x86.avx512.mask.cvttps2udq.128(<4 x float> %x0, <4 x i32> %x1, i8 -1)
+  %res2 = add <4 x i32> %res, %res1
+  ret <4 x i32> %res2
+}
+
+declare <8 x i32> @llvm.x86.avx512.mask.cvttps2udq.256(<8 x float>, <8 x i32>, i8)
+
+define <8 x i32>@test_int_x86_avx512_mask_cvtt_ps2udq_256(<8 x float> %x0, <8 x i32> %x1, i8 %x2) {
+; CHECK-LABEL: test_int_x86_avx512_mask_cvtt_ps2udq_256:
+; CHECK:       ## BB#0:
+; CHECK-NEXT:    movzbl %dil, %eax
+; CHECK-NEXT:    kmovw %eax, %k1
+; CHECK-NEXT:    vcvttps2udq %ymm0, %ymm1 {%k1}
+; CHECK-NEXT:    vcvttps2udq %ymm0, %ymm0
+; CHECK-NEXT:    vpaddd %ymm0, %ymm1, %ymm0
+; CHECK-NEXT:    retq
+  %res = call <8 x i32> @llvm.x86.avx512.mask.cvttps2udq.256(<8 x float> %x0, <8 x i32> %x1, i8 %x2)
+  %res1 = call <8 x i32> @llvm.x86.avx512.mask.cvttps2udq.256(<8 x float> %x0, <8 x i32> %x1, i8 -1)
+  %res2 = add <8 x i32> %res, %res1
+  ret <8 x i32> %res2
+}
+
+declare <2 x double> @llvm.x86.avx512.mask.cvtudq2pd.128(<4 x i32>, <2 x double>, i8)
+
+define <2 x double>@test_int_x86_avx512_mask_cvt_udq2pd_128(<4 x i32> %x0, <2 x double> %x1, i8 %x2) {
+; CHECK-LABEL: test_int_x86_avx512_mask_cvt_udq2pd_128:
+; CHECK:       ## BB#0:
+; CHECK-NEXT:    movzbl %dil, %eax
+; CHECK-NEXT:    kmovw %eax, %k1
+; CHECK-NEXT:    vcvtudq2pd %xmm0, %xmm1 {%k1}
+; CHECK-NEXT:    vcvtudq2pd %xmm0, %xmm0
+; CHECK-NEXT:    vaddpd %xmm0, %xmm1, %xmm0
+; CHECK-NEXT:    retq
+  %res = call <2 x double> @llvm.x86.avx512.mask.cvtudq2pd.128(<4 x i32> %x0, <2 x double> %x1, i8 %x2)
+  %res1 = call <2 x double> @llvm.x86.avx512.mask.cvtudq2pd.128(<4 x i32> %x0, <2 x double> %x1, i8 -1)
+  %res2 = fadd <2 x double> %res, %res1
+  ret <2 x double> %res2
+}
+
+declare <4 x double> @llvm.x86.avx512.mask.cvtudq2pd.256(<4 x i32>, <4 x double>, i8)
+
+define <4 x double>@test_int_x86_avx512_mask_cvt_udq2pd_256(<4 x i32> %x0, <4 x double> %x1, i8 %x2) {
+; CHECK-LABEL: test_int_x86_avx512_mask_cvt_udq2pd_256:
+; CHECK:       ## BB#0:
+; CHECK-NEXT:    movzbl %dil, %eax
+; CHECK-NEXT:    kmovw %eax, %k1
+; CHECK-NEXT:    vcvtudq2pd %xmm0, %ymm1 {%k1}
+; CHECK-NEXT:    vcvtudq2pd %xmm0, %ymm0
+; CHECK-NEXT:    vaddpd %ymm0, %ymm1, %ymm0
+; CHECK-NEXT:    retq
+  %res = call <4 x double> @llvm.x86.avx512.mask.cvtudq2pd.256(<4 x i32> %x0, <4 x double> %x1, i8 %x2)
+  %res1 = call <4 x double> @llvm.x86.avx512.mask.cvtudq2pd.256(<4 x i32> %x0, <4 x double> %x1, i8 -1)
+  %res2 = fadd <4 x double> %res, %res1
+  ret <4 x double> %res2
+}
+
+declare <4 x float> @llvm.x86.avx512.mask.cvtudq2ps.128(<4 x i32>, <4 x float>, i8)
+
+define <4 x float>@test_int_x86_avx512_mask_cvt_udq2ps_128(<4 x i32> %x0, <4 x float> %x1, i8 %x2) {
+; CHECK-LABEL: test_int_x86_avx512_mask_cvt_udq2ps_128:
+; CHECK:       ## BB#0:
+; CHECK-NEXT:    movzbl %dil, %eax
+; CHECK-NEXT:    kmovw %eax, %k1
+; CHECK-NEXT:    vcvtudq2ps %xmm0, %xmm1 {%k1}
+; CHECK-NEXT:    vcvtudq2ps %xmm0, %xmm0
+; CHECK-NEXT:    vaddps %xmm0, %xmm1, %xmm0
+; CHECK-NEXT:    retq
+  %res = call <4 x float> @llvm.x86.avx512.mask.cvtudq2ps.128(<4 x i32> %x0, <4 x float> %x1, i8 %x2)
+  %res1 = call <4 x float> @llvm.x86.avx512.mask.cvtudq2ps.128(<4 x i32> %x0, <4 x float> %x1, i8 -1)
+  %res2 = fadd <4 x float> %res, %res1
+  ret <4 x float> %res2
+}
+
+declare <8 x float> @llvm.x86.avx512.mask.cvtudq2ps.256(<8 x i32>, <8 x float>, i8)
+
+define <8 x float>@test_int_x86_avx512_mask_cvt_udq2ps_256(<8 x i32> %x0, <8 x float> %x1, i8 %x2) {
+; CHECK-LABEL: test_int_x86_avx512_mask_cvt_udq2ps_256:
+; CHECK:       ## BB#0:
+; CHECK-NEXT:    movzbl %dil, %eax
+; CHECK-NEXT:    kmovw %eax, %k1
+; CHECK-NEXT:    vcvtudq2ps %ymm0, %ymm1 {%k1}
+; CHECK-NEXT:    vcvtudq2ps %ymm0, %ymm0
+; CHECK-NEXT:    vaddps %ymm0, %ymm1, %ymm0
+; CHECK-NEXT:    retq
+  %res = call <8 x float> @llvm.x86.avx512.mask.cvtudq2ps.256(<8 x i32> %x0, <8 x float> %x1, i8 %x2)
+  %res1 = call <8 x float> @llvm.x86.avx512.mask.cvtudq2ps.256(<8 x i32> %x0, <8 x float> %x1, i8 -1)
+  %res2 = fadd <8 x float> %res, %res1
+  ret <8 x float> %res2
+}
+
+declare <2 x double> @llvm.x86.avx512.mask.rndscale.pd.128(<2 x double>, i32, <2 x double>, i8)
+; CHECK-LABEL: @test_int_x86_avx512_mask_rndscale_pd_128
+; CHECK-NOT: call 
+; CHECK: kmov 
+; CHECK: vrndscalepd {{.*}}{%k1} 
+; CHECK: vrndscalepd
+define <2 x double>@test_int_x86_avx512_mask_rndscale_pd_128(<2 x double> %x0, <2 x double> %x2, i8 %x3) {
+  %res = call <2 x double> @llvm.x86.avx512.mask.rndscale.pd.128(<2 x double> %x0, i32 4, <2 x double> %x2, i8 %x3)
+  %res1 = call <2 x double> @llvm.x86.avx512.mask.rndscale.pd.128(<2 x double> %x0, i32 88, <2 x double> %x2, i8 -1)
+  %res2 = fadd <2 x double> %res, %res1
+  ret <2 x double> %res2
+}
+
+declare <4 x double> @llvm.x86.avx512.mask.rndscale.pd.256(<4 x double>, i32, <4 x double>, i8)
+; CHECK-LABEL: @test_int_x86_avx512_mask_rndscale_pd_256
+; CHECK-NOT: call 
+; CHECK: kmov 
+; CHECK: vrndscalepd {{.*}}{%k1} 
+; CHECK: vrndscalepd
+define <4 x double>@test_int_x86_avx512_mask_rndscale_pd_256(<4 x double> %x0, <4 x double> %x2, i8 %x3) {
+  %res = call <4 x double> @llvm.x86.avx512.mask.rndscale.pd.256(<4 x double> %x0, i32 4, <4 x double> %x2, i8 %x3)
+  %res1 = call <4 x double> @llvm.x86.avx512.mask.rndscale.pd.256(<4 x double> %x0, i32 88, <4 x double> %x2, i8 -1)
+  %res2 = fadd <4 x double> %res, %res1
+  ret <4 x double> %res2
+}
+
+declare <4 x float> @llvm.x86.avx512.mask.rndscale.ps.128(<4 x float>, i32, <4 x float>, i8)
+; CHECK-LABEL: @test_int_x86_avx512_mask_rndscale_ps_128
+; CHECK-NOT: call 
+; CHECK: kmov 
+; CHECK: vrndscaleps {{.*}}{%k1} 
+; CHECK: vrndscaleps
+define <4 x float>@test_int_x86_avx512_mask_rndscale_ps_128(<4 x float> %x0, <4 x float> %x2, i8 %x3) {
+  %res = call <4 x float> @llvm.x86.avx512.mask.rndscale.ps.128(<4 x float> %x0, i32 88, <4 x float> %x2, i8 %x3)
+  %res1 = call <4 x float> @llvm.x86.avx512.mask.rndscale.ps.128(<4 x float> %x0, i32 4, <4 x float> %x2, i8 -1)
+  %res2 = fadd <4 x float> %res, %res1
+  ret <4 x float> %res2
+}
+
+declare <8 x float> @llvm.x86.avx512.mask.rndscale.ps.256(<8 x float>, i32, <8 x float>, i8)
+
+; CHECK-LABEL: @test_int_x86_avx512_mask_rndscale_ps_256
+; CHECK-NOT: call 
+; CHECK: kmov 
+; CHECK: vrndscaleps {{.*}}{%k1} 
+; CHECK: vrndscaleps
+define <8 x float>@test_int_x86_avx512_mask_rndscale_ps_256(<8 x float> %x0, <8 x float> %x2, i8 %x3) {
+  %res = call <8 x float> @llvm.x86.avx512.mask.rndscale.ps.256(<8 x float> %x0, i32 5, <8 x float> %x2, i8 %x3)
+  %res1 = call <8 x float> @llvm.x86.avx512.mask.rndscale.ps.256(<8 x float> %x0, i32 66, <8 x float> %x2, i8 -1)
+  %res2 = fadd <8 x float> %res, %res1
+  ret <8 x float> %res2
+}
diff --git a/test/CodeGen/X86/block-placement.ll b/test/CodeGen/X86/block-placement.ll
index e0276e42d4d..89defa956a4 100644
--- a/test/CodeGen/X86/block-placement.ll
+++ b/test/CodeGen/X86/block-placement.ll
@@ -546,7 +546,7 @@ exit:
 
 declare i32 @__gxx_personality_v0(...)
 
-define void @test_eh_lpad_successor() {
+define void @test_eh_lpad_successor() personality i8* bitcast (i32 (...)* @__gxx_personality_v0 to i8*) {
 ; Some times the landing pad ends up as the first successor of an invoke block.
 ; When this happens, a strange result used to fall out of updateTerminators: we
 ; didn't correctly locate the fallthrough successor, assuming blindly that the
@@ -564,7 +564,7 @@ preheader:
   br label %loop
 
 lpad:
-  %lpad.val = landingpad { i8*, i32 } personality i8* bitcast (i32 (...)* @__gxx_personality_v0 to i8*)
+  %lpad.val = landingpad { i8*, i32 }
           cleanup
   resume { i8*, i32 } %lpad.val
 
@@ -574,7 +574,7 @@ loop:
 
 declare void @fake_throw() noreturn
 
-define void @test_eh_throw() {
+define void @test_eh_throw() personality i8* bitcast (i32 (...)* @__gxx_personality_v0 to i8*) {
 ; For blocks containing a 'throw' (or similar functionality), we have
 ; a no-return invoke. In this case, only EH successors will exist, and
 ; fallthrough simply won't occur. Make sure we don't crash trying to update
@@ -591,7 +591,7 @@ continue:
   unreachable
 
 cleanup:
-  %0 = landingpad { i8*, i32 } personality i8* bitcast (i32 (...)* @__gxx_personality_v0 to i8*)
+  %0 = landingpad { i8*, i32 }
           cleanup
   unreachable
 }
diff --git a/test/CodeGen/X86/branchfolding-landingpads.ll b/test/CodeGen/X86/branchfolding-landingpads.ll
new file mode 100644
index 00000000000..032b9881245
--- /dev/null
+++ b/test/CodeGen/X86/branchfolding-landingpads.ll
@@ -0,0 +1,45 @@
+; RUN: llc %s -o - -verify-machineinstrs | FileCheck %s
+
+target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-unknown"
+
+; The machine level BranchFolding pass will try to remove the 'unreachable' block
+; and rewrite 'entry' to jump to the block 'unreachable' falls through to.
+; That will be a landing pad and result in 'entry' jumping to 2 landing pads.
+; This tests that we don't do this change when the fallthrough is itself a landing
+; pad.
+
+declare i32 @__gxx_personality_v0(...)
+declare void @foo()
+
+; Function Attrs: noreturn
+declare void @_throw()
+
+; CHECK-LABEL: @main
+; CHECK: %unreachable
+
+define i32 @main(i8* %cleanup) personality i8* bitcast (i32 (...)* @__gxx_personality_v0 to i8*) {
+entry:
+  invoke void @_throw() #0
+          to label %unreachable unwind label %catch.dispatch9
+
+catch.dispatch9:                                  ; preds = %entry
+  %tmp13 = landingpad { i8*, i32 }
+          cleanup
+          catch i8* null
+  invoke void @_throw() #0
+          to label %unreachable unwind label %lpad31
+
+lpad31:                                           ; preds = %catch.dispatch9
+  %tmp20 = landingpad { i8*, i32 }
+          cleanup
+          catch i8* null
+  call void @foo()
+  unreachable
+
+unreachable:                                      ; preds = %catch.dispatch9, %entry
+  unreachable
+}
+
+attributes #0 = { noreturn }
+
diff --git a/test/CodeGen/X86/bswap-vector.ll b/test/CodeGen/X86/bswap-vector.ll
index 7d5f380c1e2..5376601a95e 100644
--- a/test/CodeGen/X86/bswap-vector.ll
+++ b/test/CodeGen/X86/bswap-vector.ll
@@ -1,6 +1,6 @@
-; RUN: llc < %s -mcpu=x86-64 | FileCheck %s --check-prefix=CHECK-NOSSSE3
-; RUN: llc < %s -mcpu=core2 | FileCheck %s --check-prefix=CHECK-SSSE3
-; RUN: llc < %s -mcpu=core-avx2 | FileCheck %s --check-prefix=CHECK-AVX2
+; RUN: llc < %s -mcpu=x86-64 | FileCheck %s --check-prefix=CHECK-ALL --check-prefix=CHECK-SSE --check-prefix=CHECK-NOSSSE3
+; RUN: llc < %s -mcpu=core2 | FileCheck %s --check-prefix=CHECK-ALL --check-prefix=CHECK-SSE --check-prefix=CHECK-SSSE3
+; RUN: llc < %s -mcpu=core-avx2 | FileCheck %s --check-prefix=CHECK-AVX --check-prefix=CHECK-AVX2
 ; RUN: llc < %s -mcpu=core-avx2 -x86-experimental-vector-widening-legalization | FileCheck %s --check-prefix=CHECK-WIDE-AVX2
 
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
@@ -285,3 +285,174 @@ entry:
   %r = call <4 x i16> @llvm.bswap.v4i16(<4 x i16> %v)
   ret <4 x i16> %r
 }
+
+;
+; Double BSWAP -> Identity
+;
+
+define <8 x i16> @identity_v8i16(<8 x i16> %v) {
+; CHECK-ALL-LABEL: identity_v8i16:
+; CHECK-ALL:       # BB#0: # %entry
+; CHECK-ALL:         retq
+entry:
+  %bs1 = call <8 x i16> @llvm.bswap.v8i16(<8 x i16> %v)
+  %bs2 = call <8 x i16> @llvm.bswap.v8i16(<8 x i16> %bs1)
+  ret <8 x i16> %bs2
+}
+
+define <4 x i32> @identity_v4i32(<4 x i32> %v) {
+; CHECK-ALL-LABEL: identity_v4i32:
+; CHECK-ALL:       # BB#0: # %entry
+; CHECK-ALL-NEXT:    retq
+entry:
+  %bs1 = call <4 x i32> @llvm.bswap.v4i32(<4 x i32> %v)
+  %bs2 = call <4 x i32> @llvm.bswap.v4i32(<4 x i32> %bs1)
+  ret <4 x i32> %bs2
+}
+
+define <2 x i64> @identity_v2i64(<2 x i64> %v) {
+; CHECK-ALL-LABEL: identity_v2i64:
+; CHECK-ALL:       # BB#0: # %entry
+; CHECK-ALL-NEXT:    retq
+entry:
+  %bs1 = call <2 x i64> @llvm.bswap.v2i64(<2 x i64> %v)
+  %bs2 = call <2 x i64> @llvm.bswap.v2i64(<2 x i64> %bs1)
+  ret <2 x i64> %bs2
+}
+
+define <16 x i16> @identity_v16i16(<16 x i16> %v) {
+; CHECK-ALL-LABEL: identity_v16i16:
+; CHECK-ALL:       # BB#0: # %entry
+; CHECK-ALL-NEXT:    retq
+entry:
+  %bs1 = call <16 x i16> @llvm.bswap.v16i16(<16 x i16> %v)
+  %bs2 = call <16 x i16> @llvm.bswap.v16i16(<16 x i16> %bs1)
+  ret <16 x i16> %bs2
+}
+
+define <8 x i32> @identity_v8i32(<8 x i32> %v) {
+; CHECK-ALL-LABEL: identity_v8i32:
+; CHECK-ALL:       # BB#0: # %entry
+; CHECK-ALL-NEXT:    retq
+entry:
+  %bs1 = call <8 x i32> @llvm.bswap.v8i32(<8 x i32> %v)
+  %bs2 = call <8 x i32> @llvm.bswap.v8i32(<8 x i32> %bs1)
+  ret <8 x i32> %bs2
+}
+
+define <4 x i64> @identity_v4i64(<4 x i64> %v) {
+; CHECK-ALL-LABEL: identity_v4i64:
+; CHECK-ALL:       # BB#0: # %entry
+; CHECK-ALL-NEXT:    retq
+entry:
+  %bs1 = call <4 x i64> @llvm.bswap.v4i64(<4 x i64> %v)
+  %bs2 = call <4 x i64> @llvm.bswap.v4i64(<4 x i64> %bs1)
+  ret <4 x i64> %bs2
+}
+
+define <4 x i16> @identity_v4i16(<4 x i16> %v) {
+; CHECK-ALL-LABEL: identity_v4i16:
+; CHECK-ALL:       # BB#0: # %entry
+; CHECK-ALL-NEXT:    retq
+entry:
+  %bs1 = call <4 x i16> @llvm.bswap.v4i16(<4 x i16> %v)
+  %bs2 = call <4 x i16> @llvm.bswap.v4i16(<4 x i16> %bs1)
+  ret <4 x i16> %bs2
+}
+
+;
+; Constant Folding
+;
+
+define <8 x i16> @fold_v8i16() {
+; CHECK-SSE-LABEL: fold_v8i16:
+; CHECK-SSE:       # BB#0: # %entry
+; CHECK-SSE-NEXT:    movaps {{.*#+}} xmm0 = [0,256,65535,512,65023,1024,64511,1536]
+; CHECK-SSE-NEXT:    retq
+;
+; CHECK-AVX-LABEL: fold_v8i16:
+; CHECK-AVX:       # BB#0: # %entry
+; CHECK-AVX-NEXT:    vmovaps {{.*#+}} xmm0 = [0,256,65535,512,65023,1024,64511,1536]
+; CHECK-AVX-NEXT:    retq
+entry:
+  %r = call <8 x i16> @llvm.bswap.v8i16(<8 x i16> <i16 0, i16 1, i16 -1, i16 2, i16 -3, i16 4, i16 -5, i16 6>)
+  ret <8 x i16> %r
+}
+
+define <4 x i32> @fold_v4i32() {
+; CHECK-SSE-LABEL: fold_v4i32:
+; CHECK-SSE:       # BB#0: # %entry
+; CHECK-SSE-NEXT:    movaps {{.*#+}} xmm0 = [0,4294967295,33554432,4261412863]
+; CHECK-SSE-NEXT:    retq
+;
+; CHECK-AVX-LABEL: fold_v4i32:
+; CHECK-AVX:       # BB#0: # %entry
+; CHECK-AVX-NEXT:    vmovaps {{.*#+}} xmm0 = [0,4294967295,33554432,4261412863]
+; CHECK-AVX-NEXT:    retq
+entry:
+  %r = call <4 x i32> @llvm.bswap.v4i32(<4 x i32> <i32 0, i32 -1, i32 2, i32 -3>)
+  ret <4 x i32> %r
+}
+
+define <2 x i64> @fold_v2i64() {
+; CHECK-SSE-LABEL: fold_v2i64:
+; CHECK-SSE:       # BB#0: # %entry
+; CHECK-SSE-NEXT:    movaps {{.*#+}} xmm0 = [18374686479671623680,18446744073709551615]
+; CHECK-SSE-NEXT:    retq
+;
+; CHECK-AVX-LABEL: fold_v2i64:
+; CHECK-AVX:       # BB#0: # %entry
+; CHECK-AVX-NEXT:    vmovaps {{.*#+}} xmm0 = [18374686479671623680,18446744073709551615]
+; CHECK-AVX-NEXT:    retq
+entry:
+  %r = call <2 x i64> @llvm.bswap.v2i64(<2 x i64> <i64 255, i64 -1>)
+  ret <2 x i64> %r
+}
+
+define <16 x i16> @fold_v16i16() {
+; CHECK-SSE-LABEL: fold_v16i16:
+; CHECK-SSE:       # BB#0: # %entry
+; CHECK-SSE-NEXT:    movaps {{.*#+}} xmm0 = [0,256,65535,512,65023,1024,64511,1536]
+; CHECK-SSE-NEXT:    movaps {{.*#+}} xmm1 = [63999,2048,63487,2560,62975,3072,62463,3584]
+; CHECK-SSE-NEXT:    retq
+;
+; CHECK-AVX-LABEL: fold_v16i16:
+; CHECK-AVX:       # BB#0: # %entry
+; CHECK-AVX-NEXT:    vmovaps {{.*#+}} ymm0 = [0,256,65535,512,65023,1024,64511,1536,63999,2048,63487,2560,62975,3072,62463,3584]
+; CHECK-AVX-NEXT:    retq
+entry:
+  %r = call <16 x i16> @llvm.bswap.v16i16(<16 x i16> <i16 0, i16 1, i16 -1, i16 2, i16 -3, i16 4, i16 -5, i16 6, i16 -7, i16 8, i16 -9, i16 10, i16 -11, i16 12, i16 -13, i16 14>)
+  ret <16 x i16> %r
+}
+
+define <8 x i32> @fold_v8i32() {
+; CHECK-SSE-LABEL: fold_v8i32:
+; CHECK-SSE:       # BB#0: # %entry
+; CHECK-SSE-NEXT:    movaps {{.*#+}} xmm0 = [0,16777216,4294967295,33554432]
+; CHECK-SSE-NEXT:    movaps {{.*#+}} xmm1 = [4261412863,67108864,4227858431,100663296]
+; CHECK-SSE-NEXT:    retq
+;
+; CHECK-AVX-LABEL: fold_v8i32:
+; CHECK-AVX:       # BB#0: # %entry
+; CHECK-AVX-NEXT:    vmovaps {{.*#+}} ymm0 = [0,16777216,4294967295,33554432,4261412863,67108864,4227858431,100663296]
+; CHECK-AVX-NEXT:    retq
+entry:
+  %r = call <8 x i32> @llvm.bswap.v8i32(<8 x i32> <i32 0, i32 1, i32 -1, i32 2, i32 -3, i32 4, i32 -5, i32 6>)
+  ret <8 x i32> %r
+}
+
+define <4 x i64> @fold_v4i64() {
+; CHECK-SSE-LABEL: fold_v4i64:
+; CHECK-SSE:       # BB#0: # %entry
+; CHECK-SSE-NEXT:    movaps {{.*#+}} xmm0 = [18374686479671623680,18446744073709551615]
+; CHECK-SSE-NEXT:    movaps {{.*#+}} xmm1 = [18446462598732840960,72056494526300160]
+; CHECK-SSE-NEXT:    retq
+;
+; CHECK-AVX-LABEL: fold_v4i64:
+; CHECK-AVX:       # BB#0: # %entry
+; CHECK-AVX-NEXT:    vmovaps {{.*#+}} ymm0 = [18374686479671623680,18446744073709551615,18446462598732840960,72056494526300160]
+; CHECK-AVX-NEXT:    retq
+entry:
+  %r = call <4 x i64> @llvm.bswap.v4i64(<4 x i64> <i64 255, i64 -1, i64 65535, i64 16776960>)
+  ret <4 x i64> %r
+}
diff --git a/test/CodeGen/X86/buildvec-insertvec.ll b/test/CodeGen/X86/buildvec-insertvec.ll
index 3fb69a48b3c..73dbe1f650a 100644
--- a/test/CodeGen/X86/buildvec-insertvec.ll
+++ b/test/CodeGen/X86/buildvec-insertvec.ll
@@ -1,15 +1,56 @@
-; RUN: llc < %s -mcpu=corei7 -mtriple=x86_64-unknown-linux-gnu | FileCheck %s
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+sse4.1 | FileCheck %s
 
 define void @foo(<3 x float> %in, <4 x i8>* nocapture %out) nounwind {
+; CHECK-LABEL: foo:
+; CHECK:       # BB#0:
+; CHECK-NEXT:    cvttps2dq %xmm0, %xmm0
+; CHECK-NEXT:    movl $255, %eax
+; CHECK-NEXT:    pinsrd $3, %eax, %xmm0
+; CHECK-NEXT:    pshufb {{.*#+}} xmm0 = xmm0[0,4,8,12,u,u,u,u,u,u,u,u,u,u,u,u]
+; CHECK-NEXT:    movd %xmm0, (%rdi)
+; CHECK-NEXT:    retq
   %t0 = fptoui <3 x float> %in to <3 x i8>
   %t1 = shufflevector <3 x i8> %t0, <3 x i8> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 undef>
   %t2 = insertelement <4 x i8> %t1, i8 -1, i32 3
   store <4 x i8> %t2, <4 x i8>* %out, align 4
   ret void
-; CHECK: foo
-; CHECK: cvttps2dq
-; CHECK-NOT: pextrd
-; CHECK: pinsrd
-; CHECK-NEXT: pshufb
-; CHECK: ret
+}
+
+; Verify that the DAGCombiner doesn't wrongly fold a build_vector into a
+; blend with a zero vector if the build_vector contains negative zero.
+;
+; TODO: the codegen for function 'test_negative_zero_1' is sub-optimal.
+; Ideally, we should generate a single shuffle blend operation.
+
+define <4 x float> @test_negative_zero_1(<4 x float> %A) {
+; CHECK-LABEL: test_negative_zero_1:
+; CHECK:       # BB#0: # %entry
+; CHECK-NEXT:    movapd %xmm0, %xmm1
+; CHECK-NEXT:    shufpd {{.*#+}} xmm1 = xmm1[1,0]
+; CHECK-NEXT:    xorps %xmm2, %xmm2
+; CHECK-NEXT:    blendps {{.*#+}} xmm2 = xmm1[0],xmm2[1,2,3]
+; CHECK-NEXT:    movss {{.*#+}} xmm1 = mem[0],zero,zero,zero
+; CHECK-NEXT:    unpcklps {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
+; CHECK-NEXT:    unpcklpd {{.*#+}} xmm0 = xmm0[0],xmm2[0]
+; CHECK-NEXT:    retq
+entry:
+  %0 = extractelement <4 x float> %A, i32 0
+  %1 = insertelement <4 x float> undef, float %0, i32 0
+  %2 = insertelement <4 x float> %1, float -0.0, i32 1
+  %3 = extractelement <4 x float> %A, i32 2
+  %4 = insertelement <4 x float> %2, float %3, i32 2
+  %5 = insertelement <4 x float> %4, float 0.0, i32 3
+  ret <4 x float> %5
+}
+
+define <2 x double> @test_negative_zero_2(<2 x double> %A) {
+; CHECK-LABEL: test_negative_zero_2:
+; CHECK:       # BB#0: # %entry
+; CHECK-NEXT:    movhpd {{.*}}(%rip), %xmm0
+; CHECK-NEXT:    retq
+entry:
+  %0 = extractelement <2 x double> %A, i32 0
+  %1 = insertelement <2 x double> undef, double %0, i32 0
+  %2 = insertelement <2 x double> %1, double -0.0, i32 1
+  ret <2 x double> %2
 }
diff --git a/test/CodeGen/X86/catch.ll b/test/CodeGen/X86/catch.ll
index 64e92783ac9..be7466e8abb 100644
--- a/test/CodeGen/X86/catch.ll
+++ b/test/CodeGen/X86/catch.ll
@@ -7,13 +7,13 @@
 ; CHECK-NEXT: .quad   .Lstr
 
 @str = private unnamed_addr constant [12 x i8] c"NSException\00"
-define void @f() {
+define void @f() personality i8* bitcast (void ()* @h to i8*) {
   invoke void @g()
           to label %invoke.cont unwind label %lpad
 invoke.cont:
   ret void
 lpad:
-  %tmp14 = landingpad { i8*, i32 } personality i8* bitcast (void ()* @h to i8*)
+  %tmp14 = landingpad { i8*, i32 }
            catch i8* getelementptr inbounds ([12 x i8], [12 x i8]* @str, i64 0, i64 0)
   ret void
 }
diff --git a/test/CodeGen/X86/cfi.ll b/test/CodeGen/X86/cfi.ll
index b57ff45f51e..d5a3a8a26a3 100644
--- a/test/CodeGen/X86/cfi.ll
+++ b/test/CodeGen/X86/cfi.ll
@@ -8,7 +8,7 @@
 ; PIC: .cfi_lsda 27, .Lexception0
 
 
-define void @bar() {
+define void @bar() personality i32 (...)* @__gxx_personality_v0 {
 entry:
   %call = invoke i32 @foo()
           to label %invoke.cont unwind label %lpad
@@ -17,7 +17,7 @@ invoke.cont:
   ret void
 
 lpad:
-  %exn = landingpad {i8*, i32} personality i32 (...)* @__gxx_personality_v0
+  %exn = landingpad {i8*, i32}
             catch i8* null
   ret void
 }
diff --git a/test/CodeGen/X86/cmovcmov.ll b/test/CodeGen/X86/cmovcmov.ll
index 982bc82c4c7..d3d9748d653 100644
--- a/test/CodeGen/X86/cmovcmov.ll
+++ b/test/CodeGen/X86/cmovcmov.ll
@@ -143,19 +143,19 @@ entry:
 ; NOCMOV-NEXT:   jp  [[TBB]]
 ; NOCMOV-NEXT:   leal  24(%esp), %eax
 ; NOCMOV-NEXT: [[TBB]]:
-; NOCMOV-NEXT:   movl  (%eax), %eax
-; NOCMOV-NEXT:   leal  44(%esp), %ecx
+; NOCMOV-NEXT:   movl  (%eax), %ecx
+; NOCMOV-NEXT:   leal  44(%esp), %edx
 ; NOCMOV-NEXT:   jne  [[TBB:.LBB[0-9_]+]]
 ; NOCMOV-NEXT:   jp  [[TBB]]
-; NOCMOV-NEXT:   leal  28(%esp), %ecx
+; NOCMOV-NEXT:   leal  28(%esp), %edx
 ; NOCMOV-NEXT: [[TBB]]:
-; NOCMOV-NEXT:   movl  (%ecx), %ecx
+; NOCMOV-NEXT:   movl  12(%esp), %eax
+; NOCMOV-NEXT:   movl  (%edx), %edx
 ; NOCMOV-NEXT:   leal  48(%esp), %esi
 ; NOCMOV-NEXT:   jne  [[TBB:.LBB[0-9_]+]]
 ; NOCMOV-NEXT:   jp  [[TBB]]
 ; NOCMOV-NEXT:   leal  32(%esp), %esi
 ; NOCMOV-NEXT: [[TBB]]:
-; NOCMOV-NEXT:   movl  12(%esp), %edx
 ; NOCMOV-NEXT:   movl  (%esi), %esi
 ; NOCMOV-NEXT:   leal  52(%esp), %edi
 ; NOCMOV-NEXT:   jne  [[TBB:.LBB[0-9_]+]]
@@ -163,10 +163,10 @@ entry:
 ; NOCMOV-NEXT:   leal  36(%esp), %edi
 ; NOCMOV-NEXT: [[TBB]]:
 ; NOCMOV-NEXT:   movl  (%edi), %edi
-; NOCMOV-NEXT:   movl  %edi, 12(%edx)
-; NOCMOV-NEXT:   movl  %esi, 8(%edx)
-; NOCMOV-NEXT:   movl  %ecx, 4(%edx)
-; NOCMOV-NEXT:   movl  %eax, (%edx)
+; NOCMOV-NEXT:   movl  %edi, 12(%eax)
+; NOCMOV-NEXT:   movl  %esi, 8(%eax)
+; NOCMOV-NEXT:   movl  %edx, 4(%eax)
+; NOCMOV-NEXT:   movl  %ecx, (%eax)
 ; NOCMOV-NEXT:   popl  %esi
 ; NOCMOV-NEXT:   popl  %edi
 ; NOCMOV-NEXT:   retl  $4
diff --git a/test/CodeGen/X86/cmpxchg-clobber-flags.ll b/test/CodeGen/X86/cmpxchg-clobber-flags.ll
index 40fe4da742e..61123930887 100644
--- a/test/CodeGen/X86/cmpxchg-clobber-flags.ll
+++ b/test/CodeGen/X86/cmpxchg-clobber-flags.ll
@@ -2,10 +2,6 @@
 ; RUN: llc -verify-machineinstrs -mtriple=i386-linux-gnu -pre-RA-sched=fast %s -o - | FileCheck %s
 ; RUN: llc -verify-machineinstrs -mtriple=x86_64-linux-gnu %s -o - | FileCheck %s
 ; RUN: llc -verify-machineinstrs -mtriple=x86_64-linux-gnu -pre-RA-sched=fast %s -o - | FileCheck %s
-; RUN: llc -mtriple=i386-unknown-nacl %s -o - | FileCheck %s -check-prefix=NACL
-; RUN: llc -mtriple=i386-unknown-nacl -pre-RA-sched=fast %s -o - | FileCheck %s -check-prefix=NACL
-; RUN: llc -mtriple=x86_64-unknown-nacl %s -o - | FileCheck %s -check-prefix=NACL
-; RUN: llc -mtriple=x86_64-unknown-nacl -pre-RA-sched=fast %s -o - | FileCheck %s -check-prefix=NACL
 
 declare i32 @bar()
 
@@ -20,23 +16,6 @@ define i64 @test_intervening_call(i64* %foo, i64 %bar, i64 %baz) {
 ; CHECK-NEXT: push[[LQ]] [[FLAGS]]
 ; CHECK-NEXT: popf[[LQ]]
 ; CHECK-NEXT: jne
-
-
-; NACL-LABEL: test_intervening_call:
-; NACL: cmpxchg
-; NACL: push[[LQ:[lq]]] [[AX:%.*]]
-; NACL-NEXT: lahf
-; NACL-NEXT: mov[[LQ]] [[AX]], [[FLAGS:%.*]]
-; NACL-NEXT: pop[[LQ]] [[AX]]
-
-; NACL-NEXT: call[[LQ]] bar
-
-; NACL-NEXT: push[[LQ]] [[AX]]
-; NACL-NEXT: mov[[LQ]] [[FLAGS]], [[AX]]
-; NACL-NEXT: sahf
-; NACL-NEXT: pop[[LQ]] [[AX]]
-; NACL-NEXT: jne
-
   %cx = cmpxchg i64* %foo, i64 %bar, i64 %baz seq_cst seq_cst
   %p = extractvalue { i64, i1 } %cx, 1
   call i32 @bar()
@@ -52,13 +31,9 @@ f:
 ; Interesting in producing a clobber without any function calls.
 define i32 @test_control_flow(i32* %p, i32 %i, i32 %j) {
 ; CHECK-LABEL: test_control_flow:
+
 ; CHECK: cmpxchg
 ; CHECK-NEXT: jne
-
-; NACL-LABEL: test_control_flow:
-; NACL: cmpxchg
-; NACL-NEXT: jne
-
 entry:
   %cmp = icmp sgt i32 %i, %j
   br i1 %cmp, label %loop_start, label %cond.end
@@ -93,6 +68,7 @@ cond.end:
 ; operand. Naive attempts to limit cmpxchg EFLAGS use are likely to fail here.
 define i32 @test_feed_cmov(i32* %addr, i32 %desired, i32 %new) {
 ; CHECK-LABEL: test_feed_cmov:
+
 ; CHECK: cmpxchg
 ; CHECK: pushf[[LQ:[lq]]]
 ; CHECK-NEXT: pop[[LQ]] [[FLAGS:%.*]]
@@ -101,21 +77,6 @@ define i32 @test_feed_cmov(i32* %addr, i32 %desired, i32 %new) {
 
 ; CHECK-NEXT: push[[LQ]] [[FLAGS]]
 ; CHECK-NEXT: popf[[LQ]]
-
-
-; NACL-LABEL: test_feed_cmov:
-; NACL: cmpxchg
-; NACL: push[[LQ:[lq]]] [[AX:%.*]]
-; NACL-NEXT: lahf
-; NACL-NEXT: mov[[LQ]] [[AX]], [[FLAGS:%.*]]
-; NACL-NEXT: pop[[LQ]] [[AX]]
-
-; NACL-NEXT: call[[LQ]] bar
-
-; NACL-NEXT: push[[LQ]] [[AX]]
-; NACL-NEXT: mov[[LQ]] [[FLAGS]], [[AX]]
-; NACL-NEXT: sahf
-; NACL-NEXT: pop[[LQ]] [[AX]]
   %res = cmpxchg i32* %addr, i32 %desired, i32 %new seq_cst seq_cst
   %success = extractvalue { i32, i1 } %res, 1
 
diff --git a/test/CodeGen/X86/cmpxchg16b.ll b/test/CodeGen/X86/cmpxchg16b.ll
index 1d5bb85f8d2..d514c0a35f5 100644
--- a/test/CodeGen/X86/cmpxchg16b.ll
+++ b/test/CodeGen/X86/cmpxchg16b.ll
@@ -4,8 +4,7 @@
 define void @t1(i128* nocapture %p) nounwind ssp {
 entry:
 ; CHECK: movl	$1, %ebx
-; CHECK: lock
-; CHECK-NEXT: cmpxchg16b
+; CHECK: lock cmpxchg16b
   %r = cmpxchg i128* %p, i128 0, i128 1 seq_cst seq_cst
   ret void
 }
diff --git a/test/CodeGen/X86/code_placement_eh.ll b/test/CodeGen/X86/code_placement_eh.ll
index 2da3f9f53ef..62fddffffc4 100644
--- a/test/CodeGen/X86/code_placement_eh.ll
+++ b/test/CodeGen/X86/code_placement_eh.ll
@@ -6,7 +6,7 @@
 target datalayout = "e-p:32:32:32-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-f32:32:32-f64:32:64-v64:64:64-v128:128:128-a0:0:64-f80:128:128-n8:16:32"
 target triple = "i386-apple-darwin10.0"
 
-define void @foo() {
+define void @foo() personality i32 (...)* @__gxx_personality_v0 {
 invcont5:
   br label %bb15
 
@@ -22,12 +22,12 @@ bb18.i5.i:                                        ; preds = %.noexc6.i.i, %bb51.
           to label %.noexc6.i.i unwind label %lpad.i.i ; <float> [#uses=0]
 
 lpad.i.i:                                         ; preds = %bb18.i5.i, %.noexc6.i.i
-  %lpadval.i.i = landingpad { i8*, i32 } personality i32 (...)* @__gxx_personality_v0
+  %lpadval.i.i = landingpad { i8*, i32 }
           catch i8* null
   unreachable
 
 lpad59.i:                                         ; preds = %bb15
-  %lpadval60.i.i = landingpad { i8*, i32 } personality i32 (...)* @__gxx_personality_v0
+  %lpadval60.i.i = landingpad { i8*, i32 }
           catch i8* null
   unreachable
 
diff --git a/test/CodeGen/X86/codegen-prepare-extload.ll b/test/CodeGen/X86/codegen-prepare-extload.ll
index 65502b312b0..c5c761ee63e 100644
--- a/test/CodeGen/X86/codegen-prepare-extload.ll
+++ b/test/CodeGen/X86/codegen-prepare-extload.ll
@@ -30,7 +30,7 @@ false:
 }
 
 ; Check that we manage to form a zextload is an operation with only one
-; argument to explicitly extend is in the the way.
+; argument to explicitly extend is in the way.
 ; OPTALL-LABEL: @promoteOneArg
 ; OPTALL: [[LD:%[a-zA-Z_0-9-]+]] = load i8, i8* %p
 ; OPT-NEXT: [[ZEXT:%[a-zA-Z_0-9-]+]] = zext i8 [[LD]] to i32
@@ -55,7 +55,7 @@ false:
 }
 
 ; Check that we manage to form a sextload is an operation with only one
-; argument to explicitly extend is in the the way.
+; argument to explicitly extend is in the way.
 ; Version with sext.
 ; OPTALL-LABEL: @promoteOneArgSExt
 ; OPTALL: [[LD:%[a-zA-Z_0-9-]+]] = load i8, i8* %p
@@ -80,7 +80,7 @@ false:
 }
 
 ; Check that we manage to form a zextload is an operation with two
-; arguments to explicitly extend is in the the way.
+; arguments to explicitly extend is in the way.
 ; Extending %add will create two extensions:
 ; 1. One for %b.
 ; 2. One for %t.
@@ -119,7 +119,7 @@ false:
 }
 
 ; Check that we manage to form a sextload is an operation with two
-; arguments to explicitly extend is in the the way.
+; arguments to explicitly extend is in the way.
 ; Version with sext.
 ; OPTALL-LABEL: @promoteTwoArgSExt
 ; OPTALL: [[LD:%[a-zA-Z_0-9-]+]] = load i8, i8* %p
diff --git a/test/CodeGen/X86/coff-weak.ll b/test/CodeGen/X86/coff-weak.ll
new file mode 100644
index 00000000000..369750147f2
--- /dev/null
+++ b/test/CodeGen/X86/coff-weak.ll
@@ -0,0 +1,9 @@
+; RUN: llc -function-sections -o - %s | FileCheck %s
+
+target datalayout = "e-m:w-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-pc-windows-msvc"
+
+; CHECK: .section{{.*}}one_only
+define linkonce_odr void @foo() {
+  ret void
+}
diff --git a/test/CodeGen/X86/combine-or.ll b/test/CodeGen/X86/combine-or.ll
index 8a0ffc128e5..970f1762c1b 100644
--- a/test/CodeGen/X86/combine-or.ll
+++ b/test/CodeGen/X86/combine-or.ll
@@ -277,6 +277,70 @@ define <2 x i64> @test21(<2 x i64> %a, <2 x i64> %b) {
   ret <2 x i64> %or
 }
 
+
+; Verify that the dag-combiner keeps the correct domain for float/double vectors
+; bitcast to use the mask-or blend combine.
+
+define <2 x double> @test22(<2 x double> %a0, <2 x double> %a1) {
+; CHECK-LABEL: test22:
+; CHECK:       # BB#0:
+; CHECK-NEXT:    blendpd {{.*#+}} xmm0 = xmm1[0],xmm0[1]
+; CHECK-NEXT:    retq
+  %bc1 = bitcast <2 x double> %a0 to <2 x i64>
+  %bc2 = bitcast <2 x double> %a1 to <2 x i64>
+  %and1 = and <2 x i64> %bc1, <i64 0, i64 -1>
+  %and2 = and <2 x i64> %bc2, <i64 -1, i64 0>
+  %or = or <2 x i64> %and1, %and2
+  %bc3 = bitcast <2 x i64> %or to <2 x double>
+  ret <2 x double> %bc3
+}
+
+
+define <4 x float> @test23(<4 x float> %a0, <4 x float> %a1) {
+; CHECK-LABEL: test23:
+; CHECK:       # BB#0:
+; CHECK-NEXT:    blendps {{.*#+}} xmm0 = xmm1[0],xmm0[1,2],xmm1[3]
+; CHECK-NEXT:    retq
+  %bc1 = bitcast <4 x float> %a0 to <4 x i32>
+  %bc2 = bitcast <4 x float> %a1 to <4 x i32>
+  %and1 = and <4 x i32> %bc1, <i32 0, i32 -1, i32 -1, i32 0>
+  %and2 = and <4 x i32> %bc2, <i32 -1, i32 0, i32 0, i32 -1>
+  %or = or <4 x i32> %and1, %and2
+  %bc3 = bitcast <4 x i32> %or to <4 x float>
+  ret <4 x float> %bc3
+}
+
+
+define <4 x float> @test24(<4 x float> %a0, <4 x float> %a1) {
+; CHECK-LABEL: test24:
+; CHECK:       # BB#0:
+; CHECK-NEXT:    blendpd {{.*#+}} xmm0 = xmm1[0],xmm0[1]
+; CHECK-NEXT:    retq
+  %bc1 = bitcast <4 x float> %a0 to <2 x i64>
+  %bc2 = bitcast <4 x float> %a1 to <2 x i64>
+  %and1 = and <2 x i64> %bc1, <i64 0, i64 -1>
+  %and2 = and <2 x i64> %bc2, <i64 -1, i64 0>
+  %or = or <2 x i64> %and1, %and2
+  %bc3 = bitcast <2 x i64> %or to <4 x float>
+  ret <4 x float> %bc3
+}
+
+
+define <4 x float> @test25(<4 x float> %a0) {
+; CHECK-LABEL: test25:
+; CHECK:       # BB#0:
+; CHECK-NEXT:    blendps {{.*#+}} xmm0 = mem[0],xmm0[1,2],mem[3]
+; CHECK-NEXT:    retq
+  %bc1 = bitcast <4 x float> %a0 to <4 x i32>
+  %bc2 = bitcast <4 x float> <float 1.0, float 1.0, float 1.0, float 1.0> to <4 x i32>
+  %and1 = and <4 x i32> %bc1, <i32 0, i32 -1, i32 -1, i32 0>
+  %and2 = and <4 x i32> %bc2, <i32 -1, i32 0, i32 0, i32 -1>
+  %or = or <4 x i32> %and1, %and2
+  %bc3 = bitcast <4 x i32> %or to <4 x float>
+  ret <4 x float> %bc3
+}
+
+
 ; Verify that the DAGCombiner doesn't crash in the attempt to check if a shuffle
 ; with illegal type has a legal mask. Method 'isShuffleMaskLegal' only knows how to
 ; handle legal vector value types.
diff --git a/test/CodeGen/X86/commute-two-addr.ll b/test/CodeGen/X86/commute-two-addr.ll
index 656c385e2bc..5b01e2f4e90 100644
--- a/test/CodeGen/X86/commute-two-addr.ll
+++ b/test/CodeGen/X86/commute-two-addr.ll
@@ -39,7 +39,7 @@ define %0 @t3(i32 %lb, i8 zeroext %has_lb, i8 zeroext %lb_inclusive, i32 %ub, i8
 entry:
 ; DARWIN-LABEL: t3:
 ; DARWIN: shlq $32, %rcx
-; DARWIN-NEXT: orq %rcx, %rax
+; DARWIN-NEXT: leaq (%rax,%rcx), %rax
 ; DARWIN-NEXT: shll $8
 ; DARWIN-NOT: leaq
   %tmp21 = zext i32 %lb to i64
diff --git a/test/CodeGen/X86/complex-fca.ll b/test/CodeGen/X86/complex-fca.ll
index 8ad38a4ee5c..78b27b7dc3f 100644
--- a/test/CodeGen/X86/complex-fca.ll
+++ b/test/CodeGen/X86/complex-fca.ll
@@ -1,17 +1,21 @@
-; RUN: llc < %s -march=x86 | grep mov | count 2
-
-; Skip this on Windows as there is no ccosl and sret behaves differently.
-; XFAIL: pc-win32
+; RUN: llc < %s -march=x86 | FileCheck %s
 
 define void @ccosl({ x86_fp80, x86_fp80 }* noalias sret %agg.result, { x86_fp80, x86_fp80 } %z) nounwind {
 entry:
-	%z8 = extractvalue { x86_fp80, x86_fp80 } %z, 0
-	%z9 = extractvalue { x86_fp80, x86_fp80 } %z, 1
-	%0 = fsub x86_fp80 0xK80000000000000000000, %z9
-	%insert = insertvalue { x86_fp80, x86_fp80 } undef, x86_fp80 %0, 0
-	%insert7 = insertvalue { x86_fp80, x86_fp80 } %insert, x86_fp80 %z8, 1
-	call void @ccoshl({ x86_fp80, x86_fp80 }* noalias sret %agg.result, { x86_fp80, x86_fp80 } %insert7) nounwind
-	ret void
+  %z8 = extractvalue { x86_fp80, x86_fp80 } %z, 0
+  %z9 = extractvalue { x86_fp80, x86_fp80 } %z, 1
+  %0 = fsub x86_fp80 0xK80000000000000000000, %z9
+  %insert = insertvalue { x86_fp80, x86_fp80 } undef, x86_fp80 %0, 0
+  %insert7 = insertvalue { x86_fp80, x86_fp80 } %insert, x86_fp80 %z8, 1
+  call void @ccoshl({ x86_fp80, x86_fp80 }* noalias sret %agg.result, { x86_fp80, x86_fp80 } %insert7) nounwind
+  ret void
 }
 
+; CHECK-LABEL: ccosl:
+; CHECK:         movl    {{[0-9]+}}(%esp), %[[sret_reg:[^ ]+]]
+; CHECK:         movl    %[[sret_reg]], (%esp)
+; CHECK:         calll   {{.*ccoshl.*}}
+; CHECK:         movl    %[[sret_reg]], %eax
+; CHECK:         retl
+
 declare void @ccoshl({ x86_fp80, x86_fp80 }* noalias sret, { x86_fp80, x86_fp80 }) nounwind
diff --git a/test/CodeGen/X86/copy-propagation.ll b/test/CodeGen/X86/copy-propagation.ll
new file mode 100644
index 00000000000..19421a06fa8
--- /dev/null
+++ b/test/CodeGen/X86/copy-propagation.ll
@@ -0,0 +1,44 @@
+; RUN: llc %s -mattr=+avx -o - | FileCheck %s
+; PR21743.
+
+target triple = "x86_64-pc-win32-elf"
+
+; Check that copy propagation conservatively assumes that undef register
+; can be rewritten by the backend to break false dependencies for the
+; hardware.
+; In this function we are in this situation:
+; reg1 = copy reg2
+; = inst reg2<undef>
+; reg2 = copy reg1
+; Copy propagation used to remove the last copy.
+; This is incorrect because the undef flag on reg2 in inst, allows next
+; passes to put whatever trashed value in reg2 that may help.
+; In practice we end up with this code:
+; reg1 = copy reg2
+; reg2 = 0
+; = inst reg2<undef>
+; reg2 = copy reg1
+; Therefore, removing the last copy is wrong.
+;
+; CHECK-LABEL: foo:
+; CHECK: movl	$339752784, %e[[INDIRECT_CALL1:[a-z]+]]
+; CHECK: callq *%r[[INDIRECT_CALL1]]
+; Copy the result in a temporary.
+; Note: Technically the regalloc could have been smarter and this move not required,
+; which would have hidden the bug.
+; CHECK-NEXT: vmovapd	%xmm0, [[TMP:%xmm[0-9]+]]
+; Crush xmm0.
+; CHECK-NEXT: vxorps %xmm0, %xmm0, %xmm0
+; CHECK: movl	$339772768, %e[[INDIRECT_CALL2:[a-z]+]]
+; Set TMP in the first argument of the second call.
+; CHECK-NEXT: vmovapd	[[TMP]], %xmm0
+; CHECK: callq *%r[[INDIRECT_CALL2]]
+; CHECK: retq
+define double @foo(i64 %arg) {
+top:
+  %tmp = call double inttoptr (i64 339752784 to double (double, double)*)(double 1.000000e+00, double 0.000000e+00)
+  %tmp1 = sitofp i64 %arg to double
+  call void inttoptr (i64 339772768 to void (double, double)*)(double %tmp, double %tmp1)
+  %tmp3 = fadd double %tmp1, %tmp
+  ret double %tmp3
+}
diff --git a/test/CodeGen/X86/cppeh-nounwind.ll b/test/CodeGen/X86/cppeh-nounwind.ll
new file mode 100644
index 00000000000..d9bc001a92d
--- /dev/null
+++ b/test/CodeGen/X86/cppeh-nounwind.ll
@@ -0,0 +1,35 @@
+; RUN: llc -mtriple=i686-pc-windows-msvc < %s | FileCheck %s
+
+; Sometimes invokes of nounwind functions make it through to CodeGen, especially
+; at -O0, where Clang sometimes optimistically annotates functions as nounwind.
+; WinEHPrepare ends up outlining functions, and emitting references to LSDA
+; labels. Make sure we emit the LSDA in that case.
+
+declare i32 @__CxxFrameHandler3(...)
+declare void @nounwind_func() nounwind
+declare void @cleanup()
+
+define void @should_emit_tables() personality i32 (...)* @__CxxFrameHandler3 {
+entry:
+  invoke void @nounwind_func()
+      to label %done unwind label %lpad
+
+done:
+  ret void
+
+lpad:
+  %vals = landingpad { i8*, i32 }
+      cleanup
+  call void @cleanup()
+  resume { i8*, i32 } %vals
+}
+
+; CHECK: _should_emit_tables:
+; CHECK: calll _nounwind_func
+; CHECK: retl
+
+; CHECK: L__ehtable$should_emit_tables:
+
+; CHECK: ___ehhandler$should_emit_tables:
+; CHECK: movl $L__ehtable$should_emit_tables, %eax
+; CHECK: jmp ___CxxFrameHandler3 # TAILCALL
diff --git a/test/CodeGen/X86/critical-anti-dep-breaker.ll b/test/CodeGen/X86/critical-anti-dep-breaker.ll
index 86afc1f245a..de5744d3a88 100644
--- a/test/CodeGen/X86/critical-anti-dep-breaker.ll
+++ b/test/CodeGen/X86/critical-anti-dep-breaker.ll
@@ -11,8 +11,7 @@
 @NullToken = external global i64
 
 ; CHECK-LABEL: Part_Create:
-; CHECK-DAG: # kill: RDI<def> 
-; CHECK-DAG: movq PartClass@GOTPCREL(%rip), %r10
+; CHECK: movq PartClass@GOTPCREL(%rip), %r10
 define i32 @Part_Create(i64* %Anchor, i32 %TypeNum, i32 %F, i32 %Z, i32* %Status, i64* %PartTkn) {
   %PartObj = alloca i64*, align 8
   %Vchunk = alloca i64, align 8
diff --git a/test/CodeGen/X86/cvt16.ll b/test/CodeGen/X86/cvt16.ll
index 9846da546e7..5ee399fc137 100644
--- a/test/CodeGen/X86/cvt16.ll
+++ b/test/CodeGen/X86/cvt16.ll
@@ -1,7 +1,7 @@
 ; RUN: llc < %s -march=x86-64 -mtriple=x86_64-unknown-linux-gnu -mcpu=corei7 -mattr=-f16c | FileCheck %s -check-prefix=CHECK -check-prefix=LIBCALL
 ; RUN: llc < %s -march=x86-64 -mtriple=x86_64-unknown-linux-gnu -mcpu=corei7 -mattr=+f16c | FileCheck %s -check-prefix=CHECK -check-prefix=F16C
-; RUN: llc < %s -march=x86-64 -mtriple=x86_64-unknown-linux-gnu -mcpu=corei7 -soft-float=1 -mattr=-f16c | FileCheck %s -check-prefix=CHECK -check-prefix=SOFTFLOAT
-; RUN: llc < %s -march=x86-64 -mtriple=x86_64-unknown-linux-gnu -mcpu=corei7 -soft-float=1 -mattr=+f16c | FileCheck %s -check-prefix=CHECK -check-prefix=SOFTFLOAT
+; RUN: llc < %s -march=x86-64 -mtriple=x86_64-unknown-linux-gnu -mcpu=corei7 -mattr=-f16c,+soft-float | FileCheck %s -check-prefix=CHECK -check-prefix=SOFTFLOAT
+; RUN: llc < %s -march=x86-64 -mtriple=x86_64-unknown-linux-gnu -mcpu=corei7 -mattr=+f16c,+soft-float | FileCheck %s -check-prefix=CHECK -check-prefix=SOFTFLOAT
 
 ; This is a test for float to half float conversions on x86-64.
 ;
diff --git a/test/CodeGen/X86/dbg-changes-codegen-branch-folding.ll b/test/CodeGen/X86/dbg-changes-codegen-branch-folding.ll
index c5085a2f2a5..20d0129c3e8 100644
--- a/test/CodeGen/X86/dbg-changes-codegen-branch-folding.ll
+++ b/test/CodeGen/X86/dbg-changes-codegen-branch-folding.ll
@@ -52,48 +52,48 @@ define void @_Z3barii(i32 %param1, i32 %param2) #0 {
 entry:
   %var1 = alloca %struct.AAA3, align 1
   %var2 = alloca %struct.AAA3, align 1
-  tail call void @llvm.dbg.value(metadata i32 %param1, i64 0, metadata !30, metadata !MDExpression()), !dbg !47
-  tail call void @llvm.dbg.value(metadata i32 %param2, i64 0, metadata !31, metadata !MDExpression()), !dbg !47
-  tail call void @llvm.dbg.value(metadata i8* null, i64 0, metadata !32, metadata !MDExpression()), !dbg !49
+  tail call void @llvm.dbg.value(metadata i32 %param1, i64 0, metadata !30, metadata !DIExpression()), !dbg !47
+  tail call void @llvm.dbg.value(metadata i32 %param2, i64 0, metadata !31, metadata !DIExpression()), !dbg !47
+  tail call void @llvm.dbg.value(metadata i8* null, i64 0, metadata !32, metadata !DIExpression()), !dbg !49
   %tobool = icmp eq i32 %param2, 0, !dbg !50
   br i1 %tobool, label %if.end, label %if.then, !dbg !50
 
 if.then:                                          ; preds = %entry
   %call = tail call i8* @_Z5i2stri(i32 %param2), !dbg !52
-  tail call void @llvm.dbg.value(metadata i8* %call, i64 0, metadata !32, metadata !MDExpression()), !dbg !49
+  tail call void @llvm.dbg.value(metadata i8* %call, i64 0, metadata !32, metadata !DIExpression()), !dbg !49
   br label %if.end, !dbg !54
 
 if.end:                                           ; preds = %entry, %if.then
-  tail call void @llvm.dbg.value(metadata %struct.AAA3* %var1, i64 0, metadata !33, metadata !MDExpression()), !dbg !55
-  tail call void @llvm.dbg.value(metadata %struct.AAA3* %var1, i64 0, metadata !56, metadata !MDExpression()), !dbg !57
-  tail call void @llvm.dbg.value(metadata !58, i64 0, metadata !59, metadata !MDExpression()), !dbg !60
+  tail call void @llvm.dbg.value(metadata %struct.AAA3* %var1, i64 0, metadata !33, metadata !DIExpression()), !dbg !55
+  tail call void @llvm.dbg.value(metadata %struct.AAA3* %var1, i64 0, metadata !56, metadata !DIExpression()), !dbg !57
+  tail call void @llvm.dbg.value(metadata !58, i64 0, metadata !59, metadata !DIExpression()), !dbg !60
   %arraydecay.i = getelementptr inbounds %struct.AAA3, %struct.AAA3* %var1, i64 0, i32 0, i64 0, !dbg !61
   call void @_Z3fooPcjPKc(i8* %arraydecay.i, i32 4, i8* getelementptr inbounds ([1 x i8], [1 x i8]* @.str, i64 0, i64 0)), !dbg !61
-  call void @llvm.dbg.value(metadata %struct.AAA3* %var2, i64 0, metadata !34, metadata !MDExpression()), !dbg !63
-  call void @llvm.dbg.value(metadata %struct.AAA3* %var2, i64 0, metadata !64, metadata !MDExpression()), !dbg !65
-  call void @llvm.dbg.value(metadata !58, i64 0, metadata !66, metadata !MDExpression()), !dbg !67
+  call void @llvm.dbg.value(metadata %struct.AAA3* %var2, i64 0, metadata !34, metadata !DIExpression()), !dbg !63
+  call void @llvm.dbg.value(metadata %struct.AAA3* %var2, i64 0, metadata !64, metadata !DIExpression()), !dbg !65
+  call void @llvm.dbg.value(metadata !58, i64 0, metadata !66, metadata !DIExpression()), !dbg !67
   %arraydecay.i5 = getelementptr inbounds %struct.AAA3, %struct.AAA3* %var2, i64 0, i32 0, i64 0, !dbg !68
   call void @_Z3fooPcjPKc(i8* %arraydecay.i5, i32 4, i8* getelementptr inbounds ([1 x i8], [1 x i8]* @.str, i64 0, i64 0)), !dbg !68
   %tobool1 = icmp eq i32 %param1, 0, !dbg !69
-  call void @llvm.dbg.value(metadata %struct.AAA3* %var2, i64 0, metadata !34, metadata !MDExpression()), !dbg !63
+  call void @llvm.dbg.value(metadata %struct.AAA3* %var2, i64 0, metadata !34, metadata !DIExpression()), !dbg !63
   br i1 %tobool1, label %if.else, label %if.then2, !dbg !69
 
 if.then2:                                         ; preds = %if.end
-  call void @llvm.dbg.value(metadata %struct.AAA3* %var2, i64 0, metadata !71, metadata !MDExpression()), !dbg !73
-  call void @llvm.dbg.value(metadata !74, i64 0, metadata !75, metadata !MDExpression()), !dbg !76
+  call void @llvm.dbg.value(metadata %struct.AAA3* %var2, i64 0, metadata !71, metadata !DIExpression()), !dbg !73
+  call void @llvm.dbg.value(metadata !74, i64 0, metadata !75, metadata !DIExpression()), !dbg !76
   call void @_Z3fooPcjPKc(i8* %arraydecay.i5, i32 4, i8* getelementptr inbounds ([2 x i8], [2 x i8]* @.str1, i64 0, i64 0)), !dbg !76
   br label %if.end3, !dbg !72
 
 if.else:                                          ; preds = %if.end
-  call void @llvm.dbg.value(metadata %struct.AAA3* %var2, i64 0, metadata !77, metadata !MDExpression()), !dbg !79
-  call void @llvm.dbg.value(metadata !80, i64 0, metadata !81, metadata !MDExpression()), !dbg !82
+  call void @llvm.dbg.value(metadata %struct.AAA3* %var2, i64 0, metadata !77, metadata !DIExpression()), !dbg !79
+  call void @llvm.dbg.value(metadata !80, i64 0, metadata !81, metadata !DIExpression()), !dbg !82
   call void @_Z3fooPcjPKc(i8* %arraydecay.i5, i32 4, i8* getelementptr inbounds ([2 x i8], [2 x i8]* @.str2, i64 0, i64 0)), !dbg !82
   br label %if.end3
 
 if.end3:                                          ; preds = %if.else, %if.then2
-  call void @llvm.dbg.value(metadata %struct.AAA3* %var1, i64 0, metadata !33, metadata !MDExpression()), !dbg !55
-  call void @llvm.dbg.value(metadata %struct.AAA3* %var1, i64 0, metadata !83, metadata !MDExpression()), !dbg !85
-  call void @llvm.dbg.value(metadata !58, i64 0, metadata !86, metadata !MDExpression()), !dbg !87
+  call void @llvm.dbg.value(metadata %struct.AAA3* %var1, i64 0, metadata !33, metadata !DIExpression()), !dbg !55
+  call void @llvm.dbg.value(metadata %struct.AAA3* %var1, i64 0, metadata !83, metadata !DIExpression()), !dbg !85
+  call void @llvm.dbg.value(metadata !58, i64 0, metadata !86, metadata !DIExpression()), !dbg !87
   call void @_Z3fooPcjPKc(i8* %arraydecay.i, i32 4, i8* getelementptr inbounds ([1 x i8], [1 x i8]* @.str, i64 0, i64 0)), !dbg !87
   ret void, !dbg !88
 }
@@ -113,92 +113,92 @@ attributes #2 = { nounwind readnone }
 !llvm.module.flags = !{!44, !45}
 !llvm.ident = !{!46}
 
-!0 = !MDCompileUnit(language: DW_LANG_C_plus_plus, producer: "clang version 3.5.0 ", isOptimized: true, emissionKind: 1, file: !1, enums: !2, retainedTypes: !3, subprograms: !23, globals: !2, imports: !2)
-!1 = !MDFile(filename: "dbg-changes-codegen-branch-folding.cpp", directory: "/tmp/dbginfo")
+!0 = !DICompileUnit(language: DW_LANG_C_plus_plus, producer: "clang version 3.5.0 ", isOptimized: true, emissionKind: 1, file: !1, enums: !2, retainedTypes: !3, subprograms: !23, globals: !2, imports: !2)
+!1 = !DIFile(filename: "dbg-changes-codegen-branch-folding.cpp", directory: "/tmp/dbginfo")
 !2 = !{}
 !3 = !{!4}
-!4 = !MDCompositeType(tag: DW_TAG_structure_type, name: "AAA3", line: 4, size: 32, align: 8, file: !1, elements: !5, identifier: "_ZTS4AAA3")
+!4 = !DICompositeType(tag: DW_TAG_structure_type, name: "AAA3", line: 4, size: 32, align: 8, file: !1, elements: !5, identifier: "_ZTS4AAA3")
 !5 = !{!6, !11, !17, !18}
-!6 = !MDDerivedType(tag: DW_TAG_member, name: "text", line: 8, size: 32, align: 8, file: !1, scope: !"_ZTS4AAA3", baseType: !7)
-!7 = !MDCompositeType(tag: DW_TAG_array_type, size: 32, align: 8, baseType: !8, elements: !9)
-!8 = !MDBasicType(tag: DW_TAG_base_type, name: "char", size: 8, align: 8, encoding: DW_ATE_signed_char)
+!6 = !DIDerivedType(tag: DW_TAG_member, name: "text", line: 8, size: 32, align: 8, file: !1, scope: !"_ZTS4AAA3", baseType: !7)
+!7 = !DICompositeType(tag: DW_TAG_array_type, size: 32, align: 8, baseType: !8, elements: !9)
+!8 = !DIBasicType(tag: DW_TAG_base_type, name: "char", size: 8, align: 8, encoding: DW_ATE_signed_char)
 !9 = !{!10}
-!10 = !MDSubrange(count: 4)
-!11 = !MDSubprogram(name: "AAA3", line: 5, isLocal: false, isDefinition: false, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: true, scopeLine: 5, file: !1, scope: !"_ZTS4AAA3", type: !12)
-!12 = !MDSubroutineType(types: !13)
+!10 = !DISubrange(count: 4)
+!11 = !DISubprogram(name: "AAA3", line: 5, isLocal: false, isDefinition: false, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: true, scopeLine: 5, file: !1, scope: !"_ZTS4AAA3", type: !12)
+!12 = !DISubroutineType(types: !13)
 !13 = !{null, !14, !15}
-!14 = !MDDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, flags: DIFlagArtificial | DIFlagObjectPointer, baseType: !"_ZTS4AAA3")
-!15 = !MDDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, baseType: !16)
-!16 = !MDDerivedType(tag: DW_TAG_const_type, baseType: !8)
-!17 = !MDSubprogram(name: "operator=", linkageName: "_ZN4AAA3aSEPKc", line: 6, isLocal: false, isDefinition: false, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: true, scopeLine: 6, file: !1, scope: !"_ZTS4AAA3", type: !12)
-!18 = !MDSubprogram(name: "operator const char *", linkageName: "_ZNK4AAA3cvPKcEv", line: 7, isLocal: false, isDefinition: false, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: true, scopeLine: 7, file: !1, scope: !"_ZTS4AAA3", type: !19)
-!19 = !MDSubroutineType(types: !20)
+!14 = !DIDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, flags: DIFlagArtificial | DIFlagObjectPointer, baseType: !"_ZTS4AAA3")
+!15 = !DIDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, baseType: !16)
+!16 = !DIDerivedType(tag: DW_TAG_const_type, baseType: !8)
+!17 = !DISubprogram(name: "operator=", linkageName: "_ZN4AAA3aSEPKc", line: 6, isLocal: false, isDefinition: false, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: true, scopeLine: 6, file: !1, scope: !"_ZTS4AAA3", type: !12)
+!18 = !DISubprogram(name: "operator const char *", linkageName: "_ZNK4AAA3cvPKcEv", line: 7, isLocal: false, isDefinition: false, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: true, scopeLine: 7, file: !1, scope: !"_ZTS4AAA3", type: !19)
+!19 = !DISubroutineType(types: !20)
 !20 = !{!15, !21}
-!21 = !MDDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, flags: DIFlagArtificial | DIFlagObjectPointer, baseType: !22)
-!22 = !MDDerivedType(tag: DW_TAG_const_type, baseType: !"_ZTS4AAA3")
+!21 = !DIDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, flags: DIFlagArtificial | DIFlagObjectPointer, baseType: !22)
+!22 = !DIDerivedType(tag: DW_TAG_const_type, baseType: !"_ZTS4AAA3")
 !23 = !{!24, !35, !40}
-!24 = !MDSubprogram(name: "bar", linkageName: "_Z3barii", line: 11, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: true, scopeLine: 11, file: !1, scope: !25, type: !26, function: void (i32, i32)* @_Z3barii, variables: !29)
-!25 = !MDFile(filename: "dbg-changes-codegen-branch-folding.cpp", directory: "/tmp/dbginfo")
-!26 = !MDSubroutineType(types: !27)
+!24 = !DISubprogram(name: "bar", linkageName: "_Z3barii", line: 11, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: true, scopeLine: 11, file: !1, scope: !25, type: !26, function: void (i32, i32)* @_Z3barii, variables: !29)
+!25 = !DIFile(filename: "dbg-changes-codegen-branch-folding.cpp", directory: "/tmp/dbginfo")
+!26 = !DISubroutineType(types: !27)
 !27 = !{null, !28, !28}
-!28 = !MDBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
+!28 = !DIBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
 !29 = !{!30, !31, !32, !33, !34}
-!30 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "param1", line: 11, arg: 1, scope: !24, file: !25, type: !28)
-!31 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "param2", line: 11, arg: 2, scope: !24, file: !25, type: !28)
-!32 = !MDLocalVariable(tag: DW_TAG_auto_variable, name: "temp", line: 12, scope: !24, file: !25, type: !15)
-!33 = !MDLocalVariable(tag: DW_TAG_auto_variable, name: "var1", line: 17, scope: !24, file: !25, type: !"_ZTS4AAA3")
-!34 = !MDLocalVariable(tag: DW_TAG_auto_variable, name: "var2", line: 18, scope: !24, file: !25, type: !"_ZTS4AAA3")
-!35 = !MDSubprogram(name: "operator=", linkageName: "_ZN4AAA3aSEPKc", line: 6, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: true, scopeLine: 6, file: !1, scope: !"_ZTS4AAA3", type: !12, declaration: !17, variables: !36)
+!30 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "param1", line: 11, arg: 1, scope: !24, file: !25, type: !28)
+!31 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "param2", line: 11, arg: 2, scope: !24, file: !25, type: !28)
+!32 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "temp", line: 12, scope: !24, file: !25, type: !15)
+!33 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "var1", line: 17, scope: !24, file: !25, type: !"_ZTS4AAA3")
+!34 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "var2", line: 18, scope: !24, file: !25, type: !"_ZTS4AAA3")
+!35 = !DISubprogram(name: "operator=", linkageName: "_ZN4AAA3aSEPKc", line: 6, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: true, scopeLine: 6, file: !1, scope: !"_ZTS4AAA3", type: !12, declaration: !17, variables: !36)
 !36 = !{!37, !39}
-!37 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "this", arg: 1, flags: DIFlagArtificial | DIFlagObjectPointer, scope: !35, type: !38)
-!38 = !MDDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, baseType: !"_ZTS4AAA3")
-!39 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "value", line: 6, arg: 2, scope: !35, file: !25, type: !15)
-!40 = !MDSubprogram(name: "AAA3", linkageName: "_ZN4AAA3C2EPKc", line: 5, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: true, scopeLine: 5, file: !1, scope: !"_ZTS4AAA3", type: !12, declaration: !11, variables: !41)
+!37 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "this", arg: 1, flags: DIFlagArtificial | DIFlagObjectPointer, scope: !35, type: !38)
+!38 = !DIDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, baseType: !"_ZTS4AAA3")
+!39 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "value", line: 6, arg: 2, scope: !35, file: !25, type: !15)
+!40 = !DISubprogram(name: "AAA3", linkageName: "_ZN4AAA3C2EPKc", line: 5, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: true, scopeLine: 5, file: !1, scope: !"_ZTS4AAA3", type: !12, declaration: !11, variables: !41)
 !41 = !{!42, !43}
-!42 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "this", arg: 1, flags: DIFlagArtificial | DIFlagObjectPointer, scope: !40, type: !38)
-!43 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "value", line: 5, arg: 2, scope: !40, file: !25, type: !15)
+!42 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "this", arg: 1, flags: DIFlagArtificial | DIFlagObjectPointer, scope: !40, type: !38)
+!43 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "value", line: 5, arg: 2, scope: !40, file: !25, type: !15)
 !44 = !{i32 2, !"Dwarf Version", i32 4}
 !45 = !{i32 2, !"Debug Info Version", i32 3}
 !46 = !{!"clang version 3.5.0 "}
-!47 = !MDLocation(line: 11, scope: !24)
+!47 = !DILocation(line: 11, scope: !24)
 !48 = !{i8* null}
-!49 = !MDLocation(line: 12, scope: !24)
-!50 = !MDLocation(line: 14, scope: !51)
-!51 = distinct !MDLexicalBlock(line: 14, column: 0, file: !1, scope: !24)
-!52 = !MDLocation(line: 15, scope: !53)
-!53 = distinct !MDLexicalBlock(line: 14, column: 0, file: !1, scope: !51)
-!54 = !MDLocation(line: 16, scope: !53)
-!55 = !MDLocation(line: 17, scope: !24)
-!56 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "this", arg: 1, flags: DIFlagArtificial | DIFlagObjectPointer, scope: !40, type: !38)
-!57 = !MDLocation(line: 0, scope: !40, inlinedAt: !55)
+!49 = !DILocation(line: 12, scope: !24)
+!50 = !DILocation(line: 14, scope: !51)
+!51 = distinct !DILexicalBlock(line: 14, column: 0, file: !1, scope: !24)
+!52 = !DILocation(line: 15, scope: !53)
+!53 = distinct !DILexicalBlock(line: 14, column: 0, file: !1, scope: !51)
+!54 = !DILocation(line: 16, scope: !53)
+!55 = !DILocation(line: 17, scope: !24)
+!56 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "this", arg: 1, flags: DIFlagArtificial | DIFlagObjectPointer, scope: !40, type: !38)
+!57 = !DILocation(line: 0, scope: !40, inlinedAt: !55)
 !58 = !{i8* getelementptr inbounds ([1 x i8], [1 x i8]* @.str, i64 0, i64 0)}
-!59 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "value", line: 5, arg: 2, scope: !40, file: !25, type: !15)
-!60 = !MDLocation(line: 5, scope: !40, inlinedAt: !55)
-!61 = !MDLocation(line: 5, scope: !62, inlinedAt: !55)
-!62 = distinct !MDLexicalBlock(line: 5, column: 0, file: !1, scope: !40)
-!63 = !MDLocation(line: 18, scope: !24)
-!64 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "this", arg: 1, flags: DIFlagArtificial | DIFlagObjectPointer, scope: !40, type: !38)
-!65 = !MDLocation(line: 0, scope: !40, inlinedAt: !63)
-!66 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "value", line: 5, arg: 2, scope: !40, file: !25, type: !15)
-!67 = !MDLocation(line: 5, scope: !40, inlinedAt: !63)
-!68 = !MDLocation(line: 5, scope: !62, inlinedAt: !63)
-!69 = !MDLocation(line: 20, scope: !70)
-!70 = distinct !MDLexicalBlock(line: 20, column: 0, file: !1, scope: !24)
-!71 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "this", arg: 1, flags: DIFlagArtificial | DIFlagObjectPointer, scope: !35, type: !38)
-!72 = !MDLocation(line: 21, scope: !70)
-!73 = !MDLocation(line: 0, scope: !35, inlinedAt: !72)
+!59 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "value", line: 5, arg: 2, scope: !40, file: !25, type: !15)
+!60 = !DILocation(line: 5, scope: !40, inlinedAt: !55)
+!61 = !DILocation(line: 5, scope: !62, inlinedAt: !55)
+!62 = distinct !DILexicalBlock(line: 5, column: 0, file: !1, scope: !40)
+!63 = !DILocation(line: 18, scope: !24)
+!64 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "this", arg: 1, flags: DIFlagArtificial | DIFlagObjectPointer, scope: !40, type: !38)
+!65 = !DILocation(line: 0, scope: !40, inlinedAt: !63)
+!66 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "value", line: 5, arg: 2, scope: !40, file: !25, type: !15)
+!67 = !DILocation(line: 5, scope: !40, inlinedAt: !63)
+!68 = !DILocation(line: 5, scope: !62, inlinedAt: !63)
+!69 = !DILocation(line: 20, scope: !70)
+!70 = distinct !DILexicalBlock(line: 20, column: 0, file: !1, scope: !24)
+!71 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "this", arg: 1, flags: DIFlagArtificial | DIFlagObjectPointer, scope: !35, type: !38)
+!72 = !DILocation(line: 21, scope: !70)
+!73 = !DILocation(line: 0, scope: !35, inlinedAt: !72)
 !74 = !{i8* getelementptr inbounds ([2 x i8], [2 x i8]* @.str1, i64 0, i64 0)}
-!75 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "value", line: 6, arg: 2, scope: !35, file: !25, type: !15)
-!76 = !MDLocation(line: 6, scope: !35, inlinedAt: !72)
-!77 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "this", arg: 1, flags: DIFlagArtificial | DIFlagObjectPointer, scope: !35, type: !38)
-!78 = !MDLocation(line: 23, scope: !70)
-!79 = !MDLocation(line: 0, scope: !35, inlinedAt: !78)
+!75 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "value", line: 6, arg: 2, scope: !35, file: !25, type: !15)
+!76 = !DILocation(line: 6, scope: !35, inlinedAt: !72)
+!77 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "this", arg: 1, flags: DIFlagArtificial | DIFlagObjectPointer, scope: !35, type: !38)
+!78 = !DILocation(line: 23, scope: !70)
+!79 = !DILocation(line: 0, scope: !35, inlinedAt: !78)
 !80 = !{i8* getelementptr inbounds ([2 x i8], [2 x i8]* @.str2, i64 0, i64 0)}
-!81 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "value", line: 6, arg: 2, scope: !35, file: !25, type: !15)
-!82 = !MDLocation(line: 6, scope: !35, inlinedAt: !78)
-!83 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "this", arg: 1, flags: DIFlagArtificial | DIFlagObjectPointer, scope: !35, type: !38)
-!84 = !MDLocation(line: 24, scope: !24)
-!85 = !MDLocation(line: 0, scope: !35, inlinedAt: !84)
-!86 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "value", line: 6, arg: 2, scope: !35, file: !25, type: !15)
-!87 = !MDLocation(line: 6, scope: !35, inlinedAt: !84)
-!88 = !MDLocation(line: 25, scope: !24)
+!81 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "value", line: 6, arg: 2, scope: !35, file: !25, type: !15)
+!82 = !DILocation(line: 6, scope: !35, inlinedAt: !78)
+!83 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "this", arg: 1, flags: DIFlagArtificial | DIFlagObjectPointer, scope: !35, type: !38)
+!84 = !DILocation(line: 24, scope: !24)
+!85 = !DILocation(line: 0, scope: !35, inlinedAt: !84)
+!86 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "value", line: 6, arg: 2, scope: !35, file: !25, type: !15)
+!87 = !DILocation(line: 6, scope: !35, inlinedAt: !84)
+!88 = !DILocation(line: 25, scope: !24)
diff --git a/test/CodeGen/X86/dbg-changes-codegen.ll b/test/CodeGen/X86/dbg-changes-codegen.ll
index 8f95338ccb5..b15e4bd4bf2 100644
--- a/test/CodeGen/X86/dbg-changes-codegen.ll
+++ b/test/CodeGen/X86/dbg-changes-codegen.ll
@@ -44,7 +44,7 @@
 define zeroext i1 @_ZN3Foo3batEv(%struct.Foo* %this) #0 align 2 {
 entry:
   %0 = load %struct.Foo*, %struct.Foo** @pfoo, align 8
-  tail call void @llvm.dbg.value(metadata %struct.Foo* %0, i64 0, metadata !62, metadata !MDExpression()), !dbg !MDLocation(scope: !MDSubprogram())
+  tail call void @llvm.dbg.value(metadata %struct.Foo* %0, i64 0, metadata !62, metadata !DIExpression()), !dbg !DILocation(scope: !DISubprogram())
   %cmp.i = icmp eq %struct.Foo* %0, %this
   ret i1 %cmp.i
 }
@@ -53,7 +53,7 @@ entry:
 define void @_Z3bazv() #1 {
 entry:
   %0 = load %struct.Wibble*, %struct.Wibble** @wibble1, align 8
-  tail call void @llvm.dbg.value(metadata %struct.Flibble* undef, i64 0, metadata !65, metadata !MDExpression()), !dbg !MDLocation(scope: !MDSubprogram())
+  tail call void @llvm.dbg.value(metadata %struct.Flibble* undef, i64 0, metadata !65, metadata !DIExpression()), !dbg !DILocation(scope: !DISubprogram())
   %1 = load %struct.Wibble*, %struct.Wibble** @wibble2, align 8
   %cmp.i = icmp ugt %struct.Wibble* %1, %0
   br i1 %cmp.i, label %if.then.i, label %_ZN7Flibble3barEP6Wibble.exit
@@ -76,8 +76,8 @@ attributes #1 = { nounwind uwtable "less-precise-fpmad"="false" "no-frame-pointe
 attributes #2 = { nounwind readnone }
 
 
-!17 = !MDDerivedType(tag: DW_TAG_reference_type, baseType: null)
-!45 = !MDDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, baseType: null)
-!62 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "arg", line: 4, arg: 2, scope: !MDSubprogram(), type: !17)
+!17 = !DIDerivedType(tag: DW_TAG_reference_type, baseType: null)
+!45 = !DIDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, baseType: null)
+!62 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "arg", line: 4, arg: 2, scope: !DISubprogram(), type: !17)
 !64 = !{%struct.Flibble* undef}
-!65 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "this", line: 13, arg: 1, flags: DIFlagArtificial | DIFlagObjectPointer, scope: !MDSubprogram(), type: !45)
+!65 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "this", line: 13, arg: 1, flags: DIFlagArtificial | DIFlagObjectPointer, scope: !DISubprogram(), type: !45)
diff --git a/test/CodeGen/X86/dbg-combine.ll b/test/CodeGen/X86/dbg-combine.ll
index b7932acd1af..5eb2ea9df51 100644
--- a/test/CodeGen/X86/dbg-combine.ll
+++ b/test/CodeGen/X86/dbg-combine.ll
@@ -74,40 +74,40 @@ attributes #2 = { nounwind }
 !llvm.module.flags = !{!9, !10}
 !llvm.ident = !{!11}
 
-!0 = !MDCompileUnit(language: DW_LANG_C99, producer: "clang version 3.7.0 (trunk 227074)", isOptimized: false, emissionKind: 1, file: !1, enums: !2, retainedTypes: !2, subprograms: !3, globals: !2, imports: !2)
-!1 = !MDFile(filename: "dbg-combine.c", directory: "/home/probinson/projects/scratch")
+!0 = !DICompileUnit(language: DW_LANG_C99, producer: "clang version 3.7.0 (trunk 227074)", isOptimized: false, emissionKind: 1, file: !1, enums: !2, retainedTypes: !2, subprograms: !3, globals: !2, imports: !2)
+!1 = !DIFile(filename: "dbg-combine.c", directory: "/home/probinson/projects/scratch")
 !2 = !{}
 !3 = !{!4}
-!4 = !MDSubprogram(name: "foo", line: 1, isLocal: false, isDefinition: true, isOptimized: false, scopeLine: 2, file: !1, scope: !5, type: !6, function: i32 ()* @foo, variables: !2)
-!5 = !MDFile(filename: "dbg-combine.c", directory: "/home/probinson/projects/scratch")
-!6 = !MDSubroutineType(types: !7)
+!4 = !DISubprogram(name: "foo", line: 1, isLocal: false, isDefinition: true, isOptimized: false, scopeLine: 2, file: !1, scope: !5, type: !6, function: i32 ()* @foo, variables: !2)
+!5 = !DIFile(filename: "dbg-combine.c", directory: "/home/probinson/projects/scratch")
+!6 = !DISubroutineType(types: !7)
 !7 = !{!8}
-!8 = !MDBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
+!8 = !DIBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
 !9 = !{i32 2, !"Dwarf Version", i32 4}
 !10 = !{i32 2, !"Debug Info Version", i32 3}
 !11 = !{!"clang version 3.7.0 (trunk 227074)"}
-!12 = !MDLocalVariable(tag: DW_TAG_auto_variable, name: "elems", line: 3, scope: !4, file: !5, type: !8)
-!13 = !MDExpression()
-!14 = !MDLocation(line: 3, column: 8, scope: !4)
-!15 = !MDLocation(line: 4, column: 15, scope: !4)
-!16 = !MDLocation(line: 4, column: 4, scope: !4)
-!17 = !MDLocalVariable(tag: DW_TAG_auto_variable, name: "array1", line: 4, scope: !4, file: !5, type: !18)
-!18 = !MDCompositeType(tag: DW_TAG_array_type, align: 32, baseType: !8, elements: !19)
+!12 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "elems", line: 3, scope: !4, file: !5, type: !8)
+!13 = !DIExpression()
+!14 = !DILocation(line: 3, column: 8, scope: !4)
+!15 = !DILocation(line: 4, column: 15, scope: !4)
+!16 = !DILocation(line: 4, column: 4, scope: !4)
+!17 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "array1", line: 4, scope: !4, file: !5, type: !18)
+!18 = !DICompositeType(tag: DW_TAG_array_type, align: 32, baseType: !8, elements: !19)
 !19 = !{!20}
-!20 = !MDSubrange(count: -1)
-!21 = !MDExpression(DW_OP_deref)
-!22 = !MDLocation(line: 4, column: 8, scope: !4)
-!23 = !MDLocation(line: 5, column: 4, scope: !4)
-!24 = !MDLocation(line: 5, column: 13, scope: !4)
-!25 = !MDLocation(line: 6, column: 4, scope: !4)
-!26 = !MDLocation(line: 6, column: 13, scope: !4)
-!27 = !MDLocation(line: 7, column: 4, scope: !4)
-!28 = !MDLocation(line: 7, column: 13, scope: !4)
-!29 = !MDLocation(line: 8, column: 15, scope: !4)
-!30 = !MDLocation(line: 8, column: 4, scope: !4)
-!31 = !MDLocalVariable(tag: DW_TAG_auto_variable, name: "array2", line: 8, scope: !4, file: !5, type: !18)
-!32 = !MDLocation(line: 8, column: 8, scope: !4)
-!33 = !MDLocation(line: 9, column: 4, scope: !4)
-!34 = !MDLocation(line: 9, column: 13, scope: !4)
-!35 = !MDLocation(line: 10, column: 11, scope: !4)
-!36 = !MDLocation(line: 11, column: 1, scope: !4)
+!20 = !DISubrange(count: -1)
+!21 = !DIExpression(DW_OP_deref)
+!22 = !DILocation(line: 4, column: 8, scope: !4)
+!23 = !DILocation(line: 5, column: 4, scope: !4)
+!24 = !DILocation(line: 5, column: 13, scope: !4)
+!25 = !DILocation(line: 6, column: 4, scope: !4)
+!26 = !DILocation(line: 6, column: 13, scope: !4)
+!27 = !DILocation(line: 7, column: 4, scope: !4)
+!28 = !DILocation(line: 7, column: 13, scope: !4)
+!29 = !DILocation(line: 8, column: 15, scope: !4)
+!30 = !DILocation(line: 8, column: 4, scope: !4)
+!31 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "array2", line: 8, scope: !4, file: !5, type: !18)
+!32 = !DILocation(line: 8, column: 8, scope: !4)
+!33 = !DILocation(line: 9, column: 4, scope: !4)
+!34 = !DILocation(line: 9, column: 13, scope: !4)
+!35 = !DILocation(line: 10, column: 11, scope: !4)
+!36 = !DILocation(line: 11, column: 1, scope: !4)
diff --git a/test/CodeGen/X86/disable-tail-calls.ll b/test/CodeGen/X86/disable-tail-calls.ll
new file mode 100644
index 00000000000..80e8fd74e92
--- /dev/null
+++ b/test/CodeGen/X86/disable-tail-calls.ll
@@ -0,0 +1,40 @@
+; RUN: llc < %s -march x86-64 | FileCheck %s --check-prefix=NO-OPTION
+; RUN: llc < %s -march x86-64 -disable-tail-calls | FileCheck %s --check-prefix=DISABLE-TRUE
+; RUN: llc < %s -march x86-64 -disable-tail-calls=false | FileCheck %s --check-prefix=DISABLE-FALSE
+
+; Check that command line option "-disable-tail-calls" overrides function
+; attribute "disable-tail-calls".
+
+; NO-OPTION-LABEL: {{\_?}}func_attr
+; NO-OPTION: callq {{\_?}}callee
+
+; DISABLE-FALSE-LABEL: {{\_?}}func_attr
+; DISABLE-FALSE: jmp {{\_?}}callee
+
+; DISABLE-TRUE-LABEL: {{\_?}}func_attr
+; DISABLE-TRUE: callq {{\_?}}callee
+
+define i32 @func_attr(i32 %a) #0 {
+entry:
+  %call = tail call i32 @callee(i32 %a)
+  ret i32 %call
+}
+
+; NO-OPTION-LABEL: {{\_?}}func_noattr
+; NO-OPTION: jmp {{\_?}}callee
+
+; DISABLE-FALSE-LABEL: {{\_?}}func_noattr
+; DISABLE-FALSE: jmp {{\_?}}callee
+
+; DISABLE-TRUE-LABEL: {{\_?}}func_noattr
+; DISABLE-TRUE: callq {{\_?}}callee
+
+define i32 @func_noattr(i32 %a) {
+entry:
+  %call = tail call i32 @callee(i32 %a)
+  ret i32 %call
+}
+
+declare i32 @callee(i32)
+
+attributes #0 = { "disable-tail-calls"="true" }
diff --git a/test/CodeGen/X86/dllexport-x86_64.ll b/test/CodeGen/X86/dllexport-x86_64.ll
index 629a5572977..bb5e92f98c7 100644
--- a/test/CodeGen/X86/dllexport-x86_64.ll
+++ b/test/CodeGen/X86/dllexport-x86_64.ll
@@ -71,33 +71,33 @@ define weak_odr dllexport void @weak1() {
 @blob_alias = dllexport alias bitcast ([6 x i8]* @blob to i32 ()*)
 
 ; CHECK: .section .drectve
-; WIN32: " /EXPORT:Var1,DATA"
-; WIN32: " /EXPORT:Var2,DATA"
-; WIN32: " /EXPORT:Var3,DATA"
-; WIN32: " /EXPORT:WeakVar1,DATA"
-; WIN32: " /EXPORT:WeakVar2,DATA"
-; WIN32: " /EXPORT:f1"
-; WIN32: " /EXPORT:f2"
-; WIN32: " /EXPORT:lnk1"
-; WIN32: " /EXPORT:lnk2"
-; WIN32: " /EXPORT:weak1"
-; WIN32: " /EXPORT:alias"
-; WIN32: " /EXPORT:alias2"
-; WIN32: " /EXPORT:alias3"
-; WIN32: " /EXPORT:weak_alias"
-; WIN32: " /EXPORT:blob_alias"
-; MINGW: " -export:Var1,data"
-; MINGW: " -export:Var2,data"
-; MINGW: " -export:Var3,data"
-; MINGW: " -export:WeakVar1,data"
-; MINGW: " -export:WeakVar2,data"
-; MINGW: " -export:f1"
-; MINGW: " -export:f2"
-; MINGW: " -export:lnk1"
-; MINGW: " -export:lnk2"
-; MINGW: " -export:weak1"
-; MINGW: " -export:alias"
-; MINGW: " -export:alias2"
-; MINGW: " -export:alias3"
-; MINGW: " -export:weak_alias"
-; MINGW: " -export:blob_alias"
+; WIN32: /EXPORT:f1
+; WIN32-SAME: /EXPORT:f2
+; WIN32-SAME: /EXPORT:lnk1
+; WIN32-SAME: /EXPORT:lnk2
+; WIN32-SAME: /EXPORT:weak1
+; WIN32-SAME: /EXPORT:Var1,DATA
+; WIN32-SAME: /EXPORT:Var2,DATA
+; WIN32-SAME: /EXPORT:Var3,DATA
+; WIN32-SAME: /EXPORT:WeakVar1,DATA
+; WIN32-SAME: /EXPORT:WeakVar2,DATA
+; WIN32-SAME: /EXPORT:alias
+; WIN32-SAME: /EXPORT:alias2
+; WIN32-SAME: /EXPORT:alias3
+; WIN32-SAME: /EXPORT:weak_alias
+; WIN32-SAME: /EXPORT:blob_alias
+; MINGW: -export:f1
+; MINGW-SAME: -export:f2
+; MINGW-SAME: -export:lnk1
+; MINGW-SAME: -export:lnk2
+; MINGW-SAME: -export:weak1
+; MINGW-SAME: -export:Var1,data
+; MINGW-SAME: -export:Var2,data
+; MINGW-SAME: -export:Var3,data
+; MINGW-SAME: -export:WeakVar1,data
+; MINGW-SAME: -export:WeakVar2,data
+; MINGW-SAME: -export:alias
+; MINGW-SAME: -export:alias2
+; MINGW-SAME: -export:alias3
+; MINGW-SAME: -export:weak_alias
+; MINGW-SAME: -export:blob_alias"
diff --git a/test/CodeGen/X86/dllexport.ll b/test/CodeGen/X86/dllexport.ll
index 02a83ae7b19..915567de5bf 100644
--- a/test/CodeGen/X86/dllexport.ll
+++ b/test/CodeGen/X86/dllexport.ll
@@ -89,40 +89,41 @@ define weak_odr dllexport void @weak1() {
 @weak_alias = weak_odr dllexport alias void()* @f1
 
 ; CHECK: .section .drectve
-; CHECK-CL: " /EXPORT:_Var1,DATA"
-; CHECK-CL: " /EXPORT:_Var2,DATA"
-; CHECK-CL: " /EXPORT:_Var3,DATA"
-; CHECK-CL: " /EXPORT:_WeakVar1,DATA"
-; CHECK-CL: " /EXPORT:_WeakVar2,DATA"
-; CHECK-CL: " /EXPORT:_f1"
-; CHECK-CL: " /EXPORT:_f2"
 ; CHECK-CL-NOT: not_exported
-; CHECK-CL: " /EXPORT:_stdfun@0"
-; CHECK-CL: " /EXPORT:@fastfun@0"
-; CHECK-CL: " /EXPORT:_thisfun"
-; CHECK-CL: " /EXPORT:_lnk1"
-; CHECK-CL: " /EXPORT:_lnk2"
-; CHECK-CL: " /EXPORT:_weak1"
-; CHECK-CL: " /EXPORT:_alias"
-; CHECK-CL: " /EXPORT:_alias2"
-; CHECK-CL: " /EXPORT:_alias3"
-; CHECK-CL: " /EXPORT:_weak_alias"
-; CHECK-GCC: " -export:Var1,data"
-; CHECK-GCC: " -export:Var2,data"
-; CHECK-GCC: " -export:Var3,data"
-; CHECK-GCC: " -export:WeakVar1,data"
-; CHECK-GCC: " -export:WeakVar2,data"
-; CHECK-GCC: " -export:f1"
-; CHECK-GCC: " -export:f2"
+; CHECK-CL: /EXPORT:_f1
+; CHECK-CL-SAME: /EXPORT:_f2
+; CHECK-CL-SAME: /EXPORT:_stdfun@0
+; CHECK-CL-SAME: /EXPORT:@fastfun@0
+; CHECK-CL-SAME: /EXPORT:_thisfun
+; CHECK-CL-SAME: /EXPORT:_lnk1
+; CHECK-CL-SAME: /EXPORT:_lnk2
+; CHECK-CL-SAME: /EXPORT:_weak1
+; CHECK-CL-SAME: /EXPORT:_Var1,DATA
+; CHECK-CL-SAME: /EXPORT:_Var2,DATA
+; CHECK-CL-SAME: /EXPORT:_Var3,DATA
+; CHECK-CL-SAME: /EXPORT:_WeakVar1,DATA
+; CHECK-CL-SAME: /EXPORT:_WeakVar2,DATA
+; CHECK-CL-SAME: /EXPORT:_alias
+; CHECK-CL-SAME: /EXPORT:_alias2
+; CHECK-CL-SAME: /EXPORT:_alias3
+; CHECK-CL-SAME: /EXPORT:_weak_alias"
 ; CHECK-CL-NOT: not_exported
-; CHECK-GCC: " -export:stdfun@0"
-; CHECK-GCC: " -export:@fastfun@0"
-; CHECK-GCC: " -export:thisfun"
-; CHECK-GCC: " -export:lnk1"
-; CHECK-GCC: " -export:lnk2"
-; CHECK-GCC: " -export:weak1"
-; CHECK-GCC: " -export:alias"
-; CHECK-GCC: " -export:alias2"
-; CHECK-GCC: " -export:alias3"
-; CHECK-GCC: " -export:weak_alias"
-
+; CHECK-GCC-NOT: not_exported
+; CHECK-GCC: -export:f1
+; CHECK-GCC-SAME: -export:f2
+; CHECK-GCC-SAME: -export:stdfun@0
+; CHECK-GCC-SAME: -export:@fastfun@0
+; CHECK-GCC-SAME: -export:thisfun
+; CHECK-GCC-SAME: -export:lnk1
+; CHECK-GCC-SAME: -export:lnk2
+; CHECK-GCC-SAME: -export:weak1
+; CHECK-GCC-SAME: -export:Var1,data
+; CHECK-GCC-SAME: -export:Var2,data
+; CHECK-GCC-SAME: -export:Var3,data
+; CHECK-GCC-SAME: -export:WeakVar1,data
+; CHECK-GCC-SAME: -export:WeakVar2,data
+; CHECK-GCC-SAME: -export:alias
+; CHECK-GCC-SAME: -export:alias2
+; CHECK-GCC-SAME: -export:alias3
+; CHECK-GCC-SAME: -export:weak_alias"
+; CHECK-GCC-NOT: not_exported
diff --git a/test/CodeGen/X86/dllimport.ll b/test/CodeGen/X86/dllimport.ll
index 9db654f2271..34faaeb6fed 100644
--- a/test/CodeGen/X86/dllimport.ll
+++ b/test/CodeGen/X86/dllimport.ll
@@ -57,3 +57,7 @@ define void @use() nounwind {
 
   ret void
 }
+
+; CHECK: _fp:
+; CHECK-NEXT: .long _fun
+@fp = constant void ()* @fun
diff --git a/test/CodeGen/X86/dwarf-comp-dir.ll b/test/CodeGen/X86/dwarf-comp-dir.ll
index fcab7f55ba4..27b8b1552ec 100644
--- a/test/CodeGen/X86/dwarf-comp-dir.ll
+++ b/test/CodeGen/X86/dwarf-comp-dir.ll
@@ -7,11 +7,11 @@ target triple = "x86_64-unknown-linux-gnu"
 !llvm.dbg.cu = !{!0}
 !llvm.module.flags = !{!5}
 
-!0 = !MDCompileUnit(language: DW_LANG_C99, producer: "clang version 3.1 (trunk 143523)", isOptimized: true, emissionKind: 0, file: !4, enums: !2, retainedTypes: !7, subprograms: !2, globals: !2)
+!0 = !DICompileUnit(language: DW_LANG_C99, producer: "clang version 3.1 (trunk 143523)", isOptimized: true, emissionKind: 0, file: !4, enums: !2, retainedTypes: !7, subprograms: !2, globals: !2)
 !2 = !{}
-!3 = !MDFile(filename: "empty.c", directory: "/home/nlewycky")
-!4 = !MDFile(filename: "empty.c", directory: "/home/nlewycky")
-!6 = !MDCompositeType(tag: DW_TAG_structure_type, name: "foo", line: 1, size: 8, align: 8, file: !4, elements: !2, identifier: "_ZTS3foo")
+!3 = !DIFile(filename: "empty.c", directory: "/home/nlewycky")
+!4 = !DIFile(filename: "empty.c", directory: "/home/nlewycky")
+!6 = !DICompositeType(tag: DW_TAG_structure_type, name: "foo", line: 1, size: 8, align: 8, file: !4, elements: !2, identifier: "_ZTS3foo")
 !7 = !{!6}
 
 ; The important part of the following check is that dir = #0.
diff --git a/test/CodeGen/X86/dwarf-eh-prepare.ll b/test/CodeGen/X86/dwarf-eh-prepare.ll
index 25572d868da..9acfaeb193e 100644
--- a/test/CodeGen/X86/dwarf-eh-prepare.ll
+++ b/test/CodeGen/X86/dwarf-eh-prepare.ll
@@ -9,7 +9,7 @@
 declare void @might_throw()
 declare void @cleanup()
 
-define i32 @simple_cleanup_catch() {
+define i32 @simple_cleanup_catch() personality i32 (...)* @__gxx_personality_v0 {
   invoke void @might_throw()
           to label %cont unwind label %lpad
 
@@ -22,7 +22,7 @@ cont:
 ; CHECK: ret i32 0
 
 lpad:
-  %ehvals = landingpad { i8*, i32 } personality i32 (...)* @__gxx_personality_v0
+  %ehvals = landingpad { i8*, i32 }
       cleanup
       catch i8* @int_typeinfo
   %ehptr = extractvalue { i8*, i32 } %ehvals, 0
@@ -33,7 +33,7 @@ lpad:
   br i1 %int_match, label %catch_int, label %eh.resume
 
 ; CHECK: lpad:
-; CHECK: landingpad { i8*, i32 } personality i32 (...)* @__gxx_personality_v0
+; CHECK: landingpad { i8*, i32 }
 ; CHECK: call void @cleanup()
 ; CHECK: call i32 @llvm.eh.typeid.for
 ; CHECK: br i1
@@ -54,7 +54,7 @@ eh.resume:
 }
 
 
-define i32 @catch_no_resume() {
+define i32 @catch_no_resume() personality i32 (...)* @__gxx_personality_v0 {
   invoke void @might_throw()
           to label %cont unwind label %lpad
 
@@ -62,7 +62,7 @@ cont:
   ret i32 0
 
 lpad:
-  %ehvals = landingpad { i8*, i32 } personality i32 (...)* @__gxx_personality_v0
+  %ehvals = landingpad { i8*, i32 }
       catch i8* @int_typeinfo
   %ehptr = extractvalue { i8*, i32 } %ehvals, 0
   %ehsel = extractvalue { i8*, i32 } %ehvals, 1
@@ -81,18 +81,18 @@ eh.resume:
 
 ; Check that we can prune the unreachable resume instruction.
 
-; CHECK-LABEL: define i32 @catch_no_resume() {
+; CHECK-LABEL: define i32 @catch_no_resume() personality i32 (...)* @__gxx_personality_v0 {
 ; CHECK: invoke void @might_throw()
 ; CHECK: ret i32 0
 ; CHECK: lpad:
-; CHECK: landingpad { i8*, i32 } personality i32 (...)* @__gxx_personality_v0
+; CHECK: landingpad { i8*, i32 }
 ; CHECK-NOT: br i1
 ; CHECK: ret i32 1
 ; CHECK-NOT: call void @_Unwind_Resume
 ; CHECK: {{^[}]}}
 
 
-define i32 @catch_cleanup_merge() {
+define i32 @catch_cleanup_merge() personality i32 (...)* @__gxx_personality_v0 {
   invoke void @might_throw()
           to label %inner_invoke unwind label %outer_lpad
 inner_invoke:
@@ -102,12 +102,12 @@ cont:
   ret i32 0
 
 outer_lpad:
-  %ehvals1 = landingpad { i8*, i32 } personality i32 (...)* @__gxx_personality_v0
+  %ehvals1 = landingpad { i8*, i32 }
       catch i8* @int_typeinfo
   br label %catch.dispatch
 
 inner_lpad:
-  %ehvals2 = landingpad { i8*, i32 } personality i32 (...)* @__gxx_personality_v0
+  %ehvals2 = landingpad { i8*, i32 }
       cleanup
       catch i8* @int_typeinfo
   call void @cleanup()
@@ -138,11 +138,11 @@ eh.resume:
 ; CHECK: ret i32 0
 ;
 ; CHECK: outer_lpad:
-; CHECK: landingpad { i8*, i32 } personality i32 (...)* @__gxx_personality_v0
+; CHECK: landingpad { i8*, i32 }
 ; CHECK: br label %catch.dispatch
 ;
 ; CHECK: inner_lpad:
-; CHECK: landingpad { i8*, i32 } personality i32 (...)* @__gxx_personality_v0
+; CHECK: landingpad { i8*, i32 }
 ; CHECK: call void @cleanup()
 ; CHECK: br label %catch.dispatch
 ;
diff --git a/test/CodeGen/X86/eh-label.ll b/test/CodeGen/X86/eh-label.ll
index aff0bcfffcf..d349174f95b 100644
--- a/test/CodeGen/X86/eh-label.ll
+++ b/test/CodeGen/X86/eh-label.ll
@@ -3,7 +3,7 @@
 
 declare void @g()
 
-define void @f() {
+define void @f() personality i8* bitcast (void ()* @g to i8*) {
 bb0:
   call void asm ".Lfunc_end0:", ""()
 ; CHECK: #APP
@@ -12,7 +12,7 @@ bb0:
 
   invoke void @g() to label %bb2 unwind label %bb1
 bb1:
-  landingpad { i8*, i32 } personality i8* bitcast (void ()* @g to i8*)
+  landingpad { i8*, i32 }
           catch i8* null
   call void @g()
   ret void
diff --git a/test/CodeGen/X86/eh-nolandingpads.ll b/test/CodeGen/X86/eh-nolandingpads.ll
new file mode 100644
index 00000000000..96295226621
--- /dev/null
+++ b/test/CodeGen/X86/eh-nolandingpads.ll
@@ -0,0 +1,12 @@
+; RUN: llc < %s -mtriple=x86_64-pc-linux | FileCheck %s
+; Test that we emit functions with explicitly specified personality,
+; even if no landing pads are left.
+
+declare i32 @__my_personality_v0(...)
+declare void @might_throw()
+
+define i32 @foo() personality i32 (...)* @__my_personality_v0 {
+; CHECK: .cfi_personality 3, __my_personality_v0
+    call void @might_throw()
+    ret i32 0
+}
diff --git a/test/CodeGen/X86/emutls-pic.ll b/test/CodeGen/X86/emutls-pic.ll
new file mode 100644
index 00000000000..7204c10f9b3
--- /dev/null
+++ b/test/CodeGen/X86/emutls-pic.ll
@@ -0,0 +1,168 @@
+; RUN: llc < %s -emulated-tls -march=x86 -mtriple=i386-linux-gnu -relocation-model=pic | FileCheck -check-prefix=X32 %s
+; RUN: llc < %s -emulated-tls -march=x86-64 -mtriple=x86_64-linux-gnu -relocation-model=pic | FileCheck -check-prefix=X64 %s
+; RUN: llc < %s -emulated-tls -march=x86 -mtriple=i386-linux-android -relocation-model=pic | FileCheck -check-prefix=X32 %s
+; RUN: llc < %s -emulated-tls -march=x86-64 -mtriple=x86_64-linux-android -relocation-model=pic | FileCheck -check-prefix=X64 %s
+
+; Use my_emutls_get_address like __emutls_get_address.
+@my_emutls_v_xyz = external global i8*, align 4
+declare i8* @my_emutls_get_address(i8*)
+
+define i32 @my_get_xyz() {
+; X32-LABEL: my_get_xyz:
+; X32:      movl my_emutls_v_xyz@GOT(%ebx), %eax
+; X32-NEXT: movl %eax, (%esp)
+; X32-NEXT: calll my_emutls_get_address@PLT
+; X64-LABEL: my_get_xyz:
+; X64:      movq my_emutls_v_xyz@GOTPCREL(%rip), %rdi
+; X64-NEXT: callq my_emutls_get_address@PLT
+; X64-NEXT: movl (%rax), %eax
+
+entry:
+  %call = call i8* @my_emutls_get_address(i8* bitcast (i8** @my_emutls_v_xyz to i8*))
+  %0 = bitcast i8* %call to i32*
+  %1 = load i32, i32* %0, align 4
+  ret i32 %1
+}
+
+@i = thread_local global i32 15
+@j = internal thread_local global i32 42
+@k = internal thread_local global i32 0, align 8
+
+define i32 @f1() {
+entry:
+  %tmp1 = load i32, i32* @i
+  ret i32 %tmp1
+}
+
+; X32-LABEL: f1:
+; X32:      movl __emutls_v.i@GOT(%ebx), %eax
+; X32-NEXT: movl %eax, (%esp)
+; X32-NEXT: calll __emutls_get_address@PLT
+; X64-LABEL: f1:
+; X64:      movq __emutls_v.i@GOTPCREL(%rip), %rdi
+; X64-NEXT: callq __emutls_get_address@PLT
+; X64-NEXT: movl (%rax), %eax
+
+@i2 = external thread_local global i32
+
+define i32* @f2() {
+entry:
+  ret i32* @i
+}
+
+; X32-LABEL: f2:
+; X64-LABEL: f2:
+
+
+define i32 @f3() {
+entry:
+  %tmp1 = load i32, i32* @i  ; <i32> [#uses=1]
+  ret i32 %tmp1
+}
+
+; X32-LABEL: f3:
+; X64-LABEL: f3:
+
+
+define i32* @f4() nounwind {
+entry:
+  ret i32* @i
+}
+
+; X32-LABEL: f4:
+; X64-LABEL: f4:
+
+
+define i32 @f5() nounwind {
+entry:
+  %0 = load i32, i32* @j, align 4
+  %1 = load i32, i32* @k, align 4
+  %add = add nsw i32 %0, %1
+  ret i32 %add
+}
+
+; X32-LABEL: f5:
+; X32:      movl __emutls_v.j@GOT(%ebx), %eax
+; X32-NEXT: movl %eax, (%esp)
+; X32-NEXT: calll __emutls_get_address@PLT
+; X32-NEXT: movl (%eax), %esi
+; X32-NEXT: movl __emutls_v.k@GOT(%ebx), %eax
+; X32-NEXT: movl %eax, (%esp)
+; X32-NEXT: calll __emutls_get_address@PLT
+; X32-NEXT: addl (%eax), %esi
+; X32-NEXT: movl %esi, %eax
+
+; X64-LABEL: f5:
+; X64:      movq __emutls_v.j@GOTPCREL(%rip), %rdi
+; X64-NEXT: callq __emutls_get_address@PLT
+; X64-NEXT: movl (%rax), %ebx
+; X64-NEXT: movq __emutls_v.k@GOTPCREL(%rip), %rdi
+; X64-NEXT: callq __emutls_get_address@PLT
+; X64-NEXT: addl (%rax), %ebx
+; X64-NEXT: movl %ebx, %eax
+
+;;;;; 32-bit targets
+
+; X32:      .section .data.rel.local,
+; X32-LABEL: __emutls_v.i:
+; X32-NEXT: .long 4
+; X32-NEXT: .long 4
+; X32-NEXT: .long 0
+; X32-NEXT: .long __emutls_t.i
+
+; X32:      .section .rodata,
+; X32-LABEL: __emutls_t.i:
+; X32-NEXT: .long 15
+
+; X32:      .section .data.rel.local,
+; X32-LABEL: __emutls_v.j:
+; X32-NEXT: .long 4
+; X32-NEXT: .long 4
+; X32-NEXT: .long 0
+; X32-NEXT: .long __emutls_t.j
+
+; X32:      .section .rodata,
+; X32-LABEL: __emutls_t.j:
+; X32-NEXT: .long 42
+
+; X32:      .data
+; X32-LABEL: __emutls_v.k:
+; X32-NEXT: .long 4
+; X32-NEXT: .long 8
+; X32-NEXT: .long 0
+; X32-NEXT: .long 0
+
+; X32-NOT:   __emutls_t.k:
+
+;;;;; 64-bit targets
+
+; X64:      .section .data.rel.local,
+; X64-LABEL: __emutls_v.i:
+; X64-NEXT: .quad 4
+; X64-NEXT: .quad 4
+; X64-NEXT: .quad 0
+; X64-NEXT: .quad __emutls_t.i
+
+; X64:      .section .rodata,
+; X64-LABEL: __emutls_t.i:
+; X64-NEXT: .long 15
+
+; X64:      .section .data.rel.local,
+; X64-LABEL: __emutls_v.j:
+; X64-NEXT: .quad 4
+; X64-NEXT: .quad 4
+; X64-NEXT: .quad 0
+; X64-NEXT: .quad __emutls_t.j
+
+; X64:      .section .rodata,
+; X64-LABEL: __emutls_t.j:
+; X64-NEXT: .long 42
+
+; X64:      .data
+; X64-LABEL: __emutls_v.k:
+; X64-NEXT: .quad 4
+; X64-NEXT: .quad 8
+; X64-NEXT: .quad 0
+; X64-NEXT: .quad 0
+
+; X64-NOT:   __emutls_t.k:
diff --git a/test/CodeGen/X86/emutls-pie.ll b/test/CodeGen/X86/emutls-pie.ll
new file mode 100644
index 00000000000..4cd3d22bce0
--- /dev/null
+++ b/test/CodeGen/X86/emutls-pie.ll
@@ -0,0 +1,131 @@
+; RUN: llc < %s -emulated-tls -march=x86 -mcpu=generic -mtriple=i386-linux-gnu -relocation-model=pic -enable-pie \
+; RUN:   | FileCheck -check-prefix=X32 %s
+; RUN: llc < %s -emulated-tls -march=x86-64 -mcpu=generic -mtriple=x86_64-linux-gnu -relocation-model=pic -enable-pie \
+; RUN:   | FileCheck -check-prefix=X64 %s
+; RUN: llc < %s -emulated-tls -march=x86 -mcpu=generic -mtriple=i386-linux-android -relocation-model=pic -enable-pie \
+; RUN:   | FileCheck -check-prefix=X32 %s
+; RUN: llc < %s -emulated-tls -march=x86-64 -mcpu=generic -mtriple=x86_64-linux-android -relocation-model=pic -enable-pie \
+; RUN:   | FileCheck -check-prefix=X64 %s
+
+; Use my_emutls_get_address like __emutls_get_address.
+@my_emutls_v_xyz = external global i8*, align 4
+declare i8* @my_emutls_get_address(i8*)
+
+define i32 @my_get_xyz() {
+; X32-LABEL: my_get_xyz:
+; X32:      movl my_emutls_v_xyz@GOT(%ebx), %eax
+; X32-NEXT: movl %eax, (%esp)
+; X32-NEXT: calll my_emutls_get_address@PLT
+; X32-NEXT: movl (%eax), %eax
+; X32-NEXT: addl $8, %esp
+; X32-NEXT: popl %ebx
+; X32-NEXT: retl
+; X64-LABEL: my_get_xyz:
+; X64:      movq my_emutls_v_xyz@GOTPCREL(%rip), %rdi
+; X64-NEXT: callq my_emutls_get_address@PLT
+; X64-NEXT: movl (%rax), %eax
+; X64-NEXT: popq %rdx
+; X64-NEXT: retq
+
+entry:
+  %call = call i8* @my_emutls_get_address(i8* bitcast (i8** @my_emutls_v_xyz to i8*))
+  %0 = bitcast i8* %call to i32*
+  %1 = load i32, i32* %0, align 4
+  ret i32 %1
+}
+
+@i = thread_local global i32 15
+@i2 = external thread_local global i32
+
+define i32 @f1() {
+; X32-LABEL: f1:
+; X32:      movl __emutls_v.i@GOT(%ebx), %eax
+; X32-NEXT: movl %eax, (%esp)
+; X32-NEXT: calll __emutls_get_address@PLT
+; X32-NEXT: movl (%eax), %eax
+; X32-NEXT: addl $8, %esp
+; X32-NEXT: popl %ebx
+; X32-NEXT: retl
+; X64-LABEL: f1:
+; X64:      movq __emutls_v.i@GOTPCREL(%rip), %rdi
+; X64-NEXT: callq __emutls_get_address@PLT
+; X64-NEXT: movl (%rax), %eax
+; X64-NEXT: popq %rdx
+; X64-NEXT: retq
+
+entry:
+  %tmp1 = load i32, i32* @i
+  ret i32 %tmp1
+}
+
+define i32* @f2() {
+; X32-LABEL: f2:
+; X32:      movl __emutls_v.i@GOT(%ebx), %eax
+; X32-NEXT: movl %eax, (%esp)
+; X32-NEXT: calll __emutls_get_address@PLT
+; X64-LABEL: f2:
+; X64:      movq __emutls_v.i@GOTPCREL(%rip), %rdi
+; X64-NEXT: callq __emutls_get_address@PLT
+
+entry:
+  ret i32* @i
+}
+
+define i32 @f3() {
+; X32-LABEL: f3:
+; X32:      movl __emutls_v.i2@GOT(%ebx), %eax
+; X32-NEXT: movl %eax, (%esp)
+; X32-NEXT: calll __emutls_get_address@PLT
+; X64-LABEL: f3:
+; X64:      movq __emutls_v.i2@GOTPCREL(%rip), %rdi
+; X64-NEXT: callq __emutls_get_address@PLT
+
+entry:
+  %tmp1 = load i32, i32* @i2
+  ret i32 %tmp1
+}
+
+define i32* @f4() {
+; X32-LABEL: f4:
+; X32:      movl __emutls_v.i2@GOT(%ebx), %eax
+; X32-NEXT: movl %eax, (%esp)
+; X32-NEXT: calll __emutls_get_address@PLT
+; X64-LABEL: f4:
+; X64:      movq __emutls_v.i2@GOTPCREL(%rip), %rdi
+; X64-NEXT: callq __emutls_get_address@PLT
+
+entry:
+  ret i32* @i2
+}
+
+;;;;; 32-bit targets
+
+; X32:      .section .data.rel.local,
+; X32-LABEL: __emutls_v.i:
+; X32-NEXT: .long 4
+; X32-NEXT: .long 4
+; X32-NEXT: .long 0
+; X32-NEXT: .long __emutls_t.i
+
+; X32:      .section .rodata,
+; X32-LABEL: __emutls_t.i:
+; X32-NEXT: .long 15
+
+; X32-NOT:   __emutls_v.i2
+; X32-NOT:   __emutls_t.i2
+
+;;;;; 64-bit targets
+
+; X64:      .section .data.rel.local,
+; X64-LABEL: __emutls_v.i:
+; X64-NEXT: .quad 4
+; X64-NEXT: .quad 4
+; X64-NEXT: .quad 0
+; X64-NEXT: .quad __emutls_t.i
+
+; X64:      .section .rodata,
+; X64-LABEL: __emutls_t.i:
+; X64-NEXT: .long 15
+
+; X64-NOT:   __emutls_v.i2
+; X64-NOT:   __emutls_t.i2
diff --git a/test/CodeGen/X86/emutls.ll b/test/CodeGen/X86/emutls.ll
new file mode 100644
index 00000000000..bb2f1e2aa82
--- /dev/null
+++ b/test/CodeGen/X86/emutls.ll
@@ -0,0 +1,347 @@
+; RUN: llc < %s -emulated-tls -march=x86 -mtriple=i386-linux-gnu | FileCheck -check-prefix=X32 %s
+; RUN: llc < %s -emulated-tls -march=x86-64 -mtriple=x86_64-linux-gnu | FileCheck -check-prefix=X64 %s
+; RUN: llc < %s -emulated-tls -march=x86 -mtriple=x86-linux-android | FileCheck -check-prefix=X32 %s
+; RUN: llc < %s -emulated-tls -march=x86-64 -mtriple=x86_64-linux-android | FileCheck -check-prefix=X64 %s
+
+; Copied from tls.ll; emulated TLS model is not implemented
+; for *-pc-win32 and *-pc-winows targets yet.
+
+; Use my_emutls_get_address like __emutls_get_address.
+@my_emutls_v_xyz = external global i8*, align 4
+declare i8* @my_emutls_get_address(i8*)
+
+define i32 @my_get_xyz() {
+; X32-LABEL: my_get_xyz:
+; X32:         movl $my_emutls_v_xyz, (%esp)
+; X32-NEXT:    calll my_emutls_get_address
+; X32-NEXT:    movl (%eax), %eax
+; X32-NEXT:    addl $12, %esp
+; X32-NEXT:    retl
+; X64-LABEL: my_get_xyz:
+; X64:         movl $my_emutls_v_xyz, %edi
+; X64-NEXT:    callq my_emutls_get_address
+; X64-NEXT:    movl (%rax), %eax
+; X64-NEXT:    popq %rdx
+; X64-NEXT:    retq
+
+entry:
+  %call = call i8* @my_emutls_get_address(i8* bitcast (i8** @my_emutls_v_xyz to i8*))
+  %0 = bitcast i8* %call to i32*
+  %1 = load i32, i32* %0, align 4
+  ret i32 %1
+}
+
+@i1 = thread_local global i32 15
+@i2 = external thread_local global i32
+@i3 = internal thread_local global i32 15
+@i4 = hidden thread_local global i32 15
+@i5 = external hidden thread_local global i32
+@s1 = thread_local global i16 15
+@b1 = thread_local global i8 0
+
+define i32 @f1() {
+; X32-LABEL: f1:
+; X32:         movl $__emutls_v.i1, (%esp)
+; X32-NEXT:    calll __emutls_get_address
+; X32-NEXT:    movl (%eax), %eax
+; X32-NEXT:    addl $12, %esp
+; X32-NEXT:    retl
+; X64-LABEL: f1:
+; X64:         movl $__emutls_v.i1, %edi
+; X64-NEXT:    callq __emutls_get_address
+; X64-NEXT:    movl (%rax), %eax
+; X64-NEXT:    popq %rdx
+; X64-NEXT:    retq
+
+entry:
+  %tmp1 = load i32, i32* @i1
+  ret i32 %tmp1
+}
+
+define i32* @f2() {
+; X32-LABEL: f2:
+; X32:         movl $__emutls_v.i1, (%esp)
+; X32-NEXT:    calll __emutls_get_address
+; X32-NEXT:    addl $12, %esp
+; X32-NEXT:    retl
+; X64-LABEL: f2:
+; X64:         movl $__emutls_v.i1, %edi
+; X64-NEXT:    callq __emutls_get_address
+; X64-NEXT:    popq %rdx
+; X64-NEXT:    retq
+
+entry:
+  ret i32* @i1
+}
+
+define i32 @f3() nounwind {
+; X32-LABEL: f3:
+; X32:         movl $__emutls_v.i2, (%esp)
+; X32-NEXT:    calll __emutls_get_address
+; X32-NEXT:    movl (%eax), %eax
+; X32-NEXT:    addl $12, %esp
+; X32-NEXT:    retl
+
+entry:
+  %tmp1 = load i32, i32* @i2
+  ret i32 %tmp1
+}
+
+define i32* @f4() {
+; X32-LABEL: f4:
+; X32:         movl $__emutls_v.i2, (%esp)
+; X32-NEXT:    calll __emutls_get_address
+; X32-NEXT:    addl $12, %esp
+; X32-NEXT:    retl
+
+entry:
+  ret i32* @i2
+}
+
+define i32 @f5() nounwind {
+; X32-LABEL: f5:
+; X32:         movl $__emutls_v.i3, (%esp)
+; X32-NEXT:    calll __emutls_get_address
+; X32-NEXT:    movl (%eax), %eax
+; X32-NEXT:    addl $12, %esp
+; X32-NEXT:    retl
+
+entry:
+  %tmp1 = load i32, i32* @i3
+  ret i32 %tmp1
+}
+
+define i32* @f6() {
+; X32-LABEL: f6:
+; X32:         movl $__emutls_v.i3, (%esp)
+; X32-NEXT:    calll __emutls_get_address
+; X32-NEXT:    addl $12, %esp
+; X32-NEXT:    retl
+
+entry:
+  ret i32* @i3
+}
+
+define i32 @f7() {
+; X32-LABEL: f7:
+; X32:         movl $__emutls_v.i4, (%esp)
+; X32-NEXT:    calll __emutls_get_address
+; X32-NEXT:    movl (%eax), %eax
+; X32-NEXT:    addl $12, %esp
+; X32-NEXT:    retl
+
+entry:
+  %tmp1 = load i32, i32* @i4
+  ret i32 %tmp1
+}
+
+define i32* @f8() {
+; X32-LABEL: f8:
+; X32:         movl $__emutls_v.i4, (%esp)
+; X32-NEXT:    calll __emutls_get_address
+; X32-NEXT:    addl $12, %esp
+; X32-NEXT:    retl
+
+entry:
+  ret i32* @i4
+}
+
+define i32 @f9() {
+; X32-LABEL: f9:
+; X32:         movl $__emutls_v.i5, (%esp)
+; X32-NEXT:    calll __emutls_get_address
+; X32-NEXT:    movl (%eax), %eax
+; X32-NEXT:    addl $12, %esp
+; X32-NEXT:    retl
+
+entry:
+  %tmp1 = load i32, i32* @i5
+  ret i32 %tmp1
+}
+
+define i32* @f10() {
+; X32-LABEL: f10:
+; X32:         movl $__emutls_v.i5, (%esp)
+; X32-NEXT:    calll __emutls_get_address
+; X32-NEXT:    addl $12, %esp
+; X32-NEXT:    retl
+
+entry:
+  ret i32* @i5
+}
+
+define i16 @f11() {
+; X32-LABEL: f11:
+; X32:         movl $__emutls_v.s1, (%esp)
+; X32-NEXT:    calll __emutls_get_address
+; X32-NEXT:    movzwl (%eax), %eax
+; X32-NEXT:    addl $12, %esp
+; X32-NEXT:    retl
+
+entry:
+  %tmp1 = load i16, i16* @s1
+  ret i16 %tmp1
+}
+
+define i32 @f12() {
+; X32-LABEL: f12:
+; X32:         movl $__emutls_v.s1, (%esp)
+; X32-NEXT:    calll __emutls_get_address
+; X32-NEXT:    movswl (%eax), %eax
+; X32-NEXT:    addl $12, %esp
+; X32-NEXT:    retl
+
+entry:
+  %tmp1 = load i16, i16* @s1
+  %tmp2 = sext i16 %tmp1 to i32
+  ret i32 %tmp2
+}
+
+define i8 @f13() {
+; X32-LABEL: f13:
+; X32:         movl $__emutls_v.b1, (%esp)
+; X32-NEXT:    calll __emutls_get_address
+; X32-NEXT:    movb (%eax), %al
+; X32-NEXT:    addl $12, %esp
+; X32-NEXT:    retl
+
+entry:
+  %tmp1 = load i8, i8* @b1
+  ret i8 %tmp1
+}
+
+define i32 @f14() {
+; X32-LABEL: f14:
+; X32:         movl $__emutls_v.b1, (%esp)
+; X32-NEXT:    calll __emutls_get_address
+; X32-NEXT:    movsbl (%eax), %eax
+; X32-NEXT:    addl $12, %esp
+; X32-NEXT:    retl
+
+entry:
+  %tmp1 = load i8, i8* @b1
+  %tmp2 = sext i8 %tmp1 to i32
+  ret i32 %tmp2
+}
+
+;;;;;;;;;;;;;; 32-bit __emutls_v. and __emutls_t.
+
+; X32       .section .data.rel.local,
+; X32-LABEL: __emutls_v.i1:
+; X32-NEXT: .long 4
+; X32-NEXT: .long 4
+; X32-NEXT: .long 0
+; X32-NEXT: .long __emutls_t.i1
+
+; X32       .section .rodata,
+; X32-LABEL: __emutls_t.i1:
+; X32-NEXT: .long 15
+
+; X32-NOT:   __emutls_v.i2
+
+; X32       .section .data.rel.local,
+; X32-LABEL: __emutls_v.i3:
+; X32-NEXT: .long 4
+; X32-NEXT: .long 4
+; X32-NEXT: .long 0
+; X32-NEXT: .long __emutls_t.i3
+
+; X32       .section .rodata,
+; X32-LABEL: __emutls_t.i3:
+; X32-NEXT: .long 15
+
+; X32       .section .data.rel.local,
+; X32-LABEL: __emutls_v.i4:
+; X32-NEXT: .long 4
+; X32-NEXT: .long 4
+; X32-NEXT: .long 0
+; X32-NEXT: .long __emutls_t.i4
+
+; X32       .section .rodata,
+; X32-LABEL: __emutls_t.i4:
+; X32-NEXT: .long 15
+
+; X32-NOT:   __emutls_v.i5:
+; X32       .hidden __emutls_v.i5
+; X32-NOT:   __emutls_v.i5:
+
+; X32 .section .data.rel.local,
+; X32-LABEL: __emutls_v.s1:
+; X32-NEXT: .long 2
+; X32-NEXT: .long 2
+; X32-NEXT: .long 0
+; X32-NEXT: .long __emutls_t.s1
+
+; X32 .section .rodata,
+; X32-LABEL: __emutls_t.s1:
+; X32-NEXT: .short 15
+
+; X32 .section .data.rel.local,
+; X32-LABEL: __emutls_v.b1:
+; X32-NEXT: .long 1
+; X32-NEXT: .long 1
+; X32-NEXT: .long 0
+; X32-NEXT: .long 0
+
+; X32-NOT:   __emutls_t.b1
+
+;;;;;;;;;;;;;; 64-bit __emutls_v. and __emutls_t.
+
+; X64       .section .data.rel.local,
+; X64-LABEL: __emutls_v.i1:
+; X64-NEXT: .quad 4
+; X64-NEXT: .quad 4
+; X64-NEXT: .quad 0
+; X64-NEXT: .quad __emutls_t.i1
+
+; X64       .section .rodata,
+; X64-LABEL: __emutls_t.i1:
+; X64-NEXT: .long 15
+
+; X64-NOT:   __emutls_v.i2
+
+; X64       .section .data.rel.local,
+; X64-LABEL: __emutls_v.i3:
+; X64-NEXT: .quad 4
+; X64-NEXT: .quad 4
+; X64-NEXT: .quad 0
+; X64-NEXT: .quad __emutls_t.i3
+
+; X64       .section .rodata,
+; X64-LABEL: __emutls_t.i3:
+; X64-NEXT: .long 15
+
+; X64       .section .data.rel.local,
+; X64-LABEL: __emutls_v.i4:
+; X64-NEXT: .quad 4
+; X64-NEXT: .quad 4
+; X64-NEXT: .quad 0
+; X64-NEXT: .quad __emutls_t.i4
+
+; X64       .section .rodata,
+; X64-LABEL: __emutls_t.i4:
+; X64-NEXT: .long 15
+
+; X64-NOT:   __emutls_v.i5:
+; X64       .hidden __emutls_v.i5
+; X64-NOT:   __emutls_v.i5:
+
+; X64       .section .data.rel.local,
+; X64-LABEL: __emutls_v.s1:
+; X64-NEXT: .quad 2
+; X64-NEXT: .quad 2
+; X64-NEXT: .quad 0
+; X64-NEXT: .quad __emutls_t.s1
+
+; X64       .section .rodata,
+; X64-LABEL: __emutls_t.s1:
+; X64-NEXT: .short 15
+
+; X64       .section .data.rel.local,
+; X64-LABEL: __emutls_v.b1:
+; X64-NEXT: .quad 1
+; X64-NEXT: .quad 1
+; X64-NEXT: .quad 0
+; X64-NEXT: .quad 0
+
+; X64-NOT:  __emutls_t.b1
diff --git a/test/CodeGen/X86/emutls_generic.ll b/test/CodeGen/X86/emutls_generic.ll
new file mode 100644
index 00000000000..c80c8977447
--- /dev/null
+++ b/test/CodeGen/X86/emutls_generic.ll
@@ -0,0 +1,107 @@
+; RUN: llc < %s -emulated-tls -mtriple=i686-linux-android -relocation-model=pic \
+; RUN:     | FileCheck -check-prefix=X86_32 %s
+; RUN: llc < %s -emulated-tls -mtriple=x86_64-linux-android -march=x86 -relocation-model=pic \
+; RUN:     | FileCheck -check-prefix=X86_32 %s
+; RUN: llc < %s -emulated-tls -mtriple=x86_64-linux-android -relocation-model=pic \
+; RUN:     | FileCheck -check-prefix=X86_64 %s
+; RUN: llc < %s -emulated-tls -march=x86 -mtriple=i386-linux-gnu -relocation-model=pic \
+; RUN:     | FileCheck %s
+
+; Make sure that TLS symbols are emitted in expected order.
+
+@external_x = external thread_local global i32, align 8
+@external_y = thread_local global i8 7, align 2
+@internal_y = internal thread_local global i64 9, align 16
+
+define i32* @get_external_x() {
+entry:
+  ret i32* @external_x
+}
+
+define i8* @get_external_y() {
+entry:
+  ret i8* @external_y
+}
+
+define i64* @get_internal_y() {
+entry:
+  ret i64* @internal_y
+}
+
+; CHECK-LABEL: get_external_x:
+; CHECK-NOT:   _tls_get_address
+; CHECK:       __emutls_get_address
+; CHECK-LABEL: get_external_y:
+; CHECK:       __emutls_get_address
+; CHECK-NOT:   _tls_get_address
+; CHECK-LABEL: get_internal_y:
+; CHECK-NOT:   __emutls_t.external_x:
+; CHECK-NOT:   __emutls_v.external_x:
+; CHECK-LABEL: __emutls_v.external_y:
+; CHECK-LABEL: __emutls_t.external_y:
+; CHECK:       __emutls_t.external_y
+; CHECK-LABEL: __emutls_v.internal_y:
+; CHECK-LABEL: __emutls_t.internal_y:
+; CHECK:       __emutls_t.internal_y
+
+; X86_32-LABEL:  get_external_x:
+; X86_32:        movl __emutls_v.external_x
+; X86_32:        calll __emutls_get_address
+; X86_32-LABEL:  get_external_y:
+; X86_32:        movl __emutls_v.external_y
+; X86_32:        calll __emutls_get_address
+; X86_32-LABEL:  get_internal_y:
+; X86_32:      movl __emutls_v.internal_y
+; X86_32:      calll __emutls_get_address
+; X86_32-NOT:   __emutls_t.external_x
+; X86_32-NOT:   __emutls_v.external_x:
+; X86_32:        .section .data.rel.local
+; X86_32:        .align 4
+; X86_32-LABEL:  __emutls_v.external_y:
+; X86_32-NEXT:   .long 1
+; X86_32-NEXT:   .long 2
+; X86_32-NEXT:   .long 0
+; X86_32-NEXT:   .long __emutls_t.external_y
+; X86_32:        .section .rodata,
+; X86_32-LABEL:  __emutls_t.external_y:
+; X86_32-NEXT:   .byte 7
+; X86_32:        .section .data.rel.local
+; X86_32:        .align 4
+; X86_32-LABEL:  __emutls_v.internal_y:
+; X86_32-NEXT:   .long 8
+; X86_32-NEXT:   .long 16
+; X86_32-NEXT:   .long 0
+; X86_32-NEXT:   .long __emutls_t.internal_y
+; X86_32-LABEL:  __emutls_t.internal_y:
+; X86_32-NEXT:   .quad 9
+; X86_64-LABEL:  get_external_x:
+; X86_64:      __emutls_v.external_x
+; X86_64:      __emutls_get_address
+; X86_64-LABEL:  get_external_y:
+; X86_64:      __emutls_v.external_y
+; X86_64:      __emutls_get_address
+; X86_64-LABEL:  get_internal_y:
+; X86_64:      __emutls_v.internal_y
+; X86_64:      __emutls_get_address
+; X86_64-NOT:   __emutls_t.external_x
+; X86_64-NOT:   __emutls_v.external_x:
+; X86_64:        .align 8
+; X86_64-LABEL:  __emutls_v.external_y:
+; X86_64-NEXT:   .quad 1
+; X86_64-NEXT:   .quad 2
+; X86_64-NEXT:   .quad 0
+; X86_64-NEXT:   .quad __emutls_t.external_y
+; X86_64-NOT:    __emutls_v.external_x:
+; X86_64:        .section .rodata,
+; X86_64-LABEL:  __emutls_t.external_y:
+; X86_64-NEXT:   .byte 7
+; X86_64:        .section .data.rel.local
+; X86_64:        .align 8
+; X86_64-LABEL:  __emutls_v.internal_y:
+; X86_64-NEXT:   .quad 8
+; X86_64-NEXT:   .quad 16
+; X86_64-NEXT:   .quad 0
+; X86_64-NEXT:   .quad __emutls_t.internal_y
+; X86_64:        .section .rodata,
+; X86_64-LABEL:  __emutls_t.internal_y:
+; X86_64-NEXT:   .quad 9
diff --git a/test/CodeGen/X86/exception-label.ll b/test/CodeGen/X86/exception-label.ll
index cafa1e630b9..2270d2da180 100644
--- a/test/CodeGen/X86/exception-label.ll
+++ b/test/CodeGen/X86/exception-label.ll
@@ -8,13 +8,13 @@
 
 declare void @g()
 
-define void @f() {
+define void @f() personality i8* bitcast (void ()* @g to i8*) {
 bb0:
   call void asm ".Lexception0:", ""()
   invoke void @g()
           to label %bb2 unwind label %bb1
 bb1:
-  landingpad { i8*, i32 } personality i8* bitcast (void ()* @g to i8*)
+  landingpad { i8*, i32 }
           catch i8* null
   br label %bb2
 
diff --git a/test/CodeGen/X86/exedeps-movq.ll b/test/CodeGen/X86/exedeps-movq.ll
index b702c8716a9..a5873be6f27 100644
--- a/test/CodeGen/X86/exedeps-movq.ll
+++ b/test/CodeGen/X86/exedeps-movq.ll
@@ -19,12 +19,7 @@ define void @store_floats(<4 x float> %x, i64* %p) {
 ; AVX-LABEL: store_floats:
 ; AVX:       # BB#0:
 ; AVX-NEXT:    vaddps %xmm0, %xmm0, %xmm0
-
-
-; !!! FIXME - the AVX version is not handled correctly.
-; AVX-NEXT:    vmovq %xmm0, (%rdi)
-
-
+; AVX-NEXT:    vmovlps %xmm0, (%rdi)
 ; AVX-NEXT:    retq
   %a = fadd <4 x float> %x, %x
   %b = shufflevector <4 x float> %a, <4 x float> undef, <2 x i32> <i32 0, i32 1>
diff --git a/test/CodeGen/X86/expand-vr64-gr64-copy.mir b/test/CodeGen/X86/expand-vr64-gr64-copy.mir
new file mode 100644
index 00000000000..98fd3f779ec
--- /dev/null
+++ b/test/CodeGen/X86/expand-vr64-gr64-copy.mir
@@ -0,0 +1,37 @@
+# RUN: llc -run-pass postrapseudos -mtriple=x86_64-unknown-unknown -mattr=+3dnow -o /dev/null %s | FileCheck %s
+# This test verifies that the ExpandPostRA pass expands the GR64 <-> VR64
+# copies into appropriate MMX_MOV instructions.
+
+--- |
+
+  define <2 x i32> @test_pswapdsi(<2 x i32> %a) nounwind readnone {
+  entry:
+    %0 = bitcast <2 x i32> %a to x86_mmx
+    %1 = tail call x86_mmx @llvm.x86.3dnowa.pswapd(x86_mmx %0)
+    %2 = bitcast x86_mmx %1 to <2 x i32>
+    ret <2 x i32> %2
+  }
+
+  declare x86_mmx @llvm.x86.3dnowa.pswapd(x86_mmx) nounwind readnone
+
+...
+---
+name:            test_pswapdsi
+tracksRegLiveness: true
+body:
+  - id:          0
+    name:        entry
+    liveins:     [ '%xmm0' ]
+    instructions:
+      - '%xmm0 = PSHUFDri killed %xmm0, -24'
+      - 'MOVPQI2QImr %rsp, 1, _, -8, _, killed %xmm0'
+      - '%mm0 = PSWAPDrm %rsp, 1, _, -8, _'
+# CHECK:      %rax = MMX_MOVD64from64rr %mm0
+# CHECK-NEXT: %mm0 = MMX_MOVD64to64rr %rax
+      - '%rax = COPY %mm0'
+      - '%mm0 = COPY %rax'
+      - 'MMX_MOVQ64mr %rsp, 1, _, -16, _, killed %mm0'
+      - '%xmm0 = MOVQI2PQIrm %rsp, 1, _, -16, _'
+      - '%xmm0 = PSHUFDri killed %xmm0, -44'
+      - 'RETQ %xmm0'
+...
diff --git a/test/CodeGen/X86/extract-store.ll b/test/CodeGen/X86/extract-store.ll
index 27d93804ba6..801ecc9ada6 100644
--- a/test/CodeGen/X86/extract-store.ll
+++ b/test/CodeGen/X86/extract-store.ll
@@ -1,7 +1,8 @@
 ; RUN: llc < %s -o - -mcpu=generic -march=x86-64 -mattr=+sse4.1 | FileCheck %s -check-prefix=SSE41
 ; RUN: llc < %s -o - -mcpu=generic -march=x86-64 -mattr=+avx | FileCheck %s -check-prefix=AVX
 
-define void @pextrb(i8* nocapture %dst, <16 x i8> %foo) {
+; CHECK-LABEL: extract_i8
+define void @extract_i8(i8* nocapture %dst, <16 x i8> %foo) {
 ; AVX: vpextrb
 ; SSE41: pextrb
 ; AVX-NOT: movb
@@ -11,12 +12,35 @@ define void @pextrb(i8* nocapture %dst, <16 x i8> %foo) {
   ret void
 }
 
-define void @pextrw(i16* nocapture %dst, <8 x i16> %foo) {
+; CHECK-LABEL: extract_i16
+define void @extract_i16(i16* nocapture %dst, <8 x i16> %foo) {
 ; AVX: vpextrw
 ; SSE41: pextrw
 ; AVX-NOT: movw
 ; SSE41-NOT: movw
-  %vecext = extractelement <8 x i16> %foo, i32 15
+  %vecext = extractelement <8 x i16> %foo, i32 7
+  store i16 %vecext, i16* %dst, align 1
+  ret void
+}
+
+; CHECK-LABEL: extract_i8_undef
+define void @extract_i8_undef(i8* nocapture %dst, <16 x i8> %foo) {
+; AVX-NOT: vpextrb
+; SSE41-NOT: pextrb
+; AVX-NOT: movb
+; SSE41-NOT: movb
+  %vecext = extractelement <16 x i8> %foo, i32 16 ; undef
+  store i8 %vecext, i8* %dst, align 1
+  ret void
+}
+
+; CHECK-LABEL: extract_i16_undef
+define void @extract_i16_undef(i16* nocapture %dst, <8 x i16> %foo) {
+; AVX-NOT: vpextrw
+; SSE41-NOT: pextrw
+; AVX-NOT: movw
+; SSE41-NOT: movw
+  %vecext = extractelement <8 x i16> %foo, i32 9 ; undef
   store i16 %vecext, i16* %dst, align 1
   ret void
 }
diff --git a/test/CodeGen/X86/extractelement-index.ll b/test/CodeGen/X86/extractelement-index.ll
new file mode 100644
index 00000000000..ab3ff8ed435
--- /dev/null
+++ b/test/CodeGen/X86/extractelement-index.ll
@@ -0,0 +1,51 @@
+; RUN: llc < %s -mtriple=x86_64-pc-linux             | FileCheck %s -check-prefix=X64
+; RUN: llc < %s -mtriple=x86_64-pc-linux -mattr=+avx | FileCheck %s -check-prefix=AVX
+
+
+; CHECK-LABEL: extractelement_index_1:
+define i8 @extractelement_index_1(<32 x i8> %a) nounwind {
+  ; X64:       movaps
+  ; AVX:       vpextrb $1
+  %b = extractelement <32 x i8> %a, i256 1
+  ret i8 %b
+}
+
+; CHECK-LABEL: extractelement_index_2:
+define i32 @extractelement_index_2(<8 x i32> %a) nounwind {
+  ; X64:       pshufd
+  ; AVX:       vextractf128 $1
+  ; AVX-NEXT:  vpextrd      $3
+  %b = extractelement <8 x i32> %a, i64 7
+  ret i32 %b
+}
+
+; CHECK-LABEL: extractelement_index_3:
+define i32 @extractelement_index_3(<8 x i32> %a) nounwind {
+  ; CHECK-NOT: pextr
+  %b = extractelement <8 x i32> %a, i64 15
+  ret i32 %b
+}
+
+; CHECK-LABEL: extractelement_index_4:
+define i32 @extractelement_index_4(<8 x i32> %a) nounwind {
+  ; X64:       movd
+  ; AVX:       vextractf128 $1
+  ; AVX-NEXT:  vmovd
+  %b = extractelement <8 x i32> %a, i256 4
+  ret i32 %b
+}
+
+; CHECK-LABEL: extractelement_index_5:
+define i8 @extractelement_index_5(<32 x i8> %a, i256 %i) nounwind {
+  ; X64:       movaps
+  ; AVX:       vmovaps
+  %b = extractelement <32 x i8> %a, i256 %i
+  ret i8 %b
+}
+
+; CHECK-LABEL: extractelement_index_6:
+define i8 @extractelement_index_6(<32 x i8> %a) nounwind {
+  ; CHECK-NOT: pextr
+  %b = extractelement <32 x i8> %a, i256 -1
+  ret i8 %b
+}
\ No newline at end of file
diff --git a/test/CodeGen/X86/f16c-intrinsics.ll b/test/CodeGen/X86/f16c-intrinsics.ll
index 02967d52048..485592aeac3 100644
--- a/test/CodeGen/X86/f16c-intrinsics.ll
+++ b/test/CodeGen/X86/f16c-intrinsics.ll
@@ -1,8 +1,8 @@
-; RUN: llc < %s -march=x86 -mattr=+avx,+f16c | FileCheck %s
-; RUN: llc < %s -march=x86-64 -mattr=+avx,+f16c | FileCheck %s
+; RUN: llc < %s -mtriple=i686-unknown-unknown   -mattr=+avx,+f16c | FileCheck %s
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx,+f16c | FileCheck %s
 
 define <4 x float> @test_x86_vcvtph2ps_128(<8 x i16> %a0) {
-  ; CHECK-LABEL: test_x86_vcvtph2ps_128
+  ; CHECK-LABEL: test_x86_vcvtph2ps_128:
   ; CHECK-NOT: vmov
   ; CHECK: vcvtph2ps
   %res = call <4 x float> @llvm.x86.vcvtph2ps.128(<8 x i16> %a0) ; <<4 x float>> [#uses=1]
@@ -12,7 +12,7 @@ declare <4 x float> @llvm.x86.vcvtph2ps.128(<8 x i16>) nounwind readonly
 
 
 define <8 x float> @test_x86_vcvtph2ps_256(<8 x i16> %a0) {
-  ; CHECK-LABEL: test_x86_vcvtph2ps_256
+  ; CHECK-LABEL: test_x86_vcvtph2ps_256:
   ; CHECK-NOT: vmov
   ; CHECK: vcvtph2ps
   %res = call <8 x float> @llvm.x86.vcvtph2ps.256(<8 x i16> %a0) ; <<8 x float>> [#uses=1]
@@ -31,7 +31,7 @@ entry:
 }
 
 define <8 x i16> @test_x86_vcvtps2ph_128(<4 x float> %a0) {
-  ; CHECK-LABEL: test_x86_vcvtps2ph_128
+  ; CHECK-LABEL: test_x86_vcvtps2ph_128:
   ; CHECK-NOT: vmov
   ; CHECK: vcvtps2ph
   %res = call <8 x i16> @llvm.x86.vcvtps2ph.128(<4 x float> %a0, i32 0) ; <<8 x i16>> [#uses=1]
@@ -39,9 +39,8 @@ define <8 x i16> @test_x86_vcvtps2ph_128(<4 x float> %a0) {
 }
 declare <8 x i16> @llvm.x86.vcvtps2ph.128(<4 x float>, i32) nounwind readonly
 
-
 define <8 x i16> @test_x86_vcvtps2ph_256(<8 x float> %a0) {
-  ; CHECK-LABEL: test_x86_vcvtps2ph_256
+  ; CHECK-LABEL: test_x86_vcvtps2ph_256:
   ; CHECK-NOT: vmov
   ; CHECK: vcvtps2ph
   %res = call <8 x i16> @llvm.x86.vcvtps2ph.256(<8 x float> %a0, i32 0) ; <<8 x i16>> [#uses=1]
@@ -50,7 +49,7 @@ define <8 x i16> @test_x86_vcvtps2ph_256(<8 x float> %a0) {
 declare <8 x i16> @llvm.x86.vcvtps2ph.256(<8 x float>, i32) nounwind readonly
 
 define <4 x float> @test_x86_vcvtps2ph_128_scalar(i64* %ptr) {
-; CHECK-LABEL: test_x86_vcvtps2ph_128_scalar
+; CHECK-LABEL: test_x86_vcvtps2ph_128_scalar:
 ; CHECK-NOT: vmov
 ; CHECK: vcvtph2ps (%
 
@@ -61,3 +60,48 @@ define <4 x float> @test_x86_vcvtps2ph_128_scalar(i64* %ptr) {
   %res = tail call <4 x float> @llvm.x86.vcvtph2ps.128(<8 x i16> %bc) #2
   ret <4 x float> %res
 }
+
+define void @test_x86_vcvtps2ph_256_m(<8 x i16>* nocapture %d, <8 x float> %a) nounwind {
+entry:
+  ; CHECK-LABEL: test_x86_vcvtps2ph_256_m:
+  ; CHECK-NOT: vmov
+  ; CHECK: vcvtps2ph  $3, %ymm0, (%
+  %0 = tail call <8 x i16> @llvm.x86.vcvtps2ph.256(<8 x float> %a, i32 3)
+  store <8 x i16> %0, <8 x i16>* %d, align 16
+  ret void
+}
+
+define void @test_x86_vcvtps2ph_128_m(<4 x i16>* nocapture %d, <4 x float> %a) nounwind {
+entry:
+  ; CHECK-LABEL: test_x86_vcvtps2ph_128_m:
+  ; CHECK-NOT: vmov
+  ; CHECK: vcvtps2ph  $3, %xmm0, (%
+  %0 = tail call <8 x i16> @llvm.x86.vcvtps2ph.128(<4 x float> %a, i32 3)
+  %1 = shufflevector <8 x i16> %0, <8 x i16> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  store <4 x i16> %1, <4 x i16>* %d, align 8
+  ret void
+}
+
+define void @test_x86_vcvtps2ph_128_m2(double* nocapture %hf4x16, <4 x float> %f4x32) #0 {
+entry:
+  ; CHECK-LABEL: test_x86_vcvtps2ph_128_m2:
+  ; CHECK-NOT: vmov
+  ; CHECK: vcvtps2ph  $3, %xmm0, (%
+  %0 = tail call <8 x i16> @llvm.x86.vcvtps2ph.128(<4 x float> %f4x32, i32 3)
+  %1 = bitcast <8 x i16> %0 to <2 x double>
+  %vecext = extractelement <2 x double> %1, i32 0
+  store double %vecext, double* %hf4x16, align 8
+  ret void
+}
+
+define void @test_x86_vcvtps2ph_128_m3(i64* nocapture %hf4x16, <4 x float> %f4x32) #0 {
+entry:
+  ; CHECK-LABEL: test_x86_vcvtps2ph_128_m3:
+  ; CHECK-NOT: vmov
+  ; CHECK: vcvtps2ph  $3, %xmm0, (%
+  %0 = tail call <8 x i16> @llvm.x86.vcvtps2ph.128(<4 x float> %f4x32, i32 3)
+  %1 = bitcast <8 x i16> %0 to <2 x i64>
+  %vecext = extractelement <2 x i64> %1, i32 0
+  store i64 %vecext, i64* %hf4x16, align 8
+  ret void
+}
diff --git a/test/CodeGen/X86/fast-isel-cmp-branch.ll b/test/CodeGen/X86/fast-isel-cmp-branch.ll
index 684647ca948..d7b64ed3a5b 100644
--- a/test/CodeGen/X86/fast-isel-cmp-branch.ll
+++ b/test/CodeGen/X86/fast-isel-cmp-branch.ll
@@ -12,7 +12,7 @@
 
 declare void @bar()
 
-define void @foo(i32 %a, i32 %b) nounwind {
+define void @foo(i32 %a, i32 %b) nounwind personality i32 (...)* @__gxx_personality_v0 {
 entry:
   %q = add i32 %a, 7
   %r = add i32 %b, 9
@@ -26,7 +26,7 @@ true:
 return:
   ret void
 unw:
-  %exn = landingpad {i8*, i32} personality i32 (...)* @__gxx_personality_v0
+  %exn = landingpad {i8*, i32}
             cleanup
   unreachable
 }
diff --git a/test/CodeGen/X86/fast-isel-constrain-store-indexreg.ll b/test/CodeGen/X86/fast-isel-constrain-store-indexreg.ll
new file mode 100644
index 00000000000..c2534f72bd0
--- /dev/null
+++ b/test/CodeGen/X86/fast-isel-constrain-store-indexreg.ll
@@ -0,0 +1,25 @@
+; RUN: llc %s -o - -verify-machineinstrs | FileCheck %s
+
+target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-apple-unknown"
+
+@TheArray = external global [100000 x double], align 16
+
+; This test ensures, via the machine verifier, that the register class for the
+; index of the double store is correctly constrained to not include SP.
+
+; CHECK: movsd
+
+define i32 @main(i32* %i, double %tmpv) {
+bb:
+  br label %bb7
+
+bb7:                                              ; preds = %bb7, %bb
+  %storemerge = phi i32 [ 0, %bb ], [ %tmp19, %bb7 ]
+  %tmp15 = zext i32 %storemerge to i64
+  %tmp16 = getelementptr inbounds [100000 x double], [100000 x double]* @TheArray, i64 0, i64 %tmp15
+  store double %tmpv, double* %tmp16, align 8
+  %tmp18 = load i32, i32* %i, align 4
+  %tmp19 = add i32 %tmp18, 1
+  br label %bb7
+}
diff --git a/test/CodeGen/X86/fast-isel-emutls.ll b/test/CodeGen/X86/fast-isel-emutls.ll
new file mode 100644
index 00000000000..7dbafba6f21
--- /dev/null
+++ b/test/CodeGen/X86/fast-isel-emutls.ll
@@ -0,0 +1,48 @@
+; RUN: llc < %s -emulated-tls -march=x86 -relocation-model=pic -mtriple=i686-unknown-linux-gnu -fast-isel | FileCheck %s
+; PR3654
+
+@v = thread_local global i32 0
+define i32 @f() nounwind {
+entry:
+          %t = load i32, i32* @v
+          %s = add i32 %t, 1
+          ret i32 %s
+}
+
+; CHECK-LABEL: f:
+; CHECK:      movl __emutls_v.v@GOT(%ebx), %eax
+; CHECK-NEXT: movl %eax, (%esp)
+; CHECK-NEXT: calll __emutls_get_address@PLT
+; CHECK-NEXT: movl (%eax), %eax
+
+@alias = internal alias i32* @v
+define i32 @f_alias() nounwind {
+entry:
+          %t = load i32, i32* @v
+          %s = add i32 %t, 1
+          ret i32 %s
+}
+
+; CHECK-LABEL: f_alias:
+; CHECK:      movl __emutls_v.v@GOT(%ebx), %eax
+; CHECK-NEXT: movl %eax, (%esp)
+; CHECK-NEXT: calll __emutls_get_address@PLT
+; CHECK-NEXT: movl (%eax), %eax
+
+; Use my_emutls_get_address like __emutls_get_address.
+@my_emutls_v_xyz = external global i8*, align 4
+declare i8* @my_emutls_get_address(i8*)
+
+define i32 @my_get_xyz() {
+entry:
+  %call = call i8* @my_emutls_get_address(i8* bitcast (i8** @my_emutls_v_xyz to i8*))
+  %0 = bitcast i8* %call to i32*
+  %1 = load i32, i32* %0, align 4
+  ret i32 %1
+}
+
+; CHECK-LABEL: my_get_xyz:
+; CHECK:      movl my_emutls_v_xyz@GOT(%ebx), %eax
+; CHECK-NEXT: movl %eax, (%esp)
+; CHECK-NEXT: calll my_emutls_get_address@PLT
+; CHECK-NEXT: movl (%eax), %eax
diff --git a/test/CodeGen/X86/fast-isel-gep.ll b/test/CodeGen/X86/fast-isel-gep.ll
index 6c60f1aa7f2..1886d3379aa 100644
--- a/test/CodeGen/X86/fast-isel-gep.ll
+++ b/test/CodeGen/X86/fast-isel-gep.ll
@@ -1,9 +1,6 @@
 ; RUN: llc < %s -mtriple=x86_64-linux -O0 | FileCheck %s --check-prefix=X64
 ; RUN: llc < %s -mtriple=x86_64-windows-itanium -O0 | FileCheck %s --check-prefix=X64
 ; RUN: llc < %s -march=x86 -O0 | FileCheck %s --check-prefix=X32
-; @LOCALMOD-BEGIN
-; RUN: pnacl-llc < %s -mtriple=i686-nacl -march=x86 -O0 | FileCheck %s --check-prefix=NACL32
-; @LOCALMOD-END
 
 ; GEP indices are interpreted as signed integers, so they
 ; should be sign-extended to 64 bits on 64-bit targets.
@@ -21,13 +18,6 @@ define i32 @test1(i32 %t3, i32* %t1) nounwind {
 ; X64:  	movl	(%r[[A1:si|dx]],%rax,4), %eax
 ; X64:  	ret
 
-; @LOCALMOD-BEGIN
-; NACL32: test1:
-; NACL32:   movl (%e{{.*}},%e{{.*}},4), %eax
-; NACL32:   popl %ecx
-; NACL32:   nacljmp %ecx
-; @LOCALMOD-END
-
 }
 define i32 @test2(i64 %t3, i32* %t1) nounwind {
        %t9 = getelementptr i32, i32* %t1, i64 %t3           ; <i32*> [#uses=1]
@@ -40,12 +30,6 @@ define i32 @test2(i64 %t3, i32* %t1) nounwind {
 ; X64-LABEL: test2:
 ; X64:  	movl	(%r[[A1]],%r[[A0]],4), %eax
 ; X64:  	ret
-
-; @LOCALMOD-BEGIN
-; NACL32: test2:
-; NACL32:   movl (%e{{.*}},%e{{.*}},4), %e
-; @LOCALMOD-END
-
 }
 
 
@@ -67,12 +51,6 @@ entry:
 ; X64:  	movb	-2(%r[[A0]]), %al
 ; X64:  	ret
 
-; @LOCALMOD-BEGIN
-; NACL32: test3:
-; NACL32:   movl 4(%esp), %[[REG:e..]]
-; NACL32:   movb -2(%{{.*}}[[REG]]), %al
-; @LOCALMOD-END
-
 }
 
 define double @test4(i64 %x, double* %p) nounwind {
@@ -92,10 +70,6 @@ entry:
 ; X32: 128(%e{{.*}},%e{{.*}},8)
 ; X64-LABEL: test4:
 ; X64: 128(%r{{.*}},%r{{.*}},8)
-; @LOCALMOD-BEGIN
-; NACL32: test4:
-; NACL32: 128(%e{{.*}},%e{{.*}},8)
-; @LOCALMOD-END
 }
 
 ; PR8961 - Make sure the sext for the GEP addressing comes before the load that
@@ -115,7 +89,7 @@ define i64 @test5(i8* %A, i32 %I, i64 %B) nounwind {
 ; PR9500, rdar://9156159 - Don't do non-local address mode folding,
 ; because it may require values which wouldn't otherwise be live out
 ; of their blocks.
-define void @test6() {
+define void @test6() personality i32 (...)* @__gxx_personality_v0 {
 if.end:                                           ; preds = %if.then, %invoke.cont
   %tmp15 = load i64, i64* undef
   %dec = add i64 %tmp15, 13
@@ -129,7 +103,7 @@ invoke.cont16:                                    ; preds = %if.then14
   unreachable
 
 lpad:                                             ; preds = %if.end19, %if.then14, %if.end, %entry
-  %exn = landingpad {i8*, i32} personality i32 (...)* @__gxx_personality_v0
+  %exn = landingpad {i8*, i32}
             cleanup
   unreachable
 }
diff --git a/test/CodeGen/X86/fast-isel-movsbl-indexreg.ll b/test/CodeGen/X86/fast-isel-movsbl-indexreg.ll
new file mode 100644
index 00000000000..100c3297ec4
--- /dev/null
+++ b/test/CodeGen/X86/fast-isel-movsbl-indexreg.ll
@@ -0,0 +1,20 @@
+; RUN: llc %s -o - -verify-machineinstrs -fast-isel=true | FileCheck %s
+
+target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-unknown"
+
+; The index register on the folded memory operand was incorrect.  Ensure we generate
+; movsbl in fast-isel, but also that it passes verification which will check the register
+; class.
+
+; CHECK: movsbl
+
+@table = external hidden global [64 x i8], align 16
+
+define i32 @test(i32 %x, i64 %offset) {
+bb:
+  %tmp37 = getelementptr inbounds [64 x i8], [64 x i8]* @table, i64 0, i64 %offset
+  %tmp38 = load i8, i8* %tmp37, align 1
+  %tmp39 = sext i8 %tmp38 to i32
+  ret i32 %tmp39
+}
diff --git a/test/CodeGen/X86/fast-isel-trunc-kill-subreg.ll b/test/CodeGen/X86/fast-isel-trunc-kill-subreg.ll
new file mode 100644
index 00000000000..96a1a2dea17
--- /dev/null
+++ b/test/CodeGen/X86/fast-isel-trunc-kill-subreg.ll
@@ -0,0 +1,40 @@
+; RUN: llc %s -o - -fast-isel=true -O1 -verify-machineinstrs | FileCheck %s
+
+target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-apple-unknown"
+
+; This test failed the machine verifier because the trunc at the start of the
+; method was extracing a subreg and killing the source register.  The kill flag was
+; invalid here as the source of the trunc could still be used elsewhere.
+
+; CHECK-LABEL: @test
+
+define i32 @test(i32 %block8x8) {
+bb:
+  %tmp9 = trunc i32 %block8x8 to i1
+  %tmp10 = zext i1 %tmp9 to i32
+  %tmp11 = mul i32 %tmp10, 8
+  %tmp12 = zext i32 %tmp11 to i64
+  br label %bb241
+
+bb241:                                            ; preds = %bb241, %bb
+  %lsr.iv3 = phi i64 [ %lsr.iv.next4, %bb241 ], [ %tmp12, %bb ]
+  %lsr.iv1 = phi i32 [ %lsr.iv.next2, %bb241 ], [ 0, %bb ]
+  %lsr.iv.next2 = add nuw nsw i32 %lsr.iv1, 1
+  %lsr.iv.next4 = add i64 %lsr.iv3, 32
+  %exitcond = icmp eq i32 %lsr.iv.next2, 8
+  br i1 %exitcond, label %.preheader.preheader, label %bb241
+
+.preheader.preheader:                             ; preds = %bb241
+  %tmp18 = lshr i32 %block8x8, 1
+  br label %bb270
+
+bb270:                                            ; preds = %bb270, %.preheader.preheader
+  %lsr.iv = phi i32 [ %lsr.iv.next, %bb270 ], [ %tmp18, %.preheader.preheader ]
+  %lsr.iv.next = add i32 %lsr.iv, 4
+  %tmp272 = icmp slt i32 %lsr.iv.next, 100
+  br i1 %tmp272, label %bb270, label %.loopexit
+
+.loopexit:                                        ; preds = %bb270
+  ret i32 0
+}
diff --git a/test/CodeGen/X86/fast-isel-x86-64.ll b/test/CodeGen/X86/fast-isel-x86-64.ll
index 6813aed3322..d748cba2f8f 100644
--- a/test/CodeGen/X86/fast-isel-x86-64.ll
+++ b/test/CodeGen/X86/fast-isel-x86-64.ll
@@ -1,7 +1,5 @@
 ; RUN: llc < %s -mattr=-avx -fast-isel -mcpu=core2 -O0 -regalloc=fast -asm-verbose=0 -fast-isel-abort=1 | FileCheck %s
 ; RUN: llc < %s -mattr=+avx -fast-isel -mcpu=core2 -O0 -regalloc=fast -asm-verbose=0 -fast-isel-abort=1 | FileCheck %s --check-prefix=AVX
-; RUN: llc < %s -fast-isel -O0 -regalloc=fast -asm-verbose=0 -fast-isel-abort=1 -mtriple=x86_64-none-nacl | FileCheck %s --check-prefix=NACL64
-; RUN: llc < %s -fast-isel -O0 -regalloc=fast -asm-verbose=0 -fast-isel-abort=1 -mtriple=x86_64-none-nacl -relocation-model=pic | FileCheck %s --check-prefix=NACL64_PIC
 
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64"
 target triple = "x86_64-apple-darwin10.0.0"
@@ -48,16 +46,6 @@ define i64 @test3() nounwind {
 ; CHECK-NEXT: ret
 }
 
-; NACL64 version uses i32 for 32-bit pointers.
-define i32 @test3_nacl64() nounwind {
-  %A = ptrtoint i32* @G to i32
-  ret i32 %A
-
-; NACL64_PIC: test3_nacl64:
-; NACL64_PIC: movl G@GOTPCREL(%rip), %eax
-; NACL64_PIC-NEXT: popq    %r11
-; NACL64_PIC-NEXT: nacljmp %r11d, %r15
-}
 
 
 ; rdar://9289558
@@ -311,11 +299,6 @@ define void @test23(i8* noalias sret %result) {
 ; CHECK: call
 ; CHECK: movq  %rdi, %rax
 ; CHECK: ret
-; NACL64: test23:
-; NACL64: call
-; NACL64: movl  %edi, %eax
-; NACL64: popq %r11
-; NACL64: nacljmp %r11d, %r15
 }
 
 declare i8* @foo23()
diff --git a/test/CodeGen/X86/fcmove.ll b/test/CodeGen/X86/fcmove.ll
new file mode 100644
index 00000000000..21cc683f734
--- /dev/null
+++ b/test/CodeGen/X86/fcmove.ll
@@ -0,0 +1,15 @@
+; RUN: llc %s -o - -verify-machineinstrs | FileCheck %s
+
+target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-unknown"
+
+; Test that we can generate an fcmove, and also that it passes verification.
+
+; CHECK-LABEL: cmove_f
+; CHECK: fcmove %st({{[0-7]}}), %st(0)
+define x86_fp80 @cmove_f(x86_fp80 %a, x86_fp80 %b, i32 %c) {
+  %test = icmp eq i32 %c, 0
+  %add = fadd x86_fp80 %a, %b
+  %ret = select i1 %test, x86_fp80 %add, x86_fp80 %b
+  ret x86_fp80 %ret
+}
\ No newline at end of file
diff --git a/test/CodeGen/X86/fdiv-combine.ll b/test/CodeGen/X86/fdiv-combine.ll
index 279bb0624ac..b65e9d01ab8 100644
--- a/test/CodeGen/X86/fdiv-combine.ll
+++ b/test/CodeGen/X86/fdiv-combine.ll
@@ -1,4 +1,4 @@
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown | FileCheck %s
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mcpu=x86-64 | FileCheck %s
 
 ; Anything more than one division using a single divisor operand
 ; should be converted into a reciprocal and multiplication.
@@ -17,9 +17,9 @@ define float @div2_arcp(float %x, float %y, float %z) #0 {
 ; CHECK:       # BB#0:
 ; CHECK-NEXT:    movss {{.*#+}} xmm3 = mem[0],zero,zero,zero
 ; CHECK-NEXT:    divss %xmm2, %xmm3
-; CHECK-NEXT:    mulss %xmm3, %xmm0
 ; CHECK-NEXT:    mulss %xmm1, %xmm0
 ; CHECK-NEXT:    mulss %xmm3, %xmm0
+; CHECK-NEXT:    mulss %xmm3, %xmm0
 ; CHECK-NEXT:    retq
   %div1 = fdiv arcp float %x, %z
   %mul = fmul arcp float %div1, %y
@@ -27,5 +27,41 @@ define float @div2_arcp(float %x, float %y, float %z) #0 {
   ret float %div2
 }
 
+; If the reciprocal is already calculated, we should not
+; generate an extra multiplication by 1.0. 
+
+define double @div3_arcp(double %x, double %y, double %z) #0 {
+; CHECK-LABEL: div3_arcp:
+; CHECK:       # BB#0:
+; CHECK-NEXT:    movsd{{.*#+}} xmm2 = mem[0],zero
+; CHECK-NEXT:    divsd %xmm1, %xmm2
+; CHECK-NEXT:    mulsd %xmm2, %xmm0
+; CHECK-NEXT:    addsd %xmm2, %xmm0
+; CHECK-NEXT:    retq
+  %div1 = fdiv fast double 1.0, %y
+  %div2 = fdiv fast double %x, %y
+  %ret = fadd fast double %div2, %div1
+  ret double %ret
+}
+
+define void @PR24141() #0 {
+; CHECK-LABEL: PR24141:
+; CHECK:	callq
+; CHECK-NEXT:	divsd
+; CHECK-NEXT:	jmp
+entry:
+  br label %while.body
+
+while.body:
+  %x.0 = phi double [ undef, %entry ], [ %div, %while.body ]
+  %call = call { double, double } @g(double %x.0)
+  %xv0 = extractvalue { double, double } %call, 0
+  %xv1 = extractvalue { double, double } %call, 1
+  %div = fdiv double %xv0, %xv1
+  br label %while.body
+}
+
+declare { double, double } @g(double)
+
 ; FIXME: If the backend understands 'arcp', then this attribute is unnecessary.
 attributes #0 = { "unsafe-fp-math"="true" }
diff --git a/test/CodeGen/X86/floor-soft-float.ll b/test/CodeGen/X86/floor-soft-float.ll
index 5644509a86f..7bb738513f5 100644
--- a/test/CodeGen/X86/floor-soft-float.ll
+++ b/test/CodeGen/X86/floor-soft-float.ll
@@ -1,5 +1,5 @@
-; RUN: llc < %s -march=x86-64 -mattr=+sse4.1,-avx -soft-float=0 | FileCheck %s --check-prefix=CHECK-HARD-FLOAT
-; RUN: llc < %s -march=x86-64 -mattr=+sse4.1,-avx -soft-float=1 | FileCheck %s --check-prefix=CHECK-SOFT-FLOAT
+; RUN: llc < %s -march=x86-64 -mattr=+sse4.1,-avx | FileCheck %s --check-prefix=CHECK-HARD-FLOAT
+; RUN: llc < %s -march=x86-64 -mattr=+sse4.1,-avx,+soft-float | FileCheck %s --check-prefix=CHECK-SOFT-FLOAT
 
 target triple = "x86_64-unknown-linux-gnu"
 
diff --git a/test/CodeGen/X86/fma-intrinsics-phi-213-to-231.ll b/test/CodeGen/X86/fma-intrinsics-phi-213-to-231.ll
new file mode 100644
index 00000000000..f7d0cdf3c65
--- /dev/null
+++ b/test/CodeGen/X86/fma-intrinsics-phi-213-to-231.ll
@@ -0,0 +1,204 @@
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx2,+fma | FileCheck %s
+
+; CHECK-LABEL: fmaddsubpd_loop:
+; CHECK:   vfmaddsub231pd        %ymm{{[0-9]+}}, %ymm{{[0-9]+}}, %ymm{{[0-9]+}}
+define <4 x double> @fmaddsubpd_loop(i32 %iter, <4 x double> %a, <4 x double> %b, <4 x double> %c) {
+entry:
+  br label %for.cond
+
+for.cond:
+  %c.addr.0 = phi <4 x double> [ %c, %entry ], [ %0, %for.inc ]
+  %i.0 = phi i32 [ 0, %entry ], [ %inc, %for.inc ]
+  %cmp = icmp slt i32 %i.0, %iter
+  br i1 %cmp, label %for.body, label %for.end
+
+for.body:
+  br label %for.inc
+
+for.inc:
+  %0 = call <4 x double> @llvm.x86.fma.vfmaddsub.pd.256(<4 x double> %a, <4 x double> %b, <4 x double> %c.addr.0)
+  %inc = add nsw i32 %i.0, 1
+  br label %for.cond
+
+for.end:
+  ret <4 x double> %c.addr.0
+}
+
+; CHECK-LABEL: fmsubaddpd_loop:
+; CHECK:   vfmsubadd231pd        %ymm{{[0-9]+}}, %ymm{{[0-9]+}}, %ymm{{[0-9]+}}
+define <4 x double> @fmsubaddpd_loop(i32 %iter, <4 x double> %a, <4 x double> %b, <4 x double> %c) {
+entry:
+  br label %for.cond
+
+for.cond:
+  %c.addr.0 = phi <4 x double> [ %c, %entry ], [ %0, %for.inc ]
+  %i.0 = phi i32 [ 0, %entry ], [ %inc, %for.inc ]
+  %cmp = icmp slt i32 %i.0, %iter
+  br i1 %cmp, label %for.body, label %for.end
+
+for.body:
+  br label %for.inc
+
+for.inc:
+  %0 = call <4 x double> @llvm.x86.fma.vfmsubadd.pd.256(<4 x double> %a, <4 x double> %b, <4 x double> %c.addr.0)
+  %inc = add nsw i32 %i.0, 1
+  br label %for.cond
+
+for.end:
+  ret <4 x double> %c.addr.0
+}
+
+; CHECK-LABEL: fmaddpd_loop:
+; CHECK:   vfmadd231pd        %ymm{{[0-9]+}}, %ymm{{[0-9]+}}, %ymm{{[0-9]+}}
+define <4 x double> @fmaddpd_loop(i32 %iter, <4 x double> %a, <4 x double> %b, <4 x double> %c) {
+entry:
+  br label %for.cond
+
+for.cond:
+  %c.addr.0 = phi <4 x double> [ %c, %entry ], [ %0, %for.inc ]
+  %i.0 = phi i32 [ 0, %entry ], [ %inc, %for.inc ]
+  %cmp = icmp slt i32 %i.0, %iter
+  br i1 %cmp, label %for.body, label %for.end
+
+for.body:
+  br label %for.inc
+
+for.inc:
+  %0 = call <4 x double> @llvm.x86.fma.vfmadd.pd.256(<4 x double> %a, <4 x double> %b, <4 x double> %c.addr.0)
+  %inc = add nsw i32 %i.0, 1
+  br label %for.cond
+
+for.end:
+  ret <4 x double> %c.addr.0
+}
+
+; CHECK-LABEL: fmsubpd_loop:
+; CHECK:   vfmsub231pd        %ymm{{[0-9]+}}, %ymm{{[0-9]+}}, %ymm{{[0-9]+}}
+define <4 x double> @fmsubpd_loop(i32 %iter, <4 x double> %a, <4 x double> %b, <4 x double> %c) {
+entry:
+  br label %for.cond
+
+for.cond:
+  %c.addr.0 = phi <4 x double> [ %c, %entry ], [ %0, %for.inc ]
+  %i.0 = phi i32 [ 0, %entry ], [ %inc, %for.inc ]
+  %cmp = icmp slt i32 %i.0, %iter
+  br i1 %cmp, label %for.body, label %for.end
+
+for.body:
+  br label %for.inc
+
+for.inc:
+  %0 = call <4 x double> @llvm.x86.fma.vfmsub.pd.256(<4 x double> %a, <4 x double> %b, <4 x double> %c.addr.0)
+  %inc = add nsw i32 %i.0, 1
+  br label %for.cond
+
+for.end:
+  ret <4 x double> %c.addr.0
+}
+
+declare <4 x double> @llvm.x86.fma.vfmaddsub.pd.256(<4 x double>, <4 x double>, <4 x double>)
+declare <4 x double> @llvm.x86.fma.vfmsubadd.pd.256(<4 x double>, <4 x double>, <4 x double>)
+declare <4 x double> @llvm.x86.fma.vfmadd.pd.256(<4 x double>, <4 x double>, <4 x double>)
+declare <4 x double> @llvm.x86.fma.vfmsub.pd.256(<4 x double>, <4 x double>, <4 x double>)
+
+
+; CHECK-LABEL: fmaddsubps_loop:
+; CHECK:   vfmaddsub231ps        %ymm{{[0-9]+}}, %ymm{{[0-9]+}}, %ymm{{[0-9]+}}
+define <8 x float> @fmaddsubps_loop(i32 %iter, <8 x float> %a, <8 x float> %b, <8 x float> %c) {
+entry:
+  br label %for.cond
+
+for.cond:
+  %c.addr.0 = phi <8 x float> [ %c, %entry ], [ %0, %for.inc ]
+  %i.0 = phi i32 [ 0, %entry ], [ %inc, %for.inc ]
+  %cmp = icmp slt i32 %i.0, %iter
+  br i1 %cmp, label %for.body, label %for.end
+
+for.body:
+  br label %for.inc
+
+for.inc:
+  %0 = call <8 x float> @llvm.x86.fma.vfmaddsub.ps.256(<8 x float> %a, <8 x float> %b, <8 x float> %c.addr.0)
+  %inc = add nsw i32 %i.0, 1
+  br label %for.cond
+
+for.end:
+  ret <8 x float> %c.addr.0
+}
+
+; CHECK-LABEL: fmsubaddps_loop:
+; CHECK:   vfmsubadd231ps        %ymm{{[0-9]+}}, %ymm{{[0-9]+}}, %ymm{{[0-9]+}}
+define <8 x float> @fmsubaddps_loop(i32 %iter, <8 x float> %a, <8 x float> %b, <8 x float> %c) {
+entry:
+  br label %for.cond
+
+for.cond:
+  %c.addr.0 = phi <8 x float> [ %c, %entry ], [ %0, %for.inc ]
+  %i.0 = phi i32 [ 0, %entry ], [ %inc, %for.inc ]
+  %cmp = icmp slt i32 %i.0, %iter
+  br i1 %cmp, label %for.body, label %for.end
+
+for.body:
+  br label %for.inc
+
+for.inc:
+  %0 = call <8 x float> @llvm.x86.fma.vfmsubadd.ps.256(<8 x float> %a, <8 x float> %b, <8 x float> %c.addr.0)
+  %inc = add nsw i32 %i.0, 1
+  br label %for.cond
+
+for.end:
+  ret <8 x float> %c.addr.0
+}
+
+; CHECK-LABEL: fmaddps_loop:
+; CHECK:   vfmadd231ps        %ymm{{[0-9]+}}, %ymm{{[0-9]+}}, %ymm{{[0-9]+}}
+define <8 x float> @fmaddps_loop(i32 %iter, <8 x float> %a, <8 x float> %b, <8 x float> %c) {
+entry:
+  br label %for.cond
+
+for.cond:
+  %c.addr.0 = phi <8 x float> [ %c, %entry ], [ %0, %for.inc ]
+  %i.0 = phi i32 [ 0, %entry ], [ %inc, %for.inc ]
+  %cmp = icmp slt i32 %i.0, %iter
+  br i1 %cmp, label %for.body, label %for.end
+
+for.body:
+  br label %for.inc
+
+for.inc:
+  %0 = call <8 x float> @llvm.x86.fma.vfmadd.ps.256(<8 x float> %a, <8 x float> %b, <8 x float> %c.addr.0)
+  %inc = add nsw i32 %i.0, 1
+  br label %for.cond
+
+for.end:
+  ret <8 x float> %c.addr.0
+}
+
+; CHECK-LABEL: fmsubps_loop:
+; CHECK:   vfmsub231ps        %ymm{{[0-9]+}}, %ymm{{[0-9]+}}, %ymm{{[0-9]+}}
+define <8 x float> @fmsubps_loop(i32 %iter, <8 x float> %a, <8 x float> %b, <8 x float> %c) {
+entry:
+  br label %for.cond
+
+for.cond:
+  %c.addr.0 = phi <8 x float> [ %c, %entry ], [ %0, %for.inc ]
+  %i.0 = phi i32 [ 0, %entry ], [ %inc, %for.inc ]
+  %cmp = icmp slt i32 %i.0, %iter
+  br i1 %cmp, label %for.body, label %for.end
+
+for.body:
+  br label %for.inc
+
+for.inc:
+  %0 = call <8 x float> @llvm.x86.fma.vfmsub.ps.256(<8 x float> %a, <8 x float> %b, <8 x float> %c.addr.0)
+  %inc = add nsw i32 %i.0, 1
+  br label %for.cond
+
+for.end:
+  ret <8 x float> %c.addr.0
+}
+
+declare <8 x float> @llvm.x86.fma.vfmaddsub.ps.256(<8 x float>, <8 x float>, <8 x float>)
+declare <8 x float> @llvm.x86.fma.vfmsubadd.ps.256(<8 x float>, <8 x float>, <8 x float>)
+declare <8 x float> @llvm.x86.fma.vfmadd.ps.256(<8 x float>, <8 x float>, <8 x float>)
+declare <8 x float> @llvm.x86.fma.vfmsub.ps.256(<8 x float>, <8 x float>, <8 x float>)
diff --git a/test/CodeGen/X86/fma-intrinsics-x86.ll b/test/CodeGen/X86/fma-intrinsics-x86.ll
new file mode 100644
index 00000000000..881436386ba
--- /dev/null
+++ b/test/CodeGen/X86/fma-intrinsics-x86.ll
@@ -0,0 +1,493 @@
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -march=x86-64 -mcpu=corei7-avx -mattr=+fma | FileCheck %s --check-prefix=CHECK --check-prefix=CHECK-FMA
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -march=x86-64 -mcpu=core-avx2 -mattr=+fma,+avx2 | FileCheck %s --check-prefix=CHECK --check-prefix=CHECK-FMA
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -march=x86-64 -mcpu=corei7-avx -mattr=+fma4 | FileCheck %s --check-prefix=CHECK --check-prefix=CHECK-FMA4
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mcpu=bdver2 -mattr=+avx,-fma | FileCheck %s --check-prefix=CHECK --check-prefix=CHECK-FMA4
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mcpu=bdver2 -mattr=-fma4 | FileCheck %s --check-prefix=CHECK --check-prefix=CHECK-FMA
+
+; VFMADD
+define <4 x float> @test_x86_fma_vfmadd_ss(<4 x float> %a0, <4 x float> %a1, <4 x float> %a2) #0 {
+; CHECK-FMA-LABEL: test_x86_fma_vfmadd_ss:
+; CHECK-FMA:       # BB#0:
+; CHECK-FMA-NEXT:    vfmadd213ss %xmm2, %xmm1, %xmm0
+; CHECK-FMA-NEXT:    retq
+;
+; CHECK-FMA4-LABEL: test_x86_fma_vfmadd_ss:
+; CHECK-FMA4:       # BB#0:
+; CHECK-FMA4-NEXT:    vfmaddss %xmm2, %xmm1, %xmm0, %xmm0
+; CHECK-FMA4-NEXT:    retq
+  %res = call <4 x float> @llvm.x86.fma.vfmadd.ss(<4 x float> %a0, <4 x float> %a1, <4 x float> %a2)
+  ret <4 x float> %res
+}
+declare <4 x float> @llvm.x86.fma.vfmadd.ss(<4 x float>, <4 x float>, <4 x float>)
+
+define <2 x double> @test_x86_fma_vfmadd_sd(<2 x double> %a0, <2 x double> %a1, <2 x double> %a2) #0 {
+; CHECK-FMA-LABEL: test_x86_fma_vfmadd_sd:
+; CHECK-FMA:       # BB#0:
+; CHECK-FMA-NEXT:    vfmadd213sd %xmm2, %xmm1, %xmm0
+; CHECK-FMA-NEXT:    retq
+;
+; CHECK-FMA4-LABEL: test_x86_fma_vfmadd_sd:
+; CHECK-FMA4:       # BB#0:
+; CHECK-FMA4-NEXT:    vfmaddsd %xmm2, %xmm1, %xmm0, %xmm0
+; CHECK-FMA4-NEXT:    retq
+  %res = call <2 x double> @llvm.x86.fma.vfmadd.sd(<2 x double> %a0, <2 x double> %a1, <2 x double> %a2)
+  ret <2 x double> %res
+}
+declare <2 x double> @llvm.x86.fma.vfmadd.sd(<2 x double>, <2 x double>, <2 x double>)
+
+define <4 x float> @test_x86_fma_vfmadd_ps(<4 x float> %a0, <4 x float> %a1, <4 x float> %a2) #0 {
+; CHECK-FMA-LABEL: test_x86_fma_vfmadd_ps:
+; CHECK-FMA:       # BB#0:
+; CHECK-FMA-NEXT:    vfmadd213ps %xmm2, %xmm1, %xmm0
+; CHECK-FMA-NEXT:    retq
+;
+; CHECK-FMA4-LABEL: test_x86_fma_vfmadd_ps:
+; CHECK-FMA4:       # BB#0:
+; CHECK-FMA4-NEXT:    vfmaddps %xmm2, %xmm1, %xmm0, %xmm0
+; CHECK-FMA4-NEXT:    retq
+  %res = call <4 x float> @llvm.x86.fma.vfmadd.ps(<4 x float> %a0, <4 x float> %a1, <4 x float> %a2)
+  ret <4 x float> %res
+}
+declare <4 x float> @llvm.x86.fma.vfmadd.ps(<4 x float>, <4 x float>, <4 x float>)
+
+define <2 x double> @test_x86_fma_vfmadd_pd(<2 x double> %a0, <2 x double> %a1, <2 x double> %a2) #0 {
+; CHECK-FMA-LABEL: test_x86_fma_vfmadd_pd:
+; CHECK-FMA:       # BB#0:
+; CHECK-FMA-NEXT:    vfmadd213pd %xmm2, %xmm1, %xmm0
+; CHECK-FMA-NEXT:    retq
+;
+; CHECK-FMA4-LABEL: test_x86_fma_vfmadd_pd:
+; CHECK-FMA4:       # BB#0:
+; CHECK-FMA4-NEXT:    vfmaddpd %xmm2, %xmm1, %xmm0, %xmm0
+; CHECK-FMA4-NEXT:    retq
+  %res = call <2 x double> @llvm.x86.fma.vfmadd.pd(<2 x double> %a0, <2 x double> %a1, <2 x double> %a2)
+  ret <2 x double> %res
+}
+declare <2 x double> @llvm.x86.fma.vfmadd.pd(<2 x double>, <2 x double>, <2 x double>)
+
+define <8 x float> @test_x86_fma_vfmadd_ps_256(<8 x float> %a0, <8 x float> %a1, <8 x float> %a2) #0 {
+; CHECK-FMA-LABEL: test_x86_fma_vfmadd_ps_256:
+; CHECK-FMA:       # BB#0:
+; CHECK-FMA-NEXT:    vfmadd213ps %ymm2, %ymm1, %ymm0
+; CHECK-FMA-NEXT:    retq
+;
+; CHECK-FMA4-LABEL: test_x86_fma_vfmadd_ps_256:
+; CHECK-FMA4:       # BB#0:
+; CHECK-FMA4-NEXT:    vfmaddps %ymm2, %ymm1, %ymm0, %ymm0
+; CHECK-FMA4-NEXT:    retq
+  %res = call <8 x float> @llvm.x86.fma.vfmadd.ps.256(<8 x float> %a0, <8 x float> %a1, <8 x float> %a2)
+  ret <8 x float> %res
+}
+declare <8 x float> @llvm.x86.fma.vfmadd.ps.256(<8 x float>, <8 x float>, <8 x float>)
+
+define <4 x double> @test_x86_fma_vfmadd_pd_256(<4 x double> %a0, <4 x double> %a1, <4 x double> %a2) #0 {
+; CHECK-FMA-LABEL: test_x86_fma_vfmadd_pd_256:
+; CHECK-FMA:       # BB#0:
+; CHECK-FMA-NEXT:    vfmadd213pd %ymm2, %ymm1, %ymm0
+; CHECK-FMA-NEXT:    retq
+;
+; CHECK-FMA4-LABEL: test_x86_fma_vfmadd_pd_256:
+; CHECK-FMA4:       # BB#0:
+; CHECK-FMA4-NEXT:    vfmaddpd %ymm2, %ymm1, %ymm0, %ymm0
+; CHECK-FMA4-NEXT:    retq
+  %res = call <4 x double> @llvm.x86.fma.vfmadd.pd.256(<4 x double> %a0, <4 x double> %a1, <4 x double> %a2)
+  ret <4 x double> %res
+}
+declare <4 x double> @llvm.x86.fma.vfmadd.pd.256(<4 x double>, <4 x double>, <4 x double>)
+
+; VFMSUB
+define <4 x float> @test_x86_fma_vfmsub_ss(<4 x float> %a0, <4 x float> %a1, <4 x float> %a2) #0 {
+; CHECK-FMA-LABEL: test_x86_fma_vfmsub_ss:
+; CHECK-FMA:       # BB#0:
+; CHECK-FMA-NEXT:    vfmsub213ss %xmm2, %xmm1, %xmm0
+; CHECK-FMA-NEXT:    retq
+;
+; CHECK-FMA4-LABEL: test_x86_fma_vfmsub_ss:
+; CHECK-FMA4:       # BB#0:
+; CHECK-FMA4-NEXT:    vfmsubss %xmm2, %xmm1, %xmm0, %xmm0
+; CHECK-FMA4-NEXT:    retq
+  %res = call <4 x float> @llvm.x86.fma.vfmsub.ss(<4 x float> %a0, <4 x float> %a1, <4 x float> %a2)
+  ret <4 x float> %res
+}
+declare <4 x float> @llvm.x86.fma.vfmsub.ss(<4 x float>, <4 x float>, <4 x float>)
+
+define <2 x double> @test_x86_fma_vfmsub_sd(<2 x double> %a0, <2 x double> %a1, <2 x double> %a2) #0 {
+; CHECK-FMA-LABEL: test_x86_fma_vfmsub_sd:
+; CHECK-FMA:       # BB#0:
+; CHECK-FMA-NEXT:    vfmsub213sd %xmm2, %xmm1, %xmm0
+; CHECK-FMA-NEXT:    retq
+;
+; CHECK-FMA4-LABEL: test_x86_fma_vfmsub_sd:
+; CHECK-FMA4:       # BB#0:
+; CHECK-FMA4-NEXT:    vfmsubsd %xmm2, %xmm1, %xmm0, %xmm0
+; CHECK-FMA4-NEXT:    retq
+  %res = call <2 x double> @llvm.x86.fma.vfmsub.sd(<2 x double> %a0, <2 x double> %a1, <2 x double> %a2)
+  ret <2 x double> %res
+}
+declare <2 x double> @llvm.x86.fma.vfmsub.sd(<2 x double>, <2 x double>, <2 x double>)
+
+define <4 x float> @test_x86_fma_vfmsub_ps(<4 x float> %a0, <4 x float> %a1, <4 x float> %a2) #0 {
+; CHECK-FMA-LABEL: test_x86_fma_vfmsub_ps:
+; CHECK-FMA:       # BB#0:
+; CHECK-FMA-NEXT:    vfmsub213ps %xmm2, %xmm1, %xmm0
+; CHECK-FMA-NEXT:    retq
+;
+; CHECK-FMA4-LABEL: test_x86_fma_vfmsub_ps:
+; CHECK-FMA4:       # BB#0:
+; CHECK-FMA4-NEXT:    vfmsubps %xmm2, %xmm1, %xmm0, %xmm0
+; CHECK-FMA4-NEXT:    retq
+  %res = call <4 x float> @llvm.x86.fma.vfmsub.ps(<4 x float> %a0, <4 x float> %a1, <4 x float> %a2)
+  ret <4 x float> %res
+}
+declare <4 x float> @llvm.x86.fma.vfmsub.ps(<4 x float>, <4 x float>, <4 x float>)
+
+define <2 x double> @test_x86_fma_vfmsub_pd(<2 x double> %a0, <2 x double> %a1, <2 x double> %a2) #0 {
+; CHECK-FMA-LABEL: test_x86_fma_vfmsub_pd:
+; CHECK-FMA:       # BB#0:
+; CHECK-FMA-NEXT:    vfmsub213pd %xmm2, %xmm1, %xmm0
+; CHECK-FMA-NEXT:    retq
+;
+; CHECK-FMA4-LABEL: test_x86_fma_vfmsub_pd:
+; CHECK-FMA4:       # BB#0:
+; CHECK-FMA4-NEXT:    vfmsubpd %xmm2, %xmm1, %xmm0, %xmm0
+; CHECK-FMA4-NEXT:    retq
+  %res = call <2 x double> @llvm.x86.fma.vfmsub.pd(<2 x double> %a0, <2 x double> %a1, <2 x double> %a2)
+  ret <2 x double> %res
+}
+declare <2 x double> @llvm.x86.fma.vfmsub.pd(<2 x double>, <2 x double>, <2 x double>)
+
+define <8 x float> @test_x86_fma_vfmsub_ps_256(<8 x float> %a0, <8 x float> %a1, <8 x float> %a2) #0 {
+; CHECK-FMA-LABEL: test_x86_fma_vfmsub_ps_256:
+; CHECK-FMA:       # BB#0:
+; CHECK-FMA-NEXT:    vfmsub213ps %ymm2, %ymm1, %ymm0
+; CHECK-FMA-NEXT:    retq
+;
+; CHECK-FMA4-LABEL: test_x86_fma_vfmsub_ps_256:
+; CHECK-FMA4:       # BB#0:
+; CHECK-FMA4-NEXT:    vfmsubps %ymm2, %ymm1, %ymm0, %ymm0
+; CHECK-FMA4-NEXT:    retq
+  %res = call <8 x float> @llvm.x86.fma.vfmsub.ps.256(<8 x float> %a0, <8 x float> %a1, <8 x float> %a2)
+  ret <8 x float> %res
+}
+declare <8 x float> @llvm.x86.fma.vfmsub.ps.256(<8 x float>, <8 x float>, <8 x float>)
+
+define <4 x double> @test_x86_fma_vfmsub_pd_256(<4 x double> %a0, <4 x double> %a1, <4 x double> %a2) #0 {
+; CHECK-FMA-LABEL: test_x86_fma_vfmsub_pd_256:
+; CHECK-FMA:       # BB#0:
+; CHECK-FMA-NEXT:    vfmsub213pd %ymm2, %ymm1, %ymm0
+; CHECK-FMA-NEXT:    retq
+;
+; CHECK-FMA4-LABEL: test_x86_fma_vfmsub_pd_256:
+; CHECK-FMA4:       # BB#0:
+; CHECK-FMA4-NEXT:    vfmsubpd %ymm2, %ymm1, %ymm0, %ymm0
+; CHECK-FMA4-NEXT:    retq
+  %res = call <4 x double> @llvm.x86.fma.vfmsub.pd.256(<4 x double> %a0, <4 x double> %a1, <4 x double> %a2)
+  ret <4 x double> %res
+}
+declare <4 x double> @llvm.x86.fma.vfmsub.pd.256(<4 x double>, <4 x double>, <4 x double>)
+
+; VFNMADD
+define <4 x float> @test_x86_fma_vfnmadd_ss(<4 x float> %a0, <4 x float> %a1, <4 x float> %a2) #0 {
+; CHECK-FMA-LABEL: test_x86_fma_vfnmadd_ss:
+; CHECK-FMA:       # BB#0:
+; CHECK-FMA-NEXT:    vfnmadd213ss %xmm2, %xmm1, %xmm0
+; CHECK-FMA-NEXT:    retq
+;
+; CHECK-FMA4-LABEL: test_x86_fma_vfnmadd_ss:
+; CHECK-FMA4:       # BB#0:
+; CHECK-FMA4-NEXT:    vfnmaddss %xmm2, %xmm1, %xmm0, %xmm0
+; CHECK-FMA4-NEXT:    retq
+  %res = call <4 x float> @llvm.x86.fma.vfnmadd.ss(<4 x float> %a0, <4 x float> %a1, <4 x float> %a2)
+  ret <4 x float> %res
+}
+declare <4 x float> @llvm.x86.fma.vfnmadd.ss(<4 x float>, <4 x float>, <4 x float>)
+
+define <2 x double> @test_x86_fma_vfnmadd_sd(<2 x double> %a0, <2 x double> %a1, <2 x double> %a2) #0 {
+; CHECK-FMA-LABEL: test_x86_fma_vfnmadd_sd:
+; CHECK-FMA:       # BB#0:
+; CHECK-FMA-NEXT:    vfnmadd213sd %xmm2, %xmm1, %xmm0
+; CHECK-FMA-NEXT:    retq
+;
+; CHECK-FMA4-LABEL: test_x86_fma_vfnmadd_sd:
+; CHECK-FMA4:       # BB#0:
+; CHECK-FMA4-NEXT:    vfnmaddsd %xmm2, %xmm1, %xmm0, %xmm0
+; CHECK-FMA4-NEXT:    retq
+  %res = call <2 x double> @llvm.x86.fma.vfnmadd.sd(<2 x double> %a0, <2 x double> %a1, <2 x double> %a2)
+  ret <2 x double> %res
+}
+declare <2 x double> @llvm.x86.fma.vfnmadd.sd(<2 x double>, <2 x double>, <2 x double>)
+
+define <4 x float> @test_x86_fma_vfnmadd_ps(<4 x float> %a0, <4 x float> %a1, <4 x float> %a2) #0 {
+; CHECK-FMA-LABEL: test_x86_fma_vfnmadd_ps:
+; CHECK-FMA:       # BB#0:
+; CHECK-FMA-NEXT:    vfnmadd213ps %xmm2, %xmm1, %xmm0
+; CHECK-FMA-NEXT:    retq
+;
+; CHECK-FMA4-LABEL: test_x86_fma_vfnmadd_ps:
+; CHECK-FMA4:       # BB#0:
+; CHECK-FMA4-NEXT:    vfnmaddps %xmm2, %xmm1, %xmm0, %xmm0
+; CHECK-FMA4-NEXT:    retq
+  %res = call <4 x float> @llvm.x86.fma.vfnmadd.ps(<4 x float> %a0, <4 x float> %a1, <4 x float> %a2)
+  ret <4 x float> %res
+}
+declare <4 x float> @llvm.x86.fma.vfnmadd.ps(<4 x float>, <4 x float>, <4 x float>)
+
+define <2 x double> @test_x86_fma_vfnmadd_pd(<2 x double> %a0, <2 x double> %a1, <2 x double> %a2) #0 {
+; CHECK-FMA-LABEL: test_x86_fma_vfnmadd_pd:
+; CHECK-FMA:       # BB#0:
+; CHECK-FMA-NEXT:    vfnmadd213pd %xmm2, %xmm1, %xmm0
+; CHECK-FMA-NEXT:    retq
+;
+; CHECK-FMA4-LABEL: test_x86_fma_vfnmadd_pd:
+; CHECK-FMA4:       # BB#0:
+; CHECK-FMA4-NEXT:    vfnmaddpd %xmm2, %xmm1, %xmm0, %xmm0
+; CHECK-FMA4-NEXT:    retq
+  %res = call <2 x double> @llvm.x86.fma.vfnmadd.pd(<2 x double> %a0, <2 x double> %a1, <2 x double> %a2)
+  ret <2 x double> %res
+}
+declare <2 x double> @llvm.x86.fma.vfnmadd.pd(<2 x double>, <2 x double>, <2 x double>)
+
+define <8 x float> @test_x86_fma_vfnmadd_ps_256(<8 x float> %a0, <8 x float> %a1, <8 x float> %a2) #0 {
+; CHECK-FMA-LABEL: test_x86_fma_vfnmadd_ps_256:
+; CHECK-FMA:       # BB#0:
+; CHECK-FMA-NEXT:    vfnmadd213ps %ymm2, %ymm1, %ymm0
+; CHECK-FMA-NEXT:    retq
+;
+; CHECK-FMA4-LABEL: test_x86_fma_vfnmadd_ps_256:
+; CHECK-FMA4:       # BB#0:
+; CHECK-FMA4-NEXT:    vfnmaddps %ymm2, %ymm1, %ymm0, %ymm0
+; CHECK-FMA4-NEXT:    retq
+  %res = call <8 x float> @llvm.x86.fma.vfnmadd.ps.256(<8 x float> %a0, <8 x float> %a1, <8 x float> %a2)
+  ret <8 x float> %res
+}
+declare <8 x float> @llvm.x86.fma.vfnmadd.ps.256(<8 x float>, <8 x float>, <8 x float>)
+
+define <4 x double> @test_x86_fma_vfnmadd_pd_256(<4 x double> %a0, <4 x double> %a1, <4 x double> %a2) #0 {
+; CHECK-FMA-LABEL: test_x86_fma_vfnmadd_pd_256:
+; CHECK-FMA:       # BB#0:
+; CHECK-FMA-NEXT:    vfnmadd213pd %ymm2, %ymm1, %ymm0
+; CHECK-FMA-NEXT:    retq
+;
+; CHECK-FMA4-LABEL: test_x86_fma_vfnmadd_pd_256:
+; CHECK-FMA4:       # BB#0:
+; CHECK-FMA4-NEXT:    vfnmaddpd %ymm2, %ymm1, %ymm0, %ymm0
+; CHECK-FMA4-NEXT:    retq
+  %res = call <4 x double> @llvm.x86.fma.vfnmadd.pd.256(<4 x double> %a0, <4 x double> %a1, <4 x double> %a2)
+  ret <4 x double> %res
+}
+declare <4 x double> @llvm.x86.fma.vfnmadd.pd.256(<4 x double>, <4 x double>, <4 x double>)
+
+; VFNMSUB
+define <4 x float> @test_x86_fma_vfnmsub_ss(<4 x float> %a0, <4 x float> %a1, <4 x float> %a2) #0 {
+; CHECK-FMA-LABEL: test_x86_fma_vfnmsub_ss:
+; CHECK-FMA:       # BB#0:
+; CHECK-FMA-NEXT:    vfnmsub213ss %xmm2, %xmm1, %xmm0
+; CHECK-FMA-NEXT:    retq
+;
+; CHECK-FMA4-LABEL: test_x86_fma_vfnmsub_ss:
+; CHECK-FMA4:       # BB#0:
+; CHECK-FMA4-NEXT:    vfnmsubss %xmm2, %xmm1, %xmm0, %xmm0
+; CHECK-FMA4-NEXT:    retq
+  %res = call <4 x float> @llvm.x86.fma.vfnmsub.ss(<4 x float> %a0, <4 x float> %a1, <4 x float> %a2)
+  ret <4 x float> %res
+}
+declare <4 x float> @llvm.x86.fma.vfnmsub.ss(<4 x float>, <4 x float>, <4 x float>)
+
+define <2 x double> @test_x86_fma_vfnmsub_sd(<2 x double> %a0, <2 x double> %a1, <2 x double> %a2) #0 {
+; CHECK-FMA-LABEL: test_x86_fma_vfnmsub_sd:
+; CHECK-FMA:       # BB#0:
+; CHECK-FMA-NEXT:    vfnmsub213sd %xmm2, %xmm1, %xmm0
+; CHECK-FMA-NEXT:    retq
+;
+; CHECK-FMA4-LABEL: test_x86_fma_vfnmsub_sd:
+; CHECK-FMA4:       # BB#0:
+; CHECK-FMA4-NEXT:    vfnmsubsd %xmm2, %xmm1, %xmm0, %xmm0
+; CHECK-FMA4-NEXT:    retq
+  %res = call <2 x double> @llvm.x86.fma.vfnmsub.sd(<2 x double> %a0, <2 x double> %a1, <2 x double> %a2)
+  ret <2 x double> %res
+}
+declare <2 x double> @llvm.x86.fma.vfnmsub.sd(<2 x double>, <2 x double>, <2 x double>)
+
+define <4 x float> @test_x86_fma_vfnmsub_ps(<4 x float> %a0, <4 x float> %a1, <4 x float> %a2) #0 {
+; CHECK-FMA-LABEL: test_x86_fma_vfnmsub_ps:
+; CHECK-FMA:       # BB#0:
+; CHECK-FMA-NEXT:    vfnmsub213ps %xmm2, %xmm1, %xmm0
+; CHECK-FMA-NEXT:    retq
+;
+; CHECK-FMA4-LABEL: test_x86_fma_vfnmsub_ps:
+; CHECK-FMA4:       # BB#0:
+; CHECK-FMA4-NEXT:    vfnmsubps %xmm2, %xmm1, %xmm0, %xmm0
+; CHECK-FMA4-NEXT:    retq
+  %res = call <4 x float> @llvm.x86.fma.vfnmsub.ps(<4 x float> %a0, <4 x float> %a1, <4 x float> %a2)
+  ret <4 x float> %res
+}
+declare <4 x float> @llvm.x86.fma.vfnmsub.ps(<4 x float>, <4 x float>, <4 x float>)
+
+define <2 x double> @test_x86_fma_vfnmsub_pd(<2 x double> %a0, <2 x double> %a1, <2 x double> %a2) #0 {
+; CHECK-FMA-LABEL: test_x86_fma_vfnmsub_pd:
+; CHECK-FMA:       # BB#0:
+; CHECK-FMA-NEXT:    vfnmsub213pd %xmm2, %xmm1, %xmm0
+; CHECK-FMA-NEXT:    retq
+;
+; CHECK-FMA4-LABEL: test_x86_fma_vfnmsub_pd:
+; CHECK-FMA4:       # BB#0:
+; CHECK-FMA4-NEXT:    vfnmsubpd %xmm2, %xmm1, %xmm0, %xmm0
+; CHECK-FMA4-NEXT:    retq
+  %res = call <2 x double> @llvm.x86.fma.vfnmsub.pd(<2 x double> %a0, <2 x double> %a1, <2 x double> %a2)
+  ret <2 x double> %res
+}
+declare <2 x double> @llvm.x86.fma.vfnmsub.pd(<2 x double>, <2 x double>, <2 x double>)
+
+define <8 x float> @test_x86_fma_vfnmsub_ps_256(<8 x float> %a0, <8 x float> %a1, <8 x float> %a2) #0 {
+; CHECK-FMA-LABEL: test_x86_fma_vfnmsub_ps_256:
+; CHECK-FMA:       # BB#0:
+; CHECK-FMA-NEXT:    vfnmsub213ps %ymm2, %ymm1, %ymm0
+; CHECK-FMA-NEXT:    retq
+;
+; CHECK-FMA4-LABEL: test_x86_fma_vfnmsub_ps_256:
+; CHECK-FMA4:       # BB#0:
+; CHECK-FMA4-NEXT:    vfnmsubps %ymm2, %ymm1, %ymm0, %ymm0
+; CHECK-FMA4-NEXT:    retq
+  %res = call <8 x float> @llvm.x86.fma.vfnmsub.ps.256(<8 x float> %a0, <8 x float> %a1, <8 x float> %a2)
+  ret <8 x float> %res
+}
+declare <8 x float> @llvm.x86.fma.vfnmsub.ps.256(<8 x float>, <8 x float>, <8 x float>)
+
+define <4 x double> @test_x86_fma_vfnmsub_pd_256(<4 x double> %a0, <4 x double> %a1, <4 x double> %a2) #0 {
+; CHECK-FMA-LABEL: test_x86_fma_vfnmsub_pd_256:
+; CHECK-FMA:       # BB#0:
+; CHECK-FMA-NEXT:    vfnmsub213pd %ymm2, %ymm1, %ymm0
+; CHECK-FMA-NEXT:    retq
+;
+; CHECK-FMA4-LABEL: test_x86_fma_vfnmsub_pd_256:
+; CHECK-FMA4:       # BB#0:
+; CHECK-FMA4-NEXT:    vfnmsubpd %ymm2, %ymm1, %ymm0, %ymm0
+; CHECK-FMA4-NEXT:    retq
+  %res = call <4 x double> @llvm.x86.fma.vfnmsub.pd.256(<4 x double> %a0, <4 x double> %a1, <4 x double> %a2)
+  ret <4 x double> %res
+}
+declare <4 x double> @llvm.x86.fma.vfnmsub.pd.256(<4 x double>, <4 x double>, <4 x double>)
+
+; VFMADDSUB
+define <4 x float> @test_x86_fma_vfmaddsub_ps(<4 x float> %a0, <4 x float> %a1, <4 x float> %a2) #0 {
+; CHECK-FMA-LABEL: test_x86_fma_vfmaddsub_ps:
+; CHECK-FMA:       # BB#0:
+; CHECK-FMA-NEXT:    vfmaddsub213ps %xmm2, %xmm1, %xmm0
+; CHECK-FMA-NEXT:    retq
+;
+; CHECK-FMA4-LABEL: test_x86_fma_vfmaddsub_ps:
+; CHECK-FMA4:       # BB#0:
+; CHECK-FMA4-NEXT:    vfmaddsubps %xmm2, %xmm1, %xmm0, %xmm0
+; CHECK-FMA4-NEXT:    retq
+  %res = call <4 x float> @llvm.x86.fma.vfmaddsub.ps(<4 x float> %a0, <4 x float> %a1, <4 x float> %a2)
+  ret <4 x float> %res
+}
+declare <4 x float> @llvm.x86.fma.vfmaddsub.ps(<4 x float>, <4 x float>, <4 x float>)
+
+define <2 x double> @test_x86_fma_vfmaddsub_pd(<2 x double> %a0, <2 x double> %a1, <2 x double> %a2) #0 {
+; CHECK-FMA-LABEL: test_x86_fma_vfmaddsub_pd:
+; CHECK-FMA:       # BB#0:
+; CHECK-FMA-NEXT:    vfmaddsub213pd %xmm2, %xmm1, %xmm0
+; CHECK-FMA-NEXT:    retq
+;
+; CHECK-FMA4-LABEL: test_x86_fma_vfmaddsub_pd:
+; CHECK-FMA4:       # BB#0:
+; CHECK-FMA4-NEXT:    vfmaddsubpd %xmm2, %xmm1, %xmm0, %xmm0
+; CHECK-FMA4-NEXT:    retq
+  %res = call <2 x double> @llvm.x86.fma.vfmaddsub.pd(<2 x double> %a0, <2 x double> %a1, <2 x double> %a2)
+  ret <2 x double> %res
+}
+declare <2 x double> @llvm.x86.fma.vfmaddsub.pd(<2 x double>, <2 x double>, <2 x double>)
+
+define <8 x float> @test_x86_fma_vfmaddsub_ps_256(<8 x float> %a0, <8 x float> %a1, <8 x float> %a2) #0 {
+; CHECK-FMA-LABEL: test_x86_fma_vfmaddsub_ps_256:
+; CHECK-FMA:       # BB#0:
+; CHECK-FMA-NEXT:    vfmaddsub213ps %ymm2, %ymm1, %ymm0
+; CHECK-FMA-NEXT:    retq
+;
+; CHECK-FMA4-LABEL: test_x86_fma_vfmaddsub_ps_256:
+; CHECK-FMA4:       # BB#0:
+; CHECK-FMA4-NEXT:    vfmaddsubps %ymm2, %ymm1, %ymm0, %ymm0
+; CHECK-FMA4-NEXT:    retq
+  %res = call <8 x float> @llvm.x86.fma.vfmaddsub.ps.256(<8 x float> %a0, <8 x float> %a1, <8 x float> %a2)
+  ret <8 x float> %res
+}
+declare <8 x float> @llvm.x86.fma.vfmaddsub.ps.256(<8 x float>, <8 x float>, <8 x float>)
+
+define <4 x double> @test_x86_fma_vfmaddsub_pd_256(<4 x double> %a0, <4 x double> %a1, <4 x double> %a2) #0 {
+; CHECK-FMA-LABEL: test_x86_fma_vfmaddsub_pd_256:
+; CHECK-FMA:       # BB#0:
+; CHECK-FMA-NEXT:    vfmaddsub213pd %ymm2, %ymm1, %ymm0
+; CHECK-FMA-NEXT:    retq
+;
+; CHECK-FMA4-LABEL: test_x86_fma_vfmaddsub_pd_256:
+; CHECK-FMA4:       # BB#0:
+; CHECK-FMA4-NEXT:    vfmaddsubpd %ymm2, %ymm1, %ymm0, %ymm0
+; CHECK-FMA4-NEXT:    retq
+  %res = call <4 x double> @llvm.x86.fma.vfmaddsub.pd.256(<4 x double> %a0, <4 x double> %a1, <4 x double> %a2)
+  ret <4 x double> %res
+}
+declare <4 x double> @llvm.x86.fma.vfmaddsub.pd.256(<4 x double>, <4 x double>, <4 x double>)
+
+; VFMSUBADD
+define <4 x float> @test_x86_fma_vfmsubadd_ps(<4 x float> %a0, <4 x float> %a1, <4 x float> %a2) #0 {
+; CHECK-FMA-LABEL: test_x86_fma_vfmsubadd_ps:
+; CHECK-FMA:       # BB#0:
+; CHECK-FMA-NEXT:    vfmsubadd213ps %xmm2, %xmm1, %xmm0
+; CHECK-FMA-NEXT:    retq
+;
+; CHECK-FMA4-LABEL: test_x86_fma_vfmsubadd_ps:
+; CHECK-FMA4:       # BB#0:
+; CHECK-FMA4-NEXT:    vfmsubaddps %xmm2, %xmm1, %xmm0, %xmm0
+; CHECK-FMA4-NEXT:    retq
+  %res = call <4 x float> @llvm.x86.fma.vfmsubadd.ps(<4 x float> %a0, <4 x float> %a1, <4 x float> %a2)
+  ret <4 x float> %res
+}
+declare <4 x float> @llvm.x86.fma.vfmsubadd.ps(<4 x float>, <4 x float>, <4 x float>)
+
+define <2 x double> @test_x86_fma_vfmsubadd_pd(<2 x double> %a0, <2 x double> %a1, <2 x double> %a2) #0 {
+; CHECK-FMA-LABEL: test_x86_fma_vfmsubadd_pd:
+; CHECK-FMA:       # BB#0:
+; CHECK-FMA-NEXT:    vfmsubadd213pd %xmm2, %xmm1, %xmm0
+; CHECK-FMA-NEXT:    retq
+;
+; CHECK-FMA4-LABEL: test_x86_fma_vfmsubadd_pd:
+; CHECK-FMA4:       # BB#0:
+; CHECK-FMA4-NEXT:    vfmsubaddpd %xmm2, %xmm1, %xmm0, %xmm0
+; CHECK-FMA4-NEXT:    retq
+  %res = call <2 x double> @llvm.x86.fma.vfmsubadd.pd(<2 x double> %a0, <2 x double> %a1, <2 x double> %a2)
+  ret <2 x double> %res
+}
+declare <2 x double> @llvm.x86.fma.vfmsubadd.pd(<2 x double>, <2 x double>, <2 x double>)
+
+define <8 x float> @test_x86_fma_vfmsubadd_ps_256(<8 x float> %a0, <8 x float> %a1, <8 x float> %a2) #0 {
+; CHECK-FMA-LABEL: test_x86_fma_vfmsubadd_ps_256:
+; CHECK-FMA:       # BB#0:
+; CHECK-FMA-NEXT:    vfmsubadd213ps %ymm2, %ymm1, %ymm0
+; CHECK-FMA-NEXT:    retq
+;
+; CHECK-FMA4-LABEL: test_x86_fma_vfmsubadd_ps_256:
+; CHECK-FMA4:       # BB#0:
+; CHECK-FMA4-NEXT:    vfmsubaddps %ymm2, %ymm1, %ymm0, %ymm0
+; CHECK-FMA4-NEXT:    retq
+  %res = call <8 x float> @llvm.x86.fma.vfmsubadd.ps.256(<8 x float> %a0, <8 x float> %a1, <8 x float> %a2)
+  ret <8 x float> %res
+}
+declare <8 x float> @llvm.x86.fma.vfmsubadd.ps.256(<8 x float>, <8 x float>, <8 x float>)
+
+define <4 x double> @test_x86_fma_vfmsubadd_pd_256(<4 x double> %a0, <4 x double> %a1, <4 x double> %a2) #0 {
+; CHECK-FMA-LABEL: test_x86_fma_vfmsubadd_pd_256:
+; CHECK-FMA:       # BB#0:
+; CHECK-FMA-NEXT:    vfmsubadd213pd %ymm2, %ymm1, %ymm0
+; CHECK-FMA-NEXT:    retq
+;
+; CHECK-FMA4-LABEL: test_x86_fma_vfmsubadd_pd_256:
+; CHECK-FMA4:       # BB#0:
+; CHECK-FMA4-NEXT:    vfmsubaddpd %ymm2, %ymm1, %ymm0, %ymm0
+; CHECK-FMA4-NEXT:    retq
+  %res = call <4 x double> @llvm.x86.fma.vfmsubadd.pd.256(<4 x double> %a0, <4 x double> %a1, <4 x double> %a2)
+  ret <4 x double> %res
+}
+declare <4 x double> @llvm.x86.fma.vfmsubadd.pd.256(<4 x double>, <4 x double>, <4 x double>)
+
+attributes #0 = { nounwind }
diff --git a/test/CodeGen/X86/fma-intrinsics-x86_64.ll b/test/CodeGen/X86/fma-intrinsics-x86_64.ll
deleted file mode 100644
index aadd7311bb8..00000000000
--- a/test/CodeGen/X86/fma-intrinsics-x86_64.ll
+++ /dev/null
@@ -1,278 +0,0 @@
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -march=x86-64 -mcpu=corei7-avx -mattr=+fma | FileCheck %s --check-prefix=CHECK-FMA --check-prefix=CHECK
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -march=x86-64 -mcpu=core-avx2 -mattr=+fma,+avx2 | FileCheck %s --check-prefix=CHECK-FMA --check-prefix=CHECK
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -march=x86-64 -mcpu=corei7-avx -mattr=+fma4 | FileCheck %s --check-prefix=CHECK-FMA4 --check-prefix=CHECK
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mcpu=bdver2 -mattr=+avx,-fma | FileCheck %s --check-prefix=CHECK-FMA4 --check-prefix=CHECK
-
-; VFMADD
-define < 4 x float > @test_x86_fma_vfmadd_ss(< 4 x float > %a0, < 4 x float > %a1, < 4 x float > %a2) {
-  ; CHECK-FMA4: vfmaddss
-  ; CHECK-FMA: vfmadd213ss
-  %res = call < 4 x float > @llvm.x86.fma.vfmadd.ss(< 4 x float > %a0, < 4 x float > %a1, < 4 x float > %a2)
-  ret < 4 x float > %res
-}
-declare < 4 x float > @llvm.x86.fma.vfmadd.ss(< 4 x float >, < 4 x float >, < 4 x float >) nounwind readnone
-
-define < 2 x double > @test_x86_fma_vfmadd_sd(< 2 x double > %a0, < 2 x double > %a1, < 2 x double > %a2) {
-  ; CHECK-FMA4: vfmaddsd
-  ; CHECK-FMA: vfmadd213sd
-  %res = call < 2 x double > @llvm.x86.fma.vfmadd.sd(< 2 x double > %a0, < 2 x double > %a1, < 2 x double > %a2)
-  ret < 2 x double > %res
-}
-declare < 2 x double > @llvm.x86.fma.vfmadd.sd(< 2 x double >, < 2 x double >, < 2 x double >) nounwind readnone
-
-define < 4 x float > @test_x86_fma_vfmadd_ps(< 4 x float > %a0, < 4 x float > %a1, < 4 x float > %a2) {
-  ; CHECK-FMA4: vfmaddps
-  ; CHECK-FMA: vfmadd213ps
-  %res = call < 4 x float > @llvm.x86.fma.vfmadd.ps(< 4 x float > %a0, < 4 x float > %a1, < 4 x float > %a2)
-  ret < 4 x float > %res
-}
-declare < 4 x float > @llvm.x86.fma.vfmadd.ps(< 4 x float >, < 4 x float >, < 4 x float >) nounwind readnone
-
-define < 2 x double > @test_x86_fma_vfmadd_pd(< 2 x double > %a0, < 2 x double > %a1, < 2 x double > %a2) {
-  ; CHECK-FMA4: vfmaddpd
-  ; CHECK-FMA: vfmadd213pd
-  %res = call < 2 x double > @llvm.x86.fma.vfmadd.pd(< 2 x double > %a0, < 2 x double > %a1, < 2 x double > %a2)
-  ret < 2 x double > %res
-}
-declare < 2 x double > @llvm.x86.fma.vfmadd.pd(< 2 x double >, < 2 x double >, < 2 x double >) nounwind readnone
-
-define < 8 x float > @test_x86_fma_vfmadd_ps_256(< 8 x float > %a0, < 8 x float > %a1, < 8 x float > %a2) {
-  ; CHECK-FMA4: vfmaddps
-  ; CHECK-FMA: vfmadd213ps
-  ; CHECK: ymm
-  %res = call < 8 x float > @llvm.x86.fma.vfmadd.ps.256(< 8 x float > %a0, < 8 x float > %a1, < 8 x float > %a2)
-  ret < 8 x float > %res
-}
-declare < 8 x float > @llvm.x86.fma.vfmadd.ps.256(< 8 x float >, < 8 x float >, < 8 x float >) nounwind readnone
-
-define < 4 x double > @test_x86_fma_vfmadd_pd_256(< 4 x double > %a0, < 4 x double > %a1, < 4 x double > %a2) {
-  ; CHECK-FMA4: vfmaddpd
-  ; CHECK-FMA: vfmadd213pd
-  ; CHECK: ymm
-  %res = call < 4 x double > @llvm.x86.fma.vfmadd.pd.256(< 4 x double > %a0, < 4 x double > %a1, < 4 x double > %a2)
-  ret < 4 x double > %res
-}
-declare < 4 x double > @llvm.x86.fma.vfmadd.pd.256(< 4 x double >, < 4 x double >, < 4 x double >) nounwind readnone
-
-; VFMSUB
-define < 4 x float > @test_x86_fma_vfmsub_ss(< 4 x float > %a0, < 4 x float > %a1, < 4 x float > %a2) {
-  ; CHECK-FMA4: vfmsubss
-  ; CHECK-FMA: vfmsub213ss
-  %res = call < 4 x float > @llvm.x86.fma.vfmsub.ss(< 4 x float > %a0, < 4 x float > %a1, < 4 x float > %a2)
-  ret < 4 x float > %res
-}
-declare < 4 x float > @llvm.x86.fma.vfmsub.ss(< 4 x float >, < 4 x float >, < 4 x float >) nounwind readnone
-
-define < 2 x double > @test_x86_fma_vfmsub_sd(< 2 x double > %a0, < 2 x double > %a1, < 2 x double > %a2) {
-  ; CHECK-FMA4: vfmsubsd
-  ; CHECK-FMA: vfmsub213sd
-  %res = call < 2 x double > @llvm.x86.fma.vfmsub.sd(< 2 x double > %a0, < 2 x double > %a1, < 2 x double > %a2)
-  ret < 2 x double > %res
-}
-declare < 2 x double > @llvm.x86.fma.vfmsub.sd(< 2 x double >, < 2 x double >, < 2 x double >) nounwind readnone
-
-define < 4 x float > @test_x86_fma_vfmsub_ps(< 4 x float > %a0, < 4 x float > %a1, < 4 x float > %a2) {
-  ; CHECK-FMA4: vfmsubps
-  ; CHECK-FMA: vfmsub213ps
-  %res = call < 4 x float > @llvm.x86.fma.vfmsub.ps(< 4 x float > %a0, < 4 x float > %a1, < 4 x float > %a2)
-  ret < 4 x float > %res
-}
-declare < 4 x float > @llvm.x86.fma.vfmsub.ps(< 4 x float >, < 4 x float >, < 4 x float >) nounwind readnone
-
-define < 2 x double > @test_x86_fma_vfmsub_pd(< 2 x double > %a0, < 2 x double > %a1, < 2 x double > %a2) {
-  ; CHECK-FMA4: vfmsubpd
-  ; CHECK-FMA: vfmsub213pd
-  %res = call < 2 x double > @llvm.x86.fma.vfmsub.pd(< 2 x double > %a0, < 2 x double > %a1, < 2 x double > %a2)
-  ret < 2 x double > %res
-}
-declare < 2 x double > @llvm.x86.fma.vfmsub.pd(< 2 x double >, < 2 x double >, < 2 x double >) nounwind readnone
-
-define < 8 x float > @test_x86_fma_vfmsub_ps_256(< 8 x float > %a0, < 8 x float > %a1, < 8 x float > %a2) {
-  ; CHECK-FMA4: vfmsubps
-  ; CHECK-FMA: vfmsub213ps
-  ; CHECK: ymm
-  %res = call < 8 x float > @llvm.x86.fma.vfmsub.ps.256(< 8 x float > %a0, < 8 x float > %a1, < 8 x float > %a2)
-  ret < 8 x float > %res
-}
-declare < 8 x float > @llvm.x86.fma.vfmsub.ps.256(< 8 x float >, < 8 x float >, < 8 x float >) nounwind readnone
-
-define < 4 x double > @test_x86_fma_vfmsub_pd_256(< 4 x double > %a0, < 4 x double > %a1, < 4 x double > %a2) {
-  ; CHECK-FMA4: vfmsubpd
-  ; CHECK-FMA: vfmsub213pd
-  ; CHECK: ymm
-  %res = call < 4 x double > @llvm.x86.fma.vfmsub.pd.256(< 4 x double > %a0, < 4 x double > %a1, < 4 x double > %a2)
-  ret < 4 x double > %res
-}
-declare < 4 x double > @llvm.x86.fma.vfmsub.pd.256(< 4 x double >, < 4 x double >, < 4 x double >) nounwind readnone
-
-; VFNMADD
-define < 4 x float > @test_x86_fma_vfnmadd_ss(< 4 x float > %a0, < 4 x float > %a1, < 4 x float > %a2) {
-  ; CHECK-FMA4: vfnmaddss
-  ; CHECK-FMA: vfnmadd213ss
-  %res = call < 4 x float > @llvm.x86.fma.vfnmadd.ss(< 4 x float > %a0, < 4 x float > %a1, < 4 x float > %a2)
-  ret < 4 x float > %res
-}
-declare < 4 x float > @llvm.x86.fma.vfnmadd.ss(< 4 x float >, < 4 x float >, < 4 x float >) nounwind readnone
-
-define < 2 x double > @test_x86_fma_vfnmadd_sd(< 2 x double > %a0, < 2 x double > %a1, < 2 x double > %a2) {
-  ; CHECK-FMA4: vfnmaddsd
-  ; CHECK-FMA: vfnmadd213sd
-  %res = call < 2 x double > @llvm.x86.fma.vfnmadd.sd(< 2 x double > %a0, < 2 x double > %a1, < 2 x double > %a2)
-  ret < 2 x double > %res
-}
-declare < 2 x double > @llvm.x86.fma.vfnmadd.sd(< 2 x double >, < 2 x double >, < 2 x double >) nounwind readnone
-
-define < 4 x float > @test_x86_fma_vfnmadd_ps(< 4 x float > %a0, < 4 x float > %a1, < 4 x float > %a2) {
-  ; CHECK-FMA4: vfnmaddps
-  ; CHECK-FMA: vfnmadd213ps
-  %res = call < 4 x float > @llvm.x86.fma.vfnmadd.ps(< 4 x float > %a0, < 4 x float > %a1, < 4 x float > %a2)
-  ret < 4 x float > %res
-}
-declare < 4 x float > @llvm.x86.fma.vfnmadd.ps(< 4 x float >, < 4 x float >, < 4 x float >) nounwind readnone
-
-define < 2 x double > @test_x86_fma_vfnmadd_pd(< 2 x double > %a0, < 2 x double > %a1, < 2 x double > %a2) {
-  ; CHECK-FMA4: vfnmaddpd
-  ; CHECK-FMA: vfnmadd213pd
-  %res = call < 2 x double > @llvm.x86.fma.vfnmadd.pd(< 2 x double > %a0, < 2 x double > %a1, < 2 x double > %a2)
-  ret < 2 x double > %res
-}
-declare < 2 x double > @llvm.x86.fma.vfnmadd.pd(< 2 x double >, < 2 x double >, < 2 x double >) nounwind readnone
-
-define < 8 x float > @test_x86_fma_vfnmadd_ps_256(< 8 x float > %a0, < 8 x float > %a1, < 8 x float > %a2) {
-  ; CHECK-FMA4: vfnmaddps
-  ; CHECK-FMA: vfnmadd213ps
-  ; CHECK: ymm
-  %res = call < 8 x float > @llvm.x86.fma.vfnmadd.ps.256(< 8 x float > %a0, < 8 x float > %a1, < 8 x float > %a2)
-  ret < 8 x float > %res
-}
-declare < 8 x float > @llvm.x86.fma.vfnmadd.ps.256(< 8 x float >, < 8 x float >, < 8 x float >) nounwind readnone
-
-define < 4 x double > @test_x86_fma_vfnmadd_pd_256(< 4 x double > %a0, < 4 x double > %a1, < 4 x double > %a2) {
-  ; CHECK-FMA4: vfnmaddpd
-  ; CHECK-FMA: vfnmadd213pd
-  ; CHECK: ymm
-  %res = call < 4 x double > @llvm.x86.fma.vfnmadd.pd.256(< 4 x double > %a0, < 4 x double > %a1, < 4 x double > %a2)
-  ret < 4 x double > %res
-}
-declare < 4 x double > @llvm.x86.fma.vfnmadd.pd.256(< 4 x double >, < 4 x double >, < 4 x double >) nounwind readnone
-
-; VFNMSUB
-define < 4 x float > @test_x86_fma_vfnmsub_ss(< 4 x float > %a0, < 4 x float > %a1, < 4 x float > %a2) {
-  ; CHECK-FMA4: vfnmsubss
-  ; CHECK-FMA: vfnmsub213ss
-  %res = call < 4 x float > @llvm.x86.fma.vfnmsub.ss(< 4 x float > %a0, < 4 x float > %a1, < 4 x float > %a2)
-  ret < 4 x float > %res
-}
-declare < 4 x float > @llvm.x86.fma.vfnmsub.ss(< 4 x float >, < 4 x float >, < 4 x float >) nounwind readnone
-
-define < 2 x double > @test_x86_fma_vfnmsub_sd(< 2 x double > %a0, < 2 x double > %a1, < 2 x double > %a2) {
-  ; CHECK-FMA4: vfnmsubsd
-  ; CHECK-FMA: vfnmsub213sd
-  %res = call < 2 x double > @llvm.x86.fma.vfnmsub.sd(< 2 x double > %a0, < 2 x double > %a1, < 2 x double > %a2)
-  ret < 2 x double > %res
-}
-declare < 2 x double > @llvm.x86.fma.vfnmsub.sd(< 2 x double >, < 2 x double >, < 2 x double >) nounwind readnone
-
-define < 4 x float > @test_x86_fma_vfnmsub_ps(< 4 x float > %a0, < 4 x float > %a1, < 4 x float > %a2) {
-  ; CHECK-FMA4: vfnmsubps
-  ; CHECK-FMA: vfnmsub213ps
-  %res = call < 4 x float > @llvm.x86.fma.vfnmsub.ps(< 4 x float > %a0, < 4 x float > %a1, < 4 x float > %a2)
-  ret < 4 x float > %res
-}
-declare < 4 x float > @llvm.x86.fma.vfnmsub.ps(< 4 x float >, < 4 x float >, < 4 x float >) nounwind readnone
-
-define < 2 x double > @test_x86_fma_vfnmsub_pd(< 2 x double > %a0, < 2 x double > %a1, < 2 x double > %a2) {
-  ; CHECK-FMA4: vfnmsubpd
-  ; CHECK-FMA: vfnmsub213pd
-  %res = call < 2 x double > @llvm.x86.fma.vfnmsub.pd(< 2 x double > %a0, < 2 x double > %a1, < 2 x double > %a2)
-  ret < 2 x double > %res
-}
-declare < 2 x double > @llvm.x86.fma.vfnmsub.pd(< 2 x double >, < 2 x double >, < 2 x double >) nounwind readnone
-
-define < 8 x float > @test_x86_fma_vfnmsub_ps_256(< 8 x float > %a0, < 8 x float > %a1, < 8 x float > %a2) {
-  ; CHECK-FMA4: vfnmsubps
-  ; CHECK-FMA: vfnmsub213ps
-  ; CHECK: ymm
-  %res = call < 8 x float > @llvm.x86.fma.vfnmsub.ps.256(< 8 x float > %a0, < 8 x float > %a1, < 8 x float > %a2)
-  ret < 8 x float > %res
-}
-declare < 8 x float > @llvm.x86.fma.vfnmsub.ps.256(< 8 x float >, < 8 x float >, < 8 x float >) nounwind readnone
-
-define < 4 x double > @test_x86_fma_vfnmsub_pd_256(< 4 x double > %a0, < 4 x double > %a1, < 4 x double > %a2) {
-  ; CHECK-FMA4: vfnmsubpd
-  ; CHECK-FMA: vfnmsub213pd
-  ; CHECK: ymm
-  %res = call < 4 x double > @llvm.x86.fma.vfnmsub.pd.256(< 4 x double > %a0, < 4 x double > %a1, < 4 x double > %a2)
-  ret < 4 x double > %res
-}
-declare < 4 x double > @llvm.x86.fma.vfnmsub.pd.256(< 4 x double >, < 4 x double >, < 4 x double >) nounwind readnone
-
-; VFMADDSUB
-define < 4 x float > @test_x86_fma_vfmaddsub_ps(< 4 x float > %a0, < 4 x float > %a1, < 4 x float > %a2) {
-  ; CHECK-FMA4: vfmaddsubps
-  ; CHECK-FMA: vfmaddsub213ps
-  %res = call < 4 x float > @llvm.x86.fma.vfmaddsub.ps(< 4 x float > %a0, < 4 x float > %a1, < 4 x float > %a2)
-  ret < 4 x float > %res
-}
-declare < 4 x float > @llvm.x86.fma.vfmaddsub.ps(< 4 x float >, < 4 x float >, < 4 x float >) nounwind readnone
-
-define < 2 x double > @test_x86_fma_vfmaddsub_pd(< 2 x double > %a0, < 2 x double > %a1, < 2 x double > %a2) {
-  ; CHECK-FMA4: vfmaddsubpd
-  ; CHECK-FMA: vfmaddsub213pd
-  %res = call < 2 x double > @llvm.x86.fma.vfmaddsub.pd(< 2 x double > %a0, < 2 x double > %a1, < 2 x double > %a2)
-  ret < 2 x double > %res
-}
-declare < 2 x double > @llvm.x86.fma.vfmaddsub.pd(< 2 x double >, < 2 x double >, < 2 x double >) nounwind readnone
-
-define < 8 x float > @test_x86_fma_vfmaddsub_ps_256(< 8 x float > %a0, < 8 x float > %a1, < 8 x float > %a2) {
-  ; CHECK-FMA4: vfmaddsubps
-  ; CHECK-FMA: vfmaddsub213ps
-  ; CHECK: ymm
-  %res = call < 8 x float > @llvm.x86.fma.vfmaddsub.ps.256(< 8 x float > %a0, < 8 x float > %a1, < 8 x float > %a2)
-  ret < 8 x float > %res
-}
-declare < 8 x float > @llvm.x86.fma.vfmaddsub.ps.256(< 8 x float >, < 8 x float >, < 8 x float >) nounwind readnone
-
-define < 4 x double > @test_x86_fma_vfmaddsub_pd_256(< 4 x double > %a0, < 4 x double > %a1, < 4 x double > %a2) {
-  ; CHECK-FMA4: vfmaddsubpd
-  ; CHECK-FMA: vfmaddsub213pd
-  ; CHECK: ymm
-  %res = call < 4 x double > @llvm.x86.fma.vfmaddsub.pd.256(< 4 x double > %a0, < 4 x double > %a1, < 4 x double > %a2)
-  ret < 4 x double > %res
-}
-declare < 4 x double > @llvm.x86.fma.vfmaddsub.pd.256(< 4 x double >, < 4 x double >, < 4 x double >) nounwind readnone
-
-; VFMSUBADD
-define < 4 x float > @test_x86_fma_vfmsubadd_ps(< 4 x float > %a0, < 4 x float > %a1, < 4 x float > %a2) {
-  ; CHECK-FMA4: vfmsubaddps
-  ; CHECK-FMA: vfmsubadd213ps
-  %res = call < 4 x float > @llvm.x86.fma.vfmsubadd.ps(< 4 x float > %a0, < 4 x float > %a1, < 4 x float > %a2)
-  ret < 4 x float > %res
-}
-declare < 4 x float > @llvm.x86.fma.vfmsubadd.ps(< 4 x float >, < 4 x float >, < 4 x float >) nounwind readnone
-
-define < 2 x double > @test_x86_fma_vfmsubadd_pd(< 2 x double > %a0, < 2 x double > %a1, < 2 x double > %a2) {
-  ; CHECK-FMA4: vfmsubaddpd
-  ; CHECK-FMA: vfmsubadd213pd
-  %res = call < 2 x double > @llvm.x86.fma.vfmsubadd.pd(< 2 x double > %a0, < 2 x double > %a1, < 2 x double > %a2)
-  ret < 2 x double > %res
-}
-declare < 2 x double > @llvm.x86.fma.vfmsubadd.pd(< 2 x double >, < 2 x double >, < 2 x double >) nounwind readnone
-
-define < 8 x float > @test_x86_fma_vfmsubadd_ps_256(< 8 x float > %a0, < 8 x float > %a1, < 8 x float > %a2) {
-  ; CHECK-FMA4: vfmsubaddps
-  ; CHECK-FMA: vfmsubadd213ps
-  ; CHECK: ymm
-  %res = call < 8 x float > @llvm.x86.fma.vfmsubadd.ps.256(< 8 x float > %a0, < 8 x float > %a1, < 8 x float > %a2)
-  ret < 8 x float > %res
-}
-declare < 8 x float > @llvm.x86.fma.vfmsubadd.ps.256(< 8 x float >, < 8 x float >, < 8 x float >) nounwind readnone
-
-define < 4 x double > @test_x86_fma_vfmsubadd_pd_256(< 4 x double > %a0, < 4 x double > %a1, < 4 x double > %a2) {
-  ; CHECK-FMA4: vfmsubaddpd
-  ; CHECK-FMA: vfmsubadd213pd
-  ; CHECK: ymm
-  %res = call < 4 x double > @llvm.x86.fma.vfmsubadd.pd.256(< 4 x double > %a0, < 4 x double > %a1, < 4 x double > %a2)
-  ret < 4 x double > %res
-}
-declare < 4 x double > @llvm.x86.fma.vfmsubadd.pd.256(< 4 x double >, < 4 x double >, < 4 x double >) nounwind readnone
diff --git a/test/CodeGen/X86/fma-phi-213-to-231.ll b/test/CodeGen/X86/fma-phi-213-to-231.ll
index 9715bc7b328..34acdfe830f 100644
--- a/test/CodeGen/X86/fma-phi-213-to-231.ll
+++ b/test/CodeGen/X86/fma-phi-213-to-231.ll
@@ -1,246 +1,37 @@
-; RUN: llc < %s -mcpu=core-avx2 | FileCheck %s
-target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
-target triple = "x86_64-apple-macosx10.10.0"
-
-; CHECK-LABEL: fmaddsubpd_loop
-; CHECK: [[BODYLBL:LBB.+]]:
-; CHECK:   vfmaddsub231pd        %ymm{{[0-9]+}}, %ymm{{[0-9]+}}, %ymm{{[0-9]+}}
-; CHECK: [[INCLBL:LBB.+]]:
-; CHECK:   addl  $1, [[INDREG:%[a-z0-9]+]]
-; CHECK:   cmpl  {{%.+}}, [[INDREG]]
-; CHECK:   jl    [[BODYLBL]]
-define <4 x double> @fmaddsubpd_loop(i32 %iter, <4 x double> %a, <4 x double> %b, <4 x double> %c) {
-entry:
-  br label %for.cond
-
-for.cond:
-  %c.addr.0 = phi <4 x double> [ %c, %entry ], [ %0, %for.inc ]
-  %i.0 = phi i32 [ 0, %entry ], [ %inc, %for.inc ]
-  %cmp = icmp slt i32 %i.0, %iter
-  br i1 %cmp, label %for.body, label %for.end
-
-for.body:
-  br label %for.inc
-
-for.inc:
-  %0 = call <4 x double> @llvm.x86.fma.vfmaddsub.pd.256(<4 x double> %a, <4 x double> %b, <4 x double> %c.addr.0)
-  %inc = add nsw i32 %i.0, 1
-  br label %for.cond
-
-for.end:
-  ret <4 x double> %c.addr.0
-}
-
-; CHECK-LABEL: fmsubaddpd_loop
-; CHECK: [[BODYLBL:LBB.+]]:
-; CHECK:   vfmsubadd231pd        %ymm{{[0-9]+}}, %ymm{{[0-9]+}}, %ymm{{[0-9]+}}
-; CHECK: [[INCLBL:LBB.+]]:
-; CHECK:   addl  $1, [[INDREG:%[a-z0-9]+]]
-; CHECK:   cmpl  {{%.+}}, [[INDREG]]
-; CHECK:   jl    [[BODYLBL]]
-define <4 x double> @fmsubaddpd_loop(i32 %iter, <4 x double> %a, <4 x double> %b, <4 x double> %c) {
-entry:
-  br label %for.cond
-
-for.cond:
-  %c.addr.0 = phi <4 x double> [ %c, %entry ], [ %0, %for.inc ]
-  %i.0 = phi i32 [ 0, %entry ], [ %inc, %for.inc ]
-  %cmp = icmp slt i32 %i.0, %iter
-  br i1 %cmp, label %for.body, label %for.end
-
-for.body:
-  br label %for.inc
-
-for.inc:
-  %0 = call <4 x double> @llvm.x86.fma.vfmsubadd.pd.256(<4 x double> %a, <4 x double> %b, <4 x double> %c.addr.0)
-  %inc = add nsw i32 %i.0, 1
-  br label %for.cond
-
-for.end:
-  ret <4 x double> %c.addr.0
-}
-
-; CHECK-LABEL: fmaddpd_loop
-; CHECK: [[BODYLBL:LBB.+]]:
-; CHECK:   vfmadd231pd        %ymm{{[0-9]+}}, %ymm{{[0-9]+}}, %ymm{{[0-9]+}}
-; CHECK: [[INCLBL:LBB.+]]:
-; CHECK:   addl  $1, [[INDREG:%[a-z0-9]+]]
-; CHECK:   cmpl  {{%.+}}, [[INDREG]]
-; CHECK:   jl    [[BODYLBL]]
-define <4 x double> @fmaddpd_loop(i32 %iter, <4 x double> %a, <4 x double> %b, <4 x double> %c) {
-entry:
-  br label %for.cond
-
-for.cond:
-  %c.addr.0 = phi <4 x double> [ %c, %entry ], [ %0, %for.inc ]
-  %i.0 = phi i32 [ 0, %entry ], [ %inc, %for.inc ]
-  %cmp = icmp slt i32 %i.0, %iter
-  br i1 %cmp, label %for.body, label %for.end
-
-for.body:
-  br label %for.inc
-
-for.inc:
-  %0 = call <4 x double> @llvm.x86.fma.vfmadd.pd.256(<4 x double> %a, <4 x double> %b, <4 x double> %c.addr.0)
-  %inc = add nsw i32 %i.0, 1
-  br label %for.cond
-
-for.end:
-  ret <4 x double> %c.addr.0
-}
-
-; CHECK-LABEL: fmsubpd_loop
-; CHECK: [[BODYLBL:LBB.+]]:
-; CHECK:   vfmsub231pd        %ymm{{[0-9]+}}, %ymm{{[0-9]+}}, %ymm{{[0-9]+}}
-; CHECK: [[INCLBL:LBB.+]]:
-; CHECK:   addl  $1, [[INDREG:%[a-z0-9]+]]
-; CHECK:   cmpl  {{%.+}}, [[INDREG]]
-; CHECK:   jl    [[BODYLBL]]
-define <4 x double> @fmsubpd_loop(i32 %iter, <4 x double> %a, <4 x double> %b, <4 x double> %c) {
-entry:
-  br label %for.cond
-
-for.cond:
-  %c.addr.0 = phi <4 x double> [ %c, %entry ], [ %0, %for.inc ]
-  %i.0 = phi i32 [ 0, %entry ], [ %inc, %for.inc ]
-  %cmp = icmp slt i32 %i.0, %iter
-  br i1 %cmp, label %for.body, label %for.end
-
-for.body:
-  br label %for.inc
-
-for.inc:
-  %0 = call <4 x double> @llvm.x86.fma.vfmsub.pd.256(<4 x double> %a, <4 x double> %b, <4 x double> %c.addr.0)
-  %inc = add nsw i32 %i.0, 1
-  br label %for.cond
-
-for.end:
-  ret <4 x double> %c.addr.0
-}
-
-declare <4 x double> @llvm.x86.fma.vfmaddsub.pd.256(<4 x double>, <4 x double>, <4 x double>)
-declare <4 x double> @llvm.x86.fma.vfmsubadd.pd.256(<4 x double>, <4 x double>, <4 x double>)
-declare <4 x double> @llvm.x86.fma.vfmadd.pd.256(<4 x double>, <4 x double>, <4 x double>)
-declare <4 x double> @llvm.x86.fma.vfmsub.pd.256(<4 x double>, <4 x double>, <4 x double>)
-
-
-; CHECK-LABEL: fmaddsubps_loop
-; CHECK: [[BODYLBL:LBB.+]]:
-; CHECK:   vfmaddsub231ps        %ymm{{[0-9]+}}, %ymm{{[0-9]+}}, %ymm{{[0-9]+}}
-; CHECK: [[INCLBL:LBB.+]]:
-; CHECK:   addl  $1, [[INDREG:%[a-z0-9]+]]
-; CHECK:   cmpl  {{%.+}}, [[INDREG]]
-; CHECK:   jl    [[BODYLBL]]
-define <8 x float> @fmaddsubps_loop(i32 %iter, <8 x float> %a, <8 x float> %b, <8 x float> %c) {
-entry:
-  br label %for.cond
-
-for.cond:
-  %c.addr.0 = phi <8 x float> [ %c, %entry ], [ %0, %for.inc ]
-  %i.0 = phi i32 [ 0, %entry ], [ %inc, %for.inc ]
-  %cmp = icmp slt i32 %i.0, %iter
-  br i1 %cmp, label %for.body, label %for.end
-
-for.body:
-  br label %for.inc
-
-for.inc:
-  %0 = call <8 x float> @llvm.x86.fma.vfmaddsub.ps.256(<8 x float> %a, <8 x float> %b, <8 x float> %c.addr.0)
-  %inc = add nsw i32 %i.0, 1
-  br label %for.cond
-
-for.end:
-  ret <8 x float> %c.addr.0
-}
-
-; CHECK-LABEL: fmsubaddps_loop
-; CHECK: [[BODYLBL:LBB.+]]:
-; CHECK:   vfmsubadd231ps        %ymm{{[0-9]+}}, %ymm{{[0-9]+}}, %ymm{{[0-9]+}}
-; CHECK: [[INCLBL:LBB.+]]:
-; CHECK:   addl  $1, [[INDREG:%[a-z0-9]+]]
-; CHECK:   cmpl  {{%.+}}, [[INDREG]]
-; CHECK:   jl    [[BODYLBL]]
-define <8 x float> @fmsubaddps_loop(i32 %iter, <8 x float> %a, <8 x float> %b, <8 x float> %c) {
-entry:
-  br label %for.cond
-
-for.cond:
-  %c.addr.0 = phi <8 x float> [ %c, %entry ], [ %0, %for.inc ]
-  %i.0 = phi i32 [ 0, %entry ], [ %inc, %for.inc ]
-  %cmp = icmp slt i32 %i.0, %iter
-  br i1 %cmp, label %for.body, label %for.end
-
-for.body:
-  br label %for.inc
-
-for.inc:
-  %0 = call <8 x float> @llvm.x86.fma.vfmsubadd.ps.256(<8 x float> %a, <8 x float> %b, <8 x float> %c.addr.0)
-  %inc = add nsw i32 %i.0, 1
-  br label %for.cond
-
-for.end:
-  ret <8 x float> %c.addr.0
-}
-
-; CHECK-LABEL: fmaddps_loop
-; CHECK: [[BODYLBL:LBB.+]]:
-; CHECK:   vfmadd231ps        %ymm{{[0-9]+}}, %ymm{{[0-9]+}}, %ymm{{[0-9]+}}
-; CHECK: [[INCLBL:LBB.+]]:
-; CHECK:   addl  $1, [[INDREG:%[a-z0-9]+]]
-; CHECK:   cmpl  {{%.+}}, [[INDREG]]
-; CHECK:   jl    [[BODYLBL]]
-define <8 x float> @fmaddps_loop(i32 %iter, <8 x float> %a, <8 x float> %b, <8 x float> %c) {
-entry:
-  br label %for.cond
-
-for.cond:
-  %c.addr.0 = phi <8 x float> [ %c, %entry ], [ %0, %for.inc ]
-  %i.0 = phi i32 [ 0, %entry ], [ %inc, %for.inc ]
-  %cmp = icmp slt i32 %i.0, %iter
-  br i1 %cmp, label %for.body, label %for.end
-
-for.body:
-  br label %for.inc
-
-for.inc:
-  %0 = call <8 x float> @llvm.x86.fma.vfmadd.ps.256(<8 x float> %a, <8 x float> %b, <8 x float> %c.addr.0)
-  %inc = add nsw i32 %i.0, 1
-  br label %for.cond
-
-for.end:
-  ret <8 x float> %c.addr.0
-}
-
-; CHECK-LABEL: fmsubps_loop
-; CHECK: [[BODYLBL:LBB.+]]:
-; CHECK:   vfmsub231ps        %ymm{{[0-9]+}}, %ymm{{[0-9]+}}, %ymm{{[0-9]+}}
-; CHECK: [[INCLBL:LBB.+]]:
-; CHECK:   addl  $1, [[INDREG:%[a-z0-9]+]]
-; CHECK:   cmpl  {{%.+}}, [[INDREG]]
-; CHECK:   jl    [[BODYLBL]]
-define <8 x float> @fmsubps_loop(i32 %iter, <8 x float> %a, <8 x float> %b, <8 x float> %c) {
-entry:
-  br label %for.cond
-
-for.cond:
-  %c.addr.0 = phi <8 x float> [ %c, %entry ], [ %0, %for.inc ]
-  %i.0 = phi i32 [ 0, %entry ], [ %inc, %for.inc ]
-  %cmp = icmp slt i32 %i.0, %iter
-  br i1 %cmp, label %for.body, label %for.end
-
-for.body:
-  br label %for.inc
-
-for.inc:
-  %0 = call <8 x float> @llvm.x86.fma.vfmsub.ps.256(<8 x float> %a, <8 x float> %b, <8 x float> %c.addr.0)
-  %inc = add nsw i32 %i.0, 1
-  br label %for.cond
-
-for.end:
-  ret <8 x float> %c.addr.0
-}
-
-declare <8 x float> @llvm.x86.fma.vfmaddsub.ps.256(<8 x float>, <8 x float>, <8 x float>)
-declare <8 x float> @llvm.x86.fma.vfmsubadd.ps.256(<8 x float>, <8 x float>, <8 x float>)
-declare <8 x float> @llvm.x86.fma.vfmadd.ps.256(<8 x float>, <8 x float>, <8 x float>)
-declare <8 x float> @llvm.x86.fma.vfmsub.ps.256(<8 x float>, <8 x float>, <8 x float>)
+; RUN: llc < %s -mtriple=i386-apple-darwin10  -mattr=+fma,-fma4  | FileCheck %s
+; RUN: llc < %s -mtriple=x86_64-apple-darwin10 -mattr=+fma,-fma4 | FileCheck %s
+; RUN: llc < %s -march=x86 -mcpu=bdver2 -mattr=-fma4  | FileCheck %s
+
+; Test FMA3 variant selection
+
+; CHECK-LABEL: fma3_select231ssX:
+; CHECK: vfmadd231ss %xmm
+define float @fma3_select231ssX(float %x, float %y) {
+entry:
+  br label %while.body
+while.body:
+  %acc.01 = phi float [ 0.000000e+00, %entry ], [ %acc, %while.body ]
+  %acc = call float @llvm.fma.f32(float %x, float %y, float %acc.01)
+  %b = fcmp ueq float %acc, 0.0
+  br i1 %b, label %while.body, label %while.end
+while.end:
+  ret float %acc
+}
+
+; CHECK-LABEL: fma3_select231pdY:
+; CHECK: vfmadd231pd %ymm
+define <4 x double> @fma3_select231pdY(<4 x double> %x, <4 x double> %y) {
+entry:
+  br label %while.body
+while.body:
+  %acc.04 = phi <4 x double> [ zeroinitializer, %entry ], [ %add, %while.body ]
+  %add = call <4 x double> @llvm.fma.v4f64(<4 x double> %x, <4 x double> %y, <4 x double> %acc.04)
+  %vecext = extractelement <4 x double> %add, i32 0
+  %cmp = fcmp oeq double %vecext, 0.000000e+00
+  br i1 %cmp, label %while.body, label %while.end
+while.end:
+  ret <4 x double> %add
+}
+
+declare float @llvm.fma.f32(float, float, float)
+declare <4 x double> @llvm.fma.v4f64(<4 x double>, <4 x double>, <4 x double>)
diff --git a/test/CodeGen/X86/fma.ll b/test/CodeGen/X86/fma.ll
index 2eb152b078e..b91479cda87 100644
--- a/test/CodeGen/X86/fma.ll
+++ b/test/CodeGen/X86/fma.ll
@@ -1,80 +1,47 @@
-; RUN: llc < %s -mtriple=i386-apple-darwin10  -mattr=+fma,-fma4  | FileCheck %s --check-prefix=CHECK-FMA-INST
-; RUN: llc < %s -mtriple=i386-apple-darwin10  -mattr=-fma,-fma4  | FileCheck %s --check-prefix=CHECK-FMA-CALL
-; RUN: llc < %s -mtriple=x86_64-apple-darwin10 -mattr=+fma,-fma4 | FileCheck %s --check-prefix=CHECK-FMA-INST
-; RUN: llc < %s -mtriple=x86_64-apple-darwin10  -mattr=-fma,-fma4 | FileCheck %s --check-prefix=CHECK-FMA-CALL
-; RUN: llc < %s -march=x86 -mcpu=bdver2 -mattr=-fma4  | FileCheck %s --check-prefix=CHECK-FMA-INST
-; RUN: llc < %s -march=x86 -mcpu=bdver2 -mattr=-fma,-fma4 | FileCheck %s --check-prefix=CHECK-FMA-CALL
-
-; CHECK: test_f32
+; RUN: llc < %s -mtriple=i386-apple-darwin10  -mattr=+fma,-fma4  | FileCheck %s --check-prefix=CHECK --check-prefix=CHECK-FMA-INST
+; RUN: llc < %s -mtriple=i386-apple-darwin10  -mattr=-fma,-fma4  | FileCheck %s --check-prefix=CHECK --check-prefix=CHECK-FMA-CALL
+; RUN: llc < %s -mtriple=x86_64-apple-darwin10 -mattr=+fma,-fma4 | FileCheck %s --check-prefix=CHECK --check-prefix=CHECK-FMA-INST
+; RUN: llc < %s -mtriple=x86_64-apple-darwin10  -mattr=-fma,-fma4 | FileCheck %s --check-prefix=CHECK --check-prefix=CHECK-FMA-CALL
+; RUN: llc < %s -mtriple=x86_64-apple-darwin10  -mattr=+avx512f,-fma,-fma4 | FileCheck %s --check-prefix=CHECK --check-prefix=CHECK-FMA-INST
+; RUN: llc < %s -march=x86 -mcpu=bdver2 -mattr=-fma4  | FileCheck %s --check-prefix=CHECK --check-prefix=CHECK-FMA-INST
+; RUN: llc < %s -march=x86 -mcpu=bdver2 -mattr=-fma,-fma4 | FileCheck %s --check-prefix=CHECK --check-prefix=CHECK-FMA-CALL
+
+; CHECK-LABEL: test_f32:
 ; CHECK-FMA-INST: vfmadd213ss
 ; CHECK-FMA-CALL: fmaf
-
-define float @test_f32(float %a, float %b, float %c) nounwind readnone ssp {
+define float @test_f32(float %a, float %b, float %c) #0 {
 entry:
-  %call = tail call float @llvm.fma.f32(float %a, float %b, float %c) nounwind readnone
+  %call = call float @llvm.fma.f32(float %a, float %b, float %c)
   ret float %call
 }
 
-; CHECK: test_f64
+; CHECK-LABEL: test_f64:
 ; CHECK-FMA-INST: vfmadd213sd
 ; CHECK-FMA-CALL: fma
-
-define double @test_f64(double %a, double %b, double %c) nounwind readnone ssp {
+define double @test_f64(double %a, double %b, double %c) #0 {
 entry:
-  %call = tail call double @llvm.fma.f64(double %a, double %b, double %c) nounwind readnone
+  %call = call double @llvm.fma.f64(double %a, double %b, double %c)
   ret double %call
 }
 
-; CHECK: test_f80
+; CHECK-LABEL: test_f80:
 ; CHECK: fmal
-
-define x86_fp80 @test_f80(x86_fp80 %a, x86_fp80 %b, x86_fp80 %c) nounwind readnone ssp {
+define x86_fp80 @test_f80(x86_fp80 %a, x86_fp80 %b, x86_fp80 %c) #0 {
 entry:
-  %call = tail call x86_fp80 @llvm.fma.f80(x86_fp80 %a, x86_fp80 %b, x86_fp80 %c) nounwind readnone
+  %call = call x86_fp80 @llvm.fma.f80(x86_fp80 %a, x86_fp80 %b, x86_fp80 %c)
   ret x86_fp80 %call
 }
 
-; CHECK: test_f32_cst
-; CHECK-NOT: fma
-define float @test_f32_cst() nounwind readnone ssp {
+; CHECK-LABEL: test_f32_cst:
+; CHECK-NOT: vfmadd
+define float @test_f32_cst() #0 {
 entry:
-  %call = tail call float @llvm.fma.f32(float 3.0, float 3.0, float 3.0) nounwind readnone
+  %call = call float @llvm.fma.f32(float 3.0, float 3.0, float 3.0)
   ret float %call
 }
 
-; Test FMA3 variant selection
-; CHECK-FMA-INST: fma3_select231ssX:
-; CHECK-FMA-INST: vfmadd231ss %xmm
-define float @fma3_select231ssX(float %x, float %y) #0 {
-entry:
-  br label %while.body
-while.body:                                       ; preds = %while.body, %while.body
-  %acc.01 = phi float [ 0.000000e+00, %entry ], [ %acc, %while.body ]
-  %acc = tail call float @llvm.fma.f32(float %x, float %y, float %acc.01) nounwind readnone
-  %b = fcmp ueq float %acc, 0.0
-  br i1 %b, label %while.body, label %while.end
-while.end:                                        ; preds = %while.body, %entry
-  ret float %acc
-}
-
-; Test FMA3 variant selection
-; CHECK-FMA-INST: fma3_select231pdY:
-; CHECK-FMA-INST: vfmadd231pd %ymm
-define <4 x double> @fma3_select231pdY(<4 x double> %x, <4 x double> %y) #0 {
-entry:
-  br label %while.body
-while.body:                                       ; preds = %entry, %while.body
-  %acc.04 = phi <4 x double> [ zeroinitializer, %entry ], [ %add, %while.body ]
-  %add = tail call <4 x double> @llvm.fma.v4f64(<4 x double> %x, <4 x double> %y, <4 x double> %acc.04)
-  %vecext = extractelement <4 x double> %add, i32 0
-  %cmp = fcmp oeq double %vecext, 0.000000e+00
-  br i1 %cmp, label %while.body, label %while.end
-
-while.end:                                        ; preds = %while.body
-  ret <4 x double> %add
-}
+declare float @llvm.fma.f32(float, float, float)
+declare double @llvm.fma.f64(double, double, double)
+declare x86_fp80 @llvm.fma.f80(x86_fp80, x86_fp80, x86_fp80)
 
-declare float @llvm.fma.f32(float, float, float) nounwind readnone
-declare double @llvm.fma.f64(double, double, double) nounwind readnone
-declare x86_fp80 @llvm.fma.f80(x86_fp80, x86_fp80, x86_fp80) nounwind readnone
-declare <4 x double> @llvm.fma.v4f64(<4 x double>, <4 x double>, <4 x double>) nounwind readnone
+attributes #0 = { nounwind }
diff --git a/test/CodeGen/X86/fma3-intrinsics.ll b/test/CodeGen/X86/fma3-intrinsics.ll
deleted file mode 100644
index 9a25096c7a5..00000000000
--- a/test/CodeGen/X86/fma3-intrinsics.ll
+++ /dev/null
@@ -1,134 +0,0 @@
-; RUN: llc < %s -mtriple=x86_64-pc-win32 -mcpu=core-avx2 | FileCheck %s
-; RUN: llc < %s -mtriple=x86_64-pc-win32 -mattr=+fma,+fma4 | FileCheck %s
-; RUN: llc < %s -mcpu=bdver2 -mtriple=x86_64-pc-win32 -mattr=-fma4 | FileCheck %s
-
-define <4 x float> @test_x86_fmadd_ss(<4 x float> %a0, <4 x float> %a1, <4 x float> %a2) {
-  ; CHECK: fmadd213ss (%r8), %xmm
-  %res = call <4 x float> @llvm.x86.fma.vfmadd.ss(<4 x float> %a0, <4 x float> %a1, <4 x float> %a2) nounwind
-  ret <4 x float> %res
-}
-declare <4 x float> @llvm.x86.fma.vfmadd.ss(<4 x float>, <4 x float>, <4 x float>) nounwind readnone
-
-define <4 x float> @test_x86_fmadd_ps(<4 x float> %a0, <4 x float> %a1, <4 x float> %a2) {
-  ; CHECK: fmadd213ps
-  %res = call <4 x float> @llvm.x86.fma.vfmadd.ps(<4 x float> %a0, <4 x float> %a1, <4 x float> %a2) nounwind
-  ret <4 x float> %res
-}
-declare <4 x float> @llvm.x86.fma.vfmadd.ps(<4 x float>, <4 x float>, <4 x float>) nounwind readnone
-
-define <8 x float> @test_x86_fmadd_ps_y(<8 x float> %a0, <8 x float> %a1, <8 x float> %a2) {
-  ; CHECK: fmadd213ps {{.*\(%r.*}}, %ymm
-  %res = call <8 x float> @llvm.x86.fma.vfmadd.ps.256(<8 x float> %a0, <8 x float> %a1, <8 x float> %a2) nounwind
-  ret <8 x float> %res
-}
-declare <8 x float> @llvm.x86.fma.vfmadd.ps.256(<8 x float>, <8 x float>, <8 x float>) nounwind readnone
-
-define <4 x float> @test_x86_fnmadd_ss(<4 x float> %a0, <4 x float> %a1, <4 x float> %a2) {
-  ; CHECK: fnmadd213ss (%r8), %xmm
-  %res = call <4 x float> @llvm.x86.fma.vfnmadd.ss(<4 x float> %a0, <4 x float> %a1, <4 x float> %a2) nounwind
-  ret <4 x float> %res
-}
-declare <4 x float> @llvm.x86.fma.vfnmadd.ss(<4 x float>, <4 x float>, <4 x float>) nounwind readnone
-
-define <4 x float> @test_x86_fnmadd_ps(<4 x float> %a0, <4 x float> %a1, <4 x float> %a2) {
-  ; CHECK: fnmadd213ps
-  %res = call <4 x float> @llvm.x86.fma.vfnmadd.ps(<4 x float> %a0, <4 x float> %a1, <4 x float> %a2) nounwind
-  ret <4 x float> %res
-}
-declare <4 x float> @llvm.x86.fma.vfnmadd.ps(<4 x float>, <4 x float>, <4 x float>) nounwind readnone
-
-define <8 x float> @test_x86_fnmadd_ps_y(<8 x float> %a0, <8 x float> %a1, <8 x float> %a2) {
-  ; CHECK: fnmadd213ps {{.*\(%r.*}}, %ymm
-  %res = call <8 x float> @llvm.x86.fma.vfnmadd.ps.256(<8 x float> %a0, <8 x float> %a1, <8 x float> %a2) nounwind
-  ret <8 x float> %res
-}
-declare <8 x float> @llvm.x86.fma.vfnmadd.ps.256(<8 x float>, <8 x float>, <8 x float>) nounwind readnone
-
-
-define <4 x float> @test_x86_fmsub_ss(<4 x float> %a0, <4 x float> %a1, <4 x float> %a2) {
-  ; CHECK: fmsub213ss
-  %res = call <4 x float> @llvm.x86.fma.vfmsub.ss(<4 x float> %a0, <4 x float> %a1, <4 x float> %a2) nounwind
-  ret <4 x float> %res
-}
-declare <4 x float> @llvm.x86.fma.vfmsub.ss(<4 x float>, <4 x float>, <4 x float>) nounwind readnone
-
-define <4 x float> @test_x86_fmsub_ps(<4 x float> %a0, <4 x float> %a1, <4 x float> %a2) {
-  ; CHECK: fmsub213ps
-  %res = call <4 x float> @llvm.x86.fma.vfmsub.ps(<4 x float> %a0, <4 x float> %a1, <4 x float> %a2) nounwind
-  ret <4 x float> %res
-}
-declare <4 x float> @llvm.x86.fma.vfmsub.ps(<4 x float>, <4 x float>, <4 x float>) nounwind readnone
-
-define <4 x float> @test_x86_fnmsub_ss(<4 x float> %a0, <4 x float> %a1, <4 x float> %a2) {
-  ; CHECK: fnmsub213ss
-  %res = call <4 x float> @llvm.x86.fma.vfnmsub.ss(<4 x float> %a0, <4 x float> %a1, <4 x float> %a2) nounwind
-  ret <4 x float> %res
-}
-declare <4 x float> @llvm.x86.fma.vfnmsub.ss(<4 x float>, <4 x float>, <4 x float>) nounwind readnone
-
-define <4 x float> @test_x86_fnmsub_ps(<4 x float> %a0, <4 x float> %a1, <4 x float> %a2) {
-  ; CHECK: fnmsub213ps
-  %res = call <4 x float> @llvm.x86.fma.vfnmsub.ps(<4 x float> %a0, <4 x float> %a1, <4 x float> %a2) nounwind
-  ret <4 x float> %res
-}
-declare <4 x float> @llvm.x86.fma.vfnmsub.ps(<4 x float>, <4 x float>, <4 x float>) nounwind readnone
-
-;;;;
-
-define <2 x double> @test_x86_fmadd_sd(<2 x double> %a0, <2 x double> %a1, <2 x double> %a2) {
-  ; CHECK: fmadd213sd
-  %res = call <2 x double> @llvm.x86.fma.vfmadd.sd(<2 x double> %a0, <2 x double> %a1, <2 x double> %a2) nounwind
-  ret <2 x double> %res
-}
-declare <2 x double> @llvm.x86.fma.vfmadd.sd(<2 x double>, <2 x double>, <2 x double>) nounwind readnone
-
-define <2 x double> @test_x86_fmadd_pd(<2 x double> %a0, <2 x double> %a1, <2 x double> %a2) {
-  ; CHECK: fmadd213pd
-  %res = call <2 x double> @llvm.x86.fma.vfmadd.pd(<2 x double> %a0, <2 x double> %a1, <2 x double> %a2) nounwind
-  ret <2 x double> %res
-}
-declare <2 x double> @llvm.x86.fma.vfmadd.pd(<2 x double>, <2 x double>, <2 x double>) nounwind readnone
-
-define <2 x double> @test_x86_fnmadd_sd(<2 x double> %a0, <2 x double> %a1, <2 x double> %a2) {
-  ; CHECK: fnmadd213sd
-  %res = call <2 x double> @llvm.x86.fma.vfnmadd.sd(<2 x double> %a0, <2 x double> %a1, <2 x double> %a2) nounwind
-  ret <2 x double> %res
-}
-declare <2 x double> @llvm.x86.fma.vfnmadd.sd(<2 x double>, <2 x double>, <2 x double>) nounwind readnone
-
-define <2 x double> @test_x86_fnmadd_pd(<2 x double> %a0, <2 x double> %a1, <2 x double> %a2) {
-  ; CHECK: fnmadd213pd
-  %res = call <2 x double> @llvm.x86.fma.vfnmadd.pd(<2 x double> %a0, <2 x double> %a1, <2 x double> %a2) nounwind
-  ret <2 x double> %res
-}
-declare <2 x double> @llvm.x86.fma.vfnmadd.pd(<2 x double>, <2 x double>, <2 x double>) nounwind readnone
-
-
-
-define <2 x double> @test_x86_fmsub_sd(<2 x double> %a0, <2 x double> %a1, <2 x double> %a2) {
-  ; CHECK: fmsub213sd
-  %res = call <2 x double> @llvm.x86.fma.vfmsub.sd(<2 x double> %a0, <2 x double> %a1, <2 x double> %a2) nounwind
-  ret <2 x double> %res
-}
-declare <2 x double> @llvm.x86.fma.vfmsub.sd(<2 x double>, <2 x double>, <2 x double>) nounwind readnone
-
-define <2 x double> @test_x86_fmsub_pd(<2 x double> %a0, <2 x double> %a1, <2 x double> %a2) {
-  ; CHECK: fmsub213pd
-  %res = call <2 x double> @llvm.x86.fma.vfmsub.pd(<2 x double> %a0, <2 x double> %a1, <2 x double> %a2) nounwind
-  ret <2 x double> %res
-}
-declare <2 x double> @llvm.x86.fma.vfmsub.pd(<2 x double>, <2 x double>, <2 x double>) nounwind readnone
-
-define <2 x double> @test_x86_fnmsub_sd(<2 x double> %a0, <2 x double> %a1, <2 x double> %a2) {
-  ; CHECK: fnmsub213sd
-  %res = call <2 x double> @llvm.x86.fma.vfnmsub.sd(<2 x double> %a0, <2 x double> %a1, <2 x double> %a2) nounwind
-  ret <2 x double> %res
-}
-declare <2 x double> @llvm.x86.fma.vfnmsub.sd(<2 x double>, <2 x double>, <2 x double>) nounwind readnone
-
-define <2 x double> @test_x86_fnmsub_pd(<2 x double> %a0, <2 x double> %a1, <2 x double> %a2) {
-  ; CHECK: fnmsub213pd
-  %res = call <2 x double> @llvm.x86.fma.vfnmsub.pd(<2 x double> %a0, <2 x double> %a1, <2 x double> %a2) nounwind
-  ret <2 x double> %res
-}
-declare <2 x double> @llvm.x86.fma.vfnmsub.pd(<2 x double>, <2 x double>, <2 x double>) nounwind readnone
diff --git a/test/CodeGen/X86/fold-and-shift.ll b/test/CodeGen/X86/fold-and-shift.ll
index 074dc261f70..00173efff69 100644
--- a/test/CodeGen/X86/fold-and-shift.ll
+++ b/test/CodeGen/X86/fold-and-shift.ll
@@ -1,6 +1,4 @@
 ; RUN: llc < %s -march=x86 | FileCheck %s
-; @LOCALMOD
-; RUN: llc < %s -mtriple=x86_64-unknown-nacl | FileCheck %s --check-prefix=NACL
 
 define i32 @t1(i8* %X, i32 %i) {
 ; CHECK-LABEL: t1:
@@ -9,14 +7,6 @@ define i32 @t1(i8* %X, i32 %i) {
 ; CHECK: movl (%{{...}},%{{...}},4),
 ; CHECK: ret
 
-; NaCl64 cannot fold everything into one memory operand, but should fold it
-; into a lea
-; NACL-LABEL: t1:
-; NACL-NOT: and
-; NACL: movzbl
-; NACL: leal (%{{...}},%{{...}},4),
-; NACL: movl %nacl:(%r15,%{{...}}),
-
 entry:
   %tmp2 = shl i32 %i, 2
   %tmp4 = and i32 %tmp2, 1020
diff --git a/test/CodeGen/X86/fold-load-binops.ll b/test/CodeGen/X86/fold-load-binops.ll
new file mode 100644
index 00000000000..6d501c74fe5
--- /dev/null
+++ b/test/CodeGen/X86/fold-load-binops.ll
@@ -0,0 +1,142 @@
+; RUN: llc -mtriple=x86_64-unknown-unknown -mattr=+sse2 < %s | FileCheck %s --check-prefix=SSE
+; RUN: llc -mtriple=x86_64-unknown-unknown -mattr=+avx < %s | FileCheck %s --check-prefix=AVX
+
+; Verify that we're folding the load into the math instruction.
+; This pattern is generated out of the simplest intrinsics usage:
+;  _mm_add_ss(a, _mm_load_ss(b));
+
+define <4 x float> @addss(<4 x float> %va, float* %pb) {
+; SSE-LABEL: addss:
+; SSE:       # BB#0:
+; SSE-NEXT:    addss (%rdi), %xmm0
+; SSE-NEXT:    retq
+;
+; AVX-LABEL: addss:
+; AVX:       # BB#0:
+; AVX-NEXT:    vaddss (%rdi), %xmm0, %xmm0
+; AVX-NEXT:    retq
+    %a = extractelement <4 x float> %va, i32 0
+    %b = load float, float* %pb
+    %r = fadd float %a, %b
+    %vr = insertelement <4 x float> %va, float %r, i32 0
+    ret <4 x float> %vr
+}
+
+define <2 x double> @addsd(<2 x double> %va, double* %pb) {
+; SSE-LABEL: addsd:
+; SSE:       # BB#0:
+; SSE-NEXT:    addsd (%rdi), %xmm0
+; SSE-NEXT:    retq
+;
+; AVX-LABEL: addsd:
+; AVX:       # BB#0:
+; AVX-NEXT:    vaddsd (%rdi), %xmm0, %xmm0
+; AVX-NEXT:    retq
+    %a = extractelement <2 x double> %va, i32 0
+    %b = load double, double* %pb
+    %r = fadd double %a, %b
+    %vr = insertelement <2 x double> %va, double %r, i32 0
+    ret <2 x double> %vr
+}
+
+define <4 x float> @subss(<4 x float> %va, float* %pb) {
+; SSE-LABEL: subss:
+; SSE:       # BB#0:
+; SSE-NEXT:    subss (%rdi), %xmm0
+; SSE-NEXT:    retq
+;
+; AVX-LABEL: subss:
+; AVX:       # BB#0:
+; AVX-NEXT:    vsubss (%rdi), %xmm0, %xmm0
+; AVX-NEXT:    retq
+    %a = extractelement <4 x float> %va, i32 0
+    %b = load float, float* %pb
+    %r = fsub float %a, %b
+    %vr = insertelement <4 x float> %va, float %r, i32 0
+    ret <4 x float> %vr
+}
+
+define <2 x double> @subsd(<2 x double> %va, double* %pb) {
+; SSE-LABEL: subsd:
+; SSE:       # BB#0:
+; SSE-NEXT:    subsd (%rdi), %xmm0
+; SSE-NEXT:    retq
+;
+; AVX-LABEL: subsd:
+; AVX:       # BB#0:
+; AVX-NEXT:    vsubsd (%rdi), %xmm0, %xmm0
+; AVX-NEXT:    retq
+    %a = extractelement <2 x double> %va, i32 0
+    %b = load double, double* %pb
+    %r = fsub double %a, %b
+    %vr = insertelement <2 x double> %va, double %r, i32 0
+    ret <2 x double> %vr
+}
+
+define <4 x float> @mulss(<4 x float> %va, float* %pb) {
+; SSE-LABEL: mulss:
+; SSE:       # BB#0:
+; SSE-NEXT:    mulss (%rdi), %xmm0
+; SSE-NEXT:    retq
+;
+; AVX-LABEL: mulss:
+; AVX:       # BB#0:
+; AVX-NEXT:    vmulss (%rdi), %xmm0, %xmm0
+; AVX-NEXT:    retq
+    %a = extractelement <4 x float> %va, i32 0
+    %b = load float, float* %pb
+    %r = fmul float %a, %b
+    %vr = insertelement <4 x float> %va, float %r, i32 0
+    ret <4 x float> %vr
+}
+
+define <2 x double> @mulsd(<2 x double> %va, double* %pb) {
+; SSE-LABEL: mulsd:
+; SSE:       # BB#0:
+; SSE-NEXT:    mulsd (%rdi), %xmm0
+; SSE-NEXT:    retq
+;
+; AVX-LABEL: mulsd:
+; AVX:       # BB#0:
+; AVX-NEXT:    vmulsd (%rdi), %xmm0, %xmm0
+; AVX-NEXT:    retq
+    %a = extractelement <2 x double> %va, i32 0
+    %b = load double, double* %pb
+    %r = fmul double %a, %b
+    %vr = insertelement <2 x double> %va, double %r, i32 0
+    ret <2 x double> %vr
+}
+
+define <4 x float> @divss(<4 x float> %va, float* %pb) {
+; SSE-LABEL: divss:
+; SSE:       # BB#0:
+; SSE-NEXT:    divss (%rdi), %xmm0
+; SSE-NEXT:    retq
+;
+; AVX-LABEL: divss:
+; AVX:       # BB#0:
+; AVX-NEXT:    vdivss (%rdi), %xmm0, %xmm0
+; AVX-NEXT:    retq
+    %a = extractelement <4 x float> %va, i32 0
+    %b = load float, float* %pb
+    %r = fdiv float %a, %b
+    %vr = insertelement <4 x float> %va, float %r, i32 0
+    ret <4 x float> %vr
+}
+
+define <2 x double> @divsd(<2 x double> %va, double* %pb) {
+; SSE-LABEL: divsd:
+; SSE:       # BB#0:
+; SSE-NEXT:    divsd (%rdi), %xmm0
+; SSE-NEXT:    retq
+;
+; AVX-LABEL: divsd:
+; AVX:       # BB#0:
+; AVX-NEXT:    vdivsd (%rdi), %xmm0, %xmm0
+; AVX-NEXT:    retq
+    %a = extractelement <2 x double> %va, i32 0
+    %b = load double, double* %pb
+    %r = fdiv double %a, %b
+    %vr = insertelement <2 x double> %va, double %r, i32 0
+    ret <2 x double> %vr
+}
diff --git a/test/CodeGen/X86/fold-load-unops.ll b/test/CodeGen/X86/fold-load-unops.ll
index b03e80be799..fcde0218158 100644
--- a/test/CodeGen/X86/fold-load-unops.ll
+++ b/test/CodeGen/X86/fold-load-unops.ll
@@ -1,14 +1,18 @@
-; RUN: llc -mtriple=x86_64-unknown-unknown -mattr=+avx < %s | FileCheck %s
+; RUN: llc -mtriple=x86_64-unknown-unknown -mattr=+sse2 < %s | FileCheck %s --check-prefix=SSE
+; RUN: llc -mtriple=x86_64-unknown-unknown -mattr=+avx < %s | FileCheck %s --check-prefix=AVX
 
 ; Verify that we're folding the load into the math instruction.
 
-; FIXME: The folding should also happen without the avx attribute; 
-; ie, when generating SSE (non-VEX-prefixed) instructions.
-
 define float @rcpss(float* %a) {
-; CHECK-LABEL: rcpss:
-; CHECK:       vrcpss (%rdi), %xmm0, %xmm0
-
+; SSE-LABEL: rcpss:
+; SSE:       # BB#0:
+; SSE-NEXT:    rcpss (%rdi), %xmm0
+; SSE-NEXT:    retq
+;
+; AVX-LABEL: rcpss:
+; AVX:       # BB#0:
+; AVX-NEXT:    vrcpss (%rdi), %xmm0, %xmm0
+; AVX-NEXT:    retq
     %ld = load float, float* %a
     %ins = insertelement <4 x float> undef, float %ld, i32 0
     %res = tail call <4 x float> @llvm.x86.sse.rcp.ss(<4 x float> %ins)
@@ -17,9 +21,15 @@ define float @rcpss(float* %a) {
 }
 
 define float @rsqrtss(float* %a) {
-; CHECK-LABEL: rsqrtss:
-; CHECK:       vrsqrtss (%rdi), %xmm0, %xmm0
-
+; SSE-LABEL: rsqrtss:
+; SSE:       # BB#0:
+; SSE-NEXT:    rsqrtss (%rdi), %xmm0
+; SSE-NEXT:    retq
+;
+; AVX-LABEL: rsqrtss:
+; AVX:       # BB#0:
+; AVX-NEXT:    vrsqrtss (%rdi), %xmm0, %xmm0
+; AVX-NEXT:    retq
     %ld = load float, float* %a
     %ins = insertelement <4 x float> undef, float %ld, i32 0
     %res = tail call <4 x float> @llvm.x86.sse.rsqrt.ss(<4 x float> %ins)
@@ -28,9 +38,15 @@ define float @rsqrtss(float* %a) {
 }
 
 define float @sqrtss(float* %a) {
-; CHECK-LABEL: sqrtss:
-; CHECK:       vsqrtss (%rdi), %xmm0, %xmm0
-
+; SSE-LABEL: sqrtss:
+; SSE:       # BB#0:
+; SSE-NEXT:    sqrtss (%rdi), %xmm0
+; SSE-NEXT:    retq
+;
+; AVX-LABEL: sqrtss:
+; AVX:       # BB#0:
+; AVX-NEXT:    vsqrtss (%rdi), %xmm0, %xmm0
+; AVX-NEXT:    retq
     %ld = load float, float* %a
     %ins = insertelement <4 x float> undef, float %ld, i32 0
     %res = tail call <4 x float> @llvm.x86.sse.sqrt.ss(<4 x float> %ins)
@@ -39,9 +55,15 @@ define float @sqrtss(float* %a) {
 }
 
 define double @sqrtsd(double* %a) {
-; CHECK-LABEL: sqrtsd:
-; CHECK:       vsqrtsd (%rdi), %xmm0, %xmm0
-
+; SSE-LABEL: sqrtsd:
+; SSE:       # BB#0:
+; SSE-NEXT:    sqrtsd (%rdi), %xmm0
+; SSE-NEXT:    retq
+;
+; AVX-LABEL: sqrtsd:
+; AVX:       # BB#0:
+; AVX-NEXT:    vsqrtsd (%rdi), %xmm0, %xmm0
+; AVX-NEXT:    retq
     %ld = load double, double* %a
     %ins = insertelement <2 x double> undef, double %ld, i32 0
     %res = tail call <2 x double> @llvm.x86.sse2.sqrt.sd(<2 x double> %ins)
diff --git a/test/CodeGen/X86/fold-push.ll b/test/CodeGen/X86/fold-push.ll
new file mode 100644
index 00000000000..f5f33321c13
--- /dev/null
+++ b/test/CodeGen/X86/fold-push.ll
@@ -0,0 +1,40 @@
+; RUN: llc < %s -mtriple=i686-windows | FileCheck %s -check-prefix=CHECK -check-prefix=NORMAL
+; RUN: llc < %s -mtriple=i686-windows -mattr=call-reg-indirect | FileCheck %s -check-prefix=CHECK -check-prefix=SLM
+
+declare void @foo(i32 %r)
+
+define void @test(i32 %a, i32 %b) optsize {
+; CHECK-LABEL: test:
+; CHECK: movl [[EAX:%e..]], (%esp)
+; CHECK-NEXT: pushl [[EAX]]
+; CHECK-NEXT: calll
+; CHECK-NEXT: addl $4, %esp
+; CHECK: nop
+; NORMAL: pushl (%esp)
+; SLM: movl (%esp), [[RELOAD:%e..]]
+; SLM-NEXT: pushl [[RELOAD]]
+; CHECK: calll
+; CHECK-NEXT: addl $4, %esp
+  %c = add i32 %a, %b
+  call void @foo(i32 %c)
+  call void asm sideeffect "nop", "~{ax},~{bx},~{cx},~{dx},~{bp},~{si},~{di}"()
+  call void @foo(i32 %c)
+  ret void
+}
+
+define void @test_min(i32 %a, i32 %b) minsize {
+; CHECK-LABEL: test_min:
+; CHECK: movl [[EAX:%e..]], (%esp)
+; CHECK-NEXT: pushl [[EAX]]
+; CHECK-NEXT: calll
+; CHECK-NEXT: addl $4, %esp
+; CHECK: nop
+; CHECK: pushl (%esp)
+; CHECK: calll
+; CHECK-NEXT: addl $4, %esp
+  %c = add i32 %a, %b
+  call void @foo(i32 %c)
+  call void asm sideeffect "nop", "~{ax},~{bx},~{cx},~{dx},~{bp},~{si},~{di}"()
+  call void @foo(i32 %c)
+  ret void
+}
diff --git a/test/CodeGen/X86/fold-vector-bv-crash.ll b/test/CodeGen/X86/fold-vector-bv-crash.ll
new file mode 100644
index 00000000000..8c72afb7e97
--- /dev/null
+++ b/test/CodeGen/X86/fold-vector-bv-crash.ll
@@ -0,0 +1,17 @@
+; RUN: llc < %s -mtriple=i686-unknown -mattr=+avx
+; RUN: llc < %s -mtriple=x86_64-unknown -mattr=+avx
+
+;
+; llvm-stress generated crash case due to build_vector implicit
+; truncation bug from constant folding after legalization.
+;
+
+@G = external global i32
+
+define void @bv_crash_test() {
+  %I = insertelement <4 x i64> zeroinitializer, i64 15910, i32 0
+  %Tr = trunc <4 x i64> %I to <4 x i8>
+  %Bc = bitcast <4 x i8> %Tr to i32
+  store volatile i32 %Bc, i32* @G
+  ret void
+}
diff --git a/test/CodeGen/X86/fold-vector-sext-crash2.ll b/test/CodeGen/X86/fold-vector-sext-crash2.ll
new file mode 100644
index 00000000000..44c836195ab
--- /dev/null
+++ b/test/CodeGen/X86/fold-vector-sext-crash2.ll
@@ -0,0 +1,92 @@
+; RUN: llc < %s -march=x86    | FileCheck %s -check-prefix=X32
+; RUN: llc < %s -march=x86-64 | FileCheck %s -check-prefix=X64
+
+; DAGCombiner crashes during sext folding
+
+define <2 x i256> @test_sext1() {
+  %Se = sext <2 x i8> <i8 -100, i8 -99> to <2 x i256>
+  %Shuff = shufflevector <2 x i256> zeroinitializer, <2 x i256> %Se, <2 x i32> <i32 1, i32 3>
+  ret <2 x i256> %Shuff
+
+  ; X64-LABEL: test_sext1
+  ; X64:       movq $-1
+  ; X64-NEXT:  movq $-1
+  ; X64-NEXT:  movq $-1
+  ; X64-NEXT:  movq $-99
+
+  ; X32-LABEL: test_sext1
+  ; X32:       movl $-1
+  ; X32-NEXT:  movl $-1
+  ; X32-NEXT:  movl $-1
+  ; X32-NEXT:  movl $-1
+  ; X32-NEXT:  movl $-1
+  ; X32-NEXT:  movl $-1
+  ; X32-NEXT:  movl $-1
+  ; X32-NEXT:  movl $-99
+}
+
+define <2 x i256> @test_sext2() {
+  %Se = sext <2 x i128> <i128 -2000, i128 -1999> to <2 x i256>
+  %Shuff = shufflevector <2 x i256> zeroinitializer, <2 x i256> %Se, <2 x i32> <i32 1, i32 3>
+  ret <2 x i256> %Shuff
+
+  ; X64-LABEL: test_sext2
+  ; X64:       movq $-1
+  ; X64-NEXT:  movq $-1
+  ; X64-NEXT:  movq $-1
+  ; X64-NEXT:  movq $-1999
+
+  ; X32-LABEL: test_sext2
+  ; X32:       movl $-1
+  ; X32-NEXT:  movl $-1
+  ; X32-NEXT:  movl $-1
+  ; X32-NEXT:  movl $-1
+  ; X32-NEXT:  movl $-1
+  ; X32-NEXT:  movl $-1
+  ; X32-NEXT:  movl $-1
+  ; X32-NEXT:  movl $-1999
+}
+
+define <2 x i256> @test_zext1() {
+  %Se = zext <2 x i8> <i8 -1, i8 -2> to <2 x i256>
+  %Shuff = shufflevector <2 x i256> zeroinitializer, <2 x i256> %Se, <2 x i32> <i32 1, i32 3>
+  ret <2 x i256> %Shuff
+
+  ; X64-LABEL: test_zext1
+  ; X64:       movq $0
+  ; X64-NEXT:  movq $0
+  ; X64-NEXT:  movq $0
+  ; X64-NEXT:  movq $254
+
+  ; X32-LABEL: test_zext1
+  ; X32:       movl $0
+  ; X32-NEXT:  movl $0
+  ; X32-NEXT:  movl $0
+  ; X32-NEXT:  movl $0
+  ; X32-NEXT:  movl $0
+  ; X32-NEXT:  movl $0
+  ; X32-NEXT:  movl $0
+  ; X32-NEXT:  movl $254
+}
+
+define <2 x i256> @test_zext2() {
+  %Se = zext <2 x i128> <i128 -1, i128 -2> to <2 x i256>
+  %Shuff = shufflevector <2 x i256> zeroinitializer, <2 x i256> %Se, <2 x i32> <i32 1, i32 3>
+  ret <2 x i256> %Shuff
+
+  ; X64-LABEL: test_zext2
+  ; X64:       movq $0
+  ; X64-NEXT:  movq $0
+  ; X64-NEXT:  movq $-1
+  ; X64-NEXT:  movq $-2
+
+  ; X32-LABEL: test_zext2
+  ; X32:       movl $0
+  ; X32-NEXT:  movl $0
+  ; X32-NEXT:  movl $0
+  ; X32-NEXT:  movl $0
+  ; X32-NEXT:  movl $-1
+  ; X32-NEXT:  movl $-1
+  ; X32-NEXT:  movl $-1
+  ; X32-NEXT:  movl $-2
+}
diff --git a/test/CodeGen/X86/fold-vector-shl-crash.ll b/test/CodeGen/X86/fold-vector-shl-crash.ll
new file mode 100644
index 00000000000..9f81e44074f
--- /dev/null
+++ b/test/CodeGen/X86/fold-vector-shl-crash.ll
@@ -0,0 +1,9 @@
+; RUN: llc < %s -march=x86-64 | FileCheck %s
+; RUN: llc < %s -march=x86    | FileCheck %s
+
+;CHECK-LABEL: test
+define <2 x i256> @test() {
+  %S = shufflevector <2 x i256> zeroinitializer, <2 x i256> <i256 -1, i256 -1>, <2 x i32> <i32 0, i32 2>
+  %B = shl <2 x i256> %S, <i256 -1, i256 -1> ; DAG Combiner crashes here
+  ret <2 x i256> %B
+}
diff --git a/test/CodeGen/X86/fold-vector-shuffle-crash.ll b/test/CodeGen/X86/fold-vector-shuffle-crash.ll
new file mode 100644
index 00000000000..df9c6924b27
--- /dev/null
+++ b/test/CodeGen/X86/fold-vector-shuffle-crash.ll
@@ -0,0 +1,386 @@
+; RUN: llc < %s -mtriple=x86_64-unknown -mcpu=corei7
+
+define void @autogen_SD13708(i32) {
+BB:
+ %Shuff7 = shufflevector <8 x i32> zeroinitializer, <8 x i32> zeroinitializer, <8 x i32> <i32 8, i32 10, i32 12, i32 14, i32 undef, i32 2, i32 4, i32 undef>
+ br label %CF
+
+CF:
+ %Tr = trunc <8 x i64> zeroinitializer to <8 x i32>
+ %Shuff20 = shufflevector <8 x i32> %Shuff7, <8 x i32> %Tr, <8 x i32> <i32 13, i32 15, i32 1, i32 3, i32 5, i32 7, i32 undef, i32 11>
+ br i1 undef, label %CF, label %CF247
+
+CF247:
+ %I171 = insertelement <8 x i32> %Shuff20, i32 %0, i32 0
+ br i1 undef, label %CF, label %CF247
+}
+
+define void @autogen_SD13800(i8*, i32*, i64*, i32, i64, i8) {
+BB:
+  %A4 = alloca <8 x i1>
+  %A3 = alloca i32
+  %A2 = alloca <2 x float>
+  %A1 = alloca <2 x double>
+  %A = alloca <8 x float>
+  %L = load <8 x i1>, <8 x i1>* %A4
+  store i8 %5, i8* %0
+  %E = extractelement <2 x i64> zeroinitializer, i32 0
+  %Shuff = shufflevector <4 x i64> zeroinitializer, <4 x i64> zeroinitializer, <4 x i32> <i32 7, i32 undef, i32 undef, i32 5>
+  %I = insertelement <8 x i64> zeroinitializer, i64 419346, i32 1
+  %B = shl i64 426618, 419346
+  %Tr = trunc <8 x i64> %I to <8 x i16>
+  %Sl = select i1 false, <4 x i64> zeroinitializer, <4 x i64> zeroinitializer
+  %Cmp = icmp eq <16 x i64> zeroinitializer, zeroinitializer
+  %L5 = load i8, i8* %0
+  store i8 17, i8* %0
+  %E6 = extractelement <4 x i64> zeroinitializer, i32 1
+  %Shuff7 = shufflevector <2 x i64> zeroinitializer, <2 x i64> <i64 -1, i64 -1>, <2 x i32> <i32 0, i32 2>
+  %I8 = insertelement <16 x i8> <i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1>, i8 %L5, i32 2
+  %B9 = mul <8 x i16> %Tr, %Tr
+  %FC = fptosi float 0xBDF7B90B80000000 to i32
+  %Sl10 = select i1 false, float 0xBDF7B90B80000000, float 0xB875A90980000000
+  %Cmp11 = icmp slt <2 x i64> zeroinitializer, %Shuff7
+  %L12 = load <8 x float>, <8 x float>* %A
+  store <2 x double> <double 0xFFFFFFFFFFFFFFFF, double 0.000000e+00>, <2 x double>* %A1
+  %E13 = extractelement <4 x i64> zeroinitializer, i32 2
+  %Shuff14 = shufflevector <2 x i32> zeroinitializer, <2 x i32> zeroinitializer, <2 x i32> <i32 1, i32 3>
+  %I15 = insertelement <16 x i8> <i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1>, i8 %L5, i32 15
+  %B16 = add <2 x i64> zeroinitializer, <i64 -1, i64 -1>
+  %BC = bitcast i64 426618 to double
+  %Sl17 = select i1 false, <2 x i32> zeroinitializer, <2 x i32> zeroinitializer
+  %Cmp18 = icmp slt <8 x i1> %L, %L
+  %L19 = load i8, i8* %0
+  store i8 %L5, i8* %0
+  %E20 = extractelement <16 x i8> %I8, i32 1
+  %Shuff21 = shufflevector <8 x i64> zeroinitializer, <8 x i64> %I, <8 x i32> <i32 undef, i32 6, i32 8, i32 10, i32 12, i32 14, i32 0, i32 2>
+  %I22 = insertelement <8 x i16> %Tr, i16 18460, i32 6
+  %B23 = sub i64 419346, %4
+  %FC24 = fptosi double 0xE603EE221901D6A0 to i32
+  %Sl25 = select i1 false, i8 %L5, i8 %5
+  %Cmp26 = icmp ugt i64 %B, %B23
+  br label %CF253
+
+CF253:                                            ; preds = %CF253, %CF271, %CF260, %BB
+  %L27 = load i8, i8* %0
+  store i8 %L5, i8* %0
+  %E28 = extractelement <2 x i64> zeroinitializer, i32 0
+  %Shuff29 = shufflevector <16 x i8> <i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1>, <16 x i8> %I8, <16 x i32> <i32 15, i32 17, i32 19, i32 21, i32 23, i32 25, i32 27, i32 undef, i32 31, i32 undef, i32 3, i32 5, i32 7, i32 9, i32 11, i32 13>
+  %I30 = insertelement <8 x i1> %Cmp18, i1 false, i32 1
+  %B31 = fsub double 0xE603EE221901D6A0, %BC
+  %Tr32 = trunc <2 x i64> <i64 -1, i64 -1> to <2 x i32>
+  %Sl33 = select i1 false, double %BC, double %B31
+  %Cmp34 = icmp sgt <2 x i32> zeroinitializer, %Shuff14
+  %L35 = load i8, i8* %0
+  store i8 %L5, i8* %0
+  %E36 = extractelement <16 x i8> %Shuff29, i32 5
+  %Shuff37 = shufflevector <8 x i64> zeroinitializer, <8 x i64> %I, <8 x i32> <i32 8, i32 undef, i32 undef, i32 14, i32 0, i32 undef, i32 4, i32 6>
+  %I38 = insertelement <4 x i64> zeroinitializer, i64 %E28, i32 2
+  %FC39 = uitofp i8 %5 to double
+  %Sl40 = select i1 %Cmp26, i32 %3, i32 %FC
+  %Cmp41 = icmp sgt <2 x i64> zeroinitializer, <i64 -1, i64 -1>
+  %L42 = load i8, i8* %0
+  store i8 17, i8* %0
+  %E43 = extractelement <2 x i1> %Cmp41, i32 1
+  br i1 %E43, label %CF253, label %CF256
+
+CF256:                                            ; preds = %CF256, %CF253
+  %Shuff44 = shufflevector <8 x i64> zeroinitializer, <8 x i64> zeroinitializer, <8 x i32> <i32 14, i32 0, i32 2, i32 4, i32 6, i32 undef, i32 undef, i32 12>
+  %I45 = insertelement <8 x i32> zeroinitializer, i32 %FC, i32 0
+  %ZE = zext i8 %L19 to i32
+  %Sl46 = select i1 %E43, i8 %L35, i8 %L35
+  %Cmp47 = icmp ult i64 %E6, 426618
+  br i1 %Cmp47, label %CF256, label %CF271
+
+CF271:                                            ; preds = %CF256
+  %L48 = load i8, i8* %0
+  store i8 %L27, i8* %0
+  %E49 = extractelement <4 x i32> <i32 -1, i32 -1, i32 -1, i32 -1>, i32 2
+  %Shuff50 = shufflevector <8 x i64> zeroinitializer, <8 x i64> zeroinitializer, <8 x i32> <i32 undef, i32 7, i32 undef, i32 11, i32 13, i32 15, i32 1, i32 3>
+  %I51 = insertelement <8 x i64> zeroinitializer, i64 %4, i32 7
+  %B52 = xor <8 x i32> %I45, zeroinitializer
+  %BC53 = bitcast <4 x i32> <i32 -1, i32 -1, i32 -1, i32 -1> to <4 x float>
+  %Sl54 = select i1 false, <2 x i64> <i64 -1, i64 -1>, <2 x i64> <i64 -1, i64 -1>
+  %Cmp55 = icmp sgt i16 0, 18460
+  br i1 %Cmp55, label %CF253, label %CF255
+
+CF255:                                            ; preds = %CF255, %CF266, %CF270, %CF271
+  %L56 = load i8, i8* %0
+  store i8 %L35, i8* %0
+  %E57 = extractelement <4 x i64> zeroinitializer, i32 3
+  %Shuff58 = shufflevector <8 x i64> zeroinitializer, <8 x i64> %Shuff37, <8 x i32> <i32 undef, i32 undef, i32 10, i32 12, i32 14, i32 0, i32 2, i32 4>
+  %I59 = insertelement <4 x i32> <i32 -1, i32 -1, i32 -1, i32 -1>, i32 %FC, i32 0
+  %B60 = lshr <4 x i64> %I38, zeroinitializer
+  %FC61 = sitofp <8 x i1> %L to <8 x float>
+  %Sl62 = select i1 false, i8 %L19, i8 17
+  %Cmp63 = icmp ult i64 %E6, %E57
+  br i1 %Cmp63, label %CF255, label %CF266
+
+CF266:                                            ; preds = %CF255
+  %L64 = load i64, i64* %2
+  store i8 17, i8* %0
+  %E65 = extractelement <8 x i64> %Shuff21, i32 6
+  %Shuff66 = shufflevector <2 x i1> %Cmp11, <2 x i1> %Cmp41, <2 x i32> <i32 1, i32 3>
+  %I67 = insertelement <8 x i1> %I30, i1 false, i32 7
+  %FC68 = uitofp i8 %Sl62 to float
+  %Sl69 = select i1 false, i8 %L42, i8 17
+  %Cmp70 = icmp eq <2 x i32> zeroinitializer, zeroinitializer
+  %L71 = load i8, i8* %0
+  store i8 %5, i8* %0
+  %E72 = extractelement <2 x i64> <i64 -1, i64 -1>, i32 1
+  %Shuff73 = shufflevector <8 x i64> zeroinitializer, <8 x i64> %Shuff44, <8 x i32> <i32 undef, i32 14, i32 0, i32 2, i32 4, i32 6, i32 undef, i32 10>
+  %I74 = insertelement <2 x i1> %Cmp70, i1 %Cmp55, i32 0
+  %B75 = add <16 x i8> %I15, <i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1>
+  %FC76 = sitofp i8 33 to double
+  %Sl77 = select i1 %E43, double %BC, double %B31
+  %Cmp78 = icmp ult <8 x i64> %Shuff44, %I
+  %L79 = load i8, i8* %0
+  store i8 17, i8* %0
+  %E80 = extractelement <2 x i64> %Shuff7, i32 0
+  %Shuff81 = shufflevector <8 x i64> zeroinitializer, <8 x i64> %Shuff73, <8 x i32> <i32 undef, i32 5, i32 7, i32 9, i32 undef, i32 13, i32 15, i32 1>
+  %I82 = insertelement <8 x i64> %Shuff81, i64 %E57, i32 5
+  %FC83 = fptosi float %FC68 to i32
+  %Sl84 = select i1 %Cmp26, <2 x i64> <i64 -1, i64 -1>, <2 x i64> <i64 -1, i64 -1>
+  %Cmp85 = icmp ugt i64 %E6, %E57
+  br i1 %Cmp85, label %CF255, label %CF261
+
+CF261:                                            ; preds = %CF261, %CF266
+  %L86 = load i8, i8* %0
+  store i8 %L42, i8* %0
+  %E87 = extractelement <16 x i8> <i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1>, i32 7
+  %Shuff88 = shufflevector <16 x i8> %Shuff29, <16 x i8> %I15, <16 x i32> <i32 26, i32 28, i32 30, i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14, i32 16, i32 18, i32 20, i32 22, i32 24>
+  %I89 = insertelement <16 x i8> <i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1>, i8 %L35, i32 12
+  %B90 = shl i32 %3, %E49
+  %BC91 = bitcast <2 x i64> %Sl84 to <2 x double>
+  %Sl92 = select i1 false, i8 %L5, i8 %L19
+  %Cmp93 = icmp ugt i32 -1, %FC24
+  br i1 %Cmp93, label %CF261, label %CF268
+
+CF268:                                            ; preds = %CF268, %CF261
+  %L94 = load i8, i8* %0
+  store i8 %L5, i8* %0
+  %E95 = extractelement <8 x i64> %Shuff58, i32 0
+  %Shuff96 = shufflevector <8 x i64> %Shuff73, <8 x i64> %Shuff73, <8 x i32> <i32 3, i32 5, i32 undef, i32 9, i32 undef, i32 undef, i32 15, i32 1>
+  %I97 = insertelement <4 x i64> zeroinitializer, i64 %B23, i32 1
+  %B98 = or <8 x i64> %Shuff58, %Shuff50
+  %FC99 = sitofp <2 x i1> %Cmp34 to <2 x float>
+  %Sl100 = select i1 %Cmp85, i64 %4, i64 %E
+  %Cmp101 = icmp ne <2 x i64> %B16, zeroinitializer
+  %L102 = load i8, i8* %0
+  store i8 %L56, i8* %0
+  %E103 = extractelement <8 x i16> %I22, i32 6
+  %Shuff104 = shufflevector <2 x double> %BC91, <2 x double> %BC91, <2 x i32> <i32 1, i32 3>
+  %I105 = insertelement <8 x i64> %Shuff96, i64 198384, i32 7
+  %B106 = sdiv <8 x i32> %B52, %I45
+  %ZE107 = zext i16 0 to i32
+  %Sl108 = select i1 %E43, <16 x i8> %Shuff29, <16 x i8> <i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1>
+  %Cmp109 = icmp slt <16 x i64> zeroinitializer, zeroinitializer
+  %L110 = load <8 x float>, <8 x float>* %A
+  store i8 %L56, i8* %0
+  %E111 = extractelement <8 x i64> zeroinitializer, i32 3
+  %Shuff112 = shufflevector <2 x i1> %Shuff66, <2 x i1> %Cmp11, <2 x i32> <i32 2, i32 0>
+  %I113 = insertelement <2 x i64> %B16, i64 %E95, i32 0
+  %B114 = mul i8 %E20, %Sl25
+  %Tr115 = trunc <8 x i64> %I105 to <8 x i16>
+  %Sl116 = select i1 %Cmp26, <16 x i32> <i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1>, <16 x i32> <i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1>
+  %Cmp117 = icmp ult <8 x i16> %Tr, %Tr115
+  %L118 = load i8, i8* %0
+  store i8 %L19, i8* %0
+  %E119 = extractelement <4 x i32> %I59, i32 3
+  %Shuff120 = shufflevector <2 x i64> <i64 -1, i64 -1>, <2 x i64> %I113, <2 x i32> <i32 2, i32 0>
+  %I121 = insertelement <2 x i1> %Shuff66, i1 %Cmp26, i32 0
+  %B122 = fmul double 0.000000e+00, 0xE603EE221901D6A0
+  %FC123 = sitofp i64 %E6 to float
+  %Sl124 = select i1 false, <2 x i1> %Cmp41, <2 x i1> %Shuff66
+  %Cmp125 = icmp ult <4 x i64> %I38, %I38
+  %L126 = load i8, i8* %0
+  store i8 %L126, i8* %0
+  %E127 = extractelement <8 x i64> zeroinitializer, i32 7
+  %Shuff128 = shufflevector <2 x i1> %Cmp101, <2 x i1> %Cmp11, <2 x i32> <i32 undef, i32 0>
+  %I129 = insertelement <8 x i1> %Cmp18, i1 %E43, i32 0
+  %B130 = lshr i8 %L71, %L56
+  %FC131 = sitofp i32 %3 to float
+  %Sl132 = select i1 false, <2 x i64> %Shuff7, <2 x i64> %Sl84
+  %Cmp133 = icmp sge <8 x i16> %Tr, %Tr115
+  %L134 = load i8, i8* %0
+  store i8 %L102, i8* %0
+  %E135 = extractelement <16 x i8> %Shuff88, i32 3
+  %Shuff136 = shufflevector <8 x i64> %Shuff21, <8 x i64> zeroinitializer, <8 x i32> <i32 6, i32 8, i32 undef, i32 12, i32 14, i32 0, i32 2, i32 4>
+  %I137 = insertelement <2 x i64> zeroinitializer, i64 %E111, i32 0
+  %B138 = shl <8 x i64> %I51, %Shuff136
+  %Se = sext <2 x i32> %Tr32 to <2 x i64>
+  %Sl139 = select i1 %E43, <8 x i16> %Tr, <8 x i16> %Tr115
+  %Cmp140 = icmp sge <2 x i32> %Sl17, %Tr32
+  %L141 = load i8, i8* %0
+  store i8 17, i8* %0
+  %E142 = extractelement <8 x i16> %Tr115, i32 6
+  %Shuff143 = shufflevector <8 x i64> zeroinitializer, <8 x i64> %Shuff21, <8 x i32> <i32 1, i32 3, i32 undef, i32 7, i32 undef, i32 11, i32 13, i32 15>
+  %I144 = insertelement <4 x i64> %Shuff, i64 %4, i32 3
+  %B145 = sub <2 x i64> <i64 -1, i64 -1>, %I113
+  %Se146 = sext i8 %E135 to i32
+  %Sl147 = select i1 %Cmp55, <2 x i32> %Tr32, <2 x i32> zeroinitializer
+  %Cmp148 = icmp eq <8 x i1> %I30, %Cmp18
+  %L149 = load i8, i8* %0
+  store i8 %L56, i8* %0
+  %E150 = extractelement <2 x i64> %I113, i32 0
+  %Shuff151 = shufflevector <8 x i64> zeroinitializer, <8 x i64> %I, <8 x i32> <i32 9, i32 11, i32 13, i32 15, i32 1, i32 3, i32 undef, i32 7>
+  %I152 = insertelement <8 x i64> %Shuff136, i64 %E6, i32 3
+  %B153 = frem float %FC68, %FC123
+  %Se154 = sext i1 false to i32
+  %Sl155 = select i1 %Cmp26, i8 %E20, i8 %L19
+  %Cmp156 = icmp eq i64 198384, %4
+  br i1 %Cmp156, label %CF268, label %CF270
+
+CF270:                                            ; preds = %CF268
+  %L157 = load i8, i8* %0
+  store i8 %L157, i8* %0
+  %E158 = extractelement <8 x i1> %Cmp78, i32 1
+  br i1 %E158, label %CF255, label %CF260
+
+CF260:                                            ; preds = %CF270
+  %Shuff159 = shufflevector <8 x i64> zeroinitializer, <8 x i64> %Shuff81, <8 x i32> <i32 undef, i32 6, i32 8, i32 undef, i32 12, i32 14, i32 0, i32 2>
+  %I160 = insertelement <2 x i1> %Cmp11, i1 %Cmp156, i32 0
+  %B161 = urem <2 x i32> zeroinitializer, %Sl147
+  %Se162 = sext i8 %L48 to i16
+  %Sl163 = select i1 %Cmp93, i32 %FC83, i32 378892
+  %Cmp164 = fcmp uge double 0xE603EE221901D6A0, 0xE603EE221901D6A0
+  br i1 %Cmp164, label %CF253, label %CF254
+
+CF254:                                            ; preds = %CF254, %CF265, %CF263, %CF260
+  %L165 = load i8, i8* %0
+  store i8 %Sl62, i8* %0
+  %E166 = extractelement <8 x i64> %Shuff58, i32 1
+  %Shuff167 = shufflevector <8 x i64> %Shuff58, <8 x i64> %Shuff96, <8 x i32> <i32 12, i32 14, i32 0, i32 undef, i32 4, i32 undef, i32 8, i32 10>
+  %I168 = insertelement <2 x double> %BC91, double %BC, i32 0
+  %B169 = ashr i16 %E142, %E103
+  %FC170 = sitofp <2 x i64> %Sl84 to <2 x float>
+  %Sl171 = select i1 %Cmp156, i8 %L165, i8 %5
+  %Cmp172 = icmp ugt i8 %E20, %L102
+  br i1 %Cmp172, label %CF254, label %CF262
+
+CF262:                                            ; preds = %CF262, %CF254
+  %L173 = load i8, i8* %0
+  store i8 %L94, i8* %0
+  %E174 = extractelement <2 x i1> %Cmp70, i32 0
+  br i1 %E174, label %CF262, label %CF264
+
+CF264:                                            ; preds = %CF264, %CF262
+  %Shuff175 = shufflevector <16 x i1> %Cmp, <16 x i1> %Cmp, <16 x i32> <i32 undef, i32 9, i32 undef, i32 13, i32 undef, i32 17, i32 19, i32 21, i32 23, i32 25, i32 27, i32 29, i32 undef, i32 1, i32 3, i32 5>
+  %I176 = insertelement <8 x i64> %Shuff21, i64 419346, i32 1
+  %B177 = lshr <2 x i32> %Sl17, zeroinitializer
+  %FC178 = sitofp <8 x i32> %B106 to <8 x float>
+  %Sl179 = select i1 %Cmp156, i8 %B114, i8 %Sl171
+  %Cmp180 = icmp ugt <4 x i64> %B60, zeroinitializer
+  %L181 = load i8, i8* %0
+  store i8 %L102, i8* %0
+  %E182 = extractelement <8 x i64> zeroinitializer, i32 0
+  %Shuff183 = shufflevector <8 x i64> zeroinitializer, <8 x i64> %I176, <8 x i32> <i32 3, i32 5, i32 undef, i32 undef, i32 11, i32 13, i32 undef, i32 1>
+  %I184 = insertelement <2 x i1> %Cmp34, i1 %Cmp63, i32 1
+  %B185 = urem i32 %Sl163, %Se146
+  %FC186 = sitofp i64 %E166 to float
+  %Sl187 = select i1 %Cmp156, i1 %E43, i1 %Cmp26
+  br i1 %Sl187, label %CF264, label %CF265
+
+CF265:                                            ; preds = %CF264
+  %Cmp188 = icmp uge <16 x i1> %Shuff175, %Cmp
+  %L189 = load i8, i8* %0
+  store i8 %L19, i8* %0
+  %E190 = extractelement <2 x i1> %Cmp11, i32 0
+  br i1 %E190, label %CF254, label %CF259
+
+CF259:                                            ; preds = %CF259, %CF265
+  %Shuff191 = shufflevector <8 x i1> %Cmp117, <8 x i1> %I30, <8 x i32> <i32 11, i32 13, i32 15, i32 1, i32 3, i32 5, i32 7, i32 9>
+  %I192 = insertelement <16 x i1> %Cmp188, i1 %Cmp85, i32 13
+  %B193 = urem <2 x i64> %Sl132, %Sl54
+  %Tr194 = trunc i64 %E166 to i8
+  %Sl195 = select i1 %Cmp93, <2 x i1> %I160, <2 x i1> %Shuff66
+  %Cmp196 = icmp ult <2 x i1> %Shuff66, %Cmp11
+  %L197 = load i8, i8* %0
+  store i8 %L5, i8* %0
+  %E198 = extractelement <8 x i64> %Shuff183, i32 0
+  %Shuff199 = shufflevector <8 x i16> %I22, <8 x i16> %Tr115, <8 x i32> <i32 3, i32 5, i32 undef, i32 9, i32 11, i32 13, i32 15, i32 undef>
+  %I200 = insertelement <16 x i8> %Shuff29, i8 %L197, i32 5
+  %B201 = and <2 x i64> %B145, %I113
+  %ZE202 = zext <2 x i1> %I74 to <2 x i64>
+  %Sl203 = select i1 %Cmp26, i8 %L126, i8 %L102
+  %Cmp204 = fcmp oeq <4 x float> %BC53, %BC53
+  %L205 = load i8, i8* %0
+  store i8 %5, i8* %0
+  %E206 = extractelement <2 x double> %Shuff104, i32 0
+  %Shuff207 = shufflevector <4 x i64> %I38, <4 x i64> zeroinitializer, <4 x i32> <i32 7, i32 undef, i32 3, i32 5>
+  %I208 = insertelement <8 x i64> %I82, i64 323142, i32 1
+  %B209 = lshr i8 %L56, %L5
+  %FC210 = fptoui double 0xE603EE221901D6A0 to i1
+  br i1 %FC210, label %CF259, label %CF263
+
+CF263:                                            ; preds = %CF259
+  %Sl211 = select i1 %E174, i32 %ZE, i32 %ZE107
+  %Cmp212 = icmp ne i32 %Se154, %Sl163
+  br i1 %Cmp212, label %CF254, label %CF257
+
+CF257:                                            ; preds = %CF263
+  %L213 = load i8, i8* %0
+  store i8 %L213, i8* %0
+  %E214 = extractelement <8 x i64> %Shuff81, i32 3
+  %Shuff215 = shufflevector <8 x i64> %Shuff159, <8 x i64> %Shuff136, <8 x i32> <i32 14, i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12>
+  %I216 = insertelement <8 x i64> %Shuff215, i64 323142, i32 0
+  %Se217 = sext i8 %L71 to i64
+  %Sl218 = select i1 %Cmp156, <8 x i16> %Tr115, <8 x i16> %Tr115
+  %Cmp219 = fcmp ole <2 x float> %FC170, %FC99
+  %L220 = load i8, i8* %0
+  store i8 %L19, i8* %0
+  %E221 = extractelement <8 x i64> zeroinitializer, i32 6
+  %Shuff222 = shufflevector <4 x i1> %Cmp204, <4 x i1> %Cmp125, <4 x i32> <i32 1, i32 undef, i32 5, i32 7>
+  %I223 = insertelement <8 x i1> %Cmp18, i1 %FC210, i32 3
+  %B224 = lshr i32 %E49, %FC24
+  %FC225 = sitofp <4 x i1> %Cmp180 to <4 x float>
+  %Sl226 = select i1 %Cmp93, i64 %E28, i64 %B23
+  %Cmp227 = icmp ugt <4 x i64> zeroinitializer, %B60
+  %L228 = load i8, i8* %0
+  store i8 %Sl46, i8* %0
+  %E229 = extractelement <1 x i32> zeroinitializer, i32 0
+  %Shuff230 = shufflevector <16 x i8> %Shuff29, <16 x i8> %I200, <16 x i32> <i32 11, i32 13, i32 15, i32 17, i32 19, i32 21, i32 23, i32 undef, i32 undef, i32 29, i32 31, i32 1, i32 undef, i32 5, i32 undef, i32 9>
+  %I231 = insertelement <8 x i64> %Shuff183, i64 %L64, i32 5
+  %B232 = fadd float %FC68, %FC68
+  %Se233 = sext i1 %Cmp172 to i64
+  %Sl234 = select i1 false, i1 %Cmp164, i1 %E43
+  br label %CF
+
+CF:                                               ; preds = %CF, %CF257
+  %Cmp235 = icmp ule i32 %Sl163, %Sl211
+  br i1 %Cmp235, label %CF, label %CF252
+
+CF252:                                            ; preds = %CF252, %CF269, %CF
+  %L236 = load i8, i8* %0
+  store i8 %L19, i8* %0
+  %E237 = extractelement <16 x i1> %Shuff175, i32 15
+  br i1 %E237, label %CF252, label %CF269
+
+CF269:                                            ; preds = %CF252
+  %Shuff238 = shufflevector <2 x i1> %I160, <2 x i1> %Cmp101, <2 x i32> undef
+  %I239 = insertelement <8 x i64> zeroinitializer, i64 %4, i32 0
+  %B240 = add i8 %L56, %Sl155
+  %Tr241 = trunc <2 x i32> %Sl147 to <2 x i1>
+  %Sl242 = select i1 %Sl234, <2 x float> %FC99, <2 x float> %FC99
+  %Cmp243 = icmp eq i8 %L5, %L118
+  br i1 %Cmp243, label %CF252, label %CF258
+
+CF258:                                            ; preds = %CF258, %CF269
+  %L244 = load i8, i8* %0
+  store i8 %L19, i8* %0
+  %E245 = extractelement <2 x i64> %B201, i32 1
+  %Shuff246 = shufflevector <4 x i64> zeroinitializer, <4 x i64> %I144, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
+  %I247 = insertelement <8 x i64> %Shuff73, i64 %E182, i32 2
+  %B248 = or i64 %Sl226, %E245
+  %Tr249 = trunc <2 x i64> <i64 -1, i64 -1> to <2 x i16>
+  %Sl250 = select i1 %FC210, i64 %E57, i64 %L64
+  %Cmp251 = icmp eq i32 %FC24, %FC
+  br i1 %Cmp251, label %CF258, label %CF267
+
+CF267:                                            ; preds = %CF258
+  store i8 %L42, i8* %0
+  store i8 %Sl69, i8* %0
+  store i8 %L5, i8* %0
+  store i8 %L134, i8* %0
+  store i8 %L141, i8* %0
+  ret void
+}
diff --git a/test/CodeGen/X86/fold-vector-trunc-sitofp.ll b/test/CodeGen/X86/fold-vector-trunc-sitofp.ll
new file mode 100644
index 00000000000..6a3be7aace1
--- /dev/null
+++ b/test/CodeGen/X86/fold-vector-trunc-sitofp.ll
@@ -0,0 +1,13 @@
+; RUN: llc < %s -mtriple=i686-unknown -mattr=+avx | FileCheck %s
+; RUN: llc < %s -mtriple=x86_64-unknown -mattr=+avx | FileCheck %s
+
+; Check that constant integers are correctly being truncated before float conversion
+
+define <4 x float> @test1() {
+; CHECK-LABEL: test1
+; CHECK: movaps {{.*#+}} xmm0 = [-1.000000e+00,0.000000e+00,-1.000000e+00,0.000000e+00]
+; CHECK-NEXT: ret
+  %1 = trunc <4 x i3> <i3 -1, i3 -22, i3 7, i3 8> to <4 x i1>
+  %2 = sitofp <4 x i1> %1 to <4 x float>
+  ret <4 x float> %2
+}
diff --git a/test/CodeGen/X86/fp-fast.ll b/test/CodeGen/X86/fp-fast.ll
index 7b08ad67220..27af5738ca3 100644
--- a/test/CodeGen/X86/fp-fast.ll
+++ b/test/CodeGen/X86/fp-fast.ll
@@ -1,126 +1,116 @@
-; RUN: llc -march=x86-64 -mcpu=corei7-avx -enable-unsafe-fp-math < %s | FileCheck %s
+; RUN: llc -mtriple=x86_64-unknown-unknown -mattr=avx -enable-unsafe-fp-math < %s | FileCheck %s
 
-; CHECK-LABEL: test1
 define float @test1(float %a) {
-; CHECK-NOT: addss
-; CHECK: mulss
-; CHECK-NOT: addss
-; CHECK: ret
+; CHECK-LABEL: test1:
+; CHECK:       # BB#0:
+; CHECK-NEXT:    vmulss {{.*}}(%rip), %xmm0, %xmm0
+; CHECK-NEXT:    retq
   %t1 = fadd float %a, %a
   %r = fadd float %t1, %t1
   ret float %r
 }
 
-; CHECK-LABEL: test2
 define float @test2(float %a) {
-; CHECK-NOT: addss
-; CHECK: mulss
-; CHECK-NOT: addss
-; CHECK: ret
+; CHECK-LABEL: test2:
+; CHECK:       # BB#0:
+; CHECK-NEXT:    vmulss {{.*}}(%rip), %xmm0, %xmm0
+; CHECK-NEXT:    retq
   %t1 = fmul float 4.0, %a
   %t2 = fadd float %a, %a
   %r = fadd float %t1, %t2
   ret float %r
 }
 
-; CHECK-LABEL: test3
 define float @test3(float %a) {
-; CHECK-NOT: addss
-; CHECK: mulss
-; CHECK-NOT: addss
-; CHECK: ret
+; CHECK-LABEL: test3:
+; CHECK:       # BB#0:
+; CHECK-NEXT:    vmulss {{.*}}(%rip), %xmm0, %xmm0
+; CHECK-NEXT:    retq
   %t1 = fmul float %a, 4.0
   %t2 = fadd float %a, %a
   %r = fadd float %t1, %t2
   ret float %r
 }
 
-; CHECK-LABEL: test4
 define float @test4(float %a) {
-; CHECK-NOT: addss
-; CHECK: mulss
-; CHECK-NOT: addss
-; CHECK: ret
+; CHECK-LABEL: test4:
+; CHECK:       # BB#0:
+; CHECK-NEXT:    vmulss {{.*}}(%rip), %xmm0, %xmm0
+; CHECK-NEXT:    retq
   %t1 = fadd float %a, %a
   %t2 = fmul float 4.0, %a
   %r = fadd float %t1, %t2
   ret float %r
 }
 
-; CHECK-LABEL: test5
 define float @test5(float %a) {
-; CHECK-NOT: addss
-; CHECK: mulss
-; CHECK-NOT: addss
-; CHECK: ret
+; CHECK-LABEL: test5:
+; CHECK:       # BB#0:
+; CHECK-NEXT:    vmulss {{.*}}(%rip), %xmm0, %xmm0
+; CHECK-NEXT:    retq
   %t1 = fadd float %a, %a
   %t2 = fmul float %a, 4.0
   %r = fadd float %t1, %t2
   ret float %r
 }
 
-; CHECK-LABEL: test6
 define float @test6(float %a) {
-; CHECK-NOT: addss
-; CHECK: xorps
-; CHECK-NOT: addss
-; CHECK: ret
+; CHECK-LABEL: test6:
+; CHECK:       # BB#0:
+; CHECK-NEXT:    vxorps %xmm0, %xmm0, %xmm0
+; CHECK-NEXT:    retq
   %t1 = fmul float 2.0, %a
   %t2 = fadd float %a, %a
   %r = fsub float %t1, %t2
   ret float %r
 }
 
-; CHECK-LABEL: test7
 define float @test7(float %a) {
-; CHECK-NOT: addss
-; CHECK: xorps
-; CHECK-NOT: addss
-; CHECK: ret
+; CHECK-LABEL: test7:
+; CHECK:       # BB#0:
+; CHECK-NEXT:    vxorps %xmm0, %xmm0, %xmm0
+; CHECK-NEXT:    retq
   %t1 = fmul float %a, 2.0
   %t2 = fadd float %a, %a
   %r = fsub float %t1, %t2
   ret float %r
 }
 
-; CHECK-LABEL: test8
 define float @test8(float %a) {
-; CHECK-NOT: fma
-; CHECK-NOT: mul
-; CHECK-NOT: add
-; CHECK: ret
+; CHECK-LABEL: test8:
+; CHECK:       # BB#0:
+; CHECK-NEXT:    retq
   %t1 = fmul float %a, 0.0
   %t2 = fadd float %a, %t1
   ret float %t2
 }
 
-; CHECK-LABEL: test9
 define float @test9(float %a) {
-; CHECK-NOT: fma
-; CHECK-NOT: mul
-; CHECK-NOT: add
-; CHECK: ret
+; CHECK-LABEL: test9:
+; CHECK:       # BB#0:
+; CHECK-NEXT:    retq
   %t1 = fmul float 0.0, %a
   %t2 = fadd float %t1, %a
   ret float %t2
 }
 
-; CHECK-LABEL: test10
 define float @test10(float %a) {
-; CHECK-NOT: add
-; CHECK: vxorps
-; CHECK: ret
+; CHECK-LABEL: test10:
+; CHECK:       # BB#0:
+; CHECK-NEXT:    vxorps %xmm0, %xmm0, %xmm0
+; CHECK-NEXT:    retq
   %t1 = fsub float -0.0, %a
   %t2 = fadd float %a, %t1
   ret float %t2
 }
 
-; CHECK-LABEL: test11
 define float @test11(float %a) {
-; CHECK-NOT: add
-; CHECK: vxorps
-; CHECK: ret
+; CHECK-LABEL: test11:
+; CHECK:       # BB#0:
+; CHECK-NEXT:    vxorps %xmm0, %xmm0, %xmm0
+; CHECK-NEXT:    retq
   %t1 = fsub float -0.0, %a
   %t2 = fadd float %a, %t1
   ret float %t2
 }
+
diff --git a/test/CodeGen/X86/fp-trunc.ll b/test/CodeGen/X86/fp-trunc.ll
index 6424bfc9c21..807a8c8fe5e 100644
--- a/test/CodeGen/X86/fp-trunc.ll
+++ b/test/CodeGen/X86/fp-trunc.ll
@@ -1,6 +1,8 @@
 ; RUN: llc < %s -march=x86 -mcpu=corei7 | FileCheck %s
 ; RUN: llc < %s -march=x86 -mcpu=core-avx-i | FileCheck %s --check-prefix=AVX
 
+target triple = "i686-pc-linux-gnu"
+
 define <1 x float> @test1(<1 x double> %x) nounwind {
 ; CHECK-LABEL: test1:
 ; CHECK:       # BB#0:
@@ -59,12 +61,14 @@ define <4 x float> @test3(<4 x double> %x) nounwind {
 define <8 x float> @test4(<8 x double> %x) nounwind {
 ; CHECK-LABEL: test4:
 ; CHECK:       # BB#0:
+; CHECK-NEXT:    subl $12, %esp
 ; CHECK-NEXT:    cvtpd2ps %xmm1, %xmm1
 ; CHECK-NEXT:    cvtpd2ps %xmm0, %xmm0
 ; CHECK-NEXT:    unpcklpd {{.*#+}} xmm0 = xmm0[0],xmm1[0]
-; CHECK-NEXT:    cvtpd2ps %xmm3, %xmm3
 ; CHECK-NEXT:    cvtpd2ps %xmm2, %xmm1
-; CHECK-NEXT:    unpcklpd {{.*#+}} xmm1 = xmm1[0],xmm3[0]
+; CHECK-NEXT:    cvtpd2ps 16(%esp), %xmm2
+; CHECK-NEXT:    unpcklpd {{.*#+}} xmm1 = xmm1[0],xmm2[0]
+; CHECK-NEXT:    addl $12, %esp
 ; CHECK-NEXT:    retl
 ;
 ; AVX-LABEL: test4:
diff --git a/test/CodeGen/X86/fpcmp-soft-fp.ll b/test/CodeGen/X86/fpcmp-soft-fp.ll
new file mode 100644
index 00000000000..58d57017d18
--- /dev/null
+++ b/test/CodeGen/X86/fpcmp-soft-fp.ll
@@ -0,0 +1,127 @@
+; RUN: llc < %s -march=x86 -mcpu=pentium -mtriple=x86-linux-gnu -float-abi=soft | FileCheck %s 
+
+define i1 @test1(double %d) #0 {
+entry:
+  %cmp = fcmp ule double %d, 0.000000e+00
+  ret i1 %cmp
+}
+; CHECK-LABEL: test1:
+; CHECK: calll __gtdf2
+; CHECK: setle
+; CHECK: retl
+ 
+define i1 @test2(double %d) #0 {
+entry:
+  %cmp = fcmp ult double %d, 0.000000e+00
+  ret i1 %cmp
+}
+; CHECK-LABEL: test2:
+; CHECK: calll __gedf2
+; CHECK: sets
+; CHECK: retl
+
+define i1 @test3(double %d) #0 {
+entry:
+  %cmp = fcmp ugt double %d, 0.000000e+00
+  ret i1 %cmp
+}
+; CHECK-LABEL: test3:
+; CHECK: calll __ledf2
+; CHECK: setg
+; CHECK: retl
+
+define i1 @test4(double %d) #0 {
+entry:
+  %cmp = fcmp uge double %d, 0.000000e+00
+  ret i1 %cmp
+}
+; CHECK-LABEL: test4:
+; CHECK: calll __ltdf2
+; CHECK: setns
+; CHECK: retl
+
+define i1 @test5(double %d) #0 {
+entry:
+  %cmp = fcmp ole double %d, 0.000000e+00
+  ret i1 %cmp
+}
+; CHECK-LABEL: test5:  
+; CHECK: calll __ledf2
+; CHECK: setle
+; CHECK: retl
+
+define i1 @test6(double %d) #0 {
+entry:
+  %cmp = fcmp olt double %d, 0.000000e+00
+  ret i1 %cmp
+}
+; CHECK-LABEL: test6:
+; CHECK: calll __ltdf2
+; CHECK: sets
+; CHECK: retl
+
+define i1 @test7(double %d) #0 {
+entry:
+  %cmp = fcmp ogt double %d, 0.000000e+00
+  ret i1 %cmp
+}
+; CHECK-LABEL: test7:
+; CHECK: calll __gtdf2
+; CHECK: setg
+; CHECK: retl
+
+define i1 @test8(double %d) #0 {
+entry:
+  %cmp = fcmp oge double %d, 0.000000e+00
+  ret i1 %cmp
+}
+; CHECK-LABEL: test8:
+; CHECK: calll __gedf2
+; CHECK: setns
+; CHECK: retl
+
+define i1 @test9(double %d) #0 {
+entry:
+  %cmp = fcmp oeq double %d, 0.000000e+00
+  ret i1 %cmp
+}
+; CHECK-LABEL: test9:
+; CHECK: calll __eqdf2
+; CHECK: sete
+; CHECK: retl
+
+define i1 @test10(double %d) #0 {
+entry:
+  %cmp = fcmp ueq double %d, 0.000000e+00
+  ret i1 %cmp
+}
+; CHECK-LABEL: test10:
+; CHECK: calll __eqdf2
+; CHECK: sete
+; CHECK: calll __unorddf2
+; CHECK: setne
+; CHECK: retl
+
+define i1 @test11(double %d) #0 {
+entry:
+  %cmp = fcmp one double %d, 0.000000e+00
+  ret i1 %cmp
+}
+; CHECK-LABEL: test11:
+; CHECK: calll __gtdf2
+; CHECK: setg
+; CHECK: calll __ltdf2
+; CHECK: sets
+; CHECK: retl
+
+define i1 @test12(double %d) #0 {
+entry:
+  %cmp = fcmp une double %d, 0.000000e+00
+  ret i1 %cmp
+}
+; CHECK-LABEL: test12:
+; CHECK: calll __nedf2
+; CHECK: setne
+; CHECK: retl
+
+attributes #0 = { "use-soft-float"="true" }
diff --git a/test/CodeGen/X86/fpstack-debuginstr-kill.ll b/test/CodeGen/X86/fpstack-debuginstr-kill.ll
index 8eb452a6b10..34398414a76 100644
--- a/test/CodeGen/X86/fpstack-debuginstr-kill.ll
+++ b/test/CodeGen/X86/fpstack-debuginstr-kill.ll
@@ -32,7 +32,7 @@ sw.bb735:                                         ; preds = %if.end511
   unreachable
 
 if.end41.i2210:                                   ; preds = %if.end511
-  call void @llvm.dbg.value(metadata x86_fp80 %src.sroa.0.0.src.sroa.0.0.2280, i64 0, metadata !20, metadata !MDExpression()), !dbg !MDLocation(scope: !4)
+  call void @llvm.dbg.value(metadata x86_fp80 %src.sroa.0.0.src.sroa.0.0.2280, i64 0, metadata !20, metadata !DIExpression()), !dbg !DILocation(scope: !4)
   unreachable
 
 sw.bb992:                                         ; preds = %if.end511
@@ -43,29 +43,29 @@ declare void @llvm.dbg.value(metadata, i64, metadata, metadata)
 
 !llvm.dbg.cu = !{!0}
 !llvm.module.flags = !{!24, !25}
-!0 = !MDCompileUnit(language: DW_LANG_C_plus_plus, producer: "clang version 3.6.0 (http://llvm.org/git/clang 8444ae7cfeaefae031f8fedf0d1435ca3b14d90b) (http://llvm.org/git/llvm 886f0101a7d176543b831f5efb74c03427244a55)", isOptimized: true, emissionKind: 1, file: !1, enums: !2, retainedTypes: !2, subprograms: !3, globals: !21, imports: !2)
-!1 = !MDFile(filename: "fpu_ieee.cpp", directory: "x87stackifier")
+!0 = !DICompileUnit(language: DW_LANG_C_plus_plus, producer: "clang version 3.6.0 (http://llvm.org/git/clang 8444ae7cfeaefae031f8fedf0d1435ca3b14d90b) (http://llvm.org/git/llvm 886f0101a7d176543b831f5efb74c03427244a55)", isOptimized: true, emissionKind: 1, file: !1, enums: !2, retainedTypes: !2, subprograms: !3, globals: !21, imports: !2)
+!1 = !DIFile(filename: "fpu_ieee.cpp", directory: "x87stackifier")
 !2 = !{}
 !3 = !{!4}
-!4 = !MDSubprogram(name: "fpuop_arithmetic", linkageName: "_Z16fpuop_arithmeticjj", line: 11, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: true, scopeLine: 13, file: !5, scope: !6, type: !7, function: void (i32, i32)* @_Z16fpuop_arithmeticjj, variables: !10)
-!5 = !MDFile(filename: "f1.cpp", directory: "x87stackifier")
-!6 = !MDFile(filename: "f1.cpp", directory: "x87stackifier")
-!7 = !MDSubroutineType(types: !8)
+!4 = !DISubprogram(name: "fpuop_arithmetic", linkageName: "_Z16fpuop_arithmeticjj", line: 11, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: true, scopeLine: 13, file: !5, scope: !6, type: !7, function: void (i32, i32)* @_Z16fpuop_arithmeticjj, variables: !10)
+!5 = !DIFile(filename: "f1.cpp", directory: "x87stackifier")
+!6 = !DIFile(filename: "f1.cpp", directory: "x87stackifier")
+!7 = !DISubroutineType(types: !8)
 !8 = !{null, !9, !9}
-!9 = !MDBasicType(tag: DW_TAG_base_type, name: "unsigned int", size: 32, align: 32, encoding: DW_ATE_unsigned)
+!9 = !DIBasicType(tag: DW_TAG_base_type, name: "unsigned int", size: 32, align: 32, encoding: DW_ATE_unsigned)
 !10 = !{!11, !12, !13, !18, !20}
-!11 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "", line: 11, arg: 1, scope: !4, file: !6, type: !9)
-!12 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "", line: 11, arg: 2, scope: !4, file: !6, type: !9)
-!13 = !MDLocalVariable(tag: DW_TAG_auto_variable, name: "x", line: 14, scope: !4, file: !6, type: !14)
-!14 = !MDDerivedType(tag: DW_TAG_typedef, name: "fpu_extended", line: 3, file: !5, baseType: !15)
-!15 = !MDDerivedType(tag: DW_TAG_typedef, name: "fpu_register", line: 2, file: !5, baseType: !16)
-!16 = !MDDerivedType(tag: DW_TAG_typedef, name: "uae_f64", line: 1, file: !5, baseType: !17)
-!17 = !MDBasicType(tag: DW_TAG_base_type, name: "double", size: 64, align: 64, encoding: DW_ATE_float)
-!18 = !MDLocalVariable(tag: DW_TAG_auto_variable, name: "a", line: 15, scope: !4, file: !6, type: !19)
-!19 = !MDBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
-!20 = !MDLocalVariable(tag: DW_TAG_auto_variable, name: "value", line: 16, scope: !4, file: !6, type: !14)
+!11 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "", line: 11, arg: 1, scope: !4, file: !6, type: !9)
+!12 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "", line: 11, arg: 2, scope: !4, file: !6, type: !9)
+!13 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "x", line: 14, scope: !4, file: !6, type: !14)
+!14 = !DIDerivedType(tag: DW_TAG_typedef, name: "fpu_extended", line: 3, file: !5, baseType: !15)
+!15 = !DIDerivedType(tag: DW_TAG_typedef, name: "fpu_register", line: 2, file: !5, baseType: !16)
+!16 = !DIDerivedType(tag: DW_TAG_typedef, name: "uae_f64", line: 1, file: !5, baseType: !17)
+!17 = !DIBasicType(tag: DW_TAG_base_type, name: "double", size: 64, align: 64, encoding: DW_ATE_float)
+!18 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "a", line: 15, scope: !4, file: !6, type: !19)
+!19 = !DIBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
+!20 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "value", line: 16, scope: !4, file: !6, type: !14)
 !21 = !{!22, !23}
-!22 = !MDGlobalVariable(name: "g1", line: 5, isLocal: false, isDefinition: true, scope: null, file: !6, type: !14, variable: double* @g1)
-!23 = !MDGlobalVariable(name: "g2", line: 6, isLocal: false, isDefinition: true, scope: null, file: !6, type: !19, variable: i32* @g2)
+!22 = !DIGlobalVariable(name: "g1", line: 5, isLocal: false, isDefinition: true, scope: null, file: !6, type: !14, variable: double* @g1)
+!23 = !DIGlobalVariable(name: "g2", line: 6, isLocal: false, isDefinition: true, scope: null, file: !6, type: !19, variable: i32* @g2)
 !24 = !{i32 2, !"Dwarf Version", i32 2}
 !25 = !{i32 2, !"Debug Info Version", i32 3}
diff --git a/test/CodeGen/X86/frameaddr.ll b/test/CodeGen/X86/frameaddr.ll
index 46e525eddcf..5ed2ba0f431 100644
--- a/test/CodeGen/X86/frameaddr.ll
+++ b/test/CodeGen/X86/frameaddr.ll
@@ -35,9 +35,7 @@ entry:
 ; CHECK-X32ABI-NEXT:  popq %rbp
 ; CHECK-X32ABI-NEXT:  ret
 ; CHECK-NACL64-LABEL: test1
-; @LOCALMOD: base address hiding avoids pushing the full rbp onto the stack.
-; CHECK-NACL64:       movl %ebp, %eax
-; CHECK-NACL64-NEXT:  pushq %rax
+; CHECK-NACL64:       pushq %rbp
 ; CHECK-NACL64-NEXT:  movq %rsp, %rbp
 ; CHECK-NACL64-NEXT:  movl %ebp, %eax
   %0 = tail call i8* @llvm.frameaddress(i32 0)
@@ -74,14 +72,10 @@ entry:
 ; CHECK-X32ABI-NEXT:  popq %rbp
 ; CHECK-X32ABI-NEXT:  ret
 ; CHECK-NACL64-LABEL: test2
-; @LOCALMOD base-address hiding.
-; CHECK-NACL64:       movl %ebp, %eax
-; CHECK-NACL64-NEXT:  pushq %rax
+; CHECK-NACL64:       pushq %rbp
 ; CHECK-NACL64-NEXT:  movq %rsp, %rbp
-; -fast-isel and -O2 are a bit different in how they load from rbp
-; (we can use NACL64-NEXT if we ever make the sequences the same).
-; CHECK-NACL64:       movl {{.*}}(%{{.*}}), %eax
-; CHECK-NACL64-NEXT:  movl %nacl:(%r15,%rax), %eax
+; CHECK-NACL64-NEXT:  movl (%ebp), %eax
+; CHECK-NACL64-NEXT:  movl (%eax), %eax
   %0 = tail call i8* @llvm.frameaddress(i32 2)
   ret i8* %0
 }
diff --git a/test/CodeGen/X86/frameescape.ll b/test/CodeGen/X86/frameescape.ll
index 3a624ae863d..179a936304b 100644
--- a/test/CodeGen/X86/frameescape.ll
+++ b/test/CodeGen/X86/frameescape.ll
@@ -1,23 +1,26 @@
 ; RUN: llc -mtriple=i686-windows-msvc < %s | FileCheck %s --check-prefix=X86
 ; RUN: llc -mtriple=x86_64-windows-msvc < %s | FileCheck %s --check-prefix=X64
 
-declare void @llvm.frameescape(...)
+declare void @llvm.localescape(...)
 declare i8* @llvm.frameaddress(i32)
-declare i8* @llvm.framerecover(i8*, i8*, i32)
+declare i8* @llvm.localrecover(i8*, i8*, i32)
 declare i32 @printf(i8*, ...)
 
 @str = internal constant [10 x i8] c"asdf: %d\0A\00"
 
 define void @print_framealloc_from_fp(i8* %fp) {
-  %a.i8 = call i8* @llvm.framerecover(i8* bitcast (void()* @alloc_func to i8*), i8* %fp, i32 0)
+  %a.i8 = call i8* @llvm.localrecover(i8* bitcast (void()* @alloc_func to i8*), i8* %fp, i32 0)
   %a = bitcast i8* %a.i8 to i32*
   %a.val = load i32, i32* %a
   call i32 (i8*, ...) @printf(i8* getelementptr ([10 x i8], [10 x i8]* @str, i32 0, i32 0), i32 %a.val)
-  %b.i8 = call i8* @llvm.framerecover(i8* bitcast (void()* @alloc_func to i8*), i8* %fp, i32 1)
+  %b.i8 = call i8* @llvm.localrecover(i8* bitcast (void()* @alloc_func to i8*), i8* %fp, i32 1)
   %b = bitcast i8* %b.i8 to i32*
   %b.val = load i32, i32* %b
   call i32 (i8*, ...) @printf(i8* getelementptr ([10 x i8], [10 x i8]* @str, i32 0, i32 0), i32 %b.val)
   store i32 42, i32* %b
+  %b2 = getelementptr i32, i32* %b, i32 1
+  %b2.val = load i32, i32* %b2
+  call i32 (i8*, ...) @printf(i8* getelementptr ([10 x i8], [10 x i8]* @str, i32 0, i32 0), i32 %b2.val)
   ret void
 }
 
@@ -37,23 +40,28 @@ define void @print_framealloc_from_fp(i8* %fp) {
 ; X86: pushl   %esi
 ; X86: subl    $8, %esp
 ; X86: movl    16(%esp), %esi
-; X86: movl    _Lalloc_func$frame_escape_0(%esi), %eax
+; X86: movl    Lalloc_func$frame_escape_0(%esi), %eax
 ; X86: movl    %eax, 4(%esp)
 ; X86: movl    $_str, (%esp)
 ; X86: calll   _printf
-; X86: movl    _Lalloc_func$frame_escape_1(%esi), %eax
+; X86: movl    Lalloc_func$frame_escape_1(%esi), %eax
+; X86: movl    %eax, 4(%esp)
+; X86: movl    $_str, (%esp)
+; X86: calll   _printf
+; X86: movl    $42, Lalloc_func$frame_escape_1(%esi)
+; X86: movl    $4, %eax
+; X86: movl    Lalloc_func$frame_escape_1(%esi,%eax), %eax
 ; X86: movl    %eax, 4(%esp)
 ; X86: movl    $_str, (%esp)
 ; X86: calll   _printf
-; X86: movl    $42, _Lalloc_func$frame_escape_1(%esi)
 ; X86: addl    $8, %esp
 ; X86: popl    %esi
 ; X86: retl
 
 define void @alloc_func() {
   %a = alloca i32
-  %b = alloca i32
-  call void (...) @llvm.frameescape(i32* %a, i32* %b)
+  %b = alloca i32, i32 2
+  call void (...) @llvm.localescape(i32* %a, i32* %b)
   store i32 42, i32* %a
   store i32 13, i32* %b
   %fp = call i8* @llvm.frameaddress(i32 0)
@@ -67,9 +75,9 @@ define void @alloc_func() {
 ; X64: leaq    48(%rsp), %rbp
 ; X64: .seh_setframe 5, 48
 ; X64: .Lalloc_func$frame_escape_0 = 44
-; X64: .Lalloc_func$frame_escape_1 = 40
+; X64: .Lalloc_func$frame_escape_1 = 36
 ; X64: movl $42, -4(%rbp)
-; X64: movl $13, -8(%rbp)
+; X64: movl $13, -12(%rbp)
 ; X64: leaq    -48(%rbp), %rcx
 ; X64: callq print_framealloc_from_fp
 ; X64: retq
@@ -77,14 +85,14 @@ define void @alloc_func() {
 ; X86-LABEL: alloc_func:
 ; X86: pushl   %ebp
 ; X86: movl    %esp, %ebp
-; X86: subl    $12, %esp
+; X86: subl    $16, %esp
 ; X86: Lalloc_func$frame_escape_0 = -4
-; X86: Lalloc_func$frame_escape_1 = -8
+; X86: Lalloc_func$frame_escape_1 = -12
 ; X86: movl    $42, -4(%ebp)
-; X86: movl    $13, -8(%ebp)
+; X86: movl    $13, -12(%ebp)
 ; X86: movl    %ebp, (%esp)
 ; X86: calll   _print_framealloc_from_fp
-; X86: addl    $12, %esp
+; X86: addl    $16, %esp
 ; X86: popl    %ebp
 ; X86: retl
 
@@ -97,7 +105,7 @@ define i32 @main() {
 define void @alloc_func_no_frameaddr() {
   %a = alloca i32
   %b = alloca i32
-  call void (...) @llvm.frameescape(i32* %a, i32* %b)
+  call void (...) @llvm.localescape(i32* %a, i32* %b)
   store i32 42, i32* %a
   store i32 13, i32* %b
   call void @print_framealloc_from_fp(i8* null)
@@ -116,3 +124,5 @@ define void @alloc_func_no_frameaddr() {
 ; X64: callq print_framealloc_from_fp
 ; X64: addq $40, %rsp
 ; X64: retq
+
+; X86-LABEL: alloc_func_no_frameaddr:
diff --git a/test/CodeGen/X86/frameregister.ll b/test/CodeGen/X86/frameregister.ll
new file mode 100644
index 00000000000..826bb9d78c9
--- /dev/null
+++ b/test/CodeGen/X86/frameregister.ll
@@ -0,0 +1,30 @@
+; RUN: llc < %s -mtriple=x86_64-apple-darwin  | FileCheck %s
+; RUN: llc < %s -mtriple=x86_64-linux-gnueabi | FileCheck %s
+; RUN: opt < %s -O3 -S -mtriple=x86_64-linux-gnueabi | FileCheck %s --check-prefix=OPT
+
+define i64 @get_frame() #0 {
+entry:
+; CHECK-LABEL: get_frame:
+; CHECK: movq	%rbp, %rax
+  %sp = call i64 @llvm.read_register.i64(metadata !0)
+; OPT: @llvm.read_register.i64
+  ret i64 %sp
+}
+
+define void @set_frame(i64 %val) #0 {
+entry:
+; CHECK-LABEL: set_frame:
+; CHECK: movq	%rdi, %rbp
+  call void @llvm.write_register.i64(metadata !0, i64 %val)
+; OPT: @llvm.write_register.i64
+  ret void
+}
+
+declare i64 @llvm.read_register.i64(metadata) nounwind
+declare void @llvm.write_register.i64(metadata, i64) nounwind
+
+; register unsigned long current_stack_pointer asm("rbp");
+; CHECK-NOT: .asciz  "rbp"
+!0 = !{!"rbp\00"}
+
+attributes #0 = { nounwind "no-frame-pointer-elim"="true" }
diff --git a/test/CodeGen/X86/gcc_except_table.ll b/test/CodeGen/X86/gcc_except_table.ll
index b656dc9d68e..82064c2a390 100644
--- a/test/CodeGen/X86/gcc_except_table.ll
+++ b/test/CodeGen/X86/gcc_except_table.ll
@@ -3,7 +3,7 @@
 ; RUN: llc -mtriple i686-pc-windows-gnu %s -o -   | FileCheck %s   --check-prefix=MINGW32
 @_ZTIi = external constant i8*
 
-define i32 @main() uwtable optsize ssp {
+define i32 @main() uwtable optsize ssp personality i8* bitcast (i32 (...)* @__gxx_personality_v0 to i8*) {
 ; APPLE: .cfi_startproc
 ; APPLE: .cfi_personality 155, ___gxx_personality_v0
 ; APPLE: .cfi_lsda 16, Lexception0
@@ -36,7 +36,7 @@ entry:
           to label %try.cont unwind label %lpad
 
 lpad:
-  %0 = landingpad { i8*, i32 } personality i8* bitcast (i32 (...)* @__gxx_personality_v0 to i8*)
+  %0 = landingpad { i8*, i32 }
           cleanup
           catch i8* bitcast (i8** @_ZTIi to i8*)
   br label %eh.resume
diff --git a/test/CodeGen/X86/gcc_except_table_functions.ll b/test/CodeGen/X86/gcc_except_table_functions.ll
index 7a64a01fa38..8e002ad142b 100644
--- a/test/CodeGen/X86/gcc_except_table_functions.ll
+++ b/test/CodeGen/X86/gcc_except_table_functions.ll
@@ -10,7 +10,7 @@ declare void @filt1()
 declare void @_Z1fv()
 declare i32 @llvm.eh.typeid.for(i8*)
 
-define i32 @main() uwtable {
+define i32 @main() uwtable personality i8* bitcast (i32 (...)* @__gxx_personality_v0 to i8*) {
 entry:
   invoke void @_Z1fv()
           to label %try.cont unwind label %lpad
@@ -19,7 +19,7 @@ try.cont:
   ret i32 0
 
 lpad:
-  %0 = landingpad { i8*, i32 } personality i8* bitcast (i32 (...)* @__gxx_personality_v0 to i8*)
+  %0 = landingpad { i8*, i32 }
           cleanup
           catch i8* bitcast (void ()* @filt0 to i8*)
           catch i8* bitcast (void ()* @filt1 to i8*)
diff --git a/test/CodeGen/X86/global-fill.ll b/test/CodeGen/X86/global-fill.ll
new file mode 100644
index 00000000000..656c8ca2c32
--- /dev/null
+++ b/test/CodeGen/X86/global-fill.ll
@@ -0,0 +1,27 @@
+; RUN: llc -mtriple=x86_64-apple-darwin < %s | FileCheck %s
+
+@test1 = global [2 x i24] [i24 -1, i24 -1]
+; CHECK-LABEL: test1:
+; CHECK-NEXT: .long	16777215
+; CHECK-NEXT: .long	16777215
+
+@test2 = global [2 x i7] [i7 1, i7 1]
+; CHECK-LABEL: test2:
+; CHECK-NEXT: .space 2,1
+
+@test3 = global [4 x i128] [i128 -1, i128 -1, i128 -1, i128 -1]
+; CHECK-LABEL: test3:
+; CHECK-NEXT: .space 64,255
+
+@test4 = global [3 x i16] [i16 257, i16 257, i16 257]
+; CHECK-LABEL: test4:
+; CHECK-NEXT: .space 6,1
+
+@test5 = global [2 x [2 x i16]] [[2 x i16] [i16 257, i16 257], [2 x i16] [i16 -1, i16 -1]]
+; CHECK-LABEL: test5:
+; CHECK-NEXT: .space 4,1
+; CHECK-NEXT: .space 4,255
+
+@test6 = global [2 x [2 x i16]] [[2 x i16] [i16 257, i16 257], [2 x i16] [i16 257, i16 257]]
+; CHECK-LABEL: test6:
+; CHECK-NEXT: .space 8,1
diff --git a/test/CodeGen/X86/global-sections.ll b/test/CodeGen/X86/global-sections.ll
index 8c61411e53e..82547a60674 100644
--- a/test/CodeGen/X86/global-sections.ll
+++ b/test/CodeGen/X86/global-sections.ll
@@ -61,12 +61,12 @@ bb5:
 
 declare void @G()
 
-define void @F3(i32 %y) {
+define void @F3(i32 %y) personality i8* bitcast (void ()* @G to i8*) {
 bb0:
   invoke void @G()
           to label %bb2 unwind label %bb1
 bb1:
-  landingpad { i8*, i32 } personality i8* bitcast (void ()* @G to i8*)
+  landingpad { i8*, i32 }
           catch i8* null
   br label %bb2
 bb2:
diff --git a/test/CodeGen/X86/haddsub.ll b/test/CodeGen/X86/haddsub.ll
index 9feb5f6ea6e..6e65c6c739c 100644
--- a/test/CodeGen/X86/haddsub.ll
+++ b/test/CodeGen/X86/haddsub.ll
@@ -283,3 +283,18 @@ define <4 x double> @vhsubpd1(<4 x double> %x, <4 x double> %y) {
   %r = fsub <4 x double> %a, %b
   ret <4 x double> %r
 }
+
+; CHECK-LABEL: haddps_v2f32
+; CHECK: haddps %xmm{{[0-9]+}}, %xmm0
+; CHECK-NEXT: retq
+define <2 x float> @haddps_v2f32(<4 x float> %v0) {
+  %v0.0 = extractelement <4 x float> %v0, i32 0
+  %v0.1 = extractelement <4 x float> %v0, i32 1
+  %v0.2 = extractelement <4 x float> %v0, i32 2
+  %v0.3 = extractelement <4 x float> %v0, i32 3
+  %op0 = fadd float %v0.0, %v0.1
+  %op1 = fadd float %v0.2, %v0.3
+  %res0 = insertelement <2 x float> undef, float %op0, i32 0
+  %res1 = insertelement <2 x float> %res0, float %op1, i32 1
+  ret <2 x float> %res1
+}
diff --git a/test/CodeGen/X86/half.ll b/test/CodeGen/X86/half.ll
index f4331bab143..8a726370f19 100644
--- a/test/CodeGen/X86/half.ll
+++ b/test/CodeGen/X86/half.ll
@@ -1,5 +1,7 @@
-; RUN: llc < %s -march=x86-64 -mtriple=x86_64-unknown-linux-gnu -mcpu=corei7 -mattr=-f16c | FileCheck %s -check-prefix=CHECK -check-prefix=CHECK-LIBCALL
-; RUN: llc < %s -march=x86-64 -mtriple=x86_64-unknown-linux-gnu -mcpu=corei7 -mattr=+f16c | FileCheck %s -check-prefix=CHECK -check-prefix=CHECK-F16C
+; RUN: llc < %s -march=x86-64 -mtriple=x86_64-unknown-linux-gnu -mcpu=corei7 -mattr=-f16c -asm-verbose=false \
+; RUN:   | FileCheck %s -check-prefix=CHECK -check-prefix=CHECK-LIBCALL
+; RUN: llc < %s -march=x86-64 -mtriple=x86_64-unknown-linux-gnu -mcpu=corei7 -mattr=+f16c -asm-verbose=false \
+; RUN:    | FileCheck %s -check-prefix=CHECK -check-prefix=CHECK-F16C
 
 define void @test_load_store(half* %in, half* %out) {
 ; CHECK-LABEL: test_load_store:
@@ -30,7 +32,7 @@ define float @test_extend32(half* %addr) {
 ; CHECK-LABEL: test_extend32:
 
 ; CHECK-LIBCALL: jmp __gnu_h2f_ieee
-; CHECK-FP16: vcvtph2ps
+; CHECK-F16C: vcvtph2ps
   %val16 = load half, half* %addr
   %val32 = fpext half %val16 to float
   ret float %val32
@@ -41,8 +43,8 @@ define double @test_extend64(half* %addr) {
 
 ; CHECK-LIBCALL: callq __gnu_h2f_ieee
 ; CHECK-LIBCALL: cvtss2sd
-; CHECK-FP16: vcvtph2ps
-; CHECK-FP16: vcvtss2sd
+; CHECK-F16C: vcvtph2ps
+; CHECK-F16C: vcvtss2sd
   %val16 = load half, half* %addr
   %val32 = fpext half %val16 to double
   ret double %val32
@@ -52,7 +54,7 @@ define void @test_trunc32(float %in, half* %addr) {
 ; CHECK-LABEL: test_trunc32:
 
 ; CHECK-LIBCALL: callq __gnu_f2h_ieee
-; CHECK-FP16: vcvtps2ph
+; CHECK-F16C: vcvtps2ph
   %val16 = fptrunc float %in to half
   store half %val16, half* %addr
   ret void
@@ -62,8 +64,200 @@ define void @test_trunc64(double %in, half* %addr) {
 ; CHECK-LABEL: test_trunc64:
 
 ; CHECK-LIBCALL: callq __truncdfhf2
-; CHECK-FP16: callq __truncdfhf2
+; CHECK-F16C: callq __truncdfhf2
   %val16 = fptrunc double %in to half
   store half %val16, half* %addr
   ret void
 }
+
+define i64 @test_fptosi_i64(half* %p) #0 {
+; CHECK-LABEL: test_fptosi_i64:
+
+; CHECK-LIBCALL-NEXT: pushq %rax
+; CHECK-LIBCALL-NEXT: movzwl (%rdi), %edi
+; CHECK-LIBCALL-NEXT: callq __gnu_h2f_ieee
+; CHECK-LIBCALL-NEXT: cvttss2si %xmm0, %rax
+; CHECK-LIBCALL-NEXT: popq %rdx
+; CHECK-LIBCALL-NEXT: retq
+
+; CHECK-F16C-NEXT: movswl (%rdi), [[REG0:%[a-z0-9]+]]
+; CHECK-F16C-NEXT: vmovd [[REG0]], [[REG1:%[a-z0-9]+]]
+; CHECK-F16C-NEXT: vcvtph2ps [[REG1]], [[REG2:%[a-z0-9]+]]
+; CHECK-F16C-NEXT: vcvttss2si [[REG2]], %rax
+; CHECK-F16C-NEXT: retq
+  %a = load half, half* %p, align 2
+  %r = fptosi half %a to i64
+  ret i64 %r
+}
+
+define void @test_sitofp_i64(i64 %a, half* %p) #0 {
+; CHECK-LABEL: test_sitofp_i64:
+
+; CHECK-LIBCALL-NEXT: pushq [[ADDR:%[a-z]+]]
+; CHECK-LIBCALL-NEXT: movq %rsi, [[ADDR]]
+; CHECK-LIBCALL-NEXT: cvtsi2ssq %rdi, %xmm0
+; CHECK-LIBCALL-NEXT: callq __gnu_f2h_ieee
+; CHECK-LIBCALL-NEXT: movw %ax, ([[ADDR]])
+; CHECK_LIBCALL-NEXT: popq [[ADDR]]
+; CHECK_LIBCALL-NEXT: retq
+
+; CHECK-F16C-NEXT: vcvtsi2ssq %rdi, [[REG0:%[a-z0-9]+]], [[REG0]]
+; CHECK-F16C-NEXT: vcvtps2ph $0, [[REG0]], [[REG0]]
+; CHECK-F16C-NEXT: vmovd [[REG0]], %eax
+; CHECK-F16C-NEXT: movw %ax, (%rsi)
+; CHECK-F16C-NEXT: retq
+  %r = sitofp i64 %a to half
+  store half %r, half* %p
+  ret void
+}
+
+define i64 @test_fptoui_i64(half* %p) #0 {
+; CHECK-LABEL: test_fptoui_i64:
+
+; FP_TO_UINT is expanded using FP_TO_SINT
+; CHECK-LIBCALL-NEXT: pushq %rax
+; CHECK-LIBCALL-NEXT: movzwl (%rdi), %edi
+; CHECK-LIBCALL-NEXT: callq __gnu_h2f_ieee
+; CHECK-LIBCALL-NEXT: movss {{.[A-Z_0-9]+}}(%rip), [[REG1:%[a-z0-9]+]]
+; CHECK-LIBCALL-NEXT: movaps %xmm0, [[REG2:%[a-z0-9]+]]
+; CHECK-LIBCALL-NEXT: subss [[REG1]], [[REG2]]
+; CHECK-LIBCALL-NEXT: cvttss2si [[REG2]], [[REG3:%[a-z0-9]+]]
+; CHECK-LIBCALL-NEXT: movabsq  $-9223372036854775808, [[REG4:%[a-z0-9]+]]
+; CHECK-LIBCALL-NEXT: xorq [[REG3]], [[REG4]]
+; CHECK-LIBCALL-NEXT: cvttss2si %xmm0, [[REG5:%[a-z0-9]+]]
+; CHECK-LIBCALL-NEXT: ucomiss [[REG1]], %xmm0
+; CHECK-LIBCALL-NEXT: cmovaeq [[REG4]], [[REG5]]
+; CHECK-LIBCALL-NEXT: popq %rdx
+; CHECK-LIBCALL-NEXT: retq
+
+; CHECK-F16C-NEXT: movswl (%rdi), [[REG0:%[a-z0-9]+]]
+; CHECK-F16C-NEXT: vmovd [[REG0]], [[REG1:%[a-z0-9]+]]
+; CHECK-F16C-NEXT: vcvtph2ps [[REG1]], [[REG2:%[a-z0-9]+]]
+; CHECK-F16C-NEXT: vmovss {{.[A-Z_0-9]+}}(%rip), [[REG3:%[a-z0-9]+]]
+; CHECK-F16C-NEXT: vsubss [[REG3]], [[REG2]], [[REG4:%[a-z0-9]+]]
+; CHECK-F16C-NEXT: vcvttss2si [[REG4]], [[REG5:%[a-z0-9]+]]
+; CHECK-F16C-NEXT: movabsq $-9223372036854775808, [[REG6:%[a-z0-9]+]]
+; CHECK-F16C-NEXT: xorq [[REG5]], [[REG6:%[a-z0-9]+]]
+; CHECK-F16C-NEXT: vcvttss2si [[REG2]], [[REG7:%[a-z0-9]+]]
+; CHECK-F16C-NEXT: vucomiss [[REG3]], [[REG2]]
+; CHECK-F16C-NEXT: cmovaeq [[REG6]], %rax
+; CHECK-F16C-NEXT: retq
+  %a = load half, half* %p, align 2
+  %r = fptoui half %a to i64
+  ret i64 %r
+}
+
+define void @test_uitofp_i64(i64 %a, half* %p) #0 {
+; CHECK-LABEL: test_uitofp_i64:
+; CHECK-LIBCALL-NEXT: pushq [[ADDR:%[a-z0-9]+]]
+; CHECK-LIBCALL-NEXT: movq %rsi, [[ADDR]]
+; CHECK-NEXT: movl %edi, [[REG0:%[a-z0-9]+]]
+; CHECK-NEXT: andl $1, [[REG0]]
+; CHECK-NEXT: testq %rdi, %rdi
+; CHECK-NEXT: js [[LABEL1:.LBB[0-9_]+]]
+
+; simple conversion to float if non-negative
+; CHECK-LIBCALL-NEXT: cvtsi2ssq %rdi, [[REG1:%[a-z0-9]+]]
+; CHECK-F16C-NEXT: vcvtsi2ssq %rdi, [[REG1:%[a-z0-9]+]], [[REG1]]
+; CHECK-NEXT: jmp [[LABEL2:.LBB[0-9_]+]]
+
+; convert using shift+or if negative
+; CHECK-NEXT: [[LABEL1]]:
+; CHECK-NEXT: shrq %rdi
+; CHECK-NEXT: orq %rdi, [[REG2:%[a-z0-9]+]]
+; CHECK-LIBCALL-NEXT: cvtsi2ssq [[REG2]], [[REG3:%[a-z0-9]+]]
+; CHECK-LIBCALL-NEXT: addss [[REG3]], [[REG1]]
+; CHECK-F16C-NEXT: vcvtsi2ssq [[REG2]], [[REG3:%[a-z0-9]+]], [[REG3]]
+; CHECK-F16C-NEXT: vaddss [[REG3]], [[REG3]], [[REG1:[%a-z0-9]+]]
+
+; convert float to half
+; CHECK-NEXT: [[LABEL2]]:
+; CHECK-LIBCALL-NEXT: callq __gnu_f2h_ieee
+; CHECK-LIBCALL-NEXT: movw %ax, ([[ADDR]])
+; CHECK-LIBCALL-NEXT: popq [[ADDR]]
+; CHECK-F16C-NEXT: vcvtps2ph $0, [[REG1]], [[REG4:%[a-z0-9]+]]
+; CHECK-F16C-NEXT: vmovd [[REG4]], %eax
+; CHECK-F16C-NEXT: movw %ax, (%rsi)
+; CHECK-NEXT: retq
+
+  %r = uitofp i64 %a to half
+  store half %r, half* %p
+  ret void
+}
+
+define <4 x float> @test_extend32_vec4(<4 x half>* %p) #0 {
+; CHECK-LABEL: test_extend32_vec4:
+
+; CHECK-LIBCALL: callq __gnu_h2f_ieee
+; CHECK-LIBCALL: callq __gnu_h2f_ieee
+; CHECK-LIBCALL: callq __gnu_h2f_ieee
+; CHECK-LIBCALL: callq __gnu_h2f_ieee
+; CHECK-F16C: vcvtph2ps
+; CHECK-F16C: vcvtph2ps
+; CHECK-F16C: vcvtph2ps
+; CHECK-F16C: vcvtph2ps
+  %a = load <4 x half>, <4 x half>* %p, align 8
+  %b = fpext <4 x half> %a to <4 x float>
+  ret <4 x float> %b
+}
+
+define <4 x double> @test_extend64_vec4(<4 x half>* %p) #0 {
+; CHECK-LABEL: test_extend64_vec4
+
+; CHECK-LIBCALL: callq __gnu_h2f_ieee
+; CHECK-LIBCALL-DAG: callq __gnu_h2f_ieee
+; CHECK-LIBCALL-DAG: callq __gnu_h2f_ieee
+; CHECK-LIBCALL-DAG: callq __gnu_h2f_ieee
+; CHECK-LIBCALL-DAG: cvtss2sd
+; CHECK-LIBCALL-DAG: cvtss2sd
+; CHECK-LIBCALL-DAG: cvtss2sd
+; CHECK-LIBCALL: cvtss2sd
+; CHECK-F16C: vcvtph2ps
+; CHECK-F16C-DAG: vcvtph2ps
+; CHECK-F16C-DAG: vcvtph2ps
+; CHECK-F16C-DAG: vcvtph2ps
+; CHECK-F16C-DAG: vcvtss2sd
+; CHECK-F16C-DAG: vcvtss2sd
+; CHECK-F16C-DAG: vcvtss2sd
+; CHECK-F16C: vcvtss2sd
+  %a = load <4 x half>, <4 x half>* %p, align 8
+  %b = fpext <4 x half> %a to <4 x double>
+  ret <4 x double> %b
+}
+
+define void @test_trunc32_vec4(<4 x float> %a, <4 x half>* %p) {
+; CHECK-LABEL: test_trunc32_vec4:
+
+; CHECK-LIBCALL: callq __gnu_f2h_ieee
+; CHECK-LIBCALL: callq __gnu_f2h_ieee
+; CHECK-LIBCALL: callq __gnu_f2h_ieee
+; CHECK-LIBCALL: callq __gnu_f2h_ieee
+; CHECK-F16C: vcvtps2ph
+; CHECK-F16C: vcvtps2ph
+; CHECK-F16C: vcvtps2ph
+; CHECK-F16C: vcvtps2ph
+; CHECK: movw
+; CHECK: movw
+; CHECK: movw
+; CHECK: movw
+  %v = fptrunc <4 x float> %a to <4 x half>
+  store <4 x half> %v, <4 x half>* %p
+  ret void
+}
+
+define void @test_trunc64_vec4(<4 x double> %a, <4 x half>* %p) {
+; CHECK-LABEL: test_trunc64_vec4:
+; CHECK: callq  __truncdfhf2
+; CHECK: callq  __truncdfhf2
+; CHECK: callq  __truncdfhf2
+; CHECK: callq  __truncdfhf2
+; CHECK: movw
+; CHECK: movw
+; CHECK: movw
+; CHECK: movw
+  %v = fptrunc <4 x double> %a to <4 x half>
+  store <4 x half> %v, <4 x half>* %p
+  ret void
+}
+
+attributes #0 = { nounwind }
diff --git a/test/CodeGen/X86/i486-fence-loop.ll b/test/CodeGen/X86/i486-fence-loop.ll
index 96ed0569cf0..936e54eddaf 100644
--- a/test/CodeGen/X86/i486-fence-loop.ll
+++ b/test/CodeGen/X86/i486-fence-loop.ll
@@ -7,8 +7,7 @@
 define void @gst_atomic_queue_push(i32* %addr) {
 ; CHECK-LABEL: gst_atomic_queue_push:
 ; CHECK: movl (%eax), [[LHS:%e[a-z]+]]
-; CHECK: lock
-; CHECK-NEXT: orl
+; CHECK: lock orl
 ; CHECK: movl (%eax), [[RHS:%e[a-z]+]]
 ; CHECK: cmpl [[LHS]], [[RHS]]
 
diff --git a/test/CodeGen/X86/i64-mem-copy.ll b/test/CodeGen/X86/i64-mem-copy.ll
index 21f88779cf5..69ec3fd806b 100644
--- a/test/CodeGen/X86/i64-mem-copy.ll
+++ b/test/CodeGen/X86/i64-mem-copy.ll
@@ -1,17 +1,76 @@
-; RUN: llc < %s -mtriple=x86_64-linux   | FileCheck %s -check-prefix=X64
-; RUN: llc < %s -mtriple=x86_64-win32   | FileCheck %s -check-prefix=X64
-; X64: movq ({{%rsi|%rdx}}), %r
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=sse2 | FileCheck %s --check-prefix=X64
+; RUN: llc < %s -mtriple=i386-unknown-unknown -mattr=sse2 | FileCheck %s --check-prefix=X32
+; RUN: llc < %s -mtriple=i386-unknown-unknown -mattr=avx2 | FileCheck %s --check-prefix=X32AVX
 
-; RUN: llc < %s -march=x86 -mattr=+sse2 | FileCheck %s -check-prefix=X32
-; X32: movsd ({{%ecx|%eax}}), %xmm
+; Use movq or movsd to load / store i64 values if sse2 is available.
+; rdar://6659858
+
+define void @foo(i64* %x, i64* %y) {
+; X64-LABEL: foo:
+; X64:       # BB#0:
+; X64-NEXT:    movq (%rsi), %rax
+; X64-NEXT:    movq %rax, (%rdi)
+; X64-NEXT:    retq
+;
+; X32-LABEL: foo:
+; X32:       # BB#0:
+; X32-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X32-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X32-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
+; X32-NEXT:    movsd %xmm0, (%eax)
+; X32-NEXT:    retl
+  %tmp1 = load i64, i64* %y, align 8
+  store i64 %tmp1, i64* %x, align 8
+  ret void
+}
 
-; Uses movsd to load / store i64 values if sse2 is available.
+; Verify that a 64-bit chunk extracted from a vector is stored with a movq
+; regardless of whether the system is 64-bit.
 
-; rdar://6659858
+define void @store_i64_from_vector(<8 x i16> %x, <8 x i16> %y, i64* %i) {
+; X64-LABEL: store_i64_from_vector:
+; X64:       # BB#0:
+; X64-NEXT:    paddw %xmm1, %xmm0
+; X64-NEXT:    movq %xmm0, (%rdi)
+; X64-NEXT:    retq
+;
+; X32-LABEL: store_i64_from_vector:
+; X32:       # BB#0:
+; X32-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X32-NEXT:    paddw %xmm1, %xmm0
+; X32-NEXT:    movq %xmm0, (%eax)
+; X32-NEXT:    retl
+  %z = add <8 x i16> %x, %y                          ; force execution domain
+  %bc = bitcast <8 x i16> %z to <2 x i64>
+  %vecext = extractelement <2 x i64> %bc, i32 0
+  store i64 %vecext, i64* %i, align 8
+  ret void
+}
+
+define void @store_i64_from_vector256(<16 x i16> %x, <16 x i16> %y, i64* %i) {
+; X32AVX-LABEL: store_i64_from_vector256:
+; X32AVX:       # BB#0:
+; X32AVX-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X32AVX-NEXT:    vpaddw %ymm1, %ymm0, %ymm0
+; X32AVX-NEXT:    vextracti128 $1, %ymm0, %xmm0
+; X32AVX-NEXT:    vmovq %xmm0, (%eax)
+; X32AVX-NEXT:    vzeroupper
+; X32AVX-NEXT:    retl
+  %z = add <16 x i16> %x, %y                          ; force execution domain
+  %bc = bitcast <16 x i16> %z to <4 x i64>
+  %vecext = extractelement <4 x i64> %bc, i32 2
+  store i64 %vecext, i64* %i, align 8
+  ret void
+}
+
+; PR23476
+; Handle extraction from a non-simple / pre-legalization type.
 
-define void @foo(i64* %x, i64* %y) nounwind  {
-entry:
-	%tmp1 = load i64, i64* %y, align 8		; <i64> [#uses=1]
-	store i64 %tmp1, i64* %x, align 8
-	ret void
+define void @PR23476(<5 x i64> %in, i64* %out, i32 %index) {
+; X32-LABEL: PR23476:
+; X32:         movsd {{.*#+}} xmm0 = mem[0],zero
+; X32-NEXT:    movsd %xmm0, (%eax)
+  %ext = extractelement <5 x i64> %in, i32 %index
+  store i64 %ext, i64* %out, align 8
+  ret void
 }
diff --git a/test/CodeGen/X86/illegal-vector-args-return.ll b/test/CodeGen/X86/illegal-vector-args-return.ll
index 62a21f4c5aa..d783d4fa1b4 100644
--- a/test/CodeGen/X86/illegal-vector-args-return.ll
+++ b/test/CodeGen/X86/illegal-vector-args-return.ll
@@ -3,6 +3,8 @@
 ; RUN: llc < %s -march=x86 -mattr=+sse2 -mcpu=nehalem | grep "addps	%xmm3, %xmm1"
 ; RUN: llc < %s -march=x86 -mattr=+sse2 -mcpu=nehalem | grep "addps	%xmm2, %xmm0"
 
+target triple = "i686-apple-darwin8"
+
 define <4 x double> @foo(<4 x double> %x, <4 x double> %z) {
   %y = fmul <4 x double> %x, %z
   ret <4 x double> %y
diff --git a/test/CodeGen/X86/implicit-null-check-negative.ll b/test/CodeGen/X86/implicit-null-check-negative.ll
new file mode 100644
index 00000000000..c8d425c3889
--- /dev/null
+++ b/test/CodeGen/X86/implicit-null-check-negative.ll
@@ -0,0 +1,96 @@
+; RUN: llc -mtriple=x86_64-apple-macosx -O3 -debug-only=faultmaps -enable-implicit-null-checks < %s 2>&1 | FileCheck %s
+; REQUIRES: asserts
+
+; List cases where we should *not* be emitting implicit null checks.
+
+; CHECK-NOT: Fault Map Output
+
+define i32 @imp_null_check_load(i32* %x, i32* %y) {
+ entry:
+  %c = icmp eq i32* %x, null
+; It isn't legal to move the load from %x from "not_null" to here --
+; the store to %y could be aliasing it.
+  br i1 %c, label %is_null, label %not_null, !make.implicit !0
+
+ is_null:
+  ret i32 42
+
+ not_null:
+  store i32 0, i32* %y
+  %t = load i32, i32* %x
+  ret i32 %t
+}
+
+define i32 @imp_null_check_gep_load(i32* %x) {
+ entry:
+  %c = icmp eq i32* %x, null
+  br i1 %c, label %is_null, label %not_null, !make.implicit !0
+
+ is_null:
+  ret i32 42
+
+ not_null:
+; null + 5000 * sizeof(i32) lies outside the null page and hence the
+; load to %t cannot be assumed to be reliably faulting.
+  %x.gep = getelementptr i32, i32* %x, i32 5000
+  %t = load i32, i32* %x.gep
+  ret i32 %t
+}
+
+define i32 @imp_null_check_load_no_md(i32* %x) {
+; This is fine, except it is missing the !make.implicit metadata.
+ entry:
+  %c = icmp eq i32* %x, null
+  br i1 %c, label %is_null, label %not_null
+
+ is_null:
+  ret i32 42
+
+ not_null:
+  %t = load i32, i32* %x
+  ret i32 %t
+}
+
+define i32 @imp_null_check_no_hoist_over_acquire_load(i32* %x, i32* %y) {
+; We cannot hoist %t1 over %t0 since %t0 is an acquire load
+ entry:
+  %c = icmp eq i32* %x, null
+  br i1 %c, label %is_null, label %not_null, !make.implicit !0
+
+ is_null:
+  ret i32 42
+
+ not_null:
+  %t0 = load atomic i32, i32* %y acquire, align 4
+  %t1 = load i32, i32* %x
+  %p = add i32 %t0, %t1
+  ret i32 %p
+}
+
+define i32 @imp_null_check_add_result(i32* %x, i32* %y) {
+; This will codegen to:
+;
+;   movl    (%rsi), %eax
+;   addl    (%rdi), %eax
+;
+; The load instruction we wish to hoist is the addl, but there is a
+; write-after-write hazard preventing that from happening.  We could
+; get fancy here and exploit the commutativity of addition, but right
+; now -implicit-null-checks isn't that smart.
+;
+
+ entry:
+  %c = icmp eq i32* %x, null
+  br i1 %c, label %is_null, label %not_null, !make.implicit !0
+
+ is_null:
+  ret i32 42
+
+ not_null:
+  %t0 = load i32, i32* %y
+  %t1 = load i32, i32* %x
+  %p = add i32 %t0, %t1
+  ret i32 %p
+}
+
+!0 = !{}
diff --git a/test/CodeGen/X86/implicit-null-check.ll b/test/CodeGen/X86/implicit-null-check.ll
new file mode 100644
index 00000000000..8b905f5d23b
--- /dev/null
+++ b/test/CodeGen/X86/implicit-null-check.ll
@@ -0,0 +1,227 @@
+; RUN: llc -O3 -mtriple=x86_64-apple-macosx -enable-implicit-null-checks < %s | FileCheck %s
+
+; RUN: llc < %s -mtriple=x86_64-apple-macosx -enable-implicit-null-checks \
+; RUN:    | llvm-mc -triple x86_64-apple-macosx -filetype=obj -o - \
+; RUN:    | llvm-objdump -triple x86_64-apple-macosx -fault-map-section - \
+; RUN:    | FileCheck %s -check-prefix OBJDUMP
+
+; RUN: llc < %s -mtriple=x86_64-unknown-linux-gnu -enable-implicit-null-checks \
+; RUN:    | llvm-mc -triple x86_64-unknown-linux-gnu -filetype=obj -o - \
+; RUN:    | llvm-objdump -triple x86_64-unknown-linux-gnu -fault-map-section - \
+; RUN:    | FileCheck %s -check-prefix OBJDUMP
+
+define i32 @imp_null_check_load(i32* %x) {
+; CHECK-LABEL: _imp_null_check_load:
+; CHECK: Ltmp1:
+; CHECK: movl (%rdi), %eax
+; CHECK: retq
+; CHECK: Ltmp0:
+; CHECK: movl $42, %eax
+; CHECK: retq
+
+ entry:
+  %c = icmp eq i32* %x, null
+  br i1 %c, label %is_null, label %not_null, !make.implicit !0
+
+ is_null:
+  ret i32 42
+
+ not_null:
+  %t = load i32, i32* %x
+  ret i32 %t
+}
+
+define i32 @imp_null_check_gep_load(i32* %x) {
+; CHECK-LABEL: _imp_null_check_gep_load:
+; CHECK: Ltmp3:
+; CHECK: movl 128(%rdi), %eax
+; CHECK: retq
+; CHECK: Ltmp2:
+; CHECK: movl $42, %eax
+; CHECK: retq
+
+ entry:
+  %c = icmp eq i32* %x, null
+  br i1 %c, label %is_null, label %not_null, !make.implicit !0
+
+ is_null:
+  ret i32 42
+
+ not_null:
+  %x.gep = getelementptr i32, i32* %x, i32 32
+  %t = load i32, i32* %x.gep
+  ret i32 %t
+}
+
+define i32 @imp_null_check_add_result(i32* %x, i32 %p) {
+; CHECK-LABEL: _imp_null_check_add_result:
+; CHECK: Ltmp5:
+; CHECK: addl (%rdi), %esi
+; CHECK: movl %esi, %eax
+; CHECK: retq
+; CHECK: Ltmp4:
+; CHECK: movl $42, %eax
+; CHECK: retq
+
+ entry:
+  %c = icmp eq i32* %x, null
+  br i1 %c, label %is_null, label %not_null, !make.implicit !0
+
+ is_null:
+  ret i32 42
+
+ not_null:
+  %t = load i32, i32* %x
+  %p1 = add i32 %t, %p
+  ret i32 %p1
+}
+
+define i32 @imp_null_check_hoist_over_unrelated_load(i32* %x, i32* %y, i32* %z) {
+; CHECK-LABEL: _imp_null_check_hoist_over_unrelated_load:
+; CHECK: Ltmp7:
+; CHECK: movl (%rdi), %eax
+; CHECK: movl (%rsi), %ecx
+; CHECK: movl %ecx, (%rdx)
+; CHECK: retq
+; CHECK: Ltmp6:
+; CHECK: movl	$42, %eax
+; CHECK: retq
+
+ entry:
+  %c = icmp eq i32* %x, null
+  br i1 %c, label %is_null, label %not_null, !make.implicit !0
+
+ is_null:
+  ret i32 42
+
+ not_null:
+  %t0 = load i32, i32* %y
+  %t1 = load i32, i32* %x
+  store i32 %t0, i32* %z
+  ret i32 %t1
+}
+
+define i32 @imp_null_check_via_mem_comparision(i32* %x, i32 %val) {
+; CHECK-LABEL: _imp_null_check_via_mem_comparision
+; CHECK: Ltmp9:
+; CHECK: cmpl   %esi, 4(%rdi)
+; CHECK: jge    LBB4_2
+; CHECK: movl   $100, %eax
+; CHECK: retq
+; CHECK: Ltmp8:
+; CHECK: movl   $42, %eax
+; CHECK: retq
+; CHECK: LBB4_2:
+; CHECK: movl   $200, %eax
+; CHECK: retq
+
+ entry:
+  %c = icmp eq i32* %x, null
+  br i1 %c, label %is_null, label %not_null, !make.implicit !0
+
+ is_null:
+  ret i32 42
+
+ not_null:
+  %x.loc = getelementptr i32, i32* %x, i32 1
+  %t = load i32, i32* %x.loc
+  %m = icmp slt i32 %t, %val
+  br i1 %m, label %ret_100, label %ret_200
+
+ ret_100:
+  ret i32 100
+
+ ret_200:
+  ret i32 200
+}
+
+!0 = !{}
+
+; CHECK-LABEL: __LLVM_FaultMaps:
+
+; Version:
+; CHECK-NEXT: .byte 1
+
+; Reserved x2
+; CHECK-NEXT: .byte 0
+; CHECK-NEXT: .short 0
+
+; # functions:
+; CHECK-NEXT: .long 5
+
+; FunctionAddr:
+; CHECK-NEXT: .quad _imp_null_check_add_result
+; NumFaultingPCs
+; CHECK-NEXT: .long 1
+; Reserved:
+; CHECK-NEXT: .long 0
+; Fault[0].Type:
+; CHECK-NEXT: .long 1
+; Fault[0].FaultOffset:
+; CHECK-NEXT: .long Ltmp5-_imp_null_check_add_result
+; Fault[0].HandlerOffset:
+; CHECK-NEXT: .long Ltmp4-_imp_null_check_add_result
+
+; FunctionAddr:
+; CHECK-NEXT: .quad _imp_null_check_gep_load
+; NumFaultingPCs
+; CHECK-NEXT: .long 1
+; Reserved:
+; CHECK-NEXT: .long 0
+; Fault[0].Type:
+; CHECK-NEXT: .long 1
+; Fault[0].FaultOffset:
+; CHECK-NEXT: .long Ltmp3-_imp_null_check_gep_load
+; Fault[0].HandlerOffset:
+; CHECK-NEXT: .long Ltmp2-_imp_null_check_gep_load
+
+; FunctionAddr:
+; CHECK-NEXT: .quad _imp_null_check_hoist_over_unrelated_load
+; NumFaultingPCs
+; CHECK-NEXT: .long 1
+; Reserved:
+; CHECK-NEXT: .long 0
+; Fault[0].Type:
+; CHECK-NEXT: .long 1
+; Fault[0].FaultOffset:
+; CHECK-NEXT: .long Ltmp7-_imp_null_check_hoist_over_unrelated_load
+; Fault[0].HandlerOffset:
+; CHECK-NEXT: .long Ltmp6-_imp_null_check_hoist_over_unrelated_load
+
+; FunctionAddr:
+; CHECK-NEXT: .quad _imp_null_check_load
+; NumFaultingPCs
+; CHECK-NEXT: .long 1
+; Reserved:
+; CHECK-NEXT: .long 0
+; Fault[0].Type:
+; CHECK-NEXT: .long 1
+; Fault[0].FaultOffset:
+; CHECK-NEXT: .long Ltmp1-_imp_null_check_load
+; Fault[0].HandlerOffset:
+; CHECK-NEXT: .long Ltmp0-_imp_null_check_load
+
+; FunctionAddr:
+; CHECK-NEXT: .quad     _imp_null_check_via_mem_comparision
+; NumFaultingPCs
+; CHECK-NEXT: .long   1
+; Reserved:
+; CHECK-NEXT: .long   0
+; Fault[0].Type:
+; CHECK-NEXT: .long   1
+; Fault[0].FaultOffset:
+; CHECK-NEXT: .long   Ltmp9-_imp_null_check_via_mem_comparision
+; Fault[0].HandlerOffset:
+; CHECK-NEXT: .long   Ltmp8-_imp_null_check_via_mem_comparision
+
+; OBJDUMP: FaultMap table:
+; OBJDUMP-NEXT: Version: 0x1
+; OBJDUMP-NEXT: NumFunctions: 5
+; OBJDUMP-NEXT: FunctionAddress: 0x000000, NumFaultingPCs: 1
+; OBJDUMP-NEXT: Fault kind: FaultingLoad, faulting PC offset: 0, handling PC offset: 5
+; OBJDUMP-NEXT: FunctionAddress: 0x000000, NumFaultingPCs: 1
+; OBJDUMP-NEXT: Fault kind: FaultingLoad, faulting PC offset: 0, handling PC offset: 7
+; OBJDUMP-NEXT: FunctionAddress: 0x000000, NumFaultingPCs: 1
+; OBJDUMP-NEXT: Fault kind: FaultingLoad, faulting PC offset: 0, handling PC offset: 7
+; OBJDUMP-NEXT: FunctionAddress: 0x000000, NumFaultingPCs: 1
+; OBJDUMP-NEXT: Fault kind: FaultingLoad, faulting PC offset: 0, handling PC offset: 3
diff --git a/test/CodeGen/X86/inalloca-invoke.ll b/test/CodeGen/X86/inalloca-invoke.ll
index 93bd910ab51..9a184e563b1 100644
--- a/test/CodeGen/X86/inalloca-invoke.ll
+++ b/test/CodeGen/X86/inalloca-invoke.ll
@@ -4,13 +4,14 @@
 
 %frame.reverse = type { %Iter, %Iter }
 
+declare i32 @pers(...)
 declare void @llvm.stackrestore(i8*)
 declare i8* @llvm.stacksave()
 declare void @begin(%Iter* sret)
 declare void @plus(%Iter* sret, %Iter*, i32)
 declare void @reverse(%frame.reverse* inalloca align 4)
 
-define i32 @main() {
+define i32 @main() personality i32 (...)* @pers {
   %temp.lvalue = alloca %Iter
   br label %blah
 
@@ -21,8 +22,8 @@ blah:
   %end = getelementptr %frame.reverse, %frame.reverse* %rev_args, i32 0, i32 1
 
 ; CHECK:  calll   __chkstk
-; CHECK:  movl    %[[beg:[^,]*]], %esp
-; CHECK:  leal    12(%[[beg]]), %[[end:[^ ]*]]
+; CHECK:  movl %esp, %[[beg:[^ ]*]]
+; CHECK:  leal 12(%[[beg]]), %[[end:[^ ]*]]
 
   call void @begin(%Iter* sret %temp.lvalue)
 ; CHECK:  calll _begin
@@ -48,7 +49,7 @@ invoke.cont5:                                     ; preds = %invoke.cont
   ret i32 0
 
 lpad:                                             ; preds = %invoke.cont, %entry
-  %lp = landingpad { i8*, i32 } personality i8* null
+  %lp = landingpad { i8*, i32 }
           cleanup
   unreachable
 }
diff --git a/test/CodeGen/X86/inalloca-stdcall.ll b/test/CodeGen/X86/inalloca-stdcall.ll
index fe99f47353a..e5f6ea70e9c 100644
--- a/test/CodeGen/X86/inalloca-stdcall.ll
+++ b/test/CodeGen/X86/inalloca-stdcall.ll
@@ -10,13 +10,12 @@ define void @g() {
   %b = alloca inalloca %Foo
 ; CHECK: movl    $8, %eax
 ; CHECK: calll   __chkstk
-; CHECK: movl   %[[REG:[^,]*]], %esp
   %f1 = getelementptr %Foo, %Foo* %b, i32 0, i32 0
   %f2 = getelementptr %Foo, %Foo* %b, i32 0, i32 1
   store i32 13, i32* %f1
   store i32 42, i32* %f2
-; CHECK: movl    $13, (%[[REG]])
-; CHECK: movl    $42, 4(%[[REG]])
+; CHECK: movl    $13, (%esp)
+; CHECK: movl    $42, 4(%esp)
   call x86_stdcallcc void @f(%Foo* inalloca %b)
 ; CHECK: calll   _f@8
 ; CHECK-NOT: %esp
diff --git a/test/CodeGen/X86/inalloca.ll b/test/CodeGen/X86/inalloca.ll
index 49ee6a23f4d..904366219ab 100644
--- a/test/CodeGen/X86/inalloca.ll
+++ b/test/CodeGen/X86/inalloca.ll
@@ -10,13 +10,12 @@ entry:
   %b = alloca inalloca %Foo
 ; CHECK: movl    $8, %eax
 ; CHECK: calll   __chkstk
-; CHECK: movl   %[[REG:[^,]*]], %esp
   %f1 = getelementptr %Foo, %Foo* %b, i32 0, i32 0
   %f2 = getelementptr %Foo, %Foo* %b, i32 0, i32 1
   store i32 13, i32* %f1
   store i32 42, i32* %f2
-; CHECK: movl    $13, (%[[REG]])
-; CHECK: movl    $42, 4(%[[REG]])
+; CHECK: movl    $13, (%esp)
+; CHECK: movl    $42, 4(%esp)
   call void @f(%Foo* inalloca %b)
 ; CHECK: calll   _f
   ret void
@@ -30,13 +29,12 @@ entry:
   %b = alloca inalloca %Foo
 ; CHECK: movl    $8, %eax
 ; CHECK: calll   __chkstk
-; CHECK: movl   %[[REG:[^,]*]], %esp
   %f1 = getelementptr %Foo, %Foo* %b, i32 0, i32 0
   %f2 = getelementptr %Foo, %Foo* %b, i32 0, i32 1
   store i32 13, i32* %f1
   store i32 42, i32* %f2
-; CHECK: movl    $13, (%[[REG]])
-; CHECK: movl    $42, 4(%[[REG]])
+; CHECK: movl    $13, (%esp)
+; CHECK: movl    $42, 4(%esp)
   call void @inreg_with_inalloca(i32 inreg 1, %Foo* inalloca %b)
 ; CHECK: movl    $1, %eax
 ; CHECK: calll   _inreg_with_inalloca
@@ -51,13 +49,12 @@ entry:
   %b = alloca inalloca %Foo
 ; CHECK: movl    $8, %eax
 ; CHECK: calll   __chkstk
-; CHECK: movl   %[[REG:[^,]*]], %esp
   %f1 = getelementptr %Foo, %Foo* %b, i32 0, i32 0
   %f2 = getelementptr %Foo, %Foo* %b, i32 0, i32 1
   store i32 13, i32* %f1
   store i32 42, i32* %f2
-; CHECK-DAG: movl    $13, (%[[REG]])
-; CHECK-DAG: movl    $42, 4(%[[REG]])
+; CHECK-DAG: movl    $13, (%esp)
+; CHECK-DAG: movl    $42, 4(%esp)
   call x86_thiscallcc void @thiscall_with_inalloca(i8* null, %Foo* inalloca %b)
 ; CHECK-DAG: xorl    %ecx, %ecx
 ; CHECK: calll   _thiscall_with_inalloca
diff --git a/test/CodeGen/X86/indirect-hidden.ll b/test/CodeGen/X86/indirect-hidden.ll
index 309375d9302..9e1b7d37355 100644
--- a/test/CodeGen/X86/indirect-hidden.ll
+++ b/test/CodeGen/X86/indirect-hidden.ll
@@ -8,10 +8,10 @@
 
 declare void @throws()
 
-define void @get_indirect_hidden() {
+define void @get_indirect_hidden() personality i8* bitcast (i32 (...)* @__gxx_personality_v0 to i8*) {
   invoke void @throws() to label %end unwind label %lpad
 lpad:
-  %tmp = landingpad { i8*, i32 } personality i8* bitcast (i32 (...)* @__gxx_personality_v0 to i8*)
+  %tmp = landingpad { i8*, i32 }
           catch i8* bitcast (i8** @hidden_typeid to i8*)
   br label %end
 
@@ -19,10 +19,10 @@ end:
   ret void
 }
 
-define void @get_indirect() {
+define void @get_indirect() personality i8* bitcast (i32 (...)* @__gxx_personality_v0 to i8*) {
   invoke void @throws() to label %end unwind label %lpad
 lpad:
-  %tmp = landingpad { i8*, i32 } personality i8* bitcast (i32 (...)* @__gxx_personality_v0 to i8*)
+  %tmp = landingpad { i8*, i32 }
           catch i8* bitcast (i8** @normal_typeid to i8*)
   br label %end
 
diff --git a/test/CodeGen/X86/inline-asm-bad-constraint-n.ll b/test/CodeGen/X86/inline-asm-bad-constraint-n.ll
new file mode 100644
index 00000000000..91b1ffed4e0
--- /dev/null
+++ b/test/CodeGen/X86/inline-asm-bad-constraint-n.ll
@@ -0,0 +1,10 @@
+; RUN: not llc -march=x86 -no-integrated-as < %s 2>&1 | FileCheck %s
+
+@x = global i32 0, align 4
+
+;CHECK:	error: invalid operand for inline asm constraint 'n'
+define void @foo() {
+  %a = getelementptr i32, i32* @x, i32 1
+  call void asm sideeffect "foo $0", "n"(i32* %a) nounwind
+  ret void
+}
diff --git a/test/CodeGen/X86/invalid-gcc-snan-conversion.ll b/test/CodeGen/X86/invalid-gcc-snan-conversion.ll
deleted file mode 100644
index 4b0b8cc82b4..00000000000
--- a/test/CodeGen/X86/invalid-gcc-snan-conversion.ll
+++ /dev/null
@@ -1,25 +0,0 @@
-; If LLVM is built in Release mode with a buggy gcc under x86-32, it
-; may transform 64-bit constants with a signaling NaN bit pattern into
-; a quiet NaN bit pattern.  See
-; http://gcc.gnu.org/bugzilla/show_bug.cgi?id=58416
- 
-; RUN: llc -march=x86-64 < %s | FileCheck %s
-
-define i32 @main() #0 {
-entry:
-  %retval = alloca i32, align 4
-  %tmp = alloca i64, align 8
-  store i32 0, i32* %retval
-; -4503599627370495 == 0xfff0000000000001
-  store i64 -4503599627370495, i64* %tmp, align 8
-  %0 = load i64, i64* %tmp, align 8
-  call void @Consume(i64 %0)
-  ret i32 0
-}
-
-; CHECK: main:
-; make sure 0xfff0000000000001 didn't change to 0xfff8000000000001
-; CHECK: 0xFFF00000
-; CHECK-NOT: 0xFFF80000
-
-declare void @Consume(i64) #1
diff --git a/test/CodeGen/X86/large-gep-chain.ll b/test/CodeGen/X86/large-gep-chain.ll
index 99f58796542..8df282983f5 100644
--- a/test/CodeGen/X86/large-gep-chain.ll
+++ b/test/CodeGen/X86/large-gep-chain.ll
@@ -13,7 +13,7 @@
 @7 = external unnamed_addr constant [27 x i8], align 1
 @8 = external unnamed_addr constant [63 x i8], align 1
 
-define void @main() uwtable ssp {
+define void @main() uwtable ssp personality i8* bitcast (i32 (...)* @__gxx_personality_v0 to i8*) {
 bb:
   br i1 undef, label %bb1, label %bb2
 
@@ -25313,7 +25313,7 @@ bb25275:                                          ; preds = %bb25274
   br label %bb25272
 
 bb25276:                                          ; preds = %bb25283, %bb25274, %bb25273
-  %tmp25277 = landingpad { i8*, i32 } personality i8* bitcast (i32 (...)* @__gxx_personality_v0 to i8*)
+  %tmp25277 = landingpad { i8*, i32 }
           cleanup
   br label %bb25361
 
@@ -25383,7 +25383,7 @@ bb25297:                                          ; preds = %bb25296
   br label %bb25300
 
 bb25298:                                          ; preds = %bb25296, %bb25295, %bb25290, %bb25287
-  %tmp25299 = landingpad { i8*, i32 } personality i8* bitcast (i32 (...)* @__gxx_personality_v0 to i8*)
+  %tmp25299 = landingpad { i8*, i32 }
           cleanup
   br label %bb25360
 
@@ -25461,7 +25461,7 @@ bb25323:                                          ; preds = %bb25319
           to label %bb25326 unwind label %bb25324
 
 bb25324:                                          ; preds = %bb25357, %bb25344, %bb25343, %bb25342, %bb25337, %bb25334, %bb25333, %bb25323, %bb25313, %bb25307, %bb25306
-  %tmp25325 = landingpad { i8*, i32 } personality i8* bitcast (i32 (...)* @__gxx_personality_v0 to i8*)
+  %tmp25325 = landingpad { i8*, i32 }
           cleanup
   br label %bb25359
 
@@ -25487,11 +25487,11 @@ bb25332:                                          ; preds = %bb25330, %bb25305
   br i1 undef, label %bb25333, label %bb25357
 
 bb25333:                                          ; preds = %bb25332
-  invoke void (...)* @printf()
+  invoke void (...) @printf()
           to label %bb25334 unwind label %bb25324
 
 bb25334:                                          ; preds = %bb25333
-  invoke void (...)* @printf(i32 undef)
+  invoke void (...) @printf(i32 undef)
           to label %bb25335 unwind label %bb25324
 
 bb25335:                                          ; preds = %bb25334
@@ -25501,7 +25501,7 @@ bb25336:                                          ; preds = %bb25338, %bb25335
   br i1 undef, label %bb25337, label %bb25339
 
 bb25337:                                          ; preds = %bb25336
-  invoke void (...)* @printf(i32 undef, double undef)
+  invoke void (...) @printf(i32 undef, double undef)
           to label %bb25338 unwind label %bb25324
 
 bb25338:                                          ; preds = %bb25337
@@ -25517,11 +25517,11 @@ bb25341:                                          ; preds = %bb25340
   br label %bb25340
 
 bb25342:                                          ; preds = %bb25340
-  invoke void (...)* @printf()
+  invoke void (...) @printf()
           to label %bb25343 unwind label %bb25324
 
 bb25343:                                          ; preds = %bb25342
-  invoke void (...)* @printf(double undef, double undef)
+  invoke void (...) @printf(double undef, double undef)
           to label %bb25344 unwind label %bb25324
 
 bb25344:                                          ; preds = %bb25343
@@ -25547,27 +25547,27 @@ bb25350:                                          ; preds = %bb25349
   br label %bb25349
 
 bb25351:                                          ; preds = %bb25349
-  invoke void (...)* @printf()
+  invoke void (...) @printf()
           to label %bb25352 unwind label %bb25355
 
 bb25352:                                          ; preds = %bb25351
-  invoke void (...)* @printf(double undef)
+  invoke void (...) @printf(double undef)
           to label %bb25353 unwind label %bb25355
 
 bb25353:                                          ; preds = %bb25352
-  invoke void (...)* @printf()
+  invoke void (...) @printf()
           to label %bb25354 unwind label %bb25355
 
 bb25354:                                          ; preds = %bb25353
   br label %bb25358
 
 bb25355:                                          ; preds = %bb25353, %bb25352, %bb25351
-  %tmp25356 = landingpad { i8*, i32 } personality i8* bitcast (i32 (...)* @__gxx_personality_v0 to i8*)
+  %tmp25356 = landingpad { i8*, i32 }
           cleanup
   br label %bb25359
 
 bb25357:                                          ; preds = %bb25332
-  invoke void (...)* @printf()
+  invoke void (...) @printf()
           to label %bb25358 unwind label %bb25324
 
 bb25358:                                          ; preds = %bb25357, %bb25354
diff --git a/test/CodeGen/X86/lea-2.ll b/test/CodeGen/X86/lea-2.ll
index 0106c278874..98c57c7d090 100644
--- a/test/CodeGen/X86/lea-2.ll
+++ b/test/CodeGen/X86/lea-2.ll
@@ -9,8 +9,7 @@ define i32 @test1(i32 %A, i32 %B) {
   %tmp4 = add i32 %tmp3, %tmp1
 ; The above computation of %tmp4 should match a single lea, without using
 ; actual add instructions.
-; @LOCALMOD: The original regex would match ".dwarf_addr_size 4".
-; CHECK-NOT: add{{.*}},
+; CHECK-NOT: add
 ; CHECK: lea {{[a-z]+}}, [{{[a-z]+}} + 4*{{[a-z]+}} - 5]
 
   ret i32 %tmp4
diff --git a/test/CodeGen/X86/lea-5.ll b/test/CodeGen/X86/lea-5.ll
index e5fbdf1b85d..b89c199e719 100644
--- a/test/CodeGen/X86/lea-5.ll
+++ b/test/CodeGen/X86/lea-5.ll
@@ -4,9 +4,7 @@
 
 ; RUN: llc < %s -mtriple=x86_64-linux -O2        | FileCheck %s
 ; RUN: llc < %s -mtriple=x86_64-linux-gnux32 -O2 | FileCheck %s -check-prefix=X32
-; @LOCALMOD -- one of the leal below are left as an add for NACL64
-; TODO -- investigate why.
-; RUN: llc < %s -mtriple=x86_64-nacl -O2 | FileCheck %s -check-prefix=NACL64
+; RUN: llc < %s -mtriple=x86_64-nacl -O2 | FileCheck %s -check-prefix=X32
 
 ; Function Attrs: nounwind readnone uwtable
 define void @foo(i32 %x, i32 %d) #0 {
@@ -20,14 +18,12 @@ while.cond:                                       ; preds = %while.cond, %entry
 
 ; CHECK: leaq	-40(%rsp,%r{{[^,]*}},4), %rax
 ; X32:   leal	-40(%rsp,%r{{[^,]*}},4), %eax
-; NACL64: leal	-40(%rsp,%r{{[^,]*}},4), %eax
   %0 = load i32, i32* %arrayidx, align 4
   %cmp1 = icmp eq i32 %0, 0
   %inc = add nsw i32 %d.addr.0, 1
 
 ; CHECK: leaq	4(%r{{[^,]*}}), %r{{[^,]*}}
 ; X32:   leal	4(%r{{[^,]*}}), %e{{[^,]*}}
-; NACL64: addl $4, %e{{[^,]*}}
   br i1 %cmp1, label %while.end, label %while.cond
 
 while.end:                                        ; preds = %while.cond
@@ -49,14 +45,12 @@ while.cond:                                       ; preds = %while.cond, %entry
 
 ; CHECK: leaq	(%rsp,%r{{[^,]*}},4), %rax
 ; X32:   leal	(%rsp,%r{{[^,]*}},4), %eax
-; NACL64: leal (%rsp,%r{{[^,]*}},4), %eax
   %0 = load i32, i32* %arrayidx, align 4
   %cmp1 = icmp eq i32 %0, 0
   %inc = add nsw i32 %d.addr.0, 1
 
 ; CHECK: leaq	4(%r{{[^,]*}}), %r{{[^,]*}}
 ; X32:   leal	4(%r{{[^,]*}}), %e{{[^,]*}}
-; NACL64: addl $4, %e{{[^,]*}}
   br i1 %cmp1, label %while.end, label %while.cond
 
 while.end:                                        ; preds = %while.cond
diff --git a/test/CodeGen/X86/legalize-shl-vec.ll b/test/CodeGen/X86/legalize-shl-vec.ll
new file mode 100644
index 00000000000..7ec2a663513
--- /dev/null
+++ b/test/CodeGen/X86/legalize-shl-vec.ll
@@ -0,0 +1,44 @@
+; RUN: llc < %s -march=x86-64 | FileCheck %s
+
+define <2 x i256> @test_shl(<2 x i256> %In) {
+  %Amt = insertelement <2 x i256> undef, i256 -1, i32 0
+  %Out = shl <2 x i256> %In, %Amt
+  ret <2 x i256> %Out
+
+  ; CHECK-LABEL: test_shl
+  ; CHECK:       movq $0
+  ; CHECK-NEXT:  movq $0
+  ; CHECK-NEXT:  movq $0
+  ; CHECK-NEXT:  movq $0
+  ; CHECK-NEXT:  movq $0
+  ; CHECK-NEXT:  movq $0
+  ; CHECK-NEXT:  movq $0
+  ; CHECK-NEXT:  movq $0
+  ; CHECK:       retq
+}
+
+define <2 x i256> @test_srl(<2 x i256> %In) {
+  %Amt = insertelement <2 x i256> undef, i256 -1, i32 0
+  %Out = lshr <2 x i256> %In, %Amt
+  ret <2 x i256> %Out
+
+  ; CHECK-LABEL: test_srl
+  ; CHECK:       movq $0
+  ; CHECK-NEXT:  movq $0
+  ; CHECK-NEXT:  movq $0
+  ; CHECK-NEXT:  movq $0
+  ; CHECK-NEXT:  movq $0
+  ; CHECK-NEXT:  movq $0
+  ; CHECK-NEXT:  movq $0
+  ; CHECK-NEXT:  movq $0
+  ; CHECK:       retq
+}
+
+define <2 x i256> @test_sra(<2 x i256> %In) {
+  %Amt = insertelement <2 x i256> undef, i256 -1, i32 0
+  %Out = ashr <2 x i256> %In, %Amt
+  ret <2 x i256> %Out
+
+  ; CHECK-LABEL: test_sra
+  ; CHECK:       sarq $63
+}
diff --git a/test/CodeGen/X86/lit.local.cfg b/test/CodeGen/X86/lit.local.cfg
index 8ed58f119c4..4a1dd86abc4 100644
--- a/test/CodeGen/X86/lit.local.cfg
+++ b/test/CodeGen/X86/lit.local.cfg
@@ -4,7 +4,7 @@
 #
 # It should be possible to remove this override once all the bots have cycled
 # cleanly.
-config.suffixes = ['.ll', '.test', '.txt']
+config.suffixes = ['.ll', '.mir', '.test', '.txt']
 
 if not 'X86' in config.root.targets:
     config.unsupported = True
diff --git a/test/CodeGen/X86/llc-override-mcpu-mattr.ll b/test/CodeGen/X86/llc-override-mcpu-mattr.ll
new file mode 100644
index 00000000000..19a5ed59186
--- /dev/null
+++ b/test/CodeGen/X86/llc-override-mcpu-mattr.ll
@@ -0,0 +1,21 @@
+; RUN: llc < %s -march x86-64 -mcpu=broadwell | FileCheck %s
+; RUN: llc < %s -march x86-64 -mattr=+avx2 | FileCheck %s
+
+; Check that llc can overide function attributes target-cpu and target-features
+; using command line options -mcpu and -mattr.
+
+; CHECK: vpsadbw (%r{{si|dx}}), %ymm{{[0-9]+}}, %ymm{{[0-9]+}}
+
+define <4 x i64> @foo1(<4 x i64>* %s1, <4 x i64>* %s2) #0 {
+entry:
+  %ps1 = load <4 x i64>, <4 x i64>* %s1
+  %ps2 = load <4 x i64>, <4 x i64>* %s2
+  %0 = bitcast <4 x i64> %ps1 to <32 x i8>
+  %1 = bitcast <4 x i64> %ps2 to <32 x i8>
+  %2 = tail call <4 x i64> @llvm.x86.avx2.psad.bw(<32 x i8> %0, <32 x i8> %1)
+  ret <4 x i64> %2
+}
+
+declare <4 x i64> @llvm.x86.avx2.psad.bw(<32 x i8>, <32 x i8>)
+
+attributes #0 = { "target-cpu"="core2" "target-features"="+ssse3,+cx16,+sse4.2,+sse4.1,+sse,+sse2,+sse3,+avx,+popcnt" }
diff --git a/test/CodeGen/X86/load-slice.ll b/test/CodeGen/X86/load-slice.ll
index 7f3dd63d2ea..2f90f819d47 100644
--- a/test/CodeGen/X86/load-slice.ll
+++ b/test/CodeGen/X86/load-slice.ll
@@ -28,7 +28,7 @@
 ; Swap Imm and Real.
 ; STRESS-NEXT: vinsertps $16, [[RES_Imm]], [[RES_Real]], [[RES_Vec:%xmm[0-9]+]]
 ; Put the results back into out[out_start].
-; STRESS-NEXT: vmovq [[RES_Vec]], ([[BASE]])
+; STRESS-NEXT: vmovlps [[RES_Vec]], ([[BASE]])
 ;
 ; Same for REGULAR, we eliminate register bank copy with each slices.
 ; REGULAR-LABEL: t1:
@@ -43,7 +43,7 @@
 ; Swap Imm and Real.
 ; REGULAR-NEXT: vinsertps $16, [[RES_Imm]], [[RES_Real]], [[RES_Vec:%xmm[0-9]+]]
 ; Put the results back into out[out_start].
-; REGULAR-NEXT: vmovq [[RES_Vec]], ([[BASE]])
+; REGULAR-NEXT: vmovlps [[RES_Vec]], ([[BASE]])
 define void @t1(%class.Complex* nocapture %out, i64 %out_start) {
 entry:
   %arrayidx = getelementptr inbounds %class.Complex, %class.Complex* %out, i64 %out_start
diff --git a/test/CodeGen/X86/machine-combiner-int.ll b/test/CodeGen/X86/machine-combiner-int.ll
new file mode 100644
index 00000000000..2f3944f98b0
--- /dev/null
+++ b/test/CodeGen/X86/machine-combiner-int.ll
@@ -0,0 +1,43 @@
+; RUN: llc -mtriple=x86_64-unknown-unknown -mcpu=x86-64 < %s | FileCheck %s
+
+; Verify that integer multiplies are reassociated. The first multiply in 
+; each test should be independent of the result of the preceding add (lea).
+
+define i16 @reassociate_muls_i16(i16 %x0, i16 %x1, i16 %x2, i16 %x3) {
+; CHECK-LABEL: reassociate_muls_i16:
+; CHECK:       # BB#0:
+; CHECK-NEXT:    leal   (%rdi,%rsi), %eax
+; CHECK-NEXT:    imull  %ecx, %edx
+; CHECK-NEXT:    imull  %edx, %eax
+; CHECK-NEXT:    retq
+  %t0 = add i16 %x0, %x1
+  %t1 = mul i16 %x2, %t0
+  %t2 = mul i16 %x3, %t1
+  ret i16 %t2
+}
+
+define i32 @reassociate_muls_i32(i32 %x0, i32 %x1, i32 %x2, i32 %x3) {
+; CHECK-LABEL: reassociate_muls_i32:
+; CHECK:       # BB#0:
+; CHECK-NEXT:    leal   (%rdi,%rsi), %eax
+; CHECK-NEXT:    imull  %ecx, %edx
+; CHECK-NEXT:    imull  %edx, %eax
+; CHECK-NEXT:    retq
+  %t0 = add i32 %x0, %x1
+  %t1 = mul i32 %x2, %t0
+  %t2 = mul i32 %x3, %t1
+  ret i32 %t2
+}
+
+define i64 @reassociate_muls_i64(i64 %x0, i64 %x1, i64 %x2, i64 %x3) {
+; CHECK-LABEL: reassociate_muls_i64:
+; CHECK:       # BB#0:
+; CHECK-NEXT:    leaq   (%rdi,%rsi), %rax
+; CHECK-NEXT:    imulq  %rcx, %rdx
+; CHECK-NEXT:    imulq  %rdx, %rax
+; CHECK-NEXT:    retq
+  %t0 = add i64 %x0, %x1
+  %t1 = mul i64 %x2, %t0
+  %t2 = mul i64 %x3, %t1
+  ret i64 %t2
+}
diff --git a/test/CodeGen/X86/machine-combiner.ll b/test/CodeGen/X86/machine-combiner.ll
new file mode 100644
index 00000000000..0943bebbb09
--- /dev/null
+++ b/test/CodeGen/X86/machine-combiner.ll
@@ -0,0 +1,212 @@
+; RUN: llc -mtriple=x86_64-unknown-unknown -mcpu=x86-64 -mattr=sse -enable-unsafe-fp-math < %s | FileCheck %s --check-prefix=SSE
+; RUN: llc -mtriple=x86_64-unknown-unknown -mcpu=x86-64 -mattr=avx -enable-unsafe-fp-math < %s | FileCheck %s --check-prefix=AVX
+
+; Verify that the first two adds are independent regardless of how the inputs are
+; commuted. The destination registers are used as source registers for the third add.
+
+define float @reassociate_adds1(float %x0, float %x1, float %x2, float %x3) {
+; SSE-LABEL: reassociate_adds1:
+; SSE:       # BB#0:
+; SSE-NEXT:    addss %xmm1, %xmm0
+; SSE-NEXT:    addss %xmm3, %xmm2
+; SSE-NEXT:    addss %xmm2, %xmm0
+; SSE-NEXT:    retq
+;
+; AVX-LABEL: reassociate_adds1:
+; AVX:       # BB#0:
+; AVX-NEXT:    vaddss %xmm1, %xmm0, %xmm0
+; AVX-NEXT:    vaddss %xmm3, %xmm2, %xmm1
+; AVX-NEXT:    vaddss %xmm1, %xmm0, %xmm0
+; AVX-NEXT:    retq
+  %t0 = fadd float %x0, %x1
+  %t1 = fadd float %t0, %x2
+  %t2 = fadd float %t1, %x3
+  ret float %t2
+}
+
+define float @reassociate_adds2(float %x0, float %x1, float %x2, float %x3) {
+; SSE-LABEL: reassociate_adds2:
+; SSE:       # BB#0:
+; SSE-NEXT:    addss %xmm1, %xmm0
+; SSE-NEXT:    addss %xmm3, %xmm2
+; SSE-NEXT:    addss %xmm2, %xmm0
+; SSE-NEXT:    retq
+;
+; AVX-LABEL: reassociate_adds2:
+; AVX:       # BB#0:
+; AVX-NEXT:    vaddss %xmm1, %xmm0, %xmm0
+; AVX-NEXT:    vaddss %xmm3, %xmm2, %xmm1
+; AVX-NEXT:    vaddss %xmm1, %xmm0, %xmm0
+; AVX-NEXT:    retq
+  %t0 = fadd float %x0, %x1
+  %t1 = fadd float %x2, %t0
+  %t2 = fadd float %t1, %x3
+  ret float %t2
+}
+
+define float @reassociate_adds3(float %x0, float %x1, float %x2, float %x3) {
+; SSE-LABEL: reassociate_adds3:
+; SSE:       # BB#0:
+; SSE-NEXT:    addss %xmm1, %xmm0
+; SSE-NEXT:    addss %xmm3, %xmm2
+; SSE-NEXT:    addss %xmm2, %xmm0
+; SSE-NEXT:    retq
+;
+; AVX-LABEL: reassociate_adds3:
+; AVX:       # BB#0:
+; AVX-NEXT:    vaddss %xmm1, %xmm0, %xmm0
+; AVX-NEXT:    vaddss %xmm3, %xmm2, %xmm1
+; AVX-NEXT:    vaddss %xmm1, %xmm0, %xmm0
+; AVX-NEXT:    retq
+  %t0 = fadd float %x0, %x1
+  %t1 = fadd float %t0, %x2
+  %t2 = fadd float %x3, %t1
+  ret float %t2
+}
+
+define float @reassociate_adds4(float %x0, float %x1, float %x2, float %x3) {
+; SSE-LABEL: reassociate_adds4:
+; SSE:       # BB#0:
+; SSE-NEXT:    addss %xmm1, %xmm0
+; SSE-NEXT:    addss %xmm3, %xmm2
+; SSE-NEXT:    addss %xmm2, %xmm0
+; SSE-NEXT:    retq
+;
+; AVX-LABEL: reassociate_adds4:
+; AVX:       # BB#0:
+; AVX-NEXT:    vaddss %xmm1, %xmm0, %xmm0
+; AVX-NEXT:    vaddss %xmm3, %xmm2, %xmm1
+; AVX-NEXT:    vaddss %xmm1, %xmm0, %xmm0
+; AVX-NEXT:    retq
+  %t0 = fadd float %x0, %x1
+  %t1 = fadd float %x2, %t0
+  %t2 = fadd float %x3, %t1
+  ret float %t2
+}
+
+; Verify that we reassociate some of these ops. The optimal balanced tree of adds is not
+; produced because that would cost more compile time.
+
+define float @reassociate_adds5(float %x0, float %x1, float %x2, float %x3, float %x4, float %x5, float %x6, float %x7) {
+; SSE-LABEL: reassociate_adds5:
+; SSE:       # BB#0:
+; SSE-NEXT:    addss %xmm1, %xmm0
+; SSE-NEXT:    addss %xmm3, %xmm2
+; SSE-NEXT:    addss %xmm2, %xmm0
+; SSE-NEXT:    addss %xmm5, %xmm4
+; SSE-NEXT:    addss %xmm6, %xmm4
+; SSE-NEXT:    addss %xmm4, %xmm0
+; SSE-NEXT:    addss %xmm7, %xmm0
+; SSE-NEXT:    retq
+;
+; AVX-LABEL: reassociate_adds5:
+; AVX:       # BB#0:
+; AVX-NEXT:    vaddss %xmm1, %xmm0, %xmm0
+; AVX-NEXT:    vaddss %xmm3, %xmm2, %xmm1
+; AVX-NEXT:    vaddss %xmm1, %xmm0, %xmm0
+; AVX-NEXT:    vaddss %xmm5, %xmm4, %xmm1
+; AVX-NEXT:    vaddss %xmm6, %xmm1, %xmm1
+; AVX-NEXT:    vaddss %xmm1, %xmm0, %xmm0
+; AVX-NEXT:    vaddss %xmm7, %xmm0, %xmm0
+; AVX-NEXT:    retq
+  %t0 = fadd float %x0, %x1
+  %t1 = fadd float %t0, %x2
+  %t2 = fadd float %t1, %x3
+  %t3 = fadd float %t2, %x4
+  %t4 = fadd float %t3, %x5
+  %t5 = fadd float %t4, %x6
+  %t6 = fadd float %t5, %x7
+  ret float %t6
+}
+
+; Verify that we only need two associative operations to reassociate the operands.
+; Also, we should reassociate such that the result of the high latency division
+; is used by the final 'add' rather than reassociating the %x3 operand with the
+; division. The latter reassociation would not improve anything.
+
+define float @reassociate_adds6(float %x0, float %x1, float %x2, float %x3) {
+; SSE-LABEL: reassociate_adds6:
+; SSE:       # BB#0:
+; SSE-NEXT:    divss %xmm1, %xmm0
+; SSE-NEXT:    addss %xmm3, %xmm2
+; SSE-NEXT:    addss %xmm2, %xmm0
+; SSE-NEXT:    retq
+;
+; AVX-LABEL: reassociate_adds6:
+; AVX:       # BB#0:
+; AVX-NEXT:    vdivss %xmm1, %xmm0, %xmm0
+; AVX-NEXT:    vaddss %xmm3, %xmm2, %xmm1
+; AVX-NEXT:    vaddss %xmm1, %xmm0, %xmm0
+; AVX-NEXT:    retq
+  %t0 = fdiv float %x0, %x1
+  %t1 = fadd float %x2, %t0
+  %t2 = fadd float %x3, %t1
+  ret float %t2
+}
+
+; Verify that SSE and AVX scalar single-precison multiplies are reassociated.
+
+define float @reassociate_muls1(float %x0, float %x1, float %x2, float %x3) {
+; SSE-LABEL: reassociate_muls1:
+; SSE:       # BB#0:
+; SSE-NEXT:    divss %xmm1, %xmm0
+; SSE-NEXT:    mulss %xmm3, %xmm2
+; SSE-NEXT:    mulss %xmm2, %xmm0
+; SSE-NEXT:    retq
+;
+; AVX-LABEL: reassociate_muls1:
+; AVX:       # BB#0:
+; AVX-NEXT:    vdivss %xmm1, %xmm0, %xmm0
+; AVX-NEXT:    vmulss %xmm3, %xmm2, %xmm1
+; AVX-NEXT:    vmulss %xmm1, %xmm0, %xmm0
+; AVX-NEXT:    retq
+  %t0 = fdiv float %x0, %x1
+  %t1 = fmul float %x2, %t0
+  %t2 = fmul float %x3, %t1
+  ret float %t2
+}
+
+; Verify that SSE and AVX scalar double-precison adds are reassociated.
+
+define double @reassociate_adds_double(double %x0, double %x1, double %x2, double %x3) {
+; SSE-LABEL: reassociate_adds_double:
+; SSE:       # BB#0:
+; SSE-NEXT:    divsd %xmm1, %xmm0
+; SSE-NEXT:    addsd %xmm3, %xmm2
+; SSE-NEXT:    addsd %xmm2, %xmm0
+; SSE-NEXT:    retq
+;
+; AVX-LABEL: reassociate_adds_double:
+; AVX:       # BB#0:
+; AVX-NEXT:    vdivsd %xmm1, %xmm0, %xmm0
+; AVX-NEXT:    vaddsd %xmm3, %xmm2, %xmm1
+; AVX-NEXT:    vaddsd %xmm1, %xmm0, %xmm0
+; AVX-NEXT:    retq
+  %t0 = fdiv double %x0, %x1
+  %t1 = fadd double %x2, %t0
+  %t2 = fadd double %x3, %t1
+  ret double %t2
+}
+
+; Verify that SSE and AVX scalar double-precison multiplies are reassociated.
+
+define double @reassociate_muls_double(double %x0, double %x1, double %x2, double %x3) {
+; SSE-LABEL: reassociate_muls_double:
+; SSE:       # BB#0:
+; SSE-NEXT:    divsd %xmm1, %xmm0
+; SSE-NEXT:    mulsd %xmm3, %xmm2
+; SSE-NEXT:    mulsd %xmm2, %xmm0
+; SSE-NEXT:    retq
+;
+; AVX-LABEL: reassociate_muls_double:
+; AVX:       # BB#0:
+; AVX-NEXT:    vdivsd %xmm1, %xmm0, %xmm0
+; AVX-NEXT:    vmulsd %xmm3, %xmm2, %xmm1
+; AVX-NEXT:    vmulsd %xmm1, %xmm0, %xmm0
+; AVX-NEXT:    retq
+  %t0 = fdiv double %x0, %x1
+  %t1 = fmul double %x2, %t0
+  %t2 = fmul double %x3, %t1
+  ret double %t2
+}
+
diff --git a/test/CodeGen/X86/machine-cp.ll b/test/CodeGen/X86/machine-cp.ll
index 0006b6ea713..aaed0f0a23d 100644
--- a/test/CodeGen/X86/machine-cp.ll
+++ b/test/CodeGen/X86/machine-cp.ll
@@ -58,3 +58,58 @@ while.end:                                        ; preds = %while.body, %entry
   %t = trunc i64 %a.addr.0.lcssa to i32
   ret i32 %t
 }
+
+; Check that copy propagation does not kill thing like:
+; dst = copy src <-- do not kill that.
+; ... = op1 dst<undef>
+; ... = op2 dst <-- this is used here.
+;
+; CHECK-LABEL: foo:
+; CHECK: psllw $7,
+; CHECK: psllw $7,
+; CHECK-NEXT: pand
+; CHECK-NEXT: pcmpgtb 
+; CHECK-NEXT: pand %xmm{{[0-9]+}}, [[SRC:%xmm[0-9]+]]
+; Machine propagation used to delete the first copy as the
+; first few uses were <undef>.
+; CHECK-NEXT: movdqa [[SRC]], [[CPY1:%xmm[0-9]+]]
+; CHECK-NEXT: movdqa [[SRC]], [[CPY2:%xmm[0-9]+]]
+; CHECK-NEXT: punpckhbw [[SRC]],
+; Check that CPY1 is not redefined.
+; CHECK-NOT: , [[CPY1]]
+; undef use, we do not care.
+; CHECK: punpcklwd [[CPY1]],
+; Check that CPY1 is not redefined.
+; CHECK-NOT: , [[CPY1]]
+; CHECK: punpcklbw [[CPY2]], [[CPY2]]
+; CHECK-NEXT: punpckhwd [[CPY2]], [[CPY2]]
+; CHECK-NEXT pslld $31, [[CPY2]]
+; Check that CPY1 is not redefined.
+; CHECK-NOT: , [[CPY1]]
+; CHECK: punpcklbw [[CPY1]], [[CPY1]]
+; CHECK-NEXT: punpcklwd [[CPY1]], [[CPY1]]
+; CHECK-NEXT pslld $31, [[CPY1]]
+define <16 x float> @foo(<16 x float> %x) {
+bb:
+  %v3 = icmp slt <16 x i32> undef, zeroinitializer
+  %v14 = zext <16 x i1> %v3 to <16 x i32>
+  %v16 = fcmp olt <16 x float> %x, zeroinitializer
+  %v17 = sext <16 x i1> %v16 to <16 x i32>
+  %v18 = zext <16 x i1> %v16 to <16 x i32>
+  %v19 = xor <16 x i32> %v14, %v18
+  %v20 = or <16 x i32> %v17, undef
+  %v21 = fptosi <16 x float> %x to <16 x i32>
+  %v22 = sitofp <16 x i32> %v21 to <16 x float>
+  %v69 = fcmp ogt <16 x float> %v22, zeroinitializer
+  %v75 = and <16 x i1> %v69, %v3
+  %v77 = bitcast <16 x float> %v22 to <16 x i32>
+  %v79 = sext <16 x i1> %v75 to <16 x i32>
+  %v80 = and <16 x i32> undef, %v79
+  %v81 = xor <16 x i32> %v77, %v80
+  %v82 = and <16 x i32> undef, %v81
+  %v83 = xor <16 x i32> %v19, %v82
+  %v84 = and <16 x i32> %v83, %v20
+  %v85 = xor <16 x i32> %v19, %v84
+  %v86 = bitcast <16 x i32> %v85 to <16 x float>
+  ret <16 x float> %v86
+}
diff --git a/test/CodeGen/X86/machine-trace-metrics-crash.ll b/test/CodeGen/X86/machine-trace-metrics-crash.ll
new file mode 100644
index 00000000000..1d0ee79f04a
--- /dev/null
+++ b/test/CodeGen/X86/machine-trace-metrics-crash.ll
@@ -0,0 +1,62 @@
+; RUN: llc -mtriple=x86_64-unknown-unknown -mcpu=x86-64 -mattr=sse -enable-unsafe-fp-math < %s | FileCheck %s
+
+; The debug info in this test case was causing a crash because machine trace metrics
+; did not correctly ignore debug instructions. The check lines ensure that the
+; machine-combiner pass has run, reassociated the add operands, and therefore
+; used machine trace metrics.
+
+define void @PR24199() {
+; CHECK-LABEL:	PR24199:
+; CHECK:	addss	%xmm1, %xmm0
+; CHECK:	addss	%xmm2, %xmm0
+
+entry:
+  %i = alloca %struct.A, align 8
+  %tobool = icmp ne i32 undef, 0
+  br i1 undef, label %if.end, label %if.then
+
+if.then:
+  br label %if.end
+
+if.end:
+  %h = phi float [ 0.0, %if.then ], [ 4.0, %entry ]
+  call void @foo(%struct.A* nonnull undef)
+  tail call void @llvm.dbg.value(metadata %struct.A* undef, i64 0, metadata !5, metadata !4), !dbg !6
+  tail call void @llvm.dbg.value(metadata float %h, i64 0, metadata !5, metadata !4), !dbg !6
+  %n0 = load float, float* undef, align 4
+  %mul = fmul fast float %n0, %h
+  %add = fadd fast float %mul, 1.0
+  tail call void @llvm.dbg.value(metadata %struct.A* undef, i64 0, metadata !5, metadata !4), !dbg !6
+  tail call void @llvm.dbg.value(metadata float %add, i64 0, metadata !5, metadata !4), !dbg !6
+  %add.i = fadd fast float %add, %n0
+  store float %add.i, float* undef, align 4
+  %n1 = bitcast %struct.A* %i to i8*
+  call void @llvm.lifetime.start(i64 16, i8* %n1)
+  %n2 = load <2 x float>, <2 x float>* undef, align 8
+  %conv = uitofp i1 %tobool to float
+  %bitcast = extractelement <2 x float> %n2, i32 0
+  %factor = fmul fast float %bitcast, 2.0
+  %add3 = fadd fast float %factor, %conv
+  call void @bar(float %add3)
+  ret void
+}
+
+%struct.A = type { float, float }
+
+declare void @bar(float)
+declare void @foo(%struct.A*)
+declare void @llvm.lifetime.start(i64, i8* nocapture)
+declare void @llvm.dbg.value(metadata, i64, metadata, metadata)
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!2}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C_plus_plus, file: !1, isOptimized: true, runtimeVersion: 0, emissionKind: 1)
+!1 = !DIFile(filename: "24199.cpp", directory: "/bin")
+!2 = !{i32 2, !"Debug Info Version", i32 3}
+!3 = !DISubprogram(linkageName: "foo", file: !1, line: 18, isLocal: false, isDefinition: true, scopeLine: 18, function: void (%struct.A*)* @foo)
+!4 = !DIExpression()
+!5 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "this", arg: 1, scope: !3, flags: DIFlagArtificial | DIFlagObjectPointer)
+!6 = !DILocation(line: 0, scope: !3)
+
+
diff --git a/test/CodeGen/X86/masked_gather_scatter.ll b/test/CodeGen/X86/masked_gather_scatter.ll
new file mode 100644
index 00000000000..de16e5ddc06
--- /dev/null
+++ b/test/CodeGen/X86/masked_gather_scatter.ll
@@ -0,0 +1,142 @@
+; RUN: llc -mtriple=x86_64-apple-darwin  -mcpu=knl < %s | FileCheck %s -check-prefix=KNL
+
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+; KNL-LABEL: test1
+; KNL: kxnorw  %k1, %k1, %k1
+; KNL: vgatherdps      (%rdi,%zmm0,4), %zmm1 {%k1}
+define <16 x float> @test1(float* %base, <16 x i32> %ind) {
+
+  %broadcast.splatinsert = insertelement <16 x float*> undef, float* %base, i32 0
+  %broadcast.splat = shufflevector <16 x float*> %broadcast.splatinsert, <16 x float*> undef, <16 x i32> zeroinitializer
+
+  %sext_ind = sext <16 x i32> %ind to <16 x i64>
+  %gep.random = getelementptr float, <16 x float*> %broadcast.splat, <16 x i64> %sext_ind
+  
+  %res = call <16 x float> @llvm.masked.gather.v16f32(<16 x float*> %gep.random, i32 4, <16 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>, <16 x float> undef)
+  ret <16 x float>%res
+}
+
+declare <16 x i32> @llvm.masked.gather.v16i32(<16 x i32*>, i32, <16 x i1>, <16 x i32>)
+declare <16 x float> @llvm.masked.gather.v16f32(<16 x float*>, i32, <16 x i1>, <16 x float>)
+declare <8 x i32> @llvm.masked.gather.v8i32(<8 x i32*> , i32, <8 x i1> , <8 x i32> )
+  
+; KNL-LABEL: test2
+; KNL: kmovw %esi, %k1
+; KNL: vgatherdps      (%rdi,%zmm0,4), %zmm1 {%k1}
+define <16 x float> @test2(float* %base, <16 x i32> %ind, i16 %mask) {
+
+  %broadcast.splatinsert = insertelement <16 x float*> undef, float* %base, i32 0
+  %broadcast.splat = shufflevector <16 x float*> %broadcast.splatinsert, <16 x float*> undef, <16 x i32> zeroinitializer
+
+  %sext_ind = sext <16 x i32> %ind to <16 x i64>
+  %gep.random = getelementptr float, <16 x float*> %broadcast.splat, <16 x i64> %sext_ind
+  %imask = bitcast i16 %mask to <16 x i1>
+  %res = call <16 x float> @llvm.masked.gather.v16f32(<16 x float*> %gep.random, i32 4, <16 x i1> %imask, <16 x float>undef)
+  ret <16 x float> %res
+}
+
+; KNL-LABEL: test3
+; KNL: kmovw %esi, %k1
+; KNL: vpgatherdd      (%rdi,%zmm0,4), %zmm1 {%k1}
+define <16 x i32> @test3(i32* %base, <16 x i32> %ind, i16 %mask) {
+
+  %broadcast.splatinsert = insertelement <16 x i32*> undef, i32* %base, i32 0
+  %broadcast.splat = shufflevector <16 x i32*> %broadcast.splatinsert, <16 x i32*> undef, <16 x i32> zeroinitializer
+
+  %sext_ind = sext <16 x i32> %ind to <16 x i64>
+  %gep.random = getelementptr i32, <16 x i32*> %broadcast.splat, <16 x i64> %sext_ind
+  %imask = bitcast i16 %mask to <16 x i1>
+  %res = call <16 x i32> @llvm.masked.gather.v16i32(<16 x i32*> %gep.random, i32 4, <16 x i1> %imask, <16 x i32>undef)
+  ret <16 x i32> %res
+}
+
+; KNL-LABEL: test4
+; KNL: kmovw %esi, %k1
+; KNL: kmovw
+; KNL: vpgatherdd
+; KNL: vpgatherdd
+
+define <16 x i32> @test4(i32* %base, <16 x i32> %ind, i16 %mask) {
+
+  %broadcast.splatinsert = insertelement <16 x i32*> undef, i32* %base, i32 0
+  %broadcast.splat = shufflevector <16 x i32*> %broadcast.splatinsert, <16 x i32*> undef, <16 x i32> zeroinitializer
+
+  %gep.random = getelementptr i32, <16 x i32*> %broadcast.splat, <16 x i32> %ind
+  %imask = bitcast i16 %mask to <16 x i1>
+  %gt1 = call <16 x i32> @llvm.masked.gather.v16i32(<16 x i32*> %gep.random, i32 4, <16 x i1> %imask, <16 x i32>undef)
+  %gt2 = call <16 x i32> @llvm.masked.gather.v16i32(<16 x i32*> %gep.random, i32 4, <16 x i1> %imask, <16 x i32>%gt1)
+  %res = add <16 x i32> %gt1, %gt2
+  ret <16 x i32> %res
+}
+
+; KNL-LABEL: test5
+; KNL: kmovw %k1, %k2
+; KNL: vpscatterdd {{.*}}%k2
+; KNL: vpscatterdd {{.*}}%k1
+
+define void @test5(i32* %base, <16 x i32> %ind, i16 %mask, <16 x i32>%val) {
+
+  %broadcast.splatinsert = insertelement <16 x i32*> undef, i32* %base, i32 0
+  %broadcast.splat = shufflevector <16 x i32*> %broadcast.splatinsert, <16 x i32*> undef, <16 x i32> zeroinitializer
+
+  %gep.random = getelementptr i32, <16 x i32*> %broadcast.splat, <16 x i32> %ind
+  %imask = bitcast i16 %mask to <16 x i1>
+  call void @llvm.masked.scatter.v16i32(<16 x i32>%val, <16 x i32*> %gep.random, i32 4, <16 x i1> %imask)
+  call void @llvm.masked.scatter.v16i32(<16 x i32>%val, <16 x i32*> %gep.random, i32 4, <16 x i1> %imask)
+  ret void
+}
+
+declare void @llvm.masked.scatter.v8i32(<8 x i32> , <8 x i32*> , i32 , <8 x i1> )
+declare void @llvm.masked.scatter.v16i32(<16 x i32> , <16 x i32*> , i32 , <16 x i1> )
+
+; KNL-LABEL: test6
+; KNL: kxnorw  %k1, %k1, %k1
+; KNL: kxnorw  %k2, %k2, %k2
+; KNL: vpgatherqd      (,%zmm{{.*}}), %ymm{{.*}} {%k2}
+; KNL: vpscatterqd     %ymm{{.*}}, (,%zmm{{.*}}) {%k1}
+define <8 x i32> @test6(<8 x i32>%a1, <8 x i32*> %ptr) {
+
+  %a = call <8 x i32> @llvm.masked.gather.v8i32(<8 x i32*> %ptr, i32 4, <8 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>, <8 x i32> undef)
+
+  call void @llvm.masked.scatter.v8i32(<8 x i32> %a1, <8 x i32*> %ptr, i32 4, <8 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>)
+  ret <8 x i32>%a
+}
+
+; In this case the index should be promoted to <8 x i64> for KNL
+; KNL-LABEL: test7
+; KNL: vpmovsxdq %ymm0, %zmm0
+; KNL: kmovw   %k1, %k2
+; KNL: vpgatherqd {{.*}} {%k2}
+; KNL: vpgatherqd {{.*}} {%k1}
+define <8 x i32> @test7(i32* %base, <8 x i32> %ind, i8 %mask) {
+
+  %broadcast.splatinsert = insertelement <8 x i32*> undef, i32* %base, i32 0
+  %broadcast.splat = shufflevector <8 x i32*> %broadcast.splatinsert, <8 x i32*> undef, <8 x i32> zeroinitializer
+
+  %gep.random = getelementptr i32, <8 x i32*> %broadcast.splat, <8 x i32> %ind
+  %imask = bitcast i8 %mask to <8 x i1>
+  %gt1 = call <8 x i32> @llvm.masked.gather.v8i32(<8 x i32*> %gep.random, i32 4, <8 x i1> %imask, <8 x i32>undef)
+  %gt2 = call <8 x i32> @llvm.masked.gather.v8i32(<8 x i32*> %gep.random, i32 4, <8 x i1> %imask, <8 x i32>%gt1)
+  %res = add <8 x i32> %gt1, %gt2
+  ret <8 x i32> %res
+}
+
+; No uniform base in this case, index <8 x i64> contains addresses,
+; each gather call will be split into two
+; KNL-LABEL: test8
+; KNL: kshiftrw        $8, %k1, %k2
+; KNL: vpgatherqd
+; KNL: vpgatherqd
+; KNL: vinserti64x4
+; KNL: vpgatherqd
+; KNL: vpgatherqd
+; KNL: vinserti64x4
+define <16 x i32> @test8(<16 x i32*> %ptr.random, <16 x i32> %ind, i16 %mask) {
+  %imask = bitcast i16 %mask to <16 x i1>
+  %gt1 = call <16 x i32> @llvm.masked.gather.v16i32(<16 x i32*> %ptr.random, i32 4, <16 x i1> %imask, <16 x i32>undef)
+  %gt2 = call <16 x i32> @llvm.masked.gather.v16i32(<16 x i32*> %ptr.random, i32 4, <16 x i1> %imask, <16 x i32>%gt1)
+  %res = add <16 x i32> %gt1, %gt2
+  ret <16 x i32> %res
+}
diff --git a/test/CodeGen/X86/masked_memop.ll b/test/CodeGen/X86/masked_memop.ll
index da2f78655d1..f51d4fa103e 100644
--- a/test/CodeGen/X86/masked_memop.ll
+++ b/test/CodeGen/X86/masked_memop.ll
@@ -7,8 +7,8 @@
 ; AVX512: vmovdqu32       (%rdi), %zmm0 {%k1} {z}
 
 ; AVX2-LABEL: test1
-; AVX2: vpmaskmovd      32(%rdi)
-; AVX2: vpmaskmovd      (%rdi)
+; AVX2: vpmaskmovd      {{.*}}(%rdi)
+; AVX2: vpmaskmovd      {{.*}}(%rdi)
 ; AVX2-NOT: blend
 
 ; AVX_SCALAR-LABEL: test1
@@ -190,10 +190,13 @@ define void @test14(<2 x i32> %trigger, <2 x float>* %addr, <2 x float> %val) {
 ; AVX2-LABEL: test15
 ; AVX2: vpmaskmovd
 
-; SKX-LABEL: test15
-; SKX: kshiftl
-; SKX: kshiftr
-; SKX: vmovdqu32 {{.*}}{%k1}
+; SKX-LABEL: test15:
+; SKX:       ## BB#0:
+; SKX-NEXT:    vpandq {{.*}}(%rip), %xmm0, %xmm0
+; SKX-NEXT:    vpxor %xmm2, %xmm2, %xmm2
+; SKX-NEXT:    vpcmpeqq %xmm2, %xmm0, %k1
+; SKX-NEXT:    vpmovqd %xmm1, (%rdi) {%k1}
+; SKX-NEXT:    retq
 define void @test15(<2 x i32> %trigger, <2 x i32>* %addr, <2 x i32> %val) {
   %mask = icmp eq <2 x i32> %trigger, zeroinitializer
   call void @llvm.masked.store.v2i32(<2 x i32>%val, <2 x i32>* %addr, i32 4, <2 x i1>%mask)
diff --git a/test/CodeGen/X86/memcpy.ll b/test/CodeGen/X86/memcpy.ll
index f582571252b..00669443d6d 100644
--- a/test/CodeGen/X86/memcpy.ll
+++ b/test/CodeGen/X86/memcpy.ll
@@ -59,6 +59,26 @@ entry:
 ; DARWIN: movq
 }
 
+define void @test3_minsize(i8* nocapture %A, i8* nocapture %B) nounwind minsize noredzone {
+  tail call void @llvm.memcpy.p0i8.p0i8.i64(i8* %A, i8* %B, i64 64, i32 1, i1 false)
+  ret void
+; LINUX-LABEL: test3_minsize:
+; LINUX: memcpy
+
+; DARWIN-LABEL: test3_minsize:
+; DARWIN: memcpy
+}
+
+define void @test3_minsize_optsize(i8* nocapture %A, i8* nocapture %B) nounwind optsize minsize noredzone {
+  tail call void @llvm.memcpy.p0i8.p0i8.i64(i8* %A, i8* %B, i64 64, i32 1, i1 false)
+  ret void
+; LINUX-LABEL: test3_minsize_optsize:
+; LINUX: memcpy
+
+; DARWIN-LABEL: test3_minsize_optsize:
+; DARWIN: memcpy
+}
+
 ; Large constant memcpy's should be inlined when not optimizing for size.
 define void @test4(i8* nocapture %A, i8* nocapture %B) nounwind noredzone {
 entry:
diff --git a/test/CodeGen/X86/merge-consecutive-stores-i1.ll b/test/CodeGen/X86/merge-consecutive-stores-i1.ll
new file mode 100644
index 00000000000..a7f5c214227
--- /dev/null
+++ b/test/CodeGen/X86/merge-consecutive-stores-i1.ll
@@ -0,0 +1,15 @@
+; RUN: llc  -march=x86-64 < %s
+
+; Ensure that MergeConsecutiveStores doesn't crash when dealing with
+; i1 operands.
+
+%struct.X = type { i1, i1 }
+
+@b = common global %struct.X zeroinitializer, align 4
+
+define void @foo() {
+entry:
+  store i1 0, i1* getelementptr inbounds (%struct.X, %struct.X* @b, i64 0, i32 0), align 4
+  store i1 0, i1* getelementptr inbounds (%struct.X, %struct.X* @b, i64 0, i32 1), align 1
+  ret void
+}
diff --git a/test/CodeGen/X86/misched-code-difference-with-debug.ll b/test/CodeGen/X86/misched-code-difference-with-debug.ll
index 2cc70e1134c..0f1f382c49a 100644
--- a/test/CodeGen/X86/misched-code-difference-with-debug.ll
+++ b/test/CodeGen/X86/misched-code-difference-with-debug.ll
@@ -47,12 +47,12 @@ define void @test_with_debug() {
 entry:
   %c = alloca %class.C, align 1
   %0 = load i8, i8* @argc, align 1
-  tail call void @llvm.dbg.value(metadata i8 %0, i64 0, metadata !19, metadata !29), !dbg !MDLocation(scope: !13)
+  tail call void @llvm.dbg.value(metadata i8 %0, i64 0, metadata !19, metadata !29), !dbg !DILocation(scope: !13)
   %conv = sext i8 %0 to i32
-  tail call void @llvm.dbg.value(metadata %class.C* %c, i64 0, metadata !18, metadata !29), !dbg !MDLocation(scope: !13)
+  tail call void @llvm.dbg.value(metadata %class.C* %c, i64 0, metadata !18, metadata !29), !dbg !DILocation(scope: !13)
   %call = call i32 (%class.C*, i8, i8, i8, ...) @test_function(%class.C* %c, i8 signext 0, i8 signext %0, i8 signext 0, i32 %conv)
   %1 = load i8, i8* @argc, align 1
-  call void @llvm.dbg.value(metadata %class.C* %c, i64 0, metadata !18, metadata !29), !dbg !MDLocation(scope: !13)
+  call void @llvm.dbg.value(metadata %class.C* %c, i64 0, metadata !18, metadata !29), !dbg !DILocation(scope: !13)
   %call2 = call i32 (%class.C*, i8, i8, i8, ...) @test_function(%class.C* %c, i8 signext 0, i8 signext %1, i8 signext 0, i32 %conv)
   ret void
 }
@@ -62,29 +62,29 @@ declare void @llvm.dbg.value(metadata, i64, metadata, metadata)
 !llvm.dbg.cu = !{!0}
 !llvm.module.flags = !{!22, !23}
 
-!0 = !MDCompileUnit(language: DW_LANG_C_plus_plus, file: !1, enums: !2, retainedTypes: !3, subprograms: !12, globals: !20, imports: !2)
-!1 = !MDFile(filename: "test.cpp", directory: "")
+!0 = !DICompileUnit(language: DW_LANG_C_plus_plus, file: !1, enums: !2, retainedTypes: !3, subprograms: !12, globals: !20, imports: !2)
+!1 = !DIFile(filename: "test.cpp", directory: "")
 !2 = !{}
 !3 = !{!4}
-!4 = !MDCompositeType(tag: DW_TAG_class_type, name: "C", line: 2, size: 8, align: 8, file: !1, elements: !5, identifier: "_ZTS1C")
+!4 = !DICompositeType(tag: DW_TAG_class_type, name: "C", line: 2, size: 8, align: 8, file: !1, elements: !5, identifier: "_ZTS1C")
 !5 = !{!6}
-!6 = !MDSubprogram(name: "test", file: !1, scope: !"_ZTS1C", type: !7)
-!7 = !MDSubroutineType(types: !8)
+!6 = !DISubprogram(name: "test", file: !1, scope: !"_ZTS1C", type: !7)
+!7 = !DISubroutineType(types: !8)
 !8 = !{!9, !10, !11, !11, !11, null}
-!9 = !MDBasicType(encoding: DW_ATE_signed, size: 32, align: 32, name: "int")
-!10 = !MDDerivedType(baseType: !"_ZTS1C", tag: DW_TAG_pointer_type, size: 64, align: 64, flags: DIFlagArtificial)
-!11 = !MDBasicType(tag: DW_TAG_base_type, name: "char", size: 8, align: 8, encoding: DW_ATE_signed_char)
+!9 = !DIBasicType(encoding: DW_ATE_signed, size: 32, align: 32, name: "int")
+!10 = !DIDerivedType(baseType: !"_ZTS1C", tag: DW_TAG_pointer_type, size: 64, align: 64, flags: DIFlagArtificial)
+!11 = !DIBasicType(tag: DW_TAG_base_type, name: "char", size: 8, align: 8, encoding: DW_ATE_signed_char)
 !12 = !{!13}
-!13 = !MDSubprogram(name: "test_with_debug", linkageName: "test_with_debug", line: 6, isLocal: false, isDefinition: true, flags: DIFlagPrototyped, isOptimized: true, scopeLine: 6, file: !1, scope: !14, type: !15, function: void ()* @test_with_debug, variables: !17)
-!14 = !MDFile(filename: "test.cpp", directory: "")
-!15 = !MDSubroutineType(types: !16)
+!13 = !DISubprogram(name: "test_with_debug", linkageName: "test_with_debug", line: 6, isLocal: false, isDefinition: true, flags: DIFlagPrototyped, isOptimized: true, scopeLine: 6, file: !1, scope: !14, type: !15, function: void ()* @test_with_debug, variables: !17)
+!14 = !DIFile(filename: "test.cpp", directory: "")
+!15 = !DISubroutineType(types: !16)
 !16 = !{null}
 !17 = !{!18, !19}
-!18 = !MDLocalVariable(tag: DW_TAG_auto_variable, name: "c", line: 7, scope: !13, file: !14, type: !"_ZTS1C")
-!19 = !MDLocalVariable(tag: DW_TAG_auto_variable, name: "lc", line: 8, scope: !13, file: !14, type: !11)
+!18 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "c", line: 7, scope: !13, file: !14, type: !"_ZTS1C")
+!19 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "lc", line: 8, scope: !13, file: !14, type: !11)
 !20 = !{!21}
-!21 = !MDGlobalVariable(name: "argc", line: 1, isLocal: false, isDefinition: true, scope: null, file: !14, type: !11, variable: i8* @argc)
+!21 = !DIGlobalVariable(name: "argc", line: 1, isLocal: false, isDefinition: true, scope: null, file: !14, type: !11, variable: i8* @argc)
 !22 = !{i32 2, !"Dwarf Version", i32 4}
 !23 = !{i32 2, !"Debug Info Version", i32 3}
-!25 = !MDLocation(line: 8, column: 3, scope: !13)
-!29 = !MDExpression()
+!25 = !DILocation(line: 8, column: 3, scope: !13)
+!29 = !DIExpression()
diff --git a/test/CodeGen/X86/movtopush.ll b/test/CodeGen/X86/movtopush.ll
index f89e52457f3..b02f9ec45e7 100644
--- a/test/CodeGen/X86/movtopush.ll
+++ b/test/CodeGen/X86/movtopush.ll
@@ -2,11 +2,15 @@
 ; RUN: llc < %s -mtriple=x86_64-windows | FileCheck %s -check-prefix=X64
 ; RUN: llc < %s -mtriple=i686-windows -force-align-stack -stack-alignment=32 | FileCheck %s -check-prefix=ALIGNED 
 
+%class.Class = type { i32 }
+%struct.s = type { i64 }
+
 declare void @good(i32 %a, i32 %b, i32 %c, i32 %d)
 declare void @inreg(i32 %a, i32 inreg %b, i32 %c, i32 %d)
+declare x86_thiscallcc void @thiscall(%class.Class* %class, i32 %a, i32 %b, i32 %c, i32 %d)
 declare void @oneparam(i32 %a)
 declare void @eightparams(i32 %a, i32 %b, i32 %c, i32 %d, i32 %e, i32 %f, i32 %g, i32 %h)
-
+declare void @struct(%struct.s* byval %a, i32 %b, i32 %c, i32 %d)
 
 ; Here, we should have a reserved frame, so we don't expect pushes
 ; NORMAL-LABEL: test1:
@@ -108,13 +112,12 @@ entry:
   ret void
 }
 
-; We don't support weird calling conventions
+; We support weird calling conventions
 ; NORMAL-LABEL: test4:
-; NORMAL: subl    $12, %esp
-; NORMAL-NEXT: movl    $4, 8(%esp)
-; NORMAL-NEXT: movl    $3, 4(%esp)
-; NORMAL-NEXT: movl    $1, (%esp)
-; NORMAL-NEXT: movl    $2, %eax
+; NORMAL: movl    $2, %eax
+; NORMAL-NEXT: pushl   $4
+; NORMAL-NEXT: pushl   $3
+; NORMAL-NEXT: pushl   $1
 ; NORMAL-NEXT: call
 ; NORMAL-NEXT: addl $12, %esp
 define void @test4() optsize {
@@ -123,6 +126,20 @@ entry:
   ret void
 }
 
+; NORMAL-LABEL: test4b:
+; NORMAL: movl 4(%esp), %ecx
+; NORMAL-NEXT: pushl   $4
+; NORMAL-NEXT: pushl   $3
+; NORMAL-NEXT: pushl   $2
+; NORMAL-NEXT: pushl   $1
+; NORMAL-NEXT: call
+; NORMAL-NEXT: ret
+define void @test4b(%class.Class* %f) optsize {
+entry:
+  call x86_thiscallcc void @thiscall(%class.Class* %f, i32 1, i32 2, i32 3, i32 4)
+  ret void
+}
+
 ; When there is no reserved call frame, check that additional alignment
 ; is added when the pushes don't add up to the required alignment.
 ; ALIGNED-LABEL: test5:
@@ -229,20 +246,27 @@ entry:
 ; NORMAL-NEXT: pushl $1
 ; NORMAL-NEXT: call
 ; NORMAL-NEXT: addl $16, %esp
-; NORMAL-NEXT: subl $16, %esp
-; NORMAL-NEXT: leal 16(%esp), [[EAX:%e..]]
-; NORMAL-NEXT: movl    [[EAX]], 12(%esp)
-; NORMAL-NEXT: movl    $7, 8(%esp)
-; NORMAL-NEXT: movl    $6, 4(%esp)
-; NORMAL-NEXT: movl    $5, (%esp)
+; NORMAL-NEXT: subl $20, %esp
+; NORMAL-NEXT: movl 20(%esp), [[E1:%e..]]
+; NORMAL-NEXT: movl 24(%esp), [[E2:%e..]]
+; NORMAL-NEXT: movl    [[E2]], 4(%esp)
+; NORMAL-NEXT: movl    [[E1]], (%esp)
+; NORMAL-NEXT: leal 32(%esp), [[E3:%e..]]
+; NORMAL-NEXT: movl    [[E3]], 16(%esp)
+; NORMAL-NEXT: leal 28(%esp), [[E4:%e..]]
+; NORMAL-NEXT: movl    [[E4]], 12(%esp)
+; NORMAL-NEXT: movl    $6, 8(%esp)
 ; NORMAL-NEXT: call
-; NORMAL-NEXT: addl $16, %esp
+; NORMAL-NEXT: addl $20, %esp
 define void @test9() optsize {
 entry:
   %p = alloca i32, align 4
+  %q = alloca i32, align 4
+  %s = alloca %struct.s, align 4  
   call void @good(i32 1, i32 2, i32 3, i32 4)
-  %0 = ptrtoint i32* %p to i32
-  call void @good(i32 5, i32 6, i32 7, i32 %0)
+  %pv = ptrtoint i32* %p to i32
+  %qv = ptrtoint i32* %q to i32
+  call void @struct(%struct.s* byval %s, i32 6, i32 %qv, i32 %pv)
   ret void
 }
 
@@ -291,28 +315,17 @@ define void @test11() optsize {
 ; Converting one mov into a push isn't worth it when 
 ; doing so forces too much overhead for other calls.
 ; NORMAL-LABEL: test12:
-; NORMAL: subl    $16, %esp
-; NORMAL-NEXT: movl    $4, 8(%esp)
-; NORMAL-NEXT: movl    $3, 4(%esp)
-; NORMAL-NEXT: movl    $1, (%esp)
-; NORMAL-NEXT: movl    $2, %eax
-; NORMAL-NEXT: calll _inreg
-; NORMAL-NEXT: movl    $8, 12(%esp)
+; NORMAL: movl    $8, 12(%esp)
 ; NORMAL-NEXT: movl    $7, 8(%esp)
 ; NORMAL-NEXT: movl    $6, 4(%esp)
 ; NORMAL-NEXT: movl    $5, (%esp)
 ; NORMAL-NEXT: calll _good
-; NORMAL-NEXT: movl    $12, 8(%esp)
-; NORMAL-NEXT: movl    $11, 4(%esp)
-; NORMAL-NEXT: movl    $9, (%esp)
-; NORMAL-NEXT: movl    $10, %eax
-; NORMAL-NEXT: calll _inreg
-; NORMAL-NEXT: addl $16, %esp
 define void @test12() optsize {
 entry:
-  call void @inreg(i32 1, i32 2, i32 3, i32 4)
+  %s = alloca %struct.s, align 4  
+  call void @struct(%struct.s* %s, i32 2, i32 3, i32 4)
   call void @good(i32 5, i32 6, i32 7, i32 8)
-  call void @inreg(i32 9, i32 10, i32 11, i32 12)
+  call void @struct(%struct.s* %s, i32 10, i32 11, i32 12)
   ret void
 }
 
@@ -324,13 +337,12 @@ entry:
 ; NORMAL-NEXT: pushl    $1
 ; NORMAL-NEXT: calll _good
 ; NORMAL-NEXT: addl    $16, %esp
-; NORMAL-NEXT: subl    $12, %esp
-; NORMAL-NEXT: movl    $8, 8(%esp)
-; NORMAL-NEXT: movl    $7, 4(%esp)
-; NORMAL-NEXT: movl    $5, (%esp)
-; NORMAL-NEXT: movl    $6, %eax
-; NORMAL-NEXT: calll _inreg
-; NORMAL-NEXT: addl    $12, %esp
+; NORMAL-NEXT: subl    $20, %esp
+; NORMAL: movl    $8, 16(%esp)
+; NORMAL-NEXT: movl    $7, 12(%esp)
+; NORMAL-NEXT: movl    $6, 8(%esp)
+; NORMAL-NEXT: calll _struct
+; NORMAL-NEXT: addl    $20, %esp
 ; NORMAL-NEXT: pushl    $12
 ; NORMAL-NEXT: pushl    $11
 ; NORMAL-NEXT: pushl    $10
@@ -339,8 +351,9 @@ entry:
 ; NORMAL-NEXT: addl $16, %esp
 define void @test12b() optsize {
 entry:
-  call void @good(i32 1, i32 2, i32 3, i32 4)
-  call void @inreg(i32 5, i32 6, i32 7, i32 8)
+  %s = alloca %struct.s, align 4  
+  call void @good(i32 1, i32 2, i32 3, i32 4)  
+  call void @struct(%struct.s* %s, i32 6, i32 7, i32 8)
   call void @good(i32 9, i32 10, i32 11, i32 12)
   ret void
 }
diff --git a/test/CodeGen/X86/null-streamer.ll b/test/CodeGen/X86/null-streamer.ll
index c7be651a09c..3f5abfd40f2 100644
--- a/test/CodeGen/X86/null-streamer.ll
+++ b/test/CodeGen/X86/null-streamer.ll
@@ -14,15 +14,15 @@ define void @f1() {
 !llvm.dbg.cu = !{!0}
 !llvm.module.flags = !{!11, !13}
 
-!0 = !MDCompileUnit(language: DW_LANG_C_plus_plus, producer: " ", isOptimized: true, emissionKind: 0, file: !1, enums: !2, retainedTypes: !2, subprograms: !3, globals: !9, imports: !2)
-!1 = !MDFile(filename: "file.c", directory: "")
+!0 = !DICompileUnit(language: DW_LANG_C_plus_plus, producer: " ", isOptimized: true, emissionKind: 0, file: !1, enums: !2, retainedTypes: !2, subprograms: !3, globals: !9, imports: !2)
+!1 = !DIFile(filename: "file.c", directory: "")
 !2 = !{}
 !3 = !{!4}
-!4 = !MDSubprogram(name: "", line: 2, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: true, scopeLine: 2, file: !1, scope: !1, type: !6, function: i32 ()* null, variables: !2)
-!6 = !MDSubroutineType(types: !7)
+!4 = !DISubprogram(name: "", line: 2, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: true, scopeLine: 2, file: !1, scope: !1, type: !6, function: i32 ()* null, variables: !2)
+!6 = !DISubroutineType(types: !7)
 !7 = !{!8}
-!8 = !MDBasicType(tag: DW_TAG_base_type, size: 32, align: 32, encoding: DW_ATE_signed)
+!8 = !DIBasicType(tag: DW_TAG_base_type, size: 32, align: 32, encoding: DW_ATE_signed)
 !9 = !{!10}
-!10 = !MDGlobalVariable(name: "i", linkageName: "_ZL1i", line: 1, isLocal: true, isDefinition: true, scope: null, file: !1, type: !8)
+!10 = !DIGlobalVariable(name: "i", linkageName: "_ZL1i", line: 1, isLocal: true, isDefinition: true, scope: null, file: !1, type: !8)
 !11 = !{i32 2, !"Dwarf Version", i32 3}
 !13 = !{i32 1, !"Debug Info Version", i32 3}
diff --git a/test/CodeGen/X86/or-branch.ll b/test/CodeGen/X86/or-branch.ll
index ae3ed3f8344..9db948adb46 100644
--- a/test/CodeGen/X86/or-branch.ll
+++ b/test/CodeGen/X86/or-branch.ll
@@ -1,19 +1,28 @@
-; RUN: llc < %s -march=x86  | not grep set
+; RUN: llc < %s -mtriple=i386-unknown-unknown -jump-is-expensive=0 | FileCheck %s --check-prefix=JUMP2
+; RUN: llc < %s -mtriple=i386-unknown-unknown -jump-is-expensive=1 | FileCheck %s --check-prefix=JUMP1
 
 define void @foo(i32 %X, i32 %Y, i32 %Z) nounwind {
+; JUMP2-LABEL: foo:
+; JUMP2-DAG:     jl
+; JUMP2-DAG:     je
+;
+; JUMP1-LABEL: foo:
+; JUMP1-DAG:     sete
+; JUMP1-DAG:     setl
+; JUMP1:         orb
+; JUMP1:         jne
 entry:
-	%tmp = tail call i32 (...) @bar( )		; <i32> [#uses=0]
-	%tmp.upgrd.1 = icmp eq i32 %X, 0		; <i1> [#uses=1]
-	%tmp3 = icmp slt i32 %Y, 5		; <i1> [#uses=1]
-	%tmp4 = or i1 %tmp3, %tmp.upgrd.1		; <i1> [#uses=1]
-	br i1 %tmp4, label %cond_true, label %UnifiedReturnBlock
+  %tmp1 = icmp eq i32 %X, 0
+  %tmp3 = icmp slt i32 %Y, 5
+  %tmp4 = or i1 %tmp3, %tmp1
+  br i1 %tmp4, label %cond_true, label %UnifiedReturnBlock
 
-cond_true:		; preds = %entry
-	%tmp5 = tail call i32 (...) @bar( )		; <i32> [#uses=0]
-	ret void
+cond_true:
+  %tmp5 = tail call i32 (...) @bar( )
+  ret void
 
-UnifiedReturnBlock:		; preds = %entry
-	ret void
+UnifiedReturnBlock:
+  ret void
 }
 
 declare i32 @bar(...)
diff --git a/test/CodeGen/X86/patchpoint-invoke.ll b/test/CodeGen/X86/patchpoint-invoke.ll
index 8123f945f3f..b7f198d960a 100644
--- a/test/CodeGen/X86/patchpoint-invoke.ll
+++ b/test/CodeGen/X86/patchpoint-invoke.ll
@@ -2,7 +2,7 @@
 
 ; Test invoking of patchpoints
 ;
-define i64 @patchpoint_invoke(i64 %p1, i64 %p2) {
+define i64 @patchpoint_invoke(i64 %p1, i64 %p2) personality i8* bitcast (i32 (...)* @__gxx_personality_v0 to i8*) {
 entry:
 ; CHECK-LABEL: patchpoint_invoke:
 ; CHECK-NEXT:  [[FUNC_BEGIN:.L.*]]:
@@ -18,14 +18,14 @@ entry:
 ; CHECK-NEXT: [[PP_END:.L.*]]:
 ; CHECK:      ret
   %resolveCall = inttoptr i64 -559038736 to i8*
-  %result = invoke i64 (i64, i32, i8*, i32, ...)* @llvm.experimental.patchpoint.i64(i64 2, i32 15, i8* %resolveCall, i32 1, i64 %p1, i64 %p2)
+  %result = invoke i64 (i64, i32, i8*, i32, ...) @llvm.experimental.patchpoint.i64(i64 2, i32 15, i8* %resolveCall, i32 1, i64 %p1, i64 %p2)
             to label %success unwind label %threw
 
 success:
   ret i64 %result
 
 threw:
-  %0 = landingpad { i8*, i32 } personality i8* bitcast (i32 (...)* @__gxx_personality_v0 to i8*)
+  %0 = landingpad { i8*, i32 }
           catch i8* null
   ret i64 0
 }
diff --git a/test/CodeGen/X86/patchpoint.ll b/test/CodeGen/X86/patchpoint.ll
index eda13fd1e24..82b15c36c2b 100644
--- a/test/CodeGen/X86/patchpoint.ll
+++ b/test/CodeGen/X86/patchpoint.ll
@@ -21,6 +21,22 @@ entry:
   ret i64 %result
 }
 
+; Trivial symbolic patchpoint codegen.
+;
+
+declare i64 @foo(i64 %p1, i64 %p2)
+define i64 @trivial_symbolic_patchpoint_codegen(i64 %p1, i64 %p2) {
+entry:
+; CHECK-LABEL: trivial_symbolic_patchpoint_codegen:
+; CHECK:       movabsq $_foo, %r11
+; CHECK-NEXT:  callq   *%r11
+; CHECK-NEXT:  xchgw   %ax, %ax
+; CHECK:       retq
+  %result = tail call i64 (i64, i32, i8*, i32, ...) @llvm.experimental.patchpoint.i64(i64 9, i32 15, i8* bitcast (i64 (i64, i64)* @foo to i8*), i32 2, i64 %p1, i64 %p2)
+  ret i64 %result
+}
+
+
 ; Caller frame metadata with stackmaps. This should not be optimized
 ; as a leaf function.
 ;
diff --git a/test/CodeGen/X86/personality.ll b/test/CodeGen/X86/personality.ll
index 424a30734f0..53162ebc868 100644
--- a/test/CodeGen/X86/personality.ll
+++ b/test/CodeGen/X86/personality.ll
@@ -2,13 +2,13 @@
 ; RUN: llc < %s -mtriple=i386-apple-darwin9 | FileCheck %s -check-prefix=X32
 ; PR1632
 
-define void @_Z1fv() {
+define void @_Z1fv() personality i32 (...)* @__gxx_personality_v0 {
 entry:
   invoke void @_Z1gv()
           to label %return unwind label %unwind
 
 unwind:                                           ; preds = %entry
-  %exn = landingpad {i8*, i32} personality i32 (...)* @__gxx_personality_v0
+  %exn = landingpad {i8*, i32}
             cleanup
   br i1 false, label %eh_then, label %cleanup20
 
@@ -17,7 +17,7 @@ eh_then:                                          ; preds = %unwind
           to label %return unwind label %unwind10
 
 unwind10:                                         ; preds = %eh_then
-  %exn10 = landingpad {i8*, i32} personality i32 (...)* @__gxx_personality_v0
+  %exn10 = landingpad {i8*, i32}
             cleanup
   %upgraded.eh_select13 = extractvalue { i8*, i32 } %exn10, 1
   %upgraded.eh_select131 = sext i32 %upgraded.eh_select13 to i64
@@ -41,8 +41,10 @@ declare void @__cxa_end_catch()
 
 declare i32 @__gxx_personality_v0(...)
 
+; X64-NOT: .quad ___gxx_personality_v0
 ; X64: .cfi_personality 155, ___gxx_personality_v0
 
+; X32-NOT: .long ___gxx_personality_v0
 ; X32: .cfi_personality 155, L___gxx_personality_v0$non_lazy_ptr
 
 ; X32:        .section	__IMPORT,__pointers,non_lazy_symbol_pointers
diff --git a/test/CodeGen/X86/personality_size.ll b/test/CodeGen/X86/personality_size.ll
index 79d131b82b2..41f1ac8cad6 100644
--- a/test/CodeGen/X86/personality_size.ll
+++ b/test/CodeGen/X86/personality_size.ll
@@ -2,13 +2,13 @@
 ; RUN: llc < %s -relocation-model=pic -mtriple=i386-pc-solaris2.11 | FileCheck %s -check-prefix=X32
 ; PR1632
 
-define void @_Z1fv() {
+define void @_Z1fv() personality i32 (...)* @__gxx_personality_v0 {
 entry:
   invoke void @_Z1gv()
           to label %return unwind label %unwind
 
 unwind:                                           ; preds = %entry
-  %exn = landingpad {i8*, i32} personality i32 (...)* @__gxx_personality_v0
+  %exn = landingpad {i8*, i32}
             cleanup
   ret void
 
diff --git a/test/CodeGen/X86/pic.ll b/test/CodeGen/X86/pic.ll
index d543deb804d..73be234db81 100644
--- a/test/CodeGen/X86/pic.ll
+++ b/test/CodeGen/X86/pic.ll
@@ -196,9 +196,11 @@ bb12:
 ; LINUX-NEXT: .LJTI7_0:
 ; LINUX:   .long	 .LBB7_2@GOTOFF
 ; LINUX:   .long	 .LBB7_8@GOTOFF
-; LINUX:   .long	 .LBB7_14@GOTOFF
-; LINUX:   .long	 .LBB7_9@GOTOFF
-; LINUX:   .long	 .LBB7_10@GOTOFF
+; LINUX:   .long	 .LBB7_4@GOTOFF
+; LINUX:   .long	 .LBB7_6@GOTOFF
+; LINUX:   .long	 .LBB7_5@GOTOFF
+; LINUX:   .long	 .LBB7_8@GOTOFF
+; LINUX:   .long	 .LBB7_7@GOTOFF
 }
 
 declare void @foo1(...)
diff --git a/test/CodeGen/X86/pic_jumptable.ll b/test/CodeGen/X86/pic_jumptable.ll
index dd0a8ac0431..8c1992a24ec 100644
--- a/test/CodeGen/X86/pic_jumptable.ll
+++ b/test/CodeGen/X86/pic_jumptable.ll
@@ -55,13 +55,15 @@ entry:
 	]
 
 bb:		; preds = %entry, %entry, %entry, %entry, %entry, %entry, %entry, %entry, %entry, %entry
+	call void @_Z3bari( i32 0 )
 	br label %bb1
 
 bb1:		; preds = %bb, %entry
+	call void @_Z3bari( i32 1 )
 	br label %bb2
 
 bb2:		; preds = %bb1, %entry
-	call void @_Z3bari( i32 1 )
+	call void @_Z3bari( i32 2 )
 	br label %bb11
 
 bb3:		; preds = %entry
diff --git a/test/CodeGen/X86/pmul.ll b/test/CodeGen/X86/pmul.ll
index 6bfa656b609..dbe5bd646c7 100644
--- a/test/CodeGen/X86/pmul.ll
+++ b/test/CodeGen/X86/pmul.ll
@@ -1,5 +1,53 @@
 ; RUN: llc < %s -mtriple=x86_64-unknown-unknown | FileCheck %s --check-prefix=ALL --check-prefix=SSE2
 ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=sse4.1 | FileCheck %s --check-prefix=ALL --check-prefix=SSE41
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mcpu=core-avx2 | FileCheck %s --check-prefix=AVX2
+
+define <16 x i8> @mul8c(<16 x i8> %i) nounwind  {
+; SSE2-LABEL: mul8c:
+; SSE2:       # BB#0: # %entry
+; SSE2-NEXT:    movdqa {{.*#+}} xmm1 = [117,117,117,117,117,117,117,117,117,117,117,117,117,117,117,117]
+; SSE2-NEXT:    psraw $8, %xmm1
+; SSE2-NEXT:    movdqa %xmm0, %xmm2
+; SSE2-NEXT:    punpckhbw {{.*#+}} xmm2 = xmm2[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
+; SSE2-NEXT:    psraw $8, %xmm2
+; SSE2-NEXT:    pmullw %xmm1, %xmm2
+; SSE2-NEXT:    movdqa {{.*#+}} xmm3 = [255,255,255,255,255,255,255,255]
+; SSE2-NEXT:    pand %xmm3, %xmm2
+; SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
+; SSE2-NEXT:    psraw $8, %xmm0
+; SSE2-NEXT:    pmullw %xmm1, %xmm0
+; SSE2-NEXT:    pand %xmm3, %xmm0
+; SSE2-NEXT:    packuswb %xmm2, %xmm0
+; SSE2-NEXT:    retq
+;
+; SSE41-LABEL: mul8c:
+; SSE41:       # BB#0: # %entry
+; SSE41-NEXT:    pmovsxbw %xmm0, %xmm1
+; SSE41-NEXT:    pmovsxbw {{.*}}(%rip), %xmm2
+; SSE41-NEXT:    pmullw %xmm2, %xmm1
+; SSE41-NEXT:    movdqa {{.*#+}} xmm3 = [255,255,255,255,255,255,255,255]
+; SSE41-NEXT:    pand %xmm3, %xmm1
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
+; SSE41-NEXT:    pmovsxbw %xmm0, %xmm0
+; SSE41-NEXT:    pmullw %xmm2, %xmm0
+; SSE41-NEXT:    pand %xmm3, %xmm0
+; SSE41-NEXT:    packuswb %xmm0, %xmm1
+; SSE41-NEXT:    movdqa %xmm1, %xmm0
+; SSE41-NEXT:    retq
+entry:
+  %A = mul <16 x i8> %i, < i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117 >
+  ret <16 x i8> %A
+}
+
+define <8 x i16> @mul16c(<8 x i16> %i) nounwind  {
+; ALL-LABEL: mul16c:
+; ALL:       # BB#0: # %entry
+; ALL-NEXT:    pmullw {{.*}}(%rip), %xmm0
+; ALL-NEXT:    retq
+entry:
+  %A = mul <8 x i16> %i, < i16 117, i16 117, i16 117, i16 117, i16 117, i16 117, i16 117, i16 117 >
+  ret <8 x i16> %A
+}
 
 define <4 x i32> @a(<4 x i32> %i) nounwind  {
 ; SSE2-LABEL: a:
@@ -28,10 +76,6 @@ define <2 x i64> @b(<2 x i64> %i) nounwind  {
 ; ALL-NEXT:    movdqa {{.*#+}} xmm1 = [117,117]
 ; ALL-NEXT:    movdqa %xmm0, %xmm2
 ; ALL-NEXT:    pmuludq %xmm1, %xmm2
-; ALL-NEXT:    pxor %xmm3, %xmm3
-; ALL-NEXT:    pmuludq %xmm0, %xmm3
-; ALL-NEXT:    psllq $32, %xmm3
-; ALL-NEXT:    paddq %xmm3, %xmm2
 ; ALL-NEXT:    psrlq $32, %xmm0
 ; ALL-NEXT:    pmuludq %xmm1, %xmm0
 ; ALL-NEXT:    psllq $32, %xmm0
@@ -42,6 +86,58 @@ entry:
   ret <2 x i64> %A
 }
 
+define <16 x i8> @mul8(<16 x i8> %i, <16 x i8> %j) nounwind  {
+; SSE2-LABEL: mul8:
+; SSE2:       # BB#0: # %entry
+; SSE2-NEXT:    movdqa %xmm1, %xmm2
+; SSE2-NEXT:    punpckhbw {{.*#+}} xmm2 = xmm2[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
+; SSE2-NEXT:    psraw $8, %xmm2
+; SSE2-NEXT:    movdqa %xmm0, %xmm3
+; SSE2-NEXT:    punpckhbw {{.*#+}} xmm3 = xmm3[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
+; SSE2-NEXT:    psraw $8, %xmm3
+; SSE2-NEXT:    pmullw %xmm2, %xmm3
+; SSE2-NEXT:    movdqa {{.*#+}} xmm2 = [255,255,255,255,255,255,255,255]
+; SSE2-NEXT:    pand %xmm2, %xmm3
+; SSE2-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
+; SSE2-NEXT:    psraw $8, %xmm1
+; SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
+; SSE2-NEXT:    psraw $8, %xmm0
+; SSE2-NEXT:    pmullw %xmm1, %xmm0
+; SSE2-NEXT:    pand %xmm2, %xmm0
+; SSE2-NEXT:    packuswb %xmm3, %xmm0
+; SSE2-NEXT:    retq
+;
+; SSE41-LABEL: mul8:
+; SSE41:       # BB#0: # %entry
+; SSE41-NEXT:    pmovsxbw %xmm1, %xmm3
+; SSE41-NEXT:    pmovsxbw %xmm0, %xmm2
+; SSE41-NEXT:    pmullw %xmm3, %xmm2
+; SSE41-NEXT:    movdqa {{.*#+}} xmm3 = [255,255,255,255,255,255,255,255]
+; SSE41-NEXT:    pand %xmm3, %xmm2
+; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[2,3,0,1]
+; SSE41-NEXT:    pmovsxbw %xmm1, %xmm1
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
+; SSE41-NEXT:    pmovsxbw %xmm0, %xmm0
+; SSE41-NEXT:    pmullw %xmm1, %xmm0
+; SSE41-NEXT:    pand %xmm3, %xmm0
+; SSE41-NEXT:    packuswb %xmm0, %xmm2
+; SSE41-NEXT:    movdqa %xmm2, %xmm0
+; SSE41-NEXT:    retq
+entry:
+  %A = mul <16 x i8> %i, %j
+  ret <16 x i8> %A
+}
+
+define <8 x i16> @mul16(<8 x i16> %i, <8 x i16> %j) nounwind  {
+; ALL-LABEL: mul16:
+; ALL:       # BB#0: # %entry
+; ALL-NEXT:    pmullw %xmm1, %xmm0
+; ALL-NEXT:    retq
+entry:
+  %A = mul <8 x i16> %i, %j
+  ret <8 x i16> %A
+}
+
 define <4 x i32> @c(<4 x i32> %i, <4 x i32> %j) nounwind  {
 ; SSE2-LABEL: c:
 ; SSE2:       # BB#0: # %entry
@@ -149,3 +245,35 @@ entry:
   %A = mul <2 x i64> %i, %j
   ret <2 x i64> %A
 }
+
+define <4 x i64> @b1(<4 x i64> %i) nounwind  {
+; AVX2-LABEL: @b1
+; AVX2: vpbroadcastq
+; AVX2-NEXT: vpmuludq
+; AVX2-NEXT: vpsrlq  $32 
+; AVX2-NEXT: vpmuludq
+; AVX2-NEXT: vpsllq  $32
+; AVX2-NEXT: vpaddq
+; AVX2-NEXT: retq
+entry:
+  %A = mul <4 x i64> %i, < i64 117, i64 117, i64 117, i64 117 >
+  ret <4 x i64> %A
+}
+
+define <4 x i64> @b2(<4 x i64> %i, <4 x i64> %j) nounwind  {
+; AVX2-LABEL: @b2
+; AVX2:  vpmuludq
+; AVX2-NEXT: vpsrlq  $32
+; AVX2-NEXT: vpmuludq
+; AVX2-NEXT: vpsllq  $32
+; AVX2-NEXT: vpaddq
+; AVX2-NEXT: vpsrlq  $32
+; AVX2-NEXT: vpmuludq
+; AVX2-NEXT: vpsllq  $32
+; AVX2-NEXT: vpaddq
+; AVX2-NEXT: retq
+entry:
+  %A = mul <4 x i64> %i, %j
+  ret <4 x i64> %A
+}
+
diff --git a/test/CodeGen/X86/pr13577.ll b/test/CodeGen/X86/pr13577.ll
index faaec262cb9..a6b721a7a6f 100644
--- a/test/CodeGen/X86/pr13577.ll
+++ b/test/CodeGen/X86/pr13577.ll
@@ -1,5 +1,20 @@
-; RUN: llc < %s -march=x86-64
+; RUN: llc < %s -mtriple=x86_64-darwin | FileCheck %s
 
+; CHECK-LABEL: LCPI0_0:
+; CHECK-NEXT: .long 4286578688
+; CHECK-LABEL: LCPI0_1:
+; CHECK-NEXT: .long 2139095040
+
+; CHECK-LABEL: foo:
+; CHECK: movq {{.*}}, %rax
+; CHECK: shlq $48, %rax
+; CHECK: sets %al
+; CHECK: testb %al, %al
+; CHECK: flds LCPI0_0(%rip)
+; CHECK: flds LCPI0_1(%rip)
+; CHECK: fcmovne %st(1), %st(0)
+; CHECK: fstp %st(1)
+; CHECK: retq
 define x86_fp80 @foo(x86_fp80 %a) {
   %1 = tail call x86_fp80 @copysignl(x86_fp80 0xK7FFF8000000000000000, x86_fp80 %a) nounwind readnone
   ret x86_fp80 %1
diff --git a/test/CodeGen/X86/pr15267.ll b/test/CodeGen/X86/pr15267.ll
index e9f41d9ded4..95d7deb3417 100644
--- a/test/CodeGen/X86/pr15267.ll
+++ b/test/CodeGen/X86/pr15267.ll
@@ -50,20 +50,22 @@ define <4 x i64> @test3(<4 x i1>* %in) nounwind {
 ; CHECK: movq
 ; CHECK: shlq
 ; CHECK: sarq
-; CHECK: vmovq
 ; CHECK: movq
 ; CHECK: shlq
 ; CHECK: sarq
-; CHECK: vmovq
-; CHECK: vpunpcklqdq
+; CHECK: vmovd
+; CHECK: vpinsrd
 ; CHECK: movq
 ; CHECK: shlq
 ; CHECK: sarq
-; CHECK: vmovq
+; CHECK: vpinsrd
 ; CHECK: shlq
 ; CHECK: sarq
-; CHECK: vmovq
-; CHECK: vpunpcklqdq
+; CHECK: vpinsrd
+; CHECK: vpmovsxdq
+; CHECK: vmovd
+; CHECK: vpinsrd
+; CHECK: vpmovsxdq
 ; CHECK: vinsertf128
 ; CHECK: ret
 
diff --git a/test/CodeGen/X86/pr21099.ll b/test/CodeGen/X86/pr21099.ll
index 07292c125ee..cd8205dbc81 100644
--- a/test/CodeGen/X86/pr21099.ll
+++ b/test/CodeGen/X86/pr21099.ll
@@ -2,8 +2,7 @@
 
 define void @pr21099(i64* %p) {
 ; CHECK-LABEL: pr21099
-; CHECK: lock
-; CHECK-NEXT: addq $-2147483648
+; CHECK: lock addq $-2147483648
 ; This number is INT32_MIN: 0x80000000UL
   %1 = atomicrmw add i64* %p, i64 -2147483648 seq_cst
   ret void
diff --git a/test/CodeGen/X86/pr21529.ll b/test/CodeGen/X86/pr21529.ll
deleted file mode 100644
index 655bc844f50..00000000000
--- a/test/CodeGen/X86/pr21529.ll
+++ /dev/null
@@ -1,15 +0,0 @@
-; RUN: llc -show-mc-encoding < %s | FileCheck %s
-
-; Test that the direct object emission selects the and variant with 8 bit
-; immediate.
-; We used to get this wrong when using direct object emission, but not when
-; reading assembly.
-
-; CHECK: andq    $-32, %rsp              # encoding: [0x48,0x83,0xe4,0xe0]
-
-target triple = "x86_64-pc-linux"
-
-define void @f() {
-  %foo = alloca i8, align 32
-  ret void
-}
diff --git a/test/CodeGen/X86/pr23103.ll b/test/CodeGen/X86/pr23103.ll
new file mode 100644
index 00000000000..e0508effac0
--- /dev/null
+++ b/test/CodeGen/X86/pr23103.ll
@@ -0,0 +1,21 @@
+; RUN: llc -verify-machineinstrs -mtriple=x86_64-unknown-unknown -mcpu=generic -mattr=+avx < %s | FileCheck %s
+
+; When commuting a VADDSDrr instruction, verify that the 'IsUndef' flag is
+; correctly propagated to the operands of the resulting instruction.
+; Test for PR23103;
+
+declare zeroext i1 @foo(<1 x double>)
+
+define <1 x double> @pr23103(<1 x double>* align 8 %Vp) {
+; CHECK-LABEL: pr23103:
+; CHECK:         vmovsd (%rdi), %xmm0
+; CHECK-NEXT:    vmovsd %xmm0, {{.*}}(%rsp) {{.*#+}} 8-byte Spill
+; CHECK-NEXT:    callq foo
+; CHECK-NEXT:    vaddsd {{.*}}(%rsp), %xmm0, %xmm0 {{.*#+}} 8-byte Folded Reload
+; CHECK:         retq
+entry:
+  %V = load <1 x double>, <1 x double>* %Vp, align 8
+  %call = call zeroext i1 @foo(<1 x double> %V)
+  %fadd = fadd <1 x double> %V, undef
+  ret <1 x double> %fadd
+}
diff --git a/test/CodeGen/X86/pr23273.ll b/test/CodeGen/X86/pr23273.ll
new file mode 100644
index 00000000000..2702eb820f2
--- /dev/null
+++ b/test/CodeGen/X86/pr23273.ll
@@ -0,0 +1,17 @@
+; RUN: llc -mtriple=i386-unknown-unknown -mcpu=generic -march=x86 -mattr=-sse2 -fast-isel < %s
+
+; Verify that the backend doesn't crash during fast-isel with an assertion
+; failure when selecting a int-to-double conversion. The fast selection routine
+; for SINT_TO_FP wrongly assumed that the target had at least SSE2.
+
+@a = common global i32 0, align 4
+
+define i32 @pr23273() {
+entry:
+  %0 = load i32, i32* @a, align 4
+  %conv = sitofp i32 %0 to double
+  %call = call i32 @fn1(double %conv)
+  ret i32 0
+}
+
+declare i32 @fn1(double) #1
diff --git a/test/CodeGen/X86/pr23603.ll b/test/CodeGen/X86/pr23603.ll
new file mode 100644
index 00000000000..6f856aedb8d
--- /dev/null
+++ b/test/CodeGen/X86/pr23603.ll
@@ -0,0 +1,24 @@
+; RUN: llc < %s -mtriple=x86_64-apple-darwin | FileCheck %s
+
+declare void @free_v()
+
+define void @f(i32* %x, i32 %c32, i32* %y) {
+; CHECK-LABEL: f
+ entry:
+  %v = load i32, i32* %x, !invariant.load !0
+; CHECK: movl (%rdi), %ebx
+; CHECK: free_v
+; CHECK-NOT: movl (%rdi), %ebx
+  call void @free_v()
+  %c = icmp ne i32 %c32, 0
+  br i1 %c, label %left, label %merge
+
+ left:
+  store i32 %v, i32* %y
+  br label %merge
+
+ merge:
+  ret void
+}
+
+!0 = !{}
diff --git a/test/CodeGen/X86/pr23664.ll b/test/CodeGen/X86/pr23664.ll
new file mode 100644
index 00000000000..a501c0db837
--- /dev/null
+++ b/test/CodeGen/X86/pr23664.ll
@@ -0,0 +1,14 @@
+; RUN: llc -mtriple=x86_64-unknown-linux-gnu < %s | FileCheck %s
+
+define i2 @f(i32 %arg) {
+  %trunc = trunc i32 %arg to i1
+  %sext = sext i1 %trunc to i2
+  %or = or i2 %sext, 1
+  ret i2 %or
+
+; CHECK-LABEL: f:
+; CHECK:      addb    %dil, %dil
+; CHECK-NEXT: orb     $1, %dil
+; CHECK-NEXT: movb    %dil, %al
+; CHECK-NEXT: retq
+}
diff --git a/test/CodeGen/X86/pr23900.ll b/test/CodeGen/X86/pr23900.ll
new file mode 100644
index 00000000000..cbc77161c04
--- /dev/null
+++ b/test/CodeGen/X86/pr23900.ll
@@ -0,0 +1,29 @@
+; RUN: llc -filetype=obj %s -o %t.o
+; RUN: llvm-nm %t.o | FileCheck %s
+
+; Test that it doesn't crash (and produces an object file).
+; This use to pass a symbol with a null name to code that expected a valid
+; C string.
+
+; CHECK:         U __CxxFrameHandler3
+; CHECK:         T f
+; CHECK:         t f.cleanup
+; CHECK:         U g
+; CHECK:         U h
+
+
+target triple = "x86_64-pc-windows-msvc18.0.0"
+define void @f(i32 %x) personality i8* bitcast (i32 (...)* @__CxxFrameHandler3 to i8*) {
+  invoke void @h()
+          to label %invoke.cont unwind label %lpad
+invoke.cont:
+  ret void
+lpad:
+ landingpad { i8*, i32 }
+          cleanup
+  call void @g(i32 %x)
+  ret void
+}
+declare void @h()
+declare i32 @__CxxFrameHandler3(...)
+declare void @g(i32 %x)
diff --git a/test/CodeGen/X86/pr2656.ll b/test/CodeGen/X86/pr2656.ll
index 9a162d77ef4..095ab831d48 100644
--- a/test/CodeGen/X86/pr2656.ll
+++ b/test/CodeGen/X86/pr2656.ll
@@ -1,15 +1,24 @@
 ; RUN: llc < %s -march=x86 -mattr=+sse2 | FileCheck %s
 ; PR2656
 
-; CHECK:     {{xorps.*sp}}
-; CHECK-NOT: {{xorps.*sp}}
-
 target datalayout = "e-p:32:32:32-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-f32:32:32-f64:32:64-v64:64:64-v128:128:128-a0:0:64-f80:128:128"
 target triple = "i686-apple-darwin9.4.0"
 	%struct.anon = type <{ float, float }>
 @.str = internal constant [17 x i8] c"pt: %.0f, %.0f\0A\00\00"		; <[17 x i8]*> [#uses=1]
 
+; We can not fold either stack load into an 'xor' instruction because that
+; would change what should be a 4-byte load into a 16-byte load.
+; We can fold the 16-byte constant load into either 'xor' instruction,
+; but we do not. It has more than one use, so it gets loaded into a register.
+
 define void @foo(%struct.anon* byval %p) nounwind {
+; CHECK-LABEL: foo:
+; CHECK:         movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; CHECK-NEXT:    movss {{.*#+}} xmm1 = mem[0],zero,zero,zero
+; CHECK-NEXT:    movaps {{.*#+}} xmm2 = [2147483648,2147483648,2147483648,2147483648]
+; CHECK-NEXT:    xorps %xmm2, %xmm0
+; CHECK-NEXT:    cvtss2sd %xmm0, %xmm0
+; CHECK-NEXT:    xorps %xmm2, %xmm1
 entry:
 	%tmp = getelementptr %struct.anon, %struct.anon* %p, i32 0, i32 0		; <float*> [#uses=1]
 	%tmp1 = load float, float* %tmp		; <float> [#uses=1]
@@ -24,3 +33,20 @@ entry:
 }
 
 declare i32 @printf(...)
+
+; We can not fold the load from the stack into the 'and' instruction because
+; that changes an 8-byte load into a 16-byte load (illegal memory access).
+; We can fold the load of the constant because it is a 16-byte vector constant.
+
+define double @PR22371(double %x) {
+; CHECK-LABEL: PR22371:
+; CHECK:       movsd  16(%esp), %xmm0
+; CHECK-NEXT:  andpd  LCPI1_0, %xmm0
+; CHECK-NEXT:  movlpd  %xmm0, (%esp)
+  %call = tail call double @fabs(double %x) #0
+  ret double %call
+}
+
+declare double @fabs(double) #0
+attributes #0 = { readnone }
+
diff --git a/test/CodeGen/X86/pr3522.ll b/test/CodeGen/X86/pr3522.ll
index 867f2828d4d..9e048d59d4e 100644
--- a/test/CodeGen/X86/pr3522.ll
+++ b/test/CodeGen/X86/pr3522.ll
@@ -5,7 +5,7 @@
 target triple = "i386-pc-linux-gnu"
 @.str = external constant [13 x i8]		; <[13 x i8]*> [#uses=1]
 
-define void @_ada_c34018a() {
+define void @_ada_c34018a() personality i32 (...)* @__gxx_personality_v0 {
 entry:
 	%0 = tail call i32 @report__ident_int(i32 90)		; <i32> [#uses=1]
 	%1 = trunc i32 %0 to i8		; <i8> [#uses=1]
@@ -22,7 +22,7 @@ return:		; preds = %lpad
 	ret void
 
 lpad:		; preds = %entry
-        %exn = landingpad {i8*, i32} personality i32 (...)* @__gxx_personality_v0
+        %exn = landingpad {i8*, i32}
                  cleanup
 	%2 = icmp eq i8 %1, 90		; <i1> [#uses=1]
 	br i1 %2, label %return, label %bb22
diff --git a/test/CodeGen/X86/pr5145.ll b/test/CodeGen/X86/pr5145.ll
index 32a797ba138..4dee5f8d7d2 100644
--- a/test/CodeGen/X86/pr5145.ll
+++ b/test/CodeGen/X86/pr5145.ll
@@ -7,29 +7,25 @@ define void @atomic_maxmin_i8() {
 ; CHECK: [[LABEL1:\.?LBB[0-9]+_[0-9]+]]:
 ; CHECK: movsbl
 ; CHECK: cmpl
-; CHECK: lock
-; CHECK-NEXT: cmpxchgb
+; CHECK: lock cmpxchgb
 ; CHECK: jne [[LABEL1]]
   %2 = atomicrmw min  i8* @sc8, i8 6 acquire
 ; CHECK: [[LABEL3:\.?LBB[0-9]+_[0-9]+]]:
 ; CHECK: movsbl
 ; CHECK: cmpl
-; CHECK: lock
-; CHECK-NEXT: cmpxchgb
+; CHECK: lock cmpxchgb
 ; CHECK: jne [[LABEL3]]
   %3 = atomicrmw umax i8* @sc8, i8 7 acquire
 ; CHECK: [[LABEL5:\.?LBB[0-9]+_[0-9]+]]:
 ; CHECK: movzbl
 ; CHECK: cmpl
-; CHECK: lock
-; CHECK-NEXT: cmpxchgb
+; CHECK: lock cmpxchgb
 ; CHECK: jne [[LABEL5]]
   %4 = atomicrmw umin i8* @sc8, i8 8 acquire
 ; CHECK: [[LABEL7:\.?LBB[0-9]+_[0-9]+]]:
 ; CHECK: movzbl
 ; CHECK: cmpl
-; CHECK: lock
-; CHECK-NEXT: cmpxchgb
+; CHECK: lock cmpxchgb
 ; CHECK: jne [[LABEL7]]
   ret void
 }
diff --git a/test/CodeGen/X86/prolog-push-seq.ll b/test/CodeGen/X86/prolog-push-seq.ll
new file mode 100644
index 00000000000..f23791aef92
--- /dev/null
+++ b/test/CodeGen/X86/prolog-push-seq.ll
@@ -0,0 +1,19 @@
+; RUN: llc < %s | FileCheck %s
+
+target datalayout = "e-m:x-p:32:32-i64:64-f80:32-n8:16:32-a:0:32-S32"
+target triple = "i386-pc-windows-msvc18.0.0"
+
+declare x86_thiscallcc void @bar(i32 %a, i32 %b)
+
+define fastcc void @foo(i32 %a, i32 %b) #0 {
+; CHECK-LABEL: foo:
+; CHECK: subl $64, %esp
+; CHECK-NEXT: pushl
+; CHECK-NEXT: calll _bar
+  %local = alloca i32, i32 16
+  call x86_thiscallcc void @bar(i32 %a, i32 %b)
+  call void asm sideeffect "nop", "~{ax},~{bx},~{cx},~{dx},~{bp},~{si},~{di}"()
+  ret void
+}
+
+attributes #0 = { nounwind optsize "no-frame-pointer-elim-non-leaf"}
\ No newline at end of file
diff --git a/test/CodeGen/X86/ragreedy-hoist-spill.ll b/test/CodeGen/X86/ragreedy-hoist-spill.ll
index a958c4b9eea..e7dda534956 100644
--- a/test/CodeGen/X86/ragreedy-hoist-spill.ll
+++ b/test/CodeGen/X86/ragreedy-hoist-spill.ll
@@ -30,7 +30,7 @@ if.then:
 if.end:
   switch i64 undef, label %if.end25 [
     i64 0, label %if.then4
-    i64 1, label %land.lhs.true14
+    i64 1, label %if.end25
   ]
 
 if.then4:
diff --git a/test/CodeGen/X86/read-fp-no-frame-pointer.ll b/test/CodeGen/X86/read-fp-no-frame-pointer.ll
new file mode 100644
index 00000000000..9f78c294ce8
--- /dev/null
+++ b/test/CodeGen/X86/read-fp-no-frame-pointer.ll
@@ -0,0 +1,12 @@
+; RUN: not llc < %s -mtriple=x86_64-linux-gnueabi 2>&1 | FileCheck %s
+
+define i32 @get_frame() nounwind {
+entry:
+; CHECK: register ebp is allocatable: function has no frame pointer
+  %fp = call i32 @llvm.read_register.i32(metadata !0)
+  ret i32 %fp
+}
+
+declare i32 @llvm.read_register.i32(metadata) nounwind
+
+!0 = !{!"ebp\00"}
diff --git a/test/CodeGen/X86/recip-fastmath.ll b/test/CodeGen/X86/recip-fastmath.ll
index fcd077092da..8e02dad9d5a 100644
--- a/test/CodeGen/X86/recip-fastmath.ll
+++ b/test/CodeGen/X86/recip-fastmath.ll
@@ -1,6 +1,6 @@
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=sse2 | FileCheck %s
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=avx,use-recip-est | FileCheck %s --check-prefix=RECIP
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=avx,use-recip-est -x86-recip-refinement-steps=2 | FileCheck %s --check-prefix=REFINE
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=sse2 -recip=!divf,!vec-divf | FileCheck %s --check-prefix=NORECIP
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=avx -recip=divf,vec-divf | FileCheck %s --check-prefix=RECIP
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=avx -recip=divf:2,vec-divf:2 | FileCheck %s --check-prefix=REFINE
 
 ; If the target's divss/divps instructions are substantially
 ; slower than rcpss/rcpps with a Newton-Raphson refinement,
@@ -14,11 +14,11 @@ define float @reciprocal_estimate(float %x) #0 {
   %div = fdiv fast float 1.0, %x
   ret float %div
 
-; CHECK-LABEL: reciprocal_estimate:
-; CHECK: movss
-; CHECK-NEXT: divss
-; CHECK-NEXT: movaps
-; CHECK-NEXT: retq
+; NORECIP-LABEL: reciprocal_estimate:
+; NORECIP: movss
+; NORECIP-NEXT: divss
+; NORECIP-NEXT: movaps
+; NORECIP-NEXT: retq
 
 ; RECIP-LABEL: reciprocal_estimate:
 ; RECIP: vrcpss
@@ -45,11 +45,11 @@ define <4 x float> @reciprocal_estimate_v4f32(<4 x float> %x) #0 {
   %div = fdiv fast <4 x float> <float 1.0, float 1.0, float 1.0, float 1.0>, %x
   ret <4 x float> %div
 
-; CHECK-LABEL: reciprocal_estimate_v4f32:
-; CHECK: movaps
-; CHECK-NEXT: divps
-; CHECK-NEXT: movaps
-; CHECK-NEXT: retq
+; NORECIP-LABEL: reciprocal_estimate_v4f32:
+; NORECIP: movaps
+; NORECIP-NEXT: divps
+; NORECIP-NEXT: movaps
+; NORECIP-NEXT: retq
 
 ; RECIP-LABEL: reciprocal_estimate_v4f32:
 ; RECIP: vrcpps
@@ -76,14 +76,14 @@ define <8 x float> @reciprocal_estimate_v8f32(<8 x float> %x) #0 {
   %div = fdiv fast <8 x float> <float 1.0, float 1.0, float 1.0, float 1.0, float 1.0, float 1.0, float 1.0, float 1.0>, %x
   ret <8 x float> %div
 
-; CHECK-LABEL: reciprocal_estimate_v8f32:
-; CHECK: movaps
-; CHECK: movaps
-; CHECK-NEXT: divps
-; CHECK-NEXT: divps
-; CHECK-NEXT: movaps
-; CHECK-NEXT: movaps
-; CHECK-NEXT: retq
+; NORECIP-LABEL: reciprocal_estimate_v8f32:
+; NORECIP: movaps
+; NORECIP: movaps
+; NORECIP-NEXT: divps
+; NORECIP-NEXT: divps
+; NORECIP-NEXT: movaps
+; NORECIP-NEXT: movaps
+; NORECIP-NEXT: retq
 
 ; RECIP-LABEL: reciprocal_estimate_v8f32:
 ; RECIP: vrcpps
diff --git a/test/CodeGen/X86/rrlist-livereg-corrutpion.ll b/test/CodeGen/X86/rrlist-livereg-corrutpion.ll
new file mode 100644
index 00000000000..7191e0453a6
--- /dev/null
+++ b/test/CodeGen/X86/rrlist-livereg-corrutpion.ll
@@ -0,0 +1,26 @@
+; RUN: llc < %s -march=x86-64 | FileCheck %s
+
+; CHECK-LABEL: test
+define i64 @test(i64 %a, i256 %b, i1 %c) {
+  %u = zext i64 %a to i256
+  %s = add i256 %u, 1
+  %o = trunc i256 %s to i1
+  %j = add i256 %s, 1
+  %i = icmp ule i64 %a, 1
+  %f = select i1 %o, i256 undef, i256 %j
+  %d = select i1 %i, i256 %f, i256 1
+  %e = add i256 %b, 1
+  %n = select i1 %c, i256 %e, i256 %b
+  %m = trunc i256 %n to i64
+  %h = add i64 %m, 1
+  %r = zext i64 %h to i256
+  %v = lshr i256 %d, %r
+  %t = trunc i256 %v to i1
+  %q = shl i256 1, %r
+  %p = and i256 %d, %q
+  %w = icmp ule i256 %n, 1
+  %y = select i1 %t, i256 undef, i256 %p
+  %x = select i1 %w, i256 %y, i256 %d
+  %z = trunc i256 %x to i64
+  ret i64 %z
+}
diff --git a/test/CodeGen/X86/scev-interchange.ll b/test/CodeGen/X86/scev-interchange.ll
index e224c0858af..9cbb462e47d 100644
--- a/test/CodeGen/X86/scev-interchange.ll
+++ b/test/CodeGen/X86/scev-interchange.ll
@@ -51,7 +51,7 @@ declare fastcc void @_ZN11FE_Q_Helper12_GLOBAL__N_116invert_numberingERKSt6vecto
 
 declare fastcc void @_ZN4FE_QILi3EE14get_dpo_vectorEj(%"struct.std::vector<int,std::allocator<int> >"* noalias nocapture sret, i32)
 
-define fastcc void @_ZN4FE_QILi3EEC1Ej(i32 %degree) {
+define fastcc void @_ZN4FE_QILi3EEC1Ej(i32 %degree) personality i32 (...)* @__gxx_personality_v0 {
 entry:
 	invoke fastcc void @_ZNSt6vectorIbSaIbEEC1EmRKbRKS0_(%"struct.std::vector<bool,std::allocator<bool> >"* undef, i64 1, i8* undef)
 			to label %invcont.i unwind label %lpad.i
@@ -149,7 +149,7 @@ bb71.i:		; preds = %bb.i.i.i262.i, %bb66.i
 			to label %_ZNSt12_Vector_baseIjSaIjEEC2EmRKS0_.exit.i.i.i.i.i unwind label %lpad.i.i.i.i.i.i		; <i8*> [#uses=0]
 
 lpad.i.i.i.i.i.i:		; preds = %bb71.i
-        %exn.i.i.i.i.i.i = landingpad {i8*, i32} personality i32 (...)* @__gxx_personality_v0
+        %exn.i.i.i.i.i.i = landingpad {i8*, i32}
                  cleanup
 	unreachable
 
@@ -164,7 +164,7 @@ _ZNSt6vectorIjSaIjEED1Ev.exit.i.i:		; preds = %_ZNSt12_Vector_baseIjSaIjEEC2EmRK
 			to label %_ZNSt12_Vector_baseIjSaIjEEC2EmRKS0_.exit.i.i.i12.i.i unwind label %lpad.i.i.i.i8.i.i		; <i8*> [#uses=0]
 
 lpad.i.i.i.i8.i.i:		; preds = %_ZNSt6vectorIjSaIjEED1Ev.exit.i.i
-        %exn.i.i.i.i8.i.i = landingpad {i8*, i32} personality i32 (...)* @__gxx_personality_v0
+        %exn.i.i.i.i8.i.i = landingpad {i8*, i32}
                  cleanup
 	invoke void @_Unwind_Resume(i8* undef)
 			to label %.noexc.i9.i.i unwind label %lpad.i19.i.i
@@ -183,7 +183,7 @@ bb50.i.i.i:		; preds = %bb.i.i.i.i.i.i.i.i.i.i, %_ZNSt12_Vector_baseIjSaIjEEC2Em
 			to label %bb83.i unwind label %lpad188.i
 
 lpad.i19.i.i:		; preds = %lpad.i.i.i.i8.i.i
-        %exn.i19.i.i = landingpad {i8*, i32} personality i32 (...)* @__gxx_personality_v0
+        %exn.i19.i.i = landingpad {i8*, i32}
                  cleanup
 	unreachable
 
@@ -198,7 +198,7 @@ invcont84.i:		; preds = %bb83.i
 			to label %_ZNSt12_Vector_baseIjSaIjEEC2EmRKS0_.exit.i.i.i.i unwind label %lpad.i.i.i.i315.i		; <i8*> [#uses=0]
 
 lpad.i.i.i.i315.i:		; preds = %invcont84.i
-        %exn.i.i.i.i315.i = landingpad {i8*, i32} personality i32 (...)* @__gxx_personality_v0
+        %exn.i.i.i.i315.i = landingpad {i8*, i32}
                  cleanup
 	invoke void @_Unwind_Resume(i8* undef)
 			to label %.noexc.i316.i unwind label %lpad.i352.i
@@ -217,7 +217,7 @@ bb50.i.i:		; preds = %bb.i.i.i.i.i.i.i.i320.i, %_ZNSt12_Vector_baseIjSaIjEEC2EmR
 			to label %invcont86.i unwind label %lpad200.i
 
 lpad.i352.i:		; preds = %lpad.i.i.i.i315.i
-        %exn.i352.i = landingpad {i8*, i32} personality i32 (...)* @__gxx_personality_v0
+        %exn.i352.i = landingpad {i8*, i32}
                  cleanup
 	unreachable
 
@@ -242,7 +242,7 @@ invcont101.i:		; preds = %bb100.i
 			to label %_ZN10FullMatrixIdEC1Ejj.exit.i.i unwind label %lpad.i.i.i.i.i
 
 lpad.i.i.i.i.i:		; preds = %invcont101.i
-        %exn.i.i.i.i.i = landingpad {i8*, i32} personality i32 (...)* @__gxx_personality_v0
+        %exn.i.i.i.i.i = landingpad {i8*, i32}
                  cleanup
 	unreachable
 
@@ -251,7 +251,7 @@ _ZN10FullMatrixIdEC1Ejj.exit.i.i:		; preds = %invcont101.i
 			to label %_ZN10FullMatrixIdEC1Ejj.exit28.i.i unwind label %lpad.i.i.i27.i.i
 
 lpad.i.i.i27.i.i:		; preds = %_ZN10FullMatrixIdEC1Ejj.exit.i.i
-        %exn.i.i.i27.i.i = landingpad {i8*, i32} personality i32 (...)* @__gxx_personality_v0
+        %exn.i.i.i27.i.i = landingpad {i8*, i32}
                  cleanup
 	invoke void @_Unwind_Resume(i8* undef)
 			to label %.noexc.i.i unwind label %lpad.i.i
@@ -272,7 +272,7 @@ bb.i.i.i297.i.i:		; preds = %bb58.i.i
 	unreachable
 
 lpad.i.i:		; preds = %lpad.i.i.i27.i.i
-        %exn.i.i = landingpad {i8*, i32} personality i32 (...)* @__gxx_personality_v0
+        %exn.i.i = landingpad {i8*, i32}
                  cleanup
 	unreachable
 
@@ -312,67 +312,67 @@ bb29.loopexit.i.i:		; preds = %.noexc232.i
 	br label %bb9.i216.i
 
 lpad.i:		; preds = %entry
-        %exn.i = landingpad {i8*, i32} personality i32 (...)* @__gxx_personality_v0
+        %exn.i = landingpad {i8*, i32}
                  cleanup
 	unreachable
 
 lpad120.i:		; preds = %invcont.i
-        %exn120.i = landingpad {i8*, i32} personality i32 (...)* @__gxx_personality_v0
+        %exn120.i = landingpad {i8*, i32}
                  cleanup
 	unreachable
 
 lpad124.i:		; preds = %invcont1.i
-        %exn124.i = landingpad {i8*, i32} personality i32 (...)* @__gxx_personality_v0
+        %exn124.i = landingpad {i8*, i32}
                  cleanup
 	unreachable
 
 lpad128.i:		; preds = %invcont3.i
-        %exn128.i = landingpad {i8*, i32} personality i32 (...)* @__gxx_personality_v0
+        %exn128.i = landingpad {i8*, i32}
                  cleanup
 	unreachable
 
 lpad132.i:		; preds = %invcont4.i
-        %exn132.i = landingpad {i8*, i32} personality i32 (...)* @__gxx_personality_v0
+        %exn132.i = landingpad {i8*, i32}
                  cleanup
 	unreachable
 
 lpad136.i:		; preds = %invcont6.i
-        %exn136.i = landingpad {i8*, i32} personality i32 (...)* @__gxx_personality_v0
+        %exn136.i = landingpad {i8*, i32}
                  cleanup
 	unreachable
 
 lpad140.i:		; preds = %bb21.i, %invcont7.i
-        %exn140.i = landingpad {i8*, i32} personality i32 (...)* @__gxx_personality_v0
+        %exn140.i = landingpad {i8*, i32}
                  cleanup
 	unreachable
 
 lpad144.i:		; preds = %bb10.i168.i, %invcont9.i
-        %exn144.i = landingpad {i8*, i32} personality i32 (...)* @__gxx_personality_v0
+        %exn144.i = landingpad {i8*, i32}
                  cleanup
 	unreachable
 
 lpad148.i:		; preds = %invcont10.i
-        %exn148.i = landingpad {i8*, i32} personality i32 (...)* @__gxx_personality_v0
+        %exn148.i = landingpad {i8*, i32}
                  cleanup
 	unreachable
 
 lpad188.i:		; preds = %bb50.i.i.i
-        %exn188.i = landingpad {i8*, i32} personality i32 (...)* @__gxx_personality_v0
+        %exn188.i = landingpad {i8*, i32}
                  cleanup
 	unreachable
 
 lpad196.i:		; preds = %bb.i191.i
-        %exn196 = landingpad {i8*, i32} personality i32 (...)* @__gxx_personality_v0
+        %exn196 = landingpad {i8*, i32}
                  cleanup
 	unreachable
 
 lpad200.i:		; preds = %bb50.i.i
-        %exn200.i = landingpad {i8*, i32} personality i32 (...)* @__gxx_personality_v0
+        %exn200.i = landingpad {i8*, i32}
                  cleanup
 	unreachable
 
 lpad204.i:		; preds = %invcont86.i
-        %exn204.i = landingpad {i8*, i32} personality i32 (...)* @__gxx_personality_v0
+        %exn204.i = landingpad {i8*, i32}
                  cleanup
 	unreachable
 }
diff --git a/test/CodeGen/X86/sdiv-exact.ll b/test/CodeGen/X86/sdiv-exact.ll
index 4f8d3f05351..a6ace5bc31c 100644
--- a/test/CodeGen/X86/sdiv-exact.ll
+++ b/test/CodeGen/X86/sdiv-exact.ll
@@ -1,4 +1,4 @@
-; RUN: llc -march=x86 < %s | FileCheck %s
+; RUN: llc -march=x86 -mattr=+sse2 < %s | FileCheck %s
 
 define i32 @test1(i32 %x) {
   %div = sdiv exact i32 %x, 25
@@ -16,3 +16,14 @@ define i32 @test2(i32 %x) {
 ; CHECK-NEXT: imull	$-1431655765
 ; CHECK-NEXT: ret
 }
+
+define <4 x i32> @test3(<4 x i32> %x) {
+  %div = sdiv exact <4 x i32> %x, <i32 24, i32 24, i32 24, i32 24>
+  ret <4 x i32> %div
+; CHECK-LABEL: test3:
+; CHECK: psrad	$3,
+; CHECK: pmuludq
+; CHECK: pmuludq
+; CHECK-NOT: psrad
+; CHECK: ret
+}
diff --git a/test/CodeGen/X86/seh-basic.ll b/test/CodeGen/X86/seh-basic.ll
deleted file mode 100644
index 69d70d70948..00000000000
--- a/test/CodeGen/X86/seh-basic.ll
+++ /dev/null
@@ -1,175 +0,0 @@
-; RUN: llc -mtriple x86_64-pc-windows-msvc < %s | FileCheck %s
-
-define void @two_invoke_merged() {
-entry:
-  invoke void @try_body()
-          to label %again unwind label %lpad
-
-again:
-  invoke void @try_body()
-          to label %done unwind label %lpad
-
-done:
-  ret void
-
-lpad:
-  %vals = landingpad { i8*, i32 } personality i8* bitcast (i32 (...)* @__C_specific_handler to i8*)
-          catch i8* bitcast (i32 (i8*, i8*)* @filt0 to i8*)
-          catch i8* bitcast (i32 (i8*, i8*)* @filt1 to i8*)
-  %sel = extractvalue { i8*, i32 } %vals, 1
-  call void @use_selector(i32 %sel)
-  ret void
-}
-
-; Normal path code
-
-; CHECK-LABEL: {{^}}two_invoke_merged:
-; CHECK: .seh_proc two_invoke_merged
-; CHECK: .seh_handler __C_specific_handler, @unwind, @except
-; CHECK: .Ltmp0:
-; CHECK: callq try_body
-; CHECK-NEXT: .Ltmp1:
-; CHECK: .Ltmp2:
-; CHECK: callq try_body
-; CHECK-NEXT: .Ltmp3:
-; CHECK: retq
-
-; Landing pad code
-
-; CHECK: .Ltmp5:
-; CHECK: movl $1, %ecx
-; CHECK: jmp
-; CHECK: .Ltmp6:
-; CHECK: movl $2, %ecx
-; CHECK: callq use_selector
-
-; CHECK: .seh_handlerdata
-; CHECK-NEXT: .long 2
-; CHECK-NEXT: .long .Ltmp0@IMGREL
-; CHECK-NEXT: .long .Ltmp3@IMGREL+1
-; CHECK-NEXT: .long filt0@IMGREL
-; CHECK-NEXT: .long .Ltmp5@IMGREL
-; CHECK-NEXT: .long .Ltmp0@IMGREL
-; CHECK-NEXT: .long .Ltmp3@IMGREL+1
-; CHECK-NEXT: .long filt1@IMGREL
-; CHECK-NEXT: .long .Ltmp6@IMGREL
-; CHECK: .text
-; CHECK: .seh_endproc
-
-define void @two_invoke_gap() {
-entry:
-  invoke void @try_body()
-          to label %again unwind label %lpad
-
-again:
-  call void @do_nothing_on_unwind()
-  invoke void @try_body()
-          to label %done unwind label %lpad
-
-done:
-  ret void
-
-lpad:
-  %vals = landingpad { i8*, i32 } personality i8* bitcast (i32 (...)* @__C_specific_handler to i8*)
-          catch i8* bitcast (i32 (i8*, i8*)* @filt0 to i8*)
-  %sel = extractvalue { i8*, i32 } %vals, 1
-  call void @use_selector(i32 %sel)
-  ret void
-}
-
-; Normal path code
-
-; CHECK-LABEL: {{^}}two_invoke_gap:
-; CHECK: .seh_proc two_invoke_gap
-; CHECK: .seh_handler __C_specific_handler, @unwind, @except
-; CHECK: .Ltmp11:
-; CHECK: callq try_body
-; CHECK-NEXT: .Ltmp12:
-; CHECK: callq do_nothing_on_unwind
-; CHECK: .Ltmp13:
-; CHECK: callq try_body
-; CHECK-NEXT: .Ltmp14:
-; CHECK: retq
-
-; Landing pad code
-
-; CHECK: .Ltmp16:
-; CHECK: movl $1, %ecx
-; CHECK: callq use_selector
-
-; CHECK: .seh_handlerdata
-; CHECK-NEXT: .long 2
-; CHECK-NEXT: .long .Ltmp11@IMGREL
-; CHECK-NEXT: .long .Ltmp12@IMGREL+1
-; CHECK-NEXT: .long filt0@IMGREL
-; CHECK-NEXT: .long .Ltmp16@IMGREL
-; CHECK-NEXT: .long .Ltmp13@IMGREL
-; CHECK-NEXT: .long .Ltmp14@IMGREL+1
-; CHECK-NEXT: .long filt0@IMGREL
-; CHECK-NEXT: .long .Ltmp16@IMGREL
-; CHECK: .text
-; CHECK: .seh_endproc
-
-define void @two_invoke_nounwind_gap() {
-entry:
-  invoke void @try_body()
-          to label %again unwind label %lpad
-
-again:
-  call void @cannot_unwind()
-  invoke void @try_body()
-          to label %done unwind label %lpad
-
-done:
-  ret void
-
-lpad:
-  %vals = landingpad { i8*, i32 } personality i8* bitcast (i32 (...)* @__C_specific_handler to i8*)
-          catch i8* bitcast (i32 (i8*, i8*)* @filt0 to i8*)
-  %sel = extractvalue { i8*, i32 } %vals, 1
-  call void @use_selector(i32 %sel)
-  ret void
-}
-
-; Normal path code
-
-; CHECK-LABEL: {{^}}two_invoke_nounwind_gap:
-; CHECK: .seh_proc two_invoke_nounwind_gap
-; CHECK: .seh_handler __C_specific_handler, @unwind, @except
-; CHECK: .Ltmp21:
-; CHECK: callq try_body
-; CHECK-NEXT: .Ltmp22:
-; CHECK: callq cannot_unwind
-; CHECK: .Ltmp23:
-; CHECK: callq try_body
-; CHECK-NEXT: .Ltmp24:
-; CHECK: retq
-
-; Landing pad code
-
-; CHECK: .Ltmp26:
-; CHECK: movl $1, %ecx
-; CHECK: callq use_selector
-
-; CHECK: .seh_handlerdata
-; CHECK-NEXT: .long 1
-; CHECK-NEXT: .long .Ltmp21@IMGREL
-; CHECK-NEXT: .long .Ltmp24@IMGREL+1
-; CHECK-NEXT: .long filt0@IMGREL
-; CHECK-NEXT: .long .Ltmp26@IMGREL
-; CHECK: .text
-; CHECK: .seh_endproc
-
-declare void @try_body()
-declare void @do_nothing_on_unwind()
-declare void @cannot_unwind() nounwind
-declare void @use_selector(i32)
-
-declare i32 @filt0(i8* %eh_info, i8* %rsp)
-declare i32 @filt1(i8* %eh_info, i8* %rsp)
-
-declare void @handler0()
-declare void @handler1()
-
-declare i32 @__C_specific_handler(...)
-declare i32 @llvm.eh.typeid.for(i8*) readnone nounwind
diff --git a/test/CodeGen/X86/seh-catch-all-win32.ll b/test/CodeGen/X86/seh-catch-all-win32.ll
new file mode 100644
index 00000000000..a4ea8ab78c7
--- /dev/null
+++ b/test/CodeGen/X86/seh-catch-all-win32.ll
@@ -0,0 +1,106 @@
+; RUN: llc -mtriple=i686-windows-msvc < %s | FileCheck %s
+
+; 32-bit catch-all has to use a filter function because that's how it saves the
+; exception code.
+
+@str = linkonce_odr unnamed_addr constant [27 x i8] c"GetExceptionCode(): 0x%lx\0A\00", align 1
+
+declare i32 @_except_handler3(...)
+declare void @crash()
+declare i32 @printf(i8* nocapture readonly, ...) nounwind
+declare i32 @llvm.eh.typeid.for(i8*)
+declare i8* @llvm.frameaddress(i32)
+declare i8* @llvm.localrecover(i8*, i8*, i32)
+declare void @llvm.localescape(...)
+declare i8* @llvm.x86.seh.recoverfp(i8*, i8*)
+
+define i32 @main() personality i8* bitcast (i32 (...)* @_except_handler3 to i8*) {
+entry:
+  %__exceptioncode = alloca i32, align 4
+  call void (...) @llvm.localescape(i32* %__exceptioncode)
+  invoke void @crash() #5
+          to label %__try.cont unwind label %lpad
+
+lpad:                                             ; preds = %entry
+  %0 = landingpad { i8*, i32 }
+          catch i8* bitcast (i32 ()* @"filt$main" to i8*)
+  %1 = extractvalue { i8*, i32 } %0, 1
+  %2 = call i32 @llvm.eh.typeid.for(i8* bitcast (i32 ()* @"filt$main" to i8*)) #4
+  %matches = icmp eq i32 %1, %2
+  br i1 %matches, label %__except, label %eh.resume
+
+__except:                                         ; preds = %lpad
+  %3 = load i32, i32* %__exceptioncode, align 4
+  %call = call i32 (i8*, ...) @printf(i8* getelementptr inbounds ([27 x i8], [27 x i8]* @str, i32 0, i32 0), i32 %3) #4
+  br label %__try.cont
+
+__try.cont:                                       ; preds = %entry, %__except
+  ret i32 0
+
+eh.resume:                                        ; preds = %lpad
+  resume { i8*, i32 } %0
+}
+
+define internal i32 @"filt$main"() {
+entry:
+  %ebp = tail call i8* @llvm.frameaddress(i32 1)
+  %parentfp = tail call i8* @llvm.x86.seh.recoverfp(i8* bitcast (i32 ()* @main to i8*), i8* %ebp)
+  %code.i8 = tail call i8* @llvm.localrecover(i8* bitcast (i32 ()* @main to i8*), i8* %parentfp, i32 0)
+  %__exceptioncode = bitcast i8* %code.i8 to i32*
+  %info.addr = getelementptr inbounds i8, i8* %ebp, i32 -20
+  %0 = bitcast i8* %info.addr to i32***
+  %1 = load i32**, i32*** %0, align 4
+  %2 = load i32*, i32** %1, align 4
+  %3 = load i32, i32* %2, align 4
+  store i32 %3, i32* %__exceptioncode, align 4
+  ret i32 1
+}
+
+; Check that we can get the exception code from eax to the printf.
+
+; CHECK-LABEL: _main:
+; CHECK: pushl %ebp
+; CHECK: movl %esp, %ebp
+;       Ensure that we push *all* the CSRs, since they are clobbered by the
+;       __except block.
+; CHECK: pushl %ebx
+; CHECK: pushl %edi
+; CHECK: pushl %esi
+
+; CHECK: Lmain$frame_escape_0 = [[code_offs:[-0-9]+]]
+; CHECK: Lmain$frame_escape_1 = [[reg_offs:[-0-9]+]]
+; CHECK: movl %esp, [[reg_offs]](%ebp)
+; CHECK: movl $L__ehtable$main,
+; 	EH state 0
+; CHECK: movl $0, -16(%ebp)
+; CHECK: calll _crash
+; CHECK: popl %esi
+; CHECK: popl %edi
+; CHECK: popl %ebx
+; CHECK: retl
+; CHECK: # Block address taken
+; 	stackrestore
+; CHECK: movl -24(%ebp), %esp
+; 	EH state -1
+; CHECK: movl [[code_offs]](%ebp), %[[code:[a-z]+]]
+; CHECK: movl $-1, -16(%ebp)
+; CHECK-DAG: movl %[[code]], 4(%esp)
+; CHECK-DAG: movl $_str, (%esp)
+; CHECK: calll _printf
+
+; CHECK: .section .xdata,"dr"
+; CHECK: Lmain$parent_frame_offset = Lmain$frame_escape_1
+; CHECK: .align 4
+; CHECK: L__ehtable$main
+; CHECK-NEXT: .long -1
+; CHECK-NEXT: .long _filt$main
+; CHECK-NEXT: .long Ltmp{{[0-9]+}}
+
+; CHECK-LABEL: _filt$main:
+; CHECK: pushl %ebp
+; CHECK: movl %esp, %ebp
+; CHECK: movl (%ebp), %[[oldebp:[a-z]+]]
+; CHECK: movl -20(%[[oldebp]]), %[[ehinfo:[a-z]+]]
+; CHECK: movl (%[[ehinfo]]), %[[ehrec:[a-z]+]]
+; CHECK: movl (%[[ehrec]]), %[[ehcode:[a-z]+]]
+; CHECK: movl %[[ehcode]], {{.*}}(%{{.*}})
diff --git a/test/CodeGen/X86/seh-catch-all.ll b/test/CodeGen/X86/seh-catch-all.ll
index 931046e5115..1c1a3c2139d 100644
--- a/test/CodeGen/X86/seh-catch-all.ll
+++ b/test/CodeGen/X86/seh-catch-all.ll
@@ -1,20 +1,23 @@
 ; RUN: llc -mtriple=x86_64-windows-msvc < %s | FileCheck %s
 
-@str = internal unnamed_addr constant [10 x i8] c"recovered\00", align 1
+@str = linkonce_odr unnamed_addr constant [27 x i8] c"GetExceptionCode(): 0x%lx\0A\00", align 1
 
 declare i32 @__C_specific_handler(...)
 declare void @crash()
-declare i32 @puts(i8*)
+declare i32 @printf(i8* nocapture readonly, ...) nounwind
 
-define i32 @main() {
+define i32 @main() personality i8* bitcast (i32 (...)* @__C_specific_handler to i8*) {
 entry:
   invoke void @crash()
           to label %__try.cont unwind label %lpad
 
 lpad:
-  %0 = landingpad { i8*, i32 } personality i8* bitcast (i32 (...)* @__C_specific_handler to i8*)
+  %0 = landingpad { i8*, i32 }
           catch i8* null
-  call i32 @puts(i8* getelementptr inbounds ([10 x i8], [10 x i8]* @str, i64 0, i64 0))
+  %1 = extractvalue { i8*, i32 } %0, 0
+  %2 = ptrtoint i8* %1 to i64
+  %3 = trunc i64 %2 to i32
+  call i32 (i8*, ...) @printf(i8* getelementptr inbounds ([27 x i8], [27 x i8]* @str, i64 0, i64 0), i32 %3)
   br label %__try.cont
 
 __try.cont:
@@ -24,10 +27,19 @@ eh.resume:
   resume { i8*, i32 } %0
 }
 
+; Check that we can get the exception code from eax to the printf.
+
 ; CHECK-LABEL: main:
+; CHECK: callq crash
+; CHECK: retq
+; CHECK: # Block address taken
+; CHECK: leaq str(%rip), %rcx
+; CHECK: movl %eax, %edx
+; CHECK: callq printf
+
 ; CHECK: .seh_handlerdata
 ; CHECK-NEXT: .long 1
-; CHECK-NEXT: .Ltmp{{[0-9]+}}@IMGREL
-; CHECK-NEXT: .Ltmp{{[0-9]+}}@IMGREL+1
-; CHECK-NEXT: 1
-; CHECK-NEXT: .Ltmp{{[0-9]+}}@IMGREL
+; CHECK-NEXT: .long .Ltmp{{[0-9]+}}@IMGREL
+; CHECK-NEXT: .long .Ltmp{{[0-9]+}}@IMGREL+1
+; CHECK-NEXT: .long 1
+; CHECK-NEXT: .long .Ltmp{{[0-9]+}}@IMGREL
diff --git a/test/CodeGen/X86/seh-except-finally.ll b/test/CodeGen/X86/seh-except-finally.ll
new file mode 100644
index 00000000000..0630d001bb7
--- /dev/null
+++ b/test/CodeGen/X86/seh-except-finally.ll
@@ -0,0 +1,167 @@
+; RUN: llc < %s | FileCheck %s
+
+; Test case based on this source:
+; int puts(const char*);
+; __declspec(noinline) void crash() {
+;   *(volatile int*)0 = 42;
+; }
+; int filt();
+; void use_both() {
+;   __try {
+;     __try {
+;       crash();
+;     } __finally {
+;       puts("__finally");
+;     }
+;   } __except (filt()) {
+;     puts("__except");
+;   }
+; }
+
+target datalayout = "e-m:w-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-pc-windows-msvc"
+
+$"\01??_C@_09KJEHOMHG@__finally?$AA@" = comdat any
+
+$"\01??_C@_08MLCMLGHM@__except?$AA@" = comdat any
+
+@"\01??_C@_09KJEHOMHG@__finally?$AA@" = linkonce_odr unnamed_addr constant [10 x i8] c"__finally\00", comdat, align 1
+@"\01??_C@_08MLCMLGHM@__except?$AA@" = linkonce_odr unnamed_addr constant [9 x i8] c"__except\00", comdat, align 1
+
+declare void @crash()
+
+declare i32 @filt()
+
+; Function Attrs: nounwind uwtable
+define void @use_both() #1 personality i8* bitcast (i32 (...)* @__C_specific_handler to i8*) {
+entry:
+  %exn.slot = alloca i8*
+  %ehselector.slot = alloca i32
+  invoke void @crash() #5
+          to label %invoke.cont unwind label %lpad
+
+invoke.cont:                                      ; preds = %entry
+  %0 = call i8* @llvm.localaddress()
+  invoke void @"\01?fin$0@0@use_both@@"(i1 zeroext false, i8* %0) #5
+          to label %invoke.cont2 unwind label %lpad1
+
+invoke.cont2:                                     ; preds = %invoke.cont
+  br label %__try.cont
+
+lpad:                                             ; preds = %entry
+  %1 = landingpad { i8*, i32 }
+          cleanup
+          catch i8* bitcast (i32 (i8*, i8*)* @"\01?filt$0@0@use_both@@" to i8*)
+  %2 = extractvalue { i8*, i32 } %1, 0
+  store i8* %2, i8** %exn.slot
+  %3 = extractvalue { i8*, i32 } %1, 1
+  store i32 %3, i32* %ehselector.slot
+  %4 = call i8* @llvm.localaddress()
+  invoke void @"\01?fin$0@0@use_both@@"(i1 zeroext true, i8* %4) #5
+          to label %invoke.cont3 unwind label %lpad1
+
+lpad1:                                            ; preds = %lpad, %invoke.cont
+  %5 = landingpad { i8*, i32 }
+          catch i8* bitcast (i32 (i8*, i8*)* @"\01?filt$0@0@use_both@@" to i8*)
+  %6 = extractvalue { i8*, i32 } %5, 0
+  store i8* %6, i8** %exn.slot
+  %7 = extractvalue { i8*, i32 } %5, 1
+  store i32 %7, i32* %ehselector.slot
+  br label %catch.dispatch
+
+invoke.cont3:                                     ; preds = %lpad
+  br label %catch.dispatch
+
+catch.dispatch:                                   ; preds = %invoke.cont3, %lpad1
+  %sel = load i32, i32* %ehselector.slot
+  %8 = call i32 @llvm.eh.typeid.for(i8* bitcast (i32 (i8*, i8*)* @"\01?filt$0@0@use_both@@" to i8*)) #6
+  %matches = icmp eq i32 %sel, %8
+  br i1 %matches, label %__except, label %eh.resume
+
+__except:                                         ; preds = %catch.dispatch
+  %call = call i32 @puts(i8* getelementptr inbounds ([9 x i8], [9 x i8]* @"\01??_C@_08MLCMLGHM@__except?$AA@", i32 0, i32 0))
+  br label %__try.cont
+
+__try.cont:                                       ; preds = %__except, %invoke.cont2
+  ret void
+
+eh.resume:                                        ; preds = %catch.dispatch
+  %exn = load i8*, i8** %exn.slot
+  %sel4 = load i32, i32* %ehselector.slot
+  %lpad.val = insertvalue { i8*, i32 } undef, i8* %exn, 0
+  %lpad.val5 = insertvalue { i8*, i32 } %lpad.val, i32 %sel4, 1
+  resume { i8*, i32 } %lpad.val5
+}
+
+; CHECK-LABEL: use_both:
+; CHECK: .Ltmp0
+; CHECK: callq crash
+; CHECK: .Ltmp1
+; CHECK: .Ltmp3
+; CHECK: callq "?fin$0@0@use_both@@"
+; CHECK: .Ltmp4
+; CHECK: retq
+;
+; CHECK: .seh_handlerdata
+; CHECK-NEXT: .long 3
+; CHECK-NEXT: .long .Ltmp0@IMGREL
+; CHECK-NEXT: .long .Ltmp1@IMGREL+1
+; CHECK-NEXT: .long "?fin$0@0@use_both@@"@IMGREL
+; CHECK-NEXT: .long 0
+; CHECK-NEXT: .long .Ltmp0@IMGREL
+; CHECK-NEXT: .long .Ltmp1@IMGREL+1
+; CHECK-NEXT: .long "?filt$0@0@use_both@@"@IMGREL
+; CHECK-NEXT: .long .Ltmp{{[0-9]+}}@IMGREL
+; CHECK-NEXT: .long .Ltmp3@IMGREL
+; CHECK-NEXT: .long .Ltmp4@IMGREL+1
+; CHECK-NEXT: .long "?filt$0@0@use_both@@"@IMGREL
+; CHECK-NEXT: .long .Ltmp{{[0-9]+}}@IMGREL
+
+; Function Attrs: noinline nounwind
+define internal i32 @"\01?filt$0@0@use_both@@"(i8* %exception_pointers, i8* %frame_pointer) #2 {
+entry:
+  %frame_pointer.addr = alloca i8*, align 8
+  %exception_pointers.addr = alloca i8*, align 8
+  %exn.slot = alloca i8*
+  store i8* %frame_pointer, i8** %frame_pointer.addr, align 8
+  store i8* %exception_pointers, i8** %exception_pointers.addr, align 8
+  %0 = load i8*, i8** %exception_pointers.addr
+  %1 = bitcast i8* %0 to { i32*, i8* }*
+  %2 = getelementptr inbounds { i32*, i8* }, { i32*, i8* }* %1, i32 0, i32 0
+  %3 = load i32*, i32** %2
+  %4 = load i32, i32* %3
+  %5 = zext i32 %4 to i64
+  %6 = inttoptr i64 %5 to i8*
+  store i8* %6, i8** %exn.slot
+  %call = call i32 @filt()
+  ret i32 %call
+}
+
+define internal void @"\01?fin$0@0@use_both@@"(i1 zeroext %abnormal_termination, i8* %frame_pointer) #3 {
+entry:
+  %frame_pointer.addr = alloca i8*, align 8
+  %abnormal_termination.addr = alloca i8, align 1
+  store i8* %frame_pointer, i8** %frame_pointer.addr, align 8
+  %frombool = zext i1 %abnormal_termination to i8
+  store i8 %frombool, i8* %abnormal_termination.addr, align 1
+  %call = call i32 @puts(i8* getelementptr inbounds ([10 x i8], [10 x i8]* @"\01??_C@_09KJEHOMHG@__finally?$AA@", i32 0, i32 0))
+  ret void
+}
+
+declare i32 @puts(i8*) #3
+
+declare i32 @__C_specific_handler(...)
+
+; Function Attrs: nounwind readnone
+declare i8* @llvm.localaddress() #4
+
+; Function Attrs: nounwind readnone
+declare i32 @llvm.eh.typeid.for(i8*) #4
+
+attributes #0 = { noinline nounwind uwtable "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #1 = { nounwind uwtable "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #2 = { noinline nounwind "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #3 = { "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #4 = { nounwind readnone }
+attributes #5 = { noinline }
+attributes #6 = { nounwind }
diff --git a/test/CodeGen/X86/seh-filter-no-personality.ll b/test/CodeGen/X86/seh-filter-no-personality.ll
new file mode 100644
index 00000000000..87bc9c93f40
--- /dev/null
+++ b/test/CodeGen/X86/seh-filter-no-personality.ll
@@ -0,0 +1,33 @@
+; RUN: llc -mtriple=i686-windows-msvc < %s | FileCheck %s
+
+; Mostly make sure that llvm.x86.seh.recoverfp doesn't crash if the parent
+; function lacks a personality.
+
+declare i8* @llvm.frameaddress(i32)
+declare i8* @llvm.x86.seh.recoverfp(i8*, i8*)
+
+define i32 @main() {
+entry:
+  ret i32 0
+}
+
+define internal i32 @"filt$main"() {
+entry:
+  %ebp = tail call i8* @llvm.frameaddress(i32 1)
+  %parentfp = tail call i8* @llvm.x86.seh.recoverfp(i8* bitcast (i32 ()* @main to i8*), i8* %ebp)
+  %info.addr = getelementptr inbounds i8, i8* %ebp, i32 -20
+  %0 = bitcast i8* %info.addr to i32***
+  %1 = load i32**, i32*** %0, align 4
+  %2 = load i32*, i32** %1, align 4
+  %3 = load i32, i32* %2, align 4
+  %matches = icmp eq i32 %3, u0xC0000005
+  %r = zext i1 %matches to i32
+  ret i32 %r
+}
+
+; CHECK: _main:
+; CHECK: xorl %eax, %eax
+; CHECK: retl
+
+; CHECK: _filt$main:
+; CHECK: retl
diff --git a/test/CodeGen/X86/seh-filter.ll b/test/CodeGen/X86/seh-filter.ll
index 6a3a23edb1a..37ed15841a9 100644
--- a/test/CodeGen/X86/seh-filter.ll
+++ b/test/CodeGen/X86/seh-filter.ll
@@ -1,14 +1,14 @@
 ; RUN: llc -O0 -mtriple=x86_64-windows-msvc < %s | FileCheck %s
 
 declare void @g()
-define void @f() {
+define void @f() personality i8* bitcast (i32 (...)* @__C_specific_handler to i8*) {
   invoke void @g() to label %return unwind label %lpad
 
 return:
   ret void
 
 lpad:
-  %ehptrs = landingpad {i8*, i32} personality i8* bitcast (i32 (...)* @__C_specific_handler to i8*)
+  %ehptrs = landingpad {i8*, i32}
     filter [0 x i8*] zeroinitializer
   call void @__cxa_call_unexpected(i8* null)
   unreachable
diff --git a/test/CodeGen/X86/seh-finally.ll b/test/CodeGen/X86/seh-finally.ll
index 00601dcad54..350cd932f48 100644
--- a/test/CodeGen/X86/seh-finally.ll
+++ b/test/CodeGen/X86/seh-finally.ll
@@ -1,10 +1,12 @@
-; RUN: llc -mtriple=x86_64-windows-msvc < %s | FileCheck %s
+; RUN: llc -mtriple=x86_64-windows-msvc < %s | FileCheck %s --check-prefix=X64
+; RUN: sed -e 's/__C_specific_handler/_except_handler3/' %s | \
+; RUN:        llc -mtriple=i686-windows-msvc | FileCheck %s --check-prefix=X86
 
 @str_recovered = internal unnamed_addr constant [10 x i8] c"recovered\00", align 1
 
 declare void @crash()
 
-define i32 @main() {
+define i32 @main() personality i8* bitcast (i32 (...)* @__C_specific_handler to i8*) {
 entry:
   invoke void @crash()
           to label %invoke.cont unwind label %lpad
@@ -15,7 +17,7 @@ invoke.cont:                                      ; preds = %entry
   ret i32 0
 
 lpad:                                             ; preds = %entry
-  %0 = landingpad { i8*, i32 } personality i8* bitcast (i32 (...)* @__C_specific_handler to i8*)
+  %0 = landingpad { i8*, i32 }
           cleanup
   %1 = extractvalue { i8*, i32 } %0, 0
   %2 = extractvalue { i8*, i32 } %0, 1
@@ -26,17 +28,38 @@ invoke.cont1:                                     ; preds = %lpad
   resume { i8*, i32 } %0
 
 terminate.lpad:                                   ; preds = %lpad
-  %3 = landingpad { i8*, i32 } personality i8* bitcast (i32 (...)* @__C_specific_handler to i8*)
+  %3 = landingpad { i8*, i32 }
           catch i8* null
   call void @abort()
   unreachable
 }
 
-; CHECK: main:
+; X64-LABEL: main:
+; X64: retq
 
-; FIXME: No handlers yet!
-; CHECK: .seh_handlerdata
-; CHECK-NEXT: .long 0
+; X64: .seh_handlerdata
+; X64-NEXT: .long 1
+; X64-NEXT: .long .Ltmp0@IMGREL
+; X64-NEXT: .long .Ltmp1@IMGREL
+; X64-NEXT: .long main.cleanup@IMGREL
+; X64-NEXT: .long 0
+
+; X64-LABEL: main.cleanup:
+; X64: callq puts
+; X64: retq
+
+; X86-LABEL: _main:
+; X86: retl
+
+; X86: .section .xdata,"dr"
+; X86: L__ehtable$main:
+; X86-NEXT: .long -1
+; X86-NEXT: .long 0
+; X86-NEXT: .long _main.cleanup
+
+; X86-LABEL: _main.cleanup:
+; X86: calll _puts
+; X86: retl
 
 declare i32 @__C_specific_handler(...)
 
diff --git a/test/CodeGen/X86/seh-safe-div-win32.ll b/test/CodeGen/X86/seh-safe-div-win32.ll
new file mode 100644
index 00000000000..b1bcde2c7ff
--- /dev/null
+++ b/test/CodeGen/X86/seh-safe-div-win32.ll
@@ -0,0 +1,176 @@
+; RUN: llc -mtriple i686-pc-windows-msvc < %s | FileCheck %s
+
+; This test case is also intended to be run manually as a complete functional
+; test. It should link, print something, and exit zero rather than crashing.
+; It is the hypothetical lowering of a C source program that looks like:
+;
+;   int safe_div(int *n, int *d) {
+;     int r;
+;     __try {
+;       __try {
+;         r = *n / *d;
+;       } __except(GetExceptionCode() == EXCEPTION_ACCESS_VIOLATION) {
+;         puts("EXCEPTION_ACCESS_VIOLATION");
+;         r = -1;
+;       }
+;     } __except(GetExceptionCode() == EXCEPTION_INT_DIVIDE_BY_ZERO) {
+;       puts("EXCEPTION_INT_DIVIDE_BY_ZERO");
+;       r = -2;
+;     }
+;     return r;
+;   }
+
+@str1 = internal constant [27 x i8] c"EXCEPTION_ACCESS_VIOLATION\00"
+@str2 = internal constant [29 x i8] c"EXCEPTION_INT_DIVIDE_BY_ZERO\00"
+
+define i32 @safe_div(i32* %n, i32* %d) personality i8* bitcast (i32 (...)* @_except_handler3 to i8*) {
+entry:
+  %r = alloca i32, align 4
+  store i32 42, i32* %r
+  invoke void @try_body(i32* %r, i32* %n, i32* %d)
+          to label %__try.cont unwind label %lpad
+
+lpad:
+  %vals = landingpad { i8*, i32 }
+          catch i8* bitcast (i32 ()* @safe_div_filt0 to i8*)
+          catch i8* bitcast (i32 ()* @safe_div_filt1 to i8*)
+  %ehptr = extractvalue { i8*, i32 } %vals, 0
+  %sel = extractvalue { i8*, i32 } %vals, 1
+  %filt0_val = call i32 @llvm.eh.typeid.for(i8* bitcast (i32 ()* @safe_div_filt0 to i8*))
+  %is_filt0 = icmp eq i32 %sel, %filt0_val
+  br i1 %is_filt0, label %handler0, label %eh.dispatch1
+
+eh.dispatch1:
+  %filt1_val = call i32 @llvm.eh.typeid.for(i8* bitcast (i32 ()* @safe_div_filt1 to i8*))
+  %is_filt1 = icmp eq i32 %sel, %filt1_val
+  br i1 %is_filt1, label %handler1, label %eh.resume
+
+handler0:
+  call void @puts(i8* getelementptr ([27 x i8], [27 x i8]* @str1, i32 0, i32 0))
+  store i32 -1, i32* %r, align 4
+  br label %__try.cont
+
+handler1:
+  call void @puts(i8* getelementptr ([29 x i8], [29 x i8]* @str2, i32 0, i32 0))
+  store i32 -2, i32* %r, align 4
+  br label %__try.cont
+
+eh.resume:
+  resume { i8*, i32 } %vals
+
+__try.cont:
+  %safe_ret = load i32, i32* %r, align 4
+  ret i32 %safe_ret
+}
+
+; Normal path code
+
+; CHECK: {{^}}_safe_div:
+; CHECK: movl $42, [[rloc:.*\(%ebp\)]]
+; CHECK: leal [[rloc]],
+; CHECK: calll _try_body
+; CHECK: [[cont_bb:LBB0_[0-9]+]]:
+; CHECK: movl [[rloc]], %eax
+; CHECK: retl
+
+; Landing pad code
+
+; CHECK: [[handler0:Ltmp[0-9]+]]: # Block address taken
+; CHECK: # %handler0
+; 	Restore SP
+; CHECK: movl {{.*}}(%ebp), %esp
+; CHECK: calll _puts
+; CHECK: jmp [[cont_bb]]
+
+; CHECK: [[handler1:Ltmp[0-9]+]]: # Block address taken
+; CHECK: # %handler1
+; 	Restore SP
+; CHECK: movl {{.*}}(%ebp), %esp
+; CHECK: calll _puts
+; CHECK: jmp [[cont_bb]]
+
+; CHECK: .section .xdata,"dr"
+; CHECK: L__ehtable$safe_div:
+; CHECK-NEXT: .long -1
+; CHECK-NEXT: .long _safe_div_filt1
+; CHECK-NEXT: .long [[handler1]]
+; CHECK-NEXT: .long 0
+; CHECK-NEXT: .long _safe_div_filt0
+; CHECK-NEXT: .long [[handler0]]
+
+define void @try_body(i32* %r, i32* %n, i32* %d) {
+entry:
+  %0 = load i32, i32* %n, align 4
+  %1 = load i32, i32* %d, align 4
+  %div = sdiv i32 %0, %1
+  store i32 %div, i32* %r, align 4
+  ret void
+}
+
+; The prototype of these filter functions is:
+; int filter(EXCEPTION_POINTERS *eh_ptrs, void *rbp);
+
+; The definition of EXCEPTION_POINTERS is:
+;   typedef struct _EXCEPTION_POINTERS {
+;     EXCEPTION_RECORD *ExceptionRecord;
+;     CONTEXT          *ContextRecord;
+;   } EXCEPTION_POINTERS;
+
+; The definition of EXCEPTION_RECORD is:
+;   typedef struct _EXCEPTION_RECORD {
+;     DWORD ExceptionCode;
+;     ...
+;   } EXCEPTION_RECORD;
+
+define i32 @safe_div_filt0() {
+  %ebp = call i8* @llvm.frameaddress(i32 1)
+  %eh_ptrs.addr.i8 = getelementptr inbounds i8, i8* %ebp, i32 -20
+  %eh_ptrs.addr = bitcast i8* %eh_ptrs.addr.i8 to i32***
+  %eh_ptrs = load i32**, i32*** %eh_ptrs.addr
+  %eh_rec = load i32*, i32** %eh_ptrs
+  %eh_code = load i32, i32* %eh_rec
+  ; EXCEPTION_ACCESS_VIOLATION = 0xC0000005
+  %cmp = icmp eq i32 %eh_code, 3221225477
+  %filt.res = zext i1 %cmp to i32
+  ret i32 %filt.res
+}
+define i32 @safe_div_filt1() {
+  %ebp = call i8* @llvm.frameaddress(i32 1)
+  %eh_ptrs.addr.i8 = getelementptr inbounds i8, i8* %ebp, i32 -20
+  %eh_ptrs.addr = bitcast i8* %eh_ptrs.addr.i8 to i32***
+  %eh_ptrs = load i32**, i32*** %eh_ptrs.addr
+  %eh_rec = load i32*, i32** %eh_ptrs
+  %eh_code = load i32, i32* %eh_rec
+  ; EXCEPTION_INT_DIVIDE_BY_ZERO = 0xC0000094
+  %cmp = icmp eq i32 %eh_code, 3221225620
+  %filt.res = zext i1 %cmp to i32
+  ret i32 %filt.res
+}
+
+@str_result = internal constant [21 x i8] c"safe_div result: %d\0A\00"
+
+define i32 @main() {
+  %d.addr = alloca i32, align 4
+  %n.addr = alloca i32, align 4
+
+  store i32 10, i32* %n.addr, align 4
+  store i32 2, i32* %d.addr, align 4
+  %r1 = call i32 @safe_div(i32* %n.addr, i32* %d.addr)
+  call void (i8*, ...) @printf(i8* getelementptr ([21 x i8], [21 x i8]* @str_result, i32 0, i32 0), i32 %r1)
+
+  store i32 10, i32* %n.addr, align 4
+  store i32 0, i32* %d.addr, align 4
+  %r2 = call i32 @safe_div(i32* %n.addr, i32* %d.addr)
+  call void (i8*, ...) @printf(i8* getelementptr ([21 x i8], [21 x i8]* @str_result, i32 0, i32 0), i32 %r2)
+
+  %r3 = call i32 @safe_div(i32* %n.addr, i32* null)
+  call void (i8*, ...) @printf(i8* getelementptr ([21 x i8], [21 x i8]* @str_result, i32 0, i32 0), i32 %r3)
+  ret i32 0
+}
+
+declare i32 @_except_handler3(...)
+declare i32 @llvm.eh.typeid.for(i8*) readnone nounwind
+declare void @puts(i8*)
+declare void @printf(i8*, ...)
+declare void @abort()
+declare i8* @llvm.frameaddress(i32)
diff --git a/test/CodeGen/X86/seh-safe-div.ll b/test/CodeGen/X86/seh-safe-div.ll
index ba54f1cca60..699e58ee8ba 100644
--- a/test/CodeGen/X86/seh-safe-div.ll
+++ b/test/CodeGen/X86/seh-safe-div.ll
@@ -23,14 +23,14 @@
 @str1 = internal constant [27 x i8] c"EXCEPTION_ACCESS_VIOLATION\00"
 @str2 = internal constant [29 x i8] c"EXCEPTION_INT_DIVIDE_BY_ZERO\00"
 
-define i32 @safe_div(i32* %n, i32* %d) {
+define i32 @safe_div(i32* %n, i32* %d) personality i8* bitcast (i32 (...)* @__C_specific_handler to i8*) {
 entry:
   %r = alloca i32, align 4
   invoke void @try_body(i32* %r, i32* %n, i32* %d)
           to label %__try.cont unwind label %lpad
 
 lpad:
-  %vals = landingpad { i8*, i32 } personality i8* bitcast (i32 (...)* @__C_specific_handler to i8*)
+  %vals = landingpad { i8*, i32 }
           catch i8* bitcast (i32 (i8*, i8*)* @safe_div_filt0 to i8*)
           catch i8* bitcast (i32 (i8*, i8*)* @safe_div_filt1 to i8*)
   %ehptr = extractvalue { i8*, i32 } %vals, 0
@@ -71,46 +71,34 @@ __try.cont:
 ; CHECK: leaq [[rloc:.*\(%rsp\)]], %rcx
 ; CHECK: callq try_body
 ; CHECK-NEXT: .Ltmp1
-; CHECK: .LBB0_7:
+; CHECK: [[cont_bb:\.LBB0_[0-9]+]]:
 ; CHECK: movl [[rloc]], %eax
 ; CHECK: retq
 
 ; Landing pad code
 
-; CHECK: .Ltmp3:
-; CHECK: movl $1, %[[sel:[a-z]+]]
-; CHECK: .Ltmp4
-; CHECK: movl $2, %[[sel]]
-; CHECK: .L{{.*}}:
-; CHECK: cmpl $1, %[[sel]]
-
+; CHECK: [[handler0:\.Ltmp[0-9]+]]: # Block address taken
 ; CHECK: # %handler0
 ; CHECK: callq puts
 ; CHECK: movl $-1, [[rloc]]
-; CHECK: jmp .LBB0_7
-
-; CHECK: cmpl $2, %[[sel]]
+; CHECK: jmp [[cont_bb]]
 
+; CHECK: [[handler1:\.Ltmp[0-9]+]]: # Block address taken
 ; CHECK: # %handler1
 ; CHECK: callq puts
 ; CHECK: movl $-2, [[rloc]]
-; CHECK: jmp .LBB0_7
-
-; FIXME: EH preparation should eliminate the 'resume' instr and we should not do
-; the previous 'cmp;jeq'.
-; CHECK-NOT: _Unwind_Resume
-; CHECK: ud2
+; CHECK: jmp [[cont_bb]]
 
 ; CHECK: .seh_handlerdata
-; CHECK: .long 2
-; CHECK: .long .Ltmp0@IMGREL
-; CHECK: .long .Ltmp1@IMGREL+1
-; CHECK: .long safe_div_filt0@IMGREL
-; CHECK: .long .Ltmp3@IMGREL
-; CHECK: .long .Ltmp0@IMGREL
-; CHECK: .long .Ltmp1@IMGREL+1
-; CHECK: .long safe_div_filt1@IMGREL
-; CHECK: .long .Ltmp4@IMGREL
+; CHECK-NEXT: .long 2
+; CHECK-NEXT: .long .Ltmp0@IMGREL
+; CHECK-NEXT: .long .Ltmp1@IMGREL+1
+; CHECK-NEXT: .long safe_div_filt0@IMGREL
+; CHECK-NEXT: .long [[handler0]]@IMGREL
+; CHECK-NEXT: .long .Ltmp0@IMGREL
+; CHECK-NEXT: .long .Ltmp1@IMGREL+1
+; CHECK-NEXT: .long safe_div_filt1@IMGREL
+; CHECK-NEXT: .long [[handler1]]@IMGREL
 ; CHECK: .text
 ; CHECK: .seh_endproc
 
@@ -185,11 +173,6 @@ define i32 @main() {
   ret i32 0
 }
 
-define void @_Unwind_Resume() {
-  call void @abort()
-  unreachable
-}
-
 declare i32 @__C_specific_handler(...)
 declare i32 @llvm.eh.typeid.for(i8*) readnone nounwind
 declare void @puts(i8*)
diff --git a/test/CodeGen/X86/seh-stack-realign-win32.ll b/test/CodeGen/X86/seh-stack-realign-win32.ll
new file mode 100644
index 00000000000..f3ab71803ca
--- /dev/null
+++ b/test/CodeGen/X86/seh-stack-realign-win32.ll
@@ -0,0 +1,99 @@
+; RUN: llc -mtriple=i686-windows-msvc < %s | FileCheck %s
+
+; 32-bit catch-all has to use a filter function because that's how it saves the
+; exception code.
+
+@str = linkonce_odr unnamed_addr constant [27 x i8] c"GetExceptionCode(): 0x%lx\0A\00", align 1
+
+declare i32 @_except_handler3(...)
+declare void @crash()
+declare i32 @printf(i8* nocapture readonly, ...) nounwind
+declare i32 @llvm.eh.typeid.for(i8*)
+declare i8* @llvm.frameaddress(i32)
+declare i8* @llvm.localrecover(i8*, i8*, i32)
+declare void @llvm.localescape(...)
+declare i8* @llvm.x86.seh.recoverfp(i8*, i8*)
+
+define i32 @main() personality i8* bitcast (i32 (...)* @_except_handler3 to i8*) {
+entry:
+  ; The EH code allocation is overaligned, triggering realignment.
+  %__exceptioncode = alloca i32, align 8
+  call void (...) @llvm.localescape(i32* %__exceptioncode)
+  invoke void @crash() #5
+          to label %__try.cont unwind label %lpad
+
+lpad:                                             ; preds = %entry
+  %0 = landingpad { i8*, i32 }
+          catch i8* bitcast (i32 ()* @"filt$main" to i8*)
+  %1 = extractvalue { i8*, i32 } %0, 1
+  %2 = call i32 @llvm.eh.typeid.for(i8* bitcast (i32 ()* @"filt$main" to i8*)) #4
+  %matches = icmp eq i32 %1, %2
+  br i1 %matches, label %__except, label %eh.resume
+
+__except:                                         ; preds = %lpad
+  %3 = load i32, i32* %__exceptioncode, align 4
+  %call = call i32 (i8*, ...) @printf(i8* getelementptr inbounds ([27 x i8], [27 x i8]* @str, i32 0, i32 0), i32 %3) #4
+  br label %__try.cont
+
+__try.cont:                                       ; preds = %entry, %__except
+  ret i32 0
+
+eh.resume:                                        ; preds = %lpad
+  resume { i8*, i32 } %0
+}
+
+define internal i32 @"filt$main"() {
+entry:
+  %ebp = tail call i8* @llvm.frameaddress(i32 1)
+  %parentfp = tail call i8* @llvm.x86.seh.recoverfp(i8* bitcast (i32 ()* @main to i8*), i8* %ebp)
+  %code.i8 = tail call i8* @llvm.localrecover(i8* bitcast (i32 ()* @main to i8*), i8* %parentfp, i32 0)
+  %__exceptioncode = bitcast i8* %code.i8 to i32*
+  %info.addr = getelementptr inbounds i8, i8* %ebp, i32 -20
+  %0 = bitcast i8* %info.addr to i32***
+  %1 = load i32**, i32*** %0, align 4
+  %2 = load i32*, i32** %1, align 4
+  %3 = load i32, i32* %2, align 4
+  store i32 %3, i32* %__exceptioncode, align 4
+  ret i32 1
+}
+
+; Check that we can get the exception code from eax to the printf.
+
+; CHECK-LABEL: _main:
+; CHECK: Lmain$frame_escape_0 = [[code_offs:[-0-9]+]]
+; CHECK: Lmain$frame_escape_1 = [[reg_offs:[-0-9]+]]
+; CHECK: movl %esp, [[reg_offs]](%esi)
+; CHECK: movl $L__ehtable$main,
+;       EH state 0
+; CHECK: movl $0, 40(%esi)
+; CHECK: calll _crash
+; CHECK: retl
+; CHECK: # Block address taken
+;       stackrestore
+; CHECK: movl -24(%ebp), %esp
+; CHECK: movl $Lmain$parent_frame_offset, %eax
+; CHECK: negl %eax
+; CHECK: leal -24(%ebp,%eax), %esi
+; CHECK: movl 12(%esi), %ebp    # 4-byte Reload
+;       EH state -1
+; CHECK: movl [[code_offs]](%esi), %[[code:[a-z]+]]
+; CHECK: movl $-1, 40(%esi)
+; CHECK-DAG: movl %[[code]], 4(%esp)
+; CHECK-DAG: movl $_str, (%esp)
+; CHECK: calll _printf
+
+; CHECK: .section .xdata,"dr"
+; CHECK: Lmain$parent_frame_offset = Lmain$frame_escape_1
+; CHECK: L__ehtable$main
+; CHECK-NEXT: .long -1
+; CHECK-NEXT: .long _filt$main
+; CHECK-NEXT: .long Ltmp{{[0-9]+}}
+
+; CHECK-LABEL: _filt$main:
+; CHECK: pushl %ebp
+; CHECK: movl %esp, %ebp
+; CHECK: movl (%ebp), %[[oldebp:[a-z]+]]
+; CHECK: movl -20(%[[oldebp]]), %[[ehinfo:[a-z]+]]
+; CHECK: movl (%[[ehinfo]]), %[[ehrec:[a-z]+]]
+; CHECK: movl (%[[ehrec]]), %[[ehcode:[a-z]+]]
+; CHECK: movl %[[ehcode]], {{.*}}(%{{.*}})
diff --git a/test/CodeGen/X86/seh-stack-realign.ll b/test/CodeGen/X86/seh-stack-realign.ll
new file mode 100644
index 00000000000..f2fb28a081f
--- /dev/null
+++ b/test/CodeGen/X86/seh-stack-realign.ll
@@ -0,0 +1,101 @@
+; RUN: llc -mtriple=i686-windows-msvc < %s | FileCheck %s
+
+; 32-bit catch-all has to use a filter function because that's how it saves the
+; exception code.
+
+@str = linkonce_odr unnamed_addr constant [27 x i8] c"GetExceptionCode(): 0x%lx\0A\00", align 1
+
+declare i32 @_except_handler3(...)
+declare void @crash()
+declare i32 @printf(i8* nocapture readonly, ...) nounwind
+declare i32 @llvm.eh.typeid.for(i8*)
+declare i8* @llvm.frameaddress(i32)
+declare i8* @llvm.localrecover(i8*, i8*, i32)
+declare void @llvm.localescape(...)
+declare i8* @llvm.x86.seh.recoverfp(i8*, i8*)
+
+define i32 @main() personality i8* bitcast (i32 (...)* @_except_handler3 to i8*) {
+entry:
+  ; The EH code allocation is overaligned, triggering realignment.
+  %__exceptioncode = alloca i32, align 8
+  call void (...) @llvm.localescape(i32* %__exceptioncode)
+  invoke void @crash() #5
+          to label %__try.cont unwind label %lpad
+
+lpad:                                             ; preds = %entry
+  %0 = landingpad { i8*, i32 }
+          catch i8* bitcast (i32 ()* @"filt$main" to i8*)
+  %1 = extractvalue { i8*, i32 } %0, 1
+  %2 = call i32 @llvm.eh.typeid.for(i8* bitcast (i32 ()* @"filt$main" to i8*)) #4
+  %matches = icmp eq i32 %1, %2
+  br i1 %matches, label %__except, label %eh.resume
+
+__except:                                         ; preds = %lpad
+  %3 = load i32, i32* %__exceptioncode, align 4
+  %call = call i32 (i8*, ...) @printf(i8* getelementptr inbounds ([27 x i8], [27 x i8]* @str, i32 0, i32 0), i32 %3) #4
+  br label %__try.cont
+
+__try.cont:                                       ; preds = %entry, %__except
+  ret i32 0
+
+eh.resume:                                        ; preds = %lpad
+  resume { i8*, i32 } %0
+}
+
+define internal i32 @"filt$main"() {
+entry:
+  %ebp = tail call i8* @llvm.frameaddress(i32 1)
+  %parentfp = tail call i8* @llvm.x86.seh.recoverfp(i8* bitcast (i32 ()* @main to i8*), i8* %ebp)
+  %code.i8 = tail call i8* @llvm.localrecover(i8* bitcast (i32 ()* @main to i8*), i8* %parentfp, i32 0)
+  %__exceptioncode = bitcast i8* %code.i8 to i32*
+  %info.addr = getelementptr inbounds i8, i8* %ebp, i32 -20
+  %0 = bitcast i8* %info.addr to i32***
+  %1 = load i32**, i32*** %0, align 4
+  %2 = load i32*, i32** %1, align 4
+  %3 = load i32, i32* %2, align 4
+  store i32 %3, i32* %__exceptioncode, align 4
+  ret i32 1
+}
+
+; Check that we can get the exception code from eax to the printf.
+
+; CHECK-LABEL: _main:
+; CHECK: Lmain$frame_escape_0 = [[code_offs:[-0-9]+]]
+; CHECK: Lmain$frame_escape_1 = [[reg_offs:[-0-9]+]]
+; CHECK: movl %esp, [[reg_offs]](%esi)
+; CHECK: movl $L__ehtable$main,
+;       EH state 0
+; CHECK: movl $0, 40(%esi)
+; CHECK: calll _crash
+; CHECK: retl
+; CHECK: # Block address taken
+;       Restore ESP
+; CHECK: movl -24(%ebp), %esp
+;       Restore ESI
+; CHECK: movl $Lmain$parent_frame_offset, %eax
+; CHECK: negl %eax
+; CHECK: leal -24(%ebp,%eax), %esi
+;       Restore EBP
+; CHECK: movl 12(%esi), %ebp    # 4-byte Reload
+;       EH state -1
+; CHECK: movl [[code_offs]](%esi), %[[code:[a-z]+]]
+; CHECK: movl $-1, 40(%esi)
+; CHECK-DAG: movl %[[code]], 4(%esp)
+; CHECK-DAG: movl $_str, (%esp)
+; CHECK: calll _printf
+
+; CHECK: .section .xdata,"dr"
+; CHECK: Lmain$parent_frame_offset = Lmain$frame_escape_1
+; CHECK: L__ehtable$main
+; CHECK-NEXT: .long -1
+; CHECK-NEXT: .long _filt$main
+; CHECK-NEXT: .long Ltmp{{[0-9]+}}
+
+; CHECK-LABEL: _filt$main:
+; CHECK: pushl %ebp
+; CHECK: movl %esp, %ebp
+; CHECK: movl (%ebp), %[[oldebp:[a-z]+]]
+; CHECK: movl -20(%[[oldebp]]), %[[ehinfo:[a-z]+]]
+; CHECK: movl (%[[ehinfo]]), %[[ehrec:[a-z]+]]
+; CHECK: movl (%[[ehrec]]), %[[ehcode:[a-z]+]]
+; CHECK: movl %[[ehcode]], {{.*}}(%{{.*}})
diff --git a/test/CodeGen/X86/setjmp-spills.ll b/test/CodeGen/X86/setjmp-spills.ll
index c35caae97af..43136e018c8 100644
--- a/test/CodeGen/X86/setjmp-spills.ll
+++ b/test/CodeGen/X86/setjmp-spills.ll
@@ -78,7 +78,7 @@ second:
 ; This is the same as above, but using "invoke" rather than "call" to
 ; call setjmp().
 
-define void @setjmp_invoker() {
+define void @setjmp_invoker() personality void ()* @personality {
 ; X86-32-LABEL: setjmp_invoker:
 ; X86-64-LABEL: setjmp_invoker:
   %a1 = call i32 @get_val()
@@ -103,7 +103,7 @@ cont:
   br i1 %setjmp_result, label %second, label %first
 
 lpad:
-  %lp = landingpad { i8*, i32 } personality void ()* @personality cleanup
+  %lp = landingpad { i8*, i32 } cleanup
   unreachable
 
 first:
diff --git a/test/CodeGen/X86/shift-avx2-crash.ll b/test/CodeGen/X86/shift-avx2-crash.ll
new file mode 100755
index 00000000000..b23d6e3359e
--- /dev/null
+++ b/test/CodeGen/X86/shift-avx2-crash.ll
@@ -0,0 +1,38 @@
+; RUN: llc < %s  -mcpu=core-avx2 > /dev/null
+; This test crashed on variable shift creation on AVX2
+
+target datalayout = "e-m:e-p:32:32-f64:32:64-f80:32-n8:16:32-S128"
+target triple = "i386-unknown-linux-gnu"
+
+define void @f_f(float* noalias nocapture %RET, float %aFOO, i32 %div)  {
+allocas:
+  %__idiv_table_u32_offset10_offset_load.i = add i64 0, -2
+  br label %if_then18.i
+
+if_then18.i:
+  %aFOO_load_to_uint32 = fptoui float %aFOO to i32
+  %aFOO_load_to_uint32_broadcast_init = insertelement <8 x i32> undef, i32 %aFOO_load_to_uint32, i32 0
+  %aFOO_load_to_uint32_broadcast = shufflevector <8 x i32> %aFOO_load_to_uint32_broadcast_init, <8 x i32> undef, <8 x i32> zeroinitializer
+
+  %multiplier_load_broadcast_init.i = insertelement <8 x i64> undef, i64 2, i32 0
+  %multiplier_load_broadcast.i = shufflevector <8 x i64> %multiplier_load_broadcast_init.i, <8 x i64> undef, <8 x i32> zeroinitializer
+  %numerator_load_to_uint64.i = zext <8 x i32> %aFOO_load_to_uint32_broadcast to <8 x i64>
+
+  ;if replace '%__idiv_table_u32_offset10_offset_load.i' with '-2' or remove 'if_then18.i' label the error disappears
+  %add__shift_load21.i = add i64 %__idiv_table_u32_offset10_offset_load.i, 32
+  %add__shift_load21_broadcast_init.i = insertelement <8 x i64> undef, i64 %add__shift_load21.i, i32 0
+  %add__shift_load21_broadcast.i = shufflevector <8 x i64> %add__shift_load21_broadcast_init.i, <8 x i64> undef, <8 x i32> zeroinitializer
+
+  %mul_val_load_mult_load.i = mul <8 x i64> %numerator_load_to_uint64.i, %multiplier_load_broadcast.i
+  %bitop22.i = lshr <8 x i64> %mul_val_load_mult_load.i, %add__shift_load21_broadcast.i
+  %bitop22_to_uint32.i = trunc <8 x i64> %bitop22.i to <8 x i32>
+  br label %__fast_idiv___UM_vyuunu.exit
+
+
+__fast_idiv___UM_vyuunu.exit:                   
+  %calltmp_to_float = uitofp <8 x i32> %bitop22_to_uint32.i to <8 x float>
+  %ptrcast = bitcast float* %RET to <8 x float>*
+  store <8 x float> %calltmp_to_float, <8 x float>* %ptrcast, align 4
+  ret void
+}
+
diff --git a/test/CodeGen/X86/shift-combine.ll b/test/CodeGen/X86/shift-combine.ll
index ec62bcdcdba..43301041a0b 100644
--- a/test/CodeGen/X86/shift-combine.ll
+++ b/test/CodeGen/X86/shift-combine.ll
@@ -17,3 +17,62 @@ entry:
   ret i32 %tmp5
 }
 
+define i32* @test_exact1(i32 %a, i32 %b, i32* %x)  {
+; CHECK-LABEL: test_exact1:
+; CHECK: sarl %
+
+  %sub = sub i32 %b, %a
+  %shr = ashr exact i32 %sub, 3
+  %gep = getelementptr inbounds i32, i32* %x, i32 %shr
+  ret i32* %gep
+}
+
+define i32* @test_exact2(i32 %a, i32 %b, i32* %x)  {
+; CHECK-LABEL: test_exact2:
+; CHECK: sarl %
+
+  %sub = sub i32 %b, %a
+  %shr = ashr exact i32 %sub, 3
+  %gep = getelementptr inbounds i32, i32* %x, i32 %shr
+  ret i32* %gep
+}
+
+define i32* @test_exact3(i32 %a, i32 %b, i32* %x)  {
+; CHECK-LABEL: test_exact3:
+; CHECK-NOT: sarl
+
+  %sub = sub i32 %b, %a
+  %shr = ashr exact i32 %sub, 2
+  %gep = getelementptr inbounds i32, i32* %x, i32 %shr
+  ret i32* %gep
+}
+
+define i32* @test_exact4(i32 %a, i32 %b, i32* %x)  {
+; CHECK-LABEL: test_exact4:
+; CHECK: shrl %
+
+  %sub = sub i32 %b, %a
+  %shr = lshr exact i32 %sub, 3
+  %gep = getelementptr inbounds i32, i32* %x, i32 %shr
+  ret i32* %gep
+}
+
+define i32* @test_exact5(i32 %a, i32 %b, i32* %x)  {
+; CHECK-LABEL: test_exact5:
+; CHECK: shrl %
+
+  %sub = sub i32 %b, %a
+  %shr = lshr exact i32 %sub, 3
+  %gep = getelementptr inbounds i32, i32* %x, i32 %shr
+  ret i32* %gep
+}
+
+define i32* @test_exact6(i32 %a, i32 %b, i32* %x)  {
+; CHECK-LABEL: test_exact6:
+; CHECK-NOT: shrl
+
+  %sub = sub i32 %b, %a
+  %shr = lshr exact i32 %sub, 2
+  %gep = getelementptr inbounds i32, i32* %x, i32 %shr
+  ret i32* %gep
+}
diff --git a/test/CodeGen/X86/shift-i256.ll b/test/CodeGen/X86/shift-i256.ll
index d5f65a6ed18..866e7e67fb0 100644
--- a/test/CodeGen/X86/shift-i256.ll
+++ b/test/CodeGen/X86/shift-i256.ll
@@ -1,9 +1,21 @@
-; RUN: llc < %s -march=x86
-; RUN: llc < %s -march=x86-64
+; RUN: llc < %s -march=x86        | FileCheck %s
+; RUN: llc < %s -march=x86-64 -O0 | FileCheck %s -check-prefix=CHECK-X64
+; RUN: llc < %s -march=x86-64 -O2 | FileCheck %s -check-prefix=CHECK-X64
 
-define void @t(i256 %x, i256 %a, i256* nocapture %r) nounwind readnone {
+; CHECK-LABEL: shift1
+define void @shift1(i256 %x, i256 %a, i256* nocapture %r) nounwind readnone {
 entry:
 	%0 = ashr i256 %x, %a
 	store i256 %0, i256* %r
         ret void
 }
+
+; CHECK-LABEL: shift2
+define i256 @shift2(i256 %c) nounwind
+{
+  %b = shl i256 1, %c  ; %c must not be a constant
+  ; Special case when %c is 0:
+  ; CHECK-X64: testb [[REG:%r[0-9]+b]], [[REG]]
+  ; CHECK-X64: cmoveq
+  ret i256 %b
+}
diff --git a/test/CodeGen/X86/sibcall-win64.ll b/test/CodeGen/X86/sibcall-win64.ll
index f7038726f9c..204e1f8b050 100644
--- a/test/CodeGen/X86/sibcall-win64.ll
+++ b/test/CodeGen/X86/sibcall-win64.ll
@@ -1,7 +1,11 @@
 ; RUN: llc < %s -mtriple=x86_64-pc-linux | FileCheck %s
 
 declare x86_64_win64cc void @win64_callee(i32)
+declare x86_64_win64cc void (i32)* @win64_indirect()
+declare x86_64_win64cc void @win64_other(i32)
 declare void @sysv_callee(i32)
+declare void (i32)* @sysv_indirect()
+declare void @sysv_other(i32)
 
 define void @sysv_caller(i32 %p1) {
 entry:
@@ -40,3 +44,23 @@ define x86_64_win64cc void @win64_matched(i32 %p1) {
 
 ; CHECK-LABEL: win64_matched:
 ; CHECK: jmp win64_callee # TAILCALL
+
+define x86_64_win64cc void @win64_indirect_caller(i32 %p1) {
+  %1 = call x86_64_win64cc void (i32)* @win64_indirect()
+  call x86_64_win64cc void @win64_other(i32 0)
+  tail call x86_64_win64cc void %1(i32 %p1)
+  ret void
+}
+
+; CHECK-LABEL: win64_indirect_caller:
+; CHECK: jmpq *%{{rax|rcx|rdx|r8|r9|r11}} # TAILCALL
+
+define void @sysv_indirect_caller(i32 %p1) {
+  %1 = call void (i32)* @sysv_indirect()
+  call void @sysv_other(i32 0)
+  tail call void %1(i32 %p1)
+  ret void
+}
+
+; CHECK-LABEL: sysv_indirect_caller:
+; CHECK: jmpq *%{{rax|rcx|rdx|rsi|rdi|r8|r9|r11}} # TAILCALL
diff --git a/test/CodeGen/X86/sink-hoist.ll b/test/CodeGen/X86/sink-hoist.ll
index 2f70a83f528..972fbdf48cb 100644
--- a/test/CodeGen/X86/sink-hoist.ll
+++ b/test/CodeGen/X86/sink-hoist.ll
@@ -9,7 +9,7 @@
 ; CHECK-NEXT: testb $1, %dil
 ; CHECK-NEXT: jne
 ; CHECK-NEXT: divsd
-; CHECK-NEXT: movaps
+; CHECK-NEXT: movapd
 ; CHECK-NEXT: ret
 ; CHECK:      divsd
 
@@ -28,7 +28,7 @@ define double @foo(double %x, double %y, i1 %c) nounwind {
 ; CHECK-NEXT: testb $1, %dil
 ; CHECK-NEXT: je
 ; CHECK:      divsd
-; CHECK:      movaps
+; CHECK:      movapd
 ; CHECK:      ret
 define double @split(double %x, double %y, i1 %c) nounwind {
   %a = fdiv double %x, 3.2
diff --git a/test/CodeGen/X86/soft-fp.ll b/test/CodeGen/X86/soft-fp.ll
index 79e5ebb5fca..fa38d1044a4 100644
--- a/test/CodeGen/X86/soft-fp.ll
+++ b/test/CodeGen/X86/soft-fp.ll
@@ -1,7 +1,9 @@
-; RUN: llc < %s -march=x86    -mattr=+sse2 -soft-float | not grep xmm
-; RUN: llc < %s -march=x86-64 -mattr=+sse2 -soft-float | not grep xmm
+; RUN: llc < %s -march=x86    -mattr=+sse2,+soft-float | FileCheck %s
+; RUN: llc < %s -march=x86-64 -mattr=+sse2,+soft-float | FileCheck %s
 
-	%struct.__va_list_tag = type { i32, i32, i8*, i8* }
+; CHECK-NOT: xmm{[0-9]+}
+
+%struct.__va_list_tag = type { i32, i32, i8*, i8* }
 
 define i32 @t1(i32 %a, ...) nounwind {
 entry:
diff --git a/test/CodeGen/X86/splat-for-size.ll b/test/CodeGen/X86/splat-for-size.ll
index c052ad2aa14..635aa821d78 100644
--- a/test/CodeGen/X86/splat-for-size.ll
+++ b/test/CodeGen/X86/splat-for-size.ll
@@ -138,4 +138,18 @@ define <32 x i8> @splat_v32i8(<32 x i8> %x) #0 {
 ; CHECK: retq
 }
 
+; PR23259: Verify that ISel doesn't crash with a 'fatal error in backend'
+; due to a missing AVX pattern to select a v2i64 X86ISD::BROADCAST of a
+; loadi64 with multiple uses.
+
+@A = common global <3 x i64> zeroinitializer, align 32
+
+define <8 x i64> @pr23259() #0 {
+entry:
+  %0 = load <4 x i64>, <4 x i64>* bitcast (<3 x i64>* @A to <4 x i64>*), align 32
+  %1 = shufflevector <4 x i64> %0, <4 x i64> undef, <3 x i32> <i32 undef, i32 undef, i32 2>
+  %shuffle = shufflevector <3 x i64> <i64 1, i64 undef, i64 undef>, <3 x i64> %1, <8 x i32> <i32 5, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0>
+  ret <8 x i64> %shuffle
+}
+
 attributes #0 = { optsize }
diff --git a/test/CodeGen/X86/split-eh-lpad-edges.ll b/test/CodeGen/X86/split-eh-lpad-edges.ll
index 756a3ddc86b..82dd3b7674f 100644
--- a/test/CodeGen/X86/split-eh-lpad-edges.ll
+++ b/test/CodeGen/X86/split-eh-lpad-edges.ll
@@ -10,13 +10,13 @@
 	%struct.objc_selector = type opaque
 @"\01l_objc_msgSend_fixup_alloc" = external global %struct._message_ref_t, align 16		; <%struct._message_ref_t*> [#uses=2]
 
-define %struct.NSArray* @newFetchedRowsForFetchPlan_MT(%struct.FetchPlanHeader* %fetchPlan, %struct.objc_selector* %selectionMethod, %struct.NSObject* %selectionParameter) ssp {
+define %struct.NSArray* @newFetchedRowsForFetchPlan_MT(%struct.FetchPlanHeader* %fetchPlan, %struct.objc_selector* %selectionMethod, %struct.NSObject* %selectionParameter) ssp personality i32 (...)* @__gxx_personality_v0 {
 entry:
 	%0 = invoke %struct.NSObject* null(%struct.NSObject* null, %struct._message_ref_t* @"\01l_objc_msgSend_fixup_alloc")
 			to label %invcont unwind label %lpad		; <%struct.NSObject*> [#uses=1]
 
 invcont:		; preds = %entry
-	%1 = invoke %struct.NSObject* (%struct.NSObject*, %struct.objc_selector*, ...)* @objc_msgSend(%struct.NSObject* %0, %struct.objc_selector* null)
+	%1 = invoke %struct.NSObject* (%struct.NSObject*, %struct.objc_selector*, ...) @objc_msgSend(%struct.NSObject* %0, %struct.objc_selector* null)
 			to label %invcont26 unwind label %lpad		; <%struct.NSObject*> [#uses=0]
 
 invcont26:		; preds = %invcont
@@ -28,7 +28,7 @@ invcont27:		; preds = %invcont26
 
 lpad:		; preds = %invcont26, %invcont, %entry
 	%pool.1 = phi %struct.NSAutoreleasePool* [ null, %entry ], [ null, %invcont ], [ null, %invcont26 ]		; <%struct.NSAutoreleasePool*> [#uses=0]
-        %exn = landingpad {i8*, i32} personality i32 (...)* @__gxx_personality_v0
+        %exn = landingpad {i8*, i32}
                  cleanup
 	unreachable
 }
diff --git a/test/CodeGen/X86/sqrt-fastmath.ll b/test/CodeGen/X86/sqrt-fastmath.ll
index 4c6b521156e..9b851db8121 100644
--- a/test/CodeGen/X86/sqrt-fastmath.ll
+++ b/test/CodeGen/X86/sqrt-fastmath.ll
@@ -1,5 +1,5 @@
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=sse2 | FileCheck %s
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=avx,use-sqrt-est | FileCheck %s --check-prefix=ESTIMATE
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=sse2 -recip=!sqrtf,!vec-sqrtf,!divf,!vec-divf | FileCheck %s --check-prefix=NORECIP
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=avx -recip=sqrtf,vec-sqrtf | FileCheck %s --check-prefix=ESTIMATE
 
 declare double @__sqrt_finite(double) #0
 declare float @__sqrtf_finite(float) #0
@@ -10,10 +10,10 @@ declare <8 x float> @llvm.sqrt.v8f32(<8 x float>) #0
 
 
 define double @fd(double %d) #0 {
-; CHECK-LABEL: fd:
-; CHECK:       # BB#0:
-; CHECK-NEXT:    sqrtsd %xmm0, %xmm0
-; CHECK-NEXT:    retq
+; NORECIP-LABEL: fd:
+; NORECIP:       # BB#0:
+; NORECIP-NEXT:    sqrtsd %xmm0, %xmm0
+; NORECIP-NEXT:    retq
 ;
 ; ESTIMATE-LABEL: fd:
 ; ESTIMATE:       # BB#0:
@@ -25,20 +25,20 @@ define double @fd(double %d) #0 {
 
 
 define float @ff(float %f) #0 {
-; CHECK-LABEL: ff:
-; CHECK:       # BB#0:
-; CHECK-NEXT:    sqrtss %xmm0, %xmm0
-; CHECK-NEXT:    retq
+; NORECIP-LABEL: ff:
+; NORECIP:       # BB#0:
+; NORECIP-NEXT:    sqrtss %xmm0, %xmm0
+; NORECIP-NEXT:    retq
 ;
 ; ESTIMATE-LABEL: ff:
 ; ESTIMATE:       # BB#0:
 ; ESTIMATE-NEXT:    vrsqrtss %xmm0, %xmm0, %xmm1
 ; ESTIMATE-NEXT:    vmulss {{.*}}(%rip), %xmm1, %xmm2
-; ESTIMATE-NEXT:    vmulss %xmm1, %xmm1, %xmm1
-; ESTIMATE-NEXT:    vmulss %xmm0, %xmm1, %xmm1
+; ESTIMATE-NEXT:    vmulss %xmm0, %xmm1, %xmm3
+; ESTIMATE-NEXT:    vmulss %xmm3, %xmm1, %xmm1
 ; ESTIMATE-NEXT:    vaddss {{.*}}(%rip), %xmm1, %xmm1
+; ESTIMATE-NEXT:    vmulss %xmm0, %xmm2, %xmm2
 ; ESTIMATE-NEXT:    vmulss %xmm2, %xmm1, %xmm1
-; ESTIMATE-NEXT:    vmulss %xmm1, %xmm0, %xmm1
 ; ESTIMATE-NEXT:    vxorps %xmm2, %xmm2, %xmm2
 ; ESTIMATE-NEXT:    vcmpeqss %xmm2, %xmm0, %xmm0
 ; ESTIMATE-NEXT:    vandnps %xmm1, %xmm0, %xmm0
@@ -49,11 +49,11 @@ define float @ff(float %f) #0 {
 
 
 define x86_fp80 @fld(x86_fp80 %ld) #0 {
-; CHECK-LABEL: fld:
-; CHECK:       # BB#0:
-; CHECK-NEXT:    fldt {{[0-9]+}}(%rsp)
-; CHECK-NEXT:    fsqrt
-; CHECK-NEXT:    retq
+; NORECIP-LABEL: fld:
+; NORECIP:       # BB#0:
+; NORECIP-NEXT:    fldt {{[0-9]+}}(%rsp)
+; NORECIP-NEXT:    fsqrt
+; NORECIP-NEXT:    retq
 ;
 ; ESTIMATE-LABEL: fld:
 ; ESTIMATE:       # BB#0:
@@ -67,18 +67,18 @@ define x86_fp80 @fld(x86_fp80 %ld) #0 {
 
 
 define float @reciprocal_square_root(float %x) #0 {
-; CHECK-LABEL: reciprocal_square_root:
-; CHECK:       # BB#0:
-; CHECK-NEXT:    sqrtss %xmm0, %xmm1
-; CHECK-NEXT:    movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
-; CHECK-NEXT:    divss %xmm1, %xmm0
-; CHECK-NEXT:    retq
+; NORECIP-LABEL: reciprocal_square_root:
+; NORECIP:       # BB#0:
+; NORECIP-NEXT:    sqrtss %xmm0, %xmm1
+; NORECIP-NEXT:    movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; NORECIP-NEXT:    divss %xmm1, %xmm0
+; NORECIP-NEXT:    retq
 ;
 ; ESTIMATE-LABEL: reciprocal_square_root:
 ; ESTIMATE:       # BB#0:
 ; ESTIMATE-NEXT:    vrsqrtss %xmm0, %xmm0, %xmm1
 ; ESTIMATE-NEXT:    vmulss {{.*}}(%rip), %xmm1, %xmm2
-; ESTIMATE-NEXT:    vmulss %xmm1, %xmm1, %xmm1
+; ESTIMATE-NEXT:    vmulss %xmm0, %xmm1, %xmm0
 ; ESTIMATE-NEXT:    vmulss %xmm0, %xmm1, %xmm0
 ; ESTIMATE-NEXT:    vaddss {{.*}}(%rip), %xmm0, %xmm0
 ; ESTIMATE-NEXT:    vmulss %xmm2, %xmm0, %xmm0
@@ -89,12 +89,12 @@ define float @reciprocal_square_root(float %x) #0 {
 }
 
 define <4 x float> @reciprocal_square_root_v4f32(<4 x float> %x) #0 {
-; CHECK-LABEL: reciprocal_square_root_v4f32:
-; CHECK:       # BB#0:
-; CHECK-NEXT:    sqrtps %xmm0, %xmm1
-; CHECK-NEXT:    movaps {{.*#+}} xmm0 = [1.000000e+00,1.000000e+00,1.000000e+00,1.000000e+00]
-; CHECK-NEXT:    divps %xmm1, %xmm0
-; CHECK-NEXT:    retq
+; NORECIP-LABEL: reciprocal_square_root_v4f32:
+; NORECIP:       # BB#0:
+; NORECIP-NEXT:    sqrtps %xmm0, %xmm1
+; NORECIP-NEXT:    movaps {{.*#+}} xmm0 = [1.000000e+00,1.000000e+00,1.000000e+00,1.000000e+00]
+; NORECIP-NEXT:    divps %xmm1, %xmm0
+; NORECIP-NEXT:    retq
 ;
 ; ESTIMATE-LABEL: reciprocal_square_root_v4f32:
 ; ESTIMATE:       # BB#0:
@@ -111,15 +111,15 @@ define <4 x float> @reciprocal_square_root_v4f32(<4 x float> %x) #0 {
 }
 
 define <8 x float> @reciprocal_square_root_v8f32(<8 x float> %x) #0 {
-; CHECK-LABEL: reciprocal_square_root_v8f32:
-; CHECK:       # BB#0:
-; CHECK-NEXT:    sqrtps %xmm1, %xmm2
-; CHECK-NEXT:    sqrtps %xmm0, %xmm3
-; CHECK-NEXT:    movaps {{.*#+}} xmm1 = [1.000000e+00,1.000000e+00,1.000000e+00,1.000000e+00]
-; CHECK-NEXT:    movaps %xmm1, %xmm0
-; CHECK-NEXT:    divps %xmm3, %xmm0
-; CHECK-NEXT:    divps %xmm2, %xmm1
-; CHECK-NEXT:    retq
+; NORECIP-LABEL: reciprocal_square_root_v8f32:
+; NORECIP:       # BB#0:
+; NORECIP-NEXT:    sqrtps %xmm1, %xmm2
+; NORECIP-NEXT:    sqrtps %xmm0, %xmm3
+; NORECIP-NEXT:    movaps {{.*#+}} xmm1 = [1.000000e+00,1.000000e+00,1.000000e+00,1.000000e+00]
+; NORECIP-NEXT:    movaps %xmm1, %xmm0
+; NORECIP-NEXT:    divps %xmm3, %xmm0
+; NORECIP-NEXT:    divps %xmm2, %xmm1
+; NORECIP-NEXT:    retq
 ;
 ; ESTIMATE-LABEL: reciprocal_square_root_v8f32:
 ; ESTIMATE:       # BB#0:
diff --git a/test/CodeGen/X86/sret-implicit.ll b/test/CodeGen/X86/sret-implicit.ll
index 56809525407..2a998fc6b6c 100644
--- a/test/CodeGen/X86/sret-implicit.ll
+++ b/test/CodeGen/X86/sret-implicit.ll
@@ -1,12 +1,34 @@
-; RUN: llc -mtriple=x86_64-apple-darwin8 < %s | FileCheck %s
-; RUN: llc -mtriple=x86_64-pc-linux < %s | FileCheck %s
-; RUN: llc -mtriple=x86_64-apple-darwin8 -terminal-rule < %s | FileCheck %s
-; RUN: llc -mtriple=x86_64-pc-linux -terminal-rule < %s | FileCheck %s
-
-; CHECK-LABEL: return32
-; CHECK-DAG: movq	$0, (%rdi)
-; CHECK-DAG: movq	%rdi, %rax
-; CHECK: retq
-define i256 @return32() {
+; RUN: llc -mtriple=x86_64-apple-darwin8 < %s | FileCheck %s --check-prefix=X64
+; RUN: llc -mtriple=x86_64-pc-linux < %s | FileCheck %s --check-prefix=X64
+; RUN: llc -mtriple=i686-pc-linux < %s | FileCheck %s --check-prefix=X86
+; RUN: llc -mtriple=x86_64-apple-darwin8 -terminal-rule < %s | FileCheck %s --check-prefix=X64
+; RUN: llc -mtriple=x86_64-pc-linux -terminal-rule < %s | FileCheck %s --check-prefix=X64
+
+define void @sret_void(i32* sret %p) {
+  store i32 0, i32* %p
+  ret void
+}
+
+; X64-LABEL: sret_void
+; X64-DAG: movl $0, (%rdi)
+; X64-DAG: movq %rdi, %rax
+; X64: retq
+
+; X86-LABEL: sret_void
+; X86: movl 4(%esp), %eax
+; X86: movl $0, (%eax)
+; X86: retl
+
+define i256 @sret_demoted() {
   ret i256 0
 }
+
+; X64-LABEL: sret_demoted
+; X64-DAG: movq $0, (%rdi)
+; X64-DAG: movq %rdi, %rax
+; X64: retq
+
+; X86-LABEL: sret_demoted
+; X86: movl 4(%esp), %eax
+; X86: movl $0, (%eax)
+; X86: retl
diff --git a/test/CodeGen/X86/sse-fcopysign.ll b/test/CodeGen/X86/sse-fcopysign.ll
index 0e0e4a9a86c..8a5462bea82 100644
--- a/test/CodeGen/X86/sse-fcopysign.ll
+++ b/test/CodeGen/X86/sse-fcopysign.ll
@@ -1,16 +1,134 @@
-; RUN: llc < %s -march=x86 -mattr=+sse2 | not grep test
+; RUN: llc < %s -mtriple=i686-unknown -mattr=+sse2 | FileCheck %s --check-prefix=ALL --check-prefix=X32
+; RUN: llc < %s -mtriple=x86_64-unknown -mattr=+sse2 | FileCheck %s --check-prefix=ALL --check-prefix=X64
+
+;
+; Library Functions
+;
 
 define float @tst1(float %a, float %b) {
-	%tmp = tail call float @copysignf( float %b, float %a )
-	ret float %tmp
+; X32-LABEL: @tst1
+; X32:       movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; X32-NEXT:  movss {{.*#+}} xmm1 = mem[0],zero,zero,zero
+; X32-NEXT:  movss %xmm1, 4(%esp)
+; X32-NEXT:  movss %xmm0, (%esp)
+; X32-NEXT:  calll copysignf
+; X32-NEXT:  addl $8, %esp
+; X32-NEXT:  retl
+;
+; X64-LABEL: @tst1
+; X64:       movaps  %xmm0, %xmm2
+; X64-NEXT:  movaps  %xmm1, %xmm0
+; X64-NEXT:  movaps  %xmm2, %xmm1
+; X64-NEXT:  jmp copysignf
+  %tmp = tail call float @copysignf( float %b, float %a )
+  ret float %tmp
 }
 
 define double @tst2(double %a, float %b, float %c) {
-	%tmp1 = fadd float %b, %c
-	%tmp2 = fpext float %tmp1 to double
-	%tmp = tail call double @copysign( double %a, double %tmp2 )
-	ret double %tmp
+; X32-LABEL: @tst2
+; X32:       movsd {{.*#+}} xmm0 = mem[0],zero
+; X32-NEXT:  movss {{.*#+}} xmm1 = mem[0],zero,zero,zero
+; X32-NEXT:  addss 32(%esp), %xmm1
+; X32-NEXT:  cvtss2sd %xmm1, %xmm1
+; X32-NEXT:  movsd %xmm0, (%esp)
+; X32-NEXT:  movsd %xmm1, 8(%esp)
+; X32-NEXT:  calll copysign
+; X32-NEXT:  addl $16, %esp
+; X32-NEXT:  retl
+;
+; X64-LABEL: @tst2
+; X64:       addss   %xmm2, %xmm1
+; X64-NEXT:  cvtss2sd        %xmm1, %xmm1
+; X64-NEXT:  jmp copysign
+  %tmp1 = fadd float %b, %c
+  %tmp2 = fpext float %tmp1 to double
+  %tmp = tail call double @copysign( double %a, double %tmp2 )
+  ret double %tmp
 }
 
 declare float @copysignf(float, float)
 declare double @copysign(double, double)
+
+;
+; LLVM Intrinsic
+;
+
+define float @int1(float %a, float %b) {
+; X32-LABEL: @int1
+; X32:       movss  8(%esp), %xmm0 {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; X32-NEXT:  andps .LCPI2_0, %xmm0
+; X32-NEXT:  movss 12(%esp), %xmm1 {{.*#+}} xmm1 = mem[0],zero,zero,zero
+; X32-NEXT:  andps .LCPI2_1, %xmm1
+; X32-NEXT:  orps  %xmm0, %xmm1
+; X32-NEXT:  movss %xmm1, (%esp)
+; X32-NEXT:  flds  (%esp)
+; X32-NEXT:  popl %eax
+; X32-NEXT:  retl
+;
+; X64-LABEL: @int1
+; X64:       andps .LCPI2_0(%rip), %xmm0
+; X64-NEXT:  andps .LCPI2_1(%rip), %xmm1
+; X64-NEXT:  orps  %xmm1, %xmm0
+; X64-NEXT:  retq
+  %tmp = tail call float @llvm.copysign.f32( float %b, float %a )
+  ret float %tmp
+}
+
+define double @int2(double %a, float %b, float %c) {
+; X32-LABEL: @int2
+; X32:       movss 16(%ebp), %xmm0 {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; X32-NEXT:  addss 20(%ebp), %xmm0
+; X32-NEXT:  movsd  8(%ebp), %xmm1 {{.*#+}} xmm1 = mem[0],zero
+; X32-NEXT:  andpd .LCPI3_0, %xmm1
+; X32-NEXT:  cvtss2sd %xmm0, %xmm0
+; X32-NEXT:  andpd .LCPI3_1, %xmm0
+; X32-NEXT:  orpd  %xmm1, %xmm0
+; X32-NEXT:  movlpd %xmm0, (%esp)
+; X32-NEXT:  fldl  (%esp)
+; X32-NEXT:  movl %ebp, %esp
+; X32-NEXT:  popl %ebp
+; X32-NEXT:  retl
+;
+; X64-LABEL: @int2
+; X64:       addss %xmm2, %xmm1
+; X64-NEXT:  cvtss2sd %xmm1, %xmm1
+; X64-NEXT:  andpd .LCPI3_0(%rip), %xmm1
+; X64-NEXT:  andpd .LCPI3_1(%rip), %xmm0
+; X64-NEXT:  orpd %xmm1, %xmm0
+; X64-NEXT:  retq
+  %tmp1 = fadd float %b, %c
+  %tmp2 = fpext float %tmp1 to double
+  %tmp = tail call double @llvm.copysign.f64( double %a, double %tmp2 )
+  ret double %tmp
+}
+
+define float @cst1() {
+; X32-LABEL: @cst1
+; X32:       fld1
+; X32-NEXT:  fchs
+; X32-NEXT:  retl
+;
+; X64-LABEL: @cst1
+; X64:       movss .LCPI4_0(%rip), %xmm0 {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; X64-NEXT:  retq
+  %tmp = tail call float @llvm.copysign.f32( float 1.0, float -2.0 )
+  ret float %tmp
+}
+
+define double @cst2() {
+; X32-LABEL: @cst2
+; X32:       fldz
+; X32-NEXT:  fchs
+; X32-NEXT:  retl
+;
+; X64-LABEL: @cst2
+; X64:       movsd .LCPI5_0(%rip), %xmm0 {{.*#+}} xmm0 = mem[0],zero
+; X64-NEXT:  retq
+  %tmp1 = fadd float -1.0, -1.0
+  %tmp2 = fpext float %tmp1 to double
+  %tmp = tail call double @llvm.copysign.f64( double 0.0, double %tmp2 )
+  ret double %tmp
+}
+
+declare float     @llvm.copysign.f32(float  %Mag, float  %Sgn)
+declare double    @llvm.copysign.f64(double %Mag, double %Sgn)
diff --git a/test/CodeGen/X86/sse-intrinsics-x86.ll b/test/CodeGen/X86/sse-intrinsics-x86.ll
index 65d44bfb5ba..0857189be73 100644
--- a/test/CodeGen/X86/sse-intrinsics-x86.ll
+++ b/test/CodeGen/X86/sse-intrinsics-x86.ll
@@ -1,4 +1,5 @@
 ; RUN: llc < %s -mtriple=i386-apple-darwin -mattr=-avx,+sse | FileCheck %s
+; RUN: llc < %s -mtriple=i386-apple-darwin -mcpu=knl | FileCheck %s
 
 define <4 x float> @test_x86_sse_add_ss(<4 x float> %a0, <4 x float> %a1) {
   ; CHECK: addss
diff --git a/test/CodeGen/X86/sse-minmax.ll b/test/CodeGen/X86/sse-minmax.ll
index 4dcb54ca4b0..e4d0373299f 100644
--- a/test/CodeGen/X86/sse-minmax.ll
+++ b/test/CodeGen/X86/sse-minmax.ll
@@ -805,7 +805,7 @@ define double @ule_inverse_y(double %x) nounwind {
 ; CHECK-LABEL: clampTo3k_a:
 ; CHECK-NEXT: movsd {{[^,]*}}, %xmm1
 ; CHECK-NEXT: minsd %xmm0, %xmm1
-; CHECK-NEXT: movaps %xmm1, %xmm0
+; CHECK-NEXT: movapd %xmm1, %xmm0
 ; CHECK-NEXT: ret
 ; UNSAFE-LABEL: clampTo3k_a:
 ; UNSAFE-NEXT: minsd {{[^,]*}}, %xmm0
@@ -813,7 +813,7 @@ define double @ule_inverse_y(double %x) nounwind {
 ; FINITE-LABEL: clampTo3k_a:
 ; FINITE-NEXT: movsd {{[^,]*}}, %xmm1
 ; FINITE-NEXT: minsd %xmm0, %xmm1
-; FINITE-NEXT: movaps %xmm1, %xmm0
+; FINITE-NEXT: movapd %xmm1, %xmm0
 ; FINITE-NEXT: ret
 define double @clampTo3k_a(double %x) nounwind readnone {
 entry:
@@ -831,7 +831,7 @@ entry:
 ; FINITE-LABEL: clampTo3k_b:
 ; FINITE-NEXT: movsd {{[^,]*}}, %xmm1
 ; FINITE-NEXT: minsd %xmm0, %xmm1
-; FINITE-NEXT: movaps %xmm1, %xmm0
+; FINITE-NEXT: movapd %xmm1, %xmm0
 ; FINITE-NEXT: ret
 define double @clampTo3k_b(double %x) nounwind readnone {
 entry:
@@ -843,7 +843,7 @@ entry:
 ; CHECK-LABEL: clampTo3k_c:
 ; CHECK-NEXT: movsd {{[^,]*}}, %xmm1
 ; CHECK-NEXT: maxsd %xmm0, %xmm1
-; CHECK-NEXT: movaps %xmm1, %xmm0
+; CHECK-NEXT: movapd %xmm1, %xmm0
 ; CHECK-NEXT: ret
 ; UNSAFE-LABEL: clampTo3k_c:
 ; UNSAFE-NEXT: maxsd {{[^,]*}}, %xmm0
@@ -851,7 +851,7 @@ entry:
 ; FINITE-LABEL: clampTo3k_c:
 ; FINITE-NEXT: movsd {{[^,]*}}, %xmm1
 ; FINITE-NEXT: maxsd %xmm0, %xmm1
-; FINITE-NEXT: movaps %xmm1, %xmm0
+; FINITE-NEXT: movapd %xmm1, %xmm0
 ; FINITE-NEXT: ret
 define double @clampTo3k_c(double %x) nounwind readnone {
 entry:
@@ -869,7 +869,7 @@ entry:
 ; FINITE-LABEL: clampTo3k_d:
 ; FINITE-NEXT: movsd {{[^,]*}}, %xmm1
 ; FINITE-NEXT: maxsd %xmm0, %xmm1
-; FINITE-NEXT: movaps %xmm1, %xmm0
+; FINITE-NEXT: movapd %xmm1, %xmm0
 ; FINITE-NEXT: ret
 define double @clampTo3k_d(double %x) nounwind readnone {
 entry:
@@ -881,7 +881,7 @@ entry:
 ; CHECK-LABEL: clampTo3k_e:
 ; CHECK-NEXT: movsd {{[^,]*}}, %xmm1
 ; CHECK-NEXT: maxsd %xmm0, %xmm1
-; CHECK-NEXT: movaps %xmm1, %xmm0
+; CHECK-NEXT: movapd %xmm1, %xmm0
 ; CHECK-NEXT: ret
 ; UNSAFE-LABEL: clampTo3k_e:
 ; UNSAFE-NEXT: maxsd {{[^,]*}}, %xmm0
@@ -889,7 +889,7 @@ entry:
 ; FINITE-LABEL: clampTo3k_e:
 ; FINITE-NEXT: movsd {{[^,]*}}, %xmm1
 ; FINITE-NEXT: maxsd %xmm0, %xmm1
-; FINITE-NEXT: movaps %xmm1, %xmm0
+; FINITE-NEXT: movapd %xmm1, %xmm0
 ; FINITE-NEXT: ret
 define double @clampTo3k_e(double %x) nounwind readnone {
 entry:
@@ -907,7 +907,7 @@ entry:
 ; FINITE-LABEL: clampTo3k_f:
 ; FINITE-NEXT: movsd {{[^,]*}}, %xmm1
 ; FINITE-NEXT: maxsd %xmm0, %xmm1
-; FINITE-NEXT: movaps %xmm1, %xmm0
+; FINITE-NEXT: movapd %xmm1, %xmm0
 ; FINITE-NEXT: ret
 define double @clampTo3k_f(double %x) nounwind readnone {
 entry:
@@ -919,7 +919,7 @@ entry:
 ; CHECK-LABEL: clampTo3k_g:
 ; CHECK-NEXT: movsd {{[^,]*}}, %xmm1
 ; CHECK-NEXT: minsd %xmm0, %xmm1
-; CHECK-NEXT: movaps %xmm1, %xmm0
+; CHECK-NEXT: movapd %xmm1, %xmm0
 ; CHECK-NEXT: ret
 ; UNSAFE-LABEL: clampTo3k_g:
 ; UNSAFE-NEXT: minsd {{[^,]*}}, %xmm0
@@ -927,7 +927,7 @@ entry:
 ; FINITE-LABEL: clampTo3k_g:
 ; FINITE-NEXT: movsd {{[^,]*}}, %xmm1
 ; FINITE-NEXT: minsd %xmm0, %xmm1
-; FINITE-NEXT: movaps %xmm1, %xmm0
+; FINITE-NEXT: movapd %xmm1, %xmm0
 ; FINITE-NEXT: ret
 define double @clampTo3k_g(double %x) nounwind readnone {
 entry:
@@ -945,7 +945,7 @@ entry:
 ; FINITE-LABEL: clampTo3k_h:
 ; FINITE-NEXT: movsd {{[^,]*}}, %xmm1
 ; FINITE-NEXT: minsd %xmm0, %xmm1
-; FINITE-NEXT: movaps %xmm1, %xmm0
+; FINITE-NEXT: movapd %xmm1, %xmm0
 ; FINITE-NEXT: ret
 define double @clampTo3k_h(double %x) nounwind readnone {
 entry:
diff --git a/test/CodeGen/X86/sse-scalar-fp-arith-unary.ll b/test/CodeGen/X86/sse-scalar-fp-arith-unary.ll
new file mode 100644
index 00000000000..fab4f90279e
--- /dev/null
+++ b/test/CodeGen/X86/sse-scalar-fp-arith-unary.ll
@@ -0,0 +1,73 @@
+; RUN: llc -mtriple=x86_64-unknown-unknown -mattr=sse2 < %s   | FileCheck --check-prefix=SSE %s
+; RUN: llc -mtriple=x86_64-unknown-unknown -mattr=sse4.1 < %s | FileCheck --check-prefix=SSE %s
+; RUN: llc -mtriple=x86_64-unknown-unknown -mattr=avx < %s    | FileCheck --check-prefix=AVX %s
+
+; PR21507 - https://llvm.org/bugs/show_bug.cgi?id=21507
+; Each function should be a single math op; no extra moves.
+
+
+define <4 x float> @recip(<4 x float> %x) {
+; SSE-LABEL: recip:
+; SSE:       # BB#0:
+; SSE-NEXT:    rcpss %xmm0, %xmm0
+; SSE-NEXT:    retq
+;
+; AVX-LABEL: recip:
+; AVX:       # BB#0:
+; AVX-NEXT:    vrcpss %xmm0, %xmm0, %xmm0
+; AVX-NEXT:    retq
+  %y = tail call <4 x float> @llvm.x86.sse.rcp.ss(<4 x float> %x)
+  %shuf = shufflevector <4 x float> %y, <4 x float> %x, <4 x i32> <i32 0, i32 5, i32 6, i32 7>
+  ret <4 x float> %shuf
+}
+
+define <4 x float> @recip_square_root(<4 x float> %x) {
+; SSE-LABEL: recip_square_root:
+; SSE:       # BB#0:
+; SSE-NEXT:    rsqrtss %xmm0, %xmm0
+; SSE-NEXT:    retq
+;
+; AVX-LABEL: recip_square_root:
+; AVX:       # BB#0:
+; AVX-NEXT:    vrsqrtss %xmm0, %xmm0, %xmm0
+; AVX-NEXT:    retq
+  %y = tail call <4 x float> @llvm.x86.sse.rsqrt.ss(<4 x float> %x)
+  %shuf = shufflevector <4 x float> %y, <4 x float> %x, <4 x i32> <i32 0, i32 5, i32 6, i32 7>
+  ret <4 x float> %shuf
+}
+
+define <4 x float> @square_root(<4 x float> %x) {
+; SSE-LABEL: square_root:
+; SSE:       # BB#0:
+; SSE-NEXT:    sqrtss %xmm0, %xmm0
+; SSE-NEXT:    retq
+;
+; AVX-LABEL: square_root:
+; AVX:       # BB#0:
+; AVX-NEXT:    vsqrtss %xmm0, %xmm0, %xmm0
+; AVX-NEXT:    retq
+  %y = tail call <4 x float> @llvm.x86.sse.sqrt.ss(<4 x float> %x)
+  %shuf = shufflevector <4 x float> %y, <4 x float> %x, <4 x i32> <i32 0, i32 5, i32 6, i32 7>
+  ret <4 x float> %shuf
+}
+
+define <2 x double> @square_root_double(<2 x double> %x) {
+; SSE-LABEL: square_root_double:
+; SSE:       # BB#0:
+; SSE-NEXT:    sqrtsd %xmm0, %xmm0
+; SSE-NEXT:    retq
+;
+; AVX-LABEL: square_root_double:
+; AVX:       # BB#0:
+; AVX-NEXT:    vsqrtsd %xmm0, %xmm0, %xmm0
+; AVX-NEXT:    retq
+  %y = tail call <2 x double> @llvm.x86.sse2.sqrt.sd(<2 x double> %x)
+  %shuf = shufflevector <2 x double> %y, <2 x double> %x, <2 x i32> <i32 0, i32 3>
+  ret <2 x double> %shuf
+}
+
+declare <4 x float> @llvm.x86.sse.rcp.ss(<4 x float>)
+declare <4 x float> @llvm.x86.sse.rsqrt.ss(<4 x float>)
+declare <4 x float> @llvm.x86.sse.sqrt.ss(<4 x float>)
+declare <2 x double> @llvm.x86.sse2.sqrt.sd(<2 x double>)
+
diff --git a/test/CodeGen/X86/sse-scalar-fp-arith.ll b/test/CodeGen/X86/sse-scalar-fp-arith.ll
index 8b1c6d0c882..7162d2715bc 100644
--- a/test/CodeGen/X86/sse-scalar-fp-arith.ll
+++ b/test/CodeGen/X86/sse-scalar-fp-arith.ll
@@ -76,6 +76,31 @@ define <4 x float> @test_div_ss(<4 x float> %a, <4 x float> %b) {
   ret <4 x float> %3
 }
 
+define <4 x float> @test_sqrt_ss(<4 x float> %a) {
+; SSE2-LABEL: test_sqrt_ss:
+; SSE2:       # BB#0:
+; SSE2-NEXT:   sqrtss %xmm0, %xmm1
+; SSE2-NEXT:   movss %xmm1, %xmm0
+; SSE2-NEXT:   retq
+;
+; SSE41-LABEL: test_sqrt_ss:
+; SSE41:       # BB#0:
+; SSE41-NEXT:  sqrtss %xmm0, %xmm1
+; SSE41-NEXT:  blendps {{.*#+}} xmm0 = xmm1[0],xmm0[1,2,3]
+; SSE41-NEXT:  retq
+;
+; AVX-LABEL: test_sqrt_ss:
+; AVX:       # BB#0:
+; AVX-NEXT:    vsqrtss %xmm0, %xmm0, %xmm1
+; AVX-NEXT:    vblendps {{.*#+}} xmm0 = xmm1[0],xmm0[1,2,3]
+; AVX-NEXT:    retq
+  %1 = extractelement <4 x float> %a, i32 0
+  %2 = call float @llvm.sqrt.f32(float %1)
+  %3 = insertelement <4 x float> %a, float %2, i32 0
+  ret <4 x float> %3
+}
+declare float @llvm.sqrt.f32(float)
+
 define <2 x double> @test_add_sd(<2 x double> %a, <2 x double> %b) {
 ; SSE-LABEL: test_add_sd:
 ; SSE:       # BB#0:
@@ -144,6 +169,25 @@ define <2 x double> @test_div_sd(<2 x double> %a, <2 x double> %b) {
   ret <2 x double> %3
 }
 
+define <2 x double> @test_sqrt_sd(<2 x double> %a) {
+; SSE-LABEL: test_sqrt_sd:
+; SSE:       # BB#0:
+; SSE-NEXT:    sqrtsd %xmm0, %xmm1
+; SSE-NEXT:    movsd %xmm1, %xmm0
+; SSE-NEXT:    retq
+;
+; AVX-LABEL: test_sqrt_sd:
+; AVX:       # BB#0:
+; AVX-NEXT:    vsqrtsd %xmm0, %xmm0, %xmm1
+; AVX-NEXT:    vmovsd %xmm1, %xmm0, %xmm0
+; AVX-NEXT:    retq
+  %1 = extractelement <2 x double> %a, i32 0
+  %2 = call double @llvm.sqrt.f64(double %1)
+  %3 = insertelement <2 x double> %a, double %2, i32 0
+  ret <2 x double> %3
+}
+declare double @llvm.sqrt.f64(double)
+
 define <4 x float> @test2_add_ss(<4 x float> %a, <4 x float> %b) {
 ; SSE-LABEL: test2_add_ss:
 ; SSE:       # BB#0:
@@ -220,7 +264,7 @@ define <2 x double> @test2_add_sd(<2 x double> %a, <2 x double> %b) {
 ; SSE-LABEL: test2_add_sd:
 ; SSE:       # BB#0:
 ; SSE-NEXT:    addsd %xmm0, %xmm1
-; SSE-NEXT:    movaps %xmm1, %xmm0
+; SSE-NEXT:    movapd %xmm1, %xmm0
 ; SSE-NEXT:    retq
 ;
 ; AVX-LABEL: test2_add_sd:
@@ -238,7 +282,7 @@ define <2 x double> @test2_sub_sd(<2 x double> %a, <2 x double> %b) {
 ; SSE-LABEL: test2_sub_sd:
 ; SSE:       # BB#0:
 ; SSE-NEXT:    subsd %xmm0, %xmm1
-; SSE-NEXT:    movaps %xmm1, %xmm0
+; SSE-NEXT:    movapd %xmm1, %xmm0
 ; SSE-NEXT:    retq
 ;
 ; AVX-LABEL: test2_sub_sd:
@@ -256,7 +300,7 @@ define <2 x double> @test2_mul_sd(<2 x double> %a, <2 x double> %b) {
 ; SSE-LABEL: test2_mul_sd:
 ; SSE:       # BB#0:
 ; SSE-NEXT:    mulsd %xmm0, %xmm1
-; SSE-NEXT:    movaps %xmm1, %xmm0
+; SSE-NEXT:    movapd %xmm1, %xmm0
 ; SSE-NEXT:    retq
 ;
 ; AVX-LABEL: test2_mul_sd:
@@ -274,7 +318,7 @@ define <2 x double> @test2_div_sd(<2 x double> %a, <2 x double> %b) {
 ; SSE-LABEL: test2_div_sd:
 ; SSE:       # BB#0:
 ; SSE-NEXT:    divsd %xmm0, %xmm1
-; SSE-NEXT:    movaps %xmm1, %xmm0
+; SSE-NEXT:    movapd %xmm1, %xmm0
 ; SSE-NEXT:    retq
 ;
 ; AVX-LABEL: test2_div_sd:
@@ -371,7 +415,7 @@ define <4 x float> @test_multiple_div_ss(<4 x float> %a, <4 x float> %b) {
 }
 
 ; With SSE4.1 or greater, the shuffles in the following tests may
-; be lowered to X86Blendi nodes. 
+; be lowered to X86Blendi nodes.
 
 define <4 x float> @blend_add_ss(<4 x float> %a, float %b) {
 ; SSE-LABEL: blend_add_ss:
@@ -708,7 +752,7 @@ define <2 x double> @insert_test2_add_sd(<2 x double> %a, <2 x double> %b) {
 ; SSE-LABEL: insert_test2_add_sd:
 ; SSE:       # BB#0:
 ; SSE-NEXT:    addsd %xmm0, %xmm1
-; SSE-NEXT:    movaps %xmm1, %xmm0
+; SSE-NEXT:    movapd %xmm1, %xmm0
 ; SSE-NEXT:    retq
 ;
 ; AVX-LABEL: insert_test2_add_sd:
@@ -724,7 +768,7 @@ define <2 x double> @insert_test2_sub_sd(<2 x double> %a, <2 x double> %b) {
 ; SSE-LABEL: insert_test2_sub_sd:
 ; SSE:       # BB#0:
 ; SSE-NEXT:    subsd %xmm0, %xmm1
-; SSE-NEXT:    movaps %xmm1, %xmm0
+; SSE-NEXT:    movapd %xmm1, %xmm0
 ; SSE-NEXT:    retq
 ;
 ; AVX-LABEL: insert_test2_sub_sd:
@@ -740,7 +784,7 @@ define <2 x double> @insert_test2_mul_sd(<2 x double> %a, <2 x double> %b) {
 ; SSE-LABEL: insert_test2_mul_sd:
 ; SSE:       # BB#0:
 ; SSE-NEXT:    mulsd %xmm0, %xmm1
-; SSE-NEXT:    movaps %xmm1, %xmm0
+; SSE-NEXT:    movapd %xmm1, %xmm0
 ; SSE-NEXT:    retq
 ;
 ; AVX-LABEL: insert_test2_mul_sd:
@@ -756,7 +800,7 @@ define <2 x double> @insert_test2_div_sd(<2 x double> %a, <2 x double> %b) {
 ; SSE-LABEL: insert_test2_div_sd:
 ; SSE:       # BB#0:
 ; SSE-NEXT:    divsd %xmm0, %xmm1
-; SSE-NEXT:    movaps %xmm1, %xmm0
+; SSE-NEXT:    movapd %xmm1, %xmm0
 ; SSE-NEXT:    retq
 ;
 ; AVX-LABEL: insert_test2_div_sd:
@@ -956,7 +1000,7 @@ define <2 x double> @insert_test4_add_sd(<2 x double> %a, <2 x double> %b) {
 ; SSE-LABEL: insert_test4_add_sd:
 ; SSE:       # BB#0:
 ; SSE-NEXT:    addsd %xmm0, %xmm1
-; SSE-NEXT:    movaps %xmm1, %xmm0
+; SSE-NEXT:    movapd %xmm1, %xmm0
 ; SSE-NEXT:    retq
 ;
 ; AVX-LABEL: insert_test4_add_sd:
@@ -972,7 +1016,7 @@ define <2 x double> @insert_test4_sub_sd(<2 x double> %a, <2 x double> %b) {
 ; SSE-LABEL: insert_test4_sub_sd:
 ; SSE:       # BB#0:
 ; SSE-NEXT:    subsd %xmm0, %xmm1
-; SSE-NEXT:    movaps %xmm1, %xmm0
+; SSE-NEXT:    movapd %xmm1, %xmm0
 ; SSE-NEXT:    retq
 ;
 ; AVX-LABEL: insert_test4_sub_sd:
@@ -988,7 +1032,7 @@ define <2 x double> @insert_test4_mul_sd(<2 x double> %a, <2 x double> %b) {
 ; SSE-LABEL: insert_test4_mul_sd:
 ; SSE:       # BB#0:
 ; SSE-NEXT:    mulsd %xmm0, %xmm1
-; SSE-NEXT:    movaps %xmm1, %xmm0
+; SSE-NEXT:    movapd %xmm1, %xmm0
 ; SSE-NEXT:    retq
 ;
 ; AVX-LABEL: insert_test4_mul_sd:
@@ -1004,7 +1048,7 @@ define <2 x double> @insert_test4_div_sd(<2 x double> %a, <2 x double> %b) {
 ; SSE-LABEL: insert_test4_div_sd:
 ; SSE:       # BB#0:
 ; SSE-NEXT:    divsd %xmm0, %xmm1
-; SSE-NEXT:    movaps %xmm1, %xmm0
+; SSE-NEXT:    movapd %xmm1, %xmm0
 ; SSE-NEXT:    retq
 ;
 ; AVX-LABEL: insert_test4_div_sd:
diff --git a/test/CodeGen/X86/sse2-intrinsics-x86.ll b/test/CodeGen/X86/sse2-intrinsics-x86.ll
index 5afebd24bb4..53132a167fb 100644
--- a/test/CodeGen/X86/sse2-intrinsics-x86.ll
+++ b/test/CodeGen/X86/sse2-intrinsics-x86.ll
@@ -1,4 +1,5 @@
-; RUN: llc < %s -mtriple=i386-apple-darwin -mattr=-avx,+sse2 | FileCheck %s
+; RUN: llc < %s -mtriple=i386-apple-darwin -mattr=-avx,+sse2 | FileCheck %s --check-prefix=CHECK --check-prefix=SSE
+; RUN: llc < %s -mtriple=i386-apple-darwin -mcpu=knl | FileCheck %s
 
 define <2 x double> @test_x86_sse2_add_sd(<2 x double> %a0, <2 x double> %a1) {
   ; CHECK: addsd
@@ -142,7 +143,7 @@ declare i32 @llvm.x86.sse2.cvtsd2si(<2 x double>) nounwind readnone
 
 define <4 x float> @test_x86_sse2_cvtsd2ss(<4 x float> %a0, <2 x double> %a1) {
   ; CHECK: cvtsd2ss 
-  ; CHECK-NOT: cvtsd2ss %xmm{{[0-9]+}}, %xmm{{[0-9]+}}, %xmm{{[0-9]+}} 
+  ; SSE-NOT: cvtsd2ss %xmm{{[0-9]+}}, %xmm{{[0-9]+}}, %xmm{{[0-9]+}} 
   %res = call <4 x float> @llvm.x86.sse2.cvtsd2ss(<4 x float> %a0, <2 x double> %a1) ; <<4 x float>> [#uses=1]
   ret <4 x float> %res
 }
diff --git a/test/CodeGen/X86/sse2-vector-shifts.ll b/test/CodeGen/X86/sse2-vector-shifts.ll
index 7c8d5e57889..45028cf4bd3 100644
--- a/test/CodeGen/X86/sse2-vector-shifts.ll
+++ b/test/CodeGen/X86/sse2-vector-shifts.ll
@@ -1,4 +1,4 @@
-; RUN: llc < %s -mtriple=x86_64-pc-linux -mattr=+sse2 -mcpu=corei7 | FileCheck %s
+; RUN: llc < %s -mtriple=x86_64-pc-linux -mattr=+sse2 | FileCheck %s
 
 ; SSE2 Logical Shift Left
 
diff --git a/test/CodeGen/X86/sse3-avx-addsub-2.ll b/test/CodeGen/X86/sse3-avx-addsub-2.ll
index 5b2de28c0f5..71efa3f8f10 100644
--- a/test/CodeGen/X86/sse3-avx-addsub-2.ll
+++ b/test/CodeGen/X86/sse3-avx-addsub-2.ll
@@ -315,4 +315,17 @@ define <4 x float> @test16(<4 x float> %A, <4 x float> %B) {
 ; CHECK-NOT: addsubps
 ; CHECK: ret
 
-
+define <2 x float> @test_v2f32(<2 x float> %v0, <2 x float> %v1) {
+  %v2 = extractelement <2 x float> %v0, i32 0
+  %v3 = extractelement <2 x float> %v1, i32 0
+  %v4 = extractelement <2 x float> %v0, i32 1
+  %v5 = extractelement <2 x float> %v1, i32 1
+  %sub = fsub float %v2, %v3
+  %add = fadd float %v5, %v4
+  %res0 = insertelement <2 x float> undef, float %sub, i32 0
+  %res1 = insertelement <2 x float> %res0, float %add, i32 1
+  ret <2 x float> %res1
+}
+; CHECK-LABEL: test_v2f32
+; CHECK: addsubps %xmm1, %xmm0
+; CHECK-NEXT: retq
diff --git a/test/CodeGen/X86/sse3.ll b/test/CodeGen/X86/sse3.ll
index c1cd91beaf5..398675276c6 100644
--- a/test/CodeGen/X86/sse3.ll
+++ b/test/CodeGen/X86/sse3.ll
@@ -1,6 +1,6 @@
 ; These are tests for SSE3 codegen.
 
-; RUN: llc < %s -march=x86-64 -mcpu=nocona -mtriple=i686-apple-darwin9 -O3 | FileCheck %s --check-prefix=X64
+; RUN: llc < %s -mtriple=x86_64-apple-darwin9 --mattr=+sse3 | FileCheck %s --check-prefix=X64
 
 ; Test for v8xi16 lowering where we extract the first element of the vector and
 ; placed it in the second element of the result.
diff --git a/test/CodeGen/X86/sse41-intrinsics-x86.ll b/test/CodeGen/X86/sse41-intrinsics-x86.ll
index 5f25a16380d..771e4024336 100644
--- a/test/CodeGen/X86/sse41-intrinsics-x86.ll
+++ b/test/CodeGen/X86/sse41-intrinsics-x86.ll
@@ -1,4 +1,5 @@
 ; RUN: llc < %s -mtriple=i386-apple-darwin -mattr=-avx,+sse4.1 | FileCheck %s
+; RUN: llc < %s -mtriple=i386-apple-darwin -mcpu=knl | FileCheck %s
 
 define <2 x double> @test_x86_sse41_blendpd(<2 x double> %a0, <2 x double> %a1) {
   ; CHECK: blendpd
diff --git a/test/CodeGen/X86/sse41.ll b/test/CodeGen/X86/sse41.ll
index 3bde991c337..8532c012aa9 100644
--- a/test/CodeGen/X86/sse41.ll
+++ b/test/CodeGen/X86/sse41.ll
@@ -1035,15 +1035,15 @@ define void @insertps_pr20411(<4 x i32> %shuffle109, <4 x i32> %shuffle116, i32*
 ; X32-NEXT:    movdqu %xmm1, (%eax)
 ; X32-NEXT:    retl
 ;
-; X64-LABEL: insertps_pr20411:
-; X64:       ## BB#0:
-; X64-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[2,3,0,1]
-; X64-NEXT:    pblendw {{.*#+}} xmm1 = xmm0[0,1],xmm1[2,3],xmm0[4,5,6,7]
-; X64-NEXT:    movdqu %xmm1, (%rdi)
+; X64-LABEL: insertps_pr20411:
+; X64:       ## BB#0:
+; X64-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[2,3,0,1]
+; X64-NEXT:    pblendw {{.*#+}} xmm1 = xmm0[0,1],xmm1[2,3],xmm0[4,5,6,7]
+; X64-NEXT:    movdqu %xmm1, (%rdi)
 ; X64-NEXT:    retq
-  %shuffle117 = shufflevector <4 x i32> %shuffle109, <4 x i32> %shuffle116, <4 x i32> <i32 0, i32 7, i32 undef, i32 undef>
-  %ptrcast = bitcast i32* %RET to <4 x i32>*
-  store <4 x i32> %shuffle117, <4 x i32>* %ptrcast, align 4
+  %shuffle117 = shufflevector <4 x i32> %shuffle109, <4 x i32> %shuffle116, <4 x i32> <i32 0, i32 7, i32 undef, i32 undef>
+  %ptrcast = bitcast i32* %RET to <4 x i32>*
+  store <4 x i32> %shuffle117, <4 x i32>* %ptrcast, align 4
   ret void
 }
 
diff --git a/test/CodeGen/X86/stack-folding-fp-avx1.ll b/test/CodeGen/X86/stack-folding-fp-avx1.ll
index 18cd4179de8..63aa742bdf0 100644
--- a/test/CodeGen/X86/stack-folding-fp-avx1.ll
+++ b/test/CodeGen/X86/stack-folding-fp-avx1.ll
@@ -1409,12 +1409,26 @@ define <8 x float> @stack_fold_roundps_ymm(<8 x float> %a0) {
 }
 declare <8 x float> @llvm.x86.avx.round.ps.256(<8 x float>, i32) nounwind readnone
 
-; TODO stack_fold_roundsd
+define double @stack_fold_roundsd(double %a0) optsize {
+  ;CHECK-LABEL: stack_fold_roundsd
+  ;CHECK:       vroundsd $1, {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}}, {{%xmm[0-9][0-9]*}} {{.*#+}} 8-byte Folded Reload
+  %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
+  %2 = call double @llvm.floor.f64(double %a0)
+  ret double %2
+}
+declare double @llvm.floor.f64(double) nounwind readnone
 
 ; TODO stack_fold_roundsd_int
 declare <2 x double> @llvm.x86.sse41.round.sd(<2 x double>, <2 x double>, i32) nounwind readnone
 
-; TODO stack_fold_roundss
+define float @stack_fold_roundss(float %a0) optsize {
+  ;CHECK-LABEL: stack_fold_roundss
+  ;CHECK:       vroundss $1, {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}}, {{%xmm[0-9][0-9]*}} {{.*#+}} 4-byte Folded Reload
+  %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
+  %2 = call float @llvm.floor.f32(float %a0)
+  ret float %2
+}
+declare float @llvm.floor.f32(float) nounwind readnone
 
 ; TODO stack_fold_roundss_int
 declare <4 x float> @llvm.x86.sse41.round.ss(<4 x float>, <4 x float>, i32) nounwind readnone
@@ -1699,7 +1713,9 @@ define <2 x double> @stack_fold_unpckhpd(<2 x double> %a0, <2 x double> %a1) {
   ;CHECK:       vunpckhpd {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}}, {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = shufflevector <2 x double> %a0, <2 x double> %a1, <2 x i32> <i32 1, i32 3>
-  ret <2 x double> %2
+  ; fadd forces execution domain
+  %3 = fadd <2 x double> %2, <double 0x0, double 0x0>
+  ret <2 x double> %3
 }
 
 define <4 x double> @stack_fold_unpckhpd_ymm(<4 x double> %a0, <4 x double> %a1) {
@@ -1707,7 +1723,9 @@ define <4 x double> @stack_fold_unpckhpd_ymm(<4 x double> %a0, <4 x double> %a1)
   ;CHECK:       vunpckhpd {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}}, {{%ymm[0-9][0-9]*}} {{.*#+}} 32-byte Folded Reload
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = shufflevector <4 x double> %a0, <4 x double> %a1, <4 x i32> <i32 1, i32 5, i32 3, i32 7>
-  ret <4 x double> %2
+  ; fadd forces execution domain
+  %3 = fadd <4 x double> %2, <double 0x0, double 0x0, double 0x0, double 0x0>
+  ret <4 x double> %3
 }
 
 define <4 x float> @stack_fold_unpckhps(<4 x float> %a0, <4 x float> %a1) {
@@ -1715,7 +1733,9 @@ define <4 x float> @stack_fold_unpckhps(<4 x float> %a0, <4 x float> %a1) {
   ;CHECK:       vunpckhps {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}}, {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = shufflevector <4 x float> %a0, <4 x float> %a1, <4 x i32> <i32 2, i32 6, i32 3, i32 7>
-  ret <4 x float> %2
+  ; fadd forces execution domain
+  %3 = fadd <4 x float> %2, <float 0x0, float 0x0, float 0x0, float 0x0>
+  ret <4 x float> %3
 }
 
 define <8 x float> @stack_fold_unpckhps_ymm(<8 x float> %a0, <8 x float> %a1) {
@@ -1723,7 +1743,9 @@ define <8 x float> @stack_fold_unpckhps_ymm(<8 x float> %a0, <8 x float> %a1) {
   ;CHECK:       vunpckhps {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}}, {{%ymm[0-9][0-9]*}} {{.*#+}} 32-byte Folded Reload
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = shufflevector <8 x float> %a0, <8 x float> %a1, <8 x i32> <i32 2, i32 10, i32 3, i32 11, i32 6, i32 14, i32 7, i32 15>
-  ret <8 x float> %2
+  ; fadd forces execution domain
+  %3 = fadd <8 x float> %2, <float 0x0, float 0x0, float 0x0, float 0x0, float 0x0, float 0x0, float 0x0, float 0x0>
+  ret <8 x float> %3
 }
 
 define <2 x double> @stack_fold_unpcklpd(<2 x double> %a0, <2 x double> %a1) {
@@ -1731,7 +1753,9 @@ define <2 x double> @stack_fold_unpcklpd(<2 x double> %a0, <2 x double> %a1) {
   ;CHECK:       vunpcklpd {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}}, {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = shufflevector <2 x double> %a0, <2 x double> %a1, <2 x i32> <i32 0, i32 2>
-  ret <2 x double> %2
+  ; fadd forces execution domain
+  %3 = fadd <2 x double> %2, <double 0x0, double 0x0>
+  ret <2 x double> %3
 }
 
 define <4 x double> @stack_fold_unpcklpd_ymm(<4 x double> %a0, <4 x double> %a1) {
@@ -1739,7 +1763,9 @@ define <4 x double> @stack_fold_unpcklpd_ymm(<4 x double> %a0, <4 x double> %a1)
   ;CHECK:       vunpcklpd {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}}, {{%ymm[0-9][0-9]*}} {{.*#+}} 32-byte Folded Reload
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = shufflevector <4 x double> %a0, <4 x double> %a1, <4 x i32> <i32 0, i32 4, i32 2, i32 6>
-  ret <4 x double> %2
+  ; fadd forces execution domain
+  %3 = fadd <4 x double> %2, <double 0x0, double 0x0, double 0x0, double 0x0>
+  ret <4 x double> %3
 }
 
 define <4 x float> @stack_fold_unpcklps(<4 x float> %a0, <4 x float> %a1) {
@@ -1747,7 +1773,9 @@ define <4 x float> @stack_fold_unpcklps(<4 x float> %a0, <4 x float> %a1) {
   ;CHECK:       vunpcklps {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}}, {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = shufflevector <4 x float> %a0, <4 x float> %a1, <4 x i32> <i32 0, i32 4, i32 1, i32 5>
-  ret <4 x float> %2
+  ; fadd forces execution domain
+  %3 = fadd <4 x float> %2, <float 0x0, float 0x0, float 0x0, float 0x0>
+  ret <4 x float> %3
 }
 
 define <8 x float> @stack_fold_unpcklps_ymm(<8 x float> %a0, <8 x float> %a1) {
@@ -1755,7 +1783,9 @@ define <8 x float> @stack_fold_unpcklps_ymm(<8 x float> %a0, <8 x float> %a1) {
   ;CHECK:       vunpcklps {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}}, {{%ymm[0-9][0-9]*}} {{.*#+}} 32-byte Folded Reload
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = shufflevector <8 x float> %a0, <8 x float> %a1, <8 x i32> <i32 0, i32 8, i32 1, i32 9, i32 4, i32 12, i32 5, i32 13>
-  ret <8 x float> %2
+  ; fadd forces execution domain
+  %3 = fadd <8 x float> %2, <float 0x0, float 0x0, float 0x0, float 0x0, float 0x0, float 0x0, float 0x0, float 0x0>
+  ret <8 x float> %3
 }
 
 define <2 x double> @stack_fold_xorpd(<2 x double> %a0, <2 x double> %a1) {
diff --git a/test/CodeGen/X86/stack-folding-fp-sse42.ll b/test/CodeGen/X86/stack-folding-fp-sse42.ll
index c26cc9df989..f9fcbaabdeb 100644
--- a/test/CodeGen/X86/stack-folding-fp-sse42.ll
+++ b/test/CodeGen/X86/stack-folding-fp-sse42.ll
@@ -314,7 +314,13 @@ define i64 @stack_fold_cvtsd2si64_int(<2 x double> %a0) {
 }
 declare i64 @llvm.x86.sse2.cvtsd2si64(<2 x double>) nounwind readnone
 
-; TODO stack_fold_cvtsd2ss
+define float @stack_fold_cvtsd2ss(double %a0) optsize {
+  ;CHECK-LABEL: stack_fold_cvtsd2ss
+  ;CHECK:       cvtsd2ss {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 8-byte Folded Reload
+  %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
+  %2 = fptrunc double %a0 to float
+  ret float %2
+}
 
 define <4 x float> @stack_fold_cvtsd2ss_int(<2 x double> %a0) optsize {
   ;CHECK-LABEL: stack_fold_cvtsd2ss_int
@@ -878,11 +884,29 @@ define <4 x float> @stack_fold_roundps(<4 x float> %a0) {
 }
 declare <4 x float> @llvm.x86.sse41.round.ps(<4 x float>, i32) nounwind readnone
 
-; TODO stack_fold_roundsd
+define double @stack_fold_roundsd(double %a0) optsize {
+  ;CHECK-LABEL: stack_fold_roundsd
+  ;CHECK:       roundsd $1, {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 8-byte Folded Reload
+  %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
+  %2 = call double @llvm.floor.f64(double %a0)
+  ret double %2
+}
+declare double @llvm.floor.f64(double) nounwind readnone
+
 ; TODO stack_fold_roundsd_int
+declare <2 x double> @llvm.x86.sse41.round.sd(<2 x double>, <2 x double>, i32) nounwind readnone
+
+define float @stack_fold_roundss(float %a0) optsize {
+  ;CHECK-LABEL: stack_fold_roundss
+  ;CHECK:       roundss $1, {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 4-byte Folded Reload
+  %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
+  %2 = call float @llvm.floor.f32(float %a0)
+  ret float %2
+}
+declare float @llvm.floor.f32(float) nounwind readnone
 
-; TODO stack_fold_roundss
 ; TODO stack_fold_roundss_int
+declare <4 x float> @llvm.x86.sse41.round.ss(<4 x float>, <4 x float>, i32) nounwind readnone
 
 ; TODO stack_fold_rsqrtps
 
@@ -932,13 +956,25 @@ define <4 x float> @stack_fold_sqrtps(<4 x float> %a0) {
 }
 declare <4 x float> @llvm.x86.sse.sqrt.ps(<4 x float>) nounwind readnone
 
-; TODO stack_fold_sqrtsd
+define double @stack_fold_sqrtsd(double %a0) optsize {
+  ;CHECK-LABEL: stack_fold_sqrtsd
+  ;CHECK:       sqrtsd {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 8-byte Folded Reload
+  %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
+  %2 = call double @llvm.sqrt.f64(double %a0)
+  ret double %2
+}
 declare double @llvm.sqrt.f64(double) nounwind readnone
 
 ; TODO stack_fold_sqrtsd_int
 declare <2 x double> @llvm.x86.sse2.sqrt.sd(<2 x double>) nounwind readnone
 
-; TODO stack_fold_sqrtss
+define float @stack_fold_sqrtss(float %a0) optsize {
+  ;CHECK-LABEL: stack_fold_sqrtss
+  ;CHECK:       sqrtss {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 4-byte Folded Reload
+  %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
+  %2 = call float @llvm.sqrt.f32(float %a0)
+  ret float %2
+}
 declare float @llvm.sqrt.f32(float) nounwind readnone
 
 ; TODO stack_fold_sqrtss_int
@@ -1035,7 +1071,9 @@ define <2 x double> @stack_fold_unpckhpd(<2 x double> %a0, <2 x double> %a1) {
   ;CHECK:       unpckhpd {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = shufflevector <2 x double> %a0, <2 x double> %a1, <2 x i32> <i32 1, i32 3>
-  ret <2 x double> %2
+  ; fadd forces execution domain
+  %3 = fadd <2 x double> %2, <double 0x0, double 0x0>
+  ret <2 x double> %3
 }
 
 define <4 x float> @stack_fold_unpckhps(<4 x float> %a0, <4 x float> %a1) {
@@ -1043,7 +1081,9 @@ define <4 x float> @stack_fold_unpckhps(<4 x float> %a0, <4 x float> %a1) {
   ;CHECK:       unpckhps {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = shufflevector <4 x float> %a0, <4 x float> %a1, <4 x i32> <i32 2, i32 6, i32 3, i32 7>
-  ret <4 x float> %2
+  ; fadd forces execution domain
+  %3 = fadd <4 x float> %2, <float 0x0, float 0x0, float 0x0, float 0x0>
+  ret <4 x float> %3
 }
 
 define <2 x double> @stack_fold_unpcklpd(<2 x double> %a0, <2 x double> %a1) {
@@ -1051,7 +1091,9 @@ define <2 x double> @stack_fold_unpcklpd(<2 x double> %a0, <2 x double> %a1) {
   ;CHECK:       unpcklpd {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = shufflevector <2 x double> %a0, <2 x double> %a1, <2 x i32> <i32 0, i32 2>
-  ret <2 x double> %2
+  ; fadd forces execution domain
+  %3 = fadd <2 x double> %2, <double 0x0, double 0x0>
+  ret <2 x double> %3
 }
 
 define <4 x float> @stack_fold_unpcklps(<4 x float> %a0, <4 x float> %a1) {
@@ -1059,7 +1101,9 @@ define <4 x float> @stack_fold_unpcklps(<4 x float> %a0, <4 x float> %a1) {
   ;CHECK:       unpcklps {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = shufflevector <4 x float> %a0, <4 x float> %a1, <4 x i32> <i32 0, i32 4, i32 1, i32 5>
-  ret <4 x float> %2
+  ; fadd forces execution domain
+  %3 = fadd <4 x float> %2, <float 0x0, float 0x0, float 0x0, float 0x0>
+  ret <4 x float> %3
 }
 
 define <2 x double> @stack_fold_xorpd(<2 x double> %a0, <2 x double> %a1) {
diff --git a/test/CodeGen/X86/stack-folding-int-avx2.ll b/test/CodeGen/X86/stack-folding-int-avx2.ll
index 7f384940d85..a164fbbc7a6 100644
--- a/test/CodeGen/X86/stack-folding-int-avx2.ll
+++ b/test/CodeGen/X86/stack-folding-int-avx2.ll
@@ -13,7 +13,9 @@ define <4 x double> @stack_fold_broadcastsd_ymm(<2 x double> %a0) {
   ;CHECK:       vbroadcastsd {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = call <4 x double> @llvm.x86.avx2.vbroadcast.sd.pd.256(<2 x double> %a0)
-  ret <4 x double> %2
+  ; fadd forces execution domain
+  %3 = fadd <4 x double> %2, <double 0x0, double 0x0, double 0x0, double 0x0>
+  ret <4 x double> %3
 }
 declare <4 x double> @llvm.x86.avx2.vbroadcast.sd.pd.256(<2 x double>) nounwind readonly
 
@@ -22,7 +24,9 @@ define <4 x float> @stack_fold_broadcastss(<4 x float> %a0) {
   ;CHECK:       vbroadcastss {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = call <4 x float> @llvm.x86.avx2.vbroadcast.ss.ps(<4 x float> %a0)
-  ret <4 x float> %2
+  ; fadd forces execution domain
+  %3 = fadd <4 x float> %2, <float 0x0, float 0x0, float 0x0, float 0x0>
+  ret <4 x float> %3
 }
 declare <4 x float> @llvm.x86.avx2.vbroadcast.ss.ps(<4 x float>) nounwind readonly
 
@@ -31,7 +35,9 @@ define <8 x float> @stack_fold_broadcastss_ymm(<4 x float> %a0) {
   ;CHECK:       vbroadcastss {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = call <8 x float> @llvm.x86.avx2.vbroadcast.ss.ps.256(<4 x float> %a0)
-  ret <8 x float> %2
+  ; fadd forces execution domain
+  %3 = fadd <8 x float> %2, <float 0x0, float 0x0, float 0x0, float 0x0, float 0x0, float 0x0, float 0x0, float 0x0>
+  ret <8 x float> %3
 }
 declare <8 x float> @llvm.x86.avx2.vbroadcast.ss.ps.256(<4 x float>) nounwind readonly
 
@@ -861,9 +867,21 @@ define <8 x i32> @stack_fold_pshufd(<8 x i32> %a0) {
   ret <8 x i32> %2
 }
 
-; TODO stack_fold_pshufhw
+define <16 x i16> @stack_fold_vpshufhw(<16 x i16> %a0) {
+  ;CHECK-LABEL: stack_fold_vpshufhw
+  ;CHECK:       vpshufhw $27, {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}} {{.*#+}} 32-byte Folded Reload
+  %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
+  %2 = shufflevector <16 x i16> %a0, <16 x i16> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 7, i32 6, i32 5, i32 4, i32 8, i32 9, i32 10, i32 11, i32 15, i32 14, i32 13, i32 12>
+  ret <16 x i16> %2
+}
 
-; TODO stack_fold_pshuflw
+define <16 x i16> @stack_fold_vpshuflw(<16 x i16> %a0) {
+  ;CHECK-LABEL: stack_fold_vpshuflw
+  ;CHECK:       vpshuflw $27, {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}} {{.*#+}} 32-byte Folded Reload
+  %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
+  %2 = shufflevector <16 x i16> %a0, <16 x i16> undef, <16 x i32> <i32 3, i32 2, i32 1, i32 0, i32 4, i32 5, i32 6, i32 7, i32 11, i32 10, i32 9, i32 8, i32 12, i32 13, i32 14, i32 15>
+  ret <16 x i16> %2
+}
 
 define <32 x i8> @stack_fold_psignb(<32 x i8> %a0, <32 x i8> %a1) {
   ;CHECK-LABEL: stack_fold_psignb
diff --git a/test/CodeGen/X86/stack-folding-x86_64.ll b/test/CodeGen/X86/stack-folding-x86_64.ll
new file mode 100644
index 00000000000..211227916a0
--- /dev/null
+++ b/test/CodeGen/X86/stack-folding-x86_64.ll
@@ -0,0 +1,51 @@
+; RUN: llc -O3 -disable-peephole -mtriple=x86_64-unknown-unknown -mcpu=x86-64 < %s | FileCheck %s
+
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-unknown"
+
+; Stack reload folding tests.
+;
+; By including a nop call with sideeffects we can force a partial register spill of the
+; relevant registers and check that the reload is correctly folded into the instruction.
+
+;TODO stack_fold_bsf_i16
+declare i16 @llvm.cttz.i16(i16, i1)
+
+define i32 @stack_fold_bsf_i32(i32 %a0) {
+  ;CHECK-LABEL: stack_fold_bsf_i32
+  ;CHECK:       bsfl {{-?[0-9]*}}(%rsp), %eax {{.*#+}} 4-byte Folded Reload
+  %1 = tail call i64 asm sideeffect "nop", "=x,~{rax},~{rbx},~{rcx},~{rdx},~{rsi},~{rdi},~{rbp},~{r8},~{r9},~{r10},~{r11},~{r12},~{r13},~{r14},~{r15}"()
+  %2 = call i32 @llvm.cttz.i32(i32 %a0, i1 -1)
+  ret i32 %2
+}
+declare i32 @llvm.cttz.i32(i32, i1)
+
+define i64 @stack_fold_bsf_i64(i64 %a0) {
+  ;CHECK-LABEL: stack_fold_bsf_i64
+  ;CHECK:       bsfq {{-?[0-9]*}}(%rsp), %rax {{.*#+}} 8-byte Folded Reload
+  %1 = tail call i64 asm sideeffect "nop", "=x,~{rax},~{rbx},~{rcx},~{rdx},~{rsi},~{rdi},~{rbp},~{r8},~{r9},~{r10},~{r11},~{r12},~{r13},~{r14},~{r15}"()
+  %2 = call i64 @llvm.cttz.i64(i64 %a0, i1 -1)
+  ret i64 %2
+}
+declare i64 @llvm.cttz.i64(i64, i1)
+
+;TODO stack_fold_bsr_i16
+declare i16 @llvm.ctlz.i16(i16, i1)
+
+define i32 @stack_fold_bsr_i32(i32 %a0) {
+  ;CHECK-LABEL: stack_fold_bsr_i32
+  ;CHECK:       bsrl {{-?[0-9]*}}(%rsp), %eax {{.*#+}} 4-byte Folded Reload
+  %1 = tail call i64 asm sideeffect "nop", "=x,~{rax},~{rbx},~{rcx},~{rdx},~{rsi},~{rdi},~{rbp},~{r8},~{r9},~{r10},~{r11},~{r12},~{r13},~{r14},~{r15}"()
+  %2 = call i32 @llvm.ctlz.i32(i32 %a0, i1 -1)
+  ret i32 %2
+}
+declare i32 @llvm.ctlz.i32(i32, i1)
+
+define i64 @stack_fold_bsr_i64(i64 %a0) {
+  ;CHECK-LABEL: stack_fold_bsr_i64
+  ;CHECK:       bsrq {{-?[0-9]*}}(%rsp), %rax {{.*#+}} 8-byte Folded Reload
+  %1 = tail call i64 asm sideeffect "nop", "=x,~{rax},~{rbx},~{rcx},~{rdx},~{rsi},~{rdi},~{rbp},~{r8},~{r9},~{r10},~{r11},~{r12},~{r13},~{r14},~{r15}"()
+  %2 = call i64 @llvm.ctlz.i64(i64 %a0, i1 -1)
+  ret i64 %2
+}
+declare i64 @llvm.ctlz.i64(i64, i1)
diff --git a/test/CodeGen/X86/stack-protector-dbginfo.ll b/test/CodeGen/X86/stack-protector-dbginfo.ll
index 6275c8d65f4..3aba19464b9 100644
--- a/test/CodeGen/X86/stack-protector-dbginfo.ll
+++ b/test/CodeGen/X86/stack-protector-dbginfo.ll
@@ -10,9 +10,9 @@
 ; Function Attrs: nounwind sspreq
 define i32 @_Z18read_response_sizev() #0 {
 entry:
-  tail call void @llvm.dbg.value(metadata !22, i64 0, metadata !23, metadata !MDExpression()), !dbg !39
+  tail call void @llvm.dbg.value(metadata !22, i64 0, metadata !23, metadata !DIExpression()), !dbg !39
   %0 = load i64, i64* getelementptr inbounds ({ i64, [56 x i8] }, { i64, [56 x i8] }* @a, i32 0, i32 0), align 8, !dbg !40
-  tail call void @llvm.dbg.value(metadata i32 undef, i64 0, metadata !64, metadata !MDExpression()), !dbg !71
+  tail call void @llvm.dbg.value(metadata i32 undef, i64 0, metadata !64, metadata !DIExpression()), !dbg !71
   %1 = trunc i64 %0 to i32
   ret i32 %1
 }
@@ -25,72 +25,72 @@ attributes #0 = { sspreq }
 !llvm.dbg.cu = !{!0}
 !llvm.module.flags = !{!21, !72}
 
-!0 = !MDCompileUnit(language: DW_LANG_C_plus_plus, producer: "clang version 3.4 ", isOptimized: true, emissionKind: 1, file: !1, enums: !2, retainedTypes: !5, subprograms: !8, globals: !20, imports: !5)
-!1 = !MDFile(filename: "<unknown>", directory: "/Users/matt/ryan_bug")
+!0 = !DICompileUnit(language: DW_LANG_C_plus_plus, producer: "clang version 3.4 ", isOptimized: true, emissionKind: 1, file: !1, enums: !2, retainedTypes: !5, subprograms: !8, globals: !20, imports: !5)
+!1 = !DIFile(filename: "<unknown>", directory: "/Users/matt/ryan_bug")
 !2 = !{!3}
-!3 = !MDCompositeType(tag: DW_TAG_enumeration_type, line: 20, size: 32, align: 32, file: !1, scope: !4, elements: !6)
-!4 = !MDCompositeType(tag: DW_TAG_structure_type, name: "C", line: 19, size: 8, align: 8, file: !1, elements: !5)
+!3 = !DICompositeType(tag: DW_TAG_enumeration_type, line: 20, size: 32, align: 32, file: !1, scope: !4, elements: !6)
+!4 = !DICompositeType(tag: DW_TAG_structure_type, name: "C", line: 19, size: 8, align: 8, file: !1, elements: !5)
 !5 = !{}
 !6 = !{!7}
-!7 = !MDEnumerator(name: "max_frame_size", value: 0) ; [ DW_TAG_enumerator ] [max_frame_size :: 0]
+!7 = !DIEnumerator(name: "max_frame_size", value: 0) ; [ DW_TAG_enumerator ] [max_frame_size :: 0]
 !8 = !{!9, !24, !41, !65}
-!9 = !MDSubprogram(name: "read_response_size", linkageName: "_Z18read_response_sizev", line: 27, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: true, scopeLine: 27, file: !1, scope: !10, type: !11, function: i32 ()* @_Z18read_response_sizev, variables: !14)
-!10 = !MDFile(filename: "<unknown>", directory: "/Users/matt/ryan_bug")
-!11 = !MDSubroutineType(types: !12)
+!9 = !DISubprogram(name: "read_response_size", linkageName: "_Z18read_response_sizev", line: 27, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: true, scopeLine: 27, file: !1, scope: !10, type: !11, function: i32 ()* @_Z18read_response_sizev, variables: !14)
+!10 = !DIFile(filename: "<unknown>", directory: "/Users/matt/ryan_bug")
+!11 = !DISubroutineType(types: !12)
 !12 = !{!13}
-!13 = !MDBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
+!13 = !DIBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
 !14 = !{!15, !19}
-!15 = !MDLocalVariable(tag: DW_TAG_auto_variable, name: "b", line: 28, scope: !9, file: !10, type: !16)
-!16 = !MDCompositeType(tag: DW_TAG_structure_type, name: "B", line: 16, size: 32, align: 32, file: !1, elements: !17)
+!15 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "b", line: 28, scope: !9, file: !10, type: !16)
+!16 = !DICompositeType(tag: DW_TAG_structure_type, name: "B", line: 16, size: 32, align: 32, file: !1, elements: !17)
 !17 = !{!18}
-!18 = !MDDerivedType(tag: DW_TAG_member, name: "end_of_file", line: 17, size: 32, align: 32, file: !1, scope: !16, baseType: !13)
-!19 = !MDLocalVariable(tag: DW_TAG_auto_variable, name: "c", line: 29, scope: !9, file: !10, type: !13)
+!18 = !DIDerivedType(tag: DW_TAG_member, name: "end_of_file", line: 17, size: 32, align: 32, file: !1, scope: !16, baseType: !13)
+!19 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "c", line: 29, scope: !9, file: !10, type: !13)
 !20 = !{}
 !21 = !{i32 2, !"Dwarf Version", i32 2}
 !22 = !{i64* getelementptr inbounds ({ i64, [56 x i8] }, { i64, [56 x i8] }* @a, i32 0, i32 0)}
-!23 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "p2", line: 12, arg: 2, scope: !24, file: !10, type: !32)
-!24 = !MDSubprogram(name: "min<unsigned long long>", linkageName: "_ZN3__13minIyEERKT_S3_RS1_", line: 12, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: true, scopeLine: 12, file: !1, scope: !25, type: !27, templateParams: !33, variables: !35)
-!25 = !MDNamespace(name: "__1", line: 1, file: !26, scope: null)
-!26 = !MDFile(filename: "main.cpp", directory: "/Users/matt/ryan_bug")
-!27 = !MDSubroutineType(types: !28)
+!23 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "p2", line: 12, arg: 2, scope: !24, file: !10, type: !32)
+!24 = !DISubprogram(name: "min<unsigned long long>", linkageName: "_ZN3__13minIyEERKT_S3_RS1_", line: 12, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: true, scopeLine: 12, file: !1, scope: !25, type: !27, templateParams: !33, variables: !35)
+!25 = !DINamespace(name: "__1", line: 1, file: !26, scope: null)
+!26 = !DIFile(filename: "main.cpp", directory: "/Users/matt/ryan_bug")
+!27 = !DISubroutineType(types: !28)
 !28 = !{!29, !29, !32}
-!29 = !MDDerivedType(tag: DW_TAG_reference_type, baseType: !30)
-!30 = !MDDerivedType(tag: DW_TAG_const_type, baseType: !31)
-!31 = !MDBasicType(tag: DW_TAG_base_type, name: "long long unsigned int", size: 64, align: 64, encoding: DW_ATE_unsigned)
-!32 = !MDDerivedType(tag: DW_TAG_reference_type, baseType: !31)
+!29 = !DIDerivedType(tag: DW_TAG_reference_type, baseType: !30)
+!30 = !DIDerivedType(tag: DW_TAG_const_type, baseType: !31)
+!31 = !DIBasicType(tag: DW_TAG_base_type, name: "long long unsigned int", size: 64, align: 64, encoding: DW_ATE_unsigned)
+!32 = !DIDerivedType(tag: DW_TAG_reference_type, baseType: !31)
 !33 = !{!34}
-!34 = !MDTemplateTypeParameter(name: "_Tp", type: !31)
+!34 = !DITemplateTypeParameter(name: "_Tp", type: !31)
 !35 = !{!36, !37}
-!36 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "p1", line: 12, arg: 1, scope: !24, file: !10, type: !29)
-!37 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "p2", line: 12, arg: 2, scope: !24, file: !10, type: !32)
-!38 = !MDLocation(line: 33, scope: !9)
-!39 = !MDLocation(line: 12, scope: !24, inlinedAt: !38)
-!40 = !MDLocation(line: 9, scope: !41, inlinedAt: !59)
-!41 = !MDSubprogram(name: "min<unsigned long long, __1::A>", linkageName: "_ZN3__13minIyNS_1AEEERKT_S4_RS2_T0_", line: 7, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: true, scopeLine: 8, file: !1, scope: !25, type: !42, templateParams: !53, variables: !55)
-!42 = !MDSubroutineType(types: !43)
+!36 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "p1", line: 12, arg: 1, scope: !24, file: !10, type: !29)
+!37 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "p2", line: 12, arg: 2, scope: !24, file: !10, type: !32)
+!38 = !DILocation(line: 33, scope: !9)
+!39 = !DILocation(line: 12, scope: !24, inlinedAt: !38)
+!40 = !DILocation(line: 9, scope: !41, inlinedAt: !59)
+!41 = !DISubprogram(name: "min<unsigned long long, __1::A>", linkageName: "_ZN3__13minIyNS_1AEEERKT_S4_RS2_T0_", line: 7, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: true, scopeLine: 8, file: !1, scope: !25, type: !42, templateParams: !53, variables: !55)
+!42 = !DISubroutineType(types: !43)
 !43 = !{!29, !29, !32, !44}
-!44 = !MDCompositeType(tag: DW_TAG_structure_type, name: "A", size: 8, align: 8, file: !1, scope: !25, elements: !45)
+!44 = !DICompositeType(tag: DW_TAG_structure_type, name: "A", size: 8, align: 8, file: !1, scope: !25, elements: !45)
 !45 = !{!46}
-!46 = !MDSubprogram(name: "operator()", linkageName: "_ZN3__11AclERKiS2_", line: 1, isLocal: false, isDefinition: false, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: true, scopeLine: 1, file: !1, scope: !44, type: !47)
-!47 = !MDSubroutineType(types: !48)
+!46 = !DISubprogram(name: "operator()", linkageName: "_ZN3__11AclERKiS2_", line: 1, isLocal: false, isDefinition: false, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: true, scopeLine: 1, file: !1, scope: !44, type: !47)
+!47 = !DISubroutineType(types: !48)
 !48 = !{!13, !49, !50, !50}
-!49 = !MDDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, flags: DIFlagArtificial | DIFlagObjectPointer, baseType: !44)
-!50 = !MDDerivedType(tag: DW_TAG_reference_type, baseType: !51)
-!51 = !MDDerivedType(tag: DW_TAG_const_type, baseType: !13)
+!49 = !DIDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, flags: DIFlagArtificial | DIFlagObjectPointer, baseType: !44)
+!50 = !DIDerivedType(tag: DW_TAG_reference_type, baseType: !51)
+!51 = !DIDerivedType(tag: DW_TAG_const_type, baseType: !13)
 !53 = !{!34, !54}
-!54 = !MDTemplateTypeParameter(name: "_Compare", type: !44)
+!54 = !DITemplateTypeParameter(name: "_Compare", type: !44)
 !55 = !{!56, !57, !58}
-!56 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "p1", line: 7, arg: 1, scope: !41, file: !10, type: !29)
-!57 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "p2", line: 7, arg: 2, scope: !41, file: !10, type: !32)
-!58 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "p3", line: 8, arg: 3, scope: !41, file: !10, type: !44)
-!59 = !MDLocation(line: 13, scope: !24, inlinedAt: !38)
+!56 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "p1", line: 7, arg: 1, scope: !41, file: !10, type: !29)
+!57 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "p2", line: 7, arg: 2, scope: !41, file: !10, type: !32)
+!58 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "p3", line: 8, arg: 3, scope: !41, file: !10, type: !44)
+!59 = !DILocation(line: 13, scope: !24, inlinedAt: !38)
 !63 = !{i32 undef}
-!64 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "p1", line: 1, arg: 2, scope: !65, file: !10, type: !50)
-!65 = !MDSubprogram(name: "operator()", linkageName: "_ZN3__11AclERKiS2_", line: 1, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: true, scopeLine: 2, file: !1, scope: !25, type: !47, declaration: !46, variables: !66)
+!64 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "p1", line: 1, arg: 2, scope: !65, file: !10, type: !50)
+!65 = !DISubprogram(name: "operator()", linkageName: "_ZN3__11AclERKiS2_", line: 1, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: true, scopeLine: 2, file: !1, scope: !25, type: !47, declaration: !46, variables: !66)
 !66 = !{!67, !69, !70}
-!67 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "this", arg: 1, flags: DIFlagArtificial | DIFlagObjectPointer, scope: !65, type: !68)
-!68 = !MDDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, baseType: !44)
-!69 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "p1", line: 1, arg: 2, scope: !65, file: !10, type: !50)
-!70 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "", line: 2, arg: 3, scope: !65, file: !10, type: !50)
-!71 = !MDLocation(line: 1, scope: !65, inlinedAt: !40)
+!67 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "this", arg: 1, flags: DIFlagArtificial | DIFlagObjectPointer, scope: !65, type: !68)
+!68 = !DIDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, baseType: !44)
+!69 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "p1", line: 1, arg: 2, scope: !65, file: !10, type: !50)
+!70 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "", line: 2, arg: 3, scope: !65, file: !10, type: !50)
+!71 = !DILocation(line: 1, scope: !65, inlinedAt: !40)
 !72 = !{i32 1, !"Debug Info Version", i32 3}
diff --git a/test/CodeGen/X86/stack-protector.ll b/test/CodeGen/X86/stack-protector.ll
index acaba6dc17f..398b8548747 100644
--- a/test/CodeGen/X86/stack-protector.ll
+++ b/test/CodeGen/X86/stack-protector.ll
@@ -2097,7 +2097,7 @@ entry:
 ; test18a: Addr-of a variable passed into an invoke instruction.
 ;          no ssp attribute
 ; Requires no protector.
-define i32 @test18a()  {
+define i32 @test18a() personality i8* bitcast (i32 (...)* @__gxx_personality_v0 to i8*) {
 entry:
 ; LINUX-I386-LABEL: test18a:
 ; LINUX-I386-NOT: calll __stack_chk_fail
@@ -2125,7 +2125,7 @@ invoke.cont:
   ret i32 0
 
 lpad:
-  %0 = landingpad { i8*, i32 } personality i8* bitcast (i32 (...)* @__gxx_personality_v0 to i8*)
+  %0 = landingpad { i8*, i32 }
           catch i8* null
   ret i32 0
 }
@@ -2134,7 +2134,7 @@ lpad:
 ;          ssp attribute
 ; Requires no protector.
 ; Function Attrs: ssp 
-define i32 @test18b() #0 {
+define i32 @test18b() #0 personality i8* bitcast (i32 (...)* @__gxx_personality_v0 to i8*) {
 entry:
 ; LINUX-I386-LABEL: test18b:
 ; LINUX-I386-NOT: calll __stack_chk_fail
@@ -2162,7 +2162,7 @@ invoke.cont:
   ret i32 0
 
 lpad:
-  %0 = landingpad { i8*, i32 } personality i8* bitcast (i32 (...)* @__gxx_personality_v0 to i8*)
+  %0 = landingpad { i8*, i32 }
           catch i8* null
   ret i32 0
 }
@@ -2171,7 +2171,7 @@ lpad:
 ;          sspstrong attribute
 ; Requires protector.
 ; Function Attrs: sspstrong 
-define i32 @test18c() #1 {
+define i32 @test18c() #1 personality i8* bitcast (i32 (...)* @__gxx_personality_v0 to i8*) {
 entry:
 ; LINUX-I386-LABEL: test18c:
 ; LINUX-I386: mov{{l|q}} %gs:
@@ -2199,7 +2199,7 @@ invoke.cont:
   ret i32 0
 
 lpad:
-  %0 = landingpad { i8*, i32 } personality i8* bitcast (i32 (...)* @__gxx_personality_v0 to i8*)
+  %0 = landingpad { i8*, i32 }
           catch i8* null
   ret i32 0
 }
@@ -2208,7 +2208,7 @@ lpad:
 ;          sspreq attribute
 ; Requires protector.
 ; Function Attrs: sspreq 
-define i32 @test18d() #2 {
+define i32 @test18d() #2 personality i8* bitcast (i32 (...)* @__gxx_personality_v0 to i8*) {
 entry:
 ; LINUX-I386-LABEL: test18d:
 ; LINUX-I386: mov{{l|q}} %gs:
@@ -2236,7 +2236,7 @@ invoke.cont:
   ret i32 0
 
 lpad:
-  %0 = landingpad { i8*, i32 } personality i8* bitcast (i32 (...)* @__gxx_personality_v0 to i8*)
+  %0 = landingpad { i8*, i32 }
           catch i8* null
   ret i32 0
 }
@@ -2244,7 +2244,7 @@ lpad:
 ;           (GEP followed by an invoke)
 ;          no ssp attribute
 ; Requires no protector.
-define i32 @test19a()  {
+define i32 @test19a() personality i8* bitcast (i32 (...)* @__gxx_personality_v0 to i8*) {
 entry:
 ; LINUX-I386-LABEL: test19a:
 ; LINUX-I386-NOT: calll __stack_chk_fail
@@ -2274,7 +2274,7 @@ invoke.cont:
   ret i32 0
 
 lpad:
-  %0 = landingpad { i8*, i32 } personality i8* bitcast (i32 (...)* @__gxx_personality_v0 to i8*)
+  %0 = landingpad { i8*, i32 }
           catch i8* null
   ret i32 0
 }
@@ -2284,7 +2284,7 @@ lpad:
 ;          ssp attribute
 ; Requires no protector.
 ; Function Attrs: ssp 
-define i32 @test19b() #0 {
+define i32 @test19b() #0 personality i8* bitcast (i32 (...)* @__gxx_personality_v0 to i8*) {
 entry:
 ; LINUX-I386-LABEL: test19b:
 ; LINUX-I386-NOT: calll __stack_chk_fail
@@ -2314,7 +2314,7 @@ invoke.cont:
   ret i32 0
 
 lpad:
-  %0 = landingpad { i8*, i32 } personality i8* bitcast (i32 (...)* @__gxx_personality_v0 to i8*)
+  %0 = landingpad { i8*, i32 }
           catch i8* null
   ret i32 0
 }
@@ -2324,7 +2324,7 @@ lpad:
 ;          sspstrong attribute
 ; Requires protector.
 ; Function Attrs: sspstrong 
-define i32 @test19c() #1 {
+define i32 @test19c() #1 personality i8* bitcast (i32 (...)* @__gxx_personality_v0 to i8*) {
 entry:
 ; LINUX-I386-LABEL: test19c:
 ; LINUX-I386: mov{{l|q}} %gs:
@@ -2354,7 +2354,7 @@ invoke.cont:
   ret i32 0
 
 lpad:
-  %0 = landingpad { i8*, i32 } personality i8* bitcast (i32 (...)* @__gxx_personality_v0 to i8*)
+  %0 = landingpad { i8*, i32 }
           catch i8* null
   ret i32 0
 }
@@ -2364,7 +2364,7 @@ lpad:
 ;          sspreq attribute
 ; Requires protector.
 ; Function Attrs: sspreq 
-define i32 @test19d() #2 {
+define i32 @test19d() #2 personality i8* bitcast (i32 (...)* @__gxx_personality_v0 to i8*) {
 entry:
 ; LINUX-I386-LABEL: test19d:
 ; LINUX-I386: mov{{l|q}} %gs:
@@ -2398,7 +2398,7 @@ invoke.cont:
   ret i32 0
 
 lpad:
-  %0 = landingpad { i8*, i32 } personality i8* bitcast (i32 (...)* @__gxx_personality_v0 to i8*)
+  %0 = landingpad { i8*, i32 }
           catch i8* null
   ret i32 0
 }
diff --git a/test/CodeGen/X86/statepoint-allocas.ll b/test/CodeGen/X86/statepoint-allocas.ll
index eb34a42f462..4af33e1f547 100644
--- a/test/CodeGen/X86/statepoint-allocas.ll
+++ b/test/CodeGen/X86/statepoint-allocas.ll
@@ -21,7 +21,7 @@ define i32 addrspace(1)* @test(i32 addrspace(1)* %ptr) gc "statepoint-example" {
 entry:
   %alloca = alloca i32 addrspace(1)*, align 8
   store i32 addrspace(1)* %ptr, i32 addrspace(1)** %alloca
-  call i32 (i1 ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_i1f(i1 ()* @return_i1, i32 0, i32 0, i32 0, i32 addrspace(1)** %alloca)
+  call i32 (i64, i32, i1 ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_i1f(i64 0, i32 0, i1 ()* @return_i1, i32 0, i32 0, i32 0, i32 0, i32 addrspace(1)** %alloca)
   %rel = load i32 addrspace(1)*, i32 addrspace(1)** %alloca
   ret i32 addrspace(1)* %rel
 }
@@ -38,11 +38,11 @@ define i32 addrspace(1)* @test2(i32 addrspace(1)* %ptr) gc "statepoint-example"
 entry:
   %alloca = alloca i32 addrspace(1)*, align 8
   store i32 addrspace(1)* %ptr, i32 addrspace(1)** %alloca
-  call i32 (i1 ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_i1f(i1 ()* @return_i1, i32 0, i32 0, i32 1, i32 addrspace(1)** %alloca)
+  call i32 (i64, i32, i1 ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_i1f(i64 0, i32 0, i1 ()* @return_i1, i32 0, i32 0, i32 0, i32 1, i32 addrspace(1)** %alloca)
   ret i32 addrspace(1)* null
 }
 
-declare i32 @llvm.experimental.gc.statepoint.p0f_i1f(i1 ()*, i32, i32, ...)
+declare i32 @llvm.experimental.gc.statepoint.p0f_i1f(i64, i32, i1 ()*, i32, i32, ...)
 
 
 ; CHECK-LABEL: .section .llvm_stackmaps
@@ -66,13 +66,18 @@ declare i32 @llvm.experimental.gc.statepoint.p0f_i1f(i1 ()*, i32, i32, ...)
 
 ; Large Constants
 ; Statepoint ID only
-; CHECK: .quad	2882400000
+; CHECK: .quad	0
 
 ; Callsites
 ; The GC one
 ; CHECK: .long	.Ltmp1-test
 ; CHECK: .short	0
-; CHECK: .short	3
+; CHECK: .short	4
+; SmallConstant (0)
+; CHECK: .byte	4
+; CHECK: .byte	8
+; CHECK: .short	0
+; CHECK: .long	0
 ; SmallConstant (0)
 ; CHECK: .byte	4
 ; CHECK: .byte	8
@@ -96,7 +101,12 @@ declare i32 @llvm.experimental.gc.statepoint.p0f_i1f(i1 ()*, i32, i32, ...)
 ; The Deopt one
 ; CHECK: .long	.Ltmp3-test2
 ; CHECK: .short	0
-; CHECK: .short	3
+; CHECK: .short	4
+; SmallConstant (0)
+; CHECK: .byte	4
+; CHECK: .byte	8
+; CHECK: .short	0
+; CHECK: .long	0
 ; SmallConstant (0)
 ; CHECK: .byte	4
 ; CHECK: .byte	8
@@ -118,4 +128,3 @@ declare i32 @llvm.experimental.gc.statepoint.p0f_i1f(i1 ()*, i32, i32, ...)
 ; CHECK: .short	0
 ; CHECK: .align	8
 
-
diff --git a/test/CodeGen/X86/statepoint-call-lowering.ll b/test/CodeGen/X86/statepoint-call-lowering.ll
index 9f6e4f967c9..8f352b7728c 100644
--- a/test/CodeGen/X86/statepoint-call-lowering.ll
+++ b/test/CodeGen/X86/statepoint-call-lowering.ll
@@ -20,7 +20,7 @@ define i1 @test_i1_return() gc "statepoint-example" {
 ; CHECK: popq %rdx
 ; CHECK: retq
 entry:
-  %safepoint_token = tail call i32 (i1 ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_i1f(i1 ()* @return_i1, i32 0, i32 0, i32 0)
+  %safepoint_token = tail call i32 (i64, i32, i1 ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_i1f(i64 0, i32 0, i1 ()* @return_i1, i32 0, i32 0, i32 0, i32 0)
   %call1 = call zeroext i1 @llvm.experimental.gc.result.i1(i32 %safepoint_token)
   ret i1 %call1
 }
@@ -32,7 +32,7 @@ define i32 @test_i32_return() gc "statepoint-example" {
 ; CHECK: popq %rdx
 ; CHECK: retq
 entry:
-  %safepoint_token = tail call i32 (i32 ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_i32f(i32 ()* @return_i32, i32 0, i32 0, i32 0)
+  %safepoint_token = tail call i32 (i64, i32, i32 ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_i32f(i64 0, i32 0, i32 ()* @return_i32, i32 0, i32 0, i32 0, i32 0)
   %call1 = call zeroext i32 @llvm.experimental.gc.result.i32(i32 %safepoint_token)
   ret i32 %call1
 }
@@ -44,7 +44,7 @@ define i32* @test_i32ptr_return() gc "statepoint-example" {
 ; CHECK: popq %rdx
 ; CHECK: retq
 entry:
-  %safepoint_token = tail call i32 (i32* ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_p0i32f(i32* ()* @return_i32ptr, i32 0, i32 0, i32 0)
+  %safepoint_token = tail call i32 (i64, i32, i32* ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_p0i32f(i64 0, i32 0, i32* ()* @return_i32ptr, i32 0, i32 0, i32 0, i32 0)
   %call1 = call i32* @llvm.experimental.gc.result.p0i32(i32 %safepoint_token)
   ret i32* %call1
 }
@@ -56,7 +56,7 @@ define float @test_float_return() gc "statepoint-example" {
 ; CHECK: popq %rax
 ; CHECK: retq
 entry:
-  %safepoint_token = tail call i32 (float ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_f32f(float ()* @return_float, i32 0, i32 0, i32 0)
+  %safepoint_token = tail call i32 (i64, i32, float ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_f32f(i64 0, i32 0, float ()* @return_float, i32 0, i32 0, i32 0, i32 0)
   %call1 = call float @llvm.experimental.gc.result.f32(i32 %safepoint_token)
   ret float %call1
 }
@@ -70,8 +70,8 @@ define i1 @test_relocate(i32 addrspace(1)* %a) gc "statepoint-example" {
 ; CHECK-NEXT: popq %rdx
 ; CHECK-NEXT: retq
 entry:
-  %safepoint_token = tail call i32 (i1 ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_i1f(i1 ()* @return_i1, i32 0, i32 0, i32 0, i32 addrspace(1)* %a)
-  %call1 = call i32 addrspace(1)* @llvm.experimental.gc.relocate.p1i32(i32 %safepoint_token, i32 4, i32 4)
+  %safepoint_token = tail call i32 (i64, i32, i1 ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_i1f(i64 0, i32 0, i1 ()* @return_i1, i32 0, i32 0, i32 0, i32 0, i32 addrspace(1)* %a)
+  %call1 = call i32 addrspace(1)* @llvm.experimental.gc.relocate.p1i32(i32 %safepoint_token,  i32 7, i32 7)
   %call2 = call zeroext i1 @llvm.experimental.gc.result.i1(i32 %safepoint_token)
   ret i1 %call2
 }
@@ -81,24 +81,37 @@ define void @test_void_vararg() gc "statepoint-example" {
 ; Check a statepoint wrapping a *void* returning vararg function works
 ; CHECK: callq varargf
 entry:
-  %safepoint_token = tail call i32 (void (i32, ...)*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidi32varargf(void (i32, ...)* @varargf, i32 2, i32 0, i32 42, i32 43, i32 0)
+  %safepoint_token = tail call i32 (i64, i32, void (i32, ...)*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidi32varargf(i64 0, i32 0, void (i32, ...)* @varargf, i32 2, i32 0, i32 42, i32 43, i32 0, i32 0)
   ;; if we try to use the result from a statepoint wrapping a
   ;; non-void-returning varargf, we will experience a crash.
   ret void
 }
 
-declare i32 @llvm.experimental.gc.statepoint.p0f_i1f(i1 ()*, i32, i32, ...)
+define i1 @test_i1_return_patchable() gc "statepoint-example" {
+; CHECK-LABEL: test_i1_return_patchable
+; A patchable variant of test_i1_return
+; CHECK: pushq %rax
+; CHECK: nopl
+; CHECK: popq %rdx
+; CHECK: retq
+entry:
+  %safepoint_token = tail call i32 (i64, i32, i1 ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_i1f(i64 0, i32 3, i1 ()*null, i32 0, i32 0, i32 0, i32 0)
+  %call1 = call zeroext i1 @llvm.experimental.gc.result.i1(i32 %safepoint_token)
+  ret i1 %call1
+}
+
+declare i32 @llvm.experimental.gc.statepoint.p0f_i1f(i64, i32, i1 ()*, i32, i32, ...)
 declare i1 @llvm.experimental.gc.result.i1(i32)
 
-declare i32 @llvm.experimental.gc.statepoint.p0f_i32f(i32 ()*, i32, i32, ...)
+declare i32 @llvm.experimental.gc.statepoint.p0f_i32f(i64, i32, i32 ()*, i32, i32, ...)
 declare i32 @llvm.experimental.gc.result.i32(i32)
 
-declare i32 @llvm.experimental.gc.statepoint.p0f_p0i32f(i32* ()*, i32, i32, ...)
+declare i32 @llvm.experimental.gc.statepoint.p0f_p0i32f(i64, i32, i32* ()*, i32, i32, ...)
 declare i32* @llvm.experimental.gc.result.p0i32(i32)
 
-declare i32 @llvm.experimental.gc.statepoint.p0f_f32f(float ()*, i32, i32, ...)
+declare i32 @llvm.experimental.gc.statepoint.p0f_f32f(i64, i32, float ()*, i32, i32, ...)
 declare float @llvm.experimental.gc.result.f32(i32)
 
-declare i32 @llvm.experimental.gc.statepoint.p0f_isVoidi32varargf(void (i32, ...)*, i32, i32, ...)
+declare i32 @llvm.experimental.gc.statepoint.p0f_isVoidi32varargf(i64, i32, void (i32, ...)*, i32, i32, ...)
 
 declare i32 addrspace(1)* @llvm.experimental.gc.relocate.p1i32(i32, i32, i32)
diff --git a/test/CodeGen/X86/statepoint-far-call.ll b/test/CodeGen/X86/statepoint-far-call.ll
new file mode 100644
index 00000000000..cd8dd0f35a2
--- /dev/null
+++ b/test/CodeGen/X86/statepoint-far-call.ll
@@ -0,0 +1,22 @@
+; RUN: llc < %s | FileCheck %s
+; Test to check that Statepoints with X64 far-immediate targets
+; are lowered correctly to an indirect call via a scratch register.
+
+target datalayout = "e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-pc-win64"
+
+define void @test_far_call() gc "statepoint-example" {
+; CHECK-LABEL: test_far_call
+; CHECK: pushq %rax
+; CHECK: movabsq $140727162896504, %rax 
+; CHECK: callq *%rax
+; CHECK: popq %rax
+; CHECK: retq
+
+entry:
+  %safepoint_token = call i32 (i64, i32, void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 0, i32 0, void ()* inttoptr (i64 140727162896504 to void ()*), i32 0, i32 0, i32 0, i32 0)
+  ret void
+}
+
+declare i32 @llvm.experimental.gc.statepoint.p0f_isVoidf(i64, i32, void ()*, i32, i32, ...) 
+
diff --git a/test/CodeGen/X86/statepoint-forward.ll b/test/CodeGen/X86/statepoint-forward.ll
index 0b296cf0a84..698229e705f 100644
--- a/test/CodeGen/X86/statepoint-forward.ll
+++ b/test/CodeGen/X86/statepoint-forward.ll
@@ -25,8 +25,8 @@ entry:
   %before = load i32 addrspace(1)*, i32 addrspace(1)* addrspace(1)* %p
   %cmp1 = call i1 @f(i32 addrspace(1)* %before)
   call void @llvm.assume(i1 %cmp1)
-  %safepoint_token = tail call i32 (void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(void ()* @func, i32 0, i32 0, i32 0, i32 addrspace(1)* addrspace(1)* %p)
-  %pnew = call i32 addrspace(1)* addrspace(1)* @llvm.experimental.gc.relocate.p1p1i32(i32 %safepoint_token, i32 4, i32 4)
+  %safepoint_token = tail call i32 (i64, i32, void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 0, i32 0, void ()* @func, i32 0, i32 0, i32 0, i32 0, i32 addrspace(1)* addrspace(1)* %p)
+  %pnew = call i32 addrspace(1)* addrspace(1)* @llvm.experimental.gc.relocate.p1p1i32(i32 %safepoint_token,  i32 7, i32 7)
   %after = load i32 addrspace(1)*, i32 addrspace(1)* addrspace(1)* %pnew
   %cmp2 = call i1 @f(i32 addrspace(1)* %after)
   ret i1 %cmp2
@@ -44,8 +44,8 @@ entry:
   %cmp1 = call i1 @f(i32 addrspace(1)* %v)
   call void @llvm.assume(i1 %cmp1)
   store i32 addrspace(1)* %v, i32 addrspace(1)* addrspace(1)* %p
-  %safepoint_token = tail call i32 (void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(void ()* @func, i32 0, i32 0, i32 0, i32 addrspace(1)* addrspace(1)* %p)
-  %pnew = call i32 addrspace(1)* addrspace(1)* @llvm.experimental.gc.relocate.p1p1i32(i32 %safepoint_token, i32 4, i32 4)
+  %safepoint_token = tail call i32 (i64, i32, void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 0, i32 0, void ()* @func, i32 0, i32 0, i32 0, i32 0, i32 addrspace(1)* addrspace(1)* %p)
+  %pnew = call i32 addrspace(1)* addrspace(1)* @llvm.experimental.gc.relocate.p1p1i32(i32 %safepoint_token,  i32 7, i32 7)
   %after = load i32 addrspace(1)*, i32 addrspace(1)* addrspace(1)* %pnew
   %cmp2 = call i1 @f(i32 addrspace(1)* %after)
   ret i1 %cmp2
@@ -72,7 +72,7 @@ entry:
   %before = load i32 addrspace(1)*, i32 addrspace(1)** %p
   %cmp1 = call i1 @f(i32 addrspace(1)* %before)
   call void @llvm.assume(i1 %cmp1)
-  call i32 (void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(void ()* @func, i32 0, i32 0, i32 0)
+  call i32 (i64, i32, void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 0, i32 0, void ()* @func, i32 0, i32 0, i32 0, i32 0)
   %after = load i32 addrspace(1)*, i32 addrspace(1)** %p
   %cmp2 = call i1 @f(i32 addrspace(1)* %after)
   ret i1 %cmp2
@@ -90,7 +90,7 @@ entry:
   %cmp1 = call i1 @f(i32 addrspace(1)* %v)
   call void @llvm.assume(i1 %cmp1)
   store i32 addrspace(1)* %v, i32 addrspace(1)** %p
-  call i32 (void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(void ()* @func, i32 0, i32 0, i32 0)
+  call i32 (i64, i32, void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 0, i32 0, void ()* @func, i32 0, i32 0, i32 0, i32 0)
   %after = load i32 addrspace(1)*, i32 addrspace(1)** %p
   %cmp2 = call i1 @f(i32 addrspace(1)* %after)
   ret i1 %cmp2
@@ -102,6 +102,5 @@ entry:
 }
 
 declare void @llvm.assume(i1)
-declare i32 @llvm.experimental.gc.statepoint.p0f_isVoidf(void ()*, i32, i32, ...)
+declare i32 @llvm.experimental.gc.statepoint.p0f_isVoidf(i64, i32, void ()*, i32, i32, ...)
 declare i32 addrspace(1)* addrspace(1)* @llvm.experimental.gc.relocate.p1p1i32(i32, i32, i32) #3
-
diff --git a/test/CodeGen/X86/statepoint-gctransition-call-lowering.ll b/test/CodeGen/X86/statepoint-gctransition-call-lowering.ll
new file mode 100644
index 00000000000..61b8ded2c47
--- /dev/null
+++ b/test/CodeGen/X86/statepoint-gctransition-call-lowering.ll
@@ -0,0 +1,133 @@
+; RUN: llc < %s | FileCheck %s
+; This file contains a collection of basic tests to ensure we didn't
+; screw up normal call lowering when a statepoint is a GC transition.
+
+target datalayout = "e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-pc-linux-gnu"
+
+declare zeroext i1 @return_i1()
+declare zeroext i32 @return_i32()
+declare zeroext i32 @return_i32_with_args(i32, i8*)
+declare i32* @return_i32ptr()
+declare float @return_float()
+declare void @varargf(i32, ...)
+
+define i1 @test_i1_return() gc "statepoint-example" {
+; CHECK-LABEL: test_i1_return
+; This is just checking that a i1 gets lowered normally when there's no extra
+; state arguments to the statepoint
+; CHECK: pushq %rax
+; CHECK: callq return_i1
+; CHECK: popq %rdx
+; CHECK: retq
+entry:
+  %safepoint_token = tail call i32 (i64, i32, i1 ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_i1f(i64 0, i32 0, i1 ()* @return_i1, i32 0, i32 1, i32 0, i32 0)
+  %call1 = call zeroext i1 @llvm.experimental.gc.result.i1(i32 %safepoint_token)
+  ret i1 %call1
+}
+
+define i32 @test_i32_return() gc "statepoint-example" {
+; CHECK-LABEL: test_i32_return
+; CHECK: pushq %rax
+; CHECK: callq return_i32
+; CHECK: popq %rdx
+; CHECK: retq
+entry:
+  %safepoint_token = tail call i32 (i64, i32, i32 ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_i32f(i64 0, i32 0, i32 ()* @return_i32, i32 0, i32 1, i32 0, i32 0)
+  %call1 = call zeroext i32 @llvm.experimental.gc.result.i32(i32 %safepoint_token)
+  ret i32 %call1
+}
+
+define i32* @test_i32ptr_return() gc "statepoint-example" {
+; CHECK-LABEL: test_i32ptr_return
+; CHECK: pushq %rax
+; CHECK: callq return_i32ptr
+; CHECK: popq %rdx
+; CHECK: retq
+entry:
+  %safepoint_token = tail call i32 (i64, i32, i32* ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_p0i32f(i64 0, i32 0, i32* ()* @return_i32ptr, i32 0, i32 1, i32 0, i32 0)
+  %call1 = call i32* @llvm.experimental.gc.result.p0i32(i32 %safepoint_token)
+  ret i32* %call1
+}
+
+define float @test_float_return() gc "statepoint-example" {
+; CHECK-LABEL: test_float_return
+; CHECK: pushq %rax
+; CHECK: callq return_float
+; CHECK: popq %rax
+; CHECK: retq
+entry:
+  %safepoint_token = tail call i32 (i64, i32, float ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_f32f(i64 0, i32 0, float ()* @return_float, i32 0, i32 1, i32 0, i32 0)
+  %call1 = call float @llvm.experimental.gc.result.f32(i32 %safepoint_token)
+  ret float %call1
+}
+
+define i1 @test_relocate(i32 addrspace(1)* %a) gc "statepoint-example" {
+; CHECK-LABEL: test_relocate
+; Check that an ununsed relocate has no code-generation impact
+; CHECK: pushq %rax
+; CHECK: callq return_i1
+; CHECK-NEXT: .Ltmp9:
+; CHECK-NEXT: popq %rdx
+; CHECK-NEXT: retq
+entry:
+  %safepoint_token = tail call i32 (i64, i32, i1 ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_i1f(i64 0, i32 0, i1 ()* @return_i1, i32 0, i32 1, i32 0, i32 0, i32 addrspace(1)* %a)
+  %call1 = call i32 addrspace(1)* @llvm.experimental.gc.relocate.p1i32(i32 %safepoint_token, i32 7, i32 7)
+  %call2 = call zeroext i1 @llvm.experimental.gc.result.i1(i32 %safepoint_token)
+  ret i1 %call2
+}
+
+define void @test_void_vararg() gc "statepoint-example" {
+; CHECK-LABEL: test_void_vararg
+; Check a statepoint wrapping a *void* returning vararg function works
+; CHECK: callq varargf
+entry:
+  %safepoint_token = tail call i32 (i64, i32, void (i32, ...)*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidi32varargf(i64 0, i32 0, void (i32, ...)* @varargf, i32 2, i32 1, i32 42, i32 43, i32 0, i32 0)
+  ;; if we try to use the result from a statepoint wrapping a
+  ;; non-void-returning varargf, we will experience a crash.
+  ret void
+}
+
+define i32 @test_transition_args() gc "statepoint-example" {
+; CHECK-LABEL: test_transition_args
+; CHECK: pushq %rax
+; CHECK: callq return_i32
+; CHECK: popq %rdx
+; CHECK: retq
+entry:
+  %val = alloca i32
+  %safepoint_token = call i32 (i64, i32, i32 ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_i32f(i64 0, i32 0, i32 ()* @return_i32, i32 0, i32 1, i32 2, i32* %val, i64 42, i32 0)
+  %call1 = call i32 @llvm.experimental.gc.result.i32(i32 %safepoint_token)
+  ret i32 %call1
+}
+
+define i32 @test_transition_args_2() gc "statepoint-example" {
+; CHECK-LABEL: test_transition_args_2
+; CHECK: pushq %rax
+; CHECK: callq return_i32
+; CHECK: popq %rdx
+; CHECK: retq
+entry:
+  %val = alloca i32
+  %arg = alloca i8
+  %safepoint_token = call i32 (i64, i32, i32 (i32, i8*)*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_i32i32p0i8f(i64 0, i32 0, i32 (i32, i8*)* @return_i32_with_args, i32 2, i32 1, i32 0, i8* %arg, i32 2, i32* %val, i64 42, i32 0)
+  %call1 = call i32 @llvm.experimental.gc.result.i32(i32 %safepoint_token)
+  ret i32 %call1
+}
+
+declare i32 @llvm.experimental.gc.statepoint.p0f_i1f(i64, i32, i1 ()*, i32, i32, ...)
+declare i1 @llvm.experimental.gc.result.i1(i32)
+
+declare i32 @llvm.experimental.gc.statepoint.p0f_i32f(i64, i32, i32 ()*, i32, i32, ...)
+declare i32 @llvm.experimental.gc.statepoint.p0f_i32i32p0i8f(i64, i32, i32 (i32, i8*)*, i32, i32, ...)
+declare i32 @llvm.experimental.gc.result.i32(i32)
+
+declare i32 @llvm.experimental.gc.statepoint.p0f_p0i32f(i64, i32, i32* ()*, i32, i32, ...)
+declare i32* @llvm.experimental.gc.result.p0i32(i32)
+
+declare i32 @llvm.experimental.gc.statepoint.p0f_f32f(i64, i32, float ()*, i32, i32, ...)
+declare float @llvm.experimental.gc.result.f32(i32)
+
+declare i32 @llvm.experimental.gc.statepoint.p0f_isVoidi32varargf(i64, i32, void (i32, ...)*, i32, i32, ...)
+
+declare i32 addrspace(1)* @llvm.experimental.gc.relocate.p1i32(i32, i32, i32)
\ No newline at end of file
diff --git a/test/CodeGen/X86/statepoint-invoke.ll b/test/CodeGen/X86/statepoint-invoke.ll
index 177eb96e0d6..81b9ab89ebc 100644
--- a/test/CodeGen/X86/statepoint-invoke.ll
+++ b/test/CodeGen/X86/statepoint-invoke.ll
@@ -2,18 +2,56 @@
 
 target triple = "x86_64-pc-linux-gnu"
 
+declare void @"some_call"(i64 addrspace(1)*)
 declare i64 addrspace(1)* @"some_other_call"(i64 addrspace(1)*)
 
 declare i32 @"personality_function"()
 
-define i64 addrspace(1)* @test_result(i64 addrspace(1)* %obj, 
+define i64 addrspace(1)* @test_basic(i64 addrspace(1)* %obj,
+                                     i64 addrspace(1)* %obj1)
+gc "statepoint-example" personality i32 ()* @"personality_function" {
+entry:
+  ; CHECK: Ltmp{{[0-9]+}}:
+  ; CHECK: callq some_call
+  ; CHECK: Ltmp{{[0-9]+}}:
+  %0 = invoke i32 (i64, i32, void (i64 addrspace(1)*)*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidp1i64f(i64 0, i32 0, void (i64 addrspace(1)*)* @some_call, i32 1, i32 0, i64 addrspace(1)* %obj, i32 0, i32 5, i32 0, i32 -1, i32 0, i32 0, i32 0, i64 addrspace(1)* %obj, i64 addrspace(1)* %obj1)
+          to label %invoke_safepoint_normal_dest unwind label %exceptional_return
+
+invoke_safepoint_normal_dest:
+  ; CHECK: movq
+  %obj.relocated = call coldcc i64 addrspace(1)* @llvm.experimental.gc.relocate.p1i64(i32 %0, i32 13, i32 13)
+  %obj1.relocated = call coldcc i64 addrspace(1)* @llvm.experimental.gc.relocate.p1i64(i32 %0, i32 14, i32 14)
+  br label %normal_return
+
+normal_return:
+  ; CHECK: retq
+  ret i64 addrspace(1)* %obj.relocated
+
+exceptional_return:
+  ; CHECK: Ltmp{{[0-9]+}}:
+  ; CHECK: movq
+  ; CHECK: retq
+  %landing_pad = landingpad { i8*, i32 }
+          cleanup
+  %relocate_token = extractvalue { i8*, i32 } %landing_pad, 1
+  %obj.relocated1 = call coldcc i64 addrspace(1)* @llvm.experimental.gc.relocate.p1i64(i32 %relocate_token, i32 13, i32 13)
+  %obj1.relocated1 = call coldcc i64 addrspace(1)* @llvm.experimental.gc.relocate.p1i64(i32 %relocate_token, i32 14, i32 14)
+  ret i64 addrspace(1)* %obj1.relocated1
+}
+; CHECK-LABEL: GCC_except_table{{[0-9]+}}:
+; CHECK: .long  .Ltmp{{[0-9]+}}-.Ltmp{{[0-9]+}}
+; CHECK: .long  .Ltmp{{[0-9]+}}-.Lfunc_begin{{[0-9]+}}
+; CHECK: .byte  0
+; CHECK: .align 4
+
+define i64 addrspace(1)* @test_result(i64 addrspace(1)* %obj,
                                       i64 addrspace(1)* %obj1)
-  gc "statepoint-example" {
+  gc "statepoint-example" personality i32 ()* @personality_function {
 entry:
   ; CHECK: .Ltmp{{[0-9]+}}:
   ; CHECK: callq some_other_call
   ; CHECK: .Ltmp{{[0-9]+}}:
-  %0 = invoke i32 (i64 addrspace(1)* (i64 addrspace(1)*)*, i32, i32, ...)* @llvm.experimental.gc.statepoint.p0f_p1i64p1i64f(i64 addrspace(1)* (i64 addrspace(1)*)* @some_other_call, i32 1, i32 0, i64 addrspace(1)* %obj, i32 5, i32 0, i32 -1, i32 0, i32 0, i32 0, i64 addrspace(1)* %obj, i64 addrspace(1)* %obj1)
+  %0 = invoke i32 (i64, i32, i64 addrspace(1)* (i64 addrspace(1)*)*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_p1i64p1i64f(i64 0, i32 0, i64 addrspace(1)* (i64 addrspace(1)*)* @some_other_call, i32 1, i32 0, i64 addrspace(1)* %obj, i32 0, i32 5, i32 0, i32 -1, i32 0, i32 0, i32 0, i64 addrspace(1)* %obj, i64 addrspace(1)* %obj1)
           to label %normal_return unwind label %exceptional_return
 
 normal_return:
@@ -24,11 +62,12 @@ normal_return:
 
 exceptional_return:
   ; CHECK: .Ltmp{{[0-9]+}}:
-  ; CHECK: popq
-  ; CHECK: retq
-  %landing_pad = landingpad { i8*, i32 } personality i32 ()* @personality_function
+  ; CHECK: movq
+  %landing_pad = landingpad { i8*, i32 }
           cleanup
-  ret i64 addrspace(1)* %obj
+  %relocate_token = extractvalue { i8*, i32 } %landing_pad, 1
+  %obj.relocated = call coldcc i64 addrspace(1)* @llvm.experimental.gc.relocate.p1i64(i32 %relocate_token, i32 13, i32 13)
+  ret i64 addrspace(1)* %obj.relocated
 }
 ; CHECK-LABEL: GCC_except_table{{[0-9]+}}:
 ; CHECK: .long .Ltmp{{[0-9]+}}-.Ltmp{{[0-9]+}}
@@ -36,5 +75,124 @@ exceptional_return:
 ; CHECK: .byte 0
 ; CHECK: .align 4
 
-declare i32 @llvm.experimental.gc.statepoint.p0f_p1i64p1i64f(i64 addrspace(1)* (i64 addrspace(1)*)*, i32, i32, ...)
+define i64 addrspace(1)* @test_same_val(i1 %cond, i64 addrspace(1)* %val1, i64 addrspace(1)* %val2, i64 addrspace(1)* %val3)
+  gc "statepoint-example" personality i32 ()* @"personality_function" {
+entry:
+  br i1 %cond, label %left, label %right
+
+left:
+  ; CHECK-LABEL: %left
+  ; CHECK: movq %rdx, 8(%rsp)
+  ; CHECK: movq
+  ; CHECK: callq some_call
+  %sp1 = invoke i32 (i64, i32, void (i64 addrspace(1)*)*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidp1i64f(i64 0, i32 0, void (i64 addrspace(1)*)* @some_call, i32 1, i32 0, i64 addrspace(1)* %val1, i32 0, i32 5, i32 0, i32 -1, i32 0, i32 0, i32 0, i64 addrspace(1)* %val1, i64 addrspace(1)* %val2)
+           to label %left.relocs unwind label %exceptional_return.left
+
+left.relocs:
+  ; CHECK: movq (%rsp),
+  ; CHECK: movq 8(%rsp), [[REGVAL2:%[a-z]+]]
+  %val1.relocated = call coldcc i64 addrspace(1)* @llvm.experimental.gc.relocate.p1i64(i32 %sp1, i32 13, i32 13)
+  %val2.relocated_left = call coldcc i64 addrspace(1)* @llvm.experimental.gc.relocate.p1i64(i32 %sp1, i32 14, i32 14)
+  br label %normal_return
+
+right:
+  ; CHECK-LABEL: %right
+  ; CHECK: movq
+  ; CHECK: movq %rdx, (%rsp)
+  ; CHECK: callq some_call
+  %sp2 = invoke i32 (i64, i32, void (i64 addrspace(1)*)*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidp1i64f(i64 0, i32 0, void (i64 addrspace(1)*)* @some_call, i32 1, i32 0, i64 addrspace(1)* %val1, i32 0, i32 5, i32 0, i32 -1, i32 0, i32 0, i32 0, i64 addrspace(1)* %val2, i64 addrspace(1)* %val3)
+           to label %right.relocs unwind label %exceptional_return.right
+
+right.relocs:
+  ; CHECK: movq (%rsp), [[REGVAL2]]
+  ; CHECK: movq
+  %val2.relocated_right = call coldcc i64 addrspace(1)* @llvm.experimental.gc.relocate.p1i64(i32 %sp2, i32 13, i32 13)
+  %val3.relocated = call coldcc i64 addrspace(1)* @llvm.experimental.gc.relocate.p1i64(i32 %sp2, i32 14, i32 14)
+  br label %normal_return
+
+normal_return:
+  ; CHECK-LABEL: %normal_return
+  ; CHECK: cmoveq {{.*}}[[REGVAL2]]{{.*}}
+  ; CHECK retq
+  %a1 = phi i64 addrspace(1)* [%val1.relocated, %left.relocs], [%val3.relocated, %right.relocs]
+  %a2 = phi i64 addrspace(1)* [%val2.relocated_left, %left.relocs], [%val2.relocated_right, %right.relocs]
+  %ret = select i1 %cond, i64 addrspace(1)* %a1, i64 addrspace(1)* %a2
+  ret i64 addrspace(1)* %ret
+
+exceptional_return.left:
+  %landing_pad = landingpad { i8*, i32 }
+          cleanup
+  %relocate_token = extractvalue { i8*, i32 } %landing_pad, 1
+  %val.relocated2 = call coldcc i64 addrspace(1)* @llvm.experimental.gc.relocate.p1i64(i32 %relocate_token, i32 13, i32 13)
+  ret i64 addrspace(1)* %val.relocated2
+
+exceptional_return.right:
+  %landing_pad1 = landingpad { i8*, i32 }
+          cleanup
+  %relocate_token1 = extractvalue { i8*, i32 } %landing_pad1, 1
+  %val.relocated3 = call coldcc i64 addrspace(1)* @llvm.experimental.gc.relocate.p1i64(i32 %relocate_token1, i32 13, i32 13)
+  ret i64 addrspace(1)* %val.relocated3
+}
+
+define i64 addrspace(1)* @test_null_undef(i64 addrspace(1)* %val1)
+       gc "statepoint-example" personality i32 ()* @"personality_function" {
+; CHECK-LABEL: test_null_undef:
+entry:
+  ; CHECK: callq some_call
+  %sp1 = invoke i32 (i64, i32, void (i64 addrspace(1)*)*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidp1i64f(i64 0, i32 0, void (i64 addrspace(1)*)* @some_call, i32 1, i32 0, i64 addrspace(1)* %val1, i32 0, i32 5, i32 0, i32 -1, i32 0, i32 0, i32 0, i64 addrspace(1)* null, i64 addrspace(1)* undef)
+           to label %normal_return unwind label %exceptional_return
+
+normal_return:
+  ; CHECK-LABEL: %normal_return
+  ; CHECK: xorl %eax, %eax
+  ; CHECK-NEXT: popq
+  ; CHECK-NEXT: retq
+  %null.relocated = call coldcc i64 addrspace(1)* @llvm.experimental.gc.relocate.p1i64(i32 %sp1, i32 13, i32 13)
+  %undef.relocated = call coldcc i64 addrspace(1)* @llvm.experimental.gc.relocate.p1i64(i32 %sp1, i32 14, i32 14)
+  ret i64 addrspace(1)* %null.relocated
+
+exceptional_return:
+  %landing_pad = landingpad { i8*, i32 }
+          cleanup
+  %relocate_token = extractvalue { i8*, i32 } %landing_pad, 1
+  %null.relocated2 = call coldcc i64 addrspace(1)* @llvm.experimental.gc.relocate.p1i64(i32 %relocate_token, i32 13, i32 13)
+  %undef.relocated2 = call coldcc i64 addrspace(1)* @llvm.experimental.gc.relocate.p1i64(i32 %relocate_token, i32 14, i32 14)
+  ret i64 addrspace(1)* %null.relocated2
+}
+
+define i64 addrspace(1)* @test_alloca_and_const(i64 addrspace(1)* %val1)
+       gc "statepoint-example" personality i32 ()* @"personality_function" {
+; CHECK-LABEL: test_alloca_and_const:
+entry:
+  %a = alloca i32
+  %aa = addrspacecast i32* %a to i32 addrspace(1)*
+  %c = inttoptr i64 15 to i64 addrspace(1)*
+  ; CHECK: callq
+  %sp = invoke i32 (i64, i32, void (i64 addrspace(1)*)*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidp1i64f(i64 0, i32 0, void (i64 addrspace(1)*)* @some_call, i32 1, i32 0, i64 addrspace(1)* %val1, i32 0, i32 5, i32 0, i32 -1, i32 0, i32 0, i32 0, i32 addrspace(1)* %aa, i64 addrspace(1)* %c)
+           to label %normal_return unwind label %exceptional_return
+
+normal_return:
+  ; CHECK: leaq
+  ; CHECK-NEXT: popq
+  ; CHECK-NEXT: retq
+  %aa.rel = call coldcc i32 addrspace(1)* @llvm.experimental.gc.relocate.p1i32(i32 %sp, i32 13, i32 13)
+  %aa.converted = bitcast i32 addrspace(1)* %aa.rel to i64 addrspace(1)*
+  ret i64 addrspace(1)* %aa.converted
+
+exceptional_return:
+  ; CHECK: movl	$15
+  ; CHECK-NEXT: popq
+  ; CHECK-NEXT: retq
+  %landing_pad = landingpad { i8*, i32 }
+          cleanup
+  %relocate_token = extractvalue { i8*, i32 } %landing_pad, 1
+  %aa.rel2 = call coldcc i64 addrspace(1)* @llvm.experimental.gc.relocate.p1i64(i32 %relocate_token, i32 14, i32 14)
+  ret i64 addrspace(1)* %aa.rel2
+}
+
+declare i32 @llvm.experimental.gc.statepoint.p0f_isVoidp1i64f(i64, i32, void (i64 addrspace(1)*)*, i32, i32, ...)
+declare i32 @llvm.experimental.gc.statepoint.p0f_p1i64p1i64f(i64, i32, i64 addrspace(1)* (i64 addrspace(1)*)*, i32, i32, ...)
+
+declare i64 addrspace(1)* @llvm.experimental.gc.relocate.p1i64(i32, i32, i32)
+declare i32 addrspace(1)* @llvm.experimental.gc.relocate.p1i32(i32, i32, i32)
 declare i64 addrspace(1)* @llvm.experimental.gc.result.p1i64(i32)
diff --git a/test/CodeGen/X86/statepoint-stack-usage.ll b/test/CodeGen/X86/statepoint-stack-usage.ll
index a968c037b27..a4aa747af8c 100644
--- a/test/CodeGen/X86/statepoint-stack-usage.ll
+++ b/test/CodeGen/X86/statepoint-stack-usage.ll
@@ -11,20 +11,22 @@ target triple = "x86_64-pc-linux-gnu"
 define i32 @back_to_back_calls(i32 addrspace(1)* %a, i32 addrspace(1)* %b, i32 addrspace(1)* %c) #1 gc "statepoint-example" {
 ; CHECK-LABEL: back_to_back_calls
 ; The exact stores don't matter, but there need to be three stack slots created
-; CHECK: movq	%rdx, 16(%rsp)
-; CHECK: movq	%rdi, 8(%rsp)
+; CHECK: movq	%rdi, 16(%rsp)
+; CHECK: movq	%rdx, 8(%rsp)
 ; CHECK: movq	%rsi, (%rsp)
-  %safepoint_token = tail call i32 (void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(void ()* undef, i32 0, i32 0, i32 5, i32 0, i32 -1, i32 0, i32 0, i32 0, i32 addrspace(1)* %a, i32 addrspace(1)* %b, i32 addrspace(1)* %c)
-  %a1 = tail call coldcc i32 addrspace(1)* @llvm.experimental.gc.relocate.p1i32(i32 %safepoint_token, i32 9, i32 9)
-  %b1 = tail call coldcc i32 addrspace(1)* @llvm.experimental.gc.relocate.p1i32(i32 %safepoint_token, i32 9, i32 10)
-  %c1 = tail call coldcc i32 addrspace(1)* @llvm.experimental.gc.relocate.p1i32(i32 %safepoint_token, i32 9, i32 11)
+; There should be no more than three moves
+; CHECK-NOT: movq
+  %safepoint_token = tail call i32 (i64, i32, void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 0, i32 0, void ()* undef, i32 0, i32 0, i32 0, i32 5, i32 0, i32 -1, i32 0, i32 0, i32 0, i32 addrspace(1)* %a, i32 addrspace(1)* %b, i32 addrspace(1)* %c)
+  %a1 = tail call coldcc i32 addrspace(1)* @llvm.experimental.gc.relocate.p1i32(i32 %safepoint_token, i32 12, i32 12)
+  %b1 = tail call coldcc i32 addrspace(1)* @llvm.experimental.gc.relocate.p1i32(i32 %safepoint_token, i32 12, i32 13)
+  %c1 = tail call coldcc i32 addrspace(1)* @llvm.experimental.gc.relocate.p1i32(i32 %safepoint_token, i32 12, i32 14)
 ; CHECK: callq
 ; This is the key check.  There should NOT be any memory moves here
 ; CHECK-NOT: movq
-  %safepoint_token2 = tail call i32 (void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(void ()* undef, i32 0, i32 0, i32 5, i32 0, i32 -1, i32 0, i32 0, i32 0, i32 addrspace(1)* %c1, i32 addrspace(1)* %b1, i32 addrspace(1)* %a1)
-  %a2 = tail call coldcc i32 addrspace(1)* @llvm.experimental.gc.relocate.p1i32(i32 %safepoint_token2, i32 9, i32 11)
-  %b2 = tail call coldcc i32 addrspace(1)* @llvm.experimental.gc.relocate.p1i32(i32 %safepoint_token2, i32 9, i32 10)
-  %c2 = tail call coldcc i32 addrspace(1)* @llvm.experimental.gc.relocate.p1i32(i32 %safepoint_token2, i32 9, i32 9)
+  %safepoint_token2 = tail call i32 (i64, i32, void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 0, i32 0, void ()* undef, i32 0, i32 0, i32 0, i32 5, i32 0, i32 -1, i32 0, i32 0, i32 0, i32 addrspace(1)* %c1, i32 addrspace(1)* %b1, i32 addrspace(1)* %a1)
+  %a2 = tail call coldcc i32 addrspace(1)* @llvm.experimental.gc.relocate.p1i32(i32 %safepoint_token2, i32 12, i32 14)
+  %b2 = tail call coldcc i32 addrspace(1)* @llvm.experimental.gc.relocate.p1i32(i32 %safepoint_token2, i32 12, i32 13)
+  %c2 = tail call coldcc i32 addrspace(1)* @llvm.experimental.gc.relocate.p1i32(i32 %safepoint_token2, i32 12, i32 12)
 ; CHECK: callq
   ret i32 1
 }
@@ -34,27 +36,71 @@ define i32 @back_to_back_calls(i32 addrspace(1)* %a, i32 addrspace(1)* %b, i32 a
 define i32 @reserve_first(i32 addrspace(1)* %a, i32 addrspace(1)* %b, i32 addrspace(1)* %c) #1 gc "statepoint-example" {
 ; CHECK-LABEL: reserve_first
 ; The exact stores don't matter, but there need to be three stack slots created
-; CHECK: movq	%rdx, 16(%rsp)
-; CHECK: movq	%rdi, 8(%rsp)
+; CHECK: movq	%rdi, 16(%rsp)
+; CHECK: movq	%rdx, 8(%rsp)
 ; CHECK: movq	%rsi, (%rsp)
-  %safepoint_token = tail call i32 (void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(void ()* undef, i32 0, i32 0, i32 5, i32 0, i32 -1, i32 0, i32 0, i32 0, i32 addrspace(1)* %a, i32 addrspace(1)* %b, i32 addrspace(1)* %c)
-  %a1 = tail call coldcc i32 addrspace(1)* @llvm.experimental.gc.relocate.p1i32(i32 %safepoint_token, i32 9, i32 9)
-  %b1 = tail call coldcc i32 addrspace(1)* @llvm.experimental.gc.relocate.p1i32(i32 %safepoint_token, i32 9, i32 10)
-  %c1 = tail call coldcc i32 addrspace(1)* @llvm.experimental.gc.relocate.p1i32(i32 %safepoint_token, i32 9, i32 11)
+  %safepoint_token = tail call i32 (i64, i32, void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 0, i32 0, void ()* undef, i32 0, i32 0, i32 0, i32 5, i32 0, i32 -1, i32 0, i32 0, i32 0, i32 addrspace(1)* %a, i32 addrspace(1)* %b, i32 addrspace(1)* %c)
+  %a1 = tail call coldcc i32 addrspace(1)* @llvm.experimental.gc.relocate.p1i32(i32 %safepoint_token, i32 12, i32 12)
+  %b1 = tail call coldcc i32 addrspace(1)* @llvm.experimental.gc.relocate.p1i32(i32 %safepoint_token, i32 12, i32 13)
+  %c1 = tail call coldcc i32 addrspace(1)* @llvm.experimental.gc.relocate.p1i32(i32 %safepoint_token, i32 12, i32 14)
 ; CHECK: callq
 ; This is the key check.  There should NOT be any memory moves here
 ; CHECK-NOT: movq
-  %safepoint_token2 = tail call i32 (void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(void ()* undef, i32 0, i32 0, i32 5, i32 addrspace(1)* %a1, i32 0, i32 addrspace(1)* %c1, i32 0, i32 0, i32 addrspace(1)* %c1, i32 addrspace(1)* %b1, i32 addrspace(1)* %a1)
-  %a2 = tail call coldcc i32 addrspace(1)* @llvm.experimental.gc.relocate.p1i32(i32 %safepoint_token2, i32 9, i32 11)
-  %b2 = tail call coldcc i32 addrspace(1)* @llvm.experimental.gc.relocate.p1i32(i32 %safepoint_token2, i32 9, i32 10)
-  %c2 = tail call coldcc i32 addrspace(1)* @llvm.experimental.gc.relocate.p1i32(i32 %safepoint_token2, i32 9, i32 9)
+  %safepoint_token2 = tail call i32 (i64, i32, void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 0, i32 0, void ()* undef, i32 0, i32 0, i32 0, i32 5, i32 addrspace(1)* %a1, i32 0, i32 addrspace(1)* %c1, i32 0, i32 0, i32 addrspace(1)* %c1, i32 addrspace(1)* %b1, i32 addrspace(1)* %a1)
+  %a2 = tail call coldcc i32 addrspace(1)* @llvm.experimental.gc.relocate.p1i32(i32 %safepoint_token2, i32 12, i32 14)
+  %b2 = tail call coldcc i32 addrspace(1)* @llvm.experimental.gc.relocate.p1i32(i32 %safepoint_token2, i32 12, i32 13)
+  %c2 = tail call coldcc i32 addrspace(1)* @llvm.experimental.gc.relocate.p1i32(i32 %safepoint_token2, i32 12, i32 12)
 ; CHECK: callq
   ret i32 1
 }
 
+; Test that stack slots are reused for invokes
+define i32 @back_to_back_invokes(i32 addrspace(1)* %a, i32 addrspace(1)* %b, i32 addrspace(1)* %c) #1 gc "statepoint-example" personality i32 ()* @"personality_function" {
+; CHECK-LABEL: back_to_back_invokes
+entry:
+  ; The exact stores don't matter, but there need to be three stack slots created
+  ; CHECK: movq	%rdi, 16(%rsp)
+  ; CHECK: movq	%rdx, 8(%rsp)
+  ; CHECK: movq	%rsi, (%rsp)
+  ; CHECK: callq
+  %safepoint_token = invoke i32 (i64, i32, void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 0, i32 0, void ()* undef, i32 0, i32 0, i32 0, i32 5, i32 0, i32 -1, i32 0, i32 0, i32 0, i32 addrspace(1)* %a, i32 addrspace(1)* %b, i32 addrspace(1)* %c)
+                   to label %normal_return unwind label %exceptional_return
+
+normal_return:
+  %a1 = tail call coldcc i32 addrspace(1)* @llvm.experimental.gc.relocate.p1i32(i32 %safepoint_token, i32 12, i32 12)
+  %b1 = tail call coldcc i32 addrspace(1)* @llvm.experimental.gc.relocate.p1i32(i32 %safepoint_token, i32 12, i32 13)
+  %c1 = tail call coldcc i32 addrspace(1)* @llvm.experimental.gc.relocate.p1i32(i32 %safepoint_token, i32 12, i32 14)
+  ; Should work even through bitcasts
+  %c1.casted = bitcast i32 addrspace(1)* %c1 to i8 addrspace(1)*
+  ; This is the key check.  There should NOT be any memory moves here
+  ; CHECK-NOT: movq
+  ; CHECK: callq
+  %safepoint_token2 = invoke i32 (i64, i32, void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 0, i32 0, void ()* undef, i32 0, i32 0, i32 0, i32 5, i32 0, i32 -1, i32 0, i32 0, i32 0, i8 addrspace(1)* %c1.casted, i32 addrspace(1)* %b1, i32 addrspace(1)* %a1)
+                    to label %normal_return2 unwind label %exceptional_return2
+
+normal_return2:
+  %a2 = tail call coldcc i32 addrspace(1)* @llvm.experimental.gc.relocate.p1i32(i32 %safepoint_token2, i32 12, i32 14)
+  %b2 = tail call coldcc i32 addrspace(1)* @llvm.experimental.gc.relocate.p1i32(i32 %safepoint_token2, i32 12, i32 13)
+  %c2 = tail call coldcc i8 addrspace(1)* @llvm.experimental.gc.relocate.p1i8(i32 %safepoint_token2, i32 12, i32 12)
+  ret i32 1
+
+exceptional_return:
+  %landing_pad = landingpad { i8*, i32 }
+          cleanup
+  ret i32 0
+
+exceptional_return2:
+  %landing_pad2 = landingpad { i8*, i32 }
+          cleanup
+  ret i32 0
+}
+
 ; Function Attrs: nounwind
 declare i32 addrspace(1)* @llvm.experimental.gc.relocate.p1i32(i32, i32, i32) #3
+declare i8 addrspace(1)* @llvm.experimental.gc.relocate.p1i8(i32, i32, i32) #3
+
+declare i32 @llvm.experimental.gc.statepoint.p0f_isVoidf(i64, i32, void ()*, i32, i32, ...)
 
-declare i32 @llvm.experimental.gc.statepoint.p0f_isVoidf(void ()*, i32, i32, ...)
+declare i32 @"personality_function"()
 
 attributes #1 = { uwtable }
diff --git a/test/CodeGen/X86/statepoint-stackmap-format.ll b/test/CodeGen/X86/statepoint-stackmap-format.ll
index 9593c4082ea..e18476cee53 100644
--- a/test/CodeGen/X86/statepoint-stackmap-format.ll
+++ b/test/CodeGen/X86/statepoint-stackmap-format.ll
@@ -1,39 +1,81 @@
-; RUN: llc < %s | FileCheck %s
+; RUN: llc < %s -mtriple="x86_64-pc-linux-gnu" | FileCheck %s
+; RUN: llc < %s -mtriple="x86_64-pc-win64-coff" | FileCheck %s
+
 ; This test is a sanity check to ensure statepoints are generating StackMap
 ; sections correctly.  This is not intended to be a rigorous test of the 
 ; StackMap format (see the stackmap tests for that).
 
 target datalayout = "e-i64:64-f80:128-n8:16:32:64-S128"
-target triple = "x86_64-pc-linux-gnu"
 
 declare zeroext i1 @return_i1()
 
-define i1 @test(i32 addrspace(1)* %ptr) gc "statepoint-example" {
+define i1 @test(i32 addrspace(1)* %ptr_base, i32 %arg)
+  gc "statepoint-example" {
 ; CHECK-LABEL: test
-; Do we see one spill for the local value and the store to the
+; Do we see two spills for the local values and the store to the
+; alloca?
+; CHECK: subq	$40, %rsp
+; CHECK: movq	$0,   24(%rsp)
+; CHECK: movq	%rdi, 16(%rsp)
+; CHECK: movq	%rax, 8(%rsp)
+; CHECK: callq return_i1
+; CHECK: addq	$40, %rsp
+; CHECK: retq
+entry:
+  %metadata1 = alloca i32 addrspace(1)*, i32 2, align 8
+  store i32 addrspace(1)* null, i32 addrspace(1)** %metadata1
+  %ptr_derived = getelementptr i32, i32 addrspace(1)* %ptr_base, i32 %arg
+  %safepoint_token = tail call i32 (i64, i32, i1 ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_i1f(i64 0, i32 0, i1 ()* @return_i1, i32 0, i32 0, i32 0, i32 2, i32 addrspace(1)* %ptr_base, i32 addrspace(1)* null, i32 addrspace(1)* %ptr_base, i32 addrspace(1)* %ptr_derived, i32 addrspace(1)* null)
+  %call1 = call zeroext i1 @llvm.experimental.gc.result.i1(i32 %safepoint_token)
+  %a = call i32 addrspace(1)* @llvm.experimental.gc.relocate.p1i32(i32 %safepoint_token, i32 9, i32 9)
+  %b = call i32 addrspace(1)* @llvm.experimental.gc.relocate.p1i32(i32 %safepoint_token, i32 9, i32 10)
+  %c = call i32 addrspace(1)* @llvm.experimental.gc.relocate.p1i32(i32 %safepoint_token, i32 11, i32 11)
+; 
+  ret i1 %call1
+}
+
+; This is similar to the previous test except that we have derived pointer as
+; argument to the function. Despite that this can not happen after the
+; RewriteSafepointForGC pass, lowering should be able to handle it anyway.
+define i1 @test_derived_arg(i32 addrspace(1)* %ptr_base,
+                            i32 addrspace(1)* %ptr_derived)
+  gc "statepoint-example" {
+; CHECK-LABEL: test_derived_arg
+; Do we see two spills for the local values and the store to the
 ; alloca?
-; CHECK: subq	$24, %rsp
-; CHECK: movq	$0, 8(%rsp)
-; CHECK: movq	%rdi, (%rsp)
+; CHECK: subq	$40, %rsp
+; CHECK: movq	$0,   24(%rsp)
+; CHECK: movq	%rdi, 16(%rsp)
+; CHECK: movq	%rsi, 8(%rsp)
 ; CHECK: callq return_i1
-; CHECK: addq	$24, %rsp
+; CHECK: addq	$40, %rsp
 ; CHECK: retq
 entry:
   %metadata1 = alloca i32 addrspace(1)*, i32 2, align 8
   store i32 addrspace(1)* null, i32 addrspace(1)** %metadata1
-  %safepoint_token = tail call i32 (i1 ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_i1f(i1 ()* @return_i1, i32 0, i32 0, i32 2, i32 addrspace(1)* %ptr, i32 addrspace(1)* null, i32 addrspace(1)* %ptr, i32 addrspace(1)* null)
+  %safepoint_token = tail call i32 (i64, i32, i1 ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_i1f(i64 0, i32 0, i1 ()* @return_i1, i32 0, i32 0, i32 0, i32 2, i32 addrspace(1)* %ptr_base, i32 addrspace(1)* null, i32 addrspace(1)* %ptr_base, i32 addrspace(1)* %ptr_derived, i32 addrspace(1)* null)
   %call1 = call zeroext i1 @llvm.experimental.gc.result.i1(i32 %safepoint_token)
-  %a = call i32 addrspace(1)* @llvm.experimental.gc.relocate.p1i32(i32 %safepoint_token, i32 6, i32 6)
-  %b = call i32 addrspace(1)* @llvm.experimental.gc.relocate.p1i32(i32 %safepoint_token, i32 7, i32 7)
+  %a = call i32 addrspace(1)* @llvm.experimental.gc.relocate.p1i32(i32 %safepoint_token, i32 9, i32 9)
+  %b = call i32 addrspace(1)* @llvm.experimental.gc.relocate.p1i32(i32 %safepoint_token, i32 9, i32 10)
+  %c = call i32 addrspace(1)* @llvm.experimental.gc.relocate.p1i32(i32 %safepoint_token, i32 11, i32 11)
 ; 
   ret i1 %call1
 }
 
-declare i32 @llvm.experimental.gc.statepoint.p0f_i1f(i1 ()*, i32, i32, ...)
+; Simple test case to check that we emit the ID field correctly
+define i1 @test_id() gc "statepoint-example" {
+; CHECK-LABEL: test_id
+entry:
+  %safepoint_token = tail call i32 (i64, i32, i1 ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_i1f(i64 237, i32 0, i1 ()* @return_i1, i32 0, i32 0, i32 0, i32 0)
+  %call1 = call zeroext i1 @llvm.experimental.gc.result.i1(i32 %safepoint_token)
+  ret i1 %call1
+}
+
+
+declare i32 @llvm.experimental.gc.statepoint.p0f_i1f(i64, i32, i1 ()*, i32, i32, ...)
 declare i1 @llvm.experimental.gc.result.i1(i32)
 declare i32 addrspace(1)* @llvm.experimental.gc.relocate.p1i32(i32, i32, i32) #3
 
-
 ; CHECK-LABEL: .section .llvm_stackmaps
 ; CHECK-NEXT:  __LLVM_StackMaps:
 ; Header
@@ -41,25 +83,36 @@ declare i32 addrspace(1)* @llvm.experimental.gc.relocate.p1i32(i32, i32, i32) #3
 ; CHECK-NEXT:   .byte 0
 ; CHECK-NEXT:   .short 0
 ; Num Functions
-; CHECK-NEXT:   .long 1
+; CHECK-NEXT:   .long 3
 ; Num LargeConstants
 ; CHECK-NEXT:   .long 0
 ; Num Callsites
-; CHECK-NEXT:   .long 1
+; CHECK-NEXT:   .long 3
 
 ; Functions and stack size
 ; CHECK-NEXT:   .quad test
-; CHECK-NEXT:   .quad 24
+; CHECK-NEXT:   .quad 40
+; CHECK-NEXT:   .quad test_derived_arg
+; CHECK-NEXT:   .quad 40
+
+;
+; test
+;
 
 ; Large Constants
 ; Statepoint ID only
-; CHECK: .quad	2882400000
+; CHECK: .quad	0
 
 ; Callsites
 ; Constant arguments
 ; CHECK: .long	.Ltmp1-test
 ; CHECK: .short	0
-; CHECK: .short	8
+; CHECK: .short	11
+; SmallConstant (0)
+; CHECK: .byte	4
+; CHECK: .byte	8
+; CHECK: .short	0
+; CHECK: .long	0
 ; SmallConstant (0)
 ; CHECK: .byte	4
 ; CHECK: .byte	8
@@ -74,7 +127,7 @@ declare i32 addrspace(1)* @llvm.experimental.gc.relocate.p1i32(i32, i32, i32) #3
 ; CHECK: .byte	2
 ; CHECK: .byte	8
 ; CHECK: .short	7
-; CHECK: .long	0
+; CHECK: .long	16
 ; SmallConstant  (0)
 ; CHECK: .byte	4
 ; CHECK: .byte	8
@@ -90,20 +143,139 @@ declare i32 addrspace(1)* @llvm.experimental.gc.relocate.p1i32(i32, i32, i32) #3
 ; CHECK: .byte	8
 ; CHECK: .short	0
 ; CHECK: .long	0
-; Direct Spill Slot [RSP+0]
+; Direct Spill Slot [RSP+16]
 ; CHECK: .byte	2
 ; CHECK: .byte	8
 ; CHECK: .short	7
+; CHECK: .long	16
+; Direct Spill Slot [RSP+8]
+; CHECK: .byte	2
+; CHECK: .byte	8
+; CHECK: .short	7
+; CHECK: .long	8
+; Direct Spill Slot [RSP+16]
+; CHECK: .byte	2
+; CHECK: .byte	8
+; CHECK: .short	7
+; CHECK: .long	16
+; Direct Spill Slot [RSP+16]
+; CHECK: .byte	2
+; CHECK: .byte	8
+; CHECK: .short	7
+; CHECK: .long	16
+
+; No Padding or LiveOuts
+; CHECK: .short	0
+; CHECK: .short	0
+; CHECK: .align	8
+
+;
+; test_derived_arg
+;
+
+; Large Constants
+; Statepoint ID only
+; CHECK: .quad	0
+
+; Callsites
+; Constant arguments
+; CHECK: .long	.Ltmp3-test_derived_arg
+; CHECK: .short	0
+; CHECK: .short	11
+; SmallConstant (0)
+; CHECK: .byte	4
+; CHECK: .byte	8
+; CHECK: .short	0
 ; CHECK: .long	0
+; SmallConstant (2)
+; CHECK: .byte	4
+; CHECK: .byte	8
+; CHECK: .short	0
+; CHECK: .long	2
 ; Direct Spill Slot [RSP+0]
 ; CHECK: .byte	2
 ; CHECK: .byte	8
 ; CHECK: .short	7
+; CHECK: .long	16
+; SmallConstant  (0)
+; CHECK: .byte	4
+; CHECK: .byte	8
+; CHECK: .short	0
 ; CHECK: .long	0
+; SmallConstant  (0)
+; CHECK: .byte	4
+; CHECK: .byte	8
+; CHECK: .short	0
+; CHECK: .long	0
+; SmallConstant  (0)
+; CHECK: .byte	4
+; CHECK: .byte	8
+; CHECK: .short	0
+; CHECK: .long	0
+; Direct Spill Slot [RSP+16]
+; CHECK: .byte	2
+; CHECK: .byte	8
+; CHECK: .short	7
+; CHECK: .long	16
+; Direct Spill Slot [RSP+8]
+; CHECK: .byte	2
+; CHECK: .byte	8
+; CHECK: .short	7
+; CHECK: .long	8
+; Direct Spill Slot [RSP+16]
+; CHECK: .byte	2
+; CHECK: .byte	8
+; CHECK: .short	7
+; CHECK: .long	16
+; Direct Spill Slot [RSP+16]
+; CHECK: .byte	2
+; CHECK: .byte	8
+; CHECK: .short	7
+; CHECK: .long	16
 
 ; No Padding or LiveOuts
 ; CHECK: .short	0
 ; CHECK: .short	0
 ; CHECK: .align	8
 
+; Records for the test_id function:
+; No large constants
+
+; The Statepoint ID:
+; CHECK: .quad	237
+
+; Instruction Offset
+; CHECK: .long	.Ltmp5-test_id
+
+; Reserved:
+; CHECK: .short	0
+
+; NumLocations:
+; CHECK: .short	3
+
+; StkMapRecord[0]:
+; SmallConstant(0):
+; CHECK: .byte	4
+; CHECK: .byte	8
+; CHECK: .short	0
+; CHECK: .long	0
+
+; StkMapRecord[1]:
+; SmallConstant(0):
+; CHECK: .byte	4
+; CHECK: .byte	8
+; CHECK: .short	0
+; CHECK: .long	0
+
+; StkMapRecord[2]:
+; SmallConstant(0):
+; CHECK: .byte	4
+; CHECK: .byte	8
+; CHECK: .short	0
+; CHECK: .long	0
+
+; No padding or LiveOuts
+; CHECK: .short	0
+; CHECK: .short	0
+; CHECK: .align	8
 
diff --git a/test/CodeGen/X86/switch-bt.ll b/test/CodeGen/X86/switch-bt.ll
index 113782169b0..2cf3aafe547 100644
--- a/test/CodeGen/X86/switch-bt.ll
+++ b/test/CodeGen/X86/switch-bt.ll
@@ -140,19 +140,17 @@ sw.epilog:
 
 ; The balanced binary switch here would start with a comparison against 39, but
 ; it is currently starting with 29 because of the density-sum heuristic.
-; CHECK: cmpl $29
+; CHECK: cmpl $39
 ; CHECK: jg
 ; CHECK: cmpl $10
-; CHECK: jne
-; CHECK: cmpl $49
-; CHECK: jg
-; CHECK: cmpl $30
-; CHECK: jne
+; CHECK: je
 ; CHECK: cmpl $20
 ; CHECK: jne
+; CHECK: cmpl $40
+; CHECK: je
 ; CHECK: cmpl $50
 ; CHECK: jne
-; CHECK: cmpl $40
+; CHECK: cmpl $30
 ; CHECK: jne
 ; CHECK: cmpl $60
 ; CHECK: jne
diff --git a/test/CodeGen/X86/switch-or.ll b/test/CodeGen/X86/switch-or.ll
index 6e6b013d9fa..4642accfff8 100644
--- a/test/CodeGen/X86/switch-or.ll
+++ b/test/CodeGen/X86/switch-or.ll
@@ -1,10 +1,11 @@
 ; RUN: llc -march=x86 -asm-verbose=false < %s | FileCheck %s
 
 ; Check that merging switch cases that differ in one bit works.
+; CHECK-LABEL: test1
 ; CHECK: orl $2
 ; CHECK-NEXT: cmpl $6
 
-define void @foo(i32 %variable) nounwind {
+define void @test1(i32 %variable) nounwind {
 entry:
   switch i32 %variable, label %if.end [
     i32 4, label %if.then
@@ -19,4 +20,22 @@ if.end:
   ret void
 }
 
+; CHECK-LABEL: test2
+; CHECK: orl $-2147483648
+; CHECK-NEXT: cmpl $-2147483648
+define void @test2(i32 %variable) nounwind {
+entry:
+  switch i32 %variable, label %if.end [
+    i32 0, label %if.then
+    i32 -2147483648, label %if.then
+  ]
+
+if.then:
+  %call = tail call i32 (...) @bar() nounwind
+  ret void
+
+if.end:
+  ret void
+}
+
 declare i32 @bar(...) nounwind
diff --git a/test/CodeGen/X86/switch.ll b/test/CodeGen/X86/switch.ll
new file mode 100644
index 00000000000..748fd6f238b
--- /dev/null
+++ b/test/CodeGen/X86/switch.ll
@@ -0,0 +1,634 @@
+; RUN: llc -mtriple=x86_64-linux-gnu %s -o - | FileCheck %s
+; RUN: llc -mtriple=x86_64-linux-gnu %s -o - -O0 | FileCheck --check-prefix=NOOPT %s
+
+declare void @g(i32)
+
+define void @basic(i32 %x) {
+entry:
+  switch i32 %x, label %return [
+    i32 3, label %bb0
+    i32 1, label %bb1
+    i32 4, label %bb1
+    i32 5, label %bb2
+  ]
+bb0: tail call void @g(i32 0) br label %return
+bb1: tail call void @g(i32 1) br label %return
+bb2: tail call void @g(i32 1) br label %return
+return: ret void
+
+; Lowered as a jump table, both with and without optimization.
+; CHECK-LABEL: basic
+; CHECK: decl
+; CHECK: cmpl $4
+; CHECK: ja
+; CHECK: jmpq *.LJTI
+; NOOPT-LABEL: basic
+; NOOPT: decl
+; NOOPT: subl $4
+; NOOPT: ja
+; NOOPT: movq .LJTI
+; NOOPT: jmpq
+}
+
+
+define void @simple_ranges(i32 %x) {
+entry:
+  switch i32 %x, label %return [
+    i32 0, label %bb0
+    i32 1, label %bb0
+    i32 2, label %bb0
+    i32 3, label %bb0
+    i32 100, label %bb1
+    i32 101, label %bb1
+    i32 102, label %bb1
+    i32 103, label %bb1
+  ]
+bb0: tail call void @g(i32 0) br label %return
+bb1: tail call void @g(i32 1) br label %return
+return: ret void
+
+; Should be lowered to two range checks.
+; CHECK-LABEL: simple_ranges
+; CHECK: leal -100
+; CHECK: cmpl $4
+; CHECK: jae
+; CHECK: cmpl $3
+; CHECK: ja
+
+; We do this even at -O0, because it's cheap and makes codegen faster.
+; NOOPT-LABEL: simple_ranges
+; NOOPT: subl $4
+; NOOPT: jb
+; NOOPT: addl $-100
+; NOOPT: subl $4
+; NOOPT: jb
+}
+
+
+define void @jt_is_better(i32 %x) {
+entry:
+  switch i32 %x, label %return [
+    i32 0, label %bb0
+    i32 2, label %bb0
+    i32 4, label %bb0
+    i32 1, label %bb1
+    i32 3, label %bb1
+    i32 5, label %bb1
+
+    i32 6, label %bb2
+    i32 7, label %bb3
+    i32 8, label %bb4
+  ]
+bb0: tail call void @g(i32 0) br label %return
+bb1: tail call void @g(i32 1) br label %return
+bb2: tail call void @g(i32 2) br label %return
+bb3: tail call void @g(i32 3) br label %return
+bb4: tail call void @g(i32 4) br label %return
+return: ret void
+
+; Cases 0-5 could be lowered with two bit tests,
+; but with 6-8, the whole switch is suitable for a jump table.
+; CHECK-LABEL: jt_is_better
+; CHECK: cmpl $8
+; CHECK: jbe
+; CHECK: jmpq *.LJTI
+}
+
+
+define void @bt_is_better(i32 %x) {
+entry:
+  switch i32 %x, label %return [
+    i32 0, label %bb0
+    i32 3, label %bb0
+    i32 6, label %bb0
+    i32 1, label %bb1
+    i32 4, label %bb1
+    i32 7, label %bb1
+    i32 2, label %bb2
+    i32 5, label %bb2
+    i32 8, label %bb2
+
+  ]
+bb0: tail call void @g(i32 0) br label %return
+bb1: tail call void @g(i32 1) br label %return
+bb2: tail call void @g(i32 2) br label %return
+return: ret void
+
+; This could be lowered as a jump table, but bit tests is more efficient.
+; CHECK-LABEL: bt_is_better
+; 73 = 2^0 + 2^3 + 2^6
+; CHECK: movl $73
+; CHECK: btl
+; 146 = 2^1 + 2^4 + 2^7
+; CHECK: movl $146
+; CHECK: btl
+; 292 = 2^2 + 2^5 + 2^8
+; CHECK: movl $292
+; CHECK: btl
+}
+
+
+define void @optimal_pivot1(i32 %x) {
+entry:
+  switch i32 %x, label %return [
+    i32 100, label %bb0
+    i32 200, label %bb1
+    i32 300, label %bb0
+    i32 400, label %bb1
+    i32 500, label %bb0
+    i32 600, label %bb1
+
+  ]
+bb0: tail call void @g(i32 0) br label %return
+bb1: tail call void @g(i32 1) br label %return
+return: ret void
+
+; Should pivot around 400 for two subtrees of equal size.
+; CHECK-LABEL: optimal_pivot1
+; CHECK-NOT: cmpl
+; CHECK: cmpl $399
+}
+
+
+define void @optimal_pivot2(i32 %x) {
+entry:
+  switch i32 %x, label %return [
+    i32 100, label %bb0   i32 101, label %bb1   i32 102, label %bb2   i32 103, label %bb3
+    i32 200, label %bb0   i32 201, label %bb1   i32 202, label %bb2   i32 203, label %bb3
+    i32 300, label %bb0   i32 301, label %bb1   i32 302, label %bb2   i32 303, label %bb3
+    i32 400, label %bb0   i32 401, label %bb1   i32 402, label %bb2   i32 403, label %bb3
+
+  ]
+bb0: tail call void @g(i32 0) br label %return
+bb1: tail call void @g(i32 1) br label %return
+bb2: tail call void @g(i32 2) br label %return
+bb3: tail call void @g(i32 3) br label %return
+return: ret void
+
+; Should pivot around 300 for two subtrees with two jump tables each.
+; CHECK-LABEL: optimal_pivot2
+; CHECK-NOT: cmpl
+; CHECK: cmpl $299
+; CHECK: jmpq *.LJTI
+; CHECK: jmpq *.LJTI
+; CHECK: jmpq *.LJTI
+; CHECK: jmpq *.LJTI
+}
+
+
+define void @optimal_jump_table1(i32 %x) {
+entry:
+  switch i32 %x, label %return [
+    i32 0,  label %bb0
+    i32 5,  label %bb1
+    i32 6,  label %bb2
+    i32 12, label %bb3
+    i32 13, label %bb4
+    i32 15, label %bb5
+  ]
+bb0: tail call void @g(i32 0) br label %return
+bb1: tail call void @g(i32 1) br label %return
+bb2: tail call void @g(i32 2) br label %return
+bb3: tail call void @g(i32 3) br label %return
+bb4: tail call void @g(i32 4) br label %return
+bb5: tail call void @g(i32 5) br label %return
+return: ret void
+
+; Splitting in the largest gap (between 6 and 12) would yield suboptimal result.
+; Expecting a jump table from 5 to 15.
+; CHECK-LABEL: optimal_jump_table1
+; CHECK: leal -5
+; CHECK: cmpl $10
+; CHECK: jmpq *.LJTI
+
+; At -O0, we don't build jump tables for only parts of a switch.
+; NOOPT-LABEL: optimal_jump_table1
+; NOOPT: testl %edi, %edi
+; NOOPT: je
+; NOOPT: subl $5, %eax
+; NOOPT: je
+; NOOPT: subl $6, %eax
+; NOOPT: je
+; NOOPT: subl $12, %eax
+; NOOPT: je
+; NOOPT: subl $13, %eax
+; NOOPT: je
+; NOOPT: subl $15, %eax
+; NOOPT: je
+}
+
+
+define void @optimal_jump_table2(i32 %x) {
+entry:
+  switch i32 %x, label %return [
+    i32 0,  label %bb0
+    i32 1,  label %bb1
+    i32 2,  label %bb2
+    i32 9,  label %bb3
+    i32 14, label %bb4
+    i32 15, label %bb5
+  ]
+bb0: tail call void @g(i32 0) br label %return
+bb1: tail call void @g(i32 1) br label %return
+bb2: tail call void @g(i32 2) br label %return
+bb3: tail call void @g(i32 3) br label %return
+bb4: tail call void @g(i32 4) br label %return
+bb5: tail call void @g(i32 5) br label %return
+return: ret void
+
+; Partitioning the cases to the minimum number of dense sets is not good enough.
+; This can be partitioned as {0,1,2,9},{14,15} or {0,1,2},{9,14,15}. The former
+; should be preferred. Expecting a table from 0-9.
+; CHECK-LABEL: optimal_jump_table2
+; CHECK: cmpl $9
+; CHECK: jmpq *.LJTI
+}
+
+
+define void @optimal_jump_table3(i32 %x) {
+entry:
+  switch i32 %x, label %return [
+    i32 1,  label %bb0
+    i32 2,  label %bb1
+    i32 3,  label %bb2
+    i32 10, label %bb3
+    i32 13, label %bb0
+    i32 14, label %bb1
+    i32 15, label %bb2
+    i32 20, label %bb3
+    i32 25, label %bb4
+  ]
+bb0: tail call void @g(i32 0) br label %return
+bb1: tail call void @g(i32 1) br label %return
+bb2: tail call void @g(i32 2) br label %return
+bb3: tail call void @g(i32 3) br label %return
+bb4: tail call void @g(i32 4) br label %return
+return: ret void
+
+; Splitting to maximize left-right density sum and gap size would split this
+; between 3 and 10, and then between 20 and 25. It's better to build a table
+; from 1-20.
+; CHECK-LABEL: optimal_jump_table3
+; CHECK: leal -1
+; CHECK: cmpl $19
+; CHECK: jmpq *.LJTI
+}
+
+%struct.S = type { %struct.S*, i32 }
+define void @phi_node_trouble(%struct.S* %s) {
+entry:
+  br label %header
+header:
+  %ptr = phi %struct.S* [ %s, %entry ], [ %next, %loop ]
+  %bool = icmp eq %struct.S* %ptr, null
+  br i1 %bool, label %exit, label %loop
+loop:
+  %nextptr = getelementptr inbounds %struct.S, %struct.S* %ptr, i64 0, i32 0
+  %next = load %struct.S*, %struct.S** %nextptr
+  %xptr = getelementptr inbounds %struct.S, %struct.S* %next, i64 0, i32 1
+  %x = load i32, i32* %xptr
+  switch i32 %x, label %exit [
+    i32 4, label %header
+    i32 36, label %exit2
+    i32 69, label %exit2
+    i32 25, label %exit2
+  ]
+exit:
+  ret void
+exit2:
+  ret void
+
+; This will be lowered to a comparison with 4 and then bit tests. Make sure
+; that the phi node in %header gets a value from the comparison block.
+; CHECK-LABEL: phi_node_trouble
+; CHECK: movq (%[[REG1:[a-z]+]]), %[[REG1]]
+; CHECK: movl 8(%[[REG1]]), %[[REG2:[a-z]+]]
+; CHECK: cmpl $4, %[[REG2]]
+}
+
+
+define void @default_only(i32 %x) {
+entry:
+  br label %sw
+return:
+  ret void
+sw:
+  switch i32 %x, label %return [
+  ]
+
+; Branch directly to the default.
+; (In optimized builds the switch is removed earlier.)
+; NOOPT-LABEL: default_only
+; NOOPT: .[[L:[A-Z0-9_]+]]:
+; NOOPT-NEXT: retq
+; NOOPT: jmp .[[L]]
+}
+
+
+define void @int_max_table_cluster(i8 %x) {
+entry:
+  switch i8 %x, label %return [
+    i8 0, label %bb0 i8 1, label %bb0 i8 2, label %bb0 i8 3, label %bb0
+    i8 4, label %bb0 i8 5, label %bb0 i8 6, label %bb0 i8 7, label %bb0
+    i8 8, label %bb0 i8 9, label %bb0 i8 10, label %bb0 i8 11, label %bb0
+    i8 12, label %bb0 i8 13, label %bb0 i8 14, label %bb0 i8 15, label %bb0
+    i8 16, label %bb0 i8 17, label %bb0 i8 18, label %bb0 i8 19, label %bb0
+    i8 20, label %bb0 i8 21, label %bb0 i8 22, label %bb0 i8 23, label %bb0
+    i8 24, label %bb0 i8 25, label %bb0 i8 26, label %bb0 i8 27, label %bb0
+    i8 28, label %bb0 i8 29, label %bb0 i8 30, label %bb0 i8 31, label %bb0
+    i8 32, label %bb0 i8 33, label %bb0 i8 34, label %bb0 i8 35, label %bb0
+    i8 36, label %bb0 i8 37, label %bb0 i8 38, label %bb0 i8 39, label %bb0
+    i8 40, label %bb0 i8 41, label %bb0 i8 42, label %bb0 i8 43, label %bb0
+    i8 44, label %bb0 i8 45, label %bb0 i8 46, label %bb0 i8 47, label %bb0
+    i8 48, label %bb0 i8 49, label %bb0 i8 50, label %bb0 i8 51, label %bb0
+    i8 52, label %bb0 i8 53, label %bb0 i8 54, label %bb0 i8 55, label %bb0
+    i8 56, label %bb0 i8 57, label %bb0 i8 58, label %bb0 i8 59, label %bb0
+    i8 60, label %bb0 i8 61, label %bb0 i8 62, label %bb0 i8 63, label %bb0
+    i8 64, label %bb0 i8 65, label %bb0 i8 66, label %bb0 i8 67, label %bb0
+    i8 68, label %bb0 i8 69, label %bb0 i8 70, label %bb0 i8 71, label %bb0
+    i8 72, label %bb0 i8 73, label %bb0 i8 74, label %bb0 i8 75, label %bb0
+    i8 76, label %bb0 i8 77, label %bb0 i8 78, label %bb0 i8 79, label %bb0
+    i8 80, label %bb0 i8 81, label %bb0 i8 82, label %bb0 i8 83, label %bb0
+    i8 84, label %bb0 i8 85, label %bb0 i8 86, label %bb0 i8 87, label %bb0
+    i8 88, label %bb0 i8 89, label %bb0 i8 90, label %bb0 i8 91, label %bb0
+    i8 92, label %bb0 i8 93, label %bb0 i8 94, label %bb0 i8 95, label %bb0
+    i8 96, label %bb0 i8 97, label %bb0 i8 98, label %bb0 i8 99, label %bb0
+    i8 100, label %bb0 i8 101, label %bb0 i8 102, label %bb0 i8 103, label %bb0
+    i8 104, label %bb0 i8 105, label %bb0 i8 106, label %bb0 i8 107, label %bb0
+    i8 108, label %bb0 i8 109, label %bb0 i8 110, label %bb0 i8 111, label %bb0
+    i8 112, label %bb0 i8 113, label %bb0 i8 114, label %bb0 i8 115, label %bb0
+    i8 116, label %bb0 i8 117, label %bb0 i8 118, label %bb0 i8 119, label %bb0
+    i8 120, label %bb0 i8 121, label %bb0 i8 122, label %bb0 i8 123, label %bb0
+    i8 124, label %bb0 i8 125, label %bb0 i8 126, label %bb0 i8 127, label %bb0
+    i8 -64, label %bb1 i8 -63, label %bb1 i8 -62, label %bb1 i8 -61, label %bb1
+    i8 -60, label %bb1 i8 -59, label %bb1 i8 -58, label %bb1 i8 -57, label %bb1
+    i8 -56, label %bb1 i8 -55, label %bb1 i8 -54, label %bb1 i8 -53, label %bb1
+    i8 -52, label %bb1 i8 -51, label %bb1 i8 -50, label %bb1 i8 -49, label %bb1
+    i8 -48, label %bb1 i8 -47, label %bb1 i8 -46, label %bb1 i8 -45, label %bb1
+    i8 -44, label %bb1 i8 -43, label %bb1 i8 -42, label %bb1 i8 -41, label %bb1
+    i8 -40, label %bb1 i8 -39, label %bb1 i8 -38, label %bb1 i8 -37, label %bb1
+    i8 -36, label %bb1 i8 -35, label %bb1 i8 -34, label %bb1 i8 -33, label %bb1
+    i8 -32, label %bb2 i8 -31, label %bb2 i8 -30, label %bb2 i8 -29, label %bb2
+    i8 -28, label %bb2 i8 -27, label %bb2 i8 -26, label %bb2 i8 -25, label %bb2
+    i8 -24, label %bb2 i8 -23, label %bb2 i8 -22, label %bb2 i8 -21, label %bb2
+    i8 -20, label %bb2 i8 -19, label %bb2 i8 -18, label %bb2 i8 -17, label %bb2
+    i8 -16, label %bb3 i8 -15, label %bb3 i8 -14, label %bb3 i8 -13, label %bb3
+    i8 -12, label %bb3 i8 -11, label %bb3 i8 -10, label %bb3 i8 -9, label %bb3
+  ]
+bb0: tail call void @g(i32 0) br label %return
+bb1: tail call void @g(i32 1) br label %return
+bb2: tail call void @g(i32 1) br label %return
+bb3: tail call void @g(i32 1) br label %return
+return: ret void
+
+; Don't infloop on jump tables where the upper bound is the max value of the
+; input type (in this case 127).
+; CHECK-LABEL: int_max_table_cluster
+; CHECK: jmpq *.LJTI
+}
+
+
+define void @bt_order_by_weight(i32 %x) {
+entry:
+  switch i32 %x, label %return [
+    i32 0, label %bb0
+    i32 3, label %bb0
+    i32 6, label %bb0
+    i32 1, label %bb1
+    i32 4, label %bb1
+    i32 7, label %bb1
+    i32 2, label %bb2
+    i32 5, label %bb2
+    i32 8, label %bb2
+    i32 9, label %bb2
+  ], !prof !1
+bb0: tail call void @g(i32 0) br label %return
+bb1: tail call void @g(i32 1) br label %return
+bb2: tail call void @g(i32 2) br label %return
+return: ret void
+
+; Cases 1,4,7 have a very large branch weight (which shouldn't overflow), so
+; their bit test should come first. 0,3,6 and 2,5,8,9 both have a weight of 12,
+; but the latter set has more cases, so should be tested for earlier.
+
+; CHECK-LABEL: bt_order_by_weight
+; 146 = 2^1 + 2^4 + 2^7
+; CHECK: movl $146
+; CHECK: btl
+; 292 = 2^2 + 2^5 + 2^8 + 2^9
+; CHECK: movl $804
+; CHECK: btl
+; 73 = 2^0 + 2^3 + 2^6
+; CHECK: movl $73
+; CHECK: btl
+}
+
+!1 = !{!"branch_weights",
+       ; Default:
+       i32 1,
+       ; Cases 0,3,6:
+       i32 4, i32 4, i32 4,
+       ; Cases 1,4,7:
+       i32 4294967295, i32 2, i32 4294967295,
+       ; Cases 2,5,8,9:
+       i32 3, i32 3, i32 3, i32 3}
+
+define void @order_by_weight_and_fallthrough(i32 %x) {
+entry:
+  switch i32 %x, label %return [
+    i32 100, label %bb1
+    i32 200, label %bb0
+    i32 300, label %bb0
+  ], !prof !2
+bb0: tail call void @g(i32 0) br label %return
+bb1: tail call void @g(i32 1) br label %return
+return: ret void
+
+; Case 200 has the highest weight and should come first. 100 and 300 have the
+; same weight, but 300 goes to the 'next' block, so should be last.
+; CHECK-LABEL: order_by_weight_and_fallthrough
+; CHECK: cmpl $200
+; CHECK: cmpl $100
+; CHECK: cmpl $300
+}
+
+!2 = !{!"branch_weights",
+       ; Default:
+       i32 1,
+       ; Case 100:
+       i32 10,
+       ; Case 200:
+       i32 1000,
+       ; Case 300:
+       i32 10}
+
+
+define void @zero_weight_tree(i32 %x) {
+entry:
+  switch i32 %x, label %return [
+    i32 0,  label %bb0
+    i32 10, label %bb1
+    i32 20, label %bb2
+    i32 30, label %bb3
+    i32 40, label %bb4
+    i32 50, label %bb5
+  ], !prof !3
+bb0: tail call void @g(i32 0) br label %return
+bb1: tail call void @g(i32 1) br label %return
+bb2: tail call void @g(i32 2) br label %return
+bb3: tail call void @g(i32 3) br label %return
+bb4: tail call void @g(i32 4) br label %return
+bb5: tail call void @g(i32 5) br label %return
+return: ret void
+
+; Make sure to pick a pivot in the middle also with zero-weight cases.
+; CHECK-LABEL: zero_weight_tree
+; CHECK-NOT: cmpl
+; CHECK: cmpl $29
+}
+
+!3 = !{!"branch_weights", i32 1, i32 10, i32 0, i32 0, i32 0, i32 0, i32 10}
+
+
+define void @left_leaning_weight_balanced_tree(i32 %x) {
+entry:
+  switch i32 %x, label %return [
+    i32 0,  label %bb0
+    i32 10, label %bb1
+    i32 20, label %bb2
+    i32 30, label %bb3
+    i32 40, label %bb4
+    i32 50, label %bb5
+    i32 60, label %bb6
+    i32 70, label %bb6
+  ], !prof !4
+bb0: tail call void @g(i32 0) br label %return
+bb1: tail call void @g(i32 1) br label %return
+bb2: tail call void @g(i32 2) br label %return
+bb3: tail call void @g(i32 3) br label %return
+bb4: tail call void @g(i32 4) br label %return
+bb5: tail call void @g(i32 5) br label %return
+bb6: tail call void @g(i32 6) br label %return
+bb7: tail call void @g(i32 7) br label %return
+return: ret void
+
+; Without branch probabilities, the pivot would be 40, since that would yield
+; equal-sized sub-trees. When taking weights into account, case 70 becomes the
+; pivot. Since there is room for 3 cases in a leaf, cases 50 and 60 are also
+; included in the right-hand side because that doesn't reduce their rank.
+
+; CHECK-LABEL: left_leaning_weight_balanced_tree
+; CHECK-NOT: cmpl
+; CHECK: cmpl $49
+}
+
+!4 = !{!"branch_weights", i32 1, i32 10, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1000}
+
+
+define void @left_leaning_weight_balanced_tree2(i32 %x) {
+entry:
+  switch i32 %x, label %return [
+    i32 0,  label %bb0
+    i32 10, label %bb1
+    i32 20, label %bb2
+    i32 30, label %bb3
+    i32 40, label %bb4
+    i32 50, label %bb5
+    i32 60, label %bb6
+    i32 70, label %bb6
+  ], !prof !5
+bb0: tail call void @g(i32 0) br label %return
+bb1: tail call void @g(i32 1) br label %return
+bb2: tail call void @g(i32 2) br label %return
+bb3: tail call void @g(i32 3) br label %return
+bb4: tail call void @g(i32 4) br label %return
+bb5: tail call void @g(i32 5) br label %return
+bb6: tail call void @g(i32 6) br label %return
+bb7: tail call void @g(i32 7) br label %return
+return: ret void
+
+; Same as the previous test, except case 50 has higher rank to the left than it
+; would have on the right. Case 60 would have the same rank on both sides, so is
+; moved into the leaf.
+
+; CHECK-LABEL: left_leaning_weight_balanced_tree2
+; CHECK-NOT: cmpl
+; CHECK: cmpl $59
+}
+
+!5 = !{!"branch_weights", i32 1, i32 10, i32 1, i32 1, i32 1, i32 1, i32 90, i32 70, i32 1000}
+
+
+define void @right_leaning_weight_balanced_tree(i32 %x) {
+entry:
+  switch i32 %x, label %return [
+    i32 0,  label %bb0
+    i32 10, label %bb1
+    i32 20, label %bb2
+    i32 30, label %bb3
+    i32 40, label %bb4
+    i32 50, label %bb5
+    i32 60, label %bb6
+    i32 70, label %bb6
+  ], !prof !6
+bb0: tail call void @g(i32 0) br label %return
+bb1: tail call void @g(i32 1) br label %return
+bb2: tail call void @g(i32 2) br label %return
+bb3: tail call void @g(i32 3) br label %return
+bb4: tail call void @g(i32 4) br label %return
+bb5: tail call void @g(i32 5) br label %return
+bb6: tail call void @g(i32 6) br label %return
+bb7: tail call void @g(i32 7) br label %return
+return: ret void
+
+; Analogous to left_leaning_weight_balanced_tree.
+
+; CHECK-LABEL: right_leaning_weight_balanced_tree
+; CHECK-NOT: cmpl
+; CHECK: cmpl $19
+}
+
+!6 = !{!"branch_weights", i32 1, i32 1000, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 10}
+
+
+define void @jump_table_affects_balance(i32 %x) {
+entry:
+  switch i32 %x, label %return [
+    ; Jump table:
+    i32 0,  label %bb0
+    i32 1,  label %bb1
+    i32 2,  label %bb2
+    i32 3,  label %bb3
+
+    i32 100, label %bb0
+    i32 200, label %bb1
+    i32 300, label %bb2
+  ]
+bb0: tail call void @g(i32 0) br label %return
+bb1: tail call void @g(i32 1) br label %return
+bb2: tail call void @g(i32 2) br label %return
+bb3: tail call void @g(i32 3) br label %return
+return: ret void
+
+; CHECK-LABEL: jump_table_affects_balance
+; If the tree were balanced based on number of clusters, {0-3,100} would go on
+; the left and {200,300} on the right. However, the jump table weights as much
+; as its components, so 100 is selected as the pivot.
+; CHECK-NOT: cmpl
+; CHECK: cmpl $99
+}
+
+
+define void @pr23738(i4 %x) {
+entry:
+  switch i4 %x, label %bb0 [
+    i4 0, label %bb1
+    i4 1, label %bb1
+    i4 -5, label %bb1
+  ]
+bb0: tail call void @g(i32 0) br label %return
+bb1: tail call void @g(i32 1) br label %return
+return: ret void
+; Don't assert due to truncating the bitwidth (64) to i4 when checking
+; that the bit-test range fits in a word.
+}
diff --git a/test/CodeGen/X86/system-intrinsics-64.ll b/test/CodeGen/X86/system-intrinsics-64.ll
new file mode 100644
index 00000000000..96c44177339
--- /dev/null
+++ b/test/CodeGen/X86/system-intrinsics-64.ll
@@ -0,0 +1,33 @@
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown | FileCheck %s
+
+define void @test_fxsave(i8* %ptr) {
+; CHECK-LABEL: test_fxsave
+; CHECK: fxsave
+  call void @llvm.x86.fxsave(i8* %ptr)
+  ret void;
+}
+declare void @llvm.x86.fxsave(i8*)
+
+define void @test_fxsave64(i8* %ptr) {
+; CHECK-LABEL: test_fxsave64
+; CHECK: fxsave64
+  call void @llvm.x86.fxsave64(i8* %ptr)
+  ret void;
+}
+declare void @llvm.x86.fxsave64(i8*)
+
+define void @test_fxrstor(i8* %ptr) {
+; CHECK-LABEL: test_fxrstor
+; CHECK: fxrstor
+  call void @llvm.x86.fxrstor(i8* %ptr)
+  ret void;
+}
+declare void @llvm.x86.fxrstor(i8*)
+
+define void @test_fxrstor64(i8* %ptr) {
+; CHECK-LABEL: test_fxrstor64
+; CHECK: fxrstor64
+  call void @llvm.x86.fxrstor64(i8* %ptr)
+  ret void;
+}
+declare void @llvm.x86.fxrstor64(i8*)
diff --git a/test/CodeGen/X86/system-intrinsics.ll b/test/CodeGen/X86/system-intrinsics.ll
new file mode 100644
index 00000000000..84fcd052d7d
--- /dev/null
+++ b/test/CodeGen/X86/system-intrinsics.ll
@@ -0,0 +1,17 @@
+; RUN: llc < %s -mtriple=i686-unknown-unknown   | FileCheck %s
+
+define void @test_fxsave(i8* %ptr) {
+; CHECK-LABEL: test_fxsave
+; CHECK: fxsave
+  call void @llvm.x86.fxsave(i8* %ptr)
+  ret void;
+}
+declare void @llvm.x86.fxsave(i8*)
+
+define void @test_fxrstor(i8* %ptr) {
+; CHECK-LABEL: test_fxrstor
+; CHECK: fxrstor
+  call void @llvm.x86.fxrstor(i8* %ptr)
+  ret void;
+}
+declare void @llvm.x86.fxrstor(i8*)
diff --git a/test/CodeGen/X86/tail-call-got.ll b/test/CodeGen/X86/tail-call-got.ll
index 84d561dcd8c..20d1a87b626 100644
--- a/test/CodeGen/X86/tail-call-got.ll
+++ b/test/CodeGen/X86/tail-call-got.ll
@@ -1,12 +1,14 @@
 ; RUN: llc < %s -relocation-model=pic -mattr=+sse2 | FileCheck %s
 
+; We used to do tail calls through the GOT for these symbols, but it was
+; disabled due to PR15086.
+
 target datalayout = "e-p:32:32:32-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-f32:32:32-f64:32:64-v64:64:64-v128:128:128-a0:0:64-f80:32:32-n8:16:32"
 target triple = "i386-unknown-freebsd9.0"
 
 define double @test1(double %x) nounwind readnone {
 ; CHECK-LABEL: test1:
-; CHECK: movl foo@GOT
-; CHECK-NEXT: jmpl
+; CHECK:  calll foo@PLT
   %1 = tail call double @foo(double %x) nounwind readnone
   ret double %1
 }
@@ -15,10 +17,18 @@ declare double @foo(double) readnone
 
 define double @test2(double %x) nounwind readnone {
 ; CHECK-LABEL: test2:
-; CHECK: movl sin@GOT
-; CHECK-NEXT: jmpl
+; CHECK:  calll sin@PLT
   %1 = tail call double @sin(double %x) nounwind readnone
   ret double %1
 }
 
 declare double @sin(double) readnone
+
+define double @test3(double %x) nounwind readnone {
+; CHECK-LABEL: test3:
+; CHECK:  calll sin2@PLT
+  %1 = tail call double @sin2(double %x) nounwind readnone
+  ret double %1
+}
+
+declare double @sin2(double) readnone
diff --git a/test/CodeGen/X86/tailcall-64.ll b/test/CodeGen/X86/tailcall-64.ll
index 3ababf4d983..9e054fea5b3 100644
--- a/test/CodeGen/X86/tailcall-64.ll
+++ b/test/CodeGen/X86/tailcall-64.ll
@@ -1,8 +1,4 @@
 ; RUN: llc -mtriple=x86_64-apple-macosx -mcpu=core2 < %s | FileCheck %s
-; @LOCALMOD
-; TODO: NaCl tailcall "support" is a mess and has no tests. This only tests for
-; Bug https://code.google.com/p/nativeclient/issues/detail?id=4123
-; RUN: llc -mtriple=x86_64-unknown-nacl -mcpu=core2 < %s | FileCheck %s --check-prefix=NACL
 
 declare i64 @testi()
 
@@ -212,10 +208,6 @@ entry:
 ;
 ; CHECK-LABEL: rdar12282281
 ; CHECK: jmpq *%r11 ## TAILCALL
-
-; For NaCl, ensure we do not tail-call-optimize 6-arg functions
-; NACL-LABEL: rdar12282281
-; NACL: naclcall
 @funcs = external constant [0 x i32 (i8*, ...)*]
 
 define i32 @rdar12282281(i32 %n) nounwind uwtable ssp {
@@ -231,7 +223,6 @@ define x86_fp80 @fp80_call(x86_fp80 %x) nounwind  {
 entry:
 ; CHECK-LABEL: fp80_call:
 ; CHECK: jmp _fp80_callee
-; NACL-LABEL: fp80_call
   %call = tail call x86_fp80 @fp80_callee(x86_fp80 %x) nounwind
   ret x86_fp80 %call
 }
diff --git a/test/CodeGen/X86/tailcallpic1.ll b/test/CodeGen/X86/tailcallpic1.ll
index ff590a1fd3e..ed101fcccd2 100644
--- a/test/CodeGen/X86/tailcallpic1.ll
+++ b/test/CodeGen/X86/tailcallpic1.ll
@@ -1,5 +1,8 @@
 ; RUN: llc < %s  -tailcallopt -mtriple=i686-pc-linux-gnu -relocation-model=pic | FileCheck %s
 
+; This test uses guaranteed TCO so these will be tail calls, despite the early
+; binding issues.
+
 define protected fastcc i32 @tailcallee(i32 %a1, i32 %a2, i32 %a3, i32 %a4) {
 entry:
 	ret i32 %a3
diff --git a/test/CodeGen/X86/tailcallpic3.ll b/test/CodeGen/X86/tailcallpic3.ll
new file mode 100644
index 00000000000..edc58052d82
--- /dev/null
+++ b/test/CodeGen/X86/tailcallpic3.ll
@@ -0,0 +1,73 @@
+; RUN: llc < %s -mtriple=i686-pc-linux-gnu -relocation-model=pic | FileCheck %s
+
+; While many of these could be tail called, we don't do it because it forces
+; early binding.
+
+declare void @external()
+
+define hidden void @tailcallee_hidden() {
+entry:
+  ret void
+}
+
+define void @tailcall_hidden() {
+entry:
+  tail call void @tailcallee_hidden()
+  ret void
+}
+; CHECK: tailcall_hidden:
+; CHECK: jmp tailcallee_hidden
+
+define internal void @tailcallee_internal() {
+entry:
+  ret void
+}
+
+define void @tailcall_internal() {
+entry:
+  tail call void @tailcallee_internal()
+  ret void
+}
+; CHECK: tailcall_internal:
+; CHECK: jmp tailcallee_internal
+
+define default void @tailcallee_default() {
+entry:
+  ret void
+}
+
+define void @tailcall_default() {
+entry:
+  tail call void @tailcallee_default()
+  ret void
+}
+; CHECK: tailcall_default:
+; CHECK: calll tailcallee_default@PLT
+
+define void @tailcallee_default_implicit() {
+entry:
+  ret void
+}
+
+define void @tailcall_default_implicit() {
+entry:
+  tail call void @tailcallee_default_implicit()
+  ret void
+}
+; CHECK: tailcall_default_implicit:
+; CHECK: calll tailcallee_default_implicit@PLT
+
+define void @tailcall_external() {
+  tail call void @external()
+  ret void
+}
+; CHECK: tailcall_external:
+; CHECK: calll external@PLT
+
+define void @musttail_external() {
+  musttail call void @external()
+  ret void
+}
+; CHECK: musttail_external:
+; CHECK: movl external@GOT
+; CHECK: jmpl
diff --git a/test/CodeGen/X86/tailcallstack64.ll b/test/CodeGen/X86/tailcallstack64.ll
index bff5f9924f6..158b777fe1f 100644
--- a/test/CodeGen/X86/tailcallstack64.ll
+++ b/test/CodeGen/X86/tailcallstack64.ll
@@ -12,9 +12,9 @@
 ; Add %in1 %p1 to a different temporary register (%eax).
 ; CHECK: addl {{%edi|%ecx}}, [[R1]]
 ; Move param %in2 to stack.
-; CHECK: movl  [[R2]], [[A1]](%rsp)
+; CHECK-DAG: movl  [[R2]], [[A1]](%rsp)
 ; Move result of addition to stack.
-; CHECK: movl  [[R1]], [[A2]](%rsp)
+; CHECK-DAG: movl  [[R1]], [[A2]](%rsp)
 ; Eventually, do a TAILCALL
 ; CHECK: TAILCALL
 
diff --git a/test/CodeGen/X86/tls-android-negative.ll b/test/CodeGen/X86/tls-android-negative.ll
new file mode 100644
index 00000000000..e90b8914ab2
--- /dev/null
+++ b/test/CodeGen/X86/tls-android-negative.ll
@@ -0,0 +1,65 @@
+; RUN: llc < %s -emulated-tls -march=x86 -mtriple=x86_64-linux-android -relocation-model=pic | FileCheck  %s
+; RUN: llc < %s -emulated-tls -march=x86-64 -mtriple=x86_64-linux-android -relocation-model=pic | FileCheck  %s
+
+; Make sure that some symboles are not emitted in emulated TLS model.
+
+@external_x = external thread_local global i32
+@external_y = thread_local global i32 7
+@internal_y = internal thread_local global i32 9
+@internal_y0 = internal thread_local global i32 0
+
+define i32* @get_external_x() {
+entry:
+  ret i32* @external_x
+}
+
+define i32* @get_external_y() {
+entry:
+  ret i32* @external_y
+}
+
+define i32* @get_internal_y() {
+entry:
+  ret i32* @internal_y
+}
+
+define i32* @get_internal_y0() {
+entry:
+  ret i32* @internal_y0
+}
+
+; no direct access to emulated TLS variables.
+; no definition of emulated TLS variables.
+; no initializer for external TLS variables, __emutls_t.external_x
+; no initializer for 0-initialized TLS variables, __emutls_t.internal_y0
+; not global linkage for __emutls_t.external_y
+
+; CHECK-NOT: external_x@TLS
+; CHECK-NOT: external_y@TLS
+; CHECK-NOT: internal_y@TLS
+; CHECK-NOT: .size external_x
+; CHECK-NOT: .size external_y
+; CHECK-NOT: .size internal_y
+; CHECK-NOT: .size internal_y0
+; CHECK-NOT: __emutls_v.external_x:
+; CHECK-NOT: __emutls_t.external_x:
+; CHECK-NOT: __emutls_t.internal_y0:
+; CHECK-NOT: global __emutls_t.external_y
+; CHECK-NOT: global __emutls_v.internal_y
+; CHECK-NOT: global __emutls_v.internal_y0
+
+; CHECK:     __emutls_t.external_y
+
+; CHECK-NOT: external_x@TLS
+; CHECK-NOT: external_y@TLS
+; CHECK-NOT: internal_y@TLS
+; CHECK-NOT: .size external_x
+; CHECK-NOT: .size external_y
+; CHECK-NOT: .size internal_y
+; CHECK-NOT: .size internal_y0
+; CHECK-NOT: __emutls_v.external_x:
+; CHECK-NOT: __emutls_t.external_x:
+; CHECK-NOT: __emutls_t.internal_y0:
+; CHECK-NOT: global __emutls_t.external_y
+; CHECK-NOT: global __emutls_v.internal_y
+; CHECK-NOT: global __emutls_v.internal_y0
diff --git a/test/CodeGen/X86/tls-android.ll b/test/CodeGen/X86/tls-android.ll
new file mode 100644
index 00000000000..4156c7b3f5b
--- /dev/null
+++ b/test/CodeGen/X86/tls-android.ll
@@ -0,0 +1,89 @@
+; RUN: llc < %s -emulated-tls -march=x86 -mtriple=x86_64-linux-android -relocation-model=pic | FileCheck  %s
+; RUN: llc < %s -emulated-tls -march=x86-64 -mtriple=x86_64-linux-android -relocation-model=pic | FileCheck -check-prefix=X64 %s
+
+; Make sure that TLS symboles are emitted in expected order.
+
+@external_x = external thread_local global i32
+@external_y = thread_local global i32 7
+@internal_y = internal thread_local global i32 9
+
+define i32* @get_external_x() {
+entry:
+  ret i32* @external_x
+}
+
+define i32* @get_external_y() {
+entry:
+  ret i32* @external_y
+}
+
+define i32* @get_internal_y() {
+entry:
+  ret i32* @internal_y
+}
+
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; 32-bit mode
+; CHECK-LABEL: get_external_x:
+; CHECK:  __emutls_v.external_x
+; CHECK:  __emutls_get_address
+
+; CHECK-LABEL: get_external_y:
+; CHECK:  __emutls_v.external_y
+; CHECK:  __emutls_get_address
+
+; CHECK-LABEL: get_internal_y:
+; CHECK:  __emutls_v.internal_y
+; CHECK:  __emutls_get_address
+
+; CHECK-NOT: __emutls_v.external_x:
+
+; CHECK:       .align 4
+; CHECK-LABEL: __emutls_v.external_y:
+; CHECK-NEXT:  .long 4
+; CHECK-NEXT:  .long 4
+; CHECK-NEXT:  .long 0
+; CHECK-NEXT:  .long __emutls_t.external_y
+; CHECK-LABEL: __emutls_t.external_y:
+; CHECK-NEXT:  .long 7
+
+; CHECK:       .align 4
+; CHECK-LABEL: __emutls_v.internal_y:
+; CHECK-NEXT:  .long 4
+; CHECK-NEXT:  .long 4
+; CHECK-NEXT:  .long 0
+; CHECK-NEXT:  .long __emutls_t.internal_y
+; CHECK-LABEL: __emutls_t.internal_y:
+; CHECK-NEXT:  .long 9
+
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; 64-bit mode
+; X64-LABEL: get_external_x:
+; X64:  __emutls_v.external_x
+; X64:  __emutls_get_address
+
+; X64-LABEL: get_external_y:
+; X64:  __emutls_v.external_y
+; X64:  __emutls_get_address
+
+; X64-LABEL: get_internal_y:
+; X64:  __emutls_v.internal_y
+; X64:  __emutls_get_address
+
+; X64-NOT: __emutls_v.external_x:
+
+; X64:       .align 8
+; X64-LABEL: __emutls_v.external_y:
+; X64-NEXT:  .quad 4
+; X64-NEXT:  .quad 4
+; X64-NEXT:  .quad 0
+; X64-NEXT:  .quad __emutls_t.external_y
+; X64-LABEL: __emutls_t.external_y:
+; X64-NEXT:  .long 7
+
+; X64:       .align 8
+; X64-LABEL: __emutls_v.internal_y:
+; X64-NEXT:  .quad 4
+; X64-NEXT:  .quad 4
+; X64-NEXT:  .quad 0
+; X64-NEXT:  .quad __emutls_t.internal_y
+; X64-LABEL: __emutls_t.internal_y:
+; X64-NEXT:  .long 9
diff --git a/test/CodeGen/X86/tls-models.ll b/test/CodeGen/X86/tls-models.ll
index 0fd78532821..2377da4f025 100644
--- a/test/CodeGen/X86/tls-models.ll
+++ b/test/CodeGen/X86/tls-models.ll
@@ -18,6 +18,8 @@
 @external_le = external thread_local(localexec) global i32
 @internal_le = internal thread_local(localexec) global i32 42
 
+; See test cases for emulated model in emutls.ll, emutls-pic.ll and emutls-pie.ll.
+
 ; ----- no model specified -----
 
 define i32* @f1() {
diff --git a/test/CodeGen/X86/tls.ll b/test/CodeGen/X86/tls.ll
index 93ea8333d5c..0f3d3adec4c 100644
--- a/test/CodeGen/X86/tls.ll
+++ b/test/CodeGen/X86/tls.ll
@@ -12,6 +12,7 @@
 @i5 = external hidden thread_local global i32
 @s1 = thread_local global i16 15
 @b1 = thread_local global i8 0
+@b2 = thread_local(localexec) global i8 0
 
 define i32 @f1() {
 ; X32_LINUX-LABEL: f1:
@@ -409,3 +410,31 @@ entry:
 	ret i32 %tmp2
 }
 
+define i8* @f15() {
+; X32_LINUX-LABEL: f15:
+; X32_LINUX:      movl %gs:0, %eax
+; X32_LINUX-NEXT: leal b2@NTPOFF(%eax), %eax
+; X32_LINUX-NEXT: ret
+; X64_LINUX-LABEL: f15:
+; X64_LINUX:      movq %fs:0, %rax
+; X64_LINUX-NEXT: leaq b2@TPOFF(%rax), %rax
+; X64_LINUX-NEXT: ret
+; X32_WIN-LABEL: f15:
+; X32_WIN:      movl %fs:__tls_array, %eax
+; X32_WIN-NEXT: movl (%eax), %eax
+; X32_WIN-NEXT: leal _b2@SECREL32(%eax), %eax
+; X32_WIN-NEXT: ret
+; X64_WIN-LABEL: f15:
+; X64_WIN:      movq %gs:88, %rax
+; X64_WIN-NEXT: movq (%rax), %rax
+; X64_WIN-NEXT: leaq b2@SECREL32(%rax), %rax
+; X64_WIN-NEXT: ret
+; MINGW32-LABEL: f15:
+; MINGW32:      movl %fs:44, %eax
+; MINGW32-NEXT: movl (%eax), %eax
+; MINGW32-NEXT: leal _b2@SECREL32(%eax), %eax
+; MINGW32-NEXT: ret
+entry:
+	ret i8* @b2
+}
+
diff --git a/test/CodeGen/X86/twoaddr-lea.ll b/test/CodeGen/X86/twoaddr-lea.ll
index b5ca0275d8d..5779cf33ac8 100644
--- a/test/CodeGen/X86/twoaddr-lea.ll
+++ b/test/CodeGen/X86/twoaddr-lea.ll
@@ -25,8 +25,7 @@ define i32 @test2(i32 inreg %a, i32 inreg %b, i32 %c, i32 %d) nounwind {
 entry:
 ; CHECK-LABEL: test2:
 ; CHECK: leal
-; CHECK-NOT: leal
-; CHECK-NOT: mov
+; CHECK-NEXT: addl
 ; CHECK-NEXT: addl
 ; CHECK-NEXT: ret
  %add = add i32 %b, %a
diff --git a/test/CodeGen/X86/unaligned-32-byte-memops.ll b/test/CodeGen/X86/unaligned-32-byte-memops.ll
index b337a80b84b..d979c16f4ab 100644
--- a/test/CodeGen/X86/unaligned-32-byte-memops.ll
+++ b/test/CodeGen/X86/unaligned-32-byte-memops.ll
@@ -1,66 +1,72 @@
-; RUN: llc < %s -mtriple=x86_64-apple-darwin -mcpu=corei7-avx | FileCheck %s --check-prefix=SANDYB --check-prefix=CHECK
-; RUN: llc < %s -mtriple=x86_64-apple-darwin -mcpu=core-avx-i | FileCheck %s --check-prefix=SANDYB --check-prefix=CHECK
-; RUN: llc < %s -mtriple=x86_64-apple-darwin -mcpu=btver2 | FileCheck %s --check-prefix=BTVER2 --check-prefix=CHECK
-; RUN: llc < %s -mtriple=x86_64-apple-darwin -mcpu=core-avx2 | FileCheck %s --check-prefix=HASWELL --check-prefix=CHECK
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=avx,+slow-unaligned-mem-32 | FileCheck %s --check-prefix=AVXSLOW
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=avx,-slow-unaligned-mem-32 | FileCheck %s --check-prefix=AVXFAST
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=avx2 | FileCheck %s --check-prefix=AVX2
 
-; On Sandy Bridge or Ivy Bridge, we should not generate an unaligned 32-byte load
-; because that is slower than two 16-byte loads. 
-; Other AVX-capable chips don't have that problem.
+; Don't generate an unaligned 32-byte load on this test if that is slower than two 16-byte loads.
 
 define <8 x float> @load32bytes(<8 x float>* %Ap) {
-  ; CHECK-LABEL: load32bytes
-
-  ; SANDYB: vmovaps
-  ; SANDYB: vinsertf128
-  ; SANDYB: retq
-
-  ; BTVER2: vmovups
-  ; BTVER2: retq
-
-  ; HASWELL: vmovups
-  ; HASWELL: retq
-
+; AVXSLOW-LABEL: load32bytes:
+; AVXSLOW:       # BB#0:
+; AVXSLOW-NEXT:    vmovaps (%rdi), %xmm0
+; AVXSLOW-NEXT:    vinsertf128 $1, 16(%rdi), %ymm0, %ymm0
+; AVXSLOW-NEXT:    retq
+;
+; AVXFAST-LABEL: load32bytes:
+; AVXFAST:       # BB#0:
+; AVXFAST-NEXT:    vmovups (%rdi), %ymm0
+; AVXFAST-NEXT:    retq
+;
+; AVX2-LABEL: load32bytes:
+; AVX2:       # BB#0:
+; AVX2-NEXT:    vmovups (%rdi), %ymm0
+; AVX2-NEXT:    retq
   %A = load <8 x float>, <8 x float>* %Ap, align 16
   ret <8 x float> %A
 }
 
-; On Sandy Bridge or Ivy Bridge, we should not generate an unaligned 32-byte store
-; because that is slowerthan two 16-byte stores. 
-; Other AVX-capable chips don't have that problem.
+; Don't generate an unaligned 32-byte store on this test if that is slower than two 16-byte loads.
 
 define void @store32bytes(<8 x float> %A, <8 x float>* %P) {
-  ; CHECK-LABEL: store32bytes
-
-  ; SANDYB: vextractf128
-  ; SANDYB: vmovaps
-  ; SANDYB: retq
-
-  ; BTVER2: vmovups
-  ; BTVER2: retq
-
-  ; HASWELL: vmovups
-  ; HASWELL: retq
-
+; AVXSLOW-LABEL: store32bytes:
+; AVXSLOW:       # BB#0:
+; AVXSLOW-NEXT:    vextractf128 $1, %ymm0, 16(%rdi)
+; AVXSLOW-NEXT:    vmovaps %xmm0, (%rdi)
+; AVXSLOW-NEXT:    vzeroupper
+; AVXSLOW-NEXT:    retq
+;
+; AVXFAST-LABEL: store32bytes:
+; AVXFAST:       # BB#0:
+; AVXFAST-NEXT:    vmovups %ymm0, (%rdi)
+; AVXFAST-NEXT:    vzeroupper
+; AVXFAST-NEXT:    retq
+;
+; AVX2-LABEL: store32bytes:
+; AVX2:       # BB#0:
+; AVX2-NEXT:    vmovups %ymm0, (%rdi)
+; AVX2-NEXT:    vzeroupper
+; AVX2-NEXT:    retq
   store <8 x float> %A, <8 x float>* %P, align 16
   ret void
 }
 
-; Merge two consecutive 16-byte subvector loads into a single 32-byte load
-; if it's faster.
+; Merge two consecutive 16-byte subvector loads into a single 32-byte load if it's faster.
 
 define <8 x float> @combine_16_byte_loads_no_intrinsic(<4 x float>* %ptr) {
-  ; CHECK-LABEL: combine_16_byte_loads_no_intrinsic
-
-  ; SANDYB: vmovups
-  ; SANDYB-NEXT: vinsertf128
-  ; SANDYB-NEXT: retq
-
-  ; BTVER2: vmovups
-  ; BTVER2-NEXT: retq
-
-  ; HASWELL: vmovups
-  ; HASWELL-NEXT: retq
-
+; AVXSLOW-LABEL: combine_16_byte_loads_no_intrinsic:
+; AVXSLOW:       # BB#0:
+; AVXSLOW-NEXT:    vmovups 48(%rdi), %xmm0
+; AVXSLOW-NEXT:    vinsertf128 $1, 64(%rdi), %ymm0, %ymm0
+; AVXSLOW-NEXT:    retq
+;
+; AVXFAST-LABEL: combine_16_byte_loads_no_intrinsic:
+; AVXFAST:       # BB#0:
+; AVXFAST-NEXT:    vmovups 48(%rdi), %ymm0
+; AVXFAST-NEXT:    retq
+;
+; AVX2-LABEL: combine_16_byte_loads_no_intrinsic:
+; AVX2:       # BB#0:
+; AVX2-NEXT:    vmovups 48(%rdi), %ymm0
+; AVX2-NEXT:    retq
   %ptr1 = getelementptr inbounds <4 x float>, <4 x float>* %ptr, i64 3
   %ptr2 = getelementptr inbounds <4 x float>, <4 x float>* %ptr, i64 4
   %v1 = load <4 x float>, <4 x float>* %ptr1, align 1
@@ -69,21 +75,49 @@ define <8 x float> @combine_16_byte_loads_no_intrinsic(<4 x float>* %ptr) {
   ret <8 x float> %v3
 }
 
-; Swap the order of the shufflevector operands to ensure that the
-; pattern still matches.
-define <8 x float> @combine_16_byte_loads_no_intrinsic_swap(<4 x float>* %ptr) {
-  ; CHECK-LABEL: combine_16_byte_loads_no_intrinsic_swap
-
-  ; SANDYB: vmovups
-  ; SANDYB-NEXT: vinsertf128
-  ; SANDYB-NEXT: retq
-
-  ; BTVER2: vmovups
-  ; BTVER2-NEXT: retq
+define <8 x float> @combine_16_byte_loads_aligned(<4 x float>* %ptr) {
+;; FIXME: The first load is 32-byte aligned, so the second load should get merged.
+; AVXSLOW-LABEL: combine_16_byte_loads_aligned:
+; AVXSLOW:       # BB#0:
+; AVXSLOW-NEXT:    vmovaps 48(%rdi), %xmm0
+; AVXSLOW-NEXT:    vinsertf128 $1, 64(%rdi), %ymm0, %ymm0
+; AVXSLOW-NEXT:    retq
+;
+; AVXFAST-LABEL: combine_16_byte_loads_aligned:
+; AVXFAST:       # BB#0:
+; AVXFAST-NEXT:    vmovaps 48(%rdi), %ymm0
+; AVXFAST-NEXT:    retq
+;
+; AVX2-LABEL: combine_16_byte_loads_aligned:
+; AVX2:       # BB#0:
+; AVX2-NEXT:    vmovaps 48(%rdi), %ymm0
+; AVX2-NEXT:    retq
+  %ptr1 = getelementptr inbounds <4 x float>, <4 x float>* %ptr, i64 3
+  %ptr2 = getelementptr inbounds <4 x float>, <4 x float>* %ptr, i64 4
+  %v1 = load <4 x float>, <4 x float>* %ptr1, align 32
+  %v2 = load <4 x float>, <4 x float>* %ptr2, align 1
+  %v3 = shufflevector <4 x float> %v1, <4 x float> %v2, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+  ret <8 x float> %v3
+}
 
-  ; HASWELL: vmovups
-  ; HASWELL-NEXT: retq
+; Swap the order of the shufflevector operands to ensure that the pattern still matches.
 
+define <8 x float> @combine_16_byte_loads_no_intrinsic_swap(<4 x float>* %ptr) {
+; AVXSLOW-LABEL: combine_16_byte_loads_no_intrinsic_swap:
+; AVXSLOW:       # BB#0:
+; AVXSLOW-NEXT:    vmovups 64(%rdi), %xmm0
+; AVXSLOW-NEXT:    vinsertf128 $1, 80(%rdi), %ymm0, %ymm0
+; AVXSLOW-NEXT:    retq
+;
+; AVXFAST-LABEL: combine_16_byte_loads_no_intrinsic_swap:
+; AVXFAST:       # BB#0:
+; AVXFAST-NEXT:    vmovups 64(%rdi), %ymm0
+; AVXFAST-NEXT:    retq
+;
+; AVX2-LABEL: combine_16_byte_loads_no_intrinsic_swap:
+; AVX2:       # BB#0:
+; AVX2-NEXT:    vmovups 64(%rdi), %ymm0
+; AVX2-NEXT:    retq
   %ptr1 = getelementptr inbounds <4 x float>, <4 x float>* %ptr, i64 4
   %ptr2 = getelementptr inbounds <4 x float>, <4 x float>* %ptr, i64 5
   %v1 = load <4 x float>, <4 x float>* %ptr1, align 1
@@ -94,28 +128,29 @@ define <8 x float> @combine_16_byte_loads_no_intrinsic_swap(<4 x float>* %ptr) {
 
 ; Check each element type other than float to make sure it is handled correctly.
 ; Use the loaded values with an 'add' to make sure we're using the correct load type.
-; Even though BtVer2 has fast 32-byte loads, we should not generate those for
-; 256-bit integer vectors because BtVer2 doesn't have AVX2.
+; Don't generate 32-byte loads for integer ops unless we have AVX2.
 
 define <4 x i64> @combine_16_byte_loads_i64(<2 x i64>* %ptr, <4 x i64> %x) {
-  ; CHECK-LABEL: combine_16_byte_loads_i64
-
-  ; SANDYB: vextractf128
-  ; SANDYB-NEXT: vpaddq
-  ; SANDYB-NEXT: vpaddq
-  ; SANDYB-NEXT: vinsertf128
-  ; SANDYB-NEXT: retq
-
-  ; BTVER2: vextractf128
-  ; BTVER2-NEXT: vpaddq
-  ; BTVER2-NEXT: vpaddq
-  ; BTVER2-NEXT: vinsertf128
-  ; BTVER2-NEXT: retq
-
-  ; HASWELL-NOT: vextract
-  ; HASWELL: vpaddq
-  ; HASWELL-NEXT: retq
-
+; AVXSLOW-LABEL: combine_16_byte_loads_i64:
+; AVXSLOW:       # BB#0:
+; AVXSLOW-NEXT:    vextractf128 $1, %ymm0, %xmm1
+; AVXSLOW-NEXT:    vpaddq 96(%rdi), %xmm1, %xmm1
+; AVXSLOW-NEXT:    vpaddq 80(%rdi), %xmm0, %xmm0
+; AVXSLOW-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
+; AVXSLOW-NEXT:    retq
+;
+; AVXFAST-LABEL: combine_16_byte_loads_i64:
+; AVXFAST:       # BB#0:
+; AVXFAST-NEXT:    vextractf128 $1, %ymm0, %xmm1
+; AVXFAST-NEXT:    vpaddq 96(%rdi), %xmm1, %xmm1
+; AVXFAST-NEXT:    vpaddq 80(%rdi), %xmm0, %xmm0
+; AVXFAST-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
+; AVXFAST-NEXT:    retq
+;
+; AVX2-LABEL: combine_16_byte_loads_i64:
+; AVX2:       # BB#0:
+; AVX2-NEXT:    vpaddq 80(%rdi), %ymm0, %ymm0
+; AVX2-NEXT:    retq
   %ptr1 = getelementptr inbounds <2 x i64>, <2 x i64>* %ptr, i64 5
   %ptr2 = getelementptr inbounds <2 x i64>, <2 x i64>* %ptr, i64 6
   %v1 = load <2 x i64>, <2 x i64>* %ptr1, align 1
@@ -126,24 +161,26 @@ define <4 x i64> @combine_16_byte_loads_i64(<2 x i64>* %ptr, <4 x i64> %x) {
 }
 
 define <8 x i32> @combine_16_byte_loads_i32(<4 x i32>* %ptr, <8 x i32> %x) {
-  ; CHECK-LABEL: combine_16_byte_loads_i32
-
-  ; SANDYB: vextractf128
-  ; SANDYB-NEXT: vpaddd
-  ; SANDYB-NEXT: vpaddd
-  ; SANDYB-NEXT: vinsertf128
-  ; SANDYB-NEXT: retq
-
-  ; BTVER2: vextractf128
-  ; BTVER2-NEXT: vpaddd
-  ; BTVER2-NEXT: vpaddd
-  ; BTVER2-NEXT: vinsertf128
-  ; BTVER2-NEXT: retq
-
-  ; HASWELL-NOT: vextract
-  ; HASWELL: vpaddd
-  ; HASWELL-NEXT: retq
-
+; AVXSLOW-LABEL: combine_16_byte_loads_i32:
+; AVXSLOW:       # BB#0:
+; AVXSLOW-NEXT:    vextractf128 $1, %ymm0, %xmm1
+; AVXSLOW-NEXT:    vpaddd 112(%rdi), %xmm1, %xmm1
+; AVXSLOW-NEXT:    vpaddd 96(%rdi), %xmm0, %xmm0
+; AVXSLOW-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
+; AVXSLOW-NEXT:    retq
+;
+; AVXFAST-LABEL: combine_16_byte_loads_i32:
+; AVXFAST:       # BB#0:
+; AVXFAST-NEXT:    vextractf128 $1, %ymm0, %xmm1
+; AVXFAST-NEXT:    vpaddd 112(%rdi), %xmm1, %xmm1
+; AVXFAST-NEXT:    vpaddd 96(%rdi), %xmm0, %xmm0
+; AVXFAST-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
+; AVXFAST-NEXT:    retq
+;
+; AVX2-LABEL: combine_16_byte_loads_i32:
+; AVX2:       # BB#0:
+; AVX2-NEXT:    vpaddd 96(%rdi), %ymm0, %ymm0
+; AVX2-NEXT:    retq
   %ptr1 = getelementptr inbounds <4 x i32>, <4 x i32>* %ptr, i64 6
   %ptr2 = getelementptr inbounds <4 x i32>, <4 x i32>* %ptr, i64 7
   %v1 = load <4 x i32>, <4 x i32>* %ptr1, align 1
@@ -154,24 +191,26 @@ define <8 x i32> @combine_16_byte_loads_i32(<4 x i32>* %ptr, <8 x i32> %x) {
 }
 
 define <16 x i16> @combine_16_byte_loads_i16(<8 x i16>* %ptr, <16 x i16> %x) {
-  ; CHECK-LABEL: combine_16_byte_loads_i16
-
-  ; SANDYB: vextractf128
-  ; SANDYB-NEXT: vpaddw
-  ; SANDYB-NEXT: vpaddw
-  ; SANDYB-NEXT: vinsertf128
-  ; SANDYB-NEXT: retq
-
-  ; BTVER2: vextractf128
-  ; BTVER2-NEXT: vpaddw
-  ; BTVER2-NEXT: vpaddw
-  ; BTVER2-NEXT: vinsertf128
-  ; BTVER2-NEXT: retq
-
-  ; HASWELL-NOT: vextract
-  ; HASWELL: vpaddw
-  ; HASWELL-NEXT: retq
-
+; AVXSLOW-LABEL: combine_16_byte_loads_i16:
+; AVXSLOW:       # BB#0:
+; AVXSLOW-NEXT:    vextractf128 $1, %ymm0, %xmm1
+; AVXSLOW-NEXT:    vpaddw 128(%rdi), %xmm1, %xmm1
+; AVXSLOW-NEXT:    vpaddw 112(%rdi), %xmm0, %xmm0
+; AVXSLOW-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
+; AVXSLOW-NEXT:    retq
+;
+; AVXFAST-LABEL: combine_16_byte_loads_i16:
+; AVXFAST:       # BB#0:
+; AVXFAST-NEXT:    vextractf128 $1, %ymm0, %xmm1
+; AVXFAST-NEXT:    vpaddw 128(%rdi), %xmm1, %xmm1
+; AVXFAST-NEXT:    vpaddw 112(%rdi), %xmm0, %xmm0
+; AVXFAST-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
+; AVXFAST-NEXT:    retq
+;
+; AVX2-LABEL: combine_16_byte_loads_i16:
+; AVX2:       # BB#0:
+; AVX2-NEXT:    vpaddw 112(%rdi), %ymm0, %ymm0
+; AVX2-NEXT:    retq
   %ptr1 = getelementptr inbounds <8 x i16>, <8 x i16>* %ptr, i64 7
   %ptr2 = getelementptr inbounds <8 x i16>, <8 x i16>* %ptr, i64 8
   %v1 = load <8 x i16>, <8 x i16>* %ptr1, align 1
@@ -182,24 +221,26 @@ define <16 x i16> @combine_16_byte_loads_i16(<8 x i16>* %ptr, <16 x i16> %x) {
 }
 
 define <32 x i8> @combine_16_byte_loads_i8(<16 x i8>* %ptr, <32 x i8> %x) {
-  ; CHECK-LABEL: combine_16_byte_loads_i8
-
-  ; SANDYB: vextractf128
-  ; SANDYB-NEXT: vpaddb
-  ; SANDYB-NEXT: vpaddb
-  ; SANDYB-NEXT: vinsertf128
-  ; SANDYB-NEXT: retq
-
-  ; BTVER2: vextractf128
-  ; BTVER2-NEXT: vpaddb
-  ; BTVER2-NEXT: vpaddb
-  ; BTVER2-NEXT: vinsertf128
-  ; BTVER2-NEXT: retq
-
-  ; HASWELL-NOT: vextract
-  ; HASWELL: vpaddb
-  ; HASWELL-NEXT: retq
-
+; AVXSLOW-LABEL: combine_16_byte_loads_i8:
+; AVXSLOW:       # BB#0:
+; AVXSLOW-NEXT:    vextractf128 $1, %ymm0, %xmm1
+; AVXSLOW-NEXT:    vpaddb 144(%rdi), %xmm1, %xmm1
+; AVXSLOW-NEXT:    vpaddb 128(%rdi), %xmm0, %xmm0
+; AVXSLOW-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
+; AVXSLOW-NEXT:    retq
+;
+; AVXFAST-LABEL: combine_16_byte_loads_i8:
+; AVXFAST:       # BB#0:
+; AVXFAST-NEXT:    vextractf128 $1, %ymm0, %xmm1
+; AVXFAST-NEXT:    vpaddb 144(%rdi), %xmm1, %xmm1
+; AVXFAST-NEXT:    vpaddb 128(%rdi), %xmm0, %xmm0
+; AVXFAST-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
+; AVXFAST-NEXT:    retq
+;
+; AVX2-LABEL: combine_16_byte_loads_i8:
+; AVX2:       # BB#0:
+; AVX2-NEXT:    vpaddb 128(%rdi), %ymm0, %ymm0
+; AVX2-NEXT:    retq
   %ptr1 = getelementptr inbounds <16 x i8>, <16 x i8>* %ptr, i64 8
   %ptr2 = getelementptr inbounds <16 x i8>, <16 x i8>* %ptr, i64 9
   %v1 = load <16 x i8>, <16 x i8>* %ptr1, align 1
@@ -210,21 +251,22 @@ define <32 x i8> @combine_16_byte_loads_i8(<16 x i8>* %ptr, <32 x i8> %x) {
 }
 
 define <4 x double> @combine_16_byte_loads_double(<2 x double>* %ptr, <4 x double> %x) {
-  ; CHECK-LABEL: combine_16_byte_loads_double
-
-  ; SANDYB: vmovupd
-  ; SANDYB-NEXT: vinsertf128
-  ; SANDYB-NEXT: vaddpd
-  ; SANDYB-NEXT: retq
-
-  ; BTVER2-NOT: vinsertf128
-  ; BTVER2: vaddpd
-  ; BTVER2-NEXT: retq
-
-  ; HASWELL-NOT: vinsertf128
-  ; HASWELL: vaddpd
-  ; HASWELL-NEXT: retq
-
+; AVXSLOW-LABEL: combine_16_byte_loads_double:
+; AVXSLOW:       # BB#0:
+; AVXSLOW-NEXT:    vmovupd 144(%rdi), %xmm1
+; AVXSLOW-NEXT:    vinsertf128 $1, 160(%rdi), %ymm1, %ymm1
+; AVXSLOW-NEXT:    vaddpd %ymm0, %ymm1, %ymm0
+; AVXSLOW-NEXT:    retq
+;
+; AVXFAST-LABEL: combine_16_byte_loads_double:
+; AVXFAST:       # BB#0:
+; AVXFAST-NEXT:    vaddpd 144(%rdi), %ymm0, %ymm0
+; AVXFAST-NEXT:    retq
+;
+; AVX2-LABEL: combine_16_byte_loads_double:
+; AVX2:       # BB#0:
+; AVX2-NEXT:    vaddpd 144(%rdi), %ymm0, %ymm0
+; AVX2-NEXT:    retq
   %ptr1 = getelementptr inbounds <2 x double>, <2 x double>* %ptr, i64 9
   %ptr2 = getelementptr inbounds <2 x double>, <2 x double>* %ptr, i64 10
   %v1 = load <2 x double>, <2 x double>* %ptr1, align 1
diff --git a/test/CodeGen/X86/unknown-location.ll b/test/CodeGen/X86/unknown-location.ll
index 2ed1acf85fe..1b73e649630 100644
--- a/test/CodeGen/X86/unknown-location.ll
+++ b/test/CodeGen/X86/unknown-location.ll
@@ -21,16 +21,16 @@ entry:
 !llvm.dbg.cu = !{!3}
 !llvm.module.flags = !{!12}
 
-!0 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "x", line: 1, arg: 0, scope: !1, file: !2, type: !6)
-!1 = !MDSubprogram(name: "foo", linkageName: "foo", line: 1, isLocal: false, isDefinition: true, virtualIndex: 6, isOptimized: false, scopeLine: 1, file: !10, scope: !2, type: !4, function: i32 (i32, i32, i32, i32)* @foo)
-!2 = !MDFile(filename: "test.c", directory: "/dir")
-!3 = !MDCompileUnit(language: DW_LANG_C99, producer: "producer", isOptimized: false, emissionKind: 0, file: !10, enums: !11, retainedTypes: !11, subprograms: !9)
-!4 = !MDSubroutineType(types: !5)
+!0 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "x", line: 1, arg: 2, scope: !1, file: !2, type: !6)
+!1 = !DISubprogram(name: "foo", linkageName: "foo", line: 1, isLocal: false, isDefinition: true, virtualIndex: 6, isOptimized: false, scopeLine: 1, file: !10, scope: !2, type: !4, function: i32 (i32, i32, i32, i32)* @foo)
+!2 = !DIFile(filename: "test.c", directory: "/dir")
+!3 = !DICompileUnit(language: DW_LANG_C99, producer: "producer", isOptimized: false, emissionKind: 0, file: !10, enums: !11, retainedTypes: !11, subprograms: !9)
+!4 = !DISubroutineType(types: !5)
 !5 = !{!6}
-!6 = !MDBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
-!7 = distinct !MDLexicalBlock(line: 1, column: 30, file: !10, scope: !1)
-!8 = !MDLocation(line: 4, column: 3, scope: !7)
+!6 = !DIBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
+!7 = distinct !DILexicalBlock(line: 1, column: 30, file: !10, scope: !1)
+!8 = !DILocation(line: 4, column: 3, scope: !7)
 !9 = !{!1}
-!10 = !MDFile(filename: "test.c", directory: "/dir")
+!10 = !DIFile(filename: "test.c", directory: "/dir")
 !11 = !{}
 !12 = !{i32 1, !"Debug Info Version", i32 3}
diff --git a/test/CodeGen/X86/vec_cast2.ll b/test/CodeGen/X86/vec_cast2.ll
index e50789570ed..1ba11f51baa 100644
--- a/test/CodeGen/X86/vec_cast2.ll
+++ b/test/CodeGen/X86/vec_cast2.ll
@@ -16,13 +16,9 @@ define <8 x float> @foo1_8(<8 x i8> %src) {
 ;
 ; CHECK-WIDE-LABEL: foo1_8:
 ; CHECK-WIDE:       ## BB#0:
-; CHECK-WIDE-NEXT:    vpmovzxbd {{.*#+}} xmm1 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
-; CHECK-WIDE-NEXT:    vpslld $24, %xmm1, %xmm1
-; CHECK-WIDE-NEXT:    vpsrad $24, %xmm1, %xmm1
-; CHECK-WIDE-NEXT:    vpunpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
-; CHECK-WIDE-NEXT:    vpunpckhwd {{.*#+}} xmm0 = xmm0[4,4,5,5,6,6,7,7]
-; CHECK-WIDE-NEXT:    vpslld $24, %xmm0, %xmm0
-; CHECK-WIDE-NEXT:    vpsrad $24, %xmm0, %xmm0
+; CHECK-WIDE-NEXT:    vpmovsxbd %xmm0, %xmm1
+; CHECK-WIDE-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[1,1,2,3]
+; CHECK-WIDE-NEXT:    vpmovsxbd %xmm0, %xmm0
 ; CHECK-WIDE-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
 ; CHECK-WIDE-NEXT:    vcvtdq2ps %ymm0, %ymm0
 ; CHECK-WIDE-NEXT:    retl
@@ -40,9 +36,7 @@ define <4 x float> @foo1_4(<4 x i8> %src) {
 ;
 ; CHECK-WIDE-LABEL: foo1_4:
 ; CHECK-WIDE:       ## BB#0:
-; CHECK-WIDE-NEXT:    vpmovzxbd {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
-; CHECK-WIDE-NEXT:    vpslld $24, %xmm0, %xmm0
-; CHECK-WIDE-NEXT:    vpsrad $24, %xmm0, %xmm0
+; CHECK-WIDE-NEXT:    vpmovsxbd %xmm0, %xmm0
 ; CHECK-WIDE-NEXT:    vcvtdq2ps %xmm0, %xmm0
 ; CHECK-WIDE-NEXT:    retl
   %res = sitofp <4 x i8> %src to <4 x float>
diff --git a/test/CodeGen/X86/vec_ctbits.ll b/test/CodeGen/X86/vec_ctbits.ll
index 318aca1d54c..6b3e73b6201 100644
--- a/test/CodeGen/X86/vec_ctbits.ll
+++ b/test/CodeGen/X86/vec_ctbits.ll
@@ -5,25 +5,63 @@ declare <2 x i64> @llvm.ctlz.v2i64(<2 x i64>, i1)
 declare <2 x i64> @llvm.ctpop.v2i64(<2 x i64>)
 
 define <2 x i64> @footz(<2 x i64> %a) nounwind {
+; CHECK-LABEL: footz:
+; CHECK:       # BB#0:
+; CHECK-NEXT:    movd %xmm0, %rax
+; CHECK-NEXT:    bsfq %rax, %rax
+; CHECK-NEXT:    movd %rax, %xmm1
+; CHECK-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
+; CHECK-NEXT:    movd %xmm0, %rax
+; CHECK-NEXT:    bsfq %rax, %rax
+; CHECK-NEXT:    movd %rax, %xmm0
+; CHECK-NEXT:    punpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm0[0]
+; CHECK-NEXT:    movdqa %xmm1, %xmm0
+; CHECK-NEXT:    retq
   %c = call <2 x i64> @llvm.cttz.v2i64(<2 x i64> %a, i1 true)
   ret <2 x i64> %c
 
-; CHECK-LABEL: footz
-; CHECK: bsfq
-; CHECK: bsfq
 }
 define <2 x i64> @foolz(<2 x i64> %a) nounwind {
+; CHECK-LABEL: foolz:
+; CHECK:       # BB#0:
+; CHECK-NEXT:    movd %xmm0, %rax
+; CHECK-NEXT:    bsrq %rax, %rax
+; CHECK-NEXT:    xorq $63, %rax
+; CHECK-NEXT:    movd %rax, %xmm1
+; CHECK-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
+; CHECK-NEXT:    movd %xmm0, %rax
+; CHECK-NEXT:    bsrq %rax, %rax
+; CHECK-NEXT:    xorq $63, %rax
+; CHECK-NEXT:    movd %rax, %xmm0
+; CHECK-NEXT:    punpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm0[0]
+; CHECK-NEXT:    movdqa %xmm1, %xmm0
+; CHECK-NEXT:    retq
   %c = call <2 x i64> @llvm.ctlz.v2i64(<2 x i64> %a, i1 true)
   ret <2 x i64> %c
 
-; CHECK-LABEL: foolz
-; CHECK: bsrq
-; CHECK: xorq $63
-; CHECK: bsrq
-; CHECK: xorq $63
 }
 
 define <2 x i64> @foopop(<2 x i64> %a) nounwind {
+; CHECK-LABEL: foopop:
+; CHECK:       # BB#0:
+; CHECK-NEXT:    movdqa %xmm0, %xmm1
+; CHECK-NEXT:    psrlq $1, %xmm1
+; CHECK-NEXT:    pand {{.*}}(%rip), %xmm1
+; CHECK-NEXT:    psubq %xmm1, %xmm0
+; CHECK-NEXT:    movdqa {{.*#+}} xmm1 = [3689348814741910323,3689348814741910323]
+; CHECK-NEXT:    movdqa %xmm0, %xmm2
+; CHECK-NEXT:    pand %xmm1, %xmm2
+; CHECK-NEXT:    psrlq $2, %xmm0
+; CHECK-NEXT:    pand %xmm1, %xmm0
+; CHECK-NEXT:    paddq %xmm2, %xmm0
+; CHECK-NEXT:    movdqa %xmm0, %xmm1
+; CHECK-NEXT:    psrlq $4, %xmm1
+; CHECK-NEXT:    paddq %xmm0, %xmm1
+; CHECK-NEXT:    pand {{.*}}(%rip), %xmm1
+; CHECK-NEXT:    pxor %xmm0, %xmm0
+; CHECK-NEXT:    psadbw %xmm0, %xmm1
+; CHECK-NEXT:    movdqa %xmm1, %xmm0
+; CHECK-NEXT:    retq
   %c = call <2 x i64> @llvm.ctpop.v2i64(<2 x i64> %a)
   ret <2 x i64> %c
 }
@@ -33,35 +71,73 @@ declare <2 x i32> @llvm.ctlz.v2i32(<2 x i32>, i1)
 declare <2 x i32> @llvm.ctpop.v2i32(<2 x i32>)
 
 define <2 x i32> @promtz(<2 x i32> %a) nounwind {
+; CHECK-LABEL: promtz:
+; CHECK:       # BB#0:
+; CHECK-NEXT:    por {{.*}}(%rip), %xmm0
+; CHECK-NEXT:    movd %xmm0, %rax
+; CHECK-NEXT:    bsfq %rax, %rax
+; CHECK-NEXT:    movl $64, %ecx
+; CHECK-NEXT:    cmoveq %rcx, %rax
+; CHECK-NEXT:    movd %rax, %xmm1
+; CHECK-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
+; CHECK-NEXT:    movd %xmm0, %rax
+; CHECK-NEXT:    bsfq %rax, %rax
+; CHECK-NEXT:    cmoveq %rcx, %rax
+; CHECK-NEXT:    movd %rax, %xmm0
+; CHECK-NEXT:    punpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm0[0]
+; CHECK-NEXT:    movdqa %xmm1, %xmm0
+; CHECK-NEXT:    retq
   %c = call <2 x i32> @llvm.cttz.v2i32(<2 x i32> %a, i1 false)
   ret <2 x i32> %c
 
-; CHECK: .quad 4294967296
-; CHECK: .quad 4294967296
-; CHECK-LABEL: promtz
-; CHECK: bsfq
-; CHECK: cmov
-; CHECK: bsfq
-; CHECK: cmov
 }
 define <2 x i32> @promlz(<2 x i32> %a) nounwind {
+; CHECK-LABEL: promlz:
+; CHECK:       # BB#0:
+; CHECK-NEXT:    pand {{.*}}(%rip), %xmm0
+; CHECK-NEXT:    movd %xmm0, %rax
+; CHECK-NEXT:    bsrq %rax, %rax
+; CHECK-NEXT:    movl $127, %ecx
+; CHECK-NEXT:    cmoveq %rcx, %rax
+; CHECK-NEXT:    xorq $63, %rax
+; CHECK-NEXT:    movd %rax, %xmm1
+; CHECK-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
+; CHECK-NEXT:    movd %xmm0, %rax
+; CHECK-NEXT:    bsrq %rax, %rax
+; CHECK-NEXT:    cmoveq %rcx, %rax
+; CHECK-NEXT:    xorq $63, %rax
+; CHECK-NEXT:    movd %rax, %xmm0
+; CHECK-NEXT:    punpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm0[0]
+; CHECK-NEXT:    psubq {{.*}}(%rip), %xmm1
+; CHECK-NEXT:    movdqa %xmm1, %xmm0
+; CHECK-NEXT:    retq
   %c = call <2 x i32> @llvm.ctlz.v2i32(<2 x i32> %a, i1 false)
   ret <2 x i32> %c
 
-; CHECK: .quad 4294967295
-; CHECK: .quad 4294967295
-; CHECK: .quad 32
-; CHECK: .quad 32
-; CHECK-LABEL: promlz
-; CHECK: pand
-; CHECK: bsrq
-; CHECK: xorq $63
-; CHECK: bsrq
-; CHECK: xorq $63
-; CHECK: psub
 }
 
 define <2 x i32> @prompop(<2 x i32> %a) nounwind {
+; CHECK-LABEL: prompop:
+; CHECK:       # BB#0:
+; CHECK-NEXT:    pand {{.*}}(%rip), %xmm0
+; CHECK-NEXT:    movdqa %xmm0, %xmm1
+; CHECK-NEXT:    psrlq $1, %xmm1
+; CHECK-NEXT:    pand {{.*}}(%rip), %xmm1
+; CHECK-NEXT:    psubq %xmm1, %xmm0
+; CHECK-NEXT:    movdqa {{.*#+}} xmm1 = [3689348814741910323,3689348814741910323]
+; CHECK-NEXT:    movdqa %xmm0, %xmm2
+; CHECK-NEXT:    pand %xmm1, %xmm2
+; CHECK-NEXT:    psrlq $2, %xmm0
+; CHECK-NEXT:    pand %xmm1, %xmm0
+; CHECK-NEXT:    paddq %xmm2, %xmm0
+; CHECK-NEXT:    movdqa %xmm0, %xmm1
+; CHECK-NEXT:    psrlq $4, %xmm1
+; CHECK-NEXT:    paddq %xmm0, %xmm1
+; CHECK-NEXT:    pand {{.*}}(%rip), %xmm1
+; CHECK-NEXT:    pxor %xmm0, %xmm0
+; CHECK-NEXT:    psadbw %xmm0, %xmm1
+; CHECK-NEXT:    movdqa %xmm1, %xmm0
+; CHECK-NEXT:    retq
   %c = call <2 x i32> @llvm.ctpop.v2i32(<2 x i32> %a)
   ret <2 x i32> %c
 }
diff --git a/test/CodeGen/X86/vec_fabs.ll b/test/CodeGen/X86/vec_fabs.ll
index bfefbcf5ebd..960b5f27cf5 100644
--- a/test/CodeGen/X86/vec_fabs.ll
+++ b/test/CodeGen/X86/vec_fabs.ll
@@ -4,7 +4,7 @@
 define <2 x double> @fabs_v2f64(<2 x double> %p)
 {
   ; CHECK-LABEL: fabs_v2f64
-  ; CHECK: vandps
+  ; CHECK: vandpd
   %t = call <2 x double> @llvm.fabs.v2f64(<2 x double> %p)
   ret <2 x double> %t
 }
@@ -22,7 +22,7 @@ declare <4 x float> @llvm.fabs.v4f32(<4 x float> %p)
 define <4 x double> @fabs_v4f64(<4 x double> %p)
 {
   ; CHECK-LABEL: fabs_v4f64
-  ; CHECK: vandps
+  ; CHECK: vandpd
   %t = call <4 x double> @llvm.fabs.v4f64(<4 x double> %p)
   ret <4 x double> %t
 }
diff --git a/test/CodeGen/X86/vec_fp_to_int.ll b/test/CodeGen/X86/vec_fp_to_int.ll
new file mode 100644
index 00000000000..7834b280424
--- /dev/null
+++ b/test/CodeGen/X86/vec_fp_to_int.ll
@@ -0,0 +1,1247 @@
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+sse2 | FileCheck %s --check-prefix=ALL --check-prefix=SSE --check-prefix=SSE2
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx | FileCheck %s --check-prefix=ALL --check-prefix=AVX --check-prefix=AVX1
+;
+; 32-bit tests to make sure we're not doing anything stupid.
+; RUN: llc < %s -mtriple=i686-unknown-unknown
+; RUN: llc < %s -mtriple=i686-unknown-unknown -mattr=+sse
+; RUN: llc < %s -mtriple=i686-unknown-unknown -mattr=+sse2
+
+;
+; Double to Signed Integer
+;
+
+define <2 x i64> @fptosi_2f64_to_2i64(<2 x double> %a) {
+; SSE-LABEL: fptosi_2f64_to_2i64:
+; SSE:       # BB#0:
+; SSE-NEXT:    cvttsd2si %xmm0, %rax
+; SSE-NEXT:    movd %rax, %xmm1
+; SSE-NEXT:    shufpd {{.*#+}} xmm0 = xmm0[1,0]
+; SSE-NEXT:    cvttsd2si %xmm0, %rax
+; SSE-NEXT:    movd %rax, %xmm0
+; SSE-NEXT:    punpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm0[0]
+; SSE-NEXT:    movdqa %xmm1, %xmm0
+; SSE-NEXT:    retq
+;
+; AVX-LABEL: fptosi_2f64_to_2i64:
+; AVX:       # BB#0:
+; AVX-NEXT:    vcvttsd2si %xmm0, %rax
+; AVX-NEXT:    vmovq %rax, %xmm1
+; AVX-NEXT:    vpermilpd {{.*#+}} xmm0 = xmm0[1,0]
+; AVX-NEXT:    vcvttsd2si %xmm0, %rax
+; AVX-NEXT:    vmovq %rax, %xmm0
+; AVX-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm1[0],xmm0[0]
+; AVX-NEXT:    retq
+  %cvt = fptosi <2 x double> %a to <2 x i64>
+  ret <2 x i64> %cvt
+}
+
+define <4 x i32> @fptosi_2f64_to_2i32(<2 x double> %a) {
+; SSE-LABEL: fptosi_2f64_to_2i32:
+; SSE:       # BB#0:
+; SSE-NEXT:    cvttsd2si %xmm0, %rax
+; SSE-NEXT:    movd %rax, %xmm1
+; SSE-NEXT:    shufpd {{.*#+}} xmm0 = xmm0[1,0]
+; SSE-NEXT:    cvttsd2si %xmm0, %rax
+; SSE-NEXT:    movd %rax, %xmm0
+; SSE-NEXT:    punpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm0[0]
+; SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[0,2,2,3]
+; SSE-NEXT:    retq
+;
+; AVX-LABEL: fptosi_2f64_to_2i32:
+; AVX:       # BB#0:
+; AVX-NEXT:    vcvttsd2si %xmm0, %rax
+; AVX-NEXT:    vmovq %rax, %xmm1
+; AVX-NEXT:    vpermilpd {{.*#+}} xmm0 = xmm0[1,0]
+; AVX-NEXT:    vcvttsd2si %xmm0, %rax
+; AVX-NEXT:    vmovq %rax, %xmm0
+; AVX-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm1[0],xmm0[0]
+; AVX-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
+; AVX-NEXT:    retq
+  %cvt = fptosi <2 x double> %a to <2 x i32>
+  %ext = shufflevector <2 x i32> %cvt, <2 x i32> undef, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
+  ret <4 x i32> %ext
+}
+
+define <4 x i32> @fptosi_4f64_to_2i32(<2 x double> %a) {
+; SSE-LABEL: fptosi_4f64_to_2i32:
+; SSE:       # BB#0:
+; SSE-NEXT:    cvttsd2si %xmm0, %rax
+; SSE-NEXT:    movd %rax, %xmm1
+; SSE-NEXT:    shufpd {{.*#+}} xmm0 = xmm0[1,0]
+; SSE-NEXT:    cvttsd2si %xmm0, %rax
+; SSE-NEXT:    movd %rax, %xmm0
+; SSE-NEXT:    punpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm0[0]
+; SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[0,2,2,3]
+; SSE-NEXT:    cvttsd2si %xmm0, %rax
+; SSE-NEXT:    movd %rax, %xmm1
+; SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,1,0,1]
+; SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3]
+; SSE-NEXT:    punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
+; SSE-NEXT:    retq
+;
+; AVX-LABEL: fptosi_4f64_to_2i32:
+; AVX:       # BB#0:
+; AVX-NEXT:    vcvttpd2dqy %ymm0, %xmm0
+; AVX-NEXT:    vzeroupper
+; AVX-NEXT:    retq
+  %ext = shufflevector <2 x double> %a, <2 x double> undef, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
+  %cvt = fptosi <4 x double> %ext to <4 x i32>
+  ret <4 x i32> %cvt
+}
+
+define <4 x i64> @fptosi_4f64_to_4i64(<4 x double> %a) {
+; SSE-LABEL: fptosi_4f64_to_4i64:
+; SSE:       # BB#0:
+; SSE-NEXT:    cvttsd2si %xmm0, %rax
+; SSE-NEXT:    movd %rax, %xmm2
+; SSE-NEXT:    shufpd {{.*#+}} xmm0 = xmm0[1,0]
+; SSE-NEXT:    cvttsd2si %xmm0, %rax
+; SSE-NEXT:    movd %rax, %xmm0
+; SSE-NEXT:    punpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm0[0]
+; SSE-NEXT:    cvttsd2si %xmm1, %rax
+; SSE-NEXT:    movd %rax, %xmm3
+; SSE-NEXT:    shufpd {{.*#+}} xmm1 = xmm1[1,0]
+; SSE-NEXT:    cvttsd2si %xmm1, %rax
+; SSE-NEXT:    movd %rax, %xmm0
+; SSE-NEXT:    punpcklqdq {{.*#+}} xmm3 = xmm3[0],xmm0[0]
+; SSE-NEXT:    movdqa %xmm2, %xmm0
+; SSE-NEXT:    movdqa %xmm3, %xmm1
+; SSE-NEXT:    retq
+;
+; AVX-LABEL: fptosi_4f64_to_4i64:
+; AVX:       # BB#0:
+; AVX-NEXT:    vextractf128 $1, %ymm0, %xmm1
+; AVX-NEXT:    vcvttsd2si %xmm1, %rax
+; AVX-NEXT:    vmovq %rax, %xmm2
+; AVX-NEXT:    vpermilpd {{.*#+}} xmm1 = xmm1[1,0]
+; AVX-NEXT:    vcvttsd2si %xmm1, %rax
+; AVX-NEXT:    vmovq %rax, %xmm1
+; AVX-NEXT:    vpunpcklqdq {{.*#+}} xmm1 = xmm2[0],xmm1[0]
+; AVX-NEXT:    vcvttsd2si %xmm0, %rax
+; AVX-NEXT:    vmovq %rax, %xmm2
+; AVX-NEXT:    vpermilpd {{.*#+}} xmm0 = xmm0[1,0]
+; AVX-NEXT:    vcvttsd2si %xmm0, %rax
+; AVX-NEXT:    vmovq %rax, %xmm0
+; AVX-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm2[0],xmm0[0]
+; AVX-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
+; AVX-NEXT:    retq
+  %cvt = fptosi <4 x double> %a to <4 x i64>
+  ret <4 x i64> %cvt
+}
+
+define <4 x i32> @fptosi_4f64_to_4i32(<4 x double> %a) {
+; SSE-LABEL: fptosi_4f64_to_4i32:
+; SSE:       # BB#0:
+; SSE-NEXT:    cvttsd2si %xmm1, %rax
+; SSE-NEXT:    movd %rax, %xmm2
+; SSE-NEXT:    shufpd {{.*#+}} xmm1 = xmm1[1,0]
+; SSE-NEXT:    cvttsd2si %xmm1, %rax
+; SSE-NEXT:    movd %rax, %xmm1
+; SSE-NEXT:    punpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm1[0]
+; SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm2[0,2,2,3]
+; SSE-NEXT:    cvttsd2si %xmm0, %rax
+; SSE-NEXT:    movd %rax, %xmm2
+; SSE-NEXT:    shufpd {{.*#+}} xmm0 = xmm0[1,0]
+; SSE-NEXT:    cvttsd2si %xmm0, %rax
+; SSE-NEXT:    movd %rax, %xmm0
+; SSE-NEXT:    punpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm0[0]
+; SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[0,2,2,3]
+; SSE-NEXT:    punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
+; SSE-NEXT:    retq
+;
+; AVX-LABEL: fptosi_4f64_to_4i32:
+; AVX:       # BB#0:
+; AVX-NEXT:    vcvttpd2dqy %ymm0, %xmm0
+; AVX-NEXT:    vzeroupper
+; AVX-NEXT:    retq
+  %cvt = fptosi <4 x double> %a to <4 x i32>
+  ret <4 x i32> %cvt
+}
+
+;
+; Double to Unsigned Integer
+;
+
+define <2 x i64> @fptoui_2f64_to_2i64(<2 x double> %a) {
+; SSE-LABEL: fptoui_2f64_to_2i64:
+; SSE:       # BB#0:
+; SSE-NEXT:    movsd {{.*#+}} xmm2 = mem[0],zero
+; SSE-NEXT:    movapd %xmm0, %xmm1
+; SSE-NEXT:    subsd %xmm2, %xmm1
+; SSE-NEXT:    cvttsd2si %xmm1, %rax
+; SSE-NEXT:    movabsq $-9223372036854775808, %rcx # imm = 0x8000000000000000
+; SSE-NEXT:    xorq %rcx, %rax
+; SSE-NEXT:    cvttsd2si %xmm0, %rdx
+; SSE-NEXT:    ucomisd %xmm2, %xmm0
+; SSE-NEXT:    cmovaeq %rax, %rdx
+; SSE-NEXT:    movd %rdx, %xmm1
+; SSE-NEXT:    shufpd {{.*#+}} xmm0 = xmm0[1,0]
+; SSE-NEXT:    movapd %xmm0, %xmm3
+; SSE-NEXT:    subsd %xmm2, %xmm3
+; SSE-NEXT:    cvttsd2si %xmm3, %rax
+; SSE-NEXT:    xorq %rcx, %rax
+; SSE-NEXT:    cvttsd2si %xmm0, %rcx
+; SSE-NEXT:    ucomisd %xmm2, %xmm0
+; SSE-NEXT:    cmovaeq %rax, %rcx
+; SSE-NEXT:    movd %rcx, %xmm0
+; SSE-NEXT:    punpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm0[0]
+; SSE-NEXT:    movdqa %xmm1, %xmm0
+; SSE-NEXT:    retq
+;
+; AVX-LABEL: fptoui_2f64_to_2i64:
+; AVX:       # BB#0:
+; AVX-NEXT:    vmovsd {{.*#+}} xmm1 = mem[0],zero
+; AVX-NEXT:    vsubsd %xmm1, %xmm0, %xmm2
+; AVX-NEXT:    vcvttsd2si %xmm2, %rax
+; AVX-NEXT:    movabsq $-9223372036854775808, %rcx # imm = 0x8000000000000000
+; AVX-NEXT:    xorq %rcx, %rax
+; AVX-NEXT:    vcvttsd2si %xmm0, %rdx
+; AVX-NEXT:    vucomisd %xmm1, %xmm0
+; AVX-NEXT:    cmovaeq %rax, %rdx
+; AVX-NEXT:    vmovq %rdx, %xmm2
+; AVX-NEXT:    vpermilpd {{.*#+}} xmm0 = xmm0[1,0]
+; AVX-NEXT:    vsubsd %xmm1, %xmm0, %xmm3
+; AVX-NEXT:    vcvttsd2si %xmm3, %rax
+; AVX-NEXT:    xorq %rcx, %rax
+; AVX-NEXT:    vcvttsd2si %xmm0, %rcx
+; AVX-NEXT:    vucomisd %xmm1, %xmm0
+; AVX-NEXT:    cmovaeq %rax, %rcx
+; AVX-NEXT:    vmovq %rcx, %xmm0
+; AVX-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm2[0],xmm0[0]
+; AVX-NEXT:    retq
+  %cvt = fptoui <2 x double> %a to <2 x i64>
+  ret <2 x i64> %cvt
+}
+
+define <4 x i32> @fptoui_2f64_to_2i32(<2 x double> %a) {
+; SSE-LABEL: fptoui_2f64_to_2i32:
+; SSE:       # BB#0:
+; SSE-NEXT:    movsd {{.*#+}} xmm1 = mem[0],zero
+; SSE-NEXT:    movapd %xmm0, %xmm2
+; SSE-NEXT:    subsd %xmm1, %xmm2
+; SSE-NEXT:    cvttsd2si %xmm2, %rax
+; SSE-NEXT:    movabsq $-9223372036854775808, %rcx # imm = 0x8000000000000000
+; SSE-NEXT:    xorq %rcx, %rax
+; SSE-NEXT:    cvttsd2si %xmm0, %rdx
+; SSE-NEXT:    ucomisd %xmm1, %xmm0
+; SSE-NEXT:    cmovaeq %rax, %rdx
+; SSE-NEXT:    movd %rdx, %xmm2
+; SSE-NEXT:    shufpd {{.*#+}} xmm0 = xmm0[1,0]
+; SSE-NEXT:    movapd %xmm0, %xmm3
+; SSE-NEXT:    subsd %xmm1, %xmm3
+; SSE-NEXT:    cvttsd2si %xmm3, %rax
+; SSE-NEXT:    xorq %rcx, %rax
+; SSE-NEXT:    cvttsd2si %xmm0, %rcx
+; SSE-NEXT:    ucomisd %xmm1, %xmm0
+; SSE-NEXT:    cmovaeq %rax, %rcx
+; SSE-NEXT:    movd %rcx, %xmm0
+; SSE-NEXT:    punpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm0[0]
+; SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[0,2,2,3]
+; SSE-NEXT:    retq
+;
+; AVX-LABEL: fptoui_2f64_to_2i32:
+; AVX:       # BB#0:
+; AVX-NEXT:    vmovsd {{.*#+}} xmm1 = mem[0],zero
+; AVX-NEXT:    vsubsd %xmm1, %xmm0, %xmm2
+; AVX-NEXT:    vcvttsd2si %xmm2, %rax
+; AVX-NEXT:    movabsq $-9223372036854775808, %rcx # imm = 0x8000000000000000
+; AVX-NEXT:    xorq %rcx, %rax
+; AVX-NEXT:    vcvttsd2si %xmm0, %rdx
+; AVX-NEXT:    vucomisd %xmm1, %xmm0
+; AVX-NEXT:    cmovaeq %rax, %rdx
+; AVX-NEXT:    vmovq %rdx, %xmm2
+; AVX-NEXT:    vpermilpd {{.*#+}} xmm0 = xmm0[1,0]
+; AVX-NEXT:    vsubsd %xmm1, %xmm0, %xmm3
+; AVX-NEXT:    vcvttsd2si %xmm3, %rax
+; AVX-NEXT:    xorq %rcx, %rax
+; AVX-NEXT:    vcvttsd2si %xmm0, %rcx
+; AVX-NEXT:    vucomisd %xmm1, %xmm0
+; AVX-NEXT:    cmovaeq %rax, %rcx
+; AVX-NEXT:    vmovq %rcx, %xmm0
+; AVX-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm2[0],xmm0[0]
+; AVX-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
+; AVX-NEXT:    retq
+  %cvt = fptoui <2 x double> %a to <2 x i32>
+  %ext = shufflevector <2 x i32> %cvt, <2 x i32> undef, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
+  ret <4 x i32> %ext
+}
+
+define <4 x i32> @fptoui_4f64_to_2i32(<2 x double> %a) {
+; SSE-LABEL: fptoui_4f64_to_2i32:
+; SSE:       # BB#0:
+; SSE-NEXT:    movsd {{.*#+}} xmm1 = mem[0],zero
+; SSE-NEXT:    movapd %xmm0, %xmm2
+; SSE-NEXT:    subsd %xmm1, %xmm2
+; SSE-NEXT:    cvttsd2si %xmm2, %rax
+; SSE-NEXT:    movabsq $-9223372036854775808, %rcx # imm = 0x8000000000000000
+; SSE-NEXT:    xorq %rcx, %rax
+; SSE-NEXT:    cvttsd2si %xmm0, %rdx
+; SSE-NEXT:    ucomisd %xmm1, %xmm0
+; SSE-NEXT:    cmovaeq %rax, %rdx
+; SSE-NEXT:    movd %rdx, %xmm2
+; SSE-NEXT:    shufpd {{.*#+}} xmm0 = xmm0[1,0]
+; SSE-NEXT:    movapd %xmm0, %xmm3
+; SSE-NEXT:    subsd %xmm1, %xmm3
+; SSE-NEXT:    cvttsd2si %xmm3, %rax
+; SSE-NEXT:    xorq %rcx, %rax
+; SSE-NEXT:    cvttsd2si %xmm0, %rdx
+; SSE-NEXT:    ucomisd %xmm1, %xmm0
+; SSE-NEXT:    cmovaeq %rax, %rdx
+; SSE-NEXT:    movd %rdx, %xmm0
+; SSE-NEXT:    punpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm0[0]
+; SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[0,2,2,3]
+; SSE-NEXT:    cvttsd2si %xmm0, %rax
+; SSE-NEXT:    xorq %rax, %rcx
+; SSE-NEXT:    ucomisd %xmm1, %xmm0
+; SSE-NEXT:    cmovbq %rax, %rcx
+; SSE-NEXT:    movd %rcx, %xmm1
+; SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,1,0,1]
+; SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3]
+; SSE-NEXT:    punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
+; SSE-NEXT:    retq
+;
+; AVX-LABEL: fptoui_4f64_to_2i32:
+; AVX:       # BB#0:
+; AVX-NEXT:    vpermilpd {{.*#+}} xmm1 = xmm0[1,0]
+; AVX-NEXT:    vcvttsd2si %xmm1, %rax
+; AVX-NEXT:    vcvttsd2si %xmm0, %rcx
+; AVX-NEXT:    vmovd %ecx, %xmm0
+; AVX-NEXT:    vpinsrd $1, %eax, %xmm0, %xmm0
+; AVX-NEXT:    vcvttsd2si %xmm0, %rax
+; AVX-NEXT:    vpinsrd $2, %eax, %xmm0, %xmm0
+; AVX-NEXT:    vpinsrd $3, %eax, %xmm0, %xmm0
+; AVX-NEXT:    retq
+  %ext = shufflevector <2 x double> %a, <2 x double> undef, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
+  %cvt = fptoui <4 x double> %ext to <4 x i32>
+  ret <4 x i32> %cvt
+}
+
+define <4 x i64> @fptoui_4f64_to_4i64(<4 x double> %a) {
+; SSE-LABEL: fptoui_4f64_to_4i64:
+; SSE:       # BB#0:
+; SSE-NEXT:    movapd %xmm0, %xmm2
+; SSE-NEXT:    movsd {{.*#+}} xmm3 = mem[0],zero
+; SSE-NEXT:    subsd %xmm3, %xmm0
+; SSE-NEXT:    cvttsd2si %xmm0, %rcx
+; SSE-NEXT:    movabsq $-9223372036854775808, %rax # imm = 0x8000000000000000
+; SSE-NEXT:    xorq %rax, %rcx
+; SSE-NEXT:    cvttsd2si %xmm2, %rdx
+; SSE-NEXT:    ucomisd %xmm3, %xmm2
+; SSE-NEXT:    cmovaeq %rcx, %rdx
+; SSE-NEXT:    movd %rdx, %xmm0
+; SSE-NEXT:    shufpd {{.*#+}} xmm2 = xmm2[1,0]
+; SSE-NEXT:    movapd %xmm2, %xmm4
+; SSE-NEXT:    subsd %xmm3, %xmm4
+; SSE-NEXT:    cvttsd2si %xmm4, %rcx
+; SSE-NEXT:    xorq %rax, %rcx
+; SSE-NEXT:    cvttsd2si %xmm2, %rdx
+; SSE-NEXT:    ucomisd %xmm3, %xmm2
+; SSE-NEXT:    cmovaeq %rcx, %rdx
+; SSE-NEXT:    movd %rdx, %xmm2
+; SSE-NEXT:    punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm2[0]
+; SSE-NEXT:    movapd %xmm1, %xmm2
+; SSE-NEXT:    subsd %xmm3, %xmm2
+; SSE-NEXT:    cvttsd2si %xmm2, %rcx
+; SSE-NEXT:    xorq %rax, %rcx
+; SSE-NEXT:    cvttsd2si %xmm1, %rdx
+; SSE-NEXT:    ucomisd %xmm3, %xmm1
+; SSE-NEXT:    cmovaeq %rcx, %rdx
+; SSE-NEXT:    movd %rdx, %xmm2
+; SSE-NEXT:    shufpd {{.*#+}} xmm1 = xmm1[1,0]
+; SSE-NEXT:    movapd %xmm1, %xmm4
+; SSE-NEXT:    subsd %xmm3, %xmm4
+; SSE-NEXT:    cvttsd2si %xmm4, %rcx
+; SSE-NEXT:    xorq %rax, %rcx
+; SSE-NEXT:    cvttsd2si %xmm1, %rax
+; SSE-NEXT:    ucomisd %xmm3, %xmm1
+; SSE-NEXT:    cmovaeq %rcx, %rax
+; SSE-NEXT:    movd %rax, %xmm1
+; SSE-NEXT:    punpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm1[0]
+; SSE-NEXT:    movdqa %xmm2, %xmm1
+; SSE-NEXT:    retq
+;
+; AVX-LABEL: fptoui_4f64_to_4i64:
+; AVX:       # BB#0:
+; AVX-NEXT:    vextractf128 $1, %ymm0, %xmm2
+; AVX-NEXT:    vmovsd {{.*#+}} xmm1 = mem[0],zero
+; AVX-NEXT:    vsubsd %xmm1, %xmm2, %xmm3
+; AVX-NEXT:    vcvttsd2si %xmm3, %rax
+; AVX-NEXT:    movabsq $-9223372036854775808, %rcx # imm = 0x8000000000000000
+; AVX-NEXT:    xorq %rcx, %rax
+; AVX-NEXT:    vcvttsd2si %xmm2, %rdx
+; AVX-NEXT:    vucomisd %xmm1, %xmm2
+; AVX-NEXT:    cmovaeq %rax, %rdx
+; AVX-NEXT:    vmovq %rdx, %xmm3
+; AVX-NEXT:    vpermilpd {{.*#+}} xmm2 = xmm2[1,0]
+; AVX-NEXT:    vsubsd %xmm1, %xmm2, %xmm4
+; AVX-NEXT:    vcvttsd2si %xmm4, %rax
+; AVX-NEXT:    xorq %rcx, %rax
+; AVX-NEXT:    vcvttsd2si %xmm2, %rdx
+; AVX-NEXT:    vucomisd %xmm1, %xmm2
+; AVX-NEXT:    cmovaeq %rax, %rdx
+; AVX-NEXT:    vmovq %rdx, %xmm2
+; AVX-NEXT:    vpunpcklqdq {{.*#+}} xmm2 = xmm3[0],xmm2[0]
+; AVX-NEXT:    vsubsd %xmm1, %xmm0, %xmm3
+; AVX-NEXT:    vcvttsd2si %xmm3, %rax
+; AVX-NEXT:    xorq %rcx, %rax
+; AVX-NEXT:    vcvttsd2si %xmm0, %rdx
+; AVX-NEXT:    vucomisd %xmm1, %xmm0
+; AVX-NEXT:    cmovaeq %rax, %rdx
+; AVX-NEXT:    vmovq %rdx, %xmm3
+; AVX-NEXT:    vpermilpd {{.*#+}} xmm0 = xmm0[1,0]
+; AVX-NEXT:    vsubsd %xmm1, %xmm0, %xmm4
+; AVX-NEXT:    vcvttsd2si %xmm4, %rax
+; AVX-NEXT:    xorq %rcx, %rax
+; AVX-NEXT:    vcvttsd2si %xmm0, %rcx
+; AVX-NEXT:    vucomisd %xmm1, %xmm0
+; AVX-NEXT:    cmovaeq %rax, %rcx
+; AVX-NEXT:    vmovq %rcx, %xmm0
+; AVX-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm3[0],xmm0[0]
+; AVX-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm0
+; AVX-NEXT:    retq
+  %cvt = fptoui <4 x double> %a to <4 x i64>
+  ret <4 x i64> %cvt
+}
+
+define <4 x i32> @fptoui_4f64_to_4i32(<4 x double> %a) {
+; SSE-LABEL: fptoui_4f64_to_4i32:
+; SSE:       # BB#0:
+; SSE-NEXT:    movsd {{.*#+}} xmm2 = mem[0],zero
+; SSE-NEXT:    movapd %xmm1, %xmm3
+; SSE-NEXT:    subsd %xmm2, %xmm3
+; SSE-NEXT:    cvttsd2si %xmm3, %rcx
+; SSE-NEXT:    movabsq $-9223372036854775808, %rax # imm = 0x8000000000000000
+; SSE-NEXT:    xorq %rax, %rcx
+; SSE-NEXT:    cvttsd2si %xmm1, %rdx
+; SSE-NEXT:    ucomisd %xmm2, %xmm1
+; SSE-NEXT:    cmovaeq %rcx, %rdx
+; SSE-NEXT:    movd %rdx, %xmm3
+; SSE-NEXT:    shufpd {{.*#+}} xmm1 = xmm1[1,0]
+; SSE-NEXT:    movapd %xmm1, %xmm4
+; SSE-NEXT:    subsd %xmm2, %xmm4
+; SSE-NEXT:    cvttsd2si %xmm4, %rcx
+; SSE-NEXT:    xorq %rax, %rcx
+; SSE-NEXT:    cvttsd2si %xmm1, %rdx
+; SSE-NEXT:    ucomisd %xmm2, %xmm1
+; SSE-NEXT:    cmovaeq %rcx, %rdx
+; SSE-NEXT:    movd %rdx, %xmm1
+; SSE-NEXT:    punpcklqdq {{.*#+}} xmm3 = xmm3[0],xmm1[0]
+; SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[0,2,2,3]
+; SSE-NEXT:    movapd %xmm0, %xmm3
+; SSE-NEXT:    subsd %xmm2, %xmm3
+; SSE-NEXT:    cvttsd2si %xmm3, %rcx
+; SSE-NEXT:    xorq %rax, %rcx
+; SSE-NEXT:    cvttsd2si %xmm0, %rdx
+; SSE-NEXT:    ucomisd %xmm2, %xmm0
+; SSE-NEXT:    cmovaeq %rcx, %rdx
+; SSE-NEXT:    movd %rdx, %xmm3
+; SSE-NEXT:    shufpd {{.*#+}} xmm0 = xmm0[1,0]
+; SSE-NEXT:    movapd %xmm0, %xmm4
+; SSE-NEXT:    subsd %xmm2, %xmm4
+; SSE-NEXT:    cvttsd2si %xmm4, %rcx
+; SSE-NEXT:    xorq %rax, %rcx
+; SSE-NEXT:    cvttsd2si %xmm0, %rax
+; SSE-NEXT:    ucomisd %xmm2, %xmm0
+; SSE-NEXT:    cmovaeq %rcx, %rax
+; SSE-NEXT:    movd %rax, %xmm0
+; SSE-NEXT:    punpcklqdq {{.*#+}} xmm3 = xmm3[0],xmm0[0]
+; SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm3[0,2,2,3]
+; SSE-NEXT:    punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
+; SSE-NEXT:    retq
+;
+; AVX-LABEL: fptoui_4f64_to_4i32:
+; AVX:       # BB#0:
+; AVX-NEXT:    vpermilpd {{.*#+}} xmm1 = xmm0[1,0]
+; AVX-NEXT:    vcvttsd2si %xmm1, %rax
+; AVX-NEXT:    vcvttsd2si %xmm0, %rcx
+; AVX-NEXT:    vmovd %ecx, %xmm1
+; AVX-NEXT:    vpinsrd $1, %eax, %xmm1, %xmm1
+; AVX-NEXT:    vextractf128 $1, %ymm0, %xmm0
+; AVX-NEXT:    vcvttsd2si %xmm0, %rax
+; AVX-NEXT:    vpinsrd $2, %eax, %xmm1, %xmm1
+; AVX-NEXT:    vpermilpd {{.*#+}} xmm0 = xmm0[1,0]
+; AVX-NEXT:    vcvttsd2si %xmm0, %rax
+; AVX-NEXT:    vpinsrd $3, %eax, %xmm1, %xmm0
+; AVX-NEXT:    vzeroupper
+; AVX-NEXT:    retq
+  %cvt = fptoui <4 x double> %a to <4 x i32>
+  ret <4 x i32> %cvt
+}
+
+;
+; Float to Signed Integer
+;
+
+define <4 x i32> @fptosi_4f32_to_4i32(<4 x float> %a) {
+; SSE-LABEL: fptosi_4f32_to_4i32:
+; SSE:       # BB#0:
+; SSE-NEXT:    cvttps2dq %xmm0, %xmm0
+; SSE-NEXT:    retq
+;
+; AVX-LABEL: fptosi_4f32_to_4i32:
+; AVX:       # BB#0:
+; AVX-NEXT:    vcvttps2dq %xmm0, %xmm0
+; AVX-NEXT:    retq
+  %cvt = fptosi <4 x float> %a to <4 x i32>
+  ret <4 x i32> %cvt
+}
+
+define <2 x i64> @fptosi_2f32_to_2i64(<4 x float> %a) {
+; SSE-LABEL: fptosi_2f32_to_2i64:
+; SSE:       # BB#0:
+; SSE-NEXT:    cvttss2si %xmm0, %rax
+; SSE-NEXT:    movd %rax, %xmm1
+; SSE-NEXT:    shufps {{.*#+}} xmm0 = xmm0[1,1,2,3]
+; SSE-NEXT:    cvttss2si %xmm0, %rax
+; SSE-NEXT:    movd %rax, %xmm0
+; SSE-NEXT:    punpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm0[0]
+; SSE-NEXT:    movdqa %xmm1, %xmm0
+; SSE-NEXT:    retq
+;
+; AVX-LABEL: fptosi_2f32_to_2i64:
+; AVX:       # BB#0:
+; AVX-NEXT:    vcvttss2si %xmm0, %rax
+; AVX-NEXT:    vmovq %rax, %xmm1
+; AVX-NEXT:    vmovshdup {{.*#+}} xmm0 = xmm0[1,1,3,3]
+; AVX-NEXT:    vcvttss2si %xmm0, %rax
+; AVX-NEXT:    vmovq %rax, %xmm0
+; AVX-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm1[0],xmm0[0]
+; AVX-NEXT:    retq
+  %shuf = shufflevector <4 x float> %a, <4 x float> undef, <2 x i32> <i32 0, i32 1>
+  %cvt = fptosi <2 x float> %shuf to <2 x i64>
+  ret <2 x i64> %cvt
+}
+
+define <2 x i64> @fptosi_4f32_to_2i64(<4 x float> %a) {
+; SSE-LABEL: fptosi_4f32_to_2i64:
+; SSE:       # BB#0:
+; SSE-NEXT:    cvttss2si %xmm0, %rax
+; SSE-NEXT:    movd %rax, %xmm1
+; SSE-NEXT:    shufps {{.*#+}} xmm0 = xmm0[1,1,2,3]
+; SSE-NEXT:    cvttss2si %xmm0, %rax
+; SSE-NEXT:    movd %rax, %xmm0
+; SSE-NEXT:    punpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm0[0]
+; SSE-NEXT:    movdqa %xmm1, %xmm0
+; SSE-NEXT:    retq
+;
+; AVX-LABEL: fptosi_4f32_to_2i64:
+; AVX:       # BB#0:
+; AVX-NEXT:    vmovshdup {{.*#+}} xmm1 = xmm0[1,1,3,3]
+; AVX-NEXT:    vcvttss2si %xmm1, %rax
+; AVX-NEXT:    vcvttss2si %xmm0, %rcx
+; AVX-NEXT:    vmovq %rcx, %xmm0
+; AVX-NEXT:    vmovq %rax, %xmm1
+; AVX-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
+; AVX-NEXT:    retq
+  %cvt = fptosi <4 x float> %a to <4 x i64>
+  %shuf = shufflevector <4 x i64> %cvt, <4 x i64> undef, <2 x i32> <i32 0, i32 1>
+  ret <2 x i64> %shuf
+}
+
+define <8 x i32> @fptosi_8f32_to_8i32(<8 x float> %a) {
+; SSE-LABEL: fptosi_8f32_to_8i32:
+; SSE:       # BB#0:
+; SSE-NEXT:    cvttps2dq %xmm0, %xmm0
+; SSE-NEXT:    cvttps2dq %xmm1, %xmm1
+; SSE-NEXT:    retq
+;
+; AVX-LABEL: fptosi_8f32_to_8i32:
+; AVX:       # BB#0:
+; AVX-NEXT:    vcvttps2dq %ymm0, %ymm0
+; AVX-NEXT:    retq
+  %cvt = fptosi <8 x float> %a to <8 x i32>
+  ret <8 x i32> %cvt
+}
+
+define <4 x i64> @fptosi_4f32_to_4i64(<8 x float> %a) {
+; SSE-LABEL: fptosi_4f32_to_4i64:
+; SSE:       # BB#0:
+; SSE-NEXT:    cvttss2si %xmm0, %rax
+; SSE-NEXT:    movd %rax, %xmm2
+; SSE-NEXT:    movaps %xmm0, %xmm1
+; SSE-NEXT:    shufps {{.*#+}} xmm1 = xmm1[1,1,2,3]
+; SSE-NEXT:    cvttss2si %xmm1, %rax
+; SSE-NEXT:    movd %rax, %xmm1
+; SSE-NEXT:    punpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm1[0]
+; SSE-NEXT:    movaps %xmm0, %xmm1
+; SSE-NEXT:    shufps {{.*#+}} xmm1 = xmm1[3,1,2,3]
+; SSE-NEXT:    cvttss2si %xmm1, %rax
+; SSE-NEXT:    movd %rax, %xmm3
+; SSE-NEXT:    shufpd {{.*#+}} xmm0 = xmm0[1,0]
+; SSE-NEXT:    cvttss2si %xmm0, %rax
+; SSE-NEXT:    movd %rax, %xmm1
+; SSE-NEXT:    punpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm3[0]
+; SSE-NEXT:    movdqa %xmm2, %xmm0
+; SSE-NEXT:    retq
+;
+; AVX-LABEL: fptosi_4f32_to_4i64:
+; AVX:       # BB#0:
+; AVX-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[3,1,2,3]
+; AVX-NEXT:    vcvttss2si %xmm1, %rax
+; AVX-NEXT:    vmovq %rax, %xmm1
+; AVX-NEXT:    vpermilpd {{.*#+}} xmm2 = xmm0[1,0]
+; AVX-NEXT:    vcvttss2si %xmm2, %rax
+; AVX-NEXT:    vmovq %rax, %xmm2
+; AVX-NEXT:    vpunpcklqdq {{.*#+}} xmm1 = xmm2[0],xmm1[0]
+; AVX-NEXT:    vcvttss2si %xmm0, %rax
+; AVX-NEXT:    vmovq %rax, %xmm2
+; AVX-NEXT:    vmovshdup {{.*#+}} xmm0 = xmm0[1,1,3,3]
+; AVX-NEXT:    vcvttss2si %xmm0, %rax
+; AVX-NEXT:    vmovq %rax, %xmm0
+; AVX-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm2[0],xmm0[0]
+; AVX-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
+; AVX-NEXT:    retq
+  %shuf = shufflevector <8 x float> %a, <8 x float> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %cvt = fptosi <4 x float> %shuf to <4 x i64>
+  ret <4 x i64> %cvt
+}
+
+define <4 x i64> @fptosi_8f32_to_4i64(<8 x float> %a) {
+; SSE-LABEL: fptosi_8f32_to_4i64:
+; SSE:       # BB#0:
+; SSE-NEXT:    cvttss2si %xmm0, %rax
+; SSE-NEXT:    movd %rax, %xmm2
+; SSE-NEXT:    movaps %xmm0, %xmm1
+; SSE-NEXT:    shufps {{.*#+}} xmm1 = xmm1[1,1,2,3]
+; SSE-NEXT:    cvttss2si %xmm1, %rax
+; SSE-NEXT:    movd %rax, %xmm1
+; SSE-NEXT:    punpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm1[0]
+; SSE-NEXT:    movaps %xmm0, %xmm1
+; SSE-NEXT:    shufps {{.*#+}} xmm1 = xmm1[3,1,2,3]
+; SSE-NEXT:    cvttss2si %xmm1, %rax
+; SSE-NEXT:    movd %rax, %xmm3
+; SSE-NEXT:    shufpd {{.*#+}} xmm0 = xmm0[1,0]
+; SSE-NEXT:    cvttss2si %xmm0, %rax
+; SSE-NEXT:    movd %rax, %xmm1
+; SSE-NEXT:    punpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm3[0]
+; SSE-NEXT:    movdqa %xmm2, %xmm0
+; SSE-NEXT:    retq
+;
+; AVX-LABEL: fptosi_8f32_to_4i64:
+; AVX:       # BB#0:
+; AVX-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[3,1,2,3]
+; AVX-NEXT:    vcvttss2si %xmm1, %rax
+; AVX-NEXT:    vmovq %rax, %xmm1
+; AVX-NEXT:    vpermilpd {{.*#+}} xmm2 = xmm0[1,0]
+; AVX-NEXT:    vcvttss2si %xmm2, %rax
+; AVX-NEXT:    vmovq %rax, %xmm2
+; AVX-NEXT:    vpunpcklqdq {{.*#+}} xmm1 = xmm2[0],xmm1[0]
+; AVX-NEXT:    vcvttss2si %xmm0, %rax
+; AVX-NEXT:    vmovq %rax, %xmm2
+; AVX-NEXT:    vmovshdup {{.*#+}} xmm0 = xmm0[1,1,3,3]
+; AVX-NEXT:    vcvttss2si %xmm0, %rax
+; AVX-NEXT:    vmovq %rax, %xmm0
+; AVX-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm2[0],xmm0[0]
+; AVX-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
+; AVX-NEXT:    retq
+  %cvt = fptosi <8 x float> %a to <8 x i64>
+  %shuf = shufflevector <8 x i64> %cvt, <8 x i64> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  ret <4 x i64> %shuf
+}
+
+;
+; Float to Unsigned Integer
+;
+
+define <4 x i32> @fptoui_4f32_to_4i32(<4 x float> %a) {
+; SSE-LABEL: fptoui_4f32_to_4i32:
+; SSE:       # BB#0:
+; SSE-NEXT:    movaps %xmm0, %xmm1
+; SSE-NEXT:    shufps {{.*#+}} xmm1 = xmm1[3,1,2,3]
+; SSE-NEXT:    cvttss2si %xmm1, %rax
+; SSE-NEXT:    movd %eax, %xmm1
+; SSE-NEXT:    movaps %xmm0, %xmm2
+; SSE-NEXT:    shufps {{.*#+}} xmm2 = xmm2[1,1,2,3]
+; SSE-NEXT:    cvttss2si %xmm2, %rax
+; SSE-NEXT:    movd %eax, %xmm2
+; SSE-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]
+; SSE-NEXT:    cvttss2si %xmm0, %rax
+; SSE-NEXT:    movd %eax, %xmm1
+; SSE-NEXT:    shufpd {{.*#+}} xmm0 = xmm0[1,0]
+; SSE-NEXT:    cvttss2si %xmm0, %rax
+; SSE-NEXT:    movd %eax, %xmm0
+; SSE-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
+; SSE-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1]
+; SSE-NEXT:    movdqa %xmm1, %xmm0
+; SSE-NEXT:    retq
+;
+; AVX-LABEL: fptoui_4f32_to_4i32:
+; AVX:       # BB#0:
+; AVX-NEXT:    vmovshdup {{.*#+}} xmm1 = xmm0[1,1,3,3]
+; AVX-NEXT:    vcvttss2si %xmm1, %rax
+; AVX-NEXT:    vcvttss2si %xmm0, %rcx
+; AVX-NEXT:    vmovd %ecx, %xmm1
+; AVX-NEXT:    vpinsrd $1, %eax, %xmm1, %xmm1
+; AVX-NEXT:    vpermilpd {{.*#+}} xmm2 = xmm0[1,0]
+; AVX-NEXT:    vcvttss2si %xmm2, %rax
+; AVX-NEXT:    vpinsrd $2, %eax, %xmm1, %xmm1
+; AVX-NEXT:    vpermilps {{.*#+}} xmm0 = xmm0[3,1,2,3]
+; AVX-NEXT:    vcvttss2si %xmm0, %rax
+; AVX-NEXT:    vpinsrd $3, %eax, %xmm1, %xmm0
+; AVX-NEXT:    retq
+  %cvt = fptoui <4 x float> %a to <4 x i32>
+  ret <4 x i32> %cvt
+}
+
+define <2 x i64> @fptoui_2f32_to_2i64(<4 x float> %a) {
+; SSE-LABEL: fptoui_2f32_to_2i64:
+; SSE:       # BB#0:
+; SSE-NEXT:    movss {{.*#+}} xmm2 = mem[0],zero,zero,zero
+; SSE-NEXT:    movaps %xmm0, %xmm1
+; SSE-NEXT:    subss %xmm2, %xmm1
+; SSE-NEXT:    cvttss2si %xmm1, %rax
+; SSE-NEXT:    movabsq $-9223372036854775808, %rcx # imm = 0x8000000000000000
+; SSE-NEXT:    xorq %rcx, %rax
+; SSE-NEXT:    cvttss2si %xmm0, %rdx
+; SSE-NEXT:    ucomiss %xmm2, %xmm0
+; SSE-NEXT:    cmovaeq %rax, %rdx
+; SSE-NEXT:    movd %rdx, %xmm1
+; SSE-NEXT:    shufps {{.*#+}} xmm0 = xmm0[1,1,2,3]
+; SSE-NEXT:    movaps %xmm0, %xmm3
+; SSE-NEXT:    subss %xmm2, %xmm3
+; SSE-NEXT:    cvttss2si %xmm3, %rax
+; SSE-NEXT:    xorq %rcx, %rax
+; SSE-NEXT:    cvttss2si %xmm0, %rcx
+; SSE-NEXT:    ucomiss %xmm2, %xmm0
+; SSE-NEXT:    cmovaeq %rax, %rcx
+; SSE-NEXT:    movd %rcx, %xmm0
+; SSE-NEXT:    punpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm0[0]
+; SSE-NEXT:    movdqa %xmm1, %xmm0
+; SSE-NEXT:    retq
+;
+; AVX-LABEL: fptoui_2f32_to_2i64:
+; AVX:       # BB#0:
+; AVX-NEXT:    vmovss {{.*#+}} xmm1 = mem[0],zero,zero,zero
+; AVX-NEXT:    vsubss %xmm1, %xmm0, %xmm2
+; AVX-NEXT:    vcvttss2si %xmm2, %rax
+; AVX-NEXT:    movabsq $-9223372036854775808, %rcx # imm = 0x8000000000000000
+; AVX-NEXT:    xorq %rcx, %rax
+; AVX-NEXT:    vcvttss2si %xmm0, %rdx
+; AVX-NEXT:    vucomiss %xmm1, %xmm0
+; AVX-NEXT:    cmovaeq %rax, %rdx
+; AVX-NEXT:    vmovq %rdx, %xmm2
+; AVX-NEXT:    vmovshdup {{.*#+}} xmm0 = xmm0[1,1,3,3]
+; AVX-NEXT:    vsubss %xmm1, %xmm0, %xmm3
+; AVX-NEXT:    vcvttss2si %xmm3, %rax
+; AVX-NEXT:    xorq %rcx, %rax
+; AVX-NEXT:    vcvttss2si %xmm0, %rcx
+; AVX-NEXT:    vucomiss %xmm1, %xmm0
+; AVX-NEXT:    cmovaeq %rax, %rcx
+; AVX-NEXT:    vmovq %rcx, %xmm0
+; AVX-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm2[0],xmm0[0]
+; AVX-NEXT:    retq
+  %shuf = shufflevector <4 x float> %a, <4 x float> undef, <2 x i32> <i32 0, i32 1>
+  %cvt = fptoui <2 x float> %shuf to <2 x i64>
+  ret <2 x i64> %cvt
+}
+
+define <2 x i64> @fptoui_4f32_to_2i64(<4 x float> %a) {
+; SSE-LABEL: fptoui_4f32_to_2i64:
+; SSE:       # BB#0:
+; SSE-NEXT:    movss {{.*#+}} xmm2 = mem[0],zero,zero,zero
+; SSE-NEXT:    movaps %xmm0, %xmm1
+; SSE-NEXT:    subss %xmm2, %xmm1
+; SSE-NEXT:    cvttss2si %xmm1, %rax
+; SSE-NEXT:    movabsq $-9223372036854775808, %rcx # imm = 0x8000000000000000
+; SSE-NEXT:    xorq %rcx, %rax
+; SSE-NEXT:    cvttss2si %xmm0, %rdx
+; SSE-NEXT:    ucomiss %xmm2, %xmm0
+; SSE-NEXT:    cmovaeq %rax, %rdx
+; SSE-NEXT:    movd %rdx, %xmm1
+; SSE-NEXT:    shufps {{.*#+}} xmm0 = xmm0[1,1,2,3]
+; SSE-NEXT:    movaps %xmm0, %xmm3
+; SSE-NEXT:    subss %xmm2, %xmm3
+; SSE-NEXT:    cvttss2si %xmm3, %rax
+; SSE-NEXT:    xorq %rcx, %rax
+; SSE-NEXT:    cvttss2si %xmm0, %rcx
+; SSE-NEXT:    ucomiss %xmm2, %xmm0
+; SSE-NEXT:    cmovaeq %rax, %rcx
+; SSE-NEXT:    movd %rcx, %xmm0
+; SSE-NEXT:    punpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm0[0]
+; SSE-NEXT:    movdqa %xmm1, %xmm0
+; SSE-NEXT:    retq
+;
+; AVX-LABEL: fptoui_4f32_to_2i64:
+; AVX:       # BB#0:
+; AVX-NEXT:    vmovshdup {{.*#+}} xmm1 = xmm0[1,1,3,3]
+; AVX-NEXT:    vmovss {{.*#+}} xmm2 = mem[0],zero,zero,zero
+; AVX-NEXT:    vsubss %xmm2, %xmm1, %xmm3
+; AVX-NEXT:    vcvttss2si %xmm3, %rax
+; AVX-NEXT:    movabsq $-9223372036854775808, %rcx # imm = 0x8000000000000000
+; AVX-NEXT:    xorq %rcx, %rax
+; AVX-NEXT:    vcvttss2si %xmm1, %rdx
+; AVX-NEXT:    vucomiss %xmm2, %xmm1
+; AVX-NEXT:    cmovaeq %rax, %rdx
+; AVX-NEXT:    vsubss %xmm2, %xmm0, %xmm1
+; AVX-NEXT:    vcvttss2si %xmm1, %rax
+; AVX-NEXT:    xorq %rcx, %rax
+; AVX-NEXT:    vcvttss2si %xmm0, %rcx
+; AVX-NEXT:    vucomiss %xmm2, %xmm0
+; AVX-NEXT:    cmovaeq %rax, %rcx
+; AVX-NEXT:    vmovq %rcx, %xmm0
+; AVX-NEXT:    vmovq %rdx, %xmm1
+; AVX-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
+; AVX-NEXT:    retq
+  %cvt = fptoui <4 x float> %a to <4 x i64>
+  %shuf = shufflevector <4 x i64> %cvt, <4 x i64> undef, <2 x i32> <i32 0, i32 1>
+  ret <2 x i64> %shuf
+}
+
+define <8 x i32> @fptoui_8f32_to_8i32(<8 x float> %a) {
+; SSE-LABEL: fptoui_8f32_to_8i32:
+; SSE:       # BB#0:
+; SSE-NEXT:    movaps %xmm0, %xmm2
+; SSE-NEXT:    shufps {{.*#+}} xmm0 = xmm0[3,1,2,3]
+; SSE-NEXT:    cvttss2si %xmm0, %rax
+; SSE-NEXT:    movd %eax, %xmm0
+; SSE-NEXT:    movaps %xmm2, %xmm3
+; SSE-NEXT:    shufps {{.*#+}} xmm3 = xmm3[1,1,2,3]
+; SSE-NEXT:    cvttss2si %xmm3, %rax
+; SSE-NEXT:    movd %eax, %xmm3
+; SSE-NEXT:    punpckldq {{.*#+}} xmm3 = xmm3[0],xmm0[0],xmm3[1],xmm0[1]
+; SSE-NEXT:    cvttss2si %xmm2, %rax
+; SSE-NEXT:    movd %eax, %xmm0
+; SSE-NEXT:    shufpd {{.*#+}} xmm2 = xmm2[1,0]
+; SSE-NEXT:    cvttss2si %xmm2, %rax
+; SSE-NEXT:    movd %eax, %xmm2
+; SSE-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1]
+; SSE-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1]
+; SSE-NEXT:    movaps %xmm1, %xmm2
+; SSE-NEXT:    shufps {{.*#+}} xmm2 = xmm2[3,1,2,3]
+; SSE-NEXT:    cvttss2si %xmm2, %rax
+; SSE-NEXT:    movd %eax, %xmm2
+; SSE-NEXT:    movaps %xmm1, %xmm3
+; SSE-NEXT:    shufps {{.*#+}} xmm3 = xmm3[1,1,2,3]
+; SSE-NEXT:    cvttss2si %xmm3, %rax
+; SSE-NEXT:    movd %eax, %xmm3
+; SSE-NEXT:    punpckldq {{.*#+}} xmm3 = xmm3[0],xmm2[0],xmm3[1],xmm2[1]
+; SSE-NEXT:    cvttss2si %xmm1, %rax
+; SSE-NEXT:    movd %eax, %xmm2
+; SSE-NEXT:    shufpd {{.*#+}} xmm1 = xmm1[1,0]
+; SSE-NEXT:    cvttss2si %xmm1, %rax
+; SSE-NEXT:    movd %eax, %xmm1
+; SSE-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]
+; SSE-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1]
+; SSE-NEXT:    movdqa %xmm2, %xmm1
+; SSE-NEXT:    retq
+;
+; AVX-LABEL: fptoui_8f32_to_8i32:
+; AVX:       # BB#0:
+; AVX-NEXT:    vextractf128 $1, %ymm0, %xmm1
+; AVX-NEXT:    vmovshdup {{.*#+}} xmm2 = xmm1[1,1,3,3]
+; AVX-NEXT:    vcvttss2si %xmm2, %rax
+; AVX-NEXT:    vcvttss2si %xmm1, %rcx
+; AVX-NEXT:    vmovd %ecx, %xmm2
+; AVX-NEXT:    vpinsrd $1, %eax, %xmm2, %xmm2
+; AVX-NEXT:    vpermilpd {{.*#+}} xmm3 = xmm1[1,0]
+; AVX-NEXT:    vcvttss2si %xmm3, %rax
+; AVX-NEXT:    vpinsrd $2, %eax, %xmm2, %xmm2
+; AVX-NEXT:    vpermilps {{.*#+}} xmm1 = xmm1[3,1,2,3]
+; AVX-NEXT:    vcvttss2si %xmm1, %rax
+; AVX-NEXT:    vpinsrd $3, %eax, %xmm2, %xmm1
+; AVX-NEXT:    vmovshdup {{.*#+}} xmm2 = xmm0[1,1,3,3]
+; AVX-NEXT:    vcvttss2si %xmm2, %rax
+; AVX-NEXT:    vcvttss2si %xmm0, %rcx
+; AVX-NEXT:    vmovd %ecx, %xmm2
+; AVX-NEXT:    vpinsrd $1, %eax, %xmm2, %xmm2
+; AVX-NEXT:    vpermilpd {{.*#+}} xmm3 = xmm0[1,0]
+; AVX-NEXT:    vcvttss2si %xmm3, %rax
+; AVX-NEXT:    vpinsrd $2, %eax, %xmm2, %xmm2
+; AVX-NEXT:    vpermilps {{.*#+}} xmm0 = xmm0[3,1,2,3]
+; AVX-NEXT:    vcvttss2si %xmm0, %rax
+; AVX-NEXT:    vpinsrd $3, %eax, %xmm2, %xmm0
+; AVX-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
+; AVX-NEXT:    retq
+  %cvt = fptoui <8 x float> %a to <8 x i32>
+  ret <8 x i32> %cvt
+}
+
+define <4 x i64> @fptoui_4f32_to_4i64(<8 x float> %a) {
+; SSE-LABEL: fptoui_4f32_to_4i64:
+; SSE:       # BB#0:
+; SSE-NEXT:    movss {{.*#+}} xmm1 = mem[0],zero,zero,zero
+; SSE-NEXT:    movaps %xmm0, %xmm2
+; SSE-NEXT:    subss %xmm1, %xmm2
+; SSE-NEXT:    cvttss2si %xmm2, %rcx
+; SSE-NEXT:    movabsq $-9223372036854775808, %rax # imm = 0x8000000000000000
+; SSE-NEXT:    xorq %rax, %rcx
+; SSE-NEXT:    cvttss2si %xmm0, %rdx
+; SSE-NEXT:    ucomiss %xmm1, %xmm0
+; SSE-NEXT:    cmovaeq %rcx, %rdx
+; SSE-NEXT:    movd %rdx, %xmm2
+; SSE-NEXT:    movaps %xmm0, %xmm3
+; SSE-NEXT:    shufps {{.*#+}} xmm3 = xmm3[1,1,2,3]
+; SSE-NEXT:    movaps %xmm3, %xmm4
+; SSE-NEXT:    subss %xmm1, %xmm4
+; SSE-NEXT:    cvttss2si %xmm4, %rcx
+; SSE-NEXT:    xorq %rax, %rcx
+; SSE-NEXT:    cvttss2si %xmm3, %rdx
+; SSE-NEXT:    ucomiss %xmm1, %xmm3
+; SSE-NEXT:    cmovaeq %rcx, %rdx
+; SSE-NEXT:    movd %rdx, %xmm3
+; SSE-NEXT:    punpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm3[0]
+; SSE-NEXT:    movaps %xmm0, %xmm3
+; SSE-NEXT:    shufps {{.*#+}} xmm3 = xmm3[3,1,2,3]
+; SSE-NEXT:    movaps %xmm3, %xmm4
+; SSE-NEXT:    subss %xmm1, %xmm4
+; SSE-NEXT:    cvttss2si %xmm4, %rcx
+; SSE-NEXT:    xorq %rax, %rcx
+; SSE-NEXT:    cvttss2si %xmm3, %rdx
+; SSE-NEXT:    ucomiss %xmm1, %xmm3
+; SSE-NEXT:    cmovaeq %rcx, %rdx
+; SSE-NEXT:    movd %rdx, %xmm3
+; SSE-NEXT:    shufpd {{.*#+}} xmm0 = xmm0[1,0]
+; SSE-NEXT:    movapd %xmm0, %xmm4
+; SSE-NEXT:    subss %xmm1, %xmm4
+; SSE-NEXT:    cvttss2si %xmm4, %rcx
+; SSE-NEXT:    xorq %rax, %rcx
+; SSE-NEXT:    cvttss2si %xmm0, %rax
+; SSE-NEXT:    ucomiss %xmm1, %xmm0
+; SSE-NEXT:    cmovaeq %rcx, %rax
+; SSE-NEXT:    movd %rax, %xmm1
+; SSE-NEXT:    punpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm3[0]
+; SSE-NEXT:    movdqa %xmm2, %xmm0
+; SSE-NEXT:    retq
+;
+; AVX-LABEL: fptoui_4f32_to_4i64:
+; AVX:       # BB#0:
+; AVX-NEXT:    vpermilps {{.*#+}} xmm2 = xmm0[3,1,2,3]
+; AVX-NEXT:    vmovss {{.*#+}} xmm1 = mem[0],zero,zero,zero
+; AVX-NEXT:    vsubss %xmm1, %xmm2, %xmm3
+; AVX-NEXT:    vcvttss2si %xmm3, %rax
+; AVX-NEXT:    movabsq $-9223372036854775808, %rcx # imm = 0x8000000000000000
+; AVX-NEXT:    xorq %rcx, %rax
+; AVX-NEXT:    vcvttss2si %xmm2, %rdx
+; AVX-NEXT:    vucomiss %xmm1, %xmm2
+; AVX-NEXT:    cmovaeq %rax, %rdx
+; AVX-NEXT:    vmovq %rdx, %xmm2
+; AVX-NEXT:    vpermilpd {{.*#+}} xmm3 = xmm0[1,0]
+; AVX-NEXT:    vsubss %xmm1, %xmm3, %xmm4
+; AVX-NEXT:    vcvttss2si %xmm4, %rax
+; AVX-NEXT:    xorq %rcx, %rax
+; AVX-NEXT:    vcvttss2si %xmm3, %rdx
+; AVX-NEXT:    vucomiss %xmm1, %xmm3
+; AVX-NEXT:    cmovaeq %rax, %rdx
+; AVX-NEXT:    vmovq %rdx, %xmm3
+; AVX-NEXT:    vpunpcklqdq {{.*#+}} xmm2 = xmm3[0],xmm2[0]
+; AVX-NEXT:    vsubss %xmm1, %xmm0, %xmm3
+; AVX-NEXT:    vcvttss2si %xmm3, %rax
+; AVX-NEXT:    xorq %rcx, %rax
+; AVX-NEXT:    vcvttss2si %xmm0, %rdx
+; AVX-NEXT:    vucomiss %xmm1, %xmm0
+; AVX-NEXT:    cmovaeq %rax, %rdx
+; AVX-NEXT:    vmovq %rdx, %xmm3
+; AVX-NEXT:    vmovshdup {{.*#+}} xmm0 = xmm0[1,1,3,3]
+; AVX-NEXT:    vsubss %xmm1, %xmm0, %xmm4
+; AVX-NEXT:    vcvttss2si %xmm4, %rax
+; AVX-NEXT:    xorq %rcx, %rax
+; AVX-NEXT:    vcvttss2si %xmm0, %rcx
+; AVX-NEXT:    vucomiss %xmm1, %xmm0
+; AVX-NEXT:    cmovaeq %rax, %rcx
+; AVX-NEXT:    vmovq %rcx, %xmm0
+; AVX-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm3[0],xmm0[0]
+; AVX-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm0
+; AVX-NEXT:    retq
+  %shuf = shufflevector <8 x float> %a, <8 x float> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %cvt = fptoui <4 x float> %shuf to <4 x i64>
+  ret <4 x i64> %cvt
+}
+
+define <4 x i64> @fptoui_8f32_to_4i64(<8 x float> %a) {
+; SSE-LABEL: fptoui_8f32_to_4i64:
+; SSE:       # BB#0:
+; SSE-NEXT:    movss {{.*#+}} xmm1 = mem[0],zero,zero,zero
+; SSE-NEXT:    movaps %xmm0, %xmm2
+; SSE-NEXT:    subss %xmm1, %xmm2
+; SSE-NEXT:    cvttss2si %xmm2, %rcx
+; SSE-NEXT:    movabsq $-9223372036854775808, %rax # imm = 0x8000000000000000
+; SSE-NEXT:    xorq %rax, %rcx
+; SSE-NEXT:    cvttss2si %xmm0, %rdx
+; SSE-NEXT:    ucomiss %xmm1, %xmm0
+; SSE-NEXT:    cmovaeq %rcx, %rdx
+; SSE-NEXT:    movd %rdx, %xmm2
+; SSE-NEXT:    movaps %xmm0, %xmm3
+; SSE-NEXT:    shufps {{.*#+}} xmm3 = xmm3[1,1,2,3]
+; SSE-NEXT:    movaps %xmm3, %xmm4
+; SSE-NEXT:    subss %xmm1, %xmm4
+; SSE-NEXT:    cvttss2si %xmm4, %rcx
+; SSE-NEXT:    xorq %rax, %rcx
+; SSE-NEXT:    cvttss2si %xmm3, %rdx
+; SSE-NEXT:    ucomiss %xmm1, %xmm3
+; SSE-NEXT:    cmovaeq %rcx, %rdx
+; SSE-NEXT:    movd %rdx, %xmm3
+; SSE-NEXT:    punpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm3[0]
+; SSE-NEXT:    movaps %xmm0, %xmm3
+; SSE-NEXT:    shufps {{.*#+}} xmm3 = xmm3[3,1,2,3]
+; SSE-NEXT:    movaps %xmm3, %xmm4
+; SSE-NEXT:    subss %xmm1, %xmm4
+; SSE-NEXT:    cvttss2si %xmm4, %rcx
+; SSE-NEXT:    xorq %rax, %rcx
+; SSE-NEXT:    cvttss2si %xmm3, %rdx
+; SSE-NEXT:    ucomiss %xmm1, %xmm3
+; SSE-NEXT:    cmovaeq %rcx, %rdx
+; SSE-NEXT:    movd %rdx, %xmm3
+; SSE-NEXT:    shufpd {{.*#+}} xmm0 = xmm0[1,0]
+; SSE-NEXT:    movapd %xmm0, %xmm4
+; SSE-NEXT:    subss %xmm1, %xmm4
+; SSE-NEXT:    cvttss2si %xmm4, %rcx
+; SSE-NEXT:    xorq %rax, %rcx
+; SSE-NEXT:    cvttss2si %xmm0, %rax
+; SSE-NEXT:    ucomiss %xmm1, %xmm0
+; SSE-NEXT:    cmovaeq %rcx, %rax
+; SSE-NEXT:    movd %rax, %xmm1
+; SSE-NEXT:    punpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm3[0]
+; SSE-NEXT:    movdqa %xmm2, %xmm0
+; SSE-NEXT:    retq
+;
+; AVX-LABEL: fptoui_8f32_to_4i64:
+; AVX:       # BB#0:
+; AVX-NEXT:    vpermilps {{.*#+}} xmm2 = xmm0[3,1,2,3]
+; AVX-NEXT:    vmovss {{.*#+}} xmm1 = mem[0],zero,zero,zero
+; AVX-NEXT:    vsubss %xmm1, %xmm2, %xmm3
+; AVX-NEXT:    vcvttss2si %xmm3, %rax
+; AVX-NEXT:    movabsq $-9223372036854775808, %rcx # imm = 0x8000000000000000
+; AVX-NEXT:    xorq %rcx, %rax
+; AVX-NEXT:    vcvttss2si %xmm2, %rdx
+; AVX-NEXT:    vucomiss %xmm1, %xmm2
+; AVX-NEXT:    cmovaeq %rax, %rdx
+; AVX-NEXT:    vmovq %rdx, %xmm2
+; AVX-NEXT:    vpermilpd {{.*#+}} xmm3 = xmm0[1,0]
+; AVX-NEXT:    vsubss %xmm1, %xmm3, %xmm4
+; AVX-NEXT:    vcvttss2si %xmm4, %rax
+; AVX-NEXT:    xorq %rcx, %rax
+; AVX-NEXT:    vcvttss2si %xmm3, %rdx
+; AVX-NEXT:    vucomiss %xmm1, %xmm3
+; AVX-NEXT:    cmovaeq %rax, %rdx
+; AVX-NEXT:    vmovq %rdx, %xmm3
+; AVX-NEXT:    vpunpcklqdq {{.*#+}} xmm2 = xmm3[0],xmm2[0]
+; AVX-NEXT:    vsubss %xmm1, %xmm0, %xmm3
+; AVX-NEXT:    vcvttss2si %xmm3, %rax
+; AVX-NEXT:    xorq %rcx, %rax
+; AVX-NEXT:    vcvttss2si %xmm0, %rdx
+; AVX-NEXT:    vucomiss %xmm1, %xmm0
+; AVX-NEXT:    cmovaeq %rax, %rdx
+; AVX-NEXT:    vmovq %rdx, %xmm3
+; AVX-NEXT:    vmovshdup {{.*#+}} xmm0 = xmm0[1,1,3,3]
+; AVX-NEXT:    vsubss %xmm1, %xmm0, %xmm4
+; AVX-NEXT:    vcvttss2si %xmm4, %rax
+; AVX-NEXT:    xorq %rcx, %rax
+; AVX-NEXT:    vcvttss2si %xmm0, %rcx
+; AVX-NEXT:    vucomiss %xmm1, %xmm0
+; AVX-NEXT:    cmovaeq %rax, %rcx
+; AVX-NEXT:    vmovq %rcx, %xmm0
+; AVX-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm3[0],xmm0[0]
+; AVX-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm0
+; AVX-NEXT:    retq
+  %cvt = fptoui <8 x float> %a to <8 x i64>
+  %shuf = shufflevector <8 x i64> %cvt, <8 x i64> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  ret <4 x i64> %shuf
+}
+
+;
+; Constant Folding
+;
+
+define <2 x i64> @fptosi_2f64_to_2i64_const() {
+; SSE-LABEL: fptosi_2f64_to_2i64_const:
+; SSE:       # BB#0:
+; SSE-NEXT:    movaps {{.*#+}} xmm0 = [1,18446744073709551615]
+; SSE-NEXT:    retq
+;
+; AVX-LABEL: fptosi_2f64_to_2i64_const:
+; AVX:       # BB#0:
+; AVX-NEXT:    vmovaps {{.*#+}} xmm0 = [1,18446744073709551615]
+; AVX-NEXT:    retq
+  %cvt = fptosi <2 x double> <double 1.0, double -1.0> to <2 x i64>
+  ret <2 x i64> %cvt
+}
+
+define <4 x i32> @fptosi_2f64_to_2i32_const() {
+; SSE-LABEL: fptosi_2f64_to_2i32_const:
+; SSE:       # BB#0:
+; SSE-NEXT:    movaps {{.*#+}} xmm0 = <4294967295,1,u,u>
+; SSE-NEXT:    retq
+;
+; AVX-LABEL: fptosi_2f64_to_2i32_const:
+; AVX:       # BB#0:
+; AVX-NEXT:    vmovaps {{.*#+}} xmm0 = <4294967295,1,u,u>
+; AVX-NEXT:    retq
+  %cvt = fptosi <2 x double> <double -1.0, double 1.0> to <2 x i32>
+  %ext = shufflevector <2 x i32> %cvt, <2 x i32> undef, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
+  ret <4 x i32> %ext
+}
+
+define <4 x i64> @fptosi_4f64_to_4i64_const() {
+; SSE-LABEL: fptosi_4f64_to_4i64_const:
+; SSE:       # BB#0:
+; SSE-NEXT:    movaps {{.*#+}} xmm0 = [1,18446744073709551615]
+; SSE-NEXT:    movaps {{.*#+}} xmm1 = [2,18446744073709551613]
+; SSE-NEXT:    retq
+;
+; AVX-LABEL: fptosi_4f64_to_4i64_const:
+; AVX:       # BB#0:
+; AVX-NEXT:    vmovaps {{.*#+}} ymm0 = [1,18446744073709551615,2,18446744073709551613]
+; AVX-NEXT:    retq
+  %cvt = fptosi <4 x double> <double 1.0, double -1.0, double 2.0, double -3.0> to <4 x i64>
+  ret <4 x i64> %cvt
+}
+
+define <4 x i32> @fptosi_4f64_to_4i32_const() {
+; SSE-LABEL: fptosi_4f64_to_4i32_const:
+; SSE:       # BB#0:
+; SSE-NEXT:    movaps {{.*#+}} xmm0 = [4294967295,1,4294967294,3]
+; SSE-NEXT:    retq
+;
+; AVX-LABEL: fptosi_4f64_to_4i32_const:
+; AVX:       # BB#0:
+; AVX-NEXT:    vmovaps {{.*#+}} xmm0 = [4294967295,1,4294967294,3]
+; AVX-NEXT:    retq
+  %cvt = fptosi <4 x double> <double -1.0, double 1.0, double -2.0, double 3.0> to <4 x i32>
+  ret <4 x i32> %cvt
+}
+
+define <2 x i64> @fptoui_2f64_to_2i64_const() {
+; SSE-LABEL: fptoui_2f64_to_2i64_const:
+; SSE:       # BB#0:
+; SSE-NEXT:    movaps {{.*#+}} xmm0 = [2,4]
+; SSE-NEXT:    retq
+;
+; AVX-LABEL: fptoui_2f64_to_2i64_const:
+; AVX:       # BB#0:
+; AVX-NEXT:    vmovaps {{.*#+}} xmm0 = [2,4]
+; AVX-NEXT:    retq
+  %cvt = fptoui <2 x double> <double 2.0, double 4.0> to <2 x i64>
+  ret <2 x i64> %cvt
+}
+
+define <4 x i32> @fptoui_2f64_to_2i32_const(<2 x double> %a) {
+; SSE-LABEL: fptoui_2f64_to_2i32_const:
+; SSE:       # BB#0:
+; SSE-NEXT:    movaps {{.*#+}} xmm0 = <2,4,u,u>
+; SSE-NEXT:    retq
+;
+; AVX-LABEL: fptoui_2f64_to_2i32_const:
+; AVX:       # BB#0:
+; AVX-NEXT:    vmovaps {{.*#+}} xmm0 = <2,4,u,u>
+; AVX-NEXT:    retq
+  %cvt = fptoui <2 x double> <double 2.0, double 4.0> to <2 x i32>
+  %ext = shufflevector <2 x i32> %cvt, <2 x i32> undef, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
+  ret <4 x i32> %ext
+}
+
+define <4 x i64> @fptoui_4f64_to_4i64_const(<4 x double> %a) {
+; SSE-LABEL: fptoui_4f64_to_4i64_const:
+; SSE:       # BB#0:
+; SSE-NEXT:    movaps {{.*#+}} xmm0 = [2,4]
+; SSE-NEXT:    movaps {{.*#+}} xmm1 = [6,8]
+; SSE-NEXT:    retq
+;
+; AVX-LABEL: fptoui_4f64_to_4i64_const:
+; AVX:       # BB#0:
+; AVX-NEXT:    vmovaps {{.*#+}} ymm0 = [2,4,6,8]
+; AVX-NEXT:    retq
+  %cvt = fptoui <4 x double> <double 2.0, double 4.0, double 6.0, double 8.0> to <4 x i64>
+  ret <4 x i64> %cvt
+}
+
+define <4 x i32> @fptoui_4f64_to_4i32_const(<4 x double> %a) {
+; SSE-LABEL: fptoui_4f64_to_4i32_const:
+; SSE:       # BB#0:
+; SSE-NEXT:    movaps {{.*#+}} xmm0 = [2,4,6,8]
+; SSE-NEXT:    retq
+;
+; AVX-LABEL: fptoui_4f64_to_4i32_const:
+; AVX:       # BB#0:
+; AVX-NEXT:    vmovaps {{.*#+}} xmm0 = [2,4,6,8]
+; AVX-NEXT:    retq
+  %cvt = fptoui <4 x double> <double 2.0, double 4.0, double 6.0, double 8.0> to <4 x i32>
+  ret <4 x i32> %cvt
+}
+
+define <4 x i32> @fptosi_4f32_to_4i32_const() {
+; SSE-LABEL: fptosi_4f32_to_4i32_const:
+; SSE:       # BB#0:
+; SSE-NEXT:    movaps {{.*#+}} xmm0 = [1,4294967295,2,3]
+; SSE-NEXT:    retq
+;
+; AVX-LABEL: fptosi_4f32_to_4i32_const:
+; AVX:       # BB#0:
+; AVX-NEXT:    vmovaps {{.*#+}} xmm0 = [1,4294967295,2,3]
+; AVX-NEXT:    retq
+  %cvt = fptosi <4 x float> <float 1.0, float -1.0, float 2.0, float 3.0> to <4 x i32>
+  ret <4 x i32> %cvt
+}
+
+define <4 x i64> @fptosi_4f32_to_4i64_const() {
+; SSE-LABEL: fptosi_4f32_to_4i64_const:
+; SSE:       # BB#0:
+; SSE-NEXT:    movaps {{.*#+}} xmm0 = [1,18446744073709551615]
+; SSE-NEXT:    movaps {{.*#+}} xmm1 = [2,3]
+; SSE-NEXT:    retq
+;
+; AVX-LABEL: fptosi_4f32_to_4i64_const:
+; AVX:       # BB#0:
+; AVX-NEXT:    vmovaps {{.*#+}} ymm0 = [1,18446744073709551615,2,3]
+; AVX-NEXT:    retq
+  %cvt = fptosi <4 x float> <float 1.0, float -1.0, float 2.0, float 3.0> to <4 x i64>
+  ret <4 x i64> %cvt
+}
+
+define <8 x i32> @fptosi_8f32_to_8i32_const(<8 x float> %a) {
+; SSE-LABEL: fptosi_8f32_to_8i32_const:
+; SSE:       # BB#0:
+; SSE-NEXT:    movaps {{.*#+}} xmm0 = [1,4294967295,2,3]
+; SSE-NEXT:    movaps {{.*#+}} xmm1 = [6,4294967288,2,4294967295]
+; SSE-NEXT:    retq
+;
+; AVX-LABEL: fptosi_8f32_to_8i32_const:
+; AVX:       # BB#0:
+; AVX-NEXT:    vmovaps {{.*#+}} ymm0 = [1,4294967295,2,3,6,4294967288,2,4294967295]
+; AVX-NEXT:    retq
+  %cvt = fptosi <8 x float> <float 1.0, float -1.0, float 2.0, float 3.0, float 6.0, float -8.0, float 2.0, float -1.0> to <8 x i32>
+  ret <8 x i32> %cvt
+}
+
+define <4 x i32> @fptoui_4f32_to_4i32_const(<4 x float> %a) {
+; SSE-LABEL: fptoui_4f32_to_4i32_const:
+; SSE:       # BB#0:
+; SSE-NEXT:    movaps {{.*#+}} xmm0 = [1,2,4,6]
+; SSE-NEXT:    retq
+;
+; AVX-LABEL: fptoui_4f32_to_4i32_const:
+; AVX:       # BB#0:
+; AVX-NEXT:    vmovaps {{.*#+}} xmm0 = [1,2,4,6]
+; AVX-NEXT:    retq
+  %cvt = fptoui <4 x float> <float 1.0, float 2.0, float 4.0, float 6.0> to <4 x i32>
+  ret <4 x i32> %cvt
+}
+
+define <4 x i64> @fptoui_4f32_to_4i64_const() {
+; SSE-LABEL: fptoui_4f32_to_4i64_const:
+; SSE:       # BB#0:
+; SSE-NEXT:    movaps {{.*#+}} xmm0 = [1,2]
+; SSE-NEXT:    movaps {{.*#+}} xmm1 = [4,8]
+; SSE-NEXT:    retq
+;
+; AVX-LABEL: fptoui_4f32_to_4i64_const:
+; AVX:       # BB#0:
+; AVX-NEXT:    vmovaps {{.*#+}} ymm0 = [1,2,4,8]
+; AVX-NEXT:    retq
+  %cvt = fptoui <4 x float> <float 1.0, float 2.0, float 4.0, float 8.0> to <4 x i64>
+  ret <4 x i64> %cvt
+}
+
+define <8 x i32> @fptoui_8f32_to_8i32_const(<8 x float> %a) {
+; SSE-LABEL: fptoui_8f32_to_8i32_const:
+; SSE:       # BB#0:
+; SSE-NEXT:    movaps {{.*#+}} xmm0 = [1,2,4,6]
+; SSE-NEXT:    movaps {{.*#+}} xmm1 = [8,6,4,1]
+; SSE-NEXT:    retq
+;
+; AVX-LABEL: fptoui_8f32_to_8i32_const:
+; AVX:       # BB#0:
+; AVX-NEXT:    vmovaps {{.*#+}} ymm0 = [1,2,4,6,8,6,4,1]
+; AVX-NEXT:    retq
+  %cvt = fptoui <8 x float> <float 1.0, float 2.0, float 4.0, float 6.0, float 8.0, float 6.0, float 4.0, float 1.0> to <8 x i32>
+  ret <8 x i32> %cvt
+}
diff --git a/test/CodeGen/X86/vec_int_to_fp.ll b/test/CodeGen/X86/vec_int_to_fp.ll
new file mode 100644
index 00000000000..a8a3206a49f
--- /dev/null
+++ b/test/CodeGen/X86/vec_int_to_fp.ll
@@ -0,0 +1,1873 @@
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+sse2 | FileCheck %s --check-prefix=ALL --check-prefix=SSE --check-prefix=SSE2
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx | FileCheck %s --check-prefix=ALL --check-prefix=AVX --check-prefix=AVX1
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx2 | FileCheck %s --check-prefix=ALL --check-prefix=AVX --check-prefix=AVX2
+;
+; 32-bit tests to make sure we're not doing anything stupid.
+; RUN: llc < %s -mtriple=i686-unknown-unknown
+; RUN: llc < %s -mtriple=i686-unknown-unknown -mattr=+sse
+; RUN: llc < %s -mtriple=i686-unknown-unknown -mattr=+sse2
+
+;
+; Signed Integer to Double
+;
+
+define <2 x double> @sitofp_2i64_to_2f64(<2 x i64> %a) {
+; SSE-LABEL: sitofp_2i64_to_2f64:
+; SSE:       # BB#0:
+; SSE-NEXT:    movd %xmm0, %rax
+; SSE-NEXT:    cvtsi2sdq %rax, %xmm1
+; SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
+; SSE-NEXT:    movd %xmm0, %rax
+; SSE-NEXT:    xorps %xmm0, %xmm0
+; SSE-NEXT:    cvtsi2sdq %rax, %xmm0
+; SSE-NEXT:    unpcklpd {{.*#+}} xmm1 = xmm1[0],xmm0[0]
+; SSE-NEXT:    movapd %xmm1, %xmm0
+; SSE-NEXT:    retq
+;
+; AVX-LABEL: sitofp_2i64_to_2f64:
+; AVX:       # BB#0:
+; AVX-NEXT:    vpextrq $1, %xmm0, %rax
+; AVX-NEXT:    vcvtsi2sdq %rax, %xmm0, %xmm1
+; AVX-NEXT:    vmovq %xmm0, %rax
+; AVX-NEXT:    vxorps %xmm0, %xmm0, %xmm0
+; AVX-NEXT:    vcvtsi2sdq %rax, %xmm0, %xmm0
+; AVX-NEXT:    vunpcklpd {{.*#+}} xmm0 = xmm0[0],xmm1[0]
+; AVX-NEXT:    retq
+  %cvt = sitofp <2 x i64> %a to <2 x double>
+  ret <2 x double> %cvt
+}
+
+define <2 x double> @sitofp_2i32_to_2f64(<4 x i32> %a) {
+; SSE-LABEL: sitofp_2i32_to_2f64:
+; SSE:       # BB#0:
+; SSE-NEXT:    cvtdq2pd %xmm0, %xmm0
+; SSE-NEXT:    retq
+;
+; AVX-LABEL: sitofp_2i32_to_2f64:
+; AVX:       # BB#0:
+; AVX-NEXT:    vcvtdq2pd %xmm0, %xmm0
+; AVX-NEXT:    retq
+  %shuf = shufflevector <4 x i32> %a, <4 x i32> undef, <2 x i32> <i32 0, i32 1>
+  %cvt = sitofp <2 x i32> %shuf to <2 x double>
+  ret <2 x double> %cvt
+}
+
+define <2 x double> @sitofp_4i32_to_2f64(<4 x i32> %a) {
+; SSE-LABEL: sitofp_4i32_to_2f64:
+; SSE:       # BB#0:
+; SSE-NEXT:    cvtdq2pd %xmm0, %xmm0
+; SSE-NEXT:    retq
+;
+; AVX-LABEL: sitofp_4i32_to_2f64:
+; AVX:       # BB#0:
+; AVX-NEXT:    vcvtdq2pd %xmm0, %ymm0
+; AVX-NEXT:    vzeroupper
+; AVX-NEXT:    retq
+  %cvt = sitofp <4 x i32> %a to <4 x double>
+  %shuf = shufflevector <4 x double> %cvt, <4 x double> undef, <2 x i32> <i32 0, i32 1>
+  ret <2 x double> %shuf
+}
+
+define <2 x double> @sitofp_2i16_to_2f64(<8 x i16> %a) {
+; SSE-LABEL: sitofp_2i16_to_2f64:
+; SSE:       # BB#0:
+; SSE-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3]
+; SSE-NEXT:    psrad $16, %xmm0
+; SSE-NEXT:    cvtdq2pd %xmm0, %xmm0
+; SSE-NEXT:    retq
+;
+; AVX-LABEL: sitofp_2i16_to_2f64:
+; AVX:       # BB#0:
+; AVX-NEXT:    vpmovsxwd %xmm0, %xmm0
+; AVX-NEXT:    vcvtdq2pd %xmm0, %xmm0
+; AVX-NEXT:    retq
+  %shuf = shufflevector <8 x i16> %a, <8 x i16> undef, <2 x i32> <i32 0, i32 1>
+  %cvt = sitofp <2 x i16> %shuf to <2 x double>
+  ret <2 x double> %cvt
+}
+
+define <2 x double> @sitofp_8i16_to_2f64(<8 x i16> %a) {
+; SSE-LABEL: sitofp_8i16_to_2f64:
+; SSE:       # BB#0:
+; SSE-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3]
+; SSE-NEXT:    psrad $16, %xmm0
+; SSE-NEXT:    cvtdq2pd %xmm0, %xmm0
+; SSE-NEXT:    retq
+;
+; AVX1-LABEL: sitofp_8i16_to_2f64:
+; AVX1:       # BB#0:
+; AVX1-NEXT:    vpmovsxwd %xmm0, %xmm0
+; AVX1-NEXT:    vcvtdq2pd %xmm0, %ymm0
+; AVX1-NEXT:    vzeroupper
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: sitofp_8i16_to_2f64:
+; AVX2:       # BB#0:
+; AVX2-NEXT:    vpmovsxwd %xmm0, %ymm0
+; AVX2-NEXT:    vcvtdq2pd %xmm0, %ymm0
+; AVX2-NEXT:    vzeroupper
+; AVX2-NEXT:    retq
+  %cvt = sitofp <8 x i16> %a to <8 x double>
+  %shuf = shufflevector <8 x double> %cvt, <8 x double> undef, <2 x i32> <i32 0, i32 1>
+  ret <2 x double> %shuf
+}
+
+define <2 x double> @sitofp_2i8_to_2f64(<16 x i8> %a) {
+; SSE-LABEL: sitofp_2i8_to_2f64:
+; SSE:       # BB#0:
+; SSE-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
+; SSE-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3]
+; SSE-NEXT:    psrad $24, %xmm0
+; SSE-NEXT:    cvtdq2pd %xmm0, %xmm0
+; SSE-NEXT:    retq
+;
+; AVX-LABEL: sitofp_2i8_to_2f64:
+; AVX:       # BB#0:
+; AVX-NEXT:    vpmovsxbd %xmm0, %xmm0
+; AVX-NEXT:    vcvtdq2pd %xmm0, %xmm0
+; AVX-NEXT:    retq
+  %shuf = shufflevector <16 x i8> %a, <16 x i8> undef, <2 x i32> <i32 0, i32 1>
+  %cvt = sitofp <2 x i8> %shuf to <2 x double>
+  ret <2 x double> %cvt
+}
+
+define <2 x double> @sitofp_16i8_to_2f64(<16 x i8> %a) {
+; SSE-LABEL: sitofp_16i8_to_2f64:
+; SSE:       # BB#0:
+; SSE-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
+; SSE-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3]
+; SSE-NEXT:    psrad $24, %xmm0
+; SSE-NEXT:    cvtdq2pd %xmm0, %xmm0
+; SSE-NEXT:    retq
+;
+; AVX1-LABEL: sitofp_16i8_to_2f64:
+; AVX1:       # BB#0:
+; AVX1-NEXT:    vpmovsxbd %xmm0, %xmm0
+; AVX1-NEXT:    vcvtdq2pd %xmm0, %ymm0
+; AVX1-NEXT:    vzeroupper
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: sitofp_16i8_to_2f64:
+; AVX2:       # BB#0:
+; AVX2-NEXT:    vpmovsxbw %xmm0, %ymm0
+; AVX2-NEXT:    vpmovsxwd %xmm0, %ymm0
+; AVX2-NEXT:    vcvtdq2pd %xmm0, %ymm0
+; AVX2-NEXT:    vzeroupper
+; AVX2-NEXT:    retq
+  %cvt = sitofp <16 x i8> %a to <16 x double>
+  %shuf = shufflevector <16 x double> %cvt, <16 x double> undef, <2 x i32> <i32 0, i32 1>
+  ret <2 x double> %shuf
+}
+
+define <4 x double> @sitofp_4i64_to_4f64(<4 x i64> %a) {
+; SSE-LABEL: sitofp_4i64_to_4f64:
+; SSE:       # BB#0:
+; SSE-NEXT:    movd %xmm0, %rax
+; SSE-NEXT:    cvtsi2sdq %rax, %xmm2
+; SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
+; SSE-NEXT:    movd %xmm0, %rax
+; SSE-NEXT:    xorps %xmm0, %xmm0
+; SSE-NEXT:    cvtsi2sdq %rax, %xmm0
+; SSE-NEXT:    unpcklpd {{.*#+}} xmm2 = xmm2[0],xmm0[0]
+; SSE-NEXT:    movd %xmm1, %rax
+; SSE-NEXT:    cvtsi2sdq %rax, %xmm3
+; SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[2,3,0,1]
+; SSE-NEXT:    movd %xmm0, %rax
+; SSE-NEXT:    xorps %xmm0, %xmm0
+; SSE-NEXT:    cvtsi2sdq %rax, %xmm0
+; SSE-NEXT:    unpcklpd {{.*#+}} xmm3 = xmm3[0],xmm0[0]
+; SSE-NEXT:    movapd %xmm2, %xmm0
+; SSE-NEXT:    movapd %xmm3, %xmm1
+; SSE-NEXT:    retq
+;
+; AVX1-LABEL: sitofp_4i64_to_4f64:
+; AVX1:       # BB#0:
+; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
+; AVX1-NEXT:    vpextrq $1, %xmm1, %rax
+; AVX1-NEXT:    vcvtsi2sdq %rax, %xmm0, %xmm2
+; AVX1-NEXT:    vmovq %xmm1, %rax
+; AVX1-NEXT:    vcvtsi2sdq %rax, %xmm0, %xmm1
+; AVX1-NEXT:    vunpcklpd {{.*#+}} xmm1 = xmm1[0],xmm2[0]
+; AVX1-NEXT:    vpextrq $1, %xmm0, %rax
+; AVX1-NEXT:    vcvtsi2sdq %rax, %xmm0, %xmm2
+; AVX1-NEXT:    vmovq %xmm0, %rax
+; AVX1-NEXT:    vxorps %xmm0, %xmm0, %xmm0
+; AVX1-NEXT:    vcvtsi2sdq %rax, %xmm0, %xmm0
+; AVX1-NEXT:    vunpcklpd {{.*#+}} xmm0 = xmm0[0],xmm2[0]
+; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: sitofp_4i64_to_4f64:
+; AVX2:       # BB#0:
+; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
+; AVX2-NEXT:    vpextrq $1, %xmm1, %rax
+; AVX2-NEXT:    vcvtsi2sdq %rax, %xmm0, %xmm2
+; AVX2-NEXT:    vmovq %xmm1, %rax
+; AVX2-NEXT:    vcvtsi2sdq %rax, %xmm0, %xmm1
+; AVX2-NEXT:    vunpcklpd {{.*#+}} xmm1 = xmm1[0],xmm2[0]
+; AVX2-NEXT:    vpextrq $1, %xmm0, %rax
+; AVX2-NEXT:    vcvtsi2sdq %rax, %xmm0, %xmm2
+; AVX2-NEXT:    vmovq %xmm0, %rax
+; AVX2-NEXT:    vxorps %xmm0, %xmm0, %xmm0
+; AVX2-NEXT:    vcvtsi2sdq %rax, %xmm0, %xmm0
+; AVX2-NEXT:    vunpcklpd {{.*#+}} xmm0 = xmm0[0],xmm2[0]
+; AVX2-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
+; AVX2-NEXT:    retq
+  %cvt = sitofp <4 x i64> %a to <4 x double>
+  ret <4 x double> %cvt
+}
+
+define <4 x double> @sitofp_4i32_to_4f64(<4 x i32> %a) {
+; SSE-LABEL: sitofp_4i32_to_4f64:
+; SSE:       # BB#0:
+; SSE-NEXT:    cvtdq2pd %xmm0, %xmm2
+; SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
+; SSE-NEXT:    cvtdq2pd %xmm0, %xmm1
+; SSE-NEXT:    movaps %xmm2, %xmm0
+; SSE-NEXT:    retq
+;
+; AVX-LABEL: sitofp_4i32_to_4f64:
+; AVX:       # BB#0:
+; AVX-NEXT:    vcvtdq2pd %xmm0, %ymm0
+; AVX-NEXT:    retq
+  %cvt = sitofp <4 x i32> %a to <4 x double>
+  ret <4 x double> %cvt
+}
+
+define <4 x double> @sitofp_4i16_to_4f64(<8 x i16> %a) {
+; SSE-LABEL: sitofp_4i16_to_4f64:
+; SSE:       # BB#0:
+; SSE-NEXT:    punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]
+; SSE-NEXT:    psrad $16, %xmm1
+; SSE-NEXT:    cvtdq2pd %xmm1, %xmm0
+; SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[2,3,0,1]
+; SSE-NEXT:    cvtdq2pd %xmm1, %xmm1
+; SSE-NEXT:    retq
+;
+; AVX-LABEL: sitofp_4i16_to_4f64:
+; AVX:       # BB#0:
+; AVX-NEXT:    vpmovsxwd %xmm0, %xmm0
+; AVX-NEXT:    vcvtdq2pd %xmm0, %ymm0
+; AVX-NEXT:    retq
+  %shuf = shufflevector <8 x i16> %a, <8 x i16> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %cvt = sitofp <4 x i16> %shuf to <4 x double>
+  ret <4 x double> %cvt
+}
+
+define <4 x double> @sitofp_8i16_to_4f64(<8 x i16> %a) {
+; SSE-LABEL: sitofp_8i16_to_4f64:
+; SSE:       # BB#0:
+; SSE-NEXT:    punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]
+; SSE-NEXT:    psrad $16, %xmm1
+; SSE-NEXT:    cvtdq2pd %xmm1, %xmm0
+; SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[2,3,0,1]
+; SSE-NEXT:    cvtdq2pd %xmm1, %xmm1
+; SSE-NEXT:    retq
+;
+; AVX1-LABEL: sitofp_8i16_to_4f64:
+; AVX1:       # BB#0:
+; AVX1-NEXT:    vpmovsxwd %xmm0, %xmm0
+; AVX1-NEXT:    vcvtdq2pd %xmm0, %ymm0
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: sitofp_8i16_to_4f64:
+; AVX2:       # BB#0:
+; AVX2-NEXT:    vpmovsxwd %xmm0, %ymm0
+; AVX2-NEXT:    vcvtdq2pd %xmm0, %ymm0
+; AVX2-NEXT:    retq
+  %cvt = sitofp <8 x i16> %a to <8 x double>
+  %shuf = shufflevector <8 x double> %cvt, <8 x double> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  ret <4 x double> %shuf
+}
+
+define <4 x double> @sitofp_4i8_to_4f64(<16 x i8> %a) {
+; SSE-LABEL: sitofp_4i8_to_4f64:
+; SSE:       # BB#0:
+; SSE-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
+; SSE-NEXT:    punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]
+; SSE-NEXT:    psrad $24, %xmm1
+; SSE-NEXT:    cvtdq2pd %xmm1, %xmm0
+; SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[2,3,0,1]
+; SSE-NEXT:    cvtdq2pd %xmm1, %xmm1
+; SSE-NEXT:    retq
+;
+; AVX-LABEL: sitofp_4i8_to_4f64:
+; AVX:       # BB#0:
+; AVX-NEXT:    vpmovsxbd %xmm0, %xmm0
+; AVX-NEXT:    vcvtdq2pd %xmm0, %ymm0
+; AVX-NEXT:    retq
+  %shuf = shufflevector <16 x i8> %a, <16 x i8> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %cvt = sitofp <4 x i8> %shuf to <4 x double>
+  ret <4 x double> %cvt
+}
+
+define <4 x double> @sitofp_16i8_to_4f64(<16 x i8> %a) {
+; SSE-LABEL: sitofp_16i8_to_4f64:
+; SSE:       # BB#0:
+; SSE-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
+; SSE-NEXT:    punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]
+; SSE-NEXT:    psrad $24, %xmm1
+; SSE-NEXT:    cvtdq2pd %xmm1, %xmm0
+; SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[2,3,0,1]
+; SSE-NEXT:    cvtdq2pd %xmm1, %xmm1
+; SSE-NEXT:    retq
+;
+; AVX1-LABEL: sitofp_16i8_to_4f64:
+; AVX1:       # BB#0:
+; AVX1-NEXT:    vpmovsxbd %xmm0, %xmm0
+; AVX1-NEXT:    vcvtdq2pd %xmm0, %ymm0
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: sitofp_16i8_to_4f64:
+; AVX2:       # BB#0:
+; AVX2-NEXT:    vpmovsxbw %xmm0, %ymm0
+; AVX2-NEXT:    vpmovsxwd %xmm0, %ymm0
+; AVX2-NEXT:    vcvtdq2pd %xmm0, %ymm0
+; AVX2-NEXT:    retq
+  %cvt = sitofp <16 x i8> %a to <16 x double>
+  %shuf = shufflevector <16 x double> %cvt, <16 x double> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  ret <4 x double> %shuf
+}
+
+;
+; Unsigned Integer to Double
+;
+
+define <2 x double> @uitofp_2i64_to_2f64(<2 x i64> %a) {
+; SSE-LABEL: uitofp_2i64_to_2f64:
+; SSE:       # BB#0:
+; SSE-NEXT:    movdqa {{.*#+}} xmm1 = [1127219200,1160773632,0,0]
+; SSE-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[2,3,0,1]
+; SSE-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
+; SSE-NEXT:    movapd {{.*#+}} xmm3 = [4.503600e+15,1.934281e+25]
+; SSE-NEXT:    subpd %xmm3, %xmm0
+; SSE-NEXT:    pshufd {{.*#+}} xmm4 = xmm0[2,3,0,1]
+; SSE-NEXT:    addpd %xmm4, %xmm0
+; SSE-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]
+; SSE-NEXT:    subpd %xmm3, %xmm2
+; SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm2[2,3,0,1]
+; SSE-NEXT:    addpd %xmm2, %xmm1
+; SSE-NEXT:    unpcklpd {{.*#+}} xmm0 = xmm0[0],xmm1[0]
+; SSE-NEXT:    retq
+;
+; AVX-LABEL: uitofp_2i64_to_2f64:
+; AVX:       # BB#0:
+; AVX-NEXT:    vmovdqa {{.*#+}} xmm1 = [1127219200,1160773632,0,0]
+; AVX-NEXT:    vpunpckldq {{.*#+}} xmm2 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
+; AVX-NEXT:    vmovapd {{.*#+}} xmm3 = [4.503600e+15,1.934281e+25]
+; AVX-NEXT:    vsubpd %xmm3, %xmm2, %xmm2
+; AVX-NEXT:    vhaddpd %xmm2, %xmm2, %xmm2
+; AVX-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
+; AVX-NEXT:    vpunpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
+; AVX-NEXT:    vsubpd %xmm3, %xmm0, %xmm0
+; AVX-NEXT:    vhaddpd %xmm0, %xmm0, %xmm0
+; AVX-NEXT:    vunpcklpd {{.*#+}} xmm0 = xmm2[0],xmm0[0]
+; AVX-NEXT:    retq
+  %cvt = uitofp <2 x i64> %a to <2 x double>
+  ret <2 x double> %cvt
+}
+
+define <2 x double> @uitofp_2i32_to_2f64(<4 x i32> %a) {
+; SSE-LABEL: uitofp_2i32_to_2f64:
+; SSE:       # BB#0:
+; SSE-NEXT:    pxor %xmm1, %xmm1
+; SSE-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
+; SSE-NEXT:    movdqa {{.*#+}} xmm1 = [1127219200,1160773632,0,0]
+; SSE-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[2,3,0,1]
+; SSE-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
+; SSE-NEXT:    movapd {{.*#+}} xmm3 = [4.503600e+15,1.934281e+25]
+; SSE-NEXT:    subpd %xmm3, %xmm0
+; SSE-NEXT:    pshufd {{.*#+}} xmm4 = xmm0[2,3,0,1]
+; SSE-NEXT:    addpd %xmm4, %xmm0
+; SSE-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]
+; SSE-NEXT:    subpd %xmm3, %xmm2
+; SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm2[2,3,0,1]
+; SSE-NEXT:    addpd %xmm2, %xmm1
+; SSE-NEXT:    unpcklpd {{.*#+}} xmm0 = xmm0[0],xmm1[0]
+; SSE-NEXT:    retq
+;
+; AVX-LABEL: uitofp_2i32_to_2f64:
+; AVX:       # BB#0:
+; AVX-NEXT:    vpmovzxdq {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero
+; AVX-NEXT:    vmovdqa {{.*#+}} xmm1 = [1127219200,1160773632,0,0]
+; AVX-NEXT:    vpunpckldq {{.*#+}} xmm2 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
+; AVX-NEXT:    vmovapd {{.*#+}} xmm3 = [4.503600e+15,1.934281e+25]
+; AVX-NEXT:    vsubpd %xmm3, %xmm2, %xmm2
+; AVX-NEXT:    vhaddpd %xmm2, %xmm2, %xmm2
+; AVX-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
+; AVX-NEXT:    vpunpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
+; AVX-NEXT:    vsubpd %xmm3, %xmm0, %xmm0
+; AVX-NEXT:    vhaddpd %xmm0, %xmm0, %xmm0
+; AVX-NEXT:    vunpcklpd {{.*#+}} xmm0 = xmm2[0],xmm0[0]
+; AVX-NEXT:    retq
+  %shuf = shufflevector <4 x i32> %a, <4 x i32> undef, <2 x i32> <i32 0, i32 1>
+  %cvt = uitofp <2 x i32> %shuf to <2 x double>
+  ret <2 x double> %cvt
+}
+
+define <2 x double> @uitofp_4i32_to_2f64(<4 x i32> %a) {
+; SSE-LABEL: uitofp_4i32_to_2f64:
+; SSE:       # BB#0:
+; SSE-NEXT:    pxor %xmm1, %xmm1
+; SSE-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
+; SSE-NEXT:    movdqa {{.*#+}} xmm1 = [1127219200,1160773632,0,0]
+; SSE-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[2,3,0,1]
+; SSE-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
+; SSE-NEXT:    movapd {{.*#+}} xmm3 = [4.503600e+15,1.934281e+25]
+; SSE-NEXT:    subpd %xmm3, %xmm0
+; SSE-NEXT:    pshufd {{.*#+}} xmm4 = xmm0[2,3,0,1]
+; SSE-NEXT:    addpd %xmm4, %xmm0
+; SSE-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]
+; SSE-NEXT:    subpd %xmm3, %xmm2
+; SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm2[2,3,0,1]
+; SSE-NEXT:    addpd %xmm2, %xmm1
+; SSE-NEXT:    unpcklpd {{.*#+}} xmm0 = xmm0[0],xmm1[0]
+; SSE-NEXT:    retq
+;
+; AVX1-LABEL: uitofp_4i32_to_2f64:
+; AVX1:       # BB#0:
+; AVX1-NEXT:    vpand {{.*}}(%rip), %xmm0, %xmm1
+; AVX1-NEXT:    vcvtdq2pd %xmm1, %ymm1
+; AVX1-NEXT:    vpsrld $16, %xmm0, %xmm0
+; AVX1-NEXT:    vcvtdq2pd %xmm0, %ymm0
+; AVX1-NEXT:    vmulpd {{.*}}(%rip), %ymm0, %ymm0
+; AVX1-NEXT:    vaddpd %ymm1, %ymm0, %ymm0
+; AVX1-NEXT:    vzeroupper
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: uitofp_4i32_to_2f64:
+; AVX2:       # BB#0:
+; AVX2-NEXT:    vpsrld $16, %xmm0, %xmm1
+; AVX2-NEXT:    vcvtdq2pd %xmm1, %ymm1
+; AVX2-NEXT:    vbroadcastsd {{.*}}(%rip), %ymm2
+; AVX2-NEXT:    vmulpd %ymm2, %ymm1, %ymm1
+; AVX2-NEXT:    vpbroadcastd {{.*}}(%rip), %xmm2
+; AVX2-NEXT:    vpand %xmm2, %xmm0, %xmm0
+; AVX2-NEXT:    vcvtdq2pd %xmm0, %ymm0
+; AVX2-NEXT:    vaddpd %ymm0, %ymm1, %ymm0
+; AVX2-NEXT:    vzeroupper
+; AVX2-NEXT:    retq
+  %cvt = uitofp <4 x i32> %a to <4 x double>
+  %shuf = shufflevector <4 x double> %cvt, <4 x double> undef, <2 x i32> <i32 0, i32 1>
+  ret <2 x double> %shuf
+}
+
+define <2 x double> @uitofp_2i16_to_2f64(<8 x i16> %a) {
+; SSE-LABEL: uitofp_2i16_to_2f64:
+; SSE:       # BB#0:
+; SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,1,0,3]
+; SSE-NEXT:    pshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,5,5,6,7]
+; SSE-NEXT:    pand {{.*}}(%rip), %xmm0
+; SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
+; SSE-NEXT:    cvtdq2pd %xmm0, %xmm0
+; SSE-NEXT:    retq
+;
+; AVX-LABEL: uitofp_2i16_to_2f64:
+; AVX:       # BB#0:
+; AVX-NEXT:    vpmovzxwq {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero
+; AVX-NEXT:    vpand {{.*}}(%rip), %xmm0, %xmm0
+; AVX-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
+; AVX-NEXT:    vcvtdq2pd %xmm0, %xmm0
+; AVX-NEXT:    retq
+  %shuf = shufflevector <8 x i16> %a, <8 x i16> undef, <2 x i32> <i32 0, i32 1>
+  %cvt = uitofp <2 x i16> %shuf to <2 x double>
+  ret <2 x double> %cvt
+}
+
+define <2 x double> @uitofp_8i16_to_2f64(<8 x i16> %a) {
+; SSE-LABEL: uitofp_8i16_to_2f64:
+; SSE:       # BB#0:
+; SSE-NEXT:    pxor %xmm1, %xmm1
+; SSE-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
+; SSE-NEXT:    cvtdq2pd %xmm0, %xmm0
+; SSE-NEXT:    retq
+;
+; AVX1-LABEL: uitofp_8i16_to_2f64:
+; AVX1:       # BB#0:
+; AVX1-NEXT:    vpmovzxwd {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero
+; AVX1-NEXT:    vcvtdq2pd %xmm0, %ymm0
+; AVX1-NEXT:    vzeroupper
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: uitofp_8i16_to_2f64:
+; AVX2:       # BB#0:
+; AVX2-NEXT:    vpmovzxwd {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
+; AVX2-NEXT:    vcvtdq2pd %xmm0, %ymm0
+; AVX2-NEXT:    vzeroupper
+; AVX2-NEXT:    retq
+  %cvt = uitofp <8 x i16> %a to <8 x double>
+  %shuf = shufflevector <8 x double> %cvt, <8 x double> undef, <2 x i32> <i32 0, i32 1>
+  ret <2 x double> %shuf
+}
+
+define <2 x double> @uitofp_2i8_to_2f64(<16 x i8> %a) {
+; SSE-LABEL: uitofp_2i8_to_2f64:
+; SSE:       # BB#0:
+; SSE-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
+; SSE-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3]
+; SSE-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0,0,1,1]
+; SSE-NEXT:    pand {{.*}}(%rip), %xmm0
+; SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
+; SSE-NEXT:    cvtdq2pd %xmm0, %xmm0
+; SSE-NEXT:    retq
+;
+; AVX-LABEL: uitofp_2i8_to_2f64:
+; AVX:       # BB#0:
+; AVX-NEXT:    vpmovzxbq {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,zero,zero,zero,zero,xmm0[1],zero,zero,zero,zero,zero,zero,zero
+; AVX-NEXT:    vpand {{.*}}(%rip), %xmm0, %xmm0
+; AVX-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
+; AVX-NEXT:    vcvtdq2pd %xmm0, %xmm0
+; AVX-NEXT:    retq
+  %shuf = shufflevector <16 x i8> %a, <16 x i8> undef, <2 x i32> <i32 0, i32 1>
+  %cvt = uitofp <2 x i8> %shuf to <2 x double>
+  ret <2 x double> %cvt
+}
+
+define <2 x double> @uitofp_16i8_to_2f64(<16 x i8> %a) {
+; SSE-LABEL: uitofp_16i8_to_2f64:
+; SSE:       # BB#0:
+; SSE-NEXT:    pxor %xmm1, %xmm1
+; SSE-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3],xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7]
+; SSE-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
+; SSE-NEXT:    cvtdq2pd %xmm0, %xmm0
+; SSE-NEXT:    retq
+;
+; AVX1-LABEL: uitofp_16i8_to_2f64:
+; AVX1:       # BB#0:
+; AVX1-NEXT:    vpmovzxbd {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
+; AVX1-NEXT:    vcvtdq2pd %xmm0, %ymm0
+; AVX1-NEXT:    vzeroupper
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: uitofp_16i8_to_2f64:
+; AVX2:       # BB#0:
+; AVX2-NEXT:    vpmovzxbw {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero,xmm0[8],zero,xmm0[9],zero,xmm0[10],zero,xmm0[11],zero,xmm0[12],zero,xmm0[13],zero,xmm0[14],zero,xmm0[15],zero
+; AVX2-NEXT:    vpmovzxwd {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
+; AVX2-NEXT:    vcvtdq2pd %xmm0, %ymm0
+; AVX2-NEXT:    vzeroupper
+; AVX2-NEXT:    retq
+  %cvt = uitofp <16 x i8> %a to <16 x double>
+  %shuf = shufflevector <16 x double> %cvt, <16 x double> undef, <2 x i32> <i32 0, i32 1>
+  ret <2 x double> %shuf
+}
+
+define <4 x double> @uitofp_4i64_to_4f64(<4 x i64> %a) {
+; SSE-LABEL: uitofp_4i64_to_4f64:
+; SSE:       # BB#0:
+; SSE-NEXT:    movdqa {{.*#+}} xmm2 = [1127219200,1160773632,0,0]
+; SSE-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[2,3,0,1]
+; SSE-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1]
+; SSE-NEXT:    movapd {{.*#+}} xmm4 = [4.503600e+15,1.934281e+25]
+; SSE-NEXT:    subpd %xmm4, %xmm0
+; SSE-NEXT:    pshufd {{.*#+}} xmm5 = xmm0[2,3,0,1]
+; SSE-NEXT:    addpd %xmm5, %xmm0
+; SSE-NEXT:    punpckldq {{.*#+}} xmm3 = xmm3[0],xmm2[0],xmm3[1],xmm2[1]
+; SSE-NEXT:    subpd %xmm4, %xmm3
+; SSE-NEXT:    pshufd {{.*#+}} xmm5 = xmm3[2,3,0,1]
+; SSE-NEXT:    addpd %xmm3, %xmm5
+; SSE-NEXT:    unpcklpd {{.*#+}} xmm0 = xmm0[0],xmm5[0]
+; SSE-NEXT:    pshufd {{.*#+}} xmm3 = xmm1[2,3,0,1]
+; SSE-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1]
+; SSE-NEXT:    subpd %xmm4, %xmm1
+; SSE-NEXT:    pshufd {{.*#+}} xmm5 = xmm1[2,3,0,1]
+; SSE-NEXT:    addpd %xmm5, %xmm1
+; SSE-NEXT:    punpckldq {{.*#+}} xmm3 = xmm3[0],xmm2[0],xmm3[1],xmm2[1]
+; SSE-NEXT:    subpd %xmm4, %xmm3
+; SSE-NEXT:    pshufd {{.*#+}} xmm2 = xmm3[2,3,0,1]
+; SSE-NEXT:    addpd %xmm3, %xmm2
+; SSE-NEXT:    unpcklpd {{.*#+}} xmm1 = xmm1[0],xmm2[0]
+; SSE-NEXT:    retq
+;
+; AVX1-LABEL: uitofp_4i64_to_4f64:
+; AVX1:       # BB#0:
+; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
+; AVX1-NEXT:    vmovdqa {{.*#+}} xmm2 = [1127219200,1160773632,0,0]
+; AVX1-NEXT:    vpunpckldq {{.*#+}} xmm3 = xmm1[0],xmm2[0],xmm1[1],xmm2[1]
+; AVX1-NEXT:    vmovapd {{.*#+}} xmm4 = [4.503600e+15,1.934281e+25]
+; AVX1-NEXT:    vsubpd %xmm4, %xmm3, %xmm3
+; AVX1-NEXT:    vhaddpd %xmm3, %xmm3, %xmm3
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[2,3,0,1]
+; AVX1-NEXT:    vpunpckldq {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1]
+; AVX1-NEXT:    vsubpd %xmm4, %xmm1, %xmm1
+; AVX1-NEXT:    vhaddpd %xmm1, %xmm1, %xmm1
+; AVX1-NEXT:    vunpcklpd {{.*#+}} xmm1 = xmm3[0],xmm1[0]
+; AVX1-NEXT:    vpunpckldq {{.*#+}} xmm3 = xmm0[0],xmm2[0],xmm0[1],xmm2[1]
+; AVX1-NEXT:    vsubpd %xmm4, %xmm3, %xmm3
+; AVX1-NEXT:    vhaddpd %xmm3, %xmm3, %xmm3
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
+; AVX1-NEXT:    vpunpckldq {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1]
+; AVX1-NEXT:    vsubpd %xmm4, %xmm0, %xmm0
+; AVX1-NEXT:    vhaddpd %xmm0, %xmm0, %xmm0
+; AVX1-NEXT:    vunpcklpd {{.*#+}} xmm0 = xmm3[0],xmm0[0]
+; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: uitofp_4i64_to_4f64:
+; AVX2:       # BB#0:
+; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
+; AVX2-NEXT:    vmovdqa {{.*#+}} xmm2 = [1127219200,1160773632,0,0]
+; AVX2-NEXT:    vpunpckldq {{.*#+}} xmm3 = xmm1[0],xmm2[0],xmm1[1],xmm2[1]
+; AVX2-NEXT:    vmovapd {{.*#+}} xmm4 = [4.503600e+15,1.934281e+25]
+; AVX2-NEXT:    vsubpd %xmm4, %xmm3, %xmm3
+; AVX2-NEXT:    vhaddpd %xmm3, %xmm3, %xmm3
+; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[2,3,0,1]
+; AVX2-NEXT:    vpunpckldq {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1]
+; AVX2-NEXT:    vsubpd %xmm4, %xmm1, %xmm1
+; AVX2-NEXT:    vhaddpd %xmm1, %xmm1, %xmm1
+; AVX2-NEXT:    vunpcklpd {{.*#+}} xmm1 = xmm3[0],xmm1[0]
+; AVX2-NEXT:    vpunpckldq {{.*#+}} xmm3 = xmm0[0],xmm2[0],xmm0[1],xmm2[1]
+; AVX2-NEXT:    vsubpd %xmm4, %xmm3, %xmm3
+; AVX2-NEXT:    vhaddpd %xmm3, %xmm3, %xmm3
+; AVX2-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
+; AVX2-NEXT:    vpunpckldq {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1]
+; AVX2-NEXT:    vsubpd %xmm4, %xmm0, %xmm0
+; AVX2-NEXT:    vhaddpd %xmm0, %xmm0, %xmm0
+; AVX2-NEXT:    vunpcklpd {{.*#+}} xmm0 = xmm3[0],xmm0[0]
+; AVX2-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
+; AVX2-NEXT:    retq
+  %cvt = uitofp <4 x i64> %a to <4 x double>
+  ret <4 x double> %cvt
+}
+
+define <4 x double> @uitofp_4i32_to_4f64(<4 x i32> %a) {
+; SSE-LABEL: uitofp_4i32_to_4f64:
+; SSE:       # BB#0:
+; SSE-NEXT:    pxor %xmm1, %xmm1
+; SSE-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[2,2,3,3]
+; SSE-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
+; SSE-NEXT:    movdqa {{.*#+}} xmm3 = [1127219200,1160773632,0,0]
+; SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; SSE-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1]
+; SSE-NEXT:    movapd {{.*#+}} xmm4 = [4.503600e+15,1.934281e+25]
+; SSE-NEXT:    subpd %xmm4, %xmm0
+; SSE-NEXT:    pshufd {{.*#+}} xmm5 = xmm0[2,3,0,1]
+; SSE-NEXT:    addpd %xmm5, %xmm0
+; SSE-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm3[0],xmm1[1],xmm3[1]
+; SSE-NEXT:    subpd %xmm4, %xmm1
+; SSE-NEXT:    pshufd {{.*#+}} xmm5 = xmm1[2,3,0,1]
+; SSE-NEXT:    addpd %xmm1, %xmm5
+; SSE-NEXT:    unpcklpd {{.*#+}} xmm0 = xmm0[0],xmm5[0]
+; SSE-NEXT:    pand {{.*}}(%rip), %xmm2
+; SSE-NEXT:    pshufd {{.*#+}} xmm5 = xmm2[2,3,0,1]
+; SSE-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1]
+; SSE-NEXT:    subpd %xmm4, %xmm2
+; SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm2[2,3,0,1]
+; SSE-NEXT:    addpd %xmm2, %xmm1
+; SSE-NEXT:    punpckldq {{.*#+}} xmm5 = xmm5[0],xmm3[0],xmm5[1],xmm3[1]
+; SSE-NEXT:    subpd %xmm4, %xmm5
+; SSE-NEXT:    pshufd {{.*#+}} xmm2 = xmm5[2,3,0,1]
+; SSE-NEXT:    addpd %xmm5, %xmm2
+; SSE-NEXT:    unpcklpd {{.*#+}} xmm1 = xmm1[0],xmm2[0]
+; SSE-NEXT:    retq
+;
+; AVX1-LABEL: uitofp_4i32_to_4f64:
+; AVX1:       # BB#0:
+; AVX1-NEXT:    vpand {{.*}}(%rip), %xmm0, %xmm1
+; AVX1-NEXT:    vcvtdq2pd %xmm1, %ymm1
+; AVX1-NEXT:    vpsrld $16, %xmm0, %xmm0
+; AVX1-NEXT:    vcvtdq2pd %xmm0, %ymm0
+; AVX1-NEXT:    vmulpd {{.*}}(%rip), %ymm0, %ymm0
+; AVX1-NEXT:    vaddpd %ymm1, %ymm0, %ymm0
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: uitofp_4i32_to_4f64:
+; AVX2:       # BB#0:
+; AVX2-NEXT:    vpsrld $16, %xmm0, %xmm1
+; AVX2-NEXT:    vcvtdq2pd %xmm1, %ymm1
+; AVX2-NEXT:    vbroadcastsd {{.*}}(%rip), %ymm2
+; AVX2-NEXT:    vmulpd %ymm2, %ymm1, %ymm1
+; AVX2-NEXT:    vpbroadcastd {{.*}}(%rip), %xmm2
+; AVX2-NEXT:    vpand %xmm2, %xmm0, %xmm0
+; AVX2-NEXT:    vcvtdq2pd %xmm0, %ymm0
+; AVX2-NEXT:    vaddpd %ymm0, %ymm1, %ymm0
+; AVX2-NEXT:    retq
+  %cvt = uitofp <4 x i32> %a to <4 x double>
+  ret <4 x double> %cvt
+}
+
+define <4 x double> @uitofp_4i16_to_4f64(<8 x i16> %a) {
+; SSE-LABEL: uitofp_4i16_to_4f64:
+; SSE:       # BB#0:
+; SSE-NEXT:    pxor %xmm1, %xmm1
+; SSE-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
+; SSE-NEXT:    cvtdq2pd %xmm0, %xmm2
+; SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
+; SSE-NEXT:    cvtdq2pd %xmm0, %xmm1
+; SSE-NEXT:    movaps %xmm2, %xmm0
+; SSE-NEXT:    retq
+;
+; AVX-LABEL: uitofp_4i16_to_4f64:
+; AVX:       # BB#0:
+; AVX-NEXT:    vpmovzxwd {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero
+; AVX-NEXT:    vcvtdq2pd %xmm0, %ymm0
+; AVX-NEXT:    retq
+  %shuf = shufflevector <8 x i16> %a, <8 x i16> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %cvt = uitofp <4 x i16> %shuf to <4 x double>
+  ret <4 x double> %cvt
+}
+
+define <4 x double> @uitofp_8i16_to_4f64(<8 x i16> %a) {
+; SSE-LABEL: uitofp_8i16_to_4f64:
+; SSE:       # BB#0:
+; SSE-NEXT:    pxor %xmm1, %xmm1
+; SSE-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
+; SSE-NEXT:    cvtdq2pd %xmm0, %xmm2
+; SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
+; SSE-NEXT:    cvtdq2pd %xmm0, %xmm1
+; SSE-NEXT:    movaps %xmm2, %xmm0
+; SSE-NEXT:    retq
+;
+; AVX1-LABEL: uitofp_8i16_to_4f64:
+; AVX1:       # BB#0:
+; AVX1-NEXT:    vpmovzxwd {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero
+; AVX1-NEXT:    vcvtdq2pd %xmm0, %ymm0
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: uitofp_8i16_to_4f64:
+; AVX2:       # BB#0:
+; AVX2-NEXT:    vpmovzxwd {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
+; AVX2-NEXT:    vcvtdq2pd %xmm0, %ymm0
+; AVX2-NEXT:    retq
+  %cvt = uitofp <8 x i16> %a to <8 x double>
+  %shuf = shufflevector <8 x double> %cvt, <8 x double> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  ret <4 x double> %shuf
+}
+
+define <4 x double> @uitofp_4i8_to_4f64(<16 x i8> %a) {
+; SSE-LABEL: uitofp_4i8_to_4f64:
+; SSE:       # BB#0:
+; SSE-NEXT:    pxor %xmm1, %xmm1
+; SSE-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3],xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7]
+; SSE-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
+; SSE-NEXT:    cvtdq2pd %xmm0, %xmm2
+; SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
+; SSE-NEXT:    cvtdq2pd %xmm0, %xmm1
+; SSE-NEXT:    movaps %xmm2, %xmm0
+; SSE-NEXT:    retq
+;
+; AVX-LABEL: uitofp_4i8_to_4f64:
+; AVX:       # BB#0:
+; AVX-NEXT:    vpmovzxbd {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
+; AVX-NEXT:    vcvtdq2pd %xmm0, %ymm0
+; AVX-NEXT:    retq
+  %shuf = shufflevector <16 x i8> %a, <16 x i8> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %cvt = uitofp <4 x i8> %shuf to <4 x double>
+  ret <4 x double> %cvt
+}
+
+define <4 x double> @uitofp_16i8_to_4f64(<16 x i8> %a) {
+; SSE-LABEL: uitofp_16i8_to_4f64:
+; SSE:       # BB#0:
+; SSE-NEXT:    pxor %xmm1, %xmm1
+; SSE-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3],xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7]
+; SSE-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
+; SSE-NEXT:    cvtdq2pd %xmm0, %xmm2
+; SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
+; SSE-NEXT:    cvtdq2pd %xmm0, %xmm1
+; SSE-NEXT:    movaps %xmm2, %xmm0
+; SSE-NEXT:    retq
+;
+; AVX1-LABEL: uitofp_16i8_to_4f64:
+; AVX1:       # BB#0:
+; AVX1-NEXT:    vpmovzxbd {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
+; AVX1-NEXT:    vcvtdq2pd %xmm0, %ymm0
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: uitofp_16i8_to_4f64:
+; AVX2:       # BB#0:
+; AVX2-NEXT:    vpmovzxbw {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero,xmm0[8],zero,xmm0[9],zero,xmm0[10],zero,xmm0[11],zero,xmm0[12],zero,xmm0[13],zero,xmm0[14],zero,xmm0[15],zero
+; AVX2-NEXT:    vpmovzxwd {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
+; AVX2-NEXT:    vcvtdq2pd %xmm0, %ymm0
+; AVX2-NEXT:    retq
+  %cvt = uitofp <16 x i8> %a to <16 x double>
+  %shuf = shufflevector <16 x double> %cvt, <16 x double> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  ret <4 x double> %shuf
+}
+
+;
+; Signed Integer to Float
+;
+
+define <4 x float> @sitofp_2i64_to_4f32(<2 x i64> %a) {
+; SSE-LABEL: sitofp_2i64_to_4f32:
+; SSE:       # BB#0:
+; SSE-NEXT:    movd %xmm0, %rax
+; SSE-NEXT:    cvtsi2ssq %rax, %xmm1
+; SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
+; SSE-NEXT:    movd %xmm0, %rax
+; SSE-NEXT:    xorps %xmm0, %xmm0
+; SSE-NEXT:    cvtsi2ssq %rax, %xmm0
+; SSE-NEXT:    unpcklps {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
+; SSE-NEXT:    movaps %xmm1, %xmm0
+; SSE-NEXT:    retq
+;
+; AVX-LABEL: sitofp_2i64_to_4f32:
+; AVX:       # BB#0:
+; AVX-NEXT:    vpextrq $1, %xmm0, %rax
+; AVX-NEXT:    vcvtsi2ssq %rax, %xmm0, %xmm1
+; AVX-NEXT:    vmovq %xmm0, %rax
+; AVX-NEXT:    vxorps %xmm0, %xmm0, %xmm0
+; AVX-NEXT:    vcvtsi2ssq %rax, %xmm0, %xmm0
+; AVX-NEXT:    vinsertps {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[2,3]
+; AVX-NEXT:    vcvtsi2ssq %rax, %xmm0, %xmm1
+; AVX-NEXT:    vinsertps {{.*#+}} xmm0 = xmm0[0,1],xmm1[0],xmm0[3]
+; AVX-NEXT:    vinsertps {{.*#+}} xmm0 = xmm0[0,1,2],xmm1[0]
+; AVX-NEXT:    retq
+  %cvt = sitofp <2 x i64> %a to <2 x float>
+  %ext = shufflevector <2 x float> %cvt, <2 x float> undef, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
+  ret <4 x float> %ext
+}
+
+define <4 x float> @sitofp_4i64_to_4f32_undef(<2 x i64> %a) {
+; SSE-LABEL: sitofp_4i64_to_4f32_undef:
+; SSE:       # BB#0:
+; SSE-NEXT:    cvtsi2ssq %rax, %xmm2
+; SSE-NEXT:    movd %xmm0, %rax
+; SSE-NEXT:    cvtsi2ssq %rax, %xmm1
+; SSE-NEXT:    unpcklps {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1]
+; SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
+; SSE-NEXT:    movd %xmm0, %rax
+; SSE-NEXT:    xorps %xmm0, %xmm0
+; SSE-NEXT:    cvtsi2ssq %rax, %xmm0
+; SSE-NEXT:    unpcklps {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1]
+; SSE-NEXT:    unpcklps {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
+; SSE-NEXT:    movaps %xmm1, %xmm0
+; SSE-NEXT:    retq
+;
+; AVX-LABEL: sitofp_4i64_to_4f32_undef:
+; AVX:       # BB#0:
+; AVX-NEXT:    vpextrq $1, %xmm0, %rax
+; AVX-NEXT:    vcvtsi2ssq %rax, %xmm0, %xmm1
+; AVX-NEXT:    vmovq %xmm0, %rax
+; AVX-NEXT:    vxorps %xmm0, %xmm0, %xmm0
+; AVX-NEXT:    vcvtsi2ssq %rax, %xmm0, %xmm0
+; AVX-NEXT:    vinsertps {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[2,3]
+; AVX-NEXT:    vcvtsi2ssq %rax, %xmm0, %xmm1
+; AVX-NEXT:    vinsertps {{.*#+}} xmm0 = xmm0[0,1],xmm1[0],xmm0[3]
+; AVX-NEXT:    vinsertps {{.*#+}} xmm0 = xmm0[0,1,2],xmm1[0]
+; AVX-NEXT:    retq
+  %ext = shufflevector <2 x i64> %a, <2 x i64> undef, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
+  %cvt = sitofp <4 x i64> %ext to <4 x float>
+  ret <4 x float> %cvt
+}
+
+define <4 x float> @sitofp_4i32_to_4f32(<4 x i32> %a) {
+; SSE-LABEL: sitofp_4i32_to_4f32:
+; SSE:       # BB#0:
+; SSE-NEXT:    cvtdq2ps %xmm0, %xmm0
+; SSE-NEXT:    retq
+;
+; AVX-LABEL: sitofp_4i32_to_4f32:
+; AVX:       # BB#0:
+; AVX-NEXT:    vcvtdq2ps %xmm0, %xmm0
+; AVX-NEXT:    retq
+  %cvt = sitofp <4 x i32> %a to <4 x float>
+  ret <4 x float> %cvt
+}
+
+define <4 x float> @sitofp_4i16_to_4f32(<8 x i16> %a) {
+; SSE-LABEL: sitofp_4i16_to_4f32:
+; SSE:       # BB#0:
+; SSE-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3]
+; SSE-NEXT:    psrad $16, %xmm0
+; SSE-NEXT:    cvtdq2ps %xmm0, %xmm0
+; SSE-NEXT:    retq
+;
+; AVX-LABEL: sitofp_4i16_to_4f32:
+; AVX:       # BB#0:
+; AVX-NEXT:    vpmovsxwd %xmm0, %xmm0
+; AVX-NEXT:    vcvtdq2ps %xmm0, %xmm0
+; AVX-NEXT:    retq
+  %shuf = shufflevector <8 x i16> %a, <8 x i16> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %cvt = sitofp <4 x i16> %shuf to <4 x float>
+  ret <4 x float> %cvt
+}
+
+define <4 x float> @sitofp_8i16_to_4f32(<8 x i16> %a) {
+; SSE-LABEL: sitofp_8i16_to_4f32:
+; SSE:       # BB#0:
+; SSE-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3]
+; SSE-NEXT:    psrad $16, %xmm0
+; SSE-NEXT:    cvtdq2ps %xmm0, %xmm0
+; SSE-NEXT:    retq
+;
+; AVX1-LABEL: sitofp_8i16_to_4f32:
+; AVX1:       # BB#0:
+; AVX1-NEXT:    vpmovsxwd %xmm0, %xmm1
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
+; AVX1-NEXT:    vpmovsxwd %xmm0, %xmm0
+; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
+; AVX1-NEXT:    vcvtdq2ps %ymm0, %ymm0
+; AVX1-NEXT:    vzeroupper
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: sitofp_8i16_to_4f32:
+; AVX2:       # BB#0:
+; AVX2-NEXT:    vpmovsxwd %xmm0, %ymm0
+; AVX2-NEXT:    vcvtdq2ps %ymm0, %ymm0
+; AVX2-NEXT:    vzeroupper
+; AVX2-NEXT:    retq
+  %cvt = sitofp <8 x i16> %a to <8 x float>
+  %shuf = shufflevector <8 x float> %cvt, <8 x float> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  ret <4 x float> %shuf
+}
+
+define <4 x float> @sitofp_4i8_to_4f32(<16 x i8> %a) {
+; SSE-LABEL: sitofp_4i8_to_4f32:
+; SSE:       # BB#0:
+; SSE-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
+; SSE-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3]
+; SSE-NEXT:    psrad $24, %xmm0
+; SSE-NEXT:    cvtdq2ps %xmm0, %xmm0
+; SSE-NEXT:    retq
+;
+; AVX-LABEL: sitofp_4i8_to_4f32:
+; AVX:       # BB#0:
+; AVX-NEXT:    vpmovsxbd %xmm0, %xmm0
+; AVX-NEXT:    vcvtdq2ps %xmm0, %xmm0
+; AVX-NEXT:    retq
+  %shuf = shufflevector <16 x i8> %a, <16 x i8> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %cvt = sitofp <4 x i8> %shuf to <4 x float>
+  ret <4 x float> %cvt
+}
+
+define <4 x float> @sitofp_16i8_to_4f32(<16 x i8> %a) {
+; SSE-LABEL: sitofp_16i8_to_4f32:
+; SSE:       # BB#0:
+; SSE-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
+; SSE-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3]
+; SSE-NEXT:    psrad $24, %xmm0
+; SSE-NEXT:    cvtdq2ps %xmm0, %xmm0
+; SSE-NEXT:    retq
+;
+; AVX1-LABEL: sitofp_16i8_to_4f32:
+; AVX1:       # BB#0:
+; AVX1-NEXT:    vpmovsxbd %xmm0, %xmm1
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[1,1,2,3]
+; AVX1-NEXT:    vpmovsxbd %xmm0, %xmm0
+; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
+; AVX1-NEXT:    vcvtdq2ps %ymm0, %ymm0
+; AVX1-NEXT:    vzeroupper
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: sitofp_16i8_to_4f32:
+; AVX2:       # BB#0:
+; AVX2-NEXT:    vpmovsxbw %xmm0, %ymm0
+; AVX2-NEXT:    vpmovsxwd %xmm0, %ymm0
+; AVX2-NEXT:    vcvtdq2ps %ymm0, %ymm0
+; AVX2-NEXT:    vzeroupper
+; AVX2-NEXT:    retq
+  %cvt = sitofp <16 x i8> %a to <16 x float>
+  %shuf = shufflevector <16 x float> %cvt, <16 x float> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  ret <4 x float> %shuf
+}
+
+define <4 x float> @sitofp_4i64_to_4f32(<4 x i64> %a) {
+; SSE-LABEL: sitofp_4i64_to_4f32:
+; SSE:       # BB#0:
+; SSE-NEXT:    movd %xmm1, %rax
+; SSE-NEXT:    cvtsi2ssq %rax, %xmm3
+; SSE-NEXT:    movd %xmm0, %rax
+; SSE-NEXT:    cvtsi2ssq %rax, %xmm2
+; SSE-NEXT:    unpcklps {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1]
+; SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[2,3,0,1]
+; SSE-NEXT:    movd %xmm1, %rax
+; SSE-NEXT:    xorps %xmm1, %xmm1
+; SSE-NEXT:    cvtsi2ssq %rax, %xmm1
+; SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
+; SSE-NEXT:    movd %xmm0, %rax
+; SSE-NEXT:    xorps %xmm0, %xmm0
+; SSE-NEXT:    cvtsi2ssq %rax, %xmm0
+; SSE-NEXT:    unpcklps {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
+; SSE-NEXT:    unpcklps {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1]
+; SSE-NEXT:    movaps %xmm2, %xmm0
+; SSE-NEXT:    retq
+;
+; AVX1-LABEL: sitofp_4i64_to_4f32:
+; AVX1:       # BB#0:
+; AVX1-NEXT:    vpextrq $1, %xmm0, %rax
+; AVX1-NEXT:    vcvtsi2ssq %rax, %xmm0, %xmm1
+; AVX1-NEXT:    vmovq %xmm0, %rax
+; AVX1-NEXT:    vcvtsi2ssq %rax, %xmm0, %xmm2
+; AVX1-NEXT:    vinsertps {{.*#+}} xmm1 = xmm2[0],xmm1[0],xmm2[2,3]
+; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm0
+; AVX1-NEXT:    vmovq %xmm0, %rax
+; AVX1-NEXT:    vcvtsi2ssq %rax, %xmm0, %xmm2
+; AVX1-NEXT:    vinsertps {{.*#+}} xmm1 = xmm1[0,1],xmm2[0],xmm1[3]
+; AVX1-NEXT:    vpextrq $1, %xmm0, %rax
+; AVX1-NEXT:    vxorps %xmm0, %xmm0, %xmm0
+; AVX1-NEXT:    vcvtsi2ssq %rax, %xmm0, %xmm0
+; AVX1-NEXT:    vinsertps {{.*#+}} xmm0 = xmm1[0,1,2],xmm0[0]
+; AVX1-NEXT:    vzeroupper
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: sitofp_4i64_to_4f32:
+; AVX2:       # BB#0:
+; AVX2-NEXT:    vpextrq $1, %xmm0, %rax
+; AVX2-NEXT:    vcvtsi2ssq %rax, %xmm0, %xmm1
+; AVX2-NEXT:    vmovq %xmm0, %rax
+; AVX2-NEXT:    vcvtsi2ssq %rax, %xmm0, %xmm2
+; AVX2-NEXT:    vinsertps {{.*#+}} xmm1 = xmm2[0],xmm1[0],xmm2[2,3]
+; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm0
+; AVX2-NEXT:    vmovq %xmm0, %rax
+; AVX2-NEXT:    vcvtsi2ssq %rax, %xmm0, %xmm2
+; AVX2-NEXT:    vinsertps {{.*#+}} xmm1 = xmm1[0,1],xmm2[0],xmm1[3]
+; AVX2-NEXT:    vpextrq $1, %xmm0, %rax
+; AVX2-NEXT:    vxorps %xmm0, %xmm0, %xmm0
+; AVX2-NEXT:    vcvtsi2ssq %rax, %xmm0, %xmm0
+; AVX2-NEXT:    vinsertps {{.*#+}} xmm0 = xmm1[0,1,2],xmm0[0]
+; AVX2-NEXT:    vzeroupper
+; AVX2-NEXT:    retq
+  %cvt = sitofp <4 x i64> %a to <4 x float>
+  ret <4 x float> %cvt
+}
+
+define <8 x float> @sitofp_8i32_to_8f32(<8 x i32> %a) {
+; SSE-LABEL: sitofp_8i32_to_8f32:
+; SSE:       # BB#0:
+; SSE-NEXT:    cvtdq2ps %xmm0, %xmm0
+; SSE-NEXT:    cvtdq2ps %xmm1, %xmm1
+; SSE-NEXT:    retq
+;
+; AVX-LABEL: sitofp_8i32_to_8f32:
+; AVX:       # BB#0:
+; AVX-NEXT:    vcvtdq2ps %ymm0, %ymm0
+; AVX-NEXT:    retq
+  %cvt = sitofp <8 x i32> %a to <8 x float>
+  ret <8 x float> %cvt
+}
+
+define <8 x float> @sitofp_8i16_to_8f32(<8 x i16> %a) {
+; SSE-LABEL: sitofp_8i16_to_8f32:
+; SSE:       # BB#0:
+; SSE-NEXT:    punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]
+; SSE-NEXT:    psrad $16, %xmm1
+; SSE-NEXT:    cvtdq2ps %xmm1, %xmm2
+; SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
+; SSE-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3]
+; SSE-NEXT:    psrad $16, %xmm0
+; SSE-NEXT:    cvtdq2ps %xmm0, %xmm1
+; SSE-NEXT:    movaps %xmm2, %xmm0
+; SSE-NEXT:    retq
+;
+; AVX1-LABEL: sitofp_8i16_to_8f32:
+; AVX1:       # BB#0:
+; AVX1-NEXT:    vpmovsxwd %xmm0, %xmm1
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
+; AVX1-NEXT:    vpmovsxwd %xmm0, %xmm0
+; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
+; AVX1-NEXT:    vcvtdq2ps %ymm0, %ymm0
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: sitofp_8i16_to_8f32:
+; AVX2:       # BB#0:
+; AVX2-NEXT:    vpmovsxwd %xmm0, %ymm0
+; AVX2-NEXT:    vcvtdq2ps %ymm0, %ymm0
+; AVX2-NEXT:    retq
+  %cvt = sitofp <8 x i16> %a to <8 x float>
+  ret <8 x float> %cvt
+}
+
+define <8 x float> @sitofp_8i8_to_8f32(<16 x i8> %a) {
+; SSE-LABEL: sitofp_8i8_to_8f32:
+; SSE:       # BB#0:
+; SSE-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3],xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
+; SSE-NEXT:    punpcklwd {{.*#+}} xmm1 = xmm1[0,0,1,1,2,2,3,3]
+; SSE-NEXT:    psrad $24, %xmm1
+; SSE-NEXT:    cvtdq2ps %xmm1, %xmm2
+; SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,2,3]
+; SSE-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
+; SSE-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3]
+; SSE-NEXT:    psrad $24, %xmm0
+; SSE-NEXT:    cvtdq2ps %xmm0, %xmm1
+; SSE-NEXT:    movaps %xmm2, %xmm0
+; SSE-NEXT:    retq
+;
+; AVX1-LABEL: sitofp_8i8_to_8f32:
+; AVX1:       # BB#0:
+; AVX1-NEXT:    vpmovsxbd %xmm0, %xmm1
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[1,1,2,3]
+; AVX1-NEXT:    vpmovsxbd %xmm0, %xmm0
+; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
+; AVX1-NEXT:    vcvtdq2ps %ymm0, %ymm0
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: sitofp_8i8_to_8f32:
+; AVX2:       # BB#0:
+; AVX2-NEXT:    vpmovzxbd {{.*#+}} ymm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero,xmm0[4],zero,zero,zero,xmm0[5],zero,zero,zero,xmm0[6],zero,zero,zero,xmm0[7],zero,zero,zero
+; AVX2-NEXT:    vpslld $24, %ymm0, %ymm0
+; AVX2-NEXT:    vpsrad $24, %ymm0, %ymm0
+; AVX2-NEXT:    vcvtdq2ps %ymm0, %ymm0
+; AVX2-NEXT:    retq
+  %shuf = shufflevector <16 x i8> %a, <16 x i8> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+  %cvt = sitofp <8 x i8> %shuf to <8 x float>
+  ret <8 x float> %cvt
+}
+
+define <8 x float> @sitofp_16i8_to_8f32(<16 x i8> %a) {
+; SSE-LABEL: sitofp_16i8_to_8f32:
+; SSE:       # BB#0:
+; SSE-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3],xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
+; SSE-NEXT:    punpcklwd {{.*#+}} xmm1 = xmm1[0,0,1,1,2,2,3,3]
+; SSE-NEXT:    psrad $24, %xmm1
+; SSE-NEXT:    cvtdq2ps %xmm1, %xmm2
+; SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,2,3]
+; SSE-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
+; SSE-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3]
+; SSE-NEXT:    psrad $24, %xmm0
+; SSE-NEXT:    cvtdq2ps %xmm0, %xmm1
+; SSE-NEXT:    movaps %xmm2, %xmm0
+; SSE-NEXT:    retq
+;
+; AVX1-LABEL: sitofp_16i8_to_8f32:
+; AVX1:       # BB#0:
+; AVX1-NEXT:    vpmovsxbd %xmm0, %xmm1
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[1,1,2,3]
+; AVX1-NEXT:    vpmovsxbd %xmm0, %xmm0
+; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
+; AVX1-NEXT:    vcvtdq2ps %ymm0, %ymm0
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: sitofp_16i8_to_8f32:
+; AVX2:       # BB#0:
+; AVX2-NEXT:    vpmovsxbw %xmm0, %ymm0
+; AVX2-NEXT:    vpmovsxwd %xmm0, %ymm0
+; AVX2-NEXT:    vcvtdq2ps %ymm0, %ymm0
+; AVX2-NEXT:    retq
+  %cvt = sitofp <16 x i8> %a to <16 x float>
+  %shuf = shufflevector <16 x float> %cvt, <16 x float> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+  ret <8 x float> %shuf
+}
+
+;
+; Unsigned Integer to Float
+;
+
+define <4 x float> @uitofp_2i64_to_4f32(<2 x i64> %a) {
+; SSE-LABEL: uitofp_2i64_to_4f32:
+; SSE:       # BB#0:
+; SSE-NEXT:    movdqa %xmm0, %xmm1
+; SSE-NEXT:    movd %xmm1, %rax
+; SSE-NEXT:    movl %eax, %ecx
+; SSE-NEXT:    andl $1, %ecx
+; SSE-NEXT:    testq %rax, %rax
+; SSE-NEXT:    js .LBB38_1
+; SSE-NEXT:  # BB#2:
+; SSE-NEXT:    xorps %xmm0, %xmm0
+; SSE-NEXT:    cvtsi2ssq %rax, %xmm0
+; SSE-NEXT:    jmp .LBB38_3
+; SSE-NEXT:  .LBB38_1:
+; SSE-NEXT:    shrq %rax
+; SSE-NEXT:    orq %rax, %rcx
+; SSE-NEXT:    xorps %xmm0, %xmm0
+; SSE-NEXT:    cvtsi2ssq %rcx, %xmm0
+; SSE-NEXT:    addss %xmm0, %xmm0
+; SSE-NEXT:  .LBB38_3:
+; SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[2,3,0,1]
+; SSE-NEXT:    movd %xmm1, %rax
+; SSE-NEXT:    movl %eax, %ecx
+; SSE-NEXT:    andl $1, %ecx
+; SSE-NEXT:    testq %rax, %rax
+; SSE-NEXT:    js .LBB38_4
+; SSE-NEXT:  # BB#5:
+; SSE-NEXT:    xorps %xmm1, %xmm1
+; SSE-NEXT:    cvtsi2ssq %rax, %xmm1
+; SSE-NEXT:    unpcklps {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
+; SSE-NEXT:    retq
+; SSE-NEXT:  .LBB38_4:
+; SSE-NEXT:    shrq %rax
+; SSE-NEXT:    orq %rax, %rcx
+; SSE-NEXT:    xorps %xmm1, %xmm1
+; SSE-NEXT:    cvtsi2ssq %rcx, %xmm1
+; SSE-NEXT:    addss %xmm1, %xmm1
+; SSE-NEXT:    unpcklps {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
+; SSE-NEXT:    retq
+;
+; AVX-LABEL: uitofp_2i64_to_4f32:
+; AVX:       # BB#0:
+; AVX-NEXT:    vpextrq $1, %xmm0, %rax
+; AVX-NEXT:    movl %eax, %ecx
+; AVX-NEXT:    andl $1, %ecx
+; AVX-NEXT:    testq %rax, %rax
+; AVX-NEXT:    js .LBB38_1
+; AVX-NEXT:  # BB#2:
+; AVX-NEXT:    vcvtsi2ssq %rax, %xmm0, %xmm1
+; AVX-NEXT:    jmp .LBB38_3
+; AVX-NEXT:  .LBB38_1:
+; AVX-NEXT:    shrq %rax
+; AVX-NEXT:    orq %rax, %rcx
+; AVX-NEXT:    vcvtsi2ssq %rcx, %xmm0, %xmm1
+; AVX-NEXT:    vaddss %xmm1, %xmm1, %xmm1
+; AVX-NEXT:  .LBB38_3:
+; AVX-NEXT:    vmovq %xmm0, %rax
+; AVX-NEXT:    movl %eax, %ecx
+; AVX-NEXT:    andl $1, %ecx
+; AVX-NEXT:    testq %rax, %rax
+; AVX-NEXT:    js .LBB38_4
+; AVX-NEXT:  # BB#5:
+; AVX-NEXT:    vxorps %xmm0, %xmm0, %xmm0
+; AVX-NEXT:    vcvtsi2ssq %rax, %xmm0, %xmm0
+; AVX-NEXT:    jmp .LBB38_6
+; AVX-NEXT:  .LBB38_4:
+; AVX-NEXT:    shrq %rax
+; AVX-NEXT:    orq %rax, %rcx
+; AVX-NEXT:    vxorps %xmm0, %xmm0, %xmm0
+; AVX-NEXT:    vcvtsi2ssq %rcx, %xmm0, %xmm0
+; AVX-NEXT:    vaddss %xmm0, %xmm0, %xmm0
+; AVX-NEXT:  .LBB38_6:
+; AVX-NEXT:    vinsertps {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[2,3]
+; AVX-NEXT:    vxorps %xmm1, %xmm1, %xmm1
+; AVX-NEXT:    testq %rax, %rax
+; AVX-NEXT:    js .LBB38_8
+; AVX-NEXT:  # BB#7:
+; AVX-NEXT:    vcvtsi2ssq %rax, %xmm0, %xmm1
+; AVX-NEXT:  .LBB38_8:
+; AVX-NEXT:    vinsertps {{.*#+}} xmm0 = xmm0[0,1],xmm1[0],xmm0[3]
+; AVX-NEXT:    vinsertps {{.*#+}} xmm0 = xmm0[0,1,2],xmm1[0]
+; AVX-NEXT:    retq
+  %cvt = uitofp <2 x i64> %a to <2 x float>
+  %ext = shufflevector <2 x float> %cvt, <2 x float> undef, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
+  ret <4 x float> %ext
+}
+
+define <4 x float> @uitofp_4i64_to_4f32_undef(<2 x i64> %a) {
+; SSE-LABEL: uitofp_4i64_to_4f32_undef:
+; SSE:       # BB#0:
+; SSE-NEXT:    movdqa %xmm0, %xmm1
+; SSE-NEXT:    testq %rax, %rax
+; SSE-NEXT:    xorps %xmm2, %xmm2
+; SSE-NEXT:    js .LBB39_2
+; SSE-NEXT:  # BB#1:
+; SSE-NEXT:    xorps %xmm2, %xmm2
+; SSE-NEXT:    cvtsi2ssq %rax, %xmm2
+; SSE-NEXT:  .LBB39_2:
+; SSE-NEXT:    movd %xmm1, %rax
+; SSE-NEXT:    movl %eax, %ecx
+; SSE-NEXT:    andl $1, %ecx
+; SSE-NEXT:    testq %rax, %rax
+; SSE-NEXT:    js .LBB39_3
+; SSE-NEXT:  # BB#4:
+; SSE-NEXT:    xorps %xmm0, %xmm0
+; SSE-NEXT:    cvtsi2ssq %rax, %xmm0
+; SSE-NEXT:    jmp .LBB39_5
+; SSE-NEXT:  .LBB39_3:
+; SSE-NEXT:    shrq %rax
+; SSE-NEXT:    orq %rax, %rcx
+; SSE-NEXT:    xorps %xmm0, %xmm0
+; SSE-NEXT:    cvtsi2ssq %rcx, %xmm0
+; SSE-NEXT:    addss %xmm0, %xmm0
+; SSE-NEXT:  .LBB39_5:
+; SSE-NEXT:    unpcklps {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1]
+; SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[2,3,0,1]
+; SSE-NEXT:    movd %xmm1, %rax
+; SSE-NEXT:    movl %eax, %ecx
+; SSE-NEXT:    andl $1, %ecx
+; SSE-NEXT:    testq %rax, %rax
+; SSE-NEXT:    js .LBB39_6
+; SSE-NEXT:  # BB#7:
+; SSE-NEXT:    xorps %xmm1, %xmm1
+; SSE-NEXT:    cvtsi2ssq %rax, %xmm1
+; SSE-NEXT:    jmp .LBB39_8
+; SSE-NEXT:  .LBB39_6:
+; SSE-NEXT:    shrq %rax
+; SSE-NEXT:    orq %rax, %rcx
+; SSE-NEXT:    xorps %xmm1, %xmm1
+; SSE-NEXT:    cvtsi2ssq %rcx, %xmm1
+; SSE-NEXT:    addss %xmm1, %xmm1
+; SSE-NEXT:  .LBB39_8:
+; SSE-NEXT:    unpcklps {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1]
+; SSE-NEXT:    unpcklps {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
+; SSE-NEXT:    retq
+;
+; AVX-LABEL: uitofp_4i64_to_4f32_undef:
+; AVX:       # BB#0:
+; AVX-NEXT:    vpextrq $1, %xmm0, %rax
+; AVX-NEXT:    movl %eax, %ecx
+; AVX-NEXT:    andl $1, %ecx
+; AVX-NEXT:    testq %rax, %rax
+; AVX-NEXT:    js .LBB39_1
+; AVX-NEXT:  # BB#2:
+; AVX-NEXT:    vcvtsi2ssq %rax, %xmm0, %xmm1
+; AVX-NEXT:    jmp .LBB39_3
+; AVX-NEXT:  .LBB39_1:
+; AVX-NEXT:    shrq %rax
+; AVX-NEXT:    orq %rax, %rcx
+; AVX-NEXT:    vcvtsi2ssq %rcx, %xmm0, %xmm1
+; AVX-NEXT:    vaddss %xmm1, %xmm1, %xmm1
+; AVX-NEXT:  .LBB39_3:
+; AVX-NEXT:    vmovq %xmm0, %rax
+; AVX-NEXT:    movl %eax, %ecx
+; AVX-NEXT:    andl $1, %ecx
+; AVX-NEXT:    testq %rax, %rax
+; AVX-NEXT:    js .LBB39_4
+; AVX-NEXT:  # BB#5:
+; AVX-NEXT:    vxorps %xmm0, %xmm0, %xmm0
+; AVX-NEXT:    vcvtsi2ssq %rax, %xmm0, %xmm0
+; AVX-NEXT:    jmp .LBB39_6
+; AVX-NEXT:  .LBB39_4:
+; AVX-NEXT:    shrq %rax
+; AVX-NEXT:    orq %rax, %rcx
+; AVX-NEXT:    vxorps %xmm0, %xmm0, %xmm0
+; AVX-NEXT:    vcvtsi2ssq %rcx, %xmm0, %xmm0
+; AVX-NEXT:    vaddss %xmm0, %xmm0, %xmm0
+; AVX-NEXT:  .LBB39_6:
+; AVX-NEXT:    vinsertps {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[2,3]
+; AVX-NEXT:    vxorps %xmm1, %xmm1, %xmm1
+; AVX-NEXT:    testq %rax, %rax
+; AVX-NEXT:    js .LBB39_8
+; AVX-NEXT:  # BB#7:
+; AVX-NEXT:    vcvtsi2ssq %rax, %xmm0, %xmm1
+; AVX-NEXT:  .LBB39_8:
+; AVX-NEXT:    vinsertps {{.*#+}} xmm0 = xmm0[0,1],xmm1[0],xmm0[3]
+; AVX-NEXT:    vinsertps {{.*#+}} xmm0 = xmm0[0,1,2],xmm1[0]
+; AVX-NEXT:    retq
+  %ext = shufflevector <2 x i64> %a, <2 x i64> undef, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
+  %cvt = uitofp <4 x i64> %ext to <4 x float>
+  ret <4 x float> %cvt
+}
+
+define <4 x float> @uitofp_4i32_to_4f32(<4 x i32> %a) {
+; SSE-LABEL: uitofp_4i32_to_4f32:
+; SSE:       # BB#0:
+; SSE-NEXT:    movdqa {{.*#+}} xmm1 = [65535,65535,65535,65535]
+; SSE-NEXT:    pand %xmm0, %xmm1
+; SSE-NEXT:    por {{.*}}(%rip), %xmm1
+; SSE-NEXT:    psrld $16, %xmm0
+; SSE-NEXT:    por {{.*}}(%rip), %xmm0
+; SSE-NEXT:    addps {{.*}}(%rip), %xmm0
+; SSE-NEXT:    addps %xmm1, %xmm0
+; SSE-NEXT:    retq
+;
+; AVX1-LABEL: uitofp_4i32_to_4f32:
+; AVX1:       # BB#0:
+; AVX1-NEXT:    vpblendw {{.*#+}} xmm1 = xmm0[0],mem[1],xmm0[2],mem[3],xmm0[4],mem[5],xmm0[6],mem[7]
+; AVX1-NEXT:    vpsrld $16, %xmm0, %xmm0
+; AVX1-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0],mem[1],xmm0[2],mem[3],xmm0[4],mem[5],xmm0[6],mem[7]
+; AVX1-NEXT:    vaddps {{.*}}(%rip), %xmm0, %xmm0
+; AVX1-NEXT:    vaddps %xmm0, %xmm1, %xmm0
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: uitofp_4i32_to_4f32:
+; AVX2:       # BB#0:
+; AVX2-NEXT:    vpbroadcastd {{.*}}(%rip), %xmm1
+; AVX2-NEXT:    vpblendw {{.*#+}} xmm1 = xmm0[0],xmm1[1],xmm0[2],xmm1[3],xmm0[4],xmm1[5],xmm0[6],xmm1[7]
+; AVX2-NEXT:    vpsrld $16, %xmm0, %xmm0
+; AVX2-NEXT:    vpbroadcastd {{.*}}(%rip), %xmm2
+; AVX2-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0],xmm2[1],xmm0[2],xmm2[3],xmm0[4],xmm2[5],xmm0[6],xmm2[7]
+; AVX2-NEXT:    vbroadcastss {{.*}}(%rip), %xmm2
+; AVX2-NEXT:    vaddps %xmm2, %xmm0, %xmm0
+; AVX2-NEXT:    vaddps %xmm0, %xmm1, %xmm0
+; AVX2-NEXT:    retq
+  %cvt = uitofp <4 x i32> %a to <4 x float>
+  ret <4 x float> %cvt
+}
+
+define <4 x float> @uitofp_4i16_to_4f32(<8 x i16> %a) {
+; SSE-LABEL: uitofp_4i16_to_4f32:
+; SSE:       # BB#0:
+; SSE-NEXT:    pxor %xmm1, %xmm1
+; SSE-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
+; SSE-NEXT:    cvtdq2ps %xmm0, %xmm0
+; SSE-NEXT:    retq
+;
+; AVX-LABEL: uitofp_4i16_to_4f32:
+; AVX:       # BB#0:
+; AVX-NEXT:    vpmovzxwd {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero
+; AVX-NEXT:    vcvtdq2ps %xmm0, %xmm0
+; AVX-NEXT:    retq
+  %shuf = shufflevector <8 x i16> %a, <8 x i16> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %cvt = uitofp <4 x i16> %shuf to <4 x float>
+  ret <4 x float> %cvt
+}
+
+define <4 x float> @uitofp_8i16_to_4f32(<8 x i16> %a) {
+; SSE-LABEL: uitofp_8i16_to_4f32:
+; SSE:       # BB#0:
+; SSE-NEXT:    pxor %xmm1, %xmm1
+; SSE-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
+; SSE-NEXT:    cvtdq2ps %xmm0, %xmm0
+; SSE-NEXT:    retq
+;
+; AVX1-LABEL: uitofp_8i16_to_4f32:
+; AVX1:       # BB#0:
+; AVX1-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; AVX1-NEXT:    vpunpckhwd {{.*#+}} xmm1 = xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7]
+; AVX1-NEXT:    vpmovzxwd {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero
+; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
+; AVX1-NEXT:    vcvtdq2ps %ymm0, %ymm0
+; AVX1-NEXT:    vzeroupper
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: uitofp_8i16_to_4f32:
+; AVX2:       # BB#0:
+; AVX2-NEXT:    vpmovzxwd {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
+; AVX2-NEXT:    vcvtdq2ps %ymm0, %ymm0
+; AVX2-NEXT:    vzeroupper
+; AVX2-NEXT:    retq
+  %cvt = uitofp <8 x i16> %a to <8 x float>
+  %shuf = shufflevector <8 x float> %cvt, <8 x float> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  ret <4 x float> %shuf
+}
+
+define <4 x float> @uitofp_4i8_to_4f32(<16 x i8> %a) {
+; SSE-LABEL: uitofp_4i8_to_4f32:
+; SSE:       # BB#0:
+; SSE-NEXT:    pxor %xmm1, %xmm1
+; SSE-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3],xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7]
+; SSE-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
+; SSE-NEXT:    cvtdq2ps %xmm0, %xmm0
+; SSE-NEXT:    retq
+;
+; AVX-LABEL: uitofp_4i8_to_4f32:
+; AVX:       # BB#0:
+; AVX-NEXT:    vpmovzxbd {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
+; AVX-NEXT:    vcvtdq2ps %xmm0, %xmm0
+; AVX-NEXT:    retq
+  %shuf = shufflevector <16 x i8> %a, <16 x i8> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %cvt = uitofp <4 x i8> %shuf to <4 x float>
+  ret <4 x float> %cvt
+}
+
+define <4 x float> @uitofp_16i8_to_4f32(<16 x i8> %a) {
+; SSE-LABEL: uitofp_16i8_to_4f32:
+; SSE:       # BB#0:
+; SSE-NEXT:    pxor %xmm1, %xmm1
+; SSE-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3],xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7]
+; SSE-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
+; SSE-NEXT:    cvtdq2ps %xmm0, %xmm0
+; SSE-NEXT:    retq
+;
+; AVX1-LABEL: uitofp_16i8_to_4f32:
+; AVX1:       # BB#0:
+; AVX1-NEXT:    vpmovzxbw {{.*#+}} xmm1 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
+; AVX1-NEXT:    vpxor %xmm2, %xmm2, %xmm2
+; AVX1-NEXT:    vpunpckhwd {{.*#+}} xmm1 = xmm1[4],xmm2[4],xmm1[5],xmm2[5],xmm1[6],xmm2[6],xmm1[7],xmm2[7]
+; AVX1-NEXT:    vpmovzxbd {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
+; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
+; AVX1-NEXT:    vcvtdq2ps %ymm0, %ymm0
+; AVX1-NEXT:    vzeroupper
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: uitofp_16i8_to_4f32:
+; AVX2:       # BB#0:
+; AVX2-NEXT:    vpmovzxbw {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero,xmm0[8],zero,xmm0[9],zero,xmm0[10],zero,xmm0[11],zero,xmm0[12],zero,xmm0[13],zero,xmm0[14],zero,xmm0[15],zero
+; AVX2-NEXT:    vpmovzxwd {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
+; AVX2-NEXT:    vcvtdq2ps %ymm0, %ymm0
+; AVX2-NEXT:    vzeroupper
+; AVX2-NEXT:    retq
+  %cvt = uitofp <16 x i8> %a to <16 x float>
+  %shuf = shufflevector <16 x float> %cvt, <16 x float> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  ret <4 x float> %shuf
+}
+
+define <4 x float> @uitofp_4i64_to_4f32(<4 x i64> %a) {
+; SSE-LABEL: uitofp_4i64_to_4f32:
+; SSE:       # BB#0:
+; SSE-NEXT:    movd %xmm1, %rax
+; SSE-NEXT:    movl %eax, %ecx
+; SSE-NEXT:    andl $1, %ecx
+; SSE-NEXT:    testq %rax, %rax
+; SSE-NEXT:    js .LBB45_1
+; SSE-NEXT:  # BB#2:
+; SSE-NEXT:    cvtsi2ssq %rax, %xmm3
+; SSE-NEXT:    jmp .LBB45_3
+; SSE-NEXT:  .LBB45_1:
+; SSE-NEXT:    shrq %rax
+; SSE-NEXT:    orq %rax, %rcx
+; SSE-NEXT:    cvtsi2ssq %rcx, %xmm3
+; SSE-NEXT:    addss %xmm3, %xmm3
+; SSE-NEXT:  .LBB45_3:
+; SSE-NEXT:    movd %xmm0, %rax
+; SSE-NEXT:    movl %eax, %ecx
+; SSE-NEXT:    andl $1, %ecx
+; SSE-NEXT:    testq %rax, %rax
+; SSE-NEXT:    js .LBB45_4
+; SSE-NEXT:  # BB#5:
+; SSE-NEXT:    cvtsi2ssq %rax, %xmm2
+; SSE-NEXT:    jmp .LBB45_6
+; SSE-NEXT:  .LBB45_4:
+; SSE-NEXT:    shrq %rax
+; SSE-NEXT:    orq %rax, %rcx
+; SSE-NEXT:    cvtsi2ssq %rcx, %xmm2
+; SSE-NEXT:    addss %xmm2, %xmm2
+; SSE-NEXT:  .LBB45_6:
+; SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[2,3,0,1]
+; SSE-NEXT:    movd %xmm1, %rax
+; SSE-NEXT:    movl %eax, %ecx
+; SSE-NEXT:    andl $1, %ecx
+; SSE-NEXT:    testq %rax, %rax
+; SSE-NEXT:    js .LBB45_7
+; SSE-NEXT:  # BB#8:
+; SSE-NEXT:    xorps %xmm1, %xmm1
+; SSE-NEXT:    cvtsi2ssq %rax, %xmm1
+; SSE-NEXT:    jmp .LBB45_9
+; SSE-NEXT:  .LBB45_7:
+; SSE-NEXT:    shrq %rax
+; SSE-NEXT:    orq %rax, %rcx
+; SSE-NEXT:    xorps %xmm1, %xmm1
+; SSE-NEXT:    cvtsi2ssq %rcx, %xmm1
+; SSE-NEXT:    addss %xmm1, %xmm1
+; SSE-NEXT:  .LBB45_9:
+; SSE-NEXT:    unpcklps {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1]
+; SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
+; SSE-NEXT:    movd %xmm0, %rax
+; SSE-NEXT:    movl %eax, %ecx
+; SSE-NEXT:    andl $1, %ecx
+; SSE-NEXT:    testq %rax, %rax
+; SSE-NEXT:    js .LBB45_10
+; SSE-NEXT:  # BB#11:
+; SSE-NEXT:    xorps %xmm0, %xmm0
+; SSE-NEXT:    cvtsi2ssq %rax, %xmm0
+; SSE-NEXT:    jmp .LBB45_12
+; SSE-NEXT:  .LBB45_10:
+; SSE-NEXT:    shrq %rax
+; SSE-NEXT:    orq %rax, %rcx
+; SSE-NEXT:    xorps %xmm0, %xmm0
+; SSE-NEXT:    cvtsi2ssq %rcx, %xmm0
+; SSE-NEXT:    addss %xmm0, %xmm0
+; SSE-NEXT:  .LBB45_12:
+; SSE-NEXT:    unpcklps {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
+; SSE-NEXT:    unpcklps {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1]
+; SSE-NEXT:    movaps %xmm2, %xmm0
+; SSE-NEXT:    retq
+;
+; AVX1-LABEL: uitofp_4i64_to_4f32:
+; AVX1:       # BB#0:
+; AVX1-NEXT:    vpextrq $1, %xmm0, %rax
+; AVX1-NEXT:    movl %eax, %ecx
+; AVX1-NEXT:    andl $1, %ecx
+; AVX1-NEXT:    testq %rax, %rax
+; AVX1-NEXT:    js .LBB45_1
+; AVX1-NEXT:  # BB#2:
+; AVX1-NEXT:    vcvtsi2ssq %rax, %xmm0, %xmm1
+; AVX1-NEXT:    jmp .LBB45_3
+; AVX1-NEXT:  .LBB45_1:
+; AVX1-NEXT:    shrq %rax
+; AVX1-NEXT:    orq %rax, %rcx
+; AVX1-NEXT:    vcvtsi2ssq %rcx, %xmm0, %xmm1
+; AVX1-NEXT:    vaddss %xmm1, %xmm1, %xmm1
+; AVX1-NEXT:  .LBB45_3:
+; AVX1-NEXT:    vmovq %xmm0, %rax
+; AVX1-NEXT:    movl %eax, %ecx
+; AVX1-NEXT:    andl $1, %ecx
+; AVX1-NEXT:    testq %rax, %rax
+; AVX1-NEXT:    js .LBB45_4
+; AVX1-NEXT:  # BB#5:
+; AVX1-NEXT:    vcvtsi2ssq %rax, %xmm0, %xmm2
+; AVX1-NEXT:    jmp .LBB45_6
+; AVX1-NEXT:  .LBB45_4:
+; AVX1-NEXT:    shrq %rax
+; AVX1-NEXT:    orq %rax, %rcx
+; AVX1-NEXT:    vcvtsi2ssq %rcx, %xmm0, %xmm2
+; AVX1-NEXT:    vaddss %xmm2, %xmm2, %xmm2
+; AVX1-NEXT:  .LBB45_6:
+; AVX1-NEXT:    vinsertps {{.*#+}} xmm1 = xmm2[0],xmm1[0],xmm2[2,3]
+; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm0
+; AVX1-NEXT:    vmovq %xmm0, %rax
+; AVX1-NEXT:    movl %eax, %ecx
+; AVX1-NEXT:    andl $1, %ecx
+; AVX1-NEXT:    testq %rax, %rax
+; AVX1-NEXT:    js .LBB45_7
+; AVX1-NEXT:  # BB#8:
+; AVX1-NEXT:    vcvtsi2ssq %rax, %xmm0, %xmm2
+; AVX1-NEXT:    jmp .LBB45_9
+; AVX1-NEXT:  .LBB45_7:
+; AVX1-NEXT:    shrq %rax
+; AVX1-NEXT:    orq %rax, %rcx
+; AVX1-NEXT:    vcvtsi2ssq %rcx, %xmm0, %xmm2
+; AVX1-NEXT:    vaddss %xmm2, %xmm2, %xmm2
+; AVX1-NEXT:  .LBB45_9:
+; AVX1-NEXT:    vinsertps {{.*#+}} xmm1 = xmm1[0,1],xmm2[0],xmm1[3]
+; AVX1-NEXT:    vpextrq $1, %xmm0, %rax
+; AVX1-NEXT:    movl %eax, %ecx
+; AVX1-NEXT:    andl $1, %ecx
+; AVX1-NEXT:    testq %rax, %rax
+; AVX1-NEXT:    js .LBB45_10
+; AVX1-NEXT:  # BB#11:
+; AVX1-NEXT:    vxorps %xmm0, %xmm0, %xmm0
+; AVX1-NEXT:    vcvtsi2ssq %rax, %xmm0, %xmm0
+; AVX1-NEXT:    vinsertps {{.*#+}} xmm0 = xmm1[0,1,2],xmm0[0]
+; AVX1-NEXT:    vzeroupper
+; AVX1-NEXT:    retq
+; AVX1-NEXT:  .LBB45_10:
+; AVX1-NEXT:    shrq %rax
+; AVX1-NEXT:    orq %rax, %rcx
+; AVX1-NEXT:    vcvtsi2ssq %rcx, %xmm0, %xmm0
+; AVX1-NEXT:    vaddss %xmm0, %xmm0, %xmm0
+; AVX1-NEXT:    vinsertps {{.*#+}} xmm0 = xmm1[0,1,2],xmm0[0]
+; AVX1-NEXT:    vzeroupper
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: uitofp_4i64_to_4f32:
+; AVX2:       # BB#0:
+; AVX2-NEXT:    vpextrq $1, %xmm0, %rax
+; AVX2-NEXT:    movl %eax, %ecx
+; AVX2-NEXT:    andl $1, %ecx
+; AVX2-NEXT:    testq %rax, %rax
+; AVX2-NEXT:    js .LBB45_1
+; AVX2-NEXT:  # BB#2:
+; AVX2-NEXT:    vcvtsi2ssq %rax, %xmm0, %xmm1
+; AVX2-NEXT:    jmp .LBB45_3
+; AVX2-NEXT:  .LBB45_1:
+; AVX2-NEXT:    shrq %rax
+; AVX2-NEXT:    orq %rax, %rcx
+; AVX2-NEXT:    vcvtsi2ssq %rcx, %xmm0, %xmm1
+; AVX2-NEXT:    vaddss %xmm1, %xmm1, %xmm1
+; AVX2-NEXT:  .LBB45_3:
+; AVX2-NEXT:    vmovq %xmm0, %rax
+; AVX2-NEXT:    movl %eax, %ecx
+; AVX2-NEXT:    andl $1, %ecx
+; AVX2-NEXT:    testq %rax, %rax
+; AVX2-NEXT:    js .LBB45_4
+; AVX2-NEXT:  # BB#5:
+; AVX2-NEXT:    vcvtsi2ssq %rax, %xmm0, %xmm2
+; AVX2-NEXT:    jmp .LBB45_6
+; AVX2-NEXT:  .LBB45_4:
+; AVX2-NEXT:    shrq %rax
+; AVX2-NEXT:    orq %rax, %rcx
+; AVX2-NEXT:    vcvtsi2ssq %rcx, %xmm0, %xmm2
+; AVX2-NEXT:    vaddss %xmm2, %xmm2, %xmm2
+; AVX2-NEXT:  .LBB45_6:
+; AVX2-NEXT:    vinsertps {{.*#+}} xmm1 = xmm2[0],xmm1[0],xmm2[2,3]
+; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm0
+; AVX2-NEXT:    vmovq %xmm0, %rax
+; AVX2-NEXT:    movl %eax, %ecx
+; AVX2-NEXT:    andl $1, %ecx
+; AVX2-NEXT:    testq %rax, %rax
+; AVX2-NEXT:    js .LBB45_7
+; AVX2-NEXT:  # BB#8:
+; AVX2-NEXT:    vcvtsi2ssq %rax, %xmm0, %xmm2
+; AVX2-NEXT:    jmp .LBB45_9
+; AVX2-NEXT:  .LBB45_7:
+; AVX2-NEXT:    shrq %rax
+; AVX2-NEXT:    orq %rax, %rcx
+; AVX2-NEXT:    vcvtsi2ssq %rcx, %xmm0, %xmm2
+; AVX2-NEXT:    vaddss %xmm2, %xmm2, %xmm2
+; AVX2-NEXT:  .LBB45_9:
+; AVX2-NEXT:    vinsertps {{.*#+}} xmm1 = xmm1[0,1],xmm2[0],xmm1[3]
+; AVX2-NEXT:    vpextrq $1, %xmm0, %rax
+; AVX2-NEXT:    movl %eax, %ecx
+; AVX2-NEXT:    andl $1, %ecx
+; AVX2-NEXT:    testq %rax, %rax
+; AVX2-NEXT:    js .LBB45_10
+; AVX2-NEXT:  # BB#11:
+; AVX2-NEXT:    vxorps %xmm0, %xmm0, %xmm0
+; AVX2-NEXT:    vcvtsi2ssq %rax, %xmm0, %xmm0
+; AVX2-NEXT:    vinsertps {{.*#+}} xmm0 = xmm1[0,1,2],xmm0[0]
+; AVX2-NEXT:    vzeroupper
+; AVX2-NEXT:    retq
+; AVX2-NEXT:  .LBB45_10:
+; AVX2-NEXT:    shrq %rax
+; AVX2-NEXT:    orq %rax, %rcx
+; AVX2-NEXT:    vcvtsi2ssq %rcx, %xmm0, %xmm0
+; AVX2-NEXT:    vaddss %xmm0, %xmm0, %xmm0
+; AVX2-NEXT:    vinsertps {{.*#+}} xmm0 = xmm1[0,1,2],xmm0[0]
+; AVX2-NEXT:    vzeroupper
+; AVX2-NEXT:    retq
+  %cvt = uitofp <4 x i64> %a to <4 x float>
+  ret <4 x float> %cvt
+}
+
+define <8 x float> @uitofp_8i32_to_8f32(<8 x i32> %a) {
+; SSE-LABEL: uitofp_8i32_to_8f32:
+; SSE:       # BB#0:
+; SSE-NEXT:    movdqa {{.*#+}} xmm2 = [65535,65535,65535,65535]
+; SSE-NEXT:    movdqa %xmm0, %xmm3
+; SSE-NEXT:    pand %xmm2, %xmm3
+; SSE-NEXT:    movdqa {{.*#+}} xmm4 = [1258291200,1258291200,1258291200,1258291200]
+; SSE-NEXT:    por %xmm4, %xmm3
+; SSE-NEXT:    psrld $16, %xmm0
+; SSE-NEXT:    movdqa {{.*#+}} xmm5 = [1392508928,1392508928,1392508928,1392508928]
+; SSE-NEXT:    por %xmm5, %xmm0
+; SSE-NEXT:    movaps {{.*#+}} xmm6 = [-5.497642e+11,-5.497642e+11,-5.497642e+11,-5.497642e+11]
+; SSE-NEXT:    addps %xmm6, %xmm0
+; SSE-NEXT:    addps %xmm3, %xmm0
+; SSE-NEXT:    pand %xmm1, %xmm2
+; SSE-NEXT:    por %xmm4, %xmm2
+; SSE-NEXT:    psrld $16, %xmm1
+; SSE-NEXT:    por %xmm5, %xmm1
+; SSE-NEXT:    addps %xmm6, %xmm1
+; SSE-NEXT:    addps %xmm2, %xmm1
+; SSE-NEXT:    retq
+;
+; AVX1-LABEL: uitofp_8i32_to_8f32:
+; AVX1:       # BB#0:
+; AVX1-NEXT:    vandps {{.*}}(%rip), %ymm0, %ymm1
+; AVX1-NEXT:    vcvtdq2ps %ymm1, %ymm1
+; AVX1-NEXT:    vpsrld $16, %xmm0, %xmm2
+; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm0
+; AVX1-NEXT:    vpsrld $16, %xmm0, %xmm0
+; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm2, %ymm0
+; AVX1-NEXT:    vcvtdq2ps %ymm0, %ymm0
+; AVX1-NEXT:    vmulps {{.*}}(%rip), %ymm0, %ymm0
+; AVX1-NEXT:    vaddps %ymm1, %ymm0, %ymm0
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: uitofp_8i32_to_8f32:
+; AVX2:       # BB#0:
+; AVX2-NEXT:    vpbroadcastd {{.*}}(%rip), %ymm1
+; AVX2-NEXT:    vpblendw {{.*#+}} ymm1 = ymm0[0],ymm1[1],ymm0[2],ymm1[3],ymm0[4],ymm1[5],ymm0[6],ymm1[7],ymm0[8],ymm1[9],ymm0[10],ymm1[11],ymm0[12],ymm1[13],ymm0[14],ymm1[15]
+; AVX2-NEXT:    vpsrld $16, %ymm0, %ymm0
+; AVX2-NEXT:    vpbroadcastd {{.*}}(%rip), %ymm2
+; AVX2-NEXT:    vpblendw {{.*#+}} ymm0 = ymm0[0],ymm2[1],ymm0[2],ymm2[3],ymm0[4],ymm2[5],ymm0[6],ymm2[7],ymm0[8],ymm2[9],ymm0[10],ymm2[11],ymm0[12],ymm2[13],ymm0[14],ymm2[15]
+; AVX2-NEXT:    vbroadcastss {{.*}}(%rip), %ymm2
+; AVX2-NEXT:    vaddps %ymm2, %ymm0, %ymm0
+; AVX2-NEXT:    vaddps %ymm0, %ymm1, %ymm0
+; AVX2-NEXT:    retq
+  %cvt = uitofp <8 x i32> %a to <8 x float>
+  ret <8 x float> %cvt
+}
+
+define <8 x float> @uitofp_8i16_to_8f32(<8 x i16> %a) {
+; SSE-LABEL: uitofp_8i16_to_8f32:
+; SSE:       # BB#0:
+; SSE-NEXT:    pxor %xmm1, %xmm1
+; SSE-NEXT:    movdqa %xmm0, %xmm2
+; SSE-NEXT:    punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3]
+; SSE-NEXT:    cvtdq2ps %xmm2, %xmm2
+; SSE-NEXT:    punpckhwd {{.*#+}} xmm0 = xmm0[4,4,5,5,6,6,7,7]
+; SSE-NEXT:    pand {{.*}}(%rip), %xmm0
+; SSE-NEXT:    cvtdq2ps %xmm0, %xmm1
+; SSE-NEXT:    movaps %xmm2, %xmm0
+; SSE-NEXT:    retq
+;
+; AVX1-LABEL: uitofp_8i16_to_8f32:
+; AVX1:       # BB#0:
+; AVX1-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; AVX1-NEXT:    vpunpckhwd {{.*#+}} xmm1 = xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7]
+; AVX1-NEXT:    vpmovzxwd {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero
+; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
+; AVX1-NEXT:    vcvtdq2ps %ymm0, %ymm0
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: uitofp_8i16_to_8f32:
+; AVX2:       # BB#0:
+; AVX2-NEXT:    vpmovzxwd {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
+; AVX2-NEXT:    vcvtdq2ps %ymm0, %ymm0
+; AVX2-NEXT:    retq
+  %cvt = uitofp <8 x i16> %a to <8 x float>
+  ret <8 x float> %cvt
+}
+
+define <8 x float> @uitofp_8i8_to_8f32(<16 x i8> %a) {
+; SSE-LABEL: uitofp_8i8_to_8f32:
+; SSE:       # BB#0:
+; SSE-NEXT:    pxor %xmm1, %xmm1
+; SSE-NEXT:    movdqa %xmm0, %xmm2
+; SSE-NEXT:    punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3],xmm2[4],xmm1[4],xmm2[5],xmm1[5],xmm2[6],xmm1[6],xmm2[7],xmm1[7]
+; SSE-NEXT:    punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3]
+; SSE-NEXT:    cvtdq2ps %xmm2, %xmm2
+; SSE-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
+; SSE-NEXT:    punpckhwd {{.*#+}} xmm0 = xmm0[4,4,5,5,6,6,7,7]
+; SSE-NEXT:    pand {{.*}}(%rip), %xmm0
+; SSE-NEXT:    cvtdq2ps %xmm0, %xmm1
+; SSE-NEXT:    movaps %xmm2, %xmm0
+; SSE-NEXT:    retq
+;
+; AVX1-LABEL: uitofp_8i8_to_8f32:
+; AVX1:       # BB#0:
+; AVX1-NEXT:    vpmovzxbw {{.*#+}} xmm1 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
+; AVX1-NEXT:    vpmovzxbd {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
+; AVX1-NEXT:    vpunpckhwd {{.*#+}} xmm1 = xmm1[4,4,5,5,6,6,7,7]
+; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
+; AVX1-NEXT:    vandps {{.*}}(%rip), %ymm0, %ymm0
+; AVX1-NEXT:    vcvtdq2ps %ymm0, %ymm0
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: uitofp_8i8_to_8f32:
+; AVX2:       # BB#0:
+; AVX2-NEXT:    vpmovzxbd {{.*#+}} ymm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero,xmm0[4],zero,zero,zero,xmm0[5],zero,zero,zero,xmm0[6],zero,zero,zero,xmm0[7],zero,zero,zero
+; AVX2-NEXT:    vpbroadcastd {{.*}}(%rip), %ymm1
+; AVX2-NEXT:    vpand %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vcvtdq2ps %ymm0, %ymm0
+; AVX2-NEXT:    retq
+  %shuf = shufflevector <16 x i8> %a, <16 x i8> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+  %cvt = uitofp <8 x i8> %shuf to <8 x float>
+  ret <8 x float> %cvt
+}
+
+define <8 x float> @uitofp_16i8_to_8f32(<16 x i8> %a) {
+; SSE-LABEL: uitofp_16i8_to_8f32:
+; SSE:       # BB#0:
+; SSE-NEXT:    pxor %xmm1, %xmm1
+; SSE-NEXT:    movdqa %xmm0, %xmm2
+; SSE-NEXT:    punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3],xmm2[4],xmm1[4],xmm2[5],xmm1[5],xmm2[6],xmm1[6],xmm2[7],xmm1[7]
+; SSE-NEXT:    punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3]
+; SSE-NEXT:    cvtdq2ps %xmm2, %xmm2
+; SSE-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
+; SSE-NEXT:    punpckhwd {{.*#+}} xmm0 = xmm0[4,4,5,5,6,6,7,7]
+; SSE-NEXT:    pand {{.*}}(%rip), %xmm0
+; SSE-NEXT:    cvtdq2ps %xmm0, %xmm1
+; SSE-NEXT:    movaps %xmm2, %xmm0
+; SSE-NEXT:    retq
+;
+; AVX1-LABEL: uitofp_16i8_to_8f32:
+; AVX1:       # BB#0:
+; AVX1-NEXT:    vpmovzxbw {{.*#+}} xmm1 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
+; AVX1-NEXT:    vpxor %xmm2, %xmm2, %xmm2
+; AVX1-NEXT:    vpunpckhwd {{.*#+}} xmm1 = xmm1[4],xmm2[4],xmm1[5],xmm2[5],xmm1[6],xmm2[6],xmm1[7],xmm2[7]
+; AVX1-NEXT:    vpmovzxbd {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
+; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
+; AVX1-NEXT:    vcvtdq2ps %ymm0, %ymm0
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: uitofp_16i8_to_8f32:
+; AVX2:       # BB#0:
+; AVX2-NEXT:    vpmovzxbw {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero,xmm0[8],zero,xmm0[9],zero,xmm0[10],zero,xmm0[11],zero,xmm0[12],zero,xmm0[13],zero,xmm0[14],zero,xmm0[15],zero
+; AVX2-NEXT:    vpmovzxwd {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
+; AVX2-NEXT:    vcvtdq2ps %ymm0, %ymm0
+; AVX2-NEXT:    retq
+  %cvt = uitofp <16 x i8> %a to <16 x float>
+  %shuf = shufflevector <16 x float> %cvt, <16 x float> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+  ret <8 x float> %shuf
+}
+
+;
+; Aggregates
+;
+
+%Arguments = type <{ <8 x i8>, <8 x i16>, <8 x float>* }>
+define void @aggregate_sitofp_8i16_to_8f32(%Arguments* nocapture readonly %a0) {
+; SSE-LABEL: aggregate_sitofp_8i16_to_8f32:
+; SSE:       # BB#0:
+; SSE-NEXT:    movq 24(%rdi), %rax
+; SSE-NEXT:    movdqu 8(%rdi), %xmm0
+; SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; SSE-NEXT:    punpcklwd {{.*#+}} xmm1 = xmm1[0,0,1,1,2,2,3,3]
+; SSE-NEXT:    psrad $16, %xmm1
+; SSE-NEXT:    cvtdq2ps %xmm1, %xmm1
+; SSE-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3]
+; SSE-NEXT:    psrad $16, %xmm0
+; SSE-NEXT:    cvtdq2ps %xmm0, %xmm0
+; SSE-NEXT:    movaps %xmm0, (%rax)
+; SSE-NEXT:    movaps %xmm1, 16(%rax)
+; SSE-NEXT:    retq
+;
+; AVX1-LABEL: aggregate_sitofp_8i16_to_8f32:
+; AVX1:       # BB#0:
+; AVX1-NEXT:    movq 24(%rdi), %rax
+; AVX1-NEXT:    vmovdqu 8(%rdi), %xmm0
+; AVX1-NEXT:    vpmovsxwd %xmm0, %xmm1
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
+; AVX1-NEXT:    vpmovsxwd %xmm0, %xmm0
+; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
+; AVX1-NEXT:    vcvtdq2ps %ymm0, %ymm0
+; AVX1-NEXT:    vmovaps %ymm0, (%rax)
+; AVX1-NEXT:    vzeroupper
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: aggregate_sitofp_8i16_to_8f32:
+; AVX2:       # BB#0:
+; AVX2-NEXT:    movq 24(%rdi), %rax
+; AVX2-NEXT:    vpmovsxwd 8(%rdi), %ymm0
+; AVX2-NEXT:    vcvtdq2ps %ymm0, %ymm0
+; AVX2-NEXT:    vmovaps %ymm0, (%rax)
+; AVX2-NEXT:    vzeroupper
+; AVX2-NEXT:    retq
+ %1 = load %Arguments, %Arguments* %a0, align 1
+ %2 = extractvalue %Arguments %1, 1
+ %3 = extractvalue %Arguments %1, 2
+ %4 = sitofp <8 x i16> %2 to <8 x float>
+ store <8 x float> %4, <8 x float>* %3, align 32
+ ret void
+}
diff --git a/test/CodeGen/X86/vector-ctpop.ll b/test/CodeGen/X86/vector-ctpop.ll
deleted file mode 100644
index 59d67928c6f..00000000000
--- a/test/CodeGen/X86/vector-ctpop.ll
+++ /dev/null
@@ -1,159 +0,0 @@
-; RUN: llc < %s -mtriple=x86_64-apple-darwin -mattr=avx2                | FileCheck -check-prefix=AVX2 %s
-; RUN: llc < %s -mtriple=x86_64-apple-darwin -mattr=avx  -mattr=-popcnt | FileCheck -check-prefix=AVX1-NOPOPCNT %s
-; RUN: llc < %s -mtriple=x86_64-apple-darwin -mattr=avx2 -mattr=-popcnt | FileCheck -check-prefix=AVX2-NOPOPCNT %s
-
-; Vector version of:
-; v = v - ((v >> 1) & 0x55555555)
-; v = (v & 0x33333333) + ((v >> 2) & 0x33333333)
-; v = (v + (v >> 4) & 0xF0F0F0F)
-; v = v + (v >> 8)
-; v = v + (v >> 16)
-; v = v + (v >> 32) ; i64 only
-
-define <8 x i32> @test0(<8 x i32> %x) {
-; AVX2-LABEL: @test0
-entry:
-; AVX2:  vpsrld  $1, %ymm
-; AVX2-NEXT:  vpbroadcastd
-; AVX2-NEXT:  vpand
-; AVX2-NEXT:  vpsubd
-; AVX2-NEXT:  vpbroadcastd
-; AVX2-NEXT:  vpand
-; AVX2-NEXT:  vpsrld  $2
-; AVX2-NEXT:  vpand
-; AVX2-NEXT:  vpaddd
-; AVX2-NEXT:  vpsrld  $4
-; AVX2-NEXT:  vpaddd
-; AVX2-NEXT:  vpbroadcastd
-; AVX2-NEXT:	vpand
-; AVX2-NEXT:	vpsrld	$8
-; AVX2-NEXT:	vpaddd
-; AVX2-NEXT:	vpsrld	$16
-; AVX2-NEXT:	vpaddd
-; AVX2-NEXT:	vpbroadcastd
-; AVX2-NEXT:	vpand
-  %y = call <8 x i32> @llvm.ctpop.v8i32(<8 x i32> %x)
-  ret <8 x i32> %y
-}
-
-define <4 x i64> @test1(<4 x i64> %x) {
-; AVX2-NOPOPCNT-LABEL: @test1
-entry:
-;	AVX2-NOPOPCNT: vpsrlq	$1, %ymm
-;	AVX2-NOPOPCNT-NEXT: vpbroadcastq
-;	AVX2-NOPOPCNT-NEXT: vpand
-;	AVX2-NOPOPCNT-NEXT: vpsubq
-;	AVX2-NOPOPCNT-NEXT: vpbroadcastq
-;	AVX2-NOPOPCNT-NEXT: vpand
-;	AVX2-NOPOPCNT-NEXT: vpsrlq	$2
-;	AVX2-NOPOPCNT-NEXT: vpand
-;	AVX2-NOPOPCNT-NEXT: vpaddq
-;	AVX2-NOPOPCNT-NEXT: vpsrlq	$4
-;	AVX2-NOPOPCNT-NEXT: vpaddq
-;	AVX2-NOPOPCNT-NEXT: vpbroadcastq
-;	AVX2-NOPOPCNT-NEXT: vpand
-;	AVX2-NOPOPCNT-NEXT: vpsrlq	$8
-;	AVX2-NOPOPCNT-NEXT: vpaddq
-;	AVX2-NOPOPCNT-NEXT: vpsrlq	$16
-;	AVX2-NOPOPCNT-NEXT: vpaddq
-;	AVX2-NOPOPCNT-NEXT: vpsrlq	$32
-;	AVX2-NOPOPCNT-NEXT: vpaddq
-;	AVX2-NOPOPCNT-NEXT: vpbroadcastq
-;	AVX2-NOPOPCNT-NEXT: vpand
-  %y = call <4 x i64> @llvm.ctpop.v4i64(<4 x i64> %x)
-  ret <4 x i64> %y
-}
-
-define <4 x i32> @test2(<4 x i32> %x) {
-; AVX2-NOPOPCNT-LABEL: @test2
-; AVX1-NOPOPCNT-LABEL: @test2
-entry:
-; AVX2-NOPOPCNT:	vpsrld	$1, %xmm
-; AVX2-NOPOPCNT-NEXT:	vpbroadcastd
-; AVX2-NOPOPCNT-NEXT:	vpand
-; AVX2-NOPOPCNT-NEXT:	vpsubd
-; AVX2-NOPOPCNT-NEXT:	vpbroadcastd
-; AVX2-NOPOPCNT-NEXT:	vpand
-; AVX2-NOPOPCNT-NEXT:	vpsrld	$2
-; AVX2-NOPOPCNT-NEXT:	vpand
-; AVX2-NOPOPCNT-NEXT:	vpaddd
-; AVX2-NOPOPCNT-NEXT:	vpsrld	$4
-; AVX2-NOPOPCNT-NEXT:	vpaddd
-; AVX2-NOPOPCNT-NEXT:	vpbroadcastd
-; AVX2-NOPOPCNT-NEXT:	vpand
-; AVX2-NOPOPCNT-NEXT:	vpsrld	$8
-; AVX2-NOPOPCNT-NEXT:	vpaddd
-; AVX2-NOPOPCNT-NEXT:	vpsrld	$16
-; AVX2-NOPOPCNT-NEXT:	vpaddd
-; AVX2-NOPOPCNT-NEXT:	vpbroadcastd
-; AVX2-NOPOPCNT-NEXT:	vpand
-; AVX1-NOPOPCNT:	vpsrld	$1, %xmm
-; AVX1-NOPOPCNT-NEXT:	vpand
-; AVX1-NOPOPCNT-NEXT:	vpsubd
-; AVX1-NOPOPCNT-NEXT:	vmovdqa
-; AVX1-NOPOPCNT-NEXT:	vpand
-; AVX1-NOPOPCNT-NEXT:	vpsrld	$2
-; AVX1-NOPOPCNT-NEXT:	vpand
-; AVX1-NOPOPCNT-NEXT:	vpaddd
-; AVX1-NOPOPCNT-NEXT:	vpsrld	$4
-; AVX1-NOPOPCNT-NEXT:	vpaddd
-; AVX1-NOPOPCNT-NEXT:	vpand
-; AVX1-NOPOPCNT-NEXT:	vpsrld	$8
-; AVX1-NOPOPCNT-NEXT:	vpaddd
-; AVX1-NOPOPCNT-NEXT:	vpsrld	$16
-; AVX1-NOPOPCNT-NEXT:	vpaddd
-; AVX1-NOPOPCNT-NEXT:	vpand
-  %y = call <4 x i32> @llvm.ctpop.v4i32(<4 x i32> %x)
-  ret <4 x i32> %y
-}
-
-define <2 x i64> @test3(<2 x i64> %x) {
-; AVX2-NOPOPCNT-LABEL: @test3
-; AVX1-NOPOPCNT-LABEL: @test3
-entry:
-; AVX2-NOPOPCNT:	vpsrlq	$1, %xmm
-; AVX2-NOPOPCNT-NEXT:	vpand
-; AVX2-NOPOPCNT-NEXT:	vpsubq
-; AVX2-NOPOPCNT-NEXT:	vmovdqa
-; AVX2-NOPOPCNT-NEXT:	vpand
-; AVX2-NOPOPCNT-NEXT:	vpsrlq	$2
-; AVX2-NOPOPCNT-NEXT:	vpand
-; AVX2-NOPOPCNT-NEXT:	vpaddq
-; AVX2-NOPOPCNT-NEXT:	vpsrlq	$4
-; AVX2-NOPOPCNT-NEXT:	vpaddq
-; AVX2-NOPOPCNT-NEXT:	vpand
-; AVX2-NOPOPCNT-NEXT:	vpsrlq	$8
-; AVX2-NOPOPCNT-NEXT:	vpaddq
-; AVX2-NOPOPCNT-NEXT:	vpsrlq	$16
-; AVX2-NOPOPCNT-NEXT:	vpaddq
-; AVX2-NOPOPCNT-NEXT:	vpsrlq	$32
-; AVX2-NOPOPCNT-NEXT:	vpaddq
-; AVX2-NOPOPCNT-NEXT:	vpand
-; AVX1-NOPOPCNT:	vpsrlq	$1, %xmm
-; AVX1-NOPOPCNT-NEXT:	vpand
-; AVX1-NOPOPCNT-NEXT:	vpsubq
-; AVX1-NOPOPCNT-NEXT:	vmovdqa
-; AVX1-NOPOPCNT-NEXT:	vpand
-; AVX1-NOPOPCNT-NEXT:	vpsrlq	$2
-; AVX1-NOPOPCNT-NEXT:	vpand
-; AVX1-NOPOPCNT-NEXT:	vpaddq
-; AVX1-NOPOPCNT-NEXT:	vpsrlq	$4
-; AVX1-NOPOPCNT-NEXT:	vpaddq
-; AVX1-NOPOPCNT-NEXT:	vpand
-; AVX1-NOPOPCNT-NEXT:	vpsrlq	$8
-; AVX1-NOPOPCNT-NEXT:	vpaddq
-; AVX1-NOPOPCNT-NEXT:	vpsrlq	$16
-; AVX1-NOPOPCNT-NEXT:	vpaddq
-; AVX1-NOPOPCNT-NEXT:	vpsrlq	$32
-; AVX1-NOPOPCNT-NEXT:	vpaddq
-; AVX1-NOPOPCNT-NEXT:	vpand
-  %y = call <2 x i64> @llvm.ctpop.v2i64(<2 x i64> %x)
-  ret <2 x i64> %y
-}
-
-declare <4 x i32> @llvm.ctpop.v4i32(<4 x i32>)
-declare <2 x i64> @llvm.ctpop.v2i64(<2 x i64>)
-
-declare <8 x i32> @llvm.ctpop.v8i32(<8 x i32>)
-declare <4 x i64> @llvm.ctpop.v4i64(<4 x i64>)
-
diff --git a/test/CodeGen/X86/vector-gep.ll b/test/CodeGen/X86/vector-gep.ll
index ce98e6759b6..47878360ca0 100644
--- a/test/CodeGen/X86/vector-gep.ll
+++ b/test/CodeGen/X86/vector-gep.ll
@@ -92,3 +92,25 @@ entry:
 ;CHECK: ret
 }
 
+;CHECK-LABEL: AGEP7:
+define <4 x i8*> @AGEP7(<4 x i8*> %param, i32 %off) nounwind {
+entry:
+;CHECK: vbroadcastss
+;CHECK: vpadd
+  %A = getelementptr i8, <4 x i8*> %param, i32 %off
+  ret <4 x i8*> %A
+;CHECK: ret
+}
+
+;CHECK-LABEL: AGEP8:
+define <4 x i16*> @AGEP8(i16* %param, <4 x i32> %off) nounwind {
+entry:
+; Multiply offset by two (add it to itself).
+;CHECK: vpadd
+; add the base to the offset
+;CHECK: vbroadcastss
+;CHECK-NEXT: vpadd
+  %A = getelementptr i16, i16* %param, <4 x i32> %off
+  ret <4 x i16*> %A
+;CHECK: ret
+}
diff --git a/test/CodeGen/X86/vector-lzcnt-128.ll b/test/CodeGen/X86/vector-lzcnt-128.ll
new file mode 100644
index 00000000000..c18bcfb95b7
--- /dev/null
+++ b/test/CodeGen/X86/vector-lzcnt-128.ll
@@ -0,0 +1,1917 @@
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mcpu=x86-64 | FileCheck %s --check-prefix=ALL --check-prefix=SSE --check-prefix=SSE2
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mcpu=x86-64 -mattr=+sse3 | FileCheck %s --check-prefix=ALL --check-prefix=SSE --check-prefix=SSE3
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mcpu=x86-64 -mattr=+ssse3 | FileCheck %s --check-prefix=ALL --check-prefix=SSE --check-prefix=SSSE3
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mcpu=x86-64 -mattr=+sse4.1 | FileCheck %s --check-prefix=ALL --check-prefix=SSE --check-prefix=SSE41
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mcpu=x86-64 -mattr=+avx | FileCheck %s --check-prefix=ALL --check-prefix=AVX --check-prefix=AVX1
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mcpu=x86-64 -mattr=+avx2 | FileCheck %s --check-prefix=ALL --check-prefix=AVX --check-prefix=AVX2
+
+target triple = "x86_64-unknown-unknown"
+
+define <2 x i64> @testv2i64(<2 x i64> %in) nounwind {
+; SSE2-LABEL: testv2i64:
+; SSE2:       # BB#0:
+; SSE2-NEXT:    movd %xmm0, %rax
+; SSE2-NEXT:    bsrq %rax, %rax
+; SSE2-NEXT:    movl $127, %ecx
+; SSE2-NEXT:    cmoveq %rcx, %rax
+; SSE2-NEXT:    xorq $63, %rax
+; SSE2-NEXT:    movd %rax, %xmm1
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
+; SSE2-NEXT:    movd %xmm0, %rax
+; SSE2-NEXT:    bsrq %rax, %rax
+; SSE2-NEXT:    cmoveq %rcx, %rax
+; SSE2-NEXT:    xorq $63, %rax
+; SSE2-NEXT:    movd %rax, %xmm0
+; SSE2-NEXT:    punpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm0[0]
+; SSE2-NEXT:    movdqa %xmm1, %xmm0
+; SSE2-NEXT:    retq
+;
+; SSE3-LABEL: testv2i64:
+; SSE3:       # BB#0:
+; SSE3-NEXT:    movd %xmm0, %rax
+; SSE3-NEXT:    bsrq %rax, %rax
+; SSE3-NEXT:    movl $127, %ecx
+; SSE3-NEXT:    cmoveq %rcx, %rax
+; SSE3-NEXT:    xorq $63, %rax
+; SSE3-NEXT:    movd %rax, %xmm1
+; SSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
+; SSE3-NEXT:    movd %xmm0, %rax
+; SSE3-NEXT:    bsrq %rax, %rax
+; SSE3-NEXT:    cmoveq %rcx, %rax
+; SSE3-NEXT:    xorq $63, %rax
+; SSE3-NEXT:    movd %rax, %xmm0
+; SSE3-NEXT:    punpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm0[0]
+; SSE3-NEXT:    movdqa %xmm1, %xmm0
+; SSE3-NEXT:    retq
+;
+; SSSE3-LABEL: testv2i64:
+; SSSE3:       # BB#0:
+; SSSE3-NEXT:    movd %xmm0, %rax
+; SSSE3-NEXT:    bsrq %rax, %rax
+; SSSE3-NEXT:    movl $127, %ecx
+; SSSE3-NEXT:    cmoveq %rcx, %rax
+; SSSE3-NEXT:    xorq $63, %rax
+; SSSE3-NEXT:    movd %rax, %xmm1
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
+; SSSE3-NEXT:    movd %xmm0, %rax
+; SSSE3-NEXT:    bsrq %rax, %rax
+; SSSE3-NEXT:    cmoveq %rcx, %rax
+; SSSE3-NEXT:    xorq $63, %rax
+; SSSE3-NEXT:    movd %rax, %xmm0
+; SSSE3-NEXT:    punpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm0[0]
+; SSSE3-NEXT:    movdqa %xmm1, %xmm0
+; SSSE3-NEXT:    retq
+;
+; SSE41-LABEL: testv2i64:
+; SSE41:       # BB#0:
+; SSE41-NEXT:    pextrq $1, %xmm0, %rax
+; SSE41-NEXT:    bsrq %rax, %rax
+; SSE41-NEXT:    movl $127, %ecx
+; SSE41-NEXT:    cmoveq %rcx, %rax
+; SSE41-NEXT:    xorq $63, %rax
+; SSE41-NEXT:    movd %rax, %xmm1
+; SSE41-NEXT:    movd %xmm0, %rax
+; SSE41-NEXT:    bsrq %rax, %rax
+; SSE41-NEXT:    cmoveq %rcx, %rax
+; SSE41-NEXT:    xorq $63, %rax
+; SSE41-NEXT:    movd %rax, %xmm0
+; SSE41-NEXT:    punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
+; SSE41-NEXT:    retq
+;
+; AVX-LABEL: testv2i64:
+; AVX:       # BB#0:
+; AVX-NEXT:    vpextrq $1, %xmm0, %rax
+; AVX-NEXT:    bsrq %rax, %rax
+; AVX-NEXT:    movl $127, %ecx
+; AVX-NEXT:    cmoveq %rcx, %rax
+; AVX-NEXT:    xorq $63, %rax
+; AVX-NEXT:    vmovq %rax, %xmm1
+; AVX-NEXT:    vmovq %xmm0, %rax
+; AVX-NEXT:    bsrq %rax, %rax
+; AVX-NEXT:    cmoveq %rcx, %rax
+; AVX-NEXT:    xorq $63, %rax
+; AVX-NEXT:    vmovq %rax, %xmm0
+; AVX-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
+; AVX-NEXT:    retq
+  %out = call <2 x i64> @llvm.ctlz.v2i64(<2 x i64> %in, i1 0)
+  ret <2 x i64> %out
+}
+
+define <2 x i64> @testv2i64u(<2 x i64> %in) nounwind {
+; SSE2-LABEL: testv2i64u:
+; SSE2:       # BB#0:
+; SSE2-NEXT:    movd %xmm0, %rax
+; SSE2-NEXT:    bsrq %rax, %rax
+; SSE2-NEXT:    xorq $63, %rax
+; SSE2-NEXT:    movd %rax, %xmm1
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
+; SSE2-NEXT:    movd %xmm0, %rax
+; SSE2-NEXT:    bsrq %rax, %rax
+; SSE2-NEXT:    xorq $63, %rax
+; SSE2-NEXT:    movd %rax, %xmm0
+; SSE2-NEXT:    punpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm0[0]
+; SSE2-NEXT:    movdqa %xmm1, %xmm0
+; SSE2-NEXT:    retq
+;
+; SSE3-LABEL: testv2i64u:
+; SSE3:       # BB#0:
+; SSE3-NEXT:    movd %xmm0, %rax
+; SSE3-NEXT:    bsrq %rax, %rax
+; SSE3-NEXT:    xorq $63, %rax
+; SSE3-NEXT:    movd %rax, %xmm1
+; SSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
+; SSE3-NEXT:    movd %xmm0, %rax
+; SSE3-NEXT:    bsrq %rax, %rax
+; SSE3-NEXT:    xorq $63, %rax
+; SSE3-NEXT:    movd %rax, %xmm0
+; SSE3-NEXT:    punpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm0[0]
+; SSE3-NEXT:    movdqa %xmm1, %xmm0
+; SSE3-NEXT:    retq
+;
+; SSSE3-LABEL: testv2i64u:
+; SSSE3:       # BB#0:
+; SSSE3-NEXT:    movd %xmm0, %rax
+; SSSE3-NEXT:    bsrq %rax, %rax
+; SSSE3-NEXT:    xorq $63, %rax
+; SSSE3-NEXT:    movd %rax, %xmm1
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
+; SSSE3-NEXT:    movd %xmm0, %rax
+; SSSE3-NEXT:    bsrq %rax, %rax
+; SSSE3-NEXT:    xorq $63, %rax
+; SSSE3-NEXT:    movd %rax, %xmm0
+; SSSE3-NEXT:    punpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm0[0]
+; SSSE3-NEXT:    movdqa %xmm1, %xmm0
+; SSSE3-NEXT:    retq
+;
+; SSE41-LABEL: testv2i64u:
+; SSE41:       # BB#0:
+; SSE41-NEXT:    pextrq $1, %xmm0, %rax
+; SSE41-NEXT:    bsrq %rax, %rax
+; SSE41-NEXT:    xorq $63, %rax
+; SSE41-NEXT:    movd %rax, %xmm1
+; SSE41-NEXT:    movd %xmm0, %rax
+; SSE41-NEXT:    bsrq %rax, %rax
+; SSE41-NEXT:    xorq $63, %rax
+; SSE41-NEXT:    movd %rax, %xmm0
+; SSE41-NEXT:    punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
+; SSE41-NEXT:    retq
+;
+; AVX-LABEL: testv2i64u:
+; AVX:       # BB#0:
+; AVX-NEXT:    vpextrq $1, %xmm0, %rax
+; AVX-NEXT:    bsrq %rax, %rax
+; AVX-NEXT:    xorq $63, %rax
+; AVX-NEXT:    vmovq %rax, %xmm1
+; AVX-NEXT:    vmovq %xmm0, %rax
+; AVX-NEXT:    bsrq %rax, %rax
+; AVX-NEXT:    xorq $63, %rax
+; AVX-NEXT:    vmovq %rax, %xmm0
+; AVX-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
+; AVX-NEXT:    retq
+  %out = call <2 x i64> @llvm.ctlz.v2i64(<2 x i64> %in, i1 -1)
+  ret <2 x i64> %out
+}
+
+define <4 x i32> @testv4i32(<4 x i32> %in) nounwind {
+; SSE2-LABEL: testv4i32:
+; SSE2:       # BB#0:
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[3,1,2,3]
+; SSE2-NEXT:    movd %xmm1, %eax
+; SSE2-NEXT:    bsrl %eax, %eax
+; SSE2-NEXT:    movl $63, %ecx
+; SSE2-NEXT:    cmovel %ecx, %eax
+; SSE2-NEXT:    xorl $31, %eax
+; SSE2-NEXT:    movd %eax, %xmm1
+; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[1,1,2,3]
+; SSE2-NEXT:    movd %xmm2, %eax
+; SSE2-NEXT:    bsrl %eax, %eax
+; SSE2-NEXT:    cmovel %ecx, %eax
+; SSE2-NEXT:    xorl $31, %eax
+; SSE2-NEXT:    movd %eax, %xmm2
+; SSE2-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]
+; SSE2-NEXT:    movd %xmm0, %eax
+; SSE2-NEXT:    bsrl %eax, %eax
+; SSE2-NEXT:    cmovel %ecx, %eax
+; SSE2-NEXT:    xorl $31, %eax
+; SSE2-NEXT:    movd %eax, %xmm1
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
+; SSE2-NEXT:    movd %xmm0, %eax
+; SSE2-NEXT:    bsrl %eax, %eax
+; SSE2-NEXT:    cmovel %ecx, %eax
+; SSE2-NEXT:    xorl $31, %eax
+; SSE2-NEXT:    movd %eax, %xmm0
+; SSE2-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
+; SSE2-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1]
+; SSE2-NEXT:    movdqa %xmm1, %xmm0
+; SSE2-NEXT:    retq
+;
+; SSE3-LABEL: testv4i32:
+; SSE3:       # BB#0:
+; SSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[3,1,2,3]
+; SSE3-NEXT:    movd %xmm1, %eax
+; SSE3-NEXT:    bsrl %eax, %eax
+; SSE3-NEXT:    movl $63, %ecx
+; SSE3-NEXT:    cmovel %ecx, %eax
+; SSE3-NEXT:    xorl $31, %eax
+; SSE3-NEXT:    movd %eax, %xmm1
+; SSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[1,1,2,3]
+; SSE3-NEXT:    movd %xmm2, %eax
+; SSE3-NEXT:    bsrl %eax, %eax
+; SSE3-NEXT:    cmovel %ecx, %eax
+; SSE3-NEXT:    xorl $31, %eax
+; SSE3-NEXT:    movd %eax, %xmm2
+; SSE3-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]
+; SSE3-NEXT:    movd %xmm0, %eax
+; SSE3-NEXT:    bsrl %eax, %eax
+; SSE3-NEXT:    cmovel %ecx, %eax
+; SSE3-NEXT:    xorl $31, %eax
+; SSE3-NEXT:    movd %eax, %xmm1
+; SSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
+; SSE3-NEXT:    movd %xmm0, %eax
+; SSE3-NEXT:    bsrl %eax, %eax
+; SSE3-NEXT:    cmovel %ecx, %eax
+; SSE3-NEXT:    xorl $31, %eax
+; SSE3-NEXT:    movd %eax, %xmm0
+; SSE3-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
+; SSE3-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1]
+; SSE3-NEXT:    movdqa %xmm1, %xmm0
+; SSE3-NEXT:    retq
+;
+; SSSE3-LABEL: testv4i32:
+; SSSE3:       # BB#0:
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[3,1,2,3]
+; SSSE3-NEXT:    movd %xmm1, %eax
+; SSSE3-NEXT:    bsrl %eax, %eax
+; SSSE3-NEXT:    movl $63, %ecx
+; SSSE3-NEXT:    cmovel %ecx, %eax
+; SSSE3-NEXT:    xorl $31, %eax
+; SSSE3-NEXT:    movd %eax, %xmm1
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[1,1,2,3]
+; SSSE3-NEXT:    movd %xmm2, %eax
+; SSSE3-NEXT:    bsrl %eax, %eax
+; SSSE3-NEXT:    cmovel %ecx, %eax
+; SSSE3-NEXT:    xorl $31, %eax
+; SSSE3-NEXT:    movd %eax, %xmm2
+; SSSE3-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]
+; SSSE3-NEXT:    movd %xmm0, %eax
+; SSSE3-NEXT:    bsrl %eax, %eax
+; SSSE3-NEXT:    cmovel %ecx, %eax
+; SSSE3-NEXT:    xorl $31, %eax
+; SSSE3-NEXT:    movd %eax, %xmm1
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
+; SSSE3-NEXT:    movd %xmm0, %eax
+; SSSE3-NEXT:    bsrl %eax, %eax
+; SSSE3-NEXT:    cmovel %ecx, %eax
+; SSSE3-NEXT:    xorl $31, %eax
+; SSSE3-NEXT:    movd %eax, %xmm0
+; SSSE3-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
+; SSSE3-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1]
+; SSSE3-NEXT:    movdqa %xmm1, %xmm0
+; SSSE3-NEXT:    retq
+;
+; SSE41-LABEL: testv4i32:
+; SSE41:       # BB#0:
+; SSE41-NEXT:    pextrd $1, %xmm0, %eax
+; SSE41-NEXT:    bsrl %eax, %eax
+; SSE41-NEXT:    movl $63, %ecx
+; SSE41-NEXT:    cmovel %ecx, %eax
+; SSE41-NEXT:    xorl $31, %eax
+; SSE41-NEXT:    movd %xmm0, %edx
+; SSE41-NEXT:    bsrl %edx, %edx
+; SSE41-NEXT:    cmovel %ecx, %edx
+; SSE41-NEXT:    xorl $31, %edx
+; SSE41-NEXT:    movd %edx, %xmm1
+; SSE41-NEXT:    pinsrd $1, %eax, %xmm1
+; SSE41-NEXT:    pextrd $2, %xmm0, %eax
+; SSE41-NEXT:    bsrl %eax, %eax
+; SSE41-NEXT:    cmovel %ecx, %eax
+; SSE41-NEXT:    xorl $31, %eax
+; SSE41-NEXT:    pinsrd $2, %eax, %xmm1
+; SSE41-NEXT:    pextrd $3, %xmm0, %eax
+; SSE41-NEXT:    bsrl %eax, %eax
+; SSE41-NEXT:    cmovel %ecx, %eax
+; SSE41-NEXT:    xorl $31, %eax
+; SSE41-NEXT:    pinsrd $3, %eax, %xmm1
+; SSE41-NEXT:    movdqa %xmm1, %xmm0
+; SSE41-NEXT:    retq
+;
+; AVX-LABEL: testv4i32:
+; AVX:       # BB#0:
+; AVX-NEXT:    vpextrd $1, %xmm0, %eax
+; AVX-NEXT:    bsrl %eax, %eax
+; AVX-NEXT:    movl $63, %ecx
+; AVX-NEXT:    cmovel %ecx, %eax
+; AVX-NEXT:    xorl $31, %eax
+; AVX-NEXT:    vmovd %xmm0, %edx
+; AVX-NEXT:    bsrl %edx, %edx
+; AVX-NEXT:    cmovel %ecx, %edx
+; AVX-NEXT:    xorl $31, %edx
+; AVX-NEXT:    vmovd %edx, %xmm1
+; AVX-NEXT:    vpinsrd $1, %eax, %xmm1, %xmm1
+; AVX-NEXT:    vpextrd $2, %xmm0, %eax
+; AVX-NEXT:    bsrl %eax, %eax
+; AVX-NEXT:    cmovel %ecx, %eax
+; AVX-NEXT:    xorl $31, %eax
+; AVX-NEXT:    vpinsrd $2, %eax, %xmm1, %xmm1
+; AVX-NEXT:    vpextrd $3, %xmm0, %eax
+; AVX-NEXT:    bsrl %eax, %eax
+; AVX-NEXT:    cmovel %ecx, %eax
+; AVX-NEXT:    xorl $31, %eax
+; AVX-NEXT:    vpinsrd $3, %eax, %xmm1, %xmm0
+; AVX-NEXT:    retq
+  %out = call <4 x i32> @llvm.ctlz.v4i32(<4 x i32> %in, i1 0)
+  ret <4 x i32> %out
+}
+
+define <4 x i32> @testv4i32u(<4 x i32> %in) nounwind {
+; SSE2-LABEL: testv4i32u:
+; SSE2:       # BB#0:
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[3,1,2,3]
+; SSE2-NEXT:    movd %xmm1, %eax
+; SSE2-NEXT:    bsrl %eax, %eax
+; SSE2-NEXT:    xorl $31, %eax
+; SSE2-NEXT:    movd %eax, %xmm1
+; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[1,1,2,3]
+; SSE2-NEXT:    movd %xmm2, %eax
+; SSE2-NEXT:    bsrl %eax, %eax
+; SSE2-NEXT:    xorl $31, %eax
+; SSE2-NEXT:    movd %eax, %xmm2
+; SSE2-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]
+; SSE2-NEXT:    movd %xmm0, %eax
+; SSE2-NEXT:    bsrl %eax, %eax
+; SSE2-NEXT:    xorl $31, %eax
+; SSE2-NEXT:    movd %eax, %xmm1
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
+; SSE2-NEXT:    movd %xmm0, %eax
+; SSE2-NEXT:    bsrl %eax, %eax
+; SSE2-NEXT:    xorl $31, %eax
+; SSE2-NEXT:    movd %eax, %xmm0
+; SSE2-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
+; SSE2-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1]
+; SSE2-NEXT:    movdqa %xmm1, %xmm0
+; SSE2-NEXT:    retq
+;
+; SSE3-LABEL: testv4i32u:
+; SSE3:       # BB#0:
+; SSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[3,1,2,3]
+; SSE3-NEXT:    movd %xmm1, %eax
+; SSE3-NEXT:    bsrl %eax, %eax
+; SSE3-NEXT:    xorl $31, %eax
+; SSE3-NEXT:    movd %eax, %xmm1
+; SSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[1,1,2,3]
+; SSE3-NEXT:    movd %xmm2, %eax
+; SSE3-NEXT:    bsrl %eax, %eax
+; SSE3-NEXT:    xorl $31, %eax
+; SSE3-NEXT:    movd %eax, %xmm2
+; SSE3-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]
+; SSE3-NEXT:    movd %xmm0, %eax
+; SSE3-NEXT:    bsrl %eax, %eax
+; SSE3-NEXT:    xorl $31, %eax
+; SSE3-NEXT:    movd %eax, %xmm1
+; SSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
+; SSE3-NEXT:    movd %xmm0, %eax
+; SSE3-NEXT:    bsrl %eax, %eax
+; SSE3-NEXT:    xorl $31, %eax
+; SSE3-NEXT:    movd %eax, %xmm0
+; SSE3-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
+; SSE3-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1]
+; SSE3-NEXT:    movdqa %xmm1, %xmm0
+; SSE3-NEXT:    retq
+;
+; SSSE3-LABEL: testv4i32u:
+; SSSE3:       # BB#0:
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[3,1,2,3]
+; SSSE3-NEXT:    movd %xmm1, %eax
+; SSSE3-NEXT:    bsrl %eax, %eax
+; SSSE3-NEXT:    xorl $31, %eax
+; SSSE3-NEXT:    movd %eax, %xmm1
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[1,1,2,3]
+; SSSE3-NEXT:    movd %xmm2, %eax
+; SSSE3-NEXT:    bsrl %eax, %eax
+; SSSE3-NEXT:    xorl $31, %eax
+; SSSE3-NEXT:    movd %eax, %xmm2
+; SSSE3-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]
+; SSSE3-NEXT:    movd %xmm0, %eax
+; SSSE3-NEXT:    bsrl %eax, %eax
+; SSSE3-NEXT:    xorl $31, %eax
+; SSSE3-NEXT:    movd %eax, %xmm1
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
+; SSSE3-NEXT:    movd %xmm0, %eax
+; SSSE3-NEXT:    bsrl %eax, %eax
+; SSSE3-NEXT:    xorl $31, %eax
+; SSSE3-NEXT:    movd %eax, %xmm0
+; SSSE3-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
+; SSSE3-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1]
+; SSSE3-NEXT:    movdqa %xmm1, %xmm0
+; SSSE3-NEXT:    retq
+;
+; SSE41-LABEL: testv4i32u:
+; SSE41:       # BB#0:
+; SSE41-NEXT:    pextrd $1, %xmm0, %eax
+; SSE41-NEXT:    bsrl %eax, %eax
+; SSE41-NEXT:    xorl $31, %eax
+; SSE41-NEXT:    movd %xmm0, %ecx
+; SSE41-NEXT:    bsrl %ecx, %ecx
+; SSE41-NEXT:    xorl $31, %ecx
+; SSE41-NEXT:    movd %ecx, %xmm1
+; SSE41-NEXT:    pinsrd $1, %eax, %xmm1
+; SSE41-NEXT:    pextrd $2, %xmm0, %eax
+; SSE41-NEXT:    bsrl %eax, %eax
+; SSE41-NEXT:    xorl $31, %eax
+; SSE41-NEXT:    pinsrd $2, %eax, %xmm1
+; SSE41-NEXT:    pextrd $3, %xmm0, %eax
+; SSE41-NEXT:    bsrl %eax, %eax
+; SSE41-NEXT:    xorl $31, %eax
+; SSE41-NEXT:    pinsrd $3, %eax, %xmm1
+; SSE41-NEXT:    movdqa %xmm1, %xmm0
+; SSE41-NEXT:    retq
+;
+; AVX-LABEL: testv4i32u:
+; AVX:       # BB#0:
+; AVX-NEXT:    vpextrd $1, %xmm0, %eax
+; AVX-NEXT:    bsrl %eax, %eax
+; AVX-NEXT:    xorl $31, %eax
+; AVX-NEXT:    vmovd %xmm0, %ecx
+; AVX-NEXT:    bsrl %ecx, %ecx
+; AVX-NEXT:    xorl $31, %ecx
+; AVX-NEXT:    vmovd %ecx, %xmm1
+; AVX-NEXT:    vpinsrd $1, %eax, %xmm1, %xmm1
+; AVX-NEXT:    vpextrd $2, %xmm0, %eax
+; AVX-NEXT:    bsrl %eax, %eax
+; AVX-NEXT:    xorl $31, %eax
+; AVX-NEXT:    vpinsrd $2, %eax, %xmm1, %xmm1
+; AVX-NEXT:    vpextrd $3, %xmm0, %eax
+; AVX-NEXT:    bsrl %eax, %eax
+; AVX-NEXT:    xorl $31, %eax
+; AVX-NEXT:    vpinsrd $3, %eax, %xmm1, %xmm0
+; AVX-NEXT:    retq
+  %out = call <4 x i32> @llvm.ctlz.v4i32(<4 x i32> %in, i1 -1)
+  ret <4 x i32> %out
+}
+
+define <8 x i16> @testv8i16(<8 x i16> %in) nounwind {
+; SSE2-LABEL: testv8i16:
+; SSE2:       # BB#0:
+; SSE2-NEXT:    pextrw $7, %xmm0, %eax
+; SSE2-NEXT:    bsrw %ax, %cx
+; SSE2-NEXT:    movw $31, %ax
+; SSE2-NEXT:    cmovew %ax, %cx
+; SSE2-NEXT:    xorl $15, %ecx
+; SSE2-NEXT:    movd %ecx, %xmm1
+; SSE2-NEXT:    pextrw $3, %xmm0, %ecx
+; SSE2-NEXT:    bsrw %cx, %cx
+; SSE2-NEXT:    cmovew %ax, %cx
+; SSE2-NEXT:    xorl $15, %ecx
+; SSE2-NEXT:    movd %ecx, %xmm2
+; SSE2-NEXT:    punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3]
+; SSE2-NEXT:    pextrw $5, %xmm0, %ecx
+; SSE2-NEXT:    bsrw %cx, %cx
+; SSE2-NEXT:    cmovew %ax, %cx
+; SSE2-NEXT:    xorl $15, %ecx
+; SSE2-NEXT:    movd %ecx, %xmm3
+; SSE2-NEXT:    pextrw $1, %xmm0, %ecx
+; SSE2-NEXT:    bsrw %cx, %cx
+; SSE2-NEXT:    cmovew %ax, %cx
+; SSE2-NEXT:    xorl $15, %ecx
+; SSE2-NEXT:    movd %ecx, %xmm1
+; SSE2-NEXT:    punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm3[0],xmm1[1],xmm3[1],xmm1[2],xmm3[2],xmm1[3],xmm3[3]
+; SSE2-NEXT:    punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1],xmm1[2],xmm2[2],xmm1[3],xmm2[3]
+; SSE2-NEXT:    pextrw $6, %xmm0, %ecx
+; SSE2-NEXT:    bsrw %cx, %cx
+; SSE2-NEXT:    cmovew %ax, %cx
+; SSE2-NEXT:    xorl $15, %ecx
+; SSE2-NEXT:    movd %ecx, %xmm2
+; SSE2-NEXT:    pextrw $2, %xmm0, %ecx
+; SSE2-NEXT:    bsrw %cx, %cx
+; SSE2-NEXT:    cmovew %ax, %cx
+; SSE2-NEXT:    xorl $15, %ecx
+; SSE2-NEXT:    movd %ecx, %xmm3
+; SSE2-NEXT:    punpcklwd {{.*#+}} xmm3 = xmm3[0],xmm2[0],xmm3[1],xmm2[1],xmm3[2],xmm2[2],xmm3[3],xmm2[3]
+; SSE2-NEXT:    pextrw $4, %xmm0, %ecx
+; SSE2-NEXT:    bsrw %cx, %cx
+; SSE2-NEXT:    cmovew %ax, %cx
+; SSE2-NEXT:    xorl $15, %ecx
+; SSE2-NEXT:    movd %ecx, %xmm2
+; SSE2-NEXT:    movd %xmm0, %ecx
+; SSE2-NEXT:    bsrw %cx, %cx
+; SSE2-NEXT:    cmovew %ax, %cx
+; SSE2-NEXT:    xorl $15, %ecx
+; SSE2-NEXT:    movd %ecx, %xmm0
+; SSE2-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1],xmm0[2],xmm2[2],xmm0[3],xmm2[3]
+; SSE2-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1],xmm0[2],xmm3[2],xmm0[3],xmm3[3]
+; SSE2-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
+; SSE2-NEXT:    retq
+;
+; SSE3-LABEL: testv8i16:
+; SSE3:       # BB#0:
+; SSE3-NEXT:    pextrw $7, %xmm0, %eax
+; SSE3-NEXT:    bsrw %ax, %cx
+; SSE3-NEXT:    movw $31, %ax
+; SSE3-NEXT:    cmovew %ax, %cx
+; SSE3-NEXT:    xorl $15, %ecx
+; SSE3-NEXT:    movd %ecx, %xmm1
+; SSE3-NEXT:    pextrw $3, %xmm0, %ecx
+; SSE3-NEXT:    bsrw %cx, %cx
+; SSE3-NEXT:    cmovew %ax, %cx
+; SSE3-NEXT:    xorl $15, %ecx
+; SSE3-NEXT:    movd %ecx, %xmm2
+; SSE3-NEXT:    punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3]
+; SSE3-NEXT:    pextrw $5, %xmm0, %ecx
+; SSE3-NEXT:    bsrw %cx, %cx
+; SSE3-NEXT:    cmovew %ax, %cx
+; SSE3-NEXT:    xorl $15, %ecx
+; SSE3-NEXT:    movd %ecx, %xmm3
+; SSE3-NEXT:    pextrw $1, %xmm0, %ecx
+; SSE3-NEXT:    bsrw %cx, %cx
+; SSE3-NEXT:    cmovew %ax, %cx
+; SSE3-NEXT:    xorl $15, %ecx
+; SSE3-NEXT:    movd %ecx, %xmm1
+; SSE3-NEXT:    punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm3[0],xmm1[1],xmm3[1],xmm1[2],xmm3[2],xmm1[3],xmm3[3]
+; SSE3-NEXT:    punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1],xmm1[2],xmm2[2],xmm1[3],xmm2[3]
+; SSE3-NEXT:    pextrw $6, %xmm0, %ecx
+; SSE3-NEXT:    bsrw %cx, %cx
+; SSE3-NEXT:    cmovew %ax, %cx
+; SSE3-NEXT:    xorl $15, %ecx
+; SSE3-NEXT:    movd %ecx, %xmm2
+; SSE3-NEXT:    pextrw $2, %xmm0, %ecx
+; SSE3-NEXT:    bsrw %cx, %cx
+; SSE3-NEXT:    cmovew %ax, %cx
+; SSE3-NEXT:    xorl $15, %ecx
+; SSE3-NEXT:    movd %ecx, %xmm3
+; SSE3-NEXT:    punpcklwd {{.*#+}} xmm3 = xmm3[0],xmm2[0],xmm3[1],xmm2[1],xmm3[2],xmm2[2],xmm3[3],xmm2[3]
+; SSE3-NEXT:    pextrw $4, %xmm0, %ecx
+; SSE3-NEXT:    bsrw %cx, %cx
+; SSE3-NEXT:    cmovew %ax, %cx
+; SSE3-NEXT:    xorl $15, %ecx
+; SSE3-NEXT:    movd %ecx, %xmm2
+; SSE3-NEXT:    movd %xmm0, %ecx
+; SSE3-NEXT:    bsrw %cx, %cx
+; SSE3-NEXT:    cmovew %ax, %cx
+; SSE3-NEXT:    xorl $15, %ecx
+; SSE3-NEXT:    movd %ecx, %xmm0
+; SSE3-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1],xmm0[2],xmm2[2],xmm0[3],xmm2[3]
+; SSE3-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1],xmm0[2],xmm3[2],xmm0[3],xmm3[3]
+; SSE3-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
+; SSE3-NEXT:    retq
+;
+; SSSE3-LABEL: testv8i16:
+; SSSE3:       # BB#0:
+; SSSE3-NEXT:    pextrw $7, %xmm0, %eax
+; SSSE3-NEXT:    bsrw %ax, %cx
+; SSSE3-NEXT:    movw $31, %ax
+; SSSE3-NEXT:    cmovew %ax, %cx
+; SSSE3-NEXT:    xorl $15, %ecx
+; SSSE3-NEXT:    movd %ecx, %xmm1
+; SSSE3-NEXT:    pextrw $3, %xmm0, %ecx
+; SSSE3-NEXT:    bsrw %cx, %cx
+; SSSE3-NEXT:    cmovew %ax, %cx
+; SSSE3-NEXT:    xorl $15, %ecx
+; SSSE3-NEXT:    movd %ecx, %xmm2
+; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3]
+; SSSE3-NEXT:    pextrw $5, %xmm0, %ecx
+; SSSE3-NEXT:    bsrw %cx, %cx
+; SSSE3-NEXT:    cmovew %ax, %cx
+; SSSE3-NEXT:    xorl $15, %ecx
+; SSSE3-NEXT:    movd %ecx, %xmm3
+; SSSE3-NEXT:    pextrw $1, %xmm0, %ecx
+; SSSE3-NEXT:    bsrw %cx, %cx
+; SSSE3-NEXT:    cmovew %ax, %cx
+; SSSE3-NEXT:    xorl $15, %ecx
+; SSSE3-NEXT:    movd %ecx, %xmm1
+; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm3[0],xmm1[1],xmm3[1],xmm1[2],xmm3[2],xmm1[3],xmm3[3]
+; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1],xmm1[2],xmm2[2],xmm1[3],xmm2[3]
+; SSSE3-NEXT:    pextrw $6, %xmm0, %ecx
+; SSSE3-NEXT:    bsrw %cx, %cx
+; SSSE3-NEXT:    cmovew %ax, %cx
+; SSSE3-NEXT:    xorl $15, %ecx
+; SSSE3-NEXT:    movd %ecx, %xmm2
+; SSSE3-NEXT:    pextrw $2, %xmm0, %ecx
+; SSSE3-NEXT:    bsrw %cx, %cx
+; SSSE3-NEXT:    cmovew %ax, %cx
+; SSSE3-NEXT:    xorl $15, %ecx
+; SSSE3-NEXT:    movd %ecx, %xmm3
+; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm3 = xmm3[0],xmm2[0],xmm3[1],xmm2[1],xmm3[2],xmm2[2],xmm3[3],xmm2[3]
+; SSSE3-NEXT:    pextrw $4, %xmm0, %ecx
+; SSSE3-NEXT:    bsrw %cx, %cx
+; SSSE3-NEXT:    cmovew %ax, %cx
+; SSSE3-NEXT:    xorl $15, %ecx
+; SSSE3-NEXT:    movd %ecx, %xmm2
+; SSSE3-NEXT:    movd %xmm0, %ecx
+; SSSE3-NEXT:    bsrw %cx, %cx
+; SSSE3-NEXT:    cmovew %ax, %cx
+; SSSE3-NEXT:    xorl $15, %ecx
+; SSSE3-NEXT:    movd %ecx, %xmm0
+; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1],xmm0[2],xmm2[2],xmm0[3],xmm2[3]
+; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1],xmm0[2],xmm3[2],xmm0[3],xmm3[3]
+; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
+; SSSE3-NEXT:    retq
+;
+; SSE41-LABEL: testv8i16:
+; SSE41:       # BB#0:
+; SSE41-NEXT:    pextrw $1, %xmm0, %eax
+; SSE41-NEXT:    bsrw %ax, %cx
+; SSE41-NEXT:    movw $31, %ax
+; SSE41-NEXT:    cmovew %ax, %cx
+; SSE41-NEXT:    xorl $15, %ecx
+; SSE41-NEXT:    movd %xmm0, %edx
+; SSE41-NEXT:    bsrw %dx, %dx
+; SSE41-NEXT:    cmovew %ax, %dx
+; SSE41-NEXT:    xorl $15, %edx
+; SSE41-NEXT:    movd %edx, %xmm1
+; SSE41-NEXT:    pinsrw $1, %ecx, %xmm1
+; SSE41-NEXT:    pextrw $2, %xmm0, %ecx
+; SSE41-NEXT:    bsrw %cx, %cx
+; SSE41-NEXT:    cmovew %ax, %cx
+; SSE41-NEXT:    xorl $15, %ecx
+; SSE41-NEXT:    pinsrw $2, %ecx, %xmm1
+; SSE41-NEXT:    pextrw $3, %xmm0, %ecx
+; SSE41-NEXT:    bsrw %cx, %cx
+; SSE41-NEXT:    cmovew %ax, %cx
+; SSE41-NEXT:    xorl $15, %ecx
+; SSE41-NEXT:    pinsrw $3, %ecx, %xmm1
+; SSE41-NEXT:    pextrw $4, %xmm0, %ecx
+; SSE41-NEXT:    bsrw %cx, %cx
+; SSE41-NEXT:    cmovew %ax, %cx
+; SSE41-NEXT:    xorl $15, %ecx
+; SSE41-NEXT:    pinsrw $4, %ecx, %xmm1
+; SSE41-NEXT:    pextrw $5, %xmm0, %ecx
+; SSE41-NEXT:    bsrw %cx, %cx
+; SSE41-NEXT:    cmovew %ax, %cx
+; SSE41-NEXT:    xorl $15, %ecx
+; SSE41-NEXT:    pinsrw $5, %ecx, %xmm1
+; SSE41-NEXT:    pextrw $6, %xmm0, %ecx
+; SSE41-NEXT:    bsrw %cx, %cx
+; SSE41-NEXT:    cmovew %ax, %cx
+; SSE41-NEXT:    xorl $15, %ecx
+; SSE41-NEXT:    pinsrw $6, %ecx, %xmm1
+; SSE41-NEXT:    pextrw $7, %xmm0, %ecx
+; SSE41-NEXT:    bsrw %cx, %cx
+; SSE41-NEXT:    cmovew %ax, %cx
+; SSE41-NEXT:    xorl $15, %ecx
+; SSE41-NEXT:    pinsrw $7, %ecx, %xmm1
+; SSE41-NEXT:    movdqa %xmm1, %xmm0
+; SSE41-NEXT:    retq
+;
+; AVX-LABEL: testv8i16:
+; AVX:       # BB#0:
+; AVX-NEXT:    vpextrw $1, %xmm0, %eax
+; AVX-NEXT:    bsrw %ax, %cx
+; AVX-NEXT:    movw $31, %ax
+; AVX-NEXT:    cmovew %ax, %cx
+; AVX-NEXT:    xorl $15, %ecx
+; AVX-NEXT:    vmovd %xmm0, %edx
+; AVX-NEXT:    bsrw %dx, %dx
+; AVX-NEXT:    cmovew %ax, %dx
+; AVX-NEXT:    xorl $15, %edx
+; AVX-NEXT:    vmovd %edx, %xmm1
+; AVX-NEXT:    vpinsrw $1, %ecx, %xmm1, %xmm1
+; AVX-NEXT:    vpextrw $2, %xmm0, %ecx
+; AVX-NEXT:    bsrw %cx, %cx
+; AVX-NEXT:    cmovew %ax, %cx
+; AVX-NEXT:    xorl $15, %ecx
+; AVX-NEXT:    vpinsrw $2, %ecx, %xmm1, %xmm1
+; AVX-NEXT:    vpextrw $3, %xmm0, %ecx
+; AVX-NEXT:    bsrw %cx, %cx
+; AVX-NEXT:    cmovew %ax, %cx
+; AVX-NEXT:    xorl $15, %ecx
+; AVX-NEXT:    vpinsrw $3, %ecx, %xmm1, %xmm1
+; AVX-NEXT:    vpextrw $4, %xmm0, %ecx
+; AVX-NEXT:    bsrw %cx, %cx
+; AVX-NEXT:    cmovew %ax, %cx
+; AVX-NEXT:    xorl $15, %ecx
+; AVX-NEXT:    vpinsrw $4, %ecx, %xmm1, %xmm1
+; AVX-NEXT:    vpextrw $5, %xmm0, %ecx
+; AVX-NEXT:    bsrw %cx, %cx
+; AVX-NEXT:    cmovew %ax, %cx
+; AVX-NEXT:    xorl $15, %ecx
+; AVX-NEXT:    vpinsrw $5, %ecx, %xmm1, %xmm1
+; AVX-NEXT:    vpextrw $6, %xmm0, %ecx
+; AVX-NEXT:    bsrw %cx, %cx
+; AVX-NEXT:    cmovew %ax, %cx
+; AVX-NEXT:    xorl $15, %ecx
+; AVX-NEXT:    vpinsrw $6, %ecx, %xmm1, %xmm1
+; AVX-NEXT:    vpextrw $7, %xmm0, %ecx
+; AVX-NEXT:    bsrw %cx, %cx
+; AVX-NEXT:    cmovew %ax, %cx
+; AVX-NEXT:    xorl $15, %ecx
+; AVX-NEXT:    vpinsrw $7, %ecx, %xmm1, %xmm0
+; AVX-NEXT:    retq
+  %out = call <8 x i16> @llvm.ctlz.v8i16(<8 x i16> %in, i1 0)
+  ret <8 x i16> %out
+}
+
+define <8 x i16> @testv8i16u(<8 x i16> %in) nounwind {
+; SSE2-LABEL: testv8i16u:
+; SSE2:       # BB#0:
+; SSE2-NEXT:    pextrw $7, %xmm0, %eax
+; SSE2-NEXT:    bsrw %ax, %ax
+; SSE2-NEXT:    xorl $15, %eax
+; SSE2-NEXT:    movd %eax, %xmm1
+; SSE2-NEXT:    pextrw $3, %xmm0, %eax
+; SSE2-NEXT:    bsrw %ax, %ax
+; SSE2-NEXT:    xorl $15, %eax
+; SSE2-NEXT:    movd %eax, %xmm2
+; SSE2-NEXT:    punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3]
+; SSE2-NEXT:    pextrw $5, %xmm0, %eax
+; SSE2-NEXT:    bsrw %ax, %ax
+; SSE2-NEXT:    xorl $15, %eax
+; SSE2-NEXT:    movd %eax, %xmm3
+; SSE2-NEXT:    pextrw $1, %xmm0, %eax
+; SSE2-NEXT:    bsrw %ax, %ax
+; SSE2-NEXT:    xorl $15, %eax
+; SSE2-NEXT:    movd %eax, %xmm1
+; SSE2-NEXT:    punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm3[0],xmm1[1],xmm3[1],xmm1[2],xmm3[2],xmm1[3],xmm3[3]
+; SSE2-NEXT:    punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1],xmm1[2],xmm2[2],xmm1[3],xmm2[3]
+; SSE2-NEXT:    pextrw $6, %xmm0, %eax
+; SSE2-NEXT:    bsrw %ax, %ax
+; SSE2-NEXT:    xorl $15, %eax
+; SSE2-NEXT:    movd %eax, %xmm2
+; SSE2-NEXT:    pextrw $2, %xmm0, %eax
+; SSE2-NEXT:    bsrw %ax, %ax
+; SSE2-NEXT:    xorl $15, %eax
+; SSE2-NEXT:    movd %eax, %xmm3
+; SSE2-NEXT:    punpcklwd {{.*#+}} xmm3 = xmm3[0],xmm2[0],xmm3[1],xmm2[1],xmm3[2],xmm2[2],xmm3[3],xmm2[3]
+; SSE2-NEXT:    pextrw $4, %xmm0, %eax
+; SSE2-NEXT:    bsrw %ax, %ax
+; SSE2-NEXT:    xorl $15, %eax
+; SSE2-NEXT:    movd %eax, %xmm2
+; SSE2-NEXT:    movd %xmm0, %eax
+; SSE2-NEXT:    bsrw %ax, %ax
+; SSE2-NEXT:    xorl $15, %eax
+; SSE2-NEXT:    movd %eax, %xmm0
+; SSE2-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1],xmm0[2],xmm2[2],xmm0[3],xmm2[3]
+; SSE2-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1],xmm0[2],xmm3[2],xmm0[3],xmm3[3]
+; SSE2-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
+; SSE2-NEXT:    retq
+;
+; SSE3-LABEL: testv8i16u:
+; SSE3:       # BB#0:
+; SSE3-NEXT:    pextrw $7, %xmm0, %eax
+; SSE3-NEXT:    bsrw %ax, %ax
+; SSE3-NEXT:    xorl $15, %eax
+; SSE3-NEXT:    movd %eax, %xmm1
+; SSE3-NEXT:    pextrw $3, %xmm0, %eax
+; SSE3-NEXT:    bsrw %ax, %ax
+; SSE3-NEXT:    xorl $15, %eax
+; SSE3-NEXT:    movd %eax, %xmm2
+; SSE3-NEXT:    punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3]
+; SSE3-NEXT:    pextrw $5, %xmm0, %eax
+; SSE3-NEXT:    bsrw %ax, %ax
+; SSE3-NEXT:    xorl $15, %eax
+; SSE3-NEXT:    movd %eax, %xmm3
+; SSE3-NEXT:    pextrw $1, %xmm0, %eax
+; SSE3-NEXT:    bsrw %ax, %ax
+; SSE3-NEXT:    xorl $15, %eax
+; SSE3-NEXT:    movd %eax, %xmm1
+; SSE3-NEXT:    punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm3[0],xmm1[1],xmm3[1],xmm1[2],xmm3[2],xmm1[3],xmm3[3]
+; SSE3-NEXT:    punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1],xmm1[2],xmm2[2],xmm1[3],xmm2[3]
+; SSE3-NEXT:    pextrw $6, %xmm0, %eax
+; SSE3-NEXT:    bsrw %ax, %ax
+; SSE3-NEXT:    xorl $15, %eax
+; SSE3-NEXT:    movd %eax, %xmm2
+; SSE3-NEXT:    pextrw $2, %xmm0, %eax
+; SSE3-NEXT:    bsrw %ax, %ax
+; SSE3-NEXT:    xorl $15, %eax
+; SSE3-NEXT:    movd %eax, %xmm3
+; SSE3-NEXT:    punpcklwd {{.*#+}} xmm3 = xmm3[0],xmm2[0],xmm3[1],xmm2[1],xmm3[2],xmm2[2],xmm3[3],xmm2[3]
+; SSE3-NEXT:    pextrw $4, %xmm0, %eax
+; SSE3-NEXT:    bsrw %ax, %ax
+; SSE3-NEXT:    xorl $15, %eax
+; SSE3-NEXT:    movd %eax, %xmm2
+; SSE3-NEXT:    movd %xmm0, %eax
+; SSE3-NEXT:    bsrw %ax, %ax
+; SSE3-NEXT:    xorl $15, %eax
+; SSE3-NEXT:    movd %eax, %xmm0
+; SSE3-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1],xmm0[2],xmm2[2],xmm0[3],xmm2[3]
+; SSE3-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1],xmm0[2],xmm3[2],xmm0[3],xmm3[3]
+; SSE3-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
+; SSE3-NEXT:    retq
+;
+; SSSE3-LABEL: testv8i16u:
+; SSSE3:       # BB#0:
+; SSSE3-NEXT:    pextrw $7, %xmm0, %eax
+; SSSE3-NEXT:    bsrw %ax, %ax
+; SSSE3-NEXT:    xorl $15, %eax
+; SSSE3-NEXT:    movd %eax, %xmm1
+; SSSE3-NEXT:    pextrw $3, %xmm0, %eax
+; SSSE3-NEXT:    bsrw %ax, %ax
+; SSSE3-NEXT:    xorl $15, %eax
+; SSSE3-NEXT:    movd %eax, %xmm2
+; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3]
+; SSSE3-NEXT:    pextrw $5, %xmm0, %eax
+; SSSE3-NEXT:    bsrw %ax, %ax
+; SSSE3-NEXT:    xorl $15, %eax
+; SSSE3-NEXT:    movd %eax, %xmm3
+; SSSE3-NEXT:    pextrw $1, %xmm0, %eax
+; SSSE3-NEXT:    bsrw %ax, %ax
+; SSSE3-NEXT:    xorl $15, %eax
+; SSSE3-NEXT:    movd %eax, %xmm1
+; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm3[0],xmm1[1],xmm3[1],xmm1[2],xmm3[2],xmm1[3],xmm3[3]
+; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1],xmm1[2],xmm2[2],xmm1[3],xmm2[3]
+; SSSE3-NEXT:    pextrw $6, %xmm0, %eax
+; SSSE3-NEXT:    bsrw %ax, %ax
+; SSSE3-NEXT:    xorl $15, %eax
+; SSSE3-NEXT:    movd %eax, %xmm2
+; SSSE3-NEXT:    pextrw $2, %xmm0, %eax
+; SSSE3-NEXT:    bsrw %ax, %ax
+; SSSE3-NEXT:    xorl $15, %eax
+; SSSE3-NEXT:    movd %eax, %xmm3
+; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm3 = xmm3[0],xmm2[0],xmm3[1],xmm2[1],xmm3[2],xmm2[2],xmm3[3],xmm2[3]
+; SSSE3-NEXT:    pextrw $4, %xmm0, %eax
+; SSSE3-NEXT:    bsrw %ax, %ax
+; SSSE3-NEXT:    xorl $15, %eax
+; SSSE3-NEXT:    movd %eax, %xmm2
+; SSSE3-NEXT:    movd %xmm0, %eax
+; SSSE3-NEXT:    bsrw %ax, %ax
+; SSSE3-NEXT:    xorl $15, %eax
+; SSSE3-NEXT:    movd %eax, %xmm0
+; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1],xmm0[2],xmm2[2],xmm0[3],xmm2[3]
+; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1],xmm0[2],xmm3[2],xmm0[3],xmm3[3]
+; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
+; SSSE3-NEXT:    retq
+;
+; SSE41-LABEL: testv8i16u:
+; SSE41:       # BB#0:
+; SSE41-NEXT:    pextrw $1, %xmm0, %eax
+; SSE41-NEXT:    bsrw %ax, %ax
+; SSE41-NEXT:    xorl $15, %eax
+; SSE41-NEXT:    movd %xmm0, %ecx
+; SSE41-NEXT:    bsrw %cx, %cx
+; SSE41-NEXT:    xorl $15, %ecx
+; SSE41-NEXT:    movd %ecx, %xmm1
+; SSE41-NEXT:    pinsrw $1, %eax, %xmm1
+; SSE41-NEXT:    pextrw $2, %xmm0, %eax
+; SSE41-NEXT:    bsrw %ax, %ax
+; SSE41-NEXT:    xorl $15, %eax
+; SSE41-NEXT:    pinsrw $2, %eax, %xmm1
+; SSE41-NEXT:    pextrw $3, %xmm0, %eax
+; SSE41-NEXT:    bsrw %ax, %ax
+; SSE41-NEXT:    xorl $15, %eax
+; SSE41-NEXT:    pinsrw $3, %eax, %xmm1
+; SSE41-NEXT:    pextrw $4, %xmm0, %eax
+; SSE41-NEXT:    bsrw %ax, %ax
+; SSE41-NEXT:    xorl $15, %eax
+; SSE41-NEXT:    pinsrw $4, %eax, %xmm1
+; SSE41-NEXT:    pextrw $5, %xmm0, %eax
+; SSE41-NEXT:    bsrw %ax, %ax
+; SSE41-NEXT:    xorl $15, %eax
+; SSE41-NEXT:    pinsrw $5, %eax, %xmm1
+; SSE41-NEXT:    pextrw $6, %xmm0, %eax
+; SSE41-NEXT:    bsrw %ax, %ax
+; SSE41-NEXT:    xorl $15, %eax
+; SSE41-NEXT:    pinsrw $6, %eax, %xmm1
+; SSE41-NEXT:    pextrw $7, %xmm0, %eax
+; SSE41-NEXT:    bsrw %ax, %ax
+; SSE41-NEXT:    xorl $15, %eax
+; SSE41-NEXT:    pinsrw $7, %eax, %xmm1
+; SSE41-NEXT:    movdqa %xmm1, %xmm0
+; SSE41-NEXT:    retq
+;
+; AVX-LABEL: testv8i16u:
+; AVX:       # BB#0:
+; AVX-NEXT:    vpextrw $1, %xmm0, %eax
+; AVX-NEXT:    bsrw %ax, %ax
+; AVX-NEXT:    xorl $15, %eax
+; AVX-NEXT:    vmovd %xmm0, %ecx
+; AVX-NEXT:    bsrw %cx, %cx
+; AVX-NEXT:    xorl $15, %ecx
+; AVX-NEXT:    vmovd %ecx, %xmm1
+; AVX-NEXT:    vpinsrw $1, %eax, %xmm1, %xmm1
+; AVX-NEXT:    vpextrw $2, %xmm0, %eax
+; AVX-NEXT:    bsrw %ax, %ax
+; AVX-NEXT:    xorl $15, %eax
+; AVX-NEXT:    vpinsrw $2, %eax, %xmm1, %xmm1
+; AVX-NEXT:    vpextrw $3, %xmm0, %eax
+; AVX-NEXT:    bsrw %ax, %ax
+; AVX-NEXT:    xorl $15, %eax
+; AVX-NEXT:    vpinsrw $3, %eax, %xmm1, %xmm1
+; AVX-NEXT:    vpextrw $4, %xmm0, %eax
+; AVX-NEXT:    bsrw %ax, %ax
+; AVX-NEXT:    xorl $15, %eax
+; AVX-NEXT:    vpinsrw $4, %eax, %xmm1, %xmm1
+; AVX-NEXT:    vpextrw $5, %xmm0, %eax
+; AVX-NEXT:    bsrw %ax, %ax
+; AVX-NEXT:    xorl $15, %eax
+; AVX-NEXT:    vpinsrw $5, %eax, %xmm1, %xmm1
+; AVX-NEXT:    vpextrw $6, %xmm0, %eax
+; AVX-NEXT:    bsrw %ax, %ax
+; AVX-NEXT:    xorl $15, %eax
+; AVX-NEXT:    vpinsrw $6, %eax, %xmm1, %xmm1
+; AVX-NEXT:    vpextrw $7, %xmm0, %eax
+; AVX-NEXT:    bsrw %ax, %ax
+; AVX-NEXT:    xorl $15, %eax
+; AVX-NEXT:    vpinsrw $7, %eax, %xmm1, %xmm0
+; AVX-NEXT:    retq
+  %out = call <8 x i16> @llvm.ctlz.v8i16(<8 x i16> %in, i1 -1)
+  ret <8 x i16> %out
+}
+
+define <16 x i8> @testv16i8(<16 x i8> %in) nounwind {
+; SSE2-LABEL: testv16i8:
+; SSE2:       # BB#0:
+; SSE2-NEXT:    pushq %rbp
+; SSE2-NEXT:    pushq %rbx
+; SSE2-NEXT:    movaps %xmm0, -{{[0-9]+}}(%rsp)
+; SSE2-NEXT:    movzbl -{{[0-9]+}}(%rsp), %eax
+; SSE2-NEXT:    bsrl %eax, %ecx
+; SSE2-NEXT:    movl $15, %eax
+; SSE2-NEXT:    cmovel %eax, %ecx
+; SSE2-NEXT:    xorl $7, %ecx
+; SSE2-NEXT:    movd %ecx, %xmm0
+; SSE2-NEXT:    movzbl -{{[0-9]+}}(%rsp), %ebx
+; SSE2-NEXT:    movzbl -{{[0-9]+}}(%rsp), %edi
+; SSE2-NEXT:    movzbl -{{[0-9]+}}(%rsp), %r9d
+; SSE2-NEXT:    movzbl -{{[0-9]+}}(%rsp), %edx
+; SSE2-NEXT:    movzbl -{{[0-9]+}}(%rsp), %r11d
+; SSE2-NEXT:    movzbl -{{[0-9]+}}(%rsp), %esi
+; SSE2-NEXT:    movzbl -{{[0-9]+}}(%rsp), %r8d
+; SSE2-NEXT:    movzbl -{{[0-9]+}}(%rsp), %ecx
+; SSE2-NEXT:    bsrl %ecx, %ecx
+; SSE2-NEXT:    cmovel %eax, %ecx
+; SSE2-NEXT:    xorl $7, %ecx
+; SSE2-NEXT:    movd %ecx, %xmm1
+; SSE2-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3],xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
+; SSE2-NEXT:    bsrl %edx, %ecx
+; SSE2-NEXT:    cmovel %eax, %ecx
+; SSE2-NEXT:    xorl $7, %ecx
+; SSE2-NEXT:    movd %ecx, %xmm2
+; SSE2-NEXT:    movzbl -{{[0-9]+}}(%rsp), %edx
+; SSE2-NEXT:    movzbl -{{[0-9]+}}(%rsp), %ecx
+; SSE2-NEXT:    movzbl -{{[0-9]+}}(%rsp), %r10d
+; SSE2-NEXT:    movzbl -{{[0-9]+}}(%rsp), %ebp
+; SSE2-NEXT:    bsrl %ebp, %ebp
+; SSE2-NEXT:    cmovel %eax, %ebp
+; SSE2-NEXT:    xorl $7, %ebp
+; SSE2-NEXT:    movd %ebp, %xmm0
+; SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1],xmm0[2],xmm2[2],xmm0[3],xmm2[3],xmm0[4],xmm2[4],xmm0[5],xmm2[5],xmm0[6],xmm2[6],xmm0[7],xmm2[7]
+; SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3],xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7]
+; SSE2-NEXT:    bsrl %edi, %edi
+; SSE2-NEXT:    cmovel %eax, %edi
+; SSE2-NEXT:    xorl $7, %edi
+; SSE2-NEXT:    movd %edi, %xmm1
+; SSE2-NEXT:    bsrl %ecx, %ecx
+; SSE2-NEXT:    cmovel %eax, %ecx
+; SSE2-NEXT:    xorl $7, %ecx
+; SSE2-NEXT:    movd %ecx, %xmm2
+; SSE2-NEXT:    punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3],xmm2[4],xmm1[4],xmm2[5],xmm1[5],xmm2[6],xmm1[6],xmm2[7],xmm1[7]
+; SSE2-NEXT:    bsrl %esi, %ecx
+; SSE2-NEXT:    cmovel %eax, %ecx
+; SSE2-NEXT:    xorl $7, %ecx
+; SSE2-NEXT:    movd %ecx, %xmm3
+; SSE2-NEXT:    movzbl -{{[0-9]+}}(%rsp), %esi
+; SSE2-NEXT:    movzbl -{{[0-9]+}}(%rsp), %ecx
+; SSE2-NEXT:    bsrl %ecx, %ecx
+; SSE2-NEXT:    cmovel %eax, %ecx
+; SSE2-NEXT:    xorl $7, %ecx
+; SSE2-NEXT:    movd %ecx, %xmm1
+; SSE2-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm3[0],xmm1[1],xmm3[1],xmm1[2],xmm3[2],xmm1[3],xmm3[3],xmm1[4],xmm3[4],xmm1[5],xmm3[5],xmm1[6],xmm3[6],xmm1[7],xmm3[7]
+; SSE2-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1],xmm1[2],xmm2[2],xmm1[3],xmm2[3],xmm1[4],xmm2[4],xmm1[5],xmm2[5],xmm1[6],xmm2[6],xmm1[7],xmm2[7]
+; SSE2-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3],xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
+; SSE2-NEXT:    bsrl %ebx, %ecx
+; SSE2-NEXT:    cmovel %eax, %ecx
+; SSE2-NEXT:    xorl $7, %ecx
+; SSE2-NEXT:    movd %ecx, %xmm0
+; SSE2-NEXT:    bsrl %edx, %ecx
+; SSE2-NEXT:    cmovel %eax, %ecx
+; SSE2-NEXT:    xorl $7, %ecx
+; SSE2-NEXT:    movd %ecx, %xmm3
+; SSE2-NEXT:    punpcklbw {{.*#+}} xmm3 = xmm3[0],xmm0[0],xmm3[1],xmm0[1],xmm3[2],xmm0[2],xmm3[3],xmm0[3],xmm3[4],xmm0[4],xmm3[5],xmm0[5],xmm3[6],xmm0[6],xmm3[7],xmm0[7]
+; SSE2-NEXT:    bsrl %r11d, %ecx
+; SSE2-NEXT:    cmovel %eax, %ecx
+; SSE2-NEXT:    xorl $7, %ecx
+; SSE2-NEXT:    movd %ecx, %xmm0
+; SSE2-NEXT:    bsrl %esi, %ecx
+; SSE2-NEXT:    cmovel %eax, %ecx
+; SSE2-NEXT:    xorl $7, %ecx
+; SSE2-NEXT:    movd %ecx, %xmm2
+; SSE2-NEXT:    punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3],xmm2[4],xmm0[4],xmm2[5],xmm0[5],xmm2[6],xmm0[6],xmm2[7],xmm0[7]
+; SSE2-NEXT:    punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1],xmm2[2],xmm3[2],xmm2[3],xmm3[3],xmm2[4],xmm3[4],xmm2[5],xmm3[5],xmm2[6],xmm3[6],xmm2[7],xmm3[7]
+; SSE2-NEXT:    bsrl %r9d, %ecx
+; SSE2-NEXT:    cmovel %eax, %ecx
+; SSE2-NEXT:    xorl $7, %ecx
+; SSE2-NEXT:    movd %ecx, %xmm0
+; SSE2-NEXT:    bsrl %r10d, %ecx
+; SSE2-NEXT:    cmovel %eax, %ecx
+; SSE2-NEXT:    xorl $7, %ecx
+; SSE2-NEXT:    movd %ecx, %xmm3
+; SSE2-NEXT:    punpcklbw {{.*#+}} xmm3 = xmm3[0],xmm0[0],xmm3[1],xmm0[1],xmm3[2],xmm0[2],xmm3[3],xmm0[3],xmm3[4],xmm0[4],xmm3[5],xmm0[5],xmm3[6],xmm0[6],xmm3[7],xmm0[7]
+; SSE2-NEXT:    bsrl %r8d, %ecx
+; SSE2-NEXT:    cmovel %eax, %ecx
+; SSE2-NEXT:    xorl $7, %ecx
+; SSE2-NEXT:    movd %ecx, %xmm4
+; SSE2-NEXT:    movzbl -{{[0-9]+}}(%rsp), %ecx
+; SSE2-NEXT:    bsrl %ecx, %ecx
+; SSE2-NEXT:    cmovel %eax, %ecx
+; SSE2-NEXT:    xorl $7, %ecx
+; SSE2-NEXT:    movd %ecx, %xmm0
+; SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm4[0],xmm0[1],xmm4[1],xmm0[2],xmm4[2],xmm0[3],xmm4[3],xmm0[4],xmm4[4],xmm0[5],xmm4[5],xmm0[6],xmm4[6],xmm0[7],xmm4[7]
+; SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1],xmm0[2],xmm3[2],xmm0[3],xmm3[3],xmm0[4],xmm3[4],xmm0[5],xmm3[5],xmm0[6],xmm3[6],xmm0[7],xmm3[7]
+; SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1],xmm0[2],xmm2[2],xmm0[3],xmm2[3],xmm0[4],xmm2[4],xmm0[5],xmm2[5],xmm0[6],xmm2[6],xmm0[7],xmm2[7]
+; SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3],xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7]
+; SSE2-NEXT:    popq %rbx
+; SSE2-NEXT:    popq %rbp
+; SSE2-NEXT:    retq
+;
+; SSE3-LABEL: testv16i8:
+; SSE3:       # BB#0:
+; SSE3-NEXT:    pushq %rbp
+; SSE3-NEXT:    pushq %rbx
+; SSE3-NEXT:    movaps %xmm0, -{{[0-9]+}}(%rsp)
+; SSE3-NEXT:    movzbl -{{[0-9]+}}(%rsp), %eax
+; SSE3-NEXT:    bsrl %eax, %ecx
+; SSE3-NEXT:    movl $15, %eax
+; SSE3-NEXT:    cmovel %eax, %ecx
+; SSE3-NEXT:    xorl $7, %ecx
+; SSE3-NEXT:    movd %ecx, %xmm0
+; SSE3-NEXT:    movzbl -{{[0-9]+}}(%rsp), %ebx
+; SSE3-NEXT:    movzbl -{{[0-9]+}}(%rsp), %edi
+; SSE3-NEXT:    movzbl -{{[0-9]+}}(%rsp), %r9d
+; SSE3-NEXT:    movzbl -{{[0-9]+}}(%rsp), %edx
+; SSE3-NEXT:    movzbl -{{[0-9]+}}(%rsp), %r11d
+; SSE3-NEXT:    movzbl -{{[0-9]+}}(%rsp), %esi
+; SSE3-NEXT:    movzbl -{{[0-9]+}}(%rsp), %r8d
+; SSE3-NEXT:    movzbl -{{[0-9]+}}(%rsp), %ecx
+; SSE3-NEXT:    bsrl %ecx, %ecx
+; SSE3-NEXT:    cmovel %eax, %ecx
+; SSE3-NEXT:    xorl $7, %ecx
+; SSE3-NEXT:    movd %ecx, %xmm1
+; SSE3-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3],xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
+; SSE3-NEXT:    bsrl %edx, %ecx
+; SSE3-NEXT:    cmovel %eax, %ecx
+; SSE3-NEXT:    xorl $7, %ecx
+; SSE3-NEXT:    movd %ecx, %xmm2
+; SSE3-NEXT:    movzbl -{{[0-9]+}}(%rsp), %edx
+; SSE3-NEXT:    movzbl -{{[0-9]+}}(%rsp), %ecx
+; SSE3-NEXT:    movzbl -{{[0-9]+}}(%rsp), %r10d
+; SSE3-NEXT:    movzbl -{{[0-9]+}}(%rsp), %ebp
+; SSE3-NEXT:    bsrl %ebp, %ebp
+; SSE3-NEXT:    cmovel %eax, %ebp
+; SSE3-NEXT:    xorl $7, %ebp
+; SSE3-NEXT:    movd %ebp, %xmm0
+; SSE3-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1],xmm0[2],xmm2[2],xmm0[3],xmm2[3],xmm0[4],xmm2[4],xmm0[5],xmm2[5],xmm0[6],xmm2[6],xmm0[7],xmm2[7]
+; SSE3-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3],xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7]
+; SSE3-NEXT:    bsrl %edi, %edi
+; SSE3-NEXT:    cmovel %eax, %edi
+; SSE3-NEXT:    xorl $7, %edi
+; SSE3-NEXT:    movd %edi, %xmm1
+; SSE3-NEXT:    bsrl %ecx, %ecx
+; SSE3-NEXT:    cmovel %eax, %ecx
+; SSE3-NEXT:    xorl $7, %ecx
+; SSE3-NEXT:    movd %ecx, %xmm2
+; SSE3-NEXT:    punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3],xmm2[4],xmm1[4],xmm2[5],xmm1[5],xmm2[6],xmm1[6],xmm2[7],xmm1[7]
+; SSE3-NEXT:    bsrl %esi, %ecx
+; SSE3-NEXT:    cmovel %eax, %ecx
+; SSE3-NEXT:    xorl $7, %ecx
+; SSE3-NEXT:    movd %ecx, %xmm3
+; SSE3-NEXT:    movzbl -{{[0-9]+}}(%rsp), %esi
+; SSE3-NEXT:    movzbl -{{[0-9]+}}(%rsp), %ecx
+; SSE3-NEXT:    bsrl %ecx, %ecx
+; SSE3-NEXT:    cmovel %eax, %ecx
+; SSE3-NEXT:    xorl $7, %ecx
+; SSE3-NEXT:    movd %ecx, %xmm1
+; SSE3-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm3[0],xmm1[1],xmm3[1],xmm1[2],xmm3[2],xmm1[3],xmm3[3],xmm1[4],xmm3[4],xmm1[5],xmm3[5],xmm1[6],xmm3[6],xmm1[7],xmm3[7]
+; SSE3-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1],xmm1[2],xmm2[2],xmm1[3],xmm2[3],xmm1[4],xmm2[4],xmm1[5],xmm2[5],xmm1[6],xmm2[6],xmm1[7],xmm2[7]
+; SSE3-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3],xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
+; SSE3-NEXT:    bsrl %ebx, %ecx
+; SSE3-NEXT:    cmovel %eax, %ecx
+; SSE3-NEXT:    xorl $7, %ecx
+; SSE3-NEXT:    movd %ecx, %xmm0
+; SSE3-NEXT:    bsrl %edx, %ecx
+; SSE3-NEXT:    cmovel %eax, %ecx
+; SSE3-NEXT:    xorl $7, %ecx
+; SSE3-NEXT:    movd %ecx, %xmm3
+; SSE3-NEXT:    punpcklbw {{.*#+}} xmm3 = xmm3[0],xmm0[0],xmm3[1],xmm0[1],xmm3[2],xmm0[2],xmm3[3],xmm0[3],xmm3[4],xmm0[4],xmm3[5],xmm0[5],xmm3[6],xmm0[6],xmm3[7],xmm0[7]
+; SSE3-NEXT:    bsrl %r11d, %ecx
+; SSE3-NEXT:    cmovel %eax, %ecx
+; SSE3-NEXT:    xorl $7, %ecx
+; SSE3-NEXT:    movd %ecx, %xmm0
+; SSE3-NEXT:    bsrl %esi, %ecx
+; SSE3-NEXT:    cmovel %eax, %ecx
+; SSE3-NEXT:    xorl $7, %ecx
+; SSE3-NEXT:    movd %ecx, %xmm2
+; SSE3-NEXT:    punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3],xmm2[4],xmm0[4],xmm2[5],xmm0[5],xmm2[6],xmm0[6],xmm2[7],xmm0[7]
+; SSE3-NEXT:    punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1],xmm2[2],xmm3[2],xmm2[3],xmm3[3],xmm2[4],xmm3[4],xmm2[5],xmm3[5],xmm2[6],xmm3[6],xmm2[7],xmm3[7]
+; SSE3-NEXT:    bsrl %r9d, %ecx
+; SSE3-NEXT:    cmovel %eax, %ecx
+; SSE3-NEXT:    xorl $7, %ecx
+; SSE3-NEXT:    movd %ecx, %xmm0
+; SSE3-NEXT:    bsrl %r10d, %ecx
+; SSE3-NEXT:    cmovel %eax, %ecx
+; SSE3-NEXT:    xorl $7, %ecx
+; SSE3-NEXT:    movd %ecx, %xmm3
+; SSE3-NEXT:    punpcklbw {{.*#+}} xmm3 = xmm3[0],xmm0[0],xmm3[1],xmm0[1],xmm3[2],xmm0[2],xmm3[3],xmm0[3],xmm3[4],xmm0[4],xmm3[5],xmm0[5],xmm3[6],xmm0[6],xmm3[7],xmm0[7]
+; SSE3-NEXT:    bsrl %r8d, %ecx
+; SSE3-NEXT:    cmovel %eax, %ecx
+; SSE3-NEXT:    xorl $7, %ecx
+; SSE3-NEXT:    movd %ecx, %xmm4
+; SSE3-NEXT:    movzbl -{{[0-9]+}}(%rsp), %ecx
+; SSE3-NEXT:    bsrl %ecx, %ecx
+; SSE3-NEXT:    cmovel %eax, %ecx
+; SSE3-NEXT:    xorl $7, %ecx
+; SSE3-NEXT:    movd %ecx, %xmm0
+; SSE3-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm4[0],xmm0[1],xmm4[1],xmm0[2],xmm4[2],xmm0[3],xmm4[3],xmm0[4],xmm4[4],xmm0[5],xmm4[5],xmm0[6],xmm4[6],xmm0[7],xmm4[7]
+; SSE3-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1],xmm0[2],xmm3[2],xmm0[3],xmm3[3],xmm0[4],xmm3[4],xmm0[5],xmm3[5],xmm0[6],xmm3[6],xmm0[7],xmm3[7]
+; SSE3-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1],xmm0[2],xmm2[2],xmm0[3],xmm2[3],xmm0[4],xmm2[4],xmm0[5],xmm2[5],xmm0[6],xmm2[6],xmm0[7],xmm2[7]
+; SSE3-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3],xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7]
+; SSE3-NEXT:    popq %rbx
+; SSE3-NEXT:    popq %rbp
+; SSE3-NEXT:    retq
+;
+; SSSE3-LABEL: testv16i8:
+; SSSE3:       # BB#0:
+; SSSE3-NEXT:    pushq %rbp
+; SSSE3-NEXT:    pushq %rbx
+; SSSE3-NEXT:    movaps %xmm0, -{{[0-9]+}}(%rsp)
+; SSSE3-NEXT:    movzbl -{{[0-9]+}}(%rsp), %eax
+; SSSE3-NEXT:    bsrl %eax, %ecx
+; SSSE3-NEXT:    movl $15, %eax
+; SSSE3-NEXT:    cmovel %eax, %ecx
+; SSSE3-NEXT:    xorl $7, %ecx
+; SSSE3-NEXT:    movd %ecx, %xmm0
+; SSSE3-NEXT:    movzbl -{{[0-9]+}}(%rsp), %ebx
+; SSSE3-NEXT:    movzbl -{{[0-9]+}}(%rsp), %edi
+; SSSE3-NEXT:    movzbl -{{[0-9]+}}(%rsp), %r9d
+; SSSE3-NEXT:    movzbl -{{[0-9]+}}(%rsp), %edx
+; SSSE3-NEXT:    movzbl -{{[0-9]+}}(%rsp), %r11d
+; SSSE3-NEXT:    movzbl -{{[0-9]+}}(%rsp), %esi
+; SSSE3-NEXT:    movzbl -{{[0-9]+}}(%rsp), %r8d
+; SSSE3-NEXT:    movzbl -{{[0-9]+}}(%rsp), %ecx
+; SSSE3-NEXT:    bsrl %ecx, %ecx
+; SSSE3-NEXT:    cmovel %eax, %ecx
+; SSSE3-NEXT:    xorl $7, %ecx
+; SSSE3-NEXT:    movd %ecx, %xmm1
+; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3],xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
+; SSSE3-NEXT:    bsrl %edx, %ecx
+; SSSE3-NEXT:    cmovel %eax, %ecx
+; SSSE3-NEXT:    xorl $7, %ecx
+; SSSE3-NEXT:    movd %ecx, %xmm2
+; SSSE3-NEXT:    movzbl -{{[0-9]+}}(%rsp), %edx
+; SSSE3-NEXT:    movzbl -{{[0-9]+}}(%rsp), %ecx
+; SSSE3-NEXT:    movzbl -{{[0-9]+}}(%rsp), %r10d
+; SSSE3-NEXT:    movzbl -{{[0-9]+}}(%rsp), %ebp
+; SSSE3-NEXT:    bsrl %ebp, %ebp
+; SSSE3-NEXT:    cmovel %eax, %ebp
+; SSSE3-NEXT:    xorl $7, %ebp
+; SSSE3-NEXT:    movd %ebp, %xmm0
+; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1],xmm0[2],xmm2[2],xmm0[3],xmm2[3],xmm0[4],xmm2[4],xmm0[5],xmm2[5],xmm0[6],xmm2[6],xmm0[7],xmm2[7]
+; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3],xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7]
+; SSSE3-NEXT:    bsrl %edi, %edi
+; SSSE3-NEXT:    cmovel %eax, %edi
+; SSSE3-NEXT:    xorl $7, %edi
+; SSSE3-NEXT:    movd %edi, %xmm1
+; SSSE3-NEXT:    bsrl %ecx, %ecx
+; SSSE3-NEXT:    cmovel %eax, %ecx
+; SSSE3-NEXT:    xorl $7, %ecx
+; SSSE3-NEXT:    movd %ecx, %xmm2
+; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3],xmm2[4],xmm1[4],xmm2[5],xmm1[5],xmm2[6],xmm1[6],xmm2[7],xmm1[7]
+; SSSE3-NEXT:    bsrl %esi, %ecx
+; SSSE3-NEXT:    cmovel %eax, %ecx
+; SSSE3-NEXT:    xorl $7, %ecx
+; SSSE3-NEXT:    movd %ecx, %xmm3
+; SSSE3-NEXT:    movzbl -{{[0-9]+}}(%rsp), %esi
+; SSSE3-NEXT:    movzbl -{{[0-9]+}}(%rsp), %ecx
+; SSSE3-NEXT:    bsrl %ecx, %ecx
+; SSSE3-NEXT:    cmovel %eax, %ecx
+; SSSE3-NEXT:    xorl $7, %ecx
+; SSSE3-NEXT:    movd %ecx, %xmm1
+; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm3[0],xmm1[1],xmm3[1],xmm1[2],xmm3[2],xmm1[3],xmm3[3],xmm1[4],xmm3[4],xmm1[5],xmm3[5],xmm1[6],xmm3[6],xmm1[7],xmm3[7]
+; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1],xmm1[2],xmm2[2],xmm1[3],xmm2[3],xmm1[4],xmm2[4],xmm1[5],xmm2[5],xmm1[6],xmm2[6],xmm1[7],xmm2[7]
+; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3],xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
+; SSSE3-NEXT:    bsrl %ebx, %ecx
+; SSSE3-NEXT:    cmovel %eax, %ecx
+; SSSE3-NEXT:    xorl $7, %ecx
+; SSSE3-NEXT:    movd %ecx, %xmm0
+; SSSE3-NEXT:    bsrl %edx, %ecx
+; SSSE3-NEXT:    cmovel %eax, %ecx
+; SSSE3-NEXT:    xorl $7, %ecx
+; SSSE3-NEXT:    movd %ecx, %xmm3
+; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm3 = xmm3[0],xmm0[0],xmm3[1],xmm0[1],xmm3[2],xmm0[2],xmm3[3],xmm0[3],xmm3[4],xmm0[4],xmm3[5],xmm0[5],xmm3[6],xmm0[6],xmm3[7],xmm0[7]
+; SSSE3-NEXT:    bsrl %r11d, %ecx
+; SSSE3-NEXT:    cmovel %eax, %ecx
+; SSSE3-NEXT:    xorl $7, %ecx
+; SSSE3-NEXT:    movd %ecx, %xmm0
+; SSSE3-NEXT:    bsrl %esi, %ecx
+; SSSE3-NEXT:    cmovel %eax, %ecx
+; SSSE3-NEXT:    xorl $7, %ecx
+; SSSE3-NEXT:    movd %ecx, %xmm2
+; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3],xmm2[4],xmm0[4],xmm2[5],xmm0[5],xmm2[6],xmm0[6],xmm2[7],xmm0[7]
+; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1],xmm2[2],xmm3[2],xmm2[3],xmm3[3],xmm2[4],xmm3[4],xmm2[5],xmm3[5],xmm2[6],xmm3[6],xmm2[7],xmm3[7]
+; SSSE3-NEXT:    bsrl %r9d, %ecx
+; SSSE3-NEXT:    cmovel %eax, %ecx
+; SSSE3-NEXT:    xorl $7, %ecx
+; SSSE3-NEXT:    movd %ecx, %xmm0
+; SSSE3-NEXT:    bsrl %r10d, %ecx
+; SSSE3-NEXT:    cmovel %eax, %ecx
+; SSSE3-NEXT:    xorl $7, %ecx
+; SSSE3-NEXT:    movd %ecx, %xmm3
+; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm3 = xmm3[0],xmm0[0],xmm3[1],xmm0[1],xmm3[2],xmm0[2],xmm3[3],xmm0[3],xmm3[4],xmm0[4],xmm3[5],xmm0[5],xmm3[6],xmm0[6],xmm3[7],xmm0[7]
+; SSSE3-NEXT:    bsrl %r8d, %ecx
+; SSSE3-NEXT:    cmovel %eax, %ecx
+; SSSE3-NEXT:    xorl $7, %ecx
+; SSSE3-NEXT:    movd %ecx, %xmm4
+; SSSE3-NEXT:    movzbl -{{[0-9]+}}(%rsp), %ecx
+; SSSE3-NEXT:    bsrl %ecx, %ecx
+; SSSE3-NEXT:    cmovel %eax, %ecx
+; SSSE3-NEXT:    xorl $7, %ecx
+; SSSE3-NEXT:    movd %ecx, %xmm0
+; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm4[0],xmm0[1],xmm4[1],xmm0[2],xmm4[2],xmm0[3],xmm4[3],xmm0[4],xmm4[4],xmm0[5],xmm4[5],xmm0[6],xmm4[6],xmm0[7],xmm4[7]
+; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1],xmm0[2],xmm3[2],xmm0[3],xmm3[3],xmm0[4],xmm3[4],xmm0[5],xmm3[5],xmm0[6],xmm3[6],xmm0[7],xmm3[7]
+; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1],xmm0[2],xmm2[2],xmm0[3],xmm2[3],xmm0[4],xmm2[4],xmm0[5],xmm2[5],xmm0[6],xmm2[6],xmm0[7],xmm2[7]
+; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3],xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7]
+; SSSE3-NEXT:    popq %rbx
+; SSSE3-NEXT:    popq %rbp
+; SSSE3-NEXT:    retq
+;
+; SSE41-LABEL: testv16i8:
+; SSE41:       # BB#0:
+; SSE41-NEXT:    pextrb $1, %xmm0, %eax
+; SSE41-NEXT:    bsrl %eax, %ecx
+; SSE41-NEXT:    movl $15, %eax
+; SSE41-NEXT:    cmovel %eax, %ecx
+; SSE41-NEXT:    xorl $7, %ecx
+; SSE41-NEXT:    pextrb $0, %xmm0, %edx
+; SSE41-NEXT:    bsrl %edx, %edx
+; SSE41-NEXT:    cmovel %eax, %edx
+; SSE41-NEXT:    xorl $7, %edx
+; SSE41-NEXT:    movd %edx, %xmm1
+; SSE41-NEXT:    pinsrb $1, %ecx, %xmm1
+; SSE41-NEXT:    pextrb $2, %xmm0, %ecx
+; SSE41-NEXT:    bsrl %ecx, %ecx
+; SSE41-NEXT:    cmovel %eax, %ecx
+; SSE41-NEXT:    xorl $7, %ecx
+; SSE41-NEXT:    pinsrb $2, %ecx, %xmm1
+; SSE41-NEXT:    pextrb $3, %xmm0, %ecx
+; SSE41-NEXT:    bsrl %ecx, %ecx
+; SSE41-NEXT:    cmovel %eax, %ecx
+; SSE41-NEXT:    xorl $7, %ecx
+; SSE41-NEXT:    pinsrb $3, %ecx, %xmm1
+; SSE41-NEXT:    pextrb $4, %xmm0, %ecx
+; SSE41-NEXT:    bsrl %ecx, %ecx
+; SSE41-NEXT:    cmovel %eax, %ecx
+; SSE41-NEXT:    xorl $7, %ecx
+; SSE41-NEXT:    pinsrb $4, %ecx, %xmm1
+; SSE41-NEXT:    pextrb $5, %xmm0, %ecx
+; SSE41-NEXT:    bsrl %ecx, %ecx
+; SSE41-NEXT:    cmovel %eax, %ecx
+; SSE41-NEXT:    xorl $7, %ecx
+; SSE41-NEXT:    pinsrb $5, %ecx, %xmm1
+; SSE41-NEXT:    pextrb $6, %xmm0, %ecx
+; SSE41-NEXT:    bsrl %ecx, %ecx
+; SSE41-NEXT:    cmovel %eax, %ecx
+; SSE41-NEXT:    xorl $7, %ecx
+; SSE41-NEXT:    pinsrb $6, %ecx, %xmm1
+; SSE41-NEXT:    pextrb $7, %xmm0, %ecx
+; SSE41-NEXT:    bsrl %ecx, %ecx
+; SSE41-NEXT:    cmovel %eax, %ecx
+; SSE41-NEXT:    xorl $7, %ecx
+; SSE41-NEXT:    pinsrb $7, %ecx, %xmm1
+; SSE41-NEXT:    pextrb $8, %xmm0, %ecx
+; SSE41-NEXT:    bsrl %ecx, %ecx
+; SSE41-NEXT:    cmovel %eax, %ecx
+; SSE41-NEXT:    xorl $7, %ecx
+; SSE41-NEXT:    pinsrb $8, %ecx, %xmm1
+; SSE41-NEXT:    pextrb $9, %xmm0, %ecx
+; SSE41-NEXT:    bsrl %ecx, %ecx
+; SSE41-NEXT:    cmovel %eax, %ecx
+; SSE41-NEXT:    xorl $7, %ecx
+; SSE41-NEXT:    pinsrb $9, %ecx, %xmm1
+; SSE41-NEXT:    pextrb $10, %xmm0, %ecx
+; SSE41-NEXT:    bsrl %ecx, %ecx
+; SSE41-NEXT:    cmovel %eax, %ecx
+; SSE41-NEXT:    xorl $7, %ecx
+; SSE41-NEXT:    pinsrb $10, %ecx, %xmm1
+; SSE41-NEXT:    pextrb $11, %xmm0, %ecx
+; SSE41-NEXT:    bsrl %ecx, %ecx
+; SSE41-NEXT:    cmovel %eax, %ecx
+; SSE41-NEXT:    xorl $7, %ecx
+; SSE41-NEXT:    pinsrb $11, %ecx, %xmm1
+; SSE41-NEXT:    pextrb $12, %xmm0, %ecx
+; SSE41-NEXT:    bsrl %ecx, %ecx
+; SSE41-NEXT:    cmovel %eax, %ecx
+; SSE41-NEXT:    xorl $7, %ecx
+; SSE41-NEXT:    pinsrb $12, %ecx, %xmm1
+; SSE41-NEXT:    pextrb $13, %xmm0, %ecx
+; SSE41-NEXT:    bsrl %ecx, %ecx
+; SSE41-NEXT:    cmovel %eax, %ecx
+; SSE41-NEXT:    xorl $7, %ecx
+; SSE41-NEXT:    pinsrb $13, %ecx, %xmm1
+; SSE41-NEXT:    pextrb $14, %xmm0, %ecx
+; SSE41-NEXT:    bsrl %ecx, %ecx
+; SSE41-NEXT:    cmovel %eax, %ecx
+; SSE41-NEXT:    xorl $7, %ecx
+; SSE41-NEXT:    pinsrb $14, %ecx, %xmm1
+; SSE41-NEXT:    pextrb $15, %xmm0, %ecx
+; SSE41-NEXT:    bsrl %ecx, %ecx
+; SSE41-NEXT:    cmovel %eax, %ecx
+; SSE41-NEXT:    xorl $7, %ecx
+; SSE41-NEXT:    pinsrb $15, %ecx, %xmm1
+; SSE41-NEXT:    movdqa %xmm1, %xmm0
+; SSE41-NEXT:    retq
+;
+; AVX-LABEL: testv16i8:
+; AVX:       # BB#0:
+; AVX-NEXT:    vpextrb $1, %xmm0, %eax
+; AVX-NEXT:    bsrl %eax, %ecx
+; AVX-NEXT:    movl $15, %eax
+; AVX-NEXT:    cmovel %eax, %ecx
+; AVX-NEXT:    xorl $7, %ecx
+; AVX-NEXT:    vpextrb $0, %xmm0, %edx
+; AVX-NEXT:    bsrl %edx, %edx
+; AVX-NEXT:    cmovel %eax, %edx
+; AVX-NEXT:    xorl $7, %edx
+; AVX-NEXT:    vmovd %edx, %xmm1
+; AVX-NEXT:    vpinsrb $1, %ecx, %xmm1, %xmm1
+; AVX-NEXT:    vpextrb $2, %xmm0, %ecx
+; AVX-NEXT:    bsrl %ecx, %ecx
+; AVX-NEXT:    cmovel %eax, %ecx
+; AVX-NEXT:    xorl $7, %ecx
+; AVX-NEXT:    vpinsrb $2, %ecx, %xmm1, %xmm1
+; AVX-NEXT:    vpextrb $3, %xmm0, %ecx
+; AVX-NEXT:    bsrl %ecx, %ecx
+; AVX-NEXT:    cmovel %eax, %ecx
+; AVX-NEXT:    xorl $7, %ecx
+; AVX-NEXT:    vpinsrb $3, %ecx, %xmm1, %xmm1
+; AVX-NEXT:    vpextrb $4, %xmm0, %ecx
+; AVX-NEXT:    bsrl %ecx, %ecx
+; AVX-NEXT:    cmovel %eax, %ecx
+; AVX-NEXT:    xorl $7, %ecx
+; AVX-NEXT:    vpinsrb $4, %ecx, %xmm1, %xmm1
+; AVX-NEXT:    vpextrb $5, %xmm0, %ecx
+; AVX-NEXT:    bsrl %ecx, %ecx
+; AVX-NEXT:    cmovel %eax, %ecx
+; AVX-NEXT:    xorl $7, %ecx
+; AVX-NEXT:    vpinsrb $5, %ecx, %xmm1, %xmm1
+; AVX-NEXT:    vpextrb $6, %xmm0, %ecx
+; AVX-NEXT:    bsrl %ecx, %ecx
+; AVX-NEXT:    cmovel %eax, %ecx
+; AVX-NEXT:    xorl $7, %ecx
+; AVX-NEXT:    vpinsrb $6, %ecx, %xmm1, %xmm1
+; AVX-NEXT:    vpextrb $7, %xmm0, %ecx
+; AVX-NEXT:    bsrl %ecx, %ecx
+; AVX-NEXT:    cmovel %eax, %ecx
+; AVX-NEXT:    xorl $7, %ecx
+; AVX-NEXT:    vpinsrb $7, %ecx, %xmm1, %xmm1
+; AVX-NEXT:    vpextrb $8, %xmm0, %ecx
+; AVX-NEXT:    bsrl %ecx, %ecx
+; AVX-NEXT:    cmovel %eax, %ecx
+; AVX-NEXT:    xorl $7, %ecx
+; AVX-NEXT:    vpinsrb $8, %ecx, %xmm1, %xmm1
+; AVX-NEXT:    vpextrb $9, %xmm0, %ecx
+; AVX-NEXT:    bsrl %ecx, %ecx
+; AVX-NEXT:    cmovel %eax, %ecx
+; AVX-NEXT:    xorl $7, %ecx
+; AVX-NEXT:    vpinsrb $9, %ecx, %xmm1, %xmm1
+; AVX-NEXT:    vpextrb $10, %xmm0, %ecx
+; AVX-NEXT:    bsrl %ecx, %ecx
+; AVX-NEXT:    cmovel %eax, %ecx
+; AVX-NEXT:    xorl $7, %ecx
+; AVX-NEXT:    vpinsrb $10, %ecx, %xmm1, %xmm1
+; AVX-NEXT:    vpextrb $11, %xmm0, %ecx
+; AVX-NEXT:    bsrl %ecx, %ecx
+; AVX-NEXT:    cmovel %eax, %ecx
+; AVX-NEXT:    xorl $7, %ecx
+; AVX-NEXT:    vpinsrb $11, %ecx, %xmm1, %xmm1
+; AVX-NEXT:    vpextrb $12, %xmm0, %ecx
+; AVX-NEXT:    bsrl %ecx, %ecx
+; AVX-NEXT:    cmovel %eax, %ecx
+; AVX-NEXT:    xorl $7, %ecx
+; AVX-NEXT:    vpinsrb $12, %ecx, %xmm1, %xmm1
+; AVX-NEXT:    vpextrb $13, %xmm0, %ecx
+; AVX-NEXT:    bsrl %ecx, %ecx
+; AVX-NEXT:    cmovel %eax, %ecx
+; AVX-NEXT:    xorl $7, %ecx
+; AVX-NEXT:    vpinsrb $13, %ecx, %xmm1, %xmm1
+; AVX-NEXT:    vpextrb $14, %xmm0, %ecx
+; AVX-NEXT:    bsrl %ecx, %ecx
+; AVX-NEXT:    cmovel %eax, %ecx
+; AVX-NEXT:    xorl $7, %ecx
+; AVX-NEXT:    vpinsrb $14, %ecx, %xmm1, %xmm1
+; AVX-NEXT:    vpextrb $15, %xmm0, %ecx
+; AVX-NEXT:    bsrl %ecx, %ecx
+; AVX-NEXT:    cmovel %eax, %ecx
+; AVX-NEXT:    xorl $7, %ecx
+; AVX-NEXT:    vpinsrb $15, %ecx, %xmm1, %xmm0
+; AVX-NEXT:    retq
+  %out = call <16 x i8> @llvm.ctlz.v16i8(<16 x i8> %in, i1 0)
+  ret <16 x i8> %out
+}
+
+define <16 x i8> @testv16i8u(<16 x i8> %in) nounwind {
+; SSE2-LABEL: testv16i8u:
+; SSE2:       # BB#0:
+; SSE2-NEXT:    pushq %rbx
+; SSE2-NEXT:    movaps %xmm0, -{{[0-9]+}}(%rsp)
+; SSE2-NEXT:    movzbl -{{[0-9]+}}(%rsp), %eax
+; SSE2-NEXT:    bsrl %eax, %eax
+; SSE2-NEXT:    xorl $7, %eax
+; SSE2-NEXT:    movd %eax, %xmm0
+; SSE2-NEXT:    movzbl -{{[0-9]+}}(%rsp), %edi
+; SSE2-NEXT:    movzbl -{{[0-9]+}}(%rsp), %edx
+; SSE2-NEXT:    movzbl -{{[0-9]+}}(%rsp), %r9d
+; SSE2-NEXT:    movzbl -{{[0-9]+}}(%rsp), %eax
+; SSE2-NEXT:    movzbl -{{[0-9]+}}(%rsp), %r10d
+; SSE2-NEXT:    movzbl -{{[0-9]+}}(%rsp), %ecx
+; SSE2-NEXT:    movzbl -{{[0-9]+}}(%rsp), %r8d
+; SSE2-NEXT:    movzbl -{{[0-9]+}}(%rsp), %esi
+; SSE2-NEXT:    bsrl %esi, %esi
+; SSE2-NEXT:    xorl $7, %esi
+; SSE2-NEXT:    movd %esi, %xmm1
+; SSE2-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3],xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
+; SSE2-NEXT:    bsrl %eax, %eax
+; SSE2-NEXT:    xorl $7, %eax
+; SSE2-NEXT:    movd %eax, %xmm0
+; SSE2-NEXT:    movzbl -{{[0-9]+}}(%rsp), %eax
+; SSE2-NEXT:    movzbl -{{[0-9]+}}(%rsp), %esi
+; SSE2-NEXT:    movzbl -{{[0-9]+}}(%rsp), %r11d
+; SSE2-NEXT:    movzbl -{{[0-9]+}}(%rsp), %ebx
+; SSE2-NEXT:    bsrl %ebx, %ebx
+; SSE2-NEXT:    xorl $7, %ebx
+; SSE2-NEXT:    movd %ebx, %xmm2
+; SSE2-NEXT:    punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3],xmm2[4],xmm0[4],xmm2[5],xmm0[5],xmm2[6],xmm0[6],xmm2[7],xmm0[7]
+; SSE2-NEXT:    punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3],xmm2[4],xmm1[4],xmm2[5],xmm1[5],xmm2[6],xmm1[6],xmm2[7],xmm1[7]
+; SSE2-NEXT:    bsrl %edx, %edx
+; SSE2-NEXT:    xorl $7, %edx
+; SSE2-NEXT:    movd %edx, %xmm0
+; SSE2-NEXT:    bsrl %esi, %edx
+; SSE2-NEXT:    xorl $7, %edx
+; SSE2-NEXT:    movd %edx, %xmm3
+; SSE2-NEXT:    punpcklbw {{.*#+}} xmm3 = xmm3[0],xmm0[0],xmm3[1],xmm0[1],xmm3[2],xmm0[2],xmm3[3],xmm0[3],xmm3[4],xmm0[4],xmm3[5],xmm0[5],xmm3[6],xmm0[6],xmm3[7],xmm0[7]
+; SSE2-NEXT:    bsrl %ecx, %ecx
+; SSE2-NEXT:    xorl $7, %ecx
+; SSE2-NEXT:    movd %ecx, %xmm0
+; SSE2-NEXT:    movzbl -{{[0-9]+}}(%rsp), %ecx
+; SSE2-NEXT:    movzbl -{{[0-9]+}}(%rsp), %edx
+; SSE2-NEXT:    bsrl %edx, %edx
+; SSE2-NEXT:    xorl $7, %edx
+; SSE2-NEXT:    movd %edx, %xmm1
+; SSE2-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3],xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
+; SSE2-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm3[0],xmm1[1],xmm3[1],xmm1[2],xmm3[2],xmm1[3],xmm3[3],xmm1[4],xmm3[4],xmm1[5],xmm3[5],xmm1[6],xmm3[6],xmm1[7],xmm3[7]
+; SSE2-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1],xmm1[2],xmm2[2],xmm1[3],xmm2[3],xmm1[4],xmm2[4],xmm1[5],xmm2[5],xmm1[6],xmm2[6],xmm1[7],xmm2[7]
+; SSE2-NEXT:    bsrl %edi, %edx
+; SSE2-NEXT:    xorl $7, %edx
+; SSE2-NEXT:    movd %edx, %xmm0
+; SSE2-NEXT:    bsrl %eax, %eax
+; SSE2-NEXT:    xorl $7, %eax
+; SSE2-NEXT:    movd %eax, %xmm2
+; SSE2-NEXT:    punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3],xmm2[4],xmm0[4],xmm2[5],xmm0[5],xmm2[6],xmm0[6],xmm2[7],xmm0[7]
+; SSE2-NEXT:    bsrl %r10d, %eax
+; SSE2-NEXT:    xorl $7, %eax
+; SSE2-NEXT:    movd %eax, %xmm0
+; SSE2-NEXT:    bsrl %ecx, %eax
+; SSE2-NEXT:    xorl $7, %eax
+; SSE2-NEXT:    movd %eax, %xmm3
+; SSE2-NEXT:    punpcklbw {{.*#+}} xmm3 = xmm3[0],xmm0[0],xmm3[1],xmm0[1],xmm3[2],xmm0[2],xmm3[3],xmm0[3],xmm3[4],xmm0[4],xmm3[5],xmm0[5],xmm3[6],xmm0[6],xmm3[7],xmm0[7]
+; SSE2-NEXT:    punpcklbw {{.*#+}} xmm3 = xmm3[0],xmm2[0],xmm3[1],xmm2[1],xmm3[2],xmm2[2],xmm3[3],xmm2[3],xmm3[4],xmm2[4],xmm3[5],xmm2[5],xmm3[6],xmm2[6],xmm3[7],xmm2[7]
+; SSE2-NEXT:    bsrl %r9d, %eax
+; SSE2-NEXT:    xorl $7, %eax
+; SSE2-NEXT:    movd %eax, %xmm0
+; SSE2-NEXT:    bsrl %r11d, %eax
+; SSE2-NEXT:    xorl $7, %eax
+; SSE2-NEXT:    movd %eax, %xmm2
+; SSE2-NEXT:    punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3],xmm2[4],xmm0[4],xmm2[5],xmm0[5],xmm2[6],xmm0[6],xmm2[7],xmm0[7]
+; SSE2-NEXT:    bsrl %r8d, %eax
+; SSE2-NEXT:    xorl $7, %eax
+; SSE2-NEXT:    movd %eax, %xmm4
+; SSE2-NEXT:    movzbl -{{[0-9]+}}(%rsp), %eax
+; SSE2-NEXT:    bsrl %eax, %eax
+; SSE2-NEXT:    xorl $7, %eax
+; SSE2-NEXT:    movd %eax, %xmm0
+; SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm4[0],xmm0[1],xmm4[1],xmm0[2],xmm4[2],xmm0[3],xmm4[3],xmm0[4],xmm4[4],xmm0[5],xmm4[5],xmm0[6],xmm4[6],xmm0[7],xmm4[7]
+; SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1],xmm0[2],xmm2[2],xmm0[3],xmm2[3],xmm0[4],xmm2[4],xmm0[5],xmm2[5],xmm0[6],xmm2[6],xmm0[7],xmm2[7]
+; SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1],xmm0[2],xmm3[2],xmm0[3],xmm3[3],xmm0[4],xmm3[4],xmm0[5],xmm3[5],xmm0[6],xmm3[6],xmm0[7],xmm3[7]
+; SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3],xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7]
+; SSE2-NEXT:    popq %rbx
+; SSE2-NEXT:    retq
+;
+; SSE3-LABEL: testv16i8u:
+; SSE3:       # BB#0:
+; SSE3-NEXT:    pushq %rbx
+; SSE3-NEXT:    movaps %xmm0, -{{[0-9]+}}(%rsp)
+; SSE3-NEXT:    movzbl -{{[0-9]+}}(%rsp), %eax
+; SSE3-NEXT:    bsrl %eax, %eax
+; SSE3-NEXT:    xorl $7, %eax
+; SSE3-NEXT:    movd %eax, %xmm0
+; SSE3-NEXT:    movzbl -{{[0-9]+}}(%rsp), %edi
+; SSE3-NEXT:    movzbl -{{[0-9]+}}(%rsp), %edx
+; SSE3-NEXT:    movzbl -{{[0-9]+}}(%rsp), %r9d
+; SSE3-NEXT:    movzbl -{{[0-9]+}}(%rsp), %eax
+; SSE3-NEXT:    movzbl -{{[0-9]+}}(%rsp), %r10d
+; SSE3-NEXT:    movzbl -{{[0-9]+}}(%rsp), %ecx
+; SSE3-NEXT:    movzbl -{{[0-9]+}}(%rsp), %r8d
+; SSE3-NEXT:    movzbl -{{[0-9]+}}(%rsp), %esi
+; SSE3-NEXT:    bsrl %esi, %esi
+; SSE3-NEXT:    xorl $7, %esi
+; SSE3-NEXT:    movd %esi, %xmm1
+; SSE3-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3],xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
+; SSE3-NEXT:    bsrl %eax, %eax
+; SSE3-NEXT:    xorl $7, %eax
+; SSE3-NEXT:    movd %eax, %xmm0
+; SSE3-NEXT:    movzbl -{{[0-9]+}}(%rsp), %eax
+; SSE3-NEXT:    movzbl -{{[0-9]+}}(%rsp), %esi
+; SSE3-NEXT:    movzbl -{{[0-9]+}}(%rsp), %r11d
+; SSE3-NEXT:    movzbl -{{[0-9]+}}(%rsp), %ebx
+; SSE3-NEXT:    bsrl %ebx, %ebx
+; SSE3-NEXT:    xorl $7, %ebx
+; SSE3-NEXT:    movd %ebx, %xmm2
+; SSE3-NEXT:    punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3],xmm2[4],xmm0[4],xmm2[5],xmm0[5],xmm2[6],xmm0[6],xmm2[7],xmm0[7]
+; SSE3-NEXT:    punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3],xmm2[4],xmm1[4],xmm2[5],xmm1[5],xmm2[6],xmm1[6],xmm2[7],xmm1[7]
+; SSE3-NEXT:    bsrl %edx, %edx
+; SSE3-NEXT:    xorl $7, %edx
+; SSE3-NEXT:    movd %edx, %xmm0
+; SSE3-NEXT:    bsrl %esi, %edx
+; SSE3-NEXT:    xorl $7, %edx
+; SSE3-NEXT:    movd %edx, %xmm3
+; SSE3-NEXT:    punpcklbw {{.*#+}} xmm3 = xmm3[0],xmm0[0],xmm3[1],xmm0[1],xmm3[2],xmm0[2],xmm3[3],xmm0[3],xmm3[4],xmm0[4],xmm3[5],xmm0[5],xmm3[6],xmm0[6],xmm3[7],xmm0[7]
+; SSE3-NEXT:    bsrl %ecx, %ecx
+; SSE3-NEXT:    xorl $7, %ecx
+; SSE3-NEXT:    movd %ecx, %xmm0
+; SSE3-NEXT:    movzbl -{{[0-9]+}}(%rsp), %ecx
+; SSE3-NEXT:    movzbl -{{[0-9]+}}(%rsp), %edx
+; SSE3-NEXT:    bsrl %edx, %edx
+; SSE3-NEXT:    xorl $7, %edx
+; SSE3-NEXT:    movd %edx, %xmm1
+; SSE3-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3],xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
+; SSE3-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm3[0],xmm1[1],xmm3[1],xmm1[2],xmm3[2],xmm1[3],xmm3[3],xmm1[4],xmm3[4],xmm1[5],xmm3[5],xmm1[6],xmm3[6],xmm1[7],xmm3[7]
+; SSE3-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1],xmm1[2],xmm2[2],xmm1[3],xmm2[3],xmm1[4],xmm2[4],xmm1[5],xmm2[5],xmm1[6],xmm2[6],xmm1[7],xmm2[7]
+; SSE3-NEXT:    bsrl %edi, %edx
+; SSE3-NEXT:    xorl $7, %edx
+; SSE3-NEXT:    movd %edx, %xmm0
+; SSE3-NEXT:    bsrl %eax, %eax
+; SSE3-NEXT:    xorl $7, %eax
+; SSE3-NEXT:    movd %eax, %xmm2
+; SSE3-NEXT:    punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3],xmm2[4],xmm0[4],xmm2[5],xmm0[5],xmm2[6],xmm0[6],xmm2[7],xmm0[7]
+; SSE3-NEXT:    bsrl %r10d, %eax
+; SSE3-NEXT:    xorl $7, %eax
+; SSE3-NEXT:    movd %eax, %xmm0
+; SSE3-NEXT:    bsrl %ecx, %eax
+; SSE3-NEXT:    xorl $7, %eax
+; SSE3-NEXT:    movd %eax, %xmm3
+; SSE3-NEXT:    punpcklbw {{.*#+}} xmm3 = xmm3[0],xmm0[0],xmm3[1],xmm0[1],xmm3[2],xmm0[2],xmm3[3],xmm0[3],xmm3[4],xmm0[4],xmm3[5],xmm0[5],xmm3[6],xmm0[6],xmm3[7],xmm0[7]
+; SSE3-NEXT:    punpcklbw {{.*#+}} xmm3 = xmm3[0],xmm2[0],xmm3[1],xmm2[1],xmm3[2],xmm2[2],xmm3[3],xmm2[3],xmm3[4],xmm2[4],xmm3[5],xmm2[5],xmm3[6],xmm2[6],xmm3[7],xmm2[7]
+; SSE3-NEXT:    bsrl %r9d, %eax
+; SSE3-NEXT:    xorl $7, %eax
+; SSE3-NEXT:    movd %eax, %xmm0
+; SSE3-NEXT:    bsrl %r11d, %eax
+; SSE3-NEXT:    xorl $7, %eax
+; SSE3-NEXT:    movd %eax, %xmm2
+; SSE3-NEXT:    punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3],xmm2[4],xmm0[4],xmm2[5],xmm0[5],xmm2[6],xmm0[6],xmm2[7],xmm0[7]
+; SSE3-NEXT:    bsrl %r8d, %eax
+; SSE3-NEXT:    xorl $7, %eax
+; SSE3-NEXT:    movd %eax, %xmm4
+; SSE3-NEXT:    movzbl -{{[0-9]+}}(%rsp), %eax
+; SSE3-NEXT:    bsrl %eax, %eax
+; SSE3-NEXT:    xorl $7, %eax
+; SSE3-NEXT:    movd %eax, %xmm0
+; SSE3-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm4[0],xmm0[1],xmm4[1],xmm0[2],xmm4[2],xmm0[3],xmm4[3],xmm0[4],xmm4[4],xmm0[5],xmm4[5],xmm0[6],xmm4[6],xmm0[7],xmm4[7]
+; SSE3-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1],xmm0[2],xmm2[2],xmm0[3],xmm2[3],xmm0[4],xmm2[4],xmm0[5],xmm2[5],xmm0[6],xmm2[6],xmm0[7],xmm2[7]
+; SSE3-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1],xmm0[2],xmm3[2],xmm0[3],xmm3[3],xmm0[4],xmm3[4],xmm0[5],xmm3[5],xmm0[6],xmm3[6],xmm0[7],xmm3[7]
+; SSE3-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3],xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7]
+; SSE3-NEXT:    popq %rbx
+; SSE3-NEXT:    retq
+;
+; SSSE3-LABEL: testv16i8u:
+; SSSE3:       # BB#0:
+; SSSE3-NEXT:    pushq %rbx
+; SSSE3-NEXT:    movaps %xmm0, -{{[0-9]+}}(%rsp)
+; SSSE3-NEXT:    movzbl -{{[0-9]+}}(%rsp), %eax
+; SSSE3-NEXT:    bsrl %eax, %eax
+; SSSE3-NEXT:    xorl $7, %eax
+; SSSE3-NEXT:    movd %eax, %xmm0
+; SSSE3-NEXT:    movzbl -{{[0-9]+}}(%rsp), %edi
+; SSSE3-NEXT:    movzbl -{{[0-9]+}}(%rsp), %edx
+; SSSE3-NEXT:    movzbl -{{[0-9]+}}(%rsp), %r9d
+; SSSE3-NEXT:    movzbl -{{[0-9]+}}(%rsp), %eax
+; SSSE3-NEXT:    movzbl -{{[0-9]+}}(%rsp), %r10d
+; SSSE3-NEXT:    movzbl -{{[0-9]+}}(%rsp), %ecx
+; SSSE3-NEXT:    movzbl -{{[0-9]+}}(%rsp), %r8d
+; SSSE3-NEXT:    movzbl -{{[0-9]+}}(%rsp), %esi
+; SSSE3-NEXT:    bsrl %esi, %esi
+; SSSE3-NEXT:    xorl $7, %esi
+; SSSE3-NEXT:    movd %esi, %xmm1
+; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3],xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
+; SSSE3-NEXT:    bsrl %eax, %eax
+; SSSE3-NEXT:    xorl $7, %eax
+; SSSE3-NEXT:    movd %eax, %xmm0
+; SSSE3-NEXT:    movzbl -{{[0-9]+}}(%rsp), %eax
+; SSSE3-NEXT:    movzbl -{{[0-9]+}}(%rsp), %esi
+; SSSE3-NEXT:    movzbl -{{[0-9]+}}(%rsp), %r11d
+; SSSE3-NEXT:    movzbl -{{[0-9]+}}(%rsp), %ebx
+; SSSE3-NEXT:    bsrl %ebx, %ebx
+; SSSE3-NEXT:    xorl $7, %ebx
+; SSSE3-NEXT:    movd %ebx, %xmm2
+; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3],xmm2[4],xmm0[4],xmm2[5],xmm0[5],xmm2[6],xmm0[6],xmm2[7],xmm0[7]
+; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3],xmm2[4],xmm1[4],xmm2[5],xmm1[5],xmm2[6],xmm1[6],xmm2[7],xmm1[7]
+; SSSE3-NEXT:    bsrl %edx, %edx
+; SSSE3-NEXT:    xorl $7, %edx
+; SSSE3-NEXT:    movd %edx, %xmm0
+; SSSE3-NEXT:    bsrl %esi, %edx
+; SSSE3-NEXT:    xorl $7, %edx
+; SSSE3-NEXT:    movd %edx, %xmm3
+; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm3 = xmm3[0],xmm0[0],xmm3[1],xmm0[1],xmm3[2],xmm0[2],xmm3[3],xmm0[3],xmm3[4],xmm0[4],xmm3[5],xmm0[5],xmm3[6],xmm0[6],xmm3[7],xmm0[7]
+; SSSE3-NEXT:    bsrl %ecx, %ecx
+; SSSE3-NEXT:    xorl $7, %ecx
+; SSSE3-NEXT:    movd %ecx, %xmm0
+; SSSE3-NEXT:    movzbl -{{[0-9]+}}(%rsp), %ecx
+; SSSE3-NEXT:    movzbl -{{[0-9]+}}(%rsp), %edx
+; SSSE3-NEXT:    bsrl %edx, %edx
+; SSSE3-NEXT:    xorl $7, %edx
+; SSSE3-NEXT:    movd %edx, %xmm1
+; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3],xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
+; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm3[0],xmm1[1],xmm3[1],xmm1[2],xmm3[2],xmm1[3],xmm3[3],xmm1[4],xmm3[4],xmm1[5],xmm3[5],xmm1[6],xmm3[6],xmm1[7],xmm3[7]
+; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1],xmm1[2],xmm2[2],xmm1[3],xmm2[3],xmm1[4],xmm2[4],xmm1[5],xmm2[5],xmm1[6],xmm2[6],xmm1[7],xmm2[7]
+; SSSE3-NEXT:    bsrl %edi, %edx
+; SSSE3-NEXT:    xorl $7, %edx
+; SSSE3-NEXT:    movd %edx, %xmm0
+; SSSE3-NEXT:    bsrl %eax, %eax
+; SSSE3-NEXT:    xorl $7, %eax
+; SSSE3-NEXT:    movd %eax, %xmm2
+; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3],xmm2[4],xmm0[4],xmm2[5],xmm0[5],xmm2[6],xmm0[6],xmm2[7],xmm0[7]
+; SSSE3-NEXT:    bsrl %r10d, %eax
+; SSSE3-NEXT:    xorl $7, %eax
+; SSSE3-NEXT:    movd %eax, %xmm0
+; SSSE3-NEXT:    bsrl %ecx, %eax
+; SSSE3-NEXT:    xorl $7, %eax
+; SSSE3-NEXT:    movd %eax, %xmm3
+; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm3 = xmm3[0],xmm0[0],xmm3[1],xmm0[1],xmm3[2],xmm0[2],xmm3[3],xmm0[3],xmm3[4],xmm0[4],xmm3[5],xmm0[5],xmm3[6],xmm0[6],xmm3[7],xmm0[7]
+; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm3 = xmm3[0],xmm2[0],xmm3[1],xmm2[1],xmm3[2],xmm2[2],xmm3[3],xmm2[3],xmm3[4],xmm2[4],xmm3[5],xmm2[5],xmm3[6],xmm2[6],xmm3[7],xmm2[7]
+; SSSE3-NEXT:    bsrl %r9d, %eax
+; SSSE3-NEXT:    xorl $7, %eax
+; SSSE3-NEXT:    movd %eax, %xmm0
+; SSSE3-NEXT:    bsrl %r11d, %eax
+; SSSE3-NEXT:    xorl $7, %eax
+; SSSE3-NEXT:    movd %eax, %xmm2
+; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3],xmm2[4],xmm0[4],xmm2[5],xmm0[5],xmm2[6],xmm0[6],xmm2[7],xmm0[7]
+; SSSE3-NEXT:    bsrl %r8d, %eax
+; SSSE3-NEXT:    xorl $7, %eax
+; SSSE3-NEXT:    movd %eax, %xmm4
+; SSSE3-NEXT:    movzbl -{{[0-9]+}}(%rsp), %eax
+; SSSE3-NEXT:    bsrl %eax, %eax
+; SSSE3-NEXT:    xorl $7, %eax
+; SSSE3-NEXT:    movd %eax, %xmm0
+; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm4[0],xmm0[1],xmm4[1],xmm0[2],xmm4[2],xmm0[3],xmm4[3],xmm0[4],xmm4[4],xmm0[5],xmm4[5],xmm0[6],xmm4[6],xmm0[7],xmm4[7]
+; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1],xmm0[2],xmm2[2],xmm0[3],xmm2[3],xmm0[4],xmm2[4],xmm0[5],xmm2[5],xmm0[6],xmm2[6],xmm0[7],xmm2[7]
+; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1],xmm0[2],xmm3[2],xmm0[3],xmm3[3],xmm0[4],xmm3[4],xmm0[5],xmm3[5],xmm0[6],xmm3[6],xmm0[7],xmm3[7]
+; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3],xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7]
+; SSSE3-NEXT:    popq %rbx
+; SSSE3-NEXT:    retq
+;
+; SSE41-LABEL: testv16i8u:
+; SSE41:       # BB#0:
+; SSE41-NEXT:    pextrb $1, %xmm0, %eax
+; SSE41-NEXT:    bsrl %eax, %eax
+; SSE41-NEXT:    xorl $7, %eax
+; SSE41-NEXT:    pextrb $0, %xmm0, %ecx
+; SSE41-NEXT:    bsrl %ecx, %ecx
+; SSE41-NEXT:    xorl $7, %ecx
+; SSE41-NEXT:    movd %ecx, %xmm1
+; SSE41-NEXT:    pinsrb $1, %eax, %xmm1
+; SSE41-NEXT:    pextrb $2, %xmm0, %eax
+; SSE41-NEXT:    bsrl %eax, %eax
+; SSE41-NEXT:    xorl $7, %eax
+; SSE41-NEXT:    pinsrb $2, %eax, %xmm1
+; SSE41-NEXT:    pextrb $3, %xmm0, %eax
+; SSE41-NEXT:    bsrl %eax, %eax
+; SSE41-NEXT:    xorl $7, %eax
+; SSE41-NEXT:    pinsrb $3, %eax, %xmm1
+; SSE41-NEXT:    pextrb $4, %xmm0, %eax
+; SSE41-NEXT:    bsrl %eax, %eax
+; SSE41-NEXT:    xorl $7, %eax
+; SSE41-NEXT:    pinsrb $4, %eax, %xmm1
+; SSE41-NEXT:    pextrb $5, %xmm0, %eax
+; SSE41-NEXT:    bsrl %eax, %eax
+; SSE41-NEXT:    xorl $7, %eax
+; SSE41-NEXT:    pinsrb $5, %eax, %xmm1
+; SSE41-NEXT:    pextrb $6, %xmm0, %eax
+; SSE41-NEXT:    bsrl %eax, %eax
+; SSE41-NEXT:    xorl $7, %eax
+; SSE41-NEXT:    pinsrb $6, %eax, %xmm1
+; SSE41-NEXT:    pextrb $7, %xmm0, %eax
+; SSE41-NEXT:    bsrl %eax, %eax
+; SSE41-NEXT:    xorl $7, %eax
+; SSE41-NEXT:    pinsrb $7, %eax, %xmm1
+; SSE41-NEXT:    pextrb $8, %xmm0, %eax
+; SSE41-NEXT:    bsrl %eax, %eax
+; SSE41-NEXT:    xorl $7, %eax
+; SSE41-NEXT:    pinsrb $8, %eax, %xmm1
+; SSE41-NEXT:    pextrb $9, %xmm0, %eax
+; SSE41-NEXT:    bsrl %eax, %eax
+; SSE41-NEXT:    xorl $7, %eax
+; SSE41-NEXT:    pinsrb $9, %eax, %xmm1
+; SSE41-NEXT:    pextrb $10, %xmm0, %eax
+; SSE41-NEXT:    bsrl %eax, %eax
+; SSE41-NEXT:    xorl $7, %eax
+; SSE41-NEXT:    pinsrb $10, %eax, %xmm1
+; SSE41-NEXT:    pextrb $11, %xmm0, %eax
+; SSE41-NEXT:    bsrl %eax, %eax
+; SSE41-NEXT:    xorl $7, %eax
+; SSE41-NEXT:    pinsrb $11, %eax, %xmm1
+; SSE41-NEXT:    pextrb $12, %xmm0, %eax
+; SSE41-NEXT:    bsrl %eax, %eax
+; SSE41-NEXT:    xorl $7, %eax
+; SSE41-NEXT:    pinsrb $12, %eax, %xmm1
+; SSE41-NEXT:    pextrb $13, %xmm0, %eax
+; SSE41-NEXT:    bsrl %eax, %eax
+; SSE41-NEXT:    xorl $7, %eax
+; SSE41-NEXT:    pinsrb $13, %eax, %xmm1
+; SSE41-NEXT:    pextrb $14, %xmm0, %eax
+; SSE41-NEXT:    bsrl %eax, %eax
+; SSE41-NEXT:    xorl $7, %eax
+; SSE41-NEXT:    pinsrb $14, %eax, %xmm1
+; SSE41-NEXT:    pextrb $15, %xmm0, %eax
+; SSE41-NEXT:    bsrl %eax, %eax
+; SSE41-NEXT:    xorl $7, %eax
+; SSE41-NEXT:    pinsrb $15, %eax, %xmm1
+; SSE41-NEXT:    movdqa %xmm1, %xmm0
+; SSE41-NEXT:    retq
+;
+; AVX-LABEL: testv16i8u:
+; AVX:       # BB#0:
+; AVX-NEXT:    vpextrb $1, %xmm0, %eax
+; AVX-NEXT:    bsrl %eax, %eax
+; AVX-NEXT:    xorl $7, %eax
+; AVX-NEXT:    vpextrb $0, %xmm0, %ecx
+; AVX-NEXT:    bsrl %ecx, %ecx
+; AVX-NEXT:    xorl $7, %ecx
+; AVX-NEXT:    vmovd %ecx, %xmm1
+; AVX-NEXT:    vpinsrb $1, %eax, %xmm1, %xmm1
+; AVX-NEXT:    vpextrb $2, %xmm0, %eax
+; AVX-NEXT:    bsrl %eax, %eax
+; AVX-NEXT:    xorl $7, %eax
+; AVX-NEXT:    vpinsrb $2, %eax, %xmm1, %xmm1
+; AVX-NEXT:    vpextrb $3, %xmm0, %eax
+; AVX-NEXT:    bsrl %eax, %eax
+; AVX-NEXT:    xorl $7, %eax
+; AVX-NEXT:    vpinsrb $3, %eax, %xmm1, %xmm1
+; AVX-NEXT:    vpextrb $4, %xmm0, %eax
+; AVX-NEXT:    bsrl %eax, %eax
+; AVX-NEXT:    xorl $7, %eax
+; AVX-NEXT:    vpinsrb $4, %eax, %xmm1, %xmm1
+; AVX-NEXT:    vpextrb $5, %xmm0, %eax
+; AVX-NEXT:    bsrl %eax, %eax
+; AVX-NEXT:    xorl $7, %eax
+; AVX-NEXT:    vpinsrb $5, %eax, %xmm1, %xmm1
+; AVX-NEXT:    vpextrb $6, %xmm0, %eax
+; AVX-NEXT:    bsrl %eax, %eax
+; AVX-NEXT:    xorl $7, %eax
+; AVX-NEXT:    vpinsrb $6, %eax, %xmm1, %xmm1
+; AVX-NEXT:    vpextrb $7, %xmm0, %eax
+; AVX-NEXT:    bsrl %eax, %eax
+; AVX-NEXT:    xorl $7, %eax
+; AVX-NEXT:    vpinsrb $7, %eax, %xmm1, %xmm1
+; AVX-NEXT:    vpextrb $8, %xmm0, %eax
+; AVX-NEXT:    bsrl %eax, %eax
+; AVX-NEXT:    xorl $7, %eax
+; AVX-NEXT:    vpinsrb $8, %eax, %xmm1, %xmm1
+; AVX-NEXT:    vpextrb $9, %xmm0, %eax
+; AVX-NEXT:    bsrl %eax, %eax
+; AVX-NEXT:    xorl $7, %eax
+; AVX-NEXT:    vpinsrb $9, %eax, %xmm1, %xmm1
+; AVX-NEXT:    vpextrb $10, %xmm0, %eax
+; AVX-NEXT:    bsrl %eax, %eax
+; AVX-NEXT:    xorl $7, %eax
+; AVX-NEXT:    vpinsrb $10, %eax, %xmm1, %xmm1
+; AVX-NEXT:    vpextrb $11, %xmm0, %eax
+; AVX-NEXT:    bsrl %eax, %eax
+; AVX-NEXT:    xorl $7, %eax
+; AVX-NEXT:    vpinsrb $11, %eax, %xmm1, %xmm1
+; AVX-NEXT:    vpextrb $12, %xmm0, %eax
+; AVX-NEXT:    bsrl %eax, %eax
+; AVX-NEXT:    xorl $7, %eax
+; AVX-NEXT:    vpinsrb $12, %eax, %xmm1, %xmm1
+; AVX-NEXT:    vpextrb $13, %xmm0, %eax
+; AVX-NEXT:    bsrl %eax, %eax
+; AVX-NEXT:    xorl $7, %eax
+; AVX-NEXT:    vpinsrb $13, %eax, %xmm1, %xmm1
+; AVX-NEXT:    vpextrb $14, %xmm0, %eax
+; AVX-NEXT:    bsrl %eax, %eax
+; AVX-NEXT:    xorl $7, %eax
+; AVX-NEXT:    vpinsrb $14, %eax, %xmm1, %xmm1
+; AVX-NEXT:    vpextrb $15, %xmm0, %eax
+; AVX-NEXT:    bsrl %eax, %eax
+; AVX-NEXT:    xorl $7, %eax
+; AVX-NEXT:    vpinsrb $15, %eax, %xmm1, %xmm0
+; AVX-NEXT:    retq
+  %out = call <16 x i8> @llvm.ctlz.v16i8(<16 x i8> %in, i1 -1)
+  ret <16 x i8> %out
+}
+
+define <2 x i64> @foldv2i64() nounwind {
+; SSE-LABEL: foldv2i64:
+; SSE:       # BB#0:
+; SSE-NEXT:    movl $55, %eax
+; SSE-NEXT:    movd %rax, %xmm0
+; SSE-NEXT:    retq
+;
+; AVX-LABEL: foldv2i64:
+; AVX:       # BB#0:
+; AVX-NEXT:    movl $55, %eax
+; AVX-NEXT:    vmovq %rax, %xmm0
+; AVX-NEXT:    retq
+  %out = call <2 x i64> @llvm.ctlz.v2i64(<2 x i64> <i64 256, i64 -1>, i1 0)
+  ret <2 x i64> %out
+}
+
+define <2 x i64> @foldv2i64u() nounwind {
+; SSE-LABEL: foldv2i64u:
+; SSE:       # BB#0:
+; SSE-NEXT:    movl $55, %eax
+; SSE-NEXT:    movd %rax, %xmm0
+; SSE-NEXT:    retq
+;
+; AVX-LABEL: foldv2i64u:
+; AVX:       # BB#0:
+; AVX-NEXT:    movl $55, %eax
+; AVX-NEXT:    vmovq %rax, %xmm0
+; AVX-NEXT:    retq
+  %out = call <2 x i64> @llvm.ctlz.v2i64(<2 x i64> <i64 256, i64 -1>, i1 -1)
+  ret <2 x i64> %out
+}
+
+define <4 x i32> @foldv4i32() nounwind {
+; SSE-LABEL: foldv4i32:
+; SSE:       # BB#0:
+; SSE-NEXT:    movaps {{.*#+}} xmm0 = [23,0,32,24]
+; SSE-NEXT:    retq
+;
+; AVX-LABEL: foldv4i32:
+; AVX:       # BB#0:
+; AVX-NEXT:    vmovaps {{.*#+}} xmm0 = [23,0,32,24]
+; AVX-NEXT:    retq
+  %out = call <4 x i32> @llvm.ctlz.v4i32(<4 x i32> <i32 256, i32 -1, i32 0, i32 255>, i1 0)
+  ret <4 x i32> %out
+}
+
+define <4 x i32> @foldv4i32u() nounwind {
+; SSE-LABEL: foldv4i32u:
+; SSE:       # BB#0:
+; SSE-NEXT:    movaps {{.*#+}} xmm0 = [23,0,32,24]
+; SSE-NEXT:    retq
+;
+; AVX-LABEL: foldv4i32u:
+; AVX:       # BB#0:
+; AVX-NEXT:    vmovaps {{.*#+}} xmm0 = [23,0,32,24]
+; AVX-NEXT:    retq
+  %out = call <4 x i32> @llvm.ctlz.v4i32(<4 x i32> <i32 256, i32 -1, i32 0, i32 255>, i1 -1)
+  ret <4 x i32> %out
+}
+
+define <8 x i16> @foldv8i16() nounwind {
+; SSE-LABEL: foldv8i16:
+; SSE:       # BB#0:
+; SSE-NEXT:    movaps {{.*#+}} xmm0 = [7,0,16,8,16,13,11,9]
+; SSE-NEXT:    retq
+;
+; AVX-LABEL: foldv8i16:
+; AVX:       # BB#0:
+; AVX-NEXT:    vmovaps {{.*#+}} xmm0 = [7,0,16,8,16,13,11,9]
+; AVX-NEXT:    retq
+  %out = call <8 x i16> @llvm.ctlz.v8i16(<8 x i16> <i16 256, i16 -1, i16 0, i16 255, i16 -65536, i16 7, i16 24, i16 88>, i1 0)
+  ret <8 x i16> %out
+}
+
+define <8 x i16> @foldv8i16u() nounwind {
+; SSE-LABEL: foldv8i16u:
+; SSE:       # BB#0:
+; SSE-NEXT:    movaps {{.*#+}} xmm0 = [7,0,16,8,16,13,11,9]
+; SSE-NEXT:    retq
+;
+; AVX-LABEL: foldv8i16u:
+; AVX:       # BB#0:
+; AVX-NEXT:    vmovaps {{.*#+}} xmm0 = [7,0,16,8,16,13,11,9]
+; AVX-NEXT:    retq
+  %out = call <8 x i16> @llvm.ctlz.v8i16(<8 x i16> <i16 256, i16 -1, i16 0, i16 255, i16 -65536, i16 7, i16 24, i16 88>, i1 -1)
+  ret <8 x i16> %out
+}
+
+define <16 x i8> @foldv16i8() nounwind {
+; SSE-LABEL: foldv16i8:
+; SSE:       # BB#0:
+; SSE-NEXT:    movaps {{.*#+}} xmm0 = [8,0,8,0,8,5,3,1,0,0,7,6,5,4,3,2]
+; SSE-NEXT:    retq
+;
+; AVX-LABEL: foldv16i8:
+; AVX:       # BB#0:
+; AVX-NEXT:    vmovaps {{.*#+}} xmm0 = [8,0,8,0,8,5,3,1,0,0,7,6,5,4,3,2]
+; AVX-NEXT:    retq
+  %out = call <16 x i8> @llvm.ctlz.v16i8(<16 x i8> <i8 256, i8 -1, i8 0, i8 255, i8 -65536, i8 7, i8 24, i8 88, i8 -2, i8 254, i8 1, i8 2, i8 4, i8 8, i8 16, i8 32>, i1 0)
+  ret <16 x i8> %out
+}
+
+define <16 x i8> @foldv16i8u() nounwind {
+; SSE-LABEL: foldv16i8u:
+; SSE:       # BB#0:
+; SSE-NEXT:    movaps {{.*#+}} xmm0 = [8,0,8,0,8,5,3,1,0,0,7,6,5,4,3,2]
+; SSE-NEXT:    retq
+;
+; AVX-LABEL: foldv16i8u:
+; AVX:       # BB#0:
+; AVX-NEXT:    vmovaps {{.*#+}} xmm0 = [8,0,8,0,8,5,3,1,0,0,7,6,5,4,3,2]
+; AVX-NEXT:    retq
+  %out = call <16 x i8> @llvm.ctlz.v16i8(<16 x i8> <i8 256, i8 -1, i8 0, i8 255, i8 -65536, i8 7, i8 24, i8 88, i8 -2, i8 254, i8 1, i8 2, i8 4, i8 8, i8 16, i8 32>, i1 -1)
+  ret <16 x i8> %out
+}
+
+declare <2 x i64> @llvm.ctlz.v2i64(<2 x i64>, i1)
+declare <4 x i32> @llvm.ctlz.v4i32(<4 x i32>, i1)
+declare <8 x i16> @llvm.ctlz.v8i16(<8 x i16>, i1)
+declare <16 x i8> @llvm.ctlz.v16i8(<16 x i8>, i1)
diff --git a/test/CodeGen/X86/vector-lzcnt-256.ll b/test/CodeGen/X86/vector-lzcnt-256.ll
new file mode 100644
index 00000000000..2083fa8d436
--- /dev/null
+++ b/test/CodeGen/X86/vector-lzcnt-256.ll
@@ -0,0 +1,1305 @@
+; RUN: llc < %s -mcpu=x86-64 -mattr=+avx | FileCheck %s --check-prefix=ALL --check-prefix=AVX --check-prefix=AVX1
+; RUN: llc < %s -mcpu=x86-64 -mattr=+avx2 | FileCheck %s --check-prefix=ALL --check-prefix=AVX --check-prefix=AVX2
+
+target triple = "x86_64-unknown-unknown"
+
+define <4 x i64> @testv4i64(<4 x i64> %in) nounwind {
+; AVX1-LABEL: testv4i64:
+; AVX1:       # BB#0:
+; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
+; AVX1-NEXT:    vpextrq $1, %xmm1, %rax
+; AVX1-NEXT:    bsrq %rax, %rax
+; AVX1-NEXT:    movl $127, %ecx
+; AVX1-NEXT:    cmoveq %rcx, %rax
+; AVX1-NEXT:    xorq $63, %rax
+; AVX1-NEXT:    vmovq %rax, %xmm2
+; AVX1-NEXT:    vmovq %xmm1, %rax
+; AVX1-NEXT:    bsrq %rax, %rax
+; AVX1-NEXT:    cmoveq %rcx, %rax
+; AVX1-NEXT:    xorq $63, %rax
+; AVX1-NEXT:    vmovq %rax, %xmm1
+; AVX1-NEXT:    vpunpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm2[0]
+; AVX1-NEXT:    vpextrq $1, %xmm0, %rax
+; AVX1-NEXT:    bsrq %rax, %rax
+; AVX1-NEXT:    cmoveq %rcx, %rax
+; AVX1-NEXT:    xorq $63, %rax
+; AVX1-NEXT:    vmovq %rax, %xmm2
+; AVX1-NEXT:    vmovq %xmm0, %rax
+; AVX1-NEXT:    bsrq %rax, %rax
+; AVX1-NEXT:    cmoveq %rcx, %rax
+; AVX1-NEXT:    xorq $63, %rax
+; AVX1-NEXT:    vmovq %rax, %xmm0
+; AVX1-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm2[0]
+; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: testv4i64:
+; AVX2:       # BB#0:
+; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
+; AVX2-NEXT:    vpextrq $1, %xmm1, %rax
+; AVX2-NEXT:    bsrq %rax, %rax
+; AVX2-NEXT:    movl $127, %ecx
+; AVX2-NEXT:    cmoveq %rcx, %rax
+; AVX2-NEXT:    xorq $63, %rax
+; AVX2-NEXT:    vmovq %rax, %xmm2
+; AVX2-NEXT:    vmovq %xmm1, %rax
+; AVX2-NEXT:    bsrq %rax, %rax
+; AVX2-NEXT:    cmoveq %rcx, %rax
+; AVX2-NEXT:    xorq $63, %rax
+; AVX2-NEXT:    vmovq %rax, %xmm1
+; AVX2-NEXT:    vpunpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm2[0]
+; AVX2-NEXT:    vpextrq $1, %xmm0, %rax
+; AVX2-NEXT:    bsrq %rax, %rax
+; AVX2-NEXT:    cmoveq %rcx, %rax
+; AVX2-NEXT:    xorq $63, %rax
+; AVX2-NEXT:    vmovq %rax, %xmm2
+; AVX2-NEXT:    vmovq %xmm0, %rax
+; AVX2-NEXT:    bsrq %rax, %rax
+; AVX2-NEXT:    cmoveq %rcx, %rax
+; AVX2-NEXT:    xorq $63, %rax
+; AVX2-NEXT:    vmovq %rax, %xmm0
+; AVX2-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm2[0]
+; AVX2-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm0
+; AVX2-NEXT:    retq
+  %out = call <4 x i64> @llvm.ctlz.v4i64(<4 x i64> %in, i1 0)
+  ret <4 x i64> %out
+}
+
+define <4 x i64> @testv4i64u(<4 x i64> %in) nounwind {
+; AVX1-LABEL: testv4i64u:
+; AVX1:       # BB#0:
+; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
+; AVX1-NEXT:    vpextrq $1, %xmm1, %rax
+; AVX1-NEXT:    bsrq %rax, %rax
+; AVX1-NEXT:    xorq $63, %rax
+; AVX1-NEXT:    vmovq %rax, %xmm2
+; AVX1-NEXT:    vmovq %xmm1, %rax
+; AVX1-NEXT:    bsrq %rax, %rax
+; AVX1-NEXT:    xorq $63, %rax
+; AVX1-NEXT:    vmovq %rax, %xmm1
+; AVX1-NEXT:    vpunpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm2[0]
+; AVX1-NEXT:    vpextrq $1, %xmm0, %rax
+; AVX1-NEXT:    bsrq %rax, %rax
+; AVX1-NEXT:    xorq $63, %rax
+; AVX1-NEXT:    vmovq %rax, %xmm2
+; AVX1-NEXT:    vmovq %xmm0, %rax
+; AVX1-NEXT:    bsrq %rax, %rax
+; AVX1-NEXT:    xorq $63, %rax
+; AVX1-NEXT:    vmovq %rax, %xmm0
+; AVX1-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm2[0]
+; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: testv4i64u:
+; AVX2:       # BB#0:
+; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
+; AVX2-NEXT:    vpextrq $1, %xmm1, %rax
+; AVX2-NEXT:    bsrq %rax, %rax
+; AVX2-NEXT:    xorq $63, %rax
+; AVX2-NEXT:    vmovq %rax, %xmm2
+; AVX2-NEXT:    vmovq %xmm1, %rax
+; AVX2-NEXT:    bsrq %rax, %rax
+; AVX2-NEXT:    xorq $63, %rax
+; AVX2-NEXT:    vmovq %rax, %xmm1
+; AVX2-NEXT:    vpunpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm2[0]
+; AVX2-NEXT:    vpextrq $1, %xmm0, %rax
+; AVX2-NEXT:    bsrq %rax, %rax
+; AVX2-NEXT:    xorq $63, %rax
+; AVX2-NEXT:    vmovq %rax, %xmm2
+; AVX2-NEXT:    vmovq %xmm0, %rax
+; AVX2-NEXT:    bsrq %rax, %rax
+; AVX2-NEXT:    xorq $63, %rax
+; AVX2-NEXT:    vmovq %rax, %xmm0
+; AVX2-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm2[0]
+; AVX2-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm0
+; AVX2-NEXT:    retq
+  %out = call <4 x i64> @llvm.ctlz.v4i64(<4 x i64> %in, i1 -1)
+  ret <4 x i64> %out
+}
+
+define <8 x i32> @testv8i32(<8 x i32> %in) nounwind {
+; AVX1-LABEL: testv8i32:
+; AVX1:       # BB#0:
+; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
+; AVX1-NEXT:    vpextrd $1, %xmm1, %eax
+; AVX1-NEXT:    bsrl %eax, %ecx
+; AVX1-NEXT:    movl $63, %eax
+; AVX1-NEXT:    cmovel %eax, %ecx
+; AVX1-NEXT:    xorl $31, %ecx
+; AVX1-NEXT:    vmovd %xmm1, %edx
+; AVX1-NEXT:    bsrl %edx, %edx
+; AVX1-NEXT:    cmovel %eax, %edx
+; AVX1-NEXT:    xorl $31, %edx
+; AVX1-NEXT:    vmovd %edx, %xmm2
+; AVX1-NEXT:    vpinsrd $1, %ecx, %xmm2, %xmm2
+; AVX1-NEXT:    vpextrd $2, %xmm1, %ecx
+; AVX1-NEXT:    bsrl %ecx, %ecx
+; AVX1-NEXT:    cmovel %eax, %ecx
+; AVX1-NEXT:    xorl $31, %ecx
+; AVX1-NEXT:    vpinsrd $2, %ecx, %xmm2, %xmm2
+; AVX1-NEXT:    vpextrd $3, %xmm1, %ecx
+; AVX1-NEXT:    bsrl %ecx, %ecx
+; AVX1-NEXT:    cmovel %eax, %ecx
+; AVX1-NEXT:    xorl $31, %ecx
+; AVX1-NEXT:    vpinsrd $3, %ecx, %xmm2, %xmm1
+; AVX1-NEXT:    vpextrd $1, %xmm0, %ecx
+; AVX1-NEXT:    bsrl %ecx, %ecx
+; AVX1-NEXT:    cmovel %eax, %ecx
+; AVX1-NEXT:    xorl $31, %ecx
+; AVX1-NEXT:    vmovd %xmm0, %edx
+; AVX1-NEXT:    bsrl %edx, %edx
+; AVX1-NEXT:    cmovel %eax, %edx
+; AVX1-NEXT:    xorl $31, %edx
+; AVX1-NEXT:    vmovd %edx, %xmm2
+; AVX1-NEXT:    vpinsrd $1, %ecx, %xmm2, %xmm2
+; AVX1-NEXT:    vpextrd $2, %xmm0, %ecx
+; AVX1-NEXT:    bsrl %ecx, %ecx
+; AVX1-NEXT:    cmovel %eax, %ecx
+; AVX1-NEXT:    xorl $31, %ecx
+; AVX1-NEXT:    vpinsrd $2, %ecx, %xmm2, %xmm2
+; AVX1-NEXT:    vpextrd $3, %xmm0, %ecx
+; AVX1-NEXT:    bsrl %ecx, %ecx
+; AVX1-NEXT:    cmovel %eax, %ecx
+; AVX1-NEXT:    xorl $31, %ecx
+; AVX1-NEXT:    vpinsrd $3, %ecx, %xmm2, %xmm0
+; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: testv8i32:
+; AVX2:       # BB#0:
+; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
+; AVX2-NEXT:    vpextrd $1, %xmm1, %eax
+; AVX2-NEXT:    bsrl %eax, %ecx
+; AVX2-NEXT:    movl $63, %eax
+; AVX2-NEXT:    cmovel %eax, %ecx
+; AVX2-NEXT:    xorl $31, %ecx
+; AVX2-NEXT:    vmovd %xmm1, %edx
+; AVX2-NEXT:    bsrl %edx, %edx
+; AVX2-NEXT:    cmovel %eax, %edx
+; AVX2-NEXT:    xorl $31, %edx
+; AVX2-NEXT:    vmovd %edx, %xmm2
+; AVX2-NEXT:    vpinsrd $1, %ecx, %xmm2, %xmm2
+; AVX2-NEXT:    vpextrd $2, %xmm1, %ecx
+; AVX2-NEXT:    bsrl %ecx, %ecx
+; AVX2-NEXT:    cmovel %eax, %ecx
+; AVX2-NEXT:    xorl $31, %ecx
+; AVX2-NEXT:    vpinsrd $2, %ecx, %xmm2, %xmm2
+; AVX2-NEXT:    vpextrd $3, %xmm1, %ecx
+; AVX2-NEXT:    bsrl %ecx, %ecx
+; AVX2-NEXT:    cmovel %eax, %ecx
+; AVX2-NEXT:    xorl $31, %ecx
+; AVX2-NEXT:    vpinsrd $3, %ecx, %xmm2, %xmm1
+; AVX2-NEXT:    vpextrd $1, %xmm0, %ecx
+; AVX2-NEXT:    bsrl %ecx, %ecx
+; AVX2-NEXT:    cmovel %eax, %ecx
+; AVX2-NEXT:    xorl $31, %ecx
+; AVX2-NEXT:    vmovd %xmm0, %edx
+; AVX2-NEXT:    bsrl %edx, %edx
+; AVX2-NEXT:    cmovel %eax, %edx
+; AVX2-NEXT:    xorl $31, %edx
+; AVX2-NEXT:    vmovd %edx, %xmm2
+; AVX2-NEXT:    vpinsrd $1, %ecx, %xmm2, %xmm2
+; AVX2-NEXT:    vpextrd $2, %xmm0, %ecx
+; AVX2-NEXT:    bsrl %ecx, %ecx
+; AVX2-NEXT:    cmovel %eax, %ecx
+; AVX2-NEXT:    xorl $31, %ecx
+; AVX2-NEXT:    vpinsrd $2, %ecx, %xmm2, %xmm2
+; AVX2-NEXT:    vpextrd $3, %xmm0, %ecx
+; AVX2-NEXT:    bsrl %ecx, %ecx
+; AVX2-NEXT:    cmovel %eax, %ecx
+; AVX2-NEXT:    xorl $31, %ecx
+; AVX2-NEXT:    vpinsrd $3, %ecx, %xmm2, %xmm0
+; AVX2-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm0
+; AVX2-NEXT:    retq
+  %out = call <8 x i32> @llvm.ctlz.v8i32(<8 x i32> %in, i1 0)
+  ret <8 x i32> %out
+}
+
+define <8 x i32> @testv8i32u(<8 x i32> %in) nounwind {
+; AVX1-LABEL: testv8i32u:
+; AVX1:       # BB#0:
+; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
+; AVX1-NEXT:    vpextrd $1, %xmm1, %eax
+; AVX1-NEXT:    bsrl %eax, %eax
+; AVX1-NEXT:    xorl $31, %eax
+; AVX1-NEXT:    vmovd %xmm1, %ecx
+; AVX1-NEXT:    bsrl %ecx, %ecx
+; AVX1-NEXT:    xorl $31, %ecx
+; AVX1-NEXT:    vmovd %ecx, %xmm2
+; AVX1-NEXT:    vpinsrd $1, %eax, %xmm2, %xmm2
+; AVX1-NEXT:    vpextrd $2, %xmm1, %eax
+; AVX1-NEXT:    bsrl %eax, %eax
+; AVX1-NEXT:    xorl $31, %eax
+; AVX1-NEXT:    vpinsrd $2, %eax, %xmm2, %xmm2
+; AVX1-NEXT:    vpextrd $3, %xmm1, %eax
+; AVX1-NEXT:    bsrl %eax, %eax
+; AVX1-NEXT:    xorl $31, %eax
+; AVX1-NEXT:    vpinsrd $3, %eax, %xmm2, %xmm1
+; AVX1-NEXT:    vpextrd $1, %xmm0, %eax
+; AVX1-NEXT:    bsrl %eax, %eax
+; AVX1-NEXT:    xorl $31, %eax
+; AVX1-NEXT:    vmovd %xmm0, %ecx
+; AVX1-NEXT:    bsrl %ecx, %ecx
+; AVX1-NEXT:    xorl $31, %ecx
+; AVX1-NEXT:    vmovd %ecx, %xmm2
+; AVX1-NEXT:    vpinsrd $1, %eax, %xmm2, %xmm2
+; AVX1-NEXT:    vpextrd $2, %xmm0, %eax
+; AVX1-NEXT:    bsrl %eax, %eax
+; AVX1-NEXT:    xorl $31, %eax
+; AVX1-NEXT:    vpinsrd $2, %eax, %xmm2, %xmm2
+; AVX1-NEXT:    vpextrd $3, %xmm0, %eax
+; AVX1-NEXT:    bsrl %eax, %eax
+; AVX1-NEXT:    xorl $31, %eax
+; AVX1-NEXT:    vpinsrd $3, %eax, %xmm2, %xmm0
+; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: testv8i32u:
+; AVX2:       # BB#0:
+; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
+; AVX2-NEXT:    vpextrd $1, %xmm1, %eax
+; AVX2-NEXT:    bsrl %eax, %eax
+; AVX2-NEXT:    xorl $31, %eax
+; AVX2-NEXT:    vmovd %xmm1, %ecx
+; AVX2-NEXT:    bsrl %ecx, %ecx
+; AVX2-NEXT:    xorl $31, %ecx
+; AVX2-NEXT:    vmovd %ecx, %xmm2
+; AVX2-NEXT:    vpinsrd $1, %eax, %xmm2, %xmm2
+; AVX2-NEXT:    vpextrd $2, %xmm1, %eax
+; AVX2-NEXT:    bsrl %eax, %eax
+; AVX2-NEXT:    xorl $31, %eax
+; AVX2-NEXT:    vpinsrd $2, %eax, %xmm2, %xmm2
+; AVX2-NEXT:    vpextrd $3, %xmm1, %eax
+; AVX2-NEXT:    bsrl %eax, %eax
+; AVX2-NEXT:    xorl $31, %eax
+; AVX2-NEXT:    vpinsrd $3, %eax, %xmm2, %xmm1
+; AVX2-NEXT:    vpextrd $1, %xmm0, %eax
+; AVX2-NEXT:    bsrl %eax, %eax
+; AVX2-NEXT:    xorl $31, %eax
+; AVX2-NEXT:    vmovd %xmm0, %ecx
+; AVX2-NEXT:    bsrl %ecx, %ecx
+; AVX2-NEXT:    xorl $31, %ecx
+; AVX2-NEXT:    vmovd %ecx, %xmm2
+; AVX2-NEXT:    vpinsrd $1, %eax, %xmm2, %xmm2
+; AVX2-NEXT:    vpextrd $2, %xmm0, %eax
+; AVX2-NEXT:    bsrl %eax, %eax
+; AVX2-NEXT:    xorl $31, %eax
+; AVX2-NEXT:    vpinsrd $2, %eax, %xmm2, %xmm2
+; AVX2-NEXT:    vpextrd $3, %xmm0, %eax
+; AVX2-NEXT:    bsrl %eax, %eax
+; AVX2-NEXT:    xorl $31, %eax
+; AVX2-NEXT:    vpinsrd $3, %eax, %xmm2, %xmm0
+; AVX2-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm0
+; AVX2-NEXT:    retq
+  %out = call <8 x i32> @llvm.ctlz.v8i32(<8 x i32> %in, i1 -1)
+  ret <8 x i32> %out
+}
+
+define <16 x i16> @testv16i16(<16 x i16> %in) nounwind {
+; AVX1-LABEL: testv16i16:
+; AVX1:       # BB#0:
+; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
+; AVX1-NEXT:    vpextrw $1, %xmm1, %eax
+; AVX1-NEXT:    bsrw %ax, %cx
+; AVX1-NEXT:    movw $31, %ax
+; AVX1-NEXT:    cmovew %ax, %cx
+; AVX1-NEXT:    xorl $15, %ecx
+; AVX1-NEXT:    vmovd %xmm1, %edx
+; AVX1-NEXT:    bsrw %dx, %dx
+; AVX1-NEXT:    cmovew %ax, %dx
+; AVX1-NEXT:    xorl $15, %edx
+; AVX1-NEXT:    vmovd %edx, %xmm2
+; AVX1-NEXT:    vpinsrw $1, %ecx, %xmm2, %xmm2
+; AVX1-NEXT:    vpextrw $2, %xmm1, %ecx
+; AVX1-NEXT:    bsrw %cx, %cx
+; AVX1-NEXT:    cmovew %ax, %cx
+; AVX1-NEXT:    xorl $15, %ecx
+; AVX1-NEXT:    vpinsrw $2, %ecx, %xmm2, %xmm2
+; AVX1-NEXT:    vpextrw $3, %xmm1, %ecx
+; AVX1-NEXT:    bsrw %cx, %cx
+; AVX1-NEXT:    cmovew %ax, %cx
+; AVX1-NEXT:    xorl $15, %ecx
+; AVX1-NEXT:    vpinsrw $3, %ecx, %xmm2, %xmm2
+; AVX1-NEXT:    vpextrw $4, %xmm1, %ecx
+; AVX1-NEXT:    bsrw %cx, %cx
+; AVX1-NEXT:    cmovew %ax, %cx
+; AVX1-NEXT:    xorl $15, %ecx
+; AVX1-NEXT:    vpinsrw $4, %ecx, %xmm2, %xmm2
+; AVX1-NEXT:    vpextrw $5, %xmm1, %ecx
+; AVX1-NEXT:    bsrw %cx, %cx
+; AVX1-NEXT:    cmovew %ax, %cx
+; AVX1-NEXT:    xorl $15, %ecx
+; AVX1-NEXT:    vpinsrw $5, %ecx, %xmm2, %xmm2
+; AVX1-NEXT:    vpextrw $6, %xmm1, %ecx
+; AVX1-NEXT:    bsrw %cx, %cx
+; AVX1-NEXT:    cmovew %ax, %cx
+; AVX1-NEXT:    xorl $15, %ecx
+; AVX1-NEXT:    vpinsrw $6, %ecx, %xmm2, %xmm2
+; AVX1-NEXT:    vpextrw $7, %xmm1, %ecx
+; AVX1-NEXT:    bsrw %cx, %cx
+; AVX1-NEXT:    cmovew %ax, %cx
+; AVX1-NEXT:    xorl $15, %ecx
+; AVX1-NEXT:    vpinsrw $7, %ecx, %xmm2, %xmm1
+; AVX1-NEXT:    vpextrw $1, %xmm0, %ecx
+; AVX1-NEXT:    bsrw %cx, %cx
+; AVX1-NEXT:    cmovew %ax, %cx
+; AVX1-NEXT:    xorl $15, %ecx
+; AVX1-NEXT:    vmovd %xmm0, %edx
+; AVX1-NEXT:    bsrw %dx, %dx
+; AVX1-NEXT:    cmovew %ax, %dx
+; AVX1-NEXT:    xorl $15, %edx
+; AVX1-NEXT:    vmovd %edx, %xmm2
+; AVX1-NEXT:    vpinsrw $1, %ecx, %xmm2, %xmm2
+; AVX1-NEXT:    vpextrw $2, %xmm0, %ecx
+; AVX1-NEXT:    bsrw %cx, %cx
+; AVX1-NEXT:    cmovew %ax, %cx
+; AVX1-NEXT:    xorl $15, %ecx
+; AVX1-NEXT:    vpinsrw $2, %ecx, %xmm2, %xmm2
+; AVX1-NEXT:    vpextrw $3, %xmm0, %ecx
+; AVX1-NEXT:    bsrw %cx, %cx
+; AVX1-NEXT:    cmovew %ax, %cx
+; AVX1-NEXT:    xorl $15, %ecx
+; AVX1-NEXT:    vpinsrw $3, %ecx, %xmm2, %xmm2
+; AVX1-NEXT:    vpextrw $4, %xmm0, %ecx
+; AVX1-NEXT:    bsrw %cx, %cx
+; AVX1-NEXT:    cmovew %ax, %cx
+; AVX1-NEXT:    xorl $15, %ecx
+; AVX1-NEXT:    vpinsrw $4, %ecx, %xmm2, %xmm2
+; AVX1-NEXT:    vpextrw $5, %xmm0, %ecx
+; AVX1-NEXT:    bsrw %cx, %cx
+; AVX1-NEXT:    cmovew %ax, %cx
+; AVX1-NEXT:    xorl $15, %ecx
+; AVX1-NEXT:    vpinsrw $5, %ecx, %xmm2, %xmm2
+; AVX1-NEXT:    vpextrw $6, %xmm0, %ecx
+; AVX1-NEXT:    bsrw %cx, %cx
+; AVX1-NEXT:    cmovew %ax, %cx
+; AVX1-NEXT:    xorl $15, %ecx
+; AVX1-NEXT:    vpinsrw $6, %ecx, %xmm2, %xmm2
+; AVX1-NEXT:    vpextrw $7, %xmm0, %ecx
+; AVX1-NEXT:    bsrw %cx, %cx
+; AVX1-NEXT:    cmovew %ax, %cx
+; AVX1-NEXT:    xorl $15, %ecx
+; AVX1-NEXT:    vpinsrw $7, %ecx, %xmm2, %xmm0
+; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: testv16i16:
+; AVX2:       # BB#0:
+; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
+; AVX2-NEXT:    vpextrw $1, %xmm1, %eax
+; AVX2-NEXT:    bsrw %ax, %cx
+; AVX2-NEXT:    movw $31, %ax
+; AVX2-NEXT:    cmovew %ax, %cx
+; AVX2-NEXT:    xorl $15, %ecx
+; AVX2-NEXT:    vmovd %xmm1, %edx
+; AVX2-NEXT:    bsrw %dx, %dx
+; AVX2-NEXT:    cmovew %ax, %dx
+; AVX2-NEXT:    xorl $15, %edx
+; AVX2-NEXT:    vmovd %edx, %xmm2
+; AVX2-NEXT:    vpinsrw $1, %ecx, %xmm2, %xmm2
+; AVX2-NEXT:    vpextrw $2, %xmm1, %ecx
+; AVX2-NEXT:    bsrw %cx, %cx
+; AVX2-NEXT:    cmovew %ax, %cx
+; AVX2-NEXT:    xorl $15, %ecx
+; AVX2-NEXT:    vpinsrw $2, %ecx, %xmm2, %xmm2
+; AVX2-NEXT:    vpextrw $3, %xmm1, %ecx
+; AVX2-NEXT:    bsrw %cx, %cx
+; AVX2-NEXT:    cmovew %ax, %cx
+; AVX2-NEXT:    xorl $15, %ecx
+; AVX2-NEXT:    vpinsrw $3, %ecx, %xmm2, %xmm2
+; AVX2-NEXT:    vpextrw $4, %xmm1, %ecx
+; AVX2-NEXT:    bsrw %cx, %cx
+; AVX2-NEXT:    cmovew %ax, %cx
+; AVX2-NEXT:    xorl $15, %ecx
+; AVX2-NEXT:    vpinsrw $4, %ecx, %xmm2, %xmm2
+; AVX2-NEXT:    vpextrw $5, %xmm1, %ecx
+; AVX2-NEXT:    bsrw %cx, %cx
+; AVX2-NEXT:    cmovew %ax, %cx
+; AVX2-NEXT:    xorl $15, %ecx
+; AVX2-NEXT:    vpinsrw $5, %ecx, %xmm2, %xmm2
+; AVX2-NEXT:    vpextrw $6, %xmm1, %ecx
+; AVX2-NEXT:    bsrw %cx, %cx
+; AVX2-NEXT:    cmovew %ax, %cx
+; AVX2-NEXT:    xorl $15, %ecx
+; AVX2-NEXT:    vpinsrw $6, %ecx, %xmm2, %xmm2
+; AVX2-NEXT:    vpextrw $7, %xmm1, %ecx
+; AVX2-NEXT:    bsrw %cx, %cx
+; AVX2-NEXT:    cmovew %ax, %cx
+; AVX2-NEXT:    xorl $15, %ecx
+; AVX2-NEXT:    vpinsrw $7, %ecx, %xmm2, %xmm1
+; AVX2-NEXT:    vpextrw $1, %xmm0, %ecx
+; AVX2-NEXT:    bsrw %cx, %cx
+; AVX2-NEXT:    cmovew %ax, %cx
+; AVX2-NEXT:    xorl $15, %ecx
+; AVX2-NEXT:    vmovd %xmm0, %edx
+; AVX2-NEXT:    bsrw %dx, %dx
+; AVX2-NEXT:    cmovew %ax, %dx
+; AVX2-NEXT:    xorl $15, %edx
+; AVX2-NEXT:    vmovd %edx, %xmm2
+; AVX2-NEXT:    vpinsrw $1, %ecx, %xmm2, %xmm2
+; AVX2-NEXT:    vpextrw $2, %xmm0, %ecx
+; AVX2-NEXT:    bsrw %cx, %cx
+; AVX2-NEXT:    cmovew %ax, %cx
+; AVX2-NEXT:    xorl $15, %ecx
+; AVX2-NEXT:    vpinsrw $2, %ecx, %xmm2, %xmm2
+; AVX2-NEXT:    vpextrw $3, %xmm0, %ecx
+; AVX2-NEXT:    bsrw %cx, %cx
+; AVX2-NEXT:    cmovew %ax, %cx
+; AVX2-NEXT:    xorl $15, %ecx
+; AVX2-NEXT:    vpinsrw $3, %ecx, %xmm2, %xmm2
+; AVX2-NEXT:    vpextrw $4, %xmm0, %ecx
+; AVX2-NEXT:    bsrw %cx, %cx
+; AVX2-NEXT:    cmovew %ax, %cx
+; AVX2-NEXT:    xorl $15, %ecx
+; AVX2-NEXT:    vpinsrw $4, %ecx, %xmm2, %xmm2
+; AVX2-NEXT:    vpextrw $5, %xmm0, %ecx
+; AVX2-NEXT:    bsrw %cx, %cx
+; AVX2-NEXT:    cmovew %ax, %cx
+; AVX2-NEXT:    xorl $15, %ecx
+; AVX2-NEXT:    vpinsrw $5, %ecx, %xmm2, %xmm2
+; AVX2-NEXT:    vpextrw $6, %xmm0, %ecx
+; AVX2-NEXT:    bsrw %cx, %cx
+; AVX2-NEXT:    cmovew %ax, %cx
+; AVX2-NEXT:    xorl $15, %ecx
+; AVX2-NEXT:    vpinsrw $6, %ecx, %xmm2, %xmm2
+; AVX2-NEXT:    vpextrw $7, %xmm0, %ecx
+; AVX2-NEXT:    bsrw %cx, %cx
+; AVX2-NEXT:    cmovew %ax, %cx
+; AVX2-NEXT:    xorl $15, %ecx
+; AVX2-NEXT:    vpinsrw $7, %ecx, %xmm2, %xmm0
+; AVX2-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm0
+; AVX2-NEXT:    retq
+  %out = call <16 x i16> @llvm.ctlz.v16i16(<16 x i16> %in, i1 0)
+  ret <16 x i16> %out
+}
+
+define <16 x i16> @testv16i16u(<16 x i16> %in) nounwind {
+; AVX1-LABEL: testv16i16u:
+; AVX1:       # BB#0:
+; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
+; AVX1-NEXT:    vpextrw $1, %xmm1, %eax
+; AVX1-NEXT:    bsrw %ax, %ax
+; AVX1-NEXT:    xorl $15, %eax
+; AVX1-NEXT:    vmovd %xmm1, %ecx
+; AVX1-NEXT:    bsrw %cx, %cx
+; AVX1-NEXT:    xorl $15, %ecx
+; AVX1-NEXT:    vmovd %ecx, %xmm2
+; AVX1-NEXT:    vpinsrw $1, %eax, %xmm2, %xmm2
+; AVX1-NEXT:    vpextrw $2, %xmm1, %eax
+; AVX1-NEXT:    bsrw %ax, %ax
+; AVX1-NEXT:    xorl $15, %eax
+; AVX1-NEXT:    vpinsrw $2, %eax, %xmm2, %xmm2
+; AVX1-NEXT:    vpextrw $3, %xmm1, %eax
+; AVX1-NEXT:    bsrw %ax, %ax
+; AVX1-NEXT:    xorl $15, %eax
+; AVX1-NEXT:    vpinsrw $3, %eax, %xmm2, %xmm2
+; AVX1-NEXT:    vpextrw $4, %xmm1, %eax
+; AVX1-NEXT:    bsrw %ax, %ax
+; AVX1-NEXT:    xorl $15, %eax
+; AVX1-NEXT:    vpinsrw $4, %eax, %xmm2, %xmm2
+; AVX1-NEXT:    vpextrw $5, %xmm1, %eax
+; AVX1-NEXT:    bsrw %ax, %ax
+; AVX1-NEXT:    xorl $15, %eax
+; AVX1-NEXT:    vpinsrw $5, %eax, %xmm2, %xmm2
+; AVX1-NEXT:    vpextrw $6, %xmm1, %eax
+; AVX1-NEXT:    bsrw %ax, %ax
+; AVX1-NEXT:    xorl $15, %eax
+; AVX1-NEXT:    vpinsrw $6, %eax, %xmm2, %xmm2
+; AVX1-NEXT:    vpextrw $7, %xmm1, %eax
+; AVX1-NEXT:    bsrw %ax, %ax
+; AVX1-NEXT:    xorl $15, %eax
+; AVX1-NEXT:    vpinsrw $7, %eax, %xmm2, %xmm1
+; AVX1-NEXT:    vpextrw $1, %xmm0, %eax
+; AVX1-NEXT:    bsrw %ax, %ax
+; AVX1-NEXT:    xorl $15, %eax
+; AVX1-NEXT:    vmovd %xmm0, %ecx
+; AVX1-NEXT:    bsrw %cx, %cx
+; AVX1-NEXT:    xorl $15, %ecx
+; AVX1-NEXT:    vmovd %ecx, %xmm2
+; AVX1-NEXT:    vpinsrw $1, %eax, %xmm2, %xmm2
+; AVX1-NEXT:    vpextrw $2, %xmm0, %eax
+; AVX1-NEXT:    bsrw %ax, %ax
+; AVX1-NEXT:    xorl $15, %eax
+; AVX1-NEXT:    vpinsrw $2, %eax, %xmm2, %xmm2
+; AVX1-NEXT:    vpextrw $3, %xmm0, %eax
+; AVX1-NEXT:    bsrw %ax, %ax
+; AVX1-NEXT:    xorl $15, %eax
+; AVX1-NEXT:    vpinsrw $3, %eax, %xmm2, %xmm2
+; AVX1-NEXT:    vpextrw $4, %xmm0, %eax
+; AVX1-NEXT:    bsrw %ax, %ax
+; AVX1-NEXT:    xorl $15, %eax
+; AVX1-NEXT:    vpinsrw $4, %eax, %xmm2, %xmm2
+; AVX1-NEXT:    vpextrw $5, %xmm0, %eax
+; AVX1-NEXT:    bsrw %ax, %ax
+; AVX1-NEXT:    xorl $15, %eax
+; AVX1-NEXT:    vpinsrw $5, %eax, %xmm2, %xmm2
+; AVX1-NEXT:    vpextrw $6, %xmm0, %eax
+; AVX1-NEXT:    bsrw %ax, %ax
+; AVX1-NEXT:    xorl $15, %eax
+; AVX1-NEXT:    vpinsrw $6, %eax, %xmm2, %xmm2
+; AVX1-NEXT:    vpextrw $7, %xmm0, %eax
+; AVX1-NEXT:    bsrw %ax, %ax
+; AVX1-NEXT:    xorl $15, %eax
+; AVX1-NEXT:    vpinsrw $7, %eax, %xmm2, %xmm0
+; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: testv16i16u:
+; AVX2:       # BB#0:
+; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
+; AVX2-NEXT:    vpextrw $1, %xmm1, %eax
+; AVX2-NEXT:    bsrw %ax, %ax
+; AVX2-NEXT:    xorl $15, %eax
+; AVX2-NEXT:    vmovd %xmm1, %ecx
+; AVX2-NEXT:    bsrw %cx, %cx
+; AVX2-NEXT:    xorl $15, %ecx
+; AVX2-NEXT:    vmovd %ecx, %xmm2
+; AVX2-NEXT:    vpinsrw $1, %eax, %xmm2, %xmm2
+; AVX2-NEXT:    vpextrw $2, %xmm1, %eax
+; AVX2-NEXT:    bsrw %ax, %ax
+; AVX2-NEXT:    xorl $15, %eax
+; AVX2-NEXT:    vpinsrw $2, %eax, %xmm2, %xmm2
+; AVX2-NEXT:    vpextrw $3, %xmm1, %eax
+; AVX2-NEXT:    bsrw %ax, %ax
+; AVX2-NEXT:    xorl $15, %eax
+; AVX2-NEXT:    vpinsrw $3, %eax, %xmm2, %xmm2
+; AVX2-NEXT:    vpextrw $4, %xmm1, %eax
+; AVX2-NEXT:    bsrw %ax, %ax
+; AVX2-NEXT:    xorl $15, %eax
+; AVX2-NEXT:    vpinsrw $4, %eax, %xmm2, %xmm2
+; AVX2-NEXT:    vpextrw $5, %xmm1, %eax
+; AVX2-NEXT:    bsrw %ax, %ax
+; AVX2-NEXT:    xorl $15, %eax
+; AVX2-NEXT:    vpinsrw $5, %eax, %xmm2, %xmm2
+; AVX2-NEXT:    vpextrw $6, %xmm1, %eax
+; AVX2-NEXT:    bsrw %ax, %ax
+; AVX2-NEXT:    xorl $15, %eax
+; AVX2-NEXT:    vpinsrw $6, %eax, %xmm2, %xmm2
+; AVX2-NEXT:    vpextrw $7, %xmm1, %eax
+; AVX2-NEXT:    bsrw %ax, %ax
+; AVX2-NEXT:    xorl $15, %eax
+; AVX2-NEXT:    vpinsrw $7, %eax, %xmm2, %xmm1
+; AVX2-NEXT:    vpextrw $1, %xmm0, %eax
+; AVX2-NEXT:    bsrw %ax, %ax
+; AVX2-NEXT:    xorl $15, %eax
+; AVX2-NEXT:    vmovd %xmm0, %ecx
+; AVX2-NEXT:    bsrw %cx, %cx
+; AVX2-NEXT:    xorl $15, %ecx
+; AVX2-NEXT:    vmovd %ecx, %xmm2
+; AVX2-NEXT:    vpinsrw $1, %eax, %xmm2, %xmm2
+; AVX2-NEXT:    vpextrw $2, %xmm0, %eax
+; AVX2-NEXT:    bsrw %ax, %ax
+; AVX2-NEXT:    xorl $15, %eax
+; AVX2-NEXT:    vpinsrw $2, %eax, %xmm2, %xmm2
+; AVX2-NEXT:    vpextrw $3, %xmm0, %eax
+; AVX2-NEXT:    bsrw %ax, %ax
+; AVX2-NEXT:    xorl $15, %eax
+; AVX2-NEXT:    vpinsrw $3, %eax, %xmm2, %xmm2
+; AVX2-NEXT:    vpextrw $4, %xmm0, %eax
+; AVX2-NEXT:    bsrw %ax, %ax
+; AVX2-NEXT:    xorl $15, %eax
+; AVX2-NEXT:    vpinsrw $4, %eax, %xmm2, %xmm2
+; AVX2-NEXT:    vpextrw $5, %xmm0, %eax
+; AVX2-NEXT:    bsrw %ax, %ax
+; AVX2-NEXT:    xorl $15, %eax
+; AVX2-NEXT:    vpinsrw $5, %eax, %xmm2, %xmm2
+; AVX2-NEXT:    vpextrw $6, %xmm0, %eax
+; AVX2-NEXT:    bsrw %ax, %ax
+; AVX2-NEXT:    xorl $15, %eax
+; AVX2-NEXT:    vpinsrw $6, %eax, %xmm2, %xmm2
+; AVX2-NEXT:    vpextrw $7, %xmm0, %eax
+; AVX2-NEXT:    bsrw %ax, %ax
+; AVX2-NEXT:    xorl $15, %eax
+; AVX2-NEXT:    vpinsrw $7, %eax, %xmm2, %xmm0
+; AVX2-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm0
+; AVX2-NEXT:    retq
+  %out = call <16 x i16> @llvm.ctlz.v16i16(<16 x i16> %in, i1 -1)
+  ret <16 x i16> %out
+}
+
+define <32 x i8> @testv32i8(<32 x i8> %in) nounwind {
+; AVX1-LABEL: testv32i8:
+; AVX1:       # BB#0:
+; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
+; AVX1-NEXT:    vpextrb $1, %xmm1, %eax
+; AVX1-NEXT:    bsrl %eax, %ecx
+; AVX1-NEXT:    movl $15, %eax
+; AVX1-NEXT:    cmovel %eax, %ecx
+; AVX1-NEXT:    xorl $7, %ecx
+; AVX1-NEXT:    vpextrb $0, %xmm1, %edx
+; AVX1-NEXT:    bsrl %edx, %edx
+; AVX1-NEXT:    cmovel %eax, %edx
+; AVX1-NEXT:    xorl $7, %edx
+; AVX1-NEXT:    vmovd %edx, %xmm2
+; AVX1-NEXT:    vpinsrb $1, %ecx, %xmm2, %xmm2
+; AVX1-NEXT:    vpextrb $2, %xmm1, %ecx
+; AVX1-NEXT:    bsrl %ecx, %ecx
+; AVX1-NEXT:    cmovel %eax, %ecx
+; AVX1-NEXT:    xorl $7, %ecx
+; AVX1-NEXT:    vpinsrb $2, %ecx, %xmm2, %xmm2
+; AVX1-NEXT:    vpextrb $3, %xmm1, %ecx
+; AVX1-NEXT:    bsrl %ecx, %ecx
+; AVX1-NEXT:    cmovel %eax, %ecx
+; AVX1-NEXT:    xorl $7, %ecx
+; AVX1-NEXT:    vpinsrb $3, %ecx, %xmm2, %xmm2
+; AVX1-NEXT:    vpextrb $4, %xmm1, %ecx
+; AVX1-NEXT:    bsrl %ecx, %ecx
+; AVX1-NEXT:    cmovel %eax, %ecx
+; AVX1-NEXT:    xorl $7, %ecx
+; AVX1-NEXT:    vpinsrb $4, %ecx, %xmm2, %xmm2
+; AVX1-NEXT:    vpextrb $5, %xmm1, %ecx
+; AVX1-NEXT:    bsrl %ecx, %ecx
+; AVX1-NEXT:    cmovel %eax, %ecx
+; AVX1-NEXT:    xorl $7, %ecx
+; AVX1-NEXT:    vpinsrb $5, %ecx, %xmm2, %xmm2
+; AVX1-NEXT:    vpextrb $6, %xmm1, %ecx
+; AVX1-NEXT:    bsrl %ecx, %ecx
+; AVX1-NEXT:    cmovel %eax, %ecx
+; AVX1-NEXT:    xorl $7, %ecx
+; AVX1-NEXT:    vpinsrb $6, %ecx, %xmm2, %xmm2
+; AVX1-NEXT:    vpextrb $7, %xmm1, %ecx
+; AVX1-NEXT:    bsrl %ecx, %ecx
+; AVX1-NEXT:    cmovel %eax, %ecx
+; AVX1-NEXT:    xorl $7, %ecx
+; AVX1-NEXT:    vpinsrb $7, %ecx, %xmm2, %xmm2
+; AVX1-NEXT:    vpextrb $8, %xmm1, %ecx
+; AVX1-NEXT:    bsrl %ecx, %ecx
+; AVX1-NEXT:    cmovel %eax, %ecx
+; AVX1-NEXT:    xorl $7, %ecx
+; AVX1-NEXT:    vpinsrb $8, %ecx, %xmm2, %xmm2
+; AVX1-NEXT:    vpextrb $9, %xmm1, %ecx
+; AVX1-NEXT:    bsrl %ecx, %ecx
+; AVX1-NEXT:    cmovel %eax, %ecx
+; AVX1-NEXT:    xorl $7, %ecx
+; AVX1-NEXT:    vpinsrb $9, %ecx, %xmm2, %xmm2
+; AVX1-NEXT:    vpextrb $10, %xmm1, %ecx
+; AVX1-NEXT:    bsrl %ecx, %ecx
+; AVX1-NEXT:    cmovel %eax, %ecx
+; AVX1-NEXT:    xorl $7, %ecx
+; AVX1-NEXT:    vpinsrb $10, %ecx, %xmm2, %xmm2
+; AVX1-NEXT:    vpextrb $11, %xmm1, %ecx
+; AVX1-NEXT:    bsrl %ecx, %ecx
+; AVX1-NEXT:    cmovel %eax, %ecx
+; AVX1-NEXT:    xorl $7, %ecx
+; AVX1-NEXT:    vpinsrb $11, %ecx, %xmm2, %xmm2
+; AVX1-NEXT:    vpextrb $12, %xmm1, %ecx
+; AVX1-NEXT:    bsrl %ecx, %ecx
+; AVX1-NEXT:    cmovel %eax, %ecx
+; AVX1-NEXT:    xorl $7, %ecx
+; AVX1-NEXT:    vpinsrb $12, %ecx, %xmm2, %xmm2
+; AVX1-NEXT:    vpextrb $13, %xmm1, %ecx
+; AVX1-NEXT:    bsrl %ecx, %ecx
+; AVX1-NEXT:    cmovel %eax, %ecx
+; AVX1-NEXT:    xorl $7, %ecx
+; AVX1-NEXT:    vpinsrb $13, %ecx, %xmm2, %xmm2
+; AVX1-NEXT:    vpextrb $14, %xmm1, %ecx
+; AVX1-NEXT:    bsrl %ecx, %ecx
+; AVX1-NEXT:    cmovel %eax, %ecx
+; AVX1-NEXT:    xorl $7, %ecx
+; AVX1-NEXT:    vpinsrb $14, %ecx, %xmm2, %xmm2
+; AVX1-NEXT:    vpextrb $15, %xmm1, %ecx
+; AVX1-NEXT:    bsrl %ecx, %ecx
+; AVX1-NEXT:    cmovel %eax, %ecx
+; AVX1-NEXT:    xorl $7, %ecx
+; AVX1-NEXT:    vpinsrb $15, %ecx, %xmm2, %xmm1
+; AVX1-NEXT:    vpextrb $1, %xmm0, %ecx
+; AVX1-NEXT:    bsrl %ecx, %ecx
+; AVX1-NEXT:    cmovel %eax, %ecx
+; AVX1-NEXT:    xorl $7, %ecx
+; AVX1-NEXT:    vpextrb $0, %xmm0, %edx
+; AVX1-NEXT:    bsrl %edx, %edx
+; AVX1-NEXT:    cmovel %eax, %edx
+; AVX1-NEXT:    xorl $7, %edx
+; AVX1-NEXT:    vmovd %edx, %xmm2
+; AVX1-NEXT:    vpinsrb $1, %ecx, %xmm2, %xmm2
+; AVX1-NEXT:    vpextrb $2, %xmm0, %ecx
+; AVX1-NEXT:    bsrl %ecx, %ecx
+; AVX1-NEXT:    cmovel %eax, %ecx
+; AVX1-NEXT:    xorl $7, %ecx
+; AVX1-NEXT:    vpinsrb $2, %ecx, %xmm2, %xmm2
+; AVX1-NEXT:    vpextrb $3, %xmm0, %ecx
+; AVX1-NEXT:    bsrl %ecx, %ecx
+; AVX1-NEXT:    cmovel %eax, %ecx
+; AVX1-NEXT:    xorl $7, %ecx
+; AVX1-NEXT:    vpinsrb $3, %ecx, %xmm2, %xmm2
+; AVX1-NEXT:    vpextrb $4, %xmm0, %ecx
+; AVX1-NEXT:    bsrl %ecx, %ecx
+; AVX1-NEXT:    cmovel %eax, %ecx
+; AVX1-NEXT:    xorl $7, %ecx
+; AVX1-NEXT:    vpinsrb $4, %ecx, %xmm2, %xmm2
+; AVX1-NEXT:    vpextrb $5, %xmm0, %ecx
+; AVX1-NEXT:    bsrl %ecx, %ecx
+; AVX1-NEXT:    cmovel %eax, %ecx
+; AVX1-NEXT:    xorl $7, %ecx
+; AVX1-NEXT:    vpinsrb $5, %ecx, %xmm2, %xmm2
+; AVX1-NEXT:    vpextrb $6, %xmm0, %ecx
+; AVX1-NEXT:    bsrl %ecx, %ecx
+; AVX1-NEXT:    cmovel %eax, %ecx
+; AVX1-NEXT:    xorl $7, %ecx
+; AVX1-NEXT:    vpinsrb $6, %ecx, %xmm2, %xmm2
+; AVX1-NEXT:    vpextrb $7, %xmm0, %ecx
+; AVX1-NEXT:    bsrl %ecx, %ecx
+; AVX1-NEXT:    cmovel %eax, %ecx
+; AVX1-NEXT:    xorl $7, %ecx
+; AVX1-NEXT:    vpinsrb $7, %ecx, %xmm2, %xmm2
+; AVX1-NEXT:    vpextrb $8, %xmm0, %ecx
+; AVX1-NEXT:    bsrl %ecx, %ecx
+; AVX1-NEXT:    cmovel %eax, %ecx
+; AVX1-NEXT:    xorl $7, %ecx
+; AVX1-NEXT:    vpinsrb $8, %ecx, %xmm2, %xmm2
+; AVX1-NEXT:    vpextrb $9, %xmm0, %ecx
+; AVX1-NEXT:    bsrl %ecx, %ecx
+; AVX1-NEXT:    cmovel %eax, %ecx
+; AVX1-NEXT:    xorl $7, %ecx
+; AVX1-NEXT:    vpinsrb $9, %ecx, %xmm2, %xmm2
+; AVX1-NEXT:    vpextrb $10, %xmm0, %ecx
+; AVX1-NEXT:    bsrl %ecx, %ecx
+; AVX1-NEXT:    cmovel %eax, %ecx
+; AVX1-NEXT:    xorl $7, %ecx
+; AVX1-NEXT:    vpinsrb $10, %ecx, %xmm2, %xmm2
+; AVX1-NEXT:    vpextrb $11, %xmm0, %ecx
+; AVX1-NEXT:    bsrl %ecx, %ecx
+; AVX1-NEXT:    cmovel %eax, %ecx
+; AVX1-NEXT:    xorl $7, %ecx
+; AVX1-NEXT:    vpinsrb $11, %ecx, %xmm2, %xmm2
+; AVX1-NEXT:    vpextrb $12, %xmm0, %ecx
+; AVX1-NEXT:    bsrl %ecx, %ecx
+; AVX1-NEXT:    cmovel %eax, %ecx
+; AVX1-NEXT:    xorl $7, %ecx
+; AVX1-NEXT:    vpinsrb $12, %ecx, %xmm2, %xmm2
+; AVX1-NEXT:    vpextrb $13, %xmm0, %ecx
+; AVX1-NEXT:    bsrl %ecx, %ecx
+; AVX1-NEXT:    cmovel %eax, %ecx
+; AVX1-NEXT:    xorl $7, %ecx
+; AVX1-NEXT:    vpinsrb $13, %ecx, %xmm2, %xmm2
+; AVX1-NEXT:    vpextrb $14, %xmm0, %ecx
+; AVX1-NEXT:    bsrl %ecx, %ecx
+; AVX1-NEXT:    cmovel %eax, %ecx
+; AVX1-NEXT:    xorl $7, %ecx
+; AVX1-NEXT:    vpinsrb $14, %ecx, %xmm2, %xmm2
+; AVX1-NEXT:    vpextrb $15, %xmm0, %ecx
+; AVX1-NEXT:    bsrl %ecx, %ecx
+; AVX1-NEXT:    cmovel %eax, %ecx
+; AVX1-NEXT:    xorl $7, %ecx
+; AVX1-NEXT:    vpinsrb $15, %ecx, %xmm2, %xmm0
+; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: testv32i8:
+; AVX2:       # BB#0:
+; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
+; AVX2-NEXT:    vpextrb $1, %xmm1, %eax
+; AVX2-NEXT:    bsrl %eax, %ecx
+; AVX2-NEXT:    movl $15, %eax
+; AVX2-NEXT:    cmovel %eax, %ecx
+; AVX2-NEXT:    xorl $7, %ecx
+; AVX2-NEXT:    vpextrb $0, %xmm1, %edx
+; AVX2-NEXT:    bsrl %edx, %edx
+; AVX2-NEXT:    cmovel %eax, %edx
+; AVX2-NEXT:    xorl $7, %edx
+; AVX2-NEXT:    vmovd %edx, %xmm2
+; AVX2-NEXT:    vpinsrb $1, %ecx, %xmm2, %xmm2
+; AVX2-NEXT:    vpextrb $2, %xmm1, %ecx
+; AVX2-NEXT:    bsrl %ecx, %ecx
+; AVX2-NEXT:    cmovel %eax, %ecx
+; AVX2-NEXT:    xorl $7, %ecx
+; AVX2-NEXT:    vpinsrb $2, %ecx, %xmm2, %xmm2
+; AVX2-NEXT:    vpextrb $3, %xmm1, %ecx
+; AVX2-NEXT:    bsrl %ecx, %ecx
+; AVX2-NEXT:    cmovel %eax, %ecx
+; AVX2-NEXT:    xorl $7, %ecx
+; AVX2-NEXT:    vpinsrb $3, %ecx, %xmm2, %xmm2
+; AVX2-NEXT:    vpextrb $4, %xmm1, %ecx
+; AVX2-NEXT:    bsrl %ecx, %ecx
+; AVX2-NEXT:    cmovel %eax, %ecx
+; AVX2-NEXT:    xorl $7, %ecx
+; AVX2-NEXT:    vpinsrb $4, %ecx, %xmm2, %xmm2
+; AVX2-NEXT:    vpextrb $5, %xmm1, %ecx
+; AVX2-NEXT:    bsrl %ecx, %ecx
+; AVX2-NEXT:    cmovel %eax, %ecx
+; AVX2-NEXT:    xorl $7, %ecx
+; AVX2-NEXT:    vpinsrb $5, %ecx, %xmm2, %xmm2
+; AVX2-NEXT:    vpextrb $6, %xmm1, %ecx
+; AVX2-NEXT:    bsrl %ecx, %ecx
+; AVX2-NEXT:    cmovel %eax, %ecx
+; AVX2-NEXT:    xorl $7, %ecx
+; AVX2-NEXT:    vpinsrb $6, %ecx, %xmm2, %xmm2
+; AVX2-NEXT:    vpextrb $7, %xmm1, %ecx
+; AVX2-NEXT:    bsrl %ecx, %ecx
+; AVX2-NEXT:    cmovel %eax, %ecx
+; AVX2-NEXT:    xorl $7, %ecx
+; AVX2-NEXT:    vpinsrb $7, %ecx, %xmm2, %xmm2
+; AVX2-NEXT:    vpextrb $8, %xmm1, %ecx
+; AVX2-NEXT:    bsrl %ecx, %ecx
+; AVX2-NEXT:    cmovel %eax, %ecx
+; AVX2-NEXT:    xorl $7, %ecx
+; AVX2-NEXT:    vpinsrb $8, %ecx, %xmm2, %xmm2
+; AVX2-NEXT:    vpextrb $9, %xmm1, %ecx
+; AVX2-NEXT:    bsrl %ecx, %ecx
+; AVX2-NEXT:    cmovel %eax, %ecx
+; AVX2-NEXT:    xorl $7, %ecx
+; AVX2-NEXT:    vpinsrb $9, %ecx, %xmm2, %xmm2
+; AVX2-NEXT:    vpextrb $10, %xmm1, %ecx
+; AVX2-NEXT:    bsrl %ecx, %ecx
+; AVX2-NEXT:    cmovel %eax, %ecx
+; AVX2-NEXT:    xorl $7, %ecx
+; AVX2-NEXT:    vpinsrb $10, %ecx, %xmm2, %xmm2
+; AVX2-NEXT:    vpextrb $11, %xmm1, %ecx
+; AVX2-NEXT:    bsrl %ecx, %ecx
+; AVX2-NEXT:    cmovel %eax, %ecx
+; AVX2-NEXT:    xorl $7, %ecx
+; AVX2-NEXT:    vpinsrb $11, %ecx, %xmm2, %xmm2
+; AVX2-NEXT:    vpextrb $12, %xmm1, %ecx
+; AVX2-NEXT:    bsrl %ecx, %ecx
+; AVX2-NEXT:    cmovel %eax, %ecx
+; AVX2-NEXT:    xorl $7, %ecx
+; AVX2-NEXT:    vpinsrb $12, %ecx, %xmm2, %xmm2
+; AVX2-NEXT:    vpextrb $13, %xmm1, %ecx
+; AVX2-NEXT:    bsrl %ecx, %ecx
+; AVX2-NEXT:    cmovel %eax, %ecx
+; AVX2-NEXT:    xorl $7, %ecx
+; AVX2-NEXT:    vpinsrb $13, %ecx, %xmm2, %xmm2
+; AVX2-NEXT:    vpextrb $14, %xmm1, %ecx
+; AVX2-NEXT:    bsrl %ecx, %ecx
+; AVX2-NEXT:    cmovel %eax, %ecx
+; AVX2-NEXT:    xorl $7, %ecx
+; AVX2-NEXT:    vpinsrb $14, %ecx, %xmm2, %xmm2
+; AVX2-NEXT:    vpextrb $15, %xmm1, %ecx
+; AVX2-NEXT:    bsrl %ecx, %ecx
+; AVX2-NEXT:    cmovel %eax, %ecx
+; AVX2-NEXT:    xorl $7, %ecx
+; AVX2-NEXT:    vpinsrb $15, %ecx, %xmm2, %xmm1
+; AVX2-NEXT:    vpextrb $1, %xmm0, %ecx
+; AVX2-NEXT:    bsrl %ecx, %ecx
+; AVX2-NEXT:    cmovel %eax, %ecx
+; AVX2-NEXT:    xorl $7, %ecx
+; AVX2-NEXT:    vpextrb $0, %xmm0, %edx
+; AVX2-NEXT:    bsrl %edx, %edx
+; AVX2-NEXT:    cmovel %eax, %edx
+; AVX2-NEXT:    xorl $7, %edx
+; AVX2-NEXT:    vmovd %edx, %xmm2
+; AVX2-NEXT:    vpinsrb $1, %ecx, %xmm2, %xmm2
+; AVX2-NEXT:    vpextrb $2, %xmm0, %ecx
+; AVX2-NEXT:    bsrl %ecx, %ecx
+; AVX2-NEXT:    cmovel %eax, %ecx
+; AVX2-NEXT:    xorl $7, %ecx
+; AVX2-NEXT:    vpinsrb $2, %ecx, %xmm2, %xmm2
+; AVX2-NEXT:    vpextrb $3, %xmm0, %ecx
+; AVX2-NEXT:    bsrl %ecx, %ecx
+; AVX2-NEXT:    cmovel %eax, %ecx
+; AVX2-NEXT:    xorl $7, %ecx
+; AVX2-NEXT:    vpinsrb $3, %ecx, %xmm2, %xmm2
+; AVX2-NEXT:    vpextrb $4, %xmm0, %ecx
+; AVX2-NEXT:    bsrl %ecx, %ecx
+; AVX2-NEXT:    cmovel %eax, %ecx
+; AVX2-NEXT:    xorl $7, %ecx
+; AVX2-NEXT:    vpinsrb $4, %ecx, %xmm2, %xmm2
+; AVX2-NEXT:    vpextrb $5, %xmm0, %ecx
+; AVX2-NEXT:    bsrl %ecx, %ecx
+; AVX2-NEXT:    cmovel %eax, %ecx
+; AVX2-NEXT:    xorl $7, %ecx
+; AVX2-NEXT:    vpinsrb $5, %ecx, %xmm2, %xmm2
+; AVX2-NEXT:    vpextrb $6, %xmm0, %ecx
+; AVX2-NEXT:    bsrl %ecx, %ecx
+; AVX2-NEXT:    cmovel %eax, %ecx
+; AVX2-NEXT:    xorl $7, %ecx
+; AVX2-NEXT:    vpinsrb $6, %ecx, %xmm2, %xmm2
+; AVX2-NEXT:    vpextrb $7, %xmm0, %ecx
+; AVX2-NEXT:    bsrl %ecx, %ecx
+; AVX2-NEXT:    cmovel %eax, %ecx
+; AVX2-NEXT:    xorl $7, %ecx
+; AVX2-NEXT:    vpinsrb $7, %ecx, %xmm2, %xmm2
+; AVX2-NEXT:    vpextrb $8, %xmm0, %ecx
+; AVX2-NEXT:    bsrl %ecx, %ecx
+; AVX2-NEXT:    cmovel %eax, %ecx
+; AVX2-NEXT:    xorl $7, %ecx
+; AVX2-NEXT:    vpinsrb $8, %ecx, %xmm2, %xmm2
+; AVX2-NEXT:    vpextrb $9, %xmm0, %ecx
+; AVX2-NEXT:    bsrl %ecx, %ecx
+; AVX2-NEXT:    cmovel %eax, %ecx
+; AVX2-NEXT:    xorl $7, %ecx
+; AVX2-NEXT:    vpinsrb $9, %ecx, %xmm2, %xmm2
+; AVX2-NEXT:    vpextrb $10, %xmm0, %ecx
+; AVX2-NEXT:    bsrl %ecx, %ecx
+; AVX2-NEXT:    cmovel %eax, %ecx
+; AVX2-NEXT:    xorl $7, %ecx
+; AVX2-NEXT:    vpinsrb $10, %ecx, %xmm2, %xmm2
+; AVX2-NEXT:    vpextrb $11, %xmm0, %ecx
+; AVX2-NEXT:    bsrl %ecx, %ecx
+; AVX2-NEXT:    cmovel %eax, %ecx
+; AVX2-NEXT:    xorl $7, %ecx
+; AVX2-NEXT:    vpinsrb $11, %ecx, %xmm2, %xmm2
+; AVX2-NEXT:    vpextrb $12, %xmm0, %ecx
+; AVX2-NEXT:    bsrl %ecx, %ecx
+; AVX2-NEXT:    cmovel %eax, %ecx
+; AVX2-NEXT:    xorl $7, %ecx
+; AVX2-NEXT:    vpinsrb $12, %ecx, %xmm2, %xmm2
+; AVX2-NEXT:    vpextrb $13, %xmm0, %ecx
+; AVX2-NEXT:    bsrl %ecx, %ecx
+; AVX2-NEXT:    cmovel %eax, %ecx
+; AVX2-NEXT:    xorl $7, %ecx
+; AVX2-NEXT:    vpinsrb $13, %ecx, %xmm2, %xmm2
+; AVX2-NEXT:    vpextrb $14, %xmm0, %ecx
+; AVX2-NEXT:    bsrl %ecx, %ecx
+; AVX2-NEXT:    cmovel %eax, %ecx
+; AVX2-NEXT:    xorl $7, %ecx
+; AVX2-NEXT:    vpinsrb $14, %ecx, %xmm2, %xmm2
+; AVX2-NEXT:    vpextrb $15, %xmm0, %ecx
+; AVX2-NEXT:    bsrl %ecx, %ecx
+; AVX2-NEXT:    cmovel %eax, %ecx
+; AVX2-NEXT:    xorl $7, %ecx
+; AVX2-NEXT:    vpinsrb $15, %ecx, %xmm2, %xmm0
+; AVX2-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm0
+; AVX2-NEXT:    retq
+  %out = call <32 x i8> @llvm.ctlz.v32i8(<32 x i8> %in, i1 0)
+  ret <32 x i8> %out
+}
+
+define <32 x i8> @testv32i8u(<32 x i8> %in) nounwind {
+; AVX1-LABEL: testv32i8u:
+; AVX1:       # BB#0:
+; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
+; AVX1-NEXT:    vpextrb $1, %xmm1, %eax
+; AVX1-NEXT:    bsrl %eax, %eax
+; AVX1-NEXT:    xorl $7, %eax
+; AVX1-NEXT:    vpextrb $0, %xmm1, %ecx
+; AVX1-NEXT:    bsrl %ecx, %ecx
+; AVX1-NEXT:    xorl $7, %ecx
+; AVX1-NEXT:    vmovd %ecx, %xmm2
+; AVX1-NEXT:    vpinsrb $1, %eax, %xmm2, %xmm2
+; AVX1-NEXT:    vpextrb $2, %xmm1, %eax
+; AVX1-NEXT:    bsrl %eax, %eax
+; AVX1-NEXT:    xorl $7, %eax
+; AVX1-NEXT:    vpinsrb $2, %eax, %xmm2, %xmm2
+; AVX1-NEXT:    vpextrb $3, %xmm1, %eax
+; AVX1-NEXT:    bsrl %eax, %eax
+; AVX1-NEXT:    xorl $7, %eax
+; AVX1-NEXT:    vpinsrb $3, %eax, %xmm2, %xmm2
+; AVX1-NEXT:    vpextrb $4, %xmm1, %eax
+; AVX1-NEXT:    bsrl %eax, %eax
+; AVX1-NEXT:    xorl $7, %eax
+; AVX1-NEXT:    vpinsrb $4, %eax, %xmm2, %xmm2
+; AVX1-NEXT:    vpextrb $5, %xmm1, %eax
+; AVX1-NEXT:    bsrl %eax, %eax
+; AVX1-NEXT:    xorl $7, %eax
+; AVX1-NEXT:    vpinsrb $5, %eax, %xmm2, %xmm2
+; AVX1-NEXT:    vpextrb $6, %xmm1, %eax
+; AVX1-NEXT:    bsrl %eax, %eax
+; AVX1-NEXT:    xorl $7, %eax
+; AVX1-NEXT:    vpinsrb $6, %eax, %xmm2, %xmm2
+; AVX1-NEXT:    vpextrb $7, %xmm1, %eax
+; AVX1-NEXT:    bsrl %eax, %eax
+; AVX1-NEXT:    xorl $7, %eax
+; AVX1-NEXT:    vpinsrb $7, %eax, %xmm2, %xmm2
+; AVX1-NEXT:    vpextrb $8, %xmm1, %eax
+; AVX1-NEXT:    bsrl %eax, %eax
+; AVX1-NEXT:    xorl $7, %eax
+; AVX1-NEXT:    vpinsrb $8, %eax, %xmm2, %xmm2
+; AVX1-NEXT:    vpextrb $9, %xmm1, %eax
+; AVX1-NEXT:    bsrl %eax, %eax
+; AVX1-NEXT:    xorl $7, %eax
+; AVX1-NEXT:    vpinsrb $9, %eax, %xmm2, %xmm2
+; AVX1-NEXT:    vpextrb $10, %xmm1, %eax
+; AVX1-NEXT:    bsrl %eax, %eax
+; AVX1-NEXT:    xorl $7, %eax
+; AVX1-NEXT:    vpinsrb $10, %eax, %xmm2, %xmm2
+; AVX1-NEXT:    vpextrb $11, %xmm1, %eax
+; AVX1-NEXT:    bsrl %eax, %eax
+; AVX1-NEXT:    xorl $7, %eax
+; AVX1-NEXT:    vpinsrb $11, %eax, %xmm2, %xmm2
+; AVX1-NEXT:    vpextrb $12, %xmm1, %eax
+; AVX1-NEXT:    bsrl %eax, %eax
+; AVX1-NEXT:    xorl $7, %eax
+; AVX1-NEXT:    vpinsrb $12, %eax, %xmm2, %xmm2
+; AVX1-NEXT:    vpextrb $13, %xmm1, %eax
+; AVX1-NEXT:    bsrl %eax, %eax
+; AVX1-NEXT:    xorl $7, %eax
+; AVX1-NEXT:    vpinsrb $13, %eax, %xmm2, %xmm2
+; AVX1-NEXT:    vpextrb $14, %xmm1, %eax
+; AVX1-NEXT:    bsrl %eax, %eax
+; AVX1-NEXT:    xorl $7, %eax
+; AVX1-NEXT:    vpinsrb $14, %eax, %xmm2, %xmm2
+; AVX1-NEXT:    vpextrb $15, %xmm1, %eax
+; AVX1-NEXT:    bsrl %eax, %eax
+; AVX1-NEXT:    xorl $7, %eax
+; AVX1-NEXT:    vpinsrb $15, %eax, %xmm2, %xmm1
+; AVX1-NEXT:    vpextrb $1, %xmm0, %eax
+; AVX1-NEXT:    bsrl %eax, %eax
+; AVX1-NEXT:    xorl $7, %eax
+; AVX1-NEXT:    vpextrb $0, %xmm0, %ecx
+; AVX1-NEXT:    bsrl %ecx, %ecx
+; AVX1-NEXT:    xorl $7, %ecx
+; AVX1-NEXT:    vmovd %ecx, %xmm2
+; AVX1-NEXT:    vpinsrb $1, %eax, %xmm2, %xmm2
+; AVX1-NEXT:    vpextrb $2, %xmm0, %eax
+; AVX1-NEXT:    bsrl %eax, %eax
+; AVX1-NEXT:    xorl $7, %eax
+; AVX1-NEXT:    vpinsrb $2, %eax, %xmm2, %xmm2
+; AVX1-NEXT:    vpextrb $3, %xmm0, %eax
+; AVX1-NEXT:    bsrl %eax, %eax
+; AVX1-NEXT:    xorl $7, %eax
+; AVX1-NEXT:    vpinsrb $3, %eax, %xmm2, %xmm2
+; AVX1-NEXT:    vpextrb $4, %xmm0, %eax
+; AVX1-NEXT:    bsrl %eax, %eax
+; AVX1-NEXT:    xorl $7, %eax
+; AVX1-NEXT:    vpinsrb $4, %eax, %xmm2, %xmm2
+; AVX1-NEXT:    vpextrb $5, %xmm0, %eax
+; AVX1-NEXT:    bsrl %eax, %eax
+; AVX1-NEXT:    xorl $7, %eax
+; AVX1-NEXT:    vpinsrb $5, %eax, %xmm2, %xmm2
+; AVX1-NEXT:    vpextrb $6, %xmm0, %eax
+; AVX1-NEXT:    bsrl %eax, %eax
+; AVX1-NEXT:    xorl $7, %eax
+; AVX1-NEXT:    vpinsrb $6, %eax, %xmm2, %xmm2
+; AVX1-NEXT:    vpextrb $7, %xmm0, %eax
+; AVX1-NEXT:    bsrl %eax, %eax
+; AVX1-NEXT:    xorl $7, %eax
+; AVX1-NEXT:    vpinsrb $7, %eax, %xmm2, %xmm2
+; AVX1-NEXT:    vpextrb $8, %xmm0, %eax
+; AVX1-NEXT:    bsrl %eax, %eax
+; AVX1-NEXT:    xorl $7, %eax
+; AVX1-NEXT:    vpinsrb $8, %eax, %xmm2, %xmm2
+; AVX1-NEXT:    vpextrb $9, %xmm0, %eax
+; AVX1-NEXT:    bsrl %eax, %eax
+; AVX1-NEXT:    xorl $7, %eax
+; AVX1-NEXT:    vpinsrb $9, %eax, %xmm2, %xmm2
+; AVX1-NEXT:    vpextrb $10, %xmm0, %eax
+; AVX1-NEXT:    bsrl %eax, %eax
+; AVX1-NEXT:    xorl $7, %eax
+; AVX1-NEXT:    vpinsrb $10, %eax, %xmm2, %xmm2
+; AVX1-NEXT:    vpextrb $11, %xmm0, %eax
+; AVX1-NEXT:    bsrl %eax, %eax
+; AVX1-NEXT:    xorl $7, %eax
+; AVX1-NEXT:    vpinsrb $11, %eax, %xmm2, %xmm2
+; AVX1-NEXT:    vpextrb $12, %xmm0, %eax
+; AVX1-NEXT:    bsrl %eax, %eax
+; AVX1-NEXT:    xorl $7, %eax
+; AVX1-NEXT:    vpinsrb $12, %eax, %xmm2, %xmm2
+; AVX1-NEXT:    vpextrb $13, %xmm0, %eax
+; AVX1-NEXT:    bsrl %eax, %eax
+; AVX1-NEXT:    xorl $7, %eax
+; AVX1-NEXT:    vpinsrb $13, %eax, %xmm2, %xmm2
+; AVX1-NEXT:    vpextrb $14, %xmm0, %eax
+; AVX1-NEXT:    bsrl %eax, %eax
+; AVX1-NEXT:    xorl $7, %eax
+; AVX1-NEXT:    vpinsrb $14, %eax, %xmm2, %xmm2
+; AVX1-NEXT:    vpextrb $15, %xmm0, %eax
+; AVX1-NEXT:    bsrl %eax, %eax
+; AVX1-NEXT:    xorl $7, %eax
+; AVX1-NEXT:    vpinsrb $15, %eax, %xmm2, %xmm0
+; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: testv32i8u:
+; AVX2:       # BB#0:
+; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
+; AVX2-NEXT:    vpextrb $1, %xmm1, %eax
+; AVX2-NEXT:    bsrl %eax, %eax
+; AVX2-NEXT:    xorl $7, %eax
+; AVX2-NEXT:    vpextrb $0, %xmm1, %ecx
+; AVX2-NEXT:    bsrl %ecx, %ecx
+; AVX2-NEXT:    xorl $7, %ecx
+; AVX2-NEXT:    vmovd %ecx, %xmm2
+; AVX2-NEXT:    vpinsrb $1, %eax, %xmm2, %xmm2
+; AVX2-NEXT:    vpextrb $2, %xmm1, %eax
+; AVX2-NEXT:    bsrl %eax, %eax
+; AVX2-NEXT:    xorl $7, %eax
+; AVX2-NEXT:    vpinsrb $2, %eax, %xmm2, %xmm2
+; AVX2-NEXT:    vpextrb $3, %xmm1, %eax
+; AVX2-NEXT:    bsrl %eax, %eax
+; AVX2-NEXT:    xorl $7, %eax
+; AVX2-NEXT:    vpinsrb $3, %eax, %xmm2, %xmm2
+; AVX2-NEXT:    vpextrb $4, %xmm1, %eax
+; AVX2-NEXT:    bsrl %eax, %eax
+; AVX2-NEXT:    xorl $7, %eax
+; AVX2-NEXT:    vpinsrb $4, %eax, %xmm2, %xmm2
+; AVX2-NEXT:    vpextrb $5, %xmm1, %eax
+; AVX2-NEXT:    bsrl %eax, %eax
+; AVX2-NEXT:    xorl $7, %eax
+; AVX2-NEXT:    vpinsrb $5, %eax, %xmm2, %xmm2
+; AVX2-NEXT:    vpextrb $6, %xmm1, %eax
+; AVX2-NEXT:    bsrl %eax, %eax
+; AVX2-NEXT:    xorl $7, %eax
+; AVX2-NEXT:    vpinsrb $6, %eax, %xmm2, %xmm2
+; AVX2-NEXT:    vpextrb $7, %xmm1, %eax
+; AVX2-NEXT:    bsrl %eax, %eax
+; AVX2-NEXT:    xorl $7, %eax
+; AVX2-NEXT:    vpinsrb $7, %eax, %xmm2, %xmm2
+; AVX2-NEXT:    vpextrb $8, %xmm1, %eax
+; AVX2-NEXT:    bsrl %eax, %eax
+; AVX2-NEXT:    xorl $7, %eax
+; AVX2-NEXT:    vpinsrb $8, %eax, %xmm2, %xmm2
+; AVX2-NEXT:    vpextrb $9, %xmm1, %eax
+; AVX2-NEXT:    bsrl %eax, %eax
+; AVX2-NEXT:    xorl $7, %eax
+; AVX2-NEXT:    vpinsrb $9, %eax, %xmm2, %xmm2
+; AVX2-NEXT:    vpextrb $10, %xmm1, %eax
+; AVX2-NEXT:    bsrl %eax, %eax
+; AVX2-NEXT:    xorl $7, %eax
+; AVX2-NEXT:    vpinsrb $10, %eax, %xmm2, %xmm2
+; AVX2-NEXT:    vpextrb $11, %xmm1, %eax
+; AVX2-NEXT:    bsrl %eax, %eax
+; AVX2-NEXT:    xorl $7, %eax
+; AVX2-NEXT:    vpinsrb $11, %eax, %xmm2, %xmm2
+; AVX2-NEXT:    vpextrb $12, %xmm1, %eax
+; AVX2-NEXT:    bsrl %eax, %eax
+; AVX2-NEXT:    xorl $7, %eax
+; AVX2-NEXT:    vpinsrb $12, %eax, %xmm2, %xmm2
+; AVX2-NEXT:    vpextrb $13, %xmm1, %eax
+; AVX2-NEXT:    bsrl %eax, %eax
+; AVX2-NEXT:    xorl $7, %eax
+; AVX2-NEXT:    vpinsrb $13, %eax, %xmm2, %xmm2
+; AVX2-NEXT:    vpextrb $14, %xmm1, %eax
+; AVX2-NEXT:    bsrl %eax, %eax
+; AVX2-NEXT:    xorl $7, %eax
+; AVX2-NEXT:    vpinsrb $14, %eax, %xmm2, %xmm2
+; AVX2-NEXT:    vpextrb $15, %xmm1, %eax
+; AVX2-NEXT:    bsrl %eax, %eax
+; AVX2-NEXT:    xorl $7, %eax
+; AVX2-NEXT:    vpinsrb $15, %eax, %xmm2, %xmm1
+; AVX2-NEXT:    vpextrb $1, %xmm0, %eax
+; AVX2-NEXT:    bsrl %eax, %eax
+; AVX2-NEXT:    xorl $7, %eax
+; AVX2-NEXT:    vpextrb $0, %xmm0, %ecx
+; AVX2-NEXT:    bsrl %ecx, %ecx
+; AVX2-NEXT:    xorl $7, %ecx
+; AVX2-NEXT:    vmovd %ecx, %xmm2
+; AVX2-NEXT:    vpinsrb $1, %eax, %xmm2, %xmm2
+; AVX2-NEXT:    vpextrb $2, %xmm0, %eax
+; AVX2-NEXT:    bsrl %eax, %eax
+; AVX2-NEXT:    xorl $7, %eax
+; AVX2-NEXT:    vpinsrb $2, %eax, %xmm2, %xmm2
+; AVX2-NEXT:    vpextrb $3, %xmm0, %eax
+; AVX2-NEXT:    bsrl %eax, %eax
+; AVX2-NEXT:    xorl $7, %eax
+; AVX2-NEXT:    vpinsrb $3, %eax, %xmm2, %xmm2
+; AVX2-NEXT:    vpextrb $4, %xmm0, %eax
+; AVX2-NEXT:    bsrl %eax, %eax
+; AVX2-NEXT:    xorl $7, %eax
+; AVX2-NEXT:    vpinsrb $4, %eax, %xmm2, %xmm2
+; AVX2-NEXT:    vpextrb $5, %xmm0, %eax
+; AVX2-NEXT:    bsrl %eax, %eax
+; AVX2-NEXT:    xorl $7, %eax
+; AVX2-NEXT:    vpinsrb $5, %eax, %xmm2, %xmm2
+; AVX2-NEXT:    vpextrb $6, %xmm0, %eax
+; AVX2-NEXT:    bsrl %eax, %eax
+; AVX2-NEXT:    xorl $7, %eax
+; AVX2-NEXT:    vpinsrb $6, %eax, %xmm2, %xmm2
+; AVX2-NEXT:    vpextrb $7, %xmm0, %eax
+; AVX2-NEXT:    bsrl %eax, %eax
+; AVX2-NEXT:    xorl $7, %eax
+; AVX2-NEXT:    vpinsrb $7, %eax, %xmm2, %xmm2
+; AVX2-NEXT:    vpextrb $8, %xmm0, %eax
+; AVX2-NEXT:    bsrl %eax, %eax
+; AVX2-NEXT:    xorl $7, %eax
+; AVX2-NEXT:    vpinsrb $8, %eax, %xmm2, %xmm2
+; AVX2-NEXT:    vpextrb $9, %xmm0, %eax
+; AVX2-NEXT:    bsrl %eax, %eax
+; AVX2-NEXT:    xorl $7, %eax
+; AVX2-NEXT:    vpinsrb $9, %eax, %xmm2, %xmm2
+; AVX2-NEXT:    vpextrb $10, %xmm0, %eax
+; AVX2-NEXT:    bsrl %eax, %eax
+; AVX2-NEXT:    xorl $7, %eax
+; AVX2-NEXT:    vpinsrb $10, %eax, %xmm2, %xmm2
+; AVX2-NEXT:    vpextrb $11, %xmm0, %eax
+; AVX2-NEXT:    bsrl %eax, %eax
+; AVX2-NEXT:    xorl $7, %eax
+; AVX2-NEXT:    vpinsrb $11, %eax, %xmm2, %xmm2
+; AVX2-NEXT:    vpextrb $12, %xmm0, %eax
+; AVX2-NEXT:    bsrl %eax, %eax
+; AVX2-NEXT:    xorl $7, %eax
+; AVX2-NEXT:    vpinsrb $12, %eax, %xmm2, %xmm2
+; AVX2-NEXT:    vpextrb $13, %xmm0, %eax
+; AVX2-NEXT:    bsrl %eax, %eax
+; AVX2-NEXT:    xorl $7, %eax
+; AVX2-NEXT:    vpinsrb $13, %eax, %xmm2, %xmm2
+; AVX2-NEXT:    vpextrb $14, %xmm0, %eax
+; AVX2-NEXT:    bsrl %eax, %eax
+; AVX2-NEXT:    xorl $7, %eax
+; AVX2-NEXT:    vpinsrb $14, %eax, %xmm2, %xmm2
+; AVX2-NEXT:    vpextrb $15, %xmm0, %eax
+; AVX2-NEXT:    bsrl %eax, %eax
+; AVX2-NEXT:    xorl $7, %eax
+; AVX2-NEXT:    vpinsrb $15, %eax, %xmm2, %xmm0
+; AVX2-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm0
+; AVX2-NEXT:    retq
+  %out = call <32 x i8> @llvm.ctlz.v32i8(<32 x i8> %in, i1 -1)
+  ret <32 x i8> %out
+}
+
+define <4 x i64> @foldv4i64() nounwind {
+; ALL-LABEL: foldv4i64:
+; ALL:       # BB#0:
+; ALL-NEXT:    vmovaps {{.*#+}} ymm0 = [55,0,64,56]
+; ALL-NEXT:    retq
+  %out = call <4 x i64> @llvm.ctlz.v4i64(<4 x i64> <i64 256, i64 -1, i64 0, i64 255>, i1 0)
+  ret <4 x i64> %out
+}
+
+define <4 x i64> @foldv4i64u() nounwind {
+; ALL-LABEL: foldv4i64u:
+; ALL:       # BB#0:
+; ALL-NEXT:    vmovaps {{.*#+}} ymm0 = [55,0,64,56]
+; ALL-NEXT:    retq
+  %out = call <4 x i64> @llvm.ctlz.v4i64(<4 x i64> <i64 256, i64 -1, i64 0, i64 255>, i1 -1)
+  ret <4 x i64> %out
+}
+
+define <8 x i32> @foldv8i32() nounwind {
+; ALL-LABEL: foldv8i32:
+; ALL:       # BB#0:
+; ALL-NEXT:    vmovaps {{.*#+}} ymm0 = [23,0,32,24,0,29,27,25]
+; ALL-NEXT:    retq
+  %out = call <8 x i32> @llvm.ctlz.v8i32(<8 x i32> <i32 256, i32 -1, i32 0, i32 255, i32 -65536, i32 7, i32 24, i32 88>, i1 0)
+  ret <8 x i32> %out
+}
+
+define <8 x i32> @foldv8i32u() nounwind {
+; ALL-LABEL: foldv8i32u:
+; ALL:       # BB#0:
+; ALL-NEXT:    vmovaps {{.*#+}} ymm0 = [23,0,32,24,0,29,27,25]
+; ALL-NEXT:    retq
+  %out = call <8 x i32> @llvm.ctlz.v8i32(<8 x i32> <i32 256, i32 -1, i32 0, i32 255, i32 -65536, i32 7, i32 24, i32 88>, i1 -1)
+  ret <8 x i32> %out
+}
+
+define <16 x i16> @foldv16i16() nounwind {
+; ALL-LABEL: foldv16i16:
+; ALL:       # BB#0:
+; ALL-NEXT:    vmovaps {{.*#+}} ymm0 = [7,0,16,8,16,13,11,9,0,8,15,14,13,12,11,10]
+; ALL-NEXT:    retq
+  %out = call <16 x i16> @llvm.ctlz.v16i16(<16 x i16> <i16 256, i16 -1, i16 0, i16 255, i16 -65536, i16 7, i16 24, i16 88, i16 -2, i16 254, i16 1, i16 2, i16 4, i16 8, i16 16, i16 32>, i1 0)
+  ret <16 x i16> %out
+}
+
+define <16 x i16> @foldv16i16u() nounwind {
+; ALL-LABEL: foldv16i16u:
+; ALL:       # BB#0:
+; ALL-NEXT:    vmovaps {{.*#+}} ymm0 = [7,0,16,8,16,13,11,9,0,8,15,14,13,12,11,10]
+; ALL-NEXT:    retq
+  %out = call <16 x i16> @llvm.ctlz.v16i16(<16 x i16> <i16 256, i16 -1, i16 0, i16 255, i16 -65536, i16 7, i16 24, i16 88, i16 -2, i16 254, i16 1, i16 2, i16 4, i16 8, i16 16, i16 32>, i1 -1)
+  ret <16 x i16> %out
+}
+
+define <32 x i8> @foldv32i8() nounwind {
+; ALL-LABEL: foldv32i8:
+; ALL:       # BB#0:
+; ALL-NEXT:    vmovaps {{.*#+}} ymm0 = [8,0,8,0,8,5,3,1,0,0,7,6,5,4,3,2,1,0,8,8,0,0,0,0,0,0,0,0,6,5,5,1]
+; ALL-NEXT:    retq
+  %out = call <32 x i8> @llvm.ctlz.v32i8(<32 x i8> <i8 256, i8 -1, i8 0, i8 255, i8 -65536, i8 7, i8 24, i8 88, i8 -2, i8 254, i8 1, i8 2, i8 4, i8 8, i8 16, i8 32, i8 64, i8 128, i8 256, i8 -256, i8 -128, i8 -64, i8 -32, i8 -16, i8 -8, i8 -4, i8 -2, i8 -1, i8 3, i8 5, i8 7, i8 127>, i1 0)
+  ret <32 x i8> %out
+}
+
+define <32 x i8> @foldv32i8u() nounwind {
+; ALL-LABEL: foldv32i8u:
+; ALL:       # BB#0:
+; ALL-NEXT:    vmovaps {{.*#+}} ymm0 = [8,0,8,0,8,5,3,1,0,0,7,6,5,4,3,2,1,0,8,8,0,0,0,0,0,0,0,0,6,5,5,1]
+; ALL-NEXT:    retq
+  %out = call <32 x i8> @llvm.ctlz.v32i8(<32 x i8> <i8 256, i8 -1, i8 0, i8 255, i8 -65536, i8 7, i8 24, i8 88, i8 -2, i8 254, i8 1, i8 2, i8 4, i8 8, i8 16, i8 32, i8 64, i8 128, i8 256, i8 -256, i8 -128, i8 -64, i8 -32, i8 -16, i8 -8, i8 -4, i8 -2, i8 -1, i8 3, i8 5, i8 7, i8 127>, i1 -1)
+  ret <32 x i8> %out
+}
+
+declare <4 x i64> @llvm.ctlz.v4i64(<4 x i64>, i1)
+declare <8 x i32> @llvm.ctlz.v8i32(<8 x i32>, i1)
+declare <16 x i16> @llvm.ctlz.v16i16(<16 x i16>, i1)
+declare <32 x i8> @llvm.ctlz.v32i8(<32 x i8>, i1)
diff --git a/test/CodeGen/X86/vector-popcnt-128.ll b/test/CodeGen/X86/vector-popcnt-128.ll
new file mode 100644
index 00000000000..fef445de04a
--- /dev/null
+++ b/test/CodeGen/X86/vector-popcnt-128.ll
@@ -0,0 +1,462 @@
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mcpu=x86-64 | FileCheck %s --check-prefix=ALL --check-prefix=SSE --check-prefix=SSE2
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mcpu=x86-64 -mattr=+sse3 | FileCheck %s --check-prefix=ALL --check-prefix=SSE --check-prefix=SSE3
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mcpu=x86-64 -mattr=+ssse3 | FileCheck %s --check-prefix=ALL --check-prefix=SSE --check-prefix=SSSE3
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mcpu=x86-64 -mattr=+sse4.1 | FileCheck %s --check-prefix=ALL --check-prefix=SSE --check-prefix=SSE41
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mcpu=x86-64 -mattr=+avx | FileCheck %s --check-prefix=ALL --check-prefix=AVX --check-prefix=AVX1
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mcpu=x86-64 -mattr=+avx2 | FileCheck %s --check-prefix=ALL --check-prefix=AVX --check-prefix=AVX2
+
+target triple = "x86_64-unknown-unknown"
+
+define <2 x i64> @testv2i64(<2 x i64> %in) {
+; SSE2-LABEL: testv2i64:
+; SSE2:       # BB#0:
+; SSE2-NEXT:    movdqa %xmm0, %xmm1
+; SSE2-NEXT:    psrlq $1, %xmm1
+; SSE2-NEXT:    pand {{.*}}(%rip), %xmm1
+; SSE2-NEXT:    psubq %xmm1, %xmm0
+; SSE2-NEXT:    movdqa {{.*#+}} xmm1 = [3689348814741910323,3689348814741910323]
+; SSE2-NEXT:    movdqa %xmm0, %xmm2
+; SSE2-NEXT:    pand %xmm1, %xmm2
+; SSE2-NEXT:    psrlq $2, %xmm0
+; SSE2-NEXT:    pand %xmm1, %xmm0
+; SSE2-NEXT:    paddq %xmm2, %xmm0
+; SSE2-NEXT:    movdqa %xmm0, %xmm1
+; SSE2-NEXT:    psrlq $4, %xmm1
+; SSE2-NEXT:    paddq %xmm0, %xmm1
+; SSE2-NEXT:    pand {{.*}}(%rip), %xmm1
+; SSE2-NEXT:    pxor %xmm0, %xmm0
+; SSE2-NEXT:    psadbw %xmm0, %xmm1
+; SSE2-NEXT:    movdqa %xmm1, %xmm0
+; SSE2-NEXT:    retq
+;
+; SSE3-LABEL: testv2i64:
+; SSE3:       # BB#0:
+; SSE3-NEXT:    movdqa %xmm0, %xmm1
+; SSE3-NEXT:    psrlq $1, %xmm1
+; SSE3-NEXT:    pand {{.*}}(%rip), %xmm1
+; SSE3-NEXT:    psubq %xmm1, %xmm0
+; SSE3-NEXT:    movdqa {{.*#+}} xmm1 = [3689348814741910323,3689348814741910323]
+; SSE3-NEXT:    movdqa %xmm0, %xmm2
+; SSE3-NEXT:    pand %xmm1, %xmm2
+; SSE3-NEXT:    psrlq $2, %xmm0
+; SSE3-NEXT:    pand %xmm1, %xmm0
+; SSE3-NEXT:    paddq %xmm2, %xmm0
+; SSE3-NEXT:    movdqa %xmm0, %xmm1
+; SSE3-NEXT:    psrlq $4, %xmm1
+; SSE3-NEXT:    paddq %xmm0, %xmm1
+; SSE3-NEXT:    pand {{.*}}(%rip), %xmm1
+; SSE3-NEXT:    pxor %xmm0, %xmm0
+; SSE3-NEXT:    psadbw %xmm0, %xmm1
+; SSE3-NEXT:    movdqa %xmm1, %xmm0
+; SSE3-NEXT:    retq
+;
+; SSSE3-LABEL: testv2i64:
+; SSSE3:       # BB#0:
+; SSSE3-NEXT:    movdqa {{.*#+}} xmm1 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
+; SSSE3-NEXT:    movdqa %xmm0, %xmm2
+; SSSE3-NEXT:    pand %xmm1, %xmm2
+; SSSE3-NEXT:    movdqa {{.*#+}} xmm3 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4]
+; SSSE3-NEXT:    movdqa %xmm3, %xmm4
+; SSSE3-NEXT:    pshufb %xmm2, %xmm4
+; SSSE3-NEXT:    psrlw $4, %xmm0
+; SSSE3-NEXT:    pand %xmm1, %xmm0
+; SSSE3-NEXT:    pshufb %xmm0, %xmm3
+; SSSE3-NEXT:    paddb %xmm4, %xmm3
+; SSSE3-NEXT:    pxor %xmm0, %xmm0
+; SSSE3-NEXT:    psadbw %xmm3, %xmm0
+; SSSE3-NEXT:    retq
+;
+; SSE41-LABEL: testv2i64:
+; SSE41:       # BB#0:
+; SSE41-NEXT:    movdqa {{.*#+}} xmm1 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
+; SSE41-NEXT:    movdqa %xmm0, %xmm2
+; SSE41-NEXT:    pand %xmm1, %xmm2
+; SSE41-NEXT:    movdqa {{.*#+}} xmm3 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4]
+; SSE41-NEXT:    movdqa %xmm3, %xmm4
+; SSE41-NEXT:    pshufb %xmm2, %xmm4
+; SSE41-NEXT:    psrlw $4, %xmm0
+; SSE41-NEXT:    pand %xmm1, %xmm0
+; SSE41-NEXT:    pshufb %xmm0, %xmm3
+; SSE41-NEXT:    paddb %xmm4, %xmm3
+; SSE41-NEXT:    pxor %xmm0, %xmm0
+; SSE41-NEXT:    psadbw %xmm3, %xmm0
+; SSE41-NEXT:    retq
+;
+; AVX-LABEL: testv2i64:
+; AVX:       # BB#0:
+; AVX-NEXT:    vmovdqa {{.*#+}} xmm1 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
+; AVX-NEXT:    vpand %xmm1, %xmm0, %xmm2
+; AVX-NEXT:    vmovdqa {{.*#+}} xmm3 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4]
+; AVX-NEXT:    vpshufb %xmm2, %xmm3, %xmm2
+; AVX-NEXT:    vpsrlw $4, %xmm0, %xmm0
+; AVX-NEXT:    vpand %xmm1, %xmm0, %xmm0
+; AVX-NEXT:    vpshufb %xmm0, %xmm3, %xmm0
+; AVX-NEXT:    vpaddb %xmm2, %xmm0, %xmm0
+; AVX-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; AVX-NEXT:    vpsadbw %xmm0, %xmm1, %xmm0
+; AVX-NEXT:    retq
+  %out = call <2 x i64> @llvm.ctpop.v2i64(<2 x i64> %in)
+  ret <2 x i64> %out
+}
+
+define <4 x i32> @testv4i32(<4 x i32> %in) {
+; SSE2-LABEL: testv4i32:
+; SSE2:       # BB#0:
+; SSE2-NEXT:    movdqa %xmm0, %xmm1
+; SSE2-NEXT:    psrld $1, %xmm1
+; SSE2-NEXT:    pand {{.*}}(%rip), %xmm1
+; SSE2-NEXT:    psubd %xmm1, %xmm0
+; SSE2-NEXT:    movdqa {{.*#+}} xmm1 = [858993459,858993459,858993459,858993459]
+; SSE2-NEXT:    movdqa %xmm0, %xmm2
+; SSE2-NEXT:    pand %xmm1, %xmm2
+; SSE2-NEXT:    psrld $2, %xmm0
+; SSE2-NEXT:    pand %xmm1, %xmm0
+; SSE2-NEXT:    paddd %xmm2, %xmm0
+; SSE2-NEXT:    movdqa %xmm0, %xmm1
+; SSE2-NEXT:    psrld $4, %xmm1
+; SSE2-NEXT:    paddd %xmm0, %xmm1
+; SSE2-NEXT:    pand {{.*}}(%rip), %xmm1
+; SSE2-NEXT:    pxor %xmm0, %xmm0
+; SSE2-NEXT:    movdqa %xmm1, %xmm2
+; SSE2-NEXT:    punpckhdq {{.*#+}} xmm2 = xmm2[2],xmm0[2],xmm2[3],xmm0[3]
+; SSE2-NEXT:    psadbw %xmm0, %xmm2
+; SSE2-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
+; SSE2-NEXT:    psadbw %xmm0, %xmm1
+; SSE2-NEXT:    packuswb %xmm2, %xmm1
+; SSE2-NEXT:    movdqa %xmm1, %xmm0
+; SSE2-NEXT:    retq
+;
+; SSE3-LABEL: testv4i32:
+; SSE3:       # BB#0:
+; SSE3-NEXT:    movdqa %xmm0, %xmm1
+; SSE3-NEXT:    psrld $1, %xmm1
+; SSE3-NEXT:    pand {{.*}}(%rip), %xmm1
+; SSE3-NEXT:    psubd %xmm1, %xmm0
+; SSE3-NEXT:    movdqa {{.*#+}} xmm1 = [858993459,858993459,858993459,858993459]
+; SSE3-NEXT:    movdqa %xmm0, %xmm2
+; SSE3-NEXT:    pand %xmm1, %xmm2
+; SSE3-NEXT:    psrld $2, %xmm0
+; SSE3-NEXT:    pand %xmm1, %xmm0
+; SSE3-NEXT:    paddd %xmm2, %xmm0
+; SSE3-NEXT:    movdqa %xmm0, %xmm1
+; SSE3-NEXT:    psrld $4, %xmm1
+; SSE3-NEXT:    paddd %xmm0, %xmm1
+; SSE3-NEXT:    pand {{.*}}(%rip), %xmm1
+; SSE3-NEXT:    pxor %xmm0, %xmm0
+; SSE3-NEXT:    movdqa %xmm1, %xmm2
+; SSE3-NEXT:    punpckhdq {{.*#+}} xmm2 = xmm2[2],xmm0[2],xmm2[3],xmm0[3]
+; SSE3-NEXT:    psadbw %xmm0, %xmm2
+; SSE3-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
+; SSE3-NEXT:    psadbw %xmm0, %xmm1
+; SSE3-NEXT:    packuswb %xmm2, %xmm1
+; SSE3-NEXT:    movdqa %xmm1, %xmm0
+; SSE3-NEXT:    retq
+;
+; SSSE3-LABEL: testv4i32:
+; SSSE3:       # BB#0:
+; SSSE3-NEXT:    movdqa {{.*#+}} xmm2 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
+; SSSE3-NEXT:    movdqa %xmm0, %xmm3
+; SSSE3-NEXT:    pand %xmm2, %xmm3
+; SSSE3-NEXT:    movdqa {{.*#+}} xmm1 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4]
+; SSSE3-NEXT:    movdqa %xmm1, %xmm4
+; SSSE3-NEXT:    pshufb %xmm3, %xmm4
+; SSSE3-NEXT:    psrlw $4, %xmm0
+; SSSE3-NEXT:    pand %xmm2, %xmm0
+; SSSE3-NEXT:    pshufb %xmm0, %xmm1
+; SSSE3-NEXT:    paddb %xmm4, %xmm1
+; SSSE3-NEXT:    pxor %xmm0, %xmm0
+; SSSE3-NEXT:    movdqa %xmm1, %xmm2
+; SSSE3-NEXT:    punpckhdq {{.*#+}} xmm2 = xmm2[2],xmm0[2],xmm2[3],xmm0[3]
+; SSSE3-NEXT:    psadbw %xmm0, %xmm2
+; SSSE3-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
+; SSSE3-NEXT:    psadbw %xmm0, %xmm1
+; SSSE3-NEXT:    packuswb %xmm2, %xmm1
+; SSSE3-NEXT:    movdqa %xmm1, %xmm0
+; SSSE3-NEXT:    retq
+;
+; SSE41-LABEL: testv4i32:
+; SSE41:       # BB#0:
+; SSE41-NEXT:    movdqa {{.*#+}} xmm2 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
+; SSE41-NEXT:    movdqa %xmm0, %xmm3
+; SSE41-NEXT:    pand %xmm2, %xmm3
+; SSE41-NEXT:    movdqa {{.*#+}} xmm1 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4]
+; SSE41-NEXT:    movdqa %xmm1, %xmm4
+; SSE41-NEXT:    pshufb %xmm3, %xmm4
+; SSE41-NEXT:    psrlw $4, %xmm0
+; SSE41-NEXT:    pand %xmm2, %xmm0
+; SSE41-NEXT:    pshufb %xmm0, %xmm1
+; SSE41-NEXT:    paddb %xmm4, %xmm1
+; SSE41-NEXT:    pxor %xmm0, %xmm0
+; SSE41-NEXT:    movdqa %xmm1, %xmm2
+; SSE41-NEXT:    punpckhdq {{.*#+}} xmm2 = xmm2[2],xmm0[2],xmm2[3],xmm0[3]
+; SSE41-NEXT:    psadbw %xmm0, %xmm2
+; SSE41-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
+; SSE41-NEXT:    psadbw %xmm0, %xmm1
+; SSE41-NEXT:    packuswb %xmm2, %xmm1
+; SSE41-NEXT:    movdqa %xmm1, %xmm0
+; SSE41-NEXT:    retq
+;
+; AVX-LABEL: testv4i32:
+; AVX:       # BB#0:
+; AVX-NEXT:    vmovdqa {{.*#+}} xmm1 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
+; AVX-NEXT:    vpand %xmm1, %xmm0, %xmm2
+; AVX-NEXT:    vmovdqa {{.*#+}} xmm3 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4]
+; AVX-NEXT:    vpshufb %xmm2, %xmm3, %xmm2
+; AVX-NEXT:    vpsrlw $4, %xmm0, %xmm0
+; AVX-NEXT:    vpand %xmm1, %xmm0, %xmm0
+; AVX-NEXT:    vpshufb %xmm0, %xmm3, %xmm0
+; AVX-NEXT:    vpaddb %xmm2, %xmm0, %xmm0
+; AVX-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; AVX-NEXT:    vpunpckhdq {{.*#+}} xmm2 = xmm0[2],xmm1[2],xmm0[3],xmm1[3]
+; AVX-NEXT:    vpsadbw %xmm2, %xmm1, %xmm2
+; AVX-NEXT:    vpunpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
+; AVX-NEXT:    vpsadbw %xmm0, %xmm1, %xmm0
+; AVX-NEXT:    vpackuswb %xmm2, %xmm0, %xmm0
+; AVX-NEXT:    retq
+  %out = call <4 x i32> @llvm.ctpop.v4i32(<4 x i32> %in)
+  ret <4 x i32> %out
+}
+
+define <8 x i16> @testv8i16(<8 x i16> %in) {
+; SSE2-LABEL: testv8i16:
+; SSE2:       # BB#0:
+; SSE2-NEXT:    movdqa %xmm0, %xmm1
+; SSE2-NEXT:    psrlw $1, %xmm1
+; SSE2-NEXT:    pand {{.*}}(%rip), %xmm1
+; SSE2-NEXT:    psubw %xmm1, %xmm0
+; SSE2-NEXT:    movdqa {{.*#+}} xmm1 = [13107,13107,13107,13107,13107,13107,13107,13107]
+; SSE2-NEXT:    movdqa %xmm0, %xmm2
+; SSE2-NEXT:    pand %xmm1, %xmm2
+; SSE2-NEXT:    psrlw $2, %xmm0
+; SSE2-NEXT:    pand %xmm1, %xmm0
+; SSE2-NEXT:    paddw %xmm2, %xmm0
+; SSE2-NEXT:    movdqa %xmm0, %xmm1
+; SSE2-NEXT:    psrlw $4, %xmm1
+; SSE2-NEXT:    paddw %xmm0, %xmm1
+; SSE2-NEXT:    pand {{.*}}(%rip), %xmm1
+; SSE2-NEXT:    movdqa %xmm1, %xmm0
+; SSE2-NEXT:    psllw $8, %xmm0
+; SSE2-NEXT:    paddb %xmm1, %xmm0
+; SSE2-NEXT:    psrlw $8, %xmm0
+; SSE2-NEXT:    retq
+;
+; SSE3-LABEL: testv8i16:
+; SSE3:       # BB#0:
+; SSE3-NEXT:    movdqa %xmm0, %xmm1
+; SSE3-NEXT:    psrlw $1, %xmm1
+; SSE3-NEXT:    pand {{.*}}(%rip), %xmm1
+; SSE3-NEXT:    psubw %xmm1, %xmm0
+; SSE3-NEXT:    movdqa {{.*#+}} xmm1 = [13107,13107,13107,13107,13107,13107,13107,13107]
+; SSE3-NEXT:    movdqa %xmm0, %xmm2
+; SSE3-NEXT:    pand %xmm1, %xmm2
+; SSE3-NEXT:    psrlw $2, %xmm0
+; SSE3-NEXT:    pand %xmm1, %xmm0
+; SSE3-NEXT:    paddw %xmm2, %xmm0
+; SSE3-NEXT:    movdqa %xmm0, %xmm1
+; SSE3-NEXT:    psrlw $4, %xmm1
+; SSE3-NEXT:    paddw %xmm0, %xmm1
+; SSE3-NEXT:    pand {{.*}}(%rip), %xmm1
+; SSE3-NEXT:    movdqa %xmm1, %xmm0
+; SSE3-NEXT:    psllw $8, %xmm0
+; SSE3-NEXT:    paddb %xmm1, %xmm0
+; SSE3-NEXT:    psrlw $8, %xmm0
+; SSE3-NEXT:    retq
+;
+; SSSE3-LABEL: testv8i16:
+; SSSE3:       # BB#0:
+; SSSE3-NEXT:    movdqa {{.*#+}} xmm1 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
+; SSSE3-NEXT:    movdqa %xmm0, %xmm2
+; SSSE3-NEXT:    pand %xmm1, %xmm2
+; SSSE3-NEXT:    movdqa {{.*#+}} xmm3 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4]
+; SSSE3-NEXT:    movdqa %xmm3, %xmm4
+; SSSE3-NEXT:    pshufb %xmm2, %xmm4
+; SSSE3-NEXT:    psrlw $4, %xmm0
+; SSSE3-NEXT:    pand %xmm1, %xmm0
+; SSSE3-NEXT:    pshufb %xmm0, %xmm3
+; SSSE3-NEXT:    paddb %xmm4, %xmm3
+; SSSE3-NEXT:    movdqa %xmm3, %xmm0
+; SSSE3-NEXT:    psllw $8, %xmm0
+; SSSE3-NEXT:    paddb %xmm3, %xmm0
+; SSSE3-NEXT:    psrlw $8, %xmm0
+; SSSE3-NEXT:    retq
+;
+; SSE41-LABEL: testv8i16:
+; SSE41:       # BB#0:
+; SSE41-NEXT:    movdqa {{.*#+}} xmm1 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
+; SSE41-NEXT:    movdqa %xmm0, %xmm2
+; SSE41-NEXT:    pand %xmm1, %xmm2
+; SSE41-NEXT:    movdqa {{.*#+}} xmm3 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4]
+; SSE41-NEXT:    movdqa %xmm3, %xmm4
+; SSE41-NEXT:    pshufb %xmm2, %xmm4
+; SSE41-NEXT:    psrlw $4, %xmm0
+; SSE41-NEXT:    pand %xmm1, %xmm0
+; SSE41-NEXT:    pshufb %xmm0, %xmm3
+; SSE41-NEXT:    paddb %xmm4, %xmm3
+; SSE41-NEXT:    movdqa %xmm3, %xmm0
+; SSE41-NEXT:    psllw $8, %xmm0
+; SSE41-NEXT:    paddb %xmm3, %xmm0
+; SSE41-NEXT:    psrlw $8, %xmm0
+; SSE41-NEXT:    retq
+;
+; AVX-LABEL: testv8i16:
+; AVX:       # BB#0:
+; AVX-NEXT:    vmovdqa {{.*#+}} xmm1 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
+; AVX-NEXT:    vpand %xmm1, %xmm0, %xmm2
+; AVX-NEXT:    vmovdqa {{.*#+}} xmm3 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4]
+; AVX-NEXT:    vpshufb %xmm2, %xmm3, %xmm2
+; AVX-NEXT:    vpsrlw $4, %xmm0, %xmm0
+; AVX-NEXT:    vpand %xmm1, %xmm0, %xmm0
+; AVX-NEXT:    vpshufb %xmm0, %xmm3, %xmm0
+; AVX-NEXT:    vpaddb %xmm2, %xmm0, %xmm0
+; AVX-NEXT:    vpsllw $8, %xmm0, %xmm1
+; AVX-NEXT:    vpaddb %xmm0, %xmm1, %xmm0
+; AVX-NEXT:    vpsrlw $8, %xmm0, %xmm0
+; AVX-NEXT:    retq
+  %out = call <8 x i16> @llvm.ctpop.v8i16(<8 x i16> %in)
+  ret <8 x i16> %out
+}
+
+define <16 x i8> @testv16i8(<16 x i8> %in) {
+; SSE2-LABEL: testv16i8:
+; SSE2:       # BB#0:
+; SSE2-NEXT:    movdqa %xmm0, %xmm1
+; SSE2-NEXT:    psrlw $1, %xmm1
+; SSE2-NEXT:    pand {{.*}}(%rip), %xmm1
+; SSE2-NEXT:    psubb %xmm1, %xmm0
+; SSE2-NEXT:    movdqa {{.*#+}} xmm1 = [51,51,51,51,51,51,51,51,51,51,51,51,51,51,51,51]
+; SSE2-NEXT:    movdqa %xmm0, %xmm2
+; SSE2-NEXT:    pand %xmm1, %xmm2
+; SSE2-NEXT:    psrlw $2, %xmm0
+; SSE2-NEXT:    pand %xmm1, %xmm0
+; SSE2-NEXT:    paddb %xmm2, %xmm0
+; SSE2-NEXT:    movdqa %xmm0, %xmm1
+; SSE2-NEXT:    psrlw $4, %xmm1
+; SSE2-NEXT:    paddb %xmm0, %xmm1
+; SSE2-NEXT:    pand {{.*}}(%rip), %xmm1
+; SSE2-NEXT:    movdqa %xmm1, %xmm0
+; SSE2-NEXT:    retq
+;
+; SSE3-LABEL: testv16i8:
+; SSE3:       # BB#0:
+; SSE3-NEXT:    movdqa %xmm0, %xmm1
+; SSE3-NEXT:    psrlw $1, %xmm1
+; SSE3-NEXT:    pand {{.*}}(%rip), %xmm1
+; SSE3-NEXT:    psubb %xmm1, %xmm0
+; SSE3-NEXT:    movdqa {{.*#+}} xmm1 = [51,51,51,51,51,51,51,51,51,51,51,51,51,51,51,51]
+; SSE3-NEXT:    movdqa %xmm0, %xmm2
+; SSE3-NEXT:    pand %xmm1, %xmm2
+; SSE3-NEXT:    psrlw $2, %xmm0
+; SSE3-NEXT:    pand %xmm1, %xmm0
+; SSE3-NEXT:    paddb %xmm2, %xmm0
+; SSE3-NEXT:    movdqa %xmm0, %xmm1
+; SSE3-NEXT:    psrlw $4, %xmm1
+; SSE3-NEXT:    paddb %xmm0, %xmm1
+; SSE3-NEXT:    pand {{.*}}(%rip), %xmm1
+; SSE3-NEXT:    movdqa %xmm1, %xmm0
+; SSE3-NEXT:    retq
+;
+; SSSE3-LABEL: testv16i8:
+; SSSE3:       # BB#0:
+; SSSE3-NEXT:    movdqa {{.*#+}} xmm2 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
+; SSSE3-NEXT:    movdqa %xmm0, %xmm3
+; SSSE3-NEXT:    pand %xmm2, %xmm3
+; SSSE3-NEXT:    movdqa {{.*#+}} xmm1 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4]
+; SSSE3-NEXT:    movdqa %xmm1, %xmm4
+; SSSE3-NEXT:    pshufb %xmm3, %xmm4
+; SSSE3-NEXT:    psrlw $4, %xmm0
+; SSSE3-NEXT:    pand %xmm2, %xmm0
+; SSSE3-NEXT:    pshufb %xmm0, %xmm1
+; SSSE3-NEXT:    paddb %xmm4, %xmm1
+; SSSE3-NEXT:    movdqa %xmm1, %xmm0
+; SSSE3-NEXT:    retq
+;
+; SSE41-LABEL: testv16i8:
+; SSE41:       # BB#0:
+; SSE41-NEXT:    movdqa {{.*#+}} xmm2 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
+; SSE41-NEXT:    movdqa %xmm0, %xmm3
+; SSE41-NEXT:    pand %xmm2, %xmm3
+; SSE41-NEXT:    movdqa {{.*#+}} xmm1 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4]
+; SSE41-NEXT:    movdqa %xmm1, %xmm4
+; SSE41-NEXT:    pshufb %xmm3, %xmm4
+; SSE41-NEXT:    psrlw $4, %xmm0
+; SSE41-NEXT:    pand %xmm2, %xmm0
+; SSE41-NEXT:    pshufb %xmm0, %xmm1
+; SSE41-NEXT:    paddb %xmm4, %xmm1
+; SSE41-NEXT:    movdqa %xmm1, %xmm0
+; SSE41-NEXT:    retq
+;
+; AVX-LABEL: testv16i8:
+; AVX:       # BB#0:
+; AVX-NEXT:    vmovdqa {{.*#+}} xmm1 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
+; AVX-NEXT:    vpand %xmm1, %xmm0, %xmm2
+; AVX-NEXT:    vmovdqa {{.*#+}} xmm3 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4]
+; AVX-NEXT:    vpshufb %xmm2, %xmm3, %xmm2
+; AVX-NEXT:    vpsrlw $4, %xmm0, %xmm0
+; AVX-NEXT:    vpand %xmm1, %xmm0, %xmm0
+; AVX-NEXT:    vpshufb %xmm0, %xmm3, %xmm0
+; AVX-NEXT:    vpaddb %xmm2, %xmm0, %xmm0
+; AVX-NEXT:    retq
+  %out = call <16 x i8> @llvm.ctpop.v16i8(<16 x i8> %in)
+  ret <16 x i8> %out
+}
+
+define <2 x i64> @foldv2i64() {
+; SSE-LABEL: foldv2i64:
+; SSE:       # BB#0:
+; SSE-NEXT:    movaps {{.*#+}} xmm0 = [1,64]
+; SSE-NEXT:    retq
+;
+; AVX-LABEL: foldv2i64:
+; AVX:       # BB#0:
+; AVX-NEXT:    vmovaps {{.*#+}} xmm0 = [1,64]
+; AVX-NEXT:    retq
+  %out = call <2 x i64> @llvm.ctpop.v2i64(<2 x i64> <i64 256, i64 -1>)
+  ret <2 x i64> %out
+}
+
+define <4 x i32> @foldv4i32() {
+; SSE-LABEL: foldv4i32:
+; SSE:       # BB#0:
+; SSE-NEXT:    movaps {{.*#+}} xmm0 = [1,32,0,8]
+; SSE-NEXT:    retq
+;
+; AVX-LABEL: foldv4i32:
+; AVX:       # BB#0:
+; AVX-NEXT:    vmovaps {{.*#+}} xmm0 = [1,32,0,8]
+; AVX-NEXT:    retq
+  %out = call <4 x i32> @llvm.ctpop.v4i32(<4 x i32> <i32 256, i32 -1, i32 0, i32 255>)
+  ret <4 x i32> %out
+}
+
+define <8 x i16> @foldv8i16() {
+; SSE-LABEL: foldv8i16:
+; SSE:       # BB#0:
+; SSE-NEXT:    movaps {{.*#+}} xmm0 = [1,16,0,8,0,3,2,3]
+; SSE-NEXT:    retq
+;
+; AVX-LABEL: foldv8i16:
+; AVX:       # BB#0:
+; AVX-NEXT:    vmovaps {{.*#+}} xmm0 = [1,16,0,8,0,3,2,3]
+; AVX-NEXT:    retq
+  %out = call <8 x i16> @llvm.ctpop.v8i16(<8 x i16> <i16 256, i16 -1, i16 0, i16 255, i16 -65536, i16 7, i16 24, i16 88>)
+  ret <8 x i16> %out
+}
+
+define <16 x i8> @foldv16i8() {
+; SSE-LABEL: foldv16i8:
+; SSE:       # BB#0:
+; SSE-NEXT:    movaps {{.*#+}} xmm0 = [0,8,0,8,0,3,2,3,7,7,1,1,1,1,1,1]
+; SSE-NEXT:    retq
+;
+; AVX-LABEL: foldv16i8:
+; AVX:       # BB#0:
+; AVX-NEXT:    vmovaps {{.*#+}} xmm0 = [0,8,0,8,0,3,2,3,7,7,1,1,1,1,1,1]
+; AVX-NEXT:    retq
+  %out = call <16 x i8> @llvm.ctpop.v16i8(<16 x i8> <i8 256, i8 -1, i8 0, i8 255, i8 -65536, i8 7, i8 24, i8 88, i8 -2, i8 254, i8 1, i8 2, i8 4, i8 8, i8 16, i8 32>)
+  ret <16 x i8> %out
+}
+
+declare <2 x i64> @llvm.ctpop.v2i64(<2 x i64>)
+declare <4 x i32> @llvm.ctpop.v4i32(<4 x i32>)
+declare <8 x i16> @llvm.ctpop.v8i16(<8 x i16>)
+declare <16 x i8> @llvm.ctpop.v16i8(<16 x i8>)
diff --git a/test/CodeGen/X86/vector-popcnt-256.ll b/test/CodeGen/X86/vector-popcnt-256.ll
new file mode 100644
index 00000000000..7ce4f712483
--- /dev/null
+++ b/test/CodeGen/X86/vector-popcnt-256.ll
@@ -0,0 +1,220 @@
+; RUN: llc < %s -mcpu=x86-64 -mattr=+avx | FileCheck %s --check-prefix=ALL --check-prefix=AVX --check-prefix=AVX1
+; RUN: llc < %s -mcpu=x86-64 -mattr=+avx2 | FileCheck %s --check-prefix=ALL --check-prefix=AVX --check-prefix=AVX2
+
+target triple = "x86_64-unknown-unknown"
+
+define <4 x i64> @testv4i64(<4 x i64> %in) {
+; AVX1-LABEL: testv4i64:
+; AVX1:       # BB#0:
+; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
+; AVX1-NEXT:    vmovaps {{.*#+}} xmm2 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
+; AVX1-NEXT:    vandps %xmm2, %xmm1, %xmm3
+; AVX1-NEXT:    vmovdqa {{.*#+}} xmm4 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4]
+; AVX1-NEXT:    vpshufb %xmm3, %xmm4, %xmm3
+; AVX1-NEXT:    vpsrlw $4, %xmm1, %xmm1
+; AVX1-NEXT:    vpand %xmm2, %xmm1, %xmm1
+; AVX1-NEXT:    vpshufb %xmm1, %xmm4, %xmm1
+; AVX1-NEXT:    vpaddb %xmm3, %xmm1, %xmm1
+; AVX1-NEXT:    vpxor %xmm3, %xmm3, %xmm3
+; AVX1-NEXT:    vpsadbw %xmm1, %xmm3, %xmm1
+; AVX1-NEXT:    vandps %xmm2, %xmm0, %xmm5
+; AVX1-NEXT:    vpshufb %xmm5, %xmm4, %xmm5
+; AVX1-NEXT:    vpsrlw $4, %xmm0, %xmm0
+; AVX1-NEXT:    vpand %xmm2, %xmm0, %xmm0
+; AVX1-NEXT:    vpshufb %xmm0, %xmm4, %xmm0
+; AVX1-NEXT:    vpaddb %xmm5, %xmm0, %xmm0
+; AVX1-NEXT:    vpsadbw %xmm0, %xmm3, %xmm0
+; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: testv4i64:
+; AVX2:       # BB#0:
+; AVX2-NEXT:    vmovdqa {{.*#+}} ymm1 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
+; AVX2-NEXT:    vpand %ymm1, %ymm0, %ymm2
+; AVX2-NEXT:    vmovdqa {{.*#+}} ymm3 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4,0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4]
+; AVX2-NEXT:    vpshufb %ymm2, %ymm3, %ymm2
+; AVX2-NEXT:    vpsrlw $4, %ymm0, %ymm0
+; AVX2-NEXT:    vpand %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vpshufb %ymm0, %ymm3, %ymm0
+; AVX2-NEXT:    vpaddb %ymm2, %ymm0, %ymm0
+; AVX2-NEXT:    vpxor %ymm1, %ymm1, %ymm1
+; AVX2-NEXT:    vpsadbw %ymm0, %ymm1, %ymm0
+; AVX2-NEXT:    retq
+  %out = call <4 x i64> @llvm.ctpop.v4i64(<4 x i64> %in)
+  ret <4 x i64> %out
+}
+
+define <8 x i32> @testv8i32(<8 x i32> %in) {
+; AVX1-LABEL: testv8i32:
+; AVX1:       # BB#0:
+; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
+; AVX1-NEXT:    vmovaps {{.*#+}} xmm2 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
+; AVX1-NEXT:    vandps %xmm2, %xmm1, %xmm3
+; AVX1-NEXT:    vmovdqa {{.*#+}} xmm4 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4]
+; AVX1-NEXT:    vpshufb %xmm3, %xmm4, %xmm3
+; AVX1-NEXT:    vpsrlw $4, %xmm1, %xmm1
+; AVX1-NEXT:    vpand %xmm2, %xmm1, %xmm1
+; AVX1-NEXT:    vpshufb %xmm1, %xmm4, %xmm1
+; AVX1-NEXT:    vpaddb %xmm3, %xmm1, %xmm1
+; AVX1-NEXT:    vpxor %xmm3, %xmm3, %xmm3
+; AVX1-NEXT:    vpunpckhdq {{.*#+}} xmm5 = xmm1[2],xmm3[2],xmm1[3],xmm3[3]
+; AVX1-NEXT:    vpsadbw %xmm5, %xmm3, %xmm5
+; AVX1-NEXT:    vpunpckldq {{.*#+}} xmm1 = xmm1[0],xmm3[0],xmm1[1],xmm3[1]
+; AVX1-NEXT:    vpsadbw %xmm1, %xmm3, %xmm1
+; AVX1-NEXT:    vpackuswb %xmm5, %xmm1, %xmm1
+; AVX1-NEXT:    vandps %xmm2, %xmm0, %xmm5
+; AVX1-NEXT:    vpshufb %xmm5, %xmm4, %xmm5
+; AVX1-NEXT:    vpsrlw $4, %xmm0, %xmm0
+; AVX1-NEXT:    vpand %xmm2, %xmm0, %xmm0
+; AVX1-NEXT:    vpshufb %xmm0, %xmm4, %xmm0
+; AVX1-NEXT:    vpaddb %xmm5, %xmm0, %xmm0
+; AVX1-NEXT:    vpunpckhdq {{.*#+}} xmm2 = xmm0[2],xmm3[2],xmm0[3],xmm3[3]
+; AVX1-NEXT:    vpsadbw %xmm2, %xmm3, %xmm2
+; AVX1-NEXT:    vpunpckldq {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1]
+; AVX1-NEXT:    vpsadbw %xmm0, %xmm3, %xmm0
+; AVX1-NEXT:    vpackuswb %xmm2, %xmm0, %xmm0
+; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: testv8i32:
+; AVX2:       # BB#0:
+; AVX2-NEXT:    vmovdqa {{.*#+}} ymm1 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
+; AVX2-NEXT:    vpand %ymm1, %ymm0, %ymm2
+; AVX2-NEXT:    vmovdqa {{.*#+}} ymm3 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4,0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4]
+; AVX2-NEXT:    vpshufb %ymm2, %ymm3, %ymm2
+; AVX2-NEXT:    vpsrlw $4, %ymm0, %ymm0
+; AVX2-NEXT:    vpand %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vpshufb %ymm0, %ymm3, %ymm0
+; AVX2-NEXT:    vpaddb %ymm2, %ymm0, %ymm0
+; AVX2-NEXT:    vpxor %ymm1, %ymm1, %ymm1
+; AVX2-NEXT:    vpunpckhdq {{.*#+}} ymm2 = ymm0[2],ymm1[2],ymm0[3],ymm1[3],ymm0[6],ymm1[6],ymm0[7],ymm1[7]
+; AVX2-NEXT:    vpsadbw %ymm2, %ymm1, %ymm2
+; AVX2-NEXT:    vpunpckldq {{.*#+}} ymm0 = ymm0[0],ymm1[0],ymm0[1],ymm1[1],ymm0[4],ymm1[4],ymm0[5],ymm1[5]
+; AVX2-NEXT:    vpsadbw %ymm0, %ymm1, %ymm0
+; AVX2-NEXT:    vpackuswb %ymm2, %ymm0, %ymm0
+; AVX2-NEXT:    retq
+  %out = call <8 x i32> @llvm.ctpop.v8i32(<8 x i32> %in)
+  ret <8 x i32> %out
+}
+
+define <16 x i16> @testv16i16(<16 x i16> %in) {
+; AVX1-LABEL: testv16i16:
+; AVX1:       # BB#0:
+; AVX1-NEXT:    vmovdqa {{.*#+}} xmm1 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
+; AVX1-NEXT:    vpand %xmm1, %xmm0, %xmm2
+; AVX1-NEXT:    vmovdqa {{.*#+}} xmm3 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4]
+; AVX1-NEXT:    vpshufb %xmm2, %xmm3, %xmm2
+; AVX1-NEXT:    vpsrlw $4, %xmm0, %xmm4
+; AVX1-NEXT:    vpand %xmm1, %xmm4, %xmm4
+; AVX1-NEXT:    vpshufb %xmm4, %xmm3, %xmm4
+; AVX1-NEXT:    vpaddb %xmm2, %xmm4, %xmm2
+; AVX1-NEXT:    vpsllw $8, %xmm2, %xmm4
+; AVX1-NEXT:    vpaddb %xmm2, %xmm4, %xmm2
+; AVX1-NEXT:    vpsrlw $8, %xmm2, %xmm2
+; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm0
+; AVX1-NEXT:    vpand %xmm1, %xmm0, %xmm4
+; AVX1-NEXT:    vpshufb %xmm4, %xmm3, %xmm4
+; AVX1-NEXT:    vpsrlw $4, %xmm0, %xmm0
+; AVX1-NEXT:    vpand %xmm1, %xmm0, %xmm0
+; AVX1-NEXT:    vpshufb %xmm0, %xmm3, %xmm0
+; AVX1-NEXT:    vpaddb %xmm4, %xmm0, %xmm0
+; AVX1-NEXT:    vpsllw $8, %xmm0, %xmm1
+; AVX1-NEXT:    vpaddb %xmm0, %xmm1, %xmm0
+; AVX1-NEXT:    vpsrlw $8, %xmm0, %xmm0
+; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm2, %ymm0
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: testv16i16:
+; AVX2:       # BB#0:
+; AVX2-NEXT:    vmovdqa {{.*#+}} ymm1 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
+; AVX2-NEXT:    vpand %ymm1, %ymm0, %ymm2
+; AVX2-NEXT:    vmovdqa {{.*#+}} ymm3 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4,0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4]
+; AVX2-NEXT:    vpshufb %ymm2, %ymm3, %ymm2
+; AVX2-NEXT:    vpsrlw $4, %ymm0, %ymm0
+; AVX2-NEXT:    vpand %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vpshufb %ymm0, %ymm3, %ymm0
+; AVX2-NEXT:    vpaddb %ymm2, %ymm0, %ymm0
+; AVX2-NEXT:    vpsllw $8, %ymm0, %ymm1
+; AVX2-NEXT:    vpaddb %ymm0, %ymm1, %ymm0
+; AVX2-NEXT:    vpsrlw $8, %ymm0, %ymm0
+; AVX2-NEXT:    retq
+  %out = call <16 x i16> @llvm.ctpop.v16i16(<16 x i16> %in)
+  ret <16 x i16> %out
+}
+
+define <32 x i8> @testv32i8(<32 x i8> %in) {
+; AVX1-LABEL: testv32i8:
+; AVX1:       # BB#0:
+; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
+; AVX1-NEXT:    vmovaps {{.*#+}} xmm2 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
+; AVX1-NEXT:    vandps %xmm2, %xmm1, %xmm3
+; AVX1-NEXT:    vmovdqa {{.*#+}} xmm4 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4]
+; AVX1-NEXT:    vpshufb %xmm3, %xmm4, %xmm3
+; AVX1-NEXT:    vpsrlw $4, %xmm1, %xmm1
+; AVX1-NEXT:    vpand %xmm2, %xmm1, %xmm1
+; AVX1-NEXT:    vpshufb %xmm1, %xmm4, %xmm1
+; AVX1-NEXT:    vpaddb %xmm3, %xmm1, %xmm1
+; AVX1-NEXT:    vandps %xmm2, %xmm0, %xmm3
+; AVX1-NEXT:    vpshufb %xmm3, %xmm4, %xmm3
+; AVX1-NEXT:    vpsrlw $4, %xmm0, %xmm0
+; AVX1-NEXT:    vpand %xmm2, %xmm0, %xmm0
+; AVX1-NEXT:    vpshufb %xmm0, %xmm4, %xmm0
+; AVX1-NEXT:    vpaddb %xmm3, %xmm0, %xmm0
+; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: testv32i8:
+; AVX2:       # BB#0:
+; AVX2-NEXT:    vmovdqa {{.*#+}} ymm1 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
+; AVX2-NEXT:    vpand %ymm1, %ymm0, %ymm2
+; AVX2-NEXT:    vmovdqa {{.*#+}} ymm3 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4,0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4]
+; AVX2-NEXT:    vpshufb %ymm2, %ymm3, %ymm2
+; AVX2-NEXT:    vpsrlw $4, %ymm0, %ymm0
+; AVX2-NEXT:    vpand %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vpshufb %ymm0, %ymm3, %ymm0
+; AVX2-NEXT:    vpaddb %ymm2, %ymm0, %ymm0
+; AVX2-NEXT:    retq
+  %out = call <32 x i8> @llvm.ctpop.v32i8(<32 x i8> %in)
+  ret <32 x i8> %out
+}
+
+define <4 x i64> @foldv4i64() {
+; AVX-LABEL: foldv4i64:
+; AVX:       # BB#0:
+; AVX-NEXT:    vmovaps {{.*#+}} ymm0 = [1,64,0,8]
+; AVX-NEXT:    retq
+  %out = call <4 x i64> @llvm.ctpop.v4i64(<4 x i64> <i64 256, i64 -1, i64 0, i64 255>)
+  ret <4 x i64> %out
+}
+
+define <8 x i32> @foldv8i32() {
+; AVX-LABEL: foldv8i32:
+; AVX:       # BB#0:
+; AVX-NEXT:    vmovaps {{.*#+}} ymm0 = [1,32,0,8,16,3,2,3]
+; AVX-NEXT:    retq
+  %out = call <8 x i32> @llvm.ctpop.v8i32(<8 x i32> <i32 256, i32 -1, i32 0, i32 255, i32 -65536, i32 7, i32 24, i32 88>)
+  ret <8 x i32> %out
+}
+
+define <16 x i16> @foldv16i16() {
+; AVX-LABEL: foldv16i16:
+; AVX:       # BB#0:
+; AVX-NEXT:    vmovaps {{.*#+}} ymm0 = [1,16,0,8,0,3,2,3,15,7,1,1,1,1,1,1]
+; AVX-NEXT:    retq
+  %out = call <16 x i16> @llvm.ctpop.v16i16(<16 x i16> <i16 256, i16 -1, i16 0, i16 255, i16 -65536, i16 7, i16 24, i16 88, i16 -2, i16 254, i16 1, i16 2, i16 4, i16 8, i16 16, i16 32>)
+  ret <16 x i16> %out
+}
+
+define <32 x i8> @foldv32i8() {
+; AVX-LABEL: foldv32i8:
+; AVX:       # BB#0:
+; AVX-NEXT:    vmovaps {{.*#+}} ymm0 = [0,8,0,8,0,3,2,3,7,7,1,1,1,1,1,1,1,1,0,0,1,2,3,4,5,6,7,8,2,2,3,7]
+; AVX-NEXT:    retq
+  %out = call <32 x i8> @llvm.ctpop.v32i8(<32 x i8> <i8 256, i8 -1, i8 0, i8 255, i8 -65536, i8 7, i8 24, i8 88, i8 -2, i8 254, i8 1, i8 2, i8 4, i8 8, i8 16, i8 32, i8 64, i8 128, i8 256, i8 -256, i8 -128, i8 -64, i8 -32, i8 -16, i8 -8, i8 -4, i8 -2, i8 -1, i8 3, i8 5, i8 7, i8 127>)
+  ret <32 x i8> %out
+}
+
+declare <4 x i64> @llvm.ctpop.v4i64(<4 x i64>)
+declare <8 x i32> @llvm.ctpop.v8i32(<8 x i32>)
+declare <16 x i16> @llvm.ctpop.v16i16(<16 x i16>)
+declare <32 x i8> @llvm.ctpop.v32i8(<32 x i8>)
diff --git a/test/CodeGen/X86/vector-sext.ll b/test/CodeGen/X86/vector-sext.ll
index c4273710a91..8d931a3109a 100644
--- a/test/CodeGen/X86/vector-sext.ll
+++ b/test/CodeGen/X86/vector-sext.ll
@@ -7,683 +7,673 @@
 ; Just one 32-bit run to make sure we do reasonable things there.
 ; RUN: llc < %s -mtriple=i686-unknown-unknown -mcpu=i686 -mattr=+sse4.1 | FileCheck %s --check-prefix=X32-SSE41
 
-define <8 x i32> @sext_8i16_to_8i32(<8 x i16> %A) nounwind uwtable readnone ssp {
-; SSE2-LABEL: sext_8i16_to_8i32:
+define <8 x i16> @sext_16i8_to_8i16(<16 x i8> %A) nounwind uwtable readnone ssp {
+; SSE2-LABEL: sext_16i8_to_8i16:
 ; SSE2:       # BB#0: # %entry
-; SSE2-NEXT:    movdqa %xmm0, %xmm1
-; SSE2-NEXT:   # kill: XMM0<def> XMM1<kill>
-; SSE2-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3]
-; SSE2-NEXT:    pslld $16, %xmm0
-; SSE2-NEXT:    psrad $16, %xmm0
-; SSE2-NEXT:    punpckhwd {{.*#+}} xmm1 = xmm1[4,4,5,5,6,6,7,7]
-; SSE2-NEXT:    pslld $16, %xmm1
-; SSE2-NEXT:    psrad $16, %xmm1
+; SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
+; SSE2-NEXT:    psraw $8, %xmm0
 ; SSE2-NEXT:    retq
 ;
-; SSSE3-LABEL: sext_8i16_to_8i32:
+; SSSE3-LABEL: sext_16i8_to_8i16:
 ; SSSE3:       # BB#0: # %entry
-; SSSE3-NEXT:    movdqa %xmm0, %xmm1
-; SSSE3-NEXT:   # kill: XMM0<def> XMM1<kill>
-; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3]
-; SSSE3-NEXT:    pslld $16, %xmm0
-; SSSE3-NEXT:    psrad $16, %xmm0
-; SSSE3-NEXT:    punpckhwd {{.*#+}} xmm1 = xmm1[4,4,5,5,6,6,7,7]
-; SSSE3-NEXT:    pslld $16, %xmm1
-; SSSE3-NEXT:    psrad $16, %xmm1
+; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
+; SSSE3-NEXT:    psraw $8, %xmm0
 ; SSSE3-NEXT:    retq
 ;
-; SSE41-LABEL: sext_8i16_to_8i32:
+; SSE41-LABEL: sext_16i8_to_8i16:
 ; SSE41:       # BB#0: # %entry
-; SSE41-NEXT:    movdqa %xmm0, %xmm1
-; SSE41-NEXT:    pmovzxwd %xmm1, %xmm0
-; SSE41-NEXT:    pslld $16, %xmm0
-; SSE41-NEXT:    psrad $16, %xmm0
-; SSE41-NEXT:    punpckhwd {{.*#+}} xmm1 = xmm1[4,4,5,5,6,6,7,7]
-; SSE41-NEXT:    pslld $16, %xmm1
-; SSE41-NEXT:    psrad $16, %xmm1
+; SSE41-NEXT:    pmovsxbw %xmm0, %xmm0
 ; SSE41-NEXT:    retq
 ;
-; AVX1-LABEL: sext_8i16_to_8i32:
-; AVX1:       # BB#0: # %entry
-; AVX1-NEXT:    vpmovsxwd %xmm0, %xmm1
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
-; AVX1-NEXT:    vpmovsxwd %xmm0, %xmm0
-; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
-; AVX1-NEXT:    retq
-;
-; AVX2-LABEL: sext_8i16_to_8i32:
-; AVX2:       # BB#0: # %entry
-; AVX2-NEXT:    vpmovsxwd %xmm0, %ymm0
-; AVX2-NEXT:    retq
+; AVX-LABEL: sext_16i8_to_8i16:
+; AVX:       # BB#0: # %entry
+; AVX-NEXT:    vpmovsxbw %xmm0, %xmm0
+; AVX-NEXT:    retq
 ;
-; X32-SSE41-LABEL: sext_8i16_to_8i32:
+; X32-SSE41-LABEL: sext_16i8_to_8i16:
 ; X32-SSE41:       # BB#0: # %entry
-; X32-SSE41-NEXT:    movdqa %xmm0, %xmm1
-; X32-SSE41-NEXT:    pmovzxwd %xmm1, %xmm0
-; X32-SSE41-NEXT:    pslld $16, %xmm0
-; X32-SSE41-NEXT:    psrad $16, %xmm0
-; X32-SSE41-NEXT:    punpckhwd {{.*#+}} xmm1 = xmm1[4,4,5,5,6,6,7,7]
-; X32-SSE41-NEXT:    pslld $16, %xmm1
-; X32-SSE41-NEXT:    psrad $16, %xmm1
+; X32-SSE41-NEXT:    pmovsxbw %xmm0, %xmm0
 ; X32-SSE41-NEXT:    retl
 entry:
-  %B = sext <8 x i16> %A to <8 x i32>
-  ret <8 x i32>%B
+  %B = shufflevector <16 x i8> %A, <16 x i8> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+  %C = sext <8 x i8> %B to <8 x i16>
+  ret <8 x i16> %C
 }
 
-define <4 x i64> @sext_4i32_to_4i64(<4 x i32> %A) nounwind uwtable readnone ssp {
-; SSE2-LABEL: sext_4i32_to_4i64:
+define <16 x i16> @sext_16i8_to_16i16(<16 x i8> %A) nounwind uwtable readnone ssp {
+; SSE2-LABEL: sext_16i8_to_16i16:
 ; SSE2:       # BB#0: # %entry
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,1,1,3]
-; SSE2-NEXT:    movd %xmm1, %rax
-; SSE2-NEXT:    cltq
-; SSE2-NEXT:    movd %rax, %xmm2
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[2,3,0,1]
-; SSE2-NEXT:    movd %xmm1, %rax
-; SSE2-NEXT:    cltq
-; SSE2-NEXT:    movd %rax, %xmm1
-; SSE2-NEXT:    punpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm1[0]
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,2,3,3]
-; SSE2-NEXT:    movd %xmm0, %rax
-; SSE2-NEXT:    cltq
-; SSE2-NEXT:    movd %rax, %xmm1
+; SSE2-NEXT:    punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3],xmm2[4],xmm0[4],xmm2[5],xmm0[5],xmm2[6],xmm0[6],xmm2[7],xmm0[7]
+; SSE2-NEXT:    psraw $8, %xmm2
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
-; SSE2-NEXT:    movd %xmm0, %rax
-; SSE2-NEXT:    cltq
-; SSE2-NEXT:    movd %rax, %xmm0
-; SSE2-NEXT:    punpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm0[0]
+; SSE2-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3],xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
+; SSE2-NEXT:    psraw $8, %xmm1
 ; SSE2-NEXT:    movdqa %xmm2, %xmm0
 ; SSE2-NEXT:    retq
 ;
-; SSSE3-LABEL: sext_4i32_to_4i64:
+; SSSE3-LABEL: sext_16i8_to_16i16:
 ; SSSE3:       # BB#0: # %entry
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,1,1,3]
-; SSSE3-NEXT:    movd %xmm1, %rax
-; SSSE3-NEXT:    cltq
-; SSSE3-NEXT:    movd %rax, %xmm2
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[2,3,0,1]
-; SSSE3-NEXT:    movd %xmm1, %rax
-; SSSE3-NEXT:    cltq
-; SSSE3-NEXT:    movd %rax, %xmm1
-; SSSE3-NEXT:    punpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm1[0]
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,2,3,3]
-; SSSE3-NEXT:    movd %xmm0, %rax
-; SSSE3-NEXT:    cltq
-; SSSE3-NEXT:    movd %rax, %xmm1
+; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3],xmm2[4],xmm0[4],xmm2[5],xmm0[5],xmm2[6],xmm0[6],xmm2[7],xmm0[7]
+; SSSE3-NEXT:    psraw $8, %xmm2
 ; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
-; SSSE3-NEXT:    movd %xmm0, %rax
-; SSSE3-NEXT:    cltq
-; SSSE3-NEXT:    movd %rax, %xmm0
-; SSSE3-NEXT:    punpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm0[0]
+; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3],xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
+; SSSE3-NEXT:    psraw $8, %xmm1
 ; SSSE3-NEXT:    movdqa %xmm2, %xmm0
 ; SSSE3-NEXT:    retq
 ;
-; SSE41-LABEL: sext_4i32_to_4i64:
+; SSE41-LABEL: sext_16i8_to_16i16:
 ; SSE41:       # BB#0: # %entry
-; SSE41-NEXT:    pmovzxdq %xmm0, %xmm1
-; SSE41-NEXT:    pextrq $1, %xmm1, %rax
-; SSE41-NEXT:    cltq
-; SSE41-NEXT:    movd %rax, %xmm3
-; SSE41-NEXT:    movd %xmm1, %rax
-; SSE41-NEXT:    cltq
-; SSE41-NEXT:    movd %rax, %xmm2
-; SSE41-NEXT:    punpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm3[0]
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,2,3,3]
-; SSE41-NEXT:    pextrq $1, %xmm0, %rax
-; SSE41-NEXT:    cltq
-; SSE41-NEXT:    movd %rax, %xmm3
-; SSE41-NEXT:    movd %xmm0, %rax
-; SSE41-NEXT:    cltq
-; SSE41-NEXT:    movd %rax, %xmm1
-; SSE41-NEXT:    punpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm3[0]
+; SSE41-NEXT:    pmovsxbw %xmm0, %xmm2
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
+; SSE41-NEXT:    pmovsxbw %xmm0, %xmm1
 ; SSE41-NEXT:    movdqa %xmm2, %xmm0
 ; SSE41-NEXT:    retq
 ;
-; AVX1-LABEL: sext_4i32_to_4i64:
+; AVX1-LABEL: sext_16i8_to_16i16:
 ; AVX1:       # BB#0: # %entry
-; AVX1-NEXT:    vpmovsxdq %xmm0, %xmm1
+; AVX1-NEXT:    vpmovsxbw %xmm0, %xmm1
 ; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
-; AVX1-NEXT:    vpmovsxdq %xmm0, %xmm0
+; AVX1-NEXT:    vpmovsxbw %xmm0, %xmm0
 ; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
 ; AVX1-NEXT:    retq
 ;
-; AVX2-LABEL: sext_4i32_to_4i64:
+; AVX2-LABEL: sext_16i8_to_16i16:
 ; AVX2:       # BB#0: # %entry
-; AVX2-NEXT:    vpmovsxdq %xmm0, %ymm0
+; AVX2-NEXT:    vpmovsxbw %xmm0, %ymm0
 ; AVX2-NEXT:    retq
 ;
-; X32-SSE41-LABEL: sext_4i32_to_4i64:
+; X32-SSE41-LABEL: sext_16i8_to_16i16:
 ; X32-SSE41:       # BB#0: # %entry
-; X32-SSE41-NEXT:    pmovzxdq %xmm0, %xmm2
-; X32-SSE41-NEXT:    movd %xmm2, %eax
-; X32-SSE41-NEXT:    sarl $31, %eax
-; X32-SSE41-NEXT:    pextrd $2, %xmm2, %ecx
-; X32-SSE41-NEXT:    pinsrd $1, %eax, %xmm2
-; X32-SSE41-NEXT:    sarl $31, %ecx
-; X32-SSE41-NEXT:    pinsrd $3, %ecx, %xmm2
-; X32-SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,2,3,3]
-; X32-SSE41-NEXT:    movd %xmm1, %eax
-; X32-SSE41-NEXT:    sarl $31, %eax
-; X32-SSE41-NEXT:    pextrd $2, %xmm1, %ecx
-; X32-SSE41-NEXT:    pinsrd $1, %eax, %xmm1
-; X32-SSE41-NEXT:    sarl $31, %ecx
-; X32-SSE41-NEXT:    pinsrd $3, %ecx, %xmm1
+; X32-SSE41-NEXT:    pmovsxbw %xmm0, %xmm2
+; X32-SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
+; X32-SSE41-NEXT:    pmovsxbw %xmm0, %xmm1
 ; X32-SSE41-NEXT:    movdqa %xmm2, %xmm0
 ; X32-SSE41-NEXT:    retl
 entry:
-  %B = sext <4 x i32> %A to <4 x i64>
-  ret <4 x i64>%B
+  %B = sext <16 x i8> %A to <16 x i16>
+  ret <16 x i16> %B
 }
 
-define <4 x i32> @load_sext_test1(<4 x i16> *%ptr) {
-; SSE2-LABEL: load_sext_test1:
+define <4 x i32> @sext_16i8_to_4i32(<16 x i8> %A) nounwind uwtable readnone ssp {
+; SSE2-LABEL: sext_16i8_to_4i32:
 ; SSE2:       # BB#0: # %entry
-; SSE2-NEXT:    movq (%rdi), %xmm0
+; SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
 ; SSE2-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3]
-; SSE2-NEXT:    psrad $16, %xmm0
+; SSE2-NEXT:    psrad $24, %xmm0
 ; SSE2-NEXT:    retq
 ;
-; SSSE3-LABEL: load_sext_test1:
+; SSSE3-LABEL: sext_16i8_to_4i32:
 ; SSSE3:       # BB#0: # %entry
-; SSSE3-NEXT:    movq (%rdi), %xmm0
+; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
 ; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3]
-; SSSE3-NEXT:    psrad $16, %xmm0
+; SSSE3-NEXT:    psrad $24, %xmm0
 ; SSSE3-NEXT:    retq
 ;
-; SSE41-LABEL: load_sext_test1:
+; SSE41-LABEL: sext_16i8_to_4i32:
 ; SSE41:       # BB#0: # %entry
-; SSE41-NEXT:    pmovsxwd (%rdi), %xmm0
+; SSE41-NEXT:    pmovsxbd %xmm0, %xmm0
 ; SSE41-NEXT:    retq
 ;
-; AVX-LABEL: load_sext_test1:
+; AVX-LABEL: sext_16i8_to_4i32:
 ; AVX:       # BB#0: # %entry
-; AVX-NEXT:    vpmovsxwd (%rdi), %xmm0
+; AVX-NEXT:    vpmovsxbd %xmm0, %xmm0
 ; AVX-NEXT:    retq
 ;
-; X32-SSE41-LABEL: load_sext_test1:
+; X32-SSE41-LABEL: sext_16i8_to_4i32:
 ; X32-SSE41:       # BB#0: # %entry
-; X32-SSE41-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X32-SSE41-NEXT:    pmovsxwd (%eax), %xmm0
+; X32-SSE41-NEXT:    pmovsxbd %xmm0, %xmm0
 ; X32-SSE41-NEXT:    retl
 entry:
- %X = load <4 x i16>, <4 x i16>* %ptr
- %Y = sext <4 x i16> %X to <4 x i32>
- ret <4 x i32>%Y
+  %B = shufflevector <16 x i8> %A, <16 x i8> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %C = sext <4 x i8> %B to <4 x i32>
+  ret <4 x i32> %C
+}
+
+define <8 x i32> @sext_16i8_to_8i32(<16 x i8> %A) nounwind uwtable readnone ssp {
+; SSE2-LABEL: sext_16i8_to_8i32:
+; SSE2:       # BB#0: # %entry
+; SSE2-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3],xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
+; SSE2-NEXT:    punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3]
+; SSE2-NEXT:    psrad $24, %xmm2
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,2,3]
+; SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
+; SSE2-NEXT:    punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]
+; SSE2-NEXT:    psrad $24, %xmm1
+; SSE2-NEXT:    movdqa %xmm2, %xmm0
+; SSE2-NEXT:    retq
+;
+; SSSE3-LABEL: sext_16i8_to_8i32:
+; SSSE3:       # BB#0: # %entry
+; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3],xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
+; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3]
+; SSSE3-NEXT:    psrad $24, %xmm2
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,2,3]
+; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
+; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]
+; SSSE3-NEXT:    psrad $24, %xmm1
+; SSSE3-NEXT:    movdqa %xmm2, %xmm0
+; SSSE3-NEXT:    retq
+;
+; SSE41-LABEL: sext_16i8_to_8i32:
+; SSE41:       # BB#0: # %entry
+; SSE41-NEXT:    pmovsxbd %xmm0, %xmm2
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,2,3]
+; SSE41-NEXT:    pmovsxbd %xmm0, %xmm1
+; SSE41-NEXT:    movdqa %xmm2, %xmm0
+; SSE41-NEXT:    retq
+;
+; AVX1-LABEL: sext_16i8_to_8i32:
+; AVX1:       # BB#0: # %entry
+; AVX1-NEXT:    vpmovsxbd %xmm0, %xmm1
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[1,1,2,3]
+; AVX1-NEXT:    vpmovsxbd %xmm0, %xmm0
+; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: sext_16i8_to_8i32:
+; AVX2:       # BB#0: # %entry
+; AVX2-NEXT:    vpmovzxbd {{.*#+}} ymm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero,xmm0[4],zero,zero,zero,xmm0[5],zero,zero,zero,xmm0[6],zero,zero,zero,xmm0[7],zero,zero,zero
+; AVX2-NEXT:    vpslld $24, %ymm0, %ymm0
+; AVX2-NEXT:    vpsrad $24, %ymm0, %ymm0
+; AVX2-NEXT:    retq
+;
+; X32-SSE41-LABEL: sext_16i8_to_8i32:
+; X32-SSE41:       # BB#0: # %entry
+; X32-SSE41-NEXT:    pmovsxbd %xmm0, %xmm2
+; X32-SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,2,3]
+; X32-SSE41-NEXT:    pmovsxbd %xmm0, %xmm1
+; X32-SSE41-NEXT:    movdqa %xmm2, %xmm0
+; X32-SSE41-NEXT:    retl
+entry:
+  %B = shufflevector <16 x i8> %A, <16 x i8> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+  %C = sext <8 x i8> %B to <8 x i32>
+  ret <8 x i32> %C
 }
 
-define <4 x i32> @load_sext_test2(<4 x i8> *%ptr) {
-; SSE2-LABEL: load_sext_test2:
+define <2 x i64> @sext_16i8_to_2i64(<16 x i8> %A) nounwind uwtable readnone ssp {
+; SSE2-LABEL: sext_16i8_to_2i64:
 ; SSE2:       # BB#0: # %entry
-; SSE2-NEXT:    movd (%rdi), %xmm0
 ; SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
 ; SSE2-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3]
+; SSE2-NEXT:    movdqa %xmm0, %xmm1
+; SSE2-NEXT:    psrad $31, %xmm1
 ; SSE2-NEXT:    psrad $24, %xmm0
+; SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
 ; SSE2-NEXT:    retq
 ;
-; SSSE3-LABEL: load_sext_test2:
+; SSSE3-LABEL: sext_16i8_to_2i64:
 ; SSSE3:       # BB#0: # %entry
-; SSSE3-NEXT:    movd (%rdi), %xmm0
 ; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
 ; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3]
+; SSSE3-NEXT:    movdqa %xmm0, %xmm1
+; SSSE3-NEXT:    psrad $31, %xmm1
 ; SSSE3-NEXT:    psrad $24, %xmm0
+; SSSE3-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
 ; SSSE3-NEXT:    retq
 ;
-; SSE41-LABEL: load_sext_test2:
+; SSE41-LABEL: sext_16i8_to_2i64:
 ; SSE41:       # BB#0: # %entry
-; SSE41-NEXT:    pmovsxbd (%rdi), %xmm0
+; SSE41-NEXT:    pmovsxbq %xmm0, %xmm0
 ; SSE41-NEXT:    retq
 ;
-; AVX-LABEL: load_sext_test2:
+; AVX-LABEL: sext_16i8_to_2i64:
 ; AVX:       # BB#0: # %entry
-; AVX-NEXT:    vpmovsxbd (%rdi), %xmm0
+; AVX-NEXT:    vpmovsxbq %xmm0, %xmm0
 ; AVX-NEXT:    retq
 ;
-; X32-SSE41-LABEL: load_sext_test2:
+; X32-SSE41-LABEL: sext_16i8_to_2i64:
 ; X32-SSE41:       # BB#0: # %entry
-; X32-SSE41-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X32-SSE41-NEXT:    pmovsxbd (%eax), %xmm0
+; X32-SSE41-NEXT:    pmovsxbq %xmm0, %xmm0
 ; X32-SSE41-NEXT:    retl
 entry:
- %X = load <4 x i8>, <4 x i8>* %ptr
- %Y = sext <4 x i8> %X to <4 x i32>
- ret <4 x i32>%Y
+  %B = shufflevector <16 x i8> %A, <16 x i8> undef, <2 x i32> <i32 0, i32 1>
+  %C = sext <2 x i8> %B to <2 x i64>
+  ret <2 x i64> %C
 }
 
-define <2 x i64> @load_sext_test3(<2 x i8> *%ptr) {
-; SSE2-LABEL: load_sext_test3:
+define <4 x i64> @sext_16i8_to_4i64(<16 x i8> %A) nounwind uwtable readnone ssp {
+; SSE2-LABEL: sext_16i8_to_4i64:
 ; SSE2:       # BB#0: # %entry
-; SSE2-NEXT:    movsbq 1(%rdi), %rax
-; SSE2-NEXT:    movd %rax, %xmm1
-; SSE2-NEXT:    movsbq (%rdi), %rax
-; SSE2-NEXT:    movd %rax, %xmm0
-; SSE2-NEXT:    punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
+; SSE2-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3],xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
+; SSE2-NEXT:    punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3]
+; SSE2-NEXT:    movdqa %xmm2, %xmm1
+; SSE2-NEXT:    psrad $31, %xmm1
+; SSE2-NEXT:    psrad $24, %xmm2
+; SSE2-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]
+; SSE2-NEXT:    psrld $16, %xmm0
+; SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
+; SSE2-NEXT:    punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]
+; SSE2-NEXT:    movdqa %xmm1, %xmm0
+; SSE2-NEXT:    psrad $31, %xmm0
+; SSE2-NEXT:    psrad $24, %xmm1
+; SSE2-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
+; SSE2-NEXT:    movdqa %xmm2, %xmm0
 ; SSE2-NEXT:    retq
 ;
-; SSSE3-LABEL: load_sext_test3:
+; SSSE3-LABEL: sext_16i8_to_4i64:
 ; SSSE3:       # BB#0: # %entry
-; SSSE3-NEXT:    movsbq 1(%rdi), %rax
-; SSSE3-NEXT:    movd %rax, %xmm1
-; SSSE3-NEXT:    movsbq (%rdi), %rax
-; SSSE3-NEXT:    movd %rax, %xmm0
-; SSSE3-NEXT:    punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
+; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3],xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
+; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3]
+; SSSE3-NEXT:    movdqa %xmm2, %xmm1
+; SSSE3-NEXT:    psrad $31, %xmm1
+; SSSE3-NEXT:    psrad $24, %xmm2
+; SSSE3-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]
+; SSSE3-NEXT:    psrld $16, %xmm0
+; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
+; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]
+; SSSE3-NEXT:    movdqa %xmm1, %xmm0
+; SSSE3-NEXT:    psrad $31, %xmm0
+; SSSE3-NEXT:    psrad $24, %xmm1
+; SSSE3-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
+; SSSE3-NEXT:    movdqa %xmm2, %xmm0
 ; SSSE3-NEXT:    retq
 ;
-; SSE41-LABEL: load_sext_test3:
+; SSE41-LABEL: sext_16i8_to_4i64:
 ; SSE41:       # BB#0: # %entry
-; SSE41-NEXT:    pmovsxbq (%rdi), %xmm0
+; SSE41-NEXT:    pmovsxbq %xmm0, %xmm2
+; SSE41-NEXT:    psrld $16, %xmm0
+; SSE41-NEXT:    pmovsxbq %xmm0, %xmm1
+; SSE41-NEXT:    movdqa %xmm2, %xmm0
 ; SSE41-NEXT:    retq
 ;
-; AVX-LABEL: load_sext_test3:
-; AVX:       # BB#0: # %entry
-; AVX-NEXT:    vpmovsxbq (%rdi), %xmm0
-; AVX-NEXT:    retq
+; AVX1-LABEL: sext_16i8_to_4i64:
+; AVX1:       # BB#0: # %entry
+; AVX1-NEXT:    vpmovsxbq %xmm0, %xmm1
+; AVX1-NEXT:    vpsrld $16, %xmm0, %xmm0
+; AVX1-NEXT:    vpmovsxbq %xmm0, %xmm0
+; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: sext_16i8_to_4i64:
+; AVX2:       # BB#0: # %entry
+; AVX2-NEXT:    vpmovzxbd {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
+; AVX2-NEXT:    vpslld $24, %xmm0, %xmm0
+; AVX2-NEXT:    vpsrad $24, %xmm0, %xmm0
+; AVX2-NEXT:    vpmovsxdq %xmm0, %ymm0
+; AVX2-NEXT:    retq
 ;
-; X32-SSE41-LABEL: load_sext_test3:
+; X32-SSE41-LABEL: sext_16i8_to_4i64:
 ; X32-SSE41:       # BB#0: # %entry
-; X32-SSE41-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X32-SSE41-NEXT:    pmovsxbq (%eax), %xmm0
+; X32-SSE41-NEXT:    pmovsxbq %xmm0, %xmm2
+; X32-SSE41-NEXT:    psrld $16, %xmm0
+; X32-SSE41-NEXT:    pmovsxbq %xmm0, %xmm1
+; X32-SSE41-NEXT:    movdqa %xmm2, %xmm0
 ; X32-SSE41-NEXT:    retl
 entry:
- %X = load <2 x i8>, <2 x i8>* %ptr
- %Y = sext <2 x i8> %X to <2 x i64>
- ret <2 x i64>%Y
+  %B = shufflevector <16 x i8> %A, <16 x i8> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %C = sext <4 x i8> %B to <4 x i64>
+  ret <4 x i64> %C
 }
 
-define <2 x i64> @load_sext_test4(<2 x i16> *%ptr) {
-; SSE2-LABEL: load_sext_test4:
+define <4 x i32> @sext_8i16_to_4i32(<8 x i16> %A) nounwind uwtable readnone ssp {
+; SSE2-LABEL: sext_8i16_to_4i32:
 ; SSE2:       # BB#0: # %entry
-; SSE2-NEXT:    movswq 2(%rdi), %rax
-; SSE2-NEXT:    movd %rax, %xmm1
-; SSE2-NEXT:    movswq (%rdi), %rax
-; SSE2-NEXT:    movd %rax, %xmm0
-; SSE2-NEXT:    punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
+; SSE2-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3]
+; SSE2-NEXT:    psrad $16, %xmm0
 ; SSE2-NEXT:    retq
 ;
-; SSSE3-LABEL: load_sext_test4:
+; SSSE3-LABEL: sext_8i16_to_4i32:
 ; SSSE3:       # BB#0: # %entry
-; SSSE3-NEXT:    movswq 2(%rdi), %rax
-; SSSE3-NEXT:    movd %rax, %xmm1
-; SSSE3-NEXT:    movswq (%rdi), %rax
-; SSSE3-NEXT:    movd %rax, %xmm0
-; SSSE3-NEXT:    punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
+; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3]
+; SSSE3-NEXT:    psrad $16, %xmm0
 ; SSSE3-NEXT:    retq
 ;
-; SSE41-LABEL: load_sext_test4:
+; SSE41-LABEL: sext_8i16_to_4i32:
 ; SSE41:       # BB#0: # %entry
-; SSE41-NEXT:    pmovsxwq (%rdi), %xmm0
+; SSE41-NEXT:    pmovsxwd %xmm0, %xmm0
 ; SSE41-NEXT:    retq
 ;
-; AVX-LABEL: load_sext_test4:
+; AVX-LABEL: sext_8i16_to_4i32:
 ; AVX:       # BB#0: # %entry
-; AVX-NEXT:    vpmovsxwq (%rdi), %xmm0
+; AVX-NEXT:    vpmovsxwd %xmm0, %xmm0
 ; AVX-NEXT:    retq
 ;
-; X32-SSE41-LABEL: load_sext_test4:
+; X32-SSE41-LABEL: sext_8i16_to_4i32:
 ; X32-SSE41:       # BB#0: # %entry
-; X32-SSE41-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X32-SSE41-NEXT:    pmovsxwq (%eax), %xmm0
+; X32-SSE41-NEXT:    pmovsxwd %xmm0, %xmm0
 ; X32-SSE41-NEXT:    retl
 entry:
- %X = load <2 x i16>, <2 x i16>* %ptr
- %Y = sext <2 x i16> %X to <2 x i64>
- ret <2 x i64>%Y
+  %B = shufflevector <8 x i16> %A, <8 x i16> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %C = sext <4 x i16> %B to <4 x i32>
+  ret <4 x i32> %C
 }
 
-define <2 x i64> @load_sext_test5(<2 x i32> *%ptr) {
-; SSE2-LABEL: load_sext_test5:
+define <8 x i32> @sext_8i16_to_8i32(<8 x i16> %A) nounwind uwtable readnone ssp {
+; SSE2-LABEL: sext_8i16_to_8i32:
 ; SSE2:       # BB#0: # %entry
-; SSE2-NEXT:    movslq 4(%rdi), %rax
-; SSE2-NEXT:    movd %rax, %xmm1
-; SSE2-NEXT:    movslq (%rdi), %rax
-; SSE2-NEXT:    movd %rax, %xmm0
-; SSE2-NEXT:    punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
+; SSE2-NEXT:    punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3]
+; SSE2-NEXT:    psrad $16, %xmm2
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
+; SSE2-NEXT:    punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]
+; SSE2-NEXT:    psrad $16, %xmm1
+; SSE2-NEXT:    movdqa %xmm2, %xmm0
 ; SSE2-NEXT:    retq
 ;
-; SSSE3-LABEL: load_sext_test5:
+; SSSE3-LABEL: sext_8i16_to_8i32:
 ; SSSE3:       # BB#0: # %entry
-; SSSE3-NEXT:    movslq 4(%rdi), %rax
-; SSSE3-NEXT:    movd %rax, %xmm1
-; SSSE3-NEXT:    movslq (%rdi), %rax
-; SSSE3-NEXT:    movd %rax, %xmm0
-; SSSE3-NEXT:    punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
+; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3]
+; SSSE3-NEXT:    psrad $16, %xmm2
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
+; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]
+; SSSE3-NEXT:    psrad $16, %xmm1
+; SSSE3-NEXT:    movdqa %xmm2, %xmm0
 ; SSSE3-NEXT:    retq
 ;
-; SSE41-LABEL: load_sext_test5:
+; SSE41-LABEL: sext_8i16_to_8i32:
 ; SSE41:       # BB#0: # %entry
-; SSE41-NEXT:    pmovsxdq (%rdi), %xmm0
+; SSE41-NEXT:    pmovsxwd %xmm0, %xmm2
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
+; SSE41-NEXT:    pmovsxwd %xmm0, %xmm1
+; SSE41-NEXT:    movdqa %xmm2, %xmm0
 ; SSE41-NEXT:    retq
 ;
-; AVX-LABEL: load_sext_test5:
-; AVX:       # BB#0: # %entry
-; AVX-NEXT:    vpmovsxdq (%rdi), %xmm0
-; AVX-NEXT:    retq
+; AVX1-LABEL: sext_8i16_to_8i32:
+; AVX1:       # BB#0: # %entry
+; AVX1-NEXT:    vpmovsxwd %xmm0, %xmm1
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
+; AVX1-NEXT:    vpmovsxwd %xmm0, %xmm0
+; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: sext_8i16_to_8i32:
+; AVX2:       # BB#0: # %entry
+; AVX2-NEXT:    vpmovsxwd %xmm0, %ymm0
+; AVX2-NEXT:    retq
 ;
-; X32-SSE41-LABEL: load_sext_test5:
+; X32-SSE41-LABEL: sext_8i16_to_8i32:
 ; X32-SSE41:       # BB#0: # %entry
-; X32-SSE41-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X32-SSE41-NEXT:    pmovsxdq (%eax), %xmm0
+; X32-SSE41-NEXT:    pmovsxwd %xmm0, %xmm2
+; X32-SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
+; X32-SSE41-NEXT:    pmovsxwd %xmm0, %xmm1
+; X32-SSE41-NEXT:    movdqa %xmm2, %xmm0
 ; X32-SSE41-NEXT:    retl
 entry:
- %X = load <2 x i32>, <2 x i32>* %ptr
- %Y = sext <2 x i32> %X to <2 x i64>
- ret <2 x i64>%Y
+  %B = sext <8 x i16> %A to <8 x i32>
+  ret <8 x i32> %B
 }
 
-define <8 x i16> @load_sext_test6(<8 x i8> *%ptr) {
-; SSE2-LABEL: load_sext_test6:
+define <2 x i64> @sext_8i16_to_2i64(<8 x i16> %A) nounwind uwtable readnone ssp {
+; SSE2-LABEL: sext_8i16_to_2i64:
 ; SSE2:       # BB#0: # %entry
-; SSE2-NEXT:    movq (%rdi), %xmm0
-; SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
-; SSE2-NEXT:    psraw $8, %xmm0
+; SSE2-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3]
+; SSE2-NEXT:    movdqa %xmm0, %xmm1
+; SSE2-NEXT:    psrad $31, %xmm1
+; SSE2-NEXT:    psrad $16, %xmm0
+; SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
 ; SSE2-NEXT:    retq
 ;
-; SSSE3-LABEL: load_sext_test6:
+; SSSE3-LABEL: sext_8i16_to_2i64:
 ; SSSE3:       # BB#0: # %entry
-; SSSE3-NEXT:    movq (%rdi), %xmm0
-; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
-; SSSE3-NEXT:    psraw $8, %xmm0
+; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3]
+; SSSE3-NEXT:    movdqa %xmm0, %xmm1
+; SSSE3-NEXT:    psrad $31, %xmm1
+; SSSE3-NEXT:    psrad $16, %xmm0
+; SSSE3-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
 ; SSSE3-NEXT:    retq
 ;
-; SSE41-LABEL: load_sext_test6:
+; SSE41-LABEL: sext_8i16_to_2i64:
 ; SSE41:       # BB#0: # %entry
-; SSE41-NEXT:    pmovsxbw (%rdi), %xmm0
+; SSE41-NEXT:    pmovsxwq %xmm0, %xmm0
 ; SSE41-NEXT:    retq
 ;
-; AVX-LABEL: load_sext_test6:
+; AVX-LABEL: sext_8i16_to_2i64:
 ; AVX:       # BB#0: # %entry
-; AVX-NEXT:    vpmovsxbw (%rdi), %xmm0
+; AVX-NEXT:    vpmovsxwq %xmm0, %xmm0
 ; AVX-NEXT:    retq
 ;
-; X32-SSE41-LABEL: load_sext_test6:
+; X32-SSE41-LABEL: sext_8i16_to_2i64:
 ; X32-SSE41:       # BB#0: # %entry
-; X32-SSE41-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X32-SSE41-NEXT:    pmovsxbw (%eax), %xmm0
+; X32-SSE41-NEXT:    pmovsxwq %xmm0, %xmm0
 ; X32-SSE41-NEXT:    retl
 entry:
- %X = load <8 x i8>, <8 x i8>* %ptr
- %Y = sext <8 x i8> %X to <8 x i16>
- ret <8 x i16>%Y
+  %B = shufflevector <8 x i16> %A, <8 x i16> undef, <2 x i32> <i32 0, i32 1>
+  %C = sext <2 x i16> %B to <2 x i64>
+  ret <2 x i64> %C
 }
 
-define <4 x i64> @sext_4i1_to_4i64(<4 x i1> %mask) {
-; SSE2-LABEL: sext_4i1_to_4i64:
-; SSE2:       # BB#0:
-; SSE2-NEXT:    pslld $31, %xmm0
+define <4 x i64> @sext_8i16_to_4i64(<8 x i16> %A) nounwind uwtable readnone ssp {
+; SSE2-LABEL: sext_8i16_to_4i64:
+; SSE2:       # BB#0: # %entry
+; SSE2-NEXT:    punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3]
+; SSE2-NEXT:    movdqa %xmm2, %xmm1
+; SSE2-NEXT:    psrad $31, %xmm1
+; SSE2-NEXT:    psrad $16, %xmm2
+; SSE2-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,2,3]
+; SSE2-NEXT:    punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]
+; SSE2-NEXT:    movdqa %xmm1, %xmm0
 ; SSE2-NEXT:    psrad $31, %xmm0
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,1,1,3]
-; SSE2-NEXT:    movd %xmm1, %rax
-; SSE2-NEXT:    cltq
-; SSE2-NEXT:    movd %rax, %xmm2
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[2,3,0,1]
-; SSE2-NEXT:    movd %xmm1, %rax
-; SSE2-NEXT:    cltq
-; SSE2-NEXT:    movd %rax, %xmm1
-; SSE2-NEXT:    punpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm1[0]
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,2,3,3]
-; SSE2-NEXT:    movd %xmm0, %rax
-; SSE2-NEXT:    cltq
-; SSE2-NEXT:    movd %rax, %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
-; SSE2-NEXT:    movd %xmm0, %rax
-; SSE2-NEXT:    cltq
-; SSE2-NEXT:    movd %rax, %xmm0
-; SSE2-NEXT:    punpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm0[0]
+; SSE2-NEXT:    psrad $16, %xmm1
+; SSE2-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
 ; SSE2-NEXT:    movdqa %xmm2, %xmm0
 ; SSE2-NEXT:    retq
 ;
-; SSSE3-LABEL: sext_4i1_to_4i64:
-; SSSE3:       # BB#0:
-; SSSE3-NEXT:    pslld $31, %xmm0
+; SSSE3-LABEL: sext_8i16_to_4i64:
+; SSSE3:       # BB#0: # %entry
+; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3]
+; SSSE3-NEXT:    movdqa %xmm2, %xmm1
+; SSSE3-NEXT:    psrad $31, %xmm1
+; SSSE3-NEXT:    psrad $16, %xmm2
+; SSSE3-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,2,3]
+; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]
+; SSSE3-NEXT:    movdqa %xmm1, %xmm0
 ; SSSE3-NEXT:    psrad $31, %xmm0
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,1,1,3]
-; SSSE3-NEXT:    movd %xmm1, %rax
-; SSSE3-NEXT:    cltq
-; SSSE3-NEXT:    movd %rax, %xmm2
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[2,3,0,1]
-; SSSE3-NEXT:    movd %xmm1, %rax
-; SSSE3-NEXT:    cltq
-; SSSE3-NEXT:    movd %rax, %xmm1
-; SSSE3-NEXT:    punpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm1[0]
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,2,3,3]
-; SSSE3-NEXT:    movd %xmm0, %rax
-; SSSE3-NEXT:    cltq
-; SSSE3-NEXT:    movd %rax, %xmm1
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
-; SSSE3-NEXT:    movd %xmm0, %rax
-; SSSE3-NEXT:    cltq
-; SSSE3-NEXT:    movd %rax, %xmm0
-; SSSE3-NEXT:    punpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm0[0]
+; SSSE3-NEXT:    psrad $16, %xmm1
+; SSSE3-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
 ; SSSE3-NEXT:    movdqa %xmm2, %xmm0
 ; SSSE3-NEXT:    retq
 ;
-; SSE41-LABEL: sext_4i1_to_4i64:
-; SSE41:       # BB#0:
-; SSE41-NEXT:    pslld $31, %xmm0
-; SSE41-NEXT:    psrad $31, %xmm0
-; SSE41-NEXT:    pmovzxdq %xmm0, %xmm1
-; SSE41-NEXT:    pextrq $1, %xmm1, %rax
-; SSE41-NEXT:    cltq
-; SSE41-NEXT:    movd %rax, %xmm3
-; SSE41-NEXT:    movd %xmm1, %rax
-; SSE41-NEXT:    cltq
-; SSE41-NEXT:    movd %rax, %xmm2
-; SSE41-NEXT:    punpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm3[0]
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,2,3,3]
-; SSE41-NEXT:    pextrq $1, %xmm0, %rax
-; SSE41-NEXT:    cltq
-; SSE41-NEXT:    movd %rax, %xmm3
-; SSE41-NEXT:    movd %xmm0, %rax
-; SSE41-NEXT:    cltq
-; SSE41-NEXT:    movd %rax, %xmm1
-; SSE41-NEXT:    punpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm3[0]
+; SSE41-LABEL: sext_8i16_to_4i64:
+; SSE41:       # BB#0: # %entry
+; SSE41-NEXT:    pmovsxwq %xmm0, %xmm2
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,2,3]
+; SSE41-NEXT:    pmovsxwq %xmm0, %xmm1
 ; SSE41-NEXT:    movdqa %xmm2, %xmm0
 ; SSE41-NEXT:    retq
 ;
-; AVX1-LABEL: sext_4i1_to_4i64:
-; AVX1:       # BB#0:
-; AVX1-NEXT:    vpslld $31, %xmm0, %xmm0
-; AVX1-NEXT:    vpsrad $31, %xmm0, %xmm0
-; AVX1-NEXT:    vpmovsxdq %xmm0, %xmm1
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
-; AVX1-NEXT:    vpmovsxdq %xmm0, %xmm0
+; AVX1-LABEL: sext_8i16_to_4i64:
+; AVX1:       # BB#0: # %entry
+; AVX1-NEXT:    vpmovsxwq %xmm0, %xmm1
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[1,1,2,3]
+; AVX1-NEXT:    vpmovsxwq %xmm0, %xmm0
 ; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
 ; AVX1-NEXT:    retq
 ;
-; AVX2-LABEL: sext_4i1_to_4i64:
-; AVX2:       # BB#0:
-; AVX2-NEXT:    vpslld $31, %xmm0, %xmm0
-; AVX2-NEXT:    vpsrad $31, %xmm0, %xmm0
+; AVX2-LABEL: sext_8i16_to_4i64:
+; AVX2:       # BB#0: # %entry
+; AVX2-NEXT:    vpmovzxwd {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero
+; AVX2-NEXT:    vpslld $16, %xmm0, %xmm0
+; AVX2-NEXT:    vpsrad $16, %xmm0, %xmm0
 ; AVX2-NEXT:    vpmovsxdq %xmm0, %ymm0
 ; AVX2-NEXT:    retq
 ;
-; X32-SSE41-LABEL: sext_4i1_to_4i64:
-; X32-SSE41:       # BB#0:
-; X32-SSE41-NEXT:    pslld $31, %xmm0
-; X32-SSE41-NEXT:    psrad $31, %xmm0
-; X32-SSE41-NEXT:    pmovzxdq %xmm0, %xmm2
-; X32-SSE41-NEXT:    movd %xmm2, %eax
-; X32-SSE41-NEXT:    sarl $31, %eax
-; X32-SSE41-NEXT:    pextrd $2, %xmm2, %ecx
-; X32-SSE41-NEXT:    pinsrd $1, %eax, %xmm2
-; X32-SSE41-NEXT:    sarl $31, %ecx
-; X32-SSE41-NEXT:    pinsrd $3, %ecx, %xmm2
-; X32-SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,2,3,3]
-; X32-SSE41-NEXT:    movd %xmm1, %eax
-; X32-SSE41-NEXT:    sarl $31, %eax
-; X32-SSE41-NEXT:    pextrd $2, %xmm1, %ecx
-; X32-SSE41-NEXT:    pinsrd $1, %eax, %xmm1
-; X32-SSE41-NEXT:    sarl $31, %ecx
-; X32-SSE41-NEXT:    pinsrd $3, %ecx, %xmm1
+; X32-SSE41-LABEL: sext_8i16_to_4i64:
+; X32-SSE41:       # BB#0: # %entry
+; X32-SSE41-NEXT:    pmovsxwq %xmm0, %xmm2
+; X32-SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,2,3]
+; X32-SSE41-NEXT:    pmovsxwq %xmm0, %xmm1
 ; X32-SSE41-NEXT:    movdqa %xmm2, %xmm0
 ; X32-SSE41-NEXT:    retl
-  %extmask = sext <4 x i1> %mask to <4 x i64>
-  ret <4 x i64> %extmask
+entry:
+  %B = shufflevector <8 x i16> %A, <8 x i16> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %C = sext <4 x i16> %B to <4 x i64>
+  ret <4 x i64> %C
 }
 
-define <16 x i16> @sext_16i8_to_16i16(<16 x i8> *%ptr) {
-; SSE2-LABEL: sext_16i8_to_16i16:
+define <2 x i64> @sext_4i32_to_2i64(<4 x i32> %A) nounwind uwtable readnone ssp {
+; SSE2-LABEL: sext_4i32_to_2i64:
 ; SSE2:       # BB#0: # %entry
-; SSE2-NEXT:    movq (%rdi), %xmm0
-; SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
-; SSE2-NEXT:    psraw $8, %xmm0
-; SSE2-NEXT:    movq 8(%rdi), %xmm1
-; SSE2-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
-; SSE2-NEXT:    psraw $8, %xmm1
+; SSE2-NEXT:    movdqa %xmm0, %xmm1
+; SSE2-NEXT:    psrad $31, %xmm1
+; SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
 ; SSE2-NEXT:    retq
 ;
-; SSSE3-LABEL: sext_16i8_to_16i16:
+; SSSE3-LABEL: sext_4i32_to_2i64:
 ; SSSE3:       # BB#0: # %entry
-; SSSE3-NEXT:    movq (%rdi), %xmm0
-; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
-; SSSE3-NEXT:    psraw $8, %xmm0
-; SSSE3-NEXT:    movq 8(%rdi), %xmm1
-; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
-; SSSE3-NEXT:    psraw $8, %xmm1
+; SSSE3-NEXT:    movdqa %xmm0, %xmm1
+; SSSE3-NEXT:    psrad $31, %xmm1
+; SSSE3-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
 ; SSSE3-NEXT:    retq
 ;
-; SSE41-LABEL: sext_16i8_to_16i16:
+; SSE41-LABEL: sext_4i32_to_2i64:
 ; SSE41:       # BB#0: # %entry
-; SSE41-NEXT:    pmovsxbw (%rdi), %xmm0
-; SSE41-NEXT:    pmovsxbw 8(%rdi), %xmm1
+; SSE41-NEXT:    pmovsxdq %xmm0, %xmm0
 ; SSE41-NEXT:    retq
 ;
-; AVX1-LABEL: sext_16i8_to_16i16:
-; AVX1:       # BB#0: # %entry
-; AVX1-NEXT:    vpmovsxbw (%rdi), %xmm0
-; AVX1-NEXT:    vpmovsxbw 8(%rdi), %xmm1
-; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
-; AVX1-NEXT:    retq
-;
-; AVX2-LABEL: sext_16i8_to_16i16:
-; AVX2:       # BB#0: # %entry
-; AVX2-NEXT:    vpmovsxbw (%rdi), %ymm0
-; AVX2-NEXT:    retq
+; AVX-LABEL: sext_4i32_to_2i64:
+; AVX:       # BB#0: # %entry
+; AVX-NEXT:    vpmovsxdq %xmm0, %xmm0
+; AVX-NEXT:    retq
 ;
-; X32-SSE41-LABEL: sext_16i8_to_16i16:
+; X32-SSE41-LABEL: sext_4i32_to_2i64:
 ; X32-SSE41:       # BB#0: # %entry
-; X32-SSE41-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X32-SSE41-NEXT:    pmovsxbw (%eax), %xmm0
-; X32-SSE41-NEXT:    pmovsxbw 8(%eax), %xmm1
+; X32-SSE41-NEXT:    pmovsxdq %xmm0, %xmm0
 ; X32-SSE41-NEXT:    retl
 entry:
- %X = load <16 x i8>, <16 x i8>* %ptr
- %Y = sext <16 x i8> %X to <16 x i16>
- ret <16 x i16> %Y
+  %B = shufflevector <4 x i32> %A, <4 x i32> undef, <2 x i32> <i32 0, i32 1>
+  %C = sext <2 x i32> %B to <2 x i64>
+  ret <2 x i64> %C
 }
 
-define <4 x i64> @sext_4i8_to_4i64(<4 x i8> %mask) {
-; SSE2-LABEL: sext_4i8_to_4i64:
-; SSE2:       # BB#0:
-; SSE2-NEXT:    pslld $24, %xmm0
-; SSE2-NEXT:    psrad $24, %xmm0
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,1,1,3]
-; SSE2-NEXT:    movd %xmm1, %rax
-; SSE2-NEXT:    cltq
-; SSE2-NEXT:    movd %rax, %xmm2
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[2,3,0,1]
-; SSE2-NEXT:    movd %xmm1, %rax
-; SSE2-NEXT:    cltq
-; SSE2-NEXT:    movd %rax, %xmm1
-; SSE2-NEXT:    punpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm1[0]
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,2,3,3]
-; SSE2-NEXT:    movd %xmm0, %rax
-; SSE2-NEXT:    cltq
-; SSE2-NEXT:    movd %rax, %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
-; SSE2-NEXT:    movd %xmm0, %rax
-; SSE2-NEXT:    cltq
-; SSE2-NEXT:    movd %rax, %xmm0
-; SSE2-NEXT:    punpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm0[0]
-; SSE2-NEXT:    movdqa %xmm2, %xmm0
+define <4 x i64> @sext_4i32_to_4i64(<4 x i32> %A) nounwind uwtable readnone ssp {
+; SSE2-LABEL: sext_4i32_to_4i64:
+; SSE2:       # BB#0: # %entry
+; SSE2-NEXT:    movdqa %xmm0, %xmm2
+; SSE2-NEXT:    psrad $31, %xmm2
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1]
+; SSE2-NEXT:    movdqa %xmm1, %xmm2
+; SSE2-NEXT:    psrad $31, %xmm2
+; SSE2-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1]
 ; SSE2-NEXT:    retq
 ;
-; SSSE3-LABEL: sext_4i8_to_4i64:
-; SSSE3:       # BB#0:
-; SSSE3-NEXT:    pslld $24, %xmm0
-; SSSE3-NEXT:    psrad $24, %xmm0
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,1,1,3]
-; SSSE3-NEXT:    movd %xmm1, %rax
-; SSSE3-NEXT:    cltq
-; SSSE3-NEXT:    movd %rax, %xmm2
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[2,3,0,1]
-; SSSE3-NEXT:    movd %xmm1, %rax
-; SSSE3-NEXT:    cltq
-; SSSE3-NEXT:    movd %rax, %xmm1
-; SSSE3-NEXT:    punpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm1[0]
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,2,3,3]
-; SSSE3-NEXT:    movd %xmm0, %rax
-; SSSE3-NEXT:    cltq
-; SSSE3-NEXT:    movd %rax, %xmm1
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
-; SSSE3-NEXT:    movd %xmm0, %rax
-; SSSE3-NEXT:    cltq
-; SSSE3-NEXT:    movd %rax, %xmm0
-; SSSE3-NEXT:    punpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm0[0]
-; SSSE3-NEXT:    movdqa %xmm2, %xmm0
+; SSSE3-LABEL: sext_4i32_to_4i64:
+; SSSE3:       # BB#0: # %entry
+; SSSE3-NEXT:    movdqa %xmm0, %xmm2
+; SSSE3-NEXT:    psrad $31, %xmm2
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; SSSE3-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1]
+; SSSE3-NEXT:    movdqa %xmm1, %xmm2
+; SSSE3-NEXT:    psrad $31, %xmm2
+; SSSE3-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1]
 ; SSSE3-NEXT:    retq
 ;
-; SSE41-LABEL: sext_4i8_to_4i64:
-; SSE41:       # BB#0:
-; SSE41-NEXT:    pslld $24, %xmm0
-; SSE41-NEXT:    psrad $24, %xmm0
-; SSE41-NEXT:    pmovzxdq %xmm0, %xmm1
-; SSE41-NEXT:    pextrq $1, %xmm1, %rax
-; SSE41-NEXT:    cltq
-; SSE41-NEXT:    movd %rax, %xmm3
-; SSE41-NEXT:    movd %xmm1, %rax
-; SSE41-NEXT:    cltq
-; SSE41-NEXT:    movd %rax, %xmm2
-; SSE41-NEXT:    punpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm3[0]
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,2,3,3]
-; SSE41-NEXT:    pextrq $1, %xmm0, %rax
-; SSE41-NEXT:    cltq
-; SSE41-NEXT:    movd %rax, %xmm3
-; SSE41-NEXT:    movd %xmm0, %rax
-; SSE41-NEXT:    cltq
-; SSE41-NEXT:    movd %rax, %xmm1
-; SSE41-NEXT:    punpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm3[0]
+; SSE41-LABEL: sext_4i32_to_4i64:
+; SSE41:       # BB#0: # %entry
+; SSE41-NEXT:    pmovsxdq %xmm0, %xmm2
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
+; SSE41-NEXT:    pmovsxdq %xmm0, %xmm1
 ; SSE41-NEXT:    movdqa %xmm2, %xmm0
 ; SSE41-NEXT:    retq
 ;
-; AVX1-LABEL: sext_4i8_to_4i64:
-; AVX1:       # BB#0:
-; AVX1-NEXT:    vpslld $24, %xmm0, %xmm0
-; AVX1-NEXT:    vpsrad $24, %xmm0, %xmm0
+; AVX1-LABEL: sext_4i32_to_4i64:
+; AVX1:       # BB#0: # %entry
 ; AVX1-NEXT:    vpmovsxdq %xmm0, %xmm1
 ; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
 ; AVX1-NEXT:    vpmovsxdq %xmm0, %xmm0
 ; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
 ; AVX1-NEXT:    retq
 ;
-; AVX2-LABEL: sext_4i8_to_4i64:
-; AVX2:       # BB#0:
-; AVX2-NEXT:    vpslld $24, %xmm0, %xmm0
-; AVX2-NEXT:    vpsrad $24, %xmm0, %xmm0
+; AVX2-LABEL: sext_4i32_to_4i64:
+; AVX2:       # BB#0: # %entry
 ; AVX2-NEXT:    vpmovsxdq %xmm0, %ymm0
 ; AVX2-NEXT:    retq
 ;
-; X32-SSE41-LABEL: sext_4i8_to_4i64:
-; X32-SSE41:       # BB#0:
-; X32-SSE41-NEXT:    pslld $24, %xmm0
-; X32-SSE41-NEXT:    psrad $24, %xmm0
-; X32-SSE41-NEXT:    pmovzxdq %xmm0, %xmm2
-; X32-SSE41-NEXT:    movd %xmm2, %eax
-; X32-SSE41-NEXT:    sarl $31, %eax
-; X32-SSE41-NEXT:    pextrd $2, %xmm2, %ecx
-; X32-SSE41-NEXT:    pinsrd $1, %eax, %xmm2
-; X32-SSE41-NEXT:    sarl $31, %ecx
-; X32-SSE41-NEXT:    pinsrd $3, %ecx, %xmm2
-; X32-SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,2,3,3]
-; X32-SSE41-NEXT:    movd %xmm1, %eax
-; X32-SSE41-NEXT:    sarl $31, %eax
-; X32-SSE41-NEXT:    pextrd $2, %xmm1, %ecx
-; X32-SSE41-NEXT:    pinsrd $1, %eax, %xmm1
-; X32-SSE41-NEXT:    sarl $31, %ecx
-; X32-SSE41-NEXT:    pinsrd $3, %ecx, %xmm1
+; X32-SSE41-LABEL: sext_4i32_to_4i64:
+; X32-SSE41:       # BB#0: # %entry
+; X32-SSE41-NEXT:    pmovsxdq %xmm0, %xmm2
+; X32-SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
+; X32-SSE41-NEXT:    pmovsxdq %xmm0, %xmm1
 ; X32-SSE41-NEXT:    movdqa %xmm2, %xmm0
 ; X32-SSE41-NEXT:    retl
-  %extmask = sext <4 x i8> %mask to <4 x i64>
-  ret <4 x i64> %extmask
+entry:
+  %B = sext <4 x i32> %A to <4 x i64>
+  ret <4 x i64> %B
+}
+
+define <2 x i64> @load_sext_2i8_to_2i64(<2 x i8> *%ptr) {
+; SSE2-LABEL: load_sext_2i8_to_2i64:
+; SSE2:       # BB#0: # %entry
+; SSE2-NEXT:    movzwl (%rdi), %eax
+; SSE2-NEXT:    movd %eax, %xmm0
+; SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
+; SSE2-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3]
+; SSE2-NEXT:    movdqa %xmm0, %xmm1
+; SSE2-NEXT:    psrad $31, %xmm1
+; SSE2-NEXT:    psrad $24, %xmm0
+; SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
+; SSE2-NEXT:    retq
+;
+; SSSE3-LABEL: load_sext_2i8_to_2i64:
+; SSSE3:       # BB#0: # %entry
+; SSSE3-NEXT:    movzwl (%rdi), %eax
+; SSSE3-NEXT:    movd %eax, %xmm0
+; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
+; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3]
+; SSSE3-NEXT:    movdqa %xmm0, %xmm1
+; SSSE3-NEXT:    psrad $31, %xmm1
+; SSSE3-NEXT:    psrad $24, %xmm0
+; SSSE3-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
+; SSSE3-NEXT:    retq
+;
+; SSE41-LABEL: load_sext_2i8_to_2i64:
+; SSE41:       # BB#0: # %entry
+; SSE41-NEXT:    pmovsxbq (%rdi), %xmm0
+; SSE41-NEXT:    retq
+;
+; AVX-LABEL: load_sext_2i8_to_2i64:
+; AVX:       # BB#0: # %entry
+; AVX-NEXT:    vpmovsxbq (%rdi), %xmm0
+; AVX-NEXT:    retq
+;
+; X32-SSE41-LABEL: load_sext_2i8_to_2i64:
+; X32-SSE41:       # BB#0: # %entry
+; X32-SSE41-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X32-SSE41-NEXT:    pmovsxbq (%eax), %xmm0
+; X32-SSE41-NEXT:    retl
+entry:
+ %X = load <2 x i8>, <2 x i8>* %ptr
+ %Y = sext <2 x i8> %X to <2 x i64>
+ ret <2 x i64> %Y
+}
+
+define <4 x i32> @load_sext_4i8_to_4i32(<4 x i8> *%ptr) {
+; SSE2-LABEL: load_sext_4i8_to_4i32:
+; SSE2:       # BB#0: # %entry
+; SSE2-NEXT:    movd {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
+; SSE2-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3]
+; SSE2-NEXT:    psrad $24, %xmm0
+; SSE2-NEXT:    retq
+;
+; SSSE3-LABEL: load_sext_4i8_to_4i32:
+; SSSE3:       # BB#0: # %entry
+; SSSE3-NEXT:    movd {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
+; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3]
+; SSSE3-NEXT:    psrad $24, %xmm0
+; SSSE3-NEXT:    retq
+;
+; SSE41-LABEL: load_sext_4i8_to_4i32:
+; SSE41:       # BB#0: # %entry
+; SSE41-NEXT:    pmovsxbd (%rdi), %xmm0
+; SSE41-NEXT:    retq
+;
+; AVX-LABEL: load_sext_4i8_to_4i32:
+; AVX:       # BB#0: # %entry
+; AVX-NEXT:    vpmovsxbd (%rdi), %xmm0
+; AVX-NEXT:    retq
+;
+; X32-SSE41-LABEL: load_sext_4i8_to_4i32:
+; X32-SSE41:       # BB#0: # %entry
+; X32-SSE41-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X32-SSE41-NEXT:    pmovsxbd (%eax), %xmm0
+; X32-SSE41-NEXT:    retl
+entry:
+ %X = load <4 x i8>, <4 x i8>* %ptr
+ %Y = sext <4 x i8> %X to <4 x i32>
+ ret <4 x i32> %Y
 }
 
 define <4 x i64> @load_sext_4i8_to_4i64(<4 x i8> *%ptr) {
@@ -744,7 +734,229 @@ define <4 x i64> @load_sext_4i8_to_4i64(<4 x i8> *%ptr) {
 entry:
  %X = load <4 x i8>, <4 x i8>* %ptr
  %Y = sext <4 x i8> %X to <4 x i64>
- ret <4 x i64>%Y
+ ret <4 x i64> %Y
+}
+
+define <8 x i16> @load_sext_8i8_to_8i16(<8 x i8> *%ptr) {
+; SSE2-LABEL: load_sext_8i8_to_8i16:
+; SSE2:       # BB#0: # %entry
+; SSE2-NEXT:    movq {{.*#+}} xmm0 = mem[0],zero
+; SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
+; SSE2-NEXT:    psraw $8, %xmm0
+; SSE2-NEXT:    retq
+;
+; SSSE3-LABEL: load_sext_8i8_to_8i16:
+; SSSE3:       # BB#0: # %entry
+; SSSE3-NEXT:    movq {{.*#+}} xmm0 = mem[0],zero
+; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
+; SSSE3-NEXT:    psraw $8, %xmm0
+; SSSE3-NEXT:    retq
+;
+; SSE41-LABEL: load_sext_8i8_to_8i16:
+; SSE41:       # BB#0: # %entry
+; SSE41-NEXT:    pmovsxbw (%rdi), %xmm0
+; SSE41-NEXT:    retq
+;
+; AVX-LABEL: load_sext_8i8_to_8i16:
+; AVX:       # BB#0: # %entry
+; AVX-NEXT:    vpmovsxbw (%rdi), %xmm0
+; AVX-NEXT:    retq
+;
+; X32-SSE41-LABEL: load_sext_8i8_to_8i16:
+; X32-SSE41:       # BB#0: # %entry
+; X32-SSE41-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X32-SSE41-NEXT:    pmovsxbw (%eax), %xmm0
+; X32-SSE41-NEXT:    retl
+entry:
+ %X = load <8 x i8>, <8 x i8>* %ptr
+ %Y = sext <8 x i8> %X to <8 x i16>
+ ret <8 x i16> %Y
+}
+
+define <8 x i32> @load_sext_8i8_to_8i32(<8 x i8> *%ptr) {
+; SSE2-LABEL: load_sext_8i8_to_8i32:
+; SSE2:       # BB#0: # %entry
+; SSE2-NEXT:    movd {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
+; SSE2-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3]
+; SSE2-NEXT:    psrad $24, %xmm0
+; SSE2-NEXT:    movd {{.*#+}} xmm1 = mem[0],zero,zero,zero
+; SSE2-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
+; SSE2-NEXT:    punpcklwd {{.*#+}} xmm1 = xmm1[0,0,1,1,2,2,3,3]
+; SSE2-NEXT:    psrad $24, %xmm1
+; SSE2-NEXT:    retq
+;
+; SSSE3-LABEL: load_sext_8i8_to_8i32:
+; SSSE3:       # BB#0: # %entry
+; SSSE3-NEXT:    movd {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
+; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3]
+; SSSE3-NEXT:    psrad $24, %xmm0
+; SSSE3-NEXT:    movd {{.*#+}} xmm1 = mem[0],zero,zero,zero
+; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
+; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm1 = xmm1[0,0,1,1,2,2,3,3]
+; SSSE3-NEXT:    psrad $24, %xmm1
+; SSSE3-NEXT:    retq
+;
+; SSE41-LABEL: load_sext_8i8_to_8i32:
+; SSE41:       # BB#0: # %entry
+; SSE41-NEXT:    pmovsxbd (%rdi), %xmm0
+; SSE41-NEXT:    pmovsxbd 4(%rdi), %xmm1
+; SSE41-NEXT:    retq
+;
+; AVX1-LABEL: load_sext_8i8_to_8i32:
+; AVX1:       # BB#0: # %entry
+; AVX1-NEXT:    vpmovsxbw (%rdi), %xmm0
+; AVX1-NEXT:    vpmovsxwd %xmm0, %xmm1
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
+; AVX1-NEXT:    vpmovsxwd %xmm0, %xmm0
+; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: load_sext_8i8_to_8i32:
+; AVX2:       # BB#0: # %entry
+; AVX2-NEXT:    vpmovsxbd (%rdi), %ymm0
+; AVX2-NEXT:    retq
+;
+; X32-SSE41-LABEL: load_sext_8i8_to_8i32:
+; X32-SSE41:       # BB#0: # %entry
+; X32-SSE41-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X32-SSE41-NEXT:    pmovsxbd (%eax), %xmm0
+; X32-SSE41-NEXT:    pmovsxbd 4(%eax), %xmm1
+; X32-SSE41-NEXT:    retl
+entry:
+ %X = load <8 x i8>, <8 x i8>* %ptr
+ %Y = sext <8 x i8> %X to <8 x i32>
+ ret <8 x i32> %Y
+}
+
+define <16 x i16> @load_sext_16i8_to_16i16(<16 x i8> *%ptr) {
+; SSE2-LABEL: load_sext_16i8_to_16i16:
+; SSE2:       # BB#0: # %entry
+; SSE2-NEXT:    movq {{.*#+}} xmm0 = mem[0],zero
+; SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
+; SSE2-NEXT:    psraw $8, %xmm0
+; SSE2-NEXT:    movq {{.*#+}} xmm1 = mem[0],zero
+; SSE2-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
+; SSE2-NEXT:    psraw $8, %xmm1
+; SSE2-NEXT:    retq
+;
+; SSSE3-LABEL: load_sext_16i8_to_16i16:
+; SSSE3:       # BB#0: # %entry
+; SSSE3-NEXT:    movq {{.*#+}} xmm0 = mem[0],zero
+; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
+; SSSE3-NEXT:    psraw $8, %xmm0
+; SSSE3-NEXT:    movq {{.*#+}} xmm1 = mem[0],zero
+; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
+; SSSE3-NEXT:    psraw $8, %xmm1
+; SSSE3-NEXT:    retq
+;
+; SSE41-LABEL: load_sext_16i8_to_16i16:
+; SSE41:       # BB#0: # %entry
+; SSE41-NEXT:    pmovsxbw (%rdi), %xmm0
+; SSE41-NEXT:    pmovsxbw 8(%rdi), %xmm1
+; SSE41-NEXT:    retq
+;
+; AVX1-LABEL: load_sext_16i8_to_16i16:
+; AVX1:       # BB#0: # %entry
+; AVX1-NEXT:    vpmovsxbw (%rdi), %xmm0
+; AVX1-NEXT:    vpmovsxbw 8(%rdi), %xmm1
+; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: load_sext_16i8_to_16i16:
+; AVX2:       # BB#0: # %entry
+; AVX2-NEXT:    vpmovsxbw (%rdi), %ymm0
+; AVX2-NEXT:    retq
+;
+; X32-SSE41-LABEL: load_sext_16i8_to_16i16:
+; X32-SSE41:       # BB#0: # %entry
+; X32-SSE41-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X32-SSE41-NEXT:    pmovsxbw (%eax), %xmm0
+; X32-SSE41-NEXT:    pmovsxbw 8(%eax), %xmm1
+; X32-SSE41-NEXT:    retl
+entry:
+ %X = load <16 x i8>, <16 x i8>* %ptr
+ %Y = sext <16 x i8> %X to <16 x i16>
+ ret <16 x i16> %Y
+}
+
+define <2 x i64> @load_sext_2i16_to_2i64(<2 x i16> *%ptr) {
+; SSE2-LABEL: load_sext_2i16_to_2i64:
+; SSE2:       # BB#0: # %entry
+; SSE2-NEXT:    movd {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; SSE2-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3]
+; SSE2-NEXT:    movdqa %xmm0, %xmm1
+; SSE2-NEXT:    psrad $31, %xmm1
+; SSE2-NEXT:    psrad $16, %xmm0
+; SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
+; SSE2-NEXT:    retq
+;
+; SSSE3-LABEL: load_sext_2i16_to_2i64:
+; SSSE3:       # BB#0: # %entry
+; SSSE3-NEXT:    movd {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3]
+; SSSE3-NEXT:    movdqa %xmm0, %xmm1
+; SSSE3-NEXT:    psrad $31, %xmm1
+; SSSE3-NEXT:    psrad $16, %xmm0
+; SSSE3-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
+; SSSE3-NEXT:    retq
+;
+; SSE41-LABEL: load_sext_2i16_to_2i64:
+; SSE41:       # BB#0: # %entry
+; SSE41-NEXT:    pmovsxwq (%rdi), %xmm0
+; SSE41-NEXT:    retq
+;
+; AVX-LABEL: load_sext_2i16_to_2i64:
+; AVX:       # BB#0: # %entry
+; AVX-NEXT:    vpmovsxwq (%rdi), %xmm0
+; AVX-NEXT:    retq
+;
+; X32-SSE41-LABEL: load_sext_2i16_to_2i64:
+; X32-SSE41:       # BB#0: # %entry
+; X32-SSE41-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X32-SSE41-NEXT:    pmovsxwq (%eax), %xmm0
+; X32-SSE41-NEXT:    retl
+entry:
+ %X = load <2 x i16>, <2 x i16>* %ptr
+ %Y = sext <2 x i16> %X to <2 x i64>
+ ret <2 x i64> %Y
+}
+
+define <4 x i32> @load_sext_4i16_to_4i32(<4 x i16> *%ptr) {
+; SSE2-LABEL: load_sext_4i16_to_4i32:
+; SSE2:       # BB#0: # %entry
+; SSE2-NEXT:    movq {{.*#+}} xmm0 = mem[0],zero
+; SSE2-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3]
+; SSE2-NEXT:    psrad $16, %xmm0
+; SSE2-NEXT:    retq
+;
+; SSSE3-LABEL: load_sext_4i16_to_4i32:
+; SSSE3:       # BB#0: # %entry
+; SSSE3-NEXT:    movq {{.*#+}} xmm0 = mem[0],zero
+; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3]
+; SSSE3-NEXT:    psrad $16, %xmm0
+; SSSE3-NEXT:    retq
+;
+; SSE41-LABEL: load_sext_4i16_to_4i32:
+; SSE41:       # BB#0: # %entry
+; SSE41-NEXT:    pmovsxwd (%rdi), %xmm0
+; SSE41-NEXT:    retq
+;
+; AVX-LABEL: load_sext_4i16_to_4i32:
+; AVX:       # BB#0: # %entry
+; AVX-NEXT:    vpmovsxwd (%rdi), %xmm0
+; AVX-NEXT:    retq
+;
+; X32-SSE41-LABEL: load_sext_4i16_to_4i32:
+; X32-SSE41:       # BB#0: # %entry
+; X32-SSE41-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X32-SSE41-NEXT:    pmovsxwd (%eax), %xmm0
+; X32-SSE41-NEXT:    retl
+entry:
+ %X = load <4 x i16>, <4 x i16>* %ptr
+ %Y = sext <4 x i16> %X to <4 x i32>
+ ret <4 x i32> %Y
 }
 
 define <4 x i64> @load_sext_4i16_to_4i64(<4 x i16> *%ptr) {
@@ -805,5 +1017,326 @@ define <4 x i64> @load_sext_4i16_to_4i64(<4 x i16> *%ptr) {
 entry:
  %X = load <4 x i16>, <4 x i16>* %ptr
  %Y = sext <4 x i16> %X to <4 x i64>
- ret <4 x i64>%Y
+ ret <4 x i64> %Y
+}
+
+define <8 x i32> @load_sext_8i16_to_8i32(<8 x i16> *%ptr) {
+; SSE2-LABEL: load_sext_8i16_to_8i32:
+; SSE2:       # BB#0: # %entry
+; SSE2-NEXT:    movq {{.*#+}} xmm0 = mem[0],zero
+; SSE2-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3]
+; SSE2-NEXT:    psrad $16, %xmm0
+; SSE2-NEXT:    movq {{.*#+}} xmm1 = mem[0],zero
+; SSE2-NEXT:    punpcklwd {{.*#+}} xmm1 = xmm1[0,0,1,1,2,2,3,3]
+; SSE2-NEXT:    psrad $16, %xmm1
+; SSE2-NEXT:    retq
+;
+; SSSE3-LABEL: load_sext_8i16_to_8i32:
+; SSSE3:       # BB#0: # %entry
+; SSSE3-NEXT:    movq {{.*#+}} xmm0 = mem[0],zero
+; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3]
+; SSSE3-NEXT:    psrad $16, %xmm0
+; SSSE3-NEXT:    movq {{.*#+}} xmm1 = mem[0],zero
+; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm1 = xmm1[0,0,1,1,2,2,3,3]
+; SSSE3-NEXT:    psrad $16, %xmm1
+; SSSE3-NEXT:    retq
+;
+; SSE41-LABEL: load_sext_8i16_to_8i32:
+; SSE41:       # BB#0: # %entry
+; SSE41-NEXT:    pmovsxwd (%rdi), %xmm0
+; SSE41-NEXT:    pmovsxwd 8(%rdi), %xmm1
+; SSE41-NEXT:    retq
+;
+; AVX1-LABEL: load_sext_8i16_to_8i32:
+; AVX1:       # BB#0: # %entry
+; AVX1-NEXT:    vpmovsxwd (%rdi), %xmm0
+; AVX1-NEXT:    vpmovsxwd 8(%rdi), %xmm1
+; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: load_sext_8i16_to_8i32:
+; AVX2:       # BB#0: # %entry
+; AVX2-NEXT:    vpmovsxwd (%rdi), %ymm0
+; AVX2-NEXT:    retq
+;
+; X32-SSE41-LABEL: load_sext_8i16_to_8i32:
+; X32-SSE41:       # BB#0: # %entry
+; X32-SSE41-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X32-SSE41-NEXT:    pmovsxwd (%eax), %xmm0
+; X32-SSE41-NEXT:    pmovsxwd 8(%eax), %xmm1
+; X32-SSE41-NEXT:    retl
+entry:
+ %X = load <8 x i16>, <8 x i16>* %ptr
+ %Y = sext <8 x i16> %X to <8 x i32>
+ ret <8 x i32> %Y
+}
+
+define <2 x i64> @load_sext_2i32_to_2i64(<2 x i32> *%ptr) {
+; SSE2-LABEL: load_sext_2i32_to_2i64:
+; SSE2:       # BB#0: # %entry
+; SSE2-NEXT:    movq {{.*#+}} xmm0 = mem[0],zero
+; SSE2-NEXT:    movdqa %xmm0, %xmm1
+; SSE2-NEXT:    psrad $31, %xmm1
+; SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
+; SSE2-NEXT:    retq
+;
+; SSSE3-LABEL: load_sext_2i32_to_2i64:
+; SSSE3:       # BB#0: # %entry
+; SSSE3-NEXT:    movq {{.*#+}} xmm0 = mem[0],zero
+; SSSE3-NEXT:    movdqa %xmm0, %xmm1
+; SSSE3-NEXT:    psrad $31, %xmm1
+; SSSE3-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
+; SSSE3-NEXT:    retq
+;
+; SSE41-LABEL: load_sext_2i32_to_2i64:
+; SSE41:       # BB#0: # %entry
+; SSE41-NEXT:    pmovsxdq (%rdi), %xmm0
+; SSE41-NEXT:    retq
+;
+; AVX-LABEL: load_sext_2i32_to_2i64:
+; AVX:       # BB#0: # %entry
+; AVX-NEXT:    vpmovsxdq (%rdi), %xmm0
+; AVX-NEXT:    retq
+;
+; X32-SSE41-LABEL: load_sext_2i32_to_2i64:
+; X32-SSE41:       # BB#0: # %entry
+; X32-SSE41-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X32-SSE41-NEXT:    pmovsxdq (%eax), %xmm0
+; X32-SSE41-NEXT:    retl
+entry:
+ %X = load <2 x i32>, <2 x i32>* %ptr
+ %Y = sext <2 x i32> %X to <2 x i64>
+ ret <2 x i64> %Y
+}
+
+define <4 x i64> @load_sext_4i32_to_4i64(<4 x i32> *%ptr) {
+; SSE2-LABEL: load_sext_4i32_to_4i64:
+; SSE2:       # BB#0: # %entry
+; SSE2-NEXT:    movdqa (%rdi), %xmm0
+; SSE2-NEXT:    movdqa %xmm0, %xmm2
+; SSE2-NEXT:    psrad $31, %xmm2
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1]
+; SSE2-NEXT:    movdqa %xmm1, %xmm2
+; SSE2-NEXT:    psrad $31, %xmm2
+; SSE2-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1]
+; SSE2-NEXT:    retq
+;
+; SSSE3-LABEL: load_sext_4i32_to_4i64:
+; SSSE3:       # BB#0: # %entry
+; SSSE3-NEXT:    movdqa (%rdi), %xmm0
+; SSSE3-NEXT:    movdqa %xmm0, %xmm2
+; SSSE3-NEXT:    psrad $31, %xmm2
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; SSSE3-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1]
+; SSSE3-NEXT:    movdqa %xmm1, %xmm2
+; SSSE3-NEXT:    psrad $31, %xmm2
+; SSSE3-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1]
+; SSSE3-NEXT:    retq
+;
+; SSE41-LABEL: load_sext_4i32_to_4i64:
+; SSE41:       # BB#0: # %entry
+; SSE41-NEXT:    pmovsxdq (%rdi), %xmm0
+; SSE41-NEXT:    pmovsxdq 8(%rdi), %xmm1
+; SSE41-NEXT:    retq
+;
+; AVX1-LABEL: load_sext_4i32_to_4i64:
+; AVX1:       # BB#0: # %entry
+; AVX1-NEXT:    vpmovsxdq (%rdi), %xmm0
+; AVX1-NEXT:    vpmovsxdq 8(%rdi), %xmm1
+; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: load_sext_4i32_to_4i64:
+; AVX2:       # BB#0: # %entry
+; AVX2-NEXT:    vpmovsxdq (%rdi), %ymm0
+; AVX2-NEXT:    retq
+;
+; X32-SSE41-LABEL: load_sext_4i32_to_4i64:
+; X32-SSE41:       # BB#0: # %entry
+; X32-SSE41-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X32-SSE41-NEXT:    pmovsxdq (%eax), %xmm0
+; X32-SSE41-NEXT:    pmovsxdq 8(%eax), %xmm1
+; X32-SSE41-NEXT:    retl
+entry:
+ %X = load <4 x i32>, <4 x i32>* %ptr
+ %Y = sext <4 x i32> %X to <4 x i64>
+ ret <4 x i64> %Y
+}
+
+define i32 @sext_2i8_to_i32(<16 x i8> %A) nounwind uwtable readnone ssp {
+; SSE2-LABEL: sext_2i8_to_i32:
+; SSE2:       # BB#0: # %entry
+; SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
+; SSE2-NEXT:    psraw $8, %xmm0
+; SSE2-NEXT:    movd %xmm0, %eax
+; SSE2-NEXT:    retq
+;
+; SSSE3-LABEL: sext_2i8_to_i32:
+; SSSE3:       # BB#0: # %entry
+; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
+; SSSE3-NEXT:    psraw $8, %xmm0
+; SSSE3-NEXT:    movd %xmm0, %eax
+; SSSE3-NEXT:    retq
+;
+; SSE41-LABEL: sext_2i8_to_i32:
+; SSE41:       # BB#0: # %entry
+; SSE41-NEXT:    pmovsxbw %xmm0, %xmm0
+; SSE41-NEXT:    movd %xmm0, %eax
+; SSE41-NEXT:    retq
+;
+; AVX-LABEL: sext_2i8_to_i32:
+; AVX:       # BB#0: # %entry
+; AVX-NEXT:    vpmovsxbw %xmm0, %xmm0
+; AVX-NEXT:    vmovd %xmm0, %eax
+; AVX-NEXT:    retq
+;
+; X32-SSE41-LABEL: sext_2i8_to_i32:
+; X32-SSE41:       # BB#0: # %entry
+; X32-SSE41-NEXT:    pushl %eax
+; X32-SSE41-NEXT:  .Ltmp0:
+; X32-SSE41-NEXT:    .cfi_def_cfa_offset 8
+; X32-SSE41-NEXT:    pmovsxbw %xmm0, %xmm0
+; X32-SSE41-NEXT:    movd %xmm0, %eax
+; X32-SSE41-NEXT:    popl %edx
+; X32-SSE41-NEXT:    retl
+entry:
+  %Shuf = shufflevector <16 x i8> %A, <16 x i8> undef, <2 x i32> <i32 0, i32 1>
+  %Ex = sext <2 x i8> %Shuf to <2 x i16>
+  %Bc = bitcast <2 x i16> %Ex to i32
+  ret i32 %Bc
+}
+
+define <4 x i64> @sext_4i1_to_4i64(<4 x i1> %mask) {
+; SSE2-LABEL: sext_4i1_to_4i64:
+; SSE2:       # BB#0:
+; SSE2-NEXT:    pslld $31, %xmm0
+; SSE2-NEXT:    psrad $31, %xmm0
+; SSE2-NEXT:    movdqa %xmm0, %xmm2
+; SSE2-NEXT:    psrad $31, %xmm2
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1]
+; SSE2-NEXT:    movdqa %xmm1, %xmm2
+; SSE2-NEXT:    psrad $31, %xmm2
+; SSE2-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1]
+; SSE2-NEXT:    retq
+;
+; SSSE3-LABEL: sext_4i1_to_4i64:
+; SSSE3:       # BB#0:
+; SSSE3-NEXT:    pslld $31, %xmm0
+; SSSE3-NEXT:    psrad $31, %xmm0
+; SSSE3-NEXT:    movdqa %xmm0, %xmm2
+; SSSE3-NEXT:    psrad $31, %xmm2
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; SSSE3-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1]
+; SSSE3-NEXT:    movdqa %xmm1, %xmm2
+; SSSE3-NEXT:    psrad $31, %xmm2
+; SSSE3-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1]
+; SSSE3-NEXT:    retq
+;
+; SSE41-LABEL: sext_4i1_to_4i64:
+; SSE41:       # BB#0:
+; SSE41-NEXT:    pslld $31, %xmm0
+; SSE41-NEXT:    psrad $31, %xmm0
+; SSE41-NEXT:    pmovsxdq %xmm0, %xmm2
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
+; SSE41-NEXT:    pmovsxdq %xmm0, %xmm1
+; SSE41-NEXT:    movdqa %xmm2, %xmm0
+; SSE41-NEXT:    retq
+;
+; AVX1-LABEL: sext_4i1_to_4i64:
+; AVX1:       # BB#0:
+; AVX1-NEXT:    vpslld $31, %xmm0, %xmm0
+; AVX1-NEXT:    vpsrad $31, %xmm0, %xmm0
+; AVX1-NEXT:    vpmovsxdq %xmm0, %xmm1
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
+; AVX1-NEXT:    vpmovsxdq %xmm0, %xmm0
+; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: sext_4i1_to_4i64:
+; AVX2:       # BB#0:
+; AVX2-NEXT:    vpslld $31, %xmm0, %xmm0
+; AVX2-NEXT:    vpsrad $31, %xmm0, %xmm0
+; AVX2-NEXT:    vpmovsxdq %xmm0, %ymm0
+; AVX2-NEXT:    retq
+;
+; X32-SSE41-LABEL: sext_4i1_to_4i64:
+; X32-SSE41:       # BB#0:
+; X32-SSE41-NEXT:    pslld $31, %xmm0
+; X32-SSE41-NEXT:    psrad $31, %xmm0
+; X32-SSE41-NEXT:    pmovsxdq %xmm0, %xmm2
+; X32-SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
+; X32-SSE41-NEXT:    pmovsxdq %xmm0, %xmm1
+; X32-SSE41-NEXT:    movdqa %xmm2, %xmm0
+; X32-SSE41-NEXT:    retl
+  %extmask = sext <4 x i1> %mask to <4 x i64>
+  ret <4 x i64> %extmask
+}
+
+define <4 x i64> @sext_4i8_to_4i64(<4 x i8> %mask) {
+; SSE2-LABEL: sext_4i8_to_4i64:
+; SSE2:       # BB#0:
+; SSE2-NEXT:    pslld $24, %xmm0
+; SSE2-NEXT:    psrad $24, %xmm0
+; SSE2-NEXT:    movdqa %xmm0, %xmm2
+; SSE2-NEXT:    psrad $31, %xmm2
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1]
+; SSE2-NEXT:    movdqa %xmm1, %xmm2
+; SSE2-NEXT:    psrad $31, %xmm2
+; SSE2-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1]
+; SSE2-NEXT:    retq
+;
+; SSSE3-LABEL: sext_4i8_to_4i64:
+; SSSE3:       # BB#0:
+; SSSE3-NEXT:    pslld $24, %xmm0
+; SSSE3-NEXT:    psrad $24, %xmm0
+; SSSE3-NEXT:    movdqa %xmm0, %xmm2
+; SSSE3-NEXT:    psrad $31, %xmm2
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; SSSE3-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1]
+; SSSE3-NEXT:    movdqa %xmm1, %xmm2
+; SSSE3-NEXT:    psrad $31, %xmm2
+; SSSE3-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1]
+; SSSE3-NEXT:    retq
+;
+; SSE41-LABEL: sext_4i8_to_4i64:
+; SSE41:       # BB#0:
+; SSE41-NEXT:    pslld $24, %xmm0
+; SSE41-NEXT:    psrad $24, %xmm0
+; SSE41-NEXT:    pmovsxdq %xmm0, %xmm2
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
+; SSE41-NEXT:    pmovsxdq %xmm0, %xmm1
+; SSE41-NEXT:    movdqa %xmm2, %xmm0
+; SSE41-NEXT:    retq
+;
+; AVX1-LABEL: sext_4i8_to_4i64:
+; AVX1:       # BB#0:
+; AVX1-NEXT:    vpslld $24, %xmm0, %xmm0
+; AVX1-NEXT:    vpsrad $24, %xmm0, %xmm0
+; AVX1-NEXT:    vpmovsxdq %xmm0, %xmm1
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
+; AVX1-NEXT:    vpmovsxdq %xmm0, %xmm0
+; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: sext_4i8_to_4i64:
+; AVX2:       # BB#0:
+; AVX2-NEXT:    vpslld $24, %xmm0, %xmm0
+; AVX2-NEXT:    vpsrad $24, %xmm0, %xmm0
+; AVX2-NEXT:    vpmovsxdq %xmm0, %ymm0
+; AVX2-NEXT:    retq
+;
+; X32-SSE41-LABEL: sext_4i8_to_4i64:
+; X32-SSE41:       # BB#0:
+; X32-SSE41-NEXT:    pslld $24, %xmm0
+; X32-SSE41-NEXT:    psrad $24, %xmm0
+; X32-SSE41-NEXT:    pmovsxdq %xmm0, %xmm2
+; X32-SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
+; X32-SSE41-NEXT:    pmovsxdq %xmm0, %xmm1
+; X32-SSE41-NEXT:    movdqa %xmm2, %xmm0
+; X32-SSE41-NEXT:    retl
+  %extmask = sext <4 x i8> %mask to <4 x i64>
+  ret <4 x i64> %extmask
 }
diff --git a/test/CodeGen/X86/vector-shift-ashr-128.ll b/test/CodeGen/X86/vector-shift-ashr-128.ll
new file mode 100644
index 00000000000..64d4ea03762
--- /dev/null
+++ b/test/CodeGen/X86/vector-shift-ashr-128.ll
@@ -0,0 +1,1354 @@
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mcpu=x86-64 | FileCheck %s --check-prefix=ALL --check-prefix=SSE --check-prefix=SSE2
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mcpu=x86-64 -mattr=+sse4.1 | FileCheck %s --check-prefix=ALL --check-prefix=SSE --check-prefix=SSE41
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mcpu=x86-64 -mattr=+avx | FileCheck %s --check-prefix=ALL --check-prefix=AVX --check-prefix=AVX1
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mcpu=x86-64 -mattr=+avx2 | FileCheck %s --check-prefix=ALL --check-prefix=AVX --check-prefix=AVX2
+;
+; Just one 32-bit run to make sure we do reasonable things for i64 shifts.
+; RUN: llc < %s -mtriple=i686-unknown-unknown -mcpu=x86-64 | FileCheck %s --check-prefix=ALL --check-prefix=X32-SSE --check-prefix=X32-SSE2
+
+;
+; Variable Shifts
+;
+
+define <2 x i64> @var_shift_v2i64(<2 x i64> %a, <2 x i64> %b) nounwind {
+; SSE2-LABEL: var_shift_v2i64:
+; SSE2:       # BB#0:
+; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm1[2,3,0,1]
+; SSE2-NEXT:    movdqa {{.*#+}} xmm2 = [9223372036854775808,9223372036854775808]
+; SSE2-NEXT:    movdqa %xmm2, %xmm4
+; SSE2-NEXT:    psrlq %xmm3, %xmm4
+; SSE2-NEXT:    psrlq %xmm1, %xmm2
+; SSE2-NEXT:    movsd {{.*#+}} xmm4 = xmm2[0],xmm4[1]
+; SSE2-NEXT:    movdqa %xmm0, %xmm2
+; SSE2-NEXT:    psrlq %xmm3, %xmm2
+; SSE2-NEXT:    psrlq %xmm1, %xmm0
+; SSE2-NEXT:    movsd {{.*#+}} xmm2 = xmm0[0],xmm2[1]
+; SSE2-NEXT:    xorpd %xmm4, %xmm2
+; SSE2-NEXT:    psubq %xmm4, %xmm2
+; SSE2-NEXT:    movdqa %xmm2, %xmm0
+; SSE2-NEXT:    retq
+;
+; SSE41-LABEL: var_shift_v2i64:
+; SSE41:       # BB#0:
+; SSE41-NEXT:    movdqa {{.*#+}} xmm2 = [9223372036854775808,9223372036854775808]
+; SSE41-NEXT:    movdqa %xmm2, %xmm3
+; SSE41-NEXT:    psrlq %xmm1, %xmm3
+; SSE41-NEXT:    pshufd {{.*#+}} xmm4 = xmm1[2,3,0,1]
+; SSE41-NEXT:    psrlq %xmm4, %xmm2
+; SSE41-NEXT:    pblendw {{.*#+}} xmm2 = xmm3[0,1,2,3],xmm2[4,5,6,7]
+; SSE41-NEXT:    movdqa %xmm0, %xmm3
+; SSE41-NEXT:    psrlq %xmm1, %xmm3
+; SSE41-NEXT:    psrlq %xmm4, %xmm0
+; SSE41-NEXT:    pblendw {{.*#+}} xmm0 = xmm3[0,1,2,3],xmm0[4,5,6,7]
+; SSE41-NEXT:    pxor %xmm2, %xmm0
+; SSE41-NEXT:    psubq %xmm2, %xmm0
+; SSE41-NEXT:    retq
+;
+; AVX1-LABEL: var_shift_v2i64:
+; AVX1:       # BB#0:
+; AVX1-NEXT:    vmovdqa {{.*#+}} xmm2 = [9223372036854775808,9223372036854775808]
+; AVX1-NEXT:    vpsrlq %xmm1, %xmm2, %xmm3
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm4 = xmm1[2,3,0,1]
+; AVX1-NEXT:    vpsrlq %xmm4, %xmm2, %xmm2
+; AVX1-NEXT:    vpblendw {{.*#+}} xmm2 = xmm3[0,1,2,3],xmm2[4,5,6,7]
+; AVX1-NEXT:    vpsrlq %xmm1, %xmm0, %xmm1
+; AVX1-NEXT:    vpsrlq %xmm4, %xmm0, %xmm0
+; AVX1-NEXT:    vpblendw {{.*#+}} xmm0 = xmm1[0,1,2,3],xmm0[4,5,6,7]
+; AVX1-NEXT:    vpxor %xmm2, %xmm0, %xmm0
+; AVX1-NEXT:    vpsubq %xmm2, %xmm0, %xmm0
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: var_shift_v2i64:
+; AVX2:       # BB#0:
+; AVX2-NEXT:    vmovdqa {{.*#+}} xmm2 = [9223372036854775808,9223372036854775808]
+; AVX2-NEXT:    vpsrlvq %xmm1, %xmm2, %xmm3
+; AVX2-NEXT:    vpxor %xmm2, %xmm0, %xmm0
+; AVX2-NEXT:    vpsrlvq %xmm1, %xmm0, %xmm0
+; AVX2-NEXT:    vpsubq %xmm3, %xmm0, %xmm0
+; AVX2-NEXT:    retq
+;
+; X32-SSE-LABEL: var_shift_v2i64:
+; X32-SSE:       # BB#0:
+; X32-SSE-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[2,3,0,1]
+; X32-SSE-NEXT:    movdqa {{.*#+}} xmm3 = [0,2147483648,0,2147483648]
+; X32-SSE-NEXT:    movdqa %xmm3, %xmm4
+; X32-SSE-NEXT:    psrlq %xmm2, %xmm4
+; X32-SSE-NEXT:    movq {{.*#+}} xmm5 = xmm1[0],zero
+; X32-SSE-NEXT:    psrlq %xmm5, %xmm3
+; X32-SSE-NEXT:    movsd {{.*#+}} xmm4 = xmm3[0],xmm4[1]
+; X32-SSE-NEXT:    movdqa %xmm0, %xmm1
+; X32-SSE-NEXT:    psrlq %xmm2, %xmm1
+; X32-SSE-NEXT:    psrlq %xmm5, %xmm0
+; X32-SSE-NEXT:    movsd {{.*#+}} xmm1 = xmm0[0],xmm1[1]
+; X32-SSE-NEXT:    xorpd %xmm4, %xmm1
+; X32-SSE-NEXT:    psubq %xmm4, %xmm1
+; X32-SSE-NEXT:    movdqa %xmm1, %xmm0
+; X32-SSE-NEXT:    retl
+  %shift = ashr <2 x i64> %a, %b
+  ret <2 x i64> %shift
+}
+
+define <4 x i32> @var_shift_v4i32(<4 x i32> %a, <4 x i32> %b) nounwind {
+; SSE2-LABEL: var_shift_v4i32:
+; SSE2:       # BB#0:
+; SSE2-NEXT:    movdqa %xmm1, %xmm2
+; SSE2-NEXT:    psrldq {{.*#+}} xmm2 = xmm2[12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
+; SSE2-NEXT:    movdqa %xmm0, %xmm3
+; SSE2-NEXT:    psrad %xmm2, %xmm3
+; SSE2-NEXT:    movdqa %xmm1, %xmm2
+; SSE2-NEXT:    psrlq $32, %xmm2
+; SSE2-NEXT:    movdqa %xmm0, %xmm4
+; SSE2-NEXT:    psrad %xmm2, %xmm4
+; SSE2-NEXT:    movsd {{.*#+}} xmm3 = xmm4[0],xmm3[1]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm3[1,3,2,3]
+; SSE2-NEXT:    pxor %xmm3, %xmm3
+; SSE2-NEXT:    movdqa %xmm1, %xmm4
+; SSE2-NEXT:    punpckhdq {{.*#+}} xmm4 = xmm4[2],xmm3[2],xmm4[3],xmm3[3]
+; SSE2-NEXT:    movdqa %xmm0, %xmm5
+; SSE2-NEXT:    psrad %xmm4, %xmm5
+; SSE2-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm3[0],xmm1[1],xmm3[1]
+; SSE2-NEXT:    psrad %xmm1, %xmm0
+; SSE2-NEXT:    movsd {{.*#+}} xmm5 = xmm0[0],xmm5[1]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm5[0,2,2,3]
+; SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1]
+; SSE2-NEXT:    retq
+;
+; SSE41-LABEL: var_shift_v4i32:
+; SSE41:       # BB#0:
+; SSE41-NEXT:    movdqa %xmm1, %xmm2
+; SSE41-NEXT:    psrldq {{.*#+}} xmm2 = xmm2[12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
+; SSE41-NEXT:    movdqa %xmm0, %xmm3
+; SSE41-NEXT:    psrad %xmm2, %xmm3
+; SSE41-NEXT:    movdqa %xmm1, %xmm2
+; SSE41-NEXT:    psrlq $32, %xmm2
+; SSE41-NEXT:    movdqa %xmm0, %xmm4
+; SSE41-NEXT:    psrad %xmm2, %xmm4
+; SSE41-NEXT:    pblendw {{.*#+}} xmm4 = xmm4[0,1,2,3],xmm3[4,5,6,7]
+; SSE41-NEXT:    pxor %xmm2, %xmm2
+; SSE41-NEXT:    pmovzxdq {{.*#+}} xmm3 = xmm1[0],zero,xmm1[1],zero
+; SSE41-NEXT:    punpckhdq {{.*#+}} xmm1 = xmm1[2],xmm2[2],xmm1[3],xmm2[3]
+; SSE41-NEXT:    movdqa %xmm0, %xmm2
+; SSE41-NEXT:    psrad %xmm1, %xmm2
+; SSE41-NEXT:    psrad %xmm3, %xmm0
+; SSE41-NEXT:    pblendw {{.*#+}} xmm0 = xmm0[0,1,2,3],xmm2[4,5,6,7]
+; SSE41-NEXT:    pblendw {{.*#+}} xmm0 = xmm0[0,1],xmm4[2,3],xmm0[4,5],xmm4[6,7]
+; SSE41-NEXT:    retq
+;
+; AVX1-LABEL: var_shift_v4i32:
+; AVX1:       # BB#0:
+; AVX1-NEXT:    vpsrldq {{.*#+}} xmm2 = xmm1[12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
+; AVX1-NEXT:    vpsrad %xmm2, %xmm0, %xmm2
+; AVX1-NEXT:    vpsrlq $32, %xmm1, %xmm3
+; AVX1-NEXT:    vpsrad %xmm3, %xmm0, %xmm3
+; AVX1-NEXT:    vpblendw {{.*#+}} xmm2 = xmm3[0,1,2,3],xmm2[4,5,6,7]
+; AVX1-NEXT:    vpxor %xmm3, %xmm3, %xmm3
+; AVX1-NEXT:    vpunpckhdq {{.*#+}} xmm3 = xmm1[2],xmm3[2],xmm1[3],xmm3[3]
+; AVX1-NEXT:    vpsrad %xmm3, %xmm0, %xmm3
+; AVX1-NEXT:    vpmovzxdq {{.*#+}} xmm1 = xmm1[0],zero,xmm1[1],zero
+; AVX1-NEXT:    vpsrad %xmm1, %xmm0, %xmm0
+; AVX1-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0,1,2,3],xmm3[4,5,6,7]
+; AVX1-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0,1],xmm2[2,3],xmm0[4,5],xmm2[6,7]
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: var_shift_v4i32:
+; AVX2:       # BB#0:
+; AVX2-NEXT:    vpsravd %xmm1, %xmm0, %xmm0
+; AVX2-NEXT:    retq
+;
+; X32-SSE-LABEL: var_shift_v4i32:
+; X32-SSE:       # BB#0:
+; X32-SSE-NEXT:    movdqa %xmm1, %xmm2
+; X32-SSE-NEXT:    psrldq {{.*#+}} xmm2 = xmm2[12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
+; X32-SSE-NEXT:    movdqa %xmm0, %xmm3
+; X32-SSE-NEXT:    psrad %xmm2, %xmm3
+; X32-SSE-NEXT:    movdqa %xmm1, %xmm2
+; X32-SSE-NEXT:    psrlq $32, %xmm2
+; X32-SSE-NEXT:    movdqa %xmm0, %xmm4
+; X32-SSE-NEXT:    psrad %xmm2, %xmm4
+; X32-SSE-NEXT:    movsd {{.*#+}} xmm3 = xmm4[0],xmm3[1]
+; X32-SSE-NEXT:    pshufd {{.*#+}} xmm2 = xmm3[1,3,2,3]
+; X32-SSE-NEXT:    pxor %xmm3, %xmm3
+; X32-SSE-NEXT:    movdqa %xmm1, %xmm4
+; X32-SSE-NEXT:    punpckhdq {{.*#+}} xmm4 = xmm4[2],xmm3[2],xmm4[3],xmm3[3]
+; X32-SSE-NEXT:    movdqa %xmm0, %xmm5
+; X32-SSE-NEXT:    psrad %xmm4, %xmm5
+; X32-SSE-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm3[0],xmm1[1],xmm3[1]
+; X32-SSE-NEXT:    psrad %xmm1, %xmm0
+; X32-SSE-NEXT:    movsd {{.*#+}} xmm5 = xmm0[0],xmm5[1]
+; X32-SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm5[0,2,2,3]
+; X32-SSE-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1]
+; X32-SSE-NEXT:    retl
+  %shift = ashr <4 x i32> %a, %b
+  ret <4 x i32> %shift
+}
+
+define <8 x i16> @var_shift_v8i16(<8 x i16> %a, <8 x i16> %b) nounwind {
+; SSE2-LABEL: var_shift_v8i16:
+; SSE2:       # BB#0:
+; SSE2-NEXT:    psllw $12, %xmm1
+; SSE2-NEXT:    movdqa %xmm1, %xmm2
+; SSE2-NEXT:    psraw $15, %xmm2
+; SSE2-NEXT:    movdqa %xmm2, %xmm3
+; SSE2-NEXT:    pandn %xmm0, %xmm3
+; SSE2-NEXT:    psraw $8, %xmm0
+; SSE2-NEXT:    pand %xmm2, %xmm0
+; SSE2-NEXT:    por %xmm3, %xmm0
+; SSE2-NEXT:    paddw %xmm1, %xmm1
+; SSE2-NEXT:    movdqa %xmm1, %xmm2
+; SSE2-NEXT:    psraw $15, %xmm2
+; SSE2-NEXT:    movdqa %xmm2, %xmm3
+; SSE2-NEXT:    pandn %xmm0, %xmm3
+; SSE2-NEXT:    psraw $4, %xmm0
+; SSE2-NEXT:    pand %xmm2, %xmm0
+; SSE2-NEXT:    por %xmm3, %xmm0
+; SSE2-NEXT:    paddw %xmm1, %xmm1
+; SSE2-NEXT:    movdqa %xmm1, %xmm2
+; SSE2-NEXT:    psraw $15, %xmm2
+; SSE2-NEXT:    movdqa %xmm2, %xmm3
+; SSE2-NEXT:    pandn %xmm0, %xmm3
+; SSE2-NEXT:    psraw $2, %xmm0
+; SSE2-NEXT:    pand %xmm2, %xmm0
+; SSE2-NEXT:    por %xmm3, %xmm0
+; SSE2-NEXT:    paddw %xmm1, %xmm1
+; SSE2-NEXT:    psraw $15, %xmm1
+; SSE2-NEXT:    movdqa %xmm1, %xmm2
+; SSE2-NEXT:    pandn %xmm0, %xmm2
+; SSE2-NEXT:    psraw $1, %xmm0
+; SSE2-NEXT:    pand %xmm1, %xmm0
+; SSE2-NEXT:    por %xmm2, %xmm0
+; SSE2-NEXT:    retq
+;
+; SSE41-LABEL: var_shift_v8i16:
+; SSE41:       # BB#0:
+; SSE41-NEXT:    movdqa %xmm0, %xmm2
+; SSE41-NEXT:    movdqa %xmm1, %xmm0
+; SSE41-NEXT:    psllw $12, %xmm0
+; SSE41-NEXT:    psllw $4, %xmm1
+; SSE41-NEXT:    por %xmm0, %xmm1
+; SSE41-NEXT:    movdqa %xmm1, %xmm3
+; SSE41-NEXT:    paddw %xmm3, %xmm3
+; SSE41-NEXT:    movdqa %xmm2, %xmm4
+; SSE41-NEXT:    psraw $8, %xmm4
+; SSE41-NEXT:    movdqa %xmm1, %xmm0
+; SSE41-NEXT:    pblendvb %xmm4, %xmm2
+; SSE41-NEXT:    movdqa %xmm2, %xmm1
+; SSE41-NEXT:    psraw $4, %xmm1
+; SSE41-NEXT:    movdqa %xmm3, %xmm0
+; SSE41-NEXT:    pblendvb %xmm1, %xmm2
+; SSE41-NEXT:    movdqa %xmm2, %xmm1
+; SSE41-NEXT:    psraw $2, %xmm1
+; SSE41-NEXT:    paddw %xmm3, %xmm3
+; SSE41-NEXT:    movdqa %xmm3, %xmm0
+; SSE41-NEXT:    pblendvb %xmm1, %xmm2
+; SSE41-NEXT:    movdqa %xmm2, %xmm1
+; SSE41-NEXT:    psraw $1, %xmm1
+; SSE41-NEXT:    paddw %xmm3, %xmm3
+; SSE41-NEXT:    movdqa %xmm3, %xmm0
+; SSE41-NEXT:    pblendvb %xmm1, %xmm2
+; SSE41-NEXT:    movdqa %xmm2, %xmm0
+; SSE41-NEXT:    retq
+;
+; AVX1-LABEL: var_shift_v8i16:
+; AVX1:       # BB#0:
+; AVX1-NEXT:    vpsllw $12, %xmm1, %xmm2
+; AVX1-NEXT:    vpsllw $4, %xmm1, %xmm1
+; AVX1-NEXT:    vpor %xmm2, %xmm1, %xmm1
+; AVX1-NEXT:    vpaddw %xmm1, %xmm1, %xmm2
+; AVX1-NEXT:    vpsraw $8, %xmm0, %xmm3
+; AVX1-NEXT:    vpblendvb %xmm1, %xmm3, %xmm0, %xmm0
+; AVX1-NEXT:    vpsraw $4, %xmm0, %xmm1
+; AVX1-NEXT:    vpblendvb %xmm2, %xmm1, %xmm0, %xmm0
+; AVX1-NEXT:    vpsraw $2, %xmm0, %xmm1
+; AVX1-NEXT:    vpaddw %xmm2, %xmm2, %xmm2
+; AVX1-NEXT:    vpblendvb %xmm2, %xmm1, %xmm0, %xmm0
+; AVX1-NEXT:    vpsraw $1, %xmm0, %xmm1
+; AVX1-NEXT:    vpaddw %xmm2, %xmm2, %xmm2
+; AVX1-NEXT:    vpblendvb %xmm2, %xmm1, %xmm0, %xmm0
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: var_shift_v8i16:
+; AVX2:       # BB#0:
+; AVX2-NEXT:    vpmovzxwd {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
+; AVX2-NEXT:    vpmovsxwd %xmm0, %ymm0
+; AVX2-NEXT:    vpsravd %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vpshufb {{.*#+}} ymm0 = ymm0[0,1,4,5,8,9,12,13],zero,zero,zero,zero,zero,zero,zero,zero,ymm0[16,17,20,21,24,25,28,29],zero,zero,zero,zero,zero,zero,zero,zero
+; AVX2-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,2,2,3]
+; AVX2-NEXT:    vzeroupper
+; AVX2-NEXT:    retq
+;
+; X32-SSE-LABEL: var_shift_v8i16:
+; X32-SSE:       # BB#0:
+; X32-SSE-NEXT:    psllw $12, %xmm1
+; X32-SSE-NEXT:    movdqa %xmm1, %xmm2
+; X32-SSE-NEXT:    psraw $15, %xmm2
+; X32-SSE-NEXT:    movdqa %xmm2, %xmm3
+; X32-SSE-NEXT:    pandn %xmm0, %xmm3
+; X32-SSE-NEXT:    psraw $8, %xmm0
+; X32-SSE-NEXT:    pand %xmm2, %xmm0
+; X32-SSE-NEXT:    por %xmm3, %xmm0
+; X32-SSE-NEXT:    paddw %xmm1, %xmm1
+; X32-SSE-NEXT:    movdqa %xmm1, %xmm2
+; X32-SSE-NEXT:    psraw $15, %xmm2
+; X32-SSE-NEXT:    movdqa %xmm2, %xmm3
+; X32-SSE-NEXT:    pandn %xmm0, %xmm3
+; X32-SSE-NEXT:    psraw $4, %xmm0
+; X32-SSE-NEXT:    pand %xmm2, %xmm0
+; X32-SSE-NEXT:    por %xmm3, %xmm0
+; X32-SSE-NEXT:    paddw %xmm1, %xmm1
+; X32-SSE-NEXT:    movdqa %xmm1, %xmm2
+; X32-SSE-NEXT:    psraw $15, %xmm2
+; X32-SSE-NEXT:    movdqa %xmm2, %xmm3
+; X32-SSE-NEXT:    pandn %xmm0, %xmm3
+; X32-SSE-NEXT:    psraw $2, %xmm0
+; X32-SSE-NEXT:    pand %xmm2, %xmm0
+; X32-SSE-NEXT:    por %xmm3, %xmm0
+; X32-SSE-NEXT:    paddw %xmm1, %xmm1
+; X32-SSE-NEXT:    psraw $15, %xmm1
+; X32-SSE-NEXT:    movdqa %xmm1, %xmm2
+; X32-SSE-NEXT:    pandn %xmm0, %xmm2
+; X32-SSE-NEXT:    psraw $1, %xmm0
+; X32-SSE-NEXT:    pand %xmm1, %xmm0
+; X32-SSE-NEXT:    por %xmm2, %xmm0
+; X32-SSE-NEXT:    retl
+  %shift = ashr <8 x i16> %a, %b
+  ret <8 x i16> %shift
+}
+
+define <16 x i8> @var_shift_v16i8(<16 x i8> %a, <16 x i8> %b) nounwind {
+; SSE2-LABEL: var_shift_v16i8:
+; SSE2:       # BB#0:
+; SSE2-NEXT:    punpckhbw {{.*#+}} xmm2 = xmm2[8],xmm0[8],xmm2[9],xmm0[9],xmm2[10],xmm0[10],xmm2[11],xmm0[11],xmm2[12],xmm0[12],xmm2[13],xmm0[13],xmm2[14],xmm0[14],xmm2[15],xmm0[15]
+; SSE2-NEXT:    psllw $5, %xmm1
+; SSE2-NEXT:    punpckhbw {{.*#+}} xmm4 = xmm4[8],xmm1[8],xmm4[9],xmm1[9],xmm4[10],xmm1[10],xmm4[11],xmm1[11],xmm4[12],xmm1[12],xmm4[13],xmm1[13],xmm4[14],xmm1[14],xmm4[15],xmm1[15]
+; SSE2-NEXT:    pxor %xmm3, %xmm3
+; SSE2-NEXT:    pxor %xmm5, %xmm5
+; SSE2-NEXT:    pcmpgtw %xmm4, %xmm5
+; SSE2-NEXT:    movdqa %xmm5, %xmm6
+; SSE2-NEXT:    pandn %xmm2, %xmm6
+; SSE2-NEXT:    psraw $4, %xmm2
+; SSE2-NEXT:    pand %xmm5, %xmm2
+; SSE2-NEXT:    por %xmm6, %xmm2
+; SSE2-NEXT:    paddw %xmm4, %xmm4
+; SSE2-NEXT:    pxor %xmm5, %xmm5
+; SSE2-NEXT:    pcmpgtw %xmm4, %xmm5
+; SSE2-NEXT:    movdqa %xmm5, %xmm6
+; SSE2-NEXT:    pandn %xmm2, %xmm6
+; SSE2-NEXT:    psraw $2, %xmm2
+; SSE2-NEXT:    pand %xmm5, %xmm2
+; SSE2-NEXT:    por %xmm6, %xmm2
+; SSE2-NEXT:    paddw %xmm4, %xmm4
+; SSE2-NEXT:    pxor %xmm5, %xmm5
+; SSE2-NEXT:    pcmpgtw %xmm4, %xmm5
+; SSE2-NEXT:    movdqa %xmm5, %xmm4
+; SSE2-NEXT:    pandn %xmm2, %xmm4
+; SSE2-NEXT:    psraw $1, %xmm2
+; SSE2-NEXT:    pand %xmm5, %xmm2
+; SSE2-NEXT:    por %xmm4, %xmm2
+; SSE2-NEXT:    psrlw $8, %xmm2
+; SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
+; SSE2-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
+; SSE2-NEXT:    pxor %xmm4, %xmm4
+; SSE2-NEXT:    pcmpgtw %xmm1, %xmm4
+; SSE2-NEXT:    movdqa %xmm4, %xmm5
+; SSE2-NEXT:    pandn %xmm0, %xmm5
+; SSE2-NEXT:    psraw $4, %xmm0
+; SSE2-NEXT:    pand %xmm4, %xmm0
+; SSE2-NEXT:    por %xmm5, %xmm0
+; SSE2-NEXT:    paddw %xmm1, %xmm1
+; SSE2-NEXT:    pxor %xmm4, %xmm4
+; SSE2-NEXT:    pcmpgtw %xmm1, %xmm4
+; SSE2-NEXT:    movdqa %xmm4, %xmm5
+; SSE2-NEXT:    pandn %xmm0, %xmm5
+; SSE2-NEXT:    psraw $2, %xmm0
+; SSE2-NEXT:    pand %xmm4, %xmm0
+; SSE2-NEXT:    por %xmm5, %xmm0
+; SSE2-NEXT:    paddw %xmm1, %xmm1
+; SSE2-NEXT:    pcmpgtw %xmm1, %xmm3
+; SSE2-NEXT:    movdqa %xmm3, %xmm1
+; SSE2-NEXT:    pandn %xmm0, %xmm1
+; SSE2-NEXT:    psraw $1, %xmm0
+; SSE2-NEXT:    pand %xmm3, %xmm0
+; SSE2-NEXT:    por %xmm1, %xmm0
+; SSE2-NEXT:    psrlw $8, %xmm0
+; SSE2-NEXT:    packuswb %xmm2, %xmm0
+; SSE2-NEXT:    retq
+;
+; SSE41-LABEL: var_shift_v16i8:
+; SSE41:       # BB#0:
+; SSE41-NEXT:    movdqa %xmm0, %xmm2
+; SSE41-NEXT:    psllw $5, %xmm1
+; SSE41-NEXT:    punpckhbw {{.*#+}} xmm0 = xmm0[8],xmm1[8],xmm0[9],xmm1[9],xmm0[10],xmm1[10],xmm0[11],xmm1[11],xmm0[12],xmm1[12],xmm0[13],xmm1[13],xmm0[14],xmm1[14],xmm0[15],xmm1[15]
+; SSE41-NEXT:    punpckhbw {{.*#+}} xmm3 = xmm3[8],xmm2[8],xmm3[9],xmm2[9],xmm3[10],xmm2[10],xmm3[11],xmm2[11],xmm3[12],xmm2[12],xmm3[13],xmm2[13],xmm3[14],xmm2[14],xmm3[15],xmm2[15]
+; SSE41-NEXT:    movdqa %xmm3, %xmm4
+; SSE41-NEXT:    psraw $4, %xmm4
+; SSE41-NEXT:    pblendvb %xmm4, %xmm3
+; SSE41-NEXT:    movdqa %xmm3, %xmm4
+; SSE41-NEXT:    psraw $2, %xmm4
+; SSE41-NEXT:    paddw %xmm0, %xmm0
+; SSE41-NEXT:    pblendvb %xmm4, %xmm3
+; SSE41-NEXT:    movdqa %xmm3, %xmm4
+; SSE41-NEXT:    psraw $1, %xmm4
+; SSE41-NEXT:    paddw %xmm0, %xmm0
+; SSE41-NEXT:    pblendvb %xmm4, %xmm3
+; SSE41-NEXT:    psrlw $8, %xmm3
+; SSE41-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3],xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7]
+; SSE41-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1],xmm1[2],xmm2[2],xmm1[3],xmm2[3],xmm1[4],xmm2[4],xmm1[5],xmm2[5],xmm1[6],xmm2[6],xmm1[7],xmm2[7]
+; SSE41-NEXT:    movdqa %xmm1, %xmm2
+; SSE41-NEXT:    psraw $4, %xmm2
+; SSE41-NEXT:    pblendvb %xmm2, %xmm1
+; SSE41-NEXT:    movdqa %xmm1, %xmm2
+; SSE41-NEXT:    psraw $2, %xmm2
+; SSE41-NEXT:    paddw %xmm0, %xmm0
+; SSE41-NEXT:    pblendvb %xmm2, %xmm1
+; SSE41-NEXT:    movdqa %xmm1, %xmm2
+; SSE41-NEXT:    psraw $1, %xmm2
+; SSE41-NEXT:    paddw %xmm0, %xmm0
+; SSE41-NEXT:    pblendvb %xmm2, %xmm1
+; SSE41-NEXT:    psrlw $8, %xmm1
+; SSE41-NEXT:    packuswb %xmm3, %xmm1
+; SSE41-NEXT:    movdqa %xmm1, %xmm0
+; SSE41-NEXT:    retq
+;
+; AVX-LABEL: var_shift_v16i8:
+; AVX:       # BB#0:
+; AVX-NEXT:    vpsllw $5, %xmm1, %xmm1
+; AVX-NEXT:    vpunpckhbw {{.*#+}} xmm2 = xmm0[8],xmm1[8],xmm0[9],xmm1[9],xmm0[10],xmm1[10],xmm0[11],xmm1[11],xmm0[12],xmm1[12],xmm0[13],xmm1[13],xmm0[14],xmm1[14],xmm0[15],xmm1[15]
+; AVX-NEXT:    vpunpckhbw {{.*#+}} xmm3 = xmm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
+; AVX-NEXT:    vpsraw $4, %xmm3, %xmm4
+; AVX-NEXT:    vpblendvb %xmm2, %xmm4, %xmm3, %xmm3
+; AVX-NEXT:    vpsraw $2, %xmm3, %xmm4
+; AVX-NEXT:    vpaddw %xmm2, %xmm2, %xmm2
+; AVX-NEXT:    vpblendvb %xmm2, %xmm4, %xmm3, %xmm3
+; AVX-NEXT:    vpsraw $1, %xmm3, %xmm4
+; AVX-NEXT:    vpaddw %xmm2, %xmm2, %xmm2
+; AVX-NEXT:    vpblendvb %xmm2, %xmm4, %xmm3, %xmm2
+; AVX-NEXT:    vpsrlw $8, %xmm2, %xmm2
+; AVX-NEXT:    vpunpcklbw {{.*#+}} xmm1 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3],xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7]
+; AVX-NEXT:    vpunpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
+; AVX-NEXT:    vpsraw $4, %xmm0, %xmm3
+; AVX-NEXT:    vpblendvb %xmm1, %xmm3, %xmm0, %xmm0
+; AVX-NEXT:    vpsraw $2, %xmm0, %xmm3
+; AVX-NEXT:    vpaddw %xmm1, %xmm1, %xmm1
+; AVX-NEXT:    vpblendvb %xmm1, %xmm3, %xmm0, %xmm0
+; AVX-NEXT:    vpsraw $1, %xmm0, %xmm3
+; AVX-NEXT:    vpaddw %xmm1, %xmm1, %xmm1
+; AVX-NEXT:    vpblendvb %xmm1, %xmm3, %xmm0, %xmm0
+; AVX-NEXT:    vpsrlw $8, %xmm0, %xmm0
+; AVX-NEXT:    vpackuswb %xmm2, %xmm0, %xmm0
+; AVX-NEXT:    retq
+;
+; X32-SSE-LABEL: var_shift_v16i8:
+; X32-SSE:       # BB#0:
+; X32-SSE-NEXT:    punpckhbw {{.*#+}} xmm2 = xmm2[8],xmm0[8],xmm2[9],xmm0[9],xmm2[10],xmm0[10],xmm2[11],xmm0[11],xmm2[12],xmm0[12],xmm2[13],xmm0[13],xmm2[14],xmm0[14],xmm2[15],xmm0[15]
+; X32-SSE-NEXT:    psllw $5, %xmm1
+; X32-SSE-NEXT:    punpckhbw {{.*#+}} xmm4 = xmm4[8],xmm1[8],xmm4[9],xmm1[9],xmm4[10],xmm1[10],xmm4[11],xmm1[11],xmm4[12],xmm1[12],xmm4[13],xmm1[13],xmm4[14],xmm1[14],xmm4[15],xmm1[15]
+; X32-SSE-NEXT:    pxor %xmm3, %xmm3
+; X32-SSE-NEXT:    pxor %xmm5, %xmm5
+; X32-SSE-NEXT:    pcmpgtw %xmm4, %xmm5
+; X32-SSE-NEXT:    movdqa %xmm5, %xmm6
+; X32-SSE-NEXT:    pandn %xmm2, %xmm6
+; X32-SSE-NEXT:    psraw $4, %xmm2
+; X32-SSE-NEXT:    pand %xmm5, %xmm2
+; X32-SSE-NEXT:    por %xmm6, %xmm2
+; X32-SSE-NEXT:    paddw %xmm4, %xmm4
+; X32-SSE-NEXT:    pxor %xmm5, %xmm5
+; X32-SSE-NEXT:    pcmpgtw %xmm4, %xmm5
+; X32-SSE-NEXT:    movdqa %xmm5, %xmm6
+; X32-SSE-NEXT:    pandn %xmm2, %xmm6
+; X32-SSE-NEXT:    psraw $2, %xmm2
+; X32-SSE-NEXT:    pand %xmm5, %xmm2
+; X32-SSE-NEXT:    por %xmm6, %xmm2
+; X32-SSE-NEXT:    paddw %xmm4, %xmm4
+; X32-SSE-NEXT:    pxor %xmm5, %xmm5
+; X32-SSE-NEXT:    pcmpgtw %xmm4, %xmm5
+; X32-SSE-NEXT:    movdqa %xmm5, %xmm4
+; X32-SSE-NEXT:    pandn %xmm2, %xmm4
+; X32-SSE-NEXT:    psraw $1, %xmm2
+; X32-SSE-NEXT:    pand %xmm5, %xmm2
+; X32-SSE-NEXT:    por %xmm4, %xmm2
+; X32-SSE-NEXT:    psrlw $8, %xmm2
+; X32-SSE-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
+; X32-SSE-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
+; X32-SSE-NEXT:    pxor %xmm4, %xmm4
+; X32-SSE-NEXT:    pcmpgtw %xmm1, %xmm4
+; X32-SSE-NEXT:    movdqa %xmm4, %xmm5
+; X32-SSE-NEXT:    pandn %xmm0, %xmm5
+; X32-SSE-NEXT:    psraw $4, %xmm0
+; X32-SSE-NEXT:    pand %xmm4, %xmm0
+; X32-SSE-NEXT:    por %xmm5, %xmm0
+; X32-SSE-NEXT:    paddw %xmm1, %xmm1
+; X32-SSE-NEXT:    pxor %xmm4, %xmm4
+; X32-SSE-NEXT:    pcmpgtw %xmm1, %xmm4
+; X32-SSE-NEXT:    movdqa %xmm4, %xmm5
+; X32-SSE-NEXT:    pandn %xmm0, %xmm5
+; X32-SSE-NEXT:    psraw $2, %xmm0
+; X32-SSE-NEXT:    pand %xmm4, %xmm0
+; X32-SSE-NEXT:    por %xmm5, %xmm0
+; X32-SSE-NEXT:    paddw %xmm1, %xmm1
+; X32-SSE-NEXT:    pcmpgtw %xmm1, %xmm3
+; X32-SSE-NEXT:    movdqa %xmm3, %xmm1
+; X32-SSE-NEXT:    pandn %xmm0, %xmm1
+; X32-SSE-NEXT:    psraw $1, %xmm0
+; X32-SSE-NEXT:    pand %xmm3, %xmm0
+; X32-SSE-NEXT:    por %xmm1, %xmm0
+; X32-SSE-NEXT:    psrlw $8, %xmm0
+; X32-SSE-NEXT:    packuswb %xmm2, %xmm0
+; X32-SSE-NEXT:    retl
+  %shift = ashr <16 x i8> %a, %b
+  ret <16 x i8> %shift
+}
+
+;
+; Uniform Variable Shifts
+;
+
+define <2 x i64> @splatvar_shift_v2i64(<2 x i64> %a, <2 x i64> %b) nounwind {
+; SSE-LABEL: splatvar_shift_v2i64:
+; SSE:       # BB#0:
+; SSE-NEXT:    movdqa {{.*#+}} xmm2 = [9223372036854775808,9223372036854775808]
+; SSE-NEXT:    psrlq %xmm1, %xmm2
+; SSE-NEXT:    psrlq %xmm1, %xmm0
+; SSE-NEXT:    pxor %xmm2, %xmm0
+; SSE-NEXT:    psubq %xmm2, %xmm0
+; SSE-NEXT:    retq
+;
+; AVX-LABEL: splatvar_shift_v2i64:
+; AVX:       # BB#0:
+; AVX-NEXT:    vmovdqa {{.*#+}} xmm2 = [9223372036854775808,9223372036854775808]
+; AVX-NEXT:    vpsrlq %xmm1, %xmm2, %xmm2
+; AVX-NEXT:    vpsrlq %xmm1, %xmm0, %xmm0
+; AVX-NEXT:    vpxor %xmm2, %xmm0, %xmm0
+; AVX-NEXT:    vpsubq %xmm2, %xmm0, %xmm0
+; AVX-NEXT:    retq
+;
+; X32-SSE-LABEL: splatvar_shift_v2i64:
+; X32-SSE:       # BB#0:
+; X32-SSE-NEXT:    movq {{.*#+}} xmm1 = xmm1[0],zero
+; X32-SSE-NEXT:    movdqa {{.*#+}} xmm2 = [0,2147483648,0,2147483648]
+; X32-SSE-NEXT:    psrlq %xmm1, %xmm2
+; X32-SSE-NEXT:    psrlq %xmm1, %xmm0
+; X32-SSE-NEXT:    pxor %xmm2, %xmm0
+; X32-SSE-NEXT:    psubq %xmm2, %xmm0
+; X32-SSE-NEXT:    retl
+  %splat = shufflevector <2 x i64> %b, <2 x i64> undef, <2 x i32> zeroinitializer
+  %shift = ashr <2 x i64> %a, %splat
+  ret <2 x i64> %shift
+}
+
+define <4 x i32> @splatvar_shift_v4i32(<4 x i32> %a, <4 x i32> %b) nounwind {
+; SSE2-LABEL: splatvar_shift_v4i32:
+; SSE2:       # BB#0:
+; SSE2-NEXT:    xorps %xmm2, %xmm2
+; SSE2-NEXT:    movss {{.*#+}} xmm2 = xmm1[0],xmm2[1,2,3]
+; SSE2-NEXT:    psrad %xmm2, %xmm0
+; SSE2-NEXT:    retq
+;
+; SSE41-LABEL: splatvar_shift_v4i32:
+; SSE41:       # BB#0:
+; SSE41-NEXT:    pxor %xmm2, %xmm2
+; SSE41-NEXT:    pblendw {{.*#+}} xmm2 = xmm1[0,1],xmm2[2,3,4,5,6,7]
+; SSE41-NEXT:    psrad %xmm2, %xmm0
+; SSE41-NEXT:    retq
+;
+; AVX-LABEL: splatvar_shift_v4i32:
+; AVX:       # BB#0:
+; AVX-NEXT:    vpxor %xmm2, %xmm2, %xmm2
+; AVX-NEXT:    vpblendw {{.*#+}} xmm1 = xmm1[0,1],xmm2[2,3,4,5,6,7]
+; AVX-NEXT:    vpsrad %xmm1, %xmm0, %xmm0
+; AVX-NEXT:    retq
+;
+; X32-SSE-LABEL: splatvar_shift_v4i32:
+; X32-SSE:       # BB#0:
+; X32-SSE-NEXT:    xorps %xmm2, %xmm2
+; X32-SSE-NEXT:    movss {{.*#+}} xmm2 = xmm1[0],xmm2[1,2,3]
+; X32-SSE-NEXT:    psrad %xmm2, %xmm0
+; X32-SSE-NEXT:    retl
+  %splat = shufflevector <4 x i32> %b, <4 x i32> undef, <4 x i32> zeroinitializer
+  %shift = ashr <4 x i32> %a, %splat
+  ret <4 x i32> %shift
+}
+
+define <8 x i16> @splatvar_shift_v8i16(<8 x i16> %a, <8 x i16> %b) nounwind {
+; SSE2-LABEL: splatvar_shift_v8i16:
+; SSE2:       # BB#0:
+; SSE2-NEXT:    movd %xmm1, %eax
+; SSE2-NEXT:    movzwl %ax, %eax
+; SSE2-NEXT:    movd %eax, %xmm1
+; SSE2-NEXT:    psraw %xmm1, %xmm0
+; SSE2-NEXT:    retq
+;
+; SSE41-LABEL: splatvar_shift_v8i16:
+; SSE41:       # BB#0:
+; SSE41-NEXT:    pxor %xmm2, %xmm2
+; SSE41-NEXT:    pblendw {{.*#+}} xmm2 = xmm1[0],xmm2[1,2,3,4,5,6,7]
+; SSE41-NEXT:    psraw %xmm2, %xmm0
+; SSE41-NEXT:    retq
+;
+; AVX-LABEL: splatvar_shift_v8i16:
+; AVX:       # BB#0:
+; AVX-NEXT:    vpxor %xmm2, %xmm2, %xmm2
+; AVX-NEXT:    vpblendw {{.*#+}} xmm1 = xmm1[0],xmm2[1,2,3,4,5,6,7]
+; AVX-NEXT:    vpsraw %xmm1, %xmm0, %xmm0
+; AVX-NEXT:    retq
+;
+; X32-SSE-LABEL: splatvar_shift_v8i16:
+; X32-SSE:       # BB#0:
+; X32-SSE-NEXT:    movd %xmm1, %eax
+; X32-SSE-NEXT:    movzwl %ax, %eax
+; X32-SSE-NEXT:    movd %eax, %xmm1
+; X32-SSE-NEXT:    psraw %xmm1, %xmm0
+; X32-SSE-NEXT:    retl
+  %splat = shufflevector <8 x i16> %b, <8 x i16> undef, <8 x i32> zeroinitializer
+  %shift = ashr <8 x i16> %a, %splat
+  ret <8 x i16> %shift
+}
+
+define <16 x i8> @splatvar_shift_v16i8(<16 x i8> %a, <16 x i8> %b) nounwind {
+; SSE2-LABEL: splatvar_shift_v16i8:
+; SSE2:       # BB#0:
+; SSE2-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,1,0,3]
+; SSE2-NEXT:    pshuflw {{.*#+}} xmm1 = xmm1[0,0,0,0,4,5,6,7]
+; SSE2-NEXT:    pshufhw {{.*#+}} xmm3 = xmm1[0,1,2,3,4,4,4,4]
+; SSE2-NEXT:    punpckhbw {{.*#+}} xmm1 = xmm1[8],xmm0[8],xmm1[9],xmm0[9],xmm1[10],xmm0[10],xmm1[11],xmm0[11],xmm1[12],xmm0[12],xmm1[13],xmm0[13],xmm1[14],xmm0[14],xmm1[15],xmm0[15]
+; SSE2-NEXT:    psllw $5, %xmm3
+; SSE2-NEXT:    punpckhbw {{.*#+}} xmm4 = xmm4[8],xmm3[8],xmm4[9],xmm3[9],xmm4[10],xmm3[10],xmm4[11],xmm3[11],xmm4[12],xmm3[12],xmm4[13],xmm3[13],xmm4[14],xmm3[14],xmm4[15],xmm3[15]
+; SSE2-NEXT:    pxor %xmm2, %xmm2
+; SSE2-NEXT:    pxor %xmm5, %xmm5
+; SSE2-NEXT:    pcmpgtw %xmm4, %xmm5
+; SSE2-NEXT:    movdqa %xmm5, %xmm6
+; SSE2-NEXT:    pandn %xmm1, %xmm6
+; SSE2-NEXT:    psraw $4, %xmm1
+; SSE2-NEXT:    pand %xmm5, %xmm1
+; SSE2-NEXT:    por %xmm6, %xmm1
+; SSE2-NEXT:    paddw %xmm4, %xmm4
+; SSE2-NEXT:    pxor %xmm5, %xmm5
+; SSE2-NEXT:    pcmpgtw %xmm4, %xmm5
+; SSE2-NEXT:    movdqa %xmm5, %xmm6
+; SSE2-NEXT:    pandn %xmm1, %xmm6
+; SSE2-NEXT:    psraw $2, %xmm1
+; SSE2-NEXT:    pand %xmm5, %xmm1
+; SSE2-NEXT:    por %xmm6, %xmm1
+; SSE2-NEXT:    paddw %xmm4, %xmm4
+; SSE2-NEXT:    pxor %xmm5, %xmm5
+; SSE2-NEXT:    pcmpgtw %xmm4, %xmm5
+; SSE2-NEXT:    movdqa %xmm5, %xmm4
+; SSE2-NEXT:    pandn %xmm1, %xmm4
+; SSE2-NEXT:    psraw $1, %xmm1
+; SSE2-NEXT:    pand %xmm5, %xmm1
+; SSE2-NEXT:    por %xmm4, %xmm1
+; SSE2-NEXT:    psrlw $8, %xmm1
+; SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
+; SSE2-NEXT:    punpcklbw {{.*#+}} xmm3 = xmm3[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
+; SSE2-NEXT:    pxor %xmm4, %xmm4
+; SSE2-NEXT:    pcmpgtw %xmm3, %xmm4
+; SSE2-NEXT:    movdqa %xmm4, %xmm5
+; SSE2-NEXT:    pandn %xmm0, %xmm5
+; SSE2-NEXT:    psraw $4, %xmm0
+; SSE2-NEXT:    pand %xmm4, %xmm0
+; SSE2-NEXT:    por %xmm5, %xmm0
+; SSE2-NEXT:    paddw %xmm3, %xmm3
+; SSE2-NEXT:    pxor %xmm4, %xmm4
+; SSE2-NEXT:    pcmpgtw %xmm3, %xmm4
+; SSE2-NEXT:    movdqa %xmm4, %xmm5
+; SSE2-NEXT:    pandn %xmm0, %xmm5
+; SSE2-NEXT:    psraw $2, %xmm0
+; SSE2-NEXT:    pand %xmm4, %xmm0
+; SSE2-NEXT:    por %xmm5, %xmm0
+; SSE2-NEXT:    paddw %xmm3, %xmm3
+; SSE2-NEXT:    pcmpgtw %xmm3, %xmm2
+; SSE2-NEXT:    movdqa %xmm2, %xmm3
+; SSE2-NEXT:    pandn %xmm0, %xmm3
+; SSE2-NEXT:    psraw $1, %xmm0
+; SSE2-NEXT:    pand %xmm2, %xmm0
+; SSE2-NEXT:    por %xmm3, %xmm0
+; SSE2-NEXT:    psrlw $8, %xmm0
+; SSE2-NEXT:    packuswb %xmm1, %xmm0
+; SSE2-NEXT:    retq
+;
+; SSE41-LABEL: splatvar_shift_v16i8:
+; SSE41:       # BB#0:
+; SSE41-NEXT:    movdqa %xmm0, %xmm2
+; SSE41-NEXT:    pxor %xmm0, %xmm0
+; SSE41-NEXT:    pshufb %xmm0, %xmm1
+; SSE41-NEXT:    psllw $5, %xmm1
+; SSE41-NEXT:    punpckhbw {{.*#+}} xmm0 = xmm0[8],xmm1[8],xmm0[9],xmm1[9],xmm0[10],xmm1[10],xmm0[11],xmm1[11],xmm0[12],xmm1[12],xmm0[13],xmm1[13],xmm0[14],xmm1[14],xmm0[15],xmm1[15]
+; SSE41-NEXT:    punpckhbw {{.*#+}} xmm3 = xmm3[8],xmm2[8],xmm3[9],xmm2[9],xmm3[10],xmm2[10],xmm3[11],xmm2[11],xmm3[12],xmm2[12],xmm3[13],xmm2[13],xmm3[14],xmm2[14],xmm3[15],xmm2[15]
+; SSE41-NEXT:    movdqa %xmm3, %xmm4
+; SSE41-NEXT:    psraw $4, %xmm4
+; SSE41-NEXT:    pblendvb %xmm4, %xmm3
+; SSE41-NEXT:    movdqa %xmm3, %xmm4
+; SSE41-NEXT:    psraw $2, %xmm4
+; SSE41-NEXT:    paddw %xmm0, %xmm0
+; SSE41-NEXT:    pblendvb %xmm4, %xmm3
+; SSE41-NEXT:    movdqa %xmm3, %xmm4
+; SSE41-NEXT:    psraw $1, %xmm4
+; SSE41-NEXT:    paddw %xmm0, %xmm0
+; SSE41-NEXT:    pblendvb %xmm4, %xmm3
+; SSE41-NEXT:    psrlw $8, %xmm3
+; SSE41-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3],xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7]
+; SSE41-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1],xmm1[2],xmm2[2],xmm1[3],xmm2[3],xmm1[4],xmm2[4],xmm1[5],xmm2[5],xmm1[6],xmm2[6],xmm1[7],xmm2[7]
+; SSE41-NEXT:    movdqa %xmm1, %xmm2
+; SSE41-NEXT:    psraw $4, %xmm2
+; SSE41-NEXT:    pblendvb %xmm2, %xmm1
+; SSE41-NEXT:    movdqa %xmm1, %xmm2
+; SSE41-NEXT:    psraw $2, %xmm2
+; SSE41-NEXT:    paddw %xmm0, %xmm0
+; SSE41-NEXT:    pblendvb %xmm2, %xmm1
+; SSE41-NEXT:    movdqa %xmm1, %xmm2
+; SSE41-NEXT:    psraw $1, %xmm2
+; SSE41-NEXT:    paddw %xmm0, %xmm0
+; SSE41-NEXT:    pblendvb %xmm2, %xmm1
+; SSE41-NEXT:    psrlw $8, %xmm1
+; SSE41-NEXT:    packuswb %xmm3, %xmm1
+; SSE41-NEXT:    movdqa %xmm1, %xmm0
+; SSE41-NEXT:    retq
+;
+; AVX1-LABEL: splatvar_shift_v16i8:
+; AVX1:       # BB#0:
+; AVX1-NEXT:    vpxor %xmm2, %xmm2, %xmm2
+; AVX1-NEXT:    vpshufb %xmm2, %xmm1, %xmm1
+; AVX1-NEXT:    vpsllw $5, %xmm1, %xmm1
+; AVX1-NEXT:    vpunpckhbw {{.*#+}} xmm2 = xmm0[8],xmm1[8],xmm0[9],xmm1[9],xmm0[10],xmm1[10],xmm0[11],xmm1[11],xmm0[12],xmm1[12],xmm0[13],xmm1[13],xmm0[14],xmm1[14],xmm0[15],xmm1[15]
+; AVX1-NEXT:    vpunpckhbw {{.*#+}} xmm3 = xmm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
+; AVX1-NEXT:    vpsraw $4, %xmm3, %xmm4
+; AVX1-NEXT:    vpblendvb %xmm2, %xmm4, %xmm3, %xmm3
+; AVX1-NEXT:    vpsraw $2, %xmm3, %xmm4
+; AVX1-NEXT:    vpaddw %xmm2, %xmm2, %xmm2
+; AVX1-NEXT:    vpblendvb %xmm2, %xmm4, %xmm3, %xmm3
+; AVX1-NEXT:    vpsraw $1, %xmm3, %xmm4
+; AVX1-NEXT:    vpaddw %xmm2, %xmm2, %xmm2
+; AVX1-NEXT:    vpblendvb %xmm2, %xmm4, %xmm3, %xmm2
+; AVX1-NEXT:    vpsrlw $8, %xmm2, %xmm2
+; AVX1-NEXT:    vpunpcklbw {{.*#+}} xmm1 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3],xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7]
+; AVX1-NEXT:    vpunpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
+; AVX1-NEXT:    vpsraw $4, %xmm0, %xmm3
+; AVX1-NEXT:    vpblendvb %xmm1, %xmm3, %xmm0, %xmm0
+; AVX1-NEXT:    vpsraw $2, %xmm0, %xmm3
+; AVX1-NEXT:    vpaddw %xmm1, %xmm1, %xmm1
+; AVX1-NEXT:    vpblendvb %xmm1, %xmm3, %xmm0, %xmm0
+; AVX1-NEXT:    vpsraw $1, %xmm0, %xmm3
+; AVX1-NEXT:    vpaddw %xmm1, %xmm1, %xmm1
+; AVX1-NEXT:    vpblendvb %xmm1, %xmm3, %xmm0, %xmm0
+; AVX1-NEXT:    vpsrlw $8, %xmm0, %xmm0
+; AVX1-NEXT:    vpackuswb %xmm2, %xmm0, %xmm0
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: splatvar_shift_v16i8:
+; AVX2:       # BB#0:
+; AVX2-NEXT:    vpbroadcastb %xmm1, %xmm1
+; AVX2-NEXT:    vpsllw $5, %xmm1, %xmm1
+; AVX2-NEXT:    vpunpckhbw {{.*#+}} xmm2 = xmm0[8],xmm1[8],xmm0[9],xmm1[9],xmm0[10],xmm1[10],xmm0[11],xmm1[11],xmm0[12],xmm1[12],xmm0[13],xmm1[13],xmm0[14],xmm1[14],xmm0[15],xmm1[15]
+; AVX2-NEXT:    vpunpckhbw {{.*#+}} xmm3 = xmm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
+; AVX2-NEXT:    vpsraw $4, %xmm3, %xmm4
+; AVX2-NEXT:    vpblendvb %xmm2, %xmm4, %xmm3, %xmm3
+; AVX2-NEXT:    vpsraw $2, %xmm3, %xmm4
+; AVX2-NEXT:    vpaddw %xmm2, %xmm2, %xmm2
+; AVX2-NEXT:    vpblendvb %xmm2, %xmm4, %xmm3, %xmm3
+; AVX2-NEXT:    vpsraw $1, %xmm3, %xmm4
+; AVX2-NEXT:    vpaddw %xmm2, %xmm2, %xmm2
+; AVX2-NEXT:    vpblendvb %xmm2, %xmm4, %xmm3, %xmm2
+; AVX2-NEXT:    vpsrlw $8, %xmm2, %xmm2
+; AVX2-NEXT:    vpunpcklbw {{.*#+}} xmm1 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3],xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7]
+; AVX2-NEXT:    vpunpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
+; AVX2-NEXT:    vpsraw $4, %xmm0, %xmm3
+; AVX2-NEXT:    vpblendvb %xmm1, %xmm3, %xmm0, %xmm0
+; AVX2-NEXT:    vpsraw $2, %xmm0, %xmm3
+; AVX2-NEXT:    vpaddw %xmm1, %xmm1, %xmm1
+; AVX2-NEXT:    vpblendvb %xmm1, %xmm3, %xmm0, %xmm0
+; AVX2-NEXT:    vpsraw $1, %xmm0, %xmm3
+; AVX2-NEXT:    vpaddw %xmm1, %xmm1, %xmm1
+; AVX2-NEXT:    vpblendvb %xmm1, %xmm3, %xmm0, %xmm0
+; AVX2-NEXT:    vpsrlw $8, %xmm0, %xmm0
+; AVX2-NEXT:    vpackuswb %xmm2, %xmm0, %xmm0
+; AVX2-NEXT:    retq
+;
+; X32-SSE-LABEL: splatvar_shift_v16i8:
+; X32-SSE:       # BB#0:
+; X32-SSE-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
+; X32-SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,1,0,3]
+; X32-SSE-NEXT:    pshuflw {{.*#+}} xmm1 = xmm1[0,0,0,0,4,5,6,7]
+; X32-SSE-NEXT:    pshufhw {{.*#+}} xmm3 = xmm1[0,1,2,3,4,4,4,4]
+; X32-SSE-NEXT:    punpckhbw {{.*#+}} xmm1 = xmm1[8],xmm0[8],xmm1[9],xmm0[9],xmm1[10],xmm0[10],xmm1[11],xmm0[11],xmm1[12],xmm0[12],xmm1[13],xmm0[13],xmm1[14],xmm0[14],xmm1[15],xmm0[15]
+; X32-SSE-NEXT:    psllw $5, %xmm3
+; X32-SSE-NEXT:    punpckhbw {{.*#+}} xmm4 = xmm4[8],xmm3[8],xmm4[9],xmm3[9],xmm4[10],xmm3[10],xmm4[11],xmm3[11],xmm4[12],xmm3[12],xmm4[13],xmm3[13],xmm4[14],xmm3[14],xmm4[15],xmm3[15]
+; X32-SSE-NEXT:    pxor %xmm2, %xmm2
+; X32-SSE-NEXT:    pxor %xmm5, %xmm5
+; X32-SSE-NEXT:    pcmpgtw %xmm4, %xmm5
+; X32-SSE-NEXT:    movdqa %xmm5, %xmm6
+; X32-SSE-NEXT:    pandn %xmm1, %xmm6
+; X32-SSE-NEXT:    psraw $4, %xmm1
+; X32-SSE-NEXT:    pand %xmm5, %xmm1
+; X32-SSE-NEXT:    por %xmm6, %xmm1
+; X32-SSE-NEXT:    paddw %xmm4, %xmm4
+; X32-SSE-NEXT:    pxor %xmm5, %xmm5
+; X32-SSE-NEXT:    pcmpgtw %xmm4, %xmm5
+; X32-SSE-NEXT:    movdqa %xmm5, %xmm6
+; X32-SSE-NEXT:    pandn %xmm1, %xmm6
+; X32-SSE-NEXT:    psraw $2, %xmm1
+; X32-SSE-NEXT:    pand %xmm5, %xmm1
+; X32-SSE-NEXT:    por %xmm6, %xmm1
+; X32-SSE-NEXT:    paddw %xmm4, %xmm4
+; X32-SSE-NEXT:    pxor %xmm5, %xmm5
+; X32-SSE-NEXT:    pcmpgtw %xmm4, %xmm5
+; X32-SSE-NEXT:    movdqa %xmm5, %xmm4
+; X32-SSE-NEXT:    pandn %xmm1, %xmm4
+; X32-SSE-NEXT:    psraw $1, %xmm1
+; X32-SSE-NEXT:    pand %xmm5, %xmm1
+; X32-SSE-NEXT:    por %xmm4, %xmm1
+; X32-SSE-NEXT:    psrlw $8, %xmm1
+; X32-SSE-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
+; X32-SSE-NEXT:    punpcklbw {{.*#+}} xmm3 = xmm3[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
+; X32-SSE-NEXT:    pxor %xmm4, %xmm4
+; X32-SSE-NEXT:    pcmpgtw %xmm3, %xmm4
+; X32-SSE-NEXT:    movdqa %xmm4, %xmm5
+; X32-SSE-NEXT:    pandn %xmm0, %xmm5
+; X32-SSE-NEXT:    psraw $4, %xmm0
+; X32-SSE-NEXT:    pand %xmm4, %xmm0
+; X32-SSE-NEXT:    por %xmm5, %xmm0
+; X32-SSE-NEXT:    paddw %xmm3, %xmm3
+; X32-SSE-NEXT:    pxor %xmm4, %xmm4
+; X32-SSE-NEXT:    pcmpgtw %xmm3, %xmm4
+; X32-SSE-NEXT:    movdqa %xmm4, %xmm5
+; X32-SSE-NEXT:    pandn %xmm0, %xmm5
+; X32-SSE-NEXT:    psraw $2, %xmm0
+; X32-SSE-NEXT:    pand %xmm4, %xmm0
+; X32-SSE-NEXT:    por %xmm5, %xmm0
+; X32-SSE-NEXT:    paddw %xmm3, %xmm3
+; X32-SSE-NEXT:    pcmpgtw %xmm3, %xmm2
+; X32-SSE-NEXT:    movdqa %xmm2, %xmm3
+; X32-SSE-NEXT:    pandn %xmm0, %xmm3
+; X32-SSE-NEXT:    psraw $1, %xmm0
+; X32-SSE-NEXT:    pand %xmm2, %xmm0
+; X32-SSE-NEXT:    por %xmm3, %xmm0
+; X32-SSE-NEXT:    psrlw $8, %xmm0
+; X32-SSE-NEXT:    packuswb %xmm1, %xmm0
+; X32-SSE-NEXT:    retl
+  %splat = shufflevector <16 x i8> %b, <16 x i8> undef, <16 x i32> zeroinitializer
+  %shift = ashr <16 x i8> %a, %splat
+  ret <16 x i8> %shift
+}
+
+;
+; Constant Shifts
+;
+
+define <2 x i64> @constant_shift_v2i64(<2 x i64> %a) nounwind {
+; SSE2-LABEL: constant_shift_v2i64:
+; SSE2:       # BB#0:
+; SSE2-NEXT:    movdqa %xmm0, %xmm1
+; SSE2-NEXT:    psrlq $7, %xmm1
+; SSE2-NEXT:    psrlq $1, %xmm0
+; SSE2-NEXT:    movsd {{.*#+}} xmm1 = xmm0[0],xmm1[1]
+; SSE2-NEXT:    movapd {{.*#+}} xmm0 = [4611686018427387904,72057594037927936]
+; SSE2-NEXT:    xorpd %xmm0, %xmm1
+; SSE2-NEXT:    psubq %xmm0, %xmm1
+; SSE2-NEXT:    movdqa %xmm1, %xmm0
+; SSE2-NEXT:    retq
+;
+; SSE41-LABEL: constant_shift_v2i64:
+; SSE41:       # BB#0:
+; SSE41-NEXT:    movdqa %xmm0, %xmm1
+; SSE41-NEXT:    psrlq $7, %xmm1
+; SSE41-NEXT:    psrlq $1, %xmm0
+; SSE41-NEXT:    pblendw {{.*#+}} xmm0 = xmm0[0,1,2,3],xmm1[4,5,6,7]
+; SSE41-NEXT:    movdqa {{.*#+}} xmm1 = [4611686018427387904,72057594037927936]
+; SSE41-NEXT:    pxor %xmm1, %xmm0
+; SSE41-NEXT:    psubq %xmm1, %xmm0
+; SSE41-NEXT:    retq
+;
+; AVX1-LABEL: constant_shift_v2i64:
+; AVX1:       # BB#0:
+; AVX1-NEXT:    vpsrlq $7, %xmm0, %xmm1
+; AVX1-NEXT:    vpsrlq $1, %xmm0, %xmm0
+; AVX1-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0,1,2,3],xmm1[4,5,6,7]
+; AVX1-NEXT:    vmovdqa {{.*#+}} xmm1 = [4611686018427387904,72057594037927936]
+; AVX1-NEXT:    vpxor %xmm1, %xmm0, %xmm0
+; AVX1-NEXT:    vpsubq %xmm1, %xmm0, %xmm0
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: constant_shift_v2i64:
+; AVX2:       # BB#0:
+; AVX2-NEXT:    vpsrlvq {{.*}}(%rip), %xmm0, %xmm0
+; AVX2-NEXT:    vmovdqa {{.*#+}} xmm1 = [4611686018427387904,72057594037927936]
+; AVX2-NEXT:    vpxor %xmm1, %xmm0, %xmm0
+; AVX2-NEXT:    vpsubq %xmm1, %xmm0, %xmm0
+; AVX2-NEXT:    retq
+;
+; X32-SSE-LABEL: constant_shift_v2i64:
+; X32-SSE:       # BB#0:
+; X32-SSE-NEXT:    movdqa {{.*#+}} xmm1 = [0,2147483648,0,2147483648]
+; X32-SSE-NEXT:    movdqa %xmm1, %xmm2
+; X32-SSE-NEXT:    psrlq $7, %xmm2
+; X32-SSE-NEXT:    psrlq $1, %xmm1
+; X32-SSE-NEXT:    movsd {{.*#+}} xmm2 = xmm1[0],xmm2[1]
+; X32-SSE-NEXT:    movdqa %xmm0, %xmm1
+; X32-SSE-NEXT:    psrlq $7, %xmm1
+; X32-SSE-NEXT:    psrlq $1, %xmm0
+; X32-SSE-NEXT:    movsd {{.*#+}} xmm1 = xmm0[0],xmm1[1]
+; X32-SSE-NEXT:    xorpd %xmm2, %xmm1
+; X32-SSE-NEXT:    psubq %xmm2, %xmm1
+; X32-SSE-NEXT:    movdqa %xmm1, %xmm0
+; X32-SSE-NEXT:    retl
+  %shift = ashr <2 x i64> %a, <i64 1, i64 7>
+  ret <2 x i64> %shift
+}
+
+define <4 x i32> @constant_shift_v4i32(<4 x i32> %a) nounwind {
+; SSE2-LABEL: constant_shift_v4i32:
+; SSE2:       # BB#0:
+; SSE2-NEXT:    movdqa %xmm0, %xmm1
+; SSE2-NEXT:    psrad $7, %xmm1
+; SSE2-NEXT:    movdqa %xmm0, %xmm2
+; SSE2-NEXT:    psrad $5, %xmm2
+; SSE2-NEXT:    movsd {{.*#+}} xmm1 = xmm2[0],xmm1[1]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,3,2,3]
+; SSE2-NEXT:    movdqa %xmm0, %xmm2
+; SSE2-NEXT:    psrad $6, %xmm2
+; SSE2-NEXT:    psrad $4, %xmm0
+; SSE2-NEXT:    movsd {{.*#+}} xmm2 = xmm0[0],xmm2[1]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[0,2,2,3]
+; SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
+; SSE2-NEXT:    retq
+;
+; SSE41-LABEL: constant_shift_v4i32:
+; SSE41:       # BB#0:
+; SSE41-NEXT:    movdqa %xmm0, %xmm1
+; SSE41-NEXT:    psrad $7, %xmm1
+; SSE41-NEXT:    movdqa %xmm0, %xmm2
+; SSE41-NEXT:    psrad $5, %xmm2
+; SSE41-NEXT:    pblendw {{.*#+}} xmm2 = xmm2[0,1,2,3],xmm1[4,5,6,7]
+; SSE41-NEXT:    movdqa %xmm0, %xmm1
+; SSE41-NEXT:    psrad $6, %xmm1
+; SSE41-NEXT:    psrad $4, %xmm0
+; SSE41-NEXT:    pblendw {{.*#+}} xmm0 = xmm0[0,1,2,3],xmm1[4,5,6,7]
+; SSE41-NEXT:    pblendw {{.*#+}} xmm0 = xmm0[0,1],xmm2[2,3],xmm0[4,5],xmm2[6,7]
+; SSE41-NEXT:    retq
+;
+; AVX1-LABEL: constant_shift_v4i32:
+; AVX1:       # BB#0:
+; AVX1-NEXT:    vpsrad $7, %xmm0, %xmm1
+; AVX1-NEXT:    vpsrad $5, %xmm0, %xmm2
+; AVX1-NEXT:    vpblendw {{.*#+}} xmm1 = xmm2[0,1,2,3],xmm1[4,5,6,7]
+; AVX1-NEXT:    vpsrad $6, %xmm0, %xmm2
+; AVX1-NEXT:    vpsrad $4, %xmm0, %xmm0
+; AVX1-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0,1,2,3],xmm2[4,5,6,7]
+; AVX1-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0,1],xmm1[2,3],xmm0[4,5],xmm1[6,7]
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: constant_shift_v4i32:
+; AVX2:       # BB#0:
+; AVX2-NEXT:    vpsravd {{.*}}(%rip), %xmm0, %xmm0
+; AVX2-NEXT:    retq
+;
+; X32-SSE-LABEL: constant_shift_v4i32:
+; X32-SSE:       # BB#0:
+; X32-SSE-NEXT:    movdqa %xmm0, %xmm1
+; X32-SSE-NEXT:    psrad $7, %xmm1
+; X32-SSE-NEXT:    movdqa %xmm0, %xmm2
+; X32-SSE-NEXT:    psrad $5, %xmm2
+; X32-SSE-NEXT:    movsd {{.*#+}} xmm1 = xmm2[0],xmm1[1]
+; X32-SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,3,2,3]
+; X32-SSE-NEXT:    movdqa %xmm0, %xmm2
+; X32-SSE-NEXT:    psrad $6, %xmm2
+; X32-SSE-NEXT:    psrad $4, %xmm0
+; X32-SSE-NEXT:    movsd {{.*#+}} xmm2 = xmm0[0],xmm2[1]
+; X32-SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[0,2,2,3]
+; X32-SSE-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
+; X32-SSE-NEXT:    retl
+  %shift = ashr <4 x i32> %a, <i32 4, i32 5, i32 6, i32 7>
+  ret <4 x i32> %shift
+}
+
+define <8 x i16> @constant_shift_v8i16(<8 x i16> %a) nounwind {
+; SSE2-LABEL: constant_shift_v8i16:
+; SSE2:       # BB#0:
+; SSE2-NEXT:    movdqa %xmm0, %xmm1
+; SSE2-NEXT:    psraw $4, %xmm1
+; SSE2-NEXT:    movsd {{.*#+}} xmm1 = xmm0[0],xmm1[1]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[0,2,2,3]
+; SSE2-NEXT:    psraw $2, %xmm1
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,3,2,3]
+; SSE2-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1]
+; SSE2-NEXT:    movdqa {{.*#+}} xmm0 = [65535,0,65535,0,65535,0,65535,0]
+; SSE2-NEXT:    movdqa %xmm2, %xmm1
+; SSE2-NEXT:    pand %xmm0, %xmm1
+; SSE2-NEXT:    psraw $1, %xmm2
+; SSE2-NEXT:    pandn %xmm2, %xmm0
+; SSE2-NEXT:    por %xmm1, %xmm0
+; SSE2-NEXT:    retq
+;
+; SSE41-LABEL: constant_shift_v8i16:
+; SSE41:       # BB#0:
+; SSE41-NEXT:    movdqa %xmm0, %xmm1
+; SSE41-NEXT:    movdqa %xmm1, %xmm2
+; SSE41-NEXT:    psraw $8, %xmm2
+; SSE41-NEXT:    movaps {{.*#+}} xmm0 = [0,4112,8224,12336,16448,20560,24672,28784]
+; SSE41-NEXT:    pblendvb %xmm2, %xmm1
+; SSE41-NEXT:    movdqa %xmm1, %xmm2
+; SSE41-NEXT:    psraw $4, %xmm2
+; SSE41-NEXT:    movaps {{.*#+}} xmm0 = [0,8224,16448,24672,32896,41120,49344,57568]
+; SSE41-NEXT:    pblendvb %xmm2, %xmm1
+; SSE41-NEXT:    movdqa %xmm1, %xmm2
+; SSE41-NEXT:    psraw $2, %xmm2
+; SSE41-NEXT:    movaps {{.*#+}} xmm0 = [0,16448,32896,49344,256,16704,33152,49600]
+; SSE41-NEXT:    pblendvb %xmm2, %xmm1
+; SSE41-NEXT:    movdqa %xmm1, %xmm2
+; SSE41-NEXT:    psraw $1, %xmm2
+; SSE41-NEXT:    movaps {{.*#+}} xmm0 = [0,32896,256,33152,512,33408,768,33664]
+; SSE41-NEXT:    pblendvb %xmm2, %xmm1
+; SSE41-NEXT:    movdqa %xmm1, %xmm0
+; SSE41-NEXT:    retq
+;
+; AVX1-LABEL: constant_shift_v8i16:
+; AVX1:       # BB#0:
+; AVX1-NEXT:    vpsraw $8, %xmm0, %xmm1
+; AVX1-NEXT:    vmovdqa {{.*#+}} xmm2 = [0,4112,8224,12336,16448,20560,24672,28784]
+; AVX1-NEXT:    vpblendvb %xmm2, %xmm1, %xmm0, %xmm0
+; AVX1-NEXT:    vpsraw $4, %xmm0, %xmm1
+; AVX1-NEXT:    vmovdqa {{.*#+}} xmm2 = [0,8224,16448,24672,32896,41120,49344,57568]
+; AVX1-NEXT:    vpblendvb %xmm2, %xmm1, %xmm0, %xmm0
+; AVX1-NEXT:    vpsraw $2, %xmm0, %xmm1
+; AVX1-NEXT:    vmovdqa {{.*#+}} xmm2 = [0,16448,32896,49344,256,16704,33152,49600]
+; AVX1-NEXT:    vpblendvb %xmm2, %xmm1, %xmm0, %xmm0
+; AVX1-NEXT:    vpsraw $1, %xmm0, %xmm1
+; AVX1-NEXT:    vmovdqa {{.*#+}} xmm2 = [0,32896,256,33152,512,33408,768,33664]
+; AVX1-NEXT:    vpblendvb %xmm2, %xmm1, %xmm0, %xmm0
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: constant_shift_v8i16:
+; AVX2:       # BB#0:
+; AVX2-NEXT:    vpmovsxwd %xmm0, %ymm0
+; AVX2-NEXT:    vpmovzxwd {{.*#+}} ymm1 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero,mem[4],zero,mem[5],zero,mem[6],zero,mem[7],zero
+; AVX2-NEXT:    vpsravd %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vpshufb {{.*#+}} ymm0 = ymm0[0,1,4,5,8,9,12,13],zero,zero,zero,zero,zero,zero,zero,zero,ymm0[16,17,20,21,24,25,28,29],zero,zero,zero,zero,zero,zero,zero,zero
+; AVX2-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,2,2,3]
+; AVX2-NEXT:    vzeroupper
+; AVX2-NEXT:    retq
+;
+; X32-SSE-LABEL: constant_shift_v8i16:
+; X32-SSE:       # BB#0:
+; X32-SSE-NEXT:    movdqa %xmm0, %xmm1
+; X32-SSE-NEXT:    psraw $4, %xmm1
+; X32-SSE-NEXT:    movsd {{.*#+}} xmm1 = xmm0[0],xmm1[1]
+; X32-SSE-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[0,2,2,3]
+; X32-SSE-NEXT:    psraw $2, %xmm1
+; X32-SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,3,2,3]
+; X32-SSE-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1]
+; X32-SSE-NEXT:    movdqa {{.*#+}} xmm0 = [65535,0,65535,0,65535,0,65535,0]
+; X32-SSE-NEXT:    movdqa %xmm2, %xmm1
+; X32-SSE-NEXT:    pand %xmm0, %xmm1
+; X32-SSE-NEXT:    psraw $1, %xmm2
+; X32-SSE-NEXT:    pandn %xmm2, %xmm0
+; X32-SSE-NEXT:    por %xmm1, %xmm0
+; X32-SSE-NEXT:    retl
+  %shift = ashr <8 x i16> %a, <i16 0, i16 1, i16 2, i16 3, i16 4, i16 5, i16 6, i16 7>
+  ret <8 x i16> %shift
+}
+
+define <16 x i8> @constant_shift_v16i8(<16 x i8> %a) nounwind {
+; SSE2-LABEL: constant_shift_v16i8:
+; SSE2:       # BB#0:
+; SSE2-NEXT:    punpckhbw {{.*#+}} xmm1 = xmm1[8],xmm0[8],xmm1[9],xmm0[9],xmm1[10],xmm0[10],xmm1[11],xmm0[11],xmm1[12],xmm0[12],xmm1[13],xmm0[13],xmm1[14],xmm0[14],xmm1[15],xmm0[15]
+; SSE2-NEXT:    movdqa {{.*#+}} xmm3 = [0,1,2,3,4,5,6,7,7,6,5,4,3,2,1,0]
+; SSE2-NEXT:    psllw $5, %xmm3
+; SSE2-NEXT:    punpckhbw {{.*#+}} xmm4 = xmm4[8],xmm3[8],xmm4[9],xmm3[9],xmm4[10],xmm3[10],xmm4[11],xmm3[11],xmm4[12],xmm3[12],xmm4[13],xmm3[13],xmm4[14],xmm3[14],xmm4[15],xmm3[15]
+; SSE2-NEXT:    pxor %xmm2, %xmm2
+; SSE2-NEXT:    pxor %xmm5, %xmm5
+; SSE2-NEXT:    pcmpgtw %xmm4, %xmm5
+; SSE2-NEXT:    movdqa %xmm5, %xmm6
+; SSE2-NEXT:    pandn %xmm1, %xmm6
+; SSE2-NEXT:    psraw $4, %xmm1
+; SSE2-NEXT:    pand %xmm5, %xmm1
+; SSE2-NEXT:    por %xmm6, %xmm1
+; SSE2-NEXT:    paddw %xmm4, %xmm4
+; SSE2-NEXT:    pxor %xmm5, %xmm5
+; SSE2-NEXT:    pcmpgtw %xmm4, %xmm5
+; SSE2-NEXT:    movdqa %xmm5, %xmm6
+; SSE2-NEXT:    pandn %xmm1, %xmm6
+; SSE2-NEXT:    psraw $2, %xmm1
+; SSE2-NEXT:    pand %xmm5, %xmm1
+; SSE2-NEXT:    por %xmm6, %xmm1
+; SSE2-NEXT:    paddw %xmm4, %xmm4
+; SSE2-NEXT:    pxor %xmm5, %xmm5
+; SSE2-NEXT:    pcmpgtw %xmm4, %xmm5
+; SSE2-NEXT:    movdqa %xmm5, %xmm4
+; SSE2-NEXT:    pandn %xmm1, %xmm4
+; SSE2-NEXT:    psraw $1, %xmm1
+; SSE2-NEXT:    pand %xmm5, %xmm1
+; SSE2-NEXT:    por %xmm4, %xmm1
+; SSE2-NEXT:    psrlw $8, %xmm1
+; SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
+; SSE2-NEXT:    punpcklbw {{.*#+}} xmm3 = xmm3[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
+; SSE2-NEXT:    pxor %xmm4, %xmm4
+; SSE2-NEXT:    pcmpgtw %xmm3, %xmm4
+; SSE2-NEXT:    movdqa %xmm4, %xmm5
+; SSE2-NEXT:    pandn %xmm0, %xmm5
+; SSE2-NEXT:    psraw $4, %xmm0
+; SSE2-NEXT:    pand %xmm4, %xmm0
+; SSE2-NEXT:    por %xmm5, %xmm0
+; SSE2-NEXT:    paddw %xmm3, %xmm3
+; SSE2-NEXT:    pxor %xmm4, %xmm4
+; SSE2-NEXT:    pcmpgtw %xmm3, %xmm4
+; SSE2-NEXT:    movdqa %xmm4, %xmm5
+; SSE2-NEXT:    pandn %xmm0, %xmm5
+; SSE2-NEXT:    psraw $2, %xmm0
+; SSE2-NEXT:    pand %xmm4, %xmm0
+; SSE2-NEXT:    por %xmm5, %xmm0
+; SSE2-NEXT:    paddw %xmm3, %xmm3
+; SSE2-NEXT:    pcmpgtw %xmm3, %xmm2
+; SSE2-NEXT:    movdqa %xmm2, %xmm3
+; SSE2-NEXT:    pandn %xmm0, %xmm3
+; SSE2-NEXT:    psraw $1, %xmm0
+; SSE2-NEXT:    pand %xmm2, %xmm0
+; SSE2-NEXT:    por %xmm3, %xmm0
+; SSE2-NEXT:    psrlw $8, %xmm0
+; SSE2-NEXT:    packuswb %xmm1, %xmm0
+; SSE2-NEXT:    retq
+;
+; SSE41-LABEL: constant_shift_v16i8:
+; SSE41:       # BB#0:
+; SSE41-NEXT:    movdqa %xmm0, %xmm1
+; SSE41-NEXT:    movdqa {{.*#+}} xmm3 = [0,1,2,3,4,5,6,7,7,6,5,4,3,2,1,0]
+; SSE41-NEXT:    psllw $5, %xmm3
+; SSE41-NEXT:    punpckhbw {{.*#+}} xmm0 = xmm0[8],xmm3[8],xmm0[9],xmm3[9],xmm0[10],xmm3[10],xmm0[11],xmm3[11],xmm0[12],xmm3[12],xmm0[13],xmm3[13],xmm0[14],xmm3[14],xmm0[15],xmm3[15]
+; SSE41-NEXT:    punpckhbw {{.*#+}} xmm2 = xmm2[8],xmm1[8],xmm2[9],xmm1[9],xmm2[10],xmm1[10],xmm2[11],xmm1[11],xmm2[12],xmm1[12],xmm2[13],xmm1[13],xmm2[14],xmm1[14],xmm2[15],xmm1[15]
+; SSE41-NEXT:    movdqa %xmm2, %xmm4
+; SSE41-NEXT:    psraw $4, %xmm4
+; SSE41-NEXT:    pblendvb %xmm4, %xmm2
+; SSE41-NEXT:    movdqa %xmm2, %xmm4
+; SSE41-NEXT:    psraw $2, %xmm4
+; SSE41-NEXT:    paddw %xmm0, %xmm0
+; SSE41-NEXT:    pblendvb %xmm4, %xmm2
+; SSE41-NEXT:    movdqa %xmm2, %xmm4
+; SSE41-NEXT:    psraw $1, %xmm4
+; SSE41-NEXT:    paddw %xmm0, %xmm0
+; SSE41-NEXT:    pblendvb %xmm4, %xmm2
+; SSE41-NEXT:    psrlw $8, %xmm2
+; SSE41-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1],xmm0[2],xmm3[2],xmm0[3],xmm3[3],xmm0[4],xmm3[4],xmm0[5],xmm3[5],xmm0[6],xmm3[6],xmm0[7],xmm3[7]
+; SSE41-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
+; SSE41-NEXT:    movdqa %xmm1, %xmm3
+; SSE41-NEXT:    psraw $4, %xmm3
+; SSE41-NEXT:    pblendvb %xmm3, %xmm1
+; SSE41-NEXT:    movdqa %xmm1, %xmm3
+; SSE41-NEXT:    psraw $2, %xmm3
+; SSE41-NEXT:    paddw %xmm0, %xmm0
+; SSE41-NEXT:    pblendvb %xmm3, %xmm1
+; SSE41-NEXT:    movdqa %xmm1, %xmm3
+; SSE41-NEXT:    psraw $1, %xmm3
+; SSE41-NEXT:    paddw %xmm0, %xmm0
+; SSE41-NEXT:    pblendvb %xmm3, %xmm1
+; SSE41-NEXT:    psrlw $8, %xmm1
+; SSE41-NEXT:    packuswb %xmm2, %xmm1
+; SSE41-NEXT:    movdqa %xmm1, %xmm0
+; SSE41-NEXT:    retq
+;
+; AVX-LABEL: constant_shift_v16i8:
+; AVX:       # BB#0:
+; AVX-NEXT:    vmovdqa {{.*#+}} xmm1 = [0,1,2,3,4,5,6,7,7,6,5,4,3,2,1,0]
+; AVX-NEXT:    vpsllw $5, %xmm1, %xmm1
+; AVX-NEXT:    vpunpckhbw {{.*#+}} xmm2 = xmm0[8],xmm1[8],xmm0[9],xmm1[9],xmm0[10],xmm1[10],xmm0[11],xmm1[11],xmm0[12],xmm1[12],xmm0[13],xmm1[13],xmm0[14],xmm1[14],xmm0[15],xmm1[15]
+; AVX-NEXT:    vpunpckhbw {{.*#+}} xmm3 = xmm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
+; AVX-NEXT:    vpsraw $4, %xmm3, %xmm4
+; AVX-NEXT:    vpblendvb %xmm2, %xmm4, %xmm3, %xmm3
+; AVX-NEXT:    vpsraw $2, %xmm3, %xmm4
+; AVX-NEXT:    vpaddw %xmm2, %xmm2, %xmm2
+; AVX-NEXT:    vpblendvb %xmm2, %xmm4, %xmm3, %xmm3
+; AVX-NEXT:    vpsraw $1, %xmm3, %xmm4
+; AVX-NEXT:    vpaddw %xmm2, %xmm2, %xmm2
+; AVX-NEXT:    vpblendvb %xmm2, %xmm4, %xmm3, %xmm2
+; AVX-NEXT:    vpsrlw $8, %xmm2, %xmm2
+; AVX-NEXT:    vpunpcklbw {{.*#+}} xmm1 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3],xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7]
+; AVX-NEXT:    vpunpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
+; AVX-NEXT:    vpsraw $4, %xmm0, %xmm3
+; AVX-NEXT:    vpblendvb %xmm1, %xmm3, %xmm0, %xmm0
+; AVX-NEXT:    vpsraw $2, %xmm0, %xmm3
+; AVX-NEXT:    vpaddw %xmm1, %xmm1, %xmm1
+; AVX-NEXT:    vpblendvb %xmm1, %xmm3, %xmm0, %xmm0
+; AVX-NEXT:    vpsraw $1, %xmm0, %xmm3
+; AVX-NEXT:    vpaddw %xmm1, %xmm1, %xmm1
+; AVX-NEXT:    vpblendvb %xmm1, %xmm3, %xmm0, %xmm0
+; AVX-NEXT:    vpsrlw $8, %xmm0, %xmm0
+; AVX-NEXT:    vpackuswb %xmm2, %xmm0, %xmm0
+; AVX-NEXT:    retq
+;
+; X32-SSE-LABEL: constant_shift_v16i8:
+; X32-SSE:       # BB#0:
+; X32-SSE-NEXT:    punpckhbw {{.*#+}} xmm1 = xmm1[8],xmm0[8],xmm1[9],xmm0[9],xmm1[10],xmm0[10],xmm1[11],xmm0[11],xmm1[12],xmm0[12],xmm1[13],xmm0[13],xmm1[14],xmm0[14],xmm1[15],xmm0[15]
+; X32-SSE-NEXT:    movdqa {{.*#+}} xmm3 = [0,1,2,3,4,5,6,7,7,6,5,4,3,2,1,0]
+; X32-SSE-NEXT:    psllw $5, %xmm3
+; X32-SSE-NEXT:    punpckhbw {{.*#+}} xmm4 = xmm4[8],xmm3[8],xmm4[9],xmm3[9],xmm4[10],xmm3[10],xmm4[11],xmm3[11],xmm4[12],xmm3[12],xmm4[13],xmm3[13],xmm4[14],xmm3[14],xmm4[15],xmm3[15]
+; X32-SSE-NEXT:    pxor %xmm2, %xmm2
+; X32-SSE-NEXT:    pxor %xmm5, %xmm5
+; X32-SSE-NEXT:    pcmpgtw %xmm4, %xmm5
+; X32-SSE-NEXT:    movdqa %xmm5, %xmm6
+; X32-SSE-NEXT:    pandn %xmm1, %xmm6
+; X32-SSE-NEXT:    psraw $4, %xmm1
+; X32-SSE-NEXT:    pand %xmm5, %xmm1
+; X32-SSE-NEXT:    por %xmm6, %xmm1
+; X32-SSE-NEXT:    paddw %xmm4, %xmm4
+; X32-SSE-NEXT:    pxor %xmm5, %xmm5
+; X32-SSE-NEXT:    pcmpgtw %xmm4, %xmm5
+; X32-SSE-NEXT:    movdqa %xmm5, %xmm6
+; X32-SSE-NEXT:    pandn %xmm1, %xmm6
+; X32-SSE-NEXT:    psraw $2, %xmm1
+; X32-SSE-NEXT:    pand %xmm5, %xmm1
+; X32-SSE-NEXT:    por %xmm6, %xmm1
+; X32-SSE-NEXT:    paddw %xmm4, %xmm4
+; X32-SSE-NEXT:    pxor %xmm5, %xmm5
+; X32-SSE-NEXT:    pcmpgtw %xmm4, %xmm5
+; X32-SSE-NEXT:    movdqa %xmm5, %xmm4
+; X32-SSE-NEXT:    pandn %xmm1, %xmm4
+; X32-SSE-NEXT:    psraw $1, %xmm1
+; X32-SSE-NEXT:    pand %xmm5, %xmm1
+; X32-SSE-NEXT:    por %xmm4, %xmm1
+; X32-SSE-NEXT:    psrlw $8, %xmm1
+; X32-SSE-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
+; X32-SSE-NEXT:    punpcklbw {{.*#+}} xmm3 = xmm3[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
+; X32-SSE-NEXT:    pxor %xmm4, %xmm4
+; X32-SSE-NEXT:    pcmpgtw %xmm3, %xmm4
+; X32-SSE-NEXT:    movdqa %xmm4, %xmm5
+; X32-SSE-NEXT:    pandn %xmm0, %xmm5
+; X32-SSE-NEXT:    psraw $4, %xmm0
+; X32-SSE-NEXT:    pand %xmm4, %xmm0
+; X32-SSE-NEXT:    por %xmm5, %xmm0
+; X32-SSE-NEXT:    paddw %xmm3, %xmm3
+; X32-SSE-NEXT:    pxor %xmm4, %xmm4
+; X32-SSE-NEXT:    pcmpgtw %xmm3, %xmm4
+; X32-SSE-NEXT:    movdqa %xmm4, %xmm5
+; X32-SSE-NEXT:    pandn %xmm0, %xmm5
+; X32-SSE-NEXT:    psraw $2, %xmm0
+; X32-SSE-NEXT:    pand %xmm4, %xmm0
+; X32-SSE-NEXT:    por %xmm5, %xmm0
+; X32-SSE-NEXT:    paddw %xmm3, %xmm3
+; X32-SSE-NEXT:    pcmpgtw %xmm3, %xmm2
+; X32-SSE-NEXT:    movdqa %xmm2, %xmm3
+; X32-SSE-NEXT:    pandn %xmm0, %xmm3
+; X32-SSE-NEXT:    psraw $1, %xmm0
+; X32-SSE-NEXT:    pand %xmm2, %xmm0
+; X32-SSE-NEXT:    por %xmm3, %xmm0
+; X32-SSE-NEXT:    psrlw $8, %xmm0
+; X32-SSE-NEXT:    packuswb %xmm1, %xmm0
+; X32-SSE-NEXT:    retl
+  %shift = ashr <16 x i8> %a, <i8 0, i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 7, i8 6, i8 5, i8 4, i8 3, i8 2, i8 1, i8 0>
+  ret <16 x i8> %shift
+}
+
+;
+; Uniform Constant Shifts
+;
+
+define <2 x i64> @splatconstant_shift_v2i64(<2 x i64> %a) nounwind {
+; SSE2-LABEL: splatconstant_shift_v2i64:
+; SSE2:       # BB#0:
+; SSE2-NEXT:    movdqa %xmm0, %xmm1
+; SSE2-NEXT:    psrad $7, %xmm1
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,3,2,3]
+; SSE2-NEXT:    psrlq $7, %xmm0
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
+; SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
+; SSE2-NEXT:    retq
+;
+; SSE41-LABEL: splatconstant_shift_v2i64:
+; SSE41:       # BB#0:
+; SSE41-NEXT:    movdqa %xmm0, %xmm1
+; SSE41-NEXT:    psrad $7, %xmm1
+; SSE41-NEXT:    psrlq $7, %xmm0
+; SSE41-NEXT:    pblendw {{.*#+}} xmm0 = xmm0[0,1],xmm1[2,3],xmm0[4,5],xmm1[6,7]
+; SSE41-NEXT:    retq
+;
+; AVX1-LABEL: splatconstant_shift_v2i64:
+; AVX1:       # BB#0:
+; AVX1-NEXT:    vpsrad $7, %xmm0, %xmm1
+; AVX1-NEXT:    vpsrlq $7, %xmm0, %xmm0
+; AVX1-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0,1],xmm1[2,3],xmm0[4,5],xmm1[6,7]
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: splatconstant_shift_v2i64:
+; AVX2:       # BB#0:
+; AVX2-NEXT:    vpsrad $7, %xmm0, %xmm1
+; AVX2-NEXT:    vpsrlq $7, %xmm0, %xmm0
+; AVX2-NEXT:    vpblendd {{.*#+}} xmm0 = xmm0[0],xmm1[1],xmm0[2],xmm1[3]
+; AVX2-NEXT:    retq
+;
+; X32-SSE-LABEL: splatconstant_shift_v2i64:
+; X32-SSE:       # BB#0:
+; X32-SSE-NEXT:    movdqa %xmm0, %xmm1
+; X32-SSE-NEXT:    psrad $7, %xmm1
+; X32-SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,3,2,3]
+; X32-SSE-NEXT:    psrlq $7, %xmm0
+; X32-SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
+; X32-SSE-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
+; X32-SSE-NEXT:    retl
+  %shift = ashr <2 x i64> %a, <i64 7, i64 7>
+  ret <2 x i64> %shift
+}
+
+define <4 x i32> @splatconstant_shift_v4i32(<4 x i32> %a) nounwind {
+; SSE-LABEL: splatconstant_shift_v4i32:
+; SSE:       # BB#0:
+; SSE-NEXT:    psrad $5, %xmm0
+; SSE-NEXT:    retq
+;
+; AVX-LABEL: splatconstant_shift_v4i32:
+; AVX:       # BB#0:
+; AVX-NEXT:    vpsrad $5, %xmm0, %xmm0
+; AVX-NEXT:    retq
+;
+; X32-SSE-LABEL: splatconstant_shift_v4i32:
+; X32-SSE:       # BB#0:
+; X32-SSE-NEXT:    psrad $5, %xmm0
+; X32-SSE-NEXT:    retl
+  %shift = ashr <4 x i32> %a, <i32 5, i32 5, i32 5, i32 5>
+  ret <4 x i32> %shift
+}
+
+define <8 x i16> @splatconstant_shift_v8i16(<8 x i16> %a) nounwind {
+; SSE-LABEL: splatconstant_shift_v8i16:
+; SSE:       # BB#0:
+; SSE-NEXT:    psraw $3, %xmm0
+; SSE-NEXT:    retq
+;
+; AVX-LABEL: splatconstant_shift_v8i16:
+; AVX:       # BB#0:
+; AVX-NEXT:    vpsraw $3, %xmm0, %xmm0
+; AVX-NEXT:    retq
+;
+; X32-SSE-LABEL: splatconstant_shift_v8i16:
+; X32-SSE:       # BB#0:
+; X32-SSE-NEXT:    psraw $3, %xmm0
+; X32-SSE-NEXT:    retl
+  %shift = ashr <8 x i16> %a, <i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3>
+  ret <8 x i16> %shift
+}
+
+define <16 x i8> @splatconstant_shift_v16i8(<16 x i8> %a) nounwind {
+; SSE-LABEL: splatconstant_shift_v16i8:
+; SSE:       # BB#0:
+; SSE-NEXT:    psrlw $3, %xmm0
+; SSE-NEXT:    pand {{.*}}(%rip), %xmm0
+; SSE-NEXT:    movdqa {{.*#+}} xmm1 = [16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16]
+; SSE-NEXT:    pxor %xmm1, %xmm0
+; SSE-NEXT:    psubb %xmm1, %xmm0
+; SSE-NEXT:    retq
+;
+; AVX-LABEL: splatconstant_shift_v16i8:
+; AVX:       # BB#0:
+; AVX-NEXT:    vpsrlw $3, %xmm0, %xmm0
+; AVX-NEXT:    vpand {{.*}}(%rip), %xmm0, %xmm0
+; AVX-NEXT:    vmovdqa {{.*#+}} xmm1 = [16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16]
+; AVX-NEXT:    vpxor %xmm1, %xmm0, %xmm0
+; AVX-NEXT:    vpsubb %xmm1, %xmm0, %xmm0
+; AVX-NEXT:    retq
+;
+; X32-SSE-LABEL: splatconstant_shift_v16i8:
+; X32-SSE:       # BB#0:
+; X32-SSE-NEXT:    psrlw $3, %xmm0
+; X32-SSE-NEXT:    pand .LCPI15_0, %xmm0
+; X32-SSE-NEXT:    movdqa {{.*#+}} xmm1 = [16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16]
+; X32-SSE-NEXT:    pxor %xmm1, %xmm0
+; X32-SSE-NEXT:    psubb %xmm1, %xmm0
+; X32-SSE-NEXT:    retl
+  %shift = ashr <16 x i8> %a, <i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3>
+  ret <16 x i8> %shift
+}
diff --git a/test/CodeGen/X86/vector-shift-ashr-256.ll b/test/CodeGen/X86/vector-shift-ashr-256.ll
new file mode 100644
index 00000000000..51ded9ef5ba
--- /dev/null
+++ b/test/CodeGen/X86/vector-shift-ashr-256.ll
@@ -0,0 +1,666 @@
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mcpu=x86-64 -mattr=+avx | FileCheck %s --check-prefix=ALL --check-prefix=AVX --check-prefix=AVX1
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mcpu=x86-64 -mattr=+avx2 | FileCheck %s --check-prefix=ALL --check-prefix=AVX --check-prefix=AVX2
+
+;
+; Variable Shifts
+;
+
+define <4 x i64> @var_shift_v4i64(<4 x i64> %a, <4 x i64> %b) nounwind {
+; AVX1-LABEL: var_shift_v4i64:
+; AVX1:       # BB#0:
+; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm2
+; AVX1-NEXT:    vmovdqa {{.*#+}} xmm3 = [9223372036854775808,9223372036854775808]
+; AVX1-NEXT:    vpsrlq %xmm2, %xmm3, %xmm4
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm5 = xmm2[2,3,0,1]
+; AVX1-NEXT:    vpsrlq %xmm5, %xmm3, %xmm6
+; AVX1-NEXT:    vpblendw {{.*#+}} xmm4 = xmm4[0,1,2,3],xmm6[4,5,6,7]
+; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm6
+; AVX1-NEXT:    vpsrlq %xmm2, %xmm6, %xmm2
+; AVX1-NEXT:    vpsrlq %xmm5, %xmm6, %xmm5
+; AVX1-NEXT:    vpblendw {{.*#+}} xmm2 = xmm2[0,1,2,3],xmm5[4,5,6,7]
+; AVX1-NEXT:    vpxor %xmm4, %xmm2, %xmm2
+; AVX1-NEXT:    vpsubq %xmm4, %xmm2, %xmm2
+; AVX1-NEXT:    vpsrlq %xmm1, %xmm3, %xmm4
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm5 = xmm1[2,3,0,1]
+; AVX1-NEXT:    vpsrlq %xmm5, %xmm3, %xmm3
+; AVX1-NEXT:    vpblendw {{.*#+}} xmm3 = xmm4[0,1,2,3],xmm3[4,5,6,7]
+; AVX1-NEXT:    vpsrlq %xmm1, %xmm0, %xmm1
+; AVX1-NEXT:    vpsrlq %xmm5, %xmm0, %xmm0
+; AVX1-NEXT:    vpblendw {{.*#+}} xmm0 = xmm1[0,1,2,3],xmm0[4,5,6,7]
+; AVX1-NEXT:    vpxor %xmm3, %xmm0, %xmm0
+; AVX1-NEXT:    vpsubq %xmm3, %xmm0, %xmm0
+; AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm0
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: var_shift_v4i64:
+; AVX2:       # BB#0:
+; AVX2-NEXT:    vpbroadcastq {{.*}}(%rip), %ymm2
+; AVX2-NEXT:    vpsrlvq %ymm1, %ymm2, %ymm3
+; AVX2-NEXT:    vpxor %ymm2, %ymm0, %ymm0
+; AVX2-NEXT:    vpsrlvq %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vpsubq %ymm3, %ymm0, %ymm0
+; AVX2-NEXT:    retq
+  %shift = ashr <4 x i64> %a, %b
+  ret <4 x i64> %shift
+}
+
+define <8 x i32> @var_shift_v8i32(<8 x i32> %a, <8 x i32> %b) nounwind {
+; AVX1-LABEL: var_shift_v8i32:
+; AVX1:       # BB#0:
+; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm2
+; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm3
+; AVX1-NEXT:    vpsrldq {{.*#+}} xmm4 = xmm3[12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
+; AVX1-NEXT:    vpsrad %xmm4, %xmm2, %xmm4
+; AVX1-NEXT:    vpsrlq $32, %xmm3, %xmm5
+; AVX1-NEXT:    vpsrad %xmm5, %xmm2, %xmm5
+; AVX1-NEXT:    vpblendw {{.*#+}} xmm4 = xmm5[0,1,2,3],xmm4[4,5,6,7]
+; AVX1-NEXT:    vpxor %xmm5, %xmm5, %xmm5
+; AVX1-NEXT:    vpunpckhdq {{.*#+}} xmm6 = xmm3[2],xmm5[2],xmm3[3],xmm5[3]
+; AVX1-NEXT:    vpsrad %xmm6, %xmm2, %xmm6
+; AVX1-NEXT:    vpmovzxdq {{.*#+}} xmm3 = xmm3[0],zero,xmm3[1],zero
+; AVX1-NEXT:    vpsrad %xmm3, %xmm2, %xmm2
+; AVX1-NEXT:    vpblendw {{.*#+}} xmm2 = xmm2[0,1,2,3],xmm6[4,5,6,7]
+; AVX1-NEXT:    vpblendw {{.*#+}} xmm2 = xmm2[0,1],xmm4[2,3],xmm2[4,5],xmm4[6,7]
+; AVX1-NEXT:    vpsrldq {{.*#+}} xmm3 = xmm1[12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
+; AVX1-NEXT:    vpsrad %xmm3, %xmm0, %xmm3
+; AVX1-NEXT:    vpsrlq $32, %xmm1, %xmm4
+; AVX1-NEXT:    vpsrad %xmm4, %xmm0, %xmm4
+; AVX1-NEXT:    vpblendw {{.*#+}} xmm3 = xmm4[0,1,2,3],xmm3[4,5,6,7]
+; AVX1-NEXT:    vpunpckhdq {{.*#+}} xmm4 = xmm1[2],xmm5[2],xmm1[3],xmm5[3]
+; AVX1-NEXT:    vpsrad %xmm4, %xmm0, %xmm4
+; AVX1-NEXT:    vpmovzxdq {{.*#+}} xmm1 = xmm1[0],zero,xmm1[1],zero
+; AVX1-NEXT:    vpsrad %xmm1, %xmm0, %xmm0
+; AVX1-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0,1,2,3],xmm4[4,5,6,7]
+; AVX1-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0,1],xmm3[2,3],xmm0[4,5],xmm3[6,7]
+; AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm0
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: var_shift_v8i32:
+; AVX2:       # BB#0:
+; AVX2-NEXT:    vpsravd %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    retq
+  %shift = ashr <8 x i32> %a, %b
+  ret <8 x i32> %shift
+}
+
+define <16 x i16> @var_shift_v16i16(<16 x i16> %a, <16 x i16> %b) nounwind {
+; AVX1-LABEL: var_shift_v16i16:
+; AVX1:       # BB#0:
+; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm2
+; AVX1-NEXT:    vpsllw $12, %xmm2, %xmm3
+; AVX1-NEXT:    vpsllw $4, %xmm2, %xmm2
+; AVX1-NEXT:    vpor %xmm3, %xmm2, %xmm2
+; AVX1-NEXT:    vpaddw %xmm2, %xmm2, %xmm3
+; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm4
+; AVX1-NEXT:    vpsraw $8, %xmm4, %xmm5
+; AVX1-NEXT:    vpblendvb %xmm2, %xmm5, %xmm4, %xmm2
+; AVX1-NEXT:    vpsraw $4, %xmm2, %xmm4
+; AVX1-NEXT:    vpblendvb %xmm3, %xmm4, %xmm2, %xmm2
+; AVX1-NEXT:    vpsraw $2, %xmm2, %xmm4
+; AVX1-NEXT:    vpaddw %xmm3, %xmm3, %xmm3
+; AVX1-NEXT:    vpblendvb %xmm3, %xmm4, %xmm2, %xmm2
+; AVX1-NEXT:    vpsraw $1, %xmm2, %xmm4
+; AVX1-NEXT:    vpaddw %xmm3, %xmm3, %xmm3
+; AVX1-NEXT:    vpblendvb %xmm3, %xmm4, %xmm2, %xmm2
+; AVX1-NEXT:    vpsllw $12, %xmm1, %xmm3
+; AVX1-NEXT:    vpsllw $4, %xmm1, %xmm1
+; AVX1-NEXT:    vpor %xmm3, %xmm1, %xmm1
+; AVX1-NEXT:    vpaddw %xmm1, %xmm1, %xmm3
+; AVX1-NEXT:    vpsraw $8, %xmm0, %xmm4
+; AVX1-NEXT:    vpblendvb %xmm1, %xmm4, %xmm0, %xmm0
+; AVX1-NEXT:    vpsraw $4, %xmm0, %xmm1
+; AVX1-NEXT:    vpblendvb %xmm3, %xmm1, %xmm0, %xmm0
+; AVX1-NEXT:    vpsraw $2, %xmm0, %xmm1
+; AVX1-NEXT:    vpaddw %xmm3, %xmm3, %xmm3
+; AVX1-NEXT:    vpblendvb %xmm3, %xmm1, %xmm0, %xmm0
+; AVX1-NEXT:    vpsraw $1, %xmm0, %xmm1
+; AVX1-NEXT:    vpaddw %xmm3, %xmm3, %xmm3
+; AVX1-NEXT:    vpblendvb %xmm3, %xmm1, %xmm0, %xmm0
+; AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm0
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: var_shift_v16i16:
+; AVX2:       # BB#0:
+; AVX2-NEXT:    vpxor %ymm2, %ymm2, %ymm2
+; AVX2-NEXT:    vpunpckhwd {{.*#+}} ymm3 = ymm1[4],ymm2[4],ymm1[5],ymm2[5],ymm1[6],ymm2[6],ymm1[7],ymm2[7],ymm1[12],ymm2[12],ymm1[13],ymm2[13],ymm1[14],ymm2[14],ymm1[15],ymm2[15]
+; AVX2-NEXT:    vpunpckhwd {{.*#+}} ymm4 = ymm0[4,4,5,5,6,6,7,7,12,12,13,13,14,14,15,15]
+; AVX2-NEXT:    vpsravd %ymm3, %ymm4, %ymm3
+; AVX2-NEXT:    vpsrld $16, %ymm3, %ymm3
+; AVX2-NEXT:    vpunpcklwd {{.*#+}} ymm1 = ymm1[0],ymm2[0],ymm1[1],ymm2[1],ymm1[2],ymm2[2],ymm1[3],ymm2[3],ymm1[8],ymm2[8],ymm1[9],ymm2[9],ymm1[10],ymm2[10],ymm1[11],ymm2[11]
+; AVX2-NEXT:    vpunpcklwd {{.*#+}} ymm0 = ymm0[0,0,1,1,2,2,3,3,8,8,9,9,10,10,11,11]
+; AVX2-NEXT:    vpsravd %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vpsrld $16, %ymm0, %ymm0
+; AVX2-NEXT:    vpackusdw %ymm3, %ymm0, %ymm0
+; AVX2-NEXT:    retq
+  %shift = ashr <16 x i16> %a, %b
+  ret <16 x i16> %shift
+}
+
+define <32 x i8> @var_shift_v32i8(<32 x i8> %a, <32 x i8> %b) nounwind {
+; AVX1-LABEL: var_shift_v32i8:
+; AVX1:       # BB#0:
+; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm2
+; AVX1-NEXT:    vpsllw $5, %xmm2, %xmm2
+; AVX1-NEXT:    vpunpckhbw {{.*#+}} xmm3 = xmm0[8],xmm2[8],xmm0[9],xmm2[9],xmm0[10],xmm2[10],xmm0[11],xmm2[11],xmm0[12],xmm2[12],xmm0[13],xmm2[13],xmm0[14],xmm2[14],xmm0[15],xmm2[15]
+; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm4
+; AVX1-NEXT:    vpunpckhbw {{.*#+}} xmm5 = xmm0[8],xmm4[8],xmm0[9],xmm4[9],xmm0[10],xmm4[10],xmm0[11],xmm4[11],xmm0[12],xmm4[12],xmm0[13],xmm4[13],xmm0[14],xmm4[14],xmm0[15],xmm4[15]
+; AVX1-NEXT:    vpsraw $4, %xmm5, %xmm6
+; AVX1-NEXT:    vpblendvb %xmm3, %xmm6, %xmm5, %xmm5
+; AVX1-NEXT:    vpsraw $2, %xmm5, %xmm6
+; AVX1-NEXT:    vpaddw %xmm3, %xmm3, %xmm3
+; AVX1-NEXT:    vpblendvb %xmm3, %xmm6, %xmm5, %xmm5
+; AVX1-NEXT:    vpsraw $1, %xmm5, %xmm6
+; AVX1-NEXT:    vpaddw %xmm3, %xmm3, %xmm3
+; AVX1-NEXT:    vpblendvb %xmm3, %xmm6, %xmm5, %xmm3
+; AVX1-NEXT:    vpsrlw $8, %xmm3, %xmm3
+; AVX1-NEXT:    vpunpcklbw {{.*#+}} xmm2 = xmm0[0],xmm2[0],xmm0[1],xmm2[1],xmm0[2],xmm2[2],xmm0[3],xmm2[3],xmm0[4],xmm2[4],xmm0[5],xmm2[5],xmm0[6],xmm2[6],xmm0[7],xmm2[7]
+; AVX1-NEXT:    vpunpcklbw {{.*#+}} xmm4 = xmm0[0],xmm4[0],xmm0[1],xmm4[1],xmm0[2],xmm4[2],xmm0[3],xmm4[3],xmm0[4],xmm4[4],xmm0[5],xmm4[5],xmm0[6],xmm4[6],xmm0[7],xmm4[7]
+; AVX1-NEXT:    vpsraw $4, %xmm4, %xmm5
+; AVX1-NEXT:    vpblendvb %xmm2, %xmm5, %xmm4, %xmm4
+; AVX1-NEXT:    vpsraw $2, %xmm4, %xmm5
+; AVX1-NEXT:    vpaddw %xmm2, %xmm2, %xmm2
+; AVX1-NEXT:    vpblendvb %xmm2, %xmm5, %xmm4, %xmm4
+; AVX1-NEXT:    vpsraw $1, %xmm4, %xmm5
+; AVX1-NEXT:    vpaddw %xmm2, %xmm2, %xmm2
+; AVX1-NEXT:    vpblendvb %xmm2, %xmm5, %xmm4, %xmm2
+; AVX1-NEXT:    vpsrlw $8, %xmm2, %xmm2
+; AVX1-NEXT:    vpackuswb %xmm3, %xmm2, %xmm2
+; AVX1-NEXT:    vpsllw $5, %xmm1, %xmm1
+; AVX1-NEXT:    vpunpckhbw {{.*#+}} xmm3 = xmm0[8],xmm1[8],xmm0[9],xmm1[9],xmm0[10],xmm1[10],xmm0[11],xmm1[11],xmm0[12],xmm1[12],xmm0[13],xmm1[13],xmm0[14],xmm1[14],xmm0[15],xmm1[15]
+; AVX1-NEXT:    vpunpckhbw {{.*#+}} xmm4 = xmm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
+; AVX1-NEXT:    vpsraw $4, %xmm4, %xmm5
+; AVX1-NEXT:    vpblendvb %xmm3, %xmm5, %xmm4, %xmm4
+; AVX1-NEXT:    vpsraw $2, %xmm4, %xmm5
+; AVX1-NEXT:    vpaddw %xmm3, %xmm3, %xmm3
+; AVX1-NEXT:    vpblendvb %xmm3, %xmm5, %xmm4, %xmm4
+; AVX1-NEXT:    vpsraw $1, %xmm4, %xmm5
+; AVX1-NEXT:    vpaddw %xmm3, %xmm3, %xmm3
+; AVX1-NEXT:    vpblendvb %xmm3, %xmm5, %xmm4, %xmm3
+; AVX1-NEXT:    vpsrlw $8, %xmm3, %xmm3
+; AVX1-NEXT:    vpunpcklbw {{.*#+}} xmm1 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3],xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7]
+; AVX1-NEXT:    vpunpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
+; AVX1-NEXT:    vpsraw $4, %xmm0, %xmm4
+; AVX1-NEXT:    vpblendvb %xmm1, %xmm4, %xmm0, %xmm0
+; AVX1-NEXT:    vpsraw $2, %xmm0, %xmm4
+; AVX1-NEXT:    vpaddw %xmm1, %xmm1, %xmm1
+; AVX1-NEXT:    vpblendvb %xmm1, %xmm4, %xmm0, %xmm0
+; AVX1-NEXT:    vpsraw $1, %xmm0, %xmm4
+; AVX1-NEXT:    vpaddw %xmm1, %xmm1, %xmm1
+; AVX1-NEXT:    vpblendvb %xmm1, %xmm4, %xmm0, %xmm0
+; AVX1-NEXT:    vpsrlw $8, %xmm0, %xmm0
+; AVX1-NEXT:    vpackuswb %xmm3, %xmm0, %xmm0
+; AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm0
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: var_shift_v32i8:
+; AVX2:       # BB#0:
+; AVX2-NEXT:    vpsllw $5, %ymm1, %ymm1
+; AVX2-NEXT:    vpunpckhbw {{.*#+}} ymm2 = ymm0[8],ymm1[8],ymm0[9],ymm1[9],ymm0[10],ymm1[10],ymm0[11],ymm1[11],ymm0[12],ymm1[12],ymm0[13],ymm1[13],ymm0[14],ymm1[14],ymm0[15],ymm1[15],ymm0[24],ymm1[24],ymm0[25],ymm1[25],ymm0[26],ymm1[26],ymm0[27],ymm1[27],ymm0[28],ymm1[28],ymm0[29],ymm1[29],ymm0[30],ymm1[30],ymm0[31],ymm1[31]
+; AVX2-NEXT:    vpunpckhbw {{.*#+}} ymm3 = ymm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31]
+; AVX2-NEXT:    vpsraw $4, %ymm3, %ymm4
+; AVX2-NEXT:    vpblendvb %ymm2, %ymm4, %ymm3, %ymm3
+; AVX2-NEXT:    vpsraw $2, %ymm3, %ymm4
+; AVX2-NEXT:    vpaddw %ymm2, %ymm2, %ymm2
+; AVX2-NEXT:    vpblendvb %ymm2, %ymm4, %ymm3, %ymm3
+; AVX2-NEXT:    vpsraw $1, %ymm3, %ymm4
+; AVX2-NEXT:    vpaddw %ymm2, %ymm2, %ymm2
+; AVX2-NEXT:    vpblendvb %ymm2, %ymm4, %ymm3, %ymm2
+; AVX2-NEXT:    vpsrlw $8, %ymm2, %ymm2
+; AVX2-NEXT:    vpunpcklbw {{.*#+}} ymm1 = ymm0[0],ymm1[0],ymm0[1],ymm1[1],ymm0[2],ymm1[2],ymm0[3],ymm1[3],ymm0[4],ymm1[4],ymm0[5],ymm1[5],ymm0[6],ymm1[6],ymm0[7],ymm1[7],ymm0[16],ymm1[16],ymm0[17],ymm1[17],ymm0[18],ymm1[18],ymm0[19],ymm1[19],ymm0[20],ymm1[20],ymm0[21],ymm1[21],ymm0[22],ymm1[22],ymm0[23],ymm1[23]
+; AVX2-NEXT:    vpunpcklbw {{.*#+}} ymm0 = ymm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]
+; AVX2-NEXT:    vpsraw $4, %ymm0, %ymm3
+; AVX2-NEXT:    vpblendvb %ymm1, %ymm3, %ymm0, %ymm0
+; AVX2-NEXT:    vpsraw $2, %ymm0, %ymm3
+; AVX2-NEXT:    vpaddw %ymm1, %ymm1, %ymm1
+; AVX2-NEXT:    vpblendvb %ymm1, %ymm3, %ymm0, %ymm0
+; AVX2-NEXT:    vpsraw $1, %ymm0, %ymm3
+; AVX2-NEXT:    vpaddw %ymm1, %ymm1, %ymm1
+; AVX2-NEXT:    vpblendvb %ymm1, %ymm3, %ymm0, %ymm0
+; AVX2-NEXT:    vpsrlw $8, %ymm0, %ymm0
+; AVX2-NEXT:    vpackuswb %ymm2, %ymm0, %ymm0
+; AVX2-NEXT:    retq
+  %shift = ashr <32 x i8> %a, %b
+  ret <32 x i8> %shift
+}
+
+;
+; Uniform Variable Shifts
+;
+
+define <4 x i64> @splatvar_shift_v4i64(<4 x i64> %a, <4 x i64> %b) nounwind {
+; AVX1-LABEL: splatvar_shift_v4i64:
+; AVX1:       # BB#0:
+; AVX1-NEXT:    vmovdqa {{.*#+}} xmm2 = [9223372036854775808,9223372036854775808]
+; AVX1-NEXT:    vpsrlq %xmm1, %xmm2, %xmm2
+; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm3
+; AVX1-NEXT:    vpsrlq %xmm1, %xmm3, %xmm3
+; AVX1-NEXT:    vpxor %xmm2, %xmm3, %xmm3
+; AVX1-NEXT:    vpsubq %xmm2, %xmm3, %xmm3
+; AVX1-NEXT:    vpsrlq %xmm1, %xmm0, %xmm0
+; AVX1-NEXT:    vpxor %xmm2, %xmm0, %xmm0
+; AVX1-NEXT:    vpsubq %xmm2, %xmm0, %xmm0
+; AVX1-NEXT:    vinsertf128 $1, %xmm3, %ymm0, %ymm0
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: splatvar_shift_v4i64:
+; AVX2:       # BB#0:
+; AVX2-NEXT:    vpbroadcastq {{.*}}(%rip), %ymm2
+; AVX2-NEXT:    vpsrlq %xmm1, %ymm2, %ymm2
+; AVX2-NEXT:    vpsrlq %xmm1, %ymm0, %ymm0
+; AVX2-NEXT:    vpxor %ymm2, %ymm0, %ymm0
+; AVX2-NEXT:    vpsubq %ymm2, %ymm0, %ymm0
+; AVX2-NEXT:    retq
+  %splat = shufflevector <4 x i64> %b, <4 x i64> undef, <4 x i32> zeroinitializer
+  %shift = ashr <4 x i64> %a, %splat
+  ret <4 x i64> %shift
+}
+
+define <8 x i32> @splatvar_shift_v8i32(<8 x i32> %a, <8 x i32> %b) nounwind {
+; AVX1-LABEL: splatvar_shift_v8i32:
+; AVX1:       # BB#0:
+; AVX1-NEXT:    vpxor %xmm2, %xmm2, %xmm2
+; AVX1-NEXT:    vpblendw {{.*#+}} xmm1 = xmm1[0,1],xmm2[2,3,4,5,6,7]
+; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm2
+; AVX1-NEXT:    vpsrad %xmm1, %xmm2, %xmm2
+; AVX1-NEXT:    vpsrad %xmm1, %xmm0, %xmm0
+; AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm0
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: splatvar_shift_v8i32:
+; AVX2:       # BB#0:
+; AVX2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
+; AVX2-NEXT:    vpblendw {{.*#+}} xmm1 = xmm1[0,1],xmm2[2,3,4,5,6,7]
+; AVX2-NEXT:    vpsrad %xmm1, %ymm0, %ymm0
+; AVX2-NEXT:    retq
+  %splat = shufflevector <8 x i32> %b, <8 x i32> undef, <8 x i32> zeroinitializer
+  %shift = ashr <8 x i32> %a, %splat
+  ret <8 x i32> %shift
+}
+
+define <16 x i16> @splatvar_shift_v16i16(<16 x i16> %a, <16 x i16> %b) nounwind {
+; AVX1-LABEL: splatvar_shift_v16i16:
+; AVX1:       # BB#0:
+; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm2
+; AVX1-NEXT:    vmovd %xmm1, %eax
+; AVX1-NEXT:    movzwl %ax, %eax
+; AVX1-NEXT:    vmovd %eax, %xmm1
+; AVX1-NEXT:    vpsraw %xmm1, %xmm2, %xmm2
+; AVX1-NEXT:    vpsraw %xmm1, %xmm0, %xmm0
+; AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm0
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: splatvar_shift_v16i16:
+; AVX2:       # BB#0:
+; AVX2-NEXT:    vmovd %xmm1, %eax
+; AVX2-NEXT:    movzwl %ax, %eax
+; AVX2-NEXT:    vmovd %eax, %xmm1
+; AVX2-NEXT:    vpsraw %xmm1, %ymm0, %ymm0
+; AVX2-NEXT:    retq
+  %splat = shufflevector <16 x i16> %b, <16 x i16> undef, <16 x i32> zeroinitializer
+  %shift = ashr <16 x i16> %a, %splat
+  ret <16 x i16> %shift
+}
+
+define <32 x i8> @splatvar_shift_v32i8(<32 x i8> %a, <32 x i8> %b) nounwind {
+; AVX1-LABEL: splatvar_shift_v32i8:
+; AVX1:       # BB#0:
+; AVX1-NEXT:    vpxor %xmm2, %xmm2, %xmm2
+; AVX1-NEXT:    vpshufb %xmm2, %xmm1, %xmm1
+; AVX1-NEXT:    vpsllw $5, %xmm1, %xmm1
+; AVX1-NEXT:    vpunpckhbw {{.*#+}} xmm2 = xmm0[8],xmm1[8],xmm0[9],xmm1[9],xmm0[10],xmm1[10],xmm0[11],xmm1[11],xmm0[12],xmm1[12],xmm0[13],xmm1[13],xmm0[14],xmm1[14],xmm0[15],xmm1[15]
+; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm3
+; AVX1-NEXT:    vpunpckhbw {{.*#+}} xmm4 = xmm0[8],xmm3[8],xmm0[9],xmm3[9],xmm0[10],xmm3[10],xmm0[11],xmm3[11],xmm0[12],xmm3[12],xmm0[13],xmm3[13],xmm0[14],xmm3[14],xmm0[15],xmm3[15]
+; AVX1-NEXT:    vpsraw $4, %xmm4, %xmm5
+; AVX1-NEXT:    vpblendvb %xmm2, %xmm5, %xmm4, %xmm4
+; AVX1-NEXT:    vpsraw $2, %xmm4, %xmm5
+; AVX1-NEXT:    vpaddw %xmm2, %xmm2, %xmm6
+; AVX1-NEXT:    vpblendvb %xmm6, %xmm5, %xmm4, %xmm4
+; AVX1-NEXT:    vpsraw $1, %xmm4, %xmm5
+; AVX1-NEXT:    vpaddw %xmm6, %xmm6, %xmm9
+; AVX1-NEXT:    vpblendvb %xmm9, %xmm5, %xmm4, %xmm4
+; AVX1-NEXT:    vpsrlw $8, %xmm4, %xmm8
+; AVX1-NEXT:    vpunpcklbw {{.*#+}} xmm1 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3],xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7]
+; AVX1-NEXT:    vpunpcklbw {{.*#+}} xmm3 = xmm0[0],xmm3[0],xmm0[1],xmm3[1],xmm0[2],xmm3[2],xmm0[3],xmm3[3],xmm0[4],xmm3[4],xmm0[5],xmm3[5],xmm0[6],xmm3[6],xmm0[7],xmm3[7]
+; AVX1-NEXT:    vpsraw $4, %xmm3, %xmm5
+; AVX1-NEXT:    vpblendvb %xmm1, %xmm5, %xmm3, %xmm3
+; AVX1-NEXT:    vpsraw $2, %xmm3, %xmm5
+; AVX1-NEXT:    vpaddw %xmm1, %xmm1, %xmm4
+; AVX1-NEXT:    vpblendvb %xmm4, %xmm5, %xmm3, %xmm3
+; AVX1-NEXT:    vpsraw $1, %xmm3, %xmm5
+; AVX1-NEXT:    vpaddw %xmm4, %xmm4, %xmm7
+; AVX1-NEXT:    vpblendvb %xmm7, %xmm5, %xmm3, %xmm3
+; AVX1-NEXT:    vpsrlw $8, %xmm3, %xmm3
+; AVX1-NEXT:    vpackuswb %xmm8, %xmm3, %xmm8
+; AVX1-NEXT:    vpunpckhbw {{.*#+}} xmm5 = xmm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
+; AVX1-NEXT:    vpsraw $4, %xmm5, %xmm3
+; AVX1-NEXT:    vpblendvb %xmm2, %xmm3, %xmm5, %xmm2
+; AVX1-NEXT:    vpsraw $2, %xmm2, %xmm3
+; AVX1-NEXT:    vpblendvb %xmm6, %xmm3, %xmm2, %xmm2
+; AVX1-NEXT:    vpsraw $1, %xmm2, %xmm3
+; AVX1-NEXT:    vpblendvb %xmm9, %xmm3, %xmm2, %xmm2
+; AVX1-NEXT:    vpsrlw $8, %xmm2, %xmm2
+; AVX1-NEXT:    vpunpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
+; AVX1-NEXT:    vpsraw $4, %xmm0, %xmm3
+; AVX1-NEXT:    vpblendvb %xmm1, %xmm3, %xmm0, %xmm0
+; AVX1-NEXT:    vpsraw $2, %xmm0, %xmm1
+; AVX1-NEXT:    vpblendvb %xmm4, %xmm1, %xmm0, %xmm0
+; AVX1-NEXT:    vpsraw $1, %xmm0, %xmm1
+; AVX1-NEXT:    vpblendvb %xmm7, %xmm1, %xmm0, %xmm0
+; AVX1-NEXT:    vpsrlw $8, %xmm0, %xmm0
+; AVX1-NEXT:    vpackuswb %xmm2, %xmm0, %xmm0
+; AVX1-NEXT:    vinsertf128 $1, %xmm8, %ymm0, %ymm0
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: splatvar_shift_v32i8:
+; AVX2:       # BB#0:
+; AVX2-NEXT:    vpbroadcastb %xmm1, %ymm1
+; AVX2-NEXT:    vpsllw $5, %ymm1, %ymm1
+; AVX2-NEXT:    vpunpckhbw {{.*#+}} ymm2 = ymm0[8],ymm1[8],ymm0[9],ymm1[9],ymm0[10],ymm1[10],ymm0[11],ymm1[11],ymm0[12],ymm1[12],ymm0[13],ymm1[13],ymm0[14],ymm1[14],ymm0[15],ymm1[15],ymm0[24],ymm1[24],ymm0[25],ymm1[25],ymm0[26],ymm1[26],ymm0[27],ymm1[27],ymm0[28],ymm1[28],ymm0[29],ymm1[29],ymm0[30],ymm1[30],ymm0[31],ymm1[31]
+; AVX2-NEXT:    vpunpckhbw {{.*#+}} ymm3 = ymm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31]
+; AVX2-NEXT:    vpsraw $4, %ymm3, %ymm4
+; AVX2-NEXT:    vpblendvb %ymm2, %ymm4, %ymm3, %ymm3
+; AVX2-NEXT:    vpsraw $2, %ymm3, %ymm4
+; AVX2-NEXT:    vpaddw %ymm2, %ymm2, %ymm2
+; AVX2-NEXT:    vpblendvb %ymm2, %ymm4, %ymm3, %ymm3
+; AVX2-NEXT:    vpsraw $1, %ymm3, %ymm4
+; AVX2-NEXT:    vpaddw %ymm2, %ymm2, %ymm2
+; AVX2-NEXT:    vpblendvb %ymm2, %ymm4, %ymm3, %ymm2
+; AVX2-NEXT:    vpsrlw $8, %ymm2, %ymm2
+; AVX2-NEXT:    vpunpcklbw {{.*#+}} ymm1 = ymm0[0],ymm1[0],ymm0[1],ymm1[1],ymm0[2],ymm1[2],ymm0[3],ymm1[3],ymm0[4],ymm1[4],ymm0[5],ymm1[5],ymm0[6],ymm1[6],ymm0[7],ymm1[7],ymm0[16],ymm1[16],ymm0[17],ymm1[17],ymm0[18],ymm1[18],ymm0[19],ymm1[19],ymm0[20],ymm1[20],ymm0[21],ymm1[21],ymm0[22],ymm1[22],ymm0[23],ymm1[23]
+; AVX2-NEXT:    vpunpcklbw {{.*#+}} ymm0 = ymm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]
+; AVX2-NEXT:    vpsraw $4, %ymm0, %ymm3
+; AVX2-NEXT:    vpblendvb %ymm1, %ymm3, %ymm0, %ymm0
+; AVX2-NEXT:    vpsraw $2, %ymm0, %ymm3
+; AVX2-NEXT:    vpaddw %ymm1, %ymm1, %ymm1
+; AVX2-NEXT:    vpblendvb %ymm1, %ymm3, %ymm0, %ymm0
+; AVX2-NEXT:    vpsraw $1, %ymm0, %ymm3
+; AVX2-NEXT:    vpaddw %ymm1, %ymm1, %ymm1
+; AVX2-NEXT:    vpblendvb %ymm1, %ymm3, %ymm0, %ymm0
+; AVX2-NEXT:    vpsrlw $8, %ymm0, %ymm0
+; AVX2-NEXT:    vpackuswb %ymm2, %ymm0, %ymm0
+; AVX2-NEXT:    retq
+  %splat = shufflevector <32 x i8> %b, <32 x i8> undef, <32 x i32> zeroinitializer
+  %shift = ashr <32 x i8> %a, %splat
+  ret <32 x i8> %shift
+}
+
+;
+; Constant Shifts
+;
+
+define <4 x i64> @constant_shift_v4i64(<4 x i64> %a) nounwind {
+; AVX1-LABEL: constant_shift_v4i64:
+; AVX1:       # BB#0:
+; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
+; AVX1-NEXT:    vpsrlq $62, %xmm1, %xmm2
+; AVX1-NEXT:    vpsrlq $31, %xmm1, %xmm1
+; AVX1-NEXT:    vpblendw {{.*#+}} xmm1 = xmm1[0,1,2,3],xmm2[4,5,6,7]
+; AVX1-NEXT:    vmovdqa {{.*#+}} xmm2 = [4294967296,2]
+; AVX1-NEXT:    vpxor %xmm2, %xmm1, %xmm1
+; AVX1-NEXT:    vpsubq %xmm2, %xmm1, %xmm1
+; AVX1-NEXT:    vpsrlq $7, %xmm0, %xmm2
+; AVX1-NEXT:    vpsrlq $1, %xmm0, %xmm0
+; AVX1-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0,1,2,3],xmm2[4,5,6,7]
+; AVX1-NEXT:    vmovdqa {{.*#+}} xmm2 = [4611686018427387904,72057594037927936]
+; AVX1-NEXT:    vpxor %xmm2, %xmm0, %xmm0
+; AVX1-NEXT:    vpsubq %xmm2, %xmm0, %xmm0
+; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: constant_shift_v4i64:
+; AVX2:       # BB#0:
+; AVX2-NEXT:    vpsrlvq {{.*}}(%rip), %ymm0, %ymm0
+; AVX2-NEXT:    vmovdqa {{.*#+}} ymm1 = [4611686018427387904,72057594037927936,4294967296,2]
+; AVX2-NEXT:    vpxor %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vpsubq %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    retq
+  %shift = ashr <4 x i64> %a, <i64 1, i64 7, i64 31, i64 62>
+  ret <4 x i64> %shift
+}
+
+define <8 x i32> @constant_shift_v8i32(<8 x i32> %a) nounwind {
+; AVX1-LABEL: constant_shift_v8i32:
+; AVX1:       # BB#0:
+; AVX1-NEXT:    vpsrad $7, %xmm0, %xmm1
+; AVX1-NEXT:    vpsrad $5, %xmm0, %xmm2
+; AVX1-NEXT:    vpblendw {{.*#+}} xmm1 = xmm2[0,1,2,3],xmm1[4,5,6,7]
+; AVX1-NEXT:    vpsrad $6, %xmm0, %xmm2
+; AVX1-NEXT:    vpsrad $4, %xmm0, %xmm3
+; AVX1-NEXT:    vpblendw {{.*#+}} xmm2 = xmm3[0,1,2,3],xmm2[4,5,6,7]
+; AVX1-NEXT:    vpblendw {{.*#+}} xmm1 = xmm2[0,1],xmm1[2,3],xmm2[4,5],xmm1[6,7]
+; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm0
+; AVX1-NEXT:    vpsrad $7, %xmm0, %xmm2
+; AVX1-NEXT:    vpsrad $9, %xmm0, %xmm3
+; AVX1-NEXT:    vpblendw {{.*#+}} xmm2 = xmm3[0,1,2,3],xmm2[4,5,6,7]
+; AVX1-NEXT:    vpsrad $8, %xmm0, %xmm0
+; AVX1-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0,1],xmm2[2,3],xmm0[4,5],xmm2[6,7]
+; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: constant_shift_v8i32:
+; AVX2:       # BB#0:
+; AVX2-NEXT:    vpsravd {{.*}}(%rip), %ymm0, %ymm0
+; AVX2-NEXT:    retq
+  %shift = ashr <8 x i32> %a, <i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 8, i32 7>
+  ret <8 x i32> %shift
+}
+
+define <16 x i16> @constant_shift_v16i16(<16 x i16> %a) nounwind {
+; AVX1-LABEL: constant_shift_v16i16:
+; AVX1:       # BB#0:
+; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
+; AVX1-NEXT:    vpsraw $8, %xmm1, %xmm2
+; AVX1-NEXT:    vmovdqa {{.*#+}} xmm3 = [32896,37008,41120,45232,49344,53456,57568,61680]
+; AVX1-NEXT:    vpblendvb %xmm3, %xmm2, %xmm1, %xmm1
+; AVX1-NEXT:    vpsraw $4, %xmm1, %xmm2
+; AVX1-NEXT:    vmovdqa {{.*#+}} xmm3 = [256,8480,16704,24928,33152,41376,49600,57824]
+; AVX1-NEXT:    vpblendvb %xmm3, %xmm2, %xmm1, %xmm1
+; AVX1-NEXT:    vpsraw $2, %xmm1, %xmm2
+; AVX1-NEXT:    vmovdqa {{.*#+}} xmm3 = [512,16960,33408,49856,768,17216,33664,50112]
+; AVX1-NEXT:    vpblendvb %xmm3, %xmm2, %xmm1, %xmm1
+; AVX1-NEXT:    vpsraw $1, %xmm1, %xmm2
+; AVX1-NEXT:    vmovdqa {{.*#+}} xmm3 = [1024,33920,1280,34176,1536,34432,1792,34688]
+; AVX1-NEXT:    vpblendvb %xmm3, %xmm2, %xmm1, %xmm1
+; AVX1-NEXT:    vpsraw $8, %xmm0, %xmm2
+; AVX1-NEXT:    vmovdqa {{.*#+}} xmm3 = [0,4112,8224,12336,16448,20560,24672,28784]
+; AVX1-NEXT:    vpblendvb %xmm3, %xmm2, %xmm0, %xmm0
+; AVX1-NEXT:    vpsraw $4, %xmm0, %xmm2
+; AVX1-NEXT:    vmovdqa {{.*#+}} xmm3 = [0,8224,16448,24672,32896,41120,49344,57568]
+; AVX1-NEXT:    vpblendvb %xmm3, %xmm2, %xmm0, %xmm0
+; AVX1-NEXT:    vpsraw $2, %xmm0, %xmm2
+; AVX1-NEXT:    vmovdqa {{.*#+}} xmm3 = [0,16448,32896,49344,256,16704,33152,49600]
+; AVX1-NEXT:    vpblendvb %xmm3, %xmm2, %xmm0, %xmm0
+; AVX1-NEXT:    vpsraw $1, %xmm0, %xmm2
+; AVX1-NEXT:    vmovdqa {{.*#+}} xmm3 = [0,32896,256,33152,512,33408,768,33664]
+; AVX1-NEXT:    vpblendvb %xmm3, %xmm2, %xmm0, %xmm0
+; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: constant_shift_v16i16:
+; AVX2:       # BB#0:
+; AVX2-NEXT:    vpxor %ymm1, %ymm1, %ymm1
+; AVX2-NEXT:    vmovdqa {{.*#+}} ymm2 = [0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15]
+; AVX2-NEXT:    vpunpckhwd {{.*#+}} ymm3 = ymm2[4],ymm1[4],ymm2[5],ymm1[5],ymm2[6],ymm1[6],ymm2[7],ymm1[7],ymm2[12],ymm1[12],ymm2[13],ymm1[13],ymm2[14],ymm1[14],ymm2[15],ymm1[15]
+; AVX2-NEXT:    vpunpckhwd {{.*#+}} ymm4 = ymm0[4,4,5,5,6,6,7,7,12,12,13,13,14,14,15,15]
+; AVX2-NEXT:    vpsravd %ymm3, %ymm4, %ymm3
+; AVX2-NEXT:    vpsrld $16, %ymm3, %ymm3
+; AVX2-NEXT:    vpunpcklwd {{.*#+}} ymm1 = ymm2[0],ymm1[0],ymm2[1],ymm1[1],ymm2[2],ymm1[2],ymm2[3],ymm1[3],ymm2[8],ymm1[8],ymm2[9],ymm1[9],ymm2[10],ymm1[10],ymm2[11],ymm1[11]
+; AVX2-NEXT:    vpunpcklwd {{.*#+}} ymm0 = ymm0[0,0,1,1,2,2,3,3,8,8,9,9,10,10,11,11]
+; AVX2-NEXT:    vpsravd %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vpsrld $16, %ymm0, %ymm0
+; AVX2-NEXT:    vpackusdw %ymm3, %ymm0, %ymm0
+; AVX2-NEXT:    retq
+  %shift = ashr <16 x i16> %a, <i16 0, i16 1, i16 2, i16 3, i16 4, i16 5, i16 6, i16 7, i16 8, i16 9, i16 10, i16 11, i16 12, i16 13, i16 14, i16 15>
+  ret <16 x i16> %shift
+}
+
+define <32 x i8> @constant_shift_v32i8(<32 x i8> %a) nounwind {
+; AVX1-LABEL: constant_shift_v32i8:
+; AVX1:       # BB#0:
+; AVX1-NEXT:    vmovdqa {{.*#+}} xmm1 = [0,1,2,3,4,5,6,7,7,6,5,4,3,2,1,0]
+; AVX1-NEXT:    vpsllw $5, %xmm1, %xmm1
+; AVX1-NEXT:    vpunpckhbw {{.*#+}} xmm2 = xmm0[8],xmm1[8],xmm0[9],xmm1[9],xmm0[10],xmm1[10],xmm0[11],xmm1[11],xmm0[12],xmm1[12],xmm0[13],xmm1[13],xmm0[14],xmm1[14],xmm0[15],xmm1[15]
+; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm3
+; AVX1-NEXT:    vpunpckhbw {{.*#+}} xmm4 = xmm0[8],xmm3[8],xmm0[9],xmm3[9],xmm0[10],xmm3[10],xmm0[11],xmm3[11],xmm0[12],xmm3[12],xmm0[13],xmm3[13],xmm0[14],xmm3[14],xmm0[15],xmm3[15]
+; AVX1-NEXT:    vpsraw $4, %xmm4, %xmm5
+; AVX1-NEXT:    vpblendvb %xmm2, %xmm5, %xmm4, %xmm4
+; AVX1-NEXT:    vpsraw $2, %xmm4, %xmm5
+; AVX1-NEXT:    vpaddw %xmm2, %xmm2, %xmm6
+; AVX1-NEXT:    vpblendvb %xmm6, %xmm5, %xmm4, %xmm4
+; AVX1-NEXT:    vpsraw $1, %xmm4, %xmm5
+; AVX1-NEXT:    vpaddw %xmm6, %xmm6, %xmm9
+; AVX1-NEXT:    vpblendvb %xmm9, %xmm5, %xmm4, %xmm4
+; AVX1-NEXT:    vpsrlw $8, %xmm4, %xmm8
+; AVX1-NEXT:    vpunpcklbw {{.*#+}} xmm1 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3],xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7]
+; AVX1-NEXT:    vpunpcklbw {{.*#+}} xmm3 = xmm0[0],xmm3[0],xmm0[1],xmm3[1],xmm0[2],xmm3[2],xmm0[3],xmm3[3],xmm0[4],xmm3[4],xmm0[5],xmm3[5],xmm0[6],xmm3[6],xmm0[7],xmm3[7]
+; AVX1-NEXT:    vpsraw $4, %xmm3, %xmm5
+; AVX1-NEXT:    vpblendvb %xmm1, %xmm5, %xmm3, %xmm3
+; AVX1-NEXT:    vpsraw $2, %xmm3, %xmm5
+; AVX1-NEXT:    vpaddw %xmm1, %xmm1, %xmm4
+; AVX1-NEXT:    vpblendvb %xmm4, %xmm5, %xmm3, %xmm3
+; AVX1-NEXT:    vpsraw $1, %xmm3, %xmm5
+; AVX1-NEXT:    vpaddw %xmm4, %xmm4, %xmm7
+; AVX1-NEXT:    vpblendvb %xmm7, %xmm5, %xmm3, %xmm3
+; AVX1-NEXT:    vpsrlw $8, %xmm3, %xmm3
+; AVX1-NEXT:    vpackuswb %xmm8, %xmm3, %xmm8
+; AVX1-NEXT:    vpunpckhbw {{.*#+}} xmm5 = xmm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
+; AVX1-NEXT:    vpsraw $4, %xmm5, %xmm3
+; AVX1-NEXT:    vpblendvb %xmm2, %xmm3, %xmm5, %xmm2
+; AVX1-NEXT:    vpsraw $2, %xmm2, %xmm3
+; AVX1-NEXT:    vpblendvb %xmm6, %xmm3, %xmm2, %xmm2
+; AVX1-NEXT:    vpsraw $1, %xmm2, %xmm3
+; AVX1-NEXT:    vpblendvb %xmm9, %xmm3, %xmm2, %xmm2
+; AVX1-NEXT:    vpsrlw $8, %xmm2, %xmm2
+; AVX1-NEXT:    vpunpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
+; AVX1-NEXT:    vpsraw $4, %xmm0, %xmm3
+; AVX1-NEXT:    vpblendvb %xmm1, %xmm3, %xmm0, %xmm0
+; AVX1-NEXT:    vpsraw $2, %xmm0, %xmm1
+; AVX1-NEXT:    vpblendvb %xmm4, %xmm1, %xmm0, %xmm0
+; AVX1-NEXT:    vpsraw $1, %xmm0, %xmm1
+; AVX1-NEXT:    vpblendvb %xmm7, %xmm1, %xmm0, %xmm0
+; AVX1-NEXT:    vpsrlw $8, %xmm0, %xmm0
+; AVX1-NEXT:    vpackuswb %xmm2, %xmm0, %xmm0
+; AVX1-NEXT:    vinsertf128 $1, %xmm8, %ymm0, %ymm0
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: constant_shift_v32i8:
+; AVX2:       # BB#0:
+; AVX2-NEXT:    vmovdqa {{.*#+}} ymm1 = [0,1,2,3,4,5,6,7,7,6,5,4,3,2,1,0,0,1,2,3,4,5,6,7,7,6,5,4,3,2,1,0]
+; AVX2-NEXT:    vpsllw $5, %ymm1, %ymm1
+; AVX2-NEXT:    vpunpckhbw {{.*#+}} ymm2 = ymm0[8],ymm1[8],ymm0[9],ymm1[9],ymm0[10],ymm1[10],ymm0[11],ymm1[11],ymm0[12],ymm1[12],ymm0[13],ymm1[13],ymm0[14],ymm1[14],ymm0[15],ymm1[15],ymm0[24],ymm1[24],ymm0[25],ymm1[25],ymm0[26],ymm1[26],ymm0[27],ymm1[27],ymm0[28],ymm1[28],ymm0[29],ymm1[29],ymm0[30],ymm1[30],ymm0[31],ymm1[31]
+; AVX2-NEXT:    vpunpckhbw {{.*#+}} ymm3 = ymm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31]
+; AVX2-NEXT:    vpsraw $4, %ymm3, %ymm4
+; AVX2-NEXT:    vpblendvb %ymm2, %ymm4, %ymm3, %ymm3
+; AVX2-NEXT:    vpsraw $2, %ymm3, %ymm4
+; AVX2-NEXT:    vpaddw %ymm2, %ymm2, %ymm2
+; AVX2-NEXT:    vpblendvb %ymm2, %ymm4, %ymm3, %ymm3
+; AVX2-NEXT:    vpsraw $1, %ymm3, %ymm4
+; AVX2-NEXT:    vpaddw %ymm2, %ymm2, %ymm2
+; AVX2-NEXT:    vpblendvb %ymm2, %ymm4, %ymm3, %ymm2
+; AVX2-NEXT:    vpsrlw $8, %ymm2, %ymm2
+; AVX2-NEXT:    vpunpcklbw {{.*#+}} ymm1 = ymm0[0],ymm1[0],ymm0[1],ymm1[1],ymm0[2],ymm1[2],ymm0[3],ymm1[3],ymm0[4],ymm1[4],ymm0[5],ymm1[5],ymm0[6],ymm1[6],ymm0[7],ymm1[7],ymm0[16],ymm1[16],ymm0[17],ymm1[17],ymm0[18],ymm1[18],ymm0[19],ymm1[19],ymm0[20],ymm1[20],ymm0[21],ymm1[21],ymm0[22],ymm1[22],ymm0[23],ymm1[23]
+; AVX2-NEXT:    vpunpcklbw {{.*#+}} ymm0 = ymm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]
+; AVX2-NEXT:    vpsraw $4, %ymm0, %ymm3
+; AVX2-NEXT:    vpblendvb %ymm1, %ymm3, %ymm0, %ymm0
+; AVX2-NEXT:    vpsraw $2, %ymm0, %ymm3
+; AVX2-NEXT:    vpaddw %ymm1, %ymm1, %ymm1
+; AVX2-NEXT:    vpblendvb %ymm1, %ymm3, %ymm0, %ymm0
+; AVX2-NEXT:    vpsraw $1, %ymm0, %ymm3
+; AVX2-NEXT:    vpaddw %ymm1, %ymm1, %ymm1
+; AVX2-NEXT:    vpblendvb %ymm1, %ymm3, %ymm0, %ymm0
+; AVX2-NEXT:    vpsrlw $8, %ymm0, %ymm0
+; AVX2-NEXT:    vpackuswb %ymm2, %ymm0, %ymm0
+; AVX2-NEXT:    retq
+  %shift = ashr <32 x i8> %a, <i8 0, i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 7, i8 6, i8 5, i8 4, i8 3, i8 2, i8 1, i8 0, i8 0, i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 7, i8 6, i8 5, i8 4, i8 3, i8 2, i8 1, i8 0>
+  ret <32 x i8> %shift
+}
+
+;
+; Uniform Constant Shifts
+;
+
+define <4 x i64> @splatconstant_shift_v4i64(<4 x i64> %a) nounwind {
+; AVX1-LABEL: splatconstant_shift_v4i64:
+; AVX1:       # BB#0:
+; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
+; AVX1-NEXT:    vpsrad $7, %xmm1, %xmm2
+; AVX1-NEXT:    vpsrlq $7, %xmm1, %xmm1
+; AVX1-NEXT:    vpblendw {{.*#+}} xmm1 = xmm1[0,1],xmm2[2,3],xmm1[4,5],xmm2[6,7]
+; AVX1-NEXT:    vpsrad $7, %xmm0, %xmm2
+; AVX1-NEXT:    vpsrlq $7, %xmm0, %xmm0
+; AVX1-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0,1],xmm2[2,3],xmm0[4,5],xmm2[6,7]
+; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: splatconstant_shift_v4i64:
+; AVX2:       # BB#0:
+; AVX2-NEXT:    vpsrad $7, %ymm0, %ymm1
+; AVX2-NEXT:    vpsrlq $7, %ymm0, %ymm0
+; AVX2-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0],ymm1[1],ymm0[2],ymm1[3],ymm0[4],ymm1[5],ymm0[6],ymm1[7]
+; AVX2-NEXT:    retq
+  %shift = ashr <4 x i64> %a, <i64 7, i64 7, i64 7, i64 7>
+  ret <4 x i64> %shift
+}
+
+define <8 x i32> @splatconstant_shift_v8i32(<8 x i32> %a) nounwind {
+; AVX1-LABEL: splatconstant_shift_v8i32:
+; AVX1:       # BB#0:
+; AVX1-NEXT:    vpsrad $5, %xmm0, %xmm1
+; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm0
+; AVX1-NEXT:    vpsrad $5, %xmm0, %xmm0
+; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: splatconstant_shift_v8i32:
+; AVX2:       # BB#0:
+; AVX2-NEXT:    vpsrad $5, %ymm0, %ymm0
+; AVX2-NEXT:    retq
+  %shift = ashr <8 x i32> %a, <i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5>
+  ret <8 x i32> %shift
+}
+
+define <16 x i16> @splatconstant_shift_v16i16(<16 x i16> %a) nounwind {
+; AVX1-LABEL: splatconstant_shift_v16i16:
+; AVX1:       # BB#0:
+; AVX1-NEXT:    vpsraw $3, %xmm0, %xmm1
+; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm0
+; AVX1-NEXT:    vpsraw $3, %xmm0, %xmm0
+; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: splatconstant_shift_v16i16:
+; AVX2:       # BB#0:
+; AVX2-NEXT:    vpsraw $3, %ymm0, %ymm0
+; AVX2-NEXT:    retq
+  %shift = ashr <16 x i16> %a, <i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3>
+  ret <16 x i16> %shift
+}
+
+define <32 x i8> @splatconstant_shift_v32i8(<32 x i8> %a) nounwind {
+; AVX1-LABEL: splatconstant_shift_v32i8:
+; AVX1:       # BB#0:
+; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
+; AVX1-NEXT:    vpsrlw $3, %xmm1, %xmm1
+; AVX1-NEXT:    vmovdqa {{.*#+}} xmm2 = [31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31]
+; AVX1-NEXT:    vpand %xmm2, %xmm1, %xmm1
+; AVX1-NEXT:    vmovdqa {{.*#+}} xmm3 = [16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16]
+; AVX1-NEXT:    vpxor %xmm3, %xmm1, %xmm1
+; AVX1-NEXT:    vpsubb %xmm3, %xmm1, %xmm1
+; AVX1-NEXT:    vpsrlw $3, %xmm0, %xmm0
+; AVX1-NEXT:    vpand %xmm2, %xmm0, %xmm0
+; AVX1-NEXT:    vpxor %xmm3, %xmm0, %xmm0
+; AVX1-NEXT:    vpsubb %xmm3, %xmm0, %xmm0
+; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: splatconstant_shift_v32i8:
+; AVX2:       # BB#0:
+; AVX2-NEXT:    vpsrlw $3, %ymm0, %ymm0
+; AVX2-NEXT:    vpand {{.*}}(%rip), %ymm0, %ymm0
+; AVX2-NEXT:    vmovdqa {{.*#+}} ymm1 = [16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16]
+; AVX2-NEXT:    vpxor %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vpsubb %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    retq
+  %shift = ashr <32 x i8> %a, <i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3>
+  ret <32 x i8> %shift
+}
diff --git a/test/CodeGen/X86/vector-shift-lshr-128.ll b/test/CodeGen/X86/vector-shift-lshr-128.ll
new file mode 100644
index 00000000000..1c988946a46
--- /dev/null
+++ b/test/CodeGen/X86/vector-shift-lshr-128.ll
@@ -0,0 +1,1008 @@
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mcpu=x86-64 | FileCheck %s --check-prefix=ALL --check-prefix=SSE --check-prefix=SSE2
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mcpu=x86-64 -mattr=+sse4.1 | FileCheck %s --check-prefix=ALL --check-prefix=SSE --check-prefix=SSE41
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mcpu=x86-64 -mattr=+avx | FileCheck %s --check-prefix=ALL --check-prefix=AVX --check-prefix=AVX1
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mcpu=x86-64 -mattr=+avx2 | FileCheck %s --check-prefix=ALL --check-prefix=AVX --check-prefix=AVX2
+;
+; Just one 32-bit run to make sure we do reasonable things for i64 shifts.
+; RUN: llc < %s -mtriple=i686-unknown-unknown -mcpu=x86-64 | FileCheck %s --check-prefix=ALL --check-prefix=X32-SSE --check-prefix=X32-SSE2
+
+;
+; Variable Shifts
+;
+
+define <2 x i64> @var_shift_v2i64(<2 x i64> %a, <2 x i64> %b) nounwind {
+; SSE2-LABEL: var_shift_v2i64:
+; SSE2:       # BB#0:
+; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm1[2,3,0,1]
+; SSE2-NEXT:    movdqa %xmm0, %xmm2
+; SSE2-NEXT:    psrlq %xmm3, %xmm2
+; SSE2-NEXT:    psrlq %xmm1, %xmm0
+; SSE2-NEXT:    movsd {{.*#+}} xmm2 = xmm0[0],xmm2[1]
+; SSE2-NEXT:    movapd %xmm2, %xmm0
+; SSE2-NEXT:    retq
+;
+; SSE41-LABEL: var_shift_v2i64:
+; SSE41:       # BB#0:
+; SSE41-NEXT:    movdqa %xmm0, %xmm2
+; SSE41-NEXT:    psrlq %xmm1, %xmm2
+; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[2,3,0,1]
+; SSE41-NEXT:    psrlq %xmm1, %xmm0
+; SSE41-NEXT:    pblendw {{.*#+}} xmm0 = xmm2[0,1,2,3],xmm0[4,5,6,7]
+; SSE41-NEXT:    retq
+;
+; AVX1-LABEL: var_shift_v2i64:
+; AVX1:       # BB#0:
+; AVX1-NEXT:    vpsrlq %xmm1, %xmm0, %xmm2
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[2,3,0,1]
+; AVX1-NEXT:    vpsrlq %xmm1, %xmm0, %xmm0
+; AVX1-NEXT:    vpblendw {{.*#+}} xmm0 = xmm2[0,1,2,3],xmm0[4,5,6,7]
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: var_shift_v2i64:
+; AVX2:       # BB#0:
+; AVX2-NEXT:    vpsrlvq %xmm1, %xmm0, %xmm0
+; AVX2-NEXT:    retq
+;
+; X32-SSE-LABEL: var_shift_v2i64:
+; X32-SSE:       # BB#0:
+; X32-SSE-NEXT:    pshufd {{.*#+}} xmm3 = xmm1[2,3,0,1]
+; X32-SSE-NEXT:    movdqa %xmm0, %xmm2
+; X32-SSE-NEXT:    psrlq %xmm3, %xmm2
+; X32-SSE-NEXT:    movq {{.*#+}} xmm1 = xmm1[0],zero
+; X32-SSE-NEXT:    psrlq %xmm1, %xmm0
+; X32-SSE-NEXT:    movsd {{.*#+}} xmm2 = xmm0[0],xmm2[1]
+; X32-SSE-NEXT:    movapd %xmm2, %xmm0
+; X32-SSE-NEXT:    retl
+  %shift = lshr <2 x i64> %a, %b
+  ret <2 x i64> %shift
+}
+
+define <4 x i32> @var_shift_v4i32(<4 x i32> %a, <4 x i32> %b) nounwind {
+; SSE2-LABEL: var_shift_v4i32:
+; SSE2:       # BB#0:
+; SSE2-NEXT:    movdqa %xmm1, %xmm2
+; SSE2-NEXT:    psrldq {{.*#+}} xmm2 = xmm2[12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
+; SSE2-NEXT:    movdqa %xmm0, %xmm3
+; SSE2-NEXT:    psrld %xmm2, %xmm3
+; SSE2-NEXT:    movdqa %xmm1, %xmm2
+; SSE2-NEXT:    psrlq $32, %xmm2
+; SSE2-NEXT:    movdqa %xmm0, %xmm4
+; SSE2-NEXT:    psrld %xmm2, %xmm4
+; SSE2-NEXT:    movsd {{.*#+}} xmm3 = xmm4[0],xmm3[1]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm3[1,3,2,3]
+; SSE2-NEXT:    pxor %xmm3, %xmm3
+; SSE2-NEXT:    movdqa %xmm1, %xmm4
+; SSE2-NEXT:    punpckhdq {{.*#+}} xmm4 = xmm4[2],xmm3[2],xmm4[3],xmm3[3]
+; SSE2-NEXT:    movdqa %xmm0, %xmm5
+; SSE2-NEXT:    psrld %xmm4, %xmm5
+; SSE2-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm3[0],xmm1[1],xmm3[1]
+; SSE2-NEXT:    psrld %xmm1, %xmm0
+; SSE2-NEXT:    movsd {{.*#+}} xmm5 = xmm0[0],xmm5[1]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm5[0,2,2,3]
+; SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1]
+; SSE2-NEXT:    retq
+;
+; SSE41-LABEL: var_shift_v4i32:
+; SSE41:       # BB#0:
+; SSE41-NEXT:    movdqa %xmm1, %xmm2
+; SSE41-NEXT:    psrldq {{.*#+}} xmm2 = xmm2[12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
+; SSE41-NEXT:    movdqa %xmm0, %xmm3
+; SSE41-NEXT:    psrld %xmm2, %xmm3
+; SSE41-NEXT:    movdqa %xmm1, %xmm2
+; SSE41-NEXT:    psrlq $32, %xmm2
+; SSE41-NEXT:    movdqa %xmm0, %xmm4
+; SSE41-NEXT:    psrld %xmm2, %xmm4
+; SSE41-NEXT:    pblendw {{.*#+}} xmm4 = xmm4[0,1,2,3],xmm3[4,5,6,7]
+; SSE41-NEXT:    pxor %xmm2, %xmm2
+; SSE41-NEXT:    pmovzxdq {{.*#+}} xmm3 = xmm1[0],zero,xmm1[1],zero
+; SSE41-NEXT:    punpckhdq {{.*#+}} xmm1 = xmm1[2],xmm2[2],xmm1[3],xmm2[3]
+; SSE41-NEXT:    movdqa %xmm0, %xmm2
+; SSE41-NEXT:    psrld %xmm1, %xmm2
+; SSE41-NEXT:    psrld %xmm3, %xmm0
+; SSE41-NEXT:    pblendw {{.*#+}} xmm0 = xmm0[0,1,2,3],xmm2[4,5,6,7]
+; SSE41-NEXT:    pblendw {{.*#+}} xmm0 = xmm0[0,1],xmm4[2,3],xmm0[4,5],xmm4[6,7]
+; SSE41-NEXT:    retq
+;
+; AVX1-LABEL: var_shift_v4i32:
+; AVX1:       # BB#0:
+; AVX1-NEXT:    vpsrldq {{.*#+}} xmm2 = xmm1[12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
+; AVX1-NEXT:    vpsrld %xmm2, %xmm0, %xmm2
+; AVX1-NEXT:    vpsrlq $32, %xmm1, %xmm3
+; AVX1-NEXT:    vpsrld %xmm3, %xmm0, %xmm3
+; AVX1-NEXT:    vpblendw {{.*#+}} xmm2 = xmm3[0,1,2,3],xmm2[4,5,6,7]
+; AVX1-NEXT:    vpxor %xmm3, %xmm3, %xmm3
+; AVX1-NEXT:    vpunpckhdq {{.*#+}} xmm3 = xmm1[2],xmm3[2],xmm1[3],xmm3[3]
+; AVX1-NEXT:    vpsrld %xmm3, %xmm0, %xmm3
+; AVX1-NEXT:    vpmovzxdq {{.*#+}} xmm1 = xmm1[0],zero,xmm1[1],zero
+; AVX1-NEXT:    vpsrld %xmm1, %xmm0, %xmm0
+; AVX1-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0,1,2,3],xmm3[4,5,6,7]
+; AVX1-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0,1],xmm2[2,3],xmm0[4,5],xmm2[6,7]
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: var_shift_v4i32:
+; AVX2:       # BB#0:
+; AVX2-NEXT:    vpsrlvd %xmm1, %xmm0, %xmm0
+; AVX2-NEXT:    retq
+;
+; X32-SSE-LABEL: var_shift_v4i32:
+; X32-SSE:       # BB#0:
+; X32-SSE-NEXT:    movdqa %xmm1, %xmm2
+; X32-SSE-NEXT:    psrldq {{.*#+}} xmm2 = xmm2[12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
+; X32-SSE-NEXT:    movdqa %xmm0, %xmm3
+; X32-SSE-NEXT:    psrld %xmm2, %xmm3
+; X32-SSE-NEXT:    movdqa %xmm1, %xmm2
+; X32-SSE-NEXT:    psrlq $32, %xmm2
+; X32-SSE-NEXT:    movdqa %xmm0, %xmm4
+; X32-SSE-NEXT:    psrld %xmm2, %xmm4
+; X32-SSE-NEXT:    movsd {{.*#+}} xmm3 = xmm4[0],xmm3[1]
+; X32-SSE-NEXT:    pshufd {{.*#+}} xmm2 = xmm3[1,3,2,3]
+; X32-SSE-NEXT:    pxor %xmm3, %xmm3
+; X32-SSE-NEXT:    movdqa %xmm1, %xmm4
+; X32-SSE-NEXT:    punpckhdq {{.*#+}} xmm4 = xmm4[2],xmm3[2],xmm4[3],xmm3[3]
+; X32-SSE-NEXT:    movdqa %xmm0, %xmm5
+; X32-SSE-NEXT:    psrld %xmm4, %xmm5
+; X32-SSE-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm3[0],xmm1[1],xmm3[1]
+; X32-SSE-NEXT:    psrld %xmm1, %xmm0
+; X32-SSE-NEXT:    movsd {{.*#+}} xmm5 = xmm0[0],xmm5[1]
+; X32-SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm5[0,2,2,3]
+; X32-SSE-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1]
+; X32-SSE-NEXT:    retl
+  %shift = lshr <4 x i32> %a, %b
+  ret <4 x i32> %shift
+}
+
+define <8 x i16> @var_shift_v8i16(<8 x i16> %a, <8 x i16> %b) nounwind {
+; SSE2-LABEL: var_shift_v8i16:
+; SSE2:       # BB#0:
+; SSE2-NEXT:    psllw $12, %xmm1
+; SSE2-NEXT:    movdqa %xmm1, %xmm2
+; SSE2-NEXT:    psraw $15, %xmm2
+; SSE2-NEXT:    movdqa %xmm2, %xmm3
+; SSE2-NEXT:    pandn %xmm0, %xmm3
+; SSE2-NEXT:    psrlw $8, %xmm0
+; SSE2-NEXT:    pand %xmm2, %xmm0
+; SSE2-NEXT:    por %xmm3, %xmm0
+; SSE2-NEXT:    paddw %xmm1, %xmm1
+; SSE2-NEXT:    movdqa %xmm1, %xmm2
+; SSE2-NEXT:    psraw $15, %xmm2
+; SSE2-NEXT:    movdqa %xmm2, %xmm3
+; SSE2-NEXT:    pandn %xmm0, %xmm3
+; SSE2-NEXT:    psrlw $4, %xmm0
+; SSE2-NEXT:    pand %xmm2, %xmm0
+; SSE2-NEXT:    por %xmm3, %xmm0
+; SSE2-NEXT:    paddw %xmm1, %xmm1
+; SSE2-NEXT:    movdqa %xmm1, %xmm2
+; SSE2-NEXT:    psraw $15, %xmm2
+; SSE2-NEXT:    movdqa %xmm2, %xmm3
+; SSE2-NEXT:    pandn %xmm0, %xmm3
+; SSE2-NEXT:    psrlw $2, %xmm0
+; SSE2-NEXT:    pand %xmm2, %xmm0
+; SSE2-NEXT:    por %xmm3, %xmm0
+; SSE2-NEXT:    paddw %xmm1, %xmm1
+; SSE2-NEXT:    psraw $15, %xmm1
+; SSE2-NEXT:    movdqa %xmm1, %xmm2
+; SSE2-NEXT:    pandn %xmm0, %xmm2
+; SSE2-NEXT:    psrlw $1, %xmm0
+; SSE2-NEXT:    pand %xmm1, %xmm0
+; SSE2-NEXT:    por %xmm2, %xmm0
+; SSE2-NEXT:    retq
+;
+; SSE41-LABEL: var_shift_v8i16:
+; SSE41:       # BB#0:
+; SSE41-NEXT:    movdqa %xmm0, %xmm2
+; SSE41-NEXT:    movdqa %xmm1, %xmm0
+; SSE41-NEXT:    psllw $12, %xmm0
+; SSE41-NEXT:    psllw $4, %xmm1
+; SSE41-NEXT:    por %xmm0, %xmm1
+; SSE41-NEXT:    movdqa %xmm1, %xmm3
+; SSE41-NEXT:    paddw %xmm3, %xmm3
+; SSE41-NEXT:    movdqa %xmm2, %xmm4
+; SSE41-NEXT:    psrlw $8, %xmm4
+; SSE41-NEXT:    movdqa %xmm1, %xmm0
+; SSE41-NEXT:    pblendvb %xmm4, %xmm2
+; SSE41-NEXT:    movdqa %xmm2, %xmm1
+; SSE41-NEXT:    psrlw $4, %xmm1
+; SSE41-NEXT:    movdqa %xmm3, %xmm0
+; SSE41-NEXT:    pblendvb %xmm1, %xmm2
+; SSE41-NEXT:    movdqa %xmm2, %xmm1
+; SSE41-NEXT:    psrlw $2, %xmm1
+; SSE41-NEXT:    paddw %xmm3, %xmm3
+; SSE41-NEXT:    movdqa %xmm3, %xmm0
+; SSE41-NEXT:    pblendvb %xmm1, %xmm2
+; SSE41-NEXT:    movdqa %xmm2, %xmm1
+; SSE41-NEXT:    psrlw $1, %xmm1
+; SSE41-NEXT:    paddw %xmm3, %xmm3
+; SSE41-NEXT:    movdqa %xmm3, %xmm0
+; SSE41-NEXT:    pblendvb %xmm1, %xmm2
+; SSE41-NEXT:    movdqa %xmm2, %xmm0
+; SSE41-NEXT:    retq
+;
+; AVX1-LABEL: var_shift_v8i16:
+; AVX1:       # BB#0:
+; AVX1-NEXT:    vpsllw $12, %xmm1, %xmm2
+; AVX1-NEXT:    vpsllw $4, %xmm1, %xmm1
+; AVX1-NEXT:    vpor %xmm2, %xmm1, %xmm1
+; AVX1-NEXT:    vpaddw %xmm1, %xmm1, %xmm2
+; AVX1-NEXT:    vpsrlw $8, %xmm0, %xmm3
+; AVX1-NEXT:    vpblendvb %xmm1, %xmm3, %xmm0, %xmm0
+; AVX1-NEXT:    vpsrlw $4, %xmm0, %xmm1
+; AVX1-NEXT:    vpblendvb %xmm2, %xmm1, %xmm0, %xmm0
+; AVX1-NEXT:    vpsrlw $2, %xmm0, %xmm1
+; AVX1-NEXT:    vpaddw %xmm2, %xmm2, %xmm2
+; AVX1-NEXT:    vpblendvb %xmm2, %xmm1, %xmm0, %xmm0
+; AVX1-NEXT:    vpsrlw $1, %xmm0, %xmm1
+; AVX1-NEXT:    vpaddw %xmm2, %xmm2, %xmm2
+; AVX1-NEXT:    vpblendvb %xmm2, %xmm1, %xmm0, %xmm0
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: var_shift_v8i16:
+; AVX2:       # BB#0:
+; AVX2-NEXT:    vpmovzxwd {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
+; AVX2-NEXT:    vpmovzxwd {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
+; AVX2-NEXT:    vpsrlvd %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vpshufb {{.*#+}} ymm0 = ymm0[0,1,4,5,8,9,12,13],zero,zero,zero,zero,zero,zero,zero,zero,ymm0[16,17,20,21,24,25,28,29],zero,zero,zero,zero,zero,zero,zero,zero
+; AVX2-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,2,2,3]
+; AVX2-NEXT:    vzeroupper
+; AVX2-NEXT:    retq
+;
+; X32-SSE-LABEL: var_shift_v8i16:
+; X32-SSE:       # BB#0:
+; X32-SSE-NEXT:    psllw $12, %xmm1
+; X32-SSE-NEXT:    movdqa %xmm1, %xmm2
+; X32-SSE-NEXT:    psraw $15, %xmm2
+; X32-SSE-NEXT:    movdqa %xmm2, %xmm3
+; X32-SSE-NEXT:    pandn %xmm0, %xmm3
+; X32-SSE-NEXT:    psrlw $8, %xmm0
+; X32-SSE-NEXT:    pand %xmm2, %xmm0
+; X32-SSE-NEXT:    por %xmm3, %xmm0
+; X32-SSE-NEXT:    paddw %xmm1, %xmm1
+; X32-SSE-NEXT:    movdqa %xmm1, %xmm2
+; X32-SSE-NEXT:    psraw $15, %xmm2
+; X32-SSE-NEXT:    movdqa %xmm2, %xmm3
+; X32-SSE-NEXT:    pandn %xmm0, %xmm3
+; X32-SSE-NEXT:    psrlw $4, %xmm0
+; X32-SSE-NEXT:    pand %xmm2, %xmm0
+; X32-SSE-NEXT:    por %xmm3, %xmm0
+; X32-SSE-NEXT:    paddw %xmm1, %xmm1
+; X32-SSE-NEXT:    movdqa %xmm1, %xmm2
+; X32-SSE-NEXT:    psraw $15, %xmm2
+; X32-SSE-NEXT:    movdqa %xmm2, %xmm3
+; X32-SSE-NEXT:    pandn %xmm0, %xmm3
+; X32-SSE-NEXT:    psrlw $2, %xmm0
+; X32-SSE-NEXT:    pand %xmm2, %xmm0
+; X32-SSE-NEXT:    por %xmm3, %xmm0
+; X32-SSE-NEXT:    paddw %xmm1, %xmm1
+; X32-SSE-NEXT:    psraw $15, %xmm1
+; X32-SSE-NEXT:    movdqa %xmm1, %xmm2
+; X32-SSE-NEXT:    pandn %xmm0, %xmm2
+; X32-SSE-NEXT:    psrlw $1, %xmm0
+; X32-SSE-NEXT:    pand %xmm1, %xmm0
+; X32-SSE-NEXT:    por %xmm2, %xmm0
+; X32-SSE-NEXT:    retl
+  %shift = lshr <8 x i16> %a, %b
+  ret <8 x i16> %shift
+}
+
+define <16 x i8> @var_shift_v16i8(<16 x i8> %a, <16 x i8> %b) nounwind {
+; SSE2-LABEL: var_shift_v16i8:
+; SSE2:       # BB#0:
+; SSE2-NEXT:    psllw $5, %xmm1
+; SSE2-NEXT:    pxor %xmm2, %xmm2
+; SSE2-NEXT:    pxor %xmm3, %xmm3
+; SSE2-NEXT:    pcmpgtb %xmm1, %xmm3
+; SSE2-NEXT:    movdqa %xmm3, %xmm4
+; SSE2-NEXT:    pandn %xmm0, %xmm4
+; SSE2-NEXT:    psrlw $4, %xmm0
+; SSE2-NEXT:    pand {{.*}}(%rip), %xmm0
+; SSE2-NEXT:    pand %xmm3, %xmm0
+; SSE2-NEXT:    por %xmm4, %xmm0
+; SSE2-NEXT:    paddb %xmm1, %xmm1
+; SSE2-NEXT:    pxor %xmm3, %xmm3
+; SSE2-NEXT:    pcmpgtb %xmm1, %xmm3
+; SSE2-NEXT:    movdqa %xmm3, %xmm4
+; SSE2-NEXT:    pandn %xmm0, %xmm4
+; SSE2-NEXT:    psrlw $2, %xmm0
+; SSE2-NEXT:    pand {{.*}}(%rip), %xmm0
+; SSE2-NEXT:    pand %xmm3, %xmm0
+; SSE2-NEXT:    por %xmm4, %xmm0
+; SSE2-NEXT:    paddb %xmm1, %xmm1
+; SSE2-NEXT:    pcmpgtb %xmm1, %xmm2
+; SSE2-NEXT:    movdqa %xmm2, %xmm1
+; SSE2-NEXT:    pandn %xmm0, %xmm1
+; SSE2-NEXT:    psrlw $1, %xmm0
+; SSE2-NEXT:    pand {{.*}}(%rip), %xmm0
+; SSE2-NEXT:    pand %xmm2, %xmm0
+; SSE2-NEXT:    por %xmm1, %xmm0
+; SSE2-NEXT:    retq
+;
+; SSE41-LABEL: var_shift_v16i8:
+; SSE41:       # BB#0:
+; SSE41-NEXT:    movdqa %xmm0, %xmm2
+; SSE41-NEXT:    psllw $5, %xmm1
+; SSE41-NEXT:    movdqa %xmm2, %xmm3
+; SSE41-NEXT:    psrlw $4, %xmm3
+; SSE41-NEXT:    pand {{.*}}(%rip), %xmm3
+; SSE41-NEXT:    movdqa %xmm1, %xmm0
+; SSE41-NEXT:    pblendvb %xmm3, %xmm2
+; SSE41-NEXT:    movdqa %xmm2, %xmm3
+; SSE41-NEXT:    psrlw $2, %xmm3
+; SSE41-NEXT:    pand {{.*}}(%rip), %xmm3
+; SSE41-NEXT:    paddb %xmm1, %xmm1
+; SSE41-NEXT:    movdqa %xmm1, %xmm0
+; SSE41-NEXT:    pblendvb %xmm3, %xmm2
+; SSE41-NEXT:    movdqa %xmm2, %xmm3
+; SSE41-NEXT:    psrlw $1, %xmm3
+; SSE41-NEXT:    pand {{.*}}(%rip), %xmm3
+; SSE41-NEXT:    paddb %xmm1, %xmm1
+; SSE41-NEXT:    movdqa %xmm1, %xmm0
+; SSE41-NEXT:    pblendvb %xmm3, %xmm2
+; SSE41-NEXT:    movdqa %xmm2, %xmm0
+; SSE41-NEXT:    retq
+;
+; AVX-LABEL: var_shift_v16i8:
+; AVX:       # BB#0:
+; AVX-NEXT:    vpsllw $5, %xmm1, %xmm1
+; AVX-NEXT:    vpsrlw $4, %xmm0, %xmm2
+; AVX-NEXT:    vpand {{.*}}(%rip), %xmm2, %xmm2
+; AVX-NEXT:    vpblendvb %xmm1, %xmm2, %xmm0, %xmm0
+; AVX-NEXT:    vpsrlw $2, %xmm0, %xmm2
+; AVX-NEXT:    vpand {{.*}}(%rip), %xmm2, %xmm2
+; AVX-NEXT:    vpaddb %xmm1, %xmm1, %xmm1
+; AVX-NEXT:    vpblendvb %xmm1, %xmm2, %xmm0, %xmm0
+; AVX-NEXT:    vpsrlw $1, %xmm0, %xmm2
+; AVX-NEXT:    vpand {{.*}}(%rip), %xmm2, %xmm2
+; AVX-NEXT:    vpaddb %xmm1, %xmm1, %xmm1
+; AVX-NEXT:    vpblendvb %xmm1, %xmm2, %xmm0, %xmm0
+; AVX-NEXT:    retq
+;
+; X32-SSE-LABEL: var_shift_v16i8:
+; X32-SSE:       # BB#0:
+; X32-SSE-NEXT:    psllw $5, %xmm1
+; X32-SSE-NEXT:    pxor %xmm2, %xmm2
+; X32-SSE-NEXT:    pxor %xmm3, %xmm3
+; X32-SSE-NEXT:    pcmpgtb %xmm1, %xmm3
+; X32-SSE-NEXT:    movdqa %xmm3, %xmm4
+; X32-SSE-NEXT:    pandn %xmm0, %xmm4
+; X32-SSE-NEXT:    psrlw $4, %xmm0
+; X32-SSE-NEXT:    pand .LCPI3_0, %xmm0
+; X32-SSE-NEXT:    pand %xmm3, %xmm0
+; X32-SSE-NEXT:    por %xmm4, %xmm0
+; X32-SSE-NEXT:    paddb %xmm1, %xmm1
+; X32-SSE-NEXT:    pxor %xmm3, %xmm3
+; X32-SSE-NEXT:    pcmpgtb %xmm1, %xmm3
+; X32-SSE-NEXT:    movdqa %xmm3, %xmm4
+; X32-SSE-NEXT:    pandn %xmm0, %xmm4
+; X32-SSE-NEXT:    psrlw $2, %xmm0
+; X32-SSE-NEXT:    pand .LCPI3_1, %xmm0
+; X32-SSE-NEXT:    pand %xmm3, %xmm0
+; X32-SSE-NEXT:    por %xmm4, %xmm0
+; X32-SSE-NEXT:    paddb %xmm1, %xmm1
+; X32-SSE-NEXT:    pcmpgtb %xmm1, %xmm2
+; X32-SSE-NEXT:    movdqa %xmm2, %xmm1
+; X32-SSE-NEXT:    pandn %xmm0, %xmm1
+; X32-SSE-NEXT:    psrlw $1, %xmm0
+; X32-SSE-NEXT:    pand .LCPI3_2, %xmm0
+; X32-SSE-NEXT:    pand %xmm2, %xmm0
+; X32-SSE-NEXT:    por %xmm1, %xmm0
+; X32-SSE-NEXT:    retl
+  %shift = lshr <16 x i8> %a, %b
+  ret <16 x i8> %shift
+}
+
+;
+; Uniform Variable Shifts
+;
+
+define <2 x i64> @splatvar_shift_v2i64(<2 x i64> %a, <2 x i64> %b) nounwind {
+; SSE-LABEL: splatvar_shift_v2i64:
+; SSE:       # BB#0:
+; SSE-NEXT:    psrlq %xmm1, %xmm0
+; SSE-NEXT:    retq
+;
+; AVX-LABEL: splatvar_shift_v2i64:
+; AVX:       # BB#0:
+; AVX-NEXT:    vpsrlq %xmm1, %xmm0, %xmm0
+; AVX-NEXT:    retq
+;
+; X32-SSE-LABEL: splatvar_shift_v2i64:
+; X32-SSE:       # BB#0:
+; X32-SSE-NEXT:    movq {{.*#+}} xmm1 = xmm1[0],zero
+; X32-SSE-NEXT:    psrlq %xmm1, %xmm0
+; X32-SSE-NEXT:    retl
+  %splat = shufflevector <2 x i64> %b, <2 x i64> undef, <2 x i32> zeroinitializer
+  %shift = lshr <2 x i64> %a, %splat
+  ret <2 x i64> %shift
+}
+
+define <4 x i32> @splatvar_shift_v4i32(<4 x i32> %a, <4 x i32> %b) nounwind {
+; SSE2-LABEL: splatvar_shift_v4i32:
+; SSE2:       # BB#0:
+; SSE2-NEXT:    xorps %xmm2, %xmm2
+; SSE2-NEXT:    movss {{.*#+}} xmm2 = xmm1[0],xmm2[1,2,3]
+; SSE2-NEXT:    psrld %xmm2, %xmm0
+; SSE2-NEXT:    retq
+;
+; SSE41-LABEL: splatvar_shift_v4i32:
+; SSE41:       # BB#0:
+; SSE41-NEXT:    pxor %xmm2, %xmm2
+; SSE41-NEXT:    pblendw {{.*#+}} xmm2 = xmm1[0,1],xmm2[2,3,4,5,6,7]
+; SSE41-NEXT:    psrld %xmm2, %xmm0
+; SSE41-NEXT:    retq
+;
+; AVX-LABEL: splatvar_shift_v4i32:
+; AVX:       # BB#0:
+; AVX-NEXT:    vpxor %xmm2, %xmm2, %xmm2
+; AVX-NEXT:    vpblendw {{.*#+}} xmm1 = xmm1[0,1],xmm2[2,3,4,5,6,7]
+; AVX-NEXT:    vpsrld %xmm1, %xmm0, %xmm0
+; AVX-NEXT:    retq
+;
+; X32-SSE-LABEL: splatvar_shift_v4i32:
+; X32-SSE:       # BB#0:
+; X32-SSE-NEXT:    xorps %xmm2, %xmm2
+; X32-SSE-NEXT:    movss {{.*#+}} xmm2 = xmm1[0],xmm2[1,2,3]
+; X32-SSE-NEXT:    psrld %xmm2, %xmm0
+; X32-SSE-NEXT:    retl
+  %splat = shufflevector <4 x i32> %b, <4 x i32> undef, <4 x i32> zeroinitializer
+  %shift = lshr <4 x i32> %a, %splat
+  ret <4 x i32> %shift
+}
+
+define <8 x i16> @splatvar_shift_v8i16(<8 x i16> %a, <8 x i16> %b) nounwind {
+; SSE2-LABEL: splatvar_shift_v8i16:
+; SSE2:       # BB#0:
+; SSE2-NEXT:    movd %xmm1, %eax
+; SSE2-NEXT:    movzwl %ax, %eax
+; SSE2-NEXT:    movd %eax, %xmm1
+; SSE2-NEXT:    psrlw %xmm1, %xmm0
+; SSE2-NEXT:    retq
+;
+; SSE41-LABEL: splatvar_shift_v8i16:
+; SSE41:       # BB#0:
+; SSE41-NEXT:    pxor %xmm2, %xmm2
+; SSE41-NEXT:    pblendw {{.*#+}} xmm2 = xmm1[0],xmm2[1,2,3,4,5,6,7]
+; SSE41-NEXT:    psrlw %xmm2, %xmm0
+; SSE41-NEXT:    retq
+;
+; AVX-LABEL: splatvar_shift_v8i16:
+; AVX:       # BB#0:
+; AVX-NEXT:    vpxor %xmm2, %xmm2, %xmm2
+; AVX-NEXT:    vpblendw {{.*#+}} xmm1 = xmm1[0],xmm2[1,2,3,4,5,6,7]
+; AVX-NEXT:    vpsrlw %xmm1, %xmm0, %xmm0
+; AVX-NEXT:    retq
+;
+; X32-SSE-LABEL: splatvar_shift_v8i16:
+; X32-SSE:       # BB#0:
+; X32-SSE-NEXT:    movd %xmm1, %eax
+; X32-SSE-NEXT:    movzwl %ax, %eax
+; X32-SSE-NEXT:    movd %eax, %xmm1
+; X32-SSE-NEXT:    psrlw %xmm1, %xmm0
+; X32-SSE-NEXT:    retl
+  %splat = shufflevector <8 x i16> %b, <8 x i16> undef, <8 x i32> zeroinitializer
+  %shift = lshr <8 x i16> %a, %splat
+  ret <8 x i16> %shift
+}
+
+define <16 x i8> @splatvar_shift_v16i8(<16 x i8> %a, <16 x i8> %b) nounwind {
+; SSE2-LABEL: splatvar_shift_v16i8:
+; SSE2:       # BB#0:
+; SSE2-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,1,0,3]
+; SSE2-NEXT:    pshuflw {{.*#+}} xmm1 = xmm1[0,0,0,0,4,5,6,7]
+; SSE2-NEXT:    pshufhw {{.*#+}} xmm2 = xmm1[0,1,2,3,4,4,4,4]
+; SSE2-NEXT:    psllw $5, %xmm2
+; SSE2-NEXT:    pxor %xmm1, %xmm1
+; SSE2-NEXT:    pxor %xmm3, %xmm3
+; SSE2-NEXT:    pcmpgtb %xmm2, %xmm3
+; SSE2-NEXT:    movdqa %xmm3, %xmm4
+; SSE2-NEXT:    pandn %xmm0, %xmm4
+; SSE2-NEXT:    psrlw $4, %xmm0
+; SSE2-NEXT:    pand {{.*}}(%rip), %xmm0
+; SSE2-NEXT:    pand %xmm3, %xmm0
+; SSE2-NEXT:    por %xmm4, %xmm0
+; SSE2-NEXT:    paddb %xmm2, %xmm2
+; SSE2-NEXT:    pxor %xmm3, %xmm3
+; SSE2-NEXT:    pcmpgtb %xmm2, %xmm3
+; SSE2-NEXT:    movdqa %xmm3, %xmm4
+; SSE2-NEXT:    pandn %xmm0, %xmm4
+; SSE2-NEXT:    psrlw $2, %xmm0
+; SSE2-NEXT:    pand {{.*}}(%rip), %xmm0
+; SSE2-NEXT:    pand %xmm3, %xmm0
+; SSE2-NEXT:    por %xmm4, %xmm0
+; SSE2-NEXT:    paddb %xmm2, %xmm2
+; SSE2-NEXT:    pcmpgtb %xmm2, %xmm1
+; SSE2-NEXT:    movdqa %xmm1, %xmm2
+; SSE2-NEXT:    pandn %xmm0, %xmm2
+; SSE2-NEXT:    psrlw $1, %xmm0
+; SSE2-NEXT:    pand {{.*}}(%rip), %xmm0
+; SSE2-NEXT:    pand %xmm1, %xmm0
+; SSE2-NEXT:    por %xmm2, %xmm0
+; SSE2-NEXT:    retq
+;
+; SSE41-LABEL: splatvar_shift_v16i8:
+; SSE41:       # BB#0:
+; SSE41-NEXT:    movdqa %xmm0, %xmm2
+; SSE41-NEXT:    pxor %xmm0, %xmm0
+; SSE41-NEXT:    pshufb %xmm0, %xmm1
+; SSE41-NEXT:    psllw $5, %xmm1
+; SSE41-NEXT:    movdqa %xmm1, %xmm3
+; SSE41-NEXT:    paddb %xmm3, %xmm3
+; SSE41-NEXT:    movdqa %xmm2, %xmm4
+; SSE41-NEXT:    psrlw $4, %xmm4
+; SSE41-NEXT:    pand {{.*}}(%rip), %xmm4
+; SSE41-NEXT:    movdqa %xmm1, %xmm0
+; SSE41-NEXT:    pblendvb %xmm4, %xmm2
+; SSE41-NEXT:    movdqa %xmm2, %xmm1
+; SSE41-NEXT:    psrlw $2, %xmm1
+; SSE41-NEXT:    pand {{.*}}(%rip), %xmm1
+; SSE41-NEXT:    movdqa %xmm3, %xmm0
+; SSE41-NEXT:    pblendvb %xmm1, %xmm2
+; SSE41-NEXT:    movdqa %xmm2, %xmm1
+; SSE41-NEXT:    psrlw $1, %xmm1
+; SSE41-NEXT:    pand {{.*}}(%rip), %xmm1
+; SSE41-NEXT:    paddb %xmm3, %xmm3
+; SSE41-NEXT:    movdqa %xmm3, %xmm0
+; SSE41-NEXT:    pblendvb %xmm1, %xmm2
+; SSE41-NEXT:    movdqa %xmm2, %xmm0
+; SSE41-NEXT:    retq
+;
+; AVX1-LABEL: splatvar_shift_v16i8:
+; AVX1:       # BB#0:
+; AVX1-NEXT:    vpxor %xmm2, %xmm2, %xmm2
+; AVX1-NEXT:    vpshufb %xmm2, %xmm1, %xmm1
+; AVX1-NEXT:    vpsllw $5, %xmm1, %xmm1
+; AVX1-NEXT:    vpaddb %xmm1, %xmm1, %xmm2
+; AVX1-NEXT:    vpsrlw $4, %xmm0, %xmm3
+; AVX1-NEXT:    vpand {{.*}}(%rip), %xmm3, %xmm3
+; AVX1-NEXT:    vpblendvb %xmm1, %xmm3, %xmm0, %xmm0
+; AVX1-NEXT:    vpsrlw $2, %xmm0, %xmm1
+; AVX1-NEXT:    vpand {{.*}}(%rip), %xmm1, %xmm1
+; AVX1-NEXT:    vpblendvb %xmm2, %xmm1, %xmm0, %xmm0
+; AVX1-NEXT:    vpsrlw $1, %xmm0, %xmm1
+; AVX1-NEXT:    vpand {{.*}}(%rip), %xmm1, %xmm1
+; AVX1-NEXT:    vpaddb %xmm2, %xmm2, %xmm2
+; AVX1-NEXT:    vpblendvb %xmm2, %xmm1, %xmm0, %xmm0
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: splatvar_shift_v16i8:
+; AVX2:       # BB#0:
+; AVX2-NEXT:    vpbroadcastb %xmm1, %xmm1
+; AVX2-NEXT:    vpsllw $5, %xmm1, %xmm1
+; AVX2-NEXT:    vpsrlw $4, %xmm0, %xmm2
+; AVX2-NEXT:    vpand {{.*}}(%rip), %xmm2, %xmm2
+; AVX2-NEXT:    vpblendvb %xmm1, %xmm2, %xmm0, %xmm0
+; AVX2-NEXT:    vpsrlw $2, %xmm0, %xmm2
+; AVX2-NEXT:    vpand {{.*}}(%rip), %xmm2, %xmm2
+; AVX2-NEXT:    vpaddb %xmm1, %xmm1, %xmm1
+; AVX2-NEXT:    vpblendvb %xmm1, %xmm2, %xmm0, %xmm0
+; AVX2-NEXT:    vpsrlw $1, %xmm0, %xmm2
+; AVX2-NEXT:    vpand {{.*}}(%rip), %xmm2, %xmm2
+; AVX2-NEXT:    vpaddb %xmm1, %xmm1, %xmm1
+; AVX2-NEXT:    vpblendvb %xmm1, %xmm2, %xmm0, %xmm0
+; AVX2-NEXT:    retq
+;
+; X32-SSE-LABEL: splatvar_shift_v16i8:
+; X32-SSE:       # BB#0:
+; X32-SSE-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
+; X32-SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,1,0,3]
+; X32-SSE-NEXT:    pshuflw {{.*#+}} xmm1 = xmm1[0,0,0,0,4,5,6,7]
+; X32-SSE-NEXT:    pshufhw {{.*#+}} xmm2 = xmm1[0,1,2,3,4,4,4,4]
+; X32-SSE-NEXT:    psllw $5, %xmm2
+; X32-SSE-NEXT:    pxor %xmm1, %xmm1
+; X32-SSE-NEXT:    pxor %xmm3, %xmm3
+; X32-SSE-NEXT:    pcmpgtb %xmm2, %xmm3
+; X32-SSE-NEXT:    movdqa %xmm3, %xmm4
+; X32-SSE-NEXT:    pandn %xmm0, %xmm4
+; X32-SSE-NEXT:    psrlw $4, %xmm0
+; X32-SSE-NEXT:    pand .LCPI7_0, %xmm0
+; X32-SSE-NEXT:    pand %xmm3, %xmm0
+; X32-SSE-NEXT:    por %xmm4, %xmm0
+; X32-SSE-NEXT:    paddb %xmm2, %xmm2
+; X32-SSE-NEXT:    pxor %xmm3, %xmm3
+; X32-SSE-NEXT:    pcmpgtb %xmm2, %xmm3
+; X32-SSE-NEXT:    movdqa %xmm3, %xmm4
+; X32-SSE-NEXT:    pandn %xmm0, %xmm4
+; X32-SSE-NEXT:    psrlw $2, %xmm0
+; X32-SSE-NEXT:    pand .LCPI7_1, %xmm0
+; X32-SSE-NEXT:    pand %xmm3, %xmm0
+; X32-SSE-NEXT:    por %xmm4, %xmm0
+; X32-SSE-NEXT:    paddb %xmm2, %xmm2
+; X32-SSE-NEXT:    pcmpgtb %xmm2, %xmm1
+; X32-SSE-NEXT:    movdqa %xmm1, %xmm2
+; X32-SSE-NEXT:    pandn %xmm0, %xmm2
+; X32-SSE-NEXT:    psrlw $1, %xmm0
+; X32-SSE-NEXT:    pand .LCPI7_2, %xmm0
+; X32-SSE-NEXT:    pand %xmm1, %xmm0
+; X32-SSE-NEXT:    por %xmm2, %xmm0
+; X32-SSE-NEXT:    retl
+  %splat = shufflevector <16 x i8> %b, <16 x i8> undef, <16 x i32> zeroinitializer
+  %shift = lshr <16 x i8> %a, %splat
+  ret <16 x i8> %shift
+}
+
+;
+; Constant Shifts
+;
+
+define <2 x i64> @constant_shift_v2i64(<2 x i64> %a) nounwind {
+; SSE2-LABEL: constant_shift_v2i64:
+; SSE2:       # BB#0:
+; SSE2-NEXT:    movdqa %xmm0, %xmm1
+; SSE2-NEXT:    psrlq $7, %xmm1
+; SSE2-NEXT:    psrlq $1, %xmm0
+; SSE2-NEXT:    movsd {{.*#+}} xmm1 = xmm0[0],xmm1[1]
+; SSE2-NEXT:    movapd %xmm1, %xmm0
+; SSE2-NEXT:    retq
+;
+; SSE41-LABEL: constant_shift_v2i64:
+; SSE41:       # BB#0:
+; SSE41-NEXT:    movdqa %xmm0, %xmm1
+; SSE41-NEXT:    psrlq $7, %xmm1
+; SSE41-NEXT:    psrlq $1, %xmm0
+; SSE41-NEXT:    pblendw {{.*#+}} xmm0 = xmm0[0,1,2,3],xmm1[4,5,6,7]
+; SSE41-NEXT:    retq
+;
+; AVX1-LABEL: constant_shift_v2i64:
+; AVX1:       # BB#0:
+; AVX1-NEXT:    vpsrlq $7, %xmm0, %xmm1
+; AVX1-NEXT:    vpsrlq $1, %xmm0, %xmm0
+; AVX1-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0,1,2,3],xmm1[4,5,6,7]
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: constant_shift_v2i64:
+; AVX2:       # BB#0:
+; AVX2-NEXT:    vpsrlvq {{.*}}(%rip), %xmm0, %xmm0
+; AVX2-NEXT:    retq
+;
+; X32-SSE-LABEL: constant_shift_v2i64:
+; X32-SSE:       # BB#0:
+; X32-SSE-NEXT:    movdqa %xmm0, %xmm1
+; X32-SSE-NEXT:    psrlq $7, %xmm1
+; X32-SSE-NEXT:    psrlq $1, %xmm0
+; X32-SSE-NEXT:    movsd {{.*#+}} xmm1 = xmm0[0],xmm1[1]
+; X32-SSE-NEXT:    movapd %xmm1, %xmm0
+; X32-SSE-NEXT:    retl
+  %shift = lshr <2 x i64> %a, <i64 1, i64 7>
+  ret <2 x i64> %shift
+}
+
+define <4 x i32> @constant_shift_v4i32(<4 x i32> %a) nounwind {
+; SSE2-LABEL: constant_shift_v4i32:
+; SSE2:       # BB#0:
+; SSE2-NEXT:    movdqa %xmm0, %xmm1
+; SSE2-NEXT:    psrld $7, %xmm1
+; SSE2-NEXT:    movdqa %xmm0, %xmm2
+; SSE2-NEXT:    psrld $5, %xmm2
+; SSE2-NEXT:    movsd {{.*#+}} xmm1 = xmm2[0],xmm1[1]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,3,2,3]
+; SSE2-NEXT:    movdqa %xmm0, %xmm2
+; SSE2-NEXT:    psrld $6, %xmm2
+; SSE2-NEXT:    psrld $4, %xmm0
+; SSE2-NEXT:    movsd {{.*#+}} xmm2 = xmm0[0],xmm2[1]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[0,2,2,3]
+; SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
+; SSE2-NEXT:    retq
+;
+; SSE41-LABEL: constant_shift_v4i32:
+; SSE41:       # BB#0:
+; SSE41-NEXT:    movdqa %xmm0, %xmm1
+; SSE41-NEXT:    psrld $7, %xmm1
+; SSE41-NEXT:    movdqa %xmm0, %xmm2
+; SSE41-NEXT:    psrld $5, %xmm2
+; SSE41-NEXT:    pblendw {{.*#+}} xmm2 = xmm2[0,1,2,3],xmm1[4,5,6,7]
+; SSE41-NEXT:    movdqa %xmm0, %xmm1
+; SSE41-NEXT:    psrld $6, %xmm1
+; SSE41-NEXT:    psrld $4, %xmm0
+; SSE41-NEXT:    pblendw {{.*#+}} xmm0 = xmm0[0,1,2,3],xmm1[4,5,6,7]
+; SSE41-NEXT:    pblendw {{.*#+}} xmm0 = xmm0[0,1],xmm2[2,3],xmm0[4,5],xmm2[6,7]
+; SSE41-NEXT:    retq
+;
+; AVX1-LABEL: constant_shift_v4i32:
+; AVX1:       # BB#0:
+; AVX1-NEXT:    vpsrld $7, %xmm0, %xmm1
+; AVX1-NEXT:    vpsrld $5, %xmm0, %xmm2
+; AVX1-NEXT:    vpblendw {{.*#+}} xmm1 = xmm2[0,1,2,3],xmm1[4,5,6,7]
+; AVX1-NEXT:    vpsrld $6, %xmm0, %xmm2
+; AVX1-NEXT:    vpsrld $4, %xmm0, %xmm0
+; AVX1-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0,1,2,3],xmm2[4,5,6,7]
+; AVX1-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0,1],xmm1[2,3],xmm0[4,5],xmm1[6,7]
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: constant_shift_v4i32:
+; AVX2:       # BB#0:
+; AVX2-NEXT:    vpsrlvd {{.*}}(%rip), %xmm0, %xmm0
+; AVX2-NEXT:    retq
+;
+; X32-SSE-LABEL: constant_shift_v4i32:
+; X32-SSE:       # BB#0:
+; X32-SSE-NEXT:    movdqa %xmm0, %xmm1
+; X32-SSE-NEXT:    psrld $7, %xmm1
+; X32-SSE-NEXT:    movdqa %xmm0, %xmm2
+; X32-SSE-NEXT:    psrld $5, %xmm2
+; X32-SSE-NEXT:    movsd {{.*#+}} xmm1 = xmm2[0],xmm1[1]
+; X32-SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,3,2,3]
+; X32-SSE-NEXT:    movdqa %xmm0, %xmm2
+; X32-SSE-NEXT:    psrld $6, %xmm2
+; X32-SSE-NEXT:    psrld $4, %xmm0
+; X32-SSE-NEXT:    movsd {{.*#+}} xmm2 = xmm0[0],xmm2[1]
+; X32-SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[0,2,2,3]
+; X32-SSE-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
+; X32-SSE-NEXT:    retl
+  %shift = lshr <4 x i32> %a, <i32 4, i32 5, i32 6, i32 7>
+  ret <4 x i32> %shift
+}
+
+define <8 x i16> @constant_shift_v8i16(<8 x i16> %a) nounwind {
+; SSE2-LABEL: constant_shift_v8i16:
+; SSE2:       # BB#0:
+; SSE2-NEXT:    movdqa %xmm0, %xmm1
+; SSE2-NEXT:    psrlw $4, %xmm1
+; SSE2-NEXT:    movsd {{.*#+}} xmm1 = xmm0[0],xmm1[1]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[0,2,2,3]
+; SSE2-NEXT:    psrlw $2, %xmm1
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,3,2,3]
+; SSE2-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1]
+; SSE2-NEXT:    movdqa {{.*#+}} xmm0 = [65535,0,65535,0,65535,0,65535,0]
+; SSE2-NEXT:    movdqa %xmm2, %xmm1
+; SSE2-NEXT:    pand %xmm0, %xmm1
+; SSE2-NEXT:    psrlw $1, %xmm2
+; SSE2-NEXT:    pandn %xmm2, %xmm0
+; SSE2-NEXT:    por %xmm1, %xmm0
+; SSE2-NEXT:    retq
+;
+; SSE41-LABEL: constant_shift_v8i16:
+; SSE41:       # BB#0:
+; SSE41-NEXT:    movdqa %xmm0, %xmm1
+; SSE41-NEXT:    movdqa %xmm1, %xmm2
+; SSE41-NEXT:    psrlw $8, %xmm2
+; SSE41-NEXT:    movaps {{.*#+}} xmm0 = [0,4112,8224,12336,16448,20560,24672,28784]
+; SSE41-NEXT:    pblendvb %xmm2, %xmm1
+; SSE41-NEXT:    movdqa %xmm1, %xmm2
+; SSE41-NEXT:    psrlw $4, %xmm2
+; SSE41-NEXT:    movaps {{.*#+}} xmm0 = [0,8224,16448,24672,32896,41120,49344,57568]
+; SSE41-NEXT:    pblendvb %xmm2, %xmm1
+; SSE41-NEXT:    movdqa %xmm1, %xmm2
+; SSE41-NEXT:    psrlw $2, %xmm2
+; SSE41-NEXT:    movaps {{.*#+}} xmm0 = [0,16448,32896,49344,256,16704,33152,49600]
+; SSE41-NEXT:    pblendvb %xmm2, %xmm1
+; SSE41-NEXT:    movdqa %xmm1, %xmm2
+; SSE41-NEXT:    psrlw $1, %xmm2
+; SSE41-NEXT:    movaps {{.*#+}} xmm0 = [0,32896,256,33152,512,33408,768,33664]
+; SSE41-NEXT:    pblendvb %xmm2, %xmm1
+; SSE41-NEXT:    movdqa %xmm1, %xmm0
+; SSE41-NEXT:    retq
+;
+; AVX1-LABEL: constant_shift_v8i16:
+; AVX1:       # BB#0:
+; AVX1-NEXT:    vpsrlw $8, %xmm0, %xmm1
+; AVX1-NEXT:    vmovdqa {{.*#+}} xmm2 = [0,4112,8224,12336,16448,20560,24672,28784]
+; AVX1-NEXT:    vpblendvb %xmm2, %xmm1, %xmm0, %xmm0
+; AVX1-NEXT:    vpsrlw $4, %xmm0, %xmm1
+; AVX1-NEXT:    vmovdqa {{.*#+}} xmm2 = [0,8224,16448,24672,32896,41120,49344,57568]
+; AVX1-NEXT:    vpblendvb %xmm2, %xmm1, %xmm0, %xmm0
+; AVX1-NEXT:    vpsrlw $2, %xmm0, %xmm1
+; AVX1-NEXT:    vmovdqa {{.*#+}} xmm2 = [0,16448,32896,49344,256,16704,33152,49600]
+; AVX1-NEXT:    vpblendvb %xmm2, %xmm1, %xmm0, %xmm0
+; AVX1-NEXT:    vpsrlw $1, %xmm0, %xmm1
+; AVX1-NEXT:    vmovdqa {{.*#+}} xmm2 = [0,32896,256,33152,512,33408,768,33664]
+; AVX1-NEXT:    vpblendvb %xmm2, %xmm1, %xmm0, %xmm0
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: constant_shift_v8i16:
+; AVX2:       # BB#0:
+; AVX2-NEXT:    vpmovzxwd {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
+; AVX2-NEXT:    vpmovzxwd {{.*#+}} ymm1 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero,mem[4],zero,mem[5],zero,mem[6],zero,mem[7],zero
+; AVX2-NEXT:    vpsrlvd %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vpshufb {{.*#+}} ymm0 = ymm0[0,1,4,5,8,9,12,13],zero,zero,zero,zero,zero,zero,zero,zero,ymm0[16,17,20,21,24,25,28,29],zero,zero,zero,zero,zero,zero,zero,zero
+; AVX2-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,2,2,3]
+; AVX2-NEXT:    vzeroupper
+; AVX2-NEXT:    retq
+;
+; X32-SSE-LABEL: constant_shift_v8i16:
+; X32-SSE:       # BB#0:
+; X32-SSE-NEXT:    movdqa %xmm0, %xmm1
+; X32-SSE-NEXT:    psrlw $4, %xmm1
+; X32-SSE-NEXT:    movsd {{.*#+}} xmm1 = xmm0[0],xmm1[1]
+; X32-SSE-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[0,2,2,3]
+; X32-SSE-NEXT:    psrlw $2, %xmm1
+; X32-SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,3,2,3]
+; X32-SSE-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1]
+; X32-SSE-NEXT:    movdqa {{.*#+}} xmm0 = [65535,0,65535,0,65535,0,65535,0]
+; X32-SSE-NEXT:    movdqa %xmm2, %xmm1
+; X32-SSE-NEXT:    pand %xmm0, %xmm1
+; X32-SSE-NEXT:    psrlw $1, %xmm2
+; X32-SSE-NEXT:    pandn %xmm2, %xmm0
+; X32-SSE-NEXT:    por %xmm1, %xmm0
+; X32-SSE-NEXT:    retl
+  %shift = lshr <8 x i16> %a, <i16 0, i16 1, i16 2, i16 3, i16 4, i16 5, i16 6, i16 7>
+  ret <8 x i16> %shift
+}
+
+define <16 x i8> @constant_shift_v16i8(<16 x i8> %a) nounwind {
+; SSE2-LABEL: constant_shift_v16i8:
+; SSE2:       # BB#0:
+; SSE2-NEXT:    movdqa {{.*#+}} xmm2 = [0,1,2,3,4,5,6,7,7,6,5,4,3,2,1,0]
+; SSE2-NEXT:    psllw $5, %xmm2
+; SSE2-NEXT:    pxor %xmm1, %xmm1
+; SSE2-NEXT:    pxor %xmm3, %xmm3
+; SSE2-NEXT:    pcmpgtb %xmm2, %xmm3
+; SSE2-NEXT:    movdqa %xmm3, %xmm4
+; SSE2-NEXT:    pandn %xmm0, %xmm4
+; SSE2-NEXT:    psrlw $4, %xmm0
+; SSE2-NEXT:    pand {{.*}}(%rip), %xmm0
+; SSE2-NEXT:    pand %xmm3, %xmm0
+; SSE2-NEXT:    por %xmm4, %xmm0
+; SSE2-NEXT:    paddb %xmm2, %xmm2
+; SSE2-NEXT:    pxor %xmm3, %xmm3
+; SSE2-NEXT:    pcmpgtb %xmm2, %xmm3
+; SSE2-NEXT:    movdqa %xmm3, %xmm4
+; SSE2-NEXT:    pandn %xmm0, %xmm4
+; SSE2-NEXT:    psrlw $2, %xmm0
+; SSE2-NEXT:    pand {{.*}}(%rip), %xmm0
+; SSE2-NEXT:    pand %xmm3, %xmm0
+; SSE2-NEXT:    por %xmm4, %xmm0
+; SSE2-NEXT:    paddb %xmm2, %xmm2
+; SSE2-NEXT:    pcmpgtb %xmm2, %xmm1
+; SSE2-NEXT:    movdqa %xmm1, %xmm2
+; SSE2-NEXT:    pandn %xmm0, %xmm2
+; SSE2-NEXT:    psrlw $1, %xmm0
+; SSE2-NEXT:    pand {{.*}}(%rip), %xmm0
+; SSE2-NEXT:    pand %xmm1, %xmm0
+; SSE2-NEXT:    por %xmm2, %xmm0
+; SSE2-NEXT:    retq
+;
+; SSE41-LABEL: constant_shift_v16i8:
+; SSE41:       # BB#0:
+; SSE41-NEXT:    movdqa %xmm0, %xmm1
+; SSE41-NEXT:    movdqa {{.*#+}} xmm0 = [0,1,2,3,4,5,6,7,7,6,5,4,3,2,1,0]
+; SSE41-NEXT:    psllw $5, %xmm0
+; SSE41-NEXT:    movdqa %xmm1, %xmm2
+; SSE41-NEXT:    psrlw $4, %xmm2
+; SSE41-NEXT:    pand {{.*}}(%rip), %xmm2
+; SSE41-NEXT:    pblendvb %xmm2, %xmm1
+; SSE41-NEXT:    movdqa %xmm1, %xmm2
+; SSE41-NEXT:    psrlw $2, %xmm2
+; SSE41-NEXT:    pand {{.*}}(%rip), %xmm2
+; SSE41-NEXT:    paddb %xmm0, %xmm0
+; SSE41-NEXT:    pblendvb %xmm2, %xmm1
+; SSE41-NEXT:    movdqa %xmm1, %xmm2
+; SSE41-NEXT:    psrlw $1, %xmm2
+; SSE41-NEXT:    pand {{.*}}(%rip), %xmm2
+; SSE41-NEXT:    paddb %xmm0, %xmm0
+; SSE41-NEXT:    pblendvb %xmm2, %xmm1
+; SSE41-NEXT:    movdqa %xmm1, %xmm0
+; SSE41-NEXT:    retq
+;
+; AVX-LABEL: constant_shift_v16i8:
+; AVX:       # BB#0:
+; AVX-NEXT:    vmovdqa {{.*#+}} xmm1 = [0,1,2,3,4,5,6,7,7,6,5,4,3,2,1,0]
+; AVX-NEXT:    vpsllw $5, %xmm1, %xmm1
+; AVX-NEXT:    vpsrlw $4, %xmm0, %xmm2
+; AVX-NEXT:    vpand {{.*}}(%rip), %xmm2, %xmm2
+; AVX-NEXT:    vpblendvb %xmm1, %xmm2, %xmm0, %xmm0
+; AVX-NEXT:    vpsrlw $2, %xmm0, %xmm2
+; AVX-NEXT:    vpand {{.*}}(%rip), %xmm2, %xmm2
+; AVX-NEXT:    vpaddb %xmm1, %xmm1, %xmm1
+; AVX-NEXT:    vpblendvb %xmm1, %xmm2, %xmm0, %xmm0
+; AVX-NEXT:    vpsrlw $1, %xmm0, %xmm2
+; AVX-NEXT:    vpand {{.*}}(%rip), %xmm2, %xmm2
+; AVX-NEXT:    vpaddb %xmm1, %xmm1, %xmm1
+; AVX-NEXT:    vpblendvb %xmm1, %xmm2, %xmm0, %xmm0
+; AVX-NEXT:    retq
+;
+; X32-SSE-LABEL: constant_shift_v16i8:
+; X32-SSE:       # BB#0:
+; X32-SSE-NEXT:    movdqa {{.*#+}} xmm2 = [0,1,2,3,4,5,6,7,7,6,5,4,3,2,1,0]
+; X32-SSE-NEXT:    psllw $5, %xmm2
+; X32-SSE-NEXT:    pxor %xmm1, %xmm1
+; X32-SSE-NEXT:    pxor %xmm3, %xmm3
+; X32-SSE-NEXT:    pcmpgtb %xmm2, %xmm3
+; X32-SSE-NEXT:    movdqa %xmm3, %xmm4
+; X32-SSE-NEXT:    pandn %xmm0, %xmm4
+; X32-SSE-NEXT:    psrlw $4, %xmm0
+; X32-SSE-NEXT:    pand .LCPI11_1, %xmm0
+; X32-SSE-NEXT:    pand %xmm3, %xmm0
+; X32-SSE-NEXT:    por %xmm4, %xmm0
+; X32-SSE-NEXT:    paddb %xmm2, %xmm2
+; X32-SSE-NEXT:    pxor %xmm3, %xmm3
+; X32-SSE-NEXT:    pcmpgtb %xmm2, %xmm3
+; X32-SSE-NEXT:    movdqa %xmm3, %xmm4
+; X32-SSE-NEXT:    pandn %xmm0, %xmm4
+; X32-SSE-NEXT:    psrlw $2, %xmm0
+; X32-SSE-NEXT:    pand .LCPI11_2, %xmm0
+; X32-SSE-NEXT:    pand %xmm3, %xmm0
+; X32-SSE-NEXT:    por %xmm4, %xmm0
+; X32-SSE-NEXT:    paddb %xmm2, %xmm2
+; X32-SSE-NEXT:    pcmpgtb %xmm2, %xmm1
+; X32-SSE-NEXT:    movdqa %xmm1, %xmm2
+; X32-SSE-NEXT:    pandn %xmm0, %xmm2
+; X32-SSE-NEXT:    psrlw $1, %xmm0
+; X32-SSE-NEXT:    pand .LCPI11_3, %xmm0
+; X32-SSE-NEXT:    pand %xmm1, %xmm0
+; X32-SSE-NEXT:    por %xmm2, %xmm0
+; X32-SSE-NEXT:    retl
+  %shift = lshr <16 x i8> %a, <i8 0, i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 7, i8 6, i8 5, i8 4, i8 3, i8 2, i8 1, i8 0>
+  ret <16 x i8> %shift
+}
+
+;
+; Uniform Constant Shifts
+;
+
+define <2 x i64> @splatconstant_shift_v2i64(<2 x i64> %a) nounwind {
+; SSE-LABEL: splatconstant_shift_v2i64:
+; SSE:       # BB#0:
+; SSE-NEXT:    psrlq $7, %xmm0
+; SSE-NEXT:    retq
+;
+; AVX-LABEL: splatconstant_shift_v2i64:
+; AVX:       # BB#0:
+; AVX-NEXT:    vpsrlq $7, %xmm0, %xmm0
+; AVX-NEXT:    retq
+;
+; X32-SSE-LABEL: splatconstant_shift_v2i64:
+; X32-SSE:       # BB#0:
+; X32-SSE-NEXT:    psrlq $7, %xmm0
+; X32-SSE-NEXT:    retl
+  %shift = lshr <2 x i64> %a, <i64 7, i64 7>
+  ret <2 x i64> %shift
+}
+
+define <4 x i32> @splatconstant_shift_v4i32(<4 x i32> %a) nounwind {
+; SSE-LABEL: splatconstant_shift_v4i32:
+; SSE:       # BB#0:
+; SSE-NEXT:    psrld $5, %xmm0
+; SSE-NEXT:    retq
+;
+; AVX-LABEL: splatconstant_shift_v4i32:
+; AVX:       # BB#0:
+; AVX-NEXT:    vpsrld $5, %xmm0, %xmm0
+; AVX-NEXT:    retq
+;
+; X32-SSE-LABEL: splatconstant_shift_v4i32:
+; X32-SSE:       # BB#0:
+; X32-SSE-NEXT:    psrld $5, %xmm0
+; X32-SSE-NEXT:    retl
+  %shift = lshr <4 x i32> %a, <i32 5, i32 5, i32 5, i32 5>
+  ret <4 x i32> %shift
+}
+
+define <8 x i16> @splatconstant_shift_v8i16(<8 x i16> %a) nounwind {
+; SSE-LABEL: splatconstant_shift_v8i16:
+; SSE:       # BB#0:
+; SSE-NEXT:    psrlw $3, %xmm0
+; SSE-NEXT:    retq
+;
+; AVX-LABEL: splatconstant_shift_v8i16:
+; AVX:       # BB#0:
+; AVX-NEXT:    vpsrlw $3, %xmm0, %xmm0
+; AVX-NEXT:    retq
+;
+; X32-SSE-LABEL: splatconstant_shift_v8i16:
+; X32-SSE:       # BB#0:
+; X32-SSE-NEXT:    psrlw $3, %xmm0
+; X32-SSE-NEXT:    retl
+  %shift = lshr <8 x i16> %a, <i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3>
+  ret <8 x i16> %shift
+}
+
+define <16 x i8> @splatconstant_shift_v16i8(<16 x i8> %a) nounwind {
+; SSE-LABEL: splatconstant_shift_v16i8:
+; SSE:       # BB#0:
+; SSE-NEXT:    psrlw $3, %xmm0
+; SSE-NEXT:    pand {{.*}}(%rip), %xmm0
+; SSE-NEXT:    retq
+;
+; AVX-LABEL: splatconstant_shift_v16i8:
+; AVX:       # BB#0:
+; AVX-NEXT:    vpsrlw $3, %xmm0, %xmm0
+; AVX-NEXT:    vpand {{.*}}(%rip), %xmm0, %xmm0
+; AVX-NEXT:    retq
+;
+; X32-SSE-LABEL: splatconstant_shift_v16i8:
+; X32-SSE:       # BB#0:
+; X32-SSE-NEXT:    psrlw $3, %xmm0
+; X32-SSE-NEXT:    pand .LCPI15_0, %xmm0
+; X32-SSE-NEXT:    retl
+  %shift = lshr <16 x i8> %a, <i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3>
+  ret <16 x i8> %shift
+}
diff --git a/test/CodeGen/X86/vector-shift-lshr-256.ll b/test/CodeGen/X86/vector-shift-lshr-256.ll
new file mode 100644
index 00000000000..03cf0f018d7
--- /dev/null
+++ b/test/CodeGen/X86/vector-shift-lshr-256.ll
@@ -0,0 +1,527 @@
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mcpu=x86-64 -mattr=+avx | FileCheck %s --check-prefix=ALL --check-prefix=AVX --check-prefix=AVX1
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mcpu=x86-64 -mattr=+avx2 | FileCheck %s --check-prefix=ALL --check-prefix=AVX --check-prefix=AVX2
+
+;
+; Variable Shifts
+;
+
+define <4 x i64> @var_shift_v4i64(<4 x i64> %a, <4 x i64> %b) nounwind {
+; AVX1-LABEL: var_shift_v4i64:
+; AVX1:       # BB#0:
+; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm2
+; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm3
+; AVX1-NEXT:    vpsrlq %xmm2, %xmm3, %xmm4
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm2 = xmm2[2,3,0,1]
+; AVX1-NEXT:    vpsrlq %xmm2, %xmm3, %xmm2
+; AVX1-NEXT:    vpblendw {{.*#+}} xmm2 = xmm4[0,1,2,3],xmm2[4,5,6,7]
+; AVX1-NEXT:    vpsrlq %xmm1, %xmm0, %xmm3
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[2,3,0,1]
+; AVX1-NEXT:    vpsrlq %xmm1, %xmm0, %xmm0
+; AVX1-NEXT:    vpblendw {{.*#+}} xmm0 = xmm3[0,1,2,3],xmm0[4,5,6,7]
+; AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm0
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: var_shift_v4i64:
+; AVX2:       # BB#0:
+; AVX2-NEXT:    vpsrlvq %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    retq
+  %shift = lshr <4 x i64> %a, %b
+  ret <4 x i64> %shift
+}
+
+define <8 x i32> @var_shift_v8i32(<8 x i32> %a, <8 x i32> %b) nounwind {
+; AVX1-LABEL: var_shift_v8i32:
+; AVX1:       # BB#0:
+; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm2
+; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm3
+; AVX1-NEXT:    vpsrldq {{.*#+}} xmm4 = xmm3[12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
+; AVX1-NEXT:    vpsrld %xmm4, %xmm2, %xmm4
+; AVX1-NEXT:    vpsrlq $32, %xmm3, %xmm5
+; AVX1-NEXT:    vpsrld %xmm5, %xmm2, %xmm5
+; AVX1-NEXT:    vpblendw {{.*#+}} xmm4 = xmm5[0,1,2,3],xmm4[4,5,6,7]
+; AVX1-NEXT:    vpxor %xmm5, %xmm5, %xmm5
+; AVX1-NEXT:    vpunpckhdq {{.*#+}} xmm6 = xmm3[2],xmm5[2],xmm3[3],xmm5[3]
+; AVX1-NEXT:    vpsrld %xmm6, %xmm2, %xmm6
+; AVX1-NEXT:    vpmovzxdq {{.*#+}} xmm3 = xmm3[0],zero,xmm3[1],zero
+; AVX1-NEXT:    vpsrld %xmm3, %xmm2, %xmm2
+; AVX1-NEXT:    vpblendw {{.*#+}} xmm2 = xmm2[0,1,2,3],xmm6[4,5,6,7]
+; AVX1-NEXT:    vpblendw {{.*#+}} xmm2 = xmm2[0,1],xmm4[2,3],xmm2[4,5],xmm4[6,7]
+; AVX1-NEXT:    vpsrldq {{.*#+}} xmm3 = xmm1[12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
+; AVX1-NEXT:    vpsrld %xmm3, %xmm0, %xmm3
+; AVX1-NEXT:    vpsrlq $32, %xmm1, %xmm4
+; AVX1-NEXT:    vpsrld %xmm4, %xmm0, %xmm4
+; AVX1-NEXT:    vpblendw {{.*#+}} xmm3 = xmm4[0,1,2,3],xmm3[4,5,6,7]
+; AVX1-NEXT:    vpunpckhdq {{.*#+}} xmm4 = xmm1[2],xmm5[2],xmm1[3],xmm5[3]
+; AVX1-NEXT:    vpsrld %xmm4, %xmm0, %xmm4
+; AVX1-NEXT:    vpmovzxdq {{.*#+}} xmm1 = xmm1[0],zero,xmm1[1],zero
+; AVX1-NEXT:    vpsrld %xmm1, %xmm0, %xmm0
+; AVX1-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0,1,2,3],xmm4[4,5,6,7]
+; AVX1-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0,1],xmm3[2,3],xmm0[4,5],xmm3[6,7]
+; AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm0
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: var_shift_v8i32:
+; AVX2:       # BB#0:
+; AVX2-NEXT:    vpsrlvd %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    retq
+  %shift = lshr <8 x i32> %a, %b
+  ret <8 x i32> %shift
+}
+
+define <16 x i16> @var_shift_v16i16(<16 x i16> %a, <16 x i16> %b) nounwind {
+; AVX1-LABEL: var_shift_v16i16:
+; AVX1:       # BB#0:
+; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm2
+; AVX1-NEXT:    vpsllw $12, %xmm2, %xmm3
+; AVX1-NEXT:    vpsllw $4, %xmm2, %xmm2
+; AVX1-NEXT:    vpor %xmm3, %xmm2, %xmm2
+; AVX1-NEXT:    vpaddw %xmm2, %xmm2, %xmm3
+; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm4
+; AVX1-NEXT:    vpsrlw $8, %xmm4, %xmm5
+; AVX1-NEXT:    vpblendvb %xmm2, %xmm5, %xmm4, %xmm2
+; AVX1-NEXT:    vpsrlw $4, %xmm2, %xmm4
+; AVX1-NEXT:    vpblendvb %xmm3, %xmm4, %xmm2, %xmm2
+; AVX1-NEXT:    vpsrlw $2, %xmm2, %xmm4
+; AVX1-NEXT:    vpaddw %xmm3, %xmm3, %xmm3
+; AVX1-NEXT:    vpblendvb %xmm3, %xmm4, %xmm2, %xmm2
+; AVX1-NEXT:    vpsrlw $1, %xmm2, %xmm4
+; AVX1-NEXT:    vpaddw %xmm3, %xmm3, %xmm3
+; AVX1-NEXT:    vpblendvb %xmm3, %xmm4, %xmm2, %xmm2
+; AVX1-NEXT:    vpsllw $12, %xmm1, %xmm3
+; AVX1-NEXT:    vpsllw $4, %xmm1, %xmm1
+; AVX1-NEXT:    vpor %xmm3, %xmm1, %xmm1
+; AVX1-NEXT:    vpaddw %xmm1, %xmm1, %xmm3
+; AVX1-NEXT:    vpsrlw $8, %xmm0, %xmm4
+; AVX1-NEXT:    vpblendvb %xmm1, %xmm4, %xmm0, %xmm0
+; AVX1-NEXT:    vpsrlw $4, %xmm0, %xmm1
+; AVX1-NEXT:    vpblendvb %xmm3, %xmm1, %xmm0, %xmm0
+; AVX1-NEXT:    vpsrlw $2, %xmm0, %xmm1
+; AVX1-NEXT:    vpaddw %xmm3, %xmm3, %xmm3
+; AVX1-NEXT:    vpblendvb %xmm3, %xmm1, %xmm0, %xmm0
+; AVX1-NEXT:    vpsrlw $1, %xmm0, %xmm1
+; AVX1-NEXT:    vpaddw %xmm3, %xmm3, %xmm3
+; AVX1-NEXT:    vpblendvb %xmm3, %xmm1, %xmm0, %xmm0
+; AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm0
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: var_shift_v16i16:
+; AVX2:       # BB#0:
+; AVX2-NEXT:    vpxor %ymm2, %ymm2, %ymm2
+; AVX2-NEXT:    vpunpckhwd {{.*#+}} ymm3 = ymm1[4],ymm2[4],ymm1[5],ymm2[5],ymm1[6],ymm2[6],ymm1[7],ymm2[7],ymm1[12],ymm2[12],ymm1[13],ymm2[13],ymm1[14],ymm2[14],ymm1[15],ymm2[15]
+; AVX2-NEXT:    vpunpckhwd {{.*#+}} ymm4 = ymm0[4,4,5,5,6,6,7,7,12,12,13,13,14,14,15,15]
+; AVX2-NEXT:    vpsrlvd %ymm3, %ymm4, %ymm3
+; AVX2-NEXT:    vpsrld $16, %ymm3, %ymm3
+; AVX2-NEXT:    vpunpcklwd {{.*#+}} ymm1 = ymm1[0],ymm2[0],ymm1[1],ymm2[1],ymm1[2],ymm2[2],ymm1[3],ymm2[3],ymm1[8],ymm2[8],ymm1[9],ymm2[9],ymm1[10],ymm2[10],ymm1[11],ymm2[11]
+; AVX2-NEXT:    vpunpcklwd {{.*#+}} ymm0 = ymm0[0,0,1,1,2,2,3,3,8,8,9,9,10,10,11,11]
+; AVX2-NEXT:    vpsrlvd %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vpsrld $16, %ymm0, %ymm0
+; AVX2-NEXT:    vpackusdw %ymm3, %ymm0, %ymm0
+; AVX2-NEXT:    retq
+  %shift = lshr <16 x i16> %a, %b
+  ret <16 x i16> %shift
+}
+
+define <32 x i8> @var_shift_v32i8(<32 x i8> %a, <32 x i8> %b) nounwind {
+; AVX1-LABEL: var_shift_v32i8:
+; AVX1:       # BB#0:
+; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm2
+; AVX1-NEXT:    vpsrlw $4, %xmm2, %xmm3
+; AVX1-NEXT:    vmovdqa {{.*#+}} xmm4 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
+; AVX1-NEXT:    vpand %xmm4, %xmm3, %xmm3
+; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm5
+; AVX1-NEXT:    vpsllw $5, %xmm5, %xmm5
+; AVX1-NEXT:    vpblendvb %xmm5, %xmm3, %xmm2, %xmm2
+; AVX1-NEXT:    vpsrlw $2, %xmm2, %xmm3
+; AVX1-NEXT:    vmovdqa {{.*#+}} xmm6 = [63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63]
+; AVX1-NEXT:    vpand %xmm6, %xmm3, %xmm3
+; AVX1-NEXT:    vpaddb %xmm5, %xmm5, %xmm5
+; AVX1-NEXT:    vpblendvb %xmm5, %xmm3, %xmm2, %xmm2
+; AVX1-NEXT:    vpsrlw $1, %xmm2, %xmm3
+; AVX1-NEXT:    vmovdqa {{.*#+}} xmm7 = [127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127]
+; AVX1-NEXT:    vpand %xmm7, %xmm3, %xmm3
+; AVX1-NEXT:    vpaddb %xmm5, %xmm5, %xmm5
+; AVX1-NEXT:    vpblendvb %xmm5, %xmm3, %xmm2, %xmm2
+; AVX1-NEXT:    vpsrlw $4, %xmm0, %xmm3
+; AVX1-NEXT:    vpand %xmm4, %xmm3, %xmm3
+; AVX1-NEXT:    vpsllw $5, %xmm1, %xmm1
+; AVX1-NEXT:    vpblendvb %xmm1, %xmm3, %xmm0, %xmm0
+; AVX1-NEXT:    vpsrlw $2, %xmm0, %xmm3
+; AVX1-NEXT:    vpand %xmm6, %xmm3, %xmm3
+; AVX1-NEXT:    vpaddb %xmm1, %xmm1, %xmm1
+; AVX1-NEXT:    vpblendvb %xmm1, %xmm3, %xmm0, %xmm0
+; AVX1-NEXT:    vpsrlw $1, %xmm0, %xmm3
+; AVX1-NEXT:    vpand %xmm7, %xmm3, %xmm3
+; AVX1-NEXT:    vpaddb %xmm1, %xmm1, %xmm1
+; AVX1-NEXT:    vpblendvb %xmm1, %xmm3, %xmm0, %xmm0
+; AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm0
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: var_shift_v32i8:
+; AVX2:       # BB#0:
+; AVX2-NEXT:    vpsllw $5, %ymm1, %ymm1
+; AVX2-NEXT:    vpsrlw $4, %ymm0, %ymm2
+; AVX2-NEXT:    vpand {{.*}}(%rip), %ymm2, %ymm2
+; AVX2-NEXT:    vpblendvb %ymm1, %ymm2, %ymm0, %ymm0
+; AVX2-NEXT:    vpsrlw $2, %ymm0, %ymm2
+; AVX2-NEXT:    vpand {{.*}}(%rip), %ymm2, %ymm2
+; AVX2-NEXT:    vpaddb %ymm1, %ymm1, %ymm1
+; AVX2-NEXT:    vpblendvb %ymm1, %ymm2, %ymm0, %ymm0
+; AVX2-NEXT:    vpsrlw $1, %ymm0, %ymm2
+; AVX2-NEXT:    vpand {{.*}}(%rip), %ymm2, %ymm2
+; AVX2-NEXT:    vpaddb %ymm1, %ymm1, %ymm1
+; AVX2-NEXT:    vpblendvb %ymm1, %ymm2, %ymm0, %ymm0
+; AVX2-NEXT:    retq
+  %shift = lshr <32 x i8> %a, %b
+  ret <32 x i8> %shift
+}
+
+;
+; Uniform Variable Shifts
+;
+
+define <4 x i64> @splatvar_shift_v4i64(<4 x i64> %a, <4 x i64> %b) nounwind {
+; AVX1-LABEL: splatvar_shift_v4i64:
+; AVX1:       # BB#0:
+; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm2
+; AVX1-NEXT:    vpsrlq %xmm1, %xmm2, %xmm2
+; AVX1-NEXT:    vpsrlq %xmm1, %xmm0, %xmm0
+; AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm0
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: splatvar_shift_v4i64:
+; AVX2:       # BB#0:
+; AVX2-NEXT:    vpsrlq %xmm1, %ymm0, %ymm0
+; AVX2-NEXT:    retq
+  %splat = shufflevector <4 x i64> %b, <4 x i64> undef, <4 x i32> zeroinitializer
+  %shift = lshr <4 x i64> %a, %splat
+  ret <4 x i64> %shift
+}
+
+define <8 x i32> @splatvar_shift_v8i32(<8 x i32> %a, <8 x i32> %b) nounwind {
+; AVX1-LABEL: splatvar_shift_v8i32:
+; AVX1:       # BB#0:
+; AVX1-NEXT:    vpxor %xmm2, %xmm2, %xmm2
+; AVX1-NEXT:    vpblendw {{.*#+}} xmm1 = xmm1[0,1],xmm2[2,3,4,5,6,7]
+; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm2
+; AVX1-NEXT:    vpsrld %xmm1, %xmm2, %xmm2
+; AVX1-NEXT:    vpsrld %xmm1, %xmm0, %xmm0
+; AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm0
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: splatvar_shift_v8i32:
+; AVX2:       # BB#0:
+; AVX2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
+; AVX2-NEXT:    vpblendw {{.*#+}} xmm1 = xmm1[0,1],xmm2[2,3,4,5,6,7]
+; AVX2-NEXT:    vpsrld %xmm1, %ymm0, %ymm0
+; AVX2-NEXT:    retq
+  %splat = shufflevector <8 x i32> %b, <8 x i32> undef, <8 x i32> zeroinitializer
+  %shift = lshr <8 x i32> %a, %splat
+  ret <8 x i32> %shift
+}
+
+define <16 x i16> @splatvar_shift_v16i16(<16 x i16> %a, <16 x i16> %b) nounwind {
+; AVX1-LABEL: splatvar_shift_v16i16:
+; AVX1:       # BB#0:
+; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm2
+; AVX1-NEXT:    vmovd %xmm1, %eax
+; AVX1-NEXT:    movzwl %ax, %eax
+; AVX1-NEXT:    vmovd %eax, %xmm1
+; AVX1-NEXT:    vpsrlw %xmm1, %xmm2, %xmm2
+; AVX1-NEXT:    vpsrlw %xmm1, %xmm0, %xmm0
+; AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm0
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: splatvar_shift_v16i16:
+; AVX2:       # BB#0:
+; AVX2-NEXT:    vmovd %xmm1, %eax
+; AVX2-NEXT:    movzwl %ax, %eax
+; AVX2-NEXT:    vmovd %eax, %xmm1
+; AVX2-NEXT:    vpsrlw %xmm1, %ymm0, %ymm0
+; AVX2-NEXT:    retq
+  %splat = shufflevector <16 x i16> %b, <16 x i16> undef, <16 x i32> zeroinitializer
+  %shift = lshr <16 x i16> %a, %splat
+  ret <16 x i16> %shift
+}
+
+define <32 x i8> @splatvar_shift_v32i8(<32 x i8> %a, <32 x i8> %b) nounwind {
+; AVX1-LABEL: splatvar_shift_v32i8:
+; AVX1:       # BB#0:
+; AVX1-NEXT:    vpxor %xmm2, %xmm2, %xmm2
+; AVX1-NEXT:    vpshufb %xmm2, %xmm1, %xmm1
+; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm2
+; AVX1-NEXT:    vpsrlw $4, %xmm2, %xmm3
+; AVX1-NEXT:    vmovdqa {{.*#+}} xmm8 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
+; AVX1-NEXT:    vpand %xmm8, %xmm3, %xmm3
+; AVX1-NEXT:    vpsllw $5, %xmm1, %xmm1
+; AVX1-NEXT:    vpblendvb %xmm1, %xmm3, %xmm2, %xmm2
+; AVX1-NEXT:    vpsrlw $2, %xmm2, %xmm3
+; AVX1-NEXT:    vmovdqa {{.*#+}} xmm5 = [63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63]
+; AVX1-NEXT:    vpand %xmm5, %xmm3, %xmm3
+; AVX1-NEXT:    vpaddb %xmm1, %xmm1, %xmm6
+; AVX1-NEXT:    vpblendvb %xmm6, %xmm3, %xmm2, %xmm2
+; AVX1-NEXT:    vpsrlw $1, %xmm2, %xmm3
+; AVX1-NEXT:    vmovdqa {{.*#+}} xmm7 = [127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127]
+; AVX1-NEXT:    vpand %xmm7, %xmm3, %xmm3
+; AVX1-NEXT:    vpaddb %xmm6, %xmm6, %xmm4
+; AVX1-NEXT:    vpblendvb %xmm4, %xmm3, %xmm2, %xmm2
+; AVX1-NEXT:    vpsrlw $4, %xmm0, %xmm3
+; AVX1-NEXT:    vpand %xmm8, %xmm3, %xmm3
+; AVX1-NEXT:    vpblendvb %xmm1, %xmm3, %xmm0, %xmm0
+; AVX1-NEXT:    vpsrlw $2, %xmm0, %xmm1
+; AVX1-NEXT:    vpand %xmm5, %xmm1, %xmm1
+; AVX1-NEXT:    vpblendvb %xmm6, %xmm1, %xmm0, %xmm0
+; AVX1-NEXT:    vpsrlw $1, %xmm0, %xmm1
+; AVX1-NEXT:    vpand %xmm7, %xmm1, %xmm1
+; AVX1-NEXT:    vpblendvb %xmm4, %xmm1, %xmm0, %xmm0
+; AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm0
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: splatvar_shift_v32i8:
+; AVX2:       # BB#0:
+; AVX2-NEXT:    vpbroadcastb %xmm1, %ymm1
+; AVX2-NEXT:    vpsrlw $4, %ymm0, %ymm2
+; AVX2-NEXT:    vpand {{.*}}(%rip), %ymm2, %ymm2
+; AVX2-NEXT:    vpsllw $5, %ymm1, %ymm1
+; AVX2-NEXT:    vpblendvb %ymm1, %ymm2, %ymm0, %ymm0
+; AVX2-NEXT:    vpsrlw $2, %ymm0, %ymm2
+; AVX2-NEXT:    vpand {{.*}}(%rip), %ymm2, %ymm2
+; AVX2-NEXT:    vpaddb %ymm1, %ymm1, %ymm1
+; AVX2-NEXT:    vpblendvb %ymm1, %ymm2, %ymm0, %ymm0
+; AVX2-NEXT:    vpsrlw $1, %ymm0, %ymm2
+; AVX2-NEXT:    vpand {{.*}}(%rip), %ymm2, %ymm2
+; AVX2-NEXT:    vpaddb %ymm1, %ymm1, %ymm1
+; AVX2-NEXT:    vpblendvb %ymm1, %ymm2, %ymm0, %ymm0
+; AVX2-NEXT:    retq
+  %splat = shufflevector <32 x i8> %b, <32 x i8> undef, <32 x i32> zeroinitializer
+  %shift = lshr <32 x i8> %a, %splat
+  ret <32 x i8> %shift
+}
+
+;
+; Constant Shifts
+;
+
+define <4 x i64> @constant_shift_v4i64(<4 x i64> %a) nounwind {
+; AVX1-LABEL: constant_shift_v4i64:
+; AVX1:       # BB#0:
+; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
+; AVX1-NEXT:    vpsrlq $62, %xmm1, %xmm2
+; AVX1-NEXT:    vpsrlq $31, %xmm1, %xmm1
+; AVX1-NEXT:    vpblendw {{.*#+}} xmm1 = xmm1[0,1,2,3],xmm2[4,5,6,7]
+; AVX1-NEXT:    vpsrlq $7, %xmm0, %xmm2
+; AVX1-NEXT:    vpsrlq $1, %xmm0, %xmm0
+; AVX1-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0,1,2,3],xmm2[4,5,6,7]
+; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: constant_shift_v4i64:
+; AVX2:       # BB#0:
+; AVX2-NEXT:    vpsrlvq {{.*}}(%rip), %ymm0, %ymm0
+; AVX2-NEXT:    retq
+  %shift = lshr <4 x i64> %a, <i64 1, i64 7, i64 31, i64 62>
+  ret <4 x i64> %shift
+}
+
+define <8 x i32> @constant_shift_v8i32(<8 x i32> %a) nounwind {
+; AVX1-LABEL: constant_shift_v8i32:
+; AVX1:       # BB#0:
+; AVX1-NEXT:    vpsrld $7, %xmm0, %xmm1
+; AVX1-NEXT:    vpsrld $5, %xmm0, %xmm2
+; AVX1-NEXT:    vpblendw {{.*#+}} xmm1 = xmm2[0,1,2,3],xmm1[4,5,6,7]
+; AVX1-NEXT:    vpsrld $6, %xmm0, %xmm2
+; AVX1-NEXT:    vpsrld $4, %xmm0, %xmm3
+; AVX1-NEXT:    vpblendw {{.*#+}} xmm2 = xmm3[0,1,2,3],xmm2[4,5,6,7]
+; AVX1-NEXT:    vpblendw {{.*#+}} xmm1 = xmm2[0,1],xmm1[2,3],xmm2[4,5],xmm1[6,7]
+; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm0
+; AVX1-NEXT:    vpsrld $7, %xmm0, %xmm2
+; AVX1-NEXT:    vpsrld $9, %xmm0, %xmm3
+; AVX1-NEXT:    vpblendw {{.*#+}} xmm2 = xmm3[0,1,2,3],xmm2[4,5,6,7]
+; AVX1-NEXT:    vpsrld $8, %xmm0, %xmm0
+; AVX1-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0,1],xmm2[2,3],xmm0[4,5],xmm2[6,7]
+; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: constant_shift_v8i32:
+; AVX2:       # BB#0:
+; AVX2-NEXT:    vpsrlvd {{.*}}(%rip), %ymm0, %ymm0
+; AVX2-NEXT:    retq
+  %shift = lshr <8 x i32> %a, <i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 8, i32 7>
+  ret <8 x i32> %shift
+}
+
+define <16 x i16> @constant_shift_v16i16(<16 x i16> %a) nounwind {
+; AVX1-LABEL: constant_shift_v16i16:
+; AVX1:       # BB#0:
+; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
+; AVX1-NEXT:    vpsrlw $8, %xmm1, %xmm2
+; AVX1-NEXT:    vmovdqa {{.*#+}} xmm3 = [32896,37008,41120,45232,49344,53456,57568,61680]
+; AVX1-NEXT:    vpblendvb %xmm3, %xmm2, %xmm1, %xmm1
+; AVX1-NEXT:    vpsrlw $4, %xmm1, %xmm2
+; AVX1-NEXT:    vmovdqa {{.*#+}} xmm3 = [256,8480,16704,24928,33152,41376,49600,57824]
+; AVX1-NEXT:    vpblendvb %xmm3, %xmm2, %xmm1, %xmm1
+; AVX1-NEXT:    vpsrlw $2, %xmm1, %xmm2
+; AVX1-NEXT:    vmovdqa {{.*#+}} xmm3 = [512,16960,33408,49856,768,17216,33664,50112]
+; AVX1-NEXT:    vpblendvb %xmm3, %xmm2, %xmm1, %xmm1
+; AVX1-NEXT:    vpsrlw $1, %xmm1, %xmm2
+; AVX1-NEXT:    vmovdqa {{.*#+}} xmm3 = [1024,33920,1280,34176,1536,34432,1792,34688]
+; AVX1-NEXT:    vpblendvb %xmm3, %xmm2, %xmm1, %xmm1
+; AVX1-NEXT:    vpsrlw $8, %xmm0, %xmm2
+; AVX1-NEXT:    vmovdqa {{.*#+}} xmm3 = [0,4112,8224,12336,16448,20560,24672,28784]
+; AVX1-NEXT:    vpblendvb %xmm3, %xmm2, %xmm0, %xmm0
+; AVX1-NEXT:    vpsrlw $4, %xmm0, %xmm2
+; AVX1-NEXT:    vmovdqa {{.*#+}} xmm3 = [0,8224,16448,24672,32896,41120,49344,57568]
+; AVX1-NEXT:    vpblendvb %xmm3, %xmm2, %xmm0, %xmm0
+; AVX1-NEXT:    vpsrlw $2, %xmm0, %xmm2
+; AVX1-NEXT:    vmovdqa {{.*#+}} xmm3 = [0,16448,32896,49344,256,16704,33152,49600]
+; AVX1-NEXT:    vpblendvb %xmm3, %xmm2, %xmm0, %xmm0
+; AVX1-NEXT:    vpsrlw $1, %xmm0, %xmm2
+; AVX1-NEXT:    vmovdqa {{.*#+}} xmm3 = [0,32896,256,33152,512,33408,768,33664]
+; AVX1-NEXT:    vpblendvb %xmm3, %xmm2, %xmm0, %xmm0
+; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: constant_shift_v16i16:
+; AVX2:       # BB#0:
+; AVX2-NEXT:    vpxor %ymm1, %ymm1, %ymm1
+; AVX2-NEXT:    vmovdqa {{.*#+}} ymm2 = [0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15]
+; AVX2-NEXT:    vpunpckhwd {{.*#+}} ymm3 = ymm2[4],ymm1[4],ymm2[5],ymm1[5],ymm2[6],ymm1[6],ymm2[7],ymm1[7],ymm2[12],ymm1[12],ymm2[13],ymm1[13],ymm2[14],ymm1[14],ymm2[15],ymm1[15]
+; AVX2-NEXT:    vpunpckhwd {{.*#+}} ymm4 = ymm0[4,4,5,5,6,6,7,7,12,12,13,13,14,14,15,15]
+; AVX2-NEXT:    vpsrlvd %ymm3, %ymm4, %ymm3
+; AVX2-NEXT:    vpsrld $16, %ymm3, %ymm3
+; AVX2-NEXT:    vpunpcklwd {{.*#+}} ymm1 = ymm2[0],ymm1[0],ymm2[1],ymm1[1],ymm2[2],ymm1[2],ymm2[3],ymm1[3],ymm2[8],ymm1[8],ymm2[9],ymm1[9],ymm2[10],ymm1[10],ymm2[11],ymm1[11]
+; AVX2-NEXT:    vpunpcklwd {{.*#+}} ymm0 = ymm0[0,0,1,1,2,2,3,3,8,8,9,9,10,10,11,11]
+; AVX2-NEXT:    vpsrlvd %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vpsrld $16, %ymm0, %ymm0
+; AVX2-NEXT:    vpackusdw %ymm3, %ymm0, %ymm0
+; AVX2-NEXT:    retq
+  %shift = lshr <16 x i16> %a, <i16 0, i16 1, i16 2, i16 3, i16 4, i16 5, i16 6, i16 7, i16 8, i16 9, i16 10, i16 11, i16 12, i16 13, i16 14, i16 15>
+  ret <16 x i16> %shift
+}
+
+define <32 x i8> @constant_shift_v32i8(<32 x i8> %a) nounwind {
+; AVX1-LABEL: constant_shift_v32i8:
+; AVX1:       # BB#0:
+; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
+; AVX1-NEXT:    vpsrlw $4, %xmm1, %xmm2
+; AVX1-NEXT:    vmovdqa {{.*#+}} xmm8 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
+; AVX1-NEXT:    vpand %xmm8, %xmm2, %xmm2
+; AVX1-NEXT:    vmovdqa {{.*#+}} xmm4 = [0,1,2,3,4,5,6,7,7,6,5,4,3,2,1,0]
+; AVX1-NEXT:    vpsllw $5, %xmm4, %xmm4
+; AVX1-NEXT:    vpblendvb %xmm4, %xmm2, %xmm1, %xmm1
+; AVX1-NEXT:    vpsrlw $2, %xmm1, %xmm2
+; AVX1-NEXT:    vmovdqa {{.*#+}} xmm5 = [63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63]
+; AVX1-NEXT:    vpand %xmm5, %xmm2, %xmm2
+; AVX1-NEXT:    vpaddb %xmm4, %xmm4, %xmm6
+; AVX1-NEXT:    vpblendvb %xmm6, %xmm2, %xmm1, %xmm1
+; AVX1-NEXT:    vpsrlw $1, %xmm1, %xmm2
+; AVX1-NEXT:    vmovdqa {{.*#+}} xmm7 = [127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127]
+; AVX1-NEXT:    vpand %xmm7, %xmm2, %xmm2
+; AVX1-NEXT:    vpaddb %xmm6, %xmm6, %xmm3
+; AVX1-NEXT:    vpblendvb %xmm3, %xmm2, %xmm1, %xmm1
+; AVX1-NEXT:    vpsrlw $4, %xmm0, %xmm2
+; AVX1-NEXT:    vpand %xmm8, %xmm2, %xmm2
+; AVX1-NEXT:    vpblendvb %xmm4, %xmm2, %xmm0, %xmm0
+; AVX1-NEXT:    vpsrlw $2, %xmm0, %xmm2
+; AVX1-NEXT:    vpand %xmm5, %xmm2, %xmm2
+; AVX1-NEXT:    vpblendvb %xmm6, %xmm2, %xmm0, %xmm0
+; AVX1-NEXT:    vpsrlw $1, %xmm0, %xmm2
+; AVX1-NEXT:    vpand %xmm7, %xmm2, %xmm2
+; AVX1-NEXT:    vpblendvb %xmm3, %xmm2, %xmm0, %xmm0
+; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: constant_shift_v32i8:
+; AVX2:       # BB#0:
+; AVX2-NEXT:    vmovdqa {{.*#+}} ymm1 = [0,1,2,3,4,5,6,7,7,6,5,4,3,2,1,0,0,1,2,3,4,5,6,7,7,6,5,4,3,2,1,0]
+; AVX2-NEXT:    vpsllw $5, %ymm1, %ymm1
+; AVX2-NEXT:    vpsrlw $4, %ymm0, %ymm2
+; AVX2-NEXT:    vpand {{.*}}(%rip), %ymm2, %ymm2
+; AVX2-NEXT:    vpblendvb %ymm1, %ymm2, %ymm0, %ymm0
+; AVX2-NEXT:    vpsrlw $2, %ymm0, %ymm2
+; AVX2-NEXT:    vpand {{.*}}(%rip), %ymm2, %ymm2
+; AVX2-NEXT:    vpaddb %ymm1, %ymm1, %ymm1
+; AVX2-NEXT:    vpblendvb %ymm1, %ymm2, %ymm0, %ymm0
+; AVX2-NEXT:    vpsrlw $1, %ymm0, %ymm2
+; AVX2-NEXT:    vpand {{.*}}(%rip), %ymm2, %ymm2
+; AVX2-NEXT:    vpaddb %ymm1, %ymm1, %ymm1
+; AVX2-NEXT:    vpblendvb %ymm1, %ymm2, %ymm0, %ymm0
+; AVX2-NEXT:    retq
+  %shift = lshr <32 x i8> %a, <i8 0, i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 7, i8 6, i8 5, i8 4, i8 3, i8 2, i8 1, i8 0, i8 0, i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 7, i8 6, i8 5, i8 4, i8 3, i8 2, i8 1, i8 0>
+  ret <32 x i8> %shift
+}
+
+;
+; Uniform Constant Shifts
+;
+
+define <4 x i64> @splatconstant_shift_v4i64(<4 x i64> %a) nounwind {
+; AVX1-LABEL: splatconstant_shift_v4i64:
+; AVX1:       # BB#0:
+; AVX1-NEXT:    vpsrlq $7, %xmm0, %xmm1
+; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm0
+; AVX1-NEXT:    vpsrlq $7, %xmm0, %xmm0
+; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: splatconstant_shift_v4i64:
+; AVX2:       # BB#0:
+; AVX2-NEXT:    vpsrlq $7, %ymm0, %ymm0
+; AVX2-NEXT:    retq
+  %shift = lshr <4 x i64> %a, <i64 7, i64 7, i64 7, i64 7>
+  ret <4 x i64> %shift
+}
+
+define <8 x i32> @splatconstant_shift_v8i32(<8 x i32> %a) nounwind {
+; AVX1-LABEL: splatconstant_shift_v8i32:
+; AVX1:       # BB#0:
+; AVX1-NEXT:    vpsrld $5, %xmm0, %xmm1
+; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm0
+; AVX1-NEXT:    vpsrld $5, %xmm0, %xmm0
+; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: splatconstant_shift_v8i32:
+; AVX2:       # BB#0:
+; AVX2-NEXT:    vpsrld $5, %ymm0, %ymm0
+; AVX2-NEXT:    retq
+  %shift = lshr <8 x i32> %a, <i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5>
+  ret <8 x i32> %shift
+}
+
+define <16 x i16> @splatconstant_shift_v16i16(<16 x i16> %a) nounwind {
+; AVX1-LABEL: splatconstant_shift_v16i16:
+; AVX1:       # BB#0:
+; AVX1-NEXT:    vpsrlw $3, %xmm0, %xmm1
+; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm0
+; AVX1-NEXT:    vpsrlw $3, %xmm0, %xmm0
+; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: splatconstant_shift_v16i16:
+; AVX2:       # BB#0:
+; AVX2-NEXT:    vpsrlw $3, %ymm0, %ymm0
+; AVX2-NEXT:    retq
+  %shift = lshr <16 x i16> %a, <i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3>
+  ret <16 x i16> %shift
+}
+
+define <32 x i8> @splatconstant_shift_v32i8(<32 x i8> %a) nounwind {
+; AVX1-LABEL: splatconstant_shift_v32i8:
+; AVX1:       # BB#0:
+; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
+; AVX1-NEXT:    vpsrlw $3, %xmm1, %xmm1
+; AVX1-NEXT:    vmovdqa {{.*#+}} xmm2 = [31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31]
+; AVX1-NEXT:    vpand %xmm2, %xmm1, %xmm1
+; AVX1-NEXT:    vpsrlw $3, %xmm0, %xmm0
+; AVX1-NEXT:    vpand %xmm2, %xmm0, %xmm0
+; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: splatconstant_shift_v32i8:
+; AVX2:       # BB#0:
+; AVX2-NEXT:    vpsrlw $3, %ymm0, %ymm0
+; AVX2-NEXT:    vpand {{.*}}(%rip), %ymm0, %ymm0
+; AVX2-NEXT:    retq
+  %shift = lshr <32 x i8> %a, <i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3>
+  ret <32 x i8> %shift
+}
diff --git a/test/CodeGen/X86/vector-shift-shl-128.ll b/test/CodeGen/X86/vector-shift-shl-128.ll
new file mode 100644
index 00000000000..fa6533d39cb
--- /dev/null
+++ b/test/CodeGen/X86/vector-shift-shl-128.ll
@@ -0,0 +1,866 @@
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mcpu=x86-64 | FileCheck %s --check-prefix=ALL --check-prefix=SSE --check-prefix=SSE2
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mcpu=x86-64 -mattr=+sse4.1 | FileCheck %s --check-prefix=ALL --check-prefix=SSE --check-prefix=SSE41
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mcpu=x86-64 -mattr=+avx | FileCheck %s --check-prefix=ALL --check-prefix=AVX --check-prefix=AVX1
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mcpu=x86-64 -mattr=+avx2 | FileCheck %s --check-prefix=ALL --check-prefix=AVX --check-prefix=AVX2
+;
+; Just one 32-bit run to make sure we do reasonable things for i64 shifts.
+; RUN: llc < %s -mtriple=i686-unknown-unknown -mcpu=x86-64 | FileCheck %s --check-prefix=ALL --check-prefix=X32-SSE --check-prefix=X32-SSE2
+
+;
+; Variable Shifts
+;
+
+define <2 x i64> @var_shift_v2i64(<2 x i64> %a, <2 x i64> %b) nounwind {
+; SSE2-LABEL: var_shift_v2i64:
+; SSE2:       # BB#0:
+; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm1[2,3,0,1]
+; SSE2-NEXT:    movdqa %xmm0, %xmm2
+; SSE2-NEXT:    psllq %xmm3, %xmm2
+; SSE2-NEXT:    psllq %xmm1, %xmm0
+; SSE2-NEXT:    movsd {{.*#+}} xmm2 = xmm0[0],xmm2[1]
+; SSE2-NEXT:    movapd %xmm2, %xmm0
+; SSE2-NEXT:    retq
+;
+; SSE41-LABEL: var_shift_v2i64:
+; SSE41:       # BB#0:
+; SSE41-NEXT:    movdqa %xmm0, %xmm2
+; SSE41-NEXT:    psllq %xmm1, %xmm2
+; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[2,3,0,1]
+; SSE41-NEXT:    psllq %xmm1, %xmm0
+; SSE41-NEXT:    pblendw {{.*#+}} xmm0 = xmm2[0,1,2,3],xmm0[4,5,6,7]
+; SSE41-NEXT:    retq
+;
+; AVX1-LABEL: var_shift_v2i64:
+; AVX1:       # BB#0:
+; AVX1-NEXT:    vpsllq %xmm1, %xmm0, %xmm2
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[2,3,0,1]
+; AVX1-NEXT:    vpsllq %xmm1, %xmm0, %xmm0
+; AVX1-NEXT:    vpblendw {{.*#+}} xmm0 = xmm2[0,1,2,3],xmm0[4,5,6,7]
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: var_shift_v2i64:
+; AVX2:       # BB#0:
+; AVX2-NEXT:    vpsllvq %xmm1, %xmm0, %xmm0
+; AVX2-NEXT:    retq
+;
+; X32-SSE-LABEL: var_shift_v2i64:
+; X32-SSE:       # BB#0:
+; X32-SSE-NEXT:    pshufd {{.*#+}} xmm3 = xmm1[2,3,0,1]
+; X32-SSE-NEXT:    movdqa %xmm0, %xmm2
+; X32-SSE-NEXT:    psllq %xmm3, %xmm2
+; X32-SSE-NEXT:    movq {{.*#+}} xmm1 = xmm1[0],zero
+; X32-SSE-NEXT:    psllq %xmm1, %xmm0
+; X32-SSE-NEXT:    movsd {{.*#+}} xmm2 = xmm0[0],xmm2[1]
+; X32-SSE-NEXT:    movapd %xmm2, %xmm0
+; X32-SSE-NEXT:    retl
+  %shift = shl <2 x i64> %a, %b
+  ret <2 x i64> %shift
+}
+
+define <4 x i32> @var_shift_v4i32(<4 x i32> %a, <4 x i32> %b) nounwind {
+; SSE2-LABEL: var_shift_v4i32:
+; SSE2:       # BB#0:
+; SSE2-NEXT:    pslld $23, %xmm1
+; SSE2-NEXT:    paddd {{.*}}(%rip), %xmm1
+; SSE2-NEXT:    cvttps2dq %xmm1, %xmm1
+; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
+; SSE2-NEXT:    pmuludq %xmm0, %xmm1
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
+; SSE2-NEXT:    pmuludq %xmm2, %xmm0
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
+; SSE2-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
+; SSE2-NEXT:    movdqa %xmm1, %xmm0
+; SSE2-NEXT:    retq
+;
+; SSE41-LABEL: var_shift_v4i32:
+; SSE41:       # BB#0:
+; SSE41-NEXT:    pslld $23, %xmm1
+; SSE41-NEXT:    paddd {{.*}}(%rip), %xmm1
+; SSE41-NEXT:    cvttps2dq %xmm1, %xmm1
+; SSE41-NEXT:    pmulld %xmm1, %xmm0
+; SSE41-NEXT:    retq
+;
+; AVX1-LABEL: var_shift_v4i32:
+; AVX1:       # BB#0:
+; AVX1-NEXT:    vpslld $23, %xmm1, %xmm1
+; AVX1-NEXT:    vpaddd {{.*}}(%rip), %xmm1, %xmm1
+; AVX1-NEXT:    vcvttps2dq %xmm1, %xmm1
+; AVX1-NEXT:    vpmulld %xmm0, %xmm1, %xmm0
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: var_shift_v4i32:
+; AVX2:       # BB#0:
+; AVX2-NEXT:    vpsllvd %xmm1, %xmm0, %xmm0
+; AVX2-NEXT:    retq
+;
+; X32-SSE-LABEL: var_shift_v4i32:
+; X32-SSE:       # BB#0:
+; X32-SSE-NEXT:    pslld $23, %xmm1
+; X32-SSE-NEXT:    paddd .LCPI1_0, %xmm1
+; X32-SSE-NEXT:    cvttps2dq %xmm1, %xmm1
+; X32-SSE-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
+; X32-SSE-NEXT:    pmuludq %xmm0, %xmm1
+; X32-SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3]
+; X32-SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
+; X32-SSE-NEXT:    pmuludq %xmm2, %xmm0
+; X32-SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
+; X32-SSE-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
+; X32-SSE-NEXT:    movdqa %xmm1, %xmm0
+; X32-SSE-NEXT:    retl
+  %shift = shl <4 x i32> %a, %b
+  ret <4 x i32> %shift
+}
+
+define <8 x i16> @var_shift_v8i16(<8 x i16> %a, <8 x i16> %b) nounwind {
+; SSE2-LABEL: var_shift_v8i16:
+; SSE2:       # BB#0:
+; SSE2-NEXT:    psllw $12, %xmm1
+; SSE2-NEXT:    movdqa %xmm1, %xmm2
+; SSE2-NEXT:    psraw $15, %xmm2
+; SSE2-NEXT:    movdqa %xmm2, %xmm3
+; SSE2-NEXT:    pandn %xmm0, %xmm3
+; SSE2-NEXT:    psllw $8, %xmm0
+; SSE2-NEXT:    pand %xmm2, %xmm0
+; SSE2-NEXT:    por %xmm3, %xmm0
+; SSE2-NEXT:    paddw %xmm1, %xmm1
+; SSE2-NEXT:    movdqa %xmm1, %xmm2
+; SSE2-NEXT:    psraw $15, %xmm2
+; SSE2-NEXT:    movdqa %xmm2, %xmm3
+; SSE2-NEXT:    pandn %xmm0, %xmm3
+; SSE2-NEXT:    psllw $4, %xmm0
+; SSE2-NEXT:    pand %xmm2, %xmm0
+; SSE2-NEXT:    por %xmm3, %xmm0
+; SSE2-NEXT:    paddw %xmm1, %xmm1
+; SSE2-NEXT:    movdqa %xmm1, %xmm2
+; SSE2-NEXT:    psraw $15, %xmm2
+; SSE2-NEXT:    movdqa %xmm2, %xmm3
+; SSE2-NEXT:    pandn %xmm0, %xmm3
+; SSE2-NEXT:    psllw $2, %xmm0
+; SSE2-NEXT:    pand %xmm2, %xmm0
+; SSE2-NEXT:    por %xmm3, %xmm0
+; SSE2-NEXT:    paddw %xmm1, %xmm1
+; SSE2-NEXT:    psraw $15, %xmm1
+; SSE2-NEXT:    movdqa %xmm1, %xmm2
+; SSE2-NEXT:    pandn %xmm0, %xmm2
+; SSE2-NEXT:    psllw $1, %xmm0
+; SSE2-NEXT:    pand %xmm1, %xmm0
+; SSE2-NEXT:    por %xmm2, %xmm0
+; SSE2-NEXT:    retq
+;
+; SSE41-LABEL: var_shift_v8i16:
+; SSE41:       # BB#0:
+; SSE41-NEXT:    movdqa %xmm0, %xmm2
+; SSE41-NEXT:    movdqa %xmm1, %xmm0
+; SSE41-NEXT:    psllw $12, %xmm0
+; SSE41-NEXT:    psllw $4, %xmm1
+; SSE41-NEXT:    por %xmm0, %xmm1
+; SSE41-NEXT:    movdqa %xmm1, %xmm3
+; SSE41-NEXT:    paddw %xmm3, %xmm3
+; SSE41-NEXT:    movdqa %xmm2, %xmm4
+; SSE41-NEXT:    psllw $8, %xmm4
+; SSE41-NEXT:    movdqa %xmm1, %xmm0
+; SSE41-NEXT:    pblendvb %xmm4, %xmm2
+; SSE41-NEXT:    movdqa %xmm2, %xmm1
+; SSE41-NEXT:    psllw $4, %xmm1
+; SSE41-NEXT:    movdqa %xmm3, %xmm0
+; SSE41-NEXT:    pblendvb %xmm1, %xmm2
+; SSE41-NEXT:    movdqa %xmm2, %xmm1
+; SSE41-NEXT:    psllw $2, %xmm1
+; SSE41-NEXT:    paddw %xmm3, %xmm3
+; SSE41-NEXT:    movdqa %xmm3, %xmm0
+; SSE41-NEXT:    pblendvb %xmm1, %xmm2
+; SSE41-NEXT:    movdqa %xmm2, %xmm1
+; SSE41-NEXT:    psllw $1, %xmm1
+; SSE41-NEXT:    paddw %xmm3, %xmm3
+; SSE41-NEXT:    movdqa %xmm3, %xmm0
+; SSE41-NEXT:    pblendvb %xmm1, %xmm2
+; SSE41-NEXT:    movdqa %xmm2, %xmm0
+; SSE41-NEXT:    retq
+;
+; AVX1-LABEL: var_shift_v8i16:
+; AVX1:       # BB#0:
+; AVX1-NEXT:    vpsllw $12, %xmm1, %xmm2
+; AVX1-NEXT:    vpsllw $4, %xmm1, %xmm1
+; AVX1-NEXT:    vpor %xmm2, %xmm1, %xmm1
+; AVX1-NEXT:    vpaddw %xmm1, %xmm1, %xmm2
+; AVX1-NEXT:    vpsllw $8, %xmm0, %xmm3
+; AVX1-NEXT:    vpblendvb %xmm1, %xmm3, %xmm0, %xmm0
+; AVX1-NEXT:    vpsllw $4, %xmm0, %xmm1
+; AVX1-NEXT:    vpblendvb %xmm2, %xmm1, %xmm0, %xmm0
+; AVX1-NEXT:    vpsllw $2, %xmm0, %xmm1
+; AVX1-NEXT:    vpaddw %xmm2, %xmm2, %xmm2
+; AVX1-NEXT:    vpblendvb %xmm2, %xmm1, %xmm0, %xmm0
+; AVX1-NEXT:    vpsllw $1, %xmm0, %xmm1
+; AVX1-NEXT:    vpaddw %xmm2, %xmm2, %xmm2
+; AVX1-NEXT:    vpblendvb %xmm2, %xmm1, %xmm0, %xmm0
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: var_shift_v8i16:
+; AVX2:       # BB#0:
+; AVX2-NEXT:    vpmovzxwd {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
+; AVX2-NEXT:    vpmovzxwd {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
+; AVX2-NEXT:    vpsllvd %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vpshufb {{.*#+}} ymm0 = ymm0[0,1,4,5,8,9,12,13],zero,zero,zero,zero,zero,zero,zero,zero,ymm0[16,17,20,21,24,25,28,29],zero,zero,zero,zero,zero,zero,zero,zero
+; AVX2-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,2,2,3]
+; AVX2-NEXT:    vzeroupper
+; AVX2-NEXT:    retq
+;
+; X32-SSE-LABEL: var_shift_v8i16:
+; X32-SSE:       # BB#0:
+; X32-SSE-NEXT:    psllw $12, %xmm1
+; X32-SSE-NEXT:    movdqa %xmm1, %xmm2
+; X32-SSE-NEXT:    psraw $15, %xmm2
+; X32-SSE-NEXT:    movdqa %xmm2, %xmm3
+; X32-SSE-NEXT:    pandn %xmm0, %xmm3
+; X32-SSE-NEXT:    psllw $8, %xmm0
+; X32-SSE-NEXT:    pand %xmm2, %xmm0
+; X32-SSE-NEXT:    por %xmm3, %xmm0
+; X32-SSE-NEXT:    paddw %xmm1, %xmm1
+; X32-SSE-NEXT:    movdqa %xmm1, %xmm2
+; X32-SSE-NEXT:    psraw $15, %xmm2
+; X32-SSE-NEXT:    movdqa %xmm2, %xmm3
+; X32-SSE-NEXT:    pandn %xmm0, %xmm3
+; X32-SSE-NEXT:    psllw $4, %xmm0
+; X32-SSE-NEXT:    pand %xmm2, %xmm0
+; X32-SSE-NEXT:    por %xmm3, %xmm0
+; X32-SSE-NEXT:    paddw %xmm1, %xmm1
+; X32-SSE-NEXT:    movdqa %xmm1, %xmm2
+; X32-SSE-NEXT:    psraw $15, %xmm2
+; X32-SSE-NEXT:    movdqa %xmm2, %xmm3
+; X32-SSE-NEXT:    pandn %xmm0, %xmm3
+; X32-SSE-NEXT:    psllw $2, %xmm0
+; X32-SSE-NEXT:    pand %xmm2, %xmm0
+; X32-SSE-NEXT:    por %xmm3, %xmm0
+; X32-SSE-NEXT:    paddw %xmm1, %xmm1
+; X32-SSE-NEXT:    psraw $15, %xmm1
+; X32-SSE-NEXT:    movdqa %xmm1, %xmm2
+; X32-SSE-NEXT:    pandn %xmm0, %xmm2
+; X32-SSE-NEXT:    psllw $1, %xmm0
+; X32-SSE-NEXT:    pand %xmm1, %xmm0
+; X32-SSE-NEXT:    por %xmm2, %xmm0
+; X32-SSE-NEXT:    retl
+  %shift = shl <8 x i16> %a, %b
+  ret <8 x i16> %shift
+}
+
+define <16 x i8> @var_shift_v16i8(<16 x i8> %a, <16 x i8> %b) nounwind {
+; SSE2-LABEL: var_shift_v16i8:
+; SSE2:       # BB#0:
+; SSE2-NEXT:    psllw $5, %xmm1
+; SSE2-NEXT:    pxor %xmm2, %xmm2
+; SSE2-NEXT:    pxor %xmm3, %xmm3
+; SSE2-NEXT:    pcmpgtb %xmm1, %xmm3
+; SSE2-NEXT:    movdqa %xmm3, %xmm4
+; SSE2-NEXT:    pandn %xmm0, %xmm4
+; SSE2-NEXT:    psllw $4, %xmm0
+; SSE2-NEXT:    pand {{.*}}(%rip), %xmm0
+; SSE2-NEXT:    pand %xmm3, %xmm0
+; SSE2-NEXT:    por %xmm4, %xmm0
+; SSE2-NEXT:    paddb %xmm1, %xmm1
+; SSE2-NEXT:    pxor %xmm3, %xmm3
+; SSE2-NEXT:    pcmpgtb %xmm1, %xmm3
+; SSE2-NEXT:    movdqa %xmm3, %xmm4
+; SSE2-NEXT:    pandn %xmm0, %xmm4
+; SSE2-NEXT:    psllw $2, %xmm0
+; SSE2-NEXT:    pand {{.*}}(%rip), %xmm0
+; SSE2-NEXT:    pand %xmm3, %xmm0
+; SSE2-NEXT:    por %xmm4, %xmm0
+; SSE2-NEXT:    paddb %xmm1, %xmm1
+; SSE2-NEXT:    pcmpgtb %xmm1, %xmm2
+; SSE2-NEXT:    movdqa %xmm2, %xmm1
+; SSE2-NEXT:    pandn %xmm0, %xmm1
+; SSE2-NEXT:    paddb %xmm0, %xmm0
+; SSE2-NEXT:    pand %xmm2, %xmm0
+; SSE2-NEXT:    por %xmm1, %xmm0
+; SSE2-NEXT:    retq
+;
+; SSE41-LABEL: var_shift_v16i8:
+; SSE41:       # BB#0:
+; SSE41-NEXT:    movdqa %xmm0, %xmm2
+; SSE41-NEXT:    psllw $5, %xmm1
+; SSE41-NEXT:    movdqa %xmm2, %xmm3
+; SSE41-NEXT:    psllw $4, %xmm3
+; SSE41-NEXT:    pand {{.*}}(%rip), %xmm3
+; SSE41-NEXT:    movdqa %xmm1, %xmm0
+; SSE41-NEXT:    pblendvb %xmm3, %xmm2
+; SSE41-NEXT:    movdqa %xmm2, %xmm3
+; SSE41-NEXT:    psllw $2, %xmm3
+; SSE41-NEXT:    pand {{.*}}(%rip), %xmm3
+; SSE41-NEXT:    paddb %xmm1, %xmm1
+; SSE41-NEXT:    movdqa %xmm1, %xmm0
+; SSE41-NEXT:    pblendvb %xmm3, %xmm2
+; SSE41-NEXT:    movdqa %xmm2, %xmm3
+; SSE41-NEXT:    paddb %xmm3, %xmm3
+; SSE41-NEXT:    paddb %xmm1, %xmm1
+; SSE41-NEXT:    movdqa %xmm1, %xmm0
+; SSE41-NEXT:    pblendvb %xmm3, %xmm2
+; SSE41-NEXT:    movdqa %xmm2, %xmm0
+; SSE41-NEXT:    retq
+;
+; AVX-LABEL: var_shift_v16i8:
+; AVX:       # BB#0:
+; AVX-NEXT:    vpsllw $5, %xmm1, %xmm1
+; AVX-NEXT:    vpsllw $4, %xmm0, %xmm2
+; AVX-NEXT:    vpand {{.*}}(%rip), %xmm2, %xmm2
+; AVX-NEXT:    vpblendvb %xmm1, %xmm2, %xmm0, %xmm0
+; AVX-NEXT:    vpsllw $2, %xmm0, %xmm2
+; AVX-NEXT:    vpand {{.*}}(%rip), %xmm2, %xmm2
+; AVX-NEXT:    vpaddb %xmm1, %xmm1, %xmm1
+; AVX-NEXT:    vpblendvb %xmm1, %xmm2, %xmm0, %xmm0
+; AVX-NEXT:    vpaddb %xmm0, %xmm0, %xmm2
+; AVX-NEXT:    vpaddb %xmm1, %xmm1, %xmm1
+; AVX-NEXT:    vpblendvb %xmm1, %xmm2, %xmm0, %xmm0
+; AVX-NEXT:    retq
+;
+; X32-SSE-LABEL: var_shift_v16i8:
+; X32-SSE:       # BB#0:
+; X32-SSE-NEXT:    psllw $5, %xmm1
+; X32-SSE-NEXT:    pxor %xmm2, %xmm2
+; X32-SSE-NEXT:    pxor %xmm3, %xmm3
+; X32-SSE-NEXT:    pcmpgtb %xmm1, %xmm3
+; X32-SSE-NEXT:    movdqa %xmm3, %xmm4
+; X32-SSE-NEXT:    pandn %xmm0, %xmm4
+; X32-SSE-NEXT:    psllw $4, %xmm0
+; X32-SSE-NEXT:    pand .LCPI3_0, %xmm0
+; X32-SSE-NEXT:    pand %xmm3, %xmm0
+; X32-SSE-NEXT:    por %xmm4, %xmm0
+; X32-SSE-NEXT:    paddb %xmm1, %xmm1
+; X32-SSE-NEXT:    pxor %xmm3, %xmm3
+; X32-SSE-NEXT:    pcmpgtb %xmm1, %xmm3
+; X32-SSE-NEXT:    movdqa %xmm3, %xmm4
+; X32-SSE-NEXT:    pandn %xmm0, %xmm4
+; X32-SSE-NEXT:    psllw $2, %xmm0
+; X32-SSE-NEXT:    pand .LCPI3_1, %xmm0
+; X32-SSE-NEXT:    pand %xmm3, %xmm0
+; X32-SSE-NEXT:    por %xmm4, %xmm0
+; X32-SSE-NEXT:    paddb %xmm1, %xmm1
+; X32-SSE-NEXT:    pcmpgtb %xmm1, %xmm2
+; X32-SSE-NEXT:    movdqa %xmm2, %xmm1
+; X32-SSE-NEXT:    pandn %xmm0, %xmm1
+; X32-SSE-NEXT:    paddb %xmm0, %xmm0
+; X32-SSE-NEXT:    pand %xmm2, %xmm0
+; X32-SSE-NEXT:    por %xmm1, %xmm0
+; X32-SSE-NEXT:    retl
+  %shift = shl <16 x i8> %a, %b
+  ret <16 x i8> %shift
+}
+
+;
+; Uniform Variable Shifts
+;
+
+define <2 x i64> @splatvar_shift_v2i64(<2 x i64> %a, <2 x i64> %b) nounwind {
+; SSE-LABEL: splatvar_shift_v2i64:
+; SSE:       # BB#0:
+; SSE-NEXT:    psllq %xmm1, %xmm0
+; SSE-NEXT:    retq
+;
+; AVX-LABEL: splatvar_shift_v2i64:
+; AVX:       # BB#0:
+; AVX-NEXT:    vpsllq %xmm1, %xmm0, %xmm0
+; AVX-NEXT:    retq
+;
+; X32-SSE-LABEL: splatvar_shift_v2i64:
+; X32-SSE:       # BB#0:
+; X32-SSE-NEXT:    movq {{.*#+}} xmm1 = xmm1[0],zero
+; X32-SSE-NEXT:    psllq %xmm1, %xmm0
+; X32-SSE-NEXT:    retl
+  %splat = shufflevector <2 x i64> %b, <2 x i64> undef, <2 x i32> zeroinitializer
+  %shift = shl <2 x i64> %a, %splat
+  ret <2 x i64> %shift
+}
+
+define <4 x i32> @splatvar_shift_v4i32(<4 x i32> %a, <4 x i32> %b) nounwind {
+; SSE2-LABEL: splatvar_shift_v4i32:
+; SSE2:       # BB#0:
+; SSE2-NEXT:    xorps %xmm2, %xmm2
+; SSE2-NEXT:    movss {{.*#+}} xmm2 = xmm1[0],xmm2[1,2,3]
+; SSE2-NEXT:    pslld %xmm2, %xmm0
+; SSE2-NEXT:    retq
+;
+; SSE41-LABEL: splatvar_shift_v4i32:
+; SSE41:       # BB#0:
+; SSE41-NEXT:    pxor %xmm2, %xmm2
+; SSE41-NEXT:    pblendw {{.*#+}} xmm2 = xmm1[0,1],xmm2[2,3,4,5,6,7]
+; SSE41-NEXT:    pslld %xmm2, %xmm0
+; SSE41-NEXT:    retq
+;
+; AVX-LABEL: splatvar_shift_v4i32:
+; AVX:       # BB#0:
+; AVX-NEXT:    vpxor %xmm2, %xmm2, %xmm2
+; AVX-NEXT:    vpblendw {{.*#+}} xmm1 = xmm1[0,1],xmm2[2,3,4,5,6,7]
+; AVX-NEXT:    vpslld %xmm1, %xmm0, %xmm0
+; AVX-NEXT:    retq
+;
+; X32-SSE-LABEL: splatvar_shift_v4i32:
+; X32-SSE:       # BB#0:
+; X32-SSE-NEXT:    xorps %xmm2, %xmm2
+; X32-SSE-NEXT:    movss {{.*#+}} xmm2 = xmm1[0],xmm2[1,2,3]
+; X32-SSE-NEXT:    pslld %xmm2, %xmm0
+; X32-SSE-NEXT:    retl
+  %splat = shufflevector <4 x i32> %b, <4 x i32> undef, <4 x i32> zeroinitializer
+  %shift = shl <4 x i32> %a, %splat
+  ret <4 x i32> %shift
+}
+
+define <8 x i16> @splatvar_shift_v8i16(<8 x i16> %a, <8 x i16> %b) nounwind {
+; SSE2-LABEL: splatvar_shift_v8i16:
+; SSE2:       # BB#0:
+; SSE2-NEXT:    movd %xmm1, %eax
+; SSE2-NEXT:    movzwl %ax, %eax
+; SSE2-NEXT:    movd %eax, %xmm1
+; SSE2-NEXT:    psllw %xmm1, %xmm0
+; SSE2-NEXT:    retq
+;
+; SSE41-LABEL: splatvar_shift_v8i16:
+; SSE41:       # BB#0:
+; SSE41-NEXT:    pxor %xmm2, %xmm2
+; SSE41-NEXT:    pblendw {{.*#+}} xmm2 = xmm1[0],xmm2[1,2,3,4,5,6,7]
+; SSE41-NEXT:    psllw %xmm2, %xmm0
+; SSE41-NEXT:    retq
+;
+; AVX-LABEL: splatvar_shift_v8i16:
+; AVX:       # BB#0:
+; AVX-NEXT:    vpxor %xmm2, %xmm2, %xmm2
+; AVX-NEXT:    vpblendw {{.*#+}} xmm1 = xmm1[0],xmm2[1,2,3,4,5,6,7]
+; AVX-NEXT:    vpsllw %xmm1, %xmm0, %xmm0
+; AVX-NEXT:    retq
+;
+; X32-SSE-LABEL: splatvar_shift_v8i16:
+; X32-SSE:       # BB#0:
+; X32-SSE-NEXT:    movd %xmm1, %eax
+; X32-SSE-NEXT:    movzwl %ax, %eax
+; X32-SSE-NEXT:    movd %eax, %xmm1
+; X32-SSE-NEXT:    psllw %xmm1, %xmm0
+; X32-SSE-NEXT:    retl
+  %splat = shufflevector <8 x i16> %b, <8 x i16> undef, <8 x i32> zeroinitializer
+  %shift = shl <8 x i16> %a, %splat
+  ret <8 x i16> %shift
+}
+
+define <16 x i8> @splatvar_shift_v16i8(<16 x i8> %a, <16 x i8> %b) nounwind {
+; SSE2-LABEL: splatvar_shift_v16i8:
+; SSE2:       # BB#0:
+; SSE2-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,1,0,3]
+; SSE2-NEXT:    pshuflw {{.*#+}} xmm1 = xmm1[0,0,0,0,4,5,6,7]
+; SSE2-NEXT:    pshufhw {{.*#+}} xmm2 = xmm1[0,1,2,3,4,4,4,4]
+; SSE2-NEXT:    psllw $5, %xmm2
+; SSE2-NEXT:    pxor %xmm1, %xmm1
+; SSE2-NEXT:    pxor %xmm3, %xmm3
+; SSE2-NEXT:    pcmpgtb %xmm2, %xmm3
+; SSE2-NEXT:    movdqa %xmm3, %xmm4
+; SSE2-NEXT:    pandn %xmm0, %xmm4
+; SSE2-NEXT:    psllw $4, %xmm0
+; SSE2-NEXT:    pand {{.*}}(%rip), %xmm0
+; SSE2-NEXT:    pand %xmm3, %xmm0
+; SSE2-NEXT:    por %xmm4, %xmm0
+; SSE2-NEXT:    paddb %xmm2, %xmm2
+; SSE2-NEXT:    pxor %xmm3, %xmm3
+; SSE2-NEXT:    pcmpgtb %xmm2, %xmm3
+; SSE2-NEXT:    movdqa %xmm3, %xmm4
+; SSE2-NEXT:    pandn %xmm0, %xmm4
+; SSE2-NEXT:    psllw $2, %xmm0
+; SSE2-NEXT:    pand {{.*}}(%rip), %xmm0
+; SSE2-NEXT:    pand %xmm3, %xmm0
+; SSE2-NEXT:    por %xmm4, %xmm0
+; SSE2-NEXT:    paddb %xmm2, %xmm2
+; SSE2-NEXT:    pcmpgtb %xmm2, %xmm1
+; SSE2-NEXT:    movdqa %xmm1, %xmm2
+; SSE2-NEXT:    pandn %xmm0, %xmm2
+; SSE2-NEXT:    paddb %xmm0, %xmm0
+; SSE2-NEXT:    pand %xmm1, %xmm0
+; SSE2-NEXT:    por %xmm2, %xmm0
+; SSE2-NEXT:    retq
+;
+; SSE41-LABEL: splatvar_shift_v16i8:
+; SSE41:       # BB#0:
+; SSE41-NEXT:    movdqa %xmm0, %xmm2
+; SSE41-NEXT:    pxor %xmm0, %xmm0
+; SSE41-NEXT:    pshufb %xmm0, %xmm1
+; SSE41-NEXT:    psllw $5, %xmm1
+; SSE41-NEXT:    movdqa %xmm1, %xmm3
+; SSE41-NEXT:    paddb %xmm3, %xmm3
+; SSE41-NEXT:    movdqa %xmm2, %xmm4
+; SSE41-NEXT:    psllw $4, %xmm4
+; SSE41-NEXT:    pand {{.*}}(%rip), %xmm4
+; SSE41-NEXT:    movdqa %xmm1, %xmm0
+; SSE41-NEXT:    pblendvb %xmm4, %xmm2
+; SSE41-NEXT:    movdqa %xmm2, %xmm1
+; SSE41-NEXT:    psllw $2, %xmm1
+; SSE41-NEXT:    pand {{.*}}(%rip), %xmm1
+; SSE41-NEXT:    movdqa %xmm3, %xmm0
+; SSE41-NEXT:    pblendvb %xmm1, %xmm2
+; SSE41-NEXT:    movdqa %xmm2, %xmm1
+; SSE41-NEXT:    paddb %xmm1, %xmm1
+; SSE41-NEXT:    paddb %xmm3, %xmm3
+; SSE41-NEXT:    movdqa %xmm3, %xmm0
+; SSE41-NEXT:    pblendvb %xmm1, %xmm2
+; SSE41-NEXT:    movdqa %xmm2, %xmm0
+; SSE41-NEXT:    retq
+;
+; AVX1-LABEL: splatvar_shift_v16i8:
+; AVX1:       # BB#0:
+; AVX1-NEXT:    vpxor %xmm2, %xmm2, %xmm2
+; AVX1-NEXT:    vpshufb %xmm2, %xmm1, %xmm1
+; AVX1-NEXT:    vpsllw $5, %xmm1, %xmm1
+; AVX1-NEXT:    vpaddb %xmm1, %xmm1, %xmm2
+; AVX1-NEXT:    vpsllw $4, %xmm0, %xmm3
+; AVX1-NEXT:    vpand {{.*}}(%rip), %xmm3, %xmm3
+; AVX1-NEXT:    vpblendvb %xmm1, %xmm3, %xmm0, %xmm0
+; AVX1-NEXT:    vpsllw $2, %xmm0, %xmm1
+; AVX1-NEXT:    vpand {{.*}}(%rip), %xmm1, %xmm1
+; AVX1-NEXT:    vpblendvb %xmm2, %xmm1, %xmm0, %xmm0
+; AVX1-NEXT:    vpaddb %xmm0, %xmm0, %xmm1
+; AVX1-NEXT:    vpaddb %xmm2, %xmm2, %xmm2
+; AVX1-NEXT:    vpblendvb %xmm2, %xmm1, %xmm0, %xmm0
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: splatvar_shift_v16i8:
+; AVX2:       # BB#0:
+; AVX2-NEXT:    vpbroadcastb %xmm1, %xmm1
+; AVX2-NEXT:    vpsllw $5, %xmm1, %xmm1
+; AVX2-NEXT:    vpsllw $4, %xmm0, %xmm2
+; AVX2-NEXT:    vpand {{.*}}(%rip), %xmm2, %xmm2
+; AVX2-NEXT:    vpblendvb %xmm1, %xmm2, %xmm0, %xmm0
+; AVX2-NEXT:    vpsllw $2, %xmm0, %xmm2
+; AVX2-NEXT:    vpand {{.*}}(%rip), %xmm2, %xmm2
+; AVX2-NEXT:    vpaddb %xmm1, %xmm1, %xmm1
+; AVX2-NEXT:    vpblendvb %xmm1, %xmm2, %xmm0, %xmm0
+; AVX2-NEXT:    vpaddb %xmm0, %xmm0, %xmm2
+; AVX2-NEXT:    vpaddb %xmm1, %xmm1, %xmm1
+; AVX2-NEXT:    vpblendvb %xmm1, %xmm2, %xmm0, %xmm0
+; AVX2-NEXT:    retq
+;
+; X32-SSE-LABEL: splatvar_shift_v16i8:
+; X32-SSE:       # BB#0:
+; X32-SSE-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
+; X32-SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,1,0,3]
+; X32-SSE-NEXT:    pshuflw {{.*#+}} xmm1 = xmm1[0,0,0,0,4,5,6,7]
+; X32-SSE-NEXT:    pshufhw {{.*#+}} xmm2 = xmm1[0,1,2,3,4,4,4,4]
+; X32-SSE-NEXT:    psllw $5, %xmm2
+; X32-SSE-NEXT:    pxor %xmm1, %xmm1
+; X32-SSE-NEXT:    pxor %xmm3, %xmm3
+; X32-SSE-NEXT:    pcmpgtb %xmm2, %xmm3
+; X32-SSE-NEXT:    movdqa %xmm3, %xmm4
+; X32-SSE-NEXT:    pandn %xmm0, %xmm4
+; X32-SSE-NEXT:    psllw $4, %xmm0
+; X32-SSE-NEXT:    pand .LCPI7_0, %xmm0
+; X32-SSE-NEXT:    pand %xmm3, %xmm0
+; X32-SSE-NEXT:    por %xmm4, %xmm0
+; X32-SSE-NEXT:    paddb %xmm2, %xmm2
+; X32-SSE-NEXT:    pxor %xmm3, %xmm3
+; X32-SSE-NEXT:    pcmpgtb %xmm2, %xmm3
+; X32-SSE-NEXT:    movdqa %xmm3, %xmm4
+; X32-SSE-NEXT:    pandn %xmm0, %xmm4
+; X32-SSE-NEXT:    psllw $2, %xmm0
+; X32-SSE-NEXT:    pand .LCPI7_1, %xmm0
+; X32-SSE-NEXT:    pand %xmm3, %xmm0
+; X32-SSE-NEXT:    por %xmm4, %xmm0
+; X32-SSE-NEXT:    paddb %xmm2, %xmm2
+; X32-SSE-NEXT:    pcmpgtb %xmm2, %xmm1
+; X32-SSE-NEXT:    movdqa %xmm1, %xmm2
+; X32-SSE-NEXT:    pandn %xmm0, %xmm2
+; X32-SSE-NEXT:    paddb %xmm0, %xmm0
+; X32-SSE-NEXT:    pand %xmm1, %xmm0
+; X32-SSE-NEXT:    por %xmm2, %xmm0
+; X32-SSE-NEXT:    retl
+  %splat = shufflevector <16 x i8> %b, <16 x i8> undef, <16 x i32> zeroinitializer
+  %shift = shl <16 x i8> %a, %splat
+  ret <16 x i8> %shift
+}
+
+;
+; Constant Shifts
+;
+
+define <2 x i64> @constant_shift_v2i64(<2 x i64> %a) nounwind {
+; SSE2-LABEL: constant_shift_v2i64:
+; SSE2:       # BB#0:
+; SSE2-NEXT:    movdqa %xmm0, %xmm1
+; SSE2-NEXT:    psllq $7, %xmm1
+; SSE2-NEXT:    psllq $1, %xmm0
+; SSE2-NEXT:    movsd {{.*#+}} xmm1 = xmm0[0],xmm1[1]
+; SSE2-NEXT:    movapd %xmm1, %xmm0
+; SSE2-NEXT:    retq
+;
+; SSE41-LABEL: constant_shift_v2i64:
+; SSE41:       # BB#0:
+; SSE41-NEXT:    movdqa %xmm0, %xmm1
+; SSE41-NEXT:    psllq $7, %xmm1
+; SSE41-NEXT:    psllq $1, %xmm0
+; SSE41-NEXT:    pblendw {{.*#+}} xmm0 = xmm0[0,1,2,3],xmm1[4,5,6,7]
+; SSE41-NEXT:    retq
+;
+; AVX1-LABEL: constant_shift_v2i64:
+; AVX1:       # BB#0:
+; AVX1-NEXT:    vpsllq $7, %xmm0, %xmm1
+; AVX1-NEXT:    vpsllq $1, %xmm0, %xmm0
+; AVX1-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0,1,2,3],xmm1[4,5,6,7]
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: constant_shift_v2i64:
+; AVX2:       # BB#0:
+; AVX2-NEXT:    vpsllvq {{.*}}(%rip), %xmm0, %xmm0
+; AVX2-NEXT:    retq
+;
+; X32-SSE-LABEL: constant_shift_v2i64:
+; X32-SSE:       # BB#0:
+; X32-SSE-NEXT:    movdqa %xmm0, %xmm1
+; X32-SSE-NEXT:    psllq $7, %xmm1
+; X32-SSE-NEXT:    psllq $1, %xmm0
+; X32-SSE-NEXT:    movsd {{.*#+}} xmm1 = xmm0[0],xmm1[1]
+; X32-SSE-NEXT:    movapd %xmm1, %xmm0
+; X32-SSE-NEXT:    retl
+  %shift = shl <2 x i64> %a, <i64 1, i64 7>
+  ret <2 x i64> %shift
+}
+
+define <4 x i32> @constant_shift_v4i32(<4 x i32> %a) nounwind {
+; SSE2-LABEL: constant_shift_v4i32:
+; SSE2:       # BB#0:
+; SSE2-NEXT:    movdqa {{.*#+}} xmm1 = [16,32,64,128]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[1,1,3,3]
+; SSE2-NEXT:    pmuludq %xmm1, %xmm0
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
+; SSE2-NEXT:    pmuludq %xmm2, %xmm1
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3]
+; SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
+; SSE2-NEXT:    retq
+;
+; SSE41-LABEL: constant_shift_v4i32:
+; SSE41:       # BB#0:
+; SSE41-NEXT:    pmulld {{.*}}(%rip), %xmm0
+; SSE41-NEXT:    retq
+;
+; AVX1-LABEL: constant_shift_v4i32:
+; AVX1:       # BB#0:
+; AVX1-NEXT:    vpmulld {{.*}}(%rip), %xmm0, %xmm0
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: constant_shift_v4i32:
+; AVX2:       # BB#0:
+; AVX2-NEXT:    vpsllvd {{.*}}(%rip), %xmm0, %xmm0
+; AVX2-NEXT:    retq
+;
+; X32-SSE-LABEL: constant_shift_v4i32:
+; X32-SSE:       # BB#0:
+; X32-SSE-NEXT:    movdqa {{.*#+}} xmm1 = [16,32,64,128]
+; X32-SSE-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[1,1,3,3]
+; X32-SSE-NEXT:    pmuludq %xmm1, %xmm0
+; X32-SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
+; X32-SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
+; X32-SSE-NEXT:    pmuludq %xmm2, %xmm1
+; X32-SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3]
+; X32-SSE-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
+; X32-SSE-NEXT:    retl
+  %shift = shl <4 x i32> %a, <i32 4, i32 5, i32 6, i32 7>
+  ret <4 x i32> %shift
+}
+
+define <8 x i16> @constant_shift_v8i16(<8 x i16> %a) nounwind {
+; SSE-LABEL: constant_shift_v8i16:
+; SSE:       # BB#0:
+; SSE-NEXT:    pmullw {{.*}}(%rip), %xmm0
+; SSE-NEXT:    retq
+;
+; AVX-LABEL: constant_shift_v8i16:
+; AVX:       # BB#0:
+; AVX-NEXT:    vpmullw {{.*}}(%rip), %xmm0, %xmm0
+; AVX-NEXT:    retq
+;
+; X32-SSE-LABEL: constant_shift_v8i16:
+; X32-SSE:       # BB#0:
+; X32-SSE-NEXT:    pmullw .LCPI10_0, %xmm0
+; X32-SSE-NEXT:    retl
+  %shift = shl <8 x i16> %a, <i16 0, i16 1, i16 2, i16 3, i16 4, i16 5, i16 6, i16 7>
+  ret <8 x i16> %shift
+}
+
+define <16 x i8> @constant_shift_v16i8(<16 x i8> %a) nounwind {
+; SSE2-LABEL: constant_shift_v16i8:
+; SSE2:       # BB#0:
+; SSE2-NEXT:    movdqa {{.*#+}} xmm2 = [0,1,2,3,4,5,6,7,7,6,5,4,3,2,1,0]
+; SSE2-NEXT:    psllw $5, %xmm2
+; SSE2-NEXT:    pxor %xmm1, %xmm1
+; SSE2-NEXT:    pxor %xmm3, %xmm3
+; SSE2-NEXT:    pcmpgtb %xmm2, %xmm3
+; SSE2-NEXT:    movdqa %xmm3, %xmm4
+; SSE2-NEXT:    pandn %xmm0, %xmm4
+; SSE2-NEXT:    psllw $4, %xmm0
+; SSE2-NEXT:    pand {{.*}}(%rip), %xmm0
+; SSE2-NEXT:    pand %xmm3, %xmm0
+; SSE2-NEXT:    por %xmm4, %xmm0
+; SSE2-NEXT:    paddb %xmm2, %xmm2
+; SSE2-NEXT:    pxor %xmm3, %xmm3
+; SSE2-NEXT:    pcmpgtb %xmm2, %xmm3
+; SSE2-NEXT:    movdqa %xmm3, %xmm4
+; SSE2-NEXT:    pandn %xmm0, %xmm4
+; SSE2-NEXT:    psllw $2, %xmm0
+; SSE2-NEXT:    pand {{.*}}(%rip), %xmm0
+; SSE2-NEXT:    pand %xmm3, %xmm0
+; SSE2-NEXT:    por %xmm4, %xmm0
+; SSE2-NEXT:    paddb %xmm2, %xmm2
+; SSE2-NEXT:    pcmpgtb %xmm2, %xmm1
+; SSE2-NEXT:    movdqa %xmm1, %xmm2
+; SSE2-NEXT:    pandn %xmm0, %xmm2
+; SSE2-NEXT:    paddb %xmm0, %xmm0
+; SSE2-NEXT:    pand %xmm1, %xmm0
+; SSE2-NEXT:    por %xmm2, %xmm0
+; SSE2-NEXT:    retq
+;
+; SSE41-LABEL: constant_shift_v16i8:
+; SSE41:       # BB#0:
+; SSE41-NEXT:    movdqa %xmm0, %xmm1
+; SSE41-NEXT:    movdqa {{.*#+}} xmm0 = [0,1,2,3,4,5,6,7,7,6,5,4,3,2,1,0]
+; SSE41-NEXT:    psllw $5, %xmm0
+; SSE41-NEXT:    movdqa %xmm1, %xmm2
+; SSE41-NEXT:    psllw $4, %xmm2
+; SSE41-NEXT:    pand {{.*}}(%rip), %xmm2
+; SSE41-NEXT:    pblendvb %xmm2, %xmm1
+; SSE41-NEXT:    movdqa %xmm1, %xmm2
+; SSE41-NEXT:    psllw $2, %xmm2
+; SSE41-NEXT:    pand {{.*}}(%rip), %xmm2
+; SSE41-NEXT:    paddb %xmm0, %xmm0
+; SSE41-NEXT:    pblendvb %xmm2, %xmm1
+; SSE41-NEXT:    movdqa %xmm1, %xmm2
+; SSE41-NEXT:    paddb %xmm2, %xmm2
+; SSE41-NEXT:    paddb %xmm0, %xmm0
+; SSE41-NEXT:    pblendvb %xmm2, %xmm1
+; SSE41-NEXT:    movdqa %xmm1, %xmm0
+; SSE41-NEXT:    retq
+;
+; AVX-LABEL: constant_shift_v16i8:
+; AVX:       # BB#0:
+; AVX-NEXT:    vmovdqa {{.*#+}} xmm1 = [0,1,2,3,4,5,6,7,7,6,5,4,3,2,1,0]
+; AVX-NEXT:    vpsllw $5, %xmm1, %xmm1
+; AVX-NEXT:    vpsllw $4, %xmm0, %xmm2
+; AVX-NEXT:    vpand {{.*}}(%rip), %xmm2, %xmm2
+; AVX-NEXT:    vpblendvb %xmm1, %xmm2, %xmm0, %xmm0
+; AVX-NEXT:    vpsllw $2, %xmm0, %xmm2
+; AVX-NEXT:    vpand {{.*}}(%rip), %xmm2, %xmm2
+; AVX-NEXT:    vpaddb %xmm1, %xmm1, %xmm1
+; AVX-NEXT:    vpblendvb %xmm1, %xmm2, %xmm0, %xmm0
+; AVX-NEXT:    vpaddb %xmm0, %xmm0, %xmm2
+; AVX-NEXT:    vpaddb %xmm1, %xmm1, %xmm1
+; AVX-NEXT:    vpblendvb %xmm1, %xmm2, %xmm0, %xmm0
+; AVX-NEXT:    retq
+;
+; X32-SSE-LABEL: constant_shift_v16i8:
+; X32-SSE:       # BB#0:
+; X32-SSE-NEXT:    movdqa {{.*#+}} xmm2 = [0,1,2,3,4,5,6,7,7,6,5,4,3,2,1,0]
+; X32-SSE-NEXT:    psllw $5, %xmm2
+; X32-SSE-NEXT:    pxor %xmm1, %xmm1
+; X32-SSE-NEXT:    pxor %xmm3, %xmm3
+; X32-SSE-NEXT:    pcmpgtb %xmm2, %xmm3
+; X32-SSE-NEXT:    movdqa %xmm3, %xmm4
+; X32-SSE-NEXT:    pandn %xmm0, %xmm4
+; X32-SSE-NEXT:    psllw $4, %xmm0
+; X32-SSE-NEXT:    pand .LCPI11_1, %xmm0
+; X32-SSE-NEXT:    pand %xmm3, %xmm0
+; X32-SSE-NEXT:    por %xmm4, %xmm0
+; X32-SSE-NEXT:    paddb %xmm2, %xmm2
+; X32-SSE-NEXT:    pxor %xmm3, %xmm3
+; X32-SSE-NEXT:    pcmpgtb %xmm2, %xmm3
+; X32-SSE-NEXT:    movdqa %xmm3, %xmm4
+; X32-SSE-NEXT:    pandn %xmm0, %xmm4
+; X32-SSE-NEXT:    psllw $2, %xmm0
+; X32-SSE-NEXT:    pand .LCPI11_2, %xmm0
+; X32-SSE-NEXT:    pand %xmm3, %xmm0
+; X32-SSE-NEXT:    por %xmm4, %xmm0
+; X32-SSE-NEXT:    paddb %xmm2, %xmm2
+; X32-SSE-NEXT:    pcmpgtb %xmm2, %xmm1
+; X32-SSE-NEXT:    movdqa %xmm1, %xmm2
+; X32-SSE-NEXT:    pandn %xmm0, %xmm2
+; X32-SSE-NEXT:    paddb %xmm0, %xmm0
+; X32-SSE-NEXT:    pand %xmm1, %xmm0
+; X32-SSE-NEXT:    por %xmm2, %xmm0
+; X32-SSE-NEXT:    retl
+  %shift = shl <16 x i8> %a, <i8 0, i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 7, i8 6, i8 5, i8 4, i8 3, i8 2, i8 1, i8 0>
+  ret <16 x i8> %shift
+}
+
+;
+; Uniform Constant Shifts
+;
+
+define <2 x i64> @splatconstant_shift_v2i64(<2 x i64> %a) nounwind {
+; SSE-LABEL: splatconstant_shift_v2i64:
+; SSE:       # BB#0:
+; SSE-NEXT:    psllq $7, %xmm0
+; SSE-NEXT:    retq
+;
+; AVX-LABEL: splatconstant_shift_v2i64:
+; AVX:       # BB#0:
+; AVX-NEXT:    vpsllq $7, %xmm0, %xmm0
+; AVX-NEXT:    retq
+;
+; X32-SSE-LABEL: splatconstant_shift_v2i64:
+; X32-SSE:       # BB#0:
+; X32-SSE-NEXT:    psllq $7, %xmm0
+; X32-SSE-NEXT:    retl
+  %shift = shl <2 x i64> %a, <i64 7, i64 7>
+  ret <2 x i64> %shift
+}
+
+define <4 x i32> @splatconstant_shift_v4i32(<4 x i32> %a) nounwind {
+; SSE-LABEL: splatconstant_shift_v4i32:
+; SSE:       # BB#0:
+; SSE-NEXT:    pslld $5, %xmm0
+; SSE-NEXT:    retq
+;
+; AVX-LABEL: splatconstant_shift_v4i32:
+; AVX:       # BB#0:
+; AVX-NEXT:    vpslld $5, %xmm0, %xmm0
+; AVX-NEXT:    retq
+;
+; X32-SSE-LABEL: splatconstant_shift_v4i32:
+; X32-SSE:       # BB#0:
+; X32-SSE-NEXT:    pslld $5, %xmm0
+; X32-SSE-NEXT:    retl
+  %shift = shl <4 x i32> %a, <i32 5, i32 5, i32 5, i32 5>
+  ret <4 x i32> %shift
+}
+
+define <8 x i16> @splatconstant_shift_v8i16(<8 x i16> %a) nounwind {
+; SSE-LABEL: splatconstant_shift_v8i16:
+; SSE:       # BB#0:
+; SSE-NEXT:    psllw $3, %xmm0
+; SSE-NEXT:    retq
+;
+; AVX-LABEL: splatconstant_shift_v8i16:
+; AVX:       # BB#0:
+; AVX-NEXT:    vpsllw $3, %xmm0, %xmm0
+; AVX-NEXT:    retq
+;
+; X32-SSE-LABEL: splatconstant_shift_v8i16:
+; X32-SSE:       # BB#0:
+; X32-SSE-NEXT:    psllw $3, %xmm0
+; X32-SSE-NEXT:    retl
+  %shift = shl <8 x i16> %a, <i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3>
+  ret <8 x i16> %shift
+}
+
+define <16 x i8> @splatconstant_shift_v16i8(<16 x i8> %a) nounwind {
+; SSE-LABEL: splatconstant_shift_v16i8:
+; SSE:       # BB#0:
+; SSE-NEXT:    psllw $3, %xmm0
+; SSE-NEXT:    pand {{.*}}(%rip), %xmm0
+; SSE-NEXT:    retq
+;
+; AVX-LABEL: splatconstant_shift_v16i8:
+; AVX:       # BB#0:
+; AVX-NEXT:    vpsllw $3, %xmm0, %xmm0
+; AVX-NEXT:    vpand {{.*}}(%rip), %xmm0, %xmm0
+; AVX-NEXT:    retq
+;
+; X32-SSE-LABEL: splatconstant_shift_v16i8:
+; X32-SSE:       # BB#0:
+; X32-SSE-NEXT:    psllw $3, %xmm0
+; X32-SSE-NEXT:    pand .LCPI15_0, %xmm0
+; X32-SSE-NEXT:    retl
+  %shift = shl <16 x i8> %a, <i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3>
+  ret <16 x i8> %shift
+}
diff --git a/test/CodeGen/X86/vector-shift-shl-256.ll b/test/CodeGen/X86/vector-shift-shl-256.ll
new file mode 100644
index 00000000000..6e28494123c
--- /dev/null
+++ b/test/CodeGen/X86/vector-shift-shl-256.ll
@@ -0,0 +1,459 @@
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mcpu=x86-64 -mattr=+avx | FileCheck %s --check-prefix=ALL --check-prefix=AVX --check-prefix=AVX1
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mcpu=x86-64 -mattr=+avx2 | FileCheck %s --check-prefix=ALL --check-prefix=AVX --check-prefix=AVX2
+
+;
+; Variable Shifts
+;
+
+define <4 x i64> @var_shift_v4i64(<4 x i64> %a, <4 x i64> %b) nounwind {
+; AVX1-LABEL: var_shift_v4i64:
+; AVX1:       # BB#0:
+; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm2
+; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm3
+; AVX1-NEXT:    vpsllq %xmm2, %xmm3, %xmm4
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm2 = xmm2[2,3,0,1]
+; AVX1-NEXT:    vpsllq %xmm2, %xmm3, %xmm2
+; AVX1-NEXT:    vpblendw {{.*#+}} xmm2 = xmm4[0,1,2,3],xmm2[4,5,6,7]
+; AVX1-NEXT:    vpsllq %xmm1, %xmm0, %xmm3
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[2,3,0,1]
+; AVX1-NEXT:    vpsllq %xmm1, %xmm0, %xmm0
+; AVX1-NEXT:    vpblendw {{.*#+}} xmm0 = xmm3[0,1,2,3],xmm0[4,5,6,7]
+; AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm0
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: var_shift_v4i64:
+; AVX2:       # BB#0:
+; AVX2-NEXT:    vpsllvq %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    retq
+  %shift = shl <4 x i64> %a, %b
+  ret <4 x i64> %shift
+}
+
+define <8 x i32> @var_shift_v8i32(<8 x i32> %a, <8 x i32> %b) nounwind {
+; AVX1-LABEL: var_shift_v8i32:
+; AVX1:       # BB#0:
+; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm2
+; AVX1-NEXT:    vpslld $23, %xmm2, %xmm2
+; AVX1-NEXT:    vmovdqa {{.*#+}} xmm3 = [1065353216,1065353216,1065353216,1065353216]
+; AVX1-NEXT:    vpaddd %xmm3, %xmm2, %xmm2
+; AVX1-NEXT:    vcvttps2dq %xmm2, %xmm2
+; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm4
+; AVX1-NEXT:    vpmulld %xmm4, %xmm2, %xmm2
+; AVX1-NEXT:    vpslld $23, %xmm1, %xmm1
+; AVX1-NEXT:    vpaddd %xmm3, %xmm1, %xmm1
+; AVX1-NEXT:    vcvttps2dq %xmm1, %xmm1
+; AVX1-NEXT:    vpmulld %xmm0, %xmm1, %xmm0
+; AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm0
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: var_shift_v8i32:
+; AVX2:       # BB#0:
+; AVX2-NEXT:    vpsllvd %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    retq
+  %shift = shl <8 x i32> %a, %b
+  ret <8 x i32> %shift
+}
+
+define <16 x i16> @var_shift_v16i16(<16 x i16> %a, <16 x i16> %b) nounwind {
+; AVX1-LABEL: var_shift_v16i16:
+; AVX1:       # BB#0:
+; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm2
+; AVX1-NEXT:    vpsllw $12, %xmm2, %xmm3
+; AVX1-NEXT:    vpsllw $4, %xmm2, %xmm2
+; AVX1-NEXT:    vpor %xmm3, %xmm2, %xmm2
+; AVX1-NEXT:    vpaddw %xmm2, %xmm2, %xmm3
+; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm4
+; AVX1-NEXT:    vpsllw $8, %xmm4, %xmm5
+; AVX1-NEXT:    vpblendvb %xmm2, %xmm5, %xmm4, %xmm2
+; AVX1-NEXT:    vpsllw $4, %xmm2, %xmm4
+; AVX1-NEXT:    vpblendvb %xmm3, %xmm4, %xmm2, %xmm2
+; AVX1-NEXT:    vpsllw $2, %xmm2, %xmm4
+; AVX1-NEXT:    vpaddw %xmm3, %xmm3, %xmm3
+; AVX1-NEXT:    vpblendvb %xmm3, %xmm4, %xmm2, %xmm2
+; AVX1-NEXT:    vpsllw $1, %xmm2, %xmm4
+; AVX1-NEXT:    vpaddw %xmm3, %xmm3, %xmm3
+; AVX1-NEXT:    vpblendvb %xmm3, %xmm4, %xmm2, %xmm2
+; AVX1-NEXT:    vpsllw $12, %xmm1, %xmm3
+; AVX1-NEXT:    vpsllw $4, %xmm1, %xmm1
+; AVX1-NEXT:    vpor %xmm3, %xmm1, %xmm1
+; AVX1-NEXT:    vpaddw %xmm1, %xmm1, %xmm3
+; AVX1-NEXT:    vpsllw $8, %xmm0, %xmm4
+; AVX1-NEXT:    vpblendvb %xmm1, %xmm4, %xmm0, %xmm0
+; AVX1-NEXT:    vpsllw $4, %xmm0, %xmm1
+; AVX1-NEXT:    vpblendvb %xmm3, %xmm1, %xmm0, %xmm0
+; AVX1-NEXT:    vpsllw $2, %xmm0, %xmm1
+; AVX1-NEXT:    vpaddw %xmm3, %xmm3, %xmm3
+; AVX1-NEXT:    vpblendvb %xmm3, %xmm1, %xmm0, %xmm0
+; AVX1-NEXT:    vpsllw $1, %xmm0, %xmm1
+; AVX1-NEXT:    vpaddw %xmm3, %xmm3, %xmm3
+; AVX1-NEXT:    vpblendvb %xmm3, %xmm1, %xmm0, %xmm0
+; AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm0
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: var_shift_v16i16:
+; AVX2:       # BB#0:
+; AVX2-NEXT:    vpxor %ymm2, %ymm2, %ymm2
+; AVX2-NEXT:    vpunpckhwd {{.*#+}} ymm3 = ymm1[4],ymm2[4],ymm1[5],ymm2[5],ymm1[6],ymm2[6],ymm1[7],ymm2[7],ymm1[12],ymm2[12],ymm1[13],ymm2[13],ymm1[14],ymm2[14],ymm1[15],ymm2[15]
+; AVX2-NEXT:    vpunpckhwd {{.*#+}} ymm4 = ymm0[4,4,5,5,6,6,7,7,12,12,13,13,14,14,15,15]
+; AVX2-NEXT:    vpsllvd %ymm3, %ymm4, %ymm3
+; AVX2-NEXT:    vpsrld $16, %ymm3, %ymm3
+; AVX2-NEXT:    vpunpcklwd {{.*#+}} ymm1 = ymm1[0],ymm2[0],ymm1[1],ymm2[1],ymm1[2],ymm2[2],ymm1[3],ymm2[3],ymm1[8],ymm2[8],ymm1[9],ymm2[9],ymm1[10],ymm2[10],ymm1[11],ymm2[11]
+; AVX2-NEXT:    vpunpcklwd {{.*#+}} ymm0 = ymm0[0,0,1,1,2,2,3,3,8,8,9,9,10,10,11,11]
+; AVX2-NEXT:    vpsllvd %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vpsrld $16, %ymm0, %ymm0
+; AVX2-NEXT:    vpackusdw %ymm3, %ymm0, %ymm0
+; AVX2-NEXT:    retq
+  %shift = shl <16 x i16> %a, %b
+  ret <16 x i16> %shift
+}
+
+define <32 x i8> @var_shift_v32i8(<32 x i8> %a, <32 x i8> %b) nounwind {
+; AVX1-LABEL: var_shift_v32i8:
+; AVX1:       # BB#0:
+; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm2
+; AVX1-NEXT:    vpsllw $4, %xmm2, %xmm3
+; AVX1-NEXT:    vmovdqa {{.*#+}} xmm4 = [240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240]
+; AVX1-NEXT:    vpand %xmm4, %xmm3, %xmm3
+; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm5
+; AVX1-NEXT:    vpsllw $5, %xmm5, %xmm5
+; AVX1-NEXT:    vpblendvb %xmm5, %xmm3, %xmm2, %xmm2
+; AVX1-NEXT:    vpsllw $2, %xmm2, %xmm3
+; AVX1-NEXT:    vmovdqa {{.*#+}} xmm6 = [252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252]
+; AVX1-NEXT:    vpand %xmm6, %xmm3, %xmm3
+; AVX1-NEXT:    vpaddb %xmm5, %xmm5, %xmm5
+; AVX1-NEXT:    vpblendvb %xmm5, %xmm3, %xmm2, %xmm2
+; AVX1-NEXT:    vpaddb %xmm2, %xmm2, %xmm3
+; AVX1-NEXT:    vpaddb %xmm5, %xmm5, %xmm5
+; AVX1-NEXT:    vpblendvb %xmm5, %xmm3, %xmm2, %xmm2
+; AVX1-NEXT:    vpsllw $4, %xmm0, %xmm3
+; AVX1-NEXT:    vpand %xmm4, %xmm3, %xmm3
+; AVX1-NEXT:    vpsllw $5, %xmm1, %xmm1
+; AVX1-NEXT:    vpblendvb %xmm1, %xmm3, %xmm0, %xmm0
+; AVX1-NEXT:    vpsllw $2, %xmm0, %xmm3
+; AVX1-NEXT:    vpand %xmm6, %xmm3, %xmm3
+; AVX1-NEXT:    vpaddb %xmm1, %xmm1, %xmm1
+; AVX1-NEXT:    vpblendvb %xmm1, %xmm3, %xmm0, %xmm0
+; AVX1-NEXT:    vpaddb %xmm0, %xmm0, %xmm3
+; AVX1-NEXT:    vpaddb %xmm1, %xmm1, %xmm1
+; AVX1-NEXT:    vpblendvb %xmm1, %xmm3, %xmm0, %xmm0
+; AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm0
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: var_shift_v32i8:
+; AVX2:       # BB#0:
+; AVX2-NEXT:    vpsllw $5, %ymm1, %ymm1
+; AVX2-NEXT:    vpsllw $4, %ymm0, %ymm2
+; AVX2-NEXT:    vpand {{.*}}(%rip), %ymm2, %ymm2
+; AVX2-NEXT:    vpblendvb %ymm1, %ymm2, %ymm0, %ymm0
+; AVX2-NEXT:    vpsllw $2, %ymm0, %ymm2
+; AVX2-NEXT:    vpand {{.*}}(%rip), %ymm2, %ymm2
+; AVX2-NEXT:    vpaddb %ymm1, %ymm1, %ymm1
+; AVX2-NEXT:    vpblendvb %ymm1, %ymm2, %ymm0, %ymm0
+; AVX2-NEXT:    vpaddb %ymm0, %ymm0, %ymm2
+; AVX2-NEXT:    vpaddb %ymm1, %ymm1, %ymm1
+; AVX2-NEXT:    vpblendvb %ymm1, %ymm2, %ymm0, %ymm0
+; AVX2-NEXT:    retq
+  %shift = shl <32 x i8> %a, %b
+  ret <32 x i8> %shift
+}
+
+;
+; Uniform Variable Shifts
+;
+
+define <4 x i64> @splatvar_shift_v4i64(<4 x i64> %a, <4 x i64> %b) nounwind {
+; AVX1-LABEL: splatvar_shift_v4i64:
+; AVX1:       # BB#0:
+; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm2
+; AVX1-NEXT:    vpsllq %xmm1, %xmm2, %xmm2
+; AVX1-NEXT:    vpsllq %xmm1, %xmm0, %xmm0
+; AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm0
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: splatvar_shift_v4i64:
+; AVX2:       # BB#0:
+; AVX2-NEXT:    vpsllq %xmm1, %ymm0, %ymm0
+; AVX2-NEXT:    retq
+  %splat = shufflevector <4 x i64> %b, <4 x i64> undef, <4 x i32> zeroinitializer
+  %shift = shl <4 x i64> %a, %splat
+  ret <4 x i64> %shift
+}
+
+define <8 x i32> @splatvar_shift_v8i32(<8 x i32> %a, <8 x i32> %b) nounwind {
+; AVX1-LABEL: splatvar_shift_v8i32:
+; AVX1:       # BB#0:
+; AVX1-NEXT:    vpxor %xmm2, %xmm2, %xmm2
+; AVX1-NEXT:    vpblendw {{.*#+}} xmm1 = xmm1[0,1],xmm2[2,3,4,5,6,7]
+; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm2
+; AVX1-NEXT:    vpslld %xmm1, %xmm2, %xmm2
+; AVX1-NEXT:    vpslld %xmm1, %xmm0, %xmm0
+; AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm0
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: splatvar_shift_v8i32:
+; AVX2:       # BB#0:
+; AVX2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
+; AVX2-NEXT:    vpblendw {{.*#+}} xmm1 = xmm1[0,1],xmm2[2,3,4,5,6,7]
+; AVX2-NEXT:    vpslld %xmm1, %ymm0, %ymm0
+; AVX2-NEXT:    retq
+  %splat = shufflevector <8 x i32> %b, <8 x i32> undef, <8 x i32> zeroinitializer
+  %shift = shl <8 x i32> %a, %splat
+  ret <8 x i32> %shift
+}
+
+define <16 x i16> @splatvar_shift_v16i16(<16 x i16> %a, <16 x i16> %b) nounwind {
+; AVX1-LABEL: splatvar_shift_v16i16:
+; AVX1:       # BB#0:
+; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm2
+; AVX1-NEXT:    vmovd %xmm1, %eax
+; AVX1-NEXT:    movzwl %ax, %eax
+; AVX1-NEXT:    vmovd %eax, %xmm1
+; AVX1-NEXT:    vpsllw %xmm1, %xmm2, %xmm2
+; AVX1-NEXT:    vpsllw %xmm1, %xmm0, %xmm0
+; AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm0
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: splatvar_shift_v16i16:
+; AVX2:       # BB#0:
+; AVX2-NEXT:    vmovd %xmm1, %eax
+; AVX2-NEXT:    movzwl %ax, %eax
+; AVX2-NEXT:    vmovd %eax, %xmm1
+; AVX2-NEXT:    vpsllw %xmm1, %ymm0, %ymm0
+; AVX2-NEXT:    retq
+  %splat = shufflevector <16 x i16> %b, <16 x i16> undef, <16 x i32> zeroinitializer
+  %shift = shl <16 x i16> %a, %splat
+  ret <16 x i16> %shift
+}
+
+define <32 x i8> @splatvar_shift_v32i8(<32 x i8> %a, <32 x i8> %b) nounwind {
+; AVX1-LABEL: splatvar_shift_v32i8:
+; AVX1:       # BB#0:
+; AVX1-NEXT:    vpxor %xmm2, %xmm2, %xmm2
+; AVX1-NEXT:    vpshufb %xmm2, %xmm1, %xmm1
+; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm2
+; AVX1-NEXT:    vpsllw $4, %xmm2, %xmm3
+; AVX1-NEXT:    vmovdqa {{.*#+}} xmm4 = [240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240]
+; AVX1-NEXT:    vpand %xmm4, %xmm3, %xmm3
+; AVX1-NEXT:    vpsllw $5, %xmm1, %xmm1
+; AVX1-NEXT:    vpblendvb %xmm1, %xmm3, %xmm2, %xmm2
+; AVX1-NEXT:    vpsllw $2, %xmm2, %xmm3
+; AVX1-NEXT:    vmovdqa {{.*#+}} xmm5 = [252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252]
+; AVX1-NEXT:    vpand %xmm5, %xmm3, %xmm3
+; AVX1-NEXT:    vpaddb %xmm1, %xmm1, %xmm6
+; AVX1-NEXT:    vpblendvb %xmm6, %xmm3, %xmm2, %xmm2
+; AVX1-NEXT:    vpaddb %xmm2, %xmm2, %xmm3
+; AVX1-NEXT:    vpaddb %xmm6, %xmm6, %xmm7
+; AVX1-NEXT:    vpblendvb %xmm7, %xmm3, %xmm2, %xmm2
+; AVX1-NEXT:    vpsllw $4, %xmm0, %xmm3
+; AVX1-NEXT:    vpand %xmm4, %xmm3, %xmm3
+; AVX1-NEXT:    vpblendvb %xmm1, %xmm3, %xmm0, %xmm0
+; AVX1-NEXT:    vpsllw $2, %xmm0, %xmm1
+; AVX1-NEXT:    vpand %xmm5, %xmm1, %xmm1
+; AVX1-NEXT:    vpblendvb %xmm6, %xmm1, %xmm0, %xmm0
+; AVX1-NEXT:    vpaddb %xmm0, %xmm0, %xmm1
+; AVX1-NEXT:    vpblendvb %xmm7, %xmm1, %xmm0, %xmm0
+; AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm0
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: splatvar_shift_v32i8:
+; AVX2:       # BB#0:
+; AVX2-NEXT:    vpbroadcastb %xmm1, %ymm1
+; AVX2-NEXT:    vpsllw $4, %ymm0, %ymm2
+; AVX2-NEXT:    vpand {{.*}}(%rip), %ymm2, %ymm2
+; AVX2-NEXT:    vpsllw $5, %ymm1, %ymm1
+; AVX2-NEXT:    vpblendvb %ymm1, %ymm2, %ymm0, %ymm0
+; AVX2-NEXT:    vpsllw $2, %ymm0, %ymm2
+; AVX2-NEXT:    vpand {{.*}}(%rip), %ymm2, %ymm2
+; AVX2-NEXT:    vpaddb %ymm1, %ymm1, %ymm1
+; AVX2-NEXT:    vpblendvb %ymm1, %ymm2, %ymm0, %ymm0
+; AVX2-NEXT:    vpaddb %ymm0, %ymm0, %ymm2
+; AVX2-NEXT:    vpaddb %ymm1, %ymm1, %ymm1
+; AVX2-NEXT:    vpblendvb %ymm1, %ymm2, %ymm0, %ymm0
+; AVX2-NEXT:    retq
+  %splat = shufflevector <32 x i8> %b, <32 x i8> undef, <32 x i32> zeroinitializer
+  %shift = shl <32 x i8> %a, %splat
+  ret <32 x i8> %shift
+}
+
+;
+; Constant Shifts
+;
+
+define <4 x i64> @constant_shift_v4i64(<4 x i64> %a) nounwind {
+; AVX1-LABEL: constant_shift_v4i64:
+; AVX1:       # BB#0:
+; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
+; AVX1-NEXT:    vpsllq $62, %xmm1, %xmm2
+; AVX1-NEXT:    vpsllq $31, %xmm1, %xmm1
+; AVX1-NEXT:    vpblendw {{.*#+}} xmm1 = xmm1[0,1,2,3],xmm2[4,5,6,7]
+; AVX1-NEXT:    vpsllq $7, %xmm0, %xmm2
+; AVX1-NEXT:    vpsllq $1, %xmm0, %xmm0
+; AVX1-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0,1,2,3],xmm2[4,5,6,7]
+; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: constant_shift_v4i64:
+; AVX2:       # BB#0:
+; AVX2-NEXT:    vpsllvq {{.*}}(%rip), %ymm0, %ymm0
+; AVX2-NEXT:    retq
+  %shift = shl <4 x i64> %a, <i64 1, i64 7, i64 31, i64 62>
+  ret <4 x i64> %shift
+}
+
+define <8 x i32> @constant_shift_v8i32(<8 x i32> %a) nounwind {
+; AVX1-LABEL: constant_shift_v8i32:
+; AVX1:       # BB#0:
+; AVX1-NEXT:    vpmulld {{.*}}(%rip), %xmm0, %xmm1
+; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm0
+; AVX1-NEXT:    vpmulld {{.*}}(%rip), %xmm0, %xmm0
+; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: constant_shift_v8i32:
+; AVX2:       # BB#0:
+; AVX2-NEXT:    vpsllvd {{.*}}(%rip), %ymm0, %ymm0
+; AVX2-NEXT:    retq
+  %shift = shl <8 x i32> %a, <i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 8, i32 7>
+  ret <8 x i32> %shift
+}
+
+define <16 x i16> @constant_shift_v16i16(<16 x i16> %a) nounwind {
+; AVX1-LABEL: constant_shift_v16i16:
+; AVX1:       # BB#0:
+; AVX1-NEXT:    vpmullw {{.*}}(%rip), %xmm0, %xmm1
+; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm0
+; AVX1-NEXT:    vpmullw {{.*}}(%rip), %xmm0, %xmm0
+; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: constant_shift_v16i16:
+; AVX2:       # BB#0:
+; AVX2-NEXT:    vpmullw {{.*}}(%rip), %ymm0, %ymm0
+; AVX2-NEXT:    retq
+  %shift = shl <16 x i16> %a, <i16 0, i16 1, i16 2, i16 3, i16 4, i16 5, i16 6, i16 7, i16 8, i16 9, i16 10, i16 11, i16 12, i16 13, i16 14, i16 15>
+  ret <16 x i16> %shift
+}
+
+define <32 x i8> @constant_shift_v32i8(<32 x i8> %a) nounwind {
+; AVX1-LABEL: constant_shift_v32i8:
+; AVX1:       # BB#0:
+; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
+; AVX1-NEXT:    vpsllw $4, %xmm1, %xmm2
+; AVX1-NEXT:    vmovdqa {{.*#+}} xmm3 = [240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240]
+; AVX1-NEXT:    vpand %xmm3, %xmm2, %xmm2
+; AVX1-NEXT:    vmovdqa {{.*#+}} xmm4 = [0,1,2,3,4,5,6,7,7,6,5,4,3,2,1,0]
+; AVX1-NEXT:    vpsllw $5, %xmm4, %xmm4
+; AVX1-NEXT:    vpblendvb %xmm4, %xmm2, %xmm1, %xmm1
+; AVX1-NEXT:    vpsllw $2, %xmm1, %xmm2
+; AVX1-NEXT:    vmovdqa {{.*#+}} xmm5 = [252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252]
+; AVX1-NEXT:    vpand %xmm5, %xmm2, %xmm2
+; AVX1-NEXT:    vpaddb %xmm4, %xmm4, %xmm6
+; AVX1-NEXT:    vpblendvb %xmm6, %xmm2, %xmm1, %xmm1
+; AVX1-NEXT:    vpaddb %xmm1, %xmm1, %xmm2
+; AVX1-NEXT:    vpaddb %xmm6, %xmm6, %xmm7
+; AVX1-NEXT:    vpblendvb %xmm7, %xmm2, %xmm1, %xmm1
+; AVX1-NEXT:    vpsllw $4, %xmm0, %xmm2
+; AVX1-NEXT:    vpand %xmm3, %xmm2, %xmm2
+; AVX1-NEXT:    vpblendvb %xmm4, %xmm2, %xmm0, %xmm0
+; AVX1-NEXT:    vpsllw $2, %xmm0, %xmm2
+; AVX1-NEXT:    vpand %xmm5, %xmm2, %xmm2
+; AVX1-NEXT:    vpblendvb %xmm6, %xmm2, %xmm0, %xmm0
+; AVX1-NEXT:    vpaddb %xmm0, %xmm0, %xmm2
+; AVX1-NEXT:    vpblendvb %xmm7, %xmm2, %xmm0, %xmm0
+; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: constant_shift_v32i8:
+; AVX2:       # BB#0:
+; AVX2-NEXT:    vmovdqa {{.*#+}} ymm1 = [0,1,2,3,4,5,6,7,7,6,5,4,3,2,1,0,0,1,2,3,4,5,6,7,7,6,5,4,3,2,1,0]
+; AVX2-NEXT:    vpsllw $5, %ymm1, %ymm1
+; AVX2-NEXT:    vpsllw $4, %ymm0, %ymm2
+; AVX2-NEXT:    vpand {{.*}}(%rip), %ymm2, %ymm2
+; AVX2-NEXT:    vpblendvb %ymm1, %ymm2, %ymm0, %ymm0
+; AVX2-NEXT:    vpsllw $2, %ymm0, %ymm2
+; AVX2-NEXT:    vpand {{.*}}(%rip), %ymm2, %ymm2
+; AVX2-NEXT:    vpaddb %ymm1, %ymm1, %ymm1
+; AVX2-NEXT:    vpblendvb %ymm1, %ymm2, %ymm0, %ymm0
+; AVX2-NEXT:    vpaddb %ymm0, %ymm0, %ymm2
+; AVX2-NEXT:    vpaddb %ymm1, %ymm1, %ymm1
+; AVX2-NEXT:    vpblendvb %ymm1, %ymm2, %ymm0, %ymm0
+; AVX2-NEXT:    retq
+  %shift = shl <32 x i8> %a, <i8 0, i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 7, i8 6, i8 5, i8 4, i8 3, i8 2, i8 1, i8 0, i8 0, i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 7, i8 6, i8 5, i8 4, i8 3, i8 2, i8 1, i8 0>
+  ret <32 x i8> %shift
+}
+
+;
+; Uniform Constant Shifts
+;
+
+define <4 x i64> @splatconstant_shift_v4i64(<4 x i64> %a) nounwind {
+; AVX1-LABEL: splatconstant_shift_v4i64:
+; AVX1:       # BB#0:
+; AVX1-NEXT:    vpsllq $7, %xmm0, %xmm1
+; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm0
+; AVX1-NEXT:    vpsllq $7, %xmm0, %xmm0
+; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: splatconstant_shift_v4i64:
+; AVX2:       # BB#0:
+; AVX2-NEXT:    vpsllq $7, %ymm0, %ymm0
+; AVX2-NEXT:    retq
+  %shift = shl <4 x i64> %a, <i64 7, i64 7, i64 7, i64 7>
+  ret <4 x i64> %shift
+}
+
+define <8 x i32> @splatconstant_shift_v8i32(<8 x i32> %a) nounwind {
+; AVX1-LABEL: splatconstant_shift_v8i32:
+; AVX1:       # BB#0:
+; AVX1-NEXT:    vpslld $5, %xmm0, %xmm1
+; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm0
+; AVX1-NEXT:    vpslld $5, %xmm0, %xmm0
+; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: splatconstant_shift_v8i32:
+; AVX2:       # BB#0:
+; AVX2-NEXT:    vpslld $5, %ymm0, %ymm0
+; AVX2-NEXT:    retq
+  %shift = shl <8 x i32> %a, <i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5>
+  ret <8 x i32> %shift
+}
+
+define <16 x i16> @splatconstant_shift_v16i16(<16 x i16> %a) nounwind {
+; AVX1-LABEL: splatconstant_shift_v16i16:
+; AVX1:       # BB#0:
+; AVX1-NEXT:    vpsllw $3, %xmm0, %xmm1
+; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm0
+; AVX1-NEXT:    vpsllw $3, %xmm0, %xmm0
+; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: splatconstant_shift_v16i16:
+; AVX2:       # BB#0:
+; AVX2-NEXT:    vpsllw $3, %ymm0, %ymm0
+; AVX2-NEXT:    retq
+  %shift = shl <16 x i16> %a, <i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3>
+  ret <16 x i16> %shift
+}
+
+define <32 x i8> @splatconstant_shift_v32i8(<32 x i8> %a) nounwind {
+; AVX1-LABEL: splatconstant_shift_v32i8:
+; AVX1:       # BB#0:
+; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
+; AVX1-NEXT:    vpsllw $3, %xmm1, %xmm1
+; AVX1-NEXT:    vmovdqa {{.*#+}} xmm2 = [248,248,248,248,248,248,248,248,248,248,248,248,248,248,248,248]
+; AVX1-NEXT:    vpand %xmm2, %xmm1, %xmm1
+; AVX1-NEXT:    vpsllw $3, %xmm0, %xmm0
+; AVX1-NEXT:    vpand %xmm2, %xmm0, %xmm0
+; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: splatconstant_shift_v32i8:
+; AVX2:       # BB#0:
+; AVX2-NEXT:    vpsllw $3, %ymm0, %ymm0
+; AVX2-NEXT:    vpand {{.*}}(%rip), %ymm0, %ymm0
+; AVX2-NEXT:    retq
+  %shift = shl <32 x i8> %a, <i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3>
+  ret <32 x i8> %shift
+}
diff --git a/test/CodeGen/X86/vector-shuffle-128-v16.ll b/test/CodeGen/X86/vector-shuffle-128-v16.ll
index 53d13c86657..0d949f6a76c 100644
--- a/test/CodeGen/X86/vector-shuffle-128-v16.ll
+++ b/test/CodeGen/X86/vector-shuffle-128-v16.ll
@@ -469,6 +469,20 @@ define <16 x i8> @shuffle_v16i8_00_01_02_19_04_05_06_23_08_09_10_27_12_13_14_31(
   ret <16 x i8> %shuffle
 }
 
+define <16 x i8> @shuffle_v16i8_00_01_02_zz_04_05_06_zz_08_09_10_zz_12_13_14_zz(<16 x i8> %a) {
+; SSE-LABEL: shuffle_v16i8_00_01_02_zz_04_05_06_zz_08_09_10_zz_12_13_14_zz:
+; SSE:       # BB#0:
+; SSE-NEXT:    andps {{.*}}(%rip), %xmm0
+; SSE-NEXT:    retq
+;
+; AVX-LABEL: shuffle_v16i8_00_01_02_zz_04_05_06_zz_08_09_10_zz_12_13_14_zz:
+; AVX:       # BB#0:
+; AVX-NEXT:    vandps {{.*}}(%rip), %xmm0, %xmm0
+; AVX-NEXT:    retq
+  %shuffle = shufflevector <16 x i8> %a, <16 x i8> zeroinitializer, <16 x i32> <i32 0, i32 1, i32 2, i32 19, i32 4, i32 5, i32 6, i32 23, i32 8, i32 9, i32 10, i32 27, i32 12, i32 13, i32 14, i32 31>
+  ret <16 x i8> %shuffle
+}
+
 define <16 x i8> @shuffle_v16i8_00_01_02_03_20_05_06_23_08_09_10_11_28_13_14_31(<16 x i8> %a, <16 x i8> %b) {
 ; SSE2-LABEL: shuffle_v16i8_00_01_02_03_20_05_06_23_08_09_10_11_28_13_14_31:
 ; SSE2:       # BB#0:
@@ -653,28 +667,28 @@ define <16 x i8> @shuffle_v16i8_16_zz_zz_zz_zz_zz_zz_zz_zz_zz_zz_zz_zz_zz_zz_zz(
 define <16 x i8> @shuffle_v16i8_zz_zz_zz_zz_zz_16_zz_zz_zz_zz_zz_zz_zz_zz_zz_zz(i8 %i) {
 ; SSE2-LABEL: shuffle_v16i8_zz_zz_zz_zz_zz_16_zz_zz_zz_zz_zz_zz_zz_zz_zz_zz:
 ; SSE2:       # BB#0:
-; SSE2-NEXT:    shll   $8, %edi
-; SSE2-NEXT:    pxor   %xmm0, %xmm0
+; SSE2-NEXT:    shll $8, %edi
+; SSE2-NEXT:    pxor %xmm0, %xmm0
 ; SSE2-NEXT:    pinsrw $2, %edi, %xmm0
 ; SSE2-NEXT:    retq
 ;
 ; SSSE3-LABEL: shuffle_v16i8_zz_zz_zz_zz_zz_16_zz_zz_zz_zz_zz_zz_zz_zz_zz_zz:
 ; SSSE3:       # BB#0:
-; SSSE3-NEXT:    shll   $8, %edi
-; SSSE3-NEXT:    pxor   %xmm0, %xmm0
+; SSSE3-NEXT:    shll $8, %edi
+; SSSE3-NEXT:    pxor %xmm0, %xmm0
 ; SSSE3-NEXT:    pinsrw $2, %edi, %xmm0
 ; SSSE3-NEXT:    retq
 ;
 ; SSE41-LABEL: shuffle_v16i8_zz_zz_zz_zz_zz_16_zz_zz_zz_zz_zz_zz_zz_zz_zz_zz:
 ; SSE41:       # BB#0:
-; SSE41-NEXT:    pxor   %xmm0, %xmm0
+; SSE41-NEXT:    pxor %xmm0, %xmm0
 ; SSE41-NEXT:    pinsrb $5, %edi, %xmm0
 ; SSE41-NEXT:    retq
 ;
 ; AVX-LABEL: shuffle_v16i8_zz_zz_zz_zz_zz_16_zz_zz_zz_zz_zz_zz_zz_zz_zz_zz:
 ; AVX:       # BB#0:
-; AVX-NEXT:    vpxor   %xmm0, %xmm0
-; AVX-NEXT:    vpinsrb $5, %edi, %xmm0
+; AVX-NEXT:    vpxor %xmm0, %xmm0, %xmm0
+; AVX-NEXT:    vpinsrb $5, %edi, %xmm0, %xmm0
 ; AVX-NEXT:    retq
   %a = insertelement <16 x i8> undef, i8 %i, i32 0
   %shuffle = shufflevector <16 x i8> zeroinitializer, <16 x i8> %a, <16 x i32> <i32 0, i32 0, i32 0, i32 0, i32 0, i32 16, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0>
@@ -684,28 +698,28 @@ define <16 x i8> @shuffle_v16i8_zz_zz_zz_zz_zz_16_zz_zz_zz_zz_zz_zz_zz_zz_zz_zz(
 define <16 x i8> @shuffle_v16i8_zz_uu_uu_zz_uu_uu_zz_zz_zz_zz_zz_zz_zz_zz_zz_16(i8 %i) {
 ; SSE2-LABEL: shuffle_v16i8_zz_uu_uu_zz_uu_uu_zz_zz_zz_zz_zz_zz_zz_zz_zz_16:
 ; SSE2:       # BB#0:
-; SSE2-NEXT:    shll   $8, %edi
-; SSE2-NEXT:    pxor   %xmm0, %xmm0
+; SSE2-NEXT:    shll $8, %edi
+; SSE2-NEXT:    pxor %xmm0, %xmm0
 ; SSE2-NEXT:    pinsrw $7, %edi, %xmm0
 ; SSE2-NEXT:    retq
 ;
 ; SSSE3-LABEL: shuffle_v16i8_zz_uu_uu_zz_uu_uu_zz_zz_zz_zz_zz_zz_zz_zz_zz_16:
 ; SSSE3:       # BB#0:
-; SSSE3-NEXT:    shll   $8, %edi
-; SSSE3-NEXT:    pxor   %xmm0, %xmm0
+; SSSE3-NEXT:    shll $8, %edi
+; SSSE3-NEXT:    pxor %xmm0, %xmm0
 ; SSSE3-NEXT:    pinsrw $7, %edi, %xmm0
 ; SSSE3-NEXT:    retq
 ;
 ; SSE41-LABEL: shuffle_v16i8_zz_uu_uu_zz_uu_uu_zz_zz_zz_zz_zz_zz_zz_zz_zz_16:
 ; SSE41:       # BB#0:
-; SSE41-NEXT:    pxor   %xmm0, %xmm0
+; SSE41-NEXT:    pxor %xmm0, %xmm0
 ; SSE41-NEXT:    pinsrb $15, %edi, %xmm0
 ; SSE41-NEXT:    retq
 ;
 ; AVX-LABEL: shuffle_v16i8_zz_uu_uu_zz_uu_uu_zz_zz_zz_zz_zz_zz_zz_zz_zz_16:
 ; AVX:       # BB#0:
-; AVX-NEXT:    vpxor   %xmm0, %xmm0
-; AVX-NEXT:    vpinsrb $15, %edi, %xmm0
+; AVX-NEXT:    vpxor %xmm0, %xmm0, %xmm0
+; AVX-NEXT:    vpinsrb $15, %edi, %xmm0, %xmm0
 ; AVX-NEXT:    retq
   %a = insertelement <16 x i8> undef, i8 %i, i32 0
   %shuffle = shufflevector <16 x i8> zeroinitializer, <16 x i8> %a, <16 x i32> <i32 0, i32 undef, i32 undef, i32 3, i32 undef, i32 undef, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 16>
@@ -716,27 +730,27 @@ define <16 x i8> @shuffle_v16i8_zz_zz_19_zz_zz_zz_zz_zz_zz_zz_zz_zz_zz_zz_zz_zz(
 ; SSE2-LABEL: shuffle_v16i8_zz_zz_19_zz_zz_zz_zz_zz_zz_zz_zz_zz_zz_zz_zz_zz:
 ; SSE2:       # BB#0:
 ; SSE2-NEXT:    movzbl %dil, %eax
-; SSE2-NEXT:    pxor   %xmm0, %xmm0
+; SSE2-NEXT:    pxor %xmm0, %xmm0
 ; SSE2-NEXT:    pinsrw $1, %eax, %xmm0
 ; SSE2-NEXT:    retq
 ;
 ; SSSE3-LABEL: shuffle_v16i8_zz_zz_19_zz_zz_zz_zz_zz_zz_zz_zz_zz_zz_zz_zz_zz:
 ; SSSE3:       # BB#0:
 ; SSSE3-NEXT:    movzbl %dil, %eax
-; SSSE3-NEXT:    pxor   %xmm0, %xmm0
+; SSSE3-NEXT:    pxor %xmm0, %xmm0
 ; SSSE3-NEXT:    pinsrw $1, %eax, %xmm0
 ; SSSE3-NEXT:    retq
 ;
 ; SSE41-LABEL: shuffle_v16i8_zz_zz_19_zz_zz_zz_zz_zz_zz_zz_zz_zz_zz_zz_zz_zz:
 ; SSE41:       # BB#0:
-; SSE41-NEXT:    pxor   %xmm0, %xmm0
+; SSE41-NEXT:    pxor %xmm0, %xmm0
 ; SSE41-NEXT:    pinsrb $2, %edi, %xmm0
 ; SSE41-NEXT:    retq
 ;
 ; AVX-LABEL: shuffle_v16i8_zz_zz_19_zz_zz_zz_zz_zz_zz_zz_zz_zz_zz_zz_zz_zz:
 ; AVX:       # BB#0:
-; AVX-NEXT:    vpxor   %xmm0, %xmm0
-; AVX-NEXT:    vpinsrb $2, %edi, %xmm0
+; AVX-NEXT:    vpxor %xmm0, %xmm0, %xmm0
+; AVX-NEXT:    vpinsrb $2, %edi, %xmm0, %xmm0
 ; AVX-NEXT:    retq
   %a = insertelement <16 x i8> undef, i8 %i, i32 3
   %shuffle = shufflevector <16 x i8> zeroinitializer, <16 x i8> %a, <16 x i32> <i32 0, i32 1, i32 19, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
@@ -1341,12 +1355,12 @@ define <16 x i8> @shuffle_v16i8_uu_02_03_zz_uu_06_07_zz_uu_10_11_zz_uu_14_15_zz(
 define <16 x i8> @shuffle_v16i8_bitcast_unpack(<16 x i8> %a, <16 x i8> %b) {
 ; SSE-LABEL: shuffle_v16i8_bitcast_unpack:
 ; SSE:       # BB#0:
-; SSE-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
+; SSE-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3],xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7]
 ; SSE-NEXT:    retq
 ;
 ; AVX-LABEL: shuffle_v16i8_bitcast_unpack:
 ; AVX:       # BB#0:
-; AVX-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
+; AVX-NEXT:    vpunpcklbw {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3],xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7]
 ; AVX-NEXT:    retq
   %shuffle8  = shufflevector <16 x i8> %a, <16 x i8> %b, <16 x i32> <i32 7, i32 23, i32 6, i32 22, i32 5, i32 21, i32 4, i32 20, i32 3, i32 19, i32 2, i32 18, i32 1, i32 17, i32 0, i32 16>
   %bitcast32 = bitcast <16 x i8> %shuffle8 to <4 x float>
diff --git a/test/CodeGen/X86/vector-shuffle-128-v8.ll b/test/CodeGen/X86/vector-shuffle-128-v8.ll
index 4007f0b2b13..6a29d33d6c5 100644
--- a/test/CodeGen/X86/vector-shuffle-128-v8.ll
+++ b/test/CodeGen/X86/vector-shuffle-128-v8.ll
@@ -1384,14 +1384,14 @@ define <8 x i16> @shuffle_v8i16_8zzzzzzz(i16 %i) {
 define <8 x i16> @shuffle_v8i16_z8zzzzzz(i16 %i) {
 ; SSE-LABEL: shuffle_v8i16_z8zzzzzz:
 ; SSE:       # BB#0:
-; SSE-NEXT:    pxor   %xmm0, %xmm0
+; SSE-NEXT:    pxor %xmm0, %xmm0
 ; SSE-NEXT:    pinsrw $1, %edi, %xmm0
 ; SSE-NEXT:    retq
 ;
 ; AVX-LABEL: shuffle_v8i16_z8zzzzzz:
 ; AVX:       # BB#0:
-; AVX-NEXT:    vpxor   %xmm0, %xmm0
-; AVX-NEXT:    vpinsrw $1, %edi, %xmm0
+; AVX-NEXT:    vpxor %xmm0, %xmm0, %xmm0
+; AVX-NEXT:    vpinsrw $1, %edi, %xmm0, %xmm0
 ; AVX-NEXT:    retq
   %a = insertelement <8 x i16> undef, i16 %i, i32 0
   %shuffle = shufflevector <8 x i16> zeroinitializer, <8 x i16> %a, <8 x i32> <i32 2, i32 8, i32 3, i32 7, i32 6, i32 5, i32 4, i32 3>
@@ -1401,14 +1401,14 @@ define <8 x i16> @shuffle_v8i16_z8zzzzzz(i16 %i) {
 define <8 x i16> @shuffle_v8i16_zzzzz8zz(i16 %i) {
 ; SSE-LABEL: shuffle_v8i16_zzzzz8zz:
 ; SSE:       # BB#0:
-; SSE-NEXT:    pxor   %xmm0, %xmm0
+; SSE-NEXT:    pxor %xmm0, %xmm0
 ; SSE-NEXT:    pinsrw $5, %edi, %xmm0
 ; SSE-NEXT:    retq
 ;
 ; AVX-LABEL: shuffle_v8i16_zzzzz8zz:
 ; AVX:       # BB#0:
-; AVX-NEXT:    vpxor   %xmm0, %xmm0
-; AVX-NEXT:    vpinsrw $5, %edi, %xmm0
+; AVX-NEXT:    vpxor %xmm0, %xmm0, %xmm0
+; AVX-NEXT:    vpinsrw $5, %edi, %xmm0, %xmm0
 ; AVX-NEXT:    retq
   %a = insertelement <8 x i16> undef, i16 %i, i32 0
   %shuffle = shufflevector <8 x i16> zeroinitializer, <8 x i16> %a, <8 x i32> <i32 0, i32 0, i32 0, i32 0, i32 0, i32 8, i32 0, i32 0>
@@ -1418,14 +1418,14 @@ define <8 x i16> @shuffle_v8i16_zzzzz8zz(i16 %i) {
 define <8 x i16> @shuffle_v8i16_zuuzuuz8(i16 %i) {
 ; SSE-LABEL: shuffle_v8i16_zuuzuuz8:
 ; SSE:       # BB#0:
-; SSE-NEXT:    pxor   %xmm0, %xmm0
+; SSE-NEXT:    pxor %xmm0, %xmm0
 ; SSE-NEXT:    pinsrw $7, %edi, %xmm0
 ; SSE-NEXT:    retq
 ;
 ; AVX-LABEL: shuffle_v8i16_zuuzuuz8:
 ; AVX:       # BB#0:
-; AVX-NEXT:    vpxor   %xmm0, %xmm0
-; AVX-NEXT:    vpinsrw $7, %edi, %xmm0
+; AVX-NEXT:    vpxor %xmm0, %xmm0, %xmm0
+; AVX-NEXT:    vpinsrw $7, %edi, %xmm0, %xmm0
 ; AVX-NEXT:    retq
   %a = insertelement <8 x i16> undef, i16 %i, i32 0
   %shuffle = shufflevector <8 x i16> zeroinitializer, <8 x i16> %a, <8 x i32> <i32 0, i32 undef, i32 undef, i32 3, i32 undef, i32 undef, i32 6, i32 8>
@@ -1435,14 +1435,14 @@ define <8 x i16> @shuffle_v8i16_zuuzuuz8(i16 %i) {
 define <8 x i16> @shuffle_v8i16_zzBzzzzz(i16 %i) {
 ; SSE-LABEL: shuffle_v8i16_zzBzzzzz:
 ; SSE:       # BB#0:
-; SSE-NEXT:    pxor   %xmm0, %xmm0
+; SSE-NEXT:    pxor %xmm0, %xmm0
 ; SSE-NEXT:    pinsrw $2, %edi, %xmm0
 ; SSE-NEXT:    retq
 ;
 ; AVX-LABEL: shuffle_v8i16_zzBzzzzz:
 ; AVX:       # BB#0:
-; AVX-NEXT:    vpxor   %xmm0, %xmm0
-; AVX-NEXT:    vpinsrw $2, %edi, %xmm0
+; AVX-NEXT:    vpxor %xmm0, %xmm0, %xmm0
+; AVX-NEXT:    vpinsrw $2, %edi, %xmm0, %xmm0
 ; AVX-NEXT:    retq
   %a = insertelement <8 x i16> undef, i16 %i, i32 3
   %shuffle = shufflevector <8 x i16> zeroinitializer, <8 x i16> %a, <8 x i32> <i32 0, i32 1, i32 11, i32 3, i32 4, i32 5, i32 6, i32 7>
diff --git a/test/CodeGen/X86/vector-shuffle-256-v32.ll b/test/CodeGen/X86/vector-shuffle-256-v32.ll
index a0f43de7563..bbbe0cd8dfc 100644
--- a/test/CodeGen/X86/vector-shuffle-256-v32.ll
+++ b/test/CodeGen/X86/vector-shuffle-256-v32.ll
@@ -947,6 +947,24 @@ define <32 x i8> @shuffle_v32i8_32_01_34_03_36_05_38_07_40_09_42_11_44_13_46_15_
   ret <32 x i8> %shuffle
 }
 
+define <32 x i8> @shuffle_v32i8_zz_01_zz_03_zz_05_zz_07_zz_09_zz_11_zz_13_zz_15_zz_17_zz_19_zz_21_zz_23_zz_25_zz_27_zz_29_zz_31(<32 x i8> %a) {
+; AVX1-LABEL: shuffle_v32i8_zz_01_zz_03_zz_05_zz_07_zz_09_zz_11_zz_13_zz_15_zz_17_zz_19_zz_21_zz_23_zz_25_zz_27_zz_29_zz_31:
+; AVX1:       # BB#0:
+; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
+; AVX1-NEXT:    vmovaps {{.*#+}} xmm2 = [0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255]
+; AVX1-NEXT:    vandps %xmm2, %xmm1, %xmm1
+; AVX1-NEXT:    vandps %xmm2, %xmm0, %xmm0
+; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: shuffle_v32i8_zz_01_zz_03_zz_05_zz_07_zz_09_zz_11_zz_13_zz_15_zz_17_zz_19_zz_21_zz_23_zz_25_zz_27_zz_29_zz_31:
+; AVX2:       # BB#0:
+; AVX2-NEXT:    vandps {{.*}}(%rip), %ymm0, %ymm0
+; AVX2-NEXT:    retq
+  %shuffle = shufflevector <32 x i8> %a, <32 x i8> zeroinitializer, <32 x i32> <i32 32, i32 1, i32 34, i32 3, i32 36, i32 5, i32 38, i32 7, i32 40, i32 9, i32 42, i32 11, i32 44, i32 13, i32 46, i32 15, i32 48, i32 17, i32 50, i32 19, i32 52, i32 21, i32 54, i32 23, i32 56, i32 25, i32 58, i32 27, i32 60, i32 29, i32 62, i32 31>
+  ret <32 x i8> %shuffle
+}
+
 define <32 x i8> @shuffle_v32i8_00_32_00_32_00_32_00_32_00_32_00_32_00_32_00_32_00_32_00_32_00_32_00_32_00_32_00_32_00_32_00_32(<32 x i8> %a, <32 x i8> %b) {
 ; AVX1-LABEL: shuffle_v32i8_00_32_00_32_00_32_00_32_00_32_00_32_00_32_00_32_00_32_00_32_00_32_00_32_00_32_00_32_00_32_00_32:
 ; AVX1:       # BB#0:
diff --git a/test/CodeGen/X86/vector-shuffle-256-v4.ll b/test/CodeGen/X86/vector-shuffle-256-v4.ll
index 1b42a637907..62bf288a870 100644
--- a/test/CodeGen/X86/vector-shuffle-256-v4.ll
+++ b/test/CodeGen/X86/vector-shuffle-256-v4.ll
@@ -810,30 +810,20 @@ define <4 x i64> @stress_test1(<4 x i64> %a, <4 x i64> %b) {
 }
 
 define <4 x i64> @insert_reg_and_zero_v4i64(i64 %a) {
-; AVX1-LABEL: insert_reg_and_zero_v4i64:
-; AVX1:       # BB#0:
-; AVX1-NEXT:    vmovq %rdi, %xmm0
-; AVX1-NEXT:    retq
-;
-; AVX2-LABEL: insert_reg_and_zero_v4i64:
-; AVX2:       # BB#0:
-; AVX2-NEXT:    vmovq %rdi, %xmm0
-; AVX2-NEXT:    retq
+; ALL-LABEL: insert_reg_and_zero_v4i64:
+; ALL:       # BB#0:
+; ALL-NEXT:    vmovq %rdi, %xmm0
+; ALL-NEXT:    retq
   %v = insertelement <4 x i64> undef, i64 %a, i64 0
   %shuffle = shufflevector <4 x i64> %v, <4 x i64> zeroinitializer, <4 x i32> <i32 0, i32 5, i32 6, i32 7>
   ret <4 x i64> %shuffle
 }
 
 define <4 x i64> @insert_mem_and_zero_v4i64(i64* %ptr) {
-; AVX1-LABEL: insert_mem_and_zero_v4i64:
-; AVX1:       # BB#0:
-; AVX1-NEXT:    vmovq {{.*#+}} xmm0 = mem[0],zero
-; AVX1-NEXT:    retq
-;
-; AVX2-LABEL: insert_mem_and_zero_v4i64:
-; AVX2:       # BB#0:
-; AVX2-NEXT:    vmovq {{.*#+}} xmm0 = mem[0],zero
-; AVX2-NEXT:    retq
+; ALL-LABEL: insert_mem_and_zero_v4i64:
+; ALL:       # BB#0:
+; ALL-NEXT:    vmovq {{.*#+}} xmm0 = mem[0],zero
+; ALL-NEXT:    retq
   %a = load i64, i64* %ptr
   %v = insertelement <4 x i64> undef, i64 %a, i64 0
   %shuffle = shufflevector <4 x i64> %v, <4 x i64> zeroinitializer, <4 x i32> <i32 0, i32 5, i32 6, i32 7>
@@ -843,7 +833,6 @@ define <4 x i64> @insert_mem_and_zero_v4i64(i64* %ptr) {
 define <4 x double> @insert_reg_and_zero_v4f64(double %a) {
 ; ALL-LABEL: insert_reg_and_zero_v4f64:
 ; ALL:       # BB#0:
-; ALL-NEXT:    # kill: XMM0<def> XMM0<kill> YMM0<def>
 ; ALL-NEXT:    vxorpd %ymm1, %ymm1, %ymm1
 ; ALL-NEXT:    vblendpd {{.*#+}} ymm0 = ymm0[0],ymm1[1,2,3]
 ; ALL-NEXT:    retq
@@ -875,15 +864,10 @@ define <4 x double> @splat_mem_v4f64(double* %ptr) {
 }
 
 define <4 x i64> @splat_mem_v4i64(i64* %ptr) {
-; AVX1-LABEL: splat_mem_v4i64:
-; AVX1:       # BB#0:
-; AVX1-NEXT:    vbroadcastsd (%rdi), %ymm0
-; AVX1-NEXT:    retq
-;
-; AVX2-LABEL: splat_mem_v4i64:
-; AVX2:       # BB#0:
-; AVX2-NEXT:    vbroadcastsd (%rdi), %ymm0
-; AVX2-NEXT:    retq
+; ALL-LABEL: splat_mem_v4i64:
+; ALL:       # BB#0:
+; ALL-NEXT:    vbroadcastsd (%rdi), %ymm0
+; ALL-NEXT:    retq
   %a = load i64, i64* %ptr
   %v = insertelement <4 x i64> undef, i64 %a, i64 0
   %shuffle = shufflevector <4 x i64> %v, <4 x i64> undef, <4 x i32> <i32 0, i32 0, i32 0, i32 0>
@@ -916,6 +900,60 @@ define <4 x double> @splat_v4f64(<2 x double> %r) {
   ret <4 x double> %1
 }
 
+define <4 x i64> @splat_mem_v4i64_from_v2i64(<2 x i64>* %ptr) {
+; AVX1-LABEL: splat_mem_v4i64_from_v2i64:
+; AVX1:       # BB#0:
+; AVX1-NEXT:    vmovddup {{.*#+}} xmm0 = mem[0,0]
+; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm0, %ymm0
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: splat_mem_v4i64_from_v2i64:
+; AVX2:       # BB#0:
+; AVX2-NEXT:    vbroadcastsd (%rdi), %ymm0
+; AVX2-NEXT:    retq
+  %v = load <2 x i64>, <2 x i64>* %ptr
+  %shuffle = shufflevector <2 x i64> %v, <2 x i64> undef, <4 x i32> <i32 0, i32 0, i32 0, i32 0>
+  ret <4 x i64> %shuffle
+}
+
+define <4 x double> @splat_mem_v4f64_from_v2f64(<2 x double>* %ptr) {
+; AVX1-LABEL: splat_mem_v4f64_from_v2f64:
+; AVX1:       # BB#0:
+; AVX1-NEXT:    vmovddup {{.*#+}} xmm0 = mem[0,0]
+; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm0, %ymm0
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: splat_mem_v4f64_from_v2f64:
+; AVX2:       # BB#0:
+; AVX2-NEXT:    vbroadcastsd (%rdi), %ymm0
+; AVX2-NEXT:    retq
+  %v = load <2 x double>, <2 x double>* %ptr
+  %shuffle = shufflevector <2 x double> %v, <2 x double> undef, <4 x i32> <i32 0, i32 0, i32 0, i32 0>
+  ret <4 x double> %shuffle
+}
+
+define <4 x i64> @splat128_mem_v4i64_from_v2i64(<2 x i64>* %ptr) {
+; ALL-LABEL: splat128_mem_v4i64_from_v2i64:
+; ALL:       # BB#0:
+; ALL-NEXT:    vmovaps (%rdi), %xmm0
+; ALL-NEXT:    vinsertf128 $1, %xmm0, %ymm0, %ymm0
+; ALL-NEXT:    retq
+  %v = load <2 x i64>, <2 x i64>* %ptr
+  %shuffle = shufflevector <2 x i64> %v, <2 x i64> undef, <4 x i32> <i32 0, i32 1, i32 0, i32 1>
+  ret <4 x i64> %shuffle
+}
+
+define <4 x double> @splat128_mem_v4f64_from_v2f64(<2 x double>* %ptr) {
+; ALL-LABEL: splat128_mem_v4f64_from_v2f64:
+; ALL:       # BB#0:
+; ALL-NEXT:    vmovaps (%rdi), %xmm0
+; ALL-NEXT:    vinsertf128 $1, %xmm0, %ymm0, %ymm0
+; ALL-NEXT:    retq
+  %v = load <2 x double>, <2 x double>* %ptr
+  %shuffle = shufflevector <2 x double> %v, <2 x double> undef, <4 x i32> <i32 0, i32 1, i32 0, i32 1>
+  ret <4 x double> %shuffle
+}
+
 define <4 x double> @bitcast_v4f64_0426(<4 x double> %a, <4 x double> %b) {
 ; AVX1-LABEL: bitcast_v4f64_0426:
 ; AVX1:       # BB#0:
@@ -924,7 +962,7 @@ define <4 x double> @bitcast_v4f64_0426(<4 x double> %a, <4 x double> %b) {
 ;
 ; AVX2-LABEL: bitcast_v4f64_0426:
 ; AVX2:       # BB#0:
-; AVX2-NEXT:    vpunpcklqdq  {{.*#+}} ymm0 = ymm0[0],ymm1[0],ymm0[2],ymm1[2]
+; AVX2-NEXT:    vpunpcklqdq {{.*#+}} ymm0 = ymm0[0],ymm1[0],ymm0[2],ymm1[2]
 ; AVX2-NEXT:    retq
   %shuffle64 = shufflevector <4 x double> %a, <4 x double> %b, <4 x i32> <i32 4, i32 0, i32 6, i32 2>
   %bitcast32 = bitcast <4 x double> %shuffle64 to <8 x float>
diff --git a/test/CodeGen/X86/vector-shuffle-256-v8.ll b/test/CodeGen/X86/vector-shuffle-256-v8.ll
index bb07077b555..bc72e0a6617 100644
--- a/test/CodeGen/X86/vector-shuffle-256-v8.ll
+++ b/test/CodeGen/X86/vector-shuffle-256-v8.ll
@@ -2088,15 +2088,10 @@ entry:
 }
 
 define <8 x i32> @insert_mem_and_zero_v8i32(i32* %ptr) {
-; AVX1-LABEL: insert_mem_and_zero_v8i32:
-; AVX1:       # BB#0:
-; AVX1-NEXT:    vmovd {{.*#+}} xmm0 = mem[0],zero,zero,zero
-; AVX1-NEXT:    retq
-;
-; AVX2-LABEL: insert_mem_and_zero_v8i32:
-; AVX2:       # BB#0:
-; AVX2-NEXT:    vmovd {{.*#+}} xmm0 = mem[0],zero,zero,zero
-; AVX2-NEXT:    retq
+; ALL-LABEL: insert_mem_and_zero_v8i32:
+; ALL:       # BB#0:
+; ALL-NEXT:    vmovd {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; ALL-NEXT:    retq
   %a = load i32, i32* %ptr
   %v = insertelement <8 x i32> undef, i32 %a, i32 0
   %shuffle = shufflevector <8 x i32> %v, <8 x i32> zeroinitializer, <8 x i32> <i32 0, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
diff --git a/test/CodeGen/X86/vector-shuffle-sse4a.ll b/test/CodeGen/X86/vector-shuffle-sse4a.ll
new file mode 100644
index 00000000000..26062335cc1
--- /dev/null
+++ b/test/CodeGen/X86/vector-shuffle-sse4a.ll
@@ -0,0 +1,221 @@
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+ssse3,+sse4a | FileCheck %s --check-prefix=ALL --check-prefix=BTVER1
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx,+sse4a | FileCheck %s --check-prefix=ALL --check-prefix=BTVER2
+
+;
+; EXTRQI
+;
+
+define <16 x i8> @shuf_0zzzuuuuuuuuuuuu(<16 x i8> %a0) {
+; BTVER1-LABEL: shuf_0zzzuuuuuuuuuuuu:
+; BTVER1:       # BB#0:
+; BTVER1-NEXT:    extrq {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,zero,zero,zero,zero,xmm0[u,u,u,u,u,u,u,u]
+; BTVER1-NEXT:    retq
+;
+; BTVER2-LABEL: shuf_0zzzuuuuuuuuuuuu:
+; BTVER2:       # BB#0:
+; BTVER2-NEXT:    vpmovzxbq {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,zero,zero,zero,zero,xmm0[1],zero,zero,zero,zero,zero,zero,zero
+; BTVER2-NEXT:    retq
+  %s = shufflevector <16 x i8> %a0, <16 x i8> zeroinitializer, <16 x i32> <i32 0, i32 16, i32 16, i32 16, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+  ret <16 x i8> %s
+}
+
+define <16 x i8> @shuf_0zzzzzzz1zzzzzzz(<16 x i8> %a0) {
+; BTVER1-LABEL: shuf_0zzzzzzz1zzzzzzz:
+; BTVER1:       # BB#0:
+; BTVER1-NEXT:    movaps %xmm0, %xmm1
+; BTVER1-NEXT:    extrq {{.*#+}} xmm1 = xmm1[1],zero,zero,zero,zero,zero,zero,zero,xmm1[u,u,u,u,u,u,u,u]
+; BTVER1-NEXT:    extrq {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,zero,zero,zero,zero,xmm0[u,u,u,u,u,u,u,u]
+; BTVER1-NEXT:    punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
+; BTVER1-NEXT:    retq
+;
+; BTVER2-LABEL: shuf_0zzzzzzz1zzzzzzz:
+; BTVER2:       # BB#0:
+; BTVER2-NEXT:    vpmovzxbq {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,zero,zero,zero,zero,xmm0[1],zero,zero,zero,zero,zero,zero,zero
+; BTVER2-NEXT:    retq
+  %s = shufflevector <16 x i8> %a0, <16 x i8> zeroinitializer, <16 x i32> <i32 0, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 1, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16>
+  ret <16 x i8> %s
+}
+
+define <16 x i8> @shuf_01zzuuuuuuuuuuuu(<16 x i8> %a0) {
+; BTVER1-LABEL: shuf_01zzuuuuuuuuuuuu:
+; BTVER1:       # BB#0:
+; BTVER1-NEXT:    extrq {{.*#+}} xmm0 = xmm0[0,1],zero,zero,zero,zero,zero,zero,xmm0[u,u,u,u,u,u,u,u]
+; BTVER1-NEXT:    retq
+;
+; BTVER2-LABEL: shuf_01zzuuuuuuuuuuuu:
+; BTVER2:       # BB#0:
+; BTVER2-NEXT:    vpmovzxwq {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero
+; BTVER2-NEXT:    retq
+  %s = shufflevector <16 x i8> %a0, <16 x i8> zeroinitializer, <16 x i32> <i32 0, i32 1, i32 16, i32 16, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+  ret <16 x i8> %s
+}
+
+define <16 x i8> @shuf_01zzzzzz23zzzzzz(<16 x i8> %a0) {
+; BTVER1-LABEL: shuf_01zzzzzz23zzzzzz:
+; BTVER1:       # BB#0:
+; BTVER1-NEXT:    movaps %xmm0, %xmm1
+; BTVER1-NEXT:    extrq {{.*#+}} xmm1 = xmm1[2,3],zero,zero,zero,zero,zero,zero,xmm1[u,u,u,u,u,u,u,u]
+; BTVER1-NEXT:    extrq {{.*#+}} xmm0 = xmm0[0,1],zero,zero,zero,zero,zero,zero,xmm0[u,u,u,u,u,u,u,u]
+; BTVER1-NEXT:    punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
+; BTVER1-NEXT:    retq
+;
+; BTVER2-LABEL: shuf_01zzzzzz23zzzzzz:
+; BTVER2:       # BB#0:
+; BTVER2-NEXT:    vpmovzxwq {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero
+; BTVER2-NEXT:    retq
+  %s = shufflevector <16 x i8> %a0, <16 x i8> zeroinitializer, <16 x i32> <i32 0, i32 1, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 2, i32 3, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16>
+  ret <16 x i8> %s
+}
+
+define <16 x i8> @shuf_1zzzuuuuuuuuuuuu(<16 x i8> %a0) {
+; ALL-LABEL: shuf_1zzzuuuuuuuuuuuu:
+; ALL:       # BB#0:
+; ALL-NEXT:    extrq {{.*#+}} xmm0 = xmm0[1],zero,zero,zero,zero,zero,zero,zero,xmm0[u,u,u,u,u,u,u,u]
+; ALL-NEXT:    retq
+  %s = shufflevector <16 x i8> %a0, <16 x i8> zeroinitializer, <16 x i32> <i32 1, i32 16, i32 16, i32 16, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+  ret <16 x i8> %s
+}
+
+define <8 x i16> @shuf_1zzzuuuu(<8 x i16> %a0) {
+; ALL-LABEL: shuf_1zzzuuuu:
+; ALL:       # BB#0:
+; ALL-NEXT:    extrq {{.*#+}} xmm0 = xmm0[2,3],zero,zero,zero,zero,zero,zero,xmm0[u,u,u,u,u,u,u,u]
+; ALL-NEXT:    retq
+  %s = shufflevector <8 x i16> %a0, <8 x i16> zeroinitializer, <8 x i32> <i32 1, i32 8, i32 8, i32 8, i32 undef, i32 undef, i32 undef, i32 undef>
+  ret <8 x i16> %s
+}
+
+define <8 x i16> @shuf_12zzuuuu(<8 x i16> %a0) {
+; ALL-LABEL: shuf_12zzuuuu:
+; ALL:       # BB#0:
+; ALL-NEXT:    extrq {{.*#+}} xmm0 = xmm0[2,3,4,5],zero,zero,zero,zero,xmm0[u,u,u,u,u,u,u,u]
+; ALL-NEXT:    retq
+  %s = shufflevector <8 x i16> %a0, <8 x i16> zeroinitializer, <8 x i32> <i32 1, i32 2, i32 8, i32 8, i32 undef, i32 undef, i32 undef, i32 undef>
+  ret <8 x i16> %s
+}
+
+define <8 x i16> @shuf_012zuuuu(<8 x i16> %a0) {
+; ALL-LABEL: shuf_012zuuuu:
+; ALL:       # BB#0:
+; ALL-NEXT:    extrq {{.*#+}} xmm0 = xmm0[0,1,2,3,4,5],zero,zero,xmm0[u,u,u,u,u,u,u,u]
+; ALL-NEXT:    retq
+  %s = shufflevector <8 x i16> %a0, <8 x i16> zeroinitializer, <8 x i32> <i32 0, i32 1, i32 2, i32 8, i32 undef, i32 undef, i32 undef, i32 undef>
+  ret <8 x i16> %s
+}
+
+define <8 x i16> @shuf_0zzz1zzz(<8 x i16> %a0) {
+; BTVER1-LABEL: shuf_0zzz1zzz:
+; BTVER1:       # BB#0:
+; BTVER1-NEXT:    movaps %xmm0, %xmm1
+; BTVER1-NEXT:    extrq {{.*#+}} xmm1 = xmm1[2,3],zero,zero,zero,zero,zero,zero,xmm1[u,u,u,u,u,u,u,u]
+; BTVER1-NEXT:    extrq {{.*#+}} xmm0 = xmm0[0,1],zero,zero,zero,zero,zero,zero,xmm0[u,u,u,u,u,u,u,u]
+; BTVER1-NEXT:    punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
+; BTVER1-NEXT:    retq
+;
+; BTVER2-LABEL: shuf_0zzz1zzz:
+; BTVER2:       # BB#0:
+; BTVER2-NEXT:    vpmovzxwq {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero
+; BTVER2-NEXT:    retq
+  %s = shufflevector <8 x i16> %a0, <8 x i16> zeroinitializer, <8 x i32> <i32 0, i32 8, i32 8, i32 8, i32 1, i32 8, i32 8, i32 8>
+  ret <8 x i16> %s
+}
+
+define <4 x i32> @shuf_0z1z(<4 x i32> %a0) {
+; BTVER1-LABEL: shuf_0z1z:
+; BTVER1:       # BB#0:
+; BTVER1-NEXT:    pxor %xmm1, %xmm1
+; BTVER1-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
+; BTVER1-NEXT:    retq
+;
+; BTVER2-LABEL: shuf_0z1z:
+; BTVER2:       # BB#0:
+; BTVER2-NEXT:    vpmovzxdq {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero
+; BTVER2-NEXT:    retq
+  %s = shufflevector <4 x i32> %a0, <4 x i32> zeroinitializer, <4 x i32> <i32 0, i32 4, i32 1, i32 4>
+  ret <4 x i32> %s
+}
+
+;
+; INSERTQI
+;
+
+define <16 x i8> @shuf_0_0_2_3_uuuu_uuuu_uuuu(<16 x i8> %a0, <16 x i8> %a1) {
+; ALL-LABEL: shuf_0_0_2_3_uuuu_uuuu_uuuu:
+; ALL:       # BB#0:
+; ALL-NEXT:    insertq {{.*#+}} xmm0 = xmm0[0,0,2,3,4,5,6,7,u,u,u,u,u,u,u,u]
+; ALL-NEXT:    retq
+  %s = shufflevector <16 x i8> %a0, <16 x i8> %a1, <16 x i32> <i32 0, i32 0, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+  ret <16 x i8> %s
+}
+
+define <16 x i8> @shuf_0_16_2_3_uuuu_uuuu_uuuu(<16 x i8> %a0, <16 x i8> %a1) {
+; ALL-LABEL: shuf_0_16_2_3_uuuu_uuuu_uuuu:
+; ALL:       # BB#0:
+; ALL-NEXT:    insertq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[2,3,4,5,6,7,u,u,u,u,u,u,u,u]
+; ALL-NEXT:    retq
+  %s = shufflevector <16 x i8> %a0, <16 x i8> %a1, <16 x i32> <i32 0, i32 16, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+  ret <16 x i8> %s
+}
+
+define <16 x i8> @shuf_16_1_2_3_uuuu_uuuu_uuuu(<16 x i8> %a0, <16 x i8> %a1) {
+; ALL-LABEL: shuf_16_1_2_3_uuuu_uuuu_uuuu:
+; ALL:       # BB#0:
+; ALL-NEXT:    insertq {{.*#+}} xmm0 = xmm1[0],xmm0[1,2,3,4,5,6,7,u,u,u,u,u,u,u,u]
+; ALL-NEXT:    retq
+  %s = shufflevector <16 x i8> %a0, <16 x i8> %a1, <16 x i32> <i32 16, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+  ret <16 x i8> %s
+}
+
+define <8 x i16> @shuf_0823uuuu(<8 x i16> %a0, <8 x i16> %a1) {
+; ALL-LABEL: shuf_0823uuuu:
+; ALL:       # BB#0:
+; ALL-NEXT:    insertq {{.*#+}} xmm0 = xmm0[0,1],xmm1[0,1],xmm0[4,5,6,7,u,u,u,u,u,u,u,u]
+; ALL-NEXT:    retq
+  %s = shufflevector <8 x i16> %a0, <8 x i16> %a1, <8 x i32> <i32 0, i32 8, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef>
+  ret <8 x i16> %s
+}
+
+define <8 x i16> @shuf_0183uuuu(<8 x i16> %a0, <8 x i16> %a1) {
+; ALL-LABEL: shuf_0183uuuu:
+; ALL:       # BB#0:
+; ALL-NEXT:    insertq {{.*#+}} xmm0 = xmm0[0,1,2,3],xmm1[0,1],xmm0[6,7,u,u,u,u,u,u,u,u]
+; ALL-NEXT:    retq
+  %s = shufflevector <8 x i16> %a0, <8 x i16> %a1, <8 x i32> <i32 0, i32 1, i32 8, i32 3, i32 undef, i32 undef, i32 undef, i32 undef>
+  ret <8 x i16> %s
+}
+
+define <8 x i16> @shuf_0128uuuu(<8 x i16> %a0, <8 x i16> %a1) {
+; ALL-LABEL: shuf_0128uuuu:
+; ALL:       # BB#0:
+; ALL-NEXT:    insertq {{.*#+}} xmm0 = xmm0[0,1,2,3,4,5],xmm1[0,1],xmm0[u,u,u,u,u,u,u,u]
+; ALL-NEXT:    retq
+  %s = shufflevector <8 x i16> %a0, <8 x i16> %a1, <8 x i32> <i32 0, i32 1, i32 2, i32 8, i32 undef, i32 undef, i32 undef, i32 undef>
+  ret <8 x i16> %s
+}
+
+define <8 x i16> @shuf_0893uuuu(<8 x i16> %a0, <8 x i16> %a1) {
+; ALL-LABEL: shuf_0893uuuu:
+; ALL:       # BB#0:
+; ALL-NEXT:    insertq {{.*#+}} xmm0 = xmm0[0,1],xmm1[0,1,2,3],xmm0[6,7,u,u,u,u,u,u,u,u]
+; ALL-NEXT:    retq
+  %s = shufflevector <8 x i16> %a0, <8 x i16> %a1, <8 x i32> <i32 0, i32 8, i32 9, i32 3, i32 undef, i32 undef, i32 undef, i32 undef>
+  ret <8 x i16> %s
+}
+
+define <8 x i16> @shuf_089Auuuu(<8 x i16> %a0, <8 x i16> %a1) {
+; ALL-LABEL: shuf_089Auuuu:
+; ALL:       # BB#0:
+; ALL-NEXT:    insertq {{.*#+}} xmm0 = xmm0[0,1],xmm1[0,1,2,3,4,5],xmm0[u,u,u,u,u,u,u,u]
+; ALL-NEXT:    retq
+  %s = shufflevector <8 x i16> %a0, <8 x i16> %a1, <8 x i32> <i32 0, i32 8, i32 9, i32 10, i32 undef, i32 undef, i32 undef, i32 undef>
+  ret <8 x i16> %s
+}
+
+define <8 x i16> @shuf_089uuuuu(<8 x i16> %a0, <8 x i16> %a1) {
+; ALL-LABEL: shuf_089uuuuu:
+; ALL:       # BB#0:
+; ALL-NEXT:    insertq {{.*#+}} xmm0 = xmm0[0,1],xmm1[0,1,2,3],xmm0[6,7,u,u,u,u,u,u,u,u]
+; ALL-NEXT:    retq
+  %s = shufflevector <8 x i16> %a0, <8 x i16> %a1, <8 x i32> <i32 0, i32 8, i32 9, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+  ret <8 x i16> %s
+}
diff --git a/test/CodeGen/X86/vector-trunc.ll b/test/CodeGen/X86/vector-trunc.ll
index d2eef9af2a2..2480e676cad 100644
--- a/test/CodeGen/X86/vector-trunc.ll
+++ b/test/CodeGen/X86/vector-trunc.ll
@@ -223,15 +223,15 @@ entry:
 }
 
 define <16 x i8> @trunc16i64_const() {
-; SSE-LABEL:  trunc16i64_const
-; SSE:        # BB#0: # %entry
-; SSE-NEXT:   xorps %xmm0, %xmm0
-; SSE-NEXT:   retq
-;
-; AVX-LABEL:  trunc16i64_const
-; AVX:        # BB#0: # %entry
-; AVX-NEXT:   vxorps %xmm0, %xmm0, %xmm0
-; AVX-NEXT:   retq
+; SSE-LABEL: trunc16i64_const:
+; SSE:       # BB#0: # %entry
+; SSE-NEXT:    xorps %xmm0, %xmm0
+; SSE-NEXT:    retq
+;
+; AVX-LABEL: trunc16i64_const:
+; AVX:       # BB#0: # %entry
+; AVX-NEXT:    vxorps %xmm0, %xmm0, %xmm0
+; AVX-NEXT:    retq
 
 entry:
   %0 = trunc <16 x i64> zeroinitializer to <16 x i8>
diff --git a/test/CodeGen/X86/vector-tzcnt-128.ll b/test/CodeGen/X86/vector-tzcnt-128.ll
new file mode 100644
index 00000000000..adafd72d65a
--- /dev/null
+++ b/test/CodeGen/X86/vector-tzcnt-128.ll
@@ -0,0 +1,1788 @@
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mcpu=x86-64 | FileCheck %s --check-prefix=ALL --check-prefix=SSE --check-prefix=SSE2
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mcpu=x86-64 -mattr=+sse3 | FileCheck %s --check-prefix=ALL --check-prefix=SSE --check-prefix=SSE3
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mcpu=x86-64 -mattr=+ssse3 | FileCheck %s --check-prefix=ALL --check-prefix=SSE --check-prefix=SSSE3
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mcpu=x86-64 -mattr=+sse4.1 | FileCheck %s --check-prefix=ALL --check-prefix=SSE --check-prefix=SSE41
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mcpu=x86-64 -mattr=+avx | FileCheck %s --check-prefix=ALL --check-prefix=AVX --check-prefix=AVX1
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mcpu=x86-64 -mattr=+avx2 | FileCheck %s --check-prefix=ALL --check-prefix=AVX --check-prefix=AVX2
+
+target triple = "x86_64-unknown-unknown"
+
+define <2 x i64> @testv2i64(<2 x i64> %in) nounwind {
+; SSE2-LABEL: testv2i64:
+; SSE2:       # BB#0:
+; SSE2-NEXT:    movd %xmm0, %rax
+; SSE2-NEXT:    bsfq %rax, %rax
+; SSE2-NEXT:    movl $64, %ecx
+; SSE2-NEXT:    cmoveq %rcx, %rax
+; SSE2-NEXT:    movd %rax, %xmm1
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
+; SSE2-NEXT:    movd %xmm0, %rax
+; SSE2-NEXT:    bsfq %rax, %rax
+; SSE2-NEXT:    cmoveq %rcx, %rax
+; SSE2-NEXT:    movd %rax, %xmm0
+; SSE2-NEXT:    punpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm0[0]
+; SSE2-NEXT:    movdqa %xmm1, %xmm0
+; SSE2-NEXT:    retq
+;
+; SSE3-LABEL: testv2i64:
+; SSE3:       # BB#0:
+; SSE3-NEXT:    movd %xmm0, %rax
+; SSE3-NEXT:    bsfq %rax, %rax
+; SSE3-NEXT:    movl $64, %ecx
+; SSE3-NEXT:    cmoveq %rcx, %rax
+; SSE3-NEXT:    movd %rax, %xmm1
+; SSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
+; SSE3-NEXT:    movd %xmm0, %rax
+; SSE3-NEXT:    bsfq %rax, %rax
+; SSE3-NEXT:    cmoveq %rcx, %rax
+; SSE3-NEXT:    movd %rax, %xmm0
+; SSE3-NEXT:    punpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm0[0]
+; SSE3-NEXT:    movdqa %xmm1, %xmm0
+; SSE3-NEXT:    retq
+;
+; SSSE3-LABEL: testv2i64:
+; SSSE3:       # BB#0:
+; SSSE3-NEXT:    movd %xmm0, %rax
+; SSSE3-NEXT:    bsfq %rax, %rax
+; SSSE3-NEXT:    movl $64, %ecx
+; SSSE3-NEXT:    cmoveq %rcx, %rax
+; SSSE3-NEXT:    movd %rax, %xmm1
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
+; SSSE3-NEXT:    movd %xmm0, %rax
+; SSSE3-NEXT:    bsfq %rax, %rax
+; SSSE3-NEXT:    cmoveq %rcx, %rax
+; SSSE3-NEXT:    movd %rax, %xmm0
+; SSSE3-NEXT:    punpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm0[0]
+; SSSE3-NEXT:    movdqa %xmm1, %xmm0
+; SSSE3-NEXT:    retq
+;
+; SSE41-LABEL: testv2i64:
+; SSE41:       # BB#0:
+; SSE41-NEXT:    pextrq $1, %xmm0, %rax
+; SSE41-NEXT:    bsfq %rax, %rax
+; SSE41-NEXT:    movl $64, %ecx
+; SSE41-NEXT:    cmoveq %rcx, %rax
+; SSE41-NEXT:    movd %rax, %xmm1
+; SSE41-NEXT:    movd %xmm0, %rax
+; SSE41-NEXT:    bsfq %rax, %rax
+; SSE41-NEXT:    cmoveq %rcx, %rax
+; SSE41-NEXT:    movd %rax, %xmm0
+; SSE41-NEXT:    punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
+; SSE41-NEXT:    retq
+;
+; AVX-LABEL: testv2i64:
+; AVX:       # BB#0:
+; AVX-NEXT:    vpextrq $1, %xmm0, %rax
+; AVX-NEXT:    bsfq %rax, %rax
+; AVX-NEXT:    movl $64, %ecx
+; AVX-NEXT:    cmoveq %rcx, %rax
+; AVX-NEXT:    vmovq %rax, %xmm1
+; AVX-NEXT:    vmovq %xmm0, %rax
+; AVX-NEXT:    bsfq %rax, %rax
+; AVX-NEXT:    cmoveq %rcx, %rax
+; AVX-NEXT:    vmovq %rax, %xmm0
+; AVX-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
+; AVX-NEXT:    retq
+  %out = call <2 x i64> @llvm.cttz.v2i64(<2 x i64> %in, i1 0)
+  ret <2 x i64> %out
+}
+
+define <2 x i64> @testv2i64u(<2 x i64> %in) nounwind {
+; SSE2-LABEL: testv2i64u:
+; SSE2:       # BB#0:
+; SSE2-NEXT:    movd %xmm0, %rax
+; SSE2-NEXT:    bsfq %rax, %rax
+; SSE2-NEXT:    movd %rax, %xmm1
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
+; SSE2-NEXT:    movd %xmm0, %rax
+; SSE2-NEXT:    bsfq %rax, %rax
+; SSE2-NEXT:    movd %rax, %xmm0
+; SSE2-NEXT:    punpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm0[0]
+; SSE2-NEXT:    movdqa %xmm1, %xmm0
+; SSE2-NEXT:    retq
+;
+; SSE3-LABEL: testv2i64u:
+; SSE3:       # BB#0:
+; SSE3-NEXT:    movd %xmm0, %rax
+; SSE3-NEXT:    bsfq %rax, %rax
+; SSE3-NEXT:    movd %rax, %xmm1
+; SSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
+; SSE3-NEXT:    movd %xmm0, %rax
+; SSE3-NEXT:    bsfq %rax, %rax
+; SSE3-NEXT:    movd %rax, %xmm0
+; SSE3-NEXT:    punpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm0[0]
+; SSE3-NEXT:    movdqa %xmm1, %xmm0
+; SSE3-NEXT:    retq
+;
+; SSSE3-LABEL: testv2i64u:
+; SSSE3:       # BB#0:
+; SSSE3-NEXT:    movd %xmm0, %rax
+; SSSE3-NEXT:    bsfq %rax, %rax
+; SSSE3-NEXT:    movd %rax, %xmm1
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
+; SSSE3-NEXT:    movd %xmm0, %rax
+; SSSE3-NEXT:    bsfq %rax, %rax
+; SSSE3-NEXT:    movd %rax, %xmm0
+; SSSE3-NEXT:    punpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm0[0]
+; SSSE3-NEXT:    movdqa %xmm1, %xmm0
+; SSSE3-NEXT:    retq
+;
+; SSE41-LABEL: testv2i64u:
+; SSE41:       # BB#0:
+; SSE41-NEXT:    pextrq $1, %xmm0, %rax
+; SSE41-NEXT:    bsfq %rax, %rax
+; SSE41-NEXT:    movd %rax, %xmm1
+; SSE41-NEXT:    movd %xmm0, %rax
+; SSE41-NEXT:    bsfq %rax, %rax
+; SSE41-NEXT:    movd %rax, %xmm0
+; SSE41-NEXT:    punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
+; SSE41-NEXT:    retq
+;
+; AVX-LABEL: testv2i64u:
+; AVX:       # BB#0:
+; AVX-NEXT:    vpextrq $1, %xmm0, %rax
+; AVX-NEXT:    bsfq %rax, %rax
+; AVX-NEXT:    vmovq %rax, %xmm1
+; AVX-NEXT:    vmovq %xmm0, %rax
+; AVX-NEXT:    bsfq %rax, %rax
+; AVX-NEXT:    vmovq %rax, %xmm0
+; AVX-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
+; AVX-NEXT:    retq
+  %out = call <2 x i64> @llvm.cttz.v2i64(<2 x i64> %in, i1 -1)
+  ret <2 x i64> %out
+}
+
+define <4 x i32> @testv4i32(<4 x i32> %in) nounwind {
+; SSE2-LABEL: testv4i32:
+; SSE2:       # BB#0:
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[3,1,2,3]
+; SSE2-NEXT:    movd %xmm1, %eax
+; SSE2-NEXT:    bsfl %eax, %eax
+; SSE2-NEXT:    movl $32, %ecx
+; SSE2-NEXT:    cmovel %ecx, %eax
+; SSE2-NEXT:    movd %eax, %xmm1
+; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[1,1,2,3]
+; SSE2-NEXT:    movd %xmm2, %eax
+; SSE2-NEXT:    bsfl %eax, %eax
+; SSE2-NEXT:    cmovel %ecx, %eax
+; SSE2-NEXT:    movd %eax, %xmm2
+; SSE2-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]
+; SSE2-NEXT:    movd %xmm0, %eax
+; SSE2-NEXT:    bsfl %eax, %eax
+; SSE2-NEXT:    cmovel %ecx, %eax
+; SSE2-NEXT:    movd %eax, %xmm1
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
+; SSE2-NEXT:    movd %xmm0, %eax
+; SSE2-NEXT:    bsfl %eax, %eax
+; SSE2-NEXT:    cmovel %ecx, %eax
+; SSE2-NEXT:    movd %eax, %xmm0
+; SSE2-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
+; SSE2-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1]
+; SSE2-NEXT:    movdqa %xmm1, %xmm0
+; SSE2-NEXT:    retq
+;
+; SSE3-LABEL: testv4i32:
+; SSE3:       # BB#0:
+; SSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[3,1,2,3]
+; SSE3-NEXT:    movd %xmm1, %eax
+; SSE3-NEXT:    bsfl %eax, %eax
+; SSE3-NEXT:    movl $32, %ecx
+; SSE3-NEXT:    cmovel %ecx, %eax
+; SSE3-NEXT:    movd %eax, %xmm1
+; SSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[1,1,2,3]
+; SSE3-NEXT:    movd %xmm2, %eax
+; SSE3-NEXT:    bsfl %eax, %eax
+; SSE3-NEXT:    cmovel %ecx, %eax
+; SSE3-NEXT:    movd %eax, %xmm2
+; SSE3-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]
+; SSE3-NEXT:    movd %xmm0, %eax
+; SSE3-NEXT:    bsfl %eax, %eax
+; SSE3-NEXT:    cmovel %ecx, %eax
+; SSE3-NEXT:    movd %eax, %xmm1
+; SSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
+; SSE3-NEXT:    movd %xmm0, %eax
+; SSE3-NEXT:    bsfl %eax, %eax
+; SSE3-NEXT:    cmovel %ecx, %eax
+; SSE3-NEXT:    movd %eax, %xmm0
+; SSE3-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
+; SSE3-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1]
+; SSE3-NEXT:    movdqa %xmm1, %xmm0
+; SSE3-NEXT:    retq
+;
+; SSSE3-LABEL: testv4i32:
+; SSSE3:       # BB#0:
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[3,1,2,3]
+; SSSE3-NEXT:    movd %xmm1, %eax
+; SSSE3-NEXT:    bsfl %eax, %eax
+; SSSE3-NEXT:    movl $32, %ecx
+; SSSE3-NEXT:    cmovel %ecx, %eax
+; SSSE3-NEXT:    movd %eax, %xmm1
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[1,1,2,3]
+; SSSE3-NEXT:    movd %xmm2, %eax
+; SSSE3-NEXT:    bsfl %eax, %eax
+; SSSE3-NEXT:    cmovel %ecx, %eax
+; SSSE3-NEXT:    movd %eax, %xmm2
+; SSSE3-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]
+; SSSE3-NEXT:    movd %xmm0, %eax
+; SSSE3-NEXT:    bsfl %eax, %eax
+; SSSE3-NEXT:    cmovel %ecx, %eax
+; SSSE3-NEXT:    movd %eax, %xmm1
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
+; SSSE3-NEXT:    movd %xmm0, %eax
+; SSSE3-NEXT:    bsfl %eax, %eax
+; SSSE3-NEXT:    cmovel %ecx, %eax
+; SSSE3-NEXT:    movd %eax, %xmm0
+; SSSE3-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
+; SSSE3-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1]
+; SSSE3-NEXT:    movdqa %xmm1, %xmm0
+; SSSE3-NEXT:    retq
+;
+; SSE41-LABEL: testv4i32:
+; SSE41:       # BB#0:
+; SSE41-NEXT:    pextrd $1, %xmm0, %eax
+; SSE41-NEXT:    bsfl %eax, %eax
+; SSE41-NEXT:    movl $32, %ecx
+; SSE41-NEXT:    cmovel %ecx, %eax
+; SSE41-NEXT:    movd %xmm0, %edx
+; SSE41-NEXT:    bsfl %edx, %edx
+; SSE41-NEXT:    cmovel %ecx, %edx
+; SSE41-NEXT:    movd %edx, %xmm1
+; SSE41-NEXT:    pinsrd $1, %eax, %xmm1
+; SSE41-NEXT:    pextrd $2, %xmm0, %eax
+; SSE41-NEXT:    bsfl %eax, %eax
+; SSE41-NEXT:    cmovel %ecx, %eax
+; SSE41-NEXT:    pinsrd $2, %eax, %xmm1
+; SSE41-NEXT:    pextrd $3, %xmm0, %eax
+; SSE41-NEXT:    bsfl %eax, %eax
+; SSE41-NEXT:    cmovel %ecx, %eax
+; SSE41-NEXT:    pinsrd $3, %eax, %xmm1
+; SSE41-NEXT:    movdqa %xmm1, %xmm0
+; SSE41-NEXT:    retq
+;
+; AVX-LABEL: testv4i32:
+; AVX:       # BB#0:
+; AVX-NEXT:    vpextrd $1, %xmm0, %eax
+; AVX-NEXT:    bsfl %eax, %eax
+; AVX-NEXT:    movl $32, %ecx
+; AVX-NEXT:    cmovel %ecx, %eax
+; AVX-NEXT:    vmovd %xmm0, %edx
+; AVX-NEXT:    bsfl %edx, %edx
+; AVX-NEXT:    cmovel %ecx, %edx
+; AVX-NEXT:    vmovd %edx, %xmm1
+; AVX-NEXT:    vpinsrd $1, %eax, %xmm1, %xmm1
+; AVX-NEXT:    vpextrd $2, %xmm0, %eax
+; AVX-NEXT:    bsfl %eax, %eax
+; AVX-NEXT:    cmovel %ecx, %eax
+; AVX-NEXT:    vpinsrd $2, %eax, %xmm1, %xmm1
+; AVX-NEXT:    vpextrd $3, %xmm0, %eax
+; AVX-NEXT:    bsfl %eax, %eax
+; AVX-NEXT:    cmovel %ecx, %eax
+; AVX-NEXT:    vpinsrd $3, %eax, %xmm1, %xmm0
+; AVX-NEXT:    retq
+  %out = call <4 x i32> @llvm.cttz.v4i32(<4 x i32> %in, i1 0)
+  ret <4 x i32> %out
+}
+
+define <4 x i32> @testv4i32u(<4 x i32> %in) nounwind {
+; SSE2-LABEL: testv4i32u:
+; SSE2:       # BB#0:
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[3,1,2,3]
+; SSE2-NEXT:    movd %xmm1, %eax
+; SSE2-NEXT:    bsfl %eax, %eax
+; SSE2-NEXT:    movd %eax, %xmm1
+; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[1,1,2,3]
+; SSE2-NEXT:    movd %xmm2, %eax
+; SSE2-NEXT:    bsfl %eax, %eax
+; SSE2-NEXT:    movd %eax, %xmm2
+; SSE2-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]
+; SSE2-NEXT:    movd %xmm0, %eax
+; SSE2-NEXT:    bsfl %eax, %eax
+; SSE2-NEXT:    movd %eax, %xmm1
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
+; SSE2-NEXT:    movd %xmm0, %eax
+; SSE2-NEXT:    bsfl %eax, %eax
+; SSE2-NEXT:    movd %eax, %xmm0
+; SSE2-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
+; SSE2-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1]
+; SSE2-NEXT:    movdqa %xmm1, %xmm0
+; SSE2-NEXT:    retq
+;
+; SSE3-LABEL: testv4i32u:
+; SSE3:       # BB#0:
+; SSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[3,1,2,3]
+; SSE3-NEXT:    movd %xmm1, %eax
+; SSE3-NEXT:    bsfl %eax, %eax
+; SSE3-NEXT:    movd %eax, %xmm1
+; SSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[1,1,2,3]
+; SSE3-NEXT:    movd %xmm2, %eax
+; SSE3-NEXT:    bsfl %eax, %eax
+; SSE3-NEXT:    movd %eax, %xmm2
+; SSE3-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]
+; SSE3-NEXT:    movd %xmm0, %eax
+; SSE3-NEXT:    bsfl %eax, %eax
+; SSE3-NEXT:    movd %eax, %xmm1
+; SSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
+; SSE3-NEXT:    movd %xmm0, %eax
+; SSE3-NEXT:    bsfl %eax, %eax
+; SSE3-NEXT:    movd %eax, %xmm0
+; SSE3-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
+; SSE3-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1]
+; SSE3-NEXT:    movdqa %xmm1, %xmm0
+; SSE3-NEXT:    retq
+;
+; SSSE3-LABEL: testv4i32u:
+; SSSE3:       # BB#0:
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[3,1,2,3]
+; SSSE3-NEXT:    movd %xmm1, %eax
+; SSSE3-NEXT:    bsfl %eax, %eax
+; SSSE3-NEXT:    movd %eax, %xmm1
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[1,1,2,3]
+; SSSE3-NEXT:    movd %xmm2, %eax
+; SSSE3-NEXT:    bsfl %eax, %eax
+; SSSE3-NEXT:    movd %eax, %xmm2
+; SSSE3-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]
+; SSSE3-NEXT:    movd %xmm0, %eax
+; SSSE3-NEXT:    bsfl %eax, %eax
+; SSSE3-NEXT:    movd %eax, %xmm1
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
+; SSSE3-NEXT:    movd %xmm0, %eax
+; SSSE3-NEXT:    bsfl %eax, %eax
+; SSSE3-NEXT:    movd %eax, %xmm0
+; SSSE3-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
+; SSSE3-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1]
+; SSSE3-NEXT:    movdqa %xmm1, %xmm0
+; SSSE3-NEXT:    retq
+;
+; SSE41-LABEL: testv4i32u:
+; SSE41:       # BB#0:
+; SSE41-NEXT:    pextrd $1, %xmm0, %eax
+; SSE41-NEXT:    bsfl %eax, %eax
+; SSE41-NEXT:    movd %xmm0, %ecx
+; SSE41-NEXT:    bsfl %ecx, %ecx
+; SSE41-NEXT:    movd %ecx, %xmm1
+; SSE41-NEXT:    pinsrd $1, %eax, %xmm1
+; SSE41-NEXT:    pextrd $2, %xmm0, %eax
+; SSE41-NEXT:    bsfl %eax, %eax
+; SSE41-NEXT:    pinsrd $2, %eax, %xmm1
+; SSE41-NEXT:    pextrd $3, %xmm0, %eax
+; SSE41-NEXT:    bsfl %eax, %eax
+; SSE41-NEXT:    pinsrd $3, %eax, %xmm1
+; SSE41-NEXT:    movdqa %xmm1, %xmm0
+; SSE41-NEXT:    retq
+;
+; AVX-LABEL: testv4i32u:
+; AVX:       # BB#0:
+; AVX-NEXT:    vpextrd $1, %xmm0, %eax
+; AVX-NEXT:    bsfl %eax, %eax
+; AVX-NEXT:    vmovd %xmm0, %ecx
+; AVX-NEXT:    bsfl %ecx, %ecx
+; AVX-NEXT:    vmovd %ecx, %xmm1
+; AVX-NEXT:    vpinsrd $1, %eax, %xmm1, %xmm1
+; AVX-NEXT:    vpextrd $2, %xmm0, %eax
+; AVX-NEXT:    bsfl %eax, %eax
+; AVX-NEXT:    vpinsrd $2, %eax, %xmm1, %xmm1
+; AVX-NEXT:    vpextrd $3, %xmm0, %eax
+; AVX-NEXT:    bsfl %eax, %eax
+; AVX-NEXT:    vpinsrd $3, %eax, %xmm1, %xmm0
+; AVX-NEXT:    retq
+  %out = call <4 x i32> @llvm.cttz.v4i32(<4 x i32> %in, i1 -1)
+  ret <4 x i32> %out
+}
+
+define <8 x i16> @testv8i16(<8 x i16> %in) nounwind {
+; SSE2-LABEL: testv8i16:
+; SSE2:       # BB#0:
+; SSE2-NEXT:    pextrw $7, %xmm0, %eax
+; SSE2-NEXT:    bsfw %ax, %cx
+; SSE2-NEXT:    movw $16, %ax
+; SSE2-NEXT:    cmovew %ax, %cx
+; SSE2-NEXT:    movd %ecx, %xmm1
+; SSE2-NEXT:    pextrw $3, %xmm0, %ecx
+; SSE2-NEXT:    bsfw %cx, %cx
+; SSE2-NEXT:    cmovew %ax, %cx
+; SSE2-NEXT:    movd %ecx, %xmm2
+; SSE2-NEXT:    punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3]
+; SSE2-NEXT:    pextrw $5, %xmm0, %ecx
+; SSE2-NEXT:    bsfw %cx, %cx
+; SSE2-NEXT:    cmovew %ax, %cx
+; SSE2-NEXT:    movd %ecx, %xmm3
+; SSE2-NEXT:    pextrw $1, %xmm0, %ecx
+; SSE2-NEXT:    bsfw %cx, %cx
+; SSE2-NEXT:    cmovew %ax, %cx
+; SSE2-NEXT:    movd %ecx, %xmm1
+; SSE2-NEXT:    punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm3[0],xmm1[1],xmm3[1],xmm1[2],xmm3[2],xmm1[3],xmm3[3]
+; SSE2-NEXT:    punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1],xmm1[2],xmm2[2],xmm1[3],xmm2[3]
+; SSE2-NEXT:    pextrw $6, %xmm0, %ecx
+; SSE2-NEXT:    bsfw %cx, %cx
+; SSE2-NEXT:    cmovew %ax, %cx
+; SSE2-NEXT:    movd %ecx, %xmm2
+; SSE2-NEXT:    pextrw $2, %xmm0, %ecx
+; SSE2-NEXT:    bsfw %cx, %cx
+; SSE2-NEXT:    cmovew %ax, %cx
+; SSE2-NEXT:    movd %ecx, %xmm3
+; SSE2-NEXT:    punpcklwd {{.*#+}} xmm3 = xmm3[0],xmm2[0],xmm3[1],xmm2[1],xmm3[2],xmm2[2],xmm3[3],xmm2[3]
+; SSE2-NEXT:    pextrw $4, %xmm0, %ecx
+; SSE2-NEXT:    bsfw %cx, %cx
+; SSE2-NEXT:    cmovew %ax, %cx
+; SSE2-NEXT:    movd %ecx, %xmm2
+; SSE2-NEXT:    movd %xmm0, %ecx
+; SSE2-NEXT:    bsfw %cx, %cx
+; SSE2-NEXT:    cmovew %ax, %cx
+; SSE2-NEXT:    movd %ecx, %xmm0
+; SSE2-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1],xmm0[2],xmm2[2],xmm0[3],xmm2[3]
+; SSE2-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1],xmm0[2],xmm3[2],xmm0[3],xmm3[3]
+; SSE2-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
+; SSE2-NEXT:    retq
+;
+; SSE3-LABEL: testv8i16:
+; SSE3:       # BB#0:
+; SSE3-NEXT:    pextrw $7, %xmm0, %eax
+; SSE3-NEXT:    bsfw %ax, %cx
+; SSE3-NEXT:    movw $16, %ax
+; SSE3-NEXT:    cmovew %ax, %cx
+; SSE3-NEXT:    movd %ecx, %xmm1
+; SSE3-NEXT:    pextrw $3, %xmm0, %ecx
+; SSE3-NEXT:    bsfw %cx, %cx
+; SSE3-NEXT:    cmovew %ax, %cx
+; SSE3-NEXT:    movd %ecx, %xmm2
+; SSE3-NEXT:    punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3]
+; SSE3-NEXT:    pextrw $5, %xmm0, %ecx
+; SSE3-NEXT:    bsfw %cx, %cx
+; SSE3-NEXT:    cmovew %ax, %cx
+; SSE3-NEXT:    movd %ecx, %xmm3
+; SSE3-NEXT:    pextrw $1, %xmm0, %ecx
+; SSE3-NEXT:    bsfw %cx, %cx
+; SSE3-NEXT:    cmovew %ax, %cx
+; SSE3-NEXT:    movd %ecx, %xmm1
+; SSE3-NEXT:    punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm3[0],xmm1[1],xmm3[1],xmm1[2],xmm3[2],xmm1[3],xmm3[3]
+; SSE3-NEXT:    punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1],xmm1[2],xmm2[2],xmm1[3],xmm2[3]
+; SSE3-NEXT:    pextrw $6, %xmm0, %ecx
+; SSE3-NEXT:    bsfw %cx, %cx
+; SSE3-NEXT:    cmovew %ax, %cx
+; SSE3-NEXT:    movd %ecx, %xmm2
+; SSE3-NEXT:    pextrw $2, %xmm0, %ecx
+; SSE3-NEXT:    bsfw %cx, %cx
+; SSE3-NEXT:    cmovew %ax, %cx
+; SSE3-NEXT:    movd %ecx, %xmm3
+; SSE3-NEXT:    punpcklwd {{.*#+}} xmm3 = xmm3[0],xmm2[0],xmm3[1],xmm2[1],xmm3[2],xmm2[2],xmm3[3],xmm2[3]
+; SSE3-NEXT:    pextrw $4, %xmm0, %ecx
+; SSE3-NEXT:    bsfw %cx, %cx
+; SSE3-NEXT:    cmovew %ax, %cx
+; SSE3-NEXT:    movd %ecx, %xmm2
+; SSE3-NEXT:    movd %xmm0, %ecx
+; SSE3-NEXT:    bsfw %cx, %cx
+; SSE3-NEXT:    cmovew %ax, %cx
+; SSE3-NEXT:    movd %ecx, %xmm0
+; SSE3-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1],xmm0[2],xmm2[2],xmm0[3],xmm2[3]
+; SSE3-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1],xmm0[2],xmm3[2],xmm0[3],xmm3[3]
+; SSE3-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
+; SSE3-NEXT:    retq
+;
+; SSSE3-LABEL: testv8i16:
+; SSSE3:       # BB#0:
+; SSSE3-NEXT:    pextrw $7, %xmm0, %eax
+; SSSE3-NEXT:    bsfw %ax, %cx
+; SSSE3-NEXT:    movw $16, %ax
+; SSSE3-NEXT:    cmovew %ax, %cx
+; SSSE3-NEXT:    movd %ecx, %xmm1
+; SSSE3-NEXT:    pextrw $3, %xmm0, %ecx
+; SSSE3-NEXT:    bsfw %cx, %cx
+; SSSE3-NEXT:    cmovew %ax, %cx
+; SSSE3-NEXT:    movd %ecx, %xmm2
+; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3]
+; SSSE3-NEXT:    pextrw $5, %xmm0, %ecx
+; SSSE3-NEXT:    bsfw %cx, %cx
+; SSSE3-NEXT:    cmovew %ax, %cx
+; SSSE3-NEXT:    movd %ecx, %xmm3
+; SSSE3-NEXT:    pextrw $1, %xmm0, %ecx
+; SSSE3-NEXT:    bsfw %cx, %cx
+; SSSE3-NEXT:    cmovew %ax, %cx
+; SSSE3-NEXT:    movd %ecx, %xmm1
+; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm3[0],xmm1[1],xmm3[1],xmm1[2],xmm3[2],xmm1[3],xmm3[3]
+; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1],xmm1[2],xmm2[2],xmm1[3],xmm2[3]
+; SSSE3-NEXT:    pextrw $6, %xmm0, %ecx
+; SSSE3-NEXT:    bsfw %cx, %cx
+; SSSE3-NEXT:    cmovew %ax, %cx
+; SSSE3-NEXT:    movd %ecx, %xmm2
+; SSSE3-NEXT:    pextrw $2, %xmm0, %ecx
+; SSSE3-NEXT:    bsfw %cx, %cx
+; SSSE3-NEXT:    cmovew %ax, %cx
+; SSSE3-NEXT:    movd %ecx, %xmm3
+; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm3 = xmm3[0],xmm2[0],xmm3[1],xmm2[1],xmm3[2],xmm2[2],xmm3[3],xmm2[3]
+; SSSE3-NEXT:    pextrw $4, %xmm0, %ecx
+; SSSE3-NEXT:    bsfw %cx, %cx
+; SSSE3-NEXT:    cmovew %ax, %cx
+; SSSE3-NEXT:    movd %ecx, %xmm2
+; SSSE3-NEXT:    movd %xmm0, %ecx
+; SSSE3-NEXT:    bsfw %cx, %cx
+; SSSE3-NEXT:    cmovew %ax, %cx
+; SSSE3-NEXT:    movd %ecx, %xmm0
+; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1],xmm0[2],xmm2[2],xmm0[3],xmm2[3]
+; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1],xmm0[2],xmm3[2],xmm0[3],xmm3[3]
+; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
+; SSSE3-NEXT:    retq
+;
+; SSE41-LABEL: testv8i16:
+; SSE41:       # BB#0:
+; SSE41-NEXT:    pextrw $1, %xmm0, %eax
+; SSE41-NEXT:    bsfw %ax, %cx
+; SSE41-NEXT:    movw $16, %ax
+; SSE41-NEXT:    cmovew %ax, %cx
+; SSE41-NEXT:    movd %xmm0, %edx
+; SSE41-NEXT:    bsfw %dx, %dx
+; SSE41-NEXT:    cmovew %ax, %dx
+; SSE41-NEXT:    movd %edx, %xmm1
+; SSE41-NEXT:    pinsrw $1, %ecx, %xmm1
+; SSE41-NEXT:    pextrw $2, %xmm0, %ecx
+; SSE41-NEXT:    bsfw %cx, %cx
+; SSE41-NEXT:    cmovew %ax, %cx
+; SSE41-NEXT:    pinsrw $2, %ecx, %xmm1
+; SSE41-NEXT:    pextrw $3, %xmm0, %ecx
+; SSE41-NEXT:    bsfw %cx, %cx
+; SSE41-NEXT:    cmovew %ax, %cx
+; SSE41-NEXT:    pinsrw $3, %ecx, %xmm1
+; SSE41-NEXT:    pextrw $4, %xmm0, %ecx
+; SSE41-NEXT:    bsfw %cx, %cx
+; SSE41-NEXT:    cmovew %ax, %cx
+; SSE41-NEXT:    pinsrw $4, %ecx, %xmm1
+; SSE41-NEXT:    pextrw $5, %xmm0, %ecx
+; SSE41-NEXT:    bsfw %cx, %cx
+; SSE41-NEXT:    cmovew %ax, %cx
+; SSE41-NEXT:    pinsrw $5, %ecx, %xmm1
+; SSE41-NEXT:    pextrw $6, %xmm0, %ecx
+; SSE41-NEXT:    bsfw %cx, %cx
+; SSE41-NEXT:    cmovew %ax, %cx
+; SSE41-NEXT:    pinsrw $6, %ecx, %xmm1
+; SSE41-NEXT:    pextrw $7, %xmm0, %ecx
+; SSE41-NEXT:    bsfw %cx, %cx
+; SSE41-NEXT:    cmovew %ax, %cx
+; SSE41-NEXT:    pinsrw $7, %ecx, %xmm1
+; SSE41-NEXT:    movdqa %xmm1, %xmm0
+; SSE41-NEXT:    retq
+;
+; AVX-LABEL: testv8i16:
+; AVX:       # BB#0:
+; AVX-NEXT:    vpextrw $1, %xmm0, %eax
+; AVX-NEXT:    bsfw %ax, %cx
+; AVX-NEXT:    movw $16, %ax
+; AVX-NEXT:    cmovew %ax, %cx
+; AVX-NEXT:    vmovd %xmm0, %edx
+; AVX-NEXT:    bsfw %dx, %dx
+; AVX-NEXT:    cmovew %ax, %dx
+; AVX-NEXT:    vmovd %edx, %xmm1
+; AVX-NEXT:    vpinsrw $1, %ecx, %xmm1, %xmm1
+; AVX-NEXT:    vpextrw $2, %xmm0, %ecx
+; AVX-NEXT:    bsfw %cx, %cx
+; AVX-NEXT:    cmovew %ax, %cx
+; AVX-NEXT:    vpinsrw $2, %ecx, %xmm1, %xmm1
+; AVX-NEXT:    vpextrw $3, %xmm0, %ecx
+; AVX-NEXT:    bsfw %cx, %cx
+; AVX-NEXT:    cmovew %ax, %cx
+; AVX-NEXT:    vpinsrw $3, %ecx, %xmm1, %xmm1
+; AVX-NEXT:    vpextrw $4, %xmm0, %ecx
+; AVX-NEXT:    bsfw %cx, %cx
+; AVX-NEXT:    cmovew %ax, %cx
+; AVX-NEXT:    vpinsrw $4, %ecx, %xmm1, %xmm1
+; AVX-NEXT:    vpextrw $5, %xmm0, %ecx
+; AVX-NEXT:    bsfw %cx, %cx
+; AVX-NEXT:    cmovew %ax, %cx
+; AVX-NEXT:    vpinsrw $5, %ecx, %xmm1, %xmm1
+; AVX-NEXT:    vpextrw $6, %xmm0, %ecx
+; AVX-NEXT:    bsfw %cx, %cx
+; AVX-NEXT:    cmovew %ax, %cx
+; AVX-NEXT:    vpinsrw $6, %ecx, %xmm1, %xmm1
+; AVX-NEXT:    vpextrw $7, %xmm0, %ecx
+; AVX-NEXT:    bsfw %cx, %cx
+; AVX-NEXT:    cmovew %ax, %cx
+; AVX-NEXT:    vpinsrw $7, %ecx, %xmm1, %xmm0
+; AVX-NEXT:    retq
+  %out = call <8 x i16> @llvm.cttz.v8i16(<8 x i16> %in, i1 0)
+  ret <8 x i16> %out
+}
+
+define <8 x i16> @testv8i16u(<8 x i16> %in) nounwind {
+; SSE2-LABEL: testv8i16u:
+; SSE2:       # BB#0:
+; SSE2-NEXT:    pextrw $7, %xmm0, %eax
+; SSE2-NEXT:    bsfw %ax, %ax
+; SSE2-NEXT:    movd %eax, %xmm1
+; SSE2-NEXT:    pextrw $3, %xmm0, %eax
+; SSE2-NEXT:    bsfw %ax, %ax
+; SSE2-NEXT:    movd %eax, %xmm2
+; SSE2-NEXT:    punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3]
+; SSE2-NEXT:    pextrw $5, %xmm0, %eax
+; SSE2-NEXT:    bsfw %ax, %ax
+; SSE2-NEXT:    movd %eax, %xmm1
+; SSE2-NEXT:    pextrw $1, %xmm0, %eax
+; SSE2-NEXT:    bsfw %ax, %ax
+; SSE2-NEXT:    movd %eax, %xmm3
+; SSE2-NEXT:    punpcklwd {{.*#+}} xmm3 = xmm3[0],xmm1[0],xmm3[1],xmm1[1],xmm3[2],xmm1[2],xmm3[3],xmm1[3]
+; SSE2-NEXT:    punpcklwd {{.*#+}} xmm3 = xmm3[0],xmm2[0],xmm3[1],xmm2[1],xmm3[2],xmm2[2],xmm3[3],xmm2[3]
+; SSE2-NEXT:    pextrw $6, %xmm0, %eax
+; SSE2-NEXT:    bsfw %ax, %ax
+; SSE2-NEXT:    movd %eax, %xmm1
+; SSE2-NEXT:    pextrw $2, %xmm0, %eax
+; SSE2-NEXT:    bsfw %ax, %ax
+; SSE2-NEXT:    movd %eax, %xmm2
+; SSE2-NEXT:    punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3]
+; SSE2-NEXT:    pextrw $4, %xmm0, %eax
+; SSE2-NEXT:    bsfw %ax, %ax
+; SSE2-NEXT:    movd %eax, %xmm1
+; SSE2-NEXT:    movd %xmm0, %eax
+; SSE2-NEXT:    bsfw %ax, %ax
+; SSE2-NEXT:    movd %eax, %xmm0
+; SSE2-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
+; SSE2-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1],xmm0[2],xmm2[2],xmm0[3],xmm2[3]
+; SSE2-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1],xmm0[2],xmm3[2],xmm0[3],xmm3[3]
+; SSE2-NEXT:    retq
+;
+; SSE3-LABEL: testv8i16u:
+; SSE3:       # BB#0:
+; SSE3-NEXT:    pextrw $7, %xmm0, %eax
+; SSE3-NEXT:    bsfw %ax, %ax
+; SSE3-NEXT:    movd %eax, %xmm1
+; SSE3-NEXT:    pextrw $3, %xmm0, %eax
+; SSE3-NEXT:    bsfw %ax, %ax
+; SSE3-NEXT:    movd %eax, %xmm2
+; SSE3-NEXT:    punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3]
+; SSE3-NEXT:    pextrw $5, %xmm0, %eax
+; SSE3-NEXT:    bsfw %ax, %ax
+; SSE3-NEXT:    movd %eax, %xmm1
+; SSE3-NEXT:    pextrw $1, %xmm0, %eax
+; SSE3-NEXT:    bsfw %ax, %ax
+; SSE3-NEXT:    movd %eax, %xmm3
+; SSE3-NEXT:    punpcklwd {{.*#+}} xmm3 = xmm3[0],xmm1[0],xmm3[1],xmm1[1],xmm3[2],xmm1[2],xmm3[3],xmm1[3]
+; SSE3-NEXT:    punpcklwd {{.*#+}} xmm3 = xmm3[0],xmm2[0],xmm3[1],xmm2[1],xmm3[2],xmm2[2],xmm3[3],xmm2[3]
+; SSE3-NEXT:    pextrw $6, %xmm0, %eax
+; SSE3-NEXT:    bsfw %ax, %ax
+; SSE3-NEXT:    movd %eax, %xmm1
+; SSE3-NEXT:    pextrw $2, %xmm0, %eax
+; SSE3-NEXT:    bsfw %ax, %ax
+; SSE3-NEXT:    movd %eax, %xmm2
+; SSE3-NEXT:    punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3]
+; SSE3-NEXT:    pextrw $4, %xmm0, %eax
+; SSE3-NEXT:    bsfw %ax, %ax
+; SSE3-NEXT:    movd %eax, %xmm1
+; SSE3-NEXT:    movd %xmm0, %eax
+; SSE3-NEXT:    bsfw %ax, %ax
+; SSE3-NEXT:    movd %eax, %xmm0
+; SSE3-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
+; SSE3-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1],xmm0[2],xmm2[2],xmm0[3],xmm2[3]
+; SSE3-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1],xmm0[2],xmm3[2],xmm0[3],xmm3[3]
+; SSE3-NEXT:    retq
+;
+; SSSE3-LABEL: testv8i16u:
+; SSSE3:       # BB#0:
+; SSSE3-NEXT:    pextrw $7, %xmm0, %eax
+; SSSE3-NEXT:    bsfw %ax, %ax
+; SSSE3-NEXT:    movd %eax, %xmm1
+; SSSE3-NEXT:    pextrw $3, %xmm0, %eax
+; SSSE3-NEXT:    bsfw %ax, %ax
+; SSSE3-NEXT:    movd %eax, %xmm2
+; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3]
+; SSSE3-NEXT:    pextrw $5, %xmm0, %eax
+; SSSE3-NEXT:    bsfw %ax, %ax
+; SSSE3-NEXT:    movd %eax, %xmm1
+; SSSE3-NEXT:    pextrw $1, %xmm0, %eax
+; SSSE3-NEXT:    bsfw %ax, %ax
+; SSSE3-NEXT:    movd %eax, %xmm3
+; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm3 = xmm3[0],xmm1[0],xmm3[1],xmm1[1],xmm3[2],xmm1[2],xmm3[3],xmm1[3]
+; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm3 = xmm3[0],xmm2[0],xmm3[1],xmm2[1],xmm3[2],xmm2[2],xmm3[3],xmm2[3]
+; SSSE3-NEXT:    pextrw $6, %xmm0, %eax
+; SSSE3-NEXT:    bsfw %ax, %ax
+; SSSE3-NEXT:    movd %eax, %xmm1
+; SSSE3-NEXT:    pextrw $2, %xmm0, %eax
+; SSSE3-NEXT:    bsfw %ax, %ax
+; SSSE3-NEXT:    movd %eax, %xmm2
+; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3]
+; SSSE3-NEXT:    pextrw $4, %xmm0, %eax
+; SSSE3-NEXT:    bsfw %ax, %ax
+; SSSE3-NEXT:    movd %eax, %xmm1
+; SSSE3-NEXT:    movd %xmm0, %eax
+; SSSE3-NEXT:    bsfw %ax, %ax
+; SSSE3-NEXT:    movd %eax, %xmm0
+; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
+; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1],xmm0[2],xmm2[2],xmm0[3],xmm2[3]
+; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1],xmm0[2],xmm3[2],xmm0[3],xmm3[3]
+; SSSE3-NEXT:    retq
+;
+; SSE41-LABEL: testv8i16u:
+; SSE41:       # BB#0:
+; SSE41-NEXT:    pextrw $1, %xmm0, %eax
+; SSE41-NEXT:    bsfw %ax, %ax
+; SSE41-NEXT:    movd %xmm0, %ecx
+; SSE41-NEXT:    bsfw %cx, %cx
+; SSE41-NEXT:    movd %ecx, %xmm1
+; SSE41-NEXT:    pinsrw $1, %eax, %xmm1
+; SSE41-NEXT:    pextrw $2, %xmm0, %eax
+; SSE41-NEXT:    bsfw %ax, %ax
+; SSE41-NEXT:    pinsrw $2, %eax, %xmm1
+; SSE41-NEXT:    pextrw $3, %xmm0, %eax
+; SSE41-NEXT:    bsfw %ax, %ax
+; SSE41-NEXT:    pinsrw $3, %eax, %xmm1
+; SSE41-NEXT:    pextrw $4, %xmm0, %eax
+; SSE41-NEXT:    bsfw %ax, %ax
+; SSE41-NEXT:    pinsrw $4, %eax, %xmm1
+; SSE41-NEXT:    pextrw $5, %xmm0, %eax
+; SSE41-NEXT:    bsfw %ax, %ax
+; SSE41-NEXT:    pinsrw $5, %eax, %xmm1
+; SSE41-NEXT:    pextrw $6, %xmm0, %eax
+; SSE41-NEXT:    bsfw %ax, %ax
+; SSE41-NEXT:    pinsrw $6, %eax, %xmm1
+; SSE41-NEXT:    pextrw $7, %xmm0, %eax
+; SSE41-NEXT:    bsfw %ax, %ax
+; SSE41-NEXT:    pinsrw $7, %eax, %xmm1
+; SSE41-NEXT:    movdqa %xmm1, %xmm0
+; SSE41-NEXT:    retq
+;
+; AVX-LABEL: testv8i16u:
+; AVX:       # BB#0:
+; AVX-NEXT:    vpextrw $1, %xmm0, %eax
+; AVX-NEXT:    bsfw %ax, %ax
+; AVX-NEXT:    vmovd %xmm0, %ecx
+; AVX-NEXT:    bsfw %cx, %cx
+; AVX-NEXT:    vmovd %ecx, %xmm1
+; AVX-NEXT:    vpinsrw $1, %eax, %xmm1, %xmm1
+; AVX-NEXT:    vpextrw $2, %xmm0, %eax
+; AVX-NEXT:    bsfw %ax, %ax
+; AVX-NEXT:    vpinsrw $2, %eax, %xmm1, %xmm1
+; AVX-NEXT:    vpextrw $3, %xmm0, %eax
+; AVX-NEXT:    bsfw %ax, %ax
+; AVX-NEXT:    vpinsrw $3, %eax, %xmm1, %xmm1
+; AVX-NEXT:    vpextrw $4, %xmm0, %eax
+; AVX-NEXT:    bsfw %ax, %ax
+; AVX-NEXT:    vpinsrw $4, %eax, %xmm1, %xmm1
+; AVX-NEXT:    vpextrw $5, %xmm0, %eax
+; AVX-NEXT:    bsfw %ax, %ax
+; AVX-NEXT:    vpinsrw $5, %eax, %xmm1, %xmm1
+; AVX-NEXT:    vpextrw $6, %xmm0, %eax
+; AVX-NEXT:    bsfw %ax, %ax
+; AVX-NEXT:    vpinsrw $6, %eax, %xmm1, %xmm1
+; AVX-NEXT:    vpextrw $7, %xmm0, %eax
+; AVX-NEXT:    bsfw %ax, %ax
+; AVX-NEXT:    vpinsrw $7, %eax, %xmm1, %xmm0
+; AVX-NEXT:    retq
+  %out = call <8 x i16> @llvm.cttz.v8i16(<8 x i16> %in, i1 -1)
+  ret <8 x i16> %out
+}
+
+define <16 x i8> @testv16i8(<16 x i8> %in) nounwind {
+; SSE2-LABEL: testv16i8:
+; SSE2:       # BB#0:
+; SSE2-NEXT:    pushq %rbp
+; SSE2-NEXT:    pushq %r14
+; SSE2-NEXT:    pushq %rbx
+; SSE2-NEXT:    movaps %xmm0, -{{[0-9]+}}(%rsp)
+; SSE2-NEXT:    movzbl -{{[0-9]+}}(%rsp), %eax
+; SSE2-NEXT:    bsfl %eax, %edx
+; SSE2-NEXT:    movl $32, %eax
+; SSE2-NEXT:    cmovel %eax, %edx
+; SSE2-NEXT:    cmpl $32, %edx
+; SSE2-NEXT:    movl $8, %ecx
+; SSE2-NEXT:    cmovel %ecx, %edx
+; SSE2-NEXT:    movd %edx, %xmm0
+; SSE2-NEXT:    movzbl -{{[0-9]+}}(%rsp), %r14d
+; SSE2-NEXT:    movzbl -{{[0-9]+}}(%rsp), %ebx
+; SSE2-NEXT:    movzbl -{{[0-9]+}}(%rsp), %r9d
+; SSE2-NEXT:    movzbl -{{[0-9]+}}(%rsp), %edi
+; SSE2-NEXT:    movzbl -{{[0-9]+}}(%rsp), %r11d
+; SSE2-NEXT:    movzbl -{{[0-9]+}}(%rsp), %edx
+; SSE2-NEXT:    movzbl -{{[0-9]+}}(%rsp), %r8d
+; SSE2-NEXT:    movzbl -{{[0-9]+}}(%rsp), %esi
+; SSE2-NEXT:    bsfl %esi, %esi
+; SSE2-NEXT:    cmovel %eax, %esi
+; SSE2-NEXT:    cmpl $32, %esi
+; SSE2-NEXT:    cmovel %ecx, %esi
+; SSE2-NEXT:    movd %esi, %xmm1
+; SSE2-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3],xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
+; SSE2-NEXT:    bsfl %edi, %esi
+; SSE2-NEXT:    cmovel %eax, %esi
+; SSE2-NEXT:    cmpl $32, %esi
+; SSE2-NEXT:    cmovel %ecx, %esi
+; SSE2-NEXT:    movd %esi, %xmm2
+; SSE2-NEXT:    movzbl -{{[0-9]+}}(%rsp), %edi
+; SSE2-NEXT:    movzbl -{{[0-9]+}}(%rsp), %esi
+; SSE2-NEXT:    movzbl -{{[0-9]+}}(%rsp), %r10d
+; SSE2-NEXT:    movzbl -{{[0-9]+}}(%rsp), %ebp
+; SSE2-NEXT:    bsfl %ebp, %ebp
+; SSE2-NEXT:    cmovel %eax, %ebp
+; SSE2-NEXT:    cmpl $32, %ebp
+; SSE2-NEXT:    cmovel %ecx, %ebp
+; SSE2-NEXT:    movd %ebp, %xmm0
+; SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1],xmm0[2],xmm2[2],xmm0[3],xmm2[3],xmm0[4],xmm2[4],xmm0[5],xmm2[5],xmm0[6],xmm2[6],xmm0[7],xmm2[7]
+; SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3],xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7]
+; SSE2-NEXT:    bsfl %ebx, %ebx
+; SSE2-NEXT:    cmovel %eax, %ebx
+; SSE2-NEXT:    cmpl $32, %ebx
+; SSE2-NEXT:    cmovel %ecx, %ebx
+; SSE2-NEXT:    movd %ebx, %xmm1
+; SSE2-NEXT:    bsfl %esi, %esi
+; SSE2-NEXT:    cmovel %eax, %esi
+; SSE2-NEXT:    cmpl $32, %esi
+; SSE2-NEXT:    cmovel %ecx, %esi
+; SSE2-NEXT:    movd %esi, %xmm2
+; SSE2-NEXT:    punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3],xmm2[4],xmm1[4],xmm2[5],xmm1[5],xmm2[6],xmm1[6],xmm2[7],xmm1[7]
+; SSE2-NEXT:    bsfl %edx, %edx
+; SSE2-NEXT:    cmovel %eax, %edx
+; SSE2-NEXT:    cmpl $32, %edx
+; SSE2-NEXT:    cmovel %ecx, %edx
+; SSE2-NEXT:    movd %edx, %xmm3
+; SSE2-NEXT:    movzbl -{{[0-9]+}}(%rsp), %edx
+; SSE2-NEXT:    movzbl -{{[0-9]+}}(%rsp), %esi
+; SSE2-NEXT:    bsfl %esi, %esi
+; SSE2-NEXT:    cmovel %eax, %esi
+; SSE2-NEXT:    cmpl $32, %esi
+; SSE2-NEXT:    cmovel %ecx, %esi
+; SSE2-NEXT:    movd %esi, %xmm1
+; SSE2-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm3[0],xmm1[1],xmm3[1],xmm1[2],xmm3[2],xmm1[3],xmm3[3],xmm1[4],xmm3[4],xmm1[5],xmm3[5],xmm1[6],xmm3[6],xmm1[7],xmm3[7]
+; SSE2-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1],xmm1[2],xmm2[2],xmm1[3],xmm2[3],xmm1[4],xmm2[4],xmm1[5],xmm2[5],xmm1[6],xmm2[6],xmm1[7],xmm2[7]
+; SSE2-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3],xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
+; SSE2-NEXT:    bsfl %r14d, %esi
+; SSE2-NEXT:    cmovel %eax, %esi
+; SSE2-NEXT:    cmpl $32, %esi
+; SSE2-NEXT:    cmovel %ecx, %esi
+; SSE2-NEXT:    movd %esi, %xmm0
+; SSE2-NEXT:    bsfl %edi, %esi
+; SSE2-NEXT:    cmovel %eax, %esi
+; SSE2-NEXT:    cmpl $32, %esi
+; SSE2-NEXT:    cmovel %ecx, %esi
+; SSE2-NEXT:    movd %esi, %xmm3
+; SSE2-NEXT:    punpcklbw {{.*#+}} xmm3 = xmm3[0],xmm0[0],xmm3[1],xmm0[1],xmm3[2],xmm0[2],xmm3[3],xmm0[3],xmm3[4],xmm0[4],xmm3[5],xmm0[5],xmm3[6],xmm0[6],xmm3[7],xmm0[7]
+; SSE2-NEXT:    bsfl %r11d, %esi
+; SSE2-NEXT:    cmovel %eax, %esi
+; SSE2-NEXT:    cmpl $32, %esi
+; SSE2-NEXT:    cmovel %ecx, %esi
+; SSE2-NEXT:    movd %esi, %xmm0
+; SSE2-NEXT:    bsfl %edx, %edx
+; SSE2-NEXT:    cmovel %eax, %edx
+; SSE2-NEXT:    cmpl $32, %edx
+; SSE2-NEXT:    cmovel %ecx, %edx
+; SSE2-NEXT:    movd %edx, %xmm2
+; SSE2-NEXT:    punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3],xmm2[4],xmm0[4],xmm2[5],xmm0[5],xmm2[6],xmm0[6],xmm2[7],xmm0[7]
+; SSE2-NEXT:    punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1],xmm2[2],xmm3[2],xmm2[3],xmm3[3],xmm2[4],xmm3[4],xmm2[5],xmm3[5],xmm2[6],xmm3[6],xmm2[7],xmm3[7]
+; SSE2-NEXT:    bsfl %r9d, %edx
+; SSE2-NEXT:    cmovel %eax, %edx
+; SSE2-NEXT:    cmpl $32, %edx
+; SSE2-NEXT:    cmovel %ecx, %edx
+; SSE2-NEXT:    movd %edx, %xmm0
+; SSE2-NEXT:    bsfl %r10d, %edx
+; SSE2-NEXT:    cmovel %eax, %edx
+; SSE2-NEXT:    cmpl $32, %edx
+; SSE2-NEXT:    cmovel %ecx, %edx
+; SSE2-NEXT:    movd %edx, %xmm3
+; SSE2-NEXT:    punpcklbw {{.*#+}} xmm3 = xmm3[0],xmm0[0],xmm3[1],xmm0[1],xmm3[2],xmm0[2],xmm3[3],xmm0[3],xmm3[4],xmm0[4],xmm3[5],xmm0[5],xmm3[6],xmm0[6],xmm3[7],xmm0[7]
+; SSE2-NEXT:    bsfl %r8d, %edx
+; SSE2-NEXT:    cmovel %eax, %edx
+; SSE2-NEXT:    cmpl $32, %edx
+; SSE2-NEXT:    cmovel %ecx, %edx
+; SSE2-NEXT:    movd %edx, %xmm4
+; SSE2-NEXT:    movzbl -{{[0-9]+}}(%rsp), %edx
+; SSE2-NEXT:    bsfl %edx, %edx
+; SSE2-NEXT:    cmovel %eax, %edx
+; SSE2-NEXT:    cmpl $32, %edx
+; SSE2-NEXT:    cmovel %ecx, %edx
+; SSE2-NEXT:    movd %edx, %xmm0
+; SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm4[0],xmm0[1],xmm4[1],xmm0[2],xmm4[2],xmm0[3],xmm4[3],xmm0[4],xmm4[4],xmm0[5],xmm4[5],xmm0[6],xmm4[6],xmm0[7],xmm4[7]
+; SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1],xmm0[2],xmm3[2],xmm0[3],xmm3[3],xmm0[4],xmm3[4],xmm0[5],xmm3[5],xmm0[6],xmm3[6],xmm0[7],xmm3[7]
+; SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1],xmm0[2],xmm2[2],xmm0[3],xmm2[3],xmm0[4],xmm2[4],xmm0[5],xmm2[5],xmm0[6],xmm2[6],xmm0[7],xmm2[7]
+; SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3],xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7]
+; SSE2-NEXT:    popq %rbx
+; SSE2-NEXT:    popq %r14
+; SSE2-NEXT:    popq %rbp
+; SSE2-NEXT:    retq
+;
+; SSE3-LABEL: testv16i8:
+; SSE3:       # BB#0:
+; SSE3-NEXT:    pushq %rbp
+; SSE3-NEXT:    pushq %r14
+; SSE3-NEXT:    pushq %rbx
+; SSE3-NEXT:    movaps %xmm0, -{{[0-9]+}}(%rsp)
+; SSE3-NEXT:    movzbl -{{[0-9]+}}(%rsp), %eax
+; SSE3-NEXT:    bsfl %eax, %edx
+; SSE3-NEXT:    movl $32, %eax
+; SSE3-NEXT:    cmovel %eax, %edx
+; SSE3-NEXT:    cmpl $32, %edx
+; SSE3-NEXT:    movl $8, %ecx
+; SSE3-NEXT:    cmovel %ecx, %edx
+; SSE3-NEXT:    movd %edx, %xmm0
+; SSE3-NEXT:    movzbl -{{[0-9]+}}(%rsp), %r14d
+; SSE3-NEXT:    movzbl -{{[0-9]+}}(%rsp), %ebx
+; SSE3-NEXT:    movzbl -{{[0-9]+}}(%rsp), %r9d
+; SSE3-NEXT:    movzbl -{{[0-9]+}}(%rsp), %edi
+; SSE3-NEXT:    movzbl -{{[0-9]+}}(%rsp), %r11d
+; SSE3-NEXT:    movzbl -{{[0-9]+}}(%rsp), %edx
+; SSE3-NEXT:    movzbl -{{[0-9]+}}(%rsp), %r8d
+; SSE3-NEXT:    movzbl -{{[0-9]+}}(%rsp), %esi
+; SSE3-NEXT:    bsfl %esi, %esi
+; SSE3-NEXT:    cmovel %eax, %esi
+; SSE3-NEXT:    cmpl $32, %esi
+; SSE3-NEXT:    cmovel %ecx, %esi
+; SSE3-NEXT:    movd %esi, %xmm1
+; SSE3-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3],xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
+; SSE3-NEXT:    bsfl %edi, %esi
+; SSE3-NEXT:    cmovel %eax, %esi
+; SSE3-NEXT:    cmpl $32, %esi
+; SSE3-NEXT:    cmovel %ecx, %esi
+; SSE3-NEXT:    movd %esi, %xmm2
+; SSE3-NEXT:    movzbl -{{[0-9]+}}(%rsp), %edi
+; SSE3-NEXT:    movzbl -{{[0-9]+}}(%rsp), %esi
+; SSE3-NEXT:    movzbl -{{[0-9]+}}(%rsp), %r10d
+; SSE3-NEXT:    movzbl -{{[0-9]+}}(%rsp), %ebp
+; SSE3-NEXT:    bsfl %ebp, %ebp
+; SSE3-NEXT:    cmovel %eax, %ebp
+; SSE3-NEXT:    cmpl $32, %ebp
+; SSE3-NEXT:    cmovel %ecx, %ebp
+; SSE3-NEXT:    movd %ebp, %xmm0
+; SSE3-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1],xmm0[2],xmm2[2],xmm0[3],xmm2[3],xmm0[4],xmm2[4],xmm0[5],xmm2[5],xmm0[6],xmm2[6],xmm0[7],xmm2[7]
+; SSE3-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3],xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7]
+; SSE3-NEXT:    bsfl %ebx, %ebx
+; SSE3-NEXT:    cmovel %eax, %ebx
+; SSE3-NEXT:    cmpl $32, %ebx
+; SSE3-NEXT:    cmovel %ecx, %ebx
+; SSE3-NEXT:    movd %ebx, %xmm1
+; SSE3-NEXT:    bsfl %esi, %esi
+; SSE3-NEXT:    cmovel %eax, %esi
+; SSE3-NEXT:    cmpl $32, %esi
+; SSE3-NEXT:    cmovel %ecx, %esi
+; SSE3-NEXT:    movd %esi, %xmm2
+; SSE3-NEXT:    punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3],xmm2[4],xmm1[4],xmm2[5],xmm1[5],xmm2[6],xmm1[6],xmm2[7],xmm1[7]
+; SSE3-NEXT:    bsfl %edx, %edx
+; SSE3-NEXT:    cmovel %eax, %edx
+; SSE3-NEXT:    cmpl $32, %edx
+; SSE3-NEXT:    cmovel %ecx, %edx
+; SSE3-NEXT:    movd %edx, %xmm3
+; SSE3-NEXT:    movzbl -{{[0-9]+}}(%rsp), %edx
+; SSE3-NEXT:    movzbl -{{[0-9]+}}(%rsp), %esi
+; SSE3-NEXT:    bsfl %esi, %esi
+; SSE3-NEXT:    cmovel %eax, %esi
+; SSE3-NEXT:    cmpl $32, %esi
+; SSE3-NEXT:    cmovel %ecx, %esi
+; SSE3-NEXT:    movd %esi, %xmm1
+; SSE3-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm3[0],xmm1[1],xmm3[1],xmm1[2],xmm3[2],xmm1[3],xmm3[3],xmm1[4],xmm3[4],xmm1[5],xmm3[5],xmm1[6],xmm3[6],xmm1[7],xmm3[7]
+; SSE3-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1],xmm1[2],xmm2[2],xmm1[3],xmm2[3],xmm1[4],xmm2[4],xmm1[5],xmm2[5],xmm1[6],xmm2[6],xmm1[7],xmm2[7]
+; SSE3-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3],xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
+; SSE3-NEXT:    bsfl %r14d, %esi
+; SSE3-NEXT:    cmovel %eax, %esi
+; SSE3-NEXT:    cmpl $32, %esi
+; SSE3-NEXT:    cmovel %ecx, %esi
+; SSE3-NEXT:    movd %esi, %xmm0
+; SSE3-NEXT:    bsfl %edi, %esi
+; SSE3-NEXT:    cmovel %eax, %esi
+; SSE3-NEXT:    cmpl $32, %esi
+; SSE3-NEXT:    cmovel %ecx, %esi
+; SSE3-NEXT:    movd %esi, %xmm3
+; SSE3-NEXT:    punpcklbw {{.*#+}} xmm3 = xmm3[0],xmm0[0],xmm3[1],xmm0[1],xmm3[2],xmm0[2],xmm3[3],xmm0[3],xmm3[4],xmm0[4],xmm3[5],xmm0[5],xmm3[6],xmm0[6],xmm3[7],xmm0[7]
+; SSE3-NEXT:    bsfl %r11d, %esi
+; SSE3-NEXT:    cmovel %eax, %esi
+; SSE3-NEXT:    cmpl $32, %esi
+; SSE3-NEXT:    cmovel %ecx, %esi
+; SSE3-NEXT:    movd %esi, %xmm0
+; SSE3-NEXT:    bsfl %edx, %edx
+; SSE3-NEXT:    cmovel %eax, %edx
+; SSE3-NEXT:    cmpl $32, %edx
+; SSE3-NEXT:    cmovel %ecx, %edx
+; SSE3-NEXT:    movd %edx, %xmm2
+; SSE3-NEXT:    punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3],xmm2[4],xmm0[4],xmm2[5],xmm0[5],xmm2[6],xmm0[6],xmm2[7],xmm0[7]
+; SSE3-NEXT:    punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1],xmm2[2],xmm3[2],xmm2[3],xmm3[3],xmm2[4],xmm3[4],xmm2[5],xmm3[5],xmm2[6],xmm3[6],xmm2[7],xmm3[7]
+; SSE3-NEXT:    bsfl %r9d, %edx
+; SSE3-NEXT:    cmovel %eax, %edx
+; SSE3-NEXT:    cmpl $32, %edx
+; SSE3-NEXT:    cmovel %ecx, %edx
+; SSE3-NEXT:    movd %edx, %xmm0
+; SSE3-NEXT:    bsfl %r10d, %edx
+; SSE3-NEXT:    cmovel %eax, %edx
+; SSE3-NEXT:    cmpl $32, %edx
+; SSE3-NEXT:    cmovel %ecx, %edx
+; SSE3-NEXT:    movd %edx, %xmm3
+; SSE3-NEXT:    punpcklbw {{.*#+}} xmm3 = xmm3[0],xmm0[0],xmm3[1],xmm0[1],xmm3[2],xmm0[2],xmm3[3],xmm0[3],xmm3[4],xmm0[4],xmm3[5],xmm0[5],xmm3[6],xmm0[6],xmm3[7],xmm0[7]
+; SSE3-NEXT:    bsfl %r8d, %edx
+; SSE3-NEXT:    cmovel %eax, %edx
+; SSE3-NEXT:    cmpl $32, %edx
+; SSE3-NEXT:    cmovel %ecx, %edx
+; SSE3-NEXT:    movd %edx, %xmm4
+; SSE3-NEXT:    movzbl -{{[0-9]+}}(%rsp), %edx
+; SSE3-NEXT:    bsfl %edx, %edx
+; SSE3-NEXT:    cmovel %eax, %edx
+; SSE3-NEXT:    cmpl $32, %edx
+; SSE3-NEXT:    cmovel %ecx, %edx
+; SSE3-NEXT:    movd %edx, %xmm0
+; SSE3-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm4[0],xmm0[1],xmm4[1],xmm0[2],xmm4[2],xmm0[3],xmm4[3],xmm0[4],xmm4[4],xmm0[5],xmm4[5],xmm0[6],xmm4[6],xmm0[7],xmm4[7]
+; SSE3-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1],xmm0[2],xmm3[2],xmm0[3],xmm3[3],xmm0[4],xmm3[4],xmm0[5],xmm3[5],xmm0[6],xmm3[6],xmm0[7],xmm3[7]
+; SSE3-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1],xmm0[2],xmm2[2],xmm0[3],xmm2[3],xmm0[4],xmm2[4],xmm0[5],xmm2[5],xmm0[6],xmm2[6],xmm0[7],xmm2[7]
+; SSE3-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3],xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7]
+; SSE3-NEXT:    popq %rbx
+; SSE3-NEXT:    popq %r14
+; SSE3-NEXT:    popq %rbp
+; SSE3-NEXT:    retq
+;
+; SSSE3-LABEL: testv16i8:
+; SSSE3:       # BB#0:
+; SSSE3-NEXT:    pushq %rbp
+; SSSE3-NEXT:    pushq %r14
+; SSSE3-NEXT:    pushq %rbx
+; SSSE3-NEXT:    movaps %xmm0, -{{[0-9]+}}(%rsp)
+; SSSE3-NEXT:    movzbl -{{[0-9]+}}(%rsp), %eax
+; SSSE3-NEXT:    bsfl %eax, %edx
+; SSSE3-NEXT:    movl $32, %eax
+; SSSE3-NEXT:    cmovel %eax, %edx
+; SSSE3-NEXT:    cmpl $32, %edx
+; SSSE3-NEXT:    movl $8, %ecx
+; SSSE3-NEXT:    cmovel %ecx, %edx
+; SSSE3-NEXT:    movd %edx, %xmm0
+; SSSE3-NEXT:    movzbl -{{[0-9]+}}(%rsp), %r14d
+; SSSE3-NEXT:    movzbl -{{[0-9]+}}(%rsp), %ebx
+; SSSE3-NEXT:    movzbl -{{[0-9]+}}(%rsp), %r9d
+; SSSE3-NEXT:    movzbl -{{[0-9]+}}(%rsp), %edi
+; SSSE3-NEXT:    movzbl -{{[0-9]+}}(%rsp), %r11d
+; SSSE3-NEXT:    movzbl -{{[0-9]+}}(%rsp), %edx
+; SSSE3-NEXT:    movzbl -{{[0-9]+}}(%rsp), %r8d
+; SSSE3-NEXT:    movzbl -{{[0-9]+}}(%rsp), %esi
+; SSSE3-NEXT:    bsfl %esi, %esi
+; SSSE3-NEXT:    cmovel %eax, %esi
+; SSSE3-NEXT:    cmpl $32, %esi
+; SSSE3-NEXT:    cmovel %ecx, %esi
+; SSSE3-NEXT:    movd %esi, %xmm1
+; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3],xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
+; SSSE3-NEXT:    bsfl %edi, %esi
+; SSSE3-NEXT:    cmovel %eax, %esi
+; SSSE3-NEXT:    cmpl $32, %esi
+; SSSE3-NEXT:    cmovel %ecx, %esi
+; SSSE3-NEXT:    movd %esi, %xmm2
+; SSSE3-NEXT:    movzbl -{{[0-9]+}}(%rsp), %edi
+; SSSE3-NEXT:    movzbl -{{[0-9]+}}(%rsp), %esi
+; SSSE3-NEXT:    movzbl -{{[0-9]+}}(%rsp), %r10d
+; SSSE3-NEXT:    movzbl -{{[0-9]+}}(%rsp), %ebp
+; SSSE3-NEXT:    bsfl %ebp, %ebp
+; SSSE3-NEXT:    cmovel %eax, %ebp
+; SSSE3-NEXT:    cmpl $32, %ebp
+; SSSE3-NEXT:    cmovel %ecx, %ebp
+; SSSE3-NEXT:    movd %ebp, %xmm0
+; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1],xmm0[2],xmm2[2],xmm0[3],xmm2[3],xmm0[4],xmm2[4],xmm0[5],xmm2[5],xmm0[6],xmm2[6],xmm0[7],xmm2[7]
+; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3],xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7]
+; SSSE3-NEXT:    bsfl %ebx, %ebx
+; SSSE3-NEXT:    cmovel %eax, %ebx
+; SSSE3-NEXT:    cmpl $32, %ebx
+; SSSE3-NEXT:    cmovel %ecx, %ebx
+; SSSE3-NEXT:    movd %ebx, %xmm1
+; SSSE3-NEXT:    bsfl %esi, %esi
+; SSSE3-NEXT:    cmovel %eax, %esi
+; SSSE3-NEXT:    cmpl $32, %esi
+; SSSE3-NEXT:    cmovel %ecx, %esi
+; SSSE3-NEXT:    movd %esi, %xmm2
+; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3],xmm2[4],xmm1[4],xmm2[5],xmm1[5],xmm2[6],xmm1[6],xmm2[7],xmm1[7]
+; SSSE3-NEXT:    bsfl %edx, %edx
+; SSSE3-NEXT:    cmovel %eax, %edx
+; SSSE3-NEXT:    cmpl $32, %edx
+; SSSE3-NEXT:    cmovel %ecx, %edx
+; SSSE3-NEXT:    movd %edx, %xmm3
+; SSSE3-NEXT:    movzbl -{{[0-9]+}}(%rsp), %edx
+; SSSE3-NEXT:    movzbl -{{[0-9]+}}(%rsp), %esi
+; SSSE3-NEXT:    bsfl %esi, %esi
+; SSSE3-NEXT:    cmovel %eax, %esi
+; SSSE3-NEXT:    cmpl $32, %esi
+; SSSE3-NEXT:    cmovel %ecx, %esi
+; SSSE3-NEXT:    movd %esi, %xmm1
+; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm3[0],xmm1[1],xmm3[1],xmm1[2],xmm3[2],xmm1[3],xmm3[3],xmm1[4],xmm3[4],xmm1[5],xmm3[5],xmm1[6],xmm3[6],xmm1[7],xmm3[7]
+; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1],xmm1[2],xmm2[2],xmm1[3],xmm2[3],xmm1[4],xmm2[4],xmm1[5],xmm2[5],xmm1[6],xmm2[6],xmm1[7],xmm2[7]
+; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3],xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
+; SSSE3-NEXT:    bsfl %r14d, %esi
+; SSSE3-NEXT:    cmovel %eax, %esi
+; SSSE3-NEXT:    cmpl $32, %esi
+; SSSE3-NEXT:    cmovel %ecx, %esi
+; SSSE3-NEXT:    movd %esi, %xmm0
+; SSSE3-NEXT:    bsfl %edi, %esi
+; SSSE3-NEXT:    cmovel %eax, %esi
+; SSSE3-NEXT:    cmpl $32, %esi
+; SSSE3-NEXT:    cmovel %ecx, %esi
+; SSSE3-NEXT:    movd %esi, %xmm3
+; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm3 = xmm3[0],xmm0[0],xmm3[1],xmm0[1],xmm3[2],xmm0[2],xmm3[3],xmm0[3],xmm3[4],xmm0[4],xmm3[5],xmm0[5],xmm3[6],xmm0[6],xmm3[7],xmm0[7]
+; SSSE3-NEXT:    bsfl %r11d, %esi
+; SSSE3-NEXT:    cmovel %eax, %esi
+; SSSE3-NEXT:    cmpl $32, %esi
+; SSSE3-NEXT:    cmovel %ecx, %esi
+; SSSE3-NEXT:    movd %esi, %xmm0
+; SSSE3-NEXT:    bsfl %edx, %edx
+; SSSE3-NEXT:    cmovel %eax, %edx
+; SSSE3-NEXT:    cmpl $32, %edx
+; SSSE3-NEXT:    cmovel %ecx, %edx
+; SSSE3-NEXT:    movd %edx, %xmm2
+; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3],xmm2[4],xmm0[4],xmm2[5],xmm0[5],xmm2[6],xmm0[6],xmm2[7],xmm0[7]
+; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1],xmm2[2],xmm3[2],xmm2[3],xmm3[3],xmm2[4],xmm3[4],xmm2[5],xmm3[5],xmm2[6],xmm3[6],xmm2[7],xmm3[7]
+; SSSE3-NEXT:    bsfl %r9d, %edx
+; SSSE3-NEXT:    cmovel %eax, %edx
+; SSSE3-NEXT:    cmpl $32, %edx
+; SSSE3-NEXT:    cmovel %ecx, %edx
+; SSSE3-NEXT:    movd %edx, %xmm0
+; SSSE3-NEXT:    bsfl %r10d, %edx
+; SSSE3-NEXT:    cmovel %eax, %edx
+; SSSE3-NEXT:    cmpl $32, %edx
+; SSSE3-NEXT:    cmovel %ecx, %edx
+; SSSE3-NEXT:    movd %edx, %xmm3
+; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm3 = xmm3[0],xmm0[0],xmm3[1],xmm0[1],xmm3[2],xmm0[2],xmm3[3],xmm0[3],xmm3[4],xmm0[4],xmm3[5],xmm0[5],xmm3[6],xmm0[6],xmm3[7],xmm0[7]
+; SSSE3-NEXT:    bsfl %r8d, %edx
+; SSSE3-NEXT:    cmovel %eax, %edx
+; SSSE3-NEXT:    cmpl $32, %edx
+; SSSE3-NEXT:    cmovel %ecx, %edx
+; SSSE3-NEXT:    movd %edx, %xmm4
+; SSSE3-NEXT:    movzbl -{{[0-9]+}}(%rsp), %edx
+; SSSE3-NEXT:    bsfl %edx, %edx
+; SSSE3-NEXT:    cmovel %eax, %edx
+; SSSE3-NEXT:    cmpl $32, %edx
+; SSSE3-NEXT:    cmovel %ecx, %edx
+; SSSE3-NEXT:    movd %edx, %xmm0
+; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm4[0],xmm0[1],xmm4[1],xmm0[2],xmm4[2],xmm0[3],xmm4[3],xmm0[4],xmm4[4],xmm0[5],xmm4[5],xmm0[6],xmm4[6],xmm0[7],xmm4[7]
+; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1],xmm0[2],xmm3[2],xmm0[3],xmm3[3],xmm0[4],xmm3[4],xmm0[5],xmm3[5],xmm0[6],xmm3[6],xmm0[7],xmm3[7]
+; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1],xmm0[2],xmm2[2],xmm0[3],xmm2[3],xmm0[4],xmm2[4],xmm0[5],xmm2[5],xmm0[6],xmm2[6],xmm0[7],xmm2[7]
+; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3],xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7]
+; SSSE3-NEXT:    popq %rbx
+; SSSE3-NEXT:    popq %r14
+; SSSE3-NEXT:    popq %rbp
+; SSSE3-NEXT:    retq
+;
+; SSE41-LABEL: testv16i8:
+; SSE41:       # BB#0:
+; SSE41-NEXT:    pextrb $1, %xmm0, %eax
+; SSE41-NEXT:    bsfl %eax, %edx
+; SSE41-NEXT:    movl $32, %eax
+; SSE41-NEXT:    cmovel %eax, %edx
+; SSE41-NEXT:    cmpl $32, %edx
+; SSE41-NEXT:    movl $8, %ecx
+; SSE41-NEXT:    cmovel %ecx, %edx
+; SSE41-NEXT:    pextrb $0, %xmm0, %esi
+; SSE41-NEXT:    bsfl %esi, %esi
+; SSE41-NEXT:    cmovel %eax, %esi
+; SSE41-NEXT:    cmpl $32, %esi
+; SSE41-NEXT:    cmovel %ecx, %esi
+; SSE41-NEXT:    movd %esi, %xmm1
+; SSE41-NEXT:    pinsrb $1, %edx, %xmm1
+; SSE41-NEXT:    pextrb $2, %xmm0, %edx
+; SSE41-NEXT:    bsfl %edx, %edx
+; SSE41-NEXT:    cmovel %eax, %edx
+; SSE41-NEXT:    cmpl $32, %edx
+; SSE41-NEXT:    cmovel %ecx, %edx
+; SSE41-NEXT:    pinsrb $2, %edx, %xmm1
+; SSE41-NEXT:    pextrb $3, %xmm0, %edx
+; SSE41-NEXT:    bsfl %edx, %edx
+; SSE41-NEXT:    cmovel %eax, %edx
+; SSE41-NEXT:    cmpl $32, %edx
+; SSE41-NEXT:    cmovel %ecx, %edx
+; SSE41-NEXT:    pinsrb $3, %edx, %xmm1
+; SSE41-NEXT:    pextrb $4, %xmm0, %edx
+; SSE41-NEXT:    bsfl %edx, %edx
+; SSE41-NEXT:    cmovel %eax, %edx
+; SSE41-NEXT:    cmpl $32, %edx
+; SSE41-NEXT:    cmovel %ecx, %edx
+; SSE41-NEXT:    pinsrb $4, %edx, %xmm1
+; SSE41-NEXT:    pextrb $5, %xmm0, %edx
+; SSE41-NEXT:    bsfl %edx, %edx
+; SSE41-NEXT:    cmovel %eax, %edx
+; SSE41-NEXT:    cmpl $32, %edx
+; SSE41-NEXT:    cmovel %ecx, %edx
+; SSE41-NEXT:    pinsrb $5, %edx, %xmm1
+; SSE41-NEXT:    pextrb $6, %xmm0, %edx
+; SSE41-NEXT:    bsfl %edx, %edx
+; SSE41-NEXT:    cmovel %eax, %edx
+; SSE41-NEXT:    cmpl $32, %edx
+; SSE41-NEXT:    cmovel %ecx, %edx
+; SSE41-NEXT:    pinsrb $6, %edx, %xmm1
+; SSE41-NEXT:    pextrb $7, %xmm0, %edx
+; SSE41-NEXT:    bsfl %edx, %edx
+; SSE41-NEXT:    cmovel %eax, %edx
+; SSE41-NEXT:    cmpl $32, %edx
+; SSE41-NEXT:    cmovel %ecx, %edx
+; SSE41-NEXT:    pinsrb $7, %edx, %xmm1
+; SSE41-NEXT:    pextrb $8, %xmm0, %edx
+; SSE41-NEXT:    bsfl %edx, %edx
+; SSE41-NEXT:    cmovel %eax, %edx
+; SSE41-NEXT:    cmpl $32, %edx
+; SSE41-NEXT:    cmovel %ecx, %edx
+; SSE41-NEXT:    pinsrb $8, %edx, %xmm1
+; SSE41-NEXT:    pextrb $9, %xmm0, %edx
+; SSE41-NEXT:    bsfl %edx, %edx
+; SSE41-NEXT:    cmovel %eax, %edx
+; SSE41-NEXT:    cmpl $32, %edx
+; SSE41-NEXT:    cmovel %ecx, %edx
+; SSE41-NEXT:    pinsrb $9, %edx, %xmm1
+; SSE41-NEXT:    pextrb $10, %xmm0, %edx
+; SSE41-NEXT:    bsfl %edx, %edx
+; SSE41-NEXT:    cmovel %eax, %edx
+; SSE41-NEXT:    cmpl $32, %edx
+; SSE41-NEXT:    cmovel %ecx, %edx
+; SSE41-NEXT:    pinsrb $10, %edx, %xmm1
+; SSE41-NEXT:    pextrb $11, %xmm0, %edx
+; SSE41-NEXT:    bsfl %edx, %edx
+; SSE41-NEXT:    cmovel %eax, %edx
+; SSE41-NEXT:    cmpl $32, %edx
+; SSE41-NEXT:    cmovel %ecx, %edx
+; SSE41-NEXT:    pinsrb $11, %edx, %xmm1
+; SSE41-NEXT:    pextrb $12, %xmm0, %edx
+; SSE41-NEXT:    bsfl %edx, %edx
+; SSE41-NEXT:    cmovel %eax, %edx
+; SSE41-NEXT:    cmpl $32, %edx
+; SSE41-NEXT:    cmovel %ecx, %edx
+; SSE41-NEXT:    pinsrb $12, %edx, %xmm1
+; SSE41-NEXT:    pextrb $13, %xmm0, %edx
+; SSE41-NEXT:    bsfl %edx, %edx
+; SSE41-NEXT:    cmovel %eax, %edx
+; SSE41-NEXT:    cmpl $32, %edx
+; SSE41-NEXT:    cmovel %ecx, %edx
+; SSE41-NEXT:    pinsrb $13, %edx, %xmm1
+; SSE41-NEXT:    pextrb $14, %xmm0, %edx
+; SSE41-NEXT:    bsfl %edx, %edx
+; SSE41-NEXT:    cmovel %eax, %edx
+; SSE41-NEXT:    cmpl $32, %edx
+; SSE41-NEXT:    cmovel %ecx, %edx
+; SSE41-NEXT:    pinsrb $14, %edx, %xmm1
+; SSE41-NEXT:    pextrb $15, %xmm0, %edx
+; SSE41-NEXT:    bsfl %edx, %edx
+; SSE41-NEXT:    cmovel %eax, %edx
+; SSE41-NEXT:    cmpl $32, %edx
+; SSE41-NEXT:    cmovel %ecx, %edx
+; SSE41-NEXT:    pinsrb $15, %edx, %xmm1
+; SSE41-NEXT:    movdqa %xmm1, %xmm0
+; SSE41-NEXT:    retq
+;
+; AVX-LABEL: testv16i8:
+; AVX:       # BB#0:
+; AVX-NEXT:    vpextrb $1, %xmm0, %eax
+; AVX-NEXT:    bsfl %eax, %edx
+; AVX-NEXT:    movl $32, %eax
+; AVX-NEXT:    cmovel %eax, %edx
+; AVX-NEXT:    cmpl $32, %edx
+; AVX-NEXT:    movl $8, %ecx
+; AVX-NEXT:    cmovel %ecx, %edx
+; AVX-NEXT:    vpextrb $0, %xmm0, %esi
+; AVX-NEXT:    bsfl %esi, %esi
+; AVX-NEXT:    cmovel %eax, %esi
+; AVX-NEXT:    cmpl $32, %esi
+; AVX-NEXT:    cmovel %ecx, %esi
+; AVX-NEXT:    vmovd %esi, %xmm1
+; AVX-NEXT:    vpinsrb $1, %edx, %xmm1, %xmm1
+; AVX-NEXT:    vpextrb $2, %xmm0, %edx
+; AVX-NEXT:    bsfl %edx, %edx
+; AVX-NEXT:    cmovel %eax, %edx
+; AVX-NEXT:    cmpl $32, %edx
+; AVX-NEXT:    cmovel %ecx, %edx
+; AVX-NEXT:    vpinsrb $2, %edx, %xmm1, %xmm1
+; AVX-NEXT:    vpextrb $3, %xmm0, %edx
+; AVX-NEXT:    bsfl %edx, %edx
+; AVX-NEXT:    cmovel %eax, %edx
+; AVX-NEXT:    cmpl $32, %edx
+; AVX-NEXT:    cmovel %ecx, %edx
+; AVX-NEXT:    vpinsrb $3, %edx, %xmm1, %xmm1
+; AVX-NEXT:    vpextrb $4, %xmm0, %edx
+; AVX-NEXT:    bsfl %edx, %edx
+; AVX-NEXT:    cmovel %eax, %edx
+; AVX-NEXT:    cmpl $32, %edx
+; AVX-NEXT:    cmovel %ecx, %edx
+; AVX-NEXT:    vpinsrb $4, %edx, %xmm1, %xmm1
+; AVX-NEXT:    vpextrb $5, %xmm0, %edx
+; AVX-NEXT:    bsfl %edx, %edx
+; AVX-NEXT:    cmovel %eax, %edx
+; AVX-NEXT:    cmpl $32, %edx
+; AVX-NEXT:    cmovel %ecx, %edx
+; AVX-NEXT:    vpinsrb $5, %edx, %xmm1, %xmm1
+; AVX-NEXT:    vpextrb $6, %xmm0, %edx
+; AVX-NEXT:    bsfl %edx, %edx
+; AVX-NEXT:    cmovel %eax, %edx
+; AVX-NEXT:    cmpl $32, %edx
+; AVX-NEXT:    cmovel %ecx, %edx
+; AVX-NEXT:    vpinsrb $6, %edx, %xmm1, %xmm1
+; AVX-NEXT:    vpextrb $7, %xmm0, %edx
+; AVX-NEXT:    bsfl %edx, %edx
+; AVX-NEXT:    cmovel %eax, %edx
+; AVX-NEXT:    cmpl $32, %edx
+; AVX-NEXT:    cmovel %ecx, %edx
+; AVX-NEXT:    vpinsrb $7, %edx, %xmm1, %xmm1
+; AVX-NEXT:    vpextrb $8, %xmm0, %edx
+; AVX-NEXT:    bsfl %edx, %edx
+; AVX-NEXT:    cmovel %eax, %edx
+; AVX-NEXT:    cmpl $32, %edx
+; AVX-NEXT:    cmovel %ecx, %edx
+; AVX-NEXT:    vpinsrb $8, %edx, %xmm1, %xmm1
+; AVX-NEXT:    vpextrb $9, %xmm0, %edx
+; AVX-NEXT:    bsfl %edx, %edx
+; AVX-NEXT:    cmovel %eax, %edx
+; AVX-NEXT:    cmpl $32, %edx
+; AVX-NEXT:    cmovel %ecx, %edx
+; AVX-NEXT:    vpinsrb $9, %edx, %xmm1, %xmm1
+; AVX-NEXT:    vpextrb $10, %xmm0, %edx
+; AVX-NEXT:    bsfl %edx, %edx
+; AVX-NEXT:    cmovel %eax, %edx
+; AVX-NEXT:    cmpl $32, %edx
+; AVX-NEXT:    cmovel %ecx, %edx
+; AVX-NEXT:    vpinsrb $10, %edx, %xmm1, %xmm1
+; AVX-NEXT:    vpextrb $11, %xmm0, %edx
+; AVX-NEXT:    bsfl %edx, %edx
+; AVX-NEXT:    cmovel %eax, %edx
+; AVX-NEXT:    cmpl $32, %edx
+; AVX-NEXT:    cmovel %ecx, %edx
+; AVX-NEXT:    vpinsrb $11, %edx, %xmm1, %xmm1
+; AVX-NEXT:    vpextrb $12, %xmm0, %edx
+; AVX-NEXT:    bsfl %edx, %edx
+; AVX-NEXT:    cmovel %eax, %edx
+; AVX-NEXT:    cmpl $32, %edx
+; AVX-NEXT:    cmovel %ecx, %edx
+; AVX-NEXT:    vpinsrb $12, %edx, %xmm1, %xmm1
+; AVX-NEXT:    vpextrb $13, %xmm0, %edx
+; AVX-NEXT:    bsfl %edx, %edx
+; AVX-NEXT:    cmovel %eax, %edx
+; AVX-NEXT:    cmpl $32, %edx
+; AVX-NEXT:    cmovel %ecx, %edx
+; AVX-NEXT:    vpinsrb $13, %edx, %xmm1, %xmm1
+; AVX-NEXT:    vpextrb $14, %xmm0, %edx
+; AVX-NEXT:    bsfl %edx, %edx
+; AVX-NEXT:    cmovel %eax, %edx
+; AVX-NEXT:    cmpl $32, %edx
+; AVX-NEXT:    cmovel %ecx, %edx
+; AVX-NEXT:    vpinsrb $14, %edx, %xmm1, %xmm1
+; AVX-NEXT:    vpextrb $15, %xmm0, %edx
+; AVX-NEXT:    bsfl %edx, %edx
+; AVX-NEXT:    cmovel %eax, %edx
+; AVX-NEXT:    cmpl $32, %edx
+; AVX-NEXT:    cmovel %ecx, %edx
+; AVX-NEXT:    vpinsrb $15, %edx, %xmm1, %xmm0
+; AVX-NEXT:    retq
+  %out = call <16 x i8> @llvm.cttz.v16i8(<16 x i8> %in, i1 0)
+  ret <16 x i8> %out
+}
+
+define <16 x i8> @testv16i8u(<16 x i8> %in) nounwind {
+; SSE2-LABEL: testv16i8u:
+; SSE2:       # BB#0:
+; SSE2-NEXT:    pushq %rbx
+; SSE2-NEXT:    movaps %xmm0, -{{[0-9]+}}(%rsp)
+; SSE2-NEXT:    movzbl -{{[0-9]+}}(%rsp), %eax
+; SSE2-NEXT:    bsfl %eax, %eax
+; SSE2-NEXT:    movd %eax, %xmm0
+; SSE2-NEXT:    movzbl -{{[0-9]+}}(%rsp), %r11d
+; SSE2-NEXT:    movzbl -{{[0-9]+}}(%rsp), %eax
+; SSE2-NEXT:    movzbl -{{[0-9]+}}(%rsp), %r9d
+; SSE2-NEXT:    movzbl -{{[0-9]+}}(%rsp), %edi
+; SSE2-NEXT:    movzbl -{{[0-9]+}}(%rsp), %r10d
+; SSE2-NEXT:    movzbl -{{[0-9]+}}(%rsp), %ecx
+; SSE2-NEXT:    movzbl -{{[0-9]+}}(%rsp), %r8d
+; SSE2-NEXT:    movzbl -{{[0-9]+}}(%rsp), %edx
+; SSE2-NEXT:    bsfl %edx, %edx
+; SSE2-NEXT:    movd %edx, %xmm1
+; SSE2-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3],xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
+; SSE2-NEXT:    bsfl %edi, %edx
+; SSE2-NEXT:    movd %edx, %xmm0
+; SSE2-NEXT:    movzbl -{{[0-9]+}}(%rsp), %edx
+; SSE2-NEXT:    movzbl -{{[0-9]+}}(%rsp), %esi
+; SSE2-NEXT:    movzbl -{{[0-9]+}}(%rsp), %edi
+; SSE2-NEXT:    movzbl -{{[0-9]+}}(%rsp), %ebx
+; SSE2-NEXT:    bsfl %ebx, %ebx
+; SSE2-NEXT:    movd %ebx, %xmm2
+; SSE2-NEXT:    punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3],xmm2[4],xmm0[4],xmm2[5],xmm0[5],xmm2[6],xmm0[6],xmm2[7],xmm0[7]
+; SSE2-NEXT:    punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3],xmm2[4],xmm1[4],xmm2[5],xmm1[5],xmm2[6],xmm1[6],xmm2[7],xmm1[7]
+; SSE2-NEXT:    bsfl %eax, %eax
+; SSE2-NEXT:    movd %eax, %xmm0
+; SSE2-NEXT:    bsfl %esi, %eax
+; SSE2-NEXT:    movd %eax, %xmm3
+; SSE2-NEXT:    punpcklbw {{.*#+}} xmm3 = xmm3[0],xmm0[0],xmm3[1],xmm0[1],xmm3[2],xmm0[2],xmm3[3],xmm0[3],xmm3[4],xmm0[4],xmm3[5],xmm0[5],xmm3[6],xmm0[6],xmm3[7],xmm0[7]
+; SSE2-NEXT:    bsfl %ecx, %eax
+; SSE2-NEXT:    movd %eax, %xmm0
+; SSE2-NEXT:    movzbl -{{[0-9]+}}(%rsp), %eax
+; SSE2-NEXT:    movzbl -{{[0-9]+}}(%rsp), %ecx
+; SSE2-NEXT:    bsfl %ecx, %ecx
+; SSE2-NEXT:    movd %ecx, %xmm1
+; SSE2-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3],xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
+; SSE2-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm3[0],xmm1[1],xmm3[1],xmm1[2],xmm3[2],xmm1[3],xmm3[3],xmm1[4],xmm3[4],xmm1[5],xmm3[5],xmm1[6],xmm3[6],xmm1[7],xmm3[7]
+; SSE2-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1],xmm1[2],xmm2[2],xmm1[3],xmm2[3],xmm1[4],xmm2[4],xmm1[5],xmm2[5],xmm1[6],xmm2[6],xmm1[7],xmm2[7]
+; SSE2-NEXT:    bsfl %r11d, %ecx
+; SSE2-NEXT:    movd %ecx, %xmm0
+; SSE2-NEXT:    bsfl %edx, %ecx
+; SSE2-NEXT:    movd %ecx, %xmm2
+; SSE2-NEXT:    punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3],xmm2[4],xmm0[4],xmm2[5],xmm0[5],xmm2[6],xmm0[6],xmm2[7],xmm0[7]
+; SSE2-NEXT:    bsfl %r10d, %ecx
+; SSE2-NEXT:    movd %ecx, %xmm0
+; SSE2-NEXT:    bsfl %eax, %eax
+; SSE2-NEXT:    movd %eax, %xmm3
+; SSE2-NEXT:    punpcklbw {{.*#+}} xmm3 = xmm3[0],xmm0[0],xmm3[1],xmm0[1],xmm3[2],xmm0[2],xmm3[3],xmm0[3],xmm3[4],xmm0[4],xmm3[5],xmm0[5],xmm3[6],xmm0[6],xmm3[7],xmm0[7]
+; SSE2-NEXT:    punpcklbw {{.*#+}} xmm3 = xmm3[0],xmm2[0],xmm3[1],xmm2[1],xmm3[2],xmm2[2],xmm3[3],xmm2[3],xmm3[4],xmm2[4],xmm3[5],xmm2[5],xmm3[6],xmm2[6],xmm3[7],xmm2[7]
+; SSE2-NEXT:    bsfl %r9d, %eax
+; SSE2-NEXT:    movd %eax, %xmm0
+; SSE2-NEXT:    bsfl %edi, %eax
+; SSE2-NEXT:    movd %eax, %xmm2
+; SSE2-NEXT:    punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3],xmm2[4],xmm0[4],xmm2[5],xmm0[5],xmm2[6],xmm0[6],xmm2[7],xmm0[7]
+; SSE2-NEXT:    bsfl %r8d, %eax
+; SSE2-NEXT:    movd %eax, %xmm4
+; SSE2-NEXT:    movzbl -{{[0-9]+}}(%rsp), %eax
+; SSE2-NEXT:    bsfl %eax, %eax
+; SSE2-NEXT:    movd %eax, %xmm0
+; SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm4[0],xmm0[1],xmm4[1],xmm0[2],xmm4[2],xmm0[3],xmm4[3],xmm0[4],xmm4[4],xmm0[5],xmm4[5],xmm0[6],xmm4[6],xmm0[7],xmm4[7]
+; SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1],xmm0[2],xmm2[2],xmm0[3],xmm2[3],xmm0[4],xmm2[4],xmm0[5],xmm2[5],xmm0[6],xmm2[6],xmm0[7],xmm2[7]
+; SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1],xmm0[2],xmm3[2],xmm0[3],xmm3[3],xmm0[4],xmm3[4],xmm0[5],xmm3[5],xmm0[6],xmm3[6],xmm0[7],xmm3[7]
+; SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3],xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7]
+; SSE2-NEXT:    popq %rbx
+; SSE2-NEXT:    retq
+;
+; SSE3-LABEL: testv16i8u:
+; SSE3:       # BB#0:
+; SSE3-NEXT:    pushq %rbx
+; SSE3-NEXT:    movaps %xmm0, -{{[0-9]+}}(%rsp)
+; SSE3-NEXT:    movzbl -{{[0-9]+}}(%rsp), %eax
+; SSE3-NEXT:    bsfl %eax, %eax
+; SSE3-NEXT:    movd %eax, %xmm0
+; SSE3-NEXT:    movzbl -{{[0-9]+}}(%rsp), %r11d
+; SSE3-NEXT:    movzbl -{{[0-9]+}}(%rsp), %eax
+; SSE3-NEXT:    movzbl -{{[0-9]+}}(%rsp), %r9d
+; SSE3-NEXT:    movzbl -{{[0-9]+}}(%rsp), %edi
+; SSE3-NEXT:    movzbl -{{[0-9]+}}(%rsp), %r10d
+; SSE3-NEXT:    movzbl -{{[0-9]+}}(%rsp), %ecx
+; SSE3-NEXT:    movzbl -{{[0-9]+}}(%rsp), %r8d
+; SSE3-NEXT:    movzbl -{{[0-9]+}}(%rsp), %edx
+; SSE3-NEXT:    bsfl %edx, %edx
+; SSE3-NEXT:    movd %edx, %xmm1
+; SSE3-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3],xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
+; SSE3-NEXT:    bsfl %edi, %edx
+; SSE3-NEXT:    movd %edx, %xmm0
+; SSE3-NEXT:    movzbl -{{[0-9]+}}(%rsp), %edx
+; SSE3-NEXT:    movzbl -{{[0-9]+}}(%rsp), %esi
+; SSE3-NEXT:    movzbl -{{[0-9]+}}(%rsp), %edi
+; SSE3-NEXT:    movzbl -{{[0-9]+}}(%rsp), %ebx
+; SSE3-NEXT:    bsfl %ebx, %ebx
+; SSE3-NEXT:    movd %ebx, %xmm2
+; SSE3-NEXT:    punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3],xmm2[4],xmm0[4],xmm2[5],xmm0[5],xmm2[6],xmm0[6],xmm2[7],xmm0[7]
+; SSE3-NEXT:    punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3],xmm2[4],xmm1[4],xmm2[5],xmm1[5],xmm2[6],xmm1[6],xmm2[7],xmm1[7]
+; SSE3-NEXT:    bsfl %eax, %eax
+; SSE3-NEXT:    movd %eax, %xmm0
+; SSE3-NEXT:    bsfl %esi, %eax
+; SSE3-NEXT:    movd %eax, %xmm3
+; SSE3-NEXT:    punpcklbw {{.*#+}} xmm3 = xmm3[0],xmm0[0],xmm3[1],xmm0[1],xmm3[2],xmm0[2],xmm3[3],xmm0[3],xmm3[4],xmm0[4],xmm3[5],xmm0[5],xmm3[6],xmm0[6],xmm3[7],xmm0[7]
+; SSE3-NEXT:    bsfl %ecx, %eax
+; SSE3-NEXT:    movd %eax, %xmm0
+; SSE3-NEXT:    movzbl -{{[0-9]+}}(%rsp), %eax
+; SSE3-NEXT:    movzbl -{{[0-9]+}}(%rsp), %ecx
+; SSE3-NEXT:    bsfl %ecx, %ecx
+; SSE3-NEXT:    movd %ecx, %xmm1
+; SSE3-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3],xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
+; SSE3-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm3[0],xmm1[1],xmm3[1],xmm1[2],xmm3[2],xmm1[3],xmm3[3],xmm1[4],xmm3[4],xmm1[5],xmm3[5],xmm1[6],xmm3[6],xmm1[7],xmm3[7]
+; SSE3-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1],xmm1[2],xmm2[2],xmm1[3],xmm2[3],xmm1[4],xmm2[4],xmm1[5],xmm2[5],xmm1[6],xmm2[6],xmm1[7],xmm2[7]
+; SSE3-NEXT:    bsfl %r11d, %ecx
+; SSE3-NEXT:    movd %ecx, %xmm0
+; SSE3-NEXT:    bsfl %edx, %ecx
+; SSE3-NEXT:    movd %ecx, %xmm2
+; SSE3-NEXT:    punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3],xmm2[4],xmm0[4],xmm2[5],xmm0[5],xmm2[6],xmm0[6],xmm2[7],xmm0[7]
+; SSE3-NEXT:    bsfl %r10d, %ecx
+; SSE3-NEXT:    movd %ecx, %xmm0
+; SSE3-NEXT:    bsfl %eax, %eax
+; SSE3-NEXT:    movd %eax, %xmm3
+; SSE3-NEXT:    punpcklbw {{.*#+}} xmm3 = xmm3[0],xmm0[0],xmm3[1],xmm0[1],xmm3[2],xmm0[2],xmm3[3],xmm0[3],xmm3[4],xmm0[4],xmm3[5],xmm0[5],xmm3[6],xmm0[6],xmm3[7],xmm0[7]
+; SSE3-NEXT:    punpcklbw {{.*#+}} xmm3 = xmm3[0],xmm2[0],xmm3[1],xmm2[1],xmm3[2],xmm2[2],xmm3[3],xmm2[3],xmm3[4],xmm2[4],xmm3[5],xmm2[5],xmm3[6],xmm2[6],xmm3[7],xmm2[7]
+; SSE3-NEXT:    bsfl %r9d, %eax
+; SSE3-NEXT:    movd %eax, %xmm0
+; SSE3-NEXT:    bsfl %edi, %eax
+; SSE3-NEXT:    movd %eax, %xmm2
+; SSE3-NEXT:    punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3],xmm2[4],xmm0[4],xmm2[5],xmm0[5],xmm2[6],xmm0[6],xmm2[7],xmm0[7]
+; SSE3-NEXT:    bsfl %r8d, %eax
+; SSE3-NEXT:    movd %eax, %xmm4
+; SSE3-NEXT:    movzbl -{{[0-9]+}}(%rsp), %eax
+; SSE3-NEXT:    bsfl %eax, %eax
+; SSE3-NEXT:    movd %eax, %xmm0
+; SSE3-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm4[0],xmm0[1],xmm4[1],xmm0[2],xmm4[2],xmm0[3],xmm4[3],xmm0[4],xmm4[4],xmm0[5],xmm4[5],xmm0[6],xmm4[6],xmm0[7],xmm4[7]
+; SSE3-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1],xmm0[2],xmm2[2],xmm0[3],xmm2[3],xmm0[4],xmm2[4],xmm0[5],xmm2[5],xmm0[6],xmm2[6],xmm0[7],xmm2[7]
+; SSE3-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1],xmm0[2],xmm3[2],xmm0[3],xmm3[3],xmm0[4],xmm3[4],xmm0[5],xmm3[5],xmm0[6],xmm3[6],xmm0[7],xmm3[7]
+; SSE3-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3],xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7]
+; SSE3-NEXT:    popq %rbx
+; SSE3-NEXT:    retq
+;
+; SSSE3-LABEL: testv16i8u:
+; SSSE3:       # BB#0:
+; SSSE3-NEXT:    pushq %rbx
+; SSSE3-NEXT:    movaps %xmm0, -{{[0-9]+}}(%rsp)
+; SSSE3-NEXT:    movzbl -{{[0-9]+}}(%rsp), %eax
+; SSSE3-NEXT:    bsfl %eax, %eax
+; SSSE3-NEXT:    movd %eax, %xmm0
+; SSSE3-NEXT:    movzbl -{{[0-9]+}}(%rsp), %r11d
+; SSSE3-NEXT:    movzbl -{{[0-9]+}}(%rsp), %eax
+; SSSE3-NEXT:    movzbl -{{[0-9]+}}(%rsp), %r9d
+; SSSE3-NEXT:    movzbl -{{[0-9]+}}(%rsp), %edi
+; SSSE3-NEXT:    movzbl -{{[0-9]+}}(%rsp), %r10d
+; SSSE3-NEXT:    movzbl -{{[0-9]+}}(%rsp), %ecx
+; SSSE3-NEXT:    movzbl -{{[0-9]+}}(%rsp), %r8d
+; SSSE3-NEXT:    movzbl -{{[0-9]+}}(%rsp), %edx
+; SSSE3-NEXT:    bsfl %edx, %edx
+; SSSE3-NEXT:    movd %edx, %xmm1
+; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3],xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
+; SSSE3-NEXT:    bsfl %edi, %edx
+; SSSE3-NEXT:    movd %edx, %xmm0
+; SSSE3-NEXT:    movzbl -{{[0-9]+}}(%rsp), %edx
+; SSSE3-NEXT:    movzbl -{{[0-9]+}}(%rsp), %esi
+; SSSE3-NEXT:    movzbl -{{[0-9]+}}(%rsp), %edi
+; SSSE3-NEXT:    movzbl -{{[0-9]+}}(%rsp), %ebx
+; SSSE3-NEXT:    bsfl %ebx, %ebx
+; SSSE3-NEXT:    movd %ebx, %xmm2
+; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3],xmm2[4],xmm0[4],xmm2[5],xmm0[5],xmm2[6],xmm0[6],xmm2[7],xmm0[7]
+; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3],xmm2[4],xmm1[4],xmm2[5],xmm1[5],xmm2[6],xmm1[6],xmm2[7],xmm1[7]
+; SSSE3-NEXT:    bsfl %eax, %eax
+; SSSE3-NEXT:    movd %eax, %xmm0
+; SSSE3-NEXT:    bsfl %esi, %eax
+; SSSE3-NEXT:    movd %eax, %xmm3
+; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm3 = xmm3[0],xmm0[0],xmm3[1],xmm0[1],xmm3[2],xmm0[2],xmm3[3],xmm0[3],xmm3[4],xmm0[4],xmm3[5],xmm0[5],xmm3[6],xmm0[6],xmm3[7],xmm0[7]
+; SSSE3-NEXT:    bsfl %ecx, %eax
+; SSSE3-NEXT:    movd %eax, %xmm0
+; SSSE3-NEXT:    movzbl -{{[0-9]+}}(%rsp), %eax
+; SSSE3-NEXT:    movzbl -{{[0-9]+}}(%rsp), %ecx
+; SSSE3-NEXT:    bsfl %ecx, %ecx
+; SSSE3-NEXT:    movd %ecx, %xmm1
+; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3],xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
+; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm3[0],xmm1[1],xmm3[1],xmm1[2],xmm3[2],xmm1[3],xmm3[3],xmm1[4],xmm3[4],xmm1[5],xmm3[5],xmm1[6],xmm3[6],xmm1[7],xmm3[7]
+; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1],xmm1[2],xmm2[2],xmm1[3],xmm2[3],xmm1[4],xmm2[4],xmm1[5],xmm2[5],xmm1[6],xmm2[6],xmm1[7],xmm2[7]
+; SSSE3-NEXT:    bsfl %r11d, %ecx
+; SSSE3-NEXT:    movd %ecx, %xmm0
+; SSSE3-NEXT:    bsfl %edx, %ecx
+; SSSE3-NEXT:    movd %ecx, %xmm2
+; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3],xmm2[4],xmm0[4],xmm2[5],xmm0[5],xmm2[6],xmm0[6],xmm2[7],xmm0[7]
+; SSSE3-NEXT:    bsfl %r10d, %ecx
+; SSSE3-NEXT:    movd %ecx, %xmm0
+; SSSE3-NEXT:    bsfl %eax, %eax
+; SSSE3-NEXT:    movd %eax, %xmm3
+; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm3 = xmm3[0],xmm0[0],xmm3[1],xmm0[1],xmm3[2],xmm0[2],xmm3[3],xmm0[3],xmm3[4],xmm0[4],xmm3[5],xmm0[5],xmm3[6],xmm0[6],xmm3[7],xmm0[7]
+; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm3 = xmm3[0],xmm2[0],xmm3[1],xmm2[1],xmm3[2],xmm2[2],xmm3[3],xmm2[3],xmm3[4],xmm2[4],xmm3[5],xmm2[5],xmm3[6],xmm2[6],xmm3[7],xmm2[7]
+; SSSE3-NEXT:    bsfl %r9d, %eax
+; SSSE3-NEXT:    movd %eax, %xmm0
+; SSSE3-NEXT:    bsfl %edi, %eax
+; SSSE3-NEXT:    movd %eax, %xmm2
+; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3],xmm2[4],xmm0[4],xmm2[5],xmm0[5],xmm2[6],xmm0[6],xmm2[7],xmm0[7]
+; SSSE3-NEXT:    bsfl %r8d, %eax
+; SSSE3-NEXT:    movd %eax, %xmm4
+; SSSE3-NEXT:    movzbl -{{[0-9]+}}(%rsp), %eax
+; SSSE3-NEXT:    bsfl %eax, %eax
+; SSSE3-NEXT:    movd %eax, %xmm0
+; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm4[0],xmm0[1],xmm4[1],xmm0[2],xmm4[2],xmm0[3],xmm4[3],xmm0[4],xmm4[4],xmm0[5],xmm4[5],xmm0[6],xmm4[6],xmm0[7],xmm4[7]
+; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1],xmm0[2],xmm2[2],xmm0[3],xmm2[3],xmm0[4],xmm2[4],xmm0[5],xmm2[5],xmm0[6],xmm2[6],xmm0[7],xmm2[7]
+; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1],xmm0[2],xmm3[2],xmm0[3],xmm3[3],xmm0[4],xmm3[4],xmm0[5],xmm3[5],xmm0[6],xmm3[6],xmm0[7],xmm3[7]
+; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3],xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7]
+; SSSE3-NEXT:    popq %rbx
+; SSSE3-NEXT:    retq
+;
+; SSE41-LABEL: testv16i8u:
+; SSE41:       # BB#0:
+; SSE41-NEXT:    pextrb $1, %xmm0, %eax
+; SSE41-NEXT:    bsfl %eax, %eax
+; SSE41-NEXT:    pextrb $0, %xmm0, %ecx
+; SSE41-NEXT:    bsfl %ecx, %ecx
+; SSE41-NEXT:    movd %ecx, %xmm1
+; SSE41-NEXT:    pinsrb $1, %eax, %xmm1
+; SSE41-NEXT:    pextrb $2, %xmm0, %eax
+; SSE41-NEXT:    bsfl %eax, %eax
+; SSE41-NEXT:    pinsrb $2, %eax, %xmm1
+; SSE41-NEXT:    pextrb $3, %xmm0, %eax
+; SSE41-NEXT:    bsfl %eax, %eax
+; SSE41-NEXT:    pinsrb $3, %eax, %xmm1
+; SSE41-NEXT:    pextrb $4, %xmm0, %eax
+; SSE41-NEXT:    bsfl %eax, %eax
+; SSE41-NEXT:    pinsrb $4, %eax, %xmm1
+; SSE41-NEXT:    pextrb $5, %xmm0, %eax
+; SSE41-NEXT:    bsfl %eax, %eax
+; SSE41-NEXT:    pinsrb $5, %eax, %xmm1
+; SSE41-NEXT:    pextrb $6, %xmm0, %eax
+; SSE41-NEXT:    bsfl %eax, %eax
+; SSE41-NEXT:    pinsrb $6, %eax, %xmm1
+; SSE41-NEXT:    pextrb $7, %xmm0, %eax
+; SSE41-NEXT:    bsfl %eax, %eax
+; SSE41-NEXT:    pinsrb $7, %eax, %xmm1
+; SSE41-NEXT:    pextrb $8, %xmm0, %eax
+; SSE41-NEXT:    bsfl %eax, %eax
+; SSE41-NEXT:    pinsrb $8, %eax, %xmm1
+; SSE41-NEXT:    pextrb $9, %xmm0, %eax
+; SSE41-NEXT:    bsfl %eax, %eax
+; SSE41-NEXT:    pinsrb $9, %eax, %xmm1
+; SSE41-NEXT:    pextrb $10, %xmm0, %eax
+; SSE41-NEXT:    bsfl %eax, %eax
+; SSE41-NEXT:    pinsrb $10, %eax, %xmm1
+; SSE41-NEXT:    pextrb $11, %xmm0, %eax
+; SSE41-NEXT:    bsfl %eax, %eax
+; SSE41-NEXT:    pinsrb $11, %eax, %xmm1
+; SSE41-NEXT:    pextrb $12, %xmm0, %eax
+; SSE41-NEXT:    bsfl %eax, %eax
+; SSE41-NEXT:    pinsrb $12, %eax, %xmm1
+; SSE41-NEXT:    pextrb $13, %xmm0, %eax
+; SSE41-NEXT:    bsfl %eax, %eax
+; SSE41-NEXT:    pinsrb $13, %eax, %xmm1
+; SSE41-NEXT:    pextrb $14, %xmm0, %eax
+; SSE41-NEXT:    bsfl %eax, %eax
+; SSE41-NEXT:    pinsrb $14, %eax, %xmm1
+; SSE41-NEXT:    pextrb $15, %xmm0, %eax
+; SSE41-NEXT:    bsfl %eax, %eax
+; SSE41-NEXT:    pinsrb $15, %eax, %xmm1
+; SSE41-NEXT:    movdqa %xmm1, %xmm0
+; SSE41-NEXT:    retq
+;
+; AVX-LABEL: testv16i8u:
+; AVX:       # BB#0:
+; AVX-NEXT:    vpextrb $1, %xmm0, %eax
+; AVX-NEXT:    bsfl %eax, %eax
+; AVX-NEXT:    vpextrb $0, %xmm0, %ecx
+; AVX-NEXT:    bsfl %ecx, %ecx
+; AVX-NEXT:    vmovd %ecx, %xmm1
+; AVX-NEXT:    vpinsrb $1, %eax, %xmm1, %xmm1
+; AVX-NEXT:    vpextrb $2, %xmm0, %eax
+; AVX-NEXT:    bsfl %eax, %eax
+; AVX-NEXT:    vpinsrb $2, %eax, %xmm1, %xmm1
+; AVX-NEXT:    vpextrb $3, %xmm0, %eax
+; AVX-NEXT:    bsfl %eax, %eax
+; AVX-NEXT:    vpinsrb $3, %eax, %xmm1, %xmm1
+; AVX-NEXT:    vpextrb $4, %xmm0, %eax
+; AVX-NEXT:    bsfl %eax, %eax
+; AVX-NEXT:    vpinsrb $4, %eax, %xmm1, %xmm1
+; AVX-NEXT:    vpextrb $5, %xmm0, %eax
+; AVX-NEXT:    bsfl %eax, %eax
+; AVX-NEXT:    vpinsrb $5, %eax, %xmm1, %xmm1
+; AVX-NEXT:    vpextrb $6, %xmm0, %eax
+; AVX-NEXT:    bsfl %eax, %eax
+; AVX-NEXT:    vpinsrb $6, %eax, %xmm1, %xmm1
+; AVX-NEXT:    vpextrb $7, %xmm0, %eax
+; AVX-NEXT:    bsfl %eax, %eax
+; AVX-NEXT:    vpinsrb $7, %eax, %xmm1, %xmm1
+; AVX-NEXT:    vpextrb $8, %xmm0, %eax
+; AVX-NEXT:    bsfl %eax, %eax
+; AVX-NEXT:    vpinsrb $8, %eax, %xmm1, %xmm1
+; AVX-NEXT:    vpextrb $9, %xmm0, %eax
+; AVX-NEXT:    bsfl %eax, %eax
+; AVX-NEXT:    vpinsrb $9, %eax, %xmm1, %xmm1
+; AVX-NEXT:    vpextrb $10, %xmm0, %eax
+; AVX-NEXT:    bsfl %eax, %eax
+; AVX-NEXT:    vpinsrb $10, %eax, %xmm1, %xmm1
+; AVX-NEXT:    vpextrb $11, %xmm0, %eax
+; AVX-NEXT:    bsfl %eax, %eax
+; AVX-NEXT:    vpinsrb $11, %eax, %xmm1, %xmm1
+; AVX-NEXT:    vpextrb $12, %xmm0, %eax
+; AVX-NEXT:    bsfl %eax, %eax
+; AVX-NEXT:    vpinsrb $12, %eax, %xmm1, %xmm1
+; AVX-NEXT:    vpextrb $13, %xmm0, %eax
+; AVX-NEXT:    bsfl %eax, %eax
+; AVX-NEXT:    vpinsrb $13, %eax, %xmm1, %xmm1
+; AVX-NEXT:    vpextrb $14, %xmm0, %eax
+; AVX-NEXT:    bsfl %eax, %eax
+; AVX-NEXT:    vpinsrb $14, %eax, %xmm1, %xmm1
+; AVX-NEXT:    vpextrb $15, %xmm0, %eax
+; AVX-NEXT:    bsfl %eax, %eax
+; AVX-NEXT:    vpinsrb $15, %eax, %xmm1, %xmm0
+; AVX-NEXT:    retq
+  %out = call <16 x i8> @llvm.cttz.v16i8(<16 x i8> %in, i1 -1)
+  ret <16 x i8> %out
+}
+
+define <2 x i64> @foldv2i64() nounwind {
+; SSE-LABEL: foldv2i64:
+; SSE:       # BB#0:
+; SSE-NEXT:    movl $8, %eax
+; SSE-NEXT:    movd %rax, %xmm0
+; SSE-NEXT:    retq
+;
+; AVX-LABEL: foldv2i64:
+; AVX:       # BB#0:
+; AVX-NEXT:    movl $8, %eax
+; AVX-NEXT:    vmovq %rax, %xmm0
+; AVX-NEXT:    retq
+  %out = call <2 x i64> @llvm.cttz.v2i64(<2 x i64> <i64 256, i64 -1>, i1 0)
+  ret <2 x i64> %out
+}
+
+define <2 x i64> @foldv2i64u() nounwind {
+; SSE-LABEL: foldv2i64u:
+; SSE:       # BB#0:
+; SSE-NEXT:    movl $8, %eax
+; SSE-NEXT:    movd %rax, %xmm0
+; SSE-NEXT:    retq
+;
+; AVX-LABEL: foldv2i64u:
+; AVX:       # BB#0:
+; AVX-NEXT:    movl $8, %eax
+; AVX-NEXT:    vmovq %rax, %xmm0
+; AVX-NEXT:    retq
+  %out = call <2 x i64> @llvm.cttz.v2i64(<2 x i64> <i64 256, i64 -1>, i1 -1)
+  ret <2 x i64> %out
+}
+
+define <4 x i32> @foldv4i32() nounwind {
+; SSE-LABEL: foldv4i32:
+; SSE:       # BB#0:
+; SSE-NEXT:    movaps {{.*#+}} xmm0 = [8,0,32,0]
+; SSE-NEXT:    retq
+;
+; AVX-LABEL: foldv4i32:
+; AVX:       # BB#0:
+; AVX-NEXT:    vmovaps {{.*#+}} xmm0 = [8,0,32,0]
+; AVX-NEXT:    retq
+  %out = call <4 x i32> @llvm.cttz.v4i32(<4 x i32> <i32 256, i32 -1, i32 0, i32 255>, i1 0)
+  ret <4 x i32> %out
+}
+
+define <4 x i32> @foldv4i32u() nounwind {
+; SSE-LABEL: foldv4i32u:
+; SSE:       # BB#0:
+; SSE-NEXT:    movaps {{.*#+}} xmm0 = [8,0,32,0]
+; SSE-NEXT:    retq
+;
+; AVX-LABEL: foldv4i32u:
+; AVX:       # BB#0:
+; AVX-NEXT:    vmovaps {{.*#+}} xmm0 = [8,0,32,0]
+; AVX-NEXT:    retq
+  %out = call <4 x i32> @llvm.cttz.v4i32(<4 x i32> <i32 256, i32 -1, i32 0, i32 255>, i1 -1)
+  ret <4 x i32> %out
+}
+
+define <8 x i16> @foldv8i16() nounwind {
+; SSE-LABEL: foldv8i16:
+; SSE:       # BB#0:
+; SSE-NEXT:    movaps {{.*#+}} xmm0 = [8,0,16,0,16,0,3,3]
+; SSE-NEXT:    retq
+;
+; AVX-LABEL: foldv8i16:
+; AVX:       # BB#0:
+; AVX-NEXT:    vmovaps {{.*#+}} xmm0 = [8,0,16,0,16,0,3,3]
+; AVX-NEXT:    retq
+  %out = call <8 x i16> @llvm.cttz.v8i16(<8 x i16> <i16 256, i16 -1, i16 0, i16 255, i16 -65536, i16 7, i16 24, i16 88>, i1 0)
+  ret <8 x i16> %out
+}
+
+define <8 x i16> @foldv8i16u() nounwind {
+; SSE-LABEL: foldv8i16u:
+; SSE:       # BB#0:
+; SSE-NEXT:    movaps {{.*#+}} xmm0 = [8,0,16,0,16,0,3,3]
+; SSE-NEXT:    retq
+;
+; AVX-LABEL: foldv8i16u:
+; AVX:       # BB#0:
+; AVX-NEXT:    vmovaps {{.*#+}} xmm0 = [8,0,16,0,16,0,3,3]
+; AVX-NEXT:    retq
+  %out = call <8 x i16> @llvm.cttz.v8i16(<8 x i16> <i16 256, i16 -1, i16 0, i16 255, i16 -65536, i16 7, i16 24, i16 88>, i1 -1)
+  ret <8 x i16> %out
+}
+
+define <16 x i8> @foldv16i8() nounwind {
+; SSE-LABEL: foldv16i8:
+; SSE:       # BB#0:
+; SSE-NEXT:    movaps {{.*#+}} xmm0 = [8,0,8,0,8,0,3,3,1,1,0,1,2,3,4,5]
+; SSE-NEXT:    retq
+;
+; AVX-LABEL: foldv16i8:
+; AVX:       # BB#0:
+; AVX-NEXT:    vmovaps {{.*#+}} xmm0 = [8,0,8,0,8,0,3,3,1,1,0,1,2,3,4,5]
+; AVX-NEXT:    retq
+  %out = call <16 x i8> @llvm.cttz.v16i8(<16 x i8> <i8 256, i8 -1, i8 0, i8 255, i8 -65536, i8 7, i8 24, i8 88, i8 -2, i8 254, i8 1, i8 2, i8 4, i8 8, i8 16, i8 32>, i1 0)
+  ret <16 x i8> %out
+}
+
+define <16 x i8> @foldv16i8u() nounwind {
+; SSE-LABEL: foldv16i8u:
+; SSE:       # BB#0:
+; SSE-NEXT:    movaps {{.*#+}} xmm0 = [8,0,8,0,8,0,3,3,1,1,0,1,2,3,4,5]
+; SSE-NEXT:    retq
+;
+; AVX-LABEL: foldv16i8u:
+; AVX:       # BB#0:
+; AVX-NEXT:    vmovaps {{.*#+}} xmm0 = [8,0,8,0,8,0,3,3,1,1,0,1,2,3,4,5]
+; AVX-NEXT:    retq
+  %out = call <16 x i8> @llvm.cttz.v16i8(<16 x i8> <i8 256, i8 -1, i8 0, i8 255, i8 -65536, i8 7, i8 24, i8 88, i8 -2, i8 254, i8 1, i8 2, i8 4, i8 8, i8 16, i8 32>, i1 -1)
+  ret <16 x i8> %out
+}
+
+declare <2 x i64> @llvm.cttz.v2i64(<2 x i64>, i1)
+declare <4 x i32> @llvm.cttz.v4i32(<4 x i32>, i1)
+declare <8 x i16> @llvm.cttz.v8i16(<8 x i16>, i1)
+declare <16 x i8> @llvm.cttz.v16i8(<16 x i8>, i1)
diff --git a/test/CodeGen/X86/vector-tzcnt-256.ll b/test/CodeGen/X86/vector-tzcnt-256.ll
new file mode 100644
index 00000000000..872bd416d28
--- /dev/null
+++ b/test/CodeGen/X86/vector-tzcnt-256.ll
@@ -0,0 +1,1195 @@
+; RUN: llc < %s -mcpu=x86-64 -mattr=+avx | FileCheck %s --check-prefix=ALL --check-prefix=AVX --check-prefix=AVX1
+; RUN: llc < %s -mcpu=x86-64 -mattr=+avx2 | FileCheck %s --check-prefix=ALL --check-prefix=AVX --check-prefix=AVX2
+
+target triple = "x86_64-unknown-unknown"
+
+define <4 x i64> @testv4i64(<4 x i64> %in) nounwind {
+; AVX1-LABEL: testv4i64:
+; AVX1:       # BB#0:
+; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
+; AVX1-NEXT:    vpextrq $1, %xmm1, %rax
+; AVX1-NEXT:    bsfq %rax, %rax
+; AVX1-NEXT:    movl $64, %ecx
+; AVX1-NEXT:    cmoveq %rcx, %rax
+; AVX1-NEXT:    vmovq %rax, %xmm2
+; AVX1-NEXT:    vmovq %xmm1, %rax
+; AVX1-NEXT:    bsfq %rax, %rax
+; AVX1-NEXT:    cmoveq %rcx, %rax
+; AVX1-NEXT:    vmovq %rax, %xmm1
+; AVX1-NEXT:    vpunpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm2[0]
+; AVX1-NEXT:    vpextrq $1, %xmm0, %rax
+; AVX1-NEXT:    bsfq %rax, %rax
+; AVX1-NEXT:    cmoveq %rcx, %rax
+; AVX1-NEXT:    vmovq %rax, %xmm2
+; AVX1-NEXT:    vmovq %xmm0, %rax
+; AVX1-NEXT:    bsfq %rax, %rax
+; AVX1-NEXT:    cmoveq %rcx, %rax
+; AVX1-NEXT:    vmovq %rax, %xmm0
+; AVX1-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm2[0]
+; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: testv4i64:
+; AVX2:       # BB#0:
+; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
+; AVX2-NEXT:    vpextrq $1, %xmm1, %rax
+; AVX2-NEXT:    bsfq %rax, %rax
+; AVX2-NEXT:    movl $64, %ecx
+; AVX2-NEXT:    cmoveq %rcx, %rax
+; AVX2-NEXT:    vmovq %rax, %xmm2
+; AVX2-NEXT:    vmovq %xmm1, %rax
+; AVX2-NEXT:    bsfq %rax, %rax
+; AVX2-NEXT:    cmoveq %rcx, %rax
+; AVX2-NEXT:    vmovq %rax, %xmm1
+; AVX2-NEXT:    vpunpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm2[0]
+; AVX2-NEXT:    vpextrq $1, %xmm0, %rax
+; AVX2-NEXT:    bsfq %rax, %rax
+; AVX2-NEXT:    cmoveq %rcx, %rax
+; AVX2-NEXT:    vmovq %rax, %xmm2
+; AVX2-NEXT:    vmovq %xmm0, %rax
+; AVX2-NEXT:    bsfq %rax, %rax
+; AVX2-NEXT:    cmoveq %rcx, %rax
+; AVX2-NEXT:    vmovq %rax, %xmm0
+; AVX2-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm2[0]
+; AVX2-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm0
+; AVX2-NEXT:    retq
+  %out = call <4 x i64> @llvm.cttz.v4i64(<4 x i64> %in, i1 0)
+  ret <4 x i64> %out
+}
+
+define <4 x i64> @testv4i64u(<4 x i64> %in) nounwind {
+; AVX1-LABEL: testv4i64u:
+; AVX1:       # BB#0:
+; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
+; AVX1-NEXT:    vpextrq $1, %xmm1, %rax
+; AVX1-NEXT:    bsfq %rax, %rax
+; AVX1-NEXT:    vmovq %rax, %xmm2
+; AVX1-NEXT:    vmovq %xmm1, %rax
+; AVX1-NEXT:    bsfq %rax, %rax
+; AVX1-NEXT:    vmovq %rax, %xmm1
+; AVX1-NEXT:    vpunpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm2[0]
+; AVX1-NEXT:    vpextrq $1, %xmm0, %rax
+; AVX1-NEXT:    bsfq %rax, %rax
+; AVX1-NEXT:    vmovq %rax, %xmm2
+; AVX1-NEXT:    vmovq %xmm0, %rax
+; AVX1-NEXT:    bsfq %rax, %rax
+; AVX1-NEXT:    vmovq %rax, %xmm0
+; AVX1-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm2[0]
+; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: testv4i64u:
+; AVX2:       # BB#0:
+; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
+; AVX2-NEXT:    vpextrq $1, %xmm1, %rax
+; AVX2-NEXT:    bsfq %rax, %rax
+; AVX2-NEXT:    vmovq %rax, %xmm2
+; AVX2-NEXT:    vmovq %xmm1, %rax
+; AVX2-NEXT:    bsfq %rax, %rax
+; AVX2-NEXT:    vmovq %rax, %xmm1
+; AVX2-NEXT:    vpunpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm2[0]
+; AVX2-NEXT:    vpextrq $1, %xmm0, %rax
+; AVX2-NEXT:    bsfq %rax, %rax
+; AVX2-NEXT:    vmovq %rax, %xmm2
+; AVX2-NEXT:    vmovq %xmm0, %rax
+; AVX2-NEXT:    bsfq %rax, %rax
+; AVX2-NEXT:    vmovq %rax, %xmm0
+; AVX2-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm2[0]
+; AVX2-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm0
+; AVX2-NEXT:    retq
+  %out = call <4 x i64> @llvm.cttz.v4i64(<4 x i64> %in, i1 -1)
+  ret <4 x i64> %out
+}
+
+define <8 x i32> @testv8i32(<8 x i32> %in) nounwind {
+; AVX1-LABEL: testv8i32:
+; AVX1:       # BB#0:
+; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
+; AVX1-NEXT:    vpextrd $1, %xmm1, %eax
+; AVX1-NEXT:    bsfl %eax, %ecx
+; AVX1-NEXT:    movl $32, %eax
+; AVX1-NEXT:    cmovel %eax, %ecx
+; AVX1-NEXT:    vmovd %xmm1, %edx
+; AVX1-NEXT:    bsfl %edx, %edx
+; AVX1-NEXT:    cmovel %eax, %edx
+; AVX1-NEXT:    vmovd %edx, %xmm2
+; AVX1-NEXT:    vpinsrd $1, %ecx, %xmm2, %xmm2
+; AVX1-NEXT:    vpextrd $2, %xmm1, %ecx
+; AVX1-NEXT:    bsfl %ecx, %ecx
+; AVX1-NEXT:    cmovel %eax, %ecx
+; AVX1-NEXT:    vpinsrd $2, %ecx, %xmm2, %xmm2
+; AVX1-NEXT:    vpextrd $3, %xmm1, %ecx
+; AVX1-NEXT:    bsfl %ecx, %ecx
+; AVX1-NEXT:    cmovel %eax, %ecx
+; AVX1-NEXT:    vpinsrd $3, %ecx, %xmm2, %xmm1
+; AVX1-NEXT:    vpextrd $1, %xmm0, %ecx
+; AVX1-NEXT:    bsfl %ecx, %ecx
+; AVX1-NEXT:    cmovel %eax, %ecx
+; AVX1-NEXT:    vmovd %xmm0, %edx
+; AVX1-NEXT:    bsfl %edx, %edx
+; AVX1-NEXT:    cmovel %eax, %edx
+; AVX1-NEXT:    vmovd %edx, %xmm2
+; AVX1-NEXT:    vpinsrd $1, %ecx, %xmm2, %xmm2
+; AVX1-NEXT:    vpextrd $2, %xmm0, %ecx
+; AVX1-NEXT:    bsfl %ecx, %ecx
+; AVX1-NEXT:    cmovel %eax, %ecx
+; AVX1-NEXT:    vpinsrd $2, %ecx, %xmm2, %xmm2
+; AVX1-NEXT:    vpextrd $3, %xmm0, %ecx
+; AVX1-NEXT:    bsfl %ecx, %ecx
+; AVX1-NEXT:    cmovel %eax, %ecx
+; AVX1-NEXT:    vpinsrd $3, %ecx, %xmm2, %xmm0
+; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: testv8i32:
+; AVX2:       # BB#0:
+; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
+; AVX2-NEXT:    vpextrd $1, %xmm1, %eax
+; AVX2-NEXT:    bsfl %eax, %ecx
+; AVX2-NEXT:    movl $32, %eax
+; AVX2-NEXT:    cmovel %eax, %ecx
+; AVX2-NEXT:    vmovd %xmm1, %edx
+; AVX2-NEXT:    bsfl %edx, %edx
+; AVX2-NEXT:    cmovel %eax, %edx
+; AVX2-NEXT:    vmovd %edx, %xmm2
+; AVX2-NEXT:    vpinsrd $1, %ecx, %xmm2, %xmm2
+; AVX2-NEXT:    vpextrd $2, %xmm1, %ecx
+; AVX2-NEXT:    bsfl %ecx, %ecx
+; AVX2-NEXT:    cmovel %eax, %ecx
+; AVX2-NEXT:    vpinsrd $2, %ecx, %xmm2, %xmm2
+; AVX2-NEXT:    vpextrd $3, %xmm1, %ecx
+; AVX2-NEXT:    bsfl %ecx, %ecx
+; AVX2-NEXT:    cmovel %eax, %ecx
+; AVX2-NEXT:    vpinsrd $3, %ecx, %xmm2, %xmm1
+; AVX2-NEXT:    vpextrd $1, %xmm0, %ecx
+; AVX2-NEXT:    bsfl %ecx, %ecx
+; AVX2-NEXT:    cmovel %eax, %ecx
+; AVX2-NEXT:    vmovd %xmm0, %edx
+; AVX2-NEXT:    bsfl %edx, %edx
+; AVX2-NEXT:    cmovel %eax, %edx
+; AVX2-NEXT:    vmovd %edx, %xmm2
+; AVX2-NEXT:    vpinsrd $1, %ecx, %xmm2, %xmm2
+; AVX2-NEXT:    vpextrd $2, %xmm0, %ecx
+; AVX2-NEXT:    bsfl %ecx, %ecx
+; AVX2-NEXT:    cmovel %eax, %ecx
+; AVX2-NEXT:    vpinsrd $2, %ecx, %xmm2, %xmm2
+; AVX2-NEXT:    vpextrd $3, %xmm0, %ecx
+; AVX2-NEXT:    bsfl %ecx, %ecx
+; AVX2-NEXT:    cmovel %eax, %ecx
+; AVX2-NEXT:    vpinsrd $3, %ecx, %xmm2, %xmm0
+; AVX2-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm0
+; AVX2-NEXT:    retq
+  %out = call <8 x i32> @llvm.cttz.v8i32(<8 x i32> %in, i1 0)
+  ret <8 x i32> %out
+}
+
+define <8 x i32> @testv8i32u(<8 x i32> %in) nounwind {
+; AVX1-LABEL: testv8i32u:
+; AVX1:       # BB#0:
+; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
+; AVX1-NEXT:    vpextrd $1, %xmm1, %eax
+; AVX1-NEXT:    bsfl %eax, %eax
+; AVX1-NEXT:    vmovd %xmm1, %ecx
+; AVX1-NEXT:    bsfl %ecx, %ecx
+; AVX1-NEXT:    vmovd %ecx, %xmm2
+; AVX1-NEXT:    vpinsrd $1, %eax, %xmm2, %xmm2
+; AVX1-NEXT:    vpextrd $2, %xmm1, %eax
+; AVX1-NEXT:    bsfl %eax, %eax
+; AVX1-NEXT:    vpinsrd $2, %eax, %xmm2, %xmm2
+; AVX1-NEXT:    vpextrd $3, %xmm1, %eax
+; AVX1-NEXT:    bsfl %eax, %eax
+; AVX1-NEXT:    vpinsrd $3, %eax, %xmm2, %xmm1
+; AVX1-NEXT:    vpextrd $1, %xmm0, %eax
+; AVX1-NEXT:    bsfl %eax, %eax
+; AVX1-NEXT:    vmovd %xmm0, %ecx
+; AVX1-NEXT:    bsfl %ecx, %ecx
+; AVX1-NEXT:    vmovd %ecx, %xmm2
+; AVX1-NEXT:    vpinsrd $1, %eax, %xmm2, %xmm2
+; AVX1-NEXT:    vpextrd $2, %xmm0, %eax
+; AVX1-NEXT:    bsfl %eax, %eax
+; AVX1-NEXT:    vpinsrd $2, %eax, %xmm2, %xmm2
+; AVX1-NEXT:    vpextrd $3, %xmm0, %eax
+; AVX1-NEXT:    bsfl %eax, %eax
+; AVX1-NEXT:    vpinsrd $3, %eax, %xmm2, %xmm0
+; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: testv8i32u:
+; AVX2:       # BB#0:
+; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
+; AVX2-NEXT:    vpextrd $1, %xmm1, %eax
+; AVX2-NEXT:    bsfl %eax, %eax
+; AVX2-NEXT:    vmovd %xmm1, %ecx
+; AVX2-NEXT:    bsfl %ecx, %ecx
+; AVX2-NEXT:    vmovd %ecx, %xmm2
+; AVX2-NEXT:    vpinsrd $1, %eax, %xmm2, %xmm2
+; AVX2-NEXT:    vpextrd $2, %xmm1, %eax
+; AVX2-NEXT:    bsfl %eax, %eax
+; AVX2-NEXT:    vpinsrd $2, %eax, %xmm2, %xmm2
+; AVX2-NEXT:    vpextrd $3, %xmm1, %eax
+; AVX2-NEXT:    bsfl %eax, %eax
+; AVX2-NEXT:    vpinsrd $3, %eax, %xmm2, %xmm1
+; AVX2-NEXT:    vpextrd $1, %xmm0, %eax
+; AVX2-NEXT:    bsfl %eax, %eax
+; AVX2-NEXT:    vmovd %xmm0, %ecx
+; AVX2-NEXT:    bsfl %ecx, %ecx
+; AVX2-NEXT:    vmovd %ecx, %xmm2
+; AVX2-NEXT:    vpinsrd $1, %eax, %xmm2, %xmm2
+; AVX2-NEXT:    vpextrd $2, %xmm0, %eax
+; AVX2-NEXT:    bsfl %eax, %eax
+; AVX2-NEXT:    vpinsrd $2, %eax, %xmm2, %xmm2
+; AVX2-NEXT:    vpextrd $3, %xmm0, %eax
+; AVX2-NEXT:    bsfl %eax, %eax
+; AVX2-NEXT:    vpinsrd $3, %eax, %xmm2, %xmm0
+; AVX2-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm0
+; AVX2-NEXT:    retq
+  %out = call <8 x i32> @llvm.cttz.v8i32(<8 x i32> %in, i1 -1)
+  ret <8 x i32> %out
+}
+
+define <16 x i16> @testv16i16(<16 x i16> %in) nounwind {
+; AVX1-LABEL: testv16i16:
+; AVX1:       # BB#0:
+; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
+; AVX1-NEXT:    vpextrw $1, %xmm1, %eax
+; AVX1-NEXT:    bsfw %ax, %cx
+; AVX1-NEXT:    movw $16, %ax
+; AVX1-NEXT:    cmovew %ax, %cx
+; AVX1-NEXT:    vmovd %xmm1, %edx
+; AVX1-NEXT:    bsfw %dx, %dx
+; AVX1-NEXT:    cmovew %ax, %dx
+; AVX1-NEXT:    vmovd %edx, %xmm2
+; AVX1-NEXT:    vpinsrw $1, %ecx, %xmm2, %xmm2
+; AVX1-NEXT:    vpextrw $2, %xmm1, %ecx
+; AVX1-NEXT:    bsfw %cx, %cx
+; AVX1-NEXT:    cmovew %ax, %cx
+; AVX1-NEXT:    vpinsrw $2, %ecx, %xmm2, %xmm2
+; AVX1-NEXT:    vpextrw $3, %xmm1, %ecx
+; AVX1-NEXT:    bsfw %cx, %cx
+; AVX1-NEXT:    cmovew %ax, %cx
+; AVX1-NEXT:    vpinsrw $3, %ecx, %xmm2, %xmm2
+; AVX1-NEXT:    vpextrw $4, %xmm1, %ecx
+; AVX1-NEXT:    bsfw %cx, %cx
+; AVX1-NEXT:    cmovew %ax, %cx
+; AVX1-NEXT:    vpinsrw $4, %ecx, %xmm2, %xmm2
+; AVX1-NEXT:    vpextrw $5, %xmm1, %ecx
+; AVX1-NEXT:    bsfw %cx, %cx
+; AVX1-NEXT:    cmovew %ax, %cx
+; AVX1-NEXT:    vpinsrw $5, %ecx, %xmm2, %xmm2
+; AVX1-NEXT:    vpextrw $6, %xmm1, %ecx
+; AVX1-NEXT:    bsfw %cx, %cx
+; AVX1-NEXT:    cmovew %ax, %cx
+; AVX1-NEXT:    vpinsrw $6, %ecx, %xmm2, %xmm2
+; AVX1-NEXT:    vpextrw $7, %xmm1, %ecx
+; AVX1-NEXT:    bsfw %cx, %cx
+; AVX1-NEXT:    cmovew %ax, %cx
+; AVX1-NEXT:    vpinsrw $7, %ecx, %xmm2, %xmm1
+; AVX1-NEXT:    vpextrw $1, %xmm0, %ecx
+; AVX1-NEXT:    bsfw %cx, %cx
+; AVX1-NEXT:    cmovew %ax, %cx
+; AVX1-NEXT:    vmovd %xmm0, %edx
+; AVX1-NEXT:    bsfw %dx, %dx
+; AVX1-NEXT:    cmovew %ax, %dx
+; AVX1-NEXT:    vmovd %edx, %xmm2
+; AVX1-NEXT:    vpinsrw $1, %ecx, %xmm2, %xmm2
+; AVX1-NEXT:    vpextrw $2, %xmm0, %ecx
+; AVX1-NEXT:    bsfw %cx, %cx
+; AVX1-NEXT:    cmovew %ax, %cx
+; AVX1-NEXT:    vpinsrw $2, %ecx, %xmm2, %xmm2
+; AVX1-NEXT:    vpextrw $3, %xmm0, %ecx
+; AVX1-NEXT:    bsfw %cx, %cx
+; AVX1-NEXT:    cmovew %ax, %cx
+; AVX1-NEXT:    vpinsrw $3, %ecx, %xmm2, %xmm2
+; AVX1-NEXT:    vpextrw $4, %xmm0, %ecx
+; AVX1-NEXT:    bsfw %cx, %cx
+; AVX1-NEXT:    cmovew %ax, %cx
+; AVX1-NEXT:    vpinsrw $4, %ecx, %xmm2, %xmm2
+; AVX1-NEXT:    vpextrw $5, %xmm0, %ecx
+; AVX1-NEXT:    bsfw %cx, %cx
+; AVX1-NEXT:    cmovew %ax, %cx
+; AVX1-NEXT:    vpinsrw $5, %ecx, %xmm2, %xmm2
+; AVX1-NEXT:    vpextrw $6, %xmm0, %ecx
+; AVX1-NEXT:    bsfw %cx, %cx
+; AVX1-NEXT:    cmovew %ax, %cx
+; AVX1-NEXT:    vpinsrw $6, %ecx, %xmm2, %xmm2
+; AVX1-NEXT:    vpextrw $7, %xmm0, %ecx
+; AVX1-NEXT:    bsfw %cx, %cx
+; AVX1-NEXT:    cmovew %ax, %cx
+; AVX1-NEXT:    vpinsrw $7, %ecx, %xmm2, %xmm0
+; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: testv16i16:
+; AVX2:       # BB#0:
+; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
+; AVX2-NEXT:    vpextrw $1, %xmm1, %eax
+; AVX2-NEXT:    bsfw %ax, %cx
+; AVX2-NEXT:    movw $16, %ax
+; AVX2-NEXT:    cmovew %ax, %cx
+; AVX2-NEXT:    vmovd %xmm1, %edx
+; AVX2-NEXT:    bsfw %dx, %dx
+; AVX2-NEXT:    cmovew %ax, %dx
+; AVX2-NEXT:    vmovd %edx, %xmm2
+; AVX2-NEXT:    vpinsrw $1, %ecx, %xmm2, %xmm2
+; AVX2-NEXT:    vpextrw $2, %xmm1, %ecx
+; AVX2-NEXT:    bsfw %cx, %cx
+; AVX2-NEXT:    cmovew %ax, %cx
+; AVX2-NEXT:    vpinsrw $2, %ecx, %xmm2, %xmm2
+; AVX2-NEXT:    vpextrw $3, %xmm1, %ecx
+; AVX2-NEXT:    bsfw %cx, %cx
+; AVX2-NEXT:    cmovew %ax, %cx
+; AVX2-NEXT:    vpinsrw $3, %ecx, %xmm2, %xmm2
+; AVX2-NEXT:    vpextrw $4, %xmm1, %ecx
+; AVX2-NEXT:    bsfw %cx, %cx
+; AVX2-NEXT:    cmovew %ax, %cx
+; AVX2-NEXT:    vpinsrw $4, %ecx, %xmm2, %xmm2
+; AVX2-NEXT:    vpextrw $5, %xmm1, %ecx
+; AVX2-NEXT:    bsfw %cx, %cx
+; AVX2-NEXT:    cmovew %ax, %cx
+; AVX2-NEXT:    vpinsrw $5, %ecx, %xmm2, %xmm2
+; AVX2-NEXT:    vpextrw $6, %xmm1, %ecx
+; AVX2-NEXT:    bsfw %cx, %cx
+; AVX2-NEXT:    cmovew %ax, %cx
+; AVX2-NEXT:    vpinsrw $6, %ecx, %xmm2, %xmm2
+; AVX2-NEXT:    vpextrw $7, %xmm1, %ecx
+; AVX2-NEXT:    bsfw %cx, %cx
+; AVX2-NEXT:    cmovew %ax, %cx
+; AVX2-NEXT:    vpinsrw $7, %ecx, %xmm2, %xmm1
+; AVX2-NEXT:    vpextrw $1, %xmm0, %ecx
+; AVX2-NEXT:    bsfw %cx, %cx
+; AVX2-NEXT:    cmovew %ax, %cx
+; AVX2-NEXT:    vmovd %xmm0, %edx
+; AVX2-NEXT:    bsfw %dx, %dx
+; AVX2-NEXT:    cmovew %ax, %dx
+; AVX2-NEXT:    vmovd %edx, %xmm2
+; AVX2-NEXT:    vpinsrw $1, %ecx, %xmm2, %xmm2
+; AVX2-NEXT:    vpextrw $2, %xmm0, %ecx
+; AVX2-NEXT:    bsfw %cx, %cx
+; AVX2-NEXT:    cmovew %ax, %cx
+; AVX2-NEXT:    vpinsrw $2, %ecx, %xmm2, %xmm2
+; AVX2-NEXT:    vpextrw $3, %xmm0, %ecx
+; AVX2-NEXT:    bsfw %cx, %cx
+; AVX2-NEXT:    cmovew %ax, %cx
+; AVX2-NEXT:    vpinsrw $3, %ecx, %xmm2, %xmm2
+; AVX2-NEXT:    vpextrw $4, %xmm0, %ecx
+; AVX2-NEXT:    bsfw %cx, %cx
+; AVX2-NEXT:    cmovew %ax, %cx
+; AVX2-NEXT:    vpinsrw $4, %ecx, %xmm2, %xmm2
+; AVX2-NEXT:    vpextrw $5, %xmm0, %ecx
+; AVX2-NEXT:    bsfw %cx, %cx
+; AVX2-NEXT:    cmovew %ax, %cx
+; AVX2-NEXT:    vpinsrw $5, %ecx, %xmm2, %xmm2
+; AVX2-NEXT:    vpextrw $6, %xmm0, %ecx
+; AVX2-NEXT:    bsfw %cx, %cx
+; AVX2-NEXT:    cmovew %ax, %cx
+; AVX2-NEXT:    vpinsrw $6, %ecx, %xmm2, %xmm2
+; AVX2-NEXT:    vpextrw $7, %xmm0, %ecx
+; AVX2-NEXT:    bsfw %cx, %cx
+; AVX2-NEXT:    cmovew %ax, %cx
+; AVX2-NEXT:    vpinsrw $7, %ecx, %xmm2, %xmm0
+; AVX2-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm0
+; AVX2-NEXT:    retq
+  %out = call <16 x i16> @llvm.cttz.v16i16(<16 x i16> %in, i1 0)
+  ret <16 x i16> %out
+}
+
+define <16 x i16> @testv16i16u(<16 x i16> %in) nounwind {
+; AVX1-LABEL: testv16i16u:
+; AVX1:       # BB#0:
+; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
+; AVX1-NEXT:    vpextrw $1, %xmm1, %eax
+; AVX1-NEXT:    bsfw %ax, %ax
+; AVX1-NEXT:    vmovd %xmm1, %ecx
+; AVX1-NEXT:    bsfw %cx, %cx
+; AVX1-NEXT:    vmovd %ecx, %xmm2
+; AVX1-NEXT:    vpinsrw $1, %eax, %xmm2, %xmm2
+; AVX1-NEXT:    vpextrw $2, %xmm1, %eax
+; AVX1-NEXT:    bsfw %ax, %ax
+; AVX1-NEXT:    vpinsrw $2, %eax, %xmm2, %xmm2
+; AVX1-NEXT:    vpextrw $3, %xmm1, %eax
+; AVX1-NEXT:    bsfw %ax, %ax
+; AVX1-NEXT:    vpinsrw $3, %eax, %xmm2, %xmm2
+; AVX1-NEXT:    vpextrw $4, %xmm1, %eax
+; AVX1-NEXT:    bsfw %ax, %ax
+; AVX1-NEXT:    vpinsrw $4, %eax, %xmm2, %xmm2
+; AVX1-NEXT:    vpextrw $5, %xmm1, %eax
+; AVX1-NEXT:    bsfw %ax, %ax
+; AVX1-NEXT:    vpinsrw $5, %eax, %xmm2, %xmm2
+; AVX1-NEXT:    vpextrw $6, %xmm1, %eax
+; AVX1-NEXT:    bsfw %ax, %ax
+; AVX1-NEXT:    vpinsrw $6, %eax, %xmm2, %xmm2
+; AVX1-NEXT:    vpextrw $7, %xmm1, %eax
+; AVX1-NEXT:    bsfw %ax, %ax
+; AVX1-NEXT:    vpinsrw $7, %eax, %xmm2, %xmm1
+; AVX1-NEXT:    vpextrw $1, %xmm0, %eax
+; AVX1-NEXT:    bsfw %ax, %ax
+; AVX1-NEXT:    vmovd %xmm0, %ecx
+; AVX1-NEXT:    bsfw %cx, %cx
+; AVX1-NEXT:    vmovd %ecx, %xmm2
+; AVX1-NEXT:    vpinsrw $1, %eax, %xmm2, %xmm2
+; AVX1-NEXT:    vpextrw $2, %xmm0, %eax
+; AVX1-NEXT:    bsfw %ax, %ax
+; AVX1-NEXT:    vpinsrw $2, %eax, %xmm2, %xmm2
+; AVX1-NEXT:    vpextrw $3, %xmm0, %eax
+; AVX1-NEXT:    bsfw %ax, %ax
+; AVX1-NEXT:    vpinsrw $3, %eax, %xmm2, %xmm2
+; AVX1-NEXT:    vpextrw $4, %xmm0, %eax
+; AVX1-NEXT:    bsfw %ax, %ax
+; AVX1-NEXT:    vpinsrw $4, %eax, %xmm2, %xmm2
+; AVX1-NEXT:    vpextrw $5, %xmm0, %eax
+; AVX1-NEXT:    bsfw %ax, %ax
+; AVX1-NEXT:    vpinsrw $5, %eax, %xmm2, %xmm2
+; AVX1-NEXT:    vpextrw $6, %xmm0, %eax
+; AVX1-NEXT:    bsfw %ax, %ax
+; AVX1-NEXT:    vpinsrw $6, %eax, %xmm2, %xmm2
+; AVX1-NEXT:    vpextrw $7, %xmm0, %eax
+; AVX1-NEXT:    bsfw %ax, %ax
+; AVX1-NEXT:    vpinsrw $7, %eax, %xmm2, %xmm0
+; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: testv16i16u:
+; AVX2:       # BB#0:
+; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
+; AVX2-NEXT:    vpextrw $1, %xmm1, %eax
+; AVX2-NEXT:    bsfw %ax, %ax
+; AVX2-NEXT:    vmovd %xmm1, %ecx
+; AVX2-NEXT:    bsfw %cx, %cx
+; AVX2-NEXT:    vmovd %ecx, %xmm2
+; AVX2-NEXT:    vpinsrw $1, %eax, %xmm2, %xmm2
+; AVX2-NEXT:    vpextrw $2, %xmm1, %eax
+; AVX2-NEXT:    bsfw %ax, %ax
+; AVX2-NEXT:    vpinsrw $2, %eax, %xmm2, %xmm2
+; AVX2-NEXT:    vpextrw $3, %xmm1, %eax
+; AVX2-NEXT:    bsfw %ax, %ax
+; AVX2-NEXT:    vpinsrw $3, %eax, %xmm2, %xmm2
+; AVX2-NEXT:    vpextrw $4, %xmm1, %eax
+; AVX2-NEXT:    bsfw %ax, %ax
+; AVX2-NEXT:    vpinsrw $4, %eax, %xmm2, %xmm2
+; AVX2-NEXT:    vpextrw $5, %xmm1, %eax
+; AVX2-NEXT:    bsfw %ax, %ax
+; AVX2-NEXT:    vpinsrw $5, %eax, %xmm2, %xmm2
+; AVX2-NEXT:    vpextrw $6, %xmm1, %eax
+; AVX2-NEXT:    bsfw %ax, %ax
+; AVX2-NEXT:    vpinsrw $6, %eax, %xmm2, %xmm2
+; AVX2-NEXT:    vpextrw $7, %xmm1, %eax
+; AVX2-NEXT:    bsfw %ax, %ax
+; AVX2-NEXT:    vpinsrw $7, %eax, %xmm2, %xmm1
+; AVX2-NEXT:    vpextrw $1, %xmm0, %eax
+; AVX2-NEXT:    bsfw %ax, %ax
+; AVX2-NEXT:    vmovd %xmm0, %ecx
+; AVX2-NEXT:    bsfw %cx, %cx
+; AVX2-NEXT:    vmovd %ecx, %xmm2
+; AVX2-NEXT:    vpinsrw $1, %eax, %xmm2, %xmm2
+; AVX2-NEXT:    vpextrw $2, %xmm0, %eax
+; AVX2-NEXT:    bsfw %ax, %ax
+; AVX2-NEXT:    vpinsrw $2, %eax, %xmm2, %xmm2
+; AVX2-NEXT:    vpextrw $3, %xmm0, %eax
+; AVX2-NEXT:    bsfw %ax, %ax
+; AVX2-NEXT:    vpinsrw $3, %eax, %xmm2, %xmm2
+; AVX2-NEXT:    vpextrw $4, %xmm0, %eax
+; AVX2-NEXT:    bsfw %ax, %ax
+; AVX2-NEXT:    vpinsrw $4, %eax, %xmm2, %xmm2
+; AVX2-NEXT:    vpextrw $5, %xmm0, %eax
+; AVX2-NEXT:    bsfw %ax, %ax
+; AVX2-NEXT:    vpinsrw $5, %eax, %xmm2, %xmm2
+; AVX2-NEXT:    vpextrw $6, %xmm0, %eax
+; AVX2-NEXT:    bsfw %ax, %ax
+; AVX2-NEXT:    vpinsrw $6, %eax, %xmm2, %xmm2
+; AVX2-NEXT:    vpextrw $7, %xmm0, %eax
+; AVX2-NEXT:    bsfw %ax, %ax
+; AVX2-NEXT:    vpinsrw $7, %eax, %xmm2, %xmm0
+; AVX2-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm0
+; AVX2-NEXT:    retq
+  %out = call <16 x i16> @llvm.cttz.v16i16(<16 x i16> %in, i1 -1)
+  ret <16 x i16> %out
+}
+
+define <32 x i8> @testv32i8(<32 x i8> %in) nounwind {
+; AVX1-LABEL: testv32i8:
+; AVX1:       # BB#0:
+; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
+; AVX1-NEXT:    vpextrb $1, %xmm1, %eax
+; AVX1-NEXT:    bsfl %eax, %edx
+; AVX1-NEXT:    movl $32, %eax
+; AVX1-NEXT:    cmovel %eax, %edx
+; AVX1-NEXT:    cmpl $32, %edx
+; AVX1-NEXT:    movl $8, %ecx
+; AVX1-NEXT:    cmovel %ecx, %edx
+; AVX1-NEXT:    vpextrb $0, %xmm1, %esi
+; AVX1-NEXT:    bsfl %esi, %esi
+; AVX1-NEXT:    cmovel %eax, %esi
+; AVX1-NEXT:    cmpl $32, %esi
+; AVX1-NEXT:    cmovel %ecx, %esi
+; AVX1-NEXT:    vmovd %esi, %xmm2
+; AVX1-NEXT:    vpinsrb $1, %edx, %xmm2, %xmm2
+; AVX1-NEXT:    vpextrb $2, %xmm1, %edx
+; AVX1-NEXT:    bsfl %edx, %edx
+; AVX1-NEXT:    cmovel %eax, %edx
+; AVX1-NEXT:    cmpl $32, %edx
+; AVX1-NEXT:    cmovel %ecx, %edx
+; AVX1-NEXT:    vpinsrb $2, %edx, %xmm2, %xmm2
+; AVX1-NEXT:    vpextrb $3, %xmm1, %edx
+; AVX1-NEXT:    bsfl %edx, %edx
+; AVX1-NEXT:    cmovel %eax, %edx
+; AVX1-NEXT:    cmpl $32, %edx
+; AVX1-NEXT:    cmovel %ecx, %edx
+; AVX1-NEXT:    vpinsrb $3, %edx, %xmm2, %xmm2
+; AVX1-NEXT:    vpextrb $4, %xmm1, %edx
+; AVX1-NEXT:    bsfl %edx, %edx
+; AVX1-NEXT:    cmovel %eax, %edx
+; AVX1-NEXT:    cmpl $32, %edx
+; AVX1-NEXT:    cmovel %ecx, %edx
+; AVX1-NEXT:    vpinsrb $4, %edx, %xmm2, %xmm2
+; AVX1-NEXT:    vpextrb $5, %xmm1, %edx
+; AVX1-NEXT:    bsfl %edx, %edx
+; AVX1-NEXT:    cmovel %eax, %edx
+; AVX1-NEXT:    cmpl $32, %edx
+; AVX1-NEXT:    cmovel %ecx, %edx
+; AVX1-NEXT:    vpinsrb $5, %edx, %xmm2, %xmm2
+; AVX1-NEXT:    vpextrb $6, %xmm1, %edx
+; AVX1-NEXT:    bsfl %edx, %edx
+; AVX1-NEXT:    cmovel %eax, %edx
+; AVX1-NEXT:    cmpl $32, %edx
+; AVX1-NEXT:    cmovel %ecx, %edx
+; AVX1-NEXT:    vpinsrb $6, %edx, %xmm2, %xmm2
+; AVX1-NEXT:    vpextrb $7, %xmm1, %edx
+; AVX1-NEXT:    bsfl %edx, %edx
+; AVX1-NEXT:    cmovel %eax, %edx
+; AVX1-NEXT:    cmpl $32, %edx
+; AVX1-NEXT:    cmovel %ecx, %edx
+; AVX1-NEXT:    vpinsrb $7, %edx, %xmm2, %xmm2
+; AVX1-NEXT:    vpextrb $8, %xmm1, %edx
+; AVX1-NEXT:    bsfl %edx, %edx
+; AVX1-NEXT:    cmovel %eax, %edx
+; AVX1-NEXT:    cmpl $32, %edx
+; AVX1-NEXT:    cmovel %ecx, %edx
+; AVX1-NEXT:    vpinsrb $8, %edx, %xmm2, %xmm2
+; AVX1-NEXT:    vpextrb $9, %xmm1, %edx
+; AVX1-NEXT:    bsfl %edx, %edx
+; AVX1-NEXT:    cmovel %eax, %edx
+; AVX1-NEXT:    cmpl $32, %edx
+; AVX1-NEXT:    cmovel %ecx, %edx
+; AVX1-NEXT:    vpinsrb $9, %edx, %xmm2, %xmm2
+; AVX1-NEXT:    vpextrb $10, %xmm1, %edx
+; AVX1-NEXT:    bsfl %edx, %edx
+; AVX1-NEXT:    cmovel %eax, %edx
+; AVX1-NEXT:    cmpl $32, %edx
+; AVX1-NEXT:    cmovel %ecx, %edx
+; AVX1-NEXT:    vpinsrb $10, %edx, %xmm2, %xmm2
+; AVX1-NEXT:    vpextrb $11, %xmm1, %edx
+; AVX1-NEXT:    bsfl %edx, %edx
+; AVX1-NEXT:    cmovel %eax, %edx
+; AVX1-NEXT:    cmpl $32, %edx
+; AVX1-NEXT:    cmovel %ecx, %edx
+; AVX1-NEXT:    vpinsrb $11, %edx, %xmm2, %xmm2
+; AVX1-NEXT:    vpextrb $12, %xmm1, %edx
+; AVX1-NEXT:    bsfl %edx, %edx
+; AVX1-NEXT:    cmovel %eax, %edx
+; AVX1-NEXT:    cmpl $32, %edx
+; AVX1-NEXT:    cmovel %ecx, %edx
+; AVX1-NEXT:    vpinsrb $12, %edx, %xmm2, %xmm2
+; AVX1-NEXT:    vpextrb $13, %xmm1, %edx
+; AVX1-NEXT:    bsfl %edx, %edx
+; AVX1-NEXT:    cmovel %eax, %edx
+; AVX1-NEXT:    cmpl $32, %edx
+; AVX1-NEXT:    cmovel %ecx, %edx
+; AVX1-NEXT:    vpinsrb $13, %edx, %xmm2, %xmm2
+; AVX1-NEXT:    vpextrb $14, %xmm1, %edx
+; AVX1-NEXT:    bsfl %edx, %edx
+; AVX1-NEXT:    cmovel %eax, %edx
+; AVX1-NEXT:    cmpl $32, %edx
+; AVX1-NEXT:    cmovel %ecx, %edx
+; AVX1-NEXT:    vpinsrb $14, %edx, %xmm2, %xmm2
+; AVX1-NEXT:    vpextrb $15, %xmm1, %edx
+; AVX1-NEXT:    bsfl %edx, %edx
+; AVX1-NEXT:    cmovel %eax, %edx
+; AVX1-NEXT:    cmpl $32, %edx
+; AVX1-NEXT:    cmovel %ecx, %edx
+; AVX1-NEXT:    vpinsrb $15, %edx, %xmm2, %xmm1
+; AVX1-NEXT:    vpextrb $1, %xmm0, %edx
+; AVX1-NEXT:    bsfl %edx, %edx
+; AVX1-NEXT:    cmovel %eax, %edx
+; AVX1-NEXT:    cmpl $32, %edx
+; AVX1-NEXT:    cmovel %ecx, %edx
+; AVX1-NEXT:    vpextrb $0, %xmm0, %esi
+; AVX1-NEXT:    bsfl %esi, %esi
+; AVX1-NEXT:    cmovel %eax, %esi
+; AVX1-NEXT:    cmpl $32, %esi
+; AVX1-NEXT:    cmovel %ecx, %esi
+; AVX1-NEXT:    vmovd %esi, %xmm2
+; AVX1-NEXT:    vpinsrb $1, %edx, %xmm2, %xmm2
+; AVX1-NEXT:    vpextrb $2, %xmm0, %edx
+; AVX1-NEXT:    bsfl %edx, %edx
+; AVX1-NEXT:    cmovel %eax, %edx
+; AVX1-NEXT:    cmpl $32, %edx
+; AVX1-NEXT:    cmovel %ecx, %edx
+; AVX1-NEXT:    vpinsrb $2, %edx, %xmm2, %xmm2
+; AVX1-NEXT:    vpextrb $3, %xmm0, %edx
+; AVX1-NEXT:    bsfl %edx, %edx
+; AVX1-NEXT:    cmovel %eax, %edx
+; AVX1-NEXT:    cmpl $32, %edx
+; AVX1-NEXT:    cmovel %ecx, %edx
+; AVX1-NEXT:    vpinsrb $3, %edx, %xmm2, %xmm2
+; AVX1-NEXT:    vpextrb $4, %xmm0, %edx
+; AVX1-NEXT:    bsfl %edx, %edx
+; AVX1-NEXT:    cmovel %eax, %edx
+; AVX1-NEXT:    cmpl $32, %edx
+; AVX1-NEXT:    cmovel %ecx, %edx
+; AVX1-NEXT:    vpinsrb $4, %edx, %xmm2, %xmm2
+; AVX1-NEXT:    vpextrb $5, %xmm0, %edx
+; AVX1-NEXT:    bsfl %edx, %edx
+; AVX1-NEXT:    cmovel %eax, %edx
+; AVX1-NEXT:    cmpl $32, %edx
+; AVX1-NEXT:    cmovel %ecx, %edx
+; AVX1-NEXT:    vpinsrb $5, %edx, %xmm2, %xmm2
+; AVX1-NEXT:    vpextrb $6, %xmm0, %edx
+; AVX1-NEXT:    bsfl %edx, %edx
+; AVX1-NEXT:    cmovel %eax, %edx
+; AVX1-NEXT:    cmpl $32, %edx
+; AVX1-NEXT:    cmovel %ecx, %edx
+; AVX1-NEXT:    vpinsrb $6, %edx, %xmm2, %xmm2
+; AVX1-NEXT:    vpextrb $7, %xmm0, %edx
+; AVX1-NEXT:    bsfl %edx, %edx
+; AVX1-NEXT:    cmovel %eax, %edx
+; AVX1-NEXT:    cmpl $32, %edx
+; AVX1-NEXT:    cmovel %ecx, %edx
+; AVX1-NEXT:    vpinsrb $7, %edx, %xmm2, %xmm2
+; AVX1-NEXT:    vpextrb $8, %xmm0, %edx
+; AVX1-NEXT:    bsfl %edx, %edx
+; AVX1-NEXT:    cmovel %eax, %edx
+; AVX1-NEXT:    cmpl $32, %edx
+; AVX1-NEXT:    cmovel %ecx, %edx
+; AVX1-NEXT:    vpinsrb $8, %edx, %xmm2, %xmm2
+; AVX1-NEXT:    vpextrb $9, %xmm0, %edx
+; AVX1-NEXT:    bsfl %edx, %edx
+; AVX1-NEXT:    cmovel %eax, %edx
+; AVX1-NEXT:    cmpl $32, %edx
+; AVX1-NEXT:    cmovel %ecx, %edx
+; AVX1-NEXT:    vpinsrb $9, %edx, %xmm2, %xmm2
+; AVX1-NEXT:    vpextrb $10, %xmm0, %edx
+; AVX1-NEXT:    bsfl %edx, %edx
+; AVX1-NEXT:    cmovel %eax, %edx
+; AVX1-NEXT:    cmpl $32, %edx
+; AVX1-NEXT:    cmovel %ecx, %edx
+; AVX1-NEXT:    vpinsrb $10, %edx, %xmm2, %xmm2
+; AVX1-NEXT:    vpextrb $11, %xmm0, %edx
+; AVX1-NEXT:    bsfl %edx, %edx
+; AVX1-NEXT:    cmovel %eax, %edx
+; AVX1-NEXT:    cmpl $32, %edx
+; AVX1-NEXT:    cmovel %ecx, %edx
+; AVX1-NEXT:    vpinsrb $11, %edx, %xmm2, %xmm2
+; AVX1-NEXT:    vpextrb $12, %xmm0, %edx
+; AVX1-NEXT:    bsfl %edx, %edx
+; AVX1-NEXT:    cmovel %eax, %edx
+; AVX1-NEXT:    cmpl $32, %edx
+; AVX1-NEXT:    cmovel %ecx, %edx
+; AVX1-NEXT:    vpinsrb $12, %edx, %xmm2, %xmm2
+; AVX1-NEXT:    vpextrb $13, %xmm0, %edx
+; AVX1-NEXT:    bsfl %edx, %edx
+; AVX1-NEXT:    cmovel %eax, %edx
+; AVX1-NEXT:    cmpl $32, %edx
+; AVX1-NEXT:    cmovel %ecx, %edx
+; AVX1-NEXT:    vpinsrb $13, %edx, %xmm2, %xmm2
+; AVX1-NEXT:    vpextrb $14, %xmm0, %edx
+; AVX1-NEXT:    bsfl %edx, %edx
+; AVX1-NEXT:    cmovel %eax, %edx
+; AVX1-NEXT:    cmpl $32, %edx
+; AVX1-NEXT:    cmovel %ecx, %edx
+; AVX1-NEXT:    vpinsrb $14, %edx, %xmm2, %xmm2
+; AVX1-NEXT:    vpextrb $15, %xmm0, %edx
+; AVX1-NEXT:    bsfl %edx, %edx
+; AVX1-NEXT:    cmovel %eax, %edx
+; AVX1-NEXT:    cmpl $32, %edx
+; AVX1-NEXT:    cmovel %ecx, %edx
+; AVX1-NEXT:    vpinsrb $15, %edx, %xmm2, %xmm0
+; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: testv32i8:
+; AVX2:       # BB#0:
+; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
+; AVX2-NEXT:    vpextrb $1, %xmm1, %eax
+; AVX2-NEXT:    bsfl %eax, %edx
+; AVX2-NEXT:    movl $32, %eax
+; AVX2-NEXT:    cmovel %eax, %edx
+; AVX2-NEXT:    cmpl $32, %edx
+; AVX2-NEXT:    movl $8, %ecx
+; AVX2-NEXT:    cmovel %ecx, %edx
+; AVX2-NEXT:    vpextrb $0, %xmm1, %esi
+; AVX2-NEXT:    bsfl %esi, %esi
+; AVX2-NEXT:    cmovel %eax, %esi
+; AVX2-NEXT:    cmpl $32, %esi
+; AVX2-NEXT:    cmovel %ecx, %esi
+; AVX2-NEXT:    vmovd %esi, %xmm2
+; AVX2-NEXT:    vpinsrb $1, %edx, %xmm2, %xmm2
+; AVX2-NEXT:    vpextrb $2, %xmm1, %edx
+; AVX2-NEXT:    bsfl %edx, %edx
+; AVX2-NEXT:    cmovel %eax, %edx
+; AVX2-NEXT:    cmpl $32, %edx
+; AVX2-NEXT:    cmovel %ecx, %edx
+; AVX2-NEXT:    vpinsrb $2, %edx, %xmm2, %xmm2
+; AVX2-NEXT:    vpextrb $3, %xmm1, %edx
+; AVX2-NEXT:    bsfl %edx, %edx
+; AVX2-NEXT:    cmovel %eax, %edx
+; AVX2-NEXT:    cmpl $32, %edx
+; AVX2-NEXT:    cmovel %ecx, %edx
+; AVX2-NEXT:    vpinsrb $3, %edx, %xmm2, %xmm2
+; AVX2-NEXT:    vpextrb $4, %xmm1, %edx
+; AVX2-NEXT:    bsfl %edx, %edx
+; AVX2-NEXT:    cmovel %eax, %edx
+; AVX2-NEXT:    cmpl $32, %edx
+; AVX2-NEXT:    cmovel %ecx, %edx
+; AVX2-NEXT:    vpinsrb $4, %edx, %xmm2, %xmm2
+; AVX2-NEXT:    vpextrb $5, %xmm1, %edx
+; AVX2-NEXT:    bsfl %edx, %edx
+; AVX2-NEXT:    cmovel %eax, %edx
+; AVX2-NEXT:    cmpl $32, %edx
+; AVX2-NEXT:    cmovel %ecx, %edx
+; AVX2-NEXT:    vpinsrb $5, %edx, %xmm2, %xmm2
+; AVX2-NEXT:    vpextrb $6, %xmm1, %edx
+; AVX2-NEXT:    bsfl %edx, %edx
+; AVX2-NEXT:    cmovel %eax, %edx
+; AVX2-NEXT:    cmpl $32, %edx
+; AVX2-NEXT:    cmovel %ecx, %edx
+; AVX2-NEXT:    vpinsrb $6, %edx, %xmm2, %xmm2
+; AVX2-NEXT:    vpextrb $7, %xmm1, %edx
+; AVX2-NEXT:    bsfl %edx, %edx
+; AVX2-NEXT:    cmovel %eax, %edx
+; AVX2-NEXT:    cmpl $32, %edx
+; AVX2-NEXT:    cmovel %ecx, %edx
+; AVX2-NEXT:    vpinsrb $7, %edx, %xmm2, %xmm2
+; AVX2-NEXT:    vpextrb $8, %xmm1, %edx
+; AVX2-NEXT:    bsfl %edx, %edx
+; AVX2-NEXT:    cmovel %eax, %edx
+; AVX2-NEXT:    cmpl $32, %edx
+; AVX2-NEXT:    cmovel %ecx, %edx
+; AVX2-NEXT:    vpinsrb $8, %edx, %xmm2, %xmm2
+; AVX2-NEXT:    vpextrb $9, %xmm1, %edx
+; AVX2-NEXT:    bsfl %edx, %edx
+; AVX2-NEXT:    cmovel %eax, %edx
+; AVX2-NEXT:    cmpl $32, %edx
+; AVX2-NEXT:    cmovel %ecx, %edx
+; AVX2-NEXT:    vpinsrb $9, %edx, %xmm2, %xmm2
+; AVX2-NEXT:    vpextrb $10, %xmm1, %edx
+; AVX2-NEXT:    bsfl %edx, %edx
+; AVX2-NEXT:    cmovel %eax, %edx
+; AVX2-NEXT:    cmpl $32, %edx
+; AVX2-NEXT:    cmovel %ecx, %edx
+; AVX2-NEXT:    vpinsrb $10, %edx, %xmm2, %xmm2
+; AVX2-NEXT:    vpextrb $11, %xmm1, %edx
+; AVX2-NEXT:    bsfl %edx, %edx
+; AVX2-NEXT:    cmovel %eax, %edx
+; AVX2-NEXT:    cmpl $32, %edx
+; AVX2-NEXT:    cmovel %ecx, %edx
+; AVX2-NEXT:    vpinsrb $11, %edx, %xmm2, %xmm2
+; AVX2-NEXT:    vpextrb $12, %xmm1, %edx
+; AVX2-NEXT:    bsfl %edx, %edx
+; AVX2-NEXT:    cmovel %eax, %edx
+; AVX2-NEXT:    cmpl $32, %edx
+; AVX2-NEXT:    cmovel %ecx, %edx
+; AVX2-NEXT:    vpinsrb $12, %edx, %xmm2, %xmm2
+; AVX2-NEXT:    vpextrb $13, %xmm1, %edx
+; AVX2-NEXT:    bsfl %edx, %edx
+; AVX2-NEXT:    cmovel %eax, %edx
+; AVX2-NEXT:    cmpl $32, %edx
+; AVX2-NEXT:    cmovel %ecx, %edx
+; AVX2-NEXT:    vpinsrb $13, %edx, %xmm2, %xmm2
+; AVX2-NEXT:    vpextrb $14, %xmm1, %edx
+; AVX2-NEXT:    bsfl %edx, %edx
+; AVX2-NEXT:    cmovel %eax, %edx
+; AVX2-NEXT:    cmpl $32, %edx
+; AVX2-NEXT:    cmovel %ecx, %edx
+; AVX2-NEXT:    vpinsrb $14, %edx, %xmm2, %xmm2
+; AVX2-NEXT:    vpextrb $15, %xmm1, %edx
+; AVX2-NEXT:    bsfl %edx, %edx
+; AVX2-NEXT:    cmovel %eax, %edx
+; AVX2-NEXT:    cmpl $32, %edx
+; AVX2-NEXT:    cmovel %ecx, %edx
+; AVX2-NEXT:    vpinsrb $15, %edx, %xmm2, %xmm1
+; AVX2-NEXT:    vpextrb $1, %xmm0, %edx
+; AVX2-NEXT:    bsfl %edx, %edx
+; AVX2-NEXT:    cmovel %eax, %edx
+; AVX2-NEXT:    cmpl $32, %edx
+; AVX2-NEXT:    cmovel %ecx, %edx
+; AVX2-NEXT:    vpextrb $0, %xmm0, %esi
+; AVX2-NEXT:    bsfl %esi, %esi
+; AVX2-NEXT:    cmovel %eax, %esi
+; AVX2-NEXT:    cmpl $32, %esi
+; AVX2-NEXT:    cmovel %ecx, %esi
+; AVX2-NEXT:    vmovd %esi, %xmm2
+; AVX2-NEXT:    vpinsrb $1, %edx, %xmm2, %xmm2
+; AVX2-NEXT:    vpextrb $2, %xmm0, %edx
+; AVX2-NEXT:    bsfl %edx, %edx
+; AVX2-NEXT:    cmovel %eax, %edx
+; AVX2-NEXT:    cmpl $32, %edx
+; AVX2-NEXT:    cmovel %ecx, %edx
+; AVX2-NEXT:    vpinsrb $2, %edx, %xmm2, %xmm2
+; AVX2-NEXT:    vpextrb $3, %xmm0, %edx
+; AVX2-NEXT:    bsfl %edx, %edx
+; AVX2-NEXT:    cmovel %eax, %edx
+; AVX2-NEXT:    cmpl $32, %edx
+; AVX2-NEXT:    cmovel %ecx, %edx
+; AVX2-NEXT:    vpinsrb $3, %edx, %xmm2, %xmm2
+; AVX2-NEXT:    vpextrb $4, %xmm0, %edx
+; AVX2-NEXT:    bsfl %edx, %edx
+; AVX2-NEXT:    cmovel %eax, %edx
+; AVX2-NEXT:    cmpl $32, %edx
+; AVX2-NEXT:    cmovel %ecx, %edx
+; AVX2-NEXT:    vpinsrb $4, %edx, %xmm2, %xmm2
+; AVX2-NEXT:    vpextrb $5, %xmm0, %edx
+; AVX2-NEXT:    bsfl %edx, %edx
+; AVX2-NEXT:    cmovel %eax, %edx
+; AVX2-NEXT:    cmpl $32, %edx
+; AVX2-NEXT:    cmovel %ecx, %edx
+; AVX2-NEXT:    vpinsrb $5, %edx, %xmm2, %xmm2
+; AVX2-NEXT:    vpextrb $6, %xmm0, %edx
+; AVX2-NEXT:    bsfl %edx, %edx
+; AVX2-NEXT:    cmovel %eax, %edx
+; AVX2-NEXT:    cmpl $32, %edx
+; AVX2-NEXT:    cmovel %ecx, %edx
+; AVX2-NEXT:    vpinsrb $6, %edx, %xmm2, %xmm2
+; AVX2-NEXT:    vpextrb $7, %xmm0, %edx
+; AVX2-NEXT:    bsfl %edx, %edx
+; AVX2-NEXT:    cmovel %eax, %edx
+; AVX2-NEXT:    cmpl $32, %edx
+; AVX2-NEXT:    cmovel %ecx, %edx
+; AVX2-NEXT:    vpinsrb $7, %edx, %xmm2, %xmm2
+; AVX2-NEXT:    vpextrb $8, %xmm0, %edx
+; AVX2-NEXT:    bsfl %edx, %edx
+; AVX2-NEXT:    cmovel %eax, %edx
+; AVX2-NEXT:    cmpl $32, %edx
+; AVX2-NEXT:    cmovel %ecx, %edx
+; AVX2-NEXT:    vpinsrb $8, %edx, %xmm2, %xmm2
+; AVX2-NEXT:    vpextrb $9, %xmm0, %edx
+; AVX2-NEXT:    bsfl %edx, %edx
+; AVX2-NEXT:    cmovel %eax, %edx
+; AVX2-NEXT:    cmpl $32, %edx
+; AVX2-NEXT:    cmovel %ecx, %edx
+; AVX2-NEXT:    vpinsrb $9, %edx, %xmm2, %xmm2
+; AVX2-NEXT:    vpextrb $10, %xmm0, %edx
+; AVX2-NEXT:    bsfl %edx, %edx
+; AVX2-NEXT:    cmovel %eax, %edx
+; AVX2-NEXT:    cmpl $32, %edx
+; AVX2-NEXT:    cmovel %ecx, %edx
+; AVX2-NEXT:    vpinsrb $10, %edx, %xmm2, %xmm2
+; AVX2-NEXT:    vpextrb $11, %xmm0, %edx
+; AVX2-NEXT:    bsfl %edx, %edx
+; AVX2-NEXT:    cmovel %eax, %edx
+; AVX2-NEXT:    cmpl $32, %edx
+; AVX2-NEXT:    cmovel %ecx, %edx
+; AVX2-NEXT:    vpinsrb $11, %edx, %xmm2, %xmm2
+; AVX2-NEXT:    vpextrb $12, %xmm0, %edx
+; AVX2-NEXT:    bsfl %edx, %edx
+; AVX2-NEXT:    cmovel %eax, %edx
+; AVX2-NEXT:    cmpl $32, %edx
+; AVX2-NEXT:    cmovel %ecx, %edx
+; AVX2-NEXT:    vpinsrb $12, %edx, %xmm2, %xmm2
+; AVX2-NEXT:    vpextrb $13, %xmm0, %edx
+; AVX2-NEXT:    bsfl %edx, %edx
+; AVX2-NEXT:    cmovel %eax, %edx
+; AVX2-NEXT:    cmpl $32, %edx
+; AVX2-NEXT:    cmovel %ecx, %edx
+; AVX2-NEXT:    vpinsrb $13, %edx, %xmm2, %xmm2
+; AVX2-NEXT:    vpextrb $14, %xmm0, %edx
+; AVX2-NEXT:    bsfl %edx, %edx
+; AVX2-NEXT:    cmovel %eax, %edx
+; AVX2-NEXT:    cmpl $32, %edx
+; AVX2-NEXT:    cmovel %ecx, %edx
+; AVX2-NEXT:    vpinsrb $14, %edx, %xmm2, %xmm2
+; AVX2-NEXT:    vpextrb $15, %xmm0, %edx
+; AVX2-NEXT:    bsfl %edx, %edx
+; AVX2-NEXT:    cmovel %eax, %edx
+; AVX2-NEXT:    cmpl $32, %edx
+; AVX2-NEXT:    cmovel %ecx, %edx
+; AVX2-NEXT:    vpinsrb $15, %edx, %xmm2, %xmm0
+; AVX2-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm0
+; AVX2-NEXT:    retq
+  %out = call <32 x i8> @llvm.cttz.v32i8(<32 x i8> %in, i1 0)
+  ret <32 x i8> %out
+}
+
+define <32 x i8> @testv32i8u(<32 x i8> %in) nounwind {
+; AVX1-LABEL: testv32i8u:
+; AVX1:       # BB#0:
+; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
+; AVX1-NEXT:    vpextrb $1, %xmm1, %eax
+; AVX1-NEXT:    bsfl %eax, %eax
+; AVX1-NEXT:    vpextrb $0, %xmm1, %ecx
+; AVX1-NEXT:    bsfl %ecx, %ecx
+; AVX1-NEXT:    vmovd %ecx, %xmm2
+; AVX1-NEXT:    vpinsrb $1, %eax, %xmm2, %xmm2
+; AVX1-NEXT:    vpextrb $2, %xmm1, %eax
+; AVX1-NEXT:    bsfl %eax, %eax
+; AVX1-NEXT:    vpinsrb $2, %eax, %xmm2, %xmm2
+; AVX1-NEXT:    vpextrb $3, %xmm1, %eax
+; AVX1-NEXT:    bsfl %eax, %eax
+; AVX1-NEXT:    vpinsrb $3, %eax, %xmm2, %xmm2
+; AVX1-NEXT:    vpextrb $4, %xmm1, %eax
+; AVX1-NEXT:    bsfl %eax, %eax
+; AVX1-NEXT:    vpinsrb $4, %eax, %xmm2, %xmm2
+; AVX1-NEXT:    vpextrb $5, %xmm1, %eax
+; AVX1-NEXT:    bsfl %eax, %eax
+; AVX1-NEXT:    vpinsrb $5, %eax, %xmm2, %xmm2
+; AVX1-NEXT:    vpextrb $6, %xmm1, %eax
+; AVX1-NEXT:    bsfl %eax, %eax
+; AVX1-NEXT:    vpinsrb $6, %eax, %xmm2, %xmm2
+; AVX1-NEXT:    vpextrb $7, %xmm1, %eax
+; AVX1-NEXT:    bsfl %eax, %eax
+; AVX1-NEXT:    vpinsrb $7, %eax, %xmm2, %xmm2
+; AVX1-NEXT:    vpextrb $8, %xmm1, %eax
+; AVX1-NEXT:    bsfl %eax, %eax
+; AVX1-NEXT:    vpinsrb $8, %eax, %xmm2, %xmm2
+; AVX1-NEXT:    vpextrb $9, %xmm1, %eax
+; AVX1-NEXT:    bsfl %eax, %eax
+; AVX1-NEXT:    vpinsrb $9, %eax, %xmm2, %xmm2
+; AVX1-NEXT:    vpextrb $10, %xmm1, %eax
+; AVX1-NEXT:    bsfl %eax, %eax
+; AVX1-NEXT:    vpinsrb $10, %eax, %xmm2, %xmm2
+; AVX1-NEXT:    vpextrb $11, %xmm1, %eax
+; AVX1-NEXT:    bsfl %eax, %eax
+; AVX1-NEXT:    vpinsrb $11, %eax, %xmm2, %xmm2
+; AVX1-NEXT:    vpextrb $12, %xmm1, %eax
+; AVX1-NEXT:    bsfl %eax, %eax
+; AVX1-NEXT:    vpinsrb $12, %eax, %xmm2, %xmm2
+; AVX1-NEXT:    vpextrb $13, %xmm1, %eax
+; AVX1-NEXT:    bsfl %eax, %eax
+; AVX1-NEXT:    vpinsrb $13, %eax, %xmm2, %xmm2
+; AVX1-NEXT:    vpextrb $14, %xmm1, %eax
+; AVX1-NEXT:    bsfl %eax, %eax
+; AVX1-NEXT:    vpinsrb $14, %eax, %xmm2, %xmm2
+; AVX1-NEXT:    vpextrb $15, %xmm1, %eax
+; AVX1-NEXT:    bsfl %eax, %eax
+; AVX1-NEXT:    vpinsrb $15, %eax, %xmm2, %xmm1
+; AVX1-NEXT:    vpextrb $1, %xmm0, %eax
+; AVX1-NEXT:    bsfl %eax, %eax
+; AVX1-NEXT:    vpextrb $0, %xmm0, %ecx
+; AVX1-NEXT:    bsfl %ecx, %ecx
+; AVX1-NEXT:    vmovd %ecx, %xmm2
+; AVX1-NEXT:    vpinsrb $1, %eax, %xmm2, %xmm2
+; AVX1-NEXT:    vpextrb $2, %xmm0, %eax
+; AVX1-NEXT:    bsfl %eax, %eax
+; AVX1-NEXT:    vpinsrb $2, %eax, %xmm2, %xmm2
+; AVX1-NEXT:    vpextrb $3, %xmm0, %eax
+; AVX1-NEXT:    bsfl %eax, %eax
+; AVX1-NEXT:    vpinsrb $3, %eax, %xmm2, %xmm2
+; AVX1-NEXT:    vpextrb $4, %xmm0, %eax
+; AVX1-NEXT:    bsfl %eax, %eax
+; AVX1-NEXT:    vpinsrb $4, %eax, %xmm2, %xmm2
+; AVX1-NEXT:    vpextrb $5, %xmm0, %eax
+; AVX1-NEXT:    bsfl %eax, %eax
+; AVX1-NEXT:    vpinsrb $5, %eax, %xmm2, %xmm2
+; AVX1-NEXT:    vpextrb $6, %xmm0, %eax
+; AVX1-NEXT:    bsfl %eax, %eax
+; AVX1-NEXT:    vpinsrb $6, %eax, %xmm2, %xmm2
+; AVX1-NEXT:    vpextrb $7, %xmm0, %eax
+; AVX1-NEXT:    bsfl %eax, %eax
+; AVX1-NEXT:    vpinsrb $7, %eax, %xmm2, %xmm2
+; AVX1-NEXT:    vpextrb $8, %xmm0, %eax
+; AVX1-NEXT:    bsfl %eax, %eax
+; AVX1-NEXT:    vpinsrb $8, %eax, %xmm2, %xmm2
+; AVX1-NEXT:    vpextrb $9, %xmm0, %eax
+; AVX1-NEXT:    bsfl %eax, %eax
+; AVX1-NEXT:    vpinsrb $9, %eax, %xmm2, %xmm2
+; AVX1-NEXT:    vpextrb $10, %xmm0, %eax
+; AVX1-NEXT:    bsfl %eax, %eax
+; AVX1-NEXT:    vpinsrb $10, %eax, %xmm2, %xmm2
+; AVX1-NEXT:    vpextrb $11, %xmm0, %eax
+; AVX1-NEXT:    bsfl %eax, %eax
+; AVX1-NEXT:    vpinsrb $11, %eax, %xmm2, %xmm2
+; AVX1-NEXT:    vpextrb $12, %xmm0, %eax
+; AVX1-NEXT:    bsfl %eax, %eax
+; AVX1-NEXT:    vpinsrb $12, %eax, %xmm2, %xmm2
+; AVX1-NEXT:    vpextrb $13, %xmm0, %eax
+; AVX1-NEXT:    bsfl %eax, %eax
+; AVX1-NEXT:    vpinsrb $13, %eax, %xmm2, %xmm2
+; AVX1-NEXT:    vpextrb $14, %xmm0, %eax
+; AVX1-NEXT:    bsfl %eax, %eax
+; AVX1-NEXT:    vpinsrb $14, %eax, %xmm2, %xmm2
+; AVX1-NEXT:    vpextrb $15, %xmm0, %eax
+; AVX1-NEXT:    bsfl %eax, %eax
+; AVX1-NEXT:    vpinsrb $15, %eax, %xmm2, %xmm0
+; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: testv32i8u:
+; AVX2:       # BB#0:
+; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
+; AVX2-NEXT:    vpextrb $1, %xmm1, %eax
+; AVX2-NEXT:    bsfl %eax, %eax
+; AVX2-NEXT:    vpextrb $0, %xmm1, %ecx
+; AVX2-NEXT:    bsfl %ecx, %ecx
+; AVX2-NEXT:    vmovd %ecx, %xmm2
+; AVX2-NEXT:    vpinsrb $1, %eax, %xmm2, %xmm2
+; AVX2-NEXT:    vpextrb $2, %xmm1, %eax
+; AVX2-NEXT:    bsfl %eax, %eax
+; AVX2-NEXT:    vpinsrb $2, %eax, %xmm2, %xmm2
+; AVX2-NEXT:    vpextrb $3, %xmm1, %eax
+; AVX2-NEXT:    bsfl %eax, %eax
+; AVX2-NEXT:    vpinsrb $3, %eax, %xmm2, %xmm2
+; AVX2-NEXT:    vpextrb $4, %xmm1, %eax
+; AVX2-NEXT:    bsfl %eax, %eax
+; AVX2-NEXT:    vpinsrb $4, %eax, %xmm2, %xmm2
+; AVX2-NEXT:    vpextrb $5, %xmm1, %eax
+; AVX2-NEXT:    bsfl %eax, %eax
+; AVX2-NEXT:    vpinsrb $5, %eax, %xmm2, %xmm2
+; AVX2-NEXT:    vpextrb $6, %xmm1, %eax
+; AVX2-NEXT:    bsfl %eax, %eax
+; AVX2-NEXT:    vpinsrb $6, %eax, %xmm2, %xmm2
+; AVX2-NEXT:    vpextrb $7, %xmm1, %eax
+; AVX2-NEXT:    bsfl %eax, %eax
+; AVX2-NEXT:    vpinsrb $7, %eax, %xmm2, %xmm2
+; AVX2-NEXT:    vpextrb $8, %xmm1, %eax
+; AVX2-NEXT:    bsfl %eax, %eax
+; AVX2-NEXT:    vpinsrb $8, %eax, %xmm2, %xmm2
+; AVX2-NEXT:    vpextrb $9, %xmm1, %eax
+; AVX2-NEXT:    bsfl %eax, %eax
+; AVX2-NEXT:    vpinsrb $9, %eax, %xmm2, %xmm2
+; AVX2-NEXT:    vpextrb $10, %xmm1, %eax
+; AVX2-NEXT:    bsfl %eax, %eax
+; AVX2-NEXT:    vpinsrb $10, %eax, %xmm2, %xmm2
+; AVX2-NEXT:    vpextrb $11, %xmm1, %eax
+; AVX2-NEXT:    bsfl %eax, %eax
+; AVX2-NEXT:    vpinsrb $11, %eax, %xmm2, %xmm2
+; AVX2-NEXT:    vpextrb $12, %xmm1, %eax
+; AVX2-NEXT:    bsfl %eax, %eax
+; AVX2-NEXT:    vpinsrb $12, %eax, %xmm2, %xmm2
+; AVX2-NEXT:    vpextrb $13, %xmm1, %eax
+; AVX2-NEXT:    bsfl %eax, %eax
+; AVX2-NEXT:    vpinsrb $13, %eax, %xmm2, %xmm2
+; AVX2-NEXT:    vpextrb $14, %xmm1, %eax
+; AVX2-NEXT:    bsfl %eax, %eax
+; AVX2-NEXT:    vpinsrb $14, %eax, %xmm2, %xmm2
+; AVX2-NEXT:    vpextrb $15, %xmm1, %eax
+; AVX2-NEXT:    bsfl %eax, %eax
+; AVX2-NEXT:    vpinsrb $15, %eax, %xmm2, %xmm1
+; AVX2-NEXT:    vpextrb $1, %xmm0, %eax
+; AVX2-NEXT:    bsfl %eax, %eax
+; AVX2-NEXT:    vpextrb $0, %xmm0, %ecx
+; AVX2-NEXT:    bsfl %ecx, %ecx
+; AVX2-NEXT:    vmovd %ecx, %xmm2
+; AVX2-NEXT:    vpinsrb $1, %eax, %xmm2, %xmm2
+; AVX2-NEXT:    vpextrb $2, %xmm0, %eax
+; AVX2-NEXT:    bsfl %eax, %eax
+; AVX2-NEXT:    vpinsrb $2, %eax, %xmm2, %xmm2
+; AVX2-NEXT:    vpextrb $3, %xmm0, %eax
+; AVX2-NEXT:    bsfl %eax, %eax
+; AVX2-NEXT:    vpinsrb $3, %eax, %xmm2, %xmm2
+; AVX2-NEXT:    vpextrb $4, %xmm0, %eax
+; AVX2-NEXT:    bsfl %eax, %eax
+; AVX2-NEXT:    vpinsrb $4, %eax, %xmm2, %xmm2
+; AVX2-NEXT:    vpextrb $5, %xmm0, %eax
+; AVX2-NEXT:    bsfl %eax, %eax
+; AVX2-NEXT:    vpinsrb $5, %eax, %xmm2, %xmm2
+; AVX2-NEXT:    vpextrb $6, %xmm0, %eax
+; AVX2-NEXT:    bsfl %eax, %eax
+; AVX2-NEXT:    vpinsrb $6, %eax, %xmm2, %xmm2
+; AVX2-NEXT:    vpextrb $7, %xmm0, %eax
+; AVX2-NEXT:    bsfl %eax, %eax
+; AVX2-NEXT:    vpinsrb $7, %eax, %xmm2, %xmm2
+; AVX2-NEXT:    vpextrb $8, %xmm0, %eax
+; AVX2-NEXT:    bsfl %eax, %eax
+; AVX2-NEXT:    vpinsrb $8, %eax, %xmm2, %xmm2
+; AVX2-NEXT:    vpextrb $9, %xmm0, %eax
+; AVX2-NEXT:    bsfl %eax, %eax
+; AVX2-NEXT:    vpinsrb $9, %eax, %xmm2, %xmm2
+; AVX2-NEXT:    vpextrb $10, %xmm0, %eax
+; AVX2-NEXT:    bsfl %eax, %eax
+; AVX2-NEXT:    vpinsrb $10, %eax, %xmm2, %xmm2
+; AVX2-NEXT:    vpextrb $11, %xmm0, %eax
+; AVX2-NEXT:    bsfl %eax, %eax
+; AVX2-NEXT:    vpinsrb $11, %eax, %xmm2, %xmm2
+; AVX2-NEXT:    vpextrb $12, %xmm0, %eax
+; AVX2-NEXT:    bsfl %eax, %eax
+; AVX2-NEXT:    vpinsrb $12, %eax, %xmm2, %xmm2
+; AVX2-NEXT:    vpextrb $13, %xmm0, %eax
+; AVX2-NEXT:    bsfl %eax, %eax
+; AVX2-NEXT:    vpinsrb $13, %eax, %xmm2, %xmm2
+; AVX2-NEXT:    vpextrb $14, %xmm0, %eax
+; AVX2-NEXT:    bsfl %eax, %eax
+; AVX2-NEXT:    vpinsrb $14, %eax, %xmm2, %xmm2
+; AVX2-NEXT:    vpextrb $15, %xmm0, %eax
+; AVX2-NEXT:    bsfl %eax, %eax
+; AVX2-NEXT:    vpinsrb $15, %eax, %xmm2, %xmm0
+; AVX2-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm0
+; AVX2-NEXT:    retq
+  %out = call <32 x i8> @llvm.cttz.v32i8(<32 x i8> %in, i1 -1)
+  ret <32 x i8> %out
+}
+
+define <4 x i64> @foldv4i64() nounwind {
+; ALL-LABEL: foldv4i64:
+; ALL:       # BB#0:
+; ALL-NEXT:    vmovaps {{.*#+}} ymm0 = [8,0,64,0]
+; ALL-NEXT:    retq
+  %out = call <4 x i64> @llvm.cttz.v4i64(<4 x i64> <i64 256, i64 -1, i64 0, i64 255>, i1 0)
+  ret <4 x i64> %out
+}
+
+define <4 x i64> @foldv4i64u() nounwind {
+; ALL-LABEL: foldv4i64u:
+; ALL:       # BB#0:
+; ALL-NEXT:    vmovaps {{.*#+}} ymm0 = [8,0,64,0]
+; ALL-NEXT:    retq
+  %out = call <4 x i64> @llvm.cttz.v4i64(<4 x i64> <i64 256, i64 -1, i64 0, i64 255>, i1 -1)
+  ret <4 x i64> %out
+}
+
+define <8 x i32> @foldv8i32() nounwind {
+; ALL-LABEL: foldv8i32:
+; ALL:       # BB#0:
+; ALL-NEXT:    vmovaps {{.*#+}} ymm0 = [8,0,32,0,16,0,3,3]
+; ALL-NEXT:    retq
+  %out = call <8 x i32> @llvm.cttz.v8i32(<8 x i32> <i32 256, i32 -1, i32 0, i32 255, i32 -65536, i32 7, i32 24, i32 88>, i1 0)
+  ret <8 x i32> %out
+}
+
+define <8 x i32> @foldv8i32u() nounwind {
+; ALL-LABEL: foldv8i32u:
+; ALL:       # BB#0:
+; ALL-NEXT:    vmovaps {{.*#+}} ymm0 = [8,0,32,0,16,0,3,3]
+; ALL-NEXT:    retq
+  %out = call <8 x i32> @llvm.cttz.v8i32(<8 x i32> <i32 256, i32 -1, i32 0, i32 255, i32 -65536, i32 7, i32 24, i32 88>, i1 -1)
+  ret <8 x i32> %out
+}
+
+define <16 x i16> @foldv16i16() nounwind {
+; ALL-LABEL: foldv16i16:
+; ALL:       # BB#0:
+; ALL-NEXT:    vmovaps {{.*#+}} ymm0 = [8,0,16,0,16,0,3,3,1,1,0,1,2,3,4,5]
+; ALL-NEXT:    retq
+  %out = call <16 x i16> @llvm.cttz.v16i16(<16 x i16> <i16 256, i16 -1, i16 0, i16 255, i16 -65536, i16 7, i16 24, i16 88, i16 -2, i16 254, i16 1, i16 2, i16 4, i16 8, i16 16, i16 32>, i1 0)
+  ret <16 x i16> %out
+}
+
+define <16 x i16> @foldv16i16u() nounwind {
+; ALL-LABEL: foldv16i16u:
+; ALL:       # BB#0:
+; ALL-NEXT:    vmovaps {{.*#+}} ymm0 = [8,0,16,0,16,0,3,3,1,1,0,1,2,3,4,5]
+; ALL-NEXT:    retq
+  %out = call <16 x i16> @llvm.cttz.v16i16(<16 x i16> <i16 256, i16 -1, i16 0, i16 255, i16 -65536, i16 7, i16 24, i16 88, i16 -2, i16 254, i16 1, i16 2, i16 4, i16 8, i16 16, i16 32>, i1 -1)
+  ret <16 x i16> %out
+}
+
+define <32 x i8> @foldv32i8() nounwind {
+; ALL-LABEL: foldv32i8:
+; ALL:       # BB#0:
+; ALL-NEXT:    vmovaps {{.*#+}} ymm0 = [8,0,8,0,8,0,3,3,1,1,0,1,2,3,4,5,6,7,8,8,7,6,5,4,3,2,1,0,0,0,0,0]
+; ALL-NEXT:    retq
+  %out = call <32 x i8> @llvm.cttz.v32i8(<32 x i8> <i8 256, i8 -1, i8 0, i8 255, i8 -65536, i8 7, i8 24, i8 88, i8 -2, i8 254, i8 1, i8 2, i8 4, i8 8, i8 16, i8 32, i8 64, i8 128, i8 256, i8 -256, i8 -128, i8 -64, i8 -32, i8 -16, i8 -8, i8 -4, i8 -2, i8 -1, i8 3, i8 5, i8 7, i8 127>, i1 0)
+  ret <32 x i8> %out
+}
+
+define <32 x i8> @foldv32i8u() nounwind {
+; ALL-LABEL: foldv32i8u:
+; ALL:       # BB#0:
+; ALL-NEXT:    vmovaps {{.*#+}} ymm0 = [8,0,8,0,8,0,3,3,1,1,0,1,2,3,4,5,6,7,8,8,7,6,5,4,3,2,1,0,0,0,0,0]
+; ALL-NEXT:    retq
+  %out = call <32 x i8> @llvm.cttz.v32i8(<32 x i8> <i8 256, i8 -1, i8 0, i8 255, i8 -65536, i8 7, i8 24, i8 88, i8 -2, i8 254, i8 1, i8 2, i8 4, i8 8, i8 16, i8 32, i8 64, i8 128, i8 256, i8 -256, i8 -128, i8 -64, i8 -32, i8 -16, i8 -8, i8 -4, i8 -2, i8 -1, i8 3, i8 5, i8 7, i8 127>, i1 -1)
+  ret <32 x i8> %out
+}
+
+declare <4 x i64> @llvm.cttz.v4i64(<4 x i64>, i1)
+declare <8 x i32> @llvm.cttz.v8i32(<8 x i32>, i1)
+declare <16 x i16> @llvm.cttz.v16i16(<16 x i16>, i1)
+declare <32 x i8> @llvm.cttz.v32i8(<32 x i8>, i1)
diff --git a/test/CodeGen/X86/vector-zext.ll b/test/CodeGen/X86/vector-zext.ll
index 055c25679bc..dca2d976b74 100644
--- a/test/CodeGen/X86/vector-zext.ll
+++ b/test/CodeGen/X86/vector-zext.ll
@@ -4,25 +4,298 @@
 ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mcpu=x86-64 -mattr=+avx | FileCheck %s --check-prefix=AVX --check-prefix=AVX1
 ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mcpu=x86-64 -mattr=+avx2 | FileCheck %s --check-prefix=AVX --check-prefix=AVX2
 
+define <8 x i16> @zext_16i8_to_8i16(<16 x i8> %A) nounwind uwtable readnone ssp {
+; SSE2-LABEL: zext_16i8_to_8i16:
+; SSE2:       # BB#0: # %entry
+; SSE2-NEXT:    pxor %xmm1, %xmm1
+; SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3],xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7]
+; SSE2-NEXT:    retq
+;
+; SSSE3-LABEL: zext_16i8_to_8i16:
+; SSSE3:       # BB#0: # %entry
+; SSSE3-NEXT:    pxor %xmm1, %xmm1
+; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3],xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7]
+; SSSE3-NEXT:    retq
+;
+; SSE41-LABEL: zext_16i8_to_8i16:
+; SSE41:       # BB#0: # %entry
+; SSE41-NEXT:    pmovzxbw {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
+; SSE41-NEXT:    retq
+;
+; AVX-LABEL: zext_16i8_to_8i16:
+; AVX:       # BB#0: # %entry
+; AVX-NEXT:    vpmovzxbw {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
+; AVX-NEXT:    retq
+entry:
+  %B = shufflevector <16 x i8> %A, <16 x i8> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+  %C = zext <8 x i8> %B to <8 x i16>
+  ret <8 x i16> %C
+}
+
+; PR17654
+define <16 x i16> @zext_16i8_to_16i16(<16 x i8> %A) {
+; SSE2-LABEL: zext_16i8_to_16i16:
+; SSE2:       # BB#0: # %entry
+; SSE2-NEXT:    movdqa %xmm0, %xmm1
+; SSE2-NEXT:    pxor %xmm2, %xmm2
+; SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1],xmm0[2],xmm2[2],xmm0[3],xmm2[3],xmm0[4],xmm2[4],xmm0[5],xmm2[5],xmm0[6],xmm2[6],xmm0[7],xmm2[7]
+; SSE2-NEXT:    punpckhbw {{.*#+}} xmm1 = xmm1[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
+; SSE2-NEXT:    pand {{.*}}(%rip), %xmm1
+; SSE2-NEXT:    retq
+;
+; SSSE3-LABEL: zext_16i8_to_16i16:
+; SSSE3:       # BB#0: # %entry
+; SSSE3-NEXT:    movdqa %xmm0, %xmm1
+; SSSE3-NEXT:    pxor %xmm2, %xmm2
+; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1],xmm0[2],xmm2[2],xmm0[3],xmm2[3],xmm0[4],xmm2[4],xmm0[5],xmm2[5],xmm0[6],xmm2[6],xmm0[7],xmm2[7]
+; SSSE3-NEXT:    punpckhbw {{.*#+}} xmm1 = xmm1[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
+; SSSE3-NEXT:    pand {{.*}}(%rip), %xmm1
+; SSSE3-NEXT:    retq
+;
+; SSE41-LABEL: zext_16i8_to_16i16:
+; SSE41:       # BB#0: # %entry
+; SSE41-NEXT:    movdqa %xmm0, %xmm1
+; SSE41-NEXT:    pmovzxbw {{.*#+}} xmm0 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
+; SSE41-NEXT:    punpckhbw {{.*#+}} xmm1 = xmm1[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
+; SSE41-NEXT:    pand {{.*}}(%rip), %xmm1
+; SSE41-NEXT:    retq
+;
+; AVX1-LABEL: zext_16i8_to_16i16:
+; AVX1:       # BB#0: # %entry
+; AVX1-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; AVX1-NEXT:    vpunpckhbw {{.*#+}} xmm1 = xmm0[8],xmm1[8],xmm0[9],xmm1[9],xmm0[10],xmm1[10],xmm0[11],xmm1[11],xmm0[12],xmm1[12],xmm0[13],xmm1[13],xmm0[14],xmm1[14],xmm0[15],xmm1[15]
+; AVX1-NEXT:    vpmovzxbw {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
+; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: zext_16i8_to_16i16:
+; AVX2:       # BB#0: # %entry
+; AVX2-NEXT:    vpmovzxbw {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero,xmm0[8],zero,xmm0[9],zero,xmm0[10],zero,xmm0[11],zero,xmm0[12],zero,xmm0[13],zero,xmm0[14],zero,xmm0[15],zero
+; AVX2-NEXT:    retq
+entry:
+  %B = zext <16 x i8> %A to <16 x i16>
+  ret <16 x i16> %B
+}
+
+define <4 x i32> @zext_16i8_to_4i32(<16 x i8> %A) nounwind uwtable readnone ssp {
+; SSE2-LABEL: zext_16i8_to_4i32:
+; SSE2:       # BB#0: # %entry
+; SSE2-NEXT:    pxor %xmm1, %xmm1
+; SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3],xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7]
+; SSE2-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
+; SSE2-NEXT:    retq
+;
+; SSSE3-LABEL: zext_16i8_to_4i32:
+; SSSE3:       # BB#0: # %entry
+; SSSE3-NEXT:    pxor %xmm1, %xmm1
+; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3],xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7]
+; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
+; SSSE3-NEXT:    retq
+;
+; SSE41-LABEL: zext_16i8_to_4i32:
+; SSE41:       # BB#0: # %entry
+; SSE41-NEXT:    pmovzxbd {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
+; SSE41-NEXT:    retq
+;
+; AVX-LABEL: zext_16i8_to_4i32:
+; AVX:       # BB#0: # %entry
+; AVX-NEXT:    vpmovzxbd {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
+; AVX-NEXT:    retq
+entry:
+  %B = shufflevector <16 x i8> %A, <16 x i8> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %C = zext <4 x i8> %B to <4 x i32>
+  ret <4 x i32> %C
+}
+
+define <8 x i32> @zext_16i8_to_8i32(<16 x i8> %A) nounwind uwtable readnone ssp {
+; SSE2-LABEL: zext_16i8_to_8i32:
+; SSE2:       # BB#0: # %entry
+; SSE2-NEXT:    movdqa %xmm0, %xmm1
+; SSE2-NEXT:    pxor %xmm2, %xmm2
+; SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1],xmm0[2],xmm2[2],xmm0[3],xmm2[3],xmm0[4],xmm2[4],xmm0[5],xmm2[5],xmm0[6],xmm2[6],xmm0[7],xmm2[7]
+; SSE2-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1],xmm0[2],xmm2[2],xmm0[3],xmm2[3]
+; SSE2-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
+; SSE2-NEXT:    punpckhwd {{.*#+}} xmm1 = xmm1[4,4,5,5,6,6,7,7]
+; SSE2-NEXT:    pand {{.*}}(%rip), %xmm1
+; SSE2-NEXT:    retq
+;
+; SSSE3-LABEL: zext_16i8_to_8i32:
+; SSSE3:       # BB#0: # %entry
+; SSSE3-NEXT:    movdqa %xmm0, %xmm1
+; SSSE3-NEXT:    pxor %xmm2, %xmm2
+; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1],xmm0[2],xmm2[2],xmm0[3],xmm2[3],xmm0[4],xmm2[4],xmm0[5],xmm2[5],xmm0[6],xmm2[6],xmm0[7],xmm2[7]
+; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1],xmm0[2],xmm2[2],xmm0[3],xmm2[3]
+; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
+; SSSE3-NEXT:    punpckhwd {{.*#+}} xmm1 = xmm1[4,4,5,5,6,6,7,7]
+; SSSE3-NEXT:    pand {{.*}}(%rip), %xmm1
+; SSSE3-NEXT:    retq
+;
+; SSE41-LABEL: zext_16i8_to_8i32:
+; SSE41:       # BB#0: # %entry
+; SSE41-NEXT:    movdqa %xmm0, %xmm1
+; SSE41-NEXT:    pmovzxbd {{.*#+}} xmm0 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero
+; SSE41-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
+; SSE41-NEXT:    punpckhwd {{.*#+}} xmm1 = xmm1[4,4,5,5,6,6,7,7]
+; SSE41-NEXT:    pand {{.*}}(%rip), %xmm1
+; SSE41-NEXT:    retq
+;
+; AVX1-LABEL: zext_16i8_to_8i32:
+; AVX1:       # BB#0: # %entry
+; AVX1-NEXT:    vpmovzxbw {{.*#+}} xmm1 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
+; AVX1-NEXT:    vpmovzxbd {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
+; AVX1-NEXT:    vpunpckhwd {{.*#+}} xmm1 = xmm1[4,4,5,5,6,6,7,7]
+; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
+; AVX1-NEXT:    vandps {{.*}}(%rip), %ymm0, %ymm0
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: zext_16i8_to_8i32:
+; AVX2:       # BB#0: # %entry
+; AVX2-NEXT:    vpmovzxbd {{.*#+}} ymm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero,xmm0[4],zero,zero,zero,xmm0[5],zero,zero,zero,xmm0[6],zero,zero,zero,xmm0[7],zero,zero,zero
+; AVX2-NEXT:    vpbroadcastd {{.*}}(%rip), %ymm1
+; AVX2-NEXT:    vpand %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    retq
+entry:
+  %B = shufflevector <16 x i8> %A, <16 x i8> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+  %C = zext <8 x i8> %B to <8 x i32>
+  ret <8 x i32> %C
+}
+
+define <2 x i64> @zext_16i8_to_2i64(<16 x i8> %A) nounwind uwtable readnone ssp {
+; SSE2-LABEL: zext_16i8_to_2i64:
+; SSE2:       # BB#0: # %entry
+; SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
+; SSE2-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3]
+; SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0,0,1,1]
+; SSE2-NEXT:    pand {{.*}}(%rip), %xmm0
+; SSE2-NEXT:    retq
+;
+; SSSE3-LABEL: zext_16i8_to_2i64:
+; SSSE3:       # BB#0: # %entry
+; SSSE3-NEXT:    pshufb {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,zero,zero,zero,zero,xmm0[1],zero,zero,zero,zero,zero,zero,zero
+; SSSE3-NEXT:    pand {{.*}}(%rip), %xmm0
+; SSSE3-NEXT:    retq
+;
+; SSE41-LABEL: zext_16i8_to_2i64:
+; SSE41:       # BB#0: # %entry
+; SSE41-NEXT:    pmovzxbq {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,zero,zero,zero,zero,xmm0[1],zero,zero,zero,zero,zero,zero,zero
+; SSE41-NEXT:    pand {{.*}}(%rip), %xmm0
+; SSE41-NEXT:    retq
+;
+; AVX-LABEL: zext_16i8_to_2i64:
+; AVX:       # BB#0: # %entry
+; AVX-NEXT:    vpmovzxbq {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,zero,zero,zero,zero,xmm0[1],zero,zero,zero,zero,zero,zero,zero
+; AVX-NEXT:    vpand {{.*}}(%rip), %xmm0, %xmm0
+; AVX-NEXT:    retq
+entry:
+  %B = shufflevector <16 x i8> %A, <16 x i8> undef, <2 x i32> <i32 0, i32 1>
+  %C = zext <2 x i8> %B to <2 x i64>
+  ret <2 x i64> %C
+}
+
+define <4 x i64> @zext_16i8_to_4i64(<16 x i8> %A) nounwind uwtable readnone ssp {
+; SSE2-LABEL: zext_16i8_to_4i64:
+; SSE2:       # BB#0: # %entry
+; SSE2-NEXT:    movdqa %xmm0, %xmm2
+; SSE2-NEXT:    punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3],xmm2[4],xmm0[4],xmm2[5],xmm0[5],xmm2[6],xmm0[6],xmm2[7],xmm0[7]
+; SSE2-NEXT:    punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3]
+; SSE2-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1]
+; SSE2-NEXT:    movdqa {{.*#+}} xmm3 = [255,255]
+; SSE2-NEXT:    pand %xmm3, %xmm2
+; SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,1,2,1]
+; SSE2-NEXT:    pshuflw {{.*#+}} xmm0 = xmm0[2,1,2,3,4,5,6,7]
+; SSE2-NEXT:    pshufhw {{.*#+}} xmm1 = xmm0[0,1,2,3,7,5,6,7]
+; SSE2-NEXT:    pand %xmm3, %xmm1
+; SSE2-NEXT:    movdqa %xmm2, %xmm0
+; SSE2-NEXT:    retq
+;
+; SSSE3-LABEL: zext_16i8_to_4i64:
+; SSSE3:       # BB#0: # %entry
+; SSSE3-NEXT:    movdqa %xmm0, %xmm2
+; SSSE3-NEXT:    pshufb {{.*#+}} xmm2 = xmm2[0],zero,zero,zero,zero,zero,zero,zero,xmm2[1],zero,zero,zero,zero,zero,zero,zero
+; SSSE3-NEXT:    movdqa {{.*#+}} xmm1 = [255,255]
+; SSSE3-NEXT:    pand %xmm1, %xmm2
+; SSSE3-NEXT:    pshufb {{.*#+}} xmm0 = xmm0[2,2,1,1,2,2,3,3,3,3,5,5,2,2,3,3]
+; SSSE3-NEXT:    pand %xmm0, %xmm1
+; SSSE3-NEXT:    movdqa %xmm2, %xmm0
+; SSSE3-NEXT:    retq
+;
+; SSE41-LABEL: zext_16i8_to_4i64:
+; SSE41:       # BB#0: # %entry
+; SSE41-NEXT:    pmovzxbq {{.*#+}} xmm2 = xmm0[0],zero,zero,zero,zero,zero,zero,zero,xmm0[1],zero,zero,zero,zero,zero,zero,zero
+; SSE41-NEXT:    movdqa {{.*#+}} xmm1 = [255,255]
+; SSE41-NEXT:    pand %xmm1, %xmm2
+; SSE41-NEXT:    pshufb {{.*#+}} xmm0 = xmm0[2,2,1,1,2,2,3,3,3,3,5,5,2,2,3,3]
+; SSE41-NEXT:    pand %xmm0, %xmm1
+; SSE41-NEXT:    movdqa %xmm2, %xmm0
+; SSE41-NEXT:    retq
+;
+; AVX1-LABEL: zext_16i8_to_4i64:
+; AVX1:       # BB#0: # %entry
+; AVX1-NEXT:    vpmovzxbd {{.*#+}} xmm1 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
+; AVX1-NEXT:    vpmovzxbq {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,zero,zero,zero,zero,xmm0[1],zero,zero,zero,zero,zero,zero,zero
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[2,2,3,3]
+; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
+; AVX1-NEXT:    vandps {{.*}}(%rip), %ymm0, %ymm0
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: zext_16i8_to_4i64:
+; AVX2:       # BB#0: # %entry
+; AVX2-NEXT:    vpmovzxbq {{.*#+}} ymm0 = xmm0[0],zero,zero,zero,zero,zero,zero,zero,xmm0[1],zero,zero,zero,zero,zero,zero,zero,xmm0[2],zero,zero,zero,zero,zero,zero,zero,xmm0[3],zero,zero,zero,zero,zero,zero,zero
+; AVX2-NEXT:    vpbroadcastq {{.*}}(%rip), %ymm1
+; AVX2-NEXT:    vpand %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    retq
+entry:
+  %B = shufflevector <16 x i8> %A, <16 x i8> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %C = zext <4 x i8> %B to <4 x i64>
+  ret <4 x i64> %C
+}
+
+define <4 x i32> @zext_8i16_to_4i32(<8 x i16> %A) nounwind uwtable readnone ssp {
+; SSE2-LABEL: zext_8i16_to_4i32:
+; SSE2:       # BB#0: # %entry
+; SSE2-NEXT:    pxor %xmm1, %xmm1
+; SSE2-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
+; SSE2-NEXT:    retq
+;
+; SSSE3-LABEL: zext_8i16_to_4i32:
+; SSSE3:       # BB#0: # %entry
+; SSSE3-NEXT:    pxor %xmm1, %xmm1
+; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
+; SSSE3-NEXT:    retq
+;
+; SSE41-LABEL: zext_8i16_to_4i32:
+; SSE41:       # BB#0: # %entry
+; SSE41-NEXT:    pmovzxwd {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero
+; SSE41-NEXT:    retq
+;
+; AVX-LABEL: zext_8i16_to_4i32:
+; AVX:       # BB#0: # %entry
+; AVX-NEXT:    vpmovzxwd {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero
+; AVX-NEXT:    retq
+entry:
+  %B = shufflevector <8 x i16> %A, <8 x i16> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %C = zext <4 x i16> %B to <4 x i32>
+  ret <4 x i32> %C
+}
+
 define <8 x i32> @zext_8i16_to_8i32(<8 x i16> %A) nounwind uwtable readnone ssp {
 ; SSE2-LABEL: zext_8i16_to_8i32:
 ; SSE2:       # BB#0: # %entry
 ; SSE2-NEXT:    movdqa %xmm0, %xmm1
 ; SSE2-NEXT:    pxor %xmm2, %xmm2
-; SSE2-NEXT:    # kill
 ; SSE2-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1],xmm0[2],xmm2[2],xmm0[3],xmm2[3]
 ; SSE2-NEXT:    punpckhwd {{.*#+}} xmm1 = xmm1[4,4,5,5,6,6,7,7]
-; SSE2-NEXT:    pand .LCPI0_0(%rip), %xmm1
+; SSE2-NEXT:    pand {{.*}}(%rip), %xmm1
 ; SSE2-NEXT:    retq
 ;
 ; SSSE3-LABEL: zext_8i16_to_8i32:
 ; SSSE3:       # BB#0: # %entry
 ; SSSE3-NEXT:    movdqa %xmm0, %xmm1
 ; SSSE3-NEXT:    pxor %xmm2, %xmm2
-; SSSE3-NEXT:    # kill
 ; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1],xmm0[2],xmm2[2],xmm0[3],xmm2[3]
 ; SSSE3-NEXT:    punpckhwd {{.*#+}} xmm1 = xmm1[4,4,5,5,6,6,7,7]
-; SSSE3-NEXT:    pand .LCPI0_0(%rip), %xmm1
+; SSSE3-NEXT:    pand {{.*}}(%rip), %xmm1
 ; SSSE3-NEXT:    retq
 ;
 ; SSE41-LABEL: zext_8i16_to_8i32:
@@ -30,7 +303,7 @@ define <8 x i32> @zext_8i16_to_8i32(<8 x i16> %A) nounwind uwtable readnone ssp
 ; SSE41-NEXT:    movdqa %xmm0, %xmm1
 ; SSE41-NEXT:    pmovzxwd {{.*#+}} xmm0 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero
 ; SSE41-NEXT:    punpckhwd {{.*#+}} xmm1 = xmm1[4,4,5,5,6,6,7,7]
-; SSE41-NEXT:    pand .LCPI0_0(%rip), %xmm1
+; SSE41-NEXT:    pand {{.*}}(%rip), %xmm1
 ; SSE41-NEXT:    retq
 ;
 ; AVX1-LABEL: zext_8i16_to_8i32:
@@ -50,6 +323,124 @@ entry:
   ret <8 x i32>%B
 }
 
+define <2 x i64> @zext_8i16_to_2i64(<8 x i16> %A) nounwind uwtable readnone ssp {
+; SSE2-LABEL: zext_8i16_to_2i64:
+; SSE2:       # BB#0: # %entry
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,1,0,3]
+; SSE2-NEXT:    pshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,5,5,6,7]
+; SSE2-NEXT:    pand {{.*}}(%rip), %xmm0
+; SSE2-NEXT:    retq
+;
+; SSSE3-LABEL: zext_8i16_to_2i64:
+; SSSE3:       # BB#0: # %entry
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,1,0,3]
+; SSSE3-NEXT:    pshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,5,5,6,7]
+; SSSE3-NEXT:    pand {{.*}}(%rip), %xmm0
+; SSSE3-NEXT:    retq
+;
+; SSE41-LABEL: zext_8i16_to_2i64:
+; SSE41:       # BB#0: # %entry
+; SSE41-NEXT:    pmovzxwq {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero
+; SSE41-NEXT:    pand {{.*}}(%rip), %xmm0
+; SSE41-NEXT:    retq
+;
+; AVX-LABEL: zext_8i16_to_2i64:
+; AVX:       # BB#0: # %entry
+; AVX-NEXT:    vpmovzxwq {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero
+; AVX-NEXT:    vpand {{.*}}(%rip), %xmm0, %xmm0
+; AVX-NEXT:    retq
+entry:
+  %B = shufflevector <8 x i16> %A, <8 x i16> undef, <2 x i32> <i32 0, i32 1>
+  %C = zext <2 x i16> %B to <2 x i64>
+  ret <2 x i64> %C
+}
+
+define <4 x i64> @zext_8i16_to_4i64(<8 x i16> %A) nounwind uwtable readnone ssp {
+; SSE2-LABEL: zext_8i16_to_4i64:
+; SSE2:       # BB#0: # %entry
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,1,0,3]
+; SSE2-NEXT:    pshufhw {{.*#+}} xmm2 = xmm1[0,1,2,3,5,5,6,7]
+; SSE2-NEXT:    movdqa {{.*#+}} xmm3 = [65535,65535]
+; SSE2-NEXT:    pand %xmm3, %xmm2
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,1,2,1]
+; SSE2-NEXT:    pshuflw {{.*#+}} xmm0 = xmm0[2,1,2,3,4,5,6,7]
+; SSE2-NEXT:    pshufhw {{.*#+}} xmm1 = xmm0[0,1,2,3,7,5,6,7]
+; SSE2-NEXT:    pand %xmm3, %xmm1
+; SSE2-NEXT:    movdqa %xmm2, %xmm0
+; SSE2-NEXT:    retq
+;
+; SSSE3-LABEL: zext_8i16_to_4i64:
+; SSSE3:       # BB#0: # %entry
+; SSSE3-NEXT:    movdqa %xmm0, %xmm1
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[0,1,0,3]
+; SSSE3-NEXT:    pshufb {{.*#+}} xmm1 = xmm1[4,5,2,3,4,5,6,7,6,7,10,11,4,5,6,7]
+; SSSE3-NEXT:    movdqa {{.*#+}} xmm2 = [65535,65535]
+; SSSE3-NEXT:    pand %xmm2, %xmm1
+; SSSE3-NEXT:    pshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,5,5,6,7]
+; SSSE3-NEXT:    pand %xmm2, %xmm0
+; SSSE3-NEXT:    retq
+;
+; SSE41-LABEL: zext_8i16_to_4i64:
+; SSE41:       # BB#0: # %entry
+; SSE41-NEXT:    pmovzxwq {{.*#+}} xmm2 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero
+; SSE41-NEXT:    movdqa {{.*#+}} xmm1 = [65535,65535]
+; SSE41-NEXT:    pand %xmm1, %xmm2
+; SSE41-NEXT:    pshufb {{.*#+}} xmm0 = xmm0[4,5,2,3,4,5,6,7,6,7,10,11,4,5,6,7]
+; SSE41-NEXT:    pand %xmm0, %xmm1
+; SSE41-NEXT:    movdqa %xmm2, %xmm0
+; SSE41-NEXT:    retq
+;
+; AVX1-LABEL: zext_8i16_to_4i64:
+; AVX1:       # BB#0: # %entry
+; AVX1-NEXT:    vpmovzxwd {{.*#+}} xmm1 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero
+; AVX1-NEXT:    vpmovzxwq {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[2,2,3,3]
+; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
+; AVX1-NEXT:    vandps {{.*}}(%rip), %ymm0, %ymm0
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: zext_8i16_to_4i64:
+; AVX2:       # BB#0: # %entry
+; AVX2-NEXT:    vpmovzxwq {{.*#+}} ymm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
+; AVX2-NEXT:    vpbroadcastq {{.*}}(%rip), %ymm1
+; AVX2-NEXT:    vpand %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    retq
+entry:
+  %B = shufflevector <8 x i16> %A, <8 x i16> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %C = zext <4 x i16> %B to <4 x i64>
+  ret <4 x i64> %C
+}
+
+define <2 x i64> @zext_4i32_to_2i64(<4 x i32> %A) nounwind uwtable readnone ssp {
+; SSE2-LABEL: zext_4i32_to_2i64:
+; SSE2:       # BB#0: # %entry
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,1,1,3]
+; SSE2-NEXT:    pand {{.*}}(%rip), %xmm0
+; SSE2-NEXT:    retq
+;
+; SSSE3-LABEL: zext_4i32_to_2i64:
+; SSSE3:       # BB#0: # %entry
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,1,1,3]
+; SSSE3-NEXT:    pand {{.*}}(%rip), %xmm0
+; SSSE3-NEXT:    retq
+;
+; SSE41-LABEL: zext_4i32_to_2i64:
+; SSE41:       # BB#0: # %entry
+; SSE41-NEXT:    pmovzxdq {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero
+; SSE41-NEXT:    pand {{.*}}(%rip), %xmm0
+; SSE41-NEXT:    retq
+;
+; AVX-LABEL: zext_4i32_to_2i64:
+; AVX:       # BB#0: # %entry
+; AVX-NEXT:    vpmovzxdq {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero
+; AVX-NEXT:    vpand {{.*}}(%rip), %xmm0, %xmm0
+; AVX-NEXT:    retq
+entry:
+  %B = shufflevector <4 x i32> %A, <4 x i32> undef, <2 x i32> <i32 0, i32 1>
+  %C = zext <2 x i32> %B to <2 x i64>
+  ret <2 x i64> %C
+}
+
 define <4 x i64> @zext_4i32_to_4i64(<4 x i32> %A) nounwind uwtable readnone ssp {
 ; SSE2-LABEL: zext_4i32_to_4i64:
 ; SSE2:       # BB#0: # %entry
@@ -98,125 +489,217 @@ entry:
   ret <4 x i64>%B
 }
 
-define <8 x i32> @zext_8i8_to_8i32(<8 x i8> %z) {
-; SSE2-LABEL: zext_8i8_to_8i32:
+define <2 x i64> @load_zext_2i8_to_2i64(<2 x i8> *%ptr) {
+; SSE2-LABEL: load_zext_2i8_to_2i64:
 ; SSE2:       # BB#0: # %entry
-; SSE2-NEXT:    movdqa %xmm0, %xmm2
-; SSE2-NEXT:    punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3]
-; SSE2-NEXT:    movdqa {{.*#+}} xmm1 = [255,255,255,255]
-; SSE2-NEXT:    pand %xmm1, %xmm2
-; SSE2-NEXT:    punpckhwd {{.*#+}} xmm0 = xmm0[4,4,5,5,6,6,7,7]
-; SSE2-NEXT:    pand %xmm0, %xmm1
-; SSE2-NEXT:    movdqa %xmm2, %xmm0
+; SSE2-NEXT:    movzwl (%rdi), %eax
+; SSE2-NEXT:    movd %eax, %xmm0
+; SSE2-NEXT:    pxor %xmm1, %xmm1
+; SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3],xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7]
+; SSE2-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
+; SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
 ; SSE2-NEXT:    retq
 ;
-; SSSE3-LABEL: zext_8i8_to_8i32:
+; SSSE3-LABEL: load_zext_2i8_to_2i64:
 ; SSSE3:       # BB#0: # %entry
-; SSSE3-NEXT:    movdqa %xmm0, %xmm2
-; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3]
-; SSSE3-NEXT:    movdqa {{.*#+}} xmm1 = [255,255,255,255]
-; SSSE3-NEXT:    pand %xmm1, %xmm2
-; SSSE3-NEXT:    punpckhwd {{.*#+}} xmm0 = xmm0[4,4,5,5,6,6,7,7]
-; SSSE3-NEXT:    pand %xmm0, %xmm1
-; SSSE3-NEXT:    movdqa %xmm2, %xmm0
+; SSSE3-NEXT:    movzwl (%rdi), %eax
+; SSSE3-NEXT:    movd %eax, %xmm0
+; SSSE3-NEXT:    pshufb {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,zero,zero,zero,zero,xmm0[1],zero,zero,zero,zero,zero,zero,zero
 ; SSSE3-NEXT:    retq
 ;
-; SSE41-LABEL: zext_8i8_to_8i32:
+; SSE41-LABEL: load_zext_2i8_to_2i64:
 ; SSE41:       # BB#0: # %entry
-; SSE41-NEXT:    pmovzxwd {{.*#+}} xmm2 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero
-; SSE41-NEXT:    movdqa {{.*#+}} xmm1 = [255,255,255,255]
-; SSE41-NEXT:    pand %xmm1, %xmm2
-; SSE41-NEXT:    punpckhwd {{.*#+}} xmm0 = xmm0[4,4,5,5,6,6,7,7]
-; SSE41-NEXT:    pand %xmm0, %xmm1
-; SSE41-NEXT:    movdqa %xmm2, %xmm0
+; SSE41-NEXT:    pmovzxbq {{.*#+}} xmm0 = mem[0],zero,zero,zero,zero,zero,zero,zero,mem[1],zero,zero,zero,zero,zero,zero,zero
 ; SSE41-NEXT:    retq
 ;
-; AVX1-LABEL: zext_8i8_to_8i32:
+; AVX-LABEL: load_zext_2i8_to_2i64:
+; AVX:       # BB#0: # %entry
+; AVX-NEXT:    vpmovzxbq {{.*#+}} xmm0 = mem[0],zero,zero,zero,zero,zero,zero,zero,mem[1],zero,zero,zero,zero,zero,zero,zero
+; AVX-NEXT:    retq
+entry:
+ %X = load <2 x i8>, <2 x i8>* %ptr
+ %Y = zext <2 x i8> %X to <2 x i64>
+ ret <2 x i64> %Y
+}
+
+define <4 x i32> @load_zext_4i8_to_4i32(<4 x i8> *%ptr) {
+; SSE2-LABEL: load_zext_4i8_to_4i32:
+; SSE2:       # BB#0: # %entry
+; SSE2-NEXT:    movd {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; SSE2-NEXT:    pxor %xmm1, %xmm1
+; SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3],xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7]
+; SSE2-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
+; SSE2-NEXT:    retq
+;
+; SSSE3-LABEL: load_zext_4i8_to_4i32:
+; SSSE3:       # BB#0: # %entry
+; SSSE3-NEXT:    movd {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; SSSE3-NEXT:    pxor %xmm1, %xmm1
+; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3],xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7]
+; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
+; SSSE3-NEXT:    retq
+;
+; SSE41-LABEL: load_zext_4i8_to_4i32:
+; SSE41:       # BB#0: # %entry
+; SSE41-NEXT:    pmovzxbd {{.*#+}} xmm0 = mem[0],zero,zero,zero,mem[1],zero,zero,zero,mem[2],zero,zero,zero,mem[3],zero,zero,zero
+; SSE41-NEXT:    retq
+;
+; AVX-LABEL: load_zext_4i8_to_4i32:
+; AVX:       # BB#0: # %entry
+; AVX-NEXT:    vpmovzxbd {{.*#+}} xmm0 = mem[0],zero,zero,zero,mem[1],zero,zero,zero,mem[2],zero,zero,zero,mem[3],zero,zero,zero
+; AVX-NEXT:    retq
+entry:
+ %X = load <4 x i8>, <4 x i8>* %ptr
+ %Y = zext <4 x i8> %X to <4 x i32>
+ ret <4 x i32> %Y
+}
+
+define <4 x i64> @load_zext_4i8_to_4i64(<4 x i8> *%ptr) {
+; SSE2-LABEL: load_zext_4i8_to_4i64:
+; SSE2:       # BB#0: # %entry
+; SSE2-NEXT:    movd {{.*#+}} xmm1 = mem[0],zero,zero,zero
+; SSE2-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3],xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
+; SSE2-NEXT:    punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[0,1,1,3]
+; SSE2-NEXT:    movdqa {{.*#+}} xmm2 = [255,255]
+; SSE2-NEXT:    pand %xmm2, %xmm0
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[2,2,3,3]
+; SSE2-NEXT:    pand %xmm2, %xmm1
+; SSE2-NEXT:    retq
+;
+; SSSE3-LABEL: load_zext_4i8_to_4i64:
+; SSSE3:       # BB#0: # %entry
+; SSSE3-NEXT:    movd {{.*#+}} xmm1 = mem[0],zero,zero,zero
+; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3],xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
+; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[0,1,1,3]
+; SSSE3-NEXT:    movdqa {{.*#+}} xmm2 = [255,255]
+; SSSE3-NEXT:    pand %xmm2, %xmm0
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[2,2,3,3]
+; SSSE3-NEXT:    pand %xmm2, %xmm1
+; SSSE3-NEXT:    retq
+;
+; SSE41-LABEL: load_zext_4i8_to_4i64:
+; SSE41:       # BB#0: # %entry
+; SSE41-NEXT:    pmovzxbq {{.*#+}} xmm0 = mem[0],zero,zero,zero,zero,zero,zero,zero,mem[1],zero,zero,zero,zero,zero,zero,zero
+; SSE41-NEXT:    pmovzxbq {{.*#+}} xmm1 = mem[0],zero,zero,zero,zero,zero,zero,zero,mem[1],zero,zero,zero,zero,zero,zero,zero
+; SSE41-NEXT:    retq
+;
+; AVX1-LABEL: load_zext_4i8_to_4i64:
 ; AVX1:       # BB#0: # %entry
-; AVX1-NEXT:    vpmovzxwd {{.*#+}} xmm1 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero
-; AVX1-NEXT:    vpunpckhwd {{.*#+}} xmm0 = xmm0[4,4,5,5,6,6,7,7]
-; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
-; AVX1-NEXT:    vandps {{.*}}(%rip), %ymm0, %ymm0
+; AVX1-NEXT:    vpmovzxbq {{.*#+}} xmm0 = mem[0],zero,zero,zero,zero,zero,zero,zero,mem[1],zero,zero,zero,zero,zero,zero,zero
+; AVX1-NEXT:    vpmovzxbq {{.*#+}} xmm1 = mem[0],zero,zero,zero,zero,zero,zero,zero,mem[1],zero,zero,zero,zero,zero,zero,zero
+; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
 ; AVX1-NEXT:    retq
 ;
-; AVX2-LABEL: zext_8i8_to_8i32:
+; AVX2-LABEL: load_zext_4i8_to_4i64:
 ; AVX2:       # BB#0: # %entry
-; AVX2-NEXT:    vpmovzxwd {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
-; AVX2-NEXT:    vpbroadcastd {{.*}}(%rip), %ymm1
-; AVX2-NEXT:    vpand %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vpmovzxbq {{.*#+}} ymm0 = mem[0],zero,zero,zero,zero,zero,zero,zero,mem[1],zero,zero,zero,zero,zero,zero,zero,mem[2],zero,zero,zero,zero,zero,zero,zero,mem[3],zero,zero,zero,zero,zero,zero,zero
 ; AVX2-NEXT:    retq
 entry:
-  %t = zext <8 x i8> %z to <8 x i32>
-  ret <8 x i32> %t
+ %X = load <4 x i8>, <4 x i8>* %ptr
+ %Y = zext <4 x i8> %X to <4 x i64>
+ ret <4 x i64> %Y
 }
 
-; PR17654
-define <16 x i16> @zext_16i8_to_16i16(<16 x i8> %z) {
-; SSE2-LABEL: zext_16i8_to_16i16:
+define <8 x i16> @load_zext_8i8_to_8i16(<8 x i8> *%ptr) {
+; SSE2-LABEL: load_zext_8i8_to_8i16:
 ; SSE2:       # BB#0: # %entry
-; SSE2-NEXT:    movdqa %xmm0, %xmm1
-; SSE2-NEXT:    pxor %xmm2, %xmm2
-; SSE2-NEXT:    # kill
-; SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1],xmm0[2],xmm2[2],xmm0[3],xmm2[3],xmm0[4],xmm2[4],xmm0[5],xmm2[5],xmm0[6],xmm2[6],xmm0[7],xmm2[7]
-; SSE2-NEXT:    punpckhbw {{.*#+}} xmm1 = xmm1[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
-; SSE2-NEXT:    pand .LCPI3_0(%rip), %xmm1
+; SSE2-NEXT:    movq {{.*#+}} xmm0 = mem[0],zero
+; SSE2-NEXT:    pxor %xmm1, %xmm1
+; SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3],xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7]
 ; SSE2-NEXT:    retq
 ;
-; SSSE3-LABEL: zext_16i8_to_16i16:
+; SSSE3-LABEL: load_zext_8i8_to_8i16:
 ; SSSE3:       # BB#0: # %entry
-; SSSE3-NEXT:    movdqa %xmm0, %xmm1
-; SSSE3-NEXT:    pxor %xmm2, %xmm2
-; SSSE3-NEXT:    # kill
-; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1],xmm0[2],xmm2[2],xmm0[3],xmm2[3],xmm0[4],xmm2[4],xmm0[5],xmm2[5],xmm0[6],xmm2[6],xmm0[7],xmm2[7]
-; SSSE3-NEXT:    punpckhbw {{.*#+}} xmm1 = xmm1[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
-; SSSE3-NEXT:    pand .LCPI3_0(%rip), %xmm1
+; SSSE3-NEXT:    movq {{.*#+}} xmm0 = mem[0],zero
+; SSSE3-NEXT:    pxor %xmm1, %xmm1
+; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3],xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7]
 ; SSSE3-NEXT:    retq
 ;
-; SSE41-LABEL: zext_16i8_to_16i16:
+; SSE41-LABEL: load_zext_8i8_to_8i16:
 ; SSE41:       # BB#0: # %entry
-; SSE41-NEXT:    movdqa %xmm0, %xmm1
-; SSE41-NEXT:    pmovzxbw %xmm1, %xmm0 {{.*#+}} xmm0 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
-; SSE41-NEXT:    punpckhbw {{.*#+}} xmm1 = xmm1[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
-; SSE41-NEXT:    pand .LCPI3_0(%rip), %xmm1
+; SSE41-NEXT:    pmovzxbw {{.*#+}} xmm0 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero,mem[4],zero,mem[5],zero,mem[6],zero,mem[7],zero
 ; SSE41-NEXT:    retq
 ;
-; AVX1-LABEL: zext_16i8_to_16i16:
+; AVX-LABEL: load_zext_8i8_to_8i16:
+; AVX:       # BB#0: # %entry
+; AVX-NEXT:    vpmovzxbw {{.*#+}} xmm0 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero,mem[4],zero,mem[5],zero,mem[6],zero,mem[7],zero
+; AVX-NEXT:    retq
+entry:
+ %X = load <8 x i8>, <8 x i8>* %ptr
+ %Y = zext <8 x i8> %X to <8 x i16>
+ ret <8 x i16> %Y
+}
+
+define <8 x i32> @load_zext_8i8_to_8i32(<8 x i8> *%ptr) {
+; SSE2-LABEL: load_zext_8i8_to_8i32:
+; SSE2:       # BB#0: # %entry
+; SSE2-NEXT:    movq {{.*#+}} xmm1 = mem[0],zero
+; SSE2-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3],xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
+; SSE2-NEXT:    movdqa %xmm1, %xmm0
+; SSE2-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3]
+; SSE2-NEXT:    movdqa {{.*#+}} xmm2 = [255,255,255,255]
+; SSE2-NEXT:    pand %xmm2, %xmm0
+; SSE2-NEXT:    punpckhwd {{.*#+}} xmm1 = xmm1[4,4,5,5,6,6,7,7]
+; SSE2-NEXT:    pand %xmm2, %xmm1
+; SSE2-NEXT:    retq
+;
+; SSSE3-LABEL: load_zext_8i8_to_8i32:
+; SSSE3:       # BB#0: # %entry
+; SSSE3-NEXT:    movq {{.*#+}} xmm1 = mem[0],zero
+; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3],xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
+; SSSE3-NEXT:    movdqa %xmm1, %xmm0
+; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3]
+; SSSE3-NEXT:    movdqa {{.*#+}} xmm2 = [255,255,255,255]
+; SSSE3-NEXT:    pand %xmm2, %xmm0
+; SSSE3-NEXT:    punpckhwd {{.*#+}} xmm1 = xmm1[4,4,5,5,6,6,7,7]
+; SSSE3-NEXT:    pand %xmm2, %xmm1
+; SSSE3-NEXT:    retq
+;
+; SSE41-LABEL: load_zext_8i8_to_8i32:
+; SSE41:       # BB#0: # %entry
+; SSE41-NEXT:    pmovzxbd {{.*#+}} xmm0 = mem[0],zero,zero,zero,mem[1],zero,zero,zero,mem[2],zero,zero,zero,mem[3],zero,zero,zero
+; SSE41-NEXT:    pmovzxbd {{.*#+}} xmm1 = mem[0],zero,zero,zero,mem[1],zero,zero,zero,mem[2],zero,zero,zero,mem[3],zero,zero,zero
+; SSE41-NEXT:    retq
+;
+; AVX1-LABEL: load_zext_8i8_to_8i32:
 ; AVX1:       # BB#0: # %entry
-; AVX1-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; AVX1-NEXT:    vpunpckhbw {{.*#+}} xmm1 = xmm0[8],xmm1[8],xmm0[9],xmm1[9],xmm0[10],xmm1[10],xmm0[11],xmm1[11],xmm0[12],xmm1[12],xmm0[13],xmm1[13],xmm0[14],xmm1[14],xmm0[15],xmm1[15]
-; AVX1-NEXT:    vpmovzxbw {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
+; AVX1-NEXT:    vpmovzxbd {{.*#+}} xmm0 = mem[0],zero,zero,zero,mem[1],zero,zero,zero,mem[2],zero,zero,zero,mem[3],zero,zero,zero
+; AVX1-NEXT:    vpmovzxbd {{.*#+}} xmm1 = mem[0],zero,zero,zero,mem[1],zero,zero,zero,mem[2],zero,zero,zero,mem[3],zero,zero,zero
 ; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
 ; AVX1-NEXT:    retq
 ;
-; AVX2-LABEL: zext_16i8_to_16i16:
+; AVX2-LABEL: load_zext_8i8_to_8i32:
 ; AVX2:       # BB#0: # %entry
-; AVX2-NEXT:    vpmovzxbw {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero,xmm0[8],zero,xmm0[9],zero,xmm0[10],zero,xmm0[11],zero,xmm0[12],zero,xmm0[13],zero,xmm0[14],zero,xmm0[15],zero
+; AVX2-NEXT:    vpmovzxbd {{.*#+}} ymm0 = mem[0],zero,zero,zero,mem[1],zero,zero,zero,mem[2],zero,zero,zero,mem[3],zero,zero,zero,mem[4],zero,zero,zero,mem[5],zero,zero,zero,mem[6],zero,zero,zero,mem[7],zero,zero,zero
 ; AVX2-NEXT:    retq
 entry:
-  %t = zext <16 x i8> %z to <16 x i16>
-  ret <16 x i16> %t
+ %X = load <8 x i8>, <8 x i8>* %ptr
+ %Y = zext <8 x i8> %X to <8 x i32>
+ ret <8 x i32> %Y
 }
 
 define <16 x i16> @load_zext_16i8_to_16i16(<16 x i8> *%ptr) {
 ; SSE2-LABEL: load_zext_16i8_to_16i16:
-; SSE2:        # BB#0: # %entry
-; SSE2-NEXT:   movdqa (%rdi), %xmm1
-; SSE2-NEXT:   pxor %xmm2, %xmm2
-; SSE2-NEXT:   movdqa %xmm1, %xmm0
-; SSE2-NEXT:   punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1],xmm0[2],xmm2[2],xmm0[3],xmm2[3],xmm0[4],xmm2[4],xmm0[5],xmm2[5],xmm0[6],xmm2[6],xmm0[7],xmm2[7]
-; SSE2-NEXT:   punpckhbw {{.*#+}} xmm1 = xmm1[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
-; SSE2-NEXT:   pand .LCPI4_0(%rip), %xmm1
-; SSE2-NEXT:   retq
+; SSE2:       # BB#0: # %entry
+; SSE2-NEXT:    movdqa (%rdi), %xmm1
+; SSE2-NEXT:    pxor %xmm2, %xmm2
+; SSE2-NEXT:    movdqa %xmm1, %xmm0
+; SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1],xmm0[2],xmm2[2],xmm0[3],xmm2[3],xmm0[4],xmm2[4],xmm0[5],xmm2[5],xmm0[6],xmm2[6],xmm0[7],xmm2[7]
+; SSE2-NEXT:    punpckhbw {{.*#+}} xmm1 = xmm1[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
+; SSE2-NEXT:    pand {{.*}}(%rip), %xmm1
+; SSE2-NEXT:    retq
 ;
 ; SSSE3-LABEL: load_zext_16i8_to_16i16:
-; SSSE3:        # BB#0: # %entry
-; SSSE3-NEXT:   movdqa (%rdi), %xmm1
-; SSSE3-NEXT:   pxor %xmm2, %xmm2
-; SSSE3-NEXT:   movdqa %xmm1, %xmm0
-; SSSE3-NEXT:   punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1],xmm0[2],xmm2[2],xmm0[3],xmm2[3],xmm0[4],xmm2[4],xmm0[5],xmm2[5],xmm0[6],xmm2[6],xmm0[7],xmm2[7]
-; SSSE3-NEXT:   punpckhbw {{.*#+}} xmm1 = xmm1[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
-; SSSE3-NEXT:   pand .LCPI4_0(%rip), %xmm1
-; SSSE3-NEXT:   retq
+; SSSE3:       # BB#0: # %entry
+; SSSE3-NEXT:    movdqa (%rdi), %xmm1
+; SSSE3-NEXT:    pxor %xmm2, %xmm2
+; SSSE3-NEXT:    movdqa %xmm1, %xmm0
+; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1],xmm0[2],xmm2[2],xmm0[3],xmm2[3],xmm0[4],xmm2[4],xmm0[5],xmm2[5],xmm0[6],xmm2[6],xmm0[7],xmm2[7]
+; SSSE3-NEXT:    punpckhbw {{.*#+}} xmm1 = xmm1[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
+; SSSE3-NEXT:    pand {{.*}}(%rip), %xmm1
+; SSSE3-NEXT:    retq
 ;
 ; SSE41-LABEL: load_zext_16i8_to_16i16:
 ; SSE41:       # BB#0: # %entry
@@ -241,26 +724,134 @@ entry:
  ret <16 x i16> %Y
 }
 
+define <2 x i64> @load_zext_2i16_to_2i64(<2 x i16> *%ptr) {
+; SSE2-LABEL: load_zext_2i16_to_2i64:
+; SSE2:       # BB#0: # %entry
+; SSE2-NEXT:    movd {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; SSE2-NEXT:    pxor %xmm1, %xmm1
+; SSE2-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
+; SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
+; SSE2-NEXT:    retq
+;
+; SSSE3-LABEL: load_zext_2i16_to_2i64:
+; SSSE3:       # BB#0: # %entry
+; SSSE3-NEXT:    movd {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; SSSE3-NEXT:    pxor %xmm1, %xmm1
+; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
+; SSSE3-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
+; SSSE3-NEXT:    retq
+;
+; SSE41-LABEL: load_zext_2i16_to_2i64:
+; SSE41:       # BB#0: # %entry
+; SSE41-NEXT:    pmovzxwq {{.*#+}} xmm0 = mem[0],zero,zero,zero,mem[1],zero,zero,zero
+; SSE41-NEXT:    retq
+;
+; AVX-LABEL: load_zext_2i16_to_2i64:
+; AVX:       # BB#0: # %entry
+; AVX-NEXT:    vpmovzxwq {{.*#+}} xmm0 = mem[0],zero,zero,zero,mem[1],zero,zero,zero
+; AVX-NEXT:    retq
+entry:
+ %X = load <2 x i16>, <2 x i16>* %ptr
+ %Y = zext <2 x i16> %X to <2 x i64>
+ ret <2 x i64> %Y
+}
+
+define <4 x i32> @load_zext_4i16_to_4i32(<4 x i16> *%ptr) {
+; SSE2-LABEL: load_zext_4i16_to_4i32:
+; SSE2:       # BB#0: # %entry
+; SSE2-NEXT:    movq {{.*#+}} xmm0 = mem[0],zero
+; SSE2-NEXT:    pxor %xmm1, %xmm1
+; SSE2-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
+; SSE2-NEXT:    retq
+;
+; SSSE3-LABEL: load_zext_4i16_to_4i32:
+; SSSE3:       # BB#0: # %entry
+; SSSE3-NEXT:    movq {{.*#+}} xmm0 = mem[0],zero
+; SSSE3-NEXT:    pxor %xmm1, %xmm1
+; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
+; SSSE3-NEXT:    retq
+;
+; SSE41-LABEL: load_zext_4i16_to_4i32:
+; SSE41:       # BB#0: # %entry
+; SSE41-NEXT:    pmovzxwd {{.*#+}} xmm0 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero
+; SSE41-NEXT:    retq
+;
+; AVX-LABEL: load_zext_4i16_to_4i32:
+; AVX:       # BB#0: # %entry
+; AVX-NEXT:    vpmovzxwd {{.*#+}} xmm0 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero
+; AVX-NEXT:    retq
+entry:
+ %X = load <4 x i16>, <4 x i16>* %ptr
+ %Y = zext <4 x i16> %X to <4 x i32>
+ ret <4 x i32> %Y
+}
+
+define <4 x i64> @load_zext_4i16_to_4i64(<4 x i16> *%ptr) {
+; SSE2-LABEL: load_zext_4i16_to_4i64:
+; SSE2:       # BB#0: # %entry
+; SSE2-NEXT:    movq {{.*#+}} xmm1 = mem[0],zero
+; SSE2-NEXT:    punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[0,1,1,3]
+; SSE2-NEXT:    movdqa {{.*#+}} xmm2 = [65535,65535]
+; SSE2-NEXT:    pand %xmm2, %xmm0
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[2,2,3,3]
+; SSE2-NEXT:    pand %xmm2, %xmm1
+; SSE2-NEXT:    retq
+;
+; SSSE3-LABEL: load_zext_4i16_to_4i64:
+; SSSE3:       # BB#0: # %entry
+; SSSE3-NEXT:    movq {{.*#+}} xmm1 = mem[0],zero
+; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[0,1,1,3]
+; SSSE3-NEXT:    movdqa {{.*#+}} xmm2 = [65535,65535]
+; SSSE3-NEXT:    pand %xmm2, %xmm0
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[2,2,3,3]
+; SSSE3-NEXT:    pand %xmm2, %xmm1
+; SSSE3-NEXT:    retq
+;
+; SSE41-LABEL: load_zext_4i16_to_4i64:
+; SSE41:       # BB#0: # %entry
+; SSE41-NEXT:    pmovzxwq {{.*#+}} xmm0 = mem[0],zero,zero,zero,mem[1],zero,zero,zero
+; SSE41-NEXT:    pmovzxwq {{.*#+}} xmm1 = mem[0],zero,zero,zero,mem[1],zero,zero,zero
+; SSE41-NEXT:    retq
+;
+; AVX1-LABEL: load_zext_4i16_to_4i64:
+; AVX1:       # BB#0: # %entry
+; AVX1-NEXT:    vpmovzxwq {{.*#+}} xmm0 = mem[0],zero,zero,zero,mem[1],zero,zero,zero
+; AVX1-NEXT:    vpmovzxwq {{.*#+}} xmm1 = mem[0],zero,zero,zero,mem[1],zero,zero,zero
+; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: load_zext_4i16_to_4i64:
+; AVX2:       # BB#0: # %entry
+; AVX2-NEXT:    vpmovzxwq {{.*#+}} ymm0 = mem[0],zero,zero,zero,mem[1],zero,zero,zero,mem[2],zero,zero,zero,mem[3],zero,zero,zero
+; AVX2-NEXT:    retq
+entry:
+ %X = load <4 x i16>, <4 x i16>* %ptr
+ %Y = zext <4 x i16> %X to <4 x i64>
+ ret <4 x i64> %Y
+}
+
 define <8 x i32> @load_zext_8i16_to_8i32(<8 x i16> *%ptr) {
 ; SSE2-LABEL: load_zext_8i16_to_8i32:
-; SSE2:          # BB#0: # %entry
-; SSE2-NEXT:   movdqa (%rdi), %xmm1
-; SSE2-NEXT:   pxor %xmm2, %xmm2
-; SSE2-NEXT:   movdqa %xmm1, %xmm0
-; SSE2-NEXT:   punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1],xmm0[2],xmm2[2],xmm0[3],xmm2[3]
-; SSE2-NEXT:   punpckhwd {{.*#+}} xmm1 = xmm1[4,4,5,5,6,6,7,7]
-; SSE2-NEXT:   pand .LCPI5_0(%rip), %xmm1
-; SSE2-NEXT:   retq
+; SSE2:       # BB#0: # %entry
+; SSE2-NEXT:    movdqa (%rdi), %xmm1
+; SSE2-NEXT:    pxor %xmm2, %xmm2
+; SSE2-NEXT:    movdqa %xmm1, %xmm0
+; SSE2-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1],xmm0[2],xmm2[2],xmm0[3],xmm2[3]
+; SSE2-NEXT:    punpckhwd {{.*#+}} xmm1 = xmm1[4,4,5,5,6,6,7,7]
+; SSE2-NEXT:    pand {{.*}}(%rip), %xmm1
+; SSE2-NEXT:    retq
 ;
 ; SSSE3-LABEL: load_zext_8i16_to_8i32:
-; SSSE3:        # BB#0: # %entry
-; SSSE3-NEXT:   movdqa (%rdi), %xmm1
-; SSSE3-NEXT:   pxor %xmm2, %xmm2
-; SSSE3-NEXT:   movdqa %xmm1, %xmm0
-; SSSE3-NEXT:   punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1],xmm0[2],xmm2[2],xmm0[3],xmm2[3]
-; SSSE3-NEXT:   punpckhwd {{.*#+}} xmm1 = xmm1[4,4,5,5,6,6,7,7]
-; SSSE3-NEXT:   pand .LCPI5_0(%rip), %xmm1
-; SSSE3-NEXT:   retq
+; SSSE3:       # BB#0: # %entry
+; SSSE3-NEXT:    movdqa (%rdi), %xmm1
+; SSSE3-NEXT:    pxor %xmm2, %xmm2
+; SSSE3-NEXT:    movdqa %xmm1, %xmm0
+; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1],xmm0[2],xmm2[2],xmm0[3],xmm2[3]
+; SSSE3-NEXT:    punpckhwd {{.*#+}} xmm1 = xmm1[4,4,5,5,6,6,7,7]
+; SSSE3-NEXT:    pand {{.*}}(%rip), %xmm1
+; SSSE3-NEXT:    retq
 ;
 ; SSE41-LABEL: load_zext_8i16_to_8i32:
 ; SSE41:       # BB#0: # %entry
@@ -282,7 +873,37 @@ define <8 x i32> @load_zext_8i16_to_8i32(<8 x i16> *%ptr) {
 entry:
  %X = load <8 x i16>, <8 x i16>* %ptr
  %Y = zext <8 x i16> %X to <8 x i32>
- ret <8 x i32>%Y
+ ret <8 x i32> %Y
+}
+
+define <2 x i64> @load_zext_2i32_to_2i64(<2 x i32> *%ptr) {
+; SSE2-LABEL: load_zext_2i32_to_2i64:
+; SSE2:       # BB#0: # %entry
+; SSE2-NEXT:    movq {{.*#+}} xmm0 = mem[0],zero
+; SSE2-NEXT:    pxor %xmm1, %xmm1
+; SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
+; SSE2-NEXT:    retq
+;
+; SSSE3-LABEL: load_zext_2i32_to_2i64:
+; SSSE3:       # BB#0: # %entry
+; SSSE3-NEXT:    movq {{.*#+}} xmm0 = mem[0],zero
+; SSSE3-NEXT:    pxor %xmm1, %xmm1
+; SSSE3-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
+; SSSE3-NEXT:    retq
+;
+; SSE41-LABEL: load_zext_2i32_to_2i64:
+; SSE41:       # BB#0: # %entry
+; SSE41-NEXT:    pmovzxdq {{.*#+}} xmm0 = mem[0],zero,mem[1],zero
+; SSE41-NEXT:    retq
+;
+; AVX-LABEL: load_zext_2i32_to_2i64:
+; AVX:       # BB#0: # %entry
+; AVX-NEXT:    vpmovzxdq {{.*#+}} xmm0 = mem[0],zero,mem[1],zero
+; AVX-NEXT:    retq
+entry:
+ %X = load <2 x i32>, <2 x i32>* %ptr
+ %Y = zext <2 x i32> %X to <2 x i64>
+ ret <2 x i64> %Y
 }
 
 define <4 x i64> @load_zext_4i32_to_4i64(<4 x i32> *%ptr) {
@@ -326,7 +947,59 @@ define <4 x i64> @load_zext_4i32_to_4i64(<4 x i32> *%ptr) {
 entry:
  %X = load <4 x i32>, <4 x i32>* %ptr
  %Y = zext <4 x i32> %X to <4 x i64>
- ret <4 x i64>%Y
+ ret <4 x i64> %Y
+}
+
+define <8 x i32> @zext_8i8_to_8i32(<8 x i8> %z) {
+; SSE2-LABEL: zext_8i8_to_8i32:
+; SSE2:       # BB#0: # %entry
+; SSE2-NEXT:    movdqa %xmm0, %xmm2
+; SSE2-NEXT:    punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3]
+; SSE2-NEXT:    movdqa {{.*#+}} xmm1 = [255,255,255,255]
+; SSE2-NEXT:    pand %xmm1, %xmm2
+; SSE2-NEXT:    punpckhwd {{.*#+}} xmm0 = xmm0[4,4,5,5,6,6,7,7]
+; SSE2-NEXT:    pand %xmm0, %xmm1
+; SSE2-NEXT:    movdqa %xmm2, %xmm0
+; SSE2-NEXT:    retq
+;
+; SSSE3-LABEL: zext_8i8_to_8i32:
+; SSSE3:       # BB#0: # %entry
+; SSSE3-NEXT:    movdqa %xmm0, %xmm2
+; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3]
+; SSSE3-NEXT:    movdqa {{.*#+}} xmm1 = [255,255,255,255]
+; SSSE3-NEXT:    pand %xmm1, %xmm2
+; SSSE3-NEXT:    punpckhwd {{.*#+}} xmm0 = xmm0[4,4,5,5,6,6,7,7]
+; SSSE3-NEXT:    pand %xmm0, %xmm1
+; SSSE3-NEXT:    movdqa %xmm2, %xmm0
+; SSSE3-NEXT:    retq
+;
+; SSE41-LABEL: zext_8i8_to_8i32:
+; SSE41:       # BB#0: # %entry
+; SSE41-NEXT:    pmovzxwd {{.*#+}} xmm2 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero
+; SSE41-NEXT:    movdqa {{.*#+}} xmm1 = [255,255,255,255]
+; SSE41-NEXT:    pand %xmm1, %xmm2
+; SSE41-NEXT:    punpckhwd {{.*#+}} xmm0 = xmm0[4,4,5,5,6,6,7,7]
+; SSE41-NEXT:    pand %xmm0, %xmm1
+; SSE41-NEXT:    movdqa %xmm2, %xmm0
+; SSE41-NEXT:    retq
+;
+; AVX1-LABEL: zext_8i8_to_8i32:
+; AVX1:       # BB#0: # %entry
+; AVX1-NEXT:    vpmovzxwd {{.*#+}} xmm1 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero
+; AVX1-NEXT:    vpunpckhwd {{.*#+}} xmm0 = xmm0[4,4,5,5,6,6,7,7]
+; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
+; AVX1-NEXT:    vandps {{.*}}(%rip), %ymm0, %ymm0
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: zext_8i8_to_8i32:
+; AVX2:       # BB#0: # %entry
+; AVX2-NEXT:    vpmovzxwd {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
+; AVX2-NEXT:    vpbroadcastd {{.*}}(%rip), %ymm1
+; AVX2-NEXT:    vpand %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    retq
+entry:
+  %t = zext <8 x i8> %z to <8 x i32>
+  ret <8 x i32> %t
 }
 
 define <8 x i32> @shuf_zext_8i16_to_8i32(<8 x i16> %A) nounwind uwtable readnone ssp {
@@ -334,7 +1007,6 @@ define <8 x i32> @shuf_zext_8i16_to_8i32(<8 x i16> %A) nounwind uwtable readnone
 ; SSE2:       # BB#0: # %entry
 ; SSE2-NEXT:    movdqa %xmm0, %xmm1
 ; SSE2-NEXT:    pxor %xmm2, %xmm2
-; SSE2-NEXT:    # kill
 ; SSE2-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1],xmm0[2],xmm2[2],xmm0[3],xmm2[3]
 ; SSE2-NEXT:    punpckhwd {{.*#+}} xmm1 = xmm1[4],xmm2[4],xmm1[5],xmm2[5],xmm1[6],xmm2[6],xmm1[7],xmm2[7]
 ; SSE2-NEXT:    retq
@@ -343,7 +1015,6 @@ define <8 x i32> @shuf_zext_8i16_to_8i32(<8 x i16> %A) nounwind uwtable readnone
 ; SSSE3:       # BB#0: # %entry
 ; SSSE3-NEXT:    movdqa %xmm0, %xmm1
 ; SSSE3-NEXT:    pxor %xmm2, %xmm2
-; SSSE3-NEXT:    # kill
 ; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1],xmm0[2],xmm2[2],xmm0[3],xmm2[3]
 ; SSSE3-NEXT:    punpckhwd {{.*#+}} xmm1 = xmm1[4],xmm2[4],xmm1[5],xmm2[5],xmm1[6],xmm2[6],xmm1[7],xmm2[7]
 ; SSSE3-NEXT:    retq
@@ -358,15 +1029,14 @@ define <8 x i32> @shuf_zext_8i16_to_8i32(<8 x i16> %A) nounwind uwtable readnone
 ;
 ; AVX1-LABEL: shuf_zext_8i16_to_8i32:
 ; AVX1:       # BB#0: # %entry
-; AVX1-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; AVX1-NEXT:    vpunpckhwd {{.*#+}} xmm1 = xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7]
-; AVX1-NEXT:    vpmovzxwd {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero
+; AVX1-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; AVX1-NEXT:    vpunpckhwd {{.*#+}} xmm1 = xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7]
+; AVX1-NEXT:    vpmovzxwd {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero
 ; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
 ; AVX1-NEXT:    retq
 ;
 ; AVX2-LABEL: shuf_zext_8i16_to_8i32:
 ; AVX2:       # BB#0: # %entry
-; AVX2-NEXT:    # kill
 ; AVX2-NEXT:    vpmovzxwd {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
 ; AVX2-NEXT:    retq
 entry:
@@ -380,7 +1050,6 @@ define <4 x i64> @shuf_zext_4i32_to_4i64(<4 x i32> %A) nounwind uwtable readnone
 ; SSE2:       # BB#0: # %entry
 ; SSE2-NEXT:    movdqa %xmm0, %xmm1
 ; SSE2-NEXT:    pxor %xmm2, %xmm2
-; SSE2-NEXT:    # kill
 ; SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1]
 ; SSE2-NEXT:    punpckhdq {{.*#+}} xmm1 = xmm1[2],xmm2[2],xmm1[3],xmm2[3]
 ; SSE2-NEXT:    retq
@@ -389,7 +1058,6 @@ define <4 x i64> @shuf_zext_4i32_to_4i64(<4 x i32> %A) nounwind uwtable readnone
 ; SSSE3:       # BB#0: # %entry
 ; SSSE3-NEXT:    movdqa %xmm0, %xmm1
 ; SSSE3-NEXT:    pxor %xmm2, %xmm2
-; SSSE3-NEXT:    # kill
 ; SSSE3-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1]
 ; SSSE3-NEXT:    punpckhdq {{.*#+}} xmm1 = xmm1[2],xmm2[2],xmm1[3],xmm2[3]
 ; SSSE3-NEXT:    retq
@@ -404,16 +1072,15 @@ define <4 x i64> @shuf_zext_4i32_to_4i64(<4 x i32> %A) nounwind uwtable readnone
 ;
 ; AVX1-LABEL: shuf_zext_4i32_to_4i64:
 ; AVX1:       # BB#0: # %entry
-; AVX1-NEXT:    vinsertps {{.*#+}} xmm1 = xmm0[0],zero,xmm0[1],zero
-; AVX1-NEXT:    vxorpd %xmm2, %xmm2, %xmm2
-; AVX1-NEXT:    vblendpd {{.*#+}} xmm0 = xmm2[0],xmm0[1]
-; AVX1-NEXT:    vpermilps {{.*#+}} xmm0 = xmm0[2,0,3,0]
-; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
+; AVX1-NEXT:    vinsertps {{.*#+}} xmm1 = xmm0[0],zero,xmm0[1],zero
+; AVX1-NEXT:    vxorpd %xmm2, %xmm2, %xmm2
+; AVX1-NEXT:    vblendpd {{.*#+}} xmm0 = xmm2[0],xmm0[1]
+; AVX1-NEXT:    vpermilps {{.*#+}} xmm0 = xmm0[2,0,3,0]
+; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
 ; AVX1-NEXT:    retq
 ;
 ; AVX2-LABEL: shuf_zext_4i32_to_4i64:
 ; AVX2:       # BB#0: # %entry
-; AVX2-NEXT:    # kill
 ; AVX2-NEXT:    vpmovzxdq {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero
 ; AVX2-NEXT:    retq
 entry:
@@ -425,17 +1092,16 @@ entry:
 define <8 x i32> @shuf_zext_8i8_to_8i32(<8 x i8> %A) {
 ; SSE2-LABEL: shuf_zext_8i8_to_8i32:
 ; SSE2:       # BB#0: # %entry
-; SSE2-NEXT:    pand .LCPI9_0(%rip), %xmm0
-; SSE2-NEXT:    packuswb %xmm0, %xmm0
-; SSE2-NEXT:    pxor %xmm1, %xmm1
-; SSE2-NEXT:    movdqa %xmm0, %xmm2
-; SSE2-NEXT:    punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3],xmm2[4],xmm1[4],xmm2[5],xmm1[5],xmm2[6],xmm1[6],xmm2[7],xmm1[7]
-; SSE2-NEXT:    punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3]
-; SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
-; SSE2-NEXT:    punpckhwd {{.*#+}} xmm0 = xmm0[4,4,5,5,6,6,7,7]
-; SSE2-NEXT:    movdqa {{.*#+}} xmm1 = [0,255,255,255,0,255,255,255,0,255,255,255,0,255,255,255]
-; SSE2-NEXT:    pandn %xmm0, %xmm1
-; SSE2-NEXT:    movdqa %xmm2, %xmm0
+; SSE2-NEXT:    movdqa %xmm0, %xmm1
+; SSE2-NEXT:    pand {{.*}}(%rip), %xmm1
+; SSE2-NEXT:    packuswb %xmm1, %xmm1
+; SSE2-NEXT:    pxor %xmm2, %xmm2
+; SSE2-NEXT:    movdqa %xmm1, %xmm0
+; SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1],xmm0[2],xmm2[2],xmm0[3],xmm2[3],xmm0[4],xmm2[4],xmm0[5],xmm2[5],xmm0[6],xmm2[6],xmm0[7],xmm2[7]
+; SSE2-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1],xmm0[2],xmm2[2],xmm0[3],xmm2[3]
+; SSE2-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
+; SSE2-NEXT:    punpckhwd {{.*#+}} xmm1 = xmm1[4,4,5,5,6,6,7,7]
+; SSE2-NEXT:    pand {{.*}}(%rip), %xmm1
 ; SSE2-NEXT:    retq
 ;
 ; SSSE3-LABEL: shuf_zext_8i8_to_8i32:
diff --git a/test/CodeGen/X86/vector-zmov.ll b/test/CodeGen/X86/vector-zmov.ll
index cf592b1e9f4..29868355905 100644
--- a/test/CodeGen/X86/vector-zmov.ll
+++ b/test/CodeGen/X86/vector-zmov.ll
@@ -5,15 +5,16 @@
 ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mcpu=x86-64 -mattr=+avx2 | FileCheck %s --check-prefix=AVX --check-prefix=AVX2
 
 define <4 x i32> @load_zmov_4i32_to_0zzz(<4 x i32> *%ptr) {
-; SSE-LABEL:  load_zmov_4i32_to_0zzz:
-; SSE:        # BB#0: # %entry
-; SSE-NEXT:   movd (%rdi), %xmm0
-; SSE-NEXT:   retq
+; SSE-LABEL: load_zmov_4i32_to_0zzz:
+; SSE:       # BB#0: # %entry
+; SSE-NEXT:    movd {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; SSE-NEXT:    retq
+;
+; AVX-LABEL: load_zmov_4i32_to_0zzz:
+; AVX:       # BB#0: # %entry
+; AVX-NEXT:    vmovd {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; AVX-NEXT:    retq
 
-; AVX-LABEL:  load_zmov_4i32_to_0zzz:
-; AVX:        # BB#0: # %entry
-; AVX-NEXT:   vmovd (%rdi), %xmm0
-; AVX-NEXT:   retq
 entry:
   %X = load <4 x i32>, <4 x i32>* %ptr
   %Y = shufflevector <4 x i32> %X, <4 x i32> zeroinitializer, <4 x i32> <i32 0, i32 4, i32 4, i32 4>
@@ -21,15 +22,16 @@ entry:
 }
 
 define <2 x i64> @load_zmov_2i64_to_0z(<2 x i64> *%ptr) {
-; SSE-LABEL:  load_zmov_2i64_to_0z:
-; SSE:        # BB#0: # %entry
-; SSE-NEXT:   movq (%rdi), %xmm0
-; SSE-NEXT:   retq
+; SSE-LABEL: load_zmov_2i64_to_0z:
+; SSE:       # BB#0: # %entry
+; SSE-NEXT:    movq {{.*#+}} xmm0 = mem[0],zero
+; SSE-NEXT:    retq
+;
+; AVX-LABEL: load_zmov_2i64_to_0z:
+; AVX:       # BB#0: # %entry
+; AVX-NEXT:    vmovq {{.*#+}} xmm0 = mem[0],zero
+; AVX-NEXT:    retq
 
-; AVX-LABEL:  load_zmov_2i64_to_0z:
-; AVX:        # BB#0: # %entry
-; AVX-NEXT:   vmovq (%rdi), %xmm0
-; AVX-NEXT:   retq
 entry:
   %X = load <2 x i64>, <2 x i64>* %ptr
   %Y = shufflevector <2 x i64> %X, <2 x i64> zeroinitializer, <2 x i32> <i32 0, i32 2>
diff --git a/test/CodeGen/X86/virtual-registers-cleared-in-machine-functions-liveins.ll b/test/CodeGen/X86/virtual-registers-cleared-in-machine-functions-liveins.ll
new file mode 100644
index 00000000000..2ff8c3a9028
--- /dev/null
+++ b/test/CodeGen/X86/virtual-registers-cleared-in-machine-functions-liveins.ll
@@ -0,0 +1,19 @@
+; RUN: llc -mtriple=x86_64-unknown-unknown -o /dev/null -stop-after machine-scheduler %s | FileCheck %s --check-prefix=PRE-RA
+; RUN: llc -mtriple=x86_64-unknown-unknown -o /dev/null -stop-after prologepilog %s | FileCheck %s --check-prefix=POST-RA
+
+; This test verifies that the virtual register references in machine function's
+; liveins are cleared after register allocation.
+
+define i32 @test(i32 %a, i32 %b) {
+body:
+  %c = mul i32 %a, %b
+  ret i32 %c
+}
+
+; PRE-RA: liveins:
+; PRE-RA-NEXT: - { reg: '%edi', virtual-reg: '%0' }
+; PRE-RA-NEXT: - { reg: '%esi', virtual-reg: '%1' }
+
+; POST-RA: liveins:
+; POST-RA-NEXT: - { reg: '%edi' }
+; POST-RA-NEXT: - { reg: '%esi' }
diff --git a/test/CodeGen/X86/visibility.ll b/test/CodeGen/X86/visibility.ll
index 580c3dc9266..be7fd96f2dd 100644
--- a/test/CodeGen/X86/visibility.ll
+++ b/test/CodeGen/X86/visibility.ll
@@ -2,13 +2,19 @@
 
 @zed = external hidden constant i32
 
+define available_externally hidden void @baz() {
+  ret void
+}
+
 define hidden void @foo() nounwind {
 entry:
   call void @bar(i32* @zed)
+  call void @baz()
   ret void
 }
 
 declare hidden void @bar(i32*)
 
 ;CHECK: .hidden	zed
+;CHECK: .hidden	baz
 ;CHECK: .hidden	bar
diff --git a/test/CodeGen/X86/vselect-avx.ll b/test/CodeGen/X86/vselect-avx.ll
index ff26aebaac6..de04a097de0 100644
--- a/test/CodeGen/X86/vselect-avx.ll
+++ b/test/CodeGen/X86/vselect-avx.ll
@@ -14,8 +14,8 @@ target triple = "x86_64-apple-macosx"
 ; <rdar://problem/18675020>
 
 ; CHECK-LABEL: test:
-; CHECK: vmovdqa {{.*#+}} xmm0 = [65535,0,0,65535]
-; CHECK: vmovdqa {{.*#+}} xmm2 = [65533,124,125,14807]
+; CHECK: vmovdqa {{.*#+}} xmm1 = [65533,124,125,14807]
+; CHECK: vmovdqa {{.*#+}} xmm1 = [65535,0,0,65535]
 ; CHECK: ret
 define void @test(<4 x i16>* %a, <4 x i16>* %b) {
 body:
@@ -33,13 +33,14 @@ body:
 ; of the condition.
 ;
 ; CHECK-LABEL: test2:
-; CHECK:	vpslld	$31, %xmm0, %xmm0
-; CHECK-NEXT:	vpmovsxdq	%xmm0, %xmm1
-; CHECK-NEXT:	vpshufd	$78, %xmm0, %xmm0       ## xmm0 = xmm0[2,3,0,1]
-; CHECK-NEXT:	vpmovsxdq	%xmm0, %xmm0
-; CHECK-NEXT:	vinsertf128	$1, %xmm0, %ymm1, [[MASK:%ymm[0-9]+]]
-; CHECK: vblendvpd	[[MASK]]
-; CHECK: retq
+; CHECK:       vpslld $31, %xmm0, %xmm0
+; CHECK-NEXT:  vpsrad $31, %xmm0, %xmm0
+; CHECK-NEXT:  vpmovsxdq %xmm0, %xmm1
+; CHECK-NEXT:  vpshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
+; CHECK-NEXT:  vpmovsxdq %xmm0, %xmm0
+; CHECK-NEXT:  vinsertf128 $1, %xmm0, %ymm1, [[MASK:%ymm[0-9]+]]
+; CHECK:       vblendvpd [[MASK]]
+; CHECK:       retq
 define void @test2(double** %call1559, i64 %indvars.iv4198, <4 x i1> %tmp1895) {
 bb:
   %arrayidx1928 = getelementptr inbounds double*, double** %call1559, i64 %indvars.iv4198
diff --git a/test/CodeGen/X86/vshift-3.ll b/test/CodeGen/X86/vshift-3.ll
index 0bdb32fcb86..f368029e4b4 100644
--- a/test/CodeGen/X86/vshift-3.ll
+++ b/test/CodeGen/X86/vshift-3.ll
@@ -3,13 +3,12 @@
 ; test vector shifts converted to proper SSE2 vector shifts when the shift
 ; amounts are the same.
 
-; Note that x86 does have ashr 
+; Note that x86 does have ashr
 
-; shift1a can't use a packed shift
 define void @shift1a(<2 x i64> %val, <2 x i64>* %dst) nounwind {
 entry:
 ; CHECK-LABEL: shift1a:
-; CHECK: sarl
+; CHECK: psrad $31
   %ashr = ashr <2 x i64> %val, < i64 32, i64 32 >
   store <2 x i64> %ashr, <2 x i64>* %dst
   ret void
diff --git a/test/CodeGen/X86/webkit-jscc.ll b/test/CodeGen/X86/webkit-jscc.ll
new file mode 100644
index 00000000000..a58c53e024e
--- /dev/null
+++ b/test/CodeGen/X86/webkit-jscc.ll
@@ -0,0 +1,18 @@
+; RUN: llc -mtriple=x86_64-apple-darwin -mcpu=corei7 < %s | FileCheck %s
+; RUN: llc -mtriple=x86_64-linux-gnu -mcpu=corei7 < %s | FileCheck %s
+; RUN: llc -mtriple=x86_64-windows-gnu -mcpu=corei7 < %s | FileCheck %s
+; RUN: llc -mtriple=x86_64-windows-msvc -mcpu=corei7 < %s | FileCheck %s
+
+define webkit_jscc i32 @simple_jscall(i32 %a, i32 %b, i32 %c) {
+  %ab = add i32 %a, %b
+  %abc = add i32 %ab, %c
+  ret i32 %abc
+}
+
+; 32-bit integers are only aligned to 4 bytes, even on x64. They are *not*
+; promoted to i64.
+
+; CHECK: simple_jscall:
+; CHECK: addl 8(%rsp), %eax
+; CHECK-NEXT: addl 12(%rsp), %eax
+; CHECK-NEXT: retq
diff --git a/test/CodeGen/X86/widen_conv-2.ll b/test/CodeGen/X86/widen_conv-2.ll
index 906f7cdafb9..c8646c6489a 100644
--- a/test/CodeGen/X86/widen_conv-2.ll
+++ b/test/CodeGen/X86/widen_conv-2.ll
@@ -1,8 +1,9 @@
 ; RUN: llc < %s -march=x86 -mattr=+sse4.2 | FileCheck %s
-; CHECK: {{cwtl|movswl}}
-; CHECK: {{cwtl|movswl}}
+; CHECK: psllq $48, %xmm0
+; CHECK: psrad $16, %xmm0
+; CHECK: pshufd {{.*#+}} xmm0 = xmm0[1,3,2,3]
 
-; sign extension v2i32 to v2i16
+; sign extension v2i16 to v2i32
 
 define void @convert(<2 x i32>* %dst.addr, <2 x i16> %src) nounwind {
 entry:
diff --git a/test/CodeGen/X86/widen_conv-3.ll b/test/CodeGen/X86/widen_conv-3.ll
index a2f3d7b82b3..0a6eea049d3 100644
--- a/test/CodeGen/X86/widen_conv-3.ll
+++ b/test/CodeGen/X86/widen_conv-3.ll
@@ -1,5 +1,5 @@
 ; RUN: llc < %s -march=x86 -mattr=+sse4.2 | FileCheck %s
-; CHECK: cvtsi2ss
+; CHECK: cvtdq2ps
 
 ; sign to float v2i16 to v2f32
 
diff --git a/test/CodeGen/X86/widen_load-1.ll b/test/CodeGen/X86/widen_load-1.ll
index 849f9b9be26..c670b45df74 100644
--- a/test/CodeGen/X86/widen_load-1.ll
+++ b/test/CodeGen/X86/widen_load-1.ll
@@ -9,8 +9,8 @@
 ; SSE: movaps  %xmm0, (%rsp)
 ; SSE: callq   killcommon
 
-; AVX: vmovdqa    compl+128(%rip), %xmm0
-; AVX: vmovdqa  %xmm0, (%rsp)
+; AVX: vmovaps    compl+128(%rip), %xmm0
+; AVX: vmovaps  %xmm0, (%rsp)
 ; AVX: callq   killcommon
 
 @compl = linkonce global [20 x i64] zeroinitializer, align 64 ; <[20 x i64]*> [#uses=1]
diff --git a/test/CodeGen/X86/widen_load-2.ll b/test/CodeGen/X86/widen_load-2.ll
index f5ddc0eacc6..6f1bd754123 100644
--- a/test/CodeGen/X86/widen_load-2.ll
+++ b/test/CodeGen/X86/widen_load-2.ll
@@ -194,17 +194,9 @@ define void @rot(%i8vec3pack* nocapture sret %result, %i8vec3pack* %X, %i8vec3pa
 ; CHECK-NEXT:    movl    (%[[PTR0]]), [[TMP1:%e[abcd]+x]]
 ; CHECK-NEXT:    movl    [[TMP1]], [[TMP2:.*]]
 ; CHECK-NEXT:    pmovzxbd [[TMP2]], %[[X0:xmm[0-9]+]]
-; CHECK-NEXT:    pextrd  $1, %[[X0]], %e[[R0:[abcd]]]x
-; CHECK-NEXT:    shrl    %e[[R0]]x
-; CHECK-NEXT:    movd    %[[X0]], %e[[R1:[abcd]]]x
-; CHECK-NEXT:    shrl    %e[[R1]]x
-; CHECK-NEXT:    movd    %e[[R1]]x, %[[X1:xmm[0-9]+]]
-; CHECK-NEXT:    pinsrd  $1, %e[[R0]]x, %[[X1]]
-; CHECK-NEXT:    pextrd  $2, %[[X0]], %e[[R0:[abcd]]]x
-; CHECK-NEXT:    shrl    %e[[R0]]x
-; CHECK-NEXT:    pinsrd  $2, %e[[R0]]x, %[[X1]]
-; CHECK-NEXT:    pextrd  $3, %[[X0]], %e[[R0:[abcd]]]x
-; CHECK-NEXT:    pinsrd  $3, %e[[R0]]x, %[[X1]]
+; CHECK-NEXT:    movdqa  %[[X0]], %[[X1:xmm[0-9]+]]
+; CHECK-NEXT:    psrld   $1, %[[X1]]
+; CHECK-NEXT:    pblendw $192, %[[X0]], %[[X1]]
 ; CHECK-NEXT:    pextrb  $8, %[[X1]], 2(%{{.*}})
 ; CHECK-NEXT:    pshufb  %[[SHUFFLE_MASK]], %[[X1]]
 ; CHECK-NEXT:    pmovzxwq %[[X1]], %[[X3:xmm[0-9]+]]
diff --git a/test/CodeGen/X86/win32-eh-states.ll b/test/CodeGen/X86/win32-eh-states.ll
new file mode 100644
index 00000000000..0aae8c4d018
--- /dev/null
+++ b/test/CodeGen/X86/win32-eh-states.ll
@@ -0,0 +1,114 @@
+; RUN: llc -mtriple=i686-pc-windows-msvc < %s | FileCheck %s
+
+; Based on this source:
+; extern "C" void may_throw(int);
+; void f() {
+;   try {
+;     may_throw(1);
+;     try {
+;       may_throw(2);
+;     } catch (int) {
+;       may_throw(3);
+;     }
+;   } catch (int) {
+;     may_throw(4);
+;   }
+; }
+
+%rtti.TypeDescriptor2 = type { i8**, i8*, [3 x i8] }
+%eh.CatchHandlerType = type { i32, i8* }
+
+declare void @may_throw(i32)
+declare i32 @__CxxFrameHandler3(...)
+declare void @llvm.eh.begincatch(i8*, i8*)
+declare void @llvm.eh.endcatch()
+declare i32 @llvm.eh.typeid.for(i8*)
+
+$"\01??_R0H@8" = comdat any
+
+@"\01??_7type_info@@6B@" = external constant i8*
+@"\01??_R0H@8" = linkonce_odr global %rtti.TypeDescriptor2 { i8** @"\01??_7type_info@@6B@", i8* null, [3 x i8] c".H\00" }, comdat
+@llvm.eh.handlertype.H.0 = private unnamed_addr constant %eh.CatchHandlerType { i32 0, i8* bitcast (%rtti.TypeDescriptor2* @"\01??_R0H@8" to i8*) }, section "llvm.metadata"
+
+define void @f() #0 personality i8* bitcast (i32 (...)* @__CxxFrameHandler3 to i8*) {
+entry:
+  invoke void @may_throw(i32 1)
+          to label %invoke.cont unwind label %lpad
+
+invoke.cont:                                      ; preds = %entry
+  invoke void @may_throw(i32 2)
+          to label %try.cont.9 unwind label %lpad.1
+
+try.cont.9:                                       ; preds = %invoke.cont.3, %invoke.cont, %catch.7
+  ; FIXME: Something about our CFG breaks TailDuplication. This empy asm blocks
+  ; it so we can focus on testing the state numbering.
+  call void asm sideeffect "", "~{dirflag},~{fpsr},~{flags}"()
+  ret void
+
+lpad:                                             ; preds = %catch, %entry
+  %0 = landingpad { i8*, i32 }
+          catch %eh.CatchHandlerType* @llvm.eh.handlertype.H.0
+  %1 = extractvalue { i8*, i32 } %0, 0
+  %2 = extractvalue { i8*, i32 } %0, 1
+  br label %catch.dispatch.4
+
+lpad.1:                                           ; preds = %invoke.cont
+  %3 = landingpad { i8*, i32 }
+          catch i8* bitcast (%eh.CatchHandlerType* @llvm.eh.handlertype.H.0 to i8*)
+  %4 = extractvalue { i8*, i32 } %3, 0
+  %5 = extractvalue { i8*, i32 } %3, 1
+  %6 = tail call i32 @llvm.eh.typeid.for(i8* bitcast (%eh.CatchHandlerType* @llvm.eh.handlertype.H.0 to i8*)) #3
+  %matches = icmp eq i32 %5, %6
+  br i1 %matches, label %catch, label %catch.dispatch.4
+
+catch.dispatch.4:                                 ; preds = %lpad.1, %lpad
+  %exn.slot.0 = phi i8* [ %4, %lpad.1 ], [ %1, %lpad ]
+  %ehselector.slot.0 = phi i32 [ %5, %lpad.1 ], [ %2, %lpad ]
+  %.pre = tail call i32 @llvm.eh.typeid.for(i8* bitcast (%eh.CatchHandlerType* @llvm.eh.handlertype.H.0 to i8*)) #3
+  %matches6 = icmp eq i32 %ehselector.slot.0, %.pre
+  br i1 %matches6, label %catch.7, label %eh.resume
+
+catch.7:                                          ; preds = %catch.dispatch.4
+  tail call void @llvm.eh.begincatch(i8* %exn.slot.0, i8* null) #3
+  tail call void @may_throw(i32 4)
+  tail call void @llvm.eh.endcatch() #3
+  br label %try.cont.9
+
+catch:                                            ; preds = %lpad.1
+  tail call void @llvm.eh.begincatch(i8* %4, i8* null) #3
+  invoke void @may_throw(i32 3)
+          to label %invoke.cont.3 unwind label %lpad
+
+invoke.cont.3:                                    ; preds = %catch
+  tail call void @llvm.eh.endcatch() #3
+  br label %try.cont.9
+
+eh.resume:                                        ; preds = %catch.dispatch.4
+  %lpad.val = insertvalue { i8*, i32 } undef, i8* %exn.slot.0, 0
+  %lpad.val.12 = insertvalue { i8*, i32 } %lpad.val, i32 %ehselector.slot.0, 1
+  resume { i8*, i32 } %lpad.val.12
+}
+
+; CHECK-LABEL: _f:
+; CHECK: movl $-1, [[state:[-0-9]+]](%ebp)
+; CHECK: movl $___ehhandler$f, {{.*}}
+;
+; CHECK: movl $0, [[state]](%ebp)
+; CHECK: movl $1, (%esp)
+; CHECK: calll _may_throw
+;
+; CHECK: movl $1, [[state]](%ebp)
+; CHECK: movl $2, (%esp)
+; CHECK: calll _may_throw
+
+; CHECK-LABEL: _f.catch:
+; CHECK: movl $4, Lf$frame_escape_{{[0-9]+.*}}
+; CHECK: movl $4, (%esp)
+; CHECK: calll _may_throw
+
+; CHECK-LABEL: _f.catch.1:
+; CHECK: movl $3, Lf$frame_escape_{{[0-9]+.*}}
+; CHECK: movl $3, (%esp)
+; CHECK: calll _may_throw
+
+; CHECK: .safeseh ___ehhandler$f
diff --git a/test/CodeGen/X86/win32-eh.ll b/test/CodeGen/X86/win32-eh.ll
new file mode 100644
index 00000000000..3ee4723ce5f
--- /dev/null
+++ b/test/CodeGen/X86/win32-eh.ll
@@ -0,0 +1,153 @@
+; RUN: llc -mtriple=i686-pc-windows-msvc < %s | FileCheck %s
+
+declare void @may_throw_or_crash()
+declare i32 @_except_handler3(...)
+declare i32 @_except_handler4(...)
+declare i32 @__CxxFrameHandler3(...)
+declare void @llvm.eh.begincatch(i8*, i8*)
+declare void @llvm.eh.endcatch()
+declare i32 @llvm.eh.typeid.for(i8*)
+
+define internal i32 @catchall_filt() {
+  ret i32 1
+}
+
+define void @use_except_handler3() personality i32 (...)* @_except_handler3 {
+entry:
+  invoke void @may_throw_or_crash()
+      to label %cont unwind label %catchall
+cont:
+  ret void
+catchall:
+  %0 = landingpad { i8*, i32 }
+      catch i8* bitcast (i32 ()* @catchall_filt to i8*)
+  %1 = extractvalue { i8*, i32 } %0, 1
+  %2 = call i32 @llvm.eh.typeid.for(i8* bitcast (i32 ()* @catchall_filt to i8*)) #4
+  %matches = icmp eq i32 %1, %2
+  br i1 %matches, label %cont, label %eh.resume
+eh.resume:
+  resume { i8*, i32 } %0
+}
+
+; CHECK-LABEL: _use_except_handler3:
+; CHECK: pushl %ebp
+; CHECK: movl %esp, %ebp
+; CHECK: pushl %ebx
+; CHECK: pushl %edi
+; CHECK: pushl %esi
+; CHECK: subl ${{[0-9]+}}, %esp
+; CHECK: movl $-1, -16(%ebp)
+; CHECK: movl $L__ehtable$use_except_handler3, -20(%ebp)
+; CHECK: leal -28(%ebp), %[[node:[^ ,]*]]
+; CHECK: movl $__except_handler3, -24(%ebp)
+; CHECK: movl %fs:0, %[[next:[^ ,]*]]
+; CHECK: movl %[[next]], -28(%ebp)
+; CHECK: movl %[[node]], %fs:0
+; CHECK: calll _may_throw_or_crash
+; CHECK: movl -28(%ebp), %[[next:[^ ,]*]]
+; CHECK: movl %[[next]], %fs:0
+; CHECK: retl
+
+; CHECK: .section .xdata,"dr"
+; CHECK-LABEL: L__ehtable$use_except_handler3:
+; CHECK-NEXT:  .long   -1
+; CHECK-NEXT:  .long   _catchall_filt
+; CHECK-NEXT:  .long   Ltmp{{[0-9]+}}
+
+define void @use_except_handler4() personality i32 (...)* @_except_handler4 {
+entry:
+  invoke void @may_throw_or_crash()
+      to label %cont unwind label %catchall
+cont:
+  ret void
+catchall:
+  %0 = landingpad { i8*, i32 }
+      catch i8* bitcast (i32 ()* @catchall_filt to i8*)
+  %1 = extractvalue { i8*, i32 } %0, 1
+  %2 = call i32 @llvm.eh.typeid.for(i8* bitcast (i32 ()* @catchall_filt to i8*)) #4
+  %matches = icmp eq i32 %1, %2
+  br i1 %matches, label %cont, label %eh.resume
+eh.resume:
+  resume { i8*, i32 } %0
+}
+
+; CHECK-LABEL: _use_except_handler4:
+; CHECK: pushl %ebp
+; CHECK: movl %esp, %ebp
+; CHECK: subl ${{[0-9]+}}, %esp
+; CHECK: movl %esp, -36(%ebp)
+; CHECK: movl $-2, -16(%ebp)
+; CHECK: movl $L__ehtable$use_except_handler4, %[[lsda:[^ ,]*]]
+; CHECK: xorl ___security_cookie, %[[lsda]]
+; CHECK: movl %[[lsda]], -20(%ebp)
+; CHECK: leal -28(%ebp), %[[node:[^ ,]*]]
+; CHECK: movl $__except_handler4, -24(%ebp)
+; CHECK: movl %fs:0, %[[next:[^ ,]*]]
+; CHECK: movl %[[next]], -28(%ebp)
+; CHECK: movl %[[node]], %fs:0
+; CHECK: calll _may_throw_or_crash
+; CHECK: movl -28(%ebp), %[[next:[^ ,]*]]
+; CHECK: movl %[[next]], %fs:0
+; CHECK: retl
+
+; CHECK: .section .xdata,"dr"
+; CHECK-LABEL: L__ehtable$use_except_handler4:
+; CHECK-NEXT:  .long   -2
+; CHECK-NEXT:  .long   0
+; CHECK-NEXT:  .long   9999
+; CHECK-NEXT:  .long   0
+; CHECK-NEXT:  .long   -2
+; CHECK-NEXT:  .long   _catchall_filt
+; CHECK-NEXT:  .long   Ltmp{{[0-9]+}}
+
+define void @use_CxxFrameHandler3() personality i32 (...)* @__CxxFrameHandler3 {
+  invoke void @may_throw_or_crash()
+      to label %cont unwind label %catchall
+cont:
+  ret void
+catchall:
+  %ehvals = landingpad { i8*, i32 }
+      catch i8* null
+  %ehptr = extractvalue { i8*, i32 } %ehvals, 0
+  call void @llvm.eh.begincatch(i8* %ehptr, i8* null)
+  call void @llvm.eh.endcatch()
+  br label %cont
+}
+
+; CHECK-LABEL: _use_CxxFrameHandler3:
+; CHECK: pushl %ebp
+; CHECK: movl %esp, %ebp
+; CHECK: subl ${{[0-9]+}}, %esp
+; CHECK: movl %esp, -28(%ebp)
+; CHECK: movl $-1, -16(%ebp)
+; CHECK: leal -24(%ebp), %[[node:[^ ,]*]]
+; CHECK: movl $___ehhandler$use_CxxFrameHandler3, -20(%ebp)
+; CHECK: movl %fs:0, %[[next:[^ ,]*]]
+; CHECK: movl %[[next]], -24(%ebp)
+; CHECK: movl %[[node]], %fs:0
+; CHECK: movl $0, -16(%ebp)
+; CHECK: calll _may_throw_or_crash
+; CHECK: movl -24(%ebp), %[[next:[^ ,]*]]
+; CHECK: movl %[[next]], %fs:0
+; CHECK: retl
+
+; CHECK: .section .xdata,"dr"
+; CHECK: .align 4
+; CHECK-LABEL: L__ehtable$use_CxxFrameHandler3:
+; CHECK-NEXT:  .long   429065506
+; CHECK-NEXT:  .long   2
+; CHECK-NEXT:  .long   ($stateUnwindMap$use_CxxFrameHandler3)
+; CHECK-NEXT:  .long   1
+; CHECK-NEXT:  .long   ($tryMap$use_CxxFrameHandler3)
+; CHECK-NEXT:  .long   0
+; CHECK-NEXT:  .long   0
+; CHECK-NEXT:  .long   0
+; CHECK-NEXT:  .long   1
+
+; CHECK-LABEL: ___ehhandler$use_CxxFrameHandler3:
+; CHECK: movl $L__ehtable$use_CxxFrameHandler3, %eax
+; CHECK: jmp  ___CxxFrameHandler3 # TAILCALL
+
+; CHECK: .safeseh __except_handler3
+; CHECK: .safeseh __except_handler4
+; CHECK: .safeseh ___ehhandler$use_CxxFrameHandler3
diff --git a/test/CodeGen/X86/win32_sret.ll b/test/CodeGen/X86/win32_sret.ll
index ca01d3b4331..b38273ad959 100644
--- a/test/CodeGen/X86/win32_sret.ll
+++ b/test/CodeGen/X86/win32_sret.ll
@@ -155,25 +155,25 @@ define void @test6_f(%struct.test6* %x) nounwind {
 ; LINUX-LABEL: test6_f:
 
 ; The %x argument is moved to %ecx. It will be the this pointer.
-; WIN32: movl    8(%ebp), %ecx
+; WIN32: movl    20(%esp), %ecx
 
 ; The %x argument is moved to (%esp). It will be the this pointer. With -O0
 ; we copy esp to ecx and use (ecx) instead of (esp).
-; MINGW_X86: movl    8(%ebp), %eax
+; MINGW_X86: movl    20(%esp), %eax
 ; MINGW_X86: movl    %eax, (%e{{([a-d]x)|(sp)}})
 
-; CYGWIN: movl    8(%ebp), %eax
+; CYGWIN: movl    20(%esp), %eax
 ; CYGWIN: movl    %eax, (%e{{([a-d]x)|(sp)}})
 
 ; The sret pointer is (%esp)
-; WIN32:          leal    8(%esp), %[[REG:e[a-d]x]]
+; WIN32:          leal    4(%esp), %[[REG:e[a-d]x]]
 ; WIN32-NEXT:     movl    %[[REG]], (%e{{([a-d]x)|(sp)}})
 
 ; The sret pointer is %ecx
-; MINGW_X86-NEXT: leal    8(%esp), %ecx
+; MINGW_X86-NEXT: leal    4(%esp), %ecx
 ; MINGW_X86-NEXT: calll   _test6_g
 
-; CYGWIN-NEXT: leal    8(%esp), %ecx
+; CYGWIN-NEXT: leal    4(%esp), %ecx
 ; CYGWIN-NEXT: calll   _test6_g
 
   %tmp = alloca %struct.test6, align 4
@@ -191,16 +191,16 @@ define void @test7_f(%struct.test7* %x) nounwind {
 ; LINUX-LABEL: test7_f:
 
 ; The %x argument is moved to %ecx on all OSs. It will be the this pointer.
-; WIN32:      movl    8(%ebp), %ecx
-; MINGW_X86:  movl    8(%ebp), %ecx
-; CYGWIN:     movl    8(%ebp), %ecx
+; WIN32:      movl    20(%esp), %ecx
+; MINGW_X86:  movl    20(%esp), %ecx
+; CYGWIN:     movl    20(%esp), %ecx
 
 ; The sret pointer is (%esp)
-; WIN32:          leal    8(%esp), %[[REG:e[a-d]x]]
+; WIN32:          leal    4(%esp), %[[REG:e[a-d]x]]
 ; WIN32-NEXT:     movl    %[[REG]], (%e{{([a-d]x)|(sp)}})
-; MINGW_X86:      leal    8(%esp), %[[REG:e[a-d]x]]
+; MINGW_X86:      leal    4(%esp), %[[REG:e[a-d]x]]
 ; MINGW_X86-NEXT: movl    %[[REG]], (%e{{([a-d]x)|(sp)}})
-; CYGWIN:         leal    8(%esp), %[[REG:e[a-d]x]]
+; CYGWIN:         leal    4(%esp), %[[REG:e[a-d]x]]
 ; CYGWIN-NEXT:    movl    %[[REG]], (%e{{([a-d]x)|(sp)}})
 
   %tmp = alloca %struct.test7, align 4
diff --git a/test/CodeGen/X86/win64_call_epi.ll b/test/CodeGen/X86/win64_call_epi.ll
index 71c44b08500..096cbe41c54 100644
--- a/test/CodeGen/X86/win64_call_epi.ll
+++ b/test/CodeGen/X86/win64_call_epi.ll
@@ -5,7 +5,7 @@ declare void @baz()
 declare i32 @personality(...)
 
 ; Check for 'nop' between the last call and the epilogue.
-define void @foo1() {
+define void @foo1() personality i32 (...)* @personality {
 
     invoke void @bar()
         to label %normal
@@ -15,7 +15,7 @@ normal:
     ret void
 
 catch:
-    %1 = landingpad { i8*, i32 } personality i32 (...)* @personality cleanup
+    %1 = landingpad { i8*, i32 } cleanup
     resume { i8*, i32 } %1
 }
 ; WIN64-LABEL: foo1:
diff --git a/test/CodeGen/X86/win64_eh.ll b/test/CodeGen/X86/win64_eh.ll
index d668f43c895..cb9d026bec2 100644
--- a/test/CodeGen/X86/win64_eh.ll
+++ b/test/CodeGen/X86/win64_eh.ll
@@ -101,7 +101,7 @@ declare void @_d_eh_resume_unwind(i8*)
 
 declare i32 @bar()
 
-define i32 @foo4() #0 {
+define i32 @foo4() #0 personality i32 (i32, i32, i64, i8*, i8*)* @_d_eh_personality {
 entry:
   %step = alloca i32, align 4
   store i32 0, i32* %step
@@ -115,7 +115,7 @@ finally:
   br label %endtryfinally
 
 landingpad:
-  %landing_pad = landingpad { i8*, i32 } personality i32 (i32, i32, i64, i8*, i8*)* @_d_eh_personality
+  %landing_pad = landingpad { i8*, i32 }
           cleanup
   %tmp3 = extractvalue { i8*, i32 } %landing_pad, 0
   store i32 2, i32* %step
diff --git a/test/CodeGen/X86/win64_frame.ll b/test/CodeGen/X86/win64_frame.ll
index 2c62f4918a7..477b3144d9e 100644
--- a/test/CodeGen/X86/win64_frame.ll
+++ b/test/CodeGen/X86/win64_frame.ll
@@ -100,8 +100,9 @@ define i32 @f8(i32 %a, i32 %b, i32 %c, i32 %d, i32 %e) "no-frame-pointer-elim"="
 
   alloca i32, i32 %a
   ; CHECK:        movl    %ecx, %eax
-  ; CHECK:        leaq    15(,%rax,4), %rax
-  ; CHECK:        andq    $-16, %rax
+  ; CHECK:        leaq    15(,%rax,4), %rcx
+  ; CHECK:        movabsq $34359738352, %rax
+  ; CHECK:        andq    %rcx, %rax
   ; CHECK:        callq   __chkstk
   ; CHECK:        subq    %rax, %rsp
 
diff --git a/test/CodeGen/X86/win64_params.ll b/test/CodeGen/X86/win64_params.ll
index 9718c86300c..a0b552d4d58 100644
--- a/test/CodeGen/X86/win64_params.ll
+++ b/test/CodeGen/X86/win64_params.ll
@@ -7,8 +7,7 @@ define i32 @f6(i32 %p1, i32 %p2, i32 %p3, i32 %p4, i32 %p5, i32 %p6) nounwind re
 entry:
 ; CHECK: movl    48(%rsp), %eax
 ; CHECK: addl    40(%rsp), %eax
-; LINUX: addl    %r9d, %r8d
-; LINUX: movl    %r8d, %eax
+; LINUX: leal    (%r8,%r9), %eax
   %add = add nsw i32 %p6, %p5
   ret i32 %add
 }
@@ -27,10 +26,8 @@ entry:
 ; on other platforms here (note the x86_64_sysvcc calling convention).
 define x86_64_sysvcc i32 @f8(i32 %p1, i32 %p2, i32 %p3, i32 %p4, i32 %p5, i32 %p6) nounwind readnone optsize {
 entry:
-; CHECK: addl    %r9d, %r8d
-; CHECK: movl    %r8d, %eax
-; LINUX: addl    %r9d, %r8d
-; LINUX: movl    %r8d, %eax
+; CHECK: leal    (%r8,%r9), %eax
+; LINUX: leal    (%r8,%r9), %eax
   %add = add nsw i32 %p6, %p5
   ret i32 %add
 }
diff --git a/test/CodeGen/X86/win_cst_pool.ll b/test/CodeGen/X86/win_cst_pool.ll
index 199557dac20..77c37b4d348 100644
--- a/test/CodeGen/X86/win_cst_pool.ll
+++ b/test/CodeGen/X86/win_cst_pool.ll
@@ -64,3 +64,16 @@ define <4 x float> @undef1() {
 ; CHECK:               movaps  __xmm@00000000000000003f8000003f800000(%rip), %xmm0
 ; CHECK-NEXT:          ret
 }
+
+define float @pr23966(i32 %a) {
+  %tobool = icmp ne i32 %a, 0
+  %sel = select i1 %tobool, float -1.000000e+00, float 1.000000e+00
+  ret float %sel
+}
+
+; CHECK:              .globl  __real@bf8000003f800000
+; CHECK-NEXT:         .section        .rdata,"dr",discard,__real@bf8000003f800000
+; CHECK-NEXT:         .align  4
+; CHECK-NEXT: __real@bf8000003f800000:
+; CHECK-NEXT:         .long   1065353216
+; CHECK-NEXT:         .long   3212836864
diff --git a/test/CodeGen/X86/win_eh_prepare.ll b/test/CodeGen/X86/win_eh_prepare.ll
index e5a7d055a78..3e3f9af0582 100644
--- a/test/CodeGen/X86/win_eh_prepare.ll
+++ b/test/CodeGen/X86/win_eh_prepare.ll
@@ -11,7 +11,7 @@ declare i32 @__C_specific_handler(...)
 declare i32 @__gxx_personality_seh0(...)
 declare i32 @llvm.eh.typeid.for(i8*) readnone nounwind
 
-define i32 @use_seh() {
+define i32 @use_seh() personality i32 (...)* @__C_specific_handler {
 entry:
   invoke void @maybe_throw()
       to label %cont unwind label %lpad
@@ -20,7 +20,7 @@ cont:
   ret i32 0
 
 lpad:
-  %ehvals = landingpad { i8*, i32 } personality i32 (...)* @__C_specific_handler
+  %ehvals = landingpad { i8*, i32 }
       cleanup
       catch i8* bitcast (i32 (i8*, i8*)* @filt_g to i8*)
   %ehsel = extractvalue { i8*, i32 } %ehvals, 1
@@ -43,13 +43,15 @@ define internal i32 @filt_g(i8*, i8*) {
 ; CHECK-LABEL: define i32 @use_seh()
 ; CHECK: invoke void @maybe_throw()
 ; CHECK-NEXT: to label %cont unwind label %lpad
-; CHECK: eh.resume:
-; CHECK-NEXT: unreachable
+; CHECK: landingpad
+; CHECK-NEXT: cleanup
+; CHECK-NEXT: catch
+; CHECK-NEXT: call i8* (...) @llvm.eh.actions({{.*}})
 
 
 ; A MinGW64-ish EH style. It could happen if a binary uses both MSVC CRT and
 ; mingw CRT and is linked with LTO.
-define i32 @use_gcc() {
+define i32 @use_gcc() personality i32 (...)* @__gxx_personality_seh0 {
 entry:
   invoke void @maybe_throw()
       to label %cont unwind label %lpad
@@ -58,7 +60,7 @@ cont:
   ret i32 0
 
 lpad:
-  %ehvals = landingpad { i8*, i32 } personality i32 (...)* @__gxx_personality_seh0
+  %ehvals = landingpad { i8*, i32 }
       cleanup
       catch i8* bitcast (i8** @_ZTIi to i8*)
   %ehsel = extractvalue { i8*, i32 } %ehvals, 1
diff --git a/test/CodeGen/X86/win_ftol2.ll b/test/CodeGen/X86/win_ftol2.ll
index 14591248f35..dfa6e3aa76b 100644
--- a/test/CodeGen/X86/win_ftol2.ll
+++ b/test/CodeGen/X86/win_ftol2.ll
@@ -142,3 +142,25 @@ define i64 @double_ui64_5(double %X) {
   %tmp.1 = fptoui double %X to i64
   ret i64 %tmp.1
 }
+
+define double @pr23957_32(double %A) {
+; FTOL-LABEL: @pr23957_32
+; FTOL: fldl
+; FTOL-NEXT: fld %st(0)
+; FTOL-NEXT: calll __ftol2
+  %B = fptoui double %A to i32
+  %C = uitofp i32 %B to double
+  %D = fsub double %C, %A
+  ret double %D
+}
+
+define double @pr23957_64(double %A) {
+; FTOL-LABEL: @pr23957_64
+; FTOL: fldl
+; FTOL-NEXT: fld %st(0)
+; FTOL-NEXT: calll __ftol2
+  %B = fptoui double %A to i64
+  %C = uitofp i64 %B to double
+  %D = fsub double %C, %A
+  ret double %D
+}
diff --git a/test/CodeGen/X86/x32-function_pointer-3.ll b/test/CodeGen/X86/x32-function_pointer-3.ll
index 5eaf85d8f93..f5687b8a9de 100644
--- a/test/CodeGen/X86/x32-function_pointer-3.ll
+++ b/test/CodeGen/X86/x32-function_pointer-3.ll
@@ -3,7 +3,7 @@
 
 ; Test calling function pointer passed in struct
 
-;    The fuction argument `h' in
+;    The function argument `h' in
 
 ;    struct foo {
 ;      void (*f) (void);
diff --git a/test/CodeGen/X86/x86-32-vector-calling-conv.ll b/test/CodeGen/X86/x86-32-vector-calling-conv.ll
new file mode 100644
index 00000000000..b2bda7ab8d0
--- /dev/null
+++ b/test/CodeGen/X86/x86-32-vector-calling-conv.ll
@@ -0,0 +1,44 @@
+; RUN: llc < %s -mtriple=i686-apple-darwin -mattr=+avx512f | FileCheck %s --check-prefix=DARWIN
+; RUN: llc < %s -mtriple=i686-pc-linux -mattr=+avx512f | FileCheck %s --check-prefix=LINUX
+
+; CHECK-LABEL: test_sse:
+; DARWIN-DAG: vpaddd  %xmm1, %xmm0, %xmm0
+; DARWIN-DAG: vpaddd  %xmm3, %xmm2, %xmm1
+; DARWIN: vpaddd  %xmm1, %xmm0, %xmm0
+; LINUX-DAG:  vpaddd  %xmm1, %xmm0, %xmm0
+; LINUX-DAG:  vpaddd  {{[0-9]+}}(%e{{s|b}}p), %xmm2, %xmm1
+; LINUX:  vpaddd  %xmm1, %xmm0, %xmm0
+define <4 x i32> @test_sse(<4 x i32> %a, <4 x i32> %b, <4 x i32> %c, <4 x i32> %d) nounwind {
+  %r0 = add <4 x i32> %a, %b
+  %r1 = add <4 x i32> %c, %d
+  %ret = add <4 x i32> %r0, %r1
+  ret <4 x i32> %ret
+}
+
+; CHECK-LABEL: test_avx:
+; DARWIN-DAG: vpaddd  %ymm1, %ymm0, %ymm0
+; DARWIN-DAG: vpaddd  %ymm3, %ymm2, %ymm1
+; DARWIN: vpaddd  %ymm1, %ymm0, %ymm0
+; LINUX-DAG:  vpaddd  %ymm1, %ymm0, %ymm0
+; LINUX-DAG:  vpaddd  {{[0-9]+}}(%e{{s|b}}p), %ymm2, %ymm1
+; LINUX:  vpaddd  %ymm1, %ymm0, %ymm0
+define <8 x i32> @test_avx(<8 x i32> %a, <8 x i32> %b, <8 x i32> %c, <8 x i32> %d) nounwind {
+  %r0 = add <8 x i32> %a, %b
+  %r1 = add <8 x i32> %c, %d
+  %ret = add <8 x i32> %r0, %r1
+  ret <8 x i32> %ret
+}
+
+; CHECK-LABEL: test_avx512:
+; DARWIN-DAG: vpaddd  %zmm1, %zmm0, %zmm0
+; DARWIN-DAG: vpaddd  %zmm3, %zmm2, %zmm1
+; DARWIN: vpaddd  %zmm1, %zmm0, %zmm0
+; LINUX-DAG:  vpaddd  %zmm1, %zmm0, %zmm0
+; LINUX-DAG:  vpaddd  {{[0-9]+}}(%e{{s|b}}p), %zmm2, %zmm1
+; LINUX:  vpaddd  %zmm1, %zmm0, %zmm0
+define <16 x i32> @test_avx512(<16 x i32> %a, <16 x i32> %b, <16 x i32> %c, <16 x i32> %d) nounwind {
+  %r0 = add <16 x i32> %a, %b
+  %r1 = add <16 x i32> %c, %d
+  %ret = add <16 x i32> %r0, %r1
+  ret <16 x i32> %ret
+}
diff --git a/test/CodeGen/X86/x86-64-stack-and-frame-ptr.ll b/test/CodeGen/X86/x86-64-stack-and-frame-ptr.ll
index f6a7c10945f..c476ffd8405 100644
--- a/test/CodeGen/X86/x86-64-stack-and-frame-ptr.ll
+++ b/test/CodeGen/X86/x86-64-stack-and-frame-ptr.ll
@@ -14,13 +14,11 @@
 ; X32ABI: movl %esp, %ebp
 ; X32ABI: movl %edi, -4(%ebp)
 ; X32ABI: popq %rbp
-; @LOCALMOD for sandbox base address hiding.
 ; NACL-LABEL: foo
-; NACL: movl %ebp, %e[[REG:.*]]
-; NACL: pushq %r[[REG]]
+; NACL: pushq %rbp
 ; NACL: movq %rsp, %rbp
 ; NACL: movl %edi, -4(%rbp)
-; NACL: naclrestbp (%rsp), %r15
+; NACL: popq %rbp
 
 
 define void @foo(i32* %a) #0 {
diff --git a/test/CodeGen/X86/x86-fold-pshufb.ll b/test/CodeGen/X86/x86-fold-pshufb.ll
new file mode 100644
index 00000000000..c29e592bfe8
--- /dev/null
+++ b/test/CodeGen/X86/x86-fold-pshufb.ll
@@ -0,0 +1,17 @@
+; RUN: llc -relocation-model=pic -march=x86-64 -mtriple=x86_64-unknown-unknown -mattr=+ssse3 < %s | FileCheck %s
+
+; Verify that the backend correctly folds the shuffle in function 'fold_pshufb'
+; into a simple load from constant pool.
+
+define <2 x i64> @fold_pshufb() {
+; CHECK-LABEL: fold_pshufb:
+; CHECK:       # BB#0:
+; CHECK-NEXT:    movaps {{.*#+}} xmm0 = [0,0,0,0,1,0,0,0,2,0,0,0,3,0,0,0]
+; CHECK-NEXT:    retq
+entry:
+  %0 = tail call <16 x i8> @llvm.x86.ssse3.pshuf.b.128(<16 x i8> <i8 0, i8 0, i8 0, i8 0, i8 1, i8 0, i8 0, i8 0, i8 2, i8 0, i8 0, i8 0, i8 3, i8 0, i8 0, i8 0>, <16 x i8> <i8 0, i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 8, i8 9, i8 10, i8 11, i8 12, i8 13, i8 14, i8 15>)
+  %1 = bitcast <16 x i8> %0 to <2 x i64>
+  ret <2 x i64> %1
+}
+
+declare <16 x i8> @llvm.x86.ssse3.pshuf.b.128(<16 x i8>, <16 x i8>)
diff --git a/test/CodeGen/X86/x86-framelowering-trap.ll b/test/CodeGen/X86/x86-framelowering-trap.ll
new file mode 100644
index 00000000000..58a1da23a29
--- /dev/null
+++ b/test/CodeGen/X86/x86-framelowering-trap.ll
@@ -0,0 +1,15 @@
+; RUN: llc %s -o - | FileCheck %s
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+; CHECK-LABEL: bar:
+; CHECK: ud2
+; CHECK-NEXT: retq
+define void @bar() {
+entry:
+  call void @llvm.trap()
+  ret void
+}
+
+; Function Attrs: noreturn nounwind
+declare void @llvm.trap()
diff --git a/test/CodeGen/X86/x86-setcc-int-to-fp-combine.ll b/test/CodeGen/X86/x86-setcc-int-to-fp-combine.ll
index a2c5b3a6eed..248a9202e99 100644
--- a/test/CodeGen/X86/x86-setcc-int-to-fp-combine.ll
+++ b/test/CodeGen/X86/x86-setcc-int-to-fp-combine.ll
@@ -27,12 +27,8 @@ define void @foo1(<4 x float> %val, <4 x float> %test, <4 x double>* %p) nounwin
 ; CHECK-NEXT: .long 1                       ## 0x1
 ; CHECK-NEXT: .long 1                       ## 0x1
 ; CHECK-LABEL: foo1:
-;   FIXME: The operation gets scalarized. If/when the compiler learns to better
-;          use [V]CVTDQ2PD, this will need updated.
-; CHECK: cvtsi2sdq
-; CHECK: cvtsi2sdq
-; CHECK: cvtsi2sdq
-; CHECK: cvtsi2sdq
+; CHECK: cvtdq2pd
+; CHECK: cvtdq2pd
   %cmp = fcmp oeq <4 x float> %val, %test
   %ext = zext <4 x i1> %cmp to <4 x i32>
   %result = sitofp <4 x i32> %ext to <4 x double>
diff --git a/test/CodeGen/X86/x86-shrink-wrapping.ll b/test/CodeGen/X86/x86-shrink-wrapping.ll
new file mode 100644
index 00000000000..8c91335d91a
--- /dev/null
+++ b/test/CodeGen/X86/x86-shrink-wrapping.ll
@@ -0,0 +1,639 @@
+; RUN: llc %s -o - -enable-shrink-wrap=true | FileCheck %s --check-prefix=CHECK --check-prefix=ENABLE
+; RUN: llc %s -o - -enable-shrink-wrap=false | FileCheck %s --check-prefix=CHECK --check-prefix=DISABLE
+;
+; Note: Lots of tests use inline asm instead of regular calls.
+; This allows to have a better control on what the allocation will do.
+; Otherwise, we may have spill right in the entry block, defeating
+; shrink-wrapping. Moreover, some of the inline asm statement (nop)
+; are here to ensure that the related paths do not end up as critical
+; edges.
+target datalayout = "e-m:o-i64:64-i128:128-n32:64-S128"
+target triple = "x86_64-apple-macosx"
+
+
+; Initial motivating example: Simple diamond with a call just on one side.
+; CHECK-LABEL: foo:
+;
+; Compare the arguments and jump to exit.
+; No prologue needed.
+; ENABLE: movl %edi, [[ARG0CPY:%e[a-z]+]]
+; ENABLE-NEXT: cmpl %esi, [[ARG0CPY]]
+; ENABLE-NEXT: jge [[EXIT_LABEL:LBB[0-9_]+]]
+;
+; Prologue code.
+; (What we push does not matter. It should be some random sratch register.)
+; CHECK: pushq
+;
+; Compare the arguments and jump to exit.
+; After the prologue is set.
+; DISABLE: movl %edi, [[ARG0CPY:%e[a-z]+]]
+; DISABLE-NEXT: cmpl %esi, [[ARG0CPY]]
+; DISABLE-NEXT: jge [[EXIT_LABEL:LBB[0-9_]+]]
+;
+; Store %a in the alloca.
+; CHECK: movl [[ARG0CPY]], 4(%rsp)
+; Set the alloca address in the second argument.
+; CHECK-NEXT: leaq 4(%rsp), %rsi
+; Set the first argument to zero.
+; CHECK-NEXT: xorl %edi, %edi
+; CHECK-NEXT: callq _doSomething
+;
+; With shrink-wrapping, epilogue is just after the call.
+; ENABLE-NEXT: addq $8, %rsp
+;
+; CHECK: [[EXIT_LABEL]]:
+;
+; Without shrink-wrapping, epilogue is in the exit block.
+; Epilogue code. (What we pop does not matter.)
+; DISABLE-NEXT: popq
+;
+; CHECK-NEXT: retq
+define i32 @foo(i32 %a, i32 %b) {
+  %tmp = alloca i32, align 4
+  %tmp2 = icmp slt i32 %a, %b
+  br i1 %tmp2, label %true, label %false
+
+true:
+  store i32 %a, i32* %tmp, align 4
+  %tmp4 = call i32 @doSomething(i32 0, i32* %tmp)
+  br label %false
+
+false:
+  %tmp.0 = phi i32 [ %tmp4, %true ], [ %a, %0 ]
+  ret i32 %tmp.0
+}
+
+; Function Attrs: optsize
+declare i32 @doSomething(i32, i32*)
+
+
+; Check that we do not perform the restore inside the loop whereas the save
+; is outside.
+; CHECK-LABEL: freqSaveAndRestoreOutsideLoop:
+;
+; Shrink-wrapping allows to skip the prologue in the else case.
+; ENABLE: testl %edi, %edi  
+; ENABLE: je [[ELSE_LABEL:LBB[0-9_]+]]
+;
+; Prologue code.
+; Make sure we save the CSR used in the inline asm: rbx.
+; CHECK: pushq %rbx
+;
+; DISABLE: testl %edi, %edi
+; DISABLE: je [[ELSE_LABEL:LBB[0-9_]+]]
+;
+; SUM is in %esi because it is coalesced with the second
+; argument on the else path.
+; CHECK: xorl [[SUM:%esi]], [[SUM]]
+; CHECK-NEXT: movl $10, [[IV:%e[a-z]+]]
+;
+; Next BB.
+; CHECK: [[LOOP:LBB[0-9_]+]]: ## %for.body
+; CHECK: movl $1, [[TMP:%e[a-z]+]]
+; CHECK: addl [[TMP]], [[SUM]]
+; CHECK-NEXT: decl [[IV]]
+; CHECK-NEXT: jne [[LOOP]]
+;
+; Next BB.
+; SUM << 3.
+; CHECK: shll $3, [[SUM]]
+;
+; Jump to epilogue.
+; DISABLE: jmp [[EPILOG_BB:LBB[0-9_]+]]
+;
+; DISABLE: [[ELSE_LABEL]]: ## %if.else
+; Shift second argument by one and store into returned register.
+; DISABLE: addl %esi, %esi
+; DISABLE: [[EPILOG_BB]]: ## %if.end
+;
+; Epilogue code.
+; CHECK-DAG: popq %rbx
+; CHECK-DAG: movl %esi, %eax
+; CHECK: retq
+;
+; ENABLE: [[ELSE_LABEL]]: ## %if.else
+; Shift second argument by one and store into returned register.
+; ENABLE: addl %esi, %esi
+; ENABLE-NEXT: movl %esi, %eax
+; ENABLE-NEXT: retq
+define i32 @freqSaveAndRestoreOutsideLoop(i32 %cond, i32 %N) {
+entry:
+  %tobool = icmp eq i32 %cond, 0
+  br i1 %tobool, label %if.else, label %for.preheader
+
+for.preheader:
+  tail call void asm "nop", ""()
+  br label %for.body
+
+for.body:                                         ; preds = %entry, %for.body
+  %i.05 = phi i32 [ %inc, %for.body ], [ 0, %for.preheader ]
+  %sum.04 = phi i32 [ %add, %for.body ], [ 0, %for.preheader ]
+  %call = tail call i32 asm "movl $$1, $0", "=r,~{ebx}"()
+  %add = add nsw i32 %call, %sum.04
+  %inc = add nuw nsw i32 %i.05, 1
+  %exitcond = icmp eq i32 %inc, 10
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:                                          ; preds = %for.body
+  %shl = shl i32 %add, 3
+  br label %if.end
+
+if.else:                                          ; preds = %entry
+  %mul = shl nsw i32 %N, 1
+  br label %if.end
+
+if.end:                                           ; preds = %if.else, %for.end
+  %sum.1 = phi i32 [ %shl, %for.end ], [ %mul, %if.else ]
+  ret i32 %sum.1
+}
+
+declare i32 @something(...)
+
+; Check that we do not perform the shrink-wrapping inside the loop even
+; though that would be legal. The cost model must prevent that.
+; CHECK-LABEL: freqSaveAndRestoreOutsideLoop2:
+; Prologue code.
+; Make sure we save the CSR used in the inline asm: rbx.
+; CHECK: pushq %rbx
+; CHECK: nop
+; CHECK: xorl [[SUM:%e[a-z]+]], [[SUM]]
+; CHECK-NEXT: movl $10, [[IV:%e[a-z]+]]
+; Next BB.
+; CHECK: [[LOOP_LABEL:LBB[0-9_]+]]: ## %for.body
+; CHECK: movl $1, [[TMP:%e[a-z]+]]
+; CHECK: addl [[TMP]], [[SUM]]
+; CHECK-NEXT: decl [[IV]]
+; CHECK-NEXT: jne [[LOOP_LABEL]]
+; Next BB.
+; CHECK: ## %for.exit
+; CHECK: nop
+; CHECK: popq %rbx
+; CHECK-NEXT: retq
+define i32 @freqSaveAndRestoreOutsideLoop2(i32 %cond) {
+entry:
+  br label %for.preheader
+
+for.preheader:
+  tail call void asm "nop", ""()
+  br label %for.body
+
+for.body:                                         ; preds = %for.body, %entry
+  %i.04 = phi i32 [ 0, %for.preheader ], [ %inc, %for.body ]
+  %sum.03 = phi i32 [ 0, %for.preheader ], [ %add, %for.body ]
+  %call = tail call i32 asm "movl $$1, $0", "=r,~{ebx}"()
+  %add = add nsw i32 %call, %sum.03
+  %inc = add nuw nsw i32 %i.04, 1
+  %exitcond = icmp eq i32 %inc, 10
+  br i1 %exitcond, label %for.exit, label %for.body
+
+for.exit:
+  tail call void asm "nop", ""()
+  br label %for.end
+
+for.end:                                          ; preds = %for.body
+  ret i32 %add
+}
+
+; Check with a more complex case that we do not have save within the loop and
+; restore outside.
+; CHECK-LABEL: loopInfoSaveOutsideLoop:
+;
+; ENABLE: testl %edi, %edi
+; ENABLE-NEXT: je [[ELSE_LABEL:LBB[0-9_]+]]
+;
+; Prologue code.
+; Make sure we save the CSR used in the inline asm: rbx.
+; CHECK: pushq %rbx
+;
+; DISABLE: testl %edi, %edi
+; DISABLE-NEXT: je [[ELSE_LABEL:LBB[0-9_]+]]
+;
+; CHECK: nop
+; CHECK: xorl [[SUM:%esi]], [[SUM]]
+; CHECK-NEXT: movl $10, [[IV:%e[a-z]+]]
+;
+; CHECK: [[LOOP_LABEL:LBB[0-9_]+]]: ## %for.body
+; CHECK: movl $1, [[TMP:%e[a-z]+]]
+; CHECK: addl [[TMP]], [[SUM]]
+; CHECK-NEXT: decl [[IV]]
+; CHECK-NEXT: jne [[LOOP_LABEL]]
+; Next BB.
+; CHECK: nop
+; CHECK: shll $3, [[SUM]]
+;
+; DISABLE: jmp [[EPILOG_BB:LBB[0-9_]+]]
+;
+; DISABLE: [[ELSE_LABEL]]: ## %if.else
+; Shift second argument by one and store into returned register.
+; DISABLE: addl %esi, %esi
+; DISABLE: [[EPILOG_BB]]: ## %if.end
+;
+; Epilogue code.
+; CHECK-DAG: popq %rbx
+; CHECK-DAG: movl %esi, %eax
+; CHECK: retq
+;
+; ENABLE: [[ELSE_LABEL]]: ## %if.else
+; Shift second argument by one and store into returned register.
+; ENABLE: addl %esi, %esi
+; ENABLE-NEXT: movl %esi, %eax
+; ENABLE-NEXT: retq
+define i32 @loopInfoSaveOutsideLoop(i32 %cond, i32 %N) {
+entry:
+  %tobool = icmp eq i32 %cond, 0
+  br i1 %tobool, label %if.else, label %for.preheader
+
+for.preheader:
+  tail call void asm "nop", ""()
+  br label %for.body
+
+for.body:                                         ; preds = %entry, %for.body
+  %i.05 = phi i32 [ %inc, %for.body ], [ 0, %for.preheader ]
+  %sum.04 = phi i32 [ %add, %for.body ], [ 0, %for.preheader ]
+  %call = tail call i32 asm "movl $$1, $0", "=r,~{ebx}"()
+  %add = add nsw i32 %call, %sum.04
+  %inc = add nuw nsw i32 %i.05, 1
+  %exitcond = icmp eq i32 %inc, 10
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:                                          ; preds = %for.body
+  tail call void asm "nop", "~{ebx}"()
+  %shl = shl i32 %add, 3
+  br label %if.end
+
+if.else:                                          ; preds = %entry
+  %mul = shl nsw i32 %N, 1
+  br label %if.end
+
+if.end:                                           ; preds = %if.else, %for.end
+  %sum.1 = phi i32 [ %shl, %for.end ], [ %mul, %if.else ]
+  ret i32 %sum.1
+}
+
+declare void @somethingElse(...)
+
+; Check with a more complex case that we do not have restore within the loop and
+; save outside.
+; CHECK-LABEL: loopInfoRestoreOutsideLoop:
+;
+; ENABLE: testl %edi, %edi
+; ENABLE-NEXT: je [[ELSE_LABEL:LBB[0-9_]+]]
+;
+; Prologue code.
+; Make sure we save the CSR used in the inline asm: rbx.
+; CHECK: pushq %rbx
+;
+; DISABLE: testl %edi, %edi
+; DISABLE-NEXT: je [[ELSE_LABEL:LBB[0-9_]+]]
+;
+; CHECK: nop
+; CHECK: xorl [[SUM:%esi]], [[SUM]]
+; CHECK-NEXT: movl $10, [[IV:%e[a-z]+]]
+;
+; CHECK: [[LOOP_LABEL:LBB[0-9_]+]]: ## %for.body
+; CHECK: movl $1, [[TMP:%e[a-z]+]]
+; CHECK: addl [[TMP]], [[SUM]]
+; CHECK-NEXT: decl [[IV]]
+; CHECK-NEXT: jne [[LOOP_LABEL]]
+; Next BB.
+; CHECK: shll $3, [[SUM]]
+;
+; DISABLE: jmp [[EPILOG_BB:LBB[0-9_]+]]
+;
+; DISABLE: [[ELSE_LABEL]]: ## %if.else
+
+; Shift second argument by one and store into returned register.
+; DISABLE: addl %esi, %esi
+; DISABLE: [[EPILOG_BB]]: ## %if.end
+;
+; Epilogue code.
+; CHECK-DAG: popq %rbx
+; CHECK-DAG: movl %esi, %eax
+; CHECK: retq
+;
+; ENABLE: [[ELSE_LABEL]]: ## %if.else
+; Shift second argument by one and store into returned register.
+; ENABLE: addl %esi, %esi
+; ENABLE-NEXT: movl %esi, %eax
+; ENABLE-NEXT: retq
+define i32 @loopInfoRestoreOutsideLoop(i32 %cond, i32 %N) #0 {
+entry:
+  %tobool = icmp eq i32 %cond, 0
+  br i1 %tobool, label %if.else, label %if.then
+
+if.then:                                          ; preds = %entry
+  tail call void asm "nop", "~{ebx}"()
+  br label %for.body
+
+for.body:                                         ; preds = %for.body, %if.then
+  %i.05 = phi i32 [ 0, %if.then ], [ %inc, %for.body ]
+  %sum.04 = phi i32 [ 0, %if.then ], [ %add, %for.body ]
+  %call = tail call i32 asm "movl $$1, $0", "=r,~{ebx}"()
+  %add = add nsw i32 %call, %sum.04
+  %inc = add nuw nsw i32 %i.05, 1
+  %exitcond = icmp eq i32 %inc, 10
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:                                          ; preds = %for.body
+  %shl = shl i32 %add, 3
+  br label %if.end
+
+if.else:                                          ; preds = %entry
+  %mul = shl nsw i32 %N, 1
+  br label %if.end
+
+if.end:                                           ; preds = %if.else, %for.end
+  %sum.1 = phi i32 [ %shl, %for.end ], [ %mul, %if.else ]
+  ret i32 %sum.1
+}
+
+; Check that we handle function with no frame information correctly.
+; CHECK-LABEL: emptyFrame:
+; CHECK: ## %entry
+; CHECK-NEXT: xorl %eax, %eax
+; CHECK-NEXT: retq
+define i32 @emptyFrame() {
+entry:
+  ret i32 0
+}
+
+; Check that we handle inline asm correctly.
+; CHECK-LABEL: inlineAsm:
+;
+; ENABLE: testl %edi, %edi
+; ENABLE-NEXT: je [[ELSE_LABEL:LBB[0-9_]+]]
+;
+; Prologue code.
+; Make sure we save the CSR used in the inline asm: rbx.
+; CHECK: pushq %rbx
+;
+; DISABLE: testl %edi, %edi
+; DISABLE-NEXT: je [[ELSE_LABEL:LBB[0-9_]+]]
+;
+; CHECK: nop
+; CHECK: movl $10, [[IV:%e[a-z]+]]
+;
+; CHECK: [[LOOP_LABEL:LBB[0-9_]+]]: ## %for.body
+; Inline asm statement.
+; CHECK: addl $1, %ebx
+; CHECK: decl [[IV]]
+; CHECK-NEXT: jne [[LOOP_LABEL]]
+; Next BB.
+; CHECK: nop
+; CHECK: xorl %esi, %esi
+;
+; DISABLE: jmp [[EPILOG_BB:LBB[0-9_]+]]
+;
+; DISABLE: [[ELSE_LABEL]]: ## %if.else
+; Shift second argument by one and store into returned register.
+; DISABLE: addl %esi, %esi
+; DISABLE: [[EPILOG_BB]]: ## %if.end
+;
+; Epilogue code.
+; CHECK-DAG: popq %rbx
+; CHECK-DAG: movl %esi, %eax
+; CHECK: retq
+;
+; ENABLE: [[ELSE_LABEL]]: ## %if.else
+; Shift second argument by one and store into returned register.
+; ENABLE: addl %esi, %esi
+; ENABLE-NEXT: movl %esi, %eax
+; ENABLE-NEXT: retq
+define i32 @inlineAsm(i32 %cond, i32 %N) {
+entry:
+  %tobool = icmp eq i32 %cond, 0
+  br i1 %tobool, label %if.else, label %for.preheader
+
+for.preheader:
+  tail call void asm "nop", ""()
+  br label %for.body
+
+for.body:                                         ; preds = %entry, %for.body
+  %i.03 = phi i32 [ %inc, %for.body ], [ 0, %for.preheader ]
+  tail call void asm "addl $$1, %ebx", "~{ebx}"()
+  %inc = add nuw nsw i32 %i.03, 1
+  %exitcond = icmp eq i32 %inc, 10
+  br i1 %exitcond, label %for.exit, label %for.body
+
+for.exit:
+  tail call void asm "nop", ""()
+  br label %if.end
+
+if.else:                                          ; preds = %entry
+  %mul = shl nsw i32 %N, 1
+  br label %if.end
+
+if.end:                                           ; preds = %for.body, %if.else
+  %sum.0 = phi i32 [ %mul, %if.else ], [ 0, %for.exit ]
+  ret i32 %sum.0
+}
+
+; Check that we handle calls to variadic functions correctly.
+; CHECK-LABEL: callVariadicFunc:
+;
+; ENABLE: testl %edi, %edi
+; ENABLE-NEXT: je [[ELSE_LABEL:LBB[0-9_]+]]
+;
+; Prologue code.
+; CHECK: pushq
+;
+; DISABLE: testl %edi, %edi
+; DISABLE-NEXT: je [[ELSE_LABEL:LBB[0-9_]+]]
+;
+; Setup of the varags.
+; CHECK: movl %esi, (%rsp)
+; CHECK-NEXT: xorl %eax, %eax
+; CHECK-NEXT: %esi, %edi
+; CHECK-NEXT: %esi, %edx
+; CHECK-NEXT: %esi, %r8d
+; CHECK-NEXT: %esi, %r9d
+; CHECK-NEXT: %esi, %ecx
+; CHECK-NEXT: callq _someVariadicFunc
+; CHECK-NEXT: movl %eax, %esi
+; CHECK-NEXT: shll $3, %esi
+;
+; ENABLE-NEXT: addq $8, %rsp
+; ENABLE-NEXT: movl %esi, %eax
+; ENABLE-NEXT: retq
+;
+; DISABLE: jmp [[IFEND_LABEL:LBB[0-9_]+]]
+;
+; CHECK: [[ELSE_LABEL]]: ## %if.else
+; Shift second argument by one and store into returned register.
+; CHECK: addl %esi, %esi
+;
+; DISABLE: [[IFEND_LABEL]]: ## %if.end
+;
+; Epilogue code.
+; CHECK-NEXT: movl %esi, %eax
+; DISABLE-NEXT: popq
+; CHECK-NEXT: retq
+define i32 @callVariadicFunc(i32 %cond, i32 %N) {
+entry:
+  %tobool = icmp eq i32 %cond, 0
+  br i1 %tobool, label %if.else, label %if.then
+
+if.then:                                          ; preds = %entry
+  %call = tail call i32 (i32, ...) @someVariadicFunc(i32 %N, i32 %N, i32 %N, i32 %N, i32 %N, i32 %N, i32 %N)
+  %shl = shl i32 %call, 3
+  br label %if.end
+
+if.else:                                          ; preds = %entry
+  %mul = shl nsw i32 %N, 1
+  br label %if.end
+
+if.end:                                           ; preds = %if.else, %if.then
+  %sum.0 = phi i32 [ %shl, %if.then ], [ %mul, %if.else ]
+  ret i32 %sum.0
+}
+
+declare i32 @someVariadicFunc(i32, ...)
+
+; Check that we use LEA not to clobber EFLAGS.
+%struct.temp_slot = type { %struct.temp_slot*, %struct.rtx_def*, %struct.rtx_def*, i32, i64, %union.tree_node*, %union.tree_node*, i8, i8, i32, i32, i64, i64 }
+%union.tree_node = type { %struct.tree_decl }
+%struct.tree_decl = type { %struct.tree_common, i8*, i32, i32, %union.tree_node*, i48, %union.anon, %union.tree_node*, %union.tree_node*, %union.tree_node*, %union.tree_node*, %union.tree_node*, %union.tree_node*, %union.tree_node*, %union.tree_node*, %union.tree_node*, %union.tree_node*, %struct.rtx_def*, %struct.rtx_def*, %union.anon.1, %union.tree_node*, %union.tree_node*, %union.tree_node*, i64, %struct.lang_decl* }
+%struct.tree_common = type { %union.tree_node*, %union.tree_node*, i32 }
+%union.anon = type { i64 }
+%union.anon.1 = type { %struct.function* }
+%struct.function = type { %struct.eh_status*, %struct.stmt_status*, %struct.expr_status*, %struct.emit_status*, %struct.varasm_status*, i8*, %union.tree_node*, %struct.function*, i32, i32, i32, i32, %struct.rtx_def*, %struct.ix86_args, %struct.rtx_def*, %struct.rtx_def*, i8*, %struct.initial_value_struct*, i32, %union.tree_node*, %struct.rtx_def*, %struct.rtx_def*, %struct.rtx_def*, %struct.rtx_def*, %struct.rtx_def*, %struct.rtx_def*, %struct.rtx_def*, %union.tree_node*, %struct.rtx_def*, %struct.rtx_def*, %struct.rtx_def*, %struct.rtx_def*, i64, %union.tree_node*, %union.tree_node*, %struct.rtx_def*, %struct.rtx_def*, i32, %struct.rtx_def**, %struct.temp_slot*, i32, i32, i32, %struct.var_refs_queue*, i32, i32, i8*, %union.tree_node*, %struct.rtx_def*, i32, i32, %struct.machine_function*, i32, i32, %struct.language_function*, %struct.rtx_def*, i24 }
+%struct.eh_status = type opaque
+%struct.stmt_status = type opaque
+%struct.expr_status = type { i32, i32, i32, %struct.rtx_def*, %struct.rtx_def*, %struct.rtx_def*, %struct.rtx_def* }
+%struct.emit_status = type { i32, i32, %struct.rtx_def*, %struct.rtx_def*, %union.tree_node*, %struct.sequence_stack*, i32, i32, i8*, i32, i8*, %union.tree_node**, %struct.rtx_def** }
+%struct.sequence_stack = type { %struct.rtx_def*, %struct.rtx_def*, %union.tree_node*, %struct.sequence_stack* }
+%struct.varasm_status = type opaque
+%struct.ix86_args = type { i32, i32, i32, i32, i32, i32, i32 }
+%struct.initial_value_struct = type opaque
+%struct.var_refs_queue = type { %struct.rtx_def*, i32, i32, %struct.var_refs_queue* }
+%struct.machine_function = type opaque
+%struct.language_function = type opaque
+%struct.lang_decl = type opaque
+%struct.rtx_def = type { i32, [1 x %union.rtunion_def] }
+%union.rtunion_def = type { i64 }
+
+declare hidden fastcc %struct.temp_slot* @find_temp_slot_from_address(%struct.rtx_def* readonly)
+
+; CHECK-LABEL: useLEA:
+; DISABLE: pushq 
+;
+; CHECK: testq   %rdi, %rdi
+; CHECK-NEXT: je      [[CLEANUP:LBB[0-9_]+]]
+;
+; CHECK: movzwl  (%rdi), [[BF_LOAD:%e[a-z]+]]
+; CHECK-NEXT: cmpl $66, [[BF_LOAD]]
+; CHECK-NEXT: jne [[CLEANUP]]
+;
+; CHECK: movq 8(%rdi), %rdi
+; CHECK-NEXT: movzwl (%rdi), %e[[BF_LOAD2:[a-z]+]]
+; CHECK-NEXT: leal -54(%r[[BF_LOAD2]]), [[TMP:%e[a-z]+]]
+; CHECK-NEXT: cmpl $14, [[TMP]]
+; CHECK-NEXT: ja [[LOR_LHS_FALSE:LBB[0-9_]+]]
+;
+; CHECK: movl $24599, [[TMP2:%e[a-z]+]]
+; CHECK-NEXT: btl [[TMP]], [[TMP2]]
+; CHECK-NEXT: jb [[CLEANUP]]
+;
+; CHECK: [[LOR_LHS_FALSE]]: ## %lor.lhs.false
+; CHECK: cmpl $134, %e[[BF_LOAD2]]
+; CHECK-NEXT: je [[CLEANUP]]
+;
+; CHECK: cmpl $140, %e[[BF_LOAD2]]
+; CHECK-NEXT: je [[CLEANUP]]
+;
+; ENABLE: pushq
+; CHECK: callq _find_temp_slot_from_address
+; CHECK-NEXT: testq   %rax, %rax
+;
+; The adjustment must use LEA here (or be moved above the test).
+; ENABLE-NEXT: leaq 8(%rsp), %rsp
+;
+; CHECK-NEXT: je [[CLEANUP]]
+;
+; CHECK: movb $1, 57(%rax)
+;
+; CHECK: [[CLEANUP]]: ## %cleanup
+; DISABLE: popq
+; CHECK-NEXT: retq
+define void @useLEA(%struct.rtx_def* readonly %x) {
+entry:
+  %cmp = icmp eq %struct.rtx_def* %x, null
+  br i1 %cmp, label %cleanup, label %if.end
+
+if.end:                                           ; preds = %entry
+  %tmp = getelementptr inbounds %struct.rtx_def, %struct.rtx_def* %x, i64 0, i32 0
+  %bf.load = load i32, i32* %tmp, align 8
+  %bf.clear = and i32 %bf.load, 65535
+  %cmp1 = icmp eq i32 %bf.clear, 66
+  br i1 %cmp1, label %lor.lhs.false, label %cleanup
+
+lor.lhs.false:                                    ; preds = %if.end
+  %arrayidx = getelementptr inbounds %struct.rtx_def, %struct.rtx_def* %x, i64 0, i32 1, i64 0
+  %rtx = bitcast %union.rtunion_def* %arrayidx to %struct.rtx_def**
+  %tmp1 = load %struct.rtx_def*, %struct.rtx_def** %rtx, align 8
+  %tmp2 = getelementptr inbounds %struct.rtx_def, %struct.rtx_def* %tmp1, i64 0, i32 0
+  %bf.load2 = load i32, i32* %tmp2, align 8
+  %bf.clear3 = and i32 %bf.load2, 65535
+  switch i32 %bf.clear3, label %if.end.55 [
+    i32 67, label %cleanup
+    i32 68, label %cleanup
+    i32 54, label %cleanup
+    i32 55, label %cleanup
+    i32 58, label %cleanup
+    i32 134, label %cleanup
+    i32 56, label %cleanup
+    i32 140, label %cleanup
+  ]
+
+if.end.55:                                        ; preds = %lor.lhs.false
+  %call = tail call fastcc %struct.temp_slot* @find_temp_slot_from_address(%struct.rtx_def* %tmp1) #2
+  %cmp59 = icmp eq %struct.temp_slot* %call, null
+  br i1 %cmp59, label %cleanup, label %if.then.60
+
+if.then.60:                                       ; preds = %if.end.55
+  %addr_taken = getelementptr inbounds %struct.temp_slot, %struct.temp_slot* %call, i64 0, i32 8
+  store i8 1, i8* %addr_taken, align 1
+  br label %cleanup
+
+cleanup:                                          ; preds = %if.then.60, %if.end.55, %lor.lhs.false, %lor.lhs.false, %lor.lhs.false, %lor.lhs.false, %lor.lhs.false, %lor.lhs.false, %lor.lhs.false, %lor.lhs.false, %if.end, %entry
+  ret void
+}
+
+; Make sure we do not insert unreachable code after noreturn function.
+; Although this is not incorrect to insert such code, it is useless
+; and it hurts the binary size.
+;
+; CHECK-LABEL: noreturn:
+; DISABLE: pushq
+;
+; CHECK: testb   %dil, %dil
+; CHECK-NEXT: jne      [[ABORT:LBB[0-9_]+]]
+;
+; CHECK: movl $42, %eax
+;
+; DISABLE-NEXT: popq
+;
+; CHECK-NEXT: retq
+;
+; CHECK: [[ABORT]]: ## %if.abort
+;
+; ENABLE: pushq
+;
+; CHECK: callq _abort
+; ENABLE-NOT: popq
+define i32 @noreturn(i8 signext %bad_thing) {
+entry:
+  %tobool = icmp eq i8 %bad_thing, 0
+  br i1 %tobool, label %if.end, label %if.abort
+
+if.abort:
+  tail call void @abort() #0
+  unreachable
+
+if.end:
+  ret i32 42
+}
+
+declare void @abort() #0
+
+attributes #0 = { noreturn nounwind }
diff --git a/test/CodeGen/X86/xor.ll b/test/CodeGen/X86/xor.ll
index 829be41e512..f78fe275786 100644
--- a/test/CodeGen/X86/xor.ll
+++ b/test/CodeGen/X86/xor.ll
@@ -1,4 +1,4 @@
-; RUN: llc < %s -march=x86            -mattr=+sse2 | FileCheck %s -check-prefix=X32
+; RUN: llc < %s -mtriple=i686-unknown -mattr=+sse2 | FileCheck %s -check-prefix=X32
 ; RUN: llc < %s -mtriple=x86_64-linux -mattr=+sse2 | FileCheck %s -check-prefix=X64
 ; RUN: llc < %s -mtriple=x86_64-win32 -mattr=+sse2 | FileCheck %s -check-prefix=X64
 
@@ -193,3 +193,22 @@ define i32 @test11(i32 %b) {
 ; X32: movl    $-2, %[[REG:.*]]
 ; X32: roll    %{{.*}}, %[[REG]]
 }
+
+%struct.ref_s = type { %union.v, i16, i16 }
+%union.v = type { i64 }
+
+define %struct.ref_s* @test12(%struct.ref_s* %op, i64 %osbot, i64 %intval) {
+  %neg = shl i64 %intval, 32
+  %sext = xor i64 %neg, -4294967296
+  %idx.ext = ashr exact i64 %sext, 32
+  %add.ptr = getelementptr inbounds %struct.ref_s, %struct.ref_s* %op, i64 %idx.ext
+  ret %struct.ref_s* %add.ptr
+; X64-LABEL: test12:
+; X64: shlq	$32, %[[REG:.*]]
+; X64-NOT: not
+; X64: sarq	$28, %[[REG]]
+; X32-LABEL: test12:
+; X32: leal
+; X32-NOT: not
+; X32: shll	$2, %eax
+}
diff --git a/test/CodeGen/XCore/dwarf_debug.ll b/test/CodeGen/XCore/dwarf_debug.ll
index 66b642851d6..ba71dc798a0 100644
--- a/test/CodeGen/XCore/dwarf_debug.ll
+++ b/test/CodeGen/XCore/dwarf_debug.ll
@@ -13,7 +13,7 @@ define i32 @f(i32 %a) {
 entry:
   %a.addr = alloca i32, align 4
   store i32 %a, i32* %a.addr, align 4
-  call void @llvm.dbg.declare(metadata i32* %a.addr, metadata !11, metadata !MDExpression()), !dbg !12
+  call void @llvm.dbg.declare(metadata i32* %a.addr, metadata !11, metadata !DIExpression()), !dbg !12
   %0 = load i32, i32* %a.addr, align 4, !dbg !12
   %add = add nsw i32 %0, 1, !dbg !12
   ret i32 %add, !dbg !12
@@ -23,16 +23,16 @@ declare void @llvm.dbg.declare(metadata, metadata, metadata)
 
 !llvm.dbg.cu = !{!0}
 !llvm.module.flags = !{!9, !10}
-!0 = !MDCompileUnit(language: DW_LANG_C99, isOptimized: false, emissionKind: 1, file: !1, enums: !2, retainedTypes: !2, subprograms: !3, globals: !2, imports: !2)
-!1 = !MDFile(filename: "test.c", directory: "")
+!0 = !DICompileUnit(language: DW_LANG_C99, isOptimized: false, emissionKind: 1, file: !1, enums: !2, retainedTypes: !2, subprograms: !3, globals: !2, imports: !2)
+!1 = !DIFile(filename: "test.c", directory: "")
 !2 = !{}
 !3 = !{!4}
-!4 = !MDSubprogram(name: "f", line: 2, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 2, file: !1, scope: !1, type: !6, function: i32 (i32)* @f, variables: !2)
-!6 = !MDSubroutineType(types: !7)
+!4 = !DISubprogram(name: "f", line: 2, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 2, file: !1, scope: !1, type: !6, function: i32 (i32)* @f, variables: !2)
+!6 = !DISubroutineType(types: !7)
 !7 = !{!8, !8}
-!8 = !MDBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
+!8 = !DIBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
 !9 = !{i32 2, !"Dwarf Version", i32 4}
 !10 = !{i32 2, !"Debug Info Version", i32 3}
-!11 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "a", line: 2, arg: 1, scope: !4, file: !1, type: !8)
-!12 = !MDLocation(line: 2, scope: !4)
+!11 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "a", line: 2, arg: 1, scope: !4, file: !1, type: !8)
+!12 = !DILocation(line: 2, scope: !4)
 
diff --git a/test/CodeGen/XCore/exception.ll b/test/CodeGen/XCore/exception.ll
index 705c6b42ade..dd7e012f737 100644
--- a/test/CodeGen/XCore/exception.ll
+++ b/test/CodeGen/XCore/exception.ll
@@ -47,7 +47,7 @@ entry:
 ; CHECK: entsp 4
 ; CHECK: .cfi_def_cfa_offset 16
 ; CHECK: .cfi_offset 15, 0
-define void @fn_catch() {
+define void @fn_catch() personality i8* bitcast (i32 (...)* @__gxx_personality_v0 to i8*) {
 entry:
 
 ; N.B. we alloc no variables, hence force compiler to spill
@@ -77,7 +77,7 @@ cont:
 ; CHECK: ldw r6, r0[0]
 ; CHECK: bl __cxa_end_catch
 lpad:
-  %0 = landingpad { i8*, i32 } personality i8* bitcast (i32 (...)* @__gxx_personality_v0 to i8*)
+  %0 = landingpad { i8*, i32 }
           cleanup
           catch i8* bitcast (i8** @_ZTIi to i8*)
           catch i8* bitcast (i8** @_ZTId to i8*)
diff --git a/test/DebugInfo/2009-11-03-InsertExtractValue.ll b/test/DebugInfo/2009-11-03-InsertExtractValue.ll
index f816e7219fa..f8382af3abc 100644
--- a/test/DebugInfo/2009-11-03-InsertExtractValue.ll
+++ b/test/DebugInfo/2009-11-03-InsertExtractValue.ll
@@ -4,12 +4,12 @@
 !llvm.dbg.cu = !{!5}
 !llvm.module.flags = !{!6}
 
-!0 = !MDSubprogram(name: "bar", linkageName: "_ZN3foo3barEv", line: 3, isLocal: false, isDefinition: false, virtualIndex: 6, flags: DIFlagProtected | DIFlagPrototyped, isOptimized: false, scopeLine: 3, file: !4, scope: !1, type: !2)
-!1 = !MDFile(filename: "/foo", directory: "bar.cpp")
-!2 = !MDSubroutineType(types: !3)
+!0 = !DISubprogram(name: "bar", linkageName: "_ZN3foo3barEv", line: 3, isLocal: false, isDefinition: false, virtualIndex: 6, flags: DIFlagProtected | DIFlagPrototyped, isOptimized: false, scopeLine: 3, file: !4, scope: !1, type: !2)
+!1 = !DIFile(filename: "/foo", directory: "bar.cpp")
+!2 = !DISubroutineType(types: !3)
 !3 = !{null}
-!4 = !MDFile(filename: "/foo", directory: "bar.cpp")
-!5 = !MDCompileUnit(language: DW_LANG_C99, isOptimized: true, emissionKind: 0, file: !4, enums: !{}, retainedTypes: !{})
+!4 = !DIFile(filename: "/foo", directory: "bar.cpp")
+!5 = !DICompileUnit(language: DW_LANG_C99, isOptimized: true, emissionKind: 0, file: !4, enums: !{}, retainedTypes: !{})
 
 define <{i32, i32}> @f1() {
 ; CHECK: !dbgx ![[NUMBER:[0-9]+]]
diff --git a/test/DebugInfo/2009-11-05-DeadGlobalVariable.ll b/test/DebugInfo/2009-11-05-DeadGlobalVariable.ll
index eb0777dfa87..32d4c0ac6c3 100644
--- a/test/DebugInfo/2009-11-05-DeadGlobalVariable.ll
+++ b/test/DebugInfo/2009-11-05-DeadGlobalVariable.ll
@@ -10,17 +10,17 @@ entry:
 !llvm.dbg.cu = !{!0}
 !llvm.module.flags = !{!18}
 
-!0 = !MDCompileUnit(language: DW_LANG_C99, producer: "clang version 3.0 (trunk 139632)", isOptimized: true, emissionKind: 0, file: !17, enums: !1, retainedTypes: !1, subprograms: !3, globals: !12)
+!0 = !DICompileUnit(language: DW_LANG_C99, producer: "clang version 3.0 (trunk 139632)", isOptimized: true, emissionKind: 0, file: !17, enums: !1, retainedTypes: !1, subprograms: !3, globals: !12)
 !1 = !{}
 !3 = !{!5}
-!5 = !MDSubprogram(name: "foo", line: 1, isLocal: false, isDefinition: true, virtualIndex: 6, isOptimized: true, file: !17, scope: !6, type: !7, function: i32 ()* @foo)
-!6 = !MDFile(filename: "fb.c", directory: "/private/tmp")
-!7 = !MDSubroutineType(types: !8)
+!5 = !DISubprogram(name: "foo", line: 1, isLocal: false, isDefinition: true, virtualIndex: 6, isOptimized: true, file: !17, scope: !6, type: !7, function: i32 ()* @foo)
+!6 = !DIFile(filename: "fb.c", directory: "/private/tmp")
+!7 = !DISubroutineType(types: !8)
 !8 = !{!9}
-!9 = !MDBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
+!9 = !DIBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
 !12 = !{!14}
-!14 = !MDGlobalVariable(name: "bar", line: 2, isLocal: true, isDefinition: true, scope: !5, file: !6, type: !9)
-!15 = !MDLocation(line: 3, column: 3, scope: !16)
-!16 = distinct !MDLexicalBlock(line: 1, column: 11, file: !17, scope: !5)
-!17 = !MDFile(filename: "fb.c", directory: "/private/tmp")
+!14 = !DIGlobalVariable(name: "bar", line: 2, isLocal: true, isDefinition: true, scope: !5, file: !6, type: !9)
+!15 = !DILocation(line: 3, column: 3, scope: !16)
+!16 = distinct !DILexicalBlock(line: 1, column: 11, file: !17, scope: !5)
+!17 = !DIFile(filename: "fb.c", directory: "/private/tmp")
 !18 = !{i32 1, !"Debug Info Version", i32 3}
diff --git a/test/DebugInfo/2009-11-06-NamelessGlobalVariable.ll b/test/DebugInfo/2009-11-06-NamelessGlobalVariable.ll
index 69f3c73f65f..79f949b438f 100644
--- a/test/DebugInfo/2009-11-06-NamelessGlobalVariable.ll
+++ b/test/DebugInfo/2009-11-06-NamelessGlobalVariable.ll
@@ -4,11 +4,11 @@
 !llvm.dbg.cu = !{!0}
 !llvm.module.flags = !{!9}
 
-!0 = !MDCompileUnit(language: DW_LANG_C99, producer: "clang version 3.0 (trunk 139632)", isOptimized: true, emissionKind: 0, file: !8, enums: !2, retainedTypes: !2, subprograms: !2, globals: !3)
+!0 = !DICompileUnit(language: DW_LANG_C99, producer: "clang version 3.0 (trunk 139632)", isOptimized: true, emissionKind: 0, file: !8, enums: !2, retainedTypes: !2, subprograms: !2, globals: !3)
 !2 = !{}
 !3 = !{!5}
-!5 = !MDGlobalVariable(name: "a", line: 2, isLocal: false, isDefinition: true, scope: null, file: !6, type: !7, variable: i32* @0)
-!6 = !MDFile(filename: "g.c", directory: "/private/tmp")
-!7 = !MDBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
-!8 = !MDFile(filename: "g.c", directory: "/private/tmp")
+!5 = !DIGlobalVariable(name: "a", line: 2, isLocal: false, isDefinition: true, scope: null, file: !6, type: !7, variable: i32* @0)
+!6 = !DIFile(filename: "g.c", directory: "/private/tmp")
+!7 = !DIBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
+!8 = !DIFile(filename: "g.c", directory: "/private/tmp")
 !9 = !{i32 1, !"Debug Info Version", i32 3}
diff --git a/test/DebugInfo/2009-11-10-CurrentFn.ll b/test/DebugInfo/2009-11-10-CurrentFn.ll
index f6255ec7586..228edec93c4 100644
--- a/test/DebugInfo/2009-11-10-CurrentFn.ll
+++ b/test/DebugInfo/2009-11-10-CurrentFn.ll
@@ -13,19 +13,19 @@ declare void @llvm.dbg.value(metadata, i64, metadata, metadata) nounwind readnon
 !llvm.dbg.cu = !{!0}
 !llvm.module.flags = !{!18}
 
-!0 = !MDCompileUnit(language: DW_LANG_C99, producer: "clang version 3.0 (trunk 139632)", isOptimized: true, emissionKind: 0, file: !17, enums: !1, retainedTypes: !1, subprograms: !3, globals: !1)
+!0 = !DICompileUnit(language: DW_LANG_C99, producer: "clang version 3.0 (trunk 139632)", isOptimized: true, emissionKind: 0, file: !17, enums: !1, retainedTypes: !1, subprograms: !3, globals: !1)
 !1 = !{}
 !3 = !{!5}
-!5 = !MDSubprogram(name: "bar", line: 3, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: true, file: !17, scope: !6, type: !7, function: void (i32)* @bar, variables: !9)
-!6 = !MDFile(filename: "cf.c", directory: "/private/tmp")
-!7 = !MDSubroutineType(types: !8)
+!5 = !DISubprogram(name: "bar", line: 3, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: true, file: !17, scope: !6, type: !7, function: void (i32)* @bar, variables: !9)
+!6 = !DIFile(filename: "cf.c", directory: "/private/tmp")
+!7 = !DISubroutineType(types: !8)
 !8 = !{null}
 !9 = !{!11}
-!11 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "i", line: 3, arg: 1, scope: !5, file: !17, type: !12)
-!12 = !MDBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
-!13 = !MDLocation(line: 3, column: 14, scope: !5)
-!14 = !MDLocation(line: 4, column: 3, scope: !15)
-!15 = distinct !MDLexicalBlock(line: 3, column: 17, file: !17, scope: !5)
-!16 = !MDLocation(line: 5, column: 1, scope: !15)
-!17 = !MDFile(filename: "cf.c", directory: "/private/tmp")
+!11 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "i", line: 3, arg: 1, scope: !5, file: !17, type: !12)
+!12 = !DIBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
+!13 = !DILocation(line: 3, column: 14, scope: !5)
+!14 = !DILocation(line: 4, column: 3, scope: !15)
+!15 = distinct !DILexicalBlock(line: 3, column: 17, file: !17, scope: !5)
+!16 = !DILocation(line: 5, column: 1, scope: !15)
+!17 = !DIFile(filename: "cf.c", directory: "/private/tmp")
 !18 = !{i32 1, !"Debug Info Version", i32 3}
diff --git a/test/DebugInfo/2010-01-05-DbgScope.ll b/test/DebugInfo/2010-01-05-DbgScope.ll
index 19c9335bfc0..d5baa198422 100644
--- a/test/DebugInfo/2010-01-05-DbgScope.ll
+++ b/test/DebugInfo/2010-01-05-DbgScope.ll
@@ -11,15 +11,15 @@ entry:
 !llvm.dbg.cu = !{!3}
 !llvm.module.flags = !{!14}
 
-!0 = !MDLocation(line: 571, column: 3, scope: !1)
-!1 = distinct !MDLexicalBlock(line: 1, column: 1, file: !11, scope: !2)
-!2 = !MDSubprogram(name: "foo", linkageName: "foo", line: 561, isLocal: false, isDefinition: true, virtualIndex: 6, isOptimized: false, scope: !3, type: !4)
-!3 = !MDCompileUnit(language: DW_LANG_C99, producer: "clang 1.1", isOptimized: true, emissionKind: 0, file: !11, enums: !12, retainedTypes: !12, subprograms: !13)
-!4 = !MDSubroutineType(types: !5)
+!0 = !DILocation(line: 571, column: 3, scope: !1)
+!1 = distinct !DILexicalBlock(line: 1, column: 1, file: !11, scope: !2)
+!2 = !DISubprogram(name: "foo", linkageName: "foo", line: 561, isLocal: false, isDefinition: true, virtualIndex: 6, isOptimized: false, scope: !3, type: !4)
+!3 = !DICompileUnit(language: DW_LANG_C99, producer: "clang 1.1", isOptimized: true, emissionKind: 0, file: !11, enums: !12, retainedTypes: !12, subprograms: !13)
+!4 = !DISubroutineType(types: !5)
 !5 = !{!6}
-!6 = !MDBasicType(tag: DW_TAG_base_type, name: "char", size: 8, align: 8, encoding: DW_ATE_signed_char)
-!10 = !MDLocation(line: 588, column: 1, scope: !2)
-!11 = !MDFile(filename: "hashtab.c", directory: "/usr/src/gnu/usr.bin/cc/cc_tools/../../../../contrib/gcclibs/libiberty")
+!6 = !DIBasicType(tag: DW_TAG_base_type, name: "char", size: 8, align: 8, encoding: DW_ATE_signed_char)
+!10 = !DILocation(line: 588, column: 1, scope: !2)
+!11 = !DIFile(filename: "hashtab.c", directory: "/usr/src/gnu/usr.bin/cc/cc_tools/../../../../contrib/gcclibs/libiberty")
 !12 = !{}
 !13 = !{!2}
 !14 = !{i32 1, !"Debug Info Version", i32 3}
diff --git a/test/DebugInfo/2010-03-12-llc-crash.ll b/test/DebugInfo/2010-03-12-llc-crash.ll
index 57f96e3f15d..fcca6c52170 100644
--- a/test/DebugInfo/2010-03-12-llc-crash.ll
+++ b/test/DebugInfo/2010-03-12-llc-crash.ll
@@ -1,22 +1,22 @@
 ; RUN: llc -O0 < %s -o /dev/null
-; llc should not crash on this invalid input.
+; llc should not crash on this optimized out debug info.
 ; PR6588
 declare void @llvm.dbg.declare(metadata, metadata, metadata) nounwind readnone
 
 define void @foo() {
 entry:
-  call void @llvm.dbg.declare(metadata i32* undef, metadata !0, metadata !MDExpression()), !dbg !MDLocation(scope: !1)
+  call void @llvm.dbg.declare(metadata i32* undef, metadata !0, metadata !DIExpression()), !dbg !DILocation(scope: !1)
   ret void
 }
 
-!0 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "sy", line: 890, arg: 0, scope: !1, file: !2, type: !7)
-!1 = !MDSubprogram(name: "foo", linkageName: "foo", line: 892, isLocal: false, isDefinition: true, virtualIndex: 6, isOptimized: false, file: !8, scope: !3, type: !4)
-!2 = !MDFile(filename: "qpainter.h", directory: "QtGui")
-!3 = !MDCompileUnit(language: DW_LANG_C_plus_plus, producer: "clang 1.1", isOptimized: true, emissionKind: 0, file: !9, enums: !10, retainedTypes: !10)
-!4 = !MDSubroutineType(types: !6)
-!5 = !MDFile(filename: "splineeditor.cpp", directory: "src")
+!0 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "sy", line: 890, arg: 1, scope: !1, file: !2, type: !7)
+!1 = !DISubprogram(name: "foo", linkageName: "foo", line: 892, isLocal: false, isDefinition: true, virtualIndex: 6, isOptimized: false, file: !8, scope: !3, type: !4)
+!2 = !DIFile(filename: "qpainter.h", directory: "QtGui")
+!3 = !DICompileUnit(language: DW_LANG_C_plus_plus, producer: "clang 1.1", isOptimized: true, emissionKind: 0, file: !9, enums: !10, retainedTypes: !10)
+!4 = !DISubroutineType(types: !6)
+!5 = !DIFile(filename: "splineeditor.cpp", directory: "src")
 !6 = !{null}
-!7 = !MDBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
-!8 = !MDFile(filename: "qpainter.h", directory: "QtGui")
-!9 = !MDFile(filename: "splineeditor.cpp", directory: "src")
+!7 = !DIBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
+!8 = !DIFile(filename: "qpainter.h", directory: "QtGui")
+!9 = !DIFile(filename: "splineeditor.cpp", directory: "src")
 !10 = !{i32 0}
diff --git a/test/DebugInfo/2010-03-19-DbgDeclare.ll b/test/DebugInfo/2010-03-19-DbgDeclare.ll
index dff548994eb..40d96e1d443 100644
--- a/test/DebugInfo/2010-03-19-DbgDeclare.ll
+++ b/test/DebugInfo/2010-03-19-DbgDeclare.ll
@@ -4,17 +4,17 @@
 
 define void @Foo(i32 %a, i32 %b) {
 entry:
-  call void @llvm.dbg.declare(metadata i32* null, metadata !1, metadata !MDExpression()), !dbg !MDLocation(scope: !6)
+  call void @llvm.dbg.declare(metadata i32* null, metadata !1, metadata !DIExpression()), !dbg !DILocation(scope: !6)
   ret void
 }
 !llvm.dbg.cu = !{!2}
 !llvm.module.flags = !{!5}
-!2 = !MDCompileUnit(language: DW_LANG_Mips_Assembler, producer: "clang version 3.3 ", isOptimized: false, emissionKind: 1, file: !4, enums: !3, retainedTypes: !3, subprograms: !3, globals: !3, imports:  !3)
+!2 = !DICompileUnit(language: DW_LANG_Mips_Assembler, producer: "clang version 3.3 ", isOptimized: false, emissionKind: 1, file: !4, enums: !3, retainedTypes: !3, subprograms: !3, globals: !3, imports:  !3)
 !3 = !{}
-!0 = !MDLocation(line: 662302, column: 26, scope: !1)
-!1 = !MDLocalVariable(tag: DW_TAG_auto_variable, name: "foo", scope: !6)
-!4 = !MDFile(filename: "scratch.cpp", directory: "/usr/local/google/home/blaikie/dev/scratch")
-!6 = !MDSubprogram()
+!0 = !DILocation(line: 662302, column: 26, scope: !1)
+!1 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "foo", scope: !6)
+!4 = !DIFile(filename: "scratch.cpp", directory: "/usr/local/google/home/blaikie/dev/scratch")
+!6 = !DISubprogram()
 
 declare void @llvm.dbg.declare(metadata, metadata, metadata) nounwind readnone
 !5 = !{i32 1, !"Debug Info Version", i32 3}
diff --git a/test/DebugInfo/2010-03-24-MemberFn.ll b/test/DebugInfo/2010-03-24-MemberFn.ll
index f925297d789..83ff5f97805 100644
--- a/test/DebugInfo/2010-03-24-MemberFn.ll
+++ b/test/DebugInfo/2010-03-24-MemberFn.ll
@@ -8,7 +8,7 @@ entry:
   %0 = alloca i32                                 ; <i32*> [#uses=2]
   %s1 = alloca %struct.S                          ; <%struct.S*> [#uses=1]
   %"alloca point" = bitcast i32 0 to i32          ; <i32> [#uses=0]
-  call void @llvm.dbg.declare(metadata %struct.S* %s1, metadata !0, metadata !MDExpression()), !dbg !16
+  call void @llvm.dbg.declare(metadata %struct.S* %s1, metadata !0, metadata !DIExpression()), !dbg !16
   %1 = call i32 @_ZN1S3fooEv(%struct.S* %s1) nounwind, !dbg !17 ; <i32> [#uses=1]
   store i32 %1, i32* %0, align 4, !dbg !17
   %2 = load i32, i32* %0, align 4, !dbg !17            ; <i32> [#uses=1]
@@ -25,7 +25,7 @@ entry:
   %this_addr = alloca %struct.S*                  ; <%struct.S**> [#uses=1]
   %retval = alloca i32                            ; <i32*> [#uses=1]
   %"alloca point" = bitcast i32 0 to i32          ; <i32> [#uses=0]
-  call void @llvm.dbg.declare(metadata %struct.S** %this_addr, metadata !18, metadata !MDExpression()), !dbg !21
+  call void @llvm.dbg.declare(metadata %struct.S** %this_addr, metadata !18, metadata !DIExpression()), !dbg !21
   store %struct.S* %this, %struct.S** %this_addr
   br label %return, !dbg !21
 
@@ -39,32 +39,32 @@ declare void @llvm.dbg.declare(metadata, metadata, metadata) nounwind readnone
 !llvm.dbg.cu = !{!5}
 !llvm.module.flags = !{!28}
 
-!0 = !MDLocalVariable(tag: DW_TAG_auto_variable, name: "s1", line: 3, scope: !1, file: !4, type: !9)
-!1 = distinct !MDLexicalBlock(line: 3, column: 0, file: !25, scope: !2)
-!2 = distinct !MDLexicalBlock(line: 3, column: 0, file: !25, scope: !3)
-!3 = !MDSubprogram(name: "bar", linkageName: "_Z3barv", line: 3, isLocal: false, isDefinition: true, virtualIndex: 6, isOptimized: false, scopeLine: 3, file: !25, scope: !4, type: !6, function: i32 ()* @_Z3barv)
-!4 = !MDFile(filename: "one.cc", directory: "/tmp/")
-!5 = !MDCompileUnit(language: DW_LANG_C_plus_plus, producer: "4.2.1 (Based on Apple Inc. build 5658) (LLVM build)", isOptimized: false, emissionKind: 0, file: !25, enums: !27, retainedTypes: !27, subprograms: !24, imports:  null)
-!6 = !MDSubroutineType(types: !7)
+!0 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "s1", line: 3, scope: !1, file: !4, type: !9)
+!1 = distinct !DILexicalBlock(line: 3, column: 0, file: !25, scope: !2)
+!2 = distinct !DILexicalBlock(line: 3, column: 0, file: !25, scope: !3)
+!3 = !DISubprogram(name: "bar", linkageName: "_Z3barv", line: 3, isLocal: false, isDefinition: true, virtualIndex: 6, isOptimized: false, scopeLine: 3, file: !25, scope: !4, type: !6, function: i32 ()* @_Z3barv)
+!4 = !DIFile(filename: "one.cc", directory: "/tmp/")
+!5 = !DICompileUnit(language: DW_LANG_C_plus_plus, producer: "4.2.1 (Based on Apple Inc. build 5658) (LLVM build)", isOptimized: false, emissionKind: 0, file: !25, enums: !27, retainedTypes: !27, subprograms: !24, imports:  null)
+!6 = !DISubroutineType(types: !7)
 !7 = !{!8}
-!8 = !MDBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
-!9 = !MDCompositeType(tag: DW_TAG_structure_type, name: "S", line: 2, size: 8, align: 8, file: !26, scope: !4, elements: !11)
-!10 = !MDFile(filename: "one.h", directory: "/tmp/")
+!8 = !DIBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
+!9 = !DICompositeType(tag: DW_TAG_structure_type, name: "S", line: 2, size: 8, align: 8, file: !26, scope: !4, elements: !11)
+!10 = !DIFile(filename: "one.h", directory: "/tmp/")
 !11 = !{!12}
-!12 = !MDSubprogram(name: "foo", linkageName: "_ZN1S3fooEv", line: 3, isLocal: false, isDefinition: true, virtualIndex: 6, isOptimized: false, scopeLine: 3, file: !26, scope: !9, type: !13, function: i32 (%struct.S*)* @_ZN1S3fooEv)
-!13 = !MDSubroutineType(types: !14)
+!12 = !DISubprogram(name: "foo", linkageName: "_ZN1S3fooEv", line: 3, isLocal: false, isDefinition: true, virtualIndex: 6, isOptimized: false, scopeLine: 3, file: !26, scope: !9, type: !13, function: i32 (%struct.S*)* @_ZN1S3fooEv)
+!13 = !DISubroutineType(types: !14)
 !14 = !{!8, !15}
-!15 = !MDDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, flags: DIFlagArtificial, file: !25, scope: !4, baseType: !9)
-!16 = !MDLocation(line: 3, scope: !1)
-!17 = !MDLocation(line: 3, scope: !3)
-!18 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "this", line: 3, arg: 0, scope: !12, file: !10, type: !19)
-!19 = !MDDerivedType(tag: DW_TAG_const_type, size: 64, align: 64, flags: DIFlagArtificial, file: !25, scope: !4, baseType: !20)
-!20 = !MDDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, file: !25, scope: !4, baseType: !9)
-!21 = !MDLocation(line: 3, scope: !12)
-!22 = !MDLocation(line: 3, scope: !23)
-!23 = distinct !MDLexicalBlock(line: 3, column: 0, file: !26, scope: !12)
+!15 = !DIDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, flags: DIFlagArtificial, file: !25, scope: !4, baseType: !9)
+!16 = !DILocation(line: 3, scope: !1)
+!17 = !DILocation(line: 3, scope: !3)
+!18 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "this", line: 3, arg: 1, scope: !12, file: !10, type: !19)
+!19 = !DIDerivedType(tag: DW_TAG_const_type, size: 64, align: 64, flags: DIFlagArtificial, file: !25, scope: !4, baseType: !20)
+!20 = !DIDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, file: !25, scope: !4, baseType: !9)
+!21 = !DILocation(line: 3, scope: !12)
+!22 = !DILocation(line: 3, scope: !23)
+!23 = distinct !DILexicalBlock(line: 3, column: 0, file: !26, scope: !12)
 !24 = !{!3, !12}
-!25 = !MDFile(filename: "one.cc", directory: "/tmp/")
-!26 = !MDFile(filename: "one.h", directory: "/tmp/")
+!25 = !DIFile(filename: "one.cc", directory: "/tmp/")
+!26 = !DIFile(filename: "one.h", directory: "/tmp/")
 !27 = !{}
 !28 = !{i32 1, !"Debug Info Version", i32 3}
diff --git a/test/DebugInfo/2010-04-06-NestedFnDbgInfo.ll b/test/DebugInfo/2010-04-06-NestedFnDbgInfo.ll
index 9683c1bc12b..68fd0a93eac 100644
--- a/test/DebugInfo/2010-04-06-NestedFnDbgInfo.ll
+++ b/test/DebugInfo/2010-04-06-NestedFnDbgInfo.ll
@@ -26,7 +26,7 @@ entry:
   %retval = alloca i32, align 4                   ; <i32*> [#uses=3]
   %b = alloca %class.A, align 1                   ; <%class.A*> [#uses=1]
   store i32 0, i32* %retval
-  call void @llvm.dbg.declare(metadata %class.A* %b, metadata !0, metadata !MDExpression()), !dbg !14
+  call void @llvm.dbg.declare(metadata %class.A* %b, metadata !0, metadata !DIExpression()), !dbg !14
   %call = call i32 @_ZN1B2fnEv(%class.A* %b), !dbg !15 ; <i32> [#uses=1]
   store i32 %call, i32* %retval, !dbg !15
   %0 = load i32, i32* %retval, !dbg !16                ; <i32> [#uses=1]
@@ -42,10 +42,10 @@ entry:
   %a = alloca %class.A, align 1                   ; <%class.A*> [#uses=1]
   %i = alloca i32, align 4                        ; <i32*> [#uses=2]
   store %class.A* %this, %class.A** %this.addr
-  call void @llvm.dbg.declare(metadata %class.A** %this.addr, metadata !17, metadata !MDExpression()), !dbg !18
+  call void @llvm.dbg.declare(metadata %class.A** %this.addr, metadata !17, metadata !DIExpression()), !dbg !18
   %this1 = load %class.A*, %class.A** %this.addr             ; <%class.A*> [#uses=0]
-  call void @llvm.dbg.declare(metadata %class.A* %a, metadata !19, metadata !MDExpression()), !dbg !27
-  call void @llvm.dbg.declare(metadata i32* %i, metadata !28, metadata !MDExpression()), !dbg !29
+  call void @llvm.dbg.declare(metadata %class.A* %a, metadata !19, metadata !DIExpression()), !dbg !27
+  call void @llvm.dbg.declare(metadata i32* %i, metadata !28, metadata !DIExpression()), !dbg !29
   %call = call i32 @_ZZN1B2fnEvEN1A3fooEv(%class.A* %a), !dbg !30 ; <i32> [#uses=1]
   store i32 %call, i32* %i, !dbg !30
   %tmp = load i32, i32* %i, !dbg !31                   ; <i32> [#uses=1]
@@ -59,7 +59,7 @@ entry:
   %retval = alloca i32, align 4                   ; <i32*> [#uses=2]
   %this.addr = alloca %class.A*, align 8          ; <%class.A**> [#uses=2]
   store %class.A* %this, %class.A** %this.addr
-  call void @llvm.dbg.declare(metadata %class.A** %this.addr, metadata !33, metadata !MDExpression()), !dbg !34
+  call void @llvm.dbg.declare(metadata %class.A** %this.addr, metadata !33, metadata !DIExpression()), !dbg !34
   %this1 = load %class.A*, %class.A** %this.addr             ; <%class.A*> [#uses=0]
   store i32 42, i32* %retval, !dbg !35
   %0 = load i32, i32* %retval, !dbg !35                ; <i32> [#uses=1]
@@ -70,43 +70,43 @@ entry:
 !llvm.module.flags = !{!40}
 !37 = !{!2, !10, !23}
 
-!0 = !MDLocalVariable(tag: DW_TAG_auto_variable, name: "b", line: 16, scope: !1, file: !3, type: !8)
-!1 = distinct !MDLexicalBlock(line: 15, column: 12, file: !38, scope: !2)
-!2 = !MDSubprogram(name: "main", linkageName: "main", line: 15, isLocal: false, isDefinition: true, virtualIndex: 6, isOptimized: false, scopeLine: 15, file: !38, scope: !3, type: !5, function: i32 ()* @main)
-!3 = !MDFile(filename: "one.cc", directory: "/tmp")
-!4 = !MDCompileUnit(language: DW_LANG_C_plus_plus, producer: "clang 1.5", isOptimized: false, emissionKind: 0, file: !38, enums: !39, retainedTypes: !39, subprograms: !37, imports:  null)
-!5 = !MDSubroutineType(types: !6)
+!0 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "b", line: 16, scope: !1, file: !3, type: !8)
+!1 = distinct !DILexicalBlock(line: 15, column: 12, file: !38, scope: !2)
+!2 = !DISubprogram(name: "main", linkageName: "main", line: 15, isLocal: false, isDefinition: true, virtualIndex: 6, isOptimized: false, scopeLine: 15, file: !38, scope: !3, type: !5, function: i32 ()* @main)
+!3 = !DIFile(filename: "one.cc", directory: "/tmp")
+!4 = !DICompileUnit(language: DW_LANG_C_plus_plus, producer: "clang 1.5", isOptimized: false, emissionKind: 0, file: !38, enums: !39, retainedTypes: !39, subprograms: !37, imports:  null)
+!5 = !DISubroutineType(types: !6)
 !6 = !{!7}
-!7 = !MDBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
-!8 = !MDCompositeType(tag: DW_TAG_class_type, name: "B", line: 2, size: 8, align: 8, file: !38, scope: !3, elements: !9)
+!7 = !DIBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
+!8 = !DICompositeType(tag: DW_TAG_class_type, name: "B", line: 2, size: 8, align: 8, file: !38, scope: !3, elements: !9)
 !9 = !{!10}
-!10 = !MDSubprogram(name: "fn", linkageName: "_ZN1B2fnEv", line: 4, isLocal: false, isDefinition: true, virtualIndex: 6, isOptimized: false, scopeLine: 4, file: !38, scope: !8, type: !11, function: i32 (%class.A*)* @_ZN1B2fnEv)
-!11 = !MDSubroutineType(types: !12)
+!10 = !DISubprogram(name: "fn", linkageName: "_ZN1B2fnEv", line: 4, isLocal: false, isDefinition: true, virtualIndex: 6, isOptimized: false, scopeLine: 4, file: !38, scope: !8, type: !11, function: i32 (%class.A*)* @_ZN1B2fnEv)
+!11 = !DISubroutineType(types: !12)
 !12 = !{!7, !13}
-!13 = !MDDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, flags: DIFlagArtificial, file: !38, scope: !3, baseType: !8)
-!14 = !MDLocation(line: 16, column: 5, scope: !1)
-!15 = !MDLocation(line: 17, column: 3, scope: !1)
-!16 = !MDLocation(line: 18, column: 1, scope: !2)
-!17 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "this", line: 4, arg: 0, scope: !10, file: !3, type: !13)
-!18 = !MDLocation(line: 4, column: 7, scope: !10)
-!19 = !MDLocalVariable(tag: DW_TAG_auto_variable, name: "a", line: 9, scope: !20, file: !3, type: !21)
-!20 = distinct !MDLexicalBlock(line: 4, column: 12, file: !38, scope: !10)
-!21 = !MDCompositeType(tag: DW_TAG_class_type, name: "A", line: 5, size: 8, align: 8, file: !38, scope: !10, elements: !22)
+!13 = !DIDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, flags: DIFlagArtificial, file: !38, scope: !3, baseType: !8)
+!14 = !DILocation(line: 16, column: 5, scope: !1)
+!15 = !DILocation(line: 17, column: 3, scope: !1)
+!16 = !DILocation(line: 18, column: 1, scope: !2)
+!17 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "this", line: 4, arg: 1, scope: !10, file: !3, type: !13)
+!18 = !DILocation(line: 4, column: 7, scope: !10)
+!19 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "a", line: 9, scope: !20, file: !3, type: !21)
+!20 = distinct !DILexicalBlock(line: 4, column: 12, file: !38, scope: !10)
+!21 = !DICompositeType(tag: DW_TAG_class_type, name: "A", line: 5, size: 8, align: 8, file: !38, scope: !10, elements: !22)
 !22 = !{!23}
-!23 = !MDSubprogram(name: "foo", linkageName: "_ZZN1B2fnEvEN1A3fooEv", line: 7, isLocal: false, isDefinition: true, virtualIndex: 6, isOptimized: false, scopeLine: 7, file: !38, scope: !21, type: !24, function: i32 (%class.A*)* @_ZZN1B2fnEvEN1A3fooEv)
-!24 = !MDSubroutineType(types: !25)
+!23 = !DISubprogram(name: "foo", linkageName: "_ZZN1B2fnEvEN1A3fooEv", line: 7, isLocal: false, isDefinition: true, virtualIndex: 6, isOptimized: false, scopeLine: 7, file: !38, scope: !21, type: !24, function: i32 (%class.A*)* @_ZZN1B2fnEvEN1A3fooEv)
+!24 = !DISubroutineType(types: !25)
 !25 = !{!7, !26}
-!26 = !MDDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, flags: DIFlagArtificial, file: !38, scope: !3, baseType: !21)
-!27 = !MDLocation(line: 9, column: 7, scope: !20)
-!28 = !MDLocalVariable(tag: DW_TAG_auto_variable, name: "i", line: 10, scope: !20, file: !3, type: !7)
-!29 = !MDLocation(line: 10, column: 9, scope: !20)
-!30 = !MDLocation(line: 10, column: 5, scope: !20)
-!31 = !MDLocation(line: 11, column: 5, scope: !20)
-!32 = !MDLocation(line: 12, column: 3, scope: !10)
-!33 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "this", line: 7, arg: 0, scope: !23, file: !3, type: !26)
-!34 = !MDLocation(line: 7, column: 11, scope: !23)
-!35 = !MDLocation(line: 7, column: 19, scope: !36)
-!36 = distinct !MDLexicalBlock(line: 7, column: 17, file: !38, scope: !23)
-!38 = !MDFile(filename: "one.cc", directory: "/tmp")
+!26 = !DIDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, flags: DIFlagArtificial, file: !38, scope: !3, baseType: !21)
+!27 = !DILocation(line: 9, column: 7, scope: !20)
+!28 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "i", line: 10, scope: !20, file: !3, type: !7)
+!29 = !DILocation(line: 10, column: 9, scope: !20)
+!30 = !DILocation(line: 10, column: 5, scope: !20)
+!31 = !DILocation(line: 11, column: 5, scope: !20)
+!32 = !DILocation(line: 12, column: 3, scope: !10)
+!33 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "this", line: 7, arg: 1, scope: !23, file: !3, type: !26)
+!34 = !DILocation(line: 7, column: 11, scope: !23)
+!35 = !DILocation(line: 7, column: 19, scope: !36)
+!36 = distinct !DILexicalBlock(line: 7, column: 17, file: !38, scope: !23)
+!38 = !DIFile(filename: "one.cc", directory: "/tmp")
 !39 = !{}
 !40 = !{i32 1, !"Debug Info Version", i32 3}
diff --git a/test/DebugInfo/2010-04-19-FramePtr.ll b/test/DebugInfo/2010-04-19-FramePtr.ll
index 0bded546584..ed6a4fdb54e 100644
--- a/test/DebugInfo/2010-04-19-FramePtr.ll
+++ b/test/DebugInfo/2010-04-19-FramePtr.ll
@@ -23,15 +23,15 @@ return:                                           ; preds = %entry
 !llvm.module.flags = !{!12}
 !9 = !{!1}
 
-!0 = !MDLocation(line: 2, scope: !1)
-!1 = !MDSubprogram(name: "foo", linkageName: "foo", line: 2, isLocal: false, isDefinition: true, virtualIndex: 6, isOptimized: false, scopeLine: 2, file: !10, scope: null, type: !4, function: i32 ()* @foo)
-!2 = !MDFile(filename: "a.c", directory: "/tmp")
-!3 = !MDCompileUnit(language: DW_LANG_C89, producer: "4.2.1 (Based on Apple Inc. build 5658) (LLVM build)", isOptimized: false, emissionKind: 0, file: !10, enums: !11, retainedTypes: !11, subprograms: !9, imports:  null)
-!4 = !MDSubroutineType(types: !5)
+!0 = !DILocation(line: 2, scope: !1)
+!1 = !DISubprogram(name: "foo", linkageName: "foo", line: 2, isLocal: false, isDefinition: true, virtualIndex: 6, isOptimized: false, scopeLine: 2, file: !10, scope: null, type: !4, function: i32 ()* @foo)
+!2 = !DIFile(filename: "a.c", directory: "/tmp")
+!3 = !DICompileUnit(language: DW_LANG_C89, producer: "4.2.1 (Based on Apple Inc. build 5658) (LLVM build)", isOptimized: false, emissionKind: 0, file: !10, enums: !11, retainedTypes: !11, subprograms: !9, imports:  null)
+!4 = !DISubroutineType(types: !5)
 !5 = !{!6}
-!6 = !MDBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
-!7 = !MDLocation(line: 2, scope: !8)
-!8 = distinct !MDLexicalBlock(line: 2, column: 0, file: !10, scope: !1)
-!10 = !MDFile(filename: "a.c", directory: "/tmp")
+!6 = !DIBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
+!7 = !DILocation(line: 2, scope: !8)
+!8 = distinct !DILexicalBlock(line: 2, column: 0, file: !10, scope: !1)
+!10 = !DIFile(filename: "a.c", directory: "/tmp")
 !11 = !{}
 !12 = !{i32 1, !"Debug Info Version", i32 3}
diff --git a/test/DebugInfo/2010-05-03-DisableFramePtr.ll b/test/DebugInfo/2010-05-03-DisableFramePtr.ll
index e87b51e9f21..96251596d3a 100644
--- a/test/DebugInfo/2010-05-03-DisableFramePtr.ll
+++ b/test/DebugInfo/2010-05-03-DisableFramePtr.ll
@@ -6,7 +6,7 @@ define void @DisposeDMNotificationUPP(void (%struct.AppleEvent*)* %userUPP) "no-
 entry:
   %userUPP_addr = alloca void (%struct.AppleEvent*)* ; <void (%struct.AppleEvent*)**> [#uses=1]
   %"alloca point" = bitcast i32 0 to i32          ; <i32> [#uses=0]
-  call void @llvm.dbg.declare(metadata void (%struct.AppleEvent*)** %userUPP_addr, metadata !0, metadata !MDExpression()), !dbg !13
+  call void @llvm.dbg.declare(metadata void (%struct.AppleEvent*)** %userUPP_addr, metadata !0, metadata !DIExpression()), !dbg !13
   store void (%struct.AppleEvent*)* %userUPP, void (%struct.AppleEvent*)** %userUPP_addr
   br label %return, !dbg !14
 
@@ -18,23 +18,23 @@ declare void @llvm.dbg.declare(metadata, metadata, metadata) nounwind readnone
 
 !llvm.dbg.cu = !{!3}
 !llvm.module.flags = !{!19}
-!0 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "userUPP", line: 7, arg: 0, scope: !1, file: !2, type: !6)
-!1 = !MDSubprogram(name: "DisposeDMNotificationUPP", linkageName: "DisposeDMNotificationUPP", line: 7, isLocal: false, isDefinition: true, virtualIndex: 6, isOptimized: false, file: !16, scope: null, type: !4)
-!2 = !MDFile(filename: "t.c", directory: "/Users/echeng/LLVM/radars/r7937664/")
-!3 = !MDCompileUnit(language: DW_LANG_C89, producer: "4.2.1 (Based on Apple Inc. build 5658) (LLVM build 9999)", isOptimized: true, emissionKind: 0, file: !16, enums: !17, retainedTypes: !17, subprograms: !18)
-!4 = !MDSubroutineType(types: !5)
+!0 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "userUPP", line: 7, arg: 1, scope: !1, file: !2, type: !6)
+!1 = !DISubprogram(name: "DisposeDMNotificationUPP", linkageName: "DisposeDMNotificationUPP", line: 7, isLocal: false, isDefinition: true, virtualIndex: 6, isOptimized: false, file: !16, scope: null, type: !4)
+!2 = !DIFile(filename: "t.c", directory: "/Users/echeng/LLVM/radars/r7937664/")
+!3 = !DICompileUnit(language: DW_LANG_C89, producer: "4.2.1 (Based on Apple Inc. build 5658) (LLVM build 9999)", isOptimized: true, emissionKind: 0, file: !16, enums: !17, retainedTypes: !17, subprograms: !18)
+!4 = !DISubroutineType(types: !5)
 !5 = !{null, !6}
-!6 = !MDDerivedType(tag: DW_TAG_typedef, name: "DMNotificationUPP", line: 6, file: !16, scope: !2, baseType: !7)
-!7 = !MDDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, file: !16, scope: !2, baseType: !8)
-!8 = !MDSubroutineType(types: !9)
+!6 = !DIDerivedType(tag: DW_TAG_typedef, name: "DMNotificationUPP", line: 6, file: !16, scope: !2, baseType: !7)
+!7 = !DIDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, file: !16, scope: !2, baseType: !8)
+!8 = !DISubroutineType(types: !9)
 !9 = !{null, !10}
-!10 = !MDDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, file: !16, scope: !2, baseType: !11)
-!11 = !MDDerivedType(tag: DW_TAG_typedef, name: "AppleEvent", line: 4, file: !16, scope: !2, baseType: !12)
-!12 = !MDCompositeType(tag: DW_TAG_structure_type, name: "AEDesc", line: 1, flags: DIFlagFwdDecl, file: !16, scope: !2)
-!13 = !MDLocation(line: 7, scope: !1)
-!14 = !MDLocation(line: 8, scope: !15)
-!15 = distinct !MDLexicalBlock(line: 7, column: 0, file: !16, scope: !1)
-!16 = !MDFile(filename: "t.c", directory: "/Users/echeng/LLVM/radars/r7937664/")
+!10 = !DIDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, file: !16, scope: !2, baseType: !11)
+!11 = !DIDerivedType(tag: DW_TAG_typedef, name: "AppleEvent", line: 4, file: !16, scope: !2, baseType: !12)
+!12 = !DICompositeType(tag: DW_TAG_structure_type, name: "AEDesc", line: 1, flags: DIFlagFwdDecl, file: !16, scope: !2)
+!13 = !DILocation(line: 7, scope: !1)
+!14 = !DILocation(line: 8, scope: !15)
+!15 = distinct !DILexicalBlock(line: 7, column: 0, file: !16, scope: !1)
+!16 = !DIFile(filename: "t.c", directory: "/Users/echeng/LLVM/radars/r7937664/")
 !17 = !{}
 !18 = !{!1}
 !19 = !{i32 1, !"Debug Info Version", i32 3}
diff --git a/test/DebugInfo/2010-05-03-OriginDIE.ll b/test/DebugInfo/2010-05-03-OriginDIE.ll
index 914e78134f6..40b29fe31f9 100644
--- a/test/DebugInfo/2010-05-03-OriginDIE.ll
+++ b/test/DebugInfo/2010-05-03-OriginDIE.ll
@@ -23,12 +23,12 @@ entry:
   %a10 = call i64 @llvm.bswap.i64(i64 %a9) nounwind ; <i64> [#uses=1]
   %a11 = getelementptr inbounds %struct.gpt_t, %struct.gpt_t* %gpt, i32 0, i32 8, !dbg !7 ; <i64*> [#uses=1]
   %a12 = load i64, i64* %a11, align 4, !dbg !7         ; <i64> [#uses=1]
-  call void @llvm.dbg.declare(metadata i64* %data_addr.i17, metadata !8, metadata !MDExpression()) nounwind, !dbg !14
+  call void @llvm.dbg.declare(metadata i64* %data_addr.i17, metadata !8, metadata !DIExpression()) nounwind, !dbg !14
   store i64 %a12, i64* %data_addr.i17, align 8
-  call void @llvm.dbg.value(metadata !6, i64 0, metadata !15, metadata !MDExpression()) nounwind, !dbg !MDLocation(scope: !16)
-  call void @llvm.dbg.value(metadata i32 0, i64 0, metadata !19, metadata !MDExpression()) nounwind, !dbg !MDLocation(scope: !16)
-  call void @llvm.dbg.declare(metadata !6, metadata !23, metadata !MDExpression()) nounwind, !dbg !MDLocation(scope: !24)
-  call void @llvm.dbg.value(metadata i64* %data_addr.i17, i64 0, metadata !34, metadata !MDExpression()) nounwind, !dbg !MDLocation(scope: !24)
+  call void @llvm.dbg.value(metadata !6, i64 0, metadata !15, metadata !DIExpression()) nounwind, !dbg !DILocation(scope: !16)
+  call void @llvm.dbg.value(metadata i32 0, i64 0, metadata !19, metadata !DIExpression()) nounwind, !dbg !DILocation(scope: !16)
+  call void @llvm.dbg.declare(metadata !6, metadata !23, metadata !DIExpression()) nounwind, !dbg !DILocation(scope: !24)
+  call void @llvm.dbg.value(metadata i64* %data_addr.i17, i64 0, metadata !34, metadata !DIExpression()) nounwind, !dbg !DILocation(scope: !24)
   %a13 = load volatile i64, i64* %data_addr.i17, align 8 ; <i64> [#uses=1]
   %a14 = call i64 @llvm.bswap.i64(i64 %a13) nounwind ; <i64> [#uses=2]
   %a15 = add i64 %a10, %a14, !dbg !7              ; <i64> [#uses=1]
@@ -50,45 +50,45 @@ declare void @uuid_LtoB(i8*, i8*)
 
 !llvm.dbg.cu = !{!4}
 !llvm.module.flags = !{!41}
-!0 = !MDLocation(line: 808, scope: !1)
-!1 = distinct !MDLexicalBlock(line: 807, column: 0, file: !39, scope: !2)
-!2 = !MDSubprogram(name: "gpt2gpm", linkageName: "gpt2gpm", line: 807, isLocal: true, isDefinition: true, virtualIndex: 6, isOptimized: false, file: !39, scope: null, type: !5)
-!3 = !MDFile(filename: "G.c", directory: "/tmp")
-!4 = !MDCompileUnit(language: DW_LANG_C89, producer: "llvm-gcc", isOptimized: true, emissionKind: 0, file: !39, enums: !18, retainedTypes: !18, subprograms: !40)
-!5 = !MDSubroutineType(types: !6)
+!0 = !DILocation(line: 808, scope: !1)
+!1 = distinct !DILexicalBlock(line: 807, column: 0, file: !39, scope: !2)
+!2 = !DISubprogram(name: "gpt2gpm", linkageName: "gpt2gpm", line: 807, isLocal: true, isDefinition: true, virtualIndex: 6, isOptimized: false, file: !39, scope: null, type: !5)
+!3 = !DIFile(filename: "G.c", directory: "/tmp")
+!4 = !DICompileUnit(language: DW_LANG_C89, producer: "llvm-gcc", isOptimized: true, emissionKind: 0, file: !39, enums: !18, retainedTypes: !18, subprograms: !40)
+!5 = !DISubroutineType(types: !6)
 !6 = !{null}
-!7 = !MDLocation(line: 810, scope: !1)
-!8 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "data", line: 201, arg: 0, scope: !9, file: !10, type: !11)
-!9 = !MDSubprogram(name: "_OSSwapInt64", linkageName: "_OSSwapInt64", line: 202, isLocal: true, isDefinition: true, virtualIndex: 6, isOptimized: false, file: !10, scope: null, type: !5)
-!10 = !MDFile(filename: "OSByteOrder.h", directory: "/usr/include/libkern/ppc")
-!11 = !MDDerivedType(tag: DW_TAG_typedef, name: "uint64_t", line: 59, file: !36, scope: !3, baseType: !13)
-!12 = !MDFile(filename: "stdint.h", directory: "/usr/4.2.1/include")
-!13 = !MDBasicType(tag: DW_TAG_base_type, name: "long long unsigned int", size: 64, align: 64, encoding: DW_ATE_unsigned)
-!14 = !MDLocation(line: 202, scope: !9, inlinedAt: !7)
-!15 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "base", line: 92, arg: 0, scope: !16, file: !10, type: !17)
-!16 = !MDSubprogram(name: "OSReadSwapInt64", linkageName: "OSReadSwapInt64", line: 95, isLocal: true, isDefinition: true, virtualIndex: 6, isOptimized: false, file: !38, scope: null, type: !5)
-!17 = !MDDerivedType(tag: DW_TAG_pointer_type, size: 32, align: 32, file: !39, scope: !3, baseType: null)
+!7 = !DILocation(line: 810, scope: !1)
+!8 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "data", line: 201, arg: 1, scope: !9, file: !10, type: !11)
+!9 = !DISubprogram(name: "_OSSwapInt64", linkageName: "_OSSwapInt64", line: 202, isLocal: true, isDefinition: true, virtualIndex: 6, isOptimized: false, file: !10, scope: null, type: !5)
+!10 = !DIFile(filename: "OSByteOrder.h", directory: "/usr/include/libkern/ppc")
+!11 = !DIDerivedType(tag: DW_TAG_typedef, name: "uint64_t", line: 59, file: !36, scope: !3, baseType: !13)
+!12 = !DIFile(filename: "stdint.h", directory: "/usr/4.2.1/include")
+!13 = !DIBasicType(tag: DW_TAG_base_type, name: "long long unsigned int", size: 64, align: 64, encoding: DW_ATE_unsigned)
+!14 = !DILocation(line: 202, scope: !9, inlinedAt: !7)
+!15 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "base", line: 92, arg: 2, scope: !16, file: !10, type: !17)
+!16 = !DISubprogram(name: "OSReadSwapInt64", linkageName: "OSReadSwapInt64", line: 95, isLocal: true, isDefinition: true, virtualIndex: 6, isOptimized: false, file: !38, scope: null, type: !5)
+!17 = !DIDerivedType(tag: DW_TAG_pointer_type, size: 32, align: 32, file: !39, scope: !3, baseType: null)
 !18 = !{}
-!19 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "byteOffset", line: 94, arg: 0, scope: !16, file: !10, type: !20)
-!20 = !MDDerivedType(tag: DW_TAG_typedef, name: "uintptr_t", line: 114, file: !37, scope: !3, baseType: !22)
-!21 = !MDFile(filename: "types.h", directory: "/usr/include/ppc")
-!22 = !MDBasicType(tag: DW_TAG_base_type, name: "long unsigned int", size: 32, align: 32, encoding: DW_ATE_unsigned)
-!23 = !MDLocalVariable(tag: DW_TAG_auto_variable, name: "u", line: 100, scope: !24, file: !10, type: !25)
-!24 = distinct !MDLexicalBlock(line: 95, column: 0, file: !38, scope: !16)
-!25 = !MDCompositeType(tag: DW_TAG_union_type, line: 97, size: 64, align: 64, file: !38, scope: !16, elements: !26)
+!19 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "byteOffset", line: 94, arg: 3, scope: !16, file: !10, type: !20)
+!20 = !DIDerivedType(tag: DW_TAG_typedef, name: "uintptr_t", line: 114, file: !37, scope: !3, baseType: !22)
+!21 = !DIFile(filename: "types.h", directory: "/usr/include/ppc")
+!22 = !DIBasicType(tag: DW_TAG_base_type, name: "long unsigned int", size: 32, align: 32, encoding: DW_ATE_unsigned)
+!23 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "u", line: 100, scope: !24, file: !10, type: !25)
+!24 = distinct !DILexicalBlock(line: 95, column: 0, file: !38, scope: !16)
+!25 = !DICompositeType(tag: DW_TAG_union_type, line: 97, size: 64, align: 64, file: !38, scope: !16, elements: !26)
 !26 = !{!27, !28}
-!27 = !MDDerivedType(tag: DW_TAG_member, name: "u64", line: 98, size: 64, align: 64, file: !38, scope: !25, baseType: !11)
-!28 = !MDDerivedType(tag: DW_TAG_member, name: "u32", line: 99, size: 64, align: 32, file: !38, scope: !25, baseType: !29)
-!29 = !MDCompositeType(tag: DW_TAG_array_type, size: 64, align: 32, file: !39, scope: !3, baseType: !30, elements: !32)
-!30 = !MDDerivedType(tag: DW_TAG_typedef, name: "uint32_t", line: 55, file: !36, scope: !3, baseType: !31)
-!31 = !MDBasicType(tag: DW_TAG_base_type, name: "unsigned int", size: 32, align: 32, encoding: DW_ATE_unsigned)
+!27 = !DIDerivedType(tag: DW_TAG_member, name: "u64", line: 98, size: 64, align: 64, file: !38, scope: !25, baseType: !11)
+!28 = !DIDerivedType(tag: DW_TAG_member, name: "u32", line: 99, size: 64, align: 32, file: !38, scope: !25, baseType: !29)
+!29 = !DICompositeType(tag: DW_TAG_array_type, size: 64, align: 32, file: !39, scope: !3, baseType: !30, elements: !32)
+!30 = !DIDerivedType(tag: DW_TAG_typedef, name: "uint32_t", line: 55, file: !36, scope: !3, baseType: !31)
+!31 = !DIBasicType(tag: DW_TAG_base_type, name: "unsigned int", size: 32, align: 32, encoding: DW_ATE_unsigned)
 !32 = !{!33}
-!33 = !MDSubrange(count: 2)
-!34 = !MDLocalVariable(tag: DW_TAG_auto_variable, name: "addr", line: 96, scope: !24, file: !10, type: !35)
-!35 = !MDDerivedType(tag: DW_TAG_pointer_type, size: 32, align: 32, file: !39, scope: !3, baseType: !11)
-!36 = !MDFile(filename: "stdint.h", directory: "/usr/4.2.1/include")
-!37 = !MDFile(filename: "types.h", directory: "/usr/include/ppc")
-!38 = !MDFile(filename: "OSByteOrder.h", directory: "/usr/include/libkern/ppc")
-!39 = !MDFile(filename: "G.c", directory: "/tmp")
+!33 = !DISubrange(count: 2)
+!34 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "addr", line: 96, scope: !24, file: !10, type: !35)
+!35 = !DIDerivedType(tag: DW_TAG_pointer_type, size: 32, align: 32, file: !39, scope: !3, baseType: !11)
+!36 = !DIFile(filename: "stdint.h", directory: "/usr/4.2.1/include")
+!37 = !DIFile(filename: "types.h", directory: "/usr/include/ppc")
+!38 = !DIFile(filename: "OSByteOrder.h", directory: "/usr/include/libkern/ppc")
+!39 = !DIFile(filename: "G.c", directory: "/tmp")
 !40 = !{!2, !9, !16}
 !41 = !{i32 1, !"Debug Info Version", i32 3}
diff --git a/test/DebugInfo/2010-05-10-MultipleCU.ll b/test/DebugInfo/2010-05-10-MultipleCU.ll
index 3430e86b8d1..6513165ac24 100644
--- a/test/DebugInfo/2010-05-10-MultipleCU.ll
+++ b/test/DebugInfo/2010-05-10-MultipleCU.ll
@@ -22,23 +22,23 @@ return:
 !16 = !{!2}
 !17 = !{!10}
 
-!0 = !MDLocation(line: 3, scope: !1)
-!1 = distinct !MDLexicalBlock(line: 2, column: 0, file: !18, scope: !2)
-!2 = !MDSubprogram(name: "foo", linkageName: "foo", line: 2, isLocal: false, isDefinition: true, virtualIndex: 6, isOptimized: false, file: !18, scope: !3, type: !5, function: i32 ()* @foo)
-!3 = !MDFile(filename: "a.c", directory: "/tmp/")
-!4 = !MDCompileUnit(language: DW_LANG_C89, producer: "4.2.1 (Based on Apple Inc. build 5658) (LLVM build)", isOptimized: false, emissionKind: 0, file: !18, enums: !19, retainedTypes: !19, subprograms: !16)
-!5 = !MDSubroutineType(types: !6)
+!0 = !DILocation(line: 3, scope: !1)
+!1 = distinct !DILexicalBlock(line: 2, column: 0, file: !18, scope: !2)
+!2 = !DISubprogram(name: "foo", linkageName: "foo", line: 2, isLocal: false, isDefinition: true, virtualIndex: 6, isOptimized: false, file: !18, scope: !3, type: !5, function: i32 ()* @foo)
+!3 = !DIFile(filename: "a.c", directory: "/tmp/")
+!4 = !DICompileUnit(language: DW_LANG_C89, producer: "4.2.1 (Based on Apple Inc. build 5658) (LLVM build)", isOptimized: false, emissionKind: 0, file: !18, enums: !19, retainedTypes: !19, subprograms: !16)
+!5 = !DISubroutineType(types: !6)
 !6 = !{!7}
-!7 = !MDBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
-!8 = !MDLocation(line: 3, scope: !9)
-!9 = distinct !MDLexicalBlock(line: 2, column: 0, file: !20, scope: !10)
-!10 = !MDSubprogram(name: "bar", linkageName: "bar", line: 2, isLocal: false, isDefinition: true, virtualIndex: 6, isOptimized: false, file: !20, scope: !11, type: !13, function: i32 ()* @bar)
-!11 = !MDFile(filename: "b.c", directory: "/tmp/")
-!12 = !MDCompileUnit(language: DW_LANG_C89, producer: "4.2.1 (Based on Apple Inc. build 5658) (LLVM build)", isOptimized: false, emissionKind: 0, file: !20, enums: !19, retainedTypes: !19, subprograms: !17)
-!13 = !MDSubroutineType(types: !14)
+!7 = !DIBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
+!8 = !DILocation(line: 3, scope: !9)
+!9 = distinct !DILexicalBlock(line: 2, column: 0, file: !20, scope: !10)
+!10 = !DISubprogram(name: "bar", linkageName: "bar", line: 2, isLocal: false, isDefinition: true, virtualIndex: 6, isOptimized: false, file: !20, scope: !11, type: !13, function: i32 ()* @bar)
+!11 = !DIFile(filename: "b.c", directory: "/tmp/")
+!12 = !DICompileUnit(language: DW_LANG_C89, producer: "4.2.1 (Based on Apple Inc. build 5658) (LLVM build)", isOptimized: false, emissionKind: 0, file: !20, enums: !19, retainedTypes: !19, subprograms: !17)
+!13 = !DISubroutineType(types: !14)
 !14 = !{!15}
-!15 = !MDBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
-!18 = !MDFile(filename: "a.c", directory: "/tmp/")
+!15 = !DIBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
+!18 = !DIFile(filename: "a.c", directory: "/tmp/")
 !19 = !{}
-!20 = !MDFile(filename: "b.c", directory: "/tmp/")
+!20 = !DIFile(filename: "b.c", directory: "/tmp/")
 !21 = !{i32 1, !"Debug Info Version", i32 3}
diff --git a/test/DebugInfo/2010-06-29-InlinedFnLocalVar.ll b/test/DebugInfo/2010-06-29-InlinedFnLocalVar.ll
index ae05e478745..0d7dae9c8c7 100644
--- a/test/DebugInfo/2010-06-29-InlinedFnLocalVar.ll
+++ b/test/DebugInfo/2010-06-29-InlinedFnLocalVar.ll
@@ -3,7 +3,7 @@
 
 ; CHECK: section_info
 ; CHECK:	DW_TAG_structure_type
-; CHECK-NEXT:	info_string
+; CHECK-NEXT:	DW_AT_name
 
 
 @i = common global i32 0                          ; <i32*> [#uses=2]
@@ -15,8 +15,8 @@ declare void @llvm.dbg.value(metadata, i64, metadata, metadata) nounwind readnon
 define i32 @bar() nounwind ssp {
 entry:
   %0 = load i32, i32* @i, align 4, !dbg !17            ; <i32> [#uses=2]
-  tail call void @llvm.dbg.value(metadata i32 %0, i64 0, metadata !59, metadata !MDExpression()), !dbg !19
-  tail call void @llvm.dbg.declare(metadata !29, metadata !60, metadata !MDExpression()), !dbg !21
+  tail call void @llvm.dbg.value(metadata i32 %0, i64 0, metadata !59, metadata !DIExpression()), !dbg !19
+  tail call void @llvm.dbg.declare(metadata !29, metadata !60, metadata !DIExpression()), !dbg !21
   %1 = mul nsw i32 %0, %0, !dbg !22               ; <i32> [#uses=2]
   store i32 %1, i32* @i, align 4, !dbg !17
   ret i32 %1, !dbg !23
@@ -25,37 +25,37 @@ entry:
 !llvm.dbg.cu = !{!2}
 !llvm.module.flags = !{!28}
 
-!0 = !MDSubprogram(name: "foo", line: 9, isLocal: true, isDefinition: true, virtualIndex: 6, isOptimized: true, scopeLine: 9, file: !27, scope: !1, type: !3, variables: !24)
-!1 = !MDFile(filename: "bar.c", directory: "/tmp/")
-!2 = !MDCompileUnit(language: DW_LANG_C89, producer: "4.2.1 (Based on Apple Inc. build 5658) (LLVM build)", isOptimized: true, emissionKind: 0, file: !27, enums: !20, retainedTypes: !20, subprograms: !25, globals: !26, imports:  !20)
-!3 = !MDSubroutineType(types: !4)
+!0 = !DISubprogram(name: "foo", line: 9, isLocal: true, isDefinition: true, virtualIndex: 6, isOptimized: true, scopeLine: 9, file: !27, scope: !1, type: !3, variables: !24)
+!1 = !DIFile(filename: "bar.c", directory: "/tmp/")
+!2 = !DICompileUnit(language: DW_LANG_C89, producer: "4.2.1 (Based on Apple Inc. build 5658) (LLVM build)", isOptimized: true, emissionKind: 0, file: !27, enums: !20, retainedTypes: !20, subprograms: !25, globals: !26, imports:  !20)
+!3 = !DISubroutineType(types: !4)
 !4 = !{!5, !5}
-!5 = !MDBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
-!6 = !MDSubprogram(name: "bar", linkageName: "bar", line: 14, isLocal: false, isDefinition: true, virtualIndex: 6, isOptimized: true, file: !27, scope: !1, type: !7, function: i32 ()* @bar)
-!7 = !MDSubroutineType(types: !8)
+!5 = !DIBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
+!6 = !DISubprogram(name: "bar", linkageName: "bar", line: 14, isLocal: false, isDefinition: true, virtualIndex: 6, isOptimized: true, file: !27, scope: !1, type: !7, function: i32 ()* @bar)
+!7 = !DISubroutineType(types: !8)
 !8 = !{!5}
-!9 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "j", line: 9, arg: 0, scope: !0, file: !1, type: !5)
-!10 = !MDLocalVariable(tag: DW_TAG_auto_variable, name: "xyz", line: 10, scope: !11, file: !1, type: !12)
+!9 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "j", line: 9, arg: 1, scope: !0, file: !1, type: !5)
+!10 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "xyz", line: 10, scope: !11, file: !1, type: !12)
 
-!59 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "j", line: 9, arg: 0, scope: !0, file: !1, type: !5)
-!60 = !MDLocalVariable(tag: DW_TAG_auto_variable, name: "xyz", line: 10, scope: !11, file: !1, type: !12)
+!59 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "j", line: 9, arg: 1, scope: !0, file: !1, type: !5)
+!60 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "xyz", line: 10, scope: !11, file: !1, type: !12)
 
-!11 = distinct !MDLexicalBlock(line: 9, column: 0, file: !1, scope: !0)
-!12 = !MDCompositeType(tag: DW_TAG_structure_type, name: "X", line: 10, size: 64, align: 32, file: !27, scope: !0, elements: !13)
+!11 = distinct !DILexicalBlock(line: 9, column: 0, file: !1, scope: !0)
+!12 = !DICompositeType(tag: DW_TAG_structure_type, name: "X", line: 10, size: 64, align: 32, file: !27, scope: !0, elements: !13)
 !13 = !{!14, !15}
-!14 = !MDDerivedType(tag: DW_TAG_member, name: "a", line: 10, size: 32, align: 32, file: !27, scope: !12, baseType: !5)
-!15 = !MDDerivedType(tag: DW_TAG_member, name: "b", line: 10, size: 32, align: 32, offset: 32, file: !27, scope: !12, baseType: !5)
-!16 = !MDGlobalVariable(name: "i", line: 5, isLocal: false, isDefinition: true, scope: !1, file: !1, type: !5, variable: i32* @i)
-!17 = !MDLocation(line: 15, scope: !18)
-!18 = distinct !MDLexicalBlock(line: 14, column: 0, file: !1, scope: !6)
-!19 = !MDLocation(line: 9, scope: !0, inlinedAt: !17)
+!14 = !DIDerivedType(tag: DW_TAG_member, name: "a", line: 10, size: 32, align: 32, file: !27, scope: !12, baseType: !5)
+!15 = !DIDerivedType(tag: DW_TAG_member, name: "b", line: 10, size: 32, align: 32, offset: 32, file: !27, scope: !12, baseType: !5)
+!16 = !DIGlobalVariable(name: "i", line: 5, isLocal: false, isDefinition: true, scope: !1, file: !1, type: !5, variable: i32* @i)
+!17 = !DILocation(line: 15, scope: !18)
+!18 = distinct !DILexicalBlock(line: 14, column: 0, file: !1, scope: !6)
+!19 = !DILocation(line: 9, scope: !0, inlinedAt: !17)
 !20 = !{}
-!21 = !MDLocation(line: 9, scope: !11, inlinedAt: !17)
-!22 = !MDLocation(line: 11, scope: !11, inlinedAt: !17)
-!23 = !MDLocation(line: 16, scope: !18)
+!21 = !DILocation(line: 9, scope: !11, inlinedAt: !17)
+!22 = !DILocation(line: 11, scope: !11, inlinedAt: !17)
+!23 = !DILocation(line: 16, scope: !18)
 !24 = !{!9, !10}
 !25 = !{!0, !6}
 !26 = !{!16}
-!27 = !MDFile(filename: "bar.c", directory: "/tmp/")
+!27 = !DIFile(filename: "bar.c", directory: "/tmp/")
 !28 = !{i32 1, !"Debug Info Version", i32 3}
 !29 = !{null}
diff --git a/test/DebugInfo/2010-07-19-Crash.ll b/test/DebugInfo/2010-07-19-Crash.ll
index 67af58d422f..fbfe9f80189 100644
--- a/test/DebugInfo/2010-07-19-Crash.ll
+++ b/test/DebugInfo/2010-07-19-Crash.ll
@@ -12,19 +12,19 @@ entry:
 !llvm.dbg.sp = !{!0, !6, !11}
 !llvm.dbg.lv.foo = !{!7}
 
-!0 = !MDSubprogram(name: "bar", linkageName: "bar", line: 3, isLocal: false, isDefinition: true, virtualIndex: 6, isOptimized: true, file: !12, scope: !1, type: !3, function: i32 ()* @bar)
-!1 = !MDFile(filename: "one.c", directory: "/private/tmp")
-!2 = !MDCompileUnit(language: DW_LANG_C99, producer: "clang 2.8", isOptimized: true, emissionKind: 0, file: !12, enums: !14, retainedTypes: !14, subprograms: !13)
-!3 = !MDSubroutineType(types: !4)
+!0 = !DISubprogram(name: "bar", linkageName: "bar", line: 3, isLocal: false, isDefinition: true, virtualIndex: 6, isOptimized: true, file: !12, scope: !1, type: !3, function: i32 ()* @bar)
+!1 = !DIFile(filename: "one.c", directory: "/private/tmp")
+!2 = !DICompileUnit(language: DW_LANG_C99, producer: "clang 2.8", isOptimized: true, emissionKind: 0, file: !12, enums: !14, retainedTypes: !14, subprograms: !13)
+!3 = !DISubroutineType(types: !4)
 !4 = !{!5}
-!5 = !MDBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
-!6 = !MDSubprogram(name: "foo", linkageName: "foo", line: 7, isLocal: true, isDefinition: true, virtualIndex: 6, isOptimized: true, file: !12, scope: !1, type: !3)
-!7 = !MDLocalVariable(tag: DW_TAG_auto_variable, name: "one", line: 8, scope: !8, file: !1, type: !5)
-!8 = distinct !MDLexicalBlock(line: 7, column: 18, file: !12, scope: !6)
-!9 = !MDLocation(line: 4, column: 3, scope: !10)
-!10 = distinct !MDLexicalBlock(line: 3, column: 11, file: !12, scope: !0)
-!11 = !MDSubprogram(name: "foo", linkageName: "foo", line: 7, isLocal: true, isDefinition: false, virtualIndex: 6, isOptimized: true, file: !12, scope: !1, type: !3)
-!12 = !MDFile(filename: "one.c", directory: "/private/tmp")
+!5 = !DIBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
+!6 = !DISubprogram(name: "foo", linkageName: "foo", line: 7, isLocal: true, isDefinition: true, virtualIndex: 6, isOptimized: true, file: !12, scope: !1, type: !3)
+!7 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "one", line: 8, scope: !8, file: !1, type: !5)
+!8 = distinct !DILexicalBlock(line: 7, column: 18, file: !12, scope: !6)
+!9 = !DILocation(line: 4, column: 3, scope: !10)
+!10 = distinct !DILexicalBlock(line: 3, column: 11, file: !12, scope: !0)
+!11 = !DISubprogram(name: "foo", linkageName: "foo", line: 7, isLocal: true, isDefinition: false, virtualIndex: 6, isOptimized: true, file: !12, scope: !1, type: !3)
+!12 = !DIFile(filename: "one.c", directory: "/private/tmp")
 !13 = !{!0}
 !14 = !{}
 !15 = !{i32 1, !"Debug Info Version", i32 3}
diff --git a/test/DebugInfo/2010-10-01-crash.ll b/test/DebugInfo/2010-10-01-crash.ll
index 78d6ab9409f..c1d161ce586 100644
--- a/test/DebugInfo/2010-10-01-crash.ll
+++ b/test/DebugInfo/2010-10-01-crash.ll
@@ -1,10 +1,8 @@
 ; RUN: llc -O0 %s -o /dev/null
-; XFAIL: hexagon
-; PR 8235
 
 define void @CGRectStandardize(i32* sret %agg.result, i32* byval %rect) nounwind ssp {
 entry:
-  call void @llvm.dbg.declare(metadata i32* %rect, metadata !23, metadata !MDExpression()), !dbg !24
+  call void @llvm.dbg.declare(metadata i32* %rect, metadata !23, metadata !DIExpression()), !dbg !24
   ret void
 }
 
@@ -15,12 +13,12 @@ declare void @llvm.memcpy.p0i8.p0i8.i32(i8* nocapture, i8* nocapture, i32, i32,
 
 !llvm.dbg.cu = !{!2}
 !llvm.module.flags = !{!27}
-!0 = !MDSubprogram(name: "CGRectStandardize", linkageName: "CGRectStandardize", line: 54, isLocal: false, isDefinition: true, virtualIndex: 6, isOptimized: false, file: !1, scope: null, function: void (i32*, i32*)* @CGRectStandardize)
-!1 = !MDFile(filename: "GSFusedSilica.m", directory: "/Volumes/Data/Users/sabre/Desktop")
-!2 = !MDCompileUnit(language: DW_LANG_ObjC, producer: "clang version 2.9 (trunk 115292)", isOptimized: true, runtimeVersion: 1, emissionKind: 0, file: !25, enums: !26, retainedTypes: !26)
-!5 = !MDDerivedType(tag: DW_TAG_typedef, name: "CGRect", line: 49, file: !25, baseType: null)
-!23 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "rect", line: 53, arg: 0, scope: !0, file: !1, type: !5)
-!24 = !MDLocation(line: 53, column: 33, scope: !0)
-!25 = !MDFile(filename: "GSFusedSilica.m", directory: "/Volumes/Data/Users/sabre/Desktop")
+!0 = !DISubprogram(name: "CGRectStandardize", linkageName: "CGRectStandardize", line: 54, isLocal: false, isDefinition: true, virtualIndex: 6, isOptimized: false, file: !1, scope: null, function: void (i32*, i32*)* @CGRectStandardize)
+!1 = !DIFile(filename: "GSFusedSilica.m", directory: "/Volumes/Data/Users/sabre/Desktop")
+!2 = !DICompileUnit(language: DW_LANG_ObjC, producer: "clang version 2.9 (trunk 115292)", isOptimized: true, runtimeVersion: 1, emissionKind: 0, file: !25, enums: !26, retainedTypes: !26)
+!5 = !DIDerivedType(tag: DW_TAG_typedef, name: "CGRect", line: 49, file: !25, baseType: null)
+!23 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "rect", line: 53, arg: 2, scope: !0, file: !1, type: !5)
+!24 = !DILocation(line: 53, column: 33, scope: !0)
+!25 = !DIFile(filename: "GSFusedSilica.m", directory: "/Volumes/Data/Users/sabre/Desktop")
 !26 = !{}
 !27 = !{i32 1, !"Debug Info Version", i32 3}
diff --git a/test/DebugInfo/AArch64/big-endian.ll b/test/DebugInfo/AArch64/big-endian.ll
index a5916f232c9..45f49da6307 100644
--- a/test/DebugInfo/AArch64/big-endian.ll
+++ b/test/DebugInfo/AArch64/big-endian.ll
@@ -9,14 +9,14 @@ target triple = "aarch64_be--none-eabi"
 !llvm.module.flags = !{!8, !9}
 !llvm.ident = !{!10}
 
-!0 = !MDCompileUnit(language: DW_LANG_C99, producer: "clang version 3.6.0 ", isOptimized: true, emissionKind: 1, file: !1, enums: !2, retainedTypes: !2, subprograms: !2, globals: !3, imports: !2)
-!1 = !MDFile(filename: "-", directory: "/work/validation")
+!0 = !DICompileUnit(language: DW_LANG_C99, producer: "clang version 3.6.0 ", isOptimized: true, emissionKind: 1, file: !1, enums: !2, retainedTypes: !2, subprograms: !2, globals: !3, imports: !2)
+!1 = !DIFile(filename: "-", directory: "/work/validation")
 !2 = !{}
 !3 = !{!4}
-!4 = !MDGlobalVariable(name: "a", line: 1, isLocal: false, isDefinition: true, scope: null, file: !5, type: !7, variable: i32* @a)
-!5 = !MDFile(filename: "<stdin>", directory: "/work/validation")
+!4 = !DIGlobalVariable(name: "a", line: 1, isLocal: false, isDefinition: true, scope: null, file: !5, type: !7, variable: i32* @a)
+!5 = !DIFile(filename: "<stdin>", directory: "/work/validation")
 !6 = !{!"<stdin>", !"/work/validation"}
-!7 = !MDBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
+!7 = !DIBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
 !8 = !{i32 2, !"Dwarf Version", i32 4}
 !9 = !{i32 2, !"Debug Info Version", i32 3}
 !10 = !{!"clang version 3.6.0 "}
diff --git a/test/DebugInfo/AArch64/bitfields.ll b/test/DebugInfo/AArch64/bitfields.ll
new file mode 100644
index 00000000000..5f0caab286b
--- /dev/null
+++ b/test/DebugInfo/AArch64/bitfields.ll
@@ -0,0 +1,73 @@
+; RUN: llc -mtriple aarch64_be-gnu-linux -O0 -filetype=obj -o %t_be.o %s
+; RUN: llvm-dwarfdump -debug-dump=info %t_be.o | FileCheck %s
+
+; Produced at -O0 from:
+; struct bitfield {
+;   int a : 2;
+;   int b : 32;
+;   int c : 1;
+;   int d : 28;
+; };
+; struct bitfield b;
+
+; Note that DWARF 2 counts bit offsets backwards from the high end of
+; the storage unit to the high end of the bit field.
+
+; CHECK: DW_TAG_member
+; CHECK-NEXT: DW_AT_name{{.*}}"a"
+; CHECK-NOT: DW_TAG_member
+; CHECK:      DW_AT_byte_size  {{.*}} (0x04)
+; CHECK-NEXT: DW_AT_bit_size   {{.*}} (0x02)
+; CHECK-NEXT: DW_AT_bit_offset {{.*}} (0x00)
+; CHECK-NEXT: DW_AT_data_member_location {{.*}} 00
+
+; CHECK: DW_TAG_member
+; CHECK-NEXT: DW_AT_name{{.*}}"b"
+; CHECK-NOT: DW_TAG_member
+; CHECK:      DW_AT_data_member_location {{.*}} 04
+
+; CHECK: DW_TAG_member
+; CHECK-NEXT: DW_AT_name{{.*}}"c"
+; CHECK-NOT: DW_TAG_member
+; CHECK:      DW_AT_byte_size  {{.*}} (0x04)
+; CHECK-NEXT: DW_AT_bit_size   {{.*}} (0x01)
+; CHECK-NEXT: DW_AT_bit_offset {{.*}} (0x00)
+; CHECK-NEXT: DW_AT_data_member_location {{.*}} 08
+
+; CHECK: DW_TAG_member
+; CHECK-NEXT: DW_AT_name{{.*}}"d"
+; CHECK-NOT: DW_TAG_member
+; CHECK:      DW_AT_byte_size  {{.*}} (0x04)
+; CHECK-NEXT: DW_AT_bit_size   {{.*}} (0x1c)
+; CHECK-NEXT: DW_AT_bit_offset {{.*}} (0x01)
+; CHECK-NEXT: DW_AT_data_member_location {{.*}} 08
+
+; ModuleID = 'bitfields.c'
+target datalayout = "E-m:e-i64:64-i128:128-n32:64-S128"
+target triple = "aarch64_be--linux-gnu"
+
+%struct.bitfield = type <{ i8, [3 x i8], i64 }>
+
+@b = common global %struct.bitfield zeroinitializer, align 4
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!13, !14, !15}
+!llvm.ident = !{!16}
+
+!0 = !DICompileUnit(language: DW_LANG_C99, file: !1, producer: "clang version 3.7.0 (trunk 240548) (llvm/trunk 240554)", isOptimized: false, runtimeVersion: 0, emissionKind: 1, enums: !2, retainedTypes: !2, subprograms: !2, globals: !3, imports: !2)
+!1 = !DIFile(filename: "bitfields.c", directory: "/")
+!2 = !{}
+!3 = !{!4}
+!4 = !DIGlobalVariable(name: "b", scope: !0, file: !5, line: 8, type: !6, isLocal: false, isDefinition: true, variable: %struct.bitfield* @b)
+!5 = !DIFile(filename: "bitfields.c", directory: "/")
+!6 = !DICompositeType(tag: DW_TAG_structure_type, name: "bitfield", file: !5, line: 1, size: 96, align: 32, elements: !7)
+!7 = !{!8, !10, !11, !12}
+!8 = !DIDerivedType(tag: DW_TAG_member, name: "a", scope: !6, file: !5, line: 2, baseType: !9, size: 2, align: 32)
+!9 = !DIBasicType(name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
+!10 = !DIDerivedType(tag: DW_TAG_member, name: "b", scope: !6, file: !5, line: 3, baseType: !9, size: 32, align: 32, offset: 32)
+!11 = !DIDerivedType(tag: DW_TAG_member, name: "c", scope: !6, file: !5, line: 4, baseType: !9, size: 1, align: 32, offset: 64)
+!12 = !DIDerivedType(tag: DW_TAG_member, name: "d", scope: !6, file: !5, line: 5, baseType: !9, size: 28, align: 32, offset: 65)
+!13 = !{i32 2, !"Dwarf Version", i32 2}
+!14 = !{i32 2, !"Debug Info Version", i32 3}
+!15 = !{i32 1, !"PIC Level", i32 2}
+!16 = !{!"clang version 3.7.0 (trunk 240548) (llvm/trunk 240554)"}
diff --git a/test/DebugInfo/AArch64/cfi-eof-prologue.ll b/test/DebugInfo/AArch64/cfi-eof-prologue.ll
index ab4ef4e5ca3..756443a951d 100644
--- a/test/DebugInfo/AArch64/cfi-eof-prologue.ll
+++ b/test/DebugInfo/AArch64/cfi-eof-prologue.ll
@@ -61,52 +61,52 @@ attributes #3 = { nounwind }
 !llvm.module.flags = !{!35, !36}
 !llvm.ident = !{!37}
 
-!0 = !MDCompileUnit(language: DW_LANG_C_plus_plus, producer: "clang version 3.6.0 (trunk 224279) (llvm/trunk 224283)", isOptimized: true, emissionKind: 1, file: !1, enums: !2, retainedTypes: !3, subprograms: !27, globals: !2, imports: !2)
-!1 = !MDFile(filename: "<stdin>", directory: "")
+!0 = !DICompileUnit(language: DW_LANG_C_plus_plus, producer: "clang version 3.6.0 (trunk 224279) (llvm/trunk 224283)", isOptimized: true, emissionKind: 1, file: !1, enums: !2, retainedTypes: !3, subprograms: !27, globals: !2, imports: !2)
+!1 = !DIFile(filename: "<stdin>", directory: "")
 !2 = !{}
 !3 = !{!4, !13}
-!4 = !MDCompositeType(tag: DW_TAG_structure_type, name: "B", line: 5, size: 64, align: 64, file: !5, elements: !6, vtableHolder: !"_ZTS1A", identifier: "_ZTS1B")
-!5 = !MDFile(filename: "test1.cpp", directory: "")
+!4 = !DICompositeType(tag: DW_TAG_structure_type, name: "B", line: 5, size: 64, align: 64, file: !5, elements: !6, vtableHolder: !"_ZTS1A", identifier: "_ZTS1B")
+!5 = !DIFile(filename: "test1.cpp", directory: "")
 !6 = !{!7, !8, !12}
-!7 = !MDDerivedType(tag: DW_TAG_inheritance, scope: !"_ZTS1B", baseType: !"_ZTS1A")
-!8 = !MDSubprogram(name: "B", line: 6, isLocal: false, isDefinition: false, flags: DIFlagPrototyped, isOptimized: true, scopeLine: 6, file: !5, scope: !"_ZTS1B", type: !9)
-!9 = !MDSubroutineType(types: !10)
+!7 = !DIDerivedType(tag: DW_TAG_inheritance, scope: !"_ZTS1B", baseType: !"_ZTS1A")
+!8 = !DISubprogram(name: "B", line: 6, isLocal: false, isDefinition: false, flags: DIFlagPrototyped, isOptimized: true, scopeLine: 6, file: !5, scope: !"_ZTS1B", type: !9)
+!9 = !DISubroutineType(types: !10)
 !10 = !{null, !11}
-!11 = !MDDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, flags: DIFlagArtificial | DIFlagObjectPointer, baseType: !"_ZTS1B")
-!12 = !MDSubprogram(name: "~B", line: 7, isLocal: false, isDefinition: false, virtuality: DW_VIRTUALITY_virtual, flags: DIFlagPrototyped, isOptimized: true, scopeLine: 7, file: !5, scope: !"_ZTS1B", type: !9, containingType: !"_ZTS1B")
-!13 = !MDCompositeType(tag: DW_TAG_structure_type, name: "A", line: 1, size: 64, align: 64, file: !5, elements: !14, vtableHolder: !"_ZTS1A", identifier: "_ZTS1A")
+!11 = !DIDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, flags: DIFlagArtificial | DIFlagObjectPointer, baseType: !"_ZTS1B")
+!12 = !DISubprogram(name: "~B", line: 7, isLocal: false, isDefinition: false, virtuality: DW_VIRTUALITY_virtual, flags: DIFlagPrototyped, isOptimized: true, scopeLine: 7, file: !5, scope: !"_ZTS1B", type: !9, containingType: !"_ZTS1B")
+!13 = !DICompositeType(tag: DW_TAG_structure_type, name: "A", line: 1, size: 64, align: 64, file: !5, elements: !14, vtableHolder: !"_ZTS1A", identifier: "_ZTS1A")
 !14 = !{!15, !22, !26}
-!15 = !MDDerivedType(tag: DW_TAG_member, name: "_vptr$A", size: 64, flags: DIFlagArtificial, file: !5, scope: !16, baseType: !17)
-!16 = !MDFile(filename: "test1.cpp", directory: "")
-!17 = !MDDerivedType(tag: DW_TAG_pointer_type, size: 64, baseType: !18)
-!18 = !MDDerivedType(tag: DW_TAG_pointer_type, name: "__vtbl_ptr_type", size: 64, baseType: !19)
-!19 = !MDSubroutineType(types: !20)
+!15 = !DIDerivedType(tag: DW_TAG_member, name: "_vptr$A", size: 64, flags: DIFlagArtificial, file: !5, scope: !16, baseType: !17)
+!16 = !DIFile(filename: "test1.cpp", directory: "")
+!17 = !DIDerivedType(tag: DW_TAG_pointer_type, size: 64, baseType: !18)
+!18 = !DIDerivedType(tag: DW_TAG_pointer_type, name: "__vtbl_ptr_type", size: 64, baseType: !19)
+!19 = !DISubroutineType(types: !20)
 !20 = !{!21}
-!21 = !MDBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
-!22 = !MDSubprogram(name: "A", line: 2, isLocal: false, isDefinition: false, flags: DIFlagPrototyped, isOptimized: true, scopeLine: 2, file: !5, scope: !"_ZTS1A", type: !23)
-!23 = !MDSubroutineType(types: !24)
+!21 = !DIBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
+!22 = !DISubprogram(name: "A", line: 2, isLocal: false, isDefinition: false, flags: DIFlagPrototyped, isOptimized: true, scopeLine: 2, file: !5, scope: !"_ZTS1A", type: !23)
+!23 = !DISubroutineType(types: !24)
 !24 = !{null, !25}
-!25 = !MDDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, flags: DIFlagArtificial | DIFlagObjectPointer, baseType: !"_ZTS1A")
-!26 = !MDSubprogram(name: "~A", line: 3, isLocal: false, isDefinition: false, virtuality: DW_VIRTUALITY_virtual, flags: DIFlagPrototyped, isOptimized: true, scopeLine: 3, file: !5, scope: !"_ZTS1A", type: !23, containingType: !"_ZTS1A")
+!25 = !DIDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, flags: DIFlagArtificial | DIFlagObjectPointer, baseType: !"_ZTS1A")
+!26 = !DISubprogram(name: "~A", line: 3, isLocal: false, isDefinition: false, virtuality: DW_VIRTUALITY_virtual, flags: DIFlagPrototyped, isOptimized: true, scopeLine: 3, file: !5, scope: !"_ZTS1A", type: !23, containingType: !"_ZTS1A")
 !27 = !{!28, !32}
-!28 = !MDSubprogram(name: "B", linkageName: "_ZN1BC2Ev", line: 9, isLocal: false, isDefinition: true, flags: DIFlagPrototyped, isOptimized: true, scopeLine: 9, file: !5, scope: !"_ZTS1B", type: !9, function: %struct.B* (%struct.B*)* @_ZN1BC2Ev, declaration: !8, variables: !29)
+!28 = !DISubprogram(name: "B", linkageName: "_ZN1BC2Ev", line: 9, isLocal: false, isDefinition: true, flags: DIFlagPrototyped, isOptimized: true, scopeLine: 9, file: !5, scope: !"_ZTS1B", type: !9, function: %struct.B* (%struct.B*)* @_ZN1BC2Ev, declaration: !8, variables: !29)
 !29 = !{!30}
-!30 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "this", arg: 1, flags: DIFlagArtificial | DIFlagObjectPointer, scope: !28, type: !31)
-!31 = !MDDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, baseType: !"_ZTS1B")
-!32 = !MDSubprogram(name: "B", linkageName: "_ZN1BC1Ev", line: 9, isLocal: false, isDefinition: true, flags: DIFlagPrototyped, isOptimized: true, scopeLine: 9, file: !5, scope: !"_ZTS1B", type: !9, function: %struct.B* (%struct.B*)* @_ZN1BC1Ev, declaration: !8, variables: !33)
+!30 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "this", arg: 1, flags: DIFlagArtificial | DIFlagObjectPointer, scope: !28, type: !31)
+!31 = !DIDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, baseType: !"_ZTS1B")
+!32 = !DISubprogram(name: "B", linkageName: "_ZN1BC1Ev", line: 9, isLocal: false, isDefinition: true, flags: DIFlagPrototyped, isOptimized: true, scopeLine: 9, file: !5, scope: !"_ZTS1B", type: !9, function: %struct.B* (%struct.B*)* @_ZN1BC1Ev, declaration: !8, variables: !33)
 !33 = !{!34}
-!34 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "this", arg: 1, flags: DIFlagArtificial | DIFlagObjectPointer, scope: !32, type: !31)
+!34 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "this", arg: 1, flags: DIFlagArtificial | DIFlagObjectPointer, scope: !32, type: !31)
 !35 = !{i32 2, !"Dwarf Version", i32 4}
 !36 = !{i32 2, !"Debug Info Version", i32 3}
 !37 = !{!"clang version 3.6.0 (trunk 224279) (llvm/trunk 224283)"}
-!38 = !MDExpression()
-!39 = !MDLocation(line: 0, scope: !28)
-!40 = !MDLocation(line: 9, scope: !28)
+!38 = !DIExpression()
+!39 = !DILocation(line: 0, scope: !28)
+!40 = !DILocation(line: 9, scope: !28)
 !41 = !{!42, !42, i64 0}
 !42 = !{!"vtable pointer", !43, i64 0}
 !43 = !{!"Simple C/C++ TBAA"}
-!44 = !MDLocation(line: 0, scope: !32)
-!45 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "this", arg: 1, flags: DIFlagArtificial | DIFlagObjectPointer, scope: !28, type: !31)
-!46 = !MDLocation(line: 9, scope: !32)
-!47 = !MDLocation(line: 0, scope: !28, inlinedAt: !46)
-!48 = !MDLocation(line: 9, scope: !28, inlinedAt: !46)
+!44 = !DILocation(line: 0, scope: !32)
+!45 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "this", arg: 1, flags: DIFlagArtificial | DIFlagObjectPointer, scope: !28, type: !31)
+!46 = !DILocation(line: 9, scope: !32)
+!47 = !DILocation(line: 0, scope: !28, inlinedAt: !46)
+!48 = !DILocation(line: 9, scope: !28, inlinedAt: !46)
diff --git a/test/DebugInfo/AArch64/coalescing.ll b/test/DebugInfo/AArch64/coalescing.ll
index a935153617e..b1d56619712 100644
--- a/test/DebugInfo/AArch64/coalescing.ll
+++ b/test/DebugInfo/AArch64/coalescing.ll
@@ -44,22 +44,22 @@ attributes #3 = { nounwind optsize }
 !llvm.module.flags = !{!12, !13}
 !llvm.ident = !{!14}
 
-!0 = !MDCompileUnit(language: DW_LANG_C_plus_plus, producer: "clang version 3.6.0 (trunk 223149) (llvm/trunk 223115)", isOptimized: true, emissionKind: 1, file: !1, enums: !2, retainedTypes: !2, subprograms: !3, globals: !2, imports: !2)
-!1 = !MDFile(filename: "<stdin>", directory: "")
+!0 = !DICompileUnit(language: DW_LANG_C_plus_plus, producer: "clang version 3.6.0 (trunk 223149) (llvm/trunk 223115)", isOptimized: true, emissionKind: 1, file: !1, enums: !2, retainedTypes: !2, subprograms: !3, globals: !2, imports: !2)
+!1 = !DIFile(filename: "<stdin>", directory: "")
 !2 = !{}
 !3 = !{!4}
-!4 = !MDSubprogram(name: "start", linkageName: "_Z5startv", line: 2, isLocal: false, isDefinition: true, flags: DIFlagPrototyped, isOptimized: true, scopeLine: 3, file: !5, scope: !6, type: !7, function: void ()* @_Z5startv, variables: !9)
-!5 = !MDFile(filename: "test1.c", directory: "")
-!6 = !MDFile(filename: "test1.c", directory: "")
-!7 = !MDSubroutineType(types: !8)
+!4 = !DISubprogram(name: "start", linkageName: "_Z5startv", line: 2, isLocal: false, isDefinition: true, flags: DIFlagPrototyped, isOptimized: true, scopeLine: 3, file: !5, scope: !6, type: !7, function: void ()* @_Z5startv, variables: !9)
+!5 = !DIFile(filename: "test1.c", directory: "")
+!6 = !DIFile(filename: "test1.c", directory: "")
+!7 = !DISubroutineType(types: !8)
 !8 = !{null}
 !9 = !{!10}
-!10 = !MDLocalVariable(tag: DW_TAG_auto_variable, name: "size", line: 4, scope: !4, file: !6, type: !11)
-!11 = !MDBasicType(tag: DW_TAG_base_type, name: "unsigned int", size: 32, align: 32, encoding: DW_ATE_unsigned)
+!10 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "size", line: 4, scope: !4, file: !6, type: !11)
+!11 = !DIBasicType(tag: DW_TAG_base_type, name: "unsigned int", size: 32, align: 32, encoding: DW_ATE_unsigned)
 !12 = !{i32 2, !"Dwarf Version", i32 2}
 !13 = !{i32 2, !"Debug Info Version", i32 3}
 !14 = !{!"clang version 3.6.0 (trunk 223149) (llvm/trunk 223115)"}
-!15 = !MDLocation(line: 5, column: 3, scope: !4)
-!16 = !MDExpression()
-!17 = !MDLocation(line: 4, column: 12, scope: !4)
-!18 = !MDLocation(line: 8, column: 1, scope: !4)
+!15 = !DILocation(line: 5, column: 3, scope: !4)
+!16 = !DIExpression()
+!17 = !DILocation(line: 4, column: 12, scope: !4)
+!18 = !DILocation(line: 8, column: 1, scope: !4)
diff --git a/test/DebugInfo/AArch64/constant-dbgloc.ll b/test/DebugInfo/AArch64/constant-dbgloc.ll
new file mode 100644
index 00000000000..a71b869f536
--- /dev/null
+++ b/test/DebugInfo/AArch64/constant-dbgloc.ll
@@ -0,0 +1,37 @@
+; RUN: llc -filetype=asm %s -o - | FileCheck %s
+
+target datalayout = "e-m:e-p:32:32-i64:64-v128:64:128-a:0:32-n32-S64"
+target triple = "aarch64--linux-gnueabihf"
+
+; int
+; main(void)
+; {
+;     return -1;
+; }
+
+; CHECK: test.c:4:5
+; CHECK: movn
+
+; Function Attrs: nounwind
+define i32 @main() {
+entry:
+  %retval = alloca i32, align 4
+  store i32 0, i32* %retval
+  ret i32 -1, !dbg !11
+}
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!8, !9}
+
+!0 = !DICompileUnit(language: DW_LANG_C99, file: !1, producer: "", isOptimized: false, subprograms: !3)
+!1 = !DIFile(filename: "test.c", directory: "/home/user/clang/build")
+!2 = !{}
+!3 = !{!4}
+!4 = !DISubprogram(name: "main", scope: !1, file: !1, line: 2, type: !5, isLocal: false, isDefinition: true, scopeLine: 3, flags: DIFlagPrototyped, isOptimized: false, function: i32 ()* @main, variables: !2)
+!5 = !DISubroutineType(types: !6)
+!6 = !{!7}
+!7 = !DIBasicType(name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
+!8 = !{i32 2, !"Dwarf Version", i32 4}
+!9 = !{i32 2, !"Debug Info Version", i32 3}
+!10 = !{i32 1, !"wchar_size", i32 4}
+!11 = !DILocation(line: 4, column: 5, scope: !4)
diff --git a/test/DebugInfo/AArch64/dwarfdump.ll b/test/DebugInfo/AArch64/dwarfdump.ll
index 77d1b17a4ff..3e95a6fe314 100644
--- a/test/DebugInfo/AArch64/dwarfdump.ll
+++ b/test/DebugInfo/AArch64/dwarfdump.ll
@@ -27,14 +27,14 @@ attributes #0 = { nounwind }
 !llvm.dbg.cu = !{!0}
 !llvm.module.flags = !{!10}
 
-!0 = !MDCompileUnit(language: DW_LANG_C99, producer: "clang version 3.3 ", isOptimized: false, emissionKind: 0, file: !9, enums: !1, retainedTypes: !1, subprograms: !2, globals: !1, imports:  !1)
+!0 = !DICompileUnit(language: DW_LANG_C99, producer: "clang version 3.3 ", isOptimized: false, emissionKind: 0, file: !9, enums: !1, retainedTypes: !1, subprograms: !2, globals: !1, imports:  !1)
 !1 = !{}
 !2 = !{!3}
-!3 = !MDSubprogram(name: "main", line: 1, isLocal: false, isDefinition: true, virtualIndex: 6, isOptimized: false, scopeLine: 1, file: !9, scope: !4, type: !5, function: i32 ()* @main, variables: !1)
-!4 = !MDFile(filename: "tmp.c", directory: "/home/tim/llvm/build")
-!5 = !MDSubroutineType(types: !6)
+!3 = !DISubprogram(name: "main", line: 1, isLocal: false, isDefinition: true, virtualIndex: 6, isOptimized: false, scopeLine: 1, file: !9, scope: !4, type: !5, function: i32 ()* @main, variables: !1)
+!4 = !DIFile(filename: "tmp.c", directory: "/home/tim/llvm/build")
+!5 = !DISubroutineType(types: !6)
 !6 = !{!7}
-!7 = !MDBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
-!8 = !MDLocation(line: 2, scope: !3)
-!9 = !MDFile(filename: "tmp.c", directory: "/home/tim/llvm/build")
+!7 = !DIBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
+!8 = !DILocation(line: 2, scope: !3)
+!9 = !DIFile(filename: "tmp.c", directory: "/home/tim/llvm/build")
 !10 = !{i32 1, !"Debug Info Version", i32 3}
diff --git a/test/DebugInfo/AArch64/eh_frame.s b/test/DebugInfo/AArch64/eh_frame.s
index 12a58961d71..65367751223 100644
--- a/test/DebugInfo/AArch64/eh_frame.s
+++ b/test/DebugInfo/AArch64/eh_frame.s
@@ -17,7 +17,7 @@ foo:
 // Output is:
 
 // CHECK: Contents of section .eh_frame:
-// CHECK-NEXT: 0000 10000000 00000000 037a5200 017c1e01  .........zR..|..
+// CHECK-NEXT: 0000 10000000 00000000 017a5200 017c1e01  .........zR..|..
 // CHECK-NEXT: 0010 1b0c1f00 10000000 18000000 00000000  ................
 
 
diff --git a/test/DebugInfo/AArch64/eh_frame_personality.ll b/test/DebugInfo/AArch64/eh_frame_personality.ll
index 51d6bf80b95..ae5401b58fb 100644
--- a/test/DebugInfo/AArch64/eh_frame_personality.ll
+++ b/test/DebugInfo/AArch64/eh_frame_personality.ll
@@ -5,18 +5,18 @@ declare i32 @__gxx_personality_v0(...)
 
 declare void @bar()
 
-define i64 @foo(i64 %lhs, i64 %rhs) {
+define i64 @foo(i64 %lhs, i64 %rhs) personality i8* bitcast (i32 (...)* @__gxx_personality_v0 to i8*) {
   invoke void @bar() to label %end unwind label %clean
 end:
  ret i64 0
 
 clean:
-  %tst = landingpad { i8*, i32 } personality i8* bitcast (i32 (...)* @__gxx_personality_v0 to i8*) cleanup
+  %tst = landingpad { i8*, i32 } cleanup
   ret i64 42
 }
 
 ; CHECK: Contents of section .eh_frame:
-; CHECK: 0000 1c000000 00000000 037a504c 5200017c  .........zPLR..|
+; CHECK: 0000 1c000000 00000000 017a504c 5200017c  .........zPLR..|
 ; CHECK: 0010 1e0b0000 00000000 00000000 1b0c1f00  ................
 
 ; Don't really care about the rest:
diff --git a/test/DebugInfo/AArch64/frameindices.ll b/test/DebugInfo/AArch64/frameindices.ll
index 51424a0bdd4..02907742336 100644
--- a/test/DebugInfo/AArch64/frameindices.ll
+++ b/test/DebugInfo/AArch64/frameindices.ll
@@ -83,7 +83,7 @@ entry:
   ret void, !dbg !73
 }
 
-define void @_Z3f16v() #0 {
+define void @_Z3f16v() #0 personality i8* bitcast (i32 (...)* @__gxx_personality_v0 to i8*) {
 entry:
   %agg.tmp.i.i = alloca %struct.A, align 8
   %d = alloca %struct.B, align 1
@@ -127,7 +127,7 @@ invoke.cont:                                      ; preds = %call.i.i.noexc
   ret void, !dbg !94
 
 lpad:                                             ; preds = %call.i.i.noexc, %entry
-  %3 = landingpad { i8*, i32 } personality i8* bitcast (i32 (...)* @__gxx_personality_v0 to i8*)
+  %3 = landingpad { i8*, i32 }
           cleanup, !dbg !94
   call void @llvm.dbg.value(metadata %struct.B* %d, i64 0, metadata !39, metadata !79), !dbg !82
   %call2 = call %struct.B* @_ZN1BD1Ev(%struct.B* %d) #3, !dbg !94
@@ -160,98 +160,98 @@ attributes #5 = { builtin }
 !llvm.module.flags = !{!43, !44}
 !llvm.ident = !{!45}
 
-!0 = !MDCompileUnit(language: DW_LANG_C_plus_plus, producer: "clang version 3.7.0 ", isOptimized: true, emissionKind: 1, file: !1, enums: !2, retainedTypes: !3, subprograms: !24, globals: !40, imports: !2)
-!1 = !MDFile(filename: "<stdin>", directory: "")
+!0 = !DICompileUnit(language: DW_LANG_C_plus_plus, producer: "clang version 3.7.0 ", isOptimized: true, emissionKind: 1, file: !1, enums: !2, retainedTypes: !3, subprograms: !24, globals: !40, imports: !2)
+!1 = !DIFile(filename: "<stdin>", directory: "")
 !2 = !{}
 !3 = !{!4, !12, !14}
-!4 = !MDCompositeType(tag: DW_TAG_structure_type, name: "A", line: 2, size: 192, align: 64, file: !5, elements: !6, identifier: "_ZTS1A")
-!5 = !MDFile(filename: "test.cpp", directory: "")
+!4 = !DICompositeType(tag: DW_TAG_structure_type, name: "A", line: 2, size: 192, align: 64, file: !5, elements: !6, identifier: "_ZTS1A")
+!5 = !DIFile(filename: "test.cpp", directory: "")
 !6 = !{!7, !9, !11}
-!7 = !MDDerivedType(tag: DW_TAG_member, name: "x4", line: 3, size: 8, align: 8, file: !5, scope: !"_ZTS1A", baseType: !8)
-!8 = !MDBasicType(tag: DW_TAG_base_type, name: "bool", size: 8, align: 8, encoding: DW_ATE_boolean)
-!9 = !MDDerivedType(tag: DW_TAG_member, name: "x5", line: 4, size: 64, align: 64, offset: 64, file: !5, scope: !"_ZTS1A", baseType: !10)
-!10 = !MDDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, baseType: null)
-!11 = !MDDerivedType(tag: DW_TAG_member, name: "x6", line: 5, size: 8, align: 8, offset: 128, file: !5, scope: !"_ZTS1A", baseType: !8)
-!12 = !MDDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, baseType: !13)
-!13 = !MDBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
-!14 = !MDCompositeType(tag: DW_TAG_structure_type, name: "B", line: 8, size: 8, align: 8, file: !5, elements: !15, identifier: "_ZTS1B")
+!7 = !DIDerivedType(tag: DW_TAG_member, name: "x4", line: 3, size: 8, align: 8, file: !5, scope: !"_ZTS1A", baseType: !8)
+!8 = !DIBasicType(tag: DW_TAG_base_type, name: "bool", size: 8, align: 8, encoding: DW_ATE_boolean)
+!9 = !DIDerivedType(tag: DW_TAG_member, name: "x5", line: 4, size: 64, align: 64, offset: 64, file: !5, scope: !"_ZTS1A", baseType: !10)
+!10 = !DIDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, baseType: null)
+!11 = !DIDerivedType(tag: DW_TAG_member, name: "x6", line: 5, size: 8, align: 8, offset: 128, file: !5, scope: !"_ZTS1A", baseType: !8)
+!12 = !DIDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, baseType: !13)
+!13 = !DIBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
+!14 = !DICompositeType(tag: DW_TAG_structure_type, name: "B", line: 8, size: 8, align: 8, file: !5, elements: !15, identifier: "_ZTS1B")
 !15 = !{!16, !21}
-!16 = !MDSubprogram(name: "B", line: 9, isLocal: false, isDefinition: false, flags: DIFlagPrototyped, isOptimized: true, scopeLine: 9, file: !5, scope: !"_ZTS1B", type: !17)
-!17 = !MDSubroutineType(types: !18)
+!16 = !DISubprogram(name: "B", line: 9, isLocal: false, isDefinition: false, flags: DIFlagPrototyped, isOptimized: true, scopeLine: 9, file: !5, scope: !"_ZTS1B", type: !17)
+!17 = !DISubroutineType(types: !18)
 !18 = !{null, !19, !20}
-!19 = !MDDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, flags: DIFlagArtificial | DIFlagObjectPointer, baseType: !"_ZTS1B")
-!20 = !MDBasicType(tag: DW_TAG_base_type, name: "long int", size: 64, align: 64, encoding: DW_ATE_signed)
-!21 = !MDSubprogram(name: "~B", line: 10, isLocal: false, isDefinition: false, flags: DIFlagPrototyped, isOptimized: true, scopeLine: 10, file: !5, scope: !"_ZTS1B", type: !22)
-!22 = !MDSubroutineType(types: !23)
+!19 = !DIDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, flags: DIFlagArtificial | DIFlagObjectPointer, baseType: !"_ZTS1B")
+!20 = !DIBasicType(tag: DW_TAG_base_type, name: "long int", size: 64, align: 64, encoding: DW_ATE_signed)
+!21 = !DISubprogram(name: "~B", line: 10, isLocal: false, isDefinition: false, flags: DIFlagPrototyped, isOptimized: true, scopeLine: 10, file: !5, scope: !"_ZTS1B", type: !22)
+!22 = !DISubroutineType(types: !23)
 !23 = !{null, !19}
 !24 = !{!25, !31, !34}
-!25 = !MDSubprogram(name: "f13", linkageName: "_Z3f131A", line: 13, isLocal: false, isDefinition: true, flags: DIFlagPrototyped, isOptimized: true, scopeLine: 13, file: !5, scope: !26, type: !27, function: void (%struct.A*)* @_Z3f131A, variables: !29)
-!26 = !MDFile(filename: "test.cpp", directory: "")
-!27 = !MDSubroutineType(types: !28)
+!25 = !DISubprogram(name: "f13", linkageName: "_Z3f131A", line: 13, isLocal: false, isDefinition: true, flags: DIFlagPrototyped, isOptimized: true, scopeLine: 13, file: !5, scope: !26, type: !27, function: void (%struct.A*)* @_Z3f131A, variables: !29)
+!26 = !DIFile(filename: "test.cpp", directory: "")
+!27 = !DISubroutineType(types: !28)
 !28 = !{null, !"_ZTS1A"}
 !29 = !{!30}
-!30 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "p1", line: 13, arg: 1, scope: !25, file: !26, type: !"_ZTS1A")
-!31 = !MDSubprogram(name: "f11", linkageName: "_Z3f111A", line: 17, isLocal: false, isDefinition: true, flags: DIFlagPrototyped, isOptimized: true, scopeLine: 17, file: !5, scope: !26, type: !27, function: void (%struct.A*)* @_Z3f111A, variables: !32)
+!30 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "p1", line: 13, arg: 1, scope: !25, file: !26, type: !"_ZTS1A")
+!31 = !DISubprogram(name: "f11", linkageName: "_Z3f111A", line: 17, isLocal: false, isDefinition: true, flags: DIFlagPrototyped, isOptimized: true, scopeLine: 17, file: !5, scope: !26, type: !27, function: void (%struct.A*)* @_Z3f111A, variables: !32)
 !32 = !{!33}
-!33 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "p1", line: 17, arg: 1, scope: !31, file: !26, type: !"_ZTS1A")
-!34 = !MDSubprogram(name: "f16", linkageName: "_Z3f16v", line: 18, isLocal: false, isDefinition: true, flags: DIFlagPrototyped, isOptimized: true, scopeLine: 18, file: !5, scope: !26, type: !35, function: void ()* @_Z3f16v, variables: !37)
-!35 = !MDSubroutineType(types: !36)
+!33 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "p1", line: 17, arg: 1, scope: !31, file: !26, type: !"_ZTS1A")
+!34 = !DISubprogram(name: "f16", linkageName: "_Z3f16v", line: 18, isLocal: false, isDefinition: true, flags: DIFlagPrototyped, isOptimized: true, scopeLine: 18, file: !5, scope: !26, type: !35, function: void ()* @_Z3f16v, variables: !37)
+!35 = !DISubroutineType(types: !36)
 !36 = !{null}
 !37 = !{!38, !39}
-!38 = !MDLocalVariable(tag: DW_TAG_auto_variable, name: "c", line: 19, scope: !34, file: !26, type: !"_ZTS1A")
-!39 = !MDLocalVariable(tag: DW_TAG_auto_variable, name: "d", line: 20, scope: !34, file: !26, type: !"_ZTS1B")
+!38 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "c", line: 19, scope: !34, file: !26, type: !"_ZTS1A")
+!39 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "d", line: 20, scope: !34, file: !26, type: !"_ZTS1B")
 !40 = !{!41, !42}
-!41 = !MDGlobalVariable(name: "a", line: 1, isLocal: false, isDefinition: true, scope: null, file: !26, type: !20, variable: i64* @a)
-!42 = !MDGlobalVariable(name: "b", line: 7, isLocal: false, isDefinition: true, scope: null, file: !26, type: !12, variable: i32** @b)
+!41 = !DIGlobalVariable(name: "a", line: 1, isLocal: false, isDefinition: true, scope: null, file: !26, type: !20, variable: i64* @a)
+!42 = !DIGlobalVariable(name: "b", line: 7, isLocal: false, isDefinition: true, scope: null, file: !26, type: !12, variable: i32** @b)
 !43 = !{i32 2, !"Dwarf Version", i32 2}
 !44 = !{i32 2, !"Debug Info Version", i32 3}
 !45 = !{!"clang version 3.7.0 "}
-!46 = !MDExpression(DW_OP_deref)
-!47 = !MDLocation(line: 13, column: 12, scope: !25)
-!48 = !MDLocation(line: 14, column: 37, scope: !25)
+!46 = !DIExpression(DW_OP_deref)
+!47 = !DILocation(line: 13, column: 12, scope: !25)
+!48 = !DILocation(line: 14, column: 37, scope: !25)
 !49 = !{!50, !50, i64 0}
 !50 = !{!"long", !51, i64 0}
 !51 = !{!"omnipotent char", !52, i64 0}
 !52 = !{!"Simple C/C++ TBAA"}
-!53 = !MDLocation(line: 14, column: 14, scope: !25)
-!54 = !MDLocation(line: 14, column: 5, scope: !25)
+!53 = !DILocation(line: 14, column: 14, scope: !25)
+!54 = !DILocation(line: 14, column: 5, scope: !25)
 !55 = !{!56, !56, i64 0}
 !56 = !{!"any pointer", !51, i64 0}
-!57 = !MDLocation(line: 15, column: 6, scope: !25)
+!57 = !DILocation(line: 15, column: 6, scope: !25)
 !58 = !{i64 0, i64 1, !59, i64 8, i64 8, !55, i64 16, i64 1, !59}
 !59 = !{!60, !60, i64 0}
 !60 = !{!"bool", !51, i64 0}
-!61 = !MDLocation(line: 15, column: 3, scope: !25)
-!62 = !MDLocation(line: 16, column: 1, scope: !25)
-!63 = !MDLocation(line: 17, column: 12, scope: !31)
-!64 = !MDLocation(line: 17, column: 22, scope: !31)
-!65 = !MDLocation(line: 13, column: 12, scope: !25, inlinedAt: !66)
-!66 = distinct !MDLocation(line: 17, column: 18, scope: !31)
-!67 = !MDLocation(line: 14, column: 37, scope: !25, inlinedAt: !66)
-!68 = !MDLocation(line: 14, column: 14, scope: !25, inlinedAt: !66)
-!69 = !MDLocation(line: 14, column: 5, scope: !25, inlinedAt: !66)
-!70 = !MDLocation(line: 15, column: 6, scope: !25, inlinedAt: !66)
-!71 = !MDLocation(line: 15, column: 3, scope: !25, inlinedAt: !66)
-!72 = !MDLocation(line: 16, column: 1, scope: !25, inlinedAt: !66)
-!73 = !MDLocation(line: 17, column: 27, scope: !31)
-!74 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "p1", line: 17, arg: 1, scope: !31, file: !26, type: !"_ZTS1A")
-!75 = distinct !MDLocation(line: 22, column: 3, scope: !34)
-!76 = !MDExpression(DW_OP_bit_piece, 8, 120)
-!77 = !MDLocation(line: 17, column: 12, scope: !31, inlinedAt: !75)
-!78 = !MDExpression(DW_OP_bit_piece, 136, 56)
-!79 = !MDExpression()
-!80 = !MDLocation(line: 19, column: 5, scope: !34)
-!81 = !MDLocation(line: 20, column: 7, scope: !34)
-!82 = !MDLocation(line: 20, column: 5, scope: !34)
-!83 = !MDExpression(DW_OP_bit_piece, 0, 8)
-!84 = !MDExpression(DW_OP_bit_piece, 128, 8)
-!85 = !MDLocation(line: 13, column: 12, scope: !25, inlinedAt: !86)
-!86 = distinct !MDLocation(line: 17, column: 18, scope: !31, inlinedAt: !75)
-!87 = !MDLocation(line: 14, column: 37, scope: !25, inlinedAt: !86)
-!88 = !MDLocation(line: 14, column: 14, scope: !25, inlinedAt: !86)
-!89 = !MDLocation(line: 22, column: 7, scope: !34)
-!90 = !MDLocation(line: 14, column: 5, scope: !25, inlinedAt: !86)
-!91 = !MDLocation(line: 15, column: 6, scope: !25, inlinedAt: !86)
-!92 = !MDLocation(line: 15, column: 3, scope: !25, inlinedAt: !86)
-!93 = !MDLocation(line: 16, column: 1, scope: !25, inlinedAt: !86)
-!94 = !MDLocation(line: 23, column: 1, scope: !34)
+!61 = !DILocation(line: 15, column: 3, scope: !25)
+!62 = !DILocation(line: 16, column: 1, scope: !25)
+!63 = !DILocation(line: 17, column: 12, scope: !31)
+!64 = !DILocation(line: 17, column: 22, scope: !31)
+!65 = !DILocation(line: 13, column: 12, scope: !25, inlinedAt: !66)
+!66 = distinct !DILocation(line: 17, column: 18, scope: !31)
+!67 = !DILocation(line: 14, column: 37, scope: !25, inlinedAt: !66)
+!68 = !DILocation(line: 14, column: 14, scope: !25, inlinedAt: !66)
+!69 = !DILocation(line: 14, column: 5, scope: !25, inlinedAt: !66)
+!70 = !DILocation(line: 15, column: 6, scope: !25, inlinedAt: !66)
+!71 = !DILocation(line: 15, column: 3, scope: !25, inlinedAt: !66)
+!72 = !DILocation(line: 16, column: 1, scope: !25, inlinedAt: !66)
+!73 = !DILocation(line: 17, column: 27, scope: !31)
+!74 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "p1", line: 17, arg: 1, scope: !31, file: !26, type: !"_ZTS1A")
+!75 = distinct !DILocation(line: 22, column: 3, scope: !34)
+!76 = !DIExpression(DW_OP_bit_piece, 8, 120)
+!77 = !DILocation(line: 17, column: 12, scope: !31, inlinedAt: !75)
+!78 = !DIExpression(DW_OP_bit_piece, 136, 56)
+!79 = !DIExpression()
+!80 = !DILocation(line: 19, column: 5, scope: !34)
+!81 = !DILocation(line: 20, column: 7, scope: !34)
+!82 = !DILocation(line: 20, column: 5, scope: !34)
+!83 = !DIExpression(DW_OP_bit_piece, 0, 8)
+!84 = !DIExpression(DW_OP_bit_piece, 128, 8)
+!85 = !DILocation(line: 13, column: 12, scope: !25, inlinedAt: !86)
+!86 = distinct !DILocation(line: 17, column: 18, scope: !31, inlinedAt: !75)
+!87 = !DILocation(line: 14, column: 37, scope: !25, inlinedAt: !86)
+!88 = !DILocation(line: 14, column: 14, scope: !25, inlinedAt: !86)
+!89 = !DILocation(line: 22, column: 7, scope: !34)
+!90 = !DILocation(line: 14, column: 5, scope: !25, inlinedAt: !86)
+!91 = !DILocation(line: 15, column: 6, scope: !25, inlinedAt: !86)
+!92 = !DILocation(line: 15, column: 3, scope: !25, inlinedAt: !86)
+!93 = !DILocation(line: 16, column: 1, scope: !25, inlinedAt: !86)
+!94 = !DILocation(line: 23, column: 1, scope: !34)
diff --git a/test/DebugInfo/AArch64/struct_by_value.ll b/test/DebugInfo/AArch64/struct_by_value.ll
index 7fcab2bdde3..dc9ac42e8a0 100644
--- a/test/DebugInfo/AArch64/struct_by_value.ll
+++ b/test/DebugInfo/AArch64/struct_by_value.ll
@@ -33,7 +33,7 @@ target triple = "arm64-apple-ios3.0.0"
 ; Function Attrs: nounwind ssp
 define i32 @return_five_int(%struct.five* %f) #0 {
 entry:
-  call void @llvm.dbg.declare(metadata %struct.five* %f, metadata !17, metadata !MDExpression(DW_OP_deref)), !dbg !18
+  call void @llvm.dbg.declare(metadata %struct.five* %f, metadata !17, metadata !DIExpression(DW_OP_deref)), !dbg !18
   %a = getelementptr inbounds %struct.five, %struct.five* %f, i32 0, i32 0, !dbg !19
   %0 = load i32, i32* %a, align 4, !dbg !19
   ret i32 %0, !dbg !19
@@ -48,24 +48,24 @@ attributes #1 = { nounwind readnone }
 !llvm.dbg.cu = !{!0}
 !llvm.module.flags = !{!16, !20}
 
-!0 = !MDCompileUnit(language: DW_LANG_C99, producer: "LLVM version 3.4 ", isOptimized: false, emissionKind: 0, file: !1, enums: !2, retainedTypes: !2, subprograms: !3, globals: !2, imports: !2)
-!1 = !MDFile(filename: "struct_by_value.c", directory: "")
+!0 = !DICompileUnit(language: DW_LANG_C99, producer: "LLVM version 3.4 ", isOptimized: false, emissionKind: 0, file: !1, enums: !2, retainedTypes: !2, subprograms: !3, globals: !2, imports: !2)
+!1 = !DIFile(filename: "struct_by_value.c", directory: "")
 !2 = !{}
 !3 = !{!4}
-!4 = !MDSubprogram(name: "return_five_int", line: 13, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 14, file: !1, scope: !5, type: !6, function: i32 (%struct.five*)* @return_five_int, variables: !2)
-!5 = !MDFile(filename: "struct_by_value.c", directory: "")
-!6 = !MDSubroutineType(types: !7)
+!4 = !DISubprogram(name: "return_five_int", line: 13, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 14, file: !1, scope: !5, type: !6, function: i32 (%struct.five*)* @return_five_int, variables: !2)
+!5 = !DIFile(filename: "struct_by_value.c", directory: "")
+!6 = !DISubroutineType(types: !7)
 !7 = !{!8, !9}
-!8 = !MDBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
-!9 = !MDCompositeType(tag: DW_TAG_structure_type, name: "five", line: 1, size: 160, align: 32, file: !1, elements: !10)
+!8 = !DIBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
+!9 = !DICompositeType(tag: DW_TAG_structure_type, name: "five", line: 1, size: 160, align: 32, file: !1, elements: !10)
 !10 = !{!11, !12, !13, !14, !15}
-!11 = !MDDerivedType(tag: DW_TAG_member, name: "a", line: 3, size: 32, align: 32, file: !1, scope: !9, baseType: !8)
-!12 = !MDDerivedType(tag: DW_TAG_member, name: "b", line: 4, size: 32, align: 32, offset: 32, file: !1, scope: !9, baseType: !8)
-!13 = !MDDerivedType(tag: DW_TAG_member, name: "c", line: 5, size: 32, align: 32, offset: 64, file: !1, scope: !9, baseType: !8)
-!14 = !MDDerivedType(tag: DW_TAG_member, name: "d", line: 6, size: 32, align: 32, offset: 96, file: !1, scope: !9, baseType: !8)
-!15 = !MDDerivedType(tag: DW_TAG_member, name: "e", line: 7, size: 32, align: 32, offset: 128, file: !1, scope: !9, baseType: !8)
+!11 = !DIDerivedType(tag: DW_TAG_member, name: "a", line: 3, size: 32, align: 32, file: !1, scope: !9, baseType: !8)
+!12 = !DIDerivedType(tag: DW_TAG_member, name: "b", line: 4, size: 32, align: 32, offset: 32, file: !1, scope: !9, baseType: !8)
+!13 = !DIDerivedType(tag: DW_TAG_member, name: "c", line: 5, size: 32, align: 32, offset: 64, file: !1, scope: !9, baseType: !8)
+!14 = !DIDerivedType(tag: DW_TAG_member, name: "d", line: 6, size: 32, align: 32, offset: 96, file: !1, scope: !9, baseType: !8)
+!15 = !DIDerivedType(tag: DW_TAG_member, name: "e", line: 7, size: 32, align: 32, offset: 128, file: !1, scope: !9, baseType: !8)
 !16 = !{i32 2, !"Dwarf Version", i32 2}
-!17 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "f", line: 13, arg: 1, scope: !4, file: !5, type: !9)
-!18 = !MDLocation(line: 13, scope: !4)
-!19 = !MDLocation(line: 16, scope: !4)
+!17 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "f", line: 13, arg: 1, scope: !4, file: !5, type: !9)
+!18 = !DILocation(line: 13, scope: !4)
+!19 = !DILocation(line: 16, scope: !4)
 !20 = !{i32 1, !"Debug Info Version", i32 3}
diff --git a/test/DebugInfo/ARM/PR16736.ll b/test/DebugInfo/ARM/PR16736.ll
index 8ad5e95b5e7..9d4992210a2 100644
--- a/test/DebugInfo/ARM/PR16736.ll
+++ b/test/DebugInfo/ARM/PR16736.ll
@@ -15,14 +15,14 @@ target triple = "thumbv7-apple-ios"
 ; Function Attrs: nounwind
 define arm_aapcscc void @_Z1hiiiif(i32, i32, i32, i32, float %x) #0 {
 entry:
-  tail call void @llvm.dbg.value(metadata i32 %0, i64 0, metadata !12, metadata !MDExpression()), !dbg !18
-  tail call void @llvm.dbg.value(metadata i32 %1, i64 0, metadata !13, metadata !MDExpression()), !dbg !18
-  tail call void @llvm.dbg.value(metadata i32 %2, i64 0, metadata !14, metadata !MDExpression()), !dbg !18
-  tail call void @llvm.dbg.value(metadata i32 %3, i64 0, metadata !15, metadata !MDExpression()), !dbg !18
-  tail call void @llvm.dbg.value(metadata float %x, i64 0, metadata !16, metadata !MDExpression()), !dbg !18
+  tail call void @llvm.dbg.value(metadata i32 %0, i64 0, metadata !12, metadata !DIExpression()), !dbg !18
+  tail call void @llvm.dbg.value(metadata i32 %1, i64 0, metadata !13, metadata !DIExpression()), !dbg !18
+  tail call void @llvm.dbg.value(metadata i32 %2, i64 0, metadata !14, metadata !DIExpression()), !dbg !18
+  tail call void @llvm.dbg.value(metadata i32 %3, i64 0, metadata !15, metadata !DIExpression()), !dbg !18
+  tail call void @llvm.dbg.value(metadata float %x, i64 0, metadata !16, metadata !DIExpression()), !dbg !18
   %call = tail call arm_aapcscc i32 @_Z1fv() #3, !dbg !19
   %conv = sitofp i32 %call to float, !dbg !19
-  tail call void @llvm.dbg.value(metadata float %conv, i64 0, metadata !16, metadata !MDExpression()), !dbg !19
+  tail call void @llvm.dbg.value(metadata float %conv, i64 0, metadata !16, metadata !DIExpression()), !dbg !19
   tail call arm_aapcscc void @_Z1gf(float %conv) #3, !dbg !19
   ret void, !dbg !20
 }
@@ -41,25 +41,25 @@ attributes #3 = { nounwind }
 !llvm.dbg.cu = !{!0}
 !llvm.module.flags = !{!17, !21}
 
-!0 = !MDCompileUnit(language: DW_LANG_C_plus_plus, producer: "clang version 3.4 (trunk 190804) (llvm/trunk 190797)", isOptimized: true, emissionKind: 1, file: !1, enums: !2, retainedTypes: !2, subprograms: !3, globals: !2, imports: !2)
-!1 = !MDFile(filename: "/<unknown>", directory: "")
+!0 = !DICompileUnit(language: DW_LANG_C_plus_plus, producer: "clang version 3.4 (trunk 190804) (llvm/trunk 190797)", isOptimized: true, emissionKind: 1, file: !1, enums: !2, retainedTypes: !2, subprograms: !3, globals: !2, imports: !2)
+!1 = !DIFile(filename: "/<unknown>", directory: "")
 !2 = !{}
 !3 = !{!4}
-!4 = !MDSubprogram(name: "h", linkageName: "_Z1hiiiif", line: 3, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: true, scopeLine: 3, file: !5, scope: !6, type: !7, function: void (i32, i32, i32, i32, float)* @_Z1hiiiif, variables: !11)
-!5 = !MDFile(filename: "/arm.cpp", directory: "")
-!6 = !MDFile(filename: "/arm.cpp", directory: "")
-!7 = !MDSubroutineType(types: !8)
+!4 = !DISubprogram(name: "h", linkageName: "_Z1hiiiif", line: 3, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: true, scopeLine: 3, file: !5, scope: !6, type: !7, function: void (i32, i32, i32, i32, float)* @_Z1hiiiif, variables: !11)
+!5 = !DIFile(filename: "/arm.cpp", directory: "")
+!6 = !DIFile(filename: "/arm.cpp", directory: "")
+!7 = !DISubroutineType(types: !8)
 !8 = !{null, !9, !9, !9, !9, !10}
-!9 = !MDBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
-!10 = !MDBasicType(tag: DW_TAG_base_type, name: "float", size: 32, align: 32, encoding: DW_ATE_float)
+!9 = !DIBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
+!10 = !DIBasicType(tag: DW_TAG_base_type, name: "float", size: 32, align: 32, encoding: DW_ATE_float)
 !11 = !{!12, !13, !14, !15, !16}
-!12 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "", line: 3, arg: 1, scope: !4, file: !6, type: !9)
-!13 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "", line: 3, arg: 2, scope: !4, file: !6, type: !9)
-!14 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "", line: 3, arg: 3, scope: !4, file: !6, type: !9)
-!15 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "", line: 3, arg: 4, scope: !4, file: !6, type: !9)
-!16 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "x", line: 3, arg: 5, scope: !4, file: !6, type: !10)
+!12 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "", line: 3, arg: 1, scope: !4, file: !6, type: !9)
+!13 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "", line: 3, arg: 2, scope: !4, file: !6, type: !9)
+!14 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "", line: 3, arg: 3, scope: !4, file: !6, type: !9)
+!15 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "", line: 3, arg: 4, scope: !4, file: !6, type: !9)
+!16 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "x", line: 3, arg: 5, scope: !4, file: !6, type: !10)
 !17 = !{i32 2, !"Dwarf Version", i32 4}
-!18 = !MDLocation(line: 3, scope: !4)
-!19 = !MDLocation(line: 4, scope: !4)
-!20 = !MDLocation(line: 5, scope: !4)
+!18 = !DILocation(line: 3, scope: !4)
+!19 = !DILocation(line: 4, scope: !4)
+!20 = !DILocation(line: 5, scope: !4)
 !21 = !{i32 1, !"Debug Info Version", i32 3}
diff --git a/test/DebugInfo/ARM/bitfield.ll b/test/DebugInfo/ARM/bitfield.ll
new file mode 100644
index 00000000000..9b41e4e4f7b
--- /dev/null
+++ b/test/DebugInfo/ARM/bitfield.ll
@@ -0,0 +1,45 @@
+; RUN: %llc_dwarf -O0 -filetype=obj -o %t.o %s
+; RUN: llvm-dwarfdump -debug-dump=info %t.o | FileCheck %s
+; REQUIRES: object-emission
+;
+; Generated from:
+; struct {
+;   char c;
+;   int : 4;
+;   int reserved : 28;
+; } a;
+;
+; CHECK:      DW_TAG_member
+; CHECK:          DW_AT_name {{.*}} "reserved"
+; CHECK:          DW_AT_byte_size  {{.*}} (0x04)
+; CHECK:          DW_AT_bit_size   {{.*}} (0x1c)
+; CHECK:          DW_AT_bit_offset {{.*}} (0x18)
+; CHECK:          DW_AT_data_member_location {{.*}}00
+target datalayout = "e-m:o-p:32:32-f64:32:64-v64:32:64-v128:32:128-a:0:32-n32-S32"
+target triple = "thumbv7-apple-ios"
+
+%struct.anon = type { i8, [5 x i8] }
+
+@a = common global %struct.anon zeroinitializer, align 1
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!11, !12, !13, !14, !15}
+!llvm.ident = !{!16}
+
+!0 = !DICompileUnit(language: DW_LANG_C99, file: !1, producer: "clang version 3.7.0 (trunk 240548) (llvm/trunk 240554)", isOptimized: false, runtimeVersion: 0, emissionKind: 1, enums: !2, retainedTypes: !2, subprograms: !2, globals: !3, imports: !2)
+!1 = !DIFile(filename: "test.i", directory: "/")
+!2 = !{}
+!3 = !{!4}
+!4 = !DIGlobalVariable(name: "a", scope: !0, file: !1, line: 5, type: !5, isLocal: false, isDefinition: true, variable: %struct.anon* @a)
+!5 = !DICompositeType(tag: DW_TAG_structure_type, file: !1, line: 1, size: 48, align: 8, elements: !6)
+!6 = !{!7, !9}
+!7 = !DIDerivedType(tag: DW_TAG_member, name: "c", scope: !5, file: !1, line: 2, baseType: !8, size: 8, align: 8)
+!8 = !DIBasicType(name: "char", size: 8, align: 8, encoding: DW_ATE_signed_char)
+!9 = !DIDerivedType(tag: DW_TAG_member, name: "reserved", scope: !5, file: !1, line: 4, baseType: !10, size: 28, align: 32, offset: 12)
+!10 = !DIBasicType(name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
+!11 = !{i32 2, !"Dwarf Version", i32 2}
+!12 = !{i32 2, !"Debug Info Version", i32 3}
+!13 = !{i32 1, !"wchar_size", i32 4}
+!14 = !{i32 1, !"min_enum_size", i32 4}
+!15 = !{i32 1, !"PIC Level", i32 2}
+!16 = !{!"clang version 3.7.0 (trunk 240548) (llvm/trunk 240554)"}
diff --git a/test/DebugInfo/ARM/cfi-eof-prologue.ll b/test/DebugInfo/ARM/cfi-eof-prologue.ll
index fd91f9211a0..07e4d8a717b 100644
--- a/test/DebugInfo/ARM/cfi-eof-prologue.ll
+++ b/test/DebugInfo/ARM/cfi-eof-prologue.ll
@@ -62,54 +62,54 @@ attributes #3 = { nounwind }
 !llvm.module.flags = !{!35, !36, !37, !38}
 !llvm.ident = !{!39}
 
-!0 = !MDCompileUnit(language: DW_LANG_C_plus_plus, producer: "clang version 3.6.0 (trunk 224279) (llvm/trunk 224283)", isOptimized: true, emissionKind: 1, file: !1, enums: !2, retainedTypes: !3, subprograms: !27, globals: !2, imports: !2)
-!1 = !MDFile(filename: "<stdin>", directory: "")
+!0 = !DICompileUnit(language: DW_LANG_C_plus_plus, producer: "clang version 3.6.0 (trunk 224279) (llvm/trunk 224283)", isOptimized: true, emissionKind: 1, file: !1, enums: !2, retainedTypes: !3, subprograms: !27, globals: !2, imports: !2)
+!1 = !DIFile(filename: "<stdin>", directory: "")
 !2 = !{}
 !3 = !{!4, !13}
-!4 = !MDCompositeType(tag: DW_TAG_structure_type, name: "B", line: 5, size: 32, align: 32, file: !5, elements: !6, vtableHolder: !"_ZTS1A", identifier: "_ZTS1B")
-!5 = !MDFile(filename: "test1.cpp", directory: "")
+!4 = !DICompositeType(tag: DW_TAG_structure_type, name: "B", line: 5, size: 32, align: 32, file: !5, elements: !6, vtableHolder: !"_ZTS1A", identifier: "_ZTS1B")
+!5 = !DIFile(filename: "test1.cpp", directory: "")
 !6 = !{!7, !8, !12}
-!7 = !MDDerivedType(tag: DW_TAG_inheritance, scope: !"_ZTS1B", baseType: !"_ZTS1A")
-!8 = !MDSubprogram(name: "B", line: 6, isLocal: false, isDefinition: false, flags: DIFlagPrototyped, isOptimized: true, scopeLine: 6, file: !5, scope: !"_ZTS1B", type: !9)
-!9 = !MDSubroutineType(types: !10)
+!7 = !DIDerivedType(tag: DW_TAG_inheritance, scope: !"_ZTS1B", baseType: !"_ZTS1A")
+!8 = !DISubprogram(name: "B", line: 6, isLocal: false, isDefinition: false, flags: DIFlagPrototyped, isOptimized: true, scopeLine: 6, file: !5, scope: !"_ZTS1B", type: !9)
+!9 = !DISubroutineType(types: !10)
 !10 = !{null, !11}
-!11 = !MDDerivedType(tag: DW_TAG_pointer_type, size: 32, align: 32, flags: DIFlagArtificial | DIFlagObjectPointer, baseType: !"_ZTS1B")
-!12 = !MDSubprogram(name: "~B", line: 7, isLocal: false, isDefinition: false, virtuality: DW_VIRTUALITY_virtual, flags: DIFlagPrototyped, isOptimized: true, scopeLine: 7, file: !5, scope: !"_ZTS1B", type: !9, containingType: !"_ZTS1B")
-!13 = !MDCompositeType(tag: DW_TAG_structure_type, name: "A", line: 1, size: 32, align: 32, file: !5, elements: !14, vtableHolder: !"_ZTS1A", identifier: "_ZTS1A")
+!11 = !DIDerivedType(tag: DW_TAG_pointer_type, size: 32, align: 32, flags: DIFlagArtificial | DIFlagObjectPointer, baseType: !"_ZTS1B")
+!12 = !DISubprogram(name: "~B", line: 7, isLocal: false, isDefinition: false, virtuality: DW_VIRTUALITY_virtual, flags: DIFlagPrototyped, isOptimized: true, scopeLine: 7, file: !5, scope: !"_ZTS1B", type: !9, containingType: !"_ZTS1B")
+!13 = !DICompositeType(tag: DW_TAG_structure_type, name: "A", line: 1, size: 32, align: 32, file: !5, elements: !14, vtableHolder: !"_ZTS1A", identifier: "_ZTS1A")
 !14 = !{!15, !22, !26}
-!15 = !MDDerivedType(tag: DW_TAG_member, name: "_vptr$A", size: 32, flags: DIFlagArtificial, file: !5, scope: !16, baseType: !17)
-!16 = !MDFile(filename: "test1.cpp", directory: "")
-!17 = !MDDerivedType(tag: DW_TAG_pointer_type, size: 32, baseType: !18)
-!18 = !MDDerivedType(tag: DW_TAG_pointer_type, name: "__vtbl_ptr_type", size: 32, baseType: !19)
-!19 = !MDSubroutineType(types: !20)
+!15 = !DIDerivedType(tag: DW_TAG_member, name: "_vptr$A", size: 32, flags: DIFlagArtificial, file: !5, scope: !16, baseType: !17)
+!16 = !DIFile(filename: "test1.cpp", directory: "")
+!17 = !DIDerivedType(tag: DW_TAG_pointer_type, size: 32, baseType: !18)
+!18 = !DIDerivedType(tag: DW_TAG_pointer_type, name: "__vtbl_ptr_type", size: 32, baseType: !19)
+!19 = !DISubroutineType(types: !20)
 !20 = !{!21}
-!21 = !MDBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
-!22 = !MDSubprogram(name: "A", line: 2, isLocal: false, isDefinition: false, flags: DIFlagPrototyped, isOptimized: true, scopeLine: 2, file: !5, scope: !"_ZTS1A", type: !23)
-!23 = !MDSubroutineType(types: !24)
+!21 = !DIBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
+!22 = !DISubprogram(name: "A", line: 2, isLocal: false, isDefinition: false, flags: DIFlagPrototyped, isOptimized: true, scopeLine: 2, file: !5, scope: !"_ZTS1A", type: !23)
+!23 = !DISubroutineType(types: !24)
 !24 = !{null, !25}
-!25 = !MDDerivedType(tag: DW_TAG_pointer_type, size: 32, align: 32, flags: DIFlagArtificial | DIFlagObjectPointer, baseType: !"_ZTS1A")
-!26 = !MDSubprogram(name: "~A", line: 3, isLocal: false, isDefinition: false, virtuality: DW_VIRTUALITY_virtual, flags: DIFlagPrototyped, isOptimized: true, scopeLine: 3, file: !5, scope: !"_ZTS1A", type: !23, containingType: !"_ZTS1A")
+!25 = !DIDerivedType(tag: DW_TAG_pointer_type, size: 32, align: 32, flags: DIFlagArtificial | DIFlagObjectPointer, baseType: !"_ZTS1A")
+!26 = !DISubprogram(name: "~A", line: 3, isLocal: false, isDefinition: false, virtuality: DW_VIRTUALITY_virtual, flags: DIFlagPrototyped, isOptimized: true, scopeLine: 3, file: !5, scope: !"_ZTS1A", type: !23, containingType: !"_ZTS1A")
 !27 = !{!28, !32}
-!28 = !MDSubprogram(name: "B", linkageName: "_ZN1BC2Ev", line: 9, isLocal: false, isDefinition: true, flags: DIFlagPrototyped, isOptimized: true, scopeLine: 9, file: !5, scope: !"_ZTS1B", type: !9, function: %struct.B* (%struct.B*)* @_ZN1BC2Ev, declaration: !8, variables: !29)
+!28 = !DISubprogram(name: "B", linkageName: "_ZN1BC2Ev", line: 9, isLocal: false, isDefinition: true, flags: DIFlagPrototyped, isOptimized: true, scopeLine: 9, file: !5, scope: !"_ZTS1B", type: !9, function: %struct.B* (%struct.B*)* @_ZN1BC2Ev, declaration: !8, variables: !29)
 !29 = !{!30}
-!30 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "this", arg: 1, flags: DIFlagArtificial | DIFlagObjectPointer, scope: !28, type: !31)
-!31 = !MDDerivedType(tag: DW_TAG_pointer_type, size: 32, align: 32, baseType: !"_ZTS1B")
-!32 = !MDSubprogram(name: "B", linkageName: "_ZN1BC1Ev", line: 9, isLocal: false, isDefinition: true, flags: DIFlagPrototyped, isOptimized: true, scopeLine: 9, file: !5, scope: !"_ZTS1B", type: !9, function: %struct.B* (%struct.B*)* @_ZN1BC1Ev, declaration: !8, variables: !33)
+!30 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "this", arg: 1, flags: DIFlagArtificial | DIFlagObjectPointer, scope: !28, type: !31)
+!31 = !DIDerivedType(tag: DW_TAG_pointer_type, size: 32, align: 32, baseType: !"_ZTS1B")
+!32 = !DISubprogram(name: "B", linkageName: "_ZN1BC1Ev", line: 9, isLocal: false, isDefinition: true, flags: DIFlagPrototyped, isOptimized: true, scopeLine: 9, file: !5, scope: !"_ZTS1B", type: !9, function: %struct.B* (%struct.B*)* @_ZN1BC1Ev, declaration: !8, variables: !33)
 !33 = !{!34}
-!34 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "this", arg: 1, flags: DIFlagArtificial | DIFlagObjectPointer, scope: !32, type: !31)
+!34 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "this", arg: 1, flags: DIFlagArtificial | DIFlagObjectPointer, scope: !32, type: !31)
 !35 = !{i32 2, !"Dwarf Version", i32 4}
 !36 = !{i32 2, !"Debug Info Version", i32 3}
 !37 = !{i32 1, !"wchar_size", i32 4}
 !38 = !{i32 1, !"min_enum_size", i32 4}
 !39 = !{!"clang version 3.6.0 (trunk 224279) (llvm/trunk 224283)"}
-!40 = !MDExpression()
-!41 = !MDLocation(line: 0, scope: !28)
-!42 = !MDLocation(line: 9, scope: !28)
+!40 = !DIExpression()
+!41 = !DILocation(line: 0, scope: !28)
+!42 = !DILocation(line: 9, scope: !28)
 !43 = !{!44, !44, i64 0}
 !44 = !{!"vtable pointer", !45, i64 0}
 !45 = !{!"Simple C/C++ TBAA"}
-!46 = !MDLocation(line: 0, scope: !32)
-!47 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "this", arg: 1, flags: DIFlagArtificial | DIFlagObjectPointer, scope: !28, type: !31)
-!48 = !MDLocation(line: 9, scope: !32)
-!49 = !MDLocation(line: 0, scope: !28, inlinedAt: !48)
-!50 = !MDLocation(line: 9, scope: !28, inlinedAt: !48)
+!46 = !DILocation(line: 0, scope: !32)
+!47 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "this", arg: 1, flags: DIFlagArtificial | DIFlagObjectPointer, scope: !28, type: !31)
+!48 = !DILocation(line: 9, scope: !32)
+!49 = !DILocation(line: 0, scope: !28, inlinedAt: !48)
+!50 = !DILocation(line: 9, scope: !28, inlinedAt: !48)
diff --git a/test/DebugInfo/ARM/constant-dbgloc.ll b/test/DebugInfo/ARM/constant-dbgloc.ll
new file mode 100644
index 00000000000..c4a5703f3e8
--- /dev/null
+++ b/test/DebugInfo/ARM/constant-dbgloc.ll
@@ -0,0 +1,37 @@
+; RUN: llc -filetype=asm %s -o - | FileCheck %s
+
+target datalayout = "e-m:e-p:32:32-i64:64-v128:64:128-a:0:32-n32-S64"
+target triple = "armv7--linux-gnueabihf"
+
+; int
+; main(void)
+; {
+;     return -1;
+; }
+
+; CHECK: test.c:4:5
+; CHECK: mvn
+
+; Function Attrs: nounwind
+define i32 @main() {
+entry:
+  %retval = alloca i32, align 4
+  store i32 0, i32* %retval
+  ret i32 -1, !dbg !11
+}
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!8, !9}
+
+!0 = !DICompileUnit(language: DW_LANG_C99, file: !1, producer: "", isOptimized: false, subprograms: !3)
+!1 = !DIFile(filename: "test.c", directory: "/home/user/clang/build")
+!2 = !{}
+!3 = !{!4}
+!4 = !DISubprogram(name: "main", scope: !1, file: !1, line: 2, type: !5, isLocal: false, isDefinition: true, scopeLine: 3, flags: DIFlagPrototyped, isOptimized: false, function: i32 ()* @main, variables: !2)
+!5 = !DISubroutineType(types: !6)
+!6 = !{!7}
+!7 = !DIBasicType(name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
+!8 = !{i32 2, !"Dwarf Version", i32 4}
+!9 = !{i32 2, !"Debug Info Version", i32 3}
+!10 = !{i32 1, !"wchar_size", i32 4}
+!11 = !DILocation(line: 4, column: 5, scope: !4)
diff --git a/test/DebugInfo/ARM/header.ll b/test/DebugInfo/ARM/header.ll
index d9dd611e127..6723f093a59 100644
--- a/test/DebugInfo/ARM/header.ll
+++ b/test/DebugInfo/ARM/header.ll
@@ -18,13 +18,13 @@ define void @f() {
 !llvm.dbg.cu = !{!0}
 !llvm.module.flags = !{!7, !8}
 
-!0 = !MDCompileUnit(language: DW_LANG_C99, file: !1, producer: "foo", isOptimized: true, runtimeVersion: 0, emissionKind: 1, enums: !2, retainedTypes: !2, subprograms: !3, globals: !2, imports: !2)
-!1 = !MDFile(filename: "/foo/test.c", directory: "/foo")
+!0 = !DICompileUnit(language: DW_LANG_C99, file: !1, producer: "foo", isOptimized: true, runtimeVersion: 0, emissionKind: 1, enums: !2, retainedTypes: !2, subprograms: !3, globals: !2, imports: !2)
+!1 = !DIFile(filename: "/foo/test.c", directory: "/foo")
 !2 = !{}
 !3 = !{!4}
-!4 = !MDSubprogram(name: "f", scope: !1, file: !1, line: 1, type: !5, isLocal: false, isDefinition: true, scopeLine: 1, flags: DIFlagPrototyped, isOptimized: true, function: void ()* @f, variables: !2)
-!5 = !MDSubroutineType(types: !6)
+!4 = !DISubprogram(name: "f", scope: !1, file: !1, line: 1, type: !5, isLocal: false, isDefinition: true, scopeLine: 1, flags: DIFlagPrototyped, isOptimized: true, function: void ()* @f, variables: !2)
+!5 = !DISubroutineType(types: !6)
 !6 = !{null}
 !7 = !{i32 2, !"Dwarf Version", i32 4}
 !8 = !{i32 2, !"Debug Info Version", i32 3}
-!9 = !MDLocation(line: 1, column: 15, scope: !4)
+!9 = !DILocation(line: 1, column: 15, scope: !4)
diff --git a/test/DebugInfo/ARM/lowerbdgdeclare_vla.ll b/test/DebugInfo/ARM/lowerbdgdeclare_vla.ll
index f2f4cfd6227..daf26507a03 100644
--- a/test/DebugInfo/ARM/lowerbdgdeclare_vla.ll
+++ b/test/DebugInfo/ARM/lowerbdgdeclare_vla.ll
@@ -19,18 +19,18 @@ target triple = "thumbv7-apple-ios8.0.0"
 ; Function Attrs: nounwind optsize readnone
 define void @run(float %r) #0 {
 entry:
-  tail call void @llvm.dbg.declare(metadata float %r, metadata !11, metadata !MDExpression()), !dbg !22
+  tail call void @llvm.dbg.declare(metadata float %r, metadata !11, metadata !DIExpression()), !dbg !22
   %conv = fptosi float %r to i32, !dbg !23
-  tail call void @llvm.dbg.declare(metadata i32 %conv, metadata !12, metadata !MDExpression()), !dbg !23
+  tail call void @llvm.dbg.declare(metadata i32 %conv, metadata !12, metadata !DIExpression()), !dbg !23
   %vla = alloca float, i32 %conv, align 4, !dbg !24
-  tail call void @llvm.dbg.declare(metadata float* %vla, metadata !14, metadata !MDExpression(DW_OP_deref)), !dbg !24
+  tail call void @llvm.dbg.declare(metadata float* %vla, metadata !14, metadata !DIExpression(DW_OP_deref)), !dbg !24
 ; The VLA alloca should be described by a dbg.declare:
 ; CHECK: call void @llvm.dbg.declare(metadata float* %vla, metadata ![[VLA:.*]], metadata {{.*}})
 ; The VLA alloca and following store into the array should not be lowered to like this:
 ; CHECK-NOT:  call void @llvm.dbg.value(metadata float %r, i64 0, metadata ![[VLA]])
 ; the backend interprets this as "vla has the location of %r".
   store float %r, float* %vla, align 4, !dbg !25, !tbaa !26
-  tail call void @llvm.dbg.value(metadata i32 0, i64 0, metadata !18, metadata !MDExpression()), !dbg !30
+  tail call void @llvm.dbg.value(metadata i32 0, i64 0, metadata !18, metadata !DIExpression()), !dbg !30
   %cmp8 = icmp sgt i32 %conv, 0, !dbg !30
   br i1 %cmp8, label %for.body, label %for.end, !dbg !30
 
@@ -41,7 +41,7 @@ for.body:                                         ; preds = %entry, %for.body.fo
   %div = fdiv float %0, %r, !dbg !31
   store float %div, float* %arrayidx2, align 4, !dbg !31, !tbaa !26
   %inc = add nsw i32 %i.09, 1, !dbg !30
-  tail call void @llvm.dbg.value(metadata i32 %inc, i64 0, metadata !18, metadata !MDExpression()), !dbg !30
+  tail call void @llvm.dbg.value(metadata i32 %inc, i64 0, metadata !18, metadata !DIExpression()), !dbg !30
   %exitcond = icmp eq i32 %inc, %conv, !dbg !30
   br i1 %exitcond, label %for.end, label %for.body.for.body_crit_edge, !dbg !30
 
@@ -67,37 +67,37 @@ attributes #1 = { nounwind readnone }
 !llvm.module.flags = !{!20, !33}
 !llvm.ident = !{!21}
 
-!0 = !MDCompileUnit(language: DW_LANG_C99, producer: "clang version 3.4 ", isOptimized: true, emissionKind: 0, file: !1, enums: !2, retainedTypes: !2, subprograms: !3, globals: !2, imports: !2)
-!1 = !MDFile(filename: "<unknown>", directory: "/Volumes/Data/radar/15464571")
+!0 = !DICompileUnit(language: DW_LANG_C99, producer: "clang version 3.4 ", isOptimized: true, emissionKind: 0, file: !1, enums: !2, retainedTypes: !2, subprograms: !3, globals: !2, imports: !2)
+!1 = !DIFile(filename: "<unknown>", directory: "/Volumes/Data/radar/15464571")
 !2 = !{}
 !3 = !{!4}
-!4 = !MDSubprogram(name: "run", line: 1, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: true, scopeLine: 2, file: !5, scope: !6, type: !7, function: void (float)* @run, variables: !10)
-!5 = !MDFile(filename: "test.c", directory: "/Volumes/Data/radar/15464571")
-!6 = !MDFile(filename: "test.c", directory: "/Volumes/Data/radar/15464571")
-!7 = !MDSubroutineType(types: !8)
+!4 = !DISubprogram(name: "run", line: 1, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: true, scopeLine: 2, file: !5, scope: !6, type: !7, function: void (float)* @run, variables: !10)
+!5 = !DIFile(filename: "test.c", directory: "/Volumes/Data/radar/15464571")
+!6 = !DIFile(filename: "test.c", directory: "/Volumes/Data/radar/15464571")
+!7 = !DISubroutineType(types: !8)
 !8 = !{null, !9}
-!9 = !MDBasicType(tag: DW_TAG_base_type, name: "float", size: 32, align: 32, encoding: DW_ATE_float)
+!9 = !DIBasicType(tag: DW_TAG_base_type, name: "float", size: 32, align: 32, encoding: DW_ATE_float)
 !10 = !{!11, !12, !14, !18}
-!11 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "r", line: 1, arg: 1, scope: !4, file: !6, type: !9)
-!12 = !MDLocalVariable(tag: DW_TAG_auto_variable, name: "count", line: 3, scope: !4, file: !6, type: !13)
-!13 = !MDBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
-!14 = !MDLocalVariable(tag: DW_TAG_auto_variable, name: "vla", line: 4, scope: !4, file: !6, type: !15)
-!15 = !MDCompositeType(tag: DW_TAG_array_type, align: 32, baseType: !9, elements: !16)
+!11 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "r", line: 1, arg: 1, scope: !4, file: !6, type: !9)
+!12 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "count", line: 3, scope: !4, file: !6, type: !13)
+!13 = !DIBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
+!14 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "vla", line: 4, scope: !4, file: !6, type: !15)
+!15 = !DICompositeType(tag: DW_TAG_array_type, align: 32, baseType: !9, elements: !16)
 !16 = !{!17}
-!17 = !MDSubrange(count: -1)
-!18 = !MDLocalVariable(tag: DW_TAG_auto_variable, name: "i", line: 6, scope: !19, file: !6, type: !13)
-!19 = distinct !MDLexicalBlock(line: 6, column: 0, file: !5, scope: !4)
+!17 = !DISubrange(count: -1)
+!18 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "i", line: 6, scope: !19, file: !6, type: !13)
+!19 = distinct !DILexicalBlock(line: 6, column: 0, file: !5, scope: !4)
 !20 = !{i32 2, !"Dwarf Version", i32 2}
 !21 = !{!"clang version 3.4 "}
-!22 = !MDLocation(line: 1, scope: !4)
-!23 = !MDLocation(line: 3, scope: !4)
-!24 = !MDLocation(line: 4, scope: !4)
-!25 = !MDLocation(line: 5, scope: !4)
+!22 = !DILocation(line: 1, scope: !4)
+!23 = !DILocation(line: 3, scope: !4)
+!24 = !DILocation(line: 4, scope: !4)
+!25 = !DILocation(line: 5, scope: !4)
 !26 = !{!27, !27, i64 0}
 !27 = !{!"float", !28, i64 0}
 !28 = !{!"omnipotent char", !29, i64 0}
 !29 = !{!"Simple C/C++ TBAA"}
-!30 = !MDLocation(line: 6, scope: !19)
-!31 = !MDLocation(line: 7, scope: !19)
-!32 = !MDLocation(line: 8, scope: !4)
+!30 = !DILocation(line: 6, scope: !19)
+!31 = !DILocation(line: 7, scope: !19)
+!32 = !DILocation(line: 8, scope: !4)
 !33 = !{i32 1, !"Debug Info Version", i32 3}
diff --git a/test/DebugInfo/ARM/multiple-constant-uses-drops-dbgloc.ll b/test/DebugInfo/ARM/multiple-constant-uses-drops-dbgloc.ll
new file mode 100644
index 00000000000..e68c5b82416
--- /dev/null
+++ b/test/DebugInfo/ARM/multiple-constant-uses-drops-dbgloc.ll
@@ -0,0 +1,54 @@
+; RUN: llc -filetype=asm -asm-verbose=0 < %s | FileCheck %s
+
+; char ch;
+; int b;
+;
+; void proc (void)
+; {
+;     ch = 'A';
+;     b = 0; // <== this should have correct location
+; }
+
+; CHECK: .loc 1 7 7
+; CHECK: mov  r{{[0-9]}}, #0
+
+target datalayout = "e-m:e-p:32:32-i64:64-v128:64:128-a:0:32-n32-S64"
+target triple = "armv7--linux-gnueabihf"
+
+@ch = common global i8 0, align 1
+@b = common global i32 0, align 4
+
+; Function Attrs: nounwind
+define void @proc() #0 {
+entry:
+  store i8 65, i8* @ch, align 1, !dbg !17
+  store i32 0, i32* @b, align 4, !dbg !18
+  ret void, !dbg !19
+}
+
+attributes #0 = { nounwind "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="cortex-a8" "target-features"="+neon,+vfp3" "unsafe-fp-math"="false" "use-soft-float"="false" }
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!12, !13, !14, !15}
+!llvm.ident = !{!16}
+
+!0 = !DICompileUnit(language: DW_LANG_C99, file: !1, producer: "", isOptimized: false, subprograms: !3)
+!1 = !DIFile(filename: "test.c", directory: "/home/user/clang/build")
+!2 = !{}
+!3 = !{!4}
+!4 = !DISubprogram(name: "proc", scope: !1, file: !1, line: 4, type: !5, isLocal: false, isDefinition: true, scopeLine: 5, flags: DIFlagPrototyped, isOptimized: false, function: void ()* @proc, variables: !2)
+!5 = !DISubroutineType(types: !6)
+!6 = !{null}
+!7 = !{!8, !10}
+!8 = !DIGlobalVariable(name: "ch", scope: !0, file: !1, line: 1, type: !9, isLocal: false, isDefinition: true, variable: i8* @ch)
+!9 = !DIBasicType(name: "char", size: 8, align: 8, encoding: DW_ATE_unsigned_char)
+!10 = !DIGlobalVariable(name: "b", scope: !0, file: !1, line: 2, type: !11, isLocal: false, isDefinition: true, variable: i32* @b)
+!11 = !DIBasicType(name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
+!12 = !{i32 2, !"Dwarf Version", i32 4}
+!13 = !{i32 2, !"Debug Info Version", i32 3}
+!14 = !{i32 1, !"wchar_size", i32 4}
+!15 = !{i32 1, !"min_enum_size", i32 4}
+!16 = !{!"clang version 3.7.0 (http://llvm.org/git/clang.git 9b0abb9df531ef7928c8182120e1869affca17d5) (http://llvm.org/git/llvm.git b1e759524dd94f7ce1e24935daed8383927e96c1)"}
+!17 = !DILocation(line: 6, column: 8, scope: !4)
+!18 = !DILocation(line: 7, column: 7, scope: !4)
+!19 = !DILocation(line: 8, column: 1, scope: !4)
diff --git a/test/DebugInfo/ARM/s-super-register.ll b/test/DebugInfo/ARM/s-super-register.ll
index b47566c785d..def87695ccc 100644
--- a/test/DebugInfo/ARM/s-super-register.ll
+++ b/test/DebugInfo/ARM/s-super-register.ll
@@ -12,7 +12,7 @@ target triple = "thumbv7-apple-macosx10.6.7"
 define void @_Z3foov() optsize ssp {
 entry:
   %call = tail call float @_Z3barv() optsize, !dbg !11
-  tail call void @llvm.dbg.value(metadata float %call, i64 0, metadata !5, metadata !MDExpression()), !dbg !11
+  tail call void @llvm.dbg.value(metadata float %call, i64 0, metadata !5, metadata !DIExpression()), !dbg !11
   %call16 = tail call float @_Z2f2v() optsize, !dbg !12
   %cmp7 = fcmp olt float %call, %call16, !dbg !12
   br i1 %cmp7, label %for.body, label %for.end, !dbg !12
@@ -40,24 +40,24 @@ declare void @llvm.dbg.value(metadata, i64, metadata, metadata) nounwind readnon
 !llvm.dbg.cu = !{!0}
 !llvm.module.flags = !{!20}
 
-!0 = !MDCompileUnit(language: DW_LANG_C_plus_plus, producer: "clang version 3.0 (trunk 130845)", isOptimized: true, emissionKind: 1, file: !18, enums: !19, retainedTypes: !19, subprograms: !16, imports:  null)
-!1 = !MDSubprogram(name: "foo", linkageName: "_Z3foov", line: 5, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: true, scopeLine: 5, file: !18, scope: !2, type: !3, function: void ()* @_Z3foov, variables: !17)
-!2 = !MDFile(filename: "k.cc", directory: "/private/tmp")
-!3 = !MDSubroutineType(types: !4)
+!0 = !DICompileUnit(language: DW_LANG_C_plus_plus, producer: "clang version 3.0 (trunk 130845)", isOptimized: true, emissionKind: 1, file: !18, enums: !19, retainedTypes: !19, subprograms: !16, imports:  null)
+!1 = !DISubprogram(name: "foo", linkageName: "_Z3foov", line: 5, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: true, scopeLine: 5, file: !18, scope: !2, type: !3, function: void ()* @_Z3foov, variables: !17)
+!2 = !DIFile(filename: "k.cc", directory: "/private/tmp")
+!3 = !DISubroutineType(types: !4)
 !4 = !{null}
-!5 = !MDLocalVariable(tag: DW_TAG_auto_variable, name: "k", line: 6, scope: !6, file: !2, type: !7)
-!6 = distinct !MDLexicalBlock(line: 5, column: 12, file: !18, scope: !1)
-!7 = !MDBasicType(tag: DW_TAG_base_type, name: "float", size: 32, align: 32, encoding: DW_ATE_float)
-!8 = !MDLocalVariable(tag: DW_TAG_auto_variable, name: "y", line: 8, scope: !9, file: !2, type: !7)
-!9 = distinct !MDLexicalBlock(line: 7, column: 25, file: !18, scope: !10)
-!10 = distinct !MDLexicalBlock(line: 7, column: 3, file: !18, scope: !6)
-!11 = !MDLocation(line: 6, column: 18, scope: !6)
-!12 = !MDLocation(line: 7, column: 3, scope: !6)
-!13 = !MDLocation(line: 8, column: 20, scope: !9)
-!14 = !MDLocation(line: 7, column: 20, scope: !10)
-!15 = !MDLocation(line: 10, column: 1, scope: !6)
+!5 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "k", line: 6, scope: !6, file: !2, type: !7)
+!6 = distinct !DILexicalBlock(line: 5, column: 12, file: !18, scope: !1)
+!7 = !DIBasicType(tag: DW_TAG_base_type, name: "float", size: 32, align: 32, encoding: DW_ATE_float)
+!8 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "y", line: 8, scope: !9, file: !2, type: !7)
+!9 = distinct !DILexicalBlock(line: 7, column: 25, file: !18, scope: !10)
+!10 = distinct !DILexicalBlock(line: 7, column: 3, file: !18, scope: !6)
+!11 = !DILocation(line: 6, column: 18, scope: !6)
+!12 = !DILocation(line: 7, column: 3, scope: !6)
+!13 = !DILocation(line: 8, column: 20, scope: !9)
+!14 = !DILocation(line: 7, column: 20, scope: !10)
+!15 = !DILocation(line: 10, column: 1, scope: !6)
 !16 = !{!1}
 !17 = !{!5, !8}
-!18 = !MDFile(filename: "k.cc", directory: "/private/tmp")
+!18 = !DIFile(filename: "k.cc", directory: "/private/tmp")
 !19 = !{}
 !20 = !{i32 1, !"Debug Info Version", i32 3}
diff --git a/test/DebugInfo/ARM/selectiondag-deadcode.ll b/test/DebugInfo/ARM/selectiondag-deadcode.ll
index 9175eee4f52..1b6528269f7 100644
--- a/test/DebugInfo/ARM/selectiondag-deadcode.ll
+++ b/test/DebugInfo/ARM/selectiondag-deadcode.ll
@@ -13,15 +13,15 @@ _ZN7Vector39NormalizeEv.exit:                     ; preds = %1, %0
   ; and SelectionDAGISel crashes.  It should definitely not
   ; crash. Drop the dbg_value instead.
   ; CHECK-NOT: "matrix"
-  tail call void @llvm.dbg.declare(metadata %class.Matrix3.0.6.10* %agg.result, metadata !45, metadata !MDExpression(DW_OP_deref))
+  tail call void @llvm.dbg.declare(metadata %class.Matrix3.0.6.10* %agg.result, metadata !45, metadata !DIExpression(DW_OP_deref))
   %2 = getelementptr inbounds %class.Matrix3.0.6.10, %class.Matrix3.0.6.10* %agg.result, i32 0, i32 0, i32 8
   ret void
 }
 declare void @llvm.dbg.declare(metadata, metadata, metadata) #1
 declare arm_aapcscc void @_ZL4Sqrtd() #2
-!4 = !MDCompositeType(tag: DW_TAG_class_type, name: "Matrix3", line: 20, size: 288, align: 32, file: !5, identifier: "_ZTS7Matrix3")
-!5 = !MDFile(filename: "test.ii", directory: "/Volumes/Data/radar/15094721")
-!39 = !MDSubprogram(name: "GetMatrix", linkageName: "_Z9GetMatrixv", line: 32, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: true, scopeLine: 32, file: !5, scope: !40, type: !41, function: void (%class.Matrix3.0.6.10*)* @_Z9GetMatrixv)
-!40 = !MDFile(filename: "test.ii", directory: "/Volumes/Data/radar/15094721")
-!41 = !MDSubroutineType(types: null)
-!45 = !MDLocalVariable(tag: DW_TAG_auto_variable, name: "matrix", line: 35, scope: !39, file: !40, type: !4)
+!4 = !DICompositeType(tag: DW_TAG_class_type, name: "Matrix3", line: 20, size: 288, align: 32, file: !5, identifier: "_ZTS7Matrix3")
+!5 = !DIFile(filename: "test.ii", directory: "/Volumes/Data/radar/15094721")
+!39 = !DISubprogram(name: "GetMatrix", linkageName: "_Z9GetMatrixv", line: 32, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: true, scopeLine: 32, file: !5, scope: !40, type: !41, function: void (%class.Matrix3.0.6.10*)* @_Z9GetMatrixv)
+!40 = !DIFile(filename: "test.ii", directory: "/Volumes/Data/radar/15094721")
+!41 = !DISubroutineType(types: null)
+!45 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "matrix", line: 35, scope: !39, file: !40, type: !4)
diff --git a/test/DebugInfo/ARM/single-constant-use-preserves-dbgloc.ll b/test/DebugInfo/ARM/single-constant-use-preserves-dbgloc.ll
new file mode 100644
index 00000000000..396eb4fe194
--- /dev/null
+++ b/test/DebugInfo/ARM/single-constant-use-preserves-dbgloc.ll
@@ -0,0 +1,72 @@
+; RUN: llc -filetype=asm -asm-verbose=0 < %s | FileCheck %s
+
+; int main()
+; {
+;     int x = 0;
+;     if (x > 0)
+;         return x;
+;     x = -1; // <== this line should have correct debug location
+;     return -1;
+; }
+
+; CHECK: .loc 1 6 7
+; CHECK: mvn
+
+target datalayout = "e-m:e-p:32:32-i64:64-v128:64:128-a:0:32-n32-S64"
+target triple = "armv7--linux-gnueabihf"
+
+; Function Attrs: nounwind
+define i32 @main() {
+entry:
+  %retval = alloca i32, align 4
+  %x = alloca i32, align 4
+  store i32 0, i32* %retval
+  call void @llvm.dbg.declare(metadata i32* %x, metadata !10, metadata !11), !dbg !12
+  store i32 0, i32* %x, align 4, !dbg !12
+  %0 = load i32, i32* %x, align 4, !dbg !13
+  %cmp = icmp sgt i32 %0, 0, !dbg !15
+  br i1 %cmp, label %if.then, label %if.end, !dbg !16
+
+if.then:                                          ; preds = %entry
+  %1 = load i32, i32* %x, align 4, !dbg !17
+  store i32 %1, i32* %retval, !dbg !18
+  br label %return, !dbg !18
+
+if.end:                                           ; preds = %entry
+  store i32 -1, i32* %x, align 4, !dbg !19
+  store i32 -1, i32* %retval, !dbg !20
+  br label %return, !dbg !20
+
+return:                                           ; preds = %if.end, %if.then
+  %2 = load i32, i32* %retval, !dbg !21
+  ret i32 %2, !dbg !21
+}
+
+; Function Attrs: nounwind readnone
+declare void @llvm.dbg.declare(metadata, metadata, metadata)
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!8, !9}
+
+!0 = !DICompileUnit(language: DW_LANG_C99, file: !1, producer: "", isOptimized: false, subprograms: !3)
+!1 = !DIFile(filename: "test.c", directory: "/home/user/clang/build")
+!2 = !{}
+!3 = !{!4}
+!4 = !DISubprogram(name: "main", scope: !1, file: !1, line: 1, type: !5, isLocal: false, isDefinition: true, scopeLine: 2, isOptimized: false, function: i32 ()* @main, variables: !2)
+!5 = !DISubroutineType(types: !6)
+!6 = !{!7}
+!7 = !DIBasicType(name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
+!8 = !{i32 2, !"Dwarf Version", i32 4}
+!9 = !{i32 2, !"Debug Info Version", i32 3}
+!10 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "x", scope: !4, file: !1, line: 3, type: !7)
+!11 = !DIExpression()
+!12 = !DILocation(line: 3, column: 9, scope: !4)
+!13 = !DILocation(line: 4, column: 9, scope: !14)
+!14 = distinct !DILexicalBlock(scope: !4, file: !1, line: 4, column: 9)
+!15 = !DILocation(line: 4, column: 11, scope: !14)
+!16 = !DILocation(line: 4, column: 9, scope: !4)
+!17 = !DILocation(line: 5, column: 13, scope: !14)
+!18 = !DILocation(line: 5, column: 9, scope: !14)
+!19 = !DILocation(line: 6, column: 7, scope: !4)
+!20 = !DILocation(line: 7, column: 5, scope: !4)
+!21 = !DILocation(line: 8, column: 1, scope: !4)
diff --git a/test/DebugInfo/ARM/tls.ll b/test/DebugInfo/ARM/tls.ll
index 8d2359e7ab1..0482d65c58a 100644
--- a/test/DebugInfo/ARM/tls.ll
+++ b/test/DebugInfo/ARM/tls.ll
@@ -1,5 +1,8 @@
-; RUN: llc -O0 -filetype=asm -mtriple=armv7-linux-gnuehabi < %s | FileCheck %s
-;
+; RUN: llc -O0 -filetype=asm -mtriple=armv7-linux-gnuehabi < %s \
+; RUN:     | FileCheck %s --check-prefix=CHECK
+; RUN: llc -O0 -filetype=asm -mtriple=armv7-linux-gnuehabi -emulated-tls < %s \
+; RUN:     | FileCheck %s --check-prefix=EMU
+
 ; Generated with clang with source
 ; __thread int x;
 
@@ -16,13 +19,16 @@
 ; The debug relocation of the address of the tls variable
 ; CHECK: .long x(tlsldo)
 
-!0 = !MDCompileUnit(language: DW_LANG_C99, producer: "clang version 3.5 ", isOptimized: false, emissionKind: 0, file: !1, enums: !2, retainedTypes: !2, subprograms: !2, globals: !3, imports: !2)
-!1 = !MDFile(filename: "tls.c", directory: "/tmp")
+; TODO: Add expected output for -emulated-tls tests.
+; EMU-NOT: .long x(tlsldo)
+
+!0 = !DICompileUnit(language: DW_LANG_C99, producer: "clang version 3.5 ", isOptimized: false, emissionKind: 0, file: !1, enums: !2, retainedTypes: !2, subprograms: !2, globals: !3, imports: !2)
+!1 = !DIFile(filename: "tls.c", directory: "/tmp")
 !2 = !{}
 !3 = !{!4}
-!4 = !MDGlobalVariable(name: "x", line: 1, isLocal: false, isDefinition: true, scope: null, file: !5, type: !6, variable: i32* @x)
-!5 = !MDFile(filename: "tls.c", directory: "/tmp")
-!6 = !MDBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
+!4 = !DIGlobalVariable(name: "x", line: 1, isLocal: false, isDefinition: true, scope: null, file: !5, type: !6, variable: i32* @x)
+!5 = !DIFile(filename: "tls.c", directory: "/tmp")
+!6 = !DIBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
 !7 = !{i32 2, !"Dwarf Version", i32 4}
 !8 = !{i32 1, !"Debug Info Version", i32 3}
 !9 = !{!"clang version 3.5 "}
diff --git a/test/DebugInfo/COFF/asan-module-ctor.ll b/test/DebugInfo/COFF/asan-module-ctor.ll
index ee693d6ae85..3f8b129ca9b 100644
--- a/test/DebugInfo/COFF/asan-module-ctor.ll
+++ b/test/DebugInfo/COFF/asan-module-ctor.ll
@@ -82,14 +82,14 @@ attributes #0 = { nounwind sanitize_address "less-precise-fpmad"="false" "no-fra
 !llvm.module.flags = !{!7, !8}
 !llvm.ident = !{!9}
 
-!0 = !MDCompileUnit(language: DW_LANG_C99, producer: "clang version 3.5.0 ", isOptimized: false, emissionKind: 2, file: !1, enums: !2, retainedTypes: !2, subprograms: !3, globals: !2, imports: !2)
-!1 = !MDFile(filename: "asan.c", directory: "D:\5C")
+!0 = !DICompileUnit(language: DW_LANG_C99, producer: "clang version 3.5.0 ", isOptimized: false, emissionKind: 2, file: !1, enums: !2, retainedTypes: !2, subprograms: !3, globals: !2, imports: !2)
+!1 = !DIFile(filename: "asan.c", directory: "D:\5C")
 !2 = !{}
 !3 = !{!4}
-!4 = !MDSubprogram(name: "foo", line: 1, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 1, file: !1, scope: !5, type: !6, function: i32 ()* @foo, variables: !2)
-!5 = !MDFile(filename: "asan.c", directory: "D:C")
-!6 = !MDSubroutineType(types: !2)
+!4 = !DISubprogram(name: "foo", line: 1, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 1, file: !1, scope: !5, type: !6, function: i32 ()* @foo, variables: !2)
+!5 = !DIFile(filename: "asan.c", directory: "D:C")
+!6 = !DISubroutineType(types: !2)
 !7 = !{i32 2, !"Dwarf Version", i32 4}
 !8 = !{i32 1, !"Debug Info Version", i32 3}
 !9 = !{!"clang version 3.5.0 "}
-!10 = !MDLocation(line: 2, scope: !4)
+!10 = !DILocation(line: 2, scope: !4)
diff --git a/test/DebugInfo/COFF/asan-module-without-functions.ll b/test/DebugInfo/COFF/asan-module-without-functions.ll
index fbfaa744e80..88eec67a4bd 100644
--- a/test/DebugInfo/COFF/asan-module-without-functions.ll
+++ b/test/DebugInfo/COFF/asan-module-without-functions.ll
@@ -45,8 +45,8 @@ define internal void @asan.module_dtor() {
 !llvm.module.flags = !{!3, !4}
 !llvm.ident = !{!5}
 
-!0 = !MDCompileUnit(language: DW_LANG_C99, producer: "clang version 3.5.0 ", isOptimized: false, emissionKind: 2, file: !1, enums: !2, retainedTypes: !2, subprograms: !2, globals: !2, imports: !2)
-!1 = !MDFile(filename: "asan.c", directory: "D:\5C")
+!0 = !DICompileUnit(language: DW_LANG_C99, producer: "clang version 3.5.0 ", isOptimized: false, emissionKind: 2, file: !1, enums: !2, retainedTypes: !2, subprograms: !2, globals: !2, imports: !2)
+!1 = !DIFile(filename: "asan.c", directory: "D:\5C")
 !2 = !{}
 !3 = !{i32 2, !"Dwarf Version", i32 4}
 !4 = !{i32 1, !"Debug Info Version", i32 3}
diff --git a/test/DebugInfo/COFF/asm.ll b/test/DebugInfo/COFF/asm.ll
index cccf43f178d..bc2a11d066b 100644
--- a/test/DebugInfo/COFF/asm.ll
+++ b/test/DebugInfo/COFF/asm.ll
@@ -51,7 +51,7 @@
 ; X86-NEXT: [[F2_START]]:
 ; X86-NEXT: .secrel32 _f
 ; X86-NEXT: .secidx _f
-; X86-NEXT: .short 0
+; X86-NEXT: .short 1
 ; X86-NEXT: .long [[END_OF_F]]-_f
 ; X86-NEXT: [[FILE_SEGMENT_START:[^:]*]]:
 ; X86-NEXT: .long   0
@@ -63,6 +63,12 @@
 ; X86-NEXT: .long   5
 ; X86-NEXT: .long [[RETURN_STMT]]-_f
 ; X86-NEXT: .long   6
+; X86-NEXT: .short  0
+; X86-NEXT: .short  0
+; X86-NEXT: .short  0
+; X86-NEXT: .short  0
+; X86-NEXT: .short  0
+; X86-NEXT: .short  0
 ; X86-NEXT: [[FILE_SEGMENT_END]]:
 ; X86-NEXT: [[F2_END]]:
 ; File index to string table offset subsection
@@ -101,6 +107,7 @@
 ; OBJ32-NEXT: ]
 ; OBJ32:      FunctionLineTable [
 ; OBJ32-NEXT:   Name: _f
+; OBJ32-NEXT:   Flags: 0x1
 ; OBJ32-NEXT:   CodeSize: 0x6
 ; OBJ32-NEXT:   FilenameSegment [
 ; OBJ32-NEXT:     Filename: D:\asm.c
@@ -110,6 +117,12 @@
 ; OBJ32-NEXT:     +0x0: 4
 ; OBJ32-NEXT:     +0x0: 5
 ; OBJ32-NEXT:     +0x5: 6
+; OBJ32-NEXT:     ColStart: 0
+; OBJ32-NEXT:     ColEnd: 0
+; OBJ32-NEXT:     ColStart: 0
+; OBJ32-NEXT:     ColEnd: 0
+; OBJ32-NEXT:     ColStart: 0
+; OBJ32-NEXT:     ColEnd: 0
 ; OBJ32-NEXT:   ]
 ; OBJ32-NEXT: ]
 ; OBJ32:    }
@@ -157,7 +170,7 @@
 ; X64-NEXT: [[F2_START]]:
 ; X64-NEXT: .secrel32 f
 ; X64-NEXT: .secidx f
-; X64-NEXT: .short 0
+; X64-NEXT: .short 1
 ; X64-NEXT: .long [[END_OF_F]]-f
 ; X64-NEXT: [[FILE_SEGMENT_START:[^:]*]]:
 ; X64-NEXT: .long   0
@@ -171,6 +184,14 @@
 ; X64-NEXT: .long   5
 ; X64-NEXT: .long [[EPILOG_AND_RET]]-f
 ; X64-NEXT: .long   6
+; X64-NEXT: .short  0
+; X64-NEXT: .short  0
+; X64-NEXT: .short  0
+; X64-NEXT: .short  0
+; X64-NEXT: .short  0
+; X64-NEXT: .short  0
+; X64-NEXT: .short  0
+; X64-NEXT: .short  0
 ; X64-NEXT: [[FILE_SEGMENT_END]]:
 ; X64-NEXT: [[F2_END]]:
 ; File index to string table offset subsection
@@ -209,6 +230,7 @@
 ; OBJ64-NEXT: ]
 ; OBJ64:      FunctionLineTable [
 ; OBJ64-NEXT:   Name: f
+; OBJ64-NEXT:   Flags: 0x1
 ; OBJ64-NEXT:   CodeSize: 0xE
 ; OBJ64-NEXT:   FilenameSegment [
 ; OBJ64-NEXT:     Filename: D:\asm.c
@@ -218,6 +240,14 @@
 ; OBJ64-NEXT:     +0x4: 4
 ; OBJ64-NEXT:     +0x4: 5
 ; OBJ64-NEXT:     +0x9: 6
+; OBJ64-NEXT:     ColStart: 0
+; OBJ64-NEXT:     ColEnd: 0
+; OBJ64-NEXT:     ColStart: 0
+; OBJ64-NEXT:     ColEnd: 0
+; OBJ64-NEXT:     ColStart: 0
+; OBJ64-NEXT:     ColEnd: 0
+; OBJ64-NEXT:     ColStart: 0
+; OBJ64-NEXT:     ColEnd: 0
 ; OBJ64-NEXT:   ]
 ; OBJ64-NEXT: ]
 ; OBJ64:    }
@@ -240,18 +270,18 @@ attributes #2 = { nounwind }
 !llvm.module.flags = !{!9, !10}
 !llvm.ident = !{!11}
 
-!0 = !MDCompileUnit(language: DW_LANG_C99, producer: "clang version 3.5 ", isOptimized: false, emissionKind: 0, file: !1, enums: !2, retainedTypes: !2, subprograms: !3, globals: !2, imports: !2)
-!1 = !MDFile(filename: "<unknown>", directory: "D:\5C")
+!0 = !DICompileUnit(language: DW_LANG_C99, producer: "clang version 3.5 ", isOptimized: false, emissionKind: 0, file: !1, enums: !2, retainedTypes: !2, subprograms: !3, globals: !2, imports: !2)
+!1 = !DIFile(filename: "<unknown>", directory: "D:\5C")
 !2 = !{}
 !3 = !{!4}
-!4 = !MDSubprogram(name: "f", line: 3, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 3, file: !5, scope: !6, type: !7, function: void ()* @f, variables: !2)
-!5 = !MDFile(filename: "asm.c", directory: "D:\5C")
-!6 = !MDFile(filename: "asm.c", directory: "D:C")
-!7 = !MDSubroutineType(types: !8)
+!4 = !DISubprogram(name: "f", line: 3, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 3, file: !5, scope: !6, type: !7, function: void ()* @f, variables: !2)
+!5 = !DIFile(filename: "asm.c", directory: "D:\5C")
+!6 = !DIFile(filename: "asm.c", directory: "D:C")
+!7 = !DISubroutineType(types: !8)
 !8 = !{null}
 !9 = !{i32 2, !"Dwarf Version", i32 4}
 !10 = !{i32 1, !"Debug Info Version", i32 3}
 !11 = !{!"clang version 3.5 "}
-!12 = !MDLocation(line: 4, scope: !4)
-!13 = !MDLocation(line: 5, scope: !4)
-!14 = !MDLocation(line: 6, scope: !4)
+!12 = !DILocation(line: 4, scope: !4)
+!13 = !DILocation(line: 5, scope: !4)
+!14 = !DILocation(line: 6, scope: !4)
diff --git a/test/DebugInfo/COFF/cpp-mangling.ll b/test/DebugInfo/COFF/cpp-mangling.ll
index 1a52820f3bf..137377847aa 100644
--- a/test/DebugInfo/COFF/cpp-mangling.ll
+++ b/test/DebugInfo/COFF/cpp-mangling.ll
@@ -29,15 +29,15 @@ attributes #0 = { nounwind "less-precise-fpmad"="false" "no-frame-pointer-elim"=
 !llvm.module.flags = !{!8, !9}
 !llvm.ident = !{!10}
 
-!0 = !MDCompileUnit(language: DW_LANG_C_plus_plus, producer: "clang version 3.6.0 ", isOptimized: false, emissionKind: 2, file: !1, enums: !2, retainedTypes: !2, subprograms: !3, globals: !2, imports: !2)
-!1 = !MDFile(filename: "<stdin>", directory: "D:\5C")
+!0 = !DICompileUnit(language: DW_LANG_C_plus_plus, producer: "clang version 3.6.0 ", isOptimized: false, emissionKind: 2, file: !1, enums: !2, retainedTypes: !2, subprograms: !3, globals: !2, imports: !2)
+!1 = !DIFile(filename: "<stdin>", directory: "D:\5C")
 !2 = !{}
 !3 = !{!4}
-!4 = !MDSubprogram(name: "bar", line: 2, isLocal: false, isDefinition: true, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 2, file: !5, scope: !6, type: !7, function: i32 (i32)* @"\01?bar@foo@@YAHH@Z", variables: !2)
-!5 = !MDFile(filename: "src.cpp", directory: "D:\5C")
-!6 = !MDFile(filename: "src.cpp", directory: "D:C")
-!7 = !MDSubroutineType(types: !2)
+!4 = !DISubprogram(name: "bar", line: 2, isLocal: false, isDefinition: true, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 2, file: !5, scope: !6, type: !7, function: i32 (i32)* @"\01?bar@foo@@YAHH@Z", variables: !2)
+!5 = !DIFile(filename: "src.cpp", directory: "D:\5C")
+!6 = !DIFile(filename: "src.cpp", directory: "D:C")
+!7 = !DISubroutineType(types: !2)
 !8 = !{i32 2, !"Dwarf Version", i32 4}
 !9 = !{i32 2, !"Debug Info Version", i32 3}
 !10 = !{!"clang version 3.6.0 "}
-!11 = !MDLocation(line: 3, scope: !4)
+!11 = !DILocation(line: 3, scope: !4)
diff --git a/test/DebugInfo/COFF/multifile.ll b/test/DebugInfo/COFF/multifile.ll
index 91e2d9cd036..3dedacebc14 100644
--- a/test/DebugInfo/COFF/multifile.ll
+++ b/test/DebugInfo/COFF/multifile.ll
@@ -58,7 +58,7 @@
 ; X86-NEXT: [[F2_START]]:
 ; X86-NEXT: .secrel32 _f
 ; X86-NEXT: .secidx _f
-; X86-NEXT: .short 0
+; X86-NEXT: .short 1
 ; X86-NEXT: .long [[END_OF_F]]-_f
 ; Segment for file 'D:\\one.c' begins
 ; X86-NEXT: [[FILE_SEGMENT_START:[^:]*]]:
@@ -67,6 +67,8 @@
 ; X86-NEXT: .long [[FILE_SEGMENT_END:.*]]-[[FILE_SEGMENT_START]]
 ; X86-NEXT: .long [[CALL_LINE_1]]-_f
 ; X86-NEXT: .long   1
+; X86-NEXT: .short  0
+; X86-NEXT: .short  0
 ; X86-NEXT: [[FILE_SEGMENT_END]]:
 ; Segment for file 'D:\\two.c' begins
 ; X86-NEXT: [[FILE_SEGMENT_START:[^:]*]]:
@@ -75,6 +77,8 @@
 ; X86-NEXT: .long [[FILE_SEGMENT_END:.*]]-[[FILE_SEGMENT_START]]
 ; X86-NEXT: .long [[CALL_LINE_2]]-_f
 ; X86-NEXT: .long   2
+; X86-NEXT: .short  0
+; X86-NEXT: .short  0
 ; X86-NEXT: [[FILE_SEGMENT_END]]:
 ; A new segment for file 'D:\\one.c' begins
 ; X86-NEXT: [[FILE_SEGMENT_START:[^:]*]]:
@@ -85,6 +89,10 @@
 ; X86-NEXT: .long   7
 ; X86-NEXT: .long [[RETURN_STMT]]-_f
 ; X86-NEXT: .long   8
+; X86-NEXT: .short  0
+; X86-NEXT: .short  0
+; X86-NEXT: .short  0
+; X86-NEXT: .short  0
 ; X86-NEXT: [[FILE_SEGMENT_END]]:
 ; X86-NEXT: [[F2_END]]:
 ; File index to string table offset subsection
@@ -126,19 +134,28 @@
 ; OBJ32-NEXT: ]
 ; OBJ32:      FunctionLineTable [
 ; OBJ32-NEXT:   Name: _f
+; OBJ32-NEXT:   Flags: 0x1
 ; OBJ32-NEXT:   CodeSize: 0x10
 ; OBJ32-NEXT:   FilenameSegment [
 ; OBJ32-NEXT:     Filename: D:\one.c
 ; OBJ32-NEXT:     +0x0: 1
+; OBJ32-NEXT:     ColStart: 0
+; OBJ32-NEXT:     ColEnd: 0
 ; OBJ32-NEXT:   ]
 ; OBJ32-NEXT:   FilenameSegment [
 ; OBJ32-NEXT:     Filename: D:\two.c
 ; OBJ32-NEXT:     +0x5: 2
+; OBJ32-NEXT:     ColStart: 0
+; OBJ32-NEXT:     ColEnd: 0
 ; OBJ32-NEXT:   ]
 ; OBJ32-NEXT:   FilenameSegment [
 ; OBJ32-NEXT:     Filename: D:\one.c
 ; OBJ32-NEXT:     +0xA: 7
 ; OBJ32-NEXT:     +0xF: 8
+; OBJ32-NEXT:     ColStart: 0
+; OBJ32-NEXT:     ColEnd: 0
+; OBJ32-NEXT:     ColStart: 0
+; OBJ32-NEXT:     ColEnd: 0
 ; OBJ32-NEXT:   ]
 ; OBJ32-NEXT: ]
 ; OBJ32:    }
@@ -189,7 +206,7 @@
 ; X64-NEXT: [[F2_START]]:
 ; X64-NEXT: .secrel32 f
 ; X64-NEXT: .secidx f
-; X64-NEXT: .short 0
+; X64-NEXT: .short 1
 ; X64-NEXT: .long [[END_OF_F]]-f
 ; Segment for file 'D:\\input.c' begins
 ; X64-NEXT: [[FILE_SEGMENT_START:[^:]*]]:
@@ -198,6 +215,8 @@
 ; X64-NEXT: .long [[FILE_SEGMENT_END:.*]]-[[FILE_SEGMENT_START]]
 ; X64-NEXT: .long [[START]]-f
 ; X64-NEXT: .long   3
+; X64-NEXT: .short  0
+; X64-NEXT: .short  0
 ; X64-NEXT: [[FILE_SEGMENT_END]]:
 ; Segment for file 'D:\\one.c' begins
 ; X64-NEXT: [[FILE_SEGMENT_START:[^:]*]]:
@@ -206,6 +225,8 @@
 ; X64-NEXT: .long [[FILE_SEGMENT_END:.*]]-[[FILE_SEGMENT_START]]
 ; X64-NEXT: .long [[CALL_LINE_1]]-f
 ; X64-NEXT: .long   1
+; X64-NEXT: .short  0
+; X64-NEXT: .short  0
 ; X64-NEXT: [[FILE_SEGMENT_END]]:
 ; Segment for file 'D:\\two.c' begins
 ; X64-NEXT: [[FILE_SEGMENT_START:[^:]*]]:
@@ -214,6 +235,8 @@
 ; X64-NEXT: .long [[FILE_SEGMENT_END:.*]]-[[FILE_SEGMENT_START]]
 ; X64-NEXT: .long [[CALL_LINE_2]]-f
 ; X64-NEXT: .long   2
+; X64-NEXT: .short  0
+; X64-NEXT: .short  0
 ; X64-NEXT: [[FILE_SEGMENT_END]]:
 ; A new segment for file 'D:\\one.c' begins
 ; X64-NEXT: [[FILE_SEGMENT_START:[^:]*]]:
@@ -224,6 +247,10 @@
 ; X64-NEXT: .long   7
 ; X64-NEXT: .long [[EPILOG_AND_RET]]-f
 ; X64-NEXT: .long   8
+; X64-NEXT: .short  0
+; X64-NEXT: .short  0
+; X64-NEXT: .short  0
+; X64-NEXT: .short  0
 ; X64-NEXT: [[FILE_SEGMENT_END]]:
 ; X64-NEXT: [[F2_END]]:
 ; File index to string table offset subsection
@@ -269,23 +296,34 @@
 ; OBJ64-NEXT: ]
 ; OBJ64:      FunctionLineTable [
 ; OBJ64-NEXT:   Name: f
+; OBJ64-NEXT:   Flags: 0x1
 ; OBJ64-NEXT:   CodeSize: 0x18
 ; OBJ64-NEXT:   FilenameSegment [
 ; OBJ64-NEXT:     Filename: D:\input.c
 ; OBJ64-NEXT:     +0x0: 3
+; OBJ64-NEXT:     ColStart: 0
+; OBJ64-NEXT:     ColEnd: 0
 ; OBJ64-NEXT:   ]
 ; OBJ64-NEXT:   FilenameSegment [
 ; OBJ64-NEXT:     Filename: D:\one.c
 ; OBJ64-NEXT:     +0x4: 1
+; OBJ64-NEXT:     ColStart: 0
+; OBJ64-NEXT:     ColEnd: 0
 ; OBJ64-NEXT:   ]
 ; OBJ64-NEXT:   FilenameSegment [
 ; OBJ64-NEXT:     Filename: D:\two.c
 ; OBJ64-NEXT:     +0x9: 2
+; OBJ64-NEXT:     ColStart: 0
+; OBJ64-NEXT:     ColEnd: 0
 ; OBJ64-NEXT:   ]
 ; OBJ64-NEXT:   FilenameSegment [
 ; OBJ64-NEXT:     Filename: D:\one.c
 ; OBJ64-NEXT:     +0xE: 7
 ; OBJ64-NEXT:     +0x13: 8
+; OBJ64-NEXT:     ColStart: 0
+; OBJ64-NEXT:     ColEnd: 0
+; OBJ64-NEXT:     ColStart: 0
+; OBJ64-NEXT:     ColEnd: 0
 ; OBJ64-NEXT:   ]
 ; OBJ64-NEXT: ]
 ; OBJ64:    }
@@ -308,23 +346,23 @@ attributes #1 = { "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "
 !llvm.module.flags = !{!9, !10}
 !llvm.ident = !{!11}
 
-!0 = !MDCompileUnit(language: DW_LANG_C99, producer: "clang version 3.5 ", isOptimized: false, emissionKind: 0, file: !1, enums: !2, retainedTypes: !2, subprograms: !3, globals: !2, imports: !2)
-!1 = !MDFile(filename: "<unknown>", directory: "D:\5C")
+!0 = !DICompileUnit(language: DW_LANG_C99, producer: "clang version 3.5 ", isOptimized: false, emissionKind: 0, file: !1, enums: !2, retainedTypes: !2, subprograms: !3, globals: !2, imports: !2)
+!1 = !DIFile(filename: "<unknown>", directory: "D:\5C")
 !2 = !{}
 !3 = !{!4}
-!4 = !MDSubprogram(name: "f", line: 3, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 3, file: !5, scope: !6, type: !7, function: void ()* @f, variables: !2)
-!5 = !MDFile(filename: "input.c", directory: "D:\5C")
-!6 = !MDFile(filename: "input.c", directory: "D:C")
-!7 = !MDSubroutineType(types: !8)
+!4 = !DISubprogram(name: "f", line: 3, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 3, file: !5, scope: !6, type: !7, function: void ()* @f, variables: !2)
+!5 = !DIFile(filename: "input.c", directory: "D:\5C")
+!6 = !DIFile(filename: "input.c", directory: "D:C")
+!7 = !DISubroutineType(types: !8)
 !8 = !{null}
 !9 = !{i32 2, !"Dwarf Version", i32 4}
 !10 = !{i32 1, !"Debug Info Version", i32 3}
 !11 = !{!"clang version 3.5 "}
-!12 = !MDLocation(line: 1, scope: !13)
-!13 = !MDLexicalBlockFile(discriminator: 0, file: !14, scope: !4)
-!14 = !MDFile(filename: "one.c", directory: "D:\5C")
-!15 = !MDLocation(line: 2, scope: !16)
-!16 = !MDLexicalBlockFile(discriminator: 0, file: !17, scope: !4)
-!17 = !MDFile(filename: "two.c", directory: "D:\5C")
-!18 = !MDLocation(line: 7, scope: !13)
-!19 = !MDLocation(line: 8, scope: !13)
+!12 = !DILocation(line: 1, scope: !13)
+!13 = !DILexicalBlockFile(discriminator: 0, file: !14, scope: !4)
+!14 = !DIFile(filename: "one.c", directory: "D:\5C")
+!15 = !DILocation(line: 2, scope: !16)
+!16 = !DILexicalBlockFile(discriminator: 0, file: !17, scope: !4)
+!17 = !DIFile(filename: "two.c", directory: "D:\5C")
+!18 = !DILocation(line: 7, scope: !13)
+!19 = !DILocation(line: 8, scope: !13)
diff --git a/test/DebugInfo/COFF/multifunction.ll b/test/DebugInfo/COFF/multifunction.ll
index 48a927f3b01..bbf97dd4afc 100644
--- a/test/DebugInfo/COFF/multifunction.ll
+++ b/test/DebugInfo/COFF/multifunction.ll
@@ -82,7 +82,7 @@
 ; X86-NEXT: [[F2_START]]:
 ; X86-NEXT: .secrel32       _x
 ; X86-NEXT: .secidx _x
-; X86-NEXT: .short 0
+; X86-NEXT: .short 1
 ; X86-NEXT: .long [[END_OF_X]]-_x
 ; X86-NEXT: [[FILE_SEGMENT_START:[^:]*]]:
 ; X86-NEXT: .long   0
@@ -92,6 +92,10 @@
 ; X86-NEXT: .long   4
 ; X86-NEXT: .long [[X_RETURN]]-_x
 ; X86-NEXT: .long   5
+; X86-NEXT: .short 42
+; X86-NEXT: .short 42
+; X86-NEXT: .short 43
+; X86-NEXT: .short 43
 ; X86-NEXT: [[FILE_SEGMENT_END]]:
 ; X86-NEXT: [[F2_END]]:
 ; Symbol subsection for y
@@ -121,7 +125,7 @@
 ; X86-NEXT: [[F2_START]]:
 ; X86-NEXT: .secrel32       _y
 ; X86-NEXT: .secidx _y
-; X86-NEXT: .short 0
+; X86-NEXT: .short 1
 ; X86-NEXT: .long [[END_OF_Y]]-_y
 ; X86-NEXT: [[FILE_SEGMENT_START:[^:]*]]:
 ; X86-NEXT: .long   0
@@ -131,6 +135,10 @@
 ; X86-NEXT: .long   8
 ; X86-NEXT: .long [[Y_RETURN]]-_y
 ; X86-NEXT: .long   9
+; X86-NEXT: .short 52
+; X86-NEXT: .short 52
+; X86-NEXT: .short 53
+; X86-NEXT: .short 53
 ; X86-NEXT: [[FILE_SEGMENT_END]]:
 ; X86-NEXT: [[F2_END]]:
 ; Symbol subsection for f
@@ -160,7 +168,7 @@
 ; X86-NEXT: [[F2_START]]:
 ; X86-NEXT: .secrel32 _f
 ; X86-NEXT: .secidx _f
-; X86-NEXT: .short 0
+; X86-NEXT: .short 1
 ; X86-NEXT: .long [[END_OF_F]]-_f
 ; X86-NEXT: [[FILE_SEGMENT_START:[^:]*]]:
 ; X86-NEXT: .long   0
@@ -174,6 +182,14 @@
 ; X86-NEXT: .long   14
 ; X86-NEXT: .long [[F_RETURN]]-_f
 ; X86-NEXT: .long   15
+; X86-NEXT: .short 62
+; X86-NEXT: .short 62
+; X86-NEXT: .short 63
+; X86-NEXT: .short 63
+; X86-NEXT: .short 72
+; X86-NEXT: .short 72
+; X86-NEXT: .short 73
+; X86-NEXT: .short 73
 ; X86-NEXT: [[FILE_SEGMENT_END]]:
 ; X86-NEXT: [[F2_END]]:
 ; File index to string table offset subsection
@@ -198,14 +214,14 @@
 ; OBJ32-NEXT:   0x30 IMAGE_REL_I386_SECTION _x
 ; OBJ32-NEXT:   0x44 IMAGE_REL_I386_SECREL _x
 ; OBJ32-NEXT:   0x48 IMAGE_REL_I386_SECTION _x
-; OBJ32-NEXT:   0x94 IMAGE_REL_I386_SECREL _y
-; OBJ32-NEXT:   0x98 IMAGE_REL_I386_SECTION _y
-; OBJ32-NEXT:   0xAC IMAGE_REL_I386_SECREL _y
-; OBJ32-NEXT:   0xB0 IMAGE_REL_I386_SECTION _y
-; OBJ32-NEXT:   0xFC IMAGE_REL_I386_SECREL _f
-; OBJ32-NEXT:   0x100 IMAGE_REL_I386_SECTION _f
-; OBJ32-NEXT:   0x114 IMAGE_REL_I386_SECREL _f
-; OBJ32-NEXT:   0x118 IMAGE_REL_I386_SECTION _f
+; OBJ32-NEXT:   0x9C IMAGE_REL_I386_SECREL _y
+; OBJ32-NEXT:   0xA0 IMAGE_REL_I386_SECTION _y
+; OBJ32-NEXT:   0xB4 IMAGE_REL_I386_SECREL _y
+; OBJ32-NEXT:   0xB8 IMAGE_REL_I386_SECTION _y
+; OBJ32-NEXT:   0x10C IMAGE_REL_I386_SECREL _f
+; OBJ32-NEXT:   0x110 IMAGE_REL_I386_SECTION _f
+; OBJ32-NEXT:   0x124 IMAGE_REL_I386_SECREL _f
+; OBJ32-NEXT:   0x128 IMAGE_REL_I386_SECTION _f
 ; OBJ32-NEXT: ]
 ; OBJ32:      Subsection [
 ; OBJ32-NEXT:   Type: 0xF1
@@ -248,24 +264,35 @@
 ; OBJ32:      ]
 ; OBJ32:      FunctionLineTable [
 ; OBJ32-NEXT:   Name: _x
+; OBJ32-NEXT:   Flags: 0x1
 ; OBJ32-NEXT:   CodeSize: 0x6
 ; OBJ32-NEXT:   FilenameSegment [
 ; OBJ32-NEXT:     Filename: D:\source.c
 ; OBJ32-NEXT:     +0x0: 4
 ; OBJ32-NEXT:     +0x5: 5
+; OBJ32-NEXT:     ColStart: 42
+; OBJ32-NEXT:     ColEnd: 42
+; OBJ32-NEXT:     ColStart: 43
+; OBJ32-NEXT:     ColEnd: 43
 ; OBJ32-NEXT:   ]
 ; OBJ32-NEXT: ]
 ; OBJ32-NEXT: FunctionLineTable [
 ; OBJ32-NEXT:   Name: _y
+; OBJ32-NEXT:   Flags: 0x1
 ; OBJ32-NEXT:   CodeSize: 0x6
 ; OBJ32-NEXT:   FilenameSegment [
 ; OBJ32-NEXT:     Filename: D:\source.c
 ; OBJ32-NEXT:     +0x0: 8
 ; OBJ32-NEXT:     +0x5: 9
+; OBJ32-NEXT:     ColStart: 52
+; OBJ32-NEXT:     ColEnd: 52
+; OBJ32-NEXT:     ColStart: 53
+; OBJ32-NEXT:     ColEnd: 53
 ; OBJ32-NEXT:   ]
 ; OBJ32-NEXT: ]
 ; OBJ32-NEXT: FunctionLineTable [
 ; OBJ32-NEXT:   Name: _f
+; OBJ32-NEXT:   Flags: 0x1
 ; OBJ32-NEXT:   CodeSize: 0x10
 ; OBJ32-NEXT:   FilenameSegment [
 ; OBJ32-NEXT:     Filename: D:\source.c
@@ -273,6 +300,14 @@
 ; OBJ32-NEXT:     +0x5: 13
 ; OBJ32-NEXT:     +0xA: 14
 ; OBJ32-NEXT:     +0xF: 15
+; OBJ32-NEXT:     ColStart: 62
+; OBJ32-NEXT:     ColEnd: 62
+; OBJ32-NEXT:     ColStart: 63
+; OBJ32-NEXT:     ColEnd: 63
+; OBJ32-NEXT:     ColStart: 72
+; OBJ32-NEXT:     ColEnd: 72
+; OBJ32-NEXT:     ColStart: 73
+; OBJ32-NEXT:     ColEnd: 73
 ; OBJ32-NEXT:   ]
 ; OBJ32-NEXT: ]
 ; OBJ32:    }
@@ -349,7 +384,7 @@
 ; X64-NEXT: [[F2_START]]:
 ; X64-NEXT: .secrel32 x
 ; X64-NEXT: .secidx x
-; X64-NEXT: .short 0
+; X64-NEXT: .short 1
 ; X64-NEXT: .long [[END_OF_X]]-x
 ; X64-NEXT: [[FILE_SEGMENT_START:[^:]*]]:
 ; X64-NEXT: .long   0
@@ -361,6 +396,12 @@
 ; X64-NEXT: .long   4
 ; X64-NEXT: .long [[X_EPILOG_AND_RET]]-x
 ; X64-NEXT: .long   5
+; X64-NEXT: .short 0
+; X64-NEXT: .short 0
+; X64-NEXT: .short 42
+; X64-NEXT: .short 42
+; X64-NEXT: .short 43
+; X64-NEXT: .short 43
 ; X64-NEXT: [[FILE_SEGMENT_END]]:
 ; X64-NEXT: [[F2_END]]:
 ; Symbol subsection for y
@@ -390,7 +431,7 @@
 ; X64-NEXT: [[F2_START]]:
 ; X64-NEXT: .secrel32 y
 ; X64-NEXT: .secidx y
-; X64-NEXT: .short 0
+; X64-NEXT: .short 1
 ; X64-NEXT: .long [[END_OF_Y]]-y
 ; X64-NEXT: [[FILE_SEGMENT_START:[^:]*]]:
 ; X64-NEXT: .long   0
@@ -402,6 +443,12 @@
 ; X64-NEXT: .long   8
 ; X64-NEXT: .long [[Y_EPILOG_AND_RET]]-y
 ; X64-NEXT: .long   9
+; X64-NEXT: .short 0
+; X64-NEXT: .short 0
+; X64-NEXT: .short 52
+; X64-NEXT: .short 52
+; X64-NEXT: .short 53
+; X64-NEXT: .short 53
 ; X64-NEXT: [[FILE_SEGMENT_END]]:
 ; X64-NEXT: [[F2_END]]:
 ; Symbol subsection for f
@@ -431,7 +478,7 @@
 ; X64-NEXT: [[F2_START]]:
 ; X64-NEXT: .secrel32 f
 ; X64-NEXT: .secidx f
-; X64-NEXT: .short 0
+; X64-NEXT: .short 1
 ; X64-NEXT: .long [[END_OF_F]]-f
 ; X64-NEXT: [[FILE_SEGMENT_START:[^:]*]]:
 ; X64-NEXT: .long   0
@@ -447,6 +494,16 @@
 ; X64-NEXT: .long   14
 ; X64-NEXT: .long [[F_EPILOG_AND_RET]]-f
 ; X64-NEXT: .long   15
+; X64-NEXT: .short 0
+; X64-NEXT: .short 0
+; X64-NEXT: .short 62
+; X64-NEXT: .short 62
+; X64-NEXT: .short 63
+; X64-NEXT: .short 63
+; X64-NEXT: .short 72
+; X64-NEXT: .short 72
+; X64-NEXT: .short 73
+; X64-NEXT: .short 73
 ; X64-NEXT: [[FILE_SEGMENT_END]]:
 ; X64-NEXT: [[F2_END]]:
 ; File index to string table offset subsection
@@ -471,14 +528,14 @@
 ; OBJ64-NEXT:   0x30 IMAGE_REL_AMD64_SECTION x
 ; OBJ64-NEXT:   0x44 IMAGE_REL_AMD64_SECREL x
 ; OBJ64-NEXT:   0x48 IMAGE_REL_AMD64_SECTION x
-; OBJ64-NEXT:   0x9C IMAGE_REL_AMD64_SECREL y
-; OBJ64-NEXT:   0xA0 IMAGE_REL_AMD64_SECTION y
-; OBJ64-NEXT:   0xB4 IMAGE_REL_AMD64_SECREL y
-; OBJ64-NEXT:   0xB8 IMAGE_REL_AMD64_SECTION y
-; OBJ64-NEXT:   0x10C IMAGE_REL_AMD64_SECREL f
-; OBJ64-NEXT:   0x110 IMAGE_REL_AMD64_SECTION f
+; OBJ64-NEXT:   0xA8 IMAGE_REL_AMD64_SECREL y
+; OBJ64-NEXT:   0xAC IMAGE_REL_AMD64_SECTION y
+; OBJ64-NEXT:   0xC0 IMAGE_REL_AMD64_SECREL y
+; OBJ64-NEXT:   0xC4 IMAGE_REL_AMD64_SECTION y
 ; OBJ64-NEXT:   0x124 IMAGE_REL_AMD64_SECREL f
 ; OBJ64-NEXT:   0x128 IMAGE_REL_AMD64_SECTION f
+; OBJ64-NEXT:   0x13C IMAGE_REL_AMD64_SECREL f
+; OBJ64-NEXT:   0x140 IMAGE_REL_AMD64_SECTION f
 ; OBJ64-NEXT: ]
 ; OBJ64:      Subsection [
 ; OBJ64-NEXT:   Type: 0xF1
@@ -521,26 +578,41 @@
 ; OBJ64:      ]
 ; OBJ64:      FunctionLineTable [
 ; OBJ64-NEXT:   Name: x
+; OBJ64-NEXT:   Flags: 0x1
 ; OBJ64-NEXT:   CodeSize: 0xE
 ; OBJ64-NEXT:   FilenameSegment [
 ; OBJ64-NEXT:     Filename: D:\source.c
 ; OBJ64-NEXT:     +0x0: 3
 ; OBJ64-NEXT:     +0x4: 4
 ; OBJ64-NEXT:     +0x9: 5
+; OBJ64-NEXT:     ColStart: 0
+; OBJ64-NEXT:     ColEnd: 0
+; OBJ64-NEXT:     ColStart: 42
+; OBJ64-NEXT:     ColEnd: 42
+; OBJ64-NEXT:     ColStart: 43
+; OBJ64-NEXT:     ColEnd: 43
 ; OBJ64-NEXT:   ]
 ; OBJ64-NEXT: ]
 ; OBJ64-NEXT: FunctionLineTable [
 ; OBJ64-NEXT:   Name: y
+; OBJ64-NEXT:   Flags: 0x1
 ; OBJ64-NEXT:   CodeSize: 0xE
 ; OBJ64-NEXT:   FilenameSegment [
 ; OBJ64-NEXT:     Filename: D:\source.c
 ; OBJ64-NEXT:     +0x0: 7
 ; OBJ64-NEXT:     +0x4: 8
 ; OBJ64-NEXT:     +0x9: 9
+; OBJ64-NEXT:     ColStart: 0
+; OBJ64-NEXT:     ColEnd: 0
+; OBJ64-NEXT:     ColStart: 52
+; OBJ64-NEXT:     ColEnd: 52
+; OBJ64-NEXT:     ColStart: 53
+; OBJ64-NEXT:     ColEnd: 53
 ; OBJ64-NEXT:   ]
 ; OBJ64-NEXT: ]
 ; OBJ64-NEXT: FunctionLineTable [
 ; OBJ64-NEXT:   Name: f
+; OBJ64-NEXT:   Flags: 0x1
 ; OBJ64-NEXT:   CodeSize: 0x18
 ; OBJ64-NEXT:   FilenameSegment [
 ; OBJ64-NEXT:     Filename: D:\source.c
@@ -549,6 +621,16 @@
 ; OBJ64-NEXT:     +0x9: 13
 ; OBJ64-NEXT:     +0xE: 14
 ; OBJ64-NEXT:     +0x13: 15
+; OBJ64-NEXT:     ColStart: 0
+; OBJ64-NEXT:     ColEnd: 0
+; OBJ64-NEXT:     ColStart: 62
+; OBJ64-NEXT:     ColEnd: 62
+; OBJ64-NEXT:     ColStart: 63
+; OBJ64-NEXT:     ColEnd: 63
+; OBJ64-NEXT:     ColStart: 72
+; OBJ64-NEXT:     ColEnd: 72
+; OBJ64-NEXT:     ColStart: 73
+; OBJ64-NEXT:     ColEnd: 73
 ; OBJ64-NEXT:   ]
 ; OBJ64-NEXT: ]
 ; OBJ64:    }
@@ -585,25 +667,25 @@ attributes #1 = { "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "
 !llvm.module.flags = !{!11, !12}
 !llvm.ident = !{!13}
 
-!0 = !MDCompileUnit(language: DW_LANG_C99, producer: "clang version 3.5 ", isOptimized: false, emissionKind: 0, file: !1, enums: !2, retainedTypes: !2, subprograms: !3, globals: !2, imports: !2)
-!1 = !MDFile(filename: "<unknown>", directory: "D:\5C")
+!0 = !DICompileUnit(language: DW_LANG_C99, producer: "clang version 3.5 ", isOptimized: false, emissionKind: 0, file: !1, enums: !2, retainedTypes: !2, subprograms: !3, globals: !2, imports: !2)
+!1 = !DIFile(filename: "<unknown>", directory: "D:\5C")
 !2 = !{}
 !3 = !{!4, !9, !10}
-!4 = !MDSubprogram(name: "x", line: 3, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 3, file: !5, scope: !6, type: !7, function: void ()* @x, variables: !2)
-!5 = !MDFile(filename: "source.c", directory: "D:\5C")
-!6 = !MDFile(filename: "source.c", directory: "D:C")
-!7 = !MDSubroutineType(types: !8)
+!4 = !DISubprogram(name: "x", line: 3, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 3, file: !5, scope: !6, type: !7, function: void ()* @x, variables: !2)
+!5 = !DIFile(filename: "source.c", directory: "D:\5C")
+!6 = !DIFile(filename: "source.c", directory: "D:C")
+!7 = !DISubroutineType(types: !8)
 !8 = !{null}
-!9 = !MDSubprogram(name: "y", line: 7, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 7, file: !5, scope: !6, type: !7, function: void ()* @y, variables: !2)
-!10 = !MDSubprogram(name: "f", line: 11, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 11, file: !5, scope: !6, type: !7, function: void ()* @f, variables: !2)
+!9 = !DISubprogram(name: "y", line: 7, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 7, file: !5, scope: !6, type: !7, function: void ()* @y, variables: !2)
+!10 = !DISubprogram(name: "f", line: 11, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 11, file: !5, scope: !6, type: !7, function: void ()* @f, variables: !2)
 !11 = !{i32 2, !"Dwarf Version", i32 4}
 !12 = !{i32 1, !"Debug Info Version", i32 3}
 !13 = !{!"clang version 3.5 "}
-!14 = !MDLocation(line: 4, scope: !4)
-!15 = !MDLocation(line: 5, scope: !4)
-!16 = !MDLocation(line: 8, scope: !9)
-!17 = !MDLocation(line: 9, scope: !9)
-!18 = !MDLocation(line: 12, scope: !10)
-!19 = !MDLocation(line: 13, scope: !10)
-!20 = !MDLocation(line: 14, scope: !10)
-!21 = !MDLocation(line: 15, scope: !10)
+!14 = !DILocation(line: 4, column: 42, scope: !4)
+!15 = !DILocation(line: 5, column: 43, scope: !4)
+!16 = !DILocation(line: 8, column: 52, scope: !9)
+!17 = !DILocation(line: 9, column: 53, scope: !9)
+!18 = !DILocation(line: 12, column: 62, scope: !10)
+!19 = !DILocation(line: 13, column: 63, scope: !10)
+!20 = !DILocation(line: 14, column: 72, scope: !10)
+!21 = !DILocation(line: 15, column: 73, scope: !10)
diff --git a/test/DebugInfo/COFF/simple.ll b/test/DebugInfo/COFF/simple.ll
index 2f713a984b4..0d9857c7831 100644
--- a/test/DebugInfo/COFF/simple.ll
+++ b/test/DebugInfo/COFF/simple.ll
@@ -49,7 +49,7 @@
 ; X86-NEXT: [[F2_START]]:
 ; X86-NEXT: .secrel32 _f
 ; X86-NEXT: .secidx _f
-; X86-NEXT: .short  0
+; X86-NEXT: .short  1
 ; X86-NEXT: .long [[END_OF_F]]-_f
 ; X86-NEXT: [[FILE_SEGMENT_START:[^:]*]]:
 ; X86-NEXT: .long   0
@@ -59,6 +59,10 @@
 ; X86-NEXT: .long   4
 ; X86-NEXT: .long [[RETURN_STMT]]-_f
 ; X86-NEXT: .long   5
+; X86-NEXT: .short  0
+; X86-NEXT: .short  0
+; X86-NEXT: .short  0
+; X86-NEXT: .short  0
 ; X86-NEXT: [[FILE_SEGMENT_END]]:
 ; X86-NEXT: [[F2_END]]:
 ; File index to string table offset subsection
@@ -97,11 +101,16 @@
 ; OBJ32-NEXT: ]
 ; OBJ32:      FunctionLineTable [
 ; OBJ32-NEXT:   Name: _f
+; OBJ32-NEXT:   Flags: 0x1
 ; OBJ32-NEXT:   CodeSize: 0x6
 ; OBJ32-NEXT:   FilenameSegment [
 ; OBJ32-NEXT:     Filename: D:\test.c
 ; OBJ32-NEXT:     +0x0: 4
 ; OBJ32-NEXT:     +0x5: 5
+; OBJ32-NEXT:     ColStart: 0
+; OBJ32-NEXT:     ColEnd: 0
+; OBJ32-NEXT:     ColStart: 0
+; OBJ32-NEXT:     ColEnd: 0
 ; OBJ32-NEXT:   ]
 ; OBJ32-NEXT: ]
 ; OBJ32:    }
@@ -148,7 +157,7 @@
 ; X64-NEXT: [[F2_START]]:
 ; X64-NEXT: .secrel32 f
 ; X64-NEXT: .secidx f
-; X64-NEXT: .short  0
+; X64-NEXT: .short  1
 ; X64-NEXT: .long [[END_OF_F]]-f
 ; X64-NEXT: [[FILE_SEGMENT_START:[^:]*]]:
 ; X64-NEXT: .long   0
@@ -160,6 +169,12 @@
 ; X64-NEXT: .long   4
 ; X64-NEXT: .long [[EPILOG_AND_RET]]-f
 ; X64-NEXT: .long   5
+; X64-NEXT: .short  0
+; X64-NEXT: .short  0
+; X64-NEXT: .short  0
+; X64-NEXT: .short  0
+; X64-NEXT: .short  0
+; X64-NEXT: .short  0
 ; X64-NEXT: [[FILE_SEGMENT_END]]:
 ; X64-NEXT: [[F2_END]]:
 ; File index to string table offset subsection
@@ -198,12 +213,19 @@
 ; OBJ64-NEXT: ]
 ; OBJ64:      FunctionLineTable [
 ; OBJ64-NEXT:   Name: f
+; OBJ64-NEXT:   Flags: 0x1
 ; OBJ64-NEXT:   CodeSize: 0xE
 ; OBJ64-NEXT:   FilenameSegment [
 ; OBJ64-NEXT:     Filename: D:\test.c
 ; OBJ64-NEXT:     +0x0: 3
 ; OBJ64-NEXT:     +0x4: 4
 ; OBJ64-NEXT:     +0x9: 5
+; OBJ64-NEXT:     ColStart: 0
+; OBJ64-NEXT:     ColEnd: 0
+; OBJ64-NEXT:     ColStart: 0
+; OBJ64-NEXT:     ColEnd: 0
+; OBJ64-NEXT:     ColStart: 0
+; OBJ64-NEXT:     ColEnd: 0
 ; OBJ64-NEXT:   ]
 ; OBJ64-NEXT: ]
 ; OBJ64:    }
@@ -224,17 +246,17 @@ attributes #1 = { "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "
 !llvm.module.flags = !{!9, !10}
 !llvm.ident = !{!11}
 
-!0 = !MDCompileUnit(language: DW_LANG_C99, producer: "clang version 3.5 ", isOptimized: false, emissionKind: 0, file: !1, enums: !2, retainedTypes: !2, subprograms: !3, globals: !2, imports: !2)
-!1 = !MDFile(filename: "<unknown>", directory: "D:\5C")
+!0 = !DICompileUnit(language: DW_LANG_C99, producer: "clang version 3.5 ", isOptimized: false, emissionKind: 0, file: !1, enums: !2, retainedTypes: !2, subprograms: !3, globals: !2, imports: !2)
+!1 = !DIFile(filename: "<unknown>", directory: "D:\5C")
 !2 = !{}
 !3 = !{!4}
-!4 = !MDSubprogram(name: "f", line: 3, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 3, file: !5, scope: !6, type: !7, function: void ()* @f, variables: !2)
-!5 = !MDFile(filename: "test.c", directory: "D:\5C")
-!6 = !MDFile(filename: "test.c", directory: "D:C")
-!7 = !MDSubroutineType(types: !8)
+!4 = !DISubprogram(name: "f", line: 3, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 3, file: !5, scope: !6, type: !7, function: void ()* @f, variables: !2)
+!5 = !DIFile(filename: "test.c", directory: "D:\5C")
+!6 = !DIFile(filename: "test.c", directory: "D:C")
+!7 = !DISubroutineType(types: !8)
 !8 = !{null}
 !9 = !{i32 2, !"Dwarf Version", i32 4}
 !10 = !{i32 1, !"Debug Info Version", i32 3}
 !11 = !{!"clang version 3.5 "}
-!12 = !MDLocation(line: 4, scope: !4)
-!13 = !MDLocation(line: 5, scope: !4)
+!12 = !DILocation(line: 4, scope: !4)
+!13 = !DILocation(line: 5, scope: !4)
diff --git a/test/DebugInfo/COFF/tail-call-without-lexical-scopes.ll b/test/DebugInfo/COFF/tail-call-without-lexical-scopes.ll
index 4fb8a5be885..054a2e89a8f 100644
--- a/test/DebugInfo/COFF/tail-call-without-lexical-scopes.ll
+++ b/test/DebugInfo/COFF/tail-call-without-lexical-scopes.ll
@@ -62,18 +62,18 @@ attributes #2 = { nounwind }
 !llvm.module.flags = !{!8, !9}
 !llvm.ident = !{!10}
 
-!0 = !MDCompileUnit(language: DW_LANG_C_plus_plus, producer: "clang version 3.5.0 ", isOptimized: true, emissionKind: 2, file: !1, enums: !2, retainedTypes: !2, subprograms: !3, globals: !2, imports: !2)
-!1 = !MDFile(filename: "test.cpp", directory: "D:\5C")
+!0 = !DICompileUnit(language: DW_LANG_C_plus_plus, producer: "clang version 3.5.0 ", isOptimized: true, emissionKind: 2, file: !1, enums: !2, retainedTypes: !2, subprograms: !3, globals: !2, imports: !2)
+!1 = !DIFile(filename: "test.cpp", directory: "D:\5C")
 !2 = !{}
 !3 = !{!4, !7}
-!4 = !MDSubprogram(name: "spam", line: 7, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: true, scopeLine: 7, file: !1, scope: !5, type: !6, function: void ()* @"\01?spam@@YAXXZ", variables: !2)
-!5 = !MDFile(filename: "test.cpp", directory: "D:C")
-!6 = !MDSubroutineType(types: !2)
-!7 = !MDSubprogram(name: "bar", line: 3, isLocal: true, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: true, scopeLine: 3, file: !1, scope: !5, type: !6, variables: !2)
+!4 = !DISubprogram(name: "spam", line: 7, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: true, scopeLine: 7, file: !1, scope: !5, type: !6, function: void ()* @"\01?spam@@YAXXZ", variables: !2)
+!5 = !DIFile(filename: "test.cpp", directory: "D:C")
+!6 = !DISubroutineType(types: !2)
+!7 = !DISubprogram(name: "bar", line: 3, isLocal: true, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: true, scopeLine: 3, file: !1, scope: !5, type: !6, variables: !2)
 !8 = !{i32 2, !"Dwarf Version", i32 4}
 !9 = !{i32 1, !"Debug Info Version", i32 3}
 !10 = !{!"clang version 3.5.0 "}
-!11 = !MDLocation(line: 8, scope: !4)
-!12 = !MDLocation(line: 9, scope: !4)
-!13 = !MDLocation(line: 4, scope: !7)
-!14 = !MDLocation(line: 5, scope: !7)
+!11 = !DILocation(line: 8, scope: !4)
+!12 = !DILocation(line: 9, scope: !4)
+!13 = !DILocation(line: 4, scope: !7)
+!14 = !DILocation(line: 5, scope: !7)
diff --git a/test/DebugInfo/Inputs/dwarfdump-test.cc b/test/DebugInfo/Inputs/dwarfdump-test.cc
index 40899986490..14295d3cffa 100644
--- a/test/DebugInfo/Inputs/dwarfdump-test.cc
+++ b/test/DebugInfo/Inputs/dwarfdump-test.cc
@@ -21,3 +21,9 @@ int main() {
 // $ cp dwarfdump-test.cc /tmp/dbginfo
 // $ cd /tmp/dbginfo
 // $ clang++ -g dwarfdump-test.cc -o <output>
+
+// The result is also used as an input to .dwz tool:
+// $ cp <output> output1.dwz
+// $ cp <output> output2.dwz
+// $ dwz -m output.dwz -r output1.dwz output2.dwz
+// $ rm output2.dwz
diff --git a/test/DebugInfo/Inputs/dwarfdump.elf-mips64-64-bit-dwarf b/test/DebugInfo/Inputs/dwarfdump.elf-mips64-64-bit-dwarf
new file mode 100755
index 00000000000..5dbfea52212
Binary files /dev/null and b/test/DebugInfo/Inputs/dwarfdump.elf-mips64-64-bit-dwarf differ
diff --git a/test/DebugInfo/Inputs/gmlt.ll b/test/DebugInfo/Inputs/gmlt.ll
index 78f9c722c76..1436abedfab 100644
--- a/test/DebugInfo/Inputs/gmlt.ll
+++ b/test/DebugInfo/Inputs/gmlt.ll
@@ -131,23 +131,23 @@ attributes #2 = { nounwind }
 !llvm.module.flags = !{!10, !11}
 !llvm.ident = !{!12}
 
-!0 = !MDCompileUnit(language: DW_LANG_C_plus_plus, producer: "clang version 3.6.0 ", isOptimized: false, emissionKind: 2, file: !1, enums: !2, retainedTypes: !2, subprograms: !3, globals: !2, imports: !2)
-!1 = !MDFile(filename: "gmlt.cpp", directory: "/tmp/dbginfo")
+!0 = !DICompileUnit(language: DW_LANG_C_plus_plus, producer: "clang version 3.6.0 ", isOptimized: false, emissionKind: 2, file: !1, enums: !2, retainedTypes: !2, subprograms: !3, globals: !2, imports: !2)
+!1 = !DIFile(filename: "gmlt.cpp", directory: "/tmp/dbginfo")
 !2 = !{}
 !3 = !{!4, !7, !8, !9}
-!4 = !MDSubprogram(name: "f1", line: 1, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 1, file: !1, scope: !5, type: !6, function: void ()* @_Z2f1v, variables: !2)
-!5 = !MDFile(filename: "gmlt.cpp", directory: "/tmp/dbginfo")
-!6 = !MDSubroutineType(types: !2)
-!7 = !MDSubprogram(name: "f2", line: 2, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 2, file: !1, scope: !5, type: !6, function: void ()* @_Z2f2v, variables: !2)
-!8 = !MDSubprogram(name: "f3", line: 3, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 3, file: !1, scope: !5, type: !6, function: void ()* @_Z2f3v, variables: !2)
-!9 = !MDSubprogram(name: "f4", line: 4, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 4, file: !1, scope: !5, type: !6, function: void ()* @_Z2f4v, variables: !2)
+!4 = !DISubprogram(name: "f1", line: 1, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 1, file: !1, scope: !5, type: !6, function: void ()* @_Z2f1v, variables: !2)
+!5 = !DIFile(filename: "gmlt.cpp", directory: "/tmp/dbginfo")
+!6 = !DISubroutineType(types: !2)
+!7 = !DISubprogram(name: "f2", line: 2, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 2, file: !1, scope: !5, type: !6, function: void ()* @_Z2f2v, variables: !2)
+!8 = !DISubprogram(name: "f3", line: 3, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 3, file: !1, scope: !5, type: !6, function: void ()* @_Z2f3v, variables: !2)
+!9 = !DISubprogram(name: "f4", line: 4, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 4, file: !1, scope: !5, type: !6, function: void ()* @_Z2f4v, variables: !2)
 !10 = !{i32 2, !"Dwarf Version", i32 4}
 !11 = !{i32 2, !"Debug Info Version", i32 3}
 !12 = !{!"clang version 3.6.0 "}
-!13 = !MDLocation(line: 1, column: 12, scope: !4)
-!14 = !MDLocation(line: 2, column: 53, scope: !7)
-!15 = !MDLocation(line: 3, column: 44, scope: !8)
-!16 = !MDLocation(line: 3, column: 50, scope: !8)
-!17 = !MDLocation(line: 3, column: 44, scope: !8, inlinedAt: !18)
-!18 = !MDLocation(line: 4, column: 13, scope: !9)
-!19 = !MDLocation(line: 4, column: 19, scope: !9)
+!13 = !DILocation(line: 1, column: 12, scope: !4)
+!14 = !DILocation(line: 2, column: 53, scope: !7)
+!15 = !DILocation(line: 3, column: 44, scope: !8)
+!16 = !DILocation(line: 3, column: 50, scope: !8)
+!17 = !DILocation(line: 3, column: 44, scope: !8, inlinedAt: !18)
+!18 = !DILocation(line: 4, column: 13, scope: !9)
+!19 = !DILocation(line: 4, column: 19, scope: !9)
diff --git a/test/DebugInfo/Inputs/invalid.elf b/test/DebugInfo/Inputs/invalid.elf
new file mode 100644
index 00000000000..23acad80bc9
Binary files /dev/null and b/test/DebugInfo/Inputs/invalid.elf differ
diff --git a/test/DebugInfo/Inputs/invalid.elf.2 b/test/DebugInfo/Inputs/invalid.elf.2
new file mode 100644
index 00000000000..16528a55a93
--- /dev/null
+++ b/test/DebugInfo/Inputs/invalid.elf.2
@@ -0,0 +1 @@
+ELF'L:þB“ELFï



\ No newline at end of file
diff --git a/test/DebugInfo/Inputs/invalid.elf.3 b/test/DebugInfo/Inputs/invalid.elf.3
new file mode 100644
index 00000000000..f4944600004
--- /dev/null
+++ b/test/DebugInfo/Inputs/invalid.elf.3
@@ -0,0 +1 @@
+ELF‚Lþ'ELF



B
\ No newline at end of file
diff --git a/test/DebugInfo/Inputs/line.ll b/test/DebugInfo/Inputs/line.ll
index cb582abed8c..d56c7e01a17 100644
--- a/test/DebugInfo/Inputs/line.ll
+++ b/test/DebugInfo/Inputs/line.ll
@@ -39,17 +39,17 @@ attributes #0 = { nounwind uwtable "less-precise-fpmad"="false" "no-frame-pointe
 !llvm.module.flags = !{!7, !8}
 !llvm.ident = !{!9}
 
-!0 = !MDCompileUnit(language: DW_LANG_C_plus_plus, producer: "clang version 3.7.0 (trunk 227472) (llvm/trunk 227476)", isOptimized: false, emissionKind: 2, file: !1, enums: !2, retainedTypes: !2, subprograms: !3, globals: !2, imports: !2)
-!1 = !MDFile(filename: "line.cpp", directory: "/tmp/dbginfo")
+!0 = !DICompileUnit(language: DW_LANG_C_plus_plus, producer: "clang version 3.7.0 (trunk 227472) (llvm/trunk 227476)", isOptimized: false, emissionKind: 2, file: !1, enums: !2, retainedTypes: !2, subprograms: !3, globals: !2, imports: !2)
+!1 = !DIFile(filename: "line.cpp", directory: "/tmp/dbginfo")
 !2 = !{}
 !3 = !{!4}
-!4 = !MDSubprogram(name: "f", line: 1, isLocal: false, isDefinition: true, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 1, file: !1, scope: !5, type: !6, function: i32 (i32, i32)* @_Z1fii, variables: !2)
-!5 = !MDFile(filename: "line.cpp", directory: "/tmp/dbginfo")
-!6 = !MDSubroutineType(types: !2)
+!4 = !DISubprogram(name: "f", line: 1, isLocal: false, isDefinition: true, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 1, file: !1, scope: !5, type: !6, function: i32 (i32, i32)* @_Z1fii, variables: !2)
+!5 = !DIFile(filename: "line.cpp", directory: "/tmp/dbginfo")
+!6 = !DISubroutineType(types: !2)
 !7 = !{i32 2, !"Dwarf Version", i32 4}
 !8 = !{i32 2, !"Debug Info Version", i32 3}
 !9 = !{!"clang version 3.7.0 (trunk 227472) (llvm/trunk 227476)"}
-!10 = !MDLocation(line: 2, scope: !4)
-!11 = !MDLocation(line: 3, scope: !4)
-!12 = !MDLocation(line: 4, scope: !4)
-!13 = !MDLocation(line: 2, scope: !4)
+!10 = !DILocation(line: 2, scope: !4)
+!11 = !DILocation(line: 3, scope: !4)
+!12 = !DILocation(line: 4, scope: !4)
+!13 = !DILocation(line: 2, scope: !4)
diff --git a/test/DebugInfo/Inputs/test-multiple-macho.o b/test/DebugInfo/Inputs/test-multiple-macho.o
new file mode 100644
index 00000000000..428a1af6826
Binary files /dev/null and b/test/DebugInfo/Inputs/test-multiple-macho.o differ
diff --git a/test/DebugInfo/Inputs/test-simple-macho.o b/test/DebugInfo/Inputs/test-simple-macho.o
new file mode 100644
index 00000000000..8ae4154626c
Binary files /dev/null and b/test/DebugInfo/Inputs/test-simple-macho.o differ
diff --git a/test/DebugInfo/Mips/InlinedFnLocalVar.ll b/test/DebugInfo/Mips/InlinedFnLocalVar.ll
index 48344a95f40..5c67cc9ea98 100644
--- a/test/DebugInfo/Mips/InlinedFnLocalVar.ll
+++ b/test/DebugInfo/Mips/InlinedFnLocalVar.ll
@@ -15,8 +15,8 @@ declare void @llvm.dbg.value(metadata, i64, metadata, metadata) nounwind readnon
 define i32 @bar() nounwind ssp {
 entry:
   %0 = load i32, i32* @i, align 4, !dbg !17            ; <i32> [#uses=2]
-  tail call void @llvm.dbg.value(metadata i32 %0, i64 0, metadata !59, metadata !MDExpression()), !dbg !19
-  tail call void @llvm.dbg.declare(metadata !29, metadata !60, metadata !MDExpression()), !dbg !21
+  tail call void @llvm.dbg.value(metadata i32 %0, i64 0, metadata !59, metadata !DIExpression()), !dbg !19
+  tail call void @llvm.dbg.declare(metadata !29, metadata !60, metadata !DIExpression()), !dbg !21
   %1 = mul nsw i32 %0, %0, !dbg !22               ; <i32> [#uses=2]
   store i32 %1, i32* @i, align 4, !dbg !17
   ret i32 %1, !dbg !23
@@ -25,37 +25,37 @@ entry:
 !llvm.dbg.cu = !{!2}
 !llvm.module.flags = !{!28}
 
-!0 = !MDSubprogram(name: "foo", line: 9, isLocal: true, isDefinition: true, virtualIndex: 6, isOptimized: true, scopeLine: 9, file: !27, scope: !1, type: !3, variables: !24)
-!1 = !MDFile(filename: "bar.c", directory: "/tmp/")
-!2 = !MDCompileUnit(language: DW_LANG_C89, producer: "4.2.1 (Based on Apple Inc. build 5658) (LLVM build)", isOptimized: true, emissionKind: 0, file: !27, enums: !20, retainedTypes: !20, subprograms: !25, globals: !26, imports:  !20)
-!3 = !MDSubroutineType(types: !4)
+!0 = !DISubprogram(name: "foo", line: 9, isLocal: true, isDefinition: true, virtualIndex: 6, isOptimized: true, scopeLine: 9, file: !27, scope: !1, type: !3, variables: !24)
+!1 = !DIFile(filename: "bar.c", directory: "/tmp/")
+!2 = !DICompileUnit(language: DW_LANG_C89, producer: "4.2.1 (Based on Apple Inc. build 5658) (LLVM build)", isOptimized: true, emissionKind: 0, file: !27, enums: !20, retainedTypes: !20, subprograms: !25, globals: !26, imports:  !20)
+!3 = !DISubroutineType(types: !4)
 !4 = !{!5, !5}
-!5 = !MDBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
-!6 = !MDSubprogram(name: "bar", linkageName: "bar", line: 14, isLocal: false, isDefinition: true, virtualIndex: 6, isOptimized: true, file: !27, scope: !1, type: !7, function: i32 ()* @bar)
-!7 = !MDSubroutineType(types: !8)
+!5 = !DIBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
+!6 = !DISubprogram(name: "bar", linkageName: "bar", line: 14, isLocal: false, isDefinition: true, virtualIndex: 6, isOptimized: true, file: !27, scope: !1, type: !7, function: i32 ()* @bar)
+!7 = !DISubroutineType(types: !8)
 !8 = !{!5}
-!9 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "j", line: 9, arg: 0, scope: !0, file: !1, type: !5)
-!10 = !MDLocalVariable(tag: DW_TAG_auto_variable, name: "xyz", line: 10, scope: !11, file: !1, type: !12)
+!9 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "j", line: 9, arg: 1, scope: !0, file: !1, type: !5)
+!10 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "xyz", line: 10, scope: !11, file: !1, type: !12)
 
-!59 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "j", line: 9, arg: 0, scope: !0, file: !1, type: !5)
-!60 = !MDLocalVariable(tag: DW_TAG_auto_variable, name: "xyz", line: 10, scope: !11, file: !1, type: !12)
+!59 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "j", line: 9, arg: 1, scope: !0, file: !1, type: !5)
+!60 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "xyz", line: 10, scope: !11, file: !1, type: !12)
 
-!11 = distinct !MDLexicalBlock(line: 9, column: 0, file: !1, scope: !0)
-!12 = !MDCompositeType(tag: DW_TAG_structure_type, name: "X", line: 10, size: 64, align: 32, file: !27, scope: !0, elements: !13)
+!11 = distinct !DILexicalBlock(line: 9, column: 0, file: !1, scope: !0)
+!12 = !DICompositeType(tag: DW_TAG_structure_type, name: "X", line: 10, size: 64, align: 32, file: !27, scope: !0, elements: !13)
 !13 = !{!14, !15}
-!14 = !MDDerivedType(tag: DW_TAG_member, name: "a", line: 10, size: 32, align: 32, file: !27, scope: !12, baseType: !5)
-!15 = !MDDerivedType(tag: DW_TAG_member, name: "b", line: 10, size: 32, align: 32, offset: 32, file: !27, scope: !12, baseType: !5)
-!16 = !MDGlobalVariable(name: "i", line: 5, isLocal: false, isDefinition: true, scope: !1, file: !1, type: !5, variable: i32* @i)
-!17 = !MDLocation(line: 15, scope: !18)
-!18 = distinct !MDLexicalBlock(line: 14, column: 0, file: !1, scope: !6)
-!19 = !MDLocation(line: 9, scope: !0, inlinedAt: !17)
+!14 = !DIDerivedType(tag: DW_TAG_member, name: "a", line: 10, size: 32, align: 32, file: !27, scope: !12, baseType: !5)
+!15 = !DIDerivedType(tag: DW_TAG_member, name: "b", line: 10, size: 32, align: 32, offset: 32, file: !27, scope: !12, baseType: !5)
+!16 = !DIGlobalVariable(name: "i", line: 5, isLocal: false, isDefinition: true, scope: !1, file: !1, type: !5, variable: i32* @i)
+!17 = !DILocation(line: 15, scope: !18)
+!18 = distinct !DILexicalBlock(line: 14, column: 0, file: !1, scope: !6)
+!19 = !DILocation(line: 9, scope: !0, inlinedAt: !17)
 !20 = !{}
-!21 = !MDLocation(line: 9, scope: !11, inlinedAt: !17)
-!22 = !MDLocation(line: 11, scope: !11, inlinedAt: !17)
-!23 = !MDLocation(line: 16, scope: !18)
+!21 = !DILocation(line: 9, scope: !11, inlinedAt: !17)
+!22 = !DILocation(line: 11, scope: !11, inlinedAt: !17)
+!23 = !DILocation(line: 16, scope: !18)
 !24 = !{!9, !10}
 !25 = !{!0, !6}
 !26 = !{!16}
-!27 = !MDFile(filename: "bar.c", directory: "/tmp/")
+!27 = !DIFile(filename: "bar.c", directory: "/tmp/")
 !28 = !{i32 1, !"Debug Info Version", i32 3}
 !29 = !{null}
diff --git a/test/DebugInfo/Mips/delay-slot.ll b/test/DebugInfo/Mips/delay-slot.ll
index 97f3670850d..df01775a12e 100644
--- a/test/DebugInfo/Mips/delay-slot.ll
+++ b/test/DebugInfo/Mips/delay-slot.ll
@@ -13,12 +13,14 @@
 ; CHECK: Address            Line   Column File   ISA Discriminator Flags
 ; CHECK: ------------------ ------ ------ ------ --- ------------- -------------
 ; CHECK: 0x0000000000000000      1      0      1   0             0  is_stmt
-; CHECK: 0x0000000000000000      1      0      1   0             0  is_stmt prologue_end
-; CHECK: 0x0000000000000008      2      0      1   0             0  is_stmt
-; CHECK: 0x0000000000000020      3      0      1   0             0  is_stmt
-; CHECK: 0x0000000000000030      4      0      1   0             0  is_stmt
-; CHECK: 0x0000000000000040      5      0      1   0             0  is_stmt
-; CHECK: 0x0000000000000050      5      0      1   0             0  is_stmt end_sequence
+; FIXME: The next address probably ought to be 0x0000000000000004 but there's
+;        a constant initialization before the prologue's end.
+; CHECK: 0x0000000000000008      2      0      1   0             0  is_stmt prologue_end
+; CHECK: 0x0000000000000028      3      0      1   0             0  is_stmt
+; CHECK: 0x0000000000000038      4      0      1   0             0  is_stmt
+; CHECK: 0x0000000000000048      5      0      1   0             0  is_stmt
+; CHECK: 0x0000000000000058      5      0      1   0             0  is_stmt end_sequence
+
 
 target datalayout = "E-m:m-p:32:32-i8:8:32-i16:16:32-i64:64-n32-S64"
 target triple = "mips--linux-gnu"
@@ -26,7 +28,7 @@ target triple = "mips--linux-gnu"
 ; Function Attrs: nounwind
 define i32 @foo(i32 %x) #0 {
 entry:
-  call void @llvm.dbg.value(metadata i32 %x, i64 0, metadata !12, metadata !MDExpression()), !dbg !13
+  call void @llvm.dbg.value(metadata i32 %x, i64 0, metadata !12, metadata !DIExpression()), !dbg !13
   %tobool = icmp ne i32 %x, 0, !dbg !14
   br i1 %tobool, label %if.then, label %if.end, !dbg !14
 
@@ -54,22 +56,22 @@ attributes #1 = { nounwind readnone }
 !llvm.module.flags = !{!9, !10}
 !llvm.ident = !{!11}
 
-!0 = !MDCompileUnit(language: DW_LANG_C99, producer: "clang version 3.5.0 ", isOptimized: false, emissionKind: 1, file: !1, enums: !2, retainedTypes: !2, subprograms: !3, globals: !2, imports: !2)
-!1 = !MDFile(filename: "test.c", directory: "/tmp")
+!0 = !DICompileUnit(language: DW_LANG_C99, producer: "clang version 3.5.0 ", isOptimized: false, emissionKind: 1, file: !1, enums: !2, retainedTypes: !2, subprograms: !3, globals: !2, imports: !2)
+!1 = !DIFile(filename: "test.c", directory: "/tmp")
 !2 = !{}
 !3 = !{!4}
-!4 = !MDSubprogram(name: "foo", line: 1, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 1, file: !1, scope: !5, type: !6, function: i32 (i32)* @foo, variables: !2)
-!5 = !MDFile(filename: "test.c", directory: "/tmp")
-!6 = !MDSubroutineType(types: !7)
+!4 = !DISubprogram(name: "foo", line: 1, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 1, file: !1, scope: !5, type: !6, function: i32 (i32)* @foo, variables: !2)
+!5 = !DIFile(filename: "test.c", directory: "/tmp")
+!6 = !DISubroutineType(types: !7)
 !7 = !{!8, !8}
-!8 = !MDBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
+!8 = !DIBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
 !9 = !{i32 2, !"Dwarf Version", i32 4}
 !10 = !{i32 2, !"Debug Info Version", i32 3}
 !11 = !{!"clang version 3.5.0"}
-!12 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "x", line: 1, arg: 1, scope: !4, file: !5, type: !8)
-!13 = !MDLocation(line: 1, scope: !4)
-!14 = !MDLocation(line: 2, scope: !15)
-!15 = distinct !MDLexicalBlock(line: 2, column: 0, file: !1, scope: !4)
-!16 = !MDLocation(line: 3, scope: !15)
-!17 = !MDLocation(line: 4, scope: !4)
-!18 = !MDLocation(line: 5, scope: !4)
+!12 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "x", line: 1, arg: 1, scope: !4, file: !5, type: !8)
+!13 = !DILocation(line: 1, scope: !4)
+!14 = !DILocation(line: 2, scope: !15)
+!15 = distinct !DILexicalBlock(line: 2, column: 0, file: !1, scope: !4)
+!16 = !DILocation(line: 3, scope: !15)
+!17 = !DILocation(line: 4, scope: !4)
+!18 = !DILocation(line: 5, scope: !4)
diff --git a/test/DebugInfo/Mips/fn-call-line.ll b/test/DebugInfo/Mips/fn-call-line.ll
index ad90fa808ea..18ed3dfa43a 100644
--- a/test/DebugInfo/Mips/fn-call-line.ll
+++ b/test/DebugInfo/Mips/fn-call-line.ll
@@ -68,17 +68,17 @@ attributes #1 = { "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "n
 !llvm.module.flags = !{!8, !9}
 !llvm.ident = !{!10}
 
-!0 = !MDCompileUnit(language: DW_LANG_C99, producer: "clang version 3.7.0 (trunk 226641)", isOptimized: false, emissionKind: 1, file: !1, enums: !2, retainedTypes: !2, subprograms: !3, globals: !2, imports: !2)
-!1 = !MDFile(filename: "fn-call-line.c", directory: "/tmp/dbginfo")
+!0 = !DICompileUnit(language: DW_LANG_C99, producer: "clang version 3.7.0 (trunk 226641)", isOptimized: false, emissionKind: 1, file: !1, enums: !2, retainedTypes: !2, subprograms: !3, globals: !2, imports: !2)
+!1 = !DIFile(filename: "fn-call-line.c", directory: "/tmp/dbginfo")
 !2 = !{}
 !3 = !{!4}
-!4 = !MDSubprogram(name: "f2", line: 2, isLocal: false, isDefinition: true, isOptimized: false, scopeLine: 2, file: !1, scope: !5, type: !6, function: void ()* @f2, variables: !2)
-!5 = !MDFile(filename: "fn-call-line.c", directory: "/tmp/dbginfo")
-!6 = !MDSubroutineType(types: !7)
+!4 = !DISubprogram(name: "f2", line: 2, isLocal: false, isDefinition: true, isOptimized: false, scopeLine: 2, file: !1, scope: !5, type: !6, function: void ()* @f2, variables: !2)
+!5 = !DIFile(filename: "fn-call-line.c", directory: "/tmp/dbginfo")
+!6 = !DISubroutineType(types: !7)
 !7 = !{null}
 !8 = !{i32 2, !"Dwarf Version", i32 4}
 !9 = !{i32 2, !"Debug Info Version", i32 3}
 !10 = !{!"clang version 3.7.0 (trunk 226641)"}
-!11 = !MDLocation(line: 3, column: 3, scope: !4)
-!12 = !MDLocation(line: 4, column: 3, scope: !4)
-!13 = !MDLocation(line: 5, column: 1, scope: !4)
+!11 = !DILocation(line: 3, column: 3, scope: !4)
+!12 = !DILocation(line: 4, column: 3, scope: !4)
+!13 = !DILocation(line: 5, column: 1, scope: !4)
diff --git a/test/DebugInfo/PR20038.ll b/test/DebugInfo/PR20038.ll
index 61255656522..8691895f41a 100644
--- a/test/DebugInfo/PR20038.ll
+++ b/test/DebugInfo/PR20038.ll
@@ -74,10 +74,10 @@ land.end:                                         ; preds = %land.rhs, %entry
 
 cleanup.action:                                   ; preds = %land.end
   store %struct.C* %agg.tmp.ensured, %struct.C** %this.addr.i, align 8, !dbg !22
-  call void @llvm.dbg.declare(metadata %struct.C** %this.addr.i, metadata !129, metadata !MDExpression()), !dbg !31
+  call void @llvm.dbg.declare(metadata %struct.C** %this.addr.i, metadata !129, metadata !DIExpression()), !dbg !31
   %this1.i = load %struct.C*, %struct.C** %this.addr.i, !dbg !22
   store %struct.C* %this1.i, %struct.C** %this.addr.i.i, align 8, !dbg !21
-  call void @llvm.dbg.declare(metadata %struct.C** %this.addr.i.i, metadata !132, metadata !MDExpression()), !dbg !33
+  call void @llvm.dbg.declare(metadata %struct.C** %this.addr.i.i, metadata !132, metadata !DIExpression()), !dbg !33
   %this1.i.i = load %struct.C*, %struct.C** %this.addr.i.i, !dbg !21
   br label %cleanup.done, !dbg !22
 
@@ -91,10 +91,10 @@ entry:
   %this.addr.i = alloca %struct.C*, align 8, !dbg !37
   %this.addr = alloca %struct.C*, align 8
   store %struct.C* %this, %struct.C** %this.addr, align 8
-  call void @llvm.dbg.declare(metadata %struct.C** %this.addr, metadata !29, metadata !MDExpression()), !dbg !38
+  call void @llvm.dbg.declare(metadata %struct.C** %this.addr, metadata !29, metadata !DIExpression()), !dbg !38
   %this1 = load %struct.C*, %struct.C** %this.addr
   store %struct.C* %this1, %struct.C** %this.addr.i, align 8, !dbg !37
-  call void @llvm.dbg.declare(metadata %struct.C** %this.addr.i, metadata !232, metadata !MDExpression()), !dbg !39
+  call void @llvm.dbg.declare(metadata %struct.C** %this.addr.i, metadata !232, metadata !DIExpression()), !dbg !39
   %this1.i = load %struct.C*, %struct.C** %this.addr.i, !dbg !37
   ret void, !dbg !37
 }
@@ -104,7 +104,7 @@ define void @_ZN1CD2Ev(%struct.C* %this) unnamed_addr #1 align 2 {
 entry:
   %this.addr = alloca %struct.C*, align 8
   store %struct.C* %this, %struct.C** %this.addr, align 8
-  call void @llvm.dbg.declare(metadata %struct.C** %this.addr, metadata !32, metadata !MDExpression()), !dbg !40
+  call void @llvm.dbg.declare(metadata %struct.C** %this.addr, metadata !32, metadata !DIExpression()), !dbg !40
   %this1 = load %struct.C*, %struct.C** %this.addr
   ret void, !dbg !41
 }
@@ -120,50 +120,50 @@ attributes #2 = { nounwind readnone }
 !llvm.module.flags = !{!18, !19}
 !llvm.ident = !{!20}
 
-!0 = !MDCompileUnit(language: DW_LANG_C_plus_plus, producer: "clang version 3.5.0 ", isOptimized: false, emissionKind: 1, file: !1, enums: !2, retainedTypes: !3, subprograms: !11, globals: !2, imports: !2)
-!1 = !MDFile(filename: "<stdin>", directory: "/tmp/dbginfo")
+!0 = !DICompileUnit(language: DW_LANG_C_plus_plus, producer: "clang version 3.5.0 ", isOptimized: false, emissionKind: 1, file: !1, enums: !2, retainedTypes: !3, subprograms: !11, globals: !2, imports: !2)
+!1 = !DIFile(filename: "<stdin>", directory: "/tmp/dbginfo")
 !2 = !{}
 !3 = !{!4}
-!4 = !MDCompositeType(tag: DW_TAG_structure_type, name: "C", line: 1, size: 8, align: 8, file: !5, elements: !6, identifier: "_ZTS1C")
-!5 = !MDFile(filename: "PR20038.cpp", directory: "/tmp/dbginfo")
+!4 = !DICompositeType(tag: DW_TAG_structure_type, name: "C", line: 1, size: 8, align: 8, file: !5, elements: !6, identifier: "_ZTS1C")
+!5 = !DIFile(filename: "PR20038.cpp", directory: "/tmp/dbginfo")
 !6 = !{!7}
-!7 = !MDSubprogram(name: "~C", line: 2, isLocal: false, isDefinition: false, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 2, file: !5, scope: !"_ZTS1C", type: !8)
-!8 = !MDSubroutineType(types: !9)
+!7 = !DISubprogram(name: "~C", line: 2, isLocal: false, isDefinition: false, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 2, file: !5, scope: !"_ZTS1C", type: !8)
+!8 = !DISubroutineType(types: !9)
 !9 = !{null, !10}
-!10 = !MDDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, flags: DIFlagArtificial | DIFlagObjectPointer, baseType: !"_ZTS1C")
+!10 = !DIDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, flags: DIFlagArtificial | DIFlagObjectPointer, baseType: !"_ZTS1C")
 !11 = !{!12, !16, !17}
-!12 = !MDSubprogram(name: "fun4", linkageName: "_Z4fun4v", line: 5, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 5, file: !5, scope: !13, type: !14, function: void ()* @_Z4fun4v, variables: !2)
-!13 = !MDFile(filename: "PR20038.cpp", directory: "/tmp/dbginfo")
-!14 = !MDSubroutineType(types: !15)
+!12 = !DISubprogram(name: "fun4", linkageName: "_Z4fun4v", line: 5, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 5, file: !5, scope: !13, type: !14, function: void ()* @_Z4fun4v, variables: !2)
+!13 = !DIFile(filename: "PR20038.cpp", directory: "/tmp/dbginfo")
+!14 = !DISubroutineType(types: !15)
 !15 = !{null}
-!16 = !MDSubprogram(name: "~C", linkageName: "_ZN1CD2Ev", line: 6, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 6, file: !5, scope: !"_ZTS1C", type: !8, function: void (%struct.C*)* @_ZN1CD2Ev, declaration: !7, variables: !2)
-!17 = !MDSubprogram(name: "~C", linkageName: "_ZN1CD1Ev", line: 6, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 6, file: !5, scope: !"_ZTS1C", type: !8, function: void (%struct.C*)* @_ZN1CD1Ev, declaration: !7, variables: !2)
+!16 = !DISubprogram(name: "~C", linkageName: "_ZN1CD2Ev", line: 6, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 6, file: !5, scope: !"_ZTS1C", type: !8, function: void (%struct.C*)* @_ZN1CD2Ev, declaration: !7, variables: !2)
+!17 = !DISubprogram(name: "~C", linkageName: "_ZN1CD1Ev", line: 6, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 6, file: !5, scope: !"_ZTS1C", type: !8, function: void (%struct.C*)* @_ZN1CD1Ev, declaration: !7, variables: !2)
 !18 = !{i32 2, !"Dwarf Version", i32 4}
 !19 = !{i32 2, !"Debug Info Version", i32 3}
 !20 = !{!"clang version 3.5.0 "}
-!21 = !MDLocation(line: 6, scope: !17, inlinedAt: !22)
-!22 = !MDLocation(line: 5, scope: !23)
-!23 = distinct !MDLexicalBlock(line: 5, column: 0, file: !5, scope: !12)
-!24 = !MDLocation(line: 5, scope: !12)
-!25 = !MDLocation(line: 5, scope: !26)
-!26 = distinct !MDLexicalBlock(line: 5, column: 0, file: !5, scope: !12)
-!27 = !MDLocation(line: 5, scope: !28)
-!28 = distinct !MDLexicalBlock(line: 5, column: 0, file: !5, scope: !12)
-!29 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "this", arg: 1, flags: DIFlagArtificial | DIFlagObjectPointer, scope: !17, type: !30)
-!30 = !MDDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, baseType: !"_ZTS1C")
-!31 = !MDLocation(line: 0, scope: !17, inlinedAt: !22)
-!32 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "this", arg: 1, flags: DIFlagArtificial | DIFlagObjectPointer, scope: !16, type: !30)
-!33 = !MDLocation(line: 0, scope: !16, inlinedAt: !21)
-
-!129 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "this", arg: 1, flags: DIFlagArtificial | DIFlagObjectPointer, scope: !17, type: !30)
-!132 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "this", arg: 1, flags: DIFlagArtificial | DIFlagObjectPointer, scope: !16, type: !30)
-!232 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "this", arg: 1, flags: DIFlagArtificial | DIFlagObjectPointer, scope: !16, type: !30)
-
-!34 = !MDLocation(line: 5, scope: !35)
-!35 = distinct !MDLexicalBlock(line: 5, column: 0, file: !5, scope: !36)
-!36 = distinct !MDLexicalBlock(line: 5, column: 0, file: !5, scope: !12)
-!37 = !MDLocation(line: 6, scope: !17)
-!38 = !MDLocation(line: 0, scope: !17)
-!39 = !MDLocation(line: 0, scope: !16, inlinedAt: !37)
-!40 = !MDLocation(line: 0, scope: !16)
-!41 = !MDLocation(line: 6, scope: !16)
+!21 = !DILocation(line: 6, scope: !17, inlinedAt: !22)
+!22 = !DILocation(line: 5, scope: !23)
+!23 = distinct !DILexicalBlock(line: 5, column: 0, file: !5, scope: !12)
+!24 = !DILocation(line: 5, scope: !12)
+!25 = !DILocation(line: 5, scope: !26)
+!26 = distinct !DILexicalBlock(line: 5, column: 0, file: !5, scope: !12)
+!27 = !DILocation(line: 5, scope: !28)
+!28 = distinct !DILexicalBlock(line: 5, column: 0, file: !5, scope: !12)
+!29 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "this", arg: 1, flags: DIFlagArtificial | DIFlagObjectPointer, scope: !17, type: !30)
+!30 = !DIDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, baseType: !"_ZTS1C")
+!31 = !DILocation(line: 0, scope: !17, inlinedAt: !22)
+!32 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "this", arg: 1, flags: DIFlagArtificial | DIFlagObjectPointer, scope: !16, type: !30)
+!33 = !DILocation(line: 0, scope: !16, inlinedAt: !21)
+
+!129 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "this", arg: 1, flags: DIFlagArtificial | DIFlagObjectPointer, scope: !17, type: !30)
+!132 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "this", arg: 1, flags: DIFlagArtificial | DIFlagObjectPointer, scope: !16, type: !30)
+!232 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "this", arg: 1, flags: DIFlagArtificial | DIFlagObjectPointer, scope: !16, type: !30)
+
+!34 = !DILocation(line: 5, scope: !35)
+!35 = distinct !DILexicalBlock(line: 5, column: 0, file: !5, scope: !36)
+!36 = distinct !DILexicalBlock(line: 5, column: 0, file: !5, scope: !12)
+!37 = !DILocation(line: 6, scope: !17)
+!38 = !DILocation(line: 0, scope: !17)
+!39 = !DILocation(line: 0, scope: !16, inlinedAt: !37)
+!40 = !DILocation(line: 0, scope: !16)
+!41 = !DILocation(line: 6, scope: !16)
diff --git a/test/DebugInfo/PowerPC/tls-fission.ll b/test/DebugInfo/PowerPC/tls-fission.ll
index 74674858a50..25bc9959e4d 100644
--- a/test/DebugInfo/PowerPC/tls-fission.ll
+++ b/test/DebugInfo/PowerPC/tls-fission.ll
@@ -22,12 +22,12 @@
 !llvm.dbg.cu = !{!0}
 !llvm.module.flags = !{!7, !8}
 
-!0 = !MDCompileUnit(language: DW_LANG_C_plus_plus, producer: "clang version 3.4 ", isOptimized: false, splitDebugFilename: "tls.dwo", emissionKind: 0, file: !1, enums: !2, retainedTypes: !2, subprograms: !2, globals: !3, imports: !2)
-!1 = !MDFile(filename: "tls.cpp", directory: "/tmp")
+!0 = !DICompileUnit(language: DW_LANG_C_plus_plus, producer: "clang version 3.4 ", isOptimized: false, splitDebugFilename: "tls.dwo", emissionKind: 0, file: !1, enums: !2, retainedTypes: !2, subprograms: !2, globals: !3, imports: !2)
+!1 = !DIFile(filename: "tls.cpp", directory: "/tmp")
 !2 = !{}
 !3 = !{!4}
-!4 = !MDGlobalVariable(name: "tls", line: 1, isLocal: false, isDefinition: true, scope: null, file: !5, type: !6, variable: i32* @tls)
-!5 = !MDFile(filename: "tls.cpp", directory: "/tmp")
-!6 = !MDBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
+!4 = !DIGlobalVariable(name: "tls", line: 1, isLocal: false, isDefinition: true, scope: null, file: !5, type: !6, variable: i32* @tls)
+!5 = !DIFile(filename: "tls.cpp", directory: "/tmp")
+!6 = !DIBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
 !7 = !{i32 2, !"Dwarf Version", i32 3}
 !8 = !{i32 1, !"Debug Info Version", i32 3}
diff --git a/test/DebugInfo/PowerPC/tls.ll b/test/DebugInfo/PowerPC/tls.ll
index 77016f42ee5..8ba350ac60d 100644
--- a/test/DebugInfo/PowerPC/tls.ll
+++ b/test/DebugInfo/PowerPC/tls.ll
@@ -17,13 +17,13 @@
 !llvm.dbg.cu = !{!0}
 !llvm.module.flags = !{!7, !8}
 
-!0 = !MDCompileUnit(language: DW_LANG_C_plus_plus, producer: "clang version 3.4 ", isOptimized: false, emissionKind: 0, file: !1, enums: !2, retainedTypes: !2, subprograms: !2, globals: !3, imports: !2)
-!1 = !MDFile(filename: "tls.cpp", directory: "/tmp")
+!0 = !DICompileUnit(language: DW_LANG_C_plus_plus, producer: "clang version 3.4 ", isOptimized: false, emissionKind: 0, file: !1, enums: !2, retainedTypes: !2, subprograms: !2, globals: !3, imports: !2)
+!1 = !DIFile(filename: "tls.cpp", directory: "/tmp")
 !2 = !{}
 !3 = !{!4}
-!4 = !MDGlobalVariable(name: "tls", line: 1, isLocal: false, isDefinition: true, scope: null, file: !5, type: !6, variable: i32* @tls)
-!5 = !MDFile(filename: "tls.cpp", directory: "/tmp")
-!6 = !MDBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
+!4 = !DIGlobalVariable(name: "tls", line: 1, isLocal: false, isDefinition: true, scope: null, file: !5, type: !6, variable: i32* @tls)
+!5 = !DIFile(filename: "tls.cpp", directory: "/tmp")
+!6 = !DIBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
 !7 = !{i32 2, !"Dwarf Version", i32 3}
 
 !8 = !{i32 1, !"Debug Info Version", i32 3}
diff --git a/test/DebugInfo/Sparc/gnu-window-save.ll b/test/DebugInfo/Sparc/gnu-window-save.ll
index f18c6ce7461..c2e0364307f 100644
--- a/test/DebugInfo/Sparc/gnu-window-save.ll
+++ b/test/DebugInfo/Sparc/gnu-window-save.ll
@@ -55,17 +55,17 @@ attributes #1 = { "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "n
 !llvm.module.flags = !{!9, !10}
 !llvm.ident = !{!11}
 
-!0 = !MDCompileUnit(language: DW_LANG_C99, producer: "clang version 3.5 (http://llvm.org/git/clang.git 6a0714fee07fb7c4e32d3972b4fe2ce2f5678cf4) (llvm/ 672e88e934757f76d5c5e5258be41e7615094844)", isOptimized: false, emissionKind: 0, file: !1, enums: !2, retainedTypes: !2, subprograms: !3, globals: !2, imports: !2)
-!1 = !MDFile(filename: "hello.c", directory: "/home/venkatra/work/benchmarks/test/hello")
+!0 = !DICompileUnit(language: DW_LANG_C99, producer: "clang version 3.5 (http://llvm.org/git/clang.git 6a0714fee07fb7c4e32d3972b4fe2ce2f5678cf4) (llvm/ 672e88e934757f76d5c5e5258be41e7615094844)", isOptimized: false, emissionKind: 0, file: !1, enums: !2, retainedTypes: !2, subprograms: !3, globals: !2, imports: !2)
+!1 = !DIFile(filename: "hello.c", directory: "/home/venkatra/work/benchmarks/test/hello")
 !2 = !{}
 !3 = !{!4}
-!4 = !MDSubprogram(name: "main", line: 3, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 4, file: !1, scope: !5, type: !6, function: i32 ()* @main, variables: !2)
-!5 = !MDFile(filename: "hello.c", directory: "/home/venkatra/work/benchmarks/test/hello")
-!6 = !MDSubroutineType(types: !7)
+!4 = !DISubprogram(name: "main", line: 3, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 4, file: !1, scope: !5, type: !6, function: i32 ()* @main, variables: !2)
+!5 = !DIFile(filename: "hello.c", directory: "/home/venkatra/work/benchmarks/test/hello")
+!6 = !DISubroutineType(types: !7)
 !7 = !{!8}
-!8 = !MDBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
+!8 = !DIBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
 !9 = !{i32 2, !"Dwarf Version", i32 4}
 !10 = !{i32 1, !"Debug Info Version", i32 3}
 !11 = !{!"clang version 3.5 (http://llvm.org/git/clang.git 6a0714fee07fb7c4e32d3972b4fe2ce2f5678cf4) (llvm/ 672e88e934757f76d5c5e5258be41e7615094844)"}
-!12 = !MDLocation(line: 5, scope: !4)
-!13 = !MDLocation(line: 6, scope: !4)
+!12 = !DILocation(line: 5, scope: !4)
+!13 = !DILocation(line: 6, scope: !4)
diff --git a/test/DebugInfo/SystemZ/eh_frame.s b/test/DebugInfo/SystemZ/eh_frame.s
index 6189b901967..ad7862f9748 100644
--- a/test/DebugInfo/SystemZ/eh_frame.s
+++ b/test/DebugInfo/SystemZ/eh_frame.s
@@ -72,7 +72,7 @@ check_largest_class:
 #   DW_CFA_nop
 #
 # CHECK: Contents of section .eh_frame:
-# CHECK-NEXT: 0000 00000014 00000000 037a5200 01780e01  {{.*}}
+# CHECK-NEXT: 0000 00000014 00000000 017a5200 01780e01  {{.*}}
 # CHECK-NEXT: 0010 1b0c0fa0 01000000 0000002c 0000001c  {{.*}}
 # CHECK-NEXT: 0020 00000000 00000032 00468d07 8e068f05  {{.*}}
 # CHECK-NEXT: 0030 440e8003 60981c9c 1b991a9d 199a189e  {{.*}}
diff --git a/test/DebugInfo/SystemZ/eh_frame_personality.ll b/test/DebugInfo/SystemZ/eh_frame_personality.ll
index 92ba34da456..323d328d470 100644
--- a/test/DebugInfo/SystemZ/eh_frame_personality.ll
+++ b/test/DebugInfo/SystemZ/eh_frame_personality.ll
@@ -6,13 +6,13 @@ declare i32 @__gxx_personality_v0(...)
 
 declare void @bar()
 
-define i64 @foo(i64 %lhs, i64 %rhs) {
+define i64 @foo(i64 %lhs, i64 %rhs) personality i8* bitcast (i32 (...)* @__gxx_personality_v0 to i8*) {
   invoke void @bar() to label %end unwind label %clean
 end:
  ret i64 0
 
 clean:
-  %tst = landingpad { i8*, i32 } personality i8* bitcast (i32 (...)* @__gxx_personality_v0 to i8*) cleanup
+  %tst = landingpad { i8*, i32 } cleanup
   ret i64 42
 }
 
diff --git a/test/DebugInfo/SystemZ/eh_frame_personality.s b/test/DebugInfo/SystemZ/eh_frame_personality.s
index 456e0a6e6bd..c452951941a 100644
--- a/test/DebugInfo/SystemZ/eh_frame_personality.s
+++ b/test/DebugInfo/SystemZ/eh_frame_personality.s
@@ -61,7 +61,7 @@ DW.ref.__gxx_personality_v0:
 #   DW_CFA_nop
 #
 # CHECK: Contents of section .eh_frame:
-# CHECK-NEXT: 0000 0000001c 00000000 037a504c 52000178  .........zPLR..x
+# CHECK-NEXT: 0000 0000001c 00000000 017a504c 52000178  .........zPLR..x
 # CHECK-NEXT: 0010 0e079b00 0000001b 1b0c0fa0 01000000  ................
 # CHECK-NEXT: 0020 0000001c 00000024 00000000 00000012  .......$........
 # CHECK-NEXT: 0030 04000000 00468e06 8f05440e c0020000  .....F....D.....
diff --git a/test/DebugInfo/SystemZ/variable-loc.ll b/test/DebugInfo/SystemZ/variable-loc.ll
index e7987fc3551..5f4fe258b97 100644
--- a/test/DebugInfo/SystemZ/variable-loc.ll
+++ b/test/DebugInfo/SystemZ/variable-loc.ll
@@ -35,8 +35,8 @@ entry:
   %main_arr = alloca [100 x i32], align 4
   %val = alloca i32, align 4
   store volatile i32 0, i32* %retval
-  call void @llvm.dbg.declare(metadata [100 x i32]* %main_arr, metadata !17, metadata !MDExpression()), !dbg !22
-  call void @llvm.dbg.declare(metadata i32* %val, metadata !23, metadata !MDExpression()), !dbg !24
+  call void @llvm.dbg.declare(metadata [100 x i32]* %main_arr, metadata !17, metadata !DIExpression()), !dbg !22
+  call void @llvm.dbg.declare(metadata i32* %val, metadata !23, metadata !DIExpression()), !dbg !24
   %arraydecay = getelementptr inbounds [100 x i32], [100 x i32]* %main_arr, i32 0, i32 0, !dbg !25
   call void @populate_array(i32* %arraydecay, i32 100), !dbg !25
   %arraydecay1 = getelementptr inbounds [100 x i32], [100 x i32]* %main_arr, i32 0, i32 0, !dbg !26
@@ -52,31 +52,31 @@ declare i32 @printf(i8*, ...)
 !llvm.dbg.cu = !{!0}
 !llvm.module.flags = !{!30}
 
-!0 = !MDCompileUnit(language: DW_LANG_C99, producer: "clang version 3.2 ", isOptimized: false, emissionKind: 0, file: !29, enums: !1, retainedTypes: !1, subprograms: !3, globals: !1, imports:  !1)
+!0 = !DICompileUnit(language: DW_LANG_C99, producer: "clang version 3.2 ", isOptimized: false, emissionKind: 0, file: !29, enums: !1, retainedTypes: !1, subprograms: !3, globals: !1, imports:  !1)
 !1 = !{}
 !3 = !{!5, !11, !14}
-!5 = !MDSubprogram(name: "populate_array", line: 4, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 4, file: !29, scope: !6, type: !7, function: void (i32*, i32)* @populate_array, variables: !1)
-!6 = !MDFile(filename: "simple.c", directory: "/home/timnor01/a64-trunk/build")
-!7 = !MDSubroutineType(types: !8)
+!5 = !DISubprogram(name: "populate_array", line: 4, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 4, file: !29, scope: !6, type: !7, function: void (i32*, i32)* @populate_array, variables: !1)
+!6 = !DIFile(filename: "simple.c", directory: "/home/timnor01/a64-trunk/build")
+!7 = !DISubroutineType(types: !8)
 !8 = !{null, !9, !10}
-!9 = !MDDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, baseType: !10)
-!10 = !MDBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
-!11 = !MDSubprogram(name: "sum_array", line: 9, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 9, file: !29, scope: !6, type: !12, function: i32 (i32*, i32)* @sum_array, variables: !1)
-!12 = !MDSubroutineType(types: !13)
+!9 = !DIDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, baseType: !10)
+!10 = !DIBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
+!11 = !DISubprogram(name: "sum_array", line: 9, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 9, file: !29, scope: !6, type: !12, function: i32 (i32*, i32)* @sum_array, variables: !1)
+!12 = !DISubroutineType(types: !13)
 !13 = !{!10, !9, !10}
-!14 = !MDSubprogram(name: "main", line: 18, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 18, file: !29, scope: !6, type: !15, function: i32 ()* @main, variables: !1)
-!15 = !MDSubroutineType(types: !16)
+!14 = !DISubprogram(name: "main", line: 18, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 18, file: !29, scope: !6, type: !15, function: i32 ()* @main, variables: !1)
+!15 = !DISubroutineType(types: !16)
 !16 = !{!10}
-!17 = !MDLocalVariable(tag: DW_TAG_auto_variable, name: "main_arr", line: 19, scope: !18, file: !6, type: !19)
-!18 = distinct !MDLexicalBlock(line: 18, column: 16, file: !29, scope: !14)
-!19 = !MDCompositeType(tag: DW_TAG_array_type, size: 3200, align: 32, baseType: !10, elements: !{!20})
-!20 = !MDSubrange(count: 99)
-!22 = !MDLocation(line: 19, column: 7, scope: !18)
-!23 = !MDLocalVariable(tag: DW_TAG_auto_variable, name: "val", line: 20, scope: !18, file: !6, type: !10)
-!24 = !MDLocation(line: 20, column: 7, scope: !18)
-!25 = !MDLocation(line: 22, column: 3, scope: !18)
-!26 = !MDLocation(line: 23, column: 9, scope: !18)
-!27 = !MDLocation(line: 24, column: 3, scope: !18)
-!28 = !MDLocation(line: 26, column: 3, scope: !18)
-!29 = !MDFile(filename: "simple.c", directory: "/home/timnor01/a64-trunk/build")
+!17 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "main_arr", line: 19, scope: !18, file: !6, type: !19)
+!18 = distinct !DILexicalBlock(line: 18, column: 16, file: !29, scope: !14)
+!19 = !DICompositeType(tag: DW_TAG_array_type, size: 3200, align: 32, baseType: !10, elements: !{!20})
+!20 = !DISubrange(count: 99)
+!22 = !DILocation(line: 19, column: 7, scope: !18)
+!23 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "val", line: 20, scope: !18, file: !6, type: !10)
+!24 = !DILocation(line: 20, column: 7, scope: !18)
+!25 = !DILocation(line: 22, column: 3, scope: !18)
+!26 = !DILocation(line: 23, column: 9, scope: !18)
+!27 = !DILocation(line: 24, column: 3, scope: !18)
+!28 = !DILocation(line: 26, column: 3, scope: !18)
+!29 = !DIFile(filename: "simple.c", directory: "/home/timnor01/a64-trunk/build")
 !30 = !{i32 1, !"Debug Info Version", i32 3}
diff --git a/test/DebugInfo/X86/2010-04-13-PubType.ll b/test/DebugInfo/X86/2010-04-13-PubType.ll
index c990bfc7624..aa1bf4aaf5f 100644
--- a/test/DebugInfo/X86/2010-04-13-PubType.ll
+++ b/test/DebugInfo/X86/2010-04-13-PubType.ll
@@ -12,9 +12,9 @@ entry:
   %retval = alloca i32                            ; <i32*> [#uses=2]
   %0 = alloca i32                                 ; <i32*> [#uses=2]
   %"alloca point" = bitcast i32 0 to i32          ; <i32> [#uses=0]
-  call void @llvm.dbg.declare(metadata %struct.X** %x_addr, metadata !0, metadata !MDExpression()), !dbg !13
+  call void @llvm.dbg.declare(metadata %struct.X** %x_addr, metadata !0, metadata !DIExpression()), !dbg !13
   store %struct.X* %x, %struct.X** %x_addr
-  call void @llvm.dbg.declare(metadata %struct.Y** %y_addr, metadata !14, metadata !MDExpression()), !dbg !13
+  call void @llvm.dbg.declare(metadata %struct.Y** %y_addr, metadata !14, metadata !DIExpression()), !dbg !13
   store %struct.Y* %y, %struct.Y** %y_addr
   store i32 0, i32* %0, align 4, !dbg !13
   %1 = load i32, i32* %0, align 4, !dbg !13            ; <i32> [#uses=1]
@@ -31,24 +31,24 @@ declare void @llvm.dbg.declare(metadata, metadata, metadata) nounwind readnone
 !llvm.dbg.cu = !{!3}
 !llvm.module.flags = !{!20}
 
-!0 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "x", line: 7, arg: 0, scope: !1, file: !2, type: !7)
-!1 = !MDSubprogram(name: "foo", linkageName: "foo", line: 7, isLocal: false, isDefinition: true, virtualIndex: 6, isOptimized: false, scopeLine: 7, file: !18, scope: !2, type: !4, function: i32 (%struct.X*, %struct.Y*)* @foo)
-!2 = !MDFile(filename: "a.c", directory: "/tmp/")
-!3 = !MDCompileUnit(language: DW_LANG_C89, producer: "4.2.1 (Based on Apple Inc. build 5658) (LLVM build)", isOptimized: false, emissionKind: 0, file: !18, enums: !19, retainedTypes: !19, subprograms: !17, imports:  null)
-!4 = !MDSubroutineType(types: !5)
+!0 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "x", line: 7, arg: 1, scope: !1, file: !2, type: !7)
+!1 = !DISubprogram(name: "foo", linkageName: "foo", line: 7, isLocal: false, isDefinition: true, virtualIndex: 6, isOptimized: false, scopeLine: 7, file: !18, scope: !2, type: !4, function: i32 (%struct.X*, %struct.Y*)* @foo)
+!2 = !DIFile(filename: "a.c", directory: "/tmp/")
+!3 = !DICompileUnit(language: DW_LANG_C89, producer: "4.2.1 (Based on Apple Inc. build 5658) (LLVM build)", isOptimized: false, emissionKind: 0, file: !18, enums: !19, retainedTypes: !19, subprograms: !17, imports:  null)
+!4 = !DISubroutineType(types: !5)
 !5 = !{!6, !7, !9}
-!6 = !MDBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
-!7 = !MDDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, file: !18, scope: !2, baseType: !8)
-!8 = !MDCompositeType(tag: DW_TAG_structure_type, name: "X", line: 3, flags: DIFlagFwdDecl, file: !18, scope: !2)
-!9 = !MDDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, file: !18, scope: !2, baseType: !10)
-!10 = !MDCompositeType(tag: DW_TAG_structure_type, name: "Y", line: 4, size: 32, align: 32, file: !18, scope: !2, elements: !11)
+!6 = !DIBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
+!7 = !DIDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, file: !18, scope: !2, baseType: !8)
+!8 = !DICompositeType(tag: DW_TAG_structure_type, name: "X", line: 3, flags: DIFlagFwdDecl, file: !18, scope: !2)
+!9 = !DIDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, file: !18, scope: !2, baseType: !10)
+!10 = !DICompositeType(tag: DW_TAG_structure_type, name: "Y", line: 4, size: 32, align: 32, file: !18, scope: !2, elements: !11)
 !11 = !{!12}
-!12 = !MDDerivedType(tag: DW_TAG_member, name: "x", line: 5, size: 32, align: 32, file: !18, scope: !10, baseType: !6)
-!13 = !MDLocation(line: 7, scope: !1)
-!14 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "y", line: 7, arg: 0, scope: !1, file: !2, type: !9)
-!15 = !MDLocation(line: 7, scope: !16)
-!16 = distinct !MDLexicalBlock(line: 7, column: 0, file: !18, scope: !1)
+!12 = !DIDerivedType(tag: DW_TAG_member, name: "x", line: 5, size: 32, align: 32, file: !18, scope: !10, baseType: !6)
+!13 = !DILocation(line: 7, scope: !1)
+!14 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "y", line: 7, arg: 2, scope: !1, file: !2, type: !9)
+!15 = !DILocation(line: 7, scope: !16)
+!16 = distinct !DILexicalBlock(line: 7, column: 0, file: !18, scope: !1)
 !17 = !{!1}
-!18 = !MDFile(filename: "a.c", directory: "/tmp/")
+!18 = !DIFile(filename: "a.c", directory: "/tmp/")
 !19 = !{}
 !20 = !{i32 1, !"Debug Info Version", i32 3}
diff --git a/test/DebugInfo/X86/2011-09-26-GlobalVarContext.ll b/test/DebugInfo/X86/2011-09-26-GlobalVarContext.ll
index 2dd8b281379..1252f2cb32e 100644
--- a/test/DebugInfo/X86/2011-09-26-GlobalVarContext.ll
+++ b/test/DebugInfo/X86/2011-09-26-GlobalVarContext.ll
@@ -7,7 +7,7 @@
 
 define i32 @f() nounwind {
   %LOC = alloca i32, align 4
-  call void @llvm.dbg.declare(metadata i32* %LOC, metadata !15, metadata !MDExpression()), !dbg !17
+  call void @llvm.dbg.declare(metadata i32* %LOC, metadata !15, metadata !DIExpression()), !dbg !17
   %1 = load i32, i32* @GLB, align 4, !dbg !18
   store i32 %1, i32* %LOC, align 4, !dbg !18
   %2 = load i32, i32* @GLB, align 4, !dbg !19
@@ -19,22 +19,22 @@ declare void @llvm.dbg.declare(metadata, metadata, metadata) nounwind readnone
 !llvm.dbg.cu = !{!0}
 !llvm.module.flags = !{!21}
 
-!0 = !MDCompileUnit(language: DW_LANG_C99, producer: "clang version 3.0 (trunk)", isOptimized: false, emissionKind: 0, file: !20, enums: !1, retainedTypes: !1, subprograms: !3, globals: !12, imports:  !1)
+!0 = !DICompileUnit(language: DW_LANG_C99, producer: "clang version 3.0 (trunk)", isOptimized: false, emissionKind: 0, file: !20, enums: !1, retainedTypes: !1, subprograms: !3, globals: !12, imports:  !1)
 !1 = !{}
 !3 = !{!5}
-!5 = !MDSubprogram(name: "f", line: 3, isLocal: false, isDefinition: true, virtualIndex: 6, isOptimized: false, file: !6, scope: !6, type: !7, function: i32 ()* @f)
-!6 = !MDFile(filename: "test.c", directory: "/work/llvm/vanilla/test/DebugInfo")
-!7 = !MDSubroutineType(types: !8)
+!5 = !DISubprogram(name: "f", line: 3, isLocal: false, isDefinition: true, virtualIndex: 6, isOptimized: false, file: !6, scope: !6, type: !7, function: i32 ()* @f)
+!6 = !DIFile(filename: "test.c", directory: "/work/llvm/vanilla/test/DebugInfo")
+!7 = !DISubroutineType(types: !8)
 !8 = !{!9}
-!9 = !MDBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
+!9 = !DIBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
 !12 = !{!14}
-!14 = !MDGlobalVariable(name: "GLB", line: 1, isLocal: false, isDefinition: true, scope: null, file: !6, type: !9, variable: i32* @GLB)
-!15 = !MDLocalVariable(tag: DW_TAG_auto_variable, name: "LOC", line: 4, scope: !16, file: !6, type: !9)
-!16 = distinct !MDLexicalBlock(line: 3, column: 9, file: !20, scope: !5)
-!17 = !MDLocation(line: 4, column: 9, scope: !16)
-!18 = !MDLocation(line: 4, column: 23, scope: !16)
-!19 = !MDLocation(line: 5, column: 5, scope: !16)
-!20 = !MDFile(filename: "test.c", directory: "/work/llvm/vanilla/test/DebugInfo")
+!14 = !DIGlobalVariable(name: "GLB", line: 1, isLocal: false, isDefinition: true, scope: null, file: !6, type: !9, variable: i32* @GLB)
+!15 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "LOC", line: 4, scope: !16, file: !6, type: !9)
+!16 = distinct !DILexicalBlock(line: 3, column: 9, file: !20, scope: !5)
+!17 = !DILocation(line: 4, column: 9, scope: !16)
+!18 = !DILocation(line: 4, column: 23, scope: !16)
+!19 = !DILocation(line: 5, column: 5, scope: !16)
+!20 = !DIFile(filename: "test.c", directory: "/work/llvm/vanilla/test/DebugInfo")
 
 ; CHECK: DW_TAG_variable
 ; CHECK-NOT: DW_TAG
diff --git a/test/DebugInfo/X86/2011-12-16-BadStructRef.ll b/test/DebugInfo/X86/2011-12-16-BadStructRef.ll
index ff9f06cc1ff..fc3f69b3584 100644
--- a/test/DebugInfo/X86/2011-12-16-BadStructRef.ll
+++ b/test/DebugInfo/X86/2011-12-16-BadStructRef.ll
@@ -15,10 +15,10 @@ entry:
   %myBar = alloca %struct.bar, align 8
   store i32 0, i32* %retval
   store i32 %argc, i32* %argc.addr, align 4
-  call void @llvm.dbg.declare(metadata i32* %argc.addr, metadata !49, metadata !MDExpression()), !dbg !50
+  call void @llvm.dbg.declare(metadata i32* %argc.addr, metadata !49, metadata !DIExpression()), !dbg !50
   store i8** %argv, i8*** %argv.addr, align 8
-  call void @llvm.dbg.declare(metadata i8*** %argv.addr, metadata !51, metadata !MDExpression()), !dbg !52
-  call void @llvm.dbg.declare(metadata %struct.bar* %myBar, metadata !53, metadata !MDExpression()), !dbg !55
+  call void @llvm.dbg.declare(metadata i8*** %argv.addr, metadata !51, metadata !DIExpression()), !dbg !52
+  call void @llvm.dbg.declare(metadata %struct.bar* %myBar, metadata !53, metadata !DIExpression()), !dbg !55
   call void @_ZN3barC1Ei(%struct.bar* %myBar, i32 1), !dbg !56
   ret i32 0, !dbg !57
 }
@@ -30,9 +30,9 @@ entry:
   %this.addr = alloca %struct.bar*, align 8
   %x.addr = alloca i32, align 4
   store %struct.bar* %this, %struct.bar** %this.addr, align 8
-  call void @llvm.dbg.declare(metadata %struct.bar** %this.addr, metadata !58, metadata !MDExpression()), !dbg !59
+  call void @llvm.dbg.declare(metadata %struct.bar** %this.addr, metadata !58, metadata !DIExpression()), !dbg !59
   store i32 %x, i32* %x.addr, align 4
-  call void @llvm.dbg.declare(metadata i32* %x.addr, metadata !60, metadata !MDExpression()), !dbg !61
+  call void @llvm.dbg.declare(metadata i32* %x.addr, metadata !60, metadata !DIExpression()), !dbg !61
   %this1 = load %struct.bar*, %struct.bar** %this.addr
   %0 = load i32, i32* %x.addr, align 4, !dbg !62
   call void @_ZN3barC2Ei(%struct.bar* %this1, i32 %0), !dbg !62
@@ -44,9 +44,9 @@ entry:
   %this.addr = alloca %struct.bar*, align 8
   %x.addr = alloca i32, align 4
   store %struct.bar* %this, %struct.bar** %this.addr, align 8
-  call void @llvm.dbg.declare(metadata %struct.bar** %this.addr, metadata !63, metadata !MDExpression()), !dbg !64
+  call void @llvm.dbg.declare(metadata %struct.bar** %this.addr, metadata !63, metadata !DIExpression()), !dbg !64
   store i32 %x, i32* %x.addr, align 4
-  call void @llvm.dbg.declare(metadata i32* %x.addr, metadata !65, metadata !MDExpression()), !dbg !66
+  call void @llvm.dbg.declare(metadata i32* %x.addr, metadata !65, metadata !DIExpression()), !dbg !66
   %this1 = load %struct.bar*, %struct.bar** %this.addr
   %b = getelementptr inbounds %struct.bar, %struct.bar* %this1, i32 0, i32 0, !dbg !67
   %0 = load i32, i32* %x.addr, align 4, !dbg !67
@@ -62,9 +62,9 @@ entry:
   %this.addr = alloca %struct.baz*, align 8
   %a.addr = alloca i32, align 4
   store %struct.baz* %this, %struct.baz** %this.addr, align 8
-  call void @llvm.dbg.declare(metadata %struct.baz** %this.addr, metadata !70, metadata !MDExpression()), !dbg !71
+  call void @llvm.dbg.declare(metadata %struct.baz** %this.addr, metadata !70, metadata !DIExpression()), !dbg !71
   store i32 %a, i32* %a.addr, align 4
-  call void @llvm.dbg.declare(metadata i32* %a.addr, metadata !72, metadata !MDExpression()), !dbg !73
+  call void @llvm.dbg.declare(metadata i32* %a.addr, metadata !72, metadata !DIExpression()), !dbg !73
   %this1 = load %struct.baz*, %struct.baz** %this.addr
   %0 = load i32, i32* %a.addr, align 4, !dbg !74
   call void @_ZN3bazC2Ei(%struct.baz* %this1, i32 %0), !dbg !74
@@ -76,9 +76,9 @@ entry:
   %this.addr = alloca %struct.baz*, align 8
   %a.addr = alloca i32, align 4
   store %struct.baz* %this, %struct.baz** %this.addr, align 8
-  call void @llvm.dbg.declare(metadata %struct.baz** %this.addr, metadata !75, metadata !MDExpression()), !dbg !76
+  call void @llvm.dbg.declare(metadata %struct.baz** %this.addr, metadata !75, metadata !DIExpression()), !dbg !76
   store i32 %a, i32* %a.addr, align 4
-  call void @llvm.dbg.declare(metadata i32* %a.addr, metadata !77, metadata !MDExpression()), !dbg !78
+  call void @llvm.dbg.declare(metadata i32* %a.addr, metadata !77, metadata !DIExpression()), !dbg !78
   %this1 = load %struct.baz*, %struct.baz** %this.addr
   %h = getelementptr inbounds %struct.baz, %struct.baz* %this1, i32 0, i32 0, !dbg !79
   %0 = load i32, i32* %a.addr, align 4, !dbg !79
@@ -89,78 +89,78 @@ entry:
 !llvm.dbg.cu = !{!0}
 !llvm.module.flags = !{!83}
 
-!0 = !MDCompileUnit(language: DW_LANG_C_plus_plus, producer: "clang version 3.1 (trunk 146596)", isOptimized: false, emissionKind: 0, file: !82, enums: !1, retainedTypes: !3, subprograms: !27, globals: !1, imports:  !1)
+!0 = !DICompileUnit(language: DW_LANG_C_plus_plus, producer: "clang version 3.1 (trunk 146596)", isOptimized: false, emissionKind: 0, file: !82, enums: !1, retainedTypes: !3, subprograms: !27, globals: !1, imports:  !1)
 !1 = !{}
 !3 = !{!5, !9}
-!5 = !MDCompositeType(tag: DW_TAG_class_type, name: "bar", line: 9, size: 128, align: 64, file: !82, elements: !7)
-!6 = !MDFile(filename: "main.cpp", directory: "/Users/echristo/tmp/bad-struct-ref")
+!5 = !DICompositeType(tag: DW_TAG_class_type, name: "bar", line: 9, size: 128, align: 64, file: !82, elements: !7)
+!6 = !DIFile(filename: "main.cpp", directory: "/Users/echristo/tmp/bad-struct-ref")
 !7 = !{!8, !19, !21}
-!8 = !MDDerivedType(tag: DW_TAG_member, name: "b", line: 11, size: 32, align: 32, file: !82, scope: !5, baseType: !9)
-!9 = !MDCompositeType(tag: DW_TAG_class_type, name: "baz", line: 3, size: 32, align: 32, file: !82, elements: !10)
+!8 = !DIDerivedType(tag: DW_TAG_member, name: "b", line: 11, size: 32, align: 32, file: !82, scope: !5, baseType: !9)
+!9 = !DICompositeType(tag: DW_TAG_class_type, name: "baz", line: 3, size: 32, align: 32, file: !82, elements: !10)
 !10 = !{!11, !13}
-!11 = !MDDerivedType(tag: DW_TAG_member, name: "h", line: 5, size: 32, align: 32, file: !82, scope: !9, baseType: !12)
-!12 = !MDBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
-!13 = !MDSubprogram(name: "baz", line: 6, isLocal: false, isDefinition: false, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, file: !82, scope: !9, type: !14)
-!14 = !MDSubroutineType(types: !15)
+!11 = !DIDerivedType(tag: DW_TAG_member, name: "h", line: 5, size: 32, align: 32, file: !82, scope: !9, baseType: !12)
+!12 = !DIBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
+!13 = !DISubprogram(name: "baz", line: 6, isLocal: false, isDefinition: false, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, file: !82, scope: !9, type: !14)
+!14 = !DISubroutineType(types: !15)
 !15 = !{null, !16, !12}
-!16 = !MDDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, flags: DIFlagArtificial, baseType: !9)
-!19 = !MDDerivedType(tag: DW_TAG_member, name: "b_ref", line: 12, size: 64, align: 64, offset: 64, file: !82, scope: !5, baseType: !20)
-!20 = !MDDerivedType(tag: DW_TAG_reference_type, baseType: !9)
-!21 = !MDSubprogram(name: "bar", line: 13, isLocal: false, isDefinition: false, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, file: !82, scope: !5, type: !22)
-!22 = !MDSubroutineType(types: !23)
+!16 = !DIDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, flags: DIFlagArtificial, baseType: !9)
+!19 = !DIDerivedType(tag: DW_TAG_member, name: "b_ref", line: 12, size: 64, align: 64, offset: 64, file: !82, scope: !5, baseType: !20)
+!20 = !DIDerivedType(tag: DW_TAG_reference_type, baseType: !9)
+!21 = !DISubprogram(name: "bar", line: 13, isLocal: false, isDefinition: false, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, file: !82, scope: !5, type: !22)
+!22 = !DISubroutineType(types: !23)
 !23 = !{null, !24, !12}
-!24 = !MDDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, flags: DIFlagArtificial, baseType: !5)
+!24 = !DIDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, flags: DIFlagArtificial, baseType: !5)
 !27 = !{!29, !37, !40, !43, !46}
-!29 = !MDSubprogram(name: "main", line: 17, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, file: !82, scope: !6, type: !30, function: i32 (i32, i8**)* @main)
-!30 = !MDSubroutineType(types: !31)
+!29 = !DISubprogram(name: "main", line: 17, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, file: !82, scope: !6, type: !30, function: i32 (i32, i8**)* @main)
+!30 = !DISubroutineType(types: !31)
 !31 = !{!12, !12, !32}
-!32 = !MDDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, baseType: !33)
-!33 = !MDDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, baseType: !34)
-!34 = !MDBasicType(tag: DW_TAG_base_type, name: "char", size: 8, align: 8, encoding: DW_ATE_signed_char)
+!32 = !DIDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, baseType: !33)
+!33 = !DIDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, baseType: !34)
+!34 = !DIBasicType(tag: DW_TAG_base_type, name: "char", size: 8, align: 8, encoding: DW_ATE_signed_char)
 !35 = !{!36}
 !36 = !{} ; previously: invalid DW_TAG_base_type
-!37 = !MDSubprogram(name: "bar", linkageName: "_ZN3barC1Ei", line: 13, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, file: !82, scope: null, type: !22, function: void (%struct.bar*, i32)* @_ZN3barC1Ei, declaration: !21)
+!37 = !DISubprogram(name: "bar", linkageName: "_ZN3barC1Ei", line: 13, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, file: !82, scope: null, type: !22, function: void (%struct.bar*, i32)* @_ZN3barC1Ei, declaration: !21)
 !38 = !{!39}
 !39 = !{} ; previously: invalid DW_TAG_base_type
-!40 = !MDSubprogram(name: "bar", linkageName: "_ZN3barC2Ei", line: 13, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, file: !82, scope: null, type: !22, function: void (%struct.bar*, i32)* @_ZN3barC2Ei, declaration: !21)
+!40 = !DISubprogram(name: "bar", linkageName: "_ZN3barC2Ei", line: 13, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, file: !82, scope: null, type: !22, function: void (%struct.bar*, i32)* @_ZN3barC2Ei, declaration: !21)
 !41 = !{!42}
 !42 = !{} ; previously: invalid DW_TAG_base_type
-!43 = !MDSubprogram(name: "baz", linkageName: "_ZN3bazC1Ei", line: 6, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, file: !82, scope: null, type: !14, function: void (%struct.baz*, i32)* @_ZN3bazC1Ei, declaration: !13)
+!43 = !DISubprogram(name: "baz", linkageName: "_ZN3bazC1Ei", line: 6, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, file: !82, scope: null, type: !14, function: void (%struct.baz*, i32)* @_ZN3bazC1Ei, declaration: !13)
 !44 = !{!45}
 !45 = !{} ; previously: invalid DW_TAG_base_type
-!46 = !MDSubprogram(name: "baz", linkageName: "_ZN3bazC2Ei", line: 6, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, file: !82, scope: null, type: !14, function: void (%struct.baz*, i32)* @_ZN3bazC2Ei, declaration: !13)
-!49 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "argc", line: 16, arg: 1, scope: !29, file: !6, type: !12)
-!50 = !MDLocation(line: 16, column: 14, scope: !29)
-!51 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "argv", line: 16, arg: 2, scope: !29, file: !6, type: !32)
-!52 = !MDLocation(line: 16, column: 27, scope: !29)
-!53 = !MDLocalVariable(tag: DW_TAG_auto_variable, name: "myBar", line: 18, scope: !54, file: !6, type: !5)
-!54 = distinct !MDLexicalBlock(line: 17, column: 1, file: !82, scope: !29)
-!55 = !MDLocation(line: 18, column: 9, scope: !54)
-!56 = !MDLocation(line: 18, column: 17, scope: !54)
-!57 = !MDLocation(line: 19, column: 5, scope: !54)
-!58 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "this", line: 13, arg: 1, flags: DIFlagArtificial, scope: !37, file: !6, type: !24)
-!59 = !MDLocation(line: 13, column: 5, scope: !37)
-!60 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "x", line: 13, arg: 2, scope: !37, file: !6, type: !12)
-!61 = !MDLocation(line: 13, column: 13, scope: !37)
-!62 = !MDLocation(line: 13, column: 34, scope: !37)
-!63 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "this", line: 13, arg: 1, flags: DIFlagArtificial, scope: !40, file: !6, type: !24)
-!64 = !MDLocation(line: 13, column: 5, scope: !40)
-!65 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "x", line: 13, arg: 2, scope: !40, file: !6, type: !12)
-!66 = !MDLocation(line: 13, column: 13, scope: !40)
-!67 = !MDLocation(line: 13, column: 33, scope: !40)
-!68 = !MDLocation(line: 13, column: 34, scope: !69)
-!69 = distinct !MDLexicalBlock(line: 13, column: 33, file: !82, scope: !40)
-!70 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "this", line: 6, arg: 1, flags: DIFlagArtificial, scope: !43, file: !6, type: !16)
-!71 = !MDLocation(line: 6, column: 5, scope: !43)
-!72 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "a", line: 6, arg: 2, scope: !43, file: !6, type: !12)
-!73 = !MDLocation(line: 6, column: 13, scope: !43)
-!74 = !MDLocation(line: 6, column: 24, scope: !43)
-!75 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "this", line: 6, arg: 1, flags: DIFlagArtificial, scope: !46, file: !6, type: !16)
-!76 = !MDLocation(line: 6, column: 5, scope: !46)
-!77 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "a", line: 6, arg: 2, scope: !46, file: !6, type: !12)
-!78 = !MDLocation(line: 6, column: 13, scope: !46)
-!79 = !MDLocation(line: 6, column: 23, scope: !46)
-!80 = !MDLocation(line: 6, column: 24, scope: !81)
-!81 = distinct !MDLexicalBlock(line: 6, column: 23, file: !82, scope: !46)
-!82 = !MDFile(filename: "main.cpp", directory: "/Users/echristo/tmp/bad-struct-ref")
+!46 = !DISubprogram(name: "baz", linkageName: "_ZN3bazC2Ei", line: 6, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, file: !82, scope: null, type: !14, function: void (%struct.baz*, i32)* @_ZN3bazC2Ei, declaration: !13)
+!49 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "argc", line: 16, arg: 1, scope: !29, file: !6, type: !12)
+!50 = !DILocation(line: 16, column: 14, scope: !29)
+!51 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "argv", line: 16, arg: 2, scope: !29, file: !6, type: !32)
+!52 = !DILocation(line: 16, column: 27, scope: !29)
+!53 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "myBar", line: 18, scope: !54, file: !6, type: !5)
+!54 = distinct !DILexicalBlock(line: 17, column: 1, file: !82, scope: !29)
+!55 = !DILocation(line: 18, column: 9, scope: !54)
+!56 = !DILocation(line: 18, column: 17, scope: !54)
+!57 = !DILocation(line: 19, column: 5, scope: !54)
+!58 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "this", line: 13, arg: 1, flags: DIFlagArtificial, scope: !37, file: !6, type: !24)
+!59 = !DILocation(line: 13, column: 5, scope: !37)
+!60 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "x", line: 13, arg: 2, scope: !37, file: !6, type: !12)
+!61 = !DILocation(line: 13, column: 13, scope: !37)
+!62 = !DILocation(line: 13, column: 34, scope: !37)
+!63 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "this", line: 13, arg: 1, flags: DIFlagArtificial, scope: !40, file: !6, type: !24)
+!64 = !DILocation(line: 13, column: 5, scope: !40)
+!65 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "x", line: 13, arg: 2, scope: !40, file: !6, type: !12)
+!66 = !DILocation(line: 13, column: 13, scope: !40)
+!67 = !DILocation(line: 13, column: 33, scope: !40)
+!68 = !DILocation(line: 13, column: 34, scope: !69)
+!69 = distinct !DILexicalBlock(line: 13, column: 33, file: !82, scope: !40)
+!70 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "this", line: 6, arg: 1, flags: DIFlagArtificial, scope: !43, file: !6, type: !16)
+!71 = !DILocation(line: 6, column: 5, scope: !43)
+!72 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "a", line: 6, arg: 2, scope: !43, file: !6, type: !12)
+!73 = !DILocation(line: 6, column: 13, scope: !43)
+!74 = !DILocation(line: 6, column: 24, scope: !43)
+!75 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "this", line: 6, arg: 1, flags: DIFlagArtificial, scope: !46, file: !6, type: !16)
+!76 = !DILocation(line: 6, column: 5, scope: !46)
+!77 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "a", line: 6, arg: 2, scope: !46, file: !6, type: !12)
+!78 = !DILocation(line: 6, column: 13, scope: !46)
+!79 = !DILocation(line: 6, column: 23, scope: !46)
+!80 = !DILocation(line: 6, column: 24, scope: !81)
+!81 = distinct !DILexicalBlock(line: 6, column: 23, file: !82, scope: !46)
+!82 = !DIFile(filename: "main.cpp", directory: "/Users/echristo/tmp/bad-struct-ref")
 !83 = !{i32 1, !"Debug Info Version", i32 3}
diff --git a/test/DebugInfo/X86/DIModule.ll b/test/DebugInfo/X86/DIModule.ll
new file mode 100644
index 00000000000..daed43a58cd
--- /dev/null
+++ b/test/DebugInfo/X86/DIModule.ll
@@ -0,0 +1,25 @@
+; ModuleID = '/Volumes/Data/apple-internal/llvm/tools/clang/test/Modules/debug-info-moduleimport.m'
+; RUN: llc %s -o %t -filetype=obj
+; RUN: llvm-dwarfdump -debug-dump=info %t | FileCheck %s
+; CHECK: DW_TAG_module
+; CHECK-NEXT: DW_AT_name {{.*}}"DebugModule"
+; CHECK-NEXT: DW_AT_LLVM_config_macros {{.*}}"-DMODULES=0"
+; CHECK-NEXT: DW_AT_LLVM_include_path {{.*}}"/llvm/tools/clang/test/Modules/Inputs"
+; CHECK-NEXT: DW_AT_LLVM_isysroot {{.*}}"/"
+
+target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-apple-macosx"
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!6, !7}
+!llvm.ident = !{!8}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_ObjC, file: !1, producer: "LLVM version 3.7.0", isOptimized: false, runtimeVersion: 2, emissionKind: 1, enums: !2, retainedTypes: !2, subprograms: !2, globals: !2, imports: !3)
+!1 = !DIFile(filename: "/llvm/tools/clang/test/Modules/<stdin>", directory: "/")
+!2 = !{}
+!3 = !{!4}
+!4 = !DIImportedEntity(tag: DW_TAG_imported_declaration, scope: !0, entity: !5, line: 5)
+!5 = !DIModule(scope: null, name: "DebugModule", configMacros: "-DMODULES=0", includePath: "/llvm/tools/clang/test/Modules/Inputs", isysroot: "/")
+!6 = !{i32 2, !"Dwarf Version", i32 4}
+!7 = !{i32 2, !"Debug Info Version", i32 3}
+!8 = !{!"LLVM version 3.7.0"}
diff --git a/test/DebugInfo/X86/DW_AT_byte_size.ll b/test/DebugInfo/X86/DW_AT_byte_size.ll
index 19d0d990ee2..bfad4fe261d 100644
--- a/test/DebugInfo/X86/DW_AT_byte_size.ll
+++ b/test/DebugInfo/X86/DW_AT_byte_size.ll
@@ -14,7 +14,7 @@ define i32 @_Z3fooP1A(%struct.A* %a) nounwind uwtable ssp {
 entry:
   %a.addr = alloca %struct.A*, align 8
   store %struct.A* %a, %struct.A** %a.addr, align 8
-  call void @llvm.dbg.declare(metadata %struct.A** %a.addr, metadata !16, metadata !MDExpression()), !dbg !17
+  call void @llvm.dbg.declare(metadata %struct.A** %a.addr, metadata !16, metadata !DIExpression()), !dbg !17
   %0 = load %struct.A*, %struct.A** %a.addr, align 8, !dbg !18
   %b = getelementptr inbounds %struct.A, %struct.A* %0, i32 0, i32 0, !dbg !18
   %1 = load i32, i32* %b, align 4, !dbg !18
@@ -26,21 +26,21 @@ declare void @llvm.dbg.declare(metadata, metadata, metadata) nounwind readnone
 !llvm.dbg.cu = !{!0}
 !llvm.module.flags = !{!21}
 
-!0 = !MDCompileUnit(language: DW_LANG_C_plus_plus, producer: "clang version 3.1 (trunk 150996)", isOptimized: false, emissionKind: 0, file: !20, enums: !1, retainedTypes: !1, subprograms: !3, globals: !1, imports:  !1)
+!0 = !DICompileUnit(language: DW_LANG_C_plus_plus, producer: "clang version 3.1 (trunk 150996)", isOptimized: false, emissionKind: 0, file: !20, enums: !1, retainedTypes: !1, subprograms: !3, globals: !1, imports:  !1)
 !1 = !{}
 !3 = !{!5}
-!5 = !MDSubprogram(name: "foo", linkageName: "_Z3fooP1A", line: 3, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 3, file: !20, scope: !6, type: !7, function: i32 (%struct.A*)* @_Z3fooP1A)
-!6 = !MDFile(filename: "foo.cpp", directory: "/Users/echristo")
-!7 = !MDSubroutineType(types: !8)
+!5 = !DISubprogram(name: "foo", linkageName: "_Z3fooP1A", line: 3, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 3, file: !20, scope: !6, type: !7, function: i32 (%struct.A*)* @_Z3fooP1A)
+!6 = !DIFile(filename: "foo.cpp", directory: "/Users/echristo")
+!7 = !DISubroutineType(types: !8)
 !8 = !{!9, !10}
-!9 = !MDBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
-!10 = !MDDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, baseType: !11)
-!11 = !MDCompositeType(tag: DW_TAG_class_type, name: "A", line: 1, size: 32, align: 32, file: !20, elements: !12)
+!9 = !DIBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
+!10 = !DIDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, baseType: !11)
+!11 = !DICompositeType(tag: DW_TAG_class_type, name: "A", line: 1, size: 32, align: 32, file: !20, elements: !12)
 !12 = !{!13}
-!13 = !MDDerivedType(tag: DW_TAG_member, name: "b", line: 1, size: 32, align: 32, file: !20, scope: !11, baseType: !9)
-!16 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "a", line: 3, arg: 1, scope: !5, file: !6, type: !10)
-!17 = !MDLocation(line: 3, column: 13, scope: !5)
-!18 = !MDLocation(line: 4, column: 3, scope: !19)
-!19 = distinct !MDLexicalBlock(line: 3, column: 16, file: !20, scope: !5)
-!20 = !MDFile(filename: "foo.cpp", directory: "/Users/echristo")
+!13 = !DIDerivedType(tag: DW_TAG_member, name: "b", line: 1, size: 32, align: 32, file: !20, scope: !11, baseType: !9)
+!16 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "a", line: 3, arg: 1, scope: !5, file: !6, type: !10)
+!17 = !DILocation(line: 3, column: 13, scope: !5)
+!18 = !DILocation(line: 4, column: 3, scope: !19)
+!19 = distinct !DILexicalBlock(line: 3, column: 16, file: !20, scope: !5)
+!20 = !DIFile(filename: "foo.cpp", directory: "/Users/echristo")
 !21 = !{i32 1, !"Debug Info Version", i32 3}
diff --git a/test/DebugInfo/X86/DW_AT_linkage_name.ll b/test/DebugInfo/X86/DW_AT_linkage_name.ll
index daad4ab1a58..7f18b5037d5 100644
--- a/test/DebugInfo/X86/DW_AT_linkage_name.ll
+++ b/test/DebugInfo/X86/DW_AT_linkage_name.ll
@@ -38,7 +38,7 @@ define void @_ZN1AD2Ev(%struct.A* %this) unnamed_addr #0 align 2 {
 entry:
   %this.addr = alloca %struct.A*, align 8
   store %struct.A* %this, %struct.A** %this.addr, align 8
-  call void @llvm.dbg.declare(metadata %struct.A** %this.addr, metadata !26, metadata !MDExpression()), !dbg !28
+  call void @llvm.dbg.declare(metadata %struct.A** %this.addr, metadata !26, metadata !DIExpression()), !dbg !28
   %this1 = load %struct.A*, %struct.A** %this.addr
   ret void, !dbg !29
 }
@@ -51,7 +51,7 @@ define void @_ZN1AD1Ev(%struct.A* %this) unnamed_addr #0 align 2 {
 entry:
   %this.addr = alloca %struct.A*, align 8
   store %struct.A* %this, %struct.A** %this.addr, align 8
-  call void @llvm.dbg.declare(metadata %struct.A** %this.addr, metadata !30, metadata !MDExpression()), !dbg !31
+  call void @llvm.dbg.declare(metadata %struct.A** %this.addr, metadata !30, metadata !DIExpression()), !dbg !31
   %this1 = load %struct.A*, %struct.A** %this.addr
   call void @_ZN1AD2Ev(%struct.A* %this1), !dbg !32
   ret void, !dbg !33
@@ -61,7 +61,7 @@ entry:
 define void @_Z3foov() #2 {
 entry:
   %a = alloca %struct.A, align 1
-  call void @llvm.dbg.declare(metadata %struct.A* %a, metadata !34, metadata !MDExpression()), !dbg !35
+  call void @llvm.dbg.declare(metadata %struct.A* %a, metadata !34, metadata !DIExpression()), !dbg !35
   call void @_ZN1AC1Ei(%struct.A* %a, i32 1), !dbg !35
   call void @_ZN1AD1Ev(%struct.A* %a), !dbg !36
   ret void, !dbg !36
@@ -77,38 +77,38 @@ attributes #2 = { ssp uwtable }
 !llvm.module.flags = !{!23, !24}
 !llvm.ident = !{!25}
 
-!0 = !MDCompileUnit(language: DW_LANG_C_plus_plus, producer: "clang version 3.5.0 ", isOptimized: false, emissionKind: 1, file: !1, enums: !2, retainedTypes: !3, subprograms: !16, globals: !2, imports: !2)
-!1 = !MDFile(filename: "linkage-name.cpp", directory: "")
+!0 = !DICompileUnit(language: DW_LANG_C_plus_plus, producer: "clang version 3.5.0 ", isOptimized: false, emissionKind: 1, file: !1, enums: !2, retainedTypes: !3, subprograms: !16, globals: !2, imports: !2)
+!1 = !DIFile(filename: "linkage-name.cpp", directory: "")
 !2 = !{}
 !3 = !{!4}
-!4 = !MDCompositeType(tag: DW_TAG_structure_type, name: "A", line: 1, size: 8, align: 8, file: !1, elements: !5, identifier: "_ZTS1A")
+!4 = !DICompositeType(tag: DW_TAG_structure_type, name: "A", line: 1, size: 8, align: 8, file: !1, elements: !5, identifier: "_ZTS1A")
 !5 = !{!6, !12}
-!6 = !MDSubprogram(name: "A", line: 2, isLocal: false, isDefinition: false, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 2, file: !1, scope: !"_ZTS1A", type: !7)
-!7 = !MDSubroutineType(types: !8)
+!6 = !DISubprogram(name: "A", line: 2, isLocal: false, isDefinition: false, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 2, file: !1, scope: !"_ZTS1A", type: !7)
+!7 = !DISubroutineType(types: !8)
 !8 = !{null, !9, !10}
-!9 = !MDDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, flags: DIFlagArtificial | DIFlagObjectPointer, baseType: !"_ZTS1A")
-!10 = !MDBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
-!12 = !MDSubprogram(name: "~A", line: 3, isLocal: false, isDefinition: false, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 3, file: !1, scope: !"_ZTS1A", type: !13)
-!13 = !MDSubroutineType(types: !14)
+!9 = !DIDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, flags: DIFlagArtificial | DIFlagObjectPointer, baseType: !"_ZTS1A")
+!10 = !DIBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
+!12 = !DISubprogram(name: "~A", line: 3, isLocal: false, isDefinition: false, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 3, file: !1, scope: !"_ZTS1A", type: !13)
+!13 = !DISubroutineType(types: !14)
 !14 = !{null, !9}
 !16 = !{!17, !18, !19}
-!17 = !MDSubprogram(name: "~A", linkageName: "_ZN1AD2Ev", line: 6, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 6, file: !1, scope: !"_ZTS1A", type: !13, function: void (%struct.A*)* @_ZN1AD2Ev, declaration: !12, variables: !2)
-!18 = !MDSubprogram(name: "~A", linkageName: "_ZN1AD1Ev", line: 6, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 6, file: !1, scope: !"_ZTS1A", type: !13, function: void (%struct.A*)* @_ZN1AD1Ev, declaration: !12, variables: !2)
-!19 = !MDSubprogram(name: "foo", linkageName: "_Z3foov", line: 10, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 10, file: !1, scope: !20, type: !21, function: void ()* @_Z3foov, variables: !2)
-!20 = !MDFile(filename: "linkage-name.cpp", directory: "")
-!21 = !MDSubroutineType(types: !22)
+!17 = !DISubprogram(name: "~A", linkageName: "_ZN1AD2Ev", line: 6, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 6, file: !1, scope: !"_ZTS1A", type: !13, function: void (%struct.A*)* @_ZN1AD2Ev, declaration: !12, variables: !2)
+!18 = !DISubprogram(name: "~A", linkageName: "_ZN1AD1Ev", line: 6, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 6, file: !1, scope: !"_ZTS1A", type: !13, function: void (%struct.A*)* @_ZN1AD1Ev, declaration: !12, variables: !2)
+!19 = !DISubprogram(name: "foo", linkageName: "_Z3foov", line: 10, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 10, file: !1, scope: !20, type: !21, function: void ()* @_Z3foov, variables: !2)
+!20 = !DIFile(filename: "linkage-name.cpp", directory: "")
+!21 = !DISubroutineType(types: !22)
 !22 = !{null}
 !23 = !{i32 2, !"Dwarf Version", i32 2}
 !24 = !{i32 1, !"Debug Info Version", i32 3}
 !25 = !{!"clang version 3.5.0 "}
-!26 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "this", arg: 1, flags: DIFlagArtificial | DIFlagObjectPointer, scope: !17, type: !27)
-!27 = !MDDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, baseType: !"_ZTS1A")
-!28 = !MDLocation(line: 0, scope: !17)
-!29 = !MDLocation(line: 8, scope: !17)
-!30 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "this", arg: 1, flags: DIFlagArtificial | DIFlagObjectPointer, scope: !18, type: !27)
-!31 = !MDLocation(line: 0, scope: !18)
-!32 = !MDLocation(line: 6, scope: !18)
-!33 = !MDLocation(line: 8, scope: !18)
-!34 = !MDLocalVariable(tag: DW_TAG_auto_variable, name: "a", line: 11, scope: !19, file: !20, type: !"_ZTS1A")
-!35 = !MDLocation(line: 11, scope: !19)
-!36 = !MDLocation(line: 12, scope: !19)
+!26 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "this", arg: 1, flags: DIFlagArtificial | DIFlagObjectPointer, scope: !17, type: !27)
+!27 = !DIDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, baseType: !"_ZTS1A")
+!28 = !DILocation(line: 0, scope: !17)
+!29 = !DILocation(line: 8, scope: !17)
+!30 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "this", arg: 1, flags: DIFlagArtificial | DIFlagObjectPointer, scope: !18, type: !27)
+!31 = !DILocation(line: 0, scope: !18)
+!32 = !DILocation(line: 6, scope: !18)
+!33 = !DILocation(line: 8, scope: !18)
+!34 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "a", line: 11, scope: !19, file: !20, type: !"_ZTS1A")
+!35 = !DILocation(line: 11, scope: !19)
+!36 = !DILocation(line: 12, scope: !19)
diff --git a/test/DebugInfo/X86/DW_AT_location-reference.ll b/test/DebugInfo/X86/DW_AT_location-reference.ll
index 82ed0a96059..96d1e509957 100644
--- a/test/DebugInfo/X86/DW_AT_location-reference.ll
+++ b/test/DebugInfo/X86/DW_AT_location-reference.ll
@@ -64,7 +64,7 @@ define void @f() nounwind {
 entry:
   %call = tail call i32 @g(i32 0, i32 0) nounwind, !dbg !8
   store i32 %call, i32* @a, align 4, !dbg !8
-  tail call void @llvm.dbg.value(metadata i32 1, i64 0, metadata !5, metadata !MDExpression()), !dbg !13
+  tail call void @llvm.dbg.value(metadata i32 1, i64 0, metadata !5, metadata !DIExpression()), !dbg !13
   br label %while.body
 
 while.body:                                       ; preds = %entry, %while.body
@@ -75,10 +75,10 @@ while.body:                                       ; preds = %entry, %while.body
   br i1 %tobool, label %while.end, label %while.body, !dbg !14
 
 while.end:                                        ; preds = %while.body
-  tail call void @llvm.dbg.value(metadata i32 %mul, i64 0, metadata !5, metadata !MDExpression()), !dbg !14
+  tail call void @llvm.dbg.value(metadata i32 %mul, i64 0, metadata !5, metadata !DIExpression()), !dbg !14
   %call4 = tail call i32 @g(i32 %mul, i32 0) nounwind, !dbg !15
   store i32 %call4, i32* @a, align 4, !dbg !15
-  tail call void @llvm.dbg.value(metadata i32 2, i64 0, metadata !5, metadata !MDExpression()), !dbg !17
+  tail call void @llvm.dbg.value(metadata i32 2, i64 0, metadata !5, metadata !DIExpression()), !dbg !17
   br label %while.body9
 
 while.body9:                                      ; preds = %while.end, %while.body9
@@ -89,7 +89,7 @@ while.body9:                                      ; preds = %while.end, %while.b
   br i1 %tobool8, label %while.end13, label %while.body9, !dbg !18
 
 while.end13:                                      ; preds = %while.body9
-  tail call void @llvm.dbg.value(metadata i32 %mul12, i64 0, metadata !5, metadata !MDExpression()), !dbg !18
+  tail call void @llvm.dbg.value(metadata i32 %mul12, i64 0, metadata !5, metadata !DIExpression()), !dbg !18
   %call15 = tail call i32 @g(i32 0, i32 %mul12) nounwind, !dbg !19
   store i32 %call15, i32* @a, align 4, !dbg !19
   ret void, !dbg !20
@@ -102,25 +102,25 @@ declare void @llvm.dbg.value(metadata, i64, metadata, metadata) nounwind readnon
 !llvm.dbg.cu = !{!2}
 !llvm.module.flags = !{!24}
 
-!0 = !MDSubprogram(name: "f", line: 4, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: true, scopeLine: 4, file: !23, scope: !1, type: !3, function: void ()* @f, variables: !22)
-!1 = !MDFile(filename: "simple.c", directory: "/home/rengol01/temp/tests/dwarf/relocation")
-!2 = !MDCompileUnit(language: DW_LANG_C99, producer: "clang version 3.0 (trunk)", isOptimized: true, emissionKind: 1, file: !23, enums: !{}, retainedTypes: !{}, subprograms: !21, imports:  null)
-!3 = !MDSubroutineType(types: !4)
+!0 = !DISubprogram(name: "f", line: 4, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: true, scopeLine: 4, file: !23, scope: !1, type: !3, function: void ()* @f, variables: !22)
+!1 = !DIFile(filename: "simple.c", directory: "/home/rengol01/temp/tests/dwarf/relocation")
+!2 = !DICompileUnit(language: DW_LANG_C99, producer: "clang version 3.0 (trunk)", isOptimized: true, emissionKind: 1, file: !23, enums: !{}, retainedTypes: !{}, subprograms: !21, imports:  null)
+!3 = !DISubroutineType(types: !4)
 !4 = !{null}
-!5 = !MDLocalVariable(tag: DW_TAG_auto_variable, name: "x", line: 5, scope: !6, file: !1, type: !7)
-!6 = distinct !MDLexicalBlock(line: 4, column: 14, file: !23, scope: !0)
-!7 = !MDBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
-!8 = !MDLocation(line: 6, column: 3, scope: !6)
+!5 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "x", line: 5, scope: !6, file: !1, type: !7)
+!6 = distinct !DILexicalBlock(line: 4, column: 14, file: !23, scope: !0)
+!7 = !DIBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
+!8 = !DILocation(line: 6, column: 3, scope: !6)
 !12 = !{i32 1}
-!13 = !MDLocation(line: 7, column: 3, scope: !6)
-!14 = !MDLocation(line: 8, column: 3, scope: !6)
-!15 = !MDLocation(line: 9, column: 3, scope: !6)
+!13 = !DILocation(line: 7, column: 3, scope: !6)
+!14 = !DILocation(line: 8, column: 3, scope: !6)
+!15 = !DILocation(line: 9, column: 3, scope: !6)
 !16 = !{i32 2}
-!17 = !MDLocation(line: 10, column: 3, scope: !6)
-!18 = !MDLocation(line: 11, column: 3, scope: !6)
-!19 = !MDLocation(line: 12, column: 3, scope: !6)
-!20 = !MDLocation(line: 13, column: 1, scope: !6)
+!17 = !DILocation(line: 10, column: 3, scope: !6)
+!18 = !DILocation(line: 11, column: 3, scope: !6)
+!19 = !DILocation(line: 12, column: 3, scope: !6)
+!20 = !DILocation(line: 13, column: 1, scope: !6)
 !21 = !{!0}
 !22 = !{!5}
-!23 = !MDFile(filename: "simple.c", directory: "/home/rengol01/temp/tests/dwarf/relocation")
+!23 = !DIFile(filename: "simple.c", directory: "/home/rengol01/temp/tests/dwarf/relocation")
 !24 = !{i32 1, !"Debug Info Version", i32 3}
diff --git a/test/DebugInfo/X86/DW_AT_object_pointer.ll b/test/DebugInfo/X86/DW_AT_object_pointer.ll
index b8fa8b8576d..86d67cae6b0 100644
--- a/test/DebugInfo/X86/DW_AT_object_pointer.ll
+++ b/test/DebugInfo/X86/DW_AT_object_pointer.ll
@@ -17,8 +17,8 @@ entry:
   %.addr = alloca i32, align 4
   %a = alloca %class.A, align 4
   store i32 %0, i32* %.addr, align 4
-  call void @llvm.dbg.declare(metadata i32* %.addr, metadata !36, metadata !MDExpression()), !dbg !35
-  call void @llvm.dbg.declare(metadata %class.A* %a, metadata !21, metadata !MDExpression()), !dbg !23
+  call void @llvm.dbg.declare(metadata i32* %.addr, metadata !36, metadata !DIExpression()), !dbg !35
+  call void @llvm.dbg.declare(metadata %class.A* %a, metadata !21, metadata !DIExpression()), !dbg !23
   call void @_ZN1AC1Ev(%class.A* %a), !dbg !24
   %m_a = getelementptr inbounds %class.A, %class.A* %a, i32 0, i32 0, !dbg !25
   %1 = load i32, i32* %m_a, align 4, !dbg !25
@@ -31,7 +31,7 @@ define linkonce_odr void @_ZN1AC1Ev(%class.A* %this) unnamed_addr nounwind uwtab
 entry:
   %this.addr = alloca %class.A*, align 8
   store %class.A* %this, %class.A** %this.addr, align 8
-  call void @llvm.dbg.declare(metadata %class.A** %this.addr, metadata !26, metadata !MDExpression()), !dbg !28
+  call void @llvm.dbg.declare(metadata %class.A** %this.addr, metadata !26, metadata !DIExpression()), !dbg !28
   %this1 = load %class.A*, %class.A** %this.addr
   call void @_ZN1AC2Ev(%class.A* %this1), !dbg !29
   ret void, !dbg !29
@@ -41,7 +41,7 @@ define linkonce_odr void @_ZN1AC2Ev(%class.A* %this) unnamed_addr nounwind uwtab
 entry:
   %this.addr = alloca %class.A*, align 8
   store %class.A* %this, %class.A** %this.addr, align 8
-  call void @llvm.dbg.declare(metadata %class.A** %this.addr, metadata !30, metadata !MDExpression()), !dbg !31
+  call void @llvm.dbg.declare(metadata %class.A** %this.addr, metadata !30, metadata !DIExpression()), !dbg !31
   %this1 = load %class.A*, %class.A** %this.addr
   %m_a = getelementptr inbounds %class.A, %class.A* %this1, i32 0, i32 0, !dbg !32
   store i32 0, i32* %m_a, align 4, !dbg !32
@@ -51,38 +51,38 @@ entry:
 !llvm.dbg.cu = !{!0}
 !llvm.module.flags = !{!38}
 
-!0 = !MDCompileUnit(language: DW_LANG_C_plus_plus, producer: "clang version 3.2 (trunk 163586) (llvm/trunk 163570)", isOptimized: false, emissionKind: 0, file: !37, enums: !1, retainedTypes: !1, subprograms: !3, globals: !1, imports:  !1)
+!0 = !DICompileUnit(language: DW_LANG_C_plus_plus, producer: "clang version 3.2 (trunk 163586) (llvm/trunk 163570)", isOptimized: false, emissionKind: 0, file: !37, enums: !1, retainedTypes: !1, subprograms: !3, globals: !1, imports:  !1)
 !1 = !{}
 !3 = !{!5, !10, !20}
-!5 = !MDSubprogram(name: "foo", linkageName: "_Z3fooi", line: 7, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 7, file: !6, scope: !6, type: !7, function: i32 (i32)* @_Z3fooi, variables: !1)
-!6 = !MDFile(filename: "bar.cpp", directory: "/Users/echristo/debug-tests")
-!7 = !MDSubroutineType(types: !8)
+!5 = !DISubprogram(name: "foo", linkageName: "_Z3fooi", line: 7, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 7, file: !6, scope: !6, type: !7, function: i32 (i32)* @_Z3fooi, variables: !1)
+!6 = !DIFile(filename: "bar.cpp", directory: "/Users/echristo/debug-tests")
+!7 = !DISubroutineType(types: !8)
 !8 = !{!9}
-!9 = !MDBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
-!10 = !MDSubprogram(name: "A", linkageName: "_ZN1AC1Ev", line: 3, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 3, file: !6, scope: null, type: !11, function: void (%class.A*)* @_ZN1AC1Ev, declaration: !17, variables: !1)
-!11 = !MDSubroutineType(types: !12)
+!9 = !DIBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
+!10 = !DISubprogram(name: "A", linkageName: "_ZN1AC1Ev", line: 3, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 3, file: !6, scope: null, type: !11, function: void (%class.A*)* @_ZN1AC1Ev, declaration: !17, variables: !1)
+!11 = !DISubroutineType(types: !12)
 !12 = !{null, !13}
-!13 = !MDDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, flags: DIFlagArtificial | DIFlagObjectPointer, baseType: !14)
-!14 = !MDCompositeType(tag: DW_TAG_class_type, name: "A", line: 1, size: 32, align: 32, file: !37, elements: !15)
+!13 = !DIDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, flags: DIFlagArtificial | DIFlagObjectPointer, baseType: !14)
+!14 = !DICompositeType(tag: DW_TAG_class_type, name: "A", line: 1, size: 32, align: 32, file: !37, elements: !15)
 !15 = !{!16, !17}
-!16 = !MDDerivedType(tag: DW_TAG_member, name: "m_a", line: 4, size: 32, align: 32, file: !37, scope: !14, baseType: !9)
-!17 = !MDSubprogram(name: "A", line: 3, isLocal: false, isDefinition: false, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 3, file: !6, scope: !14, type: !11)
-!20 = !MDSubprogram(name: "A", linkageName: "_ZN1AC2Ev", line: 3, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 3, file: !6, scope: null, type: !11, function: void (%class.A*)* @_ZN1AC2Ev, declaration: !17, variables: !1)
-!21 = !MDLocalVariable(tag: DW_TAG_auto_variable, name: "a", line: 8, scope: !22, file: !6, type: !14)
-!22 = distinct !MDLexicalBlock(line: 7, column: 11, file: !6, scope: !5)
-!23 = !MDLocation(line: 8, column: 5, scope: !22)
-!24 = !MDLocation(line: 8, column: 6, scope: !22)
-!25 = !MDLocation(line: 9, column: 3, scope: !22)
-!26 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "this", line: 3, arg: 1, flags: DIFlagArtificial | DIFlagObjectPointer, scope: !10, file: !6, type: !27)
-!27 = !MDDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, baseType: !14)
-!28 = !MDLocation(line: 3, column: 3, scope: !10)
-!29 = !MDLocation(line: 3, column: 18, scope: !10)
-!30 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "this", line: 3, arg: 1, flags: DIFlagArtificial | DIFlagObjectPointer, scope: !20, file: !6, type: !27)
-!31 = !MDLocation(line: 3, column: 3, scope: !20)
-!32 = !MDLocation(line: 3, column: 9, scope: !33)
-!33 = distinct !MDLexicalBlock(line: 3, column: 7, file: !6, scope: !20)
-!34 = !MDLocation(line: 3, column: 18, scope: !33)
-!35 = !MDLocation(line: 7, scope: !5)
-!36 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "", line: 7, arg: 1, scope: !5, file: !6, type: !9)
-!37 = !MDFile(filename: "bar.cpp", directory: "/Users/echristo/debug-tests")
+!16 = !DIDerivedType(tag: DW_TAG_member, name: "m_a", line: 4, size: 32, align: 32, file: !37, scope: !14, baseType: !9)
+!17 = !DISubprogram(name: "A", line: 3, isLocal: false, isDefinition: false, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 3, file: !6, scope: !14, type: !11)
+!20 = !DISubprogram(name: "A", linkageName: "_ZN1AC2Ev", line: 3, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 3, file: !6, scope: null, type: !11, function: void (%class.A*)* @_ZN1AC2Ev, declaration: !17, variables: !1)
+!21 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "a", line: 8, scope: !22, file: !6, type: !14)
+!22 = distinct !DILexicalBlock(line: 7, column: 11, file: !6, scope: !5)
+!23 = !DILocation(line: 8, column: 5, scope: !22)
+!24 = !DILocation(line: 8, column: 6, scope: !22)
+!25 = !DILocation(line: 9, column: 3, scope: !22)
+!26 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "this", line: 3, arg: 1, flags: DIFlagArtificial | DIFlagObjectPointer, scope: !10, file: !6, type: !27)
+!27 = !DIDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, baseType: !14)
+!28 = !DILocation(line: 3, column: 3, scope: !10)
+!29 = !DILocation(line: 3, column: 18, scope: !10)
+!30 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "this", line: 3, arg: 1, flags: DIFlagArtificial | DIFlagObjectPointer, scope: !20, file: !6, type: !27)
+!31 = !DILocation(line: 3, column: 3, scope: !20)
+!32 = !DILocation(line: 3, column: 9, scope: !33)
+!33 = distinct !DILexicalBlock(line: 3, column: 7, file: !6, scope: !20)
+!34 = !DILocation(line: 3, column: 18, scope: !33)
+!35 = !DILocation(line: 7, scope: !5)
+!36 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "", line: 7, arg: 1, scope: !5, file: !6, type: !9)
+!37 = !DIFile(filename: "bar.cpp", directory: "/Users/echristo/debug-tests")
 !38 = !{i32 1, !"Debug Info Version", i32 3}
diff --git a/test/DebugInfo/X86/DW_AT_specification.ll b/test/DebugInfo/X86/DW_AT_specification.ll
index 52a2f2e96be..993e623c751 100644
--- a/test/DebugInfo/X86/DW_AT_specification.ll
+++ b/test/DebugInfo/X86/DW_AT_specification.ll
@@ -20,23 +20,23 @@ entry:
 !llvm.dbg.cu = !{!0}
 !llvm.module.flags = !{!28}
 
-!0 = !MDCompileUnit(language: DW_LANG_C_plus_plus, producer: "clang version 3.0 ()", isOptimized: false, emissionKind: 0, file: !27, enums: !1, retainedTypes: !1, subprograms: !3, globals: !18, imports:  !1)
+!0 = !DICompileUnit(language: DW_LANG_C_plus_plus, producer: "clang version 3.0 ()", isOptimized: false, emissionKind: 0, file: !27, enums: !1, retainedTypes: !1, subprograms: !3, globals: !18, imports:  !1)
 !1 = !{}
 !3 = !{!5}
-!5 = !MDSubprogram(name: "bar", linkageName: "_ZN3foo3barEv", line: 4, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 4, file: !6, scope: null, type: !7, function: void ()* @_ZN3foo3barEv, declaration: !11)
-!6 = !MDFile(filename: "nsNativeAppSupportBase.ii", directory: "/Users/espindola/mozilla-central/obj-x86_64-apple-darwin11.2.0/toolkit/library")
-!7 = !MDSubroutineType(types: !8)
+!5 = !DISubprogram(name: "bar", linkageName: "_ZN3foo3barEv", line: 4, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 4, file: !6, scope: null, type: !7, function: void ()* @_ZN3foo3barEv, declaration: !11)
+!6 = !DIFile(filename: "nsNativeAppSupportBase.ii", directory: "/Users/espindola/mozilla-central/obj-x86_64-apple-darwin11.2.0/toolkit/library")
+!7 = !DISubroutineType(types: !8)
 !8 = !{null, !9}
-!9 = !MDDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, flags: DIFlagArtificial, baseType: !10)
-!10 = !MDCompositeType(tag: DW_TAG_structure_type, name: "foo", line: 1, flags: DIFlagFwdDecl, file: !27)
-!11 = !MDSubprogram(name: "bar", linkageName: "_ZN3foo3barEv", line: 2, isLocal: false, isDefinition: false, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 2, file: !6, scope: !12, type: !7)
-!12 = !MDCompositeType(tag: DW_TAG_class_type, name: "foo", line: 1, size: 8, align: 8, file: !27, elements: !13)
+!9 = !DIDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, flags: DIFlagArtificial, baseType: !10)
+!10 = !DICompositeType(tag: DW_TAG_structure_type, name: "foo", line: 1, flags: DIFlagFwdDecl, file: !27)
+!11 = !DISubprogram(name: "bar", linkageName: "_ZN3foo3barEv", line: 2, isLocal: false, isDefinition: false, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 2, file: !6, scope: !12, type: !7)
+!12 = !DICompositeType(tag: DW_TAG_class_type, name: "foo", line: 1, size: 8, align: 8, file: !27, elements: !13)
 !13 = !{!11}
 !18 = !{!20}
-!20 = !MDGlobalVariable(name: "x", line: 5, isLocal: true, isDefinition: true, scope: !5, file: !6, type: !21, variable: i32* @_ZZN3foo3barEvE1x)
-!21 = !MDDerivedType(tag: DW_TAG_const_type, baseType: !22)
-!22 = !MDBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
-!25 = !MDLocation(line: 6, column: 1, scope: !26)
-!26 = distinct !MDLexicalBlock(line: 4, column: 17, file: !6, scope: !5)
-!27 = !MDFile(filename: "nsNativeAppSupportBase.ii", directory: "/Users/espindola/mozilla-central/obj-x86_64-apple-darwin11.2.0/toolkit/library")
+!20 = !DIGlobalVariable(name: "x", line: 5, isLocal: true, isDefinition: true, scope: !5, file: !6, type: !21, variable: i32* @_ZZN3foo3barEvE1x)
+!21 = !DIDerivedType(tag: DW_TAG_const_type, baseType: !22)
+!22 = !DIBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
+!25 = !DILocation(line: 6, column: 1, scope: !26)
+!26 = distinct !DILexicalBlock(line: 4, column: 17, file: !6, scope: !5)
+!27 = !DIFile(filename: "nsNativeAppSupportBase.ii", directory: "/Users/espindola/mozilla-central/obj-x86_64-apple-darwin11.2.0/toolkit/library")
 !28 = !{i32 1, !"Debug Info Version", i32 3}
diff --git a/test/DebugInfo/X86/DW_AT_stmt_list_sec_offset.ll b/test/DebugInfo/X86/DW_AT_stmt_list_sec_offset.ll
index bedc26aa17f..c8e71a26b8f 100644
--- a/test/DebugInfo/X86/DW_AT_stmt_list_sec_offset.ll
+++ b/test/DebugInfo/X86/DW_AT_stmt_list_sec_offset.ll
@@ -30,15 +30,15 @@ attributes #0 = { nounwind "less-precise-fpmad"="false" "no-frame-pointer-elim"=
 !llvm.dbg.cu = !{!0}
 !llvm.module.flags = !{!9, !11}
 
-!0 = !MDCompileUnit(language: DW_LANG_C99, producer: "clang version 3.4 ", isOptimized: false, emissionKind: 0, file: !1, enums: !2, retainedTypes: !2, subprograms: !3, globals: !2, imports: !2)
-!1 = !MDFile(filename: "test.c", directory: "C:\5CProjects")
+!0 = !DICompileUnit(language: DW_LANG_C99, producer: "clang version 3.4 ", isOptimized: false, emissionKind: 0, file: !1, enums: !2, retainedTypes: !2, subprograms: !3, globals: !2, imports: !2)
+!1 = !DIFile(filename: "test.c", directory: "C:\5CProjects")
 !2 = !{}
 !3 = !{!4}
-!4 = !MDSubprogram(name: "main", line: 1, isLocal: false, isDefinition: true, virtualIndex: 6, isOptimized: false, scopeLine: 2, file: !1, scope: !5, type: !6, function: i32 ()* @main, variables: !2)
-!5 = !MDFile(filename: "test.c", directory: "C:CProjects")
-!6 = !MDSubroutineType(types: !7)
+!4 = !DISubprogram(name: "main", line: 1, isLocal: false, isDefinition: true, virtualIndex: 6, isOptimized: false, scopeLine: 2, file: !1, scope: !5, type: !6, function: i32 ()* @main, variables: !2)
+!5 = !DIFile(filename: "test.c", directory: "C:CProjects")
+!6 = !DISubroutineType(types: !7)
 !7 = !{!8}
-!8 = !MDBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
+!8 = !DIBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
 !9 = !{i32 2, !"Dwarf Version", i32 4}
-!10 = !MDLocation(line: 3, scope: !4)
+!10 = !DILocation(line: 3, scope: !4)
 !11 = !{i32 1, !"Debug Info Version", i32 3}
diff --git a/test/DebugInfo/X86/DW_TAG_friend.ll b/test/DebugInfo/X86/DW_TAG_friend.ll
index a90687a371d..8237c4086d3 100644
--- a/test/DebugInfo/X86/DW_TAG_friend.ll
+++ b/test/DebugInfo/X86/DW_TAG_friend.ll
@@ -18,27 +18,27 @@
 !llvm.dbg.cu = !{!0}
 !llvm.module.flags = !{!29}
 
-!0 = !MDCompileUnit(language: DW_LANG_C_plus_plus, producer: "clang version 3.1 (trunk 153413) (llvm/trunk 153428)", isOptimized: false, emissionKind: 0, file: !28, enums: !1, retainedTypes: !1, subprograms: !1, globals: !3, imports:  !1)
+!0 = !DICompileUnit(language: DW_LANG_C_plus_plus, producer: "clang version 3.1 (trunk 153413) (llvm/trunk 153428)", isOptimized: false, emissionKind: 0, file: !28, enums: !1, retainedTypes: !1, subprograms: !1, globals: !3, imports:  !1)
 !1 = !{}
 !3 = !{!5, !17}
-!5 = !MDGlobalVariable(name: "a", line: 10, isLocal: false, isDefinition: true, scope: null, file: !6, type: !7, variable: %class.A* @a)
-!6 = !MDFile(filename: "foo.cpp", directory: "/Users/echristo/tmp")
-!7 = !MDCompositeType(tag: DW_TAG_class_type, name: "A", line: 1, size: 32, align: 32, file: !28, elements: !8)
+!5 = !DIGlobalVariable(name: "a", line: 10, isLocal: false, isDefinition: true, scope: null, file: !6, type: !7, variable: %class.A* @a)
+!6 = !DIFile(filename: "foo.cpp", directory: "/Users/echristo/tmp")
+!7 = !DICompositeType(tag: DW_TAG_class_type, name: "A", line: 1, size: 32, align: 32, file: !28, elements: !8)
 !8 = !{!9, !11}
-!9 = !MDDerivedType(tag: DW_TAG_member, name: "a", line: 2, size: 32, align: 32, flags: DIFlagPrivate, file: !28, scope: !7, baseType: !10)
-!10 = !MDBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
-!11 = !MDSubprogram(name: "A", line: 1, isLocal: false, isDefinition: false, virtualIndex: 6, flags: DIFlagArtificial | DIFlagPrototyped, isOptimized: false, scopeLine: 1, file: !6, scope: !7, type: !12)
-!12 = !MDSubroutineType(types: !13)
+!9 = !DIDerivedType(tag: DW_TAG_member, name: "a", line: 2, size: 32, align: 32, flags: DIFlagPrivate, file: !28, scope: !7, baseType: !10)
+!10 = !DIBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
+!11 = !DISubprogram(name: "A", line: 1, isLocal: false, isDefinition: false, virtualIndex: 6, flags: DIFlagArtificial | DIFlagPrototyped, isOptimized: false, scopeLine: 1, file: !6, scope: !7, type: !12)
+!12 = !DISubroutineType(types: !13)
 !13 = !{null, !14}
-!14 = !MDDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, flags: DIFlagArtificial, baseType: !7)
-!17 = !MDGlobalVariable(name: "b", line: 11, isLocal: false, isDefinition: true, scope: null, file: !6, type: !18, variable: %class.B* @b)
-!18 = !MDCompositeType(tag: DW_TAG_class_type, name: "B", line: 5, size: 32, align: 32, file: !28, elements: !19)
+!14 = !DIDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, flags: DIFlagArtificial, baseType: !7)
+!17 = !DIGlobalVariable(name: "b", line: 11, isLocal: false, isDefinition: true, scope: null, file: !6, type: !18, variable: %class.B* @b)
+!18 = !DICompositeType(tag: DW_TAG_class_type, name: "B", line: 5, size: 32, align: 32, file: !28, elements: !19)
 !19 = !{!20, !21, !27}
-!20 = !MDDerivedType(tag: DW_TAG_member, name: "b", line: 7, size: 32, align: 32, flags: DIFlagPrivate, file: !28, scope: !18, baseType: !10)
-!21 = !MDSubprogram(name: "B", line: 5, isLocal: false, isDefinition: false, virtualIndex: 6, flags: DIFlagArtificial | DIFlagPrototyped, isOptimized: false, scopeLine: 5, file: !6, scope: !18, type: !22)
-!22 = !MDSubroutineType(types: !23)
+!20 = !DIDerivedType(tag: DW_TAG_member, name: "b", line: 7, size: 32, align: 32, flags: DIFlagPrivate, file: !28, scope: !18, baseType: !10)
+!21 = !DISubprogram(name: "B", line: 5, isLocal: false, isDefinition: false, virtualIndex: 6, flags: DIFlagArtificial | DIFlagPrototyped, isOptimized: false, scopeLine: 5, file: !6, scope: !18, type: !22)
+!22 = !DISubroutineType(types: !23)
 !23 = !{null, !24}
-!24 = !MDDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, flags: DIFlagArtificial, baseType: !18)
-!27 = !MDDerivedType(tag: DW_TAG_friend, file: !28, baseType: !7)
-!28 = !MDFile(filename: "foo.cpp", directory: "/Users/echristo/tmp")
+!24 = !DIDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, flags: DIFlagArtificial, baseType: !18)
+!27 = !DIDerivedType(tag: DW_TAG_friend, file: !28, baseType: !7)
+!28 = !DIFile(filename: "foo.cpp", directory: "/Users/echristo/tmp")
 !29 = !{i32 1, !"Debug Info Version", i32 3}
diff --git a/test/DebugInfo/X86/InlinedFnLocalVar.ll b/test/DebugInfo/X86/InlinedFnLocalVar.ll
index 3832e4f32b8..c73a7b6a421 100644
--- a/test/DebugInfo/X86/InlinedFnLocalVar.ll
+++ b/test/DebugInfo/X86/InlinedFnLocalVar.ll
@@ -15,8 +15,8 @@ declare void @llvm.dbg.value(metadata, i64, metadata, metadata) nounwind readnon
 define i32 @bar() nounwind ssp {
 entry:
   %0 = load i32, i32* @i, align 4, !dbg !17            ; <i32> [#uses=2]
-  tail call void @llvm.dbg.value(metadata i32 %0, i64 0, metadata !109, metadata !MDExpression()), !dbg !19
-  tail call void @llvm.dbg.declare(metadata !29, metadata !110, metadata !MDExpression()), !dbg !21
+  tail call void @llvm.dbg.value(metadata i32 %0, i64 0, metadata !109, metadata !DIExpression()), !dbg !19
+  tail call void @llvm.dbg.declare(metadata !29, metadata !110, metadata !DIExpression()), !dbg !21
   %1 = mul nsw i32 %0, %0, !dbg !22               ; <i32> [#uses=2]
   store i32 %1, i32* @i, align 4, !dbg !17
   ret i32 %1, !dbg !23
@@ -25,37 +25,37 @@ entry:
 !llvm.dbg.cu = !{!2}
 !llvm.module.flags = !{!28}
 
-!0 = !MDSubprogram(name: "foo", line: 9, isLocal: true, isDefinition: true, virtualIndex: 6, isOptimized: true, scopeLine: 9, file: !27, scope: !1, type: !3, variables: !24)
-!1 = !MDFile(filename: "bar.c", directory: "/tmp/")
-!2 = !MDCompileUnit(language: DW_LANG_C89, producer: "4.2.1 (Based on Apple Inc. build 5658) (LLVM build)", isOptimized: true, emissionKind: 0, file: !27, enums: !20, retainedTypes: !20, subprograms: !25, globals: !26, imports:  !20)
-!3 = !MDSubroutineType(types: !4)
+!0 = !DISubprogram(name: "foo", line: 9, isLocal: true, isDefinition: true, virtualIndex: 6, isOptimized: true, scopeLine: 9, file: !27, scope: !1, type: !3, variables: !24)
+!1 = !DIFile(filename: "bar.c", directory: "/tmp/")
+!2 = !DICompileUnit(language: DW_LANG_C89, producer: "4.2.1 (Based on Apple Inc. build 5658) (LLVM build)", isOptimized: true, emissionKind: 0, file: !27, enums: !20, retainedTypes: !20, subprograms: !25, globals: !26, imports:  !20)
+!3 = !DISubroutineType(types: !4)
 !4 = !{!5, !5}
-!5 = !MDBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
-!6 = !MDSubprogram(name: "bar", linkageName: "bar", line: 14, isLocal: false, isDefinition: true, virtualIndex: 6, isOptimized: true, file: !27, scope: !1, type: !7, function: i32 ()* @bar)
-!7 = !MDSubroutineType(types: !8)
+!5 = !DIBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
+!6 = !DISubprogram(name: "bar", linkageName: "bar", line: 14, isLocal: false, isDefinition: true, virtualIndex: 6, isOptimized: true, file: !27, scope: !1, type: !7, function: i32 ()* @bar)
+!7 = !DISubroutineType(types: !8)
 !8 = !{!5}
-!9 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "j", line: 9, arg: 0, scope: !0, file: !1, type: !5)
-!10 = !MDLocalVariable(tag: DW_TAG_auto_variable, name: "xyz", line: 10, scope: !11, file: !1, type: !12)
+!9 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "j", line: 9, arg: 1, scope: !0, file: !1, type: !5)
+!10 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "xyz", line: 10, scope: !11, file: !1, type: !12)
 
-!109 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "j", line: 9, arg: 0, scope: !0, file: !1, type: !5)
-!110 = !MDLocalVariable(tag: DW_TAG_auto_variable, name: "xyz", line: 10, scope: !11, file: !1, type: !12)
+!109 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "j", line: 9, arg: 1, scope: !0, file: !1, type: !5)
+!110 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "xyz", line: 10, scope: !11, file: !1, type: !12)
 
-!11 = distinct !MDLexicalBlock(line: 9, column: 0, file: !1, scope: !0)
-!12 = !MDCompositeType(tag: DW_TAG_structure_type, name: "X", line: 10, size: 64, align: 32, file: !27, scope: !0, elements: !13)
+!11 = distinct !DILexicalBlock(line: 9, column: 0, file: !1, scope: !0)
+!12 = !DICompositeType(tag: DW_TAG_structure_type, name: "X", line: 10, size: 64, align: 32, file: !27, scope: !0, elements: !13)
 !13 = !{!14, !15}
-!14 = !MDDerivedType(tag: DW_TAG_member, name: "a", line: 10, size: 32, align: 32, file: !27, scope: !12, baseType: !5)
-!15 = !MDDerivedType(tag: DW_TAG_member, name: "b", line: 10, size: 32, align: 32, offset: 32, file: !27, scope: !12, baseType: !5)
-!16 = !MDGlobalVariable(name: "i", line: 5, isLocal: false, isDefinition: true, scope: !1, file: !1, type: !5, variable: i32* @i)
-!17 = !MDLocation(line: 15, scope: !18)
-!18 = distinct !MDLexicalBlock(line: 14, column: 0, file: !1, scope: !6)
-!19 = !MDLocation(line: 9, scope: !0, inlinedAt: !17)
+!14 = !DIDerivedType(tag: DW_TAG_member, name: "a", line: 10, size: 32, align: 32, file: !27, scope: !12, baseType: !5)
+!15 = !DIDerivedType(tag: DW_TAG_member, name: "b", line: 10, size: 32, align: 32, offset: 32, file: !27, scope: !12, baseType: !5)
+!16 = !DIGlobalVariable(name: "i", line: 5, isLocal: false, isDefinition: true, scope: !1, file: !1, type: !5, variable: i32* @i)
+!17 = !DILocation(line: 15, scope: !18)
+!18 = distinct !DILexicalBlock(line: 14, column: 0, file: !1, scope: !6)
+!19 = !DILocation(line: 9, scope: !0, inlinedAt: !17)
 !20 = !{}
-!21 = !MDLocation(line: 9, scope: !11, inlinedAt: !17)
-!22 = !MDLocation(line: 11, scope: !11, inlinedAt: !17)
-!23 = !MDLocation(line: 16, scope: !18)
+!21 = !DILocation(line: 9, scope: !11, inlinedAt: !17)
+!22 = !DILocation(line: 11, scope: !11, inlinedAt: !17)
+!23 = !DILocation(line: 16, scope: !18)
 !24 = !{!9, !10}
 !25 = !{!0, !6}
 !26 = !{!16}
-!27 = !MDFile(filename: "bar.c", directory: "/tmp/")
+!27 = !DIFile(filename: "bar.c", directory: "/tmp/")
 !28 = !{i32 1, !"Debug Info Version", i32 3}
 !29 = !{null}
diff --git a/test/DebugInfo/X86/aligned_stack_var.ll b/test/DebugInfo/X86/aligned_stack_var.ll
index c9f24f4d8af..560125d6f40 100644
--- a/test/DebugInfo/X86/aligned_stack_var.ll
+++ b/test/DebugInfo/X86/aligned_stack_var.ll
@@ -18,7 +18,7 @@
 define void @_Z3runv() nounwind uwtable {
 entry:
   %x = alloca i32, align 32
-  call void @llvm.dbg.declare(metadata i32* %x, metadata !9, metadata !MDExpression()), !dbg !12
+  call void @llvm.dbg.declare(metadata i32* %x, metadata !9, metadata !DIExpression()), !dbg !12
   ret void, !dbg !13
 }
 
@@ -27,17 +27,17 @@ declare void @llvm.dbg.declare(metadata, metadata, metadata) nounwind readnone
 !llvm.dbg.cu = !{!0}
 !llvm.module.flags = !{!15}
 
-!0 = !MDCompileUnit(language: DW_LANG_C_plus_plus, producer: "clang version 3.2 (trunk 155696:155697) (llvm/trunk 155696)", isOptimized: false, emissionKind: 0, file: !14, enums: !1, retainedTypes: !1, subprograms: !3, globals: !1, imports:  !1)
+!0 = !DICompileUnit(language: DW_LANG_C_plus_plus, producer: "clang version 3.2 (trunk 155696:155697) (llvm/trunk 155696)", isOptimized: false, emissionKind: 0, file: !14, enums: !1, retainedTypes: !1, subprograms: !3, globals: !1, imports:  !1)
 !1 = !{}
 !3 = !{!5}
-!5 = !MDSubprogram(name: "run", linkageName: "_Z3runv", line: 1, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 1, file: !14, scope: !6, type: !7, function: void ()* @_Z3runv, variables: !1)
-!6 = !MDFile(filename: "test.cc", directory: "/home/samsonov/debuginfo")
-!7 = !MDSubroutineType(types: !8)
+!5 = !DISubprogram(name: "run", linkageName: "_Z3runv", line: 1, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 1, file: !14, scope: !6, type: !7, function: void ()* @_Z3runv, variables: !1)
+!6 = !DIFile(filename: "test.cc", directory: "/home/samsonov/debuginfo")
+!7 = !DISubroutineType(types: !8)
 !8 = !{null}
-!9 = !MDLocalVariable(tag: DW_TAG_auto_variable, name: "x", line: 2, scope: !10, file: !6, type: !11)
-!10 = distinct !MDLexicalBlock(line: 1, column: 12, file: !14, scope: !5)
-!11 = !MDBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
-!12 = !MDLocation(line: 2, column: 7, scope: !10)
-!13 = !MDLocation(line: 3, column: 1, scope: !10)
-!14 = !MDFile(filename: "test.cc", directory: "/home/samsonov/debuginfo")
+!9 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "x", line: 2, scope: !10, file: !6, type: !11)
+!10 = distinct !DILexicalBlock(line: 1, column: 12, file: !14, scope: !5)
+!11 = !DIBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
+!12 = !DILocation(line: 2, column: 7, scope: !10)
+!13 = !DILocation(line: 3, column: 1, scope: !10)
+!14 = !DIFile(filename: "test.cc", directory: "/home/samsonov/debuginfo")
 !15 = !{i32 1, !"Debug Info Version", i32 3}
diff --git a/test/DebugInfo/X86/arange-and-stub.ll b/test/DebugInfo/X86/arange-and-stub.ll
index 1952432ef42..bbc6de5e573 100644
--- a/test/DebugInfo/X86/arange-and-stub.ll
+++ b/test/DebugInfo/X86/arange-and-stub.ll
@@ -16,7 +16,7 @@ define void @foo() {
   ret void
 }
 
-define void @bar() {
+define void @bar() personality i8* bitcast (void ()* @foo to i8*) {
   invoke void @foo()
           to label %invoke.cont unwind label %lpad
 
@@ -24,7 +24,7 @@ invoke.cont:                                      ; preds = %0
   ret void
 
 lpad:                                             ; preds = %0
-  %tmp1 = landingpad { i8*, i32 } personality i8* bitcast (void ()* @foo to i8*)
+  %tmp1 = landingpad { i8*, i32 }
           filter [1 x i8*] [i8* bitcast (i8** @_ZTId to i8*)]
   ret void
 }
@@ -32,22 +32,22 @@ lpad:                                             ; preds = %0
 !llvm.dbg.cu = !{!0}
 !llvm.module.flags = !{!17, !18}
 
-!0 = !MDCompileUnit(language: DW_LANG_C_plus_plus, file: !1, producer: "clang version 3.7.0 (trunk 234308) (llvm/trunk 234310)", isOptimized: false, runtimeVersion: 0, emissionKind: 1, enums: !2, retainedTypes: !2, subprograms: !3, globals: !10, imports: !2)
-!1 = !MDFile(filename: "/Users/espindola/llvm/<stdin>", directory: "/Users/espindola/llvm/build")
+!0 = !DICompileUnit(language: DW_LANG_C_plus_plus, file: !1, producer: "clang version 3.7.0 (trunk 234308) (llvm/trunk 234310)", isOptimized: false, runtimeVersion: 0, emissionKind: 1, enums: !2, retainedTypes: !2, subprograms: !3, globals: !10, imports: !2)
+!1 = !DIFile(filename: "/Users/espindola/llvm/<stdin>", directory: "/Users/espindola/llvm/build")
 !2 = !{}
 !3 = !{!4, !9}
-!4 = !MDSubprogram(name: "foo", linkageName: "foo", scope: !5, file: !5, line: 1, type: !6, isLocal: false, isDefinition: true, scopeLine: 1, flags: DIFlagPrototyped, isOptimized: false, function: void ()* @foo, variables: !2)
-!5 = !MDFile(filename: "/Users/espindola/llvm/test.cpp", directory: "/Users/espindola/llvm/build")
-!6 = !MDSubroutineType(types: !7)
+!4 = !DISubprogram(name: "foo", linkageName: "foo", scope: !5, file: !5, line: 1, type: !6, isLocal: false, isDefinition: true, scopeLine: 1, flags: DIFlagPrototyped, isOptimized: false, function: void ()* @foo, variables: !2)
+!5 = !DIFile(filename: "/Users/espindola/llvm/test.cpp", directory: "/Users/espindola/llvm/build")
+!6 = !DISubroutineType(types: !7)
 !7 = !{null, !8}
-!8 = !MDBasicType(name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
-!9 = !MDSubprogram(name: "bar_d", linkageName: "bar", scope: !5, file: !5, line: 3, type: !6, isLocal: false, isDefinition: true, scopeLine: 3, flags: DIFlagPrototyped, isOptimized: false, function: void ()* @bar, variables: !2)
+!8 = !DIBasicType(name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
+!9 = !DISubprogram(name: "bar_d", linkageName: "bar", scope: !5, file: !5, line: 3, type: !6, isLocal: false, isDefinition: true, scopeLine: 3, flags: DIFlagPrototyped, isOptimized: false, function: void ()* @bar, variables: !2)
 !10 = !{!11}
-!11 = !MDGlobalVariable(name: "zed", scope: !0, file: !5, line: 6, type: !12, isLocal: false, isDefinition: true, variable: [1 x void ()*]* @zed)
-!12 = !MDCompositeType(tag: DW_TAG_array_type, baseType: !13, size: 64, align: 64, elements: !15)
-!13 = !MDDerivedType(tag: DW_TAG_typedef, name: "vifunc", file: !5, line: 5, baseType: !14)
-!14 = !MDDerivedType(tag: DW_TAG_pointer_type, baseType: !6, size: 64, align: 64)
+!11 = !DIGlobalVariable(name: "zed", scope: !0, file: !5, line: 6, type: !12, isLocal: false, isDefinition: true, variable: [1 x void ()*]* @zed)
+!12 = !DICompositeType(tag: DW_TAG_array_type, baseType: !13, size: 64, align: 64, elements: !15)
+!13 = !DIDerivedType(tag: DW_TAG_typedef, name: "vifunc", file: !5, line: 5, baseType: !14)
+!14 = !DIDerivedType(tag: DW_TAG_pointer_type, baseType: !6, size: 64, align: 64)
 !15 = !{!16}
-!16 = !MDSubrange(count: 1)
+!16 = !DISubrange(count: 1)
 !17 = !{i32 2, !"Dwarf Version", i32 4}
 !18 = !{i32 2, !"Debug Info Version", i32 3}
diff --git a/test/DebugInfo/X86/arange.ll b/test/DebugInfo/X86/arange.ll
index 61743fcbe2c..0b0c2a0b3ee 100644
--- a/test/DebugInfo/X86/arange.ll
+++ b/test/DebugInfo/X86/arange.ll
@@ -29,18 +29,18 @@
 !llvm.module.flags = !{!12, !13}
 !llvm.ident = !{!14}
 
-!0 = !MDCompileUnit(language: DW_LANG_C_plus_plus, producer: "clang version 3.5 ", isOptimized: false, emissionKind: 0, file: !1, enums: !2, retainedTypes: !3, subprograms: !2, globals: !9, imports: !2)
-!1 = !MDFile(filename: "simple.cpp", directory: "/tmp/dbginfo")
+!0 = !DICompileUnit(language: DW_LANG_C_plus_plus, producer: "clang version 3.5 ", isOptimized: false, emissionKind: 0, file: !1, enums: !2, retainedTypes: !3, subprograms: !2, globals: !9, imports: !2)
+!1 = !DIFile(filename: "simple.cpp", directory: "/tmp/dbginfo")
 !2 = !{}
 !3 = !{!4}
-!4 = !MDCompositeType(tag: DW_TAG_structure_type, name: "foo<&i>", line: 3, size: 8, align: 8, file: !1, elements: !2, templateParams: !5, identifier: "_ZTS3fooIXadL_Z1iEEE")
+!4 = !DICompositeType(tag: DW_TAG_structure_type, name: "foo<&i>", line: 3, size: 8, align: 8, file: !1, elements: !2, templateParams: !5, identifier: "_ZTS3fooIXadL_Z1iEEE")
 !5 = !{!6}
-!6 = !MDTemplateValueParameter(tag: DW_TAG_template_value_parameter, name: "x", type: !7, value: i32* @i)
-!7 = !MDDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, baseType: !8)
-!8 = !MDBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
+!6 = !DITemplateValueParameter(tag: DW_TAG_template_value_parameter, name: "x", type: !7, value: i32* @i)
+!7 = !DIDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, baseType: !8)
+!8 = !DIBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
 !9 = !{!10}
-!10 = !MDGlobalVariable(name: "f", line: 6, isLocal: false, isDefinition: true, scope: null, file: !11, type: !4, variable: %struct.foo* @f)
-!11 = !MDFile(filename: "simple.cpp", directory: "/tmp/dbginfo")
+!10 = !DIGlobalVariable(name: "f", line: 6, isLocal: false, isDefinition: true, scope: null, file: !11, type: !4, variable: %struct.foo* @f)
+!11 = !DIFile(filename: "simple.cpp", directory: "/tmp/dbginfo")
 !12 = !{i32 2, !"Dwarf Version", i32 4}
 !13 = !{i32 1, !"Debug Info Version", i32 3}
 !14 = !{!"clang version 3.5 "}
diff --git a/test/DebugInfo/X86/arguments.ll b/test/DebugInfo/X86/arguments.ll
index 46327118a78..0af119b241f 100644
--- a/test/DebugInfo/X86/arguments.ll
+++ b/test/DebugInfo/X86/arguments.ll
@@ -31,8 +31,8 @@
 ; Function Attrs: nounwind uwtable
 define void @_Z4func3fooS_(%struct.foo* %f, %struct.foo* %g) #0 {
 entry:
-  call void @llvm.dbg.declare(metadata %struct.foo* %f, metadata !19, metadata !MDExpression()), !dbg !20
-  call void @llvm.dbg.declare(metadata %struct.foo* %g, metadata !21, metadata !MDExpression()), !dbg !20
+  call void @llvm.dbg.declare(metadata %struct.foo* %f, metadata !19, metadata !DIExpression()), !dbg !20
+  call void @llvm.dbg.declare(metadata %struct.foo* %g, metadata !21, metadata !DIExpression()), !dbg !20
   %i = getelementptr inbounds %struct.foo, %struct.foo* %f, i32 0, i32 0, !dbg !22
   %0 = load i32, i32* %i, align 4, !dbg !22
   %inc = add nsw i32 %0, 1, !dbg !22
@@ -49,27 +49,27 @@ attributes #1 = { nounwind readnone }
 !llvm.dbg.cu = !{!0}
 !llvm.module.flags = !{!24}
 
-!0 = !MDCompileUnit(language: DW_LANG_C_plus_plus, producer: "clang version 3.4 ", isOptimized: false, emissionKind: 1, file: !1, enums: !2, retainedTypes: !2, subprograms: !3, globals: !2, imports: !2)
-!1 = !MDFile(filename: "scratch.cpp", directory: "/usr/local/google/home/blaikie/dev/scratch")
+!0 = !DICompileUnit(language: DW_LANG_C_plus_plus, producer: "clang version 3.4 ", isOptimized: false, emissionKind: 1, file: !1, enums: !2, retainedTypes: !2, subprograms: !3, globals: !2, imports: !2)
+!1 = !DIFile(filename: "scratch.cpp", directory: "/usr/local/google/home/blaikie/dev/scratch")
 !2 = !{}
 !3 = !{!4}
-!4 = !MDSubprogram(name: "func", linkageName: "_Z4func3fooS_", line: 6, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 6, file: !1, scope: !5, type: !6, function: void (%struct.foo*, %struct.foo*)* @_Z4func3fooS_, variables: !2)
-!5 = !MDFile(filename: "scratch.cpp", directory: "/usr/local/google/home/blaikie/dev/scratch")
-!6 = !MDSubroutineType(types: !7)
+!4 = !DISubprogram(name: "func", linkageName: "_Z4func3fooS_", line: 6, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 6, file: !1, scope: !5, type: !6, function: void (%struct.foo*, %struct.foo*)* @_Z4func3fooS_, variables: !2)
+!5 = !DIFile(filename: "scratch.cpp", directory: "/usr/local/google/home/blaikie/dev/scratch")
+!6 = !DISubroutineType(types: !7)
 !7 = !{null, !8, !8}
-!8 = !MDCompositeType(tag: DW_TAG_structure_type, name: "foo", line: 1, size: 32, align: 32, file: !1, elements: !9)
+!8 = !DICompositeType(tag: DW_TAG_structure_type, name: "foo", line: 1, size: 32, align: 32, file: !1, elements: !9)
 !9 = !{!10, !12}
-!10 = !MDDerivedType(tag: DW_TAG_member, name: "i", line: 3, size: 32, align: 32, file: !1, scope: !8, baseType: !11)
-!11 = !MDBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
-!12 = !MDSubprogram(name: "foo", line: 2, isLocal: false, isDefinition: false, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 2, file: !1, scope: !8, type: !13)
-!13 = !MDSubroutineType(types: !14)
+!10 = !DIDerivedType(tag: DW_TAG_member, name: "i", line: 3, size: 32, align: 32, file: !1, scope: !8, baseType: !11)
+!11 = !DIBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
+!12 = !DISubprogram(name: "foo", line: 2, isLocal: false, isDefinition: false, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 2, file: !1, scope: !8, type: !13)
+!13 = !DISubroutineType(types: !14)
 !14 = !{null, !15, !16}
-!15 = !MDDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, flags: DIFlagArtificial | DIFlagObjectPointer, baseType: !8)
-!16 = !MDDerivedType(tag: DW_TAG_reference_type, baseType: !17)
-!17 = !MDDerivedType(tag: DW_TAG_const_type, baseType: !8)
-!19 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "f", line: 6, arg: 1, scope: !4, file: !5, type: !8)
-!20 = !MDLocation(line: 6, scope: !4)
-!21 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "g", line: 6, arg: 2, scope: !4, file: !5, type: !8)
-!22 = !MDLocation(line: 7, scope: !4)
-!23 = !MDLocation(line: 8, scope: !4)
+!15 = !DIDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, flags: DIFlagArtificial | DIFlagObjectPointer, baseType: !8)
+!16 = !DIDerivedType(tag: DW_TAG_reference_type, baseType: !17)
+!17 = !DIDerivedType(tag: DW_TAG_const_type, baseType: !8)
+!19 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "f", line: 6, arg: 1, scope: !4, file: !5, type: !8)
+!20 = !DILocation(line: 6, scope: !4)
+!21 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "g", line: 6, arg: 2, scope: !4, file: !5, type: !8)
+!22 = !DILocation(line: 7, scope: !4)
+!23 = !DILocation(line: 8, scope: !4)
 !24 = !{i32 1, !"Debug Info Version", i32 3}
diff --git a/test/DebugInfo/X86/array.ll b/test/DebugInfo/X86/array.ll
index aae8c65ac8e..8da2079a053 100644
--- a/test/DebugInfo/X86/array.ll
+++ b/test/DebugInfo/X86/array.ll
@@ -25,7 +25,7 @@ target triple = "x86_64-apple-macosx10.9.0"
 
 ; Function Attrs: nounwind ssp uwtable
 define void @f(i32* nocapture %p) #0 {
-  tail call void @llvm.dbg.value(metadata i32* %p, i64 0, metadata !11, metadata !MDExpression()), !dbg !28
+  tail call void @llvm.dbg.value(metadata i32* %p, i64 0, metadata !11, metadata !DIExpression()), !dbg !28
   store i32 42, i32* %p, align 4, !dbg !29, !tbaa !30
   ret void, !dbg !34
 }
@@ -33,15 +33,15 @@ define void @f(i32* nocapture %p) #0 {
 ; Function Attrs: nounwind ssp uwtable
 define i32 @main(i32 %argc, i8** nocapture readnone %argv) #0 {
   %array = alloca [4 x i32], align 16
-  tail call void @llvm.dbg.value(metadata i32 %argc, i64 0, metadata !19, metadata !MDExpression()), !dbg !35
-  tail call void @llvm.dbg.value(metadata i8** %argv, i64 0, metadata !20, metadata !MDExpression()), !dbg !35
-  tail call void @llvm.dbg.value(metadata [4 x i32]* %array, i64 0, metadata !21, metadata !MDExpression()), !dbg !36
+  tail call void @llvm.dbg.value(metadata i32 %argc, i64 0, metadata !19, metadata !DIExpression()), !dbg !35
+  tail call void @llvm.dbg.value(metadata i8** %argv, i64 0, metadata !20, metadata !DIExpression()), !dbg !35
+  tail call void @llvm.dbg.value(metadata [4 x i32]* %array, i64 0, metadata !21, metadata !DIExpression()), !dbg !36
   %1 = bitcast [4 x i32]* %array to i8*, !dbg !36
   call void @llvm.memcpy.p0i8.p0i8.i64(i8* %1, i8* bitcast ([4 x i32]* @main.array to i8*), i64 16, i32 16, i1 false), !dbg !36
-  tail call void @llvm.dbg.value(metadata [4 x i32]* %array, i64 0, metadata !21, metadata !MDExpression()), !dbg !36
+  tail call void @llvm.dbg.value(metadata [4 x i32]* %array, i64 0, metadata !21, metadata !DIExpression()), !dbg !36
   %2 = getelementptr inbounds [4 x i32], [4 x i32]* %array, i64 0, i64 0, !dbg !37
   call void @f(i32* %2), !dbg !37
-  tail call void @llvm.dbg.value(metadata [4 x i32]* %array, i64 0, metadata !21, metadata !MDExpression()), !dbg !36
+  tail call void @llvm.dbg.value(metadata [4 x i32]* %array, i64 0, metadata !21, metadata !DIExpression()), !dbg !36
   %3 = load i32, i32* %2, align 16, !dbg !38, !tbaa !30
   ret i32 %3, !dbg !38
 }
@@ -60,42 +60,42 @@ attributes #2 = { nounwind readnone }
 !llvm.module.flags = !{!25, !26}
 !llvm.ident = !{!27}
 
-!0 = !MDCompileUnit(language: DW_LANG_C99, producer: "clang version 3.5.0 ", isOptimized: true, emissionKind: 1, file: !1, enums: !2, retainedTypes: !2, subprograms: !3, globals: !2, imports: !2)
-!1 = !MDFile(filename: "array.c", directory: "")
+!0 = !DICompileUnit(language: DW_LANG_C99, producer: "clang version 3.5.0 ", isOptimized: true, emissionKind: 1, file: !1, enums: !2, retainedTypes: !2, subprograms: !3, globals: !2, imports: !2)
+!1 = !DIFile(filename: "array.c", directory: "")
 !2 = !{}
 !3 = !{!4, !12}
-!4 = !MDSubprogram(name: "f", line: 1, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: true, scopeLine: 1, file: !1, scope: !5, type: !6, function: void (i32*)* @f, variables: !10)
-!5 = !MDFile(filename: "array.c", directory: "")
-!6 = !MDSubroutineType(types: !7)
+!4 = !DISubprogram(name: "f", line: 1, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: true, scopeLine: 1, file: !1, scope: !5, type: !6, function: void (i32*)* @f, variables: !10)
+!5 = !DIFile(filename: "array.c", directory: "")
+!6 = !DISubroutineType(types: !7)
 !7 = !{null, !8}
-!8 = !MDDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, baseType: !9)
-!9 = !MDBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
+!8 = !DIDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, baseType: !9)
+!9 = !DIBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
 !10 = !{!11}
-!11 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "p", line: 1, arg: 1, scope: !4, file: !5, type: !8)
-!12 = !MDSubprogram(name: "main", line: 5, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: true, scopeLine: 5, file: !1, scope: !5, type: !13, function: i32 (i32, i8**)* @main, variables: !18)
-!13 = !MDSubroutineType(types: !14)
+!11 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "p", line: 1, arg: 1, scope: !4, file: !5, type: !8)
+!12 = !DISubprogram(name: "main", line: 5, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: true, scopeLine: 5, file: !1, scope: !5, type: !13, function: i32 (i32, i8**)* @main, variables: !18)
+!13 = !DISubroutineType(types: !14)
 !14 = !{!9, !9, !15}
-!15 = !MDDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, baseType: !16)
-!16 = !MDDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, baseType: !17)
-!17 = !MDBasicType(tag: DW_TAG_base_type, name: "char", size: 8, align: 8, encoding: DW_ATE_signed_char)
+!15 = !DIDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, baseType: !16)
+!16 = !DIDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, baseType: !17)
+!17 = !DIBasicType(tag: DW_TAG_base_type, name: "char", size: 8, align: 8, encoding: DW_ATE_signed_char)
 !18 = !{!19, !20, !21}
-!19 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "argc", line: 5, arg: 1, scope: !12, file: !5, type: !9)
-!20 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "argv", line: 5, arg: 2, scope: !12, file: !5, type: !15)
-!21 = !MDLocalVariable(tag: DW_TAG_auto_variable, name: "array", line: 6, scope: !12, file: !5, type: !22)
-!22 = !MDCompositeType(tag: DW_TAG_array_type, size: 128, align: 32, baseType: !9, elements: !23)
+!19 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "argc", line: 5, arg: 1, scope: !12, file: !5, type: !9)
+!20 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "argv", line: 5, arg: 2, scope: !12, file: !5, type: !15)
+!21 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "array", line: 6, scope: !12, file: !5, type: !22)
+!22 = !DICompositeType(tag: DW_TAG_array_type, size: 128, align: 32, baseType: !9, elements: !23)
 !23 = !{!24}
-!24 = !MDSubrange(count: 4)
+!24 = !DISubrange(count: 4)
 !25 = !{i32 2, !"Dwarf Version", i32 2}
 !26 = !{i32 1, !"Debug Info Version", i32 3}
 !27 = !{!"clang version 3.5.0 "}
-!28 = !MDLocation(line: 1, scope: !4)
-!29 = !MDLocation(line: 2, scope: !4)
+!28 = !DILocation(line: 1, scope: !4)
+!29 = !DILocation(line: 2, scope: !4)
 !30 = !{!31, !31, i64 0}
 !31 = !{!"int", !32, i64 0}
 !32 = !{!"omnipotent char", !33, i64 0}
 !33 = !{!"Simple C/C++ TBAA"}
-!34 = !MDLocation(line: 3, scope: !4)
-!35 = !MDLocation(line: 5, scope: !12)
-!36 = !MDLocation(line: 6, scope: !12)
-!37 = !MDLocation(line: 7, scope: !12)
-!38 = !MDLocation(line: 8, scope: !12)
+!34 = !DILocation(line: 3, scope: !4)
+!35 = !DILocation(line: 5, scope: !12)
+!36 = !DILocation(line: 6, scope: !12)
+!37 = !DILocation(line: 7, scope: !12)
+!38 = !DILocation(line: 8, scope: !12)
diff --git a/test/DebugInfo/X86/array2.ll b/test/DebugInfo/X86/array2.ll
index b54992abe90..e08c52640b3 100644
--- a/test/DebugInfo/X86/array2.ll
+++ b/test/DebugInfo/X86/array2.ll
@@ -17,8 +17,8 @@
 ;
 ; CHECK: define i32 @main
 ; CHECK: call void @llvm.dbg.value(metadata i32 42, i64 0, metadata ![[ARRAY:[0-9]+]], metadata ![[EXPR:[0-9]+]])
-; CHECK: ![[ARRAY]] = !MDLocalVariable(tag: DW_TAG_auto_variable, name: "array",{{.*}} line: 6
-; CHECK: ![[EXPR]] = !MDExpression(DW_OP_bit_piece, 0, 32)
+; CHECK: ![[ARRAY]] = !DILocalVariable(tag: DW_TAG_auto_variable, name: "array",{{.*}} line: 6
+; CHECK: ![[EXPR]] = !DIExpression(DW_OP_bit_piece, 0, 32)
 target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-apple-macosx10.9.0"
 
@@ -29,7 +29,7 @@ define void @f(i32* %p) #0 {
 entry:
   %p.addr = alloca i32*, align 8
   store i32* %p, i32** %p.addr, align 8
-  call void @llvm.dbg.declare(metadata i32** %p.addr, metadata !19, metadata !MDExpression()), !dbg !20
+  call void @llvm.dbg.declare(metadata i32** %p.addr, metadata !19, metadata !DIExpression()), !dbg !20
   %0 = load i32*, i32** %p.addr, align 8, !dbg !21
   %arrayidx = getelementptr inbounds i32, i32* %0, i64 0, !dbg !21
   store i32 42, i32* %arrayidx, align 4, !dbg !21
@@ -48,10 +48,10 @@ entry:
   %array = alloca [4 x i32], align 16
   store i32 0, i32* %retval
   store i32 %argc, i32* %argc.addr, align 4
-  call void @llvm.dbg.declare(metadata i32* %argc.addr, metadata !23, metadata !MDExpression()), !dbg !24
+  call void @llvm.dbg.declare(metadata i32* %argc.addr, metadata !23, metadata !DIExpression()), !dbg !24
   store i8** %argv, i8*** %argv.addr, align 8
-  call void @llvm.dbg.declare(metadata i8*** %argv.addr, metadata !25, metadata !MDExpression()), !dbg !24
-  call void @llvm.dbg.declare(metadata [4 x i32]* %array, metadata !26, metadata !MDExpression()), !dbg !30
+  call void @llvm.dbg.declare(metadata i8*** %argv.addr, metadata !25, metadata !DIExpression()), !dbg !24
+  call void @llvm.dbg.declare(metadata [4 x i32]* %array, metadata !26, metadata !DIExpression()), !dbg !30
   %0 = bitcast [4 x i32]* %array to i8*, !dbg !30
   call void @llvm.memcpy.p0i8.p0i8.i64(i8* %0, i8* bitcast ([4 x i32]* @main.array to i8*), i64 16, i32 16, i1 false), !dbg !30
   %arraydecay = getelementptr inbounds [4 x i32], [4 x i32]* %array, i32 0, i32 0, !dbg !31
@@ -72,36 +72,36 @@ attributes #2 = { nounwind }
 !llvm.module.flags = !{!16, !17}
 !llvm.ident = !{!18}
 
-!0 = !MDCompileUnit(language: DW_LANG_C99, producer: "clang version 3.5.0 ", isOptimized: false, emissionKind: 1, file: !1, enums: !2, retainedTypes: !2, subprograms: !3, globals: !2, imports: !2)
-!1 = !MDFile(filename: "array.c", directory: "")
+!0 = !DICompileUnit(language: DW_LANG_C99, producer: "clang version 3.5.0 ", isOptimized: false, emissionKind: 1, file: !1, enums: !2, retainedTypes: !2, subprograms: !3, globals: !2, imports: !2)
+!1 = !DIFile(filename: "array.c", directory: "")
 !2 = !{}
 !3 = !{!4, !10}
-!4 = !MDSubprogram(name: "f", line: 1, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 1, file: !1, scope: !5, type: !6, function: void (i32*)* @f, variables: !2)
-!5 = !MDFile(filename: "array.c", directory: "")
-!6 = !MDSubroutineType(types: !7)
+!4 = !DISubprogram(name: "f", line: 1, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 1, file: !1, scope: !5, type: !6, function: void (i32*)* @f, variables: !2)
+!5 = !DIFile(filename: "array.c", directory: "")
+!6 = !DISubroutineType(types: !7)
 !7 = !{null, !8}
-!8 = !MDDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, baseType: !9)
-!9 = !MDBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
-!10 = !MDSubprogram(name: "main", line: 5, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 5, file: !1, scope: !5, type: !11, function: i32 (i32, i8**)* @main, variables: !2)
-!11 = !MDSubroutineType(types: !12)
+!8 = !DIDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, baseType: !9)
+!9 = !DIBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
+!10 = !DISubprogram(name: "main", line: 5, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 5, file: !1, scope: !5, type: !11, function: i32 (i32, i8**)* @main, variables: !2)
+!11 = !DISubroutineType(types: !12)
 !12 = !{!9, !9, !13}
-!13 = !MDDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, baseType: !14)
-!14 = !MDDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, baseType: !15)
-!15 = !MDBasicType(tag: DW_TAG_base_type, name: "char", size: 8, align: 8, encoding: DW_ATE_signed_char)
+!13 = !DIDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, baseType: !14)
+!14 = !DIDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, baseType: !15)
+!15 = !DIBasicType(tag: DW_TAG_base_type, name: "char", size: 8, align: 8, encoding: DW_ATE_signed_char)
 !16 = !{i32 2, !"Dwarf Version", i32 2}
 !17 = !{i32 1, !"Debug Info Version", i32 3}
 !18 = !{!"clang version 3.5.0 "}
-!19 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "p", line: 1, arg: 1, scope: !4, file: !5, type: !8)
-!20 = !MDLocation(line: 1, scope: !4)
-!21 = !MDLocation(line: 2, scope: !4)
-!22 = !MDLocation(line: 3, scope: !4)
-!23 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "argc", line: 5, arg: 1, scope: !10, file: !5, type: !9)
-!24 = !MDLocation(line: 5, scope: !10)
-!25 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "argv", line: 5, arg: 2, scope: !10, file: !5, type: !13)
-!26 = !MDLocalVariable(tag: DW_TAG_auto_variable, name: "array", line: 6, scope: !10, file: !5, type: !27)
-!27 = !MDCompositeType(tag: DW_TAG_array_type, size: 128, align: 32, baseType: !9, elements: !28)
+!19 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "p", line: 1, arg: 1, scope: !4, file: !5, type: !8)
+!20 = !DILocation(line: 1, scope: !4)
+!21 = !DILocation(line: 2, scope: !4)
+!22 = !DILocation(line: 3, scope: !4)
+!23 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "argc", line: 5, arg: 1, scope: !10, file: !5, type: !9)
+!24 = !DILocation(line: 5, scope: !10)
+!25 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "argv", line: 5, arg: 2, scope: !10, file: !5, type: !13)
+!26 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "array", line: 6, scope: !10, file: !5, type: !27)
+!27 = !DICompositeType(tag: DW_TAG_array_type, size: 128, align: 32, baseType: !9, elements: !28)
 !28 = !{!29}
-!29 = !MDSubrange(count: 4)
-!30 = !MDLocation(line: 6, scope: !10)
-!31 = !MDLocation(line: 7, scope: !10)
-!32 = !MDLocation(line: 8, scope: !10)
+!29 = !DISubrange(count: 4)
+!30 = !DILocation(line: 6, scope: !10)
+!31 = !DILocation(line: 7, scope: !10)
+!32 = !DILocation(line: 8, scope: !10)
diff --git a/test/DebugInfo/X86/asm-macro-line-number.s b/test/DebugInfo/X86/asm-macro-line-number.s
index 0f51dbb6440..8b0843d0613 100644
--- a/test/DebugInfo/X86/asm-macro-line-number.s
+++ b/test/DebugInfo/X86/asm-macro-line-number.s
@@ -3,12 +3,18 @@
 # 1 "reduced.S"
 # 1 "<built-in>" 1
 # 1 "reduced.S" 2
+# 200 "macros.h"
 
  .macro return arg
   movl %eax, \arg
   retl
  .endm
 
+ .macro return2 arg
+  return \arg
+ .endm
+
+# 7 "reduced.S"
 function:
  return 0
 
@@ -18,3 +24,11 @@ function:
 # CHECK: .loc 2 8 0
 # CHECK: retl
 
+# 42 "reduced.S"
+function2:
+ return2 0
+
+# CHECK: .loc 2 43 0
+# CHECK: movl %eax, 0
+# CHECK: .loc 2 43 0
+# CHECK: retl
diff --git a/test/DebugInfo/X86/bitfields.ll b/test/DebugInfo/X86/bitfields.ll
new file mode 100644
index 00000000000..e895fd67e03
--- /dev/null
+++ b/test/DebugInfo/X86/bitfields.ll
@@ -0,0 +1,73 @@
+; RUN: llc -mtriple x86_64-apple-macosx -O0 -filetype=obj -o %t_le.o %s
+; RUN: llvm-dwarfdump -debug-dump=info %t_le.o | FileCheck %s
+
+; Produced at -O0 from:
+; struct bitfield {
+;   int a : 2;
+;   int b : 32;
+;   int c : 1;
+;   int d : 28;
+; };
+; struct bitfield b;
+
+; Note that DWARF 2 counts bit offsets backwards from the high end of
+; the storage unit to the high end of the bit field.
+
+; CHECK: DW_TAG_member
+; CHECK-NEXT: DW_AT_name{{.*}}"a"
+; CHECK-NOT: DW_TAG_member
+; CHECK:      DW_AT_byte_size  {{.*}} (0x04)
+; CHECK-NEXT: DW_AT_bit_size   {{.*}} (0x02)
+; CHECK-NEXT: DW_AT_bit_offset {{.*}} (0x1e)
+; CHECK-NEXT: DW_AT_data_member_location {{.*}} 00
+
+; CHECK: DW_TAG_member
+; CHECK-NEXT: DW_AT_name{{.*}}"b"
+; CHECK-NOT: DW_TAG_member
+; CHECK:      DW_AT_data_member_location {{.*}} 04
+
+; CHECK: DW_TAG_member
+; CHECK-NEXT: DW_AT_name{{.*}}"c"
+; CHECK-NOT: DW_TAG_member
+; CHECK:      DW_AT_byte_size  {{.*}} (0x04)
+; CHECK-NEXT: DW_AT_bit_size   {{.*}} (0x01)
+; CHECK-NEXT: DW_AT_bit_offset {{.*}} (0x1f)
+; CHECK-NEXT: DW_AT_data_member_location {{.*}} 08
+
+; CHECK: DW_TAG_member
+; CHECK-NEXT: DW_AT_name{{.*}}"d"
+; CHECK-NOT: DW_TAG_member
+; CHECK:      DW_AT_byte_size  {{.*}} (0x04)
+; CHECK-NEXT: DW_AT_bit_size   {{.*}} (0x1c)
+; CHECK-NEXT: DW_AT_bit_offset {{.*}} (0x03)
+; CHECK-NEXT: DW_AT_data_member_location {{.*}} 08
+
+; ModuleID = 'bitfields.c'
+target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-apple-macosx"
+
+%struct.bitfield = type <{ i8, [3 x i8], i64 }>
+
+@b = common global %struct.bitfield zeroinitializer, align 4
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!13, !14, !15}
+!llvm.ident = !{!16}
+
+!0 = !DICompileUnit(language: DW_LANG_C99, file: !1, producer: "clang version 3.7.0 (trunk 240548) (llvm/trunk 240554)", isOptimized: false, runtimeVersion: 0, emissionKind: 1, enums: !2, retainedTypes: !2, subprograms: !2, globals: !3, imports: !2)
+!1 = !DIFile(filename: "bitfields.c", directory: "/")
+!2 = !{}
+!3 = !{!4}
+!4 = !DIGlobalVariable(name: "b", scope: !0, file: !5, line: 8, type: !6, isLocal: false, isDefinition: true, variable: %struct.bitfield* @b)
+!5 = !DIFile(filename: "bitfields.c", directory: "/")
+!6 = !DICompositeType(tag: DW_TAG_structure_type, name: "bitfield", file: !5, line: 1, size: 96, align: 32, elements: !7)
+!7 = !{!8, !10, !11, !12}
+!8 = !DIDerivedType(tag: DW_TAG_member, name: "a", scope: !6, file: !5, line: 2, baseType: !9, size: 2, align: 32)
+!9 = !DIBasicType(name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
+!10 = !DIDerivedType(tag: DW_TAG_member, name: "b", scope: !6, file: !5, line: 3, baseType: !9, size: 32, align: 32, offset: 32)
+!11 = !DIDerivedType(tag: DW_TAG_member, name: "c", scope: !6, file: !5, line: 4, baseType: !9, size: 1, align: 32, offset: 64)
+!12 = !DIDerivedType(tag: DW_TAG_member, name: "d", scope: !6, file: !5, line: 5, baseType: !9, size: 28, align: 32, offset: 65)
+!13 = !{i32 2, !"Dwarf Version", i32 2}
+!14 = !{i32 2, !"Debug Info Version", i32 3}
+!15 = !{i32 1, !"PIC Level", i32 2}
+!16 = !{!"clang version 3.7.0 (trunk 240548) (llvm/trunk 240554)"}
diff --git a/test/DebugInfo/X86/block-capture.ll b/test/DebugInfo/X86/block-capture.ll
index 314879bbb0c..fe026d656bb 100644
--- a/test/DebugInfo/X86/block-capture.ll
+++ b/test/DebugInfo/X86/block-capture.ll
@@ -62,18 +62,18 @@ attributes #3 = { nounwind }
 !llvm.module.flags = !{!16, !17, !18, !19, !20, !21, !22}
 !llvm.ident = !{!23}
 
-!0 = !MDCompileUnit(language: DW_LANG_ObjC, producer: "clang version 3.6.0 (trunk 223471)", isOptimized: false, runtimeVersion: 2, emissionKind: 1, file: !1, enums: !2, retainedTypes: !2, subprograms: !3, globals: !2, imports: !2)
-!1 = !MDFile(filename: "foo.m", directory: "")
+!0 = !DICompileUnit(language: DW_LANG_ObjC, producer: "clang version 3.6.0 (trunk 223471)", isOptimized: false, runtimeVersion: 2, emissionKind: 1, file: !1, enums: !2, retainedTypes: !2, subprograms: !3, globals: !2, imports: !2)
+!1 = !DIFile(filename: "foo.m", directory: "")
 !2 = !{}
 !3 = !{!8}
-!5 = !MDFile(filename: "foo.m", directory: "")
-!6 = !MDSubroutineType(types: !7)
+!5 = !DIFile(filename: "foo.m", directory: "")
+!6 = !DISubroutineType(types: !7)
 !7 = !{null}
-!8 = !MDSubprogram(name: "__foo_block_invoke", line: 2, isLocal: true, isDefinition: true, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 2, file: !1, scope: !5, type: !9, function: void (i8*)* @__foo_block_invoke, variables: !2)
-!9 = !MDSubroutineType(types: !10)
+!8 = !DISubprogram(name: "__foo_block_invoke", line: 2, isLocal: true, isDefinition: true, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 2, file: !1, scope: !5, type: !9, function: void (i8*)* @__foo_block_invoke, variables: !2)
+!9 = !DISubroutineType(types: !10)
 !10 = !{null, !11}
-!11 = !MDDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, baseType: null)
-!13 = !MDSubroutineType(types: !14)
+!11 = !DIDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, baseType: null)
+!13 = !DISubroutineType(types: !14)
 !14 = !{null, !11, !11}
 !16 = !{i32 1, !"Objective-C Version", i32 2}
 !17 = !{i32 1, !"Objective-C Image Info Version", i32 0}
@@ -83,49 +83,49 @@ attributes #3 = { nounwind }
 !21 = !{i32 2, !"Debug Info Version", i32 3}
 !22 = !{i32 1, !"PIC Level", i32 2}
 !23 = !{!"clang version 3.6.0 (trunk 223471)"}
-!25 = !MDDerivedType(tag: DW_TAG_pointer_type, size: 64, baseType: !26)
-!26 = !MDCompositeType(tag: DW_TAG_structure_type, name: "__block_literal_generic", line: 2, size: 256, flags: DIFlagAppleBlock, file: !1, scope: !5, elements: !27)
+!25 = !DIDerivedType(tag: DW_TAG_pointer_type, size: 64, baseType: !26)
+!26 = !DICompositeType(tag: DW_TAG_structure_type, name: "__block_literal_generic", line: 2, size: 256, flags: DIFlagAppleBlock, file: !1, scope: !5, elements: !27)
 !27 = !{!28, !29, !31, !32, !36}
-!28 = !MDDerivedType(tag: DW_TAG_member, name: "__isa", size: 64, align: 64, file: !1, scope: !5, baseType: !11)
-!29 = !MDDerivedType(tag: DW_TAG_member, name: "__flags", size: 32, align: 32, offset: 64, file: !1, scope: !5, baseType: !30)
-!30 = !MDBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
-!31 = !MDDerivedType(tag: DW_TAG_member, name: "__reserved", size: 32, align: 32, offset: 96, file: !1, scope: !5, baseType: !30)
-!32 = !MDDerivedType(tag: DW_TAG_member, name: "__FuncPtr", size: 64, align: 64, offset: 128, file: !1, scope: !5, baseType: !33)
-!33 = !MDDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, baseType: !34)
-!34 = !MDSubroutineType(types: !35)
+!28 = !DIDerivedType(tag: DW_TAG_member, name: "__isa", size: 64, align: 64, file: !1, scope: !5, baseType: !11)
+!29 = !DIDerivedType(tag: DW_TAG_member, name: "__flags", size: 32, align: 32, offset: 64, file: !1, scope: !5, baseType: !30)
+!30 = !DIBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
+!31 = !DIDerivedType(tag: DW_TAG_member, name: "__reserved", size: 32, align: 32, offset: 96, file: !1, scope: !5, baseType: !30)
+!32 = !DIDerivedType(tag: DW_TAG_member, name: "__FuncPtr", size: 64, align: 64, offset: 128, file: !1, scope: !5, baseType: !33)
+!33 = !DIDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, baseType: !34)
+!34 = !DISubroutineType(types: !35)
 !35 = !{null, null}
-!36 = !MDDerivedType(tag: DW_TAG_member, name: "__descriptor", line: 2, size: 64, align: 64, offset: 192, file: !1, scope: !5, baseType: !37)
-!37 = !MDDerivedType(tag: DW_TAG_pointer_type, size: 64, baseType: !38)
-!38 = !MDCompositeType(tag: DW_TAG_structure_type, name: "__block_descriptor", line: 2, size: 128, flags: DIFlagAppleBlock, file: !1, scope: !5, elements: !39)
+!36 = !DIDerivedType(tag: DW_TAG_member, name: "__descriptor", line: 2, size: 64, align: 64, offset: 192, file: !1, scope: !5, baseType: !37)
+!37 = !DIDerivedType(tag: DW_TAG_pointer_type, size: 64, baseType: !38)
+!38 = !DICompositeType(tag: DW_TAG_structure_type, name: "__block_descriptor", line: 2, size: 128, flags: DIFlagAppleBlock, file: !1, scope: !5, elements: !39)
 !39 = !{!40, !42}
-!40 = !MDDerivedType(tag: DW_TAG_member, name: "reserved", size: 64, align: 64, file: !1, scope: !5, baseType: !41)
-!41 = !MDBasicType(tag: DW_TAG_base_type, name: "long unsigned int", size: 64, align: 64, encoding: DW_ATE_unsigned)
-!42 = !MDDerivedType(tag: DW_TAG_member, name: "Size", size: 64, align: 64, offset: 64, file: !1, scope: !5, baseType: !41)
-!43 = !MDExpression()
-!47 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: ".block_descriptor", line: 2, arg: 1, flags: DIFlagArtificial, scope: !8, file: !5, type: !48)
-!48 = !MDDerivedType(tag: DW_TAG_pointer_type, size: 64, baseType: !49)
-!49 = !MDCompositeType(tag: DW_TAG_structure_type, name: "__block_literal_1", line: 2, size: 320, align: 64, file: !1, scope: !5, elements: !50)
+!40 = !DIDerivedType(tag: DW_TAG_member, name: "reserved", size: 64, align: 64, file: !1, scope: !5, baseType: !41)
+!41 = !DIBasicType(tag: DW_TAG_base_type, name: "long unsigned int", size: 64, align: 64, encoding: DW_ATE_unsigned)
+!42 = !DIDerivedType(tag: DW_TAG_member, name: "Size", size: 64, align: 64, offset: 64, file: !1, scope: !5, baseType: !41)
+!43 = !DIExpression()
+!47 = !DILocalVariable(tag: DW_TAG_arg_variable, name: ".block_descriptor", line: 2, arg: 1, flags: DIFlagArtificial, scope: !8, file: !5, type: !48)
+!48 = !DIDerivedType(tag: DW_TAG_pointer_type, size: 64, baseType: !49)
+!49 = !DICompositeType(tag: DW_TAG_structure_type, name: "__block_literal_1", line: 2, size: 320, align: 64, file: !1, scope: !5, elements: !50)
 !50 = !{!51, !52, !53, !54, !56, !65}
-!51 = !MDDerivedType(tag: DW_TAG_member, name: "__isa", line: 2, size: 64, align: 64, flags: DIFlagPublic, file: !1, scope: !5, baseType: !11)
-!52 = !MDDerivedType(tag: DW_TAG_member, name: "__flags", line: 2, size: 32, align: 32, offset: 64, flags: DIFlagPublic, file: !1, scope: !5, baseType: !30)
-!53 = !MDDerivedType(tag: DW_TAG_member, name: "__reserved", line: 2, size: 32, align: 32, offset: 96, flags: DIFlagPublic, file: !1, scope: !5, baseType: !30)
-!54 = !MDDerivedType(tag: DW_TAG_member, name: "__FuncPtr", line: 2, size: 64, align: 64, offset: 128, flags: DIFlagPublic, file: !1, scope: !5, baseType: !55)
-!55 = !MDDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, baseType: !6)
-!56 = !MDDerivedType(tag: DW_TAG_member, name: "__descriptor", line: 2, size: 64, align: 64, offset: 192, flags: DIFlagPublic, file: !1, scope: !5, baseType: !57)
-!57 = !MDDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, baseType: !58)
-!58 = !MDCompositeType(tag: DW_TAG_structure_type, name: "__block_descriptor_withcopydispose", line: 2, size: 256, align: 64, file: !1, elements: !59)
+!51 = !DIDerivedType(tag: DW_TAG_member, name: "__isa", line: 2, size: 64, align: 64, flags: DIFlagPublic, file: !1, scope: !5, baseType: !11)
+!52 = !DIDerivedType(tag: DW_TAG_member, name: "__flags", line: 2, size: 32, align: 32, offset: 64, flags: DIFlagPublic, file: !1, scope: !5, baseType: !30)
+!53 = !DIDerivedType(tag: DW_TAG_member, name: "__reserved", line: 2, size: 32, align: 32, offset: 96, flags: DIFlagPublic, file: !1, scope: !5, baseType: !30)
+!54 = !DIDerivedType(tag: DW_TAG_member, name: "__FuncPtr", line: 2, size: 64, align: 64, offset: 128, flags: DIFlagPublic, file: !1, scope: !5, baseType: !55)
+!55 = !DIDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, baseType: !6)
+!56 = !DIDerivedType(tag: DW_TAG_member, name: "__descriptor", line: 2, size: 64, align: 64, offset: 192, flags: DIFlagPublic, file: !1, scope: !5, baseType: !57)
+!57 = !DIDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, baseType: !58)
+!58 = !DICompositeType(tag: DW_TAG_structure_type, name: "__block_descriptor_withcopydispose", line: 2, size: 256, align: 64, file: !1, elements: !59)
 !59 = !{!60, !61, !62, !64}
-!60 = !MDDerivedType(tag: DW_TAG_member, name: "reserved", line: 2, size: 64, align: 64, file: !1, scope: !58, baseType: !41)
-!61 = !MDDerivedType(tag: DW_TAG_member, name: "Size", line: 2, size: 64, align: 64, offset: 64, file: !1, scope: !58, baseType: !41)
-!62 = !MDDerivedType(tag: DW_TAG_member, name: "CopyFuncPtr", line: 2, size: 64, align: 64, offset: 128, file: !1, scope: !58, baseType: !63)
-!63 = !MDDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, baseType: !11)
-!64 = !MDDerivedType(tag: DW_TAG_member, name: "DestroyFuncPtr", line: 2, size: 64, align: 64, offset: 192, file: !1, scope: !58, baseType: !63)
-!65 = !MDDerivedType(tag: DW_TAG_member, name: "block", line: 2, size: 64, align: 64, offset: 256, flags: DIFlagPublic, file: !1, scope: !5, baseType: !25)
-!66 = !MDLocation(line: 2, column: 20, scope: !8)
-!67 = !MDLocation(line: 2, column: 21, scope: !8)
-!68 = !MDLocalVariable(tag: DW_TAG_auto_variable, name: "block", line: 2, scope: !8, file: !5, type: !25)
-!69 = !MDExpression(DW_OP_deref, DW_OP_plus, 32)
-!70 = !MDLocation(line: 2, column: 9, scope: !8)
-!71 = !MDLocation(line: 2, column: 23, scope: !72)
-!72 = distinct !MDLexicalBlock(line: 2, column: 21, file: !1, scope: !8)
-!73 = !MDLocation(line: 2, column: 32, scope: !8)
+!60 = !DIDerivedType(tag: DW_TAG_member, name: "reserved", line: 2, size: 64, align: 64, file: !1, scope: !58, baseType: !41)
+!61 = !DIDerivedType(tag: DW_TAG_member, name: "Size", line: 2, size: 64, align: 64, offset: 64, file: !1, scope: !58, baseType: !41)
+!62 = !DIDerivedType(tag: DW_TAG_member, name: "CopyFuncPtr", line: 2, size: 64, align: 64, offset: 128, file: !1, scope: !58, baseType: !63)
+!63 = !DIDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, baseType: !11)
+!64 = !DIDerivedType(tag: DW_TAG_member, name: "DestroyFuncPtr", line: 2, size: 64, align: 64, offset: 192, file: !1, scope: !58, baseType: !63)
+!65 = !DIDerivedType(tag: DW_TAG_member, name: "block", line: 2, size: 64, align: 64, offset: 256, flags: DIFlagPublic, file: !1, scope: !5, baseType: !25)
+!66 = !DILocation(line: 2, column: 20, scope: !8)
+!67 = !DILocation(line: 2, column: 21, scope: !8)
+!68 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "block", line: 2, scope: !8, file: !5, type: !25)
+!69 = !DIExpression(DW_OP_deref, DW_OP_plus, 32)
+!70 = !DILocation(line: 2, column: 9, scope: !8)
+!71 = !DILocation(line: 2, column: 23, scope: !72)
+!72 = distinct !DILexicalBlock(line: 2, column: 21, file: !1, scope: !8)
+!73 = !DILocation(line: 2, column: 32, scope: !8)
diff --git a/test/DebugInfo/X86/byvalstruct.ll b/test/DebugInfo/X86/byvalstruct.ll
index 573a64245c8..dedbfb0200b 100644
--- a/test/DebugInfo/X86/byvalstruct.ll
+++ b/test/DebugInfo/X86/byvalstruct.ll
@@ -66,14 +66,14 @@ entry:
   %otherBitmap.addr = alloca %0*, align 8
   %length.addr = alloca i64, align 8
   store %0* %self, %0** %self.addr, align 8
-  call void @llvm.dbg.declare(metadata %0** %self.addr, metadata !28, metadata !MDExpression()), !dbg !29
+  call void @llvm.dbg.declare(metadata %0** %self.addr, metadata !28, metadata !DIExpression()), !dbg !29
   store i8* %_cmd, i8** %_cmd.addr, align 8
-  call void @llvm.dbg.declare(metadata i8** %_cmd.addr, metadata !30, metadata !MDExpression()), !dbg !29
+  call void @llvm.dbg.declare(metadata i8** %_cmd.addr, metadata !30, metadata !DIExpression()), !dbg !29
   store %0* %otherBitmap, %0** %otherBitmap.addr, align 8
-  call void @llvm.dbg.declare(metadata %0** %otherBitmap.addr, metadata !32, metadata !MDExpression()), !dbg !29
-  call void @llvm.dbg.declare(metadata %struct.ImageInfo* %info, metadata !33, metadata !MDExpression()), !dbg !34
+  call void @llvm.dbg.declare(metadata %0** %otherBitmap.addr, metadata !32, metadata !DIExpression()), !dbg !29
+  call void @llvm.dbg.declare(metadata %struct.ImageInfo* %info, metadata !33, metadata !DIExpression()), !dbg !34
   store i64 %length, i64* %length.addr, align 8
-  call void @llvm.dbg.declare(metadata i64* %length.addr, metadata !35, metadata !MDExpression()), !dbg !36
+  call void @llvm.dbg.declare(metadata i64* %length.addr, metadata !35, metadata !DIExpression()), !dbg !36
   %0 = load i8*, i8** %retval, !dbg !37
   ret i8* %0, !dbg !37
 }
@@ -87,42 +87,42 @@ attributes #1 = { nounwind readnone }
 !llvm.dbg.cu = !{!0}
 !llvm.module.flags = !{!24, !25, !26, !27, !38}
 
-!0 = !MDCompileUnit(language: DW_LANG_ObjC_plus_plus, producer: "clang version 3.4 ", isOptimized: false, runtimeVersion: 2, emissionKind: 0, file: !1, enums: !2, retainedTypes: !3, subprograms: !6, globals: !2, imports: !2)
-!1 = !MDFile(filename: "t.mm", directory: "")
+!0 = !DICompileUnit(language: DW_LANG_ObjC_plus_plus, producer: "clang version 3.4 ", isOptimized: false, runtimeVersion: 2, emissionKind: 0, file: !1, enums: !2, retainedTypes: !3, subprograms: !6, globals: !2, imports: !2)
+!1 = !DIFile(filename: "t.mm", directory: "")
 !2 = !{}
 !3 = !{!4}
-!4 = !MDCompositeType(tag: DW_TAG_structure_type, name: "Bitmap", line: 8, size: 8, align: 8, flags: DIFlagObjcClassComplete, runtimeLang: DW_LANG_ObjC_plus_plus, file: !1, scope: !5, elements: !2)
-!5 = !MDFile(filename: "t.mm", directory: "")
+!4 = !DICompositeType(tag: DW_TAG_structure_type, name: "Bitmap", line: 8, size: 8, align: 8, flags: DIFlagObjcClassComplete, runtimeLang: DW_LANG_ObjC_plus_plus, file: !1, scope: !5, elements: !2)
+!5 = !DIFile(filename: "t.mm", directory: "")
 !6 = !{!7}
-!7 = !MDSubprogram(name: "-[Bitmap initWithCopy:andInfo:andLength:]", line: 9, isLocal: true, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 9, file: !1, scope: !5, type: !8, function: i8* (%0*, i8*, %0*, %struct.ImageInfo*, i64)* @"\01-[Bitmap initWithCopy:andInfo:andLength:]", variables: !2)
-!8 = !MDSubroutineType(types: !9)
+!7 = !DISubprogram(name: "-[Bitmap initWithCopy:andInfo:andLength:]", line: 9, isLocal: true, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 9, file: !1, scope: !5, type: !8, function: i8* (%0*, i8*, %0*, %struct.ImageInfo*, i64)* @"\01-[Bitmap initWithCopy:andInfo:andLength:]", variables: !2)
+!8 = !DISubroutineType(types: !9)
 !9 = !{!4, !10, !11, !14, !15, !19}
-!10 = !MDDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, flags: DIFlagArtificial | DIFlagObjectPointer, baseType: !4)
-!11 = !MDDerivedType(tag: DW_TAG_typedef, name: "SEL", line: 9, flags: DIFlagArtificial, file: !1, baseType: !12)
-!12 = !MDDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, baseType: !13)
-!13 = !MDCompositeType(tag: DW_TAG_structure_type, name: "objc_selector", flags: DIFlagFwdDecl, file: !1)
-!14 = !MDDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, baseType: !4)
-!15 = !MDDerivedType(tag: DW_TAG_typedef, name: "ImageInfo", line: 7, file: !1, baseType: !16)
-!16 = !MDCompositeType(tag: DW_TAG_structure_type, line: 2, size: 192, align: 64, file: !1, elements: !17)
+!10 = !DIDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, flags: DIFlagArtificial | DIFlagObjectPointer, baseType: !4)
+!11 = !DIDerivedType(tag: DW_TAG_typedef, name: "SEL", line: 9, flags: DIFlagArtificial, file: !1, baseType: !12)
+!12 = !DIDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, baseType: !13)
+!13 = !DICompositeType(tag: DW_TAG_structure_type, name: "objc_selector", flags: DIFlagFwdDecl, file: !1)
+!14 = !DIDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, baseType: !4)
+!15 = !DIDerivedType(tag: DW_TAG_typedef, name: "ImageInfo", line: 7, file: !1, baseType: !16)
+!16 = !DICompositeType(tag: DW_TAG_structure_type, line: 2, size: 192, align: 64, file: !1, elements: !17)
 !17 = !{!18, !21, !22}
-!18 = !MDDerivedType(tag: DW_TAG_member, name: "width", line: 4, size: 64, align: 64, file: !1, scope: !16, baseType: !19)
-!19 = !MDDerivedType(tag: DW_TAG_typedef, name: "NSUInteger", line: 1, file: !1, baseType: !20)
-!20 = !MDBasicType(tag: DW_TAG_base_type, name: "long unsigned int", size: 64, align: 64, encoding: DW_ATE_unsigned)
-!21 = !MDDerivedType(tag: DW_TAG_member, name: "height", line: 5, size: 64, align: 64, offset: 64, file: !1, scope: !16, baseType: !19)
-!22 = !MDDerivedType(tag: DW_TAG_member, name: "pixelAspect", line: 6, size: 64, align: 64, offset: 128, file: !1, scope: !16, baseType: !23)
-!23 = !MDBasicType(tag: DW_TAG_base_type, name: "double", size: 64, align: 64, encoding: DW_ATE_float)
+!18 = !DIDerivedType(tag: DW_TAG_member, name: "width", line: 4, size: 64, align: 64, file: !1, scope: !16, baseType: !19)
+!19 = !DIDerivedType(tag: DW_TAG_typedef, name: "NSUInteger", line: 1, file: !1, baseType: !20)
+!20 = !DIBasicType(tag: DW_TAG_base_type, name: "long unsigned int", size: 64, align: 64, encoding: DW_ATE_unsigned)
+!21 = !DIDerivedType(tag: DW_TAG_member, name: "height", line: 5, size: 64, align: 64, offset: 64, file: !1, scope: !16, baseType: !19)
+!22 = !DIDerivedType(tag: DW_TAG_member, name: "pixelAspect", line: 6, size: 64, align: 64, offset: 128, file: !1, scope: !16, baseType: !23)
+!23 = !DIBasicType(tag: DW_TAG_base_type, name: "double", size: 64, align: 64, encoding: DW_ATE_float)
 !24 = !{i32 1, !"Objective-C Version", i32 2}
 !25 = !{i32 1, !"Objective-C Image Info Version", i32 0}
 !26 = !{i32 1, !"Objective-C Image Info Section", !"__DATA, __objc_imageinfo, regular, no_dead_strip"}
 !27 = !{i32 4, !"Objective-C Garbage Collection", i32 0}
-!28 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "self", line: 9, arg: 1, flags: DIFlagArtificial | DIFlagObjectPointer, scope: !7, file: !5, type: !14)
-!29 = !MDLocation(line: 9, scope: !7)
-!30 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "_cmd", line: 9, arg: 2, flags: DIFlagArtificial, scope: !7, file: !5, type: !31)
-!31 = !MDDerivedType(tag: DW_TAG_typedef, name: "SEL", line: 9, file: !1, baseType: !12)
-!32 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "otherBitmap", line: 9, arg: 3, scope: !7, file: !5, type: !14)
-!33 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "info", line: 10, arg: 4, scope: !7, file: !5, type: !15)
-!34 = !MDLocation(line: 10, scope: !7)
-!35 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "length", line: 11, arg: 5, scope: !7, file: !5, type: !19)
-!36 = !MDLocation(line: 11, scope: !7)
-!37 = !MDLocation(line: 13, scope: !7)
+!28 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "self", line: 9, arg: 1, flags: DIFlagArtificial | DIFlagObjectPointer, scope: !7, file: !5, type: !14)
+!29 = !DILocation(line: 9, scope: !7)
+!30 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "_cmd", line: 9, arg: 2, flags: DIFlagArtificial, scope: !7, file: !5, type: !31)
+!31 = !DIDerivedType(tag: DW_TAG_typedef, name: "SEL", line: 9, file: !1, baseType: !12)
+!32 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "otherBitmap", line: 9, arg: 3, scope: !7, file: !5, type: !14)
+!33 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "info", line: 10, arg: 4, scope: !7, file: !5, type: !15)
+!34 = !DILocation(line: 10, scope: !7)
+!35 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "length", line: 11, arg: 5, scope: !7, file: !5, type: !19)
+!36 = !DILocation(line: 11, scope: !7)
+!37 = !DILocation(line: 13, scope: !7)
 !38 = !{i32 1, !"Debug Info Version", i32 3}
diff --git a/test/DebugInfo/X86/c-type-units.ll b/test/DebugInfo/X86/c-type-units.ll
index 79ba08bfd49..6756bd45f88 100644
--- a/test/DebugInfo/X86/c-type-units.ll
+++ b/test/DebugInfo/X86/c-type-units.ll
@@ -17,13 +17,13 @@
 !llvm.module.flags = !{!7, !8}
 !llvm.ident = !{!9}
 
-!0 = !MDCompileUnit(language: DW_LANG_C99, producer: "clang version 3.5 ", isOptimized: false, emissionKind: 0, file: !1, enums: !2, retainedTypes: !2, subprograms: !2, globals: !3, imports: !2)
-!1 = !MDFile(filename: "simple.c", directory: "/tmp/dbginfo")
+!0 = !DICompileUnit(language: DW_LANG_C99, producer: "clang version 3.5 ", isOptimized: false, emissionKind: 0, file: !1, enums: !2, retainedTypes: !2, subprograms: !2, globals: !3, imports: !2)
+!1 = !DIFile(filename: "simple.c", directory: "/tmp/dbginfo")
 !2 = !{}
 !3 = !{!4}
-!4 = !MDGlobalVariable(name: "f", line: 2, isLocal: false, isDefinition: true, scope: null, file: !5, type: !6, variable: %struct.foo* @f)
-!5 = !MDFile(filename: "simple.c", directory: "/tmp/dbginfo")
-!6 = !MDCompositeType(tag: DW_TAG_structure_type, name: "foo", line: 1, align: 8, file: !1, elements: !2)
+!4 = !DIGlobalVariable(name: "f", line: 2, isLocal: false, isDefinition: true, scope: null, file: !5, type: !6, variable: %struct.foo* @f)
+!5 = !DIFile(filename: "simple.c", directory: "/tmp/dbginfo")
+!6 = !DICompositeType(tag: DW_TAG_structure_type, name: "foo", line: 1, align: 8, file: !1, elements: !2)
 !7 = !{i32 2, !"Dwarf Version", i32 4}
 !8 = !{i32 1, !"Debug Info Version", i32 3}
 !9 = !{!"clang version 3.5 "}
diff --git a/test/DebugInfo/X86/coff_debug_info_type.ll b/test/DebugInfo/X86/coff_debug_info_type.ll
index 101281ca019..fd86c88baa1 100644
--- a/test/DebugInfo/X86/coff_debug_info_type.ll
+++ b/test/DebugInfo/X86/coff_debug_info_type.ll
@@ -29,15 +29,15 @@ attributes #0 = { nounwind "less-precise-fpmad"="false" "no-frame-pointer-elim"=
 !llvm.dbg.cu = !{!0}
 !llvm.module.flags = !{!9, !11}
 
-!0 = !MDCompileUnit(language: DW_LANG_C99, producer: "clang version 3.4 ", isOptimized: false, emissionKind: 0, file: !1, enums: !2, retainedTypes: !2, subprograms: !3, globals: !2, imports: !2)
-!1 = !MDFile(filename: "test.c", directory: "C:\5CProjects")
+!0 = !DICompileUnit(language: DW_LANG_C99, producer: "clang version 3.4 ", isOptimized: false, emissionKind: 0, file: !1, enums: !2, retainedTypes: !2, subprograms: !3, globals: !2, imports: !2)
+!1 = !DIFile(filename: "test.c", directory: "C:\5CProjects")
 !2 = !{}
 !3 = !{!4}
-!4 = !MDSubprogram(name: "main", line: 1, isLocal: false, isDefinition: true, virtualIndex: 6, isOptimized: false, scopeLine: 2, file: !1, scope: !5, type: !6, function: i32 ()* @main, variables: !2)
-!5 = !MDFile(filename: "test.c", directory: "C:CProjects")
-!6 = !MDSubroutineType(types: !7)
+!4 = !DISubprogram(name: "main", line: 1, isLocal: false, isDefinition: true, virtualIndex: 6, isOptimized: false, scopeLine: 2, file: !1, scope: !5, type: !6, function: i32 ()* @main, variables: !2)
+!5 = !DIFile(filename: "test.c", directory: "C:CProjects")
+!6 = !DISubroutineType(types: !7)
 !7 = !{!8}
-!8 = !MDBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
+!8 = !DIBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
 !9 = !{i32 2, !"Dwarf Version", i32 3}
-!10 = !MDLocation(line: 3, scope: !4)
+!10 = !DILocation(line: 3, scope: !4)
 !11 = !{i32 1, !"Debug Info Version", i32 3}
diff --git a/test/DebugInfo/X86/coff_relative_names.ll b/test/DebugInfo/X86/coff_relative_names.ll
index 06eab19ede4..6b7864ab397 100644
--- a/test/DebugInfo/X86/coff_relative_names.ll
+++ b/test/DebugInfo/X86/coff_relative_names.ll
@@ -23,15 +23,15 @@ attributes #0 = { nounwind "less-precise-fpmad"="false" "no-frame-pointer-elim"=
 !llvm.dbg.cu = !{!0}
 !llvm.module.flags = !{!9, !11}
 
-!0 = !MDCompileUnit(language: DW_LANG_C99, producer: "clang version 3.4 ", isOptimized: false, emissionKind: 0, file: !1, enums: !2, retainedTypes: !2, subprograms: !3, globals: !2, imports: !2)
-!1 = !MDFile(filename: "test.c", directory: "C:\5CProjects")
+!0 = !DICompileUnit(language: DW_LANG_C99, producer: "clang version 3.4 ", isOptimized: false, emissionKind: 0, file: !1, enums: !2, retainedTypes: !2, subprograms: !3, globals: !2, imports: !2)
+!1 = !DIFile(filename: "test.c", directory: "C:\5CProjects")
 !2 = !{}
 !3 = !{!4}
-!4 = !MDSubprogram(name: "main", line: 1, isLocal: false, isDefinition: true, virtualIndex: 6, isOptimized: false, scopeLine: 2, file: !1, scope: !5, type: !6, function: i32 ()* @main, variables: !2)
-!5 = !MDFile(filename: "test.c", directory: "C:CProjects")
-!6 = !MDSubroutineType(types: !7)
+!4 = !DISubprogram(name: "main", line: 1, isLocal: false, isDefinition: true, virtualIndex: 6, isOptimized: false, scopeLine: 2, file: !1, scope: !5, type: !6, function: i32 ()* @main, variables: !2)
+!5 = !DIFile(filename: "test.c", directory: "C:CProjects")
+!6 = !DISubroutineType(types: !7)
 !7 = !{!8}
-!8 = !MDBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
+!8 = !DIBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
 !9 = !{i32 2, !"Dwarf Version", i32 3}
-!10 = !MDLocation(line: 3, scope: !4)
+!10 = !DILocation(line: 3, scope: !4)
 !11 = !{i32 1, !"Debug Info Version", i32 3}
diff --git a/test/DebugInfo/X86/concrete_out_of_line.ll b/test/DebugInfo/X86/concrete_out_of_line.ll
index 9895324d9a4..c1e1f750e2c 100644
--- a/test/DebugInfo/X86/concrete_out_of_line.ll
+++ b/test/DebugInfo/X86/concrete_out_of_line.ll
@@ -76,56 +76,56 @@ declare void @_Z8moz_freePv(i8*)
 !llvm.dbg.cu = !{!0}
 !llvm.module.flags = !{!60}
 
-!0 = !MDCompileUnit(language: DW_LANG_C_plus_plus, producer: "clang version 3.1 ()", isOptimized: true, emissionKind: 0, file: !59, enums: !1, retainedTypes: !1, subprograms: !3, globals: !47, imports:  !1)
+!0 = !DICompileUnit(language: DW_LANG_C_plus_plus, producer: "clang version 3.1 ()", isOptimized: true, emissionKind: 0, file: !59, enums: !1, retainedTypes: !1, subprograms: !3, globals: !47, imports:  !1)
 !1 = !{}
 !3 = !{!5, !23, !27, !31}
-!5 = !MDSubprogram(name: "Release", linkageName: "_ZN17nsAutoRefCnt7ReleaseEv", line: 14, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: true, scopeLine: 14, file: !6, scope: null, type: !7, function: i32 ()* @_ZN17nsAutoRefCnt7ReleaseEv , declaration: !12, variables: !20)
-!6 = !MDFile(filename: "nsAutoRefCnt.ii", directory: "/Users/espindola/mozilla-central/obj-x86_64-apple-darwin11.2.0/netwerk/base/src")
-!7 = !MDSubroutineType(types: !8)
+!5 = !DISubprogram(name: "Release", linkageName: "_ZN17nsAutoRefCnt7ReleaseEv", line: 14, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: true, scopeLine: 14, file: !6, scope: null, type: !7, function: i32 ()* @_ZN17nsAutoRefCnt7ReleaseEv , declaration: !12, variables: !20)
+!6 = !DIFile(filename: "nsAutoRefCnt.ii", directory: "/Users/espindola/mozilla-central/obj-x86_64-apple-darwin11.2.0/netwerk/base/src")
+!7 = !DISubroutineType(types: !8)
 !8 = !{!9, !10}
-!9 = !MDBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
-!10 = !MDDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, flags: DIFlagArtificial, baseType: !11)
-!11 = !MDCompositeType(tag: DW_TAG_structure_type, name: "nsAutoRefCnt", line: 10, flags: DIFlagFwdDecl, file: !59)
-!12 = !MDSubprogram(name: "Release", linkageName: "_ZN17nsAutoRefCnt7ReleaseEv", line: 11, isLocal: false, isDefinition: false, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: true, scopeLine: 11, file: !6, scope: !13, type: !7, variables: !18)
-!13 = !MDCompositeType(tag: DW_TAG_class_type, name: "nsAutoRefCnt", line: 10, size: 8, align: 8, file: !59, elements: !14)
+!9 = !DIBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
+!10 = !DIDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, flags: DIFlagArtificial, baseType: !11)
+!11 = !DICompositeType(tag: DW_TAG_structure_type, name: "nsAutoRefCnt", line: 10, flags: DIFlagFwdDecl, file: !59)
+!12 = !DISubprogram(name: "Release", linkageName: "_ZN17nsAutoRefCnt7ReleaseEv", line: 11, isLocal: false, isDefinition: false, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: true, scopeLine: 11, file: !6, scope: !13, type: !7, variables: !18)
+!13 = !DICompositeType(tag: DW_TAG_class_type, name: "nsAutoRefCnt", line: 10, size: 8, align: 8, file: !59, elements: !14)
 !14 = !{!12, !15}
-!15 = !MDSubprogram(name: "~nsAutoRefCnt", line: 12, isLocal: false, isDefinition: false, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: true, scopeLine: 12, file: !6, scope: !13, type: !16, variables: !18)
-!16 = !MDSubroutineType(types: !17)
+!15 = !DISubprogram(name: "~nsAutoRefCnt", line: 12, isLocal: false, isDefinition: false, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: true, scopeLine: 12, file: !6, scope: !13, type: !16, variables: !18)
+!16 = !DISubroutineType(types: !17)
 !17 = !{null, !10}
 !18 = !{}
 !20 = !{!22}
-!22 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "this", line: 14, arg: 1, flags: DIFlagArtificial, scope: !5, file: !6, type: !10)
-!23 = !MDSubprogram(name: "~nsAutoRefCnt", linkageName: "_ZN17nsAutoRefCntD1Ev", line: 18, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: true, scopeLine: 18, file: !6, scope: null, type: !16, function: void ()* @_ZN17nsAutoRefCntD1Ev, declaration: !15, variables: !24)
+!22 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "this", line: 14, arg: 1, flags: DIFlagArtificial, scope: !5, file: !6, type: !10)
+!23 = !DISubprogram(name: "~nsAutoRefCnt", linkageName: "_ZN17nsAutoRefCntD1Ev", line: 18, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: true, scopeLine: 18, file: !6, scope: null, type: !16, function: void ()* @_ZN17nsAutoRefCntD1Ev, declaration: !15, variables: !24)
 !24 = !{!26}
-!26 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "this", line: 18, arg: 1, flags: DIFlagArtificial, scope: !23, file: !6, type: !10)
-!27 = !MDSubprogram(name: "~nsAutoRefCnt", linkageName: "_ZN17nsAutoRefCntD2Ev", line: 18, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: true, scopeLine: 18, file: !6, scope: null, type: !16, function: i32 ()* null, declaration: !15, variables: !28)
+!26 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "this", line: 18, arg: 1, flags: DIFlagArtificial, scope: !23, file: !6, type: !10)
+!27 = !DISubprogram(name: "~nsAutoRefCnt", linkageName: "_ZN17nsAutoRefCntD2Ev", line: 18, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: true, scopeLine: 18, file: !6, scope: null, type: !16, function: i32 ()* null, declaration: !15, variables: !28)
 !28 = !{!30}
-!30 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "this", line: 18, arg: 1, flags: DIFlagArtificial, scope: !27, file: !6, type: !10)
-!31 = !MDSubprogram(name: "operator=", linkageName: "_ZN12nsAutoRefCntaSEi", line: 4, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: true, scopeLine: 4, file: !6, scope: null, type: !32, declaration: !36, variables: !43)
-!32 = !MDSubroutineType(types: !33)
+!30 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "this", line: 18, arg: 1, flags: DIFlagArtificial, scope: !27, file: !6, type: !10)
+!31 = !DISubprogram(name: "operator=", linkageName: "_ZN12nsAutoRefCntaSEi", line: 4, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: true, scopeLine: 4, file: !6, scope: null, type: !32, declaration: !36, variables: !43)
+!32 = !DISubroutineType(types: !33)
 !33 = !{!9, !34, !9}
-!34 = !MDDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, flags: DIFlagArtificial, baseType: !35)
-!35 = !MDCompositeType(tag: DW_TAG_structure_type, name: "nsAutoRefCnt", line: 2, flags: DIFlagFwdDecl, file: !59)
-!36 = !MDSubprogram(name: "operator=", linkageName: "_ZN12nsAutoRefCntaSEi", line: 4, isLocal: false, isDefinition: false, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: true, scopeLine: 4, file: !6, scope: !37, type: !32, variables: !18)
-!37 = !MDCompositeType(tag: DW_TAG_class_type, name: "nsAutoRefCnt", line: 2, size: 32, align: 32, file: !59, elements: !38)
+!34 = !DIDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, flags: DIFlagArtificial, baseType: !35)
+!35 = !DICompositeType(tag: DW_TAG_structure_type, name: "nsAutoRefCnt", line: 2, flags: DIFlagFwdDecl, file: !59)
+!36 = !DISubprogram(name: "operator=", linkageName: "_ZN12nsAutoRefCntaSEi", line: 4, isLocal: false, isDefinition: false, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: true, scopeLine: 4, file: !6, scope: !37, type: !32, variables: !18)
+!37 = !DICompositeType(tag: DW_TAG_class_type, name: "nsAutoRefCnt", line: 2, size: 32, align: 32, file: !59, elements: !38)
 !38 = !{!39, !40, !36}
-!39 = !MDDerivedType(tag: DW_TAG_member, name: "mValue", line: 7, size: 32, align: 32, file: !59, scope: !37, baseType: !9)
-!40 = !MDSubprogram(name: "nsAutoRefCnt", line: 3, isLocal: false, isDefinition: false, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: true, scopeLine: 3, file: !6, scope: !37, type: !41, variables: !18)
-!41 = !MDSubroutineType(types: !42)
+!39 = !DIDerivedType(tag: DW_TAG_member, name: "mValue", line: 7, size: 32, align: 32, file: !59, scope: !37, baseType: !9)
+!40 = !DISubprogram(name: "nsAutoRefCnt", line: 3, isLocal: false, isDefinition: false, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: true, scopeLine: 3, file: !6, scope: !37, type: !41, variables: !18)
+!41 = !DISubroutineType(types: !42)
 !42 = !{null, !34}
 !43 = !{!45, !46}
-!45 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "this", line: 4, arg: 1, flags: DIFlagArtificial, scope: !31, file: !6, type: !34)
-!46 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "aValue", line: 4, arg: 2, scope: !31, file: !6, type: !9)
+!45 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "this", line: 4, arg: 1, flags: DIFlagArtificial, scope: !31, file: !6, type: !34)
+!46 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "aValue", line: 4, arg: 2, scope: !31, file: !6, type: !9)
 !47 = !{!49}
-!49 = !MDGlobalVariable(name: "mRefCnt", line: 9, isLocal: false, isDefinition: true, scope: null, file: !6, type: !37, variable: i32* null)
-!50 = !MDLocation(line: 5, column: 5, scope: !51, inlinedAt: !52)
-!51 = distinct !MDLexicalBlock(line: 4, column: 29, file: !6, scope: !31)
-!52 = !MDLocation(line: 15, scope: !53)
-!53 = distinct !MDLexicalBlock(line: 14, column: 34, file: !6, scope: !5)
-!54 = !MDLocation(line: 19, column: 3, scope: !55, inlinedAt: !56)
-!55 = distinct !MDLexicalBlock(line: 18, column: 41, file: !6, scope: !27)
-!56 = !MDLocation(line: 18, column: 41, scope: !23, inlinedAt: !52)
-!57 = !MDLocation(line: 19, column: 3, scope: !55, inlinedAt: !58)
-!58 = !MDLocation(line: 18, column: 41, scope: !23)
-!59 = !MDFile(filename: "nsAutoRefCnt.ii", directory: "/Users/espindola/mozilla-central/obj-x86_64-apple-darwin11.2.0/netwerk/base/src")
+!49 = !DIGlobalVariable(name: "mRefCnt", line: 9, isLocal: false, isDefinition: true, scope: null, file: !6, type: !37, variable: i32* null)
+!50 = !DILocation(line: 5, column: 5, scope: !51, inlinedAt: !52)
+!51 = distinct !DILexicalBlock(line: 4, column: 29, file: !6, scope: !31)
+!52 = !DILocation(line: 15, scope: !53)
+!53 = distinct !DILexicalBlock(line: 14, column: 34, file: !6, scope: !5)
+!54 = !DILocation(line: 19, column: 3, scope: !55, inlinedAt: !56)
+!55 = distinct !DILexicalBlock(line: 18, column: 41, file: !6, scope: !27)
+!56 = !DILocation(line: 18, column: 41, scope: !23, inlinedAt: !52)
+!57 = !DILocation(line: 19, column: 3, scope: !55, inlinedAt: !58)
+!58 = !DILocation(line: 18, column: 41, scope: !23)
+!59 = !DIFile(filename: "nsAutoRefCnt.ii", directory: "/Users/espindola/mozilla-central/obj-x86_64-apple-darwin11.2.0/netwerk/base/src")
 !60 = !{i32 1, !"Debug Info Version", i32 3}
diff --git a/test/DebugInfo/X86/constant-aggregate.ll b/test/DebugInfo/X86/constant-aggregate.ll
index d0e4c98a1ad..daec3b4e7d6 100644
--- a/test/DebugInfo/X86/constant-aggregate.ll
+++ b/test/DebugInfo/X86/constant-aggregate.ll
@@ -72,47 +72,47 @@ attributes #1 = { nounwind readnone }
 !llvm.module.flags = !{!33, !34, !35}
 !llvm.ident = !{!36}
 
-!0 = !MDCompileUnit(language: DW_LANG_C_plus_plus, producer: "clang version 3.6.0 (trunk 225364) (llvm/trunk 225366)", isOptimized: true, emissionKind: 1, file: !1, enums: !2, retainedTypes: !3, subprograms: !11, globals: !2, imports: !2)
-!1 = !MDFile(filename: "sroasplit-4.cpp", directory: "")
+!0 = !DICompileUnit(language: DW_LANG_C_plus_plus, producer: "clang version 3.6.0 (trunk 225364) (llvm/trunk 225366)", isOptimized: true, emissionKind: 1, file: !1, enums: !2, retainedTypes: !3, subprograms: !11, globals: !2, imports: !2)
+!1 = !DIFile(filename: "sroasplit-4.cpp", directory: "")
 !2 = !{}
 !3 = !{!4, !8}
-!4 = !MDCompositeType(tag: DW_TAG_structure_type, line: 1, size: 32, align: 32, file: !1, elements: !5, identifier: "_ZTS1S")
+!4 = !DICompositeType(tag: DW_TAG_structure_type, line: 1, size: 32, align: 32, file: !1, elements: !5, identifier: "_ZTS1S")
 !5 = !{!6}
-!6 = !MDDerivedType(tag: DW_TAG_member, name: "i", line: 1, size: 32, align: 32, file: !1, scope: !"_ZTS1S", baseType: !7)
-!7 = !MDBasicType(tag: DW_TAG_base_type, name: "unsigned int", size: 32, align: 32, encoding: DW_ATE_unsigned)
-!8 = !MDCompositeType(tag: DW_TAG_class_type, name: "C", line: 8, size: 32, align: 32, file: !1, elements: !9, identifier: "_ZTS1C")
+!6 = !DIDerivedType(tag: DW_TAG_member, name: "i", line: 1, size: 32, align: 32, file: !1, scope: !"_ZTS1S", baseType: !7)
+!7 = !DIBasicType(tag: DW_TAG_base_type, name: "unsigned int", size: 32, align: 32, encoding: DW_ATE_unsigned)
+!8 = !DICompositeType(tag: DW_TAG_class_type, name: "C", line: 8, size: 32, align: 32, file: !1, elements: !9, identifier: "_ZTS1C")
 !9 = !{!10}
-!10 = !MDDerivedType(tag: DW_TAG_member, name: "i", line: 8, size: 32, align: 32, flags: DIFlagPublic, file: !1, scope: !"_ZTS1C", baseType: !7)
+!10 = !DIDerivedType(tag: DW_TAG_member, name: "i", line: 8, size: 32, align: 32, flags: DIFlagPublic, file: !1, scope: !"_ZTS1C", baseType: !7)
 !11 = !{!12, !19, !24}
-!12 = !MDSubprogram(name: "foo", linkageName: "_Z3foo1S", line: 3, isLocal: false, isDefinition: true, flags: DIFlagPrototyped, isOptimized: true, scopeLine: 3, file: !1, scope: !13, type: !14, function: i32 (i32)* @_Z3foo1S, variables: !17)
-!13 = !MDFile(filename: "sroasplit-4.cpp", directory: "")
-!14 = !MDSubroutineType(types: !15)
+!12 = !DISubprogram(name: "foo", linkageName: "_Z3foo1S", line: 3, isLocal: false, isDefinition: true, flags: DIFlagPrototyped, isOptimized: true, scopeLine: 3, file: !1, scope: !13, type: !14, function: i32 (i32)* @_Z3foo1S, variables: !17)
+!13 = !DIFile(filename: "sroasplit-4.cpp", directory: "")
+!14 = !DISubroutineType(types: !15)
 !15 = !{!7, !16}
-!16 = !MDDerivedType(tag: DW_TAG_typedef, name: "S", line: 1, file: !1, baseType: !"_ZTS1S")
+!16 = !DIDerivedType(tag: DW_TAG_typedef, name: "S", line: 1, file: !1, baseType: !"_ZTS1S")
 !17 = !{!18}
-!18 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "s", line: 3, arg: 1, scope: !12, file: !13, type: !16)
-!19 = !MDSubprogram(name: "foo", linkageName: "_Z3foo1C", line: 10, isLocal: false, isDefinition: true, flags: DIFlagPrototyped, isOptimized: true, scopeLine: 10, file: !1, scope: !13, type: !20, function: i32 (i32)* @_Z3foo1C, variables: !22)
-!20 = !MDSubroutineType(types: !21)
+!18 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "s", line: 3, arg: 1, scope: !12, file: !13, type: !16)
+!19 = !DISubprogram(name: "foo", linkageName: "_Z3foo1C", line: 10, isLocal: false, isDefinition: true, flags: DIFlagPrototyped, isOptimized: true, scopeLine: 10, file: !1, scope: !13, type: !20, function: i32 (i32)* @_Z3foo1C, variables: !22)
+!20 = !DISubroutineType(types: !21)
 !21 = !{!7, !"_ZTS1C"}
 !22 = !{!23}
-!23 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "c", line: 10, arg: 1, scope: !19, file: !13, type: !"_ZTS1C")
-!24 = !MDSubprogram(name: "bar", linkageName: "_Z3barv", line: 15, isLocal: false, isDefinition: true, flags: DIFlagPrototyped, isOptimized: true, scopeLine: 15, file: !1, scope: !13, type: !25, function: i32 ()* @_Z3barv, variables: !27)
-!25 = !MDSubroutineType(types: !26)
+!23 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "c", line: 10, arg: 1, scope: !19, file: !13, type: !"_ZTS1C")
+!24 = !DISubprogram(name: "bar", linkageName: "_Z3barv", line: 15, isLocal: false, isDefinition: true, flags: DIFlagPrototyped, isOptimized: true, scopeLine: 15, file: !1, scope: !13, type: !25, function: i32 ()* @_Z3barv, variables: !27)
+!25 = !DISubroutineType(types: !26)
 !26 = !{!7}
 !27 = !{!28}
-!28 = !MDLocalVariable(tag: DW_TAG_auto_variable, name: "a", line: 16, scope: !24, file: !13, type: !29)
-!29 = !MDCompositeType(tag: DW_TAG_array_type, size: 32, align: 32, baseType: !30, elements: !31)
-!30 = !MDBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
+!28 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "a", line: 16, scope: !24, file: !13, type: !29)
+!29 = !DICompositeType(tag: DW_TAG_array_type, size: 32, align: 32, baseType: !30, elements: !31)
+!30 = !DIBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
 !31 = !{!32}
-!32 = !MDSubrange(count: 1)
+!32 = !DISubrange(count: 1)
 !33 = !{i32 2, !"Dwarf Version", i32 2}
 !34 = !{i32 2, !"Debug Info Version", i32 3}
 !35 = !{i32 1, !"PIC Level", i32 2}
 !36 = !{!"clang version 3.6.0 (trunk 225364) (llvm/trunk 225366)"}
-!37 = !MDExpression()
-!38 = !MDLocation(line: 3, column: 16, scope: !12)
-!39 = !MDLocation(line: 5, column: 3, scope: !12)
-!40 = !MDLocation(line: 10, column: 16, scope: !19)
-!41 = !MDLocation(line: 12, column: 3, scope: !19)
-!42 = !MDLocation(line: 16, column: 6, scope: !24)
-!43 = !MDLocation(line: 17, column: 3, scope: !24)
+!37 = !DIExpression()
+!38 = !DILocation(line: 3, column: 16, scope: !12)
+!39 = !DILocation(line: 5, column: 3, scope: !12)
+!40 = !DILocation(line: 10, column: 16, scope: !19)
+!41 = !DILocation(line: 12, column: 3, scope: !19)
+!42 = !DILocation(line: 16, column: 6, scope: !24)
+!43 = !DILocation(line: 17, column: 3, scope: !24)
diff --git a/test/DebugInfo/X86/cu-ranges-odr.ll b/test/DebugInfo/X86/cu-ranges-odr.ll
index 182b1fd1d3c..623e2fdd47a 100644
--- a/test/DebugInfo/X86/cu-ranges-odr.ll
+++ b/test/DebugInfo/X86/cu-ranges-odr.ll
@@ -35,9 +35,9 @@ entry:
   %this.addr = alloca %class.A*, align 8
   %i.addr = alloca i32, align 4
   store %class.A* %this, %class.A** %this.addr, align 8
-  call void @llvm.dbg.declare(metadata %class.A** %this.addr, metadata !27, metadata !MDExpression()), !dbg !29
+  call void @llvm.dbg.declare(metadata %class.A** %this.addr, metadata !27, metadata !DIExpression()), !dbg !29
   store i32 %i, i32* %i.addr, align 4
-  call void @llvm.dbg.declare(metadata i32* %i.addr, metadata !30, metadata !MDExpression()), !dbg !31
+  call void @llvm.dbg.declare(metadata i32* %i.addr, metadata !30, metadata !DIExpression()), !dbg !31
   %this1 = load %class.A*, %class.A** %this.addr
   %a = getelementptr inbounds %class.A, %class.A* %this1, i32 0, i32 0, !dbg !31
   %0 = load i32, i32* %i.addr, align 4, !dbg !31
@@ -61,35 +61,35 @@ attributes #1 = { nounwind readnone }
 !llvm.module.flags = !{!23, !24}
 !llvm.ident = !{!25}
 
-!0 = !MDCompileUnit(language: DW_LANG_C_plus_plus, producer: "clang version 3.5 (trunk 199923) (llvm/trunk 199940)", isOptimized: false, emissionKind: 1, file: !1, enums: !2, retainedTypes: !3, subprograms: !13, globals: !21, imports: !2)
-!1 = !MDFile(filename: "baz.cpp", directory: "/usr/local/google/home/echristo/tmp")
+!0 = !DICompileUnit(language: DW_LANG_C_plus_plus, producer: "clang version 3.5 (trunk 199923) (llvm/trunk 199940)", isOptimized: false, emissionKind: 1, file: !1, enums: !2, retainedTypes: !3, subprograms: !13, globals: !21, imports: !2)
+!1 = !DIFile(filename: "baz.cpp", directory: "/usr/local/google/home/echristo/tmp")
 !2 = !{}
 !3 = !{!4}
-!4 = !MDCompositeType(tag: DW_TAG_class_type, name: "A", line: 1, size: 32, align: 32, file: !1, elements: !5, identifier: "_ZTS1A")
+!4 = !DICompositeType(tag: DW_TAG_class_type, name: "A", line: 1, size: 32, align: 32, file: !1, elements: !5, identifier: "_ZTS1A")
 !5 = !{!6, !8}
-!6 = !MDDerivedType(tag: DW_TAG_member, name: "a", line: 5, size: 32, align: 32, flags: DIFlagPrivate, file: !1, scope: !"_ZTS1A", baseType: !7)
-!7 = !MDBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
-!8 = !MDSubprogram(name: "A", line: 3, isLocal: false, isDefinition: false, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 3, file: !1, scope: !"_ZTS1A", type: !9)
-!9 = !MDSubroutineType(types: !10)
+!6 = !DIDerivedType(tag: DW_TAG_member, name: "a", line: 5, size: 32, align: 32, flags: DIFlagPrivate, file: !1, scope: !"_ZTS1A", baseType: !7)
+!7 = !DIBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
+!8 = !DISubprogram(name: "A", line: 3, isLocal: false, isDefinition: false, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 3, file: !1, scope: !"_ZTS1A", type: !9)
+!9 = !DISubroutineType(types: !10)
 !10 = !{null, !11, !7}
-!11 = !MDDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, flags: DIFlagArtificial | DIFlagObjectPointer, baseType: !"_ZTS1A")
+!11 = !DIDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, flags: DIFlagArtificial | DIFlagObjectPointer, baseType: !"_ZTS1A")
 !13 = !{!14, !18, !19}
-!14 = !MDSubprogram(name: "__cxx_global_var_init", line: 8, isLocal: true, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 8, file: !1, scope: !15, type: !16, function: void ()* @__cxx_global_var_init, variables: !2)
-!15 = !MDFile(filename: "baz.cpp", directory: "/usr/local/google/home/echristo/tmp")
-!16 = !MDSubroutineType(types: !17)
+!14 = !DISubprogram(name: "__cxx_global_var_init", line: 8, isLocal: true, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 8, file: !1, scope: !15, type: !16, function: void ()* @__cxx_global_var_init, variables: !2)
+!15 = !DIFile(filename: "baz.cpp", directory: "/usr/local/google/home/echristo/tmp")
+!16 = !DISubroutineType(types: !17)
 !17 = !{null}
-!18 = !MDSubprogram(name: "A", linkageName: "_ZN1AC2Ei", line: 3, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 3, file: !1, scope: !"_ZTS1A", type: !9, function: void (%class.A*, i32)* @_ZN1AC2Ei, declaration: !8, variables: !2)
-!19 = !MDSubprogram(name: "", linkageName: "_GLOBAL__I_a", line: 3, isLocal: true, isDefinition: true, virtualIndex: 6, flags: DIFlagArtificial, isOptimized: false, scopeLine: 3, file: !1, scope: !15, type: !20, function: void ()* @_GLOBAL__I_a, variables: !2)
-!20 = !MDSubroutineType(types: !2)
+!18 = !DISubprogram(name: "A", linkageName: "_ZN1AC2Ei", line: 3, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 3, file: !1, scope: !"_ZTS1A", type: !9, function: void (%class.A*, i32)* @_ZN1AC2Ei, declaration: !8, variables: !2)
+!19 = !DISubprogram(name: "", linkageName: "_GLOBAL__I_a", line: 3, isLocal: true, isDefinition: true, virtualIndex: 6, flags: DIFlagArtificial, isOptimized: false, scopeLine: 3, file: !1, scope: !15, type: !20, function: void ()* @_GLOBAL__I_a, variables: !2)
+!20 = !DISubroutineType(types: !2)
 !21 = !{!22}
-!22 = !MDGlobalVariable(name: "a", line: 8, isLocal: false, isDefinition: true, scope: null, file: !15, type: !4, variable: %class.A* @a)
+!22 = !DIGlobalVariable(name: "a", line: 8, isLocal: false, isDefinition: true, scope: null, file: !15, type: !4, variable: %class.A* @a)
 !23 = !{i32 2, !"Dwarf Version", i32 4}
 !24 = !{i32 1, !"Debug Info Version", i32 3}
 !25 = !{!"clang version 3.5 (trunk 199923) (llvm/trunk 199940)"}
-!26 = !MDLocation(line: 8, scope: !14)
-!27 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "this", arg: 1, flags: DIFlagArtificial | DIFlagObjectPointer, scope: !18, type: !28)
-!28 = !MDDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, baseType: !"_ZTS1A")
-!29 = !MDLocation(line: 0, scope: !18)
-!30 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "i", line: 3, arg: 2, scope: !18, file: !15, type: !7)
-!31 = !MDLocation(line: 3, scope: !18)
-!32 = !MDLocation(line: 3, scope: !19)
+!26 = !DILocation(line: 8, scope: !14)
+!27 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "this", arg: 1, flags: DIFlagArtificial | DIFlagObjectPointer, scope: !18, type: !28)
+!28 = !DIDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, baseType: !"_ZTS1A")
+!29 = !DILocation(line: 0, scope: !18)
+!30 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "i", line: 3, arg: 2, scope: !18, file: !15, type: !7)
+!31 = !DILocation(line: 3, scope: !18)
+!32 = !DILocation(line: 3, scope: !19)
diff --git a/test/DebugInfo/X86/cu-ranges.ll b/test/DebugInfo/X86/cu-ranges.ll
index f44a4d16d02..5611a31cd5f 100644
--- a/test/DebugInfo/X86/cu-ranges.ll
+++ b/test/DebugInfo/X86/cu-ranges.ll
@@ -29,7 +29,7 @@ define i32 @foo(i32 %a) #0 {
 entry:
   %a.addr = alloca i32, align 4
   store i32 %a, i32* %a.addr, align 4
-  call void @llvm.dbg.declare(metadata i32* %a.addr, metadata !13, metadata !MDExpression()), !dbg !14
+  call void @llvm.dbg.declare(metadata i32* %a.addr, metadata !13, metadata !DIExpression()), !dbg !14
   %0 = load i32, i32* %a.addr, align 4, !dbg !14
   %add = add nsw i32 %0, 1, !dbg !14
   ret i32 %add, !dbg !14
@@ -43,7 +43,7 @@ define i32 @bar(i32 %b) #0 {
 entry:
   %b.addr = alloca i32, align 4
   store i32 %b, i32* %b.addr, align 4
-  call void @llvm.dbg.declare(metadata i32* %b.addr, metadata !15, metadata !MDExpression()), !dbg !16
+  call void @llvm.dbg.declare(metadata i32* %b.addr, metadata !15, metadata !DIExpression()), !dbg !16
   %0 = load i32, i32* %b.addr, align 4, !dbg !16
   %add = add nsw i32 %0, 2, !dbg !16
   ret i32 %add, !dbg !16
@@ -56,20 +56,20 @@ attributes #1 = { nounwind readnone }
 !llvm.module.flags = !{!10, !11}
 !llvm.ident = !{!12}
 
-!0 = !MDCompileUnit(language: DW_LANG_C99, producer: "clang version 3.5.0 (trunk 204164) (llvm/trunk 204183)", isOptimized: false, emissionKind: 1, file: !1, enums: !2, retainedTypes: !2, subprograms: !3, globals: !2, imports: !2)
-!1 = !MDFile(filename: "z.c", directory: "/usr/local/google/home/echristo")
+!0 = !DICompileUnit(language: DW_LANG_C99, producer: "clang version 3.5.0 (trunk 204164) (llvm/trunk 204183)", isOptimized: false, emissionKind: 1, file: !1, enums: !2, retainedTypes: !2, subprograms: !3, globals: !2, imports: !2)
+!1 = !DIFile(filename: "z.c", directory: "/usr/local/google/home/echristo")
 !2 = !{}
 !3 = !{!4, !9}
-!4 = !MDSubprogram(name: "foo", line: 1, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 1, file: !1, scope: !5, type: !6, function: i32 (i32)* @foo, variables: !2)
-!5 = !MDFile(filename: "z.c", directory: "/usr/local/google/home/echristo")
-!6 = !MDSubroutineType(types: !7)
+!4 = !DISubprogram(name: "foo", line: 1, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 1, file: !1, scope: !5, type: !6, function: i32 (i32)* @foo, variables: !2)
+!5 = !DIFile(filename: "z.c", directory: "/usr/local/google/home/echristo")
+!6 = !DISubroutineType(types: !7)
 !7 = !{!8, !8}
-!8 = !MDBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
-!9 = !MDSubprogram(name: "bar", line: 2, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 2, file: !1, scope: !5, type: !6, function: i32 (i32)* @bar, variables: !2)
+!8 = !DIBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
+!9 = !DISubprogram(name: "bar", line: 2, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 2, file: !1, scope: !5, type: !6, function: i32 (i32)* @bar, variables: !2)
 !10 = !{i32 2, !"Dwarf Version", i32 4}
 !11 = !{i32 1, !"Debug Info Version", i32 3}
 !12 = !{!"clang version 3.5.0 (trunk 204164) (llvm/trunk 204183)"}
-!13 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "a", line: 1, arg: 1, scope: !4, file: !5, type: !8)
-!14 = !MDLocation(line: 1, scope: !4)
-!15 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "b", line: 2, arg: 1, scope: !9, file: !5, type: !8)
-!16 = !MDLocation(line: 2, scope: !9)
+!13 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "a", line: 1, arg: 1, scope: !4, file: !5, type: !8)
+!14 = !DILocation(line: 1, scope: !4)
+!15 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "b", line: 2, arg: 1, scope: !9, file: !5, type: !8)
+!16 = !DILocation(line: 2, scope: !9)
diff --git a/test/DebugInfo/X86/data_member_location.ll b/test/DebugInfo/X86/data_member_location.ll
index b5361166738..c17c45bcc69 100644
--- a/test/DebugInfo/X86/data_member_location.ll
+++ b/test/DebugInfo/X86/data_member_location.ll
@@ -34,19 +34,19 @@
 !llvm.module.flags = !{!13, !15}
 !llvm.ident = !{!14}
 
-!0 = !MDCompileUnit(language: DW_LANG_C_plus_plus, producer: "clang version 3.4 ", isOptimized: false, emissionKind: 0, file: !1, enums: !2, retainedTypes: !3, subprograms: !2, globals: !10, imports: !2)
-!1 = !MDFile(filename: "data_member_location.cpp", directory: "/tmp/dbginfo")
+!0 = !DICompileUnit(language: DW_LANG_C_plus_plus, producer: "clang version 3.4 ", isOptimized: false, emissionKind: 0, file: !1, enums: !2, retainedTypes: !3, subprograms: !2, globals: !10, imports: !2)
+!1 = !DIFile(filename: "data_member_location.cpp", directory: "/tmp/dbginfo")
 !2 = !{}
 !3 = !{!4}
-!4 = !MDCompositeType(tag: DW_TAG_structure_type, name: "foo", line: 1, size: 64, align: 32, file: !1, elements: !5, identifier: "_ZTS3foo")
+!4 = !DICompositeType(tag: DW_TAG_structure_type, name: "foo", line: 1, size: 64, align: 32, file: !1, elements: !5, identifier: "_ZTS3foo")
 !5 = !{!6, !8}
-!6 = !MDDerivedType(tag: DW_TAG_member, name: "c", line: 2, size: 8, align: 8, file: !1, scope: !"_ZTS3foo", baseType: !7)
-!7 = !MDBasicType(tag: DW_TAG_base_type, name: "char", size: 8, align: 8, encoding: DW_ATE_signed_char)
-!8 = !MDDerivedType(tag: DW_TAG_member, name: "i", line: 3, size: 32, align: 32, offset: 32, file: !1, scope: !"_ZTS3foo", baseType: !9)
-!9 = !MDBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
+!6 = !DIDerivedType(tag: DW_TAG_member, name: "c", line: 2, size: 8, align: 8, file: !1, scope: !"_ZTS3foo", baseType: !7)
+!7 = !DIBasicType(tag: DW_TAG_base_type, name: "char", size: 8, align: 8, encoding: DW_ATE_signed_char)
+!8 = !DIDerivedType(tag: DW_TAG_member, name: "i", line: 3, size: 32, align: 32, offset: 32, file: !1, scope: !"_ZTS3foo", baseType: !9)
+!9 = !DIBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
 !10 = !{!11}
-!11 = !MDGlobalVariable(name: "f", line: 6, isLocal: false, isDefinition: true, scope: null, file: !12, type: !4, variable: %struct.foo* @f)
-!12 = !MDFile(filename: "data_member_location.cpp", directory: "/tmp/dbginfo")
+!11 = !DIGlobalVariable(name: "f", line: 6, isLocal: false, isDefinition: true, scope: null, file: !12, type: !4, variable: %struct.foo* @f)
+!12 = !DIFile(filename: "data_member_location.cpp", directory: "/tmp/dbginfo")
 !13 = !{i32 2, !"Dwarf Version", i32 4}
 !14 = !{!"clang version 3.4 "}
 
diff --git a/test/DebugInfo/X86/dbg-at-specficiation.ll b/test/DebugInfo/X86/dbg-at-specficiation.ll
index ee568af1ca3..ccb5ab142cd 100644
--- a/test/DebugInfo/X86/dbg-at-specficiation.ll
+++ b/test/DebugInfo/X86/dbg-at-specficiation.ll
@@ -8,14 +8,14 @@
 !llvm.dbg.cu = !{!0}
 !llvm.module.flags = !{!12}
 
-!0 = !MDCompileUnit(language: DW_LANG_C99, producer: "clang version 3.0 (trunk 140253)", isOptimized: true, emissionKind: 0, file: !11, enums: !2, retainedTypes: !2, subprograms: !2, globals: !3)
+!0 = !DICompileUnit(language: DW_LANG_C99, producer: "clang version 3.0 (trunk 140253)", isOptimized: true, emissionKind: 0, file: !11, enums: !2, retainedTypes: !2, subprograms: !2, globals: !3)
 !2 = !{}
 !3 = !{!5}
-!5 = !MDGlobalVariable(name: "a", line: 1, isLocal: false, isDefinition: true, scope: null, file: !6, type: !7, variable: [10 x i32]* @a)
-!6 = !MDFile(filename: "x.c", directory: "/private/tmp")
-!7 = !MDCompositeType(tag: DW_TAG_array_type, size: 320, align: 32, baseType: !8, elements: !9)
-!8 = !MDBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
+!5 = !DIGlobalVariable(name: "a", line: 1, isLocal: false, isDefinition: true, scope: null, file: !6, type: !7, variable: [10 x i32]* @a)
+!6 = !DIFile(filename: "x.c", directory: "/private/tmp")
+!7 = !DICompositeType(tag: DW_TAG_array_type, size: 320, align: 32, baseType: !8, elements: !9)
+!8 = !DIBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
 !9 = !{!10}
-!10 = !MDSubrange(count: 10)
-!11 = !MDFile(filename: "x.c", directory: "/private/tmp")
+!10 = !DISubrange(count: 10)
+!11 = !DIFile(filename: "x.c", directory: "/private/tmp")
 !12 = !{i32 1, !"Debug Info Version", i32 3}
diff --git a/test/DebugInfo/X86/dbg-byval-parameter.ll b/test/DebugInfo/X86/dbg-byval-parameter.ll
index c30e977651b..a592884496a 100644
--- a/test/DebugInfo/X86/dbg-byval-parameter.ll
+++ b/test/DebugInfo/X86/dbg-byval-parameter.ll
@@ -9,7 +9,7 @@ entry:
   %retval = alloca double                         ; <double*> [#uses=2]
   %0 = alloca double                              ; <double*> [#uses=2]
   %"alloca point" = bitcast i32 0 to i32          ; <i32> [#uses=0]
-  call void @llvm.dbg.declare(metadata %struct.Rect* %my_r0, metadata !0, metadata !MDExpression()), !dbg !15
+  call void @llvm.dbg.declare(metadata %struct.Rect* %my_r0, metadata !0, metadata !DIExpression()), !dbg !15
   %1 = getelementptr inbounds %struct.Rect, %struct.Rect* %my_r0, i32 0, i32 0, !dbg !16 ; <%struct.Pt*> [#uses=1]
   %2 = getelementptr inbounds %struct.Pt, %struct.Pt* %1, i32 0, i32 0, !dbg !16 ; <double*> [#uses=1]
   %3 = load double, double* %2, align 8, !dbg !16         ; <double> [#uses=1]
@@ -28,25 +28,25 @@ declare void @llvm.dbg.declare(metadata, metadata, metadata) nounwind readnone
 !llvm.dbg.cu = !{!3}
 !llvm.module.flags = !{!21}
 
-!0 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "my_r0", line: 11, arg: 0, scope: !1, file: !2, type: !7)
-!1 = !MDSubprogram(name: "foo", linkageName: "foo", line: 11, isLocal: false, isDefinition: true, virtualIndex: 6, isOptimized: false, file: !19, scope: !2, type: !4, function: double (%struct.Rect*)* @foo)
-!2 = !MDFile(filename: "b2.c", directory: "/tmp/")
-!3 = !MDCompileUnit(language: DW_LANG_C89, producer: "4.2.1 (Based on Apple Inc. build 5658) (LLVM build)", isOptimized: false, emissionKind: 0, file: !19, enums: !20, retainedTypes: !20, subprograms: !18, imports:  null)
-!4 = !MDSubroutineType(types: !5)
+!0 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "my_r0", line: 11, arg: 1, scope: !1, file: !2, type: !7)
+!1 = !DISubprogram(name: "foo", linkageName: "foo", line: 11, isLocal: false, isDefinition: true, virtualIndex: 6, isOptimized: false, file: !19, scope: !2, type: !4, function: double (%struct.Rect*)* @foo)
+!2 = !DIFile(filename: "b2.c", directory: "/tmp/")
+!3 = !DICompileUnit(language: DW_LANG_C89, producer: "4.2.1 (Based on Apple Inc. build 5658) (LLVM build)", isOptimized: false, emissionKind: 0, file: !19, enums: !20, retainedTypes: !20, subprograms: !18, imports:  null)
+!4 = !DISubroutineType(types: !5)
 !5 = !{!6, !7}
-!6 = !MDBasicType(tag: DW_TAG_base_type, name: "double", size: 64, align: 64, encoding: DW_ATE_float)
-!7 = !MDCompositeType(tag: DW_TAG_structure_type, name: "Rect", line: 6, size: 256, align: 64, file: !19, scope: !2, elements: !8)
+!6 = !DIBasicType(tag: DW_TAG_base_type, name: "double", size: 64, align: 64, encoding: DW_ATE_float)
+!7 = !DICompositeType(tag: DW_TAG_structure_type, name: "Rect", line: 6, size: 256, align: 64, file: !19, scope: !2, elements: !8)
 !8 = !{!9, !14}
-!9 = !MDDerivedType(tag: DW_TAG_member, name: "P1", line: 7, size: 128, align: 64, file: !19, scope: !7, baseType: !10)
-!10 = !MDCompositeType(tag: DW_TAG_structure_type, name: "Pt", line: 1, size: 128, align: 64, file: !19, scope: !2, elements: !11)
+!9 = !DIDerivedType(tag: DW_TAG_member, name: "P1", line: 7, size: 128, align: 64, file: !19, scope: !7, baseType: !10)
+!10 = !DICompositeType(tag: DW_TAG_structure_type, name: "Pt", line: 1, size: 128, align: 64, file: !19, scope: !2, elements: !11)
 !11 = !{!12, !13}
-!12 = !MDDerivedType(tag: DW_TAG_member, name: "x", line: 2, size: 64, align: 64, file: !19, scope: !10, baseType: !6)
-!13 = !MDDerivedType(tag: DW_TAG_member, name: "y", line: 3, size: 64, align: 64, offset: 64, file: !19, scope: !10, baseType: !6)
-!14 = !MDDerivedType(tag: DW_TAG_member, name: "P2", line: 8, size: 128, align: 64, offset: 128, file: !19, scope: !7, baseType: !10)
-!15 = !MDLocation(line: 11, scope: !1)
-!16 = !MDLocation(line: 12, scope: !17)
-!17 = distinct !MDLexicalBlock(line: 11, column: 0, file: !19, scope: !1)
+!12 = !DIDerivedType(tag: DW_TAG_member, name: "x", line: 2, size: 64, align: 64, file: !19, scope: !10, baseType: !6)
+!13 = !DIDerivedType(tag: DW_TAG_member, name: "y", line: 3, size: 64, align: 64, offset: 64, file: !19, scope: !10, baseType: !6)
+!14 = !DIDerivedType(tag: DW_TAG_member, name: "P2", line: 8, size: 128, align: 64, offset: 128, file: !19, scope: !7, baseType: !10)
+!15 = !DILocation(line: 11, scope: !1)
+!16 = !DILocation(line: 12, scope: !17)
+!17 = distinct !DILexicalBlock(line: 11, column: 0, file: !19, scope: !1)
 !18 = !{!1}
-!19 = !MDFile(filename: "b2.c", directory: "/tmp/")
+!19 = !DIFile(filename: "b2.c", directory: "/tmp/")
 !20 = !{}
 !21 = !{i32 1, !"Debug Info Version", i32 3}
diff --git a/test/DebugInfo/X86/dbg-const-int.ll b/test/DebugInfo/X86/dbg-const-int.ll
index abe4e94b645..8482abf373e 100644
--- a/test/DebugInfo/X86/dbg-const-int.ll
+++ b/test/DebugInfo/X86/dbg-const-int.ll
@@ -12,7 +12,7 @@ target triple = "x86_64-apple-macosx10.6.7"
 
 define i32 @foo() nounwind uwtable readnone optsize ssp {
 entry:
-  tail call void @llvm.dbg.value(metadata i32 42, i64 0, metadata !6, metadata !MDExpression()), !dbg !9
+  tail call void @llvm.dbg.value(metadata i32 42, i64 0, metadata !6, metadata !DIExpression()), !dbg !9
   ret i32 42, !dbg !10
 }
 
@@ -21,19 +21,19 @@ declare void @llvm.dbg.value(metadata, i64, metadata, metadata) nounwind readnon
 !llvm.dbg.cu = !{!0}
 !llvm.module.flags = !{!15}
 
-!0 = !MDCompileUnit(language: DW_LANG_C99, producer: "clang version 3.0 (trunk 132191)", isOptimized: true, emissionKind: 0, file: !13, enums: !14, retainedTypes: !14, subprograms: !11, imports:  null)
-!1 = !MDSubprogram(name: "foo", line: 1, isLocal: false, isDefinition: true, virtualIndex: 6, isOptimized: true, file: !13, scope: !2, type: !3, function: i32 ()* @foo, variables: !12)
-!2 = !MDFile(filename: "a.c", directory: "/private/tmp")
-!3 = !MDSubroutineType(types: !4)
+!0 = !DICompileUnit(language: DW_LANG_C99, producer: "clang version 3.0 (trunk 132191)", isOptimized: true, emissionKind: 0, file: !13, enums: !14, retainedTypes: !14, subprograms: !11, imports:  null)
+!1 = !DISubprogram(name: "foo", line: 1, isLocal: false, isDefinition: true, virtualIndex: 6, isOptimized: true, file: !13, scope: !2, type: !3, function: i32 ()* @foo, variables: !12)
+!2 = !DIFile(filename: "a.c", directory: "/private/tmp")
+!3 = !DISubroutineType(types: !4)
 !4 = !{!5}
-!5 = !MDBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
-!6 = !MDLocalVariable(tag: DW_TAG_auto_variable, name: "i", line: 2, scope: !7, file: !2, type: !5)
-!7 = distinct !MDLexicalBlock(line: 1, column: 11, file: !13, scope: !1)
+!5 = !DIBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
+!6 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "i", line: 2, scope: !7, file: !2, type: !5)
+!7 = distinct !DILexicalBlock(line: 1, column: 11, file: !13, scope: !1)
 !8 = !{i32 42}
-!9 = !MDLocation(line: 2, column: 12, scope: !7)
-!10 = !MDLocation(line: 3, column: 2, scope: !7)
+!9 = !DILocation(line: 2, column: 12, scope: !7)
+!10 = !DILocation(line: 3, column: 2, scope: !7)
 !11 = !{!1}
 !12 = !{!6}
-!13 = !MDFile(filename: "a.c", directory: "/private/tmp")
+!13 = !DIFile(filename: "a.c", directory: "/private/tmp")
 !14 = !{}
 !15 = !{i32 1, !"Debug Info Version", i32 3}
diff --git a/test/DebugInfo/X86/dbg-const.ll b/test/DebugInfo/X86/dbg-const.ll
index fd20c3c9e8c..27a0eabfe49 100644
--- a/test/DebugInfo/X86/dbg-const.ll
+++ b/test/DebugInfo/X86/dbg-const.ll
@@ -17,9 +17,9 @@ target triple = "x86_64-apple-darwin10.0.0"
 ;CHECK-NEXT:  .byte	42
 define i32 @foobar() nounwind readonly noinline ssp {
 entry:
-  tail call void @llvm.dbg.value(metadata i32 42, i64 0, metadata !6, metadata !MDExpression()), !dbg !9
+  tail call void @llvm.dbg.value(metadata i32 42, i64 0, metadata !6, metadata !DIExpression()), !dbg !9
   %call = tail call i32 @bar(), !dbg !11
-  tail call void @llvm.dbg.value(metadata i32 %call, i64 0, metadata !6, metadata !MDExpression()), !dbg !11
+  tail call void @llvm.dbg.value(metadata i32 %call, i64 0, metadata !6, metadata !DIExpression()), !dbg !11
   %call2 = tail call i32 @bar(), !dbg !11
   %add = add nsw i32 %call2, %call, !dbg !12
   ret i32 %add, !dbg !10
@@ -31,21 +31,21 @@ declare i32 @bar() nounwind readnone
 !llvm.dbg.cu = !{!2}
 !llvm.module.flags = !{!17}
 
-!0 = !MDSubprogram(name: "foobar", linkageName: "foobar", line: 12, isLocal: false, isDefinition: true, virtualIndex: 6, isOptimized: true, file: !15, scope: !1, type: !3, function: i32 ()* @foobar, variables: !14)
-!1 = !MDFile(filename: "mu.c", directory: "/private/tmp")
-!2 = !MDCompileUnit(language: DW_LANG_C99, producer: "clang version 2.9 (trunk 114183)", isOptimized: true, emissionKind: 1, file: !15, enums: !16, retainedTypes: !16, subprograms: !13, imports:  null)
-!3 = !MDSubroutineType(types: !4)
+!0 = !DISubprogram(name: "foobar", linkageName: "foobar", line: 12, isLocal: false, isDefinition: true, virtualIndex: 6, isOptimized: true, file: !15, scope: !1, type: !3, function: i32 ()* @foobar, variables: !14)
+!1 = !DIFile(filename: "mu.c", directory: "/private/tmp")
+!2 = !DICompileUnit(language: DW_LANG_C99, producer: "clang version 2.9 (trunk 114183)", isOptimized: true, emissionKind: 1, file: !15, enums: !16, retainedTypes: !16, subprograms: !13, imports:  null)
+!3 = !DISubroutineType(types: !4)
 !4 = !{!5}
-!5 = !MDBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
-!6 = !MDLocalVariable(tag: DW_TAG_auto_variable, name: "j", line: 15, scope: !7, file: !1, type: !5)
-!7 = distinct !MDLexicalBlock(line: 12, column: 52, file: !15, scope: !0)
+!5 = !DIBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
+!6 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "j", line: 15, scope: !7, file: !1, type: !5)
+!7 = distinct !DILexicalBlock(line: 12, column: 52, file: !15, scope: !0)
 !8 = !{i32 42}
-!9 = !MDLocation(line: 15, column: 12, scope: !7)
-!10 = !MDLocation(line: 23, column: 3, scope: !7)
-!11 = !MDLocation(line: 17, column: 3, scope: !7)
-!12 = !MDLocation(line: 18, column: 3, scope: !7)
+!9 = !DILocation(line: 15, column: 12, scope: !7)
+!10 = !DILocation(line: 23, column: 3, scope: !7)
+!11 = !DILocation(line: 17, column: 3, scope: !7)
+!12 = !DILocation(line: 18, column: 3, scope: !7)
 !13 = !{!0}
 !14 = !{!6}
-!15 = !MDFile(filename: "mu.c", directory: "/private/tmp")
+!15 = !DIFile(filename: "mu.c", directory: "/private/tmp")
 !16 = !{}
 !17 = !{i32 1, !"Debug Info Version", i32 3}
diff --git a/test/DebugInfo/X86/dbg-declare-arg.ll b/test/DebugInfo/X86/dbg-declare-arg.ll
index a3a61e57adf..85ff4b920ad 100644
--- a/test/DebugInfo/X86/dbg-declare-arg.ll
+++ b/test/DebugInfo/X86/dbg-declare-arg.ll
@@ -14,8 +14,8 @@ entry:
   %nrvo = alloca i1
   %cleanup.dest.slot = alloca i32
   store i32 %i, i32* %i.addr, align 4
-  call void @llvm.dbg.declare(metadata i32* %i.addr, metadata !26, metadata !MDExpression()), !dbg !27
-  call void @llvm.dbg.declare(metadata i32* %j, metadata !28, metadata !MDExpression()), !dbg !30
+  call void @llvm.dbg.declare(metadata i32* %i.addr, metadata !26, metadata !DIExpression()), !dbg !27
+  call void @llvm.dbg.declare(metadata i32* %j, metadata !28, metadata !DIExpression()), !dbg !30
   store i32 0, i32* %j, align 4, !dbg !31
   %tmp = load i32, i32* %i.addr, align 4, !dbg !32
   %cmp = icmp eq i32 %tmp, 42, !dbg !32
@@ -29,7 +29,7 @@ if.then:                                          ; preds = %entry
 
 if.end:                                           ; preds = %if.then, %entry
   store i1 false, i1* %nrvo, !dbg !36
-  call void @llvm.dbg.declare(metadata %class.A* %agg.result, metadata !37, metadata !MDExpression()), !dbg !39
+  call void @llvm.dbg.declare(metadata %class.A* %agg.result, metadata !37, metadata !DIExpression()), !dbg !39
   %tmp2 = load i32, i32* %j, align 4, !dbg !40
   %x = getelementptr inbounds %class.A, %class.A* %agg.result, i32 0, i32 0, !dbg !40
   store i32 %tmp2, i32* %x, align 4, !dbg !40
@@ -52,7 +52,7 @@ define linkonce_odr void @_ZN1AD1Ev(%class.A* %this) unnamed_addr ssp align 2 {
 entry:
   %this.addr = alloca %class.A*, align 8
   store %class.A* %this, %class.A** %this.addr, align 8
-  call void @llvm.dbg.declare(metadata %class.A** %this.addr, metadata !43, metadata !MDExpression()), !dbg !44
+  call void @llvm.dbg.declare(metadata %class.A** %this.addr, metadata !43, metadata !DIExpression()), !dbg !44
   %this1 = load %class.A*, %class.A** %this.addr
   call void @_ZN1AD2Ev(%class.A* %this1)
   ret void, !dbg !45
@@ -62,7 +62,7 @@ define linkonce_odr void @_ZN1AD2Ev(%class.A* %this) unnamed_addr nounwind ssp a
 entry:
   %this.addr = alloca %class.A*, align 8
   store %class.A* %this, %class.A** %this.addr, align 8
-  call void @llvm.dbg.declare(metadata %class.A** %this.addr, metadata !46, metadata !MDExpression()), !dbg !47
+  call void @llvm.dbg.declare(metadata %class.A** %this.addr, metadata !46, metadata !DIExpression()), !dbg !47
   %this1 = load %class.A*, %class.A** %this.addr
   %x = getelementptr inbounds %class.A, %class.A* %this1, i32 0, i32 0, !dbg !48
   store i32 1, i32* %x, align 4, !dbg !48
@@ -72,56 +72,56 @@ entry:
 !llvm.dbg.cu = !{!2}
 !llvm.module.flags = !{!52}
 
-!0 = !MDSubprogram(name: "~A", line: 2, isLocal: false, isDefinition: false, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, file: !51, scope: !1, type: !11)
-!1 = !MDCompositeType(tag: DW_TAG_class_type, name: "A", line: 2, size: 128, align: 32, file: !51, scope: !2, elements: !4)
-!2 = !MDCompileUnit(language: DW_LANG_C_plus_plus, producer: "clang version 3.0 (trunk 130127)", isOptimized: false, emissionKind: 1, file: !51, enums: !{}, retainedTypes: !{}, subprograms: !50)
-!3 = !MDFile(filename: "a.cc", directory: "/private/tmp")
+!0 = !DISubprogram(name: "~A", line: 2, isLocal: false, isDefinition: false, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, file: !51, scope: !1, type: !11)
+!1 = !DICompositeType(tag: DW_TAG_class_type, name: "A", line: 2, size: 128, align: 32, file: !51, scope: !2, elements: !4)
+!2 = !DICompileUnit(language: DW_LANG_C_plus_plus, producer: "clang version 3.0 (trunk 130127)", isOptimized: false, emissionKind: 1, file: !51, enums: !{}, retainedTypes: !{}, subprograms: !50)
+!3 = !DIFile(filename: "a.cc", directory: "/private/tmp")
 !4 = !{!5, !7, !8, !9, !0, !10, !14}
-!5 = !MDDerivedType(tag: DW_TAG_member, name: "x", line: 2, size: 32, align: 32, file: !51, scope: !3, baseType: !6)
-!6 = !MDBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
-!7 = !MDDerivedType(tag: DW_TAG_member, name: "y", line: 2, size: 32, align: 32, offset: 32, file: !51, scope: !3, baseType: !6)
-!8 = !MDDerivedType(tag: DW_TAG_member, name: "z", line: 2, size: 32, align: 32, offset: 64, file: !51, scope: !3, baseType: !6)
-!9 = !MDDerivedType(tag: DW_TAG_member, name: "o", line: 2, size: 32, align: 32, offset: 96, file: !51, scope: !3, baseType: !6)
-!10 = !MDSubprogram(name: "A", line: 2, isLocal: false, isDefinition: false, virtualIndex: 6, flags: DIFlagArtificial | DIFlagPrototyped, isOptimized: false, file: !51, scope: !1, type: !11)
-!11 = !MDSubroutineType(types: !12)
+!5 = !DIDerivedType(tag: DW_TAG_member, name: "x", line: 2, size: 32, align: 32, file: !51, scope: !3, baseType: !6)
+!6 = !DIBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
+!7 = !DIDerivedType(tag: DW_TAG_member, name: "y", line: 2, size: 32, align: 32, offset: 32, file: !51, scope: !3, baseType: !6)
+!8 = !DIDerivedType(tag: DW_TAG_member, name: "z", line: 2, size: 32, align: 32, offset: 64, file: !51, scope: !3, baseType: !6)
+!9 = !DIDerivedType(tag: DW_TAG_member, name: "o", line: 2, size: 32, align: 32, offset: 96, file: !51, scope: !3, baseType: !6)
+!10 = !DISubprogram(name: "A", line: 2, isLocal: false, isDefinition: false, virtualIndex: 6, flags: DIFlagArtificial | DIFlagPrototyped, isOptimized: false, file: !51, scope: !1, type: !11)
+!11 = !DISubroutineType(types: !12)
 !12 = !{null, !13}
-!13 = !MDDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, flags: DIFlagArtificial, file: !3, baseType: !1)
-!14 = !MDSubprogram(name: "A", line: 2, isLocal: false, isDefinition: false, virtualIndex: 6, flags: DIFlagArtificial | DIFlagPrototyped, isOptimized: false, file: !51, scope: !1, type: !15)
-!15 = !MDSubroutineType(types: !16)
+!13 = !DIDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, flags: DIFlagArtificial, file: !3, baseType: !1)
+!14 = !DISubprogram(name: "A", line: 2, isLocal: false, isDefinition: false, virtualIndex: 6, flags: DIFlagArtificial | DIFlagPrototyped, isOptimized: false, file: !51, scope: !1, type: !15)
+!15 = !DISubroutineType(types: !16)
 !16 = !{null, !13, !17}
-!17 = !MDDerivedType(tag: DW_TAG_reference_type, scope: !2, baseType: !18)
-!18 = !MDDerivedType(tag: DW_TAG_const_type, file: !3, baseType: !1)
-!19 = !MDSubprogram(name: "foo", linkageName: "_Z3fooi", line: 4, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, file: !51, scope: !3, type: !20, function: void (%class.A*, i32)* @_Z3fooi)
-!20 = !MDSubroutineType(types: !21)
+!17 = !DIDerivedType(tag: DW_TAG_reference_type, scope: !2, baseType: !18)
+!18 = !DIDerivedType(tag: DW_TAG_const_type, file: !3, baseType: !1)
+!19 = !DISubprogram(name: "foo", linkageName: "_Z3fooi", line: 4, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, file: !51, scope: !3, type: !20, function: void (%class.A*, i32)* @_Z3fooi)
+!20 = !DISubroutineType(types: !21)
 !21 = !{!1}
-!22 = !MDSubprogram(name: "~A", linkageName: "_ZN1AD1Ev", line: 2, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, file: !51, scope: !3, type: !23, function: void (%class.A*)* @_ZN1AD1Ev)
-!23 = !MDSubroutineType(types: !24)
+!22 = !DISubprogram(name: "~A", linkageName: "_ZN1AD1Ev", line: 2, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, file: !51, scope: !3, type: !23, function: void (%class.A*)* @_ZN1AD1Ev)
+!23 = !DISubroutineType(types: !24)
 !24 = !{null}
-!25 = !MDSubprogram(name: "~A", linkageName: "_ZN1AD2Ev", line: 2, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, file: !51, scope: !3, type: !23, function: void (%class.A*)* @_ZN1AD2Ev)
-!26 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "i", line: 4, arg: 1, scope: !19, file: !3, type: !6)
-!27 = !MDLocation(line: 4, column: 11, scope: !19)
-!28 = !MDLocalVariable(tag: DW_TAG_auto_variable, name: "j", line: 5, scope: !29, file: !3, type: !6)
-!29 = distinct !MDLexicalBlock(line: 4, column: 14, file: !51, scope: !19)
-!30 = !MDLocation(line: 5, column: 7, scope: !29)
-!31 = !MDLocation(line: 5, column: 12, scope: !29)
-!32 = !MDLocation(line: 6, column: 3, scope: !29)
-!33 = !MDLocation(line: 7, column: 5, scope: !34)
-!34 = distinct !MDLexicalBlock(line: 6, column: 16, file: !51, scope: !29)
-!35 = !MDLocation(line: 8, column: 3, scope: !34)
-!36 = !MDLocation(line: 9, column: 9, scope: !29)
-!37 = !MDLocalVariable(tag: DW_TAG_auto_variable, name: "my_a", line: 9, scope: !29, file: !3, type: !38)
-!38 = !MDDerivedType(tag: DW_TAG_reference_type, file: !3, baseType: !1)
-!39 = !MDLocation(line: 9, column: 5, scope: !29)
-!40 = !MDLocation(line: 10, column: 3, scope: !29)
-!41 = !MDLocation(line: 11, column: 3, scope: !29)
-!42 = !MDLocation(line: 12, column: 1, scope: !29)
-!43 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "this", line: 2, arg: 1, flags: DIFlagArtificial, scope: !22, file: !3, type: !13)
-!44 = !MDLocation(line: 2, column: 47, scope: !22)
-!45 = !MDLocation(line: 2, column: 61, scope: !22)
-!46 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "this", line: 2, arg: 1, flags: DIFlagArtificial, scope: !25, file: !3, type: !13)
-!47 = !MDLocation(line: 2, column: 47, scope: !25)
-!48 = !MDLocation(line: 2, column: 54, scope: !49)
-!49 = distinct !MDLexicalBlock(line: 2, column: 52, file: !51, scope: !25)
+!25 = !DISubprogram(name: "~A", linkageName: "_ZN1AD2Ev", line: 2, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, file: !51, scope: !3, type: !23, function: void (%class.A*)* @_ZN1AD2Ev)
+!26 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "i", line: 4, arg: 1, scope: !19, file: !3, type: !6)
+!27 = !DILocation(line: 4, column: 11, scope: !19)
+!28 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "j", line: 5, scope: !29, file: !3, type: !6)
+!29 = distinct !DILexicalBlock(line: 4, column: 14, file: !51, scope: !19)
+!30 = !DILocation(line: 5, column: 7, scope: !29)
+!31 = !DILocation(line: 5, column: 12, scope: !29)
+!32 = !DILocation(line: 6, column: 3, scope: !29)
+!33 = !DILocation(line: 7, column: 5, scope: !34)
+!34 = distinct !DILexicalBlock(line: 6, column: 16, file: !51, scope: !29)
+!35 = !DILocation(line: 8, column: 3, scope: !34)
+!36 = !DILocation(line: 9, column: 9, scope: !29)
+!37 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "my_a", line: 9, scope: !29, file: !3, type: !38)
+!38 = !DIDerivedType(tag: DW_TAG_reference_type, file: !3, baseType: !1)
+!39 = !DILocation(line: 9, column: 5, scope: !29)
+!40 = !DILocation(line: 10, column: 3, scope: !29)
+!41 = !DILocation(line: 11, column: 3, scope: !29)
+!42 = !DILocation(line: 12, column: 1, scope: !29)
+!43 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "this", line: 2, arg: 1, flags: DIFlagArtificial, scope: !22, file: !3, type: !13)
+!44 = !DILocation(line: 2, column: 47, scope: !22)
+!45 = !DILocation(line: 2, column: 61, scope: !22)
+!46 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "this", line: 2, arg: 1, flags: DIFlagArtificial, scope: !25, file: !3, type: !13)
+!47 = !DILocation(line: 2, column: 47, scope: !25)
+!48 = !DILocation(line: 2, column: 54, scope: !49)
+!49 = distinct !DILexicalBlock(line: 2, column: 52, file: !51, scope: !25)
 !50 = !{!19, !22, !25}
-!51 = !MDFile(filename: "a.cc", directory: "/private/tmp")
+!51 = !DIFile(filename: "a.cc", directory: "/private/tmp")
 !52 = !{i32 1, !"Debug Info Version", i32 3}
diff --git a/test/DebugInfo/X86/dbg-declare.ll b/test/DebugInfo/X86/dbg-declare.ll
index b0c92003d96..a1420a0ba2e 100644
--- a/test/DebugInfo/X86/dbg-declare.ll
+++ b/test/DebugInfo/X86/dbg-declare.ll
@@ -7,14 +7,14 @@ entry:
   %saved_stack = alloca i8*
   %cleanup.dest.slot = alloca i32
   store i32* %x, i32** %x.addr, align 8
-  call void @llvm.dbg.declare(metadata i32** %x.addr, metadata !14, metadata !MDExpression()), !dbg !15
+  call void @llvm.dbg.declare(metadata i32** %x.addr, metadata !14, metadata !DIExpression()), !dbg !15
   %0 = load i32*, i32** %x.addr, align 8, !dbg !16
   %1 = load i32, i32* %0, align 4, !dbg !16
   %2 = zext i32 %1 to i64, !dbg !16
   %3 = call i8* @llvm.stacksave(), !dbg !16
   store i8* %3, i8** %saved_stack, !dbg !16
   %vla = alloca i8, i64 %2, align 16, !dbg !16
-  call void @llvm.dbg.declare(metadata i8* %vla, metadata !18, metadata !MDExpression()), !dbg !23
+  call void @llvm.dbg.declare(metadata i8* %vla, metadata !18, metadata !DIExpression()), !dbg !23
   store i32 1, i32* %cleanup.dest.slot
   %4 = load i8*, i8** %saved_stack, !dbg !24
   call void @llvm.stackrestore(i8* %4), !dbg !24
@@ -30,27 +30,27 @@ declare void @llvm.stackrestore(i8*) nounwind
 !llvm.dbg.cu = !{!0}
 !llvm.module.flags = !{!27}
 
-!0 = !MDCompileUnit(language: DW_LANG_C99, producer: "clang version 3.1 (trunk 153698)", isOptimized: false, emissionKind: 0, file: !26, enums: !1, retainedTypes: !1, subprograms: !3, globals: !1)
+!0 = !DICompileUnit(language: DW_LANG_C99, producer: "clang version 3.1 (trunk 153698)", isOptimized: false, emissionKind: 0, file: !26, enums: !1, retainedTypes: !1, subprograms: !3, globals: !1)
 !1 = !{}
 !3 = !{!5}
-!5 = !MDSubprogram(name: "foo", line: 6, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, file: !26, scope: !0, type: !7, function: i32 (i32*)* @foo)
-!6 = !MDFile(filename: "20020104-2.c", directory: "/Volumes/Sandbox/llvm")
-!7 = !MDSubroutineType(types: !8)
+!5 = !DISubprogram(name: "foo", line: 6, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, file: !26, scope: !0, type: !7, function: i32 (i32*)* @foo)
+!6 = !DIFile(filename: "20020104-2.c", directory: "/Volumes/Sandbox/llvm")
+!7 = !DISubroutineType(types: !8)
 !8 = !{!9, !10}
-!9 = !MDBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
-!10 = !MDDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, baseType: !11)
-!11 = !MDDerivedType(tag: DW_TAG_const_type, baseType: !9)
-!14 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "x", line: 5, arg: 1, scope: !5, file: !6, type: !10)
-!15 = !MDLocation(line: 5, column: 21, scope: !5)
-!16 = !MDLocation(line: 7, column: 13, scope: !17)
-!17 = distinct !MDLexicalBlock(line: 6, column: 1, file: !26, scope: !5)
-!18 = !MDLocalVariable(tag: DW_TAG_auto_variable, name: "a", line: 7, scope: !17, file: !6, type: !19)
-!19 = !MDCompositeType(tag: DW_TAG_array_type, align: 8, baseType: !20, elements: !21)
-!20 = !MDBasicType(tag: DW_TAG_base_type, name: "char", size: 8, align: 8, encoding: DW_ATE_signed_char)
+!9 = !DIBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
+!10 = !DIDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, baseType: !11)
+!11 = !DIDerivedType(tag: DW_TAG_const_type, baseType: !9)
+!14 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "x", line: 5, arg: 1, scope: !5, file: !6, type: !10)
+!15 = !DILocation(line: 5, column: 21, scope: !5)
+!16 = !DILocation(line: 7, column: 13, scope: !17)
+!17 = distinct !DILexicalBlock(line: 6, column: 1, file: !26, scope: !5)
+!18 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "a", line: 7, scope: !17, file: !6, type: !19)
+!19 = !DICompositeType(tag: DW_TAG_array_type, align: 8, baseType: !20, elements: !21)
+!20 = !DIBasicType(tag: DW_TAG_base_type, name: "char", size: 8, align: 8, encoding: DW_ATE_signed_char)
 !21 = !{!22}
-!22 = !MDSubrange(count: -1)
-!23 = !MDLocation(line: 7, column: 8, scope: !17)
-!24 = !MDLocation(line: 9, column: 1, scope: !17)
-!25 = !MDLocation(line: 8, column: 3, scope: !17)
-!26 = !MDFile(filename: "20020104-2.c", directory: "/Volumes/Sandbox/llvm")
+!22 = !DISubrange(count: -1)
+!23 = !DILocation(line: 7, column: 8, scope: !17)
+!24 = !DILocation(line: 9, column: 1, scope: !17)
+!25 = !DILocation(line: 8, column: 3, scope: !17)
+!26 = !DIFile(filename: "20020104-2.c", directory: "/Volumes/Sandbox/llvm")
 !27 = !{i32 1, !"Debug Info Version", i32 3}
diff --git a/test/DebugInfo/X86/dbg-file-name.ll b/test/DebugInfo/X86/dbg-file-name.ll
index 7ded7c4aa72..cc18c0804db 100644
--- a/test/DebugInfo/X86/dbg-file-name.ll
+++ b/test/DebugInfo/X86/dbg-file-name.ll
@@ -12,13 +12,13 @@ define i32 @main() nounwind {
 !llvm.dbg.cu = !{!2}
 !llvm.module.flags = !{!12}
 
-!1 = !MDFile(filename: "simple.c", directory: "/Users/manav/one/two")
-!2 = !MDCompileUnit(language: DW_LANG_C89, producer: "LLVM build 00", isOptimized: true, emissionKind: 0, file: !10, enums: !11, retainedTypes: !11, subprograms: !9)
-!5 = !MDBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
-!6 = !MDSubprogram(name: "main", linkageName: "main", line: 9, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, file: !10, scope: !1, type: !7, function: i32 ()* @main)
-!7 = !MDSubroutineType(types: !8)
+!1 = !DIFile(filename: "simple.c", directory: "/Users/manav/one/two")
+!2 = !DICompileUnit(language: DW_LANG_C89, producer: "LLVM build 00", isOptimized: true, emissionKind: 0, file: !10, enums: !11, retainedTypes: !11, subprograms: !9)
+!5 = !DIBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
+!6 = !DISubprogram(name: "main", linkageName: "main", line: 9, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, file: !10, scope: !1, type: !7, function: i32 ()* @main)
+!7 = !DISubroutineType(types: !8)
 !8 = !{!5}
 !9 = !{!6}
-!10 = !MDFile(filename: "simple.c", directory: "/Users/manav/one/two")
+!10 = !DIFile(filename: "simple.c", directory: "/Users/manav/one/two")
 !11 = !{}
 !12 = !{i32 1, !"Debug Info Version", i32 3}
diff --git a/test/DebugInfo/X86/dbg-i128-const.ll b/test/DebugInfo/X86/dbg-i128-const.ll
index 1b89188f345..de804faabf4 100644
--- a/test/DebugInfo/X86/dbg-i128-const.ll
+++ b/test/DebugInfo/X86/dbg-i128-const.ll
@@ -6,7 +6,7 @@
 
 define i128 @__foo(i128 %a, i128 %b) nounwind {
 entry:
-  tail call void @llvm.dbg.value(metadata i128 42 , i64 0, metadata !1, metadata !MDExpression()), !dbg !11
+  tail call void @llvm.dbg.value(metadata i128 42 , i64 0, metadata !1, metadata !DIExpression()), !dbg !11
   %add = add i128 %a, %b, !dbg !11
   ret i128 %add, !dbg !11
 }
@@ -17,19 +17,19 @@ declare void @llvm.dbg.value(metadata, i64, metadata, metadata) nounwind readnon
 !llvm.module.flags = !{!16}
 
 !0 = !{i128 42 }
-!1 = !MDLocalVariable(tag: DW_TAG_auto_variable, name: "MAX", line: 29, scope: !2, file: !4, type: !8)
-!2 = distinct !MDLexicalBlock(line: 26, column: 0, file: !13, scope: !3)
-!3 = !MDSubprogram(name: "__foo", linkageName: "__foo", line: 26, isLocal: false, isDefinition: true, virtualIndex: 6, isOptimized: false, scopeLine: 26, file: !13, scope: !4, type: !6, function: i128 (i128, i128)* @__foo)
-!4 = !MDFile(filename: "foo.c", directory: "/tmp")
-!5 = !MDCompileUnit(language: DW_LANG_C89, producer: "clang", isOptimized: true, emissionKind: 0, file: !13, enums: !15, retainedTypes: !15, subprograms: !12, imports:  null)
-!6 = !MDSubroutineType(types: !7)
+!1 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "MAX", line: 29, scope: !2, file: !4, type: !8)
+!2 = distinct !DILexicalBlock(line: 26, column: 0, file: !13, scope: !3)
+!3 = !DISubprogram(name: "__foo", linkageName: "__foo", line: 26, isLocal: false, isDefinition: true, virtualIndex: 6, isOptimized: false, scopeLine: 26, file: !13, scope: !4, type: !6, function: i128 (i128, i128)* @__foo)
+!4 = !DIFile(filename: "foo.c", directory: "/tmp")
+!5 = !DICompileUnit(language: DW_LANG_C89, producer: "clang", isOptimized: true, emissionKind: 0, file: !13, enums: !15, retainedTypes: !15, subprograms: !12, imports:  null)
+!6 = !DISubroutineType(types: !7)
 !7 = !{!8, !8, !8}
-!8 = !MDDerivedType(tag: DW_TAG_typedef, name: "ti_int", line: 78, file: !14, scope: !4, baseType: !10)
-!9 = !MDFile(filename: "myint.h", directory: "/tmp")
-!10 = !MDBasicType(tag: DW_TAG_base_type, size: 128, align: 128, encoding: DW_ATE_signed)
-!11 = !MDLocation(line: 29, scope: !2)
+!8 = !DIDerivedType(tag: DW_TAG_typedef, name: "ti_int", line: 78, file: !14, scope: !4, baseType: !10)
+!9 = !DIFile(filename: "myint.h", directory: "/tmp")
+!10 = !DIBasicType(tag: DW_TAG_base_type, size: 128, align: 128, encoding: DW_ATE_signed)
+!11 = !DILocation(line: 29, scope: !2)
 !12 = !{!3}
-!13 = !MDFile(filename: "foo.c", directory: "/tmp")
-!14 = !MDFile(filename: "myint.h", directory: "/tmp")
+!13 = !DIFile(filename: "foo.c", directory: "/tmp")
+!14 = !DIFile(filename: "myint.h", directory: "/tmp")
 !15 = !{}
 !16 = !{i32 1, !"Debug Info Version", i32 3}
diff --git a/test/DebugInfo/X86/dbg-merge-loc-entry.ll b/test/DebugInfo/X86/dbg-merge-loc-entry.ll
index b46fe2bcc3a..bbcdd59a5b0 100644
--- a/test/DebugInfo/X86/dbg-merge-loc-entry.ll
+++ b/test/DebugInfo/X86/dbg-merge-loc-entry.ll
@@ -14,8 +14,8 @@ target triple = "x86_64-apple-darwin8"
 
 define hidden i128 @__divti3(i128 %u, i128 %v) nounwind readnone {
 entry:
-  tail call void @llvm.dbg.value(metadata i128 %u, i64 0, metadata !14, metadata !MDExpression()), !dbg !15
-  tail call void @llvm.dbg.value(metadata i64 0, i64 0, metadata !17, metadata !MDExpression()), !dbg !21
+  tail call void @llvm.dbg.value(metadata i128 %u, i64 0, metadata !14, metadata !DIExpression()), !dbg !15
+  tail call void @llvm.dbg.value(metadata i64 0, i64 0, metadata !17, metadata !DIExpression()), !dbg !21
   br i1 undef, label %bb2, label %bb4, !dbg !22
 
 bb2:                                              ; preds = %entry
@@ -40,36 +40,36 @@ declare %0 @llvm.uadd.with.overflow.i64(i64, i64) nounwind readnone
 !llvm.dbg.cu = !{!2}
 !llvm.module.flags = !{!32}
 
-!0 = !MDSubprogram(name: "__udivmodti4", line: 879, isLocal: true, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: true, scopeLine: 879, file: !29, scope: !1, type: !3)
-!1 = !MDFile(filename: "foobar.c", directory: "/tmp")
-!2 = !MDCompileUnit(language: DW_LANG_C89, producer: "4.2.1 (Based on Apple Inc. build 5658) (LLVM build)", isOptimized: true, emissionKind: 0, file: !29, enums: !31, retainedTypes: !31, subprograms: !28, imports:  null)
-!3 = !MDSubroutineType(types: !4)
+!0 = !DISubprogram(name: "__udivmodti4", line: 879, isLocal: true, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: true, scopeLine: 879, file: !29, scope: !1, type: !3)
+!1 = !DIFile(filename: "foobar.c", directory: "/tmp")
+!2 = !DICompileUnit(language: DW_LANG_C89, producer: "4.2.1 (Based on Apple Inc. build 5658) (LLVM build)", isOptimized: true, emissionKind: 0, file: !29, enums: !31, retainedTypes: !31, subprograms: !28, imports:  null)
+!3 = !DISubroutineType(types: !4)
 !4 = !{!5, !5, !5, !8}
-!5 = !MDDerivedType(tag: DW_TAG_typedef, name: "UTItype", line: 166, file: !30, scope: !6, baseType: !7)
-!6 = !MDFile(filename: "foobar.h", directory: "/tmp")
-!7 = !MDBasicType(tag: DW_TAG_base_type, size: 128, align: 128, encoding: DW_ATE_unsigned)
-!8 = !MDDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, file: !29, scope: !1, baseType: !5)
-!9 = !MDSubprogram(name: "__divti3", linkageName: "__divti3", line: 1094, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: true, scopeLine: 1094, file: !29, scope: !1, type: !10, function: i128 (i128, i128)* @__divti3)
-!10 = !MDSubroutineType(types: !11)
+!5 = !DIDerivedType(tag: DW_TAG_typedef, name: "UTItype", line: 166, file: !30, scope: !6, baseType: !7)
+!6 = !DIFile(filename: "foobar.h", directory: "/tmp")
+!7 = !DIBasicType(tag: DW_TAG_base_type, size: 128, align: 128, encoding: DW_ATE_unsigned)
+!8 = !DIDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, file: !29, scope: !1, baseType: !5)
+!9 = !DISubprogram(name: "__divti3", linkageName: "__divti3", line: 1094, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: true, scopeLine: 1094, file: !29, scope: !1, type: !10, function: i128 (i128, i128)* @__divti3)
+!10 = !DISubroutineType(types: !11)
 !11 = !{!12, !12, !12}
-!12 = !MDDerivedType(tag: DW_TAG_typedef, name: "TItype", line: 160, file: !30, scope: !6, baseType: !13)
-!13 = !MDBasicType(tag: DW_TAG_base_type, size: 128, align: 128, encoding: DW_ATE_signed)
-!14 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "u", line: 1093, arg: 0, scope: !9, file: !1, type: !12)
-!15 = !MDLocation(line: 1093, scope: !9)
+!12 = !DIDerivedType(tag: DW_TAG_typedef, name: "TItype", line: 160, file: !30, scope: !6, baseType: !13)
+!13 = !DIBasicType(tag: DW_TAG_base_type, size: 128, align: 128, encoding: DW_ATE_signed)
+!14 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "u", line: 1093, arg: 1, scope: !9, file: !1, type: !12)
+!15 = !DILocation(line: 1093, scope: !9)
 !16 = !{i64 0}
-!17 = !MDLocalVariable(tag: DW_TAG_auto_variable, name: "c", line: 1095, scope: !18, file: !1, type: !19)
-!18 = distinct !MDLexicalBlock(line: 1094, column: 0, file: !29, scope: !9)
-!19 = !MDDerivedType(tag: DW_TAG_typedef, name: "word_type", line: 424, file: !30, scope: !6, baseType: !20)
-!20 = !MDBasicType(tag: DW_TAG_base_type, name: "long int", size: 64, align: 64, encoding: DW_ATE_signed)
-!21 = !MDLocation(line: 1095, scope: !18)
-!22 = !MDLocation(line: 1103, scope: !18)
-!23 = !MDLocation(line: 1104, scope: !18)
-!24 = !MDLocation(line: 1003, scope: !25, inlinedAt: !26)
-!25 = distinct !MDLexicalBlock(line: 879, column: 0, file: !29, scope: !0)
-!26 = !MDLocation(line: 1107, scope: !18)
-!27 = !MDLocation(line: 1111, scope: !18)
+!17 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "c", line: 1095, scope: !18, file: !1, type: !19)
+!18 = distinct !DILexicalBlock(line: 1094, column: 0, file: !29, scope: !9)
+!19 = !DIDerivedType(tag: DW_TAG_typedef, name: "word_type", line: 424, file: !30, scope: !6, baseType: !20)
+!20 = !DIBasicType(tag: DW_TAG_base_type, name: "long int", size: 64, align: 64, encoding: DW_ATE_signed)
+!21 = !DILocation(line: 1095, scope: !18)
+!22 = !DILocation(line: 1103, scope: !18)
+!23 = !DILocation(line: 1104, scope: !18)
+!24 = !DILocation(line: 1003, scope: !25, inlinedAt: !26)
+!25 = distinct !DILexicalBlock(line: 879, column: 0, file: !29, scope: !0)
+!26 = !DILocation(line: 1107, scope: !18)
+!27 = !DILocation(line: 1111, scope: !18)
 !28 = !{!0, !9}
-!29 = !MDFile(filename: "foobar.c", directory: "/tmp")
-!30 = !MDFile(filename: "foobar.h", directory: "/tmp")
+!29 = !DIFile(filename: "foobar.c", directory: "/tmp")
+!30 = !DIFile(filename: "foobar.h", directory: "/tmp")
 !31 = !{}
 !32 = !{i32 1, !"Debug Info Version", i32 3}
diff --git a/test/DebugInfo/X86/dbg-prolog-end.ll b/test/DebugInfo/X86/dbg-prolog-end.ll
index 54c243495c4..870b09120bc 100644
--- a/test/DebugInfo/X86/dbg-prolog-end.ll
+++ b/test/DebugInfo/X86/dbg-prolog-end.ll
@@ -8,8 +8,8 @@ entry:
   %i.addr = alloca i32, align 4
   %j = alloca i32, align 4
   store i32 %i, i32* %i.addr, align 4
-  call void @llvm.dbg.declare(metadata i32* %i.addr, metadata !7, metadata !MDExpression()), !dbg !8
-  call void @llvm.dbg.declare(metadata i32* %j, metadata !9, metadata !MDExpression()), !dbg !11
+  call void @llvm.dbg.declare(metadata i32* %i.addr, metadata !7, metadata !DIExpression()), !dbg !8
+  call void @llvm.dbg.declare(metadata i32* %j, metadata !9, metadata !DIExpression()), !dbg !11
   store i32 2, i32* %j, align 4, !dbg !12
   %tmp = load i32, i32* %j, align 4, !dbg !13
   %inc = add nsw i32 %tmp, 1, !dbg !13
@@ -36,24 +36,24 @@ entry:
 !llvm.module.flags = !{!21}
 !18 = !{!1, !6}
 
-!0 = !MDCompileUnit(language: DW_LANG_C99, producer: "clang version 3.0 (trunk 131100)", isOptimized: false, emissionKind: 0, file: !19, enums: !20, retainedTypes: !20, subprograms: !18, imports:  null)
-!1 = !MDSubprogram(name: "foo", line: 1, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 1, file: !19, scope: !2, type: !3, function: i32 (i32)* @foo)
-!2 = !MDFile(filename: "/tmp/a.c", directory: "/private/tmp")
-!3 = !MDSubroutineType(types: !4)
+!0 = !DICompileUnit(language: DW_LANG_C99, producer: "clang version 3.0 (trunk 131100)", isOptimized: false, emissionKind: 0, file: !19, enums: !20, retainedTypes: !20, subprograms: !18, imports:  null)
+!1 = !DISubprogram(name: "foo", line: 1, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 1, file: !19, scope: !2, type: !3, function: i32 (i32)* @foo)
+!2 = !DIFile(filename: "/tmp/a.c", directory: "/private/tmp")
+!3 = !DISubroutineType(types: !4)
 !4 = !{!5}
-!5 = !MDBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
-!6 = !MDSubprogram(name: "main", line: 7, isLocal: false, isDefinition: true, virtualIndex: 6, isOptimized: false, scopeLine: 7, file: !19, scope: !2, type: !3, function: i32 ()* @main)
-!7 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "i", line: 1, arg: 1, scope: !1, file: !2, type: !5)
-!8 = !MDLocation(line: 1, column: 13, scope: !1)
-!9 = !MDLocalVariable(tag: DW_TAG_auto_variable, name: "j", line: 2, scope: !10, file: !2, type: !5)
-!10 = distinct !MDLexicalBlock(line: 1, column: 16, file: !19, scope: !1)
-!11 = !MDLocation(line: 2, column: 6, scope: !10)
-!12 = !MDLocation(line: 2, column: 11, scope: !10)
-!13 = !MDLocation(line: 3, column: 2, scope: !10)
-!14 = !MDLocation(line: 4, column: 2, scope: !10)
-!15 = !MDLocation(line: 5, column: 2, scope: !10)
-!16 = !MDLocation(line: 8, column: 2, scope: !17)
-!17 = distinct !MDLexicalBlock(line: 7, column: 12, file: !19, scope: !6)
-!19 = !MDFile(filename: "/tmp/a.c", directory: "/private/tmp")
+!5 = !DIBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
+!6 = !DISubprogram(name: "main", line: 7, isLocal: false, isDefinition: true, virtualIndex: 6, isOptimized: false, scopeLine: 7, file: !19, scope: !2, type: !3, function: i32 ()* @main)
+!7 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "i", line: 1, arg: 1, scope: !1, file: !2, type: !5)
+!8 = !DILocation(line: 1, column: 13, scope: !1)
+!9 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "j", line: 2, scope: !10, file: !2, type: !5)
+!10 = distinct !DILexicalBlock(line: 1, column: 16, file: !19, scope: !1)
+!11 = !DILocation(line: 2, column: 6, scope: !10)
+!12 = !DILocation(line: 2, column: 11, scope: !10)
+!13 = !DILocation(line: 3, column: 2, scope: !10)
+!14 = !DILocation(line: 4, column: 2, scope: !10)
+!15 = !DILocation(line: 5, column: 2, scope: !10)
+!16 = !DILocation(line: 8, column: 2, scope: !17)
+!17 = distinct !DILexicalBlock(line: 7, column: 12, file: !19, scope: !6)
+!19 = !DIFile(filename: "/tmp/a.c", directory: "/private/tmp")
 !20 = !{}
 !21 = !{i32 1, !"Debug Info Version", i32 3}
diff --git a/test/DebugInfo/X86/dbg-subrange.ll b/test/DebugInfo/X86/dbg-subrange.ll
index 873b2bc947a..e3bd07355cc 100644
--- a/test/DebugInfo/X86/dbg-subrange.ll
+++ b/test/DebugInfo/X86/dbg-subrange.ll
@@ -15,21 +15,21 @@ entry:
 !llvm.dbg.cu = !{!0}
 !llvm.module.flags = !{!22}
 
-!0 = !MDCompileUnit(language: DW_LANG_C99, producer: "clang version 3.1 (trunk 144833)", isOptimized: false, emissionKind: 0, file: !21, enums: !1, retainedTypes: !1, subprograms: !3, globals: !11, imports:  !1)
+!0 = !DICompileUnit(language: DW_LANG_C99, producer: "clang version 3.1 (trunk 144833)", isOptimized: false, emissionKind: 0, file: !21, enums: !1, retainedTypes: !1, subprograms: !3, globals: !11, imports:  !1)
 !1 = !{}
 !3 = !{!5}
-!5 = !MDSubprogram(name: "bar", line: 4, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, file: !21, scope: !6, type: !7, function: void ()* @bar)
-!6 = !MDFile(filename: "small.c", directory: "/private/tmp")
-!7 = !MDSubroutineType(types: !8)
+!5 = !DISubprogram(name: "bar", line: 4, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, file: !21, scope: !6, type: !7, function: void ()* @bar)
+!6 = !DIFile(filename: "small.c", directory: "/private/tmp")
+!7 = !DISubroutineType(types: !8)
 !8 = !{null}
 !11 = !{!13}
-!13 = !MDGlobalVariable(name: "s", line: 2, isLocal: false, isDefinition: true, scope: null, file: !6, type: !14, variable: [4294967296 x i8]* @s)
-!14 = !MDCompositeType(tag: DW_TAG_array_type, size: 34359738368, align: 8, baseType: !15, elements: !16)
-!15 = !MDBasicType(tag: DW_TAG_base_type, name: "char", size: 8, align: 8, encoding: DW_ATE_signed_char)
+!13 = !DIGlobalVariable(name: "s", line: 2, isLocal: false, isDefinition: true, scope: null, file: !6, type: !14, variable: [4294967296 x i8]* @s)
+!14 = !DICompositeType(tag: DW_TAG_array_type, size: 34359738368, align: 8, baseType: !15, elements: !16)
+!15 = !DIBasicType(tag: DW_TAG_base_type, name: "char", size: 8, align: 8, encoding: DW_ATE_signed_char)
 !16 = !{!17}
-!17 = !MDSubrange(count: 4294967296)
-!18 = !MDLocation(line: 5, column: 3, scope: !19)
-!19 = distinct !MDLexicalBlock(line: 4, column: 1, file: !21, scope: !5)
-!20 = !MDLocation(line: 6, column: 1, scope: !19)
-!21 = !MDFile(filename: "small.c", directory: "/private/tmp")
+!17 = !DISubrange(count: 4294967296)
+!18 = !DILocation(line: 5, column: 3, scope: !19)
+!19 = distinct !DILexicalBlock(line: 4, column: 1, file: !21, scope: !5)
+!20 = !DILocation(line: 6, column: 1, scope: !19)
+!21 = !DIFile(filename: "small.c", directory: "/private/tmp")
 !22 = !{i32 1, !"Debug Info Version", i32 3}
diff --git a/test/DebugInfo/X86/dbg-value-const-byref.ll b/test/DebugInfo/X86/dbg-value-const-byref.ll
index 766b72ebc41..a54ef0590e4 100644
--- a/test/DebugInfo/X86/dbg-value-const-byref.ll
+++ b/test/DebugInfo/X86/dbg-value-const-byref.ll
@@ -50,13 +50,13 @@ target triple = "x86_64-apple-macosx10.9.0"
 define i32 @foo() #0 {
 entry:
   %i = alloca i32, align 4
-  call void @llvm.dbg.value(metadata i32 3, i64 0, metadata !10, metadata !MDExpression()), !dbg !15
+  call void @llvm.dbg.value(metadata i32 3, i64 0, metadata !10, metadata !DIExpression()), !dbg !15
   %call = call i32 @f3(i32 3) #3, !dbg !16
-  call void @llvm.dbg.value(metadata i32 7, i64 0, metadata !10, metadata !MDExpression()), !dbg !18
+  call void @llvm.dbg.value(metadata i32 7, i64 0, metadata !10, metadata !DIExpression()), !dbg !18
   %call1 = call i32 (...) @f1() #3, !dbg !19
-  call void @llvm.dbg.value(metadata i32 %call1, i64 0, metadata !10, metadata !MDExpression()), !dbg !19
+  call void @llvm.dbg.value(metadata i32 %call1, i64 0, metadata !10, metadata !DIExpression()), !dbg !19
   store i32 %call1, i32* %i, align 4, !dbg !19, !tbaa !20
-  call void @llvm.dbg.value(metadata i32* %i, i64 0, metadata !10, metadata !MDExpression()), !dbg !24
+  call void @llvm.dbg.value(metadata i32* %i, i64 0, metadata !10, metadata !DIExpression()), !dbg !24
   call void @f2(i32* %i) #3, !dbg !24
   ret i32 0, !dbg !25
 }
@@ -78,29 +78,29 @@ attributes #3 = { nounwind }
 !llvm.module.flags = !{!11, !12}
 !llvm.ident = !{!13}
 
-!0 = !MDCompileUnit(language: DW_LANG_C99, producer: "clang version 3.5.0 ", isOptimized: true, emissionKind: 1, file: !1, enums: !2, retainedTypes: !2, subprograms: !3, globals: !2, imports: !2)
-!1 = !MDFile(filename: "dbg-value-const-byref.c", directory: "")
+!0 = !DICompileUnit(language: DW_LANG_C99, producer: "clang version 3.5.0 ", isOptimized: true, emissionKind: 1, file: !1, enums: !2, retainedTypes: !2, subprograms: !3, globals: !2, imports: !2)
+!1 = !DIFile(filename: "dbg-value-const-byref.c", directory: "")
 !2 = !{}
 !3 = !{!4}
-!4 = !MDSubprogram(name: "foo", line: 5, isLocal: false, isDefinition: true, virtualIndex: 6, isOptimized: true, scopeLine: 5, file: !1, scope: !5, type: !6, function: i32 ()* @foo, variables: !9)
-!5 = !MDFile(filename: "dbg-value-const-byref.c", directory: "")
-!6 = !MDSubroutineType(types: !7)
+!4 = !DISubprogram(name: "foo", line: 5, isLocal: false, isDefinition: true, virtualIndex: 6, isOptimized: true, scopeLine: 5, file: !1, scope: !5, type: !6, function: i32 ()* @foo, variables: !9)
+!5 = !DIFile(filename: "dbg-value-const-byref.c", directory: "")
+!6 = !DISubroutineType(types: !7)
 !7 = !{!8}
-!8 = !MDBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
+!8 = !DIBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
 !9 = !{!10}
-!10 = !MDLocalVariable(tag: DW_TAG_auto_variable, name: "i", line: 6, scope: !4, file: !5, type: !8)
+!10 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "i", line: 6, scope: !4, file: !5, type: !8)
 !11 = !{i32 2, !"Dwarf Version", i32 2}
 !12 = !{i32 1, !"Debug Info Version", i32 3}
 !13 = !{!"clang version 3.5.0 "}
 !14 = !{i32 3}
-!15 = !MDLocation(line: 6, scope: !4)
-!16 = !MDLocation(line: 7, scope: !4)
+!15 = !DILocation(line: 6, scope: !4)
+!16 = !DILocation(line: 7, scope: !4)
 !17 = !{i32 7}
-!18 = !MDLocation(line: 8, scope: !4)
-!19 = !MDLocation(line: 9, scope: !4)
+!18 = !DILocation(line: 8, scope: !4)
+!19 = !DILocation(line: 9, scope: !4)
 !20 = !{!21, !21, i64 0}
 !21 = !{!"int", !22, i64 0}
 !22 = !{!"omnipotent char", !23, i64 0}
 !23 = !{!"Simple C/C++ TBAA"}
-!24 = !MDLocation(line: 10, scope: !4)
-!25 = !MDLocation(line: 11, scope: !4)
+!24 = !DILocation(line: 10, scope: !4)
+!25 = !DILocation(line: 11, scope: !4)
diff --git a/test/DebugInfo/X86/dbg-value-dag-combine.ll b/test/DebugInfo/X86/dbg-value-dag-combine.ll
index 7370e8d978d..e73d161ecb7 100644
--- a/test/DebugInfo/X86/dbg-value-dag-combine.ll
+++ b/test/DebugInfo/X86/dbg-value-dag-combine.ll
@@ -8,15 +8,15 @@ declare <4 x i32> @__amdil_get_global_id_int()
 declare void @llvm.dbg.value(metadata, i64, metadata, metadata)
 define void @__OpenCL_test_kernel(i32 addrspace(1)* %ip) nounwind {
 entry:
-  call void @llvm.dbg.value(metadata i32 addrspace(1)* %ip, i64 0, metadata !7, metadata !MDExpression()), !dbg !8
+  call void @llvm.dbg.value(metadata i32 addrspace(1)* %ip, i64 0, metadata !7, metadata !DIExpression()), !dbg !8
   %0 = call <4 x i32> @__amdil_get_global_id_int() nounwind
   %1 = extractelement <4 x i32> %0, i32 0
-  call void @llvm.dbg.value(metadata i32 %1, i64 0, metadata !9, metadata !MDExpression()), !dbg !11
-  call void @llvm.dbg.value(metadata i32 0, i64 0, metadata !13, metadata !MDExpression()), !dbg !14
+  call void @llvm.dbg.value(metadata i32 %1, i64 0, metadata !9, metadata !DIExpression()), !dbg !11
+  call void @llvm.dbg.value(metadata i32 0, i64 0, metadata !13, metadata !DIExpression()), !dbg !14
   %tmp2 = load i32, i32 addrspace(1)* %ip, align 4, !dbg !15
   %tmp3 = add i32 0, %tmp2, !dbg !15
 ; CHECK:  ##DEBUG_VALUE: idx <- E{{..$}}
-  call void @llvm.dbg.value(metadata i32 %tmp3, i64 0, metadata !13, metadata !MDExpression()), !dbg !15
+  call void @llvm.dbg.value(metadata i32 %tmp3, i64 0, metadata !13, metadata !DIExpression()), !dbg !15
   %arrayidx = getelementptr i32, i32 addrspace(1)* %ip, i32 %1, !dbg !16
   store i32 %tmp3, i32 addrspace(1)* %arrayidx, align 4, !dbg !16
   ret void, !dbg !17
@@ -24,24 +24,24 @@ entry:
 !llvm.dbg.cu = !{!2}
 !llvm.module.flags = !{!20}
 
-!0 = !MDSubprogram(name: "__OpenCL_test_kernel", linkageName: "__OpenCL_test_kernel", line: 2, isLocal: false, isDefinition: true, virtualIndex: 6, isOptimized: false, file: !19, scope: !1, type: !3, function: void (i32 addrspace(1)*)* @__OpenCL_test_kernel)
-!1 = !MDFile(filename: "OCL6368.tmp.cl", directory: "E:CUsersCmvillmow.AMDCAppDataCLocalCTemp")
-!2 = !MDCompileUnit(language: DW_LANG_C89, producer: "clc", isOptimized: false, emissionKind: 1, file: !19, enums: !12, retainedTypes: !12, subprograms: !18, imports:  null)
-!3 = !MDSubroutineType(types: !4)
+!0 = !DISubprogram(name: "__OpenCL_test_kernel", linkageName: "__OpenCL_test_kernel", line: 2, isLocal: false, isDefinition: true, virtualIndex: 6, isOptimized: false, file: !19, scope: !1, type: !3, function: void (i32 addrspace(1)*)* @__OpenCL_test_kernel)
+!1 = !DIFile(filename: "OCL6368.tmp.cl", directory: "E:CUsersCmvillmow.AMDCAppDataCLocalCTemp")
+!2 = !DICompileUnit(language: DW_LANG_C89, producer: "clc", isOptimized: false, emissionKind: 1, file: !19, enums: !12, retainedTypes: !12, subprograms: !18, imports:  null)
+!3 = !DISubroutineType(types: !4)
 !4 = !{null, !5}
-!5 = !MDDerivedType(tag: DW_TAG_pointer_type, size: 32, align: 32, scope: !2, baseType: !6)
-!6 = !MDBasicType(tag: DW_TAG_base_type, name: "unsigned int", size: 32, align: 32, encoding: DW_ATE_unsigned)
-!7 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "ip", line: 1, arg: 0, scope: !0, file: !1, type: !5)
-!8 = !MDLocation(line: 1, column: 42, scope: !0)
-!9 = !MDLocalVariable(tag: DW_TAG_auto_variable, name: "gid", line: 3, scope: !10, file: !1, type: !6)
-!10 = distinct !MDLexicalBlock(line: 2, column: 1, file: !19, scope: !0)
-!11 = !MDLocation(line: 3, column: 41, scope: !10)
+!5 = !DIDerivedType(tag: DW_TAG_pointer_type, size: 32, align: 32, scope: !2, baseType: !6)
+!6 = !DIBasicType(tag: DW_TAG_base_type, name: "unsigned int", size: 32, align: 32, encoding: DW_ATE_unsigned)
+!7 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "ip", line: 1, arg: 1, scope: !0, file: !1, type: !5)
+!8 = !DILocation(line: 1, column: 42, scope: !0)
+!9 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "gid", line: 3, scope: !10, file: !1, type: !6)
+!10 = distinct !DILexicalBlock(line: 2, column: 1, file: !19, scope: !0)
+!11 = !DILocation(line: 3, column: 41, scope: !10)
 !12 = !{}
-!13 = !MDLocalVariable(tag: DW_TAG_auto_variable, name: "idx", line: 4, scope: !10, file: !1, type: !6)
-!14 = !MDLocation(line: 4, column: 20, scope: !10)
-!15 = !MDLocation(line: 5, column: 15, scope: !10)
-!16 = !MDLocation(line: 6, column: 18, scope: !10)
-!17 = !MDLocation(line: 7, column: 1, scope: !0)
+!13 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "idx", line: 4, scope: !10, file: !1, type: !6)
+!14 = !DILocation(line: 4, column: 20, scope: !10)
+!15 = !DILocation(line: 5, column: 15, scope: !10)
+!16 = !DILocation(line: 6, column: 18, scope: !10)
+!17 = !DILocation(line: 7, column: 1, scope: !0)
 !18 = !{!0}
-!19 = !MDFile(filename: "OCL6368.tmp.cl", directory: "E:\5CUsers\5Cmvillmow.AMD\5CAppData\5CLocal\5CTemp")
+!19 = !DIFile(filename: "OCL6368.tmp.cl", directory: "E:\5CUsers\5Cmvillmow.AMD\5CAppData\5CLocal\5CTemp")
 !20 = !{i32 1, !"Debug Info Version", i32 3}
diff --git a/test/DebugInfo/X86/dbg-value-inlined-parameter.ll b/test/DebugInfo/X86/dbg-value-inlined-parameter.ll
index df650b75016..3fa5cc5a98a 100644
--- a/test/DebugInfo/X86/dbg-value-inlined-parameter.ll
+++ b/test/DebugInfo/X86/dbg-value-inlined-parameter.ll
@@ -45,8 +45,8 @@
 
 define i32 @foo(%struct.S1* nocapture %sp, i32 %nums) nounwind optsize ssp {
 entry:
-  tail call void @llvm.dbg.value(metadata %struct.S1* %sp, i64 0, metadata !9, metadata !MDExpression()), !dbg !20
-  tail call void @llvm.dbg.value(metadata i32 %nums, i64 0, metadata !18, metadata !MDExpression()), !dbg !21
+  tail call void @llvm.dbg.value(metadata %struct.S1* %sp, i64 0, metadata !9, metadata !DIExpression()), !dbg !20
+  tail call void @llvm.dbg.value(metadata i32 %nums, i64 0, metadata !18, metadata !DIExpression()), !dbg !21
   %tmp2 = getelementptr inbounds %struct.S1, %struct.S1* %sp, i64 0, i32 1, !dbg !22
   store i32 %nums, i32* %tmp2, align 4, !dbg !22
   %call = tail call float* @bar(i32 %nums) nounwind optsize, !dbg !27
@@ -61,8 +61,8 @@ declare float* @bar(i32) optsize
 
 define void @foobar() nounwind optsize ssp {
 entry:
-  tail call void @llvm.dbg.value(metadata %struct.S1* @p, i64 0, metadata !9, metadata !MDExpression()) nounwind, !dbg !31
-  tail call void @llvm.dbg.value(metadata i32 1, i64 0, metadata !18, metadata !MDExpression()) nounwind, !dbg !35
+  tail call void @llvm.dbg.value(metadata %struct.S1* @p, i64 0, metadata !9, metadata !DIExpression()) nounwind, !dbg !31
+  tail call void @llvm.dbg.value(metadata i32 1, i64 0, metadata !18, metadata !DIExpression()) nounwind, !dbg !35
   store i32 1, i32* getelementptr inbounds (%struct.S1, %struct.S1* @p, i64 0, i32 1), align 8, !dbg !36
   %call.i = tail call float* @bar(i32 1) nounwind optsize, !dbg !37
   store float* %call.i, float** getelementptr inbounds (%struct.S1, %struct.S1* @p, i64 0, i32 0), align 8, !dbg !37
@@ -74,44 +74,44 @@ declare void @llvm.dbg.value(metadata, i64, metadata, metadata) nounwind readnon
 !llvm.dbg.cu = !{!2}
 !llvm.module.flags = !{!43}
 
-!0 = !MDSubprogram(name: "foo", line: 8, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: true, scopeLine: 8, file: !1, scope: !1, type: !3, function: i32 (%struct.S1*, i32)* @foo, variables: !41)
-!1 = !MDFile(filename: "nm2.c", directory: "/private/tmp")
-!2 = !MDCompileUnit(language: DW_LANG_C99, producer: "clang version 2.9 (trunk 125693)", isOptimized: true, emissionKind: 1, file: !42, enums: !{}, retainedTypes: !{}, subprograms: !39, globals: !40, imports:  !44)
-!3 = !MDSubroutineType(types: !4)
+!0 = !DISubprogram(name: "foo", line: 8, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: true, scopeLine: 8, file: !1, scope: !1, type: !3, function: i32 (%struct.S1*, i32)* @foo, variables: !41)
+!1 = !DIFile(filename: "nm2.c", directory: "/private/tmp")
+!2 = !DICompileUnit(language: DW_LANG_C99, producer: "clang version 2.9 (trunk 125693)", isOptimized: true, emissionKind: 1, file: !42, enums: !{}, retainedTypes: !{}, subprograms: !39, globals: !40, imports:  !44)
+!3 = !DISubroutineType(types: !4)
 !4 = !{!5}
-!5 = !MDBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
-!6 = !MDSubprogram(name: "foobar", line: 15, isLocal: false, isDefinition: true, virtualIndex: 6, isOptimized: true, file: !1, scope: !1, type: !7, function: void ()* @foobar)
-!7 = !MDSubroutineType(types: !8)
+!5 = !DIBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
+!6 = !DISubprogram(name: "foobar", line: 15, isLocal: false, isDefinition: true, virtualIndex: 6, isOptimized: true, file: !1, scope: !1, type: !7, function: void ()* @foobar)
+!7 = !DISubroutineType(types: !8)
 !8 = !{null}
-!9 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "sp", line: 7, arg: 1, scope: !0, file: !1, type: !10)
-!10 = !MDDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, scope: !2, baseType: !11)
-!11 = !MDDerivedType(tag: DW_TAG_typedef, name: "S1", line: 4, file: !42, scope: !2, baseType: !12)
-!12 = !MDCompositeType(tag: DW_TAG_structure_type, name: "S1", line: 1, size: 128, align: 64, file: !42, scope: !2, elements: !13)
+!9 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "sp", line: 7, arg: 1, scope: !0, file: !1, type: !10)
+!10 = !DIDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, scope: !2, baseType: !11)
+!11 = !DIDerivedType(tag: DW_TAG_typedef, name: "S1", line: 4, file: !42, scope: !2, baseType: !12)
+!12 = !DICompositeType(tag: DW_TAG_structure_type, name: "S1", line: 1, size: 128, align: 64, file: !42, scope: !2, elements: !13)
 !13 = !{!14, !17}
-!14 = !MDDerivedType(tag: DW_TAG_member, name: "m", line: 2, size: 64, align: 64, file: !42, scope: !1, baseType: !15)
-!15 = !MDDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, scope: !2, baseType: !16)
-!16 = !MDBasicType(tag: DW_TAG_base_type, name: "float", size: 32, align: 32, encoding: DW_ATE_float)
-!17 = !MDDerivedType(tag: DW_TAG_member, name: "nums", line: 3, size: 32, align: 32, offset: 64, file: !42, scope: !1, baseType: !5)
-!18 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "nums", line: 7, arg: 2, scope: !0, file: !1, type: !5)
-!19 = !MDGlobalVariable(name: "p", line: 14, isLocal: false, isDefinition: true, scope: !2, file: !1, type: !11, variable: %struct.S1* @p)
-!20 = !MDLocation(line: 7, column: 13, scope: !0)
-!21 = !MDLocation(line: 7, column: 21, scope: !0)
-!22 = !MDLocation(line: 9, column: 3, scope: !23)
-!23 = distinct !MDLexicalBlock(line: 8, column: 1, file: !1, scope: !0)
-!27 = !MDLocation(line: 10, column: 3, scope: !23)
-!29 = !MDLocation(line: 11, column: 3, scope: !23)
+!14 = !DIDerivedType(tag: DW_TAG_member, name: "m", line: 2, size: 64, align: 64, file: !42, scope: !1, baseType: !15)
+!15 = !DIDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, scope: !2, baseType: !16)
+!16 = !DIBasicType(tag: DW_TAG_base_type, name: "float", size: 32, align: 32, encoding: DW_ATE_float)
+!17 = !DIDerivedType(tag: DW_TAG_member, name: "nums", line: 3, size: 32, align: 32, offset: 64, file: !42, scope: !1, baseType: !5)
+!18 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "nums", line: 7, arg: 2, scope: !0, file: !1, type: !5)
+!19 = !DIGlobalVariable(name: "p", line: 14, isLocal: false, isDefinition: true, scope: !2, file: !1, type: !11, variable: %struct.S1* @p)
+!20 = !DILocation(line: 7, column: 13, scope: !0)
+!21 = !DILocation(line: 7, column: 21, scope: !0)
+!22 = !DILocation(line: 9, column: 3, scope: !23)
+!23 = distinct !DILexicalBlock(line: 8, column: 1, file: !1, scope: !0)
+!27 = !DILocation(line: 10, column: 3, scope: !23)
+!29 = !DILocation(line: 11, column: 3, scope: !23)
 !30 = !{%struct.S1* @p}
-!31 = !MDLocation(line: 7, column: 13, scope: !0, inlinedAt: !32)
-!32 = !MDLocation(line: 16, column: 3, scope: !33)
-!33 = distinct !MDLexicalBlock(line: 15, column: 15, file: !1, scope: !6)
+!31 = !DILocation(line: 7, column: 13, scope: !0, inlinedAt: !32)
+!32 = !DILocation(line: 16, column: 3, scope: !33)
+!33 = distinct !DILexicalBlock(line: 15, column: 15, file: !1, scope: !6)
 !34 = !{i32 1}
-!35 = !MDLocation(line: 7, column: 21, scope: !0, inlinedAt: !32)
-!36 = !MDLocation(line: 9, column: 3, scope: !23, inlinedAt: !32)
-!37 = !MDLocation(line: 10, column: 3, scope: !23, inlinedAt: !32)
-!38 = !MDLocation(line: 17, column: 1, scope: !33)
+!35 = !DILocation(line: 7, column: 21, scope: !0, inlinedAt: !32)
+!36 = !DILocation(line: 9, column: 3, scope: !23, inlinedAt: !32)
+!37 = !DILocation(line: 10, column: 3, scope: !23, inlinedAt: !32)
+!38 = !DILocation(line: 17, column: 1, scope: !33)
 !39 = !{!0, !6}
 !40 = !{!19}
 !41 = !{!9, !18}
-!42 = !MDFile(filename: "nm2.c", directory: "/private/tmp")
+!42 = !DIFile(filename: "nm2.c", directory: "/private/tmp")
 !43 = !{i32 1, !"Debug Info Version", i32 3}
 !44 = !{}
diff --git a/test/DebugInfo/X86/dbg-value-isel.ll b/test/DebugInfo/X86/dbg-value-isel.ll
index 823cacdf82b..9d5a8b76ac4 100644
--- a/test/DebugInfo/X86/dbg-value-isel.ll
+++ b/test/DebugInfo/X86/dbg-value-isel.ll
@@ -13,7 +13,7 @@ target triple = "x86_64-apple-darwin10.0.0"
 
 define void @__OpenCL_nbt02_kernel(i32 addrspace(1)* %ip) nounwind {
 entry:
-  call void @llvm.dbg.value(metadata i32 addrspace(1)* %ip, i64 0, metadata !8, metadata !MDExpression()), !dbg !9
+  call void @llvm.dbg.value(metadata i32 addrspace(1)* %ip, i64 0, metadata !8, metadata !DIExpression()), !dbg !9
   %0 = call <4 x i32> @__amdil_get_local_id_int() nounwind
   %1 = extractelement <4 x i32> %0, i32 0
   br label %2
@@ -28,7 +28,7 @@ entry:
 
 get_local_id.exit:                                ; preds = %4
   %6 = phi i32 [ %5, %4 ]
-  call void @llvm.dbg.value(metadata i32 %6, i64 0, metadata !10, metadata !MDExpression()), !dbg !12
+  call void @llvm.dbg.value(metadata i32 %6, i64 0, metadata !10, metadata !DIExpression()), !dbg !12
   %7 = call <4 x i32> @__amdil_get_global_id_int() nounwind, !dbg !12
   %8 = extractelement <4 x i32> %7, i32 0, !dbg !12
   br label %9
@@ -43,7 +43,7 @@ get_local_id.exit:                                ; preds = %4
 
 get_global_id.exit:                               ; preds = %11
   %13 = phi i32 [ %12, %11 ]
-  call void @llvm.dbg.value(metadata i32 %13, i64 0, metadata !13, metadata !MDExpression()), !dbg !14
+  call void @llvm.dbg.value(metadata i32 %13, i64 0, metadata !13, metadata !DIExpression()), !dbg !14
   %14 = call <4 x i32> @__amdil_get_local_size_int() nounwind
   %15 = extractelement <4 x i32> %14, i32 0
   br label %16
@@ -58,7 +58,7 @@ get_global_id.exit:                               ; preds = %11
 
 get_local_size.exit:                              ; preds = %18
   %20 = phi i32 [ %19, %18 ]
-  call void @llvm.dbg.value(metadata i32 %20, i64 0, metadata !15, metadata !MDExpression()), !dbg !16
+  call void @llvm.dbg.value(metadata i32 %20, i64 0, metadata !15, metadata !DIExpression()), !dbg !16
   %tmp5 = add i32 %6, %13, !dbg !17
   %tmp7 = add i32 %tmp5, %20, !dbg !17
   store i32 %tmp7, i32 addrspace(1)* %ip, align 4, !dbg !17
@@ -81,26 +81,26 @@ declare void @llvm.dbg.value(metadata, i64, metadata, metadata) nounwind readnon
 !llvm.dbg.cu = !{!2}
 !llvm.module.flags = !{!22}
 
-!0 = !MDSubprogram(name: "__OpenCL_nbt02_kernel", linkageName: "__OpenCL_nbt02_kernel", line: 2, isLocal: false, isDefinition: true, virtualIndex: 6, isOptimized: false, file: !20, scope: !1, type: !3, function: void (i32 addrspace(1)*)* @__OpenCL_nbt02_kernel)
-!1 = !MDFile(filename: "OCLlLwTXZ.cl", directory: "/tmp")
-!2 = !MDCompileUnit(language: DW_LANG_C89, producer: "clc", isOptimized: false, emissionKind: 1, file: !20, enums: !21, retainedTypes: !21, subprograms: !19, imports:  null)
-!3 = !MDSubroutineType(types: !4)
+!0 = !DISubprogram(name: "__OpenCL_nbt02_kernel", linkageName: "__OpenCL_nbt02_kernel", line: 2, isLocal: false, isDefinition: true, virtualIndex: 6, isOptimized: false, file: !20, scope: !1, type: !3, function: void (i32 addrspace(1)*)* @__OpenCL_nbt02_kernel)
+!1 = !DIFile(filename: "OCLlLwTXZ.cl", directory: "/tmp")
+!2 = !DICompileUnit(language: DW_LANG_C89, producer: "clc", isOptimized: false, emissionKind: 1, file: !20, enums: !21, retainedTypes: !21, subprograms: !19, imports:  null)
+!3 = !DISubroutineType(types: !4)
 !4 = !{null, !5}
-!5 = !MDDerivedType(tag: DW_TAG_pointer_type, size: 32, align: 32, scope: !2, baseType: !6)
-!6 = !MDDerivedType(tag: DW_TAG_typedef, name: "uint", file: !20, scope: !2, baseType: !7)
-!7 = !MDBasicType(tag: DW_TAG_base_type, name: "unsigned int", size: 32, align: 32, encoding: DW_ATE_unsigned)
-!8 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "ip", line: 1, arg: 0, scope: !0, file: !1, type: !5)
-!9 = !MDLocation(line: 1, column: 32, scope: !0)
-!10 = !MDLocalVariable(tag: DW_TAG_auto_variable, name: "tid", line: 3, scope: !11, file: !1, type: !6)
-!11 = distinct !MDLexicalBlock(line: 2, column: 1, file: !1, scope: !0)
-!12 = !MDLocation(line: 5, column: 24, scope: !11)
-!13 = !MDLocalVariable(tag: DW_TAG_auto_variable, name: "gid", line: 3, scope: !11, file: !1, type: !6)
-!14 = !MDLocation(line: 6, column: 25, scope: !11)
-!15 = !MDLocalVariable(tag: DW_TAG_auto_variable, name: "lsz", line: 3, scope: !11, file: !1, type: !6)
-!16 = !MDLocation(line: 7, column: 26, scope: !11)
-!17 = !MDLocation(line: 9, column: 24, scope: !11)
-!18 = !MDLocation(line: 10, column: 1, scope: !0)
+!5 = !DIDerivedType(tag: DW_TAG_pointer_type, size: 32, align: 32, scope: !2, baseType: !6)
+!6 = !DIDerivedType(tag: DW_TAG_typedef, name: "uint", file: !20, scope: !2, baseType: !7)
+!7 = !DIBasicType(tag: DW_TAG_base_type, name: "unsigned int", size: 32, align: 32, encoding: DW_ATE_unsigned)
+!8 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "ip", line: 1, arg: 1, scope: !0, file: !1, type: !5)
+!9 = !DILocation(line: 1, column: 32, scope: !0)
+!10 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "tid", line: 3, scope: !11, file: !1, type: !6)
+!11 = distinct !DILexicalBlock(line: 2, column: 1, file: !1, scope: !0)
+!12 = !DILocation(line: 5, column: 24, scope: !11)
+!13 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "gid", line: 3, scope: !11, file: !1, type: !6)
+!14 = !DILocation(line: 6, column: 25, scope: !11)
+!15 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "lsz", line: 3, scope: !11, file: !1, type: !6)
+!16 = !DILocation(line: 7, column: 26, scope: !11)
+!17 = !DILocation(line: 9, column: 24, scope: !11)
+!18 = !DILocation(line: 10, column: 1, scope: !0)
 !19 = !{!0}
-!20 = !MDFile(filename: "OCLlLwTXZ.cl", directory: "/tmp")
+!20 = !DIFile(filename: "OCLlLwTXZ.cl", directory: "/tmp")
 !21 = !{}
 !22 = !{i32 1, !"Debug Info Version", i32 3}
diff --git a/test/DebugInfo/X86/dbg-value-location.ll b/test/DebugInfo/X86/dbg-value-location.ll
index 2a8a91c616d..631c05e1844 100644
--- a/test/DebugInfo/X86/dbg-value-location.ll
+++ b/test/DebugInfo/X86/dbg-value-location.ll
@@ -18,7 +18,7 @@ declare void @llvm.dbg.declare(metadata, metadata, metadata) nounwind readnone
 
 define i32 @foo(i32 %dev, i64 %cmd, i8* %data, i32 %data2) nounwind optsize ssp {
 entry:
-  call void @llvm.dbg.value(metadata i32 %dev, i64 0, metadata !12, metadata !MDExpression()), !dbg !13
+  call void @llvm.dbg.value(metadata i32 %dev, i64 0, metadata !12, metadata !DIExpression()), !dbg !13
   %tmp.i = load i32, i32* @dfm, align 4, !dbg !14
   %cmp.i = icmp eq i32 %tmp.i, 0, !dbg !14
   br i1 %cmp.i, label %if.else, label %if.end.i, !dbg !14
@@ -50,30 +50,30 @@ declare void @llvm.dbg.value(metadata, i64, metadata, metadata) nounwind readnon
 !llvm.dbg.cu = !{!2}
 !llvm.module.flags = !{!29}
 
-!0 = !MDSubprogram(name: "foo", line: 19510, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: true, scopeLine: 19510, file: !26, scope: !1, type: !3, function: i32 (i32, i64, i8*, i32)* @foo)
-!1 = !MDFile(filename: "/tmp/f.c", directory: "/tmp")
-!2 = !MDCompileUnit(language: DW_LANG_C99, producer: "clang version 2.9 (trunk 124753)", isOptimized: true, emissionKind: 0, file: !27, enums: !28, retainedTypes: !28, subprograms: !24, imports:  null)
-!3 = !MDSubroutineType(types: !4)
+!0 = !DISubprogram(name: "foo", line: 19510, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: true, scopeLine: 19510, file: !26, scope: !1, type: !3, function: i32 (i32, i64, i8*, i32)* @foo)
+!1 = !DIFile(filename: "/tmp/f.c", directory: "/tmp")
+!2 = !DICompileUnit(language: DW_LANG_C99, producer: "clang version 2.9 (trunk 124753)", isOptimized: true, emissionKind: 0, file: !27, enums: !28, retainedTypes: !28, subprograms: !24, imports:  null)
+!3 = !DISubroutineType(types: !4)
 !4 = !{!5}
-!5 = !MDBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
-!6 = !MDSubprogram(name: "bar3", line: 14827, isLocal: true, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: true, file: !26, scope: !1, type: !3, function: i32 (i32)* @bar3)
-!7 = !MDSubprogram(name: "bar2", line: 15397, isLocal: true, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: true, file: !26, scope: !1, type: !3, function: i32 (i32)* @bar2)
-!8 = !MDSubprogram(name: "bar", line: 12382, isLocal: true, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: true, file: !26, scope: !1, type: !9, function: i32 (i32, i32*)* @bar)
-!9 = !MDSubroutineType(types: !10)
+!5 = !DIBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
+!6 = !DISubprogram(name: "bar3", line: 14827, isLocal: true, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: true, file: !26, scope: !1, type: !3, function: i32 (i32)* @bar3)
+!7 = !DISubprogram(name: "bar2", line: 15397, isLocal: true, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: true, file: !26, scope: !1, type: !3, function: i32 (i32)* @bar2)
+!8 = !DISubprogram(name: "bar", line: 12382, isLocal: true, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: true, file: !26, scope: !1, type: !9, function: i32 (i32, i32*)* @bar)
+!9 = !DISubroutineType(types: !10)
 !10 = !{!11}
-!11 = !MDBasicType(tag: DW_TAG_base_type, name: "unsigned char", size: 8, align: 8, encoding: DW_ATE_unsigned_char)
-!12 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "var", line: 19509, arg: 0, scope: !0, file: !1, type: !5)
-!13 = !MDLocation(line: 19509, column: 20, scope: !0)
-!14 = !MDLocation(line: 18091, column: 2, scope: !15, inlinedAt: !17)
-!15 = distinct !MDLexicalBlock(line: 18086, column: 1, file: !26, scope: !16)
-!16 = !MDSubprogram(name: "foo_bar", line: 18086, isLocal: true, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: true, file: !26, scope: !1, type: !3)
-!17 = !MDLocation(line: 19514, column: 2, scope: !18)
-!18 = distinct !MDLexicalBlock(line: 19510, column: 1, file: !26, scope: !0)
-!22 = !MDLocation(line: 18094, column: 2, scope: !15, inlinedAt: !17)
-!23 = !MDLocation(line: 19524, column: 1, scope: !18)
+!11 = !DIBasicType(tag: DW_TAG_base_type, name: "unsigned char", size: 8, align: 8, encoding: DW_ATE_unsigned_char)
+!12 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "var", line: 19509, arg: 1, scope: !0, file: !1, type: !5)
+!13 = !DILocation(line: 19509, column: 20, scope: !0)
+!14 = !DILocation(line: 18091, column: 2, scope: !15, inlinedAt: !17)
+!15 = distinct !DILexicalBlock(line: 18086, column: 1, file: !26, scope: !16)
+!16 = !DISubprogram(name: "foo_bar", line: 18086, isLocal: true, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: true, file: !26, scope: !1, type: !3)
+!17 = !DILocation(line: 19514, column: 2, scope: !18)
+!18 = distinct !DILexicalBlock(line: 19510, column: 1, file: !26, scope: !0)
+!22 = !DILocation(line: 18094, column: 2, scope: !15, inlinedAt: !17)
+!23 = !DILocation(line: 19524, column: 1, scope: !18)
 !24 = !{!0, !6, !7, !8, !16}
-!25 = !MDFile(filename: "f.i", directory: "/tmp")
-!26 = !MDFile(filename: "/tmp/f.c", directory: "/tmp")
-!27 = !MDFile(filename: "f.i", directory: "/tmp")
+!25 = !DIFile(filename: "f.i", directory: "/tmp")
+!26 = !DIFile(filename: "/tmp/f.c", directory: "/tmp")
+!27 = !DIFile(filename: "f.i", directory: "/tmp")
 !28 = !{}
 !29 = !{i32 1, !"Debug Info Version", i32 3}
diff --git a/test/DebugInfo/X86/dbg-value-range.ll b/test/DebugInfo/X86/dbg-value-range.ll
index df1ab7bb145..b7d5d520add 100644
--- a/test/DebugInfo/X86/dbg-value-range.ll
+++ b/test/DebugInfo/X86/dbg-value-range.ll
@@ -4,10 +4,10 @@
 
 define i32 @bar(%struct.a* nocapture %b) nounwind ssp {
 entry:
-  tail call void @llvm.dbg.value(metadata %struct.a* %b, i64 0, metadata !6, metadata !MDExpression()), !dbg !13
+  tail call void @llvm.dbg.value(metadata %struct.a* %b, i64 0, metadata !6, metadata !DIExpression()), !dbg !13
   %tmp1 = getelementptr inbounds %struct.a, %struct.a* %b, i64 0, i32 0, !dbg !14
   %tmp2 = load i32, i32* %tmp1, align 4, !dbg !14
-  tail call void @llvm.dbg.value(metadata i32 %tmp2, i64 0, metadata !11, metadata !MDExpression()), !dbg !14
+  tail call void @llvm.dbg.value(metadata i32 %tmp2, i64 0, metadata !11, metadata !DIExpression()), !dbg !14
   %call = tail call i32 (...) @foo(i32 %tmp2) nounwind , !dbg !18
   %add = add nsw i32 %tmp2, 1, !dbg !19
   ret i32 %add, !dbg !19
@@ -20,26 +20,26 @@ declare void @llvm.dbg.value(metadata, i64, metadata, metadata) nounwind readnon
 !llvm.dbg.cu = !{!2}
 !llvm.module.flags = !{!24}
 
-!0 = !MDSubprogram(name: "bar", line: 5, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: true, file: !22, scope: !1, type: !3, function: i32 (%struct.a*)* @bar, variables: !21)
-!1 = !MDFile(filename: "bar.c", directory: "/private/tmp")
-!2 = !MDCompileUnit(language: DW_LANG_C99, producer: "clang version 2.9 (trunk 122997)", isOptimized: true, emissionKind: 1, file: !22, enums: !23, retainedTypes: !23, subprograms: !20, imports:  null)
-!3 = !MDSubroutineType(types: !4)
+!0 = !DISubprogram(name: "bar", line: 5, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: true, file: !22, scope: !1, type: !3, function: i32 (%struct.a*)* @bar, variables: !21)
+!1 = !DIFile(filename: "bar.c", directory: "/private/tmp")
+!2 = !DICompileUnit(language: DW_LANG_C99, producer: "clang version 2.9 (trunk 122997)", isOptimized: true, emissionKind: 1, file: !22, enums: !23, retainedTypes: !23, subprograms: !20, imports:  null)
+!3 = !DISubroutineType(types: !4)
 !4 = !{!5}
-!5 = !MDBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
-!6 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "b", line: 5, arg: 0, scope: !0, file: !1, type: !7)
-!7 = !MDDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, scope: !2, baseType: !8)
-!8 = !MDCompositeType(tag: DW_TAG_structure_type, name: "a", line: 1, size: 32, align: 32, file: !22, scope: !2, elements: !9)
+!5 = !DIBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
+!6 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "b", line: 5, arg: 1, scope: !0, file: !1, type: !7)
+!7 = !DIDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, scope: !2, baseType: !8)
+!8 = !DICompositeType(tag: DW_TAG_structure_type, name: "a", line: 1, size: 32, align: 32, file: !22, scope: !2, elements: !9)
 !9 = !{!10}
-!10 = !MDDerivedType(tag: DW_TAG_member, name: "c", line: 2, size: 32, align: 32, file: !22, scope: !1, baseType: !5)
-!11 = !MDLocalVariable(tag: DW_TAG_auto_variable, name: "x", line: 6, scope: !12, file: !1, type: !5)
-!12 = distinct !MDLexicalBlock(line: 5, column: 22, file: !22, scope: !0)
-!13 = !MDLocation(line: 5, column: 19, scope: !0)
-!14 = !MDLocation(line: 6, column: 14, scope: !12)
-!18 = !MDLocation(line: 7, column: 2, scope: !12)
-!19 = !MDLocation(line: 8, column: 2, scope: !12)
+!10 = !DIDerivedType(tag: DW_TAG_member, name: "c", line: 2, size: 32, align: 32, file: !22, scope: !1, baseType: !5)
+!11 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "x", line: 6, scope: !12, file: !1, type: !5)
+!12 = distinct !DILexicalBlock(line: 5, column: 22, file: !22, scope: !0)
+!13 = !DILocation(line: 5, column: 19, scope: !0)
+!14 = !DILocation(line: 6, column: 14, scope: !12)
+!18 = !DILocation(line: 7, column: 2, scope: !12)
+!19 = !DILocation(line: 8, column: 2, scope: !12)
 !20 = !{!0}
 !21 = !{!6, !11}
-!22 = !MDFile(filename: "bar.c", directory: "/private/tmp")
+!22 = !DIFile(filename: "bar.c", directory: "/private/tmp")
 !23 = !{}
 
 ; Check that variable bar:b value range is appropriately truncated in debug info.
@@ -55,11 +55,8 @@ declare void @llvm.dbg.value(metadata, i64, metadata, metadata) nounwind readnon
 ;CHECK-NEXT:	.quad
 ;CHECK-NEXT: [[CLOBBER_OFF:Lset.*]] = [[CLOBBER]]-{{.*}}
 ;CHECK-NEXT:	.quad	[[CLOBBER_OFF]]
-;CHECK-NEXT: Lset{{.*}} = Ltmp{{.*}}-Ltmp{{.*}}
-;CHECK-NEXT:    .short  Lset
-;CHECK-NEXT: Ltmp
+;CHECK-NEXT:  .short 1 ## Loc expr size
 ;CHECK-NEXT:	.byte	85 ## DW_OP_reg
-;CHECK-NEXT: Ltmp
 ;CHECK-NEXT:	.quad	0
 ;CHECK-NEXT:	.quad	0
 !24 = !{i32 1, !"Debug Info Version", i32 3}
diff --git a/test/DebugInfo/X86/dbg-value-terminator.ll b/test/DebugInfo/X86/dbg-value-terminator.ll
index 24841abc5a9..5f50b13aa02 100644
--- a/test/DebugInfo/X86/dbg-value-terminator.ll
+++ b/test/DebugInfo/X86/dbg-value-terminator.ll
@@ -87,7 +87,7 @@ VEC_edge_base_index.exit7.i:                      ; preds = %"3.i5.i"
 "44.i":                                           ; preds = %"42.i"
   %2 = load %a*, %a** undef, align 8, !dbg !12
   %3 = bitcast %a* %2 to %a*, !dbg !12
-  call void @llvm.dbg.value(metadata %a* %3, i64 0, metadata !6, metadata !MDExpression()), !dbg !12
+  call void @llvm.dbg.value(metadata %a* %3, i64 0, metadata !6, metadata !DIExpression()), !dbg !12
   br label %may_unswitch_on.exit, !dbg !12
 
 "45.i":                                           ; preds = %"38.i"
@@ -113,21 +113,21 @@ declare void @llvm.dbg.value(metadata, i64, metadata, metadata) nounwind readnon
 !llvm.dbg.cu = !{!0}
 !llvm.module.flags = !{!22}
 
-!0 = !MDCompileUnit(language: DW_LANG_C99, producer: "Apple clang version", isOptimized: true, emissionKind: 1, file: !20, enums: !21, retainedTypes: !21, subprograms: !18, imports:  null)
-!1 = !MDSubprogram(name: "foo", line: 2, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: true, file: !20, scope: !2, type: !3, function: %a* ()* @test, variables: !19)
-!2 = !MDFile(filename: "a.c", directory: "/private/tmp")
-!3 = !MDSubroutineType(types: !4)
+!0 = !DICompileUnit(language: DW_LANG_C99, producer: "Apple clang version", isOptimized: true, emissionKind: 1, file: !20, enums: !21, retainedTypes: !21, subprograms: !18, imports:  null)
+!1 = !DISubprogram(name: "foo", line: 2, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: true, file: !20, scope: !2, type: !3, function: %a* ()* @test, variables: !19)
+!2 = !DIFile(filename: "a.c", directory: "/private/tmp")
+!3 = !DISubroutineType(types: !4)
 !4 = !{!5}
-!5 = !MDBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
-!6 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "i", line: 2, arg: 1, scope: !1, file: !2, type: !5)
-!7 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "c", line: 2, arg: 2, scope: !1, file: !2, type: !8)
-!8 = !MDDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, scope: !0, baseType: !9)
-!9 = !MDBasicType(tag: DW_TAG_base_type, name: "char", size: 8, align: 8, encoding: DW_ATE_signed_char)
-!10 = !MDLocalVariable(tag: DW_TAG_auto_variable, name: "a", line: 3, scope: !11, file: !2, type: !9)
-!11 = distinct !MDLexicalBlock(line: 2, column: 25, file: !20, scope: !1)
-!12 = !MDLocation(line: 2, column: 13, scope: !1)
+!5 = !DIBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
+!6 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "i", line: 2, arg: 1, scope: !1, file: !2, type: !5)
+!7 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "c", line: 2, arg: 2, scope: !1, file: !2, type: !8)
+!8 = !DIDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, scope: !0, baseType: !9)
+!9 = !DIBasicType(tag: DW_TAG_base_type, name: "char", size: 8, align: 8, encoding: DW_ATE_signed_char)
+!10 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "a", line: 3, scope: !11, file: !2, type: !9)
+!11 = distinct !DILexicalBlock(line: 2, column: 25, file: !20, scope: !1)
+!12 = !DILocation(line: 2, column: 13, scope: !1)
 !18 = !{!1}
 !19 = !{!6, !7, !10}
-!20 = !MDFile(filename: "a.c", directory: "/private/tmp")
+!20 = !DIFile(filename: "a.c", directory: "/private/tmp")
 !21 = !{}
 !22 = !{i32 1, !"Debug Info Version", i32 3}
diff --git a/test/DebugInfo/X86/dbg_value_direct.ll b/test/DebugInfo/X86/dbg_value_direct.ll
index 22c5ce5920c..6446dfe491a 100644
--- a/test/DebugInfo/X86/dbg_value_direct.ll
+++ b/test/DebugInfo/X86/dbg_value_direct.ll
@@ -53,7 +53,7 @@ entry:
   %19 = inttoptr i64 %18 to i8*
   %20 = load i8, i8* %19
   %21 = icmp ne i8 %20, 0
-  call void @llvm.dbg.declare(metadata i32* %3, metadata !23, metadata !28), !dbg !MDLocation(scope: !4)
+  call void @llvm.dbg.declare(metadata i32* %3, metadata !23, metadata !28), !dbg !DILocation(scope: !4)
   br i1 %21, label %22, label %28
 
 ; <label>:22                                      ; preds = %entry
@@ -70,7 +70,7 @@ entry:
 
 ; <label>:28                                      ; preds = %22, %entry
   store i32 %0, i32* %3, align 4
-  call void @llvm.dbg.declare(metadata %struct.A* %agg.result, metadata !24, metadata !MDExpression(DW_OP_deref)), !dbg !25
+  call void @llvm.dbg.declare(metadata %struct.A* %agg.result, metadata !24, metadata !DIExpression(DW_OP_deref)), !dbg !25
   call void @_ZN1AC1Ev(%struct.A* %agg.result), !dbg !25
   store i64 1172321806, i64* %4, !dbg !26
   %29 = inttoptr i64 %10 to i32*, !dbg !26
@@ -147,30 +147,30 @@ attributes #2 = { "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "n
 !llvm.dbg.cu = !{!0}
 !llvm.module.flags = !{!22, !27}
 
-!0 = !MDCompileUnit(language: DW_LANG_C_plus_plus, producer: "clang version 3.4 ", isOptimized: false, emissionKind: 1, file: !1, enums: !2, retainedTypes: !2, subprograms: !3, globals: !2, imports: !2)
-!1 = !MDFile(filename: "crash.cpp", directory: "/tmp")
+!0 = !DICompileUnit(language: DW_LANG_C_plus_plus, producer: "clang version 3.4 ", isOptimized: false, emissionKind: 1, file: !1, enums: !2, retainedTypes: !2, subprograms: !3, globals: !2, imports: !2)
+!1 = !DIFile(filename: "crash.cpp", directory: "/tmp")
 !2 = !{}
 !3 = !{!4}
-!4 = !MDSubprogram(name: "func", linkageName: "_Z4funci", line: 6, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 6, file: !1, scope: !5, type: !6, function: void (%struct.A*, i32)* @_Z4funci, variables: !2)
-!5 = !MDFile(filename: "crash.cpp", directory: "/tmp")
-!6 = !MDSubroutineType(types: !7)
+!4 = !DISubprogram(name: "func", linkageName: "_Z4funci", line: 6, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 6, file: !1, scope: !5, type: !6, function: void (%struct.A*, i32)* @_Z4funci, variables: !2)
+!5 = !DIFile(filename: "crash.cpp", directory: "/tmp")
+!6 = !DISubroutineType(types: !7)
 !7 = !{!8, !21}
-!8 = !MDCompositeType(tag: DW_TAG_structure_type, name: "A", line: 1, size: 8, align: 8, file: !1, elements: !9)
+!8 = !DICompositeType(tag: DW_TAG_structure_type, name: "A", line: 1, size: 8, align: 8, file: !1, elements: !9)
 !9 = !{!10, !15}
-!10 = !MDSubprogram(name: "A", line: 2, isLocal: false, isDefinition: false, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 2, file: !1, scope: !8, type: !11)
-!11 = !MDSubroutineType(types: !12)
+!10 = !DISubprogram(name: "A", line: 2, isLocal: false, isDefinition: false, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 2, file: !1, scope: !8, type: !11)
+!11 = !DISubroutineType(types: !12)
 !12 = !{null, !13}
-!13 = !MDDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, flags: DIFlagArtificial | DIFlagObjectPointer, baseType: !8)
-!15 = !MDSubprogram(name: "A", line: 3, isLocal: false, isDefinition: false, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 3, file: !1, scope: !8, type: !16)
-!16 = !MDSubroutineType(types: !17)
+!13 = !DIDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, flags: DIFlagArtificial | DIFlagObjectPointer, baseType: !8)
+!15 = !DISubprogram(name: "A", line: 3, isLocal: false, isDefinition: false, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 3, file: !1, scope: !8, type: !16)
+!16 = !DISubroutineType(types: !17)
 !17 = !{null, !13, !18}
-!18 = !MDDerivedType(tag: DW_TAG_reference_type, baseType: !19)
-!19 = !MDDerivedType(tag: DW_TAG_const_type, baseType: !8)
-!21 = !MDBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
+!18 = !DIDerivedType(tag: DW_TAG_reference_type, baseType: !19)
+!19 = !DIDerivedType(tag: DW_TAG_const_type, baseType: !8)
+!21 = !DIBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
 !22 = !{i32 2, !"Dwarf Version", i32 3}
-!23 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "", line: 6, arg: 1, scope: !4, file: !5, type: !21)
-!24 = !MDLocalVariable(tag: DW_TAG_auto_variable, name: "a", line: 7, scope: !4, file: !5, type: !8)
-!25 = !MDLocation(line: 7, scope: !4)
-!26 = !MDLocation(line: 8, scope: !4)
+!23 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "", line: 6, arg: 1, scope: !4, file: !5, type: !21)
+!24 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "a", line: 7, scope: !4, file: !5, type: !8)
+!25 = !DILocation(line: 7, scope: !4)
+!26 = !DILocation(line: 8, scope: !4)
 !27 = !{i32 1, !"Debug Info Version", i32 3}
-!28 = !MDExpression(DW_OP_deref)
+!28 = !DIExpression(DW_OP_deref)
diff --git a/test/DebugInfo/X86/debug-dead-local-var.ll b/test/DebugInfo/X86/debug-dead-local-var.ll
index cbbe4c8153a..d56d17d06e6 100644
--- a/test/DebugInfo/X86/debug-dead-local-var.ll
+++ b/test/DebugInfo/X86/debug-dead-local-var.ll
@@ -27,25 +27,25 @@ attributes #0 = { nounwind readnone uwtable "less-precise-fpmad"="false" "no-fra
 !llvm.module.flags = !{!18, !19}
 !llvm.ident = !{!20}
 
-!0 = !MDCompileUnit(language: DW_LANG_C99, producer: "clang version 3.5.0 (trunk 209255) (llvm/trunk 209253)", isOptimized: true, emissionKind: 1, file: !1, enums: !2, retainedTypes: !2, subprograms: !3, globals: !2, imports: !2)
-!1 = !MDFile(filename: "debug-dead-local-var.c", directory: "/usr/local/google/home/echristo")
+!0 = !DICompileUnit(language: DW_LANG_C99, producer: "clang version 3.5.0 (trunk 209255) (llvm/trunk 209253)", isOptimized: true, emissionKind: 1, file: !1, enums: !2, retainedTypes: !2, subprograms: !3, globals: !2, imports: !2)
+!1 = !DIFile(filename: "debug-dead-local-var.c", directory: "/usr/local/google/home/echristo")
 !2 = !{}
 !3 = !{!4, !9}
-!4 = !MDSubprogram(name: "bar", line: 11, isLocal: false, isDefinition: true, virtualIndex: 6, isOptimized: true, scopeLine: 11, file: !1, scope: !5, type: !6, function: i32 ()* @bar, variables: !2)
-!5 = !MDFile(filename: "debug-dead-local-var.c", directory: "/usr/local/google/home/echristo")
-!6 = !MDSubroutineType(types: !7)
+!4 = !DISubprogram(name: "bar", line: 11, isLocal: false, isDefinition: true, virtualIndex: 6, isOptimized: true, scopeLine: 11, file: !1, scope: !5, type: !6, function: i32 ()* @bar, variables: !2)
+!5 = !DIFile(filename: "debug-dead-local-var.c", directory: "/usr/local/google/home/echristo")
+!6 = !DISubroutineType(types: !7)
 !7 = !{!8}
-!8 = !MDBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
-!9 = !MDSubprogram(name: "foo", line: 6, isLocal: true, isDefinition: true, virtualIndex: 6, isOptimized: true, scopeLine: 6, file: !1, scope: !5, type: !10, variables: !12)
-!10 = !MDSubroutineType(types: !11)
+!8 = !DIBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
+!9 = !DISubprogram(name: "foo", line: 6, isLocal: true, isDefinition: true, virtualIndex: 6, isOptimized: true, scopeLine: 6, file: !1, scope: !5, type: !10, variables: !12)
+!10 = !DISubroutineType(types: !11)
 !11 = !{null}
 !12 = !{!13}
-!13 = !MDLocalVariable(tag: DW_TAG_auto_variable, name: "xyz", line: 8, scope: !9, file: !5, type: !14)
-!14 = !MDCompositeType(tag: DW_TAG_structure_type, name: "X", line: 8, size: 64, align: 32, file: !1, scope: !9, elements: !15)
+!13 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "xyz", line: 8, scope: !9, file: !5, type: !14)
+!14 = !DICompositeType(tag: DW_TAG_structure_type, name: "X", line: 8, size: 64, align: 32, file: !1, scope: !9, elements: !15)
 !15 = !{!16, !17}
-!16 = !MDDerivedType(tag: DW_TAG_member, name: "a", line: 8, size: 32, align: 32, file: !1, scope: !14, baseType: !8)
-!17 = !MDDerivedType(tag: DW_TAG_member, name: "b", line: 8, size: 32, align: 32, offset: 32, file: !1, scope: !14, baseType: !8)
+!16 = !DIDerivedType(tag: DW_TAG_member, name: "a", line: 8, size: 32, align: 32, file: !1, scope: !14, baseType: !8)
+!17 = !DIDerivedType(tag: DW_TAG_member, name: "b", line: 8, size: 32, align: 32, offset: 32, file: !1, scope: !14, baseType: !8)
 !18 = !{i32 2, !"Dwarf Version", i32 4}
 !19 = !{i32 2, !"Debug Info Version", i32 3}
 !20 = !{!"clang version 3.5.0 (trunk 209255) (llvm/trunk 209253)"}
-!21 = !MDLocation(line: 13, scope: !4)
+!21 = !DILocation(line: 13, scope: !4)
diff --git a/test/DebugInfo/X86/debug-info-access.ll b/test/DebugInfo/X86/debug-info-access.ll
index ae959099d9a..21da7b2db1b 100644
--- a/test/DebugInfo/X86/debug-info-access.ll
+++ b/test/DebugInfo/X86/debug-info-access.ll
@@ -106,45 +106,45 @@ attributes #0 = { nounwind ssp uwtable }
 !llvm.module.flags = !{!38, !39}
 !llvm.ident = !{!40}
 
-!0 = !MDCompileUnit(language: DW_LANG_C_plus_plus, producer: "clang version 3.6.0 ", isOptimized: false, emissionKind: 1, file: !1, enums: !2, retainedTypes: !3, subprograms: !29, globals: !34, imports: !2)
-!1 = !MDFile(filename: "/llvm/tools/clang/test/CodeGenCXX/debug-info-access.cpp", directory: "")
+!0 = !DICompileUnit(language: DW_LANG_C_plus_plus, producer: "clang version 3.6.0 ", isOptimized: false, emissionKind: 1, file: !1, enums: !2, retainedTypes: !3, subprograms: !29, globals: !34, imports: !2)
+!1 = !DIFile(filename: "/llvm/tools/clang/test/CodeGenCXX/debug-info-access.cpp", directory: "")
 !2 = !{}
 !3 = !{!4, !12, !22}
-!4 = !MDCompositeType(tag: DW_TAG_structure_type, name: "A", line: 3, size: 8, align: 8, file: !1, elements: !5, identifier: "_ZTS1A")
+!4 = !DICompositeType(tag: DW_TAG_structure_type, name: "A", line: 3, size: 8, align: 8, file: !1, elements: !5, identifier: "_ZTS1A")
 !5 = !{!6, !8}
-!6 = !MDDerivedType(tag: DW_TAG_member, name: "pub_default_static", line: 7, flags: DIFlagStaticMember, file: !1, scope: !"_ZTS1A", baseType: !7)
-!7 = !MDBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
-!8 = !MDSubprogram(name: "pub_default", linkageName: "_ZN1A11pub_defaultEv", line: 5, isLocal: false, isDefinition: false, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 5, file: !1, scope: !"_ZTS1A", type: !9)
-!9 = !MDSubroutineType(types: !10)
+!6 = !DIDerivedType(tag: DW_TAG_member, name: "pub_default_static", line: 7, flags: DIFlagStaticMember, file: !1, scope: !"_ZTS1A", baseType: !7)
+!7 = !DIBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
+!8 = !DISubprogram(name: "pub_default", linkageName: "_ZN1A11pub_defaultEv", line: 5, isLocal: false, isDefinition: false, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 5, file: !1, scope: !"_ZTS1A", type: !9)
+!9 = !DISubroutineType(types: !10)
 !10 = !{null, !11}
-!11 = !MDDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, flags: DIFlagArtificial | DIFlagObjectPointer, baseType: !"_ZTS1A")
-!12 = !MDCompositeType(tag: DW_TAG_class_type, name: "B", line: 11, size: 8, align: 8, file: !1, elements: !13, identifier: "_ZTS1B")
+!11 = !DIDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, flags: DIFlagArtificial | DIFlagObjectPointer, baseType: !"_ZTS1A")
+!12 = !DICompositeType(tag: DW_TAG_class_type, name: "B", line: 11, size: 8, align: 8, file: !1, elements: !13, identifier: "_ZTS1B")
 !13 = !{!14, !15, !16, !20, !21}
-!14 = !MDDerivedType(tag: DW_TAG_inheritance, flags: DIFlagPublic, scope: !"_ZTS1B", baseType: !"_ZTS1A")
-!15 = !MDDerivedType(tag: DW_TAG_member, name: "public_static", line: 16, flags: DIFlagPublic | DIFlagStaticMember, file: !1, scope: !"_ZTS1B", baseType: !7)
-!16 = !MDSubprogram(name: "pub", linkageName: "_ZN1B3pubEv", line: 14, isLocal: false, isDefinition: false, virtualIndex: 6, flags: DIFlagPublic | DIFlagPrototyped, isOptimized: false, scopeLine: 14, file: !1, scope: !"_ZTS1B", type: !17)
-!17 = !MDSubroutineType(types: !18)
+!14 = !DIDerivedType(tag: DW_TAG_inheritance, flags: DIFlagPublic, scope: !"_ZTS1B", baseType: !"_ZTS1A")
+!15 = !DIDerivedType(tag: DW_TAG_member, name: "public_static", line: 16, flags: DIFlagPublic | DIFlagStaticMember, file: !1, scope: !"_ZTS1B", baseType: !7)
+!16 = !DISubprogram(name: "pub", linkageName: "_ZN1B3pubEv", line: 14, isLocal: false, isDefinition: false, virtualIndex: 6, flags: DIFlagPublic | DIFlagPrototyped, isOptimized: false, scopeLine: 14, file: !1, scope: !"_ZTS1B", type: !17)
+!17 = !DISubroutineType(types: !18)
 !18 = !{null, !19}
-!19 = !MDDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, flags: DIFlagArtificial | DIFlagObjectPointer, baseType: !"_ZTS1B")
-!20 = !MDSubprogram(name: "prot", linkageName: "_ZN1B4protEv", line: 19, isLocal: false, isDefinition: false, virtualIndex: 6, flags: DIFlagProtected | DIFlagPrototyped, isOptimized: false, scopeLine: 19, file: !1, scope: !"_ZTS1B", type: !17)
-!21 = !MDSubprogram(name: "priv_default", linkageName: "_ZN1B12priv_defaultEv", line: 22, isLocal: false, isDefinition: false, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 22, file: !1, scope: !"_ZTS1B", type: !17)
-!22 = !MDCompositeType(tag: DW_TAG_union_type, name: "U", line: 25, size: 32, align: 32, file: !1, elements: !23, identifier: "_ZTS1U")
+!19 = !DIDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, flags: DIFlagArtificial | DIFlagObjectPointer, baseType: !"_ZTS1B")
+!20 = !DISubprogram(name: "prot", linkageName: "_ZN1B4protEv", line: 19, isLocal: false, isDefinition: false, virtualIndex: 6, flags: DIFlagProtected | DIFlagPrototyped, isOptimized: false, scopeLine: 19, file: !1, scope: !"_ZTS1B", type: !17)
+!21 = !DISubprogram(name: "priv_default", linkageName: "_ZN1B12priv_defaultEv", line: 22, isLocal: false, isDefinition: false, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 22, file: !1, scope: !"_ZTS1B", type: !17)
+!22 = !DICompositeType(tag: DW_TAG_union_type, name: "U", line: 25, size: 32, align: 32, file: !1, elements: !23, identifier: "_ZTS1U")
 !23 = !{!24, !25}
-!24 = !MDDerivedType(tag: DW_TAG_member, name: "union_priv", line: 30, size: 32, align: 32, flags: DIFlagPrivate, file: !1, scope: !"_ZTS1U", baseType: !7)
-!25 = !MDSubprogram(name: "union_pub_default", linkageName: "_ZN1U17union_pub_defaultEv", line: 27, isLocal: false, isDefinition: false, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 27, file: !1, scope: !"_ZTS1U", type: !26)
-!26 = !MDSubroutineType(types: !27)
+!24 = !DIDerivedType(tag: DW_TAG_member, name: "union_priv", line: 30, size: 32, align: 32, flags: DIFlagPrivate, file: !1, scope: !"_ZTS1U", baseType: !7)
+!25 = !DISubprogram(name: "union_pub_default", linkageName: "_ZN1U17union_pub_defaultEv", line: 27, isLocal: false, isDefinition: false, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 27, file: !1, scope: !"_ZTS1U", type: !26)
+!26 = !DISubroutineType(types: !27)
 !27 = !{null, !28}
-!28 = !MDDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, flags: DIFlagArtificial | DIFlagObjectPointer, baseType: !"_ZTS1U")
+!28 = !DIDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, flags: DIFlagArtificial | DIFlagObjectPointer, baseType: !"_ZTS1U")
 !29 = !{!30}
-!30 = !MDSubprogram(name: "free", linkageName: "_Z4freev", line: 35, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 35, file: !1, scope: !31, type: !32, function: void ()* @_Z4freev, variables: !2)
-!31 = !MDFile(filename: "/llvm/tools/clang/test/CodeGenCXX/debug-info-access.cpp", directory: "")
-!32 = !MDSubroutineType(types: !33)
+!30 = !DISubprogram(name: "free", linkageName: "_Z4freev", line: 35, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 35, file: !1, scope: !31, type: !32, function: void ()* @_Z4freev, variables: !2)
+!31 = !DIFile(filename: "/llvm/tools/clang/test/CodeGenCXX/debug-info-access.cpp", directory: "")
+!32 = !DISubroutineType(types: !33)
 !33 = !{null}
 !34 = !{!35, !36, !37}
-!35 = !MDGlobalVariable(name: "a", line: 37, isLocal: false, isDefinition: true, scope: null, file: !31, type: !"_ZTS1A", variable: %struct.A* @a)
-!36 = !MDGlobalVariable(name: "b", line: 38, isLocal: false, isDefinition: true, scope: null, file: !31, type: !"_ZTS1B", variable: %class.B* @b)
-!37 = !MDGlobalVariable(name: "u", line: 39, isLocal: false, isDefinition: true, scope: null, file: !31, type: !"_ZTS1U", variable: %union.U* @u)
+!35 = !DIGlobalVariable(name: "a", line: 37, isLocal: false, isDefinition: true, scope: null, file: !31, type: !"_ZTS1A", variable: %struct.A* @a)
+!36 = !DIGlobalVariable(name: "b", line: 38, isLocal: false, isDefinition: true, scope: null, file: !31, type: !"_ZTS1B", variable: %class.B* @b)
+!37 = !DIGlobalVariable(name: "u", line: 39, isLocal: false, isDefinition: true, scope: null, file: !31, type: !"_ZTS1U", variable: %union.U* @u)
 !38 = !{i32 2, !"Dwarf Version", i32 2}
 !39 = !{i32 2, !"Debug Info Version", i32 3}
 !40 = !{!"clang version 3.6.0 "}
-!41 = !MDLocation(line: 35, column: 14, scope: !30)
+!41 = !DILocation(line: 35, column: 14, scope: !30)
diff --git a/test/DebugInfo/X86/debug-info-block-captured-self.ll b/test/DebugInfo/X86/debug-info-block-captured-self.ll
index d6c1fc00e89..e79b093d031 100644
--- a/test/DebugInfo/X86/debug-info-block-captured-self.ll
+++ b/test/DebugInfo/X86/debug-info-block-captured-self.ll
@@ -80,33 +80,33 @@ define internal void @"__24-[Main initWithContext:]_block_invoke_2"(i8* %.block_
 
 !llvm.dbg.cu = !{!0}
 !llvm.module.flags = !{!108}
-!0 = !MDCompileUnit(language: DW_LANG_ObjC, producer: "clang version 3.3 ", isOptimized: false, runtimeVersion: 2, emissionKind: 0, file: !107, enums: !2, retainedTypes: !4, subprograms: !23, globals: !15, imports:  !15)
-!1 = !MDFile(filename: "llvm/tools/clang/test/CodeGenObjC/debug-info-block-captured-self.m", directory: "")
+!0 = !DICompileUnit(language: DW_LANG_ObjC, producer: "clang version 3.3 ", isOptimized: false, runtimeVersion: 2, emissionKind: 0, file: !107, enums: !2, retainedTypes: !4, subprograms: !23, globals: !15, imports:  !15)
+!1 = !DIFile(filename: "llvm/tools/clang/test/CodeGenObjC/debug-info-block-captured-self.m", directory: "")
 !2 = !{!3}
-!3 = !MDCompositeType(tag: DW_TAG_enumeration_type, line: 20, size: 32, align: 32, file: !107, elements: !4)
+!3 = !DICompositeType(tag: DW_TAG_enumeration_type, line: 20, size: 32, align: 32, file: !107, elements: !4)
 !4 = !{}
 !15 = !{}
 !23 = !{!38, !42}
-!27 = !MDDerivedType(tag: DW_TAG_typedef, name: "id", line: 31, file: !107, baseType: !28)
-!28 = !MDDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, baseType: !29)
-!29 = !MDCompositeType(tag: DW_TAG_structure_type, name: "objc_object", file: !107, elements: !30)
+!27 = !DIDerivedType(tag: DW_TAG_typedef, name: "id", line: 31, file: !107, baseType: !28)
+!28 = !DIDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, baseType: !29)
+!29 = !DICompositeType(tag: DW_TAG_structure_type, name: "objc_object", file: !107, elements: !30)
 !30 = !{!31}
-!31 = !MDDerivedType(tag: DW_TAG_member, name: "isa", size: 64, file: !107, scope: !29, baseType: !32)
-!32 = !MDDerivedType(tag: DW_TAG_pointer_type, size: 64, baseType: !33)
-!33 = !MDCompositeType(tag: DW_TAG_structure_type, name: "objc_class", flags: DIFlagFwdDecl, file: !107)
-!34 = !MDCompositeType(tag: DW_TAG_structure_type, name: "Main", line: 23, flags: DIFlagArtificial | DIFlagObjectPointer, runtimeLang: DW_LANG_ObjC, file: !107)
-!38 = !MDSubprogram(name: "__24-[Main initWithContext:]_block_invoke", line: 33, isLocal: true, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 33, file: !1, scope: !1, type: !39, function: void (i8*, i8*)* @"__24-[Main initWithContext:]_block_invoke", variables: !15)
-!39 = !MDSubroutineType(types: !40)
+!31 = !DIDerivedType(tag: DW_TAG_member, name: "isa", size: 64, file: !107, scope: !29, baseType: !32)
+!32 = !DIDerivedType(tag: DW_TAG_pointer_type, size: 64, baseType: !33)
+!33 = !DICompositeType(tag: DW_TAG_structure_type, name: "objc_class", flags: DIFlagFwdDecl, file: !107)
+!34 = !DICompositeType(tag: DW_TAG_structure_type, name: "Main", line: 23, flags: DIFlagArtificial | DIFlagObjectPointer, runtimeLang: DW_LANG_ObjC, file: !107)
+!38 = !DISubprogram(name: "__24-[Main initWithContext:]_block_invoke", line: 33, isLocal: true, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 33, file: !1, scope: !1, type: !39, function: void (i8*, i8*)* @"__24-[Main initWithContext:]_block_invoke", variables: !15)
+!39 = !DISubroutineType(types: !40)
 !40 = !{null, !41, !27}
-!41 = !MDDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, baseType: null)
-!42 = !MDSubprogram(name: "__24-[Main initWithContext:]_block_invoke_2", line: 35, isLocal: true, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 35, file: !1, scope: !1, type: !39, function: void (i8*, i8*)* @"__24-[Main initWithContext:]_block_invoke_2", variables: !15)
-!84 = !MDLocation(line: 33, scope: !38)
-!86 = !MDLocalVariable(tag: DW_TAG_auto_variable, name: "self", line: 41, scope: !38, file: !1, type: !34)
-!87 = !MDLocation(line: 41, scope: !38)
-!103 = !MDLocation(line: 35, scope: !42)
-!105 = !MDLocalVariable(tag: DW_TAG_auto_variable, name: "self", line: 40, scope: !42, file: !1, type: !34)
-!106 = !MDLocation(line: 40, scope: !42)
-!107 = !MDFile(filename: "llvm/tools/clang/test/CodeGenObjC/debug-info-block-captured-self.m", directory: "")
+!41 = !DIDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, baseType: null)
+!42 = !DISubprogram(name: "__24-[Main initWithContext:]_block_invoke_2", line: 35, isLocal: true, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 35, file: !1, scope: !1, type: !39, function: void (i8*, i8*)* @"__24-[Main initWithContext:]_block_invoke_2", variables: !15)
+!84 = !DILocation(line: 33, scope: !38)
+!86 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "self", line: 41, scope: !38, file: !1, type: !34)
+!87 = !DILocation(line: 41, scope: !38)
+!103 = !DILocation(line: 35, scope: !42)
+!105 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "self", line: 40, scope: !42, file: !1, type: !34)
+!106 = !DILocation(line: 40, scope: !42)
+!107 = !DIFile(filename: "llvm/tools/clang/test/CodeGenObjC/debug-info-block-captured-self.m", directory: "")
 !108 = !{i32 1, !"Debug Info Version", i32 3}
-!109 = !MDExpression(DW_OP_deref, DW_OP_plus, 32)
-!110 = !MDExpression(DW_OP_deref, DW_OP_plus, 32)
+!109 = !DIExpression(DW_OP_deref, DW_OP_plus, 32)
+!110 = !DIExpression(DW_OP_deref, DW_OP_plus, 32)
diff --git a/test/DebugInfo/X86/debug-info-blocks.ll b/test/DebugInfo/X86/debug-info-blocks.ll
index 44f945cfe33..a9aadfdb731 100644
--- a/test/DebugInfo/X86/debug-info-blocks.ll
+++ b/test/DebugInfo/X86/debug-info-blocks.ll
@@ -101,9 +101,9 @@ define internal i8* @"\01-[A init]"(%0* %self, i8* %_cmd) #0 {
   %3 = alloca %struct._objc_super
   %4 = alloca <{ i8*, i32, i32, i8*, %struct.__block_descriptor*, %0* }>, align 8
   store %0* %self, %0** %1, align 8
-  call void @llvm.dbg.declare(metadata %0** %1, metadata !60, metadata !MDExpression()), !dbg !62
+  call void @llvm.dbg.declare(metadata %0** %1, metadata !60, metadata !DIExpression()), !dbg !62
   store i8* %_cmd, i8** %2, align 8
-  call void @llvm.dbg.declare(metadata i8** %2, metadata !63, metadata !MDExpression()), !dbg !62
+  call void @llvm.dbg.declare(metadata i8** %2, metadata !63, metadata !DIExpression()), !dbg !62
   %5 = load %0*, %0** %1, !dbg !65
   %6 = bitcast %0* %5 to i8*, !dbg !65
   %7 = getelementptr inbounds %struct._objc_super, %struct._objc_super* %3, i32 0, i32 0, !dbg !65
@@ -150,7 +150,7 @@ declare i8* @objc_msgSendSuper2(%struct._objc_super*, i8*, ...)
 define internal void @run(void ()* %block) #0 {
   %1 = alloca void ()*, align 8
   store void ()* %block, void ()** %1, align 8
-  call void @llvm.dbg.declare(metadata void ()** %1, metadata !72, metadata !MDExpression()), !dbg !73
+  call void @llvm.dbg.declare(metadata void ()** %1, metadata !72, metadata !DIExpression()), !dbg !73
   %2 = load void ()*, void ()** %1, align 8, !dbg !74
   %3 = bitcast void ()* %2 to %struct.__block_literal_generic*, !dbg !74
   %4 = getelementptr inbounds %struct.__block_literal_generic, %struct.__block_literal_generic* %3, i32 0, i32 3, !dbg !74
@@ -167,13 +167,13 @@ define internal void @"__9-[A init]_block_invoke"(i8* %.block_descriptor) #0 {
   %d = alloca %1*, align 8
   store i8* %.block_descriptor, i8** %1, align 8
   %3 = load i8*, i8** %1
-  call void @llvm.dbg.value(metadata i8* %3, i64 0, metadata !76, metadata !MDExpression()), !dbg !88
-  call void @llvm.dbg.declare(metadata i8* %.block_descriptor, metadata !76, metadata !MDExpression()), !dbg !88
+  call void @llvm.dbg.value(metadata i8* %3, i64 0, metadata !76, metadata !DIExpression()), !dbg !88
+  call void @llvm.dbg.declare(metadata i8* %.block_descriptor, metadata !76, metadata !DIExpression()), !dbg !88
   %4 = bitcast i8* %.block_descriptor to <{ i8*, i32, i32, i8*, %struct.__block_descriptor*, %0* }>*, !dbg !88
   store <{ i8*, i32, i32, i8*, %struct.__block_descriptor*, %0* }>* %4, <{ i8*, i32, i32, i8*, %struct.__block_descriptor*, %0* }>** %2, align 8, !dbg !88
   %5 = getelementptr inbounds <{ i8*, i32, i32, i8*, %struct.__block_descriptor*, %0* }>, <{ i8*, i32, i32, i8*, %struct.__block_descriptor*, %0* }>* %4, i32 0, i32 5, !dbg !88
   call void @llvm.dbg.declare(metadata <{ i8*, i32, i32, i8*, %struct.__block_descriptor*, %0* }>** %2, metadata !89, metadata !111), !dbg !90
-  call void @llvm.dbg.declare(metadata %1** %d, metadata !91, metadata !MDExpression()), !dbg !100
+  call void @llvm.dbg.declare(metadata %1** %d, metadata !91, metadata !DIExpression()), !dbg !100
   %6 = load %struct._class_t*, %struct._class_t** @"\01L_OBJC_CLASSLIST_REFERENCES_$_", !dbg !100
   %7 = bitcast %struct._class_t* %6 to i8*, !dbg !100
   %8 = load i8*, i8** getelementptr inbounds (%struct._message_ref_t, %struct._message_ref_t* bitcast ({ i8* (i8*, %struct._message_ref_t*, ...)*, i8* }* @"\01l_objc_msgSend_fixup_alloc" to %struct._message_ref_t*), i32 0, i32 0), !dbg !100
@@ -210,9 +210,9 @@ define internal void @__copy_helper_block_(i8*, i8*) {
   %3 = alloca i8*, align 8
   %4 = alloca i8*, align 8
   store i8* %0, i8** %3, align 8
-  call void @llvm.dbg.declare(metadata i8** %3, metadata !102, metadata !MDExpression()), !dbg !103
+  call void @llvm.dbg.declare(metadata i8** %3, metadata !102, metadata !DIExpression()), !dbg !103
   store i8* %1, i8** %4, align 8
-  call void @llvm.dbg.declare(metadata i8** %4, metadata !104, metadata !MDExpression()), !dbg !103
+  call void @llvm.dbg.declare(metadata i8** %4, metadata !104, metadata !DIExpression()), !dbg !103
   %5 = load i8*, i8** %4, !dbg !103
   %6 = bitcast i8* %5 to <{ i8*, i32, i32, i8*, %struct.__block_descriptor*, %0* }>*, !dbg !103
   %7 = load i8*, i8** %3, !dbg !103
@@ -231,7 +231,7 @@ declare void @_Block_object_assign(i8*, i8*, i32)
 define internal void @__destroy_helper_block_(i8*) {
   %2 = alloca i8*, align 8
   store i8* %0, i8** %2, align 8
-  call void @llvm.dbg.declare(metadata i8** %2, metadata !105, metadata !MDExpression()), !dbg !106
+  call void @llvm.dbg.declare(metadata i8** %2, metadata !105, metadata !DIExpression()), !dbg !106
   %3 = load i8*, i8** %2, !dbg !106
   %4 = bitcast i8* %3 to <{ i8*, i32, i32, i8*, %struct.__block_descriptor*, %0* }>*, !dbg !106
   %5 = getelementptr inbounds <{ i8*, i32, i32, i8*, %struct.__block_descriptor*, %0* }>, <{ i8*, i32, i32, i8*, %struct.__block_descriptor*, %0* }>* %4, i32 0, i32 5, !dbg !106
@@ -247,7 +247,7 @@ define i32 @main() #0 {
   %1 = alloca i32, align 4
   %a = alloca %0*, align 8
   store i32 0, i32* %1
-  call void @llvm.dbg.declare(metadata %0** %a, metadata !107, metadata !MDExpression()), !dbg !108
+  call void @llvm.dbg.declare(metadata %0** %a, metadata !107, metadata !DIExpression()), !dbg !108
   %2 = load %struct._class_t*, %struct._class_t** @"\01L_OBJC_CLASSLIST_REFERENCES_$_5", !dbg !108
   %3 = bitcast %struct._class_t* %2 to i8*, !dbg !108
   %4 = load i8*, i8** getelementptr inbounds (%struct._message_ref_t, %struct._message_ref_t* bitcast ({ i8* (i8*, %struct._message_ref_t*, ...)*, i8* }* @"\01l_objc_msgSend_fixup_alloc" to %struct._message_ref_t*), i32 0, i32 0), !dbg !108
@@ -270,115 +270,115 @@ attributes #3 = { nounwind }
 !llvm.dbg.cu = !{!0}
 !llvm.module.flags = !{!56, !57, !58, !59, !110}
 
-!0 = !MDCompileUnit(language: DW_LANG_ObjC, producer: "clang version 3.3 ", isOptimized: false, runtimeVersion: 2, emissionKind: 1, file: !1, enums: !2, retainedTypes: !3, subprograms: !12, globals: !2, imports:  !2)
-!1 = !MDFile(filename: "llvm/tools/clang/test/CodeGenObjC/<unknown>", directory: "llvm/_build.ninja.Debug")
+!0 = !DICompileUnit(language: DW_LANG_ObjC, producer: "clang version 3.3 ", isOptimized: false, runtimeVersion: 2, emissionKind: 1, file: !1, enums: !2, retainedTypes: !3, subprograms: !12, globals: !2, imports:  !2)
+!1 = !DIFile(filename: "llvm/tools/clang/test/CodeGenObjC/<unknown>", directory: "llvm/_build.ninja.Debug")
 !2 = !{}
 !3 = !{!4}
-!4 = !MDCompositeType(tag: DW_TAG_structure_type, name: "A", line: 33, size: 32, align: 32, flags: DIFlagObjcClassComplete, runtimeLang: DW_LANG_ObjC, file: !5, scope: !6, elements: !7)
-!5 = !MDFile(filename: "llvm/tools/clang/test/CodeGenObjC/debug-info-blocks.m", directory: "llvm/_build.ninja.Debug")
-!6 = !MDFile(filename: "llvm/tools/clang/test/CodeGenObjC/debug-info-blocks.m", directory: "llvm/_build.ninja.Debug")
+!4 = !DICompositeType(tag: DW_TAG_structure_type, name: "A", line: 33, size: 32, align: 32, flags: DIFlagObjcClassComplete, runtimeLang: DW_LANG_ObjC, file: !5, scope: !6, elements: !7)
+!5 = !DIFile(filename: "llvm/tools/clang/test/CodeGenObjC/debug-info-blocks.m", directory: "llvm/_build.ninja.Debug")
+!6 = !DIFile(filename: "llvm/tools/clang/test/CodeGenObjC/debug-info-blocks.m", directory: "llvm/_build.ninja.Debug")
 !7 = !{!8, !10}
-!8 = !MDDerivedType(tag: DW_TAG_inheritance, scope: !4, baseType: !9)
-!9 = !MDCompositeType(tag: DW_TAG_structure_type, name: "NSObject", line: 21, align: 8, runtimeLang: DW_LANG_ObjC, file: !5, scope: !6, elements: !2)
-!10 = !MDDerivedType(tag: DW_TAG_member, name: "ivar", line: 35, size: 32, align: 32, file: !5, scope: !6, baseType: !11)
-!11 = !MDBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
+!8 = !DIDerivedType(tag: DW_TAG_inheritance, scope: !4, baseType: !9)
+!9 = !DICompositeType(tag: DW_TAG_structure_type, name: "NSObject", line: 21, align: 8, runtimeLang: DW_LANG_ObjC, file: !5, scope: !6, elements: !2)
+!10 = !DIDerivedType(tag: DW_TAG_member, name: "ivar", line: 35, size: 32, align: 32, file: !5, scope: !6, baseType: !11)
+!11 = !DIBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
 !12 = !{!13, !27, !31, !35, !36, !39}
-!13 = !MDSubprogram(name: "-[A init]", line: 46, isLocal: true, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 46, file: !5, scope: !6, type: !14, function: i8* (%0*, i8*)* @"\01-[A init]", variables: !2)
-!14 = !MDSubroutineType(types: !15)
+!13 = !DISubprogram(name: "-[A init]", line: 46, isLocal: true, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 46, file: !5, scope: !6, type: !14, function: i8* (%0*, i8*)* @"\01-[A init]", variables: !2)
+!14 = !DISubroutineType(types: !15)
 !15 = !{!16, !23, !24}
-!16 = !MDDerivedType(tag: DW_TAG_typedef, name: "id", line: 46, file: !5, baseType: !17)
-!17 = !MDDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, baseType: !18)
-!18 = !MDCompositeType(tag: DW_TAG_structure_type, name: "objc_object", file: !1, elements: !19)
+!16 = !DIDerivedType(tag: DW_TAG_typedef, name: "id", line: 46, file: !5, baseType: !17)
+!17 = !DIDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, baseType: !18)
+!18 = !DICompositeType(tag: DW_TAG_structure_type, name: "objc_object", file: !1, elements: !19)
 !19 = !{!20}
-!20 = !MDDerivedType(tag: DW_TAG_member, name: "isa", size: 64, file: !1, scope: !18, baseType: !21)
-!21 = !MDDerivedType(tag: DW_TAG_pointer_type, size: 64, baseType: !22)
-!22 = !MDCompositeType(tag: DW_TAG_structure_type, name: "objc_class", flags: DIFlagFwdDecl, file: !1)
-!23 = !MDDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, flags: DIFlagArtificial | DIFlagObjectPointer, baseType: !4)
-!24 = !MDDerivedType(tag: DW_TAG_typedef, name: "SEL", line: 46, flags: DIFlagArtificial, file: !5, baseType: !25)
-!25 = !MDDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, baseType: !26)
-!26 = !MDCompositeType(tag: DW_TAG_structure_type, name: "objc_selector", flags: DIFlagFwdDecl, file: !1)
-!27 = !MDSubprogram(name: "__9-[A init]_block_invoke", line: 49, isLocal: true, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 49, file: !5, scope: !6, type: !28, function: void (i8*)* @"__9-[A init]_block_invoke", variables: !2)
-!28 = !MDSubroutineType(types: !29)
+!20 = !DIDerivedType(tag: DW_TAG_member, name: "isa", size: 64, file: !1, scope: !18, baseType: !21)
+!21 = !DIDerivedType(tag: DW_TAG_pointer_type, size: 64, baseType: !22)
+!22 = !DICompositeType(tag: DW_TAG_structure_type, name: "objc_class", flags: DIFlagFwdDecl, file: !1)
+!23 = !DIDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, flags: DIFlagArtificial | DIFlagObjectPointer, baseType: !4)
+!24 = !DIDerivedType(tag: DW_TAG_typedef, name: "SEL", line: 46, flags: DIFlagArtificial, file: !5, baseType: !25)
+!25 = !DIDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, baseType: !26)
+!26 = !DICompositeType(tag: DW_TAG_structure_type, name: "objc_selector", flags: DIFlagFwdDecl, file: !1)
+!27 = !DISubprogram(name: "__9-[A init]_block_invoke", line: 49, isLocal: true, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 49, file: !5, scope: !6, type: !28, function: void (i8*)* @"__9-[A init]_block_invoke", variables: !2)
+!28 = !DISubroutineType(types: !29)
 !29 = !{null, !30}
-!30 = !MDDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, baseType: null)
-!31 = !MDSubprogram(name: "__copy_helper_block_", line: 52, isLocal: true, isDefinition: true, virtualIndex: 6, isOptimized: false, scopeLine: 52, file: !1, scope: !32, type: !33, function: void (i8*, i8*)* @__copy_helper_block_, variables: !2)
-!32 = !MDFile(filename: "llvm/tools/clang/test/CodeGenObjC/<unknown>", directory: "llvm/_build.ninja.Debug")
-!33 = !MDSubroutineType(types: !34)
+!30 = !DIDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, baseType: null)
+!31 = !DISubprogram(name: "__copy_helper_block_", line: 52, isLocal: true, isDefinition: true, virtualIndex: 6, isOptimized: false, scopeLine: 52, file: !1, scope: !32, type: !33, function: void (i8*, i8*)* @__copy_helper_block_, variables: !2)
+!32 = !DIFile(filename: "llvm/tools/clang/test/CodeGenObjC/<unknown>", directory: "llvm/_build.ninja.Debug")
+!33 = !DISubroutineType(types: !34)
 !34 = !{null, !30, !30}
-!35 = !MDSubprogram(name: "__destroy_helper_block_", line: 52, isLocal: true, isDefinition: true, virtualIndex: 6, isOptimized: false, scopeLine: 52, file: !1, scope: !32, type: !28, function: void (i8*)* @__destroy_helper_block_, variables: !2)
-!36 = !MDSubprogram(name: "main", line: 59, isLocal: false, isDefinition: true, virtualIndex: 6, isOptimized: false, scopeLine: 60, file: !5, scope: !6, type: !37, function: i32 ()* @main, variables: !2)
-!37 = !MDSubroutineType(types: !38)
+!35 = !DISubprogram(name: "__destroy_helper_block_", line: 52, isLocal: true, isDefinition: true, virtualIndex: 6, isOptimized: false, scopeLine: 52, file: !1, scope: !32, type: !28, function: void (i8*)* @__destroy_helper_block_, variables: !2)
+!36 = !DISubprogram(name: "main", line: 59, isLocal: false, isDefinition: true, virtualIndex: 6, isOptimized: false, scopeLine: 60, file: !5, scope: !6, type: !37, function: i32 ()* @main, variables: !2)
+!37 = !DISubroutineType(types: !38)
 !38 = !{!11}
-!39 = !MDSubprogram(name: "run", line: 39, isLocal: true, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 40, file: !5, scope: !6, type: !40, function: void (void ()*)* @run, variables: !2)
-!40 = !MDSubroutineType(types: !41)
+!39 = !DISubprogram(name: "run", line: 39, isLocal: true, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 40, file: !5, scope: !6, type: !40, function: void (void ()*)* @run, variables: !2)
+!40 = !DISubroutineType(types: !41)
 !41 = !{null, !42}
-!42 = !MDDerivedType(tag: DW_TAG_pointer_type, size: 64, baseType: !43)
-!43 = !MDCompositeType(tag: DW_TAG_structure_type, name: "__block_literal_generic", line: 40, size: 256, flags: DIFlagAppleBlock, file: !5, scope: !6, elements: !44)
+!42 = !DIDerivedType(tag: DW_TAG_pointer_type, size: 64, baseType: !43)
+!43 = !DICompositeType(tag: DW_TAG_structure_type, name: "__block_literal_generic", line: 40, size: 256, flags: DIFlagAppleBlock, file: !5, scope: !6, elements: !44)
 !44 = !{!45, !46, !47, !48, !49}
-!45 = !MDDerivedType(tag: DW_TAG_member, name: "__isa", size: 64, align: 64, file: !5, scope: !6, baseType: !30)
-!46 = !MDDerivedType(tag: DW_TAG_member, name: "__flags", size: 32, align: 32, offset: 64, file: !5, scope: !6, baseType: !11)
-!47 = !MDDerivedType(tag: DW_TAG_member, name: "__reserved", size: 32, align: 32, offset: 96, file: !5, scope: !6, baseType: !11)
-!48 = !MDDerivedType(tag: DW_TAG_member, name: "__FuncPtr", size: 64, align: 64, offset: 128, file: !5, scope: !6, baseType: !30)
-!49 = !MDDerivedType(tag: DW_TAG_member, name: "__descriptor", line: 40, size: 64, align: 64, offset: 192, file: !5, scope: !6, baseType: !50)
-!50 = !MDDerivedType(tag: DW_TAG_pointer_type, size: 64, baseType: !51)
-!51 = !MDCompositeType(tag: DW_TAG_structure_type, name: "__block_descriptor", line: 40, size: 128, flags: DIFlagAppleBlock, file: !5, scope: !6, elements: !52)
+!45 = !DIDerivedType(tag: DW_TAG_member, name: "__isa", size: 64, align: 64, file: !5, scope: !6, baseType: !30)
+!46 = !DIDerivedType(tag: DW_TAG_member, name: "__flags", size: 32, align: 32, offset: 64, file: !5, scope: !6, baseType: !11)
+!47 = !DIDerivedType(tag: DW_TAG_member, name: "__reserved", size: 32, align: 32, offset: 96, file: !5, scope: !6, baseType: !11)
+!48 = !DIDerivedType(tag: DW_TAG_member, name: "__FuncPtr", size: 64, align: 64, offset: 128, file: !5, scope: !6, baseType: !30)
+!49 = !DIDerivedType(tag: DW_TAG_member, name: "__descriptor", line: 40, size: 64, align: 64, offset: 192, file: !5, scope: !6, baseType: !50)
+!50 = !DIDerivedType(tag: DW_TAG_pointer_type, size: 64, baseType: !51)
+!51 = !DICompositeType(tag: DW_TAG_structure_type, name: "__block_descriptor", line: 40, size: 128, flags: DIFlagAppleBlock, file: !5, scope: !6, elements: !52)
 !52 = !{!53, !55}
-!53 = !MDDerivedType(tag: DW_TAG_member, name: "reserved", size: 64, align: 64, file: !5, scope: !6, baseType: !54)
-!54 = !MDBasicType(tag: DW_TAG_base_type, name: "long unsigned int", size: 64, align: 64, encoding: DW_ATE_unsigned)
-!55 = !MDDerivedType(tag: DW_TAG_member, name: "Size", size: 64, align: 64, offset: 64, file: !5, scope: !6, baseType: !54)
+!53 = !DIDerivedType(tag: DW_TAG_member, name: "reserved", size: 64, align: 64, file: !5, scope: !6, baseType: !54)
+!54 = !DIBasicType(tag: DW_TAG_base_type, name: "long unsigned int", size: 64, align: 64, encoding: DW_ATE_unsigned)
+!55 = !DIDerivedType(tag: DW_TAG_member, name: "Size", size: 64, align: 64, offset: 64, file: !5, scope: !6, baseType: !54)
 !56 = !{i32 1, !"Objective-C Version", i32 2}
 !57 = !{i32 1, !"Objective-C Image Info Version", i32 0}
 !58 = !{i32 1, !"Objective-C Image Info Section", !"__DATA, __objc_imageinfo, regular, no_dead_strip"}
 !59 = !{i32 4, !"Objective-C Garbage Collection", i32 0}
-!60 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "self", line: 46, arg: 1, flags: DIFlagArtificial | DIFlagObjectPointer, scope: !13, file: !32, type: !61)
-!61 = !MDDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, baseType: !4)
-!62 = !MDLocation(line: 46, scope: !13)
-!63 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "_cmd", line: 46, arg: 2, flags: DIFlagArtificial, scope: !13, file: !32, type: !64)
-!64 = !MDDerivedType(tag: DW_TAG_typedef, name: "SEL", line: 46, file: !5, baseType: !25)
-!65 = !MDLocation(line: 48, scope: !66)
-!66 = distinct !MDLexicalBlock(line: 47, column: 0, file: !5, scope: !13)
+!60 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "self", line: 46, arg: 1, flags: DIFlagArtificial | DIFlagObjectPointer, scope: !13, file: !32, type: !61)
+!61 = !DIDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, baseType: !4)
+!62 = !DILocation(line: 46, scope: !13)
+!63 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "_cmd", line: 46, arg: 2, flags: DIFlagArtificial, scope: !13, file: !32, type: !64)
+!64 = !DIDerivedType(tag: DW_TAG_typedef, name: "SEL", line: 46, file: !5, baseType: !25)
+!65 = !DILocation(line: 48, scope: !66)
+!66 = distinct !DILexicalBlock(line: 47, column: 0, file: !5, scope: !13)
 !67 = !{}
-!68 = !MDLocation(line: 49, scope: !69)
-!69 = distinct !MDLexicalBlock(line: 48, column: 0, file: !5, scope: !66)
-!70 = !MDLocation(line: 53, scope: !69)
-!71 = !MDLocation(line: 54, scope: !66)
-!72 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "block", line: 39, arg: 1, scope: !39, file: !6, type: !42)
-!73 = !MDLocation(line: 39, scope: !39)
-!74 = !MDLocation(line: 41, scope: !39)
-!75 = !MDLocation(line: 42, scope: !39)
-!76 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: ".block_descriptor", line: 49, arg: 1, flags: DIFlagArtificial, scope: !27, file: !6, type: !77)
-!77 = !MDDerivedType(tag: DW_TAG_pointer_type, size: 64, baseType: !78)
-!78 = !MDCompositeType(tag: DW_TAG_structure_type, name: "__block_literal_1", line: 49, size: 320, align: 64, file: !5, scope: !6, elements: !79)
+!68 = !DILocation(line: 49, scope: !69)
+!69 = distinct !DILexicalBlock(line: 48, column: 0, file: !5, scope: !66)
+!70 = !DILocation(line: 53, scope: !69)
+!71 = !DILocation(line: 54, scope: !66)
+!72 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "block", line: 39, arg: 1, scope: !39, file: !6, type: !42)
+!73 = !DILocation(line: 39, scope: !39)
+!74 = !DILocation(line: 41, scope: !39)
+!75 = !DILocation(line: 42, scope: !39)
+!76 = !DILocalVariable(tag: DW_TAG_arg_variable, name: ".block_descriptor", line: 49, arg: 1, flags: DIFlagArtificial, scope: !27, file: !6, type: !77)
+!77 = !DIDerivedType(tag: DW_TAG_pointer_type, size: 64, baseType: !78)
+!78 = !DICompositeType(tag: DW_TAG_structure_type, name: "__block_literal_1", line: 49, size: 320, align: 64, file: !5, scope: !6, elements: !79)
 !79 = !{!80, !81, !82, !83, !84, !87}
-!80 = !MDDerivedType(tag: DW_TAG_member, name: "__isa", line: 49, size: 64, align: 64, file: !5, scope: !6, baseType: !30)
-!81 = !MDDerivedType(tag: DW_TAG_member, name: "__flags", line: 49, size: 32, align: 32, offset: 64, file: !5, scope: !6, baseType: !11)
-!82 = !MDDerivedType(tag: DW_TAG_member, name: "__reserved", line: 49, size: 32, align: 32, offset: 96, file: !5, scope: !6, baseType: !11)
-!83 = !MDDerivedType(tag: DW_TAG_member, name: "__FuncPtr", line: 49, size: 64, align: 64, offset: 128, file: !5, scope: !6, baseType: !30)
-!84 = !MDDerivedType(tag: DW_TAG_member, name: "__descriptor", line: 49, size: 64, align: 64, offset: 192, file: !5, scope: !6, baseType: !85)
-!85 = !MDDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, baseType: !86)
-!86 = !MDCompositeType(tag: DW_TAG_structure_type, name: "__block_descriptor_withcopydispose", line: 49, flags: DIFlagFwdDecl, file: !1)
-!87 = !MDDerivedType(tag: DW_TAG_member, name: "self", line: 49, size: 64, align: 64, offset: 256, file: !5, scope: !6, baseType: !61)
-!88 = !MDLocation(line: 49, scope: !27)
-!89 = !MDLocalVariable(tag: DW_TAG_auto_variable, name: "self", line: 52, scope: !27, file: !32, type: !23)
-!90 = !MDLocation(line: 52, scope: !27)
-!91 = !MDLocalVariable(tag: DW_TAG_auto_variable, name: "d", line: 50, scope: !92, file: !6, type: !93)
-!92 = distinct !MDLexicalBlock(line: 49, column: 0, file: !5, scope: !27)
-!93 = !MDDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, baseType: !94)
-!94 = !MDCompositeType(tag: DW_TAG_structure_type, name: "NSMutableDictionary", line: 30, align: 8, runtimeLang: DW_LANG_ObjC, file: !5, scope: !6, elements: !95)
+!80 = !DIDerivedType(tag: DW_TAG_member, name: "__isa", line: 49, size: 64, align: 64, file: !5, scope: !6, baseType: !30)
+!81 = !DIDerivedType(tag: DW_TAG_member, name: "__flags", line: 49, size: 32, align: 32, offset: 64, file: !5, scope: !6, baseType: !11)
+!82 = !DIDerivedType(tag: DW_TAG_member, name: "__reserved", line: 49, size: 32, align: 32, offset: 96, file: !5, scope: !6, baseType: !11)
+!83 = !DIDerivedType(tag: DW_TAG_member, name: "__FuncPtr", line: 49, size: 64, align: 64, offset: 128, file: !5, scope: !6, baseType: !30)
+!84 = !DIDerivedType(tag: DW_TAG_member, name: "__descriptor", line: 49, size: 64, align: 64, offset: 192, file: !5, scope: !6, baseType: !85)
+!85 = !DIDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, baseType: !86)
+!86 = !DICompositeType(tag: DW_TAG_structure_type, name: "__block_descriptor_withcopydispose", line: 49, flags: DIFlagFwdDecl, file: !1)
+!87 = !DIDerivedType(tag: DW_TAG_member, name: "self", line: 49, size: 64, align: 64, offset: 256, file: !5, scope: !6, baseType: !61)
+!88 = !DILocation(line: 49, scope: !27)
+!89 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "self", line: 52, scope: !27, file: !32, type: !23)
+!90 = !DILocation(line: 52, scope: !27)
+!91 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "d", line: 50, scope: !92, file: !6, type: !93)
+!92 = distinct !DILexicalBlock(line: 49, column: 0, file: !5, scope: !27)
+!93 = !DIDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, baseType: !94)
+!94 = !DICompositeType(tag: DW_TAG_structure_type, name: "NSMutableDictionary", line: 30, align: 8, runtimeLang: DW_LANG_ObjC, file: !5, scope: !6, elements: !95)
 !95 = !{!96}
-!96 = !MDDerivedType(tag: DW_TAG_inheritance, scope: !94, baseType: !97)
-!97 = !MDCompositeType(tag: DW_TAG_structure_type, name: "NSDictionary", line: 26, align: 8, runtimeLang: DW_LANG_ObjC, file: !5, scope: !6, elements: !98)
+!96 = !DIDerivedType(tag: DW_TAG_inheritance, scope: !94, baseType: !97)
+!97 = !DICompositeType(tag: DW_TAG_structure_type, name: "NSDictionary", line: 26, align: 8, runtimeLang: DW_LANG_ObjC, file: !5, scope: !6, elements: !98)
 !98 = !{!99}
-!99 = !MDDerivedType(tag: DW_TAG_inheritance, scope: !97, baseType: !9)
-!100 = !MDLocation(line: 50, scope: !92)
-!101 = !MDLocation(line: 51, scope: !92)
-!102 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "", line: 52, arg: 1, flags: DIFlagArtificial | DIFlagObjectPointer, scope: !31, file: !32, type: !30)
-!103 = !MDLocation(line: 52, scope: !31)
-!104 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "", line: 52, arg: 2, flags: DIFlagArtificial, scope: !31, file: !32, type: !30)
-!105 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "", line: 52, arg: 1, flags: DIFlagArtificial | DIFlagObjectPointer, scope: !35, file: !32, type: !30)
-!106 = !MDLocation(line: 52, scope: !35)
-!107 = !MDLocalVariable(tag: DW_TAG_auto_variable, name: "a", line: 61, scope: !36, file: !6, type: !61)
-!108 = !MDLocation(line: 61, scope: !36)
-!109 = !MDLocation(line: 62, scope: !36)
+!99 = !DIDerivedType(tag: DW_TAG_inheritance, scope: !97, baseType: !9)
+!100 = !DILocation(line: 50, scope: !92)
+!101 = !DILocation(line: 51, scope: !92)
+!102 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "", line: 52, arg: 1, flags: DIFlagArtificial | DIFlagObjectPointer, scope: !31, file: !32, type: !30)
+!103 = !DILocation(line: 52, scope: !31)
+!104 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "", line: 52, arg: 2, flags: DIFlagArtificial, scope: !31, file: !32, type: !30)
+!105 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "", line: 52, arg: 1, flags: DIFlagArtificial | DIFlagObjectPointer, scope: !35, file: !32, type: !30)
+!106 = !DILocation(line: 52, scope: !35)
+!107 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "a", line: 61, scope: !36, file: !6, type: !61)
+!108 = !DILocation(line: 61, scope: !36)
+!109 = !DILocation(line: 62, scope: !36)
 !110 = !{i32 1, !"Debug Info Version", i32 3}
-!111 = !MDExpression(DW_OP_deref, DW_OP_plus, 32)
+!111 = !DIExpression(DW_OP_deref, DW_OP_plus, 32)
diff --git a/test/DebugInfo/X86/debug-info-packed-struct.ll b/test/DebugInfo/X86/debug-info-packed-struct.ll
new file mode 100644
index 00000000000..6829c2d1374
--- /dev/null
+++ b/test/DebugInfo/X86/debug-info-packed-struct.ll
@@ -0,0 +1,198 @@
+; Generated from tools/clang/test/CodeGen/debug-info-packed-struct.c
+; ModuleID = 'llvm/tools/clang/test/CodeGen/debug-info-packed-struct.c'
+target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-apple-darwin"
+
+; RUN: %llc_dwarf -O0 -filetype=obj -o %t.o %s
+; RUN: llvm-dwarfdump -debug-dump=info %t.o | FileCheck %s
+; REQUIRES: object-emission
+
+;  // ---------------------------------------------------------------------
+;  // Not packed.
+;  // ---------------------------------------------------------------------
+;  struct size8 {
+;    int i : 4;
+;    long long l : 60;
+;  };
+;  struct layout0 {
+;    char l0_ofs0;
+;    struct size8 l0_ofs8;
+;    int l0_ofs16 : 1;
+;  } l0;
+
+%struct.layout0 = type { i8, %struct.size8, i8 }
+%struct.size8 = type { i64 }
+; CHECK:  DW_TAG_structure_type
+; CHECK:      DW_AT_name {{.*}} "layout0"
+; CHECK:      DW_AT_byte_size {{.*}} (0x18)
+; CHECK:      DW_TAG_member
+; CHECK:          DW_AT_name {{.*}} "l0_ofs0"
+; CHECK:          DW_AT_data_member_location {{.*}}00
+; CHECK:      DW_TAG_member
+; CHECK:          DW_AT_name {{.*}} "l0_ofs8"
+; CHECK:          DW_AT_data_member_location {{.*}}08
+; CHECK:      DW_TAG_member
+; CHECK:          DW_AT_name {{.*}} "l0_ofs16"
+; CHECK:          DW_AT_bit_size   {{.*}} (0x01)
+; CHECK:          DW_AT_bit_offset {{.*}} (0x1f)
+; CHECK:          DW_AT_data_member_location {{.*}}10
+
+
+; // ---------------------------------------------------------------------
+; // Implicitly packed.
+; // ---------------------------------------------------------------------
+; struct size8_anon {
+;   int : 4;
+;   long long : 60;
+; };
+; struct layout1 {
+;   char l1_ofs0;
+;   struct size8_anon l1_ofs1;
+;   int l1_ofs9 : 1;
+; } l1;
+
+%struct.layout1 = type <{ i8, %struct.size8_anon, i8, [2 x i8] }>
+%struct.size8_anon = type { i64 }
+
+; CHECK:  DW_TAG_structure_type
+; CHECK:      DW_AT_name {{.*}} "layout1"
+; CHECK:      DW_AT_byte_size {{.*}} (0x0c)
+; CHECK:      DW_TAG_member
+; CHECK:          DW_AT_name {{.*}} "l1_ofs0"
+; CHECK:          DW_AT_data_member_location {{.*}}00
+; CHECK:      DW_TAG_member
+; CHECK:          DW_AT_name {{.*}} "l1_ofs1"
+; CHECK:          DW_AT_data_member_location {{.*}}01
+; CHECK:      DW_TAG_member
+; CHECK:          DW_AT_name {{.*}} "l1_ofs9"
+; CHECK:          DW_AT_byte_size  {{.*}} (0x04)
+; CHECK:          DW_AT_bit_size   {{.*}} (0x01)
+; CHECK:          DW_AT_bit_offset {{.*}} (0x17)
+; CHECK:          DW_AT_data_member_location {{.*}}08
+
+; // ---------------------------------------------------------------------
+; // Explicitly packed.
+; // ---------------------------------------------------------------------
+; #pragma pack(1)
+; struct size8_pack1 {
+;   int i : 4;
+;   long long l : 60;
+; };
+; struct layout2 {
+;   char l2_ofs0;
+;   struct size8_pack1 l2_ofs1;
+;   int l2_ofs9 : 1;
+; } l2;
+; #pragma pack()
+
+%struct.layout2 = type <{ i8, %struct.size8_pack1, i8 }>
+%struct.size8_pack1 = type { i64 }
+
+; CHECK:  DW_TAG_structure_type
+; CHECK:      DW_AT_name {{.*}} "layout2"
+; CHECK:      DW_AT_byte_size {{.*}} (0x0a)
+; CHECK:      DW_TAG_member
+; CHECK:          DW_AT_name {{.*}} "l2_ofs0"
+; CHECK:          DW_AT_data_member_location {{.*}}00
+; CHECK:      DW_TAG_member
+; CHECK:          DW_AT_name {{.*}} "l2_ofs1"
+; CHECK:          DW_AT_data_member_location {{.*}}01
+; CHECK:      DW_TAG_member
+; CHECK:          DW_AT_name {{.*}} "l2_ofs9"
+; CHECK:          DW_AT_byte_size  {{.*}} (0x04)
+; CHECK:          DW_AT_bit_size   {{.*}} (0x01)
+; CHECK:          DW_AT_bit_offset {{.*}} (0x17)
+; CHECK:          DW_AT_data_member_location {{.*}}08
+
+; // ---------------------------------------------------------------------
+; // Explicitly packed with different alignment.
+; // ---------------------------------------------------------------------
+; #pragma pack(4)
+; struct size8_pack4 {
+;   int i : 4;
+;   long long l : 60;
+; };
+; struct layout3 {
+;   char l3_ofs0;
+;   struct size8_pack4 l3_ofs4;
+;   int l3_ofs12 : 1;
+; } l 3;
+; #pragma pack()
+
+
+%struct.layout3 = type <{ i8, [3 x i8], %struct.size8_pack4, i8, [3 x i8] }>
+%struct.size8_pack4 = type { i64 }
+
+; CHECK:  DW_TAG_structure_type
+; CHECK:      DW_AT_name {{.*}} "layout3"
+; CHECK:      DW_AT_byte_size {{.*}} (0x10)
+; CHECK:      DW_TAG_member
+; CHECK:          DW_AT_name {{.*}} "l3_ofs0"
+; CHECK:          DW_AT_data_member_location {{.*}}00
+; CHECK:      DW_TAG_member
+; CHECK:          DW_AT_name {{.*}} "l3_ofs4"
+; CHECK:          DW_AT_data_member_location {{.*}}04
+; CHECK:      DW_TAG_member
+; CHECK:          DW_AT_name {{.*}} "l3_ofs12"
+; CHECK:          DW_AT_byte_size  {{.*}} (0x04)
+; CHECK:          DW_AT_bit_size   {{.*}} (0x01)
+; CHECK:          DW_AT_bit_offset {{.*}} (0x1f)
+; CHECK:          DW_AT_data_member_location {{.*}}0c
+
+@l0 = common global %struct.layout0 zeroinitializer, align 8
+@l1 = common global %struct.layout1 zeroinitializer, align 4
+@l2 = common global %struct.layout2 zeroinitializer, align 1
+@l3 = common global %struct.layout3 zeroinitializer, align 4
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!45, !46}
+!llvm.ident = !{!47}
+
+!0 = !DICompileUnit(language: DW_LANG_C99, file: !1, producer: "clang version 3.7.0 (trunk 240791) (llvm/trunk 240790)", isOptimized: false, runtimeVersion: 0, emissionKind: 1, enums: !2, retainedTypes: !2, subprograms: !2, globals: !3, imports: !2)
+!1 = !DIFile(filename: "/llvm/tools/clang/test/CodeGen/<stdin>", directory: "/llvm/_build.ninja.release")
+!2 = !{}
+!3 = !{!4, !18, !25, !35}
+!4 = !DIGlobalVariable(name: "l0", scope: !0, file: !5, line: 88, type: !6, isLocal: false, isDefinition: true, variable: %struct.layout0* @l0)
+!5 = !DIFile(filename: "/llvm/tools/clang/test/CodeGen/debug-info-packed-struct.c", directory: "/llvm/_build.ninja.release")
+!6 = !DICompositeType(tag: DW_TAG_structure_type, name: "layout0", file: !5, line: 15, size: 192, align: 64, elements: !7)
+!7 = !{!8, !10, !17}
+!8 = !DIDerivedType(tag: DW_TAG_member, name: "l0_ofs0", scope: !6, file: !5, line: 16, baseType: !9, size: 8, align: 8)
+!9 = !DIBasicType(name: "char", size: 8, align: 8, encoding: DW_ATE_signed_char)
+!10 = !DIDerivedType(tag: DW_TAG_member, name: "l0_ofs8", scope: !6, file: !5, line: 17, baseType: !11, size: 64, align: 64, offset: 64)
+!11 = !DICompositeType(tag: DW_TAG_structure_type, name: "size8", file: !5, line: 11, size: 64, align: 64, elements: !12)
+!12 = !{!13, !15}
+!13 = !DIDerivedType(tag: DW_TAG_member, name: "i", scope: !11, file: !5, line: 12, baseType: !14, size: 4, align: 32)
+!14 = !DIBasicType(name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
+!15 = !DIDerivedType(tag: DW_TAG_member, name: "l", scope: !11, file: !5, line: 13, baseType: !16, size: 60, offset: 4)
+!16 = !DIBasicType(name: "long long int", size: 64, align: 64, encoding: DW_ATE_signed)
+!17 = !DIDerivedType(tag: DW_TAG_member, name: "l0_ofs16", scope: !6, file: !5, line: 18, baseType: !14, size: 1, align: 32, offset: 128)
+!18 = !DIGlobalVariable(name: "l1", scope: !0, file: !5, line: 89, type: !19, isLocal: false, isDefinition: true, variable: %struct.layout1* @l1)
+!19 = !DICompositeType(tag: DW_TAG_structure_type, name: "layout1", file: !5, line: 34, size: 96, align: 32, elements: !20)
+!20 = !{!21, !22, !24}
+!21 = !DIDerivedType(tag: DW_TAG_member, name: "l1_ofs0", scope: !19, file: !5, line: 35, baseType: !9, size: 8, align: 8)
+!22 = !DIDerivedType(tag: DW_TAG_member, name: "l1_ofs1", scope: !19, file: !5, line: 36, baseType: !23, size: 64, align: 8, offset: 8)
+!23 = !DICompositeType(tag: DW_TAG_structure_type, name: "size8_anon", file: !5, line: 30, size: 64, align: 8, elements: !2)
+!24 = !DIDerivedType(tag: DW_TAG_member, name: "l1_ofs9", scope: !19, file: !5, line: 37, baseType: !14, size: 1, align: 32, offset: 72)
+!25 = !DIGlobalVariable(name: "l2", scope: !0, file: !5, line: 90, type: !26, isLocal: false, isDefinition: true, variable: %struct.layout2* @l2)
+!26 = !DICompositeType(tag: DW_TAG_structure_type, name: "layout2", file: !5, line: 54, size: 80, align: 8, elements: !27)
+!27 = !{!28, !29, !34}
+!28 = !DIDerivedType(tag: DW_TAG_member, name: "l2_ofs0", scope: !26, file: !5, line: 55, baseType: !9, size: 8, align: 8)
+!29 = !DIDerivedType(tag: DW_TAG_member, name: "l2_ofs1", scope: !26, file: !5, line: 56, baseType: !30, size: 64, align: 8, offset: 8)
+!30 = !DICompositeType(tag: DW_TAG_structure_type, name: "size8_pack1", file: !5, line: 50, size: 64, align: 8, elements: !31)
+!31 = !{!32, !33}
+!32 = !DIDerivedType(tag: DW_TAG_member, name: "i", scope: !30, file: !5, line: 51, baseType: !14, size: 4, align: 32)
+!33 = !DIDerivedType(tag: DW_TAG_member, name: "l", scope: !30, file: !5, line: 52, baseType: !16, size: 60, offset: 4)
+!34 = !DIDerivedType(tag: DW_TAG_member, name: "l2_ofs9", scope: !26, file: !5, line: 57, baseType: !14, size: 1, align: 32, offset: 72)
+!35 = !DIGlobalVariable(name: "l3", scope: !0, file: !5, line: 91, type: !36, isLocal: false, isDefinition: true, variable: %struct.layout3* @l3)
+!36 = !DICompositeType(tag: DW_TAG_structure_type, name: "layout3", file: !5, line: 76, size: 128, align: 32, elements: !37)
+!37 = !{!38, !39, !44}
+!38 = !DIDerivedType(tag: DW_TAG_member, name: "l3_ofs0", scope: !36, file: !5, line: 77, baseType: !9, size: 8, align: 8)
+!39 = !DIDerivedType(tag: DW_TAG_member, name: "l3_ofs4", scope: !36, file: !5, line: 78, baseType: !40, size: 64, align: 32, offset: 32)
+!40 = !DICompositeType(tag: DW_TAG_structure_type, name: "size8_pack4", file: !5, line: 72, size: 64, align: 32, elements: !41)
+!41 = !{!42, !43}
+!42 = !DIDerivedType(tag: DW_TAG_member, name: "i", scope: !40, file: !5, line: 73, baseType: !14, size: 4, align: 32)
+!43 = !DIDerivedType(tag: DW_TAG_member, name: "l", scope: !40, file: !5, line: 74, baseType: !16, size: 60, offset: 4)
+!44 = !DIDerivedType(tag: DW_TAG_member, name: "l3_ofs12", scope: !36, file: !5, line: 79, baseType: !14, size: 1, align: 32, offset: 96)
+!45 = !{i32 2, !"Dwarf Version", i32 2}
+!46 = !{i32 2, !"Debug Info Version", i32 3}
+!47 = !{!"clang version 3.7.0 (trunk 240791) (llvm/trunk 240790)"}
diff --git a/test/DebugInfo/X86/debug-info-static-member.ll b/test/DebugInfo/X86/debug-info-static-member.ll
index 255827f4e7c..1dc85b95487 100644
--- a/test/DebugInfo/X86/debug-info-static-member.ll
+++ b/test/DebugInfo/X86/debug-info-static-member.ll
@@ -47,7 +47,7 @@ entry:
   %retval = alloca i32, align 4
   %instance_C = alloca %class.C, align 4
   store i32 0, i32* %retval
-  call void @llvm.dbg.declare(metadata %class.C* %instance_C, metadata !29, metadata !MDExpression()), !dbg !30
+  call void @llvm.dbg.declare(metadata %class.C* %instance_C, metadata !29, metadata !DIExpression()), !dbg !30
   %d = getelementptr inbounds %class.C, %class.C* %instance_C, i32 0, i32 0, !dbg !31
   store i32 8, i32* %d, align 4, !dbg !31
   %0 = load i32, i32* @_ZN1C1cE, align 4, !dbg !32
@@ -59,37 +59,37 @@ declare void @llvm.dbg.declare(metadata, metadata, metadata) nounwind readnone
 !llvm.dbg.cu = !{!0}
 !llvm.module.flags = !{!34}
 
-!0 = !MDCompileUnit(language: DW_LANG_C_plus_plus, producer: "clang version 3.3 (trunk 171914)", isOptimized: false, emissionKind: 0, file: !33, enums: !1, retainedTypes: !1, subprograms: !3, globals: !10, imports:  !1)
+!0 = !DICompileUnit(language: DW_LANG_C_plus_plus, producer: "clang version 3.3 (trunk 171914)", isOptimized: false, emissionKind: 0, file: !33, enums: !1, retainedTypes: !1, subprograms: !3, globals: !10, imports:  !1)
 !1 = !{}
 !3 = !{!5}
-!5 = !MDSubprogram(name: "main", line: 18, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 23, file: !33, scope: !6, type: !7, function: i32 ()* @main, variables: !1)
-!6 = !MDFile(filename: "/usr/local/google/home/blaikie/Development/llvm/src/tools/clang/test/CodeGenCXX/debug-info-static-member.cpp", directory: "/home/blaikie/local/Development/llvm/build/clang/x86-64/Debug/llvm")
-!7 = !MDSubroutineType(types: !8)
+!5 = !DISubprogram(name: "main", line: 18, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 23, file: !33, scope: !6, type: !7, function: i32 ()* @main, variables: !1)
+!6 = !DIFile(filename: "/usr/local/google/home/blaikie/Development/llvm/src/tools/clang/test/CodeGenCXX/debug-info-static-member.cpp", directory: "/home/blaikie/local/Development/llvm/build/clang/x86-64/Debug/llvm")
+!7 = !DISubroutineType(types: !8)
 !8 = !{!9}
-!9 = !MDBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
+!9 = !DIBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
 !10 = !{!12, !27, !28}
-!12 = !MDGlobalVariable(name: "a", linkageName: "_ZN1C1aE", line: 14, isLocal: false, isDefinition: true, scope: null, file: !6, type: !9, variable: i32* @_ZN1C1aE, declaration: !15)
-!13 = !MDCompositeType(tag: DW_TAG_class_type, name: "C", line: 1, size: 32, align: 32, file: !33, elements: !14)
+!12 = !DIGlobalVariable(name: "a", linkageName: "_ZN1C1aE", line: 14, isLocal: false, isDefinition: true, scope: null, file: !6, type: !9, variable: i32* @_ZN1C1aE, declaration: !15)
+!13 = !DICompositeType(tag: DW_TAG_class_type, name: "C", line: 1, size: 32, align: 32, file: !33, elements: !14)
 !14 = !{!15, !16, !19, !20, !23, !24, !26}
-!15 = !MDDerivedType(tag: DW_TAG_member, name: "a", line: 3, flags: DIFlagPrivate | DIFlagStaticMember, file: !33, scope: !13, baseType: !9)
-!16 = !MDDerivedType(tag: DW_TAG_member, name: "const_a", line: 4, flags: DIFlagPrivate | DIFlagStaticMember, file: !33, scope: !13, baseType: !17, extraData: i1 true)
-!17 = !MDDerivedType(tag: DW_TAG_const_type, baseType: !18)
-!18 = !MDBasicType(tag: DW_TAG_base_type, name: "bool", size: 8, align: 8, encoding: DW_ATE_boolean)
-!19 = !MDDerivedType(tag: DW_TAG_member, name: "b", line: 6, flags: DIFlagProtected | DIFlagStaticMember, file: !33, scope: !13, baseType: !9)
-!20 = !MDDerivedType(tag: DW_TAG_member, name: "const_b", line: 7, flags: DIFlagProtected | DIFlagStaticMember, file: !33, scope: !13, baseType: !21, extraData: float 0x40091EB860000000)
-!21 = !MDDerivedType(tag: DW_TAG_const_type, baseType: !22)
-!22 = !MDBasicType(tag: DW_TAG_base_type, name: "float", size: 32, align: 32, encoding: DW_ATE_float)
-!23 = !MDDerivedType(tag: DW_TAG_member, name: "c", line: 9, flags: DIFlagPublic | DIFlagStaticMember, file: !33, scope: !13, baseType: !9)
-!24 = !MDDerivedType(tag: DW_TAG_member, name: "const_c", line: 10, flags: DIFlagPublic | DIFlagStaticMember, file: !33, scope: !13, baseType: !25, extraData: i32 18)
-!25 = !MDDerivedType(tag: DW_TAG_const_type, baseType: !9)
-!26 = !MDDerivedType(tag: DW_TAG_member, name: "d", line: 11, size: 32, align: 32, flags: DIFlagPublic, file: !33, scope: !13, baseType: !9)
-!27 = !MDGlobalVariable(name: "b", linkageName: "_ZN1C1bE", line: 15, isLocal: false, isDefinition: true, scope: null, file: !6, type: !9, variable: i32* @_ZN1C1bE, declaration: !19)
-!28 = !MDGlobalVariable(name: "c", linkageName: "_ZN1C1cE", line: 16, isLocal: false, isDefinition: true, scope: null, file: !6, type: !9, variable: i32* @_ZN1C1cE, declaration: !23)
-!29 = !MDLocalVariable(tag: DW_TAG_auto_variable, name: "instance_C", line: 20, scope: !5, file: !6, type: !13)
-!30 = !MDLocation(line: 20, scope: !5)
-!31 = !MDLocation(line: 21, scope: !5)
-!32 = !MDLocation(line: 22, scope: !5)
-!33 = !MDFile(filename: "/usr/local/google/home/blaikie/Development/llvm/src/tools/clang/test/CodeGenCXX/debug-info-static-member.cpp", directory: "/home/blaikie/local/Development/llvm/build/clang/x86-64/Debug/llvm")
+!15 = !DIDerivedType(tag: DW_TAG_member, name: "a", line: 3, flags: DIFlagPrivate | DIFlagStaticMember, file: !33, scope: !13, baseType: !9)
+!16 = !DIDerivedType(tag: DW_TAG_member, name: "const_a", line: 4, flags: DIFlagPrivate | DIFlagStaticMember, file: !33, scope: !13, baseType: !17, extraData: i1 true)
+!17 = !DIDerivedType(tag: DW_TAG_const_type, baseType: !18)
+!18 = !DIBasicType(tag: DW_TAG_base_type, name: "bool", size: 8, align: 8, encoding: DW_ATE_boolean)
+!19 = !DIDerivedType(tag: DW_TAG_member, name: "b", line: 6, flags: DIFlagProtected | DIFlagStaticMember, file: !33, scope: !13, baseType: !9)
+!20 = !DIDerivedType(tag: DW_TAG_member, name: "const_b", line: 7, flags: DIFlagProtected | DIFlagStaticMember, file: !33, scope: !13, baseType: !21, extraData: float 0x40091EB860000000)
+!21 = !DIDerivedType(tag: DW_TAG_const_type, baseType: !22)
+!22 = !DIBasicType(tag: DW_TAG_base_type, name: "float", size: 32, align: 32, encoding: DW_ATE_float)
+!23 = !DIDerivedType(tag: DW_TAG_member, name: "c", line: 9, flags: DIFlagPublic | DIFlagStaticMember, file: !33, scope: !13, baseType: !9)
+!24 = !DIDerivedType(tag: DW_TAG_member, name: "const_c", line: 10, flags: DIFlagPublic | DIFlagStaticMember, file: !33, scope: !13, baseType: !25, extraData: i32 18)
+!25 = !DIDerivedType(tag: DW_TAG_const_type, baseType: !9)
+!26 = !DIDerivedType(tag: DW_TAG_member, name: "d", line: 11, size: 32, align: 32, flags: DIFlagPublic, file: !33, scope: !13, baseType: !9)
+!27 = !DIGlobalVariable(name: "b", linkageName: "_ZN1C1bE", line: 15, isLocal: false, isDefinition: true, scope: null, file: !6, type: !9, variable: i32* @_ZN1C1bE, declaration: !19)
+!28 = !DIGlobalVariable(name: "c", linkageName: "_ZN1C1cE", line: 16, isLocal: false, isDefinition: true, scope: null, file: !6, type: !9, variable: i32* @_ZN1C1cE, declaration: !23)
+!29 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "instance_C", line: 20, scope: !5, file: !6, type: !13)
+!30 = !DILocation(line: 20, scope: !5)
+!31 = !DILocation(line: 21, scope: !5)
+!32 = !DILocation(line: 22, scope: !5)
+!33 = !DIFile(filename: "/usr/local/google/home/blaikie/Development/llvm/src/tools/clang/test/CodeGenCXX/debug-info-static-member.cpp", directory: "/home/blaikie/local/Development/llvm/build/clang/x86-64/Debug/llvm")
 ; PRESENT verifies that static member declarations have these attributes:
 ; external, declaration, accessibility, and either DW_AT_linkage_name
 ; (for variables) or DW_AT_const_value (for constants).
diff --git a/test/DebugInfo/X86/debug-loc-asan.ll b/test/DebugInfo/X86/debug-loc-asan.ll
index a30c13f0684..68bef490480 100644
--- a/test/DebugInfo/X86/debug-loc-asan.ll
+++ b/test/DebugInfo/X86/debug-loc-asan.ll
@@ -77,7 +77,7 @@ entry:
   %21 = inttoptr i64 %20 to i8*
   %22 = load i8, i8* %21
   %23 = icmp ne i8 %22, 0
-  call void @llvm.dbg.declare(metadata i32* %8, metadata !12, metadata !14), !dbg !MDLocation(scope: !4)
+  call void @llvm.dbg.declare(metadata i32* %8, metadata !12, metadata !14), !dbg !DILocation(scope: !4)
   br i1 %23, label %24, label %30
 
 ; <label>:24                                      ; preds = %5
@@ -165,18 +165,18 @@ attributes #1 = { nounwind readnone }
 !llvm.module.flags = !{!9, !10}
 !llvm.ident = !{!11}
 
-!0 = !MDCompileUnit(language: DW_LANG_C_plus_plus, producer: "clang version 3.5.0 (209308)", isOptimized: false, emissionKind: 1, file: !1, enums: !2, retainedTypes: !2, subprograms: !3, globals: !2, imports: !2)
-!1 = !MDFile(filename: "test.cc", directory: "/llvm_cmake_gcc")
+!0 = !DICompileUnit(language: DW_LANG_C_plus_plus, producer: "clang version 3.5.0 (209308)", isOptimized: false, emissionKind: 1, file: !1, enums: !2, retainedTypes: !2, subprograms: !3, globals: !2, imports: !2)
+!1 = !DIFile(filename: "test.cc", directory: "/llvm_cmake_gcc")
 !2 = !{}
 !3 = !{!4}
-!4 = !MDSubprogram(name: "bar", linkageName: "_Z3bari", line: 1, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 1, file: !1, scope: !5, type: !6, function: i32 (i32)* @_Z3bari, variables: !2)
-!5 = !MDFile(filename: "test.cc", directory: "/llvm_cmake_gcc")
-!6 = !MDSubroutineType(types: !7)
+!4 = !DISubprogram(name: "bar", linkageName: "_Z3bari", line: 1, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 1, file: !1, scope: !5, type: !6, function: i32 (i32)* @_Z3bari, variables: !2)
+!5 = !DIFile(filename: "test.cc", directory: "/llvm_cmake_gcc")
+!6 = !DISubroutineType(types: !7)
 !7 = !{!8, !8}
-!8 = !MDBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
+!8 = !DIBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
 !9 = !{i32 2, !"Dwarf Version", i32 4}
 !10 = !{i32 2, !"Debug Info Version", i32 3}
 !11 = !{!"clang version 3.5.0 (209308)"}
-!12 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "y", line: 1, arg: 1, scope: !4, file: !5, type: !8)
-!13 = !MDLocation(line: 2, scope: !4)
-!14 = !MDExpression(DW_OP_deref)
+!12 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "y", line: 1, arg: 1, scope: !4, file: !5, type: !8)
+!13 = !DILocation(line: 2, scope: !4)
+!14 = !DIExpression(DW_OP_deref)
diff --git a/test/DebugInfo/X86/debug-loc-empty-entries.ll b/test/DebugInfo/X86/debug-loc-empty-entries.ll
new file mode 100644
index 00000000000..3b997fd35e0
--- /dev/null
+++ b/test/DebugInfo/X86/debug-loc-empty-entries.ll
@@ -0,0 +1,66 @@
+; RUN: llc -mtriple=x86_64-apple-macosx <%s | FileCheck %s
+; Test that we don't generate empty .debug_loc entries.  Originally, there were
+; two empty .debug_loc entries for 'a' in an otherwise empty .debug_loc list.
+;
+; CHECK:      .section __DWARF,__debug_loc,regular,debug
+; CHECK-NEXT: Lsection_debug_loc:
+; CHECK-NEXT: .section __DWARF,__debug_abbrev,regular,debug
+;
+; Test that the variable stuck around.
+; CHECK:      .section __DWARF,__debug_info,regular,debug
+; CHECK:      DW_TAG_variable
+; CHECK-NOT:  DW_AT_location
+
+; Generated using clang -cc1 with the following args:
+;
+;   -triple x86_64-apple-macosx -emit-llvm -gdwarf-4 -O1
+;
+; From this testcase:
+;
+;;   void fn1() {
+;;     float a = 1;
+;;     for (;;)
+;;       a = 0;
+;;   }
+
+; Function Attrs: noreturn nounwind readnone
+define void @_Z3fn1v() #0 {
+entry:
+  tail call void @llvm.dbg.value(metadata float 1.000000e+00, i64 0, metadata !9, metadata !14), !dbg !15
+  br label %for.cond, !dbg !16
+
+for.cond:                                         ; preds = %for.cond, %entry
+  tail call void @llvm.dbg.value(metadata float 0.000000e+00, i64 0, metadata !9, metadata !14), !dbg !15
+  br label %for.cond, !dbg !17
+}
+
+; Function Attrs: nounwind readnone
+declare void @llvm.dbg.value(metadata, i64, metadata, metadata) #1
+
+attributes #0 = { noreturn nounwind readnone "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "no-realign-stack" "stack-protector-buffer-size"="8" "target-features"="+sse,+sse2" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #1 = { nounwind readnone }
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!11, !12}
+!llvm.ident = !{!13}
+
+!0 = !DICompileUnit(language: DW_LANG_C_plus_plus, file: !1, producer: "clang version 3.7.0 (trunk 238517) (llvm/trunk 238524)", isOptimized: true, runtimeVersion: 0, emissionKind: 1, enums: !2, retainedTypes: !2, subprograms: !3, globals: !2, imports: !2)
+!1 = !DIFile(filename: "<stdin>", directory: "/Users/dexonsmith/data/llvm/bootstrap/play/delta2/testcase")
+!2 = !{}
+!3 = !{!4}
+!4 = !DISubprogram(name: "fn1", linkageName: "_Z3fn1v", scope: !5, file: !5, line: 1, type: !6, isLocal: false, isDefinition: true, scopeLine: 1, flags: DIFlagPrototyped, isOptimized: true, function: void ()* @_Z3fn1v, variables: !8)
+!5 = !DIFile(filename: "t.cpp", directory: "/Users/dexonsmith/data/llvm/bootstrap/play/delta2/testcase")
+!6 = !DISubroutineType(types: !7)
+!7 = !{null}
+!8 = !{!9}
+!9 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "a", scope: !4, file: !5, line: 2, type: !10)
+!10 = !DIBasicType(name: "float", size: 32, align: 32, encoding: DW_ATE_float)
+!11 = !{i32 2, !"Dwarf Version", i32 4}
+!12 = !{i32 2, !"Debug Info Version", i32 3}
+!13 = !{!"clang version 3.7.0 (trunk 238517) (llvm/trunk 238524)"}
+!14 = !DIExpression()
+!15 = !DILocation(line: 2, scope: !4)
+!16 = !DILocation(line: 3, scope: !4)
+!17 = !DILocation(line: 3, scope: !18)
+!18 = distinct !DILexicalBlock(scope: !19, file: !5, line: 3)
+!19 = distinct !DILexicalBlock(scope: !4, file: !5, line: 3)
diff --git a/test/DebugInfo/X86/debug-loc-offset.ll b/test/DebugInfo/X86/debug-loc-offset.ll
index 56a0c7f4e9b..cf2fcc8e222 100644
--- a/test/DebugInfo/X86/debug-loc-offset.ll
+++ b/test/DebugInfo/X86/debug-loc-offset.ll
@@ -64,7 +64,7 @@ define i32 @_Z3bari(i32 %b) #0 {
 entry:
   %b.addr = alloca i32, align 4
   store i32 %b, i32* %b.addr, align 4
-  call void @llvm.dbg.declare(metadata i32* %b.addr, metadata !21, metadata !MDExpression()), !dbg !22
+  call void @llvm.dbg.declare(metadata i32* %b.addr, metadata !21, metadata !DIExpression()), !dbg !22
   %0 = load i32, i32* %b.addr, align 4, !dbg !23
   %add = add nsw i32 %0, 4, !dbg !23
   ret i32 %add, !dbg !23
@@ -76,8 +76,8 @@ declare void @llvm.dbg.declare(metadata, metadata, metadata) #1
 define void @_Z3baz1A(%struct.A* %a) #2 {
 entry:
   %z = alloca i32, align 4
-  call void @llvm.dbg.declare(metadata %struct.A* %a, metadata !24, metadata !MDExpression(DW_OP_deref)), !dbg !25
-  call void @llvm.dbg.declare(metadata i32* %z, metadata !26, metadata !MDExpression()), !dbg !27
+  call void @llvm.dbg.declare(metadata %struct.A* %a, metadata !24, metadata !DIExpression(DW_OP_deref)), !dbg !25
+  call void @llvm.dbg.declare(metadata i32* %z, metadata !26, metadata !DIExpression()), !dbg !27
   store i32 2, i32* %z, align 4, !dbg !27
   %var = getelementptr inbounds %struct.A, %struct.A* %a, i32 0, i32 1, !dbg !28
   %0 = load i32, i32* %var, align 4, !dbg !28
@@ -116,38 +116,38 @@ attributes #2 = { "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "n
 !llvm.module.flags = !{!18, !19}
 !llvm.ident = !{!20, !20}
 
-!0 = !MDCompileUnit(language: DW_LANG_C_plus_plus, producer: "clang version 3.5.0 (210479)", isOptimized: false, emissionKind: 1, file: !1, enums: !2, retainedTypes: !2, subprograms: !3, globals: !2, imports: !2)
-!1 = !MDFile(filename: "debug-loc-offset1.cc", directory: "/llvm_cmake_gcc")
+!0 = !DICompileUnit(language: DW_LANG_C_plus_plus, producer: "clang version 3.5.0 (210479)", isOptimized: false, emissionKind: 1, file: !1, enums: !2, retainedTypes: !2, subprograms: !3, globals: !2, imports: !2)
+!1 = !DIFile(filename: "debug-loc-offset1.cc", directory: "/llvm_cmake_gcc")
 !2 = !{}
 !3 = !{!4}
-!4 = !MDSubprogram(name: "bar", linkageName: "_Z3bari", line: 1, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 1, file: !1, scope: !5, type: !6, function: i32 (i32)* @_Z3bari, variables: !2)
-!5 = !MDFile(filename: "debug-loc-offset1.cc", directory: "/llvm_cmake_gcc")
-!6 = !MDSubroutineType(types: !7)
+!4 = !DISubprogram(name: "bar", linkageName: "_Z3bari", line: 1, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 1, file: !1, scope: !5, type: !6, function: i32 (i32)* @_Z3bari, variables: !2)
+!5 = !DIFile(filename: "debug-loc-offset1.cc", directory: "/llvm_cmake_gcc")
+!6 = !DISubroutineType(types: !7)
 !7 = !{!8, !8}
-!8 = !MDBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
-!9 = !MDCompileUnit(language: DW_LANG_C_plus_plus, producer: "clang version 3.5.0 (210479)", isOptimized: false, emissionKind: 1, file: !10, enums: !2, retainedTypes: !11, subprograms: !13, globals: !2, imports: !2)
-!10 = !MDFile(filename: "debug-loc-offset2.cc", directory: "/llvm_cmake_gcc")
+!8 = !DIBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
+!9 = !DICompileUnit(language: DW_LANG_C_plus_plus, producer: "clang version 3.5.0 (210479)", isOptimized: false, emissionKind: 1, file: !10, enums: !2, retainedTypes: !11, subprograms: !13, globals: !2, imports: !2)
+!10 = !DIFile(filename: "debug-loc-offset2.cc", directory: "/llvm_cmake_gcc")
 !11 = !{!12}
-!12 = !MDCompositeType(tag: DW_TAG_structure_type, name: "A", line: 1, flags: DIFlagFwdDecl, file: !10, identifier: "_ZTS1A")
+!12 = !DICompositeType(tag: DW_TAG_structure_type, name: "A", line: 1, flags: DIFlagFwdDecl, file: !10, identifier: "_ZTS1A")
 !13 = !{!14}
-!14 = !MDSubprogram(name: "baz", linkageName: "_Z3baz1A", line: 6, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 6, file: !10, scope: !15, type: !16, function: void (%struct.A*)* @_Z3baz1A, variables: !2)
-!15 = !MDFile(filename: "debug-loc-offset2.cc", directory: "/llvm_cmake_gcc")
-!16 = !MDSubroutineType(types: !17)
+!14 = !DISubprogram(name: "baz", linkageName: "_Z3baz1A", line: 6, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 6, file: !10, scope: !15, type: !16, function: void (%struct.A*)* @_Z3baz1A, variables: !2)
+!15 = !DIFile(filename: "debug-loc-offset2.cc", directory: "/llvm_cmake_gcc")
+!16 = !DISubroutineType(types: !17)
 !17 = !{null, !12}
 !18 = !{i32 2, !"Dwarf Version", i32 4}
 !19 = !{i32 2, !"Debug Info Version", i32 3}
 !20 = !{!"clang version 3.5.0 (210479)"}
-!21 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "b", line: 1, arg: 1, scope: !4, file: !5, type: !8)
-!22 = !MDLocation(line: 1, scope: !4)
-!23 = !MDLocation(line: 2, scope: !4)
-!24 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "a", line: 6, arg: 1, scope: !14, file: !15, type: !"_ZTS1A")
-!25 = !MDLocation(line: 6, scope: !14)
-!26 = !MDLocalVariable(tag: DW_TAG_auto_variable, name: "z", line: 7, scope: !14, file: !15, type: !8)
-!27 = !MDLocation(line: 7, scope: !14)
-!28 = !MDLocation(line: 8, scope: !29)
-!29 = distinct !MDLexicalBlock(line: 8, column: 0, file: !10, scope: !14)
-!30 = !MDLocation(line: 9, scope: !29)
-!31 = !MDLocation(line: 10, scope: !32)
-!32 = distinct !MDLexicalBlock(line: 10, column: 0, file: !10, scope: !14)
-!33 = !MDLocation(line: 11, scope: !32)
-!34 = !MDLocation(line: 12, scope: !14)
+!21 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "b", line: 1, arg: 1, scope: !4, file: !5, type: !8)
+!22 = !DILocation(line: 1, scope: !4)
+!23 = !DILocation(line: 2, scope: !4)
+!24 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "a", line: 6, arg: 1, scope: !14, file: !15, type: !"_ZTS1A")
+!25 = !DILocation(line: 6, scope: !14)
+!26 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "z", line: 7, scope: !14, file: !15, type: !8)
+!27 = !DILocation(line: 7, scope: !14)
+!28 = !DILocation(line: 8, scope: !29)
+!29 = distinct !DILexicalBlock(line: 8, column: 0, file: !10, scope: !14)
+!30 = !DILocation(line: 9, scope: !29)
+!31 = !DILocation(line: 10, scope: !32)
+!32 = distinct !DILexicalBlock(line: 10, column: 0, file: !10, scope: !14)
+!33 = !DILocation(line: 11, scope: !32)
+!34 = !DILocation(line: 12, scope: !14)
diff --git a/test/DebugInfo/X86/debug-ranges-offset.ll b/test/DebugInfo/X86/debug-ranges-offset.ll
index 55872db280e..19c3bcf3748 100644
--- a/test/DebugInfo/X86/debug-ranges-offset.ll
+++ b/test/DebugInfo/X86/debug-ranges-offset.ll
@@ -31,11 +31,11 @@ entry:
   %call = call i8* @_Znwm(i64 4) #4, !dbg !19
   %_msret = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @__msan_retval_tls, i64 0, i64 0), align 8, !dbg !19
   %3 = bitcast i8* %call to i32*, !dbg !19
-  tail call void @llvm.dbg.value(metadata i32* %3, i64 0, metadata !9, metadata !MDExpression()), !dbg !19
+  tail call void @llvm.dbg.value(metadata i32* %3, i64 0, metadata !9, metadata !DIExpression()), !dbg !19
   %4 = inttoptr i64 %1 to i64*, !dbg !19
   store i64 %_msret, i64* %4, align 8, !dbg !19
   store volatile i32* %3, i32** %p, align 8, !dbg !19
-  tail call void @llvm.dbg.value(metadata i32** %p, i64 0, metadata !9, metadata !MDExpression()), !dbg !19
+  tail call void @llvm.dbg.value(metadata i32** %p, i64 0, metadata !9, metadata !DIExpression()), !dbg !19
   %p.0.p.0. = load volatile i32*, i32** %p, align 8, !dbg !20
   %_msld = load i64, i64* %4, align 8, !dbg !20
   %_mscmp = icmp eq i64 %_msld, 0, !dbg !20
@@ -96,11 +96,11 @@ entry:
   %call.i = call i8* @_Znwm(i64 4) #4, !dbg !30
   %_msret = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @__msan_retval_tls, i64 0, i64 0), align 8, !dbg !30
   %3 = bitcast i8* %call.i to i32*, !dbg !30
-  tail call void @llvm.dbg.value(metadata i32* %3, i64 0, metadata !32, metadata !MDExpression()), !dbg !30
+  tail call void @llvm.dbg.value(metadata i32* %3, i64 0, metadata !32, metadata !DIExpression()), !dbg !30
   %4 = inttoptr i64 %1 to i64*, !dbg !30
   store i64 %_msret, i64* %4, align 8, !dbg !30
   store volatile i32* %3, i32** %p.i, align 8, !dbg !30
-  tail call void @llvm.dbg.value(metadata i32** %p.i, i64 0, metadata !32, metadata !MDExpression()), !dbg !30
+  tail call void @llvm.dbg.value(metadata i32** %p.i, i64 0, metadata !32, metadata !DIExpression()), !dbg !30
   %p.i.0.p.0.p.0..i = load volatile i32*, i32** %p.i, align 8, !dbg !33
   %_msld = load i64, i64* %4, align 8, !dbg !33
   %_mscmp = icmp eq i64 %_msld, 0, !dbg !33
@@ -202,40 +202,40 @@ attributes #4 = { builtin }
 !llvm.module.flags = !{!16, !17}
 !llvm.ident = !{!18}
 
-!0 = !MDCompileUnit(language: DW_LANG_C_plus_plus, producer: "clang version 3.5.0 (trunk 207243) (llvm/trunk 207259)", isOptimized: true, emissionKind: 1, file: !1, enums: !2, retainedTypes: !2, subprograms: !3, globals: !2, imports: !2)
-!1 = !MDFile(filename: "foo.cpp", directory: "/usr/local/google/home/echristo/tmp")
+!0 = !DICompileUnit(language: DW_LANG_C_plus_plus, producer: "clang version 3.5.0 (trunk 207243) (llvm/trunk 207259)", isOptimized: true, emissionKind: 1, file: !1, enums: !2, retainedTypes: !2, subprograms: !3, globals: !2, imports: !2)
+!1 = !DIFile(filename: "foo.cpp", directory: "/usr/local/google/home/echristo/tmp")
 !2 = !{}
 !3 = !{!4, !13}
-!4 = !MDSubprogram(name: "f", linkageName: "_Z1fv", line: 3, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: true, scopeLine: 3, file: !1, scope: !5, type: !6, function: void ()* @_Z1fv, variables: !8)
-!5 = !MDFile(filename: "foo.cpp", directory: "/usr/local/google/home/echristo/tmp")
-!6 = !MDSubroutineType(types: !7)
+!4 = !DISubprogram(name: "f", linkageName: "_Z1fv", line: 3, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: true, scopeLine: 3, file: !1, scope: !5, type: !6, function: void ()* @_Z1fv, variables: !8)
+!5 = !DIFile(filename: "foo.cpp", directory: "/usr/local/google/home/echristo/tmp")
+!6 = !DISubroutineType(types: !7)
 !7 = !{null}
 !8 = !{!9}
-!9 = !MDLocalVariable(tag: DW_TAG_auto_variable, name: "p", line: 4, scope: !4, file: !5, type: !10)
-!10 = !MDDerivedType(tag: DW_TAG_volatile_type, baseType: !11)
-!11 = !MDDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, baseType: !12)
-!12 = !MDBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
-!13 = !MDSubprogram(name: "main", line: 9, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: true, scopeLine: 9, file: !1, scope: !5, type: !14, function: i32 ()* @main, variables: !2)
-!14 = !MDSubroutineType(types: !15)
+!9 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "p", line: 4, scope: !4, file: !5, type: !10)
+!10 = !DIDerivedType(tag: DW_TAG_volatile_type, baseType: !11)
+!11 = !DIDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, baseType: !12)
+!12 = !DIBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
+!13 = !DISubprogram(name: "main", line: 9, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: true, scopeLine: 9, file: !1, scope: !5, type: !14, function: i32 ()* @main, variables: !2)
+!14 = !DISubroutineType(types: !15)
 !15 = !{!12}
 !16 = !{i32 2, !"Dwarf Version", i32 4}
 !17 = !{i32 1, !"Debug Info Version", i32 3}
 !18 = !{!"clang version 3.5.0 (trunk 207243) (llvm/trunk 207259)"}
-!19 = !MDLocation(line: 4, scope: !4)
-!20 = !MDLocation(line: 5, scope: !21)
-!21 = distinct !MDLexicalBlock(line: 5, column: 0, file: !1, scope: !4)
+!19 = !DILocation(line: 4, scope: !4)
+!20 = !DILocation(line: 5, scope: !21)
+!21 = distinct !DILexicalBlock(line: 5, column: 0, file: !1, scope: !4)
 !22 = !{!"branch_weights", i32 1000, i32 1}
 !23 = !{!24, !24, i64 0}
 !24 = !{!"int", !25, i64 0}
 !25 = !{!"omnipotent char", !26, i64 0}
 !26 = !{!"Simple C/C++ TBAA"}
 !27 = !{!"branch_weights", i32 1, i32 1000}
-!28 = !MDLocation(line: 6, scope: !21)
-!29 = !MDLocation(line: 7, scope: !4)
-!30 = !MDLocation(line: 4, scope: !4, inlinedAt: !31)
-!31 = !MDLocation(line: 10, scope: !13)
-!32 = !MDLocalVariable(tag: DW_TAG_auto_variable, name: "p", line: 4, scope: !4, file: !5, type: !10)
-!33 = !MDLocation(line: 5, scope: !21, inlinedAt: !31)
-!34 = !MDLocation(line: 6, scope: !21, inlinedAt: !31)
-!35 = !MDLocation(line: 7, scope: !4, inlinedAt: !31)
-!36 = !MDLocation(line: 11, scope: !13)
+!28 = !DILocation(line: 6, scope: !21)
+!29 = !DILocation(line: 7, scope: !4)
+!30 = !DILocation(line: 4, scope: !4, inlinedAt: !31)
+!31 = !DILocation(line: 10, scope: !13)
+!32 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "p", line: 4, scope: !4, file: !5, type: !10)
+!33 = !DILocation(line: 5, scope: !21, inlinedAt: !31)
+!34 = !DILocation(line: 6, scope: !21, inlinedAt: !31)
+!35 = !DILocation(line: 7, scope: !4, inlinedAt: !31)
+!36 = !DILocation(line: 11, scope: !13)
diff --git a/test/DebugInfo/X86/debug_frame.ll b/test/DebugInfo/X86/debug_frame.ll
index 49aea75a063..48ce554f011 100644
--- a/test/DebugInfo/X86/debug_frame.ll
+++ b/test/DebugInfo/X86/debug_frame.ll
@@ -13,10 +13,10 @@ entry:
 !llvm.module.flags = !{!7}
 !5 = !{!0}
 
-!0 = !MDSubprogram(name: "f", line: 1, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: true, scopeLine: 1, file: !6, scope: !1, type: !3, function: void ()* @f)
-!1 = !MDFile(filename: "/home/espindola/llvm/test.c", directory: "/home/espindola/llvm/build")
-!2 = !MDCompileUnit(language: DW_LANG_C99, producer: "clang version 3.0 ()", isOptimized: true, emissionKind: 0, file: !6, enums: !{}, retainedTypes: !{}, subprograms: !5)
-!3 = !MDSubroutineType(types: !4)
+!0 = !DISubprogram(name: "f", line: 1, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: true, scopeLine: 1, file: !6, scope: !1, type: !3, function: void ()* @f)
+!1 = !DIFile(filename: "/home/espindola/llvm/test.c", directory: "/home/espindola/llvm/build")
+!2 = !DICompileUnit(language: DW_LANG_C99, producer: "clang version 3.0 ()", isOptimized: true, emissionKind: 0, file: !6, enums: !{}, retainedTypes: !{}, subprograms: !5)
+!3 = !DISubroutineType(types: !4)
 !4 = !{null}
-!6 = !MDFile(filename: "/home/espindola/llvm/test.c", directory: "/home/espindola/llvm/build")
+!6 = !DIFile(filename: "/home/espindola/llvm/test.c", directory: "/home/espindola/llvm/build")
 !7 = !{i32 1, !"Debug Info Version", i32 3}
diff --git a/test/DebugInfo/X86/debugger-tune.ll b/test/DebugInfo/X86/debugger-tune.ll
new file mode 100644
index 00000000000..7d862e31517
--- /dev/null
+++ b/test/DebugInfo/X86/debugger-tune.ll
@@ -0,0 +1,44 @@
+; Verify target-based defaults for "debugger tuning," and the ability to
+; override defaults.
+; We use existence of the debug_pubnames section to distinguish the GDB case,
+; and the apple_names section to distinguish the LLDB case. SCE has neither.
+
+; Verify defaults for various targets.
+; RUN: llc -mtriple=x86_64-scei-ps4 -filetype=obj < %s | llvm-readobj -sections - | FileCheck --check-prefix=SCE %s
+; RUN: llc -mtriple=x86_64-apple-darwin12 -filetype=obj < %s | llvm-readobj -sections - | FileCheck --check-prefix=LLDB %s
+; RUN: llc -mtriple=x86_64-pc-freebsd -filetype=obj < %s | llvm-readobj -sections - | FileCheck --check-prefix=LLDB %s
+; RUN: llc -mtriple=x86_64-pc-linux -filetype=obj < %s | llvm-readobj -sections - | FileCheck --check-prefix=GDB %s
+
+; We can override defaults.
+; RUN: llc -mtriple=x86_64-scei-ps4 -filetype=obj -debugger-tune=gdb < %s | llvm-readobj -sections - | FileCheck --check-prefix=GDB %s
+; RUN: llc -mtriple=x86_64-pc-linux -filetype=obj -debugger-tune=lldb < %s | llvm-readobj -sections - | FileCheck --check-prefix=LLDB %s
+; RUN: llc -mtriple=x86_64-apple-darwin12 -filetype=obj -debugger-tune=sce < %s | llvm-readobj -sections - | FileCheck --check-prefix=SCE %s
+
+
+; GDB-NOT: apple_names
+; GDB: debug_pubnames
+; GDB-NOT: apple_names
+
+; LLDB-NOT: debug_pubnames
+; LLDB: apple_names
+; LLDB-NOT: debug_pubnames
+
+; SCE-NOT: debug_pubnames
+; SCE-NOT: apple_names
+
+
+@globalvar = global i32 0, align 4
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!6, !7}
+!llvm.ident = !{!8}
+
+!0 = !DICompileUnit(language: DW_LANG_C_plus_plus, file: !1, producer: "clang version 3.7.0 (trunk 238808)", isOptimized: false, runtimeVersion: 0, emissionKind: 1, enums: !2, retainedTypes: !2, subprograms: !2, globals: !3, imports: !2)
+!1 = !DIFile(filename: "debugger-tune.cpp", directory: "/home/probinson/projects/scratch")
+!2 = !{}
+!3 = !{!4}
+!4 = !DIGlobalVariable(name: "globalvar", scope: !0, file: !1, line: 1, type: !5, isLocal: false, isDefinition: true, variable: i32* @globalvar)
+!5 = !DIBasicType(name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
+!6 = !{i32 2, !"Dwarf Version", i32 4}
+!7 = !{i32 2, !"Debug Info Version", i32 3}
+!8 = !{!"clang version 3.7.0 (trunk 238808)"}
diff --git a/test/DebugInfo/X86/decl-derived-member.ll b/test/DebugInfo/X86/decl-derived-member.ll
index 63cae4988af..3b833580e9c 100644
--- a/test/DebugInfo/X86/decl-derived-member.ll
+++ b/test/DebugInfo/X86/decl-derived-member.ll
@@ -100,52 +100,52 @@ attributes #4 = { "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "n
 !llvm.module.flags = !{!30, !31}
 !llvm.ident = !{!32}
 
-!0 = !MDCompileUnit(language: DW_LANG_C_plus_plus, producer: "clang version 3.7.0 (trunk 227104) (llvm/trunk 227103)", isOptimized: false, emissionKind: 1, file: !1, enums: !2, retainedTypes: !3, subprograms: !9, globals: !28, imports: !2)
-!1 = !MDFile(filename: "decl-derived-member.cpp", directory: "/tmp/dbginfo")
+!0 = !DICompileUnit(language: DW_LANG_C_plus_plus, producer: "clang version 3.7.0 (trunk 227104) (llvm/trunk 227103)", isOptimized: false, emissionKind: 1, file: !1, enums: !2, retainedTypes: !3, subprograms: !9, globals: !28, imports: !2)
+!1 = !DIFile(filename: "decl-derived-member.cpp", directory: "/tmp/dbginfo")
 !2 = !{}
 !3 = !{!4, !8}
-!4 = !MDCompositeType(tag: DW_TAG_structure_type, name: "foo", line: 5, size: 64, align: 64, file: !1, elements: !5, identifier: "_ZTS3foo")
+!4 = !DICompositeType(tag: DW_TAG_structure_type, name: "foo", line: 5, size: 64, align: 64, file: !1, elements: !5, identifier: "_ZTS3foo")
 !5 = !{!6}
-!6 = !MDDerivedType(tag: DW_TAG_member, name: "b", line: 6, size: 64, align: 64, file: !1, scope: !"_ZTS3foo", baseType: !7)
-!7 = !MDDerivedType(tag: DW_TAG_typedef, name: "base_type", line: 4, file: !1, baseType: !"_ZTS4base")
-!8 = !MDCompositeType(tag: DW_TAG_structure_type, name: "base", line: 1, flags: DIFlagFwdDecl, file: !1, identifier: "_ZTS4base")
+!6 = !DIDerivedType(tag: DW_TAG_member, name: "b", line: 6, size: 64, align: 64, file: !1, scope: !"_ZTS3foo", baseType: !7)
+!7 = !DIDerivedType(tag: DW_TAG_typedef, name: "base_type", line: 4, file: !1, baseType: !"_ZTS4base")
+!8 = !DICompositeType(tag: DW_TAG_structure_type, name: "base", line: 1, flags: DIFlagFwdDecl, file: !1, identifier: "_ZTS4base")
 !9 = !{!10, !14, !19, !24, !26}
-!10 = !MDSubprogram(name: "__cxx_global_var_init", line: 8, isLocal: true, isDefinition: true, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 8, file: !1, scope: !11, type: !12, function: void ()* @__cxx_global_var_init, variables: !2)
-!11 = !MDFile(filename: "decl-derived-member.cpp", directory: "/tmp/dbginfo")
-!12 = !MDSubroutineType(types: !13)
+!10 = !DISubprogram(name: "__cxx_global_var_init", line: 8, isLocal: true, isDefinition: true, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 8, file: !1, scope: !11, type: !12, function: void ()* @__cxx_global_var_init, variables: !2)
+!11 = !DIFile(filename: "decl-derived-member.cpp", directory: "/tmp/dbginfo")
+!12 = !DISubroutineType(types: !13)
 !13 = !{null}
-!14 = !MDSubprogram(name: "foo", linkageName: "_ZN3fooC2Ev", line: 5, isLocal: false, isDefinition: true, flags: DIFlagArtificial | DIFlagPrototyped, isOptimized: false, scopeLine: 5, file: !1, scope: !"_ZTS3foo", type: !15, function: void (%struct.foo*)* @_ZN3fooC2Ev, declaration: !18, variables: !2)
-!15 = !MDSubroutineType(types: !16)
+!14 = !DISubprogram(name: "foo", linkageName: "_ZN3fooC2Ev", line: 5, isLocal: false, isDefinition: true, flags: DIFlagArtificial | DIFlagPrototyped, isOptimized: false, scopeLine: 5, file: !1, scope: !"_ZTS3foo", type: !15, function: void (%struct.foo*)* @_ZN3fooC2Ev, declaration: !18, variables: !2)
+!15 = !DISubroutineType(types: !16)
 !16 = !{null, !17}
-!17 = !MDDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, flags: DIFlagArtificial | DIFlagObjectPointer, baseType: !"_ZTS3foo")
-!18 = !MDSubprogram(name: "foo", isLocal: false, isDefinition: false, flags: DIFlagArtificial | DIFlagPrototyped, isOptimized: false, scope: !"_ZTS3foo", type: !15)
-!19 = !MDSubprogram(name: "base", linkageName: "_ZN4baseC2Ev", line: 1, isLocal: false, isDefinition: true, flags: DIFlagArtificial | DIFlagPrototyped, isOptimized: false, scopeLine: 1, file: !1, scope: !"_ZTS4base", type: !20, function: void (%struct.base*)* @_ZN4baseC2Ev, declaration: !23, variables: !2)
-!20 = !MDSubroutineType(types: !21)
+!17 = !DIDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, flags: DIFlagArtificial | DIFlagObjectPointer, baseType: !"_ZTS3foo")
+!18 = !DISubprogram(name: "foo", isLocal: false, isDefinition: false, flags: DIFlagArtificial | DIFlagPrototyped, isOptimized: false, scope: !"_ZTS3foo", type: !15)
+!19 = !DISubprogram(name: "base", linkageName: "_ZN4baseC2Ev", line: 1, isLocal: false, isDefinition: true, flags: DIFlagArtificial | DIFlagPrototyped, isOptimized: false, scopeLine: 1, file: !1, scope: !"_ZTS4base", type: !20, function: void (%struct.base*)* @_ZN4baseC2Ev, declaration: !23, variables: !2)
+!20 = !DISubroutineType(types: !21)
 !21 = !{null, !22}
-!22 = !MDDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, flags: DIFlagArtificial | DIFlagObjectPointer, baseType: !"_ZTS4base")
-!23 = !MDSubprogram(name: "base", isLocal: false, isDefinition: false, flags: DIFlagArtificial | DIFlagPrototyped, isOptimized: false, scope: !"_ZTS4base", type: !20)
-!24 = !MDSubprogram(name: "~foo", linkageName: "_ZN3fooD2Ev", line: 5, isLocal: false, isDefinition: true, flags: DIFlagArtificial | DIFlagPrototyped, isOptimized: false, scopeLine: 5, file: !1, scope: !"_ZTS3foo", type: !15, function: void (%struct.foo*)* @_ZN3fooD2Ev, declaration: !25, variables: !2)
-!25 = !MDSubprogram(name: "~foo", isLocal: false, isDefinition: false, flags: DIFlagArtificial | DIFlagPrototyped, isOptimized: false, scope: !"_ZTS3foo", type: !15)
-!26 = !MDSubprogram(name: "", linkageName: "_GLOBAL__sub_I_decl_derived_member.cpp", isLocal: true, isDefinition: true, flags: DIFlagArtificial, isOptimized: false, file: !1, scope: !11, type: !27, function: void ()* @_GLOBAL__sub_I_decl_derived_member.cpp, variables: !2)
-!27 = !MDSubroutineType(types: !2)
+!22 = !DIDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, flags: DIFlagArtificial | DIFlagObjectPointer, baseType: !"_ZTS4base")
+!23 = !DISubprogram(name: "base", isLocal: false, isDefinition: false, flags: DIFlagArtificial | DIFlagPrototyped, isOptimized: false, scope: !"_ZTS4base", type: !20)
+!24 = !DISubprogram(name: "~foo", linkageName: "_ZN3fooD2Ev", line: 5, isLocal: false, isDefinition: true, flags: DIFlagArtificial | DIFlagPrototyped, isOptimized: false, scopeLine: 5, file: !1, scope: !"_ZTS3foo", type: !15, function: void (%struct.foo*)* @_ZN3fooD2Ev, declaration: !25, variables: !2)
+!25 = !DISubprogram(name: "~foo", isLocal: false, isDefinition: false, flags: DIFlagArtificial | DIFlagPrototyped, isOptimized: false, scope: !"_ZTS3foo", type: !15)
+!26 = !DISubprogram(name: "", linkageName: "_GLOBAL__sub_I_decl_derived_member.cpp", isLocal: true, isDefinition: true, flags: DIFlagArtificial, isOptimized: false, file: !1, scope: !11, type: !27, function: void ()* @_GLOBAL__sub_I_decl_derived_member.cpp, variables: !2)
+!27 = !DISubroutineType(types: !2)
 !28 = !{!29}
-!29 = !MDGlobalVariable(name: "f", line: 8, isLocal: false, isDefinition: true, scope: null, file: !11, type: !"_ZTS3foo", variable: %struct.foo* @f)
+!29 = !DIGlobalVariable(name: "f", line: 8, isLocal: false, isDefinition: true, scope: null, file: !11, type: !"_ZTS3foo", variable: %struct.foo* @f)
 !30 = !{i32 2, !"Dwarf Version", i32 4}
 !31 = !{i32 2, !"Debug Info Version", i32 3}
 !32 = !{!"clang version 3.7.0 (trunk 227104) (llvm/trunk 227103)"}
-!33 = !MDLocation(line: 8, column: 5, scope: !10)
-!34 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "this", arg: 1, flags: DIFlagArtificial | DIFlagObjectPointer, scope: !14, type: !35)
-!35 = !MDDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, baseType: !"_ZTS3foo")
-!36 = !MDExpression()
-!37 = !MDLocation(line: 0, scope: !14)
-!38 = !MDLocation(line: 5, column: 8, scope: !14)
-!39 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "this", arg: 1, flags: DIFlagArtificial | DIFlagObjectPointer, scope: !24, type: !35)
-!40 = !MDLocation(line: 0, scope: !24)
-!41 = !MDLocation(line: 5, column: 8, scope: !42)
-!42 = distinct !MDLexicalBlock(line: 5, column: 8, file: !1, scope: !24)
-!43 = !MDLocation(line: 5, column: 8, scope: !24)
-!44 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "this", arg: 1, flags: DIFlagArtificial | DIFlagObjectPointer, scope: !19, type: !45)
-!45 = !MDDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, baseType: !"_ZTS4base")
-!46 = !MDLocation(line: 0, scope: !19)
-!47 = !MDLocation(line: 1, column: 8, scope: !19)
-!48 = !MDLocation(line: 0, scope: !26)
+!33 = !DILocation(line: 8, column: 5, scope: !10)
+!34 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "this", arg: 1, flags: DIFlagArtificial | DIFlagObjectPointer, scope: !14, type: !35)
+!35 = !DIDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, baseType: !"_ZTS3foo")
+!36 = !DIExpression()
+!37 = !DILocation(line: 0, scope: !14)
+!38 = !DILocation(line: 5, column: 8, scope: !14)
+!39 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "this", arg: 1, flags: DIFlagArtificial | DIFlagObjectPointer, scope: !24, type: !35)
+!40 = !DILocation(line: 0, scope: !24)
+!41 = !DILocation(line: 5, column: 8, scope: !42)
+!42 = distinct !DILexicalBlock(line: 5, column: 8, file: !1, scope: !24)
+!43 = !DILocation(line: 5, column: 8, scope: !24)
+!44 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "this", arg: 1, flags: DIFlagArtificial | DIFlagObjectPointer, scope: !19, type: !45)
+!45 = !DIDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, baseType: !"_ZTS4base")
+!46 = !DILocation(line: 0, scope: !19)
+!47 = !DILocation(line: 1, column: 8, scope: !19)
+!48 = !DILocation(line: 0, scope: !26)
diff --git a/test/DebugInfo/X86/deleted-bit-piece.ll b/test/DebugInfo/X86/deleted-bit-piece.ll
index b8ae9b16f50..63f3be5ea41 100644
--- a/test/DebugInfo/X86/deleted-bit-piece.ll
+++ b/test/DebugInfo/X86/deleted-bit-piece.ll
@@ -28,19 +28,19 @@ declare void @llvm.dbg.value(metadata, i64, metadata, metadata)
 
 !0 = !{i32 2, !"Dwarf Version", i32 2}
 !1 = !{i32 2, !"Debug Info Version", i32 3}
-!2 = !MDCompileUnit(language: DW_LANG_C_plus_plus, file: !3, isOptimized: true, runtimeVersion: 0, emissionKind: 1, enums: !4, retainedTypes: !5, subprograms: !11, globals: !4, imports: !4)
-!3 = !MDFile(filename: "foo.cpp", directory: "/path/to/dir")
+!2 = !DICompileUnit(language: DW_LANG_C_plus_plus, file: !3, isOptimized: true, runtimeVersion: 0, emissionKind: 1, enums: !4, retainedTypes: !5, subprograms: !11, globals: !4, imports: !4)
+!3 = !DIFile(filename: "foo.cpp", directory: "/path/to/dir")
 !4 = !{}
 !5 = !{!6}
-!6 = !MDCompositeType(tag: DW_TAG_structure_type, name: "Class", size: 64, align: 64, elements: !7, identifier: "_ZT5Class")
+!6 = !DICompositeType(tag: DW_TAG_structure_type, name: "Class", size: 64, align: 64, elements: !7, identifier: "_ZT5Class")
 !7 = !{!8, !10}
-!8 = !MDDerivedType(tag: DW_TAG_member, name: "a", scope: !"_ZT5Class", baseType: !9, size: 32, align: 32)
-!9 = !MDBasicType(name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
-!10 = !MDDerivedType(tag: DW_TAG_member, name: "b", scope: !"_ZT5Class", baseType: !9, size: 32, align: 32)
+!8 = !DIDerivedType(tag: DW_TAG_member, name: "a", scope: !"_ZT5Class", baseType: !9, size: 32, align: 32)
+!9 = !DIBasicType(name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
+!10 = !DIDerivedType(tag: DW_TAG_member, name: "b", scope: !"_ZT5Class", baseType: !9, size: 32, align: 32)
 !11 = !{!12}
-!12 = !MDSubprogram(name: "foo", scope: null, file: !3, type: !13, isLocal: false, isDefinition: true, isOptimized: false, function: void ()* @_Z3foov)
-!13 = !MDSubroutineType(types: !14)
+!12 = !DISubprogram(name: "foo", scope: null, file: !3, type: !13, isLocal: false, isDefinition: true, isOptimized: false, function: void ()* @_Z3foov)
+!13 = !DISubroutineType(types: !14)
 !14 = !{null}
-!15 = !MDLocalVariable(tag: DW_TAG_auto_variable, name: "v", scope: !12, type: !"_ZT5Class")
-!16 = !MDExpression(DW_OP_bit_piece, 32, 32)
-!17 = !MDLocation(line: 2755, column: 9, scope: !12)
+!15 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "v", scope: !12, type: !"_ZT5Class")
+!16 = !DIExpression(DW_OP_bit_piece, 32, 32)
+!17 = !DILocation(line: 2755, column: 9, scope: !12)
diff --git a/test/DebugInfo/X86/discriminator.ll b/test/DebugInfo/X86/discriminator.ll
index 19c0bc8ffd5..da5acc753c4 100644
--- a/test/DebugInfo/X86/discriminator.ll
+++ b/test/DebugInfo/X86/discriminator.ll
@@ -41,22 +41,22 @@ attributes #0 = { nounwind uwtable "less-precise-fpmad"="false" "no-frame-pointe
 !llvm.module.flags = !{!7, !8}
 !llvm.ident = !{!9}
 
-!0 = !MDCompileUnit(language: DW_LANG_C99, producer: "clang version 3.5 ", isOptimized: false, emissionKind: 0, file: !1, enums: !2, retainedTypes: !2, subprograms: !3, globals: !2, imports: !2)
-!1 = !MDFile(filename: "discriminator.c", directory: ".")
+!0 = !DICompileUnit(language: DW_LANG_C99, producer: "clang version 3.5 ", isOptimized: false, emissionKind: 0, file: !1, enums: !2, retainedTypes: !2, subprograms: !3, globals: !2, imports: !2)
+!1 = !DIFile(filename: "discriminator.c", directory: ".")
 !2 = !{}
 !3 = !{!4}
-!4 = !MDSubprogram(name: "foo", line: 1, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 1, file: !1, scope: !5, type: !6, function: i32 (i32)* @foo, variables: !2)
-!5 = !MDFile(filename: "discriminator.c", directory: ".")
-!6 = !MDSubroutineType(types: !2)
+!4 = !DISubprogram(name: "foo", line: 1, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 1, file: !1, scope: !5, type: !6, function: i32 (i32)* @foo, variables: !2)
+!5 = !DIFile(filename: "discriminator.c", directory: ".")
+!6 = !DISubroutineType(types: !2)
 !7 = !{i32 2, !"Dwarf Version", i32 4}
 !8 = !{i32 1, !"Debug Info Version", i32 3}
 !9 = !{!"clang version 3.5 "}
-!10 = !MDLocation(line: 2, scope: !11)
-!11 = distinct !MDLexicalBlock(line: 2, column: 0, file: !1, scope: !4)
-!12 = !MDLocation(line: 3, scope: !4)
-!13 = !MDLocation(line: 4, scope: !4)
-!14 = !MDLocation(line: 2, scope: !15)
-!15 = !MDLexicalBlockFile(discriminator: 42, file: !1, scope: !4)
+!10 = !DILocation(line: 2, scope: !11)
+!11 = distinct !DILexicalBlock(line: 2, column: 0, file: !1, scope: !4)
+!12 = !DILocation(line: 3, scope: !4)
+!13 = !DILocation(line: 4, scope: !4)
+!14 = !DILocation(line: 2, scope: !15)
+!15 = !DILexicalBlockFile(discriminator: 42, file: !1, scope: !4)
 
 ; CHECK: Address            Line   Column File   ISA Discriminator Flags
 ; CHECK: ------------------ ------ ------ ------ --- ------------- -------------
diff --git a/test/DebugInfo/X86/dwarf-aranges-no-dwarf-labels.ll b/test/DebugInfo/X86/dwarf-aranges-no-dwarf-labels.ll
index 82bf01e2863..aae155414bd 100644
--- a/test/DebugInfo/X86/dwarf-aranges-no-dwarf-labels.ll
+++ b/test/DebugInfo/X86/dwarf-aranges-no-dwarf-labels.ll
@@ -28,14 +28,14 @@ target triple = "x86_64-unknown-linux-gnu"
 ; Function Attrs: nounwind readnone uwtable
 define i32 @_Z3fooi(i32 %bar) #0 {
 entry:
-  tail call void @llvm.dbg.value(metadata i32 %bar, i64 0, metadata !10, metadata !MDExpression()), !dbg !20
+  tail call void @llvm.dbg.value(metadata i32 %bar, i64 0, metadata !10, metadata !DIExpression()), !dbg !20
   ret i32 %bar, !dbg !20
 }
 
 ; Function Attrs: nounwind readnone uwtable
 define i32 @_Z4foo2i(i32 %bar2) #0 {
 entry:
-  tail call void @llvm.dbg.value(metadata i32 %bar2, i64 0, metadata !13, metadata !MDExpression()), !dbg !21
+  tail call void @llvm.dbg.value(metadata i32 %bar2, i64 0, metadata !13, metadata !DIExpression()), !dbg !21
   ret i32 %bar2, !dbg !21
 }
 
@@ -60,29 +60,29 @@ attributes #2 = { nounwind readnone }
 !llvm.dbg.cu = !{!0}
 !llvm.module.flags = !{!19, !26}
 
-!0 = !MDCompileUnit(language: DW_LANG_C_plus_plus, producer: "clang version 3.4 (191881)", isOptimized: true, emissionKind: 1, file: !1, enums: !2, retainedTypes: !2, subprograms: !3, globals: !17, imports: !2)
-!1 = !MDFile(filename: "tmp/debug_ranges/a.cc", directory: "/")
+!0 = !DICompileUnit(language: DW_LANG_C_plus_plus, producer: "clang version 3.4 (191881)", isOptimized: true, emissionKind: 1, file: !1, enums: !2, retainedTypes: !2, subprograms: !3, globals: !17, imports: !2)
+!1 = !DIFile(filename: "tmp/debug_ranges/a.cc", directory: "/")
 !2 = !{}
 !3 = !{!4, !11, !14}
-!4 = !MDSubprogram(name: "foo", linkageName: "_Z3fooi", line: 2, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: true, scopeLine: 2, file: !1, scope: !5, type: !6, function: i32 (i32)* @_Z3fooi, variables: !9)
-!5 = !MDFile(filename: "tmp/debug_ranges/a.cc", directory: "/")
-!6 = !MDSubroutineType(types: !7)
+!4 = !DISubprogram(name: "foo", linkageName: "_Z3fooi", line: 2, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: true, scopeLine: 2, file: !1, scope: !5, type: !6, function: i32 (i32)* @_Z3fooi, variables: !9)
+!5 = !DIFile(filename: "tmp/debug_ranges/a.cc", directory: "/")
+!6 = !DISubroutineType(types: !7)
 !7 = !{!8, !8}
-!8 = !MDBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
+!8 = !DIBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
 !9 = !{!10}
-!10 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "bar", line: 2, arg: 1, scope: !4, file: !5, type: !8)
-!11 = !MDSubprogram(name: "foo2", linkageName: "_Z4foo2i", line: 3, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: true, scopeLine: 3, file: !1, scope: !5, type: !6, function: i32 (i32)* @_Z4foo2i, variables: !12)
+!10 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "bar", line: 2, arg: 1, scope: !4, file: !5, type: !8)
+!11 = !DISubprogram(name: "foo2", linkageName: "_Z4foo2i", line: 3, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: true, scopeLine: 3, file: !1, scope: !5, type: !6, function: i32 (i32)* @_Z4foo2i, variables: !12)
 !12 = !{!13}
-!13 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "bar2", line: 3, arg: 1, scope: !11, file: !5, type: !8)
-!14 = !MDSubprogram(name: "main", line: 5, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: true, scopeLine: 5, file: !1, scope: !5, type: !15, function: i32 ()* @main, variables: !2)
-!15 = !MDSubroutineType(types: !16)
+!13 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "bar2", line: 3, arg: 1, scope: !11, file: !5, type: !8)
+!14 = !DISubprogram(name: "main", line: 5, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: true, scopeLine: 5, file: !1, scope: !5, type: !15, function: i32 ()* @main, variables: !2)
+!15 = !DISubroutineType(types: !16)
 !16 = !{!8}
 !17 = !{!18}
-!18 = !MDGlobalVariable(name: "global", line: 1, isLocal: false, isDefinition: true, scope: null, file: !5, type: !8, variable: i32* @global)
+!18 = !DIGlobalVariable(name: "global", line: 1, isLocal: false, isDefinition: true, scope: null, file: !5, type: !8, variable: i32* @global)
 !19 = !{i32 2, !"Dwarf Version", i32 4}
-!20 = !MDLocation(line: 2, scope: !4)
-!21 = !MDLocation(line: 3, scope: !11)
-!22 = !MDLocation(line: 6, scope: !14)
+!20 = !DILocation(line: 2, scope: !4)
+!21 = !DILocation(line: 3, scope: !11)
+!22 = !DILocation(line: 6, scope: !14)
 !23 = !{!"int", !24}
 !24 = !{!"omnipotent char", !25}
 !25 = !{!"Simple C/C++ TBAA"}
diff --git a/test/DebugInfo/X86/dwarf-aranges.ll b/test/DebugInfo/X86/dwarf-aranges.ll
index 0e658a26637..9924697247b 100644
--- a/test/DebugInfo/X86/dwarf-aranges.ll
+++ b/test/DebugInfo/X86/dwarf-aranges.ll
@@ -62,20 +62,20 @@ entry:
 !llvm.dbg.cu = !{!0}
 !llvm.module.flags = !{!13, !16}
 
-!0 = !MDCompileUnit(language: DW_LANG_C99, producer: "clang version 3.4 ", isOptimized: false, emissionKind: 0, file: !1, enums: !2, retainedTypes: !2, subprograms: !3, globals: !8, imports: !2)
-!1 = !MDFile(filename: "test.c", directory: "/home/kayamon")
+!0 = !DICompileUnit(language: DW_LANG_C99, producer: "clang version 3.4 ", isOptimized: false, emissionKind: 0, file: !1, enums: !2, retainedTypes: !2, subprograms: !3, globals: !8, imports: !2)
+!1 = !DIFile(filename: "test.c", directory: "/home/kayamon")
 !2 = !{}
 !3 = !{!4}
-!4 = !MDSubprogram(name: "some_code", line: 5, isLocal: false, isDefinition: true, virtualIndex: 6, isOptimized: false, scopeLine: 6, file: !1, scope: !5, type: !6, function: void ()* @some_code, variables: !2)
-!5 = !MDFile(filename: "test.c", directory: "/home/kayamon")
-!6 = !MDSubroutineType(types: !7)
+!4 = !DISubprogram(name: "some_code", line: 5, isLocal: false, isDefinition: true, virtualIndex: 6, isOptimized: false, scopeLine: 6, file: !1, scope: !5, type: !6, function: void ()* @some_code, variables: !2)
+!5 = !DIFile(filename: "test.c", directory: "/home/kayamon")
+!6 = !DISubroutineType(types: !7)
 !7 = !{null}
 !8 = !{!9, !11, !12}
-!9 = !MDGlobalVariable(name: "some_data", line: 1, isLocal: false, isDefinition: true, scope: null, file: !5, type: !10, variable: i32* @some_data)
-!10 = !MDBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
-!11 = !MDGlobalVariable(name: "some_other", line: 3, isLocal: false, isDefinition: true, scope: null, file: !5, type: !10, variable: i32* @some_other)
-!12 = !MDGlobalVariable(name: "some_bss", line: 2, isLocal: false, isDefinition: true, scope: null, file: !5, type: !10, variable: i32* @some_bss)
+!9 = !DIGlobalVariable(name: "some_data", line: 1, isLocal: false, isDefinition: true, scope: null, file: !5, type: !10, variable: i32* @some_data)
+!10 = !DIBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
+!11 = !DIGlobalVariable(name: "some_other", line: 3, isLocal: false, isDefinition: true, scope: null, file: !5, type: !10, variable: i32* @some_other)
+!12 = !DIGlobalVariable(name: "some_bss", line: 2, isLocal: false, isDefinition: true, scope: null, file: !5, type: !10, variable: i32* @some_bss)
 !13 = !{i32 2, !"Dwarf Version", i32 4}
-!14 = !MDLocation(line: 7, scope: !4)
-!15 = !MDLocation(line: 8, scope: !4)
+!14 = !DILocation(line: 7, scope: !4)
+!15 = !DILocation(line: 8, scope: !4)
 !16 = !{i32 1, !"Debug Info Version", i32 3}
diff --git a/test/DebugInfo/X86/dwarf-public-names.ll b/test/DebugInfo/X86/dwarf-public-names.ll
index 449d0dd2aef..c72da38832e 100644
--- a/test/DebugInfo/X86/dwarf-public-names.ll
+++ b/test/DebugInfo/X86/dwarf-public-names.ll
@@ -39,11 +39,11 @@
 
 ; Darwin and PS4 shouldn't be generating the section by default
 ; NOPUB: debug_pubnames
-; NOPUB: {{^$}}
+; NOPUB-NEXT: {{^$}}
 
 ; Skip the output to the header of the pubnames section.
 ; LINUX: debug_pubnames
-; LINUX: unit_size = 0x00000128
+; LINUX-NEXT: unit_size = 0x00000128
 
 ; Check for each name in the output.
 ; LINUX-DAG: "ns"
@@ -65,7 +65,7 @@ define void @_ZN1C15member_functionEv(%struct.C* %this) nounwind uwtable align 2
 entry:
   %this.addr = alloca %struct.C*, align 8
   store %struct.C* %this, %struct.C** %this.addr, align 8
-  call void @llvm.dbg.declare(metadata %struct.C** %this.addr, metadata !28, metadata !MDExpression()), !dbg !30
+  call void @llvm.dbg.declare(metadata %struct.C** %this.addr, metadata !28, metadata !DIExpression()), !dbg !30
   %this1 = load %struct.C*, %struct.C** %this.addr
   store i32 0, i32* @_ZN1C22static_member_variableE, align 4, !dbg !31
   ret void, !dbg !32
@@ -96,42 +96,42 @@ attributes #1 = { nounwind readnone }
 !llvm.dbg.cu = !{!0}
 !llvm.module.flags = !{!38}
 
-!0 = !MDCompileUnit(language: DW_LANG_C_plus_plus, producer: "clang version 3.3 (http://llvm.org/git/clang.git a09cd8103a6a719cb2628cdf0c91682250a17bd2) (http://llvm.org/git/llvm.git 47d03cec0afca0c01ae42b82916d1d731716cd20)", isOptimized: false, emissionKind: 0, file: !37, enums: !1, retainedTypes: !1, subprograms: !2, globals: !24, imports:  !1)
+!0 = !DICompileUnit(language: DW_LANG_C_plus_plus, producer: "clang version 3.3 (http://llvm.org/git/clang.git a09cd8103a6a719cb2628cdf0c91682250a17bd2) (http://llvm.org/git/llvm.git 47d03cec0afca0c01ae42b82916d1d731716cd20)", isOptimized: false, emissionKind: 0, file: !37, enums: !1, retainedTypes: !1, subprograms: !2, globals: !24, imports:  !1)
 !1 = !{}
 !2 = !{!3, !18, !19, !20}
-!3 = !MDSubprogram(name: "member_function", linkageName: "_ZN1C15member_functionEv", line: 9, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 9, file: !4, scope: null, type: !5, function: void (%struct.C*)* @_ZN1C15member_functionEv, declaration: !12, variables: !1)
-!4 = !MDFile(filename: "dwarf-public-names.cpp", directory: "/usr2/kparzysz/s.hex/t")
-!5 = !MDSubroutineType(types: !6)
+!3 = !DISubprogram(name: "member_function", linkageName: "_ZN1C15member_functionEv", line: 9, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 9, file: !4, scope: null, type: !5, function: void (%struct.C*)* @_ZN1C15member_functionEv, declaration: !12, variables: !1)
+!4 = !DIFile(filename: "dwarf-public-names.cpp", directory: "/usr2/kparzysz/s.hex/t")
+!5 = !DISubroutineType(types: !6)
 !6 = !{null, !7}
-!7 = !MDDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, flags: DIFlagArtificial | DIFlagObjectPointer, baseType: !8)
-!8 = !MDCompositeType(tag: DW_TAG_structure_type, name: "C", line: 1, size: 8, align: 8, file: !37, elements: !9)
+!7 = !DIDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, flags: DIFlagArtificial | DIFlagObjectPointer, baseType: !8)
+!8 = !DICompositeType(tag: DW_TAG_structure_type, name: "C", line: 1, size: 8, align: 8, file: !37, elements: !9)
 !9 = !{!10, !12, !14}
-!10 = !MDDerivedType(tag: DW_TAG_member, name: "static_member_variable", line: 4, flags: DIFlagStaticMember, file: !37, scope: !8, baseType: !11)
-!11 = !MDBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
-!12 = !MDSubprogram(name: "member_function", linkageName: "_ZN1C15member_functionEv", line: 2, isLocal: false, isDefinition: false, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 2, file: !4, scope: !8, type: !5, variables: !13)
+!10 = !DIDerivedType(tag: DW_TAG_member, name: "static_member_variable", line: 4, flags: DIFlagStaticMember, file: !37, scope: !8, baseType: !11)
+!11 = !DIBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
+!12 = !DISubprogram(name: "member_function", linkageName: "_ZN1C15member_functionEv", line: 2, isLocal: false, isDefinition: false, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 2, file: !4, scope: !8, type: !5, variables: !13)
 !13 = !{} ; previously: invalid DW_TAG_base_type
-!14 = !MDSubprogram(name: "static_member_function", linkageName: "_ZN1C22static_member_functionEv", line: 3, isLocal: false, isDefinition: false, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 3, file: !4, scope: !8, type: !15, variables: !17)
-!15 = !MDSubroutineType(types: !16)
+!14 = !DISubprogram(name: "static_member_function", linkageName: "_ZN1C22static_member_functionEv", line: 3, isLocal: false, isDefinition: false, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 3, file: !4, scope: !8, type: !15, variables: !17)
+!15 = !DISubroutineType(types: !16)
 !16 = !{!11}
 !17 = !{} ; previously: invalid DW_TAG_base_type
-!18 = !MDSubprogram(name: "static_member_function", linkageName: "_ZN1C22static_member_functionEv", line: 13, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 13, file: !4, scope: null, type: !15, function: i32 ()* @_ZN1C22static_member_functionEv, declaration: !14, variables: !1)
-!19 = !MDSubprogram(name: "global_function", linkageName: "_Z15global_functionv", line: 19, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 19, file: !4, scope: !4, type: !15, function: i32 ()* @_Z15global_functionv, variables: !1)
-!20 = !MDSubprogram(name: "global_namespace_function", linkageName: "_ZN2ns25global_namespace_functionEv", line: 24, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 24, file: !4, scope: !21, type: !22, function: void ()* @_ZN2ns25global_namespace_functionEv, variables: !1)
-!21 = !MDNamespace(name: "ns", line: 23, file: !4, scope: null)
-!22 = !MDSubroutineType(types: !23)
+!18 = !DISubprogram(name: "static_member_function", linkageName: "_ZN1C22static_member_functionEv", line: 13, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 13, file: !4, scope: null, type: !15, function: i32 ()* @_ZN1C22static_member_functionEv, declaration: !14, variables: !1)
+!19 = !DISubprogram(name: "global_function", linkageName: "_Z15global_functionv", line: 19, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 19, file: !4, scope: !4, type: !15, function: i32 ()* @_Z15global_functionv, variables: !1)
+!20 = !DISubprogram(name: "global_namespace_function", linkageName: "_ZN2ns25global_namespace_functionEv", line: 24, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 24, file: !4, scope: !21, type: !22, function: void ()* @_ZN2ns25global_namespace_functionEv, variables: !1)
+!21 = !DINamespace(name: "ns", line: 23, file: !4, scope: null)
+!22 = !DISubroutineType(types: !23)
 !23 = !{null}
 !24 = !{!25, !26, !27}
-!25 = !MDGlobalVariable(name: "static_member_variable", linkageName: "_ZN1C22static_member_variableE", line: 7, isLocal: false, isDefinition: true, scope: !8, file: !4, type: !11, variable: i32* @_ZN1C22static_member_variableE, declaration: !10)
-!26 = !MDGlobalVariable(name: "global_variable", line: 17, isLocal: false, isDefinition: true, scope: null, file: !4, type: !8, variable: %struct.C* @global_variable)
-!27 = !MDGlobalVariable(name: "global_namespace_variable", linkageName: "_ZN2ns25global_namespace_variableE", line: 27, isLocal: false, isDefinition: true, scope: !21, file: !4, type: !11, variable: i32* @_ZN2ns25global_namespace_variableE)
-!28 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "this", line: 9, arg: 1, flags: DIFlagArtificial | DIFlagObjectPointer, scope: !3, file: !4, type: !29)
-!29 = !MDDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, baseType: !8)
-!30 = !MDLocation(line: 9, scope: !3)
-!31 = !MDLocation(line: 10, scope: !3)
-!32 = !MDLocation(line: 11, scope: !3)
-!33 = !MDLocation(line: 14, scope: !18)
-!34 = !MDLocation(line: 20, scope: !19)
-!35 = !MDLocation(line: 25, scope: !20)
-!36 = !MDLocation(line: 26, scope: !20)
-!37 = !MDFile(filename: "dwarf-public-names.cpp", directory: "/usr2/kparzysz/s.hex/t")
+!25 = !DIGlobalVariable(name: "static_member_variable", linkageName: "_ZN1C22static_member_variableE", line: 7, isLocal: false, isDefinition: true, scope: !8, file: !4, type: !11, variable: i32* @_ZN1C22static_member_variableE, declaration: !10)
+!26 = !DIGlobalVariable(name: "global_variable", line: 17, isLocal: false, isDefinition: true, scope: null, file: !4, type: !8, variable: %struct.C* @global_variable)
+!27 = !DIGlobalVariable(name: "global_namespace_variable", linkageName: "_ZN2ns25global_namespace_variableE", line: 27, isLocal: false, isDefinition: true, scope: !21, file: !4, type: !11, variable: i32* @_ZN2ns25global_namespace_variableE)
+!28 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "this", line: 9, arg: 1, flags: DIFlagArtificial | DIFlagObjectPointer, scope: !3, file: !4, type: !29)
+!29 = !DIDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, baseType: !8)
+!30 = !DILocation(line: 9, scope: !3)
+!31 = !DILocation(line: 10, scope: !3)
+!32 = !DILocation(line: 11, scope: !3)
+!33 = !DILocation(line: 14, scope: !18)
+!34 = !DILocation(line: 20, scope: !19)
+!35 = !DILocation(line: 25, scope: !20)
+!36 = !DILocation(line: 26, scope: !20)
+!37 = !DIFile(filename: "dwarf-public-names.cpp", directory: "/usr2/kparzysz/s.hex/t")
 !38 = !{i32 1, !"Debug Info Version", i32 3}
diff --git a/test/DebugInfo/X86/dwarf-pubnames-split.ll b/test/DebugInfo/X86/dwarf-pubnames-split.ll
index 97ff1ca426a..f17f7848f04 100644
--- a/test/DebugInfo/X86/dwarf-pubnames-split.ll
+++ b/test/DebugInfo/X86/dwarf-pubnames-split.ll
@@ -24,15 +24,15 @@ attributes #0 = { nounwind uwtable "less-precise-fpmad"="false" "no-frame-pointe
 !llvm.dbg.cu = !{!0}
 !llvm.module.flags = !{!9, !11}
 
-!0 = !MDCompileUnit(language: DW_LANG_C99, producer: "clang version 3.4 (trunk 189287) (llvm/trunk 189296)", isOptimized: false, emissionKind: 0, file: !1, enums: !2, retainedTypes: !2, subprograms: !3, globals: !2, imports: !2)
-!1 = !MDFile(filename: "foo.c", directory: "/usr/local/google/home/echristo/tmp")
+!0 = !DICompileUnit(language: DW_LANG_C99, producer: "clang version 3.4 (trunk 189287) (llvm/trunk 189296)", isOptimized: false, emissionKind: 0, file: !1, enums: !2, retainedTypes: !2, subprograms: !3, globals: !2, imports: !2)
+!1 = !DIFile(filename: "foo.c", directory: "/usr/local/google/home/echristo/tmp")
 !2 = !{}
 !3 = !{!4}
-!4 = !MDSubprogram(name: "main", line: 1, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 1, file: !1, scope: !5, type: !6, function: i32 ()* @main, variables: !2)
-!5 = !MDFile(filename: "foo.c", directory: "/usr/local/google/home/echristo/tmp")
-!6 = !MDSubroutineType(types: !7)
+!4 = !DISubprogram(name: "main", line: 1, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 1, file: !1, scope: !5, type: !6, function: i32 ()* @main, variables: !2)
+!5 = !DIFile(filename: "foo.c", directory: "/usr/local/google/home/echristo/tmp")
+!6 = !DISubroutineType(types: !7)
 !7 = !{!8}
-!8 = !MDBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
+!8 = !DIBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
 !9 = !{i32 2, !"Dwarf Version", i32 3}
-!10 = !MDLocation(line: 2, scope: !4)
+!10 = !DILocation(line: 2, scope: !4)
 !11 = !{i32 1, !"Debug Info Version", i32 3}
diff --git a/test/DebugInfo/X86/earlydup-crash.ll b/test/DebugInfo/X86/earlydup-crash.ll
index 5525a3c35b0..738b7194488 100644
--- a/test/DebugInfo/X86/earlydup-crash.ll
+++ b/test/DebugInfo/X86/earlydup-crash.ll
@@ -13,7 +13,7 @@ entry:
 bb:                                               ; preds = %entry
   %tmp = icmp eq i32 undef, 0
   %tmp1 = add i32 0, 11
-  call void @llvm.dbg.value(metadata i32 %tmp1, i64 0, metadata !0, metadata !MDExpression()), !dbg !MDLocation(scope: !1)
+  call void @llvm.dbg.value(metadata i32 %tmp1, i64 0, metadata !0, metadata !DIExpression()), !dbg !DILocation(scope: !1)
   br i1 undef, label %bb18, label %bb31.preheader
 
 bb31.preheader:                                   ; preds = %bb19, %bb
@@ -44,51 +44,51 @@ declare void @foobar(i32)
 
 !llvm.dbg.cu = !{!4}
 !llvm.module.flags = !{!47}
-!0 = !MDLocalVariable(tag: DW_TAG_auto_variable, name: "frname_len", line: 517, scope: !1, file: !3, type: !38)
-!1 = distinct !MDLexicalBlock(line: 515, column: 0, file: !44, scope: !2)
-!2 = !MDSubprogram(name: "framework_construct_pathname", line: 515, isLocal: true, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: true, file: !44, scope: null, type: !5, function: i8* (i8*, %struct.cpp_dir*)* @framework_construct_pathname)
-!3 = !MDFile(filename: "darwin-c.c", directory: "/Users/espindola/llvm/build-llvm-gcc/gcc/../../llvm-gcc-4.2/gcc/config")
-!4 = !MDCompileUnit(language: DW_LANG_C89, producer: "4.2.1 (Based on Apple Inc. build 5658) (LLVM build)", isOptimized: true, emissionKind: 0, file: !44, enums: !46, retainedTypes: !46, subprograms: !45)
-!5 = !MDSubroutineType(types: !6)
+!0 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "frname_len", line: 517, scope: !1, file: !3, type: !38)
+!1 = distinct !DILexicalBlock(line: 515, column: 0, file: !44, scope: !2)
+!2 = !DISubprogram(name: "framework_construct_pathname", line: 515, isLocal: true, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: true, file: !44, scope: null, type: !5, function: i8* (i8*, %struct.cpp_dir*)* @framework_construct_pathname)
+!3 = !DIFile(filename: "darwin-c.c", directory: "/Users/espindola/llvm/build-llvm-gcc/gcc/../../llvm-gcc-4.2/gcc/config")
+!4 = !DICompileUnit(language: DW_LANG_C89, producer: "4.2.1 (Based on Apple Inc. build 5658) (LLVM build)", isOptimized: true, emissionKind: 0, file: !44, enums: !46, retainedTypes: !46, subprograms: !45)
+!5 = !DISubroutineType(types: !6)
 !6 = !{!7, !9, !11}
-!7 = !MDDerivedType(tag: DW_TAG_pointer_type, size: 32, align: 32, file: !44, scope: !3, baseType: !8)
-!8 = !MDBasicType(tag: DW_TAG_base_type, name: "char", size: 8, align: 8, encoding: DW_ATE_signed_char)
-!9 = !MDDerivedType(tag: DW_TAG_pointer_type, size: 32, align: 32, file: !44, scope: !3, baseType: !10)
-!10 = !MDDerivedType(tag: DW_TAG_const_type, size: 8, align: 8, file: !44, scope: !3, baseType: !8)
-!11 = !MDDerivedType(tag: DW_TAG_pointer_type, size: 32, align: 32, file: !44, scope: !3, baseType: !12)
-!12 = !MDDerivedType(tag: DW_TAG_typedef, name: "cpp_dir", line: 45, file: !41, scope: !13, baseType: !14)
-!13 = !MDFile(filename: "cpplib.h", directory: "/Users/espindola/llvm/build-llvm-gcc/gcc/../../llvm-gcc-4.2/gcc/../libcpp/include")
-!14 = !MDCompositeType(tag: DW_TAG_structure_type, name: "cpp_dir", line: 43, size: 352, align: 32, file: !41, scope: !3, elements: !15)
+!7 = !DIDerivedType(tag: DW_TAG_pointer_type, size: 32, align: 32, file: !44, scope: !3, baseType: !8)
+!8 = !DIBasicType(tag: DW_TAG_base_type, name: "char", size: 8, align: 8, encoding: DW_ATE_signed_char)
+!9 = !DIDerivedType(tag: DW_TAG_pointer_type, size: 32, align: 32, file: !44, scope: !3, baseType: !10)
+!10 = !DIDerivedType(tag: DW_TAG_const_type, size: 8, align: 8, file: !44, scope: !3, baseType: !8)
+!11 = !DIDerivedType(tag: DW_TAG_pointer_type, size: 32, align: 32, file: !44, scope: !3, baseType: !12)
+!12 = !DIDerivedType(tag: DW_TAG_typedef, name: "cpp_dir", line: 45, file: !41, scope: !13, baseType: !14)
+!13 = !DIFile(filename: "cpplib.h", directory: "/Users/espindola/llvm/build-llvm-gcc/gcc/../../llvm-gcc-4.2/gcc/../libcpp/include")
+!14 = !DICompositeType(tag: DW_TAG_structure_type, name: "cpp_dir", line: 43, size: 352, align: 32, file: !41, scope: !3, elements: !15)
 !15 = !{!16, !18, !19, !21, !23, !25, !27, !29, !33, !36}
-!16 = !MDDerivedType(tag: DW_TAG_member, name: "next", line: 572, size: 32, align: 32, file: !41, scope: !14, baseType: !17)
-!17 = !MDDerivedType(tag: DW_TAG_pointer_type, size: 32, align: 32, file: !44, scope: !3, baseType: !14)
-!18 = !MDDerivedType(tag: DW_TAG_member, name: "name", line: 575, size: 32, align: 32, offset: 32, file: !41, scope: !14, baseType: !7)
-!19 = !MDDerivedType(tag: DW_TAG_member, name: "len", line: 576, size: 32, align: 32, offset: 64, file: !41, scope: !14, baseType: !20)
-!20 = !MDBasicType(tag: DW_TAG_base_type, name: "unsigned int", size: 32, align: 32, encoding: DW_ATE_unsigned)
-!21 = !MDDerivedType(tag: DW_TAG_member, name: "sysp", line: 580, size: 8, align: 8, offset: 96, file: !41, scope: !14, baseType: !22)
-!22 = !MDBasicType(tag: DW_TAG_base_type, name: "unsigned char", size: 8, align: 8, encoding: DW_ATE_unsigned_char)
-!23 = !MDDerivedType(tag: DW_TAG_member, name: "name_map", line: 584, size: 32, align: 32, offset: 128, file: !41, scope: !14, baseType: !24)
-!24 = !MDDerivedType(tag: DW_TAG_pointer_type, size: 32, align: 32, file: !44, scope: !3, baseType: !9)
-!25 = !MDDerivedType(tag: DW_TAG_member, name: "header_map", line: 590, size: 32, align: 32, offset: 160, file: !41, scope: !14, baseType: !26)
-!26 = !MDDerivedType(tag: DW_TAG_pointer_type, size: 32, align: 32, file: !44, scope: !3, baseType: null)
-!27 = !MDDerivedType(tag: DW_TAG_member, name: "construct", line: 597, size: 32, align: 32, offset: 192, file: !41, scope: !14, baseType: !28)
-!28 = !MDDerivedType(tag: DW_TAG_pointer_type, size: 32, align: 32, file: !44, scope: !3, baseType: !5)
-!29 = !MDDerivedType(tag: DW_TAG_member, name: "ino", line: 601, size: 64, align: 64, offset: 224, file: !41, scope: !14, baseType: !30)
-!30 = !MDDerivedType(tag: DW_TAG_typedef, name: "ino_t", line: 141, file: !42, scope: !31, baseType: !32)
-!31 = !MDFile(filename: "types.h", directory: "/usr/include/sys")
-!32 = !MDBasicType(tag: DW_TAG_base_type, name: "long long unsigned int", size: 64, align: 64, encoding: DW_ATE_unsigned)
-!33 = !MDDerivedType(tag: DW_TAG_member, name: "dev", line: 602, size: 32, align: 32, offset: 288, file: !41, scope: !14, baseType: !34)
-!34 = !MDDerivedType(tag: DW_TAG_typedef, name: "dev_t", line: 107, file: !42, scope: !31, baseType: !35)
-!35 = !MDBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
-!36 = !MDDerivedType(tag: DW_TAG_member, name: "user_supplied_p", line: 605, size: 8, align: 8, offset: 320, file: !41, scope: !14, baseType: !37)
-!37 = !MDBasicType(tag: DW_TAG_base_type, name: "_Bool", size: 8, align: 8, encoding: DW_ATE_boolean)
-!38 = !MDDerivedType(tag: DW_TAG_typedef, name: "size_t", line: 326, file: !43, scope: !39, baseType: !40)
-!39 = !MDFile(filename: "stddef.h", directory: "/Users/espindola/llvm/build-llvm-gcc/./prev-gcc/include")
-!40 = !MDBasicType(tag: DW_TAG_base_type, name: "long unsigned int", size: 32, align: 32, encoding: DW_ATE_unsigned)
-!41 = !MDFile(filename: "cpplib.h", directory: "/Users/espindola/llvm/build-llvm-gcc/gcc/../../llvm-gcc-4.2/gcc/../libcpp/include")
-!42 = !MDFile(filename: "types.h", directory: "/usr/include/sys")
-!43 = !MDFile(filename: "stddef.h", directory: "/Users/espindola/llvm/build-llvm-gcc/./prev-gcc/include")
-!44 = !MDFile(filename: "darwin-c.c", directory: "/Users/espindola/llvm/build-llvm-gcc/gcc/../../llvm-gcc-4.2/gcc/config")
+!16 = !DIDerivedType(tag: DW_TAG_member, name: "next", line: 572, size: 32, align: 32, file: !41, scope: !14, baseType: !17)
+!17 = !DIDerivedType(tag: DW_TAG_pointer_type, size: 32, align: 32, file: !44, scope: !3, baseType: !14)
+!18 = !DIDerivedType(tag: DW_TAG_member, name: "name", line: 575, size: 32, align: 32, offset: 32, file: !41, scope: !14, baseType: !7)
+!19 = !DIDerivedType(tag: DW_TAG_member, name: "len", line: 576, size: 32, align: 32, offset: 64, file: !41, scope: !14, baseType: !20)
+!20 = !DIBasicType(tag: DW_TAG_base_type, name: "unsigned int", size: 32, align: 32, encoding: DW_ATE_unsigned)
+!21 = !DIDerivedType(tag: DW_TAG_member, name: "sysp", line: 580, size: 8, align: 8, offset: 96, file: !41, scope: !14, baseType: !22)
+!22 = !DIBasicType(tag: DW_TAG_base_type, name: "unsigned char", size: 8, align: 8, encoding: DW_ATE_unsigned_char)
+!23 = !DIDerivedType(tag: DW_TAG_member, name: "name_map", line: 584, size: 32, align: 32, offset: 128, file: !41, scope: !14, baseType: !24)
+!24 = !DIDerivedType(tag: DW_TAG_pointer_type, size: 32, align: 32, file: !44, scope: !3, baseType: !9)
+!25 = !DIDerivedType(tag: DW_TAG_member, name: "header_map", line: 590, size: 32, align: 32, offset: 160, file: !41, scope: !14, baseType: !26)
+!26 = !DIDerivedType(tag: DW_TAG_pointer_type, size: 32, align: 32, file: !44, scope: !3, baseType: null)
+!27 = !DIDerivedType(tag: DW_TAG_member, name: "construct", line: 597, size: 32, align: 32, offset: 192, file: !41, scope: !14, baseType: !28)
+!28 = !DIDerivedType(tag: DW_TAG_pointer_type, size: 32, align: 32, file: !44, scope: !3, baseType: !5)
+!29 = !DIDerivedType(tag: DW_TAG_member, name: "ino", line: 601, size: 64, align: 64, offset: 224, file: !41, scope: !14, baseType: !30)
+!30 = !DIDerivedType(tag: DW_TAG_typedef, name: "ino_t", line: 141, file: !42, scope: !31, baseType: !32)
+!31 = !DIFile(filename: "types.h", directory: "/usr/include/sys")
+!32 = !DIBasicType(tag: DW_TAG_base_type, name: "long long unsigned int", size: 64, align: 64, encoding: DW_ATE_unsigned)
+!33 = !DIDerivedType(tag: DW_TAG_member, name: "dev", line: 602, size: 32, align: 32, offset: 288, file: !41, scope: !14, baseType: !34)
+!34 = !DIDerivedType(tag: DW_TAG_typedef, name: "dev_t", line: 107, file: !42, scope: !31, baseType: !35)
+!35 = !DIBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
+!36 = !DIDerivedType(tag: DW_TAG_member, name: "user_supplied_p", line: 605, size: 8, align: 8, offset: 320, file: !41, scope: !14, baseType: !37)
+!37 = !DIBasicType(tag: DW_TAG_base_type, name: "_Bool", size: 8, align: 8, encoding: DW_ATE_boolean)
+!38 = !DIDerivedType(tag: DW_TAG_typedef, name: "size_t", line: 326, file: !43, scope: !39, baseType: !40)
+!39 = !DIFile(filename: "stddef.h", directory: "/Users/espindola/llvm/build-llvm-gcc/./prev-gcc/include")
+!40 = !DIBasicType(tag: DW_TAG_base_type, name: "long unsigned int", size: 32, align: 32, encoding: DW_ATE_unsigned)
+!41 = !DIFile(filename: "cpplib.h", directory: "/Users/espindola/llvm/build-llvm-gcc/gcc/../../llvm-gcc-4.2/gcc/../libcpp/include")
+!42 = !DIFile(filename: "types.h", directory: "/usr/include/sys")
+!43 = !DIFile(filename: "stddef.h", directory: "/Users/espindola/llvm/build-llvm-gcc/./prev-gcc/include")
+!44 = !DIFile(filename: "darwin-c.c", directory: "/Users/espindola/llvm/build-llvm-gcc/gcc/../../llvm-gcc-4.2/gcc/config")
 !45 = !{!2}
 !46 = !{}
 !47 = !{i32 1, !"Debug Info Version", i32 3}
diff --git a/test/DebugInfo/X86/elf-names.ll b/test/DebugInfo/X86/elf-names.ll
index 4344a5be584..e4565c27cf7 100644
--- a/test/DebugInfo/X86/elf-names.ll
+++ b/test/DebugInfo/X86/elf-names.ll
@@ -22,7 +22,7 @@
 
 define void @_ZN1DC2Ev(%class.D* nocapture %this) unnamed_addr nounwind uwtable align 2 {
 entry:
-  tail call void @llvm.dbg.value(metadata %class.D* %this, i64 0, metadata !29, metadata !MDExpression()), !dbg !36
+  tail call void @llvm.dbg.value(metadata %class.D* %this, i64 0, metadata !29, metadata !DIExpression()), !dbg !36
   %c1 = getelementptr inbounds %class.D, %class.D* %this, i64 0, i32 0, !dbg !37
   store i32 1, i32* %c1, align 4, !dbg !37
   %c2 = getelementptr inbounds %class.D, %class.D* %this, i64 0, i32 1, !dbg !42
@@ -36,8 +36,8 @@ entry:
 
 define void @_ZN1DC2ERKS_(%class.D* nocapture %this, %class.D* nocapture %d) unnamed_addr nounwind uwtable align 2 {
 entry:
-  tail call void @llvm.dbg.value(metadata %class.D* %this, i64 0, metadata !34, metadata !MDExpression()), !dbg !46
-  tail call void @llvm.dbg.value(metadata %class.D* %d, i64 0, metadata !35, metadata !MDExpression()), !dbg !46
+  tail call void @llvm.dbg.value(metadata %class.D* %this, i64 0, metadata !34, metadata !DIExpression()), !dbg !46
+  tail call void @llvm.dbg.value(metadata %class.D* %d, i64 0, metadata !35, metadata !DIExpression()), !dbg !46
   %c1 = getelementptr inbounds %class.D, %class.D* %d, i64 0, i32 0, !dbg !47
   %0 = load i32, i32* %c1, align 4, !dbg !47
   %c12 = getelementptr inbounds %class.D, %class.D* %this, i64 0, i32 0, !dbg !47
@@ -62,47 +62,47 @@ declare void @llvm.dbg.value(metadata, i64, metadata, metadata) nounwind readnon
 !llvm.dbg.cu = !{!0}
 !llvm.module.flags = !{!54}
 
-!0 = !MDCompileUnit(language: DW_LANG_C_plus_plus, producer: "clang version 3.2 (trunk 167506) (llvm/trunk 167505)", isOptimized: true, emissionKind: 0, file: !53, enums: !1, retainedTypes: !1, subprograms: !3, globals: !1, imports:  !1)
+!0 = !DICompileUnit(language: DW_LANG_C_plus_plus, producer: "clang version 3.2 (trunk 167506) (llvm/trunk 167505)", isOptimized: true, emissionKind: 0, file: !53, enums: !1, retainedTypes: !1, subprograms: !3, globals: !1, imports:  !1)
 !1 = !{}
 !3 = !{!5, !31}
-!5 = !MDSubprogram(name: "D", linkageName: "_ZN1DC2Ev", line: 12, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: true, scopeLine: 12, file: !6, scope: null, type: !7, function: void (%class.D*)* @_ZN1DC2Ev, declaration: !17, variables: !27)
-!6 = !MDFile(filename: "foo.cpp", directory: "/usr/local/google/home/echristo")
-!7 = !MDSubroutineType(types: !8)
+!5 = !DISubprogram(name: "D", linkageName: "_ZN1DC2Ev", line: 12, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: true, scopeLine: 12, file: !6, scope: null, type: !7, function: void (%class.D*)* @_ZN1DC2Ev, declaration: !17, variables: !27)
+!6 = !DIFile(filename: "foo.cpp", directory: "/usr/local/google/home/echristo")
+!7 = !DISubroutineType(types: !8)
 !8 = !{null, !9}
-!9 = !MDDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, flags: DIFlagArtificial | DIFlagObjectPointer, baseType: !10)
-!10 = !MDCompositeType(tag: DW_TAG_class_type, name: "D", line: 1, size: 128, align: 32, file: !53, elements: !11)
+!9 = !DIDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, flags: DIFlagArtificial | DIFlagObjectPointer, baseType: !10)
+!10 = !DICompositeType(tag: DW_TAG_class_type, name: "D", line: 1, size: 128, align: 32, file: !53, elements: !11)
 !11 = !{!12, !14, !15, !16, !17, !20}
-!12 = !MDDerivedType(tag: DW_TAG_member, name: "c1", line: 6, size: 32, align: 32, flags: DIFlagPrivate, file: !53, scope: !10, baseType: !13)
-!13 = !MDBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
-!14 = !MDDerivedType(tag: DW_TAG_member, name: "c2", line: 7, size: 32, align: 32, offset: 32, flags: DIFlagPrivate, file: !53, scope: !10, baseType: !13)
-!15 = !MDDerivedType(tag: DW_TAG_member, name: "c3", line: 8, size: 32, align: 32, offset: 64, flags: DIFlagPrivate, file: !53, scope: !10, baseType: !13)
-!16 = !MDDerivedType(tag: DW_TAG_member, name: "c4", line: 9, size: 32, align: 32, offset: 96, flags: DIFlagPrivate, file: !53, scope: !10, baseType: !13)
-!17 = !MDSubprogram(name: "D", line: 3, isLocal: false, isDefinition: false, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: true, scopeLine: 3, file: !6, scope: !10, type: !7)
-!20 = !MDSubprogram(name: "D", line: 4, isLocal: false, isDefinition: false, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: true, scopeLine: 4, file: !6, scope: !10, type: !21)
-!21 = !MDSubroutineType(types: !22)
+!12 = !DIDerivedType(tag: DW_TAG_member, name: "c1", line: 6, size: 32, align: 32, flags: DIFlagPrivate, file: !53, scope: !10, baseType: !13)
+!13 = !DIBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
+!14 = !DIDerivedType(tag: DW_TAG_member, name: "c2", line: 7, size: 32, align: 32, offset: 32, flags: DIFlagPrivate, file: !53, scope: !10, baseType: !13)
+!15 = !DIDerivedType(tag: DW_TAG_member, name: "c3", line: 8, size: 32, align: 32, offset: 64, flags: DIFlagPrivate, file: !53, scope: !10, baseType: !13)
+!16 = !DIDerivedType(tag: DW_TAG_member, name: "c4", line: 9, size: 32, align: 32, offset: 96, flags: DIFlagPrivate, file: !53, scope: !10, baseType: !13)
+!17 = !DISubprogram(name: "D", line: 3, isLocal: false, isDefinition: false, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: true, scopeLine: 3, file: !6, scope: !10, type: !7)
+!20 = !DISubprogram(name: "D", line: 4, isLocal: false, isDefinition: false, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: true, scopeLine: 4, file: !6, scope: !10, type: !21)
+!21 = !DISubroutineType(types: !22)
 !22 = !{null, !9, !23}
-!23 = !MDDerivedType(tag: DW_TAG_reference_type, baseType: !24)
-!24 = !MDDerivedType(tag: DW_TAG_const_type, baseType: !10)
+!23 = !DIDerivedType(tag: DW_TAG_reference_type, baseType: !24)
+!24 = !DIDerivedType(tag: DW_TAG_const_type, baseType: !10)
 !27 = !{!29}
-!29 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "this", line: 12, arg: 1, flags: DIFlagArtificial | DIFlagObjectPointer, scope: !5, file: !6, type: !30)
-!30 = !MDDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, baseType: !10)
-!31 = !MDSubprogram(name: "D", linkageName: "_ZN1DC2ERKS_", line: 19, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: true, scopeLine: 19, file: !6, scope: null, type: !21, function: void (%class.D*, %class.D*)* @_ZN1DC2ERKS_, declaration: !20, variables: !32)
+!29 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "this", line: 12, arg: 1, flags: DIFlagArtificial | DIFlagObjectPointer, scope: !5, file: !6, type: !30)
+!30 = !DIDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, baseType: !10)
+!31 = !DISubprogram(name: "D", linkageName: "_ZN1DC2ERKS_", line: 19, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: true, scopeLine: 19, file: !6, scope: null, type: !21, function: void (%class.D*, %class.D*)* @_ZN1DC2ERKS_, declaration: !20, variables: !32)
 !32 = !{!34, !35}
-!34 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "this", line: 19, arg: 1, flags: DIFlagArtificial | DIFlagObjectPointer, scope: !31, file: !6, type: !30)
-!35 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "d", line: 19, arg: 2, scope: !31, file: !6, type: !23)
-!36 = !MDLocation(line: 12, scope: !5)
-!37 = !MDLocation(line: 13, scope: !38)
-!38 = distinct !MDLexicalBlock(line: 12, column: 0, file: !6, scope: !5)
-!42 = !MDLocation(line: 14, scope: !38)
-!43 = !MDLocation(line: 15, scope: !38)
-!44 = !MDLocation(line: 16, scope: !38)
-!45 = !MDLocation(line: 17, scope: !38)
-!46 = !MDLocation(line: 19, scope: !31)
-!47 = !MDLocation(line: 20, scope: !48)
-!48 = distinct !MDLexicalBlock(line: 19, column: 0, file: !6, scope: !31)
-!49 = !MDLocation(line: 21, scope: !48)
-!50 = !MDLocation(line: 22, scope: !48)
-!51 = !MDLocation(line: 23, scope: !48)
-!52 = !MDLocation(line: 24, scope: !48)
-!53 = !MDFile(filename: "foo.cpp", directory: "/usr/local/google/home/echristo")
+!34 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "this", line: 19, arg: 1, flags: DIFlagArtificial | DIFlagObjectPointer, scope: !31, file: !6, type: !30)
+!35 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "d", line: 19, arg: 2, scope: !31, file: !6, type: !23)
+!36 = !DILocation(line: 12, scope: !5)
+!37 = !DILocation(line: 13, scope: !38)
+!38 = distinct !DILexicalBlock(line: 12, column: 0, file: !6, scope: !5)
+!42 = !DILocation(line: 14, scope: !38)
+!43 = !DILocation(line: 15, scope: !38)
+!44 = !DILocation(line: 16, scope: !38)
+!45 = !DILocation(line: 17, scope: !38)
+!46 = !DILocation(line: 19, scope: !31)
+!47 = !DILocation(line: 20, scope: !48)
+!48 = distinct !DILexicalBlock(line: 19, column: 0, file: !6, scope: !31)
+!49 = !DILocation(line: 21, scope: !48)
+!50 = !DILocation(line: 22, scope: !48)
+!51 = !DILocation(line: 23, scope: !48)
+!52 = !DILocation(line: 24, scope: !48)
+!53 = !DIFile(filename: "foo.cpp", directory: "/usr/local/google/home/echristo")
 !54 = !{i32 1, !"Debug Info Version", i32 3}
diff --git a/test/DebugInfo/X86/empty-and-one-elem-array.ll b/test/DebugInfo/X86/empty-and-one-elem-array.ll
index e4e7fbd521a..1250b9c6441 100644
--- a/test/DebugInfo/X86/empty-and-one-elem-array.ll
+++ b/test/DebugInfo/X86/empty-and-one-elem-array.ll
@@ -9,8 +9,8 @@ define i32 @func() nounwind uwtable ssp {
 entry:
   %my_foo = alloca %struct.foo, align 4
   %my_bar = alloca %struct.bar, align 4
-  call void @llvm.dbg.declare(metadata %struct.foo* %my_foo, metadata !10, metadata !MDExpression()), !dbg !19
-  call void @llvm.dbg.declare(metadata %struct.bar* %my_bar, metadata !20, metadata !MDExpression()), !dbg !28
+  call void @llvm.dbg.declare(metadata %struct.foo* %my_foo, metadata !10, metadata !DIExpression()), !dbg !19
+  call void @llvm.dbg.declare(metadata %struct.bar* %my_bar, metadata !20, metadata !DIExpression()), !dbg !28
   %a = getelementptr inbounds %struct.foo, %struct.foo* %my_foo, i32 0, i32 0, !dbg !29
   store i32 3, i32* %a, align 4, !dbg !29
   %a1 = getelementptr inbounds %struct.bar, %struct.bar* %my_bar, i32 0, i32 0, !dbg !30
@@ -63,35 +63,35 @@ declare void @llvm.dbg.declare(metadata, metadata, metadata) nounwind readnone
 !llvm.dbg.cu = !{!0}
 !llvm.module.flags = !{!33}
 
-!0 = !MDCompileUnit(language: DW_LANG_C99, producer: "clang version 3.3 (trunk 169136)", isOptimized: false, emissionKind: 0, file: !32, enums: !1, retainedTypes: !1, subprograms: !3, globals: !1, imports:  !1)
+!0 = !DICompileUnit(language: DW_LANG_C99, producer: "clang version 3.3 (trunk 169136)", isOptimized: false, emissionKind: 0, file: !32, enums: !1, retainedTypes: !1, subprograms: !3, globals: !1, imports:  !1)
 !1 = !{}
 !3 = !{!5}
-!5 = !MDSubprogram(name: "func", line: 11, isLocal: false, isDefinition: true, virtualIndex: 6, isOptimized: false, scopeLine: 11, file: !6, scope: !6, type: !7, function: i32 ()* @func, variables: !1)
-!6 = !MDFile(filename: "test.c", directory: "/Volumes/Sandbox/llvm")
-!7 = !MDSubroutineType(types: !8)
+!5 = !DISubprogram(name: "func", line: 11, isLocal: false, isDefinition: true, virtualIndex: 6, isOptimized: false, scopeLine: 11, file: !6, scope: !6, type: !7, function: i32 ()* @func, variables: !1)
+!6 = !DIFile(filename: "test.c", directory: "/Volumes/Sandbox/llvm")
+!7 = !DISubroutineType(types: !8)
 !8 = !{!9}
-!9 = !MDBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
-!10 = !MDLocalVariable(tag: DW_TAG_auto_variable, name: "my_foo", line: 12, scope: !11, file: !6, type: !12)
-!11 = distinct !MDLexicalBlock(line: 11, column: 0, file: !6, scope: !5)
-!12 = !MDCompositeType(tag: DW_TAG_structure_type, name: "foo", line: 1, size: 64, align: 32, file: !32, elements: !13)
+!9 = !DIBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
+!10 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "my_foo", line: 12, scope: !11, file: !6, type: !12)
+!11 = distinct !DILexicalBlock(line: 11, column: 0, file: !6, scope: !5)
+!12 = !DICompositeType(tag: DW_TAG_structure_type, name: "foo", line: 1, size: 64, align: 32, file: !32, elements: !13)
 !13 = !{!14, !15}
-!14 = !MDDerivedType(tag: DW_TAG_member, name: "a", line: 2, size: 32, align: 32, file: !32, scope: !12, baseType: !9)
-!15 = !MDDerivedType(tag: DW_TAG_member, name: "b", line: 3, size: 32, align: 32, offset: 32, file: !32, scope: !12, baseType: !16)
-!16 = !MDCompositeType(tag: DW_TAG_array_type, size: 32, align: 32, baseType: !9, elements: !17)
+!14 = !DIDerivedType(tag: DW_TAG_member, name: "a", line: 2, size: 32, align: 32, file: !32, scope: !12, baseType: !9)
+!15 = !DIDerivedType(tag: DW_TAG_member, name: "b", line: 3, size: 32, align: 32, offset: 32, file: !32, scope: !12, baseType: !16)
+!16 = !DICompositeType(tag: DW_TAG_array_type, size: 32, align: 32, baseType: !9, elements: !17)
 !17 = !{!18}
-!18 = !MDSubrange(count: 1)
-!19 = !MDLocation(line: 12, scope: !11)
-!20 = !MDLocalVariable(tag: DW_TAG_auto_variable, name: "my_bar", line: 13, scope: !11, file: !6, type: !21)
-!21 = !MDCompositeType(tag: DW_TAG_structure_type, name: "bar", line: 6, size: 32, align: 32, file: !32, elements: !22)
+!18 = !DISubrange(count: 1)
+!19 = !DILocation(line: 12, scope: !11)
+!20 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "my_bar", line: 13, scope: !11, file: !6, type: !21)
+!21 = !DICompositeType(tag: DW_TAG_structure_type, name: "bar", line: 6, size: 32, align: 32, file: !32, elements: !22)
 !22 = !{!23, !24}
-!23 = !MDDerivedType(tag: DW_TAG_member, name: "a", line: 7, size: 32, align: 32, file: !32, scope: !21, baseType: !9)
-!24 = !MDDerivedType(tag: DW_TAG_member, name: "b", line: 8, align: 32, offset: 32, file: !32, scope: !21, baseType: !25)
-!25 = !MDCompositeType(tag: DW_TAG_array_type, align: 32, baseType: !9, elements: !26)
+!23 = !DIDerivedType(tag: DW_TAG_member, name: "a", line: 7, size: 32, align: 32, file: !32, scope: !21, baseType: !9)
+!24 = !DIDerivedType(tag: DW_TAG_member, name: "b", line: 8, align: 32, offset: 32, file: !32, scope: !21, baseType: !25)
+!25 = !DICompositeType(tag: DW_TAG_array_type, align: 32, baseType: !9, elements: !26)
 !26 = !{!27}
-!27 = !MDSubrange(count: 0)
-!28 = !MDLocation(line: 13, scope: !11)
-!29 = !MDLocation(line: 15, scope: !11)
-!30 = !MDLocation(line: 16, scope: !11)
-!31 = !MDLocation(line: 17, scope: !11)
-!32 = !MDFile(filename: "test.c", directory: "/Volumes/Sandbox/llvm")
+!27 = !DISubrange(count: 0)
+!28 = !DILocation(line: 13, scope: !11)
+!29 = !DILocation(line: 15, scope: !11)
+!30 = !DILocation(line: 16, scope: !11)
+!31 = !DILocation(line: 17, scope: !11)
+!32 = !DIFile(filename: "test.c", directory: "/Volumes/Sandbox/llvm")
 !33 = !{i32 1, !"Debug Info Version", i32 3}
diff --git a/test/DebugInfo/X86/empty-array.ll b/test/DebugInfo/X86/empty-array.ll
index 678738dd274..ce460a783c0 100644
--- a/test/DebugInfo/X86/empty-array.ll
+++ b/test/DebugInfo/X86/empty-array.ll
@@ -27,21 +27,21 @@
 !llvm.dbg.cu = !{!0}
 !llvm.module.flags = !{!21}
 
-!0 = !MDCompileUnit(language: DW_LANG_C_plus_plus, producer: "clang version 3.3 (trunk 169136)", isOptimized: false, emissionKind: 0, file: !20, enums: !1, retainedTypes: !1, subprograms: !1, globals: !3, imports:  !1)
+!0 = !DICompileUnit(language: DW_LANG_C_plus_plus, producer: "clang version 3.3 (trunk 169136)", isOptimized: false, emissionKind: 0, file: !20, enums: !1, retainedTypes: !1, subprograms: !1, globals: !3, imports:  !1)
 !1 = !{}
 !3 = !{!5}
-!5 = !MDGlobalVariable(name: "a", line: 1, isLocal: false, isDefinition: true, scope: null, file: !6, type: !7, variable: %class.A* @a)
-!6 = !MDFile(filename: "t.cpp", directory: "/Volumes/Sandbox/llvm")
-!7 = !MDCompositeType(tag: DW_TAG_class_type, name: "A", line: 1, align: 32, file: !20, elements: !8)
+!5 = !DIGlobalVariable(name: "a", line: 1, isLocal: false, isDefinition: true, scope: null, file: !6, type: !7, variable: %class.A* @a)
+!6 = !DIFile(filename: "t.cpp", directory: "/Volumes/Sandbox/llvm")
+!7 = !DICompositeType(tag: DW_TAG_class_type, name: "A", line: 1, align: 32, file: !20, elements: !8)
 !8 = !{!9, !14}
-!9 = !MDDerivedType(tag: DW_TAG_member, name: "x", line: 1, flags: DIFlagPrivate, file: !20, scope: !7, baseType: !10)
-!10 = !MDCompositeType(tag: DW_TAG_array_type, align: 32, baseType: !11, elements: !12)
-!11 = !MDBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
+!9 = !DIDerivedType(tag: DW_TAG_member, name: "x", line: 1, flags: DIFlagPrivate, file: !20, scope: !7, baseType: !10)
+!10 = !DICompositeType(tag: DW_TAG_array_type, align: 32, baseType: !11, elements: !12)
+!11 = !DIBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
 !12 = !{!13}
-!13 = !MDSubrange(count: -1)
-!14 = !MDSubprogram(name: "A", line: 1, isLocal: false, isDefinition: false, virtualIndex: 6, flags: DIFlagArtificial | DIFlagPrototyped, isOptimized: false, scopeLine: 1, file: !6, scope: !7, type: !15)
-!15 = !MDSubroutineType(types: !16)
+!13 = !DISubrange(count: -1)
+!14 = !DISubprogram(name: "A", line: 1, isLocal: false, isDefinition: false, virtualIndex: 6, flags: DIFlagArtificial | DIFlagPrototyped, isOptimized: false, scopeLine: 1, file: !6, scope: !7, type: !15)
+!15 = !DISubroutineType(types: !16)
 !16 = !{null, !17}
-!17 = !MDDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, flags: DIFlagArtificial | DIFlagObjectPointer, baseType: !7)
-!20 = !MDFile(filename: "t.cpp", directory: "/Volumes/Sandbox/llvm")
+!17 = !DIDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, flags: DIFlagArtificial | DIFlagObjectPointer, baseType: !7)
+!20 = !DIFile(filename: "t.cpp", directory: "/Volumes/Sandbox/llvm")
 !21 = !{i32 1, !"Debug Info Version", i32 3}
diff --git a/test/DebugInfo/X86/empty.ll b/test/DebugInfo/X86/empty.ll
index b3ec1473696..1f48a2f2124 100644
--- a/test/DebugInfo/X86/empty.ll
+++ b/test/DebugInfo/X86/empty.ll
@@ -19,8 +19,8 @@
 !llvm.dbg.cu = !{!0}
 !llvm.module.flags = !{!5}
 
-!0 = !MDCompileUnit(language: DW_LANG_C99, producer: "clang version 3.1 (trunk 143523)", isOptimized: true, emissionKind: 0, file: !4, enums: !2, retainedTypes: !2, subprograms: !2, globals: !2)
+!0 = !DICompileUnit(language: DW_LANG_C99, producer: "clang version 3.1 (trunk 143523)", isOptimized: true, emissionKind: 0, file: !4, enums: !2, retainedTypes: !2, subprograms: !2, globals: !2)
 !2 = !{}
-!3 = !MDFile(filename: "empty.c", directory: "/home/nlewycky")
-!4 = !MDFile(filename: "empty.c", directory: "/home/nlewycky")
+!3 = !DIFile(filename: "empty.c", directory: "/home/nlewycky")
+!4 = !DIFile(filename: "empty.c", directory: "/home/nlewycky")
 !5 = !{i32 1, !"Debug Info Version", i32 3}
diff --git a/test/DebugInfo/X86/ending-run.ll b/test/DebugInfo/X86/ending-run.ll
index a75ad0a78c1..9e3584615df 100644
--- a/test/DebugInfo/X86/ending-run.ll
+++ b/test/DebugInfo/X86/ending-run.ll
@@ -13,8 +13,8 @@ entry:
   %x.addr = alloca i32, align 4
   %y = alloca i32, align 4
   store i32 %x, i32* %x.addr, align 4
-  call void @llvm.dbg.declare(metadata i32* %x.addr, metadata !12, metadata !MDExpression()), !dbg !13
-  call void @llvm.dbg.declare(metadata i32* %y, metadata !14, metadata !MDExpression()), !dbg !16
+  call void @llvm.dbg.declare(metadata i32* %x.addr, metadata !12, metadata !DIExpression()), !dbg !13
+  call void @llvm.dbg.declare(metadata i32* %y, metadata !14, metadata !DIExpression()), !dbg !16
   %0 = load i32, i32* %x.addr, align 4, !dbg !17
   %1 = load i32, i32* %x.addr, align 4, !dbg !17
   %mul = mul nsw i32 %0, %1, !dbg !17
@@ -29,20 +29,20 @@ declare void @llvm.dbg.declare(metadata, metadata, metadata) nounwind readnone
 !llvm.dbg.cu = !{!0}
 !llvm.module.flags = !{!20}
 
-!0 = !MDCompileUnit(language: DW_LANG_C99, producer: "clang version 3.1 (trunk 153921) (llvm/trunk 153916)", isOptimized: false, emissionKind: 0, file: !19, enums: !1, retainedTypes: !1, subprograms: !3, globals: !1, imports:  !1)
+!0 = !DICompileUnit(language: DW_LANG_C99, producer: "clang version 3.1 (trunk 153921) (llvm/trunk 153916)", isOptimized: false, emissionKind: 0, file: !19, enums: !1, retainedTypes: !1, subprograms: !3, globals: !1, imports:  !1)
 !1 = !{}
 !3 = !{!5}
-!5 = !MDSubprogram(name: "callee", line: 4, isLocal: false, isDefinition: true, virtualIndex: 6, isOptimized: false, scopeLine: 7, file: !19, scope: !6, type: !7, function: i32 (i32)* @callee)
-!6 = !MDFile(filename: "ending-run.c", directory: "/Users/echristo/tmp")
-!7 = !MDSubroutineType(types: !8)
+!5 = !DISubprogram(name: "callee", line: 4, isLocal: false, isDefinition: true, virtualIndex: 6, isOptimized: false, scopeLine: 7, file: !19, scope: !6, type: !7, function: i32 (i32)* @callee)
+!6 = !DIFile(filename: "ending-run.c", directory: "/Users/echristo/tmp")
+!7 = !DISubroutineType(types: !8)
 !8 = !{!9, !9}
-!9 = !MDBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
-!12 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "x", line: 5, arg: 1, scope: !5, file: !6, type: !9)
-!13 = !MDLocation(line: 5, column: 5, scope: !5)
-!14 = !MDLocalVariable(tag: DW_TAG_auto_variable, name: "y", line: 8, scope: !15, file: !6, type: !9)
-!15 = distinct !MDLexicalBlock(line: 7, column: 1, file: !19, scope: !5)
-!16 = !MDLocation(line: 8, column: 9, scope: !15)
-!17 = !MDLocation(line: 8, column: 18, scope: !15)
-!18 = !MDLocation(line: 9, column: 5, scope: !15)
-!19 = !MDFile(filename: "ending-run.c", directory: "/Users/echristo/tmp")
+!9 = !DIBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
+!12 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "x", line: 5, arg: 1, scope: !5, file: !6, type: !9)
+!13 = !DILocation(line: 5, column: 5, scope: !5)
+!14 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "y", line: 8, scope: !15, file: !6, type: !9)
+!15 = distinct !DILexicalBlock(line: 7, column: 1, file: !19, scope: !5)
+!16 = !DILocation(line: 8, column: 9, scope: !15)
+!17 = !DILocation(line: 8, column: 18, scope: !15)
+!18 = !DILocation(line: 9, column: 5, scope: !15)
+!19 = !DIFile(filename: "ending-run.c", directory: "/Users/echristo/tmp")
 !20 = !{i32 1, !"Debug Info Version", i32 3}
diff --git a/test/DebugInfo/X86/enum-class.ll b/test/DebugInfo/X86/enum-class.ll
index 72588811c78..57b2aa404cb 100644
--- a/test/DebugInfo/X86/enum-class.ll
+++ b/test/DebugInfo/X86/enum-class.ll
@@ -8,26 +8,26 @@
 !llvm.dbg.cu = !{!0}
 !llvm.module.flags = !{!23}
 
-!0 = !MDCompileUnit(language: DW_LANG_C_plus_plus, producer: "clang version 3.2 (trunk 157269) (llvm/trunk 157264)", isOptimized: false, emissionKind: 0, file: !22, enums: !1, retainedTypes: !15, subprograms: !15, globals: !17, imports:  !15)
+!0 = !DICompileUnit(language: DW_LANG_C_plus_plus, producer: "clang version 3.2 (trunk 157269) (llvm/trunk 157264)", isOptimized: false, emissionKind: 0, file: !22, enums: !1, retainedTypes: !15, subprograms: !15, globals: !17, imports:  !15)
 !1 = !{!3, !8, !12}
-!3 = !MDCompositeType(tag: DW_TAG_enumeration_type, name: "A", line: 1, size: 32, align: 32, file: !4, baseType: !5, elements: !6)
-!4 = !MDFile(filename: "foo.cpp", directory: "/Users/echristo/tmp")
-!5 = !MDBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
+!3 = !DICompositeType(tag: DW_TAG_enumeration_type, name: "A", line: 1, size: 32, align: 32, file: !4, baseType: !5, elements: !6)
+!4 = !DIFile(filename: "foo.cpp", directory: "/Users/echristo/tmp")
+!5 = !DIBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
 !6 = !{!7}
-!7 = !MDEnumerator(name: "A1", value: 1) ; [ DW_TAG_enumerator ]
-!8 = !MDCompositeType(tag: DW_TAG_enumeration_type, name: "B", line: 2, size: 64, align: 64, file: !4, baseType: !9, elements: !10)
-!9 = !MDBasicType(tag: DW_TAG_base_type, name: "long unsigned int", size: 64, align: 64, encoding: DW_ATE_unsigned)
+!7 = !DIEnumerator(name: "A1", value: 1) ; [ DW_TAG_enumerator ]
+!8 = !DICompositeType(tag: DW_TAG_enumeration_type, name: "B", line: 2, size: 64, align: 64, file: !4, baseType: !9, elements: !10)
+!9 = !DIBasicType(tag: DW_TAG_base_type, name: "long unsigned int", size: 64, align: 64, encoding: DW_ATE_unsigned)
 !10 = !{!11}
-!11 = !MDEnumerator(name: "B1", value: 1) ; [ DW_TAG_enumerator ]
-!12 = !MDCompositeType(tag: DW_TAG_enumeration_type, name: "C", line: 3, size: 32, align: 32, file: !4, elements: !13)
+!11 = !DIEnumerator(name: "B1", value: 1) ; [ DW_TAG_enumerator ]
+!12 = !DICompositeType(tag: DW_TAG_enumeration_type, name: "C", line: 3, size: 32, align: 32, file: !4, elements: !13)
 !13 = !{!14}
-!14 = !MDEnumerator(name: "C1", value: 1) ; [ DW_TAG_enumerator ]
+!14 = !DIEnumerator(name: "C1", value: 1) ; [ DW_TAG_enumerator ]
 !15 = !{}
 !17 = !{!19, !20, !21}
-!19 = !MDGlobalVariable(name: "a", line: 4, isLocal: false, isDefinition: true, scope: null, file: !4, type: !3, variable: i32* @a)
-!20 = !MDGlobalVariable(name: "b", line: 5, isLocal: false, isDefinition: true, scope: null, file: !4, type: !8, variable: i64* @b)
-!21 = !MDGlobalVariable(name: "c", line: 6, isLocal: false, isDefinition: true, scope: null, file: !4, type: !12, variable: i32* @c)
-!22 = !MDFile(filename: "foo.cpp", directory: "/Users/echristo/tmp")
+!19 = !DIGlobalVariable(name: "a", line: 4, isLocal: false, isDefinition: true, scope: null, file: !4, type: !3, variable: i32* @a)
+!20 = !DIGlobalVariable(name: "b", line: 5, isLocal: false, isDefinition: true, scope: null, file: !4, type: !8, variable: i64* @b)
+!21 = !DIGlobalVariable(name: "c", line: 6, isLocal: false, isDefinition: true, scope: null, file: !4, type: !12, variable: i32* @c)
+!22 = !DIFile(filename: "foo.cpp", directory: "/Users/echristo/tmp")
 
 ; CHECK: DW_TAG_enumeration_type [{{.*}}]
 ; CHECK: DW_AT_type [DW_FORM_ref4]
diff --git a/test/DebugInfo/X86/enum-fwd-decl.ll b/test/DebugInfo/X86/enum-fwd-decl.ll
index 2a27eb31730..f5deeb7d671 100644
--- a/test/DebugInfo/X86/enum-fwd-decl.ll
+++ b/test/DebugInfo/X86/enum-fwd-decl.ll
@@ -6,13 +6,13 @@
 !llvm.dbg.cu = !{!0}
 !llvm.module.flags = !{!9}
 
-!0 = !MDCompileUnit(language: DW_LANG_C_plus_plus, producer: "clang version 3.2 (trunk 165274) (llvm/trunk 165272)", isOptimized: false, emissionKind: 0, file: !8, enums: !1, retainedTypes: !1, subprograms: !1, globals: !3, imports:  !1)
+!0 = !DICompileUnit(language: DW_LANG_C_plus_plus, producer: "clang version 3.2 (trunk 165274) (llvm/trunk 165272)", isOptimized: false, emissionKind: 0, file: !8, enums: !1, retainedTypes: !1, subprograms: !1, globals: !3, imports:  !1)
 !1 = !{}
 !3 = !{!5}
-!5 = !MDGlobalVariable(name: "e", line: 2, isLocal: false, isDefinition: true, scope: null, file: !6, type: !7, variable: i16* @e)
-!6 = !MDFile(filename: "foo.cpp", directory: "/tmp")
-!7 = !MDCompositeType(tag: DW_TAG_enumeration_type, name: "E", line: 1, size: 16, align: 16, flags: DIFlagFwdDecl, file: !8)
-!8 = !MDFile(filename: "foo.cpp", directory: "/tmp")
+!5 = !DIGlobalVariable(name: "e", line: 2, isLocal: false, isDefinition: true, scope: null, file: !6, type: !7, variable: i16* @e)
+!6 = !DIFile(filename: "foo.cpp", directory: "/tmp")
+!7 = !DICompositeType(tag: DW_TAG_enumeration_type, name: "E", line: 1, size: 16, align: 16, flags: DIFlagFwdDecl, file: !8)
+!8 = !DIFile(filename: "foo.cpp", directory: "/tmp")
 
 ; CHECK: DW_TAG_enumeration_type
 ; CHECK-NEXT: DW_AT_name
diff --git a/test/DebugInfo/X86/externaltyperef.ll b/test/DebugInfo/X86/externaltyperef.ll
new file mode 100644
index 00000000000..c344d5f068c
--- /dev/null
+++ b/test/DebugInfo/X86/externaltyperef.ll
@@ -0,0 +1,51 @@
+; REQUIRES: object-emission
+; RUN: %llc_dwarf -filetype=obj -O0 < %s | llvm-dwarfdump -debug-dump=info - | FileCheck %s
+; Manually derived by externalizing the composite types from:
+;
+;   namespace N { class B; }
+;   using N::B;
+;   class A;
+;   A *a;
+;
+; Test the direct use of an external type.
+; CHECK: DW_TAG_variable
+; CHECK:   DW_AT_type [DW_FORM_ref4]	  {{.*}}{[[PTR:.*]]}
+; CHECK: [[PTR]]: DW_TAG_pointer_type
+; CHECK:   DW_AT_type [DW_FORM_ref4]  	  {{.*}}{[[A:.*]]}
+; CHECK: [[A]]: DW_TAG_class_type
+; CHECK:   DW_AT_declaration [DW_FORM_flag]	(0x01)
+; CHECK:   DW_AT_signature [DW_FORM_ref_sig8]	(0x4e834ea939695c24)
+; CHECK: [[B:.*]]: DW_TAG_class_type
+; CHECK:   DW_AT_declaration [DW_FORM_flag]	(0x01)
+; CHECK:   DW_AT_signature [DW_FORM_ref_sig8]	(0x942e51c7addda5f7)
+; CHECK:   DW_TAG_imported_declaration
+; CHECK:     DW_AT_import [DW_FORM_ref4]  {{.*}}[[B]]
+
+target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-apple-macosx10.10.0"
+
+%class.A = type opaque
+
+@a = global %class.A* null, align 8
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!13, !14, !15}
+!llvm.ident = !{!16}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C_plus_plus, file: !1, producer: "clang version 3.7.0 (trunk 242039) (llvm/trunk 242046)", isOptimized: false, runtimeVersion: 0, emissionKind: 1, enums: !2, retainedTypes: !3, globals: !5, imports: !11)
+!1 = !DIFile(filename: "test.cpp", directory: "/")
+!2 = !{}
+!3 = !{!4, !9}
+!4 = !DICompositeType(tag: DW_TAG_class_type, name: "A", file: !1, flags: DIFlagExternalTypeRef, identifier: "_ZTS1A")
+!5 = !{!6}
+!6 = !DIGlobalVariable(name: "a", scope: !0, file: !1, line: 2, type: !7, isLocal: false, isDefinition: true, variable: %class.A** @a)
+!7 = !DIDerivedType(tag: DW_TAG_pointer_type, baseType: !"_ZTS1A", size: 64, align: 64)
+!8 = !DICompositeType(tag: DW_TAG_class_type, name: "B", file: !1, flags: DIFlagExternalTypeRef, identifier: "_ZTS1B")
+!9 = !DICompositeType(tag: DW_TAG_class_type, name: "A", file: !1, flags: DIFlagExternalTypeRef, identifier: "_ZTSN1N1BE")
+!10 = !DINamespace(name: "N", scope: null, file: !1, line: 1)
+!11 = !{!12}
+!12 = !DIImportedEntity(tag: DW_TAG_imported_declaration, scope: !0, entity: !"_ZTSN1N1BE", line: 4)
+!13 = !{i32 2, !"Dwarf Version", i32 2}
+!14 = !{i32 2, !"Debug Info Version", i32 3}
+!15 = !{i32 1, !"PIC Level", i32 2}
+!16 = !{!"clang version 3.7.0 (trunk 242039) (llvm/trunk 242046)"}
diff --git a/test/DebugInfo/X86/fission-cu.ll b/test/DebugInfo/X86/fission-cu.ll
index e0592b40cb4..493f84283e2 100644
--- a/test/DebugInfo/X86/fission-cu.ll
+++ b/test/DebugInfo/X86/fission-cu.ll
@@ -8,13 +8,13 @@
 !llvm.dbg.cu = !{!0}
 !llvm.module.flags = !{!9}
 
-!0 = !MDCompileUnit(language: DW_LANG_C99, producer: "clang version 3.3 (trunk 169021) (llvm/trunk 169020)", isOptimized: false, splitDebugFilename: "baz.dwo", emissionKind: 0, file: !8, enums: !1, retainedTypes: !1, subprograms: !1, globals: !3, imports:  !1)
+!0 = !DICompileUnit(language: DW_LANG_C99, producer: "clang version 3.3 (trunk 169021) (llvm/trunk 169020)", isOptimized: false, splitDebugFilename: "baz.dwo", emissionKind: 0, file: !8, enums: !1, retainedTypes: !1, subprograms: !1, globals: !3, imports:  !1)
 !1 = !{}
 !3 = !{!5}
-!5 = !MDGlobalVariable(name: "a", line: 1, isLocal: false, isDefinition: true, scope: null, file: !6, type: !7, variable: i32* @a)
-!6 = !MDFile(filename: "baz.c", directory: "/usr/local/google/home/echristo/tmp")
-!7 = !MDBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
-!8 = !MDFile(filename: "baz.c", directory: "/usr/local/google/home/echristo/tmp")
+!5 = !DIGlobalVariable(name: "a", line: 1, isLocal: false, isDefinition: true, scope: null, file: !6, type: !7, variable: i32* @a)
+!6 = !DIFile(filename: "baz.c", directory: "/usr/local/google/home/echristo/tmp")
+!7 = !DIBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
+!8 = !DIFile(filename: "baz.c", directory: "/usr/local/google/home/echristo/tmp")
 
 ; Check that the skeleton compile unit contains the proper attributes:
 ; This DIE has the following attributes: DW_AT_comp_dir, DW_AT_stmt_list,
diff --git a/test/DebugInfo/X86/fission-hash.ll b/test/DebugInfo/X86/fission-hash.ll
index d4c4a42ee0b..74764416d74 100644
--- a/test/DebugInfo/X86/fission-hash.ll
+++ b/test/DebugInfo/X86/fission-hash.ll
@@ -9,8 +9,8 @@
 !llvm.dbg.cu = !{!0}
 !llvm.module.flags = !{!3, !4}
 
-!0 = !MDCompileUnit(language: DW_LANG_C99, producer: "clang version 3.4 (trunk 188230) (llvm/trunk 188234)", isOptimized: false, splitDebugFilename: "foo.dwo", emissionKind: 0, file: !1, enums: !2, retainedTypes: !2, subprograms: !2, globals: !2, imports: !2)
-!1 = !MDFile(filename: "foo.c", directory: "/usr/local/google/home/echristo/tmp")
+!0 = !DICompileUnit(language: DW_LANG_C99, producer: "clang version 3.4 (trunk 188230) (llvm/trunk 188234)", isOptimized: false, splitDebugFilename: "foo.dwo", emissionKind: 0, file: !1, enums: !2, retainedTypes: !2, subprograms: !2, globals: !2, imports: !2)
+!1 = !DIFile(filename: "foo.c", directory: "/usr/local/google/home/echristo/tmp")
 !2 = !{}
 !3 = !{i32 2, !"Dwarf Version", i32 3}
 !4 = !{i32 1, !"Debug Info Version", i32 3}
diff --git a/test/DebugInfo/X86/fission-inline.ll b/test/DebugInfo/X86/fission-inline.ll
index f55be64c3cc..53e782756ca 100644
--- a/test/DebugInfo/X86/fission-inline.ll
+++ b/test/DebugInfo/X86/fission-inline.ll
@@ -1,4 +1,6 @@
-; RUN: llc -split-dwarf=Enable -O0 < %s -mtriple=x86_64-unknown-linux-gnu -filetype=obj | llvm-dwarfdump -debug-dump=info - | FileCheck %s
+; RUN: llc -split-dwarf=Enable -O0 < %s -mtriple=x86_64-unknown-linux-gnu -filetype=obj > %t
+; RUN: llvm-dwarfdump -debug-dump=info %t | FileCheck %s
+; RUN: llvm-objdump -r %t | FileCheck --check-prefix=RELOCS %s
 
 ; Test the emission of gmlt-like inlining information into the skeleton unit.
 ; This allows inline-aware symbolication/backtracing given only the linked
@@ -65,10 +67,13 @@
 ; CHECK:        DW_TAG_inlined_subroutine
 ; CHECK-NEXT:     DW_AT_abstract_origin {{.*}} "f2<int>"
 ; CHECK-NEXT:     DW_AT_ranges
-; CHECK-NEXT:     DW_AT_call_file
+; CHECK-NOT: {{DW_AT|DW_TAG|NULL}}
+; CHECK:     DW_AT_call_file
 ; CHECK-NEXT:     DW_AT_call_line {{.*}} (18)
 ; CHECK-NOT: DW_
 
+; RELOCS-NOT: RELOCATION RECORDS FOR [.rela.debug_ranges]
+
 ; Function Attrs: uwtable
 define void @_ZN3foo2f3Ez(...) #0 align 2 {
 entry:
@@ -87,33 +92,33 @@ attributes #1 = { "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "n
 !llvm.module.flags = !{!22, !23}
 !llvm.ident = !{!24}
 
-!0 = !MDCompileUnit(language: DW_LANG_C_plus_plus, producer: "clang version 3.6.0 ", isOptimized: false, splitDebugFilename: "fission-inline.dwo", emissionKind: 1, file: !1, enums: !2, retainedTypes: !3, subprograms: !9, globals: !2, imports: !18)
-!1 = !MDFile(filename: "fission-inline.cpp", directory: "/tmp/dbginfo")
+!0 = !DICompileUnit(language: DW_LANG_C_plus_plus, producer: "clang version 3.6.0 ", isOptimized: false, splitDebugFilename: "fission-inline.dwo", emissionKind: 1, file: !1, enums: !2, retainedTypes: !3, subprograms: !9, globals: !2, imports: !18)
+!1 = !DIFile(filename: "fission-inline.cpp", directory: "/tmp/dbginfo")
 !2 = !{}
 !3 = !{!4}
-!4 = !MDCompositeType(tag: DW_TAG_structure_type, name: "foo", line: 1, size: 8, align: 8, file: !1, elements: !5, identifier: "_ZTS3foo")
+!4 = !DICompositeType(tag: DW_TAG_structure_type, name: "foo", line: 1, size: 8, align: 8, file: !1, elements: !5, identifier: "_ZTS3foo")
 !5 = !{!6}
-!6 = !MDSubprogram(name: "f3", linkageName: "_ZN3foo2f3Ez", line: 4, isLocal: false, isDefinition: false, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 4, file: !1, scope: !"_ZTS3foo", type: !7)
-!7 = !MDSubroutineType(types: !8)
+!6 = !DISubprogram(name: "f3", linkageName: "_ZN3foo2f3Ez", line: 4, isLocal: false, isDefinition: false, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 4, file: !1, scope: !"_ZTS3foo", type: !7)
+!7 = !DISubroutineType(types: !8)
 !8 = !{null, null}
 !9 = !{!10, !11}
-!10 = !MDSubprogram(name: "f3", linkageName: "_ZN3foo2f3Ez", line: 15, isLocal: false, isDefinition: true, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 15, file: !1, scope: !"_ZTS3foo", type: !7, function: void (...)* @_ZN3foo2f3Ez, declaration: !6, variables: !2)
-!11 = !MDSubprogram(name: "f2<int>", linkageName: "_ZN3foo2f2IiEEvv", line: 10, isLocal: false, isDefinition: true, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 10, file: !1, scope: !"_ZTS3foo", type: !12, templateParams: !14, declaration: !17, variables: !2)
-!12 = !MDSubroutineType(types: !13)
+!10 = !DISubprogram(name: "f3", linkageName: "_ZN3foo2f3Ez", line: 15, isLocal: false, isDefinition: true, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 15, file: !1, scope: !"_ZTS3foo", type: !7, function: void (...)* @_ZN3foo2f3Ez, declaration: !6, variables: !2)
+!11 = !DISubprogram(name: "f2<int>", linkageName: "_ZN3foo2f2IiEEvv", line: 10, isLocal: false, isDefinition: true, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 10, file: !1, scope: !"_ZTS3foo", type: !12, templateParams: !14, declaration: !17, variables: !2)
+!12 = !DISubroutineType(types: !13)
 !13 = !{null}
 !14 = !{!15}
-!15 = !MDTemplateTypeParameter(name: "T", type: !16)
-!16 = !MDBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
-!17 = !MDSubprogram(name: "f2<int>", linkageName: "_ZN3foo2f2IiEEvv", line: 10, isLocal: false, isDefinition: false, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 10, file: !1, scope: !"_ZTS3foo", type: !12, templateParams: !14)
+!15 = !DITemplateTypeParameter(name: "T", type: !16)
+!16 = !DIBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
+!17 = !DISubprogram(name: "f2<int>", linkageName: "_ZN3foo2f2IiEEvv", line: 10, isLocal: false, isDefinition: false, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 10, file: !1, scope: !"_ZTS3foo", type: !12, templateParams: !14)
 !18 = !{!19}
-!19 = !MDImportedEntity(tag: DW_TAG_imported_declaration, line: 19, scope: !20, entity: !"_ZTS3foo")
-!20 = distinct !MDLexicalBlock(line: 16, column: 13, file: !1, scope: !21)
-!21 = distinct !MDLexicalBlock(line: 16, column: 7, file: !1, scope: !10)
+!19 = !DIImportedEntity(tag: DW_TAG_imported_declaration, line: 19, scope: !20, entity: !"_ZTS3foo")
+!20 = distinct !DILexicalBlock(line: 16, column: 13, file: !1, scope: !21)
+!21 = distinct !DILexicalBlock(line: 16, column: 7, file: !1, scope: !10)
 !22 = !{i32 2, !"Dwarf Version", i32 4}
 !23 = !{i32 2, !"Debug Info Version", i32 3}
 !24 = !{!"clang version 3.6.0 "}
-!25 = !MDLocation(line: 17, column: 5, scope: !20)
-!26 = !MDLocation(line: 11, column: 3, scope: !11, inlinedAt: !27)
-!27 = !MDLocation(line: 18, column: 5, scope: !20)
-!28 = !MDLocation(line: 12, column: 3, scope: !11, inlinedAt: !27)
-!29 = !MDLocation(line: 21, column: 1, scope: !10)
+!25 = !DILocation(line: 17, column: 5, scope: !20)
+!26 = !DILocation(line: 11, column: 3, scope: !11, inlinedAt: !27)
+!27 = !DILocation(line: 18, column: 5, scope: !20)
+!28 = !DILocation(line: 12, column: 3, scope: !11, inlinedAt: !27)
+!29 = !DILocation(line: 21, column: 1, scope: !10)
diff --git a/test/DebugInfo/X86/fission-ranges.ll b/test/DebugInfo/X86/fission-ranges.ll
index 57bce098f4f..e5eb25ce90c 100644
--- a/test/DebugInfo/X86/fission-ranges.ll
+++ b/test/DebugInfo/X86/fission-ranges.ll
@@ -91,8 +91,8 @@ entry:
 ; Function Attrs: nounwind uwtable
 define internal fastcc void @foo() #0 {
 entry:
-  tail call void @llvm.dbg.value(metadata i32 1, i64 0, metadata !13, metadata !MDExpression()), !dbg !30
-  tail call void @llvm.dbg.value(metadata i32 0, i64 0, metadata !14, metadata !MDExpression()), !dbg !31
+  tail call void @llvm.dbg.value(metadata i32 1, i64 0, metadata !13, metadata !DIExpression()), !dbg !30
+  tail call void @llvm.dbg.value(metadata i32 0, i64 0, metadata !14, metadata !DIExpression()), !dbg !31
   %c.promoted9 = load i32, i32* @c, align 4, !dbg !32, !tbaa !33
   br label %for.cond1.preheader, !dbg !31
 
@@ -114,28 +114,28 @@ for.cond7.preheader:                              ; preds = %for.inc10, %for.con
 for.body9:                                        ; preds = %for.body9, %for.cond7.preheader
   %and2 = phi i32 [ %and.lcssa5, %for.cond7.preheader ], [ %and, %for.body9 ], !dbg !40
   %e.01 = phi i32 [ 0, %for.cond7.preheader ], [ %inc, %for.body9 ]
-  tail call void @llvm.dbg.value(metadata i32* @c, i64 0, metadata !19, metadata !MDExpression()), !dbg !40
+  tail call void @llvm.dbg.value(metadata i32* @c, i64 0, metadata !19, metadata !DIExpression()), !dbg !40
   %and = and i32 %and2, 1, !dbg !32
   %inc = add i32 %e.01, 1, !dbg !39
-  tail call void @llvm.dbg.value(metadata i32 %inc, i64 0, metadata !18, metadata !MDExpression()), !dbg !39
+  tail call void @llvm.dbg.value(metadata i32 %inc, i64 0, metadata !18, metadata !DIExpression()), !dbg !39
   %exitcond = icmp eq i32 %inc, 30, !dbg !39
   br i1 %exitcond, label %for.inc10, label %for.body9, !dbg !39
 
 for.inc10:                                        ; preds = %for.body9
   %inc11 = add nsw i32 %b.03, 1, !dbg !38
-  tail call void @llvm.dbg.value(metadata i32 %inc11, i64 0, metadata !15, metadata !MDExpression()), !dbg !38
+  tail call void @llvm.dbg.value(metadata i32 %inc11, i64 0, metadata !15, metadata !DIExpression()), !dbg !38
   %exitcond11 = icmp eq i32 %inc11, 30, !dbg !38
   br i1 %exitcond11, label %for.inc13, label %for.cond7.preheader, !dbg !38
 
 for.inc13:                                        ; preds = %for.inc10
   %inc14 = add i32 %d.06, 1, !dbg !37
-  tail call void @llvm.dbg.value(metadata i32 %inc14, i64 0, metadata !16, metadata !MDExpression()), !dbg !37
+  tail call void @llvm.dbg.value(metadata i32 %inc14, i64 0, metadata !16, metadata !DIExpression()), !dbg !37
   %exitcond12 = icmp eq i32 %inc14, 30, !dbg !37
   br i1 %exitcond12, label %for.inc16, label %for.cond4.preheader, !dbg !37
 
 for.inc16:                                        ; preds = %for.inc13
   %inc17 = add nsw i32 %a.08, 1, !dbg !31
-  tail call void @llvm.dbg.value(metadata i32 %inc17, i64 0, metadata !14, metadata !MDExpression()), !dbg !31
+  tail call void @llvm.dbg.value(metadata i32 %inc17, i64 0, metadata !14, metadata !DIExpression()), !dbg !31
   %exitcond13 = icmp eq i32 %inc17, 30, !dbg !31
   br i1 %exitcond13, label %for.end18, label %for.cond1.preheader, !dbg !31
 
@@ -153,48 +153,48 @@ attributes #1 = { nounwind readnone }
 !llvm.dbg.cu = !{!0}
 !llvm.module.flags = !{!26, !43}
 
-!0 = !MDCompileUnit(language: DW_LANG_C99, producer: "clang version 3.4 (trunk 191700) (llvm/trunk 191710)", isOptimized: true, splitDebugFilename: "small.dwo", emissionKind: 0, file: !1, enums: !2, retainedTypes: !2, subprograms: !3, globals: !2, imports: !2)
-!1 = !MDFile(filename: "small.c", directory: "/usr/local/google/home/echristo/tmp")
+!0 = !DICompileUnit(language: DW_LANG_C99, producer: "clang version 3.4 (trunk 191700) (llvm/trunk 191710)", isOptimized: true, splitDebugFilename: "small.dwo", emissionKind: 0, file: !1, enums: !2, retainedTypes: !2, subprograms: !3, globals: !2, imports: !2)
+!1 = !DIFile(filename: "small.c", directory: "/usr/local/google/home/echristo/tmp")
 !2 = !{}
 !3 = !{!4, !8}
-!4 = !MDSubprogram(name: "bar", line: 18, isLocal: false, isDefinition: true, virtualIndex: 6, isOptimized: true, scopeLine: 19, file: !1, scope: !5, type: !6, function: void ()* @bar, variables: !2)
-!5 = !MDFile(filename: "small.c", directory: "/usr/local/google/home/echristo/tmp")
-!6 = !MDSubroutineType(types: !7)
+!4 = !DISubprogram(name: "bar", line: 18, isLocal: false, isDefinition: true, virtualIndex: 6, isOptimized: true, scopeLine: 19, file: !1, scope: !5, type: !6, function: void ()* @bar, variables: !2)
+!5 = !DIFile(filename: "small.c", directory: "/usr/local/google/home/echristo/tmp")
+!6 = !DISubroutineType(types: !7)
 !7 = !{null}
-!8 = !MDSubprogram(name: "foo", line: 2, isLocal: true, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: true, scopeLine: 3, file: !1, scope: !5, type: !9, function: void ()* @foo, variables: !12)
-!9 = !MDSubroutineType(types: !10)
+!8 = !DISubprogram(name: "foo", line: 2, isLocal: true, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: true, scopeLine: 3, file: !1, scope: !5, type: !9, function: void ()* @foo, variables: !12)
+!9 = !DISubroutineType(types: !10)
 !10 = !{null, !11}
-!11 = !MDBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
+!11 = !DIBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
 !12 = !{!13, !14, !15, !16, !18, !19}
-!13 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "p", line: 2, arg: 1, scope: !8, file: !5, type: !11)
-!14 = !MDLocalVariable(tag: DW_TAG_auto_variable, name: "a", line: 4, scope: !8, file: !5, type: !11)
-!15 = !MDLocalVariable(tag: DW_TAG_auto_variable, name: "b", line: 4, scope: !8, file: !5, type: !11)
-!16 = !MDLocalVariable(tag: DW_TAG_auto_variable, name: "d", line: 5, scope: !8, file: !5, type: !17)
-!17 = !MDBasicType(tag: DW_TAG_base_type, name: "unsigned int", size: 32, align: 32, encoding: DW_ATE_unsigned)
-!18 = !MDLocalVariable(tag: DW_TAG_auto_variable, name: "e", line: 5, scope: !8, file: !5, type: !17)
-!19 = !MDLocalVariable(tag: DW_TAG_auto_variable, name: "w", line: 12, scope: !20, file: !5, type: !25)
-!20 = distinct !MDLexicalBlock(line: 11, column: 0, file: !1, scope: !21)
-!21 = distinct !MDLexicalBlock(line: 10, column: 0, file: !1, scope: !22)
-!22 = distinct !MDLexicalBlock(line: 9, column: 0, file: !1, scope: !23)
-!23 = distinct !MDLexicalBlock(line: 8, column: 0, file: !1, scope: !24)
-!24 = distinct !MDLexicalBlock(line: 7, column: 0, file: !1, scope: !8)
-!25 = !MDDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, baseType: !11)
+!13 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "p", line: 2, arg: 1, scope: !8, file: !5, type: !11)
+!14 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "a", line: 4, scope: !8, file: !5, type: !11)
+!15 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "b", line: 4, scope: !8, file: !5, type: !11)
+!16 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "d", line: 5, scope: !8, file: !5, type: !17)
+!17 = !DIBasicType(tag: DW_TAG_base_type, name: "unsigned int", size: 32, align: 32, encoding: DW_ATE_unsigned)
+!18 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "e", line: 5, scope: !8, file: !5, type: !17)
+!19 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "w", line: 12, scope: !20, file: !5, type: !25)
+!20 = distinct !DILexicalBlock(line: 11, column: 0, file: !1, scope: !21)
+!21 = distinct !DILexicalBlock(line: 10, column: 0, file: !1, scope: !22)
+!22 = distinct !DILexicalBlock(line: 9, column: 0, file: !1, scope: !23)
+!23 = distinct !DILexicalBlock(line: 8, column: 0, file: !1, scope: !24)
+!24 = distinct !DILexicalBlock(line: 7, column: 0, file: !1, scope: !8)
+!25 = !DIDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, baseType: !11)
 !26 = !{i32 2, !"Dwarf Version", i32 4}
-!27 = !MDLocation(line: 20, scope: !4)
-!28 = !MDLocation(line: 21, scope: !4)
+!27 = !DILocation(line: 20, scope: !4)
+!28 = !DILocation(line: 21, scope: !4)
 !29 = !{i32 1}
-!30 = !MDLocation(line: 2, scope: !8)
-!31 = !MDLocation(line: 7, scope: !24)
-!32 = !MDLocation(line: 13, scope: !20)
+!30 = !DILocation(line: 2, scope: !8)
+!31 = !DILocation(line: 7, scope: !24)
+!32 = !DILocation(line: 13, scope: !20)
 !33 = !{!34, !34, i64 0}
 !34 = !{!"int", !35, i64 0}
 !35 = !{!"omnipotent char", !36, i64 0}
 !36 = !{!"Simple C/C++ TBAA"}
-!37 = !MDLocation(line: 8, scope: !23)
-!38 = !MDLocation(line: 9, scope: !22)
-!39 = !MDLocation(line: 10, scope: !21)
-!40 = !MDLocation(line: 12, scope: !20)
+!37 = !DILocation(line: 8, scope: !23)
+!38 = !DILocation(line: 9, scope: !22)
+!39 = !DILocation(line: 10, scope: !21)
+!40 = !DILocation(line: 12, scope: !20)
 !41 = !{i32* @c}
-!42 = !MDLocation(line: 15, scope: !8)
+!42 = !DILocation(line: 15, scope: !8)
 !43 = !{i32 1, !"Debug Info Version", i32 3}
 !44 = !{i32 0}
diff --git a/test/DebugInfo/X86/float_const.ll b/test/DebugInfo/X86/float_const.ll
index 7d5a355be76..6e99ffcc1f2 100644
--- a/test/DebugInfo/X86/float_const.ll
+++ b/test/DebugInfo/X86/float_const.ll
@@ -31,25 +31,25 @@ attributes #1 = { nounwind readnone }
 !llvm.module.flags = !{!15, !16, !17}
 !llvm.ident = !{!18}
 
-!0 = !MDCompileUnit(language: DW_LANG_C99, producer: "clang version 3.7.0 (trunk 227686)", isOptimized: true, emissionKind: 1, file: !1, enums: !2, retainedTypes: !3, subprograms: !6, globals: !2, imports: !2)
-!1 = !MDFile(filename: "foo.c", directory: "")
+!0 = !DICompileUnit(language: DW_LANG_C99, producer: "clang version 3.7.0 (trunk 227686)", isOptimized: true, emissionKind: 1, file: !1, enums: !2, retainedTypes: !3, subprograms: !6, globals: !2, imports: !2)
+!1 = !DIFile(filename: "foo.c", directory: "")
 !2 = !{}
 !3 = !{!4}
-!4 = !MDDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, baseType: !5)
-!5 = !MDBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
+!4 = !DIDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, baseType: !5)
+!5 = !DIBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
 !6 = !{!7}
-!7 = !MDSubprogram(name: "foo", line: 1, isLocal: false, isDefinition: true, isOptimized: true, scopeLine: 1, file: !8, scope: !9, type: !10, function: void ()* @foo, variables: !12)
-!8 = !MDFile(filename: "foo.c", directory: "")
-!9 = !MDFile(filename: "foo.c", directory: "")
-!10 = !MDSubroutineType(types: !11)
+!7 = !DISubprogram(name: "foo", line: 1, isLocal: false, isDefinition: true, isOptimized: true, scopeLine: 1, file: !8, scope: !9, type: !10, function: void ()* @foo, variables: !12)
+!8 = !DIFile(filename: "foo.c", directory: "")
+!9 = !DIFile(filename: "foo.c", directory: "")
+!10 = !DISubroutineType(types: !11)
 !11 = !{null}
 !12 = !{!13}
-!13 = !MDLocalVariable(tag: DW_TAG_auto_variable, name: "a", line: 2, scope: !7, file: !9, type: !14)
-!14 = !MDBasicType(tag: DW_TAG_base_type, name: "float", size: 32, align: 32, encoding: DW_ATE_float)
+!13 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "a", line: 2, scope: !7, file: !9, type: !14)
+!14 = !DIBasicType(tag: DW_TAG_base_type, name: "float", size: 32, align: 32, encoding: DW_ATE_float)
 !15 = !{i32 2, !"Dwarf Version", i32 2}
 !16 = !{i32 2, !"Debug Info Version", i32 3}
 !17 = !{i32 1, !"PIC Level", i32 2}
 !18 = !{!"clang version 3.7.0 (trunk 227686)"}
-!19 = !MDExpression()
-!20 = !MDLocation(line: 2, column: 9, scope: !7)
-!21 = !MDLocation(line: 4, column: 1, scope: !7)
+!19 = !DIExpression()
+!20 = !DILocation(line: 2, column: 9, scope: !7)
+!21 = !DILocation(line: 4, column: 1, scope: !7)
diff --git a/test/DebugInfo/X86/formal_parameter.ll b/test/DebugInfo/X86/formal_parameter.ll
index 97f45ece12b..e8a8b5f1803 100644
--- a/test/DebugInfo/X86/formal_parameter.ll
+++ b/test/DebugInfo/X86/formal_parameter.ll
@@ -28,7 +28,7 @@ define void @foo(i32 %map) #0 {
 entry:
   %map.addr = alloca i32, align 4
   store i32 %map, i32* %map.addr, align 4, !tbaa !15
-  call void @llvm.dbg.declare(metadata i32* %map.addr, metadata !10, metadata !MDExpression()), !dbg !14
+  call void @llvm.dbg.declare(metadata i32* %map.addr, metadata !10, metadata !DIExpression()), !dbg !14
   %call = call i32 (i32*, ...) bitcast (i32 (...)* @lookup to i32 (i32*, ...)*)(i32* %map.addr) #3, !dbg !19
   ; Ensure that all dbg intrinsics have the same scope after
   ; LowerDbgDeclare is finished with them.
@@ -59,26 +59,26 @@ attributes #3 = { nounwind }
 !llvm.module.flags = !{!11, !12}
 !llvm.ident = !{!13}
 
-!0 = !MDCompileUnit(language: DW_LANG_C99, producer: "clang version 3.5.0 ", isOptimized: true, emissionKind: 1, file: !1, enums: !2, retainedTypes: !2, subprograms: !3, globals: !2, imports: !2)
-!1 = !MDFile(filename: "formal_parameter.c", directory: "")
+!0 = !DICompileUnit(language: DW_LANG_C99, producer: "clang version 3.5.0 ", isOptimized: true, emissionKind: 1, file: !1, enums: !2, retainedTypes: !2, subprograms: !3, globals: !2, imports: !2)
+!1 = !DIFile(filename: "formal_parameter.c", directory: "")
 !2 = !{}
 !3 = !{!4}
-!4 = !MDSubprogram(name: "foo", line: 1, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: true, scopeLine: 2, file: !1, scope: !5, type: !6, function: void (i32)* @foo, variables: !9)
-!5 = !MDFile(filename: "formal_parameter.c", directory: "")
-!6 = !MDSubroutineType(types: !7)
+!4 = !DISubprogram(name: "foo", line: 1, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: true, scopeLine: 2, file: !1, scope: !5, type: !6, function: void (i32)* @foo, variables: !9)
+!5 = !DIFile(filename: "formal_parameter.c", directory: "")
+!6 = !DISubroutineType(types: !7)
 !7 = !{null, !8}
-!8 = !MDBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
+!8 = !DIBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
 !9 = !{!10}
-!10 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "map", line: 1, arg: 1, scope: !4, file: !5, type: !8)
+!10 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "map", line: 1, arg: 1, scope: !4, file: !5, type: !8)
 !11 = !{i32 2, !"Dwarf Version", i32 2}
 !12 = !{i32 1, !"Debug Info Version", i32 3}
 !13 = !{!"clang version 3.5.0 "}
-!14 = !MDLocation(line: 1, scope: !4)
+!14 = !DILocation(line: 1, scope: !4)
 !15 = !{!16, !16, i64 0}
 !16 = !{!"int", !17, i64 0}
 !17 = !{!"omnipotent char", !18, i64 0}
 !18 = !{!"Simple C/C++ TBAA"}
-!19 = !MDLocation(line: 3, scope: !4)
-!20 = !MDLocation(line: 4, scope: !21)
-!21 = distinct !MDLexicalBlock(line: 4, column: 0, file: !1, scope: !4)
-!22 = !MDLocation(line: 5, scope: !4)
+!19 = !DILocation(line: 3, scope: !4)
+!20 = !DILocation(line: 4, scope: !21)
+!21 = distinct !DILexicalBlock(line: 4, column: 0, file: !1, scope: !4)
+!22 = !DILocation(line: 5, scope: !4)
diff --git a/test/DebugInfo/X86/frame-register.ll b/test/DebugInfo/X86/frame-register.ll
index 7a5285edab5..bcf65a1a385 100644
--- a/test/DebugInfo/X86/frame-register.ll
+++ b/test/DebugInfo/X86/frame-register.ll
@@ -33,28 +33,28 @@ attributes #1 = { nounwind readnone }
 !llvm.module.flags = !{!12, !13}
 !llvm.ident = !{!14}
 
-!0 = !MDCompileUnit(language: DW_LANG_C99, producer: "clang version 3.7.0", isOptimized: false, emissionKind: 1, file: !1, enums: !2, retainedTypes: !2, subprograms: !3, globals: !2, imports: !2)
-!1 = !MDFile(filename: "x.c", directory: "")
+!0 = !DICompileUnit(language: DW_LANG_C99, producer: "clang version 3.7.0", isOptimized: false, emissionKind: 1, file: !1, enums: !2, retainedTypes: !2, subprograms: !3, globals: !2, imports: !2)
+!1 = !DIFile(filename: "x.c", directory: "")
 !2 = !{}
 !3 = !{!4, !9}
-!4 = !MDSubprogram(name: "foo", line: 2, isLocal: false, isDefinition: true, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 3, file: !1, scope: !5, type: !6, function: i32 (i32)* @foo, variables: !2)
-!5 = !MDFile(filename: "x.c", directory: "")
-!6 = !MDSubroutineType(types: !7)
+!4 = !DISubprogram(name: "foo", line: 2, isLocal: false, isDefinition: true, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 3, file: !1, scope: !5, type: !6, function: i32 (i32)* @foo, variables: !2)
+!5 = !DIFile(filename: "x.c", directory: "")
+!6 = !DISubroutineType(types: !7)
 !7 = !{!8, !8}
-!8 = !MDBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
-!9 = !MDSubprogram(name: "main", line: 8, isLocal: false, isDefinition: true, isOptimized: false, scopeLine: 9, file: !1, scope: !5, type: !10, function: i32 ()* @main, variables: !2)
-!10 = !MDSubroutineType(types: !11)
+!8 = !DIBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
+!9 = !DISubprogram(name: "main", line: 8, isLocal: false, isDefinition: true, isOptimized: false, scopeLine: 9, file: !1, scope: !5, type: !10, function: i32 ()* @main, variables: !2)
+!10 = !DISubroutineType(types: !11)
 !11 = !{!8}
 !12 = !{i32 2, !"Dwarf Version", i32 4}
 !13 = !{i32 2, !"Debug Info Version", i32 3}
 !14 = !{!"clang version 3.7.0"}
-!15 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "i", line: 2, arg: 1, scope: !4, file: !5, type: !8)
-!16 = !MDExpression()
-!17 = !MDLocation(line: 2, column: 10, scope: !4)
-!18 = !MDLocation(line: 4, column: 10, scope: !4)
-!19 = !MDLocation(line: 4, column: 3, scope: !4)
-!20 = !MDLocalVariable(tag: DW_TAG_auto_variable, name: "i", line: 10, scope: !9, file: !5, type: !8)
-!21 = !MDLocation(line: 10, column: 7, scope: !9)
-!22 = !MDLocation(line: 11, column: 15, scope: !9)
-!23 = !MDLocation(line: 11, column: 10, scope: !9)
-!24 = !MDLocation(line: 11, column: 3, scope: !9)
+!15 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "i", line: 2, arg: 1, scope: !4, file: !5, type: !8)
+!16 = !DIExpression()
+!17 = !DILocation(line: 2, column: 10, scope: !4)
+!18 = !DILocation(line: 4, column: 10, scope: !4)
+!19 = !DILocation(line: 4, column: 3, scope: !4)
+!20 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "i", line: 10, scope: !9, file: !5, type: !8)
+!21 = !DILocation(line: 10, column: 7, scope: !9)
+!22 = !DILocation(line: 11, column: 15, scope: !9)
+!23 = !DILocation(line: 11, column: 10, scope: !9)
+!24 = !DILocation(line: 11, column: 3, scope: !9)
diff --git a/test/DebugInfo/X86/generate-odr-hash.ll b/test/DebugInfo/X86/generate-odr-hash.ll
index b5c100a921b..7ac9a69181f 100644
--- a/test/DebugInfo/X86/generate-odr-hash.ll
+++ b/test/DebugInfo/X86/generate-odr-hash.ll
@@ -183,7 +183,7 @@
 define void @_Z3foov() #0 {
 entry:
   %b = alloca %struct.baz, align 1
-  call void @llvm.dbg.declare(metadata %struct.baz* %b, metadata !46, metadata !MDExpression()), !dbg !48
+  call void @llvm.dbg.declare(metadata %struct.baz* %b, metadata !46, metadata !DIExpression()), !dbg !48
   ret void, !dbg !49
 }
 
@@ -201,7 +201,7 @@ define internal void @_ZN12_GLOBAL__N_16walrusC2Ev(%"struct.<anonymous namespace
 entry:
   %this.addr = alloca %"struct.<anonymous namespace>::walrus"*, align 8
   store %"struct.<anonymous namespace>::walrus"* %this, %"struct.<anonymous namespace>::walrus"** %this.addr, align 8
-  call void @llvm.dbg.declare(metadata %"struct.<anonymous namespace>::walrus"** %this.addr, metadata !51, metadata !MDExpression()), !dbg !53
+  call void @llvm.dbg.declare(metadata %"struct.<anonymous namespace>::walrus"** %this.addr, metadata !51, metadata !DIExpression()), !dbg !53
   %this1 = load %"struct.<anonymous namespace>::walrus"*, %"struct.<anonymous namespace>::walrus"** %this.addr
   ret void, !dbg !54
 }
@@ -219,58 +219,58 @@ attributes #1 = { nounwind readnone }
 !llvm.module.flags = !{!43, !44}
 !llvm.ident = !{!45}
 
-!0 = !MDCompileUnit(language: DW_LANG_C_plus_plus, producer: "clang version 3.5 ", isOptimized: false, splitDebugFilename: "bar.dwo", emissionKind: 0, file: !1, enums: !2, retainedTypes: !3, subprograms: !21, globals: !38, imports: !2)
-!1 = !MDFile(filename: "bar.cpp", directory: "/tmp/dbginfo")
+!0 = !DICompileUnit(language: DW_LANG_C_plus_plus, producer: "clang version 3.5 ", isOptimized: false, splitDebugFilename: "bar.dwo", emissionKind: 0, file: !1, enums: !2, retainedTypes: !3, subprograms: !21, globals: !38, imports: !2)
+!1 = !DIFile(filename: "bar.cpp", directory: "/tmp/dbginfo")
 !2 = !{}
 !3 = !{!4, !6, !14, !17}
-!4 = !MDCompositeType(tag: DW_TAG_structure_type, name: "bar", line: 1, size: 8, align: 8, file: !5, elements: !2, identifier: "_ZTS3bar")
-!5 = !MDFile(filename: "bar.h", directory: "/tmp/dbginfo")
-!6 = !MDCompositeType(tag: DW_TAG_class_type, name: "fluffy", line: 13, size: 64, align: 32, file: !1, scope: !7, elements: !10, identifier: "_ZTSN7echidna8capybara8mongoose6fluffyE")
-!7 = !MDNamespace(name: "mongoose", line: 12, file: !1, scope: !8)
-!8 = !MDNamespace(name: "capybara", line: 11, file: !1, scope: !9)
-!9 = !MDNamespace(name: "echidna", line: 10, file: !1, scope: null)
+!4 = !DICompositeType(tag: DW_TAG_structure_type, name: "bar", line: 1, size: 8, align: 8, file: !5, elements: !2, identifier: "_ZTS3bar")
+!5 = !DIFile(filename: "bar.h", directory: "/tmp/dbginfo")
+!6 = !DICompositeType(tag: DW_TAG_class_type, name: "fluffy", line: 13, size: 64, align: 32, file: !1, scope: !7, elements: !10, identifier: "_ZTSN7echidna8capybara8mongoose6fluffyE")
+!7 = !DINamespace(name: "mongoose", line: 12, file: !1, scope: !8)
+!8 = !DINamespace(name: "capybara", line: 11, file: !1, scope: !9)
+!9 = !DINamespace(name: "echidna", line: 10, file: !1, scope: null)
 !10 = !{!11, !13}
-!11 = !MDDerivedType(tag: DW_TAG_member, name: "a", line: 14, size: 32, align: 32, flags: DIFlagPrivate, file: !1, scope: !"_ZTSN7echidna8capybara8mongoose6fluffyE", baseType: !12)
-!12 = !MDBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
-!13 = !MDDerivedType(tag: DW_TAG_member, name: "b", line: 15, size: 32, align: 32, offset: 32, flags: DIFlagPrivate, file: !1, scope: !"_ZTSN7echidna8capybara8mongoose6fluffyE", baseType: !12)
-!14 = !MDCompositeType(tag: DW_TAG_structure_type, name: "wombat", line: 31, size: 64, align: 32, file: !1, elements: !15, identifier: "_ZTS6wombat")
+!11 = !DIDerivedType(tag: DW_TAG_member, name: "a", line: 14, size: 32, align: 32, flags: DIFlagPrivate, file: !1, scope: !"_ZTSN7echidna8capybara8mongoose6fluffyE", baseType: !12)
+!12 = !DIBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
+!13 = !DIDerivedType(tag: DW_TAG_member, name: "b", line: 15, size: 32, align: 32, offset: 32, flags: DIFlagPrivate, file: !1, scope: !"_ZTSN7echidna8capybara8mongoose6fluffyE", baseType: !12)
+!14 = !DICompositeType(tag: DW_TAG_structure_type, name: "wombat", line: 31, size: 64, align: 32, file: !1, elements: !15, identifier: "_ZTS6wombat")
 !15 = !{!16}
-!16 = !MDDerivedType(tag: DW_TAG_member, name: "a_b", line: 35, size: 64, align: 32, file: !1, scope: !"_ZTS6wombat", baseType: !"_ZTSN6wombatUt_E")
-!17 = !MDCompositeType(tag: DW_TAG_structure_type, line: 32, size: 64, align: 32, file: !1, scope: !"_ZTS6wombat", elements: !18, identifier: "_ZTSN6wombatUt_E")
+!16 = !DIDerivedType(tag: DW_TAG_member, name: "a_b", line: 35, size: 64, align: 32, file: !1, scope: !"_ZTS6wombat", baseType: !"_ZTSN6wombatUt_E")
+!17 = !DICompositeType(tag: DW_TAG_structure_type, line: 32, size: 64, align: 32, file: !1, scope: !"_ZTS6wombat", elements: !18, identifier: "_ZTSN6wombatUt_E")
 !18 = !{!19, !20}
-!19 = !MDDerivedType(tag: DW_TAG_member, name: "a", line: 33, size: 32, align: 32, file: !1, scope: !"_ZTSN6wombatUt_E", baseType: !12)
-!20 = !MDDerivedType(tag: DW_TAG_member, name: "b", line: 34, size: 32, align: 32, offset: 32, file: !1, scope: !"_ZTSN6wombatUt_E", baseType: !12)
+!19 = !DIDerivedType(tag: DW_TAG_member, name: "a", line: 33, size: 32, align: 32, file: !1, scope: !"_ZTSN6wombatUt_E", baseType: !12)
+!20 = !DIDerivedType(tag: DW_TAG_member, name: "b", line: 34, size: 32, align: 32, offset: 32, file: !1, scope: !"_ZTSN6wombatUt_E", baseType: !12)
 !21 = !{!22, !26, !27, !36}
-!22 = !MDSubprogram(name: "foo", linkageName: "_Z3foov", line: 5, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 5, file: !1, scope: !23, type: !24, function: void ()* @_Z3foov, variables: !2)
-!23 = !MDFile(filename: "bar.cpp", directory: "/tmp/dbginfo")
-!24 = !MDSubroutineType(types: !25)
+!22 = !DISubprogram(name: "foo", linkageName: "_Z3foov", line: 5, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 5, file: !1, scope: !23, type: !24, function: void ()* @_Z3foov, variables: !2)
+!23 = !DIFile(filename: "bar.cpp", directory: "/tmp/dbginfo")
+!24 = !DISubroutineType(types: !25)
 !25 = !{null}
-!26 = !MDSubprogram(name: "__cxx_global_var_init", line: 29, isLocal: true, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 29, file: !1, scope: !23, type: !24, function: void ()* @__cxx_global_var_init, variables: !2)
-!27 = !MDSubprogram(name: "walrus", linkageName: "_ZN12_GLOBAL__N_16walrusC2Ev", line: 25, isLocal: true, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 25, file: !1, scope: !28, type: !32, function: void (%"struct.<anonymous namespace>::walrus"*)* @_ZN12_GLOBAL__N_16walrusC2Ev, declaration: !31, variables: !2)
-!28 = !MDCompositeType(tag: DW_TAG_structure_type, name: "walrus", line: 24, size: 8, align: 8, file: !1, scope: !29, elements: !30)
-!29 = !MDNamespace(line: 23, file: !1, scope: null)
+!26 = !DISubprogram(name: "__cxx_global_var_init", line: 29, isLocal: true, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 29, file: !1, scope: !23, type: !24, function: void ()* @__cxx_global_var_init, variables: !2)
+!27 = !DISubprogram(name: "walrus", linkageName: "_ZN12_GLOBAL__N_16walrusC2Ev", line: 25, isLocal: true, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 25, file: !1, scope: !28, type: !32, function: void (%"struct.<anonymous namespace>::walrus"*)* @_ZN12_GLOBAL__N_16walrusC2Ev, declaration: !31, variables: !2)
+!28 = !DICompositeType(tag: DW_TAG_structure_type, name: "walrus", line: 24, size: 8, align: 8, file: !1, scope: !29, elements: !30)
+!29 = !DINamespace(line: 23, file: !1, scope: null)
 !30 = !{!31}
-!31 = !MDSubprogram(name: "walrus", line: 25, isLocal: false, isDefinition: false, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 25, file: !1, scope: !28, type: !32)
-!32 = !MDSubroutineType(types: !33)
+!31 = !DISubprogram(name: "walrus", line: 25, isLocal: false, isDefinition: false, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 25, file: !1, scope: !28, type: !32)
+!32 = !DISubroutineType(types: !33)
 !33 = !{null, !34}
-!34 = !MDDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, flags: DIFlagArtificial | DIFlagObjectPointer, baseType: !28)
-!36 = !MDSubprogram(name: "", linkageName: "_GLOBAL__I_a", line: 25, isLocal: true, isDefinition: true, virtualIndex: 6, flags: DIFlagArtificial, isOptimized: false, scopeLine: 25, file: !1, scope: !23, type: !37, function: void ()* @_GLOBAL__I_a, variables: !2)
-!37 = !MDSubroutineType(types: !2)
+!34 = !DIDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, flags: DIFlagArtificial | DIFlagObjectPointer, baseType: !28)
+!36 = !DISubprogram(name: "", linkageName: "_GLOBAL__I_a", line: 25, isLocal: true, isDefinition: true, virtualIndex: 6, flags: DIFlagArtificial, isOptimized: false, scopeLine: 25, file: !1, scope: !23, type: !37, function: void ()* @_GLOBAL__I_a, variables: !2)
+!37 = !DISubroutineType(types: !2)
 !38 = !{!39, !40, !41, !42}
-!39 = !MDGlobalVariable(name: "b", line: 3, isLocal: false, isDefinition: true, scope: null, file: !23, type: !4, variable: %struct.bar* @b)
-!40 = !MDGlobalVariable(name: "animal", linkageName: "_ZN7echidna8capybara8mongoose6animalE", line: 18, isLocal: false, isDefinition: true, scope: !7, file: !23, type: !6, variable: %"class.echidna::capybara::mongoose::fluffy"* @_ZN7echidna8capybara8mongoose6animalE)
-!41 = !MDGlobalVariable(name: "w", line: 29, isLocal: true, isDefinition: true, scope: null, file: !23, type: !28, variable: %"struct.<anonymous namespace>::walrus"* @w)
-!42 = !MDGlobalVariable(name: "wom", line: 38, isLocal: false, isDefinition: true, scope: null, file: !23, type: !14, variable: %struct.wombat* @wom)
+!39 = !DIGlobalVariable(name: "b", line: 3, isLocal: false, isDefinition: true, scope: null, file: !23, type: !4, variable: %struct.bar* @b)
+!40 = !DIGlobalVariable(name: "animal", linkageName: "_ZN7echidna8capybara8mongoose6animalE", line: 18, isLocal: false, isDefinition: true, scope: !7, file: !23, type: !6, variable: %"class.echidna::capybara::mongoose::fluffy"* @_ZN7echidna8capybara8mongoose6animalE)
+!41 = !DIGlobalVariable(name: "w", line: 29, isLocal: true, isDefinition: true, scope: null, file: !23, type: !28, variable: %"struct.<anonymous namespace>::walrus"* @w)
+!42 = !DIGlobalVariable(name: "wom", line: 38, isLocal: false, isDefinition: true, scope: null, file: !23, type: !14, variable: %struct.wombat* @wom)
 !43 = !{i32 2, !"Dwarf Version", i32 4}
 !44 = !{i32 1, !"Debug Info Version", i32 3}
 !45 = !{!"clang version 3.5 "}
-!46 = !MDLocalVariable(tag: DW_TAG_auto_variable, name: "b", line: 7, scope: !22, file: !23, type: !47)
-!47 = !MDCompositeType(tag: DW_TAG_structure_type, name: "baz", line: 6, size: 8, align: 8, file: !1, scope: !22, elements: !2)
-!48 = !MDLocation(line: 7, scope: !22)
-!49 = !MDLocation(line: 8, scope: !22)
-!50 = !MDLocation(line: 29, scope: !26)
-!51 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "this", arg: 1, flags: DIFlagArtificial | DIFlagObjectPointer, scope: !27, type: !52)
-!52 = !MDDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, baseType: !28)
-!53 = !MDLocation(line: 0, scope: !27)
-!54 = !MDLocation(line: 25, scope: !27)
-!55 = !MDLocation(line: 25, scope: !36)
+!46 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "b", line: 7, scope: !22, file: !23, type: !47)
+!47 = !DICompositeType(tag: DW_TAG_structure_type, name: "baz", line: 6, size: 8, align: 8, file: !1, scope: !22, elements: !2)
+!48 = !DILocation(line: 7, scope: !22)
+!49 = !DILocation(line: 8, scope: !22)
+!50 = !DILocation(line: 29, scope: !26)
+!51 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "this", arg: 1, flags: DIFlagArtificial | DIFlagObjectPointer, scope: !27, type: !52)
+!52 = !DIDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, baseType: !28)
+!53 = !DILocation(line: 0, scope: !27)
+!54 = !DILocation(line: 25, scope: !27)
+!55 = !DILocation(line: 25, scope: !36)
diff --git a/test/DebugInfo/X86/ghost-sdnode-dbgvalues.ll b/test/DebugInfo/X86/ghost-sdnode-dbgvalues.ll
index b43702fd132..ef249e8fab9 100644
--- a/test/DebugInfo/X86/ghost-sdnode-dbgvalues.ll
+++ b/test/DebugInfo/X86/ghost-sdnode-dbgvalues.ll
@@ -68,38 +68,38 @@ attributes #1 = { nounwind readnone }
 !llvm.module.flags = !{!13, !14}
 !llvm.ident = !{!15}
 
-!0 = !MDCompileUnit(language: DW_LANG_C99, producer: "clang version 3.6.0 ", isOptimized: false, emissionKind: 1, file: !1, enums: !2, retainedTypes: !3, subprograms: !7, globals: !2, imports: !2)
-!1 = !MDFile(filename: "ghost-sdnode-dbgvalues.c", directory: "/tmp")
+!0 = !DICompileUnit(language: DW_LANG_C99, producer: "clang version 3.6.0 ", isOptimized: false, emissionKind: 1, file: !1, enums: !2, retainedTypes: !3, subprograms: !7, globals: !2, imports: !2)
+!1 = !DIFile(filename: "ghost-sdnode-dbgvalues.c", directory: "/tmp")
 !2 = !{}
 !3 = !{!4}
-!4 = !MDDerivedType(tag: DW_TAG_typedef, name: "int16_t", line: 30, file: !5, baseType: !6)
-!5 = !MDFile(filename: "/usr/include/sys/_types/_int16_t.h", directory: "/tmp")
-!6 = !MDBasicType(tag: DW_TAG_base_type, name: "short", size: 16, align: 16, encoding: DW_ATE_signed)
+!4 = !DIDerivedType(tag: DW_TAG_typedef, name: "int16_t", line: 30, file: !5, baseType: !6)
+!5 = !DIFile(filename: "/usr/include/sys/_types/_int16_t.h", directory: "/tmp")
+!6 = !DIBasicType(tag: DW_TAG_base_type, name: "short", size: 16, align: 16, encoding: DW_ATE_signed)
 !7 = !{!8}
-!8 = !MDSubprogram(name: "foo", line: 3, isLocal: false, isDefinition: true, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 3, file: !1, scope: !9, type: !10, function: i32 (i32)* @foo, variables: !2)
-!9 = !MDFile(filename: "ghost-sdnode-dbgvalues.c", directory: "/tmp")
-!10 = !MDSubroutineType(types: !11)
+!8 = !DISubprogram(name: "foo", line: 3, isLocal: false, isDefinition: true, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 3, file: !1, scope: !9, type: !10, function: i32 (i32)* @foo, variables: !2)
+!9 = !DIFile(filename: "ghost-sdnode-dbgvalues.c", directory: "/tmp")
+!10 = !DISubroutineType(types: !11)
 !11 = !{!12, !12}
-!12 = !MDBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
+!12 = !DIBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
 !13 = !{i32 2, !"Dwarf Version", i32 2}
 !14 = !{i32 2, !"Debug Info Version", i32 3}
 !15 = !{!"clang version 3.6.0 "}
-!16 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "a", line: 3, arg: 1, scope: !8, file: !9, type: !12)
-!17 = !MDExpression()
-!18 = !MDLocation(line: 3, column: 13, scope: !8)
-!19 = !MDLocation(line: 4, column: 5, scope: !8)
-!20 = !MDLocalVariable(tag: DW_TAG_auto_variable, name: "b", line: 4, scope: !8, file: !9, type: !12)
-!21 = !MDLocation(line: 4, column: 9, scope: !8)
-!22 = !MDLocation(line: 5, column: 5, scope: !8)
-!23 = !MDLocalVariable(tag: DW_TAG_auto_variable, name: "c", line: 5, scope: !8, file: !9, type: !12)
-!24 = !MDLocation(line: 5, column: 9, scope: !8)
-!25 = !MDLocation(line: 6, column: 5, scope: !8)
-!26 = !MDLocalVariable(tag: DW_TAG_auto_variable, name: "d", line: 6, scope: !8, file: !9, type: !12)
-!27 = !MDLocation(line: 6, column: 9, scope: !8)
-!28 = !MDLocation(line: 7, column: 5, scope: !8)
-!29 = !MDLocalVariable(tag: DW_TAG_auto_variable, name: "e", line: 7, scope: !8, file: !9, type: !12)
-!30 = !MDLocation(line: 7, column: 9, scope: !8)
-!31 = !MDLocation(line: 8, column: 5, scope: !8)
-!32 = !MDLocalVariable(tag: DW_TAG_auto_variable, name: "f", line: 8, scope: !8, file: !9, type: !12)
-!33 = !MDLocation(line: 8, column: 9, scope: !8)
-!34 = !MDLocation(line: 9, column: 5, scope: !8)
+!16 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "a", line: 3, arg: 1, scope: !8, file: !9, type: !12)
+!17 = !DIExpression()
+!18 = !DILocation(line: 3, column: 13, scope: !8)
+!19 = !DILocation(line: 4, column: 5, scope: !8)
+!20 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "b", line: 4, scope: !8, file: !9, type: !12)
+!21 = !DILocation(line: 4, column: 9, scope: !8)
+!22 = !DILocation(line: 5, column: 5, scope: !8)
+!23 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "c", line: 5, scope: !8, file: !9, type: !12)
+!24 = !DILocation(line: 5, column: 9, scope: !8)
+!25 = !DILocation(line: 6, column: 5, scope: !8)
+!26 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "d", line: 6, scope: !8, file: !9, type: !12)
+!27 = !DILocation(line: 6, column: 9, scope: !8)
+!28 = !DILocation(line: 7, column: 5, scope: !8)
+!29 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "e", line: 7, scope: !8, file: !9, type: !12)
+!30 = !DILocation(line: 7, column: 9, scope: !8)
+!31 = !DILocation(line: 8, column: 5, scope: !8)
+!32 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "f", line: 8, scope: !8, file: !9, type: !12)
+!33 = !DILocation(line: 8, column: 9, scope: !8)
+!34 = !DILocation(line: 9, column: 5, scope: !8)
diff --git a/test/DebugInfo/X86/gnu-public-names-empty.ll b/test/DebugInfo/X86/gnu-public-names-empty.ll
index 798cf0a269e..d87d8ef1900 100644
--- a/test/DebugInfo/X86/gnu-public-names-empty.ll
+++ b/test/DebugInfo/X86/gnu-public-names-empty.ll
@@ -12,8 +12,8 @@
 !llvm.dbg.cu = !{!0}
 !llvm.module.flags = !{!3, !4}
 
-!0 = !MDCompileUnit(language: DW_LANG_C99, producer: "clang version 3.4 (trunk 191846) (llvm/trunk 191866)", isOptimized: false, emissionKind: 0, file: !1, enums: !2, retainedTypes: !2, subprograms: !2, globals: !2, imports: !2)
-!1 = !MDFile(filename: "foo.c", directory: "/usr/local/google/home/echristo/tmp")
+!0 = !DICompileUnit(language: DW_LANG_C99, producer: "clang version 3.4 (trunk 191846) (llvm/trunk 191866)", isOptimized: false, emissionKind: 0, file: !1, enums: !2, retainedTypes: !2, subprograms: !2, globals: !2, imports: !2)
+!1 = !DIFile(filename: "foo.c", directory: "/usr/local/google/home/echristo/tmp")
 !2 = !{}
 !3 = !{i32 2, !"Dwarf Version", i32 4}
 !4 = !{i32 1, !"Debug Info Version", i32 3}
diff --git a/test/DebugInfo/X86/gnu-public-names.ll b/test/DebugInfo/X86/gnu-public-names.ll
index e57b3eba2be..2022dfaa687 100644
--- a/test/DebugInfo/X86/gnu-public-names.ll
+++ b/test/DebugInfo/X86/gnu-public-names.ll
@@ -300,66 +300,66 @@ attributes #1 = { nounwind readnone }
 !llvm.module.flags = !{!48, !49}
 !llvm.ident = !{!50}
 
-!0 = !MDCompileUnit(language: DW_LANG_C_plus_plus, file: !1, producer: "clang version 3.7.0 (trunk 234897) (llvm/trunk 234911)", isOptimized: false, runtimeVersion: 0, emissionKind: 1, enums: !2, retainedTypes: !3, subprograms: !19, globals: !31, imports: !44)
-!1 = !MDFile(filename: "gnu-public-names.cpp", directory: "/tmp/dbginfo")
+!0 = !DICompileUnit(language: DW_LANG_C_plus_plus, file: !1, producer: "clang version 3.7.0 (trunk 234897) (llvm/trunk 234911)", isOptimized: false, runtimeVersion: 0, emissionKind: 1, enums: !2, retainedTypes: !3, subprograms: !19, globals: !31, imports: !44)
+!1 = !DIFile(filename: "gnu-public-names.cpp", directory: "/tmp/dbginfo")
 !2 = !{}
 !3 = !{!4, !15}
-!4 = !MDCompositeType(tag: DW_TAG_structure_type, name: "C", file: !1, line: 1, size: 8, align: 8, elements: !5, identifier: "_ZTS1C")
+!4 = !DICompositeType(tag: DW_TAG_structure_type, name: "C", file: !1, line: 1, size: 8, align: 8, elements: !5, identifier: "_ZTS1C")
 !5 = !{!6, !8, !12}
-!6 = !MDDerivedType(tag: DW_TAG_member, name: "static_member_variable", scope: !"_ZTS1C", file: !1, line: 4, baseType: !7, flags: DIFlagStaticMember)
-!7 = !MDBasicType(name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
-!8 = !MDSubprogram(name: "member_function", linkageName: "_ZN1C15member_functionEv", scope: !"_ZTS1C", file: !1, line: 2, type: !9, isLocal: false, isDefinition: false, scopeLine: 2, flags: DIFlagPrototyped, isOptimized: false)
-!9 = !MDSubroutineType(types: !10)
+!6 = !DIDerivedType(tag: DW_TAG_member, name: "static_member_variable", scope: !"_ZTS1C", file: !1, line: 4, baseType: !7, flags: DIFlagStaticMember)
+!7 = !DIBasicType(name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
+!8 = !DISubprogram(name: "member_function", linkageName: "_ZN1C15member_functionEv", scope: !"_ZTS1C", file: !1, line: 2, type: !9, isLocal: false, isDefinition: false, scopeLine: 2, flags: DIFlagPrototyped, isOptimized: false)
+!9 = !DISubroutineType(types: !10)
 !10 = !{null, !11}
-!11 = !MDDerivedType(tag: DW_TAG_pointer_type, baseType: !"_ZTS1C", size: 64, align: 64, flags: DIFlagArtificial | DIFlagObjectPointer)
-!12 = !MDSubprogram(name: "static_member_function", linkageName: "_ZN1C22static_member_functionEv", scope: !"_ZTS1C", file: !1, line: 3, type: !13, isLocal: false, isDefinition: false, scopeLine: 3, flags: DIFlagPrototyped, isOptimized: false)
-!13 = !MDSubroutineType(types: !14)
+!11 = !DIDerivedType(tag: DW_TAG_pointer_type, baseType: !"_ZTS1C", size: 64, align: 64, flags: DIFlagArtificial | DIFlagObjectPointer)
+!12 = !DISubprogram(name: "static_member_function", linkageName: "_ZN1C22static_member_functionEv", scope: !"_ZTS1C", file: !1, line: 3, type: !13, isLocal: false, isDefinition: false, scopeLine: 3, flags: DIFlagPrototyped, isOptimized: false)
+!13 = !DISubroutineType(types: !14)
 !14 = !{!7}
-!15 = !MDCompositeType(tag: DW_TAG_structure_type, name: "D", scope: !16, file: !1, line: 29, size: 32, align: 32, elements: !17, identifier: "_ZTSN2ns1DE")
-!16 = !MDNamespace(name: "ns", scope: null, file: !1, line: 23)
+!15 = !DICompositeType(tag: DW_TAG_structure_type, name: "D", scope: !16, file: !1, line: 29, size: 32, align: 32, elements: !17, identifier: "_ZTSN2ns1DE")
+!16 = !DINamespace(name: "ns", scope: null, file: !1, line: 23)
 !17 = !{!18}
-!18 = !MDDerivedType(tag: DW_TAG_member, name: "A", scope: !"_ZTSN2ns1DE", file: !1, line: 30, baseType: !7, size: 32, align: 32)
+!18 = !DIDerivedType(tag: DW_TAG_member, name: "A", scope: !"_ZTSN2ns1DE", file: !1, line: 30, baseType: !7, size: 32, align: 32)
 !19 = !{!20, !21, !22, !23, !26, !30}
-!20 = !MDSubprogram(name: "member_function", linkageName: "_ZN1C15member_functionEv", scope: !"_ZTS1C", file: !1, line: 9, type: !9, isLocal: false, isDefinition: true, scopeLine: 9, flags: DIFlagPrototyped, isOptimized: false, function: void (%struct.C*)* @_ZN1C15member_functionEv, declaration: !8, variables: !2)
-!21 = !MDSubprogram(name: "static_member_function", linkageName: "_ZN1C22static_member_functionEv", scope: !"_ZTS1C", file: !1, line: 13, type: !13, isLocal: false, isDefinition: true, scopeLine: 13, flags: DIFlagPrototyped, isOptimized: false, function: i32 ()* @_ZN1C22static_member_functionEv, declaration: !12, variables: !2)
-!22 = !MDSubprogram(name: "global_function", linkageName: "_Z15global_functionv", scope: !1, file: !1, line: 19, type: !13, isLocal: false, isDefinition: true, scopeLine: 19, flags: DIFlagPrototyped, isOptimized: false, function: i32 ()* @_Z15global_functionv, variables: !2)
-!23 = !MDSubprogram(name: "global_namespace_function", linkageName: "_ZN2ns25global_namespace_functionEv", scope: !16, file: !1, line: 24, type: !24, isLocal: false, isDefinition: true, scopeLine: 24, flags: DIFlagPrototyped, isOptimized: false, function: void ()* @_ZN2ns25global_namespace_functionEv, variables: !2)
-!24 = !MDSubroutineType(types: !25)
+!20 = !DISubprogram(name: "member_function", linkageName: "_ZN1C15member_functionEv", scope: !"_ZTS1C", file: !1, line: 9, type: !9, isLocal: false, isDefinition: true, scopeLine: 9, flags: DIFlagPrototyped, isOptimized: false, function: void (%struct.C*)* @_ZN1C15member_functionEv, declaration: !8, variables: !2)
+!21 = !DISubprogram(name: "static_member_function", linkageName: "_ZN1C22static_member_functionEv", scope: !"_ZTS1C", file: !1, line: 13, type: !13, isLocal: false, isDefinition: true, scopeLine: 13, flags: DIFlagPrototyped, isOptimized: false, function: i32 ()* @_ZN1C22static_member_functionEv, declaration: !12, variables: !2)
+!22 = !DISubprogram(name: "global_function", linkageName: "_Z15global_functionv", scope: !1, file: !1, line: 19, type: !13, isLocal: false, isDefinition: true, scopeLine: 19, flags: DIFlagPrototyped, isOptimized: false, function: i32 ()* @_Z15global_functionv, variables: !2)
+!23 = !DISubprogram(name: "global_namespace_function", linkageName: "_ZN2ns25global_namespace_functionEv", scope: !16, file: !1, line: 24, type: !24, isLocal: false, isDefinition: true, scopeLine: 24, flags: DIFlagPrototyped, isOptimized: false, function: void ()* @_ZN2ns25global_namespace_functionEv, variables: !2)
+!24 = !DISubroutineType(types: !25)
 !25 = !{null}
-!26 = !MDSubprogram(name: "f3", linkageName: "_Z2f3v", scope: !1, file: !1, line: 40, type: !27, isLocal: false, isDefinition: true, scopeLine: 40, flags: DIFlagPrototyped, isOptimized: false, function: i32* ()* @_Z2f3v, variables: !2)
-!27 = !MDSubroutineType(types: !28)
+!26 = !DISubprogram(name: "f3", linkageName: "_Z2f3v", scope: !1, file: !1, line: 40, type: !27, isLocal: false, isDefinition: true, scopeLine: 40, flags: DIFlagPrototyped, isOptimized: false, function: i32* ()* @_Z2f3v, variables: !2)
+!27 = !DISubroutineType(types: !28)
 !28 = !{!29}
-!29 = !MDDerivedType(tag: DW_TAG_pointer_type, baseType: !7, size: 64, align: 64)
-!30 = !MDSubprogram(name: "f7", linkageName: "_Z2f7v", scope: !1, file: !1, line: 57, type: !13, isLocal: false, isDefinition: true, scopeLine: 57, flags: DIFlagPrototyped, isOptimized: false, function: i32 ()* @_Z2f7v, variables: !2)
+!29 = !DIDerivedType(tag: DW_TAG_pointer_type, baseType: !7, size: 64, align: 64)
+!30 = !DISubprogram(name: "f7", linkageName: "_Z2f7v", scope: !1, file: !1, line: 57, type: !13, isLocal: false, isDefinition: true, scopeLine: 57, flags: DIFlagPrototyped, isOptimized: false, function: i32 ()* @_Z2f7v, variables: !2)
 !31 = !{!32, !33, !34, !35, !36, !37, !39, !41}
-!32 = !MDGlobalVariable(name: "static_member_variable", linkageName: "_ZN1C22static_member_variableE", scope: !0, file: !1, line: 7, type: !7, isLocal: false, isDefinition: true, variable: i32* @_ZN1C22static_member_variableE, declaration: !6)
-!33 = !MDGlobalVariable(name: "global_variable", scope: !0, file: !1, line: 17, type: !"_ZTS1C", isLocal: false, isDefinition: true, variable: %struct.C* @global_variable)
-!34 = !MDGlobalVariable(name: "global_namespace_variable", linkageName: "_ZN2ns25global_namespace_variableE", scope: !16, file: !1, line: 27, type: !7, isLocal: false, isDefinition: true, variable: i32* @_ZN2ns25global_namespace_variableE)
-!35 = !MDGlobalVariable(name: "d", linkageName: "_ZN2ns1dE", scope: !16, file: !1, line: 31, type: !"_ZTSN2ns1DE", isLocal: false, isDefinition: true, variable: %"struct.ns::D"* @_ZN2ns1dE)
-!36 = !MDGlobalVariable(name: "z", scope: !26, file: !1, line: 41, type: !7, isLocal: true, isDefinition: true, variable: i32* @_ZZ2f3vE1z)
-!37 = !MDGlobalVariable(name: "i", linkageName: "_ZN12_GLOBAL__N_11iE", scope: !38, file: !1, line: 37, type: !7, isLocal: true, isDefinition: true, variable: i32* @_ZN12_GLOBAL__N_11iE)
-!38 = !MDNamespace(scope: null, file: !1, line: 36)
-!39 = !MDGlobalVariable(name: "b", linkageName: "_ZN12_GLOBAL__N_15inner1bE", scope: !40, file: !1, line: 47, type: !7, isLocal: true, isDefinition: true, variable: i32* @_ZN12_GLOBAL__N_15inner1bE)
-!40 = !MDNamespace(name: "inner", scope: !38, file: !1, line: 46)
-!41 = !MDGlobalVariable(name: "c", linkageName: "_ZN5outer12_GLOBAL__N_11cE", scope: !42, file: !1, line: 53, type: !7, isLocal: true, isDefinition: true, variable: i32* @_ZN5outer12_GLOBAL__N_11cE)
-!42 = !MDNamespace(scope: !43, file: !1, line: 52)
-!43 = !MDNamespace(name: "outer", scope: null, file: !1, line: 51)
+!32 = !DIGlobalVariable(name: "static_member_variable", linkageName: "_ZN1C22static_member_variableE", scope: !0, file: !1, line: 7, type: !7, isLocal: false, isDefinition: true, variable: i32* @_ZN1C22static_member_variableE, declaration: !6)
+!33 = !DIGlobalVariable(name: "global_variable", scope: !0, file: !1, line: 17, type: !"_ZTS1C", isLocal: false, isDefinition: true, variable: %struct.C* @global_variable)
+!34 = !DIGlobalVariable(name: "global_namespace_variable", linkageName: "_ZN2ns25global_namespace_variableE", scope: !16, file: !1, line: 27, type: !7, isLocal: false, isDefinition: true, variable: i32* @_ZN2ns25global_namespace_variableE)
+!35 = !DIGlobalVariable(name: "d", linkageName: "_ZN2ns1dE", scope: !16, file: !1, line: 31, type: !"_ZTSN2ns1DE", isLocal: false, isDefinition: true, variable: %"struct.ns::D"* @_ZN2ns1dE)
+!36 = !DIGlobalVariable(name: "z", scope: !26, file: !1, line: 41, type: !7, isLocal: true, isDefinition: true, variable: i32* @_ZZ2f3vE1z)
+!37 = !DIGlobalVariable(name: "i", linkageName: "_ZN12_GLOBAL__N_11iE", scope: !38, file: !1, line: 37, type: !7, isLocal: true, isDefinition: true, variable: i32* @_ZN12_GLOBAL__N_11iE)
+!38 = !DINamespace(scope: null, file: !1, line: 36)
+!39 = !DIGlobalVariable(name: "b", linkageName: "_ZN12_GLOBAL__N_15inner1bE", scope: !40, file: !1, line: 47, type: !7, isLocal: true, isDefinition: true, variable: i32* @_ZN12_GLOBAL__N_15inner1bE)
+!40 = !DINamespace(name: "inner", scope: !38, file: !1, line: 46)
+!41 = !DIGlobalVariable(name: "c", linkageName: "_ZN5outer12_GLOBAL__N_11cE", scope: !42, file: !1, line: 53, type: !7, isLocal: true, isDefinition: true, variable: i32* @_ZN5outer12_GLOBAL__N_11cE)
+!42 = !DINamespace(scope: !43, file: !1, line: 52)
+!43 = !DINamespace(name: "outer", scope: null, file: !1, line: 51)
 !44 = !{!45, !47}
-!45 = !MDImportedEntity(tag: DW_TAG_imported_declaration, scope: !0, entity: !46, line: 34)
-!46 = !MDGlobalVariable(name: "global_namespace_variable_decl", linkageName: "_ZN2ns30global_namespace_variable_declE", scope: !16, file: !1, line: 28, type: !7, isLocal: false, isDefinition: false)
-!47 = !MDImportedEntity(tag: DW_TAG_imported_module, scope: !43, entity: !42, line: 43)
+!45 = !DIImportedEntity(tag: DW_TAG_imported_declaration, scope: !0, entity: !46, line: 34)
+!46 = !DIGlobalVariable(name: "global_namespace_variable_decl", linkageName: "_ZN2ns30global_namespace_variable_declE", scope: !16, file: !1, line: 28, type: !7, isLocal: false, isDefinition: false)
+!47 = !DIImportedEntity(tag: DW_TAG_imported_module, scope: !43, entity: !42, line: 43)
 !48 = !{i32 2, !"Dwarf Version", i32 4}
 !49 = !{i32 2, !"Debug Info Version", i32 3}
 !50 = !{!"clang version 3.7.0 (trunk 234897) (llvm/trunk 234911)"}
-!51 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "this", arg: 1, scope: !20, type: !52, flags: DIFlagArtificial | DIFlagObjectPointer)
-!52 = !MDDerivedType(tag: DW_TAG_pointer_type, baseType: !"_ZTS1C", size: 64, align: 64)
-!53 = !MDExpression()
-!54 = !MDLocation(line: 0, scope: !20)
-!55 = !MDLocation(line: 10, scope: !20)
-!56 = !MDLocation(line: 11, scope: !20)
-!57 = !MDLocation(line: 14, scope: !21)
-!58 = !MDLocation(line: 20, scope: !22)
-!59 = !MDLocation(line: 25, scope: !23)
-!60 = !MDLocation(line: 26, scope: !23)
-!61 = !MDLocation(line: 42, scope: !26)
-!62 = !MDLocation(line: 58, scope: !30)
+!51 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "this", arg: 1, scope: !20, type: !52, flags: DIFlagArtificial | DIFlagObjectPointer)
+!52 = !DIDerivedType(tag: DW_TAG_pointer_type, baseType: !"_ZTS1C", size: 64, align: 64)
+!53 = !DIExpression()
+!54 = !DILocation(line: 0, scope: !20)
+!55 = !DILocation(line: 10, scope: !20)
+!56 = !DILocation(line: 11, scope: !20)
+!57 = !DILocation(line: 14, scope: !21)
+!58 = !DILocation(line: 20, scope: !22)
+!59 = !DILocation(line: 25, scope: !23)
+!60 = !DILocation(line: 26, scope: !23)
+!61 = !DILocation(line: 42, scope: !26)
+!62 = !DILocation(line: 58, scope: !30)
diff --git a/test/DebugInfo/X86/header.ll b/test/DebugInfo/X86/header.ll
index 263f9ec923c..8575c4a672a 100644
--- a/test/DebugInfo/X86/header.ll
+++ b/test/DebugInfo/X86/header.ll
@@ -17,13 +17,13 @@ define void @f() {
 !llvm.dbg.cu = !{!0}
 !llvm.module.flags = !{!7, !8}
 
-!0 = !MDCompileUnit(language: DW_LANG_C99, file: !1, producer: "foo", isOptimized: true, runtimeVersion: 0, emissionKind: 1, enums: !2, retainedTypes: !2, subprograms: !3, globals: !2, imports: !2)
-!1 = !MDFile(filename: "/foo/test.c", directory: "/foo")
+!0 = !DICompileUnit(language: DW_LANG_C99, file: !1, producer: "foo", isOptimized: true, runtimeVersion: 0, emissionKind: 1, enums: !2, retainedTypes: !2, subprograms: !3, globals: !2, imports: !2)
+!1 = !DIFile(filename: "/foo/test.c", directory: "/foo")
 !2 = !{}
 !3 = !{!4}
-!4 = !MDSubprogram(name: "f", scope: !1, file: !1, line: 1, type: !5, isLocal: false, isDefinition: true, scopeLine: 1, flags: DIFlagPrototyped, isOptimized: true, function: void ()* @f, variables: !2)
-!5 = !MDSubroutineType(types: !6)
+!4 = !DISubprogram(name: "f", scope: !1, file: !1, line: 1, type: !5, isLocal: false, isDefinition: true, scopeLine: 1, flags: DIFlagPrototyped, isOptimized: true, function: void ()* @f, variables: !2)
+!5 = !DISubroutineType(types: !6)
 !6 = !{null}
 !7 = !{i32 2, !"Dwarf Version", i32 4}
 !8 = !{i32 2, !"Debug Info Version", i32 3}
-!9 = !MDLocation(line: 1, column: 15, scope: !4)
+!9 = !DILocation(line: 1, column: 15, scope: !4)
diff --git a/test/DebugInfo/X86/inline-member-function.ll b/test/DebugInfo/X86/inline-member-function.ll
index c1a367e7e0f..73cd1947f4f 100644
--- a/test/DebugInfo/X86/inline-member-function.ll
+++ b/test/DebugInfo/X86/inline-member-function.ll
@@ -46,9 +46,9 @@ entry:
   store i32 0, i32* %retval
   %0 = load i32, i32* @i, align 4, !dbg !23
   store %struct.foo* %tmp, %struct.foo** %this.addr.i, align 8
-  call void @llvm.dbg.declare(metadata %struct.foo** %this.addr.i, metadata !24, metadata !MDExpression()), !dbg !26
+  call void @llvm.dbg.declare(metadata %struct.foo** %this.addr.i, metadata !24, metadata !DIExpression()), !dbg !26
   store i32 %0, i32* %x.addr.i, align 4
-  call void @llvm.dbg.declare(metadata i32* %x.addr.i, metadata !27, metadata !MDExpression()), !dbg !28
+  call void @llvm.dbg.declare(metadata i32* %x.addr.i, metadata !27, metadata !DIExpression()), !dbg !28
   %this1.i = load %struct.foo*, %struct.foo** %this.addr.i
   %1 = load i32, i32* %x.addr.i, align 4, !dbg !28
   %add.i = add nsw i32 %1, 2, !dbg !28
@@ -65,31 +65,31 @@ attributes #1 = { nounwind readnone }
 !llvm.module.flags = !{!20, !21}
 !llvm.ident = !{!22}
 
-!0 = !MDCompileUnit(language: DW_LANG_C_plus_plus, producer: "clang version 3.5.0 ", isOptimized: false, emissionKind: 1, file: !1, enums: !2, retainedTypes: !3, subprograms: !12, globals: !18, imports: !2)
-!1 = !MDFile(filename: "inline.cpp", directory: "/tmp/dbginfo")
+!0 = !DICompileUnit(language: DW_LANG_C_plus_plus, producer: "clang version 3.5.0 ", isOptimized: false, emissionKind: 1, file: !1, enums: !2, retainedTypes: !3, subprograms: !12, globals: !18, imports: !2)
+!1 = !DIFile(filename: "inline.cpp", directory: "/tmp/dbginfo")
 !2 = !{}
 !3 = !{!4}
-!4 = !MDCompositeType(tag: DW_TAG_structure_type, name: "foo", line: 1, size: 8, align: 8, file: !1, elements: !5, identifier: "_ZTS3foo")
+!4 = !DICompositeType(tag: DW_TAG_structure_type, name: "foo", line: 1, size: 8, align: 8, file: !1, elements: !5, identifier: "_ZTS3foo")
 !5 = !{!6}
-!6 = !MDSubprogram(name: "func", linkageName: "_ZN3foo4funcEi", line: 2, isLocal: false, isDefinition: false, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 2, file: !1, scope: !"_ZTS3foo", type: !7)
-!7 = !MDSubroutineType(types: !8)
+!6 = !DISubprogram(name: "func", linkageName: "_ZN3foo4funcEi", line: 2, isLocal: false, isDefinition: false, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 2, file: !1, scope: !"_ZTS3foo", type: !7)
+!7 = !DISubroutineType(types: !8)
 !8 = !{!9, !10, !9}
-!9 = !MDBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
-!10 = !MDDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, flags: DIFlagArtificial | DIFlagObjectPointer, baseType: !"_ZTS3foo")
+!9 = !DIBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
+!10 = !DIDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, flags: DIFlagArtificial | DIFlagObjectPointer, baseType: !"_ZTS3foo")
 !12 = !{!13, !17}
-!13 = !MDSubprogram(name: "main", line: 7, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 7, file: !1, scope: !14, type: !15, function: i32 ()* @main, variables: !2)
-!14 = !MDFile(filename: "inline.cpp", directory: "/tmp/dbginfo")
-!15 = !MDSubroutineType(types: !16)
+!13 = !DISubprogram(name: "main", line: 7, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 7, file: !1, scope: !14, type: !15, function: i32 ()* @main, variables: !2)
+!14 = !DIFile(filename: "inline.cpp", directory: "/tmp/dbginfo")
+!15 = !DISubroutineType(types: !16)
 !16 = !{!9}
-!17 = !MDSubprogram(name: "func", linkageName: "_ZN3foo4funcEi", line: 2, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 2, file: !1, scope: !"_ZTS3foo", type: !7, declaration: !6, variables: !2)
+!17 = !DISubprogram(name: "func", linkageName: "_ZN3foo4funcEi", line: 2, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 2, file: !1, scope: !"_ZTS3foo", type: !7, declaration: !6, variables: !2)
 !18 = !{!19}
-!19 = !MDGlobalVariable(name: "i", line: 5, isLocal: false, isDefinition: true, scope: null, file: !14, type: !9, variable: i32* @i)
+!19 = !DIGlobalVariable(name: "i", line: 5, isLocal: false, isDefinition: true, scope: null, file: !14, type: !9, variable: i32* @i)
 !20 = !{i32 2, !"Dwarf Version", i32 4}
 !21 = !{i32 1, !"Debug Info Version", i32 3}
 !22 = !{!"clang version 3.5.0 "}
-!23 = !MDLocation(line: 8, scope: !13)
-!24 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "this", arg: 1, flags: DIFlagArtificial | DIFlagObjectPointer, scope: !17, type: !25)
-!25 = !MDDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, baseType: !"_ZTS3foo")
-!26 = !MDLocation(line: 0, scope: !17, inlinedAt: !23)
-!27 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "x", line: 2, arg: 2, scope: !17, file: !14, type: !9)
-!28 = !MDLocation(line: 2, scope: !17, inlinedAt: !23)
+!23 = !DILocation(line: 8, scope: !13)
+!24 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "this", arg: 1, flags: DIFlagArtificial | DIFlagObjectPointer, scope: !17, type: !25)
+!25 = !DIDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, baseType: !"_ZTS3foo")
+!26 = !DILocation(line: 0, scope: !17, inlinedAt: !23)
+!27 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "x", line: 2, arg: 2, scope: !17, file: !14, type: !9)
+!28 = !DILocation(line: 2, scope: !17, inlinedAt: !23)
diff --git a/test/DebugInfo/X86/inline-seldag-test.ll b/test/DebugInfo/X86/inline-seldag-test.ll
index 8f7b107c5fb..c8e75cf3fcb 100644
--- a/test/DebugInfo/X86/inline-seldag-test.ll
+++ b/test/DebugInfo/X86/inline-seldag-test.ll
@@ -27,10 +27,10 @@ define void @func() #0 {
 entry:
   %y.addr.i = alloca i32, align 4
   %x = alloca i32, align 4
-  call void @llvm.dbg.declare(metadata i32* %x, metadata !15, metadata !MDExpression()), !dbg !17
+  call void @llvm.dbg.declare(metadata i32* %x, metadata !15, metadata !DIExpression()), !dbg !17
   %0 = load volatile i32, i32* %x, align 4, !dbg !18
   store i32 %0, i32* %y.addr.i, align 4
-  call void @llvm.dbg.declare(metadata i32* %y.addr.i, metadata !19, metadata !MDExpression()), !dbg !20
+  call void @llvm.dbg.declare(metadata i32* %y.addr.i, metadata !19, metadata !DIExpression()), !dbg !20
   %1 = load i32, i32* %y.addr.i, align 4, !dbg !21
   %tobool.i = icmp ne i32 %1, 0, !dbg !21
   %cond.i = select i1 %tobool.i, i32 4, i32 7, !dbg !21
@@ -48,26 +48,26 @@ attributes #1 = { nounwind readnone }
 !llvm.module.flags = !{!12, !13}
 !llvm.ident = !{!14}
 
-!0 = !MDCompileUnit(language: DW_LANG_C99, producer: "clang version 3.5.0 ", isOptimized: false, emissionKind: 1, file: !1, enums: !2, retainedTypes: !2, subprograms: !3, globals: !2, imports: !2)
-!1 = !MDFile(filename: "inline-seldag-test.c", directory: "/tmp/dbginfo")
+!0 = !DICompileUnit(language: DW_LANG_C99, producer: "clang version 3.5.0 ", isOptimized: false, emissionKind: 1, file: !1, enums: !2, retainedTypes: !2, subprograms: !3, globals: !2, imports: !2)
+!1 = !DIFile(filename: "inline-seldag-test.c", directory: "/tmp/dbginfo")
 !2 = !{}
 !3 = !{!4, !8}
-!4 = !MDSubprogram(name: "func", line: 4, isLocal: false, isDefinition: true, virtualIndex: 6, isOptimized: false, scopeLine: 4, file: !1, scope: !5, type: !6, function: void ()* @func, variables: !2)
-!5 = !MDFile(filename: "inline-seldag-test.c", directory: "/tmp/dbginfo")
-!6 = !MDSubroutineType(types: !7)
+!4 = !DISubprogram(name: "func", line: 4, isLocal: false, isDefinition: true, virtualIndex: 6, isOptimized: false, scopeLine: 4, file: !1, scope: !5, type: !6, function: void ()* @func, variables: !2)
+!5 = !DIFile(filename: "inline-seldag-test.c", directory: "/tmp/dbginfo")
+!6 = !DISubroutineType(types: !7)
 !7 = !{null}
-!8 = !MDSubprogram(name: "f", line: 1, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 1, file: !1, scope: !5, type: !9, variables: !2)
-!9 = !MDSubroutineType(types: !10)
+!8 = !DISubprogram(name: "f", line: 1, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 1, file: !1, scope: !5, type: !9, variables: !2)
+!9 = !DISubroutineType(types: !10)
 !10 = !{!11, !11}
-!11 = !MDBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
+!11 = !DIBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
 !12 = !{i32 2, !"Dwarf Version", i32 4}
 !13 = !{i32 1, !"Debug Info Version", i32 3}
 !14 = !{!"clang version 3.5.0 "}
-!15 = !MDLocalVariable(tag: DW_TAG_auto_variable, name: "x", line: 5, scope: !4, file: !5, type: !16)
-!16 = !MDDerivedType(tag: DW_TAG_volatile_type, baseType: !11)
-!17 = !MDLocation(line: 5, scope: !4)
-!18 = !MDLocation(line: 6, column: 7, scope: !4)
-!19 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "y", line: 1, arg: 1, scope: !8, file: !5, type: !11)
-!20 = !MDLocation(line: 1, scope: !8, inlinedAt: !18)
-!21 = !MDLocation(line: 2, scope: !8, inlinedAt: !18)
-!22 = !MDLocation(line: 7, scope: !4)
+!15 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "x", line: 5, scope: !4, file: !5, type: !16)
+!16 = !DIDerivedType(tag: DW_TAG_volatile_type, baseType: !11)
+!17 = !DILocation(line: 5, scope: !4)
+!18 = !DILocation(line: 6, column: 7, scope: !4)
+!19 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "y", line: 1, arg: 1, scope: !8, file: !5, type: !11)
+!20 = !DILocation(line: 1, scope: !8, inlinedAt: !18)
+!21 = !DILocation(line: 2, scope: !8, inlinedAt: !18)
+!22 = !DILocation(line: 7, scope: !4)
diff --git a/test/DebugInfo/X86/inlined-formal-parameter.ll b/test/DebugInfo/X86/inlined-formal-parameter.ll
index a2ae0df97cb..3cc4e5a65a6 100644
--- a/test/DebugInfo/X86/inlined-formal-parameter.ll
+++ b/test/DebugInfo/X86/inlined-formal-parameter.ll
@@ -48,28 +48,28 @@ attributes #3 = { nounwind }
 !llvm.module.flags = !{!13, !14, !15}
 !llvm.ident = !{!16}
 
-!0 = !MDCompileUnit(language: DW_LANG_C99, file: !1, producer: "clang version 3.7.0 (trunk 235110) (llvm/trunk 235108)", isOptimized: true, runtimeVersion: 0, emissionKind: 1, enums: !2, retainedTypes: !2, subprograms: !3, globals: !2, imports: !2)
-!1 = !MDFile(filename: "t.c", directory: "/path/to/dir")
+!0 = !DICompileUnit(language: DW_LANG_C99, file: !1, producer: "clang version 3.7.0 (trunk 235110) (llvm/trunk 235108)", isOptimized: true, runtimeVersion: 0, emissionKind: 1, enums: !2, retainedTypes: !2, subprograms: !3, globals: !2, imports: !2)
+!1 = !DIFile(filename: "t.c", directory: "/path/to/dir")
 !2 = !{}
 !3 = !{!4, !7}
-!4 = !MDSubprogram(name: "foo", scope: !1, file: !1, line: 3, type: !5, isLocal: false, isDefinition: true, scopeLine: 3, flags: DIFlagPrototyped, isOptimized: true, function: void ()* @foo, variables: !2)
-!5 = !MDSubroutineType(types: !6)
+!4 = !DISubprogram(name: "foo", scope: !1, file: !1, line: 3, type: !5, isLocal: false, isDefinition: true, scopeLine: 3, flags: DIFlagPrototyped, isOptimized: true, function: void ()* @foo, variables: !2)
+!5 = !DISubroutineType(types: !6)
 !6 = !{null}
-!7 = !MDSubprogram(name: "bar", scope: !1, file: !1, line: 2, type: !8, isLocal: true, isDefinition: true, scopeLine: 2, flags: DIFlagPrototyped, isOptimized: true, variables: !11)
-!8 = !MDSubroutineType(types: !9)
+!7 = !DISubprogram(name: "bar", scope: !1, file: !1, line: 2, type: !8, isLocal: true, isDefinition: true, scopeLine: 2, flags: DIFlagPrototyped, isOptimized: true, variables: !11)
+!8 = !DISubroutineType(types: !9)
 !9 = !{null, !10}
-!10 = !MDBasicType(name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
+!10 = !DIBasicType(name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
 !11 = !{!12}
-!12 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "a", arg: 1, scope: !7, file: !1, line: 2, type: !10)
+!12 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "a", arg: 1, scope: !7, file: !1, line: 2, type: !10)
 !13 = !{i32 2, !"Dwarf Version", i32 2}
 !14 = !{i32 2, !"Debug Info Version", i32 3}
 !15 = !{i32 1, !"PIC Level", i32 2}
 !16 = !{!"clang version 3.7.0 (trunk 235110) (llvm/trunk 235108)"}
-!17 = !MDExpression()
-!18 = !MDLocation(line: 2, column: 52, scope: !7, inlinedAt: !19)
-!19 = distinct !MDLocation(line: 4, column: 3, scope: !4)
-!20 = !MDLocation(line: 2, column: 57, scope: !7, inlinedAt: !19)
-!21 = !MDLocation(line: 2, column: 52, scope: !7, inlinedAt: !22)
-!22 = distinct !MDLocation(line: 5, column: 3, scope: !4)
-!23 = !MDLocation(line: 2, column: 57, scope: !7, inlinedAt: !22)
-!24 = !MDLocation(line: 6, column: 1, scope: !4)
+!17 = !DIExpression()
+!18 = !DILocation(line: 2, column: 52, scope: !7, inlinedAt: !19)
+!19 = distinct !DILocation(line: 4, column: 3, scope: !4)
+!20 = !DILocation(line: 2, column: 57, scope: !7, inlinedAt: !19)
+!21 = !DILocation(line: 2, column: 52, scope: !7, inlinedAt: !22)
+!22 = distinct !DILocation(line: 5, column: 3, scope: !4)
+!23 = !DILocation(line: 2, column: 57, scope: !7, inlinedAt: !22)
+!24 = !DILocation(line: 6, column: 1, scope: !4)
diff --git a/test/DebugInfo/X86/inlined-indirect-value.ll b/test/DebugInfo/X86/inlined-indirect-value.ll
new file mode 100644
index 00000000000..7f95691fbb8
--- /dev/null
+++ b/test/DebugInfo/X86/inlined-indirect-value.ll
@@ -0,0 +1,81 @@
+; RUN: llc -filetype=asm -asm-verbose=0 < %s | FileCheck %s
+
+; "1" from line 09 in the snippet below shouldn't be marked with location of "1"
+; from line 04.  Instead it will have location inside main() (real location is
+; just erased, so it won't be perfectly accurate).
+
+; options: -g -O3
+; 01 volatile int x;
+; 02 int y;
+; 03 static __attribute__((always_inline)) int f1() {
+; 04     if (x * 3 < 14) return 1;
+; 05     return 2;
+; 06 }
+; 07 int main() {
+; 08     x = f1();
+; 09     x = x ? 1 : 2;
+; 10 }
+
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+@x = common global i32 0, align 4
+@y = common global i32 0, align 4
+
+define i32 @main() {
+; CHECK: .loc 1 {{[89]}}
+; CHECK-NOT: .loc
+; CHECK: movl $1
+
+entry:
+  %0 = load volatile i32, i32* @x, align 4, !dbg !16, !tbaa !19
+  %mul.i = mul nsw i32 %0, 3, !dbg !23
+  %cmp.i = icmp slt i32 %mul.i, 14, !dbg !24
+  %..i = select i1 %cmp.i, i32 1, i32 2, !dbg !25
+  store volatile i32 %..i, i32* @x, align 4, !dbg !27, !tbaa !19
+  %1 = load volatile i32, i32* @x, align 4, !dbg !28, !tbaa !19
+  %tobool = icmp ne i32 %1, 0, !dbg !28
+  br i1 %tobool, label %select.end, label %select.mid
+
+select.mid:                                       ; preds = %entry
+  br label %select.end
+
+select.end:                                       ; preds = %entry, %select.mid
+  %cond = phi i32 [ 1, %entry ], [ 2, %select.mid ]
+  store volatile i32 %cond, i32* @x, align 4, !dbg !29, !tbaa !19
+  ret i32 0, !dbg !30
+}
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!13, !14}
+
+!0 = !DICompileUnit(language: DW_LANG_C99, file: !1, producer: "", isOptimized: true, runtimeVersion: 0, emissionKind: 1, enums: !2, retainedTypes: !2, subprograms: !3, globals: !9, imports: !2)
+!1 = !DIFile(filename: "inline-break.c", directory: "/build/dir")
+!2 = !{}
+!3 = !{!4, !8}
+!4 = !DISubprogram(name: "main", scope: !1, file: !1, line: 7, type: !5, isLocal: false, isDefinition: true, scopeLine: 7, isOptimized: true, function: i32 ()* @main, variables: !2)
+!5 = !DISubroutineType(types: !6)
+!6 = !{!7}
+!7 = !DIBasicType(name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
+!8 = !DISubprogram(name: "f1", scope: !1, file: !1, line: 3, type: !5, isLocal: true, isDefinition: true, scopeLine: 3, isOptimized: true, variables: !2)
+!9 = !{!10, !12}
+!10 = !DIGlobalVariable(name: "x", scope: !0, file: !1, line: 1, type: !11, isLocal: false, isDefinition: true, variable: i32* @x)
+!11 = !DIDerivedType(tag: DW_TAG_volatile_type, baseType: !7)
+!12 = !DIGlobalVariable(name: "y", scope: !0, file: !1, line: 2, type: !7, isLocal: false, isDefinition: true, variable: i32* @y)
+!13 = !{i32 2, !"Dwarf Version", i32 4}
+!14 = !{i32 2, !"Debug Info Version", i32 3}
+!16 = !DILocation(line: 4, column: 9, scope: !17, inlinedAt: !18)
+!17 = distinct !DILexicalBlock(scope: !8, file: !1, line: 4, column: 9)
+!18 = distinct !DILocation(line: 8, column: 9, scope: !4)
+!19 = !{!20, !20, i64 0}
+!20 = !{!"int", !21, i64 0}
+!21 = !{!"omnipotent char", !22, i64 0}
+!22 = !{!"Simple C/C++ TBAA"}
+!23 = !DILocation(line: 4, column: 11, scope: !17, inlinedAt: !18)
+!24 = !DILocation(line: 4, column: 15, scope: !17, inlinedAt: !18)
+!25 = !DILocation(line: 4, column: 21, scope: !26, inlinedAt: !18)
+!26 = !DILexicalBlockFile(scope: !17, file: !1, discriminator: 1)
+!27 = !DILocation(line: 8, column: 7, scope: !4)
+!28 = !DILocation(line: 9, column: 9, scope: !4)
+!29 = !DILocation(line: 9, column: 7, scope: !4)
+!30 = !DILocation(line: 10, column: 1, scope: !4)
diff --git a/test/DebugInfo/X86/instcombine-instrinsics.ll b/test/DebugInfo/X86/instcombine-instrinsics.ll
index dbf498ec82d..8241e6498f9 100644
--- a/test/DebugInfo/X86/instcombine-instrinsics.ll
+++ b/test/DebugInfo/X86/instcombine-instrinsics.ll
@@ -30,7 +30,7 @@ target triple = "x86_64-apple-macosx10.9.0"
 ; Function Attrs: nounwind ssp uwtable
 define void @init() #0 {
   %p = alloca %struct.i14*, align 8
-  call void @llvm.dbg.declare(metadata %struct.i14** %p, metadata !11, metadata !MDExpression()), !dbg !18
+  call void @llvm.dbg.declare(metadata %struct.i14** %p, metadata !11, metadata !DIExpression()), !dbg !18
   store %struct.i14* null, %struct.i14** %p, align 8, !dbg !18
   %1 = call i32 @foo(%struct.i14** %p), !dbg !19
   %2 = load %struct.i14*, %struct.i14** %p, align 8, !dbg !20
@@ -54,26 +54,26 @@ attributes #1 = { nounwind readnone }
 !llvm.module.flags = !{!8, !9}
 !llvm.ident = !{!10}
 
-!0 = !MDCompileUnit(language: DW_LANG_C99, producer: "clang version 3.5.0 ", isOptimized: false, emissionKind: 1, file: !1, enums: !2, retainedTypes: !2, subprograms: !3, globals: !2, imports: !2)
-!1 = !MDFile(filename: "instcombine_intrinsics.c", directory: "")
+!0 = !DICompileUnit(language: DW_LANG_C99, producer: "clang version 3.5.0 ", isOptimized: false, emissionKind: 1, file: !1, enums: !2, retainedTypes: !2, subprograms: !3, globals: !2, imports: !2)
+!1 = !DIFile(filename: "instcombine_intrinsics.c", directory: "")
 !2 = !{}
 !3 = !{!4}
-!4 = !MDSubprogram(name: "init", line: 7, isLocal: false, isDefinition: true, virtualIndex: 6, isOptimized: false, scopeLine: 7, file: !1, scope: !5, type: !6, function: void ()* @init, variables: !2)
-!5 = !MDFile(filename: "instcombine_intrinsics.c", directory: "")
-!6 = !MDSubroutineType(types: !7)
+!4 = !DISubprogram(name: "init", line: 7, isLocal: false, isDefinition: true, virtualIndex: 6, isOptimized: false, scopeLine: 7, file: !1, scope: !5, type: !6, function: void ()* @init, variables: !2)
+!5 = !DIFile(filename: "instcombine_intrinsics.c", directory: "")
+!6 = !DISubroutineType(types: !7)
 !7 = !{null}
 !8 = !{i32 2, !"Dwarf Version", i32 2}
 !9 = !{i32 1, !"Debug Info Version", i32 3}
 !10 = !{!"clang version 3.5.0 "}
-!11 = !MDLocalVariable(tag: DW_TAG_auto_variable, name: "p", line: 8, scope: !4, file: !5, type: !12)
-!12 = !MDDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, baseType: !13)
-!13 = !MDDerivedType(tag: DW_TAG_typedef, name: "i14", line: 3, file: !1, baseType: !14)
-!14 = !MDCompositeType(tag: DW_TAG_structure_type, line: 1, size: 64, align: 64, file: !1, elements: !15)
+!11 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "p", line: 8, scope: !4, file: !5, type: !12)
+!12 = !DIDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, baseType: !13)
+!13 = !DIDerivedType(tag: DW_TAG_typedef, name: "i14", line: 3, file: !1, baseType: !14)
+!14 = !DICompositeType(tag: DW_TAG_structure_type, line: 1, size: 64, align: 64, file: !1, elements: !15)
 !15 = !{!16}
-!16 = !MDDerivedType(tag: DW_TAG_member, name: "i", line: 2, size: 64, align: 64, file: !1, scope: !14, baseType: !17)
-!17 = !MDBasicType(tag: DW_TAG_base_type, name: "long int", size: 64, align: 64, encoding: DW_ATE_signed)
-!18 = !MDLocation(line: 8, scope: !4)
-!19 = !MDLocation(line: 9, scope: !4)
-!20 = !MDLocation(line: 10, scope: !4)
-!21 = !MDLocation(line: 11, scope: !4)
-!22 = !MDLocation(line: 12, scope: !4)
+!16 = !DIDerivedType(tag: DW_TAG_member, name: "i", line: 2, size: 64, align: 64, file: !1, scope: !14, baseType: !17)
+!17 = !DIBasicType(tag: DW_TAG_base_type, name: "long int", size: 64, align: 64, encoding: DW_ATE_signed)
+!18 = !DILocation(line: 8, scope: !4)
+!19 = !DILocation(line: 9, scope: !4)
+!20 = !DILocation(line: 10, scope: !4)
+!21 = !DILocation(line: 11, scope: !4)
+!22 = !DILocation(line: 12, scope: !4)
diff --git a/test/DebugInfo/X86/lexical_block.ll b/test/DebugInfo/X86/lexical_block.ll
index 6a661cdadd5..2619f12f69e 100644
--- a/test/DebugInfo/X86/lexical_block.ll
+++ b/test/DebugInfo/X86/lexical_block.ll
@@ -25,7 +25,7 @@
 define void @_Z1bv() #0 {
 entry:
   %i = alloca i32, align 4
-  call void @llvm.dbg.declare(metadata i32* %i, metadata !11, metadata !MDExpression()), !dbg !14
+  call void @llvm.dbg.declare(metadata i32* %i, metadata !11, metadata !DIExpression()), !dbg !14
   store i32 3, i32* %i, align 4, !dbg !14
   %0 = load i32, i32* %i, align 4, !dbg !14
   %tobool = icmp ne i32 %0, 0, !dbg !14
@@ -48,20 +48,20 @@ attributes #1 = { nounwind readnone }
 !llvm.module.flags = !{!8, !9}
 !llvm.ident = !{!10}
 
-!0 = !MDCompileUnit(language: DW_LANG_C_plus_plus, producer: "clang version 3.5.0 ", isOptimized: false, emissionKind: 1, file: !1, enums: !2, retainedTypes: !2, subprograms: !3, globals: !2, imports: !2)
-!1 = !MDFile(filename: "lexical_block.cpp", directory: "/tmp/dbginfo")
+!0 = !DICompileUnit(language: DW_LANG_C_plus_plus, producer: "clang version 3.5.0 ", isOptimized: false, emissionKind: 1, file: !1, enums: !2, retainedTypes: !2, subprograms: !3, globals: !2, imports: !2)
+!1 = !DIFile(filename: "lexical_block.cpp", directory: "/tmp/dbginfo")
 !2 = !{}
 !3 = !{!4}
-!4 = !MDSubprogram(name: "b", linkageName: "_Z1bv", line: 1, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 1, file: !1, scope: !5, type: !6, function: void ()* @_Z1bv, variables: !2)
-!5 = !MDFile(filename: "lexical_block.cpp", directory: "/tmp/dbginfo")
-!6 = !MDSubroutineType(types: !7)
+!4 = !DISubprogram(name: "b", linkageName: "_Z1bv", line: 1, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 1, file: !1, scope: !5, type: !6, function: void ()* @_Z1bv, variables: !2)
+!5 = !DIFile(filename: "lexical_block.cpp", directory: "/tmp/dbginfo")
+!6 = !DISubroutineType(types: !7)
 !7 = !{null}
 !8 = !{i32 2, !"Dwarf Version", i32 4}
 !9 = !{i32 1, !"Debug Info Version", i32 3}
 !10 = !{!"clang version 3.5.0 "}
-!11 = !MDLocalVariable(tag: DW_TAG_auto_variable, name: "i", line: 2, scope: !12, file: !5, type: !13)
-!12 = distinct !MDLexicalBlock(line: 2, column: 0, file: !1, scope: !4)
-!13 = !MDBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
-!14 = !MDLocation(line: 2, scope: !12)
-!15 = !MDLocation(line: 3, scope: !12)
-!16 = !MDLocation(line: 4, scope: !4)
+!11 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "i", line: 2, scope: !12, file: !5, type: !13)
+!12 = distinct !DILexicalBlock(line: 2, column: 0, file: !1, scope: !4)
+!13 = !DIBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
+!14 = !DILocation(line: 2, scope: !12)
+!15 = !DILocation(line: 3, scope: !12)
+!16 = !DILocation(line: 4, scope: !4)
diff --git a/test/DebugInfo/X86/line-info.ll b/test/DebugInfo/X86/line-info.ll
index b72fcf805ce..a3d957f8bc4 100644
--- a/test/DebugInfo/X86/line-info.ll
+++ b/test/DebugInfo/X86/line-info.ll
@@ -18,7 +18,7 @@ define i32 @foo(i32 %x) #0 {
 entry:
   %x.addr = alloca i32, align 4
   store i32 %x, i32* %x.addr, align 4
-  call void @llvm.dbg.declare(metadata i32* %x.addr, metadata !14, metadata !MDExpression()), !dbg !15
+  call void @llvm.dbg.declare(metadata i32* %x.addr, metadata !14, metadata !DIExpression()), !dbg !15
   %0 = load i32, i32* %x.addr, align 4, !dbg !16
   %inc = add nsw i32 %0, 1, !dbg !16
   store i32 %inc, i32* %x.addr, align 4, !dbg !16
@@ -38,23 +38,23 @@ attributes #1 = { nounwind readnone }
 !llvm.dbg.cu = !{!0}
 !llvm.module.flags = !{!19}
 
-!0 = !MDCompileUnit(language: DW_LANG_C99, producer: "clang version 3.3 ", isOptimized: false, emissionKind: 0, file: !1, enums: !2, retainedTypes: !2, subprograms: !3, globals: !2, imports:  !2)
-!1 = !MDFile(filename: "list0.c", directory: "/usr/local/google/home/blaikie/dev/scratch")
+!0 = !DICompileUnit(language: DW_LANG_C99, producer: "clang version 3.3 ", isOptimized: false, emissionKind: 0, file: !1, enums: !2, retainedTypes: !2, subprograms: !3, globals: !2, imports:  !2)
+!1 = !DIFile(filename: "list0.c", directory: "/usr/local/google/home/blaikie/dev/scratch")
 !2 = !{}
 !3 = !{!4, !10}
-!4 = !MDSubprogram(name: "foo", line: 1, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 1, file: !5, scope: !6, type: !7, function: i32 (i32)* @foo, variables: !2)
-!5 = !MDFile(filename: "./list0.h", directory: "/usr/local/google/home/blaikie/dev/scratch")
-!6 = !MDFile(filename: "./list0.h", directory: "/usr/local/google/home/blaikie/dev/scratch")
-!7 = !MDSubroutineType(types: !8)
+!4 = !DISubprogram(name: "foo", line: 1, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 1, file: !5, scope: !6, type: !7, function: i32 (i32)* @foo, variables: !2)
+!5 = !DIFile(filename: "./list0.h", directory: "/usr/local/google/home/blaikie/dev/scratch")
+!6 = !DIFile(filename: "./list0.h", directory: "/usr/local/google/home/blaikie/dev/scratch")
+!7 = !DISubroutineType(types: !8)
 !8 = !{!9, !9}
-!9 = !MDBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
-!10 = !MDSubprogram(name: "main", line: 2, isLocal: false, isDefinition: true, virtualIndex: 6, isOptimized: false, scopeLine: 2, file: !1, scope: !11, type: !12, function: i32 ()* @main, variables: !2)
-!11 = !MDFile(filename: "list0.c", directory: "/usr/local/google/home/blaikie/dev/scratch")
-!12 = !MDSubroutineType(types: !13)
+!9 = !DIBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
+!10 = !DISubprogram(name: "main", line: 2, isLocal: false, isDefinition: true, virtualIndex: 6, isOptimized: false, scopeLine: 2, file: !1, scope: !11, type: !12, function: i32 ()* @main, variables: !2)
+!11 = !DIFile(filename: "list0.c", directory: "/usr/local/google/home/blaikie/dev/scratch")
+!12 = !DISubroutineType(types: !13)
 !13 = !{!9}
-!14 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "x", line: 1, arg: 1, scope: !4, file: !6, type: !9)
-!15 = !MDLocation(line: 1, scope: !4)
-!16 = !MDLocation(line: 2, scope: !4)
-!17 = !MDLocation(line: 3, scope: !18)
-!18 = !MDLexicalBlockFile(discriminator: 0, file: !11, scope: !10)
+!14 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "x", line: 1, arg: 1, scope: !4, file: !6, type: !9)
+!15 = !DILocation(line: 1, scope: !4)
+!16 = !DILocation(line: 2, scope: !4)
+!17 = !DILocation(line: 3, scope: !18)
+!18 = !DILexicalBlockFile(discriminator: 0, file: !11, scope: !10)
 !19 = !{i32 1, !"Debug Info Version", i32 3}
diff --git a/test/DebugInfo/X86/linkage-name.ll b/test/DebugInfo/X86/linkage-name.ll
index 33ddf7cb56a..330a7f9e53a 100644
--- a/test/DebugInfo/X86/linkage-name.ll
+++ b/test/DebugInfo/X86/linkage-name.ll
@@ -14,9 +14,9 @@ entry:
   %this.addr = alloca %class.A*, align 8
   %b.addr = alloca i32, align 4
   store %class.A* %this, %class.A** %this.addr, align 8
-  call void @llvm.dbg.declare(metadata %class.A** %this.addr, metadata !21, metadata !MDExpression()), !dbg !23
+  call void @llvm.dbg.declare(metadata %class.A** %this.addr, metadata !21, metadata !DIExpression()), !dbg !23
   store i32 %b, i32* %b.addr, align 4
-  call void @llvm.dbg.declare(metadata i32* %b.addr, metadata !24, metadata !MDExpression()), !dbg !25
+  call void @llvm.dbg.declare(metadata i32* %b.addr, metadata !24, metadata !DIExpression()), !dbg !25
   %this1 = load %class.A*, %class.A** %this.addr
   %0 = load i32, i32* %b.addr, align 4, !dbg !26
   ret i32 %0, !dbg !26
@@ -27,26 +27,26 @@ declare void @llvm.dbg.declare(metadata, metadata, metadata) nounwind readnone
 !llvm.dbg.cu = !{!0}
 !llvm.module.flags = !{!29}
 
-!0 = !MDCompileUnit(language: DW_LANG_C_plus_plus, producer: "clang version 3.1 (trunk 152691) (llvm/trunk 152692)", isOptimized: false, emissionKind: 0, file: !28, enums: !1, retainedTypes: !1, subprograms: !3, globals: !18, imports:  !1)
+!0 = !DICompileUnit(language: DW_LANG_C_plus_plus, producer: "clang version 3.1 (trunk 152691) (llvm/trunk 152692)", isOptimized: false, emissionKind: 0, file: !28, enums: !1, retainedTypes: !1, subprograms: !3, globals: !18, imports:  !1)
 !1 = !{}
 !3 = !{!5}
-!5 = !MDSubprogram(name: "a", linkageName: "_ZN1A1aEi", line: 5, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 5, file: !6, scope: null, type: !7, function: i32 (%class.A*, i32)* @_ZN1A1aEi, declaration: !13)
-!6 = !MDFile(filename: "foo.cpp", directory: "/Users/echristo")
-!7 = !MDSubroutineType(types: !8)
+!5 = !DISubprogram(name: "a", linkageName: "_ZN1A1aEi", line: 5, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 5, file: !6, scope: null, type: !7, function: i32 (%class.A*, i32)* @_ZN1A1aEi, declaration: !13)
+!6 = !DIFile(filename: "foo.cpp", directory: "/Users/echristo")
+!7 = !DISubroutineType(types: !8)
 !8 = !{!9, !10, !9}
-!9 = !MDBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
-!10 = !MDDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, flags: DIFlagArtificial, baseType: !11)
-!11 = !MDCompositeType(tag: DW_TAG_class_type, name: "A", line: 1, size: 8, align: 8, file: !28, elements: !12)
+!9 = !DIBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
+!10 = !DIDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, flags: DIFlagArtificial, baseType: !11)
+!11 = !DICompositeType(tag: DW_TAG_class_type, name: "A", line: 1, size: 8, align: 8, file: !28, elements: !12)
 !12 = !{!13}
-!13 = !MDSubprogram(name: "a", linkageName: "_ZN1A1aEi", line: 2, isLocal: false, isDefinition: false, virtualIndex: 6, flags: DIFlagPrivate | DIFlagPrototyped, isOptimized: false, file: !6, scope: !11, type: !7)
+!13 = !DISubprogram(name: "a", linkageName: "_ZN1A1aEi", line: 2, isLocal: false, isDefinition: false, virtualIndex: 6, flags: DIFlagPrivate | DIFlagPrototyped, isOptimized: false, file: !6, scope: !11, type: !7)
 !18 = !{!20}
-!20 = !MDGlobalVariable(name: "a", line: 9, isLocal: false, isDefinition: true, scope: null, file: !6, type: !11, variable: %class.A* @a)
-!21 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "this", line: 5, arg: 1, flags: DIFlagArtificial, scope: !5, file: !6, type: !22)
-!22 = !MDDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, baseType: !11)
-!23 = !MDLocation(line: 5, column: 8, scope: !5)
-!24 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "b", line: 5, arg: 2, scope: !5, file: !6, type: !9)
-!25 = !MDLocation(line: 5, column: 14, scope: !5)
-!26 = !MDLocation(line: 6, column: 4, scope: !27)
-!27 = distinct !MDLexicalBlock(line: 5, column: 17, file: !6, scope: !5)
-!28 = !MDFile(filename: "foo.cpp", directory: "/Users/echristo")
+!20 = !DIGlobalVariable(name: "a", line: 9, isLocal: false, isDefinition: true, scope: null, file: !6, type: !11, variable: %class.A* @a)
+!21 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "this", line: 5, arg: 1, flags: DIFlagArtificial, scope: !5, file: !6, type: !22)
+!22 = !DIDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, baseType: !11)
+!23 = !DILocation(line: 5, column: 8, scope: !5)
+!24 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "b", line: 5, arg: 2, scope: !5, file: !6, type: !9)
+!25 = !DILocation(line: 5, column: 14, scope: !5)
+!26 = !DILocation(line: 6, column: 4, scope: !27)
+!27 = distinct !DILexicalBlock(line: 5, column: 17, file: !6, scope: !5)
+!28 = !DIFile(filename: "foo.cpp", directory: "/Users/echristo")
 !29 = !{i32 1, !"Debug Info Version", i32 3}
diff --git a/test/DebugInfo/X86/low-pc-cu.ll b/test/DebugInfo/X86/low-pc-cu.ll
index 1a80b6ad92b..727a7bb3dec 100644
--- a/test/DebugInfo/X86/low-pc-cu.ll
+++ b/test/DebugInfo/X86/low-pc-cu.ll
@@ -32,15 +32,15 @@ attributes #0 = { nounwind uwtable "less-precise-fpmad"="false" "no-frame-pointe
 !llvm.module.flags = !{!8, !9}
 !llvm.ident = !{!10}
 
-!0 = !MDCompileUnit(language: DW_LANG_C99, producer: "clang version 3.5.0 (trunk 204164) (llvm/trunk 204183)", isOptimized: false, emissionKind: 1, file: !1, enums: !2, retainedTypes: !2, subprograms: !3, globals: !2, imports: !2)
-!1 = !MDFile(filename: "z.c", directory: "/usr/local/google/home/echristo")
+!0 = !DICompileUnit(language: DW_LANG_C99, producer: "clang version 3.5.0 (trunk 204164) (llvm/trunk 204183)", isOptimized: false, emissionKind: 1, file: !1, enums: !2, retainedTypes: !2, subprograms: !3, globals: !2, imports: !2)
+!1 = !DIFile(filename: "z.c", directory: "/usr/local/google/home/echristo")
 !2 = !{}
 !3 = !{!4}
-!4 = !MDSubprogram(name: "z", line: 1, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 1, file: !1, scope: !5, type: !6, function: void ()* @z, variables: !2)
-!5 = !MDFile(filename: "z.c", directory: "/usr/local/google/home/echristo")
-!6 = !MDSubroutineType(types: !7)
+!4 = !DISubprogram(name: "z", line: 1, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 1, file: !1, scope: !5, type: !6, function: void ()* @z, variables: !2)
+!5 = !DIFile(filename: "z.c", directory: "/usr/local/google/home/echristo")
+!6 = !DISubroutineType(types: !7)
 !7 = !{null}
 !8 = !{i32 2, !"Dwarf Version", i32 4}
 !9 = !{i32 1, !"Debug Info Version", i32 3}
 !10 = !{!"clang version 3.5.0 (trunk 204164) (llvm/trunk 204183)"}
-!11 = !MDLocation(line: 1, scope: !4)
+!11 = !DILocation(line: 1, scope: !4)
diff --git a/test/DebugInfo/X86/memberfnptr.ll b/test/DebugInfo/X86/memberfnptr.ll
index f5f4845c99a..a53264d2c85 100644
--- a/test/DebugInfo/X86/memberfnptr.ll
+++ b/test/DebugInfo/X86/memberfnptr.ll
@@ -24,20 +24,20 @@ declare void @_ZN1A3fooEv(%struct.A*)
 !llvm.module.flags = !{!14, !15, !16}
 !llvm.ident = !{!17}
 
-!0 = !MDCompileUnit(language: DW_LANG_C_plus_plus, producer: "clang version 3.6.0 ", isOptimized: false, emissionKind: 1, file: !1, enums: !2, retainedTypes: !3, subprograms: !2, globals: !10, imports: !2)
-!1 = !MDFile(filename: "memberfnptr.cpp", directory: "")
+!0 = !DICompileUnit(language: DW_LANG_C_plus_plus, producer: "clang version 3.6.0 ", isOptimized: false, emissionKind: 1, file: !1, enums: !2, retainedTypes: !3, subprograms: !2, globals: !10, imports: !2)
+!1 = !DIFile(filename: "memberfnptr.cpp", directory: "")
 !2 = !{}
 !3 = !{!4}
-!4 = !MDCompositeType(tag: DW_TAG_structure_type, name: "A", line: 1, size: 8, align: 8, file: !1, elements: !5, identifier: "_ZTS1A")
+!4 = !DICompositeType(tag: DW_TAG_structure_type, name: "A", line: 1, size: 8, align: 8, file: !1, elements: !5, identifier: "_ZTS1A")
 !5 = !{!6}
-!6 = !MDSubprogram(name: "foo", linkageName: "_ZN1A3fooEv", line: 2, isLocal: false, isDefinition: false, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 2, file: !1, scope: !"_ZTS1A", type: !7)
-!7 = !MDSubroutineType(types: !8)
+!6 = !DISubprogram(name: "foo", linkageName: "_ZN1A3fooEv", line: 2, isLocal: false, isDefinition: false, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 2, file: !1, scope: !"_ZTS1A", type: !7)
+!7 = !DISubroutineType(types: !8)
 !8 = !{null, !9}
-!9 = !MDDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, flags: DIFlagArtificial | DIFlagObjectPointer, baseType: !"_ZTS1A")
+!9 = !DIDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, flags: DIFlagArtificial | DIFlagObjectPointer, baseType: !"_ZTS1A")
 !10 = !{!11}
-!11 = !MDGlobalVariable(name: "p", line: 5, isLocal: false, isDefinition: true, scope: null, file: !12, type: !13, variable: { i64, i64 }* @p)
-!12 = !MDFile(filename: "memberfnptr.cpp", directory: "")
-!13 = !MDDerivedType(tag: DW_TAG_ptr_to_member_type, size: 64, baseType: !7, extraData: !"_ZTS1A")
+!11 = !DIGlobalVariable(name: "p", line: 5, isLocal: false, isDefinition: true, scope: null, file: !12, type: !13, variable: { i64, i64 }* @p)
+!12 = !DIFile(filename: "memberfnptr.cpp", directory: "")
+!13 = !DIDerivedType(tag: DW_TAG_ptr_to_member_type, size: 64, baseType: !7, extraData: !"_ZTS1A")
 !14 = !{i32 2, !"Dwarf Version", i32 2}
 !15 = !{i32 2, !"Debug Info Version", i32 3}
 !16 = !{i32 1, !"PIC Level", i32 2}
diff --git a/test/DebugInfo/X86/mi-print.ll b/test/DebugInfo/X86/mi-print.ll
index 26dd0cbe608..b5d7b0ef067 100644
--- a/test/DebugInfo/X86/mi-print.ll
+++ b/test/DebugInfo/X86/mi-print.ll
@@ -31,26 +31,26 @@ attributes #1 = { nounwind readnone }
 !llvm.module.flags = !{!13, !14, !15}
 !llvm.ident = !{!16}
 
-!0 = !MDCompileUnit(language: DW_LANG_C99, file: !1, producer: "clang version 3.7.0 (trunk 233919) (llvm/trunk 233920)", isOptimized: true, runtimeVersion: 0, emissionKind: 1, enums: !2, retainedTypes: !2, subprograms: !3, globals: !2, imports: !2)
-!1 = !MDFile(filename: "t.c", directory: "/Users/dexonsmith/data/llvm/debug-info/test/DebugInfo/X86")
+!0 = !DICompileUnit(language: DW_LANG_C99, file: !1, producer: "clang version 3.7.0 (trunk 233919) (llvm/trunk 233920)", isOptimized: true, runtimeVersion: 0, emissionKind: 1, enums: !2, retainedTypes: !2, subprograms: !3, globals: !2, imports: !2)
+!1 = !DIFile(filename: "t.c", directory: "/Users/dexonsmith/data/llvm/debug-info/test/DebugInfo/X86")
 !2 = !{}
 !3 = !{!4, !10}
-!4 = !MDSubprogram(name: "bar", scope: !1, file: !1, line: 2, type: !5, isLocal: false, isDefinition: true, scopeLine: 2, flags: DIFlagPrototyped, isOptimized: true, function: i32 (i32)* @bar, variables: !8)
-!5 = !MDSubroutineType(types: !6)
+!4 = !DISubprogram(name: "bar", scope: !1, file: !1, line: 2, type: !5, isLocal: false, isDefinition: true, scopeLine: 2, flags: DIFlagPrototyped, isOptimized: true, function: i32 (i32)* @bar, variables: !8)
+!5 = !DISubroutineType(types: !6)
 !6 = !{!7, !7}
-!7 = !MDBasicType(name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
+!7 = !DIBasicType(name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
 !8 = !{!9}
-!9 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "x", arg: 1, scope: !4, file: !1, line: 2, type: !7)
-!10 = !MDSubprogram(name: "foo", scope: !1, file: !1, line: 1, type: !5, isLocal: true, isDefinition: true, scopeLine: 1, flags: DIFlagPrototyped, isOptimized: true, variables: !11)
+!9 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "x", arg: 1, scope: !4, file: !1, line: 2, type: !7)
+!10 = !DISubprogram(name: "foo", scope: !1, file: !1, line: 1, type: !5, isLocal: true, isDefinition: true, scopeLine: 1, flags: DIFlagPrototyped, isOptimized: true, variables: !11)
 !11 = !{!12}
-!12 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "x", arg: 1, scope: !10, file: !1, line: 1, type: !7)
+!12 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "x", arg: 1, scope: !10, file: !1, line: 1, type: !7)
 !13 = !{i32 2, !"Dwarf Version", i32 2}
 !14 = !{i32 2, !"Debug Info Version", i32 3}
 !15 = !{i32 1, !"PIC Level", i32 2}
 !16 = !{!"clang version 3.7.0 (trunk 233919) (llvm/trunk 233920)"}
-!17 = !MDExpression()
-!18 = !MDLocation(line: 2, column: 13, scope: !4)
-!19 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "x", arg: 1, scope: !10, file: !1, line: 1, type: !7)
-!20 = distinct !MDLocation(line: 2, column: 25, scope: !4)
-!21 = !MDLocation(line: 1, column: 20, scope: !10, inlinedAt: !20)
-!22 = !MDLocation(line: 2, column: 18, scope: !4)
+!17 = !DIExpression()
+!18 = !DILocation(line: 2, column: 13, scope: !4)
+!19 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "x", arg: 1, scope: !10, file: !1, line: 1, type: !7)
+!20 = distinct !DILocation(line: 2, column: 25, scope: !4)
+!21 = !DILocation(line: 1, column: 20, scope: !10, inlinedAt: !20)
+!22 = !DILocation(line: 2, column: 18, scope: !4)
diff --git a/test/DebugInfo/X86/misched-dbg-value.ll b/test/DebugInfo/X86/misched-dbg-value.ll
index 189cdd7b451..c660e1f72a9 100644
--- a/test/DebugInfo/X86/misched-dbg-value.ll
+++ b/test/DebugInfo/X86/misched-dbg-value.ll
@@ -48,12 +48,12 @@
 
 define void @Proc8(i32* nocapture %Array1Par, [51 x i32]* nocapture %Array2Par, i32 %IntParI1, i32 %IntParI2) nounwind optsize {
 entry:
-  tail call void @llvm.dbg.value(metadata i32* %Array1Par, i64 0, metadata !23, metadata !MDExpression()), !dbg !64
-  tail call void @llvm.dbg.value(metadata [51 x i32]* %Array2Par, i64 0, metadata !24, metadata !MDExpression()), !dbg !65
-  tail call void @llvm.dbg.value(metadata i32 %IntParI1, i64 0, metadata !25, metadata !MDExpression()), !dbg !66
-  tail call void @llvm.dbg.value(metadata i32 %IntParI2, i64 0, metadata !26, metadata !MDExpression()), !dbg !67
+  tail call void @llvm.dbg.value(metadata i32* %Array1Par, i64 0, metadata !23, metadata !DIExpression()), !dbg !64
+  tail call void @llvm.dbg.value(metadata [51 x i32]* %Array2Par, i64 0, metadata !24, metadata !DIExpression()), !dbg !65
+  tail call void @llvm.dbg.value(metadata i32 %IntParI1, i64 0, metadata !25, metadata !DIExpression()), !dbg !66
+  tail call void @llvm.dbg.value(metadata i32 %IntParI2, i64 0, metadata !26, metadata !DIExpression()), !dbg !67
   %add = add i32 %IntParI1, 5, !dbg !68
-  tail call void @llvm.dbg.value(metadata i32 %add, i64 0, metadata !27, metadata !MDExpression()), !dbg !68
+  tail call void @llvm.dbg.value(metadata i32 %add, i64 0, metadata !27, metadata !DIExpression()), !dbg !68
   %idxprom = sext i32 %add to i64, !dbg !69
   %arrayidx = getelementptr inbounds i32, i32* %Array1Par, i64 %idxprom, !dbg !69
   store i32 %IntParI2, i32* %arrayidx, align 4, !dbg !69
@@ -65,7 +65,7 @@ entry:
   %idxprom7 = sext i32 %add6 to i64, !dbg !74
   %arrayidx8 = getelementptr inbounds i32, i32* %Array1Par, i64 %idxprom7, !dbg !74
   store i32 %add, i32* %arrayidx8, align 4, !dbg !74
-  tail call void @llvm.dbg.value(metadata i32 %add, i64 0, metadata !28, metadata !MDExpression()), !dbg !75
+  tail call void @llvm.dbg.value(metadata i32 %add, i64 0, metadata !28, metadata !DIExpression()), !dbg !75
   br label %for.body, !dbg !75
 
 for.body:                                         ; preds = %entry, %for.body
@@ -74,7 +74,7 @@ for.body:                                         ; preds = %entry, %for.body
   %arrayidx13 = getelementptr inbounds [51 x i32], [51 x i32]* %Array2Par, i64 %idxprom, i64 %indvars.iv, !dbg !77
   store i32 %add, i32* %arrayidx13, align 4, !dbg !77
   %inc = add nsw i32 %IntIndex.046, 1, !dbg !75
-  tail call void @llvm.dbg.value(metadata i32 %inc, i64 0, metadata !28, metadata !MDExpression()), !dbg !75
+  tail call void @llvm.dbg.value(metadata i32 %inc, i64 0, metadata !28, metadata !DIExpression()), !dbg !75
   %cmp = icmp sgt i32 %inc, %add3, !dbg !75
   %indvars.iv.next = add i64 %indvars.iv, 1, !dbg !75
   br i1 %cmp, label %for.end, label %for.body, !dbg !75
@@ -103,84 +103,84 @@ attributes #1 = { nounwind readnone }
 !llvm.dbg.cu = !{!0}
 !llvm.module.flags = !{!83}
 
-!0 = !MDCompileUnit(language: DW_LANG_C99, producer: "clang version 3.3 (trunk 175015)", isOptimized: true, emissionKind: 1, file: !82, enums: !1, retainedTypes: !10, subprograms: !11, globals: !29, imports:  !10)
+!0 = !DICompileUnit(language: DW_LANG_C99, producer: "clang version 3.3 (trunk 175015)", isOptimized: true, emissionKind: 1, file: !82, enums: !1, retainedTypes: !10, subprograms: !11, globals: !29, imports:  !10)
 !1 = !{!2}
-!2 = !MDCompositeType(tag: DW_TAG_enumeration_type, line: 128, size: 32, align: 32, file: !82, elements: !4)
-!3 = !MDFile(filename: "dry.c", directory: "/Users/manmanren/test-Nov/rdar_13183203/test2")
+!2 = !DICompositeType(tag: DW_TAG_enumeration_type, line: 128, size: 32, align: 32, file: !82, elements: !4)
+!3 = !DIFile(filename: "dry.c", directory: "/Users/manmanren/test-Nov/rdar_13183203/test2")
 !4 = !{!5, !6, !7, !8, !9}
-!5 = !MDEnumerator(name: "Ident1", value: 0) ; [ DW_TAG_enumerator ] [Ident1 :: 0]
-!6 = !MDEnumerator(name: "Ident2", value: 10000) ; [ DW_TAG_enumerator ] [Ident2 :: 10000]
-!7 = !MDEnumerator(name: "Ident3", value: 10001) ; [ DW_TAG_enumerator ] [Ident3 :: 10001]
-!8 = !MDEnumerator(name: "Ident4", value: 10002) ; [ DW_TAG_enumerator ] [Ident4 :: 10002]
-!9 = !MDEnumerator(name: "Ident5", value: 10003) ; [ DW_TAG_enumerator ] [Ident5 :: 10003]
+!5 = !DIEnumerator(name: "Ident1", value: 0) ; [ DW_TAG_enumerator ] [Ident1 :: 0]
+!6 = !DIEnumerator(name: "Ident2", value: 10000) ; [ DW_TAG_enumerator ] [Ident2 :: 10000]
+!7 = !DIEnumerator(name: "Ident3", value: 10001) ; [ DW_TAG_enumerator ] [Ident3 :: 10001]
+!8 = !DIEnumerator(name: "Ident4", value: 10002) ; [ DW_TAG_enumerator ] [Ident4 :: 10002]
+!9 = !DIEnumerator(name: "Ident5", value: 10003) ; [ DW_TAG_enumerator ] [Ident5 :: 10003]
 !10 = !{}
 !11 = !{!12}
-!12 = !MDSubprogram(name: "Proc8", line: 180, isLocal: false, isDefinition: true, virtualIndex: 6, isOptimized: true, scopeLine: 185, file: !82, scope: !3, type: !13, function: void (i32*, [51 x i32]*, i32, i32)* @Proc8, variables: !22)
-!13 = !MDSubroutineType(types: !14)
+!12 = !DISubprogram(name: "Proc8", line: 180, isLocal: false, isDefinition: true, virtualIndex: 6, isOptimized: true, scopeLine: 185, file: !82, scope: !3, type: !13, function: void (i32*, [51 x i32]*, i32, i32)* @Proc8, variables: !22)
+!13 = !DISubroutineType(types: !14)
 !14 = !{null, !15, !17, !21, !21}
-!15 = !MDDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, baseType: !16)
-!16 = !MDBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
-!17 = !MDDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, baseType: !18)
-!18 = !MDCompositeType(tag: DW_TAG_array_type, size: 1632, align: 32, baseType: !16, elements: !19)
+!15 = !DIDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, baseType: !16)
+!16 = !DIBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
+!17 = !DIDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, baseType: !18)
+!18 = !DICompositeType(tag: DW_TAG_array_type, size: 1632, align: 32, baseType: !16, elements: !19)
 !19 = !{!20}
-!20 = !MDSubrange(count: 51)
-!21 = !MDDerivedType(tag: DW_TAG_typedef, name: "OneToFifty", line: 132, file: !82, baseType: !16)
+!20 = !DISubrange(count: 51)
+!21 = !DIDerivedType(tag: DW_TAG_typedef, name: "OneToFifty", line: 132, file: !82, baseType: !16)
 !22 = !{!23, !24, !25, !26, !27, !28}
-!23 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "Array1Par", line: 181, arg: 1, scope: !12, file: !3, type: !15)
-!24 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "Array2Par", line: 182, arg: 2, scope: !12, file: !3, type: !17)
-!25 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "IntParI1", line: 183, arg: 3, scope: !12, file: !3, type: !21)
-!26 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "IntParI2", line: 184, arg: 4, scope: !12, file: !3, type: !21)
-!27 = !MDLocalVariable(tag: DW_TAG_auto_variable, name: "IntLoc", line: 186, scope: !12, file: !3, type: !21)
-!28 = !MDLocalVariable(tag: DW_TAG_auto_variable, name: "IntIndex", line: 187, scope: !12, file: !3, type: !21)
+!23 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "Array1Par", line: 181, arg: 1, scope: !12, file: !3, type: !15)
+!24 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "Array2Par", line: 182, arg: 2, scope: !12, file: !3, type: !17)
+!25 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "IntParI1", line: 183, arg: 3, scope: !12, file: !3, type: !21)
+!26 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "IntParI2", line: 184, arg: 4, scope: !12, file: !3, type: !21)
+!27 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "IntLoc", line: 186, scope: !12, file: !3, type: !21)
+!28 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "IntIndex", line: 187, scope: !12, file: !3, type: !21)
 !29 = !{!30, !35, !36, !38, !39, !40, !42, !46, !63}
-!30 = !MDGlobalVariable(name: "Version", line: 111, isLocal: false, isDefinition: true, scope: null, file: !3, type: !31, variable: [4 x i8]* @Version)
-!31 = !MDCompositeType(tag: DW_TAG_array_type, size: 32, align: 8, baseType: !32, elements: !33)
-!32 = !MDBasicType(tag: DW_TAG_base_type, name: "char", size: 8, align: 8, encoding: DW_ATE_signed_char)
+!30 = !DIGlobalVariable(name: "Version", line: 111, isLocal: false, isDefinition: true, scope: null, file: !3, type: !31, variable: [4 x i8]* @Version)
+!31 = !DICompositeType(tag: DW_TAG_array_type, size: 32, align: 8, baseType: !32, elements: !33)
+!32 = !DIBasicType(tag: DW_TAG_base_type, name: "char", size: 8, align: 8, encoding: DW_ATE_signed_char)
 !33 = !{!34}
-!34 = !MDSubrange(count: 4)
-!35 = !MDGlobalVariable(name: "IntGlob", line: 171, isLocal: false, isDefinition: true, scope: null, file: !3, type: !16, variable: i32* @IntGlob)
-!36 = !MDGlobalVariable(name: "BoolGlob", line: 172, isLocal: false, isDefinition: true, scope: null, file: !3, type: !37, variable: i32* @BoolGlob)
-!37 = !MDDerivedType(tag: DW_TAG_typedef, name: "boolean", line: 149, file: !82, baseType: !16)
-!38 = !MDGlobalVariable(name: "Char1Glob", line: 173, isLocal: false, isDefinition: true, scope: null, file: !3, type: !32, variable: i8* @Char1Glob)
-!39 = !MDGlobalVariable(name: "Char2Glob", line: 174, isLocal: false, isDefinition: true, scope: null, file: !3, type: !32, variable: i8* @Char2Glob)
-!40 = !MDGlobalVariable(name: "Array1Glob", line: 175, isLocal: false, isDefinition: true, scope: null, file: !3, type: !41, variable: [51 x i32]* @Array1Glob)
-!41 = !MDDerivedType(tag: DW_TAG_typedef, name: "Array1Dim", line: 135, file: !82, baseType: !18)
-!42 = !MDGlobalVariable(name: "Array2Glob", line: 176, isLocal: false, isDefinition: true, scope: null, file: !3, type: !43, variable: [51 x [51 x i32]]* @Array2Glob)
-!43 = !MDDerivedType(tag: DW_TAG_typedef, name: "Array2Dim", line: 136, file: !82, baseType: !44)
-!44 = !MDCompositeType(tag: DW_TAG_array_type, size: 83232, align: 32, baseType: !16, elements: !45)
+!34 = !DISubrange(count: 4)
+!35 = !DIGlobalVariable(name: "IntGlob", line: 171, isLocal: false, isDefinition: true, scope: null, file: !3, type: !16, variable: i32* @IntGlob)
+!36 = !DIGlobalVariable(name: "BoolGlob", line: 172, isLocal: false, isDefinition: true, scope: null, file: !3, type: !37, variable: i32* @BoolGlob)
+!37 = !DIDerivedType(tag: DW_TAG_typedef, name: "boolean", line: 149, file: !82, baseType: !16)
+!38 = !DIGlobalVariable(name: "Char1Glob", line: 173, isLocal: false, isDefinition: true, scope: null, file: !3, type: !32, variable: i8* @Char1Glob)
+!39 = !DIGlobalVariable(name: "Char2Glob", line: 174, isLocal: false, isDefinition: true, scope: null, file: !3, type: !32, variable: i8* @Char2Glob)
+!40 = !DIGlobalVariable(name: "Array1Glob", line: 175, isLocal: false, isDefinition: true, scope: null, file: !3, type: !41, variable: [51 x i32]* @Array1Glob)
+!41 = !DIDerivedType(tag: DW_TAG_typedef, name: "Array1Dim", line: 135, file: !82, baseType: !18)
+!42 = !DIGlobalVariable(name: "Array2Glob", line: 176, isLocal: false, isDefinition: true, scope: null, file: !3, type: !43, variable: [51 x [51 x i32]]* @Array2Glob)
+!43 = !DIDerivedType(tag: DW_TAG_typedef, name: "Array2Dim", line: 136, file: !82, baseType: !44)
+!44 = !DICompositeType(tag: DW_TAG_array_type, size: 83232, align: 32, baseType: !16, elements: !45)
 !45 = !{!20, !20}
-!46 = !MDGlobalVariable(name: "PtrGlb", line: 177, isLocal: false, isDefinition: true, scope: null, file: !3, type: !47, variable: %struct.Record** @PtrGlb)
-!47 = !MDDerivedType(tag: DW_TAG_typedef, name: "RecordPtr", line: 148, file: !82, baseType: !48)
-!48 = !MDDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, baseType: !49)
-!49 = !MDDerivedType(tag: DW_TAG_typedef, name: "RecordType", line: 147, file: !82, baseType: !50)
-!50 = !MDCompositeType(tag: DW_TAG_structure_type, name: "Record", line: 138, size: 448, align: 64, file: !82, elements: !51)
+!46 = !DIGlobalVariable(name: "PtrGlb", line: 177, isLocal: false, isDefinition: true, scope: null, file: !3, type: !47, variable: %struct.Record** @PtrGlb)
+!47 = !DIDerivedType(tag: DW_TAG_typedef, name: "RecordPtr", line: 148, file: !82, baseType: !48)
+!48 = !DIDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, baseType: !49)
+!49 = !DIDerivedType(tag: DW_TAG_typedef, name: "RecordType", line: 147, file: !82, baseType: !50)
+!50 = !DICompositeType(tag: DW_TAG_structure_type, name: "Record", line: 138, size: 448, align: 64, file: !82, elements: !51)
 !51 = !{!52, !54, !56, !57, !58}
-!52 = !MDDerivedType(tag: DW_TAG_member, name: "PtrComp", line: 140, size: 64, align: 64, file: !82, scope: !50, baseType: !53)
-!53 = !MDDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, baseType: !50)
-!54 = !MDDerivedType(tag: DW_TAG_member, name: "Discr", line: 141, size: 32, align: 32, offset: 64, file: !82, scope: !50, baseType: !55)
-!55 = !MDDerivedType(tag: DW_TAG_typedef, name: "Enumeration", line: 128, file: !82, baseType: !2)
-!56 = !MDDerivedType(tag: DW_TAG_member, name: "EnumComp", line: 142, size: 32, align: 32, offset: 96, file: !82, scope: !50, baseType: !55)
-!57 = !MDDerivedType(tag: DW_TAG_member, name: "IntComp", line: 143, size: 32, align: 32, offset: 128, file: !82, scope: !50, baseType: !21)
-!58 = !MDDerivedType(tag: DW_TAG_member, name: "StringComp", line: 144, size: 248, align: 8, offset: 160, file: !82, scope: !50, baseType: !59)
-!59 = !MDDerivedType(tag: DW_TAG_typedef, name: "String30", line: 134, file: !82, baseType: !60)
-!60 = !MDCompositeType(tag: DW_TAG_array_type, size: 248, align: 8, baseType: !32, elements: !61)
+!52 = !DIDerivedType(tag: DW_TAG_member, name: "PtrComp", line: 140, size: 64, align: 64, file: !82, scope: !50, baseType: !53)
+!53 = !DIDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, baseType: !50)
+!54 = !DIDerivedType(tag: DW_TAG_member, name: "Discr", line: 141, size: 32, align: 32, offset: 64, file: !82, scope: !50, baseType: !55)
+!55 = !DIDerivedType(tag: DW_TAG_typedef, name: "Enumeration", line: 128, file: !82, baseType: !2)
+!56 = !DIDerivedType(tag: DW_TAG_member, name: "EnumComp", line: 142, size: 32, align: 32, offset: 96, file: !82, scope: !50, baseType: !55)
+!57 = !DIDerivedType(tag: DW_TAG_member, name: "IntComp", line: 143, size: 32, align: 32, offset: 128, file: !82, scope: !50, baseType: !21)
+!58 = !DIDerivedType(tag: DW_TAG_member, name: "StringComp", line: 144, size: 248, align: 8, offset: 160, file: !82, scope: !50, baseType: !59)
+!59 = !DIDerivedType(tag: DW_TAG_typedef, name: "String30", line: 134, file: !82, baseType: !60)
+!60 = !DICompositeType(tag: DW_TAG_array_type, size: 248, align: 8, baseType: !32, elements: !61)
 !61 = !{!62}
-!62 = !MDSubrange(count: 31)
-!63 = !MDGlobalVariable(name: "PtrGlbNext", line: 178, isLocal: false, isDefinition: true, scope: null, file: !3, type: !47, variable: %struct.Record** @PtrGlbNext)
-!64 = !MDLocation(line: 181, scope: !12)
-!65 = !MDLocation(line: 182, scope: !12)
-!66 = !MDLocation(line: 183, scope: !12)
-!67 = !MDLocation(line: 184, scope: !12)
-!68 = !MDLocation(line: 189, scope: !12)
-!69 = !MDLocation(line: 190, scope: !12)
-!73 = !MDLocation(line: 191, scope: !12)
-!74 = !MDLocation(line: 192, scope: !12)
-!75 = !MDLocation(line: 193, scope: !76)
-!76 = distinct !MDLexicalBlock(line: 193, column: 0, file: !82, scope: !12)
-!77 = !MDLocation(line: 194, scope: !76)
-!78 = !MDLocation(line: 195, scope: !12)
-!79 = !MDLocation(line: 196, scope: !12)
-!80 = !MDLocation(line: 197, scope: !12)
-!81 = !MDLocation(line: 198, scope: !12)
-!82 = !MDFile(filename: "dry.c", directory: "/Users/manmanren/test-Nov/rdar_13183203/test2")
+!62 = !DISubrange(count: 31)
+!63 = !DIGlobalVariable(name: "PtrGlbNext", line: 178, isLocal: false, isDefinition: true, scope: null, file: !3, type: !47, variable: %struct.Record** @PtrGlbNext)
+!64 = !DILocation(line: 181, scope: !12)
+!65 = !DILocation(line: 182, scope: !12)
+!66 = !DILocation(line: 183, scope: !12)
+!67 = !DILocation(line: 184, scope: !12)
+!68 = !DILocation(line: 189, scope: !12)
+!69 = !DILocation(line: 190, scope: !12)
+!73 = !DILocation(line: 191, scope: !12)
+!74 = !DILocation(line: 192, scope: !12)
+!75 = !DILocation(line: 193, scope: !76)
+!76 = distinct !DILexicalBlock(line: 193, column: 0, file: !82, scope: !12)
+!77 = !DILocation(line: 194, scope: !76)
+!78 = !DILocation(line: 195, scope: !12)
+!79 = !DILocation(line: 196, scope: !12)
+!80 = !DILocation(line: 197, scope: !12)
+!81 = !DILocation(line: 198, scope: !12)
+!82 = !DIFile(filename: "dry.c", directory: "/Users/manmanren/test-Nov/rdar_13183203/test2")
 !83 = !{i32 1, !"Debug Info Version", i32 3}
diff --git a/test/DebugInfo/X86/missing-file-line.ll b/test/DebugInfo/X86/missing-file-line.ll
index 6c09f4d940c..455868cb173 100644
--- a/test/DebugInfo/X86/missing-file-line.ll
+++ b/test/DebugInfo/X86/missing-file-line.ll
@@ -36,22 +36,22 @@ declare void @llvm.dbg.declare(metadata, metadata, metadata) #1
 !llvm.module.flags = !{!7, !8}
 !llvm.ident = !{!9}
 
-!0 = !MDCompileUnit(language: DW_LANG_C99, file: !1, producer: "clang", isOptimized: false, runtimeVersion: 0, emissionKind: 1, enums: !2, retainedTypes: !2, subprograms: !3, globals: !2, imports: !2)
-!1 = !MDFile(filename: "file.c", directory: "/dir")
+!0 = !DICompileUnit(language: DW_LANG_C99, file: !1, producer: "clang", isOptimized: false, runtimeVersion: 0, emissionKind: 1, enums: !2, retainedTypes: !2, subprograms: !3, globals: !2, imports: !2)
+!1 = !DIFile(filename: "file.c", directory: "/dir")
 !2 = !{}
 !3 = !{!4}
-!4 = !MDSubprogram(name: "f", scope: !1, file: !1, line: 7, type: !5, isLocal: false, isDefinition: true, scopeLine: 7, isOptimized: false, function: void ()* @f, variables: !2)
-!5 = !MDSubroutineType(types: !6)
+!4 = !DISubprogram(name: "f", scope: !1, file: !1, line: 7, type: !5, isLocal: false, isDefinition: true, scopeLine: 7, isOptimized: false, function: void ()* @f, variables: !2)
+!5 = !DISubroutineType(types: !6)
 !6 = !{null}
 !7 = !{i32 2, !"Dwarf Version", i32 4}
 !8 = !{i32 2, !"Debug Info Version", i32 3}
 !9 = !{!"clang"}
-!10 = !MDLocalVariable(tag: DW_TAG_auto_variable, name: "x", scope: !4, file: !1, line: 8, type: !11)
-!11 = !MDDerivedType(tag: DW_TAG_typedef, name: "SS", baseType: !12)
-!12 = !MDCompositeType(tag: DW_TAG_structure_type, name: "S", size: 64, align: 64, elements: !13)
+!10 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "x", scope: !4, file: !1, line: 8, type: !11)
+!11 = !DIDerivedType(tag: DW_TAG_typedef, name: "SS", baseType: !12)
+!12 = !DICompositeType(tag: DW_TAG_structure_type, name: "S", size: 64, align: 64, elements: !13)
 !13 = !{!14}
-!14 = !MDDerivedType(tag: DW_TAG_member, name: "s", scope: !12, baseType: !15, size: 64, align: 64)
-!15 = !MDDerivedType(tag: DW_TAG_pointer_type, baseType: !12, size: 64, align: 64)
-!16 = !MDExpression()
-!17 = !MDLocation(line: 8, column: 6, scope: !4)
-!18 = !MDLocation(line: 9, column: 1, scope: !4)
+!14 = !DIDerivedType(tag: DW_TAG_member, name: "s", scope: !12, baseType: !15, size: 64, align: 64)
+!15 = !DIDerivedType(tag: DW_TAG_pointer_type, baseType: !12, size: 64, align: 64)
+!16 = !DIExpression()
+!17 = !DILocation(line: 8, column: 6, scope: !4)
+!18 = !DILocation(line: 9, column: 1, scope: !4)
diff --git a/test/DebugInfo/X86/multiple-aranges.ll b/test/DebugInfo/X86/multiple-aranges.ll
index a4a2946fc5f..571454bd8d0 100644
--- a/test/DebugInfo/X86/multiple-aranges.ll
+++ b/test/DebugInfo/X86/multiple-aranges.ll
@@ -44,17 +44,17 @@ target triple = "x86_64-unknown-linux-gnu"
 !llvm.dbg.cu = !{!0, !7}
 !llvm.module.flags = !{!12, !13}
 
-!0 = !MDCompileUnit(language: DW_LANG_C99, producer: "clang version 3.4 ", isOptimized: false, emissionKind: 0, file: !1, enums: !2, retainedTypes: !2, subprograms: !2, globals: !3, imports: !2)
-!1 = !MDFile(filename: "test1.c", directory: "/home/kayamon")
+!0 = !DICompileUnit(language: DW_LANG_C99, producer: "clang version 3.4 ", isOptimized: false, emissionKind: 0, file: !1, enums: !2, retainedTypes: !2, subprograms: !2, globals: !3, imports: !2)
+!1 = !DIFile(filename: "test1.c", directory: "/home/kayamon")
 !2 = !{}
 !3 = !{!4}
-!4 = !MDGlobalVariable(name: "kittens", line: 1, isLocal: false, isDefinition: true, scope: null, file: !5, type: !6, variable: i32* @kittens)
-!5 = !MDFile(filename: "test1.c", directory: "/home/kayamon")
-!6 = !MDBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
-!7 = !MDCompileUnit(language: DW_LANG_C99, producer: "clang version 3.4 ", isOptimized: false, emissionKind: 0, file: !8, enums: !2, retainedTypes: !2, subprograms: !2, globals: !9, imports: !2)
-!8 = !MDFile(filename: "test2.c", directory: "/home/kayamon")
+!4 = !DIGlobalVariable(name: "kittens", line: 1, isLocal: false, isDefinition: true, scope: null, file: !5, type: !6, variable: i32* @kittens)
+!5 = !DIFile(filename: "test1.c", directory: "/home/kayamon")
+!6 = !DIBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
+!7 = !DICompileUnit(language: DW_LANG_C99, producer: "clang version 3.4 ", isOptimized: false, emissionKind: 0, file: !8, enums: !2, retainedTypes: !2, subprograms: !2, globals: !9, imports: !2)
+!8 = !DIFile(filename: "test2.c", directory: "/home/kayamon")
 !9 = !{!10}
-!10 = !MDGlobalVariable(name: "rainbows", line: 1, isLocal: false, isDefinition: true, scope: null, file: !11, type: !6, variable: i32* @rainbows)
-!11 = !MDFile(filename: "test2.c", directory: "/home/kayamon")
+!10 = !DIGlobalVariable(name: "rainbows", line: 1, isLocal: false, isDefinition: true, scope: null, file: !11, type: !6, variable: i32* @rainbows)
+!11 = !DIFile(filename: "test2.c", directory: "/home/kayamon")
 !12 = !{i32 2, !"Dwarf Version", i32 4}
 !13 = !{i32 1, !"Debug Info Version", i32 3}
diff --git a/test/DebugInfo/X86/multiple-at-const-val.ll b/test/DebugInfo/X86/multiple-at-const-val.ll
index b7ec9104ba0..01e3af20d2d 100644
--- a/test/DebugInfo/X86/multiple-at-const-val.ll
+++ b/test/DebugInfo/X86/multiple-at-const-val.ll
@@ -32,32 +32,32 @@ declare void @llvm.dbg.value(metadata, i64, metadata, metadata) nounwind readnon
 !llvm.dbg.cu = !{!0}
 !llvm.module.flags = !{!1803}
 
-!0 = !MDCompileUnit(language: DW_LANG_C_plus_plus, producer: "clang version 3.3 (trunk 174207)", isOptimized: true, emissionKind: 0, file: !1802, enums: !1, retainedTypes: !955, subprograms: !956, globals: !1786, imports:  !955)
+!0 = !DICompileUnit(language: DW_LANG_C_plus_plus, producer: "clang version 3.3 (trunk 174207)", isOptimized: true, emissionKind: 0, file: !1802, enums: !1, retainedTypes: !955, subprograms: !956, globals: !1786, imports:  !955)
 !1 = !{!26}
-!4 = !MDNamespace(name: "std", line: 48, scope: !5)
-!5 = !MDFile(filename: "os_base.h", directory: "/privite/tmp")
-!25 = !MDEnumerator(name: "_S_os_fmtflags_end", value: 65536) ; [ DW_TAG_enumerator ]
-!26 = !MDCompositeType(tag: DW_TAG_enumeration_type, name: "_Ios_Iostate", line: 146, size: 32, align: 32, file: !1801, scope: !4, elements: !27)
+!4 = !DINamespace(name: "std", line: 48, scope: !5)
+!5 = !DIFile(filename: "os_base.h", directory: "/privite/tmp")
+!25 = !DIEnumerator(name: "_S_os_fmtflags_end", value: 65536) ; [ DW_TAG_enumerator ]
+!26 = !DICompositeType(tag: DW_TAG_enumeration_type, name: "_Ios_Iostate", line: 146, size: 32, align: 32, file: !1801, scope: !4, elements: !27)
 !27 = !{!28, !29, !30, !31, !32}
-!28 = !MDEnumerator(name: "_S_goodbit", value: 0) ; [ DW_TAG_enumerator ] [_S_goodbit :: 0]
-!29 = !MDEnumerator(name: "_S_badbit", value: 1) ; [ DW_TAG_enumerator ] [_S_badbit :: 1]
-!30 = !MDEnumerator(name: "_S_eofbit", value: 2) ; [ DW_TAG_enumerator ] [_S_eofbit :: 2]
-!31 = !MDEnumerator(name: "_S_failbit", value: 4) ; [ DW_TAG_enumerator ] [_S_failbit :: 4]
-!32 = !MDEnumerator(name: "_S_os_ostate_end", value: 65536) ; [ DW_TAG_enumerator ] [_S_os_ostate_end :: 65536]
-!49 = !MDCompositeType(tag: DW_TAG_class_type, name: "os_base", line: 200, size: 1728, align: 64, file: !1801, scope: !4, elements: !50, vtableHolder: !49)
+!28 = !DIEnumerator(name: "_S_goodbit", value: 0) ; [ DW_TAG_enumerator ] [_S_goodbit :: 0]
+!29 = !DIEnumerator(name: "_S_badbit", value: 1) ; [ DW_TAG_enumerator ] [_S_badbit :: 1]
+!30 = !DIEnumerator(name: "_S_eofbit", value: 2) ; [ DW_TAG_enumerator ] [_S_eofbit :: 2]
+!31 = !DIEnumerator(name: "_S_failbit", value: 4) ; [ DW_TAG_enumerator ] [_S_failbit :: 4]
+!32 = !DIEnumerator(name: "_S_os_ostate_end", value: 65536) ; [ DW_TAG_enumerator ] [_S_os_ostate_end :: 65536]
+!49 = !DICompositeType(tag: DW_TAG_class_type, name: "os_base", line: 200, size: 1728, align: 64, file: !1801, scope: !4, elements: !50, vtableHolder: !49)
 !50 = !{!77}
-!54 = !MDSubroutineType(types: !55)
+!54 = !DISubroutineType(types: !55)
 !55 = !{!56}
-!56 = !MDBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
-!77 = !MDDerivedType(tag: DW_TAG_member, name: "badbit", line: 331, flags: DIFlagStaticMember, file: !1801, scope: !49, baseType: !78, extraData: i32 1)
-!78 = !MDDerivedType(tag: DW_TAG_const_type, baseType: !79)
-!79 = !MDDerivedType(tag: DW_TAG_typedef, name: "ostate", line: 327, file: !1801, scope: !49, baseType: !26)
+!56 = !DIBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
+!77 = !DIDerivedType(tag: DW_TAG_member, name: "badbit", line: 331, flags: DIFlagStaticMember, file: !1801, scope: !49, baseType: !78, extraData: i32 1)
+!78 = !DIDerivedType(tag: DW_TAG_const_type, baseType: !79)
+!79 = !DIDerivedType(tag: DW_TAG_typedef, name: "ostate", line: 327, file: !1801, scope: !49, baseType: !26)
 !955 = !{}
 !956 = !{!960}
-!960 = !MDSubprogram(name: "main", line: 73, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: true, scopeLine: 73, file: !1802, scope: null, type: !54, function: i32 ()* @main, variables: !955)
-!961 = !MDFile(filename: "student2.cpp", directory: "/privite/tmp")
+!960 = !DISubprogram(name: "main", line: 73, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: true, scopeLine: 73, file: !1802, scope: null, type: !54, function: i32 ()* @main, variables: !955)
+!961 = !DIFile(filename: "student2.cpp", directory: "/privite/tmp")
 !1786 = !{!1800}
-!1800 = !MDGlobalVariable(name: "badbit", linkageName: "badbit", line: 331, isLocal: true, isDefinition: true, scope: !5, file: !5, type: !78, variable: i32 1, declaration: !77)
-!1801 = !MDFile(filename: "os_base.h", directory: "/privite/tmp")
-!1802 = !MDFile(filename: "student2.cpp", directory: "/privite/tmp")
+!1800 = !DIGlobalVariable(name: "badbit", linkageName: "badbit", line: 331, isLocal: true, isDefinition: true, scope: !5, file: !5, type: !78, variable: i32 1, declaration: !77)
+!1801 = !DIFile(filename: "os_base.h", directory: "/privite/tmp")
+!1802 = !DIFile(filename: "student2.cpp", directory: "/privite/tmp")
 !1803 = !{i32 1, !"Debug Info Version", i32 3}
diff --git a/test/DebugInfo/X86/nodebug_with_debug_loc.ll b/test/DebugInfo/X86/nodebug_with_debug_loc.ll
index e4fb0626d9e..8153b3352d4 100644
--- a/test/DebugInfo/X86/nodebug_with_debug_loc.ll
+++ b/test/DebugInfo/X86/nodebug_with_debug_loc.ll
@@ -58,8 +58,8 @@ entry:
 for.body:                                         ; preds = %for.body, %entry
   %iter.02 = phi i32 [ 0, %entry ], [ %inc, %for.body ]
   call void @llvm.lifetime.start(i64 4, i8* %0), !dbg !26
-  call void @llvm.dbg.value(metadata %struct.string* %str2.i, i64 0, metadata !16, metadata !MDExpression()) #3, !dbg !26
-  call void @llvm.dbg.value(metadata %struct.string* %str2.i, i64 0, metadata !27, metadata !MDExpression()) #3, !dbg !29
+  call void @llvm.dbg.value(metadata %struct.string* %str2.i, i64 0, metadata !16, metadata !DIExpression()) #3, !dbg !26
+  call void @llvm.dbg.value(metadata %struct.string* %str2.i, i64 0, metadata !27, metadata !DIExpression()) #3, !dbg !29
   call void @_Z4sinkPKv(i8* undef) #3, !dbg !29
   call void @_Z4sinkPKv(i8* %0) #3, !dbg !30
   call void @llvm.lifetime.end(i64 4, i8* %0), !dbg !31
@@ -97,38 +97,38 @@ attributes #3 = { nounwind }
 !llvm.module.flags = !{!23, !24}
 !llvm.ident = !{!25}
 
-!0 = !MDCompileUnit(language: DW_LANG_C_plus_plus, producer: "clang version 3.5.0 ", isOptimized: true, emissionKind: 1, file: !1, enums: !2, retainedTypes: !3, subprograms: !10, globals: !2, imports: !2)
-!1 = !MDFile(filename: "<stdin>", directory: "/tmp/dbginfo")
+!0 = !DICompileUnit(language: DW_LANG_C_plus_plus, producer: "clang version 3.5.0 ", isOptimized: true, emissionKind: 1, file: !1, enums: !2, retainedTypes: !3, subprograms: !10, globals: !2, imports: !2)
+!1 = !DIFile(filename: "<stdin>", directory: "/tmp/dbginfo")
 !2 = !{}
 !3 = !{!4}
-!4 = !MDCompositeType(tag: DW_TAG_structure_type, name: "string", line: 7, size: 32, align: 32, file: !5, elements: !6, identifier: "_ZTS6string")
-!5 = !MDFile(filename: "repro.cpp", directory: "/tmp/dbginfo")
+!4 = !DICompositeType(tag: DW_TAG_structure_type, name: "string", line: 7, size: 32, align: 32, file: !5, elements: !6, identifier: "_ZTS6string")
+!5 = !DIFile(filename: "repro.cpp", directory: "/tmp/dbginfo")
 !6 = !{!7}
-!7 = !MDDerivedType(tag: DW_TAG_member, name: "mem", line: 8, size: 32, align: 32, file: !5, scope: !"_ZTS6string", baseType: !8)
-!8 = !MDDerivedType(tag: DW_TAG_pointer_type, size: 32, align: 32, baseType: !9)
-!9 = !MDBasicType(tag: DW_TAG_base_type, name: "unsigned int", size: 32, align: 32, encoding: DW_ATE_unsigned)
+!7 = !DIDerivedType(tag: DW_TAG_member, name: "mem", line: 8, size: 32, align: 32, file: !5, scope: !"_ZTS6string", baseType: !8)
+!8 = !DIDerivedType(tag: DW_TAG_pointer_type, size: 32, align: 32, baseType: !9)
+!9 = !DIBasicType(tag: DW_TAG_base_type, name: "unsigned int", size: 32, align: 32, encoding: DW_ATE_unsigned)
 !10 = !{!11, !17}
-!11 = !MDSubprogram(name: "f", linkageName: "_Z1fv", line: 14, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: true, scopeLine: 14, file: !5, scope: !12, type: !13, variables: !15)
-!12 = !MDFile(filename: "repro.cpp", directory: "/tmp/dbginfo")
-!13 = !MDSubroutineType(types: !14)
+!11 = !DISubprogram(name: "f", linkageName: "_Z1fv", line: 14, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: true, scopeLine: 14, file: !5, scope: !12, type: !13, variables: !15)
+!12 = !DIFile(filename: "repro.cpp", directory: "/tmp/dbginfo")
+!13 = !DISubroutineType(types: !14)
 !14 = !{null}
 !15 = !{!16}
-!16 = !MDLocalVariable(tag: DW_TAG_auto_variable, name: "str2", line: 15, scope: !11, file: !12, type: !"_ZTS6string")
-!17 = !MDSubprogram(name: "s2", linkageName: "_Z2s2P6string", line: 13, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: true, scopeLine: 13, file: !5, scope: !12, type: !18, variables: !21)
-!18 = !MDSubroutineType(types: !19)
+!16 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "str2", line: 15, scope: !11, file: !12, type: !"_ZTS6string")
+!17 = !DISubprogram(name: "s2", linkageName: "_Z2s2P6string", line: 13, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: true, scopeLine: 13, file: !5, scope: !12, type: !18, variables: !21)
+!18 = !DISubroutineType(types: !19)
 !19 = !{null, !20}
-!20 = !MDDerivedType(tag: DW_TAG_pointer_type, size: 32, align: 32, baseType: !"_ZTS6string")
+!20 = !DIDerivedType(tag: DW_TAG_pointer_type, size: 32, align: 32, baseType: !"_ZTS6string")
 !21 = !{!22}
-!22 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "lhs", line: 13, arg: 1, scope: !17, file: !12, type: !20)
+!22 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "lhs", line: 13, arg: 1, scope: !17, file: !12, type: !20)
 !23 = !{i32 2, !"Dwarf Version", i32 4}
 !24 = !{i32 2, !"Debug Info Version", i32 3}
 !25 = !{!"clang version 3.5.0 "}
-!26 = !MDLocation(line: 15, scope: !11)
-!27 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "lhs", line: 13, arg: 1, scope: !17, file: !12, type: !20)
-!28 = !MDLocation(line: 16, scope: !11)
-!29 = !MDLocation(line: 13, scope: !17, inlinedAt: !28)
-!30 = !MDLocation(line: 17, scope: !11)
-!31 = !MDLocation(line: 18, scope: !11)
+!26 = !DILocation(line: 15, scope: !11)
+!27 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "lhs", line: 13, arg: 1, scope: !17, file: !12, type: !20)
+!28 = !DILocation(line: 16, scope: !11)
+!29 = !DILocation(line: 13, scope: !17, inlinedAt: !28)
+!30 = !DILocation(line: 17, scope: !11)
+!31 = !DILocation(line: 18, scope: !11)
 !32 = !{!33, !34, i64 0}
 !33 = !{!"_ZTS6string", !34, i64 0}
 !34 = !{!"any pointer", !35, i64 0}
diff --git a/test/DebugInfo/X86/nondefault-subrange-array.ll b/test/DebugInfo/X86/nondefault-subrange-array.ll
index 172bacbb33a..20406cfa0f1 100644
--- a/test/DebugInfo/X86/nondefault-subrange-array.ll
+++ b/test/DebugInfo/X86/nondefault-subrange-array.ll
@@ -30,21 +30,21 @@
 !llvm.dbg.cu = !{!0}
 !llvm.module.flags = !{!21}
 
-!0 = !MDCompileUnit(language: DW_LANG_C_plus_plus, producer: "clang version 3.3 (trunk 169136)", isOptimized: false, emissionKind: 0, file: !20, enums: !1, retainedTypes: !1, subprograms: !1, globals: !3, imports:  !1)
+!0 = !DICompileUnit(language: DW_LANG_C_plus_plus, producer: "clang version 3.3 (trunk 169136)", isOptimized: false, emissionKind: 0, file: !20, enums: !1, retainedTypes: !1, subprograms: !1, globals: !3, imports:  !1)
 !1 = !{}
 !3 = !{!5}
-!5 = !MDGlobalVariable(name: "a", line: 1, isLocal: false, isDefinition: true, scope: null, file: !6, type: !7, variable: %class.A* @a)
-!6 = !MDFile(filename: "t.cpp", directory: "/Volumes/Sandbox/llvm")
-!7 = !MDCompositeType(tag: DW_TAG_class_type, name: "A", line: 1, align: 32, file: !20, elements: !8)
+!5 = !DIGlobalVariable(name: "a", line: 1, isLocal: false, isDefinition: true, scope: null, file: !6, type: !7, variable: %class.A* @a)
+!6 = !DIFile(filename: "t.cpp", directory: "/Volumes/Sandbox/llvm")
+!7 = !DICompositeType(tag: DW_TAG_class_type, name: "A", line: 1, align: 32, file: !20, elements: !8)
 !8 = !{!9, !14}
-!9 = !MDDerivedType(tag: DW_TAG_member, name: "x", line: 1, flags: DIFlagPrivate, file: !20, scope: !7, baseType: !10)
-!10 = !MDCompositeType(tag: DW_TAG_array_type, align: 32, baseType: !11, elements: !12)
-!11 = !MDBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
+!9 = !DIDerivedType(tag: DW_TAG_member, name: "x", line: 1, flags: DIFlagPrivate, file: !20, scope: !7, baseType: !10)
+!10 = !DICompositeType(tag: DW_TAG_array_type, align: 32, baseType: !11, elements: !12)
+!11 = !DIBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
 !12 = !{!13}
-!13 = !MDSubrange(count: 42, lowerBound: -3)
-!14 = !MDSubprogram(name: "A", line: 1, isLocal: false, isDefinition: false, virtualIndex: 6, flags: DIFlagArtificial | DIFlagPrototyped, isOptimized: false, scopeLine: 1, file: !6, scope: !7, type: !15)
-!15 = !MDSubroutineType(types: !16)
+!13 = !DISubrange(count: 42, lowerBound: -3)
+!14 = !DISubprogram(name: "A", line: 1, isLocal: false, isDefinition: false, virtualIndex: 6, flags: DIFlagArtificial | DIFlagPrototyped, isOptimized: false, scopeLine: 1, file: !6, scope: !7, type: !15)
+!15 = !DISubroutineType(types: !16)
 !16 = !{null, !17}
-!17 = !MDDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, flags: DIFlagArtificial | DIFlagObjectPointer, baseType: !7)
-!20 = !MDFile(filename: "t.cpp", directory: "/Volumes/Sandbox/llvm")
+!17 = !DIDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, flags: DIFlagArtificial | DIFlagObjectPointer, baseType: !7)
+!20 = !DIFile(filename: "t.cpp", directory: "/Volumes/Sandbox/llvm")
 !21 = !{i32 1, !"Debug Info Version", i32 3}
diff --git a/test/DebugInfo/X86/nophysreg.ll b/test/DebugInfo/X86/nophysreg.ll
index 22394c8cc16..f4478efb67f 100644
--- a/test/DebugInfo/X86/nophysreg.ll
+++ b/test/DebugInfo/X86/nophysreg.ll
@@ -136,68 +136,68 @@ attributes #3 = { ssp uwtable }
 !llvm.module.flags = !{!29, !30, !31}
 !llvm.ident = !{!32}
 
-!0 = !MDCompileUnit(language: DW_LANG_C_plus_plus, producer: "clang version 3.7.0 (trunk 227088) (llvm/trunk 227091)", isOptimized: true, emissionKind: 1, file: !1, enums: !2, retainedTypes: !3, subprograms: !10, globals: !2, imports: !2)
-!1 = !MDFile(filename: "test.cpp", directory: "")
+!0 = !DICompileUnit(language: DW_LANG_C_plus_plus, producer: "clang version 3.7.0 (trunk 227088) (llvm/trunk 227091)", isOptimized: true, emissionKind: 1, file: !1, enums: !2, retainedTypes: !3, subprograms: !10, globals: !2, imports: !2)
+!1 = !DIFile(filename: "test.cpp", directory: "")
 !2 = !{}
 !3 = !{!4}
-!4 = !MDCompositeType(tag: DW_TAG_structure_type, name: "A", line: 1, size: 128, align: 64, file: !1, elements: !5, identifier: "_ZTS1A")
+!4 = !DICompositeType(tag: DW_TAG_structure_type, name: "A", line: 1, size: 128, align: 64, file: !1, elements: !5, identifier: "_ZTS1A")
 !5 = !{!6, !9}
-!6 = !MDDerivedType(tag: DW_TAG_member, name: "m1", line: 2, size: 64, align: 64, file: !1, scope: !"_ZTS1A", baseType: !7)
-!7 = !MDDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, baseType: !8)
-!8 = !MDBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
-!9 = !MDDerivedType(tag: DW_TAG_member, name: "m2", line: 3, size: 32, align: 32, offset: 64, file: !1, scope: !"_ZTS1A", baseType: !8)
+!6 = !DIDerivedType(tag: DW_TAG_member, name: "m1", line: 2, size: 64, align: 64, file: !1, scope: !"_ZTS1A", baseType: !7)
+!7 = !DIDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, baseType: !8)
+!8 = !DIBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
+!9 = !DIDerivedType(tag: DW_TAG_member, name: "m2", line: 3, size: 32, align: 32, offset: 64, file: !1, scope: !"_ZTS1A", baseType: !8)
 !10 = !{!11, !17}
-!11 = !MDSubprogram(name: "f2", linkageName: "_Z2f21A", line: 7, isLocal: false, isDefinition: true, flags: DIFlagPrototyped, isOptimized: true, scopeLine: 7, file: !1, scope: !12, type: !13, function: void (i32*, i32)* @_Z2f21A, variables: !15)
-!12 = !MDFile(filename: "test.cpp", directory: "")
-!13 = !MDSubroutineType(types: !14)
+!11 = !DISubprogram(name: "f2", linkageName: "_Z2f21A", line: 7, isLocal: false, isDefinition: true, flags: DIFlagPrototyped, isOptimized: true, scopeLine: 7, file: !1, scope: !12, type: !13, function: void (i32*, i32)* @_Z2f21A, variables: !15)
+!12 = !DIFile(filename: "test.cpp", directory: "")
+!13 = !DISubroutineType(types: !14)
 !14 = !{null, !"_ZTS1A"}
 !15 = !{!16}
-!16 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "p5", line: 7, arg: 1, scope: !11, file: !12, type: !"_ZTS1A")
-!17 = !MDSubprogram(name: "f", linkageName: "_Z1fv", line: 12, isLocal: false, isDefinition: true, flags: DIFlagPrototyped, isOptimized: true, scopeLine: 12, file: !1, scope: !12, type: !18, function: void ()* @_Z1fv, variables: !20)
-!18 = !MDSubroutineType(types: !19)
+!16 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "p5", line: 7, arg: 1, scope: !11, file: !12, type: !"_ZTS1A")
+!17 = !DISubprogram(name: "f", linkageName: "_Z1fv", line: 12, isLocal: false, isDefinition: true, flags: DIFlagPrototyped, isOptimized: true, scopeLine: 12, file: !1, scope: !12, type: !18, function: void ()* @_Z1fv, variables: !20)
+!18 = !DISubroutineType(types: !19)
 !19 = !{null}
 !20 = !{!21, !23, !26, !27, !28}
-!21 = !MDLocalVariable(tag: DW_TAG_auto_variable, name: "x", line: 14, scope: !22, file: !12, type: !8)
-!22 = distinct !MDLexicalBlock(line: 13, column: 18, file: !1, scope: !17)
-!23 = !MDLocalVariable(tag: DW_TAG_auto_variable, name: "y", line: 17, scope: !24, file: !12, type: !25)
-!24 = distinct !MDLexicalBlock(line: 16, column: 20, file: !1, scope: !22)
-!25 = !MDBasicType(tag: DW_TAG_base_type, name: "char", size: 8, align: 8, encoding: DW_ATE_signed_char)
-!26 = !MDLocalVariable(tag: DW_TAG_auto_variable, name: "j", line: 19, scope: !24, file: !12, type: !25)
-!27 = !MDLocalVariable(tag: DW_TAG_auto_variable, name: "I", line: 21, scope: !24, file: !12, type: !25)
-!28 = !MDLocalVariable(tag: DW_TAG_auto_variable, name: "g", line: 24, scope: !24, file: !12, type: !"_ZTS1A")
+!21 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "x", line: 14, scope: !22, file: !12, type: !8)
+!22 = distinct !DILexicalBlock(line: 13, column: 18, file: !1, scope: !17)
+!23 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "y", line: 17, scope: !24, file: !12, type: !25)
+!24 = distinct !DILexicalBlock(line: 16, column: 20, file: !1, scope: !22)
+!25 = !DIBasicType(tag: DW_TAG_base_type, name: "char", size: 8, align: 8, encoding: DW_ATE_signed_char)
+!26 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "j", line: 19, scope: !24, file: !12, type: !25)
+!27 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "I", line: 21, scope: !24, file: !12, type: !25)
+!28 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "g", line: 24, scope: !24, file: !12, type: !"_ZTS1A")
 !29 = !{i32 2, !"Dwarf Version", i32 2}
 !30 = !{i32 2, !"Debug Info Version", i32 3}
 !31 = !{i32 1, !"PIC Level", i32 2}
 !32 = !{!"clang version 3.7.0 (trunk 227088) (llvm/trunk 227091)"}
-!33 = !MDExpression(DW_OP_bit_piece, 0, 8)
-!34 = !MDLocation(line: 7, column: 42, scope: !11)
-!35 = !MDExpression(DW_OP_bit_piece, 8, 4)
-!36 = !MDExpression()
-!37 = !MDLocation(line: 7, column: 48, scope: !11)
-!38 = !MDLocation(line: 7, column: 66, scope: !11)
-!39 = !MDLocation(line: 13, column: 10, scope: !17)
-!40 = !MDLocation(line: 13, column: 3, scope: !17)
-!41 = !MDLocation(line: 15, column: 13, scope: !22)
+!33 = !DIExpression(DW_OP_bit_piece, 0, 8)
+!34 = !DILocation(line: 7, column: 42, scope: !11)
+!35 = !DIExpression(DW_OP_bit_piece, 8, 4)
+!36 = !DIExpression()
+!37 = !DILocation(line: 7, column: 48, scope: !11)
+!38 = !DILocation(line: 7, column: 66, scope: !11)
+!39 = !DILocation(line: 13, column: 10, scope: !17)
+!40 = !DILocation(line: 13, column: 3, scope: !17)
+!41 = !DILocation(line: 15, column: 13, scope: !22)
 !42 = !{!43, !43, i64 0}
 !43 = !{!"int", !44, i64 0}
 !44 = !{!"omnipotent char", !45, i64 0}
 !45 = !{!"Simple C/C++ TBAA"}
-!46 = !MDLocation(line: 14, column: 9, scope: !22)
-!47 = !MDLocation(line: 15, column: 5, scope: !22)
-!48 = !MDLocation(line: 16, column: 12, scope: !22)
-!49 = !MDLocation(line: 16, column: 5, scope: !22)
-!50 = !MDLocation(line: 17, column: 12, scope: !24)
-!51 = !MDLocation(line: 18, column: 7, scope: !24)
-!52 = !MDLocation(line: 19, column: 12, scope: !24)
-!53 = !MDLocation(line: 20, column: 7, scope: !24)
-!54 = !MDLocation(line: 21, column: 12, scope: !24)
-!55 = !MDLocation(line: 22, column: 7, scope: !24)
-!56 = !MDLocation(line: 23, column: 12, scope: !24)
-!57 = !MDLocation(line: 23, column: 15, scope: !24)
-!58 = !MDLocation(line: 23, column: 7, scope: !24)
-!59 = !MDLocation(line: 24, column: 9, scope: !24)
-!60 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "p5", line: 7, arg: 1, scope: !11, file: !12, type: !"_ZTS1A")
-!61 = distinct !MDLocation(line: 26, column: 7, scope: !24)
-!62 = !MDLocation(line: 7, column: 42, scope: !11, inlinedAt: !61)
-!63 = !MDLocation(line: 7, column: 48, scope: !11, inlinedAt: !61)
-!64 = !MDLocation(line: 29, column: 1, scope: !17)
+!46 = !DILocation(line: 14, column: 9, scope: !22)
+!47 = !DILocation(line: 15, column: 5, scope: !22)
+!48 = !DILocation(line: 16, column: 12, scope: !22)
+!49 = !DILocation(line: 16, column: 5, scope: !22)
+!50 = !DILocation(line: 17, column: 12, scope: !24)
+!51 = !DILocation(line: 18, column: 7, scope: !24)
+!52 = !DILocation(line: 19, column: 12, scope: !24)
+!53 = !DILocation(line: 20, column: 7, scope: !24)
+!54 = !DILocation(line: 21, column: 12, scope: !24)
+!55 = !DILocation(line: 22, column: 7, scope: !24)
+!56 = !DILocation(line: 23, column: 12, scope: !24)
+!57 = !DILocation(line: 23, column: 15, scope: !24)
+!58 = !DILocation(line: 23, column: 7, scope: !24)
+!59 = !DILocation(line: 24, column: 9, scope: !24)
+!60 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "p5", line: 7, arg: 1, scope: !11, file: !12, type: !"_ZTS1A")
+!61 = distinct !DILocation(line: 26, column: 7, scope: !24)
+!62 = !DILocation(line: 7, column: 42, scope: !11, inlinedAt: !61)
+!63 = !DILocation(line: 7, column: 48, scope: !11, inlinedAt: !61)
+!64 = !DILocation(line: 29, column: 1, scope: !17)
diff --git a/test/DebugInfo/X86/objc-fwd-decl.ll b/test/DebugInfo/X86/objc-fwd-decl.ll
index 0d3d24898af..5b0aca5f92f 100644
--- a/test/DebugInfo/X86/objc-fwd-decl.ll
+++ b/test/DebugInfo/X86/objc-fwd-decl.ll
@@ -12,16 +12,16 @@
 !llvm.dbg.cu = !{!0}
 !llvm.module.flags = !{!9, !10, !11, !12, !14}
 
-!0 = !MDCompileUnit(language: DW_LANG_ObjC, producer: "clang version 3.1 (trunk 152054 trunk 152094)", isOptimized: false, runtimeVersion: 2, emissionKind: 0, file: !13, enums: !1, retainedTypes: !1, subprograms: !1, globals: !3, imports:  !1)
+!0 = !DICompileUnit(language: DW_LANG_ObjC, producer: "clang version 3.1 (trunk 152054 trunk 152094)", isOptimized: false, runtimeVersion: 2, emissionKind: 0, file: !13, enums: !1, retainedTypes: !1, subprograms: !1, globals: !3, imports:  !1)
 !1 = !{}
 !3 = !{!5}
-!5 = !MDGlobalVariable(name: "a", line: 3, isLocal: false, isDefinition: true, scope: null, file: !6, type: !7, variable: %0** @a)
-!6 = !MDFile(filename: "foo.m", directory: "/Users/echristo")
-!7 = !MDDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, baseType: !8)
-!8 = !MDCompositeType(tag: DW_TAG_structure_type, name: "FooBarBaz", line: 1, flags: DIFlagFwdDecl, runtimeLang: DW_LANG_ObjC, file: !13)
+!5 = !DIGlobalVariable(name: "a", line: 3, isLocal: false, isDefinition: true, scope: null, file: !6, type: !7, variable: %0** @a)
+!6 = !DIFile(filename: "foo.m", directory: "/Users/echristo")
+!7 = !DIDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, baseType: !8)
+!8 = !DICompositeType(tag: DW_TAG_structure_type, name: "FooBarBaz", line: 1, flags: DIFlagFwdDecl, runtimeLang: DW_LANG_ObjC, file: !13)
 !9 = !{i32 1, !"Objective-C Version", i32 2}
 !10 = !{i32 1, !"Objective-C Image Info Version", i32 0}
 !11 = !{i32 1, !"Objective-C Image Info Section", !"__DATA, __objc_imageinfo, regular, no_dead_strip"}
 !12 = !{i32 4, !"Objective-C Garbage Collection", i32 0}
-!13 = !MDFile(filename: "foo.m", directory: "/Users/echristo")
+!13 = !DIFile(filename: "foo.m", directory: "/Users/echristo")
 !14 = !{i32 1, !"Debug Info Version", i32 3}
diff --git a/test/DebugInfo/X86/objc-property-void.ll b/test/DebugInfo/X86/objc-property-void.ll
index 5744a7b122e..0737fe2d2cb 100644
--- a/test/DebugInfo/X86/objc-property-void.ll
+++ b/test/DebugInfo/X86/objc-property-void.ll
@@ -56,9 +56,9 @@ entry:
   %self.addr = alloca %0*, align 8
   %_cmd.addr = alloca i8*, align 8
   store %0* %self, %0** %self.addr, align 8
-  call void @llvm.dbg.declare(metadata %0** %self.addr, metadata !24, metadata !MDExpression()), !dbg !26
+  call void @llvm.dbg.declare(metadata %0** %self.addr, metadata !24, metadata !DIExpression()), !dbg !26
   store i8* %_cmd, i8** %_cmd.addr, align 8
-  call void @llvm.dbg.declare(metadata i8** %_cmd.addr, metadata !27, metadata !MDExpression()), !dbg !26
+  call void @llvm.dbg.declare(metadata i8** %_cmd.addr, metadata !27, metadata !DIExpression()), !dbg !26
   ret void, !dbg !29
 }
 
@@ -72,23 +72,23 @@ attributes #1 = { nounwind readnone }
 !llvm.module.flags = !{!17, !18, !19, !20, !21, !22}
 !llvm.ident = !{!23}
 
-!0 = !MDCompileUnit(language: DW_LANG_ObjC, isOptimized: false, runtimeVersion: 2, emissionKind: 0, file: !1, enums: !2, retainedTypes: !3, subprograms: !9, globals: !2, imports: !2)
-!1 = !MDFile(filename: "-", directory: "")
+!0 = !DICompileUnit(language: DW_LANG_ObjC, isOptimized: false, runtimeVersion: 2, emissionKind: 0, file: !1, enums: !2, retainedTypes: !3, subprograms: !9, globals: !2, imports: !2)
+!1 = !DIFile(filename: "-", directory: "")
 !2 = !{}
 !3 = !{!4}
-!4 = !MDCompositeType(tag: DW_TAG_structure_type, name: "Foo", line: 1, align: 8, flags: DIFlagObjcClassComplete, runtimeLang: DW_LANG_ObjC, file: !5, scope: !6, elements: !7)
-!5 = !MDFile(filename: "<stdin>", directory: "")
-!6 = !MDFile(filename: "<stdin>", directory: "")
+!4 = !DICompositeType(tag: DW_TAG_structure_type, name: "Foo", line: 1, align: 8, flags: DIFlagObjcClassComplete, runtimeLang: DW_LANG_ObjC, file: !5, scope: !6, elements: !7)
+!5 = !DIFile(filename: "<stdin>", directory: "")
+!6 = !DIFile(filename: "<stdin>", directory: "")
 !7 = !{!8}
-!8 = !MDObjCProperty(name: "foo", line: 2, attributes: 2117, file: !6)
+!8 = !DIObjCProperty(name: "foo", line: 2, attributes: 2117, file: !6)
 !9 = !{!10}
-!10 = !MDSubprogram(name: "-[Foo foo]", line: 5, isLocal: true, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 5, file: !5, scope: !6, type: !11, function: void (%0*, i8*)* @"\01-[Foo foo]", variables: !2)
-!11 = !MDSubroutineType(types: !12)
+!10 = !DISubprogram(name: "-[Foo foo]", line: 5, isLocal: true, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 5, file: !5, scope: !6, type: !11, function: void (%0*, i8*)* @"\01-[Foo foo]", variables: !2)
+!11 = !DISubroutineType(types: !12)
 !12 = !{null, !13, !14}
-!13 = !MDDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, flags: DIFlagArtificial | DIFlagObjectPointer, baseType: !4)
-!14 = !MDDerivedType(tag: DW_TAG_typedef, name: "SEL", line: 5, flags: DIFlagArtificial, file: !5, baseType: !15)
-!15 = !MDDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, baseType: !16)
-!16 = !MDCompositeType(tag: DW_TAG_structure_type, name: "objc_selector", flags: DIFlagFwdDecl, file: !1)
+!13 = !DIDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, flags: DIFlagArtificial | DIFlagObjectPointer, baseType: !4)
+!14 = !DIDerivedType(tag: DW_TAG_typedef, name: "SEL", line: 5, flags: DIFlagArtificial, file: !5, baseType: !15)
+!15 = !DIDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, baseType: !16)
+!16 = !DICompositeType(tag: DW_TAG_structure_type, name: "objc_selector", flags: DIFlagFwdDecl, file: !1)
 !17 = !{i32 1, !"Objective-C Version", i32 2}
 !18 = !{i32 1, !"Objective-C Image Info Version", i32 0}
 !19 = !{i32 1, !"Objective-C Image Info Section", !"__DATA, __objc_imageinfo, regular, no_dead_strip"}
@@ -96,9 +96,9 @@ attributes #1 = { nounwind readnone }
 !21 = !{i32 2, !"Dwarf Version", i32 2}
 !22 = !{i32 1, !"Debug Info Version", i32 3}
 !23 = !{!""}
-!24 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "self", arg: 1, flags: DIFlagArtificial | DIFlagObjectPointer, scope: !10, type: !25)
-!25 = !MDDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, baseType: !4)
-!26 = !MDLocation(line: 0, scope: !10)
-!27 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "_cmd", arg: 2, flags: DIFlagArtificial, scope: !10, type: !28)
-!28 = !MDDerivedType(tag: DW_TAG_typedef, name: "SEL", line: 5, file: !5, baseType: !15)
-!29 = !MDLocation(line: 5, scope: !10)
+!24 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "self", arg: 1, flags: DIFlagArtificial | DIFlagObjectPointer, scope: !10, type: !25)
+!25 = !DIDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, baseType: !4)
+!26 = !DILocation(line: 0, scope: !10)
+!27 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "_cmd", arg: 2, flags: DIFlagArtificial, scope: !10, type: !28)
+!28 = !DIDerivedType(tag: DW_TAG_typedef, name: "SEL", line: 5, file: !5, baseType: !15)
+!29 = !DILocation(line: 5, scope: !10)
diff --git a/test/DebugInfo/X86/op_deref.ll b/test/DebugInfo/X86/op_deref.ll
index 56b3aa5fe0a..86f629e9c15 100644
--- a/test/DebugInfo/X86/op_deref.ll
+++ b/test/DebugInfo/X86/op_deref.ll
@@ -24,7 +24,7 @@
 ; ASM-CHECK: DW_OP_breg2
 
 ; RUN: llvm-as %s -o - | llvm-dis - | FileCheck %s --check-prefix=PRETTY-PRINT
-; PRETTY-PRINT: MDExpression(DW_OP_deref, DW_OP_deref)
+; PRETTY-PRINT: DIExpression(DW_OP_deref, DW_OP_deref)
 
 define void @testVLAwithSize(i32 %s) nounwind uwtable ssp {
 entry:
@@ -32,14 +32,14 @@ entry:
   %saved_stack = alloca i8*
   %i = alloca i32, align 4
   store i32 %s, i32* %s.addr, align 4
-  call void @llvm.dbg.declare(metadata i32* %s.addr, metadata !10, metadata !MDExpression()), !dbg !11
+  call void @llvm.dbg.declare(metadata i32* %s.addr, metadata !10, metadata !DIExpression()), !dbg !11
   %0 = load i32, i32* %s.addr, align 4, !dbg !12
   %1 = zext i32 %0 to i64, !dbg !12
   %2 = call i8* @llvm.stacksave(), !dbg !12
   store i8* %2, i8** %saved_stack, !dbg !12
   %vla = alloca i32, i64 %1, align 16, !dbg !12
   call void @llvm.dbg.declare(metadata i32* %vla, metadata !14, metadata !30), !dbg !18
-  call void @llvm.dbg.declare(metadata i32* %i, metadata !19, metadata !MDExpression()), !dbg !20
+  call void @llvm.dbg.declare(metadata i32* %i, metadata !19, metadata !DIExpression()), !dbg !20
   store i32 0, i32* %i, align 4, !dbg !21
   br label %for.cond, !dbg !21
 
@@ -80,32 +80,32 @@ declare void @llvm.stackrestore(i8*) nounwind
 !llvm.dbg.cu = !{!0}
 !llvm.module.flags = !{!29}
 
-!0 = !MDCompileUnit(language: DW_LANG_C99, producer: "clang version 3.2 (trunk 156005) (llvm/trunk 156000)", isOptimized: false, emissionKind: 1, file: !28, enums: !1, retainedTypes: !1, subprograms: !3, globals: !1, imports:  !1)
+!0 = !DICompileUnit(language: DW_LANG_C99, producer: "clang version 3.2 (trunk 156005) (llvm/trunk 156000)", isOptimized: false, emissionKind: 1, file: !28, enums: !1, retainedTypes: !1, subprograms: !3, globals: !1, imports:  !1)
 !1 = !{}
 !3 = !{!5}
-!5 = !MDSubprogram(name: "testVLAwithSize", line: 1, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 2, file: !28, scope: !6, type: !7, function: void (i32)* @testVLAwithSize, variables: !1)
-!6 = !MDFile(filename: "bar.c", directory: "/Users/echristo/tmp")
-!7 = !MDSubroutineType(types: !8)
+!5 = !DISubprogram(name: "testVLAwithSize", line: 1, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 2, file: !28, scope: !6, type: !7, function: void (i32)* @testVLAwithSize, variables: !1)
+!6 = !DIFile(filename: "bar.c", directory: "/Users/echristo/tmp")
+!7 = !DISubroutineType(types: !8)
 !8 = !{null, !9}
-!9 = !MDBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
-!10 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "s", line: 1, arg: 1, scope: !5, file: !6, type: !9)
-!11 = !MDLocation(line: 1, column: 26, scope: !5)
-!12 = !MDLocation(line: 3, column: 13, scope: !13)
-!13 = distinct !MDLexicalBlock(line: 2, column: 1, file: !28, scope: !5)
-!14 = !MDLocalVariable(tag: DW_TAG_auto_variable, name: "vla", line: 3, scope: !13, file: !6, type: !15)
-!15 = !MDCompositeType(tag: DW_TAG_array_type, align: 32, baseType: !9, elements: !16)
+!9 = !DIBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
+!10 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "s", line: 1, arg: 1, scope: !5, file: !6, type: !9)
+!11 = !DILocation(line: 1, column: 26, scope: !5)
+!12 = !DILocation(line: 3, column: 13, scope: !13)
+!13 = distinct !DILexicalBlock(line: 2, column: 1, file: !28, scope: !5)
+!14 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "vla", line: 3, scope: !13, file: !6, type: !15)
+!15 = !DICompositeType(tag: DW_TAG_array_type, align: 32, baseType: !9, elements: !16)
 !16 = !{!17}
-!17 = !MDSubrange(count: -1)
-!18 = !MDLocation(line: 3, column: 7, scope: !13)
-!19 = !MDLocalVariable(tag: DW_TAG_auto_variable, name: "i", line: 4, scope: !13, file: !6, type: !9)
-!20 = !MDLocation(line: 4, column: 7, scope: !13)
-!21 = !MDLocation(line: 5, column: 8, scope: !22)
-!22 = distinct !MDLexicalBlock(line: 5, column: 3, file: !28, scope: !13)
-!23 = !MDLocation(line: 6, column: 5, scope: !24)
-!24 = distinct !MDLexicalBlock(line: 5, column: 27, file: !28, scope: !22)
-!25 = !MDLocation(line: 7, column: 3, scope: !24)
-!26 = !MDLocation(line: 5, column: 22, scope: !22)
-!27 = !MDLocation(line: 8, column: 1, scope: !13)
-!28 = !MDFile(filename: "bar.c", directory: "/Users/echristo/tmp")
+!17 = !DISubrange(count: -1)
+!18 = !DILocation(line: 3, column: 7, scope: !13)
+!19 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "i", line: 4, scope: !13, file: !6, type: !9)
+!20 = !DILocation(line: 4, column: 7, scope: !13)
+!21 = !DILocation(line: 5, column: 8, scope: !22)
+!22 = distinct !DILexicalBlock(line: 5, column: 3, file: !28, scope: !13)
+!23 = !DILocation(line: 6, column: 5, scope: !24)
+!24 = distinct !DILexicalBlock(line: 5, column: 27, file: !28, scope: !22)
+!25 = !DILocation(line: 7, column: 3, scope: !24)
+!26 = !DILocation(line: 5, column: 22, scope: !22)
+!27 = !DILocation(line: 8, column: 1, scope: !13)
+!28 = !DIFile(filename: "bar.c", directory: "/Users/echristo/tmp")
 !29 = !{i32 1, !"Debug Info Version", i32 3}
-!30 = !MDExpression(DW_OP_deref, DW_OP_deref)
+!30 = !DIExpression(DW_OP_deref, DW_OP_deref)
diff --git a/test/DebugInfo/X86/parameters.ll b/test/DebugInfo/X86/parameters.ll
index 419732f6d80..044f51e6e6b 100644
--- a/test/DebugInfo/X86/parameters.ll
+++ b/test/DebugInfo/X86/parameters.ll
@@ -42,7 +42,7 @@
 ; Function Attrs: uwtable
 define void @_ZN7pr147634funcENS_3fooE(%"struct.pr14763::foo"* noalias sret %agg.result, %"struct.pr14763::foo"* %f) #0 {
 entry:
-  call void @llvm.dbg.declare(metadata %"struct.pr14763::foo"* %f, metadata !22, metadata !MDExpression(DW_OP_deref)), !dbg !24
+  call void @llvm.dbg.declare(metadata %"struct.pr14763::foo"* %f, metadata !22, metadata !DIExpression(DW_OP_deref)), !dbg !24
   call void @_ZN7pr147633fooC1ERKS0_(%"struct.pr14763::foo"* %agg.result, %"struct.pr14763::foo"* %f), !dbg !25
   ret void, !dbg !25
 }
@@ -58,8 +58,8 @@ entry:
   %b.addr = alloca i8, align 1
   %frombool = zext i1 %b to i8
   store i8 %frombool, i8* %b.addr, align 1
-  call void @llvm.dbg.declare(metadata i8* %b.addr, metadata !26, metadata !MDExpression()), !dbg !27
-  call void @llvm.dbg.declare(metadata %"struct.pr14763::foo"* %g, metadata !28, metadata !MDExpression(DW_OP_deref)), !dbg !27
+  call void @llvm.dbg.declare(metadata i8* %b.addr, metadata !26, metadata !DIExpression()), !dbg !27
+  call void @llvm.dbg.declare(metadata %"struct.pr14763::foo"* %g, metadata !28, metadata !DIExpression(DW_OP_deref)), !dbg !27
   %0 = load i8, i8* %b.addr, align 1, !dbg !29
   %tobool = trunc i8 %0 to i1, !dbg !29
   br i1 %tobool, label %if.then, label %if.end, !dbg !29
@@ -82,36 +82,36 @@ attributes #2 = { "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "n
 !llvm.dbg.cu = !{!0}
 !llvm.module.flags = !{!21, !33}
 
-!0 = !MDCompileUnit(language: DW_LANG_C_plus_plus, producer: "clang version 3.4 ", isOptimized: false, emissionKind: 1, file: !1, enums: !2, retainedTypes: !2, subprograms: !3, globals: !2, imports: !2)
-!1 = !MDFile(filename: "pass.cpp", directory: "/tmp")
+!0 = !DICompileUnit(language: DW_LANG_C_plus_plus, producer: "clang version 3.4 ", isOptimized: false, emissionKind: 1, file: !1, enums: !2, retainedTypes: !2, subprograms: !3, globals: !2, imports: !2)
+!1 = !DIFile(filename: "pass.cpp", directory: "/tmp")
 !2 = !{}
 !3 = !{!4, !17}
-!4 = !MDSubprogram(name: "func", linkageName: "_ZN7pr147634funcENS_3fooE", line: 6, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 6, file: !1, scope: !5, type: !6, function: void (%"struct.pr14763::foo"*, %"struct.pr14763::foo"*)* @_ZN7pr147634funcENS_3fooE, variables: !2)
-!5 = !MDNamespace(name: "pr14763", line: 1, file: !1, scope: null)
-!6 = !MDSubroutineType(types: !7)
+!4 = !DISubprogram(name: "func", linkageName: "_ZN7pr147634funcENS_3fooE", line: 6, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 6, file: !1, scope: !5, type: !6, function: void (%"struct.pr14763::foo"*, %"struct.pr14763::foo"*)* @_ZN7pr147634funcENS_3fooE, variables: !2)
+!5 = !DINamespace(name: "pr14763", line: 1, file: !1, scope: null)
+!6 = !DISubroutineType(types: !7)
 !7 = !{!8, !8}
-!8 = !MDCompositeType(tag: DW_TAG_structure_type, name: "foo", line: 2, size: 8, align: 8, file: !1, scope: !5, elements: !9)
+!8 = !DICompositeType(tag: DW_TAG_structure_type, name: "foo", line: 2, size: 8, align: 8, file: !1, scope: !5, elements: !9)
 !9 = !{!10}
-!10 = !MDSubprogram(name: "foo", line: 3, isLocal: false, isDefinition: false, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 3, file: !1, scope: !8, type: !11)
-!11 = !MDSubroutineType(types: !12)
+!10 = !DISubprogram(name: "foo", line: 3, isLocal: false, isDefinition: false, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 3, file: !1, scope: !8, type: !11)
+!11 = !DISubroutineType(types: !12)
 !12 = !{null, !13, !14}
-!13 = !MDDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, flags: DIFlagArtificial | DIFlagObjectPointer, baseType: !8)
-!14 = !MDDerivedType(tag: DW_TAG_reference_type, baseType: !15)
-!15 = !MDDerivedType(tag: DW_TAG_const_type, baseType: !8)
-!17 = !MDSubprogram(name: "func2", linkageName: "_ZN7pr147635func2EbNS_3fooE", line: 12, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 12, file: !1, scope: !5, type: !18, function: void (i1, %"struct.pr14763::foo"*)* @_ZN7pr147635func2EbNS_3fooE, variables: !2)
-!18 = !MDSubroutineType(types: !19)
+!13 = !DIDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, flags: DIFlagArtificial | DIFlagObjectPointer, baseType: !8)
+!14 = !DIDerivedType(tag: DW_TAG_reference_type, baseType: !15)
+!15 = !DIDerivedType(tag: DW_TAG_const_type, baseType: !8)
+!17 = !DISubprogram(name: "func2", linkageName: "_ZN7pr147635func2EbNS_3fooE", line: 12, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 12, file: !1, scope: !5, type: !18, function: void (i1, %"struct.pr14763::foo"*)* @_ZN7pr147635func2EbNS_3fooE, variables: !2)
+!18 = !DISubroutineType(types: !19)
 !19 = !{null, !20, !8}
-!20 = !MDBasicType(tag: DW_TAG_base_type, name: "bool", size: 8, align: 8, encoding: DW_ATE_boolean)
+!20 = !DIBasicType(tag: DW_TAG_base_type, name: "bool", size: 8, align: 8, encoding: DW_ATE_boolean)
 !21 = !{i32 2, !"Dwarf Version", i32 3}
-!22 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "f", line: 6, arg: 1, scope: !4, file: !23, type: !8)
-!23 = !MDFile(filename: "pass.cpp", directory: "/tmp")
-!24 = !MDLocation(line: 6, scope: !4)
-!25 = !MDLocation(line: 7, scope: !4)
-!26 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "b", line: 12, arg: 1, scope: !17, file: !23, type: !20)
-!27 = !MDLocation(line: 12, scope: !17)
-!28 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "g", line: 12, arg: 2, scope: !17, file: !23, type: !8)
-!29 = !MDLocation(line: 13, scope: !30)
-!30 = distinct !MDLexicalBlock(line: 13, column: 0, file: !1, scope: !17)
-!31 = !MDLocation(line: 14, scope: !30)
-!32 = !MDLocation(line: 15, scope: !17)
+!22 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "f", line: 6, arg: 1, scope: !4, file: !23, type: !8)
+!23 = !DIFile(filename: "pass.cpp", directory: "/tmp")
+!24 = !DILocation(line: 6, scope: !4)
+!25 = !DILocation(line: 7, scope: !4)
+!26 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "b", line: 12, arg: 1, scope: !17, file: !23, type: !20)
+!27 = !DILocation(line: 12, scope: !17)
+!28 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "g", line: 12, arg: 2, scope: !17, file: !23, type: !8)
+!29 = !DILocation(line: 13, scope: !30)
+!30 = distinct !DILexicalBlock(line: 13, column: 0, file: !1, scope: !17)
+!31 = !DILocation(line: 14, scope: !30)
+!32 = !DILocation(line: 15, scope: !17)
 !33 = !{i32 1, !"Debug Info Version", i32 3}
diff --git a/test/DebugInfo/X86/pieces-1.ll b/test/DebugInfo/X86/pieces-1.ll
index 290efbefea4..f96e045f828 100644
--- a/test/DebugInfo/X86/pieces-1.ll
+++ b/test/DebugInfo/X86/pieces-1.ll
@@ -50,30 +50,30 @@ attributes #1 = { nounwind readnone }
 !llvm.module.flags = !{!17, !18}
 !llvm.ident = !{!19}
 
-!0 = !MDCompileUnit(language: DW_LANG_C99, producer: "clang version 3.5 ", isOptimized: true, emissionKind: 1, file: !1, enums: !2, retainedTypes: !2, subprograms: !3, globals: !2, imports: !2)
-!1 = !MDFile(filename: "pieces.c", directory: "")
+!0 = !DICompileUnit(language: DW_LANG_C99, producer: "clang version 3.5 ", isOptimized: true, emissionKind: 1, file: !1, enums: !2, retainedTypes: !2, subprograms: !3, globals: !2, imports: !2)
+!1 = !DIFile(filename: "pieces.c", directory: "")
 !2 = !{}
 !3 = !{!4}
-!4 = !MDSubprogram(name: "foo", line: 3, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: true, scopeLine: 3, file: !1, scope: !5, type: !6, function: i32 (i64, i32)* @foo, variables: !15)
-!5 = !MDFile(filename: "pieces.c", directory: "")
-!6 = !MDSubroutineType(types: !7)
+!4 = !DISubprogram(name: "foo", line: 3, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: true, scopeLine: 3, file: !1, scope: !5, type: !6, function: i32 (i64, i32)* @foo, variables: !15)
+!5 = !DIFile(filename: "pieces.c", directory: "")
+!6 = !DISubroutineType(types: !7)
 !7 = !{!8, !9}
-!8 = !MDBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
-!9 = !MDDerivedType(tag: DW_TAG_typedef, name: "S", line: 1, file: !1, baseType: !10)
-!10 = !MDCompositeType(tag: DW_TAG_structure_type, line: 1, size: 128, align: 64, file: !1, elements: !11)
+!8 = !DIBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
+!9 = !DIDerivedType(tag: DW_TAG_typedef, name: "S", line: 1, file: !1, baseType: !10)
+!10 = !DICompositeType(tag: DW_TAG_structure_type, line: 1, size: 128, align: 64, file: !1, elements: !11)
 !11 = !{!12, !14}
-!12 = !MDDerivedType(tag: DW_TAG_member, name: "a", line: 1, size: 64, align: 64, file: !1, scope: !10, baseType: !13)
-!13 = !MDBasicType(tag: DW_TAG_base_type, name: "long int", size: 64, align: 64, encoding: DW_ATE_signed)
-!14 = !MDDerivedType(tag: DW_TAG_member, name: "b", line: 1, size: 32, align: 32, offset: 64, file: !1, scope: !10, baseType: !8)
+!12 = !DIDerivedType(tag: DW_TAG_member, name: "a", line: 1, size: 64, align: 64, file: !1, scope: !10, baseType: !13)
+!13 = !DIBasicType(tag: DW_TAG_base_type, name: "long int", size: 64, align: 64, encoding: DW_ATE_signed)
+!14 = !DIDerivedType(tag: DW_TAG_member, name: "b", line: 1, size: 32, align: 32, offset: 64, file: !1, scope: !10, baseType: !8)
 !15 = !{!16}
-!16 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "s", line: 3, arg: 1, scope: !4, file: !5, type: !9)
+!16 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "s", line: 3, arg: 1, scope: !4, file: !5, type: !9)
 !17 = !{i32 2, !"Dwarf Version", i32 4}
 !18 = !{i32 1, !"Debug Info Version", i32 3}
 !19 = !{!"clang version 3.5 "}
-!20 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "s", line: 3, arg: 1, scope: !4, file: !5, type: !9)
-!21 = !MDLocation(line: 3, scope: !4)
-!22 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "s", line: 3, arg: 1, scope: !4, file: !5, type: !9)
-!23 = !MDLocation(line: 4, scope: !4)
-!24 = !MDExpression(DW_OP_bit_piece, 0, 64)
+!20 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "s", line: 3, arg: 1, scope: !4, file: !5, type: !9)
+!21 = !DILocation(line: 3, scope: !4)
+!22 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "s", line: 3, arg: 1, scope: !4, file: !5, type: !9)
+!23 = !DILocation(line: 4, scope: !4)
+!24 = !DIExpression(DW_OP_bit_piece, 0, 64)
 !25 = !{}
-!27 = !MDExpression(DW_OP_bit_piece, 64, 32)
+!27 = !DIExpression(DW_OP_bit_piece, 64, 32)
diff --git a/test/DebugInfo/X86/pieces-2.ll b/test/DebugInfo/X86/pieces-2.ll
index 8df96663b91..7c39d323c3e 100644
--- a/test/DebugInfo/X86/pieces-2.ll
+++ b/test/DebugInfo/X86/pieces-2.ll
@@ -31,7 +31,7 @@ target triple = "x86_64-apple-macosx10.9.0"
 ; Function Attrs: nounwind ssp uwtable
 define i32 @foo(%struct.Outer* byval align 8 %outer) #0 {
 entry:
-  call void @llvm.dbg.declare(metadata %struct.Outer* %outer, metadata !25, metadata !MDExpression()), !dbg !26
+  call void @llvm.dbg.declare(metadata %struct.Outer* %outer, metadata !25, metadata !DIExpression()), !dbg !26
   %i1.sroa.0.0..sroa_idx = getelementptr inbounds %struct.Outer, %struct.Outer* %outer, i64 0, i32 0, i64 1, i32 0, !dbg !27
   %i1.sroa.0.0.copyload = load i32, i32* %i1.sroa.0.0..sroa_idx, align 8, !dbg !27
   call void @llvm.dbg.value(metadata i32 %i1.sroa.0.0.copyload, i64 0, metadata !28, metadata !29), !dbg !27
@@ -57,35 +57,35 @@ attributes #2 = { nounwind }
 !llvm.module.flags = !{!22, !23}
 !llvm.ident = !{!24}
 
-!0 = !MDCompileUnit(language: DW_LANG_C99, producer: "clang version 3.5.0 ", isOptimized: false, emissionKind: 1, file: !1, enums: !2, retainedTypes: !2, subprograms: !3, globals: !2, imports: !2)
-!1 = !MDFile(filename: "sroasplit-1.c", directory: "")
+!0 = !DICompileUnit(language: DW_LANG_C99, producer: "clang version 3.5.0 ", isOptimized: false, emissionKind: 1, file: !1, enums: !2, retainedTypes: !2, subprograms: !3, globals: !2, imports: !2)
+!1 = !DIFile(filename: "sroasplit-1.c", directory: "")
 !2 = !{}
 !3 = !{!4}
-!4 = !MDSubprogram(name: "foo", line: 10, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 10, file: !1, scope: !5, type: !6, function: i32 (%struct.Outer*)* @foo, variables: !2)
-!5 = !MDFile(filename: "sroasplit-1.c", directory: "")
-!6 = !MDSubroutineType(types: !7)
+!4 = !DISubprogram(name: "foo", line: 10, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 10, file: !1, scope: !5, type: !6, function: i32 (%struct.Outer*)* @foo, variables: !2)
+!5 = !DIFile(filename: "sroasplit-1.c", directory: "")
+!6 = !DISubroutineType(types: !7)
 !7 = !{!8, !9}
-!8 = !MDBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
-!9 = !MDDerivedType(tag: DW_TAG_typedef, name: "Outer", line: 8, file: !1, baseType: !10)
-!10 = !MDCompositeType(tag: DW_TAG_structure_type, line: 6, size: 256, align: 64, file: !1, elements: !11)
+!8 = !DIBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
+!9 = !DIDerivedType(tag: DW_TAG_typedef, name: "Outer", line: 8, file: !1, baseType: !10)
+!10 = !DICompositeType(tag: DW_TAG_structure_type, line: 6, size: 256, align: 64, file: !1, elements: !11)
 !11 = !{!12}
-!12 = !MDDerivedType(tag: DW_TAG_member, name: "inner", line: 7, size: 256, align: 64, file: !1, scope: !10, baseType: !13)
-!13 = !MDCompositeType(tag: DW_TAG_array_type, size: 256, align: 64, baseType: !14, elements: !20)
-!14 = !MDDerivedType(tag: DW_TAG_typedef, name: "Inner", line: 4, file: !1, baseType: !15)
-!15 = !MDCompositeType(tag: DW_TAG_structure_type, line: 1, size: 128, align: 64, file: !1, elements: !16)
+!12 = !DIDerivedType(tag: DW_TAG_member, name: "inner", line: 7, size: 256, align: 64, file: !1, scope: !10, baseType: !13)
+!13 = !DICompositeType(tag: DW_TAG_array_type, size: 256, align: 64, baseType: !14, elements: !20)
+!14 = !DIDerivedType(tag: DW_TAG_typedef, name: "Inner", line: 4, file: !1, baseType: !15)
+!15 = !DICompositeType(tag: DW_TAG_structure_type, line: 1, size: 128, align: 64, file: !1, elements: !16)
 !16 = !{!17, !18}
-!17 = !MDDerivedType(tag: DW_TAG_member, name: "a", line: 2, size: 32, align: 32, file: !1, scope: !15, baseType: !8)
-!18 = !MDDerivedType(tag: DW_TAG_member, name: "b", line: 3, size: 64, align: 64, offset: 64, file: !1, scope: !15, baseType: !19)
-!19 = !MDBasicType(tag: DW_TAG_base_type, name: "long int", size: 64, align: 64, encoding: DW_ATE_signed)
+!17 = !DIDerivedType(tag: DW_TAG_member, name: "a", line: 2, size: 32, align: 32, file: !1, scope: !15, baseType: !8)
+!18 = !DIDerivedType(tag: DW_TAG_member, name: "b", line: 3, size: 64, align: 64, offset: 64, file: !1, scope: !15, baseType: !19)
+!19 = !DIBasicType(tag: DW_TAG_base_type, name: "long int", size: 64, align: 64, encoding: DW_ATE_signed)
 !20 = !{!21}
-!21 = !MDSubrange(count: 2)
+!21 = !DISubrange(count: 2)
 !22 = !{i32 2, !"Dwarf Version", i32 2}
 !23 = !{i32 1, !"Debug Info Version", i32 3}
 !24 = !{!"clang version 3.5.0 "}
-!25 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "outer", line: 10, arg: 1, scope: !4, file: !5, type: !9)
-!26 = !MDLocation(line: 10, scope: !4)
-!27 = !MDLocation(line: 11, scope: !4)
-!28 = !MDLocalVariable(tag: DW_TAG_auto_variable, name: "i1", line: 11, scope: !4, file: !5, type: !14)
-!29 = !MDExpression(DW_OP_bit_piece, 0, 32)
+!25 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "outer", line: 10, arg: 1, scope: !4, file: !5, type: !9)
+!26 = !DILocation(line: 10, scope: !4)
+!27 = !DILocation(line: 11, scope: !4)
+!28 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "i1", line: 11, scope: !4, file: !5, type: !14)
+!29 = !DIExpression(DW_OP_bit_piece, 0, 32)
 !31 = !{i32 3, i32 0, i32 12}
-!32 = !MDLocation(line: 12, scope: !4)
+!32 = !DILocation(line: 12, scope: !4)
diff --git a/test/DebugInfo/X86/pieces-3.ll b/test/DebugInfo/X86/pieces-3.ll
index b85b766d992..97ff9aadfcf 100644
--- a/test/DebugInfo/X86/pieces-3.ll
+++ b/test/DebugInfo/X86/pieces-3.ll
@@ -67,40 +67,40 @@ attributes #2 = { nounwind }
 !llvm.module.flags = !{!21, !22}
 !llvm.ident = !{!23}
 
-!0 = !MDCompileUnit(language: DW_LANG_C99, producer: "clang version 3.5.0 ", isOptimized: false, emissionKind: 1, file: !1, enums: !2, retainedTypes: !2, subprograms: !3, globals: !2, imports: !2)
-!1 = !MDFile(filename: "sroasplit-2.c", directory: "")
+!0 = !DICompileUnit(language: DW_LANG_C99, producer: "clang version 3.5.0 ", isOptimized: false, emissionKind: 1, file: !1, enums: !2, retainedTypes: !2, subprograms: !3, globals: !2, imports: !2)
+!1 = !DIFile(filename: "sroasplit-2.c", directory: "")
 !2 = !{}
 !3 = !{!4}
-!4 = !MDSubprogram(name: "foo", line: 10, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 10, file: !1, scope: !5, type: !6, function: i32 (i64, i64)* @foo, variables: !2)
-!5 = !MDFile(filename: "sroasplit-2.c", directory: "")
-!6 = !MDSubroutineType(types: !7)
+!4 = !DISubprogram(name: "foo", line: 10, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 10, file: !1, scope: !5, type: !6, function: i32 (i64, i64)* @foo, variables: !2)
+!5 = !DIFile(filename: "sroasplit-2.c", directory: "")
+!6 = !DISubroutineType(types: !7)
 !7 = !{!8, !9}
-!8 = !MDBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
-!9 = !MDDerivedType(tag: DW_TAG_typedef, name: "Outer", line: 8, file: !1, baseType: !10)
-!10 = !MDCompositeType(tag: DW_TAG_structure_type, line: 6, size: 128, align: 32, file: !1, elements: !11)
+!8 = !DIBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
+!9 = !DIDerivedType(tag: DW_TAG_typedef, name: "Outer", line: 8, file: !1, baseType: !10)
+!10 = !DICompositeType(tag: DW_TAG_structure_type, line: 6, size: 128, align: 32, file: !1, elements: !11)
 !11 = !{!12}
-!12 = !MDDerivedType(tag: DW_TAG_member, name: "inner", line: 7, size: 128, align: 32, file: !1, scope: !10, baseType: !13)
-!13 = !MDCompositeType(tag: DW_TAG_array_type, size: 128, align: 32, baseType: !14, elements: !19)
-!14 = !MDDerivedType(tag: DW_TAG_typedef, name: "Inner", line: 4, file: !1, baseType: !15)
-!15 = !MDCompositeType(tag: DW_TAG_structure_type, line: 1, size: 64, align: 32, file: !1, elements: !16)
+!12 = !DIDerivedType(tag: DW_TAG_member, name: "inner", line: 7, size: 128, align: 32, file: !1, scope: !10, baseType: !13)
+!13 = !DICompositeType(tag: DW_TAG_array_type, size: 128, align: 32, baseType: !14, elements: !19)
+!14 = !DIDerivedType(tag: DW_TAG_typedef, name: "Inner", line: 4, file: !1, baseType: !15)
+!15 = !DICompositeType(tag: DW_TAG_structure_type, line: 1, size: 64, align: 32, file: !1, elements: !16)
 !16 = !{!17, !18}
-!17 = !MDDerivedType(tag: DW_TAG_member, name: "a", line: 2, size: 32, align: 32, file: !1, scope: !15, baseType: !8)
-!18 = !MDDerivedType(tag: DW_TAG_member, name: "b", line: 3, size: 32, align: 32, offset: 32, file: !1, scope: !15, baseType: !8)
+!17 = !DIDerivedType(tag: DW_TAG_member, name: "a", line: 2, size: 32, align: 32, file: !1, scope: !15, baseType: !8)
+!18 = !DIDerivedType(tag: DW_TAG_member, name: "b", line: 3, size: 32, align: 32, offset: 32, file: !1, scope: !15, baseType: !8)
 !19 = !{!20}
-!20 = !MDSubrange(count: 2)
+!20 = !DISubrange(count: 2)
 !21 = !{i32 2, !"Dwarf Version", i32 2}
 !22 = !{i32 1, !"Debug Info Version", i32 3}
 !23 = !{!"clang version 3.5.0 "}
-!24 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "outer", line: 10, arg: 1, scope: !4, file: !5, type: !9)
-!25 = !MDExpression(DW_OP_bit_piece, 0, 64)
-!26 = !MDLocation(line: 10, scope: !4)
-!27 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "outer", line: 10, arg: 1, scope: !4, file: !5, type: !9)
-!28 = !MDExpression(DW_OP_bit_piece, 64, 64)
-!29 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "outer", line: 10, arg: 1, scope: !4, file: !5, type: !9)
-!30 = !MDExpression(DW_OP_bit_piece, 96, 32)
-!31 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "outer", line: 10, arg: 1, scope: !4, file: !5, type: !9)
-!32 = !MDExpression(DW_OP_bit_piece, 64, 32)
-!33 = !MDLocation(line: 11, scope: !4)
-!34 = !MDLocalVariable(tag: DW_TAG_auto_variable, name: "i1", line: 11, scope: !4, file: !5, type: !14)
-!35 = !MDExpression(DW_OP_bit_piece, 0, 32)
-!36 = !MDLocation(line: 12, scope: !4)
+!24 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "outer", line: 10, arg: 1, scope: !4, file: !5, type: !9)
+!25 = !DIExpression(DW_OP_bit_piece, 0, 64)
+!26 = !DILocation(line: 10, scope: !4)
+!27 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "outer", line: 10, arg: 1, scope: !4, file: !5, type: !9)
+!28 = !DIExpression(DW_OP_bit_piece, 64, 64)
+!29 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "outer", line: 10, arg: 1, scope: !4, file: !5, type: !9)
+!30 = !DIExpression(DW_OP_bit_piece, 96, 32)
+!31 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "outer", line: 10, arg: 1, scope: !4, file: !5, type: !9)
+!32 = !DIExpression(DW_OP_bit_piece, 64, 32)
+!33 = !DILocation(line: 11, scope: !4)
+!34 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "i1", line: 11, scope: !4, file: !5, type: !14)
+!35 = !DIExpression(DW_OP_bit_piece, 0, 32)
+!36 = !DILocation(line: 12, scope: !4)
diff --git a/test/DebugInfo/X86/pointer-type-size.ll b/test/DebugInfo/X86/pointer-type-size.ll
index 351bccfbb4e..f5abdfe8925 100644
--- a/test/DebugInfo/X86/pointer-type-size.ll
+++ b/test/DebugInfo/X86/pointer-type-size.ll
@@ -11,16 +11,16 @@
 !llvm.dbg.cu = !{!0}
 !llvm.module.flags = !{!14}
 
-!0 = !MDCompileUnit(language: DW_LANG_C99, producer: "clang version 3.1 (trunk 147882)", isOptimized: false, emissionKind: 0, file: !13, enums: !1, retainedTypes: !1, subprograms: !1, globals: !3, imports:  !1)
+!0 = !DICompileUnit(language: DW_LANG_C99, producer: "clang version 3.1 (trunk 147882)", isOptimized: false, emissionKind: 0, file: !13, enums: !1, retainedTypes: !1, subprograms: !1, globals: !3, imports:  !1)
 !1 = !{}
 !3 = !{!5}
-!5 = !MDGlobalVariable(name: "crass", line: 1, isLocal: false, isDefinition: true, scope: null, file: !6, type: !7, variable: %struct.crass* @crass)
-!6 = !MDFile(filename: "foo.c", directory: "/Users/echristo/tmp")
-!7 = !MDCompositeType(tag: DW_TAG_structure_type, name: "crass", line: 1, size: 64, align: 64, file: !13, elements: !8)
+!5 = !DIGlobalVariable(name: "crass", line: 1, isLocal: false, isDefinition: true, scope: null, file: !6, type: !7, variable: %struct.crass* @crass)
+!6 = !DIFile(filename: "foo.c", directory: "/Users/echristo/tmp")
+!7 = !DICompositeType(tag: DW_TAG_structure_type, name: "crass", line: 1, size: 64, align: 64, file: !13, elements: !8)
 !8 = !{!9}
-!9 = !MDDerivedType(tag: DW_TAG_member, name: "ptr", line: 1, size: 64, align: 64, file: !13, scope: !7, baseType: !10)
-!10 = !MDDerivedType(tag: DW_TAG_const_type, baseType: !11)
-!11 = !MDDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, baseType: !12)
-!12 = !MDBasicType(tag: DW_TAG_base_type, name: "char", size: 8, align: 8, encoding: DW_ATE_signed_char)
-!13 = !MDFile(filename: "foo.c", directory: "/Users/echristo/tmp")
+!9 = !DIDerivedType(tag: DW_TAG_member, name: "ptr", line: 1, size: 64, align: 64, file: !13, scope: !7, baseType: !10)
+!10 = !DIDerivedType(tag: DW_TAG_const_type, baseType: !11)
+!11 = !DIDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, baseType: !12)
+!12 = !DIBasicType(tag: DW_TAG_base_type, name: "char", size: 8, align: 8, encoding: DW_ATE_signed_char)
+!13 = !DIFile(filename: "foo.c", directory: "/Users/echristo/tmp")
 !14 = !{i32 1, !"Debug Info Version", i32 3}
diff --git a/test/DebugInfo/X86/pr11300.ll b/test/DebugInfo/X86/pr11300.ll
index aff26f9ef20..b68609dd718 100644
--- a/test/DebugInfo/X86/pr11300.ll
+++ b/test/DebugInfo/X86/pr11300.ll
@@ -18,7 +18,7 @@ define void @_Z3zedP3foo(%struct.foo* %x) uwtable {
 entry:
   %x.addr = alloca %struct.foo*, align 8
   store %struct.foo* %x, %struct.foo** %x.addr, align 8
-  call void @llvm.dbg.declare(metadata %struct.foo** %x.addr, metadata !23, metadata !MDExpression()), !dbg !24
+  call void @llvm.dbg.declare(metadata %struct.foo** %x.addr, metadata !23, metadata !DIExpression()), !dbg !24
   %0 = load %struct.foo*, %struct.foo** %x.addr, align 8, !dbg !25
   call void @_ZN3foo3barEv(%struct.foo* %0), !dbg !25
   ret void, !dbg !27
@@ -30,7 +30,7 @@ define linkonce_odr void @_ZN3foo3barEv(%struct.foo* %this) nounwind uwtable ali
 entry:
   %this.addr = alloca %struct.foo*, align 8
   store %struct.foo* %this, %struct.foo** %this.addr, align 8
-  call void @llvm.dbg.declare(metadata %struct.foo** %this.addr, metadata !28, metadata !MDExpression()), !dbg !29
+  call void @llvm.dbg.declare(metadata %struct.foo** %this.addr, metadata !28, metadata !DIExpression()), !dbg !29
   %this1 = load %struct.foo*, %struct.foo** %this.addr
   ret void, !dbg !30
 }
@@ -38,29 +38,29 @@ entry:
 !llvm.dbg.cu = !{!0}
 !llvm.module.flags = !{!33}
 
-!0 = !MDCompileUnit(language: DW_LANG_C_plus_plus, producer: "clang version 3.0 ()", isOptimized: false, emissionKind: 0, file: !32, enums: !1, retainedTypes: !1, subprograms: !3, globals: !1, imports:  !1)
+!0 = !DICompileUnit(language: DW_LANG_C_plus_plus, producer: "clang version 3.0 ()", isOptimized: false, emissionKind: 0, file: !32, enums: !1, retainedTypes: !1, subprograms: !3, globals: !1, imports:  !1)
 !1 = !{}
 !3 = !{!5, !20}
-!5 = !MDSubprogram(name: "zed", linkageName: "_Z3zedP3foo", line: 4, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 4, file: !6, scope: !6, type: !7, function: void (%struct.foo*)* @_Z3zedP3foo)
-!6 = !MDFile(filename: "/home/espindola/llvm/test.cc", directory: "/home/espindola/tmpfs/build")
-!7 = !MDSubroutineType(types: !8)
+!5 = !DISubprogram(name: "zed", linkageName: "_Z3zedP3foo", line: 4, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 4, file: !6, scope: !6, type: !7, function: void (%struct.foo*)* @_Z3zedP3foo)
+!6 = !DIFile(filename: "/home/espindola/llvm/test.cc", directory: "/home/espindola/tmpfs/build")
+!7 = !DISubroutineType(types: !8)
 !8 = !{null, !9}
-!9 = !MDDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, baseType: !10)
-!10 = !MDCompositeType(tag: DW_TAG_class_type, name: "foo", line: 1, size: 8, align: 8, file: !32, elements: !11)
+!9 = !DIDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, baseType: !10)
+!10 = !DICompositeType(tag: DW_TAG_class_type, name: "foo", line: 1, size: 8, align: 8, file: !32, elements: !11)
 !11 = !{!12}
-!12 = !MDSubprogram(name: "bar", linkageName: "_ZN3foo3barEv", line: 2, isLocal: false, isDefinition: false, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 2, file: !6, scope: !10, type: !13)
-!13 = !MDSubroutineType(types: !14)
+!12 = !DISubprogram(name: "bar", linkageName: "_ZN3foo3barEv", line: 2, isLocal: false, isDefinition: false, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 2, file: !6, scope: !10, type: !13)
+!13 = !DISubroutineType(types: !14)
 !14 = !{null, !15}
-!15 = !MDDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, flags: DIFlagArtificial, baseType: !10)
-!20 = !MDSubprogram(name: "bar", linkageName: "_ZN3foo3barEv", line: 2, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 2, file: !6, scope: null, type: !13, function: void (%struct.foo*)* @_ZN3foo3barEv, declaration: !12)
-!23 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "x", line: 4, arg: 1, scope: !5, file: !6, type: !9)
-!24 = !MDLocation(line: 4, column: 15, scope: !5)
-!25 = !MDLocation(line: 4, column: 20, scope: !26)
-!26 = distinct !MDLexicalBlock(line: 4, column: 18, file: !6, scope: !5)
-!27 = !MDLocation(line: 4, column: 30, scope: !26)
-!28 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "this", line: 2, arg: 1, flags: DIFlagArtificial, scope: !20, file: !6, type: !15)
-!29 = !MDLocation(line: 2, column: 8, scope: !20)
-!30 = !MDLocation(line: 2, column: 15, scope: !31)
-!31 = distinct !MDLexicalBlock(line: 2, column: 14, file: !6, scope: !20)
-!32 = !MDFile(filename: "/home/espindola/llvm/test.cc", directory: "/home/espindola/tmpfs/build")
+!15 = !DIDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, flags: DIFlagArtificial, baseType: !10)
+!20 = !DISubprogram(name: "bar", linkageName: "_ZN3foo3barEv", line: 2, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 2, file: !6, scope: null, type: !13, function: void (%struct.foo*)* @_ZN3foo3barEv, declaration: !12)
+!23 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "x", line: 4, arg: 1, scope: !5, file: !6, type: !9)
+!24 = !DILocation(line: 4, column: 15, scope: !5)
+!25 = !DILocation(line: 4, column: 20, scope: !26)
+!26 = distinct !DILexicalBlock(line: 4, column: 18, file: !6, scope: !5)
+!27 = !DILocation(line: 4, column: 30, scope: !26)
+!28 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "this", line: 2, arg: 1, flags: DIFlagArtificial, scope: !20, file: !6, type: !15)
+!29 = !DILocation(line: 2, column: 8, scope: !20)
+!30 = !DILocation(line: 2, column: 15, scope: !31)
+!31 = distinct !DILexicalBlock(line: 2, column: 14, file: !6, scope: !20)
+!32 = !DIFile(filename: "/home/espindola/llvm/test.cc", directory: "/home/espindola/tmpfs/build")
 !33 = !{i32 1, !"Debug Info Version", i32 3}
diff --git a/test/DebugInfo/X86/pr12831.ll b/test/DebugInfo/X86/pr12831.ll
index 887110fedd6..98b9ce48860 100644
--- a/test/DebugInfo/X86/pr12831.ll
+++ b/test/DebugInfo/X86/pr12831.ll
@@ -20,7 +20,7 @@ entry:
   %agg.tmp4 = alloca %class.function, align 1
   %agg.tmp5 = alloca %class.anon.0, align 1
   store %class.BPLFunctionWriter* %this, %class.BPLFunctionWriter** %this.addr, align 8
-  call void @llvm.dbg.declare(metadata %class.BPLFunctionWriter** %this.addr, metadata !133, metadata !MDExpression()), !dbg !135
+  call void @llvm.dbg.declare(metadata %class.BPLFunctionWriter** %this.addr, metadata !133, metadata !DIExpression()), !dbg !135
   %this1 = load %class.BPLFunctionWriter*, %class.BPLFunctionWriter** %this.addr
   %MW = getelementptr inbounds %class.BPLFunctionWriter, %class.BPLFunctionWriter* %this1, i32 0, i32 0, !dbg !136
   %0 = load %struct.BPLModuleWriter*, %struct.BPLModuleWriter** %MW, align 8, !dbg !136
@@ -42,8 +42,8 @@ entry:
   %this.addr = alloca %class.function*, align 8
   %__f = alloca %class.anon.0, align 1
   store %class.function* %this, %class.function** %this.addr, align 8
-  call void @llvm.dbg.declare(metadata %class.function** %this.addr, metadata !140, metadata !MDExpression()), !dbg !142
-  call void @llvm.dbg.declare(metadata %class.anon.0* %__f, metadata !143, metadata !MDExpression()), !dbg !144
+  call void @llvm.dbg.declare(metadata %class.function** %this.addr, metadata !140, metadata !DIExpression()), !dbg !142
+  call void @llvm.dbg.declare(metadata %class.anon.0* %__f, metadata !143, metadata !DIExpression()), !dbg !144
   %this1 = load %class.function*, %class.function** %this.addr
   call void @"_ZN13_Base_manager21_M_not_empty_functionIZN17BPLFunctionWriter9writeExprEvE3$_1_0EEvRKT_"(%class.anon.0* %__f), !dbg !145
   ret void, !dbg !147
@@ -61,8 +61,8 @@ entry:
   %this.addr = alloca %class.function*, align 8
   %__f = alloca %class.anon, align 1
   store %class.function* %this, %class.function** %this.addr, align 8
-  call void @llvm.dbg.declare(metadata %class.function** %this.addr, metadata !150, metadata !MDExpression()), !dbg !151
-  call void @llvm.dbg.declare(metadata %class.anon* %__f, metadata !152, metadata !MDExpression()), !dbg !153
+  call void @llvm.dbg.declare(metadata %class.function** %this.addr, metadata !150, metadata !DIExpression()), !dbg !151
+  call void @llvm.dbg.declare(metadata %class.anon* %__f, metadata !152, metadata !DIExpression()), !dbg !153
   %this1 = load %class.function*, %class.function** %this.addr
   call void @"_ZN13_Base_manager21_M_not_empty_functionIZN17BPLFunctionWriter9writeExprEvE3$_0EEvRKT_"(%class.anon* %__f), !dbg !154
   ret void, !dbg !156
@@ -78,133 +78,133 @@ entry:
 !llvm.dbg.cu = !{!0}
 !llvm.module.flags = !{!162}
 
-!0 = !MDCompileUnit(language: DW_LANG_C_plus_plus, producer: "clang version 3.2 ", isOptimized: false, emissionKind: 0, file: !161, enums: !1, retainedTypes: !1, subprograms: !3, globals: !128)
+!0 = !DICompileUnit(language: DW_LANG_C_plus_plus, producer: "clang version 3.2 ", isOptimized: false, emissionKind: 0, file: !161, enums: !1, retainedTypes: !1, subprograms: !3, globals: !128)
 !1 = !{}
 !3 = !{!5, !106, !107, !126, !127}
-!5 = !MDSubprogram(name: "writeExpr", linkageName: "_ZN17BPLFunctionWriter9writeExprEv", line: 19, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 19, file: !6, scope: null, type: !7, function: void (%class.BPLFunctionWriter*)* @_ZN17BPLFunctionWriter9writeExprEv, declaration: !103, variables: !1)
-!6 = !MDFile(filename: "BPLFunctionWriter2.ii", directory: "/home/peter/crashdelta")
-!7 = !MDSubroutineType(types: !8)
+!5 = !DISubprogram(name: "writeExpr", linkageName: "_ZN17BPLFunctionWriter9writeExprEv", line: 19, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 19, file: !6, scope: null, type: !7, function: void (%class.BPLFunctionWriter*)* @_ZN17BPLFunctionWriter9writeExprEv, declaration: !103, variables: !1)
+!6 = !DIFile(filename: "BPLFunctionWriter2.ii", directory: "/home/peter/crashdelta")
+!7 = !DISubroutineType(types: !8)
 !8 = !{null, !9}
-!9 = !MDDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, flags: DIFlagArtificial, baseType: !10)
-!10 = !MDCompositeType(tag: DW_TAG_class_type, name: "BPLFunctionWriter", line: 15, size: 64, align: 64, file: !160, elements: !11)
+!9 = !DIDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, flags: DIFlagArtificial, baseType: !10)
+!10 = !DICompositeType(tag: DW_TAG_class_type, name: "BPLFunctionWriter", line: 15, size: 64, align: 64, file: !160, elements: !11)
 !11 = !{!12, !103}
-!12 = !MDDerivedType(tag: DW_TAG_member, name: "MW", line: 16, size: 64, align: 64, flags: DIFlagPrivate, file: !160, scope: !10, baseType: !13)
-!13 = !MDDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, baseType: !14)
-!14 = !MDCompositeType(tag: DW_TAG_class_type, name: "BPLModuleWriter", line: 12, size: 8, align: 8, file: !160, elements: !15)
+!12 = !DIDerivedType(tag: DW_TAG_member, name: "MW", line: 16, size: 64, align: 64, flags: DIFlagPrivate, file: !160, scope: !10, baseType: !13)
+!13 = !DIDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, baseType: !14)
+!14 = !DICompositeType(tag: DW_TAG_class_type, name: "BPLModuleWriter", line: 12, size: 8, align: 8, file: !160, elements: !15)
 !15 = !{!16}
-!16 = !MDSubprogram(name: "writeIntrinsic", linkageName: "_ZN15BPLModuleWriter14writeIntrinsicE8functionIFvvEE", line: 13, isLocal: false, isDefinition: false, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 13, file: !6, scope: !14, type: !17)
-!17 = !MDSubroutineType(types: !18)
+!16 = !DISubprogram(name: "writeIntrinsic", linkageName: "_ZN15BPLModuleWriter14writeIntrinsicE8functionIFvvEE", line: 13, isLocal: false, isDefinition: false, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 13, file: !6, scope: !14, type: !17)
+!17 = !DISubroutineType(types: !18)
 !18 = !{null, !19, !20}
-!19 = !MDDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, flags: DIFlagArtificial, baseType: !14)
-!20 = !MDCompositeType(tag: DW_TAG_class_type, name: "function<void ()>", line: 6, size: 8, align: 8, file: !160, elements: !21, templateParams: !97)
+!19 = !DIDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, flags: DIFlagArtificial, baseType: !14)
+!20 = !DICompositeType(tag: DW_TAG_class_type, name: "function<void ()>", line: 6, size: 8, align: 8, file: !160, elements: !21, templateParams: !97)
 !21 = !{!22, !51, !58, !86, !92}
-!22 = !MDSubprogram(name: "function<BPLFunctionWriter::<lambda at BPLFunctionWriter2.ii:20:36> >", line: 8, isLocal: false, isDefinition: false, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 8, file: !6, scope: !20, type: !23, templateParams: !47)
-!23 = !MDSubroutineType(types: !24)
+!22 = !DISubprogram(name: "function<BPLFunctionWriter::<lambda at BPLFunctionWriter2.ii:20:36> >", line: 8, isLocal: false, isDefinition: false, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 8, file: !6, scope: !20, type: !23, templateParams: !47)
+!23 = !DISubroutineType(types: !24)
 !24 = !{null, !25, !26}
-!25 = !MDDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, flags: DIFlagArtificial, baseType: !20)
-!26 = !MDCompositeType(tag: DW_TAG_class_type, line: 20, size: 8, align: 8, file: !160, scope: !5, elements: !27)
+!25 = !DIDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, flags: DIFlagArtificial, baseType: !20)
+!26 = !DICompositeType(tag: DW_TAG_class_type, line: 20, size: 8, align: 8, file: !160, scope: !5, elements: !27)
 !27 = !{!28, !35, !41}
-!28 = !MDSubprogram(name: "operator()", line: 20, isLocal: false, isDefinition: false, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 20, file: !6, scope: !26, type: !29)
-!29 = !MDSubroutineType(types: !30)
+!28 = !DISubprogram(name: "operator()", line: 20, isLocal: false, isDefinition: false, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 20, file: !6, scope: !26, type: !29)
+!29 = !DISubroutineType(types: !30)
 !30 = !{null, !31}
-!31 = !MDDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, flags: DIFlagArtificial, baseType: !32)
-!32 = !MDDerivedType(tag: DW_TAG_const_type, baseType: !26)
-!35 = !MDSubprogram(name: "~", line: 20, isLocal: false, isDefinition: false, virtualIndex: 6, flags: DIFlagArtificial | DIFlagPrototyped, isOptimized: false, scopeLine: 20, file: !6, scope: !26, type: !36)
-!36 = !MDSubroutineType(types: !37)
+!31 = !DIDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, flags: DIFlagArtificial, baseType: !32)
+!32 = !DIDerivedType(tag: DW_TAG_const_type, baseType: !26)
+!35 = !DISubprogram(name: "~", line: 20, isLocal: false, isDefinition: false, virtualIndex: 6, flags: DIFlagArtificial | DIFlagPrototyped, isOptimized: false, scopeLine: 20, file: !6, scope: !26, type: !36)
+!36 = !DISubroutineType(types: !37)
 !37 = !{null, !38}
-!38 = !MDDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, flags: DIFlagArtificial, baseType: !26)
-!41 = !MDSubprogram(name: "", line: 20, isLocal: false, isDefinition: false, virtualIndex: 6, flags: DIFlagArtificial | DIFlagPrototyped, isOptimized: false, scopeLine: 20, file: !6, scope: !26, type: !42)
-!42 = !MDSubroutineType(types: !43)
+!38 = !DIDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, flags: DIFlagArtificial, baseType: !26)
+!41 = !DISubprogram(name: "", line: 20, isLocal: false, isDefinition: false, virtualIndex: 6, flags: DIFlagArtificial | DIFlagPrototyped, isOptimized: false, scopeLine: 20, file: !6, scope: !26, type: !42)
+!42 = !DISubroutineType(types: !43)
 !43 = !{null, !38, !44}
-!44 = !MDDerivedType(tag: DW_TAG_rvalue_reference_type, baseType: !26)
+!44 = !DIDerivedType(tag: DW_TAG_rvalue_reference_type, baseType: !26)
 !47 = !{!48}
-!48 = !MDTemplateTypeParameter(name: "_Functor", type: !26)
-!51 = !MDSubprogram(name: "function<function<void ()> >", line: 8, isLocal: false, isDefinition: false, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 8, file: !6, scope: !20, type: !52, templateParams: !54)
-!52 = !MDSubroutineType(types: !53)
+!48 = !DITemplateTypeParameter(name: "_Functor", type: !26)
+!51 = !DISubprogram(name: "function<function<void ()> >", line: 8, isLocal: false, isDefinition: false, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 8, file: !6, scope: !20, type: !52, templateParams: !54)
+!52 = !DISubroutineType(types: !53)
 !53 = !{null, !25, !20}
 !54 = !{!55}
-!55 = !MDTemplateTypeParameter(name: "_Functor", type: !20)
-!58 = !MDSubprogram(name: "function<BPLFunctionWriter::<lambda at BPLFunctionWriter2.ii:23:36> >", line: 8, isLocal: false, isDefinition: false, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 8, file: !6, scope: !20, type: !59, templateParams: !82)
-!59 = !MDSubroutineType(types: !60)
+!55 = !DITemplateTypeParameter(name: "_Functor", type: !20)
+!58 = !DISubprogram(name: "function<BPLFunctionWriter::<lambda at BPLFunctionWriter2.ii:23:36> >", line: 8, isLocal: false, isDefinition: false, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 8, file: !6, scope: !20, type: !59, templateParams: !82)
+!59 = !DISubroutineType(types: !60)
 !60 = !{null, !25, !61}
-!61 = !MDCompositeType(tag: DW_TAG_class_type, line: 23, size: 8, align: 8, file: !160, scope: !5, elements: !62)
+!61 = !DICompositeType(tag: DW_TAG_class_type, line: 23, size: 8, align: 8, file: !160, scope: !5, elements: !62)
 !62 = !{!63, !70, !76}
-!63 = !MDSubprogram(name: "operator()", line: 23, isLocal: false, isDefinition: false, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 23, file: !6, scope: !61, type: !64)
-!64 = !MDSubroutineType(types: !65)
+!63 = !DISubprogram(name: "operator()", line: 23, isLocal: false, isDefinition: false, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 23, file: !6, scope: !61, type: !64)
+!64 = !DISubroutineType(types: !65)
 !65 = !{null, !66}
-!66 = !MDDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, flags: DIFlagArtificial, baseType: !67)
-!67 = !MDDerivedType(tag: DW_TAG_const_type, baseType: !61)
-!70 = !MDSubprogram(name: "~", line: 23, isLocal: false, isDefinition: false, virtualIndex: 6, flags: DIFlagArtificial | DIFlagPrototyped, isOptimized: false, scopeLine: 23, file: !6, scope: !61, type: !71)
-!71 = !MDSubroutineType(types: !72)
+!66 = !DIDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, flags: DIFlagArtificial, baseType: !67)
+!67 = !DIDerivedType(tag: DW_TAG_const_type, baseType: !61)
+!70 = !DISubprogram(name: "~", line: 23, isLocal: false, isDefinition: false, virtualIndex: 6, flags: DIFlagArtificial | DIFlagPrototyped, isOptimized: false, scopeLine: 23, file: !6, scope: !61, type: !71)
+!71 = !DISubroutineType(types: !72)
 !72 = !{null, !73}
-!73 = !MDDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, flags: DIFlagArtificial, baseType: !61)
-!76 = !MDSubprogram(name: "", line: 23, isLocal: false, isDefinition: false, virtualIndex: 6, flags: DIFlagArtificial | DIFlagPrototyped, isOptimized: false, scopeLine: 23, file: !6, scope: !61, type: !77)
-!77 = !MDSubroutineType(types: !78)
+!73 = !DIDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, flags: DIFlagArtificial, baseType: !61)
+!76 = !DISubprogram(name: "", line: 23, isLocal: false, isDefinition: false, virtualIndex: 6, flags: DIFlagArtificial | DIFlagPrototyped, isOptimized: false, scopeLine: 23, file: !6, scope: !61, type: !77)
+!77 = !DISubroutineType(types: !78)
 !78 = !{null, !73, !79}
-!79 = !MDDerivedType(tag: DW_TAG_rvalue_reference_type, baseType: !61)
+!79 = !DIDerivedType(tag: DW_TAG_rvalue_reference_type, baseType: !61)
 !82 = !{!83}
-!83 = !MDTemplateTypeParameter(name: "_Functor", type: !61)
-!86 = !MDSubprogram(name: "function", line: 6, isLocal: false, isDefinition: false, virtualIndex: 6, flags: DIFlagArtificial | DIFlagPrototyped, isOptimized: false, scopeLine: 6, file: !6, scope: !20, type: !87)
-!87 = !MDSubroutineType(types: !88)
+!83 = !DITemplateTypeParameter(name: "_Functor", type: !61)
+!86 = !DISubprogram(name: "function", line: 6, isLocal: false, isDefinition: false, virtualIndex: 6, flags: DIFlagArtificial | DIFlagPrototyped, isOptimized: false, scopeLine: 6, file: !6, scope: !20, type: !87)
+!87 = !DISubroutineType(types: !88)
 !88 = !{null, !25, !89}
-!89 = !MDDerivedType(tag: DW_TAG_rvalue_reference_type, baseType: !20)
-!92 = !MDSubprogram(name: "~function", line: 6, isLocal: false, isDefinition: false, virtualIndex: 6, flags: DIFlagArtificial | DIFlagPrototyped, isOptimized: false, scopeLine: 6, file: !6, scope: !20, type: !93)
-!93 = !MDSubroutineType(types: !94)
+!89 = !DIDerivedType(tag: DW_TAG_rvalue_reference_type, baseType: !20)
+!92 = !DISubprogram(name: "~function", line: 6, isLocal: false, isDefinition: false, virtualIndex: 6, flags: DIFlagArtificial | DIFlagPrototyped, isOptimized: false, scopeLine: 6, file: !6, scope: !20, type: !93)
+!93 = !DISubroutineType(types: !94)
 !94 = !{null, !25}
 !97 = !{!98}
-!98 = !MDTemplateTypeParameter(name: "T", type: !99)
-!99 = !MDSubroutineType(types: !100)
+!98 = !DITemplateTypeParameter(name: "T", type: !99)
+!99 = !DISubroutineType(types: !100)
 !100 = !{null}
-!103 = !MDSubprogram(name: "writeExpr", linkageName: "_ZN17BPLFunctionWriter9writeExprEv", line: 17, isLocal: false, isDefinition: false, virtualIndex: 6, flags: DIFlagPrivate | DIFlagPrototyped, isOptimized: false, scopeLine: 17, file: !6, scope: !10, type: !7)
-!106 = !MDSubprogram(name: "function<BPLFunctionWriter::<lambda at BPLFunctionWriter2.ii:23:36> >", linkageName: "_ZN8functionIFvvEEC2IZN17BPLFunctionWriter9writeExprEvE3$_1_0EET_", line: 8, isLocal: true, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 8, file: !6, scope: null, type: !59, function: void (%class.function*)* @"_ZN8functionIFvvEEC2IZN17BPLFunctionWriter9writeExprEvE3$_1_0EET_", templateParams: !82, declaration: !58, variables: !1)
-!107 = !MDSubprogram(name: "_M_not_empty_function<BPLFunctionWriter::<lambda at BPLFunctionWriter2.ii:23:36> >", linkageName: "_ZN13_Base_manager21_M_not_empty_functionIZN17BPLFunctionWriter9writeExprEvE3$_1_0EEvRKT_", line: 3, isLocal: true, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 3, file: !6, scope: null, type: !108, function: void (%class.anon.0*)* @"_ZN13_Base_manager21_M_not_empty_functionIZN17BPLFunctionWriter9writeExprEvE3$_1_0EEvRKT_", templateParams: !111, declaration: !113, variables: !1)
-!108 = !MDSubroutineType(types: !109)
+!103 = !DISubprogram(name: "writeExpr", linkageName: "_ZN17BPLFunctionWriter9writeExprEv", line: 17, isLocal: false, isDefinition: false, virtualIndex: 6, flags: DIFlagPrivate | DIFlagPrototyped, isOptimized: false, scopeLine: 17, file: !6, scope: !10, type: !7)
+!106 = !DISubprogram(name: "function<BPLFunctionWriter::<lambda at BPLFunctionWriter2.ii:23:36> >", linkageName: "_ZN8functionIFvvEEC2IZN17BPLFunctionWriter9writeExprEvE3$_1_0EET_", line: 8, isLocal: true, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 8, file: !6, scope: null, type: !59, function: void (%class.function*)* @"_ZN8functionIFvvEEC2IZN17BPLFunctionWriter9writeExprEvE3$_1_0EET_", templateParams: !82, declaration: !58, variables: !1)
+!107 = !DISubprogram(name: "_M_not_empty_function<BPLFunctionWriter::<lambda at BPLFunctionWriter2.ii:23:36> >", linkageName: "_ZN13_Base_manager21_M_not_empty_functionIZN17BPLFunctionWriter9writeExprEvE3$_1_0EEvRKT_", line: 3, isLocal: true, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 3, file: !6, scope: null, type: !108, function: void (%class.anon.0*)* @"_ZN13_Base_manager21_M_not_empty_functionIZN17BPLFunctionWriter9writeExprEvE3$_1_0EEvRKT_", templateParams: !111, declaration: !113, variables: !1)
+!108 = !DISubroutineType(types: !109)
 !109 = !{null, !110}
-!110 = !MDDerivedType(tag: DW_TAG_reference_type, baseType: !61)
+!110 = !DIDerivedType(tag: DW_TAG_reference_type, baseType: !61)
 !111 = !{!112}
-!112 = !MDTemplateTypeParameter(name: "_Tp", type: !61)
-!113 = !MDSubprogram(name: "_M_not_empty_function<BPLFunctionWriter::<lambda at BPLFunctionWriter2.ii:23:36> >", linkageName: "_ZN13_Base_manager21_M_not_empty_functionIZN17BPLFunctionWriter9writeExprEvE3$_1_0EEvRKT_", line: 3, isLocal: false, isDefinition: false, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 3, file: !6, scope: !114, type: !108, templateParams: !111)
-!114 = !MDCompositeType(tag: DW_TAG_class_type, name: "_Base_manager", line: 1, size: 8, align: 8, file: !160, elements: !115)
+!112 = !DITemplateTypeParameter(name: "_Tp", type: !61)
+!113 = !DISubprogram(name: "_M_not_empty_function<BPLFunctionWriter::<lambda at BPLFunctionWriter2.ii:23:36> >", linkageName: "_ZN13_Base_manager21_M_not_empty_functionIZN17BPLFunctionWriter9writeExprEvE3$_1_0EEvRKT_", line: 3, isLocal: false, isDefinition: false, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 3, file: !6, scope: !114, type: !108, templateParams: !111)
+!114 = !DICompositeType(tag: DW_TAG_class_type, name: "_Base_manager", line: 1, size: 8, align: 8, file: !160, elements: !115)
 !115 = !{!116, !113}
-!116 = !MDSubprogram(name: "_M_not_empty_function<BPLFunctionWriter::<lambda at BPLFunctionWriter2.ii:20:36> >", linkageName: "_ZN13_Base_manager21_M_not_empty_functionIZN17BPLFunctionWriter9writeExprEvE3$_0EEvRKT_", line: 3, isLocal: false, isDefinition: false, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 3, file: !6, scope: !114, type: !117, templateParams: !120)
-!117 = !MDSubroutineType(types: !118)
+!116 = !DISubprogram(name: "_M_not_empty_function<BPLFunctionWriter::<lambda at BPLFunctionWriter2.ii:20:36> >", linkageName: "_ZN13_Base_manager21_M_not_empty_functionIZN17BPLFunctionWriter9writeExprEvE3$_0EEvRKT_", line: 3, isLocal: false, isDefinition: false, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 3, file: !6, scope: !114, type: !117, templateParams: !120)
+!117 = !DISubroutineType(types: !118)
 !118 = !{null, !119}
-!119 = !MDDerivedType(tag: DW_TAG_reference_type, baseType: !26)
+!119 = !DIDerivedType(tag: DW_TAG_reference_type, baseType: !26)
 !120 = !{!121}
-!121 = !MDTemplateTypeParameter(name: "_Tp", type: !26)
-!126 = !MDSubprogram(name: "function<BPLFunctionWriter::<lambda at BPLFunctionWriter2.ii:20:36> >", linkageName: "_ZN8functionIFvvEEC2IZN17BPLFunctionWriter9writeExprEvE3$_0EET_", line: 8, isLocal: true, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 8, file: !6, scope: null, type: !23, function: void (%class.function*)* @"_ZN8functionIFvvEEC2IZN17BPLFunctionWriter9writeExprEvE3$_0EET_", templateParams: !47, declaration: !22, variables: !1)
-!127 = !MDSubprogram(name: "_M_not_empty_function<BPLFunctionWriter::<lambda at BPLFunctionWriter2.ii:20:36> >", linkageName: "_ZN13_Base_manager21_M_not_empty_functionIZN17BPLFunctionWriter9writeExprEvE3$_0EEvRKT_", line: 3, isLocal: true, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 3, file: !6, scope: null, type: !117, function: void (%class.anon*)* @"_ZN13_Base_manager21_M_not_empty_functionIZN17BPLFunctionWriter9writeExprEvE3$_0EEvRKT_", templateParams: !120, declaration: !116, variables: !1)
+!121 = !DITemplateTypeParameter(name: "_Tp", type: !26)
+!126 = !DISubprogram(name: "function<BPLFunctionWriter::<lambda at BPLFunctionWriter2.ii:20:36> >", linkageName: "_ZN8functionIFvvEEC2IZN17BPLFunctionWriter9writeExprEvE3$_0EET_", line: 8, isLocal: true, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 8, file: !6, scope: null, type: !23, function: void (%class.function*)* @"_ZN8functionIFvvEEC2IZN17BPLFunctionWriter9writeExprEvE3$_0EET_", templateParams: !47, declaration: !22, variables: !1)
+!127 = !DISubprogram(name: "_M_not_empty_function<BPLFunctionWriter::<lambda at BPLFunctionWriter2.ii:20:36> >", linkageName: "_ZN13_Base_manager21_M_not_empty_functionIZN17BPLFunctionWriter9writeExprEvE3$_0EEvRKT_", line: 3, isLocal: true, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 3, file: !6, scope: null, type: !117, function: void (%class.anon*)* @"_ZN13_Base_manager21_M_not_empty_functionIZN17BPLFunctionWriter9writeExprEvE3$_0EEvRKT_", templateParams: !120, declaration: !116, variables: !1)
 !128 = !{!130}
-!130 = !MDGlobalVariable(name: "__stored_locally", linkageName: "__stored_locally", line: 2, isLocal: true, isDefinition: true, scope: !114, file: !6, type: !131, variable: i1 1)
-!131 = !MDDerivedType(tag: DW_TAG_const_type, baseType: !132)
-!132 = !MDBasicType(tag: DW_TAG_base_type, name: "bool", size: 8, align: 8, encoding: DW_ATE_boolean)
-!133 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "this", line: 19, arg: 1, flags: DIFlagArtificial, scope: !5, file: !6, type: !134)
-!134 = !MDDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, baseType: !10)
-!135 = !MDLocation(line: 19, column: 39, scope: !5)
-!136 = !MDLocation(line: 20, column: 17, scope: !137)
-!137 = distinct !MDLexicalBlock(line: 19, column: 51, file: !6, scope: !5)
-!138 = !MDLocation(line: 23, column: 17, scope: !137)
-!139 = !MDLocation(line: 26, column: 15, scope: !137)
-!140 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "this", line: 8, arg: 1, flags: DIFlagArtificial, scope: !106, file: !6, type: !141)
-!141 = !MDDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, baseType: !20)
-!142 = !MDLocation(line: 8, column: 45, scope: !106)
-!143 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "__f", line: 8, arg: 2, scope: !106, file: !6, type: !61)
-!144 = !MDLocation(line: 8, column: 63, scope: !106)
-!145 = !MDLocation(line: 9, column: 9, scope: !146)
-!146 = distinct !MDLexicalBlock(line: 8, column: 81, file: !6, scope: !106)
-!147 = !MDLocation(line: 10, column: 13, scope: !146)
-!148 = !MDLocation(line: 4, column: 5, scope: !149)
-!149 = distinct !MDLexicalBlock(line: 3, column: 105, file: !6, scope: !107)
-!150 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "this", line: 8, arg: 1, flags: DIFlagArtificial, scope: !126, file: !6, type: !141)
-!151 = !MDLocation(line: 8, column: 45, scope: !126)
-!152 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "__f", line: 8, arg: 2, scope: !126, file: !6, type: !26)
-!153 = !MDLocation(line: 8, column: 63, scope: !126)
-!154 = !MDLocation(line: 9, column: 9, scope: !155)
-!155 = distinct !MDLexicalBlock(line: 8, column: 81, file: !6, scope: !126)
-!156 = !MDLocation(line: 10, column: 13, scope: !155)
-!157 = !MDLocation(line: 4, column: 5, scope: !158)
-!158 = distinct !MDLexicalBlock(line: 3, column: 105, file: !6, scope: !127)
-!159 = !MDFile(filename: "BPLFunctionWriter.cpp", directory: "/home/peter/crashdelta")
-!160 = !MDFile(filename: "BPLFunctionWriter2.ii", directory: "/home/peter/crashdelta")
-!161 = !MDFile(filename: "BPLFunctionWriter.cpp", directory: "/home/peter/crashdelta")
+!130 = !DIGlobalVariable(name: "__stored_locally", linkageName: "__stored_locally", line: 2, isLocal: true, isDefinition: true, scope: !114, file: !6, type: !131, variable: i1 1)
+!131 = !DIDerivedType(tag: DW_TAG_const_type, baseType: !132)
+!132 = !DIBasicType(tag: DW_TAG_base_type, name: "bool", size: 8, align: 8, encoding: DW_ATE_boolean)
+!133 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "this", line: 19, arg: 1, flags: DIFlagArtificial, scope: !5, file: !6, type: !134)
+!134 = !DIDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, baseType: !10)
+!135 = !DILocation(line: 19, column: 39, scope: !5)
+!136 = !DILocation(line: 20, column: 17, scope: !137)
+!137 = distinct !DILexicalBlock(line: 19, column: 51, file: !6, scope: !5)
+!138 = !DILocation(line: 23, column: 17, scope: !137)
+!139 = !DILocation(line: 26, column: 15, scope: !137)
+!140 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "this", line: 8, arg: 1, flags: DIFlagArtificial, scope: !106, file: !6, type: !141)
+!141 = !DIDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, baseType: !20)
+!142 = !DILocation(line: 8, column: 45, scope: !106)
+!143 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "__f", line: 8, arg: 2, scope: !106, file: !6, type: !61)
+!144 = !DILocation(line: 8, column: 63, scope: !106)
+!145 = !DILocation(line: 9, column: 9, scope: !146)
+!146 = distinct !DILexicalBlock(line: 8, column: 81, file: !6, scope: !106)
+!147 = !DILocation(line: 10, column: 13, scope: !146)
+!148 = !DILocation(line: 4, column: 5, scope: !149)
+!149 = distinct !DILexicalBlock(line: 3, column: 105, file: !6, scope: !107)
+!150 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "this", line: 8, arg: 1, flags: DIFlagArtificial, scope: !126, file: !6, type: !141)
+!151 = !DILocation(line: 8, column: 45, scope: !126)
+!152 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "__f", line: 8, arg: 2, scope: !126, file: !6, type: !26)
+!153 = !DILocation(line: 8, column: 63, scope: !126)
+!154 = !DILocation(line: 9, column: 9, scope: !155)
+!155 = distinct !DILexicalBlock(line: 8, column: 81, file: !6, scope: !126)
+!156 = !DILocation(line: 10, column: 13, scope: !155)
+!157 = !DILocation(line: 4, column: 5, scope: !158)
+!158 = distinct !DILexicalBlock(line: 3, column: 105, file: !6, scope: !127)
+!159 = !DIFile(filename: "BPLFunctionWriter.cpp", directory: "/home/peter/crashdelta")
+!160 = !DIFile(filename: "BPLFunctionWriter2.ii", directory: "/home/peter/crashdelta")
+!161 = !DIFile(filename: "BPLFunctionWriter.cpp", directory: "/home/peter/crashdelta")
 !162 = !{i32 1, !"Debug Info Version", i32 3}
diff --git a/test/DebugInfo/X86/pr13303.ll b/test/DebugInfo/X86/pr13303.ll
index a6d42ff9ab7..df31a799446 100644
--- a/test/DebugInfo/X86/pr13303.ll
+++ b/test/DebugInfo/X86/pr13303.ll
@@ -15,15 +15,15 @@ entry:
 !llvm.dbg.cu = !{!0}
 !llvm.module.flags = !{!13}
 
-!0 = !MDCompileUnit(language: DW_LANG_C99, producer: "clang version 3.2 (trunk 160143)", isOptimized: false, emissionKind: 0, file: !12, enums: !1, retainedTypes: !1, subprograms: !3, globals: !1, imports:  !1)
+!0 = !DICompileUnit(language: DW_LANG_C99, producer: "clang version 3.2 (trunk 160143)", isOptimized: false, emissionKind: 0, file: !12, enums: !1, retainedTypes: !1, subprograms: !3, globals: !1, imports:  !1)
 !1 = !{}
 !3 = !{!5}
-!5 = !MDSubprogram(name: "main", line: 1, isLocal: false, isDefinition: true, virtualIndex: 6, isOptimized: false, scopeLine: 1, file: !12, scope: !6, type: !7, function: i32 ()* @main, variables: !1)
-!6 = !MDFile(filename: "PR13303.c", directory: "/home/probinson")
-!7 = !MDSubroutineType(types: !8)
+!5 = !DISubprogram(name: "main", line: 1, isLocal: false, isDefinition: true, virtualIndex: 6, isOptimized: false, scopeLine: 1, file: !12, scope: !6, type: !7, function: i32 ()* @main, variables: !1)
+!6 = !DIFile(filename: "PR13303.c", directory: "/home/probinson")
+!7 = !DISubroutineType(types: !8)
 !8 = !{!9}
-!9 = !MDBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
-!10 = !MDLocation(line: 1, column: 14, scope: !11)
-!11 = distinct !MDLexicalBlock(line: 1, column: 12, file: !12, scope: !5)
-!12 = !MDFile(filename: "PR13303.c", directory: "/home/probinson")
+!9 = !DIBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
+!10 = !DILocation(line: 1, column: 14, scope: !11)
+!11 = distinct !DILexicalBlock(line: 1, column: 12, file: !12, scope: !5)
+!12 = !DIFile(filename: "PR13303.c", directory: "/home/probinson")
 !13 = !{i32 1, !"Debug Info Version", i32 3}
diff --git a/test/DebugInfo/X86/pr19307.ll b/test/DebugInfo/X86/pr19307.ll
index 7ea4de80da5..e04131fe322 100644
--- a/test/DebugInfo/X86/pr19307.ll
+++ b/test/DebugInfo/X86/pr19307.ll
@@ -40,10 +40,10 @@ entry:
   %offset.addr = alloca i64*, align 8
   %limit.addr = alloca i64*, align 8
   store i64* %offset, i64** %offset.addr, align 8
-  call void @llvm.dbg.declare(metadata i64** %offset.addr, metadata !45, metadata !MDExpression()), !dbg !46
+  call void @llvm.dbg.declare(metadata i64** %offset.addr, metadata !45, metadata !DIExpression()), !dbg !46
   store i64* %limit, i64** %limit.addr, align 8
-  call void @llvm.dbg.declare(metadata i64** %limit.addr, metadata !47, metadata !MDExpression()), !dbg !46
-  call void @llvm.dbg.declare(metadata %"class.std::basic_string"* %range, metadata !48, metadata !MDExpression(DW_OP_deref)), !dbg !49
+  call void @llvm.dbg.declare(metadata i64** %limit.addr, metadata !47, metadata !DIExpression()), !dbg !46
+  call void @llvm.dbg.declare(metadata %"class.std::basic_string"* %range, metadata !48, metadata !DIExpression(DW_OP_deref)), !dbg !49
   %call = call i32 @_ZNKSs7compareEmmPKc(%"class.std::basic_string"* %range, i64 0, i64 6, i8* getelementptr inbounds ([7 x i8], [7 x i8]* @.str, i32 0, i32 0)), !dbg !50
   %cmp = icmp ne i32 %call, 0, !dbg !50
   br i1 %cmp, label %if.then, label %lor.lhs.false, !dbg !50
@@ -84,62 +84,62 @@ attributes #2 = { "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "n
 !llvm.module.flags = !{!42, !43}
 !llvm.ident = !{!44}
 
-!0 = !MDCompileUnit(language: DW_LANG_C_plus_plus, producer: "clang version 3.5.0 (209308)", isOptimized: false, emissionKind: 1, file: !1, enums: !2, retainedTypes: !3, subprograms: !12, globals: !2, imports: !21)
-!1 = !MDFile(filename: "pr19307.cc", directory: "/llvm_cmake_gcc")
+!0 = !DICompileUnit(language: DW_LANG_C_plus_plus, producer: "clang version 3.5.0 (209308)", isOptimized: false, emissionKind: 1, file: !1, enums: !2, retainedTypes: !3, subprograms: !12, globals: !2, imports: !21)
+!1 = !DIFile(filename: "pr19307.cc", directory: "/llvm_cmake_gcc")
 !2 = !{}
 !3 = !{!4, !6, !8}
-!4 = !MDCompositeType(tag: DW_TAG_structure_type, line: 83, flags: DIFlagFwdDecl, file: !5, identifier: "_ZTS11__mbstate_t")
-!5 = !MDFile(filename: "/usr/include/wchar.h", directory: "/llvm_cmake_gcc")
-!6 = !MDCompositeType(tag: DW_TAG_structure_type, name: "lconv", line: 54, flags: DIFlagFwdDecl, file: !7, identifier: "_ZTS5lconv")
-!7 = !MDFile(filename: "/usr/include/locale.h", directory: "/llvm_cmake_gcc")
-!8 = !MDCompositeType(tag: DW_TAG_class_type, name: "basic_string<char, std::char_traits<char>, std::allocator<char> >", line: 1134, flags: DIFlagFwdDecl, file: !9, scope: !10, identifier: "_ZTSSs")
-!9 = !MDFile(filename: "/usr/lib/gcc/x86_64-linux-gnu/4.6/../../../../include/c++/4.6/bits/basic_string.tcc", directory: "/llvm_cmake_gcc")
-!10 = !MDNamespace(name: "std", line: 153, file: !11, scope: null)
-!11 = !MDFile(filename: "/usr/lib/gcc/x86_64-linux-gnu/4.6/../../../../include/c++/4.6/x86_64-linux-gnu/bits/c++config.h", directory: "/llvm_cmake_gcc")
+!4 = !DICompositeType(tag: DW_TAG_structure_type, line: 83, flags: DIFlagFwdDecl, file: !5, identifier: "_ZTS11__mbstate_t")
+!5 = !DIFile(filename: "/usr/include/wchar.h", directory: "/llvm_cmake_gcc")
+!6 = !DICompositeType(tag: DW_TAG_structure_type, name: "lconv", line: 54, flags: DIFlagFwdDecl, file: !7, identifier: "_ZTS5lconv")
+!7 = !DIFile(filename: "/usr/include/locale.h", directory: "/llvm_cmake_gcc")
+!8 = !DICompositeType(tag: DW_TAG_class_type, name: "basic_string<char, std::char_traits<char>, std::allocator<char> >", line: 1134, flags: DIFlagFwdDecl, file: !9, scope: !10, identifier: "_ZTSSs")
+!9 = !DIFile(filename: "/usr/lib/gcc/x86_64-linux-gnu/4.6/../../../../include/c++/4.6/bits/basic_string.tcc", directory: "/llvm_cmake_gcc")
+!10 = !DINamespace(name: "std", line: 153, file: !11, scope: null)
+!11 = !DIFile(filename: "/usr/lib/gcc/x86_64-linux-gnu/4.6/../../../../include/c++/4.6/x86_64-linux-gnu/bits/c++config.h", directory: "/llvm_cmake_gcc")
 !12 = !{!13}
-!13 = !MDSubprogram(name: "parse_range", linkageName: "_Z11parse_rangeRyS_Ss", line: 3, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 4, file: !1, scope: !14, type: !15, function: void (i64*, i64*, %"class.std::basic_string"*)* @_Z11parse_rangeRyS_Ss, variables: !2)
-!14 = !MDFile(filename: "pr19307.cc", directory: "/llvm_cmake_gcc")
-!15 = !MDSubroutineType(types: !16)
+!13 = !DISubprogram(name: "parse_range", linkageName: "_Z11parse_rangeRyS_Ss", line: 3, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 4, file: !1, scope: !14, type: !15, function: void (i64*, i64*, %"class.std::basic_string"*)* @_Z11parse_rangeRyS_Ss, variables: !2)
+!14 = !DIFile(filename: "pr19307.cc", directory: "/llvm_cmake_gcc")
+!15 = !DISubroutineType(types: !16)
 !16 = !{null, !17, !17, !19}
-!17 = !MDDerivedType(tag: DW_TAG_reference_type, baseType: !18)
-!18 = !MDBasicType(tag: DW_TAG_base_type, name: "long long unsigned int", size: 64, align: 64, encoding: DW_ATE_unsigned)
-!19 = !MDDerivedType(tag: DW_TAG_typedef, name: "string", line: 65, file: !20, scope: !10, baseType: !"_ZTSSs")
-!20 = !MDFile(filename: "/usr/lib/gcc/x86_64-linux-gnu/4.6/../../../../include/c++/4.6/bits/stringfwd.h", directory: "/llvm_cmake_gcc")
+!17 = !DIDerivedType(tag: DW_TAG_reference_type, baseType: !18)
+!18 = !DIBasicType(tag: DW_TAG_base_type, name: "long long unsigned int", size: 64, align: 64, encoding: DW_ATE_unsigned)
+!19 = !DIDerivedType(tag: DW_TAG_typedef, name: "string", line: 65, file: !20, scope: !10, baseType: !"_ZTSSs")
+!20 = !DIFile(filename: "/usr/lib/gcc/x86_64-linux-gnu/4.6/../../../../include/c++/4.6/bits/stringfwd.h", directory: "/llvm_cmake_gcc")
 !21 = !{!22, !26, !29, !33, !38, !41}
-!22 = !MDImportedEntity(tag: DW_TAG_imported_module, line: 57, scope: !23, entity: !25)
-!23 = !MDNamespace(name: "__gnu_debug", line: 55, file: !24, scope: null)
-!24 = !MDFile(filename: "/usr/lib/gcc/x86_64-linux-gnu/4.6/../../../../include/c++/4.6/debug/debug.h", directory: "/llvm_cmake_gcc")
-!25 = !MDNamespace(name: "__debug", line: 49, file: !24, scope: !10)
-!26 = !MDImportedEntity(tag: DW_TAG_imported_declaration, line: 66, scope: !10, entity: !27)
-!27 = !MDDerivedType(tag: DW_TAG_typedef, name: "mbstate_t", line: 106, file: !5, baseType: !28)
-!28 = !MDDerivedType(tag: DW_TAG_typedef, name: "__mbstate_t", line: 95, file: !5, baseType: !"_ZTS11__mbstate_t")
-!29 = !MDImportedEntity(tag: DW_TAG_imported_declaration, line: 141, scope: !10, entity: !30)
-!30 = !MDDerivedType(tag: DW_TAG_typedef, name: "wint_t", line: 141, file: !31, baseType: !32)
-!31 = !MDFile(filename: "/llvm_cmake_gcc/bin/../lib/clang/3.5.0/include/stddef.h", directory: "/llvm_cmake_gcc")
-!32 = !MDBasicType(tag: DW_TAG_base_type, name: "unsigned int", size: 32, align: 32, encoding: DW_ATE_unsigned)
-!33 = !MDImportedEntity(tag: DW_TAG_imported_declaration, line: 42, scope: !34, entity: !36)
-!34 = !MDNamespace(name: "__gnu_cxx", line: 69, file: !35, scope: null)
-!35 = !MDFile(filename: "/usr/lib/gcc/x86_64-linux-gnu/4.6/../../../../include/c++/4.6/bits/cpp_type_traits.h", directory: "/llvm_cmake_gcc")
-!36 = !MDDerivedType(tag: DW_TAG_typedef, name: "size_t", line: 155, file: !11, scope: !10, baseType: !37)
-!37 = !MDBasicType(tag: DW_TAG_base_type, name: "long unsigned int", size: 64, align: 64, encoding: DW_ATE_unsigned)
-!38 = !MDImportedEntity(tag: DW_TAG_imported_declaration, line: 43, scope: !34, entity: !39)
-!39 = !MDDerivedType(tag: DW_TAG_typedef, name: "ptrdiff_t", line: 156, file: !11, scope: !10, baseType: !40)
-!40 = !MDBasicType(tag: DW_TAG_base_type, name: "long int", size: 64, align: 64, encoding: DW_ATE_signed)
-!41 = !MDImportedEntity(tag: DW_TAG_imported_declaration, line: 55, scope: !10, entity: !"_ZTS5lconv")
+!22 = !DIImportedEntity(tag: DW_TAG_imported_module, line: 57, scope: !23, entity: !25)
+!23 = !DINamespace(name: "__gnu_debug", line: 55, file: !24, scope: null)
+!24 = !DIFile(filename: "/usr/lib/gcc/x86_64-linux-gnu/4.6/../../../../include/c++/4.6/debug/debug.h", directory: "/llvm_cmake_gcc")
+!25 = !DINamespace(name: "__debug", line: 49, file: !24, scope: !10)
+!26 = !DIImportedEntity(tag: DW_TAG_imported_declaration, line: 66, scope: !10, entity: !27)
+!27 = !DIDerivedType(tag: DW_TAG_typedef, name: "mbstate_t", line: 106, file: !5, baseType: !28)
+!28 = !DIDerivedType(tag: DW_TAG_typedef, name: "__mbstate_t", line: 95, file: !5, baseType: !"_ZTS11__mbstate_t")
+!29 = !DIImportedEntity(tag: DW_TAG_imported_declaration, line: 141, scope: !10, entity: !30)
+!30 = !DIDerivedType(tag: DW_TAG_typedef, name: "wint_t", line: 141, file: !31, baseType: !32)
+!31 = !DIFile(filename: "/llvm_cmake_gcc/bin/../lib/clang/3.5.0/include/stddef.h", directory: "/llvm_cmake_gcc")
+!32 = !DIBasicType(tag: DW_TAG_base_type, name: "unsigned int", size: 32, align: 32, encoding: DW_ATE_unsigned)
+!33 = !DIImportedEntity(tag: DW_TAG_imported_declaration, line: 42, scope: !34, entity: !36)
+!34 = !DINamespace(name: "__gnu_cxx", line: 69, file: !35, scope: null)
+!35 = !DIFile(filename: "/usr/lib/gcc/x86_64-linux-gnu/4.6/../../../../include/c++/4.6/bits/cpp_type_traits.h", directory: "/llvm_cmake_gcc")
+!36 = !DIDerivedType(tag: DW_TAG_typedef, name: "size_t", line: 155, file: !11, scope: !10, baseType: !37)
+!37 = !DIBasicType(tag: DW_TAG_base_type, name: "long unsigned int", size: 64, align: 64, encoding: DW_ATE_unsigned)
+!38 = !DIImportedEntity(tag: DW_TAG_imported_declaration, line: 43, scope: !34, entity: !39)
+!39 = !DIDerivedType(tag: DW_TAG_typedef, name: "ptrdiff_t", line: 156, file: !11, scope: !10, baseType: !40)
+!40 = !DIBasicType(tag: DW_TAG_base_type, name: "long int", size: 64, align: 64, encoding: DW_ATE_signed)
+!41 = !DIImportedEntity(tag: DW_TAG_imported_declaration, line: 55, scope: !10, entity: !"_ZTS5lconv")
 !42 = !{i32 2, !"Dwarf Version", i32 4}
 !43 = !{i32 2, !"Debug Info Version", i32 3}
 !44 = !{!"clang version 3.5.0 (209308)"}
-!45 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "offset", line: 3, arg: 1, scope: !13, file: !14, type: !17)
-!46 = !MDLocation(line: 3, scope: !13)
-!47 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "limit", line: 3, arg: 2, scope: !13, file: !14, type: !17)
-!48 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "range", line: 4, arg: 3, scope: !13, file: !14, type: !19)
-!49 = !MDLocation(line: 4, scope: !13)
-!50 = !MDLocation(line: 5, scope: !51)
-!51 = distinct !MDLexicalBlock(line: 5, column: 0, file: !1, scope: !13)
-!52 = !MDLocation(line: 5, scope: !53)
-!53 = distinct !MDLexicalBlock(line: 5, column: 0, file: !1, scope: !51)
-!54 = !MDLocation(line: 6, scope: !51)
-!55 = !MDLocation(line: 7, scope: !13)
-!56 = !MDLocation(line: 8, scope: !13)
-!57 = !MDLocation(line: 9, scope: !13)
+!45 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "offset", line: 3, arg: 1, scope: !13, file: !14, type: !17)
+!46 = !DILocation(line: 3, scope: !13)
+!47 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "limit", line: 3, arg: 2, scope: !13, file: !14, type: !17)
+!48 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "range", line: 4, arg: 3, scope: !13, file: !14, type: !19)
+!49 = !DILocation(line: 4, scope: !13)
+!50 = !DILocation(line: 5, scope: !51)
+!51 = distinct !DILexicalBlock(line: 5, column: 0, file: !1, scope: !13)
+!52 = !DILocation(line: 5, scope: !53)
+!53 = distinct !DILexicalBlock(line: 5, column: 0, file: !1, scope: !51)
+!54 = !DILocation(line: 6, scope: !51)
+!55 = !DILocation(line: 7, scope: !13)
+!56 = !DILocation(line: 8, scope: !13)
+!57 = !DILocation(line: 9, scope: !13)
 
diff --git a/test/DebugInfo/X86/processes-relocations.ll b/test/DebugInfo/X86/processes-relocations.ll
index 31efe468f7e..bb49e6bef78 100644
--- a/test/DebugInfo/X86/processes-relocations.ll
+++ b/test/DebugInfo/X86/processes-relocations.ll
@@ -13,8 +13,8 @@
 !llvm.module.flags = !{!3, !4}
 !llvm.ident = !{!5}
 
-!0 = !MDCompileUnit(file: !1, language: DW_LANG_C99, producer: "clang version 3.6.0 ", isOptimized: false, enums: !2, retainedTypes: !2, subprograms: !2, globals: !2, imports: !2, emissionKind: 1)
-!1 = !MDFile(filename: "empty.c", directory: "/a")
+!0 = !DICompileUnit(file: !1, language: DW_LANG_C99, producer: "clang version 3.6.0 ", isOptimized: false, enums: !2, retainedTypes: !2, subprograms: !2, globals: !2, imports: !2, emissionKind: 1)
+!1 = !DIFile(filename: "empty.c", directory: "/a")
 !2 = !{}
 !3 = !{i32 2, !"Dwarf Version", i32 4}
 !4 = !{i32 2, !"Debug Info Version", i32 3}
diff --git a/test/DebugInfo/X86/prologue-stack.ll b/test/DebugInfo/X86/prologue-stack.ll
index b9c029366b8..cd362e38095 100644
--- a/test/DebugInfo/X86/prologue-stack.ll
+++ b/test/DebugInfo/X86/prologue-stack.ll
@@ -21,16 +21,16 @@ declare i32 @callme(i32)
 !llvm.dbg.cu = !{!0}
 !llvm.module.flags = !{!14}
 
-!0 = !MDCompileUnit(language: DW_LANG_C99, producer: "clang version 3.2 (trunk 164980) (llvm/trunk 164979)", isOptimized: false, emissionKind: 0, file: !13, enums: !1, retainedTypes: !1, subprograms: !3, globals: !1, imports:  !1)
+!0 = !DICompileUnit(language: DW_LANG_C99, producer: "clang version 3.2 (trunk 164980) (llvm/trunk 164979)", isOptimized: false, emissionKind: 0, file: !13, enums: !1, retainedTypes: !1, subprograms: !3, globals: !1, imports:  !1)
 !1 = !{}
 !3 = !{!5}
-!5 = !MDSubprogram(name: "isel_line_test2", line: 3, isLocal: false, isDefinition: true, virtualIndex: 6, isOptimized: false, scopeLine: 4, file: !13, scope: !6, type: !7, function: i32 ()* @isel_line_test2, variables: !1)
-!6 = !MDFile(filename: "bar.c", directory: "/usr/local/google/home/echristo/tmp")
-!7 = !MDSubroutineType(types: !8)
+!5 = !DISubprogram(name: "isel_line_test2", line: 3, isLocal: false, isDefinition: true, virtualIndex: 6, isOptimized: false, scopeLine: 4, file: !13, scope: !6, type: !7, function: i32 ()* @isel_line_test2, variables: !1)
+!6 = !DIFile(filename: "bar.c", directory: "/usr/local/google/home/echristo/tmp")
+!7 = !DISubroutineType(types: !8)
 !8 = !{!9}
-!9 = !MDBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
-!10 = !MDLocation(line: 5, column: 3, scope: !11)
-!11 = distinct !MDLexicalBlock(line: 4, column: 1, file: !13, scope: !5)
-!12 = !MDLocation(line: 6, column: 3, scope: !11)
-!13 = !MDFile(filename: "bar.c", directory: "/usr/local/google/home/echristo/tmp")
+!9 = !DIBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
+!10 = !DILocation(line: 5, column: 3, scope: !11)
+!11 = distinct !DILexicalBlock(line: 4, column: 1, file: !13, scope: !5)
+!12 = !DILocation(line: 6, column: 3, scope: !11)
+!13 = !DIFile(filename: "bar.c", directory: "/usr/local/google/home/echristo/tmp")
 !14 = !{i32 1, !"Debug Info Version", i32 3}
diff --git a/test/DebugInfo/X86/recursive_inlining.ll b/test/DebugInfo/X86/recursive_inlining.ll
index e20be8357eb..7825646c7c8 100644
--- a/test/DebugInfo/X86/recursive_inlining.ll
+++ b/test/DebugInfo/X86/recursive_inlining.ll
@@ -95,7 +95,7 @@ define void @_Z3fn6v() #0 {
 entry:
   tail call void @_Z3fn8v() #3, !dbg !31
   %0 = load %struct.C*, %struct.C** @x, align 8, !dbg !32, !tbaa !33
-  tail call void @llvm.dbg.value(metadata %struct.C* %0, i64 0, metadata !37, metadata !MDExpression()) #3, !dbg !38
+  tail call void @llvm.dbg.value(metadata %struct.C* %0, i64 0, metadata !37, metadata !DIExpression()) #3, !dbg !38
   tail call void @_Z3fn8v() #3, !dbg !39
   %b.i = getelementptr inbounds %struct.C, %struct.C* %0, i64 0, i32 0, !dbg !40
   %1 = load i32, i32* %b.i, align 4, !dbg !40, !tbaa !42
@@ -116,7 +116,7 @@ declare void @_Z3fn8v() #1
 ; Function Attrs: nounwind
 define linkonce_odr void @_ZN1C5m_fn2Ev(%struct.C* nocapture readonly %this) #0 align 2 {
 entry:
-  tail call void @llvm.dbg.value(metadata %struct.C* %this, i64 0, metadata !24, metadata !MDExpression()), !dbg !49
+  tail call void @llvm.dbg.value(metadata %struct.C* %this, i64 0, metadata !24, metadata !DIExpression()), !dbg !49
   tail call void @_Z3fn8v() #3, !dbg !50
   %b = getelementptr inbounds %struct.C, %struct.C* %this, i64 0, i32 0, !dbg !51
   %0 = load i32, i32* %b, align 4, !dbg !51, !tbaa !42
@@ -130,7 +130,7 @@ if.then:                                          ; preds = %entry
 if.end:                                           ; preds = %entry, %if.then
   tail call void @_Z3fn8v() #3, !dbg !53
   %1 = load %struct.C*, %struct.C** @x, align 8, !dbg !56, !tbaa !33
-  tail call void @llvm.dbg.value(metadata %struct.C* %1, i64 0, metadata !57, metadata !MDExpression()) #3, !dbg !58
+  tail call void @llvm.dbg.value(metadata %struct.C* %1, i64 0, metadata !57, metadata !DIExpression()) #3, !dbg !58
   tail call void @_Z3fn8v() #3, !dbg !59
   %b.i.i = getelementptr inbounds %struct.C, %struct.C* %1, i64 0, i32 0, !dbg !60
   %2 = load i32, i32* %b.i.i, align 4, !dbg !60, !tbaa !42
@@ -154,7 +154,7 @@ entry:
 tailrecurse:                                      ; preds = %tailrecurse.backedge, %entry
   tail call void @_Z3fn8v() #3, !dbg !64
   %0 = load %struct.C*, %struct.C** @x, align 8, !dbg !66, !tbaa !33
-  tail call void @llvm.dbg.value(metadata %struct.C* %0, i64 0, metadata !67, metadata !MDExpression()) #3, !dbg !68
+  tail call void @llvm.dbg.value(metadata %struct.C* %0, i64 0, metadata !67, metadata !DIExpression()) #3, !dbg !68
   tail call void @_Z3fn8v() #3, !dbg !69
   %b.i.i = getelementptr inbounds %struct.C, %struct.C* %0, i64 0, i32 0, !dbg !70
   %1 = load i32, i32* %b.i.i, align 4, !dbg !70, !tbaa !42
@@ -199,77 +199,77 @@ attributes #3 = { nounwind }
 !llvm.module.flags = !{!28, !29}
 !llvm.ident = !{!30}
 
-!0 = !MDCompileUnit(language: DW_LANG_C_plus_plus, producer: "clang version 3.6.0 ", isOptimized: true, emissionKind: 1, file: !1, enums: !2, retainedTypes: !3, subprograms: !13, globals: !26, imports: !2)
-!1 = !MDFile(filename: "<stdin>", directory: "/usr/local/google/home/blaikie/dev/scratch/missing_concrete_variable_on_darwin/reduce")
+!0 = !DICompileUnit(language: DW_LANG_C_plus_plus, producer: "clang version 3.6.0 ", isOptimized: true, emissionKind: 1, file: !1, enums: !2, retainedTypes: !3, subprograms: !13, globals: !26, imports: !2)
+!1 = !DIFile(filename: "<stdin>", directory: "/usr/local/google/home/blaikie/dev/scratch/missing_concrete_variable_on_darwin/reduce")
 !2 = !{}
 !3 = !{!4}
-!4 = !MDCompositeType(tag: DW_TAG_structure_type, name: "C", line: 5, size: 32, align: 32, file: !5, elements: !6, identifier: "_ZTS1C")
-!5 = !MDFile(filename: "recursive_inlining.cpp", directory: "/usr/local/google/home/blaikie/dev/scratch/missing_concrete_variable_on_darwin/reduce")
+!4 = !DICompositeType(tag: DW_TAG_structure_type, name: "C", line: 5, size: 32, align: 32, file: !5, elements: !6, identifier: "_ZTS1C")
+!5 = !DIFile(filename: "recursive_inlining.cpp", directory: "/usr/local/google/home/blaikie/dev/scratch/missing_concrete_variable_on_darwin/reduce")
 !6 = !{!7, !9}
-!7 = !MDDerivedType(tag: DW_TAG_member, name: "b", line: 6, size: 32, align: 32, file: !5, scope: !"_ZTS1C", baseType: !8)
-!8 = !MDBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
-!9 = !MDSubprogram(name: "m_fn2", linkageName: "_ZN1C5m_fn2Ev", line: 7, isLocal: false, isDefinition: false, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: true, scopeLine: 7, file: !5, scope: !"_ZTS1C", type: !10)
-!10 = !MDSubroutineType(types: !11)
+!7 = !DIDerivedType(tag: DW_TAG_member, name: "b", line: 6, size: 32, align: 32, file: !5, scope: !"_ZTS1C", baseType: !8)
+!8 = !DIBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
+!9 = !DISubprogram(name: "m_fn2", linkageName: "_ZN1C5m_fn2Ev", line: 7, isLocal: false, isDefinition: false, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: true, scopeLine: 7, file: !5, scope: !"_ZTS1C", type: !10)
+!10 = !DISubroutineType(types: !11)
 !11 = !{null, !12}
-!12 = !MDDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, flags: DIFlagArtificial | DIFlagObjectPointer, baseType: !"_ZTS1C")
+!12 = !DIDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, flags: DIFlagArtificial | DIFlagObjectPointer, baseType: !"_ZTS1C")
 !13 = !{!14, !18, !19, !20, !21, !22}
-!14 = !MDSubprogram(name: "fn6", linkageName: "_Z3fn6v", line: 15, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: true, scopeLine: 15, file: !5, scope: !15, type: !16, function: void ()* @_Z3fn6v, variables: !2)
-!15 = !MDFile(filename: "recursive_inlining.cpp", directory: "/usr/local/google/home/blaikie/dev/scratch/missing_concrete_variable_on_darwin/reduce")
-!16 = !MDSubroutineType(types: !17)
+!14 = !DISubprogram(name: "fn6", linkageName: "_Z3fn6v", line: 15, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: true, scopeLine: 15, file: !5, scope: !15, type: !16, function: void ()* @_Z3fn6v, variables: !2)
+!15 = !DIFile(filename: "recursive_inlining.cpp", directory: "/usr/local/google/home/blaikie/dev/scratch/missing_concrete_variable_on_darwin/reduce")
+!16 = !DISubroutineType(types: !17)
 !17 = !{null}
-!18 = !MDSubprogram(name: "fn3", linkageName: "_Z3fn3v", line: 20, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: true, scopeLine: 20, file: !5, scope: !15, type: !16, function: void ()* @_Z3fn3v, variables: !2)
-!19 = !MDSubprogram(name: "fn4", linkageName: "_Z3fn4v", line: 21, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: true, scopeLine: 21, file: !5, scope: !15, type: !16, function: void ()* @_Z3fn4v, variables: !2)
-!20 = !MDSubprogram(name: "fn5", linkageName: "_Z3fn5v", line: 22, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: true, scopeLine: 22, file: !5, scope: !15, type: !16, function: void ()* @_Z3fn5v, variables: !2)
-!21 = !MDSubprogram(name: "fn7", linkageName: "_Z3fn7v", line: 14, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: true, scopeLine: 14, file: !5, scope: !15, type: !16, variables: !2)
-!22 = !MDSubprogram(name: "m_fn2", linkageName: "_ZN1C5m_fn2Ev", line: 7, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: true, scopeLine: 7, file: !5, scope: !"_ZTS1C", type: !10, function: void (%struct.C*)* @_ZN1C5m_fn2Ev, declaration: !9, variables: !23)
+!18 = !DISubprogram(name: "fn3", linkageName: "_Z3fn3v", line: 20, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: true, scopeLine: 20, file: !5, scope: !15, type: !16, function: void ()* @_Z3fn3v, variables: !2)
+!19 = !DISubprogram(name: "fn4", linkageName: "_Z3fn4v", line: 21, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: true, scopeLine: 21, file: !5, scope: !15, type: !16, function: void ()* @_Z3fn4v, variables: !2)
+!20 = !DISubprogram(name: "fn5", linkageName: "_Z3fn5v", line: 22, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: true, scopeLine: 22, file: !5, scope: !15, type: !16, function: void ()* @_Z3fn5v, variables: !2)
+!21 = !DISubprogram(name: "fn7", linkageName: "_Z3fn7v", line: 14, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: true, scopeLine: 14, file: !5, scope: !15, type: !16, variables: !2)
+!22 = !DISubprogram(name: "m_fn2", linkageName: "_ZN1C5m_fn2Ev", line: 7, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: true, scopeLine: 7, file: !5, scope: !"_ZTS1C", type: !10, function: void (%struct.C*)* @_ZN1C5m_fn2Ev, declaration: !9, variables: !23)
 !23 = !{!24}
-!24 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "this", arg: 1, flags: DIFlagArtificial | DIFlagObjectPointer, scope: !22, type: !25)
-!25 = !MDDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, baseType: !"_ZTS1C")
+!24 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "this", arg: 1, flags: DIFlagArtificial | DIFlagObjectPointer, scope: !22, type: !25)
+!25 = !DIDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, baseType: !"_ZTS1C")
 !26 = !{!27}
-!27 = !MDGlobalVariable(name: "x", line: 13, isLocal: false, isDefinition: true, scope: null, file: !15, type: !25, variable: %struct.C** @x)
+!27 = !DIGlobalVariable(name: "x", line: 13, isLocal: false, isDefinition: true, scope: null, file: !15, type: !25, variable: %struct.C** @x)
 !28 = !{i32 2, !"Dwarf Version", i32 4}
 !29 = !{i32 2, !"Debug Info Version", i32 3}
 !30 = !{!"clang version 3.6.0 "}
-!31 = !MDLocation(line: 16, scope: !14)
-!32 = !MDLocation(line: 17, scope: !14)
+!31 = !DILocation(line: 16, scope: !14)
+!32 = !DILocation(line: 17, scope: !14)
 !33 = !{!34, !34, i64 0}
 !34 = !{!"any pointer", !35, i64 0}
 !35 = !{!"omnipotent char", !36, i64 0}
 !36 = !{!"Simple C/C++ TBAA"}
-!37 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "this", arg: 1, flags: DIFlagArtificial | DIFlagObjectPointer, scope: !22, type: !25)
-!38 = !MDLocation(line: 0, scope: !22, inlinedAt: !32)
-!39 = !MDLocation(line: 8, scope: !22, inlinedAt: !32)
-!40 = !MDLocation(line: 9, scope: !41, inlinedAt: !32)
-!41 = distinct !MDLexicalBlock(line: 9, column: 0, file: !5, scope: !22)
+!37 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "this", arg: 1, flags: DIFlagArtificial | DIFlagObjectPointer, scope: !22, type: !25)
+!38 = !DILocation(line: 0, scope: !22, inlinedAt: !32)
+!39 = !DILocation(line: 8, scope: !22, inlinedAt: !32)
+!40 = !DILocation(line: 9, scope: !41, inlinedAt: !32)
+!41 = distinct !DILexicalBlock(line: 9, column: 0, file: !5, scope: !22)
 !42 = !{!43, !44, i64 0}
 !43 = !{!"_ZTS1C", !44, i64 0}
 !44 = !{!"int", !35, i64 0}
-!45 = !MDLocation(line: 9, scope: !46, inlinedAt: !32)
-!46 = distinct !MDLexicalBlock(line: 9, column: 0, file: !5, scope: !41)
-!47 = !MDLocation(line: 10, scope: !22, inlinedAt: !32)
-!48 = !MDLocation(line: 19, scope: !14)
-!49 = !MDLocation(line: 0, scope: !22)
-!50 = !MDLocation(line: 8, scope: !22)
-!51 = !MDLocation(line: 9, scope: !41)
-!52 = !MDLocation(line: 9, scope: !46)
-!53 = !MDLocation(line: 16, scope: !14, inlinedAt: !54)
-!54 = !MDLocation(line: 20, scope: !18, inlinedAt: !55)
-!55 = !MDLocation(line: 10, scope: !22)
-!56 = !MDLocation(line: 17, scope: !14, inlinedAt: !54)
-!57 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "this", arg: 1, flags: DIFlagArtificial | DIFlagObjectPointer, scope: !22, type: !25)
-!58 = !MDLocation(line: 0, scope: !22, inlinedAt: !56)
-!59 = !MDLocation(line: 8, scope: !22, inlinedAt: !56)
-!60 = !MDLocation(line: 9, scope: !41, inlinedAt: !56)
-!61 = !MDLocation(line: 9, scope: !46, inlinedAt: !56)
-!62 = !MDLocation(line: 10, scope: !22, inlinedAt: !56)
-!63 = !MDLocation(line: 11, scope: !22)
-!64 = !MDLocation(line: 16, scope: !14, inlinedAt: !65)
-!65 = !MDLocation(line: 20, scope: !18)
-!66 = !MDLocation(line: 17, scope: !14, inlinedAt: !65)
-!67 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "this", arg: 1, flags: DIFlagArtificial | DIFlagObjectPointer, scope: !22, type: !25)
-!68 = !MDLocation(line: 0, scope: !22, inlinedAt: !66)
-!69 = !MDLocation(line: 8, scope: !22, inlinedAt: !66)
-!70 = !MDLocation(line: 9, scope: !41, inlinedAt: !66)
-!71 = !MDLocation(line: 9, scope: !46, inlinedAt: !66)
-!72 = !MDLocation(line: 21, scope: !19)
-!73 = !MDLocation(line: 22, scope: !20)
+!45 = !DILocation(line: 9, scope: !46, inlinedAt: !32)
+!46 = distinct !DILexicalBlock(line: 9, column: 0, file: !5, scope: !41)
+!47 = !DILocation(line: 10, scope: !22, inlinedAt: !32)
+!48 = !DILocation(line: 19, scope: !14)
+!49 = !DILocation(line: 0, scope: !22)
+!50 = !DILocation(line: 8, scope: !22)
+!51 = !DILocation(line: 9, scope: !41)
+!52 = !DILocation(line: 9, scope: !46)
+!53 = !DILocation(line: 16, scope: !14, inlinedAt: !54)
+!54 = !DILocation(line: 20, scope: !18, inlinedAt: !55)
+!55 = !DILocation(line: 10, scope: !22)
+!56 = !DILocation(line: 17, scope: !14, inlinedAt: !54)
+!57 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "this", arg: 1, flags: DIFlagArtificial | DIFlagObjectPointer, scope: !22, type: !25)
+!58 = !DILocation(line: 0, scope: !22, inlinedAt: !56)
+!59 = !DILocation(line: 8, scope: !22, inlinedAt: !56)
+!60 = !DILocation(line: 9, scope: !41, inlinedAt: !56)
+!61 = !DILocation(line: 9, scope: !46, inlinedAt: !56)
+!62 = !DILocation(line: 10, scope: !22, inlinedAt: !56)
+!63 = !DILocation(line: 11, scope: !22)
+!64 = !DILocation(line: 16, scope: !14, inlinedAt: !65)
+!65 = !DILocation(line: 20, scope: !18)
+!66 = !DILocation(line: 17, scope: !14, inlinedAt: !65)
+!67 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "this", arg: 1, flags: DIFlagArtificial | DIFlagObjectPointer, scope: !22, type: !25)
+!68 = !DILocation(line: 0, scope: !22, inlinedAt: !66)
+!69 = !DILocation(line: 8, scope: !22, inlinedAt: !66)
+!70 = !DILocation(line: 9, scope: !41, inlinedAt: !66)
+!71 = !DILocation(line: 9, scope: !46, inlinedAt: !66)
+!72 = !DILocation(line: 21, scope: !19)
+!73 = !DILocation(line: 22, scope: !20)
diff --git a/test/DebugInfo/X86/ref_addr_relocation.ll b/test/DebugInfo/X86/ref_addr_relocation.ll
index 046419062ec..488f98ef8e1 100644
--- a/test/DebugInfo/X86/ref_addr_relocation.ll
+++ b/test/DebugInfo/X86/ref_addr_relocation.ll
@@ -58,19 +58,19 @@
 !llvm.dbg.cu = !{!0, !9}
 !llvm.module.flags = !{!14, !15}
 
-!0 = !MDCompileUnit(language: DW_LANG_C_plus_plus, producer: "clang version 3.4 (trunk 191799)", isOptimized: false, emissionKind: 0, file: !1, enums: !2, retainedTypes: !3, subprograms: !2, globals: !6, imports: !2)
-!1 = !MDFile(filename: "tu1.cpp", directory: "/Users/manmanren/test-Nov/type_unique_air/ref_addr")
+!0 = !DICompileUnit(language: DW_LANG_C_plus_plus, producer: "clang version 3.4 (trunk 191799)", isOptimized: false, emissionKind: 0, file: !1, enums: !2, retainedTypes: !3, subprograms: !2, globals: !6, imports: !2)
+!1 = !DIFile(filename: "tu1.cpp", directory: "/Users/manmanren/test-Nov/type_unique_air/ref_addr")
 !2 = !{}
 !3 = !{!4}
-!4 = !MDCompositeType(tag: DW_TAG_structure_type, name: "foo", line: 1, size: 8, align: 8, file: !5, elements: !2, identifier: "_ZTS3foo")
-!5 = !MDFile(filename: "./hdr.h", directory: "/Users/manmanren/test-Nov/type_unique_air/ref_addr")
+!4 = !DICompositeType(tag: DW_TAG_structure_type, name: "foo", line: 1, size: 8, align: 8, file: !5, elements: !2, identifier: "_ZTS3foo")
+!5 = !DIFile(filename: "./hdr.h", directory: "/Users/manmanren/test-Nov/type_unique_air/ref_addr")
 !6 = !{!7}
-!7 = !MDGlobalVariable(name: "f", line: 2, isLocal: false, isDefinition: true, scope: null, file: !8, type: !4, variable: %struct.foo* @f)
-!8 = !MDFile(filename: "tu1.cpp", directory: "/Users/manmanren/test-Nov/type_unique_air/ref_addr")
-!9 = !MDCompileUnit(language: DW_LANG_C_plus_plus, producer: "clang version 3.4 (trunk 191799)", isOptimized: false, emissionKind: 0, file: !10, enums: !2, retainedTypes: !3, subprograms: !2, globals: !11, imports: !2)
-!10 = !MDFile(filename: "tu2.cpp", directory: "/Users/manmanren/test-Nov/type_unique_air/ref_addr")
+!7 = !DIGlobalVariable(name: "f", line: 2, isLocal: false, isDefinition: true, scope: null, file: !8, type: !4, variable: %struct.foo* @f)
+!8 = !DIFile(filename: "tu1.cpp", directory: "/Users/manmanren/test-Nov/type_unique_air/ref_addr")
+!9 = !DICompileUnit(language: DW_LANG_C_plus_plus, producer: "clang version 3.4 (trunk 191799)", isOptimized: false, emissionKind: 0, file: !10, enums: !2, retainedTypes: !3, subprograms: !2, globals: !11, imports: !2)
+!10 = !DIFile(filename: "tu2.cpp", directory: "/Users/manmanren/test-Nov/type_unique_air/ref_addr")
 !11 = !{!12}
-!12 = !MDGlobalVariable(name: "g", line: 2, isLocal: false, isDefinition: true, scope: null, file: !13, type: !4, variable: %struct.foo* @g)
-!13 = !MDFile(filename: "tu2.cpp", directory: "/Users/manmanren/test-Nov/type_unique_air/ref_addr")
+!12 = !DIGlobalVariable(name: "g", line: 2, isLocal: false, isDefinition: true, scope: null, file: !13, type: !4, variable: %struct.foo* @g)
+!13 = !DIFile(filename: "tu2.cpp", directory: "/Users/manmanren/test-Nov/type_unique_air/ref_addr")
 !14 = !{i32 2, !"Dwarf Version", i32 2}
 !15 = !{i32 1, !"Debug Info Version", i32 3}
diff --git a/test/DebugInfo/X86/reference-argument.ll b/test/DebugInfo/X86/reference-argument.ll
index a4e5aba13db..401553ec971 100644
--- a/test/DebugInfo/X86/reference-argument.ll
+++ b/test/DebugInfo/X86/reference-argument.ll
@@ -20,8 +20,8 @@ define linkonce_odr void @_ZN1A3fooE4SVal(%class.A* %this, %class.SVal* %v) noun
 entry:
   %this.addr = alloca %class.A*, align 8
   store %class.A* %this, %class.A** %this.addr, align 8
-  call void @llvm.dbg.declare(metadata %class.A** %this.addr, metadata !59, metadata !MDExpression()), !dbg !61
-  call void @llvm.dbg.declare(metadata %class.SVal* %v, metadata !62, metadata !MDExpression(DW_OP_deref)), !dbg !61
+  call void @llvm.dbg.declare(metadata %class.A** %this.addr, metadata !59, metadata !DIExpression()), !dbg !61
+  call void @llvm.dbg.declare(metadata %class.SVal* %v, metadata !62, metadata !DIExpression(DW_OP_deref)), !dbg !61
   %this1 = load %class.A*, %class.A** %this.addr
   call void @_Z3barR4SVal(%class.SVal* %v), !dbg !61
   ret void, !dbg !61
@@ -32,67 +32,67 @@ declare void @_ZN4SValD2Ev(%class.SVal* %this)
 !llvm.dbg.cu = !{!0}
 !llvm.module.flags = !{!47, !68}
 
-!0 = !MDCompileUnit(language: DW_LANG_C_plus_plus, producer: "clang version 3.4 ", isOptimized: false, emissionKind: 0, file: !1, enums: !2, retainedTypes: !2, subprograms: !3, globals: !2, imports: !2)
-!1 = !MDFile(filename: "aggregate-indirect-arg.cpp", directory: "")
+!0 = !DICompileUnit(language: DW_LANG_C_plus_plus, producer: "clang version 3.4 ", isOptimized: false, emissionKind: 0, file: !1, enums: !2, retainedTypes: !2, subprograms: !3, globals: !2, imports: !2)
+!1 = !DIFile(filename: "aggregate-indirect-arg.cpp", directory: "")
 !2 = !{}
 !3 = !{!4, !29, !33, !34, !35}
-!4 = !MDSubprogram(name: "bar", linkageName: "_Z3barR4SVal", line: 19, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 19, file: !1, scope: !5, type: !6, function: void (%class.SVal*)* @_Z3barR4SVal, variables: !2)
-!5 = !MDFile(filename: "aggregate-indirect-arg.cpp", directory: "")
-!6 = !MDSubroutineType(types: !7)
+!4 = !DISubprogram(name: "bar", linkageName: "_Z3barR4SVal", line: 19, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 19, file: !1, scope: !5, type: !6, function: void (%class.SVal*)* @_Z3barR4SVal, variables: !2)
+!5 = !DIFile(filename: "aggregate-indirect-arg.cpp", directory: "")
+!6 = !DISubroutineType(types: !7)
 !7 = !{null, !8}
-!8 = !MDDerivedType(tag: DW_TAG_reference_type, baseType: !9)
-!9 = !MDCompositeType(tag: DW_TAG_class_type, name: "SVal", line: 12, size: 128, align: 64, file: !1, elements: !10)
+!8 = !DIDerivedType(tag: DW_TAG_reference_type, baseType: !9)
+!9 = !DICompositeType(tag: DW_TAG_class_type, name: "SVal", line: 12, size: 128, align: 64, file: !1, elements: !10)
 !10 = !{!11, !14, !16, !21, !23}
-!11 = !MDDerivedType(tag: DW_TAG_member, name: "Data", line: 15, size: 64, align: 64, file: !1, scope: !9, baseType: !12)
-!12 = !MDDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, baseType: !13)
-!13 = !MDDerivedType(tag: DW_TAG_const_type, baseType: null)
-!14 = !MDDerivedType(tag: DW_TAG_member, name: "Kind", line: 16, size: 32, align: 32, offset: 64, file: !1, scope: !9, baseType: !15)
-!15 = !MDBasicType(tag: DW_TAG_base_type, name: "unsigned int", size: 32, align: 32, encoding: DW_ATE_unsigned)
-!16 = !MDSubprogram(name: "~SVal", line: 14, isLocal: false, isDefinition: false, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 14, file: !1, scope: !9, type: !17)
-!17 = !MDSubroutineType(types: !18)
+!11 = !DIDerivedType(tag: DW_TAG_member, name: "Data", line: 15, size: 64, align: 64, file: !1, scope: !9, baseType: !12)
+!12 = !DIDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, baseType: !13)
+!13 = !DIDerivedType(tag: DW_TAG_const_type, baseType: null)
+!14 = !DIDerivedType(tag: DW_TAG_member, name: "Kind", line: 16, size: 32, align: 32, offset: 64, file: !1, scope: !9, baseType: !15)
+!15 = !DIBasicType(tag: DW_TAG_base_type, name: "unsigned int", size: 32, align: 32, encoding: DW_ATE_unsigned)
+!16 = !DISubprogram(name: "~SVal", line: 14, isLocal: false, isDefinition: false, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 14, file: !1, scope: !9, type: !17)
+!17 = !DISubroutineType(types: !18)
 !18 = !{null, !19}
-!19 = !MDDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, flags: DIFlagArtificial | DIFlagObjectPointer, baseType: !9)
-!21 = !MDSubprogram(name: "SVal", line: 12, isLocal: false, isDefinition: false, virtualIndex: 6, flags: DIFlagArtificial | DIFlagPrototyped, isOptimized: false, scopeLine: 12, file: !1, scope: !9, type: !17)
-!23 = !MDSubprogram(name: "SVal", line: 12, isLocal: false, isDefinition: false, virtualIndex: 6, flags: DIFlagArtificial | DIFlagPrototyped, isOptimized: false, scopeLine: 12, file: !1, scope: !9, type: !24)
-!24 = !MDSubroutineType(types: !25)
+!19 = !DIDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, flags: DIFlagArtificial | DIFlagObjectPointer, baseType: !9)
+!21 = !DISubprogram(name: "SVal", line: 12, isLocal: false, isDefinition: false, virtualIndex: 6, flags: DIFlagArtificial | DIFlagPrototyped, isOptimized: false, scopeLine: 12, file: !1, scope: !9, type: !17)
+!23 = !DISubprogram(name: "SVal", line: 12, isLocal: false, isDefinition: false, virtualIndex: 6, flags: DIFlagArtificial | DIFlagPrototyped, isOptimized: false, scopeLine: 12, file: !1, scope: !9, type: !24)
+!24 = !DISubroutineType(types: !25)
 !25 = !{null, !19, !26}
-!26 = !MDDerivedType(tag: DW_TAG_reference_type, baseType: !27)
-!27 = !MDDerivedType(tag: DW_TAG_const_type, baseType: !9)
-!29 = !MDSubprogram(name: "main", line: 25, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 25, file: !1, scope: !5, type: !30, function: i32 ()* @main, variables: !2)
-!30 = !MDSubroutineType(types: !31)
+!26 = !DIDerivedType(tag: DW_TAG_reference_type, baseType: !27)
+!27 = !DIDerivedType(tag: DW_TAG_const_type, baseType: !9)
+!29 = !DISubprogram(name: "main", line: 25, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 25, file: !1, scope: !5, type: !30, function: i32 ()* @main, variables: !2)
+!30 = !DISubroutineType(types: !31)
 !31 = !{!32}
-!32 = !MDBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
-!33 = !MDSubprogram(name: "~SVal", linkageName: "_ZN4SValD1Ev", line: 14, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 14, file: !1, scope: null, type: !17, function: void (%class.SVal*)* @_ZN4SValD1Ev, declaration: !16, variables: !2)
-!34 = !MDSubprogram(name: "~SVal", linkageName: "_ZN4SValD2Ev", line: 14, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 14, file: !1, scope: null, type: !17, function: void (%class.SVal*)* @_ZN4SValD2Ev, declaration: !16, variables: !2)
-!35 = !MDSubprogram(name: "foo", linkageName: "_ZN1A3fooE4SVal", line: 22, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 22, file: !1, scope: null, type: !36, function: void (%class.A*, %class.SVal*)* @_ZN1A3fooE4SVal, declaration: !41, variables: !2)
-!36 = !MDSubroutineType(types: !37)
+!32 = !DIBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
+!33 = !DISubprogram(name: "~SVal", linkageName: "_ZN4SValD1Ev", line: 14, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 14, file: !1, scope: null, type: !17, function: void (%class.SVal*)* @_ZN4SValD1Ev, declaration: !16, variables: !2)
+!34 = !DISubprogram(name: "~SVal", linkageName: "_ZN4SValD2Ev", line: 14, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 14, file: !1, scope: null, type: !17, function: void (%class.SVal*)* @_ZN4SValD2Ev, declaration: !16, variables: !2)
+!35 = !DISubprogram(name: "foo", linkageName: "_ZN1A3fooE4SVal", line: 22, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 22, file: !1, scope: null, type: !36, function: void (%class.A*, %class.SVal*)* @_ZN1A3fooE4SVal, declaration: !41, variables: !2)
+!36 = !DISubroutineType(types: !37)
 !37 = !{null, !38, !9}
-!38 = !MDDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, flags: DIFlagArtificial | DIFlagObjectPointer, baseType: !39)
-!39 = !MDCompositeType(tag: DW_TAG_class_type, name: "A", line: 20, size: 8, align: 8, file: !1, elements: !40)
+!38 = !DIDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, flags: DIFlagArtificial | DIFlagObjectPointer, baseType: !39)
+!39 = !DICompositeType(tag: DW_TAG_class_type, name: "A", line: 20, size: 8, align: 8, file: !1, elements: !40)
 !40 = !{!41, !43}
-!41 = !MDSubprogram(name: "foo", linkageName: "_ZN1A3fooE4SVal", line: 22, isLocal: false, isDefinition: false, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 22, file: !1, scope: !39, type: !36)
-!43 = !MDSubprogram(name: "A", line: 20, isLocal: false, isDefinition: false, virtualIndex: 6, flags: DIFlagArtificial | DIFlagPrototyped, isOptimized: false, scopeLine: 20, file: !1, scope: !39, type: !44)
-!44 = !MDSubroutineType(types: !45)
+!41 = !DISubprogram(name: "foo", linkageName: "_ZN1A3fooE4SVal", line: 22, isLocal: false, isDefinition: false, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 22, file: !1, scope: !39, type: !36)
+!43 = !DISubprogram(name: "A", line: 20, isLocal: false, isDefinition: false, virtualIndex: 6, flags: DIFlagArtificial | DIFlagPrototyped, isOptimized: false, scopeLine: 20, file: !1, scope: !39, type: !44)
+!44 = !DISubroutineType(types: !45)
 !45 = !{null, !38}
 !47 = !{i32 2, !"Dwarf Version", i32 3}
-!48 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "v", line: 19, arg: 1, scope: !4, file: !5, type: !8)
-!49 = !MDLocation(line: 19, scope: !4)
-!50 = !MDLocalVariable(tag: DW_TAG_auto_variable, name: "v", line: 26, scope: !29, file: !5, type: !9)
-!51 = !MDLocation(line: 26, scope: !29)
-!52 = !MDLocation(line: 27, scope: !29)
-!53 = !MDLocation(line: 28, scope: !29)
-!54 = !MDLocalVariable(tag: DW_TAG_auto_variable, name: "a", line: 29, scope: !29, file: !5, type: !39)
-!55 = !MDLocation(line: 29, scope: !29)
-!56 = !MDLocation(line: 30, scope: !29)
-!57 = !MDLocation(line: 31, scope: !29)
-!58 = !MDLocation(line: 32, scope: !29)
-!59 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "this", line: 22, arg: 1, flags: DIFlagArtificial | DIFlagObjectPointer, scope: !35, file: !5, type: !60)
-!60 = !MDDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, baseType: !39)
-!61 = !MDLocation(line: 22, scope: !35)
-!62 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "v", line: 22, arg: 2, scope: !35, file: !5, type: !9)
-!63 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "this", line: 14, arg: 1, flags: DIFlagArtificial | DIFlagObjectPointer, scope: !33, file: !5, type: !64)
-!64 = !MDDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, baseType: !9)
-!65 = !MDLocation(line: 14, scope: !33)
-!66 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "this", line: 14, arg: 1, flags: DIFlagArtificial | DIFlagObjectPointer, scope: !34, file: !5, type: !64)
-!67 = !MDLocation(line: 14, scope: !34)
+!48 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "v", line: 19, arg: 1, scope: !4, file: !5, type: !8)
+!49 = !DILocation(line: 19, scope: !4)
+!50 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "v", line: 26, scope: !29, file: !5, type: !9)
+!51 = !DILocation(line: 26, scope: !29)
+!52 = !DILocation(line: 27, scope: !29)
+!53 = !DILocation(line: 28, scope: !29)
+!54 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "a", line: 29, scope: !29, file: !5, type: !39)
+!55 = !DILocation(line: 29, scope: !29)
+!56 = !DILocation(line: 30, scope: !29)
+!57 = !DILocation(line: 31, scope: !29)
+!58 = !DILocation(line: 32, scope: !29)
+!59 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "this", line: 22, arg: 1, flags: DIFlagArtificial | DIFlagObjectPointer, scope: !35, file: !5, type: !60)
+!60 = !DIDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, baseType: !39)
+!61 = !DILocation(line: 22, scope: !35)
+!62 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "v", line: 22, arg: 2, scope: !35, file: !5, type: !9)
+!63 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "this", line: 14, arg: 1, flags: DIFlagArtificial | DIFlagObjectPointer, scope: !33, file: !5, type: !64)
+!64 = !DIDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, baseType: !9)
+!65 = !DILocation(line: 14, scope: !33)
+!66 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "this", line: 14, arg: 1, flags: DIFlagArtificial | DIFlagObjectPointer, scope: !34, file: !5, type: !64)
+!67 = !DILocation(line: 14, scope: !34)
 !68 = !{i32 1, !"Debug Info Version", i32 3}
diff --git a/test/DebugInfo/X86/rvalue-ref.ll b/test/DebugInfo/X86/rvalue-ref.ll
index c6f00076098..4e4da67beb0 100644
--- a/test/DebugInfo/X86/rvalue-ref.ll
+++ b/test/DebugInfo/X86/rvalue-ref.ll
@@ -9,7 +9,7 @@ define void @_Z3fooOi(i32* %i) uwtable ssp {
 entry:
   %i.addr = alloca i32*, align 8
   store i32* %i, i32** %i.addr, align 8
-  call void @llvm.dbg.declare(metadata i32** %i.addr, metadata !11, metadata !MDExpression()), !dbg !12
+  call void @llvm.dbg.declare(metadata i32** %i.addr, metadata !11, metadata !DIExpression()), !dbg !12
   %0 = load i32*, i32** %i.addr, align 8, !dbg !13
   %1 = load i32, i32* %0, align 4, !dbg !13
   %call = call i32 (i8*, ...) @printf(i8* getelementptr inbounds ([4 x i8], [4 x i8]* @.str, i32 0, i32 0), i32 %1), !dbg !13
@@ -23,19 +23,19 @@ declare i32 @printf(i8*, ...)
 !llvm.dbg.cu = !{!0}
 !llvm.module.flags = !{!17}
 
-!0 = !MDCompileUnit(language: DW_LANG_C_plus_plus, producer: "clang version 3.2 (trunk 157054) (llvm/trunk 157060)", isOptimized: false, emissionKind: 0, file: !16, enums: !1, retainedTypes: !1, subprograms: !3, globals: !1, imports:  !1)
+!0 = !DICompileUnit(language: DW_LANG_C_plus_plus, producer: "clang version 3.2 (trunk 157054) (llvm/trunk 157060)", isOptimized: false, emissionKind: 0, file: !16, enums: !1, retainedTypes: !1, subprograms: !3, globals: !1, imports:  !1)
 !1 = !{}
 !3 = !{!5}
-!5 = !MDSubprogram(name: "foo", linkageName: "_Z3fooOi", line: 4, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 5, file: !16, scope: !6, type: !7, function: void (i32*)* @_Z3fooOi, variables: !1)
-!6 = !MDFile(filename: "foo.cpp", directory: "/Users/echristo/tmp")
-!7 = !MDSubroutineType(types: !8)
+!5 = !DISubprogram(name: "foo", linkageName: "_Z3fooOi", line: 4, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 5, file: !16, scope: !6, type: !7, function: void (i32*)* @_Z3fooOi, variables: !1)
+!6 = !DIFile(filename: "foo.cpp", directory: "/Users/echristo/tmp")
+!7 = !DISubroutineType(types: !8)
 !8 = !{null, !9}
-!9 = !MDDerivedType(tag: DW_TAG_rvalue_reference_type, baseType: !10)
-!10 = !MDBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
-!11 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "i", line: 4, arg: 1, scope: !5, file: !6, type: !9)
-!12 = !MDLocation(line: 4, column: 17, scope: !5)
-!13 = !MDLocation(line: 6, column: 3, scope: !14)
-!14 = distinct !MDLexicalBlock(line: 5, column: 1, file: !16, scope: !5)
-!15 = !MDLocation(line: 7, column: 1, scope: !14)
-!16 = !MDFile(filename: "foo.cpp", directory: "/Users/echristo/tmp")
+!9 = !DIDerivedType(tag: DW_TAG_rvalue_reference_type, baseType: !10)
+!10 = !DIBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
+!11 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "i", line: 4, arg: 1, scope: !5, file: !6, type: !9)
+!12 = !DILocation(line: 4, column: 17, scope: !5)
+!13 = !DILocation(line: 6, column: 3, scope: !14)
+!14 = distinct !DILexicalBlock(line: 5, column: 1, file: !16, scope: !5)
+!15 = !DILocation(line: 7, column: 1, scope: !14)
+!16 = !DIFile(filename: "foo.cpp", directory: "/Users/echristo/tmp")
 !17 = !{i32 1, !"Debug Info Version", i32 3}
diff --git a/test/DebugInfo/X86/sret.ll b/test/DebugInfo/X86/sret.ll
index 07771177dd2..ef8f2e6d65e 100644
--- a/test/DebugInfo/X86/sret.ll
+++ b/test/DebugInfo/X86/sret.ll
@@ -23,9 +23,9 @@ entry:
   %this.addr = alloca %class.A*, align 8
   %i.addr = alloca i32, align 4
   store %class.A* %this, %class.A** %this.addr, align 8
-  call void @llvm.dbg.declare(metadata %class.A** %this.addr, metadata !67, metadata !MDExpression()), !dbg !69
+  call void @llvm.dbg.declare(metadata %class.A** %this.addr, metadata !67, metadata !DIExpression()), !dbg !69
   store i32 %i, i32* %i.addr, align 4
-  call void @llvm.dbg.declare(metadata i32* %i.addr, metadata !70, metadata !MDExpression()), !dbg !71
+  call void @llvm.dbg.declare(metadata i32* %i.addr, metadata !70, metadata !DIExpression()), !dbg !71
   %this1 = load %class.A*, %class.A** %this.addr
   %0 = bitcast %class.A* %this1 to i8***, !dbg !72
   store i8** getelementptr inbounds ([4 x i8*], [4 x i8*]* @_ZTV1A, i64 0, i64 2), i8*** %0, !dbg !72
@@ -44,9 +44,9 @@ entry:
   %this.addr = alloca %class.A*, align 8
   %rhs.addr = alloca %class.A*, align 8
   store %class.A* %this, %class.A** %this.addr, align 8
-  call void @llvm.dbg.declare(metadata %class.A** %this.addr, metadata !74, metadata !MDExpression()), !dbg !75
+  call void @llvm.dbg.declare(metadata %class.A** %this.addr, metadata !74, metadata !DIExpression()), !dbg !75
   store %class.A* %rhs, %class.A** %rhs.addr, align 8
-  call void @llvm.dbg.declare(metadata %class.A** %rhs.addr, metadata !76, metadata !MDExpression()), !dbg !77
+  call void @llvm.dbg.declare(metadata %class.A** %rhs.addr, metadata !76, metadata !DIExpression()), !dbg !77
   %this1 = load %class.A*, %class.A** %this.addr
   %0 = bitcast %class.A* %this1 to i8***, !dbg !78
   store i8** getelementptr inbounds ([4 x i8*], [4 x i8*]* @_ZTV1A, i64 0, i64 2), i8*** %0, !dbg !78
@@ -64,9 +64,9 @@ entry:
   %this.addr = alloca %class.A*, align 8
   %rhs.addr = alloca %class.A*, align 8
   store %class.A* %this, %class.A** %this.addr, align 8
-  call void @llvm.dbg.declare(metadata %class.A** %this.addr, metadata !80, metadata !MDExpression()), !dbg !81
+  call void @llvm.dbg.declare(metadata %class.A** %this.addr, metadata !80, metadata !DIExpression()), !dbg !81
   store %class.A* %rhs, %class.A** %rhs.addr, align 8
-  call void @llvm.dbg.declare(metadata %class.A** %rhs.addr, metadata !82, metadata !MDExpression()), !dbg !83
+  call void @llvm.dbg.declare(metadata %class.A** %rhs.addr, metadata !82, metadata !DIExpression()), !dbg !83
   %this1 = load %class.A*, %class.A** %this.addr
   %0 = load %class.A*, %class.A** %rhs.addr, align 8, !dbg !84
   %m_int = getelementptr inbounds %class.A, %class.A* %0, i32 0, i32 1, !dbg !84
@@ -81,7 +81,7 @@ define i32 @_ZN1A7get_intEv(%class.A* %this) #0 align 2 {
 entry:
   %this.addr = alloca %class.A*, align 8
   store %class.A* %this, %class.A** %this.addr, align 8
-  call void @llvm.dbg.declare(metadata %class.A** %this.addr, metadata !86, metadata !MDExpression()), !dbg !87
+  call void @llvm.dbg.declare(metadata %class.A** %this.addr, metadata !86, metadata !DIExpression()), !dbg !87
   %this1 = load %class.A*, %class.A** %this.addr
   %m_int = getelementptr inbounds %class.A, %class.A* %this1, i32 0, i32 1, !dbg !88
   %0 = load i32, i32* %m_int, align 4, !dbg !88
@@ -95,10 +95,10 @@ entry:
   %nrvo = alloca i1
   %cleanup.dest.slot = alloca i32
   store %class.B* %this, %class.B** %this.addr, align 8
-  call void @llvm.dbg.declare(metadata %class.B** %this.addr, metadata !89, metadata !MDExpression()), !dbg !91
+  call void @llvm.dbg.declare(metadata %class.B** %this.addr, metadata !89, metadata !DIExpression()), !dbg !91
   %this1 = load %class.B*, %class.B** %this.addr
   store i1 false, i1* %nrvo, !dbg !92
-  call void @llvm.dbg.declare(metadata %class.A* %agg.result, metadata !93, metadata !MDExpression(DW_OP_deref)), !dbg !92
+  call void @llvm.dbg.declare(metadata %class.A* %agg.result, metadata !93, metadata !DIExpression(DW_OP_deref)), !dbg !92
   call void @_ZN1AC1Ei(%class.A* %agg.result, i32 12), !dbg !92
   store i1 true, i1* %nrvo, !dbg !94
   store i32 1, i32* %cleanup.dest.slot
@@ -118,13 +118,13 @@ define linkonce_odr void @_ZN1AD2Ev(%class.A* %this) unnamed_addr #0 align 2 {
 entry:
   %this.addr = alloca %class.A*, align 8
   store %class.A* %this, %class.A** %this.addr, align 8
-  call void @llvm.dbg.declare(metadata %class.A** %this.addr, metadata !101, metadata !MDExpression()), !dbg !102
+  call void @llvm.dbg.declare(metadata %class.A** %this.addr, metadata !101, metadata !DIExpression()), !dbg !102
   %this1 = load %class.A*, %class.A** %this.addr
   ret void, !dbg !103
 }
 
 ; Function Attrs: uwtable
-define i32 @main(i32 %argc, i8** %argv) #2 {
+define i32 @main(i32 %argc, i8** %argv) #2 personality i8* bitcast (i32 (...)* @__gxx_personality_v0 to i8*) {
 entry:
   %retval = alloca i32, align 4
   %argc.addr = alloca i32, align 4
@@ -138,12 +138,12 @@ entry:
   %cleanup.dest.slot = alloca i32
   store i32 0, i32* %retval
   store i32 %argc, i32* %argc.addr, align 4
-  call void @llvm.dbg.declare(metadata i32* %argc.addr, metadata !104, metadata !MDExpression()), !dbg !105
+  call void @llvm.dbg.declare(metadata i32* %argc.addr, metadata !104, metadata !DIExpression()), !dbg !105
   store i8** %argv, i8*** %argv.addr, align 8
-  call void @llvm.dbg.declare(metadata i8*** %argv.addr, metadata !106, metadata !MDExpression()), !dbg !105
-  call void @llvm.dbg.declare(metadata %class.B* %b, metadata !107, metadata !MDExpression()), !dbg !108
+  call void @llvm.dbg.declare(metadata i8*** %argv.addr, metadata !106, metadata !DIExpression()), !dbg !105
+  call void @llvm.dbg.declare(metadata %class.B* %b, metadata !107, metadata !DIExpression()), !dbg !108
   call void @_ZN1BC2Ev(%class.B* %b), !dbg !108
-  call void @llvm.dbg.declare(metadata i32* %return_val, metadata !109, metadata !MDExpression()), !dbg !110
+  call void @llvm.dbg.declare(metadata i32* %return_val, metadata !109, metadata !DIExpression()), !dbg !110
   call void @_ZN1B9AInstanceEv(%class.A* sret %temp.lvalue, %class.B* %b), !dbg !110
   %call = invoke i32 @_ZN1A7get_intEv(%class.A* %temp.lvalue)
           to label %invoke.cont unwind label %lpad, !dbg !110
@@ -151,7 +151,7 @@ entry:
 invoke.cont:                                      ; preds = %entry
   call void @_ZN1AD2Ev(%class.A* %temp.lvalue), !dbg !111
   store i32 %call, i32* %return_val, align 4, !dbg !111
-  call void @llvm.dbg.declare(metadata %class.A* %a, metadata !113, metadata !MDExpression()), !dbg !114
+  call void @llvm.dbg.declare(metadata %class.A* %a, metadata !113, metadata !DIExpression()), !dbg !114
   call void @_ZN1B9AInstanceEv(%class.A* sret %a, %class.B* %b), !dbg !114
   %0 = load i32, i32* %return_val, align 4, !dbg !115
   store i32 %0, i32* %retval, !dbg !115
@@ -161,7 +161,7 @@ invoke.cont:                                      ; preds = %entry
   ret i32 %1, !dbg !116
 
 lpad:                                             ; preds = %entry
-  %2 = landingpad { i8*, i32 } personality i8* bitcast (i32 (...)* @__gxx_personality_v0 to i8*)
+  %2 = landingpad { i8*, i32 }
           cleanup, !dbg !116
   %3 = extractvalue { i8*, i32 } %2, 0, !dbg !116
   store i8* %3, i8** %exn.slot, !dbg !116
@@ -181,7 +181,7 @@ eh.resume:                                        ; preds = %invoke.cont1
   resume { i8*, i32 } %lpad.val2, !dbg !119
 
 terminate.lpad:                                   ; preds = %lpad
-  %5 = landingpad { i8*, i32 } personality i8* bitcast (i32 (...)* @__gxx_personality_v0 to i8*)
+  %5 = landingpad { i8*, i32 }
           catch i8* null, !dbg !121
   %6 = extractvalue { i8*, i32 } %5, 0, !dbg !121
   call void @__clang_call_terminate(i8* %6) #5, !dbg !121
@@ -193,7 +193,7 @@ define linkonce_odr void @_ZN1BC2Ev(%class.B* %this) unnamed_addr #0 align 2 {
 entry:
   %this.addr = alloca %class.B*, align 8
   store %class.B* %this, %class.B** %this.addr, align 8
-  call void @llvm.dbg.declare(metadata %class.B** %this.addr, metadata !123, metadata !MDExpression()), !dbg !124
+  call void @llvm.dbg.declare(metadata %class.B** %this.addr, metadata !123, metadata !DIExpression()), !dbg !124
   %this1 = load %class.B*, %class.B** %this.addr
   ret void, !dbg !125
 }
@@ -212,13 +212,13 @@ declare i8* @__cxa_begin_catch(i8*)
 declare void @_ZSt9terminatev()
 
 ; Function Attrs: uwtable
-define linkonce_odr void @_ZN1AD0Ev(%class.A* %this) unnamed_addr #2 align 2 {
+define linkonce_odr void @_ZN1AD0Ev(%class.A* %this) unnamed_addr #2 align 2 personality i8* bitcast (i32 (...)* @__gxx_personality_v0 to i8*) {
 entry:
   %this.addr = alloca %class.A*, align 8
   %exn.slot = alloca i8*
   %ehselector.slot = alloca i32
   store %class.A* %this, %class.A** %this.addr, align 8
-  call void @llvm.dbg.declare(metadata %class.A** %this.addr, metadata !126, metadata !MDExpression()), !dbg !127
+  call void @llvm.dbg.declare(metadata %class.A** %this.addr, metadata !126, metadata !DIExpression()), !dbg !127
   %this1 = load %class.A*, %class.A** %this.addr
   invoke void @_ZN1AD2Ev(%class.A* %this1)
           to label %invoke.cont unwind label %lpad, !dbg !128
@@ -229,7 +229,7 @@ invoke.cont:                                      ; preds = %entry
   ret void, !dbg !129
 
 lpad:                                             ; preds = %entry
-  %1 = landingpad { i8*, i32 } personality i8* bitcast (i32 (...)* @__gxx_personality_v0 to i8*)
+  %1 = landingpad { i8*, i32 }
           cleanup, !dbg !131
   %2 = extractvalue { i8*, i32 } %1, 0, !dbg !131
   store i8* %2, i8** %exn.slot, !dbg !131
@@ -263,131 +263,131 @@ attributes #7 = { builtin nounwind }
 !llvm.module.flags = !{!64, !65}
 !llvm.ident = !{!66}
 
-!0 = !MDCompileUnit(language: DW_LANG_C_plus_plus, producer: "clang version 3.5.0 (trunk 203283) (llvm/trunk 203307)", isOptimized: false, splitDebugFilename: "sret.dwo", emissionKind: 1, file: !1, enums: !2, retainedTypes: !3, subprograms: !48, globals: !2, imports: !2)
-!1 = !MDFile(filename: "sret.cpp", directory: "/usr/local/google/home/echristo/tmp")
+!0 = !DICompileUnit(language: DW_LANG_C_plus_plus, producer: "clang version 3.5.0 (trunk 203283) (llvm/trunk 203307)", isOptimized: false, splitDebugFilename: "sret.dwo", emissionKind: 1, file: !1, enums: !2, retainedTypes: !3, subprograms: !48, globals: !2, imports: !2)
+!1 = !DIFile(filename: "sret.cpp", directory: "/usr/local/google/home/echristo/tmp")
 !2 = !{}
 !3 = !{!4, !37}
-!4 = !MDCompositeType(tag: DW_TAG_class_type, name: "A", line: 1, size: 128, align: 64, file: !1, elements: !5, vtableHolder: !"_ZTS1A", identifier: "_ZTS1A")
+!4 = !DICompositeType(tag: DW_TAG_class_type, name: "A", line: 1, size: 128, align: 64, file: !1, elements: !5, vtableHolder: !"_ZTS1A", identifier: "_ZTS1A")
 !5 = !{!6, !13, !14, !19, !25, !29, !33}
-!6 = !MDDerivedType(tag: DW_TAG_member, name: "_vptr$A", size: 64, flags: DIFlagArtificial, file: !1, scope: !7, baseType: !8)
-!7 = !MDFile(filename: "sret.cpp", directory: "/usr/local/google/home/echristo/tmp")
-!8 = !MDDerivedType(tag: DW_TAG_pointer_type, size: 64, baseType: !9)
-!9 = !MDDerivedType(tag: DW_TAG_pointer_type, name: "__vtbl_ptr_type", size: 64, baseType: !10)
-!10 = !MDSubroutineType(types: !11)
+!6 = !DIDerivedType(tag: DW_TAG_member, name: "_vptr$A", size: 64, flags: DIFlagArtificial, file: !1, scope: !7, baseType: !8)
+!7 = !DIFile(filename: "sret.cpp", directory: "/usr/local/google/home/echristo/tmp")
+!8 = !DIDerivedType(tag: DW_TAG_pointer_type, size: 64, baseType: !9)
+!9 = !DIDerivedType(tag: DW_TAG_pointer_type, name: "__vtbl_ptr_type", size: 64, baseType: !10)
+!10 = !DISubroutineType(types: !11)
 !11 = !{!12}
-!12 = !MDBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
-!13 = !MDDerivedType(tag: DW_TAG_member, name: "m_int", line: 13, size: 32, align: 32, offset: 64, flags: DIFlagProtected, file: !1, scope: !"_ZTS1A", baseType: !12)
-!14 = !MDSubprogram(name: "A", line: 4, isLocal: false, isDefinition: false, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 4, file: !1, scope: !"_ZTS1A", type: !15)
-!15 = !MDSubroutineType(types: !16)
+!12 = !DIBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
+!13 = !DIDerivedType(tag: DW_TAG_member, name: "m_int", line: 13, size: 32, align: 32, offset: 64, flags: DIFlagProtected, file: !1, scope: !"_ZTS1A", baseType: !12)
+!14 = !DISubprogram(name: "A", line: 4, isLocal: false, isDefinition: false, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 4, file: !1, scope: !"_ZTS1A", type: !15)
+!15 = !DISubroutineType(types: !16)
 !16 = !{null, !17, !12}
-!17 = !MDDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, flags: DIFlagArtificial | DIFlagObjectPointer, baseType: !"_ZTS1A")
-!19 = !MDSubprogram(name: "A", line: 5, isLocal: false, isDefinition: false, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 5, file: !1, scope: !"_ZTS1A", type: !20)
-!20 = !MDSubroutineType(types: !21)
+!17 = !DIDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, flags: DIFlagArtificial | DIFlagObjectPointer, baseType: !"_ZTS1A")
+!19 = !DISubprogram(name: "A", line: 5, isLocal: false, isDefinition: false, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 5, file: !1, scope: !"_ZTS1A", type: !20)
+!20 = !DISubroutineType(types: !21)
 !21 = !{null, !17, !22}
-!22 = !MDDerivedType(tag: DW_TAG_reference_type, baseType: !23)
-!23 = !MDDerivedType(tag: DW_TAG_const_type, baseType: !"_ZTS1A")
-!25 = !MDSubprogram(name: "operator=", linkageName: "_ZN1AaSERKS_", line: 7, isLocal: false, isDefinition: false, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 7, file: !1, scope: !"_ZTS1A", type: !26)
-!26 = !MDSubroutineType(types: !27)
+!22 = !DIDerivedType(tag: DW_TAG_reference_type, baseType: !23)
+!23 = !DIDerivedType(tag: DW_TAG_const_type, baseType: !"_ZTS1A")
+!25 = !DISubprogram(name: "operator=", linkageName: "_ZN1AaSERKS_", line: 7, isLocal: false, isDefinition: false, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 7, file: !1, scope: !"_ZTS1A", type: !26)
+!26 = !DISubroutineType(types: !27)
 !27 = !{!22, !17, !22}
-!29 = !MDSubprogram(name: "~A", line: 8, isLocal: false, isDefinition: false, virtuality: DW_VIRTUALITY_virtual, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 8, file: !1, scope: !"_ZTS1A", type: !30, containingType: !"_ZTS1A")
-!30 = !MDSubroutineType(types: !31)
+!29 = !DISubprogram(name: "~A", line: 8, isLocal: false, isDefinition: false, virtuality: DW_VIRTUALITY_virtual, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 8, file: !1, scope: !"_ZTS1A", type: !30, containingType: !"_ZTS1A")
+!30 = !DISubroutineType(types: !31)
 !31 = !{null, !17}
-!33 = !MDSubprogram(name: "get_int", linkageName: "_ZN1A7get_intEv", line: 10, isLocal: false, isDefinition: false, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 10, file: !1, scope: !"_ZTS1A", type: !34)
-!34 = !MDSubroutineType(types: !35)
+!33 = !DISubprogram(name: "get_int", linkageName: "_ZN1A7get_intEv", line: 10, isLocal: false, isDefinition: false, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 10, file: !1, scope: !"_ZTS1A", type: !34)
+!34 = !DISubroutineType(types: !35)
 !35 = !{!12, !17}
-!37 = !MDCompositeType(tag: DW_TAG_class_type, name: "B", line: 38, size: 8, align: 8, file: !1, elements: !38, identifier: "_ZTS1B")
+!37 = !DICompositeType(tag: DW_TAG_class_type, name: "B", line: 38, size: 8, align: 8, file: !1, elements: !38, identifier: "_ZTS1B")
 !38 = !{!39, !44}
-!39 = !MDSubprogram(name: "B", line: 41, isLocal: false, isDefinition: false, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 41, file: !1, scope: !"_ZTS1B", type: !40)
-!40 = !MDSubroutineType(types: !41)
+!39 = !DISubprogram(name: "B", line: 41, isLocal: false, isDefinition: false, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 41, file: !1, scope: !"_ZTS1B", type: !40)
+!40 = !DISubroutineType(types: !41)
 !41 = !{null, !42}
-!42 = !MDDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, flags: DIFlagArtificial | DIFlagObjectPointer, baseType: !"_ZTS1B")
-!44 = !MDSubprogram(name: "AInstance", linkageName: "_ZN1B9AInstanceEv", line: 43, isLocal: false, isDefinition: false, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 43, file: !1, scope: !"_ZTS1B", type: !45)
-!45 = !MDSubroutineType(types: !46)
+!42 = !DIDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, flags: DIFlagArtificial | DIFlagObjectPointer, baseType: !"_ZTS1B")
+!44 = !DISubprogram(name: "AInstance", linkageName: "_ZN1B9AInstanceEv", line: 43, isLocal: false, isDefinition: false, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 43, file: !1, scope: !"_ZTS1B", type: !45)
+!45 = !DISubroutineType(types: !46)
 !46 = !{!4, !42}
 !48 = !{!49, !50, !51, !52, !53, !54, !61, !62, !63}
-!49 = !MDSubprogram(name: "A", linkageName: "_ZN1AC2Ei", line: 16, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 18, file: !1, scope: !"_ZTS1A", type: !15, function: void (%class.A*, i32)* @_ZN1AC2Ei, declaration: !14, variables: !2)
-!50 = !MDSubprogram(name: "A", linkageName: "_ZN1AC2ERKS_", line: 21, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 23, file: !1, scope: !"_ZTS1A", type: !20, function: void (%class.A*, %class.A*)* @_ZN1AC2ERKS_, declaration: !19, variables: !2)
-!51 = !MDSubprogram(name: "operator=", linkageName: "_ZN1AaSERKS_", line: 27, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 28, file: !1, scope: !"_ZTS1A", type: !26, function: %class.A* (%class.A*, %class.A*)* @_ZN1AaSERKS_, declaration: !25, variables: !2)
-!52 = !MDSubprogram(name: "get_int", linkageName: "_ZN1A7get_intEv", line: 33, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 34, file: !1, scope: !"_ZTS1A", type: !34, function: i32 (%class.A*)* @_ZN1A7get_intEv, declaration: !33, variables: !2)
-!53 = !MDSubprogram(name: "AInstance", linkageName: "_ZN1B9AInstanceEv", line: 47, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 48, file: !1, scope: !"_ZTS1B", type: !45, function: void (%class.A*, %class.B*)* @_ZN1B9AInstanceEv, declaration: !44, variables: !2)
-!54 = !MDSubprogram(name: "main", line: 53, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 54, file: !1, scope: !7, type: !55, function: i32 (i32, i8**)* @main, variables: !2)
-!55 = !MDSubroutineType(types: !56)
+!49 = !DISubprogram(name: "A", linkageName: "_ZN1AC2Ei", line: 16, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 18, file: !1, scope: !"_ZTS1A", type: !15, function: void (%class.A*, i32)* @_ZN1AC2Ei, declaration: !14, variables: !2)
+!50 = !DISubprogram(name: "A", linkageName: "_ZN1AC2ERKS_", line: 21, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 23, file: !1, scope: !"_ZTS1A", type: !20, function: void (%class.A*, %class.A*)* @_ZN1AC2ERKS_, declaration: !19, variables: !2)
+!51 = !DISubprogram(name: "operator=", linkageName: "_ZN1AaSERKS_", line: 27, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 28, file: !1, scope: !"_ZTS1A", type: !26, function: %class.A* (%class.A*, %class.A*)* @_ZN1AaSERKS_, declaration: !25, variables: !2)
+!52 = !DISubprogram(name: "get_int", linkageName: "_ZN1A7get_intEv", line: 33, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 34, file: !1, scope: !"_ZTS1A", type: !34, function: i32 (%class.A*)* @_ZN1A7get_intEv, declaration: !33, variables: !2)
+!53 = !DISubprogram(name: "AInstance", linkageName: "_ZN1B9AInstanceEv", line: 47, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 48, file: !1, scope: !"_ZTS1B", type: !45, function: void (%class.A*, %class.B*)* @_ZN1B9AInstanceEv, declaration: !44, variables: !2)
+!54 = !DISubprogram(name: "main", line: 53, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 54, file: !1, scope: !7, type: !55, function: i32 (i32, i8**)* @main, variables: !2)
+!55 = !DISubroutineType(types: !56)
 !56 = !{!12, !12, !57}
-!57 = !MDDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, baseType: !58)
-!58 = !MDDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, baseType: !59)
-!59 = !MDDerivedType(tag: DW_TAG_const_type, baseType: !60)
-!60 = !MDBasicType(tag: DW_TAG_base_type, name: "char", size: 8, align: 8, encoding: DW_ATE_signed_char)
-!61 = !MDSubprogram(name: "~A", linkageName: "_ZN1AD0Ev", line: 8, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 8, file: !1, scope: !"_ZTS1A", type: !30, function: void (%class.A*)* @_ZN1AD0Ev, declaration: !29, variables: !2)
-!62 = !MDSubprogram(name: "B", linkageName: "_ZN1BC2Ev", line: 41, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 41, file: !1, scope: !"_ZTS1B", type: !40, function: void (%class.B*)* @_ZN1BC2Ev, declaration: !39, variables: !2)
-!63 = !MDSubprogram(name: "~A", linkageName: "_ZN1AD2Ev", line: 8, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 8, file: !1, scope: !"_ZTS1A", type: !30, function: void (%class.A*)* @_ZN1AD2Ev, declaration: !29, variables: !2)
+!57 = !DIDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, baseType: !58)
+!58 = !DIDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, baseType: !59)
+!59 = !DIDerivedType(tag: DW_TAG_const_type, baseType: !60)
+!60 = !DIBasicType(tag: DW_TAG_base_type, name: "char", size: 8, align: 8, encoding: DW_ATE_signed_char)
+!61 = !DISubprogram(name: "~A", linkageName: "_ZN1AD0Ev", line: 8, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 8, file: !1, scope: !"_ZTS1A", type: !30, function: void (%class.A*)* @_ZN1AD0Ev, declaration: !29, variables: !2)
+!62 = !DISubprogram(name: "B", linkageName: "_ZN1BC2Ev", line: 41, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 41, file: !1, scope: !"_ZTS1B", type: !40, function: void (%class.B*)* @_ZN1BC2Ev, declaration: !39, variables: !2)
+!63 = !DISubprogram(name: "~A", linkageName: "_ZN1AD2Ev", line: 8, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 8, file: !1, scope: !"_ZTS1A", type: !30, function: void (%class.A*)* @_ZN1AD2Ev, declaration: !29, variables: !2)
 !64 = !{i32 2, !"Dwarf Version", i32 4}
 !65 = !{i32 1, !"Debug Info Version", i32 3}
 !66 = !{!"clang version 3.5.0 (trunk 203283) (llvm/trunk 203307)"}
-!67 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "this", arg: 1, flags: DIFlagArtificial | DIFlagObjectPointer, scope: !49, type: !68)
-!68 = !MDDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, baseType: !"_ZTS1A")
-!69 = !MDLocation(line: 0, scope: !49)
-!70 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "i", line: 16, arg: 2, scope: !49, file: !7, type: !12)
-!71 = !MDLocation(line: 16, scope: !49)
-!72 = !MDLocation(line: 18, scope: !49)
-!73 = !MDLocation(line: 19, scope: !49)
-!74 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "this", arg: 1, flags: DIFlagArtificial | DIFlagObjectPointer, scope: !50, type: !68)
-!75 = !MDLocation(line: 0, scope: !50)
-!76 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "rhs", line: 21, arg: 2, scope: !50, file: !7, type: !22)
-!77 = !MDLocation(line: 21, scope: !50)
-!78 = !MDLocation(line: 23, scope: !50)
-!79 = !MDLocation(line: 24, scope: !50)
-!80 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "this", arg: 1, flags: DIFlagArtificial | DIFlagObjectPointer, scope: !51, type: !68)
-!81 = !MDLocation(line: 0, scope: !51)
-!82 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "rhs", line: 27, arg: 2, scope: !51, file: !7, type: !22)
-!83 = !MDLocation(line: 27, scope: !51)
-!84 = !MDLocation(line: 29, scope: !51)
-!85 = !MDLocation(line: 30, scope: !51)
-!86 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "this", arg: 1, flags: DIFlagArtificial | DIFlagObjectPointer, scope: !52, type: !68)
-!87 = !MDLocation(line: 0, scope: !52)
-!88 = !MDLocation(line: 35, scope: !52)
-!89 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "this", arg: 1, flags: DIFlagArtificial | DIFlagObjectPointer, scope: !53, type: !90)
-!90 = !MDDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, baseType: !"_ZTS1B")
-!91 = !MDLocation(line: 0, scope: !53)
-!92 = !MDLocation(line: 49, scope: !53)
-!93 = !MDLocalVariable(tag: DW_TAG_auto_variable, name: "a", line: 49, scope: !53, file: !7, type: !4)
-!94 = !MDLocation(line: 50, scope: !53)
-!95 = !MDLocation(line: 51, scope: !53)
-!96 = !MDLocation(line: 51, scope: !97)
-!97 = distinct !MDLexicalBlock(line: 51, column: 0, file: !1, scope: !53)
-!98 = !MDLocation(line: 51, scope: !99)
-!99 = distinct !MDLexicalBlock(line: 51, column: 0, file: !1, scope: !100)
-!100 = distinct !MDLexicalBlock(line: 51, column: 0, file: !1, scope: !53)
-!101 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "this", arg: 1, flags: DIFlagArtificial | DIFlagObjectPointer, scope: !63, type: !68)
-!102 = !MDLocation(line: 0, scope: !63)
-!103 = !MDLocation(line: 8, scope: !63)
-!104 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "argc", line: 53, arg: 1, scope: !54, file: !7, type: !12)
-!105 = !MDLocation(line: 53, scope: !54)
-!106 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "argv", line: 53, arg: 2, scope: !54, file: !7, type: !57)
-!107 = !MDLocalVariable(tag: DW_TAG_auto_variable, name: "b", line: 55, scope: !54, file: !7, type: !37)
-!108 = !MDLocation(line: 55, scope: !54)
-!109 = !MDLocalVariable(tag: DW_TAG_auto_variable, name: "return_val", line: 56, scope: !54, file: !7, type: !12)
-!110 = !MDLocation(line: 56, scope: !54)
-!111 = !MDLocation(line: 56, scope: !112)
-!112 = distinct !MDLexicalBlock(line: 56, column: 0, file: !1, scope: !54)
-!113 = !MDLocalVariable(tag: DW_TAG_auto_variable, name: "a", line: 58, scope: !54, file: !7, type: !4)
-!114 = !MDLocation(line: 58, scope: !54)
-!115 = !MDLocation(line: 59, scope: !54)
-!116 = !MDLocation(line: 60, scope: !54)
-!117 = !MDLocation(line: 60, scope: !118)
-!118 = distinct !MDLexicalBlock(line: 60, column: 0, file: !1, scope: !54)
-!119 = !MDLocation(line: 60, scope: !120)
-!120 = distinct !MDLexicalBlock(line: 60, column: 0, file: !1, scope: !54)
-!121 = !MDLocation(line: 60, scope: !122)
-!122 = distinct !MDLexicalBlock(line: 60, column: 0, file: !1, scope: !54)
-!123 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "this", arg: 1, flags: DIFlagArtificial | DIFlagObjectPointer, scope: !62, type: !90)
-!124 = !MDLocation(line: 0, scope: !62)
-!125 = !MDLocation(line: 41, scope: !62)
-!126 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "this", arg: 1, flags: DIFlagArtificial | DIFlagObjectPointer, scope: !61, type: !68)
-!127 = !MDLocation(line: 0, scope: !61)
-!128 = !MDLocation(line: 8, scope: !61)
-!129 = !MDLocation(line: 8, scope: !130)
-!130 = distinct !MDLexicalBlock(line: 8, column: 0, file: !1, scope: !61)
-!131 = !MDLocation(line: 8, scope: !132)
-!132 = distinct !MDLexicalBlock(line: 8, column: 0, file: !1, scope: !61)
-!133 = !MDLocation(line: 8, scope: !134)
-!134 = distinct !MDLexicalBlock(line: 8, column: 0, file: !1, scope: !61)
+!67 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "this", arg: 1, flags: DIFlagArtificial | DIFlagObjectPointer, scope: !49, type: !68)
+!68 = !DIDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, baseType: !"_ZTS1A")
+!69 = !DILocation(line: 0, scope: !49)
+!70 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "i", line: 16, arg: 2, scope: !49, file: !7, type: !12)
+!71 = !DILocation(line: 16, scope: !49)
+!72 = !DILocation(line: 18, scope: !49)
+!73 = !DILocation(line: 19, scope: !49)
+!74 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "this", arg: 1, flags: DIFlagArtificial | DIFlagObjectPointer, scope: !50, type: !68)
+!75 = !DILocation(line: 0, scope: !50)
+!76 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "rhs", line: 21, arg: 2, scope: !50, file: !7, type: !22)
+!77 = !DILocation(line: 21, scope: !50)
+!78 = !DILocation(line: 23, scope: !50)
+!79 = !DILocation(line: 24, scope: !50)
+!80 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "this", arg: 1, flags: DIFlagArtificial | DIFlagObjectPointer, scope: !51, type: !68)
+!81 = !DILocation(line: 0, scope: !51)
+!82 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "rhs", line: 27, arg: 2, scope: !51, file: !7, type: !22)
+!83 = !DILocation(line: 27, scope: !51)
+!84 = !DILocation(line: 29, scope: !51)
+!85 = !DILocation(line: 30, scope: !51)
+!86 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "this", arg: 1, flags: DIFlagArtificial | DIFlagObjectPointer, scope: !52, type: !68)
+!87 = !DILocation(line: 0, scope: !52)
+!88 = !DILocation(line: 35, scope: !52)
+!89 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "this", arg: 1, flags: DIFlagArtificial | DIFlagObjectPointer, scope: !53, type: !90)
+!90 = !DIDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, baseType: !"_ZTS1B")
+!91 = !DILocation(line: 0, scope: !53)
+!92 = !DILocation(line: 49, scope: !53)
+!93 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "a", line: 49, scope: !53, file: !7, type: !4)
+!94 = !DILocation(line: 50, scope: !53)
+!95 = !DILocation(line: 51, scope: !53)
+!96 = !DILocation(line: 51, scope: !97)
+!97 = distinct !DILexicalBlock(line: 51, column: 0, file: !1, scope: !53)
+!98 = !DILocation(line: 51, scope: !99)
+!99 = distinct !DILexicalBlock(line: 51, column: 0, file: !1, scope: !100)
+!100 = distinct !DILexicalBlock(line: 51, column: 0, file: !1, scope: !53)
+!101 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "this", arg: 1, flags: DIFlagArtificial | DIFlagObjectPointer, scope: !63, type: !68)
+!102 = !DILocation(line: 0, scope: !63)
+!103 = !DILocation(line: 8, scope: !63)
+!104 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "argc", line: 53, arg: 1, scope: !54, file: !7, type: !12)
+!105 = !DILocation(line: 53, scope: !54)
+!106 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "argv", line: 53, arg: 2, scope: !54, file: !7, type: !57)
+!107 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "b", line: 55, scope: !54, file: !7, type: !37)
+!108 = !DILocation(line: 55, scope: !54)
+!109 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "return_val", line: 56, scope: !54, file: !7, type: !12)
+!110 = !DILocation(line: 56, scope: !54)
+!111 = !DILocation(line: 56, scope: !112)
+!112 = distinct !DILexicalBlock(line: 56, column: 0, file: !1, scope: !54)
+!113 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "a", line: 58, scope: !54, file: !7, type: !4)
+!114 = !DILocation(line: 58, scope: !54)
+!115 = !DILocation(line: 59, scope: !54)
+!116 = !DILocation(line: 60, scope: !54)
+!117 = !DILocation(line: 60, scope: !118)
+!118 = distinct !DILexicalBlock(line: 60, column: 0, file: !1, scope: !54)
+!119 = !DILocation(line: 60, scope: !120)
+!120 = distinct !DILexicalBlock(line: 60, column: 0, file: !1, scope: !54)
+!121 = !DILocation(line: 60, scope: !122)
+!122 = distinct !DILexicalBlock(line: 60, column: 0, file: !1, scope: !54)
+!123 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "this", arg: 1, flags: DIFlagArtificial | DIFlagObjectPointer, scope: !62, type: !90)
+!124 = !DILocation(line: 0, scope: !62)
+!125 = !DILocation(line: 41, scope: !62)
+!126 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "this", arg: 1, flags: DIFlagArtificial | DIFlagObjectPointer, scope: !61, type: !68)
+!127 = !DILocation(line: 0, scope: !61)
+!128 = !DILocation(line: 8, scope: !61)
+!129 = !DILocation(line: 8, scope: !130)
+!130 = distinct !DILexicalBlock(line: 8, column: 0, file: !1, scope: !61)
+!131 = !DILocation(line: 8, scope: !132)
+!132 = distinct !DILexicalBlock(line: 8, column: 0, file: !1, scope: !61)
+!133 = !DILocation(line: 8, scope: !134)
+!134 = distinct !DILexicalBlock(line: 8, column: 0, file: !1, scope: !61)
diff --git a/test/DebugInfo/X86/sroasplit-1.ll b/test/DebugInfo/X86/sroasplit-1.ll
index a25b8b2ba71..2179aa38095 100644
--- a/test/DebugInfo/X86/sroasplit-1.ll
+++ b/test/DebugInfo/X86/sroasplit-1.ll
@@ -25,9 +25,9 @@
 ; CHECK: call void @llvm.dbg.value(metadata i32 %[[A:.*]], i64 0, metadata ![[VAR]], metadata ![[PIECE2:[0-9]+]])
 ; CHECK: ret i32 %[[A]]
 ; Read Var and Piece:
-; CHECK: ![[VAR]] = !MDLocalVariable(tag: DW_TAG_auto_variable, name: "i1",{{.*}} line: 11,
-; CHECK: ![[PIECE1]] = !MDExpression(DW_OP_bit_piece, 32, 96)
-; CHECK: ![[PIECE2]] = !MDExpression(DW_OP_bit_piece, 0, 32)
+; CHECK: ![[VAR]] = !DILocalVariable(tag: DW_TAG_auto_variable, name: "i1",{{.*}} line: 11,
+; CHECK: ![[PIECE1]] = !DIExpression(DW_OP_bit_piece, 32, 96)
+; CHECK: ![[PIECE2]] = !DIExpression(DW_OP_bit_piece, 0, 32)
 
 target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-apple-macosx10.9.0"
@@ -65,33 +65,33 @@ attributes #2 = { nounwind }
 !llvm.module.flags = !{!22, !23}
 !llvm.ident = !{!24}
 
-!0 = !MDCompileUnit(language: DW_LANG_C99, producer: "clang version 3.5.0 ", isOptimized: false, emissionKind: 1, file: !1, enums: !{}, retainedTypes: !{}, subprograms: !3, globals: !{}, imports: !{})
-!1 = !MDFile(filename: "sroasplit-1.c", directory: "")
-!2 = !MDExpression()
+!0 = !DICompileUnit(language: DW_LANG_C99, producer: "clang version 3.5.0 ", isOptimized: false, emissionKind: 1, file: !1, enums: !{}, retainedTypes: !{}, subprograms: !3, globals: !{}, imports: !{})
+!1 = !DIFile(filename: "sroasplit-1.c", directory: "")
+!2 = !DIExpression()
 !3 = !{!4}
-!4 = !MDSubprogram(name: "foo", line: 10, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 10, file: !1, scope: !5, type: !6, function: i32 (%struct.Outer*)* @foo, variables: !{})
-!5 = !MDFile(filename: "sroasplit-1.c", directory: "")
-!6 = !MDSubroutineType(types: !7)
+!4 = !DISubprogram(name: "foo", line: 10, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 10, file: !1, scope: !5, type: !6, function: i32 (%struct.Outer*)* @foo, variables: !{})
+!5 = !DIFile(filename: "sroasplit-1.c", directory: "")
+!6 = !DISubroutineType(types: !7)
 !7 = !{!8, !9}
-!8 = !MDBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
-!9 = !MDDerivedType(tag: DW_TAG_typedef, name: "Outer", line: 8, file: !1, baseType: !10)
-!10 = !MDCompositeType(tag: DW_TAG_structure_type, line: 6, size: 256, align: 64, file: !1, elements: !11)
+!8 = !DIBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
+!9 = !DIDerivedType(tag: DW_TAG_typedef, name: "Outer", line: 8, file: !1, baseType: !10)
+!10 = !DICompositeType(tag: DW_TAG_structure_type, line: 6, size: 256, align: 64, file: !1, elements: !11)
 !11 = !{!12}
-!12 = !MDDerivedType(tag: DW_TAG_member, name: "inner", line: 7, size: 256, align: 64, file: !1, scope: !10, baseType: !13)
-!13 = !MDCompositeType(tag: DW_TAG_array_type, size: 256, align: 64, baseType: !14, elements: !20)
-!14 = !MDDerivedType(tag: DW_TAG_typedef, name: "Inner", line: 4, file: !1, baseType: !15)
-!15 = !MDCompositeType(tag: DW_TAG_structure_type, line: 1, size: 128, align: 64, file: !1, elements: !16)
+!12 = !DIDerivedType(tag: DW_TAG_member, name: "inner", line: 7, size: 256, align: 64, file: !1, scope: !10, baseType: !13)
+!13 = !DICompositeType(tag: DW_TAG_array_type, size: 256, align: 64, baseType: !14, elements: !20)
+!14 = !DIDerivedType(tag: DW_TAG_typedef, name: "Inner", line: 4, file: !1, baseType: !15)
+!15 = !DICompositeType(tag: DW_TAG_structure_type, line: 1, size: 128, align: 64, file: !1, elements: !16)
 !16 = !{!17, !18}
-!17 = !MDDerivedType(tag: DW_TAG_member, name: "a", line: 2, size: 32, align: 32, file: !1, scope: !15, baseType: !8)
-!18 = !MDDerivedType(tag: DW_TAG_member, name: "b", line: 3, size: 64, align: 64, offset: 64, file: !1, scope: !15, baseType: !19)
-!19 = !MDBasicType(tag: DW_TAG_base_type, name: "long int", size: 64, align: 64, encoding: DW_ATE_signed)
+!17 = !DIDerivedType(tag: DW_TAG_member, name: "a", line: 2, size: 32, align: 32, file: !1, scope: !15, baseType: !8)
+!18 = !DIDerivedType(tag: DW_TAG_member, name: "b", line: 3, size: 64, align: 64, offset: 64, file: !1, scope: !15, baseType: !19)
+!19 = !DIBasicType(tag: DW_TAG_base_type, name: "long int", size: 64, align: 64, encoding: DW_ATE_signed)
 !20 = !{!21}
-!21 = !MDSubrange(count: 2)
+!21 = !DISubrange(count: 2)
 !22 = !{i32 2, !"Dwarf Version", i32 2}
 !23 = !{i32 1, !"Debug Info Version", i32 3}
 !24 = !{!"clang version 3.5.0 "}
-!25 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "outer", line: 10, arg: 1, scope: !4, file: !5, type: !9)
-!26 = !MDLocation(line: 10, scope: !4)
-!27 = !MDLocalVariable(tag: DW_TAG_auto_variable, name: "i1", line: 11, scope: !4, file: !5, type: !14)
-!28 = !MDLocation(line: 11, scope: !4)
-!29 = !MDLocation(line: 12, scope: !4)
+!25 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "outer", line: 10, arg: 1, scope: !4, file: !5, type: !9)
+!26 = !DILocation(line: 10, scope: !4)
+!27 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "i1", line: 11, scope: !4, file: !5, type: !14)
+!28 = !DILocation(line: 11, scope: !4)
+!29 = !DILocation(line: 12, scope: !4)
diff --git a/test/DebugInfo/X86/sroasplit-2.ll b/test/DebugInfo/X86/sroasplit-2.ll
index 1a80f3e7ad1..b209fafd4df 100644
--- a/test/DebugInfo/X86/sroasplit-2.ll
+++ b/test/DebugInfo/X86/sroasplit-2.ll
@@ -23,11 +23,11 @@
 ; CHECK:  call void @llvm.dbg.value(metadata i64 %outer.coerce0, i64 0, metadata ![[O:[0-9]+]], metadata ![[PIECE1:[0-9]+]]),
 ; CHECK:  call void @llvm.dbg.value(metadata i64 %outer.coerce1, i64 0, metadata ![[O]], metadata ![[PIECE2:[0-9]+]]),
 ; CHECK:  call void @llvm.dbg.value({{.*}}, i64 0, metadata ![[I1:[0-9]+]], metadata ![[PIECE3:[0-9]+]]),
-; CHECK-DAG: ![[O]] = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "outer",{{.*}} line: 10
-; CHECK-DAG: ![[PIECE1]] = !MDExpression(DW_OP_bit_piece, 0, 64)
-; CHECK-DAG: ![[PIECE2]] = !MDExpression(DW_OP_bit_piece, 64, 64)
-; CHECK-DAG: ![[I1]] = !MDLocalVariable(tag: DW_TAG_auto_variable, name: "i1",{{.*}} line: 11
-; CHECK-DAG: ![[PIECE3]] = !MDExpression(DW_OP_bit_piece, 0, 32)
+; CHECK-DAG: ![[O]] = !DILocalVariable(tag: DW_TAG_arg_variable, name: "outer",{{.*}} line: 10
+; CHECK-DAG: ![[PIECE1]] = !DIExpression(DW_OP_bit_piece, 0, 64)
+; CHECK-DAG: ![[PIECE2]] = !DIExpression(DW_OP_bit_piece, 64, 64)
+; CHECK-DAG: ![[I1]] = !DILocalVariable(tag: DW_TAG_auto_variable, name: "i1",{{.*}} line: 11
+; CHECK-DAG: ![[PIECE3]] = !DIExpression(DW_OP_bit_piece, 0, 32)
 
 ; ModuleID = 'sroasplit-2.c'
 target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
@@ -71,32 +71,32 @@ attributes #2 = { nounwind }
 !llvm.module.flags = !{!21, !22}
 !llvm.ident = !{!23}
 
-!0 = !MDCompileUnit(language: DW_LANG_C99, producer: "clang version 3.5.0 ", isOptimized: false, emissionKind: 1, file: !1, enums: !{}, retainedTypes: !{}, subprograms: !3, globals: !{}, imports: !{})
-!1 = !MDFile(filename: "sroasplit-2.c", directory: "")
-!2 = !MDExpression()
+!0 = !DICompileUnit(language: DW_LANG_C99, producer: "clang version 3.5.0 ", isOptimized: false, emissionKind: 1, file: !1, enums: !{}, retainedTypes: !{}, subprograms: !3, globals: !{}, imports: !{})
+!1 = !DIFile(filename: "sroasplit-2.c", directory: "")
+!2 = !DIExpression()
 !3 = !{!4}
-!4 = !MDSubprogram(name: "foo", line: 10, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 10, file: !1, scope: !5, type: !6, function: i32 (i64, i64)* @foo, variables: !{})
-!5 = !MDFile(filename: "sroasplit-2.c", directory: "")
-!6 = !MDSubroutineType(types: !7)
+!4 = !DISubprogram(name: "foo", line: 10, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 10, file: !1, scope: !5, type: !6, function: i32 (i64, i64)* @foo, variables: !{})
+!5 = !DIFile(filename: "sroasplit-2.c", directory: "")
+!6 = !DISubroutineType(types: !7)
 !7 = !{!8, !9}
-!8 = !MDBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
-!9 = !MDDerivedType(tag: DW_TAG_typedef, name: "Outer", line: 8, file: !1, baseType: !10)
-!10 = !MDCompositeType(tag: DW_TAG_structure_type, line: 6, size: 128, align: 32, file: !1, elements: !11)
+!8 = !DIBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
+!9 = !DIDerivedType(tag: DW_TAG_typedef, name: "Outer", line: 8, file: !1, baseType: !10)
+!10 = !DICompositeType(tag: DW_TAG_structure_type, line: 6, size: 128, align: 32, file: !1, elements: !11)
 !11 = !{!12}
-!12 = !MDDerivedType(tag: DW_TAG_member, name: "inner", line: 7, size: 128, align: 32, file: !1, scope: !10, baseType: !13)
-!13 = !MDCompositeType(tag: DW_TAG_array_type, size: 128, align: 32, baseType: !14, elements: !19)
-!14 = !MDDerivedType(tag: DW_TAG_typedef, name: "Inner", line: 4, file: !1, baseType: !15)
-!15 = !MDCompositeType(tag: DW_TAG_structure_type, line: 1, size: 64, align: 32, file: !1, elements: !16)
+!12 = !DIDerivedType(tag: DW_TAG_member, name: "inner", line: 7, size: 128, align: 32, file: !1, scope: !10, baseType: !13)
+!13 = !DICompositeType(tag: DW_TAG_array_type, size: 128, align: 32, baseType: !14, elements: !19)
+!14 = !DIDerivedType(tag: DW_TAG_typedef, name: "Inner", line: 4, file: !1, baseType: !15)
+!15 = !DICompositeType(tag: DW_TAG_structure_type, line: 1, size: 64, align: 32, file: !1, elements: !16)
 !16 = !{!17, !18}
-!17 = !MDDerivedType(tag: DW_TAG_member, name: "a", line: 2, size: 32, align: 32, file: !1, scope: !15, baseType: !8)
-!18 = !MDDerivedType(tag: DW_TAG_member, name: "b", line: 3, size: 32, align: 32, offset: 32, file: !1, scope: !15, baseType: !8)
+!17 = !DIDerivedType(tag: DW_TAG_member, name: "a", line: 2, size: 32, align: 32, file: !1, scope: !15, baseType: !8)
+!18 = !DIDerivedType(tag: DW_TAG_member, name: "b", line: 3, size: 32, align: 32, offset: 32, file: !1, scope: !15, baseType: !8)
 !19 = !{!20}
-!20 = !MDSubrange(count: 2)
+!20 = !DISubrange(count: 2)
 !21 = !{i32 2, !"Dwarf Version", i32 2}
 !22 = !{i32 1, !"Debug Info Version", i32 3}
 !23 = !{!"clang version 3.5.0 "}
-!24 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "outer", line: 10, arg: 1, scope: !4, file: !5, type: !9)
-!25 = !MDLocation(line: 10, scope: !4)
-!26 = !MDLocalVariable(tag: DW_TAG_auto_variable, name: "i1", line: 11, scope: !4, file: !5, type: !14)
-!27 = !MDLocation(line: 11, scope: !4)
-!28 = !MDLocation(line: 12, scope: !4)
+!24 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "outer", line: 10, arg: 1, scope: !4, file: !5, type: !9)
+!25 = !DILocation(line: 10, scope: !4)
+!26 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "i1", line: 11, scope: !4, file: !5, type: !14)
+!27 = !DILocation(line: 11, scope: !4)
+!28 = !DILocation(line: 12, scope: !4)
diff --git a/test/DebugInfo/X86/sroasplit-3.ll b/test/DebugInfo/X86/sroasplit-3.ll
index 00b4fd3a942..50b701f781d 100644
--- a/test/DebugInfo/X86/sroasplit-3.ll
+++ b/test/DebugInfo/X86/sroasplit-3.ll
@@ -4,8 +4,8 @@
 ; not partitioned into multiple allocas.
 ;
 ; CHECK: call void @llvm.dbg.value(metadata float %s.coerce, i64 0, metadata ![[VAR:[0-9]+]], metadata ![[EXPR:[0-9]+]])
-; CHECK: ![[VAR]] = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "s",{{.*}} line: 3,
-; CHECK: ![[EXPR]] = !MDExpression(
+; CHECK: ![[VAR]] = !DILocalVariable(tag: DW_TAG_arg_variable, name: "s",{{.*}} line: 3,
+; CHECK: ![[EXPR]] = !DIExpression(
 ; CHECK-NOT:                       DW_OP_bit_piece
 
 ;
@@ -41,23 +41,23 @@ attributes #1 = { nounwind readnone }
 !llvm.module.flags = !{!12, !13, !14}
 !llvm.ident = !{!15}
 
-!0 = !MDCompileUnit(language: DW_LANG_C99, producer: "clang version 3.6.0 ", isOptimized: false, emissionKind: 1, file: !1, enums: !2, retainedTypes: !2, subprograms: !3, globals: !2, imports: !2)
-!1 = !MDFile(filename: "test.c", directory: "/Volumes/Data/llvm/_build.ninja.debug")
+!0 = !DICompileUnit(language: DW_LANG_C99, producer: "clang version 3.6.0 ", isOptimized: false, emissionKind: 1, file: !1, enums: !2, retainedTypes: !2, subprograms: !3, globals: !2, imports: !2)
+!1 = !DIFile(filename: "test.c", directory: "/Volumes/Data/llvm/_build.ninja.debug")
 !2 = !{}
 !3 = !{!4}
-!4 = !MDSubprogram(name: "foo", line: 3, isLocal: false, isDefinition: true, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 3, file: !1, scope: !5, type: !6, function: float (float)* @foo, variables: !2)
-!5 = !MDFile(filename: "test.c", directory: "/Volumes/Data/llvm/_build.ninja.debug")
-!6 = !MDSubroutineType(types: !7)
+!4 = !DISubprogram(name: "foo", line: 3, isLocal: false, isDefinition: true, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 3, file: !1, scope: !5, type: !6, function: float (float)* @foo, variables: !2)
+!5 = !DIFile(filename: "test.c", directory: "/Volumes/Data/llvm/_build.ninja.debug")
+!6 = !DISubroutineType(types: !7)
 !7 = !{!8, !9}
-!8 = !MDBasicType(tag: DW_TAG_base_type, name: "float", size: 32, align: 32, encoding: DW_ATE_float)
-!9 = !MDCompositeType(tag: DW_TAG_structure_type, name: "S", line: 1, size: 32, align: 32, file: !1, elements: !10)
+!8 = !DIBasicType(tag: DW_TAG_base_type, name: "float", size: 32, align: 32, encoding: DW_ATE_float)
+!9 = !DICompositeType(tag: DW_TAG_structure_type, name: "S", line: 1, size: 32, align: 32, file: !1, elements: !10)
 !10 = !{!11}
-!11 = !MDDerivedType(tag: DW_TAG_member, name: "f", line: 1, size: 32, align: 32, file: !1, scope: !9, baseType: !8)
+!11 = !DIDerivedType(tag: DW_TAG_member, name: "f", line: 1, size: 32, align: 32, file: !1, scope: !9, baseType: !8)
 !12 = !{i32 2, !"Dwarf Version", i32 2}
 !13 = !{i32 2, !"Debug Info Version", i32 3}
 !14 = !{i32 1, !"PIC Level", i32 2}
 !15 = !{!"clang version 3.6.0 "}
-!16 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "s", line: 3, arg: 1, scope: !4, file: !5, type: !9)
-!17 = !MDExpression()
-!18 = !MDLocation(line: 3, column: 20, scope: !4)
-!19 = !MDLocation(line: 4, column: 2, scope: !4)
+!16 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "s", line: 3, arg: 1, scope: !4, file: !5, type: !9)
+!17 = !DIExpression()
+!18 = !DILocation(line: 3, column: 20, scope: !4)
+!19 = !DILocation(line: 4, column: 2, scope: !4)
diff --git a/test/DebugInfo/X86/sroasplit-4.ll b/test/DebugInfo/X86/sroasplit-4.ll
index ddb470a8cf1..e9fcdf8eac9 100644
--- a/test/DebugInfo/X86/sroasplit-4.ll
+++ b/test/DebugInfo/X86/sroasplit-4.ll
@@ -7,10 +7,10 @@
 ; CHECK: call void @llvm.dbg.value(metadata i64 %[[T1]], i64 0, metadata ![[Y]], metadata ![[P2:.*]])
 ; CHECK: call void @llvm.dbg.value(metadata i64 %[[T]], i64 0, metadata ![[R:.*]], metadata ![[P3:.*]])
 ; CHECK: call void @llvm.dbg.value(metadata i64 %[[T1]], i64 0, metadata ![[R]], metadata ![[P4:.*]])
-; CHECK: ![[P1]] = !MDExpression(DW_OP_bit_piece, 0, 64)
-; CHECK: ![[P2]] = !MDExpression(DW_OP_bit_piece, 64, 64)
-; CHECK: ![[P3]] = !MDExpression(DW_OP_bit_piece, 192, 64)
-; CHECK: ![[P4]] = !MDExpression(DW_OP_bit_piece, 256, 64)
+; CHECK: ![[P1]] = !DIExpression(DW_OP_bit_piece, 0, 64)
+; CHECK: ![[P2]] = !DIExpression(DW_OP_bit_piece, 64, 64)
+; CHECK: ![[P3]] = !DIExpression(DW_OP_bit_piece, 192, 64)
+; CHECK: ![[P4]] = !DIExpression(DW_OP_bit_piece, 256, 64)
 ; 
 ; struct p {
 ;   __SIZE_TYPE__ s;
@@ -109,38 +109,38 @@ attributes #3 = { nounwind }
 !llvm.module.flags = !{!21, !22}
 !llvm.ident = !{!23}
 
-!0 = !MDCompileUnit(language: DW_LANG_C_plus_plus, producer: "clang version 3.7.0 ", isOptimized: false, emissionKind: 1, file: !1, enums: !2, retainedTypes: !3, subprograms: !16, globals: !2, imports: !2)
-!1 = !MDFile(filename: "<stdin>", directory: "")
+!0 = !DICompileUnit(language: DW_LANG_C_plus_plus, producer: "clang version 3.7.0 ", isOptimized: false, emissionKind: 1, file: !1, enums: !2, retainedTypes: !3, subprograms: !16, globals: !2, imports: !2)
+!1 = !DIFile(filename: "<stdin>", directory: "")
 !2 = !{}
 !3 = !{!4, !10}
-!4 = !MDCompositeType(tag: DW_TAG_structure_type, name: "p", line: 3, size: 128, align: 64, file: !5, elements: !6, identifier: "_ZTS1p")
-!5 = !MDFile(filename: "pr22393.cc", directory: "")
+!4 = !DICompositeType(tag: DW_TAG_structure_type, name: "p", line: 3, size: 128, align: 64, file: !5, elements: !6, identifier: "_ZTS1p")
+!5 = !DIFile(filename: "pr22393.cc", directory: "")
 !6 = !{!7, !9}
-!7 = !MDDerivedType(tag: DW_TAG_member, name: "s", line: 4, size: 64, align: 64, file: !5, scope: !"_ZTS1p", baseType: !8)
-!8 = !MDBasicType(tag: DW_TAG_base_type, name: "long unsigned int", size: 64, align: 64, encoding: DW_ATE_unsigned)
-!9 = !MDDerivedType(tag: DW_TAG_member, name: "t", line: 5, size: 64, align: 64, offset: 64, file: !5, scope: !"_ZTS1p", baseType: !8)
-!10 = !MDCompositeType(tag: DW_TAG_structure_type, name: "r", line: 8, size: 320, align: 64, file: !5, elements: !11, identifier: "_ZTS1r")
+!7 = !DIDerivedType(tag: DW_TAG_member, name: "s", line: 4, size: 64, align: 64, file: !5, scope: !"_ZTS1p", baseType: !8)
+!8 = !DIBasicType(tag: DW_TAG_base_type, name: "long unsigned int", size: 64, align: 64, encoding: DW_ATE_unsigned)
+!9 = !DIDerivedType(tag: DW_TAG_member, name: "t", line: 5, size: 64, align: 64, offset: 64, file: !5, scope: !"_ZTS1p", baseType: !8)
+!10 = !DICompositeType(tag: DW_TAG_structure_type, name: "r", line: 8, size: 320, align: 64, file: !5, elements: !11, identifier: "_ZTS1r")
 !11 = !{!12, !14, !15}
-!12 = !MDDerivedType(tag: DW_TAG_member, name: "i", line: 9, size: 32, align: 32, file: !5, scope: !"_ZTS1r", baseType: !13)
-!13 = !MDBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
-!14 = !MDDerivedType(tag: DW_TAG_member, name: "x", line: 10, size: 128, align: 64, offset: 64, file: !5, scope: !"_ZTS1r", baseType: !"_ZTS1p")
-!15 = !MDDerivedType(tag: DW_TAG_member, name: "y", line: 11, size: 128, align: 64, offset: 192, file: !5, scope: !"_ZTS1r", baseType: !"_ZTS1p")
+!12 = !DIDerivedType(tag: DW_TAG_member, name: "i", line: 9, size: 32, align: 32, file: !5, scope: !"_ZTS1r", baseType: !13)
+!13 = !DIBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
+!14 = !DIDerivedType(tag: DW_TAG_member, name: "x", line: 10, size: 128, align: 64, offset: 64, file: !5, scope: !"_ZTS1r", baseType: !"_ZTS1p")
+!15 = !DIDerivedType(tag: DW_TAG_member, name: "y", line: 11, size: 128, align: 64, offset: 192, file: !5, scope: !"_ZTS1r", baseType: !"_ZTS1p")
 !16 = !{!17}
-!17 = !MDSubprogram(name: "test", linkageName: "_Z4testv", line: 18, isLocal: false, isDefinition: true, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 18, file: !5, scope: !18, type: !19, function: i32 ()* @_Z4testv, variables: !2)
-!18 = !MDFile(filename: "pr22393.cc", directory: "")
-!19 = !MDSubroutineType(types: !20)
+!17 = !DISubprogram(name: "test", linkageName: "_Z4testv", line: 18, isLocal: false, isDefinition: true, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 18, file: !5, scope: !18, type: !19, function: i32 ()* @_Z4testv, variables: !2)
+!18 = !DIFile(filename: "pr22393.cc", directory: "")
+!19 = !DISubroutineType(types: !20)
 !20 = !{!13}
 !21 = !{i32 2, !"Dwarf Version", i32 4}
 !22 = !{i32 2, !"Debug Info Version", i32 3}
 !23 = !{!"clang version 3.7.0 "}
-!24 = !MDLocation(line: 19, scope: !25)
-!25 = distinct !MDLexicalBlock(line: 19, column: 0, file: !5, scope: !17)
-!26 = !MDLocation(line: 19, scope: !17)
-!27 = !MDLocation(line: 20, scope: !25)
-!28 = !MDLocalVariable(tag: DW_TAG_auto_variable, name: "y", line: 21, scope: !17, file: !18, type: !"_ZTS1p")
-!29 = !MDExpression()
-!30 = !MDLocation(line: 21, scope: !17)
-!31 = !MDLocalVariable(tag: DW_TAG_auto_variable, name: "r", line: 22, scope: !17, file: !18, type: !"_ZTS1r")
-!32 = !MDLocation(line: 22, scope: !17)
-!33 = !MDLocation(line: 23, scope: !17)
-!34 = !MDLocation(line: 24, scope: !17)
+!24 = !DILocation(line: 19, scope: !25)
+!25 = distinct !DILexicalBlock(line: 19, column: 0, file: !5, scope: !17)
+!26 = !DILocation(line: 19, scope: !17)
+!27 = !DILocation(line: 20, scope: !25)
+!28 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "y", line: 21, scope: !17, file: !18, type: !"_ZTS1p")
+!29 = !DIExpression()
+!30 = !DILocation(line: 21, scope: !17)
+!31 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "r", line: 22, scope: !17, file: !18, type: !"_ZTS1r")
+!32 = !DILocation(line: 22, scope: !17)
+!33 = !DILocation(line: 23, scope: !17)
+!34 = !DILocation(line: 24, scope: !17)
diff --git a/test/DebugInfo/X86/sroasplit-5.ll b/test/DebugInfo/X86/sroasplit-5.ll
index b60e022f451..ce6b4544c56 100644
--- a/test/DebugInfo/X86/sroasplit-5.ll
+++ b/test/DebugInfo/X86/sroasplit-5.ll
@@ -20,9 +20,9 @@ target triple = "x86_64-unknown-linux-gnu"
 ;
 ; There should be no debug info for the padding.
 ; CHECK-NOT: DW_OP_bit_piece, 56
-; CHECK: MDExpression(DW_OP_bit_piece, 32, 24)
+; CHECK: DIExpression(DW_OP_bit_piece, 32, 24)
 ; CHECK-NOT: DW_OP_bit_piece, 56
-; CHECK: MDExpression(DW_OP_bit_piece, 0, 32)
+; CHECK: DIExpression(DW_OP_bit_piece, 0, 32)
 ; CHECK-NOT: DW_OP_bit_piece, 56
 %struct.prog_src_register = type { i32, i24 }
 
@@ -66,26 +66,26 @@ attributes #2 = { nounwind }
 !llvm.module.flags = !{!13, !14}
 !llvm.ident = !{!15}
 
-!0 = !MDCompileUnit(language: DW_LANG_C99, producer: "clang version 3.7.0 ", isOptimized: false, emissionKind: 1, file: !1, enums: !2, retainedTypes: !2, subprograms: !3, globals: !2, imports: !2)
-!1 = !MDFile(filename: "<stdin>", directory: "")
+!0 = !DICompileUnit(language: DW_LANG_C99, producer: "clang version 3.7.0 ", isOptimized: false, emissionKind: 1, file: !1, enums: !2, retainedTypes: !2, subprograms: !3, globals: !2, imports: !2)
+!1 = !DIFile(filename: "<stdin>", directory: "")
 !2 = !{}
 !3 = !{!4}
-!4 = !MDSubprogram(name: "src_reg_for_float", line: 7, isLocal: false, isDefinition: true, isOptimized: false, scopeLine: 7, file: !5, scope: !6, type: !7, function: i64 ()* @src_reg_for_float, variables: !2)
-!5 = !MDFile(filename: "pr22495.c", directory: "")
-!6 = !MDFile(filename: "pr22495.c", directory: "")
-!7 = !MDSubroutineType(types: !8)
+!4 = !DISubprogram(name: "src_reg_for_float", line: 7, isLocal: false, isDefinition: true, isOptimized: false, scopeLine: 7, file: !5, scope: !6, type: !7, function: i64 ()* @src_reg_for_float, variables: !2)
+!5 = !DIFile(filename: "pr22495.c", directory: "")
+!6 = !DIFile(filename: "pr22495.c", directory: "")
+!7 = !DISubroutineType(types: !8)
 !8 = !{!9}
-!9 = !MDCompositeType(tag: DW_TAG_structure_type, name: "prog_src_register", line: 1, size: 64, align: 32, file: !5, elements: !10)
+!9 = !DICompositeType(tag: DW_TAG_structure_type, name: "prog_src_register", line: 1, size: 64, align: 32, file: !5, elements: !10)
 !10 = !{!11}
-!11 = !MDDerivedType(tag: DW_TAG_member, name: "Index", line: 3, size: 13, align: 32, offset: 4, file: !5, scope: !9, baseType: !12)
-!12 = !MDBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
+!11 = !DIDerivedType(tag: DW_TAG_member, name: "Index", line: 3, size: 13, align: 32, offset: 4, file: !5, scope: !9, baseType: !12)
+!12 = !DIBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
 !13 = !{i32 2, !"Dwarf Version", i32 4}
 !14 = !{i32 2, !"Debug Info Version", i32 3}
 !15 = !{!"clang version 3.7.0 "}
-!16 = !MDLocalVariable(tag: DW_TAG_auto_variable, name: "a", line: 8, scope: !4, file: !6, type: !9)
-!17 = !MDExpression()
-!18 = !MDLocation(line: 8, scope: !4)
-!19 = !MDLocation(line: 9, scope: !4)
-!20 = !MDLocalVariable(tag: DW_TAG_auto_variable, name: "local", line: 10, scope: !4, file: !6, type: !12)
-!21 = !MDLocation(line: 10, scope: !4)
-!22 = !MDLocation(line: 11, scope: !4)
+!16 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "a", line: 8, scope: !4, file: !6, type: !9)
+!17 = !DIExpression()
+!18 = !DILocation(line: 8, scope: !4)
+!19 = !DILocation(line: 9, scope: !4)
+!20 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "local", line: 10, scope: !4, file: !6, type: !12)
+!21 = !DILocation(line: 10, scope: !4)
+!22 = !DILocation(line: 11, scope: !4)
diff --git a/test/DebugInfo/X86/stmt-list-multiple-compile-units.ll b/test/DebugInfo/X86/stmt-list-multiple-compile-units.ll
index 8e1c3c92448..86a1647d6fa 100644
--- a/test/DebugInfo/X86/stmt-list-multiple-compile-units.ll
+++ b/test/DebugInfo/X86/stmt-list-multiple-compile-units.ll
@@ -51,16 +51,18 @@
 
 ; PR15408
 ; ASM: Lcu_begin0:
-; ASM: Lset3 = Lline_table_start0-Lsection_line ## DW_AT_stmt_list
-; ASM-NEXT: .long   Lset3
+; ASM-NOT: Lcu_begin
+; ASM: Lset[[LT:[0-9]+]] = Lline_table_start0-Lsection_line ## DW_AT_stmt_list
+; ASM-NEXT: .long   Lset[[LT]]
 ; ASM: Lcu_begin1:
-; ASM: Lset13 = Lline_table_start0-Lsection_line ## DW_AT_stmt_list
-; ASM-NEXT: .long   Lset13
+; ASM-NOT: Lcu_begin
+; ASM: Lset[[LT:[0-9]+]] = Lline_table_start0-Lsection_line ## DW_AT_stmt_list
+; ASM-NEXT: .long   Lset[[LT]]
 define i32 @test(i32 %a) nounwind uwtable ssp {
 entry:
   %a.addr = alloca i32, align 4
   store i32 %a, i32* %a.addr, align 4
-  call void @llvm.dbg.declare(metadata i32* %a.addr, metadata !15, metadata !MDExpression()), !dbg !16
+  call void @llvm.dbg.declare(metadata i32* %a.addr, metadata !15, metadata !DIExpression()), !dbg !16
   %0 = load i32, i32* %a.addr, align 4, !dbg !17
   %call = call i32 @fn(i32 %0), !dbg !17
   ret i32 %call, !dbg !17
@@ -72,33 +74,33 @@ define i32 @fn(i32 %a) nounwind uwtable ssp {
 entry:
   %a.addr = alloca i32, align 4
   store i32 %a, i32* %a.addr, align 4
-  call void @llvm.dbg.declare(metadata i32* %a.addr, metadata !19, metadata !MDExpression()), !dbg !20
+  call void @llvm.dbg.declare(metadata i32* %a.addr, metadata !19, metadata !DIExpression()), !dbg !20
   %0 = load i32, i32* %a.addr, align 4, !dbg !21
   ret i32 %0, !dbg !21
 }
 
 !llvm.dbg.cu = !{!0, !10}
 !llvm.module.flags = !{!25}
-!0 = !MDCompileUnit(language: DW_LANG_C99, producer: "clang version 3.3", isOptimized: false, emissionKind: 1, file: !23, enums: !1, retainedTypes: !1, subprograms: !3, globals: !1, imports:  !1)
+!0 = !DICompileUnit(language: DW_LANG_C99, producer: "clang version 3.3", isOptimized: false, emissionKind: 1, file: !23, enums: !1, retainedTypes: !1, subprograms: !3, globals: !1, imports:  !1)
 !1 = !{}
 !3 = !{!5}
-!5 = !MDSubprogram(name: "test", line: 2, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 3, file: !23, scope: !6, type: !7, function: i32 (i32)* @test, variables: !1)
-!6 = !MDFile(filename: "simple.c", directory: "/private/tmp")
-!7 = !MDSubroutineType(types: !8)
+!5 = !DISubprogram(name: "test", line: 2, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 3, file: !23, scope: !6, type: !7, function: i32 (i32)* @test, variables: !1)
+!6 = !DIFile(filename: "simple.c", directory: "/private/tmp")
+!7 = !DISubroutineType(types: !8)
 !8 = !{!9, !9}
-!9 = !MDBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
-!10 = !MDCompileUnit(language: DW_LANG_C99, producer: "clang version 3.3 (trunk 172862)", isOptimized: false, emissionKind: 1, file: !24, enums: !1, retainedTypes: !1, subprograms: !11, globals: !1, imports:  !1)
+!9 = !DIBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
+!10 = !DICompileUnit(language: DW_LANG_C99, producer: "clang version 3.3 (trunk 172862)", isOptimized: false, emissionKind: 1, file: !24, enums: !1, retainedTypes: !1, subprograms: !11, globals: !1, imports:  !1)
 !11 = !{!13}
-!13 = !MDSubprogram(name: "fn", line: 1, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 1, file: !24, scope: !14, type: !7, function: i32 (i32)* @fn, variables: !1)
-!14 = !MDFile(filename: "simple2.c", directory: "/private/tmp")
-!15 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "a", line: 2, arg: 1, scope: !5, file: !6, type: !9)
-!16 = !MDLocation(line: 2, scope: !5)
-!17 = !MDLocation(line: 4, scope: !18)
-!18 = distinct !MDLexicalBlock(line: 3, column: 0, file: !23, scope: !5)
-!19 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "a", line: 1, arg: 1, scope: !13, file: !14, type: !9)
-!20 = !MDLocation(line: 1, scope: !13)
-!21 = !MDLocation(line: 2, scope: !22)
-!22 = distinct !MDLexicalBlock(line: 1, column: 0, file: !24, scope: !13)
-!23 = !MDFile(filename: "simple.c", directory: "/private/tmp")
-!24 = !MDFile(filename: "simple2.c", directory: "/private/tmp")
+!13 = !DISubprogram(name: "fn", line: 1, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 1, file: !24, scope: !14, type: !7, function: i32 (i32)* @fn, variables: !1)
+!14 = !DIFile(filename: "simple2.c", directory: "/private/tmp")
+!15 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "a", line: 2, arg: 1, scope: !5, file: !6, type: !9)
+!16 = !DILocation(line: 2, scope: !5)
+!17 = !DILocation(line: 4, scope: !18)
+!18 = distinct !DILexicalBlock(line: 3, column: 0, file: !23, scope: !5)
+!19 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "a", line: 1, arg: 1, scope: !13, file: !14, type: !9)
+!20 = !DILocation(line: 1, scope: !13)
+!21 = !DILocation(line: 2, scope: !22)
+!22 = distinct !DILexicalBlock(line: 1, column: 0, file: !24, scope: !13)
+!23 = !DIFile(filename: "simple.c", directory: "/private/tmp")
+!24 = !DIFile(filename: "simple2.c", directory: "/private/tmp")
 !25 = !{i32 1, !"Debug Info Version", i32 3}
diff --git a/test/DebugInfo/X86/stmt-list.ll b/test/DebugInfo/X86/stmt-list.ll
index 9bf9edcdc3a..1f5189e362a 100644
--- a/test/DebugInfo/X86/stmt-list.ll
+++ b/test/DebugInfo/X86/stmt-list.ll
@@ -14,10 +14,10 @@ entry:
 !llvm.module.flags = !{!7}
 !5 = !{!0}
 
-!0 = !MDSubprogram(name: "f", line: 1, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: true, scopeLine: 1, file: !6, scope: !1, type: !3, function: void ()* @f)
-!1 = !MDFile(filename: "test2.c", directory: "/home/espindola/llvm")
-!2 = !MDCompileUnit(language: DW_LANG_C99, producer: "clang version 3.0 ()", isOptimized: true, emissionKind: 0, file: !6, enums: !{}, retainedTypes: !{}, subprograms: !5)
-!3 = !MDSubroutineType(types: !4)
+!0 = !DISubprogram(name: "f", line: 1, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: true, scopeLine: 1, file: !6, scope: !1, type: !3, function: void ()* @f)
+!1 = !DIFile(filename: "test2.c", directory: "/home/espindola/llvm")
+!2 = !DICompileUnit(language: DW_LANG_C99, producer: "clang version 3.0 ()", isOptimized: true, emissionKind: 0, file: !6, enums: !{}, retainedTypes: !{}, subprograms: !5)
+!3 = !DISubroutineType(types: !4)
 !4 = !{null}
-!6 = !MDFile(filename: "test2.c", directory: "/home/espindola/llvm")
+!6 = !DIFile(filename: "test2.c", directory: "/home/espindola/llvm")
 !7 = !{i32 1, !"Debug Info Version", i32 3}
diff --git a/test/DebugInfo/X86/stringpool.ll b/test/DebugInfo/X86/stringpool.ll
index 4dda4872b37..62b2b1bb55c 100644
--- a/test/DebugInfo/X86/stringpool.ll
+++ b/test/DebugInfo/X86/stringpool.ll
@@ -6,19 +6,19 @@
 !llvm.dbg.cu = !{!0}
 !llvm.module.flags = !{!9}
 
-!0 = !MDCompileUnit(language: DW_LANG_C99, producer: "clang version 3.1 (trunk 143009)", isOptimized: true, emissionKind: 0, file: !8, enums: !1, retainedTypes: !1, subprograms: !1, globals: !3, imports:  !1)
+!0 = !DICompileUnit(language: DW_LANG_C99, producer: "clang version 3.1 (trunk 143009)", isOptimized: true, emissionKind: 0, file: !8, enums: !1, retainedTypes: !1, subprograms: !1, globals: !3, imports:  !1)
 !1 = !{}
 !3 = !{!5}
-!5 = !MDGlobalVariable(name: "yyyy", line: 1, isLocal: false, isDefinition: true, scope: null, file: !6, type: !7, variable: i32* @yyyy)
-!6 = !MDFile(filename: "z.c", directory: "/home/nicholas")
-!7 = !MDBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
-!8 = !MDFile(filename: "z.c", directory: "/home/nicholas")
+!5 = !DIGlobalVariable(name: "yyyy", line: 1, isLocal: false, isDefinition: true, scope: null, file: !6, type: !7, variable: i32* @yyyy)
+!6 = !DIFile(filename: "z.c", directory: "/home/nicholas")
+!7 = !DIBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
+!8 = !DIFile(filename: "z.c", directory: "/home/nicholas")
 
 ; Verify that "yyyy" ended up in the stringpool.
 ; LINUX: .section .debug_str,"MS",@progbits,1
 ; LINUX: yyyy
 ; DARWIN: .section __DWARF,__debug_str,regular,debug
-; DARWIN: yyyy
+; DARWIN: .asciz "yyyy" ## string offset=[[YYYY:[0-9]+]]
 
 ; Verify that we refer to 'yyyy' with a relocation.
 ; LINUX:      .long   .Linfo_string3          # DW_AT_name
@@ -30,9 +30,8 @@
 ; LINUX-NEXT: .byte   3
 ; LINUX-NEXT: .quad   yyyy
 
-; Verify that we refer to 'yyyy' without a relocation.
-; DARWIN: Lset[[ID:[0-9]+]] = Linfo_string3-Linfo_string ## DW_AT_name
-; DARWIN-NEXT:        .long   Lset[[ID]]
+; Verify that we refer to 'yyyy' with a direct offset.
+; DARWIN:             .long   [[YYYY]]
 ; DARWIN-NEXT:        .long   {{[0-9]+}}              ## DW_AT_type
 ; DARWIN-NEXT:                                        ## DW_AT_external
 ; DARWIN-NEXT:        .byte   1                       ## DW_AT_decl_file
diff --git a/test/DebugInfo/X86/struct-loc.ll b/test/DebugInfo/X86/struct-loc.ll
index 711eaa1719d..2c133b9b66e 100644
--- a/test/DebugInfo/X86/struct-loc.ll
+++ b/test/DebugInfo/X86/struct-loc.ll
@@ -14,14 +14,14 @@
 !llvm.dbg.cu = !{!0}
 !llvm.module.flags = !{!12}
 
-!0 = !MDCompileUnit(language: DW_LANG_C99, producer: "clang version 3.1 (trunk 152837) (llvm/trunk 152845)", isOptimized: false, emissionKind: 0, file: !11, enums: !1, retainedTypes: !1, subprograms: !1, globals: !3, imports:  !1)
+!0 = !DICompileUnit(language: DW_LANG_C99, producer: "clang version 3.1 (trunk 152837) (llvm/trunk 152845)", isOptimized: false, emissionKind: 0, file: !11, enums: !1, retainedTypes: !1, subprograms: !1, globals: !3, imports:  !1)
 !1 = !{}
 !3 = !{!5}
-!5 = !MDGlobalVariable(name: "f", line: 5, isLocal: false, isDefinition: true, scope: null, file: !6, type: !7, variable: %struct.foo* @f)
-!6 = !MDFile(filename: "struct_bug.c", directory: "/Users/echristo/tmp")
-!7 = !MDCompositeType(tag: DW_TAG_structure_type, name: "foo", line: 1, size: 32, align: 32, file: !11, elements: !8)
+!5 = !DIGlobalVariable(name: "f", line: 5, isLocal: false, isDefinition: true, scope: null, file: !6, type: !7, variable: %struct.foo* @f)
+!6 = !DIFile(filename: "struct_bug.c", directory: "/Users/echristo/tmp")
+!7 = !DICompositeType(tag: DW_TAG_structure_type, name: "foo", line: 1, size: 32, align: 32, file: !11, elements: !8)
 !8 = !{!9}
-!9 = !MDDerivedType(tag: DW_TAG_member, name: "a", line: 2, size: 32, align: 32, file: !11, scope: !7, baseType: !10)
-!10 = !MDBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
-!11 = !MDFile(filename: "struct_bug.c", directory: "/Users/echristo/tmp")
+!9 = !DIDerivedType(tag: DW_TAG_member, name: "a", line: 2, size: 32, align: 32, file: !11, scope: !7, baseType: !10)
+!10 = !DIBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
+!11 = !DIFile(filename: "struct_bug.c", directory: "/Users/echristo/tmp")
 !12 = !{i32 1, !"Debug Info Version", i32 3}
diff --git a/test/DebugInfo/X86/subrange-type.ll b/test/DebugInfo/X86/subrange-type.ll
index 4bb1ee8f31e..d83cca6c877 100644
--- a/test/DebugInfo/X86/subrange-type.ll
+++ b/test/DebugInfo/X86/subrange-type.ll
@@ -12,7 +12,7 @@ entry:
   %retval = alloca i32, align 4
   %i = alloca [2 x i32], align 4
   store i32 0, i32* %retval
-  call void @llvm.dbg.declare(metadata [2 x i32]* %i, metadata !10, metadata !MDExpression()), !dbg !15
+  call void @llvm.dbg.declare(metadata [2 x i32]* %i, metadata !10, metadata !DIExpression()), !dbg !15
   ret i32 0, !dbg !16
 }
 
@@ -21,20 +21,20 @@ declare void @llvm.dbg.declare(metadata, metadata, metadata) nounwind readnone
 !llvm.dbg.cu = !{!0}
 !llvm.module.flags = !{!18}
 
-!0 = !MDCompileUnit(language: DW_LANG_C99, producer: "clang version 3.3 (trunk 171472) (llvm/trunk 171487)", isOptimized: false, emissionKind: 0, file: !17, enums: !1, retainedTypes: !1, subprograms: !3, globals: !1, imports:  !1)
+!0 = !DICompileUnit(language: DW_LANG_C99, producer: "clang version 3.3 (trunk 171472) (llvm/trunk 171487)", isOptimized: false, emissionKind: 0, file: !17, enums: !1, retainedTypes: !1, subprograms: !3, globals: !1, imports:  !1)
 !1 = !{}
 !3 = !{!5}
-!5 = !MDSubprogram(name: "main", line: 2, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 3, file: !6, scope: !6, type: !7, function: i32 ()* @main, variables: !1)
-!6 = !MDFile(filename: "foo.c", directory: "/usr/local/google/home/echristo/tmp")
-!7 = !MDSubroutineType(types: !8)
+!5 = !DISubprogram(name: "main", line: 2, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 3, file: !6, scope: !6, type: !7, function: i32 ()* @main, variables: !1)
+!6 = !DIFile(filename: "foo.c", directory: "/usr/local/google/home/echristo/tmp")
+!7 = !DISubroutineType(types: !8)
 !8 = !{!9}
-!9 = !MDBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
-!10 = !MDLocalVariable(tag: DW_TAG_auto_variable, name: "i", line: 4, scope: !11, file: !6, type: !12)
-!11 = distinct !MDLexicalBlock(line: 3, column: 0, file: !6, scope: !5)
-!12 = !MDCompositeType(tag: DW_TAG_array_type, size: 64, align: 32, baseType: !9, elements: !13)
+!9 = !DIBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
+!10 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "i", line: 4, scope: !11, file: !6, type: !12)
+!11 = distinct !DILexicalBlock(line: 3, column: 0, file: !6, scope: !5)
+!12 = !DICompositeType(tag: DW_TAG_array_type, size: 64, align: 32, baseType: !9, elements: !13)
 !13 = !{!14}
-!14 = !MDSubrange(count: 2)
-!15 = !MDLocation(line: 4, scope: !11)
-!16 = !MDLocation(line: 6, scope: !11)
-!17 = !MDFile(filename: "foo.c", directory: "/usr/local/google/home/echristo/tmp")
+!14 = !DISubrange(count: 2)
+!15 = !DILocation(line: 4, scope: !11)
+!16 = !DILocation(line: 6, scope: !11)
+!17 = !DIFile(filename: "foo.c", directory: "/usr/local/google/home/echristo/tmp")
 !18 = !{i32 1, !"Debug Info Version", i32 3}
diff --git a/test/DebugInfo/X86/subreg.ll b/test/DebugInfo/X86/subreg.ll
index ce3870f44f0..b850ce800e3 100644
--- a/test/DebugInfo/X86/subreg.ll
+++ b/test/DebugInfo/X86/subreg.ll
@@ -9,7 +9,7 @@
 
 define i16 @f(i16 signext %zzz) nounwind {
 entry:
-  call void @llvm.dbg.value(metadata i16 %zzz, i64 0, metadata !0, metadata !MDExpression()), !dbg !MDLocation(scope: !1)
+  call void @llvm.dbg.value(metadata i16 %zzz, i64 0, metadata !0, metadata !DIExpression()), !dbg !DILocation(scope: !1)
   %conv = sext i16 %zzz to i32, !dbg !7
   %conv1 = trunc i32 %conv to i16
   ret i16 %conv1
@@ -21,14 +21,14 @@ declare void @llvm.dbg.value(metadata, i64, metadata, metadata) nounwind readnon
 !llvm.module.flags = !{!11}
 !9 = !{!1}
 
-!0 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "zzz", line: 3, arg: 1, scope: !1, file: !2, type: !6)
-!1 = !MDSubprogram(name: "f", line: 3, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 3, file: !10, scope: !2, type: !4, function: i16 (i16)* @f)
-!2 = !MDFile(filename: "/home/espindola/llvm/test.c", directory: "/home/espindola/tmpfs/build")
-!3 = !MDCompileUnit(language: DW_LANG_C99, producer: "clang version 3.0 ()", isOptimized: false, emissionKind: 1, file: !10, enums: !{}, retainedTypes: !{}, subprograms: !9, imports:  null)
-!4 = !MDSubroutineType(types: !5)
+!0 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "zzz", line: 3, arg: 1, scope: !1, file: !2, type: !6)
+!1 = !DISubprogram(name: "f", line: 3, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 3, file: !10, scope: !2, type: !4, function: i16 (i16)* @f)
+!2 = !DIFile(filename: "/home/espindola/llvm/test.c", directory: "/home/espindola/tmpfs/build")
+!3 = !DICompileUnit(language: DW_LANG_C99, producer: "clang version 3.0 ()", isOptimized: false, emissionKind: 1, file: !10, enums: !{}, retainedTypes: !{}, subprograms: !9, imports:  null)
+!4 = !DISubroutineType(types: !5)
 !5 = !{null}
-!6 = !MDBasicType(tag: DW_TAG_base_type, name: "short", size: 16, align: 16, encoding: DW_ATE_signed)
-!7 = !MDLocation(line: 4, column: 22, scope: !8)
-!8 = distinct !MDLexicalBlock(line: 3, column: 19, file: !10, scope: !1)
-!10 = !MDFile(filename: "/home/espindola/llvm/test.c", directory: "/home/espindola/tmpfs/build")
+!6 = !DIBasicType(tag: DW_TAG_base_type, name: "short", size: 16, align: 16, encoding: DW_ATE_signed)
+!7 = !DILocation(line: 4, column: 22, scope: !8)
+!8 = distinct !DILexicalBlock(line: 3, column: 19, file: !10, scope: !1)
+!10 = !DIFile(filename: "/home/espindola/llvm/test.c", directory: "/home/espindola/tmpfs/build")
 !11 = !{i32 1, !"Debug Info Version", i32 3}
diff --git a/test/DebugInfo/X86/subregisters.ll b/test/DebugInfo/X86/subregisters.ll
index 41691d21eb2..a4722aa8cd5 100644
--- a/test/DebugInfo/X86/subregisters.ll
+++ b/test/DebugInfo/X86/subregisters.ll
@@ -40,10 +40,10 @@ target triple = "x86_64-apple-macosx10.9.0"
 ; Function Attrs: noinline nounwind ssp uwtable
 define void @doSomething(%struct.bar* nocapture readonly %b) #0 {
 entry:
-  tail call void @llvm.dbg.value(metadata %struct.bar* %b, i64 0, metadata !15, metadata !MDExpression()), !dbg !25
+  tail call void @llvm.dbg.value(metadata %struct.bar* %b, i64 0, metadata !15, metadata !DIExpression()), !dbg !25
   %a1 = getelementptr inbounds %struct.bar, %struct.bar* %b, i64 0, i32 0, !dbg !26
   %0 = load i32, i32* %a1, align 4, !dbg !26, !tbaa !27
-  tail call void @llvm.dbg.value(metadata i32 %0, i64 0, metadata !16, metadata !MDExpression()), !dbg !26
+  tail call void @llvm.dbg.value(metadata i32 %0, i64 0, metadata !16, metadata !DIExpression()), !dbg !26
   %call = tail call i32 (i8*, ...) @printf(i8* getelementptr inbounds ([4 x i8], [4 x i8]* @.str, i64 0, i64 0), i32 %0) #4, !dbg !32
   ret void, !dbg !33
 }
@@ -59,7 +59,7 @@ define i32 @main() #3 {
 entry:
   %myBar = alloca i64, align 8, !dbg !34
   %tmpcast = bitcast i64* %myBar to %struct.bar*, !dbg !34
-  tail call void @llvm.dbg.declare(metadata %struct.bar* %tmpcast, metadata !21, metadata !MDExpression()), !dbg !34
+  tail call void @llvm.dbg.declare(metadata %struct.bar* %tmpcast, metadata !21, metadata !DIExpression()), !dbg !34
   store i64 17179869187, i64* %myBar, align 8, !dbg !34
   call void @doSomething(%struct.bar* %tmpcast), !dbg !35
   ret i32 0, !dbg !36
@@ -78,40 +78,40 @@ attributes #4 = { nounwind }
 !llvm.module.flags = !{!22, !23}
 !llvm.ident = !{!24}
 
-!0 = !MDCompileUnit(language: DW_LANG_C99, producer: "clang version 3.5 ", isOptimized: true, emissionKind: 1, file: !1, enums: !2, retainedTypes: !2, subprograms: !3, globals: !2, imports: !2)
-!1 = !MDFile(filename: "subregisters.c", directory: "")
+!0 = !DICompileUnit(language: DW_LANG_C99, producer: "clang version 3.5 ", isOptimized: true, emissionKind: 1, file: !1, enums: !2, retainedTypes: !2, subprograms: !3, globals: !2, imports: !2)
+!1 = !DIFile(filename: "subregisters.c", directory: "")
 !2 = !{}
 !3 = !{!4, !17}
-!4 = !MDSubprogram(name: "doSomething", line: 10, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: true, scopeLine: 11, file: !1, scope: !5, type: !6, function: void (%struct.bar*)* @doSomething, variables: !14)
-!5 = !MDFile(filename: "subregisters.c", directory: "")
-!6 = !MDSubroutineType(types: !7)
+!4 = !DISubprogram(name: "doSomething", line: 10, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: true, scopeLine: 11, file: !1, scope: !5, type: !6, function: void (%struct.bar*)* @doSomething, variables: !14)
+!5 = !DIFile(filename: "subregisters.c", directory: "")
+!6 = !DISubroutineType(types: !7)
 !7 = !{null, !8}
-!8 = !MDDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, baseType: !9)
-!9 = !MDCompositeType(tag: DW_TAG_structure_type, name: "bar", line: 3, size: 64, align: 32, file: !1, elements: !10)
+!8 = !DIDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, baseType: !9)
+!9 = !DICompositeType(tag: DW_TAG_structure_type, name: "bar", line: 3, size: 64, align: 32, file: !1, elements: !10)
 !10 = !{!11, !13}
-!11 = !MDDerivedType(tag: DW_TAG_member, name: "a", line: 4, size: 32, align: 32, file: !1, scope: !9, baseType: !12)
-!12 = !MDBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
-!13 = !MDDerivedType(tag: DW_TAG_member, name: "b", line: 5, size: 32, align: 32, offset: 32, file: !1, scope: !9, baseType: !12)
+!11 = !DIDerivedType(tag: DW_TAG_member, name: "a", line: 4, size: 32, align: 32, file: !1, scope: !9, baseType: !12)
+!12 = !DIBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
+!13 = !DIDerivedType(tag: DW_TAG_member, name: "b", line: 5, size: 32, align: 32, offset: 32, file: !1, scope: !9, baseType: !12)
 !14 = !{!15, !16}
-!15 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "b", line: 10, arg: 1, scope: !4, file: !5, type: !8)
-!16 = !MDLocalVariable(tag: DW_TAG_auto_variable, name: "a", line: 12, scope: !4, file: !5, type: !12)
-!17 = !MDSubprogram(name: "main", line: 16, isLocal: false, isDefinition: true, virtualIndex: 6, isOptimized: true, scopeLine: 17, file: !1, scope: !5, type: !18, function: i32 ()* @main, variables: !20)
-!18 = !MDSubroutineType(types: !19)
+!15 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "b", line: 10, arg: 1, scope: !4, file: !5, type: !8)
+!16 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "a", line: 12, scope: !4, file: !5, type: !12)
+!17 = !DISubprogram(name: "main", line: 16, isLocal: false, isDefinition: true, virtualIndex: 6, isOptimized: true, scopeLine: 17, file: !1, scope: !5, type: !18, function: i32 ()* @main, variables: !20)
+!18 = !DISubroutineType(types: !19)
 !19 = !{!12}
 !20 = !{!21}
-!21 = !MDLocalVariable(tag: DW_TAG_auto_variable, name: "myBar", line: 18, scope: !17, file: !5, type: !9)
+!21 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "myBar", line: 18, scope: !17, file: !5, type: !9)
 !22 = !{i32 2, !"Dwarf Version", i32 2}
 !23 = !{i32 1, !"Debug Info Version", i32 3}
 !24 = !{!"clang version 3.5 "}
-!25 = !MDLocation(line: 10, scope: !4)
-!26 = !MDLocation(line: 12, scope: !4)
+!25 = !DILocation(line: 10, scope: !4)
+!26 = !DILocation(line: 12, scope: !4)
 !27 = !{!28, !29, i64 0}
 !28 = !{!"bar", !29, i64 0, !29, i64 4}
 !29 = !{!"int", !30, i64 0}
 !30 = !{!"omnipotent char", !31, i64 0}
 !31 = !{!"Simple C/C++ TBAA"}
-!32 = !MDLocation(line: 13, scope: !4)
-!33 = !MDLocation(line: 14, scope: !4)
-!34 = !MDLocation(line: 18, scope: !17)
-!35 = !MDLocation(line: 19, scope: !17)
-!36 = !MDLocation(line: 20, scope: !17)
+!32 = !DILocation(line: 13, scope: !4)
+!33 = !DILocation(line: 14, scope: !4)
+!34 = !DILocation(line: 18, scope: !17)
+!35 = !DILocation(line: 19, scope: !17)
+!36 = !DILocation(line: 20, scope: !17)
diff --git a/test/DebugInfo/X86/template.ll b/test/DebugInfo/X86/template.ll
index fdfe383f696..f846b29f576 100644
--- a/test/DebugInfo/X86/template.ll
+++ b/test/DebugInfo/X86/template.ll
@@ -89,42 +89,42 @@ attributes #0 = { nounwind uwtable "less-precise-fpmad"="false" "no-frame-pointe
 !llvm.module.flags = !{!33, !34}
 !llvm.ident = !{!35}
 
-!0 = !MDCompileUnit(language: DW_LANG_C_plus_plus, producer: "clang version 3.6.0 (trunk 224394) (llvm/trunk 224384)", isOptimized: false, emissionKind: 1, file: !1, enums: !2, retainedTypes: !3, subprograms: !9, globals: !30, imports: !2)
-!1 = !MDFile(filename: "template.cpp", directory: "/tmp/dbginfo")
+!0 = !DICompileUnit(language: DW_LANG_C_plus_plus, producer: "clang version 3.6.0 (trunk 224394) (llvm/trunk 224384)", isOptimized: false, emissionKind: 1, file: !1, enums: !2, retainedTypes: !3, subprograms: !9, globals: !30, imports: !2)
+!1 = !DIFile(filename: "template.cpp", directory: "/tmp/dbginfo")
 !2 = !{}
 !3 = !{!4, !8}
-!4 = !MDCompositeType(tag: DW_TAG_structure_type, name: "y_impl<int>", line: 2, size: 8, align: 8, file: !1, elements: !2, templateParams: !5, identifier: "_ZTS6y_implIiE")
+!4 = !DICompositeType(tag: DW_TAG_structure_type, name: "y_impl<int>", line: 2, size: 8, align: 8, file: !1, elements: !2, templateParams: !5, identifier: "_ZTS6y_implIiE")
 !5 = !{!6}
-!6 = !MDTemplateTypeParameter(type: !7)
-!7 = !MDBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
-!8 = !MDCompositeType(tag: DW_TAG_structure_type, name: "nested", line: 2, size: 8, align: 8, file: !1, scope: !"_ZTS6y_implIiE", elements: !2, identifier: "_ZTSN6y_implIiE6nestedE")
+!6 = !DITemplateTypeParameter(type: !7)
+!7 = !DIBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
+!8 = !DICompositeType(tag: DW_TAG_structure_type, name: "nested", line: 2, size: 8, align: 8, file: !1, scope: !"_ZTS6y_implIiE", elements: !2, identifier: "_ZTSN6y_implIiE6nestedE")
 !9 = !{!10, !14, !28}
-!10 = !MDSubprogram(name: "__cxx_global_var_init", line: 3, isLocal: true, isDefinition: true, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 3, file: !1, scope: !11, type: !12, function: void ()* @__cxx_global_var_init, variables: !2)
-!11 = !MDFile(filename: "template.cpp", directory: "/tmp/dbginfo")
-!12 = !MDSubroutineType(types: !13)
+!10 = !DISubprogram(name: "__cxx_global_var_init", line: 3, isLocal: true, isDefinition: true, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 3, file: !1, scope: !11, type: !12, function: void ()* @__cxx_global_var_init, variables: !2)
+!11 = !DIFile(filename: "template.cpp", directory: "/tmp/dbginfo")
+!12 = !DISubroutineType(types: !13)
 !13 = !{null}
-!14 = !MDSubprogram(name: "func<3, &glbl, y_impl, nullptr, 1, 2>", linkageName: "_Z4funcILi3EXadL_Z4glblEE6y_implLDn0EJLi1ELi2EEEiv", line: 1, isLocal: false, isDefinition: true, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 1, file: !1, scope: !11, type: !15, function: i32 ()* @_Z4funcILi3EXadL_Z4glblEE6y_implLDn0EJLi1ELi2EEEiv, templateParams: !17, variables: !2)
-!15 = !MDSubroutineType(types: !16)
+!14 = !DISubprogram(name: "func<3, &glbl, y_impl, nullptr, 1, 2>", linkageName: "_Z4funcILi3EXadL_Z4glblEE6y_implLDn0EJLi1ELi2EEEiv", line: 1, isLocal: false, isDefinition: true, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 1, file: !1, scope: !11, type: !15, function: i32 ()* @_Z4funcILi3EXadL_Z4glblEE6y_implLDn0EJLi1ELi2EEEiv, templateParams: !17, variables: !2)
+!15 = !DISubroutineType(types: !16)
 !16 = !{!7}
 !17 = !{!18, !19, !21, !22, !24}
-!18 = !MDTemplateValueParameter(tag: DW_TAG_template_value_parameter, name: "x", type: !7, value: i32 3)
-!19 = !MDTemplateValueParameter(tag: DW_TAG_template_value_parameter, type: !20, value: i32* @glbl)
-!20 = !MDDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, baseType: !7)
-!21 = !MDTemplateValueParameter(tag: DW_TAG_GNU_template_template_param, name: "y", type: null, value: !"y_impl")
-!22 = !MDTemplateValueParameter(tag: DW_TAG_template_value_parameter, name: "n", type: !23, value: i8 0)
-!23 = !MDBasicType(tag: DW_TAG_unspecified_type, name: "decltype(nullptr)")
-!24 = !MDTemplateValueParameter(tag: DW_TAG_GNU_template_parameter_pack, name: "z", type: null, value: !25)
+!18 = !DITemplateValueParameter(tag: DW_TAG_template_value_parameter, name: "x", type: !7, value: i32 3)
+!19 = !DITemplateValueParameter(tag: DW_TAG_template_value_parameter, type: !20, value: i32* @glbl)
+!20 = !DIDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, baseType: !7)
+!21 = !DITemplateValueParameter(tag: DW_TAG_GNU_template_template_param, name: "y", type: null, value: !"y_impl")
+!22 = !DITemplateValueParameter(tag: DW_TAG_template_value_parameter, name: "n", type: !23, value: i8 0)
+!23 = !DIBasicType(tag: DW_TAG_unspecified_type, name: "decltype(nullptr)")
+!24 = !DITemplateValueParameter(tag: DW_TAG_GNU_template_parameter_pack, name: "z", type: null, value: !25)
 !25 = !{!26, !27}
-!26 = !MDTemplateValueParameter(tag: DW_TAG_template_value_parameter, type: !7, value: i32 1)
-!27 = !MDTemplateValueParameter(tag: DW_TAG_template_value_parameter, type: !7, value: i32 2)
-!28 = !MDSubprogram(name: "", linkageName: "_GLOBAL__sub_I_template.cpp", isLocal: true, isDefinition: true, flags: DIFlagArtificial, isOptimized: false, file: !1, scope: !11, type: !29, function: void ()* @_GLOBAL__sub_I_template.cpp, variables: !2)
-!29 = !MDSubroutineType(types: !2)
+!26 = !DITemplateValueParameter(tag: DW_TAG_template_value_parameter, type: !7, value: i32 1)
+!27 = !DITemplateValueParameter(tag: DW_TAG_template_value_parameter, type: !7, value: i32 2)
+!28 = !DISubprogram(name: "", linkageName: "_GLOBAL__sub_I_template.cpp", isLocal: true, isDefinition: true, flags: DIFlagArtificial, isOptimized: false, file: !1, scope: !11, type: !29, function: void ()* @_GLOBAL__sub_I_template.cpp, variables: !2)
+!29 = !DISubroutineType(types: !2)
 !30 = !{!31, !32}
-!31 = !MDGlobalVariable(name: "glbl", line: 3, isLocal: false, isDefinition: true, scope: null, file: !11, type: !7, variable: i32* @glbl)
-!32 = !MDGlobalVariable(name: "n", line: 4, isLocal: false, isDefinition: true, scope: null, file: !11, type: !"_ZTSN6y_implIiE6nestedE", variable: %"struct.y_impl<int>::nested"* @n)
+!31 = !DIGlobalVariable(name: "glbl", line: 3, isLocal: false, isDefinition: true, scope: null, file: !11, type: !7, variable: i32* @glbl)
+!32 = !DIGlobalVariable(name: "n", line: 4, isLocal: false, isDefinition: true, scope: null, file: !11, type: !"_ZTSN6y_implIiE6nestedE", variable: %"struct.y_impl<int>::nested"* @n)
 !33 = !{i32 2, !"Dwarf Version", i32 4}
 !34 = !{i32 2, !"Debug Info Version", i32 3}
 !35 = !{!"clang version 3.6.0 (trunk 224394) (llvm/trunk 224384)"}
-!36 = !MDLocation(line: 3, column: 12, scope: !10)
-!37 = !MDLocation(line: 1, column: 96, scope: !14)
-!38 = !MDLocation(line: 0, scope: !28)
+!36 = !DILocation(line: 3, column: 12, scope: !10)
+!37 = !DILocation(line: 1, column: 96, scope: !14)
+!38 = !DILocation(line: 0, scope: !28)
diff --git a/test/DebugInfo/X86/tls.ll b/test/DebugInfo/X86/tls.ll
index d43a173485c..b5544b510eb 100644
--- a/test/DebugInfo/X86/tls.ll
+++ b/test/DebugInfo/X86/tls.ll
@@ -1,20 +1,30 @@
 ; RUN: llc %s -o - -filetype=asm -O0 -mtriple=x86_64-unknown-linux-gnu \
-; RUN:   | FileCheck --check-prefix=SINGLE --check-prefix=SINGLE-64 --check-prefix=GNUOP %s
+; RUN:   | FileCheck --check-prefix=NOEMU --check-prefix=SINGLE --check-prefix=SINGLE-64 --check-prefix=GNUOP %s
 
 ; RUN: llc %s -o - -filetype=asm -O0 -mtriple=i386-linux-gnu \
-; RUN:   | FileCheck --check-prefix=SINGLE --check-prefix=SINGLE-32 --check-prefix=GNUOP %s
+; RUN:   | FileCheck --check-prefix=NOEMU --check-prefix=SINGLE --check-prefix=SINGLE-32 --check-prefix=GNUOP %s
 
 ; RUN: llc %s -o - -filetype=asm -O0 -mtriple=x86_64-unknown-linux-gnu -split-dwarf=Enable \
-; RUN:   | FileCheck --check-prefix=FISSION --check-prefix=GNUOP %s
+; RUN:   | FileCheck --check-prefix=NOEMU --check-prefix=FISSION --check-prefix=GNUOP %s
 
 ; RUN: llc %s -o - -filetype=asm -O0 -mtriple=x86_64-scei-ps4 \
-; RUN:   | FileCheck --check-prefix=SINGLE --check-prefix=SINGLE-64 --check-prefix=STDOP %s
+; RUN:   | FileCheck --check-prefix=NOEMU --check-prefix=SINGLE --check-prefix=SINGLE-64 --check-prefix=STDOP %s
 
 ; RUN: llc %s -o - -filetype=asm -O0 -mtriple=x86_64-apple-darwin \
-; RUN:   | FileCheck --check-prefix=DARWIN --check-prefix=STDOP %s
+; RUN:   | FileCheck --check-prefix=NOEMU --check-prefix=DARWIN --check-prefix=STDOP %s
 
 ; RUN: llc %s -o - -filetype=asm -O0 -mtriple=x86_64-unknown-freebsd \
-; RUN:   | FileCheck --check-prefix=SINGLE --check-prefix=SINGLE-64 --check-prefix=GNUOP %s
+; RUN:   | FileCheck --check-prefix=NOEMU --check-prefix=SINGLE --check-prefix=SINGLE-64 --check-prefix=STDOP %s
+
+; RUN: llc %s -o - -filetype=asm -O0 -mtriple=x86_64-unknown-linux-gnu -emulated-tls \
+; RUN:   | FileCheck --check-prefix=SINGLE --check-prefix=EMUSINGLE-64 \
+; RUN:     --check-prefix=EMUGNUOP --check-prefix=EMU %s
+
+; RUN: llc %s -o - -filetype=asm -O0 -mtriple=i386-linux-gnu -emulated-tls \
+; RUN:   | FileCheck --check-prefix=SINGLE --check-prefix=EMUSINGLE-32 \
+; RUN:     --check-prefix=EMUGNUOP --check-prefix=EMU %s
+
+; TODO: Add expected output for -emulated-tls tests.
 
 ; FIXME: add relocation and DWARF expression support to llvm-dwarfdump & use
 ; that here instead of raw assembly printing
@@ -101,22 +111,22 @@ attributes #0 = { nounwind uwtable "less-precise-fpmad"="false" "no-frame-pointe
 !llvm.module.flags = !{!15, !16}
 !llvm.ident = !{!17}
 
-!0 = !MDCompileUnit(language: DW_LANG_C_plus_plus, producer: "clang version 3.5 ", isOptimized: false, splitDebugFilename: "-.dwo", emissionKind: 0, file: !1, enums: !2, retainedTypes: !2, subprograms: !3, globals: !12, imports: !2)
-!1 = !MDFile(filename: "tls.cpp", directory: "/tmp/dbginfo")
+!0 = !DICompileUnit(language: DW_LANG_C_plus_plus, producer: "clang version 3.5 ", isOptimized: false, splitDebugFilename: "-.dwo", emissionKind: 0, file: !1, enums: !2, retainedTypes: !2, subprograms: !3, globals: !12, imports: !2)
+!1 = !DIFile(filename: "tls.cpp", directory: "/tmp/dbginfo")
 !2 = !{}
 !3 = !{!4}
-!4 = !MDSubprogram(name: "func<&glbl>", linkageName: "_Z4funcIXadL_Z4glblEEEiv", line: 5, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 5, file: !1, scope: !5, type: !6, function: i32 ()* @_Z4funcIXadL_Z4glblEEEiv, templateParams: !9, variables: !2)
-!5 = !MDFile(filename: "tls.cpp", directory: "/tmp/dbginfo")
-!6 = !MDSubroutineType(types: !7)
+!4 = !DISubprogram(name: "func<&glbl>", linkageName: "_Z4funcIXadL_Z4glblEEEiv", line: 5, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 5, file: !1, scope: !5, type: !6, function: i32 ()* @_Z4funcIXadL_Z4glblEEEiv, templateParams: !9, variables: !2)
+!5 = !DIFile(filename: "tls.cpp", directory: "/tmp/dbginfo")
+!6 = !DISubroutineType(types: !7)
 !7 = !{!8}
-!8 = !MDBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
+!8 = !DIBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
 !9 = !{!10}
-!10 = !MDTemplateValueParameter(tag: DW_TAG_template_value_parameter, name: "I", type: !11, value: i32* @glbl)
-!11 = !MDDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, baseType: !8)
+!10 = !DITemplateValueParameter(tag: DW_TAG_template_value_parameter, name: "I", type: !11, value: i32* @glbl)
+!11 = !DIDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, baseType: !8)
 !12 = !{!13, !14}
-!13 = !MDGlobalVariable(name: "tls", line: 1, isLocal: false, isDefinition: true, scope: null, file: !5, type: !8, variable: i32* @tls)
-!14 = !MDGlobalVariable(name: "glbl", line: 2, isLocal: false, isDefinition: true, scope: null, file: !5, type: !8, variable: i32* @glbl)
+!13 = !DIGlobalVariable(name: "tls", line: 1, isLocal: false, isDefinition: true, scope: null, file: !5, type: !8, variable: i32* @tls)
+!14 = !DIGlobalVariable(name: "glbl", line: 2, isLocal: false, isDefinition: true, scope: null, file: !5, type: !8, variable: i32* @glbl)
 !15 = !{i32 2, !"Dwarf Version", i32 4}
 !16 = !{i32 1, !"Debug Info Version", i32 3}
 !17 = !{!"clang version 3.5 "}
-!18 = !MDLocation(line: 6, scope: !4)
+!18 = !DILocation(line: 6, scope: !4)
diff --git a/test/DebugInfo/X86/type_units_with_addresses.ll b/test/DebugInfo/X86/type_units_with_addresses.ll
index 9f712b9a957..2a8d664e89b 100644
--- a/test/DebugInfo/X86/type_units_with_addresses.ll
+++ b/test/DebugInfo/X86/type_units_with_addresses.ll
@@ -112,40 +112,40 @@
 !llvm.module.flags = !{!34, !35}
 !llvm.ident = !{!36}
 
-!0 = !MDCompileUnit(language: DW_LANG_C_plus_plus, producer: "clang version 3.5.0 ", isOptimized: false, splitDebugFilename: "tu.dwo", emissionKind: 1, file: !1, enums: !2, retainedTypes: !3, subprograms: !2, globals: !27, imports: !2)
-!1 = !MDFile(filename: "tu.cpp", directory: "/tmp/dbginfo")
+!0 = !DICompileUnit(language: DW_LANG_C_plus_plus, producer: "clang version 3.5.0 ", isOptimized: false, splitDebugFilename: "tu.dwo", emissionKind: 1, file: !1, enums: !2, retainedTypes: !3, subprograms: !2, globals: !27, imports: !2)
+!1 = !DIFile(filename: "tu.cpp", directory: "/tmp/dbginfo")
 !2 = !{}
 !3 = !{!4, !9, !12, !13, !17, !18, !19, !23, !24}
-!4 = !MDCompositeType(tag: DW_TAG_structure_type, name: "S1<&i>", line: 4, size: 8, align: 8, file: !1, elements: !2, templateParams: !5, identifier: "_ZTS2S1IXadL_Z1iEEE")
+!4 = !DICompositeType(tag: DW_TAG_structure_type, name: "S1<&i>", line: 4, size: 8, align: 8, file: !1, elements: !2, templateParams: !5, identifier: "_ZTS2S1IXadL_Z1iEEE")
 !5 = !{!6}
-!6 = !MDTemplateValueParameter(tag: DW_TAG_template_value_parameter, name: "I", type: !7, value: i32* @i)
-!7 = !MDDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, baseType: !8)
-!8 = !MDBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
-!9 = !MDCompositeType(tag: DW_TAG_structure_type, name: "S2", line: 11, size: 8, align: 8, file: !1, elements: !10, identifier: "_ZTS2S2")
+!6 = !DITemplateValueParameter(tag: DW_TAG_template_value_parameter, name: "I", type: !7, value: i32* @i)
+!7 = !DIDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, baseType: !8)
+!8 = !DIBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
+!9 = !DICompositeType(tag: DW_TAG_structure_type, name: "S2", line: 11, size: 8, align: 8, file: !1, elements: !10, identifier: "_ZTS2S2")
 !10 = !{!11}
-!11 = !MDDerivedType(tag: DW_TAG_member, name: "s2_1", line: 12, size: 8, align: 8, file: !1, scope: !"_ZTS2S2", baseType: !"_ZTS4S2_1IXadL_Z1iEEE")
-!12 = !MDCompositeType(tag: DW_TAG_structure_type, name: "S2_1<&i>", line: 9, size: 8, align: 8, file: !1, elements: !2, templateParams: !5, identifier: "_ZTS4S2_1IXadL_Z1iEEE")
-!13 = !MDCompositeType(tag: DW_TAG_structure_type, name: "S3", line: 22, size: 16, align: 8, file: !1, elements: !14, identifier: "_ZTS2S3")
+!11 = !DIDerivedType(tag: DW_TAG_member, name: "s2_1", line: 12, size: 8, align: 8, file: !1, scope: !"_ZTS2S2", baseType: !"_ZTS4S2_1IXadL_Z1iEEE")
+!12 = !DICompositeType(tag: DW_TAG_structure_type, name: "S2_1<&i>", line: 9, size: 8, align: 8, file: !1, elements: !2, templateParams: !5, identifier: "_ZTS4S2_1IXadL_Z1iEEE")
+!13 = !DICompositeType(tag: DW_TAG_structure_type, name: "S3", line: 22, size: 16, align: 8, file: !1, elements: !14, identifier: "_ZTS2S3")
 !14 = !{!15, !16}
-!15 = !MDDerivedType(tag: DW_TAG_member, name: "s3_1", line: 23, size: 8, align: 8, file: !1, scope: !"_ZTS2S3", baseType: !"_ZTS4S3_1IXadL_Z1iEEE")
-!16 = !MDDerivedType(tag: DW_TAG_member, name: "s3_2", line: 24, size: 8, align: 8, offset: 8, file: !1, scope: !"_ZTS2S3", baseType: !"_ZTS4S3_2")
-!17 = !MDCompositeType(tag: DW_TAG_structure_type, name: "S3_1<&i>", line: 18, size: 8, align: 8, file: !1, elements: !2, templateParams: !5, identifier: "_ZTS4S3_1IXadL_Z1iEEE")
-!18 = !MDCompositeType(tag: DW_TAG_structure_type, name: "S3_2", line: 20, size: 8, align: 8, file: !1, elements: !2, identifier: "_ZTS4S3_2")
-!19 = !MDCompositeType(tag: DW_TAG_structure_type, name: "S4", line: 34, size: 16, align: 8, file: !1, elements: !20, identifier: "_ZTS2S4")
+!15 = !DIDerivedType(tag: DW_TAG_member, name: "s3_1", line: 23, size: 8, align: 8, file: !1, scope: !"_ZTS2S3", baseType: !"_ZTS4S3_1IXadL_Z1iEEE")
+!16 = !DIDerivedType(tag: DW_TAG_member, name: "s3_2", line: 24, size: 8, align: 8, offset: 8, file: !1, scope: !"_ZTS2S3", baseType: !"_ZTS4S3_2")
+!17 = !DICompositeType(tag: DW_TAG_structure_type, name: "S3_1<&i>", line: 18, size: 8, align: 8, file: !1, elements: !2, templateParams: !5, identifier: "_ZTS4S3_1IXadL_Z1iEEE")
+!18 = !DICompositeType(tag: DW_TAG_structure_type, name: "S3_2", line: 20, size: 8, align: 8, file: !1, elements: !2, identifier: "_ZTS4S3_2")
+!19 = !DICompositeType(tag: DW_TAG_structure_type, name: "S4", line: 34, size: 16, align: 8, file: !1, elements: !20, identifier: "_ZTS2S4")
 !20 = !{!21, !22}
-!21 = !MDDerivedType(tag: DW_TAG_member, name: "s4_1", line: 35, size: 8, align: 8, file: !1, scope: !"_ZTS2S4", baseType: !"_ZTS4S4_1")
-!22 = !MDDerivedType(tag: DW_TAG_member, name: "s4_2", line: 36, size: 8, align: 8, offset: 8, file: !1, scope: !"_ZTS2S4", baseType: !"_ZTS4S4_2IXadL_Z1iEEE")
-!23 = !MDCompositeType(tag: DW_TAG_structure_type, name: "S4_1", line: 29, size: 8, align: 8, file: !1, elements: !2, identifier: "_ZTS4S4_1")
-!24 = !MDCompositeType(tag: DW_TAG_structure_type, name: "S4_2<&i>", line: 32, size: 8, align: 8, file: !1, elements: !2, templateParams: !25, identifier: "_ZTS4S4_2IXadL_Z1iEEE")
+!21 = !DIDerivedType(tag: DW_TAG_member, name: "s4_1", line: 35, size: 8, align: 8, file: !1, scope: !"_ZTS2S4", baseType: !"_ZTS4S4_1")
+!22 = !DIDerivedType(tag: DW_TAG_member, name: "s4_2", line: 36, size: 8, align: 8, offset: 8, file: !1, scope: !"_ZTS2S4", baseType: !"_ZTS4S4_2IXadL_Z1iEEE")
+!23 = !DICompositeType(tag: DW_TAG_structure_type, name: "S4_1", line: 29, size: 8, align: 8, file: !1, elements: !2, identifier: "_ZTS4S4_1")
+!24 = !DICompositeType(tag: DW_TAG_structure_type, name: "S4_2<&i>", line: 32, size: 8, align: 8, file: !1, elements: !2, templateParams: !25, identifier: "_ZTS4S4_2IXadL_Z1iEEE")
 !25 = !{!26}
-!26 = !MDTemplateValueParameter(tag: DW_TAG_template_value_parameter, name: "T", type: !7, value: i32* @i)
+!26 = !DITemplateValueParameter(tag: DW_TAG_template_value_parameter, name: "T", type: !7, value: i32* @i)
 !27 = !{!28, !30, !31, !32, !33}
-!28 = !MDGlobalVariable(name: "i", line: 1, isLocal: false, isDefinition: true, scope: null, file: !29, type: !8, variable: i32* @i)
-!29 = !MDFile(filename: "tu.cpp", directory: "/tmp/dbginfo")
-!30 = !MDGlobalVariable(name: "a", line: 6, isLocal: false, isDefinition: true, scope: null, file: !29, type: !"_ZTS2S1IXadL_Z1iEEE", variable: %struct.S1* @a)
-!31 = !MDGlobalVariable(name: "s2", line: 15, isLocal: false, isDefinition: true, scope: null, file: !29, type: !"_ZTS2S2", variable: %struct.S2* @s2)
-!32 = !MDGlobalVariable(name: "s3", line: 27, isLocal: false, isDefinition: true, scope: null, file: !29, type: !"_ZTS2S3", variable: %struct.S3* @s3)
-!33 = !MDGlobalVariable(name: "s4", line: 39, isLocal: false, isDefinition: true, scope: null, file: !29, type: !"_ZTS2S4", variable: %struct.S4* @s4)
+!28 = !DIGlobalVariable(name: "i", line: 1, isLocal: false, isDefinition: true, scope: null, file: !29, type: !8, variable: i32* @i)
+!29 = !DIFile(filename: "tu.cpp", directory: "/tmp/dbginfo")
+!30 = !DIGlobalVariable(name: "a", line: 6, isLocal: false, isDefinition: true, scope: null, file: !29, type: !"_ZTS2S1IXadL_Z1iEEE", variable: %struct.S1* @a)
+!31 = !DIGlobalVariable(name: "s2", line: 15, isLocal: false, isDefinition: true, scope: null, file: !29, type: !"_ZTS2S2", variable: %struct.S2* @s2)
+!32 = !DIGlobalVariable(name: "s3", line: 27, isLocal: false, isDefinition: true, scope: null, file: !29, type: !"_ZTS2S3", variable: %struct.S3* @s3)
+!33 = !DIGlobalVariable(name: "s4", line: 39, isLocal: false, isDefinition: true, scope: null, file: !29, type: !"_ZTS2S4", variable: %struct.S4* @s4)
 !34 = !{i32 2, !"Dwarf Version", i32 4}
 !35 = !{i32 1, !"Debug Info Version", i32 3}
 !36 = !{!"clang version 3.5.0 "}
diff --git a/test/DebugInfo/X86/union-const.ll b/test/DebugInfo/X86/union-const.ll
index 7df9afe1db2..e051c6297b7 100644
--- a/test/DebugInfo/X86/union-const.ll
+++ b/test/DebugInfo/X86/union-const.ll
@@ -40,27 +40,27 @@ attributes #1 = { nounwind readnone }
 !llvm.module.flags = !{!17, !18, !19}
 !llvm.ident = !{!20}
 
-!0 = !MDCompileUnit(language: DW_LANG_C99, producer: "clang version 3.7.0 (trunk 226915) (llvm/trunk 226905)", isOptimized: true, emissionKind: 1, file: !1, enums: !2, retainedTypes: !2, subprograms: !3, globals: !2, imports: !2)
-!1 = !MDFile(filename: "union.c", directory: "")
+!0 = !DICompileUnit(language: DW_LANG_C99, producer: "clang version 3.7.0 (trunk 226915) (llvm/trunk 226905)", isOptimized: true, emissionKind: 1, file: !1, enums: !2, retainedTypes: !2, subprograms: !3, globals: !2, imports: !2)
+!1 = !DIFile(filename: "union.c", directory: "")
 !2 = !{}
 !3 = !{!4}
-!4 = !MDSubprogram(name: "mfi_aen_setup", line: 5, isLocal: false, isDefinition: true, isOptimized: true, scopeLine: 5, file: !1, scope: !5, type: !6, function: i32 ()* @mfi_aen_setup, variables: !15)
-!5 = !MDFile(filename: "union.c", directory: "")
-!6 = !MDSubroutineType(types: !7)
+!4 = !DISubprogram(name: "mfi_aen_setup", line: 5, isLocal: false, isDefinition: true, isOptimized: true, scopeLine: 5, file: !1, scope: !5, type: !6, function: i32 ()* @mfi_aen_setup, variables: !15)
+!5 = !DIFile(filename: "union.c", directory: "")
+!6 = !DISubroutineType(types: !7)
 !7 = !{!8}
-!8 = !MDCompositeType(tag: DW_TAG_union_type, name: "mfi_evt", line: 1, size: 32, align: 32, file: !1, elements: !9)
+!8 = !DICompositeType(tag: DW_TAG_union_type, name: "mfi_evt", line: 1, size: 32, align: 32, file: !1, elements: !9)
 !9 = !{!10}
-!10 = !MDDerivedType(tag: DW_TAG_member, name: "members", line: 4, size: 32, align: 32, file: !1, scope: !8, baseType: !11)
-!11 = !MDCompositeType(tag: DW_TAG_structure_type, line: 2, size: 32, align: 32, file: !1, scope: !8, elements: !12)
+!10 = !DIDerivedType(tag: DW_TAG_member, name: "members", line: 4, size: 32, align: 32, file: !1, scope: !8, baseType: !11)
+!11 = !DICompositeType(tag: DW_TAG_structure_type, line: 2, size: 32, align: 32, file: !1, scope: !8, elements: !12)
 !12 = !{!13}
-!13 = !MDDerivedType(tag: DW_TAG_member, name: "reserved", line: 3, size: 32, align: 32, file: !1, scope: !11, baseType: !14)
-!14 = !MDBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
+!13 = !DIDerivedType(tag: DW_TAG_member, name: "reserved", line: 3, size: 32, align: 32, file: !1, scope: !11, baseType: !14)
+!14 = !DIBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
 !15 = !{!16}
-!16 = !MDLocalVariable(tag: DW_TAG_auto_variable, name: "a", line: 6, scope: !4, file: !5, type: !8)
+!16 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "a", line: 6, scope: !4, file: !5, type: !8)
 !17 = !{i32 2, !"Dwarf Version", i32 2}
 !18 = !{i32 2, !"Debug Info Version", i32 3}
 !19 = !{i32 1, !"PIC Level", i32 2}
 !20 = !{!"clang version 3.7.0 (trunk 226915) (llvm/trunk 226905)"}
-!21 = !MDExpression()
-!22 = !MDLocation(line: 6, column: 17, scope: !4)
-!23 = !MDLocation(line: 8, column: 1, scope: !4)
+!21 = !DIExpression()
+!22 = !DILocation(line: 6, column: 17, scope: !4)
+!23 = !DILocation(line: 8, column: 1, scope: !4)
diff --git a/test/DebugInfo/X86/union-template.ll b/test/DebugInfo/X86/union-template.ll
index dfbc1c66cbe..172dcdcbb9d 100644
--- a/test/DebugInfo/X86/union-template.ll
+++ b/test/DebugInfo/X86/union-template.ll
@@ -16,8 +16,8 @@ entry:
   %value.addr = alloca float, align 4
   %tempValue = alloca %"union.PR15637::Value", align 4
   store float %value, float* %value.addr, align 4
-  call void @llvm.dbg.declare(metadata float* %value.addr, metadata !23, metadata !MDExpression()), !dbg !24
-  call void @llvm.dbg.declare(metadata %"union.PR15637::Value"* %tempValue, metadata !25, metadata !MDExpression()), !dbg !26
+  call void @llvm.dbg.declare(metadata float* %value.addr, metadata !23, metadata !DIExpression()), !dbg !24
+  call void @llvm.dbg.declare(metadata %"union.PR15637::Value"* %tempValue, metadata !25, metadata !DIExpression()), !dbg !26
   ret void, !dbg !27
 }
 
@@ -29,31 +29,31 @@ attributes #1 = { nounwind readnone }
 !llvm.dbg.cu = !{!0}
 !llvm.module.flags = !{!28}
 
-!0 = !MDCompileUnit(language: DW_LANG_C_plus_plus, producer: "clang version 3.3 (trunk 178499) (llvm/trunk 178472)", isOptimized: false, emissionKind: 0, file: !1, enums: !2, retainedTypes: !2, subprograms: !3, globals: !9, imports:  !2)
-!1 = !MDFile(filename: "foo.cc", directory: "/usr/local/google/home/echristo/tmp")
+!0 = !DICompileUnit(language: DW_LANG_C_plus_plus, producer: "clang version 3.3 (trunk 178499) (llvm/trunk 178472)", isOptimized: false, emissionKind: 0, file: !1, enums: !2, retainedTypes: !2, subprograms: !3, globals: !9, imports:  !2)
+!1 = !DIFile(filename: "foo.cc", directory: "/usr/local/google/home/echristo/tmp")
 !2 = !{}
 !3 = !{!4}
-!4 = !MDSubprogram(name: "g", linkageName: "_ZN7PR156371gEf", line: 3, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 3, file: !1, scope: !5, type: !6, function: void (float)* @_ZN7PR156371gEf, variables: !2)
-!5 = !MDNamespace(name: "PR15637", line: 1, file: !1, scope: null)
-!6 = !MDSubroutineType(types: !7)
+!4 = !DISubprogram(name: "g", linkageName: "_ZN7PR156371gEf", line: 3, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 3, file: !1, scope: !5, type: !6, function: void (float)* @_ZN7PR156371gEf, variables: !2)
+!5 = !DINamespace(name: "PR15637", line: 1, file: !1, scope: null)
+!6 = !DISubroutineType(types: !7)
 !7 = !{null, !8}
-!8 = !MDBasicType(tag: DW_TAG_base_type, name: "float", size: 32, align: 32, encoding: DW_ATE_float)
+!8 = !DIBasicType(tag: DW_TAG_base_type, name: "float", size: 32, align: 32, encoding: DW_ATE_float)
 !9 = !{!10}
-!10 = !MDGlobalVariable(name: "f", linkageName: "_ZN7PR156371fE", line: 6, isLocal: false, isDefinition: true, scope: !5, file: !11, type: !12, variable: %"union.PR15637::Value"* @_ZN7PR156371fE)
-!11 = !MDFile(filename: "foo.cc", directory: "/usr/local/google/home/echristo/tmp")
-!12 = !MDCompositeType(tag: DW_TAG_union_type, name: "Value<float>", line: 2, size: 32, align: 32, file: !1, scope: !5, elements: !13, templateParams: !21)
+!10 = !DIGlobalVariable(name: "f", linkageName: "_ZN7PR156371fE", line: 6, isLocal: false, isDefinition: true, scope: !5, file: !11, type: !12, variable: %"union.PR15637::Value"* @_ZN7PR156371fE)
+!11 = !DIFile(filename: "foo.cc", directory: "/usr/local/google/home/echristo/tmp")
+!12 = !DICompositeType(tag: DW_TAG_union_type, name: "Value<float>", line: 2, size: 32, align: 32, file: !1, scope: !5, elements: !13, templateParams: !21)
 !13 = !{!14, !16}
-!14 = !MDDerivedType(tag: DW_TAG_member, name: "a", line: 2, size: 32, align: 32, file: !1, scope: !12, baseType: !15)
-!15 = !MDBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
-!16 = !MDSubprogram(name: "Value", line: 2, isLocal: false, isDefinition: false, virtualIndex: 6, flags: DIFlagArtificial | DIFlagPrototyped, isOptimized: false, scopeLine: 2, file: !1, scope: !12, type: !17)
-!17 = !MDSubroutineType(types: !18)
+!14 = !DIDerivedType(tag: DW_TAG_member, name: "a", line: 2, size: 32, align: 32, file: !1, scope: !12, baseType: !15)
+!15 = !DIBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
+!16 = !DISubprogram(name: "Value", line: 2, isLocal: false, isDefinition: false, virtualIndex: 6, flags: DIFlagArtificial | DIFlagPrototyped, isOptimized: false, scopeLine: 2, file: !1, scope: !12, type: !17)
+!17 = !DISubroutineType(types: !18)
 !18 = !{null, !19}
-!19 = !MDDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, flags: DIFlagArtificial | DIFlagObjectPointer, baseType: !12)
+!19 = !DIDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, flags: DIFlagArtificial | DIFlagObjectPointer, baseType: !12)
 !21 = !{!22}
-!22 = !MDTemplateTypeParameter(name: "T", type: !8)
-!23 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "value", line: 3, arg: 1, scope: !4, file: !11, type: !8)
-!24 = !MDLocation(line: 3, scope: !4)
-!25 = !MDLocalVariable(tag: DW_TAG_auto_variable, name: "tempValue", line: 4, scope: !4, file: !11, type: !12)
-!26 = !MDLocation(line: 4, scope: !4)
-!27 = !MDLocation(line: 5, scope: !4)
+!22 = !DITemplateTypeParameter(name: "T", type: !8)
+!23 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "value", line: 3, arg: 1, scope: !4, file: !11, type: !8)
+!24 = !DILocation(line: 3, scope: !4)
+!25 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "tempValue", line: 4, scope: !4, file: !11, type: !12)
+!26 = !DILocation(line: 4, scope: !4)
+!27 = !DILocation(line: 5, scope: !4)
 !28 = !{i32 1, !"Debug Info Version", i32 3}
diff --git a/test/DebugInfo/X86/vector.ll b/test/DebugInfo/X86/vector.ll
index 6d2a4f67abe..f95ad956dea 100644
--- a/test/DebugInfo/X86/vector.ll
+++ b/test/DebugInfo/X86/vector.ll
@@ -12,17 +12,17 @@
 !llvm.dbg.cu = !{!0}
 !llvm.module.flags = !{!13}
 
-!0 = !MDCompileUnit(language: DW_LANG_C99, producer: "clang version 3.3 (trunk 171825) (llvm/trunk 171822)", isOptimized: false, emissionKind: 0, file: !12, enums: !1, retainedTypes: !1, subprograms: !1, globals: !3, imports:  !1)
+!0 = !DICompileUnit(language: DW_LANG_C99, producer: "clang version 3.3 (trunk 171825) (llvm/trunk 171822)", isOptimized: false, emissionKind: 0, file: !12, enums: !1, retainedTypes: !1, subprograms: !1, globals: !3, imports:  !1)
 !1 = !{}
 !3 = !{!5}
-!5 = !MDGlobalVariable(name: "a", line: 3, isLocal: false, isDefinition: true, scope: null, file: !6, type: !7, variable: <4 x i32>* @a)
-!6 = !MDFile(filename: "foo.c", directory: "/Users/echristo")
-!7 = !MDDerivedType(tag: DW_TAG_typedef, name: "v4si", line: 1, file: !12, baseType: !8)
-!8 = !MDCompositeType(tag: DW_TAG_array_type, size: 128, align: 128, flags: DIFlagVector, baseType: !9, elements: !10)
-!9 = !MDBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
+!5 = !DIGlobalVariable(name: "a", line: 3, isLocal: false, isDefinition: true, scope: null, file: !6, type: !7, variable: <4 x i32>* @a)
+!6 = !DIFile(filename: "foo.c", directory: "/Users/echristo")
+!7 = !DIDerivedType(tag: DW_TAG_typedef, name: "v4si", line: 1, file: !12, baseType: !8)
+!8 = !DICompositeType(tag: DW_TAG_array_type, size: 128, align: 128, flags: DIFlagVector, baseType: !9, elements: !10)
+!9 = !DIBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
 !10 = !{!11}
-!11 = !MDSubrange(count: 4)
-!12 = !MDFile(filename: "foo.c", directory: "/Users/echristo")
+!11 = !DISubrange(count: 4)
+!12 = !DIFile(filename: "foo.c", directory: "/Users/echristo")
 
 ; Check that we get an array type with a vector attribute.
 ; CHECK: DW_TAG_array_type
diff --git a/test/DebugInfo/X86/vla.ll b/test/DebugInfo/X86/vla.ll
index ab91854e3fb..dfcc5f1cce2 100644
--- a/test/DebugInfo/X86/vla.ll
+++ b/test/DebugInfo/X86/vla.ll
@@ -27,13 +27,13 @@ entry:
   %saved_stack = alloca i8*
   %cleanup.dest.slot = alloca i32
   store i32 %n, i32* %n.addr, align 4
-  call void @llvm.dbg.declare(metadata i32* %n.addr, metadata !15, metadata !MDExpression()), !dbg !16
+  call void @llvm.dbg.declare(metadata i32* %n.addr, metadata !15, metadata !DIExpression()), !dbg !16
   %0 = load i32, i32* %n.addr, align 4, !dbg !17
   %1 = zext i32 %0 to i64, !dbg !17
   %2 = call i8* @llvm.stacksave(), !dbg !17
   store i8* %2, i8** %saved_stack, !dbg !17
   %vla = alloca i32, i64 %1, align 16, !dbg !17
-  call void @llvm.dbg.declare(metadata i32* %vla, metadata !18, metadata !MDExpression(DW_OP_deref)), !dbg !17
+  call void @llvm.dbg.declare(metadata i32* %vla, metadata !18, metadata !DIExpression(DW_OP_deref)), !dbg !17
   %arrayidx = getelementptr inbounds i32, i32* %vla, i64 0, !dbg !22
   store i32 42, i32* %arrayidx, align 4, !dbg !22
   %3 = load i32, i32* %n.addr, align 4, !dbg !23
@@ -64,9 +64,9 @@ entry:
   %argv.addr = alloca i8**, align 8
   store i32 0, i32* %retval
   store i32 %argc, i32* %argc.addr, align 4
-  call void @llvm.dbg.declare(metadata i32* %argc.addr, metadata !25, metadata !MDExpression()), !dbg !26
+  call void @llvm.dbg.declare(metadata i32* %argc.addr, metadata !25, metadata !DIExpression()), !dbg !26
   store i8** %argv, i8*** %argv.addr, align 8
-  call void @llvm.dbg.declare(metadata i8*** %argv.addr, metadata !27, metadata !MDExpression()), !dbg !26
+  call void @llvm.dbg.declare(metadata i8*** %argv.addr, metadata !27, metadata !DIExpression()), !dbg !26
   %0 = load i32, i32* %argc.addr, align 4, !dbg !28
   %call = call i32 @vla(i32 %0), !dbg !28
   ret i32 %call, !dbg !28
@@ -75,33 +75,33 @@ entry:
 !llvm.dbg.cu = !{!0}
 !llvm.module.flags = !{!29}
 
-!0 = !MDCompileUnit(language: DW_LANG_C99, producer: "clang version 3.3 ", isOptimized: false, emissionKind: 1, file: !1, enums: !2, retainedTypes: !2, subprograms: !3, globals: !2, imports: !2)
-!1 = !MDFile(filename: "vla.c", directory: "")
+!0 = !DICompileUnit(language: DW_LANG_C99, producer: "clang version 3.3 ", isOptimized: false, emissionKind: 1, file: !1, enums: !2, retainedTypes: !2, subprograms: !3, globals: !2, imports: !2)
+!1 = !DIFile(filename: "vla.c", directory: "")
 !2 = !{}
 !3 = !{!4, !9}
-!4 = !MDSubprogram(name: "vla", line: 1, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 1, file: !1, scope: !5, type: !6, function: i32 (i32)* @vla, variables: !2)
-!5 = !MDFile(filename: "vla.c", directory: "")
-!6 = !MDSubroutineType(types: !7)
+!4 = !DISubprogram(name: "vla", line: 1, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 1, file: !1, scope: !5, type: !6, function: i32 (i32)* @vla, variables: !2)
+!5 = !DIFile(filename: "vla.c", directory: "")
+!6 = !DISubroutineType(types: !7)
 !7 = !{!8, !8}
-!8 = !MDBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
-!9 = !MDSubprogram(name: "main", line: 7, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 7, file: !1, scope: !5, type: !10, function: i32 (i32, i8**)* @main, variables: !2)
-!10 = !MDSubroutineType(types: !11)
+!8 = !DIBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
+!9 = !DISubprogram(name: "main", line: 7, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 7, file: !1, scope: !5, type: !10, function: i32 (i32, i8**)* @main, variables: !2)
+!10 = !DISubroutineType(types: !11)
 !11 = !{!8, !8, !12}
-!12 = !MDDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, baseType: !13)
-!13 = !MDDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, baseType: !14)
-!14 = !MDBasicType(tag: DW_TAG_base_type, name: "char", size: 8, align: 8, encoding: DW_ATE_signed_char)
-!15 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "n", line: 1, arg: 1, scope: !4, file: !5, type: !8)
-!16 = !MDLocation(line: 1, scope: !4)
-!17 = !MDLocation(line: 2, scope: !4)
-!18 = !MDLocalVariable(tag: DW_TAG_auto_variable, name: "a", line: 2, scope: !4, file: !5, type: !19)
-!19 = !MDCompositeType(tag: DW_TAG_array_type, align: 32, baseType: !8, elements: !20)
+!12 = !DIDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, baseType: !13)
+!13 = !DIDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, baseType: !14)
+!14 = !DIBasicType(tag: DW_TAG_base_type, name: "char", size: 8, align: 8, encoding: DW_ATE_signed_char)
+!15 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "n", line: 1, arg: 1, scope: !4, file: !5, type: !8)
+!16 = !DILocation(line: 1, scope: !4)
+!17 = !DILocation(line: 2, scope: !4)
+!18 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "a", line: 2, scope: !4, file: !5, type: !19)
+!19 = !DICompositeType(tag: DW_TAG_array_type, align: 32, baseType: !8, elements: !20)
 !20 = !{!21}
-!21 = !MDSubrange(count: -1)
-!22 = !MDLocation(line: 3, scope: !4)
-!23 = !MDLocation(line: 4, scope: !4)
-!24 = !MDLocation(line: 5, scope: !4)
-!25 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "argc", line: 7, arg: 1, scope: !9, file: !5, type: !8)
-!26 = !MDLocation(line: 7, scope: !9)
-!27 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "argv", line: 7, arg: 2, scope: !9, file: !5, type: !12)
-!28 = !MDLocation(line: 8, scope: !9)
+!21 = !DISubrange(count: -1)
+!22 = !DILocation(line: 3, scope: !4)
+!23 = !DILocation(line: 4, scope: !4)
+!24 = !DILocation(line: 5, scope: !4)
+!25 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "argc", line: 7, arg: 1, scope: !9, file: !5, type: !8)
+!26 = !DILocation(line: 7, scope: !9)
+!27 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "argv", line: 7, arg: 2, scope: !9, file: !5, type: !12)
+!28 = !DILocation(line: 8, scope: !9)
 !29 = !{i32 1, !"Debug Info Version", i32 3}
diff --git a/test/DebugInfo/accel-table-hash-collisions.ll b/test/DebugInfo/accel-table-hash-collisions.ll
index 25bd2f426c8..5e109a7de5e 100644
--- a/test/DebugInfo/accel-table-hash-collisions.ll
+++ b/test/DebugInfo/accel-table-hash-collisions.ll
@@ -69,23 +69,23 @@
 !llvm.module.flags = !{!17, !18, !19}
 !llvm.ident = !{!20}
 
-!0 = !MDCompileUnit(language: DW_LANG_C99, file: !1, producer: "clang version 3.7.0 (trunk 231548) (llvm/trunk 231547)", isOptimized: false, runtimeVersion: 0, emissionKind: 1, enums: !2, retainedTypes: !2, subprograms: !2, globals: !3, imports: !2)
-!1 = !MDFile(filename: "hash-collisions.c", directory: "/tmp")
+!0 = !DICompileUnit(language: DW_LANG_C99, file: !1, producer: "clang version 3.7.0 (trunk 231548) (llvm/trunk 231547)", isOptimized: false, runtimeVersion: 0, emissionKind: 1, enums: !2, retainedTypes: !2, subprograms: !2, globals: !3, imports: !2)
+!1 = !DIFile(filename: "hash-collisions.c", directory: "/tmp")
 !2 = !{}
 !3 = !{!4, !6, !7, !8, !9, !10, !11, !12, !13, !14, !15, !16}
-!4 = !MDGlobalVariable(name: "ForceTopDown", scope: !0, file: !1, line: 1, type: !5, isLocal: false, isDefinition: true, variable: i32* @ForceTopDown)
-!5 = !MDBasicType(name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
-!6 = !MDGlobalVariable(name: "_ZNSt3__116allocator_traitsINS_9allocatorINS_11__tree_nodeINS_12__value_typeIPN4llvm10BasicBlockEPNS4_10RegionNodeEEEPvEEEEE11__constructIS9_JNS_4pairIS6_S8_EEEEEvNS_17integral_constantIbLb1EEERSC_PT_DpOT0_", scope: !0, file: !1, line: 2, type: !5, isLocal: false, isDefinition: true, variable: i32* @_ZNSt3__116allocator_traitsINS_9allocatorINS_11__tree_nodeINS_12__value_typeIPN4llvm10BasicBlockEPNS4_10RegionNodeEEEPvEEEEE11__constructIS9_JNS_4pairIS6_S8_EEEEEvNS_17integral_constantIbLb1EEERSC_PT_DpOT0_)
-!7 = !MDGlobalVariable(name: "_ZN5clang23DataRecursiveASTVisitorIN12_GLOBAL__N_124UnusedBackingIvarCheckerEE26TraverseCUDAKernelCallExprEPNS_18CUDAKernelCallExprE", scope: !0, file: !1, line: 3, type: !5, isLocal: false, isDefinition: true, variable: i32* @_ZN5clang23DataRecursiveASTVisitorIN12_GLOBAL__N_124UnusedBackingIvarCheckerEE26TraverseCUDAKernelCallExprEPNS_18CUDAKernelCallExprE)
-!8 = !MDGlobalVariable(name: "_ZN4llvm16DenseMapIteratorIPNS_10MDLocationENS_6detail13DenseSetEmptyENS_10MDNodeInfoIS1_EENS3_12DenseSetPairIS2_EELb0EE23AdvancePastEmptyBucketsEv", scope: !0, file: !1, line: 4, type: !5, isLocal: false, isDefinition: true, variable: i32* @_ZN4llvm16DenseMapIteratorIPNS_10MDLocationENS_6detail13DenseSetEmptyENS_10MDNodeInfoIS1_EENS3_12DenseSetPairIS2_EELb0EE23AdvancePastEmptyBucketsEv)
-!9 = !MDGlobalVariable(name: "_ZNK4llvm12LivePhysRegs5printERNS_11raw_ostreamE", scope: !0, file: !1, line: 5, type: !5, isLocal: false, isDefinition: true, variable: i32* @_ZNK4llvm12LivePhysRegs5printERNS_11raw_ostreamE)
-!10 = !MDGlobalVariable(name: "_ZN4llvm15ScalarEvolution14getSignedRangeEPKNS_4SCEVE", scope: !0, file: !1, line: 6, type: !5, isLocal: false, isDefinition: true, variable: i32* @_ZN4llvm15ScalarEvolution14getSignedRangeEPKNS_4SCEVE)
-!11 = !MDGlobalVariable(name: "k1", scope: !0, file: !1, line: 7, type: !5, isLocal: false, isDefinition: true, variable: i32* @k1)
-!12 = !MDGlobalVariable(name: "is", scope: !0, file: !1, line: 8, type: !5, isLocal: false, isDefinition: true, variable: i32* @is)
-!13 = !MDGlobalVariable(name: "setStmt", scope: !0, file: !1, line: 9, type: !5, isLocal: false, isDefinition: true, variable: i32* @setStmt)
-!14 = !MDGlobalVariable(name: "_ZN4llvm5TwineC1Ei", scope: !0, file: !1, line: 10, type: !5, isLocal: false, isDefinition: true, variable: i32* @_ZN4llvm5TwineC1Ei)
-!15 = !MDGlobalVariable(name: "_ZNK5clang12OverrideAttr5cloneERNS_10ASTContextE", scope: !0, file: !1, line: 11, type: !5, isLocal: false, isDefinition: true, variable: i32* @_ZNK5clang12OverrideAttr5cloneERNS_10ASTContextE)
-!16 = !MDGlobalVariable(name: "_ZN4llvm22MachineModuleInfoMachOD2Ev", scope: !0, file: !1, line: 12, type: !5, isLocal: false, isDefinition: true, variable: i32* @_ZN4llvm22MachineModuleInfoMachOD2Ev)
+!4 = !DIGlobalVariable(name: "ForceTopDown", scope: !0, file: !1, line: 1, type: !5, isLocal: false, isDefinition: true, variable: i32* @ForceTopDown)
+!5 = !DIBasicType(name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
+!6 = !DIGlobalVariable(name: "_ZNSt3__116allocator_traitsINS_9allocatorINS_11__tree_nodeINS_12__value_typeIPN4llvm10BasicBlockEPNS4_10RegionNodeEEEPvEEEEE11__constructIS9_JNS_4pairIS6_S8_EEEEEvNS_17integral_constantIbLb1EEERSC_PT_DpOT0_", scope: !0, file: !1, line: 2, type: !5, isLocal: false, isDefinition: true, variable: i32* @_ZNSt3__116allocator_traitsINS_9allocatorINS_11__tree_nodeINS_12__value_typeIPN4llvm10BasicBlockEPNS4_10RegionNodeEEEPvEEEEE11__constructIS9_JNS_4pairIS6_S8_EEEEEvNS_17integral_constantIbLb1EEERSC_PT_DpOT0_)
+!7 = !DIGlobalVariable(name: "_ZN5clang23DataRecursiveASTVisitorIN12_GLOBAL__N_124UnusedBackingIvarCheckerEE26TraverseCUDAKernelCallExprEPNS_18CUDAKernelCallExprE", scope: !0, file: !1, line: 3, type: !5, isLocal: false, isDefinition: true, variable: i32* @_ZN5clang23DataRecursiveASTVisitorIN12_GLOBAL__N_124UnusedBackingIvarCheckerEE26TraverseCUDAKernelCallExprEPNS_18CUDAKernelCallExprE)
+!8 = !DIGlobalVariable(name: "_ZN4llvm16DenseMapIteratorIPNS_10MDLocationENS_6detail13DenseSetEmptyENS_10MDNodeInfoIS1_EENS3_12DenseSetPairIS2_EELb0EE23AdvancePastEmptyBucketsEv", scope: !0, file: !1, line: 4, type: !5, isLocal: false, isDefinition: true, variable: i32* @_ZN4llvm16DenseMapIteratorIPNS_10MDLocationENS_6detail13DenseSetEmptyENS_10MDNodeInfoIS1_EENS3_12DenseSetPairIS2_EELb0EE23AdvancePastEmptyBucketsEv)
+!9 = !DIGlobalVariable(name: "_ZNK4llvm12LivePhysRegs5printERNS_11raw_ostreamE", scope: !0, file: !1, line: 5, type: !5, isLocal: false, isDefinition: true, variable: i32* @_ZNK4llvm12LivePhysRegs5printERNS_11raw_ostreamE)
+!10 = !DIGlobalVariable(name: "_ZN4llvm15ScalarEvolution14getSignedRangeEPKNS_4SCEVE", scope: !0, file: !1, line: 6, type: !5, isLocal: false, isDefinition: true, variable: i32* @_ZN4llvm15ScalarEvolution14getSignedRangeEPKNS_4SCEVE)
+!11 = !DIGlobalVariable(name: "k1", scope: !0, file: !1, line: 7, type: !5, isLocal: false, isDefinition: true, variable: i32* @k1)
+!12 = !DIGlobalVariable(name: "is", scope: !0, file: !1, line: 8, type: !5, isLocal: false, isDefinition: true, variable: i32* @is)
+!13 = !DIGlobalVariable(name: "setStmt", scope: !0, file: !1, line: 9, type: !5, isLocal: false, isDefinition: true, variable: i32* @setStmt)
+!14 = !DIGlobalVariable(name: "_ZN4llvm5TwineC1Ei", scope: !0, file: !1, line: 10, type: !5, isLocal: false, isDefinition: true, variable: i32* @_ZN4llvm5TwineC1Ei)
+!15 = !DIGlobalVariable(name: "_ZNK5clang12OverrideAttr5cloneERNS_10ASTContextE", scope: !0, file: !1, line: 11, type: !5, isLocal: false, isDefinition: true, variable: i32* @_ZNK5clang12OverrideAttr5cloneERNS_10ASTContextE)
+!16 = !DIGlobalVariable(name: "_ZN4llvm22MachineModuleInfoMachOD2Ev", scope: !0, file: !1, line: 12, type: !5, isLocal: false, isDefinition: true, variable: i32* @_ZN4llvm22MachineModuleInfoMachOD2Ev)
 !17 = !{i32 2, !"Dwarf Version", i32 2}
 !18 = !{i32 2, !"Debug Info Version", i32 3}
 !19 = !{i32 1, !"PIC Level", i32 2}
diff --git a/test/DebugInfo/array.ll b/test/DebugInfo/array.ll
index 67011773d21..f6c79f77c6f 100644
--- a/test/DebugInfo/array.ll
+++ b/test/DebugInfo/array.ll
@@ -6,7 +6,7 @@ entry:
   %retval = alloca i32, align 4
   %a = alloca [0 x i32], align 4
   store i32 0, i32* %retval
-  call void @llvm.dbg.declare(metadata [0 x i32]* %a, metadata !6, metadata !MDExpression()), !dbg !11
+  call void @llvm.dbg.declare(metadata [0 x i32]* %a, metadata !6, metadata !DIExpression()), !dbg !11
   ret i32 0, !dbg !12
 }
 
@@ -15,15 +15,15 @@ declare void @llvm.dbg.declare(metadata, metadata, metadata) nounwind readnone
 !llvm.dbg.cu = !{!2}
 !llvm.module.flags = !{!16}
 
-!0 = !MDSubprogram(name: "main", line: 3, isLocal: false, isDefinition: true, virtualIndex: 6, isOptimized: false, scopeLine: 3, file: !14, scope: !1, type: !3, function: i32 ()* @main)
-!1 = !MDFile(filename: "array.c", directory: "/private/tmp")
-!2 = !MDCompileUnit(language: DW_LANG_C99, producer: "clang version 3.0 (trunk 129138)", isOptimized: false, emissionKind: 0, file: !14, enums: !15, retainedTypes: !15, subprograms: !13, imports:  null)
-!3 = !MDSubroutineType(types: !4)
+!0 = !DISubprogram(name: "main", line: 3, isLocal: false, isDefinition: true, virtualIndex: 6, isOptimized: false, scopeLine: 3, file: !14, scope: !1, type: !3, function: i32 ()* @main)
+!1 = !DIFile(filename: "array.c", directory: "/private/tmp")
+!2 = !DICompileUnit(language: DW_LANG_C99, producer: "clang version 3.0 (trunk 129138)", isOptimized: false, emissionKind: 0, file: !14, enums: !15, retainedTypes: !15, subprograms: !13, imports:  null)
+!3 = !DISubroutineType(types: !4)
 !4 = !{!5}
-!5 = !MDBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
-!6 = !MDLocalVariable(tag: DW_TAG_auto_variable, name: "a", line: 4, scope: !7, file: !1, type: !8)
-!7 = distinct !MDLexicalBlock(line: 3, column: 12, file: !14, scope: !0)
-!8 = !MDCompositeType(tag: DW_TAG_array_type, align: 32, file: !14, scope: !2, baseType: !5, elements: !9)
+!5 = !DIBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
+!6 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "a", line: 4, scope: !7, file: !1, type: !8)
+!7 = distinct !DILexicalBlock(line: 3, column: 12, file: !14, scope: !0)
+!8 = !DICompositeType(tag: DW_TAG_array_type, align: 32, file: !14, scope: !2, baseType: !5, elements: !9)
 !9 = !{!10}
 ;CHECK: section_info:
 ;CHECK: DW_TAG_subrange_type
@@ -31,10 +31,10 @@ declare void @llvm.dbg.declare(metadata, metadata, metadata) nounwind readnone
 ;CHECK-NOT: DW_AT_lower_bound
 ;CHECK-NOT: DW_AT_upper_bound
 ;CHECK-NEXT: End Of Children Mark
-!10 = !MDSubrange(count: -1)
-!11 = !MDLocation(line: 4, column: 7, scope: !7)
-!12 = !MDLocation(line: 5, column: 3, scope: !7)
+!10 = !DISubrange(count: -1)
+!11 = !DILocation(line: 4, column: 7, scope: !7)
+!12 = !DILocation(line: 5, column: 3, scope: !7)
 !13 = !{!0}
-!14 = !MDFile(filename: "array.c", directory: "/private/tmp")
+!14 = !DIFile(filename: "array.c", directory: "/private/tmp")
 !15 = !{}
 !16 = !{i32 1, !"Debug Info Version", i32 3}
diff --git a/test/DebugInfo/block-asan.ll b/test/DebugInfo/block-asan.ll
index 618fa62b178..393816b3485 100644
--- a/test/DebugInfo/block-asan.ll
+++ b/test/DebugInfo/block-asan.ll
@@ -13,7 +13,7 @@
 
 ; Check that the location of the ASAN instrumented __block variable is
 ; correct.
-; CHECK: !MDExpression(DW_OP_deref, DW_OP_plus, 8, DW_OP_deref, DW_OP_plus, 24)
+; CHECK: !DIExpression(DW_OP_deref, DW_OP_plus, 8, DW_OP_deref, DW_OP_plus, 24)
 
 target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
 
@@ -58,30 +58,30 @@ attributes #3 = { nounwind }
 !llvm.module.flags = !{!8, !9, !10}
 !llvm.ident = !{!11}
 
-!0 = !MDCompileUnit(language: DW_LANG_C99, producer: "clang version 3.6.0 (trunk 223120) (llvm/trunk 223119)", isOptimized: false, emissionKind: 1, file: !1, enums: !2, retainedTypes: !2, subprograms: !3, globals: !2, imports: !2)
-!1 = !MDFile(filename: "block.c", directory: "/tmp")
+!0 = !DICompileUnit(language: DW_LANG_C99, producer: "clang version 3.6.0 (trunk 223120) (llvm/trunk 223119)", isOptimized: false, emissionKind: 1, file: !1, enums: !2, retainedTypes: !2, subprograms: !3, globals: !2, imports: !2)
+!1 = !DIFile(filename: "block.c", directory: "/tmp")
 !2 = !{}
 !3 = !{!4}
-!4 = !MDSubprogram(name: "foo", line: 3, isLocal: false, isDefinition: true, isOptimized: false, scopeLine: 3, file: !1, scope: !5, type: !6, function: void ()* @foo, variables: !2)
-!5 = !MDFile(filename: "block.c", directory: "/tmp")
-!6 = !MDSubroutineType(types: !7)
+!4 = !DISubprogram(name: "foo", line: 3, isLocal: false, isDefinition: true, isOptimized: false, scopeLine: 3, file: !1, scope: !5, type: !6, function: void ()* @foo, variables: !2)
+!5 = !DIFile(filename: "block.c", directory: "/tmp")
+!6 = !DISubroutineType(types: !7)
 !7 = !{null}
 !8 = !{i32 2, !"Dwarf Version", i32 2}
 !9 = !{i32 2, !"Debug Info Version", i32 3}
 !10 = !{i32 1, !"PIC Level", i32 2}
 !11 = !{!"clang version 3.6.0 (trunk 223120) (llvm/trunk 223119)"}
-!12 = !MDLocalVariable(tag: DW_TAG_auto_variable, name: "x", line: 4, scope: !4, file: !5, type: !13)
-!13 = !MDCompositeType(tag: DW_TAG_structure_type, size: 224, flags: DIFlagBlockByrefStruct, file: !1, scope: !5, elements: !14)
+!12 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "x", line: 4, scope: !4, file: !5, type: !13)
+!13 = !DICompositeType(tag: DW_TAG_structure_type, size: 224, flags: DIFlagBlockByrefStruct, file: !1, scope: !5, elements: !14)
 !14 = !{!15, !17, !18, !20, !21}
-!15 = !MDDerivedType(tag: DW_TAG_member, name: "__isa", size: 64, align: 64, file: !1, scope: !5, baseType: !16)
-!16 = !MDDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, baseType: null)
-!17 = !MDDerivedType(tag: DW_TAG_member, name: "__forwarding", size: 64, align: 64, offset: 64, file: !1, scope: !5, baseType: !16)
-!18 = !MDDerivedType(tag: DW_TAG_member, name: "__flags", size: 32, align: 32, offset: 128, file: !1, scope: !5, baseType: !19)
-!19 = !MDBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
-!20 = !MDDerivedType(tag: DW_TAG_member, name: "__size", size: 32, align: 32, offset: 160, file: !1, scope: !5, baseType: !19)
-!21 = !MDDerivedType(tag: DW_TAG_member, name: "x", size: 32, align: 32, offset: 192, file: !1, scope: !5, baseType: !19)
-!22 = !MDExpression(DW_OP_plus, 8, DW_OP_deref, DW_OP_plus, 24)
-!23 = !MDLocation(line: 4, column: 15, scope: !4)
-!24 = !MDLocation(line: 4, column: 3, scope: !4)
-!25 = !MDLocation(line: 5, column: 3, scope: !4)
-!26 = !MDLocation(line: 6, column: 1, scope: !4)
+!15 = !DIDerivedType(tag: DW_TAG_member, name: "__isa", size: 64, align: 64, file: !1, scope: !5, baseType: !16)
+!16 = !DIDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, baseType: null)
+!17 = !DIDerivedType(tag: DW_TAG_member, name: "__forwarding", size: 64, align: 64, offset: 64, file: !1, scope: !5, baseType: !16)
+!18 = !DIDerivedType(tag: DW_TAG_member, name: "__flags", size: 32, align: 32, offset: 128, file: !1, scope: !5, baseType: !19)
+!19 = !DIBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
+!20 = !DIDerivedType(tag: DW_TAG_member, name: "__size", size: 32, align: 32, offset: 160, file: !1, scope: !5, baseType: !19)
+!21 = !DIDerivedType(tag: DW_TAG_member, name: "x", size: 32, align: 32, offset: 192, file: !1, scope: !5, baseType: !19)
+!22 = !DIExpression(DW_OP_plus, 8, DW_OP_deref, DW_OP_plus, 24)
+!23 = !DILocation(line: 4, column: 15, scope: !4)
+!24 = !DILocation(line: 4, column: 3, scope: !4)
+!25 = !DILocation(line: 5, column: 3, scope: !4)
+!26 = !DILocation(line: 6, column: 1, scope: !4)
diff --git a/test/DebugInfo/bug_null_debuginfo.ll b/test/DebugInfo/bug_null_debuginfo.ll
index d64c6a6abd8..5aaa3834a5f 100644
--- a/test/DebugInfo/bug_null_debuginfo.ll
+++ b/test/DebugInfo/bug_null_debuginfo.ll
@@ -3,6 +3,6 @@
 !llvm.dbg.cu = !{!0}
 !llvm.module.flags = !{!2}
 
-!0 = !MDCompileUnit(language: DW_LANG_C99, isOptimized: false, emissionKind: 0, file: !1, globals:  null)
-!1 = !MDFile(filename: "t", directory: "")
+!0 = !DICompileUnit(language: DW_LANG_C99, isOptimized: false, emissionKind: 0, file: !1, globals:  null)
+!1 = !DIFile(filename: "t", directory: "")
 !2 = !{i32 1, !"Debug Info Version", i32 3}
diff --git a/test/DebugInfo/constant-pointers.ll b/test/DebugInfo/constant-pointers.ll
index 255299818d7..8e0a242f1ba 100644
--- a/test/DebugInfo/constant-pointers.ll
+++ b/test/DebugInfo/constant-pointers.ll
@@ -30,22 +30,22 @@ attributes #0 = { nounwind uwtable "less-precise-fpmad"="false" "no-frame-pointe
 !llvm.module.flags = !{!15, !16}
 !llvm.ident = !{!17}
 
-!0 = !MDCompileUnit(language: DW_LANG_C_plus_plus, producer: "clang version 3.5.0 ", isOptimized: false, emissionKind: 1, file: !1, enums: !2, retainedTypes: !2, subprograms: !3, globals: !2, imports: !2)
-!1 = !MDFile(filename: "constant-pointers.cpp", directory: "/tmp/dbginfo")
+!0 = !DICompileUnit(language: DW_LANG_C_plus_plus, producer: "clang version 3.5.0 ", isOptimized: false, emissionKind: 1, file: !1, enums: !2, retainedTypes: !2, subprograms: !3, globals: !2, imports: !2)
+!1 = !DIFile(filename: "constant-pointers.cpp", directory: "/tmp/dbginfo")
 !2 = !{}
 !3 = !{!4}
-!4 = !MDSubprogram(name: "func<nullptr, nullptr, 42>", linkageName: "_Z4funcILPv0ELPFvvE0ELi42EEvv", line: 2, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 2, file: !1, scope: !5, type: !6, function: void ()* @_Z4funcILPv0ELPFvvE0ELi42EEvv, templateParams: !8, variables: !2)
-!5 = !MDFile(filename: "constant-pointers.cpp", directory: "/tmp/dbginfo")
-!6 = !MDSubroutineType(types: !7)
+!4 = !DISubprogram(name: "func<nullptr, nullptr, 42>", linkageName: "_Z4funcILPv0ELPFvvE0ELi42EEvv", line: 2, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 2, file: !1, scope: !5, type: !6, function: void ()* @_Z4funcILPv0ELPFvvE0ELi42EEvv, templateParams: !8, variables: !2)
+!5 = !DIFile(filename: "constant-pointers.cpp", directory: "/tmp/dbginfo")
+!6 = !DISubroutineType(types: !7)
 !7 = !{null}
 !8 = !{!9, !11, !13}
-!9 = !MDTemplateValueParameter(tag: DW_TAG_template_value_parameter, name: "V", type: !10, value: i8 0)
-!10 = !MDDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, baseType: null)
-!11 = !MDTemplateValueParameter(tag: DW_TAG_template_value_parameter, name: "F", type: !12, value: i8 0)
-!12 = !MDDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, baseType: !6)
-!13 = !MDTemplateValueParameter(tag: DW_TAG_template_value_parameter, name: "i", type: !14, value: i32 42)
-!14 = !MDBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
+!9 = !DITemplateValueParameter(tag: DW_TAG_template_value_parameter, name: "V", type: !10, value: i8 0)
+!10 = !DIDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, baseType: null)
+!11 = !DITemplateValueParameter(tag: DW_TAG_template_value_parameter, name: "F", type: !12, value: i8 0)
+!12 = !DIDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, baseType: !6)
+!13 = !DITemplateValueParameter(tag: DW_TAG_template_value_parameter, name: "i", type: !14, value: i32 42)
+!14 = !DIBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
 !15 = !{i32 2, !"Dwarf Version", i32 4}
 !16 = !{i32 2, !"Debug Info Version", i32 3}
 !17 = !{!"clang version 3.5.0 "}
-!18 = !MDLocation(line: 3, scope: !4)
+!18 = !DILocation(line: 3, scope: !4)
diff --git a/test/DebugInfo/constant-sdnodes-have-dbg-location.ll b/test/DebugInfo/constant-sdnodes-have-dbg-location.ll
new file mode 100644
index 00000000000..1f502a2a262
--- /dev/null
+++ b/test/DebugInfo/constant-sdnodes-have-dbg-location.ll
@@ -0,0 +1,26 @@
+; RUN: llc -debug < %s 2>&1 | FileCheck %s
+; REQUIRES: asserts
+
+; CHECK: 0x{{[0-9,a-f]+}}: i32 = Constant<-1>test.c:4:5
+
+define i32 @main() {
+entry:
+  %retval = alloca i32, align 4
+  store i32 0, i32* %retval
+  ret i32 -1, !dbg !10
+}
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!8, !9}
+
+!0 = !DICompileUnit(language: DW_LANG_C99, file: !1, producer: "", isOptimized: false, subprograms: !3)
+!1 = !DIFile(filename: "test.c", directory: "/home/user/clang-llvm/build")
+!2 = !{}
+!3 = !{!4}
+!4 = !DISubprogram(name: "main", scope: !1, file: !1, line: 2, type: !5, isLocal: false, isDefinition: true, scopeLine: 3, flags: DIFlagPrototyped, isOptimized: false, function: i32 ()* @main, variables: !2)
+!5 = !DISubroutineType(types: !6)
+!6 = !{!7}
+!7 = !DIBasicType(name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
+!8 = !{i32 2, !"Dwarf Version", i32 4}
+!9 = !{i32 2, !"Debug Info Version", i32 3}
+!10 = !DILocation(line: 4, column: 5, scope: !4)
diff --git a/test/DebugInfo/constantfp-sdnodes-have-dbg-location.ll b/test/DebugInfo/constantfp-sdnodes-have-dbg-location.ll
new file mode 100644
index 00000000000..2f7ecd8b061
--- /dev/null
+++ b/test/DebugInfo/constantfp-sdnodes-have-dbg-location.ll
@@ -0,0 +1,24 @@
+; RUN: llc -debug < %s 2>&1 | FileCheck %s
+; REQUIRES: asserts
+
+; CHECK: 0x{{[0-9,a-f]+}}: f64 = ConstantFP<1.500000e+00>test.c:3:5
+
+define double @f() {
+entry:
+  ret double 1.500000e+00, !dbg !10
+}
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!8, !9}
+
+!0 = !DICompileUnit(language: DW_LANG_C99, file: !1, producer: "", isOptimized: false, subprograms: !3)
+!1 = !DIFile(filename: "test.c", directory: "/home/user/clang-llvm/build")
+!2 = !{}
+!3 = !{!4}
+!4 = !DISubprogram(name: "f", scope: !1, file: !1, line: 1, type: !5, isLocal: false, isDefinition: true, scopeLine: 2, flags: DIFlagPrototyped, isOptimized: false, function: double ()* @f, variables: !2)
+!5 = !DISubroutineType(types: !6)
+!6 = !{!7}
+!7 = !DIBasicType(name: "double", size: 64, align: 64, encoding: DW_ATE_float)
+!8 = !{i32 2, !"Dwarf Version", i32 4}
+!9 = !{i32 2, !"Debug Info Version", i32 3}
+!10 = !DILocation(line: 3, column: 5, scope: !4)
diff --git a/test/DebugInfo/cross-cu-inlining.ll b/test/DebugInfo/cross-cu-inlining.ll
index 31f49e91bc0..436bdda6473 100644
--- a/test/DebugInfo/cross-cu-inlining.ll
+++ b/test/DebugInfo/cross-cu-inlining.ll
@@ -75,7 +75,7 @@ entry:
   %1 = bitcast i32* %x.addr.i to i8*
   call void @llvm.lifetime.start(i64 4, i8* %1)
   store i32 %0, i32* %x.addr.i, align 4
-  call void @llvm.dbg.declare(metadata i32* %x.addr.i, metadata !120, metadata !MDExpression()), !dbg !21
+  call void @llvm.dbg.declare(metadata i32* %x.addr.i, metadata !120, metadata !DIExpression()), !dbg !21
   %2 = load i32, i32* %x.addr.i, align 4, !dbg !22
   %mul.i = mul nsw i32 %2, 2, !dbg !22
   %3 = bitcast i32* %x.addr.i to i8*, !dbg !22
@@ -88,7 +88,7 @@ define i32 @_Z4funci(i32 %x) #1 {
 entry:
   %x.addr = alloca i32, align 4
   store i32 %x, i32* %x.addr, align 4
-  call void @llvm.dbg.declare(metadata i32* %x.addr, metadata !20, metadata !MDExpression()), !dbg !23
+  call void @llvm.dbg.declare(metadata i32* %x.addr, metadata !20, metadata !DIExpression()), !dbg !23
   %0 = load i32, i32* %x.addr, align 4, !dbg !24
   %mul = mul nsw i32 %0, 2, !dbg !24
   ret i32 %mul, !dbg !24
@@ -112,32 +112,32 @@ attributes #3 = { nounwind }
 !llvm.module.flags = !{!16, !17}
 !llvm.ident = !{!18, !18}
 
-!0 = !MDCompileUnit(language: DW_LANG_C_plus_plus, producer: "clang version 3.5.0 ", isOptimized: false, emissionKind: 1, file: !1, enums: !2, retainedTypes: !2, subprograms: !3, globals: !2, imports: !2)
-!1 = !MDFile(filename: "a.cpp", directory: "/tmp/dbginfo")
+!0 = !DICompileUnit(language: DW_LANG_C_plus_plus, producer: "clang version 3.5.0 ", isOptimized: false, emissionKind: 1, file: !1, enums: !2, retainedTypes: !2, subprograms: !3, globals: !2, imports: !2)
+!1 = !DIFile(filename: "a.cpp", directory: "/tmp/dbginfo")
 !2 = !{}
 !3 = !{!4}
-!4 = !MDSubprogram(name: "main", line: 3, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 3, file: !1, scope: !5, type: !6, function: i32 ()* @main, variables: !2)
-!5 = !MDFile(filename: "a.cpp", directory: "/tmp/dbginfo")
-!6 = !MDSubroutineType(types: !7)
+!4 = !DISubprogram(name: "main", line: 3, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 3, file: !1, scope: !5, type: !6, function: i32 ()* @main, variables: !2)
+!5 = !DIFile(filename: "a.cpp", directory: "/tmp/dbginfo")
+!6 = !DISubroutineType(types: !7)
 !7 = !{!8}
-!8 = !MDBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
-!9 = !MDCompileUnit(language: DW_LANG_C_plus_plus, producer: "clang version 3.5.0 ", isOptimized: false, emissionKind: 1, file: !10, enums: !2, retainedTypes: !2, subprograms: !11, globals: !2, imports: !2)
-!10 = !MDFile(filename: "b.cpp", directory: "/tmp/dbginfo")
+!8 = !DIBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
+!9 = !DICompileUnit(language: DW_LANG_C_plus_plus, producer: "clang version 3.5.0 ", isOptimized: false, emissionKind: 1, file: !10, enums: !2, retainedTypes: !2, subprograms: !11, globals: !2, imports: !2)
+!10 = !DIFile(filename: "b.cpp", directory: "/tmp/dbginfo")
 !11 = !{!12}
-!12 = !MDSubprogram(name: "func", linkageName: "_Z4funci", line: 1, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 1, file: !10, scope: !13, type: !14, function: i32 (i32)* @_Z4funci, variables: !2)
-!13 = !MDFile(filename: "b.cpp", directory: "/tmp/dbginfo")
-!14 = !MDSubroutineType(types: !15)
+!12 = !DISubprogram(name: "func", linkageName: "_Z4funci", line: 1, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 1, file: !10, scope: !13, type: !14, function: i32 (i32)* @_Z4funci, variables: !2)
+!13 = !DIFile(filename: "b.cpp", directory: "/tmp/dbginfo")
+!14 = !DISubroutineType(types: !15)
 !15 = !{!8, !8}
 !16 = !{i32 2, !"Dwarf Version", i32 4}
 !17 = !{i32 2, !"Debug Info Version", i32 3}
 !18 = !{!"clang version 3.5.0 "}
-!19 = !MDLocation(line: 4, scope: !4)
-!20 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "x", line: 1, arg: 1, scope: !12, file: !13, type: !8)
+!19 = !DILocation(line: 4, scope: !4)
+!20 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "x", line: 1, arg: 1, scope: !12, file: !13, type: !8)
 
-!120 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "x", line: 1, arg: 1, scope: !12, file: !13, type: !8)
+!120 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "x", line: 1, arg: 1, scope: !12, file: !13, type: !8)
 
-!21 = !MDLocation(line: 1, scope: !12, inlinedAt: !19)
-!22 = !MDLocation(line: 2, scope: !12, inlinedAt: !19)
-!23 = !MDLocation(line: 1, scope: !12)
-!24 = !MDLocation(line: 2, scope: !12)
+!21 = !DILocation(line: 1, scope: !12, inlinedAt: !19)
+!22 = !DILocation(line: 2, scope: !12, inlinedAt: !19)
+!23 = !DILocation(line: 1, scope: !12)
+!24 = !DILocation(line: 2, scope: !12)
 
diff --git a/test/DebugInfo/cross-cu-linkonce-distinct.ll b/test/DebugInfo/cross-cu-linkonce-distinct.ll
index 544f41bad63..b8ce57e7035 100644
--- a/test/DebugInfo/cross-cu-linkonce-distinct.ll
+++ b/test/DebugInfo/cross-cu-linkonce-distinct.ll
@@ -52,7 +52,7 @@
 define linkonce_odr i32 @_Z4funci(i32 %i) #0 {
   %1 = alloca i32, align 4
   store i32 %i, i32* %1, align 4
-  call void @llvm.dbg.declare(metadata i32* %1, metadata !22, metadata !MDExpression()), !dbg !23
+  call void @llvm.dbg.declare(metadata i32* %1, metadata !22, metadata !DIExpression()), !dbg !23
   %2 = load i32, i32* %1, align 4, !dbg !24
   %3 = mul nsw i32 %2, 2, !dbg !24
   ret i32 %3, !dbg !24
@@ -68,28 +68,28 @@ attributes #1 = { nounwind readnone }
 !llvm.module.flags = !{!19, !20}
 !llvm.ident = !{!21, !21}
 
-!0 = !MDCompileUnit(language: DW_LANG_C_plus_plus, producer: "clang version 3.5.0 ", isOptimized: false, emissionKind: 1, file: !1, enums: !2, retainedTypes: !2, subprograms: !3, globals: !9, imports: !2)
-!1 = !MDFile(filename: "a.cpp", directory: "/tmp/dbginfo")
+!0 = !DICompileUnit(language: DW_LANG_C_plus_plus, producer: "clang version 3.5.0 ", isOptimized: false, emissionKind: 1, file: !1, enums: !2, retainedTypes: !2, subprograms: !3, globals: !9, imports: !2)
+!1 = !DIFile(filename: "a.cpp", directory: "/tmp/dbginfo")
 !2 = !{}
 !3 = !{!4}
-!4 = !MDSubprogram(name: "func", linkageName: "_Z4funci", line: 1, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 1, file: !1, scope: !5, type: !6, function: i32 (i32)* @_Z4funci, variables: !2)
-!5 = !MDFile(filename: "a.cpp", directory: "/tmp/dbginfo")
-!6 = !MDSubroutineType(types: !7)
+!4 = !DISubprogram(name: "func", linkageName: "_Z4funci", line: 1, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 1, file: !1, scope: !5, type: !6, function: i32 (i32)* @_Z4funci, variables: !2)
+!5 = !DIFile(filename: "a.cpp", directory: "/tmp/dbginfo")
+!6 = !DISubroutineType(types: !7)
 !7 = !{!8, !8}
-!8 = !MDBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
+!8 = !DIBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
 !9 = !{!10}
-!10 = !MDGlobalVariable(name: "x", line: 4, isLocal: false, isDefinition: true, scope: null, file: !5, type: !11, variable: i32 (i32)** @x)
-!11 = !MDDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, baseType: !6)
-!12 = !MDCompileUnit(language: DW_LANG_C_plus_plus, producer: "clang version 3.5.0 ", isOptimized: false, emissionKind: 1, file: !13, enums: !2, retainedTypes: !2, subprograms: !14, globals: !17, imports: !2)
-!13 = !MDFile(filename: "b.cpp", directory: "/tmp/dbginfo")
+!10 = !DIGlobalVariable(name: "x", line: 4, isLocal: false, isDefinition: true, scope: null, file: !5, type: !11, variable: i32 (i32)** @x)
+!11 = !DIDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, baseType: !6)
+!12 = !DICompileUnit(language: DW_LANG_C_plus_plus, producer: "clang version 3.5.0 ", isOptimized: false, emissionKind: 1, file: !13, enums: !2, retainedTypes: !2, subprograms: !14, globals: !17, imports: !2)
+!13 = !DIFile(filename: "b.cpp", directory: "/tmp/dbginfo")
 !14 = !{!15}
-!15 = !MDSubprogram(name: "func", linkageName: "_Z4funci", line: 1, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 1, file: !13, scope: !16, type: !6, function: i32 (i32)* @_Z4funci, variables: !2)
-!16 = !MDFile(filename: "b.cpp", directory: "/tmp/dbginfo")
+!15 = !DISubprogram(name: "func", linkageName: "_Z4funci", line: 1, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 1, file: !13, scope: !16, type: !6, function: i32 (i32)* @_Z4funci, variables: !2)
+!16 = !DIFile(filename: "b.cpp", directory: "/tmp/dbginfo")
 !17 = !{!18}
-!18 = !MDGlobalVariable(name: "y", line: 4, isLocal: false, isDefinition: true, scope: null, file: !16, type: !11, variable: i32 (i32)** @y)
+!18 = !DIGlobalVariable(name: "y", line: 4, isLocal: false, isDefinition: true, scope: null, file: !16, type: !11, variable: i32 (i32)** @y)
 !19 = !{i32 2, !"Dwarf Version", i32 4}
 !20 = !{i32 1, !"Debug Info Version", i32 3}
 !21 = !{!"clang version 3.5.0 "}
-!22 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "i", line: 1, arg: 1, scope: !4, file: !5, type: !8)
-!23 = !MDLocation(line: 1, scope: !4)
-!24 = !MDLocation(line: 2, scope: !4)
+!22 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "i", line: 1, arg: 1, scope: !4, file: !5, type: !8)
+!23 = !DILocation(line: 1, scope: !4)
+!24 = !DILocation(line: 2, scope: !4)
diff --git a/test/DebugInfo/cross-cu-linkonce.ll b/test/DebugInfo/cross-cu-linkonce.ll
index d212ee2f417..3638825fc9c 100644
--- a/test/DebugInfo/cross-cu-linkonce.ll
+++ b/test/DebugInfo/cross-cu-linkonce.ll
@@ -32,7 +32,7 @@
 define linkonce_odr i32 @_Z4funci(i32 %i) #0 {
   %1 = alloca i32, align 4
   store i32 %i, i32* %1, align 4
-  call void @llvm.dbg.declare(metadata i32* %1, metadata !20, metadata !MDExpression()), !dbg !21
+  call void @llvm.dbg.declare(metadata i32* %1, metadata !20, metadata !DIExpression()), !dbg !21
   %2 = load i32, i32* %1, align 4, !dbg !22
   %3 = mul nsw i32 %2, 2, !dbg !22
   ret i32 %3, !dbg !22
@@ -48,26 +48,26 @@ attributes #1 = { nounwind readnone }
 !llvm.module.flags = !{!17, !18}
 !llvm.ident = !{!19, !19}
 
-!0 = !MDCompileUnit(language: DW_LANG_C_plus_plus, producer: "clang version 3.5.0 ", isOptimized: false, emissionKind: 1, file: !1, enums: !2, retainedTypes: !2, subprograms: !3, globals: !10, imports: !2)
-!1 = !MDFile(filename: "a.cpp", directory: "/tmp/dbginfo")
+!0 = !DICompileUnit(language: DW_LANG_C_plus_plus, producer: "clang version 3.5.0 ", isOptimized: false, emissionKind: 1, file: !1, enums: !2, retainedTypes: !2, subprograms: !3, globals: !10, imports: !2)
+!1 = !DIFile(filename: "a.cpp", directory: "/tmp/dbginfo")
 !2 = !{}
 !3 = !{!4}
-!4 = !MDSubprogram(name: "func", linkageName: "_Z4funci", line: 1, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 1, file: !5, scope: !6, type: !7, function: i32 (i32)* @_Z4funci, variables: !2)
-!5 = !MDFile(filename: "func.h", directory: "/tmp/dbginfo")
-!6 = !MDFile(filename: "func.h", directory: "/tmp/dbginfo")
-!7 = !MDSubroutineType(types: !8)
+!4 = !DISubprogram(name: "func", linkageName: "_Z4funci", line: 1, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 1, file: !5, scope: !6, type: !7, function: i32 (i32)* @_Z4funci, variables: !2)
+!5 = !DIFile(filename: "func.h", directory: "/tmp/dbginfo")
+!6 = !DIFile(filename: "func.h", directory: "/tmp/dbginfo")
+!7 = !DISubroutineType(types: !8)
 !8 = !{!9, !9}
-!9 = !MDBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
+!9 = !DIBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
 !10 = !{!11}
-!11 = !MDGlobalVariable(name: "x", line: 4, isLocal: false, isDefinition: true, scope: null, file: !6, type: !12, variable: i32 (i32)** @x)
-!12 = !MDDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, baseType: !7)
-!13 = !MDCompileUnit(language: DW_LANG_C_plus_plus, producer: "clang version 3.5.0 ", isOptimized: false, emissionKind: 1, file: !14, enums: !2, retainedTypes: !2, subprograms: !3, globals: !15, imports: !2)
-!14 = !MDFile(filename: "b.cpp", directory: "/tmp/dbginfo")
+!11 = !DIGlobalVariable(name: "x", line: 4, isLocal: false, isDefinition: true, scope: null, file: !6, type: !12, variable: i32 (i32)** @x)
+!12 = !DIDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, baseType: !7)
+!13 = !DICompileUnit(language: DW_LANG_C_plus_plus, producer: "clang version 3.5.0 ", isOptimized: false, emissionKind: 1, file: !14, enums: !2, retainedTypes: !2, subprograms: !3, globals: !15, imports: !2)
+!14 = !DIFile(filename: "b.cpp", directory: "/tmp/dbginfo")
 !15 = !{!16}
-!16 = !MDGlobalVariable(name: "y", line: 4, isLocal: false, isDefinition: true, scope: null, file: !6, type: !12, variable: i32 (i32)** @y)
+!16 = !DIGlobalVariable(name: "y", line: 4, isLocal: false, isDefinition: true, scope: null, file: !6, type: !12, variable: i32 (i32)** @y)
 !17 = !{i32 2, !"Dwarf Version", i32 4}
 !18 = !{i32 1, !"Debug Info Version", i32 3}
 !19 = !{!"clang version 3.5.0 "}
-!20 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "i", line: 1, arg: 1, scope: !4, file: !6, type: !9)
-!21 = !MDLocation(line: 1, scope: !4)
-!22 = !MDLocation(line: 2, scope: !4)
+!20 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "i", line: 1, arg: 1, scope: !4, file: !6, type: !9)
+!21 = !DILocation(line: 1, scope: !4)
+!22 = !DILocation(line: 2, scope: !4)
diff --git a/test/DebugInfo/cu-range-hole.ll b/test/DebugInfo/cu-range-hole.ll
index a354f2044aa..19a305483fb 100644
--- a/test/DebugInfo/cu-range-hole.ll
+++ b/test/DebugInfo/cu-range-hole.ll
@@ -18,7 +18,7 @@ define i32 @b(i32 %c) #0 {
 entry:
   %c.addr = alloca i32, align 4
   store i32 %c, i32* %c.addr, align 4
-  call void @llvm.dbg.declare(metadata i32* %c.addr, metadata !13, metadata !MDExpression()), !dbg !14
+  call void @llvm.dbg.declare(metadata i32* %c.addr, metadata !13, metadata !DIExpression()), !dbg !14
   %0 = load i32, i32* %c.addr, align 4, !dbg !14
   %add = add nsw i32 %0, 1, !dbg !14
   ret i32 %add, !dbg !14
@@ -42,7 +42,7 @@ define i32 @d(i32 %e) #0 {
 entry:
   %e.addr = alloca i32, align 4
   store i32 %e, i32* %e.addr, align 4
-  call void @llvm.dbg.declare(metadata i32* %e.addr, metadata !15, metadata !MDExpression()), !dbg !16
+  call void @llvm.dbg.declare(metadata i32* %e.addr, metadata !15, metadata !DIExpression()), !dbg !16
   %0 = load i32, i32* %e.addr, align 4, !dbg !16
   %add = add nsw i32 %0, 1, !dbg !16
   ret i32 %add, !dbg !16
@@ -56,19 +56,19 @@ attributes #1 = { nounwind readnone }
 !llvm.module.flags = !{!11, !12}
 
 !0 = !{!"clang version 3.5.0 (trunk 204164) (llvm/trunk 204183)"}
-!1 = !MDCompileUnit(language: DW_LANG_C99, producer: "clang version 3.5.0 (trunk 204164) (llvm/trunk 204183)", isOptimized: false, emissionKind: 1, file: !2, enums: !3, retainedTypes: !3, subprograms: !4, globals: !3, imports: !3)
-!2 = !MDFile(filename: "b.c", directory: "/usr/local/google/home/echristo")
+!1 = !DICompileUnit(language: DW_LANG_C99, producer: "clang version 3.5.0 (trunk 204164) (llvm/trunk 204183)", isOptimized: false, emissionKind: 1, file: !2, enums: !3, retainedTypes: !3, subprograms: !4, globals: !3, imports: !3)
+!2 = !DIFile(filename: "b.c", directory: "/usr/local/google/home/echristo")
 !3 = !{}
 !4 = !{!5, !10}
-!5 = !MDSubprogram(name: "b", line: 1, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 1, file: !2, scope: !6, type: !7, function: i32 (i32)* @b, variables: !3)
-!6 = !MDFile(filename: "b.c", directory: "/usr/local/google/home/echristo")
-!7 = !MDSubroutineType(types: !8)
+!5 = !DISubprogram(name: "b", line: 1, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 1, file: !2, scope: !6, type: !7, function: i32 (i32)* @b, variables: !3)
+!6 = !DIFile(filename: "b.c", directory: "/usr/local/google/home/echristo")
+!7 = !DISubroutineType(types: !8)
 !8 = !{!9, !9}
-!9 = !MDBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
-!10 = !MDSubprogram(name: "d", line: 3, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 3, file: !2, scope: !6, type: !7, function: i32 (i32)* @d, variables: !3)
+!9 = !DIBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
+!10 = !DISubprogram(name: "d", line: 3, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 3, file: !2, scope: !6, type: !7, function: i32 (i32)* @d, variables: !3)
 !11 = !{i32 2, !"Dwarf Version", i32 4}
 !12 = !{i32 1, !"Debug Info Version", i32 3}
-!13 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "c", line: 1, arg: 1, scope: !5, file: !6, type: !9)
-!14 = !MDLocation(line: 1, scope: !5)
-!15 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "e", line: 3, arg: 1, scope: !10, file: !6, type: !9)
-!16 = !MDLocation(line: 3, scope: !10)
+!13 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "c", line: 1, arg: 1, scope: !5, file: !6, type: !9)
+!14 = !DILocation(line: 1, scope: !5)
+!15 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "e", line: 3, arg: 1, scope: !10, file: !6, type: !9)
+!16 = !DILocation(line: 3, scope: !10)
diff --git a/test/DebugInfo/cu-ranges.ll b/test/DebugInfo/cu-ranges.ll
index 7b7f5f2758f..3cec5c88f38 100644
--- a/test/DebugInfo/cu-ranges.ll
+++ b/test/DebugInfo/cu-ranges.ll
@@ -22,7 +22,7 @@ define i32 @foo(i32 %a) #0 section "__TEXT,__foo" {
 entry:
   %a.addr = alloca i32, align 4
   store i32 %a, i32* %a.addr, align 4
-  call void @llvm.dbg.declare(metadata i32* %a.addr, metadata !13, metadata !MDExpression()), !dbg !14
+  call void @llvm.dbg.declare(metadata i32* %a.addr, metadata !13, metadata !DIExpression()), !dbg !14
   %0 = load i32, i32* %a.addr, align 4, !dbg !15
   %add = add nsw i32 %0, 5, !dbg !15
   ret i32 %add, !dbg !15
@@ -36,7 +36,7 @@ define i32 @bar(i32 %a) #0 {
 entry:
   %a.addr = alloca i32, align 4
   store i32 %a, i32* %a.addr, align 4
-  call void @llvm.dbg.declare(metadata i32* %a.addr, metadata !16, metadata !MDExpression()), !dbg !17
+  call void @llvm.dbg.declare(metadata i32* %a.addr, metadata !16, metadata !DIExpression()), !dbg !17
   %0 = load i32, i32* %a.addr, align 4, !dbg !18
   %add = add nsw i32 %0, 5, !dbg !18
   ret i32 %add, !dbg !18
@@ -49,23 +49,23 @@ attributes #1 = { nounwind readnone }
 !llvm.module.flags = !{!10, !11}
 !llvm.ident = !{!12}
 
-!0 = !MDCompileUnit(language: DW_LANG_C99, producer: "clang version 3.5.0 (trunk 204164) (llvm/trunk 204183)", isOptimized: false, emissionKind: 1, file: !1, enums: !2, retainedTypes: !2, subprograms: !3, globals: !2, imports: !2)
-!1 = !MDFile(filename: "foo.c", directory: "/usr/local/google/home/echristo")
+!0 = !DICompileUnit(language: DW_LANG_C99, producer: "clang version 3.5.0 (trunk 204164) (llvm/trunk 204183)", isOptimized: false, emissionKind: 1, file: !1, enums: !2, retainedTypes: !2, subprograms: !3, globals: !2, imports: !2)
+!1 = !DIFile(filename: "foo.c", directory: "/usr/local/google/home/echristo")
 !2 = !{}
 !3 = !{!4, !9}
-!4 = !MDSubprogram(name: "foo", line: 1, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 1, file: !1, scope: !5, type: !6, function: i32 (i32)* @foo, variables: !2)
-!5 = !MDFile(filename: "foo.c", directory: "/usr/local/google/home/echristo")
-!6 = !MDSubroutineType(types: !7)
+!4 = !DISubprogram(name: "foo", line: 1, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 1, file: !1, scope: !5, type: !6, function: i32 (i32)* @foo, variables: !2)
+!5 = !DIFile(filename: "foo.c", directory: "/usr/local/google/home/echristo")
+!6 = !DISubroutineType(types: !7)
 !7 = !{!8, !8}
-!8 = !MDBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
-!9 = !MDSubprogram(name: "bar", line: 5, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 5, file: !1, scope: !5, type: !6, function: i32 (i32)* @bar, variables: !2)
+!8 = !DIBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
+!9 = !DISubprogram(name: "bar", line: 5, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 5, file: !1, scope: !5, type: !6, function: i32 (i32)* @bar, variables: !2)
 !10 = !{i32 2, !"Dwarf Version", i32 4}
 !11 = !{i32 1, !"Debug Info Version", i32 3}
 !12 = !{!"clang version 3.5.0 (trunk 204164) (llvm/trunk 204183)"}
-!13 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "a", line: 1, arg: 1, scope: !4, file: !5, type: !8)
-!14 = !MDLocation(line: 1, scope: !4)
-!15 = !MDLocation(line: 2, scope: !4)
-!16 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "a", line: 5, arg: 1, scope: !9, file: !5, type: !8)
-!17 = !MDLocation(line: 5, scope: !9)
-!18 = !MDLocation(line: 6, scope: !9)
+!13 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "a", line: 1, arg: 1, scope: !4, file: !5, type: !8)
+!14 = !DILocation(line: 1, scope: !4)
+!15 = !DILocation(line: 2, scope: !4)
+!16 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "a", line: 5, arg: 1, scope: !9, file: !5, type: !8)
+!17 = !DILocation(line: 5, scope: !9)
+!18 = !DILocation(line: 6, scope: !9)
 
diff --git a/test/DebugInfo/dead-argument-order.ll b/test/DebugInfo/dead-argument-order.ll
index b2021516e03..2dd556d096d 100644
--- a/test/DebugInfo/dead-argument-order.ll
+++ b/test/DebugInfo/dead-argument-order.ll
@@ -38,8 +38,8 @@
 ; Function Attrs: nounwind readnone uwtable
 define i32 @_Z8function1Si(i32 %s.coerce, i32 %i) #0 {
 entry:
-  tail call void @llvm.dbg.declare(metadata %struct.S* undef, metadata !14, metadata !MDExpression()), !dbg !20
-  tail call void @llvm.dbg.value(metadata i32 %i, i64 0, metadata !15, metadata !MDExpression()), !dbg !20
+  tail call void @llvm.dbg.declare(metadata %struct.S* undef, metadata !14, metadata !DIExpression()), !dbg !20
+  tail call void @llvm.dbg.value(metadata i32 %i, i64 0, metadata !15, metadata !DIExpression()), !dbg !20
   %add = add nsw i32 %i, %s.coerce, !dbg !20
   ret i32 %add, !dbg !20
 }
@@ -57,25 +57,25 @@ attributes #1 = { nounwind readnone }
 !llvm.module.flags = !{!16, !17}
 !llvm.ident = !{!18}
 
-!0 = !MDCompileUnit(language: DW_LANG_C_plus_plus, producer: "clang version 3.5.0 ", isOptimized: true, emissionKind: 1, file: !1, enums: !2, retainedTypes: !3, subprograms: !8, globals: !2, imports: !2)
-!1 = !MDFile(filename: "dead-argument-order.cpp", directory: "/tmp/dbginfo")
+!0 = !DICompileUnit(language: DW_LANG_C_plus_plus, producer: "clang version 3.5.0 ", isOptimized: true, emissionKind: 1, file: !1, enums: !2, retainedTypes: !3, subprograms: !8, globals: !2, imports: !2)
+!1 = !DIFile(filename: "dead-argument-order.cpp", directory: "/tmp/dbginfo")
 !2 = !{}
 !3 = !{!4}
-!4 = !MDCompositeType(tag: DW_TAG_structure_type, name: "S", line: 1, size: 32, align: 32, file: !1, elements: !5, identifier: "_ZTS1S")
+!4 = !DICompositeType(tag: DW_TAG_structure_type, name: "S", line: 1, size: 32, align: 32, file: !1, elements: !5, identifier: "_ZTS1S")
 !5 = !{!6}
-!6 = !MDDerivedType(tag: DW_TAG_member, name: "i", line: 1, size: 32, align: 32, file: !1, scope: !"_ZTS1S", baseType: !7)
-!7 = !MDBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
+!6 = !DIDerivedType(tag: DW_TAG_member, name: "i", line: 1, size: 32, align: 32, file: !1, scope: !"_ZTS1S", baseType: !7)
+!7 = !DIBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
 !8 = !{!9}
-!9 = !MDSubprogram(name: "function", linkageName: "_Z8function1Si", line: 2, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: true, scopeLine: 2, file: !1, scope: !10, type: !11, function: i32 (i32, i32)* @_Z8function1Si, variables: !13)
-!10 = !MDFile(filename: "dead-argument-order.cpp", directory: "/tmp/dbginfo")
-!11 = !MDSubroutineType(types: !12)
+!9 = !DISubprogram(name: "function", linkageName: "_Z8function1Si", line: 2, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: true, scopeLine: 2, file: !1, scope: !10, type: !11, function: i32 (i32, i32)* @_Z8function1Si, variables: !13)
+!10 = !DIFile(filename: "dead-argument-order.cpp", directory: "/tmp/dbginfo")
+!11 = !DISubroutineType(types: !12)
 !12 = !{!7, !4, !7}
 !13 = !{!14, !15}
-!14 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "s", line: 2, arg: 1, scope: !9, file: !10, type: !"_ZTS1S")
-!15 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "i", line: 2, arg: 2, scope: !9, file: !10, type: !7)
+!14 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "s", line: 2, arg: 1, scope: !9, file: !10, type: !"_ZTS1S")
+!15 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "i", line: 2, arg: 2, scope: !9, file: !10, type: !7)
 !16 = !{i32 2, !"Dwarf Version", i32 4}
 !17 = !{i32 2, !"Debug Info Version", i32 3}
 !18 = !{!"clang version 3.5.0 "}
 !19 = !{%struct.S* undef}
-!20 = !MDLocation(line: 2, scope: !9)
+!20 = !DILocation(line: 2, scope: !9)
 
diff --git a/test/DebugInfo/debug-info-always-inline.ll b/test/DebugInfo/debug-info-always-inline.ll
index cfa046d92f1..5eaa6752218 100644
--- a/test/DebugInfo/debug-info-always-inline.ll
+++ b/test/DebugInfo/debug-info-always-inline.ll
@@ -132,12 +132,12 @@ attributes #3 = { "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "
 !15 = !{i32 786433, null, null, !"", i32 0, i64 320, i64 32, i32 0, i32 0, !9, !16, i32 0, null, null, null} ; [ DW_TAG_array_type ] [line 0, size 320, align 32, offset 0] [from int]
 !16 = !{!17}
 !17 = !{i32 786465, i64 0, i64 10}       ; [ DW_TAG_subrange_type ] [0, 9]
-!18 = !MDLocation(line: 3, scope: !4)
-!19 = !MDLocation(line: 4, scope: !4)
+!18 = !DILocation(line: 3, scope: !4)
+!19 = !DILocation(line: 4, scope: !4)
 !20 = !{i32 786688, !4, !"sum", !6, i32 5, !9, i32 0, i32 0} ; [ DW_TAG_auto_variable ] [sum] [line 5]
-!21 = !MDLocation(line: 5, scope: !4)
-!22 = !MDLocation(line: 6, scope: !4)
-!23 = !MDLocation(line: 13, scope: !10)
+!21 = !DILocation(line: 5, scope: !4)
+!22 = !DILocation(line: 6, scope: !4)
+!23 = !DILocation(line: 13, scope: !10)
 !24 = !{i32 786688, !10, !"i", !6, i32 14, !9, i32 0, i32 0} ; [ DW_TAG_auto_variable ] [i] [line 14]
-!25 = !MDLocation(line: 14, scope: !10)
-!26 = !MDLocation(line: 15, scope: !10)
+!25 = !DILocation(line: 14, scope: !10)
+!26 = !DILocation(line: 15, scope: !10)
diff --git a/test/DebugInfo/debug-info-qualifiers.ll b/test/DebugInfo/debug-info-qualifiers.ll
index 96be71e9ae4..1f8f6c41993 100644
--- a/test/DebugInfo/debug-info-qualifiers.ll
+++ b/test/DebugInfo/debug-info-qualifiers.ll
@@ -39,10 +39,10 @@ define void @_Z1gv() #0 {
   %a = alloca %class.A, align 1
   %pl = alloca { i64, i64 }, align 8
   %pr = alloca { i64, i64 }, align 8
-  call void @llvm.dbg.declare(metadata %class.A* %a, metadata !24, metadata !MDExpression()), !dbg !25
-  call void @llvm.dbg.declare(metadata { i64, i64 }* %pl, metadata !26, metadata !MDExpression()), !dbg !31
+  call void @llvm.dbg.declare(metadata %class.A* %a, metadata !24, metadata !DIExpression()), !dbg !25
+  call void @llvm.dbg.declare(metadata { i64, i64 }* %pl, metadata !26, metadata !DIExpression()), !dbg !31
   store { i64, i64 } { i64 ptrtoint (void (%class.A*)* @_ZNKR1A1lEv to i64), i64 0 }, { i64, i64 }* %pl, align 8, !dbg !31
-  call void @llvm.dbg.declare(metadata { i64, i64 }* %pr, metadata !32, metadata !MDExpression()), !dbg !35
+  call void @llvm.dbg.declare(metadata { i64, i64 }* %pr, metadata !32, metadata !DIExpression()), !dbg !35
   store { i64, i64 } { i64 ptrtoint (void (%class.A*)* @_ZNKO1A1rEv to i64), i64 0 }, { i64, i64 }* %pr, align 8, !dbg !35
   ret void, !dbg !36
 }
@@ -61,38 +61,38 @@ attributes #1 = { nounwind readnone }
 !llvm.module.flags = !{!21, !22}
 !llvm.ident = !{!23}
 
-!0 = !MDCompileUnit(language: DW_LANG_C_plus_plus, producer: "clang version 3.5 ", isOptimized: false, emissionKind: 0, file: !1, enums: !2, retainedTypes: !3, subprograms: !16, globals: !2, imports: !2)
-!1 = !MDFile(filename: "debug-info-qualifiers.cpp", directory: "")
+!0 = !DICompileUnit(language: DW_LANG_C_plus_plus, producer: "clang version 3.5 ", isOptimized: false, emissionKind: 0, file: !1, enums: !2, retainedTypes: !3, subprograms: !16, globals: !2, imports: !2)
+!1 = !DIFile(filename: "debug-info-qualifiers.cpp", directory: "")
 !2 = !{}
 !3 = !{!4}
-!4 = !MDCompositeType(tag: DW_TAG_class_type, name: "A", line: 2, size: 8, align: 8, file: !5, elements: !6, identifier: "_ZTS1A")
-!5 = !MDFile(filename: "debug-info-qualifiers.cpp", directory: "")
+!4 = !DICompositeType(tag: DW_TAG_class_type, name: "A", line: 2, size: 8, align: 8, file: !5, elements: !6, identifier: "_ZTS1A")
+!5 = !DIFile(filename: "debug-info-qualifiers.cpp", directory: "")
 !6 = !{!7, !13}
-!7 = !MDSubprogram(name: "l", linkageName: "_ZNKR1A1lEv", line: 5, isLocal: false, isDefinition: false, virtualIndex: 6, flags: DIFlagPrototyped | DIFlagLValueReference, isOptimized: false, scopeLine: 5, file: !5, scope: !"_ZTS1A", type: !8)
-!8 = !MDSubroutineType(flags: DIFlagLValueReference, types: !9)
+!7 = !DISubprogram(name: "l", linkageName: "_ZNKR1A1lEv", line: 5, isLocal: false, isDefinition: false, virtualIndex: 6, flags: DIFlagPrototyped | DIFlagLValueReference, isOptimized: false, scopeLine: 5, file: !5, scope: !"_ZTS1A", type: !8)
+!8 = !DISubroutineType(flags: DIFlagLValueReference, types: !9)
 !9 = !{null, !10}
-!10 = !MDDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, flags: DIFlagArtificial | DIFlagObjectPointer, baseType: !11)
-!11 = !MDDerivedType(tag: DW_TAG_const_type, baseType: !"_ZTS1A")
-!13 = !MDSubprogram(name: "r", linkageName: "_ZNKO1A1rEv", line: 7, isLocal: false, isDefinition: false, virtualIndex: 6, flags: DIFlagObjectPointer | DIFlagRValueReference, isOptimized: false, scopeLine: 7, file: !5, scope: !"_ZTS1A", type: !14)
-!14 = !MDSubroutineType(flags: DIFlagRValueReference, types: !9)
+!10 = !DIDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, flags: DIFlagArtificial | DIFlagObjectPointer, baseType: !11)
+!11 = !DIDerivedType(tag: DW_TAG_const_type, baseType: !"_ZTS1A")
+!13 = !DISubprogram(name: "r", linkageName: "_ZNKO1A1rEv", line: 7, isLocal: false, isDefinition: false, virtualIndex: 6, flags: DIFlagObjectPointer | DIFlagRValueReference, isOptimized: false, scopeLine: 7, file: !5, scope: !"_ZTS1A", type: !14)
+!14 = !DISubroutineType(flags: DIFlagRValueReference, types: !9)
 !16 = !{!17}
-!17 = !MDSubprogram(name: "g", linkageName: "_Z1gv", line: 10, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 10, file: !5, scope: !18, type: !19, function: void ()* @_Z1gv, variables: !2)
-!18 = !MDFile(filename: "debug-info-qualifiers.cpp", directory: "")
-!19 = !MDSubroutineType(types: !20)
+!17 = !DISubprogram(name: "g", linkageName: "_Z1gv", line: 10, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 10, file: !5, scope: !18, type: !19, function: void ()* @_Z1gv, variables: !2)
+!18 = !DIFile(filename: "debug-info-qualifiers.cpp", directory: "")
+!19 = !DISubroutineType(types: !20)
 !20 = !{null}
 !21 = !{i32 2, !"Dwarf Version", i32 4}
 !22 = !{i32 1, !"Debug Info Version", i32 3}
 !23 = !{!"clang version 3.5 "}
-!24 = !MDLocalVariable(tag: DW_TAG_auto_variable, name: "a", line: 11, scope: !17, file: !18, type: !4)
-!25 = !MDLocation(line: 11, scope: !17)
-!26 = !MDLocalVariable(tag: DW_TAG_auto_variable, name: "pl", line: 16, scope: !17, file: !18, type: !27)
-!27 = !MDDerivedType(tag: DW_TAG_ptr_to_member_type, baseType: !28, extraData: !"_ZTS1A")
-!28 = !MDSubroutineType(flags: DIFlagLValueReference, types: !29)
+!24 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "a", line: 11, scope: !17, file: !18, type: !4)
+!25 = !DILocation(line: 11, scope: !17)
+!26 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "pl", line: 16, scope: !17, file: !18, type: !27)
+!27 = !DIDerivedType(tag: DW_TAG_ptr_to_member_type, baseType: !28, extraData: !"_ZTS1A")
+!28 = !DISubroutineType(flags: DIFlagLValueReference, types: !29)
 !29 = !{null, !30}
-!30 = !MDDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, flags: DIFlagArtificial | DIFlagObjectPointer, baseType: !"_ZTS1A")
-!31 = !MDLocation(line: 16, scope: !17)
-!32 = !MDLocalVariable(tag: DW_TAG_auto_variable, name: "pr", line: 21, scope: !17, file: !18, type: !33)
-!33 = !MDDerivedType(tag: DW_TAG_ptr_to_member_type, baseType: !34, extraData: !"_ZTS1A")
-!34 = !MDSubroutineType(flags: DIFlagRValueReference, types: !29)
-!35 = !MDLocation(line: 21, scope: !17)
-!36 = !MDLocation(line: 22, scope: !17)
+!30 = !DIDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, flags: DIFlagArtificial | DIFlagObjectPointer, baseType: !"_ZTS1A")
+!31 = !DILocation(line: 16, scope: !17)
+!32 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "pr", line: 21, scope: !17, file: !18, type: !33)
+!33 = !DIDerivedType(tag: DW_TAG_ptr_to_member_type, baseType: !34, extraData: !"_ZTS1A")
+!34 = !DISubroutineType(flags: DIFlagRValueReference, types: !29)
+!35 = !DILocation(line: 21, scope: !17)
+!36 = !DILocation(line: 22, scope: !17)
diff --git a/test/DebugInfo/debuginfofinder-forward-declaration.ll b/test/DebugInfo/debuginfofinder-forward-declaration.ll
index fc5d294b2c8..db143260e02 100644
--- a/test/DebugInfo/debuginfofinder-forward-declaration.ll
+++ b/test/DebugInfo/debuginfofinder-forward-declaration.ll
@@ -27,16 +27,16 @@
 !llvm.module.flags = !{!10, !11}
 !llvm.ident = !{!12}
 
-!0 = !MDCompileUnit(language: DW_LANG_C99, file: !1, producer: "clang version 3.7.0 (http://llvm.org/git/clang.git 247b30a043eb8f39ea3708e7e995089da0a6b00f) (http://llvm.org/git/llvm.git 6ecc7365a89c771fd229bdd9ffcc178684ea1aa5)", isOptimized: false, runtimeVersion: 0, emissionKind: 1, enums: !2, retainedTypes: !2, subprograms: !2, globals: !3, imports: !2)
-!1 = !MDFile(filename: "minimal.c", directory: "/tmp")
+!0 = !DICompileUnit(language: DW_LANG_C99, file: !1, producer: "clang version 3.7.0 (http://llvm.org/git/clang.git 247b30a043eb8f39ea3708e7e995089da0a6b00f) (http://llvm.org/git/llvm.git 6ecc7365a89c771fd229bdd9ffcc178684ea1aa5)", isOptimized: false, runtimeVersion: 0, emissionKind: 1, enums: !2, retainedTypes: !2, subprograms: !2, globals: !3, imports: !2)
+!1 = !DIFile(filename: "minimal.c", directory: "/tmp")
 !2 = !{}
 !3 = !{!4}
-!4 = !MDGlobalVariable(name: "y", scope: !0, file: !1, line: 7, type: !5, isLocal: false, isDefinition: true, variable: %struct.Y* @y)
-!5 = !MDCompositeType(tag: DW_TAG_structure_type, name: "Y", file: !1, line: 3, size: 64, align: 64, elements: !6)
+!4 = !DIGlobalVariable(name: "y", scope: !0, file: !1, line: 7, type: !5, isLocal: false, isDefinition: true, variable: %struct.Y* @y)
+!5 = !DICompositeType(tag: DW_TAG_structure_type, name: "Y", file: !1, line: 3, size: 64, align: 64, elements: !6)
 !6 = !{!7}
-!7 = !MDDerivedType(tag: DW_TAG_member, name: "x", scope: !5, file: !1, line: 4, baseType: !8, size: 64, align: 64)
-!8 = !MDDerivedType(tag: DW_TAG_pointer_type, baseType: !9, size: 64, align: 64)
-!9 = !MDCompositeType(tag: DW_TAG_structure_type, name: "X", file: !1, line: 1, flags: DIFlagFwdDecl)
+!7 = !DIDerivedType(tag: DW_TAG_member, name: "x", scope: !5, file: !1, line: 4, baseType: !8, size: 64, align: 64)
+!8 = !DIDerivedType(tag: DW_TAG_pointer_type, baseType: !9, size: 64, align: 64)
+!9 = !DICompositeType(tag: DW_TAG_structure_type, name: "X", file: !1, line: 1, flags: DIFlagFwdDecl)
 !10 = !{i32 2, !"Dwarf Version", i32 4}
 !11 = !{i32 2, !"Debug Info Version", i32 3}
 !12 = !{!"clang version 3.7.0 (http://llvm.org/git/clang.git 247b30a043eb8f39ea3708e7e995089da0a6b00f) (http://llvm.org/git/llvm.git 6ecc7365a89c771fd229bdd9ffcc178684ea1aa5)"}
diff --git a/test/DebugInfo/debuginfofinder-multiple-cu.ll b/test/DebugInfo/debuginfofinder-multiple-cu.ll
index d881d43a32a..4088dd54f53 100644
--- a/test/DebugInfo/debuginfofinder-multiple-cu.ll
+++ b/test/DebugInfo/debuginfofinder-multiple-cu.ll
@@ -22,20 +22,20 @@ define void @g() {
 !llvm.dbg.cu = !{!0, !8}
 !llvm.module.flags = !{!13, !16}
 
-!0 = !MDCompileUnit(language: DW_LANG_C99, producer: "clang version 3.4 (192092)", isOptimized: false, emissionKind: 0, file: !1, enums: !2, retainedTypes: !2, subprograms: !3, globals: !2, imports: !2)
-!1 = !MDFile(filename: "test1.c", directory: "/tmp")
+!0 = !DICompileUnit(language: DW_LANG_C99, producer: "clang version 3.4 (192092)", isOptimized: false, emissionKind: 0, file: !1, enums: !2, retainedTypes: !2, subprograms: !3, globals: !2, imports: !2)
+!1 = !DIFile(filename: "test1.c", directory: "/tmp")
 !2 = !{}
 !3 = !{!4}
-!4 = !MDSubprogram(name: "f", line: 1, isLocal: false, isDefinition: true, virtualIndex: 6, isOptimized: false, scopeLine: 1, file: !1, scope: !5, type: !6, function: void ()* @f, variables: !2)
-!5 = !MDFile(filename: "test1.c", directory: "/tmp")
-!6 = !MDSubroutineType(types: !7)
+!4 = !DISubprogram(name: "f", line: 1, isLocal: false, isDefinition: true, virtualIndex: 6, isOptimized: false, scopeLine: 1, file: !1, scope: !5, type: !6, function: void ()* @f, variables: !2)
+!5 = !DIFile(filename: "test1.c", directory: "/tmp")
+!6 = !DISubroutineType(types: !7)
 !7 = !{null}
-!8 = !MDCompileUnit(language: DW_LANG_C99, producer: "clang version 3.4 (192092)", isOptimized: false, emissionKind: 0, file: !9, enums: !2, retainedTypes: !2, subprograms: !10, globals: !2, imports: !2)
-!9 = !MDFile(filename: "test2.c", directory: "/tmp")
+!8 = !DICompileUnit(language: DW_LANG_C99, producer: "clang version 3.4 (192092)", isOptimized: false, emissionKind: 0, file: !9, enums: !2, retainedTypes: !2, subprograms: !10, globals: !2, imports: !2)
+!9 = !DIFile(filename: "test2.c", directory: "/tmp")
 !10 = !{!11}
-!11 = !MDSubprogram(name: "g", line: 1, isLocal: false, isDefinition: true, virtualIndex: 6, isOptimized: false, scopeLine: 1, file: !9, scope: !12, type: !6, function: void ()* @g, variables: !2)
-!12 = !MDFile(filename: "test2.c", directory: "/tmp")
+!11 = !DISubprogram(name: "g", line: 1, isLocal: false, isDefinition: true, virtualIndex: 6, isOptimized: false, scopeLine: 1, file: !9, scope: !12, type: !6, function: void ()* @g, variables: !2)
+!12 = !DIFile(filename: "test2.c", directory: "/tmp")
 !13 = !{i32 2, !"Dwarf Version", i32 4}
-!14 = !MDLocation(line: 1, scope: !4)
-!15 = !MDLocation(line: 1, scope: !11)
+!14 = !DILocation(line: 1, scope: !4)
+!15 = !DILocation(line: 1, scope: !11)
 !16 = !{i32 1, !"Debug Info Version", i32 3}
diff --git a/test/DebugInfo/debuglineinfo-macho.test b/test/DebugInfo/debuglineinfo-macho.test
new file mode 100644
index 00000000000..0c63dd6e8b0
--- /dev/null
+++ b/test/DebugInfo/debuglineinfo-macho.test
@@ -0,0 +1,43 @@
+# Check that relocations get applied
+RUN: llvm-dwarfdump %p/Inputs/test-simple-macho.o | FileCheck %s
+RUN: llvm-dwarfdump %p/Inputs/test-multiple-macho.o | FileCheck %s
+RUN: llvm-rtdyld -printline %p/Inputs/test-multiple-macho.o | FileCheck %s
+RUN: llvm-rtdyld -printobjline %p/Inputs/test-multiple-macho.o | FileCheck %s
+
+CHECK-NOT: error: failed to compute relocation: X86_64_RELOC_UNSIGNED
+
+# Check that relocations get applied correctly
+RUN: llvm-rtdyld -printobjline %p/Inputs/test-simple-macho.o \
+RUN:   | FileCheck %s -check-prefix TEST_SIMPLE
+RUN: llvm-rtdyld -printline %p/Inputs/test-simple-macho.o \
+RUN:   | FileCheck %s -check-prefix TEST_SIMPLE
+RUN: llvm-rtdyld -printobjline %p/Inputs/test-multiple-macho.o \
+RUN:   | FileCheck %s -check-prefix TEST_MULTIPLE
+RUN: llvm-rtdyld -printline %p/Inputs/test-multiple-macho.o \
+RUN:   | FileCheck %s -check-prefix TEST_MULTIPLE
+
+TEST_SIMPLE:      Function: _foo, Size = 11
+TEST_SIMPLE-NEXT:  Line info @ 0: simple.c, line:1
+TEST_SIMPLE-NEXT:  Line info @ 7: simple.c, line:2
+TEST_SIMPLE-NOT:   Line info @ 11: simple.c, line:2
+
+TEST_MULTIPLE:      Function: _bar, Size = 48
+TEST_MULTIPLE-NEXT:   Line info @ 0: multiple.c, line:5
+TEST_MULTIPLE-NEXT:   Line info @ 7: multiple.c, line:6
+TEST_MULTIPLE-NEXT:   Line info @ 16: multiple.c, line:9
+TEST_MULTIPLE-NEXT:   Line info @ 21: multiple.c, line:9
+TEST_MULTIPLE-NEXT:   Line info @ 26: multiple.c, line:7
+TEST_MULTIPLE-NEXT:   Line info @ 33: multiple.c, line:10
+TEST_MULTIPLE-NOT:    Line info @ 48: multiple.c, line:12
+TEST_MULTIPLE-NEXT: Function: _foo, Size = 16
+TEST_MULTIPLE-NEXT:   Line info @ 0: multiple.c, line:1
+TEST_MULTIPLE-NEXT:   Line info @ 7: multiple.c, line:2
+TEST_MULTIPLE-NOT:    Line info @ 16: multiple.c, line:5
+TEST_MULTIPLE-NEXT: Function: _fubar, Size = 46
+TEST_MULTIPLE-NEXT:   Line info @ 0: multiple.c, line:12
+TEST_MULTIPLE-NEXT:   Line info @ 7: multiple.c, line:13
+TEST_MULTIPLE-NEXT:   Line info @ 12: multiple.c, line:17
+TEST_MULTIPLE-NEXT:   Line info @ 25: multiple.c, line:15
+TEST_MULTIPLE-NEXT:   Line info @ 34: multiple.c, line:19
+TEST_MULTIPLE-NEXT:   Line info @ 41: multiple.c, line:21
+TEST_MULTIPLE-NOT:    Line info @ 46: multiple.c, line:21
diff --git a/test/DebugInfo/debuglineinfo.test b/test/DebugInfo/debuglineinfo.test
index 14d2f82b04a..720ba12ed76 100644
--- a/test/DebugInfo/debuglineinfo.test
+++ b/test/DebugInfo/debuglineinfo.test
@@ -1,7 +1,11 @@
 RUN: llvm-rtdyld -printline %p/Inputs/test-inline.o \
 RUN:   | FileCheck %s -check-prefix TEST_INLINE
+RUN: llvm-rtdyld -printdebugline %p/Inputs/test-inline.o \
+RUN:   | FileCheck %s -check-prefix TEST_INLINE
 RUN: llvm-rtdyld -printline %p/Inputs/test-parameters.o \
 RUN:   | FileCheck %s -check-prefix TEST_PARAMETERS
+RUN: llvm-rtdyld -printdebugline %p/Inputs/test-parameters.o \
+RUN:   | FileCheck %s -check-prefix TEST_PARAMETERS
 
 ; This test verifies that relocations are correctly applied to the
 ; .debug_line section and exercises DIContext::getLineInfoForAddressRange().
@@ -14,7 +18,6 @@ TEST_INLINE-NEXT: Line info @ 165: test-inline.cpp, line:35
 TEST_INLINE-NEXT: Function: _Z3foov, Size = 3
 TEST_INLINE-NEXT: Line info @ 0: test-inline.cpp, line:28
 TEST_INLINE-NEXT: Line info @ 2: test-inline.cpp, line:29
-TEST_INLINE-NEXT: Line info @ 3: test-inline.cpp, line:29
 TEST_INLINE-NEXT: Function: main, Size = 146
 TEST_INLINE-NEXT: Line info @ 0: test-inline.cpp, line:39
 TEST_INLINE-NEXT: Line info @ 21: test-inline.cpp, line:41
@@ -25,7 +28,6 @@ TEST_INLINE-NEXT: Line info @ 90: test-inline.cpp, line:45
 TEST_INLINE-NEXT: Line info @ 95: test-inline.cpp, line:46
 TEST_INLINE-NEXT: Line info @ 114: test-inline.cpp, line:48 
 TEST_INLINE-NEXT: Line info @ 141: test-inline.cpp, line:49
-TEST_INLINE-NEXT: Line info @ 146: test-inline.cpp, line:49
 
 ; This test checks the case where all code is in a single section.
 TEST_PARAMETERS:      Function: _Z15test_parametersPfPA2_dR11char_structPPitm, Size = 170
@@ -45,5 +47,4 @@ TEST_PARAMETERS-NEXT: Line info @ 90: test-parameters.cpp, line:45
 TEST_PARAMETERS-NEXT: Line info @ 95: test-parameters.cpp, line:46
 TEST_PARAMETERS-NEXT: Line info @ 114: test-parameters.cpp, line:48 
 TEST_PARAMETERS-NEXT: Line info @ 141: test-parameters.cpp, line:49
-TEST_PARAMETERS-NEXT: Line info @ 146: test-parameters.cpp, line:49
 
diff --git a/test/DebugInfo/dwarf-public-names.ll b/test/DebugInfo/dwarf-public-names.ll
index 254a7129678..bd340578bfb 100644
--- a/test/DebugInfo/dwarf-public-names.ll
+++ b/test/DebugInfo/dwarf-public-names.ll
@@ -59,7 +59,7 @@ define void @_ZN1C15member_functionEv(%struct.C* %this) nounwind uwtable align 2
 entry:
   %this.addr = alloca %struct.C*, align 8
   store %struct.C* %this, %struct.C** %this.addr, align 8
-  call void @llvm.dbg.declare(metadata %struct.C** %this.addr, metadata !28, metadata !MDExpression()), !dbg !30
+  call void @llvm.dbg.declare(metadata %struct.C** %this.addr, metadata !28, metadata !DIExpression()), !dbg !30
   %this1 = load %struct.C*, %struct.C** %this.addr
   store i32 0, i32* @_ZN1C22static_member_variableE, align 4, !dbg !31
   ret void, !dbg !32
@@ -90,42 +90,42 @@ attributes #1 = { nounwind readnone }
 !llvm.dbg.cu = !{!0}
 !llvm.module.flags = !{!38}
 
-!0 = !MDCompileUnit(language: DW_LANG_C_plus_plus, producer: "clang version 3.3 (http://llvm.org/git/clang.git a09cd8103a6a719cb2628cdf0c91682250a17bd2) (http://llvm.org/git/llvm.git 47d03cec0afca0c01ae42b82916d1d731716cd20)", isOptimized: false, emissionKind: 0, file: !37, enums: !1, retainedTypes: !1, subprograms: !2, globals: !24, imports:  !1)
+!0 = !DICompileUnit(language: DW_LANG_C_plus_plus, producer: "clang version 3.3 (http://llvm.org/git/clang.git a09cd8103a6a719cb2628cdf0c91682250a17bd2) (http://llvm.org/git/llvm.git 47d03cec0afca0c01ae42b82916d1d731716cd20)", isOptimized: false, emissionKind: 0, file: !37, enums: !1, retainedTypes: !1, subprograms: !2, globals: !24, imports:  !1)
 !1 = !{}
 !2 = !{!3, !18, !19, !20}
-!3 = !MDSubprogram(name: "member_function", linkageName: "_ZN1C15member_functionEv", line: 9, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 9, file: !4, scope: null, type: !5, function: void (%struct.C*)* @_ZN1C15member_functionEv, declaration: !12, variables: !1)
-!4 = !MDFile(filename: "dwarf-public-names.cpp", directory: "/usr2/kparzysz/s.hex/t")
-!5 = !MDSubroutineType(types: !6)
+!3 = !DISubprogram(name: "member_function", linkageName: "_ZN1C15member_functionEv", line: 9, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 9, file: !4, scope: null, type: !5, function: void (%struct.C*)* @_ZN1C15member_functionEv, declaration: !12, variables: !1)
+!4 = !DIFile(filename: "dwarf-public-names.cpp", directory: "/usr2/kparzysz/s.hex/t")
+!5 = !DISubroutineType(types: !6)
 !6 = !{null, !7}
-!7 = !MDDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, flags: DIFlagArtificial | DIFlagObjectPointer, baseType: !8)
-!8 = !MDCompositeType(tag: DW_TAG_structure_type, name: "C", line: 1, size: 8, align: 8, file: !37, elements: !9)
+!7 = !DIDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, flags: DIFlagArtificial | DIFlagObjectPointer, baseType: !8)
+!8 = !DICompositeType(tag: DW_TAG_structure_type, name: "C", line: 1, size: 8, align: 8, file: !37, elements: !9)
 !9 = !{!10, !12, !14}
-!10 = !MDDerivedType(tag: DW_TAG_member, name: "static_member_variable", line: 4, flags: DIFlagStaticMember, file: !37, scope: !8, baseType: !11)
-!11 = !MDBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
-!12 = !MDSubprogram(name: "member_function", linkageName: "_ZN1C15member_functionEv", line: 2, isLocal: false, isDefinition: false, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 2, file: !4, scope: !8, type: !5, variables: !13)
+!10 = !DIDerivedType(tag: DW_TAG_member, name: "static_member_variable", line: 4, flags: DIFlagStaticMember, file: !37, scope: !8, baseType: !11)
+!11 = !DIBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
+!12 = !DISubprogram(name: "member_function", linkageName: "_ZN1C15member_functionEv", line: 2, isLocal: false, isDefinition: false, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 2, file: !4, scope: !8, type: !5, variables: !13)
 !13 = !{} ; previously: invalid DW_TAG_base_type
-!14 = !MDSubprogram(name: "static_member_function", linkageName: "_ZN1C22static_member_functionEv", line: 3, isLocal: false, isDefinition: false, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 3, file: !4, scope: !8, type: !15, variables: !17)
-!15 = !MDSubroutineType(types: !16)
+!14 = !DISubprogram(name: "static_member_function", linkageName: "_ZN1C22static_member_functionEv", line: 3, isLocal: false, isDefinition: false, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 3, file: !4, scope: !8, type: !15, variables: !17)
+!15 = !DISubroutineType(types: !16)
 !16 = !{!11}
 !17 = !{} ; previously: invalid DW_TAG_base_type
-!18 = !MDSubprogram(name: "static_member_function", linkageName: "_ZN1C22static_member_functionEv", line: 13, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 13, file: !4, scope: null, type: !15, function: i32 ()* @_ZN1C22static_member_functionEv, declaration: !14, variables: !1)
-!19 = !MDSubprogram(name: "global_function", linkageName: "_Z15global_functionv", line: 19, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 19, file: !4, scope: !4, type: !15, function: i32 ()* @_Z15global_functionv, variables: !1)
-!20 = !MDSubprogram(name: "global_namespace_function", linkageName: "_ZN2ns25global_namespace_functionEv", line: 24, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 24, file: !4, scope: !21, type: !22, function: void ()* @_ZN2ns25global_namespace_functionEv, variables: !1)
-!21 = !MDNamespace(name: "ns", line: 23, file: !4, scope: null)
-!22 = !MDSubroutineType(types: !23)
+!18 = !DISubprogram(name: "static_member_function", linkageName: "_ZN1C22static_member_functionEv", line: 13, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 13, file: !4, scope: null, type: !15, function: i32 ()* @_ZN1C22static_member_functionEv, declaration: !14, variables: !1)
+!19 = !DISubprogram(name: "global_function", linkageName: "_Z15global_functionv", line: 19, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 19, file: !4, scope: !4, type: !15, function: i32 ()* @_Z15global_functionv, variables: !1)
+!20 = !DISubprogram(name: "global_namespace_function", linkageName: "_ZN2ns25global_namespace_functionEv", line: 24, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 24, file: !4, scope: !21, type: !22, function: void ()* @_ZN2ns25global_namespace_functionEv, variables: !1)
+!21 = !DINamespace(name: "ns", line: 23, file: !4, scope: null)
+!22 = !DISubroutineType(types: !23)
 !23 = !{null}
 !24 = !{!25, !26, !27}
-!25 = !MDGlobalVariable(name: "static_member_variable", linkageName: "_ZN1C22static_member_variableE", line: 7, isLocal: false, isDefinition: true, scope: !8, file: !4, type: !11, variable: i32* @_ZN1C22static_member_variableE, declaration: !10)
-!26 = !MDGlobalVariable(name: "global_variable", line: 17, isLocal: false, isDefinition: true, scope: null, file: !4, type: !8, variable: %struct.C* @global_variable)
-!27 = !MDGlobalVariable(name: "global_namespace_variable", linkageName: "_ZN2ns25global_namespace_variableE", line: 27, isLocal: false, isDefinition: true, scope: !21, file: !4, type: !11, variable: i32* @_ZN2ns25global_namespace_variableE)
-!28 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "this", line: 9, arg: 1, flags: DIFlagArtificial | DIFlagObjectPointer, scope: !3, file: !4, type: !29)
-!29 = !MDDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, baseType: !8)
-!30 = !MDLocation(line: 9, scope: !3)
-!31 = !MDLocation(line: 10, scope: !3)
-!32 = !MDLocation(line: 11, scope: !3)
-!33 = !MDLocation(line: 14, scope: !18)
-!34 = !MDLocation(line: 20, scope: !19)
-!35 = !MDLocation(line: 25, scope: !20)
-!36 = !MDLocation(line: 26, scope: !20)
-!37 = !MDFile(filename: "dwarf-public-names.cpp", directory: "/usr2/kparzysz/s.hex/t")
+!25 = !DIGlobalVariable(name: "static_member_variable", linkageName: "_ZN1C22static_member_variableE", line: 7, isLocal: false, isDefinition: true, scope: !8, file: !4, type: !11, variable: i32* @_ZN1C22static_member_variableE, declaration: !10)
+!26 = !DIGlobalVariable(name: "global_variable", line: 17, isLocal: false, isDefinition: true, scope: null, file: !4, type: !8, variable: %struct.C* @global_variable)
+!27 = !DIGlobalVariable(name: "global_namespace_variable", linkageName: "_ZN2ns25global_namespace_variableE", line: 27, isLocal: false, isDefinition: true, scope: !21, file: !4, type: !11, variable: i32* @_ZN2ns25global_namespace_variableE)
+!28 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "this", line: 9, arg: 1, flags: DIFlagArtificial | DIFlagObjectPointer, scope: !3, file: !4, type: !29)
+!29 = !DIDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, baseType: !8)
+!30 = !DILocation(line: 9, scope: !3)
+!31 = !DILocation(line: 10, scope: !3)
+!32 = !DILocation(line: 11, scope: !3)
+!33 = !DILocation(line: 14, scope: !18)
+!34 = !DILocation(line: 20, scope: !19)
+!35 = !DILocation(line: 25, scope: !20)
+!36 = !DILocation(line: 26, scope: !20)
+!37 = !DIFile(filename: "dwarf-public-names.cpp", directory: "/usr2/kparzysz/s.hex/t")
 !38 = !{i32 1, !"Debug Info Version", i32 3}
diff --git a/test/DebugInfo/dwarfdump-64-bit-dwarf.test b/test/DebugInfo/dwarfdump-64-bit-dwarf.test
new file mode 100644
index 00000000000..0a24414c27b
--- /dev/null
+++ b/test/DebugInfo/dwarfdump-64-bit-dwarf.test
@@ -0,0 +1,15 @@
+RUN: llvm-dwarfdump %p/Inputs/dwarfdump.elf-mips64-64-bit-dwarf \
+RUN:   --debug-dump=line | FileCheck %s
+
+# FIXME: llvm-dwarfdump's support for 64-bit dwarf is currently limited to
+# .debug_line.
+
+CHECK:   total_length: 0x00000212
+CHECK:        version: 2
+CHECK:prologue_length: 0x000001ab
+CHECK:min_inst_length: 1
+CHECK:default_is_stmt: 1
+CHECK:      line_base: -5
+CHECK:     line_range: 14
+CHECK:    opcode_base: 13
+CHECK: is_stmt end_sequence
diff --git a/test/DebugInfo/dwarfdump-invalid.test b/test/DebugInfo/dwarfdump-invalid.test
new file mode 100644
index 00000000000..a36ad2f412e
--- /dev/null
+++ b/test/DebugInfo/dwarfdump-invalid.test
@@ -0,0 +1,6 @@
+; Verify that llvm-dwarfdump doesn't crash on broken input files.
+
+RUN: not llvm-dwarfdump %p/Inputs/invalid.elf 2>&1 | FileCheck %s --check-prefix=INVALID-ELF
+RUN: not llvm-dwarfdump %p/Inputs/invalid.elf.2 2>&1 | FileCheck %s --check-prefix=INVALID-ELF
+RUN: not llvm-dwarfdump %p/Inputs/invalid.elf.3 2>&1 | FileCheck %s --check-prefix=INVALID-ELF
+INVALID-ELF: Invalid data was encountered while parsing the file
diff --git a/test/DebugInfo/empty.ll b/test/DebugInfo/empty.ll
index f91105aba04..ad26b500b35 100644
--- a/test/DebugInfo/empty.ll
+++ b/test/DebugInfo/empty.ll
@@ -24,8 +24,8 @@
 !llvm.dbg.cu = !{!0}
 !llvm.module.flags = !{!5}
 
-!0 = !MDCompileUnit(language: DW_LANG_C99, producer: "clang version 3.1 (trunk 143523)", isOptimized: true, emissionKind: 0, file: !4, enums: !2, retainedTypes: !2, subprograms: !2, globals: !2)
+!0 = !DICompileUnit(language: DW_LANG_C99, producer: "clang version 3.1 (trunk 143523)", isOptimized: true, emissionKind: 0, file: !4, enums: !2, retainedTypes: !2, subprograms: !2, globals: !2)
 !2 = !{}
-!3 = !MDFile(filename: "empty.c", directory: "/home/nlewycky")
-!4 = !MDFile(filename: "empty.c", directory: "/home/nlewycky")
+!3 = !DIFile(filename: "empty.c", directory: "/home/nlewycky")
+!4 = !DIFile(filename: "empty.c", directory: "/home/nlewycky")
 !5 = !{i32 1, !"Debug Info Version", i32 3}
diff --git a/test/DebugInfo/enum-types.ll b/test/DebugInfo/enum-types.ll
index bce4dfde82b..7b50f5e9087 100644
--- a/test/DebugInfo/enum-types.ll
+++ b/test/DebugInfo/enum-types.ll
@@ -25,7 +25,7 @@ define void @_Z4topA2EA(i32 %sa) #0 {
 entry:
   %sa.addr = alloca i32, align 4
   store i32 %sa, i32* %sa.addr, align 4
-  call void @llvm.dbg.declare(metadata i32* %sa.addr, metadata !22, metadata !MDExpression()), !dbg !23
+  call void @llvm.dbg.declare(metadata i32* %sa.addr, metadata !22, metadata !DIExpression()), !dbg !23
   ret void, !dbg !24
 }
 
@@ -37,7 +37,7 @@ define void @_Z4topB2EA(i32 %sa) #0 {
 entry:
   %sa.addr = alloca i32, align 4
   store i32 %sa, i32* %sa.addr, align 4
-  call void @llvm.dbg.declare(metadata i32* %sa.addr, metadata !25, metadata !MDExpression()), !dbg !26
+  call void @llvm.dbg.declare(metadata i32* %sa.addr, metadata !25, metadata !DIExpression()), !dbg !26
   ret void, !dbg !27
 }
 
@@ -48,31 +48,31 @@ attributes #1 = { nounwind readnone }
 !llvm.module.flags = !{!19, !20}
 !llvm.ident = !{!21, !21}
 
-!0 = !MDCompileUnit(language: DW_LANG_C_plus_plus, producer: "clang version 3.5.0 (trunk 214102:214133) (llvm/trunk 214102:214132)", isOptimized: false, emissionKind: 1, file: !1, enums: !2, retainedTypes: !2, subprograms: !6, globals: !11, imports: !11)
-!1 = !MDFile(filename: "a.cpp", directory: "")
+!0 = !DICompileUnit(language: DW_LANG_C_plus_plus, producer: "clang version 3.5.0 (trunk 214102:214133) (llvm/trunk 214102:214132)", isOptimized: false, emissionKind: 1, file: !1, enums: !2, retainedTypes: !2, subprograms: !6, globals: !11, imports: !11)
+!1 = !DIFile(filename: "a.cpp", directory: "")
 !2 = !{!3}
-!3 = !MDCompositeType(tag: DW_TAG_enumeration_type, name: "EA", line: 1, size: 32, align: 32, file: !1, elements: !4, identifier: "_ZTS2EA")
+!3 = !DICompositeType(tag: DW_TAG_enumeration_type, name: "EA", line: 1, size: 32, align: 32, file: !1, elements: !4, identifier: "_ZTS2EA")
 !4 = !{!5}
-!5 = !MDEnumerator(name: "EA_0", value: 0) ; [ DW_TAG_enumerator ] [EA_0 :: 0]
+!5 = !DIEnumerator(name: "EA_0", value: 0) ; [ DW_TAG_enumerator ] [EA_0 :: 0]
 !6 = !{!7}
-!7 = !MDSubprogram(name: "topA", linkageName: "_Z4topA2EA", line: 5, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 5, file: !1, scope: !8, type: !9, function: void (i32)* @_Z4topA2EA, variables: !11)
-!8 = !MDFile(filename: "a.cpp", directory: "")
-!9 = !MDSubroutineType(types: !10)
+!7 = !DISubprogram(name: "topA", linkageName: "_Z4topA2EA", line: 5, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 5, file: !1, scope: !8, type: !9, function: void (i32)* @_Z4topA2EA, variables: !11)
+!8 = !DIFile(filename: "a.cpp", directory: "")
+!9 = !DISubroutineType(types: !10)
 !10 = !{null, !"_ZTS2EA"}
 !11 = !{}
-!12 = !MDCompileUnit(language: DW_LANG_C_plus_plus, producer: "clang version 3.5.0 (trunk 214102:214133) (llvm/trunk 214102:214132)", isOptimized: false, emissionKind: 1, file: !13, enums: !14, retainedTypes: !14, subprograms: !16, globals: !11, imports: !11)
-!13 = !MDFile(filename: "b.cpp", directory: "")
+!12 = !DICompileUnit(language: DW_LANG_C_plus_plus, producer: "clang version 3.5.0 (trunk 214102:214133) (llvm/trunk 214102:214132)", isOptimized: false, emissionKind: 1, file: !13, enums: !14, retainedTypes: !14, subprograms: !16, globals: !11, imports: !11)
+!13 = !DIFile(filename: "b.cpp", directory: "")
 !14 = !{!15}
-!15 = !MDCompositeType(tag: DW_TAG_enumeration_type, name: "EA", line: 1, size: 32, align: 32, file: !13, elements: !4, identifier: "_ZTS2EA")
+!15 = !DICompositeType(tag: DW_TAG_enumeration_type, name: "EA", line: 1, size: 32, align: 32, file: !13, elements: !4, identifier: "_ZTS2EA")
 !16 = !{!17}
-!17 = !MDSubprogram(name: "topB", linkageName: "_Z4topB2EA", line: 5, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 5, file: !13, scope: !18, type: !9, function: void (i32)* @_Z4topB2EA, variables: !11)
-!18 = !MDFile(filename: "b.cpp", directory: "")
+!17 = !DISubprogram(name: "topB", linkageName: "_Z4topB2EA", line: 5, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 5, file: !13, scope: !18, type: !9, function: void (i32)* @_Z4topB2EA, variables: !11)
+!18 = !DIFile(filename: "b.cpp", directory: "")
 !19 = !{i32 2, !"Dwarf Version", i32 2}
 !20 = !{i32 2, !"Debug Info Version", i32 3}
 !21 = !{!"clang version 3.5.0 (trunk 214102:214133) (llvm/trunk 214102:214132)"}
-!22 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "sa", line: 5, arg: 1, scope: !7, file: !8, type: !"_ZTS2EA")
-!23 = !MDLocation(line: 5, column: 14, scope: !7)
-!24 = !MDLocation(line: 6, column: 1, scope: !7)
-!25 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "sa", line: 5, arg: 1, scope: !17, file: !18, type: !"_ZTS2EA")
-!26 = !MDLocation(line: 5, column: 14, scope: !17)
-!27 = !MDLocation(line: 6, column: 1, scope: !17)
+!22 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "sa", line: 5, arg: 1, scope: !7, file: !8, type: !"_ZTS2EA")
+!23 = !DILocation(line: 5, column: 14, scope: !7)
+!24 = !DILocation(line: 6, column: 1, scope: !7)
+!25 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "sa", line: 5, arg: 1, scope: !17, file: !18, type: !"_ZTS2EA")
+!26 = !DILocation(line: 5, column: 14, scope: !17)
+!27 = !DILocation(line: 6, column: 1, scope: !17)
diff --git a/test/DebugInfo/enum.ll b/test/DebugInfo/enum.ll
index c706a5ead2e..fd07a92ae41 100644
--- a/test/DebugInfo/enum.ll
+++ b/test/DebugInfo/enum.ll
@@ -39,7 +39,7 @@
 define void @_Z4funcv() #0 {
 entry:
   %b = alloca i32, align 4
-  call void @llvm.dbg.declare(metadata i32* %b, metadata !20, metadata !MDExpression()), !dbg !22
+  call void @llvm.dbg.declare(metadata i32* %b, metadata !20, metadata !DIExpression()), !dbg !22
   store i32 0, i32* %b, align 4, !dbg !22
   ret void, !dbg !23
 }
@@ -53,28 +53,28 @@ attributes #1 = { nounwind readnone }
 !llvm.dbg.cu = !{!0}
 !llvm.module.flags = !{!19, !24}
 
-!0 = !MDCompileUnit(language: DW_LANG_C_plus_plus, producer: "clang version 3.4 ", isOptimized: false, emissionKind: 0, file: !1, enums: !2, retainedTypes: !11, subprograms: !12, globals: !17, imports: !11)
-!1 = !MDFile(filename: "enum.cpp", directory: "/tmp")
+!0 = !DICompileUnit(language: DW_LANG_C_plus_plus, producer: "clang version 3.4 ", isOptimized: false, emissionKind: 0, file: !1, enums: !2, retainedTypes: !11, subprograms: !12, globals: !17, imports: !11)
+!1 = !DIFile(filename: "enum.cpp", directory: "/tmp")
 !2 = !{!3, !8}
-!3 = !MDCompositeType(tag: DW_TAG_enumeration_type, name: "e1", line: 1, size: 64, align: 64, file: !1, elements: !4)
+!3 = !DICompositeType(tag: DW_TAG_enumeration_type, name: "e1", line: 1, size: 64, align: 64, file: !1, elements: !4)
 !4 = !{!5, !6, !7}
-!5 = !MDEnumerator(name: "I", value: 0) ; [ DW_TAG_enumerator ] [I :: 0]
-!6 = !MDEnumerator(name: "J", value: 4294967295) ; [ DW_TAG_enumerator ] [J :: 4294967295]
-!7 = !MDEnumerator(name: "K", value: -1152921504606846976) ; [ DW_TAG_enumerator ] [K :: 17293822569102704640]
-!8 = !MDCompositeType(tag: DW_TAG_enumeration_type, name: "e2", line: 2, size: 32, align: 32, file: !1, elements: !9)
+!5 = !DIEnumerator(name: "I", value: 0) ; [ DW_TAG_enumerator ] [I :: 0]
+!6 = !DIEnumerator(name: "J", value: 4294967295) ; [ DW_TAG_enumerator ] [J :: 4294967295]
+!7 = !DIEnumerator(name: "K", value: -1152921504606846976) ; [ DW_TAG_enumerator ] [K :: 17293822569102704640]
+!8 = !DICompositeType(tag: DW_TAG_enumeration_type, name: "e2", line: 2, size: 32, align: 32, file: !1, elements: !9)
 !9 = !{!10}
-!10 = !MDEnumerator(name: "X", value: 0) ; [ DW_TAG_enumerator ] [X :: 0]
+!10 = !DIEnumerator(name: "X", value: 0) ; [ DW_TAG_enumerator ] [X :: 0]
 !11 = !{}
 !12 = !{!13}
-!13 = !MDSubprogram(name: "func", linkageName: "_Z4funcv", line: 3, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 3, file: !1, scope: !14, type: !15, function: void ()* @_Z4funcv, variables: !11)
-!14 = !MDFile(filename: "enum.cpp", directory: "/tmp")
-!15 = !MDSubroutineType(types: !16)
+!13 = !DISubprogram(name: "func", linkageName: "_Z4funcv", line: 3, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 3, file: !1, scope: !14, type: !15, function: void ()* @_Z4funcv, variables: !11)
+!14 = !DIFile(filename: "enum.cpp", directory: "/tmp")
+!15 = !DISubroutineType(types: !16)
 !16 = !{null}
 !17 = !{!18}
-!18 = !MDGlobalVariable(name: "a", line: 1, isLocal: false, isDefinition: true, scope: null, file: !14, type: !3, variable: i64* @a)
+!18 = !DIGlobalVariable(name: "a", line: 1, isLocal: false, isDefinition: true, scope: null, file: !14, type: !3, variable: i64* @a)
 !19 = !{i32 2, !"Dwarf Version", i32 3}
-!20 = !MDLocalVariable(tag: DW_TAG_auto_variable, name: "b", line: 4, scope: !13, file: !14, type: !21)
-!21 = !MDBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
-!22 = !MDLocation(line: 4, scope: !13)
-!23 = !MDLocation(line: 5, scope: !13)
+!20 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "b", line: 4, scope: !13, file: !14, type: !21)
+!21 = !DIBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
+!22 = !DILocation(line: 4, scope: !13)
+!23 = !DILocation(line: 5, scope: !13)
 !24 = !{i32 1, !"Debug Info Version", i32 3}
diff --git a/test/DebugInfo/global.ll b/test/DebugInfo/global.ll
index 6383e05bccc..cf2d26b36ac 100644
--- a/test/DebugInfo/global.ll
+++ b/test/DebugInfo/global.ll
@@ -26,17 +26,17 @@ attributes #0 = { nounwind readnone uwtable "less-precise-fpmad"="false" "no-fra
 !llvm.dbg.cu = !{!0}
 !llvm.module.flags = !{!11, !13}
 
-!0 = !MDCompileUnit(language: DW_LANG_C_plus_plus, producer: "clang version 3.4 ", isOptimized: true, emissionKind: 0, file: !1, enums: !2, retainedTypes: !2, subprograms: !3, globals: !9, imports: !2)
-!1 = !MDFile(filename: "global.cpp", directory: "/tmp")
+!0 = !DICompileUnit(language: DW_LANG_C_plus_plus, producer: "clang version 3.4 ", isOptimized: true, emissionKind: 0, file: !1, enums: !2, retainedTypes: !2, subprograms: !3, globals: !9, imports: !2)
+!1 = !DIFile(filename: "global.cpp", directory: "/tmp")
 !2 = !{}
 !3 = !{!4}
-!4 = !MDSubprogram(name: "main", line: 2, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: true, scopeLine: 2, file: !1, scope: !5, type: !6, function: i32 ()* @main, variables: !2)
-!5 = !MDFile(filename: "global.cpp", directory: "/tmp")
-!6 = !MDSubroutineType(types: !7)
+!4 = !DISubprogram(name: "main", line: 2, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: true, scopeLine: 2, file: !1, scope: !5, type: !6, function: i32 ()* @main, variables: !2)
+!5 = !DIFile(filename: "global.cpp", directory: "/tmp")
+!6 = !DISubroutineType(types: !7)
 !7 = !{!8}
-!8 = !MDBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
+!8 = !DIBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
 !9 = !{!10}
-!10 = !MDGlobalVariable(name: "i", linkageName: "_ZL1i", line: 1, isLocal: true, isDefinition: true, scope: null, file: !5, type: !8)
+!10 = !DIGlobalVariable(name: "i", linkageName: "_ZL1i", line: 1, isLocal: true, isDefinition: true, scope: null, file: !5, type: !8)
 !11 = !{i32 2, !"Dwarf Version", i32 3}
-!12 = !MDLocation(line: 4, scope: !4)
+!12 = !DILocation(line: 4, scope: !4)
 !13 = !{i32 1, !"Debug Info Version", i32 3}
diff --git a/test/DebugInfo/incorrect-variable-debugloc.ll b/test/DebugInfo/incorrect-variable-debugloc.ll
index 68f0ae0b670..930f8c92cf6 100644
--- a/test/DebugInfo/incorrect-variable-debugloc.ll
+++ b/test/DebugInfo/incorrect-variable-debugloc.ll
@@ -110,7 +110,7 @@ entry:
 
 ; <label>:30                                      ; preds = %24, %5
   store i32 0, i32* %i.i, align 4, !dbg !39, !tbaa !41
-  tail call void @llvm.dbg.value(metadata %struct.C* %8, i64 0, metadata !27, metadata !MDExpression()), !dbg !46
+  tail call void @llvm.dbg.value(metadata %struct.C* %8, i64 0, metadata !27, metadata !DIExpression()), !dbg !46
   call void @_ZN1C5m_fn3Ev(%struct.C* %8), !dbg !47
   unreachable, !dbg !47
 }
@@ -145,7 +145,7 @@ entry:
   %16 = add i64 %15, 0, !dbg !48
   %17 = inttoptr i64 %16 to i64*, !dbg !48
   store i64 -868083113472691727, i64* %17, !dbg !48
-  tail call void @llvm.dbg.value(metadata %struct.C* %this, i64 0, metadata !30, metadata !MDExpression()), !dbg !48
+  tail call void @llvm.dbg.value(metadata %struct.C* %this, i64 0, metadata !30, metadata !DIExpression()), !dbg !48
   %call = call i32 @_ZN1A5m_fn1Ev(%struct.A* %8), !dbg !49
   %i.i = getelementptr inbounds %struct.C, %struct.C* %this, i64 0, i32 1, i32 0, !dbg !50
   %18 = ptrtoint i32* %i.i to i64, !dbg !50
@@ -336,56 +336,56 @@ attributes #3 = { nounwind readnone }
 !llvm.module.flags = !{!36, !37}
 !llvm.ident = !{!38}
 
-!0 = !MDCompileUnit(language: DW_LANG_C_plus_plus, producer: "clang version 3.5.0 ", isOptimized: true, emissionKind: 1, file: !1, enums: !2, retainedTypes: !3, subprograms: !21, globals: !2, imports: !2)
-!1 = !MDFile(filename: "<stdin>", directory: "/tmp/dbginfo")
+!0 = !DICompileUnit(language: DW_LANG_C_plus_plus, producer: "clang version 3.5.0 ", isOptimized: true, emissionKind: 1, file: !1, enums: !2, retainedTypes: !3, subprograms: !21, globals: !2, imports: !2)
+!1 = !DIFile(filename: "<stdin>", directory: "/tmp/dbginfo")
 !2 = !{}
 !3 = !{!4, !14}
-!4 = !MDCompositeType(tag: DW_TAG_structure_type, name: "C", line: 10, size: 64, align: 32, file: !5, elements: !6, identifier: "_ZTS1C")
-!5 = !MDFile(filename: "incorrect-variable-debug-loc.cpp", directory: "/tmp/dbginfo")
+!4 = !DICompositeType(tag: DW_TAG_structure_type, name: "C", line: 10, size: 64, align: 32, file: !5, elements: !6, identifier: "_ZTS1C")
+!5 = !DIFile(filename: "incorrect-variable-debug-loc.cpp", directory: "/tmp/dbginfo")
 !6 = !{!7, !9, !10}
-!7 = !MDDerivedType(tag: DW_TAG_member, name: "j", line: 12, size: 32, align: 32, file: !5, scope: !"_ZTS1C", baseType: !8)
-!8 = !MDBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
-!9 = !MDDerivedType(tag: DW_TAG_member, name: "b", line: 13, size: 32, align: 32, offset: 32, file: !5, scope: !"_ZTS1C", baseType: !"_ZTS1B")
-!10 = !MDSubprogram(name: "m_fn3", linkageName: "_ZN1C5m_fn3Ev", line: 11, isLocal: false, isDefinition: false, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: true, scopeLine: 11, file: !5, scope: !"_ZTS1C", type: !11)
-!11 = !MDSubroutineType(types: !12)
+!7 = !DIDerivedType(tag: DW_TAG_member, name: "j", line: 12, size: 32, align: 32, file: !5, scope: !"_ZTS1C", baseType: !8)
+!8 = !DIBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
+!9 = !DIDerivedType(tag: DW_TAG_member, name: "b", line: 13, size: 32, align: 32, offset: 32, file: !5, scope: !"_ZTS1C", baseType: !"_ZTS1B")
+!10 = !DISubprogram(name: "m_fn3", linkageName: "_ZN1C5m_fn3Ev", line: 11, isLocal: false, isDefinition: false, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: true, scopeLine: 11, file: !5, scope: !"_ZTS1C", type: !11)
+!11 = !DISubroutineType(types: !12)
 !12 = !{null, !13}
-!13 = !MDDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, flags: DIFlagArtificial | DIFlagObjectPointer, baseType: !"_ZTS1C")
-!14 = !MDCompositeType(tag: DW_TAG_structure_type, name: "B", line: 5, size: 32, align: 32, file: !5, elements: !15, identifier: "_ZTS1B")
+!13 = !DIDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, flags: DIFlagArtificial | DIFlagObjectPointer, baseType: !"_ZTS1C")
+!14 = !DICompositeType(tag: DW_TAG_structure_type, name: "B", line: 5, size: 32, align: 32, file: !5, elements: !15, identifier: "_ZTS1B")
 !15 = !{!16, !17}
-!16 = !MDDerivedType(tag: DW_TAG_member, name: "i", line: 7, size: 32, align: 32, file: !5, scope: !"_ZTS1B", baseType: !8)
-!17 = !MDSubprogram(name: "m_fn2", linkageName: "_ZN1B5m_fn2Ev", line: 6, isLocal: false, isDefinition: false, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: true, scopeLine: 6, file: !5, scope: !"_ZTS1B", type: !18)
-!18 = !MDSubroutineType(types: !19)
+!16 = !DIDerivedType(tag: DW_TAG_member, name: "i", line: 7, size: 32, align: 32, file: !5, scope: !"_ZTS1B", baseType: !8)
+!17 = !DISubprogram(name: "m_fn2", linkageName: "_ZN1B5m_fn2Ev", line: 6, isLocal: false, isDefinition: false, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: true, scopeLine: 6, file: !5, scope: !"_ZTS1B", type: !18)
+!18 = !DISubroutineType(types: !19)
 !19 = !{null, !20}
-!20 = !MDDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, flags: DIFlagArtificial | DIFlagObjectPointer, baseType: !"_ZTS1B")
+!20 = !DIDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, flags: DIFlagArtificial | DIFlagObjectPointer, baseType: !"_ZTS1B")
 !21 = !{!22, !28, !32}
-!22 = !MDSubprogram(name: "fn1", linkageName: "_Z3fn1v", line: 16, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: true, scopeLine: 16, file: !5, scope: !23, type: !24, function: i32 ()* @_Z3fn1v, variables: !26)
-!23 = !MDFile(filename: "incorrect-variable-debug-loc.cpp", directory: "/tmp/dbginfo")
-!24 = !MDSubroutineType(types: !25)
+!22 = !DISubprogram(name: "fn1", linkageName: "_Z3fn1v", line: 16, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: true, scopeLine: 16, file: !5, scope: !23, type: !24, function: i32 ()* @_Z3fn1v, variables: !26)
+!23 = !DIFile(filename: "incorrect-variable-debug-loc.cpp", directory: "/tmp/dbginfo")
+!24 = !DISubroutineType(types: !25)
 !25 = !{!8}
 !26 = !{!27}
-!27 = !MDLocalVariable(tag: DW_TAG_auto_variable, name: "A", line: 17, scope: !22, file: !23, type: !"_ZTS1C")
-!28 = !MDSubprogram(name: "m_fn3", linkageName: "_ZN1C5m_fn3Ev", line: 21, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: true, scopeLine: 21, file: !5, scope: !"_ZTS1C", type: !11, function: void (%struct.C*)* @_ZN1C5m_fn3Ev, declaration: !10, variables: !29)
+!27 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "A", line: 17, scope: !22, file: !23, type: !"_ZTS1C")
+!28 = !DISubprogram(name: "m_fn3", linkageName: "_ZN1C5m_fn3Ev", line: 21, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: true, scopeLine: 21, file: !5, scope: !"_ZTS1C", type: !11, function: void (%struct.C*)* @_ZN1C5m_fn3Ev, declaration: !10, variables: !29)
 !29 = !{!30}
-!30 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "this", arg: 1, flags: DIFlagArtificial | DIFlagObjectPointer, scope: !28, type: !31)
-!31 = !MDDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, baseType: !"_ZTS1C")
-!32 = !MDSubprogram(name: "m_fn2", linkageName: "_ZN1B5m_fn2Ev", line: 6, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: true, scopeLine: 6, file: !5, scope: !"_ZTS1B", type: !18, declaration: !17, variables: !33)
+!30 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "this", arg: 1, flags: DIFlagArtificial | DIFlagObjectPointer, scope: !28, type: !31)
+!31 = !DIDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, baseType: !"_ZTS1C")
+!32 = !DISubprogram(name: "m_fn2", linkageName: "_ZN1B5m_fn2Ev", line: 6, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: true, scopeLine: 6, file: !5, scope: !"_ZTS1B", type: !18, declaration: !17, variables: !33)
 !33 = !{!34}
-!34 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "this", arg: 1, flags: DIFlagArtificial | DIFlagObjectPointer, scope: !32, type: !35)
-!35 = !MDDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, baseType: !"_ZTS1B")
+!34 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "this", arg: 1, flags: DIFlagArtificial | DIFlagObjectPointer, scope: !32, type: !35)
+!35 = !DIDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, baseType: !"_ZTS1B")
 !36 = !{i32 2, !"Dwarf Version", i32 4}
 !37 = !{i32 2, !"Debug Info Version", i32 3}
 !38 = !{!"clang version 3.5.0 "}
-!39 = !MDLocation(line: 6, scope: !32, inlinedAt: !40)
-!40 = !MDLocation(line: 18, scope: !22)
+!39 = !DILocation(line: 6, scope: !32, inlinedAt: !40)
+!40 = !DILocation(line: 18, scope: !22)
 !41 = !{!42, !43, i64 0}
 !42 = !{!"_ZTS1B", !43, i64 0}
 !43 = !{!"int", !44, i64 0}
 !44 = !{!"omnipotent char", !45, i64 0}
 !45 = !{!"Simple C/C++ TBAA"}
-!46 = !MDLocation(line: 17, scope: !22)
-!47 = !MDLocation(line: 19, scope: !22)
-!48 = !MDLocation(line: 0, scope: !28)
-!49 = !MDLocation(line: 22, scope: !28)
-!50 = !MDLocation(line: 6, scope: !32, inlinedAt: !51)
-!51 = !MDLocation(line: 23, scope: !28)
-!52 = !MDLocation(line: 24, scope: !28)
+!46 = !DILocation(line: 17, scope: !22)
+!47 = !DILocation(line: 19, scope: !22)
+!48 = !DILocation(line: 0, scope: !28)
+!49 = !DILocation(line: 22, scope: !28)
+!50 = !DILocation(line: 6, scope: !32, inlinedAt: !51)
+!51 = !DILocation(line: 23, scope: !28)
+!52 = !DILocation(line: 24, scope: !28)
diff --git a/test/DebugInfo/incorrect-variable-debugloc1.ll b/test/DebugInfo/incorrect-variable-debugloc1.ll
index ffedeb2b36e..3ece94ac888 100644
--- a/test/DebugInfo/incorrect-variable-debugloc1.ll
+++ b/test/DebugInfo/incorrect-variable-debugloc1.ll
@@ -53,25 +53,25 @@ attributes #2 = { nounwind readnone }
 !llvm.module.flags = !{!12, !13}
 !llvm.ident = !{!14}
 
-!0 = !MDCompileUnit(language: DW_LANG_C_plus_plus, producer: "clang version 3.6.0 (trunk 223522)", isOptimized: true, emissionKind: 1, file: !1, enums: !2, retainedTypes: !2, subprograms: !3, globals: !2, imports: !2)
-!1 = !MDFile(filename: "test.cpp", directory: "/home/kromanova/ngh/ToT_latest/llvm/test/DebugInfo")
+!0 = !DICompileUnit(language: DW_LANG_C_plus_plus, producer: "clang version 3.6.0 (trunk 223522)", isOptimized: true, emissionKind: 1, file: !1, enums: !2, retainedTypes: !2, subprograms: !3, globals: !2, imports: !2)
+!1 = !DIFile(filename: "test.cpp", directory: "/home/kromanova/ngh/ToT_latest/llvm/test/DebugInfo")
 !2 = !{}
 !3 = !{!4}
-!4 = !MDSubprogram(name: "main", line: 3, isLocal: false, isDefinition: true, flags: DIFlagPrototyped, isOptimized: true, scopeLine: 4, file: !1, scope: !5, type: !6, function: i32 ()* @main, variables: !9)
-!5 = !MDFile(filename: "test.cpp", directory: "/home/kromanova/ngh/ToT_latest/llvm/test/DebugInfo")
-!6 = !MDSubroutineType(types: !7)
+!4 = !DISubprogram(name: "main", line: 3, isLocal: false, isDefinition: true, flags: DIFlagPrototyped, isOptimized: true, scopeLine: 4, file: !1, scope: !5, type: !6, function: i32 ()* @main, variables: !9)
+!5 = !DIFile(filename: "test.cpp", directory: "/home/kromanova/ngh/ToT_latest/llvm/test/DebugInfo")
+!6 = !DISubroutineType(types: !7)
 !7 = !{!8}
-!8 = !MDBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
+!8 = !DIBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
 !9 = !{!10}
-!10 = !MDLocalVariable(tag: DW_TAG_auto_variable, name: "c", line: 5, scope: !4, file: !5, type: !11)
-!11 = !MDDerivedType(tag: DW_TAG_volatile_type, baseType: !8)
+!10 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "c", line: 5, scope: !4, file: !5, type: !11)
+!11 = !DIDerivedType(tag: DW_TAG_volatile_type, baseType: !8)
 !12 = !{i32 2, !"Dwarf Version", i32 2}
 !13 = !{i32 2, !"Debug Info Version", i32 3}
 !14 = !{!"clang version 3.6.0 (trunk 223522)"}
 !15 = !{i32 13}
-!16 = !MDExpression()
-!17 = !MDLocation(line: 5, column: 16, scope: !4)
-!18 = !MDLocation(line: 5, column: 3, scope: !4)
-!19 = !MDLocation(line: 6, column: 7, scope: !4)
-!20 = !MDLocation(line: 7, column: 3, scope: !4)
+!16 = !DIExpression()
+!17 = !DILocation(line: 5, column: 16, scope: !4)
+!18 = !DILocation(line: 5, column: 3, scope: !4)
+!19 = !DILocation(line: 6, column: 7, scope: !4)
+!20 = !DILocation(line: 7, column: 3, scope: !4)
 
diff --git a/test/DebugInfo/inheritance.ll b/test/DebugInfo/inheritance.ll
index 0ecded5eb71..10d799b54f7 100644
--- a/test/DebugInfo/inheritance.ll
+++ b/test/DebugInfo/inheritance.ll
@@ -16,7 +16,7 @@ entry:
   %0 = alloca i32                                 ; <i32*> [#uses=2]
   %tst = alloca %struct.test1                     ; <%struct.test1*> [#uses=1]
   %"alloca point" = bitcast i32 0 to i32          ; <i32> [#uses=0]
-  call void @llvm.dbg.declare(metadata %struct.test1* %tst, metadata !0, metadata !MDExpression()), !dbg !21
+  call void @llvm.dbg.declare(metadata %struct.test1* %tst, metadata !0, metadata !DIExpression()), !dbg !21
   call void @_ZN5test1C1Ev(%struct.test1* %tst) nounwind, !dbg !22
   store i32 0, i32* %0, align 4, !dbg !23
   %1 = load i32, i32* %0, align 4, !dbg !23            ; <i32> [#uses=1]
@@ -32,7 +32,7 @@ define linkonce_odr void @_ZN5test1C1Ev(%struct.test1* %this) nounwind ssp align
 entry:
   %this_addr = alloca %struct.test1*              ; <%struct.test1**> [#uses=2]
   %"alloca point" = bitcast i32 0 to i32          ; <i32> [#uses=0]
-  call void @llvm.dbg.declare(metadata %struct.test1** %this_addr, metadata !24, metadata !MDExpression()), !dbg !28
+  call void @llvm.dbg.declare(metadata %struct.test1** %this_addr, metadata !24, metadata !DIExpression()), !dbg !28
   store %struct.test1* %this, %struct.test1** %this_addr
   %0 = load %struct.test1*, %struct.test1** %this_addr, align 8, !dbg !28 ; <%struct.test1*> [#uses=1]
   %1 = getelementptr inbounds %struct.test1, %struct.test1* %0, i32 0, i32 0, !dbg !28 ; <i32 (...)***> [#uses=1]
@@ -49,7 +49,7 @@ define linkonce_odr void @_ZN5test1D1Ev(%struct.test1* %this) nounwind ssp align
 entry:
   %this_addr = alloca %struct.test1*              ; <%struct.test1**> [#uses=3]
   %"alloca point" = bitcast i32 0 to i32          ; <i32> [#uses=0]
-  call void @llvm.dbg.declare(metadata %struct.test1** %this_addr, metadata !32, metadata !MDExpression()), !dbg !34
+  call void @llvm.dbg.declare(metadata %struct.test1** %this_addr, metadata !32, metadata !DIExpression()), !dbg !34
   store %struct.test1* %this, %struct.test1** %this_addr
   %0 = load %struct.test1*, %struct.test1** %this_addr, align 8, !dbg !35 ; <%struct.test1*> [#uses=1]
   %1 = getelementptr inbounds %struct.test1, %struct.test1* %0, i32 0, i32 0, !dbg !35 ; <i32 (...)***> [#uses=1]
@@ -78,7 +78,7 @@ define linkonce_odr void @_ZN5test1D0Ev(%struct.test1* %this) nounwind ssp align
 entry:
   %this_addr = alloca %struct.test1*              ; <%struct.test1**> [#uses=3]
   %"alloca point" = bitcast i32 0 to i32          ; <i32> [#uses=0]
-  call void @llvm.dbg.declare(metadata %struct.test1** %this_addr, metadata !38, metadata !MDExpression()), !dbg !40
+  call void @llvm.dbg.declare(metadata %struct.test1** %this_addr, metadata !38, metadata !DIExpression()), !dbg !40
   store %struct.test1* %this, %struct.test1** %this_addr
   %0 = load %struct.test1*, %struct.test1** %this_addr, align 8, !dbg !41 ; <%struct.test1*> [#uses=1]
   %1 = getelementptr inbounds %struct.test1, %struct.test1* %0, i32 0, i32 0, !dbg !41 ; <i32 (...)***> [#uses=1]
@@ -105,50 +105,50 @@ return:                                           ; preds = %bb2
 
 declare void @_ZdlPv(i8*) nounwind
 
-!0 = !MDLocalVariable(tag: DW_TAG_auto_variable, name: "tst", line: 13, scope: !1, file: !4, type: !8)
-!1 = distinct !MDLexicalBlock(line: 0, column: 0, file: !44, scope: !2)
-!2 = distinct !MDLexicalBlock(line: 0, column: 0, file: !44, scope: !3)
-!3 = !MDSubprogram(name: "main", linkageName: "main", line: 11, isLocal: false, isDefinition: true, virtualIndex: 6, isOptimized: false, scope: !4, type: !5)
-!4 = !MDCompileUnit(language: DW_LANG_C_plus_plus, producer: "4.2.1 (Based on Apple Inc. build 5658) (LLVM build)", isOptimized: true, emissionKind: 0, file: !44, enums: !45, retainedTypes: !45)
-!5 = !MDSubroutineType(types: !6)
+!0 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "tst", line: 13, scope: !1, file: !4, type: !8)
+!1 = distinct !DILexicalBlock(line: 0, column: 0, file: !44, scope: !2)
+!2 = distinct !DILexicalBlock(line: 0, column: 0, file: !44, scope: !3)
+!3 = !DISubprogram(name: "main", linkageName: "main", line: 11, isLocal: false, isDefinition: true, virtualIndex: 6, isOptimized: false, scope: !4, type: !5)
+!4 = !DICompileUnit(language: DW_LANG_C_plus_plus, producer: "4.2.1 (Based on Apple Inc. build 5658) (LLVM build)", isOptimized: true, emissionKind: 0, file: !44, enums: !45, retainedTypes: !45)
+!5 = !DISubroutineType(types: !6)
 !6 = !{!7}
-!7 = !MDBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
-!8 = !MDCompositeType(tag: DW_TAG_structure_type, name: "test1", line: 1, size: 64, align: 64, file: !44, scope: !4, elements: !9, vtableHolder: !8)
+!7 = !DIBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
+!8 = !DICompositeType(tag: DW_TAG_structure_type, name: "test1", line: 1, size: 64, align: 64, file: !44, scope: !4, elements: !9, vtableHolder: !8)
 !9 = !{!10, !14, !18}
-!10 = !MDDerivedType(tag: DW_TAG_member, name: "_vptr$test1", line: 1, size: 64, align: 64, file: !44, scope: !8, baseType: !11)
-!11 = !MDDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, file: !4, baseType: !12)
-!12 = !MDDerivedType(tag: DW_TAG_pointer_type, name: "__vtbl_ptr_type", scope: !4, baseType: !5)
-!13 = !MDCompileUnit(language: DW_LANG_C_plus_plus, producer: "4.2.1 (Based on Apple Inc. build 5658) (LLVM build)", isOptimized: false, emissionKind: 0, file: !46, enums: !45, retainedTypes: !45)
-!14 = !MDSubprogram(name: "test1", line: 1, isLocal: false, isDefinition: false, virtualIndex: 6, flags: DIFlagPrivate, isOptimized: false, scope: !8, type: !15)
-!15 = !MDSubroutineType(types: !16)
+!10 = !DIDerivedType(tag: DW_TAG_member, name: "_vptr$test1", line: 1, size: 64, align: 64, file: !44, scope: !8, baseType: !11)
+!11 = !DIDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, file: !4, baseType: !12)
+!12 = !DIDerivedType(tag: DW_TAG_pointer_type, name: "__vtbl_ptr_type", scope: !4, baseType: !5)
+!13 = !DICompileUnit(language: DW_LANG_C_plus_plus, producer: "4.2.1 (Based on Apple Inc. build 5658) (LLVM build)", isOptimized: false, emissionKind: 0, file: !46, enums: !45, retainedTypes: !45)
+!14 = !DISubprogram(name: "test1", line: 1, isLocal: false, isDefinition: false, virtualIndex: 6, flags: DIFlagPrivate, isOptimized: false, scope: !8, type: !15)
+!15 = !DISubroutineType(types: !16)
 !16 = !{null, !17}
-!17 = !MDDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, flags: DIFlagArtificial, file: !4, baseType: !8)
-!18 = !MDSubprogram(name: "~test1", line: 4, isLocal: false, isDefinition: false, virtuality: DW_VIRTUALITY_virtual, virtualIndex: 6, isOptimized: false, scope: !8, type: !19, containingType: !8)
-!19 = !MDSubroutineType(types: !20)
+!17 = !DIDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, flags: DIFlagArtificial, file: !4, baseType: !8)
+!18 = !DISubprogram(name: "~test1", line: 4, isLocal: false, isDefinition: false, virtuality: DW_VIRTUALITY_virtual, virtualIndex: 6, isOptimized: false, scope: !8, type: !19, containingType: !8)
+!19 = !DISubroutineType(types: !20)
 !20 = !{null, !17, !7}
-!21 = !MDLocation(line: 11, scope: !1)
-!22 = !MDLocation(line: 13, scope: !1)
-!23 = !MDLocation(line: 14, scope: !1)
-!24 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "this", line: 13, arg: 0, scope: !25, file: !4, type: !26)
-!25 = !MDSubprogram(name: "test1", linkageName: "_ZN5test1C1Ev", line: 1, isLocal: false, isDefinition: true, virtualIndex: 6, isOptimized: false, scope: !4, type: !15)
-!26 = !MDDerivedType(tag: DW_TAG_const_type, size: 64, align: 64, flags: DIFlagArtificial, file: !4, baseType: !27)
-!27 = !MDDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, file: !4, baseType: !8)
-!28 = !MDLocation(line: 1, scope: !25)
-!29 = !MDLocation(line: 1, scope: !30)
-!30 = distinct !MDLexicalBlock(line: 0, column: 0, file: !44, scope: !31)
-!31 = distinct !MDLexicalBlock(line: 0, column: 0, file: !44, scope: !25)
-!32 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "this", line: 4, arg: 0, scope: !33, file: !4, type: !26)
-!33 = !MDSubprogram(name: "~test1", linkageName: "_ZN5test1D1Ev", line: 4, isLocal: false, isDefinition: true, virtuality: DW_VIRTUALITY_virtual, virtualIndex: 6, isOptimized: false, scope: !8, type: !15, containingType: !8)
-!34 = !MDLocation(line: 4, scope: !33)
-!35 = !MDLocation(line: 5, scope: !36)
-!36 = distinct !MDLexicalBlock(line: 0, column: 0, file: !44, scope: !33)
-!37 = !MDLocation(line: 6, scope: !36)
-!38 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "this", line: 4, arg: 0, scope: !39, file: !4, type: !26)
-!39 = !MDSubprogram(name: "~test1", linkageName: "_ZN5test1D0Ev", line: 4, isLocal: false, isDefinition: true, virtuality: DW_VIRTUALITY_virtual, virtualIndex: 6, isOptimized: false, scope: !8, type: !15, containingType: !8)
-!40 = !MDLocation(line: 4, scope: !39)
-!41 = !MDLocation(line: 5, scope: !42)
-!42 = distinct !MDLexicalBlock(line: 0, column: 0, file: !44, scope: !39)
-!43 = !MDLocation(line: 6, scope: !42)
-!44 = !MDFile(filename: "inheritance.cpp", directory: "/tmp/")
+!21 = !DILocation(line: 11, scope: !1)
+!22 = !DILocation(line: 13, scope: !1)
+!23 = !DILocation(line: 14, scope: !1)
+!24 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "this", line: 13, arg: 1, scope: !25, file: !4, type: !26)
+!25 = !DISubprogram(name: "test1", linkageName: "_ZN5test1C1Ev", line: 1, isLocal: false, isDefinition: true, virtualIndex: 6, isOptimized: false, scope: !4, type: !15)
+!26 = !DIDerivedType(tag: DW_TAG_const_type, size: 64, align: 64, flags: DIFlagArtificial, file: !4, baseType: !27)
+!27 = !DIDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, file: !4, baseType: !8)
+!28 = !DILocation(line: 1, scope: !25)
+!29 = !DILocation(line: 1, scope: !30)
+!30 = distinct !DILexicalBlock(line: 0, column: 0, file: !44, scope: !31)
+!31 = distinct !DILexicalBlock(line: 0, column: 0, file: !44, scope: !25)
+!32 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "this", line: 4, arg: 1, scope: !33, file: !4, type: !26)
+!33 = !DISubprogram(name: "~test1", linkageName: "_ZN5test1D1Ev", line: 4, isLocal: false, isDefinition: true, virtuality: DW_VIRTUALITY_virtual, virtualIndex: 6, isOptimized: false, scope: !8, type: !15, containingType: !8)
+!34 = !DILocation(line: 4, scope: !33)
+!35 = !DILocation(line: 5, scope: !36)
+!36 = distinct !DILexicalBlock(line: 0, column: 0, file: !44, scope: !33)
+!37 = !DILocation(line: 6, scope: !36)
+!38 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "this", line: 4, arg: 1, scope: !39, file: !4, type: !26)
+!39 = !DISubprogram(name: "~test1", linkageName: "_ZN5test1D0Ev", line: 4, isLocal: false, isDefinition: true, virtuality: DW_VIRTUALITY_virtual, virtualIndex: 6, isOptimized: false, scope: !8, type: !15, containingType: !8)
+!40 = !DILocation(line: 4, scope: !39)
+!41 = !DILocation(line: 5, scope: !42)
+!42 = distinct !DILexicalBlock(line: 0, column: 0, file: !44, scope: !39)
+!43 = !DILocation(line: 6, scope: !42)
+!44 = !DIFile(filename: "inheritance.cpp", directory: "/tmp/")
 !45 = !{i32 0}
-!46 = !MDFile(filename: "<built-in>", directory: "/tmp/")
+!46 = !DIFile(filename: "<built-in>", directory: "/tmp/")
diff --git a/test/DebugInfo/inline-debug-info-multiret.ll b/test/DebugInfo/inline-debug-info-multiret.ll
index b4ab0b21471..d86e6abbd80 100644
--- a/test/DebugInfo/inline-debug-info-multiret.ll
+++ b/test/DebugInfo/inline-debug-info-multiret.ll
@@ -10,8 +10,8 @@
 ; CHECK: br label %invoke.cont, !dbg ![[MD]]
 ; The branch instruction has the source location of line 9 and its inlined location
 ; has the source location of line 14.
-; CHECK: ![[INL:[0-9]+]] = distinct !MDLocation(line: 14, scope: {{.*}})
-; CHECK: ![[MD]] = !MDLocation(line: 9, scope: {{.*}}, inlinedAt: ![[INL]])
+; CHECK: ![[INL:[0-9]+]] = distinct !DILocation(line: 14, scope: {{.*}})
+; CHECK: ![[MD]] = !DILocation(line: 9, scope: {{.*}}, inlinedAt: ![[INL]])
 
 ; ModuleID = 'test.cpp'
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
@@ -27,8 +27,8 @@ entry:
   %k.addr = alloca i32, align 4
   %k2 = alloca i32, align 4
   store i32 %k, i32* %k.addr, align 4
-  call void @llvm.dbg.declare(metadata i32* %k.addr, metadata !13, metadata !MDExpression()), !dbg !14
-  call void @llvm.dbg.declare(metadata i32* %k2, metadata !15, metadata !MDExpression()), !dbg !16
+  call void @llvm.dbg.declare(metadata i32* %k.addr, metadata !13, metadata !DIExpression()), !dbg !14
+  call void @llvm.dbg.declare(metadata i32* %k2, metadata !15, metadata !DIExpression()), !dbg !16
   %0 = load i32, i32* %k.addr, align 4, !dbg !16
   %call = call i32 @_Z8test_exti(i32 %0), !dbg !16
   store i32 %call, i32* %k2, align 4, !dbg !16
@@ -57,7 +57,7 @@ declare void @llvm.dbg.declare(metadata, metadata, metadata) #1
 
 declare i32 @_Z8test_exti(i32)
 
-define i32 @_Z5test2v()  {
+define i32 @_Z5test2v() personality i8* bitcast (i32 (...)* @__gxx_personality_v0 to i8*) {
 entry:
   %exn.slot = alloca i8*
   %ehselector.slot = alloca i32
@@ -70,7 +70,7 @@ invoke.cont:                                      ; preds = %entry
   br label %try.cont, !dbg !23
 
 lpad:                                             ; preds = %entry
-  %1 = landingpad { i8*, i32 } personality i8* bitcast (i32 (...)* @__gxx_personality_v0 to i8*)
+  %1 = landingpad { i8*, i32 }
           catch i8* bitcast (i8** @_ZTIi to i8*), !dbg !21
   %2 = extractvalue { i8*, i32 } %1, 0, !dbg !21
   store i8* %2, i8** %exn.slot, !dbg !21
@@ -85,7 +85,7 @@ catch.dispatch:                                   ; preds = %lpad
   br i1 %matches, label %catch, label %eh.resume, !dbg !23
 
 catch:                                            ; preds = %catch.dispatch
-  call void @llvm.dbg.declare(metadata i32* %e, metadata !24, metadata !MDExpression()), !dbg !25
+  call void @llvm.dbg.declare(metadata i32* %e, metadata !24, metadata !DIExpression()), !dbg !25
   %exn = load i8*, i8** %exn.slot, !dbg !23
   %5 = call i8* @__cxa_begin_catch(i8* %exn) #2, !dbg !23
   %6 = bitcast i8* %5 to i32*, !dbg !23
@@ -122,35 +122,35 @@ attributes #2 = { nounwind }
 !llvm.dbg.cu = !{!0}
 !llvm.module.flags = !{!31}
 
-!0 = !MDCompileUnit(language: DW_LANG_C_plus_plus, producer: "clang version 3.3 ", isOptimized: false, emissionKind: 0, file: !1, enums: !2, retainedTypes: !2, subprograms: !3, globals: !2, imports: !2)
-!1 = !MDFile(filename: "<unknown>", directory: "")
+!0 = !DICompileUnit(language: DW_LANG_C_plus_plus, producer: "clang version 3.3 ", isOptimized: false, emissionKind: 0, file: !1, enums: !2, retainedTypes: !2, subprograms: !3, globals: !2, imports: !2)
+!1 = !DIFile(filename: "<unknown>", directory: "")
 !2 = !{}
 !3 = !{!4, !10}
-!4 = !MDSubprogram(name: "test", linkageName: "_Z4testi", line: 4, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 4, file: !5, scope: !6, type: !7, function: i32 (i32)* @_Z4testi, variables: !2)
-!5 = !MDFile(filename: "test.cpp", directory: "")
-!6 = !MDFile(filename: "test.cpp", directory: "")
-!7 = !MDSubroutineType(types: !8)
+!4 = !DISubprogram(name: "test", linkageName: "_Z4testi", line: 4, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 4, file: !5, scope: !6, type: !7, function: i32 (i32)* @_Z4testi, variables: !2)
+!5 = !DIFile(filename: "test.cpp", directory: "")
+!6 = !DIFile(filename: "test.cpp", directory: "")
+!7 = !DISubroutineType(types: !8)
 !8 = !{!9, !9}
-!9 = !MDBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
-!10 = !MDSubprogram(name: "test2", linkageName: "_Z5test2v", line: 11, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 11, file: !5, scope: !6, type: !11, function: i32 ()* @_Z5test2v, variables: !2)
-!11 = !MDSubroutineType(types: !12)
+!9 = !DIBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
+!10 = !DISubprogram(name: "test2", linkageName: "_Z5test2v", line: 11, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 11, file: !5, scope: !6, type: !11, function: i32 ()* @_Z5test2v, variables: !2)
+!11 = !DISubroutineType(types: !12)
 !12 = !{!9}
-!13 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "k", line: 4, arg: 1, scope: !4, file: !6, type: !9)
-!14 = !MDLocation(line: 4, scope: !4)
-!15 = !MDLocalVariable(tag: DW_TAG_auto_variable, name: "k2", line: 5, scope: !4, file: !6, type: !9)
-!16 = !MDLocation(line: 5, scope: !4)
-!17 = !MDLocation(line: 6, scope: !4)
-!18 = !MDLocation(line: 7, scope: !4)
-!19 = !MDLocation(line: 8, scope: !4)
-!20 = !MDLocation(line: 9, scope: !4)
-!21 = !MDLocation(line: 14, scope: !22)
-!22 = distinct !MDLexicalBlock(line: 13, column: 0, file: !5, scope: !10)
-!23 = !MDLocation(line: 15, scope: !22)
-!24 = !MDLocalVariable(tag: DW_TAG_auto_variable, name: "e", line: 16, scope: !10, file: !6, type: !9)
-!25 = !MDLocation(line: 16, scope: !10)
-!26 = !MDLocation(line: 17, scope: !27)
-!27 = distinct !MDLexicalBlock(line: 16, column: 0, file: !5, scope: !10)
-!28 = !MDLocation(line: 18, scope: !27)
-!29 = !MDLocation(line: 19, scope: !10)
-!30 = !MDLocation(line: 20, scope: !10)
+!13 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "k", line: 4, arg: 1, scope: !4, file: !6, type: !9)
+!14 = !DILocation(line: 4, scope: !4)
+!15 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "k2", line: 5, scope: !4, file: !6, type: !9)
+!16 = !DILocation(line: 5, scope: !4)
+!17 = !DILocation(line: 6, scope: !4)
+!18 = !DILocation(line: 7, scope: !4)
+!19 = !DILocation(line: 8, scope: !4)
+!20 = !DILocation(line: 9, scope: !4)
+!21 = !DILocation(line: 14, scope: !22)
+!22 = distinct !DILexicalBlock(line: 13, column: 0, file: !5, scope: !10)
+!23 = !DILocation(line: 15, scope: !22)
+!24 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "e", line: 16, scope: !10, file: !6, type: !9)
+!25 = !DILocation(line: 16, scope: !10)
+!26 = !DILocation(line: 17, scope: !27)
+!27 = distinct !DILexicalBlock(line: 16, column: 0, file: !5, scope: !10)
+!28 = !DILocation(line: 18, scope: !27)
+!29 = !DILocation(line: 19, scope: !10)
+!30 = !DILocation(line: 20, scope: !10)
 !31 = !{i32 1, !"Debug Info Version", i32 3}
diff --git a/test/DebugInfo/inline-debug-info.ll b/test/DebugInfo/inline-debug-info.ll
index f1f04ff5532..908093ca255 100644
--- a/test/DebugInfo/inline-debug-info.ll
+++ b/test/DebugInfo/inline-debug-info.ll
@@ -31,8 +31,8 @@
 ; CHECK: br label %invoke.cont, !dbg [[MD:![0-9]+]]
 ; The branch instruction has the source location of line 9 and its inlined location
 ; has the source location of line 14.
-; CHECK: [[INL:![0-9]*]] = distinct !MDLocation(line: 14, scope: {{.*}})
-; CHECK: [[MD]] = !MDLocation(line: 9, scope: {{.*}}, inlinedAt: [[INL]])
+; CHECK: [[INL:![0-9]*]] = distinct !DILocation(line: 14, scope: {{.*}})
+; CHECK: [[MD]] = !DILocation(line: 9, scope: {{.*}}, inlinedAt: [[INL]])
 
 ; ModuleID = 'test.cpp'
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
@@ -47,8 +47,8 @@ entry:
   %k.addr = alloca i32, align 4
   %k2 = alloca i32, align 4
   store i32 %k, i32* %k.addr, align 4
-  call void @llvm.dbg.declare(metadata i32* %k.addr, metadata !13, metadata !MDExpression()), !dbg !14
-  call void @llvm.dbg.declare(metadata i32* %k2, metadata !15, metadata !MDExpression()), !dbg !16
+  call void @llvm.dbg.declare(metadata i32* %k.addr, metadata !13, metadata !DIExpression()), !dbg !14
+  call void @llvm.dbg.declare(metadata i32* %k2, metadata !15, metadata !DIExpression()), !dbg !16
   %0 = load i32, i32* %k.addr, align 4, !dbg !16
   %call = call i32 @_Z8test_exti(i32 %0), !dbg !16
   store i32 %call, i32* %k2, align 4, !dbg !16
@@ -75,7 +75,7 @@ declare void @llvm.dbg.declare(metadata, metadata, metadata) #1
 
 declare i32 @_Z8test_exti(i32)
 
-define i32 @_Z5test2v()  {
+define i32 @_Z5test2v() personality i8* bitcast (i32 (...)* @__gxx_personality_v0 to i8*) {
 entry:
   %exn.slot = alloca i8*
   %ehselector.slot = alloca i32
@@ -88,7 +88,7 @@ invoke.cont:                                      ; preds = %entry
   br label %try.cont, !dbg !23
 
 lpad:                                             ; preds = %entry
-  %1 = landingpad { i8*, i32 } personality i8* bitcast (i32 (...)* @__gxx_personality_v0 to i8*)
+  %1 = landingpad { i8*, i32 }
           catch i8* bitcast (i8** @_ZTIi to i8*), !dbg !21
   %2 = extractvalue { i8*, i32 } %1, 0, !dbg !21
   store i8* %2, i8** %exn.slot, !dbg !21
@@ -103,7 +103,7 @@ catch.dispatch:                                   ; preds = %lpad
   br i1 %matches, label %catch, label %eh.resume, !dbg !23
 
 catch:                                            ; preds = %catch.dispatch
-  call void @llvm.dbg.declare(metadata i32* %e, metadata !24, metadata !MDExpression()), !dbg !25
+  call void @llvm.dbg.declare(metadata i32* %e, metadata !24, metadata !DIExpression()), !dbg !25
   %exn = load i8*, i8** %exn.slot, !dbg !23
   %5 = call i8* @__cxa_begin_catch(i8* %exn) #2, !dbg !23
   %6 = bitcast i8* %5 to i32*, !dbg !23
@@ -140,35 +140,35 @@ attributes #2 = { nounwind }
 !llvm.dbg.cu = !{!0}
 !llvm.module.flags = !{!31}
 
-!0 = !MDCompileUnit(language: DW_LANG_C_plus_plus, producer: "clang version 3.3 ", isOptimized: false, emissionKind: 0, file: !1, enums: !2, retainedTypes: !2, subprograms: !3, globals: !2, imports: !2)
-!1 = !MDFile(filename: "<unknown>", directory: "")
+!0 = !DICompileUnit(language: DW_LANG_C_plus_plus, producer: "clang version 3.3 ", isOptimized: false, emissionKind: 0, file: !1, enums: !2, retainedTypes: !2, subprograms: !3, globals: !2, imports: !2)
+!1 = !DIFile(filename: "<unknown>", directory: "")
 !2 = !{}
 !3 = !{!4, !10}
-!4 = !MDSubprogram(name: "test", linkageName: "_Z4testi", line: 4, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 4, file: !5, scope: !6, type: !7, function: i32 (i32)* @_Z4testi, variables: !2)
-!5 = !MDFile(filename: "test.cpp", directory: "")
-!6 = !MDFile(filename: "test.cpp", directory: "")
-!7 = !MDSubroutineType(types: !8)
+!4 = !DISubprogram(name: "test", linkageName: "_Z4testi", line: 4, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 4, file: !5, scope: !6, type: !7, function: i32 (i32)* @_Z4testi, variables: !2)
+!5 = !DIFile(filename: "test.cpp", directory: "")
+!6 = !DIFile(filename: "test.cpp", directory: "")
+!7 = !DISubroutineType(types: !8)
 !8 = !{!9, !9}
-!9 = !MDBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
-!10 = !MDSubprogram(name: "test2", linkageName: "_Z5test2v", line: 11, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 11, file: !5, scope: !6, type: !11, function: i32 ()* @_Z5test2v, variables: !2)
-!11 = !MDSubroutineType(types: !12)
+!9 = !DIBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
+!10 = !DISubprogram(name: "test2", linkageName: "_Z5test2v", line: 11, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 11, file: !5, scope: !6, type: !11, function: i32 ()* @_Z5test2v, variables: !2)
+!11 = !DISubroutineType(types: !12)
 !12 = !{!9}
-!13 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "k", line: 4, arg: 1, scope: !4, file: !6, type: !9)
-!14 = !MDLocation(line: 4, scope: !4)
-!15 = !MDLocalVariable(tag: DW_TAG_auto_variable, name: "k2", line: 5, scope: !4, file: !6, type: !9)
-!16 = !MDLocation(line: 5, scope: !4)
-!17 = !MDLocation(line: 6, scope: !4)
-!18 = !MDLocation(line: 7, scope: !4)
-!19 = !MDLocation(line: 8, scope: !4)
-!20 = !MDLocation(line: 9, scope: !4)
-!21 = !MDLocation(line: 14, scope: !22)
-!22 = distinct !MDLexicalBlock(line: 13, column: 0, file: !5, scope: !10)
-!23 = !MDLocation(line: 15, scope: !22)
-!24 = !MDLocalVariable(tag: DW_TAG_auto_variable, name: "e", line: 16, scope: !10, file: !6, type: !9)
-!25 = !MDLocation(line: 16, scope: !10)
-!26 = !MDLocation(line: 17, scope: !27)
-!27 = distinct !MDLexicalBlock(line: 16, column: 0, file: !5, scope: !10)
-!28 = !MDLocation(line: 18, scope: !27)
-!29 = !MDLocation(line: 19, scope: !10)
-!30 = !MDLocation(line: 20, scope: !10)
+!13 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "k", line: 4, arg: 1, scope: !4, file: !6, type: !9)
+!14 = !DILocation(line: 4, scope: !4)
+!15 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "k2", line: 5, scope: !4, file: !6, type: !9)
+!16 = !DILocation(line: 5, scope: !4)
+!17 = !DILocation(line: 6, scope: !4)
+!18 = !DILocation(line: 7, scope: !4)
+!19 = !DILocation(line: 8, scope: !4)
+!20 = !DILocation(line: 9, scope: !4)
+!21 = !DILocation(line: 14, scope: !22)
+!22 = distinct !DILexicalBlock(line: 13, column: 0, file: !5, scope: !10)
+!23 = !DILocation(line: 15, scope: !22)
+!24 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "e", line: 16, scope: !10, file: !6, type: !9)
+!25 = !DILocation(line: 16, scope: !10)
+!26 = !DILocation(line: 17, scope: !27)
+!27 = distinct !DILexicalBlock(line: 16, column: 0, file: !5, scope: !10)
+!28 = !DILocation(line: 18, scope: !27)
+!29 = !DILocation(line: 19, scope: !10)
+!30 = !DILocation(line: 20, scope: !10)
 !31 = !{i32 1, !"Debug Info Version", i32 3}
diff --git a/test/DebugInfo/inline-no-debug-info.ll b/test/DebugInfo/inline-no-debug-info.ll
index 229b9b50a41..9ecd58031eb 100644
--- a/test/DebugInfo/inline-no-debug-info.ll
+++ b/test/DebugInfo/inline-no-debug-info.ll
@@ -21,11 +21,11 @@
 
 ; Debug location of the code in caller() and of the inlined code that did not
 ; have any debug location before.
-; CHECK-DAG: [[A]] = !MDLocation(line: 4, scope: !{{[0-9]+}})
+; CHECK-DAG: [[A]] = !DILocation(line: 4, scope: !{{[0-9]+}})
 
 ; Debug location of the inlined code.
-; CHECK-DAG: [[B]] = !MDLocation(line: 2, scope: !{{[0-9]+}}, inlinedAt: [[A_INL:![0-9]*]])
-; CHECK-DAG: [[A_INL]] = distinct !MDLocation(line: 4, scope: !{{[0-9]+}})
+; CHECK-DAG: [[B]] = !DILocation(line: 2, scope: !{{[0-9]+}}, inlinedAt: [[A_INL:![0-9]*]])
+; CHECK-DAG: [[A_INL]] = distinct !DILocation(line: 4, scope: !{{[0-9]+}})
 
 
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
@@ -55,16 +55,16 @@ attributes #0 = { nounwind uwtable "less-precise-fpmad"="false" "no-frame-pointe
 !llvm.module.flags = !{!8, !9}
 !llvm.ident = !{!10}
 
-!0 = !MDCompileUnit(language: DW_LANG_C99, producer: "clang version 3.5.0 (210174)", isOptimized: true, emissionKind: 2, file: !1, enums: !2, retainedTypes: !2, subprograms: !3, globals: !2, imports: !2)
-!1 = !MDFile(filename: "test.c", directory: "/code/llvm/build0")
+!0 = !DICompileUnit(language: DW_LANG_C99, producer: "clang version 3.5.0 (210174)", isOptimized: true, emissionKind: 2, file: !1, enums: !2, retainedTypes: !2, subprograms: !3, globals: !2, imports: !2)
+!1 = !DIFile(filename: "test.c", directory: "/code/llvm/build0")
 !2 = !{}
 !3 = !{!4, !7}
-!4 = !MDSubprogram(name: "caller", line: 4, isLocal: false, isDefinition: true, virtualIndex: 6, isOptimized: true, scopeLine: 4, file: !1, scope: !5, type: !6, function: void ()* @caller, variables: !2)
-!5 = !MDFile(filename: "test.c", directory: "/code/llvm/build0")
-!6 = !MDSubroutineType(types: !2)
-!7 = !MDSubprogram(name: "callee2", line: 2, isLocal: true, isDefinition: true, virtualIndex: 6, isOptimized: true, scopeLine: 2, file: !1, scope: !5, type: !6, variables: !2)
+!4 = !DISubprogram(name: "caller", line: 4, isLocal: false, isDefinition: true, virtualIndex: 6, isOptimized: true, scopeLine: 4, file: !1, scope: !5, type: !6, function: void ()* @caller, variables: !2)
+!5 = !DIFile(filename: "test.c", directory: "/code/llvm/build0")
+!6 = !DISubroutineType(types: !2)
+!7 = !DISubprogram(name: "callee2", line: 2, isLocal: true, isDefinition: true, virtualIndex: 6, isOptimized: true, scopeLine: 2, file: !1, scope: !5, type: !6, variables: !2)
 !8 = !{i32 2, !"Dwarf Version", i32 4}
 !9 = !{i32 2, !"Debug Info Version", i32 3}
 !10 = !{!"clang version 3.5.0 (210174)"}
-!11 = !MDLocation(line: 2, scope: !7)
-!12 = !MDLocation(line: 4, scope: !4)
+!11 = !DILocation(line: 2, scope: !7)
+!12 = !DILocation(line: 4, scope: !4)
diff --git a/test/DebugInfo/inline-scopes.ll b/test/DebugInfo/inline-scopes.ll
index 55c035b6d85..45324f6a4b9 100644
--- a/test/DebugInfo/inline-scopes.ll
+++ b/test/DebugInfo/inline-scopes.ll
@@ -43,7 +43,7 @@ entry:
   %b.i3 = alloca i8, align 1
   %retval.i = alloca i32, align 4
   %b.i = alloca i8, align 1
-  call void @llvm.dbg.declare(metadata i8* %b.i, metadata !16, metadata !MDExpression()), !dbg !19
+  call void @llvm.dbg.declare(metadata i8* %b.i, metadata !16, metadata !DIExpression()), !dbg !19
   %call.i = call zeroext i1 @_Z1fv(), !dbg !19
   %frombool.i = zext i1 %call.i to i8, !dbg !19
   store i8 %frombool.i, i8* %b.i, align 1, !dbg !19
@@ -61,7 +61,7 @@ if.end.i:                                         ; preds = %entry
 
 _Z2f1v.exit:                                      ; preds = %if.then.i, %if.end.i
   %1 = load i32, i32* %retval.i, !dbg !23
-  call void @llvm.dbg.declare(metadata i8* %b.i3, metadata !24, metadata !MDExpression()), !dbg !27
+  call void @llvm.dbg.declare(metadata i8* %b.i3, metadata !24, metadata !DIExpression()), !dbg !27
   %call.i4 = call zeroext i1 @_Z1fv(), !dbg !27
   %frombool.i5 = zext i1 %call.i4 to i8, !dbg !27
   store i8 %frombool.i5, i8* %b.i3, align 1, !dbg !27
@@ -95,36 +95,36 @@ attributes #2 = { "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "n
 !llvm.module.flags = !{!13, !14}
 !llvm.ident = !{!15}
 
-!0 = !MDCompileUnit(language: DW_LANG_C_plus_plus, producer: "clang version 3.5.0 ", isOptimized: false, emissionKind: 1, file: !1, enums: !2, retainedTypes: !2, subprograms: !3, globals: !2, imports: !2)
-!1 = !MDFile(filename: "inline-scopes.cpp", directory: "/tmp/dbginfo")
+!0 = !DICompileUnit(language: DW_LANG_C_plus_plus, producer: "clang version 3.5.0 ", isOptimized: false, emissionKind: 1, file: !1, enums: !2, retainedTypes: !2, subprograms: !3, globals: !2, imports: !2)
+!1 = !DIFile(filename: "inline-scopes.cpp", directory: "/tmp/dbginfo")
 !2 = !{}
 !3 = !{!4, !10, !12}
-!4 = !MDSubprogram(name: "main", line: 7, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 7, file: !5, scope: !6, type: !7, function: i32 ()* @main, variables: !2)
-!5 = !MDFile(filename: "y.cc", directory: "/tmp/dbginfo")
-!6 = !MDFile(filename: "y.cc", directory: "/tmp/dbginfo")
-!7 = !MDSubroutineType(types: !8)
+!4 = !DISubprogram(name: "main", line: 7, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 7, file: !5, scope: !6, type: !7, function: i32 ()* @main, variables: !2)
+!5 = !DIFile(filename: "y.cc", directory: "/tmp/dbginfo")
+!6 = !DIFile(filename: "y.cc", directory: "/tmp/dbginfo")
+!7 = !DISubroutineType(types: !8)
 !8 = !{!9}
-!9 = !MDBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
-!10 = !MDSubprogram(name: "f2", linkageName: "_Z2f2v", line: 8, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 8, file: !1, scope: !11, type: !7, variables: !2)
-!11 = !MDFile(filename: "inline-scopes.cpp", directory: "/tmp/dbginfo")
-!12 = !MDSubprogram(name: "f1", linkageName: "_Z2f1v", line: 2, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 2, file: !1, scope: !11, type: !7, variables: !2)
+!9 = !DIBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
+!10 = !DISubprogram(name: "f2", linkageName: "_Z2f2v", line: 8, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 8, file: !1, scope: !11, type: !7, variables: !2)
+!11 = !DIFile(filename: "inline-scopes.cpp", directory: "/tmp/dbginfo")
+!12 = !DISubprogram(name: "f1", linkageName: "_Z2f1v", line: 2, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 2, file: !1, scope: !11, type: !7, variables: !2)
 !13 = !{i32 2, !"Dwarf Version", i32 4}
 !14 = !{i32 1, !"Debug Info Version", i32 3}
 !15 = !{!"clang version 3.5.0 "}
-!16 = !MDLocalVariable(tag: DW_TAG_auto_variable, name: "b", line: 3, scope: !17, file: !11, type: !18)
-!17 = distinct !MDLexicalBlock(line: 3, column: 0, file: !1, scope: !12)
-!18 = !MDBasicType(tag: DW_TAG_base_type, name: "bool", size: 8, align: 8, encoding: DW_ATE_boolean)
-!19 = !MDLocation(line: 3, scope: !17, inlinedAt: !20)
-!20 = !MDLocation(line: 8, scope: !4)
-!21 = !MDLocation(line: 4, scope: !17, inlinedAt: !20)
-!22 = !MDLocation(line: 5, scope: !12, inlinedAt: !20)
-!23 = !MDLocation(line: 6, scope: !12, inlinedAt: !20)
-!24 = !MDLocalVariable(tag: DW_TAG_auto_variable, name: "b", line: 2, scope: !25, file: !6, type: !18)
-!25 = distinct !MDLexicalBlock(line: 2, column: 0, file: !5, scope: !26)
-!26 = !MDLexicalBlockFile(discriminator: 0, file: !5, scope: !10)
-!27 = !MDLocation(line: 2, scope: !25, inlinedAt: !28)
-!28 = !MDLocation(line: 9, scope: !4)
-!29 = !MDLocation(line: 3, scope: !25, inlinedAt: !28)
-!30 = !MDLocation(line: 4, scope: !26, inlinedAt: !28)
-!31 = !MDLocation(line: 5, scope: !26, inlinedAt: !28)
-!32 = !MDLocation(line: 10, scope: !4)
+!16 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "b", line: 3, scope: !17, file: !11, type: !18)
+!17 = distinct !DILexicalBlock(line: 3, column: 0, file: !1, scope: !12)
+!18 = !DIBasicType(tag: DW_TAG_base_type, name: "bool", size: 8, align: 8, encoding: DW_ATE_boolean)
+!19 = !DILocation(line: 3, scope: !17, inlinedAt: !20)
+!20 = !DILocation(line: 8, scope: !4)
+!21 = !DILocation(line: 4, scope: !17, inlinedAt: !20)
+!22 = !DILocation(line: 5, scope: !12, inlinedAt: !20)
+!23 = !DILocation(line: 6, scope: !12, inlinedAt: !20)
+!24 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "b", line: 2, scope: !25, file: !6, type: !18)
+!25 = distinct !DILexicalBlock(line: 2, column: 0, file: !5, scope: !26)
+!26 = !DILexicalBlockFile(discriminator: 0, file: !5, scope: !10)
+!27 = !DILocation(line: 2, scope: !25, inlinedAt: !28)
+!28 = !DILocation(line: 9, scope: !4)
+!29 = !DILocation(line: 3, scope: !25, inlinedAt: !28)
+!30 = !DILocation(line: 4, scope: !26, inlinedAt: !28)
+!31 = !DILocation(line: 5, scope: !26, inlinedAt: !28)
+!32 = !DILocation(line: 10, scope: !4)
diff --git a/test/DebugInfo/inlined-arguments.ll b/test/DebugInfo/inlined-arguments.ll
index 5d416f7dc7f..912ebb1321e 100644
--- a/test/DebugInfo/inlined-arguments.ll
+++ b/test/DebugInfo/inlined-arguments.ll
@@ -24,16 +24,16 @@
 
 ; Function Attrs: uwtable
 define void @_Z2f2v() #0 {
-  tail call void @llvm.dbg.value(metadata i32 undef, i64 0, metadata !16, metadata !MDExpression()), !dbg !18
-  tail call void @llvm.dbg.value(metadata i32 2, i64 0, metadata !20, metadata !MDExpression()), !dbg !18
+  tail call void @llvm.dbg.value(metadata i32 undef, i64 0, metadata !16, metadata !DIExpression()), !dbg !18
+  tail call void @llvm.dbg.value(metadata i32 2, i64 0, metadata !20, metadata !DIExpression()), !dbg !18
   tail call void @_Z2f3i(i32 2), !dbg !21
   ret void, !dbg !22
 }
 
 ; Function Attrs: uwtable
 define void @_Z2f1ii(i32 %x, i32 %y) #0 {
-  tail call void @llvm.dbg.value(metadata i32 %x, i64 0, metadata !13, metadata !MDExpression()), !dbg !23
-  tail call void @llvm.dbg.value(metadata i32 %y, i64 0, metadata !14, metadata !MDExpression()), !dbg !23
+  tail call void @llvm.dbg.value(metadata i32 %x, i64 0, metadata !13, metadata !DIExpression()), !dbg !23
+  tail call void @llvm.dbg.value(metadata i32 %y, i64 0, metadata !14, metadata !DIExpression()), !dbg !23
   tail call void @_Z2f3i(i32 %y), !dbg !24
   ret void, !dbg !25
 }
@@ -50,30 +50,30 @@ attributes #2 = { nounwind readnone }
 !llvm.dbg.cu = !{!0}
 !llvm.module.flags = !{!26}
 
-!0 = !MDCompileUnit(language: DW_LANG_C_plus_plus, producer: "clang version 3.4 ", isOptimized: true, emissionKind: 1, file: !1, enums: !2, retainedTypes: !2, subprograms: !3, globals: !2, imports: !2)
-!1 = !MDFile(filename: "exp.cpp", directory: "/usr/local/google/home/blaikie/dev/scratch")
+!0 = !DICompileUnit(language: DW_LANG_C_plus_plus, producer: "clang version 3.4 ", isOptimized: true, emissionKind: 1, file: !1, enums: !2, retainedTypes: !2, subprograms: !3, globals: !2, imports: !2)
+!1 = !DIFile(filename: "exp.cpp", directory: "/usr/local/google/home/blaikie/dev/scratch")
 !2 = !{}
 !3 = !{!4, !8}
-!4 = !MDSubprogram(name: "f2", linkageName: "_Z2f2v", line: 3, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: true, scopeLine: 3, file: !1, scope: !5, type: !6, function: void ()* @_Z2f2v, variables: !2)
-!5 = !MDFile(filename: "exp.cpp", directory: "/usr/local/google/home/blaikie/dev/scratch")
-!6 = !MDSubroutineType(types: !7)
+!4 = !DISubprogram(name: "f2", linkageName: "_Z2f2v", line: 3, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: true, scopeLine: 3, file: !1, scope: !5, type: !6, function: void ()* @_Z2f2v, variables: !2)
+!5 = !DIFile(filename: "exp.cpp", directory: "/usr/local/google/home/blaikie/dev/scratch")
+!6 = !DISubroutineType(types: !7)
 !7 = !{null}
-!8 = !MDSubprogram(name: "f1", linkageName: "_Z2f1ii", line: 6, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: true, scopeLine: 6, file: !1, scope: !5, type: !9, function: void (i32, i32)* @_Z2f1ii, variables: !12)
-!9 = !MDSubroutineType(types: !10)
+!8 = !DISubprogram(name: "f1", linkageName: "_Z2f1ii", line: 6, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: true, scopeLine: 6, file: !1, scope: !5, type: !9, function: void (i32, i32)* @_Z2f1ii, variables: !12)
+!9 = !DISubroutineType(types: !10)
 !10 = !{null, !11, !11}
-!11 = !MDBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
+!11 = !DIBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
 !12 = !{!13, !14}
-!13 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "x", line: 6, arg: 1, scope: !8, file: !5, type: !11)
-!14 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "y", line: 6, arg: 2, scope: !8, file: !5, type: !11)
+!13 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "x", line: 6, arg: 1, scope: !8, file: !5, type: !11)
+!14 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "y", line: 6, arg: 2, scope: !8, file: !5, type: !11)
 !15 = !{i32 undef}
-!16 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "x", line: 6, arg: 1, scope: !8, file: !5, type: !11)
-!17 = !MDLocation(line: 4, scope: !4)
-!18 = !MDLocation(line: 6, scope: !8, inlinedAt: !17)
+!16 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "x", line: 6, arg: 1, scope: !8, file: !5, type: !11)
+!17 = !DILocation(line: 4, scope: !4)
+!18 = !DILocation(line: 6, scope: !8, inlinedAt: !17)
 !19 = !{i32 2}
-!20 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "y", line: 6, arg: 2, scope: !8, file: !5, type: !11)
-!21 = !MDLocation(line: 7, scope: !8, inlinedAt: !17)
-!22 = !MDLocation(line: 5, scope: !4)
-!23 = !MDLocation(line: 6, scope: !8)
-!24 = !MDLocation(line: 7, scope: !8)
-!25 = !MDLocation(line: 8, scope: !8)
+!20 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "y", line: 6, arg: 2, scope: !8, file: !5, type: !11)
+!21 = !DILocation(line: 7, scope: !8, inlinedAt: !17)
+!22 = !DILocation(line: 5, scope: !4)
+!23 = !DILocation(line: 6, scope: !8)
+!24 = !DILocation(line: 7, scope: !8)
+!25 = !DILocation(line: 8, scope: !8)
 !26 = !{i32 1, !"Debug Info Version", i32 3}
diff --git a/test/DebugInfo/inlined-vars.ll b/test/DebugInfo/inlined-vars.ll
index 2cf59cd005a..8c782374886 100644
--- a/test/DebugInfo/inlined-vars.ll
+++ b/test/DebugInfo/inlined-vars.ll
@@ -4,8 +4,8 @@
 
 define i32 @main() uwtable {
 entry:
-  tail call void @llvm.dbg.value(metadata i32 0, i64 0, metadata !18, metadata !MDExpression()), !dbg !21
-  tail call void @llvm.dbg.value(metadata i32 0, i64 0, metadata !22, metadata !MDExpression()), !dbg !23
+  tail call void @llvm.dbg.value(metadata i32 0, i64 0, metadata !18, metadata !DIExpression()), !dbg !21
+  tail call void @llvm.dbg.value(metadata i32 0, i64 0, metadata !22, metadata !DIExpression()), !dbg !23
   tail call void @smth(i32 0), !dbg !24
   tail call void @smth(i32 0), !dbg !25
   ret i32 0, !dbg !19
@@ -18,39 +18,39 @@ declare void @llvm.dbg.value(metadata, i64, metadata, metadata) nounwind readnon
 !llvm.dbg.cu = !{!0}
 !llvm.module.flags = !{!27}
 
-!0 = !MDCompileUnit(language: DW_LANG_C_plus_plus, producer: "clang version 3.2 (trunk 159419)", isOptimized: true, emissionKind: 0, file: !26, enums: !2, retainedTypes: !2, subprograms: !3, globals: !2, imports:  !2)
+!0 = !DICompileUnit(language: DW_LANG_C_plus_plus, producer: "clang version 3.2 (trunk 159419)", isOptimized: true, emissionKind: 0, file: !26, enums: !2, retainedTypes: !2, subprograms: !3, globals: !2, imports:  !2)
 !1 = !{i32 0}
 !2 = !{}
 !3 = !{!5, !10}
-!5 = !MDSubprogram(name: "main", line: 10, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: true, scopeLine: 10, file: !26, scope: !6, type: !7, function: i32 ()* @main, variables: !2)
-!6 = !MDFile(filename: "inline-bug.cc", directory: "/tmp/dbginfo/pr13202")
-!7 = !MDSubroutineType(types: !8)
+!5 = !DISubprogram(name: "main", line: 10, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: true, scopeLine: 10, file: !26, scope: !6, type: !7, function: i32 ()* @main, variables: !2)
+!6 = !DIFile(filename: "inline-bug.cc", directory: "/tmp/dbginfo/pr13202")
+!7 = !DISubroutineType(types: !8)
 !8 = !{!9}
-!9 = !MDBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
-!10 = !MDSubprogram(name: "f", linkageName: "_ZL1fi", line: 3, isLocal: true, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: true, scopeLine: 3, file: !26, scope: !6, type: !11, variables: !13)
-!11 = !MDSubroutineType(types: !12)
+!9 = !DIBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
+!10 = !DISubprogram(name: "f", linkageName: "_ZL1fi", line: 3, isLocal: true, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: true, scopeLine: 3, file: !26, scope: !6, type: !11, variables: !13)
+!11 = !DISubroutineType(types: !12)
 !12 = !{!9, !9}
 !13 = !{!15, !16}
-!15 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "argument", line: 3, arg: 1, scope: !10, file: !6, type: !9)
+!15 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "argument", line: 3, arg: 1, scope: !10, file: !6, type: !9)
 
 ; Two DW_TAG_formal_parameter: one abstract and one inlined.
 ; ARGUMENT: {{.*Abbrev.*DW_TAG_formal_parameter}}
 ; ARGUMENT: {{.*Abbrev.*DW_TAG_formal_parameter}}
 ; ARGUMENT-NOT: {{.*Abbrev.*DW_TAG_formal_parameter}}
 
-!16 = !MDLocalVariable(tag: DW_TAG_auto_variable, name: "local", line: 4, scope: !10, file: !6, type: !9)
+!16 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "local", line: 4, scope: !10, file: !6, type: !9)
 
 ; Two DW_TAG_variable: one abstract and one inlined.
 ; VARIABLE: {{.*Abbrev.*DW_TAG_variable}}
 ; VARIABLE: {{.*Abbrev.*DW_TAG_variable}}
 ; VARIABLE-NOT: {{.*Abbrev.*DW_TAG_variable}}
 
-!18 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "argument", line: 3, arg: 1, scope: !10, file: !6, type: !9)
-!19 = !MDLocation(line: 11, column: 10, scope: !5)
-!21 = !MDLocation(line: 3, column: 25, scope: !10, inlinedAt: !19)
-!22 = !MDLocalVariable(tag: DW_TAG_auto_variable, name: "local", line: 4, scope: !10, file: !6, type: !9)
-!23 = !MDLocation(line: 4, column: 16, scope: !10, inlinedAt: !19)
-!24 = !MDLocation(line: 5, column: 3, scope: !10, inlinedAt: !19)
-!25 = !MDLocation(line: 6, column: 3, scope: !10, inlinedAt: !19)
-!26 = !MDFile(filename: "inline-bug.cc", directory: "/tmp/dbginfo/pr13202")
+!18 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "argument", line: 3, arg: 1, scope: !10, file: !6, type: !9)
+!19 = !DILocation(line: 11, column: 10, scope: !5)
+!21 = !DILocation(line: 3, column: 25, scope: !10, inlinedAt: !19)
+!22 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "local", line: 4, scope: !10, file: !6, type: !9)
+!23 = !DILocation(line: 4, column: 16, scope: !10, inlinedAt: !19)
+!24 = !DILocation(line: 5, column: 3, scope: !10, inlinedAt: !19)
+!25 = !DILocation(line: 6, column: 3, scope: !10, inlinedAt: !19)
+!26 = !DIFile(filename: "inline-bug.cc", directory: "/tmp/dbginfo/pr13202")
 !27 = !{i32 1, !"Debug Info Version", i32 3}
diff --git a/test/DebugInfo/location-verifier.ll b/test/DebugInfo/location-verifier.ll
index f49fd591578..f44a37d7fca 100644
--- a/test/DebugInfo/location-verifier.ll
+++ b/test/DebugInfo/location-verifier.ll
@@ -15,19 +15,19 @@ attributes #0 = { nounwind ssp uwtable }
 !llvm.module.flags = !{!9, !10, !11}
 !llvm.ident = !{!12}
 
-!0 = !MDCompileUnit(language: DW_LANG_C99, producer: "clang version 3.7.0 ", isOptimized: false, emissionKind: 1, file: !1, enums: !2, retainedTypes: !2, subprograms: !3, globals: !2, imports: !2)
-!1 = !MDFile(filename: "test.c", directory: "")
+!0 = !DICompileUnit(language: DW_LANG_C99, producer: "clang version 3.7.0 ", isOptimized: false, emissionKind: 1, file: !1, enums: !2, retainedTypes: !2, subprograms: !3, globals: !2, imports: !2)
+!1 = !DIFile(filename: "test.c", directory: "")
 !2 = !{}
 !3 = !{!4}
-!4 = !MDSubprogram(name: "foo", line: 1, isLocal: false, isDefinition: true, isOptimized: false, scopeLine: 1, file: !1, scope: !5, type: !6, function: i32 ()* @foo, variables: !2)
-!5 = !MDFile(filename: "test.c", directory: "")
-!6 = !MDSubroutineType(types: !7)
+!4 = !DISubprogram(name: "foo", line: 1, isLocal: false, isDefinition: true, isOptimized: false, scopeLine: 1, file: !1, scope: !5, type: !6, function: i32 ()* @foo, variables: !2)
+!5 = !DIFile(filename: "test.c", directory: "")
+!6 = !DISubroutineType(types: !7)
 !7 = !{!8}
-!8 = !MDBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
+!8 = !DIBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
 !9 = !{i32 2, !"Dwarf Version", i32 2}
 !10 = !{i32 2, !"Debug Info Version", i32 3}
 !11 = !{i32 1, !"PIC Level", i32 2}
 !12 = !{!"clang version 3.7.0 "}
-; An old-style MDLocation should not pass verify.
+; An old-style DILocation should not pass verify.
 ; CHECK: invalid !dbg metadata attachment
 !13 = !{i32 2, i32 2, !4, null}
diff --git a/test/DebugInfo/lto-comp-dir.ll b/test/DebugInfo/lto-comp-dir.ll
index 7d75c071a7d..4bafb7bdd2e 100644
--- a/test/DebugInfo/lto-comp-dir.ll
+++ b/test/DebugInfo/lto-comp-dir.ll
@@ -59,26 +59,26 @@ attributes #1 = { uwtable "less-precise-fpmad"="false" "no-frame-pointer-elim"="
 !llvm.module.flags = !{!16, !17}
 !llvm.ident = !{!18, !18}
 
-!0 = !MDCompileUnit(language: DW_LANG_C_plus_plus, producer: "clang version 3.5.0 ", isOptimized: false, emissionKind: 1, file: !1, enums: !2, retainedTypes: !2, subprograms: !3, globals: !2, imports: !2)
-!1 = !MDFile(filename: "a.cpp", directory: "/tmp/dbginfo/a")
+!0 = !DICompileUnit(language: DW_LANG_C_plus_plus, producer: "clang version 3.5.0 ", isOptimized: false, emissionKind: 1, file: !1, enums: !2, retainedTypes: !2, subprograms: !3, globals: !2, imports: !2)
+!1 = !DIFile(filename: "a.cpp", directory: "/tmp/dbginfo/a")
 !2 = !{}
 !3 = !{!4}
-!4 = !MDSubprogram(name: "func", linkageName: "_Z4funcv", line: 1, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 1, file: !1, scope: !5, type: !6, function: void ()* @_Z4funcv, variables: !2)
-!5 = !MDFile(filename: "a.cpp", directory: "/tmp/dbginfo/a")
-!6 = !MDSubroutineType(types: !7)
+!4 = !DISubprogram(name: "func", linkageName: "_Z4funcv", line: 1, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 1, file: !1, scope: !5, type: !6, function: void ()* @_Z4funcv, variables: !2)
+!5 = !DIFile(filename: "a.cpp", directory: "/tmp/dbginfo/a")
+!6 = !DISubroutineType(types: !7)
 !7 = !{null}
-!8 = !MDCompileUnit(language: DW_LANG_C_plus_plus, producer: "clang version 3.5.0 ", isOptimized: false, emissionKind: 1, file: !9, enums: !2, retainedTypes: !2, subprograms: !10, globals: !2, imports: !2)
-!9 = !MDFile(filename: "b.cpp", directory: "/tmp/dbginfo/b")
+!8 = !DICompileUnit(language: DW_LANG_C_plus_plus, producer: "clang version 3.5.0 ", isOptimized: false, emissionKind: 1, file: !9, enums: !2, retainedTypes: !2, subprograms: !10, globals: !2, imports: !2)
+!9 = !DIFile(filename: "b.cpp", directory: "/tmp/dbginfo/b")
 !10 = !{!11}
-!11 = !MDSubprogram(name: "main", line: 2, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 2, file: !9, scope: !12, type: !13, function: i32 ()* @main, variables: !2)
-!12 = !MDFile(filename: "b.cpp", directory: "/tmp/dbginfo/b")
-!13 = !MDSubroutineType(types: !14)
+!11 = !DISubprogram(name: "main", line: 2, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 2, file: !9, scope: !12, type: !13, function: i32 ()* @main, variables: !2)
+!12 = !DIFile(filename: "b.cpp", directory: "/tmp/dbginfo/b")
+!13 = !DISubroutineType(types: !14)
 !14 = !{!15}
-!15 = !MDBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
+!15 = !DIBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
 !16 = !{i32 2, !"Dwarf Version", i32 4}
 !17 = !{i32 1, !"Debug Info Version", i32 3}
 !18 = !{!"clang version 3.5.0 "}
-!19 = !MDLocation(line: 2, scope: !4)
-!20 = !MDLocation(line: 3, scope: !11)
-!21 = !MDLocation(line: 4, scope: !11)
+!19 = !DILocation(line: 2, scope: !4)
+!20 = !DILocation(line: 3, scope: !11)
+!21 = !DILocation(line: 4, scope: !11)
 
diff --git a/test/DebugInfo/member-order.ll b/test/DebugInfo/member-order.ll
index 0e1397c1113..f07e4f85cde 100644
--- a/test/DebugInfo/member-order.ll
+++ b/test/DebugInfo/member-order.ll
@@ -29,7 +29,7 @@ define void @_ZN3foo2f1Ev(%struct.foo* %this) #0 align 2 {
 entry:
   %this.addr = alloca %struct.foo*, align 8
   store %struct.foo* %this, %struct.foo** %this.addr, align 8
-  call void @llvm.dbg.declare(metadata %struct.foo** %this.addr, metadata !16, metadata !MDExpression()), !dbg !18
+  call void @llvm.dbg.declare(metadata %struct.foo** %this.addr, metadata !16, metadata !DIExpression()), !dbg !18
   %this1 = load %struct.foo*, %struct.foo** %this.addr
   ret void, !dbg !19
 }
@@ -43,24 +43,24 @@ attributes #1 = { nounwind readnone }
 !llvm.dbg.cu = !{!0}
 !llvm.module.flags = !{!15, !20}
 
-!0 = !MDCompileUnit(language: DW_LANG_C_plus_plus, producer: "clang version 3.4 ", isOptimized: false, emissionKind: 0, file: !1, enums: !2, retainedTypes: !3, subprograms: !13, globals: !2, imports: !2)
-!1 = !MDFile(filename: "member-order.cpp", directory: "/tmp/dbginfo")
+!0 = !DICompileUnit(language: DW_LANG_C_plus_plus, producer: "clang version 3.4 ", isOptimized: false, emissionKind: 0, file: !1, enums: !2, retainedTypes: !3, subprograms: !13, globals: !2, imports: !2)
+!1 = !DIFile(filename: "member-order.cpp", directory: "/tmp/dbginfo")
 !2 = !{}
 !3 = !{!4}
-!4 = !MDCompositeType(tag: DW_TAG_structure_type, name: "foo", line: 1, size: 8, align: 8, file: !1, elements: !5, identifier: "_ZTS3foo")
+!4 = !DICompositeType(tag: DW_TAG_structure_type, name: "foo", line: 1, size: 8, align: 8, file: !1, elements: !5, identifier: "_ZTS3foo")
 !5 = !{!6, !11}
-!6 = !MDSubprogram(name: "f1", linkageName: "_ZN3foo2f1Ev", line: 2, isLocal: false, isDefinition: false, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 2, file: !1, scope: !4, type: !7)
-!7 = !MDSubroutineType(types: !8)
+!6 = !DISubprogram(name: "f1", linkageName: "_ZN3foo2f1Ev", line: 2, isLocal: false, isDefinition: false, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 2, file: !1, scope: !4, type: !7)
+!7 = !DISubroutineType(types: !8)
 !8 = !{null, !9}
-!9 = !MDDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, flags: DIFlagArtificial | DIFlagObjectPointer, baseType: !"_ZTS3foo")
+!9 = !DIDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, flags: DIFlagArtificial | DIFlagObjectPointer, baseType: !"_ZTS3foo")
 !10 = !{i32 786468}
-!11 = !MDSubprogram(name: "f2", linkageName: "_ZN3foo2f2Ev", line: 3, isLocal: false, isDefinition: false, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 3, file: !1, scope: !4, type: !7)
+!11 = !DISubprogram(name: "f2", linkageName: "_ZN3foo2f2Ev", line: 3, isLocal: false, isDefinition: false, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 3, file: !1, scope: !4, type: !7)
 !12 = !{i32 786468}
 !13 = !{!14}
-!14 = !MDSubprogram(name: "f1", linkageName: "_ZN3foo2f1Ev", line: 6, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 6, file: !1, scope: null, type: !7, function: void (%struct.foo*)* @_ZN3foo2f1Ev, declaration: !6, variables: !2)
+!14 = !DISubprogram(name: "f1", linkageName: "_ZN3foo2f1Ev", line: 6, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 6, file: !1, scope: null, type: !7, function: void (%struct.foo*)* @_ZN3foo2f1Ev, declaration: !6, variables: !2)
 !15 = !{i32 2, !"Dwarf Version", i32 4}
-!16 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "this", arg: 1, flags: DIFlagArtificial | DIFlagObjectPointer, scope: !14, type: !17)
-!17 = !MDDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, baseType: !"_ZTS3foo")
-!18 = !MDLocation(line: 0, scope: !14)
-!19 = !MDLocation(line: 7, scope: !14)
+!16 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "this", arg: 1, flags: DIFlagArtificial | DIFlagObjectPointer, scope: !14, type: !17)
+!17 = !DIDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, baseType: !"_ZTS3foo")
+!18 = !DILocation(line: 0, scope: !14)
+!19 = !DILocation(line: 7, scope: !14)
 !20 = !{i32 1, !"Debug Info Version", i32 3}
diff --git a/test/DebugInfo/member-pointers.ll b/test/DebugInfo/member-pointers.ll
index a3c438792d6..0cf9ead8421 100644
--- a/test/DebugInfo/member-pointers.ll
+++ b/test/DebugInfo/member-pointers.ll
@@ -23,18 +23,18 @@
 !llvm.dbg.cu = !{!0}
 !llvm.module.flags = !{!16}
 
-!0 = !MDCompileUnit(language: DW_LANG_C_plus_plus, producer: "clang version 3.3 ", isOptimized: false, emissionKind: 0, file: !15, enums: !1, retainedTypes: !1, subprograms: !1, globals: !3, imports:  !1)
+!0 = !DICompileUnit(language: DW_LANG_C_plus_plus, producer: "clang version 3.3 ", isOptimized: false, emissionKind: 0, file: !15, enums: !1, retainedTypes: !1, subprograms: !1, globals: !3, imports:  !1)
 !1 = !{}
 !3 = !{!5, !10}
-!5 = !MDGlobalVariable(name: "x", line: 4, isLocal: false, isDefinition: true, scope: null, file: !6, type: !7, variable: i64* @x)
-!6 = !MDFile(filename: "simple.cpp", directory: "/home/blaikie/Development/scratch")
-!7 = !MDDerivedType(tag: DW_TAG_ptr_to_member_type, baseType: !8, extraData: !9)
-!8 = !MDBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
-!9 = !MDCompositeType(tag: DW_TAG_structure_type, name: "S", line: 1, size: 8, align: 8, file: !15, elements: !1)
-!10 = !MDGlobalVariable(name: "y", line: 5, isLocal: false, isDefinition: true, scope: null, file: !6, type: !11, variable: { i64, i64 }* @y)
-!11 = !MDDerivedType(tag: DW_TAG_ptr_to_member_type, baseType: !12, extraData: !9)
-!12 = !MDSubroutineType(types: !13)
+!5 = !DIGlobalVariable(name: "x", line: 4, isLocal: false, isDefinition: true, scope: null, file: !6, type: !7, variable: i64* @x)
+!6 = !DIFile(filename: "simple.cpp", directory: "/home/blaikie/Development/scratch")
+!7 = !DIDerivedType(tag: DW_TAG_ptr_to_member_type, baseType: !8, extraData: !9)
+!8 = !DIBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
+!9 = !DICompositeType(tag: DW_TAG_structure_type, name: "S", line: 1, size: 8, align: 8, file: !15, elements: !1)
+!10 = !DIGlobalVariable(name: "y", line: 5, isLocal: false, isDefinition: true, scope: null, file: !6, type: !11, variable: { i64, i64 }* @y)
+!11 = !DIDerivedType(tag: DW_TAG_ptr_to_member_type, baseType: !12, extraData: !9)
+!12 = !DISubroutineType(types: !13)
 !13 = !{null, !14, !8}
-!14 = !MDDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, flags: DIFlagArtificial | DIFlagObjectPointer, baseType: !9)
-!15 = !MDFile(filename: "simple.cpp", directory: "/home/blaikie/Development/scratch")
+!14 = !DIDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, flags: DIFlagArtificial | DIFlagObjectPointer, baseType: !9)
+!15 = !DIFile(filename: "simple.cpp", directory: "/home/blaikie/Development/scratch")
 !16 = !{i32 1, !"Debug Info Version", i32 3}
diff --git a/test/DebugInfo/missing-abstract-variable.ll b/test/DebugInfo/missing-abstract-variable.ll
index a26afb73029..8a576c2bfd3 100644
--- a/test/DebugInfo/missing-abstract-variable.ll
+++ b/test/DebugInfo/missing-abstract-variable.ll
@@ -99,7 +99,7 @@
 ; Function Attrs: uwtable
 define void @_Z1bv() #0 {
 entry:
-  tail call void @llvm.dbg.value(metadata i1 false, i64 0, metadata !25, metadata !MDExpression()), !dbg !27
+  tail call void @llvm.dbg.value(metadata i1 false, i64 0, metadata !25, metadata !DIExpression()), !dbg !27
   tail call void @_Z1fi(i32 0), !dbg !28
   ret void, !dbg !29
 }
@@ -107,13 +107,13 @@ entry:
 ; Function Attrs: uwtable
 define void @_Z1ab(i1 zeroext %u) #0 {
 entry:
-  tail call void @llvm.dbg.value(metadata i1 %u, i64 0, metadata !13, metadata !MDExpression()), !dbg !30
-  tail call void @llvm.dbg.value(metadata i1 %u, i64 0, metadata !31, metadata !MDExpression()), !dbg !33
+  tail call void @llvm.dbg.value(metadata i1 %u, i64 0, metadata !13, metadata !DIExpression()), !dbg !30
+  tail call void @llvm.dbg.value(metadata i1 %u, i64 0, metadata !31, metadata !DIExpression()), !dbg !33
   br i1 %u, label %if.then.i, label %_Z1xb.exit, !dbg !34
 
 if.then.i:                                        ; preds = %entry
   %0 = load i32, i32* @t, align 4, !dbg !35, !tbaa !36
-  tail call void @llvm.dbg.value(metadata i32 %0, i64 0, metadata !40, metadata !MDExpression()), !dbg !35
+  tail call void @llvm.dbg.value(metadata i32 %0, i64 0, metadata !40, metadata !DIExpression()), !dbg !35
   tail call void @_Z1fi(i32 %0), !dbg !41
   br label %_Z1xb.exit, !dbg !42
 
@@ -135,48 +135,48 @@ attributes #2 = { nounwind readnone }
 !llvm.module.flags = !{!21, !22}
 !llvm.ident = !{!23}
 
-!0 = !MDCompileUnit(language: DW_LANG_C_plus_plus, producer: "clang version 3.5.0 ", isOptimized: true, emissionKind: 1, file: !1, enums: !2, retainedTypes: !2, subprograms: !3, globals: !2, imports: !2)
-!1 = !MDFile(filename: "missing-abstract-variables.cc", directory: "/tmp/dbginfo")
+!0 = !DICompileUnit(language: DW_LANG_C_plus_plus, producer: "clang version 3.5.0 ", isOptimized: true, emissionKind: 1, file: !1, enums: !2, retainedTypes: !2, subprograms: !3, globals: !2, imports: !2)
+!1 = !DIFile(filename: "missing-abstract-variables.cc", directory: "/tmp/dbginfo")
 !2 = !{}
 !3 = !{!4, !8, !14}
-!4 = !MDSubprogram(name: "b", linkageName: "_Z1bv", line: 13, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: true, scopeLine: 13, file: !1, scope: !5, type: !6, function: void ()* @_Z1bv, variables: !2)
-!5 = !MDFile(filename: "missing-abstract-variables.cc", directory: "/tmp/dbginfo")
-!6 = !MDSubroutineType(types: !7)
+!4 = !DISubprogram(name: "b", linkageName: "_Z1bv", line: 13, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: true, scopeLine: 13, file: !1, scope: !5, type: !6, function: void ()* @_Z1bv, variables: !2)
+!5 = !DIFile(filename: "missing-abstract-variables.cc", directory: "/tmp/dbginfo")
+!6 = !DISubroutineType(types: !7)
 !7 = !{null}
-!8 = !MDSubprogram(name: "a", linkageName: "_Z1ab", line: 17, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: true, scopeLine: 17, file: !1, scope: !5, type: !9, function: void (i1)* @_Z1ab, variables: !12)
-!9 = !MDSubroutineType(types: !10)
+!8 = !DISubprogram(name: "a", linkageName: "_Z1ab", line: 17, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: true, scopeLine: 17, file: !1, scope: !5, type: !9, function: void (i1)* @_Z1ab, variables: !12)
+!9 = !DISubroutineType(types: !10)
 !10 = !{null, !11}
-!11 = !MDBasicType(tag: DW_TAG_base_type, name: "bool", size: 8, align: 8, encoding: DW_ATE_boolean)
+!11 = !DIBasicType(tag: DW_TAG_base_type, name: "bool", size: 8, align: 8, encoding: DW_ATE_boolean)
 !12 = !{!13}
-!13 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "u", line: 17, arg: 1, scope: !8, file: !5, type: !11)
-!14 = !MDSubprogram(name: "x", linkageName: "_Z1xb", line: 5, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: true, scopeLine: 5, file: !1, scope: !5, type: !9, variables: !15)
+!13 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "u", line: 17, arg: 1, scope: !8, file: !5, type: !11)
+!14 = !DISubprogram(name: "x", linkageName: "_Z1xb", line: 5, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: true, scopeLine: 5, file: !1, scope: !5, type: !9, variables: !15)
 !15 = !{!16, !17}
-!16 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "b", line: 5, arg: 1, scope: !14, file: !5, type: !11)
-!17 = !MDLocalVariable(tag: DW_TAG_auto_variable, name: "s", line: 7, scope: !18, file: !5, type: !20)
-!18 = distinct !MDLexicalBlock(line: 6, column: 0, file: !1, scope: !19)
-!19 = distinct !MDLexicalBlock(line: 6, column: 0, file: !1, scope: !14)
-!20 = !MDBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
+!16 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "b", line: 5, arg: 1, scope: !14, file: !5, type: !11)
+!17 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "s", line: 7, scope: !18, file: !5, type: !20)
+!18 = distinct !DILexicalBlock(line: 6, column: 0, file: !1, scope: !19)
+!19 = distinct !DILexicalBlock(line: 6, column: 0, file: !1, scope: !14)
+!20 = !DIBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
 !21 = !{i32 2, !"Dwarf Version", i32 4}
 !22 = !{i32 2, !"Debug Info Version", i32 3}
 !23 = !{!"clang version 3.5.0 "}
 !24 = !{i1 false}
-!25 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "b", line: 5, arg: 1, scope: !14, file: !5, type: !11)
-!26 = !MDLocation(line: 14, scope: !4)
-!27 = !MDLocation(line: 5, scope: !14, inlinedAt: !26)
-!28 = !MDLocation(line: 10, scope: !14, inlinedAt: !26)
-!29 = !MDLocation(line: 15, scope: !4)
-!30 = !MDLocation(line: 17, scope: !8)
-!31 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "b", line: 5, arg: 1, scope: !14, file: !5, type: !11)
-!32 = !MDLocation(line: 18, scope: !8)
-!33 = !MDLocation(line: 5, scope: !14, inlinedAt: !32)
-!34 = !MDLocation(line: 6, scope: !19, inlinedAt: !32)
-!35 = !MDLocation(line: 7, scope: !18, inlinedAt: !32)
+!25 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "b", line: 5, arg: 1, scope: !14, file: !5, type: !11)
+!26 = !DILocation(line: 14, scope: !4)
+!27 = !DILocation(line: 5, scope: !14, inlinedAt: !26)
+!28 = !DILocation(line: 10, scope: !14, inlinedAt: !26)
+!29 = !DILocation(line: 15, scope: !4)
+!30 = !DILocation(line: 17, scope: !8)
+!31 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "b", line: 5, arg: 1, scope: !14, file: !5, type: !11)
+!32 = !DILocation(line: 18, scope: !8)
+!33 = !DILocation(line: 5, scope: !14, inlinedAt: !32)
+!34 = !DILocation(line: 6, scope: !19, inlinedAt: !32)
+!35 = !DILocation(line: 7, scope: !18, inlinedAt: !32)
 !36 = !{!37, !37, i64 0}
 !37 = !{!"int", !38, i64 0}
 !38 = !{!"omnipotent char", !39, i64 0}
 !39 = !{!"Simple C/C++ TBAA"}
-!40 = !MDLocalVariable(tag: DW_TAG_auto_variable, name: "s", line: 7, scope: !18, file: !5, type: !20)
-!41 = !MDLocation(line: 8, scope: !18, inlinedAt: !32)
-!42 = !MDLocation(line: 9, scope: !18, inlinedAt: !32)
-!43 = !MDLocation(line: 10, scope: !14, inlinedAt: !32)
-!44 = !MDLocation(line: 19, scope: !8)
+!40 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "s", line: 7, scope: !18, file: !5, type: !20)
+!41 = !DILocation(line: 8, scope: !18, inlinedAt: !32)
+!42 = !DILocation(line: 9, scope: !18, inlinedAt: !32)
+!43 = !DILocation(line: 10, scope: !14, inlinedAt: !32)
+!44 = !DILocation(line: 19, scope: !8)
diff --git a/test/DebugInfo/multiline.ll b/test/DebugInfo/multiline.ll
index 1fa5979fc0e..e6b43239fda 100644
--- a/test/DebugInfo/multiline.ll
+++ b/test/DebugInfo/multiline.ll
@@ -62,21 +62,21 @@ attributes #1 = { "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "n
 !llvm.module.flags = !{!8, !9}
 !llvm.ident = !{!10}
 
-!0 = !MDCompileUnit(language: DW_LANG_C99, producer: "clang version 3.6.0 (trunk 225000) (llvm/trunk 224999)", isOptimized: false, emissionKind: 1, file: !1, enums: !2, retainedTypes: !2, subprograms: !3, globals: !2, imports: !2)
-!1 = !MDFile(filename: "multiline.c", directory: "/tmp/dbginfo")
+!0 = !DICompileUnit(language: DW_LANG_C99, producer: "clang version 3.6.0 (trunk 225000) (llvm/trunk 224999)", isOptimized: false, emissionKind: 1, file: !1, enums: !2, retainedTypes: !2, subprograms: !3, globals: !2, imports: !2)
+!1 = !DIFile(filename: "multiline.c", directory: "/tmp/dbginfo")
 !2 = !{}
 !3 = !{!4}
-!4 = !MDSubprogram(name: "f2", line: 2, isLocal: false, isDefinition: true, isOptimized: false, scopeLine: 2, file: !1, scope: !5, type: !6, function: void ()* @f2, variables: !2)
-!5 = !MDFile(filename: "multiline.c", directory: "/tmp/dbginfo")
-!6 = !MDSubroutineType(types: !7)
+!4 = !DISubprogram(name: "f2", line: 2, isLocal: false, isDefinition: true, isOptimized: false, scopeLine: 2, file: !1, scope: !5, type: !6, function: void ()* @f2, variables: !2)
+!5 = !DIFile(filename: "multiline.c", directory: "/tmp/dbginfo")
+!6 = !DISubroutineType(types: !7)
 !7 = !{null}
 !8 = !{i32 2, !"Dwarf Version", i32 4}
 !9 = !{i32 2, !"Debug Info Version", i32 3}
 !10 = !{!"clang version 3.6.0 (trunk 225000) (llvm/trunk 224999)"}
-!11 = !MDLocation(line: 3, column: 3, scope: !4)
-!12 = !MDLocation(line: 3, column: 9, scope: !4)
-!13 = !MDLocation(line: 3, column: 15, scope: !4)
-!14 = !MDLocation(line: 4, column: 3, scope: !4)
-!15 = !MDLocation(line: 4, column: 9, scope: !4)
-!16 = !MDLocation(line: 4, column: 15, scope: !4)
-!17 = !MDLocation(line: 5, column: 1, scope: !4)
+!11 = !DILocation(line: 3, column: 3, scope: !4)
+!12 = !DILocation(line: 3, column: 9, scope: !4)
+!13 = !DILocation(line: 3, column: 15, scope: !4)
+!14 = !DILocation(line: 4, column: 3, scope: !4)
+!15 = !DILocation(line: 4, column: 9, scope: !4)
+!16 = !DILocation(line: 4, column: 15, scope: !4)
+!17 = !DILocation(line: 5, column: 1, scope: !4)
diff --git a/test/DebugInfo/namespace.ll b/test/DebugInfo/namespace.ll
index fae75952d03..85ef7356205 100644
--- a/test/DebugInfo/namespace.ll
+++ b/test/DebugInfo/namespace.ll
@@ -288,79 +288,79 @@ attributes #1 = { nounwind readnone }
 !llvm.module.flags = !{!57, !58}
 !llvm.ident = !{!59}
 
-!0 = !MDCompileUnit(language: DW_LANG_C_plus_plus, producer: "clang version 3.6.0 ", isOptimized: false, emissionKind: 1, file: !1, enums: !2, retainedTypes: !3, subprograms: !9, globals: !30, imports: !33)
-!1 = !MDFile(filename: "debug-info-namespace.cpp", directory: "/tmp")
+!0 = !DICompileUnit(language: DW_LANG_C_plus_plus, producer: "clang version 3.6.0 ", isOptimized: false, emissionKind: 1, file: !1, enums: !2, retainedTypes: !3, subprograms: !9, globals: !30, imports: !33)
+!1 = !DIFile(filename: "debug-info-namespace.cpp", directory: "/tmp")
 !2 = !{}
 !3 = !{!4, !8}
-!4 = !MDCompositeType(tag: DW_TAG_structure_type, name: "foo", line: 5, flags: DIFlagFwdDecl, file: !5, scope: !6, identifier: "_ZTSN1A1B3fooE")
-!5 = !MDFile(filename: "foo.cpp", directory: "/tmp")
-!6 = !MDNamespace(name: "B", line: 1, file: !5, scope: !7)
-!7 = !MDNamespace(name: "A", line: 5, file: !1, scope: null)
-!8 = !MDCompositeType(tag: DW_TAG_structure_type, name: "bar", line: 6, size: 8, align: 8, file: !5, scope: !6, elements: !2, identifier: "_ZTSN1A1B3barE")
+!4 = !DICompositeType(tag: DW_TAG_structure_type, name: "foo", line: 5, flags: DIFlagFwdDecl, file: !5, scope: !6, identifier: "_ZTSN1A1B3fooE")
+!5 = !DIFile(filename: "foo.cpp", directory: "/tmp")
+!6 = !DINamespace(name: "B", line: 1, file: !5, scope: !7)
+!7 = !DINamespace(name: "A", line: 5, file: !1, scope: null)
+!8 = !DICompositeType(tag: DW_TAG_structure_type, name: "bar", line: 6, size: 8, align: 8, file: !5, scope: !6, elements: !2, identifier: "_ZTSN1A1B3barE")
 !9 = !{!10, !14, !17, !21, !25, !26, !27}
-!10 = !MDSubprogram(name: "f1", linkageName: "_ZN1A1B2f1Ev", line: 3, isLocal: false, isDefinition: true, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 3, file: !5, scope: !6, type: !11, function: i32 ()* @_ZN1A1B2f1Ev, variables: !2)
-!11 = !MDSubroutineType(types: !12)
+!10 = !DISubprogram(name: "f1", linkageName: "_ZN1A1B2f1Ev", line: 3, isLocal: false, isDefinition: true, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 3, file: !5, scope: !6, type: !11, function: i32 ()* @_ZN1A1B2f1Ev, variables: !2)
+!11 = !DISubroutineType(types: !12)
 !12 = !{!13}
-!13 = !MDBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
-!14 = !MDSubprogram(name: "f1", linkageName: "_ZN1A1B2f1Ei", line: 4, isLocal: false, isDefinition: true, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 4, file: !5, scope: !6, type: !15, function: void (i32)* @_ZN1A1B2f1Ei, variables: !2)
-!15 = !MDSubroutineType(types: !16)
+!13 = !DIBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
+!14 = !DISubprogram(name: "f1", linkageName: "_ZN1A1B2f1Ei", line: 4, isLocal: false, isDefinition: true, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 4, file: !5, scope: !6, type: !15, function: void (i32)* @_ZN1A1B2f1Ei, variables: !2)
+!15 = !DISubroutineType(types: !16)
 !16 = !{null, !13}
-!17 = !MDSubprogram(name: "__cxx_global_var_init", line: 20, isLocal: true, isDefinition: true, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 20, file: !5, scope: !18, type: !19, function: void ()* @__cxx_global_var_init, variables: !2)
-!18 = !MDFile(filename: "foo.cpp", directory: "/tmp")
-!19 = !MDSubroutineType(types: !20)
+!17 = !DISubprogram(name: "__cxx_global_var_init", line: 20, isLocal: true, isDefinition: true, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 20, file: !5, scope: !18, type: !19, function: void ()* @__cxx_global_var_init, variables: !2)
+!18 = !DIFile(filename: "foo.cpp", directory: "/tmp")
+!19 = !DISubroutineType(types: !20)
 !20 = !{null}
-!21 = !MDSubprogram(name: "func", linkageName: "_Z4funcb", line: 21, isLocal: false, isDefinition: true, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 21, file: !5, scope: !18, type: !22, function: i32 (i1)* @_Z4funcb, variables: !2)
-!22 = !MDSubroutineType(types: !23)
+!21 = !DISubprogram(name: "func", linkageName: "_Z4funcb", line: 21, isLocal: false, isDefinition: true, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 21, file: !5, scope: !18, type: !22, function: i32 (i1)* @_Z4funcb, variables: !2)
+!22 = !DISubroutineType(types: !23)
 !23 = !{!13, !24}
-!24 = !MDBasicType(tag: DW_TAG_base_type, name: "bool", size: 8, align: 8, encoding: DW_ATE_boolean)
-!25 = !MDSubprogram(name: "__cxx_global_var_init1", line: 44, isLocal: true, isDefinition: true, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 44, file: !5, scope: !18, type: !19, function: void ()* @__cxx_global_var_init1, variables: !2)
-!26 = !MDSubprogram(name: "func_fwd", linkageName: "_ZN1A1B8func_fwdEv", line: 47, isLocal: false, isDefinition: true, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 47, file: !5, scope: !6, type: !19, function: void ()* @_ZN1A1B8func_fwdEv, variables: !2)
-!27 = !MDSubprogram(name: "", linkageName: "_GLOBAL__sub_I_debug_info_namespace.cpp", isLocal: true, isDefinition: true, flags: DIFlagArtificial, isOptimized: false, file: !1, scope: !28, type: !29, function: void ()* @_GLOBAL__sub_I_debug_info_namespace.cpp, variables: !2)
-!28 = !MDFile(filename: "debug-info-namespace.cpp", directory: "/tmp")
-!29 = !MDSubroutineType(types: !2)
+!24 = !DIBasicType(tag: DW_TAG_base_type, name: "bool", size: 8, align: 8, encoding: DW_ATE_boolean)
+!25 = !DISubprogram(name: "__cxx_global_var_init1", line: 44, isLocal: true, isDefinition: true, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 44, file: !5, scope: !18, type: !19, function: void ()* @__cxx_global_var_init1, variables: !2)
+!26 = !DISubprogram(name: "func_fwd", linkageName: "_ZN1A1B8func_fwdEv", line: 47, isLocal: false, isDefinition: true, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 47, file: !5, scope: !6, type: !19, function: void ()* @_ZN1A1B8func_fwdEv, variables: !2)
+!27 = !DISubprogram(name: "", linkageName: "_GLOBAL__sub_I_debug_info_namespace.cpp", isLocal: true, isDefinition: true, flags: DIFlagArtificial, isOptimized: false, file: !1, scope: !28, type: !29, function: void ()* @_GLOBAL__sub_I_debug_info_namespace.cpp, variables: !2)
+!28 = !DIFile(filename: "debug-info-namespace.cpp", directory: "/tmp")
+!29 = !DISubroutineType(types: !2)
 !30 = !{!31, !32}
-!31 = !MDGlobalVariable(name: "i", linkageName: "_ZN1A1B1iE", line: 20, isLocal: false, isDefinition: true, scope: !6, file: !18, type: !13, variable: i32* @_ZN1A1B1iE)
-!32 = !MDGlobalVariable(name: "var_fwd", linkageName: "_ZN1A1B7var_fwdE", line: 44, isLocal: false, isDefinition: true, scope: !6, file: !18, type: !13, variable: i32* @_ZN1A1B7var_fwdE)
+!31 = !DIGlobalVariable(name: "i", linkageName: "_ZN1A1B1iE", line: 20, isLocal: false, isDefinition: true, scope: !6, file: !18, type: !13, variable: i32* @_ZN1A1B1iE)
+!32 = !DIGlobalVariable(name: "var_fwd", linkageName: "_ZN1A1B7var_fwdE", line: 44, isLocal: false, isDefinition: true, scope: !6, file: !18, type: !13, variable: i32* @_ZN1A1B7var_fwdE)
 !33 = !{!34, !35, !36, !37, !40, !41, !42, !43, !44, !45, !47, !48, !49, !51, !54, !55, !56}
-!34 = !MDImportedEntity(tag: DW_TAG_imported_module, line: 15, scope: !7, entity: !6)
-!35 = !MDImportedEntity(tag: DW_TAG_imported_module, line: 18, scope: !0, entity: !7)
-!36 = !MDImportedEntity(tag: DW_TAG_imported_declaration, line: 19, name: "E", scope: !0, entity: !7)
-!37 = !MDImportedEntity(tag: DW_TAG_imported_module, line: 23, scope: !38, entity: !6)
-!38 = distinct !MDLexicalBlock(line: 22, column: 10, file: !5, scope: !39)
-!39 = distinct !MDLexicalBlock(line: 22, column: 7, file: !5, scope: !21)
-!40 = !MDImportedEntity(tag: DW_TAG_imported_module, line: 26, scope: !21, entity: !7)
-!41 = !MDImportedEntity(tag: DW_TAG_imported_declaration, line: 27, scope: !21, entity: !"_ZTSN1A1B3fooE")
-!42 = !MDImportedEntity(tag: DW_TAG_imported_declaration, line: 28, scope: !21, entity: !"_ZTSN1A1B3barE")
-!43 = !MDImportedEntity(tag: DW_TAG_imported_declaration, line: 29, scope: !21, entity: !14)
-!44 = !MDImportedEntity(tag: DW_TAG_imported_declaration, line: 30, scope: !21, entity: !31)
-!45 = !MDImportedEntity(tag: DW_TAG_imported_declaration, line: 31, scope: !21, entity: !46)
-!46 = !MDDerivedType(tag: DW_TAG_typedef, name: "baz", line: 7, file: !5, scope: !6, baseType: !"_ZTSN1A1B3barE")
-!47 = !MDImportedEntity(tag: DW_TAG_imported_declaration, line: 32, name: "X", scope: !21, entity: !7)
-!48 = !MDImportedEntity(tag: DW_TAG_imported_declaration, line: 33, name: "Y", scope: !21, entity: !47)
-!49 = !MDImportedEntity(tag: DW_TAG_imported_declaration, line: 34, scope: !21, entity: !50)
-!50 = !MDGlobalVariable(name: "var_decl", linkageName: "_ZN1A1B8var_declE", line: 8, isLocal: false, isDefinition: false, scope: !6, file: !18, type: !13)
-!51 = !MDImportedEntity(tag: DW_TAG_imported_declaration, line: 35, scope: !21, entity: !52)
-!52 = !MDSubprogram(name: "func_decl", linkageName: "_ZN1A1B9func_declEv", line: 9, isLocal: false, isDefinition: false, flags: DIFlagPrototyped, isOptimized: false, file: !5, scope: !6, type: !19, variables: !53)
+!34 = !DIImportedEntity(tag: DW_TAG_imported_module, line: 15, scope: !7, entity: !6)
+!35 = !DIImportedEntity(tag: DW_TAG_imported_module, line: 18, scope: !0, entity: !7)
+!36 = !DIImportedEntity(tag: DW_TAG_imported_declaration, line: 19, name: "E", scope: !0, entity: !7)
+!37 = !DIImportedEntity(tag: DW_TAG_imported_module, line: 23, scope: !38, entity: !6)
+!38 = distinct !DILexicalBlock(line: 22, column: 10, file: !5, scope: !39)
+!39 = distinct !DILexicalBlock(line: 22, column: 7, file: !5, scope: !21)
+!40 = !DIImportedEntity(tag: DW_TAG_imported_module, line: 26, scope: !21, entity: !7)
+!41 = !DIImportedEntity(tag: DW_TAG_imported_declaration, line: 27, scope: !21, entity: !"_ZTSN1A1B3fooE")
+!42 = !DIImportedEntity(tag: DW_TAG_imported_declaration, line: 28, scope: !21, entity: !"_ZTSN1A1B3barE")
+!43 = !DIImportedEntity(tag: DW_TAG_imported_declaration, line: 29, scope: !21, entity: !14)
+!44 = !DIImportedEntity(tag: DW_TAG_imported_declaration, line: 30, scope: !21, entity: !31)
+!45 = !DIImportedEntity(tag: DW_TAG_imported_declaration, line: 31, scope: !21, entity: !46)
+!46 = !DIDerivedType(tag: DW_TAG_typedef, name: "baz", line: 7, file: !5, scope: !6, baseType: !"_ZTSN1A1B3barE")
+!47 = !DIImportedEntity(tag: DW_TAG_imported_declaration, line: 32, name: "X", scope: !21, entity: !7)
+!48 = !DIImportedEntity(tag: DW_TAG_imported_declaration, line: 33, name: "Y", scope: !21, entity: !47)
+!49 = !DIImportedEntity(tag: DW_TAG_imported_declaration, line: 34, scope: !21, entity: !50)
+!50 = !DIGlobalVariable(name: "var_decl", linkageName: "_ZN1A1B8var_declE", line: 8, isLocal: false, isDefinition: false, scope: !6, file: !18, type: !13)
+!51 = !DIImportedEntity(tag: DW_TAG_imported_declaration, line: 35, scope: !21, entity: !52)
+!52 = !DISubprogram(name: "func_decl", linkageName: "_ZN1A1B9func_declEv", line: 9, isLocal: false, isDefinition: false, flags: DIFlagPrototyped, isOptimized: false, file: !5, scope: !6, type: !19, variables: !53)
 !53 = !{} ; previously: invalid DW_TAG_base_type
-!54 = !MDImportedEntity(tag: DW_TAG_imported_declaration, line: 36, scope: !21, entity: !32)
-!55 = !MDImportedEntity(tag: DW_TAG_imported_declaration, line: 37, scope: !21, entity: !26)
-!56 = !MDImportedEntity(tag: DW_TAG_imported_declaration, line: 42, scope: !7, entity: !31)
+!54 = !DIImportedEntity(tag: DW_TAG_imported_declaration, line: 36, scope: !21, entity: !32)
+!55 = !DIImportedEntity(tag: DW_TAG_imported_declaration, line: 37, scope: !21, entity: !26)
+!56 = !DIImportedEntity(tag: DW_TAG_imported_declaration, line: 42, scope: !7, entity: !31)
 !57 = !{i32 2, !"Dwarf Version", i32 2}
 !58 = !{i32 2, !"Debug Info Version", i32 3}
 !59 = !{!"clang version 3.6.0 "}
-!60 = !MDLocation(line: 3, column: 12, scope: !10)
-!61 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "", line: 4, arg: 1, scope: !14, file: !18, type: !13)
-!62 = !MDExpression()
-!63 = !MDLocation(line: 4, column: 12, scope: !14)
-!64 = !MDLocation(line: 4, column: 16, scope: !14)
-!65 = !MDLocation(line: 20, column: 12, scope: !17)
-!66 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "b", line: 21, arg: 1, scope: !21, file: !18, type: !24)
-!67 = !MDLocation(line: 21, column: 15, scope: !21)
-!68 = !MDLocation(line: 22, column: 7, scope: !21)
-!69 = !MDLocation(line: 24, column: 5, scope: !38)
-!70 = !MDLocation(line: 38, column: 3, scope: !21)
-!71 = !MDLocation(line: 39, column: 1, scope: !21)
-!72 = !MDLocation(line: 44, column: 15, scope: !25)
-!73 = !MDLocation(line: 47, column: 21, scope: !26)
-!74 = !MDLocation(line: 0, scope: !75)
-!75 = !MDLexicalBlockFile(discriminator: 0, file: !5, scope: !27)
+!60 = !DILocation(line: 3, column: 12, scope: !10)
+!61 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "", line: 4, arg: 1, scope: !14, file: !18, type: !13)
+!62 = !DIExpression()
+!63 = !DILocation(line: 4, column: 12, scope: !14)
+!64 = !DILocation(line: 4, column: 16, scope: !14)
+!65 = !DILocation(line: 20, column: 12, scope: !17)
+!66 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "b", line: 21, arg: 1, scope: !21, file: !18, type: !24)
+!67 = !DILocation(line: 21, column: 15, scope: !21)
+!68 = !DILocation(line: 22, column: 7, scope: !21)
+!69 = !DILocation(line: 24, column: 5, scope: !38)
+!70 = !DILocation(line: 38, column: 3, scope: !21)
+!71 = !DILocation(line: 39, column: 1, scope: !21)
+!72 = !DILocation(line: 44, column: 15, scope: !25)
+!73 = !DILocation(line: 47, column: 21, scope: !26)
+!74 = !DILocation(line: 0, scope: !75)
+!75 = !DILexicalBlockFile(discriminator: 0, file: !5, scope: !27)
diff --git a/test/DebugInfo/namespace_function_definition.ll b/test/DebugInfo/namespace_function_definition.ll
index 1558636ff01..44340635484 100644
--- a/test/DebugInfo/namespace_function_definition.ll
+++ b/test/DebugInfo/namespace_function_definition.ll
@@ -30,15 +30,15 @@ attributes #0 = { nounwind uwtable "less-precise-fpmad"="false" "no-frame-pointe
 !llvm.module.flags = !{!8, !9}
 !llvm.ident = !{!10}
 
-!0 = !MDCompileUnit(language: DW_LANG_C_plus_plus, producer: "clang version 3.5.0 ", isOptimized: false, emissionKind: 1, file: !1, enums: !2, retainedTypes: !2, subprograms: !3, globals: !2, imports: !2)
-!1 = !MDFile(filename: "namespace_function_definition.cpp", directory: "/tmp/dbginfo")
+!0 = !DICompileUnit(language: DW_LANG_C_plus_plus, producer: "clang version 3.5.0 ", isOptimized: false, emissionKind: 1, file: !1, enums: !2, retainedTypes: !2, subprograms: !3, globals: !2, imports: !2)
+!1 = !DIFile(filename: "namespace_function_definition.cpp", directory: "/tmp/dbginfo")
 !2 = !{}
 !3 = !{!4}
-!4 = !MDSubprogram(name: "func", linkageName: "_ZN2ns4funcEv", line: 2, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 2, file: !1, scope: !5, type: !6, function: void ()* @_ZN2ns4funcEv, variables: !2)
-!5 = !MDNamespace(name: "ns", line: 1, file: !1, scope: null)
-!6 = !MDSubroutineType(types: !7)
+!4 = !DISubprogram(name: "func", linkageName: "_ZN2ns4funcEv", line: 2, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 2, file: !1, scope: !5, type: !6, function: void ()* @_ZN2ns4funcEv, variables: !2)
+!5 = !DINamespace(name: "ns", line: 1, file: !1, scope: null)
+!6 = !DISubroutineType(types: !7)
 !7 = !{null}
 !8 = !{i32 2, !"Dwarf Version", i32 4}
 !9 = !{i32 1, !"Debug Info Version", i32 3}
 !10 = !{!"clang version 3.5.0 "}
-!11 = !MDLocation(line: 3, scope: !4)
+!11 = !DILocation(line: 3, scope: !4)
diff --git a/test/DebugInfo/namespace_inline_function_definition.ll b/test/DebugInfo/namespace_inline_function_definition.ll
index 8628b3d465e..c14152065b7 100644
--- a/test/DebugInfo/namespace_inline_function_definition.ll
+++ b/test/DebugInfo/namespace_inline_function_definition.ll
@@ -42,7 +42,7 @@ entry:
   store i32 0, i32* %retval
   %0 = load i32, i32* @x, align 4, !dbg !16
   store i32 %0, i32* %i.addr.i, align 4
-  call void @llvm.dbg.declare(metadata i32* %i.addr.i, metadata !117, metadata !MDExpression()), !dbg !18
+  call void @llvm.dbg.declare(metadata i32* %i.addr.i, metadata !117, metadata !DIExpression()), !dbg !18
   %1 = load i32, i32* %i.addr.i, align 4, !dbg !18
   %mul.i = mul nsw i32 %1, 2, !dbg !18
   ret i32 %mul.i, !dbg !16
@@ -53,7 +53,7 @@ define i32 @_ZN2ns4funcEi(i32 %i) #1 {
 entry:
   %i.addr = alloca i32, align 4
   store i32 %i, i32* %i.addr, align 4
-  call void @llvm.dbg.declare(metadata i32* %i.addr, metadata !17, metadata !MDExpression()), !dbg !19
+  call void @llvm.dbg.declare(metadata i32* %i.addr, metadata !17, metadata !DIExpression()), !dbg !19
   %0 = load i32, i32* %i.addr, align 4, !dbg !19
   %mul = mul nsw i32 %0, 2, !dbg !19
   ret i32 %mul, !dbg !19
@@ -70,26 +70,26 @@ attributes #2 = { nounwind readnone }
 !llvm.module.flags = !{!13, !14}
 !llvm.ident = !{!15}
 
-!0 = !MDCompileUnit(language: DW_LANG_C_plus_plus, producer: "clang version 3.5.0 ", isOptimized: false, emissionKind: 1, file: !1, enums: !2, retainedTypes: !2, subprograms: !3, globals: !2, imports: !2)
-!1 = !MDFile(filename: "namespace_inline_function_definition.cpp", directory: "/tmp/dbginfo")
+!0 = !DICompileUnit(language: DW_LANG_C_plus_plus, producer: "clang version 3.5.0 ", isOptimized: false, emissionKind: 1, file: !1, enums: !2, retainedTypes: !2, subprograms: !3, globals: !2, imports: !2)
+!1 = !DIFile(filename: "namespace_inline_function_definition.cpp", directory: "/tmp/dbginfo")
 !2 = !{}
 !3 = !{!4, !9}
-!4 = !MDSubprogram(name: "main", line: 5, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 5, file: !1, scope: !5, type: !6, function: i32 ()* @main, variables: !2)
-!5 = !MDFile(filename: "namespace_inline_function_definition.cpp", directory: "/tmp/dbginfo")
-!6 = !MDSubroutineType(types: !7)
+!4 = !DISubprogram(name: "main", line: 5, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 5, file: !1, scope: !5, type: !6, function: i32 ()* @main, variables: !2)
+!5 = !DIFile(filename: "namespace_inline_function_definition.cpp", directory: "/tmp/dbginfo")
+!6 = !DISubroutineType(types: !7)
 !7 = !{!8}
-!8 = !MDBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
-!9 = !MDSubprogram(name: "func", linkageName: "_ZN2ns4funcEi", line: 6, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 6, file: !1, scope: !10, type: !11, function: i32 (i32)* @_ZN2ns4funcEi, variables: !2)
-!10 = !MDNamespace(name: "ns", line: 1, file: !1, scope: null)
-!11 = !MDSubroutineType(types: !12)
+!8 = !DIBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
+!9 = !DISubprogram(name: "func", linkageName: "_ZN2ns4funcEi", line: 6, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 6, file: !1, scope: !10, type: !11, function: i32 (i32)* @_ZN2ns4funcEi, variables: !2)
+!10 = !DINamespace(name: "ns", line: 1, file: !1, scope: null)
+!11 = !DISubroutineType(types: !12)
 !12 = !{!8, !8}
 !13 = !{i32 2, !"Dwarf Version", i32 4}
 !14 = !{i32 2, !"Debug Info Version", i32 3}
 !15 = !{!"clang version 3.5.0 "}
-!16 = !MDLocation(line: 5, scope: !4)
-!17 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "i", line: 6, arg: 1, scope: !9, file: !5, type: !8)
+!16 = !DILocation(line: 5, scope: !4)
+!17 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "i", line: 6, arg: 1, scope: !9, file: !5, type: !8)
 
-!117 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "i", line: 6, arg: 1, scope: !9, file: !5, type: !8)
+!117 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "i", line: 6, arg: 1, scope: !9, file: !5, type: !8)
 
-!18 = !MDLocation(line: 6, scope: !9, inlinedAt: !16)
-!19 = !MDLocation(line: 6, scope: !9)
+!18 = !DILocation(line: 6, scope: !9, inlinedAt: !16)
+!19 = !DILocation(line: 6, scope: !9)
diff --git a/test/DebugInfo/nodebug.ll b/test/DebugInfo/nodebug.ll
index d241114ed3c..3ef3119a0c7 100644
--- a/test/DebugInfo/nodebug.ll
+++ b/test/DebugInfo/nodebug.ll
@@ -37,15 +37,15 @@ attributes #0 = { uwtable "less-precise-fpmad"="false" "no-frame-pointer-elim"="
 !llvm.module.flags = !{!8, !9}
 !llvm.ident = !{!10}
 
-!0 = !MDCompileUnit(language: DW_LANG_C_plus_plus, producer: "clang version 3.5.0 ", isOptimized: false, emissionKind: 1, file: !1, enums: !2, retainedTypes: !2, subprograms: !3, globals: !2, imports: !2)
-!1 = !MDFile(filename: "nodebug.cpp", directory: "/tmp/dbginfo")
+!0 = !DICompileUnit(language: DW_LANG_C_plus_plus, producer: "clang version 3.5.0 ", isOptimized: false, emissionKind: 1, file: !1, enums: !2, retainedTypes: !2, subprograms: !3, globals: !2, imports: !2)
+!1 = !DIFile(filename: "nodebug.cpp", directory: "/tmp/dbginfo")
 !2 = !{}
 !3 = !{!4}
-!4 = !MDSubprogram(name: "f1", linkageName: "_Z2f1v", line: 2, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 2, file: !1, scope: !5, type: !6, variables: !2)
-!5 = !MDFile(filename: "nodebug.cpp", directory: "/tmp/dbginfo")
-!6 = !MDSubroutineType(types: !7)
+!4 = !DISubprogram(name: "f1", linkageName: "_Z2f1v", line: 2, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 2, file: !1, scope: !5, type: !6, variables: !2)
+!5 = !DIFile(filename: "nodebug.cpp", directory: "/tmp/dbginfo")
+!6 = !DISubroutineType(types: !7)
 !7 = !{null}
 !8 = !{i32 2, !"Dwarf Version", i32 4}
 !9 = !{i32 2, !"Debug Info Version", i32 3}
 !10 = !{!"clang version 3.5.0 "}
-!11 = !MDLocation(line: 3, scope: !4)
+!11 = !DILocation(line: 3, scope: !4)
diff --git a/test/DebugInfo/piece-verifier.ll b/test/DebugInfo/piece-verifier.ll
index fce88cdf815..9d7efd6a396 100644
--- a/test/DebugInfo/piece-verifier.ll
+++ b/test/DebugInfo/piece-verifier.ll
@@ -23,34 +23,34 @@ attributes #1 = { nounwind readnone }
 !llvm.module.flags = !{!17, !18}
 !llvm.ident = !{!19}
 
-!0 = !MDCompileUnit(language: DW_LANG_C99, producer: "clang version 3.5 ", isOptimized: true, emissionKind: 1, file: !1, enums: !2, retainedTypes: !2, subprograms: !3, globals: !2, imports: !2)
-!1 = !MDFile(filename: "pieces.c", directory: "")
+!0 = !DICompileUnit(language: DW_LANG_C99, producer: "clang version 3.5 ", isOptimized: true, emissionKind: 1, file: !1, enums: !2, retainedTypes: !2, subprograms: !3, globals: !2, imports: !2)
+!1 = !DIFile(filename: "pieces.c", directory: "")
 !2 = !{}
 !3 = !{!4}
-!4 = !MDSubprogram(name: "foo", line: 3, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: true, scopeLine: 3, file: !1, scope: !5, type: !6, function: i32 (i64, i32)* @foo, variables: !15)
-!5 = !MDFile(filename: "pieces.c", directory: "")
-!6 = !MDSubroutineType(types: !7)
+!4 = !DISubprogram(name: "foo", line: 3, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: true, scopeLine: 3, file: !1, scope: !5, type: !6, function: i32 (i64, i32)* @foo, variables: !15)
+!5 = !DIFile(filename: "pieces.c", directory: "")
+!6 = !DISubroutineType(types: !7)
 !7 = !{!8, !9}
-!8 = !MDBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
-!9 = !MDDerivedType(tag: DW_TAG_typedef, name: "S", line: 1, file: !1, baseType: !10)
-!10 = !MDCompositeType(tag: DW_TAG_structure_type, line: 1, size: 128, align: 64, file: !1, elements: !11)
+!8 = !DIBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
+!9 = !DIDerivedType(tag: DW_TAG_typedef, name: "S", line: 1, file: !1, baseType: !10)
+!10 = !DICompositeType(tag: DW_TAG_structure_type, line: 1, size: 128, align: 64, file: !1, elements: !11)
 !11 = !{!12, !14}
-!12 = !MDDerivedType(tag: DW_TAG_member, name: "a", line: 1, size: 64, align: 64, file: !1, scope: !10, baseType: !13)
-!13 = !MDBasicType(tag: DW_TAG_base_type, name: "long int", size: 64, align: 64, encoding: DW_ATE_signed)
-!14 = !MDDerivedType(tag: DW_TAG_member, name: "b", line: 1, size: 32, align: 32, offset: 64, file: !1, scope: !10, baseType: !8)
+!12 = !DIDerivedType(tag: DW_TAG_member, name: "a", line: 1, size: 64, align: 64, file: !1, scope: !10, baseType: !13)
+!13 = !DIBasicType(tag: DW_TAG_base_type, name: "long int", size: 64, align: 64, encoding: DW_ATE_signed)
+!14 = !DIDerivedType(tag: DW_TAG_member, name: "b", line: 1, size: 32, align: 32, offset: 64, file: !1, scope: !10, baseType: !8)
 !15 = !{!16}
-!16 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "s", line: 3, arg: 1, scope: !4, file: !5, type: !9)
+!16 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "s", line: 3, arg: 1, scope: !4, file: !5, type: !9)
 !17 = !{i32 2, !"Dwarf Version", i32 4}
 !18 = !{i32 1, !"Debug Info Version", i32 3}
 !19 = !{!"clang version 3.5 "}
-!20 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "s", line: 3, arg: 1, scope: !4, file: !5, type: !9)
-!21 = !MDLocation(line: 3, scope: !4)
-!22 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "s", line: 3, arg: 1, scope: !4, file: !5, type: !9)
-!23 = !MDLocation(line: 4, scope: !4)
-!24 = !MDExpression(DW_OP_deref, DW_OP_bit_piece, 0, 64)
+!20 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "s", line: 3, arg: 1, scope: !4, file: !5, type: !9)
+!21 = !DILocation(line: 3, scope: !4)
+!22 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "s", line: 3, arg: 1, scope: !4, file: !5, type: !9)
+!23 = !DILocation(line: 4, scope: !4)
+!24 = !DIExpression(DW_OP_deref, DW_OP_bit_piece, 0, 64)
 !25 = !{}
 ; This expression has elements after DW_OP_bit_piece.
 ; CHECK: invalid expression
-; CHECK-NEXT: !MDExpression({{[0-9]+}}, 64, 32, {{[0-9]+}})
+; CHECK-NEXT: !DIExpression({{[0-9]+}}, 64, 32, {{[0-9]+}})
 ; CHECK-NOT: invalid expression
-!27 = !MDExpression(DW_OP_bit_piece, 64, 32, DW_OP_deref)
+!27 = !DIExpression(DW_OP_bit_piece, 64, 32, DW_OP_deref)
diff --git a/test/DebugInfo/restrict.ll b/test/DebugInfo/restrict.ll
index 8192d2c3f27..71d94f60c96 100644
--- a/test/DebugInfo/restrict.ll
+++ b/test/DebugInfo/restrict.ll
@@ -21,7 +21,7 @@ define void @_Z3fooPv(i8* noalias %dst) #0 {
 entry:
   %dst.addr = alloca i8*, align 8
   store i8* %dst, i8** %dst.addr, align 8
-  call void @llvm.dbg.declare(metadata i8** %dst.addr, metadata !13, metadata !MDExpression()), !dbg !14
+  call void @llvm.dbg.declare(metadata i8** %dst.addr, metadata !13, metadata !DIExpression()), !dbg !14
   ret void, !dbg !15
 }
 
@@ -35,19 +35,19 @@ attributes #1 = { nounwind readnone }
 !llvm.module.flags = !{!10, !11}
 !llvm.ident = !{!12}
 
-!0 = !MDCompileUnit(language: DW_LANG_C_plus_plus, producer: "clang version 3.5.0 ", isOptimized: false, emissionKind: 1, file: !1, enums: !2, retainedTypes: !2, subprograms: !3, globals: !2, imports: !2)
-!1 = !MDFile(filename: "restrict.c", directory: "/tmp/dbginfo")
+!0 = !DICompileUnit(language: DW_LANG_C_plus_plus, producer: "clang version 3.5.0 ", isOptimized: false, emissionKind: 1, file: !1, enums: !2, retainedTypes: !2, subprograms: !3, globals: !2, imports: !2)
+!1 = !DIFile(filename: "restrict.c", directory: "/tmp/dbginfo")
 !2 = !{}
 !3 = !{!4}
-!4 = !MDSubprogram(name: "foo", linkageName: "_Z3fooPv", line: 1, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 1, file: !1, scope: !5, type: !6, function: void (i8*)* @_Z3fooPv, variables: !2)
-!5 = !MDFile(filename: "restrict.c", directory: "/tmp/dbginfo")
-!6 = !MDSubroutineType(types: !7)
+!4 = !DISubprogram(name: "foo", linkageName: "_Z3fooPv", line: 1, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 1, file: !1, scope: !5, type: !6, function: void (i8*)* @_Z3fooPv, variables: !2)
+!5 = !DIFile(filename: "restrict.c", directory: "/tmp/dbginfo")
+!6 = !DISubroutineType(types: !7)
 !7 = !{null, !8}
-!8 = !MDDerivedType(tag: DW_TAG_restrict_type, baseType: !9)
-!9 = !MDDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, baseType: null)
+!8 = !DIDerivedType(tag: DW_TAG_restrict_type, baseType: !9)
+!9 = !DIDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, baseType: null)
 !10 = !{i32 2, !"Dwarf Version", i32 4}
 !11 = !{i32 1, !"Debug Info Version", i32 3}
 !12 = !{!"clang version 3.5.0 "}
-!13 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "dst", line: 1, arg: 1, scope: !4, file: !5, type: !8)
-!14 = !MDLocation(line: 1, scope: !4)
-!15 = !MDLocation(line: 2, scope: !4)
+!13 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "dst", line: 1, arg: 1, scope: !4, file: !5, type: !8)
+!14 = !DILocation(line: 1, scope: !4)
+!15 = !DILocation(line: 2, scope: !4)
diff --git a/test/DebugInfo/sugared-constants.ll b/test/DebugInfo/sugared-constants.ll
index c5bb528b063..421fe1dcd85 100644
--- a/test/DebugInfo/sugared-constants.ll
+++ b/test/DebugInfo/sugared-constants.ll
@@ -24,11 +24,11 @@
 ; Function Attrs: uwtable
 define i32 @main() #0 {
 entry:
-  tail call void @llvm.dbg.value(metadata i32 42, i64 0, metadata !10, metadata !MDExpression()), !dbg !21
+  tail call void @llvm.dbg.value(metadata i32 42, i64 0, metadata !10, metadata !DIExpression()), !dbg !21
   tail call void @_Z4funci(i32 42), !dbg !22
-  tail call void @llvm.dbg.value(metadata i32 117, i64 0, metadata !12, metadata !MDExpression()), !dbg !24
+  tail call void @llvm.dbg.value(metadata i32 117, i64 0, metadata !12, metadata !DIExpression()), !dbg !24
   tail call void @_Z4funcj(i32 117), !dbg !25
-  tail call void @llvm.dbg.value(metadata i16 7, i64 0, metadata !15, metadata !MDExpression()), !dbg !27
+  tail call void @llvm.dbg.value(metadata i16 7, i64 0, metadata !15, metadata !DIExpression()), !dbg !27
   tail call void @_Z4funcDs(i16 zeroext 7), !dbg !28
   ret i32 0, !dbg !29
 }
@@ -50,33 +50,33 @@ attributes #2 = { nounwind readnone }
 !llvm.module.flags = !{!17, !18}
 !llvm.ident = !{!19}
 
-!0 = !MDCompileUnit(language: DW_LANG_C_plus_plus, producer: "clang version 3.5.0 ", isOptimized: true, emissionKind: 1, file: !1, enums: !2, retainedTypes: !2, subprograms: !3, globals: !2, imports: !2)
-!1 = !MDFile(filename: "const.cpp", directory: "/tmp/dbginfo")
+!0 = !DICompileUnit(language: DW_LANG_C_plus_plus, producer: "clang version 3.5.0 ", isOptimized: true, emissionKind: 1, file: !1, enums: !2, retainedTypes: !2, subprograms: !3, globals: !2, imports: !2)
+!1 = !DIFile(filename: "const.cpp", directory: "/tmp/dbginfo")
 !2 = !{}
 !3 = !{!4}
-!4 = !MDSubprogram(name: "main", line: 4, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: true, scopeLine: 4, file: !1, scope: !5, type: !6, function: i32 ()* @main, variables: !9)
-!5 = !MDFile(filename: "const.cpp", directory: "/tmp/dbginfo")
-!6 = !MDSubroutineType(types: !7)
+!4 = !DISubprogram(name: "main", line: 4, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: true, scopeLine: 4, file: !1, scope: !5, type: !6, function: i32 ()* @main, variables: !9)
+!5 = !DIFile(filename: "const.cpp", directory: "/tmp/dbginfo")
+!6 = !DISubroutineType(types: !7)
 !7 = !{!8}
-!8 = !MDBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
+!8 = !DIBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
 !9 = !{!10, !12, !15}
-!10 = !MDLocalVariable(tag: DW_TAG_auto_variable, name: "i", line: 5, scope: !4, file: !5, type: !11)
-!11 = !MDDerivedType(tag: DW_TAG_const_type, baseType: !8)
-!12 = !MDLocalVariable(tag: DW_TAG_auto_variable, name: "j", line: 7, scope: !4, file: !5, type: !13)
-!13 = !MDDerivedType(tag: DW_TAG_const_type, baseType: !14)
-!14 = !MDBasicType(tag: DW_TAG_base_type, name: "unsigned int", size: 32, align: 32, encoding: DW_ATE_unsigned)
-!15 = !MDLocalVariable(tag: DW_TAG_auto_variable, name: "c", line: 9, scope: !4, file: !5, type: !16)
-!16 = !MDBasicType(tag: DW_TAG_base_type, name: "char16_t", size: 16, align: 16, encoding: 16)
+!10 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "i", line: 5, scope: !4, file: !5, type: !11)
+!11 = !DIDerivedType(tag: DW_TAG_const_type, baseType: !8)
+!12 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "j", line: 7, scope: !4, file: !5, type: !13)
+!13 = !DIDerivedType(tag: DW_TAG_const_type, baseType: !14)
+!14 = !DIBasicType(tag: DW_TAG_base_type, name: "unsigned int", size: 32, align: 32, encoding: DW_ATE_unsigned)
+!15 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "c", line: 9, scope: !4, file: !5, type: !16)
+!16 = !DIBasicType(tag: DW_TAG_base_type, name: "char16_t", size: 16, align: 16, encoding: 16)
 !17 = !{i32 2, !"Dwarf Version", i32 4}
 !18 = !{i32 1, !"Debug Info Version", i32 3}
 !19 = !{!"clang version 3.5.0 "}
 !20 = !{i32 42}
-!21 = !MDLocation(line: 5, scope: !4)
-!22 = !MDLocation(line: 6, scope: !4)
+!21 = !DILocation(line: 5, scope: !4)
+!22 = !DILocation(line: 6, scope: !4)
 !23 = !{i32 117}
-!24 = !MDLocation(line: 7, scope: !4)
-!25 = !MDLocation(line: 8, scope: !4)
+!24 = !DILocation(line: 7, scope: !4)
+!25 = !DILocation(line: 8, scope: !4)
 !26 = !{i16 7}
-!27 = !MDLocation(line: 9, scope: !4)
-!28 = !MDLocation(line: 10, scope: !4)
-!29 = !MDLocation(line: 11, scope: !4)
+!27 = !DILocation(line: 9, scope: !4)
+!28 = !DILocation(line: 10, scope: !4)
+!29 = !DILocation(line: 11, scope: !4)
diff --git a/test/DebugInfo/template-recursive-void.ll b/test/DebugInfo/template-recursive-void.ll
index d33b8daab73..30eaee3f75d 100644
--- a/test/DebugInfo/template-recursive-void.ll
+++ b/test/DebugInfo/template-recursive-void.ll
@@ -25,37 +25,37 @@
 !llvm.dbg.cu = !{!0}
 !llvm.module.flags = !{!36, !37}
 
-!0 = !MDCompileUnit(language: DW_LANG_C_plus_plus, producer: "clang version 3.4 (trunk 187958) (llvm/trunk 187964)", isOptimized: false, emissionKind: 0, file: !1, enums: !2, retainedTypes: !2, subprograms: !2, globals: !3, imports: !2)
-!1 = !MDFile(filename: "debug-info-template-recursive.cpp", directory: "/usr/local/google/home/echristo/tmp")
+!0 = !DICompileUnit(language: DW_LANG_C_plus_plus, producer: "clang version 3.4 (trunk 187958) (llvm/trunk 187964)", isOptimized: false, emissionKind: 0, file: !1, enums: !2, retainedTypes: !2, subprograms: !2, globals: !3, imports: !2)
+!1 = !DIFile(filename: "debug-info-template-recursive.cpp", directory: "/usr/local/google/home/echristo/tmp")
 !2 = !{}
 !3 = !{!4}
-!4 = !MDGlobalVariable(name: "filters", line: 10, isLocal: false, isDefinition: true, scope: null, file: !5, type: !6, variable: %class.bar* @filters)
-!5 = !MDFile(filename: "debug-info-template-recursive.cpp", directory: "/usr/local/google/home/echristo/tmp")
-!6 = !MDCompositeType(tag: DW_TAG_class_type, name: "bar", line: 9, size: 8, align: 8, file: !1, elements: !7)
+!4 = !DIGlobalVariable(name: "filters", line: 10, isLocal: false, isDefinition: true, scope: null, file: !5, type: !6, variable: %class.bar* @filters)
+!5 = !DIFile(filename: "debug-info-template-recursive.cpp", directory: "/usr/local/google/home/echristo/tmp")
+!6 = !DICompositeType(tag: DW_TAG_class_type, name: "bar", line: 9, size: 8, align: 8, file: !1, elements: !7)
 !7 = !{!8, !31}
-!8 = !MDDerivedType(tag: DW_TAG_inheritance, scope: !6, baseType: !9)
-!9 = !MDCompositeType(tag: DW_TAG_class_type, name: "foo<void>", line: 5, size: 8, align: 8, file: !1, elements: !10, templateParams: !29)
+!8 = !DIDerivedType(tag: DW_TAG_inheritance, scope: !6, baseType: !9)
+!9 = !DICompositeType(tag: DW_TAG_class_type, name: "foo<void>", line: 5, size: 8, align: 8, file: !1, elements: !10, templateParams: !29)
 !10 = !{!11, !19, !25}
-!11 = !MDDerivedType(tag: DW_TAG_inheritance, scope: !9, baseType: !12)
-!12 = !MDCompositeType(tag: DW_TAG_class_type, name: "base", line: 3, size: 8, align: 8, file: !1, elements: !13)
+!11 = !DIDerivedType(tag: DW_TAG_inheritance, scope: !9, baseType: !12)
+!12 = !DICompositeType(tag: DW_TAG_class_type, name: "base", line: 3, size: 8, align: 8, file: !1, elements: !13)
 !13 = !{!14}
-!14 = !MDSubprogram(name: "base", line: 3, isLocal: false, isDefinition: false, virtualIndex: 6, flags: DIFlagArtificial | DIFlagPrototyped, isOptimized: false, scopeLine: 3, file: !1, scope: !12, type: !15)
-!15 = !MDSubroutineType(types: !16)
+!14 = !DISubprogram(name: "base", line: 3, isLocal: false, isDefinition: false, virtualIndex: 6, flags: DIFlagArtificial | DIFlagPrototyped, isOptimized: false, scopeLine: 3, file: !1, scope: !12, type: !15)
+!15 = !DISubroutineType(types: !16)
 !16 = !{null, !17}
-!17 = !MDDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, flags: DIFlagArtificial | DIFlagObjectPointer, baseType: !12)
-!19 = !MDSubprogram(name: "operator=", linkageName: "_ZN3fooIvEaSES0_", line: 6, isLocal: false, isDefinition: false, virtualIndex: 6, flags: DIFlagPrivate | DIFlagPrototyped, isOptimized: false, scopeLine: 6, file: !1, scope: !9, type: !20)
-!20 = !MDSubroutineType(types: !21)
+!17 = !DIDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, flags: DIFlagArtificial | DIFlagObjectPointer, baseType: !12)
+!19 = !DISubprogram(name: "operator=", linkageName: "_ZN3fooIvEaSES0_", line: 6, isLocal: false, isDefinition: false, virtualIndex: 6, flags: DIFlagPrivate | DIFlagPrototyped, isOptimized: false, scopeLine: 6, file: !1, scope: !9, type: !20)
+!20 = !DISubroutineType(types: !21)
 !21 = !{null, !22, !23}
-!22 = !MDDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, flags: DIFlagArtificial | DIFlagObjectPointer, baseType: !9)
-!23 = !MDDerivedType(tag: DW_TAG_const_type, baseType: !9)
-!25 = !MDSubprogram(name: "foo", line: 5, isLocal: false, isDefinition: false, virtualIndex: 6, flags: DIFlagArtificial | DIFlagPrototyped, isOptimized: false, scopeLine: 5, file: !1, scope: !9, type: !26)
-!26 = !MDSubroutineType(types: !27)
+!22 = !DIDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, flags: DIFlagArtificial | DIFlagObjectPointer, baseType: !9)
+!23 = !DIDerivedType(tag: DW_TAG_const_type, baseType: !9)
+!25 = !DISubprogram(name: "foo", line: 5, isLocal: false, isDefinition: false, virtualIndex: 6, flags: DIFlagArtificial | DIFlagPrototyped, isOptimized: false, scopeLine: 5, file: !1, scope: !9, type: !26)
+!26 = !DISubroutineType(types: !27)
 !27 = !{null, !22}
 !29 = !{!30}
-!30 = !MDTemplateTypeParameter(name: "T", type: null)
-!31 = !MDSubprogram(name: "bar", line: 9, isLocal: false, isDefinition: false, virtualIndex: 6, flags: DIFlagArtificial | DIFlagPrototyped, isOptimized: false, scopeLine: 9, file: !1, scope: !6, type: !32)
-!32 = !MDSubroutineType(types: !33)
+!30 = !DITemplateTypeParameter(name: "T", type: null)
+!31 = !DISubprogram(name: "bar", line: 9, isLocal: false, isDefinition: false, virtualIndex: 6, flags: DIFlagArtificial | DIFlagPrototyped, isOptimized: false, scopeLine: 9, file: !1, scope: !6, type: !32)
+!32 = !DISubroutineType(types: !33)
 !33 = !{null, !34}
-!34 = !MDDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, flags: DIFlagArtificial | DIFlagObjectPointer, baseType: !6)
+!34 = !DIDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, flags: DIFlagArtificial | DIFlagObjectPointer, baseType: !6)
 !36 = !{i32 2, !"Dwarf Version", i32 3}
 !37 = !{i32 1, !"Debug Info Version", i32 3}
diff --git a/test/DebugInfo/tu-composite.ll b/test/DebugInfo/tu-composite.ll
index a5f25fba17a..47fb8bc3148 100644
--- a/test/DebugInfo/tu-composite.ll
+++ b/test/DebugInfo/tu-composite.ll
@@ -91,7 +91,7 @@ define void @_ZN1C3fooEv(%struct.C* %this) unnamed_addr #0 align 2 {
 entry:
   %this.addr = alloca %struct.C*, align 8
   store %struct.C* %this, %struct.C** %this.addr, align 8
-  call void @llvm.dbg.declare(metadata %struct.C** %this.addr, metadata !36, metadata !MDExpression()), !dbg !38
+  call void @llvm.dbg.declare(metadata %struct.C** %this.addr, metadata !36, metadata !DIExpression()), !dbg !38
   %this1 = load %struct.C*, %struct.C** %this.addr
   ret void, !dbg !39
 }
@@ -108,12 +108,12 @@ entry:
   %e = alloca %"struct.D::Nested", align 1
   %p = alloca %"struct.D::Nested2"*, align 8
   %t = alloca %"struct.D::virt", align 8
-  call void @llvm.dbg.declare(metadata %struct.bar* %B, metadata !40, metadata !MDExpression()), !dbg !42
-  call void @llvm.dbg.declare(metadata [3 x %struct.bar]* %A, metadata !43, metadata !MDExpression()), !dbg !47
-  call void @llvm.dbg.declare(metadata %struct.bar* %B2, metadata !48, metadata !MDExpression()), !dbg !50
-  call void @llvm.dbg.declare(metadata %"struct.D::Nested"* %e, metadata !51, metadata !MDExpression()), !dbg !52
-  call void @llvm.dbg.declare(metadata %"struct.D::Nested2"** %p, metadata !53, metadata !MDExpression()), !dbg !55
-  call void @llvm.dbg.declare(metadata %"struct.D::virt"* %t, metadata !56, metadata !MDExpression()), !dbg !57
+  call void @llvm.dbg.declare(metadata %struct.bar* %B, metadata !40, metadata !DIExpression()), !dbg !42
+  call void @llvm.dbg.declare(metadata [3 x %struct.bar]* %A, metadata !43, metadata !DIExpression()), !dbg !47
+  call void @llvm.dbg.declare(metadata %struct.bar* %B2, metadata !48, metadata !DIExpression()), !dbg !50
+  call void @llvm.dbg.declare(metadata %"struct.D::Nested"* %e, metadata !51, metadata !DIExpression()), !dbg !52
+  call void @llvm.dbg.declare(metadata %"struct.D::Nested2"** %p, metadata !53, metadata !DIExpression()), !dbg !55
+  call void @llvm.dbg.declare(metadata %"struct.D::virt"* %t, metadata !56, metadata !DIExpression()), !dbg !57
   ret void, !dbg !58
 }
 
@@ -123,62 +123,62 @@ attributes #1 = { nounwind readnone }
 !llvm.dbg.cu = !{!0}
 !llvm.module.flags = !{!35, !59}
 
-!0 = !MDCompileUnit(language: DW_LANG_C_plus_plus, producer: "clang version 3.4", isOptimized: false, emissionKind: 0, file: !1, enums: !2, retainedTypes: !3, subprograms: !30, globals: !2, imports: !2)
-!1 = !MDFile(filename: "tmp.cpp", directory: ".")
+!0 = !DICompileUnit(language: DW_LANG_C_plus_plus, producer: "clang version 3.4", isOptimized: false, emissionKind: 0, file: !1, enums: !2, retainedTypes: !3, subprograms: !30, globals: !2, imports: !2)
+!1 = !DIFile(filename: "tmp.cpp", directory: ".")
 !2 = !{}
 !3 = !{!4, !18, !19, !22, !23, !24}
-!4 = !MDCompositeType(tag: DW_TAG_structure_type, name: "C", line: 1, size: 64, align: 64, file: !1, elements: !5, vtableHolder: !"_ZTS1C", identifier: "_ZTS1C")
+!4 = !DICompositeType(tag: DW_TAG_structure_type, name: "C", line: 1, size: 64, align: 64, file: !1, elements: !5, vtableHolder: !"_ZTS1C", identifier: "_ZTS1C")
 !5 = !{!6, !13}
-!6 = !MDDerivedType(tag: DW_TAG_member, name: "_vptr$C", size: 64, flags: DIFlagArtificial, file: !1, scope: !7, baseType: !8)
-!7 = !MDFile(filename: "tmp.cpp", directory: ".")
-!8 = !MDDerivedType(tag: DW_TAG_pointer_type, size: 64, baseType: !9)
-!9 = !MDDerivedType(tag: DW_TAG_pointer_type, name: "__vtbl_ptr_type", size: 64, baseType: !10)
-!10 = !MDSubroutineType(types: !11)
+!6 = !DIDerivedType(tag: DW_TAG_member, name: "_vptr$C", size: 64, flags: DIFlagArtificial, file: !1, scope: !7, baseType: !8)
+!7 = !DIFile(filename: "tmp.cpp", directory: ".")
+!8 = !DIDerivedType(tag: DW_TAG_pointer_type, size: 64, baseType: !9)
+!9 = !DIDerivedType(tag: DW_TAG_pointer_type, name: "__vtbl_ptr_type", size: 64, baseType: !10)
+!10 = !DISubroutineType(types: !11)
 !11 = !{!12}
-!12 = !MDBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
-!13 = !MDSubprogram(name: "foo", linkageName: "_ZN1C3fooEv", line: 2, isLocal: false, isDefinition: false, virtuality: DW_VIRTUALITY_virtual, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 2, file: !1, scope: !"_ZTS1C", type: !14, containingType: !"_ZTS1C")
-!14 = !MDSubroutineType(types: !15)
+!12 = !DIBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
+!13 = !DISubprogram(name: "foo", linkageName: "_ZN1C3fooEv", line: 2, isLocal: false, isDefinition: false, virtuality: DW_VIRTUALITY_virtual, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 2, file: !1, scope: !"_ZTS1C", type: !14, containingType: !"_ZTS1C")
+!14 = !DISubroutineType(types: !15)
 !15 = !{null, !16}
-!16 = !MDDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, flags: DIFlagArtificial | DIFlagObjectPointer, baseType: !"_ZTS1C")
-!18 = !MDCompositeType(tag: DW_TAG_structure_type, name: "bar", line: 7, size: 8, align: 8, file: !1, elements: !2, identifier: "_ZTS3bar")
-!19 = !MDCompositeType(tag: DW_TAG_structure_type, name: "D", line: 9, size: 8, align: 8, file: !1, elements: !20, identifier: "_ZTS1D")
+!16 = !DIDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, flags: DIFlagArtificial | DIFlagObjectPointer, baseType: !"_ZTS1C")
+!18 = !DICompositeType(tag: DW_TAG_structure_type, name: "bar", line: 7, size: 8, align: 8, file: !1, elements: !2, identifier: "_ZTS3bar")
+!19 = !DICompositeType(tag: DW_TAG_structure_type, name: "D", line: 9, size: 8, align: 8, file: !1, elements: !20, identifier: "_ZTS1D")
 !20 = !{!21}
-!21 = !MDDerivedType(tag: DW_TAG_member, name: "a", line: 11, flags: DIFlagStaticMember, file: !1, scope: !"_ZTS1D", baseType: !12)
-!22 = !MDCompositeType(tag: DW_TAG_structure_type, name: "Nested", line: 12, size: 8, align: 8, file: !1, scope: !"_ZTS1D", elements: !2, identifier: "_ZTSN1D6NestedE")
-!23 = !MDCompositeType(tag: DW_TAG_structure_type, name: "Nested2", line: 13, flags: DIFlagFwdDecl, file: !1, scope: !"_ZTS1D", identifier: "_ZTSN1D7Nested2E")
-!24 = !MDCompositeType(tag: DW_TAG_structure_type, name: "virt<bar>", line: 15, size: 64, align: 64, file: !1, scope: !"_ZTS1D", elements: !25, templateParams: !28, identifier: "_ZTSN1D4virtI3barEE")
+!21 = !DIDerivedType(tag: DW_TAG_member, name: "a", line: 11, flags: DIFlagStaticMember, file: !1, scope: !"_ZTS1D", baseType: !12)
+!22 = !DICompositeType(tag: DW_TAG_structure_type, name: "Nested", line: 12, size: 8, align: 8, file: !1, scope: !"_ZTS1D", elements: !2, identifier: "_ZTSN1D6NestedE")
+!23 = !DICompositeType(tag: DW_TAG_structure_type, name: "Nested2", line: 13, flags: DIFlagFwdDecl, file: !1, scope: !"_ZTS1D", identifier: "_ZTSN1D7Nested2E")
+!24 = !DICompositeType(tag: DW_TAG_structure_type, name: "virt<bar>", line: 15, size: 64, align: 64, file: !1, scope: !"_ZTS1D", elements: !25, templateParams: !28, identifier: "_ZTSN1D4virtI3barEE")
 !25 = !{!26}
-!26 = !MDDerivedType(tag: DW_TAG_member, name: "values", line: 16, size: 64, align: 64, file: !1, scope: !"_ZTSN1D4virtI3barEE", baseType: !27)
-!27 = !MDDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, baseType: !"_ZTS3bar")
+!26 = !DIDerivedType(tag: DW_TAG_member, name: "values", line: 16, size: 64, align: 64, file: !1, scope: !"_ZTSN1D4virtI3barEE", baseType: !27)
+!27 = !DIDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, baseType: !"_ZTS3bar")
 !28 = !{!29}
-!29 = !MDTemplateTypeParameter(name: "T", type: !"_ZTS3bar")
+!29 = !DITemplateTypeParameter(name: "T", type: !"_ZTS3bar")
 !30 = !{!31, !32}
-!31 = !MDSubprogram(name: "foo", linkageName: "_ZN1C3fooEv", line: 4, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 4, file: !1, scope: null, type: !14, function: void (%struct.C*)* @_ZN1C3fooEv, declaration: !13, variables: !2)
-!32 = !MDSubprogram(name: "test", linkageName: "_Z4testv", line: 20, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 20, file: !1, scope: !7, type: !33, function: void ()* @_Z4testv, variables: !2)
-!33 = !MDSubroutineType(types: !34)
+!31 = !DISubprogram(name: "foo", linkageName: "_ZN1C3fooEv", line: 4, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 4, file: !1, scope: null, type: !14, function: void (%struct.C*)* @_ZN1C3fooEv, declaration: !13, variables: !2)
+!32 = !DISubprogram(name: "test", linkageName: "_Z4testv", line: 20, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 20, file: !1, scope: !7, type: !33, function: void ()* @_Z4testv, variables: !2)
+!33 = !DISubroutineType(types: !34)
 !34 = !{null}
 !35 = !{i32 2, !"Dwarf Version", i32 2}
-!36 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "this", arg: 1, flags: DIFlagArtificial | DIFlagObjectPointer, scope: !31, type: !37)
-!37 = !MDDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, baseType: !"_ZTS1C")
-!38 = !MDLocation(line: 0, scope: !31)
-!39 = !MDLocation(line: 5, scope: !31)
-!40 = !MDLocalVariable(tag: DW_TAG_auto_variable, name: "B", line: 21, scope: !32, file: !7, type: !41)
-!41 = !MDDerivedType(tag: DW_TAG_typedef, name: "baz", line: 8, file: !1, baseType: !"_ZTS3bar")
-!42 = !MDLocation(line: 21, scope: !32)
-!43 = !MDLocalVariable(tag: DW_TAG_auto_variable, name: "A", line: 22, scope: !32, file: !7, type: !44)
-!44 = !MDCompositeType(tag: DW_TAG_array_type, size: 24, align: 8, baseType: !"_ZTS3bar", elements: !45)
+!36 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "this", arg: 1, flags: DIFlagArtificial | DIFlagObjectPointer, scope: !31, type: !37)
+!37 = !DIDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, baseType: !"_ZTS1C")
+!38 = !DILocation(line: 0, scope: !31)
+!39 = !DILocation(line: 5, scope: !31)
+!40 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "B", line: 21, scope: !32, file: !7, type: !41)
+!41 = !DIDerivedType(tag: DW_TAG_typedef, name: "baz", line: 8, file: !1, baseType: !"_ZTS3bar")
+!42 = !DILocation(line: 21, scope: !32)
+!43 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "A", line: 22, scope: !32, file: !7, type: !44)
+!44 = !DICompositeType(tag: DW_TAG_array_type, size: 24, align: 8, baseType: !"_ZTS3bar", elements: !45)
 !45 = !{!46}
-!46 = !MDSubrange(count: 3)
-!47 = !MDLocation(line: 22, scope: !32)
-!48 = !MDLocalVariable(tag: DW_TAG_auto_variable, name: "B2", line: 23, scope: !32, file: !7, type: !49)
-!49 = !MDDerivedType(tag: DW_TAG_typedef, name: "baz2", line: 10, file: !1, scope: !"_ZTS1D", baseType: !"_ZTS3bar")
-!50 = !MDLocation(line: 23, scope: !32)
-!51 = !MDLocalVariable(tag: DW_TAG_auto_variable, name: "e", line: 24, scope: !32, file: !7, type: !22)
-!52 = !MDLocation(line: 24, scope: !32)
-!53 = !MDLocalVariable(tag: DW_TAG_auto_variable, name: "p", line: 25, scope: !32, file: !7, type: !54)
-!54 = !MDDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, baseType: !"_ZTSN1D7Nested2E")
-!55 = !MDLocation(line: 25, scope: !32)
-!56 = !MDLocalVariable(tag: DW_TAG_auto_variable, name: "t", line: 26, scope: !32, file: !7, type: !24)
-!57 = !MDLocation(line: 26, scope: !32)
-!58 = !MDLocation(line: 27, scope: !32)
+!46 = !DISubrange(count: 3)
+!47 = !DILocation(line: 22, scope: !32)
+!48 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "B2", line: 23, scope: !32, file: !7, type: !49)
+!49 = !DIDerivedType(tag: DW_TAG_typedef, name: "baz2", line: 10, file: !1, scope: !"_ZTS1D", baseType: !"_ZTS3bar")
+!50 = !DILocation(line: 23, scope: !32)
+!51 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "e", line: 24, scope: !32, file: !7, type: !22)
+!52 = !DILocation(line: 24, scope: !32)
+!53 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "p", line: 25, scope: !32, file: !7, type: !54)
+!54 = !DIDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, baseType: !"_ZTSN1D7Nested2E")
+!55 = !DILocation(line: 25, scope: !32)
+!56 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "t", line: 26, scope: !32, file: !7, type: !24)
+!57 = !DILocation(line: 26, scope: !32)
+!58 = !DILocation(line: 27, scope: !32)
 !59 = !{i32 1, !"Debug Info Version", i32 3}
diff --git a/test/DebugInfo/tu-member-pointer.ll b/test/DebugInfo/tu-member-pointer.ll
index 3c4ce347b15..a46c41313cf 100644
--- a/test/DebugInfo/tu-member-pointer.ll
+++ b/test/DebugInfo/tu-member-pointer.ll
@@ -16,15 +16,15 @@
 !llvm.dbg.cu = !{!0}
 !llvm.module.flags = !{!10, !11}
 
-!0 = !MDCompileUnit(language: DW_LANG_C_plus_plus, producer: "clang version 3.4", isOptimized: false, emissionKind: 0, file: !1, enums: !2, retainedTypes: !3, subprograms: !2, globals: !5, imports: !2)
-!1 = !MDFile(filename: "foo.cpp", directory: ".")
+!0 = !DICompileUnit(language: DW_LANG_C_plus_plus, producer: "clang version 3.4", isOptimized: false, emissionKind: 0, file: !1, enums: !2, retainedTypes: !3, subprograms: !2, globals: !5, imports: !2)
+!1 = !DIFile(filename: "foo.cpp", directory: ".")
 !2 = !{}
 !3 = !{!4}
-!4 = !MDCompositeType(tag: DW_TAG_structure_type, name: "Foo", line: 1, flags: DIFlagFwdDecl, file: !1, identifier: "_ZTS3Foo")
+!4 = !DICompositeType(tag: DW_TAG_structure_type, name: "Foo", line: 1, flags: DIFlagFwdDecl, file: !1, identifier: "_ZTS3Foo")
 !5 = !{!6}
-!6 = !MDGlobalVariable(name: "x", line: 4, isLocal: false, isDefinition: true, scope: null, file: !7, type: !8, variable: i64* @x)
-!7 = !MDFile(filename: "foo.cpp", directory: ".")
-!8 = !MDDerivedType(tag: DW_TAG_ptr_to_member_type, baseType: !9, extraData: !"_ZTS3Foo")
-!9 = !MDBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
+!6 = !DIGlobalVariable(name: "x", line: 4, isLocal: false, isDefinition: true, scope: null, file: !7, type: !8, variable: i64* @x)
+!7 = !DIFile(filename: "foo.cpp", directory: ".")
+!8 = !DIDerivedType(tag: DW_TAG_ptr_to_member_type, baseType: !9, extraData: !"_ZTS3Foo")
+!9 = !DIBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
 !10 = !{i32 2, !"Dwarf Version", i32 2}
 !11 = !{i32 1, !"Debug Info Version", i32 3}
diff --git a/test/DebugInfo/two-cus-from-same-file.ll b/test/DebugInfo/two-cus-from-same-file.ll
index 0ed11073820..9dbd64a3fb9 100644
--- a/test/DebugInfo/two-cus-from-same-file.ll
+++ b/test/DebugInfo/two-cus-from-same-file.ll
@@ -23,8 +23,8 @@ declare i32 @puts(i8* nocapture) nounwind
 
 define i32 @main(i32 %argc, i8** nocapture %argv) nounwind {
 entry:
-  tail call void @llvm.dbg.value(metadata i32 %argc, i64 0, metadata !21, metadata !MDExpression()), !dbg !26
-  tail call void @llvm.dbg.value(metadata i8** %argv, i64 0, metadata !22, metadata !MDExpression()), !dbg !27
+  tail call void @llvm.dbg.value(metadata i32 %argc, i64 0, metadata !21, metadata !DIExpression()), !dbg !26
+  tail call void @llvm.dbg.value(metadata i8** %argv, i64 0, metadata !22, metadata !DIExpression()), !dbg !27
   %puts = tail call i32 @puts(i8* getelementptr inbounds ([6 x i8], [6 x i8]* @str1, i32 0, i32 0)), !dbg !28
   tail call void @foo() nounwind, !dbg !30
   ret i32 0, !dbg !31
@@ -35,35 +35,35 @@ declare void @llvm.dbg.value(metadata, i64, metadata, metadata) nounwind readnon
 !llvm.dbg.cu = !{!0, !9}
 !llvm.module.flags = !{!33}
 
-!0 = !MDCompileUnit(language: DW_LANG_C99, producer: "clang version 3.2 (trunk 156513)", isOptimized: true, emissionKind: 1, file: !32, enums: !1, retainedTypes: !1, subprograms: !3, globals: !1, imports: !1)
+!0 = !DICompileUnit(language: DW_LANG_C99, producer: "clang version 3.2 (trunk 156513)", isOptimized: true, emissionKind: 1, file: !32, enums: !1, retainedTypes: !1, subprograms: !3, globals: !1, imports: !1)
 !1 = !{}
 !3 = !{!5}
-!5 = !MDSubprogram(name: "foo", line: 5, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: true, scopeLine: 5, file: !32, scope: !6, type: !7, function: void ()* @foo, variables: !1)
-!6 = !MDFile(filename: "foo.c", directory: "/tmp")
-!7 = !MDSubroutineType(types: !8)
+!5 = !DISubprogram(name: "foo", line: 5, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: true, scopeLine: 5, file: !32, scope: !6, type: !7, function: void ()* @foo, variables: !1)
+!6 = !DIFile(filename: "foo.c", directory: "/tmp")
+!7 = !DISubroutineType(types: !8)
 !8 = !{null}
-!9 = !MDCompileUnit(language: DW_LANG_C99, producer: "clang version 3.2 (trunk 156513)", isOptimized: true, emissionKind: 1, file: !32, enums: !1, retainedTypes: !1, subprograms: !10, globals: !1, imports: !1)
+!9 = !DICompileUnit(language: DW_LANG_C99, producer: "clang version 3.2 (trunk 156513)", isOptimized: true, emissionKind: 1, file: !32, enums: !1, retainedTypes: !1, subprograms: !10, globals: !1, imports: !1)
 !10 = !{!12}
-!12 = !MDSubprogram(name: "main", line: 11, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: true, scopeLine: 11, file: !32, scope: !6, type: !13, function: i32 (i32, i8**)* @main, variables: !19)
-!13 = !MDSubroutineType(types: !14)
+!12 = !DISubprogram(name: "main", line: 11, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: true, scopeLine: 11, file: !32, scope: !6, type: !13, function: i32 (i32, i8**)* @main, variables: !19)
+!13 = !DISubroutineType(types: !14)
 !14 = !{!15, !15, !16}
-!15 = !MDBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
-!16 = !MDDerivedType(tag: DW_TAG_pointer_type, size: 32, align: 32, baseType: !17)
-!17 = !MDDerivedType(tag: DW_TAG_pointer_type, size: 32, align: 32, baseType: !18)
-!18 = !MDBasicType(tag: DW_TAG_base_type, name: "char", size: 8, align: 8, encoding: DW_ATE_signed_char)
+!15 = !DIBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
+!16 = !DIDerivedType(tag: DW_TAG_pointer_type, size: 32, align: 32, baseType: !17)
+!17 = !DIDerivedType(tag: DW_TAG_pointer_type, size: 32, align: 32, baseType: !18)
+!18 = !DIBasicType(tag: DW_TAG_base_type, name: "char", size: 8, align: 8, encoding: DW_ATE_signed_char)
 !19 = !{!21, !22}
-!21 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "argc", line: 11, arg: 1, scope: !12, file: !6, type: !15)
-!22 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "argv", line: 11, arg: 2, scope: !12, file: !6, type: !16)
-!23 = !MDLocation(line: 6, column: 3, scope: !24)
-!24 = distinct !MDLexicalBlock(line: 5, column: 16, file: !32, scope: !5)
-!25 = !MDLocation(line: 7, column: 1, scope: !24)
-!26 = !MDLocation(line: 11, column: 14, scope: !12)
-!27 = !MDLocation(line: 11, column: 26, scope: !12)
-!28 = !MDLocation(line: 12, column: 3, scope: !29)
-!29 = distinct !MDLexicalBlock(line: 11, column: 34, file: !32, scope: !12)
-!30 = !MDLocation(line: 13, column: 3, scope: !29)
-!31 = !MDLocation(line: 14, column: 3, scope: !29)
-!32 = !MDFile(filename: "foo.c", directory: "/tmp")
+!21 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "argc", line: 11, arg: 1, scope: !12, file: !6, type: !15)
+!22 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "argv", line: 11, arg: 2, scope: !12, file: !6, type: !16)
+!23 = !DILocation(line: 6, column: 3, scope: !24)
+!24 = distinct !DILexicalBlock(line: 5, column: 16, file: !32, scope: !5)
+!25 = !DILocation(line: 7, column: 1, scope: !24)
+!26 = !DILocation(line: 11, column: 14, scope: !12)
+!27 = !DILocation(line: 11, column: 26, scope: !12)
+!28 = !DILocation(line: 12, column: 3, scope: !29)
+!29 = distinct !DILexicalBlock(line: 11, column: 34, file: !32, scope: !12)
+!30 = !DILocation(line: 13, column: 3, scope: !29)
+!31 = !DILocation(line: 14, column: 3, scope: !29)
+!32 = !DIFile(filename: "foo.c", directory: "/tmp")
 
 ; This test is simple to be cross platform (many targets don't yet have
 ; sufficiently good DWARF emission and/or dumping)
diff --git a/test/DebugInfo/typedef.ll b/test/DebugInfo/typedef.ll
index 416e680c303..591995e9b25 100644
--- a/test/DebugInfo/typedef.ll
+++ b/test/DebugInfo/typedef.ll
@@ -18,14 +18,14 @@
 !llvm.module.flags = !{!8, !9}
 !llvm.ident = !{!10}
 
-!0 = !MDCompileUnit(language: DW_LANG_C_plus_plus, producer: "clang version 3.5.0 ", isOptimized: false, emissionKind: 1, file: !1, enums: !2, retainedTypes: !2, subprograms: !2, globals: !3, imports: !2)
-!1 = !MDFile(filename: "typedef.cpp", directory: "/tmp/dbginfo")
+!0 = !DICompileUnit(language: DW_LANG_C_plus_plus, producer: "clang version 3.5.0 ", isOptimized: false, emissionKind: 1, file: !1, enums: !2, retainedTypes: !2, subprograms: !2, globals: !3, imports: !2)
+!1 = !DIFile(filename: "typedef.cpp", directory: "/tmp/dbginfo")
 !2 = !{}
 !3 = !{!4}
-!4 = !MDGlobalVariable(name: "y", line: 2, isLocal: false, isDefinition: true, scope: null, file: !5, type: !6, variable: i8** @y)
-!5 = !MDFile(filename: "typedef.cpp", directory: "/tmp/dbginfo")
-!6 = !MDDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, baseType: !7)
-!7 = !MDDerivedType(tag: DW_TAG_typedef, name: "x", line: 1, file: !1, baseType: null)
+!4 = !DIGlobalVariable(name: "y", line: 2, isLocal: false, isDefinition: true, scope: null, file: !5, type: !6, variable: i8** @y)
+!5 = !DIFile(filename: "typedef.cpp", directory: "/tmp/dbginfo")
+!6 = !DIDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, baseType: !7)
+!7 = !DIDerivedType(tag: DW_TAG_typedef, name: "x", line: 1, file: !1, baseType: null)
 !8 = !{i32 2, !"Dwarf Version", i32 4}
 !9 = !{i32 1, !"Debug Info Version", i32 3}
 !10 = !{!"clang version 3.5.0 "}
diff --git a/test/DebugInfo/unconditional-branch.ll b/test/DebugInfo/unconditional-branch.ll
index 1399bdcee1b..220a0e3dbe3 100644
--- a/test/DebugInfo/unconditional-branch.ll
+++ b/test/DebugInfo/unconditional-branch.ll
@@ -22,7 +22,7 @@ define void @foo(i32 %i) #0 {
 entry:
   %i.addr = alloca i32, align 4
   store i32 %i, i32* %i.addr, align 4
-  call void @llvm.dbg.declare(metadata i32* %i.addr, metadata !12, metadata !MDExpression()), !dbg !13
+  call void @llvm.dbg.declare(metadata i32* %i.addr, metadata !12, metadata !DIExpression()), !dbg !13
   %0 = load i32, i32* %i.addr, align 4, !dbg !14
   switch i32 %0, label %sw.default [
   ], !dbg !14
@@ -45,21 +45,21 @@ attributes #1 = { nounwind readnone }
 !llvm.module.flags = !{!9, !10}
 !llvm.ident = !{!11}
 
-!0 = !MDCompileUnit(language: DW_LANG_C99, producer: "clang version 3.5.0 (204712)", isOptimized: false, emissionKind: 1, file: !1, enums: !2, retainedTypes: !2, subprograms: !3, globals: !2, imports: !2)
-!1 = !MDFile(filename: "test.c", directory: "D:\5Cwork\5CEPRs\5C396363")
+!0 = !DICompileUnit(language: DW_LANG_C99, producer: "clang version 3.5.0 (204712)", isOptimized: false, emissionKind: 1, file: !1, enums: !2, retainedTypes: !2, subprograms: !3, globals: !2, imports: !2)
+!1 = !DIFile(filename: "test.c", directory: "D:\5Cwork\5CEPRs\5C396363")
 !2 = !{}
 !3 = !{!4}
-!4 = !MDSubprogram(name: "foo", line: 1, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 1, file: !1, scope: !5, type: !6, function: void (i32)* @foo, variables: !2)
-!5 = !MDFile(filename: "test.c", directory: "D:CworkCEPRsC396363")
-!6 = !MDSubroutineType(types: !7)
+!4 = !DISubprogram(name: "foo", line: 1, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 1, file: !1, scope: !5, type: !6, function: void (i32)* @foo, variables: !2)
+!5 = !DIFile(filename: "test.c", directory: "D:CworkCEPRsC396363")
+!6 = !DISubroutineType(types: !7)
 !7 = !{null, !8}
-!8 = !MDBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
+!8 = !DIBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
 !9 = !{i32 2, !"Dwarf Version", i32 4}
 !10 = !{i32 1, !"Debug Info Version", i32 3}
 !11 = !{!"clang version 3.5.0 (204712)"}
-!12 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "i", line: 1, arg: 1, scope: !4, file: !5, type: !8)
-!13 = !MDLocation(line: 1, scope: !4)
-!14 = !MDLocation(line: 2, scope: !4)
-!15 = !MDLocation(line: 4, scope: !16)
-!16 = distinct !MDLexicalBlock(line: 2, column: 0, file: !1, scope: !4)
-!17 = !MDLocation(line: 6, scope: !4)
+!12 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "i", line: 1, arg: 1, scope: !4, file: !5, type: !8)
+!13 = !DILocation(line: 1, scope: !4)
+!14 = !DILocation(line: 2, scope: !4)
+!15 = !DILocation(line: 4, scope: !16)
+!16 = distinct !DILexicalBlock(line: 2, column: 0, file: !1, scope: !4)
+!17 = !DILocation(line: 6, scope: !4)
diff --git a/test/DebugInfo/varargs.ll b/test/DebugInfo/varargs.ll
index 8bdfef5a993..93dbfa1e0ac 100644
--- a/test/DebugInfo/varargs.ll
+++ b/test/DebugInfo/varargs.ll
@@ -55,9 +55,9 @@ define void @_Z1biz(i32 %c, ...) #0 {
   %a = alloca %struct.A, align 1
   %fptr = alloca void (i32, ...)*, align 8
   store i32 %c, i32* %1, align 4
-  call void @llvm.dbg.declare(metadata i32* %1, metadata !21, metadata !MDExpression()), !dbg !22
-  call void @llvm.dbg.declare(metadata %struct.A* %a, metadata !23, metadata !MDExpression()), !dbg !24
-  call void @llvm.dbg.declare(metadata void (i32, ...)** %fptr, metadata !25, metadata !MDExpression()), !dbg !27
+  call void @llvm.dbg.declare(metadata i32* %1, metadata !21, metadata !DIExpression()), !dbg !22
+  call void @llvm.dbg.declare(metadata %struct.A* %a, metadata !23, metadata !DIExpression()), !dbg !24
+  call void @llvm.dbg.declare(metadata void (i32, ...)** %fptr, metadata !25, metadata !DIExpression()), !dbg !27
   store void (i32, ...)* @_Z1biz, void (i32, ...)** %fptr, align 8, !dbg !27
   ret void, !dbg !28
 }
@@ -72,30 +72,30 @@ attributes #1 = { nounwind readnone }
 !llvm.module.flags = !{!18, !19}
 !llvm.ident = !{!20}
 
-!0 = !MDCompileUnit(language: DW_LANG_C_plus_plus, producer: "clang version 3.5 ", isOptimized: false, emissionKind: 0, file: !1, enums: !2, retainedTypes: !3, subprograms: !13, globals: !2, imports: !2)
-!1 = !MDFile(filename: "llvm/tools/clang/test/CodeGenCXX/debug-info-varargs.cpp", directory: "radar/13690847")
+!0 = !DICompileUnit(language: DW_LANG_C_plus_plus, producer: "clang version 3.5 ", isOptimized: false, emissionKind: 0, file: !1, enums: !2, retainedTypes: !3, subprograms: !13, globals: !2, imports: !2)
+!1 = !DIFile(filename: "llvm/tools/clang/test/CodeGenCXX/debug-info-varargs.cpp", directory: "radar/13690847")
 !2 = !{}
 !3 = !{!4}
-!4 = !MDCompositeType(tag: DW_TAG_structure_type, name: "A", line: 3, size: 8, align: 8, file: !1, elements: !5, identifier: "_ZTS1A")
+!4 = !DICompositeType(tag: DW_TAG_structure_type, name: "A", line: 3, size: 8, align: 8, file: !1, elements: !5, identifier: "_ZTS1A")
 !5 = !{!6}
-!6 = !MDSubprogram(name: "a", linkageName: "_ZN1A1aEiz", line: 6, isLocal: false, isDefinition: false, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 6, file: !1, scope: !"_ZTS1A", type: !7)
-!7 = !MDSubroutineType(types: !8)
+!6 = !DISubprogram(name: "a", linkageName: "_ZN1A1aEiz", line: 6, isLocal: false, isDefinition: false, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 6, file: !1, scope: !"_ZTS1A", type: !7)
+!7 = !DISubroutineType(types: !8)
 !8 = !{null, !9, !10, null}
-!9 = !MDDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, flags: DIFlagArtificial | DIFlagObjectPointer, baseType: !"_ZTS1A")
-!10 = !MDBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
+!9 = !DIDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, flags: DIFlagArtificial | DIFlagObjectPointer, baseType: !"_ZTS1A")
+!10 = !DIBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
 !13 = !{!14}
-!14 = !MDSubprogram(name: "b", linkageName: "_Z1biz", line: 13, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 13, file: !1, scope: !15, type: !16, function: void (i32, ...)* @_Z1biz, variables: !2)
-!15 = !MDFile(filename: "llvm/tools/clang/test/CodeGenCXX/debug-info-varargs.cpp", directory: "radar/13690847")
-!16 = !MDSubroutineType(types: !17)
+!14 = !DISubprogram(name: "b", linkageName: "_Z1biz", line: 13, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 13, file: !1, scope: !15, type: !16, function: void (i32, ...)* @_Z1biz, variables: !2)
+!15 = !DIFile(filename: "llvm/tools/clang/test/CodeGenCXX/debug-info-varargs.cpp", directory: "radar/13690847")
+!16 = !DISubroutineType(types: !17)
 !17 = !{null, !10, null}
 !18 = !{i32 2, !"Dwarf Version", i32 2}
 !19 = !{i32 1, !"Debug Info Version", i32 3}
 !20 = !{!"clang version 3.5 "}
-!21 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "c", line: 13, arg: 1, scope: !14, file: !15, type: !10)
-!22 = !MDLocation(line: 13, scope: !14)
-!23 = !MDLocalVariable(tag: DW_TAG_auto_variable, name: "a", line: 16, scope: !14, file: !15, type: !4)
-!24 = !MDLocation(line: 16, scope: !14)
-!25 = !MDLocalVariable(tag: DW_TAG_auto_variable, name: "fptr", line: 18, scope: !14, file: !15, type: !26)
-!26 = !MDDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, baseType: !16)
-!27 = !MDLocation(line: 18, scope: !14)
-!28 = !MDLocation(line: 22, scope: !14)
+!21 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "c", line: 13, arg: 1, scope: !14, file: !15, type: !10)
+!22 = !DILocation(line: 13, scope: !14)
+!23 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "a", line: 16, scope: !14, file: !15, type: !4)
+!24 = !DILocation(line: 16, scope: !14)
+!25 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "fptr", line: 18, scope: !14, file: !15, type: !26)
+!26 = !DIDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, baseType: !16)
+!27 = !DILocation(line: 18, scope: !14)
+!28 = !DILocation(line: 22, scope: !14)
diff --git a/test/DebugInfo/version.ll b/test/DebugInfo/version.ll
index 635c0a7a955..f18a3e2871e 100644
--- a/test/DebugInfo/version.ll
+++ b/test/DebugInfo/version.ll
@@ -18,15 +18,15 @@ attributes #0 = { nounwind uwtable "less-precise-fpmad"="false" "no-frame-pointe
 !llvm.dbg.cu = !{!0}
 !llvm.module.flags = !{!9, !11}
 
-!0 = !MDCompileUnit(language: DW_LANG_C99, producer: "clang version 3.4 (trunk 185475)", isOptimized: false, emissionKind: 0, file: !1, enums: !2, retainedTypes: !2, subprograms: !3, globals: !2, imports: !2)
-!1 = !MDFile(filename: "CodeGen/dwarf-version.c", directory: "test")
+!0 = !DICompileUnit(language: DW_LANG_C99, producer: "clang version 3.4 (trunk 185475)", isOptimized: false, emissionKind: 0, file: !1, enums: !2, retainedTypes: !2, subprograms: !3, globals: !2, imports: !2)
+!1 = !DIFile(filename: "CodeGen/dwarf-version.c", directory: "test")
 !2 = !{}
 !3 = !{!4}
-!4 = !MDSubprogram(name: "main", line: 6, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 6, file: !1, scope: !5, type: !6, function: i32 ()* @main, variables: !2)
-!5 = !MDFile(filename: "CodeGen/dwarf-version.c", directory: "test")
-!6 = !MDSubroutineType(types: !7)
+!4 = !DISubprogram(name: "main", line: 6, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 6, file: !1, scope: !5, type: !6, function: i32 ()* @main, variables: !2)
+!5 = !DIFile(filename: "CodeGen/dwarf-version.c", directory: "test")
+!6 = !DISubroutineType(types: !7)
 !7 = !{!8}
-!8 = !MDBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
+!8 = !DIBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
 !9 = !{i32 2, !"Dwarf Version", i32 3}
-!10 = !MDLocation(line: 7, scope: !4)
+!10 = !DILocation(line: 7, scope: !4)
 !11 = !{i32 1, !"Debug Info Version", i32 3}
diff --git a/test/ExecutionEngine/MCJIT/Inputs/multi-module-eh-b.ll b/test/ExecutionEngine/MCJIT/Inputs/multi-module-eh-b.ll
index d7dbb032b5d..d7beeb61cb4 100644
--- a/test/ExecutionEngine/MCJIT/Inputs/multi-module-eh-b.ll
+++ b/test/ExecutionEngine/MCJIT/Inputs/multi-module-eh-b.ll
@@ -12,13 +12,13 @@ define void @throwException_B() {
   unreachable
 }
 
-define i32 @FB() {
+define i32 @FB() personality i8* bitcast (i32 (...)* @__gxx_personality_v0 to i8*) {
 entry:
   invoke void @throwException_B()
           to label %try.cont unwind label %lpad
 
 lpad:
-  %p = landingpad { i8*, i32 } personality i8* bitcast (i32 (...)* @__gxx_personality_v0 to i8*)
+  %p = landingpad { i8*, i32 }
           catch i8* bitcast (i8** @_ZTIi to i8*)
   %e = extractvalue { i8*, i32 } %p, 0
   call i8* @__cxa_begin_catch(i8* %e)
diff --git a/test/ExecutionEngine/MCJIT/cross-module-sm-pic-a.ll b/test/ExecutionEngine/MCJIT/cross-module-sm-pic-a.ll
index eb414249bdc..04331990db3 100644
--- a/test/ExecutionEngine/MCJIT/cross-module-sm-pic-a.ll
+++ b/test/ExecutionEngine/MCJIT/cross-module-sm-pic-a.ll
@@ -1,5 +1,5 @@
 ; RUN: %lli -extra-module=%p/Inputs/cross-module-b.ll -relocation-model=pic -code-model=small %s > /dev/null
-; XFAIL: mips, i686, i386
+; XFAIL: mips-, mipsel-, i686, i386
 
 declare i32 @FB()
 
diff --git a/test/ExecutionEngine/MCJIT/eh-lg-pic.ll b/test/ExecutionEngine/MCJIT/eh-lg-pic.ll
index 9277ec4a8ca..cd4834b3f12 100644
--- a/test/ExecutionEngine/MCJIT/eh-lg-pic.ll
+++ b/test/ExecutionEngine/MCJIT/eh-lg-pic.ll
@@ -1,5 +1,5 @@
 ; RUN: %lli -relocation-model=pic -code-model=large %s
-; XFAIL: cygwin, win32, mingw, mips, i686, i386, aarch64, arm, asan, msan
+; XFAIL: cygwin, win32, mingw, mips-, mipsel-, i686, i386, aarch64, arm, asan, msan
 declare i8* @__cxa_allocate_exception(i64)
 declare void @__cxa_throw(i8*, i8*, i8*)
 declare i32 @__gxx_personality_v0(...)
@@ -14,13 +14,13 @@ define void @throwException() {
   unreachable
 }
 
-define i32 @main() {
+define i32 @main() personality i8* bitcast (i32 (...)* @__gxx_personality_v0 to i8*) {
 entry:
   invoke void @throwException()
           to label %try.cont unwind label %lpad
 
 lpad:
-  %p = landingpad { i8*, i32 } personality i8* bitcast (i32 (...)* @__gxx_personality_v0 to i8*)
+  %p = landingpad { i8*, i32 }
           catch i8* bitcast (i8** @_ZTIi to i8*)
   %e = extractvalue { i8*, i32 } %p, 0
   call i8* @__cxa_begin_catch(i8* %e)
diff --git a/test/ExecutionEngine/MCJIT/eh-sm-pic.ll b/test/ExecutionEngine/MCJIT/eh-sm-pic.ll
index 37fb628cf0a..24d8b2ceb4f 100644
--- a/test/ExecutionEngine/MCJIT/eh-sm-pic.ll
+++ b/test/ExecutionEngine/MCJIT/eh-sm-pic.ll
@@ -1,5 +1,5 @@
 ; RUN: %lli -relocation-model=pic -code-model=small %s
-; XFAIL: cygwin, win32, mingw, mips, i686, i386, darwin, aarch64, arm, asan, msan
+; XFAIL: cygwin, win32, mingw, mips-, mipsel-, i686, i386, darwin, aarch64, arm, asan, msan
 declare i8* @__cxa_allocate_exception(i64)
 declare void @__cxa_throw(i8*, i8*, i8*)
 declare i32 @__gxx_personality_v0(...)
@@ -14,13 +14,13 @@ define void @throwException() {
   unreachable
 }
 
-define i32 @main() {
+define i32 @main() personality i8* bitcast (i32 (...)* @__gxx_personality_v0 to i8*) {
 entry:
   invoke void @throwException()
           to label %try.cont unwind label %lpad
 
 lpad:
-  %p = landingpad { i8*, i32 } personality i8* bitcast (i32 (...)* @__gxx_personality_v0 to i8*)
+  %p = landingpad { i8*, i32 }
           catch i8* bitcast (i8** @_ZTIi to i8*)
   %e = extractvalue { i8*, i32 } %p, 0
   call i8* @__cxa_begin_catch(i8* %e)
diff --git a/test/ExecutionEngine/MCJIT/eh.ll b/test/ExecutionEngine/MCJIT/eh.ll
index 9f73e3a9937..b301b64767c 100644
--- a/test/ExecutionEngine/MCJIT/eh.ll
+++ b/test/ExecutionEngine/MCJIT/eh.ll
@@ -14,13 +14,13 @@ define void @throwException() {
   unreachable
 }
 
-define i32 @main() {
+define i32 @main() personality i8* bitcast (i32 (...)* @__gxx_personality_v0 to i8*) {
 entry:
   invoke void @throwException()
           to label %try.cont unwind label %lpad
 
 lpad:
-  %p = landingpad { i8*, i32 } personality i8* bitcast (i32 (...)* @__gxx_personality_v0 to i8*)
+  %p = landingpad { i8*, i32 }
           catch i8* bitcast (i8** @_ZTIi to i8*)
   %e = extractvalue { i8*, i32 } %p, 0
   call i8* @__cxa_begin_catch(i8* %e)
diff --git a/test/ExecutionEngine/MCJIT/multi-module-eh-a.ll b/test/ExecutionEngine/MCJIT/multi-module-eh-a.ll
index 8626626e75f..50ed321a0d6 100644
--- a/test/ExecutionEngine/MCJIT/multi-module-eh-a.ll
+++ b/test/ExecutionEngine/MCJIT/multi-module-eh-a.ll
@@ -16,13 +16,13 @@ define void @throwException() {
   unreachable
 }
 
-define i32 @main() {
+define i32 @main() personality i8* bitcast (i32 (...)* @__gxx_personality_v0 to i8*) {
 entry:
   invoke void @throwException()
           to label %try.cont unwind label %lpad
 
 lpad:
-  %p = landingpad { i8*, i32 } personality i8* bitcast (i32 (...)* @__gxx_personality_v0 to i8*)
+  %p = landingpad { i8*, i32 }
           catch i8* bitcast (i8** @_ZTIi to i8*)
   %e = extractvalue { i8*, i32 } %p, 0
   call i8* @__cxa_begin_catch(i8* %e)
diff --git a/test/ExecutionEngine/MCJIT/multi-module-sm-pic-a.ll b/test/ExecutionEngine/MCJIT/multi-module-sm-pic-a.ll
index 01faecc6495..7e5710dbf9c 100644
--- a/test/ExecutionEngine/MCJIT/multi-module-sm-pic-a.ll
+++ b/test/ExecutionEngine/MCJIT/multi-module-sm-pic-a.ll
@@ -1,5 +1,5 @@
 ; RUN: %lli -extra-module=%p/Inputs/multi-module-b.ll -extra-module=%p/Inputs/multi-module-c.ll -relocation-model=pic -code-model=small %s > /dev/null
-; XFAIL: mips, i686, i386
+; XFAIL: mips-, mipsel-, i686, i386
 
 declare i32 @FB()
 
diff --git a/test/ExecutionEngine/MCJIT/remote/cross-module-sm-pic-a.ll b/test/ExecutionEngine/MCJIT/remote/cross-module-sm-pic-a.ll
index d47fc6cffbb..415fd25409d 100644
--- a/test/ExecutionEngine/MCJIT/remote/cross-module-sm-pic-a.ll
+++ b/test/ExecutionEngine/MCJIT/remote/cross-module-sm-pic-a.ll
@@ -1,5 +1,5 @@
 ; RUN: %lli -extra-module=%p/Inputs/cross-module-b.ll -disable-lazy-compilation=true -remote-mcjit -mcjit-remote-process=lli-child-target%exeext -relocation-model=pic -code-model=small %s > /dev/null
-; XFAIL: mips, i686, i386, arm
+; XFAIL: mips-, mipsel-, i686, i386, arm
 
 declare i32 @FB()
 
diff --git a/test/ExecutionEngine/MCJIT/remote/multi-module-sm-pic-a.ll b/test/ExecutionEngine/MCJIT/remote/multi-module-sm-pic-a.ll
index d248c4b2a94..3e020dc853a 100644
--- a/test/ExecutionEngine/MCJIT/remote/multi-module-sm-pic-a.ll
+++ b/test/ExecutionEngine/MCJIT/remote/multi-module-sm-pic-a.ll
@@ -1,5 +1,5 @@
 ; RUN: %lli -extra-module=%p/Inputs/multi-module-b.ll -extra-module=%p/Inputs/multi-module-c.ll -disable-lazy-compilation=true -remote-mcjit -mcjit-remote-process=lli-child-target%exeext -relocation-model=pic -code-model=small %s > /dev/null
-; XFAIL: mips, i686, i386, arm
+; XFAIL: mips-, mipsel-, i686, i386, arm
 
 declare i32 @FB()
 
diff --git a/test/ExecutionEngine/MCJIT/remote/test-global-init-nonzero-sm-pic.ll b/test/ExecutionEngine/MCJIT/remote/test-global-init-nonzero-sm-pic.ll
index f9184b4077b..e350b85a8bc 100644
--- a/test/ExecutionEngine/MCJIT/remote/test-global-init-nonzero-sm-pic.ll
+++ b/test/ExecutionEngine/MCJIT/remote/test-global-init-nonzero-sm-pic.ll
@@ -1,5 +1,5 @@
 ; RUN: %lli -remote-mcjit -relocation-model=pic -code-model=small %s > /dev/null
-; XFAIL: mips, aarch64, arm, i686, i386
+; XFAIL: mips-, mipsel-, aarch64, arm, i686, i386
 
 @count = global i32 1, align 4
 
diff --git a/test/ExecutionEngine/MCJIT/remote/test-ptr-reloc-sm-pic.ll b/test/ExecutionEngine/MCJIT/remote/test-ptr-reloc-sm-pic.ll
index d9a4faa8e6b..7162e927de0 100644
--- a/test/ExecutionEngine/MCJIT/remote/test-ptr-reloc-sm-pic.ll
+++ b/test/ExecutionEngine/MCJIT/remote/test-ptr-reloc-sm-pic.ll
@@ -1,5 +1,5 @@
 ; RUN: %lli -remote-mcjit -O0 -relocation-model=pic -code-model=small %s
-; XFAIL: mips, aarch64, arm, i686, i386
+; XFAIL: mips-, mipsel-, aarch64, arm, i686, i386
 
 @.str = private unnamed_addr constant [6 x i8] c"data1\00", align 1
 @ptr = global i8* getelementptr inbounds ([6 x i8], [6 x i8]* @.str, i32 0, i32 0), align 4
diff --git a/test/ExecutionEngine/MCJIT/stubs-sm-pic.ll b/test/ExecutionEngine/MCJIT/stubs-sm-pic.ll
index 6eebe44e3ff..46545ce5342 100644
--- a/test/ExecutionEngine/MCJIT/stubs-sm-pic.ll
+++ b/test/ExecutionEngine/MCJIT/stubs-sm-pic.ll
@@ -1,5 +1,5 @@
 ; RUN: %lli -disable-lazy-compilation=false -relocation-model=pic -code-model=small %s
-; XFAIL: mips, i686, i386, aarch64, arm
+; XFAIL: mips-, mipsel-, i686, i386, aarch64, arm
 
 define i32 @main() nounwind {
 entry:
diff --git a/test/ExecutionEngine/MCJIT/test-global-init-nonzero-sm-pic.ll b/test/ExecutionEngine/MCJIT/test-global-init-nonzero-sm-pic.ll
index 8ae496d0fab..e54135fc7cb 100644
--- a/test/ExecutionEngine/MCJIT/test-global-init-nonzero-sm-pic.ll
+++ b/test/ExecutionEngine/MCJIT/test-global-init-nonzero-sm-pic.ll
@@ -1,5 +1,5 @@
 ; RUN: %lli -relocation-model=pic -code-model=small %s > /dev/null
-; XFAIL: mips, aarch64, arm, i686, i386
+; XFAIL: mips-, mipsel-, aarch64, arm, i686, i386
 
 @count = global i32 1, align 4
 
diff --git a/test/ExecutionEngine/MCJIT/test-ptr-reloc-sm-pic.ll b/test/ExecutionEngine/MCJIT/test-ptr-reloc-sm-pic.ll
index d50ba9dd5a0..eea6fde2e2c 100644
--- a/test/ExecutionEngine/MCJIT/test-ptr-reloc-sm-pic.ll
+++ b/test/ExecutionEngine/MCJIT/test-ptr-reloc-sm-pic.ll
@@ -1,5 +1,5 @@
 ; RUN: %lli -O0 -relocation-model=pic -code-model=small %s
-; XFAIL: mips, aarch64, arm, i686, i386
+; XFAIL: mips-, mipsel-, aarch64, arm, i686, i386
 
 @.str = private unnamed_addr constant [6 x i8] c"data1\00", align 1
 @ptr = global i8* getelementptr inbounds ([6 x i8], [6 x i8]* @.str, i32 0, i32 0), align 4
diff --git a/test/ExecutionEngine/OrcLazy/hello.ll b/test/ExecutionEngine/OrcLazy/hello.ll
index 795224e019b..c3cf0f346a1 100644
--- a/test/ExecutionEngine/OrcLazy/hello.ll
+++ b/test/ExecutionEngine/OrcLazy/hello.ll
@@ -1,7 +1,7 @@
 ; RUN: lli -jit-kind=orc-lazy -orc-lazy-debug=funcs-to-stdout %s | FileCheck %s
 ;
 ; CHECK: Hello
-; CHECK: [ {{.*}}main$orc_body ]
+; CHECK: [ {{.*}}main ]
 ; CHECK: Goodbye
 
 %class.Foo = type { i8 }
diff --git a/test/ExecutionEngine/OrcMCJIT/Inputs/multi-module-eh-b.ll b/test/ExecutionEngine/OrcMCJIT/Inputs/multi-module-eh-b.ll
index d7dbb032b5d..d7beeb61cb4 100644
--- a/test/ExecutionEngine/OrcMCJIT/Inputs/multi-module-eh-b.ll
+++ b/test/ExecutionEngine/OrcMCJIT/Inputs/multi-module-eh-b.ll
@@ -12,13 +12,13 @@ define void @throwException_B() {
   unreachable
 }
 
-define i32 @FB() {
+define i32 @FB() personality i8* bitcast (i32 (...)* @__gxx_personality_v0 to i8*) {
 entry:
   invoke void @throwException_B()
           to label %try.cont unwind label %lpad
 
 lpad:
-  %p = landingpad { i8*, i32 } personality i8* bitcast (i32 (...)* @__gxx_personality_v0 to i8*)
+  %p = landingpad { i8*, i32 }
           catch i8* bitcast (i8** @_ZTIi to i8*)
   %e = extractvalue { i8*, i32 } %p, 0
   call i8* @__cxa_begin_catch(i8* %e)
diff --git a/test/ExecutionEngine/OrcMCJIT/cross-module-sm-pic-a.ll b/test/ExecutionEngine/OrcMCJIT/cross-module-sm-pic-a.ll
index 4f5afd0ed47..9af82e7486f 100644
--- a/test/ExecutionEngine/OrcMCJIT/cross-module-sm-pic-a.ll
+++ b/test/ExecutionEngine/OrcMCJIT/cross-module-sm-pic-a.ll
@@ -1,5 +1,5 @@
 ; RUN: %lli -jit-kind=orc-mcjit -extra-module=%p/Inputs/cross-module-b.ll -relocation-model=pic -code-model=small %s > /dev/null
-; XFAIL: mips, i686, i386
+; XFAIL: mips-, mipsel-, i686, i386
 
 declare i32 @FB()
 
diff --git a/test/ExecutionEngine/OrcMCJIT/eh-lg-pic.ll b/test/ExecutionEngine/OrcMCJIT/eh-lg-pic.ll
index 32b53c4847b..936d7eebe09 100644
--- a/test/ExecutionEngine/OrcMCJIT/eh-lg-pic.ll
+++ b/test/ExecutionEngine/OrcMCJIT/eh-lg-pic.ll
@@ -1,5 +1,5 @@
 ; RUN: %lli -jit-kind=orc-mcjit -relocation-model=pic -code-model=large %s
-; XFAIL: cygwin, win32, mingw, mips, i686, i386, aarch64, arm, asan, msan
+; XFAIL: cygwin, win32, mingw, mips-, mipsel-, i686, i386, aarch64, arm, asan, msan
 declare i8* @__cxa_allocate_exception(i64)
 declare void @__cxa_throw(i8*, i8*, i8*)
 declare i32 @__gxx_personality_v0(...)
@@ -14,13 +14,13 @@ define void @throwException() {
   unreachable
 }
 
-define i32 @main() {
+define i32 @main() personality i8* bitcast (i32 (...)* @__gxx_personality_v0 to i8*) {
 entry:
   invoke void @throwException()
           to label %try.cont unwind label %lpad
 
 lpad:
-  %p = landingpad { i8*, i32 } personality i8* bitcast (i32 (...)* @__gxx_personality_v0 to i8*)
+  %p = landingpad { i8*, i32 }
           catch i8* bitcast (i8** @_ZTIi to i8*)
   %e = extractvalue { i8*, i32 } %p, 0
   call i8* @__cxa_begin_catch(i8* %e)
diff --git a/test/ExecutionEngine/OrcMCJIT/eh-sm-pic.ll b/test/ExecutionEngine/OrcMCJIT/eh-sm-pic.ll
index a54795ead56..02279226dd5 100644
--- a/test/ExecutionEngine/OrcMCJIT/eh-sm-pic.ll
+++ b/test/ExecutionEngine/OrcMCJIT/eh-sm-pic.ll
@@ -1,5 +1,5 @@
 ; RUN: %lli -jit-kind=orc-mcjit -relocation-model=pic -code-model=small %s
-; XFAIL: cygwin, win32, mingw, mips, i686, i386, darwin, aarch64, arm, asan, msan
+; XFAIL: cygwin, win32, mingw, mips-, mipsel-, i686, i386, darwin, aarch64, arm, asan, msan
 declare i8* @__cxa_allocate_exception(i64)
 declare void @__cxa_throw(i8*, i8*, i8*)
 declare i32 @__gxx_personality_v0(...)
@@ -14,13 +14,13 @@ define void @throwException() {
   unreachable
 }
 
-define i32 @main() {
+define i32 @main() personality i8* bitcast (i32 (...)* @__gxx_personality_v0 to i8*) {
 entry:
   invoke void @throwException()
           to label %try.cont unwind label %lpad
 
 lpad:
-  %p = landingpad { i8*, i32 } personality i8* bitcast (i32 (...)* @__gxx_personality_v0 to i8*)
+  %p = landingpad { i8*, i32 }
           catch i8* bitcast (i8** @_ZTIi to i8*)
   %e = extractvalue { i8*, i32 } %p, 0
   call i8* @__cxa_begin_catch(i8* %e)
diff --git a/test/ExecutionEngine/OrcMCJIT/eh.ll b/test/ExecutionEngine/OrcMCJIT/eh.ll
index 2de6a517b2b..8a1b4d8f6df 100644
--- a/test/ExecutionEngine/OrcMCJIT/eh.ll
+++ b/test/ExecutionEngine/OrcMCJIT/eh.ll
@@ -14,13 +14,13 @@ define void @throwException() {
   unreachable
 }
 
-define i32 @main() {
+define i32 @main() personality i8* bitcast (i32 (...)* @__gxx_personality_v0 to i8*) {
 entry:
   invoke void @throwException()
           to label %try.cont unwind label %lpad
 
 lpad:
-  %p = landingpad { i8*, i32 } personality i8* bitcast (i32 (...)* @__gxx_personality_v0 to i8*)
+  %p = landingpad { i8*, i32 }
           catch i8* bitcast (i8** @_ZTIi to i8*)
   %e = extractvalue { i8*, i32 } %p, 0
   call i8* @__cxa_begin_catch(i8* %e)
diff --git a/test/ExecutionEngine/OrcMCJIT/multi-module-eh-a.ll b/test/ExecutionEngine/OrcMCJIT/multi-module-eh-a.ll
index f77cb44a878..ccde9aefe8e 100644
--- a/test/ExecutionEngine/OrcMCJIT/multi-module-eh-a.ll
+++ b/test/ExecutionEngine/OrcMCJIT/multi-module-eh-a.ll
@@ -16,13 +16,13 @@ define void @throwException() {
   unreachable
 }
 
-define i32 @main() {
+define i32 @main() personality i8* bitcast (i32 (...)* @__gxx_personality_v0 to i8*) {
 entry:
   invoke void @throwException()
           to label %try.cont unwind label %lpad
 
 lpad:
-  %p = landingpad { i8*, i32 } personality i8* bitcast (i32 (...)* @__gxx_personality_v0 to i8*)
+  %p = landingpad { i8*, i32 }
           catch i8* bitcast (i8** @_ZTIi to i8*)
   %e = extractvalue { i8*, i32 } %p, 0
   call i8* @__cxa_begin_catch(i8* %e)
diff --git a/test/ExecutionEngine/OrcMCJIT/multi-module-sm-pic-a.ll b/test/ExecutionEngine/OrcMCJIT/multi-module-sm-pic-a.ll
index 80b8e163d5d..0052c012652 100644
--- a/test/ExecutionEngine/OrcMCJIT/multi-module-sm-pic-a.ll
+++ b/test/ExecutionEngine/OrcMCJIT/multi-module-sm-pic-a.ll
@@ -1,5 +1,5 @@
 ; RUN: %lli -jit-kind=orc-mcjit -extra-module=%p/Inputs/multi-module-b.ll -extra-module=%p/Inputs/multi-module-c.ll -relocation-model=pic -code-model=small %s > /dev/null
-; XFAIL: mips, i686, i386
+; XFAIL: mips-, mipsel-, i686, i386
 
 declare i32 @FB()
 
diff --git a/test/ExecutionEngine/OrcMCJIT/remote/cross-module-sm-pic-a.ll b/test/ExecutionEngine/OrcMCJIT/remote/cross-module-sm-pic-a.ll
index c8979043857..4326fc1e526 100644
--- a/test/ExecutionEngine/OrcMCJIT/remote/cross-module-sm-pic-a.ll
+++ b/test/ExecutionEngine/OrcMCJIT/remote/cross-module-sm-pic-a.ll
@@ -1,5 +1,5 @@
 ; RUN: %lli -jit-kind=orc-mcjit -extra-module=%p/Inputs/cross-module-b.ll -disable-lazy-compilation=true -remote-mcjit -mcjit-remote-process=lli-child-target%exeext -relocation-model=pic -code-model=small %s > /dev/null
-; XFAIL: mips, i686, i386, arm
+; XFAIL: mips-, mipsel-, i686, i386, arm
 
 declare i32 @FB()
 
diff --git a/test/ExecutionEngine/OrcMCJIT/remote/multi-module-sm-pic-a.ll b/test/ExecutionEngine/OrcMCJIT/remote/multi-module-sm-pic-a.ll
index 41812d25169..18a2d7e87d0 100644
--- a/test/ExecutionEngine/OrcMCJIT/remote/multi-module-sm-pic-a.ll
+++ b/test/ExecutionEngine/OrcMCJIT/remote/multi-module-sm-pic-a.ll
@@ -1,5 +1,5 @@
 ; RUN: %lli -jit-kind=orc-mcjit -extra-module=%p/Inputs/multi-module-b.ll -extra-module=%p/Inputs/multi-module-c.ll -disable-lazy-compilation=true -remote-mcjit -mcjit-remote-process=lli-child-target%exeext -relocation-model=pic -code-model=small %s > /dev/null
-; XFAIL: mips, i686, i386, arm
+; XFAIL: mips-, mipsel-, i686, i386, arm
 
 declare i32 @FB()
 
diff --git a/test/ExecutionEngine/OrcMCJIT/remote/test-global-init-nonzero-sm-pic.ll b/test/ExecutionEngine/OrcMCJIT/remote/test-global-init-nonzero-sm-pic.ll
index ec9e1ae1a03..f1e93133b22 100644
--- a/test/ExecutionEngine/OrcMCJIT/remote/test-global-init-nonzero-sm-pic.ll
+++ b/test/ExecutionEngine/OrcMCJIT/remote/test-global-init-nonzero-sm-pic.ll
@@ -1,5 +1,5 @@
 ; RUN: %lli -jit-kind=orc-mcjit -remote-mcjit -relocation-model=pic -code-model=small %s > /dev/null
-; XFAIL: mips, aarch64, arm, i686, i386
+; XFAIL: mips-, mipsel-, aarch64, arm, i686, i386
 
 @count = global i32 1, align 4
 
diff --git a/test/ExecutionEngine/OrcMCJIT/remote/test-ptr-reloc-sm-pic.ll b/test/ExecutionEngine/OrcMCJIT/remote/test-ptr-reloc-sm-pic.ll
index 61321ac1b94..cac800ef5e7 100644
--- a/test/ExecutionEngine/OrcMCJIT/remote/test-ptr-reloc-sm-pic.ll
+++ b/test/ExecutionEngine/OrcMCJIT/remote/test-ptr-reloc-sm-pic.ll
@@ -1,5 +1,5 @@
 ; RUN: %lli -jit-kind=orc-mcjit -remote-mcjit -O0 -relocation-model=pic -code-model=small %s
-; XFAIL: mips, aarch64, arm, i686, i386
+; XFAIL: mips-, mipsel-, aarch64, arm, i686, i386
 
 @.str = private unnamed_addr constant [6 x i8] c"data1\00", align 1
 @ptr = global i8* getelementptr inbounds ([6 x i8], [6 x i8]* @.str, i32 0, i32 0), align 4
diff --git a/test/ExecutionEngine/OrcMCJIT/stubs-sm-pic.ll b/test/ExecutionEngine/OrcMCJIT/stubs-sm-pic.ll
index f354a0de093..ec4183bc724 100644
--- a/test/ExecutionEngine/OrcMCJIT/stubs-sm-pic.ll
+++ b/test/ExecutionEngine/OrcMCJIT/stubs-sm-pic.ll
@@ -1,5 +1,5 @@
 ; RUN: %lli -jit-kind=orc-mcjit -disable-lazy-compilation=false -relocation-model=pic -code-model=small %s
-; XFAIL: mips, i686, i386, aarch64, arm
+; XFAIL: mips-, mipsel-, i686, i386, aarch64, arm
 
 define i32 @main() nounwind {
 entry:
diff --git a/test/ExecutionEngine/OrcMCJIT/test-global-init-nonzero-sm-pic.ll b/test/ExecutionEngine/OrcMCJIT/test-global-init-nonzero-sm-pic.ll
index 7f66aec7707..c9b6cf755b7 100644
--- a/test/ExecutionEngine/OrcMCJIT/test-global-init-nonzero-sm-pic.ll
+++ b/test/ExecutionEngine/OrcMCJIT/test-global-init-nonzero-sm-pic.ll
@@ -1,5 +1,5 @@
 ; RUN: %lli -jit-kind=orc-mcjit -relocation-model=pic -code-model=small %s > /dev/null
-; XFAIL: mips, aarch64, arm, i686, i386
+; XFAIL: mips-, mipsel-, aarch64, arm, i686, i386
 
 @count = global i32 1, align 4
 
diff --git a/test/ExecutionEngine/OrcMCJIT/test-ptr-reloc-sm-pic.ll b/test/ExecutionEngine/OrcMCJIT/test-ptr-reloc-sm-pic.ll
index 6e0a425623b..d0bb070110c 100644
--- a/test/ExecutionEngine/OrcMCJIT/test-ptr-reloc-sm-pic.ll
+++ b/test/ExecutionEngine/OrcMCJIT/test-ptr-reloc-sm-pic.ll
@@ -1,5 +1,5 @@
 ; RUN: %lli -jit-kind=orc-mcjit -O0 -relocation-model=pic -code-model=small %s
-; XFAIL: mips, aarch64, arm, i686, i386
+; XFAIL: mips-, mipsel-, aarch64, arm, i686, i386
 
 @.str = private unnamed_addr constant [6 x i8] c"data1\00", align 1
 @ptr = global i8* getelementptr inbounds ([6 x i8], [6 x i8]* @.str, i32 0, i32 0), align 4
diff --git a/test/ExecutionEngine/RuntimeDyld/ARM/MachO_ARM_PIC_relocations.s b/test/ExecutionEngine/RuntimeDyld/ARM/MachO_ARM_PIC_relocations.s
index 7ff3a897576..1f5ec673597 100644
--- a/test/ExecutionEngine/RuntimeDyld/ARM/MachO_ARM_PIC_relocations.s
+++ b/test/ExecutionEngine/RuntimeDyld/ARM/MachO_ARM_PIC_relocations.s
@@ -40,6 +40,13 @@ insn4:
         .comm   baz, 4, 2
         .comm   foo, 4, 2
 
+	.section        __DATA,__data
+	.globl  _a
+	.align  2
+# rtdyld-check: *{4}bar_ofs = bar + 4
+bar_ofs:
+	.long   bar + 4
+
 # Check that the symbol pointer section entries are fixed up properly:
 # rtdyld-check: *{4}foo$non_lazy_ptr = foo
         .section	__DATA,__nl_symbol_ptr,non_lazy_symbol_pointers
diff --git a/test/ExecutionEngine/RuntimeDyld/Mips/ELF_Mips64r2N64_PIC_relocations.s b/test/ExecutionEngine/RuntimeDyld/Mips/ELF_Mips64r2N64_PIC_relocations.s
new file mode 100644
index 00000000000..1d8d293a26a
--- /dev/null
+++ b/test/ExecutionEngine/RuntimeDyld/Mips/ELF_Mips64r2N64_PIC_relocations.s
@@ -0,0 +1,164 @@
+# RUN: llvm-mc -triple=mips64el-unknown-linux -relocation-model=pic -code-model=small -filetype=obj -o %T/test_ELF_Mips64N64.o %s
+# RUN: llc -mtriple=mips64el-unknown-linux -relocation-model=pic -filetype=obj -o %T/test_ELF_ExternalFunction_Mips64N64.o %S/Inputs/ExternalFunction.ll
+# RUN: llvm-rtdyld -triple=mips64el-unknown-linux -verify -map-section test_ELF_Mips64N64.o,.text=0x1000 -map-section test_ELF_ExternalFunction_Mips64N64.o,.text=0x10000 -check=%s %/T/test_ELF_Mips64N64.o %T/test_ELF_ExternalFunction_Mips64N64.o
+
+	.data
+# Test R_MIPS_PC32 relocation.
+# rtdyld-check: *{4}(R_MIPS_PC32) = (foo - R_MIPS_PC32)[31:0]
+R_MIPS_PC32:
+	.word foo-.
+# rtdyld-check: *{4}(R_MIPS_PC32 + 4) = (foo - tmp1)[31:0]
+tmp1:
+	.4byte foo-tmp1
+
+	.text
+	.abicalls
+	.section	.mdebug.abi64,"",@progbits
+	.nan	legacy
+	.file	"ELF_Mips64N64_PIC_relocations.ll"
+	.text
+	.globl	bar
+	.align	3
+	.type	bar,@function
+	.set	nomicromips
+	.set	nomips16
+	.ent	bar
+bar:
+	.frame	$fp,40,$ra
+	.mask 	0x00000000,0
+	.fmask	0x00000000,0
+	.set	noreorder
+	.set	nomacro
+	.set	noat
+	daddiu	$sp, $sp, -40
+	sd	$ra, 32($sp)
+	sd	$fp, 24($sp)
+	move	 $fp, $sp
+	sd	$4, 16($fp)
+	lb	$2, 0($4)
+	sd	$4, 8($fp)
+
+# Test R_MIPS_26 relocation.
+# rtdyld-check:  decode_operand(insn1, 0)[25:0] = foo
+insn1:
+	jal   foo
+	nop
+
+# Test R_MIPS_PC16 relocation.
+# rtdyld-check:  decode_operand(insn2, 1)[15:0] = foo - insn2
+insn2:
+	bal   foo
+	nop
+
+	move	 $sp, $fp
+	ld	$ra, 32($sp)
+	ld	$fp, 24($sp)
+	daddiu	$sp, $sp, 32
+	jr	$ra
+	nop
+	.set	at
+	.set	macro
+	.set	reorder
+	.end	bar
+$func_end0:
+	.size	bar, ($func_end0)-bar
+
+	.globl	main
+	.align	3
+	.type	main,@function
+	.set	nomicromips
+	.set	nomips16
+	.ent	main
+main:
+	.frame	$fp,32,$ra
+	.mask 	0x00000000,0
+	.fmask	0x00000000,0
+	.set	noreorder
+	.set	nomacro
+	.set	noat
+	daddiu	$sp, $sp, -32
+	sd	$ra, 24($sp)
+	sd	$fp, 16($sp)
+	sd	$gp, 8($sp)
+	move	 $fp, $sp
+
+# Check upper 16-bits of offset between the address of main function
+# and the global offset table.
+# rtdyld-check:  decode_operand(insn3, 1)[15:0] = ((section_addr(test_ELF_Mips64N64.o, .got) + 0x7ff0) - main + 0x8000)[31:16]
+insn3:
+	lui	$1, %hi(%neg(%gp_rel(main)))
+	daddu	$1, $1, $25
+
+# Check lower 16-bits of offset between the address of main function
+# and the global offset table.
+# rtdyld-check:  decode_operand(insn4, 2)[15:0] = ((section_addr(test_ELF_Mips64N64.o, .got) + 0x7ff0) - main)[15:0]
+insn4:
+	daddiu	$1, $1, %lo(%neg(%gp_rel(main)))
+	sw	$zero, 4($fp)
+
+# $gp register contains address of the .got section + 0x7FF0. 0x7FF0 is
+# the offset of $gp from the beginning of the .got section. Check that we are
+# loading address of the page pointer from correct offset. In this case
+# the page pointer is the first entry in the .got section, so offset will be
+# 0 - 0x7FF0.
+# rtdyld-check:  decode_operand(insn5, 2)[15:0] = 0x8010
+#
+# Check that the global offset table contains the page pointer.
+# rtdyld-check: *{8}(section_addr(test_ELF_Mips64N64.o, .got)) = (_str + 0x8000) & 0xffffffffffff0000
+insn5:
+	ld	$25, %got_page(_str)($1)
+
+# Check the offset of _str from the page pointer.
+# rtdyld-check:  decode_operand(insn6, 2)[15:0] = _str[15:0]
+insn6:
+	daddiu	$25, $25, %got_ofst(_str)
+
+# Check that we are loading address of var from correct offset. In this case
+# var is the second entry in the .got section, so offset will be 8 - 0x7FF0.
+# rtdyld-check:  decode_operand(insn7, 2)[15:0] = 0x8018
+#
+# Check that the global offset table contains the address of the var.
+# rtdyld-check: *{8}(section_addr(test_ELF_Mips64N64.o, .got) + 8) = var
+insn7:
+	ld	$2, %got_disp(var)($1)
+	sd	$25, 0($2)
+
+# Check that we are loading address of bar from correct offset. In this case
+# bar is the third entry in the .got section, so offset will be 16 - 0x7FF0.
+# rtdyld-check:  decode_operand(insn8, 2)[15:0] = 0x8020
+#
+# Check that the global offset table contains the address of the bar.
+# rtdyld-check: *{8}(section_addr(test_ELF_Mips64N64.o, .got) + 16) = bar
+insn8:
+	ld	$2, %call16(bar)($1)
+
+	move	 $4, $25
+	move	 $gp, $1
+	move	 $25, $2
+	jalr	$25
+	nop
+	move	 $sp, $fp
+	ld	$gp, 8($sp)
+	ld	$fp, 16($sp)
+	ld	$ra, 24($sp)
+	daddiu	$sp, $sp, 32
+	jr	$ra
+	nop
+	.set	at
+	.set	macro
+	.set	reorder
+	.end	main
+$func_end1:
+	.size	main, ($func_end1)-main
+
+	.type	_str,@object
+	.section	.rodata.str1.1,"aMS",@progbits,1
+_str:
+	.asciz	"test"
+	.size	_str, 5
+
+	.type	var,@object
+	.comm	var,8,8
+
+	.section	".note.GNU-stack","",@progbits
+	.text
diff --git a/test/ExecutionEngine/RuntimeDyld/Mips/ELF_O32R6_relocations.s b/test/ExecutionEngine/RuntimeDyld/Mips/ELF_O32R6_relocations.s
new file mode 100644
index 00000000000..3d20db21964
--- /dev/null
+++ b/test/ExecutionEngine/RuntimeDyld/Mips/ELF_O32R6_relocations.s
@@ -0,0 +1,49 @@
+# RUN: llvm-mc -triple=mipsel-unknown-linux -mcpu=mips32r6 -relocation-model=pic -code-model=small -filetype=obj -o %T/test_ELF_O32R6.o %s
+# RUN: llc -mtriple=mipsel-unknown-linux -mcpu=mips32r6 -relocation-model=pic -filetype=obj -o %T/test_ELF_ExternalFunction_O32R6.o %S/Inputs/ExternalFunction.ll
+# RUN: llvm-rtdyld -triple=mipsel-unknown-linux -mcpu=mips32r6 -verify -map-section test_ELF_O32R6.o,.text=0x1000 -map-section test_ELF_ExternalFunction_O32R6.o,.text=0x10000 -check=%s %/T/test_ELF_O32R6.o %T/test_ELF_ExternalFunction_O32R6.o
+
+# RUN: llvm-mc -triple=mips-unknown-linux -mcpu=mips32r6 -relocation-model=pic -code-model=small -filetype=obj -o %T/test_ELF_O32R6.o %s
+# RUN: llc -mtriple=mips-unknown-linux -mcpu=mips32r6 -relocation-model=pic -filetype=obj -o %T/test_ELF_ExternalFunction_O32R6.o %S/Inputs/ExternalFunction.ll
+# RUN: llvm-rtdyld -triple=mips-unknown-linux -mcpu=mips32r6 -verify -map-section test_ELF_O32R6.o,.text=0x1000 -map-section test_ELF_ExternalFunction_O32R6.o,.text=0x10000 -check=%s %/T/test_ELF_O32R6.o %T/test_ELF_ExternalFunction_O32R6.o
+
+	.text
+	.abicalls
+	.nan	2008
+	.text
+	.set	nomicromips
+	.set	nomips16
+	.set	noreorder
+	.set	nomacro
+	.set	noat
+
+	.align	3
+	.globl	bar
+	.type	bar,@function
+
+bar:
+# Test R_MIPS_PC19_S2 relocation.
+# rtdyld-check:  decode_operand(R_MIPS_PC19_S2, 1)[20:0] = (foo - R_MIPS_PC19_S2)[20:0]
+R_MIPS_PC19_S2:
+	lwpc $6,foo
+
+# Test R_MIPS_PC21_S2 relocation.
+# rtdyld-check:  decode_operand(R_MIPS_PC21_S2, 1)[22:0] = (foo - next_pc(R_MIPS_PC21_S2))[22:0]
+R_MIPS_PC21_S2:
+	bnezc	$5,foo
+
+# Test R_MIPS_PC26_S2 relocation.
+# rtdyld-check:  decode_operand(R_MIPS_PC26_S2, 0)[27:0] = (foo - next_pc(R_MIPS_PC26_S2))[27:0]
+R_MIPS_PC26_S2:
+	balc	foo
+
+# Test R_MIPS_PCHI16 relocation.
+# rtdyld-check:  decode_operand(R_MIPS_PCHI16, 1)[15:0] = (foo - R_MIPS_PCHI16 + 0x8000)[31:16]
+R_MIPS_PCHI16:
+	aluipc $5, %pcrel_hi(foo)
+
+# Test R_MIPS_PCLO16 relocation.
+# rtdyld-check:  decode_operand(R_MIPS_PCLO16, 2)[15:0] = (foo - R_MIPS_PCLO16)[15:0]
+R_MIPS_PCLO16:
+	addiu  $5, $5, %pcrel_lo(foo)
+
+	.size	bar, .-bar
diff --git a/test/ExecutionEngine/RuntimeDyld/Mips/ELF_O32_PIC_relocations.s b/test/ExecutionEngine/RuntimeDyld/Mips/ELF_O32_PIC_relocations.s
new file mode 100644
index 00000000000..6c47262246a
--- /dev/null
+++ b/test/ExecutionEngine/RuntimeDyld/Mips/ELF_O32_PIC_relocations.s
@@ -0,0 +1,55 @@
+# RUN: llvm-mc -triple=mipsel-unknown-linux -relocation-model=pic -code-model=small -filetype=obj -o %T/test_ELF_O32.o %s
+# RUN: llc -mtriple=mipsel-unknown-linux -relocation-model=pic -filetype=obj -o %T/test_ELF_ExternalFunction_O32.o %S/Inputs/ExternalFunction.ll
+# RUN: llvm-rtdyld -triple=mipsel-unknown-linux -verify -map-section test_ELF_O32.o,.text=0x1000 -map-section test_ELF_ExternalFunction_O32.o,.text=0x10000 -check=%s %T/test_ELF_O32.o %T/test_ELF_ExternalFunction_O32.o
+
+# RUN: llvm-mc -triple=mips-unknown-linux -relocation-model=pic -code-model=small -filetype=obj -o %T/test_ELF_O32.o %s
+# RUN: llc -mtriple=mips-unknown-linux -relocation-model=pic -filetype=obj -o %/T/test_ELF_ExternalFunction_O32.o %S/Inputs/ExternalFunction.ll
+# RUN: llvm-rtdyld -triple=mips-unknown-linux -verify -map-section test_ELF_O32.o,.text=0x1000 -map-section test_ELF_ExternalFunction_O32.o,.text=0x10000 -check=%s %T/test_ELF_O32.o %T/test_ELF_ExternalFunction_O32.o
+
+        .data
+# rtdyld-check: *{4}R_MIPS_32 = foo[31:0]
+R_MIPS_32:
+        .word foo
+# rtdyld-check: *{4}(R_MIPS_32+4) = foo[31:0]
+        .4byte foo
+# rtdyld-check: *{4}(R_MIPS_PC32) = (foo - R_MIPS_PC32)[31:0]
+R_MIPS_PC32:
+        .word foo-.
+# rtdyld-check: *{4}(R_MIPS_PC32 + 4) = (foo - tmp1)[31:0]
+tmp1:
+        .4byte foo-tmp1
+
+	.text
+	.abicalls
+	.nan	legacy
+	.text
+	.set	nomicromips
+	.set	nomips16
+	.set	noreorder
+	.set	nomacro
+	.set	noat
+
+	.align	3
+	.globl	bar
+	.type	bar,@function
+bar:
+# rtdyld-check:  decode_operand(R_MIPS_26, 0)[27:0] = stub_addr(test_ELF_O32.o, .text, foo)[27:0]
+# rtdyld-check:  decode_operand(R_MIPS_26, 0)[1:0] = 0
+R_MIPS_26:
+	j   foo
+	nop
+
+# rtdyld-check:  decode_operand(R_MIPS_PC16, 1)[17:0] = (foo - R_MIPS_PC16)[17:0]
+R_MIPS_PC16:
+	bal   foo
+	nop
+
+# rtdyld-check:  decode_operand(R_MIPS_HI16, 1)[15:0] = foo[31:16]
+R_MIPS_HI16:
+	lui	$1, %hi(foo)
+
+# rtdyld-check:  decode_operand(R_MIPS_LO16, 1)[15:0] = foo[15:0]
+R_MIPS_LO16:
+	lui	$1, %lo(foo)
+
+	.size	bar, .-bar
diff --git a/test/ExecutionEngine/RuntimeDyld/Mips/Inputs/ExternalFunction.ll b/test/ExecutionEngine/RuntimeDyld/Mips/Inputs/ExternalFunction.ll
new file mode 100644
index 00000000000..a59d68c1e18
--- /dev/null
+++ b/test/ExecutionEngine/RuntimeDyld/Mips/Inputs/ExternalFunction.ll
@@ -0,0 +1,4 @@
+define void @foo() {
+entry:
+  ret void
+}
diff --git a/test/ExecutionEngine/RuntimeDyld/Mips/lit.local.cfg b/test/ExecutionEngine/RuntimeDyld/Mips/lit.local.cfg
new file mode 100644
index 00000000000..a3183a25afa
--- /dev/null
+++ b/test/ExecutionEngine/RuntimeDyld/Mips/lit.local.cfg
@@ -0,0 +1,3 @@
+if not 'Mips' in config.root.targets:
+    config.unsupported = True
+
diff --git a/test/ExecutionEngine/RuntimeDyld/X86/MachO_i386_DynNoPIC_relocations.s b/test/ExecutionEngine/RuntimeDyld/X86/MachO_i386_DynNoPIC_relocations.s
index f427b985b58..17c332a11c3 100644
--- a/test/ExecutionEngine/RuntimeDyld/X86/MachO_i386_DynNoPIC_relocations.s
+++ b/test/ExecutionEngine/RuntimeDyld/X86/MachO_i386_DynNoPIC_relocations.s
@@ -1,7 +1,10 @@
 # RUN: llvm-mc -triple=i386-apple-macosx10.4 -relocation-model=dynamic-no-pic -filetype=obj -o %T/test_i386.o %s
 # RUN: llvm-rtdyld -triple=i386-apple-macosx10.4 -verify -check=%s %/T/test_i386.o
 
-	.section	__TEXT,__text,regular,pure_instructions
+// Put the section used in the test at a non zero address.
+	.long 4
+
+	.section	__TEXT,__text2,regular,pure_instructions
 	.globl	bar
 	.align	4, 0x90
 bar:
@@ -9,9 +12,9 @@ bar:
 tmp0$pb:
 	popl	%eax
 # Test section difference relocation to non-lazy ptr section.
-# rtdyld-check: decode_operand(inst1, 4) = x$non_lazy_ptr - tmp0$pb
+# rtdyld-check: decode_operand(inst1, 4) = x$non_lazy_ptr - tmp0$pb + 8
 inst1:
-	movl	x$non_lazy_ptr-tmp0$pb(%eax), %eax
+	movl	(x$non_lazy_ptr-tmp0$pb)+8(%eax), %eax
         movl    (%eax), %ebx
 
 # Test VANILLA relocation to jump table.
diff --git a/test/ExecutionEngine/RuntimeDyld/X86/MachO_x86-64_PIC_relocations.s b/test/ExecutionEngine/RuntimeDyld/X86/MachO_x86-64_PIC_relocations.s
index f28e4d24599..2ef8cc439df 100644
--- a/test/ExecutionEngine/RuntimeDyld/X86/MachO_x86-64_PIC_relocations.s
+++ b/test/ExecutionEngine/RuntimeDyld/X86/MachO_x86-64_PIC_relocations.s
@@ -1,5 +1,5 @@
 # RUN: llvm-mc -triple=x86_64-apple-macosx10.9 -relocation-model=pic -filetype=obj -o %T/test_x86-64.o %s
-# RUN: llvm-rtdyld -triple=x86_64-apple-macosx10.9 -verify -check=%s %/T/test_x86-64.o
+# RUN: llvm-rtdyld -triple=x86_64-apple-macosx10.9 -dummy-extern ds1=0xfffffffffffffffe -dummy-extern ds2=0xffffffffffffffff -verify -check=%s %/T/test_x86-64.o
 
         .section	__TEXT,__text,regular,pure_instructions
 	.globl	foo
@@ -46,4 +46,15 @@ eh_frame_test:
 x:
         .long   5
 
+# Test dummy-extern relocation.
+# rtdyld-check: *{8}z1 = ds1
+z1:
+        .quad   ds1
+
+# Test external-symbol relocation bypass: symbols with addr 0xffffffffffffffff
+# don't have their relocations applied.
+# rtdyld-check: *{8}z2 = 0
+z2:
+        .quad   ds2
+
 .subsections_via_symbols
diff --git a/test/Feature/callingconventions.ll b/test/Feature/callingconventions.ll
index 8b339d43fcd..9aafb36c757 100644
--- a/test/Feature/callingconventions.ll
+++ b/test/Feature/callingconventions.ll
@@ -25,7 +25,7 @@ define coldcc void @bar2() {
         ret void
 }
 
-define cc42 void @bar3() {
+define cc42 void @bar3() personality i32 (...)* @__gxx_personality_v0 {
   invoke fastcc void @foo( )
     to label %Ok unwind label %U
 
@@ -33,12 +33,12 @@ Ok:
   ret void
 
 U:
-  %exn = landingpad {i8*, i32} personality i32 (...)* @__gxx_personality_v0
+  %exn = landingpad {i8*, i32}
             cleanup
   resume { i8*, i32 } %exn
 }
 
-define void @bar4() {
+define void @bar4() personality i32 (...)* @__gxx_personality_v0 {
   call cc42 void @bar( )
   invoke cc42 void @bar3( )
     to label %Ok unwind label %U
@@ -47,7 +47,7 @@ Ok:
   ret void
 
 U:
-  %exn = landingpad {i8*, i32} personality i32 (...)* @__gxx_personality_v0
+  %exn = landingpad {i8*, i32}
             cleanup
   resume { i8*, i32 } %exn
 }
diff --git a/test/Feature/calltest.ll b/test/Feature/calltest.ll
index e7d8e8d00b3..a53c3a1215a 100644
--- a/test/Feature/calltest.ll
+++ b/test/Feature/calltest.ll
@@ -10,7 +10,7 @@ define void @invoke(%FunTy* %x) {
   ret void
 }
 
-define i32 @main(i32 %argc) {
+define i32 @main(i32 %argc) personality i32 (...)* @__gxx_personality_v0 {
   %retval = call i32 @test( i32 %argc )           ; <i32> [#uses=2]
   %two = add i32 %retval, %retval         ; <i32> [#uses=1]
   %retval2 = invoke i32 @test( i32 %argc )
@@ -22,7 +22,7 @@ Next:
   ret i32 %two2
 
 Error:
-  %exn = landingpad {i8*, i32} personality i32 (...)* @__gxx_personality_v0
+  %exn = landingpad {i8*, i32}
             cleanup
   ret i32 -1
 }
diff --git a/test/Feature/exception.ll b/test/Feature/exception.ll
index 6e18a81bcfc..027f9619470 100644
--- a/test/Feature/exception.ll
+++ b/test/Feature/exception.ll
@@ -6,7 +6,7 @@
 @_ZTId = external constant i8*
 @_ZTIPKc = external constant i8*
 
-define void @_Z3barv() uwtable optsize ssp {
+define void @_Z3barv() uwtable optsize ssp personality i32 (...)* @__gxx_personality_v0 {
 entry:
   invoke void @_Z3quxv() optsize
           to label %try.cont unwind label %lpad
@@ -15,7 +15,7 @@ try.cont:                                         ; preds = %entry, %invoke.cont
   ret void
 
 lpad:                                             ; preds = %entry
-  %exn = landingpad {i8*, i32} personality i32 (...)* @__gxx_personality_v0
+  %exn = landingpad {i8*, i32}
             cleanup
             catch i8** @_ZTIc
             filter [2 x i8**] [i8** @_ZTIPKc, i8** @_ZTId]
@@ -25,3 +25,74 @@ lpad:                                             ; preds = %entry
 declare void @_Z3quxv() optsize
 
 declare i32 @__gxx_personality_v0(...)
+
+define void @cleanupret0() personality i32 (...)* @__gxx_personality_v0 {
+entry:
+  br label %bb
+bb:
+  cleanuppad void [i7 4]
+  cleanupret i8 0 unwind label %bb
+}
+
+define void @cleanupret1() personality i32 (...)* @__gxx_personality_v0 {
+entry:
+  br label %bb
+bb:
+  cleanuppad void [i7 4]
+  cleanupret void unwind label %bb
+}
+
+define void @cleanupret2() personality i32 (...)* @__gxx_personality_v0 {
+entry:
+  cleanupret i8 0 unwind to caller
+}
+
+define void @cleanupret3() personality i32 (...)* @__gxx_personality_v0 {
+  cleanupret void unwind to caller
+}
+
+define void @catchret() personality i32 (...)* @__gxx_personality_v0 {
+entry:
+  br label %bb
+bb:
+  catchret label %bb
+}
+
+define i8 @catchpad() personality i32 (...)* @__gxx_personality_v0 {
+entry:
+  br label %bb2
+bb:
+  ret i8 %cbv
+bb2:
+  %cbv = catchpad i8 [i7 4] to label %bb unwind label %bb2
+}
+
+define void @terminatepad0() personality i32 (...)* @__gxx_personality_v0 {
+entry:
+  br label %bb
+bb:
+  terminatepad [i7 4] unwind label %bb
+}
+
+define void @terminatepad1() personality i32 (...)* @__gxx_personality_v0 {
+entry:
+  terminatepad [i7 4] unwind to caller
+}
+
+define void @cleanuppad() personality i32 (...)* @__gxx_personality_v0 {
+entry:
+  cleanuppad void [i7 4]
+  ret void
+}
+
+define void @catchendpad0() personality i32 (...)* @__gxx_personality_v0 {
+entry:
+  br label %bb
+bb:
+  catchendpad unwind label %bb
+}
+
+define void @catchendpad1() personality i32 (...)* @__gxx_personality_v0 {
+entry:
+  catchendpad unwind to caller
+}
diff --git a/test/Feature/seh-nounwind.ll b/test/Feature/seh-nounwind.ll
index 203471649df..2afd1004647 100644
--- a/test/Feature/seh-nounwind.ll
+++ b/test/Feature/seh-nounwind.ll
@@ -11,13 +11,13 @@ entry:
   ret i32 %div
 }
 
-define i32 @main() nounwind {
+define i32 @main() nounwind personality i8* bitcast (i32 (...)* @__C_specific_handler to i8*) {
 entry:
   %call = invoke i32 @div(i32 10, i32 0)
           to label %__try.cont unwind label %lpad
 
 lpad:
-  %0 = landingpad { i8*, i32 } personality i8* bitcast (i32 (...)* @__C_specific_handler to i8*)
+  %0 = landingpad { i8*, i32 }
           catch i8* null
   br label %__try.cont
 
diff --git a/test/Instrumentation/AddressSanitizer/X86/asm_attr.ll b/test/Instrumentation/AddressSanitizer/X86/asm_attr.ll
index 0667a147487..95e4125bc74 100644
--- a/test/Instrumentation/AddressSanitizer/X86/asm_attr.ll
+++ b/test/Instrumentation/AddressSanitizer/X86/asm_attr.ll
@@ -4,8 +4,8 @@ target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
 
 ; CHECK-LABEL: mov_no_attr
-; CHECK-NOT: callq __asan_report_load@PLT
-; CHECK-NOT: callq __asan_report_store@PLT
+; CHECK-NOT: callq __asan_report_load8@PLT
+; CHECK-NOT: callq __asan_report_store8@PLT
 define void @mov_no_attr(i64* %dst, i64* %src) {
   tail call void asm sideeffect "movq ($1), %rax  \0A\09movq %rax, ($0)  \0A\09", "r,r,~{memory},~{rax},~{dirflag},~{fpsr},~{flags}"(i64* %dst, i64* %src)
   ret void
diff --git a/test/Instrumentation/AddressSanitizer/X86/asm_more_registers_than_available.ll b/test/Instrumentation/AddressSanitizer/X86/asm_more_registers_than_available.ll
new file mode 100644
index 00000000000..7827f3fbf27
--- /dev/null
+++ b/test/Instrumentation/AddressSanitizer/X86/asm_more_registers_than_available.ll
@@ -0,0 +1,56 @@
+; RUN: opt < %s -asan -S -o %t.ll
+; RUN: FileCheck %s < %t.ll
+
+; Don't do stack malloc on functions containing inline assembly on 64-bit
+; platforms. It makes LLVM run out of registers.
+
+; CHECK-LABEL: define void @TestAbsenceOfStackMalloc(i8* %S, i32 %pS, i8* %D, i32 %pD, i32 %h)
+; CHECK: %MyAlloca
+; CHECK-NOT: call {{.*}} @__asan_stack_malloc
+
+target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-apple-macosx10.10.0"
+
+define void @TestAbsenceOfStackMalloc(i8* %S, i32 %pS, i8* %D, i32 %pD, i32 %h) #0 {
+entry:
+  %S.addr = alloca i8*, align 8
+  %pS.addr = alloca i32, align 4
+  %D.addr = alloca i8*, align 8
+  %pD.addr = alloca i32, align 4
+  %h.addr = alloca i32, align 4
+  %sr = alloca i32, align 4
+  %pDiffD = alloca i32, align 4
+  %pDiffS = alloca i32, align 4
+  %flagSA = alloca i8, align 1
+  %flagDA = alloca i8, align 1
+  store i8* %S, i8** %S.addr, align 8
+  store i32 %pS, i32* %pS.addr, align 4
+  store i8* %D, i8** %D.addr, align 8
+  store i32 %pD, i32* %pD.addr, align 4
+  store i32 %h, i32* %h.addr, align 4
+  store i32 4, i32* %sr, align 4
+  %0 = load i32, i32* %pD.addr, align 4
+  %sub = sub i32 %0, 5
+  store i32 %sub, i32* %pDiffD, align 4
+  %1 = load i32, i32* %pS.addr, align 4
+  %shl = shl i32 %1, 1
+  %sub1 = sub i32 %shl, 5
+  store i32 %sub1, i32* %pDiffS, align 4
+  %2 = load i32, i32* %pS.addr, align 4
+  %and = and i32 %2, 15
+  %cmp = icmp eq i32 %and, 0
+  %conv = zext i1 %cmp to i32
+  %conv2 = trunc i32 %conv to i8
+  store i8 %conv2, i8* %flagSA, align 1
+  %3 = load i32, i32* %pD.addr, align 4
+  %and3 = and i32 %3, 15
+  %cmp4 = icmp eq i32 %and3, 0
+  %conv5 = zext i1 %cmp4 to i32
+  %conv6 = trunc i32 %conv5 to i8
+  store i8 %conv6, i8* %flagDA, align 1
+  call void asm sideeffect "mov\09\09\09$0,\09\09\09\09\09\09\09\09\09\09%rsi\0Amov\09\09\09$2,\09\09\09\09\09\09\09\09\09\09%rcx\0Amov\09\09\09$1,\09\09\09\09\09\09\09\09\09\09%rdi\0Amov\09\09\09$8,\09\09\09\09\09\09\09\09\09\09%rax\0A", "*m,*m,*m,*m,*m,*m,*m,*m,*m,~{rsi},~{rdi},~{rax},~{rcx},~{rdx},~{memory},~{dirflag},~{fpsr},~{flags}"(i8** %S.addr, i8** %D.addr, i32* %pS.addr, i32* %pDiffS, i32* %pDiffD, i32* %sr, i8* %flagSA, i8* %flagDA, i32* %h.addr) #1
+  ret void
+}
+
+attributes #0 = { nounwind sanitize_address }
+attributes #1 = { nounwind }
diff --git a/test/Instrumentation/AddressSanitizer/basic.ll b/test/Instrumentation/AddressSanitizer/basic.ll
index 21153aeaa64..9c4d416a1ef 100644
--- a/test/Instrumentation/AddressSanitizer/basic.ll
+++ b/test/Instrumentation/AddressSanitizer/basic.ll
@@ -4,6 +4,7 @@
 
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64"
 target triple = "x86_64-unknown-linux-gnu"
+; CHECK: @llvm.global_ctors = {{.*}}@asan.module_ctor
 
 define i32 @test_load(i32* %a) sanitize_address {
 ; CHECK-LABEL: @test_load
@@ -169,5 +170,8 @@ define void @memintr_test(i8* %a, i8* %b) nounwind uwtable sanitize_address {
 ; CHECK: __asan_memcpy
 ; CHECK: ret void
 
+; CHECK: define internal void @asan.module_ctor()
+; CHECK: call void @__asan_init()
+
 ; PROF
 ; CHECK: ![[PROF]] = !{!"branch_weights", i32 1, i32 100000}
diff --git a/test/Instrumentation/AddressSanitizer/debug_info.ll b/test/Instrumentation/AddressSanitizer/debug_info.ll
index 86703443309..19d627857ab 100644
--- a/test/Instrumentation/AddressSanitizer/debug_info.ll
+++ b/test/Instrumentation/AddressSanitizer/debug_info.ll
@@ -11,8 +11,8 @@ entry:
   %p.addr = alloca i32, align 4
   %r = alloca i32, align 4
   store volatile i32 %p, i32* %p.addr, align 4
-  call void @llvm.dbg.declare(metadata i32* %p.addr, metadata !10, metadata !MDExpression()), !dbg !11
-  call void @llvm.dbg.declare(metadata i32* %r, metadata !12, metadata !MDExpression()), !dbg !14
+  call void @llvm.dbg.declare(metadata i32* %p.addr, metadata !10, metadata !DIExpression()), !dbg !11
+  call void @llvm.dbg.declare(metadata i32* %r, metadata !12, metadata !DIExpression()), !dbg !14
   %0 = load i32, i32* %p.addr, align 4, !dbg !14
   %add = add nsw i32 %0, 1, !dbg !14
   store volatile i32 %add, i32* %r, align 4, !dbg !14
@@ -33,30 +33,30 @@ declare void @llvm.dbg.declare(metadata, metadata, metadata) nounwind readnone
 !llvm.dbg.cu = !{!0}
 !llvm.module.flags = !{!17}
 
-!0 = !MDCompileUnit(language: DW_LANG_C_plus_plus, producer: "clang version 3.3 (trunk 169314)", isOptimized: true, emissionKind: 0, file: !16, enums: !1, retainedTypes: !1, subprograms: !3, globals: !1)
+!0 = !DICompileUnit(language: DW_LANG_C_plus_plus, producer: "clang version 3.3 (trunk 169314)", isOptimized: true, emissionKind: 0, file: !16, enums: !1, retainedTypes: !1, subprograms: !3, globals: !1)
 !1 = !{}
 !3 = !{!5}
-!5 = !MDSubprogram(name: "zzz", linkageName: "_Z3zzzi", line: 1, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 1, file: !16, scope: !6, type: !7, function: i32 (i32)* @_Z3zzzi, variables: !1)
-!6 = !MDFile(filename: "a.cc", directory: "/usr/local/google/llvm_cmake_clang/tmp/debuginfo")
-!7 = !MDSubroutineType(types: !8)
+!5 = !DISubprogram(name: "zzz", linkageName: "_Z3zzzi", line: 1, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 1, file: !16, scope: !6, type: !7, function: i32 (i32)* @_Z3zzzi, variables: !1)
+!6 = !DIFile(filename: "a.cc", directory: "/usr/local/google/llvm_cmake_clang/tmp/debuginfo")
+!7 = !DISubroutineType(types: !8)
 !8 = !{!9, !9}
-!9 = !MDBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
-!10 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "p", line: 1, arg: 1, scope: !5, file: !6, type: !9)
-!11 = !MDLocation(line: 1, scope: !5)
-!12 = !MDLocalVariable(tag: DW_TAG_auto_variable, name: "r", line: 2, scope: !13, file: !6, type: !9)
+!9 = !DIBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
+!10 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "p", line: 1, arg: 1, scope: !5, file: !6, type: !9)
+!11 = !DILocation(line: 1, scope: !5)
+!12 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "r", line: 2, scope: !13, file: !6, type: !9)
 
 ; Verify that debug descriptors for argument and local variable will be replaced
 ; with descriptors that end with OpDeref (encoded as 2).
-;   CHECK: ![[ARG_ID]] = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "p", arg: 1,{{.*}} line: 1
-;   CHECK: ![[OPDEREF]] = !MDExpression(DW_OP_deref)
-;   CHECK: ![[VAR_ID]] = !MDLocalVariable(tag: DW_TAG_auto_variable, name: "r",{{.*}} line: 2
+;   CHECK: ![[ARG_ID]] = !DILocalVariable(tag: DW_TAG_arg_variable, name: "p", arg: 1,{{.*}} line: 1
+;   CHECK: ![[OPDEREF]] = !DIExpression(DW_OP_deref)
+;   CHECK: ![[VAR_ID]] = !DILocalVariable(tag: DW_TAG_auto_variable, name: "r",{{.*}} line: 2
 ; Verify that there are no more variable descriptors.
-;   CHECK-NOT: !MDLocalVariable(tag: DW_TAG_arg_variable
-;   CHECK-NOT: !MDLocalVariable(tag: DW_TAG_auto_variable
+;   CHECK-NOT: !DILocalVariable(tag: DW_TAG_arg_variable
+;   CHECK-NOT: !DILocalVariable(tag: DW_TAG_auto_variable
 
 
-!13 = distinct !MDLexicalBlock(line: 1, column: 0, file: !16, scope: !5)
-!14 = !MDLocation(line: 2, scope: !13)
-!15 = !MDLocation(line: 3, scope: !13)
-!16 = !MDFile(filename: "a.cc", directory: "/usr/local/google/llvm_cmake_clang/tmp/debuginfo")
+!13 = distinct !DILexicalBlock(line: 1, column: 0, file: !16, scope: !5)
+!14 = !DILocation(line: 2, scope: !13)
+!15 = !DILocation(line: 3, scope: !13)
+!16 = !DIFile(filename: "a.cc", directory: "/usr/local/google/llvm_cmake_clang/tmp/debuginfo")
 !17 = !{i32 1, !"Debug Info Version", i32 3}
diff --git a/test/Instrumentation/AddressSanitizer/debug_info_noninstrumented_alloca.ll b/test/Instrumentation/AddressSanitizer/debug_info_noninstrumented_alloca.ll
new file mode 100644
index 00000000000..844da79f708
--- /dev/null
+++ b/test/Instrumentation/AddressSanitizer/debug_info_noninstrumented_alloca.ll
@@ -0,0 +1,39 @@
+; This test checks that non-instrumented allocas stay in the first basic block.
+; Only first-basic-block allocas are considered stack slots, and moving them
+; breaks debug info.
+
+; RUN: opt < %s -asan -asan-module -S | FileCheck %s
+
+target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-apple-macosx10.10.0"
+
+define i32 @foo() sanitize_address {
+entry:
+  ; Won't be instrumented because of asan-skip-promotable-allocas.
+  %non_instrumented1 = alloca i32, align 4
+
+  ; Regular alloca, will get instrumented (forced by the ptrtoint below).
+  %instrumented = alloca i32, align 4
+
+  ; Won't be instrumented because of asan-skip-promotable-allocas.
+  %non_instrumented2 = alloca i32, align 4
+
+  br label %bb0
+
+bb0:
+  ; Won't be instrumented because of asan-skip-promotable-allocas.
+  %non_instrumented3 = alloca i32, align 4
+
+  %ptr = ptrtoint i32* %instrumented to i32
+  br label %bb1
+
+bb1:
+  ret i32 %ptr
+}
+
+; CHECK: entry:
+; CHECK: %non_instrumented1 = alloca i32, align 4
+; CHECK: %non_instrumented2 = alloca i32, align 4
+; CHECK: load i32, i32* @__asan_option_detect_stack_use_after_return
+; CHECK: bb0:
+; CHECK: %non_instrumented3 = alloca i32, align 4
diff --git a/test/Instrumentation/AddressSanitizer/do-not-instrument-llvm-metadata-darwin.ll b/test/Instrumentation/AddressSanitizer/do-not-instrument-llvm-metadata-darwin.ll
new file mode 100644
index 00000000000..f67155a29c2
--- /dev/null
+++ b/test/Instrumentation/AddressSanitizer/do-not-instrument-llvm-metadata-darwin.ll
@@ -0,0 +1,15 @@
+; This test checks that we are not instrumenting globals in llvm.metadata
+; and other llvm internal globals.
+; RUN: opt < %s -asan -asan-module -S | FileCheck %s
+
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-apple-macosx10.10.0"
+
+@.str_noinst = private unnamed_addr constant [4 x i8] c"aaa\00", section "llvm.metadata"
+@.str_noinst_prof = private unnamed_addr constant [4 x i8] c"aaa\00", section "__DATA,__llvm_covmap"
+@.str_inst = private unnamed_addr constant [4 x i8] c"aaa\00"
+
+; CHECK-NOT: {{asan_gen.*str_noinst}}
+; CHECK-NOT: {{asan_gen.*str_noinst_prof}}
+; CHECK: {{asan_gen.*str_inst}}
+; CHECK: @asan.module_ctor
diff --git a/test/Instrumentation/AddressSanitizer/do-not-instrument-llvm-metadata.ll b/test/Instrumentation/AddressSanitizer/do-not-instrument-llvm-metadata.ll
index d02f12aec98..93eca5bfd82 100644
--- a/test/Instrumentation/AddressSanitizer/do-not-instrument-llvm-metadata.ll
+++ b/test/Instrumentation/AddressSanitizer/do-not-instrument-llvm-metadata.ll
@@ -1,12 +1,15 @@
-; This test checks that we are not instrumenting globals in llvm.metadata.
+; This test checks that we are not instrumenting globals in llvm.metadata
+; and other llvm internal globals.
 ; RUN: opt < %s -asan -asan-module -S | FileCheck %s
 
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
 
 @.str_noinst = private unnamed_addr constant [4 x i8] c"aaa\00", section "llvm.metadata"
+@.str_noinst_prof = private unnamed_addr constant [4 x i8] c"aaa\00", section "__llvm_prf_data"
 @.str_inst = private unnamed_addr constant [4 x i8] c"aaa\00"
 
 ; CHECK-NOT: {{asan_gen.*str_noinst}}
+; CHECK-NOT: {{asan_gen.*str_noinst_prof}}
 ; CHECK: {{asan_gen.*str_inst}}
 ; CHECK: @asan.module_ctor
diff --git a/test/Instrumentation/AddressSanitizer/global_metadata.ll b/test/Instrumentation/AddressSanitizer/global_metadata.ll
index 2d09073a637..6cf1ee8e8d4 100644
--- a/test/Instrumentation/AddressSanitizer/global_metadata.ll
+++ b/test/Instrumentation/AddressSanitizer/global_metadata.ll
@@ -16,9 +16,9 @@ target triple = "x86_64-unknown-linux-gnu"
 ; CHECK: @.str = internal unnamed_addr constant { [14 x i8], [50 x i8] } { [14 x i8] c"Hello, world!\00", [50 x i8] zeroinitializer }, align 32
 
 ; Check emitted location descriptions:
-; CHECK: [[VARNAME:@__asan_gen_[0-9]+]] = private unnamed_addr constant [7 x i8] c"global\00", align 1
-; CHECK: [[FILENAME:@__asan_gen_[0-9]+]] = private unnamed_addr constant [22 x i8] c"/tmp/asan-globals.cpp\00", align 1
-; CHECK: [[LOCDESCR:@__asan_gen_[0-9]+]] = private unnamed_addr constant { [22 x i8]*, i32, i32 } { [22 x i8]* [[FILENAME]], i32 5, i32 5 }
+; CHECK: [[VARNAME:@__asan_gen_.[0-9]+]] = private unnamed_addr constant [7 x i8] c"global\00", align 1
+; CHECK: [[FILENAME:@__asan_gen_.[0-9]+]] = private unnamed_addr constant [22 x i8] c"/tmp/asan-globals.cpp\00", align 1
+; CHECK: [[LOCDESCR:@__asan_gen_.[0-9]+]] = private unnamed_addr constant { [22 x i8]*, i32, i32 } { [22 x i8]* [[FILENAME]], i32 5, i32 5 }
 
 ; Check that location decriptors and global names were passed into __asan_register_globals:
 ; CHECK: i64 ptrtoint ([7 x i8]* [[VARNAME]] to i64)
diff --git a/test/Instrumentation/AddressSanitizer/instrument-dynamic-allocas.ll b/test/Instrumentation/AddressSanitizer/instrument-dynamic-allocas.ll
index 18a86a9b88f..ceaf0e6fcfb 100644
--- a/test/Instrumentation/AddressSanitizer/instrument-dynamic-allocas.ll
+++ b/test/Instrumentation/AddressSanitizer/instrument-dynamic-allocas.ll
@@ -2,17 +2,13 @@
 ;   -asan-instrument-allocas=1
 
 ; RUN: opt < %s -asan -asan-module -asan-instrument-allocas=1 -S | FileCheck %s --check-prefix=CHECK-ALLOCA
-; RUN: opt < %s -asan -asan-module -asan-instrument-allocas=0 -S | FileCheck %s --check-prefix=CHECK-NOALLOCA
-; RUN: opt < %s -asan -asan-module -S | FileCheck %s --check-prefix=CHECK-NOALLOCA
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64"
 target triple = "x86_64-unknown-linux-gnu"
 
 define void @foo(i32 %len) sanitize_address {
 entry:
-; CHECK-ALLOCA: store i32 -892679478
-; CHECK-ALLOCA: store i32 -875836469
-; CHECK-NOALLOCA-NOT: store i32 -892679478
-; CHECK-NOALLOCA-NOT: store i32 -875836469
+; CHECK-ALLOCA: __asan_alloca_poison
+; CHECK-ALLOCA: __asan_allocas_unpoison
   %0 = alloca i32, align 4
   %1 = alloca i8*
   store volatile i32 %len, i32* %0, align 4
diff --git a/test/Instrumentation/AddressSanitizer/instrument-no-return.ll b/test/Instrumentation/AddressSanitizer/instrument-no-return.ll
index 5d5c592c3f4..2e90bfc64b2 100644
--- a/test/Instrumentation/AddressSanitizer/instrument-no-return.ll
+++ b/test/Instrumentation/AddressSanitizer/instrument-no-return.ll
@@ -29,7 +29,7 @@ entry:
 
 declare i32 @__gxx_personality_v0(...)
 
-define i64 @Invoke1(i8** %esc) nounwind uwtable ssp sanitize_address {
+define i64 @Invoke1(i8** %esc) nounwind uwtable ssp sanitize_address personality i8* bitcast (i32 (...)* @__gxx_personality_v0 to i8*) {
 entry:
   invoke void @MyNoReturnFunc(i32 1)
           to label %invoke.cont unwind label %lpad
@@ -38,7 +38,7 @@ invoke.cont:
   ret i64 0
 
 lpad:
-  %0 = landingpad { i8*, i32 } personality i8* bitcast (i32 (...)* @__gxx_personality_v0 to i8*)
+  %0 = landingpad { i8*, i32 }
           filter [0 x i8*] zeroinitializer
   ret i64 1
 }
diff --git a/test/Instrumentation/AddressSanitizer/localescape.ll b/test/Instrumentation/AddressSanitizer/localescape.ll
new file mode 100644
index 00000000000..d9daa8c96b1
--- /dev/null
+++ b/test/Instrumentation/AddressSanitizer/localescape.ll
@@ -0,0 +1,86 @@
+; RUN: opt < %s -asan -asan-module -asan-use-after-return -asan-stack-dynamic-alloca -S | FileCheck %s
+; RUN: opt < %s -asan -asan-module -asan-use-after-return=0 -asan-stack-dynamic-alloca=0 -S | FileCheck %s
+
+target datalayout = "e-m:x-p:32:32-i64:64-f80:32-n8:16:32-a:0:32-S32"
+target triple = "i686-pc-windows-msvc18.0.0"
+
+declare i32 @llvm.eh.typeid.for(i8*) #2
+declare i8* @llvm.frameaddress(i32)
+declare i8* @llvm.x86.seh.recoverfp(i8*, i8*)
+declare i8* @llvm.localrecover(i8*, i8*, i32)
+declare void @llvm.localescape(...) #1
+
+declare i32 @_except_handler3(...)
+declare void @may_throw(i32* %r)
+
+define i32 @main() sanitize_address personality i8* bitcast (i32 (...)* @_except_handler3 to i8*) {
+entry:
+  %r = alloca i32, align 4
+  %__exception_code = alloca i32, align 4
+  call void (...) @llvm.localescape(i32* nonnull %__exception_code)
+  %0 = bitcast i32* %r to i8*
+  store i32 0, i32* %r, align 4
+  invoke void @may_throw(i32* nonnull %r) #4
+          to label %__try.cont unwind label %lpad
+
+lpad:                                             ; preds = %entry
+  %1 = landingpad { i8*, i32 }
+          catch i8* bitcast (i32 ()* @"\01?filt$0@0@main@@" to i8*)
+  %2 = extractvalue { i8*, i32 } %1, 1
+  %3 = call i32 @llvm.eh.typeid.for(i8* bitcast (i32 ()* @"\01?filt$0@0@main@@" to i8*)) #1
+  %matches = icmp eq i32 %2, %3
+  br i1 %matches, label %__except, label %eh.resume
+
+__except:                                         ; preds = %lpad
+  store i32 1, i32* %r, align 4
+  br label %__try.cont
+
+__try.cont:                                       ; preds = %entry, %__except
+  %4 = load i32, i32* %r, align 4
+  ret i32 %4
+
+eh.resume:                                        ; preds = %lpad
+  resume { i8*, i32 } %1
+}
+
+; Check that the alloca remains static and the localescape call remains in the
+; entry block.
+
+; CHECK-LABEL: define i32 @main()
+; CHECK-NOT: br {{.*}}label
+; CHECK: %__exception_code = alloca i32, align 4
+; CHECK-NOT: br {{.*}}label
+; CHECK: call void (...) @llvm.localescape(i32* nonnull %__exception_code)
+
+; Function Attrs: nounwind
+define internal i32 @"\01?filt$0@0@main@@"() #1 {
+entry:
+  %0 = tail call i8* @llvm.frameaddress(i32 1)
+  %1 = tail call i8* @llvm.x86.seh.recoverfp(i8* bitcast (i32 ()* @main to i8*), i8* %0)
+  %2 = tail call i8* @llvm.localrecover(i8* bitcast (i32 ()* @main to i8*), i8* %1, i32 0)
+  %__exception_code = bitcast i8* %2 to i32*
+  %3 = getelementptr inbounds i8, i8* %0, i32 -20
+  %4 = bitcast i8* %3 to { i32*, i8* }**
+  %5 = load { i32*, i8* }*, { i32*, i8* }** %4, align 4
+  %6 = getelementptr inbounds { i32*, i8* }, { i32*, i8* }* %5, i32 0, i32 0
+  %7 = load i32*, i32** %6, align 4
+  %8 = load i32, i32* %7, align 4
+  store i32 %8, i32* %__exception_code, align 4
+  ret i32 1
+}
+
+; CHECK-LABEL: define internal i32 @"\01?filt$0@0@main@@"()
+; CHECK: tail call i8* @llvm.localrecover(i8* bitcast (i32 ()* @main to i8*), i8* {{.*}}, i32 0)
+
+define void @ScaleFilterCols_SSSE3(i8* %dst_ptr, i8* %src_ptr, i32 %dst_width, i32 %x, i32 %dx) sanitize_address {
+entry:
+  %dst_width.addr = alloca i32, align 4
+  store i32 %dst_width, i32* %dst_width.addr, align 4
+  %0 = call { i8*, i8*, i32, i32, i32 } asm sideeffect "", "=r,=r,={ax},=r,=r,=*rm,rm,rm,0,1,2,3,4,5,~{memory},~{cc},~{xmm0},~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{dirflag},~{fpsr},~{flags}"(i32* nonnull %dst_width.addr, i32 %x, i32 %dx, i8* %dst_ptr, i8* %src_ptr, i32 0, i32 0, i32 0, i32 %dst_width)
+  ret void
+}
+
+define void @ScaleColsUp2_SSE2() sanitize_address {
+entry:
+  ret void
+}
diff --git a/test/Instrumentation/AddressSanitizer/stack_dynamic_alloca.ll b/test/Instrumentation/AddressSanitizer/stack_dynamic_alloca.ll
index 9c058742dbf..a8292204d27 100644
--- a/test/Instrumentation/AddressSanitizer/stack_dynamic_alloca.ll
+++ b/test/Instrumentation/AddressSanitizer/stack_dynamic_alloca.ll
@@ -44,3 +44,34 @@ entry:
   call void asm sideeffect "mov %%rbx, %%rcx", "~{dirflag},~{fpsr},~{flags}"() nounwind
   ret void
 }
+
+; Test that dynamic alloca is not used when setjmp is present.
+%struct.__jmp_buf_tag = type { [8 x i64], i32, %struct.__sigset_t }
+%struct.__sigset_t = type { [16 x i64] }
+@_ZL3buf = internal global [1 x %struct.__jmp_buf_tag] zeroinitializer, align 16
+
+define void @Func3() uwtable sanitize_address {
+; CHECK-LABEL: define void @Func3
+; CHECK-NOT: __asan_option_detect_stack_use_after_return
+; CHECK-NOT: __asan_stack_malloc
+; CHECK: call void @__asan_handle_no_return
+; CHECK: call void @longjmp
+; CHECK: ret void
+entry:
+  %a = alloca i32, align 4
+  %call = call i32 @_setjmp(%struct.__jmp_buf_tag* getelementptr inbounds ([1 x %struct.__jmp_buf_tag], [1 x %struct.__jmp_buf_tag]* @_ZL3buf, i32 0, i32 0)) nounwind returns_twice
+  %cmp = icmp eq i32 0, %call
+  br i1 %cmp, label %if.then, label %if.end
+
+if.then:                                          ; preds = %entry
+  call void @longjmp(%struct.__jmp_buf_tag* getelementptr inbounds ([1 x %struct.__jmp_buf_tag], [1 x %struct.__jmp_buf_tag]* @_ZL3buf, i32 0, i32 0), i32 1) noreturn nounwind
+  unreachable
+
+if.end:                                           ; preds = %entry
+  call void @_Z10escape_ptrPi(i32* %a)
+  ret void
+}
+
+declare i32 @_setjmp(%struct.__jmp_buf_tag*) nounwind returns_twice
+declare void @longjmp(%struct.__jmp_buf_tag*, i32) noreturn nounwind
+declare void @_Z10escape_ptrPi(i32*)
diff --git a/test/Instrumentation/AddressSanitizer/undecidable-dynamic-alloca-1.ll b/test/Instrumentation/AddressSanitizer/undecidable-dynamic-alloca-1.ll
deleted file mode 100644
index c67fb501463..00000000000
--- a/test/Instrumentation/AddressSanitizer/undecidable-dynamic-alloca-1.ll
+++ /dev/null
@@ -1,23 +0,0 @@
-; Test that undecidable dynamic allocas are skipped by ASan.
-
-; RUN: opt < %s -asan -asan-module -asan-instrument-allocas=1 -S | FileCheck %s
-target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64"
-target triple = "x86_64-unknown-linux-gnu"
-
-define void @g(i64 %n) sanitize_address {
-entry:
-  %cmp = icmp sgt i64 %n, 100
-  br i1 %cmp, label %do_alloca, label %done
-
-do_alloca:
-; CHECK-NOT: store i32 -892679478
-  %0 = alloca i8, i64 %n, align 1
-  call void @f(i8* %0)
-  br label %done
-
-done:
-  ret void
-}
-
-declare void @f(i8*)
-
diff --git a/test/Instrumentation/DataFlowSanitizer/debug.ll b/test/Instrumentation/DataFlowSanitizer/debug.ll
index 9bab258dac3..c18f5920fa3 100644
--- a/test/Instrumentation/DataFlowSanitizer/debug.ll
+++ b/test/Instrumentation/DataFlowSanitizer/debug.ll
@@ -1,6 +1,6 @@
 ; RUN: opt < %s -dfsan -dfsan-abilist=%S/Inputs/debuglist.txt -S | FileCheck %s
 
-; CHECK: !MDSubprogram(name: "main",{{.*}} function: i32 ()* @main{{[,)]}}
+; CHECK: !DISubprogram(name: "main",{{.*}} function: i32 ()* @main{{[,)]}}
 
 ; Generated from a simple source file compiled with clang -g:
 ; int main() {
@@ -21,16 +21,16 @@ attributes #0 = { nounwind uwtable "less-precise-fpmad"="false" "no-frame-pointe
 !llvm.module.flags = !{!9, !10}
 !llvm.ident = !{!11}
 
-!0 = !MDCompileUnit(language: DW_LANG_C_plus_plus, producer: "clang version 3.6.0 ", isOptimized: false, emissionKind: 1, file: !1, enums: !2, retainedTypes: !2, subprograms: !3, globals: !2, imports: !2)
-!1 = !MDFile(filename: "debug.cpp", directory: "/tmp/dbginfo")
+!0 = !DICompileUnit(language: DW_LANG_C_plus_plus, producer: "clang version 3.6.0 ", isOptimized: false, emissionKind: 1, file: !1, enums: !2, retainedTypes: !2, subprograms: !3, globals: !2, imports: !2)
+!1 = !DIFile(filename: "debug.cpp", directory: "/tmp/dbginfo")
 !2 = !{}
 !3 = !{!4}
-!4 = !MDSubprogram(name: "main", line: 1, isLocal: false, isDefinition: true, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 1, file: !1, scope: !5, type: !6, function: i32 ()* @main, variables: !2)
-!5 = !MDFile(filename: "debug.cpp", directory: "/tmp/dbginfo")
-!6 = !MDSubroutineType(types: !7)
+!4 = !DISubprogram(name: "main", line: 1, isLocal: false, isDefinition: true, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 1, file: !1, scope: !5, type: !6, function: i32 ()* @main, variables: !2)
+!5 = !DIFile(filename: "debug.cpp", directory: "/tmp/dbginfo")
+!6 = !DISubroutineType(types: !7)
 !7 = !{!8}
-!8 = !MDBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
+!8 = !DIBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
 !9 = !{i32 2, !"Dwarf Version", i32 4}
 !10 = !{i32 2, !"Debug Info Version", i32 3}
 !11 = !{!"clang version 3.6.0 "}
-!12 = !MDLocation(line: 2, column: 1, scope: !4)
+!12 = !DILocation(line: 2, column: 1, scope: !4)
diff --git a/test/Instrumentation/InstrProfiling/PR23499.ll b/test/Instrumentation/InstrProfiling/PR23499.ll
new file mode 100644
index 00000000000..5aae735120b
--- /dev/null
+++ b/test/Instrumentation/InstrProfiling/PR23499.ll
@@ -0,0 +1,21 @@
+;; Check that data associated with linkonce odr functions are placed in
+;; the same comdat section as their associated function.
+
+; RUN: opt < %s -mtriple=x86_64-apple-macosx10.10.0 -instrprof -S | FileCheck %s
+; RUN: opt < %s -mtriple=x86_64-unknown-linux -instrprof -S | FileCheck %s
+
+$_Z3barIvEvv = comdat any
+
+@__llvm_profile_name__Z3barIvEvv = linkonce_odr hidden constant [11 x i8] c"_Z3barIvEvv", align 1
+
+; CHECK: @__llvm_profile_name__Z3barIvEvv = linkonce_odr hidden constant [11 x i8] c"_Z3barIvEvv", section "{{.*}}__llvm_prf_names", comdat($_Z3barIvEvv), align 1
+; CHECK: @__llvm_profile_counters__Z3barIvEvv = linkonce_odr hidden global [1 x i64] zeroinitializer, section "{{.*}}__llvm_prf_cnts", comdat($_Z3barIvEvv), align 8
+; CHECK: @__llvm_profile_data__Z3barIvEvv = linkonce_odr hidden constant { i32, i32, i64, i8*, i64* } { i32 11, i32 1, i64 0, i8* getelementptr inbounds ([11 x i8], [11 x i8]* @__llvm_profile_name__Z3barIvEvv, i32 0, i32 0), i64* getelementptr inbounds ([1 x i64], [1 x i64]* @__llvm_profile_counters__Z3barIvEvv, i32 0, i32 0) }, section "{{.*}}__llvm_prf_data", comdat($_Z3barIvEvv), align 8
+
+declare void @llvm.instrprof.increment(i8*, i64, i32, i32) #1
+
+define linkonce_odr void @_Z3barIvEvv() comdat {
+entry:
+  call void @llvm.instrprof.increment(i8* getelementptr inbounds ([11 x i8], [11 x i8]* @__llvm_profile_name__Z3barIvEvv, i32 0, i32 0), i64 0, i32 1, i32 0)
+  ret void
+}
diff --git a/test/Instrumentation/MemorySanitizer/msan_basic.ll b/test/Instrumentation/MemorySanitizer/msan_basic.ll
index 4e84aeff2d0..8b8e2970959 100644
--- a/test/Instrumentation/MemorySanitizer/msan_basic.ll
+++ b/test/Instrumentation/MemorySanitizer/msan_basic.ll
@@ -4,8 +4,7 @@
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
 
-; Check the presence of __msan_init
-; CHECK: @llvm.global_ctors {{.*}} @__msan_init
+; CHECK: @llvm.global_ctors {{.*}} @msan.module_ctor
 
 ; Check the presence and the linkage type of __msan_track_origins and
 ; other interface symbols.
@@ -878,3 +877,6 @@ define void @MismatchedReturnTypeTailCall(i32 %a) sanitize_memory {
 ; CHECK-LABEL: define void @MismatchedReturnTypeTailCall
 ; CHECK: tail call i32 @InnerTailCall
 ; CHECK: ret void
+
+; CHECK-LABEL: define internal void @msan.module_ctor
+; CHECK: call void @__msan_init()
diff --git a/test/Instrumentation/MemorySanitizer/store-origin.ll b/test/Instrumentation/MemorySanitizer/store-origin.ll
index 92c0b2ce48f..cdfe280bc8c 100644
--- a/test/Instrumentation/MemorySanitizer/store-origin.ll
+++ b/test/Instrumentation/MemorySanitizer/store-origin.ll
@@ -11,8 +11,8 @@ target triple = "x86_64-unknown-linux-gnu"
 ; Function Attrs: nounwind
 define void @Store(i32* nocapture %p, i32 %x) #0 {
 entry:
-  tail call void @llvm.dbg.value(metadata i32* %p, i64 0, metadata !11, metadata !MDExpression()), !dbg !16
-  tail call void @llvm.dbg.value(metadata i32 %x, i64 0, metadata !12, metadata !MDExpression()), !dbg !16
+  tail call void @llvm.dbg.value(metadata i32* %p, i64 0, metadata !11, metadata !DIExpression()), !dbg !16
+  tail call void @llvm.dbg.value(metadata i32 %x, i64 0, metadata !12, metadata !DIExpression()), !dbg !16
   store i32 %x, i32* %p, align 4, !dbg !17, !tbaa !18
   ret void, !dbg !22
 }
@@ -27,29 +27,29 @@ attributes #1 = { nounwind readnone }
 !llvm.module.flags = !{!13, !14}
 !llvm.ident = !{!15}
 
-!0 = !MDCompileUnit(language: DW_LANG_C99, producer: "clang version 3.5.0 (204220)", isOptimized: true, emissionKind: 1, file: !1, enums: !2, retainedTypes: !2, subprograms: !3, globals: !2, imports: !2)
-!1 = !MDFile(filename: "../2.cc", directory: "/tmp/build0")
+!0 = !DICompileUnit(language: DW_LANG_C99, producer: "clang version 3.5.0 (204220)", isOptimized: true, emissionKind: 1, file: !1, enums: !2, retainedTypes: !2, subprograms: !3, globals: !2, imports: !2)
+!1 = !DIFile(filename: "../2.cc", directory: "/tmp/build0")
 !2 = !{}
 !3 = !{!4}
-!4 = !MDSubprogram(name: "Store", line: 1, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: true, scopeLine: 1, file: !1, scope: !5, type: !6, function: void (i32*, i32)* @Store, variables: !10)
-!5 = !MDFile(filename: "../2.cc", directory: "/tmp/build0")
-!6 = !MDSubroutineType(types: !7)
+!4 = !DISubprogram(name: "Store", line: 1, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: true, scopeLine: 1, file: !1, scope: !5, type: !6, function: void (i32*, i32)* @Store, variables: !10)
+!5 = !DIFile(filename: "../2.cc", directory: "/tmp/build0")
+!6 = !DISubroutineType(types: !7)
 !7 = !{null, !8, !9}
-!8 = !MDDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, baseType: !9)
-!9 = !MDBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
+!8 = !DIDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, baseType: !9)
+!9 = !DIBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
 !10 = !{!11, !12}
-!11 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "p", line: 1, arg: 1, scope: !4, file: !5, type: !8)
-!12 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "x", line: 1, arg: 2, scope: !4, file: !5, type: !9)
+!11 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "p", line: 1, arg: 1, scope: !4, file: !5, type: !8)
+!12 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "x", line: 1, arg: 2, scope: !4, file: !5, type: !9)
 !13 = !{i32 2, !"Dwarf Version", i32 4}
 !14 = !{i32 1, !"Debug Info Version", i32 3}
 !15 = !{!"clang version 3.5.0 (204220)"}
-!16 = !MDLocation(line: 1, scope: !4)
-!17 = !MDLocation(line: 2, scope: !4)
+!16 = !DILocation(line: 1, scope: !4)
+!17 = !DILocation(line: 2, scope: !4)
 !18 = !{!19, !19, i64 0}
 !19 = !{!"int", !20, i64 0}
 !20 = !{!"omnipotent char", !21, i64 0}
 !21 = !{!"Simple C/C++ TBAA"}
-!22 = !MDLocation(line: 3, scope: !4)
+!22 = !DILocation(line: 3, scope: !4)
 
 
 ; CHECK: @Store
diff --git a/test/Instrumentation/SanitizerCoverage/coverage-dbg.ll b/test/Instrumentation/SanitizerCoverage/coverage-dbg.ll
index d3ee7b32543..45b4c8cb0a6 100644
--- a/test/Instrumentation/SanitizerCoverage/coverage-dbg.ll
+++ b/test/Instrumentation/SanitizerCoverage/coverage-dbg.ll
@@ -16,7 +16,7 @@
 
 ; Test that __sanitizer_cov call has !dbg pointing to the opening { of A::f().
 ; CHECK: call void @__sanitizer_cov(i32*{{.*}}), !dbg [[A:!.*]]
-; CHECK: [[A]] = !MDLocation(line: 6, scope: !{{.*}})
+; CHECK: [[A]] = !DILocation(line: 6, scope: !{{.*}})
 
 
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
@@ -27,7 +27,7 @@ target triple = "x86_64-unknown-linux-gnu"
 ; Function Attrs: nounwind readonly uwtable
 define i32 @_ZN1A1fEv(%struct.A* nocapture readonly %this) #0 align 2 {
 entry:
-  tail call void @llvm.dbg.value(metadata %struct.A* %this, i64 0, metadata !15, metadata !MDExpression()), !dbg !20
+  tail call void @llvm.dbg.value(metadata %struct.A* %this, i64 0, metadata !15, metadata !DIExpression()), !dbg !20
   %x = getelementptr inbounds %struct.A, %struct.A* %this, i64 0, i32 0, !dbg !21
   %0 = load i32, i32* %x, align 4, !dbg !21
   ret i32 %0, !dbg !21
@@ -43,25 +43,25 @@ attributes #1 = { nounwind readnone }
 !llvm.module.flags = !{!17, !18}
 !llvm.ident = !{!19}
 
-!0 = !MDCompileUnit(language: DW_LANG_C_plus_plus, producer: "clang version 3.5.0 (210251)", isOptimized: true, emissionKind: 1, file: !1, enums: !2, retainedTypes: !3, subprograms: !12, globals: !2, imports: !2)
-!1 = !MDFile(filename: "../1.cc", directory: "/code/llvm/build0")
+!0 = !DICompileUnit(language: DW_LANG_C_plus_plus, producer: "clang version 3.5.0 (210251)", isOptimized: true, emissionKind: 1, file: !1, enums: !2, retainedTypes: !3, subprograms: !12, globals: !2, imports: !2)
+!1 = !DIFile(filename: "../1.cc", directory: "/code/llvm/build0")
 !2 = !{}
 !3 = !{!4}
-!4 = !MDCompositeType(tag: DW_TAG_structure_type, name: "A", line: 1, size: 32, align: 32, file: !1, elements: !5, identifier: "_ZTS1A")
+!4 = !DICompositeType(tag: DW_TAG_structure_type, name: "A", line: 1, size: 32, align: 32, file: !1, elements: !5, identifier: "_ZTS1A")
 !5 = !{!6, !8}
-!6 = !MDDerivedType(tag: DW_TAG_member, name: "x", line: 3, size: 32, align: 32, file: !1, scope: !"_ZTS1A", baseType: !7)
-!7 = !MDBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
-!8 = !MDSubprogram(name: "f", linkageName: "_ZN1A1fEv", line: 2, isLocal: false, isDefinition: false, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: true, scopeLine: 2, file: !1, scope: !"_ZTS1A", type: !9)
-!9 = !MDSubroutineType(types: !10)
+!6 = !DIDerivedType(tag: DW_TAG_member, name: "x", line: 3, size: 32, align: 32, file: !1, scope: !"_ZTS1A", baseType: !7)
+!7 = !DIBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
+!8 = !DISubprogram(name: "f", linkageName: "_ZN1A1fEv", line: 2, isLocal: false, isDefinition: false, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: true, scopeLine: 2, file: !1, scope: !"_ZTS1A", type: !9)
+!9 = !DISubroutineType(types: !10)
 !10 = !{!7, !11}
-!11 = !MDDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, flags: DIFlagArtificial | DIFlagObjectPointer, baseType: !"_ZTS1A")
+!11 = !DIDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, flags: DIFlagArtificial | DIFlagObjectPointer, baseType: !"_ZTS1A")
 !12 = !{!13}
-!13 = !MDSubprogram(name: "f", linkageName: "_ZN1A1fEv", line: 6, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: true, scopeLine: 6, file: !1, scope: !"_ZTS1A", type: !9, function: i32 (%struct.A*)* @_ZN1A1fEv, declaration: !8, variables: !14)
+!13 = !DISubprogram(name: "f", linkageName: "_ZN1A1fEv", line: 6, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: true, scopeLine: 6, file: !1, scope: !"_ZTS1A", type: !9, function: i32 (%struct.A*)* @_ZN1A1fEv, declaration: !8, variables: !14)
 !14 = !{!15}
-!15 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "this", arg: 1, flags: DIFlagArtificial | DIFlagObjectPointer, scope: !13, type: !16)
-!16 = !MDDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, baseType: !"_ZTS1A")
+!15 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "this", arg: 1, flags: DIFlagArtificial | DIFlagObjectPointer, scope: !13, type: !16)
+!16 = !DIDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, baseType: !"_ZTS1A")
 !17 = !{i32 2, !"Dwarf Version", i32 4}
 !18 = !{i32 2, !"Debug Info Version", i32 3}
 !19 = !{!"clang version 3.5.0 (210251)"}
-!20 = !MDLocation(line: 0, scope: !13)
-!21 = !MDLocation(line: 7, scope: !13)
+!20 = !DILocation(line: 0, scope: !13)
+!21 = !DILocation(line: 7, scope: !13)
diff --git a/test/Instrumentation/SanitizerCoverage/coverage.ll b/test/Instrumentation/SanitizerCoverage/coverage.ll
index b2f0ab0680b..659c03040f2 100644
--- a/test/Instrumentation/SanitizerCoverage/coverage.ll
+++ b/test/Instrumentation/SanitizerCoverage/coverage.ll
@@ -119,3 +119,12 @@ entry:
 ; CHECK4: call void @__sanitizer_cov_indir_call16({{.*}},[[CACHE:.*]])
 ; CHECK4-NOT: call void @__sanitizer_cov_indir_call16({{.*}},[[CACHE]])
 ; CHECK4: ret void
+
+define void @call_unreachable() uwtable sanitize_address {
+entry:
+  unreachable
+}
+
+; CHECK4-LABEL: define void @call_unreachable
+; CHECK4-NOT: __sanitizer_cov
+; CHECK4: unreachable
diff --git a/test/Instrumentation/SanitizerCoverage/coverage2-dbg.ll b/test/Instrumentation/SanitizerCoverage/coverage2-dbg.ll
index ce85073f7ef..f9b8e712688 100644
--- a/test/Instrumentation/SanitizerCoverage/coverage2-dbg.ll
+++ b/test/Instrumentation/SanitizerCoverage/coverage2-dbg.ll
@@ -21,13 +21,13 @@ target triple = "x86_64-unknown-linux-gnu"
 ; CHECK: call void @__sanitizer_cov(i32*{{.*}}), !dbg [[B:!.*]]
 ; CHECK: call void @__sanitizer_cov(i32*{{.*}}), !dbg [[C:!.*]]
 ; CHECK: ret void
-; CHECK: [[A]] = !MDLocation(line: 1, scope: !{{.*}})
-; CHECK: [[B]] = !MDLocation(line: 3, column: 5, scope: !{{.*}})
-; CHECK: [[C]] = !MDLocation(line: 4, column: 1, scope: !{{.*}})
+; CHECK: [[A]] = !DILocation(line: 1, scope: !{{.*}})
+; CHECK: [[B]] = !DILocation(line: 3, column: 5, scope: !{{.*}})
+; CHECK: [[C]] = !DILocation(line: 4, column: 1, scope: !{{.*}})
 
 define void @_Z3fooPi(i32* %a) #0 {
 entry:
-  tail call void @llvm.dbg.value(metadata i32* %a, i64 0, metadata !11, metadata !MDExpression()), !dbg !15
+  tail call void @llvm.dbg.value(metadata i32* %a, i64 0, metadata !11, metadata !DIExpression()), !dbg !15
   %tobool = icmp eq i32* %a, null, !dbg !16
   br i1 %tobool, label %if.end, label %if.then, !dbg !16
 
@@ -49,27 +49,27 @@ attributes #1 = { nounwind readnone }
 !llvm.module.flags = !{!12, !13}
 !llvm.ident = !{!14}
 
-!0 = !MDCompileUnit(language: DW_LANG_C_plus_plus, producer: "clang version 3.6.0 (217079)", isOptimized: true, emissionKind: 1, file: !1, enums: !2, retainedTypes: !2, subprograms: !3, globals: !2, imports: !2)
-!1 = !MDFile(filename: "if.cc", directory: "FOO")
+!0 = !DICompileUnit(language: DW_LANG_C_plus_plus, producer: "clang version 3.6.0 (217079)", isOptimized: true, emissionKind: 1, file: !1, enums: !2, retainedTypes: !2, subprograms: !3, globals: !2, imports: !2)
+!1 = !DIFile(filename: "if.cc", directory: "FOO")
 !2 = !{}
 !3 = !{!4}
-!4 = !MDSubprogram(name: "foo", linkageName: "_Z3fooPi", line: 1, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: true, scopeLine: 1, file: !1, scope: !5, type: !6, function: void (i32*)* @_Z3fooPi, variables: !10)
-!5 = !MDFile(filename: "if.cc", directory: "FOO")
-!6 = !MDSubroutineType(types: !7)
+!4 = !DISubprogram(name: "foo", linkageName: "_Z3fooPi", line: 1, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: true, scopeLine: 1, file: !1, scope: !5, type: !6, function: void (i32*)* @_Z3fooPi, variables: !10)
+!5 = !DIFile(filename: "if.cc", directory: "FOO")
+!6 = !DISubroutineType(types: !7)
 !7 = !{null, !8}
-!8 = !MDDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, baseType: !9)
-!9 = !MDBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
+!8 = !DIDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, baseType: !9)
+!9 = !DIBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
 !10 = !{!11}
-!11 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "a", line: 1, arg: 1, scope: !4, file: !5, type: !8)
+!11 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "a", line: 1, arg: 1, scope: !4, file: !5, type: !8)
 !12 = !{i32 2, !"Dwarf Version", i32 4}
 !13 = !{i32 2, !"Debug Info Version", i32 3}
 !14 = !{!"clang version 3.6.0 (217079)"}
-!15 = !MDLocation(line: 1, column: 15, scope: !4)
-!16 = !MDLocation(line: 2, column: 7, scope: !17)
-!17 = distinct !MDLexicalBlock(line: 2, column: 7, file: !1, scope: !4)
-!18 = !MDLocation(line: 3, column: 5, scope: !17)
+!15 = !DILocation(line: 1, column: 15, scope: !4)
+!16 = !DILocation(line: 2, column: 7, scope: !17)
+!17 = distinct !DILexicalBlock(line: 2, column: 7, file: !1, scope: !4)
+!18 = !DILocation(line: 3, column: 5, scope: !17)
 !19 = !{!20, !20, i64 0}
 !20 = !{!"int", !21, i64 0}
 !21 = !{!"omnipotent char", !22, i64 0}
 !22 = !{!"Simple C/C++ TBAA"}
-!23 = !MDLocation(line: 4, column: 1, scope: !4)
+!23 = !DILocation(line: 4, column: 1, scope: !4)
diff --git a/test/Instrumentation/SanitizerCoverage/switch-tracing.ll b/test/Instrumentation/SanitizerCoverage/switch-tracing.ll
new file mode 100644
index 00000000000..9ad1c7acf63
--- /dev/null
+++ b/test/Instrumentation/SanitizerCoverage/switch-tracing.ll
@@ -0,0 +1,32 @@
+; Test -sanitizer-coverage-experimental-trace-compares=1 (instrumenting a switch)
+; RUN: opt < %s -sancov -sanitizer-coverage-level=1 -sanitizer-coverage-experimental-trace-compares=1  -S | FileCheck %s --check-prefix=CHECK
+
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64"
+target triple = "x86_64-unknown-linux-gnu"
+declare void @_Z3bari(i32)
+define void @foo(i32 %x) {
+entry:
+; CHECK: __sancov_gen_cov_switch_values = internal global [5 x i64] [i64 3, i64 32, i64 1, i64 101, i64 1001]
+; CHECK: [[TMP:%[0-9]*]] = zext i32 %x to i64
+; CHECK-NEXT: call void @__sanitizer_cov_trace_switch(i64 [[TMP]], i64* getelementptr inbounds ([5 x i64], [5 x i64]* @__sancov_gen_cov_switch_values, i32 0, i32 0))
+  switch i32 %x, label %sw.epilog [
+    i32 1, label %sw.bb
+    i32 101, label %sw.bb.1
+    i32 1001, label %sw.bb.2
+  ]
+
+sw.bb:                                            ; preds = %entry
+  tail call void @_Z3bari(i32 4)
+  br label %sw.epilog
+
+sw.bb.1:                                          ; preds = %entry
+  tail call void @_Z3bari(i32 5)
+  br label %sw.epilog
+
+sw.bb.2:                                          ; preds = %entry
+  tail call void @_Z3bari(i32 6)
+  br label %sw.epilog
+
+sw.epilog:                                        ; preds = %entry, %sw.bb.2, %sw.bb.1, %sw.bb
+  ret void
+}
diff --git a/test/Instrumentation/ThreadSanitizer/atomic.ll b/test/Instrumentation/ThreadSanitizer/atomic.ll
index 1d6ac3851e2..db01bab8fe5 100644
--- a/test/Instrumentation/ThreadSanitizer/atomic.ll
+++ b/test/Instrumentation/ThreadSanitizer/atomic.ll
@@ -4,1984 +4,1995 @@ target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f3
 
 define i8 @atomic8_load_unordered(i8* %a) nounwind uwtable {
 entry:
-  %0 = load atomic i8, i8* %a unordered, align 1
-  ret i8 %0
+  %0 = load atomic i8, i8* %a unordered, align 1, !dbg !7
+  ret i8 %0, !dbg !7
 }
-; CHECK: atomic8_load_unordered
-; CHECK: call i8 @__tsan_atomic8_load(i8* %a, i32 0)
+; CHECK-LABEL: atomic8_load_unordered
+; CHECK: call i8 @__tsan_atomic8_load(i8* %a, i32 0), !dbg
 
 define i8 @atomic8_load_monotonic(i8* %a) nounwind uwtable {
 entry:
-  %0 = load atomic i8, i8* %a monotonic, align 1
-  ret i8 %0
+  %0 = load atomic i8, i8* %a monotonic, align 1, !dbg !7
+  ret i8 %0, !dbg !7
 }
-; CHECK: atomic8_load_monotonic
-; CHECK: call i8 @__tsan_atomic8_load(i8* %a, i32 0)
+; CHECK-LABEL: atomic8_load_monotonic
+; CHECK: call i8 @__tsan_atomic8_load(i8* %a, i32 0), !dbg
 
 define i8 @atomic8_load_acquire(i8* %a) nounwind uwtable {
 entry:
-  %0 = load atomic i8, i8* %a acquire, align 1
-  ret i8 %0
+  %0 = load atomic i8, i8* %a acquire, align 1, !dbg !7
+  ret i8 %0, !dbg !7
 }
-; CHECK: atomic8_load_acquire
-; CHECK: call i8 @__tsan_atomic8_load(i8* %a, i32 2)
+; CHECK-LABEL: atomic8_load_acquire
+; CHECK: call i8 @__tsan_atomic8_load(i8* %a, i32 2), !dbg
 
 define i8 @atomic8_load_seq_cst(i8* %a) nounwind uwtable {
 entry:
-  %0 = load atomic i8, i8* %a seq_cst, align 1
-  ret i8 %0
+  %0 = load atomic i8, i8* %a seq_cst, align 1, !dbg !7
+  ret i8 %0, !dbg !7
 }
-; CHECK: atomic8_load_seq_cst
-; CHECK: call i8 @__tsan_atomic8_load(i8* %a, i32 5)
+; CHECK-LABEL: atomic8_load_seq_cst
+; CHECK: call i8 @__tsan_atomic8_load(i8* %a, i32 5), !dbg
 
 define void @atomic8_store_unordered(i8* %a) nounwind uwtable {
 entry:
-  store atomic i8 0, i8* %a unordered, align 1
-  ret void
+  store atomic i8 0, i8* %a unordered, align 1, !dbg !7
+  ret void, !dbg !7
 }
-; CHECK: atomic8_store_unordered
-; CHECK: call void @__tsan_atomic8_store(i8* %a, i8 0, i32 0)
+; CHECK-LABEL: atomic8_store_unordered
+; CHECK: call void @__tsan_atomic8_store(i8* %a, i8 0, i32 0), !dbg
 
 define void @atomic8_store_monotonic(i8* %a) nounwind uwtable {
 entry:
-  store atomic i8 0, i8* %a monotonic, align 1
-  ret void
+  store atomic i8 0, i8* %a monotonic, align 1, !dbg !7
+  ret void, !dbg !7
 }
-; CHECK: atomic8_store_monotonic
-; CHECK: call void @__tsan_atomic8_store(i8* %a, i8 0, i32 0)
+; CHECK-LABEL: atomic8_store_monotonic
+; CHECK: call void @__tsan_atomic8_store(i8* %a, i8 0, i32 0), !dbg
 
 define void @atomic8_store_release(i8* %a) nounwind uwtable {
 entry:
-  store atomic i8 0, i8* %a release, align 1
-  ret void
+  store atomic i8 0, i8* %a release, align 1, !dbg !7
+  ret void, !dbg !7
 }
-; CHECK: atomic8_store_release
-; CHECK: call void @__tsan_atomic8_store(i8* %a, i8 0, i32 3)
+; CHECK-LABEL: atomic8_store_release
+; CHECK: call void @__tsan_atomic8_store(i8* %a, i8 0, i32 3), !dbg
 
 define void @atomic8_store_seq_cst(i8* %a) nounwind uwtable {
 entry:
-  store atomic i8 0, i8* %a seq_cst, align 1
-  ret void
+  store atomic i8 0, i8* %a seq_cst, align 1, !dbg !7
+  ret void, !dbg !7
 }
-; CHECK: atomic8_store_seq_cst
-; CHECK: call void @__tsan_atomic8_store(i8* %a, i8 0, i32 5)
+; CHECK-LABEL: atomic8_store_seq_cst
+; CHECK: call void @__tsan_atomic8_store(i8* %a, i8 0, i32 5), !dbg
 
 define void @atomic8_xchg_monotonic(i8* %a) nounwind uwtable {
 entry:
-  atomicrmw xchg i8* %a, i8 0 monotonic
-  ret void
+  atomicrmw xchg i8* %a, i8 0 monotonic, !dbg !7
+  ret void, !dbg !7
 }
-; CHECK: atomic8_xchg_monotonic
-; CHECK: call i8 @__tsan_atomic8_exchange(i8* %a, i8 0, i32 0)
+; CHECK-LABEL: atomic8_xchg_monotonic
+; CHECK: call i8 @__tsan_atomic8_exchange(i8* %a, i8 0, i32 0), !dbg
 
 define void @atomic8_add_monotonic(i8* %a) nounwind uwtable {
 entry:
-  atomicrmw add i8* %a, i8 0 monotonic
-  ret void
+  atomicrmw add i8* %a, i8 0 monotonic, !dbg !7
+  ret void, !dbg !7
 }
-; CHECK: atomic8_add_monotonic
-; CHECK: call i8 @__tsan_atomic8_fetch_add(i8* %a, i8 0, i32 0)
+; CHECK-LABEL: atomic8_add_monotonic
+; CHECK: call i8 @__tsan_atomic8_fetch_add(i8* %a, i8 0, i32 0), !dbg
 
 define void @atomic8_sub_monotonic(i8* %a) nounwind uwtable {
 entry:
-  atomicrmw sub i8* %a, i8 0 monotonic
-  ret void
+  atomicrmw sub i8* %a, i8 0 monotonic, !dbg !7
+  ret void, !dbg !7
 }
-; CHECK: atomic8_sub_monotonic
-; CHECK: call i8 @__tsan_atomic8_fetch_sub(i8* %a, i8 0, i32 0)
+; CHECK-LABEL: atomic8_sub_monotonic
+; CHECK: call i8 @__tsan_atomic8_fetch_sub(i8* %a, i8 0, i32 0), !dbg
 
 define void @atomic8_and_monotonic(i8* %a) nounwind uwtable {
 entry:
-  atomicrmw and i8* %a, i8 0 monotonic
-  ret void
+  atomicrmw and i8* %a, i8 0 monotonic, !dbg !7
+  ret void, !dbg !7
 }
-; CHECK: atomic8_and_monotonic
-; CHECK: call i8 @__tsan_atomic8_fetch_and(i8* %a, i8 0, i32 0)
+; CHECK-LABEL: atomic8_and_monotonic
+; CHECK: call i8 @__tsan_atomic8_fetch_and(i8* %a, i8 0, i32 0), !dbg
 
 define void @atomic8_or_monotonic(i8* %a) nounwind uwtable {
 entry:
-  atomicrmw or i8* %a, i8 0 monotonic
-  ret void
+  atomicrmw or i8* %a, i8 0 monotonic, !dbg !7
+  ret void, !dbg !7
 }
-; CHECK: atomic8_or_monotonic
-; CHECK: call i8 @__tsan_atomic8_fetch_or(i8* %a, i8 0, i32 0)
+; CHECK-LABEL: atomic8_or_monotonic
+; CHECK: call i8 @__tsan_atomic8_fetch_or(i8* %a, i8 0, i32 0), !dbg
 
 define void @atomic8_xor_monotonic(i8* %a) nounwind uwtable {
 entry:
-  atomicrmw xor i8* %a, i8 0 monotonic
-  ret void
+  atomicrmw xor i8* %a, i8 0 monotonic, !dbg !7
+  ret void, !dbg !7
 }
-; CHECK: atomic8_xor_monotonic
-; CHECK: call i8 @__tsan_atomic8_fetch_xor(i8* %a, i8 0, i32 0)
+; CHECK-LABEL: atomic8_xor_monotonic
+; CHECK: call i8 @__tsan_atomic8_fetch_xor(i8* %a, i8 0, i32 0), !dbg
 
 define void @atomic8_nand_monotonic(i8* %a) nounwind uwtable {
 entry:
-  atomicrmw nand i8* %a, i8 0 monotonic
-  ret void
+  atomicrmw nand i8* %a, i8 0 monotonic, !dbg !7
+  ret void, !dbg !7
 }
-; CHECK: atomic8_nand_monotonic
-; CHECK: call i8 @__tsan_atomic8_fetch_nand(i8* %a, i8 0, i32 0)
+; CHECK-LABEL: atomic8_nand_monotonic
+; CHECK: call i8 @__tsan_atomic8_fetch_nand(i8* %a, i8 0, i32 0), !dbg
 
 define void @atomic8_xchg_acquire(i8* %a) nounwind uwtable {
 entry:
-  atomicrmw xchg i8* %a, i8 0 acquire
-  ret void
+  atomicrmw xchg i8* %a, i8 0 acquire, !dbg !7
+  ret void, !dbg !7
 }
-; CHECK: atomic8_xchg_acquire
-; CHECK: call i8 @__tsan_atomic8_exchange(i8* %a, i8 0, i32 2)
+; CHECK-LABEL: atomic8_xchg_acquire
+; CHECK: call i8 @__tsan_atomic8_exchange(i8* %a, i8 0, i32 2), !dbg
 
 define void @atomic8_add_acquire(i8* %a) nounwind uwtable {
 entry:
-  atomicrmw add i8* %a, i8 0 acquire
-  ret void
+  atomicrmw add i8* %a, i8 0 acquire, !dbg !7
+  ret void, !dbg !7
 }
-; CHECK: atomic8_add_acquire
-; CHECK: call i8 @__tsan_atomic8_fetch_add(i8* %a, i8 0, i32 2)
+; CHECK-LABEL: atomic8_add_acquire
+; CHECK: call i8 @__tsan_atomic8_fetch_add(i8* %a, i8 0, i32 2), !dbg
 
 define void @atomic8_sub_acquire(i8* %a) nounwind uwtable {
 entry:
-  atomicrmw sub i8* %a, i8 0 acquire
-  ret void
+  atomicrmw sub i8* %a, i8 0 acquire, !dbg !7
+  ret void, !dbg !7
 }
-; CHECK: atomic8_sub_acquire
-; CHECK: call i8 @__tsan_atomic8_fetch_sub(i8* %a, i8 0, i32 2)
+; CHECK-LABEL: atomic8_sub_acquire
+; CHECK: call i8 @__tsan_atomic8_fetch_sub(i8* %a, i8 0, i32 2), !dbg
 
 define void @atomic8_and_acquire(i8* %a) nounwind uwtable {
 entry:
-  atomicrmw and i8* %a, i8 0 acquire
-  ret void
+  atomicrmw and i8* %a, i8 0 acquire, !dbg !7
+  ret void, !dbg !7
 }
-; CHECK: atomic8_and_acquire
-; CHECK: call i8 @__tsan_atomic8_fetch_and(i8* %a, i8 0, i32 2)
+; CHECK-LABEL: atomic8_and_acquire
+; CHECK: call i8 @__tsan_atomic8_fetch_and(i8* %a, i8 0, i32 2), !dbg
 
 define void @atomic8_or_acquire(i8* %a) nounwind uwtable {
 entry:
-  atomicrmw or i8* %a, i8 0 acquire
-  ret void
+  atomicrmw or i8* %a, i8 0 acquire, !dbg !7
+  ret void, !dbg !7
 }
-; CHECK: atomic8_or_acquire
-; CHECK: call i8 @__tsan_atomic8_fetch_or(i8* %a, i8 0, i32 2)
+; CHECK-LABEL: atomic8_or_acquire
+; CHECK: call i8 @__tsan_atomic8_fetch_or(i8* %a, i8 0, i32 2), !dbg
 
 define void @atomic8_xor_acquire(i8* %a) nounwind uwtable {
 entry:
-  atomicrmw xor i8* %a, i8 0 acquire
-  ret void
+  atomicrmw xor i8* %a, i8 0 acquire, !dbg !7
+  ret void, !dbg !7
 }
-; CHECK: atomic8_xor_acquire
-; CHECK: call i8 @__tsan_atomic8_fetch_xor(i8* %a, i8 0, i32 2)
+; CHECK-LABEL: atomic8_xor_acquire
+; CHECK: call i8 @__tsan_atomic8_fetch_xor(i8* %a, i8 0, i32 2), !dbg
 
 define void @atomic8_nand_acquire(i8* %a) nounwind uwtable {
 entry:
-  atomicrmw nand i8* %a, i8 0 acquire
-  ret void
+  atomicrmw nand i8* %a, i8 0 acquire, !dbg !7
+  ret void, !dbg !7
 }
-; CHECK: atomic8_nand_acquire
-; CHECK: call i8 @__tsan_atomic8_fetch_nand(i8* %a, i8 0, i32 2)
+; CHECK-LABEL: atomic8_nand_acquire
+; CHECK: call i8 @__tsan_atomic8_fetch_nand(i8* %a, i8 0, i32 2), !dbg
 
 define void @atomic8_xchg_release(i8* %a) nounwind uwtable {
 entry:
-  atomicrmw xchg i8* %a, i8 0 release
-  ret void
+  atomicrmw xchg i8* %a, i8 0 release, !dbg !7
+  ret void, !dbg !7
 }
-; CHECK: atomic8_xchg_release
-; CHECK: call i8 @__tsan_atomic8_exchange(i8* %a, i8 0, i32 3)
+; CHECK-LABEL: atomic8_xchg_release
+; CHECK: call i8 @__tsan_atomic8_exchange(i8* %a, i8 0, i32 3), !dbg
 
 define void @atomic8_add_release(i8* %a) nounwind uwtable {
 entry:
-  atomicrmw add i8* %a, i8 0 release
-  ret void
+  atomicrmw add i8* %a, i8 0 release, !dbg !7
+  ret void, !dbg !7
 }
-; CHECK: atomic8_add_release
-; CHECK: call i8 @__tsan_atomic8_fetch_add(i8* %a, i8 0, i32 3)
+; CHECK-LABEL: atomic8_add_release
+; CHECK: call i8 @__tsan_atomic8_fetch_add(i8* %a, i8 0, i32 3), !dbg
 
 define void @atomic8_sub_release(i8* %a) nounwind uwtable {
 entry:
-  atomicrmw sub i8* %a, i8 0 release
-  ret void
+  atomicrmw sub i8* %a, i8 0 release, !dbg !7
+  ret void, !dbg !7
 }
-; CHECK: atomic8_sub_release
-; CHECK: call i8 @__tsan_atomic8_fetch_sub(i8* %a, i8 0, i32 3)
+; CHECK-LABEL: atomic8_sub_release
+; CHECK: call i8 @__tsan_atomic8_fetch_sub(i8* %a, i8 0, i32 3), !dbg
 
 define void @atomic8_and_release(i8* %a) nounwind uwtable {
 entry:
-  atomicrmw and i8* %a, i8 0 release
-  ret void
+  atomicrmw and i8* %a, i8 0 release, !dbg !7
+  ret void, !dbg !7
 }
-; CHECK: atomic8_and_release
-; CHECK: call i8 @__tsan_atomic8_fetch_and(i8* %a, i8 0, i32 3)
+; CHECK-LABEL: atomic8_and_release
+; CHECK: call i8 @__tsan_atomic8_fetch_and(i8* %a, i8 0, i32 3), !dbg
 
 define void @atomic8_or_release(i8* %a) nounwind uwtable {
 entry:
-  atomicrmw or i8* %a, i8 0 release
-  ret void
+  atomicrmw or i8* %a, i8 0 release, !dbg !7
+  ret void, !dbg !7
 }
-; CHECK: atomic8_or_release
-; CHECK: call i8 @__tsan_atomic8_fetch_or(i8* %a, i8 0, i32 3)
+; CHECK-LABEL: atomic8_or_release
+; CHECK: call i8 @__tsan_atomic8_fetch_or(i8* %a, i8 0, i32 3), !dbg
 
 define void @atomic8_xor_release(i8* %a) nounwind uwtable {
 entry:
-  atomicrmw xor i8* %a, i8 0 release
-  ret void
+  atomicrmw xor i8* %a, i8 0 release, !dbg !7
+  ret void, !dbg !7
 }
-; CHECK: atomic8_xor_release
-; CHECK: call i8 @__tsan_atomic8_fetch_xor(i8* %a, i8 0, i32 3)
+; CHECK-LABEL: atomic8_xor_release
+; CHECK: call i8 @__tsan_atomic8_fetch_xor(i8* %a, i8 0, i32 3), !dbg
 
 define void @atomic8_nand_release(i8* %a) nounwind uwtable {
 entry:
-  atomicrmw nand i8* %a, i8 0 release
-  ret void
+  atomicrmw nand i8* %a, i8 0 release, !dbg !7
+  ret void, !dbg !7
 }
-; CHECK: atomic8_nand_release
-; CHECK: call i8 @__tsan_atomic8_fetch_nand(i8* %a, i8 0, i32 3)
+; CHECK-LABEL: atomic8_nand_release
+; CHECK: call i8 @__tsan_atomic8_fetch_nand(i8* %a, i8 0, i32 3), !dbg
 
 define void @atomic8_xchg_acq_rel(i8* %a) nounwind uwtable {
 entry:
-  atomicrmw xchg i8* %a, i8 0 acq_rel
-  ret void
+  atomicrmw xchg i8* %a, i8 0 acq_rel, !dbg !7
+  ret void, !dbg !7
 }
-; CHECK: atomic8_xchg_acq_rel
-; CHECK: call i8 @__tsan_atomic8_exchange(i8* %a, i8 0, i32 4)
+; CHECK-LABEL: atomic8_xchg_acq_rel
+; CHECK: call i8 @__tsan_atomic8_exchange(i8* %a, i8 0, i32 4), !dbg
 
 define void @atomic8_add_acq_rel(i8* %a) nounwind uwtable {
 entry:
-  atomicrmw add i8* %a, i8 0 acq_rel
-  ret void
+  atomicrmw add i8* %a, i8 0 acq_rel, !dbg !7
+  ret void, !dbg !7
 }
-; CHECK: atomic8_add_acq_rel
-; CHECK: call i8 @__tsan_atomic8_fetch_add(i8* %a, i8 0, i32 4)
+; CHECK-LABEL: atomic8_add_acq_rel
+; CHECK: call i8 @__tsan_atomic8_fetch_add(i8* %a, i8 0, i32 4), !dbg
 
 define void @atomic8_sub_acq_rel(i8* %a) nounwind uwtable {
 entry:
-  atomicrmw sub i8* %a, i8 0 acq_rel
-  ret void
+  atomicrmw sub i8* %a, i8 0 acq_rel, !dbg !7
+  ret void, !dbg !7
 }
-; CHECK: atomic8_sub_acq_rel
-; CHECK: call i8 @__tsan_atomic8_fetch_sub(i8* %a, i8 0, i32 4)
+; CHECK-LABEL: atomic8_sub_acq_rel
+; CHECK: call i8 @__tsan_atomic8_fetch_sub(i8* %a, i8 0, i32 4), !dbg
 
 define void @atomic8_and_acq_rel(i8* %a) nounwind uwtable {
 entry:
-  atomicrmw and i8* %a, i8 0 acq_rel
-  ret void
+  atomicrmw and i8* %a, i8 0 acq_rel, !dbg !7
+  ret void, !dbg !7
 }
-; CHECK: atomic8_and_acq_rel
-; CHECK: call i8 @__tsan_atomic8_fetch_and(i8* %a, i8 0, i32 4)
+; CHECK-LABEL: atomic8_and_acq_rel
+; CHECK: call i8 @__tsan_atomic8_fetch_and(i8* %a, i8 0, i32 4), !dbg
 
 define void @atomic8_or_acq_rel(i8* %a) nounwind uwtable {
 entry:
-  atomicrmw or i8* %a, i8 0 acq_rel
-  ret void
+  atomicrmw or i8* %a, i8 0 acq_rel, !dbg !7
+  ret void, !dbg !7
 }
-; CHECK: atomic8_or_acq_rel
-; CHECK: call i8 @__tsan_atomic8_fetch_or(i8* %a, i8 0, i32 4)
+; CHECK-LABEL: atomic8_or_acq_rel
+; CHECK: call i8 @__tsan_atomic8_fetch_or(i8* %a, i8 0, i32 4), !dbg
 
 define void @atomic8_xor_acq_rel(i8* %a) nounwind uwtable {
 entry:
-  atomicrmw xor i8* %a, i8 0 acq_rel
-  ret void
+  atomicrmw xor i8* %a, i8 0 acq_rel, !dbg !7
+  ret void, !dbg !7
 }
-; CHECK: atomic8_xor_acq_rel
-; CHECK: call i8 @__tsan_atomic8_fetch_xor(i8* %a, i8 0, i32 4)
+; CHECK-LABEL: atomic8_xor_acq_rel
+; CHECK: call i8 @__tsan_atomic8_fetch_xor(i8* %a, i8 0, i32 4), !dbg
 
 define void @atomic8_nand_acq_rel(i8* %a) nounwind uwtable {
 entry:
-  atomicrmw nand i8* %a, i8 0 acq_rel
-  ret void
+  atomicrmw nand i8* %a, i8 0 acq_rel, !dbg !7
+  ret void, !dbg !7
 }
-; CHECK: atomic8_nand_acq_rel
-; CHECK: call i8 @__tsan_atomic8_fetch_nand(i8* %a, i8 0, i32 4)
+; CHECK-LABEL: atomic8_nand_acq_rel
+; CHECK: call i8 @__tsan_atomic8_fetch_nand(i8* %a, i8 0, i32 4), !dbg
 
 define void @atomic8_xchg_seq_cst(i8* %a) nounwind uwtable {
 entry:
-  atomicrmw xchg i8* %a, i8 0 seq_cst
-  ret void
+  atomicrmw xchg i8* %a, i8 0 seq_cst, !dbg !7
+  ret void, !dbg !7
 }
-; CHECK: atomic8_xchg_seq_cst
-; CHECK: call i8 @__tsan_atomic8_exchange(i8* %a, i8 0, i32 5)
+; CHECK-LABEL: atomic8_xchg_seq_cst
+; CHECK: call i8 @__tsan_atomic8_exchange(i8* %a, i8 0, i32 5), !dbg
 
 define void @atomic8_add_seq_cst(i8* %a) nounwind uwtable {
 entry:
-  atomicrmw add i8* %a, i8 0 seq_cst
-  ret void
+  atomicrmw add i8* %a, i8 0 seq_cst, !dbg !7
+  ret void, !dbg !7
 }
-; CHECK: atomic8_add_seq_cst
-; CHECK: call i8 @__tsan_atomic8_fetch_add(i8* %a, i8 0, i32 5)
+; CHECK-LABEL: atomic8_add_seq_cst
+; CHECK: call i8 @__tsan_atomic8_fetch_add(i8* %a, i8 0, i32 5), !dbg
 
 define void @atomic8_sub_seq_cst(i8* %a) nounwind uwtable {
 entry:
-  atomicrmw sub i8* %a, i8 0 seq_cst
-  ret void
+  atomicrmw sub i8* %a, i8 0 seq_cst, !dbg !7
+  ret void, !dbg !7
 }
-; CHECK: atomic8_sub_seq_cst
-; CHECK: call i8 @__tsan_atomic8_fetch_sub(i8* %a, i8 0, i32 5)
+; CHECK-LABEL: atomic8_sub_seq_cst
+; CHECK: call i8 @__tsan_atomic8_fetch_sub(i8* %a, i8 0, i32 5), !dbg
 
 define void @atomic8_and_seq_cst(i8* %a) nounwind uwtable {
 entry:
-  atomicrmw and i8* %a, i8 0 seq_cst
-  ret void
+  atomicrmw and i8* %a, i8 0 seq_cst, !dbg !7
+  ret void, !dbg !7
 }
-; CHECK: atomic8_and_seq_cst
-; CHECK: call i8 @__tsan_atomic8_fetch_and(i8* %a, i8 0, i32 5)
+; CHECK-LABEL: atomic8_and_seq_cst
+; CHECK: call i8 @__tsan_atomic8_fetch_and(i8* %a, i8 0, i32 5), !dbg
 
 define void @atomic8_or_seq_cst(i8* %a) nounwind uwtable {
 entry:
-  atomicrmw or i8* %a, i8 0 seq_cst
-  ret void
+  atomicrmw or i8* %a, i8 0 seq_cst, !dbg !7
+  ret void, !dbg !7
 }
-; CHECK: atomic8_or_seq_cst
-; CHECK: call i8 @__tsan_atomic8_fetch_or(i8* %a, i8 0, i32 5)
+; CHECK-LABEL: atomic8_or_seq_cst
+; CHECK: call i8 @__tsan_atomic8_fetch_or(i8* %a, i8 0, i32 5), !dbg
 
 define void @atomic8_xor_seq_cst(i8* %a) nounwind uwtable {
 entry:
-  atomicrmw xor i8* %a, i8 0 seq_cst
-  ret void
+  atomicrmw xor i8* %a, i8 0 seq_cst, !dbg !7
+  ret void, !dbg !7
 }
-; CHECK: atomic8_xor_seq_cst
-; CHECK: call i8 @__tsan_atomic8_fetch_xor(i8* %a, i8 0, i32 5)
+; CHECK-LABEL: atomic8_xor_seq_cst
+; CHECK: call i8 @__tsan_atomic8_fetch_xor(i8* %a, i8 0, i32 5), !dbg
 
 define void @atomic8_nand_seq_cst(i8* %a) nounwind uwtable {
 entry:
-  atomicrmw nand i8* %a, i8 0 seq_cst
-  ret void
+  atomicrmw nand i8* %a, i8 0 seq_cst, !dbg !7
+  ret void, !dbg !7
 }
-; CHECK: atomic8_nand_seq_cst
-; CHECK: call i8 @__tsan_atomic8_fetch_nand(i8* %a, i8 0, i32 5)
+; CHECK-LABEL: atomic8_nand_seq_cst
+; CHECK: call i8 @__tsan_atomic8_fetch_nand(i8* %a, i8 0, i32 5), !dbg
 
 define void @atomic8_cas_monotonic(i8* %a) nounwind uwtable {
 entry:
-  cmpxchg i8* %a, i8 0, i8 1 monotonic monotonic
-  ret void
+  cmpxchg i8* %a, i8 0, i8 1 monotonic monotonic, !dbg !7
+  ret void, !dbg !7
 }
-; CHECK: atomic8_cas_monotonic
-; CHECK: call i8 @__tsan_atomic8_compare_exchange_val(i8* %a, i8 0, i8 1, i32 0, i32 0)
+; CHECK-LABEL: atomic8_cas_monotonic
+; CHECK: call i8 @__tsan_atomic8_compare_exchange_val(i8* %a, i8 0, i8 1, i32 0, i32 0), !dbg
 
 define void @atomic8_cas_acquire(i8* %a) nounwind uwtable {
 entry:
-  cmpxchg i8* %a, i8 0, i8 1 acquire acquire
-  ret void
+  cmpxchg i8* %a, i8 0, i8 1 acquire acquire, !dbg !7
+  ret void, !dbg !7
 }
-; CHECK: atomic8_cas_acquire
-; CHECK: call i8 @__tsan_atomic8_compare_exchange_val(i8* %a, i8 0, i8 1, i32 2, i32 2)
+; CHECK-LABEL: atomic8_cas_acquire
+; CHECK: call i8 @__tsan_atomic8_compare_exchange_val(i8* %a, i8 0, i8 1, i32 2, i32 2), !dbg
 
 define void @atomic8_cas_release(i8* %a) nounwind uwtable {
 entry:
-  cmpxchg i8* %a, i8 0, i8 1 release monotonic
-  ret void
+  cmpxchg i8* %a, i8 0, i8 1 release monotonic, !dbg !7
+  ret void, !dbg !7
 }
-; CHECK: atomic8_cas_release
-; CHECK: call i8 @__tsan_atomic8_compare_exchange_val(i8* %a, i8 0, i8 1, i32 3, i32 0)
+; CHECK-LABEL: atomic8_cas_release
+; CHECK: call i8 @__tsan_atomic8_compare_exchange_val(i8* %a, i8 0, i8 1, i32 3, i32 0), !dbg
 
 define void @atomic8_cas_acq_rel(i8* %a) nounwind uwtable {
 entry:
-  cmpxchg i8* %a, i8 0, i8 1 acq_rel acquire
-  ret void
+  cmpxchg i8* %a, i8 0, i8 1 acq_rel acquire, !dbg !7
+  ret void, !dbg !7
 }
-; CHECK: atomic8_cas_acq_rel
-; CHECK: call i8 @__tsan_atomic8_compare_exchange_val(i8* %a, i8 0, i8 1, i32 4, i32 2)
+; CHECK-LABEL: atomic8_cas_acq_rel
+; CHECK: call i8 @__tsan_atomic8_compare_exchange_val(i8* %a, i8 0, i8 1, i32 4, i32 2), !dbg
 
 define void @atomic8_cas_seq_cst(i8* %a) nounwind uwtable {
 entry:
-  cmpxchg i8* %a, i8 0, i8 1 seq_cst seq_cst
-  ret void
+  cmpxchg i8* %a, i8 0, i8 1 seq_cst seq_cst, !dbg !7
+  ret void, !dbg !7
 }
-; CHECK: atomic8_cas_seq_cst
-; CHECK: call i8 @__tsan_atomic8_compare_exchange_val(i8* %a, i8 0, i8 1, i32 5, i32 5)
+; CHECK-LABEL: atomic8_cas_seq_cst
+; CHECK: call i8 @__tsan_atomic8_compare_exchange_val(i8* %a, i8 0, i8 1, i32 5, i32 5), !dbg
 
 define i16 @atomic16_load_unordered(i16* %a) nounwind uwtable {
 entry:
-  %0 = load atomic i16, i16* %a unordered, align 2
-  ret i16 %0
+  %0 = load atomic i16, i16* %a unordered, align 2, !dbg !7
+  ret i16 %0, !dbg !7
 }
-; CHECK: atomic16_load_unordered
-; CHECK: call i16 @__tsan_atomic16_load(i16* %a, i32 0)
+; CHECK-LABEL: atomic16_load_unordered
+; CHECK: call i16 @__tsan_atomic16_load(i16* %a, i32 0), !dbg
 
 define i16 @atomic16_load_monotonic(i16* %a) nounwind uwtable {
 entry:
-  %0 = load atomic i16, i16* %a monotonic, align 2
-  ret i16 %0
+  %0 = load atomic i16, i16* %a monotonic, align 2, !dbg !7
+  ret i16 %0, !dbg !7
 }
-; CHECK: atomic16_load_monotonic
-; CHECK: call i16 @__tsan_atomic16_load(i16* %a, i32 0)
+; CHECK-LABEL: atomic16_load_monotonic
+; CHECK: call i16 @__tsan_atomic16_load(i16* %a, i32 0), !dbg
 
 define i16 @atomic16_load_acquire(i16* %a) nounwind uwtable {
 entry:
-  %0 = load atomic i16, i16* %a acquire, align 2
-  ret i16 %0
+  %0 = load atomic i16, i16* %a acquire, align 2, !dbg !7
+  ret i16 %0, !dbg !7
 }
-; CHECK: atomic16_load_acquire
-; CHECK: call i16 @__tsan_atomic16_load(i16* %a, i32 2)
+; CHECK-LABEL: atomic16_load_acquire
+; CHECK: call i16 @__tsan_atomic16_load(i16* %a, i32 2), !dbg
 
 define i16 @atomic16_load_seq_cst(i16* %a) nounwind uwtable {
 entry:
-  %0 = load atomic i16, i16* %a seq_cst, align 2
-  ret i16 %0
+  %0 = load atomic i16, i16* %a seq_cst, align 2, !dbg !7
+  ret i16 %0, !dbg !7
 }
-; CHECK: atomic16_load_seq_cst
-; CHECK: call i16 @__tsan_atomic16_load(i16* %a, i32 5)
+; CHECK-LABEL: atomic16_load_seq_cst
+; CHECK: call i16 @__tsan_atomic16_load(i16* %a, i32 5), !dbg
 
 define void @atomic16_store_unordered(i16* %a) nounwind uwtable {
 entry:
-  store atomic i16 0, i16* %a unordered, align 2
-  ret void
+  store atomic i16 0, i16* %a unordered, align 2, !dbg !7
+  ret void, !dbg !7
 }
-; CHECK: atomic16_store_unordered
-; CHECK: call void @__tsan_atomic16_store(i16* %a, i16 0, i32 0)
+; CHECK-LABEL: atomic16_store_unordered
+; CHECK: call void @__tsan_atomic16_store(i16* %a, i16 0, i32 0), !dbg
 
 define void @atomic16_store_monotonic(i16* %a) nounwind uwtable {
 entry:
-  store atomic i16 0, i16* %a monotonic, align 2
-  ret void
+  store atomic i16 0, i16* %a monotonic, align 2, !dbg !7
+  ret void, !dbg !7
 }
-; CHECK: atomic16_store_monotonic
-; CHECK: call void @__tsan_atomic16_store(i16* %a, i16 0, i32 0)
+; CHECK-LABEL: atomic16_store_monotonic
+; CHECK: call void @__tsan_atomic16_store(i16* %a, i16 0, i32 0), !dbg
 
 define void @atomic16_store_release(i16* %a) nounwind uwtable {
 entry:
-  store atomic i16 0, i16* %a release, align 2
-  ret void
+  store atomic i16 0, i16* %a release, align 2, !dbg !7
+  ret void, !dbg !7
 }
-; CHECK: atomic16_store_release
-; CHECK: call void @__tsan_atomic16_store(i16* %a, i16 0, i32 3)
+; CHECK-LABEL: atomic16_store_release
+; CHECK: call void @__tsan_atomic16_store(i16* %a, i16 0, i32 3), !dbg
 
 define void @atomic16_store_seq_cst(i16* %a) nounwind uwtable {
 entry:
-  store atomic i16 0, i16* %a seq_cst, align 2
-  ret void
+  store atomic i16 0, i16* %a seq_cst, align 2, !dbg !7
+  ret void, !dbg !7
 }
-; CHECK: atomic16_store_seq_cst
-; CHECK: call void @__tsan_atomic16_store(i16* %a, i16 0, i32 5)
+; CHECK-LABEL: atomic16_store_seq_cst
+; CHECK: call void @__tsan_atomic16_store(i16* %a, i16 0, i32 5), !dbg
 
 define void @atomic16_xchg_monotonic(i16* %a) nounwind uwtable {
 entry:
-  atomicrmw xchg i16* %a, i16 0 monotonic
-  ret void
+  atomicrmw xchg i16* %a, i16 0 monotonic, !dbg !7
+  ret void, !dbg !7
 }
-; CHECK: atomic16_xchg_monotonic
-; CHECK: call i16 @__tsan_atomic16_exchange(i16* %a, i16 0, i32 0)
+; CHECK-LABEL: atomic16_xchg_monotonic
+; CHECK: call i16 @__tsan_atomic16_exchange(i16* %a, i16 0, i32 0), !dbg
 
 define void @atomic16_add_monotonic(i16* %a) nounwind uwtable {
 entry:
-  atomicrmw add i16* %a, i16 0 monotonic
-  ret void
+  atomicrmw add i16* %a, i16 0 monotonic, !dbg !7
+  ret void, !dbg !7
 }
-; CHECK: atomic16_add_monotonic
-; CHECK: call i16 @__tsan_atomic16_fetch_add(i16* %a, i16 0, i32 0)
+; CHECK-LABEL: atomic16_add_monotonic
+; CHECK: call i16 @__tsan_atomic16_fetch_add(i16* %a, i16 0, i32 0), !dbg
 
 define void @atomic16_sub_monotonic(i16* %a) nounwind uwtable {
 entry:
-  atomicrmw sub i16* %a, i16 0 monotonic
-  ret void
+  atomicrmw sub i16* %a, i16 0 monotonic, !dbg !7
+  ret void, !dbg !7
 }
-; CHECK: atomic16_sub_monotonic
-; CHECK: call i16 @__tsan_atomic16_fetch_sub(i16* %a, i16 0, i32 0)
+; CHECK-LABEL: atomic16_sub_monotonic
+; CHECK: call i16 @__tsan_atomic16_fetch_sub(i16* %a, i16 0, i32 0), !dbg
 
 define void @atomic16_and_monotonic(i16* %a) nounwind uwtable {
 entry:
-  atomicrmw and i16* %a, i16 0 monotonic
-  ret void
+  atomicrmw and i16* %a, i16 0 monotonic, !dbg !7
+  ret void, !dbg !7
 }
-; CHECK: atomic16_and_monotonic
-; CHECK: call i16 @__tsan_atomic16_fetch_and(i16* %a, i16 0, i32 0)
+; CHECK-LABEL: atomic16_and_monotonic
+; CHECK: call i16 @__tsan_atomic16_fetch_and(i16* %a, i16 0, i32 0), !dbg
 
 define void @atomic16_or_monotonic(i16* %a) nounwind uwtable {
 entry:
-  atomicrmw or i16* %a, i16 0 monotonic
-  ret void
+  atomicrmw or i16* %a, i16 0 monotonic, !dbg !7
+  ret void, !dbg !7
 }
-; CHECK: atomic16_or_monotonic
-; CHECK: call i16 @__tsan_atomic16_fetch_or(i16* %a, i16 0, i32 0)
+; CHECK-LABEL: atomic16_or_monotonic
+; CHECK: call i16 @__tsan_atomic16_fetch_or(i16* %a, i16 0, i32 0), !dbg
 
 define void @atomic16_xor_monotonic(i16* %a) nounwind uwtable {
 entry:
-  atomicrmw xor i16* %a, i16 0 monotonic
-  ret void
+  atomicrmw xor i16* %a, i16 0 monotonic, !dbg !7
+  ret void, !dbg !7
 }
-; CHECK: atomic16_xor_monotonic
-; CHECK: call i16 @__tsan_atomic16_fetch_xor(i16* %a, i16 0, i32 0)
+; CHECK-LABEL: atomic16_xor_monotonic
+; CHECK: call i16 @__tsan_atomic16_fetch_xor(i16* %a, i16 0, i32 0), !dbg
 
 define void @atomic16_nand_monotonic(i16* %a) nounwind uwtable {
 entry:
-  atomicrmw nand i16* %a, i16 0 monotonic
-  ret void
+  atomicrmw nand i16* %a, i16 0 monotonic, !dbg !7
+  ret void, !dbg !7
 }
-; CHECK: atomic16_nand_monotonic
-; CHECK: call i16 @__tsan_atomic16_fetch_nand(i16* %a, i16 0, i32 0)
+; CHECK-LABEL: atomic16_nand_monotonic
+; CHECK: call i16 @__tsan_atomic16_fetch_nand(i16* %a, i16 0, i32 0), !dbg
 
 define void @atomic16_xchg_acquire(i16* %a) nounwind uwtable {
 entry:
-  atomicrmw xchg i16* %a, i16 0 acquire
-  ret void
+  atomicrmw xchg i16* %a, i16 0 acquire, !dbg !7
+  ret void, !dbg !7
 }
-; CHECK: atomic16_xchg_acquire
-; CHECK: call i16 @__tsan_atomic16_exchange(i16* %a, i16 0, i32 2)
+; CHECK-LABEL: atomic16_xchg_acquire
+; CHECK: call i16 @__tsan_atomic16_exchange(i16* %a, i16 0, i32 2), !dbg
 
 define void @atomic16_add_acquire(i16* %a) nounwind uwtable {
 entry:
-  atomicrmw add i16* %a, i16 0 acquire
-  ret void
+  atomicrmw add i16* %a, i16 0 acquire, !dbg !7
+  ret void, !dbg !7
 }
-; CHECK: atomic16_add_acquire
-; CHECK: call i16 @__tsan_atomic16_fetch_add(i16* %a, i16 0, i32 2)
+; CHECK-LABEL: atomic16_add_acquire
+; CHECK: call i16 @__tsan_atomic16_fetch_add(i16* %a, i16 0, i32 2), !dbg
 
 define void @atomic16_sub_acquire(i16* %a) nounwind uwtable {
 entry:
-  atomicrmw sub i16* %a, i16 0 acquire
-  ret void
+  atomicrmw sub i16* %a, i16 0 acquire, !dbg !7
+  ret void, !dbg !7
 }
-; CHECK: atomic16_sub_acquire
-; CHECK: call i16 @__tsan_atomic16_fetch_sub(i16* %a, i16 0, i32 2)
+; CHECK-LABEL: atomic16_sub_acquire
+; CHECK: call i16 @__tsan_atomic16_fetch_sub(i16* %a, i16 0, i32 2), !dbg
 
 define void @atomic16_and_acquire(i16* %a) nounwind uwtable {
 entry:
-  atomicrmw and i16* %a, i16 0 acquire
-  ret void
+  atomicrmw and i16* %a, i16 0 acquire, !dbg !7
+  ret void, !dbg !7
 }
-; CHECK: atomic16_and_acquire
-; CHECK: call i16 @__tsan_atomic16_fetch_and(i16* %a, i16 0, i32 2)
+; CHECK-LABEL: atomic16_and_acquire
+; CHECK: call i16 @__tsan_atomic16_fetch_and(i16* %a, i16 0, i32 2), !dbg
 
 define void @atomic16_or_acquire(i16* %a) nounwind uwtable {
 entry:
-  atomicrmw or i16* %a, i16 0 acquire
-  ret void
+  atomicrmw or i16* %a, i16 0 acquire, !dbg !7
+  ret void, !dbg !7
 }
-; CHECK: atomic16_or_acquire
-; CHECK: call i16 @__tsan_atomic16_fetch_or(i16* %a, i16 0, i32 2)
+; CHECK-LABEL: atomic16_or_acquire
+; CHECK: call i16 @__tsan_atomic16_fetch_or(i16* %a, i16 0, i32 2), !dbg
 
 define void @atomic16_xor_acquire(i16* %a) nounwind uwtable {
 entry:
-  atomicrmw xor i16* %a, i16 0 acquire
-  ret void
+  atomicrmw xor i16* %a, i16 0 acquire, !dbg !7
+  ret void, !dbg !7
 }
-; CHECK: atomic16_xor_acquire
-; CHECK: call i16 @__tsan_atomic16_fetch_xor(i16* %a, i16 0, i32 2)
+; CHECK-LABEL: atomic16_xor_acquire
+; CHECK: call i16 @__tsan_atomic16_fetch_xor(i16* %a, i16 0, i32 2), !dbg
 
 define void @atomic16_nand_acquire(i16* %a) nounwind uwtable {
 entry:
-  atomicrmw nand i16* %a, i16 0 acquire
-  ret void
+  atomicrmw nand i16* %a, i16 0 acquire, !dbg !7
+  ret void, !dbg !7
 }
-; CHECK: atomic16_nand_acquire
-; CHECK: call i16 @__tsan_atomic16_fetch_nand(i16* %a, i16 0, i32 2)
+; CHECK-LABEL: atomic16_nand_acquire
+; CHECK: call i16 @__tsan_atomic16_fetch_nand(i16* %a, i16 0, i32 2), !dbg
 
 define void @atomic16_xchg_release(i16* %a) nounwind uwtable {
 entry:
-  atomicrmw xchg i16* %a, i16 0 release
-  ret void
+  atomicrmw xchg i16* %a, i16 0 release, !dbg !7
+  ret void, !dbg !7
 }
-; CHECK: atomic16_xchg_release
-; CHECK: call i16 @__tsan_atomic16_exchange(i16* %a, i16 0, i32 3)
+; CHECK-LABEL: atomic16_xchg_release
+; CHECK: call i16 @__tsan_atomic16_exchange(i16* %a, i16 0, i32 3), !dbg
 
 define void @atomic16_add_release(i16* %a) nounwind uwtable {
 entry:
-  atomicrmw add i16* %a, i16 0 release
-  ret void
+  atomicrmw add i16* %a, i16 0 release, !dbg !7
+  ret void, !dbg !7
 }
-; CHECK: atomic16_add_release
-; CHECK: call i16 @__tsan_atomic16_fetch_add(i16* %a, i16 0, i32 3)
+; CHECK-LABEL: atomic16_add_release
+; CHECK: call i16 @__tsan_atomic16_fetch_add(i16* %a, i16 0, i32 3), !dbg
 
 define void @atomic16_sub_release(i16* %a) nounwind uwtable {
 entry:
-  atomicrmw sub i16* %a, i16 0 release
-  ret void
+  atomicrmw sub i16* %a, i16 0 release, !dbg !7
+  ret void, !dbg !7
 }
-; CHECK: atomic16_sub_release
-; CHECK: call i16 @__tsan_atomic16_fetch_sub(i16* %a, i16 0, i32 3)
+; CHECK-LABEL: atomic16_sub_release
+; CHECK: call i16 @__tsan_atomic16_fetch_sub(i16* %a, i16 0, i32 3), !dbg
 
 define void @atomic16_and_release(i16* %a) nounwind uwtable {
 entry:
-  atomicrmw and i16* %a, i16 0 release
-  ret void
+  atomicrmw and i16* %a, i16 0 release, !dbg !7
+  ret void, !dbg !7
 }
-; CHECK: atomic16_and_release
-; CHECK: call i16 @__tsan_atomic16_fetch_and(i16* %a, i16 0, i32 3)
+; CHECK-LABEL: atomic16_and_release
+; CHECK: call i16 @__tsan_atomic16_fetch_and(i16* %a, i16 0, i32 3), !dbg
 
 define void @atomic16_or_release(i16* %a) nounwind uwtable {
 entry:
-  atomicrmw or i16* %a, i16 0 release
-  ret void
+  atomicrmw or i16* %a, i16 0 release, !dbg !7
+  ret void, !dbg !7
 }
-; CHECK: atomic16_or_release
-; CHECK: call i16 @__tsan_atomic16_fetch_or(i16* %a, i16 0, i32 3)
+; CHECK-LABEL: atomic16_or_release
+; CHECK: call i16 @__tsan_atomic16_fetch_or(i16* %a, i16 0, i32 3), !dbg
 
 define void @atomic16_xor_release(i16* %a) nounwind uwtable {
 entry:
-  atomicrmw xor i16* %a, i16 0 release
-  ret void
+  atomicrmw xor i16* %a, i16 0 release, !dbg !7
+  ret void, !dbg !7
 }
-; CHECK: atomic16_xor_release
-; CHECK: call i16 @__tsan_atomic16_fetch_xor(i16* %a, i16 0, i32 3)
+; CHECK-LABEL: atomic16_xor_release
+; CHECK: call i16 @__tsan_atomic16_fetch_xor(i16* %a, i16 0, i32 3), !dbg
 
 define void @atomic16_nand_release(i16* %a) nounwind uwtable {
 entry:
-  atomicrmw nand i16* %a, i16 0 release
-  ret void
+  atomicrmw nand i16* %a, i16 0 release, !dbg !7
+  ret void, !dbg !7
 }
-; CHECK: atomic16_nand_release
-; CHECK: call i16 @__tsan_atomic16_fetch_nand(i16* %a, i16 0, i32 3)
+; CHECK-LABEL: atomic16_nand_release
+; CHECK: call i16 @__tsan_atomic16_fetch_nand(i16* %a, i16 0, i32 3), !dbg
 
 define void @atomic16_xchg_acq_rel(i16* %a) nounwind uwtable {
 entry:
-  atomicrmw xchg i16* %a, i16 0 acq_rel
-  ret void
+  atomicrmw xchg i16* %a, i16 0 acq_rel, !dbg !7
+  ret void, !dbg !7
 }
-; CHECK: atomic16_xchg_acq_rel
-; CHECK: call i16 @__tsan_atomic16_exchange(i16* %a, i16 0, i32 4)
+; CHECK-LABEL: atomic16_xchg_acq_rel
+; CHECK: call i16 @__tsan_atomic16_exchange(i16* %a, i16 0, i32 4), !dbg
 
 define void @atomic16_add_acq_rel(i16* %a) nounwind uwtable {
 entry:
-  atomicrmw add i16* %a, i16 0 acq_rel
-  ret void
+  atomicrmw add i16* %a, i16 0 acq_rel, !dbg !7
+  ret void, !dbg !7
 }
-; CHECK: atomic16_add_acq_rel
-; CHECK: call i16 @__tsan_atomic16_fetch_add(i16* %a, i16 0, i32 4)
+; CHECK-LABEL: atomic16_add_acq_rel
+; CHECK: call i16 @__tsan_atomic16_fetch_add(i16* %a, i16 0, i32 4), !dbg
 
 define void @atomic16_sub_acq_rel(i16* %a) nounwind uwtable {
 entry:
-  atomicrmw sub i16* %a, i16 0 acq_rel
-  ret void
+  atomicrmw sub i16* %a, i16 0 acq_rel, !dbg !7
+  ret void, !dbg !7
 }
-; CHECK: atomic16_sub_acq_rel
-; CHECK: call i16 @__tsan_atomic16_fetch_sub(i16* %a, i16 0, i32 4)
+; CHECK-LABEL: atomic16_sub_acq_rel
+; CHECK: call i16 @__tsan_atomic16_fetch_sub(i16* %a, i16 0, i32 4), !dbg
 
 define void @atomic16_and_acq_rel(i16* %a) nounwind uwtable {
 entry:
-  atomicrmw and i16* %a, i16 0 acq_rel
-  ret void
+  atomicrmw and i16* %a, i16 0 acq_rel, !dbg !7
+  ret void, !dbg !7
 }
-; CHECK: atomic16_and_acq_rel
-; CHECK: call i16 @__tsan_atomic16_fetch_and(i16* %a, i16 0, i32 4)
+; CHECK-LABEL: atomic16_and_acq_rel
+; CHECK: call i16 @__tsan_atomic16_fetch_and(i16* %a, i16 0, i32 4), !dbg
 
 define void @atomic16_or_acq_rel(i16* %a) nounwind uwtable {
 entry:
-  atomicrmw or i16* %a, i16 0 acq_rel
-  ret void
+  atomicrmw or i16* %a, i16 0 acq_rel, !dbg !7
+  ret void, !dbg !7
 }
-; CHECK: atomic16_or_acq_rel
-; CHECK: call i16 @__tsan_atomic16_fetch_or(i16* %a, i16 0, i32 4)
+; CHECK-LABEL: atomic16_or_acq_rel
+; CHECK: call i16 @__tsan_atomic16_fetch_or(i16* %a, i16 0, i32 4), !dbg
 
 define void @atomic16_xor_acq_rel(i16* %a) nounwind uwtable {
 entry:
-  atomicrmw xor i16* %a, i16 0 acq_rel
-  ret void
+  atomicrmw xor i16* %a, i16 0 acq_rel, !dbg !7
+  ret void, !dbg !7
 }
-; CHECK: atomic16_xor_acq_rel
-; CHECK: call i16 @__tsan_atomic16_fetch_xor(i16* %a, i16 0, i32 4)
+; CHECK-LABEL: atomic16_xor_acq_rel
+; CHECK: call i16 @__tsan_atomic16_fetch_xor(i16* %a, i16 0, i32 4), !dbg
 
 define void @atomic16_nand_acq_rel(i16* %a) nounwind uwtable {
 entry:
-  atomicrmw nand i16* %a, i16 0 acq_rel
-  ret void
+  atomicrmw nand i16* %a, i16 0 acq_rel, !dbg !7
+  ret void, !dbg !7
 }
-; CHECK: atomic16_nand_acq_rel
-; CHECK: call i16 @__tsan_atomic16_fetch_nand(i16* %a, i16 0, i32 4)
+; CHECK-LABEL: atomic16_nand_acq_rel
+; CHECK: call i16 @__tsan_atomic16_fetch_nand(i16* %a, i16 0, i32 4), !dbg
 
 define void @atomic16_xchg_seq_cst(i16* %a) nounwind uwtable {
 entry:
-  atomicrmw xchg i16* %a, i16 0 seq_cst
-  ret void
+  atomicrmw xchg i16* %a, i16 0 seq_cst, !dbg !7
+  ret void, !dbg !7
 }
-; CHECK: atomic16_xchg_seq_cst
-; CHECK: call i16 @__tsan_atomic16_exchange(i16* %a, i16 0, i32 5)
+; CHECK-LABEL: atomic16_xchg_seq_cst
+; CHECK: call i16 @__tsan_atomic16_exchange(i16* %a, i16 0, i32 5), !dbg
 
 define void @atomic16_add_seq_cst(i16* %a) nounwind uwtable {
 entry:
-  atomicrmw add i16* %a, i16 0 seq_cst
-  ret void
+  atomicrmw add i16* %a, i16 0 seq_cst, !dbg !7
+  ret void, !dbg !7
 }
-; CHECK: atomic16_add_seq_cst
-; CHECK: call i16 @__tsan_atomic16_fetch_add(i16* %a, i16 0, i32 5)
+; CHECK-LABEL: atomic16_add_seq_cst
+; CHECK: call i16 @__tsan_atomic16_fetch_add(i16* %a, i16 0, i32 5), !dbg
 
 define void @atomic16_sub_seq_cst(i16* %a) nounwind uwtable {
 entry:
-  atomicrmw sub i16* %a, i16 0 seq_cst
-  ret void
+  atomicrmw sub i16* %a, i16 0 seq_cst, !dbg !7
+  ret void, !dbg !7
 }
-; CHECK: atomic16_sub_seq_cst
-; CHECK: call i16 @__tsan_atomic16_fetch_sub(i16* %a, i16 0, i32 5)
+; CHECK-LABEL: atomic16_sub_seq_cst
+; CHECK: call i16 @__tsan_atomic16_fetch_sub(i16* %a, i16 0, i32 5), !dbg
 
 define void @atomic16_and_seq_cst(i16* %a) nounwind uwtable {
 entry:
-  atomicrmw and i16* %a, i16 0 seq_cst
-  ret void
+  atomicrmw and i16* %a, i16 0 seq_cst, !dbg !7
+  ret void, !dbg !7
 }
-; CHECK: atomic16_and_seq_cst
-; CHECK: call i16 @__tsan_atomic16_fetch_and(i16* %a, i16 0, i32 5)
+; CHECK-LABEL: atomic16_and_seq_cst
+; CHECK: call i16 @__tsan_atomic16_fetch_and(i16* %a, i16 0, i32 5), !dbg
 
 define void @atomic16_or_seq_cst(i16* %a) nounwind uwtable {
 entry:
-  atomicrmw or i16* %a, i16 0 seq_cst
-  ret void
+  atomicrmw or i16* %a, i16 0 seq_cst, !dbg !7
+  ret void, !dbg !7
 }
-; CHECK: atomic16_or_seq_cst
-; CHECK: call i16 @__tsan_atomic16_fetch_or(i16* %a, i16 0, i32 5)
+; CHECK-LABEL: atomic16_or_seq_cst
+; CHECK: call i16 @__tsan_atomic16_fetch_or(i16* %a, i16 0, i32 5), !dbg
 
 define void @atomic16_xor_seq_cst(i16* %a) nounwind uwtable {
 entry:
-  atomicrmw xor i16* %a, i16 0 seq_cst
-  ret void
+  atomicrmw xor i16* %a, i16 0 seq_cst, !dbg !7
+  ret void, !dbg !7
 }
-; CHECK: atomic16_xor_seq_cst
-; CHECK: call i16 @__tsan_atomic16_fetch_xor(i16* %a, i16 0, i32 5)
+; CHECK-LABEL: atomic16_xor_seq_cst
+; CHECK: call i16 @__tsan_atomic16_fetch_xor(i16* %a, i16 0, i32 5), !dbg
 
 define void @atomic16_nand_seq_cst(i16* %a) nounwind uwtable {
 entry:
-  atomicrmw nand i16* %a, i16 0 seq_cst
-  ret void
+  atomicrmw nand i16* %a, i16 0 seq_cst, !dbg !7
+  ret void, !dbg !7
 }
-; CHECK: atomic16_nand_seq_cst
-; CHECK: call i16 @__tsan_atomic16_fetch_nand(i16* %a, i16 0, i32 5)
+; CHECK-LABEL: atomic16_nand_seq_cst
+; CHECK: call i16 @__tsan_atomic16_fetch_nand(i16* %a, i16 0, i32 5), !dbg
 
 define void @atomic16_cas_monotonic(i16* %a) nounwind uwtable {
 entry:
-  cmpxchg i16* %a, i16 0, i16 1 monotonic monotonic
-  ret void
+  cmpxchg i16* %a, i16 0, i16 1 monotonic monotonic, !dbg !7
+  ret void, !dbg !7
 }
-; CHECK: atomic16_cas_monotonic
-; CHECK: call i16 @__tsan_atomic16_compare_exchange_val(i16* %a, i16 0, i16 1, i32 0, i32 0)
+; CHECK-LABEL: atomic16_cas_monotonic
+; CHECK: call i16 @__tsan_atomic16_compare_exchange_val(i16* %a, i16 0, i16 1, i32 0, i32 0), !dbg
 
 define void @atomic16_cas_acquire(i16* %a) nounwind uwtable {
 entry:
-  cmpxchg i16* %a, i16 0, i16 1 acquire acquire
-  ret void
+  cmpxchg i16* %a, i16 0, i16 1 acquire acquire, !dbg !7
+  ret void, !dbg !7
 }
-; CHECK: atomic16_cas_acquire
-; CHECK: call i16 @__tsan_atomic16_compare_exchange_val(i16* %a, i16 0, i16 1, i32 2, i32 2)
+; CHECK-LABEL: atomic16_cas_acquire
+; CHECK: call i16 @__tsan_atomic16_compare_exchange_val(i16* %a, i16 0, i16 1, i32 2, i32 2), !dbg
 
 define void @atomic16_cas_release(i16* %a) nounwind uwtable {
 entry:
-  cmpxchg i16* %a, i16 0, i16 1 release monotonic
-  ret void
+  cmpxchg i16* %a, i16 0, i16 1 release monotonic, !dbg !7
+  ret void, !dbg !7
 }
-; CHECK: atomic16_cas_release
-; CHECK: call i16 @__tsan_atomic16_compare_exchange_val(i16* %a, i16 0, i16 1, i32 3, i32 0)
+; CHECK-LABEL: atomic16_cas_release
+; CHECK: call i16 @__tsan_atomic16_compare_exchange_val(i16* %a, i16 0, i16 1, i32 3, i32 0), !dbg
 
 define void @atomic16_cas_acq_rel(i16* %a) nounwind uwtable {
 entry:
-  cmpxchg i16* %a, i16 0, i16 1 acq_rel acquire
-  ret void
+  cmpxchg i16* %a, i16 0, i16 1 acq_rel acquire, !dbg !7
+  ret void, !dbg !7
 }
-; CHECK: atomic16_cas_acq_rel
-; CHECK: call i16 @__tsan_atomic16_compare_exchange_val(i16* %a, i16 0, i16 1, i32 4, i32 2)
+; CHECK-LABEL: atomic16_cas_acq_rel
+; CHECK: call i16 @__tsan_atomic16_compare_exchange_val(i16* %a, i16 0, i16 1, i32 4, i32 2), !dbg
 
 define void @atomic16_cas_seq_cst(i16* %a) nounwind uwtable {
 entry:
-  cmpxchg i16* %a, i16 0, i16 1 seq_cst seq_cst
-  ret void
+  cmpxchg i16* %a, i16 0, i16 1 seq_cst seq_cst, !dbg !7
+  ret void, !dbg !7
 }
-; CHECK: atomic16_cas_seq_cst
-; CHECK: call i16 @__tsan_atomic16_compare_exchange_val(i16* %a, i16 0, i16 1, i32 5, i32 5)
+; CHECK-LABEL: atomic16_cas_seq_cst
+; CHECK: call i16 @__tsan_atomic16_compare_exchange_val(i16* %a, i16 0, i16 1, i32 5, i32 5), !dbg
 
 define i32 @atomic32_load_unordered(i32* %a) nounwind uwtable {
 entry:
-  %0 = load atomic i32, i32* %a unordered, align 4
-  ret i32 %0
+  %0 = load atomic i32, i32* %a unordered, align 4, !dbg !7
+  ret i32 %0, !dbg !7
 }
-; CHECK: atomic32_load_unordered
-; CHECK: call i32 @__tsan_atomic32_load(i32* %a, i32 0)
+; CHECK-LABEL: atomic32_load_unordered
+; CHECK: call i32 @__tsan_atomic32_load(i32* %a, i32 0), !dbg
 
 define i32 @atomic32_load_monotonic(i32* %a) nounwind uwtable {
 entry:
-  %0 = load atomic i32, i32* %a monotonic, align 4
-  ret i32 %0
+  %0 = load atomic i32, i32* %a monotonic, align 4, !dbg !7
+  ret i32 %0, !dbg !7
 }
-; CHECK: atomic32_load_monotonic
-; CHECK: call i32 @__tsan_atomic32_load(i32* %a, i32 0)
+; CHECK-LABEL: atomic32_load_monotonic
+; CHECK: call i32 @__tsan_atomic32_load(i32* %a, i32 0), !dbg
 
 define i32 @atomic32_load_acquire(i32* %a) nounwind uwtable {
 entry:
-  %0 = load atomic i32, i32* %a acquire, align 4
-  ret i32 %0
+  %0 = load atomic i32, i32* %a acquire, align 4, !dbg !7
+  ret i32 %0, !dbg !7
 }
-; CHECK: atomic32_load_acquire
-; CHECK: call i32 @__tsan_atomic32_load(i32* %a, i32 2)
+; CHECK-LABEL: atomic32_load_acquire
+; CHECK: call i32 @__tsan_atomic32_load(i32* %a, i32 2), !dbg
 
 define i32 @atomic32_load_seq_cst(i32* %a) nounwind uwtable {
 entry:
-  %0 = load atomic i32, i32* %a seq_cst, align 4
-  ret i32 %0
+  %0 = load atomic i32, i32* %a seq_cst, align 4, !dbg !7
+  ret i32 %0, !dbg !7
 }
-; CHECK: atomic32_load_seq_cst
-; CHECK: call i32 @__tsan_atomic32_load(i32* %a, i32 5)
+; CHECK-LABEL: atomic32_load_seq_cst
+; CHECK: call i32 @__tsan_atomic32_load(i32* %a, i32 5), !dbg
 
 define void @atomic32_store_unordered(i32* %a) nounwind uwtable {
 entry:
-  store atomic i32 0, i32* %a unordered, align 4
-  ret void
+  store atomic i32 0, i32* %a unordered, align 4, !dbg !7
+  ret void, !dbg !7
 }
-; CHECK: atomic32_store_unordered
-; CHECK: call void @__tsan_atomic32_store(i32* %a, i32 0, i32 0)
+; CHECK-LABEL: atomic32_store_unordered
+; CHECK: call void @__tsan_atomic32_store(i32* %a, i32 0, i32 0), !dbg
 
 define void @atomic32_store_monotonic(i32* %a) nounwind uwtable {
 entry:
-  store atomic i32 0, i32* %a monotonic, align 4
-  ret void
+  store atomic i32 0, i32* %a monotonic, align 4, !dbg !7
+  ret void, !dbg !7
 }
-; CHECK: atomic32_store_monotonic
-; CHECK: call void @__tsan_atomic32_store(i32* %a, i32 0, i32 0)
+; CHECK-LABEL: atomic32_store_monotonic
+; CHECK: call void @__tsan_atomic32_store(i32* %a, i32 0, i32 0), !dbg
 
 define void @atomic32_store_release(i32* %a) nounwind uwtable {
 entry:
-  store atomic i32 0, i32* %a release, align 4
-  ret void
+  store atomic i32 0, i32* %a release, align 4, !dbg !7
+  ret void, !dbg !7
 }
-; CHECK: atomic32_store_release
-; CHECK: call void @__tsan_atomic32_store(i32* %a, i32 0, i32 3)
+; CHECK-LABEL: atomic32_store_release
+; CHECK: call void @__tsan_atomic32_store(i32* %a, i32 0, i32 3), !dbg
 
 define void @atomic32_store_seq_cst(i32* %a) nounwind uwtable {
 entry:
-  store atomic i32 0, i32* %a seq_cst, align 4
-  ret void
+  store atomic i32 0, i32* %a seq_cst, align 4, !dbg !7
+  ret void, !dbg !7
 }
-; CHECK: atomic32_store_seq_cst
-; CHECK: call void @__tsan_atomic32_store(i32* %a, i32 0, i32 5)
+; CHECK-LABEL: atomic32_store_seq_cst
+; CHECK: call void @__tsan_atomic32_store(i32* %a, i32 0, i32 5), !dbg
 
 define void @atomic32_xchg_monotonic(i32* %a) nounwind uwtable {
 entry:
-  atomicrmw xchg i32* %a, i32 0 monotonic
-  ret void
+  atomicrmw xchg i32* %a, i32 0 monotonic, !dbg !7
+  ret void, !dbg !7
 }
-; CHECK: atomic32_xchg_monotonic
-; CHECK: call i32 @__tsan_atomic32_exchange(i32* %a, i32 0, i32 0)
+; CHECK-LABEL: atomic32_xchg_monotonic
+; CHECK: call i32 @__tsan_atomic32_exchange(i32* %a, i32 0, i32 0), !dbg
 
 define void @atomic32_add_monotonic(i32* %a) nounwind uwtable {
 entry:
-  atomicrmw add i32* %a, i32 0 monotonic
-  ret void
+  atomicrmw add i32* %a, i32 0 monotonic, !dbg !7
+  ret void, !dbg !7
 }
-; CHECK: atomic32_add_monotonic
-; CHECK: call i32 @__tsan_atomic32_fetch_add(i32* %a, i32 0, i32 0)
+; CHECK-LABEL: atomic32_add_monotonic
+; CHECK: call i32 @__tsan_atomic32_fetch_add(i32* %a, i32 0, i32 0), !dbg
 
 define void @atomic32_sub_monotonic(i32* %a) nounwind uwtable {
 entry:
-  atomicrmw sub i32* %a, i32 0 monotonic
-  ret void
+  atomicrmw sub i32* %a, i32 0 monotonic, !dbg !7
+  ret void, !dbg !7
 }
-; CHECK: atomic32_sub_monotonic
-; CHECK: call i32 @__tsan_atomic32_fetch_sub(i32* %a, i32 0, i32 0)
+; CHECK-LABEL: atomic32_sub_monotonic
+; CHECK: call i32 @__tsan_atomic32_fetch_sub(i32* %a, i32 0, i32 0), !dbg
 
 define void @atomic32_and_monotonic(i32* %a) nounwind uwtable {
 entry:
-  atomicrmw and i32* %a, i32 0 monotonic
-  ret void
+  atomicrmw and i32* %a, i32 0 monotonic, !dbg !7
+  ret void, !dbg !7
 }
-; CHECK: atomic32_and_monotonic
-; CHECK: call i32 @__tsan_atomic32_fetch_and(i32* %a, i32 0, i32 0)
+; CHECK-LABEL: atomic32_and_monotonic
+; CHECK: call i32 @__tsan_atomic32_fetch_and(i32* %a, i32 0, i32 0), !dbg
 
 define void @atomic32_or_monotonic(i32* %a) nounwind uwtable {
 entry:
-  atomicrmw or i32* %a, i32 0 monotonic
-  ret void
+  atomicrmw or i32* %a, i32 0 monotonic, !dbg !7
+  ret void, !dbg !7
 }
-; CHECK: atomic32_or_monotonic
-; CHECK: call i32 @__tsan_atomic32_fetch_or(i32* %a, i32 0, i32 0)
+; CHECK-LABEL: atomic32_or_monotonic
+; CHECK: call i32 @__tsan_atomic32_fetch_or(i32* %a, i32 0, i32 0), !dbg
 
 define void @atomic32_xor_monotonic(i32* %a) nounwind uwtable {
 entry:
-  atomicrmw xor i32* %a, i32 0 monotonic
-  ret void
+  atomicrmw xor i32* %a, i32 0 monotonic, !dbg !7
+  ret void, !dbg !7
 }
-; CHECK: atomic32_xor_monotonic
-; CHECK: call i32 @__tsan_atomic32_fetch_xor(i32* %a, i32 0, i32 0)
+; CHECK-LABEL: atomic32_xor_monotonic
+; CHECK: call i32 @__tsan_atomic32_fetch_xor(i32* %a, i32 0, i32 0), !dbg
 
 define void @atomic32_nand_monotonic(i32* %a) nounwind uwtable {
 entry:
-  atomicrmw nand i32* %a, i32 0 monotonic
-  ret void
+  atomicrmw nand i32* %a, i32 0 monotonic, !dbg !7
+  ret void, !dbg !7
 }
-; CHECK: atomic32_nand_monotonic
-; CHECK: call i32 @__tsan_atomic32_fetch_nand(i32* %a, i32 0, i32 0)
+; CHECK-LABEL: atomic32_nand_monotonic
+; CHECK: call i32 @__tsan_atomic32_fetch_nand(i32* %a, i32 0, i32 0), !dbg
 
 define void @atomic32_xchg_acquire(i32* %a) nounwind uwtable {
 entry:
-  atomicrmw xchg i32* %a, i32 0 acquire
-  ret void
+  atomicrmw xchg i32* %a, i32 0 acquire, !dbg !7
+  ret void, !dbg !7
 }
-; CHECK: atomic32_xchg_acquire
-; CHECK: call i32 @__tsan_atomic32_exchange(i32* %a, i32 0, i32 2)
+; CHECK-LABEL: atomic32_xchg_acquire
+; CHECK: call i32 @__tsan_atomic32_exchange(i32* %a, i32 0, i32 2), !dbg
 
 define void @atomic32_add_acquire(i32* %a) nounwind uwtable {
 entry:
-  atomicrmw add i32* %a, i32 0 acquire
-  ret void
+  atomicrmw add i32* %a, i32 0 acquire, !dbg !7
+  ret void, !dbg !7
 }
-; CHECK: atomic32_add_acquire
-; CHECK: call i32 @__tsan_atomic32_fetch_add(i32* %a, i32 0, i32 2)
+; CHECK-LABEL: atomic32_add_acquire
+; CHECK: call i32 @__tsan_atomic32_fetch_add(i32* %a, i32 0, i32 2), !dbg
 
 define void @atomic32_sub_acquire(i32* %a) nounwind uwtable {
 entry:
-  atomicrmw sub i32* %a, i32 0 acquire
-  ret void
+  atomicrmw sub i32* %a, i32 0 acquire, !dbg !7
+  ret void, !dbg !7
 }
-; CHECK: atomic32_sub_acquire
-; CHECK: call i32 @__tsan_atomic32_fetch_sub(i32* %a, i32 0, i32 2)
+; CHECK-LABEL: atomic32_sub_acquire
+; CHECK: call i32 @__tsan_atomic32_fetch_sub(i32* %a, i32 0, i32 2), !dbg
 
 define void @atomic32_and_acquire(i32* %a) nounwind uwtable {
 entry:
-  atomicrmw and i32* %a, i32 0 acquire
-  ret void
+  atomicrmw and i32* %a, i32 0 acquire, !dbg !7
+  ret void, !dbg !7
 }
-; CHECK: atomic32_and_acquire
-; CHECK: call i32 @__tsan_atomic32_fetch_and(i32* %a, i32 0, i32 2)
+; CHECK-LABEL: atomic32_and_acquire
+; CHECK: call i32 @__tsan_atomic32_fetch_and(i32* %a, i32 0, i32 2), !dbg
 
 define void @atomic32_or_acquire(i32* %a) nounwind uwtable {
 entry:
-  atomicrmw or i32* %a, i32 0 acquire
-  ret void
+  atomicrmw or i32* %a, i32 0 acquire, !dbg !7
+  ret void, !dbg !7
 }
-; CHECK: atomic32_or_acquire
-; CHECK: call i32 @__tsan_atomic32_fetch_or(i32* %a, i32 0, i32 2)
+; CHECK-LABEL: atomic32_or_acquire
+; CHECK: call i32 @__tsan_atomic32_fetch_or(i32* %a, i32 0, i32 2), !dbg
 
 define void @atomic32_xor_acquire(i32* %a) nounwind uwtable {
 entry:
-  atomicrmw xor i32* %a, i32 0 acquire
-  ret void
+  atomicrmw xor i32* %a, i32 0 acquire, !dbg !7
+  ret void, !dbg !7
 }
-; CHECK: atomic32_xor_acquire
-; CHECK: call i32 @__tsan_atomic32_fetch_xor(i32* %a, i32 0, i32 2)
+; CHECK-LABEL: atomic32_xor_acquire
+; CHECK: call i32 @__tsan_atomic32_fetch_xor(i32* %a, i32 0, i32 2), !dbg
 
 define void @atomic32_nand_acquire(i32* %a) nounwind uwtable {
 entry:
-  atomicrmw nand i32* %a, i32 0 acquire
-  ret void
+  atomicrmw nand i32* %a, i32 0 acquire, !dbg !7
+  ret void, !dbg !7
 }
-; CHECK: atomic32_nand_acquire
-; CHECK: call i32 @__tsan_atomic32_fetch_nand(i32* %a, i32 0, i32 2)
+; CHECK-LABEL: atomic32_nand_acquire
+; CHECK: call i32 @__tsan_atomic32_fetch_nand(i32* %a, i32 0, i32 2), !dbg
 
 define void @atomic32_xchg_release(i32* %a) nounwind uwtable {
 entry:
-  atomicrmw xchg i32* %a, i32 0 release
-  ret void
+  atomicrmw xchg i32* %a, i32 0 release, !dbg !7
+  ret void, !dbg !7
 }
-; CHECK: atomic32_xchg_release
-; CHECK: call i32 @__tsan_atomic32_exchange(i32* %a, i32 0, i32 3)
+; CHECK-LABEL: atomic32_xchg_release
+; CHECK: call i32 @__tsan_atomic32_exchange(i32* %a, i32 0, i32 3), !dbg
 
 define void @atomic32_add_release(i32* %a) nounwind uwtable {
 entry:
-  atomicrmw add i32* %a, i32 0 release
-  ret void
+  atomicrmw add i32* %a, i32 0 release, !dbg !7
+  ret void, !dbg !7
 }
-; CHECK: atomic32_add_release
-; CHECK: call i32 @__tsan_atomic32_fetch_add(i32* %a, i32 0, i32 3)
+; CHECK-LABEL: atomic32_add_release
+; CHECK: call i32 @__tsan_atomic32_fetch_add(i32* %a, i32 0, i32 3), !dbg
 
 define void @atomic32_sub_release(i32* %a) nounwind uwtable {
 entry:
-  atomicrmw sub i32* %a, i32 0 release
-  ret void
+  atomicrmw sub i32* %a, i32 0 release, !dbg !7
+  ret void, !dbg !7
 }
-; CHECK: atomic32_sub_release
-; CHECK: call i32 @__tsan_atomic32_fetch_sub(i32* %a, i32 0, i32 3)
+; CHECK-LABEL: atomic32_sub_release
+; CHECK: call i32 @__tsan_atomic32_fetch_sub(i32* %a, i32 0, i32 3), !dbg
 
 define void @atomic32_and_release(i32* %a) nounwind uwtable {
 entry:
-  atomicrmw and i32* %a, i32 0 release
-  ret void
+  atomicrmw and i32* %a, i32 0 release, !dbg !7
+  ret void, !dbg !7
 }
-; CHECK: atomic32_and_release
-; CHECK: call i32 @__tsan_atomic32_fetch_and(i32* %a, i32 0, i32 3)
+; CHECK-LABEL: atomic32_and_release
+; CHECK: call i32 @__tsan_atomic32_fetch_and(i32* %a, i32 0, i32 3), !dbg
 
 define void @atomic32_or_release(i32* %a) nounwind uwtable {
 entry:
-  atomicrmw or i32* %a, i32 0 release
-  ret void
+  atomicrmw or i32* %a, i32 0 release, !dbg !7
+  ret void, !dbg !7
 }
-; CHECK: atomic32_or_release
-; CHECK: call i32 @__tsan_atomic32_fetch_or(i32* %a, i32 0, i32 3)
+; CHECK-LABEL: atomic32_or_release
+; CHECK: call i32 @__tsan_atomic32_fetch_or(i32* %a, i32 0, i32 3), !dbg
 
 define void @atomic32_xor_release(i32* %a) nounwind uwtable {
 entry:
-  atomicrmw xor i32* %a, i32 0 release
-  ret void
+  atomicrmw xor i32* %a, i32 0 release, !dbg !7
+  ret void, !dbg !7
 }
-; CHECK: atomic32_xor_release
-; CHECK: call i32 @__tsan_atomic32_fetch_xor(i32* %a, i32 0, i32 3)
+; CHECK-LABEL: atomic32_xor_release
+; CHECK: call i32 @__tsan_atomic32_fetch_xor(i32* %a, i32 0, i32 3), !dbg
 
 define void @atomic32_nand_release(i32* %a) nounwind uwtable {
 entry:
-  atomicrmw nand i32* %a, i32 0 release
-  ret void
+  atomicrmw nand i32* %a, i32 0 release, !dbg !7
+  ret void, !dbg !7
 }
-; CHECK: atomic32_nand_release
-; CHECK: call i32 @__tsan_atomic32_fetch_nand(i32* %a, i32 0, i32 3)
+; CHECK-LABEL: atomic32_nand_release
+; CHECK: call i32 @__tsan_atomic32_fetch_nand(i32* %a, i32 0, i32 3), !dbg
 
 define void @atomic32_xchg_acq_rel(i32* %a) nounwind uwtable {
 entry:
-  atomicrmw xchg i32* %a, i32 0 acq_rel
-  ret void
+  atomicrmw xchg i32* %a, i32 0 acq_rel, !dbg !7
+  ret void, !dbg !7
 }
-; CHECK: atomic32_xchg_acq_rel
-; CHECK: call i32 @__tsan_atomic32_exchange(i32* %a, i32 0, i32 4)
+; CHECK-LABEL: atomic32_xchg_acq_rel
+; CHECK: call i32 @__tsan_atomic32_exchange(i32* %a, i32 0, i32 4), !dbg
 
 define void @atomic32_add_acq_rel(i32* %a) nounwind uwtable {
 entry:
-  atomicrmw add i32* %a, i32 0 acq_rel
-  ret void
+  atomicrmw add i32* %a, i32 0 acq_rel, !dbg !7
+  ret void, !dbg !7
 }
-; CHECK: atomic32_add_acq_rel
-; CHECK: call i32 @__tsan_atomic32_fetch_add(i32* %a, i32 0, i32 4)
+; CHECK-LABEL: atomic32_add_acq_rel
+; CHECK: call i32 @__tsan_atomic32_fetch_add(i32* %a, i32 0, i32 4), !dbg
 
 define void @atomic32_sub_acq_rel(i32* %a) nounwind uwtable {
 entry:
-  atomicrmw sub i32* %a, i32 0 acq_rel
-  ret void
+  atomicrmw sub i32* %a, i32 0 acq_rel, !dbg !7
+  ret void, !dbg !7
 }
-; CHECK: atomic32_sub_acq_rel
-; CHECK: call i32 @__tsan_atomic32_fetch_sub(i32* %a, i32 0, i32 4)
+; CHECK-LABEL: atomic32_sub_acq_rel
+; CHECK: call i32 @__tsan_atomic32_fetch_sub(i32* %a, i32 0, i32 4), !dbg
 
 define void @atomic32_and_acq_rel(i32* %a) nounwind uwtable {
 entry:
-  atomicrmw and i32* %a, i32 0 acq_rel
-  ret void
+  atomicrmw and i32* %a, i32 0 acq_rel, !dbg !7
+  ret void, !dbg !7
 }
-; CHECK: atomic32_and_acq_rel
-; CHECK: call i32 @__tsan_atomic32_fetch_and(i32* %a, i32 0, i32 4)
+; CHECK-LABEL: atomic32_and_acq_rel
+; CHECK: call i32 @__tsan_atomic32_fetch_and(i32* %a, i32 0, i32 4), !dbg
 
 define void @atomic32_or_acq_rel(i32* %a) nounwind uwtable {
 entry:
-  atomicrmw or i32* %a, i32 0 acq_rel
-  ret void
+  atomicrmw or i32* %a, i32 0 acq_rel, !dbg !7
+  ret void, !dbg !7
 }
-; CHECK: atomic32_or_acq_rel
-; CHECK: call i32 @__tsan_atomic32_fetch_or(i32* %a, i32 0, i32 4)
+; CHECK-LABEL: atomic32_or_acq_rel
+; CHECK: call i32 @__tsan_atomic32_fetch_or(i32* %a, i32 0, i32 4), !dbg
 
 define void @atomic32_xor_acq_rel(i32* %a) nounwind uwtable {
 entry:
-  atomicrmw xor i32* %a, i32 0 acq_rel
-  ret void
+  atomicrmw xor i32* %a, i32 0 acq_rel, !dbg !7
+  ret void, !dbg !7
 }
-; CHECK: atomic32_xor_acq_rel
-; CHECK: call i32 @__tsan_atomic32_fetch_xor(i32* %a, i32 0, i32 4)
+; CHECK-LABEL: atomic32_xor_acq_rel
+; CHECK: call i32 @__tsan_atomic32_fetch_xor(i32* %a, i32 0, i32 4), !dbg
 
 define void @atomic32_nand_acq_rel(i32* %a) nounwind uwtable {
 entry:
-  atomicrmw nand i32* %a, i32 0 acq_rel
-  ret void
+  atomicrmw nand i32* %a, i32 0 acq_rel, !dbg !7
+  ret void, !dbg !7
 }
-; CHECK: atomic32_nand_acq_rel
-; CHECK: call i32 @__tsan_atomic32_fetch_nand(i32* %a, i32 0, i32 4)
+; CHECK-LABEL: atomic32_nand_acq_rel
+; CHECK: call i32 @__tsan_atomic32_fetch_nand(i32* %a, i32 0, i32 4), !dbg
 
 define void @atomic32_xchg_seq_cst(i32* %a) nounwind uwtable {
 entry:
-  atomicrmw xchg i32* %a, i32 0 seq_cst
-  ret void
+  atomicrmw xchg i32* %a, i32 0 seq_cst, !dbg !7
+  ret void, !dbg !7
 }
-; CHECK: atomic32_xchg_seq_cst
-; CHECK: call i32 @__tsan_atomic32_exchange(i32* %a, i32 0, i32 5)
+; CHECK-LABEL: atomic32_xchg_seq_cst
+; CHECK: call i32 @__tsan_atomic32_exchange(i32* %a, i32 0, i32 5), !dbg
 
 define void @atomic32_add_seq_cst(i32* %a) nounwind uwtable {
 entry:
-  atomicrmw add i32* %a, i32 0 seq_cst
-  ret void
+  atomicrmw add i32* %a, i32 0 seq_cst, !dbg !7
+  ret void, !dbg !7
 }
-; CHECK: atomic32_add_seq_cst
-; CHECK: call i32 @__tsan_atomic32_fetch_add(i32* %a, i32 0, i32 5)
+; CHECK-LABEL: atomic32_add_seq_cst
+; CHECK: call i32 @__tsan_atomic32_fetch_add(i32* %a, i32 0, i32 5), !dbg
 
 define void @atomic32_sub_seq_cst(i32* %a) nounwind uwtable {
 entry:
-  atomicrmw sub i32* %a, i32 0 seq_cst
-  ret void
+  atomicrmw sub i32* %a, i32 0 seq_cst, !dbg !7
+  ret void, !dbg !7
 }
-; CHECK: atomic32_sub_seq_cst
-; CHECK: call i32 @__tsan_atomic32_fetch_sub(i32* %a, i32 0, i32 5)
+; CHECK-LABEL: atomic32_sub_seq_cst
+; CHECK: call i32 @__tsan_atomic32_fetch_sub(i32* %a, i32 0, i32 5), !dbg
 
 define void @atomic32_and_seq_cst(i32* %a) nounwind uwtable {
 entry:
-  atomicrmw and i32* %a, i32 0 seq_cst
-  ret void
+  atomicrmw and i32* %a, i32 0 seq_cst, !dbg !7
+  ret void, !dbg !7
 }
-; CHECK: atomic32_and_seq_cst
-; CHECK: call i32 @__tsan_atomic32_fetch_and(i32* %a, i32 0, i32 5)
+; CHECK-LABEL: atomic32_and_seq_cst
+; CHECK: call i32 @__tsan_atomic32_fetch_and(i32* %a, i32 0, i32 5), !dbg
 
 define void @atomic32_or_seq_cst(i32* %a) nounwind uwtable {
 entry:
-  atomicrmw or i32* %a, i32 0 seq_cst
-  ret void
+  atomicrmw or i32* %a, i32 0 seq_cst, !dbg !7
+  ret void, !dbg !7
 }
-; CHECK: atomic32_or_seq_cst
-; CHECK: call i32 @__tsan_atomic32_fetch_or(i32* %a, i32 0, i32 5)
+; CHECK-LABEL: atomic32_or_seq_cst
+; CHECK: call i32 @__tsan_atomic32_fetch_or(i32* %a, i32 0, i32 5), !dbg
 
 define void @atomic32_xor_seq_cst(i32* %a) nounwind uwtable {
 entry:
-  atomicrmw xor i32* %a, i32 0 seq_cst
-  ret void
+  atomicrmw xor i32* %a, i32 0 seq_cst, !dbg !7
+  ret void, !dbg !7
 }
-; CHECK: atomic32_xor_seq_cst
-; CHECK: call i32 @__tsan_atomic32_fetch_xor(i32* %a, i32 0, i32 5)
+; CHECK-LABEL: atomic32_xor_seq_cst
+; CHECK: call i32 @__tsan_atomic32_fetch_xor(i32* %a, i32 0, i32 5), !dbg
 
 define void @atomic32_nand_seq_cst(i32* %a) nounwind uwtable {
 entry:
-  atomicrmw nand i32* %a, i32 0 seq_cst
-  ret void
+  atomicrmw nand i32* %a, i32 0 seq_cst, !dbg !7
+  ret void, !dbg !7
 }
-; CHECK: atomic32_nand_seq_cst
-; CHECK: call i32 @__tsan_atomic32_fetch_nand(i32* %a, i32 0, i32 5)
+; CHECK-LABEL: atomic32_nand_seq_cst
+; CHECK: call i32 @__tsan_atomic32_fetch_nand(i32* %a, i32 0, i32 5), !dbg
 
 define void @atomic32_cas_monotonic(i32* %a) nounwind uwtable {
 entry:
-  cmpxchg i32* %a, i32 0, i32 1 monotonic monotonic
-  ret void
+  cmpxchg i32* %a, i32 0, i32 1 monotonic monotonic, !dbg !7
+  ret void, !dbg !7
 }
-; CHECK: atomic32_cas_monotonic
-; CHECK: call i32 @__tsan_atomic32_compare_exchange_val(i32* %a, i32 0, i32 1, i32 0, i32 0)
+; CHECK-LABEL: atomic32_cas_monotonic
+; CHECK: call i32 @__tsan_atomic32_compare_exchange_val(i32* %a, i32 0, i32 1, i32 0, i32 0), !dbg
 
 define void @atomic32_cas_acquire(i32* %a) nounwind uwtable {
 entry:
-  cmpxchg i32* %a, i32 0, i32 1 acquire acquire
-  ret void
+  cmpxchg i32* %a, i32 0, i32 1 acquire acquire, !dbg !7
+  ret void, !dbg !7
 }
-; CHECK: atomic32_cas_acquire
-; CHECK: call i32 @__tsan_atomic32_compare_exchange_val(i32* %a, i32 0, i32 1, i32 2, i32 2)
+; CHECK-LABEL: atomic32_cas_acquire
+; CHECK: call i32 @__tsan_atomic32_compare_exchange_val(i32* %a, i32 0, i32 1, i32 2, i32 2), !dbg
 
 define void @atomic32_cas_release(i32* %a) nounwind uwtable {
 entry:
-  cmpxchg i32* %a, i32 0, i32 1 release monotonic
-  ret void
+  cmpxchg i32* %a, i32 0, i32 1 release monotonic, !dbg !7
+  ret void, !dbg !7
 }
-; CHECK: atomic32_cas_release
-; CHECK: call i32 @__tsan_atomic32_compare_exchange_val(i32* %a, i32 0, i32 1, i32 3, i32 0)
+; CHECK-LABEL: atomic32_cas_release
+; CHECK: call i32 @__tsan_atomic32_compare_exchange_val(i32* %a, i32 0, i32 1, i32 3, i32 0), !dbg
 
 define void @atomic32_cas_acq_rel(i32* %a) nounwind uwtable {
 entry:
-  cmpxchg i32* %a, i32 0, i32 1 acq_rel acquire
-  ret void
+  cmpxchg i32* %a, i32 0, i32 1 acq_rel acquire, !dbg !7
+  ret void, !dbg !7
 }
-; CHECK: atomic32_cas_acq_rel
-; CHECK: call i32 @__tsan_atomic32_compare_exchange_val(i32* %a, i32 0, i32 1, i32 4, i32 2)
+; CHECK-LABEL: atomic32_cas_acq_rel
+; CHECK: call i32 @__tsan_atomic32_compare_exchange_val(i32* %a, i32 0, i32 1, i32 4, i32 2), !dbg
 
 define void @atomic32_cas_seq_cst(i32* %a) nounwind uwtable {
 entry:
-  cmpxchg i32* %a, i32 0, i32 1 seq_cst seq_cst
-  ret void
+  cmpxchg i32* %a, i32 0, i32 1 seq_cst seq_cst, !dbg !7
+  ret void, !dbg !7
 }
-; CHECK: atomic32_cas_seq_cst
-; CHECK: call i32 @__tsan_atomic32_compare_exchange_val(i32* %a, i32 0, i32 1, i32 5, i32 5)
+; CHECK-LABEL: atomic32_cas_seq_cst
+; CHECK: call i32 @__tsan_atomic32_compare_exchange_val(i32* %a, i32 0, i32 1, i32 5, i32 5), !dbg
 
 define i64 @atomic64_load_unordered(i64* %a) nounwind uwtable {
 entry:
-  %0 = load atomic i64, i64* %a unordered, align 8
-  ret i64 %0
+  %0 = load atomic i64, i64* %a unordered, align 8, !dbg !7
+  ret i64 %0, !dbg !7
 }
-; CHECK: atomic64_load_unordered
-; CHECK: call i64 @__tsan_atomic64_load(i64* %a, i32 0)
+; CHECK-LABEL: atomic64_load_unordered
+; CHECK: call i64 @__tsan_atomic64_load(i64* %a, i32 0), !dbg
 
 define i64 @atomic64_load_monotonic(i64* %a) nounwind uwtable {
 entry:
-  %0 = load atomic i64, i64* %a monotonic, align 8
-  ret i64 %0
+  %0 = load atomic i64, i64* %a monotonic, align 8, !dbg !7
+  ret i64 %0, !dbg !7
 }
-; CHECK: atomic64_load_monotonic
-; CHECK: call i64 @__tsan_atomic64_load(i64* %a, i32 0)
+; CHECK-LABEL: atomic64_load_monotonic
+; CHECK: call i64 @__tsan_atomic64_load(i64* %a, i32 0), !dbg
 
 define i64 @atomic64_load_acquire(i64* %a) nounwind uwtable {
 entry:
-  %0 = load atomic i64, i64* %a acquire, align 8
-  ret i64 %0
+  %0 = load atomic i64, i64* %a acquire, align 8, !dbg !7
+  ret i64 %0, !dbg !7
 }
-; CHECK: atomic64_load_acquire
-; CHECK: call i64 @__tsan_atomic64_load(i64* %a, i32 2)
+; CHECK-LABEL: atomic64_load_acquire
+; CHECK: call i64 @__tsan_atomic64_load(i64* %a, i32 2), !dbg
 
 define i64 @atomic64_load_seq_cst(i64* %a) nounwind uwtable {
 entry:
-  %0 = load atomic i64, i64* %a seq_cst, align 8
-  ret i64 %0
+  %0 = load atomic i64, i64* %a seq_cst, align 8, !dbg !7
+  ret i64 %0, !dbg !7
 }
-; CHECK: atomic64_load_seq_cst
-; CHECK: call i64 @__tsan_atomic64_load(i64* %a, i32 5)
+; CHECK-LABEL: atomic64_load_seq_cst
+; CHECK: call i64 @__tsan_atomic64_load(i64* %a, i32 5), !dbg
 
 define void @atomic64_store_unordered(i64* %a) nounwind uwtable {
 entry:
-  store atomic i64 0, i64* %a unordered, align 8
-  ret void
+  store atomic i64 0, i64* %a unordered, align 8, !dbg !7
+  ret void, !dbg !7
 }
-; CHECK: atomic64_store_unordered
-; CHECK: call void @__tsan_atomic64_store(i64* %a, i64 0, i32 0)
+; CHECK-LABEL: atomic64_store_unordered
+; CHECK: call void @__tsan_atomic64_store(i64* %a, i64 0, i32 0), !dbg
 
 define void @atomic64_store_monotonic(i64* %a) nounwind uwtable {
 entry:
-  store atomic i64 0, i64* %a monotonic, align 8
-  ret void
+  store atomic i64 0, i64* %a monotonic, align 8, !dbg !7
+  ret void, !dbg !7
 }
-; CHECK: atomic64_store_monotonic
-; CHECK: call void @__tsan_atomic64_store(i64* %a, i64 0, i32 0)
+; CHECK-LABEL: atomic64_store_monotonic
+; CHECK: call void @__tsan_atomic64_store(i64* %a, i64 0, i32 0), !dbg
 
 define void @atomic64_store_release(i64* %a) nounwind uwtable {
 entry:
-  store atomic i64 0, i64* %a release, align 8
-  ret void
+  store atomic i64 0, i64* %a release, align 8, !dbg !7
+  ret void, !dbg !7
 }
-; CHECK: atomic64_store_release
-; CHECK: call void @__tsan_atomic64_store(i64* %a, i64 0, i32 3)
+; CHECK-LABEL: atomic64_store_release
+; CHECK: call void @__tsan_atomic64_store(i64* %a, i64 0, i32 3), !dbg
 
 define void @atomic64_store_seq_cst(i64* %a) nounwind uwtable {
 entry:
-  store atomic i64 0, i64* %a seq_cst, align 8
-  ret void
+  store atomic i64 0, i64* %a seq_cst, align 8, !dbg !7
+  ret void, !dbg !7
 }
-; CHECK: atomic64_store_seq_cst
-; CHECK: call void @__tsan_atomic64_store(i64* %a, i64 0, i32 5)
+; CHECK-LABEL: atomic64_store_seq_cst
+; CHECK: call void @__tsan_atomic64_store(i64* %a, i64 0, i32 5), !dbg
 
 define void @atomic64_xchg_monotonic(i64* %a) nounwind uwtable {
 entry:
-  atomicrmw xchg i64* %a, i64 0 monotonic
-  ret void
+  atomicrmw xchg i64* %a, i64 0 monotonic, !dbg !7
+  ret void, !dbg !7
 }
-; CHECK: atomic64_xchg_monotonic
-; CHECK: call i64 @__tsan_atomic64_exchange(i64* %a, i64 0, i32 0)
+; CHECK-LABEL: atomic64_xchg_monotonic
+; CHECK: call i64 @__tsan_atomic64_exchange(i64* %a, i64 0, i32 0), !dbg
 
 define void @atomic64_add_monotonic(i64* %a) nounwind uwtable {
 entry:
-  atomicrmw add i64* %a, i64 0 monotonic
-  ret void
+  atomicrmw add i64* %a, i64 0 monotonic, !dbg !7
+  ret void, !dbg !7
 }
-; CHECK: atomic64_add_monotonic
-; CHECK: call i64 @__tsan_atomic64_fetch_add(i64* %a, i64 0, i32 0)
+; CHECK-LABEL: atomic64_add_monotonic
+; CHECK: call i64 @__tsan_atomic64_fetch_add(i64* %a, i64 0, i32 0), !dbg
 
 define void @atomic64_sub_monotonic(i64* %a) nounwind uwtable {
 entry:
-  atomicrmw sub i64* %a, i64 0 monotonic
-  ret void
+  atomicrmw sub i64* %a, i64 0 monotonic, !dbg !7
+  ret void, !dbg !7
 }
-; CHECK: atomic64_sub_monotonic
-; CHECK: call i64 @__tsan_atomic64_fetch_sub(i64* %a, i64 0, i32 0)
+; CHECK-LABEL: atomic64_sub_monotonic
+; CHECK: call i64 @__tsan_atomic64_fetch_sub(i64* %a, i64 0, i32 0), !dbg
 
 define void @atomic64_and_monotonic(i64* %a) nounwind uwtable {
 entry:
-  atomicrmw and i64* %a, i64 0 monotonic
-  ret void
+  atomicrmw and i64* %a, i64 0 monotonic, !dbg !7
+  ret void, !dbg !7
 }
-; CHECK: atomic64_and_monotonic
-; CHECK: call i64 @__tsan_atomic64_fetch_and(i64* %a, i64 0, i32 0)
+; CHECK-LABEL: atomic64_and_monotonic
+; CHECK: call i64 @__tsan_atomic64_fetch_and(i64* %a, i64 0, i32 0), !dbg
 
 define void @atomic64_or_monotonic(i64* %a) nounwind uwtable {
 entry:
-  atomicrmw or i64* %a, i64 0 monotonic
-  ret void
+  atomicrmw or i64* %a, i64 0 monotonic, !dbg !7
+  ret void, !dbg !7
 }
-; CHECK: atomic64_or_monotonic
-; CHECK: call i64 @__tsan_atomic64_fetch_or(i64* %a, i64 0, i32 0)
+; CHECK-LABEL: atomic64_or_monotonic
+; CHECK: call i64 @__tsan_atomic64_fetch_or(i64* %a, i64 0, i32 0), !dbg
 
 define void @atomic64_xor_monotonic(i64* %a) nounwind uwtable {
 entry:
-  atomicrmw xor i64* %a, i64 0 monotonic
-  ret void
+  atomicrmw xor i64* %a, i64 0 monotonic, !dbg !7
+  ret void, !dbg !7
 }
-; CHECK: atomic64_xor_monotonic
-; CHECK: call i64 @__tsan_atomic64_fetch_xor(i64* %a, i64 0, i32 0)
+; CHECK-LABEL: atomic64_xor_monotonic
+; CHECK: call i64 @__tsan_atomic64_fetch_xor(i64* %a, i64 0, i32 0), !dbg
 
 define void @atomic64_nand_monotonic(i64* %a) nounwind uwtable {
 entry:
-  atomicrmw nand i64* %a, i64 0 monotonic
-  ret void
+  atomicrmw nand i64* %a, i64 0 monotonic, !dbg !7
+  ret void, !dbg !7
 }
-; CHECK: atomic64_nand_monotonic
-; CHECK: call i64 @__tsan_atomic64_fetch_nand(i64* %a, i64 0, i32 0)
+; CHECK-LABEL: atomic64_nand_monotonic
+; CHECK: call i64 @__tsan_atomic64_fetch_nand(i64* %a, i64 0, i32 0), !dbg
 
 define void @atomic64_xchg_acquire(i64* %a) nounwind uwtable {
 entry:
-  atomicrmw xchg i64* %a, i64 0 acquire
-  ret void
+  atomicrmw xchg i64* %a, i64 0 acquire, !dbg !7
+  ret void, !dbg !7
 }
-; CHECK: atomic64_xchg_acquire
-; CHECK: call i64 @__tsan_atomic64_exchange(i64* %a, i64 0, i32 2)
+; CHECK-LABEL: atomic64_xchg_acquire
+; CHECK: call i64 @__tsan_atomic64_exchange(i64* %a, i64 0, i32 2), !dbg
 
 define void @atomic64_add_acquire(i64* %a) nounwind uwtable {
 entry:
-  atomicrmw add i64* %a, i64 0 acquire
-  ret void
+  atomicrmw add i64* %a, i64 0 acquire, !dbg !7
+  ret void, !dbg !7
 }
-; CHECK: atomic64_add_acquire
-; CHECK: call i64 @__tsan_atomic64_fetch_add(i64* %a, i64 0, i32 2)
+; CHECK-LABEL: atomic64_add_acquire
+; CHECK: call i64 @__tsan_atomic64_fetch_add(i64* %a, i64 0, i32 2), !dbg
 
 define void @atomic64_sub_acquire(i64* %a) nounwind uwtable {
 entry:
-  atomicrmw sub i64* %a, i64 0 acquire
-  ret void
+  atomicrmw sub i64* %a, i64 0 acquire, !dbg !7
+  ret void, !dbg !7
 }
-; CHECK: atomic64_sub_acquire
-; CHECK: call i64 @__tsan_atomic64_fetch_sub(i64* %a, i64 0, i32 2)
+; CHECK-LABEL: atomic64_sub_acquire
+; CHECK: call i64 @__tsan_atomic64_fetch_sub(i64* %a, i64 0, i32 2), !dbg
 
 define void @atomic64_and_acquire(i64* %a) nounwind uwtable {
 entry:
-  atomicrmw and i64* %a, i64 0 acquire
-  ret void
+  atomicrmw and i64* %a, i64 0 acquire, !dbg !7
+  ret void, !dbg !7
 }
-; CHECK: atomic64_and_acquire
-; CHECK: call i64 @__tsan_atomic64_fetch_and(i64* %a, i64 0, i32 2)
+; CHECK-LABEL: atomic64_and_acquire
+; CHECK: call i64 @__tsan_atomic64_fetch_and(i64* %a, i64 0, i32 2), !dbg
 
 define void @atomic64_or_acquire(i64* %a) nounwind uwtable {
 entry:
-  atomicrmw or i64* %a, i64 0 acquire
-  ret void
+  atomicrmw or i64* %a, i64 0 acquire, !dbg !7
+  ret void, !dbg !7
 }
-; CHECK: atomic64_or_acquire
-; CHECK: call i64 @__tsan_atomic64_fetch_or(i64* %a, i64 0, i32 2)
+; CHECK-LABEL: atomic64_or_acquire
+; CHECK: call i64 @__tsan_atomic64_fetch_or(i64* %a, i64 0, i32 2), !dbg
 
 define void @atomic64_xor_acquire(i64* %a) nounwind uwtable {
 entry:
-  atomicrmw xor i64* %a, i64 0 acquire
-  ret void
+  atomicrmw xor i64* %a, i64 0 acquire, !dbg !7
+  ret void, !dbg !7
 }
-; CHECK: atomic64_xor_acquire
-; CHECK: call i64 @__tsan_atomic64_fetch_xor(i64* %a, i64 0, i32 2)
+; CHECK-LABEL: atomic64_xor_acquire
+; CHECK: call i64 @__tsan_atomic64_fetch_xor(i64* %a, i64 0, i32 2), !dbg
 
 define void @atomic64_nand_acquire(i64* %a) nounwind uwtable {
 entry:
-  atomicrmw nand i64* %a, i64 0 acquire
-  ret void
+  atomicrmw nand i64* %a, i64 0 acquire, !dbg !7
+  ret void, !dbg !7
 }
-; CHECK: atomic64_nand_acquire
-; CHECK: call i64 @__tsan_atomic64_fetch_nand(i64* %a, i64 0, i32 2)
+; CHECK-LABEL: atomic64_nand_acquire
+; CHECK: call i64 @__tsan_atomic64_fetch_nand(i64* %a, i64 0, i32 2), !dbg
 
 define void @atomic64_xchg_release(i64* %a) nounwind uwtable {
 entry:
-  atomicrmw xchg i64* %a, i64 0 release
-  ret void
+  atomicrmw xchg i64* %a, i64 0 release, !dbg !7
+  ret void, !dbg !7
 }
-; CHECK: atomic64_xchg_release
-; CHECK: call i64 @__tsan_atomic64_exchange(i64* %a, i64 0, i32 3)
+; CHECK-LABEL: atomic64_xchg_release
+; CHECK: call i64 @__tsan_atomic64_exchange(i64* %a, i64 0, i32 3), !dbg
 
 define void @atomic64_add_release(i64* %a) nounwind uwtable {
 entry:
-  atomicrmw add i64* %a, i64 0 release
-  ret void
+  atomicrmw add i64* %a, i64 0 release, !dbg !7
+  ret void, !dbg !7
 }
-; CHECK: atomic64_add_release
-; CHECK: call i64 @__tsan_atomic64_fetch_add(i64* %a, i64 0, i32 3)
+; CHECK-LABEL: atomic64_add_release
+; CHECK: call i64 @__tsan_atomic64_fetch_add(i64* %a, i64 0, i32 3), !dbg
 
 define void @atomic64_sub_release(i64* %a) nounwind uwtable {
 entry:
-  atomicrmw sub i64* %a, i64 0 release
-  ret void
+  atomicrmw sub i64* %a, i64 0 release, !dbg !7
+  ret void, !dbg !7
 }
-; CHECK: atomic64_sub_release
-; CHECK: call i64 @__tsan_atomic64_fetch_sub(i64* %a, i64 0, i32 3)
+; CHECK-LABEL: atomic64_sub_release
+; CHECK: call i64 @__tsan_atomic64_fetch_sub(i64* %a, i64 0, i32 3), !dbg
 
 define void @atomic64_and_release(i64* %a) nounwind uwtable {
 entry:
-  atomicrmw and i64* %a, i64 0 release
-  ret void
+  atomicrmw and i64* %a, i64 0 release, !dbg !7
+  ret void, !dbg !7
 }
-; CHECK: atomic64_and_release
-; CHECK: call i64 @__tsan_atomic64_fetch_and(i64* %a, i64 0, i32 3)
+; CHECK-LABEL: atomic64_and_release
+; CHECK: call i64 @__tsan_atomic64_fetch_and(i64* %a, i64 0, i32 3), !dbg
 
 define void @atomic64_or_release(i64* %a) nounwind uwtable {
 entry:
-  atomicrmw or i64* %a, i64 0 release
-  ret void
+  atomicrmw or i64* %a, i64 0 release, !dbg !7
+  ret void, !dbg !7
 }
-; CHECK: atomic64_or_release
-; CHECK: call i64 @__tsan_atomic64_fetch_or(i64* %a, i64 0, i32 3)
+; CHECK-LABEL: atomic64_or_release
+; CHECK: call i64 @__tsan_atomic64_fetch_or(i64* %a, i64 0, i32 3), !dbg
 
 define void @atomic64_xor_release(i64* %a) nounwind uwtable {
 entry:
-  atomicrmw xor i64* %a, i64 0 release
-  ret void
+  atomicrmw xor i64* %a, i64 0 release, !dbg !7
+  ret void, !dbg !7
 }
-; CHECK: atomic64_xor_release
-; CHECK: call i64 @__tsan_atomic64_fetch_xor(i64* %a, i64 0, i32 3)
+; CHECK-LABEL: atomic64_xor_release
+; CHECK: call i64 @__tsan_atomic64_fetch_xor(i64* %a, i64 0, i32 3), !dbg
 
 define void @atomic64_nand_release(i64* %a) nounwind uwtable {
 entry:
-  atomicrmw nand i64* %a, i64 0 release
-  ret void
+  atomicrmw nand i64* %a, i64 0 release, !dbg !7
+  ret void, !dbg !7
 }
-; CHECK: atomic64_nand_release
-; CHECK: call i64 @__tsan_atomic64_fetch_nand(i64* %a, i64 0, i32 3)
+; CHECK-LABEL: atomic64_nand_release
+; CHECK: call i64 @__tsan_atomic64_fetch_nand(i64* %a, i64 0, i32 3), !dbg
 
 define void @atomic64_xchg_acq_rel(i64* %a) nounwind uwtable {
 entry:
-  atomicrmw xchg i64* %a, i64 0 acq_rel
-  ret void
+  atomicrmw xchg i64* %a, i64 0 acq_rel, !dbg !7
+  ret void, !dbg !7
 }
-; CHECK: atomic64_xchg_acq_rel
-; CHECK: call i64 @__tsan_atomic64_exchange(i64* %a, i64 0, i32 4)
+; CHECK-LABEL: atomic64_xchg_acq_rel
+; CHECK: call i64 @__tsan_atomic64_exchange(i64* %a, i64 0, i32 4), !dbg
 
 define void @atomic64_add_acq_rel(i64* %a) nounwind uwtable {
 entry:
-  atomicrmw add i64* %a, i64 0 acq_rel
-  ret void
+  atomicrmw add i64* %a, i64 0 acq_rel, !dbg !7
+  ret void, !dbg !7
 }
-; CHECK: atomic64_add_acq_rel
-; CHECK: call i64 @__tsan_atomic64_fetch_add(i64* %a, i64 0, i32 4)
+; CHECK-LABEL: atomic64_add_acq_rel
+; CHECK: call i64 @__tsan_atomic64_fetch_add(i64* %a, i64 0, i32 4), !dbg
 
 define void @atomic64_sub_acq_rel(i64* %a) nounwind uwtable {
 entry:
-  atomicrmw sub i64* %a, i64 0 acq_rel
-  ret void
+  atomicrmw sub i64* %a, i64 0 acq_rel, !dbg !7
+  ret void, !dbg !7
 }
-; CHECK: atomic64_sub_acq_rel
-; CHECK: call i64 @__tsan_atomic64_fetch_sub(i64* %a, i64 0, i32 4)
+; CHECK-LABEL: atomic64_sub_acq_rel
+; CHECK: call i64 @__tsan_atomic64_fetch_sub(i64* %a, i64 0, i32 4), !dbg
 
 define void @atomic64_and_acq_rel(i64* %a) nounwind uwtable {
 entry:
-  atomicrmw and i64* %a, i64 0 acq_rel
-  ret void
+  atomicrmw and i64* %a, i64 0 acq_rel, !dbg !7
+  ret void, !dbg !7
 }
-; CHECK: atomic64_and_acq_rel
-; CHECK: call i64 @__tsan_atomic64_fetch_and(i64* %a, i64 0, i32 4)
+; CHECK-LABEL: atomic64_and_acq_rel
+; CHECK: call i64 @__tsan_atomic64_fetch_and(i64* %a, i64 0, i32 4), !dbg
 
 define void @atomic64_or_acq_rel(i64* %a) nounwind uwtable {
 entry:
-  atomicrmw or i64* %a, i64 0 acq_rel
-  ret void
+  atomicrmw or i64* %a, i64 0 acq_rel, !dbg !7
+  ret void, !dbg !7
 }
-; CHECK: atomic64_or_acq_rel
-; CHECK: call i64 @__tsan_atomic64_fetch_or(i64* %a, i64 0, i32 4)
+; CHECK-LABEL: atomic64_or_acq_rel
+; CHECK: call i64 @__tsan_atomic64_fetch_or(i64* %a, i64 0, i32 4), !dbg
 
 define void @atomic64_xor_acq_rel(i64* %a) nounwind uwtable {
 entry:
-  atomicrmw xor i64* %a, i64 0 acq_rel
-  ret void
+  atomicrmw xor i64* %a, i64 0 acq_rel, !dbg !7
+  ret void, !dbg !7
 }
-; CHECK: atomic64_xor_acq_rel
-; CHECK: call i64 @__tsan_atomic64_fetch_xor(i64* %a, i64 0, i32 4)
+; CHECK-LABEL: atomic64_xor_acq_rel
+; CHECK: call i64 @__tsan_atomic64_fetch_xor(i64* %a, i64 0, i32 4), !dbg
 
 define void @atomic64_nand_acq_rel(i64* %a) nounwind uwtable {
 entry:
-  atomicrmw nand i64* %a, i64 0 acq_rel
-  ret void
+  atomicrmw nand i64* %a, i64 0 acq_rel, !dbg !7
+  ret void, !dbg !7
 }
-; CHECK: atomic64_nand_acq_rel
-; CHECK: call i64 @__tsan_atomic64_fetch_nand(i64* %a, i64 0, i32 4)
+; CHECK-LABEL: atomic64_nand_acq_rel
+; CHECK: call i64 @__tsan_atomic64_fetch_nand(i64* %a, i64 0, i32 4), !dbg
 
 define void @atomic64_xchg_seq_cst(i64* %a) nounwind uwtable {
 entry:
-  atomicrmw xchg i64* %a, i64 0 seq_cst
-  ret void
+  atomicrmw xchg i64* %a, i64 0 seq_cst, !dbg !7
+  ret void, !dbg !7
 }
-; CHECK: atomic64_xchg_seq_cst
-; CHECK: call i64 @__tsan_atomic64_exchange(i64* %a, i64 0, i32 5)
+; CHECK-LABEL: atomic64_xchg_seq_cst
+; CHECK: call i64 @__tsan_atomic64_exchange(i64* %a, i64 0, i32 5), !dbg
 
 define void @atomic64_add_seq_cst(i64* %a) nounwind uwtable {
 entry:
-  atomicrmw add i64* %a, i64 0 seq_cst
-  ret void
+  atomicrmw add i64* %a, i64 0 seq_cst, !dbg !7
+  ret void, !dbg !7
 }
-; CHECK: atomic64_add_seq_cst
-; CHECK: call i64 @__tsan_atomic64_fetch_add(i64* %a, i64 0, i32 5)
+; CHECK-LABEL: atomic64_add_seq_cst
+; CHECK: call i64 @__tsan_atomic64_fetch_add(i64* %a, i64 0, i32 5), !dbg
 
 define void @atomic64_sub_seq_cst(i64* %a) nounwind uwtable {
 entry:
-  atomicrmw sub i64* %a, i64 0 seq_cst
-  ret void
+  atomicrmw sub i64* %a, i64 0 seq_cst, !dbg !7
+  ret void, !dbg !7
 }
-; CHECK: atomic64_sub_seq_cst
-; CHECK: call i64 @__tsan_atomic64_fetch_sub(i64* %a, i64 0, i32 5)
+; CHECK-LABEL: atomic64_sub_seq_cst
+; CHECK: call i64 @__tsan_atomic64_fetch_sub(i64* %a, i64 0, i32 5), !dbg
 
 define void @atomic64_and_seq_cst(i64* %a) nounwind uwtable {
 entry:
-  atomicrmw and i64* %a, i64 0 seq_cst
-  ret void
+  atomicrmw and i64* %a, i64 0 seq_cst, !dbg !7
+  ret void, !dbg !7
 }
-; CHECK: atomic64_and_seq_cst
-; CHECK: call i64 @__tsan_atomic64_fetch_and(i64* %a, i64 0, i32 5)
+; CHECK-LABEL: atomic64_and_seq_cst
+; CHECK: call i64 @__tsan_atomic64_fetch_and(i64* %a, i64 0, i32 5), !dbg
 
 define void @atomic64_or_seq_cst(i64* %a) nounwind uwtable {
 entry:
-  atomicrmw or i64* %a, i64 0 seq_cst
-  ret void
+  atomicrmw or i64* %a, i64 0 seq_cst, !dbg !7
+  ret void, !dbg !7
 }
-; CHECK: atomic64_or_seq_cst
-; CHECK: call i64 @__tsan_atomic64_fetch_or(i64* %a, i64 0, i32 5)
+; CHECK-LABEL: atomic64_or_seq_cst
+; CHECK: call i64 @__tsan_atomic64_fetch_or(i64* %a, i64 0, i32 5), !dbg
 
 define void @atomic64_xor_seq_cst(i64* %a) nounwind uwtable {
 entry:
-  atomicrmw xor i64* %a, i64 0 seq_cst
-  ret void
+  atomicrmw xor i64* %a, i64 0 seq_cst, !dbg !7
+  ret void, !dbg !7
 }
-; CHECK: atomic64_xor_seq_cst
-; CHECK: call i64 @__tsan_atomic64_fetch_xor(i64* %a, i64 0, i32 5)
+; CHECK-LABEL: atomic64_xor_seq_cst
+; CHECK: call i64 @__tsan_atomic64_fetch_xor(i64* %a, i64 0, i32 5), !dbg
 
 define void @atomic64_nand_seq_cst(i64* %a) nounwind uwtable {
 entry:
-  atomicrmw nand i64* %a, i64 0 seq_cst
-  ret void
+  atomicrmw nand i64* %a, i64 0 seq_cst, !dbg !7
+  ret void, !dbg !7
 }
-; CHECK: atomic64_nand_seq_cst
-; CHECK: call i64 @__tsan_atomic64_fetch_nand(i64* %a, i64 0, i32 5)
+; CHECK-LABEL: atomic64_nand_seq_cst
+; CHECK: call i64 @__tsan_atomic64_fetch_nand(i64* %a, i64 0, i32 5), !dbg
 
 define void @atomic64_cas_monotonic(i64* %a) nounwind uwtable {
 entry:
-  cmpxchg i64* %a, i64 0, i64 1 monotonic monotonic
-  ret void
+  cmpxchg i64* %a, i64 0, i64 1 monotonic monotonic, !dbg !7
+  ret void, !dbg !7
 }
-; CHECK: atomic64_cas_monotonic
-; CHECK: call i64 @__tsan_atomic64_compare_exchange_val(i64* %a, i64 0, i64 1, i32 0, i32 0)
+; CHECK-LABEL: atomic64_cas_monotonic
+; CHECK: call i64 @__tsan_atomic64_compare_exchange_val(i64* %a, i64 0, i64 1, i32 0, i32 0), !dbg
 
 define void @atomic64_cas_acquire(i64* %a) nounwind uwtable {
 entry:
-  cmpxchg i64* %a, i64 0, i64 1 acquire acquire
-  ret void
+  cmpxchg i64* %a, i64 0, i64 1 acquire acquire, !dbg !7
+  ret void, !dbg !7
 }
-; CHECK: atomic64_cas_acquire
-; CHECK: call i64 @__tsan_atomic64_compare_exchange_val(i64* %a, i64 0, i64 1, i32 2, i32 2)
+; CHECK-LABEL: atomic64_cas_acquire
+; CHECK: call i64 @__tsan_atomic64_compare_exchange_val(i64* %a, i64 0, i64 1, i32 2, i32 2), !dbg
 
 define void @atomic64_cas_release(i64* %a) nounwind uwtable {
 entry:
-  cmpxchg i64* %a, i64 0, i64 1 release monotonic
-  ret void
+  cmpxchg i64* %a, i64 0, i64 1 release monotonic, !dbg !7
+  ret void, !dbg !7
 }
-; CHECK: atomic64_cas_release
-; CHECK: call i64 @__tsan_atomic64_compare_exchange_val(i64* %a, i64 0, i64 1, i32 3, i32 0)
+; CHECK-LABEL: atomic64_cas_release
+; CHECK: call i64 @__tsan_atomic64_compare_exchange_val(i64* %a, i64 0, i64 1, i32 3, i32 0), !dbg
 
 define void @atomic64_cas_acq_rel(i64* %a) nounwind uwtable {
 entry:
-  cmpxchg i64* %a, i64 0, i64 1 acq_rel acquire
-  ret void
+  cmpxchg i64* %a, i64 0, i64 1 acq_rel acquire, !dbg !7
+  ret void, !dbg !7
 }
-; CHECK: atomic64_cas_acq_rel
-; CHECK: call i64 @__tsan_atomic64_compare_exchange_val(i64* %a, i64 0, i64 1, i32 4, i32 2)
+; CHECK-LABEL: atomic64_cas_acq_rel
+; CHECK: call i64 @__tsan_atomic64_compare_exchange_val(i64* %a, i64 0, i64 1, i32 4, i32 2), !dbg
 
 define void @atomic64_cas_seq_cst(i64* %a) nounwind uwtable {
 entry:
-  cmpxchg i64* %a, i64 0, i64 1 seq_cst seq_cst
-  ret void
+  cmpxchg i64* %a, i64 0, i64 1 seq_cst seq_cst, !dbg !7
+  ret void, !dbg !7
 }
-; CHECK: atomic64_cas_seq_cst
-; CHECK: call i64 @__tsan_atomic64_compare_exchange_val(i64* %a, i64 0, i64 1, i32 5, i32 5)
+; CHECK-LABEL: atomic64_cas_seq_cst
+; CHECK: call i64 @__tsan_atomic64_compare_exchange_val(i64* %a, i64 0, i64 1, i32 5, i32 5), !dbg
 
 define i128 @atomic128_load_unordered(i128* %a) nounwind uwtable {
 entry:
-  %0 = load atomic i128, i128* %a unordered, align 16
-  ret i128 %0
+  %0 = load atomic i128, i128* %a unordered, align 16, !dbg !7
+  ret i128 %0, !dbg !7
 }
-; CHECK: atomic128_load_unordered
-; CHECK: call i128 @__tsan_atomic128_load(i128* %a, i32 0)
+; CHECK-LABEL: atomic128_load_unordered
+; CHECK: call i128 @__tsan_atomic128_load(i128* %a, i32 0), !dbg
 
 define i128 @atomic128_load_monotonic(i128* %a) nounwind uwtable {
 entry:
-  %0 = load atomic i128, i128* %a monotonic, align 16
-  ret i128 %0
+  %0 = load atomic i128, i128* %a monotonic, align 16, !dbg !7
+  ret i128 %0, !dbg !7
 }
-; CHECK: atomic128_load_monotonic
-; CHECK: call i128 @__tsan_atomic128_load(i128* %a, i32 0)
+; CHECK-LABEL: atomic128_load_monotonic
+; CHECK: call i128 @__tsan_atomic128_load(i128* %a, i32 0), !dbg
 
 define i128 @atomic128_load_acquire(i128* %a) nounwind uwtable {
 entry:
-  %0 = load atomic i128, i128* %a acquire, align 16
-  ret i128 %0
+  %0 = load atomic i128, i128* %a acquire, align 16, !dbg !7
+  ret i128 %0, !dbg !7
 }
-; CHECK: atomic128_load_acquire
-; CHECK: call i128 @__tsan_atomic128_load(i128* %a, i32 2)
+; CHECK-LABEL: atomic128_load_acquire
+; CHECK: call i128 @__tsan_atomic128_load(i128* %a, i32 2), !dbg
 
 define i128 @atomic128_load_seq_cst(i128* %a) nounwind uwtable {
 entry:
-  %0 = load atomic i128, i128* %a seq_cst, align 16
-  ret i128 %0
+  %0 = load atomic i128, i128* %a seq_cst, align 16, !dbg !7
+  ret i128 %0, !dbg !7
 }
-; CHECK: atomic128_load_seq_cst
-; CHECK: call i128 @__tsan_atomic128_load(i128* %a, i32 5)
+; CHECK-LABEL: atomic128_load_seq_cst
+; CHECK: call i128 @__tsan_atomic128_load(i128* %a, i32 5), !dbg
 
 define void @atomic128_store_unordered(i128* %a) nounwind uwtable {
 entry:
-  store atomic i128 0, i128* %a unordered, align 16
-  ret void
+  store atomic i128 0, i128* %a unordered, align 16, !dbg !7
+  ret void, !dbg !7
 }
-; CHECK: atomic128_store_unordered
-; CHECK: call void @__tsan_atomic128_store(i128* %a, i128 0, i32 0)
+; CHECK-LABEL: atomic128_store_unordered
+; CHECK: call void @__tsan_atomic128_store(i128* %a, i128 0, i32 0), !dbg
 
 define void @atomic128_store_monotonic(i128* %a) nounwind uwtable {
 entry:
-  store atomic i128 0, i128* %a monotonic, align 16
-  ret void
+  store atomic i128 0, i128* %a monotonic, align 16, !dbg !7
+  ret void, !dbg !7
 }
-; CHECK: atomic128_store_monotonic
-; CHECK: call void @__tsan_atomic128_store(i128* %a, i128 0, i32 0)
+; CHECK-LABEL: atomic128_store_monotonic
+; CHECK: call void @__tsan_atomic128_store(i128* %a, i128 0, i32 0), !dbg
 
 define void @atomic128_store_release(i128* %a) nounwind uwtable {
 entry:
-  store atomic i128 0, i128* %a release, align 16
-  ret void
+  store atomic i128 0, i128* %a release, align 16, !dbg !7
+  ret void, !dbg !7
 }
-; CHECK: atomic128_store_release
-; CHECK: call void @__tsan_atomic128_store(i128* %a, i128 0, i32 3)
+; CHECK-LABEL: atomic128_store_release
+; CHECK: call void @__tsan_atomic128_store(i128* %a, i128 0, i32 3), !dbg
 
 define void @atomic128_store_seq_cst(i128* %a) nounwind uwtable {
 entry:
-  store atomic i128 0, i128* %a seq_cst, align 16
-  ret void
+  store atomic i128 0, i128* %a seq_cst, align 16, !dbg !7
+  ret void, !dbg !7
 }
-; CHECK: atomic128_store_seq_cst
-; CHECK: call void @__tsan_atomic128_store(i128* %a, i128 0, i32 5)
+; CHECK-LABEL: atomic128_store_seq_cst
+; CHECK: call void @__tsan_atomic128_store(i128* %a, i128 0, i32 5), !dbg
 
 define void @atomic128_xchg_monotonic(i128* %a) nounwind uwtable {
 entry:
-  atomicrmw xchg i128* %a, i128 0 monotonic
-  ret void
+  atomicrmw xchg i128* %a, i128 0 monotonic, !dbg !7
+  ret void, !dbg !7
 }
-; CHECK: atomic128_xchg_monotonic
-; CHECK: call i128 @__tsan_atomic128_exchange(i128* %a, i128 0, i32 0)
+; CHECK-LABEL: atomic128_xchg_monotonic
+; CHECK: call i128 @__tsan_atomic128_exchange(i128* %a, i128 0, i32 0), !dbg
 
 define void @atomic128_add_monotonic(i128* %a) nounwind uwtable {
 entry:
-  atomicrmw add i128* %a, i128 0 monotonic
-  ret void
+  atomicrmw add i128* %a, i128 0 monotonic, !dbg !7
+  ret void, !dbg !7
 }
-; CHECK: atomic128_add_monotonic
-; CHECK: call i128 @__tsan_atomic128_fetch_add(i128* %a, i128 0, i32 0)
+; CHECK-LABEL: atomic128_add_monotonic
+; CHECK: call i128 @__tsan_atomic128_fetch_add(i128* %a, i128 0, i32 0), !dbg
 
 define void @atomic128_sub_monotonic(i128* %a) nounwind uwtable {
 entry:
-  atomicrmw sub i128* %a, i128 0 monotonic
-  ret void
+  atomicrmw sub i128* %a, i128 0 monotonic, !dbg !7
+  ret void, !dbg !7
 }
-; CHECK: atomic128_sub_monotonic
-; CHECK: call i128 @__tsan_atomic128_fetch_sub(i128* %a, i128 0, i32 0)
+; CHECK-LABEL: atomic128_sub_monotonic
+; CHECK: call i128 @__tsan_atomic128_fetch_sub(i128* %a, i128 0, i32 0), !dbg
 
 define void @atomic128_and_monotonic(i128* %a) nounwind uwtable {
 entry:
-  atomicrmw and i128* %a, i128 0 monotonic
-  ret void
+  atomicrmw and i128* %a, i128 0 monotonic, !dbg !7
+  ret void, !dbg !7
 }
-; CHECK: atomic128_and_monotonic
-; CHECK: call i128 @__tsan_atomic128_fetch_and(i128* %a, i128 0, i32 0)
+; CHECK-LABEL: atomic128_and_monotonic
+; CHECK: call i128 @__tsan_atomic128_fetch_and(i128* %a, i128 0, i32 0), !dbg
 
 define void @atomic128_or_monotonic(i128* %a) nounwind uwtable {
 entry:
-  atomicrmw or i128* %a, i128 0 monotonic
-  ret void
+  atomicrmw or i128* %a, i128 0 monotonic, !dbg !7
+  ret void, !dbg !7
 }
-; CHECK: atomic128_or_monotonic
-; CHECK: call i128 @__tsan_atomic128_fetch_or(i128* %a, i128 0, i32 0)
+; CHECK-LABEL: atomic128_or_monotonic
+; CHECK: call i128 @__tsan_atomic128_fetch_or(i128* %a, i128 0, i32 0), !dbg
 
 define void @atomic128_xor_monotonic(i128* %a) nounwind uwtable {
 entry:
-  atomicrmw xor i128* %a, i128 0 monotonic
-  ret void
+  atomicrmw xor i128* %a, i128 0 monotonic, !dbg !7
+  ret void, !dbg !7
 }
-; CHECK: atomic128_xor_monotonic
-; CHECK: call i128 @__tsan_atomic128_fetch_xor(i128* %a, i128 0, i32 0)
+; CHECK-LABEL: atomic128_xor_monotonic
+; CHECK: call i128 @__tsan_atomic128_fetch_xor(i128* %a, i128 0, i32 0), !dbg
 
 define void @atomic128_nand_monotonic(i128* %a) nounwind uwtable {
 entry:
-  atomicrmw nand i128* %a, i128 0 monotonic
-  ret void
+  atomicrmw nand i128* %a, i128 0 monotonic, !dbg !7
+  ret void, !dbg !7
 }
-; CHECK: atomic128_nand_monotonic
-; CHECK: call i128 @__tsan_atomic128_fetch_nand(i128* %a, i128 0, i32 0)
+; CHECK-LABEL: atomic128_nand_monotonic
+; CHECK: call i128 @__tsan_atomic128_fetch_nand(i128* %a, i128 0, i32 0), !dbg
 
 define void @atomic128_xchg_acquire(i128* %a) nounwind uwtable {
 entry:
-  atomicrmw xchg i128* %a, i128 0 acquire
-  ret void
+  atomicrmw xchg i128* %a, i128 0 acquire, !dbg !7
+  ret void, !dbg !7
 }
-; CHECK: atomic128_xchg_acquire
-; CHECK: call i128 @__tsan_atomic128_exchange(i128* %a, i128 0, i32 2)
+; CHECK-LABEL: atomic128_xchg_acquire
+; CHECK: call i128 @__tsan_atomic128_exchange(i128* %a, i128 0, i32 2), !dbg
 
 define void @atomic128_add_acquire(i128* %a) nounwind uwtable {
 entry:
-  atomicrmw add i128* %a, i128 0 acquire
-  ret void
+  atomicrmw add i128* %a, i128 0 acquire, !dbg !7
+  ret void, !dbg !7
 }
-; CHECK: atomic128_add_acquire
-; CHECK: call i128 @__tsan_atomic128_fetch_add(i128* %a, i128 0, i32 2)
+; CHECK-LABEL: atomic128_add_acquire
+; CHECK: call i128 @__tsan_atomic128_fetch_add(i128* %a, i128 0, i32 2), !dbg
 
 define void @atomic128_sub_acquire(i128* %a) nounwind uwtable {
 entry:
-  atomicrmw sub i128* %a, i128 0 acquire
-  ret void
+  atomicrmw sub i128* %a, i128 0 acquire, !dbg !7
+  ret void, !dbg !7
 }
-; CHECK: atomic128_sub_acquire
-; CHECK: call i128 @__tsan_atomic128_fetch_sub(i128* %a, i128 0, i32 2)
+; CHECK-LABEL: atomic128_sub_acquire
+; CHECK: call i128 @__tsan_atomic128_fetch_sub(i128* %a, i128 0, i32 2), !dbg
 
 define void @atomic128_and_acquire(i128* %a) nounwind uwtable {
 entry:
-  atomicrmw and i128* %a, i128 0 acquire
-  ret void
+  atomicrmw and i128* %a, i128 0 acquire, !dbg !7
+  ret void, !dbg !7
 }
-; CHECK: atomic128_and_acquire
-; CHECK: call i128 @__tsan_atomic128_fetch_and(i128* %a, i128 0, i32 2)
+; CHECK-LABEL: atomic128_and_acquire
+; CHECK: call i128 @__tsan_atomic128_fetch_and(i128* %a, i128 0, i32 2), !dbg
 
 define void @atomic128_or_acquire(i128* %a) nounwind uwtable {
 entry:
-  atomicrmw or i128* %a, i128 0 acquire
-  ret void
+  atomicrmw or i128* %a, i128 0 acquire, !dbg !7
+  ret void, !dbg !7
 }
-; CHECK: atomic128_or_acquire
-; CHECK: call i128 @__tsan_atomic128_fetch_or(i128* %a, i128 0, i32 2)
+; CHECK-LABEL: atomic128_or_acquire
+; CHECK: call i128 @__tsan_atomic128_fetch_or(i128* %a, i128 0, i32 2), !dbg
 
 define void @atomic128_xor_acquire(i128* %a) nounwind uwtable {
 entry:
-  atomicrmw xor i128* %a, i128 0 acquire
-  ret void
+  atomicrmw xor i128* %a, i128 0 acquire, !dbg !7
+  ret void, !dbg !7
 }
-; CHECK: atomic128_xor_acquire
-; CHECK: call i128 @__tsan_atomic128_fetch_xor(i128* %a, i128 0, i32 2)
+; CHECK-LABEL: atomic128_xor_acquire
+; CHECK: call i128 @__tsan_atomic128_fetch_xor(i128* %a, i128 0, i32 2), !dbg
 
 define void @atomic128_nand_acquire(i128* %a) nounwind uwtable {
 entry:
-  atomicrmw nand i128* %a, i128 0 acquire
-  ret void
+  atomicrmw nand i128* %a, i128 0 acquire, !dbg !7
+  ret void, !dbg !7
 }
-; CHECK: atomic128_nand_acquire
-; CHECK: call i128 @__tsan_atomic128_fetch_nand(i128* %a, i128 0, i32 2)
+; CHECK-LABEL: atomic128_nand_acquire
+; CHECK: call i128 @__tsan_atomic128_fetch_nand(i128* %a, i128 0, i32 2), !dbg
 
 define void @atomic128_xchg_release(i128* %a) nounwind uwtable {
 entry:
-  atomicrmw xchg i128* %a, i128 0 release
-  ret void
+  atomicrmw xchg i128* %a, i128 0 release, !dbg !7
+  ret void, !dbg !7
 }
-; CHECK: atomic128_xchg_release
-; CHECK: call i128 @__tsan_atomic128_exchange(i128* %a, i128 0, i32 3)
+; CHECK-LABEL: atomic128_xchg_release
+; CHECK: call i128 @__tsan_atomic128_exchange(i128* %a, i128 0, i32 3), !dbg
 
 define void @atomic128_add_release(i128* %a) nounwind uwtable {
 entry:
-  atomicrmw add i128* %a, i128 0 release
-  ret void
+  atomicrmw add i128* %a, i128 0 release, !dbg !7
+  ret void, !dbg !7
 }
-; CHECK: atomic128_add_release
-; CHECK: call i128 @__tsan_atomic128_fetch_add(i128* %a, i128 0, i32 3)
+; CHECK-LABEL: atomic128_add_release
+; CHECK: call i128 @__tsan_atomic128_fetch_add(i128* %a, i128 0, i32 3), !dbg
 
 define void @atomic128_sub_release(i128* %a) nounwind uwtable {
 entry:
-  atomicrmw sub i128* %a, i128 0 release
-  ret void
+  atomicrmw sub i128* %a, i128 0 release, !dbg !7
+  ret void, !dbg !7
 }
-; CHECK: atomic128_sub_release
-; CHECK: call i128 @__tsan_atomic128_fetch_sub(i128* %a, i128 0, i32 3)
+; CHECK-LABEL: atomic128_sub_release
+; CHECK: call i128 @__tsan_atomic128_fetch_sub(i128* %a, i128 0, i32 3), !dbg
 
 define void @atomic128_and_release(i128* %a) nounwind uwtable {
 entry:
-  atomicrmw and i128* %a, i128 0 release
-  ret void
+  atomicrmw and i128* %a, i128 0 release, !dbg !7
+  ret void, !dbg !7
 }
-; CHECK: atomic128_and_release
-; CHECK: call i128 @__tsan_atomic128_fetch_and(i128* %a, i128 0, i32 3)
+; CHECK-LABEL: atomic128_and_release
+; CHECK: call i128 @__tsan_atomic128_fetch_and(i128* %a, i128 0, i32 3), !dbg
 
 define void @atomic128_or_release(i128* %a) nounwind uwtable {
 entry:
-  atomicrmw or i128* %a, i128 0 release
-  ret void
+  atomicrmw or i128* %a, i128 0 release, !dbg !7
+  ret void, !dbg !7
 }
-; CHECK: atomic128_or_release
-; CHECK: call i128 @__tsan_atomic128_fetch_or(i128* %a, i128 0, i32 3)
+; CHECK-LABEL: atomic128_or_release
+; CHECK: call i128 @__tsan_atomic128_fetch_or(i128* %a, i128 0, i32 3), !dbg
 
 define void @atomic128_xor_release(i128* %a) nounwind uwtable {
 entry:
-  atomicrmw xor i128* %a, i128 0 release
-  ret void
+  atomicrmw xor i128* %a, i128 0 release, !dbg !7
+  ret void, !dbg !7
 }
-; CHECK: atomic128_xor_release
-; CHECK: call i128 @__tsan_atomic128_fetch_xor(i128* %a, i128 0, i32 3)
+; CHECK-LABEL: atomic128_xor_release
+; CHECK: call i128 @__tsan_atomic128_fetch_xor(i128* %a, i128 0, i32 3), !dbg
 
 define void @atomic128_nand_release(i128* %a) nounwind uwtable {
 entry:
-  atomicrmw nand i128* %a, i128 0 release
-  ret void
+  atomicrmw nand i128* %a, i128 0 release, !dbg !7
+  ret void, !dbg !7
 }
-; CHECK: atomic128_nand_release
-; CHECK: call i128 @__tsan_atomic128_fetch_nand(i128* %a, i128 0, i32 3)
+; CHECK-LABEL: atomic128_nand_release
+; CHECK: call i128 @__tsan_atomic128_fetch_nand(i128* %a, i128 0, i32 3), !dbg
 
 define void @atomic128_xchg_acq_rel(i128* %a) nounwind uwtable {
 entry:
-  atomicrmw xchg i128* %a, i128 0 acq_rel
-  ret void
+  atomicrmw xchg i128* %a, i128 0 acq_rel, !dbg !7
+  ret void, !dbg !7
 }
-; CHECK: atomic128_xchg_acq_rel
-; CHECK: call i128 @__tsan_atomic128_exchange(i128* %a, i128 0, i32 4)
+; CHECK-LABEL: atomic128_xchg_acq_rel
+; CHECK: call i128 @__tsan_atomic128_exchange(i128* %a, i128 0, i32 4), !dbg
 
 define void @atomic128_add_acq_rel(i128* %a) nounwind uwtable {
 entry:
-  atomicrmw add i128* %a, i128 0 acq_rel
-  ret void
+  atomicrmw add i128* %a, i128 0 acq_rel, !dbg !7
+  ret void, !dbg !7
 }
-; CHECK: atomic128_add_acq_rel
-; CHECK: call i128 @__tsan_atomic128_fetch_add(i128* %a, i128 0, i32 4)
+; CHECK-LABEL: atomic128_add_acq_rel
+; CHECK: call i128 @__tsan_atomic128_fetch_add(i128* %a, i128 0, i32 4), !dbg
 
 define void @atomic128_sub_acq_rel(i128* %a) nounwind uwtable {
 entry:
-  atomicrmw sub i128* %a, i128 0 acq_rel
-  ret void
+  atomicrmw sub i128* %a, i128 0 acq_rel, !dbg !7
+  ret void, !dbg !7
 }
-; CHECK: atomic128_sub_acq_rel
-; CHECK: call i128 @__tsan_atomic128_fetch_sub(i128* %a, i128 0, i32 4)
+; CHECK-LABEL: atomic128_sub_acq_rel
+; CHECK: call i128 @__tsan_atomic128_fetch_sub(i128* %a, i128 0, i32 4), !dbg
 
 define void @atomic128_and_acq_rel(i128* %a) nounwind uwtable {
 entry:
-  atomicrmw and i128* %a, i128 0 acq_rel
-  ret void
+  atomicrmw and i128* %a, i128 0 acq_rel, !dbg !7
+  ret void, !dbg !7
 }
-; CHECK: atomic128_and_acq_rel
-; CHECK: call i128 @__tsan_atomic128_fetch_and(i128* %a, i128 0, i32 4)
+; CHECK-LABEL: atomic128_and_acq_rel
+; CHECK: call i128 @__tsan_atomic128_fetch_and(i128* %a, i128 0, i32 4), !dbg
 
 define void @atomic128_or_acq_rel(i128* %a) nounwind uwtable {
 entry:
-  atomicrmw or i128* %a, i128 0 acq_rel
-  ret void
+  atomicrmw or i128* %a, i128 0 acq_rel, !dbg !7
+  ret void, !dbg !7
 }
-; CHECK: atomic128_or_acq_rel
-; CHECK: call i128 @__tsan_atomic128_fetch_or(i128* %a, i128 0, i32 4)
+; CHECK-LABEL: atomic128_or_acq_rel
+; CHECK: call i128 @__tsan_atomic128_fetch_or(i128* %a, i128 0, i32 4), !dbg
 
 define void @atomic128_xor_acq_rel(i128* %a) nounwind uwtable {
 entry:
-  atomicrmw xor i128* %a, i128 0 acq_rel
-  ret void
+  atomicrmw xor i128* %a, i128 0 acq_rel, !dbg !7
+  ret void, !dbg !7
 }
-; CHECK: atomic128_xor_acq_rel
-; CHECK: call i128 @__tsan_atomic128_fetch_xor(i128* %a, i128 0, i32 4)
+; CHECK-LABEL: atomic128_xor_acq_rel
+; CHECK: call i128 @__tsan_atomic128_fetch_xor(i128* %a, i128 0, i32 4), !dbg
 
 define void @atomic128_nand_acq_rel(i128* %a) nounwind uwtable {
 entry:
-  atomicrmw nand i128* %a, i128 0 acq_rel
-  ret void
+  atomicrmw nand i128* %a, i128 0 acq_rel, !dbg !7
+  ret void, !dbg !7
 }
-; CHECK: atomic128_nand_acq_rel
-; CHECK: call i128 @__tsan_atomic128_fetch_nand(i128* %a, i128 0, i32 4)
+; CHECK-LABEL: atomic128_nand_acq_rel
+; CHECK: call i128 @__tsan_atomic128_fetch_nand(i128* %a, i128 0, i32 4), !dbg
 
 define void @atomic128_xchg_seq_cst(i128* %a) nounwind uwtable {
 entry:
-  atomicrmw xchg i128* %a, i128 0 seq_cst
-  ret void
+  atomicrmw xchg i128* %a, i128 0 seq_cst, !dbg !7
+  ret void, !dbg !7
 }
-; CHECK: atomic128_xchg_seq_cst
-; CHECK: call i128 @__tsan_atomic128_exchange(i128* %a, i128 0, i32 5)
+; CHECK-LABEL: atomic128_xchg_seq_cst
+; CHECK: call i128 @__tsan_atomic128_exchange(i128* %a, i128 0, i32 5), !dbg
 
 define void @atomic128_add_seq_cst(i128* %a) nounwind uwtable {
 entry:
-  atomicrmw add i128* %a, i128 0 seq_cst
-  ret void
+  atomicrmw add i128* %a, i128 0 seq_cst, !dbg !7
+  ret void, !dbg !7
 }
-; CHECK: atomic128_add_seq_cst
-; CHECK: call i128 @__tsan_atomic128_fetch_add(i128* %a, i128 0, i32 5)
+; CHECK-LABEL: atomic128_add_seq_cst
+; CHECK: call i128 @__tsan_atomic128_fetch_add(i128* %a, i128 0, i32 5), !dbg
 
 define void @atomic128_sub_seq_cst(i128* %a) nounwind uwtable {
 entry:
-  atomicrmw sub i128* %a, i128 0 seq_cst
-  ret void
+  atomicrmw sub i128* %a, i128 0 seq_cst, !dbg !7
+  ret void, !dbg !7
 }
-; CHECK: atomic128_sub_seq_cst
-; CHECK: call i128 @__tsan_atomic128_fetch_sub(i128* %a, i128 0, i32 5)
+; CHECK-LABEL: atomic128_sub_seq_cst
+; CHECK: call i128 @__tsan_atomic128_fetch_sub(i128* %a, i128 0, i32 5), !dbg
 
 define void @atomic128_and_seq_cst(i128* %a) nounwind uwtable {
 entry:
-  atomicrmw and i128* %a, i128 0 seq_cst
-  ret void
+  atomicrmw and i128* %a, i128 0 seq_cst, !dbg !7
+  ret void, !dbg !7
 }
-; CHECK: atomic128_and_seq_cst
-; CHECK: call i128 @__tsan_atomic128_fetch_and(i128* %a, i128 0, i32 5)
+; CHECK-LABEL: atomic128_and_seq_cst
+; CHECK: call i128 @__tsan_atomic128_fetch_and(i128* %a, i128 0, i32 5), !dbg
 
 define void @atomic128_or_seq_cst(i128* %a) nounwind uwtable {
 entry:
-  atomicrmw or i128* %a, i128 0 seq_cst
-  ret void
+  atomicrmw or i128* %a, i128 0 seq_cst, !dbg !7
+  ret void, !dbg !7
 }
-; CHECK: atomic128_or_seq_cst
-; CHECK: call i128 @__tsan_atomic128_fetch_or(i128* %a, i128 0, i32 5)
+; CHECK-LABEL: atomic128_or_seq_cst
+; CHECK: call i128 @__tsan_atomic128_fetch_or(i128* %a, i128 0, i32 5), !dbg
 
 define void @atomic128_xor_seq_cst(i128* %a) nounwind uwtable {
 entry:
-  atomicrmw xor i128* %a, i128 0 seq_cst
-  ret void
+  atomicrmw xor i128* %a, i128 0 seq_cst, !dbg !7
+  ret void, !dbg !7
 }
-; CHECK: atomic128_xor_seq_cst
-; CHECK: call i128 @__tsan_atomic128_fetch_xor(i128* %a, i128 0, i32 5)
+; CHECK-LABEL: atomic128_xor_seq_cst
+; CHECK: call i128 @__tsan_atomic128_fetch_xor(i128* %a, i128 0, i32 5), !dbg
 
 define void @atomic128_nand_seq_cst(i128* %a) nounwind uwtable {
 entry:
-  atomicrmw nand i128* %a, i128 0 seq_cst
-  ret void
+  atomicrmw nand i128* %a, i128 0 seq_cst, !dbg !7
+  ret void, !dbg !7
 }
-; CHECK: atomic128_nand_seq_cst
-; CHECK: call i128 @__tsan_atomic128_fetch_nand(i128* %a, i128 0, i32 5)
+; CHECK-LABEL: atomic128_nand_seq_cst
+; CHECK: call i128 @__tsan_atomic128_fetch_nand(i128* %a, i128 0, i32 5), !dbg
 
 define void @atomic128_cas_monotonic(i128* %a) nounwind uwtable {
 entry:
-  cmpxchg i128* %a, i128 0, i128 1 monotonic monotonic
-  ret void
+  cmpxchg i128* %a, i128 0, i128 1 monotonic monotonic, !dbg !7
+  ret void, !dbg !7
 }
-; CHECK: atomic128_cas_monotonic
-; CHECK: call i128 @__tsan_atomic128_compare_exchange_val(i128* %a, i128 0, i128 1, i32 0, i32 0)
+; CHECK-LABEL: atomic128_cas_monotonic
+; CHECK: call i128 @__tsan_atomic128_compare_exchange_val(i128* %a, i128 0, i128 1, i32 0, i32 0), !dbg
 
 define void @atomic128_cas_acquire(i128* %a) nounwind uwtable {
 entry:
-  cmpxchg i128* %a, i128 0, i128 1 acquire acquire
-  ret void
+  cmpxchg i128* %a, i128 0, i128 1 acquire acquire, !dbg !7
+  ret void, !dbg !7
 }
-; CHECK: atomic128_cas_acquire
-; CHECK: call i128 @__tsan_atomic128_compare_exchange_val(i128* %a, i128 0, i128 1, i32 2, i32 2)
+; CHECK-LABEL: atomic128_cas_acquire
+; CHECK: call i128 @__tsan_atomic128_compare_exchange_val(i128* %a, i128 0, i128 1, i32 2, i32 2), !dbg
 
 define void @atomic128_cas_release(i128* %a) nounwind uwtable {
 entry:
-  cmpxchg i128* %a, i128 0, i128 1 release monotonic
-  ret void
+  cmpxchg i128* %a, i128 0, i128 1 release monotonic, !dbg !7
+  ret void, !dbg !7
 }
-; CHECK: atomic128_cas_release
-; CHECK: call i128 @__tsan_atomic128_compare_exchange_val(i128* %a, i128 0, i128 1, i32 3, i32 0)
+; CHECK-LABEL: atomic128_cas_release
+; CHECK: call i128 @__tsan_atomic128_compare_exchange_val(i128* %a, i128 0, i128 1, i32 3, i32 0), !dbg
 
 define void @atomic128_cas_acq_rel(i128* %a) nounwind uwtable {
 entry:
-  cmpxchg i128* %a, i128 0, i128 1 acq_rel acquire
-  ret void
+  cmpxchg i128* %a, i128 0, i128 1 acq_rel acquire, !dbg !7
+  ret void, !dbg !7
 }
-; CHECK: atomic128_cas_acq_rel
-; CHECK: call i128 @__tsan_atomic128_compare_exchange_val(i128* %a, i128 0, i128 1, i32 4, i32 2)
+; CHECK-LABEL: atomic128_cas_acq_rel
+; CHECK: call i128 @__tsan_atomic128_compare_exchange_val(i128* %a, i128 0, i128 1, i32 4, i32 2), !dbg
 
 define void @atomic128_cas_seq_cst(i128* %a) nounwind uwtable {
 entry:
-  cmpxchg i128* %a, i128 0, i128 1 seq_cst seq_cst
-  ret void
+  cmpxchg i128* %a, i128 0, i128 1 seq_cst seq_cst, !dbg !7
+  ret void, !dbg !7
 }
-; CHECK: atomic128_cas_seq_cst
-; CHECK: call i128 @__tsan_atomic128_compare_exchange_val(i128* %a, i128 0, i128 1, i32 5, i32 5)
+; CHECK-LABEL: atomic128_cas_seq_cst
+; CHECK: call i128 @__tsan_atomic128_compare_exchange_val(i128* %a, i128 0, i128 1, i32 5, i32 5), !dbg
 
 define void @atomic_signal_fence_acquire() nounwind uwtable {
 entry:
-  fence singlethread acquire
-  ret void
+  fence singlethread acquire, !dbg !7
+  ret void, !dbg !7
 }
-; CHECK: atomic_signal_fence_acquire
-; CHECK: call void @__tsan_atomic_signal_fence(i32 2)
+; CHECK-LABEL: atomic_signal_fence_acquire
+; CHECK: call void @__tsan_atomic_signal_fence(i32 2), !dbg
 
 define void @atomic_thread_fence_acquire() nounwind uwtable {
 entry:
-  fence  acquire
-  ret void
+  fence  acquire, !dbg !7
+  ret void, !dbg !7
 }
-; CHECK: atomic_thread_fence_acquire
-; CHECK: call void @__tsan_atomic_thread_fence(i32 2)
+; CHECK-LABEL: atomic_thread_fence_acquire
+; CHECK: call void @__tsan_atomic_thread_fence(i32 2), !dbg
 
 define void @atomic_signal_fence_release() nounwind uwtable {
 entry:
-  fence singlethread release
-  ret void
+  fence singlethread release, !dbg !7
+  ret void, !dbg !7
 }
-; CHECK: atomic_signal_fence_release
-; CHECK: call void @__tsan_atomic_signal_fence(i32 3)
+; CHECK-LABEL: atomic_signal_fence_release
+; CHECK: call void @__tsan_atomic_signal_fence(i32 3), !dbg
 
 define void @atomic_thread_fence_release() nounwind uwtable {
 entry:
-  fence  release
-  ret void
+  fence  release, !dbg !7
+  ret void, !dbg !7
 }
-; CHECK: atomic_thread_fence_release
-; CHECK: call void @__tsan_atomic_thread_fence(i32 3)
+; CHECK-LABEL: atomic_thread_fence_release
+; CHECK: call void @__tsan_atomic_thread_fence(i32 3), !dbg
 
 define void @atomic_signal_fence_acq_rel() nounwind uwtable {
 entry:
-  fence singlethread acq_rel
-  ret void
+  fence singlethread acq_rel, !dbg !7
+  ret void, !dbg !7
 }
-; CHECK: atomic_signal_fence_acq_rel
-; CHECK: call void @__tsan_atomic_signal_fence(i32 4)
+; CHECK-LABEL: atomic_signal_fence_acq_rel
+; CHECK: call void @__tsan_atomic_signal_fence(i32 4), !dbg
 
 define void @atomic_thread_fence_acq_rel() nounwind uwtable {
 entry:
-  fence  acq_rel
-  ret void
+  fence  acq_rel, !dbg !7
+  ret void, !dbg !7
 }
-; CHECK: atomic_thread_fence_acq_rel
-; CHECK: call void @__tsan_atomic_thread_fence(i32 4)
+; CHECK-LABEL: atomic_thread_fence_acq_rel
+; CHECK: call void @__tsan_atomic_thread_fence(i32 4), !dbg
 
 define void @atomic_signal_fence_seq_cst() nounwind uwtable {
 entry:
-  fence singlethread seq_cst
-  ret void
+  fence singlethread seq_cst, !dbg !7
+  ret void, !dbg !7
 }
-; CHECK: atomic_signal_fence_seq_cst
-; CHECK: call void @__tsan_atomic_signal_fence(i32 5)
+; CHECK-LABEL: atomic_signal_fence_seq_cst
+; CHECK: call void @__tsan_atomic_signal_fence(i32 5), !dbg
 
 define void @atomic_thread_fence_seq_cst() nounwind uwtable {
 entry:
-  fence  seq_cst
-  ret void
+  fence  seq_cst, !dbg !7
+  ret void, !dbg !7
 }
-; CHECK: atomic_thread_fence_seq_cst
-; CHECK: call void @__tsan_atomic_thread_fence(i32 5)
+; CHECK-LABEL: atomic_thread_fence_seq_cst
+; CHECK: call void @__tsan_atomic_thread_fence(i32 5), !dbg
+
+!llvm.module.flags = !{!0, !1, !2}
+!0 = !{i32 2, !"Dwarf Version", i32 4}
+!1 = !{i32 2, !"Debug Info Version", i32 3}
+!2 = !{i32 1, !"PIC Level", i32 2}
+
+!3 = !{}
+!4 = !DISubroutineType(types: !3)
+!5 = !DIFile(filename: "atomic.cpp", directory: "/tmp")
+!6 = !DISubprogram(name: "test", scope: !5, file: !5, line: 99, type: !4, isLocal: false, isDefinition: true, scopeLine: 100, flags: DIFlagPrototyped, isOptimized: false, variables: !3)
+!7 = !DILocation(line: 100, column: 1, scope: !6)
diff --git a/test/Instrumentation/ThreadSanitizer/tsan_basic.ll b/test/Instrumentation/ThreadSanitizer/tsan_basic.ll
index 22582ebda73..7e049c548f2 100644
--- a/test/Instrumentation/ThreadSanitizer/tsan_basic.ll
+++ b/test/Instrumentation/ThreadSanitizer/tsan_basic.ll
@@ -9,7 +9,7 @@ entry:
   ret i32 %tmp1
 }
 
-; CHECK: @llvm.global_ctors = {{.*}}@__tsan_init
+; CHECK: @llvm.global_ctors = {{.*}}@tsan.module_ctor
 
 ; CHECK: define i32 @read_4_bytes(i32* %a)
 ; CHECK:        call void @__tsan_func_entry(i8* %0)
@@ -53,3 +53,6 @@ entry:
 ; CHECK: call i8* @memset
 ; CHECK: ret void
 }
+
+; CHECK: define internal void @tsan.module_ctor()
+; CHECK: call void @__tsan_init()
diff --git a/test/JitListener/multiple.ll b/test/JitListener/multiple.ll
index 912f4be9c04..2ba06ef1f0d 100644
--- a/test/JitListener/multiple.ll
+++ b/test/JitListener/multiple.ll
@@ -125,43 +125,43 @@ attributes #1 = { nounwind readnone }
 !llvm.module.flags = !{!11, !12, !13}
 !llvm.ident = !{!14}
 
-!0 = !MDCompileUnit(language: DW_LANG_C99, producer: "clang version 3.6.0 (trunk)", isOptimized: false, emissionKind: 1, file: !1, enums: !2, retainedTypes: !2, subprograms: !3, globals: !2, imports: !2)
-!1 = !MDFile(filename: "multiple.c", directory: "F:\5Cusers\5Cakaylor\5Cllvm-s\5Cllvm\5Ctest\5CJitListener")
+!0 = !DICompileUnit(language: DW_LANG_C99, producer: "clang version 3.6.0 (trunk)", isOptimized: false, emissionKind: 1, file: !1, enums: !2, retainedTypes: !2, subprograms: !3, globals: !2, imports: !2)
+!1 = !DIFile(filename: "multiple.c", directory: "F:\5Cusers\5Cakaylor\5Cllvm-s\5Cllvm\5Ctest\5CJitListener")
 !2 = !{}
 !3 = !{!4, !9, !10}
-!4 = !MDSubprogram(name: "foo", line: 1, isLocal: false, isDefinition: true, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 1, file: !1, scope: !5, type: !6, function: i32 (i32)* @foo, variables: !2)
-!5 = !MDFile(filename: "multiple.c", directory: "F:CusersCakaylorCllvm-sCllvmCtestCJitListener")
-!6 = !MDSubroutineType(types: !7)
+!4 = !DISubprogram(name: "foo", line: 1, isLocal: false, isDefinition: true, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 1, file: !1, scope: !5, type: !6, function: i32 (i32)* @foo, variables: !2)
+!5 = !DIFile(filename: "multiple.c", directory: "F:CusersCakaylorCllvm-sCllvmCtestCJitListener")
+!6 = !DISubroutineType(types: !7)
 !7 = !{!8, !8}
-!8 = !MDBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
-!9 = !MDSubprogram(name: "bar", line: 5, isLocal: false, isDefinition: true, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 5, file: !1, scope: !5, type: !6, function: i32 (i32)* @bar, variables: !2)
-!10 = !MDSubprogram(name: "fubar", line: 12, isLocal: false, isDefinition: true, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 12, file: !1, scope: !5, type: !6, function: i32 (i32)* @fubar, variables: !2)
+!8 = !DIBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
+!9 = !DISubprogram(name: "bar", line: 5, isLocal: false, isDefinition: true, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 5, file: !1, scope: !5, type: !6, function: i32 (i32)* @bar, variables: !2)
+!10 = !DISubprogram(name: "fubar", line: 12, isLocal: false, isDefinition: true, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 12, file: !1, scope: !5, type: !6, function: i32 (i32)* @fubar, variables: !2)
 !11 = !{i32 2, !"Dwarf Version", i32 4}
 !12 = !{i32 2, !"Debug Info Version", i32 3}
 !13 = !{i32 1, !"PIC Level", i32 2}
 !14 = !{!"clang version 3.6.0 (trunk)"}
-!15 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "a", line: 1, arg: 1, scope: !4, file: !5, type: !8)
-!16 = !MDExpression()
-!17 = !MDLocation(line: 1, column: 13, scope: !4)
-!18 = !MDLocation(line: 2, column: 10, scope: !4)
-!19 = !MDLocation(line: 2, column: 3, scope: !4)
-!20 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "a", line: 5, arg: 1, scope: !9, file: !5, type: !8)
-!21 = !MDLocation(line: 5, column: 13, scope: !9)
-!22 = !MDLocation(line: 6, column: 7, scope: !23)
-!23 = distinct !MDLexicalBlock(line: 6, column: 7, file: !1, scope: !9)
-!24 = !MDLocation(line: 6, column: 7, scope: !9)
-!25 = !MDLocation(line: 7, column: 5, scope: !26)
-!26 = distinct !MDLexicalBlock(line: 6, column: 15, file: !1, scope: !23)
-!27 = !MDLocation(line: 9, column: 14, scope: !9)
-!28 = !MDLocation(line: 9, column: 10, scope: !9)
-!29 = !MDLocation(line: 9, column: 3, scope: !9)
-!30 = !MDLocation(line: 10, column: 1, scope: !9)
-!31 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "a", line: 12, arg: 1, scope: !10, file: !5, type: !8)
-!32 = !MDLocation(line: 12, column: 15, scope: !10)
-!33 = !MDLocation(line: 13, column: 11, scope: !10)
-!34 = !MDLocation(line: 13, column: 3, scope: !10)
-!35 = !MDLocation(line: 15, column: 7, scope: !36)
-!36 = distinct !MDLexicalBlock(line: 13, column: 14, file: !1, scope: !10)
-!37 = !MDLocation(line: 17, column: 7, scope: !36)
-!38 = !MDLocation(line: 19, column: 7, scope: !36)
-!39 = !MDLocation(line: 21, column: 1, scope: !10)
+!15 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "a", line: 1, arg: 1, scope: !4, file: !5, type: !8)
+!16 = !DIExpression()
+!17 = !DILocation(line: 1, column: 13, scope: !4)
+!18 = !DILocation(line: 2, column: 10, scope: !4)
+!19 = !DILocation(line: 2, column: 3, scope: !4)
+!20 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "a", line: 5, arg: 1, scope: !9, file: !5, type: !8)
+!21 = !DILocation(line: 5, column: 13, scope: !9)
+!22 = !DILocation(line: 6, column: 7, scope: !23)
+!23 = distinct !DILexicalBlock(line: 6, column: 7, file: !1, scope: !9)
+!24 = !DILocation(line: 6, column: 7, scope: !9)
+!25 = !DILocation(line: 7, column: 5, scope: !26)
+!26 = distinct !DILexicalBlock(line: 6, column: 15, file: !1, scope: !23)
+!27 = !DILocation(line: 9, column: 14, scope: !9)
+!28 = !DILocation(line: 9, column: 10, scope: !9)
+!29 = !DILocation(line: 9, column: 3, scope: !9)
+!30 = !DILocation(line: 10, column: 1, scope: !9)
+!31 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "a", line: 12, arg: 1, scope: !10, file: !5, type: !8)
+!32 = !DILocation(line: 12, column: 15, scope: !10)
+!33 = !DILocation(line: 13, column: 11, scope: !10)
+!34 = !DILocation(line: 13, column: 3, scope: !10)
+!35 = !DILocation(line: 15, column: 7, scope: !36)
+!36 = distinct !DILexicalBlock(line: 13, column: 14, file: !1, scope: !10)
+!37 = !DILocation(line: 17, column: 7, scope: !36)
+!38 = !DILocation(line: 19, column: 7, scope: !36)
+!39 = !DILocation(line: 21, column: 1, scope: !10)
diff --git a/test/JitListener/simple.ll b/test/JitListener/simple.ll
index 4586d584031..d98eef54e4a 100644
--- a/test/JitListener/simple.ll
+++ b/test/JitListener/simple.ll
@@ -35,20 +35,20 @@ attributes #1 = { nounwind readnone }
 !llvm.module.flags = !{!9, !10}
 !llvm.ident = !{!11}
 
-!0 = !MDCompileUnit(language: DW_LANG_C99, producer: "clang version 3.6.0 (trunk)", isOptimized: false, emissionKind: 1, file: !1, enums: !2, retainedTypes: !2, subprograms: !3, globals: !2, imports: !2)
-!1 = !MDFile(filename: "simple.c", directory: "F:\5Cusers\5Cakaylor\5Cllvm-s\5Cllvm\5Ctest\5CJitListener")
+!0 = !DICompileUnit(language: DW_LANG_C99, producer: "clang version 3.6.0 (trunk)", isOptimized: false, emissionKind: 1, file: !1, enums: !2, retainedTypes: !2, subprograms: !3, globals: !2, imports: !2)
+!1 = !DIFile(filename: "simple.c", directory: "F:\5Cusers\5Cakaylor\5Cllvm-s\5Cllvm\5Ctest\5CJitListener")
 !2 = !{}
 !3 = !{!4}
-!4 = !MDSubprogram(name: "foo", line: 1, isLocal: false, isDefinition: true, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 1, file: !1, scope: !5, type: !6, function: i32 (i32)* @foo, variables: !2)
-!5 = !MDFile(filename: "simple.c", directory: "F:CusersCakaylorCllvm-sCllvmCtestCJitListener")
-!6 = !MDSubroutineType(types: !7)
+!4 = !DISubprogram(name: "foo", line: 1, isLocal: false, isDefinition: true, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 1, file: !1, scope: !5, type: !6, function: i32 (i32)* @foo, variables: !2)
+!5 = !DIFile(filename: "simple.c", directory: "F:CusersCakaylorCllvm-sCllvmCtestCJitListener")
+!6 = !DISubroutineType(types: !7)
 !7 = !{!8, !8}
-!8 = !MDBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
+!8 = !DIBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
 !9 = !{i32 2, !"Dwarf Version", i32 4}
 !10 = !{i32 2, !"Debug Info Version", i32 3}
 !11 = !{!"clang version 3.6.0 (trunk)"}
-!12 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "a", line: 1, arg: 1, scope: !4, file: !5, type: !8)
-!13 = !MDExpression()
-!14 = !MDLocation(line: 1, column: 13, scope: !4)
-!15 = !MDLocation(line: 2, column: 10, scope: !4)
-!16 = !MDLocation(line: 2, column: 3, scope: !4)
+!12 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "a", line: 1, arg: 1, scope: !4, file: !5, type: !8)
+!13 = !DIExpression()
+!14 = !DILocation(line: 1, column: 13, scope: !4)
+!15 = !DILocation(line: 2, column: 10, scope: !4)
+!16 = !DILocation(line: 2, column: 3, scope: !4)
diff --git a/test/LTO/X86/linkonce_odr_func.ll b/test/LTO/X86/linkonce_odr_func.ll
index 7fa6527f47c..671b30a460c 100644
--- a/test/LTO/X86/linkonce_odr_func.ll
+++ b/test/LTO/X86/linkonce_odr_func.ll
@@ -46,7 +46,7 @@ declare void @f(void()*)
 
 declare void @p()
 
-define void @bar() {
+define void @bar() personality void()* @p {
 bb0:
   call void @foo1()
   call void @f(void()* @foo2)
@@ -56,6 +56,6 @@ bb1:
 bb2:
   ret void
 clean:
-  landingpad {i32, i32} personality void()* @p cleanup
+  landingpad {i32, i32} cleanup
   ret void
 }
diff --git a/test/LibDriver/Inputs/a.s b/test/LibDriver/Inputs/a.s
new file mode 100644
index 00000000000..88258e2797f
--- /dev/null
+++ b/test/LibDriver/Inputs/a.s
@@ -0,0 +1,2 @@
+.globl a
+a:
diff --git a/test/LibDriver/Inputs/b.s b/test/LibDriver/Inputs/b.s
new file mode 100644
index 00000000000..4890c9247c7
--- /dev/null
+++ b/test/LibDriver/Inputs/b.s
@@ -0,0 +1,2 @@
+.globl b
+b:
diff --git a/test/LibDriver/infer-output-path.test b/test/LibDriver/infer-output-path.test
new file mode 100644
index 00000000000..7a1bbcbbd18
--- /dev/null
+++ b/test/LibDriver/infer-output-path.test
@@ -0,0 +1,15 @@
+RUN: llvm-mc -triple=x86_64-pc-windows-msvc -filetype=obj -o %T/a.obj %S/Inputs/a.s
+RUN: llvm-mc -triple=x86_64-pc-windows-msvc -filetype=obj -o %T/b.o %S/Inputs/b.s
+RUN: llvm-mc -triple=x86_64-pc-windows-msvc -filetype=obj -o %T/c %S/Inputs/b.s
+
+RUN: rm -f %T/a.lib
+RUN: llvm-lib %T/a.obj
+RUN: test -e %T/a.lib
+
+RUN: rm -f %T/b.lib
+RUN: llvm-lib /libpath:%T b.o
+RUN: test -e %T/b.lib
+
+RUN: rm -f %T/c.lib
+RUN: llvm-lib /libpath:%T c
+RUN: test -e %T/c.lib
diff --git a/test/LibDriver/libpath.test b/test/LibDriver/libpath.test
new file mode 100644
index 00000000000..2cfca2456d9
--- /dev/null
+++ b/test/LibDriver/libpath.test
@@ -0,0 +1,15 @@
+RUN: mkdir -p %T/a %T/b
+RUN: llvm-mc -triple=x86_64-pc-windows-msvc -filetype=obj -o %T/a/foo.obj %S/Inputs/a.s
+RUN: llvm-mc -triple=x86_64-pc-windows-msvc -filetype=obj -o %T/b/foo.obj %S/Inputs/b.s
+
+RUN: env "LIB=%T/a;%T/b" llvm-lib /out:%t1.lib foo.obj
+RUN: llvm-nm %t1.lib | FileCheck --check-prefix=A %s
+
+RUN: llvm-lib /out:%t2.lib /libpath:%T/a /libpath:%T/b foo.obj
+RUN: llvm-nm %t2.lib | FileCheck --check-prefix=A %s
+
+RUN: env LIB=%T/a llvm-lib /libpath:%T/b /out:%t3.lib foo.obj
+RUN: llvm-nm %t3.lib | FileCheck --check-prefix=B %s
+
+A: T a
+B: T b
diff --git a/test/LibDriver/lit.local.cfg b/test/LibDriver/lit.local.cfg
new file mode 100644
index 00000000000..e71f3cc4c41
--- /dev/null
+++ b/test/LibDriver/lit.local.cfg
@@ -0,0 +1,3 @@
+if not 'X86' in config.root.targets:
+    config.unsupported = True
+
diff --git a/test/LibDriver/thin.test b/test/LibDriver/thin.test
new file mode 100644
index 00000000000..4ab9d61952b
--- /dev/null
+++ b/test/LibDriver/thin.test
@@ -0,0 +1,9 @@
+RUN: echo foo > %t
+
+RUN: llvm-lib  -out:%t.a %t
+RUN: FileCheck --check-prefix=FAT %s < %t.a
+FAT: !<arch>
+
+RUN: llvm-lib  -out:%t.thin.a -llvmlibthin %t
+RUN: FileCheck --check-prefix=THIN %s < %t.thin.a
+THIN: !<thin>
diff --git a/test/Linker/2009-09-03-mdnode.ll b/test/Linker/2009-09-03-mdnode.ll
index bdb2b1e15e8..ec444d316b7 100644
--- a/test/Linker/2009-09-03-mdnode.ll
+++ b/test/Linker/2009-09-03-mdnode.ll
@@ -26,6 +26,6 @@ declare void @llvm.dbg.stoppoint(i32, i32, metadata) nounwind readnone
 
 declare void @llvm.dbg.region.end(metadata) nounwind readnone
 
-!0 = !MDSubprogram(name: "main", linkageName: "main", line: 2, isLocal: false, isDefinition: true, virtualIndex: 6, isOptimized: false, scope: !1)
-!1 = !MDCompileUnit(language: DW_LANG_C99, producer: "ellcc 0.1.0", isOptimized: true, emissionKind: 0, file: !2)
-!2 = !MDFile(filename: "a.c", directory: "/home/rich/ellcc/test/source")
+!0 = !DISubprogram(name: "main", linkageName: "main", line: 2, isLocal: false, isDefinition: true, virtualIndex: 6, isOptimized: false, scope: !1)
+!1 = !DICompileUnit(language: DW_LANG_C99, producer: "ellcc 0.1.0", isOptimized: true, emissionKind: 0, file: !2)
+!2 = !DIFile(filename: "a.c", directory: "/home/rich/ellcc/test/source")
diff --git a/test/Linker/2009-09-03-mdnode2.ll b/test/Linker/2009-09-03-mdnode2.ll
index 3b046a9fe7c..d9d52680b7b 100644
--- a/test/Linker/2009-09-03-mdnode2.ll
+++ b/test/Linker/2009-09-03-mdnode2.ll
@@ -21,6 +21,6 @@ declare void @llvm.dbg.stoppoint(i32, i32, metadata) nounwind readnone
 
 declare void @llvm.dbg.region.end(metadata) nounwind readnone
 
-!0 = !MDSubprogram(name: "f", linkageName: "f", line: 1, isLocal: false, isDefinition: true, virtualIndex: 6, isOptimized: false, scope: !1)
-!1 = !MDCompileUnit(language: DW_LANG_C99, producer: "ellcc 0.1.0", isOptimized: true, emissionKind: 0, file: !2)
-!2 = !MDFile(filename: "b.c", directory: "/home/rich/ellcc/test/source")
+!0 = !DISubprogram(name: "f", linkageName: "f", line: 1, isLocal: false, isDefinition: true, virtualIndex: 6, isOptimized: false, scope: !1)
+!1 = !DICompileUnit(language: DW_LANG_C99, producer: "ellcc 0.1.0", isOptimized: true, emissionKind: 0, file: !2)
+!2 = !DIFile(filename: "b.c", directory: "/home/rich/ellcc/test/source")
diff --git a/test/Linker/2011-08-04-DebugLoc.ll b/test/Linker/2011-08-04-DebugLoc.ll
index 25d08cdffed..710ddf7774f 100644
--- a/test/Linker/2011-08-04-DebugLoc.ll
+++ b/test/Linker/2011-08-04-DebugLoc.ll
@@ -17,15 +17,15 @@ define i32 @foo() nounwind ssp {
 !llvm.module.flags = !{!11}
 !llvm.dbg.sp = !{!1}
 
-!0 = !MDCompileUnit(language: DW_LANG_C99, producer: "Apple clang version 3.0 (tags/Apple/clang-209.11) (based on LLVM 3.0svn)", isOptimized: true, emissionKind: 0, file: !8, enums: !9, retainedTypes: !9, subprograms: !10)
-!1 = !MDSubprogram(name: "foo", line: 2, isLocal: false, isDefinition: true, virtualIndex: 6, isOptimized: false, file: !8, scope: !2, type: !3, function: i32 ()* @foo)
-!2 = !MDFile(filename: "a.c", directory: "/private/tmp")
-!3 = !MDSubroutineType(types: !4)
+!0 = !DICompileUnit(language: DW_LANG_C99, producer: "Apple clang version 3.0 (tags/Apple/clang-209.11) (based on LLVM 3.0svn)", isOptimized: true, emissionKind: 0, file: !8, enums: !9, retainedTypes: !9, subprograms: !10)
+!1 = !DISubprogram(name: "foo", line: 2, isLocal: false, isDefinition: true, virtualIndex: 6, isOptimized: false, file: !8, scope: !2, type: !3, function: i32 ()* @foo)
+!2 = !DIFile(filename: "a.c", directory: "/private/tmp")
+!3 = !DISubroutineType(types: !4)
 !4 = !{!5}
-!5 = !MDBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
-!6 = !MDLocation(line: 2, column: 13, scope: !7)
-!7 = distinct !MDLexicalBlock(line: 2, column: 11, file: !8, scope: !1)
-!8 = !MDFile(filename: "a.c", directory: "/private/tmp")
+!5 = !DIBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
+!6 = !DILocation(line: 2, column: 13, scope: !7)
+!7 = distinct !DILexicalBlock(line: 2, column: 11, file: !8, scope: !1)
+!8 = !DIFile(filename: "a.c", directory: "/private/tmp")
 !9 = !{}
 !10 = !{!1}
 !11 = !{i32 1, !"Debug Info Version", i32 3}
diff --git a/test/Linker/2011-08-04-DebugLoc2.ll b/test/Linker/2011-08-04-DebugLoc2.ll
index 76e73dde27a..bc524dae91d 100644
--- a/test/Linker/2011-08-04-DebugLoc2.ll
+++ b/test/Linker/2011-08-04-DebugLoc2.ll
@@ -14,15 +14,15 @@ define i32 @bar() nounwind ssp {
 !llvm.module.flags = !{!11}
 !llvm.dbg.sp = !{!1}
 
-!0 = !MDCompileUnit(language: DW_LANG_C99, producer: "Apple clang version 3.0 (tags/Apple/clang-209.11) (based on LLVM 3.0svn)", isOptimized: true, emissionKind: 0, file: !8, enums: !9, retainedTypes: !9, subprograms: !10)
-!1 = !MDSubprogram(name: "bar", line: 1, isLocal: false, isDefinition: true, virtualIndex: 6, isOptimized: false, file: !8, scope: !2, type: !3, function: i32 ()* @bar)
-!2 = !MDFile(filename: "b.c", directory: "/private/tmp")
-!3 = !MDSubroutineType(types: !4)
+!0 = !DICompileUnit(language: DW_LANG_C99, producer: "Apple clang version 3.0 (tags/Apple/clang-209.11) (based on LLVM 3.0svn)", isOptimized: true, emissionKind: 0, file: !8, enums: !9, retainedTypes: !9, subprograms: !10)
+!1 = !DISubprogram(name: "bar", line: 1, isLocal: false, isDefinition: true, virtualIndex: 6, isOptimized: false, file: !8, scope: !2, type: !3, function: i32 ()* @bar)
+!2 = !DIFile(filename: "b.c", directory: "/private/tmp")
+!3 = !DISubroutineType(types: !4)
 !4 = !{!5}
-!5 = !MDBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
-!6 = !MDLocation(line: 1, column: 13, scope: !7)
-!7 = distinct !MDLexicalBlock(line: 1, column: 11, file: !8, scope: !1)
-!8 = !MDFile(filename: "b.c", directory: "/private/tmp")
+!5 = !DIBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
+!6 = !DILocation(line: 1, column: 13, scope: !7)
+!7 = distinct !DILexicalBlock(line: 1, column: 11, file: !8, scope: !1)
+!8 = !DIFile(filename: "b.c", directory: "/private/tmp")
 !9 = !{}
 !10 = !{!1}
 !11 = !{i32 1, !"Debug Info Version", i32 3}
diff --git a/test/Linker/2011-08-04-Metadata.ll b/test/Linker/2011-08-04-Metadata.ll
index 7b398003a2e..fc1af754abd 100644
--- a/test/Linker/2011-08-04-Metadata.ll
+++ b/test/Linker/2011-08-04-Metadata.ll
@@ -2,12 +2,12 @@
 ; RUN: llvm-dis < %t.bc | FileCheck %s
 ; Test if internal global variable's debug info is merged appropriately or not.
 
-; CHECK: !MDGlobalVariable(name: "x",
+; CHECK: !DIGlobalVariable(name: "x",
 ; CHECK-NOT:               linkageName:
 ; CHECK-SAME:              variable: i32* @x{{[,)]}}
-; CHECK: !MDGlobalVariable(name: "x",
+; CHECK: !DIGlobalVariable(name: "x",
 ; CHECK-NOT:               linkageName:
-; CHECK-SAME:              variable: i32* @x1{{[,)]}}
+; CHECK-SAME:              variable: i32* @x.1{{[,)]}}
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64"
 target triple = "x86_64-apple-macosx10.7.0"
 
@@ -24,15 +24,15 @@ entry:
 !llvm.dbg.sp = !{!1}
 !llvm.dbg.gv = !{!5}
 
-!0 = !MDCompileUnit(language: DW_LANG_C99, producer: "clang version 3.0 ()", isOptimized: true, emissionKind: 0, file: !9, enums: !{}, retainedTypes: !{}, subprograms: !10)
-!1 = !MDSubprogram(name: "foo", line: 3, isLocal: false, isDefinition: true, virtualIndex: 6, isOptimized: false, file: !9, scope: !2, type: !3, function: void ()* @foo)
-!2 = !MDFile(filename: "/tmp/one.c", directory: "/Volumes/Lalgate/Slate/D")
-!3 = !MDSubroutineType(types: !4)
+!0 = !DICompileUnit(language: DW_LANG_C99, producer: "clang version 3.0 ()", isOptimized: true, emissionKind: 0, file: !9, enums: !{}, retainedTypes: !{}, subprograms: !10)
+!1 = !DISubprogram(name: "foo", line: 3, isLocal: false, isDefinition: true, virtualIndex: 6, isOptimized: false, file: !9, scope: !2, type: !3, function: void ()* @foo)
+!2 = !DIFile(filename: "/tmp/one.c", directory: "/Volumes/Lalgate/Slate/D")
+!3 = !DISubroutineType(types: !4)
 !4 = !{null}
-!5 = !MDGlobalVariable(name: "x", line: 2, isLocal: true, isDefinition: true, scope: !0, file: !2, type: !6, variable: i32* @x)
-!6 = !MDBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
-!7 = !MDLocation(line: 3, column: 14, scope: !8)
-!8 = distinct !MDLexicalBlock(line: 3, column: 12, file: !9, scope: !1)
-!9 = !MDFile(filename: "/tmp/one.c", directory: "/Volumes/Lalgate/Slate/D")
+!5 = !DIGlobalVariable(name: "x", line: 2, isLocal: true, isDefinition: true, scope: !0, file: !2, type: !6, variable: i32* @x)
+!6 = !DIBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
+!7 = !DILocation(line: 3, column: 14, scope: !8)
+!8 = distinct !DILexicalBlock(line: 3, column: 12, file: !9, scope: !1)
+!9 = !DIFile(filename: "/tmp/one.c", directory: "/Volumes/Lalgate/Slate/D")
 !10 = !{!1}
 !11 = !{i32 1, !"Debug Info Version", i32 3}
diff --git a/test/Linker/2011-08-04-Metadata2.ll b/test/Linker/2011-08-04-Metadata2.ll
index ba8c6db6d16..5eb231b2e36 100644
--- a/test/Linker/2011-08-04-Metadata2.ll
+++ b/test/Linker/2011-08-04-Metadata2.ll
@@ -19,15 +19,15 @@ entry:
 !llvm.dbg.sp = !{!1}
 !llvm.dbg.gv = !{!5}
 
-!0 = !MDCompileUnit(language: DW_LANG_C99, producer: "clang version 3.0 ()", isOptimized: true, emissionKind: 0, file: !9, enums: !{}, retainedTypes: !{}, subprograms: !10)
-!1 = !MDSubprogram(name: "bar", line: 2, isLocal: false, isDefinition: true, virtualIndex: 6, isOptimized: false, file: !9, scope: !2, type: !3, function: void ()* @bar)
-!2 = !MDFile(filename: "/tmp/two.c", directory: "/Volumes/Lalgate/Slate/D")
-!3 = !MDSubroutineType(types: !4)
+!0 = !DICompileUnit(language: DW_LANG_C99, producer: "clang version 3.0 ()", isOptimized: true, emissionKind: 0, file: !9, enums: !{}, retainedTypes: !{}, subprograms: !10)
+!1 = !DISubprogram(name: "bar", line: 2, isLocal: false, isDefinition: true, virtualIndex: 6, isOptimized: false, file: !9, scope: !2, type: !3, function: void ()* @bar)
+!2 = !DIFile(filename: "/tmp/two.c", directory: "/Volumes/Lalgate/Slate/D")
+!3 = !DISubroutineType(types: !4)
 !4 = !{null}
-!5 = !MDGlobalVariable(name: "x", line: 1, isLocal: true, isDefinition: true, scope: !0, file: !2, type: !6, variable: i32* @x)
-!6 = !MDBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
-!7 = !MDLocation(line: 2, column: 14, scope: !8)
-!8 = distinct !MDLexicalBlock(line: 2, column: 12, file: !9, scope: !1)
-!9 = !MDFile(filename: "/tmp/two.c", directory: "/Volumes/Lalgate/Slate/D")
+!5 = !DIGlobalVariable(name: "x", line: 1, isLocal: true, isDefinition: true, scope: !0, file: !2, type: !6, variable: i32* @x)
+!6 = !DIBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
+!7 = !DILocation(line: 2, column: 14, scope: !8)
+!8 = distinct !DILexicalBlock(line: 2, column: 12, file: !9, scope: !1)
+!9 = !DIFile(filename: "/tmp/two.c", directory: "/Volumes/Lalgate/Slate/D")
 !10 = !{!1}
 !11 = !{i32 1, !"Debug Info Version", i32 3}
diff --git a/test/Linker/2011-08-18-unique-class-type.ll b/test/Linker/2011-08-18-unique-class-type.ll
index ee62e40092e..1692a90bf7e 100644
--- a/test/Linker/2011-08-18-unique-class-type.ll
+++ b/test/Linker/2011-08-18-unique-class-type.ll
@@ -11,7 +11,7 @@ target triple = "x86_64-apple-macosx10.7.0"
 define void @_Z3fooN2N11AE() nounwind uwtable ssp {
 entry:
   %mya = alloca %"class.N1::A", align 1
-  call void @llvm.dbg.declare(metadata %"class.N1::A"* %mya, metadata !9, metadata !MDExpression()), !dbg !13
+  call void @llvm.dbg.declare(metadata %"class.N1::A"* %mya, metadata !9, metadata !DIExpression()), !dbg !13
   ret void, !dbg !14
 }
 
@@ -20,21 +20,21 @@ declare void @llvm.dbg.declare(metadata, metadata, metadata) nounwind readnone
 !llvm.dbg.cu = !{!0}
 !llvm.module.flags = !{!18}
 
-!0 = !MDCompileUnit(language: DW_LANG_C_plus_plus, producer: "clang version 3.0 (trunk 137954)", isOptimized: true, emissionKind: 0, file: !16, enums: !2, retainedTypes: !2, subprograms: !3, globals: !2)
+!0 = !DICompileUnit(language: DW_LANG_C_plus_plus, producer: "clang version 3.0 (trunk 137954)", isOptimized: true, emissionKind: 0, file: !16, enums: !2, retainedTypes: !2, subprograms: !3, globals: !2)
 !1 = !{!2}
 !2 = !{}
 !3 = !{!5}
-!5 = !MDSubprogram(name: "foo", linkageName: "_Z3fooN2N11AE", line: 4, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, file: !16, scope: !6, type: !7, function: void ()* @_Z3fooN2N11AE)
-!6 = !MDFile(filename: "n1.c", directory: "/private/tmp")
-!7 = !MDSubroutineType(types: !8)
+!5 = !DISubprogram(name: "foo", linkageName: "_Z3fooN2N11AE", line: 4, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, file: !16, scope: !6, type: !7, function: void ()* @_Z3fooN2N11AE)
+!6 = !DIFile(filename: "n1.c", directory: "/private/tmp")
+!7 = !DISubroutineType(types: !8)
 !8 = !{null}
-!9 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "mya", line: 4, arg: 1, scope: !5, file: !6, type: !10)
-!10 = !MDCompositeType(tag: DW_TAG_class_type, name: "A", line: 3, size: 8, align: 8, file: !17, scope: !11, elements: !2)
-!11 = !MDNamespace(name: "N1", line: 2, file: !17, scope: null)
-!12 = !MDFile(filename: "./n.h", directory: "/private/tmp")
-!13 = !MDLocation(line: 4, column: 12, scope: !5)
-!14 = !MDLocation(line: 4, column: 18, scope: !15)
-!15 = distinct !MDLexicalBlock(line: 4, column: 17, file: !16, scope: !5)
-!16 = !MDFile(filename: "n1.c", directory: "/private/tmp")
-!17 = !MDFile(filename: "./n.h", directory: "/private/tmp")
+!9 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "mya", line: 4, arg: 1, scope: !5, file: !6, type: !10)
+!10 = !DICompositeType(tag: DW_TAG_class_type, name: "A", line: 3, size: 8, align: 8, file: !17, scope: !11, elements: !2)
+!11 = !DINamespace(name: "N1", line: 2, file: !17, scope: null)
+!12 = !DIFile(filename: "./n.h", directory: "/private/tmp")
+!13 = !DILocation(line: 4, column: 12, scope: !5)
+!14 = !DILocation(line: 4, column: 18, scope: !15)
+!15 = distinct !DILexicalBlock(line: 4, column: 17, file: !16, scope: !5)
+!16 = !DIFile(filename: "n1.c", directory: "/private/tmp")
+!17 = !DIFile(filename: "./n.h", directory: "/private/tmp")
 !18 = !{i32 1, !"Debug Info Version", i32 3}
diff --git a/test/Linker/2011-08-18-unique-class-type2.ll b/test/Linker/2011-08-18-unique-class-type2.ll
index a603aee8fa6..1befb4deacc 100644
--- a/test/Linker/2011-08-18-unique-class-type2.ll
+++ b/test/Linker/2011-08-18-unique-class-type2.ll
@@ -9,7 +9,7 @@ target triple = "x86_64-apple-macosx10.7.0"
 define void @_Z3barN2N11AE() nounwind uwtable ssp {
 entry:
   %youra = alloca %"class.N1::A", align 1
-  call void @llvm.dbg.declare(metadata %"class.N1::A"* %youra, metadata !9, metadata !MDExpression()), !dbg !13
+  call void @llvm.dbg.declare(metadata %"class.N1::A"* %youra, metadata !9, metadata !DIExpression()), !dbg !13
   ret void, !dbg !14
 }
 
@@ -18,21 +18,21 @@ declare void @llvm.dbg.declare(metadata, metadata, metadata) nounwind readnone
 !llvm.dbg.cu = !{!0}
 !llvm.module.flags = !{!18}
 
-!0 = !MDCompileUnit(language: DW_LANG_C_plus_plus, producer: "clang version 3.0 (trunk 137954)", isOptimized: true, emissionKind: 0, file: !16, enums: !2, retainedTypes: !2, subprograms: !3, globals: !2)
+!0 = !DICompileUnit(language: DW_LANG_C_plus_plus, producer: "clang version 3.0 (trunk 137954)", isOptimized: true, emissionKind: 0, file: !16, enums: !2, retainedTypes: !2, subprograms: !3, globals: !2)
 !1 = !{!2}
 !2 = !{}
 !3 = !{!5}
-!5 = !MDSubprogram(name: "bar", linkageName: "_Z3barN2N11AE", line: 4, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scope: !6, type: !7, function: void ()* @_Z3barN2N11AE)
-!6 = !MDFile(filename: "n2.c", directory: "/private/tmp")
-!7 = !MDSubroutineType(types: !8)
+!5 = !DISubprogram(name: "bar", linkageName: "_Z3barN2N11AE", line: 4, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scope: !6, type: !7, function: void ()* @_Z3barN2N11AE)
+!6 = !DIFile(filename: "n2.c", directory: "/private/tmp")
+!7 = !DISubroutineType(types: !8)
 !8 = !{null}
-!9 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "youra", line: 4, arg: 1, scope: !5, file: !6, type: !10)
-!10 = !MDCompositeType(tag: DW_TAG_class_type, name: "A", line: 3, size: 8, align: 8, file: !17, scope: !11, elements: !2)
-!11 = !MDNamespace(name: "N1", line: 2, file: !17, scope: null)
-!12 = !MDFile(filename: "./n.h", directory: "/private/tmp")
-!13 = !MDLocation(line: 4, column: 12, scope: !5)
-!14 = !MDLocation(line: 4, column: 20, scope: !15)
-!15 = distinct !MDLexicalBlock(line: 4, column: 19, file: !16, scope: !5)
-!16 = !MDFile(filename: "n2.c", directory: "/private/tmp")
-!17 = !MDFile(filename: "./n.h", directory: "/private/tmp")
+!9 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "youra", line: 4, arg: 1, scope: !5, file: !6, type: !10)
+!10 = !DICompositeType(tag: DW_TAG_class_type, name: "A", line: 3, size: 8, align: 8, file: !17, scope: !11, elements: !2)
+!11 = !DINamespace(name: "N1", line: 2, file: !17, scope: null)
+!12 = !DIFile(filename: "./n.h", directory: "/private/tmp")
+!13 = !DILocation(line: 4, column: 12, scope: !5)
+!14 = !DILocation(line: 4, column: 20, scope: !15)
+!15 = distinct !DILexicalBlock(line: 4, column: 19, file: !16, scope: !5)
+!16 = !DIFile(filename: "n2.c", directory: "/private/tmp")
+!17 = !DIFile(filename: "./n.h", directory: "/private/tmp")
 !18 = !{i32 1, !"Debug Info Version", i32 3}
diff --git a/test/Linker/2011-08-18-unique-debug-type.ll b/test/Linker/2011-08-18-unique-debug-type.ll
index a8ccd27acba..b1da9354f8c 100644
--- a/test/Linker/2011-08-18-unique-debug-type.ll
+++ b/test/Linker/2011-08-18-unique-debug-type.ll
@@ -1,6 +1,6 @@
 ; RUN: llvm-link %s %p/2011-08-18-unique-debug-type2.ll -S -o - | FileCheck %s
 ; Test to check only one MDNode for "int" after linking.
-; CHECK: !MDBasicType(name: "int"
+; CHECK: !DIBasicType(name: "int"
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64"
 target triple = "x86_64-apple-macosx10.7.0"
 
@@ -12,16 +12,16 @@ entry:
 !llvm.dbg.cu = !{!0}
 !llvm.module.flags = !{!13}
 
-!0 = !MDCompileUnit(language: DW_LANG_C99, producer: "clang version 3.0 (trunk 137954)", isOptimized: true, emissionKind: 0, file: !12, enums: !2, retainedTypes: !2, subprograms: !3, globals: !2)
+!0 = !DICompileUnit(language: DW_LANG_C99, producer: "clang version 3.0 (trunk 137954)", isOptimized: true, emissionKind: 0, file: !12, enums: !2, retainedTypes: !2, subprograms: !3, globals: !2)
 !1 = !{!2}
 !2 = !{}
 !3 = !{!5}
-!5 = !MDSubprogram(name: "foo", line: 1, isLocal: false, isDefinition: true, virtualIndex: 6, isOptimized: false, file: !12, scope: !6, type: !7, function: i32 ()* @foo)
-!6 = !MDFile(filename: "one.c", directory: "/private/tmp")
-!7 = !MDSubroutineType(types: !8)
+!5 = !DISubprogram(name: "foo", line: 1, isLocal: false, isDefinition: true, virtualIndex: 6, isOptimized: false, file: !12, scope: !6, type: !7, function: i32 ()* @foo)
+!6 = !DIFile(filename: "one.c", directory: "/private/tmp")
+!7 = !DISubroutineType(types: !8)
 !8 = !{!9}
-!9 = !MDBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
-!10 = !MDLocation(line: 1, column: 13, scope: !11)
-!11 = distinct !MDLexicalBlock(line: 1, column: 11, file: !12, scope: !5)
-!12 = !MDFile(filename: "one.c", directory: "/private/tmp")
+!9 = !DIBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
+!10 = !DILocation(line: 1, column: 13, scope: !11)
+!11 = distinct !DILexicalBlock(line: 1, column: 11, file: !12, scope: !5)
+!12 = !DIFile(filename: "one.c", directory: "/private/tmp")
 !13 = !{i32 1, !"Debug Info Version", i32 3}
diff --git a/test/Linker/2011-08-18-unique-debug-type2.ll b/test/Linker/2011-08-18-unique-debug-type2.ll
index 242f953459b..40958dabf88 100644
--- a/test/Linker/2011-08-18-unique-debug-type2.ll
+++ b/test/Linker/2011-08-18-unique-debug-type2.ll
@@ -12,16 +12,16 @@ entry:
 !llvm.dbg.cu = !{!0}
 !llvm.module.flags = !{!13}
 
-!0 = !MDCompileUnit(language: DW_LANG_C99, producer: "clang version 3.0 (trunk 137954)", isOptimized: true, emissionKind: 0, file: !12, enums: !2, retainedTypes: !2, subprograms: !3, globals: !2)
+!0 = !DICompileUnit(language: DW_LANG_C99, producer: "clang version 3.0 (trunk 137954)", isOptimized: true, emissionKind: 0, file: !12, enums: !2, retainedTypes: !2, subprograms: !3, globals: !2)
 !1 = !{!2}
 !2 = !{}
 !3 = !{!5}
-!5 = !MDSubprogram(name: "bar", line: 1, isLocal: false, isDefinition: true, virtualIndex: 6, isOptimized: false, file: !12, scope: !6, type: !7, function: i32 ()* @bar)
-!6 = !MDFile(filename: "two.c", directory: "/private/tmp")
-!7 = !MDSubroutineType(types: !8)
+!5 = !DISubprogram(name: "bar", line: 1, isLocal: false, isDefinition: true, virtualIndex: 6, isOptimized: false, file: !12, scope: !6, type: !7, function: i32 ()* @bar)
+!6 = !DIFile(filename: "two.c", directory: "/private/tmp")
+!7 = !DISubroutineType(types: !8)
 !8 = !{!9}
-!9 = !MDBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
-!10 = !MDLocation(line: 1, column: 13, scope: !11)
-!11 = distinct !MDLexicalBlock(line: 1, column: 11, file: !12, scope: !5)
-!12 = !MDFile(filename: "two.c", directory: "/private/tmp")
+!9 = !DIBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
+!10 = !DILocation(line: 1, column: 13, scope: !11)
+!11 = distinct !DILexicalBlock(line: 1, column: 11, file: !12, scope: !5)
+!12 = !DIFile(filename: "two.c", directory: "/private/tmp")
 !13 = !{i32 1, !"Debug Info Version", i32 3}
diff --git a/test/Linker/DbgDeclare.ll b/test/Linker/DbgDeclare.ll
index 6a9239f206a..a6ffa915c07 100644
--- a/test/Linker/DbgDeclare.ll
+++ b/test/Linker/DbgDeclare.ll
@@ -21,9 +21,9 @@ entry:
   %argv.addr = alloca i8**, align 8
   store i32 0, i32* %retval
   store i32 %argc, i32* %argc.addr, align 4
-  call void @llvm.dbg.declare(metadata i32* %argc.addr, metadata !14, metadata !MDExpression()), !dbg !15
+  call void @llvm.dbg.declare(metadata i32* %argc.addr, metadata !14, metadata !DIExpression()), !dbg !15
   store i8** %argv, i8*** %argv.addr, align 8
-  call void @llvm.dbg.declare(metadata i8*** %argv.addr, metadata !16, metadata !MDExpression()), !dbg !15
+  call void @llvm.dbg.declare(metadata i8*** %argv.addr, metadata !16, metadata !DIExpression()), !dbg !15
   %0 = load i32, i32* %argc.addr, align 4, !dbg !17
   %1 = load i8**, i8*** %argv.addr, align 8, !dbg !17
   call void @test(i32 %0, i8** %1), !dbg !17
@@ -37,23 +37,23 @@ declare void @test(i32, i8**)
 !llvm.dbg.cu = !{!0}
 !llvm.module.flags = !{!21}
 
-!0 = !MDCompileUnit(language: DW_LANG_C_plus_plus, producer: "clang version 3.3 (trunk 173515)", isOptimized: true, emissionKind: 0, file: !20, enums: !2, retainedTypes: !2, subprograms: !3, globals: !2)
+!0 = !DICompileUnit(language: DW_LANG_C_plus_plus, producer: "clang version 3.3 (trunk 173515)", isOptimized: true, emissionKind: 0, file: !20, enums: !2, retainedTypes: !2, subprograms: !3, globals: !2)
 !2 = !{}
 !3 = !{!5}
-!5 = !MDSubprogram(name: "main", line: 3, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 4, file: !20, scope: null, type: !7, function: i32 (i32, i8**)* @main, variables: !2)
-!6 = !MDFile(filename: "main.cpp", directory: "/private/tmp")
-!7 = !MDSubroutineType(types: !8)
+!5 = !DISubprogram(name: "main", line: 3, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 4, file: !20, scope: null, type: !7, function: i32 (i32, i8**)* @main, variables: !2)
+!6 = !DIFile(filename: "main.cpp", directory: "/private/tmp")
+!7 = !DISubroutineType(types: !8)
 !8 = !{!9, !9, !10}
-!9 = !MDBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
-!10 = !MDDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, baseType: !11)
-!11 = !MDDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, baseType: !12)
-!12 = !MDDerivedType(tag: DW_TAG_const_type, baseType: !13)
-!13 = !MDBasicType(tag: DW_TAG_base_type, name: "char", size: 8, align: 8, encoding: DW_ATE_signed_char)
-!14 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "argc", line: 3, arg: 1, scope: !5, file: !6, type: !9)
-!15 = !MDLocation(line: 3, scope: !5)
-!16 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "argv", line: 3, arg: 2, scope: !5, file: !6, type: !10)
-!17 = !MDLocation(line: 5, scope: !18)
-!18 = distinct !MDLexicalBlock(line: 4, column: 0, file: !20, scope: !5)
-!19 = !MDLocation(line: 6, scope: !18)
-!20 = !MDFile(filename: "main.cpp", directory: "/private/tmp")
+!9 = !DIBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
+!10 = !DIDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, baseType: !11)
+!11 = !DIDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, baseType: !12)
+!12 = !DIDerivedType(tag: DW_TAG_const_type, baseType: !13)
+!13 = !DIBasicType(tag: DW_TAG_base_type, name: "char", size: 8, align: 8, encoding: DW_ATE_signed_char)
+!14 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "argc", line: 3, arg: 1, scope: !5, file: !6, type: !9)
+!15 = !DILocation(line: 3, scope: !5)
+!16 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "argv", line: 3, arg: 2, scope: !5, file: !6, type: !10)
+!17 = !DILocation(line: 5, scope: !18)
+!18 = distinct !DILexicalBlock(line: 4, column: 0, file: !20, scope: !5)
+!19 = !DILocation(line: 6, scope: !18)
+!20 = !DIFile(filename: "main.cpp", directory: "/private/tmp")
 !21 = !{i32 1, !"Debug Info Version", i32 3}
diff --git a/test/Linker/DbgDeclare2.ll b/test/Linker/DbgDeclare2.ll
index 2afe0c22194..b716833ecc6 100644
--- a/test/Linker/DbgDeclare2.ll
+++ b/test/Linker/DbgDeclare2.ll
@@ -11,10 +11,10 @@ entry:
   %argv.addr = alloca i8**, align 8
   %i = alloca i32, align 4
   store i32 %argc, i32* %argc.addr, align 4
-  call void @llvm.dbg.declare(metadata i32* %argc.addr, metadata !14, metadata !MDExpression()), !dbg !15
+  call void @llvm.dbg.declare(metadata i32* %argc.addr, metadata !14, metadata !DIExpression()), !dbg !15
   store i8** %argv, i8*** %argv.addr, align 8
-  call void @llvm.dbg.declare(metadata i8*** %argv.addr, metadata !16, metadata !MDExpression()), !dbg !15
-  call void @llvm.dbg.declare(metadata i32* %i, metadata !17, metadata !MDExpression()), !dbg !20
+  call void @llvm.dbg.declare(metadata i8*** %argv.addr, metadata !16, metadata !DIExpression()), !dbg !15
+  call void @llvm.dbg.declare(metadata i32* %i, metadata !17, metadata !DIExpression()), !dbg !20
   store i32 0, i32* %i, align 4, !dbg !20
   br label %for.cond, !dbg !20
 
@@ -50,29 +50,29 @@ declare i32 @puts(i8*)
 !llvm.dbg.cu = !{!0}
 !llvm.module.flags = !{!27}
 
-!0 = !MDCompileUnit(language: DW_LANG_C_plus_plus, producer: "clang version 3.3 (trunk 173515)", isOptimized: true, emissionKind: 0, file: !25, enums: !2, retainedTypes: !2, subprograms: !3, globals: !2)
+!0 = !DICompileUnit(language: DW_LANG_C_plus_plus, producer: "clang version 3.3 (trunk 173515)", isOptimized: true, emissionKind: 0, file: !25, enums: !2, retainedTypes: !2, subprograms: !3, globals: !2)
 !2 = !{}
 !3 = !{!5}
-!5 = !MDSubprogram(name: "print_args", linkageName: "test", line: 4, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 5, file: !26, scope: null, type: !7, function: void (i32, i8**)* @test, variables: !2)
-!6 = !MDFile(filename: "test.cpp", directory: "/private/tmp")
-!7 = !MDSubroutineType(types: !8)
+!5 = !DISubprogram(name: "print_args", linkageName: "test", line: 4, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 5, file: !26, scope: null, type: !7, function: void (i32, i8**)* @test, variables: !2)
+!6 = !DIFile(filename: "test.cpp", directory: "/private/tmp")
+!7 = !DISubroutineType(types: !8)
 !8 = !{null, !9, !10}
-!9 = !MDBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
-!10 = !MDDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, baseType: !11)
-!11 = !MDDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, baseType: !12)
-!12 = !MDDerivedType(tag: DW_TAG_const_type, baseType: !13)
-!13 = !MDBasicType(tag: DW_TAG_base_type, name: "char", size: 8, align: 8, encoding: DW_ATE_signed_char)
-!14 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "argc", line: 4, arg: 1, scope: !5, file: !6, type: !9)
-!15 = !MDLocation(line: 4, scope: !5)
-!16 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "argv", line: 4, arg: 2, scope: !5, file: !6, type: !10)
-!17 = !MDLocalVariable(tag: DW_TAG_auto_variable, name: "i", line: 6, scope: !18, file: !6, type: !9)
-!18 = distinct !MDLexicalBlock(line: 6, column: 0, file: !26, scope: !19)
-!19 = distinct !MDLexicalBlock(line: 5, column: 0, file: !26, scope: !5)
-!20 = !MDLocation(line: 6, scope: !18)
-!21 = !MDLocation(line: 8, scope: !22)
-!22 = distinct !MDLexicalBlock(line: 7, column: 0, file: !26, scope: !18)
-!23 = !MDLocation(line: 9, scope: !22)
-!24 = !MDLocation(line: 10, scope: !19)
-!25 = !MDFile(filename: "main.cpp", directory: "/private/tmp")
-!26 = !MDFile(filename: "test.cpp", directory: "/private/tmp")
+!9 = !DIBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
+!10 = !DIDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, baseType: !11)
+!11 = !DIDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, baseType: !12)
+!12 = !DIDerivedType(tag: DW_TAG_const_type, baseType: !13)
+!13 = !DIBasicType(tag: DW_TAG_base_type, name: "char", size: 8, align: 8, encoding: DW_ATE_signed_char)
+!14 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "argc", line: 4, arg: 1, scope: !5, file: !6, type: !9)
+!15 = !DILocation(line: 4, scope: !5)
+!16 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "argv", line: 4, arg: 2, scope: !5, file: !6, type: !10)
+!17 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "i", line: 6, scope: !18, file: !6, type: !9)
+!18 = distinct !DILexicalBlock(line: 6, column: 0, file: !26, scope: !19)
+!19 = distinct !DILexicalBlock(line: 5, column: 0, file: !26, scope: !5)
+!20 = !DILocation(line: 6, scope: !18)
+!21 = !DILocation(line: 8, scope: !22)
+!22 = distinct !DILexicalBlock(line: 7, column: 0, file: !26, scope: !18)
+!23 = !DILocation(line: 9, scope: !22)
+!24 = !DILocation(line: 10, scope: !19)
+!25 = !DIFile(filename: "main.cpp", directory: "/private/tmp")
+!26 = !DIFile(filename: "test.cpp", directory: "/private/tmp")
 !27 = !{i32 1, !"Debug Info Version", i32 3}
diff --git a/test/Linker/Inputs/mdlocation.ll b/test/Linker/Inputs/mdlocation.ll
index 69353866b63..22473db4609 100644
--- a/test/Linker/Inputs/mdlocation.ll
+++ b/test/Linker/Inputs/mdlocation.ll
@@ -1,13 +1,13 @@
 !named = !{!0, !1, !2, !3, !4, !5, !6, !7, !8, !9}
 
-!0 = !MDSubprogram() ; Use this as a scope.
-!1 = !MDLocation(line: 3, column: 7, scope: !0)
-!2 = !MDLocation(line: 3, column: 7, scope: !0, inlinedAt: !1)
-!3 = !MDLocation(line: 3, column: 7, scope: !0, inlinedAt: !2)
-!4 = distinct !MDSubprogram() ; Test actual remapping.
-!5 = !MDLocation(line: 3, column: 7, scope: !4)
-!6 = !MDLocation(line: 3, column: 7, scope: !4, inlinedAt: !5)
-!7 = !MDLocation(line: 3, column: 7, scope: !4, inlinedAt: !6)
+!0 = !DISubprogram() ; Use this as a scope.
+!1 = !DILocation(line: 3, column: 7, scope: !0)
+!2 = !DILocation(line: 3, column: 7, scope: !0, inlinedAt: !1)
+!3 = !DILocation(line: 3, column: 7, scope: !0, inlinedAt: !2)
+!4 = distinct !DISubprogram() ; Test actual remapping.
+!5 = !DILocation(line: 3, column: 7, scope: !4)
+!6 = !DILocation(line: 3, column: 7, scope: !4, inlinedAt: !5)
+!7 = !DILocation(line: 3, column: 7, scope: !4, inlinedAt: !6)
 ; Test distinct nodes.
-!8 = distinct !MDLocation(line: 3, column: 7, scope: !0)
-!9 = distinct !MDLocation(line: 3, column: 7, scope: !0, inlinedAt: !8)
+!8 = distinct !DILocation(line: 3, column: 7, scope: !0)
+!9 = distinct !DILocation(line: 3, column: 7, scope: !0, inlinedAt: !8)
diff --git a/test/Linker/Inputs/metadata-function.ll b/test/Linker/Inputs/metadata-function.ll
new file mode 100644
index 00000000000..8572ff11983
--- /dev/null
+++ b/test/Linker/Inputs/metadata-function.ll
@@ -0,0 +1,13 @@
+define weak void @foo() !weak !0 {
+  unreachable
+}
+
+define void @baz() !baz !0 {
+  unreachable
+}
+
+define void @b() !b !0 {
+  unreachable
+}
+
+!0 = !{!"b"}
diff --git a/test/Linker/Inputs/override-different-linkage.ll b/test/Linker/Inputs/override-different-linkage.ll
new file mode 100644
index 00000000000..bc151409180
--- /dev/null
+++ b/test/Linker/Inputs/override-different-linkage.ll
@@ -0,0 +1,4 @@
+define linkonce i32 @foo(i32 %i) {
+entry:
+  ret i32 4
+}
diff --git a/test/Linker/Inputs/override-with-internal-linkage-2.ll b/test/Linker/Inputs/override-with-internal-linkage-2.ll
new file mode 100644
index 00000000000..2f54a1511fa
--- /dev/null
+++ b/test/Linker/Inputs/override-with-internal-linkage-2.ll
@@ -0,0 +1,4 @@
+define internal i32 @foo(i32 %i) {
+entry:
+  ret i32 4
+}
diff --git a/test/Linker/Inputs/override-with-internal-linkage.ll b/test/Linker/Inputs/override-with-internal-linkage.ll
new file mode 100644
index 00000000000..6e06fa5c4f7
--- /dev/null
+++ b/test/Linker/Inputs/override-with-internal-linkage.ll
@@ -0,0 +1,4 @@
+define i32 @foo(i32 %i) {
+entry:
+  ret i32 4
+}
diff --git a/test/Linker/Inputs/override.ll b/test/Linker/Inputs/override.ll
new file mode 100644
index 00000000000..6e06fa5c4f7
--- /dev/null
+++ b/test/Linker/Inputs/override.ll
@@ -0,0 +1,4 @@
+define i32 @foo(i32 %i) {
+entry:
+  ret i32 4
+}
diff --git a/test/Linker/Inputs/replaced-function-matches-first-subprogram.ll b/test/Linker/Inputs/replaced-function-matches-first-subprogram.ll
index f0cf1d3e4a2..efa8ec9c67b 100644
--- a/test/Linker/Inputs/replaced-function-matches-first-subprogram.ll
+++ b/test/Linker/Inputs/replaced-function-matches-first-subprogram.ll
@@ -12,16 +12,16 @@ entry:
 !llvm.module.flags = !{!8, !9, !10}
 !llvm.ident = !{!11}
 
-!0 = !MDCompileUnit(language: DW_LANG_C_plus_plus, producer: "clang version 3.6.0 (trunk 224193) (llvm/trunk 224197)", isOptimized: false, emissionKind: 2, file: !1, enums: !2, retainedTypes: !2, subprograms: !3, globals: !2, imports: !2)
-!1 = !MDFile(filename: "t2.cpp", directory: "/Users/dexonsmith/data/llvm/staging/test/Linker/repro/d2")
+!0 = !DICompileUnit(language: DW_LANG_C_plus_plus, producer: "clang version 3.6.0 (trunk 224193) (llvm/trunk 224197)", isOptimized: false, emissionKind: 2, file: !1, enums: !2, retainedTypes: !2, subprograms: !3, globals: !2, imports: !2)
+!1 = !DIFile(filename: "t2.cpp", directory: "/Users/dexonsmith/data/llvm/staging/test/Linker/repro/d2")
 !2 = !{}
 !3 = !{!4}
-!4 = !MDSubprogram(name: "foo", line: 2, isLocal: false, isDefinition: true, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 2, file: !5, scope: !6, type: !7, function: i32 (%struct.Class*)* @_ZN5ClassIiE3fooEv, variables: !2)
-!5 = !MDFile(filename: "../t.h", directory: "/Users/dexonsmith/data/llvm/staging/test/Linker/repro/d2")
-!6 = !MDFile(filename: "../t.h", directory: "/Users/dexonsmith/data/llvm/staging/test/Linker/repro/d2")
-!7 = !MDSubroutineType(types: !2)
+!4 = !DISubprogram(name: "foo", line: 2, isLocal: false, isDefinition: true, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 2, file: !5, scope: !6, type: !7, function: i32 (%struct.Class*)* @_ZN5ClassIiE3fooEv, variables: !2)
+!5 = !DIFile(filename: "../t.h", directory: "/Users/dexonsmith/data/llvm/staging/test/Linker/repro/d2")
+!6 = !DIFile(filename: "../t.h", directory: "/Users/dexonsmith/data/llvm/staging/test/Linker/repro/d2")
+!7 = !DISubroutineType(types: !2)
 !8 = !{i32 2, !"Dwarf Version", i32 2}
 !9 = !{i32 2, !"Debug Info Version", i32 3}
 !10 = !{i32 1, !"PIC Level", i32 2}
 !11 = !{!"clang version 3.6.0 (trunk 224193) (llvm/trunk 224197)"}
-!12 = !MDLocation(line: 2, column: 15, scope: !4)
+!12 = !DILocation(line: 2, column: 15, scope: !4)
diff --git a/test/Linker/Inputs/subprogram-linkonce-weak-odr.ll b/test/Linker/Inputs/subprogram-linkonce-weak-odr.ll
index b6518a81eab..71c080173a3 100644
--- a/test/Linker/Inputs/subprogram-linkonce-weak-odr.ll
+++ b/test/Linker/Inputs/subprogram-linkonce-weak-odr.ll
@@ -1,15 +1,15 @@
 define weak_odr i32 @foo(i32 %a, i32 %b) {
 entry:
-  %sum = add i32 %a, %b, !dbg !MDLocation(line: 2, scope: !3)
-  ret i32 %sum, !dbg !MDLocation(line: 3, scope: !3)
+  %sum = add i32 %a, %b, !dbg !DILocation(line: 2, scope: !3)
+  ret i32 %sum, !dbg !DILocation(line: 3, scope: !3)
 }
 
 !llvm.module.flags = !{!0}
 !0 = !{i32 2, !"Debug Info Version", i32 3}
 
 !llvm.dbg.cu = !{!1}
-!1 = !MDCompileUnit(language: DW_LANG_C99, file: !2, subprograms: !{!3}, emissionKind: 1)
-!2 = !MDFile(filename: "foo.c", directory: "/path/to/dir")
-!3 = !MDSubprogram(file: !4, scope: !4, line: 1, name: "foo", function: i32 (i32, i32)* @foo, type: !5)
-!4 = !MDFile(filename: "foo.h", directory: "/path/to/dir")
-!5 = !MDSubroutineType(types: !{})
+!1 = !DICompileUnit(language: DW_LANG_C99, file: !2, subprograms: !{!3}, emissionKind: 1)
+!2 = !DIFile(filename: "foo.c", directory: "/path/to/dir")
+!3 = !DISubprogram(file: !4, scope: !4, line: 1, name: "foo", function: i32 (i32, i32)* @foo, type: !5)
+!4 = !DIFile(filename: "foo.h", directory: "/path/to/dir")
+!5 = !DISubroutineType(types: !{})
diff --git a/test/Linker/Inputs/subprogram-linkonce-weak.ll b/test/Linker/Inputs/subprogram-linkonce-weak.ll
index e73143dae8c..f5399e244a9 100644
--- a/test/Linker/Inputs/subprogram-linkonce-weak.ll
+++ b/test/Linker/Inputs/subprogram-linkonce-weak.ll
@@ -1,7 +1,7 @@
 define weak i32 @foo(i32 %a, i32 %b) {
 entry:
-  %sum = call i32 @fastadd(i32 %a, i32 %b), !dbg !MDLocation(line: 52, scope: !3)
-  ret i32 %sum, !dbg !MDLocation(line: 53, scope: !3)
+  %sum = call i32 @fastadd(i32 %a, i32 %b), !dbg !DILocation(line: 52, scope: !3)
+  ret i32 %sum, !dbg !DILocation(line: 53, scope: !3)
 }
 
 declare i32 @fastadd(i32, i32)
@@ -10,7 +10,7 @@ declare i32 @fastadd(i32, i32)
 !0 = !{i32 2, !"Debug Info Version", i32 3}
 
 !llvm.dbg.cu = !{!1}
-!1 = !MDCompileUnit(language: DW_LANG_C99, file: !2, subprograms: !{!3}, emissionKind: 1)
-!2 = !MDFile(filename: "foo.c", directory: "/path/to/dir")
-!3 = !MDSubprogram(file: !2, scope: !2, line: 51, name: "foo", function: i32 (i32, i32)* @foo, type: !4)
-!4 = !MDSubroutineType(types: !{})
+!1 = !DICompileUnit(language: DW_LANG_C99, file: !2, subprograms: !{!3}, emissionKind: 1)
+!2 = !DIFile(filename: "foo.c", directory: "/path/to/dir")
+!3 = !DISubprogram(file: !2, scope: !2, line: 51, name: "foo", function: i32 (i32, i32)* @foo, type: !4)
+!4 = !DISubroutineType(types: !{})
diff --git a/test/Linker/Inputs/type-unique-inheritance-a.ll b/test/Linker/Inputs/type-unique-inheritance-a.ll
index 82f9d55ba31..75cc7e1c838 100644
--- a/test/Linker/Inputs/type-unique-inheritance-a.ll
+++ b/test/Linker/Inputs/type-unique-inheritance-a.ll
@@ -1,7 +1,7 @@
-; CHECK: !MDCompositeType(tag: DW_TAG_class_type, name: "A"
-; CHECK: !MDCompositeType(tag: DW_TAG_class_type, name: "Base"
-; CHECK: !MDCompositeType(tag: DW_TAG_class_type, name: "B"
-; CHECK-NOT: !MDCompositeType(tag: DW_TAG_class_type
+; CHECK: !DICompositeType(tag: DW_TAG_class_type, name: "A"
+; CHECK: !DICompositeType(tag: DW_TAG_class_type, name: "Base"
+; CHECK: !DICompositeType(tag: DW_TAG_class_type, name: "B"
+; CHECK-NOT: !DICompositeType(tag: DW_TAG_class_type
 ; Content of header files:
 ; 
 ; class Base;
@@ -52,8 +52,8 @@ entry:
   %a.addr = alloca i32, align 4
   %t = alloca %class.A, align 4
   store i32 %a, i32* %a.addr, align 4
-  call void @llvm.dbg.declare(metadata i32* %a.addr, metadata !20, metadata !MDExpression()), !dbg !21
-  call void @llvm.dbg.declare(metadata %class.A* %t, metadata !22, metadata !MDExpression()), !dbg !23
+  call void @llvm.dbg.declare(metadata i32* %a.addr, metadata !20, metadata !DIExpression()), !dbg !21
+  call void @llvm.dbg.declare(metadata %class.A* %t, metadata !22, metadata !DIExpression()), !dbg !23
   ret void, !dbg !24
 }
 
@@ -66,29 +66,29 @@ attributes #1 = { nounwind readnone }
 !llvm.dbg.cu = !{!0}
 !llvm.module.flags = !{!19, !25}
 
-!0 = !MDCompileUnit(language: DW_LANG_C_plus_plus, producer: "clang version 3.4 (http://llvm.org/git/clang.git f54e02f969d02d640103db73efc30c45439fceab) (http://llvm.org/git/llvm.git 284353b55896cb1babfaa7add7c0a363245342d2)", isOptimized: false, emissionKind: 0, file: !1, enums: !2, retainedTypes: !3, subprograms: !14, globals: !2, imports: !2)
-!1 = !MDFile(filename: "foo.cpp", directory: "/Users/mren/c_testing/type_unique_air/inher")
+!0 = !DICompileUnit(language: DW_LANG_C_plus_plus, producer: "clang version 3.4 (http://llvm.org/git/clang.git f54e02f969d02d640103db73efc30c45439fceab) (http://llvm.org/git/llvm.git 284353b55896cb1babfaa7add7c0a363245342d2)", isOptimized: false, emissionKind: 0, file: !1, enums: !2, retainedTypes: !3, subprograms: !14, globals: !2, imports: !2)
+!1 = !DIFile(filename: "foo.cpp", directory: "/Users/mren/c_testing/type_unique_air/inher")
 !2 = !{}
 !3 = !{!4, !8}
-!4 = !MDCompositeType(tag: DW_TAG_class_type, name: "A", line: 3, size: 64, align: 32, file: !5, elements: !6, identifier: "_ZTS1A")
-!5 = !MDFile(filename: "./a.hpp", directory: "/Users/mren/c_testing/type_unique_air/inher")
+!4 = !DICompositeType(tag: DW_TAG_class_type, name: "A", line: 3, size: 64, align: 32, file: !5, elements: !6, identifier: "_ZTS1A")
+!5 = !DIFile(filename: "./a.hpp", directory: "/Users/mren/c_testing/type_unique_air/inher")
 !6 = !{!7, !13}
-!7 = !MDDerivedType(tag: DW_TAG_inheritance, flags: DIFlagPrivate, scope: !"_ZTS1A", baseType: !8)
-!8 = !MDCompositeType(tag: DW_TAG_class_type, name: "Base", line: 3, size: 32, align: 32, file: !9, elements: !10, identifier: "_ZTS4Base")
-!9 = !MDFile(filename: "./b.hpp", directory: "/Users/mren/c_testing/type_unique_air/inher")
+!7 = !DIDerivedType(tag: DW_TAG_inheritance, flags: DIFlagPrivate, scope: !"_ZTS1A", baseType: !8)
+!8 = !DICompositeType(tag: DW_TAG_class_type, name: "Base", line: 3, size: 32, align: 32, file: !9, elements: !10, identifier: "_ZTS4Base")
+!9 = !DIFile(filename: "./b.hpp", directory: "/Users/mren/c_testing/type_unique_air/inher")
 !10 = !{!11}
-!11 = !MDDerivedType(tag: DW_TAG_member, name: "b", line: 4, size: 32, align: 32, flags: DIFlagPrivate, file: !9, scope: !"_ZTS4Base", baseType: !12)
-!12 = !MDBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
-!13 = !MDDerivedType(tag: DW_TAG_member, name: "x", line: 4, size: 32, align: 32, offset: 32, flags: DIFlagPrivate, file: !5, scope: !"_ZTS1A", baseType: !12)
+!11 = !DIDerivedType(tag: DW_TAG_member, name: "b", line: 4, size: 32, align: 32, flags: DIFlagPrivate, file: !9, scope: !"_ZTS4Base", baseType: !12)
+!12 = !DIBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
+!13 = !DIDerivedType(tag: DW_TAG_member, name: "x", line: 4, size: 32, align: 32, offset: 32, flags: DIFlagPrivate, file: !5, scope: !"_ZTS1A", baseType: !12)
 !14 = !{!15}
-!15 = !MDSubprogram(name: "f", linkageName: "_Z1fi", line: 5, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 5, file: !1, scope: !16, type: !17, function: void (i32)* @_Z1fi, variables: !2)
-!16 = !MDFile(filename: "foo.cpp", directory: "/Users/mren/c_testing/type_unique_air/inher")
-!17 = !MDSubroutineType(types: !18)
+!15 = !DISubprogram(name: "f", linkageName: "_Z1fi", line: 5, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 5, file: !1, scope: !16, type: !17, function: void (i32)* @_Z1fi, variables: !2)
+!16 = !DIFile(filename: "foo.cpp", directory: "/Users/mren/c_testing/type_unique_air/inher")
+!17 = !DISubroutineType(types: !18)
 !18 = !{null, !12}
 !19 = !{i32 2, !"Dwarf Version", i32 2}
-!20 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "a", line: 5, arg: 1, scope: !15, file: !16, type: !12)
-!21 = !MDLocation(line: 5, scope: !15)
-!22 = !MDLocalVariable(tag: DW_TAG_auto_variable, name: "t", line: 6, scope: !15, file: !16, type: !4)
-!23 = !MDLocation(line: 6, scope: !15)
-!24 = !MDLocation(line: 7, scope: !15)
+!20 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "a", line: 5, arg: 1, scope: !15, file: !16, type: !12)
+!21 = !DILocation(line: 5, scope: !15)
+!22 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "t", line: 6, scope: !15, file: !16, type: !4)
+!23 = !DILocation(line: 6, scope: !15)
+!24 = !DILocation(line: 7, scope: !15)
 !25 = !{i32 1, !"Debug Info Version", i32 3}
diff --git a/test/Linker/Inputs/type-unique-inheritance-b.ll b/test/Linker/Inputs/type-unique-inheritance-b.ll
index 2a27d7c8111..dd89b8dd42d 100644
--- a/test/Linker/Inputs/type-unique-inheritance-b.ll
+++ b/test/Linker/Inputs/type-unique-inheritance-b.ll
@@ -10,8 +10,8 @@ entry:
   %a.addr = alloca i32, align 4
   %t = alloca %class.B, align 8
   store i32 %a, i32* %a.addr, align 4
-  call void @llvm.dbg.declare(metadata i32* %a.addr, metadata !28, metadata !MDExpression()), !dbg !29
-  call void @llvm.dbg.declare(metadata %class.B* %t, metadata !30, metadata !MDExpression()), !dbg !31
+  call void @llvm.dbg.declare(metadata i32* %a.addr, metadata !28, metadata !DIExpression()), !dbg !29
+  call void @llvm.dbg.declare(metadata %class.B* %t, metadata !30, metadata !DIExpression()), !dbg !31
   ret void, !dbg !32
 }
 
@@ -24,7 +24,7 @@ entry:
   %retval = alloca i32, align 4
   %a = alloca %class.A, align 4
   store i32 0, i32* %retval
-  call void @llvm.dbg.declare(metadata %class.A* %a, metadata !33, metadata !MDExpression()), !dbg !34
+  call void @llvm.dbg.declare(metadata %class.A* %a, metadata !33, metadata !DIExpression()), !dbg !34
   call void @_Z1fi(i32 0), !dbg !35
   call void @_Z1gi(i32 1), !dbg !36
   ret i32 0, !dbg !37
@@ -40,42 +40,42 @@ attributes #3 = { "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "n
 !llvm.dbg.cu = !{!0}
 !llvm.module.flags = !{!27, !38}
 
-!0 = !MDCompileUnit(language: DW_LANG_C_plus_plus, producer: "clang version 3.4 (http://llvm.org/git/clang.git f54e02f969d02d640103db73efc30c45439fceab) (http://llvm.org/git/llvm.git 284353b55896cb1babfaa7add7c0a363245342d2)", isOptimized: false, emissionKind: 0, file: !1, enums: !2, retainedTypes: !3, subprograms: !19, globals: !2, imports: !2)
-!1 = !MDFile(filename: "bar.cpp", directory: "/Users/mren/c_testing/type_unique_air/inher")
+!0 = !DICompileUnit(language: DW_LANG_C_plus_plus, producer: "clang version 3.4 (http://llvm.org/git/clang.git f54e02f969d02d640103db73efc30c45439fceab) (http://llvm.org/git/llvm.git 284353b55896cb1babfaa7add7c0a363245342d2)", isOptimized: false, emissionKind: 0, file: !1, enums: !2, retainedTypes: !3, subprograms: !19, globals: !2, imports: !2)
+!1 = !DIFile(filename: "bar.cpp", directory: "/Users/mren/c_testing/type_unique_air/inher")
 !2 = !{}
 !3 = !{!4, !11, !15}
-!4 = !MDCompositeType(tag: DW_TAG_class_type, name: "B", line: 7, size: 128, align: 64, file: !5, elements: !6, identifier: "_ZTS1B")
-!5 = !MDFile(filename: "./b.hpp", directory: "/Users/mren/c_testing/type_unique_air/inher")
+!4 = !DICompositeType(tag: DW_TAG_class_type, name: "B", line: 7, size: 128, align: 64, file: !5, elements: !6, identifier: "_ZTS1B")
+!5 = !DIFile(filename: "./b.hpp", directory: "/Users/mren/c_testing/type_unique_air/inher")
 !6 = !{!7, !9}
-!7 = !MDDerivedType(tag: DW_TAG_member, name: "bb", line: 8, size: 32, align: 32, flags: DIFlagPrivate, file: !5, scope: !"_ZTS1B", baseType: !8)
-!8 = !MDBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
-!9 = !MDDerivedType(tag: DW_TAG_member, name: "a", line: 9, size: 64, align: 64, offset: 64, flags: DIFlagPrivate, file: !5, scope: !"_ZTS1B", baseType: !10)
-!10 = !MDDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, baseType: !11)
-!11 = !MDCompositeType(tag: DW_TAG_class_type, name: "A", line: 3, size: 64, align: 32, file: !12, elements: !13, identifier: "_ZTS1A")
-!12 = !MDFile(filename: "./a.hpp", directory: "/Users/mren/c_testing/type_unique_air/inher")
+!7 = !DIDerivedType(tag: DW_TAG_member, name: "bb", line: 8, size: 32, align: 32, flags: DIFlagPrivate, file: !5, scope: !"_ZTS1B", baseType: !8)
+!8 = !DIBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
+!9 = !DIDerivedType(tag: DW_TAG_member, name: "a", line: 9, size: 64, align: 64, offset: 64, flags: DIFlagPrivate, file: !5, scope: !"_ZTS1B", baseType: !10)
+!10 = !DIDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, baseType: !11)
+!11 = !DICompositeType(tag: DW_TAG_class_type, name: "A", line: 3, size: 64, align: 32, file: !12, elements: !13, identifier: "_ZTS1A")
+!12 = !DIFile(filename: "./a.hpp", directory: "/Users/mren/c_testing/type_unique_air/inher")
 !13 = !{!14, !18}
-!14 = !MDDerivedType(tag: DW_TAG_inheritance, flags: DIFlagPrivate, scope: !"_ZTS1A", baseType: !15)
-!15 = !MDCompositeType(tag: DW_TAG_class_type, name: "Base", line: 3, size: 32, align: 32, file: !5, elements: !16, identifier: "_ZTS4Base")
+!14 = !DIDerivedType(tag: DW_TAG_inheritance, flags: DIFlagPrivate, scope: !"_ZTS1A", baseType: !15)
+!15 = !DICompositeType(tag: DW_TAG_class_type, name: "Base", line: 3, size: 32, align: 32, file: !5, elements: !16, identifier: "_ZTS4Base")
 !16 = !{!17}
-!17 = !MDDerivedType(tag: DW_TAG_member, name: "b", line: 4, size: 32, align: 32, flags: DIFlagPrivate, file: !5, scope: !"_ZTS4Base", baseType: !8)
-!18 = !MDDerivedType(tag: DW_TAG_member, name: "x", line: 4, size: 32, align: 32, offset: 32, flags: DIFlagPrivate, file: !12, scope: !"_ZTS1A", baseType: !8)
+!17 = !DIDerivedType(tag: DW_TAG_member, name: "b", line: 4, size: 32, align: 32, flags: DIFlagPrivate, file: !5, scope: !"_ZTS4Base", baseType: !8)
+!18 = !DIDerivedType(tag: DW_TAG_member, name: "x", line: 4, size: 32, align: 32, offset: 32, flags: DIFlagPrivate, file: !12, scope: !"_ZTS1A", baseType: !8)
 !19 = !{!20, !24}
-!20 = !MDSubprogram(name: "g", linkageName: "_Z1gi", line: 4, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 4, file: !1, scope: !21, type: !22, function: void (i32)* @_Z1gi, variables: !2)
-!21 = !MDFile(filename: "bar.cpp", directory: "/Users/mren/c_testing/type_unique_air/inher")
-!22 = !MDSubroutineType(types: !23)
+!20 = !DISubprogram(name: "g", linkageName: "_Z1gi", line: 4, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 4, file: !1, scope: !21, type: !22, function: void (i32)* @_Z1gi, variables: !2)
+!21 = !DIFile(filename: "bar.cpp", directory: "/Users/mren/c_testing/type_unique_air/inher")
+!22 = !DISubroutineType(types: !23)
 !23 = !{null, !8}
-!24 = !MDSubprogram(name: "main", line: 9, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 9, file: !1, scope: !21, type: !25, function: i32 ()* @main, variables: !2)
-!25 = !MDSubroutineType(types: !26)
+!24 = !DISubprogram(name: "main", line: 9, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 9, file: !1, scope: !21, type: !25, function: i32 ()* @main, variables: !2)
+!25 = !DISubroutineType(types: !26)
 !26 = !{!8}
 !27 = !{i32 2, !"Dwarf Version", i32 2}
-!28 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "a", line: 4, arg: 1, scope: !20, file: !21, type: !8)
-!29 = !MDLocation(line: 4, scope: !20)
-!30 = !MDLocalVariable(tag: DW_TAG_auto_variable, name: "t", line: 5, scope: !20, file: !21, type: !4)
-!31 = !MDLocation(line: 5, scope: !20)
-!32 = !MDLocation(line: 6, scope: !20)
-!33 = !MDLocalVariable(tag: DW_TAG_auto_variable, name: "a", line: 10, scope: !24, file: !21, type: !11)
-!34 = !MDLocation(line: 10, scope: !24)
-!35 = !MDLocation(line: 11, scope: !24)
-!36 = !MDLocation(line: 12, scope: !24)
-!37 = !MDLocation(line: 13, scope: !24)
+!28 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "a", line: 4, arg: 1, scope: !20, file: !21, type: !8)
+!29 = !DILocation(line: 4, scope: !20)
+!30 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "t", line: 5, scope: !20, file: !21, type: !4)
+!31 = !DILocation(line: 5, scope: !20)
+!32 = !DILocation(line: 6, scope: !20)
+!33 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "a", line: 10, scope: !24, file: !21, type: !11)
+!34 = !DILocation(line: 10, scope: !24)
+!35 = !DILocation(line: 11, scope: !24)
+!36 = !DILocation(line: 12, scope: !24)
+!37 = !DILocation(line: 13, scope: !24)
 !38 = !{i32 1, !"Debug Info Version", i32 3}
diff --git a/test/Linker/Inputs/type-unique-simple2-a.ll b/test/Linker/Inputs/type-unique-simple2-a.ll
index 3a5695eb943..c07157bf87b 100644
--- a/test/Linker/Inputs/type-unique-simple2-a.ll
+++ b/test/Linker/Inputs/type-unique-simple2-a.ll
@@ -49,8 +49,8 @@ entry:
   %a.addr = alloca i32, align 4
   %t = alloca %struct.Base, align 8
   store i32 %a, i32* %a.addr, align 4
-  call void @llvm.dbg.declare(metadata i32* %a.addr, metadata !17, metadata !MDExpression()), !dbg !18
-  call void @llvm.dbg.declare(metadata %struct.Base* %t, metadata !19, metadata !MDExpression()), !dbg !20
+  call void @llvm.dbg.declare(metadata i32* %a.addr, metadata !17, metadata !DIExpression()), !dbg !18
+  call void @llvm.dbg.declare(metadata %struct.Base* %t, metadata !19, metadata !DIExpression()), !dbg !20
   ret void, !dbg !21
 }
 
@@ -63,26 +63,26 @@ attributes #1 = { nounwind readnone }
 !llvm.dbg.cu = !{!0}
 !llvm.module.flags = !{!16, !22}
 
-!0 = !MDCompileUnit(language: DW_LANG_C_plus_plus, producer: "clang version 3.4 (http://llvm.org/git/clang.git 8a3f9e46cb988d2c664395b21910091e3730ae82) (http://llvm.org/git/llvm.git 4699e9549358bc77824a59114548eecc3f7c523c)", isOptimized: false, emissionKind: 0, file: !1, enums: !2, retainedTypes: !3, subprograms: !11, globals: !2, imports: !2)
-!1 = !MDFile(filename: "foo.cpp", directory: ".")
+!0 = !DICompileUnit(language: DW_LANG_C_plus_plus, producer: "clang version 3.4 (http://llvm.org/git/clang.git 8a3f9e46cb988d2c664395b21910091e3730ae82) (http://llvm.org/git/llvm.git 4699e9549358bc77824a59114548eecc3f7c523c)", isOptimized: false, emissionKind: 0, file: !1, enums: !2, retainedTypes: !3, subprograms: !11, globals: !2, imports: !2)
+!1 = !DIFile(filename: "foo.cpp", directory: ".")
 !2 = !{}
 !3 = !{!4}
-!4 = !MDCompositeType(tag: DW_TAG_structure_type, name: "Base", line: 1, size: 128, align: 64, file: !5, elements: !6, identifier: "_ZTS4Base")
-!5 = !MDFile(filename: "./a.hpp", directory: ".")
+!4 = !DICompositeType(tag: DW_TAG_structure_type, name: "Base", line: 1, size: 128, align: 64, file: !5, elements: !6, identifier: "_ZTS4Base")
+!5 = !DIFile(filename: "./a.hpp", directory: ".")
 !6 = !{!7, !9}
-!7 = !MDDerivedType(tag: DW_TAG_member, name: "a", line: 2, size: 32, align: 32, file: !5, scope: !"_ZTS4Base", baseType: !8)
-!8 = !MDBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
-!9 = !MDDerivedType(tag: DW_TAG_member, name: "b", line: 3, size: 64, align: 64, offset: 64, file: !5, scope: !"_ZTS4Base", baseType: !10)
-!10 = !MDDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, baseType: !"_ZTS4Base")
+!7 = !DIDerivedType(tag: DW_TAG_member, name: "a", line: 2, size: 32, align: 32, file: !5, scope: !"_ZTS4Base", baseType: !8)
+!8 = !DIBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
+!9 = !DIDerivedType(tag: DW_TAG_member, name: "b", line: 3, size: 64, align: 64, offset: 64, file: !5, scope: !"_ZTS4Base", baseType: !10)
+!10 = !DIDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, baseType: !"_ZTS4Base")
 !11 = !{!12}
-!12 = !MDSubprogram(name: "f", linkageName: "_Z1fi", line: 3, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 3, file: !1, scope: !13, type: !14, function: void (i32)* @_Z1fi, variables: !2)
-!13 = !MDFile(filename: "foo.cpp", directory: ".")
-!14 = !MDSubroutineType(types: !15)
+!12 = !DISubprogram(name: "f", linkageName: "_Z1fi", line: 3, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 3, file: !1, scope: !13, type: !14, function: void (i32)* @_Z1fi, variables: !2)
+!13 = !DIFile(filename: "foo.cpp", directory: ".")
+!14 = !DISubroutineType(types: !15)
 !15 = !{null, !8}
 !16 = !{i32 2, !"Dwarf Version", i32 2}
-!17 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "a", line: 3, arg: 1, scope: !12, file: !13, type: !8)
-!18 = !MDLocation(line: 3, scope: !12)
-!19 = !MDLocalVariable(tag: DW_TAG_auto_variable, name: "t", line: 4, scope: !12, file: !13, type: !4)
-!20 = !MDLocation(line: 4, scope: !12)
-!21 = !MDLocation(line: 5, scope: !12)
+!17 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "a", line: 3, arg: 1, scope: !12, file: !13, type: !8)
+!18 = !DILocation(line: 3, scope: !12)
+!19 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "t", line: 4, scope: !12, file: !13, type: !4)
+!20 = !DILocation(line: 4, scope: !12)
+!21 = !DILocation(line: 5, scope: !12)
 !22 = !{i32 1, !"Debug Info Version", i32 3}
diff --git a/test/Linker/Inputs/type-unique-simple2-b.ll b/test/Linker/Inputs/type-unique-simple2-b.ll
index 0c1facad261..817f88704de 100644
--- a/test/Linker/Inputs/type-unique-simple2-b.ll
+++ b/test/Linker/Inputs/type-unique-simple2-b.ll
@@ -8,8 +8,8 @@ entry:
   %a.addr = alloca i32, align 4
   %t = alloca %struct.Base, align 8
   store i32 %a, i32* %a.addr, align 4
-  call void @llvm.dbg.declare(metadata i32* %a.addr, metadata !20, metadata !MDExpression()), !dbg !21
-  call void @llvm.dbg.declare(metadata %struct.Base* %t, metadata !22, metadata !MDExpression()), !dbg !23
+  call void @llvm.dbg.declare(metadata i32* %a.addr, metadata !20, metadata !DIExpression()), !dbg !21
+  call void @llvm.dbg.declare(metadata %struct.Base* %t, metadata !22, metadata !DIExpression()), !dbg !23
   ret void, !dbg !24
 }
 
@@ -36,32 +36,32 @@ attributes #3 = { "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "n
 !llvm.dbg.cu = !{!0}
 !llvm.module.flags = !{!19, !28}
 
-!0 = !MDCompileUnit(language: DW_LANG_C_plus_plus, producer: "clang version 3.4 (http://llvm.org/git/clang.git 8a3f9e46cb988d2c664395b21910091e3730ae82) (http://llvm.org/git/llvm.git 4699e9549358bc77824a59114548eecc3f7c523c)", isOptimized: false, emissionKind: 0, file: !1, enums: !2, retainedTypes: !3, subprograms: !11, globals: !2, imports: !2)
-!1 = !MDFile(filename: "bar.cpp", directory: ".")
+!0 = !DICompileUnit(language: DW_LANG_C_plus_plus, producer: "clang version 3.4 (http://llvm.org/git/clang.git 8a3f9e46cb988d2c664395b21910091e3730ae82) (http://llvm.org/git/llvm.git 4699e9549358bc77824a59114548eecc3f7c523c)", isOptimized: false, emissionKind: 0, file: !1, enums: !2, retainedTypes: !3, subprograms: !11, globals: !2, imports: !2)
+!1 = !DIFile(filename: "bar.cpp", directory: ".")
 !2 = !{}
 !3 = !{!4}
-!4 = !MDCompositeType(tag: DW_TAG_structure_type, name: "Base", line: 1, size: 128, align: 64, file: !5, elements: !6, identifier: "_ZTS4Base")
-!5 = !MDFile(filename: "./a.hpp", directory: ".")
+!4 = !DICompositeType(tag: DW_TAG_structure_type, name: "Base", line: 1, size: 128, align: 64, file: !5, elements: !6, identifier: "_ZTS4Base")
+!5 = !DIFile(filename: "./a.hpp", directory: ".")
 !6 = !{!7, !9}
-!7 = !MDDerivedType(tag: DW_TAG_member, name: "a", line: 2, size: 32, align: 32, file: !5, scope: !"_ZTS4Base", baseType: !8)
-!8 = !MDBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
-!9 = !MDDerivedType(tag: DW_TAG_member, name: "b", line: 3, size: 64, align: 64, offset: 64, file: !5, scope: !"_ZTS4Base", baseType: !10)
-!10 = !MDDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, baseType: !"_ZTS4Base")
+!7 = !DIDerivedType(tag: DW_TAG_member, name: "a", line: 2, size: 32, align: 32, file: !5, scope: !"_ZTS4Base", baseType: !8)
+!8 = !DIBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
+!9 = !DIDerivedType(tag: DW_TAG_member, name: "b", line: 3, size: 64, align: 64, offset: 64, file: !5, scope: !"_ZTS4Base", baseType: !10)
+!10 = !DIDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, baseType: !"_ZTS4Base")
 !11 = !{!12, !16}
-!12 = !MDSubprogram(name: "g", linkageName: "_Z1gi", line: 4, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 4, file: !1, scope: !13, type: !14, function: void (i32)* @_Z1gi, variables: !2)
-!13 = !MDFile(filename: "bar.cpp", directory: ".")
-!14 = !MDSubroutineType(types: !15)
+!12 = !DISubprogram(name: "g", linkageName: "_Z1gi", line: 4, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 4, file: !1, scope: !13, type: !14, function: void (i32)* @_Z1gi, variables: !2)
+!13 = !DIFile(filename: "bar.cpp", directory: ".")
+!14 = !DISubroutineType(types: !15)
 !15 = !{null, !8}
-!16 = !MDSubprogram(name: "main", line: 7, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 7, file: !1, scope: !13, type: !17, function: i32 ()* @main, variables: !2)
-!17 = !MDSubroutineType(types: !18)
+!16 = !DISubprogram(name: "main", line: 7, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 7, file: !1, scope: !13, type: !17, function: i32 ()* @main, variables: !2)
+!17 = !DISubroutineType(types: !18)
 !18 = !{!8}
 !19 = !{i32 2, !"Dwarf Version", i32 2}
-!20 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "a", line: 4, arg: 1, scope: !12, file: !13, type: !8)
-!21 = !MDLocation(line: 4, scope: !12)
-!22 = !MDLocalVariable(tag: DW_TAG_auto_variable, name: "t", line: 5, scope: !12, file: !13, type: !4)
-!23 = !MDLocation(line: 5, scope: !12)
-!24 = !MDLocation(line: 6, scope: !12)
-!25 = !MDLocation(line: 8, scope: !16)
-!26 = !MDLocation(line: 9, scope: !16)
-!27 = !MDLocation(line: 10, scope: !16)
+!20 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "a", line: 4, arg: 1, scope: !12, file: !13, type: !8)
+!21 = !DILocation(line: 4, scope: !12)
+!22 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "t", line: 5, scope: !12, file: !13, type: !4)
+!23 = !DILocation(line: 5, scope: !12)
+!24 = !DILocation(line: 6, scope: !12)
+!25 = !DILocation(line: 8, scope: !16)
+!26 = !DILocation(line: 9, scope: !16)
+!27 = !DILocation(line: 10, scope: !16)
 !28 = !{i32 1, !"Debug Info Version", i32 3}
diff --git a/test/Linker/comdat10.ll b/test/Linker/comdat10.ll
new file mode 100644
index 00000000000..8a32c426739
--- /dev/null
+++ b/test/Linker/comdat10.ll
@@ -0,0 +1,6 @@
+; RUN: llvm-link %s /dev/null -S -o - | FileCheck %s
+
+$c = comdat largest
+
+; CHECK: @c = global i32 0, comdat
+@c = global i32 0, comdat
diff --git a/test/Linker/debug-info-version-a.ll b/test/Linker/debug-info-version-a.ll
index 7d73e86a2bf..8cc85b167f7 100644
--- a/test/Linker/debug-info-version-a.ll
+++ b/test/Linker/debug-info-version-a.ll
@@ -3,14 +3,14 @@
 ; Test linking of incompatible debug info versions. The debug info
 ; from the other file should be dropped.
 
-; CHECK-NOT: !MDFile(filename: "b.c", directory: "")
-; CHECK:     !MDFile(filename: "a.c", directory: "")
-; CHECK-NOT: !MDFile(filename: "b.c", directory: "")
+; CHECK-NOT: !DIFile(filename: "b.c", directory: "")
+; CHECK:     !DIFile(filename: "a.c", directory: "")
+; CHECK-NOT: !DIFile(filename: "b.c", directory: "")
 
 !llvm.module.flags = !{ !0 }
 !llvm.dbg.cu = !{!1}
 
 !0 = !{i32 2, !"Debug Info Version", i32 3}
-!1 = !MDCompileUnit(language: DW_LANG_C99, producer: "clang", isOptimized: true, emissionKind: 0, file: !2, enums: !3, retainedTypes: !3, subprograms: !3)
-!2 = !MDFile(filename: "a.c", directory: "")
+!1 = !DICompileUnit(language: DW_LANG_C99, producer: "clang", isOptimized: true, emissionKind: 0, file: !2, enums: !3, retainedTypes: !3, subprograms: !3)
+!2 = !DIFile(filename: "a.c", directory: "")
 !3 = !{}
diff --git a/test/Linker/debug-info-version-b.ll b/test/Linker/debug-info-version-b.ll
index cc1c625653c..b2452f43702 100644
--- a/test/Linker/debug-info-version-b.ll
+++ b/test/Linker/debug-info-version-b.ll
@@ -5,6 +5,6 @@
 !llvm.dbg.cu = !{!1}
 
 !0 = !{i32 2, !"Debug Info Version", i32 42}
-!1 = !MDCompileUnit(language: DW_LANG_C99, producer: "clang", isOptimized: false, file: !"I AM UNEXPECTED!")
+!1 = !DICompileUnit(language: DW_LANG_C99, producer: "clang", isOptimized: false, file: !"I AM UNEXPECTED!")
 !2 = !{!"b.c", !""}
 !3 = !{}
diff --git a/test/Linker/mdlocation.ll b/test/Linker/mdlocation.ll
index 9aea4b9027d..9acc6701599 100644
--- a/test/Linker/mdlocation.ll
+++ b/test/Linker/mdlocation.ll
@@ -1,34 +1,34 @@
 ; RUN: llvm-link %s %S/Inputs/mdlocation.ll -o - -S | FileCheck %s
 
-; Test that MDLocations are remapped properly.
+; Test that DILocations are remapped properly.
 
 ; CHECK: !named = !{!0, !1, !2, !3, !4, !5, !6, !7, !8, !9, !0, !1, !2, !3, !10, !11, !12, !13, !14, !15}
 !named = !{!0, !1, !2, !3, !4, !5, !6, !7, !8, !9}
 
-; CHECK:      !0 = !MDSubprogram(
-; CHECK-NEXT: !1 = !MDLocation(line: 3, column: 7, scope: !0)
-; CHECK-NEXT: !2 = !MDLocation(line: 3, column: 7, scope: !0, inlinedAt: !1)
-; CHECK-NEXT: !3 = !MDLocation(line: 3, column: 7, scope: !0, inlinedAt: !2)
-; CHECK-NEXT: !4 = distinct !MDSubprogram(
-; CHECK-NEXT: !5 = !MDLocation(line: 3, column: 7, scope: !4)
-; CHECK-NEXT: !6 = !MDLocation(line: 3, column: 7, scope: !4, inlinedAt: !5)
-; CHECK-NEXT: !7 = !MDLocation(line: 3, column: 7, scope: !4, inlinedAt: !6)
-; CHECK-NEXT: !8 = distinct !MDLocation(line: 3, column: 7, scope: !0)
-; CHECK-NEXT: !9 = distinct !MDLocation(line: 3, column: 7, scope: !0, inlinedAt: !8)
-; CHECK-NEXT: !10 = distinct !MDSubprogram(
-; CHECK-NEXT: !11 = !MDLocation(line: 3, column: 7, scope: !10)
-; CHECK-NEXT: !12 = !MDLocation(line: 3, column: 7, scope: !10, inlinedAt: !11)
-; CHECK-NEXT: !13 = !MDLocation(line: 3, column: 7, scope: !10, inlinedAt: !12)
-; CHECK-NEXT: !14 = distinct !MDLocation(line: 3, column: 7, scope: !0)
-; CHECK-NEXT: !15 = distinct !MDLocation(line: 3, column: 7, scope: !0, inlinedAt: !14)
-!0 = !MDSubprogram() ; Use this as a scope.
-!1 = !MDLocation(line: 3, column: 7, scope: !0)
-!2 = !MDLocation(line: 3, column: 7, scope: !0, inlinedAt: !1)
-!3 = !MDLocation(line: 3, column: 7, scope: !0, inlinedAt: !2)
-!4 = distinct !MDSubprogram() ; Test actual remapping.
-!5 = !MDLocation(line: 3, column: 7, scope: !4)
-!6 = !MDLocation(line: 3, column: 7, scope: !4, inlinedAt: !5)
-!7 = !MDLocation(line: 3, column: 7, scope: !4, inlinedAt: !6)
+; CHECK:      !0 = !DISubprogram(
+; CHECK-NEXT: !1 = !DILocation(line: 3, column: 7, scope: !0)
+; CHECK-NEXT: !2 = !DILocation(line: 3, column: 7, scope: !0, inlinedAt: !1)
+; CHECK-NEXT: !3 = !DILocation(line: 3, column: 7, scope: !0, inlinedAt: !2)
+; CHECK-NEXT: !4 = distinct !DISubprogram(
+; CHECK-NEXT: !5 = !DILocation(line: 3, column: 7, scope: !4)
+; CHECK-NEXT: !6 = !DILocation(line: 3, column: 7, scope: !4, inlinedAt: !5)
+; CHECK-NEXT: !7 = !DILocation(line: 3, column: 7, scope: !4, inlinedAt: !6)
+; CHECK-NEXT: !8 = distinct !DILocation(line: 3, column: 7, scope: !0)
+; CHECK-NEXT: !9 = distinct !DILocation(line: 3, column: 7, scope: !0, inlinedAt: !8)
+; CHECK-NEXT: !10 = distinct !DISubprogram(
+; CHECK-NEXT: !11 = !DILocation(line: 3, column: 7, scope: !10)
+; CHECK-NEXT: !12 = !DILocation(line: 3, column: 7, scope: !10, inlinedAt: !11)
+; CHECK-NEXT: !13 = !DILocation(line: 3, column: 7, scope: !10, inlinedAt: !12)
+; CHECK-NEXT: !14 = distinct !DILocation(line: 3, column: 7, scope: !0)
+; CHECK-NEXT: !15 = distinct !DILocation(line: 3, column: 7, scope: !0, inlinedAt: !14)
+!0 = !DISubprogram() ; Use this as a scope.
+!1 = !DILocation(line: 3, column: 7, scope: !0)
+!2 = !DILocation(line: 3, column: 7, scope: !0, inlinedAt: !1)
+!3 = !DILocation(line: 3, column: 7, scope: !0, inlinedAt: !2)
+!4 = distinct !DISubprogram() ; Test actual remapping.
+!5 = !DILocation(line: 3, column: 7, scope: !4)
+!6 = !DILocation(line: 3, column: 7, scope: !4, inlinedAt: !5)
+!7 = !DILocation(line: 3, column: 7, scope: !4, inlinedAt: !6)
 ; Test distinct nodes.
-!8 = distinct !MDLocation(line: 3, column: 7, scope: !0)
-!9 = distinct !MDLocation(line: 3, column: 7, scope: !0, inlinedAt: !8)
+!8 = distinct !DILocation(line: 3, column: 7, scope: !0)
+!9 = distinct !DILocation(line: 3, column: 7, scope: !0, inlinedAt: !8)
diff --git a/test/Linker/metadata-function.ll b/test/Linker/metadata-function.ll
new file mode 100644
index 00000000000..0b17e0c5858
--- /dev/null
+++ b/test/Linker/metadata-function.ll
@@ -0,0 +1,26 @@
+; RUN: llvm-link %s %S/Inputs/metadata-function.ll -S | FileCheck %s
+; RUN: llvm-link %S/Inputs/metadata-function.ll %s -S | FileCheck %s
+
+; CHECK-DAG: define weak void @foo() !weak ![[B:[0-9]+]] {
+define linkonce void @foo() !linkonce !0 {
+  unreachable
+}
+
+; CHECK-DAG: define void @bar() !bar ![[A:[0-9]+]] {
+define void @bar() !bar !0 {
+  call void @baz()
+  unreachable
+}
+
+; CHECK-DAG: define void @baz() !baz ![[B]] {
+declare void @baz()
+
+; CHECK-DAG: define void @a() !a ![[A]] {
+; CHECK-DAG: define void @b() !b ![[B]] {
+define void @a() !a !0 {
+  unreachable
+}
+
+; CHECK-DAG: ![[A]] = !{!"a"}
+; CHECK-DAG: ![[B]] = !{!"b"}
+!0 = !{!"a"}
diff --git a/test/Linker/override-different-linkage.ll b/test/Linker/override-different-linkage.ll
new file mode 100644
index 00000000000..19961f2486a
--- /dev/null
+++ b/test/Linker/override-different-linkage.ll
@@ -0,0 +1,19 @@
+; RUN: llvm-link %s -override %S/Inputs/override-different-linkage.ll -S | FileCheck %s
+; RUN: llvm-link -override %S/Inputs/override-different-linkage.ll %s -S | FileCheck %s
+
+
+; CHECK-LABEL: define linkonce i32 @foo
+; CHECK-NEXT: entry:
+; CHECK-NEXT: ret i32 4
+define weak i32 @foo(i32 %i) {
+entry:
+  %add = add nsw i32 %i, %i
+  ret i32 %add
+}
+
+; Function Attrs: nounwind ssp uwtable
+define i32 @main(i32 %argc, i8** %argv) {
+entry:
+  %a = call i32 @foo(i32 2)
+  ret i32 %a
+}
diff --git a/test/Linker/override-with-internal-linkage-2.ll b/test/Linker/override-with-internal-linkage-2.ll
new file mode 100644
index 00000000000..45b131fe579
--- /dev/null
+++ b/test/Linker/override-with-internal-linkage-2.ll
@@ -0,0 +1,23 @@
+; RUN: llvm-link %s -override %S/Inputs/override-with-internal-linkage-2.ll -S | FileCheck %s
+; RUN: llvm-link -override %S/Inputs/override-with-internal-linkage-2.ll %s -S | FileCheck %s
+
+; CHECK-LABEL: define i32 @foo
+; CHECK-NEXT: entry:
+; CHECK-NEXT: %add = add nsw i32 %i, %i
+; CHECK-NEXT: ret i32 %add
+define i32 @foo(i32 %i) {
+entry:
+  %add = add nsw i32 %i, %i
+  ret i32 %add
+}
+
+; CHECK-LABEL: define internal i32 @foo.1
+; CHECK-NEXT: entry:
+; CHECK-NEXT: ret i32 4
+
+; Function Attrs: nounwind ssp uwtable
+define i32 @main(i32 %argc, i8** %argv) {
+entry:
+  %a = call i32 @foo(i32 2)
+  ret i32 %a
+}
diff --git a/test/Linker/override-with-internal-linkage.ll b/test/Linker/override-with-internal-linkage.ll
new file mode 100644
index 00000000000..d3a79479932
--- /dev/null
+++ b/test/Linker/override-with-internal-linkage.ll
@@ -0,0 +1,25 @@
+; RUN: llvm-link %s -override %S/Inputs/override-with-internal-linkage.ll -S | FileCheck %s
+; RUN: llvm-link -override %S/Inputs/override-with-internal-linkage.ll %s -S | FileCheck %s
+
+; CHECK-LABEL: define i32 @main(
+; CHECK-NEXT: entry:
+; CHECK-NEXT: call i32 @foo2(
+define i32 @main(i32 %argc, i8** %argv) {
+entry:
+  %a = call i32 @foo(i32 2)
+  ret i32 %a
+}
+
+; CHECK-LABEL: define internal i32 @foo2(
+; CHECK-NEXT: entry:
+; CHECK-NEXT: %add = add nsw i32 %i, %i
+; CHECK-NEXT: ret i32 %add
+define internal i32 @foo(i32 %i) {
+entry:
+  %add = add nsw i32 %i, %i
+  ret i32 %add
+}
+
+; CHECK-LABEL: define i32 @foo(
+; CHECK-NEXT: entry:
+; CHECK-NEXT: ret i32 4
diff --git a/test/Linker/override.ll b/test/Linker/override.ll
new file mode 100644
index 00000000000..015cb4c382a
--- /dev/null
+++ b/test/Linker/override.ll
@@ -0,0 +1,19 @@
+; RUN: llvm-link %s -override %S/Inputs/override.ll -S | FileCheck %s
+; RUN: llvm-link -override %S/Inputs/override.ll %s -S | FileCheck %s
+
+
+; CHECK-LABEL: define i32 @foo
+; CHECK-NEXT: entry:
+; CHECK-NEXT: ret i32 4
+define i32 @foo(i32 %i) {
+entry:
+  %add = add nsw i32 %i, %i
+  ret i32 %add
+}
+
+; Function Attrs: nounwind ssp uwtable
+define i32 @main(i32 %argc, i8** %argv) {
+entry:
+  %a = call i32 @foo(i32 2)
+  ret i32 %a
+}
diff --git a/test/Linker/replaced-function-matches-first-subprogram.ll b/test/Linker/replaced-function-matches-first-subprogram.ll
index ac6633d1c01..fc7e653786e 100644
--- a/test/Linker/replaced-function-matches-first-subprogram.ll
+++ b/test/Linker/replaced-function-matches-first-subprogram.ll
@@ -39,10 +39,10 @@ entry:
 !llvm.ident = !{!13}
 
 ; Extract out the list of subprograms from each compile unit.
-; CHECK-DAG: ![[CU1]] = !MDCompileUnit({{.*}} subprograms: ![[SPs1:[0-9]+]]
-; CHECK-DAG: ![[CU2]] = !MDCompileUnit({{.*}} subprograms: ![[SPs2:[0-9]+]]
-!0 = !MDCompileUnit(language: DW_LANG_C_plus_plus, producer: "clang version 3.6.0 (trunk 224193) (llvm/trunk 224197)", isOptimized: false, emissionKind: 2, file: !1, enums: !2, retainedTypes: !2, subprograms: !3, globals: !2, imports: !2)
-!1 = !MDFile(filename: "t1.cpp", directory: "/Users/dexonsmith/data/llvm/staging/test/Linker/repro/d1")
+; CHECK-DAG: ![[CU1]] = !DICompileUnit({{.*}} subprograms: ![[SPs1:[0-9]+]]
+; CHECK-DAG: ![[CU2]] = !DICompileUnit({{.*}} subprograms: ![[SPs2:[0-9]+]]
+!0 = !DICompileUnit(language: DW_LANG_C_plus_plus, producer: "clang version 3.6.0 (trunk 224193) (llvm/trunk 224197)", isOptimized: false, emissionKind: 2, file: !1, enums: !2, retainedTypes: !2, subprograms: !3, globals: !2, imports: !2)
+!1 = !DIFile(filename: "t1.cpp", directory: "/Users/dexonsmith/data/llvm/staging/test/Linker/repro/d1")
 !2 = !{}
 
 ; Extract out each compile unit's single subprogram.  The replaced subprogram's
@@ -50,30 +50,30 @@ entry:
 ; CHECK-DAG: ![[SPs1]] = !{![[SP1:[0-9]+]], ![[SP2r:[0-9]+]]}
 ; CHECK-DAG: ![[SPs2]] = !{![[SP2:[0-9]+]]}
 !3 = !{!4, !7}
-!4 = !MDSubprogram(name: "foo", line: 2, isLocal: false, isDefinition: true, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 2, file: !1, scope: !5, type: !6, function: i32 ()* @_Z3foov, variables: !2)
-!5 = !MDFile(filename: "t1.cpp", directory: "/Users/dexonsmith/data/llvm/staging/test/Linker/repro/d1")
-!6 = !MDSubroutineType(types: !2)
+!4 = !DISubprogram(name: "foo", line: 2, isLocal: false, isDefinition: true, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 2, file: !1, scope: !5, type: !6, function: i32 ()* @_Z3foov, variables: !2)
+!5 = !DIFile(filename: "t1.cpp", directory: "/Users/dexonsmith/data/llvm/staging/test/Linker/repro/d1")
+!6 = !DISubroutineType(types: !2)
 
 ; Extract out the file from the replaced subprogram.  Confirm that each
 ; subprogram is pointing at the correct function.
-; CHECK-DAG: ![[SP1]] = !MDSubprogram({{.*}} function: i32 ()* @_Z3foov
-; CHECK-DAG: ![[SP2]] = !MDSubprogram({{.*}} file: ![[FILE:[0-9]+]],{{.*}} function: i32 (%struct.Class*)* @_ZN5ClassIiE3fooEv
+; CHECK-DAG: ![[SP1]] = !DISubprogram({{.*}} function: i32 ()* @_Z3foov
+; CHECK-DAG: ![[SP2]] = !DISubprogram({{.*}} file: ![[FILE:[0-9]+]],{{.*}} function: i32 (%struct.Class*)* @_ZN5ClassIiE3fooEv
 
 ; We can't use CHECK-NOT/CHECK-SAME with a CHECK-DAG, so rely on field order to
 ; prove that there's no function: here.
-; CHECK-DAG: ![[SP2r]] = {{.*}}!MDSubprogram({{.*}} isOptimized: false, variables:
-!7 = !MDSubprogram(name: "foo", line: 2, isLocal: false, isDefinition: true, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 2, file: !8, scope: !9, type: !6, function: i32 (%struct.Class*)* @_ZN5ClassIiE3fooEv, variables: !2)
+; CHECK-DAG: ![[SP2r]] = {{.*}}!DISubprogram({{.*}} isOptimized: false, variables:
+!7 = !DISubprogram(name: "foo", line: 2, isLocal: false, isDefinition: true, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 2, file: !8, scope: !9, type: !6, function: i32 (%struct.Class*)* @_ZN5ClassIiE3fooEv, variables: !2)
 
 ; The new subprogram should be pointing at the new directory.
-; CHECK-DAG: ![[FILE]] = !MDFile(filename: "../t.h", directory: "/Users/dexonsmith/data/llvm/staging/test/Linker/repro/d2")
-!8 = !MDFile(filename: "../t.h", directory: "/Users/dexonsmith/data/llvm/staging/test/Linker/repro/d1")
-!9 = !MDFile(filename: "../t.h", directory: "/Users/dexonsmith/data/llvm/staging/test/Linker/repro/d1")
+; CHECK-DAG: ![[FILE]] = !DIFile(filename: "../t.h", directory: "/Users/dexonsmith/data/llvm/staging/test/Linker/repro/d2")
+!8 = !DIFile(filename: "../t.h", directory: "/Users/dexonsmith/data/llvm/staging/test/Linker/repro/d1")
+!9 = !DIFile(filename: "../t.h", directory: "/Users/dexonsmith/data/llvm/staging/test/Linker/repro/d1")
 !10 = !{i32 2, !"Dwarf Version", i32 2}
 !11 = !{i32 2, !"Debug Info Version", i32 3}
 !12 = !{i32 1, !"PIC Level", i32 2}
 !13 = !{!"clang version 3.6.0 (trunk 224193) (llvm/trunk 224197)"}
-!14 = !MDLocation(line: 2, column: 20, scope: !4)
+!14 = !DILocation(line: 2, column: 20, scope: !4)
 
 ; The same subprogram should be pointed to by inside the !dbg reference.
-; CHECK: ![[LOC]] = !MDLocation(line: 2, column: 15, scope: ![[SP2]])
-!15 = !MDLocation(line: 2, column: 15, scope: !7)
+; CHECK: ![[LOC]] = !DILocation(line: 2, column: 15, scope: ![[SP2]])
+!15 = !DILocation(line: 2, column: 15, scope: !7)
diff --git a/test/Linker/subprogram-linkonce-weak-odr.ll b/test/Linker/subprogram-linkonce-weak-odr.ll
index 52ce67969a0..c1fa0a5adeb 100644
--- a/test/Linker/subprogram-linkonce-weak-odr.ll
+++ b/test/Linker/subprogram-linkonce-weak-odr.ll
@@ -36,15 +36,15 @@
 
 define i32 @bar(i32 %a, i32 %b) {
 entry:
-  %sum = add i32 %a, %b, !dbg !MDLocation(line: 2, scope: !4,
-                                          inlinedAt: !MDLocation(line: 12, scope: !3))
-  ret i32 %sum, !dbg !MDLocation(line: 13, scope: !3)
+  %sum = add i32 %a, %b, !dbg !DILocation(line: 2, scope: !4,
+                                          inlinedAt: !DILocation(line: 12, scope: !3))
+  ret i32 %sum, !dbg !DILocation(line: 13, scope: !3)
 }
 
 define linkonce_odr i32 @foo(i32 %a, i32 %b) {
 entry:
-  %sum = add i32 %a, %b, !dbg !MDLocation(line: 2, scope: !4)
-  ret i32 %sum, !dbg !MDLocation(line: 3, scope: !4)
+  %sum = add i32 %a, %b, !dbg !DILocation(line: 2, scope: !4)
+  ret i32 %sum, !dbg !DILocation(line: 3, scope: !4)
 }
 
 !llvm.module.flags = !{!0}
@@ -55,45 +55,45 @@ entry:
 ; WL-SAME: !{![[WCU:[0-9]+]], ![[LCU:[0-9]+]]}
 !llvm.dbg.cu = !{!1}
 
-; LW: ![[LCU]] = !MDCompileUnit({{.*}} subprograms: ![[LSPs:[0-9]+]]
+; LW: ![[LCU]] = !DICompileUnit({{.*}} subprograms: ![[LSPs:[0-9]+]]
 ; LW: ![[LSPs]] = !{![[BARSP:[0-9]+]], ![[FOOSP:[0-9]+]]}
-; LW: ![[BARSP]] = !MDSubprogram(name: "bar",
+; LW: ![[BARSP]] = !DISubprogram(name: "bar",
 ; LW-SAME: function: i32 (i32, i32)* @bar
-; LW: ![[FOOSP]] = {{.*}}!MDSubprogram(name: "foo",
+; LW: ![[FOOSP]] = {{.*}}!DISubprogram(name: "foo",
 ; LW-NOT: function:
 ; LW-SAME: ){{$}}
-; LW: ![[WCU]] = !MDCompileUnit({{.*}} subprograms: ![[WSPs:[0-9]+]]
+; LW: ![[WCU]] = !DICompileUnit({{.*}} subprograms: ![[WSPs:[0-9]+]]
 ; LW: ![[WSPs]] = !{![[WEAKFOOSP:[0-9]+]]}
-; LW: ![[WEAKFOOSP]] = !MDSubprogram(name: "foo",
+; LW: ![[WEAKFOOSP]] = !DISubprogram(name: "foo",
 ; LW-SAME: function: i32 (i32, i32)* @foo
-; LW: ![[FOOINBAR]] = !MDLocation(line: 2, scope: ![[FOOSP]], inlinedAt: ![[BARIA:[0-9]+]])
-; LW: ![[BARIA]] = !MDLocation(line: 12, scope: ![[BARSP]])
-; LW: ![[BARRET]] = !MDLocation(line: 13, scope: ![[BARSP]])
-; LW: ![[FOOADD]] = !MDLocation(line: 2, scope: ![[WEAKFOOSP]])
-; LW: ![[FOORET]] = !MDLocation(line: 3, scope: ![[WEAKFOOSP]])
+; LW: ![[FOOINBAR]] = !DILocation(line: 2, scope: ![[FOOSP]], inlinedAt: ![[BARIA:[0-9]+]])
+; LW: ![[BARIA]] = !DILocation(line: 12, scope: ![[BARSP]])
+; LW: ![[BARRET]] = !DILocation(line: 13, scope: ![[BARSP]])
+; LW: ![[FOOADD]] = !DILocation(line: 2, scope: ![[WEAKFOOSP]])
+; LW: ![[FOORET]] = !DILocation(line: 3, scope: ![[WEAKFOOSP]])
 
 ; Same as above, but reordered.
-; WL: ![[WCU]] = !MDCompileUnit({{.*}} subprograms: ![[WSPs:[0-9]+]]
+; WL: ![[WCU]] = !DICompileUnit({{.*}} subprograms: ![[WSPs:[0-9]+]]
 ; WL: ![[WSPs]] = !{![[WEAKFOOSP:[0-9]+]]}
-; WL: ![[WEAKFOOSP]] = !MDSubprogram(name: "foo",
+; WL: ![[WEAKFOOSP]] = !DISubprogram(name: "foo",
 ; WL-SAME: function: i32 (i32, i32)* @foo
-; WL: ![[LCU]] = !MDCompileUnit({{.*}} subprograms: ![[LSPs:[0-9]+]]
+; WL: ![[LCU]] = !DICompileUnit({{.*}} subprograms: ![[LSPs:[0-9]+]]
 ; Note: for symmetry, LSPs would have a different copy of the subprogram.
 ; WL: ![[LSPs]] = !{![[BARSP:[0-9]+]], ![[WEAKFOOSP:[0-9]+]]}
-; WL: ![[BARSP]] = !MDSubprogram(name: "bar",
+; WL: ![[BARSP]] = !DISubprogram(name: "bar",
 ; WL-SAME: function: i32 (i32, i32)* @bar
-; WL: ![[FOOADD]] = !MDLocation(line: 2, scope: ![[WEAKFOOSP]])
-; WL: ![[FOORET]] = !MDLocation(line: 3, scope: ![[WEAKFOOSP]])
-; WL: ![[FOOINBAR]] = !MDLocation(line: 2, scope: ![[WEAKFOOSP]], inlinedAt: ![[BARIA:[0-9]+]])
-; WL: ![[BARIA]] = !MDLocation(line: 12, scope: ![[BARSP]])
-; WL: ![[BARRET]] = !MDLocation(line: 13, scope: ![[BARSP]])
-
-!1 = !MDCompileUnit(language: DW_LANG_C99, file: !2, subprograms: !{!3, !4}, emissionKind: 1)
-!2 = !MDFile(filename: "bar.c", directory: "/path/to/dir")
-!3 = !MDSubprogram(file: !2, scope: !2, line: 11, name: "bar", function: i32 (i32, i32)* @bar, type: !6)
-!4 = !MDSubprogram(file: !5, scope: !5, line: 1, name: "foo", function: i32 (i32, i32)* @foo, type: !6)
-!5 = !MDFile(filename: "foo.h", directory: "/path/to/dir")
-!6 = !MDSubroutineType(types: !{})
+; WL: ![[FOOADD]] = !DILocation(line: 2, scope: ![[WEAKFOOSP]])
+; WL: ![[FOORET]] = !DILocation(line: 3, scope: ![[WEAKFOOSP]])
+; WL: ![[FOOINBAR]] = !DILocation(line: 2, scope: ![[WEAKFOOSP]], inlinedAt: ![[BARIA:[0-9]+]])
+; WL: ![[BARIA]] = !DILocation(line: 12, scope: ![[BARSP]])
+; WL: ![[BARRET]] = !DILocation(line: 13, scope: ![[BARSP]])
+
+!1 = !DICompileUnit(language: DW_LANG_C99, file: !2, subprograms: !{!3, !4}, emissionKind: 1)
+!2 = !DIFile(filename: "bar.c", directory: "/path/to/dir")
+!3 = !DISubprogram(file: !2, scope: !2, line: 11, name: "bar", function: i32 (i32, i32)* @bar, type: !6)
+!4 = !DISubprogram(file: !5, scope: !5, line: 1, name: "foo", function: i32 (i32, i32)* @foo, type: !6)
+!5 = !DIFile(filename: "foo.h", directory: "/path/to/dir")
+!6 = !DISubroutineType(types: !{})
 
 ; Crasher for llc.
 ; REQUIRES: object-emission
diff --git a/test/Linker/subprogram-linkonce-weak.ll b/test/Linker/subprogram-linkonce-weak.ll
index 94d436594a5..598cea7afeb 100644
--- a/test/Linker/subprogram-linkonce-weak.ll
+++ b/test/Linker/subprogram-linkonce-weak.ll
@@ -33,15 +33,15 @@
 
 define i32 @bar(i32 %a, i32 %b) {
 entry:
-  %sum = add i32 %a, %b, !dbg !MDLocation(line: 2, scope: !4,
-                                          inlinedAt: !MDLocation(line: 12, scope: !3))
-  ret i32 %sum, !dbg !MDLocation(line: 13, scope: !3)
+  %sum = add i32 %a, %b, !dbg !DILocation(line: 2, scope: !4,
+                                          inlinedAt: !DILocation(line: 12, scope: !3))
+  ret i32 %sum, !dbg !DILocation(line: 13, scope: !3)
 }
 
 define linkonce i32 @foo(i32 %a, i32 %b) {
 entry:
-  %sum = add i32 %a, %b, !dbg !MDLocation(line: 2, scope: !4)
-  ret i32 %sum, !dbg !MDLocation(line: 3, scope: !4)
+  %sum = add i32 %a, %b, !dbg !DILocation(line: 2, scope: !4)
+  ret i32 %sum, !dbg !DILocation(line: 3, scope: !4)
 }
 
 !llvm.module.flags = !{!0}
@@ -52,46 +52,46 @@ entry:
 ; WL-SAME: !{![[WCU:[0-9]+]], ![[LCU:[0-9]+]]}
 !llvm.dbg.cu = !{!1}
 
-; LW: ![[LCU]] = !MDCompileUnit({{.*}} subprograms: ![[LSPs:[0-9]+]]
+; LW: ![[LCU]] = !DICompileUnit({{.*}} subprograms: ![[LSPs:[0-9]+]]
 ; LW: ![[LSPs]] = !{![[BARSP:[0-9]+]], ![[FOOSP:[0-9]+]]}
-; LW: ![[BARSP]] = !MDSubprogram(name: "bar",
+; LW: ![[BARSP]] = !DISubprogram(name: "bar",
 ; LW-SAME: function: i32 (i32, i32)* @bar
-; LW: ![[FOOSP]] = {{.*}}!MDSubprogram(name: "foo",
+; LW: ![[FOOSP]] = {{.*}}!DISubprogram(name: "foo",
 ; LW-NOT: function:
 ; LW-SAME: ){{$}}
-; LW: ![[WCU]] = !MDCompileUnit({{.*}} subprograms: ![[WSPs:[0-9]+]]
+; LW: ![[WCU]] = !DICompileUnit({{.*}} subprograms: ![[WSPs:[0-9]+]]
 ; LW: ![[WSPs]] = !{![[WEAKFOOSP:[0-9]+]]}
-; LW: ![[WEAKFOOSP]] = !MDSubprogram(name: "foo",
+; LW: ![[WEAKFOOSP]] = !DISubprogram(name: "foo",
 ; LW-SAME: function: i32 (i32, i32)* @foo
-; LW: ![[FOOINBAR]] = !MDLocation(line: 2, scope: ![[FOOSP]], inlinedAt: ![[BARIA:[0-9]+]])
-; LW: ![[BARIA]] = !MDLocation(line: 12, scope: ![[BARSP]])
-; LW: ![[BARRET]] = !MDLocation(line: 13, scope: ![[BARSP]])
-; LW: ![[FOOCALL]] = !MDLocation(line: 52, scope: ![[WEAKFOOSP]])
-; LW: ![[FOORET]] = !MDLocation(line: 53, scope: ![[WEAKFOOSP]])
+; LW: ![[FOOINBAR]] = !DILocation(line: 2, scope: ![[FOOSP]], inlinedAt: ![[BARIA:[0-9]+]])
+; LW: ![[BARIA]] = !DILocation(line: 12, scope: ![[BARSP]])
+; LW: ![[BARRET]] = !DILocation(line: 13, scope: ![[BARSP]])
+; LW: ![[FOOCALL]] = !DILocation(line: 52, scope: ![[WEAKFOOSP]])
+; LW: ![[FOORET]] = !DILocation(line: 53, scope: ![[WEAKFOOSP]])
 
 ; Same as above, but reordered.
-; WL: ![[WCU]] = !MDCompileUnit({{.*}} subprograms: ![[WSPs:[0-9]+]]
+; WL: ![[WCU]] = !DICompileUnit({{.*}} subprograms: ![[WSPs:[0-9]+]]
 ; WL: ![[WSPs]] = !{![[WEAKFOOSP:[0-9]+]]}
-; WL: ![[WEAKFOOSP]] = !MDSubprogram(name: "foo",
+; WL: ![[WEAKFOOSP]] = !DISubprogram(name: "foo",
 ; WL-SAME: function: i32 (i32, i32)* @foo
-; WL: ![[LCU]] = !MDCompileUnit({{.*}} subprograms: ![[LSPs:[0-9]+]]
+; WL: ![[LCU]] = !DICompileUnit({{.*}} subprograms: ![[LSPs:[0-9]+]]
 ; WL: ![[LSPs]] = !{![[BARSP:[0-9]+]], ![[FOOSP:[0-9]+]]}
-; WL: ![[BARSP]] = !MDSubprogram(name: "bar",
+; WL: ![[BARSP]] = !DISubprogram(name: "bar",
 ; WL-SAME: function: i32 (i32, i32)* @bar
-; WL: ![[FOOSP]] = {{.*}}!MDSubprogram(name: "foo",
+; WL: ![[FOOSP]] = {{.*}}!DISubprogram(name: "foo",
 ; Note, for symmetry, this should be "NOT: function:" and "SAME: ){{$}}".
 ; WL-SAME: function: i32 (i32, i32)* @foo
-; WL: ![[FOOCALL]] = !MDLocation(line: 52, scope: ![[WEAKFOOSP]])
-; WL: ![[FOORET]] = !MDLocation(line: 53, scope: ![[WEAKFOOSP]])
-; WL: ![[FOOINBAR]] = !MDLocation(line: 2, scope: ![[FOOSP]], inlinedAt: ![[BARIA:[0-9]+]])
-; WL: ![[BARIA]] = !MDLocation(line: 12, scope: ![[BARSP]])
-; WL: ![[BARRET]] = !MDLocation(line: 13, scope: ![[BARSP]])
-
-!1 = !MDCompileUnit(language: DW_LANG_C99, file: !2, subprograms: !{!3, !4}, emissionKind: 1)
-!2 = !MDFile(filename: "bar.c", directory: "/path/to/dir")
-!3 = !MDSubprogram(file: !2, scope: !2, line: 11, name: "bar", function: i32 (i32, i32)* @bar, type: !5)
-!4 = !MDSubprogram(file: !2, scope: !2, line: 1, name: "foo", function: i32 (i32, i32)* @foo, type: !5)
-!5 = !MDSubroutineType(types: !{})
+; WL: ![[FOOCALL]] = !DILocation(line: 52, scope: ![[WEAKFOOSP]])
+; WL: ![[FOORET]] = !DILocation(line: 53, scope: ![[WEAKFOOSP]])
+; WL: ![[FOOINBAR]] = !DILocation(line: 2, scope: ![[FOOSP]], inlinedAt: ![[BARIA:[0-9]+]])
+; WL: ![[BARIA]] = !DILocation(line: 12, scope: ![[BARSP]])
+; WL: ![[BARRET]] = !DILocation(line: 13, scope: ![[BARSP]])
+
+!1 = !DICompileUnit(language: DW_LANG_C99, file: !2, subprograms: !{!3, !4}, emissionKind: 1)
+!2 = !DIFile(filename: "bar.c", directory: "/path/to/dir")
+!3 = !DISubprogram(file: !2, scope: !2, line: 11, name: "bar", function: i32 (i32, i32)* @bar, type: !5)
+!4 = !DISubprogram(file: !2, scope: !2, line: 1, name: "foo", function: i32 (i32, i32)* @foo, type: !5)
+!5 = !DISubroutineType(types: !{})
 
 ; Crasher for llc.
 ; REQUIRES: object-emission
diff --git a/test/Linker/testlink.ll b/test/Linker/testlink.ll
index 5488fccca59..2e9447ddfd3 100644
--- a/test/Linker/testlink.ll
+++ b/test/Linker/testlink.ll
@@ -61,7 +61,7 @@
 @AConst = linkonce constant i32 123
 
 ; Renamed version of Intern1.
-; CHECK-DAG: @Intern1{{[0-9]+}} = internal constant i32 52
+; CHECK-DAG: @Intern1.{{[0-9]+}} = internal constant i32 52
 
 
 ; Globals linked from testlink2.
diff --git a/test/Linker/type-unique-odr-a.ll b/test/Linker/type-unique-odr-a.ll
index a44654752df..fa6b6bb4f22 100644
--- a/test/Linker/type-unique-odr-a.ll
+++ b/test/Linker/type-unique-odr-a.ll
@@ -59,7 +59,7 @@ entry:
 define internal void @_ZL3barv() #0 {
 entry:
   %a = alloca %class.A, align 4
-  call void @llvm.dbg.declare(metadata %class.A* %a, metadata !24, metadata !MDExpression()), !dbg !25
+  call void @llvm.dbg.declare(metadata %class.A* %a, metadata !24, metadata !DIExpression()), !dbg !25
   ret void, !dbg !26
 }
 
@@ -73,29 +73,29 @@ attributes #1 = { nounwind readnone }
 !llvm.module.flags = !{!20, !21}
 !llvm.ident = !{!22}
 
-!0 = !MDCompileUnit(language: DW_LANG_C_plus_plus, producer: "clang version 3.5.0 ", isOptimized: false, emissionKind: 1, file: !1, enums: !2, retainedTypes: !3, subprograms: !14, globals: !2, imports: !2)
-!1 = !MDFile(filename: "<unknown>", directory: "")
+!0 = !DICompileUnit(language: DW_LANG_C_plus_plus, producer: "clang version 3.5.0 ", isOptimized: false, emissionKind: 1, file: !1, enums: !2, retainedTypes: !3, subprograms: !14, globals: !2, imports: !2)
+!1 = !DIFile(filename: "<unknown>", directory: "")
 !2 = !{}
 !3 = !{!4}
-!4 = !MDCompositeType(tag: DW_TAG_class_type, name: "A", line: 1, size: 32, align: 32, file: !5, elements: !6, identifier: "_ZTS1A")
-!5 = !MDFile(filename: "type-unique-odr-a.cpp", directory: "")
+!4 = !DICompositeType(tag: DW_TAG_class_type, name: "A", line: 1, size: 32, align: 32, file: !5, elements: !6, identifier: "_ZTS1A")
+!5 = !DIFile(filename: "type-unique-odr-a.cpp", directory: "")
 !6 = !{!7, !9}
-!7 = !MDDerivedType(tag: DW_TAG_member, name: "data", line: 2, size: 32, align: 32, flags: DIFlagPrivate, file: !5, scope: !"_ZTS1A", baseType: !8)
-!8 = !MDBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
-!9 = !MDSubprogram(name: "getFoo", linkageName: "_ZN1A6getFooEv", line: 4, isLocal: false, isDefinition: false, virtualIndex: 6, flags: DIFlagProtected | DIFlagPrototyped, isOptimized: false, scopeLine: 4, file: !5, scope: !"_ZTS1A", type: !10)
-!10 = !MDSubroutineType(types: !11)
+!7 = !DIDerivedType(tag: DW_TAG_member, name: "data", line: 2, size: 32, align: 32, flags: DIFlagPrivate, file: !5, scope: !"_ZTS1A", baseType: !8)
+!8 = !DIBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
+!9 = !DISubprogram(name: "getFoo", linkageName: "_ZN1A6getFooEv", line: 4, isLocal: false, isDefinition: false, virtualIndex: 6, flags: DIFlagProtected | DIFlagPrototyped, isOptimized: false, scopeLine: 4, file: !5, scope: !"_ZTS1A", type: !10)
+!10 = !DISubroutineType(types: !11)
 !11 = !{null, !12}
-!12 = !MDDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, flags: DIFlagArtificial | DIFlagObjectPointer, baseType: !"_ZTS1A")
+!12 = !DIDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, flags: DIFlagArtificial | DIFlagObjectPointer, baseType: !"_ZTS1A")
 !14 = !{!15, !19}
-!15 = !MDSubprogram(name: "baz", linkageName: "_Z3bazv", line: 11, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 11, file: !5, scope: !16, type: !17, function: void ()* @_Z3bazv, variables: !2)
-!16 = !MDFile(filename: "type-unique-odr-a.cpp", directory: "")
-!17 = !MDSubroutineType(types: !18)
+!15 = !DISubprogram(name: "baz", linkageName: "_Z3bazv", line: 11, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 11, file: !5, scope: !16, type: !17, function: void ()* @_Z3bazv, variables: !2)
+!16 = !DIFile(filename: "type-unique-odr-a.cpp", directory: "")
+!17 = !DISubroutineType(types: !18)
 !18 = !{null}
-!19 = !MDSubprogram(name: "bar", linkageName: "_ZL3barv", line: 7, isLocal: true, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 7, file: !5, scope: !16, type: !17, function: void ()* @_ZL3barv, variables: !2)
+!19 = !DISubprogram(name: "bar", linkageName: "_ZL3barv", line: 7, isLocal: true, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 7, file: !5, scope: !16, type: !17, function: void ()* @_ZL3barv, variables: !2)
 !20 = !{i32 2, !"Dwarf Version", i32 4}
 !21 = !{i32 1, !"Debug Info Version", i32 3}
 !22 = !{!"clang version 3.5.0 "}
-!23 = !MDLocation(line: 11, scope: !15)
-!24 = !MDLocalVariable(tag: DW_TAG_auto_variable, name: "a", line: 8, scope: !19, file: !16, type: !"_ZTS1A")
-!25 = !MDLocation(line: 8, scope: !19)
-!26 = !MDLocation(line: 9, scope: !19)
+!23 = !DILocation(line: 11, scope: !15)
+!24 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "a", line: 8, scope: !19, file: !16, type: !"_ZTS1A")
+!25 = !DILocation(line: 8, scope: !19)
+!26 = !DILocation(line: 9, scope: !19)
diff --git a/test/Linker/type-unique-odr-b.ll b/test/Linker/type-unique-odr-b.ll
index c7a48aac39c..f57c21da824 100644
--- a/test/Linker/type-unique-odr-b.ll
+++ b/test/Linker/type-unique-odr-b.ll
@@ -26,7 +26,7 @@ define void @_ZN1A6getFooEv(%class.A* %this) #0 align 2 {
 entry:
   %this.addr = alloca %class.A*, align 8
   store %class.A* %this, %class.A** %this.addr, align 8
-  call void @llvm.dbg.declare(metadata %class.A** %this.addr, metadata !24, metadata !MDExpression()), !dbg !26
+  call void @llvm.dbg.declare(metadata %class.A** %this.addr, metadata !24, metadata !DIExpression()), !dbg !26
   %this1 = load %class.A*, %class.A** %this.addr
   ret void, !dbg !27
 }
@@ -54,32 +54,32 @@ attributes #1 = { nounwind readnone }
 !llvm.module.flags = !{!21, !22}
 !llvm.ident = !{!23}
 
-!0 = !MDCompileUnit(language: DW_LANG_C_plus_plus, producer: "clang version 3.5.0 ", isOptimized: false, emissionKind: 1, file: !1, enums: !2, retainedTypes: !3, subprograms: !14, globals: !2, imports: !2)
-!1 = !MDFile(filename: "<unknown>", directory: "")
+!0 = !DICompileUnit(language: DW_LANG_C_plus_plus, producer: "clang version 3.5.0 ", isOptimized: false, emissionKind: 1, file: !1, enums: !2, retainedTypes: !3, subprograms: !14, globals: !2, imports: !2)
+!1 = !DIFile(filename: "<unknown>", directory: "")
 !2 = !{}
 !3 = !{!4}
-!4 = !MDCompositeType(tag: DW_TAG_class_type, name: "A", line: 2, size: 32, align: 32, file: !5, elements: !6, identifier: "_ZTS1A")
-!5 = !MDFile(filename: "type-unique-odr-b.cpp", directory: "")
+!4 = !DICompositeType(tag: DW_TAG_class_type, name: "A", line: 2, size: 32, align: 32, file: !5, elements: !6, identifier: "_ZTS1A")
+!5 = !DIFile(filename: "type-unique-odr-b.cpp", directory: "")
 !6 = !{!7, !9}
-!7 = !MDDerivedType(tag: DW_TAG_member, name: "data", line: 3, size: 32, align: 32, flags: DIFlagPrivate, file: !5, scope: !"_ZTS1A", baseType: !8)
-!8 = !MDBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
-!9 = !MDSubprogram(name: "getFoo", linkageName: "_ZN1A6getFooEv", line: 5, isLocal: false, isDefinition: false, virtualIndex: 6, flags: DIFlagProtected | DIFlagPrototyped, isOptimized: false, scopeLine: 5, file: !5, scope: !"_ZTS1A", type: !10)
-!10 = !MDSubroutineType(types: !11)
+!7 = !DIDerivedType(tag: DW_TAG_member, name: "data", line: 3, size: 32, align: 32, flags: DIFlagPrivate, file: !5, scope: !"_ZTS1A", baseType: !8)
+!8 = !DIBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
+!9 = !DISubprogram(name: "getFoo", linkageName: "_ZN1A6getFooEv", line: 5, isLocal: false, isDefinition: false, virtualIndex: 6, flags: DIFlagProtected | DIFlagPrototyped, isOptimized: false, scopeLine: 5, file: !5, scope: !"_ZTS1A", type: !10)
+!10 = !DISubroutineType(types: !11)
 !11 = !{null, !12}
-!12 = !MDDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, flags: DIFlagArtificial | DIFlagObjectPointer, baseType: !"_ZTS1A")
+!12 = !DIDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, flags: DIFlagArtificial | DIFlagObjectPointer, baseType: !"_ZTS1A")
 !14 = !{!15, !16, !20}
-!15 = !MDSubprogram(name: "getFoo", linkageName: "_ZN1A6getFooEv", line: 8, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 8, file: !5, scope: !"_ZTS1A", type: !10, function: void (%class.A*)* @_ZN1A6getFooEv, declaration: !9, variables: !2)
-!16 = !MDSubprogram(name: "f", linkageName: "_Z1fv", line: 11, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 11, file: !5, scope: !17, type: !18, function: void ()* @_Z1fv, variables: !2)
-!17 = !MDFile(filename: "type-unique-odr-b.cpp", directory: "")
-!18 = !MDSubroutineType(types: !19)
+!15 = !DISubprogram(name: "getFoo", linkageName: "_ZN1A6getFooEv", line: 8, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 8, file: !5, scope: !"_ZTS1A", type: !10, function: void (%class.A*)* @_ZN1A6getFooEv, declaration: !9, variables: !2)
+!16 = !DISubprogram(name: "f", linkageName: "_Z1fv", line: 11, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 11, file: !5, scope: !17, type: !18, function: void ()* @_Z1fv, variables: !2)
+!17 = !DIFile(filename: "type-unique-odr-b.cpp", directory: "")
+!18 = !DISubroutineType(types: !19)
 !19 = !{null}
-!20 = !MDSubprogram(name: "bar", linkageName: "_ZL3barv", line: 10, isLocal: true, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 10, file: !5, scope: !17, type: !18, function: void ()* @_ZL3barv, variables: !2)
+!20 = !DISubprogram(name: "bar", linkageName: "_ZL3barv", line: 10, isLocal: true, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 10, file: !5, scope: !17, type: !18, function: void ()* @_ZL3barv, variables: !2)
 !21 = !{i32 2, !"Dwarf Version", i32 4}
 !22 = !{i32 1, !"Debug Info Version", i32 3}
 !23 = !{!"clang version 3.5.0 "}
-!24 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "this", arg: 1, flags: DIFlagArtificial | DIFlagObjectPointer, scope: !15, type: !25)
-!25 = !MDDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, baseType: !"_ZTS1A")
-!26 = !MDLocation(line: 0, scope: !15)
-!27 = !MDLocation(line: 8, scope: !15)
-!28 = !MDLocation(line: 11, scope: !16)
-!29 = !MDLocation(line: 10, scope: !20)
+!24 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "this", arg: 1, flags: DIFlagArtificial | DIFlagObjectPointer, scope: !15, type: !25)
+!25 = !DIDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, baseType: !"_ZTS1A")
+!26 = !DILocation(line: 0, scope: !15)
+!27 = !DILocation(line: 8, scope: !15)
+!28 = !DILocation(line: 11, scope: !16)
+!29 = !DILocation(line: 10, scope: !20)
diff --git a/test/Linker/type-unique-simple-a.ll b/test/Linker/type-unique-simple-a.ll
index 2ff60a12ba4..1ab43d657e1 100644
--- a/test/Linker/type-unique-simple-a.ll
+++ b/test/Linker/type-unique-simple-a.ll
@@ -54,8 +54,8 @@ entry:
   %a.addr = alloca i32, align 4
   %t = alloca %struct.Base, align 4
   store i32 %a, i32* %a.addr, align 4
-  call void @llvm.dbg.declare(metadata i32* %a.addr, metadata !15, metadata !MDExpression()), !dbg !16
-  call void @llvm.dbg.declare(metadata %struct.Base* %t, metadata !17, metadata !MDExpression()), !dbg !18
+  call void @llvm.dbg.declare(metadata i32* %a.addr, metadata !15, metadata !DIExpression()), !dbg !16
+  call void @llvm.dbg.declare(metadata %struct.Base* %t, metadata !17, metadata !DIExpression()), !dbg !18
   ret void, !dbg !19
 }
 
@@ -68,24 +68,24 @@ attributes #1 = { nounwind readnone }
 !llvm.dbg.cu = !{!0}
 !llvm.module.flags = !{!14, !20}
 
-!0 = !MDCompileUnit(language: DW_LANG_C_plus_plus, producer: "clang version 3.4 (http://llvm.org/git/clang.git c23b1db6268c8e7ce64026d57d1510c1aac200a0) (http://llvm.org/git/llvm.git 09b98fe3978eddefc2145adc1056cf21580ce945)", isOptimized: false, emissionKind: 0, file: !1, enums: !2, retainedTypes: !3, subprograms: !9, globals: !2, imports: !2)
-!1 = !MDFile(filename: "foo.cpp", directory: "/Users/mren/c_testing/type_unique_air/simple")
+!0 = !DICompileUnit(language: DW_LANG_C_plus_plus, producer: "clang version 3.4 (http://llvm.org/git/clang.git c23b1db6268c8e7ce64026d57d1510c1aac200a0) (http://llvm.org/git/llvm.git 09b98fe3978eddefc2145adc1056cf21580ce945)", isOptimized: false, emissionKind: 0, file: !1, enums: !2, retainedTypes: !3, subprograms: !9, globals: !2, imports: !2)
+!1 = !DIFile(filename: "foo.cpp", directory: "/Users/mren/c_testing/type_unique_air/simple")
 !2 = !{}
 !3 = !{!4}
-!4 = !MDCompositeType(tag: DW_TAG_structure_type, name: "Base", line: 1, size: 32, align: 32, file: !5, elements: !6, identifier: "_ZTS4Base")
-!5 = !MDFile(filename: "./a.hpp", directory: "/Users/mren/c_testing/type_unique_air/simple")
+!4 = !DICompositeType(tag: DW_TAG_structure_type, name: "Base", line: 1, size: 32, align: 32, file: !5, elements: !6, identifier: "_ZTS4Base")
+!5 = !DIFile(filename: "./a.hpp", directory: "/Users/mren/c_testing/type_unique_air/simple")
 !6 = !{!7}
-!7 = !MDDerivedType(tag: DW_TAG_member, name: "a", line: 2, size: 32, align: 32, file: !5, scope: !"_ZTS4Base", baseType: !8)
-!8 = !MDBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
+!7 = !DIDerivedType(tag: DW_TAG_member, name: "a", line: 2, size: 32, align: 32, file: !5, scope: !"_ZTS4Base", baseType: !8)
+!8 = !DIBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
 !9 = !{!10}
-!10 = !MDSubprogram(name: "f", linkageName: "_Z1fi", line: 3, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 3, file: !1, scope: !11, type: !12, function: void (i32)* @_Z1fi, variables: !2)
-!11 = !MDFile(filename: "foo.cpp", directory: "/Users/mren/c_testing/type_unique_air/simple")
-!12 = !MDSubroutineType(types: !13)
+!10 = !DISubprogram(name: "f", linkageName: "_Z1fi", line: 3, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 3, file: !1, scope: !11, type: !12, function: void (i32)* @_Z1fi, variables: !2)
+!11 = !DIFile(filename: "foo.cpp", directory: "/Users/mren/c_testing/type_unique_air/simple")
+!12 = !DISubroutineType(types: !13)
 !13 = !{null, !8}
 !14 = !{i32 2, !"Dwarf Version", i32 2}
-!15 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "a", line: 3, arg: 1, scope: !10, file: !11, type: !8)
-!16 = !MDLocation(line: 3, scope: !10)
-!17 = !MDLocalVariable(tag: DW_TAG_auto_variable, name: "t", line: 4, scope: !10, file: !11, type: !4)
-!18 = !MDLocation(line: 4, scope: !10)
-!19 = !MDLocation(line: 5, scope: !10)
+!15 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "a", line: 3, arg: 1, scope: !10, file: !11, type: !8)
+!16 = !DILocation(line: 3, scope: !10)
+!17 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "t", line: 4, scope: !10, file: !11, type: !4)
+!18 = !DILocation(line: 4, scope: !10)
+!19 = !DILocation(line: 5, scope: !10)
 !20 = !{i32 1, !"Debug Info Version", i32 3}
diff --git a/test/Linker/type-unique-simple-b.ll b/test/Linker/type-unique-simple-b.ll
index 4cbfc8a5441..c12b9184529 100644
--- a/test/Linker/type-unique-simple-b.ll
+++ b/test/Linker/type-unique-simple-b.ll
@@ -10,8 +10,8 @@ entry:
   %a.addr = alloca i32, align 4
   %t = alloca %struct.Base, align 4
   store i32 %a, i32* %a.addr, align 4
-  call void @llvm.dbg.declare(metadata i32* %a.addr, metadata !18, metadata !MDExpression()), !dbg !19
-  call void @llvm.dbg.declare(metadata %struct.Base* %t, metadata !20, metadata !MDExpression()), !dbg !21
+  call void @llvm.dbg.declare(metadata i32* %a.addr, metadata !18, metadata !DIExpression()), !dbg !19
+  call void @llvm.dbg.declare(metadata %struct.Base* %t, metadata !20, metadata !DIExpression()), !dbg !21
   ret void, !dbg !22
 }
 
@@ -38,30 +38,30 @@ attributes #3 = { "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "n
 !llvm.dbg.cu = !{!0}
 !llvm.module.flags = !{!17, !26}
 
-!0 = !MDCompileUnit(language: DW_LANG_C_plus_plus, producer: "clang version 3.4 (http://llvm.org/git/clang.git c23b1db6268c8e7ce64026d57d1510c1aac200a0) (http://llvm.org/git/llvm.git 09b98fe3978eddefc2145adc1056cf21580ce945)", isOptimized: false, emissionKind: 0, file: !1, enums: !2, retainedTypes: !3, subprograms: !9, globals: !2, imports: !2)
-!1 = !MDFile(filename: "bar.cpp", directory: "/Users/mren/c_testing/type_unique_air/simple")
+!0 = !DICompileUnit(language: DW_LANG_C_plus_plus, producer: "clang version 3.4 (http://llvm.org/git/clang.git c23b1db6268c8e7ce64026d57d1510c1aac200a0) (http://llvm.org/git/llvm.git 09b98fe3978eddefc2145adc1056cf21580ce945)", isOptimized: false, emissionKind: 0, file: !1, enums: !2, retainedTypes: !3, subprograms: !9, globals: !2, imports: !2)
+!1 = !DIFile(filename: "bar.cpp", directory: "/Users/mren/c_testing/type_unique_air/simple")
 !2 = !{}
 !3 = !{!4}
-!4 = !MDCompositeType(tag: DW_TAG_structure_type, name: "Base", line: 1, size: 32, align: 32, file: !5, elements: !6, identifier: "_ZTS4Base")
-!5 = !MDFile(filename: "./a.hpp", directory: "/Users/mren/c_testing/type_unique_air/simple")
+!4 = !DICompositeType(tag: DW_TAG_structure_type, name: "Base", line: 1, size: 32, align: 32, file: !5, elements: !6, identifier: "_ZTS4Base")
+!5 = !DIFile(filename: "./a.hpp", directory: "/Users/mren/c_testing/type_unique_air/simple")
 !6 = !{!7}
-!7 = !MDDerivedType(tag: DW_TAG_member, name: "a", line: 2, size: 32, align: 32, file: !5, scope: !"_ZTS4Base", baseType: !8)
-!8 = !MDBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
+!7 = !DIDerivedType(tag: DW_TAG_member, name: "a", line: 2, size: 32, align: 32, file: !5, scope: !"_ZTS4Base", baseType: !8)
+!8 = !DIBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
 !9 = !{!10, !14}
-!10 = !MDSubprogram(name: "g", linkageName: "_Z1gi", line: 4, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 4, file: !1, scope: !11, type: !12, function: void (i32)* @_Z1gi, variables: !2)
-!11 = !MDFile(filename: "bar.cpp", directory: "/Users/mren/c_testing/type_unique_air/simple")
-!12 = !MDSubroutineType(types: !13)
+!10 = !DISubprogram(name: "g", linkageName: "_Z1gi", line: 4, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 4, file: !1, scope: !11, type: !12, function: void (i32)* @_Z1gi, variables: !2)
+!11 = !DIFile(filename: "bar.cpp", directory: "/Users/mren/c_testing/type_unique_air/simple")
+!12 = !DISubroutineType(types: !13)
 !13 = !{null, !8}
-!14 = !MDSubprogram(name: "main", line: 7, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 7, file: !1, scope: !11, type: !15, function: i32 ()* @main, variables: !2)
-!15 = !MDSubroutineType(types: !16)
+!14 = !DISubprogram(name: "main", line: 7, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 7, file: !1, scope: !11, type: !15, function: i32 ()* @main, variables: !2)
+!15 = !DISubroutineType(types: !16)
 !16 = !{!8}
 !17 = !{i32 2, !"Dwarf Version", i32 2}
-!18 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "a", line: 4, arg: 1, scope: !10, file: !11, type: !8)
-!19 = !MDLocation(line: 4, scope: !10)
-!20 = !MDLocalVariable(tag: DW_TAG_auto_variable, name: "t", line: 5, scope: !10, file: !11, type: !4)
-!21 = !MDLocation(line: 5, scope: !10)
-!22 = !MDLocation(line: 6, scope: !10)
-!23 = !MDLocation(line: 8, scope: !14)
-!24 = !MDLocation(line: 9, scope: !14)
-!25 = !MDLocation(line: 10, scope: !14)
+!18 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "a", line: 4, arg: 1, scope: !10, file: !11, type: !8)
+!19 = !DILocation(line: 4, scope: !10)
+!20 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "t", line: 5, scope: !10, file: !11, type: !4)
+!21 = !DILocation(line: 5, scope: !10)
+!22 = !DILocation(line: 6, scope: !10)
+!23 = !DILocation(line: 8, scope: !14)
+!24 = !DILocation(line: 9, scope: !14)
+!25 = !DILocation(line: 10, scope: !14)
 !26 = !{i32 1, !"Debug Info Version", i32 3}
diff --git a/test/Linker/type-unique-simple2-a.ll b/test/Linker/type-unique-simple2-a.ll
index 8a1cc204b8d..f9170ab5f7f 100644
--- a/test/Linker/type-unique-simple2-a.ll
+++ b/test/Linker/type-unique-simple2-a.ll
@@ -48,7 +48,7 @@ define linkonce_odr void @_ZN1AC1Ev(%class.A* %this) unnamed_addr #2 align 2 {
 entry:
   %this.addr = alloca %class.A*, align 8
   store %class.A* %this, %class.A** %this.addr, align 8
-  call void @llvm.dbg.declare(metadata %class.A** %this.addr, metadata !39, metadata !MDExpression()), !dbg !41
+  call void @llvm.dbg.declare(metadata %class.A** %this.addr, metadata !39, metadata !DIExpression()), !dbg !41
   %this1 = load %class.A*, %class.A** %this.addr
   call void @_ZN1AC2Ev(%class.A* %this1) #1, !dbg !42
   ret void, !dbg !42
@@ -64,7 +64,7 @@ define linkonce_odr void @_ZN1AC2Ev(%class.A* %this) unnamed_addr #2 align 2 {
 entry:
   %this.addr = alloca %class.A*, align 8
   store %class.A* %this, %class.A** %this.addr, align 8
-  call void @llvm.dbg.declare(metadata %class.A** %this.addr, metadata !44, metadata !MDExpression()), !dbg !45
+  call void @llvm.dbg.declare(metadata %class.A** %this.addr, metadata !44, metadata !DIExpression()), !dbg !45
   %this1 = load %class.A*, %class.A** %this.addr
   %0 = bitcast %class.A* %this1 to i8***, !dbg !46
   store i8** getelementptr inbounds ([4 x i8*], [4 x i8*]* @_ZTV1A, i64 0, i64 2), i8*** %0, !dbg !46
@@ -80,47 +80,47 @@ attributes #4 = { nounwind readnone }
 !llvm.module.flags = !{!35, !36}
 !llvm.ident = !{!37}
 
-!0 = !MDCompileUnit(language: DW_LANG_C_plus_plus, producer: "clang version 3.5 ", isOptimized: false, emissionKind: 0, file: !1, enums: !2, retainedTypes: !3, subprograms: !26, globals: !2, imports: !2)
-!1 = !MDFile(filename: "<unknown>", directory: "")
+!0 = !DICompileUnit(language: DW_LANG_C_plus_plus, producer: "clang version 3.5 ", isOptimized: false, emissionKind: 0, file: !1, enums: !2, retainedTypes: !3, subprograms: !26, globals: !2, imports: !2)
+!1 = !DIFile(filename: "<unknown>", directory: "")
 !2 = !{}
 !3 = !{!4}
-!4 = !MDCompositeType(tag: DW_TAG_class_type, name: "A", line: 2, size: 64, align: 64, file: !5, elements: !6, vtableHolder: !"_ZTS1A", identifier: "_ZTS1A")
-!5 = !MDFile(filename: "./ab.h", directory: "")
+!4 = !DICompositeType(tag: DW_TAG_class_type, name: "A", line: 2, size: 64, align: 64, file: !5, elements: !6, vtableHolder: !"_ZTS1A", identifier: "_ZTS1A")
+!5 = !DIFile(filename: "./ab.h", directory: "")
 !6 = !{!7, !14, !19}
-!7 = !MDDerivedType(tag: DW_TAG_member, name: "_vptr$A", size: 64, flags: DIFlagArtificial, file: !5, scope: !8, baseType: !9)
-!8 = !MDFile(filename: "./ab.h", directory: "")
-!9 = !MDDerivedType(tag: DW_TAG_pointer_type, size: 64, baseType: !10)
-!10 = !MDDerivedType(tag: DW_TAG_pointer_type, name: "__vtbl_ptr_type", size: 64, baseType: !11)
-!11 = !MDSubroutineType(types: !12)
+!7 = !DIDerivedType(tag: DW_TAG_member, name: "_vptr$A", size: 64, flags: DIFlagArtificial, file: !5, scope: !8, baseType: !9)
+!8 = !DIFile(filename: "./ab.h", directory: "")
+!9 = !DIDerivedType(tag: DW_TAG_pointer_type, size: 64, baseType: !10)
+!10 = !DIDerivedType(tag: DW_TAG_pointer_type, name: "__vtbl_ptr_type", size: 64, baseType: !11)
+!11 = !DISubroutineType(types: !12)
 !12 = !{!13}
-!13 = !MDBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
-!14 = !MDSubprogram(name: "setFoo", linkageName: "_ZN1A6setFooEv", line: 4, isLocal: false, isDefinition: false, virtuality: DW_VIRTUALITY_virtual, virtualIndex: 6, flags: DIFlagPublic | DIFlagPrototyped, isOptimized: false, scopeLine: 4, file: !5, scope: !"_ZTS1A", type: !15, containingType: !"_ZTS1A")
-!15 = !MDSubroutineType(types: !16)
+!13 = !DIBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
+!14 = !DISubprogram(name: "setFoo", linkageName: "_ZN1A6setFooEv", line: 4, isLocal: false, isDefinition: false, virtuality: DW_VIRTUALITY_virtual, virtualIndex: 6, flags: DIFlagPublic | DIFlagPrototyped, isOptimized: false, scopeLine: 4, file: !5, scope: !"_ZTS1A", type: !15, containingType: !"_ZTS1A")
+!15 = !DISubroutineType(types: !16)
 !16 = !{null, !17}
-!17 = !MDDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, flags: DIFlagArtificial | DIFlagObjectPointer, baseType: !"_ZTS1A")
-!19 = !MDSubprogram(name: "getFoo", linkageName: "_ZN1A6getFooEv", line: 5, isLocal: false, isDefinition: false, virtuality: DW_VIRTUALITY_virtual, virtualIndex: 6, flags: DIFlagPublic | DIFlagPrototyped, isOptimized: false, scopeLine: 5, file: !5, scope: !"_ZTS1A", type: !20, containingType: !"_ZTS1A")
-!20 = !MDSubroutineType(types: !21)
+!17 = !DIDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, flags: DIFlagArtificial | DIFlagObjectPointer, baseType: !"_ZTS1A")
+!19 = !DISubprogram(name: "getFoo", linkageName: "_ZN1A6getFooEv", line: 5, isLocal: false, isDefinition: false, virtuality: DW_VIRTUALITY_virtual, virtualIndex: 6, flags: DIFlagPublic | DIFlagPrototyped, isOptimized: false, scopeLine: 5, file: !5, scope: !"_ZTS1A", type: !20, containingType: !"_ZTS1A")
+!20 = !DISubroutineType(types: !21)
 !21 = !{!22, !17}
-!22 = !MDDerivedType(tag: DW_TAG_const_type, baseType: !23)
-!23 = !MDDerivedType(tag: DW_TAG_typedef, name: "foo_t", line: 1, file: !24, baseType: !13)
-!24 = !MDFile(filename: "a.cpp", directory: "")
+!22 = !DIDerivedType(tag: DW_TAG_const_type, baseType: !23)
+!23 = !DIDerivedType(tag: DW_TAG_typedef, name: "foo_t", line: 1, file: !24, baseType: !13)
+!24 = !DIFile(filename: "a.cpp", directory: "")
 !26 = !{!27, !31, !34}
-!27 = !MDSubprogram(name: "bar", linkageName: "_Z3barv", line: 2, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 2, file: !24, scope: !28, type: !29, function: i32 ()* @_Z3barv, variables: !2)
-!28 = !MDFile(filename: "a.cpp", directory: "")
-!29 = !MDSubroutineType(types: !30)
+!27 = !DISubprogram(name: "bar", linkageName: "_Z3barv", line: 2, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 2, file: !24, scope: !28, type: !29, function: i32 ()* @_Z3barv, variables: !2)
+!28 = !DIFile(filename: "a.cpp", directory: "")
+!29 = !DISubroutineType(types: !30)
 !30 = !{!23}
-!31 = !MDSubprogram(name: "A", linkageName: "_ZN1AC1Ev", line: 2, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagArtificial | DIFlagPrototyped, isOptimized: false, scopeLine: 2, file: !5, scope: !"_ZTS1A", type: !15, function: void (%class.A*)* @_ZN1AC1Ev, declaration: !32, variables: !2)
-!32 = !MDSubprogram(name: "A", isLocal: false, isDefinition: false, virtualIndex: 6, flags: DIFlagArtificial | DIFlagPrototyped, isOptimized: false, scope: !"_ZTS1A", type: !15)
-!34 = !MDSubprogram(name: "A", linkageName: "_ZN1AC2Ev", line: 2, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagArtificial | DIFlagPrototyped, isOptimized: false, scopeLine: 2, file: !5, scope: !"_ZTS1A", type: !15, function: void (%class.A*)* @_ZN1AC2Ev, declaration: !32, variables: !2)
+!31 = !DISubprogram(name: "A", linkageName: "_ZN1AC1Ev", line: 2, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagArtificial | DIFlagPrototyped, isOptimized: false, scopeLine: 2, file: !5, scope: !"_ZTS1A", type: !15, function: void (%class.A*)* @_ZN1AC1Ev, declaration: !32, variables: !2)
+!32 = !DISubprogram(name: "A", isLocal: false, isDefinition: false, virtualIndex: 6, flags: DIFlagArtificial | DIFlagPrototyped, isOptimized: false, scope: !"_ZTS1A", type: !15)
+!34 = !DISubprogram(name: "A", linkageName: "_ZN1AC2Ev", line: 2, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagArtificial | DIFlagPrototyped, isOptimized: false, scopeLine: 2, file: !5, scope: !"_ZTS1A", type: !15, function: void (%class.A*)* @_ZN1AC2Ev, declaration: !32, variables: !2)
 !35 = !{i32 2, !"Dwarf Version", i32 2}
 !36 = !{i32 1, !"Debug Info Version", i32 3}
 !37 = !{!"clang version 3.5 "}
-!38 = !MDLocation(line: 3, scope: !27)
-!39 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "this", arg: 1, flags: DIFlagArtificial | DIFlagObjectPointer, scope: !31, type: !40)
-!40 = !MDDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, baseType: !"_ZTS1A")
-!41 = !MDLocation(line: 0, scope: !31)
-!42 = !MDLocation(line: 2, scope: !43)
-!43 = !MDLexicalBlockFile(discriminator: 0, file: !5, scope: !31)
-!44 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "this", arg: 1, flags: DIFlagArtificial | DIFlagObjectPointer, scope: !34, type: !40)
-!45 = !MDLocation(line: 0, scope: !34)
-!46 = !MDLocation(line: 2, scope: !34)
+!38 = !DILocation(line: 3, scope: !27)
+!39 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "this", arg: 1, flags: DIFlagArtificial | DIFlagObjectPointer, scope: !31, type: !40)
+!40 = !DIDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, baseType: !"_ZTS1A")
+!41 = !DILocation(line: 0, scope: !31)
+!42 = !DILocation(line: 2, scope: !43)
+!43 = !DILexicalBlockFile(discriminator: 0, file: !5, scope: !31)
+!44 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "this", arg: 1, flags: DIFlagArtificial | DIFlagObjectPointer, scope: !34, type: !40)
+!45 = !DILocation(line: 0, scope: !34)
+!46 = !DILocation(line: 2, scope: !34)
diff --git a/test/Linker/type-unique-simple2-b.ll b/test/Linker/type-unique-simple2-b.ll
index 1679405412c..5539fb4adb3 100644
--- a/test/Linker/type-unique-simple2-b.ll
+++ b/test/Linker/type-unique-simple2-b.ll
@@ -22,7 +22,7 @@ define void @_ZN1A6setFooEv(%class.A* %this) unnamed_addr #0 align 2 {
 entry:
   %this.addr = alloca %class.A*, align 8
   store %class.A* %this, %class.A** %this.addr, align 8
-  call void @llvm.dbg.declare(metadata %class.A** %this.addr, metadata !32, metadata !MDExpression()), !dbg !34
+  call void @llvm.dbg.declare(metadata %class.A** %this.addr, metadata !32, metadata !DIExpression()), !dbg !34
   %this1 = load %class.A*, %class.A** %this.addr
   ret void, !dbg !35
 }
@@ -35,7 +35,7 @@ define i32 @_ZN1A6getFooEv(%class.A* %this) unnamed_addr #0 align 2 {
 entry:
   %this.addr = alloca %class.A*, align 8
   store %class.A* %this, %class.A** %this.addr, align 8
-  call void @llvm.dbg.declare(metadata %class.A** %this.addr, metadata !36, metadata !MDExpression()), !dbg !37
+  call void @llvm.dbg.declare(metadata %class.A** %this.addr, metadata !36, metadata !DIExpression()), !dbg !37
   %this1 = load %class.A*, %class.A** %this.addr
   ret i32 1, !dbg !38
 }
@@ -47,40 +47,40 @@ attributes #1 = { nounwind readnone }
 !llvm.module.flags = !{!29, !30}
 !llvm.ident = !{!31}
 
-!0 = !MDCompileUnit(language: DW_LANG_C_plus_plus, producer: "clang version 3.5 ", isOptimized: false, emissionKind: 0, file: !1, enums: !2, retainedTypes: !3, subprograms: !25, globals: !2, imports: !2)
-!1 = !MDFile(filename: "<unknown>", directory: "")
+!0 = !DICompileUnit(language: DW_LANG_C_plus_plus, producer: "clang version 3.5 ", isOptimized: false, emissionKind: 0, file: !1, enums: !2, retainedTypes: !3, subprograms: !25, globals: !2, imports: !2)
+!1 = !DIFile(filename: "<unknown>", directory: "")
 !2 = !{}
 !3 = !{!4}
-!4 = !MDCompositeType(tag: DW_TAG_class_type, name: "A", line: 2, size: 64, align: 64, file: !5, elements: !6, vtableHolder: !"_ZTS1A", identifier: "_ZTS1A")
-!5 = !MDFile(filename: "./ab.h", directory: "")
+!4 = !DICompositeType(tag: DW_TAG_class_type, name: "A", line: 2, size: 64, align: 64, file: !5, elements: !6, vtableHolder: !"_ZTS1A", identifier: "_ZTS1A")
+!5 = !DIFile(filename: "./ab.h", directory: "")
 !6 = !{!7, !14, !19}
-!7 = !MDDerivedType(tag: DW_TAG_member, name: "_vptr$A", size: 64, flags: DIFlagArtificial, file: !5, scope: !8, baseType: !9)
-!8 = !MDFile(filename: "./ab.h", directory: "")
-!9 = !MDDerivedType(tag: DW_TAG_pointer_type, size: 64, baseType: !10)
-!10 = !MDDerivedType(tag: DW_TAG_pointer_type, name: "__vtbl_ptr_type", size: 64, baseType: !11)
-!11 = !MDSubroutineType(types: !12)
+!7 = !DIDerivedType(tag: DW_TAG_member, name: "_vptr$A", size: 64, flags: DIFlagArtificial, file: !5, scope: !8, baseType: !9)
+!8 = !DIFile(filename: "./ab.h", directory: "")
+!9 = !DIDerivedType(tag: DW_TAG_pointer_type, size: 64, baseType: !10)
+!10 = !DIDerivedType(tag: DW_TAG_pointer_type, name: "__vtbl_ptr_type", size: 64, baseType: !11)
+!11 = !DISubroutineType(types: !12)
 !12 = !{!13}
-!13 = !MDBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
-!14 = !MDSubprogram(name: "setFoo", linkageName: "_ZN1A6setFooEv", line: 4, isLocal: false, isDefinition: false, virtuality: DW_VIRTUALITY_virtual, virtualIndex: 6, flags: DIFlagPublic | DIFlagPrototyped, isOptimized: false, scopeLine: 4, file: !5, scope: !"_ZTS1A", type: !15, containingType: !"_ZTS1A")
-!15 = !MDSubroutineType(types: !16)
+!13 = !DIBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
+!14 = !DISubprogram(name: "setFoo", linkageName: "_ZN1A6setFooEv", line: 4, isLocal: false, isDefinition: false, virtuality: DW_VIRTUALITY_virtual, virtualIndex: 6, flags: DIFlagPublic | DIFlagPrototyped, isOptimized: false, scopeLine: 4, file: !5, scope: !"_ZTS1A", type: !15, containingType: !"_ZTS1A")
+!15 = !DISubroutineType(types: !16)
 !16 = !{null, !17}
-!17 = !MDDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, flags: DIFlagArtificial | DIFlagObjectPointer, baseType: !"_ZTS1A")
-!19 = !MDSubprogram(name: "getFoo", linkageName: "_ZN1A6getFooEv", line: 5, isLocal: false, isDefinition: false, virtuality: DW_VIRTUALITY_virtual, virtualIndex: 6, flags: DIFlagPublic | DIFlagPrototyped, isOptimized: false, scopeLine: 5, file: !5, scope: !"_ZTS1A", type: !20, containingType: !"_ZTS1A")
-!20 = !MDSubroutineType(types: !21)
+!17 = !DIDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, flags: DIFlagArtificial | DIFlagObjectPointer, baseType: !"_ZTS1A")
+!19 = !DISubprogram(name: "getFoo", linkageName: "_ZN1A6getFooEv", line: 5, isLocal: false, isDefinition: false, virtuality: DW_VIRTUALITY_virtual, virtualIndex: 6, flags: DIFlagPublic | DIFlagPrototyped, isOptimized: false, scopeLine: 5, file: !5, scope: !"_ZTS1A", type: !20, containingType: !"_ZTS1A")
+!20 = !DISubroutineType(types: !21)
 !21 = !{!22, !17}
-!22 = !MDDerivedType(tag: DW_TAG_const_type, baseType: !23)
-!23 = !MDDerivedType(tag: DW_TAG_typedef, name: "foo_t", line: 1, file: !5, baseType: !13)
+!22 = !DIDerivedType(tag: DW_TAG_const_type, baseType: !23)
+!23 = !DIDerivedType(tag: DW_TAG_typedef, name: "foo_t", line: 1, file: !5, baseType: !13)
 !25 = !{!26, !28}
-!26 = !MDSubprogram(name: "setFoo", linkageName: "_ZN1A6setFooEv", line: 2, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPublic | DIFlagPrototyped, isOptimized: false, scopeLine: 2, file: !27, scope: !"_ZTS1A", type: !15, function: void (%class.A*)* @_ZN1A6setFooEv, declaration: !14, variables: !2)
-!27 = !MDFile(filename: "b.cpp", directory: "")
-!28 = !MDSubprogram(name: "getFoo", linkageName: "_ZN1A6getFooEv", line: 4, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPublic | DIFlagPrototyped, isOptimized: false, scopeLine: 4, file: !27, scope: !"_ZTS1A", type: !20, function: i32 (%class.A*)* @_ZN1A6getFooEv, declaration: !19, variables: !2)
+!26 = !DISubprogram(name: "setFoo", linkageName: "_ZN1A6setFooEv", line: 2, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPublic | DIFlagPrototyped, isOptimized: false, scopeLine: 2, file: !27, scope: !"_ZTS1A", type: !15, function: void (%class.A*)* @_ZN1A6setFooEv, declaration: !14, variables: !2)
+!27 = !DIFile(filename: "b.cpp", directory: "")
+!28 = !DISubprogram(name: "getFoo", linkageName: "_ZN1A6getFooEv", line: 4, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPublic | DIFlagPrototyped, isOptimized: false, scopeLine: 4, file: !27, scope: !"_ZTS1A", type: !20, function: i32 (%class.A*)* @_ZN1A6getFooEv, declaration: !19, variables: !2)
 !29 = !{i32 2, !"Dwarf Version", i32 2}
 !30 = !{i32 1, !"Debug Info Version", i32 3}
 !31 = !{!"clang version 3.5 "}
-!32 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "this", arg: 1, flags: DIFlagArtificial | DIFlagObjectPointer, scope: !26, type: !33)
-!33 = !MDDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, baseType: !"_ZTS1A")
-!34 = !MDLocation(line: 0, scope: !26)
-!35 = !MDLocation(line: 2, scope: !26)
-!36 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "this", arg: 1, flags: DIFlagArtificial | DIFlagObjectPointer, scope: !28, type: !33)
-!37 = !MDLocation(line: 0, scope: !28)
-!38 = !MDLocation(line: 4, scope: !28)
+!32 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "this", arg: 1, flags: DIFlagArtificial | DIFlagObjectPointer, scope: !26, type: !33)
+!33 = !DIDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, baseType: !"_ZTS1A")
+!34 = !DILocation(line: 0, scope: !26)
+!35 = !DILocation(line: 2, scope: !26)
+!36 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "this", arg: 1, flags: DIFlagArtificial | DIFlagObjectPointer, scope: !28, type: !33)
+!37 = !DILocation(line: 0, scope: !28)
+!38 = !DILocation(line: 4, scope: !28)
diff --git a/test/Linker/type-unique-type-array-a.ll b/test/Linker/type-unique-type-array-a.ll
index a2b1316cd80..98c8d657743 100644
--- a/test/Linker/type-unique-type-array-a.ll
+++ b/test/Linker/type-unique-type-array-a.ll
@@ -51,8 +51,8 @@ entry:
   %coerce.dive = getelementptr %struct.SA, %struct.SA* %sa, i32 0, i32 0
   store i32 %sa.coerce, i32* %coerce.dive
   store %class.A* %a, %class.A** %a.addr, align 8
-  call void @llvm.dbg.declare(metadata %class.A** %a.addr, metadata !24, metadata !MDExpression()), !dbg !25
-  call void @llvm.dbg.declare(metadata %struct.SA* %sa, metadata !26, metadata !MDExpression()), !dbg !27
+  call void @llvm.dbg.declare(metadata %class.A** %a.addr, metadata !24, metadata !DIExpression()), !dbg !25
+  call void @llvm.dbg.declare(metadata %struct.SA* %sa, metadata !26, metadata !DIExpression()), !dbg !27
   %0 = load %class.A*, %class.A** %a.addr, align 8, !dbg !28
   %1 = bitcast %struct.SA* %agg.tmp to i8*, !dbg !28
   %2 = bitcast %struct.SA* %sa to i8*, !dbg !28
@@ -74,8 +74,8 @@ entry:
   %coerce.dive = getelementptr %struct.SA, %struct.SA* %a, i32 0, i32 0
   store i32 %a.coerce, i32* %coerce.dive
   store %class.A* %this, %class.A** %this.addr, align 8
-  call void @llvm.dbg.declare(metadata %class.A** %this.addr, metadata !30, metadata !MDExpression()), !dbg !31
-  call void @llvm.dbg.declare(metadata %struct.SA* %a, metadata !32, metadata !MDExpression()), !dbg !33
+  call void @llvm.dbg.declare(metadata %class.A** %this.addr, metadata !30, metadata !DIExpression()), !dbg !31
+  call void @llvm.dbg.declare(metadata %struct.SA* %a, metadata !32, metadata !DIExpression()), !dbg !33
   %this1 = load %class.A*, %class.A** %this.addr
   ret void, !dbg !34
 }
@@ -92,38 +92,38 @@ attributes #3 = { nounwind }
 !llvm.module.flags = !{!21, !22}
 !llvm.ident = !{!23}
 
-!0 = !MDCompileUnit(language: DW_LANG_C_plus_plus, producer: "clang version 3.5.0 (trunk 214102:214113M) (llvm/trunk 214102:214115M)", isOptimized: false, emissionKind: 1, file: !1, enums: !2, retainedTypes: !3, subprograms: !14, globals: !2, imports: !2)
-!1 = !MDFile(filename: "a.cpp", directory: "/Users/manmanren/test-Nov/type_unique/rdar_di_array")
+!0 = !DICompileUnit(language: DW_LANG_C_plus_plus, producer: "clang version 3.5.0 (trunk 214102:214113M) (llvm/trunk 214102:214115M)", isOptimized: false, emissionKind: 1, file: !1, enums: !2, retainedTypes: !3, subprograms: !14, globals: !2, imports: !2)
+!1 = !DIFile(filename: "a.cpp", directory: "/Users/manmanren/test-Nov/type_unique/rdar_di_array")
 !2 = !{}
 !3 = !{!4, !10}
-!4 = !MDCompositeType(tag: DW_TAG_class_type, name: "A", line: 5, size: 8, align: 8, file: !1, elements: !5, identifier: "_ZTS1A")
+!4 = !DICompositeType(tag: DW_TAG_class_type, name: "A", line: 5, size: 8, align: 8, file: !1, elements: !5, identifier: "_ZTS1A")
 !5 = !{!6}
-!6 = !MDSubprogram(name: "testA", linkageName: "_ZN1A5testAE2SA", line: 7, isLocal: false, isDefinition: false, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 7, file: !1, scope: !"_ZTS1A", type: !7)
-!7 = !MDSubroutineType(types: !8)
+!6 = !DISubprogram(name: "testA", linkageName: "_ZN1A5testAE2SA", line: 7, isLocal: false, isDefinition: false, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 7, file: !1, scope: !"_ZTS1A", type: !7)
+!7 = !DISubroutineType(types: !8)
 !8 = !{null, !9, !"_ZTS2SA"}
-!9 = !MDDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, flags: DIFlagArtificial | DIFlagObjectPointer, baseType: !"_ZTS1A")
-!10 = !MDCompositeType(tag: DW_TAG_structure_type, name: "SA", line: 1, size: 32, align: 32, file: !1, elements: !11, identifier: "_ZTS2SA")
+!9 = !DIDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, flags: DIFlagArtificial | DIFlagObjectPointer, baseType: !"_ZTS1A")
+!10 = !DICompositeType(tag: DW_TAG_structure_type, name: "SA", line: 1, size: 32, align: 32, file: !1, elements: !11, identifier: "_ZTS2SA")
 !11 = !{!12}
-!12 = !MDDerivedType(tag: DW_TAG_member, name: "a", line: 2, size: 32, align: 32, file: !1, scope: !"_ZTS2SA", baseType: !13)
-!13 = !MDBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
+!12 = !DIDerivedType(tag: DW_TAG_member, name: "a", line: 2, size: 32, align: 32, file: !1, scope: !"_ZTS2SA", baseType: !13)
+!13 = !DIBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
 !14 = !{!15, !20}
-!15 = !MDSubprogram(name: "topA", linkageName: "_Z4topAP1A2SA", line: 11, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 11, file: !1, scope: !16, type: !17, function: void (%class.A*, i32)* @_Z4topAP1A2SA, variables: !2)
-!16 = !MDFile(filename: "a.cpp", directory: "/Users/manmanren/test-Nov/type_unique/rdar_di_array")
-!17 = !MDSubroutineType(types: !18)
+!15 = !DISubprogram(name: "topA", linkageName: "_Z4topAP1A2SA", line: 11, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 11, file: !1, scope: !16, type: !17, function: void (%class.A*, i32)* @_Z4topAP1A2SA, variables: !2)
+!16 = !DIFile(filename: "a.cpp", directory: "/Users/manmanren/test-Nov/type_unique/rdar_di_array")
+!17 = !DISubroutineType(types: !18)
 !18 = !{null, !19, !"_ZTS2SA"}
-!19 = !MDDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, baseType: !"_ZTS1A")
-!20 = !MDSubprogram(name: "testA", linkageName: "_ZN1A5testAE2SA", line: 7, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 7, file: !1, scope: !"_ZTS1A", type: !7, function: void (%class.A*, i32)* @_ZN1A5testAE2SA, declaration: !6, variables: !2)
+!19 = !DIDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, baseType: !"_ZTS1A")
+!20 = !DISubprogram(name: "testA", linkageName: "_ZN1A5testAE2SA", line: 7, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 7, file: !1, scope: !"_ZTS1A", type: !7, function: void (%class.A*, i32)* @_ZN1A5testAE2SA, declaration: !6, variables: !2)
 !21 = !{i32 2, !"Dwarf Version", i32 2}
 !22 = !{i32 2, !"Debug Info Version", i32 3}
 !23 = !{!"clang version 3.5.0 (trunk 214102:214113M) (llvm/trunk 214102:214115M)"}
-!24 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "a", line: 11, arg: 1, scope: !15, file: !16, type: !19)
-!25 = !MDLocation(line: 11, column: 14, scope: !15)
-!26 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "sa", line: 11, arg: 2, scope: !15, file: !16, type: !"_ZTS2SA")
-!27 = !MDLocation(line: 11, column: 20, scope: !15)
-!28 = !MDLocation(line: 12, column: 3, scope: !15)
-!29 = !MDLocation(line: 13, column: 1, scope: !15)
-!30 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "this", arg: 1, flags: DIFlagArtificial | DIFlagObjectPointer, scope: !20, type: !19)
-!31 = !MDLocation(line: 0, scope: !20)
-!32 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "a", line: 7, arg: 2, scope: !20, file: !16, type: !"_ZTS2SA")
-!33 = !MDLocation(line: 7, column: 17, scope: !20)
-!34 = !MDLocation(line: 8, column: 3, scope: !20)
+!24 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "a", line: 11, arg: 1, scope: !15, file: !16, type: !19)
+!25 = !DILocation(line: 11, column: 14, scope: !15)
+!26 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "sa", line: 11, arg: 2, scope: !15, file: !16, type: !"_ZTS2SA")
+!27 = !DILocation(line: 11, column: 20, scope: !15)
+!28 = !DILocation(line: 12, column: 3, scope: !15)
+!29 = !DILocation(line: 13, column: 1, scope: !15)
+!30 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "this", arg: 1, flags: DIFlagArtificial | DIFlagObjectPointer, scope: !20, type: !19)
+!31 = !DILocation(line: 0, scope: !20)
+!32 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "a", line: 7, arg: 2, scope: !20, file: !16, type: !"_ZTS2SA")
+!33 = !DILocation(line: 7, column: 17, scope: !20)
+!34 = !DILocation(line: 8, column: 3, scope: !20)
diff --git a/test/Linker/type-unique-type-array-b.ll b/test/Linker/type-unique-type-array-b.ll
index 11b0a20d1fc..14ce59b3c12 100644
--- a/test/Linker/type-unique-type-array-b.ll
+++ b/test/Linker/type-unique-type-array-b.ll
@@ -30,8 +30,8 @@ entry:
   %coerce.dive = getelementptr %struct.SA, %struct.SA* %sa, i32 0, i32 0
   store i32 %sa.coerce, i32* %coerce.dive
   store %class.B* %b, %class.B** %b.addr, align 8
-  call void @llvm.dbg.declare(metadata %class.B** %b.addr, metadata !24, metadata !MDExpression()), !dbg !25
-  call void @llvm.dbg.declare(metadata %struct.SA* %sa, metadata !26, metadata !MDExpression()), !dbg !27
+  call void @llvm.dbg.declare(metadata %class.B** %b.addr, metadata !24, metadata !DIExpression()), !dbg !25
+  call void @llvm.dbg.declare(metadata %struct.SA* %sa, metadata !26, metadata !DIExpression()), !dbg !27
   %0 = load %class.B*, %class.B** %b.addr, align 8, !dbg !28
   %1 = bitcast %struct.SA* %agg.tmp to i8*, !dbg !28
   %2 = bitcast %struct.SA* %sa to i8*, !dbg !28
@@ -53,8 +53,8 @@ entry:
   %coerce.dive = getelementptr %struct.SA, %struct.SA* %sa, i32 0, i32 0
   store i32 %sa.coerce, i32* %coerce.dive
   store %class.B* %this, %class.B** %this.addr, align 8
-  call void @llvm.dbg.declare(metadata %class.B** %this.addr, metadata !30, metadata !MDExpression()), !dbg !31
-  call void @llvm.dbg.declare(metadata %struct.SA* %sa, metadata !32, metadata !MDExpression()), !dbg !33
+  call void @llvm.dbg.declare(metadata %class.B** %this.addr, metadata !30, metadata !DIExpression()), !dbg !31
+  call void @llvm.dbg.declare(metadata %struct.SA* %sa, metadata !32, metadata !DIExpression()), !dbg !33
   %this1 = load %class.B*, %class.B** %this.addr
   ret void, !dbg !34
 }
@@ -71,38 +71,38 @@ attributes #3 = { nounwind }
 !llvm.module.flags = !{!21, !22}
 !llvm.ident = !{!23}
 
-!0 = !MDCompileUnit(language: DW_LANG_C_plus_plus, producer: "clang version 3.5.0 (trunk 214102:214113M) (llvm/trunk 214102:214115M)", isOptimized: false, emissionKind: 1, file: !1, enums: !2, retainedTypes: !3, subprograms: !14, globals: !2, imports: !2)
-!1 = !MDFile(filename: "b.cpp", directory: "/Users/manmanren/test-Nov/type_unique/rdar_di_array")
+!0 = !DICompileUnit(language: DW_LANG_C_plus_plus, producer: "clang version 3.5.0 (trunk 214102:214113M) (llvm/trunk 214102:214115M)", isOptimized: false, emissionKind: 1, file: !1, enums: !2, retainedTypes: !3, subprograms: !14, globals: !2, imports: !2)
+!1 = !DIFile(filename: "b.cpp", directory: "/Users/manmanren/test-Nov/type_unique/rdar_di_array")
 !2 = !{}
 !3 = !{!4, !10}
-!4 = !MDCompositeType(tag: DW_TAG_class_type, name: "B", line: 5, size: 8, align: 8, file: !1, elements: !5, identifier: "_ZTS1B")
+!4 = !DICompositeType(tag: DW_TAG_class_type, name: "B", line: 5, size: 8, align: 8, file: !1, elements: !5, identifier: "_ZTS1B")
 !5 = !{!6}
-!6 = !MDSubprogram(name: "testB", linkageName: "_ZN1B5testBE2SA", line: 7, isLocal: false, isDefinition: false, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 7, file: !1, scope: !"_ZTS1B", type: !7)
-!7 = !MDSubroutineType(types: !8)
+!6 = !DISubprogram(name: "testB", linkageName: "_ZN1B5testBE2SA", line: 7, isLocal: false, isDefinition: false, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 7, file: !1, scope: !"_ZTS1B", type: !7)
+!7 = !DISubroutineType(types: !8)
 !8 = !{null, !9, !"_ZTS2SA"}
-!9 = !MDDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, flags: DIFlagArtificial | DIFlagObjectPointer, baseType: !"_ZTS1B")
-!10 = !MDCompositeType(tag: DW_TAG_structure_type, name: "SA", line: 1, size: 32, align: 32, file: !1, elements: !11, identifier: "_ZTS2SA")
+!9 = !DIDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, flags: DIFlagArtificial | DIFlagObjectPointer, baseType: !"_ZTS1B")
+!10 = !DICompositeType(tag: DW_TAG_structure_type, name: "SA", line: 1, size: 32, align: 32, file: !1, elements: !11, identifier: "_ZTS2SA")
 !11 = !{!12}
-!12 = !MDDerivedType(tag: DW_TAG_member, name: "a", line: 2, size: 32, align: 32, file: !1, scope: !"_ZTS2SA", baseType: !13)
-!13 = !MDBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
+!12 = !DIDerivedType(tag: DW_TAG_member, name: "a", line: 2, size: 32, align: 32, file: !1, scope: !"_ZTS2SA", baseType: !13)
+!13 = !DIBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
 !14 = !{!15, !20}
-!15 = !MDSubprogram(name: "topB", linkageName: "_Z4topBP1B2SA", line: 11, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 11, file: !1, scope: !16, type: !17, function: void (%class.B*, i32)* @_Z4topBP1B2SA, variables: !2)
-!16 = !MDFile(filename: "b.cpp", directory: "/Users/manmanren/test-Nov/type_unique/rdar_di_array")
-!17 = !MDSubroutineType(types: !18)
+!15 = !DISubprogram(name: "topB", linkageName: "_Z4topBP1B2SA", line: 11, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 11, file: !1, scope: !16, type: !17, function: void (%class.B*, i32)* @_Z4topBP1B2SA, variables: !2)
+!16 = !DIFile(filename: "b.cpp", directory: "/Users/manmanren/test-Nov/type_unique/rdar_di_array")
+!17 = !DISubroutineType(types: !18)
 !18 = !{null, !19, !"_ZTS2SA"}
-!19 = !MDDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, baseType: !"_ZTS1B")
-!20 = !MDSubprogram(name: "testB", linkageName: "_ZN1B5testBE2SA", line: 7, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 7, file: !1, scope: !"_ZTS1B", type: !7, function: void (%class.B*, i32)* @_ZN1B5testBE2SA, declaration: !6, variables: !2)
+!19 = !DIDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, baseType: !"_ZTS1B")
+!20 = !DISubprogram(name: "testB", linkageName: "_ZN1B5testBE2SA", line: 7, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 7, file: !1, scope: !"_ZTS1B", type: !7, function: void (%class.B*, i32)* @_ZN1B5testBE2SA, declaration: !6, variables: !2)
 !21 = !{i32 2, !"Dwarf Version", i32 2}
 !22 = !{i32 2, !"Debug Info Version", i32 3}
 !23 = !{!"clang version 3.5.0 (trunk 214102:214113M) (llvm/trunk 214102:214115M)"}
-!24 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "b", line: 11, arg: 1, scope: !15, file: !16, type: !19)
-!25 = !MDLocation(line: 11, column: 14, scope: !15)
-!26 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "sa", line: 11, arg: 2, scope: !15, file: !16, type: !"_ZTS2SA")
-!27 = !MDLocation(line: 11, column: 20, scope: !15)
-!28 = !MDLocation(line: 12, column: 3, scope: !15)
-!29 = !MDLocation(line: 13, column: 1, scope: !15)
-!30 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "this", arg: 1, flags: DIFlagArtificial | DIFlagObjectPointer, scope: !20, type: !19)
-!31 = !MDLocation(line: 0, scope: !20)
-!32 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "sa", line: 7, arg: 2, scope: !20, file: !16, type: !"_ZTS2SA")
-!33 = !MDLocation(line: 7, column: 17, scope: !20)
-!34 = !MDLocation(line: 8, column: 3, scope: !20)
+!24 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "b", line: 11, arg: 1, scope: !15, file: !16, type: !19)
+!25 = !DILocation(line: 11, column: 14, scope: !15)
+!26 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "sa", line: 11, arg: 2, scope: !15, file: !16, type: !"_ZTS2SA")
+!27 = !DILocation(line: 11, column: 20, scope: !15)
+!28 = !DILocation(line: 12, column: 3, scope: !15)
+!29 = !DILocation(line: 13, column: 1, scope: !15)
+!30 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "this", arg: 1, flags: DIFlagArtificial | DIFlagObjectPointer, scope: !20, type: !19)
+!31 = !DILocation(line: 0, scope: !20)
+!32 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "sa", line: 7, arg: 2, scope: !20, file: !16, type: !"_ZTS2SA")
+!33 = !DILocation(line: 7, column: 17, scope: !20)
+!34 = !DILocation(line: 8, column: 3, scope: !20)
diff --git a/test/MC/AArch64/alias-addsubimm.s b/test/MC/AArch64/alias-addsubimm.s
new file mode 100644
index 00000000000..75e0a185572
--- /dev/null
+++ b/test/MC/AArch64/alias-addsubimm.s
@@ -0,0 +1,94 @@
+// RUN: llvm-mc -triple=aarch64-none-linux-gnu < %s | FileCheck %s
+
+// CHECK: sub w0, w2, #2, lsl #12
+// CHECK: sub w0, w2, #2, lsl #12
+        sub w0, w2, #2, lsl 12
+        add w0, w2, #-2, lsl 12
+// CHECK: sub x1, x3, #2, lsl #12
+// CHECK: sub x1, x3, #2, lsl #12
+        sub x1, x3, #2, lsl 12
+        add x1, x3, #-2, lsl 12
+// CHECK: sub x1, x3, #4
+// CHECK: sub x1, x3, #4
+        sub x1, x3, #4
+        add x1, x3, #-4
+// CHECK: sub x1, x3, #4095
+// CHECK: sub x1, x3, #4095
+        sub x1, x3, #4095, lsl 0
+        add x1, x3, #-4095, lsl 0
+// CHECK: sub x3, x4, #0
+        sub x3, x4, #0
+
+// CHECK: add w0, w2, #2, lsl #12
+// CHECK: add w0, w2, #2, lsl #12
+        add w0, w2, #2, lsl 12
+        sub w0, w2, #-2, lsl 12
+// CHECK: add x1, x3, #2, lsl #12
+// CHECK: add x1, x3, #2, lsl #12
+        add x1, x3, #2, lsl 12
+        sub x1, x3, #-2, lsl 12
+// CHECK: add x1, x3, #4
+// CHECK: add x1, x3, #4
+        add x1, x3, #4
+        sub x1, x3, #-4
+// CHECK: add x1, x3, #4095
+// CHECK: add x1, x3, #4095
+        add x1, x3, #4095, lsl 0
+        sub x1, x3, #-4095, lsl 0
+// CHECK: add x2, x5, #0
+        add x2, x5, #0
+
+// CHECK: subs w0, w2, #2, lsl #12
+// CHECK: subs w0, w2, #2, lsl #12
+        subs w0, w2, #2, lsl 12
+        adds w0, w2, #-2, lsl 12
+// CHECK: subs x1, x3, #2, lsl #12
+// CHECK: subs x1, x3, #2, lsl #12
+        subs x1, x3, #2, lsl 12
+        adds x1, x3, #-2, lsl 12
+// CHECK: subs x1, x3, #4
+// CHECK: subs x1, x3, #4
+        subs x1, x3, #4
+        adds x1, x3, #-4
+// CHECK: subs x1, x3, #4095
+// CHECK: subs x1, x3, #4095
+        subs x1, x3, #4095, lsl 0
+        adds x1, x3, #-4095, lsl 0
+// CHECK: subs x3, x4, #0
+        subs x3, x4, #0
+
+// CHECK: adds w0, w2, #2, lsl #12
+// CHECK: adds w0, w2, #2, lsl #12
+        adds w0, w2, #2, lsl 12
+        subs w0, w2, #-2, lsl 12
+// CHECK: adds x1, x3, #2, lsl #12
+// CHECK: adds x1, x3, #2, lsl #12
+        adds x1, x3, #2, lsl 12
+        subs x1, x3, #-2, lsl 12
+// CHECK: adds x1, x3, #4
+// CHECK: adds x1, x3, #4
+        adds x1, x3, #4
+        subs x1, x3, #-4
+// CHECK: adds x1, x3, #4095
+// CHECK: adds x1, x3, #4095
+        adds x1, x3, #4095, lsl 0
+        subs x1, x3, #-4095, lsl 0
+// CHECK: adds x2, x5, #0
+        adds x2, x5, #0
+
+// CHECK: {{adds xzr,|cmn}} x5, #5
+// CHECK: {{adds xzr,|cmn}} x5, #5
+        cmn x5, #5
+        cmp x5, #-5
+// CHECK: {{subs xzr,|cmp}} x6, #4095
+// CHECK: {{subs xzr,|cmp}} x6, #4095
+        cmp x6, #4095
+        cmn x6, #-4095
+// CHECK: {{adds wzr,|cmn}} w7, #5
+// CHECK: {{adds wzr,|cmn}} w7, #5
+        cmn w7, #5
+        cmp w7, #-5
+// CHECK: {{subs wzr,|cmp}} w8, #4095
+// CHECK: {{subs wzr,|cmp}} w8, #4095
+        cmp w8, #4095
+        cmn w8, #-4095
diff --git a/test/MC/AArch64/arm64-leaf-compact-unwind.s b/test/MC/AArch64/arm64-leaf-compact-unwind.s
index 27d3d51c293..a0703f6360d 100644
--- a/test/MC/AArch64/arm64-leaf-compact-unwind.s
+++ b/test/MC/AArch64/arm64-leaf-compact-unwind.s
@@ -1,5 +1,5 @@
 // RUN: llvm-mc -triple=arm64-apple-ios -filetype=obj < %s | \
-// RUN: llvm-readobj -sections -section-relocations -section-data | \
+// RUN: llvm-readobj --expand-relocs -sections -section-relocations -section-data | \
 // RUN: FileCheck %s
 //
 // rdar://13070556
@@ -23,10 +23,34 @@
 // CHECK-NEXT:   Reserved1:
 // CHECK-NEXT:   Reserved2:
 // CHECK-NEXT:   Relocations [
-// CHECK-NEXT:     0x60 0 3 0 ARM64_RELOC_UNSIGNED 0 0x1
-// CHECK-NEXT:     0x40 0 3 0 ARM64_RELOC_UNSIGNED 0 0x1
-// CHECK-NEXT:     0x20 0 3 0 ARM64_RELOC_UNSIGNED 0 0x1
-// CHECK-NEXT:     0x0 0 3 0 ARM64_RELOC_UNSIGNED 0 0x1
+// CHECK-NEXT:     Relocation {
+// CHECK-NEXT:       Offset: 0x60
+// CHECK-NEXT:       PCRel: 0
+// CHECK-NEXT:       Length: 3
+// CHECK-NEXT:       Type: ARM64_RELOC_UNSIGNED (0)
+// CHECK-NEXT:       Section: __text (1)
+// CHECK-NEXT:     }
+// CHECK-NEXT:     Relocation {
+// CHECK-NEXT:       Offset: 0x40
+// CHECK-NEXT:       PCRel: 0
+// CHECK-NEXT:       Length: 3
+// CHECK-NEXT:       Type: ARM64_RELOC_UNSIGNED (0)
+// CHECK-NEXT:       Section: __text (1)
+// CHECK-NEXT:     }
+// CHECK-NEXT:     Relocation {
+// CHECK-NEXT:       Offset: 0x20
+// CHECK-NEXT:       PCRel: 0
+// CHECK-NEXT:       Length: 3
+// CHECK-NEXT:       Type: ARM64_RELOC_UNSIGNED (0)
+// CHECK-NEXT:       Section: __text (1)
+// CHECK-NEXT:     }
+// CHECK-NEXT:     Relocation {
+// CHECK-NEXT:       Offset: 0x0
+// CHECK-NEXT:       PCRel: 0
+// CHECK-NEXT:       Length: 3
+// CHECK-NEXT:       Type: ARM64_RELOC_UNSIGNED (0)
+// CHECK-NEXT:       Section: __text (1)
+// CHECK-NEXT:     }
 // CHECK-NEXT:   ]
 // CHECK-NEXT:   SectionData (
 // CHECK-NEXT:     0000: 00000000 00000000 08000000 00000002
diff --git a/test/MC/AArch64/arm64-small-data-fixups.s b/test/MC/AArch64/arm64-small-data-fixups.s
index 3fe7c75c011..6debe0b8fb0 100644
--- a/test/MC/AArch64/arm64-small-data-fixups.s
+++ b/test/MC/AArch64/arm64-small-data-fixups.s
@@ -1,4 +1,4 @@
-; RUN: llvm-mc -triple arm64-apple-darwin -filetype=obj -o - %s | macho-dump | FileCheck %s
+; RUN: llvm-mc -triple arm64-apple-darwin -filetype=obj -o - %s | llvm-readobj -r | FileCheck %s
 
 foo:
   .long 0
@@ -9,16 +9,15 @@ baz:
   .byte foo - bar
   .short foo - bar
 
-; CHECK: # Relocation 0
-; CHECK: (('word-0', 0x9),
-; CHECK:  ('word-1', 0x1a000002)),
-; CHECK: # Relocation 1
-; CHECK: (('word-0', 0x9),
-; CHECK:  ('word-1', 0xa000001)),
-; CHECK: # Relocation 2
-; CHECK: (('word-0', 0x8),
-; CHECK:  ('word-1', 0x18000002)),
-; CHECK: # Relocation 3
-; CHECK: (('word-0', 0x8),
-; CHECK:  ('word-1', 0x8000001)),
-
+; CHECK: File: <stdin>
+; CHECK: Format: Mach-O arm64
+; CHECK: Arch: aarch64
+; CHECK: AddressSize: 64bit
+; CHECK: Relocations [
+; CHECK:  Section __text {
+; CHECK:    0x9 0 1 1 ARM64_RELOC_SUBTRACTOR 0 bar
+; CHECK:    0x9 0 1 1 ARM64_RELOC_UNSIGNED 0 foo
+; CHECK:    0x8 0 0 1 ARM64_RELOC_SUBTRACTOR 0 bar
+; CHECK:    0x8 0 0 1 ARM64_RELOC_UNSIGNED 0 foo
+; CHECK:  }
+; CHECK: ]
diff --git a/test/MC/AArch64/armv8.1a-atomic.s b/test/MC/AArch64/armv8.1a-atomic.s
new file mode 100644
index 00000000000..bcfd3e7d166
--- /dev/null
+++ b/test/MC/AArch64/armv8.1a-atomic.s
@@ -0,0 +1,184 @@
+// RUN: not llvm-mc -triple aarch64-none-linux-gnu -mattr=+v8.1a -show-encoding < %s 2> %t | FileCheck %s
+// RUN: FileCheck --check-prefix=CHECK-ERROR <%t %s
+  .text
+
+  //8 bits
+  casb   w0, w1, [x2]
+  casab  w0, w1, [x2]
+  caslb  w0, w1, [x2]
+  casalb   w0, w1, [x2]
+
+//CHECK:  casb   w0, w1, [x2]        //      encoding: [0x41,0x7c,0xa0,0x08]
+//CHECK:  casab  w0, w1, [x2]        //      encoding: [0x41,0x7c,0xe0,0x08]
+//CHECK:  caslb   w0, w1, [x2]       //      encoding: [0x41,0xfc,0xa0,0x08]
+//CHECK:  casalb   w0, w1, [x2]      //      encoding: [0x41,0xfc,0xe0,0x08]
+
+  casb w0, w1, [w2]
+  casalb x0, x1, [x2]
+//CHECK-ERROR: error: invalid operand for instruction
+//CHECK-ERROR:   casb w0, w1, [w2]
+//CHECK-ERROR:                 ^
+//CHECK-ERROR: error: invalid operand for instruction
+//CHECK-ERROR:   casalb x0, x1, [x2]
+//CHECK-ERROR:          ^
+
+  //16 bits
+  cash   w0, w1, [x2]
+  casah  w0, w1, [x2]
+  caslh  w0, w1, [x2]
+  casalh   w0, w1, [x2]
+
+//CHECK:  cash   w0, w1, [x2]        //      encoding: [0x41,0x7c,0xa0,0x48]
+//CHECK:  casah  w0, w1, [x2]        //      encoding: [0x41,0x7c,0xe0,0x48]
+//CHECK:  caslh   w0, w1, [x2]       //      encoding: [0x41,0xfc,0xa0,0x48]
+//CHECK:  casalh   w0, w1, [x2]      //      encoding: [0x41,0xfc,0xe0,0x48]
+
+  //32 bits
+  cas   w0, w1, [x2]
+  casa  w0, w1, [x2]
+  casl  w0, w1, [x2]
+  casal   w0, w1, [x2]
+
+//CHECK:  cas   w0, w1, [x2]        //      encoding: [0x41,0x7c,0xa0,0x88]
+//CHECK:  casa  w0, w1, [x2]        //      encoding: [0x41,0x7c,0xe0,0x88]
+//CHECK:  casl   w0, w1, [x2]       //      encoding: [0x41,0xfc,0xa0,0x88]
+//CHECK:  casal   w0, w1, [x2]      //      encoding: [0x41,0xfc,0xe0,0x88]
+
+  cas   w0, w1, [w2]
+  casl  w0, x1, [x2]
+
+//CHECK-ERROR: error: invalid operand for instruction
+//CHECK-ERROR:   cas   w0, w1, [w2]
+//CHECK-ERROR:                  ^
+//CHECK-ERROR: error: invalid operand for instruction
+//CHECK-ERROR:   casl  w0, x1, [x2]
+//CHECK-ERROR:             ^
+
+  //64 bits
+  cas   x0, x1, [x2]
+  casa  x0, x1, [x2]
+  casl   x0, x1, [x2]
+  casal   x0, x1, [x2]
+
+//CHECK:  cas   x0, x1, [x2]        //      encoding: [0x41,0x7c,0xa0,0xc8]
+//CHECK:  casa  x0, x1, [x2]        //      encoding: [0x41,0x7c,0xe0,0xc8]
+//CHECK:  casl   x0, x1, [x2]       //      encoding: [0x41,0xfc,0xa0,0xc8]
+//CHECK:  casal   x0, x1, [x2]      //      encoding: [0x41,0xfc,0xe0,0xc8]
+
+  casa   x0, x1, [w2]
+  casal  x0, w1, [x2]
+
+//CHECK-ERROR: error: invalid operand for instruction
+//CHECK-ERROR:   casa   x0, x1, [w2]
+//CHECK-ERROR:                   ^
+//CHECK-ERROR: error: invalid operand for instruction
+//CHECK-ERROR:   casal  x0, w1, [x2]
+//CHECK-ERROR:              ^
+
+  // LD<OP> intructions
+  ldadda x0, x1, [x2]
+  ldclrl x0, x1, [x2]
+  ldeoral x0, x1, [x2]
+  ldset x0, x1, [x2]
+  ldsmaxa w0, w1, [x2]
+  ldsminlb w0, w1, [x2]
+  ldumaxalh w0, w1, [x2]
+  ldumin w0, w1, [x2]
+  ldsminb w2, w3, [x5]
+//CHECK: ldadda     x0, x1, [x2]  // encoding: [0x41,0x00,0xa0,0xf8]
+//CHECK: ldclrl     x0, x1, [x2]  // encoding: [0x41,0x10,0x60,0xf8]
+//CHECK: ldeoral    x0, x1, [x2]  // encoding: [0x41,0x20,0xe0,0xf8]
+//CHECK: ldset      x0, x1, [x2]  // encoding: [0x41,0x30,0x20,0xf8]
+//CHECK: ldsmaxa    w0, w1, [x2]  // encoding: [0x41,0x40,0xa0,0xb8]
+//CHECK: ldsminlb   w0, w1, [x2]  // encoding: [0x41,0x50,0x60,0x38]
+//CHECK: ldumaxalh  w0, w1, [x2]  // encoding: [0x41,0x60,0xe0,0x78]
+//CHECK: ldumin     w0, w1, [x2]  // encoding: [0x41,0x70,0x20,0xb8]
+//CHECK: ldsminb    w2, w3, [x5]  // encoding: [0xa3,0x50,0x22,0x38]
+
+  // ST<OP> intructions: aliases to LD<OP>
+  stADDlb w0, [x2]
+  stclrlh w0, [x2]
+  steorl  w0, [x2]
+  stsetl  x0, [x2]
+  stsmaxb  w0, [x2]
+  stsminh  w0, [x2]
+  stumax   w0, [x2]
+  stumin   x0, [x2]
+  stsminl x29, [sp]
+//CHECK: staddlb    w0, [x2]  // encoding: [0x5f,0x00,0x60,0x38]
+//CHECK: stclrlh    w0, [x2]  // encoding: [0x5f,0x10,0x60,0x78]
+//CHECK: steorl     w0, [x2]  // encoding: [0x5f,0x20,0x60,0xb8]
+//CHECK: stsetl     x0, [x2]  // encoding: [0x5f,0x30,0x60,0xf8]
+//CHECK: stsmaxb     w0, [x2]  // encoding: [0x5f,0x40,0x20,0x38]
+//CHECK: stsminh     w0, [x2]  // encoding: [0x5f,0x50,0x20,0x78]
+//CHECK: stumax      w0, [x2]  // encoding: [0x5f,0x60,0x20,0xb8]
+//CHECK: stumin      x0, [x2]  // encoding: [0x5f,0x70,0x20,0xf8]
+//CHECK: stsminl     x29, [sp] // encoding: [0xff,0x53,0x7d,0xf8]
+
+
+  ldsmax x0, x1, [w2]
+  ldeorl w0, w1, [w2]
+//CHECK-ERROR: error: invalid operand for instruction
+//CHECK-ERROR:   ldsmax x0, x1, [w2]
+//CHECK-ERROR:                   ^
+//CHECK-ERROR: error: invalid operand for instruction
+//CHECK-ERROR:   ldeorl w0, w1, [w2]
+//CHECK-ERROR:                   ^
+
+  //SWP instruction
+  swp   x0, x1, [x2]
+  swpb  w0, w1, [x2]
+  swplh w0, w1, [x2]
+  swpal x0, x1, [sp]
+//CHECK: swp   x0, x1, [x2]       // encoding: [0x41,0x80,0x20,0xf8]
+//CHECK: swpb  w0, w1, [x2]       // encoding: [0x41,0x80,0x20,0x38]
+//CHECK: swplh w0, w1, [x2]       // encoding: [0x41,0x80,0x60,0x78]
+//CHECK: swpal x0, x1, [sp]       // encoding: [0xe1,0x83,0xe0,0xf8]
+
+  swp   x0, x1, [w2]
+  swp   x0, x1, [xzr]
+//CHECK-ERROR: error: invalid operand for instruction
+//CHECK-ERROR:   swp   x0, x1, [w2]
+//CHECK-ERROR:                  ^
+//CHECK-ERROR: error: invalid operand for instruction
+//CHECK-ERROR:   swp   x0, x1, [xzr]
+//CHECK-ERROR:                  ^
+
+  //CASP instruction
+  casp x0, x1, x2, x3, [x4]
+  casp w0, w1, w2, w3, [x4]
+//CHECK: casp x0, x1, x2, x3, [x4]      // encoding: [0x82,0x7c,0x20,0x48]
+//CHECK: casp w0, w1, w2, w3, [x4]      // encoding: [0x82,0x7c,0x20,0x08]
+
+  casp x1, x2, x4, x5, [x6]
+  casp x0, x1, x3, x4, [x5]
+  casp x0, x2, x4, x5, [x6]
+  casp x0, x1, x2, x4, [x5]
+  casp x0, w1, x2, x3, [x5]
+  casp w0, x1, x2, x3, [x5]
+  casp w0, x1, w2, w3, [x5]
+  casp x0, x1, w2, w3, [x5]
+//CHECK-ERROR: error: expected first even register of a consecutive same-size even/odd register pair
+//CHECK-ERROR:  casp x1, x2, x4, x5, [x6]
+//CHECK-ERROR:       ^
+//CHECK-ERROR: error: expected first even register of a consecutive same-size even/odd register pair
+//CHECK-ERROR:  casp x0, x1, x3, x4, [x5]
+//CHECK-ERROR:               ^
+//CHECK-ERROR: error:  expected second odd register of a consecutive same-size even/odd register pair
+//CHECK-ERROR:  casp x0, x2, x4, x5, [x6]
+//CHECK-ERROR:           ^
+//CHECK-ERROR: error: expected second odd register of a consecutive same-size even/odd register pair
+//CHECK-ERROR:  casp x0, x1, x2, x4, [x5]
+//CHECK-ERROR:                   ^
+//CHECK-ERROR: error: expected second odd register of a consecutive same-size even/odd register pair
+//CHECK-ERROR:  casp x0, w1, x2, x3, [x5]
+//CHECK-ERROR:           ^
+//CHECK-ERROR: error: expected second odd register of a consecutive same-size even/odd register pair
+//CHECK-ERROR:  casp w0, x1, x2, x3, [x5]
+//CHECK-ERROR:           ^
+//CHECK-ERROR: error: expected second odd register of a consecutive same-size even/odd register pair
+//CHECK-ERROR:  casp w0, x1, w2, w3, [x5]
+//CHECK-ERROR:           ^
+//CHECK-ERROR: error: invalid operand for instruction
+//CHECK-ERROR:  casp x0, x1, w2, w3, [x5]
+//CHECK-ERROR:               ^
diff --git a/test/MC/AArch64/armv8.1a-lor.s b/test/MC/AArch64/armv8.1a-lor.s
index 309cb325bce..1db2c02e272 100644
--- a/test/MC/AArch64/armv8.1a-lor.s
+++ b/test/MC/AArch64/armv8.1a-lor.s
@@ -1,4 +1,5 @@
-// RUN: llvm-mc -triple aarch64-none-linux-gnu -show-encoding -mattr=+v8.1a < %s | FileCheck %s
+// RUN: not llvm-mc -triple aarch64-none-linux-gnu -show-encoding -mattr=+v8.1a < %s 2>%t | FileCheck %s
+// RUN: FileCheck --check-prefix=CHECK-ERROR %s <%t
 
 
 //------------------------------------------------------------------------------
@@ -31,3 +32,48 @@
 // CHECK:   msr    LORN_EL1, x0  // encoding: [0x40,0xa4,0x18,0xd5]
 // CHECK:   msr    LORC_EL1, x0  // encoding: [0x60,0xa4,0x18,0xd5]
 // CHECK:   mrs    x0, LORID_EL1 // encoding: [0xe0,0xa4,0x38,0xd5]
+
+        ldlarb w0,[w1]
+        ldlarh x0,[x1]
+        stllrb w0,[w1]
+        stllrh x0,[x1]
+        stllr  w0,[w1]
+        msr    LORSA_EL1, #0
+        msr    LOREA_EL1, #0
+        msr    LORN_EL1, #0
+        msr    LORC_EL1, #0
+        msr    LORID_EL1, x0
+        mrs    LORID_EL1, #0
+// CHECK-ERROR: error: invalid operand for instruction
+// CHECK-ERROR:         ldlarb w0,[w1]
+// CHECK-ERROR:                    ^
+// CHECK-ERROR: error: invalid operand for instruction
+// CHECK-ERROR:         ldlarh x0,[x1]
+// CHECK-ERROR:                ^
+// CHECK-ERROR: error: invalid operand for instruction
+// CHECK-ERROR:         stllrb w0,[w1]
+// CHECK-ERROR:                    ^
+// CHECK-ERROR: error: invalid operand for instruction
+// CHECK-ERROR:         stllrh x0,[x1]
+// CHECK-ERROR:                ^
+// CHECK-ERROR: error: invalid operand for instruction
+// CHECK-ERROR:         stllr  w0,[w1]
+// CHECK-ERROR:                    ^
+// CHECK-ERROR: error: invalid operand for instruction
+// CHECK-ERROR:         msr     LORSA_EL1, #0
+// CHECK-ERROR:                            ^
+// CHECK-ERROR: error: invalid operand for instruction
+// CHECK-ERROR:         msr     LOREA_EL1, #0
+// CHECK-ERROR:                            ^
+// CHECK-ERROR: error: invalid operand for instruction
+// CHECK-ERROR:         msr     LORN_EL1, #0
+// CHECK-ERROR:                           ^
+// CHECK-ERROR: error: invalid operand for instruction
+// CHECK-ERROR:         msr     LORC_EL1, #0
+// CHECK-ERROR:                           ^
+// CHECK-ERROR: error: expected writable system register or pstate
+// CHECK-ERROR:         msr     LORID_EL1, x0
+// CHECK-ERROR:                 ^
+// CHECK-ERROR: error: invalid operand for instruction
+// CHECK-ERROR:         mrs     LORID_EL1, #0
+// CHECK-ERROR:                 ^
diff --git a/test/MC/AArch64/basic-a64-diagnostics.s b/test/MC/AArch64/basic-a64-diagnostics.s
index 07e6d012e93..0c2bc689663 100644
--- a/test/MC/AArch64/basic-a64-diagnostics.s
+++ b/test/MC/AArch64/basic-a64-diagnostics.s
@@ -75,19 +75,19 @@
 // Add/sub (immediate)
 //------------------------------------------------------------------------------
 
-// Out of range immediates: < 0 or more than 12 bits
-        add w4, w5, #-1
+// Out of range immediates: more than 12 bits
+        add w4, w5, #-4096
         add w5, w6, #0x1000
-        add w4, w5, #-1, lsl #12
+        add w4, w5, #-4096, lsl #12
         add w5, w6, #0x1000, lsl #12
 // CHECK-ERROR: error: expected compatible register, symbol or integer in range [0, 4095]
-// CHECK-ERROR-NEXT:         add w4, w5, #-1
+// CHECK-ERROR-NEXT:         add w4, w5, #-4096
 // CHECK-ERROR-NEXT:                     ^
 // CHECK-ERROR-AARCH64-NEXT: error: expected compatible register, symbol or integer in range [0, 4095]
 // CHECK-ERROR-AARCH64-NEXT:         add w5, w6, #0x1000
 // CHECK-ERROR-AARCH64-NEXT:                     ^
 // CHECK-ERROR-NEXT: error: expected compatible register, symbol or integer in range [0, 4095]
-// CHECK-ERROR-NEXT:         add w4, w5, #-1, lsl #12
+// CHECK-ERROR-NEXT:         add w4, w5, #-4096, lsl #12
 // CHECK-ERROR-NEXT:                     ^
 // CHECK-ERROR-NEXT: error: expected compatible register, symbol or integer in range [0, 4095]
 // CHECK-ERROR-NEXT:         add w5, w6, #0x1000, lsl #12
@@ -1088,6 +1088,23 @@
 // CHECK-ERROR-NEXT:         ubfx w3, wsp, #10, #8
 // CHECK-ERROR-NEXT:                  ^
 
+        bfc wsp, #3, #6
+        bfc w4, #2, #31
+        bfc sp, #0, #1
+        bfc x6, #0, #0
+// CHECK-ERROR: error: invalid operand for instruction
+// CHECK-ERROR-NEXT:        bfc wsp, #3, #6
+// CHECK-ERROR-NEXT:            ^
+// CHECK-ERROR-NEXT: error: requested insert overflows register
+// CHECK-ERROR-NEXT:         bfc w4, #2, #31
+// CHECK-ERROR-NEXT:                     ^
+// CHECK-ERROR-NEXT: error: invalid operand for instruction
+// CHECK-ERROR-NEXT:         bfc sp, #0, #1
+// CHECK-ERROR-NEXT:             ^
+// CHECK-ERROR-NEXT: error: expected integer in range [1, 32]
+// CHECK-ERROR-NEXT:         bfc x6, #0, #0
+// CHECK-ERROR-NEXT:                     ^
+
 //------------------------------------------------------------------------------
 // Compare & branch (immediate)
 //------------------------------------------------------------------------------
@@ -3477,6 +3494,7 @@
         msr ID_MMFR1_EL1, x12
         msr ID_MMFR2_EL1, x12
         msr ID_MMFR3_EL1, x12
+        msr ID_MMFR4_EL1, x12
         msr ID_ISAR0_EL1, x12
         msr ID_ISAR1_EL1, x12
         msr ID_ISAR2_EL1, x12
@@ -3570,6 +3588,9 @@
 // CHECK-ERROR-NEXT:         msr ID_MMFR3_EL1, x12
 // CHECK-ERROR-NEXT:             ^
 // CHECK-ERROR-NEXT: error: expected writable system register or pstate
+// CHECK-ERROR-NEXT:         msr ID_MMFR4_EL1, x12
+// CHECK-ERROR-NEXT:             ^
+// CHECK-ERROR-NEXT: error: expected writable system register or pstate
 // CHECK-ERROR-NEXT:         msr ID_ISAR0_EL1, x12
 // CHECK-ERROR-NEXT:             ^
 // CHECK-ERROR-NEXT: error: expected writable system register or pstate
diff --git a/test/MC/AArch64/basic-a64-instructions.s b/test/MC/AArch64/basic-a64-instructions.s
index dd8dfd4e36d..f8e49432145 100644
--- a/test/MC/AArch64/basic-a64-instructions.s
+++ b/test/MC/AArch64/basic-a64-instructions.s
@@ -978,7 +978,7 @@ _func:
         bfm x5, x6, #12, #63
 // CHECK: bfi      x4, x5, #52, #11           // encoding: [0xa4,0x28,0x4c,0xb3]
 // CHECK: bfxil    xzr, x4, #0, #1            // encoding: [0x9f,0x00,0x40,0xb3]
-// CHECK: bfi      x4, xzr, #1, #6            // encoding: [0xe4,0x17,0x7f,0xb3]
+// CHECK: bfc      x4, #1, #6                 // encoding: [0xe4,0x17,0x7f,0xb3]
 // CHECK: bfxil    x5, x6, #12, #52           // encoding: [0xc5,0xfc,0x4c,0xb3]
 
         sxtb w1, w2
@@ -1078,7 +1078,7 @@ _func:
 // CHECK: bfxil    w9, w10, #0, #32           // encoding: [0x49,0x7d,0x00,0x33]
 // CHECK: bfi      w11, w12, #31, #1          // encoding: [0x8b,0x01,0x01,0x33]
 // CHECK: bfi      w13, w14, #29, #3          // encoding: [0xcd,0x09,0x03,0x33]
-// CHECK: bfi      xzr, xzr, #10, #11         // encoding: [0xff,0x2b,0x76,0xb3]
+// CHECK: bfc      xzr, #10, #11              // encoding: [0xff,0x2b,0x76,0xb3]
 
         bfxil w9, w10, #0, #1
         bfxil x2, x3, #63, #1
@@ -1132,6 +1132,16 @@ _func:
 // CHECK: lsr     w11, w12, #31           // encoding: [0x8b,0x7d,0x1f,0x53]
 // CHECK: lsr     w13, w14, #29           // encoding: [0xcd,0x7d,0x1d,0x53]
 // CHECK: ubfx    xzr, xzr, #10, #11      // encoding: [0xff,0x53,0x4a,0xd3]
+
+        bfc w3, #0, #32
+        bfc wzr, #31, #1
+        bfc x0, #5, #9
+        bfc xzr, #63, #1
+// CHECK: bfc w3, #0, #32             // encoding: [0xe3,0x7f,0x00,0x33]
+// CHECK: bfc wzr, #31, #1            // encoding: [0xff,0x03,0x01,0x33]
+// CHECK: bfc x0, #5, #9              // encoding: [0xe0,0x23,0x7b,0xb3]
+// CHECK: bfc xzr, #63, #1            // encoding: [0xff,0x03,0x41,0xb3]
+
 //------------------------------------------------------------------------------
 // Compare & branch (immediate)
 //------------------------------------------------------------------------------
@@ -1479,6 +1489,9 @@ _func:
 // CHECK: clz	w24, wzr                     // encoding: [0xf8,0x13,0xc0,0x5a]
 // CHECK: rev	x22, xzr                     // encoding: [0xf6,0x0f,0xc0,0xda]
 
+	rev64	x13, x12
+// CHECK: rev	x13, x12                     // encoding: [0x8d,0x0d,0xc0,0xda]
+
 //------------------------------------------------------------------------------
 // Data-processing (2 source)
 //------------------------------------------------------------------------------
@@ -4296,6 +4309,7 @@ _func:
 	mrs x9, ID_MMFR1_EL1
 	mrs x9, ID_MMFR2_EL1
 	mrs x9, ID_MMFR3_EL1
+	mrs x9, ID_MMFR4_EL1
 	mrs x9, ID_ISAR0_EL1
 	mrs x9, ID_ISAR1_EL1
 	mrs x9, ID_ISAR2_EL1
@@ -4596,6 +4610,7 @@ _func:
 // CHECK: mrs      x9, {{id_mmfr1_el1|ID_MMFR1_EL1}}           // encoding: [0xa9,0x01,0x38,0xd5]
 // CHECK: mrs      x9, {{id_mmfr2_el1|ID_MMFR2_EL1}}           // encoding: [0xc9,0x01,0x38,0xd5]
 // CHECK: mrs      x9, {{id_mmfr3_el1|ID_MMFR3_EL1}}           // encoding: [0xe9,0x01,0x38,0xd5]
+// CHECK: mrs      x9, {{id_mmfr4_el1|ID_MMFR4_EL1}}           // encoding: [0xc9,0x02,0x38,0xd5]
 // CHECK: mrs      x9, {{id_isar0_el1|ID_ISAR0_EL1}}           // encoding: [0x09,0x02,0x38,0xd5]
 // CHECK: mrs      x9, {{id_isar1_el1|ID_ISAR1_EL1}}           // encoding: [0x29,0x02,0x38,0xd5]
 // CHECK: mrs      x9, {{id_isar2_el1|ID_ISAR2_EL1}}           // encoding: [0x49,0x02,0x38,0xd5]
diff --git a/test/MC/AArch64/case-insen-reg-names.s b/test/MC/AArch64/case-insen-reg-names.s
new file mode 100644
index 00000000000..b31ab675a7e
--- /dev/null
+++ b/test/MC/AArch64/case-insen-reg-names.s
@@ -0,0 +1,8 @@
+// RUN: llvm-mc -triple aarch64-none-linux-gnu -show-encoding < %s | FileCheck %s
+
+fadd v0.2d, v5.2d, v6.2d
+fadd V0.2d, V5.2d, V6.2d
+fadd v0.2d, V5.2d, v6.2d
+// CHECK: fadd v0.2d, v5.2d, v6.2d          // encoding: [0xa0,0xd4,0x66,0x4e]
+// CHECK: fadd v0.2d, v5.2d, v6.2d          // encoding: [0xa0,0xd4,0x66,0x4e]
+// CHECK: fadd v0.2d, v5.2d, v6.2d          // encoding: [0xa0,0xd4,0x66,0x4e]
diff --git a/test/MC/AArch64/expr-shr.s b/test/MC/AArch64/expr-shr.s
new file mode 100644
index 00000000000..aa23878e5ad
--- /dev/null
+++ b/test/MC/AArch64/expr-shr.s
@@ -0,0 +1,8 @@
+// RUN: llvm-mc -triple aarch64-unknown-unknown-elf %s | FileCheck %s --check-prefix=ELF
+// RUN: llvm-mc -triple aarch64-unknown-darwin %s | FileCheck %s --check-prefix=DARWIN
+
+.data
+
+// ELF: .xword 3
+// DARWIN: .quad 3
+.quad (~0 >> 62)
diff --git a/test/MC/AArch64/inst-directive.s b/test/MC/AArch64/inst-directive.s
index 6a4b64eeaaf..3bb620f689d 100644
--- a/test/MC/AArch64/inst-directive.s
+++ b/test/MC/AArch64/inst-directive.s
@@ -15,7 +15,7 @@ aarch64_inst:
 // CHECK-ASM:        .globl  aarch64_inst
 // CHECK-ASM:        .type   aarch64_inst,@function
 // CHECK-ASM: aarch64_inst:
-// CHECK-ASM:        .inst   0x5E104020
+// CHECK-ASM:        .inst   0x5e104020
 
 // CHECK-OBJ: Section {
 // CHECK-OBJ:   Name: .inst.aarch64_inst
diff --git a/test/MC/R600/ds-err.s b/test/MC/AMDGPU/ds-err.s
similarity index 100%
rename from test/MC/R600/ds-err.s
rename to test/MC/AMDGPU/ds-err.s
diff --git a/test/MC/R600/ds.s b/test/MC/AMDGPU/ds.s
similarity index 100%
rename from test/MC/R600/ds.s
rename to test/MC/AMDGPU/ds.s
diff --git a/test/MC/AMDGPU/flat.s b/test/MC/AMDGPU/flat.s
new file mode 100644
index 00000000000..adad29a5595
--- /dev/null
+++ b/test/MC/AMDGPU/flat.s
@@ -0,0 +1,477 @@
+// RUN: llvm-mc -arch=amdgcn -mcpu=bonaire -show-encoding %s | FileCheck %s --check-prefix=CIVI --check-prefix=CI
+// RUN: llvm-mc -arch=amdgcn -mcpu=tonga -show-encoding %s | FileCheck %s --check-prefix=CIVI
+
+// FIXME: These instructions give an 'invalid operand' error on SI and should
+// instead be reporting an 'instruction not supported' error.
+
+// XUN: not llvm-mc -arch=amdgcn -mcpu=tonga -show-encoding %s | FileCheck %s --check-prefix=NOVI
+// XUN: not llvm-mc -arch=amdgcn -show-encoding %s 2>&1 | FileCheck %s --check-prefix=NOSI
+// XUN: not llvm-mc -arch=amdgcn -mcpu=SI -show-encoding %s 2>&1 | FileCheck %s --check-prefix=NOSI
+
+//===----------------------------------------------------------------------===//
+// Operands
+//===----------------------------------------------------------------------===//
+
+flat_load_dword v1, v[3:4]
+// NOSI: error: instruction not supported on this GPU
+// CIVI: flat_load_dword v1, v[3:4] ; encoding: [0x00,0x00,0x30,0xdc,0x03,0x00,0x00,0x01]
+
+flat_load_dword v1, v[3:4] glc
+// NOSI: error: instruction not supported on this GPU
+// CIVI: flat_load_dword v1, v[3:4] glc ; encoding: [0x00,0x00,0x31,0xdc,0x03,0x00,0x00,0x01]
+
+flat_load_dword v1, v[3:4] glc slc
+// NOSI: error: instruction not supported on this GPU
+// CIVI: flat_load_dword v1, v[3:4] glc slc ; encoding: [0x00,0x00,0x33,0xdc,0x03,0x00,0x00,0x01]
+
+flat_load_dword v1, v[3:4] glc tfe
+// NOSI: error: instruction not supported on this GPU
+// CIVI: flat_load_dword v1, v[3:4] glc tfe ; encoding: [0x00,0x00,0x31,0xdc,0x03,0x00,0x80,0x01]
+
+flat_load_dword v1, v[3:4] glc slc tfe
+// NOSI: error: instruction not supported on this GPU
+// CIVI: flat_load_dword v1, v[3:4] glc slc tfe ; encoding: [0x00,0x00,0x33,0xdc,0x03,0x00,0x80,0x01]
+
+flat_load_dword v1, v[3:4] glc tfe slc
+// NOSI: error: instruction not supported on this GPU
+// CIVI: flat_load_dword v1, v[3:4] glc slc tfe ; encoding: [0x00,0x00,0x33,0xdc,0x03,0x00,0x80,0x01]
+
+flat_load_dword v1, v[3:4] slc
+// NOSI: error: instruction not supported on this GPU
+// CIVI: flat_load_dword v1, v[3:4] slc ; encoding: [0x00,0x00,0x32,0xdc,0x03,0x00,0x00,0x01]
+
+flat_load_dword v1, v[3:4] slc glc
+// NOSI: error: instruction not supported on this GPU
+// CIVI: flat_load_dword v1, v[3:4] glc slc ; encoding: [0x00,0x00,0x33,0xdc,0x03,0x00,0x00,0x01]
+
+flat_load_dword v1, v[3:4] slc tfe
+// NOSI: error: instruction not supported on this GPU
+// CIVI: flat_load_dword v1, v[3:4] slc tfe ; encoding: [0x00,0x00,0x32,0xdc,0x03,0x00,0x80,0x01]
+
+flat_load_dword v1, v[3:4] slc glc tfe
+// NOSI: error: instruction not supported on this GPU
+// CIVI: flat_load_dword v1, v[3:4] glc slc tfe ; encoding: [0x00,0x00,0x33,0xdc,0x03,0x00,0x80,0x01]
+
+flat_load_dword v1, v[3:4] slc tfe glc
+// NOSI: error: instruction not supported on this GPU
+// CIVI: flat_load_dword v1, v[3:4] glc slc tfe ; encoding: [0x00,0x00,0x33,0xdc,0x03,0x00,0x80,0x01]
+
+flat_load_dword v1, v[3:4] tfe
+// NOSI: error: instruction not supported on this GPU
+// CIVI: flat_load_dword v1, v[3:4] tfe ; encoding: [0x00,0x00,0x30,0xdc,0x03,0x00,0x80,0x01]
+
+flat_load_dword v1, v[3:4] tfe glc
+// NOSI: error: instruction not supported on this GPU
+// CIVI: flat_load_dword v1, v[3:4] glc tfe ; encoding: [0x00,0x00,0x31,0xdc,0x03,0x00,0x80,0x01]
+
+flat_load_dword v1, v[3:4] tfe slc
+// NOSI: error: instruction not supported on this GPU
+// CIVI: flat_load_dword v1, v[3:4] slc tfe ; encoding: [0x00,0x00,0x32,0xdc,0x03,0x00,0x80,0x01]
+
+flat_load_dword v1, v[3:4] tfe glc slc
+// NOSI: error: instruction not supported on this GPU
+// CIVI: flat_load_dword v1, v[3:4] glc slc tfe ; encoding: [0x00,0x00,0x33,0xdc,0x03,0x00,0x80,0x01]
+
+flat_load_dword v1, v[3:4] tfe slc glc
+// NOSI: error: instruction not supported on this GPU
+// CIVI: flat_load_dword v1, v[3:4] glc slc tfe ; encoding: [0x00,0x00,0x33,0xdc,0x03,0x00,0x80,0x01]
+
+flat_store_dword v1, v[3:4]
+// NOSI: error: instruction not supported on this GPU
+// CIVI: flat_store_dword v1, v[3:4] ; encoding: [0x00,0x00,0x70,0xdc,0x03,0x01,0x00,0x00]
+
+flat_store_dword v1, v[3:4] glc
+// NOSI: error: instruction not supported on this GPU
+// CIVI: flat_store_dword v1, v[3:4] glc ; encoding: [0x00,0x00,0x71,0xdc,0x03,0x01,0x00,0x00]
+
+flat_store_dword v1, v[3:4] glc slc
+// NOSI: error: instruction not supported on this GPU
+// CIVI: flat_store_dword v1, v[3:4] glc slc ; encoding: [0x00,0x00,0x73,0xdc,0x03,0x01,0x00,0x00]
+
+flat_store_dword v1, v[3:4] glc tfe
+// NOSI: error: instruction not supported on this GPU
+// CIVI: flat_store_dword v1, v[3:4] glc tfe ; encoding: [0x00,0x00,0x71,0xdc,0x03,0x01,0x80,0x00]
+
+flat_store_dword v1, v[3:4] glc slc tfe
+// NOSI: error: instruction not supported on this GPU
+// CIVI: flat_store_dword v1, v[3:4] glc slc tfe ; encoding: [0x00,0x00,0x73,0xdc,0x03,0x01,0x80,0x00]
+
+flat_store_dword v1, v[3:4] glc tfe slc
+// NOSI: error: instruction not supported on this GPU
+// CIVI: flat_store_dword v1, v[3:4] glc slc tfe ; encoding: [0x00,0x00,0x73,0xdc,0x03,0x01,0x80,0x00]
+
+flat_store_dword v1, v[3:4] slc
+// NOSI: error: instruction not supported on this GPU
+// CIVI: flat_store_dword v1, v[3:4] slc ; encoding: [0x00,0x00,0x72,0xdc,0x03,0x01,0x00,0x00]
+
+flat_store_dword v1, v[3:4] slc glc
+// NOSI: error: instruction not supported on this GPU
+// CIVI: flat_store_dword v1, v[3:4] glc slc ; encoding: [0x00,0x00,0x73,0xdc,0x03,0x01,0x00,0x00]
+
+flat_store_dword v1, v[3:4] slc tfe
+// NOSI: error: instruction not supported on this GPU
+// CIVI: flat_store_dword v1, v[3:4] slc tfe ; encoding: [0x00,0x00,0x72,0xdc,0x03,0x01,0x80,0x00]
+
+flat_store_dword v1, v[3:4] slc glc tfe
+// NOSI: error: instruction not supported on this GPU
+// CIVI: flat_store_dword v1, v[3:4] glc slc tfe ; encoding: [0x00,0x00,0x73,0xdc,0x03,0x01,0x80,0x00]
+
+flat_store_dword v1, v[3:4] slc tfe glc
+// NOSI: error: instruction not supported on this GPU
+// CIVI: flat_store_dword v1, v[3:4] glc slc tfe ; encoding: [0x00,0x00,0x73,0xdc,0x03,0x01,0x80,0x00]
+
+flat_store_dword v1, v[3:4] tfe
+// NOSI: error: instruction not supported on this GPU
+// CIVI: flat_store_dword v1, v[3:4] tfe ; encoding: [0x00,0x00,0x70,0xdc,0x03,0x01,0x80,0x00]
+
+flat_store_dword v1, v[3:4] tfe glc
+// NOSI: error: instruction not supported on this GPU
+// CIVI: flat_store_dword v1, v[3:4] glc tfe ; encoding: [0x00,0x00,0x71,0xdc,0x03,0x01,0x80,0x00]
+
+flat_store_dword v1, v[3:4] tfe slc
+// NOSI: error: instruction not supported on this GPU
+// CIVI: flat_store_dword v1, v[3:4] slc tfe ; encoding: [0x00,0x00,0x72,0xdc,0x03,0x01,0x80,0x00]
+
+flat_store_dword v1, v[3:4] tfe glc slc
+// NOSI: error: instruction not supported on this GPU
+// CIVI: flat_store_dword v1, v[3:4] glc slc tfe ; encoding: [0x00,0x00,0x73,0xdc,0x03,0x01,0x80,0x00]
+
+flat_store_dword v1, v[3:4] tfe slc glc
+// NOSI: error: instruction not supported on this GPU
+// CIVI: flat_store_dword v1, v[3:4] glc slc tfe ; encoding: [0x00,0x00,0x73,0xdc,0x03,0x01,0x80,0x00]
+
+// FIXME: For atomic instructions, glc must be placed immediately following
+// the data regiser.  These forms aren't currently supported:
+// flat_atomic_add v1, v[3:4], v5 slc glc
+// flat_atomic_add v1, v[3:4], v5 slc glc tfe
+// flat_atomic_add v1, v[3:4], v5 slc tfe glc
+// flat_atomic_add v1, v[3:4], v5 tfe glc
+// flat_atomic_add v[3:4], v5 tfe glc
+// flat_atomic_add v1, v[3:4], v5 tfe glc slc
+// flat_atomic_add v1, v[3:4], v5 tfe slc glc
+
+flat_atomic_add v1 v[3:4], v5 glc slc
+// NOSI: error: instruction not supported on this GPU
+// CIVI: flat_atomic_add v1, v[3:4], v5 glc slc ; encoding: [0x00,0x00,0xcb,0xdc,0x03,0x05,0x00,0x01]
+
+flat_atomic_add v1 v[3:4], v5 glc tfe
+// NOSI: error: instruction not supported on this GPU
+// CIVI: flat_atomic_add v1, v[3:4], v5 glc tfe ; encoding: [0x00,0x00,0xc9,0xdc,0x03,0x05,0x80,0x01]
+
+flat_atomic_add v1 v[3:4], v5 glc slc tfe
+// NOSI: error: instruction not supported on this GPU
+// CIVI: flat_atomic_add v1, v[3:4], v5 glc slc tfe ; encoding: [0x00,0x00,0xcb,0xdc,0x03,0x05,0x80,0x01]
+
+flat_atomic_add v1 v[3:4], v5 glc tfe slc
+// NOSI: error: instruction not supported on this GPU
+// CIVI: flat_atomic_add v1, v[3:4], v5 glc slc tfe ; encoding: [0x00,0x00,0xcb,0xdc,0x03,0x05,0x80,0x01]
+
+flat_atomic_add v[3:4], v5 slc
+// NOSI: error: instruction not supported on this GPU
+// CIVI: flat_atomic_add v[3:4], v5 slc ; encoding: [0x00,0x00,0xca,0xdc,0x03,0x05,0x00,0x00]
+
+flat_atomic_add v[3:4], v5 slc tfe
+// NOSI: error: instruction not supported on this GPU
+// CIVI: flat_atomic_add v[3:4], v5 slc tfe ; encoding: [0x00,0x00,0xca,0xdc,0x03,0x05,0x80,0x00]
+
+flat_atomic_add v[3:4], v5 tfe
+// NOSI: error: instruction not supported on this GPU
+// CIVI: flat_atomic_add v[3:4], v5 tfe ; encoding: [0x00,0x00,0xc8,0xdc,0x03,0x05,0x80,0x00]
+
+//===----------------------------------------------------------------------===//
+// Instructions
+//===----------------------------------------------------------------------===//
+
+flat_load_ubyte v1, v[3:4]
+// NOSI: error: instruction not supported on this GPU
+// CIVI: flat_load_ubyte v1, v[3:4] ; encoding: [0x00,0x00,0x20,0xdc,0x03,0x00,0x00,0x01]
+
+flat_load_sbyte v1, v[3:4]
+// NOSI: error: instruction not supported on this GPU
+// CIVI: flat_load_sbyte v1, v[3:4] ; encoding: [0x00,0x00,0x24,0xdc,0x03,0x00,0x00,0x01]
+
+flat_load_ushort v1, v[3:4]
+// NOSI: error: instruction not supported on this GPU
+// CIVI: flat_load_ushort v1, v[3:4] ; encoding: [0x00,0x00,0x28,0xdc,0x03,0x00,0x00,0x01]
+
+flat_load_sshort v1, v[3:4]
+// NOSI: error: instruction not supported on this GPU
+// CIVI: flat_load_sshort v1, v[3:4] ; encoding: [0x00,0x00,0x2c,0xdc,0x03,0x00,0x00,0x01]
+
+flat_load_dword v1, v[3:4]
+// NOSI: error: instruction not supported on this GPU
+// CIVI: flat_load_dword v1, v[3:4] ; encoding: [0x00,0x00,0x30,0xdc,0x03,0x00,0x00,0x01]
+
+flat_load_dwordx2 v[1:2], v[3:4]
+// NOSI: error: instruction not supported on this GPU
+// CIVI: flat_load_dwordx2 v[1:2], v[3:4] ; encoding: [0x00,0x00,0x34,0xdc,0x03,0x00,0x00,0x01]
+
+flat_load_dwordx4 v[5:8], v[3:4]
+// NOSI: error: instruction not supported on this GPU
+// CIVI: flat_load_dwordx4 v[5:8], v[3:4] ; encoding: [0x00,0x00,0x38,0xdc,0x03,0x00,0x00,0x05]
+
+flat_load_dwordx3 v[5:7], v[3:4]
+// NOSI: error: instruction not supported on this GPU
+// CIVI: flat_load_dwordx3 v[5:7], v[3:4] ; encoding: [0x00,0x00,0x3c,0xdc,0x03,0x00,0x00,0x05]
+
+flat_store_byte v1, v[3:4]
+// NOSI: error: instruction not supported on this GPU
+// CIVI: flat_store_byte v1, v[3:4] ; encoding: [0x00,0x00,0x60,0xdc,0x03,0x01,0x00,0x00]
+
+flat_store_short v1, v[3:4]
+// NOSI: error: instruction not supported on this GPU
+// CIVI: flat_store_short v1, v[3:4] ; encoding: [0x00,0x00,0x68,0xdc,0x03,0x01,0x00,0x00]
+
+flat_store_dword v1, v[3:4]
+// NOSI: error: instruction not supported on this GPU
+// CIVI: flat_store_dword v1, v[3:4] ; encoding: [0x00,0x00,0x70,0xdc,0x03,0x01,0x00,0x00]
+
+flat_store_dwordx2 v[1:2], v[3:4]
+// NOSI: error: instruction not supported on this GPU
+// CIVI: flat_store_dwordx2 v[1:2], v[3:4] ; encoding: [0x00,0x00,0x74,0xdc,0x03,0x01,0x00,0x00]
+
+flat_store_dwordx4 v[5:8], v[3:4]
+// NOSI: error: instruction not supported on this GPU
+// CIVI: flat_store_dwordx4 v[5:8], v[3:4] ; encoding: [0x00,0x00,0x78,0xdc,0x03,0x05,0x00,0x00]
+
+flat_store_dwordx3 v[5:7], v[3:4]
+// NOSI: error: instruction not supported on this GPU
+// CIVI: flat_store_dwordx3 v[5:7], v[3:4] ; encoding: [0x00,0x00,0x7c,0xdc,0x03,0x05,0x00,0x00]
+
+flat_atomic_swap v[3:4], v5
+// NOSI: error: instruction not supported on this GPU
+// CIVI: flat_atomic_swap v[3:4], v5 ; encoding: [0x00,0x00,0xc0,0xdc,0x03,0x05,0x00,0x00]
+
+flat_atomic_swap v1, v[3:4], v5 glc
+// NOSI: error: instruction not supported on this GPU
+// CIVI: flat_atomic_swap v1, v[3:4], v5 glc ; encoding: [0x00,0x00,0xc1,0xdc,0x03,0x05,0x00,0x01]
+
+flat_atomic_cmpswap v[3:4], v[5:6]
+// NOSI: error: instruction not supported on this GPU
+// CIVI: flat_atomic_cmpswap v[3:4], v[5:6] ; encoding: [0x00,0x00,0xc4,0xdc,0x03,0x05,0x00,0x00]
+
+flat_atomic_cmpswap v1, v[3:4], v[5:6] glc
+// NOSI: error: instruction not supported on this GPU
+// CIVI: flat_atomic_cmpswap v1, v[3:4], v[5:6] glc ; encoding: [0x00,0x00,0xc5,0xdc,0x03,0x05,0x00,0x01]
+
+flat_atomic_add v[3:4], v5
+// NOSI: error: instruction not supported on this GPU
+// CIVI: flat_atomic_add v[3:4], v5 ; encoding: [0x00,0x00,0xc8,0xdc,0x03,0x05,0x00,0x00]
+
+flat_atomic_add v1, v[3:4], v5 glc
+// NOSI: error: instruction not supported on this GPU
+// CIVI: flat_atomic_add v1, v[3:4], v5 glc ; encoding: [0x00,0x00,0xc9,0xdc,0x03,0x05,0x00,0x01]
+
+flat_atomic_sub v[3:4], v5
+// NOSI: error: instruction not supported on this GPU
+// CIVI: flat_atomic_sub v[3:4], v5 ; encoding: [0x00,0x00,0xcc,0xdc,0x03,0x05,0x00,0x00]
+
+flat_atomic_sub v1, v[3:4], v5 glc
+// NOSI: error: instruction not supported on this GPU
+// CIVI: flat_atomic_sub v1, v[3:4], v5 glc ; encoding: [0x00,0x00,0xcd,0xdc,0x03,0x05,0x00,0x01]
+
+flat_atomic_smin v[3:4], v5
+// NOSI: error: instruction not supported on this GPU
+// CIVI: flat_atomic_smin v[3:4], v5 ; encoding: [0x00,0x00,0xd4,0xdc,0x03,0x05,0x00,0x00]
+
+flat_atomic_smin v1, v[3:4], v5 glc
+// NOSI: error: instruction not supported on this GPU
+// CIVI: flat_atomic_smin v1, v[3:4], v5 glc ; encoding: [0x00,0x00,0xd5,0xdc,0x03,0x05,0x00,0x01]
+
+flat_atomic_umin v[3:4], v5
+// NOSI: error: instruction not supported on this GPU
+// CIVI: flat_atomic_umin v[3:4], v5 ; encoding: [0x00,0x00,0xd8,0xdc,0x03,0x05,0x00,0x00]
+
+flat_atomic_umin v1, v[3:4], v5 glc
+// NOSI: error: instruction not supported on this GPU
+// CIVI: flat_atomic_umin v1, v[3:4], v5 glc ; encoding: [0x00,0x00,0xd9,0xdc,0x03,0x05,0x00,0x01]
+
+flat_atomic_smax v[3:4], v5
+// NOSI: error: instruction not supported on this GPU
+// CIVI: flat_atomic_smax v[3:4], v5 ; encoding: [0x00,0x00,0xdc,0xdc,0x03,0x05,0x00,0x00]
+
+flat_atomic_smax v1, v[3:4], v5 glc
+// NOSI: error: instruction not supported on this GPU
+// CIVI: flat_atomic_smax v1, v[3:4], v5 glc ; encoding: [0x00,0x00,0xdd,0xdc,0x03,0x05,0x00,0x01]
+
+flat_atomic_umax v[3:4], v5
+// NOSI: error: instruction not supported on this GPU
+// CIVI: flat_atomic_umax v[3:4], v5 ; encoding: [0x00,0x00,0xe0,0xdc,0x03,0x05,0x00,0x00]
+
+flat_atomic_umax v1, v[3:4], v5 glc
+// NOSI: error: instruction not supported on this GPU
+// CIVI: flat_atomic_umax v1, v[3:4], v5 glc ; encoding: [0x00,0x00,0xe1,0xdc,0x03,0x05,0x00,0x01]
+
+flat_atomic_and v[3:4], v5
+// NOSI: error: instruction not supported on this GPU
+// CIVI: flat_atomic_and v[3:4], v5 ; encoding: [0x00,0x00,0xe4,0xdc,0x03,0x05,0x00,0x00]
+
+flat_atomic_and v1, v[3:4], v5 glc
+// NOSI: error: instruction not supported on this GPU
+// CIVI: flat_atomic_and v1, v[3:4], v5 glc ; encoding: [0x00,0x00,0xe5,0xdc,0x03,0x05,0x00,0x01]
+
+flat_atomic_or v[3:4], v5
+// NOSI: error: instruction not supported on this GPU
+// CIVI: flat_atomic_or v[3:4], v5 ; encoding: [0x00,0x00,0xe8,0xdc,0x03,0x05,0x00,0x00]
+
+flat_atomic_or v1, v[3:4], v5 glc
+// NOSI: error: instruction not supported on this GPU
+// CIVI: flat_atomic_or v1, v[3:4], v5 glc ; encoding: [0x00,0x00,0xe9,0xdc,0x03,0x05,0x00,0x01]
+
+flat_atomic_xor v[3:4], v5
+// NOSI: error: instruction not supported on this GPU
+// CIVI: flat_atomic_xor v[3:4], v5 ; encoding: [0x00,0x00,0xec,0xdc,0x03,0x05,0x00,0x00]
+
+flat_atomic_xor v1, v[3:4], v5 glc
+// NOSI: error: instruction not supported on this GPU
+// CIVI: flat_atomic_xor v1, v[3:4], v5 glc ; encoding: [0x00,0x00,0xed,0xdc,0x03,0x05,0x00,0x01]
+
+flat_atomic_inc v[3:4], v5
+// NOSI: error: instruction not supported on this GPU
+// CIVI: flat_atomic_inc v[3:4], v5 ; encoding: [0x00,0x00,0xf0,0xdc,0x03,0x05,0x00,0x00]
+
+flat_atomic_inc v1, v[3:4], v5 glc
+// NOSI: error: instruction not supported on this GPU
+// CIVI: flat_atomic_inc v1, v[3:4], v5 glc ; encoding: [0x00,0x00,0xf1,0xdc,0x03,0x05,0x00,0x01]
+
+flat_atomic_dec v[3:4], v5
+// NOSI: error: instruction not supported on this GPU
+// CIVI: flat_atomic_dec v[3:4], v5 ; encoding: [0x00,0x00,0xf4,0xdc,0x03,0x05,0x00,0x00]
+
+flat_atomic_dec v1, v[3:4], v5 glc
+// NOSI: error: instruction not supported on this GPU
+// CIVI: flat_atomic_dec v1, v[3:4], v5 glc ; encoding: [0x00,0x00,0xf5,0xdc,0x03,0x05,0x00,0x01]
+
+flat_atomic_swap_x2 v[3:4], v[5:6]
+// NOSI: error: instruction not supported on this GPU
+// CIVI: flat_atomic_swap_x2 v[3:4], v[5:6] ; encoding: [0x00,0x00,0x40,0xdd,0x03,0x05,0x00,0x00]
+
+flat_atomic_swap_x2 v[1:2], v[3:4], v[5:6] glc
+// NOSI: error: instruction not supported on this GPU
+// CIVI: flat_atomic_swap_x2 v[1:2], v[3:4], v[5:6] glc ; encoding: [0x00,0x00,0x41,0xdd,0x03,0x05,0x00,0x01]
+
+flat_atomic_cmpswap_x2 v[3:4], v[5:8]
+// NOSI: error: instruction not supported on this GPU
+// CIVI: flat_atomic_cmpswap_x2 v[3:4], v[5:8] ; encoding: [0x00,0x00,0x44,0xdd,0x03,0x05,0x00,0x00]
+
+flat_atomic_cmpswap_x2 v[1:2], v[3:4], v[5:8] glc
+// NOSI: error: instruction not supported on this GPU
+// CIVI: flat_atomic_cmpswap_x2 v[1:2], v[3:4], v[5:8] glc ; encoding: [0x00,0x00,0x45,0xdd,0x03,0x05,0x00,0x01]
+
+flat_atomic_add_x2 v[3:4], v[5:6]
+// NOSI: error: instruction not supported on this GPU
+// CIVI: flat_atomic_add_x2 v[3:4], v[5:6] ; encoding: [0x00,0x00,0x48,0xdd,0x03,0x05,0x00,0x00]
+
+flat_atomic_add_x2 v[1:2], v[3:4], v[5:6] glc
+// NOSI: error: instruction not supported on this GPU
+// CIVI: flat_atomic_add_x2 v[1:2], v[3:4], v[5:6] glc ; encoding: [0x00,0x00,0x49,0xdd,0x03,0x05,0x00,0x01]
+
+flat_atomic_sub_x2 v[3:4], v[5:6]
+// NOSI: error: instruction not supported on this GPU
+// CIVI: flat_atomic_sub_x2 v[3:4], v[5:6] ; encoding: [0x00,0x00,0x4c,0xdd,0x03,0x05,0x00,0x00]
+
+flat_atomic_sub_x2 v[1:2], v[3:4], v[5:6] glc
+// NOSI: error: instruction not supported on this GPU
+// CIVI: flat_atomic_sub_x2 v[1:2], v[3:4], v[5:6] glc ; encoding: [0x00,0x00,0x4d,0xdd,0x03,0x05,0x00,0x01]
+
+flat_atomic_smin_x2 v[3:4], v[5:6]
+// NOSI: error: instruction not supported on this GPU
+// CIVI: flat_atomic_smin_x2 v[3:4], v[5:6] ; encoding: [0x00,0x00,0x54,0xdd,0x03,0x05,0x00,0x00]
+
+flat_atomic_smin_x2 v[1:2], v[3:4], v[5:6] glc
+// NOSI: error: instruction not supported on this GPU
+// CIVI: flat_atomic_smin_x2 v[1:2], v[3:4], v[5:6] glc ; encoding: [0x00,0x00,0x55,0xdd,0x03,0x05,0x00,0x01]
+
+flat_atomic_umin_x2 v[3:4], v[5:6]
+// NOSI: error: instruction not supported on this GPU
+// CIVI: flat_atomic_umin_x2 v[3:4], v[5:6] ; encoding: [0x00,0x00,0x58,0xdd,0x03,0x05,0x00,0x00]
+
+flat_atomic_umin_x2 v[1:2], v[3:4], v[5:6] glc
+// NOSI: error: instruction not supported on this GPU
+// CIVI: flat_atomic_umin_x2 v[1:2], v[3:4], v[5:6] glc ; encoding: [0x00,0x00,0x59,0xdd,0x03,0x05,0x00,0x01] 
+
+flat_atomic_smax_x2 v[3:4], v[5:6]
+// NOSI: error: instruction not supported on this GPU
+// CIVI: flat_atomic_smax_x2 v[3:4], v[5:6] ; encoding: [0x00,0x00,0x5c,0xdd,0x03,0x05,0x00,0x00]
+
+flat_atomic_smax_x2 v[1:2], v[3:4], v[5:6] glc
+// NOSI: error: instruction not supported on this GPU
+// CIVI: flat_atomic_smax_x2 v[1:2], v[3:4], v[5:6] glc ; encoding: [0x00,0x00,0x5d,0xdd,0x03,0x05,0x00,0x01]
+
+flat_atomic_umax_x2 v[3:4], v[5:6]
+// NOSI: error: instruction not supported on this GPU
+// CIVI: flat_atomic_umax_x2 v[3:4], v[5:6] ; encoding: [0x00,0x00,0x60,0xdd,0x03,0x05,0x00,0x00]
+
+flat_atomic_umax_x2 v[1:2], v[3:4], v[5:6] glc
+// NOSI: error: instruction not supported on this GPU
+// CIVI: flat_atomic_umax_x2 v[1:2], v[3:4], v[5:6] glc ; encoding: [0x00,0x00,0x61,0xdd,0x03,0x05,0x00,0x01]
+
+flat_atomic_and_x2 v[3:4], v[5:6]
+// NOSI: error: instruction not supported on this GPU
+// CIVI: flat_atomic_and_x2 v[3:4], v[5:6] ; encoding: [0x00,0x00,0x64,0xdd,0x03,0x05,0x00,0x00]
+
+flat_atomic_and_x2 v[1:2], v[3:4], v[5:6] glc
+// NOSI: error: instruction not supported on this GPU
+// CIVI: flat_atomic_and_x2 v[1:2], v[3:4], v[5:6] glc ; encoding: [0x00,0x00,0x65,0xdd,0x03,0x05,0x00,0x01]
+
+flat_atomic_or_x2 v[3:4], v[5:6]
+// NOSI: error: instruction not supported on this GPU
+// CIVI: flat_atomic_or_x2 v[3:4], v[5:6] ; encoding: [0x00,0x00,0x68,0xdd,0x03,0x05,0x00,0x00]
+
+flat_atomic_or_x2 v[1:2], v[3:4], v[5:6] glc
+// NOSI: error: instruction not supported on this GPU
+// CIVI: flat_atomic_or_x2 v[1:2], v[3:4], v[5:6] glc ; encoding: [0x00,0x00,0x69,0xdd,0x03,0x05,0x00,0x01]
+
+flat_atomic_xor_x2 v[3:4], v[5:6]
+// NOSI: error: instruction not supported on this GPU
+// CIVI: flat_atomic_xor_x2 v[3:4], v[5:6] ; encoding: [0x00,0x00,0x6c,0xdd,0x03,0x05,0x00,0x00]
+
+flat_atomic_xor_x2 v[1:2], v[3:4], v[5:6] glc
+// NOSI: error: instruction not supported on this GPU
+// CIVI: flat_atomic_xor_x2 v[1:2], v[3:4], v[5:6] glc ; encoding: [0x00,0x00,0x6d,0xdd,0x03,0x05,0x00,0x01]
+
+flat_atomic_inc_x2 v[3:4], v[5:6]
+// NOSI: error: instruction not supported on this GPU
+// CIVI: flat_atomic_inc_x2 v[3:4], v[5:6] ; encoding: [0x00,0x00,0x70,0xdd,0x03,0x05,0x00,0x00]
+
+flat_atomic_inc_x2 v[1:2], v[3:4], v[5:6] glc
+// NOSI: error: instruction not supported on this GPU
+// CIVI: flat_atomic_inc_x2 v[1:2], v[3:4], v[5:6] glc ; encoding: [0x00,0x00,0x71,0xdd,0x03,0x05,0x00,0x01]
+
+flat_atomic_dec_x2 v[3:4], v[5:6]
+// NOSI: error: instruction not supported on this GPU
+// CIVI: flat_atomic_dec_x2 v[3:4], v[5:6] ; encoding: [0x00,0x00,0x74,0xdd,0x03,0x05,0x00,0x00]
+
+flat_atomic_dec_x2 v[1:2], v[3:4], v[5:6] glc
+// NOSI: error: instruction not supported on this GPU
+// CIVI: flat_atomic_dec_x2 v[1:2], v[3:4], v[5:6] glc ; encoding: [0x00,0x00,0x75,0xdd,0x03,0x05,0x00,0x01]
+
+flat_atomic_fcmpswap_x2 v[3:4], v[5:8]
+// NOSI: error: instruction not supported on this GPU
+// CI: flat_atomic_fcmpswap_x2 v[3:4], v[5:8] ; encoding: [0x00,0x00,0x78,0xdd,0x03,0x05,0x00,0x00]
+// NOVI: error: instruction not supported on this GPU
+
+flat_atomic_fcmpswap_x2 v[1:2], v[3:4], v[5:8] glc
+// NOSI: error: instruction not supported on this GPU
+// CI: flat_atomic_fcmpswap_x2 v[1:2], v[3:4], v[5:8] glc ; encoding: [0x00,0x00,0x79,0xdd,0x03,0x05,0x00,0x01]
+// NOVI: error: instruction not supported on this GPU
+
+flat_atomic_fmin_x2 v[3:4], v[5:6]
+// NOSI: error: instruction not supported on this GPU
+// CI: flat_atomic_fmin_x2 v[3:4], v[5:6] ; encoding: [0x00,0x00,0x7c,0xdd,0x03,0x05,0x00,0x00]
+// NOVI: error: instruction not supported on this GPU
+
+flat_atomic_fmin_x2 v[1:2], v[3:4], v[5:6] glc
+// NOSI: error: instruction not supported on this GPU
+// CI: flat_atomic_fmin_x2 v[1:2], v[3:4], v[5:6] glc ; encoding: [0x00,0x00,0x7d,0xdd,0x03,0x05,0x00,0x01]
+// NOVI: error: instruction not supported on this GPU
+
+flat_atomic_fmax_x2 v[3:4], v[5:6]
+// NOSI: error: instruction not supported on this GPU
+// CI: flat_atomic_fmax_x2 v[3:4], v[5:6] ; encoding: [0x00,0x00,0x80,0xdd,0x03,0x05,0x00,0x00]
+// NOVI: error: instruction not supported on this GPU
+
+flat_atomic_fmax_x2 v[1:2], v[3:4], v[5:6] glc
+// NOSI: error: instruction not supported on this GPU
+// CI: flat_atomic_fmax_x2 v[1:2], v[3:4], v[5:6] glc ; encoding: [0x00,0x00,0x81,0xdd,0x03,0x05,0x00,0x01]
+// NOVI: error: instruction not supported on this GPU
diff --git a/test/MC/AMDGPU/hsa.s b/test/MC/AMDGPU/hsa.s
new file mode 100644
index 00000000000..7dfea0fe787
--- /dev/null
+++ b/test/MC/AMDGPU/hsa.s
@@ -0,0 +1,233 @@
+// RUN: llvm-mc -triple amdgcn--amdhsa -mcpu=kaveri -show-encoding %s | FileCheck %s --check-prefix=ASM
+// RUN: llvm-mc -filetype=obj -triple amdgcn--amdhsa -mcpu=kaveri -show-encoding %s | llvm-readobj -s -sd | FileCheck %s --check-prefix=ELF
+
+// ELF: SHT_NOTE
+// ELF: 0000: 04000000 08000000 01000000 414D4400
+// ELF: 0010: 01000000 00000000 04000000 1B000000
+// ELF: 0020: 03000000 414D4400 04000700 07000000
+// ELF: 0030: 00000000 00000000 414D4400 414D4447
+// ELF: 0040: 50550000
+
+.hsa_code_object_version 1,0
+// ASM: .hsa_code_object_version 1,0
+
+.hsa_code_object_isa 7,0,0,"AMD","AMDGPU"
+// ASM: .hsa_code_object_isa 7,0,0,"AMD","AMDGPU"
+
+.text
+amd_kernel_code_t_test_all:
+; Test all amd_kernel_code_t members with non-default values.
+.amd_kernel_code_t
+	kernel_code_version_major = 100
+	kernel_code_version_minor = 100
+	machine_kind = 0
+	machine_version_major = 5
+	machine_version_minor = 5
+	machine_version_stepping = 5
+	kernel_code_entry_byte_offset = 512
+	kernel_code_prefetch_byte_size = 1
+	max_scratch_backing_memory_byte_size = 1
+	compute_pgm_rsrc1_vgprs = 1
+	compute_pgm_rsrc1_sgprs = 1
+	compute_pgm_rsrc1_priority = 1
+	compute_pgm_rsrc1_float_mode = 1
+	compute_pgm_rsrc1_priv = 1
+	compute_pgm_rsrc1_dx10_clamp = 1
+	compute_pgm_rsrc1_debug_mode = 1
+	compute_pgm_rsrc1_ieee_mode = 1
+	compute_pgm_rsrc2_scratch_en = 1
+	compute_pgm_rsrc2_user_sgpr = 1
+	compute_pgm_rsrc2_tgid_x_en = 1
+	compute_pgm_rsrc2_tgid_y_en = 1
+	compute_pgm_rsrc2_tgid_z_en = 1
+	compute_pgm_rsrc2_tg_size_en = 1
+	compute_pgm_rsrc2_tidig_comp_cnt = 1
+	compute_pgm_rsrc2_excp_en_msb = 1
+	compute_pgm_rsrc2_lds_size = 1
+	compute_pgm_rsrc2_excp_en = 1
+	enable_sgpr_private_segment_buffer = 1
+	enable_sgpr_dispatch_ptr = 1
+	enable_sgpr_queue_ptr = 1
+	enable_sgpr_kernarg_segment_ptr = 1
+	enable_sgpr_dispatch_id = 1
+	enable_sgpr_flat_scratch_init = 1
+	enable_sgpr_private_segment_size = 1
+	enable_sgpr_grid_workgroup_count_x = 1
+	enable_sgpr_grid_workgroup_count_y = 1
+	enable_sgpr_grid_workgroup_count_z = 1
+	enable_ordered_append_gds = 1
+	private_element_size = 1
+	is_ptr64 = 1
+	is_dynamic_callstack = 1
+	is_debug_enabled = 1
+	is_xnack_enabled = 1
+	workitem_private_segment_byte_size = 1
+	workgroup_group_segment_byte_size = 1
+	gds_segment_byte_size = 1
+	kernarg_segment_byte_size = 1
+	workgroup_fbarrier_count = 1
+	wavefront_sgpr_count = 1
+	workitem_vgpr_count = 1
+	reserved_vgpr_first = 1
+	reserved_vgpr_count = 1
+	reserved_sgpr_first = 1
+	reserved_sgpr_count = 1
+	debug_wavefront_private_segment_offset_sgpr = 1
+	debug_private_segment_buffer_sgpr = 1
+	kernarg_segment_alignment = 5
+	group_segment_alignment = 5
+	private_segment_alignment = 5
+	wavefront_size = 5
+	call_convention = 1
+	runtime_loader_kernel_symbol = 1
+.end_amd_kernel_code_t
+
+// ASM-LABEL: {{^}}amd_kernel_code_t_test_all:
+// ASM: .amd_kernel_code_t
+// ASM: kernel_code_version_major = 100
+// ASM: kernel_code_version_minor = 100
+// ASM: machine_kind = 0
+// ASM: machine_version_major = 5
+// ASM: machine_version_minor = 5
+// ASM: machine_version_stepping = 5
+// ASM: kernel_code_entry_byte_offset = 512
+// ASM: kernel_code_prefetch_byte_size = 1
+// ASM: max_scratch_backing_memory_byte_size = 1
+// ASM: compute_pgm_rsrc1_vgprs = 1
+// ASM: compute_pgm_rsrc1_sgprs = 1
+// ASM: compute_pgm_rsrc1_priority = 1
+// ASM: compute_pgm_rsrc1_float_mode = 1 
+// ASM: compute_pgm_rsrc1_priv = 1
+// ASM: compute_pgm_rsrc1_dx10_clamp = 1
+// ASM: compute_pgm_rsrc1_debug_mode = 1 
+// ASM: compute_pgm_rsrc1_ieee_mode = 1
+// ASM: compute_pgm_rsrc2_scratch_en = 1
+// ASM: compute_pgm_rsrc2_user_sgpr = 1
+// ASM: compute_pgm_rsrc2_tgid_x_en = 1
+// ASM: compute_pgm_rsrc2_tgid_y_en = 1
+// ASM: compute_pgm_rsrc2_tgid_z_en = 1
+// ASM: compute_pgm_rsrc2_tg_size_en = 1
+// ASM: compute_pgm_rsrc2_tidig_comp_cnt = 1
+// ASM: compute_pgm_rsrc2_excp_en_msb = 1
+// ASM: compute_pgm_rsrc2_lds_size = 1
+// ASM: compute_pgm_rsrc2_excp_en = 1
+// ASM: enable_sgpr_private_segment_buffer = 1
+// ASM: enable_sgpr_dispatch_ptr = 1
+// ASM: enable_sgpr_queue_ptr = 1
+// ASM: enable_sgpr_kernarg_segment_ptr = 1
+// ASM: enable_sgpr_dispatch_id = 1
+// ASM: enable_sgpr_flat_scratch_init = 1
+// ASM: enable_sgpr_private_segment_size = 1
+// ASM: enable_sgpr_grid_workgroup_count_x = 1
+// ASM: enable_sgpr_grid_workgroup_count_y = 1
+// ASM: enable_sgpr_grid_workgroup_count_z = 1
+// ASM: enable_ordered_append_gds = 1
+// ASM: private_element_size = 1
+// ASM: is_ptr64 = 1
+// ASM: is_dynamic_callstack = 1
+// ASM: is_debug_enabled = 1
+// ASM: is_xnack_enabled = 1
+// ASM: workitem_private_segment_byte_size = 1
+// ASM: workgroup_group_segment_byte_size = 1
+// ASM: gds_segment_byte_size = 1
+// ASM: kernarg_segment_byte_size = 1
+// ASM: workgroup_fbarrier_count = 1
+// ASM: wavefront_sgpr_count = 1
+// ASM: workitem_vgpr_count = 1
+// ASM: reserved_vgpr_first = 1
+// ASM: reserved_vgpr_count = 1
+// ASM: reserved_sgpr_first = 1
+// ASM: reserved_sgpr_count = 1
+// ASM: debug_wavefront_private_segment_offset_sgpr = 1
+// ASM: debug_private_segment_buffer_sgpr = 1
+// ASM: kernarg_segment_alignment = 5
+// ASM: group_segment_alignment = 5
+// ASM: private_segment_alignment = 5
+// ASM: wavefront_size = 5
+// ASM: call_convention = 1
+// ASM: runtime_loader_kernel_symbol = 1
+// ASM: .end_amd_kernel_code_t
+
+amd_kernel_code_t_minimal:
+.amd_kernel_code_t
+	enable_sgpr_kernarg_segment_ptr = 1
+	is_ptr64 = 1
+	compute_pgm_rsrc1_vgprs = 1
+	compute_pgm_rsrc1_sgprs = 1
+	compute_pgm_rsrc2_user_sgpr = 2
+	kernarg_segment_byte_size = 16
+	wavefront_sgpr_count = 8
+//      wavefront_sgpr_count = 7
+;	wavefront_sgpr_count = 7
+// Make sure a blank line won't break anything:
+
+// Make sure a line with whitespace won't break anything:
+   
+	workitem_vgpr_count = 16
+.end_amd_kernel_code_t
+
+// ASM-LABEL: {{^}}amd_kernel_code_t_minimal:
+// ASM: .amd_kernel_code_t
+// ASM:	kernel_code_version_major = 1
+// ASM:	kernel_code_version_minor = 0
+// ASM:	machine_kind = 1
+// ASM:	machine_version_major = 7
+// ASM:	machine_version_minor = 0
+// ASM:	machine_version_stepping = 0
+// ASM:	kernel_code_entry_byte_offset = 256
+// ASM:	kernel_code_prefetch_byte_size = 0
+// ASM:	max_scratch_backing_memory_byte_size = 0
+// ASM:	compute_pgm_rsrc1_vgprs = 1
+// ASM:	compute_pgm_rsrc1_sgprs = 1
+// ASM:	compute_pgm_rsrc1_priority = 0
+// ASM:	compute_pgm_rsrc1_float_mode = 0
+// ASM:	compute_pgm_rsrc1_priv = 0
+// ASM:	compute_pgm_rsrc1_dx10_clamp = 0
+// ASM:	compute_pgm_rsrc1_debug_mode = 0
+// ASM:	compute_pgm_rsrc1_ieee_mode = 0
+// ASM:	compute_pgm_rsrc2_scratch_en = 0
+// ASM:	compute_pgm_rsrc2_user_sgpr = 2
+// ASM:	compute_pgm_rsrc2_tgid_x_en = 0
+// ASM:	compute_pgm_rsrc2_tgid_y_en = 0
+// ASM:	compute_pgm_rsrc2_tgid_z_en = 0
+// ASM:	compute_pgm_rsrc2_tg_size_en = 0
+// ASM:	compute_pgm_rsrc2_tidig_comp_cnt = 0
+// ASM:	compute_pgm_rsrc2_excp_en_msb = 0
+// ASM:	compute_pgm_rsrc2_lds_size = 0
+// ASM:	compute_pgm_rsrc2_excp_en = 0
+// ASM:	enable_sgpr_private_segment_buffer = 0
+// ASM:	enable_sgpr_dispatch_ptr = 0
+// ASM:	enable_sgpr_queue_ptr = 0
+// ASM:	enable_sgpr_kernarg_segment_ptr = 1
+// ASM:	enable_sgpr_dispatch_id = 0
+// ASM:	enable_sgpr_flat_scratch_init = 0
+// ASM:	enable_sgpr_private_segment_size = 0
+// ASM:	enable_sgpr_grid_workgroup_count_x = 0
+// ASM:	enable_sgpr_grid_workgroup_count_y = 0
+// ASM:	enable_sgpr_grid_workgroup_count_z = 0
+// ASM:	enable_ordered_append_gds = 0
+// ASM:	private_element_size = 0
+// ASM:	is_ptr64 = 1
+// ASM:	is_dynamic_callstack = 0
+// ASM:	is_debug_enabled = 0
+// ASM:	is_xnack_enabled = 0
+// ASM:	workitem_private_segment_byte_size = 0
+// ASM:	workgroup_group_segment_byte_size = 0
+// ASM:	gds_segment_byte_size = 0
+// ASM:	kernarg_segment_byte_size = 16
+// ASM:	workgroup_fbarrier_count = 0
+// ASM:	wavefront_sgpr_count = 8
+// ASM:	workitem_vgpr_count = 16
+// ASM:	reserved_vgpr_first = 0
+// ASM:	reserved_vgpr_count = 0
+// ASM:	reserved_sgpr_first = 0
+// ASM:	reserved_sgpr_count = 0
+// ASM:	debug_wavefront_private_segment_offset_sgpr = 0
+// ASM:	debug_private_segment_buffer_sgpr = 0
+// ASM:	kernarg_segment_alignment = 4
+// ASM:	group_segment_alignment = 4
+// ASM:	private_segment_alignment = 4
+// ASM:	wavefront_size = 6
+// ASM:	call_convention = 0
+// ASM:	runtime_loader_kernel_symbol = 0
+// ASM: .end_amd_kernel_code_t
diff --git a/test/MC/AMDGPU/hsa_code_object_isa_noargs.s b/test/MC/AMDGPU/hsa_code_object_isa_noargs.s
new file mode 100644
index 00000000000..85f53bb697b
--- /dev/null
+++ b/test/MC/AMDGPU/hsa_code_object_isa_noargs.s
@@ -0,0 +1,16 @@
+// RUN: llvm-mc -triple amdgcn--amdhsa -mcpu=kaveri -show-encoding %s | FileCheck %s --check-prefix=ASM
+// RUN: llvm-mc -filetype=obj -triple amdgcn--amdhsa -mcpu=kaveri -show-encoding %s | llvm-readobj -s -sd | FileCheck %s --check-prefix=ELF
+
+// ELF: SHT_NOTE
+// ELF: 0000: 04000000 08000000 01000000 414D4400
+// ELF: 0010: 01000000 00000000 04000000 1B000000
+// ELF: 0020: 03000000 414D4400 04000700 07000000
+// ELF: 0030: 00000000 00000000 414D4400 414D4447
+// ELF: 0040: 50550000
+
+.hsa_code_object_version 1,0
+// ASM: .hsa_code_object_version 1,0
+
+.hsa_code_object_isa
+// ASM: .hsa_code_object_isa 7,0,0,"AMD","AMDGPU"
+
diff --git a/test/MC/AMDGPU/lit.local.cfg b/test/MC/AMDGPU/lit.local.cfg
new file mode 100644
index 00000000000..2a665f06be7
--- /dev/null
+++ b/test/MC/AMDGPU/lit.local.cfg
@@ -0,0 +1,2 @@
+if not 'AMDGPU' in config.root.targets:
+    config.unsupported = True
diff --git a/test/MC/R600/mubuf.s b/test/MC/AMDGPU/mubuf.s
similarity index 100%
rename from test/MC/R600/mubuf.s
rename to test/MC/AMDGPU/mubuf.s
diff --git a/test/MC/R600/smrd.s b/test/MC/AMDGPU/smrd.s
similarity index 100%
rename from test/MC/R600/smrd.s
rename to test/MC/AMDGPU/smrd.s
diff --git a/test/MC/R600/sop1-err.s b/test/MC/AMDGPU/sop1-err.s
similarity index 100%
rename from test/MC/R600/sop1-err.s
rename to test/MC/AMDGPU/sop1-err.s
diff --git a/test/MC/R600/sop1.s b/test/MC/AMDGPU/sop1.s
similarity index 100%
rename from test/MC/R600/sop1.s
rename to test/MC/AMDGPU/sop1.s
diff --git a/test/MC/R600/sop2.s b/test/MC/AMDGPU/sop2.s
similarity index 100%
rename from test/MC/R600/sop2.s
rename to test/MC/AMDGPU/sop2.s
diff --git a/test/MC/R600/sopc.s b/test/MC/AMDGPU/sopc.s
similarity index 100%
rename from test/MC/R600/sopc.s
rename to test/MC/AMDGPU/sopc.s
diff --git a/test/MC/R600/sopk.s b/test/MC/AMDGPU/sopk.s
similarity index 100%
rename from test/MC/R600/sopk.s
rename to test/MC/AMDGPU/sopk.s
diff --git a/test/MC/R600/sopp.s b/test/MC/AMDGPU/sopp.s
similarity index 100%
rename from test/MC/R600/sopp.s
rename to test/MC/AMDGPU/sopp.s
diff --git a/test/MC/AMDGPU/vop1.s b/test/MC/AMDGPU/vop1.s
new file mode 100644
index 00000000000..d0b00fcd189
--- /dev/null
+++ b/test/MC/AMDGPU/vop1.s
@@ -0,0 +1,357 @@
+// RUN: not llvm-mc -arch=amdgcn -show-encoding %s | FileCheck %s --check-prefix=GCN --check-prefix=SI --check-prefix=SICI
+// RUN: not llvm-mc -arch=amdgcn -mcpu=SI -show-encoding %s | FileCheck %s --check-prefix=GCN --check-prefix=SI --check-prefix=SICI
+// RUN: not llvm-mc -arch=amdgcn -mcpu=bonaire -show-encoding %s | FileCheck %s --check-prefix=GCN --check-prefix=SICI --check-prefix=CIVI
+// RUN: not llvm-mc -arch=amdgcn -mcpu=tonga -show-encoding %s | FileCheck %s --check-prefix=GCN --check-prefix=CIVI --check-prefix=VI
+
+// RUN: not llvm-mc -arch=amdgcn -show-encoding %s 2>&1 | FileCheck %s --check-prefix=NOSI --check-prefix=NOSICI
+// RUN: not llvm-mc -arch=amdgcn -mcpu=SI -show-encoding %s 2>&1 | FileCheck %s --check-prefix=NOSI --check-prefix=NOSICI
+// RUN: not llvm-mc -arch=amdgcn -mcpu=bonaire -show-encoding %s 2>&1 | FileCheck %s --check-prefix=NOSICI
+// RUN: not llvm-mc -arch=amdgcn -mcpu=tonga -show-encoding %s 2>&1 | FileCheck %s -check-prefix=NOVI
+
+
+// GCN: v_nop ; encoding: [0x00,0x00,0x00,0x7e]
+v_nop
+
+// GCN: v_mov_b32_e32 v1, v2 ; encoding: [0x02,0x03,0x02,0x7e]
+v_mov_b32 v1, v2
+
+// GCN: v_readfirstlane_b32 s1, v2 ; encoding: [0x02,0x05,0x02,0x7e]
+v_readfirstlane_b32 s1, v2
+
+// GCN: v_cvt_i32_f64_e32 v1, v[2:3] ; encoding: [0x02,0x07,0x02,0x7e]
+v_cvt_i32_f64 v1, v[2:3]
+
+// GCN: v_cvt_f64_i32_e32 v[1:2], v2 ; encoding: [0x02,0x09,0x02,0x7e]
+v_cvt_f64_i32 v[1:2], v2
+
+// GCN: v_cvt_f32_i32_e32 v1, v2 ; encoding: [0x02,0x0b,0x02,0x7e]
+v_cvt_f32_i32 v1, v2
+
+// GCN: v_cvt_f32_u32_e32 v1, v2 ; encoding: [0x02,0x0d,0x02,0x7e]
+v_cvt_f32_u32 v1, v2
+
+// GCN: v_cvt_u32_f32_e32 v1, v2 ; encoding: [0x02,0x0f,0x02,0x7e
+v_cvt_u32_f32 v1, v2
+
+// GCN: v_cvt_i32_f32_e32 v1, v2 ; encoding: [0x02,0x11,0x02,0x7e]
+v_cvt_i32_f32 v1, v2
+
+// SICI: v_mov_fed_b32_e32 v1, v2 ; encoding: [0x02,0x13,0x02,0x7e]
+// NOVI: error: instruction not supported on this GPU
+v_mov_fed_b32 v1, v2
+
+// GCN: v_cvt_f16_f32_e32 v1, v2 ; encoding: [0x02,0x15,0x02,0x7e]
+v_cvt_f16_f32 v1, v2
+
+// GCN: v_cvt_f32_f16_e32 v1, v2 ; encoding: [0x02,0x17,0x02,0x7e]
+v_cvt_f32_f16 v1, v2
+
+// GCN: v_cvt_rpi_i32_f32_e32 v1, v2 ; encoding: [0x02,0x19,0x02,0x7e]
+v_cvt_rpi_i32_f32 v1, v2
+
+// GCN: v_cvt_flr_i32_f32_e32 v1, v2 ; encoding: [0x02,0x1b,0x02,0x7e]
+v_cvt_flr_i32_f32 v1, v2
+
+// GCN: v_cvt_off_f32_i4_e32 v1, v2 ; encoding: [0x02,0x1d,0x02,0x7e]
+v_cvt_off_f32_i4_e32 v1, v2
+
+// GCN: v_cvt_f32_f64_e32 v1, v[2:3] ; encoding: [0x02,0x1f,0x02,0x7e]
+v_cvt_f32_f64 v1, v[2:3]
+
+// GCN: v_cvt_f64_f32_e32 v[1:2], v2 ; encoding: [0x02,0x21,0x02,0x7e]
+v_cvt_f64_f32 v[1:2], v2
+
+// GCN: v_cvt_f32_ubyte0_e32 v1, v2 ; encoding: [0x02,0x23,0x02,0x7e]
+v_cvt_f32_ubyte0 v1, v2
+
+// GCN: v_cvt_f32_ubyte1_e32 v1, v2 ; encoding: [0x02,0x25,0x02,0x7e]
+v_cvt_f32_ubyte1_e32 v1, v2
+
+// GCN: v_cvt_f32_ubyte2_e32 v1, v2 ; encoding: [0x02,0x27,0x02,0x7e]
+v_cvt_f32_ubyte2 v1, v2
+
+// GCN: v_cvt_f32_ubyte3_e32 v1, v2 ; encoding: [0x02,0x29,0x02,0x7e]
+v_cvt_f32_ubyte3 v1, v2
+
+// GCN: v_cvt_u32_f64_e32 v1, v[2:3] ; encoding: [0x02,0x2b,0x02,0x7e]
+v_cvt_u32_f64 v1, v[2:3]
+
+// GCN: v_cvt_f64_u32_e32 v[1:2], v2 ; encoding: [0x02,0x2d,0x02,0x7e]
+v_cvt_f64_u32 v[1:2], v2
+
+// NOSI: error: instruction not supported on this GPU
+// NOSI: v_trunc_f64_e32 v[1:2], v[2:3]
+// CIVI: v_trunc_f64_e32 v[1:2], v[2:3] ; encoding: [0x02,0x2f,0x02,0x7e]
+v_trunc_f64_e32 v[1:2], v[2:3]
+
+// NOSI: error: instruction not supported on this GPU
+// NOSI: v_ceil_f64_e32 v[1:2], v[2:3]
+// CIVI: v_ceil_f64_e32 v[1:2], v[2:3] ; encoding: [0x02,0x31,0x02,0x7e]
+v_ceil_f64_e32 v[1:2], v[2:3]
+
+// NOSI: error: instruction not supported on this GPU
+// NOSI: v_rndne_f64_e32 v[1:2], v[2:3]
+// CIVI: v_rndne_f64_e32 v[1:2], v[2:3] ; encoding: [0x02,0x33,0x02,0x7e]
+v_rndne_f64_e32 v[1:2], v[2:3]
+
+// NOSI: error: instruction not supported on this GPU
+// NOSI: v_floor_f64_e32 v[1:2], v[2:3]
+// CIVI: v_floor_f64_e32 v[1:2], v[2:3] ; encoding: [0x02,0x35,0x02,0x7e]
+v_floor_f64_e32 v[1:2], v[2:3]
+
+// SICI: v_fract_f32_e32 v1, v2 ; encoding: [0x02,0x41,0x02,0x7e]
+// VI:   v_fract_f32_e32 v1, v2 ; encoding: [0x02,0x37,0x02,0x7e]
+v_fract_f32 v1, v2
+
+// SICI: v_trunc_f32_e32 v1, v2 ; encoding: [0x02,0x43,0x02,0x7e]
+// VI:   v_trunc_f32_e32 v1, v2 ; encoding: [0x02,0x39,0x02,0x7e]
+v_trunc_f32 v1, v2
+
+// SICI: v_ceil_f32_e32 v1, v2 ; encoding: [0x02,0x45,0x02,0x7e]
+// VI:   v_ceil_f32_e32 v1, v2 ; encoding: [0x02,0x3b,0x02,0x7e]
+v_ceil_f32 v1, v2
+
+// SICI: v_rndne_f32_e32 v1, v2 ; encoding: [0x02,0x47,0x02,0x7e]
+// VI:   v_rndne_f32_e32 v1, v2 ; encoding: [0x02,0x3d,0x02,0x7e]
+v_rndne_f32 v1, v2
+
+// SICI: v_floor_f32_e32 v1, v2 ; encoding: [0x02,0x49,0x02,0x7e]
+// VI:   v_floor_f32_e32 v1, v2 ; encoding: [0x02,0x3f,0x02,0x7e]
+v_floor_f32_e32 v1, v2
+
+// SICI: v_exp_f32_e32 v1, v2 ; encoding: [0x02,0x4b,0x02,0x7e]
+// VI:   v_exp_f32_e32 v1, v2 ; encoding: [0x02,0x41,0x02,0x7e]
+v_exp_f32 v1, v2
+
+// SICI: v_log_clamp_f32_e32 v1, v2 ; encoding: [0x02,0x4d,0x02,0x7e]
+// NOVI: error: instruction not supported on this GPU
+// NOVI: v_log_clamp_f32 v1, v2
+v_log_clamp_f32 v1, v2
+
+// SICI: v_log_f32_e32 v1, v2 ; encoding: [0x02,0x4f,0x02,0x7e]
+// VI:   v_log_f32_e32 v1, v2 ; encoding: [0x02,0x43,0x02,0x7e]
+v_log_f32 v1, v2
+
+// SICI: v_rcp_clamp_f32_e32 v1, v2 ; encoding: [0x02,0x51,0x02,0x7e]
+// NOVI: error: instruction not supported on this GPU
+// NOVI: v_rcp_clamp_f32 v1, v2
+v_rcp_clamp_f32 v1, v2
+
+// SICI: v_rcp_legacy_f32_e32 v1, v2 ; encoding: [0x02,0x53,0x02,0x7e]
+// NOVI: error: instruction not supported on this GPU
+// NOVI: v_rcp_legacy_f32 v1, v2
+v_rcp_legacy_f32 v1, v2
+
+// SICI: v_rcp_f32_e32 v1, v2 ; encoding: [0x02,0x55,0x02,0x7e]
+// VI:   v_rcp_f32_e32 v1, v2 ; encoding: [0x02,0x45,0x02,0x7e]
+v_rcp_f32 v1, v2
+
+// SICI: v_rcp_iflag_f32_e32 v1, v2 ; encoding: [0x02,0x57,0x02,0x7e]
+// VI:   v_rcp_iflag_f32_e32 v1, v2 ; encoding: [0x02,0x47,0x02,0x7e]
+v_rcp_iflag_f32 v1, v2
+
+// SICI: v_rsq_clamp_f32_e32 v1, v2 ; encoding: [0x02,0x59,0x02,0x7e]
+// NOVI: error: instruction not supported on this GPU
+// NOVI: v_rsq_clamp_f32 v1, v2
+v_rsq_clamp_f32 v1, v2
+
+// SICI: v_rsq_legacy_f32_e32 v1, v2 ; encoding: [0x02,0x5b,0x02,0x7e]
+// NOVI: error: instruction not supported on this GPU
+// NOVI: v_rsq_legacy_f32 v1, v2
+v_rsq_legacy_f32 v1, v2
+
+// SICI: v_rsq_f32_e32 v1, v2 ; encoding: [0x02,0x5d,0x02,0x7e]
+// VI:   v_rsq_f32_e32 v1, v2 ; encoding: [0x02,0x49,0x02,0x7e]
+v_rsq_f32_e32 v1, v2
+
+// SICI: v_rcp_f64_e32 v[1:2], v[2:3] ; encoding: [0x02,0x5f,0x02,0x7e]
+// VI:   v_rcp_f64_e32 v[1:2], v[2:3] ; encoding: [0x02,0x4b,0x02,0x7e]
+v_rcp_f64 v[1:2], v[2:3]
+
+// SICI: v_rcp_clamp_f64_e32 v[1:2], v[2:3] ; encoding: [0x02,0x61,0x02,0x7e]
+// NOVI: error: instruction not supported on this GPU
+// NOVI: v_rcp_clamp_f64 v[1:2], v[2:3]
+v_rcp_clamp_f64 v[1:2], v[2:3]
+
+// SICI: v_rsq_f64_e32 v[1:2], v[2:3] ; encoding: [0x02,0x63,0x02,0x7e]
+// VI:   v_rsq_f64_e32 v[1:2], v[2:3] ; encoding: [0x02,0x4d,0x02,0x7e]
+v_rsq_f64 v[1:2], v[2:3]
+
+// SICI: v_rsq_clamp_f64_e32 v[1:2], v[2:3] ; encoding: [0x02,0x65,0x02,0x7e]
+// NOVI: error: instruction not supported on this GPU
+// NOVI: v_rsq_clamp_f64 v[1:2], v[2:3]
+v_rsq_clamp_f64 v[1:2], v[2:3]
+
+// SICI: v_sqrt_f32_e32 v1, v2 ; encoding: [0x02,0x67,0x02,0x7e]
+// VI:   v_sqrt_f32_e32 v1, v2 ; encoding: [0x02,0x4f,0x02,0x7e]
+v_sqrt_f32 v1, v2
+
+// SICI: v_sqrt_f64_e32 v[1:2], v[2:3] ; encoding: [0x02,0x69,0x02,0x7e]
+// VI:   v_sqrt_f64_e32 v[1:2], v[2:3] ; encoding: [0x02,0x51,0x02,0x7e]
+v_sqrt_f64 v[1:2], v[2:3]
+
+// SICI: v_sin_f32_e32 v1, v2 ; encoding: [0x02,0x6b,0x02,0x7e]
+// VI:   v_sin_f32_e32 v1, v2 ; encoding: [0x02,0x53,0x02,0x7e]
+v_sin_f32 v1, v2
+
+// SICI: v_cos_f32_e32 v1, v2 ; encoding: [0x02,0x6d,0x02,0x7e]
+// VI:   v_cos_f32_e32 v1, v2 ; encoding: [0x02,0x55,0x02,0x7e]
+v_cos_f32 v1, v2
+
+// SICI: v_not_b32_e32 v1, v2 ; encoding: [0x02,0x6f,0x02,0x7e]
+// VI:   v_not_b32_e32 v1, v2 ; encoding: [0x02,0x57,0x02,0x7e]
+v_not_b32 v1, v2
+
+// SICI: v_bfrev_b32_e32 v1, v2 ; encoding: [0x02,0x71,0x02,0x7e]
+// VI:   v_bfrev_b32_e32 v1, v2 ; encoding: [0x02,0x59,0x02,0x7e]
+v_bfrev_b32 v1, v2
+
+// SICI: v_ffbh_u32_e32 v1, v2 ; encoding: [0x02,0x73,0x02,0x7e]
+// VI:   v_ffbh_u32_e32 v1, v2 ; encoding: [0x02,0x5b,0x02,0x7e]
+v_ffbh_u32 v1, v2
+
+// SICI: v_ffbl_b32_e32 v1, v2 ; encoding: [0x02,0x75,0x02,0x7e]
+// VI:   v_ffbl_b32_e32 v1, v2 ; encoding: [0x02,0x5d,0x02,0x7e]
+v_ffbl_b32 v1, v2
+
+// SICI: v_ffbh_i32_e32 v1, v2 ; encoding: [0x02,0x77,0x02,0x7e]
+// VI:   v_ffbh_i32_e32 v1, v2 ; encoding: [0x02,0x5f,0x02,0x7e]
+v_ffbh_i32_e32 v1, v2
+
+// SICI: v_frexp_exp_i32_f64_e32 v1, v[2:3] ; encoding: [0x02,0x79,0x02,0x7e]
+// VI:   v_frexp_exp_i32_f64_e32 v1, v[2:3] ; encoding: [0x02,0x61,0x02,0x7e]
+v_frexp_exp_i32_f64 v1, v[2:3]
+
+// SICI: v_frexp_mant_f64_e32 v[1:2], v[2:3] ; encoding: [0x02,0x7b,0x02,0x7e]
+// VI;   v_frexp_mant_f64_e32 v[1:2], v[2:3] ; encoding: [0x02,0x63,0x02,0x7e]
+v_frexp_mant_f64 v[1:2], v[2:3]
+
+// SICI: v_fract_f64_e32 v[1:2], v[2:3] ; encoding: [0x02,0x7d,0x02,0x7e]
+// VI:   v_fract_f64_e32 v[1:2], v[2:3] ; encoding: [0x02,0x65,0x02,0x7e]
+v_fract_f64 v[1:2], v[2:3]
+
+// SICI: v_frexp_exp_i32_f32_e32 v1, v2 ; encoding: [0x02,0x7f,0x02,0x7e]
+// VI:   v_frexp_exp_i32_f32_e32 v1, v2 ; encoding: [0x02,0x67,0x02,0x7e]
+v_frexp_exp_i32_f32 v1, v2
+
+// SICI: v_frexp_mant_f32_e32 v1, v2 ; encoding: [0x02,0x81,0x02,0x7e]
+// VI:   v_frexp_mant_f32_e32 v1, v2 ; encoding: [0x02,0x69,0x02,0x7e]
+v_frexp_mant_f32 v1, v2
+
+// SICI: v_clrexcp ; encoding: [0x00,0x82,0x00,0x7e]
+// VI:   v_clrexcp ; encoding: [0x00,0x6a,0x00,0x7e]
+v_clrexcp
+
+// SICI: v_movreld_b32_e32 v1, v2 ; encoding: [0x02,0x85,0x02,0x7e]
+// VI:   v_movreld_b32_e32 v1, v2 ; encoding: [0x02,0x6d,0x02,0x7e]
+v_movreld_b32 v1, v2
+
+// SICI: v_movrels_b32_e32 v1, v2 ; encoding: [0x02,0x87,0x02,0x7e]
+// VI:   v_movrels_b32_e32 v1, v2 ; encoding: [0x02,0x6f,0x02,0x7e]
+v_movrels_b32 v1, v2
+
+// SICI: v_movrelsd_b32_e32 v1, v2 ; encoding: [0x02,0x89,0x02,0x7e]
+// VI:   v_movrelsd_b32_e32 v1, v2 ; encoding: [0x02,0x71,0x02,0x7e]
+v_movrelsd_b32 v1, v2
+
+// NOSI: error: instruction not supported on this GPU
+// NOSI: v_log_legacy_f32 v1, v2
+// CI: v_log_legacy_f32_e32 v1, v2 ; encoding: [0x02,0x8b,0x02,0x7e]
+// VI: v_log_legacy_f32_e32 v1, v2 ; encoding: [0x02,0x99,0x02,0x7e]
+v_log_legacy_f32 v1, v2
+
+// NOSI: error: instruction not supported on this GPU
+// NOSI: v_exp_legacy_f32 v1, v2
+// CI: v_exp_legacy_f32_e32 v1, v2 ; encoding: [0x02,0x8d,0x02,0x7e]
+// VI: v_exp_legacy_f32_e32 v1, v2 ; encoding: [0x02,0x97,0x02,0x7e]
+v_exp_legacy_f32 v1, v2
+
+// NOSICI: error: instruction not supported on this GPU
+// NOSICI: v_cvt_f16_u16 v1, v2
+// VI: v_cvt_f16_u16_e32 v1, v2 ; encoding: [0x02,0x73,0x02,0x7e]
+v_cvt_f16_u16 v1, v2
+
+// NOSICI: error: instruction not supported on this GPU
+// NOSICI: v_cvt_f16_i16 v1, v2
+// VI: v_cvt_f16_i16_e32 v1, v2 ; encoding: [0x02,0x75,0x02,0x7e]
+v_cvt_f16_i16 v1, v2
+
+// NOSICI: error: instruction not supported on this GPU
+// NOSICI: v_cvt_u16_f16 v1, v2
+// VI: v_cvt_u16_f16_e32 v1, v2 ; encoding: [0x02,0x77,0x02,0x7e]
+v_cvt_u16_f16 v1, v2
+
+// NOSICI: error: instruction not supported on this GPU
+// NOSICI: v_cvt_i16_f16 v1, v2
+// VI: v_cvt_i16_f16_e32 v1, v2 ; encoding: [0x02,0x79,0x02,0x7e]
+v_cvt_i16_f16 v1, v2
+
+// NOSICI: error: instruction not supported on this GPU
+// NOSICI: v_rcp_f16 v1, v2
+// VI: v_rcp_f16_e32 v1, v2 ; encoding: [0x02,0x7b,0x02,0x7e]
+v_rcp_f16 v1, v2
+
+// NOSICI: error: instruction not supported on this GPU
+// NOSICI: v_sqrt_f16 v1, v2
+// VI: v_sqrt_f16_e32 v1, v2 ; encoding: [0x02,0x7d,0x02,0x7e]
+v_sqrt_f16 v1, v2
+
+// NOSICI: error: instruction not supported on this GPU
+// NOSICI: v_rsq_f16 v1, v2
+// VI: v_rsq_f16_e32 v1, v2 ; encoding: [0x02,0x7f,0x02,0x7e]
+v_rsq_f16 v1, v2
+
+// NOSICI: error: instruction not supported on this GPU
+// NOSICI: v_log_f16 v1, v2
+// VI: v_log_f16_e32 v1, v2 ; encoding: [0x02,0x81,0x02,0x7e]
+v_log_f16 v1, v2
+
+// NOSICI: error: instruction not supported on this GPU
+// NOSICI: v_exp_f16 v1, v2
+// VI: v_exp_f16_e32 v1, v2 ; encoding: [0x02,0x83,0x02,0x7e]
+v_exp_f16 v1, v2
+
+// NOSICI: error: instruction not supported on this GPU
+// NOSICI: v_frexp_mant_f16 v1, v2
+// VI: v_frexp_mant_f16_e32 v1, v2 ; encoding: [0x02,0x85,0x02,0x7e]
+v_frexp_mant_f16 v1, v2
+
+// NOSICI: error: instruction not supported on this GPU
+// NOSICI: v_frexp_exp_i16_f16 v1, v2
+// VI: v_frexp_exp_i16_f16_e32 v1, v2 ; encoding: [0x02,0x87,0x02,0x7e]
+v_frexp_exp_i16_f16 v1, v2
+
+// NOSICI: error: instruction not supported on this GPU
+// NOSICI: v_floor_f16 v1, v2
+// VI: v_floor_f16_e32 v1, v2 ; encoding: [0x02,0x89,0x02,0x7e]
+v_floor_f16 v1, v2
+
+// NOSICI: error: instruction not supported on this GPU
+// NOSICI: v_ceil_f16 v1, v2
+// VI: v_ceil_f16_e32 v1, v2 ; encoding: [0x02,0x8b,0x02,0x7e]
+v_ceil_f16 v1, v2
+
+// NOSICI: error: instruction not supported on this GPU
+// NOSICI: v_trunc_f16 v1, v2
+// VI: v_trunc_f16_e32 v1, v2 ; encoding: [0x02,0x8d,0x02,0x7e]
+v_trunc_f16 v1, v2
+
+// NOSICI: error: instruction not supported on this GPU
+// NOSICI: v_rndne_f16 v1, v2
+// VI: v_rndne_f16_e32 v1, v2 ; encoding: [0x02,0x8f,0x02,0x7e]
+v_rndne_f16 v1, v2
+
+// NOSICI: error: instruction not supported on this GPU
+// NOSICI: v_fract_f16 v1, v2
+// VI: v_fract_f16_e32 v1, v2 ; encoding: [0x02,0x91,0x02,0x7e]
+v_fract_f16 v1, v2
+
+// NOSICI: error: instruction not supported on this GPU
+// NOSICI: v_sin_f16 v1, v2
+// VI: v_sin_f16_e32 v1, v2 ; encoding: [0x02,0x93,0x02,0x7e]
+v_sin_f16 v1, v2
+
+// NOSICI: error: instruction not supported on this GPU
+// NOSICI: v_cos_f16 v1, v2
+// VI: v_cos_f16_e32 v1, v2 ; encoding: [0x02,0x95,0x02,0x7e]
+v_cos_f16 v1, v2
diff --git a/test/MC/R600/vop2-err.s b/test/MC/AMDGPU/vop2-err.s
similarity index 100%
rename from test/MC/R600/vop2-err.s
rename to test/MC/AMDGPU/vop2-err.s
diff --git a/test/MC/AMDGPU/vop2.s b/test/MC/AMDGPU/vop2.s
new file mode 100644
index 00000000000..a1f3b8d8936
--- /dev/null
+++ b/test/MC/AMDGPU/vop2.s
@@ -0,0 +1,421 @@
+// RUN: not llvm-mc -arch=amdgcn -show-encoding %s | FileCheck %s --check-prefix=GCN --check-prefix=SICI
+// RUN: not llvm-mc -arch=amdgcn -mcpu=SI -show-encoding %s | FileCheck %s --check-prefix=GCN --check-prefix=SICI
+// RUN: not llvm-mc -arch=amdgcn -mcpu=bonaire -show-encoding %s | FileCheck %s --check-prefix=GCN --check-prefix=SICI
+// RUN: not llvm-mc -arch=amdgcn -mcpu=tonga -show-encoding %s | FileCheck %s --check-prefix=GCN --check-prefix=CIVI --check-prefix=VI
+
+// RUN: not llvm-mc -arch=amdgcn -show-encoding %s 2>&1 | FileCheck %s --check-prefix=NOSICI
+// RUN: not llvm-mc -arch=amdgcn -mcpu=SI -show-encoding %s 2>&1 | FileCheck %s --check-prefix=NOSICI
+// RUN: not llvm-mc -arch=amdgcn -mcpu=bonaire -show-encoding %s 2>&1 | FileCheck %s --check-prefix=NOSICI
+// RUN: not llvm-mc -arch=amdgcn -mcpu=tonga -show-encoding %s 2>&1 | FileCheck %s -check-prefix=NOVI
+
+//===----------------------------------------------------------------------===//
+// Generic Checks for floating-point instructions (These have modifiers).
+//===----------------------------------------------------------------------===//
+
+// TODO: 64-bit encoding of instructions with modifiers
+
+// _e32 suffix
+// SICI: v_add_f32_e32 v1, v2, v3 ; encoding: [0x02,0x07,0x02,0x06]
+v_add_f32_e32 v1, v2, v3
+
+// src0 inline immediate
+// SICI: v_add_f32_e32 v1, 1.0, v3 ; encoding: [0xf2,0x06,0x02,0x06]
+v_add_f32 v1, 1.0, v3
+
+// src0 negative inline immediate
+// SICI: v_add_f32_e32 v1, -1.0, v3 ; encoding: [0xf3,0x06,0x02,0x06]
+v_add_f32 v1, -1.0, v3
+
+// src0 literal
+// SICI: v_add_f32_e32 v1, 0x42c80000, v3 ; encoding: [0xff,0x06,0x02,0x06,0x00,0x00,0xc8,0x42]
+v_add_f32 v1, 100.0, v3
+
+// src0 negative literal
+// SICI: v_add_f32_e32 v1, 0xc2c80000, v3 ; encoding: [0xff,0x06,0x02,0x06,0x00,0x00,0xc8,0xc2]
+v_add_f32 v1, -100.0, v3
+
+//===----------------------------------------------------------------------===//
+// Generic Checks for integer instructions (These don't have modifiers).
+//===----------------------------------------------------------------------===//
+
+// _e32 suffix
+// SICI: v_mul_i32_i24_e32 v1, v2, v3 ; encoding: [0x02,0x07,0x02,0x12]
+v_mul_i32_i24_e32 v1, v2, v3
+
+// _e64 suffix
+// SICI: v_mul_i32_i24_e64 v1, v2, v3 ; encoding: [0x01,0x00,0x12,0xd2,0x02,0x07,0x02,0x00]
+v_mul_i32_i24_e64 v1, v2, v3
+
+// src0 inline
+// SICI: v_mul_i32_i24_e32 v1, 3, v3 ; encoding: [0x83,0x06,0x02,0x12]
+v_mul_i32_i24 v1, 3, v3
+
+// src0 negative inline
+// SICI: v_mul_i32_i24_e32 v1, -3, v3 ; encoding: [0xc3,0x06,0x02,0x12]
+v_mul_i32_i24 v1, -3, v3
+
+// src1 inline
+// SICI: v_mul_i32_i24_e64 v1, v2, 3 ; encoding: [0x01,0x00,0x12,0xd2,0x02,0x07,0x01,0x00]
+v_mul_i32_i24 v1, v2, 3
+
+// src1 negative inline
+// SICI: v_mul_i32_i24_e64 v1, v2, -3 ; encoding: [0x01,0x00,0x12,0xd2,0x02,0x87,0x01,0x00]
+v_mul_i32_i24 v1, v2, -3
+
+// src0 literal
+// SICI: v_mul_i32_i24_e32 v1, 0x64, v3 ; encoding: [0xff,0x06,0x02,0x12,0x64,0x00,0x00,0x00]
+v_mul_i32_i24 v1, 100, v3
+
+// src1 negative literal
+// SICI: v_mul_i32_i24_e32 v1, 0xffffff9c, v3 ; encoding: [0xff,0x06,0x02,0x12,0x9c,0xff,0xff,0xff]
+v_mul_i32_i24 v1, -100, v3
+
+//===----------------------------------------------------------------------===//
+// Checks for legal operands
+//===----------------------------------------------------------------------===//
+
+// src0 sgpr
+// SICI: v_mul_i32_i24_e32 v1, s2, v3 ; encoding: [0x02,0x06,0x02,0x12]
+v_mul_i32_i24 v1, s2, v3
+
+// src1 sgpr
+// SICI: v_mul_i32_i24_e64 v1, v2, s3 ; encoding: [0x01,0x00,0x12,0xd2,0x02,0x07,0x00,0x00]
+v_mul_i32_i24 v1, v2, s3
+
+// src0, src1 same sgpr
+// SICI: v_mul_i32_i24_e64 v1, s2, s2 ; encoding: [0x01,0x00,0x12,0xd2,0x02,0x04,0x00,0x00]
+v_mul_i32_i24 v1, s2, s2
+
+// src0 sgpr, src1 inline
+// SICI: v_mul_i32_i24_e64 v1, s2, 3 ; encoding: [0x01,0x00,0x12,0xd2,0x02,0x06,0x01,0x00]
+v_mul_i32_i24 v1, s2, 3
+
+// src0 inline src1 sgpr
+// SICI: v_mul_i32_i24_e64 v1, 3, s3 ; encoding: [0x01,0x00,0x12,0xd2,0x83,0x06,0x00,0x00]
+v_mul_i32_i24 v1, 3, s3
+
+//===----------------------------------------------------------------------===//
+// Instructions
+//===----------------------------------------------------------------------===//
+
+// GCN: v_cndmask_b32_e32 v1, v2, v3 ; encoding: [0x02,0x07,0x02,0x00]
+v_cndmask_b32 v1, v2, v3
+
+// SICI: v_readlane_b32 s1, v2, s3 ; encoding: [0x02,0x07,0x02,0x02]
+// VI:   v_readlane_b32 s1, v2, s3 ; encoding: [0x01,0x00,0x89,0xd2,0x02,0x07,0x00,0x00]
+v_readlane_b32 s1, v2, s3
+
+// SICI: v_writelane_b32 v1, s2, s3 ; encoding: [0x02,0x06,0x02,0x04]
+// VI:   v_writelane_b32 v1, s2, s3 ; encoding: [0x01,0x00,0x8a,0xd2,0x02,0x06,0x00,0x00]
+v_writelane_b32 v1, s2, s3
+
+// SICI: v_add_f32_e32 v1, v2, v3 ; encoding: [0x02,0x07,0x02,0x06]
+// VI:   v_add_f32_e32 v1, v2, v3 ; encoding: [0x02,0x07,0x02,0x02]
+v_add_f32 v1, v2, v3
+
+// SICI: v_sub_f32_e32 v1, v2, v3 ; encoding: [0x02,0x07,0x02,0x08]
+// VI:   v_sub_f32_e32 v1, v2, v3 ; encoding: [0x02,0x07,0x02,0x04]
+v_sub_f32 v1, v2, v3
+
+// SICI: v_subrev_f32_e32 v1, v2, v3 ; encoding: [0x02,0x07,0x02,0x0a]
+// VI:   v_subrev_f32_e32 v1, v2, v3 ; encoding: [0x02,0x07,0x02,0x06]
+v_subrev_f32 v1, v2, v3
+
+// SICI: v_mac_legacy_f32_e32 v1, v2, v3 ; encoding: [0x02,0x07,0x02,0x0c]
+// NOVI: error: instruction not supported on this GPU
+// NOVI: v_mac_legacy_f32 v1, v2, v3
+v_mac_legacy_f32 v1, v2, v3
+
+// SICI: v_mul_legacy_f32_e32 v1, v2, v3 ; encoding: [0x02,0x07,0x02,0x0e]
+// VI:   v_mul_legacy_f32_e32 v1, v2, v3 ; encoding: [0x02,0x07,0x02,0x08]
+v_mul_legacy_f32_e32 v1, v2, v3
+
+// SICI: v_mul_f32_e32 v1, v2, v3 ; encoding: [0x02,0x07,0x02,0x10]
+// VI:   v_mul_f32_e32 v1, v2, v3 ; encoding: [0x02,0x07,0x02,0x0a]
+v_mul_f32 v1, v2, v3
+
+// SICI: v_mul_i32_i24_e32 v1, v2, v3 ; encoding: [0x02,0x07,0x02,0x12]
+// VI:   v_mul_i32_i24_e32 v1, v2, v3 ; encoding: [0x02,0x07,0x02,0x0c]
+v_mul_i32_i24 v1, v2, v3
+
+// SICI: v_mul_hi_i32_i24_e32 v1, v2, v3 ; encoding: [0x02,0x07,0x02,0x14]
+// VI:   v_mul_hi_i32_i24_e32 v1, v2, v3 ; encoding: [0x02,0x07,0x02,0x0e]
+v_mul_hi_i32_i24 v1, v2, v3
+
+// SICI: v_mul_u32_u24_e32 v1, v2, v3 ; encoding: [0x02,0x07,0x02,0x16]
+// VI:   v_mul_u32_u24_e32 v1, v2, v3 ; encoding: [0x02,0x07,0x02,0x10]
+v_mul_u32_u24 v1, v2, v3
+
+// SICI: v_mul_hi_u32_u24_e32 v1, v2, v3 ; encoding: [0x02,0x07,0x02,0x18]
+// VI:   v_mul_hi_u32_u24_e32 v1, v2, v3 ; encoding: [0x02,0x07,0x02,0x12]
+v_mul_hi_u32_u24 v1, v2, v3
+
+// SICI: v_min_legacy_f32_e32 v1, v2, v3 ; encoding: [0x02,0x07,0x02,0x1a]
+// NOVI: error: instruction not supported on this GPU
+// NOVI: v_min_legacy_f32_e32 v1, v2, v3
+v_min_legacy_f32_e32 v1, v2, v3
+
+// SICI: v_max_legacy_f32_e32 v1, v2, v3 ; encoding: [0x02,0x07,0x02,0x1c]
+// NOVI: error: instruction not supported on this GPU
+// NOVI: v_max_legacy_f32 v1, v2, v3
+v_max_legacy_f32 v1, v2, v3
+
+// SICI: v_min_f32_e32 v1, v2, v3 ; encoding: [0x02,0x07,0x02,0x1e]
+// VI:   v_min_f32_e32 v1, v2, v3 ; encoding: [0x02,0x07,0x02,0x14]
+v_min_f32_e32 v1, v2, v3
+
+// SICI: v_max_f32_e32 v1, v2, v3 ; encoding: [0x02,0x07,0x02,0x20]
+// VI:   v_max_f32_e32 v1, v2, v3 ; encoding: [0x02,0x07,0x02,0x16]
+v_max_f32 v1, v2 v3
+
+// SICI: v_min_i32_e32 v1, v2, v3 ; encoding: [0x02,0x07,0x02,0x22]
+// VI:   v_min_i32_e32 v1, v2, v3 ; encoding: [0x02,0x07,0x02,0x18]
+v_min_i32 v1, v2, v3
+
+// SICI: v_max_i32_e32 v1, v2, v3 ; encoding: [0x02,0x07,0x02,0x24]
+// VI:   v_max_i32_e32 v1, v2, v3 ; encoding: [0x02,0x07,0x02,0x1a]
+v_max_i32 v1, v2, v3
+
+// SICI: v_min_u32_e32 v1, v2, v3 ; encoding: [0x02,0x07,0x02,0x26]
+// VI:   v_min_u32_e32 v1, v2, v3 ; encoding: [0x02,0x07,0x02,0x1c]
+v_min_u32 v1, v2, v3
+
+// SICI: v_max_u32_e32 v1, v2, v3 ; encoding: [0x02,0x07,0x02,0x28]
+// VI:   v_max_u32_e32 v1, v2, v3 ; encoding: [0x02,0x07,0x02,0x1e]
+v_max_u32 v1, v2, v3
+
+// SICI: v_lshr_b32_e32 v1, v2, v3 ; encoding: [0x02,0x07,0x02,0x2a]
+// NOVI: error: instruction not supported on this GPU
+// NOVI: v_lshr_b32 v1, v2, v3
+v_lshr_b32 v1, v2, v3
+
+// SICI: v_lshrrev_b32_e32 v1, v2, v3 ; encoding: [0x02,0x07,0x02,0x2c]
+// VI:   v_lshrrev_b32_e32 v1, v2, v3 ; encoding: [0x02,0x07,0x02,0x20]
+v_lshrrev_b32 v1, v2, v3
+
+// SICI: v_ashr_i32_e32 v1, v2, v3 ; encoding: [0x02,0x07,0x02,0x2e]
+// NOVI: error: instruction not supported on this GPU
+// NOVI: v_ashr_i32 v1, v2, v3
+v_ashr_i32 v1, v2, v3
+
+// SICI: v_ashrrev_i32_e32 v1, v2, v3 ; encoding: [0x02,0x07,0x02,0x30]
+// VI:   v_ashrrev_i32_e32 v1, v2, v3 ; encoding: [0x02,0x07,0x02,0x22]
+v_ashrrev_i32 v1, v2, v3
+
+// SICI: v_lshl_b32_e32 v1, v2, v3 ; encoding: [0x02,0x07,0x02,0x32]
+// NOVI: error: instruction not supported on this GPU
+// NOVI: v_lshl_b32_e32 v1, v2, v3
+v_lshl_b32_e32 v1, v2, v3
+
+// SICI: v_lshlrev_b32_e32 v1, v2, v3 ; encoding: [0x02,0x07,0x02,0x34]
+// VI:   v_lshlrev_b32_e32 v1, v2, v3 ; encoding: [0x02,0x07,0x02,0x24]
+v_lshlrev_b32 v1, v2, v3
+
+// SICI: v_and_b32_e32 v1, v2, v3 ; encoding: [0x02,0x07,0x02,0x36]
+// VI:   v_and_b32_e32 v1, v2, v3 ; encoding: [0x02,0x07,0x02,0x26]
+v_and_b32 v1, v2, v3
+
+// SICI: v_or_b32_e32 v1, v2, v3 ; encoding: [0x02,0x07,0x02,0x38]
+// VI:   v_or_b32_e32 v1, v2, v3 ; encoding: [0x02,0x07,0x02,0x28]
+v_or_b32 v1, v2, v3
+
+// SICI: v_xor_b32_e32 v1, v2, v3 ; encoding: [0x02,0x07,0x02,0x3a]
+// VI:   v_xor_b32_e32 v1, v2, v3 ; encoding: [0x02,0x07,0x02,0x2a]
+v_xor_b32 v1, v2, v3
+
+// SICI: v_bfm_b32_e32 v1, v2, v3 ; encoding: [0x02,0x07,0x02,0x3c]
+// VI:   v_bfm_b32_e64 v1, v2, v3 ; encoding: [0x01,0x00,0x93,0xd2,0x02,0x07,0x02,0x00]
+v_bfm_b32 v1, v2, v3
+
+// SICI: v_mac_f32_e32 v1, v2, v3 ; encoding: [0x02,0x07,0x02,0x3e]
+// VI:   v_mac_f32_e32 v1, v2, v3 ; encoding: [0x02,0x07,0x02,0x2c]
+v_mac_f32 v1, v2, v3
+
+// SICI: v_madmk_f32_e32 v1, v2, v3, 0x42800000 ; encoding: [0x02,0x07,0x02,0x40,0x00,0x00,0x80,0x42]
+// VI:   v_madmk_f32_e32 v1, v2, v3, 0x42800000 ; encoding: [0x02,0x07,0x02,0x2e,0x00,0x00,0x80,0x42]
+v_madmk_f32 v1, v2, v3, 64.0
+
+// SICI: v_madak_f32_e32 v1, v2, v3, 0x42800000 ; encoding: [0x02,0x07,0x02,0x42,0x00,0x00,0x80,0x42]
+// VI:  v_madak_f32_e32 v1, v2, v3, 0x42800000 ; encoding: [0x02,0x07,0x02,0x30,0x00,0x00,0x80,0x42]
+v_madak_f32 v1, v2, v3, 64.0
+
+// SICI: v_bcnt_u32_b32_e32 v1, v2, v3 ; encoding: [0x02,0x07,0x02,0x44]
+// VI:   v_bcnt_u32_b32_e64 v1, v2, v3 ; encoding: [0x01,0x00,0x8b,0xd2,0x02,0x07,0x02,0x00]
+v_bcnt_u32_b32 v1, v2, v3
+
+// SICI: v_mbcnt_lo_u32_b32_e32 v1, v2, v3 ; encoding: [0x02,0x07,0x02,0x46]
+// VI:   v_mbcnt_lo_u32_b32_e64 v1, v2, v3 ; encoding: [0x01,0x00,0x8c,0xd2,0x02,0x07,0x02,0x00]
+v_mbcnt_lo_u32_b32 v1, v2, v3
+
+// SICI: v_mbcnt_hi_u32_b32_e32 v1, v2, v3 ; encoding: [0x02,0x07,0x02,0x48]
+// VI:   v_mbcnt_hi_u32_b32_e64 v1, v2, v3 ; encoding: [0x01,0x00,0x8d,0xd2,0x02,0x07,0x02,0x00]
+v_mbcnt_hi_u32_b32 v1, v2, v3
+
+// SICI: v_add_i32_e32 v1, v2, v3 ; encoding: [0x02,0x07,0x02,0x4a]
+// VI:   v_add_i32_e32 v1, v2, v3 ; encoding: [0x02,0x07,0x02,0x32]
+v_add_i32 v1, v2, v3
+
+// SICI: v_add_i32_e32 v1, v2, v3 ; encoding: [0x02,0x07,0x02,0x4a]
+// VI:   v_add_i32_e32 v1, v2, v3 ; encoding: [0x02,0x07,0x02,0x32]
+v_add_u32 v1, v2, v3
+
+// SICI: v_sub_i32_e32 v1, v2, v3 ; encoding: [0x02,0x07,0x02,0x4c]
+// VI:   v_sub_i32_e32 v1, v2, v3 ; encoding: [0x02,0x07,0x02,0x34]
+v_sub_i32 v1, v2, v3
+
+// SICI: v_sub_i32_e32 v1, v2, v3 ; encoding: [0x02,0x07,0x02,0x4c]
+// VI:   v_sub_i32_e32 v1, v2, v3 ; encoding: [0x02,0x07,0x02,0x34]
+v_sub_u32 v1, v2, v3
+
+// SICI: v_subrev_i32_e32 v1, v2, v3 ; encoding: [0x02,0x07,0x02,0x4e]
+// VI:   v_subrev_i32_e32 v1, v2, v3 ; encoding: [0x02,0x07,0x02,0x36]
+v_subrev_i32 v1, v2, v3
+
+// SICI: v_subrev_i32_e32 v1, v2, v3 ; encoding: [0x02,0x07,0x02,0x4e]
+// VI:   v_subrev_i32_e32 v1, v2, v3 ; encoding: [0x02,0x07,0x02,0x36]
+v_subrev_u32 v1, v2, v3
+
+// SICI: v_addc_u32_e32 v1, v2, v3 ; encoding: [0x02,0x07,0x02,0x50]
+// VI:   v_addc_u32_e32 v1, v2, v3 ; encoding: [0x02,0x07,0x02,0x38]
+v_addc_u32 v1, v2, v3
+
+// SICI: v_subb_u32_e32 v1, v2, v3 ; encoding: [0x02,0x07,0x02,0x52]
+// VI:   v_subb_u32_e32 v1, v2, v3 ; encoding: [0x02,0x07,0x02,0x3a]
+v_subb_u32 v1, v2, v3
+
+// SICI: v_subbrev_u32_e32 v1, v2, v3 ; encoding: [0x02,0x07,0x02,0x54]
+// VI:   v_subbrev_u32_e32 v1, v2, v3 ; encoding: [0x02,0x07,0x02,0x3c]
+v_subbrev_u32 v1, v2, v3
+
+// SICI: v_ldexp_f32_e32 v1, v2, v3 ; encoding: [0x02,0x07,0x02,0x56]
+// VI:   v_ldexp_f32_e64 v1, v2, v3 ; encoding: [0x01,0x00,0x88,0xd2,0x02,0x07,0x02,0x00]
+v_ldexp_f32 v1, v2, v3
+
+// SICI: v_cvt_pkaccum_u8_f32_e32 v1, v2, v3 ; encoding: [0x02,0x07,0x02,0x58]
+// VI:   v_cvt_pkaccum_u8_f32_e64 v1, v2, v3 ; encoding: [0x01,0x00,0xf0,0xd1,0x02,0x07,0x02,0x00]
+v_cvt_pkaccum_u8_f32 v1, v2, v3
+
+// SICI: v_cvt_pknorm_i16_f32_e32 v1, v2, v3 ; encoding: [0x02,0x07,0x02,0x5a]
+// VI:   v_cvt_pknorm_i16_f32_e64 v1, v2, v3 ; encoding: [0x01,0x00,0x94,0xd2,0x02,0x07,0x02,0x00]
+v_cvt_pknorm_i16_f32 v1, v2, v3
+
+// SICI: v_cvt_pknorm_u16_f32_e32 v1, v2, v3 ; encoding: [0x02,0x07,0x02,0x5c]
+// VI:   v_cvt_pknorm_u16_f32_e64 v1, v2, v3 ; encoding: [0x01,0x00,0x95,0xd2,0x02,0x07,0x02,0x00]
+v_cvt_pknorm_u16_f32 v1, v2, v3
+
+// SICI: v_cvt_pkrtz_f16_f32_e32 v1, v2, v3 ; encoding: [0x02,0x07,0x02,0x5e]
+// VI:   v_cvt_pkrtz_f16_f32_e64 v1, v2, v3 ; encoding: [0x01,0x00,0x96,0xd2,0x02,0x07,0x02,0x00]
+v_cvt_pkrtz_f16_f32 v1, v2, v3
+
+// SICI: v_cvt_pk_u16_u32_e32 v1, v2, v3 ; encoding: [0x02,0x07,0x02,0x60]
+// VI:   v_cvt_pk_u16_u32_e64 v1, v2, v3 ; encoding: [0x01,0x00,0x97,0xd2,0x02,0x07,0x02,0x00]
+v_cvt_pk_u16_u32 v1, v2, v3
+
+// SICI: v_cvt_pk_i16_i32_e32 v1, v2, v3 ; encoding: [0x02,0x07,0x02,0x62]
+// VI:   v_cvt_pk_i16_i32_e64 v1, v2, v3 ; encoding: [0x01,0x00,0x98,0xd2,0x02,0x07,0x02,0x00]
+v_cvt_pk_i16_i32 v1, v2, v3
+
+// NOSICI: error: instruction not supported on this GPU
+// NOSICI: v_add_f16 v1, v2, v3
+// VI:     v_add_f16_e32 v1, v2, v3 ; encoding: [0x02,0x07,0x02,0x3e]
+v_add_f16 v1, v2, v3
+
+// NOSICI: error: instruction not supported on this GPU
+// NOSICI: v_sub_f16 v1, v2, v3
+// VI:     v_sub_f16_e32 v1, v2, v3 ; encoding: [0x02,0x07,0x02,0x40]
+v_sub_f16 v1, v2, v3
+
+// NOSICI: error: instruction not supported on this GPU
+// NOSICI: v_subrev_f16 v1, v2, v3
+// VI:     v_subrev_f16_e32 v1, v2, v3 ; encoding: [0x02,0x07,0x02,0x42]
+v_subrev_f16 v1, v2, v3
+
+// NOSICI: error: instruction not supported on this GPU
+// NOSICI: v_mul_f16 v1, v2, v3
+// VI:     v_mul_f16_e32 v1, v2, v3 ; encoding: [0x02,0x07,0x02,0x44]
+v_mul_f16 v1, v2, v3
+
+// NOSICI: error: instruction not supported on this GPU
+// NOSICI: v_mac_f16 v1, v2, v3
+// VI:     v_mac_f16_e32 v1, v2, v3 ; encoding: [0x02,0x07,0x02,0x46]
+v_mac_f16 v1, v2, v3
+
+// NOSICI: error: instruction not supported on this GPU
+// NOSICI: v_madmk_f16 v1, v2, v3, 64.0
+// VI:     v_madmk_f16_e32 v1, v2, v3, 0x42800000 ; encoding: [0x02,0x07,0x02,0x48,0x00,0x00,0x80,0x42]
+v_madmk_f16 v1, v2, v3, 64.0
+
+// NOSICI: error: instruction not supported on this GPU
+// NOSICI: v_madak_f16 v1, v2, v3, 64.0
+// VI:     v_madak_f16_e32 v1, v2, v3, 0x42800000 ; encoding: [0x02,0x07,0x02,0x4a,0x00,0x00,0x80,0x42]
+v_madak_f16 v1, v2, v3, 64.0
+
+// NOSICI: error: instruction not supported on this GPU
+// NOSICI: v_add_u16 v1, v2, v3
+// VI:     v_add_u16_e32 v1, v2, v3 ; encoding: [0x02,0x07,0x02,0x4c]
+v_add_u16 v1, v2, v3
+
+// NOSICI: error: instruction not supported on this GPU
+// NOSICI: v_sub_u16 v1, v2, v3
+// VI:     v_sub_u16_e32 v1, v2, v3 ; encoding: [0x02,0x07,0x02,0x4e]
+v_sub_u16 v1, v2, v3
+
+// NOSICI: error: instruction not supported on this GPU
+// NOSICI: v_subrev_u16 v1, v2, v3
+// VI:     v_subrev_u16_e32 v1, v2, v3 ; encoding: [0x02,0x07,0x02,0x50]
+v_subrev_u16 v1, v2, v3
+
+// NOSICI: error: instruction not supported on this GPU
+// NOSICI: v_mul_lo_u16 v1, v2, v3
+// VI:     v_mul_lo_u16_e32 v1, v2, v3 ; encoding: [0x02,0x07,0x02,0x52]
+v_mul_lo_u16 v1, v2, v3
+
+// NOSICI: error: instruction not supported on this GPU
+// NOSICI: v_lshlrev_b16 v1, v2, v3
+// VI:     v_lshlrev_b16_e32 v1, v2, v3 ; encoding: [0x02,0x07,0x02,0x54]
+v_lshlrev_b16 v1, v2, v3
+
+// NOSICI: error: instruction not supported on this GPU
+// NOSICI: v_lshrrev_b16 v1, v2, v3
+// VI: v_lshrrev_b16_e32 v1, v2, v3 ; encoding: [0x02,0x07,0x02,0x56]
+v_lshrrev_b16 v1, v2, v3
+
+// NOSICI: error: instruction not supported on this GPU
+// NOSICI: v_ashrrev_b16 v1, v2, v3
+// VI:     v_ashrrev_b16_e32 v1, v2, v3 ; encoding: [0x02,0x07,0x02,0x58]
+v_ashrrev_b16 v1, v2, v3
+
+// NOSICI: error: instruction not supported on this GPU
+// NOSICI: v_max_f16 v1, v2, v3
+// VI:     v_max_f16_e32 v1, v2, v3 ; encoding: [0x02,0x07,0x02,0x5a]
+v_max_f16 v1, v2, v3
+
+// NOSICI: error: instruction not supported on this GPU
+// NOSICI: v_min_f16 v1, v2, v3
+// VI:     v_min_f16_e32 v1, v2, v3 ; encoding: [0x02,0x07,0x02,0x5c]
+v_min_f16 v1, v2, v3
+
+// NOSICI: error: instruction not supported on this GPU
+// NOSICI: v_max_u16 v1, v2, v3
+// VI:     v_max_u16_e32 v1, v2, v3 ; encoding: [0x02,0x07,0x02,0x5e]
+v_max_u16 v1, v2, v3
+
+// NOSICI: error: instruction not supported on this GPU
+// NOSICI: v_max_i16 v1, v2, v3
+// VI:     v_max_i16_e32 v1, v2, v3 ; encoding: [0x02,0x07,0x02,0x60]
+v_max_i16 v1, v2, v3
+
+// NOSICI: error: instruction not supported on this GPU
+// NOSICI: v_min_u16 v1, v2, v3
+// VI:     v_min_u16_e32 v1, v2, v3 ; encoding: [0x02,0x07,0x02,0x62]
+v_min_u16 v1, v2, v3
+
+// NOSICI: error: instruction not supported on this GPU
+// NOSICI: v_min_i16 v1, v2, v3
+// VI:     v_min_i16_e32 v1, v2, v3 ; encoding: [0x02,0x07,0x02,0x64]
+v_min_i16 v1, v2, v3
+
+// NOSICI: error: instruction not supported on this GPU
+// NOSICI: v_ldexp_f16 v1, v2, v3
+// VI:     v_ldexp_f16_e32 v1, v2, v3 ; encoding: [0x02,0x07,0x02,0x66]
+v_ldexp_f16 v1, v2, v3
diff --git a/test/MC/AMDGPU/vop3-errs.s b/test/MC/AMDGPU/vop3-errs.s
new file mode 100644
index 00000000000..b57fe6d5314
--- /dev/null
+++ b/test/MC/AMDGPU/vop3-errs.s
@@ -0,0 +1,5 @@
+// RUN: not llvm-mc -arch=amdgcn -show-encoding %s 2>&1 | FileCheck %s
+// RUN: not llvm-mc -arch=amdgcn -mcpu=SI -show-encoding %s 2>&1 | FileCheck %s
+
+v_add_f32_e64 v0, v1
+// CHECK: error: too few operands for instruction
diff --git a/test/MC/R600/vop3.s b/test/MC/AMDGPU/vop3.s
similarity index 92%
rename from test/MC/R600/vop3.s
rename to test/MC/AMDGPU/vop3.s
index 7d1ba0b4d22..20562335974 100644
--- a/test/MC/R600/vop3.s
+++ b/test/MC/AMDGPU/vop3.s
@@ -1,10 +1,15 @@
 // RUN: llvm-mc -arch=amdgcn -show-encoding %s | FileCheck %s
-// RUN: llvm-mc -arch=amdgcn -show-encoding %s | FileCheck %s
+// RUN: llvm-mc -arch=amdgcn -mcpu=SI -show-encoding %s | FileCheck %s
 
 //===----------------------------------------------------------------------===//
 // VOPC Instructions
 //===----------------------------------------------------------------------===//
 
+// Test forced e64 encoding
+
+v_cmp_lt_f32_e64 s[2:3], v4, -v6
+// CHECK: v_cmp_lt_f32_e64 s[2:3], v4, -v6 ; encoding: [0x02,0x00,0x02,0xd0,0x04,0x0d,0x02,0x40]
+
 //
 // Modifier tests:
 //
@@ -95,6 +100,12 @@ v_fract_f32 v1, v2, div:2 clamp
 // VOP2 Instructions
 ///===---------------------------------------------------------------------===//
 
+// Test forced e64 encoding with e32 operands
+
+v_ldexp_f32_e64 v1, v3, v5
+// CHECK: v_ldexp_f32_e64 v1, v3, v5 ; encoding: [0x01,0x00,0x56,0xd2,0x03,0x0b,0x02,0x00]
+
+
 // TODO: Modifier tests
 
 v_cndmask_b32 v1, v3, v5, s[4:5]
diff --git a/test/MC/R600/vopc.s b/test/MC/AMDGPU/vopc.s
similarity index 100%
rename from test/MC/R600/vopc.s
rename to test/MC/AMDGPU/vopc.s
diff --git a/test/MC/ARM/arm-elf-symver.s b/test/MC/ARM/arm-elf-symver.s
index 26d76556324..ce9c4fe5f00 100644
--- a/test/MC/ARM/arm-elf-symver.s
+++ b/test/MC/ARM/arm-elf-symver.s
@@ -39,7 +39,7 @@ global1:
 @ CHECK-NEXT:     Binding: Local (0x0)
 @ CHECK-NEXT:     Type: None (0x0)
 @ CHECK-NEXT:     Other: 0
-@ CHECK-NEXT:     Section: .text (0x1)
+@ CHECK-NEXT:     Section: .text
 @ CHECK-NEXT:   }
 @ CHECK-NEXT:   Symbol {
 @ CHECK-NEXT:     Name: bar3@@zed
@@ -48,7 +48,7 @@ global1:
 @ CHECK-NEXT:     Binding: Local (0x0)
 @ CHECK-NEXT:     Type: None (0x0)
 @ CHECK-NEXT:     Other: 0
-@ CHECK-NEXT:     Section: .text (0x1)
+@ CHECK-NEXT:     Section: .text
 @ CHECK-NEXT:   }
 @ CHECK-NEXT:   Symbol {
 @ CHECK-NEXT:     Name: bar5@@zed
@@ -57,7 +57,7 @@ global1:
 @ CHECK-NEXT:     Binding: Local (0x0)
 @ CHECK-NEXT:     Type: None (0x0)
 @ CHECK-NEXT:     Other: 0
-@ CHECK-NEXT:     Section: .text (0x1)
+@ CHECK-NEXT:     Section: .text
 @ CHECK-NEXT:   }
 @ CHECK-NEXT:   Symbol {
 @ CHECK-NEXT:     Name: defined1
@@ -66,7 +66,7 @@ global1:
 @ CHECK-NEXT:     Binding: Local (0x0)
 @ CHECK-NEXT:     Type: None (0x0)
 @ CHECK-NEXT:     Other: 0
-@ CHECK-NEXT:     Section: .text (0x1)
+@ CHECK-NEXT:     Section: .text
 @ CHECK-NEXT:   }
 @ CHECK-NEXT:   Symbol {
 @ CHECK-NEXT:     Name: defined2
@@ -75,34 +75,34 @@ global1:
 @ CHECK-NEXT:     Binding: Local (0x0)
 @ CHECK-NEXT:     Type: None (0x0)
 @ CHECK-NEXT:     Other: 0
-@ CHECK-NEXT:     Section: .text (0x1)
+@ CHECK-NEXT:     Section: .text
 @ CHECK-NEXT:   }
 @ CHECK-NEXT:   Symbol {
-@ CHECK-NEXT:     Name: .text (0)
+@ CHECK-NEXT:     Name: (0)
 @ CHECK-NEXT:     Value: 0x0
 @ CHECK-NEXT:     Size: 0
 @ CHECK-NEXT:     Binding: Local (0x0)
 @ CHECK-NEXT:     Type: Section (0x3)
 @ CHECK-NEXT:     Other: 0
-@ CHECK-NEXT:     Section: .text (0x1)
+@ CHECK-NEXT:     Section: .text
 @ CHECK-NEXT:   }
 @ CHECK-NEXT:   Symbol {
-@ CHECK-NEXT:     Name: .data (0)
+@ CHECK-NEXT:     Name: bar2@zed
 @ CHECK-NEXT:     Value: 0x0
 @ CHECK-NEXT:     Size: 0
-@ CHECK-NEXT:     Binding: Local (0x0)
-@ CHECK-NEXT:     Type: Section (0x3)
+@ CHECK-NEXT:     Binding: Global (0x1)
+@ CHECK-NEXT:     Type: None (0x0)
 @ CHECK-NEXT:     Other: 0
-@ CHECK-NEXT:     Section: .data
+@ CHECK-NEXT:     Section: Undefined (0x0)
 @ CHECK-NEXT:   }
 @ CHECK-NEXT:   Symbol {
-@ CHECK-NEXT:     Name: .bss (0)
+@ CHECK-NEXT:     Name: bar6@zed
 @ CHECK-NEXT:     Value: 0x0
 @ CHECK-NEXT:     Size: 0
-@ CHECK-NEXT:     Binding: Local (0x0)
-@ CHECK-NEXT:     Type: Section (0x3)
+@ CHECK-NEXT:     Binding: Global (0x1)
+@ CHECK-NEXT:     Type: None (0x0)
 @ CHECK-NEXT:     Other: 0
-@ CHECK-NEXT:     Section: .bss
+@ CHECK-NEXT:     Section: Undefined (0x0)
 @ CHECK-NEXT:   }
 @ CHECK-NEXT:   Symbol {
 @ CHECK-NEXT:     Name: g1@@zed
@@ -111,7 +111,7 @@ global1:
 @ CHECK-NEXT:     Binding: Global (0x1)
 @ CHECK-NEXT:     Type: None (0x0)
 @ CHECK-NEXT:     Other: 0
-@ CHECK-NEXT:     Section: .text (0x1)
+@ CHECK-NEXT:     Section: .text
 @ CHECK-NEXT:   }
 @ CHECK-NEXT:   Symbol {
 @ CHECK-NEXT:     Name: global1
@@ -120,24 +120,6 @@ global1:
 @ CHECK-NEXT:     Binding: Global (0x1)
 @ CHECK-NEXT:     Type: None (0x0)
 @ CHECK-NEXT:     Other: 0
-@ CHECK-NEXT:     Section: .text (0x1)
-@ CHECK-NEXT:   }
-@ CHECK-NEXT:   Symbol {
-@ CHECK-NEXT:     Name: bar2@zed
-@ CHECK-NEXT:     Value: 0x0
-@ CHECK-NEXT:     Size: 0
-@ CHECK-NEXT:     Binding: Global (0x1)
-@ CHECK-NEXT:     Type: None (0x0)
-@ CHECK-NEXT:     Other: 0
-@ CHECK-NEXT:     Section: Undefined (0x0)
-@ CHECK-NEXT:   }
-@ CHECK-NEXT:   Symbol {
-@ CHECK-NEXT:     Name: bar6@zed
-@ CHECK-NEXT:     Value: 0x0
-@ CHECK-NEXT:     Size: 0
-@ CHECK-NEXT:     Binding: Global (0x1)
-@ CHECK-NEXT:     Type: None (0x0)
-@ CHECK-NEXT:     Other: 0
-@ CHECK-NEXT:     Section: Undefined (0x0)
+@ CHECK-NEXT:     Section: .text
 @ CHECK-NEXT:   }
 @ CHECK-NEXT: ]
diff --git a/test/MC/ARM/basic-thumb2-instructions.s b/test/MC/ARM/basic-thumb2-instructions.s
index 05e0b2b574e..c289325d6d1 100644
--- a/test/MC/ARM/basic-thumb2-instructions.s
+++ b/test/MC/ARM/basic-thumb2-instructions.s
@@ -49,7 +49,6 @@ _func:
         adcs	r0, r1, r3, lsl #7
         adc.w	r0, r1, r3, lsr #31
         adcs.w	r0, r1, r3, asr #32
-        add r2, sp, ip
 
 @ CHECK: adc.w	r4, r5, r6              @ encoding: [0x45,0xeb,0x06,0x04]
 @ CHECK: adcs.w	r4, r5, r6              @ encoding: [0x55,0xeb,0x06,0x04]
@@ -59,7 +58,6 @@ _func:
 @ CHECK: adcs.w	r0, r1, r3, lsl #7      @ encoding: [0x51,0xeb,0xc3,0x10]
 @ CHECK: adc.w	r0, r1, r3, lsr #31     @ encoding: [0x41,0xeb,0xd3,0x70]
 @ CHECK: adcs.w	r0, r1, r3, asr #32     @ encoding: [0x51,0xeb,0x23,0x00]
-@ CHECK: add.w	r2, sp, r12             @ encoding: [0x0d,0xeb,0x0c,0x02]
 
 
 @------------------------------------------------------------------------------
@@ -115,23 +113,99 @@ _func:
 
 
 @------------------------------------------------------------------------------
-@ ADD (register)
+@ ADD (register, not SP) A8.8.6
 @------------------------------------------------------------------------------
         add r1, r2, r8
         add r5, r9, r2, asr #32
         adds r7, r3, r1, lsl #31
         adds.w r0, r3, r6, lsr #25
         add.w r4, r8, r1, ror #12
+        adds r1, r1, r7              // T1
+        it eq
+        addeq r1, r3, r5             // T1
+        it eq
+        addeq r1, r1, r5             // T1
+        it eq
+        addseq r1, r3, r5            // T3
+        it eq
+        addseq r1, r1, r5            // T3
         add r10, r8
         add r10, r10, r8
+        it eq
+        addeq r1, r10                // T2
+        it eq
+        addseq r1, r10               // T3
 
 @ CHECK: add.w	r1, r2, r8              @ encoding: [0x02,0xeb,0x08,0x01]
 @ CHECK: add.w	r5, r9, r2, asr #32     @ encoding: [0x09,0xeb,0x22,0x05]
 @ CHECK: adds.w	r7, r3, r1, lsl #31     @ encoding: [0x13,0xeb,0xc1,0x77]
 @ CHECK: adds.w	r0, r3, r6, lsr #25     @ encoding: [0x13,0xeb,0x56,0x60]
 @ CHECK: add.w	r4, r8, r1, ror #12     @ encoding: [0x08,0xeb,0x31,0x34]
+@ CHECK: adds r1, r1, r7                @ encoding: [0xc9,0x19]
+@ CHECK: it eq                          @ encoding: [0x08,0xbf]
+@ CHECK: addeq r1, r3, r5               @ encoding: [0x59,0x19]
+@ CHECK: it eq                          @ encoding: [0x08,0xbf]
+@ CHECK: addeq r1, r1, r5               @ encoding: [0x49,0x19]
+@ CHECK: it eq                          @ encoding: [0x08,0xbf]
+@ CHECK: addseq.w r1, r3, r5            @ encoding: [0x13,0xeb,0x05,0x01]
+@ CHECK: it eq                          @ encoding: [0x08,0xbf]
+@ CHECK: addseq.w r1, r1, r5            @ encoding: [0x11,0xeb,0x05,0x01]
 @ CHECK: add	r10, r8                 @ encoding: [0xc2,0x44]
 @ CHECK: add	r10, r8                 @ encoding: [0xc2,0x44]
+@ CHECK: it eq                          @ encoding: [0x08,0xbf]
+@ CHECK: addeq r1, r10                  @ encoding: [0x51,0x44]
+@ CHECK: it eq                          @ encoding: [0x08,0xbf]
+@ CHECK: addseq.w r1, r1, r10           @ encoding: [0x11,0xeb,0x0a,0x01]
+
+@------------------------------------------------------------------------------
+@ ADD (SP plus immediate) A8.8.9
+@------------------------------------------------------------------------------
+        it eq
+@ CHECK: it eq                          @ encoding: [0x08,0xbf]
+        addeq r7, sp, #1020          // T1
+@ CHECK: addeq	r7, sp, #1020           @ encoding: [0xff,0xaf]
+
+        it eq
+@ CHECK: it eq                          @ encoding: [0x08,0xbf]
+        addeq sp, sp, #508           // T2
+@ FIXME: ARMARM says 'addeq sp, sp, #508'
+@ CHECK: addeq	sp, #508                @ encoding: [0x7f,0xb0]
+
+        add r7, sp, #15              // T3
+@ CHECK: add.w	r7, sp, #15             @ encoding: [0x0d,0xf1,0x0f,0x07]
+        adds r7, sp, #16             // T3
+@ CHECK: adds.w	r7, sp, #16             @ encoding: [0x1d,0xf1,0x10,0x07]
+        add r8, sp, #16              // T3
+@ CHECK: add.w	r8, sp, #16             @ encoding: [0x0d,0xf1,0x10,0x08]
+
+        addw r6, sp, #1020           // T4
+@ CHECK: addw	r6, sp, #1020           @ encoding: [0x0d,0xf2,0xfc,0x36]
+        add r6, sp, #1019            // T4
+@ CHECK: addw	r6, sp, #1019           @ encoding: [0x0d,0xf2,0xfb,0x36]
+
+@------------------------------------------------------------------------------
+@ ADD (SP plus register) A8.8.10
+@------------------------------------------------------------------------------
+        it eq
+@ CHECK: it eq                          @ encoding: [0x08,0xbf]
+        addeq r8, sp, r8             // T1
+@ CHECK: addeq	r8, sp, r8              @ encoding: [0xe8,0x44]
+        it eq
+@ CHECK: it eq                          @ encoding: [0x08,0xbf]
+        addeq r8, sp                 // T1
+@ CHECK: addeq	r8, sp                  @ encoding: [0xe8,0x44]
+
+        it eq
+@ CHECK: it eq                          @ encoding: [0x08,0xbf]
+        addeq sp, r9                 // T2
+@ CHECK: addeq	sp, r9                  @ encoding: [0xcd,0x44]
+
+        add r2, sp, ip               // T3
+@ CHECK: add.w r2, sp, r12              @ encoding: [0x0d,0xeb,0x0c,0x02]
+        it eq
+@ CHECK: it eq                          @ encoding: [0x08,0xbf]
+        addeq r2, sp, ip             // T3
+@ CHECK: addeq.w r2, sp, r12            @ encoding: [0x0d,0xeb,0x0c,0x02]
 
 
 @------------------------------------------------------------------------------
diff --git a/test/MC/ARM/bracket-exprs.s b/test/MC/ARM/bracket-exprs.s
index 922bf703701..74f3323cd2d 100644
--- a/test/MC/ARM/bracket-exprs.s
+++ b/test/MC/ARM/bracket-exprs.s
@@ -1,7 +1,7 @@
 // RUN: llvm-mc -triple arm-unknown-linux %s | FileCheck %s
 
 // CHECK: .byte 1
-.if [~0 >> 1] == -1
+.if [~0 >> 63] == 1
 .byte 1
 .else
 .byte 2
diff --git a/test/MC/ARM/coff-debugging-secrel.ll b/test/MC/ARM/coff-debugging-secrel.ll
index 4f897e1adf7..1b8b7310171 100644
--- a/test/MC/ARM/coff-debugging-secrel.ll
+++ b/test/MC/ARM/coff-debugging-secrel.ll
@@ -16,14 +16,14 @@ entry:
 !llvm.dbg.cu = !{!7}
 !llvm.module.flags = !{!9, !10}
 
-!0 = !MDLocation(line: 1, scope: !1)
-!1 = !MDSubprogram(name: "function", line: 1, isLocal: false, isDefinition: true, virtualIndex: 6, isOptimized: false, scopeLine: 1, file: !2, scope: !3, type: !4, function: void ()* @function, variables: !6)
-!2 = !MDFile(filename: "/Users/compnerd/work/llvm/test/MC/ARM/reduced.c", directory: "/Users/compnerd/work/llvm")
-!3 = !MDFile(filename: "/Users/compnerd/work/llvm/test/MC/ARM/reduced.c", directory: "/Users/compnerd/work/llvm")
-!4 = !MDSubroutineType(types: !5)
+!0 = !DILocation(line: 1, scope: !1)
+!1 = !DISubprogram(name: "function", line: 1, isLocal: false, isDefinition: true, virtualIndex: 6, isOptimized: false, scopeLine: 1, file: !2, scope: !3, type: !4, function: void ()* @function, variables: !6)
+!2 = !DIFile(filename: "/Users/compnerd/work/llvm/test/MC/ARM/reduced.c", directory: "/Users/compnerd/work/llvm")
+!3 = !DIFile(filename: "/Users/compnerd/work/llvm/test/MC/ARM/reduced.c", directory: "/Users/compnerd/work/llvm")
+!4 = !DISubroutineType(types: !5)
 !5 = !{null}
 !6 = !{}
-!7 = !MDCompileUnit(language: DW_LANG_C99, producer: "clang version 3.5.0", isOptimized: false, emissionKind: 1, file: !2, enums: !6, retainedTypes: !6, subprograms: !8, globals: !6, imports: !6)
+!7 = !DICompileUnit(language: DW_LANG_C99, producer: "clang version 3.5.0", isOptimized: false, emissionKind: 1, file: !2, enums: !6, retainedTypes: !6, subprograms: !8, globals: !6, imports: !6)
 !8 = !{!1}
 !9 = !{i32 2, !"Dwarf Version", i32 4}
 !10 = !{i32 1, !"Debug Info Version", i32 3}
diff --git a/test/MC/ARM/data-in-code.ll b/test/MC/ARM/data-in-code.ll
index 724577bb0da..c4910ff20e6 100644
--- a/test/MC/ARM/data-in-code.ll
+++ b/test/MC/ARM/data-in-code.ll
@@ -1,8 +1,8 @@
-;; RUN: llc -O0 -verify-machineinstrs -fast-isel-abort=1 \
+;; RUN: llc -verify-machineinstrs \
 ;; RUN:   -mtriple=armv7-linux-gnueabi -filetype=obj %s -o - | \
 ;; RUN:   llvm-readobj -t | FileCheck -check-prefix=ARM %s
 
-;; RUN: llc -O0 -verify-machineinstrs -fast-isel-abort=1 \
+;; RUN: llc -verify-machineinstrs \
 ;; RUN:   -mtriple=thumbv7-linux-gnueabi -filetype=obj %s -o - | \
 ;; RUN:   llvm-readobj -t | FileCheck -check-prefix=TMB %s
 
@@ -11,102 +11,25 @@
 
 define void @foo(i32* %ptr) nounwind ssp {
   %tmp = load i32, i32* %ptr, align 4
-  switch i32 %tmp, label %default [
-    i32 11, label %bb0
-    i32 10, label %bb1
-    i32 8, label %bb2
-    i32 4, label %bb3
-    i32 2, label %bb4
-    i32 6, label %bb5
-    i32 9, label %bb6
-    i32 15, label %bb7
-    i32 1, label %bb8
-    i32 3, label %bb9
-    i32 5, label %bb10
-    i32 30, label %bb11
-    i32 31, label %bb12
-    i32 13, label %bb13
-    i32 14, label %bb14
-    i32 20, label %bb15
-    i32 19, label %bb16
-    i32 17, label %bb17
-    i32 18, label %bb18
-    i32 21, label %bb19
-    i32 22, label %bb20
-    i32 16, label %bb21
-    i32 24, label %bb22
-    i32 25, label %bb23
-    i32 26, label %bb24
-    i32 27, label %bb25
-    i32 28, label %bb26
-    i32 23, label %bb27
-    i32 12, label %bb28
+  switch i32 %tmp, label %exit [
+    i32 0, label %bb0
+    i32 1, label %bb1
+    i32 2, label %bb2
+    i32 3, label %bb3
   ]
-
-default:
-  br label %exit
 bb0:
+  store i32 0, i32* %ptr, align 4
   br label %exit
 bb1:
+  store i32 1, i32* %ptr, align 4
   br label %exit
 bb2:
+  store i32 2, i32* %ptr, align 4
   br label %exit
 bb3:
+  store i32 3, i32* %ptr, align 4
   br label %exit
-bb4:
-  br label %exit
-bb5:
-  br label %exit
-bb6:
-  br label %exit
-bb7:
-  br label %exit
-bb8:
-  br label %exit
-bb9:
-  br label %exit
-bb10:
-  br label %exit
-bb11:
-  br label %exit
-bb12:
-  br label %exit
-bb13:
-  br label %exit
-bb14:
-  br label %exit
-bb15:
-  br label %exit
-bb16:
-  br label %exit
-bb17:
-  br label %exit
-bb18:
-  br label %exit
-bb19:
-  br label %exit
-bb20:
-  br label %exit
-bb21:
-  br label %exit
-bb22:
-  br label %exit
-bb23:
-  br label %exit
-bb24:
-  br label %exit
-bb25:
-  br label %exit
-bb26:
-  br label %exit
-bb27:
-  br label %exit
-bb28:
-  br label %exit
-
-
 exit:
-
   ret void
 }
 
diff --git a/test/MC/ARM/directive-arch-armv2.s b/test/MC/ARM/directive-arch-armv2.s
index 40857ca9fad..f6dc20c0a36 100644
--- a/test/MC/ARM/directive-arch-armv2.s
+++ b/test/MC/ARM/directive-arch-armv2.s
@@ -20,7 +20,7 @@
 @ CHECK-ATTR:   }
 @ CHECK-ATTR:   Attribute {
 @ CHECK-ATTR:     TagName: CPU_arch
-@ CHECK-ATTR:     Description: ARM v4
+@ CHECK-ATTR:     Description: Pre-v4
 @ CHECK-ATTR:   }
 @ CHECK-ATTR:   Attribute {
 @ CHECK-ATTR:     TagName: ARM_ISA_use
diff --git a/test/MC/ARM/directive-arch-armv2a.s b/test/MC/ARM/directive-arch-armv2a.s
index 62c2ace796f..bb0a693dc84 100644
--- a/test/MC/ARM/directive-arch-armv2a.s
+++ b/test/MC/ARM/directive-arch-armv2a.s
@@ -20,7 +20,7 @@
 @ CHECK-ATTR:   }
 @ CHECK-ATTR:   Attribute {
 @ CHECK-ATTR:     TagName: CPU_arch
-@ CHECK-ATTR:     Description: ARM v4
+@ CHECK-ATTR:     Description: Pre-v4
 @ CHECK-ATTR:   }
 @ CHECK-ATTR:   Attribute {
 @ CHECK-ATTR:     TagName: ARM_ISA_use
diff --git a/test/MC/ARM/directive-arch-armv3.s b/test/MC/ARM/directive-arch-armv3.s
index 41cce659246..aeec638eac8 100644
--- a/test/MC/ARM/directive-arch-armv3.s
+++ b/test/MC/ARM/directive-arch-armv3.s
@@ -20,7 +20,7 @@
 @ CHECK-ATTR:   }
 @ CHECK-ATTR:   Attribute {
 @ CHECK-ATTR:     TagName: CPU_arch
-@ CHECK-ATTR:     Description: ARM v4
+@ CHECK-ATTR:     Description: Pre-v4
 @ CHECK-ATTR:   }
 @ CHECK-ATTR:   Attribute {
 @ CHECK-ATTR:     TagName: ARM_ISA_use
diff --git a/test/MC/ARM/directive-arch-armv3m.s b/test/MC/ARM/directive-arch-armv3m.s
index 8041da2e1e5..fda8db52e7b 100644
--- a/test/MC/ARM/directive-arch-armv3m.s
+++ b/test/MC/ARM/directive-arch-armv3m.s
@@ -20,7 +20,7 @@
 @ CHECK-ATTR:   }
 @ CHECK-ATTR:   Attribute {
 @ CHECK-ATTR:     TagName: CPU_arch
-@ CHECK-ATTR:     Description: ARM v4
+@ CHECK-ATTR:     Description: Pre-v4
 @ CHECK-ATTR:   }
 @ CHECK-ATTR:   Attribute {
 @ CHECK-ATTR:     TagName: ARM_ISA_use
diff --git a/test/MC/ARM/directive-fpu-multiple.s b/test/MC/ARM/directive-fpu-multiple.s
index de2baaf2716..50389a11532 100644
--- a/test/MC/ARM/directive-fpu-multiple.s
+++ b/test/MC/ARM/directive-fpu-multiple.s
@@ -1,12 +1,33 @@
 @ Check multiple .fpu directives.
 
 @ The later .fpu directive should overwrite the earlier one.
-@ See also: directive-fpu-multiple2.s.
+@ We also check here that all the .fpu directives that we expect to work do work
 
 @ RUN: llvm-mc -triple arm-eabi -filetype obj %s | llvm-readobj -arm-attributes \
 @ RUN:   | FileCheck %s -check-prefix CHECK-ATTR
 
+	.fpu none
+	.fpu vfp
+	.fpu vfpv2
+	.fpu vfpv3
+	.fpu vfpv3-fp16
+	.fpu vfpv3-d16
+	.fpu vfpv3-d16-fp16
+	.fpu vfpv3xd
+	.fpu vfpv3xd-fp16
+	.fpu vfpv4
+	.fpu vfpv4-d16
+	.fpu fpv4-sp-d16
+	.fpu fpv5-d16
+	.fpu fpv5-sp-d16
+	.fpu fp-armv8
 	.fpu neon
+	.fpu neon-fp16
+	.fpu neon-vfpv4
+	.fpu neon-fp-armv8
+	.fpu crypto-neon-fp-armv8
+	.fpu softvfp
+
 	.fpu vfpv4
 
 @ CHECK-ATTR: FileAttributes {
diff --git a/test/MC/ARM/directive-type-diagnostics.s b/test/MC/ARM/directive-type-diagnostics.s
new file mode 100644
index 00000000000..b166ffd06aa
--- /dev/null
+++ b/test/MC/ARM/directive-type-diagnostics.s
@@ -0,0 +1,10 @@
+// RUN: not llvm-mc -triple arm-elf -filetype asm -o /dev/null %s 2>&1 | FileCheck %s
+// RUN: not llvm-mc -triple armeb-elf -filetype asm -o /dev/null %s 2>&1 | FileCheck %s
+// RUN: not llvm-mc -triple thumb-elf -filetype asm -o /dev/null %s 2>&1 | FileCheck %s
+// RUN: not llvm-mc -triple thumbeb-elf -filetype asm -o /dev/null %s 2>&1 | FileCheck %s
+
+        .type symbol 32
+// CHECK: error: expected STT_<TYPE_IN_UPPER_CASE>, '#<type>', '%<type>' or "<type>"
+// CHECK: .type symbol 32
+// CHECK:              ^
+
diff --git a/test/MC/ARM/dwarf-cfi-initial-state.s b/test/MC/ARM/dwarf-cfi-initial-state.s
index 0d1c08af728..d3f7ce420f9 100644
--- a/test/MC/ARM/dwarf-cfi-initial-state.s
+++ b/test/MC/ARM/dwarf-cfi-initial-state.s
@@ -13,5 +13,8 @@ bx lr
 # When llvm-dwarfdump prints the full info for the DW_CFA_def_cfa
 # field, we can check that here too.
 # CHECK: DW_CFA_def_cfa:
+# The following 2 DW_CFA_nop instructions are "padding"
+# CHECK: DW_CFA_nop:
+# CHECK: DW_CFA_nop:
 # CHECK-NOT: DW_CFA
 # CHECK: FDE
diff --git a/test/MC/ARM/eh-directive-section-comdat.s b/test/MC/ARM/eh-directive-section-comdat.s
index 296718f096c..9c7160ea5e7 100644
--- a/test/MC/ARM/eh-directive-section-comdat.s
+++ b/test/MC/ARM/eh-directive-section-comdat.s
@@ -47,13 +47,15 @@ func1:
 @ CHECK:     Type: SHT_GROUP (0x11)
 @ CHECK:     Flags [ (0x0)
 @ CHECK:     ]
+@ CHECK:     Size: 24
 @ CHECK:     SectionData (
 @-------------------------------------------------------------------------------
-@ The second, third, and fourth word should correspond to the section index
-@ of .TEST1, .ARM.extab.TEST1, and .ARM.exidx.TEST1.
+@ These are the section indexes of .TEST1, .ARM.extab.TEST1, .ARM.exidx.TEST1,
+@ .rel.ARM.extab.TEST1, and .rel.ARM.exidx.TEST1.
 @-------------------------------------------------------------------------------
-@ CHECK:       0000: 01000000 05000000 06000000 08000000  |................|
-@ CHECK:     )
+@ CHECK-NEXT:     0000: 01000000 06000000 07000000 08000000
+@ CHECK-NEXT:     0010: 09000000 0A000000
+@ CHECK-NEXT:     )
 @ CHECK:   }
 
 
@@ -61,8 +63,8 @@ func1:
 @ Check the .TEST1 section
 @-------------------------------------------------------------------------------
 @ CHECK:   Section {
-@ CHECK:     Index: 5
-@ CHECK:     Name: .TEST1
+@ CHECK:     Index: 6
+@ CHECK-NEXT:     Name: .TEST1
 @ CHECK:     Type: SHT_PROGBITS (0x1)
 @-------------------------------------------------------------------------------
 @ The flags should contain SHF_GROUP.
@@ -79,8 +81,8 @@ func1:
 @ Check the .ARM.extab.TEST1 section
 @-------------------------------------------------------------------------------
 @ CHECK:   Section {
-@ CHECK:     Index: 6
-@ CHECK:     Name: .ARM.extab.TEST1
+@ CHECK:     Index: 7
+@ CHECK-NEXT:     Name: .ARM.extab.TEST1
 @ CHECK:     Type: SHT_PROGBITS (0x1)
 @-------------------------------------------------------------------------------
 @ The flags should contain SHF_GROUP.
@@ -91,13 +93,17 @@ func1:
 @ CHECK:     ]
 @ CHECK:   }
 
+@ CHECK:   Section {
+@ CHECK:     Index: 8
+@ CHECK-NEXT:     Name: .rel.ARM.extab.TEST1
+@ CHECK: }
 
 @-------------------------------------------------------------------------------
 @ Check the .ARM.exidx.TEST1 section
 @-------------------------------------------------------------------------------
 @ CHECK:   Section {
-@ CHECK:     Index: 8
-@ CHECK:     Name: .ARM.exidx.TEST1
+@ CHECK:     Index: 9
+@ CHECK-NEXT:     Name: .ARM.exidx.TEST1
 @ CHECK:     Type: SHT_ARM_EXIDX (0x70000001)
 @-------------------------------------------------------------------------------
 @ The flags should contain SHF_GROUP.
@@ -107,11 +113,16 @@ func1:
 @ CHECK:       SHF_GROUP (0x200)
 @ CHECK:       SHF_LINK_ORDER (0x80)
 @ CHECK:     ]
-@ CHECK:     Link: 5
+@ CHECK:     Link: 6
 @ CHECK:   }
-@ CHECK: ]
 
 
+@ CHECK:   Section {
+@ CHECK:     Index: 10
+@ CHECK-NEXT:     Name: .rel.ARM.exidx.TEST1
+@ CHECK: }
+
+@ CHECK: ]
 
 @-------------------------------------------------------------------------------
 @ Check symbol func1.  It should be weak binding, and belong to .TEST1 section.
@@ -121,6 +132,6 @@ func1:
 @ CHECK:     Name: func1
 @ CHECK:     Binding: Weak (0x2)
 @ CHECK:     Type: Function (0x2)
-@ CHECK:     Section: .TEST1 (0x5)
+@ CHECK:     Section: .TEST1
 @ CHECK:   }
 @ CHECK: ]
diff --git a/test/MC/ARM/eh-directive-section-multiple-func.s b/test/MC/ARM/eh-directive-section-multiple-func.s
index e7198a4aecb..e5307cf9190 100644
--- a/test/MC/ARM/eh-directive-section-multiple-func.s
+++ b/test/MC/ARM/eh-directive-section-multiple-func.s
@@ -54,7 +54,8 @@ func2:
 @ Check the .TEST1 section.  There should be two "bx lr" instructions.
 @-------------------------------------------------------------------------------
 @ CHECK:   Section {
-@ CHECK:     Name: .TEST1
+@ CHECK:     Index: 5
+@ CHECK-NEXT:     Name: .TEST1
 @ CHECK:     SectionData (
 @ CHECK:       0000: 1EFF2FE1 1EFF2FE1                    |../.../.|
 @ CHECK:     )
@@ -86,7 +87,7 @@ func2:
 @-------------------------------------------------------------------------------
 @ CHECK:   Section {
 @ CHECK:     Name: .ARM.exidx.TEST1
-@ CHECK:     Link: 4
+@ CHECK:     Link: 5
 @-------------------------------------------------------------------------------
 @ The first word should be the offset to .TEST1.
 @ The second word should be the offset to .ARM.extab.TEST1
@@ -123,7 +124,7 @@ func2:
 @ CHECK:     Binding: Global (0x1)
 @ CHECK:     Type: Function (0x2)
 @ CHECK:     Other: 0
-@ CHECK:     Section: .TEST1 (0x4)
+@ CHECK:     Section: .TEST1
 @ CHECK:   }
 @ CHECK:   Symbol {
 @ CHECK:     Name: func2
@@ -132,6 +133,6 @@ func2:
 @ CHECK:     Binding: Global (0x1)
 @ CHECK:     Type: Function (0x2)
 @ CHECK:     Other: 0
-@ CHECK:     Section: .TEST1 (0x4)
+@ CHECK:     Section: .TEST1
 @ CHECK:   }
 @ CHECK: ]
diff --git a/test/MC/ARM/eh-directive-section.s b/test/MC/ARM/eh-directive-section.s
index 671d10679d1..e36d9a99aaf 100644
--- a/test/MC/ARM/eh-directive-section.s
+++ b/test/MC/ARM/eh-directive-section.s
@@ -50,8 +50,8 @@ func2:
 @-------------------------------------------------------------------------------
 @ CHECK: Sections [
 @ CHECK:   Section {
-@ CHECK:     Index: 4
-@ CHECK:     Name: .TEST1
+@ CHECK:     Index: 5
+@ CHECK-NEXT:     Name: .TEST1
 @ CHECK:     SectionData (
 @ CHECK:       0000: 1EFF2FE1                             |../.|
 @ CHECK:     )
@@ -84,7 +84,7 @@ func2:
 @-------------------------------------------------------------------------------
 @ This section should linked with .TEST1 section.
 @-------------------------------------------------------------------------------
-@ CHECK:     Link: 4
+@ CHECK:     Link: 5
 
 @-------------------------------------------------------------------------------
 @ The first word should be relocated to the code address in .TEST1 section.
@@ -109,8 +109,8 @@ func2:
 @ Check the TEST2 section (without the dot in the beginning)
 @-------------------------------------------------------------------------------
 @ CHECK:   Section {
-@ CHECK:     Index: 9
-@ CHECK:     Name: TEST2
+@ CHECK:     Index: 10
+@ CHECK-NEXT:     Name: TEST2
 @ CHECK:     SectionData (
 @ CHECK:       0000: 1EFF2FE1                             |../.|
 @ CHECK:     )
@@ -143,7 +143,7 @@ func2:
 @-------------------------------------------------------------------------------
 @ This section should linked with TEST2 section.
 @-------------------------------------------------------------------------------
-@ CHECK:     Link: 9
+@ CHECK:     Link: 10
 
 @-------------------------------------------------------------------------------
 @ The first word should be relocated to the code address in TEST2 section.
@@ -171,10 +171,10 @@ func2:
 @ CHECK: Symbols [
 @ CHECK:   Symbol {
 @ CHECK:     Name: func1
-@ CHECK:     Section: .TEST1 (0x4)
+@ CHECK:     Section: .TEST1
 @ CHECK:   }
 @ CHECK:   Symbol {
 @ CHECK:     Name: func2
-@ CHECK:     Section: TEST2 (0x9)
+@ CHECK:     Section: TEST2
 @ CHECK:   }
 @ CHECK: ]
diff --git a/test/MC/ARM/eh-directive-text-section-multiple-func.s b/test/MC/ARM/eh-directive-text-section-multiple-func.s
index 50e09f5f11b..7ab401c5043 100644
--- a/test/MC/ARM/eh-directive-text-section-multiple-func.s
+++ b/test/MC/ARM/eh-directive-text-section-multiple-func.s
@@ -72,10 +72,10 @@ func2:
 @ CHECK: Symbols [
 @ CHECK:   Symbol {
 @ CHECK:     Name: func1
-@ CHECK:     Section: .text (0x1)
+@ CHECK:     Section: .text
 @ CHECK:   }
 @ CHECK:   Symbol {
 @ CHECK:     Name: func2
-@ CHECK:     Section: .text (0x1)
+@ CHECK:     Section: .text
 @ CHECK:   }
 @ CHECK: ]
diff --git a/test/MC/ARM/eh-directive-text-section.s b/test/MC/ARM/eh-directive-text-section.s
index 4a073aa8115..32696d5a1da 100644
--- a/test/MC/ARM/eh-directive-text-section.s
+++ b/test/MC/ARM/eh-directive-text-section.s
@@ -29,8 +29,8 @@ func1:
 @-------------------------------------------------------------------------------
 @ Check the index of .text section.  This will be used in .ARM.exidx.
 @-------------------------------------------------------------------------------
-@ CHECK:     Index: 1
-@ CHECK:     Name: .text
+@ CHECK:     Index: 2
+@ CHECK-NEXT:     Name: .text
 @ CHECK:     Type: SHT_PROGBITS (0x1)
 @ CHECK:     Flags [ (0x6)
 @ CHECK:       SHF_ALLOC (0x2)
@@ -58,7 +58,7 @@ func1:
 @ Check the linked section of the EXIDX section.  This should be the index
 @ of the .text section.
 @-------------------------------------------------------------------------------
-@ CHECK:     Link: 1
+@ CHECK:     Link: 2
 
 @-------------------------------------------------------------------------------
 @ The first word should be the offset to .text.  The second word should be
diff --git a/test/MC/ARM/eh-link.s b/test/MC/ARM/eh-link.s
index 0c44c0e2c06..e14fb0638b9 100644
--- a/test/MC/ARM/eh-link.s
+++ b/test/MC/ARM/eh-link.s
@@ -17,7 +17,7 @@
 @ CHECK-NEXT:     SHF_GROUP
 @ CHECK-NEXT:   ]
 @ CHECK-NEXT:   Address: 0x0
-@ CHECK-NEXT:   Offset: 0x54
+@ CHECK-NEXT:   Offset:
 @ CHECK-NEXT:   Size: 4
 @ CHECK-NEXT:   Link: 0
 @ CHECK-NEXT:   Info: 0
@@ -34,7 +34,7 @@
 @ CHECK-NEXT:     SHF_LINK_ORDER
 @ CHECK-NEXT:   ]
 @ CHECK-NEXT:   Address: 0x0
-@ CHECK-NEXT:   Offset: 0x58
+@ CHECK-NEXT:   Offset:
 @ CHECK-NEXT:   Size: 8
 @ CHECK-NEXT:   Link: 6
 @ CHECK-NEXT:   Info: 0
@@ -43,7 +43,7 @@
 @ CHECK-NEXT: }
 
 @ CHECK:      Section {
-@ CHECK:        Index: 9
+@ CHECK:        Index: 10
 @ CHECK-NEXT:   Name: .text
 @ CHECK-NEXT:   Type: SHT_PROGBITS
 @ CHECK-NEXT:   Flags [
@@ -60,7 +60,7 @@
 @ CHECK-NEXT:   EntrySize: 0
 @ CHECK-NEXT: }
 @ CHECK-NEXT: Section {
-@ CHECK-NEXT:   Index: 10
+@ CHECK-NEXT:   Index: 11
 @ CHECK-NEXT:   Name: .ARM.exidx
 @ CHECK-NEXT:   Type: SHT_ARM_EXIDX
 @ CHECK-NEXT:   Flags [
@@ -71,7 +71,7 @@
 @ CHECK-NEXT:   Address: 0x0
 @ CHECK-NEXT:   Offset:
 @ CHECK-NEXT:   Size: 8
-@ CHECK-NEXT:   Link: 9
+@ CHECK-NEXT:   Link: 10
 @ CHECK-NEXT:   Info: 0
 @ CHECK-NEXT:   AddressAlignment: 4
 @ CHECK-NEXT:   EntrySize: 0
diff --git a/test/MC/ARM/elf-movt.s b/test/MC/ARM/elf-movt.s
index 1ff5da593b4..9b46cafe3fb 100644
--- a/test/MC/ARM/elf-movt.s
+++ b/test/MC/ARM/elf-movt.s
@@ -44,15 +44,11 @@ barf:                                   @ @barf
 @ OBJ-NEXT:     Address: 0x0
 @ OBJ-NEXT:     Offset:
 @ OBJ-NEXT:     Size: 16
-@ OBJ-NEXT:     Link: 6
-@ OBJ-NEXT:     Info: 1
+@ OBJ-NEXT:     Link:
+@ OBJ-NEXT:     Info:
 @ OBJ-NEXT:     AddressAlignment: 4
 @ OBJ-NEXT:     EntrySize: 8
 @ OBJ-NEXT:     Relocations [
-@ OBJ-NEXT:       0x0 R_ARM_MOVW_PREL_NC
-@ OBJ-NEXT:       0x4 R_ARM_MOVT_PREL
+@ OBJ-NEXT:       0x0 R_ARM_MOVW_PREL_NC GOT 0x0
+@ OBJ-NEXT:       0x4 R_ARM_MOVT_PREL GOT 0x0
 @ OBJ-NEXT:   ]
-@ OBJ-NEXT:     SectionData (
-@ OBJ-NEXT:       0000: 00000000 2D060000 04000000 2E060000  |....-...........|
-@ OBJ-NEXT:     )
-@ OBJ-NEXT:   }
diff --git a/test/MC/ARM/elf-note-nacl.ll b/test/MC/ARM/elf-note-nacl.ll
deleted file mode 100644
index 8551cf52b4a..00000000000
--- a/test/MC/ARM/elf-note-nacl.ll
+++ /dev/null
@@ -1,22 +0,0 @@
-; RUN: llc -filetype=obj -mtriple armv7-none-nacl-gnueabi %s -o - \
-; RUN:   | llvm-objdump -triple armv7 -s - | FileCheck %s
-
-; Tests that NaCl object files contain an ELF note section that identifies them
-; to the binutils gold linker
-
-define void @main() {
-  ret void
-}
-
-; There appears to be no way for llvm-objdump to show flags for sections, or
-; to dump groups like readelf.
-; CHECK: .group
-; CHECK: .note.NaCl.ABI.arm
-; The contents of the words in the note section should be:
-;   sizeof "NaCl"
-;   sizeof "arm"
-;   1 (NT_VERSION)
-;   "NaCl" with nul termination and padding to align 4
-;   "arm" with nul termination and padding to align 4
-; CHECK-NEXT: 0000 05000000 04000000 01000000 4e61436c
-; CHECK-NEXT: 0010 00000000 61726d00
diff --git a/test/MC/ARM/elf-reloc-01.ll b/test/MC/ARM/elf-reloc-01.ll
deleted file mode 100644
index 7f3cc185af1..00000000000
--- a/test/MC/ARM/elf-reloc-01.ll
+++ /dev/null
@@ -1,67 +0,0 @@
-;; RUN: llc -mtriple=armv7-linux-gnueabi -O3  \
-;; RUN:    -mcpu=cortex-a8 -mattr=-neon -mattr=+vfp2  -arm-reserve-r9  \
-;; RUN:    -filetype=obj %s -o - | \
-;; RUN:   llvm-readobj -r | FileCheck -check-prefix=OBJ %s
-
-;; FIXME: This file needs to be in .s form!
-;; The args to llc are there to constrain the codegen only.
-;; 
-;; Ensure no regression on ARM/gcc compatibility for 
-;; emitting explicit symbol relocs for nonexternal symbols 
-;; versus section symbol relocs (with offset) - 
-;;
-;; Default llvm behavior is to emit as section symbol relocs nearly
-;; everything that is not an undefined external. Unfortunately, this 
-;; diverges from what codesourcery ARM/gcc does!
-;;
-;; Tests that reloc to _MergedGlobals show up as explicit symbol reloc
-
-
-target triple = "armv7-none-linux-gnueabi"
-
-@var_tls = thread_local global i32 1
-@var_tls_double = thread_local global double 1.000000e+00
-@var_static = internal global i32 1
-@var_static_double = internal global double 1.000000e+00
-@var_global = global i32 1
-@var_global_double = global double 1.000000e+00
-
-declare i32 @mystrlen(i8* nocapture %s) nounwind  
-
-declare void @myhextochar(i32 %n, i8* nocapture %buffer)
-
-declare void @__aeabi_read_tp() nounwind 
-
-declare void @__nacl_read_tp() nounwind  
-
-define i32 @main(i32 %argc, i8** nocapture %argv) nounwind {
-entry:
-  switch i32 %argc, label %bb3 [
-    i32 555, label %bb
-    i32 6666, label %bb2
-  ]
-
-bb:                                               ; preds = %entry
-  store volatile i32 11, i32* @var_tls, align 4
-  store volatile double 2.200000e+01, double* @var_tls_double, align 8
-  store volatile i32 33, i32* @var_static, align 4
-  store volatile double 4.400000e+01, double* @var_static_double, align 8
-  store volatile i32 55, i32* @var_global, align 4
-  store volatile double 6.600000e+01, double* @var_global_double, align 8
-  br label %bb3
-
-bb2:                                              ; preds = %entry
-  ret i32 add (i32 add (i32 add (i32 ptrtoint (i32* @var_tls to i32), i32 add (i32 ptrtoint (i32* @var_static to i32), i32 ptrtoint (i32* @var_global to i32))), i32 ptrtoint (double* @var_tls_double to i32)), i32 add (i32 ptrtoint (double* @var_static_double to i32), i32 ptrtoint (double* @var_global_double to i32)))
-
-bb3:                                              ; preds = %bb, %entry
-  tail call void @exit(i32 55) noreturn nounwind
-  unreachable
-}
-
-declare void @exit(i32) noreturn nounwind
-
-; OBJ: Relocations [
-; OBJ:   Section {{.*}} .rel.text {
-; OBJ:     0x{{[0-9,A-F]+}} R_ARM_MOVW_ABS_NC _MergedGlobals
-; OBJ:   }
-; OBJ: ]
diff --git a/test/MC/ARM/elf-reloc-01.s b/test/MC/ARM/elf-reloc-01.s
new file mode 100644
index 00000000000..f3019cdff3c
--- /dev/null
+++ b/test/MC/ARM/elf-reloc-01.s
@@ -0,0 +1,26 @@
+// RUN: llvm-mc -triple=armv7-linux-gnueabi \
+// RUN:    -mcpu=cortex-a8 -mattr=-neon -mattr=+vfp2  \
+// RUN:    -filetype=obj %s -o - | \
+// RUN:   llvm-readobj -r | FileCheck -check-prefix=OBJ %s
+
+// Ensure no regression on ARM/gcc compatibility for
+// emitting explicit symbol relocs for nonexternal symbols
+// versus section symbol relocs (with offset) -
+//
+// Default llvm behavior is to emit as section symbol relocs nearly
+// everything that is not an undefined external. Unfortunately, this
+// diverges from what codesourcery ARM/gcc does!
+//
+// Tests that reloc to _MergedGlobals show up as explicit symbol reloc
+
+	movw	r2, :lower16:_MergedGlobals
+
+_MergedGlobals:
+	.long	1
+
+
+// OBJ: Relocations [
+// OBJ:   Section {{.*}} .rel.text {
+// OBJ:     0x{{[0-9,A-F]+}} R_ARM_MOVW_ABS_NC _MergedGlobals
+// OBJ:   }
+// OBJ: ]
diff --git a/test/MC/ARM/elf-reloc-02.ll b/test/MC/ARM/elf-reloc-02.ll
deleted file mode 100644
index 0ffb6237d61..00000000000
--- a/test/MC/ARM/elf-reloc-02.ll
+++ /dev/null
@@ -1,48 +0,0 @@
-;; RUN: llc -mtriple=armv7-linux-gnueabi -O3  \
-;; RUN:    -mcpu=cortex-a8 -mattr=-neon -mattr=+vfp2  -arm-reserve-r9  \
-;; RUN:    -filetype=obj %s -o - | \
-;; RUN:   llvm-readobj -r | FileCheck -check-prefix=OBJ %s
-
-;; FIXME: This file needs to be in .s form!
-;; The args to llc are there to constrain the codegen only.
-;; 
-;; Ensure no regression on ARM/gcc compatibility for 
-;; emitting explicit symbol relocs for nonexternal symbols 
-;; versus section symbol relocs (with offset) - 
-;;
-;; Default llvm behavior is to emit as section symbol relocs nearly
-;; everything that is not an undefined external. Unfortunately, this 
-;; diverges from what codesourcery ARM/gcc does!
-;;
-;; Tests that reloc to .L.str* show up as explicit symbols
-
-target triple = "armv7-none-linux-gnueabi"
-
-@.str = private constant [7 x i8] c"@null\0A\00", align 4
-@.str1 = private constant [8 x i8] c"@write\0A\00", align 4
-@.str2 = private constant [13 x i8] c"hello worldn\00", align 4
-@.str3 = private constant [7 x i8] c"@exit\0A\00", align 4
-
-declare i32 @mystrlen(i8* nocapture %s) nounwind readonly 
-
-declare void @myhextochar(i32 %n, i8* nocapture %buffer) nounwind 
-
-define i32 @main() nounwind {
-entry:
-  %0 = tail call i32 (...) @write(i32 1, i8* getelementptr inbounds ([7 x i8], [7 x i8]* @.str, i32 0, i32 0), i32 6) nounwind
-  %1 = tail call i32 (...) @write(i32 1, i8* getelementptr inbounds ([8 x i8], [8 x i8]* @.str1, i32 0, i32 0), i32 7) nounwind
-  %2 = tail call i32 (...) @write(i32 1, i8* getelementptr inbounds ([13 x i8], [13 x i8]* @.str2, i32 0, i32 0), i32 12) nounwind
-  %3 = tail call i32 (...) @write(i32 1, i8* getelementptr inbounds ([7 x i8], [7 x i8]* @.str3, i32 0, i32 0), i32 6) nounwind
-  tail call void @exit(i32 55) noreturn nounwind
-  unreachable
-}
-
-declare i32 @write(...)
-
-declare void @exit(i32) noreturn nounwind
-
-;; OBJ:      Relocations [
-;; OBJ:        Section {{.*}} .rel.text {
-;; OBJ-NEXT:     0x{{[0-9,A-F]+}} R_ARM_MOVW_ABS_NC .L.str
-;; OBJ:        }
-;; OBJ:      ]
diff --git a/test/MC/ARM/elf-reloc-02.s b/test/MC/ARM/elf-reloc-02.s
new file mode 100644
index 00000000000..24e2bb3b6fd
--- /dev/null
+++ b/test/MC/ARM/elf-reloc-02.s
@@ -0,0 +1,27 @@
+// RUN: llvm-mc -triple=armv7-linux-gnueabi \
+// RUN:    -mcpu=cortex-a8 -mattr=-neon -mattr=+vfp2  \
+// RUN:    -filetype=obj %s -o - | \
+// RUN:   llvm-readobj -r | FileCheck -check-prefix=OBJ %s
+
+// Ensure no regression on ARM/gcc compatibility for
+// emitting explicit symbol relocs for nonexternal symbols
+// versus section symbol relocs (with offset) -
+//
+// Default llvm behavior is to emit as section symbol relocs nearly
+// everything that is not an undefined external. Unfortunately, this
+// diverges from what codesourcery ARM/gcc does!
+//
+// Tests that reloc to .L.str* show up as explicit symbols
+
+	movw	r1, :lower16:.L.str
+	movt	r1, :upper16:.L.str
+
+	.section	.rodata,"a",%progbits
+.L.str:
+	.asciz	"@null\n"
+
+// OBJ:      Relocations [
+// OBJ:        Section {{.*}} .rel.text {
+// OBJ-NEXT:     0x{{[0-9,A-F]+}} R_ARM_MOVW_ABS_NC .L.str
+// OBJ:        }
+// OBJ:      ]
diff --git a/test/MC/ARM/elf-reloc-03.ll b/test/MC/ARM/elf-reloc-03.ll
deleted file mode 100644
index 4beb91f193f..00000000000
--- a/test/MC/ARM/elf-reloc-03.ll
+++ /dev/null
@@ -1,95 +0,0 @@
-;; RUN: llc -mtriple=armv7-linux-gnueabi -O3  \
-;; RUN:    -mcpu=cortex-a8 -mattr=-neon -mattr=+vfp2  -arm-reserve-r9  \
-;; RUN:    -filetype=obj %s -o - | \
-;; RUN:   llvm-readobj -r | FileCheck -check-prefix=OBJ %s
-
-;; FIXME: This file needs to be in .s form!
-;; The args to llc are there to constrain the codegen only.
-;; 
-;; Ensure no regression on ARM/gcc compatibility for 
-;; emitting explicit symbol relocs for nonexternal symbols 
-;; versus section symbol relocs (with offset) - 
-;;
-;; Default llvm behavior is to emit as section symbol relocs nearly
-;; everything that is not an undefined external. Unfortunately, this 
-;; diverges from what codesourcery ARM/gcc does!
-;;
-;; Verifies that internal constants appear as explict symbol relocs
-
-
-target triple = "armv7-none-linux-gnueabi"
-
-@startval = global i32 5
-@vtable = internal constant [10 x i32 (...)*] [i32 (...)* bitcast (i32 ()* @foo0 to i32 (...)*), i32 (...)* bitcast (i32 ()* @foo1 to i32 (...)*), i32 (...)* bitcast (i32 ()* @foo2 to i32 (...)*), i32 (...)* bitcast (i32 ()* @foo3 to i32 (...)*), i32 (...)* bitcast (i32 ()* @foo4 to i32 (...)*), i32 (...)* bitcast (i32 ()* @foo5 to i32 (...)*), i32 (...)* bitcast (i32 ()* @foo6 to i32 (...)*), i32 (...)* bitcast (i32 ()* @foo7 to i32 (...)*), i32 (...)* bitcast (i32 ()* @foo8 to i32 (...)*), i32 (...)* bitcast (i32 ()* @foo9 to i32 (...)*)]
-
-declare i32 @mystrlen(i8* nocapture %s) nounwind readonly 
-
-declare void @myhextochar(i32 %n, i8* nocapture %buffer) nounwind 
-
-define internal i32 @foo0() nounwind readnone {
-entry:
-  ret i32 0
-}
-
-define internal i32 @foo1() nounwind readnone {
-entry:
-  ret i32 1
-}
-
-define internal i32 @foo2() nounwind readnone {
-entry:
-  ret i32 2
-}
-
-define internal i32 @foo3() nounwind readnone {
-entry:
-  ret i32 3
-}
-
-define internal i32 @foo4() nounwind readnone {
-entry:
-  ret i32 4
-}
-
-define internal i32 @foo5() nounwind readnone {
-entry:
-  ret i32 55
-}
-
-define internal i32 @foo6() nounwind readnone {
-entry:
-  ret i32 6
-}
-
-define internal i32 @foo7() nounwind readnone {
-entry:
-  ret i32 7
-}
-
-define internal i32 @foo8() nounwind readnone {
-entry:
-  ret i32 8
-}
-
-define internal i32 @foo9() nounwind readnone {
-entry:
-  ret i32 9
-}
-
-define i32 @main() nounwind {
-entry:
-  %0 = load i32, i32* @startval, align 4
-  %1 = getelementptr inbounds [10 x i32 (...)*], [10 x i32 (...)*]* @vtable, i32 0, i32 %0
-  %2 = load i32 (...)*, i32 (...)** %1, align 4
-  %3 = tail call i32 (...) %2() nounwind
-  tail call void @exit(i32 %3) noreturn nounwind
-  unreachable
-}
-
-declare void @exit(i32) noreturn nounwind
-
-;; OBJ: Relocations [
-;; OBJ:   Section {{.*}} .rel.text {
-;; OBJ:     0x{{[0-9,A-F]+}} R_ARM_MOVW_ABS_NC vtable
-;; OBJ:   }
-;; OBJ: ]
diff --git a/test/MC/ARM/elf-reloc-03.s b/test/MC/ARM/elf-reloc-03.s
new file mode 100644
index 00000000000..e55b1273769
--- /dev/null
+++ b/test/MC/ARM/elf-reloc-03.s
@@ -0,0 +1,27 @@
+// RUN: llvm-mc -triple=armv7-linux-gnueabi  \
+// RUN:    -mcpu=cortex-a8 -mattr=-neon -mattr=+vfp2   \
+// RUN:    -filetype=obj %s -o - | \
+// RUN:   llvm-readobj -r | FileCheck -check-prefix=OBJ %s
+
+// Ensure no regression on ARM/gcc compatibility for
+// emitting explicit symbol relocs for nonexternal symbols
+// versus section symbol relocs (with offset) -
+//
+// Default llvm behavior is to emit as section symbol relocs nearly
+// everything that is not an undefined external. Unfortunately, this
+// diverges from what codesourcery ARM/gcc does!
+//
+// Verifies that internal constants appear as explict symbol relocs
+
+	movw	r1, :lower16:vtable
+
+
+	.section	.data.rel.ro.local,"aw",%progbits
+vtable:
+	.long	0
+
+// OBJ: Relocations [
+// OBJ:   Section {{.*}} .rel.text {
+// OBJ:     0x{{[0-9,A-F]+}} R_ARM_MOVW_ABS_NC vtable
+// OBJ:   }
+// OBJ: ]
diff --git a/test/MC/ARM/elf-thumbfunc-reloc.ll b/test/MC/ARM/elf-thumbfunc-reloc.ll
deleted file mode 100644
index f35971a6b88..00000000000
--- a/test/MC/ARM/elf-thumbfunc-reloc.ll
+++ /dev/null
@@ -1,45 +0,0 @@
-; RUN: llc %s -mtriple=thumbv7-linux-gnueabi -relocation-model=pic \
-; RUN: -filetype=obj -o - | llvm-readobj -s -sd -r -t | \
-; RUN: FileCheck %s
-
-; FIXME: This file needs to be in .s form!
-; We want to test relocatable thumb function call,
-; but ARMAsmParser cannot handle "bl foo(PLT)" yet
-
-target datalayout = "e-p:32:32:32-i1:8:32-i8:8:32-i16:16:32-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:64:128-a0:0:32-n32"
-target triple = "thumbv7-none--gnueabi"
-
-define void @foo() nounwind {
-entry:
-  ret void
-}
-
-define void @bar() nounwind {
-entry:
-  call void @foo()
-  ret void
-}
-
-
-; make sure that bl 0 <foo> (fff7feff) is correctly encoded
-; CHECK: Sections [
-; CHECK:   SectionData (
-; CHECK:     0000: 704700BF 2DE90048 FFF7FEFF BDE80088
-; CHECK:   )
-; CHECK: ]
-
-; CHECK:      Relocations [
-; CHECK-NEXT:   Section {{.*}} .rel.text {
-; CHECK-NEXT:     0x8 R_ARM_THM_CALL foo 0x0
-; CHECK-NEXT:   }
-; CHECK-NEXT:   Section {{.*}} .rel.ARM.exidx {
-; CHECK-NEXT:     0x0 R_ARM_PREL31 .text 0x0
-; CHECK-NEXT:     0x8 R_ARM_PREL31 .text 0x0
-; CHECK-NEXT:   }
-; CHECK-NEXT: ]
-
-; make sure foo is thumb function: bit 0 = 1
-; CHECK:      Symbols [
-; CHECK:        Symbol {
-; CHECK:          Name: foo
-; CHECK-NEXT:     Value: 0x1
diff --git a/test/MC/ARM/elf-thumbfunc-reloc.s b/test/MC/ARM/elf-thumbfunc-reloc.s
index dd380c36226..70aa8c66b96 100644
--- a/test/MC/ARM/elf-thumbfunc-reloc.s
+++ b/test/MC/ARM/elf-thumbfunc-reloc.s
@@ -29,7 +29,7 @@ ptr:
 
 @@ make sure the relocation is with f. That is one way to make sure it includes
 @@ the thumb bit.
-@CHECK-NEXT:   Section (6) .rel.data.rel.local {
+@CHECK-NEXT:   Section ({{.*}}) .rel.data.rel.local {
 @CHECK-NEXT:     0x0 R_ARM_ABS32 f 0x0
 @CHECK-NEXT:   }
 @CHECK-NEXT: ]
diff --git a/test/MC/ARM/elf-thumbfunc-reloc2.s b/test/MC/ARM/elf-thumbfunc-reloc2.s
new file mode 100644
index 00000000000..54eedcd9575
--- /dev/null
+++ b/test/MC/ARM/elf-thumbfunc-reloc2.s
@@ -0,0 +1,44 @@
+// RUN: llvm-mc %s -triple=thumbv7-linux-gnueabi -relocation-model=pic \
+// RUN: -filetype=obj -o - | llvm-readobj -s -sd -r -t | \
+// RUN: FileCheck %s
+
+// We want to test relocatable thumb function call.
+
+	.thumb_func
+foo:
+	.fnstart
+	bx	lr
+	.cantunwind
+	.fnend
+
+	.align	1
+bar:
+	.fnstart
+	push	{r7, lr}
+	bl	foo(PLT)
+	pop	{r7, pc}
+	.cantunwind
+	.fnend
+
+// make sure that bl 0 <foo> (fff7feff) is correctly encoded
+// CHECK: Sections [
+// CHECK:   SectionData (
+// CHECK:     0000: 704780B5 FFF7FEFF 80BD
+// CHECK:   )
+// CHECK: ]
+
+// CHECK:      Relocations [
+// CHECK-NEXT:   Section {{.*}} .rel.text {
+// CHECK-NEXT:     0x4 R_ARM_THM_CALL foo 0x0
+// CHECK-NEXT:   }
+// CHECK-NEXT:   Section {{.*}} .rel.ARM.exidx {
+// CHECK-NEXT:     0x0 R_ARM_PREL31 .text 0x0
+// CHECK-NEXT:     0x8 R_ARM_PREL31 .text 0x0
+// CHECK-NEXT:   }
+// CHECK-NEXT: ]
+
+// make sure foo is thumb function: bit 0 = 1
+// CHECK:      Symbols [
+// CHECK:        Symbol {
+// CHECK:          Name: foo
+// CHECK-NEXT:     Value: 0x1
diff --git a/test/MC/ARM/inst-directive-emit.s b/test/MC/ARM/inst-directive-emit.s
index 13b7edfa1f3..63fdd15ee38 100644
--- a/test/MC/ARM/inst-directive-emit.s
+++ b/test/MC/ARM/inst-directive-emit.s
@@ -15,6 +15,6 @@ emit_asm:
 @ CHECK: 	.globl	emit_asm
 @ CHECK: 	.type	emit_asm,%function
 @ CHECK: emit_asm:
-@ CHECK: 	inst.w 0xF2400000
-@ CHECK: 	inst.w 0xF2C00000
+@ CHECK: 	inst.w 0xf2400000
+@ CHECK: 	inst.w 0xf2c00000
 
diff --git a/test/MC/ARM/thumb2-dsp-diag.s b/test/MC/ARM/thumb2-dsp-diag.s
index cb0e7744ef4..a87c2120d5a 100644
--- a/test/MC/ARM/thumb2-dsp-diag.s
+++ b/test/MC/ARM/thumb2-dsp-diag.s
@@ -1,24 +1,34 @@
-; RUN: not llvm-mc -triple=thumbv7m < %s 2> %t
-; RUN: FileCheck --check-prefix=CHECK-ERRORS < %t %s
+@ RUN: not llvm-mc -triple=thumbv7m 2>&1 < %s | FileCheck --check-prefix=CHECK-ERRORS %s
+@ RUN: llvm-mc -triple=thumbv7em -show-encoding < %s | FileCheck --check-prefix=CHECK-7EM %s
 
 sxtab r0, r0, r0
 sxtah r0, r0, r0
 sxtab16 r0, r0, r0
 sxtb16 r0, r0
 sxtb16 r0, r0, ror #8
-; CHECK-ERRORS: error: instruction requires: arm-mode
-; CHECK-ERRORS: error: instruction requires: arm-mode
-; CHECK-ERRORS: error: instruction requires: arm-mode
-; CHECK-ERRORS: error: instruction requires: arm-mode
-; CHECK-ERRORS: error: invalid operand for instruction
+@ CHECK-ERRORS: error: instruction requires: arm-mode
+@ CHECK-ERRORS: error: instruction requires: arm-mode
+@ CHECK-ERRORS: error: instruction requires: arm-mode
+@ CHECK-ERRORS: error: instruction requires: arm-mode
+@ CHECK-ERRORS: error: invalid operand for instruction
+@ CHECK-7EM: sxtab	r0, r0, r0              @ encoding: [0x40,0xfa,0x80,0xf0]
+@ CHECK-7EM: sxtah	r0, r0, r0              @ encoding: [0x00,0xfa,0x80,0xf0]
+@ CHECK-7EM: sxtab16	r0, r0, r0              @ encoding: [0x20,0xfa,0x80,0xf0]
+@ CHECK-7EM: sxtb16	r0, r0                  @ encoding: [0x2f,0xfa,0x80,0xf0]
+@ CHECK-7EM: sxtb16	r0, r0, ror #8          @ encoding: [0x2f,0xfa,0x90,0xf0]
 
 uxtab r0, r0, r0
 uxtah r0, r0, r0
 uxtab16 r0, r0, r0
 uxtb16 r0, r0
 uxtb16 r0, r0, ror #8
-; CHECK-ERRORS: error: instruction requires: arm-mode
-; CHECK-ERRORS: error: instruction requires: arm-mode
-; CHECK-ERRORS: error: instruction requires: arm-mode
-; CHECK-ERRORS: error: instruction requires: arm-mode
-; CHECK-ERRORS: error: invalid operand for instruction
+@ CHECK-ERRORS: error: instruction requires: arm-mode
+@ CHECK-ERRORS: error: instruction requires: arm-mode
+@ CHECK-ERRORS: error: instruction requires: arm-mode
+@ CHECK-ERRORS: error: instruction requires: arm-mode
+@ CHECK-ERRORS: error: invalid operand for instruction
+@ CHECK-7EM: uxtab	r0, r0, r0              @ encoding: [0x50,0xfa,0x80,0xf0]
+@ CHECK-7EM: uxtah	r0, r0, r0              @ encoding: [0x10,0xfa,0x80,0xf0]
+@ CHECK-7EM: uxtab16	r0, r0, r0              @ encoding: [0x30,0xfa,0x80,0xf0]
+@ CHECK-7EM: uxtb16	r0, r0                  @ encoding: [0x3f,0xfa,0x80,0xf0]
+@ CHECK-7EM: uxtb16	r0, r0, ror #8          @ encoding: [0x3f,0xfa,0x90,0xf0]
diff --git a/test/MC/ARM/thumb2-narrow-dp.ll b/test/MC/ARM/thumb2-narrow-dp.ll
index ae2ba355d41..050e9713399 100644
--- a/test/MC/ARM/thumb2-narrow-dp.ll
+++ b/test/MC/ARM/thumb2-narrow-dp.ll
@@ -6,11 +6,102 @@
 // Assemblers should chose the narrow thumb encoding when possible, i.e.
 //   - Rd == Rn 
 //   - Rd, Rn and Rm are < r8
-// In addition, some operations are commutative, allowing the transormation 
+// In addition, some operations are commutative, allowing the transformation
 // when:
 //   - Rd == Rn || Rd == Rm
 //   - Rd, Rn and Rm are < r8
 
+// ADD immediate (not SP) A8.8.4
+    ADDS     r0, r0, #5          // T1
+// CHECK: adds  r0, r0, #5          @ encoding: [0x40,0x1d]
+    ADDS     r1, r1, #8          // T2
+// CHECK: adds  r1, #8              @ encoding: [0x08,0x31]
+    ADDS.W   r1, r1, #8          // .w => T3
+// CHECK: adds.w r1, r1, #8         @ encoding: [0x11,0xf1,0x08,0x01]
+    ADDS     r8, r8, #8          // T3
+// CHECK: adds.w r8, r8, #8         @ encoding: [0x18,0xf1,0x08,0x08]
+
+    IT EQ
+// CHECK: it eq                     @ encoding: [0x08,0xbf]
+    ADDEQ    r0, r0, #5          // T1
+// CHECK: addeq r0, r0, #5          @ encoding: [0x40,0x1d]
+    IT EQ
+// CHECK: it eq                     @ encoding: [0x08,0xbf]
+    ADDEQ    r1, r1, #8          // T2
+// CHECK: addeq r1, #8              @ encoding: [0x08,0x31]
+
+    IT EQ
+// CHECK: it eq                     @ encoding: [0x08,0xbf]
+    ADDSEQ   r0, r0, #5          // T3
+// CHECK: addseq.w r0, r0, #5       @ encoding: [0x10,0xf1,0x05,0x00]
+    IT EQ
+// CHECK: it eq                     @ encoding: [0x08,0xbf]
+    ADDSEQ   r1, r1, #8          // T3
+// CHECK: addseq.w r1, r1, #8       @ encoding: [0x11,0xf1,0x08,0x01]
+
+// ADD register (not SP) A8.8.6 (commutative)
+    ADDS     r0, r2, r1          // ADDS has T1 narrow 3 operand
+// CHECK: adds  r0, r2, r1          @ encoding: [0x50,0x18]
+    ADDS     r2, r2, r1          // ADDS has T1 narrow 3 operand
+// CHECK: adds  r2, r2, r1          @ encoding: [0x52,0x18]
+    ADD      r3, r1, r3          // T2
+// CHECK: add  r3, r1               @ encoding: [0x0b,0x44]
+
+    IT EQ
+// CHECK: it eq                     @ encoding: [0x08,0xbf]
+    ADDEQ    r0, r2, r1          // (In IT) ADD has T1 narrow 3 operand
+// CHECK: addeq r0, r2, r1          @ encoding: [0x50,0x18]
+    IT EQ
+// CHECK: it eq                     @ encoding: [0x08,0xbf]
+    ADDEQ    r2, r2, r1          // (In IT) ADD has T1 narrow 3 operand
+// CHECK: addeq r2, r2, r1          @ encoding: [0x52,0x18]
+
+    IT EQ
+// CHECK: it eq                     @ encoding: [0x08,0xbf]
+    ADDSEQ   r0, r2, r1          // T3
+// CHECK: addseq.w r0, r2, r1       @ encoding: [0x12,0xeb,0x01,0x00]
+    IT EQ
+// CHECK: it eq                     @ encoding: [0x08,0xbf]
+    ADDSEQ   r2, r2, r1          // T3
+// CHECK: addseq.w r2, r2, r1       @ encoding: [0x12,0xeb,0x01,0x02]
+
+    ADD      r3, r3, r1          // T2
+// CHECK: add  r3, r1               @ encoding: [0x0b,0x44]
+    ADD      r4, r4, pc          // T2
+// CHECK: add  r4, pc               @ encoding: [0x7c,0x44]
+    ADD      r4, pc, r4          // T2
+// CHECK: add  r4, pc               @ encoding: [0x7c,0x44]
+    ADD      pc, pc, r2          // T2
+// CHECK: add  pc, r2               @ encoding: [0x97,0x44]
+    ADD      pc, r2, pc          // T2
+// CHECK: add  pc, r2               @ encoding: [0x97,0x44]
+    ADD      pc, pc, sp          // T2
+// CHECK: add   pc, sp              @ encoding: [0xef,0x44]
+    ADD      pc, sp, pc          // T2
+// CHECK: add   pc, sp, pc          @ encoding: [0xef,0x44]
+
+// ADD (SP plus immediate) A8.8.9
+    ADD      sp, sp, #20         // T2
+// FIXME: ARMARM says 'add   sp, sp, #20'
+// CHECK: add   sp, #20             @ encoding: [0x05,0xb0]
+    ADD      sp, sp, #508        // T2
+// CHECK: add   sp, #508            @ encoding: [0x7f,0xb0]
+    ADD      sp, sp, #512        // T3
+// CHECK: add.w sp, sp, #512        @ encoding: [0x0d,0xf5,0x00,0x7d]
+
+// ADD (SP plus register) A8.8.10 (commutative)
+    ADD      r9, sp, r9          // T1
+// CHECK: add   r9, sp, r9          @ encoding: [0xe9,0x44]
+    ADD      r9, r9, sp          // T1
+// FIXME: ARMARM says 'add   r9, sp, r9'
+// CHECK: add   r9, sp              @ encoding: [0xe9,0x44]
+    ADD      sp, sp, r10         // T2
+// CHECK: add   sp, r10             @ encoding: [0xd5,0x44]
+    ADD      sp, r10, sp         // T2
+// CHECK: add   sp, r10             @ encoding: [0xd5,0x44]
+    ADD      sp, sp, pc          // T2
+// CHECK: add   sp, pc              @ encoding: [0xfd,0x44]
+
 // AND (commutative)
     ANDS     r0, r2, r1          // Must be wide - 3 distinct registers
     ANDS     r2, r2, r1          // Should choose narrow
diff --git a/test/MC/ARM/thumb_rewrites.s b/test/MC/ARM/thumb_rewrites.s
index c9d625e60de..06c77e89862 100644
--- a/test/MC/ARM/thumb_rewrites.s
+++ b/test/MC/ARM/thumb_rewrites.s
@@ -1,4 +1,11 @@
 @ RUN: llvm-mc -triple thumbv6m -show-encoding < %s | FileCheck %s
+@ RUN: llvm-mc -triple thumbv7m -show-encoding < %s | FileCheck %s
+
+    adds    r1, r1, #3
+@ CHECK: adds   r1, r1, #3          @ encoding: [0xc9,0x1c]
+
+    adds    r1, #3
+@ CHECK: adds   r1, #3              @ encoding: [0x03,0x31]
 
     adds    r0, r0, #8
 @ CHECK: adds   r0, #8              @ encoding: [0x08,0x30]
@@ -9,9 +16,25 @@
     add     r0, r0, r8
 @ CHECK: add    r0, r8              @ encoding: [0x40,0x44]
 
+    add     r1, r8, r1
+@ CHECK: add    r1, r8              @ encoding: [0x41,0x44]
+
     add     sp, sp, r0
 @ CHECK: add    sp, r0              @ encoding: [0x85,0x44]
 
+    add     r4, sp, r4
+@ CHECK: add    r4, sp, r4          @ encoding: [0x6c,0x44]
+
+    add     r4, r4, sp
+@ CHECK: add    r4, sp              @ encoding: [0x6c,0x44]
+
+    add     sp, sp, #32
+@ FIXME: ARMARM says 'add   sp, sp, #32'
+@ CHECK: add    sp, #32             @ encoding: [0x08,0xb0]
+
+    add     r5, sp, #1016
+@ CHECK: add    r5, sp, #1016       @ encoding: [0xfe,0xad]
+
     add     r0, r0, r1
 @ CHECK: add    r0, r1              @ encoding: [0x08,0x44]
 
@@ -21,12 +44,30 @@
     subs    r0, r0, r0
 @ CHECK: subs   r0, r0, r0          @ encoding: [0x00,0x1a]
 
+    subs    r3, r3, #5
+@ CHECK: subs   r3, r3, #5          @ encoding: [0x5b,0x1f]
+
+    subs    r3, #5
+@ CHECK: subs   r3, #5              @ encoding: [0x05,0x3b]
+
+    subs    r2, r2, #8
+@ CHECK: subs   r2, #8              @ encoding: [0x08,0x3a]
+
+    sub     sp, sp, #16
+@ CHECK: sub    sp, #16             @ encoding: [0x84,0xb0]
+
+    ands    r0, r1, r0
+@ CHECK: ands   r0, r1              @ encoding: [0x08,0x40]
+
     ands    r0, r0, r1
 @ CHECK: ands   r0, r1              @ encoding: [0x08,0x40]
 
     eors    r0, r0, r1
 @ CHECK: eors   r0, r1              @ encoding: [0x48,0x40]
 
+    eors    r0, r1, r0
+@ CHECK: eors   r0, r1              @ encoding: [0x48,0x40]
+
     lsls    r0, r0, r1
 @ CHECK: lsls   r0, r1              @ encoding: [0x88,0x40]
 
@@ -39,6 +80,9 @@
     adcs    r0, r0, r1
 @ CHECK: adcs   r0, r1              @ encoding: [0x48,0x41]
 
+    adcs    r0, r1, r0
+@ CHECK: adcs   r0, r1              @ encoding: [0x48,0x41]
+
     sbcs    r0, r0, r1
 @ CHECK: sbcs   r0, r1              @ encoding: [0x88,0x41]
 
@@ -48,5 +92,8 @@
     orrs    r0, r0, r1
 @ CHECK: orrs   r0, r1              @ encoding: [0x08,0x43]
 
+    orrs    r0, r1, r0
+@ CHECK: orrs   r0, r1              @ encoding: [0x08,0x43]
+
     bics    r0, r0, r1
 @ CHECK: bics   r0, r1              @ encoding: [0x88,0x43]
diff --git a/test/MC/ARM/thumb_set-diagnostics.s b/test/MC/ARM/thumb_set-diagnostics.s
index 5f1844de01f..86f1ee5245c 100644
--- a/test/MC/ARM/thumb_set-diagnostics.s
+++ b/test/MC/ARM/thumb_set-diagnostics.s
@@ -41,3 +41,31 @@
 @ CHECK: 	.thumb_set trailer_trash, 0x11fe1e55,
 @ CHECK:                                            ^
 
+	.type alpha,%function
+alpha:
+	nop
+
+        .type beta,%function
+beta:
+	bkpt
+
+	.thumb_set beta, alpha
+
+@ CHECK: error: redefinition of 'beta'
+@ CHECK: 	.thumb_set beta, alpha
+@ CHECK:                                            ^
+
+	.type recursive_use,%function
+	.thumb_set recursive_use, recursive_use + 1
+
+@ CHECK: error: Recursive use of 'recursive_use'
+@ CHECK: 	.thumb_set recursive_use, recursive_use + 1
+@ CHECK:                                            ^
+
+  variable_result = alpha + 1
+  .long variable_result
+	.thumb_set variable_result, 1
+
+@ CHECK: error: invalid reassignment of non-absolute variable 'variable_result'
+@ CHECK: 	.thumb_set variable_result, 1
+@ CHECK:                                            ^
\ No newline at end of file
diff --git a/test/MC/ARM/thumb_set.s b/test/MC/ARM/thumb_set.s
index d2a0dc04730..00b3e53e724 100644
--- a/test/MC/ARM/thumb_set.s
+++ b/test/MC/ARM/thumb_set.s
@@ -54,8 +54,6 @@ alpha:
 	nop
 
         .type beta,%function
-beta:
-	bkpt
 
 	.thumb_set beta, alpha
 
diff --git a/test/MC/AsmParser/at-pseudo-variable-bad.s b/test/MC/AsmParser/at-pseudo-variable-bad.s
new file mode 100644
index 00000000000..885f18b2d4f
--- /dev/null
+++ b/test/MC/AsmParser/at-pseudo-variable-bad.s
@@ -0,0 +1,23 @@
+# RUN: not llvm-mc -triple i386-unknown-unknown < %s 2>&1 | FileCheck %s
+
+add  $1\@, %eax
+# CHECK: :[[@LINE-1]]:8: error: unexpected token in argument list
+
+.macro A @
+  mov  %eax, %eax
+.endm
+# CHECK: :[[@LINE-3]]:11: error: expected identifier in '.macro' directive
+
+.rept 2
+  addi $8, $8, \@
+.endr
+# CHECK: error: unknown token in expression
+# CHECK: :[[@LINE-4]]:1: note: while in macro instantiation
+# CHECK-NEXT: .rept 2
+
+.rep 3
+  addi $9, $9, \@
+.endr
+# CHECK: error: unknown token in expression
+# CHECK: :[[@LINE-4]]:1: note: while in macro instantiation
+# CHECK-NEXT: .rep 3
diff --git a/test/MC/AsmParser/at-pseudo-variable.s b/test/MC/AsmParser/at-pseudo-variable.s
new file mode 100644
index 00000000000..b0966e13717
--- /dev/null
+++ b/test/MC/AsmParser/at-pseudo-variable.s
@@ -0,0 +1,64 @@
+# RUN: llvm-mc -triple i386-unknown-unknown %s | FileCheck %s
+
+.macro A
+  add  $1\@, %eax
+.endm
+
+.macro B
+  sub  $1\@, %eax
+.endm
+
+  A
+# CHECK: addl  $10, %eax
+  A
+# CHECK: addl  $11, %eax
+  B
+# CHECK: subl  $12, %eax
+  B
+# CHECK: subl  $13, %eax
+
+# The following uses of \@ are undocumented, but valid:
+.irpc foo,234
+  add  $\foo\@, %eax
+.endr
+# CHECK: addl  $24, %eax
+# CHECK: addl  $34, %eax
+# CHECK: addl  $44, %eax
+
+.irp reg,%eax,%ebx
+  sub  $2\@, \reg
+.endr
+# CHECK: subl  $24, %eax
+# CHECK: subl  $24, %ebx
+
+# Test that .irp(c) and .rep(t) do not increase \@.
+# Only the use of A should increase \@, so we can test that it increases by 1
+# each time.
+
+.irpc foo,123
+  sub  $\foo, %eax
+.endr
+
+  A
+# CHECK: addl  $14, %eax
+
+.irp reg,%eax,%ebx
+  sub  $4, \reg
+.endr
+
+  A
+# CHECK: addl  $15, %eax
+
+.rept 2
+  sub  $5, %eax
+.endr
+
+  A
+# CHECK: addl  $16, %eax
+
+.rep 3
+  sub  $6, %eax
+.endr
+
+  A
+# CHECK: addl  $17, %eax
diff --git a/test/MC/AsmParser/defsym.s b/test/MC/AsmParser/defsym.s
new file mode 100644
index 00000000000..06981f54046
--- /dev/null
+++ b/test/MC/AsmParser/defsym.s
@@ -0,0 +1,20 @@
+# RUN: llvm-mc -filetype=obj -triple=i386-unknown-elf -defsym a=7 -defsym b=11 %s | llvm-objdump -t - | FileCheck %s
+
+.ifndef a
+.err 
+.endif
+
+.if a<>7
+.err
+.endif
+
+.ifndef b
+.err
+.endif
+
+.if b<>11
+.err
+.endif
+
+# CHECK: 00000007         *ABS*  00000000 a
+# CHECK: 0000000b         *ABS*  00000000 b
\ No newline at end of file
diff --git a/test/MC/AsmParser/defsym_error1.s b/test/MC/AsmParser/defsym_error1.s
new file mode 100644
index 00000000000..872577844d4
--- /dev/null
+++ b/test/MC/AsmParser/defsym_error1.s
@@ -0,0 +1,2 @@
+# RUN: not llvm-mc -filetype=obj -triple=i386-unknown-elf -defsym aaoeuaoeu %s 2>&1 | FileCheck %s
+# CHECK: defsym must be of the form: sym=value
diff --git a/test/MC/AsmParser/defsym_error2.s b/test/MC/AsmParser/defsym_error2.s
new file mode 100644
index 00000000000..ec4cc79182f
--- /dev/null
+++ b/test/MC/AsmParser/defsym_error2.s
@@ -0,0 +1,2 @@
+# RUN: not llvm-mc -filetype=obj -triple=i386-unknown-elf -defsym a=a %s 2>&1 | FileCheck %s
+# CHECK: error: Value is not an integer: a
diff --git a/test/MC/AsmParser/expr-shr.s b/test/MC/AsmParser/expr-shr.s
new file mode 100644
index 00000000000..fc117b64e95
--- /dev/null
+++ b/test/MC/AsmParser/expr-shr.s
@@ -0,0 +1,13 @@
+// RUN: llvm-mc -triple x86_64-unknown-unknown-elf %s | FileCheck %s --check-prefix=CHECK
+// RUN: llvm-mc -triple x86_64-pc-windows-msvc %s | FileCheck %s --check-prefix=MSVC
+// RUN: llvm-mc -triple x86_64-unknown-darwin %s | FileCheck %s --check-prefix=DARWIN
+
+.data
+
+// CHECK: .quad 3
+
+// Both COFF and Darwin still use AShr.
+// MSVC: .quad -1
+// DARWIN: .quad -1
+
+.quad (~0 >> 62)
diff --git a/test/MC/AsmParser/section.s b/test/MC/AsmParser/section.s
index 0c3828d714a..2ef545b822c 100644
--- a/test/MC/AsmParser/section.s
+++ b/test/MC/AsmParser/section.s
@@ -48,7 +48,7 @@
 
 # CHECK:      Sections [
 # CHECK:        Section {
-# CHECK:          Name: test1 (68)
+# CHECK:          Name: test1
 # CHECK-NEXT:     Type: SHT_PROGBITS
 # CHECK-NEXT:     Flags [ (0x0)
 # CHECK-NEXT:     ]
@@ -64,7 +64,7 @@
 # CHECK-NEXT:     )
 # CHECK-NEXT:   }
 # CHECK:        Section {
-# CHECK:          Name: test2 (62)
+# CHECK:          Name: test2
 # CHECK-NEXT:     Type: SHT_PROGBITS
 # CHECK-NEXT:     Flags [ (0x0)
 # CHECK-NEXT:     ]
@@ -80,7 +80,7 @@
 # CHECK-NEXT:     )
 # CHECK-NEXT:   }
 # CHECK:        Section {
-# CHECK:          Name: test3 (56)
+# CHECK:          Name: test3
 # CHECK-NEXT:     Type: SHT_PROGBITS
 # CHECK-NEXT:     Flags [ (0x0)
 # CHECK-NEXT:     ]
@@ -96,7 +96,7 @@
 # CHECK-NEXT:     )
 # CHECK-NEXT:   }
 # CHECK:        Section {
-# CHECK:          Name: test4 (50)
+# CHECK:          Name: test4
 # CHECK-NEXT:     Type: SHT_PROGBITS
 # CHECK-NEXT:     Flags [ (0x0)
 # CHECK-NEXT:     ]
@@ -112,7 +112,7 @@
 # CHECK-NEXT:     )
 # CHECK-NEXT:   }
 # CHECK:        Section {
-# CHECK:          Name: test5 (44)
+# CHECK:          Name: test5
 # CHECK-NEXT:     Type: SHT_PROGBITS
 # CHECK-NEXT:     Flags [ (0x0)
 # CHECK-NEXT:     ]
diff --git a/test/MC/COFF/ARM/directive-type-diagnostics.s b/test/MC/COFF/ARM/directive-type-diagnostics.s
new file mode 100644
index 00000000000..f8a52cd43e4
--- /dev/null
+++ b/test/MC/COFF/ARM/directive-type-diagnostics.s
@@ -0,0 +1,10 @@
+// RUN: not llvm-mc -triple arm-coff -filetype asm -o /dev/null %s 2>&1 | FileCheck %s
+// RUN: not llvm-mc -triple armeb-coff -filetype asm -o /dev/null %s 2>&1 | FileCheck %s
+// RUN: not llvm-mc -triple thumb-coff -filetype asm -o /dev/null %s 2>&1 | FileCheck %s
+// RUN: not llvm-mc -triple thumbeb-coff -filetype asm -o /dev/null %s 2>&1 | FileCheck %s
+
+        .type symbol 32
+// CHECK: error: expected STT_<TYPE_IN_UPPER_CASE>, '#<type>', '%<type>' or "<type>"
+// CHECK: .type symbol 32
+// CHECK:              ^
+
diff --git a/test/MC/COFF/ARM/lit.local.cfg b/test/MC/COFF/ARM/lit.local.cfg
new file mode 100644
index 00000000000..98c6700c209
--- /dev/null
+++ b/test/MC/COFF/ARM/lit.local.cfg
@@ -0,0 +1,3 @@
+if not 'ARM' in config.root.targets:
+    config.unsupported = True
+
diff --git a/test/MC/COFF/cross-section-relative.ll b/test/MC/COFF/cross-section-relative.ll
new file mode 100644
index 00000000000..6b7a3d7b40b
--- /dev/null
+++ b/test/MC/COFF/cross-section-relative.ll
@@ -0,0 +1,37 @@
+; Verify the assembler produces the expected expressions
+; RUN: llc -mtriple=x86_64-pc-win32 %s -o - | FileCheck %s
+
+;;;; some globals
+
+@g1 = constant i32 1;
+@g2 = constant i32 2;
+@g3 = constant i32 3;
+@g4 = constant i32 4;
+@__ImageBase = external global i64*;
+
+;;;; cross-section relative relocations
+
+; CHECK: .quad (g3-t1)+4
+@t1 = global i64 add(i64 sub(i64 ptrtoint(i32* @g3 to i64), i64 ptrtoint(i64* @t1 to i64)), i64 4), section ".fix"
+; CHECK: .quad g3-t2
+@t2 = global i64 sub(i64 ptrtoint(i32* @g3 to i64), i64 ptrtoint(i64* @t2 to i64)), section ".fix"
+; CHECK: .quad (g3-t3)-4
+@t3 = global i64 sub(i64 sub(i64 ptrtoint(i32* @g3 to i64), i64 ptrtoint(i64* @t3 to i64)), i64 4), section ".fix"
+; CHECK: .long g3-t4
+@t4 = global i32 trunc(i64 sub(i64 ptrtoint(i32* @g3 to i64), i64 ptrtoint(i32* @t4 to i64)) to i32), section ".fix"
+
+;;;; image base relocation
+
+; CHECK: .long g3@IMGREL
+@t5 = global i32 trunc(i64 sub(i64 ptrtoint(i32* @g3 to i64), i64 ptrtoint(i64** @__ImageBase to i64)) to i32), section ".fix"
+
+;;;; cross-section relative with source offset
+
+%struct.EEType = type { [2 x i8], i64, i32}
+
+; CHECK: .long g3-(t6+16)
+@t6 = global %struct.EEType { 
+        [2 x i8] c"\01\02", 
+        i64 256,
+        i32 trunc(i64 sub(i64 ptrtoint(i32* @g3 to i64), i64 ptrtoint(i32* getelementptr inbounds (%struct.EEType, %struct.EEType* @t6, i32 0, i32 2) to i64)) to i32 )
+}, section ".fix"
diff --git a/test/MC/COFF/cross-section-relative.s b/test/MC/COFF/cross-section-relative.s
new file mode 100644
index 00000000000..dd94b0a5e83
--- /dev/null
+++ b/test/MC/COFF/cross-section-relative.s
@@ -0,0 +1,118 @@
+// Verify the .fix data section conveys the right offsets and the right relocations
+// RUN: llvm-mc -filetype=obj -triple x86_64-pc-win32 %s -o - | llvm-readobj -relocations -expand-relocs -sections -section-data | FileCheck %s --check-prefix=READOBJ
+
+        .text
+	.section	.rdata,"dr"
+	.globl	g1                      # @g1
+	.align	4
+g1:
+	.long	1                       # 0x1
+
+	.globl	g2                      # @g2
+	.align	4
+g2:
+	.long	2                       # 0x2
+
+	.globl	g3                      # @g3
+	.align	4
+g3:
+	.long	3                       # 0x3
+
+	.globl	g4                      # @g4
+	.align	4
+g4:
+	.long	4                       # 0x4
+
+	.section	.fix,"dw"
+	.globl	t1                      # @t1
+	.align	8
+t1:
+	.quad	(g3-t1)+4
+
+	.globl	t2                      # @t2
+	.align	8
+t2:
+	.quad	g3-t2
+
+	.globl	t3                      # @t3
+	.align	8
+t3:
+	.quad	(g3-t3)-4
+
+	.globl	t4                      # @t4
+	.align	4
+t4:
+	.long	g3-t4
+
+	.globl	t5                      # @t5
+	.align	4
+t5:
+	.long	g3@IMGREL
+
+	.globl	t6                      # @t6
+	.align	16
+t6:
+	.ascii	"\001\002"
+	.zero	6
+	.quad	256                     # 0x100
+	.long	g3-(t6+16)
+	.zero	4
+
+
+// READOBJ:  Section {
+// READOBJ:    Number: 5
+// READOBJ:    Name: .fix (2E 66 69 78 00 00 00 00)
+// READOBJ:    VirtualSize: 0x0
+// READOBJ:    VirtualAddress: 0x0
+// READOBJ:    RawDataSize: 56
+// READOBJ:    PointerToRawData: 0xEC
+// READOBJ:    PointerToRelocations: 0x124
+// READOBJ:    PointerToLineNumbers: 0x0
+// READOBJ:    RelocationCount: 6
+// READOBJ:    LineNumberCount: 0
+// READOBJ:    Characteristics [ (0xC0500040)
+// READOBJ:      IMAGE_SCN_ALIGN_16BYTES (0x500000)
+// READOBJ:      IMAGE_SCN_CNT_INITIALIZED_DATA (0x40)
+// READOBJ:      IMAGE_SCN_MEM_READ (0x40000000)
+// READOBJ:      IMAGE_SCN_MEM_WRITE (0x80000000)
+// READOBJ:    ]
+// READOBJ:    SectionData (
+// READOBJ:      0000: 10000000 00000000 0C000000 00000000  |................|
+// READOBJ:      0010: 08000000 00000000 0C000000 00000000  |................|
+// READOBJ:      0020: 01020000 00000000 00010000 00000000  |................|
+// READOBJ:      0030: 0C000000 00000000                    |........|
+// READOBJ:    )
+// READOBJ:  }
+// READOBJ:  ]
+// READOBJ:  Relocations [
+// READOBJ:  Section (5) .fix {
+// READOBJ:    Relocation {
+// READOBJ:      Offset: 0x0
+// READOBJ:      Type: IMAGE_REL_AMD64_REL32 (4)
+// READOBJ:      Symbol: .rdata
+// READOBJ:    }
+// READOBJ:    Relocation {
+// READOBJ:      Offset: 0x8
+// READOBJ:      Type: IMAGE_REL_AMD64_REL32 (4)
+// READOBJ:      Symbol: .rdata
+// READOBJ:    }
+// READOBJ:    Relocation {
+// READOBJ:      Offset: 0x10
+// READOBJ:      Type: IMAGE_REL_AMD64_REL32 (4)
+// READOBJ:      Symbol: .rdata
+// READOBJ:    }
+// READOBJ:    Relocation {
+// READOBJ:      Offset: 0x18
+// READOBJ:      Type: IMAGE_REL_AMD64_REL32 (4)
+// READOBJ:      Symbol: .rdata
+// READOBJ:    }
+// READOBJ:    Relocation {
+// READOBJ:      Offset: 0x1C
+// READOBJ:      Type: IMAGE_REL_AMD64_ADDR32NB (3)
+// READOBJ:      Symbol: g3
+// READOBJ:    }
+// READOBJ:    Relocation {
+// READOBJ:      Offset: 0x30
+// READOBJ:      Type: IMAGE_REL_AMD64_REL32 (4)
+// READOBJ:      Symbol: .rdata
+// READOBJ:    }
diff --git a/test/MC/COFF/pr23025.s b/test/MC/COFF/pr23025.s
new file mode 100644
index 00000000000..f0d007543a4
--- /dev/null
+++ b/test/MC/COFF/pr23025.s
@@ -0,0 +1,23 @@
+// RUN: llvm-mc -filetype=obj -triple x86_64-pc-windows-msvc < %s | llvm-readobj -r | FileCheck %s
+
+// CHECK:      Relocations [
+// CHECK-NEXT:   Section {{.*}} .text {
+// CHECK-NEXT:     0x3 IMAGE_REL_AMD64_REL32 zed
+// CHECK-NEXT:     0xA IMAGE_REL_AMD64_REL32 zed2
+// CHECK-NEXT:   }
+// CHECK-NEXT: ]
+
+foo:
+	leaq	zed(%rip), %rax
+	leaq	zed2(%rip), %rax
+	retq
+
+	.section	.rdata,"dr",discard,zed
+Lbar:
+	.zero	2
+
+	.globl	zed
+zed = Lbar+1
+
+	.globl	zed2
+zed2 = Lbar
diff --git a/test/MC/COFF/safeseh.s b/test/MC/COFF/safeseh.s
new file mode 100644
index 00000000000..d21628daff5
--- /dev/null
+++ b/test/MC/COFF/safeseh.s
@@ -0,0 +1,6 @@
+// RUN: llvm-mc -triple i686-pc-win32 %s | FileCheck %s
+
+// check that we quote the output of .safeseh
+
+.safeseh "\01foo"
+// CHECK: .safeseh "\01foo"
diff --git a/test/MC/Disassembler/AArch64/armv8.1a-atomic.txt b/test/MC/Disassembler/AArch64/armv8.1a-atomic.txt
new file mode 100644
index 00000000000..b20fabb8c6f
--- /dev/null
+++ b/test/MC/Disassembler/AArch64/armv8.1a-atomic.txt
@@ -0,0 +1,87 @@
+# RUN: llvm-mc -triple aarch64-none-linux-gnu -mattr=+v8.1a --disassemble < %s | FileCheck %s
+
+0x41,0x7c,0xa0,0x08
+0x41,0x7c,0xe0,0x08
+0x41,0xfc,0xa0,0x08
+0x41,0xfc,0xe0,0x08
+0x41,0x7c,0xa0,0x48
+0x41,0x7c,0xe0,0x48
+0x41,0xfc,0xa0,0x48
+0x41,0xfc,0xe0,0x48
+# CHECK:  casb   w0, w1, [x2]
+# CHECK:  casab  w0, w1, [x2]
+# CHECK:  caslb   w0, w1, [x2]
+# CHECK:  casalb   w0, w1, [x2]
+# CHECK:  cash   w0, w1, [x2]
+# CHECK:  casah  w0, w1, [x2]
+# CHECK:  caslh   w0, w1, [x2]
+# CHECK:  casalh   w0, w1, [x2]
+
+0x41,0x7c,0xa0,0x88
+0x41,0x7c,0xe0,0x88
+0x41,0xfc,0xa0,0x88
+0x41,0xfc,0xe0,0x88
+0x41,0x7c,0xa0,0xc8
+0x41,0x7c,0xe0,0xc8
+0x41,0xfc,0xa0,0xc8
+0x41,0xfc,0xe0,0xc8
+# CHECK:  cas   w0, w1, [x2]
+# CHECK:  casa  w0, w1, [x2]
+# CHECK:  casl   w0, w1, [x2]
+# CHECK:  casal   w0, w1, [x2]
+# CHECK:  cas   x0, x1, [x2]
+# CHECK:  casa  x0, x1, [x2]
+# CHECK:  casl   x0, x1, [x2]
+# CHECK:  casal   x0, x1, [x2]
+
+0x41,0x80,0x20,0xf8
+0x41,0x80,0x20,0x38
+0x41,0x80,0x60,0x78
+0xe1,0x83,0xe0,0xf8
+# CHECK: swp   x0, x1, [x2]
+# CHECK: swpb  w0, w1, [x2]
+# CHECK: swplh w0, w1, [x2]
+# CHECK: swpal x0, x1, [sp]
+
+0x41,0x00,0xa0,0xf8
+0x41,0x10,0x60,0xf8
+0x41,0x20,0xe0,0xf8
+0x41,0x30,0x20,0xf8
+0x41,0x40,0xa0,0xb8
+0x41,0x50,0x60,0x38
+0x41,0x60,0xe0,0x78
+0x41,0x70,0x20,0xb8
+0xab,0x51,0xe7,0x78
+# CHECK:  ldadda    x0, x1, [x2]
+# CHECK:  ldclrl    x0, x1, [x2]
+# CHECK:  ldeoral   x0, x1, [x2]
+# CHECK:  ldset     x0, x1, [x2]
+# CHECK:  ldsmaxa   w0, w1, [x2]
+# CHECK:  ldsminlb  w0, w1, [x2]
+# CHECK:  ldumaxalh w0, w1, [x2]
+# CHECK:  ldumin    w0, w1, [x2]
+# CHECK:  ldsminalh w7, w11, [x13]
+
+0x5f,0x00,0x60,0x38
+0x5f,0x10,0x60,0x78
+0x5f,0x20,0x60,0xb8
+0x5f,0x30,0x60,0xf8
+0x5f,0x40,0x20,0x38
+0x5f,0x50,0x20,0x78
+0x5f,0x60,0x20,0xb8
+0x5f,0x70,0x20,0xf8
+0xff,0x53,0x7d,0xf8
+# CHECK: staddlb    w0, [x2]
+# CHECK: stclrlh    w0, [x2]
+# CHECK: steorl     w0, [x2]
+# CHECK: stsetl     x0, [x2]
+# CHECK: stsmaxb     w0, [x2]
+# CHECK: stsminh     w0, [x2]
+# CHECK: stumax      w0, [x2]
+# CHECK: stumin      x0, [x2]
+# CHECK: stsminl     x29, [sp]
+
+0x82,0x7c,0x20,0x48
+0x82,0x7c,0x20,0x08
+# CHECK: casp x0, x1, x2, x3, [x4]
+# CHECK: casp w0, w1, w2, w3, [x4]
diff --git a/test/MC/Disassembler/AArch64/armv8.1a-lor.txt b/test/MC/Disassembler/AArch64/armv8.1a-lor.txt
index 5f8e725ba20..1194351eab9 100644
--- a/test/MC/Disassembler/AArch64/armv8.1a-lor.txt
+++ b/test/MC/Disassembler/AArch64/armv8.1a-lor.txt
@@ -20,9 +20,19 @@
 0x20,0xa4,0x18,0xd5
 0x40,0xa4,0x18,0xd5
 0x60,0xa4,0x18,0xd5
-0xe0,0xa4,0x38,0xd5
+0xe0,0xa4,0x18,0xd5
 # CHECK:   msr LORSA_EL1, x0
 # CHECK:   msr LOREA_EL1, x0
 # CHECK:   msr LORN_EL1, x0
 # CHECK:   msr LORC_EL1, x0
+# CHECK:   msr S3_0_C10_C4_7, x0
+0x00,0xa4,0x38,0xd5
+0x20,0xa4,0x38,0xd5
+0x40,0xa4,0x38,0xd5
+0x60,0xa4,0x38,0xd5
+0xe0,0xa4,0x38,0xd5
+# CHECK:   mrs x0, LORSA_EL1
+# CHECK:   mrs x0, LOREA_EL1
+# CHECK:   mrs x0, LORN_EL1
+# CHECK:   mrs x0, LORC_EL1
 # CHECK:   mrs x0, LORID_EL1
diff --git a/test/MC/Disassembler/AArch64/basic-a64-instructions.txt b/test/MC/Disassembler/AArch64/basic-a64-instructions.txt
index 23da001accb..089fc821097 100644
--- a/test/MC/Disassembler/AArch64/basic-a64-instructions.txt
+++ b/test/MC/Disassembler/AArch64/basic-a64-instructions.txt
@@ -622,7 +622,7 @@
 
 # CHECK: bfi      x4, x5, #52, #11
 # CHECK: bfxil    xzr, x4, #0, #1
-# CHECK: bfi      x4, xzr, #1, #6
+# CHECK: bfc      x4, #1, #6
 # CHECK: bfxil    x5, x6, #12, #52
 0xa4 0x28 0x4c 0xb3
 0x9f 0x0 0x40 0xb3
@@ -715,7 +715,7 @@
 # CHECK: bfxil    w9, w10, #0, #32
 # CHECK: bfi      w11, w12, #31, #1
 # CHECK: bfi      w13, w14, #29, #3
-# CHECK: bfi      xzr, xzr, #10, #11
+# CHECK: bfc      xzr, #10, #11
 0x49 0x1 0x0 0x33
 0x62 0x0 0x41 0xb3
 0x93 0xfe 0x40 0xb3
@@ -3414,6 +3414,7 @@
 # CHECK: mrs      x9, {{id_mmfr1_el1|ID_MMFR1_EL1}}
 # CHECK: mrs      x9, {{id_mmfr2_el1|ID_MMFR2_EL1}}
 # CHECK: mrs      x9, {{id_mmfr3_el1|ID_MMFR3_EL1}}
+# CHECK: mrs      x9, {{id_mmfr4_el1|ID_MMFR4_EL1}}
 # CHECK: mrs      x9, {{id_isar0_el1|ID_ISAR0_EL1}}
 # CHECK: mrs      x9, {{id_isar1_el1|ID_ISAR1_EL1}}
 # CHECK: mrs      x9, {{id_isar2_el1|ID_ISAR2_EL1}}
@@ -3968,6 +3969,7 @@
 0xa9 0x1 0x38 0xd5
 0xc9 0x1 0x38 0xd5
 0xe9 0x1 0x38 0xd5
+0xc9 0x2 0x38 0xd5
 0x9 0x2 0x38 0xd5
 0x29 0x2 0x38 0xd5
 0x49 0x2 0x38 0xd5
@@ -4170,12 +4172,16 @@
 
 # CHECK: mrs     x12, {{s3_7_c15_c1_5|S3_7_C15_C1_5}}
 # CHECK: mrs     x13, {{s3_2_c11_c15_7|S3_2_C11_C15_7}}
+# CHECK: mrs     xzr, {{s0_0_c4_c0_0|S0_0_C4_C0_0}}
 # CHECK: msr     {{s3_0_c15_c0_0|S3_0_C15_C0_0}}, x12
 # CHECK: msr     {{s3_7_c11_c13_7|S3_7_C11_C13_7}}, x5
+# CHECK: msr     {{s0_0_c4_c0_0|S0_0_C4_C0_0}}, xzr
 0xac 0xf1 0x3f 0xd5
 0xed 0xbf 0x3a 0xd5
+0x1f 0x40 0x20 0xd5
 0x0c 0xf0 0x18 0xd5
 0xe5 0xbd 0x1f 0xd5
+0x1f 0x40 0x00 0xd5
 
 #------------------------------------------------------------------------------
 # Test and branch (immediate)
diff --git a/test/MC/Disassembler/Hexagon/alu32_alu.txt b/test/MC/Disassembler/Hexagon/alu32_alu.txt
index 4dde7df0759..26b320ecde0 100644
--- a/test/MC/Disassembler/Hexagon/alu32_alu.txt
+++ b/test/MC/Disassembler/Hexagon/alu32_alu.txt
@@ -49,7 +49,7 @@
 0xf1 0xff 0x5f 0x78
 # CHECK: r17 = #32767
 0xf1 0xff 0xdf 0x78
-# CHECK: r17 = ##65535
+# CHECK: r17 = #-1
 
 # Transfer register
 0x11 0xc0 0x75 0x70
diff --git a/test/MC/Disassembler/Mips/micromips.txt b/test/MC/Disassembler/Mips/micromips.txt
index 2b75d46b3bc..5809ac28113 100644
--- a/test/MC/Disassembler/Mips/micromips.txt
+++ b/test/MC/Disassembler/Mips/micromips.txt
@@ -1,506 +1,342 @@
 # RUN: llvm-mc --disassemble %s -triple=mips-unknown-linux -mattr=micromips \
 # RUN: | FileCheck %s
 
-# CHECK: add $9, $6, $7
-0x00 0xe6 0x49 0x10
+0x00 0xe6 0x49 0x10 # CHECK: add $9, $6, $7
 
-# CHECK: addi $9, $6, 17767
-0x11 0x26 0x45 0x67
+0x11 0x26 0x45 0x67 # CHECK: addi $9, $6, 17767
 
-# CHECK: addiu $9, $6, -15001
-0x31 0x26 0xc5 0x67
+0x31 0x26 0xc5 0x67 # CHECK: addiu $9, $6, -15001
 
-# CHECK: addi $9, $6, 17767
-0x11 0x26 0x45 0x67
+0x11 0x26 0x45 0x67 # CHECK: addi $9, $6, 17767
 
-# CHECK: addiu $9, $6, -15001
-0x31 0x26 0xc5 0x67
+0x31 0x26 0xc5 0x67 # CHECK: addiu $9, $6, -15001
 
-# CHECK: addiusp -16
-0x4f 0xf9
+0x4f 0xf9 # CHECK: addiusp -16
 
-# CHECK: addiusp -1028
-0x4f 0xff
+0x4f 0xff # CHECK: addiusp -1028
 
-# CHECK: addiusp -1032
-0x4f 0xfd
+0x4f 0xfd # CHECK: addiusp -1032
 
-# CHECK: addiusp 1024
-0x4c 0x01
+0x4c 0x01 # CHECK: addiusp 1024
 
-# CHECK: addiusp 1028
-0x4c 0x03
+0x4c 0x03 # CHECK: addiusp 1028
 
-# CHECK: addu $9, $6, $7
-0x00 0xe6 0x49 0x50
+0x00 0xe6 0x49 0x50 # CHECK: addu $9, $6, $7
 
-# CHECK: sub $9, $6, $7
-0x00 0xe6 0x49 0x90
+0x00 0xe6 0x49 0x90 # CHECK: sub $9, $6, $7
 
-# CHECK: subu $4, $3, $5
-0x00 0xa3 0x21 0xd0
+0x00 0xa3 0x21 0xd0 # CHECK: subu $4, $3, $5
 
-# CHECK: sub $6, $zero, $7
-0x00 0xe0 0x31 0x90
+0x00 0xe0 0x31 0x90 # CHECK: sub $6, $zero, $7
 
-# CHECK: subu $6, $zero, $7
-0x00 0xe0 0x31 0xd0
+0x00 0xe0 0x31 0xd0 # CHECK: subu $6, $zero, $7
 
-# CHECK: addu $7, $8, $zero
-0x00 0x08 0x39 0x50
+0x00 0x08 0x39 0x50 # CHECK: addu $7, $8, $zero
 
-# CHECK: slt $3, $3, $5
-0x00 0xa3 0x1b 0x50
+0x00 0xa3 0x1b 0x50 # CHECK: slt $3, $3, $5
 
-# CHECK: slti $3, $3, 103
-0x90 0x63 0x00 0x67
+0x90 0x63 0x00 0x67 # CHECK: slti $3, $3, 103
 
-# CHECK: slti $3, $3, 103
-0x90 0x63 0x00 0x67
+0x90 0x63 0x00 0x67 # CHECK: slti $3, $3, 103
 
-# CHECK: sltiu $3, $3, 103
-0xb0 0x63 0x00 0x67
+0xb0 0x63 0x00 0x67 # CHECK: sltiu $3, $3, 103
 
-# CHECK: sltu $3, $3, $5
-0x00 0xa3 0x1b 0x90
+0x00 0xa3 0x1b 0x90 # CHECK: sltu $3, $3, $5
 
-# CHECK: lui $9, 17767
-0x41 0xa9 0x45 0x67
+0x41 0xa9 0x45 0x67 # CHECK: lui $9, 17767
 
-# CHECK: and $9, $6, $7
-0x00 0xe6 0x4a 0x50
+0x00 0xe6 0x4a 0x50 # CHECK: and $9, $6, $7
 
-# CHECK: andi $9, $6, 17767
-0xd1 0x26 0x45 0x67
+0xd1 0x26 0x45 0x67 # CHECK: andi $9, $6, 17767
 
-# CHECK: andi $9, $6, 17767
-0xd1 0x26 0x45 0x67
+0xd1 0x26 0x45 0x67 # CHECK: andi $9, $6, 17767
 
-# CHECK: andi16 $16, $2, 31
-0x2c 0x29
+0x2c 0x29 # CHECK: andi16 $16, $2, 31
 
-# CHECK: or $3, $4, $5
-0x00 0xa4 0x1a 0x90
+0x00 0xa4 0x1a 0x90 # CHECK: or $3, $4, $5
 
-# CHECK: ori $9, $6, 17767
-0x51 0x26 0x45 0x67
+0x51 0x26 0x45 0x67 # CHECK: ori $9, $6, 17767
 
-# CHECK: xor $3, $3, $5
-0x00 0xa3 0x1b 0x10
+0x00 0xa3 0x1b 0x10 # CHECK: xor $3, $3, $5
 
-# CHECK: xori $9, $6, 17767
-0x71 0x26 0x45 0x67
+0x71 0x26 0x45 0x67 # CHECK: xori $9, $6, 17767
 
-# CHECK: xori $9, $6, 17767
-0x71 0x26 0x45 0x67
+0x71 0x26 0x45 0x67 # CHECK: xori $9, $6, 17767
 
-# CHECK: nor $9, $6, $7
-0x00 0xe6 0x4a 0xd0
+0x00 0xe6 0x4a 0xd0 # CHECK: nor $9, $6, $7
 
-# CHECK: not $7, $8
-0x00 0x08 0x3a 0xd0
+0x00 0x08 0x3a 0xd0 # CHECK: not $7, $8
 
-# CHECK: mul $9, $6, $7
-0x00 0xe6 0x4a 0x10
+0x00 0xe6 0x4a 0x10 # CHECK: mul $9, $6, $7
 
-# CHECK: mult $9, $7
-0x00 0xe9 0x8b 0x3c
+0x00 0xe9 0x8b 0x3c # CHECK: mult $9, $7
 
-# CHECK: multu $9, $7
-0x00 0xe9 0x9b 0x3c
+0x00 0xe9 0x9b 0x3c # CHECK: multu $9, $7
 
-# CHECK-EB: div $zero, $9, $7
-0x00 0xe9 0xab 0x3c
+0x00 0xe9 0xab 0x3c # CHECK-EB: div $zero, $9, $7
 
-# CHECK-EB: divu $zero, $9, $7
-0x00 0xe9 0xbb 0x3c
+0x00 0xe9 0xbb 0x3c # CHECK-EB: divu $zero, $9, $7
 
-# CHECK: sll $4, $3, 7
-0x00 0x83 0x38 0x00
+0x00 0x83 0x38 0x00 # CHECK: sll $4, $3, 7
 
-# CHECK: sllv $2, $3, $5
-0x00 0x65 0x10 0x10
+0x00 0x65 0x10 0x10 # CHECK: sllv $2, $3, $5
 
-# CHECK: sra $4, $3, 7
-0x00 0x83 0x38 0x80
+0x00 0x83 0x38 0x80 # CHECK: sra $4, $3, 7
 
-# CHECK: srav $2, $3, $5
-0x00 0x65 0x10 0x90
+0x00 0x65 0x10 0x90 # CHECK: srav $2, $3, $5
 
-# CHECK: srl $4, $3, 7
-0x00 0x83 0x38 0x40
+0x00 0x83 0x38 0x40 # CHECK: srl $4, $3, 7
 
-# CHECK: srlv $2, $3, $5
-0x00 0x65 0x10 0x50
+0x00 0x65 0x10 0x50 # CHECK: srlv $2, $3, $5
 
-# CHECK: rotr $9, $6, 7
-0x01 0x26 0x38 0xc0
+0x01 0x26 0x38 0xc0 # CHECK: rotr $9, $6, 7
 
-# CHECK: rotrv $9, $6, $7
-0x00 0xc7 0x48 0xd0
+0x00 0xc7 0x48 0xd0 # CHECK: rotrv $9, $6, $7
 
-# CHECK: lb $5, 8($4)
-0x1c 0xa4 0x00 0x08
+0x1c 0xa4 0x00 0x08 # CHECK: lb $5, 8($4)
 
-# CHECK: lbu $6, 8($4)
-0x14 0xc4 0x00 0x08
+0x14 0xc4 0x00 0x08 # CHECK: lbu $6, 8($4)
 
-# CHECK: lh $2, 8($4)
-0x3c 0x44 0x00 0x08
+0x3c 0x44 0x00 0x08 # CHECK: lh $2, 8($4)
 
-# CHECK: lhu $4, 8($2)
-0x34 0x82 0x00 0x08
+0x34 0x82 0x00 0x08 # CHECK: lhu $4, 8($2)
 
-# CHECK: lw  $6, 4($5)
-0xfc 0xc5 0x00 0x04
+0xfc 0xc5 0x00 0x04 # CHECK: lw  $6, 4($5)
 
-# CHECK: lw $6, 123($sp)
-0xfc 0xdd 0x00 0x7b
+0xfc 0xdd 0x00 0x7b # CHECK: lw $6, 123($sp)
 
-# CHECK: sb $5, 8($4)
-0x18 0xa4 0x00 0x08
+0x18 0xa4 0x00 0x08 # CHECK: sb $5, 8($4)
 
-# CHECK: sh  $2, 8($4)
-0x38 0x44 0x00 0x08
+0x38 0x44 0x00 0x08 # CHECK: sh  $2, 8($4)
 
-# CHECK: sw  $5, 4($6)
-0xf8 0xa6 0x00 0x04
+0xf8 0xa6 0x00 0x04 # CHECK: sw  $5, 4($6)
 
-# CHECK: sw $5, 123($sp)
-0xf8 0xbd 0x00 0x7b
+0xf8 0xbd 0x00 0x7b # CHECK: sw $5, 123($sp)
 
-# CHECK: lwu $2, 8($4)
-0x60 0x44 0xe0 0x08
+0x60 0x44 0xe0 0x08 # CHECK: lwu $2, 8($4)
 
-# CHECK: lwl $4, 16($5)
-0x60 0x85 0x00 0x10
+0x60 0x85 0x00 0x10 # CHECK: lwl $4, 16($5)
 
-# CHECK: lwr $4, 16($5)
-0x60 0x85 0x10 0x10
+0x60 0x85 0x10 0x10 # CHECK: lwr $4, 16($5)
 
-# CHECK: swl $4, 16($5)
-0x60 0x85 0x80 0x10
+0x60 0x85 0x80 0x10 # CHECK: swl $4, 16($5)
 
-# CHECK: swr $4, 16($5)
-0x60 0x85 0x90 0x10
+0x60 0x85 0x90 0x10 # CHECK: swr $4, 16($5)
 
-# CHECK: movz $9, $6, $7
-0x00 0xe6 0x48 0x58
+0x00 0xe6 0x48 0x58 # CHECK: movz $9, $6, $7
 
-# CHECK: movn $9, $6, $7
-0x00 0xe6 0x48 0x18
+0x00 0xe6 0x48 0x18 # CHECK: movn $9, $6, $7
 
-# CHECK: movt $9, $6, $fcc0
-0x55 0x26 0x09 0x7b
+0x55 0x26 0x09 0x7b # CHECK: movt $9, $6, $fcc0
 
-# CHECK: movf $9, $6, $fcc0
-0x55 0x26 0x01 0x7b
+0x55 0x26 0x01 0x7b # CHECK: movf $9, $6, $fcc0
 
-# CHECK: mthi   $6
-0x00 0x06 0x2d 0x7c
+0x00 0x06 0x2d 0x7c # CHECK: mthi   $6
 
-# CHECK: mfhi   $6
-0x00 0x06 0x0d 0x7c
+0x00 0x06 0x0d 0x7c # CHECK: mfhi   $6
 
-# CHECK: mtlo   $6
-0x00 0x06 0x3d 0x7c
+0x00 0x06 0x3d 0x7c # CHECK: mtlo   $6
 
-# CHECK: mflo   $6
-0x00 0x06 0x1d 0x7c
+0x00 0x06 0x1d 0x7c # CHECK: mflo   $6
 
-# CHECK: madd   $4, $5
-0x00 0xa4 0xcb 0x3c
+0x00 0xa4 0xcb 0x3c # CHECK: madd   $4, $5
 
-# CHECK: maddu  $4, $5
-0x00 0xa4 0xdb 0x3c
+0x00 0xa4 0xdb 0x3c # CHECK: maddu  $4, $5
 
-# CHECK: msub   $4, $5
-0x00 0xa4 0xeb 0x3c
+0x00 0xa4 0xeb 0x3c # CHECK: msub   $4, $5
 
-# CHECK: msubu  $4, $5
-0x00 0xa4 0xfb 0x3c
+0x00 0xa4 0xfb 0x3c # CHECK: msubu  $4, $5
 
-# CHECK: clz $9, $6
-0x01 0x26 0x5b 0x3c
+0x01 0x26 0x5b 0x3c # CHECK: clz $9, $6
 
-# CHECK: clo $9, $6
-0x01 0x26 0x4b 0x3c
+0x01 0x26 0x4b 0x3c # CHECK: clo $9, $6
 
-# CHECK: seb $9, $6
-0x01 0x26 0x2b 0x3c
+0x01 0x26 0x2b 0x3c # CHECK: seb $9, $6
 
-# CHECK: seh $9, $6
-0x01 0x26 0x3b 0x3c
+0x01 0x26 0x3b 0x3c # CHECK: seh $9, $6
 
-# CHECK: wsbh $9, $6
-0x01 0x26 0x7b 0x3c
+0x01 0x26 0x7b 0x3c # CHECK: wsbh $9, $6
 
-# CHECK: ext $9, $6, 3, 7
-0x01 0x26 0x30 0xec
+0x01 0x26 0x30 0xec # CHECK: ext $9, $6, 3, 7
 
-# CHECK: ins $9, $6, 3, 7
-0x01 0x26 0x48 0xcc
+0x01 0x26 0x48 0xcc # CHECK: ins $9, $6, 3, 7
 
-# CHECK: j 1328
-0xd4 0x00 0x02 0x98
+0xd4 0x00 0x02 0x98 # CHECK: j 1328
 
-# CHECK: jal 1328
-0xf4 0x00 0x02 0x98
+0xf4 0x00 0x02 0x98 # CHECK: jal 1328
 
-# CHECK: jalr $ra, $6
-0x03 0xe6 0x0f 0x3c
+0x03 0xe6 0x0f 0x3c # CHECK: jalr $ra, $6
 
-# CHECK: jr $7
-0x00 0x07 0x0f 0x3c
+0x00 0x07 0x0f 0x3c # CHECK: jr $7
 
-# CHECK: jraddiusp 20
-0x47 0x05
+0x47 0x05 # CHECK: jraddiusp 20
 
-# CHECK: beq $9, $6, 1332
-0x94 0xc9 0x02 0x9a
+0x94 0xc9 0x02 0x9a # CHECK: beq $9, $6, 1332
 
-# CHECK: bgez $6, 1332
-0x40 0x46 0x02 0x9a
+0x40 0x46 0x02 0x9a # CHECK: bgez $6, 1332
 
-# CHECK: bgezal $6, 1332
-0x40 0x66 0x02 0x9a
+0x40 0x66 0x02 0x9a # CHECK: bgezal $6, 1332
 
-# CHECK: bltzal $6, 1332
-0x40 0x26 0x02 0x9a
+0x40 0x26 0x02 0x9a # CHECK: bltzal $6, 1332
 
-# CHECK: bgtz $6, 1332
-0x40 0xc6 0x02 0x9a
+0x40 0xc6 0x02 0x9a # CHECK: bgtz $6, 1332
 
-# CHECK: blez $6, 1332
-0x40 0x86 0x02 0x9a
+0x40 0x86 0x02 0x9a # CHECK: blez $6, 1332
 
-# CHECK: bne $9, $6, 1332
-0xb4 0xc9 0x02 0x9a
+0xb4 0xc9 0x02 0x9a # CHECK: bne $9, $6, 1332
 
-# CHECK: bltz $6, 1332
-0x40 0x06 0x02 0x9a
+0x40 0x06 0x02 0x9a # CHECK: bltz $6, 1332
 
-# CHECK: teq $8, $9, 0
-0x01 0x28 0x00 0x3c
+0x01 0x28 0x00 0x3c # CHECK: teq $8, $9, 0
 
-# CHECK: tge $8, $9, 0
-0x01 0x28 0x02 0x3c
+0x01 0x28 0x02 0x3c # CHECK: tge $8, $9, 0
 
-# CHECK: tgeu $8, $9, 0
-0x01 0x28 0x04 0x3c
+0x01 0x28 0x04 0x3c # CHECK: tgeu $8, $9, 0
 
-# CHECK: tlt $8, $9, 0
-0x01 0x28 0x08 0x3c
+0x01 0x28 0x08 0x3c # CHECK: tlt $8, $9, 0
 
-# CHECK: tltu $8, $9, 0
-0x01 0x28 0x0a 0x3c
+0x01 0x28 0x0a 0x3c # CHECK: tltu $8, $9, 0
 
-# CHECK: tne $8, $9, 0
-0x01 0x28 0x0c 0x3c
+0x01 0x28 0x0c 0x3c # CHECK: tne $8, $9, 0
 
-# CHECK: teqi $9, 17767
-0x41,0xc9,0x45,0x67
+0x41,0xc9,0x45,0x67 # CHECK: teqi $9, 17767
 
-# CHECK: tgei $9, 17767
-0x41 0x29 0x45 0x67
+0x41 0x29 0x45 0x67 # CHECK: tgei $9, 17767
 
-# CHECK: tgeiu $9, 17767
-0x41 0x69 0x45 0x67
+0x41 0x69 0x45 0x67 # CHECK: tgeiu $9, 17767
 
-# CHECK: tlti $9, 17767
-0x41 0x09 0x45 0x67
+0x41 0x09 0x45 0x67 # CHECK: tlti $9, 17767
 
-# CHECK: tltiu $9, 17767
-0x41 0x49 0x45 0x67
+0x41 0x49 0x45 0x67 # CHECK: tltiu $9, 17767
 
-# CHECK: tnei $9, 17767
-0x41 0x89 0x45 0x67
+0x41 0x89 0x45 0x67 # CHECK: tnei $9, 17767
 
-# CHECK: cache 1, 8($5)
-0x20 0x25 0x60 0x08
+0x20 0x25 0x60 0x08 # CHECK: cache 1, 8($5)
 
-# CHECK: pref 1, 8($5)
-0x60 0x25 0x20 0x08
+0x60 0x25 0x20 0x08 # CHECK: pref 1, 8($5)
 
-# CHECK: ssnop
-0x00 0x00 0x08 0x00
+0x00 0x00 0x08 0x00 # CHECK: ssnop
 
-# CHECK: ehb
-0x00 0x00 0x18 0x00
+0x00 0x00 0x18 0x00 # CHECK: ehb
 
-# CHECK: pause
-0x00 0x00 0x28 0x00
+0x00 0x00 0x28 0x00 # CHECK: pause
 
-# CHECK: ll $2, 8($4)
-0x60 0x44 0x30 0x08
+0x60 0x44 0x30 0x08 # CHECK: ll $2, 8($4)
 
-# CHECK: sc $2, 8($4)
-0x60 0x44 0xb0 0x08
+0x60 0x44 0xb0 0x08 # CHECK: sc $2, 8($4)
 
-# CHECK: lwxs $2, $3($4)
-0x00 0x64 0x11 0x18
+0x00 0x64 0x11 0x18 # CHECK: lwxs $2, $3($4)
 
-# CHECK: bgezals $6, 1332
-0x42 0x66 0x02 0x9a
+0x42 0x66 0x02 0x9a # CHECK: bgezals $6, 1332
 
-# CHECK: bltzals $6, 1332
-0x42 0x26 0x02 0x9a
+0x42 0x26 0x02 0x9a # CHECK: bltzals $6, 1332
 
-# CHECK: beqzc $9, 1332
-0x40 0xe9 0x02 0x9a
+0x40 0xe9 0x02 0x9a # CHECK: beqzc $9, 1332
 
-# CHECK: bnezc $9, 1332
-0x40 0xa9 0x02 0x9a
+0x40 0xa9 0x02 0x9a # CHECK: bnezc $9, 1332
 
-# CHECK: jals 1328
-0x74 0x00 0x02 0x98
+0x74 0x00 0x02 0x98 # CHECK: jals 1328
 
-# CHECK: jalrs $ra, $6
-0x03 0xe6 0x4f 0x3c
+0x03 0xe6 0x4f 0x3c # CHECK: jalrs $ra, $6
 
-# CHECK: lwm32 $16, $17, 8($4)
-0x20 0x44 0x50 0x08
+0x20 0x44 0x50 0x08 # CHECK: lwm32 $16, $17, 8($4)
 
-# CHECK: swm32 $16, $17, 8($4)
-0x20 0x44 0xd0 0x08
+0x20 0x44 0xd0 0x08 # CHECK: swm32 $16, $17, 8($4)
 
-# CHECK: swp $16, 8($4)
-0x22 0x04 0x90 0x08
+0x22 0x04 0x90 0x08 # CHECK: swp $16, 8($4)
 
-# CHECK: lwp $16, 8($4)
-0x22 0x04 0x10 0x08
+0x22 0x04 0x10 0x08 # CHECK: lwp $16, 8($4)
 
-# CHECK: nop
-0x00 0x00 0x00 0x00
+0x00 0x00 0x00 0x00 # CHECK: nop
 
-# CHECK: addiupc $2, 20
-0x79 0x00 0x00 0x05
+0x79 0x00 0x00 0x05 # CHECK: addiupc $2, 20
 
-# CHECK: addiupc $7, 16777212
-0x7b 0xbf 0xff 0xff
+0x7b 0xbf 0xff 0xff # CHECK: addiupc $7, 16777212
 
-# CHECK: addiupc $7, -16777216
-0x7b 0xc0 0x00 0x00
+0x7b 0xc0 0x00 0x00 # CHECK: addiupc $7, -16777216
 
-# CHECK: addu16 $6, $17, $4
-0x07 0x42
+0x07 0x42 # CHECK: addu16 $6, $17, $4
 
-# CHECK: subu16 $5, $16, $3
-0x06 0xb1
+0x06 0xb1 # CHECK: subu16 $5, $16, $3
 
-# CHECK: and16 $16, $2
-0x44 0x82
+0x44 0x82 # CHECK: and16 $16, $2
 
-# CHECK: not16 $17, $3
-0x44 0x0b
+0x44 0x0b # CHECK: not16 $17, $3
 
-# CHECK: or16 $16, $4
-0x44 0xc4
+0x44 0xc4 # CHECK: or16 $16, $4
 
-# CHECK: xor16 $17, $5
-0x44 0x4d
+0x44 0x4d # CHECK: xor16 $17, $5
 
-# CHECK: sll16 $3, $16, 5
-0x25 0x8a
+0x25 0x8a # CHECK: sll16 $3, $16, 5
 
-# CHECK: srl16 $4, $17, 6
-0x26 0x1d
+0x26 0x1d # CHECK: srl16 $4, $17, 6
 
-# CHECK: lbu16 $3, 4($17)
-0x09 0x94
+0x09 0x94 # CHECK: lbu16 $3, 4($17)
 
-# CHECK: lbu16 $3, -1($16)
-0x09 0x8f
+0x09 0x8f # CHECK: lbu16 $3, -1($16)
 
-# CHECK: lhu16 $3, 4($16)
-0x29 0x82
+0x29 0x82 # CHECK: lhu16 $3, 4($16)
 
-# CHECK: lw16 $4, 8($17)
-0x6a 0x12
+0x6a 0x12 # CHECK: lw16 $4, 8($17)
 
-# CHECK: sb16 $3, 4($16)
-0x89 0x84
+0x89 0x84 # CHECK: sb16 $3, 4($16)
 
-# CHECK: sh16 $4, 8($17)
-0xaa 0x14
+0xaa 0x14 # CHECK: sh16 $4, 8($17)
 
-# CHECK: sw16 $4, 4($17)
-0xea 0x11
+0xea 0x11 # CHECK: sw16 $4, 4($17)
 
-# CHECK: sw16 $zero, 4($17)
-0xe8 0x11
+0xe8 0x11 # CHECK: sw16 $zero, 4($17)
 
-# CHECK: mfhi $9
-0x46 0x09
+0x46 0x09 # CHECK: mfhi $9
 
-# CHECK: mflo $9
-0x46 0x49
+0x46 0x49 # CHECK: mflo $9
 
-# CHECK: move $25, $1
-0x0f 0x21
+0x0f 0x21 # CHECK: move $25, $1
 
-# CHECK: jrc $9
-0x45 0xa9
+0x45 0xa9 # CHECK: jrc $9
 
-# CHECK: jalr $9
-0x45 0xc9
+0x45 0xc9 # CHECK: jalr $9
 
-# CHECK: jalrs16 $9
-0x45 0xe9
+0x45 0xe9 # CHECK: jalrs16 $9
 
-# CHECK: jr16 $9
-0x45 0x89
+0x45 0x89 # CHECK: jr16 $9
 
-# CHECK: li16 $3, -1
-0xed 0xff
+0xed 0xff # CHECK: li16 $3, -1
 
-# CHECK: li16 $3, 126
-0xed 0xfe
+0xed 0xfe # CHECK: li16 $3, 126
 
-# CHECK: addiur1sp $7, 4
-0x6f 0x83
+0x6f 0x83 # CHECK: addiur1sp $7, 4
 
-# CHECK: addiur2 $6, $7, -1
-0x6f 0x7e
+0x6f 0x7e # CHECK: addiur2 $6, $7, -1
 
-# CHECK: addiur2 $6, $7, 12
-0x6f 0x76
+0x6f 0x76 # CHECK: addiur2 $6, $7, 12
 
-# CHECK: addius5 $7, -2
-0x4c 0xfc
+0x4c 0xfc # CHECK: addius5 $7, -2
 
-# CHECK: nop
-0x0c 0x00
+0x0c 0x00 # CHECK: nop
 
-# CHECK: lw $3, 32($sp)
-0x48 0x68
+0x48 0x68 # CHECK: lw $3, 32($sp)
 
-# CHECK: sw $4, 124($sp)
-0xc8 0x9f
+0xc8 0x9f # CHECK: sw $4, 124($sp)
 
-# CHECK: beqz16 $6, 20
-0x8f 0x0a
+0x8f 0x0a # CHECK: beqz16 $6, 20
 
-# CHECK: bnez16 $6, 20
-0xaf 0x0a
+0xaf 0x0a # CHECK: bnez16 $6, 20
 
-# CHECK: b16 132
-0xcc 0x42
+0xcc 0x42 # CHECK: b16 132
 
-# CHECK: lw $3, 32($gp)
-0x65 0x88
+0x65 0x88 # CHECK: lw $3, 32($gp)
 
-# CHECK: lwm16 $16, $17, $ra, 8($sp)
-0x45 0x12
+0x45 0x12 # CHECK: lwm16 $16, $17, $ra, 8($sp)
 
-# CHECK: swm16 $16, $17, $ra, 8($sp)
-0x45 0x52
+0x45 0x52 # CHECK: swm16 $16, $17, $ra, 8($sp)
 
-# CHECK: break16 8
-0x46 0x88
+0x46 0x88 # CHECK: break16 8
 
-# CHECK: sdbbp16 14
-0x46 0xce
+0x46 0xce # CHECK: sdbbp16 14
 
-# CHECK: movep $5, $6, $2, $3
-0x84 0x34
+0x84 0x34 # CHECK: movep $5, $6, $2, $3
+
+0x00 0x00 0x57 0x7c # CHECK: ei
+
+0x00 0x0a 0x57 0x7c # CHECK: ei $10
diff --git a/test/MC/Disassembler/Mips/micromips32r6.txt b/test/MC/Disassembler/Mips/micromips32r6.txt
new file mode 100644
index 00000000000..a2691ee6bc3
--- /dev/null
+++ b/test/MC/Disassembler/Mips/micromips32r6.txt
@@ -0,0 +1,114 @@
+# RUN: llvm-mc --disassemble %s -triple=mips-unknown-linux -mcpu=mips32r6 -mattr=micromips | FileCheck %s
+
+0x00 0xa4 0x19 0x10 # CHECK: add $3, $4, $5
+
+0x30 0x64 0x04 0xd2 # CHECK: addiu $3, $4, 1234
+
+0x00 0xa4 0x19 0x50 # CHECK: addu $3, $4, $5
+
+0x78 0x80 0x00 0x19 # CHECK: addiupc $4, 100
+
+0x78 0x7f 0x00 0x38 # CHECK: aluipc $3, 56
+
+0x78 0x7e 0xff 0xff # CHECK: auipc $3, -1
+
+0x00 0x43 0x24 0x1f # CHECK: align $4, $2, $3, 2
+
+0x00 0xa4 0x1a 0x50 # CHECK: and $3, $4, $5
+
+0xd0 0x64 0x04 0xd2 # CHECK: andi $3, $4, 1234
+
+0x10 0x62 0xff 0xe9 # CHECK: aui $3, $2, -23
+
+0x74 0x40 0x02 0x9a # CHECK: beqzalc $2, 1332
+
+0x7c 0x40 0x02 0x9a # CHECK: bnezalc $2, 1332
+
+0xc0 0x42 0x02 0x9a # CHECK: bgezalc $2, 1332
+
+0xe0 0x40 0x02 0x9a # CHECK: bgtzalc $2, 1332
+
+0xe0 0x42 0x02 0x9a # CHECK: bltzalc $2, 1332
+
+0xc0 0x40 0x02 0x9a # CHECK: blezalc $2, 1332
+
+0xb4 0x37 0x96 0xb8 # CHECK: balc 14572256
+
+0x94 0x37 0x96 0xb8 # CHECK: bc 14572256
+
+0x00 0x44 0x0b 0x3c # CHECK: bitswap $4, $2
+
+0x00 0x00 0x00 0x07 # CHECK: break
+
+0x00 0x07 0x00 0x07 # CHECK: break 7
+
+0x00 0x07 0x01 0x47 # CHECK: break 7, 5
+
+0x20 0x25 0x60 0x08 # CHECK: cache 1, 8($5)
+
+0x01 0x65 0x4b 0x3c # CHECK: clo $11, $5
+
+0x03 0x80 0xe8 0x50 # CHECK: clz $sp, $gp
+
+0x00 0xa4 0x19 0x18 # CHECK: div $3, $4, $5
+
+0x00 0xa4 0x19 0x98 # CHECK: divu $3, $4, $5
+
+0x00 0x00 0x18 0x00 # CHECK: ehb
+
+0x00 0x00 0x57 0x7c # CHECK: ei
+
+0x00 0x0a 0x57 0x7c # CHECK: ei $10
+
+0x00 0x00 0xf3 0x7c # CHECK: eret
+
+0x00 0x01 0xf3 0x7c # CHECK: eretnc
+
+0x80 0x05 0x01 0x00 # CHECK: jialc $5, 256
+
+0xa0 0x05 0x01 0x00 # CHECK: jic $5, 256
+
+0x78 0x48 0x00 0x43 # CHECK: lwpc $2, 268
+
+0x00 0x43 0x26 0x0f # CHECK: lsa $2, $3, $4, 3
+
+0x00 0xa4 0x19 0x58 # CHECK: mod $3, $4, $5
+
+0x00 0xa4 0x19 0xd8 # CHECK: modu $3, $4, $5
+
+0x00 0xa4 0x18 0x18 # CHECK: mul $3, $4, $5
+
+0x00 0xa4 0x18 0x58 # CHECK: muh $3, $4, $5
+
+0x00 0xa4 0x18 0x98 # CHECK: mulu $3, $4, $5
+
+0x00 0xa4,0x18,0xd8 # CHECK: muhu $3, $4, $5
+
+0x00 0x00 0x00 0x00 # CHECK: nop
+
+0x00 0xa4 0x1a 0xd0 # CHECK: nor $3, $4, $5
+
+0x00,0xa4,0x1a,0x90 # CHECK: or $3, $4, $5
+
+0x50 0x64 0x04 0xd2 # CHECK: ori $3, $4, 1234
+
+0x60 0x25 0x20 0x08 # CHECK: pref 1, 8($5)
+
+0x00 0x83 0x11 0x40 # CHECK: seleqz $2, $3, $4
+
+0x00 0x83 0x11 0x80 # CHECK: selnez $2, $3, $4
+
+0x00 0x83 0x38 0x00 # CHECK: sll $4, $3, 7
+
+0x00 0xa4 0x19 0x90 # CHECK: sub $3, $4, $5
+
+0x00 0xa4 0x19 0xd0 # CHECK: subu $3, $4, $5
+
+0x00 0xa4 0x1b 0x10 # CHECK: xor $3, $4, $5
+
+0x70 0x64 0x04 0xd2 # CHECK: xori $3, $4, 1234
+
+0x00 0x64 0x2b 0x3c # CHECK: seb $3, $4
+
+0x00 0x64 0x3b 0x3c # CHECK: seh $3, $4
+
diff --git a/test/MC/Disassembler/Mips/micromips_le.txt b/test/MC/Disassembler/Mips/micromips_le.txt
index 3f3b3255972..3058bd06106 100644
--- a/test/MC/Disassembler/Mips/micromips_le.txt
+++ b/test/MC/Disassembler/Mips/micromips_le.txt
@@ -1,506 +1,342 @@
 # RUN: llvm-mc --disassemble %s -triple=mipsel-unknown-linux -mattr=micromips \
 # RUN: | FileCheck %s
 
-# CHECK: add $9, $6, $7
-0xe6 0x00 0x10 0x49
+0xe6 0x00 0x10 0x49 # CHECK: add $9, $6, $7
 
-# CHECK: addi $9, $6, 17767
-0x26 0x11 0x67 0x45
+0x26 0x11 0x67 0x45 # CHECK: addi $9, $6, 17767
 
-# CHECK: addiu $9, $6, -15001
-0x26 0x31 0x67 0xc5
+0x26 0x31 0x67 0xc5 # CHECK: addiu $9, $6, -15001
 
-# CHECK: addi $9, $6, 17767
-0x26 0x11 0x67 0x45
+0x26 0x11 0x67 0x45 # CHECK: addi $9, $6, 17767
 
-# CHECK: addiu $9, $6, -15001
-0x26 0x31 0x67 0xc5
+0x26 0x31 0x67 0xc5 # CHECK: addiu $9, $6, -15001
 
-# CHECK: addiusp -16
-0xf9 0x4f
+0xf9 0x4f # CHECK: addiusp -16
 
-# CHECK: addiusp -1028
-0xff 0x4f
+0xff 0x4f # CHECK: addiusp -1028
 
-# CHECK: addiusp -1032
-0xfd 0x4f
+0xfd 0x4f # CHECK: addiusp -1032
 
-# CHECK: addiusp 1024
-0x01 0x4c
+0x01 0x4c # CHECK: addiusp 1024
 
-# CHECK: addiusp 1028
-0x03 0x4c
+0x03 0x4c # CHECK: addiusp 1028
 
-# CHECK: addu $9, $6, $7
-0xe6 0x00 0x50 0x49
+0xe6 0x00 0x50 0x49 # CHECK: addu $9, $6, $7
 
-# CHECK: andi16 $16, $2, 31
-0x29 0x2c
+0x29 0x2c # CHECK: andi16 $16, $2, 31
 
-# CHECK: sub $9, $6, $7
-0xe6 0x00 0x90 0x49
+0xe6 0x00 0x90 0x49 # CHECK: sub $9, $6, $7
 
-# CHECK: subu  $4, $3, $5
-0xa3 0x00 0xd0 0x21
+0xa3 0x00 0xd0 0x21 # CHECK: subu  $4, $3, $5
 
-# CHECK: sub $6, $zero, $7
-0xe0 0x00 0x90 0x31
+0xe0 0x00 0x90 0x31 # CHECK: sub $6, $zero, $7
 
-# CHECK: subu $6, $zero, $7
-0xe0 0x00 0xd0 0x31
+0xe0 0x00 0xd0 0x31 # CHECK: subu $6, $zero, $7
 
-# CHECK: addu $7, $8, $zero
-0x08 0x00 0x50 0x39
+0x08 0x00 0x50 0x39 # CHECK: addu $7, $8, $zero
 
-# CHECK: slt $3, $3, $5
-0xa3 0x00 0x50 0x1b
+0xa3 0x00 0x50 0x1b # CHECK: slt $3, $3, $5
 
-# CHECK: slti $3, $3, 103
-0x63 0x90 0x67 0x00
+0x63 0x90 0x67 0x00 # CHECK: slti $3, $3, 103
 
-# CHECK: slti $3, $3, 103
-0x63 0x90 0x67 0x00
+0x63 0x90 0x67 0x00 # CHECK: slti $3, $3, 103
 
-# CHECK: sltiu $3, $3, 103
-0x63 0xb0 0x67 0x00
+0x63 0xb0 0x67 0x00 # CHECK: sltiu $3, $3, 103
 
-# CHECK: sltu $3, $3, $5
-0xa3 0x00 0x90 0x1b
+0xa3 0x00 0x90 0x1b # CHECK: sltu $3, $3, $5
 
-# CHECK: lui $9, 17767
-0xa9 0x41 0x67 0x45
+0xa9 0x41 0x67 0x45 # CHECK: lui $9, 17767
 
-# CHECK: and $9, $6, $7
-0xe6 0x00 0x50 0x4a
+0xe6 0x00 0x50 0x4a # CHECK: and $9, $6, $7
 
-# CHECK: andi $9, $6, 17767
-0x26 0xd1 0x67 0x45
+0x26 0xd1 0x67 0x45 # CHECK: andi $9, $6, 17767
 
-# CHECK: andi $9, $6, 17767
-0x26 0xd1 0x67 0x45
+0x26 0xd1 0x67 0x45 # CHECK: andi $9, $6, 17767
 
-# CHECK: or $3, $4, $5
-0xa4 0x00 0x90 0x1a
+0xa4 0x00 0x90 0x1a # CHECK: or $3, $4, $5
 
-# CHECK: ori $9, $6, 17767
-0x26 0x51 0x67 0x45
+0x26 0x51 0x67 0x45 # CHECK: ori $9, $6, 17767
 
-# CHECK: xor $3, $3, $5
-0xa3 0x00 0x10 0x1b
+0xa3 0x00 0x10 0x1b # CHECK: xor $3, $3, $5
 
-# CHECK: xori $9, $6, 17767
-0x26 0x71 0x67 0x45
+0x26 0x71 0x67 0x45 # CHECK: xori $9, $6, 17767
 
-# CHECK: xori $9, $6, 17767
-0x26 0x71 0x67 0x45
+0x26 0x71 0x67 0x45 # CHECK: xori $9, $6, 17767
 
-# CHECK: nor $9, $6, $7
-0xe6 0x00 0xd0 0x4a
+0xe6 0x00 0xd0 0x4a # CHECK: nor $9, $6, $7
 
-# CHECK: not $7, $8
-0x08 0x00 0xd0 0x3a
+0x08 0x00 0xd0 0x3a # CHECK: not $7, $8
 
-# CHECK: mul $9, $6, $7
-0xe6 0x00 0x10 0x4a
+0xe6 0x00 0x10 0x4a # CHECK: mul $9, $6, $7
 
-# CHECK: mult $9, $7
-0xe9 0x00 0x3c 0x8b
+0xe9 0x00 0x3c 0x8b # CHECK: mult $9, $7
 
-# CHECK: multu $9, $7
-0xe9 0x00 0x3c 0x9b
+0xe9 0x00 0x3c 0x9b # CHECK: multu $9, $7
 
-# CHECK: div $zero, $9, $7
-0xe9 0x00 0x3c 0xab
+0xe9 0x00 0x3c 0xab # CHECK: div $zero, $9, $7
 
-# CHECK: divu $zero, $9, $7
-0xe9 0x00 0x3c 0xbb
+0xe9 0x00 0x3c 0xbb # CHECK: divu $zero, $9, $7
 
-# CHECK: sll $4, $3, 7
-0x83 0x00 0x00 0x38
+0x83 0x00 0x00 0x38 # CHECK: sll $4, $3, 7
 
-# CHECK: sllv $2, $3, $5
-0x65 0x00 0x10 0x10
+0x65 0x00 0x10 0x10 # CHECK: sllv $2, $3, $5
 
-# CHECK: sra $4, $3, 7
-0x83 0x00 0x80 0x38
+0x83 0x00 0x80 0x38 # CHECK: sra $4, $3, 7
 
-# CHECK: srav $2, $3, $5
-0x65 0x00 0x90 0x10
+0x65 0x00 0x90 0x10 # CHECK: srav $2, $3, $5
 
-# CHECK: srl $4, $3, 7
-0x83 0x00 0x40 0x38
+0x83 0x00 0x40 0x38 # CHECK: srl $4, $3, 7
 
-# CHECK: srlv $2, $3, $5
-0x65 0x00 0x50 0x10
+0x65 0x00 0x50 0x10 # CHECK: srlv $2, $3, $5
 
-# CHECK: rotr $9, $6, 7
-0x26 0x01 0xc0 0x38
+0x26 0x01 0xc0 0x38 # CHECK: rotr $9, $6, 7
 
-# CHECK: rotrv $9, $6, $7
-0xc7 0x00 0xd0 0x48
+0xc7 0x00 0xd0 0x48 # CHECK: rotrv $9, $6, $7
 
-# CHECK: lb $5, 8($4)
-0xa4 0x1c 0x08 0x00
+0xa4 0x1c 0x08 0x00 # CHECK: lb $5, 8($4)
 
-# CHECK: lbu $6, 8($4)
-0xc4 0x14 0x08 0x00
+0xc4 0x14 0x08 0x00 # CHECK: lbu $6, 8($4)
 
-# CHECK: lh $2, 8($4)
-0x44 0x3c 0x08 0x00
+0x44 0x3c 0x08 0x00 # CHECK: lh $2, 8($4)
 
-# CHECK: lhu $4, 8($2)
-0x82 0x34 0x08 0x00
+0x82 0x34 0x08 0x00 # CHECK: lhu $4, 8($2)
 
-# CHECK: lw $6, 4($5)
-0xc5 0xfc 0x04 0x00
+0xc5 0xfc 0x04 0x00 # CHECK: lw $6, 4($5)
 
-# CHECK: lw $6, 123($sp)
-0xdd 0xfc 0x7b 0x00
+0xdd 0xfc 0x7b 0x00 # CHECK: lw $6, 123($sp)
 
-# CHECK: sb $5, 8($4)
-0xa4 0x18 0x08 0x00
+0xa4 0x18 0x08 0x00 # CHECK: sb $5, 8($4)
 
-# CHECK: sh $2, 8($4)
-0x44 0x38 0x08 0x00
+0x44 0x38 0x08 0x00 # CHECK: sh $2, 8($4)
 
-# CHECK: sw $5, 4($6)
-0xa6 0xf8 0x04 0x00
+0xa6 0xf8 0x04 0x00 # CHECK: sw $5, 4($6)
 
-# CHECK: sw $5, 123($sp)
-0xbd 0xf8 0x7b 0x00
+0xbd 0xf8 0x7b 0x00 # CHECK: sw $5, 123($sp)
 
-# CHECK: lwu $2, 8($4)
-0x44 0x60 0x08 0xe0
+0x44 0x60 0x08 0xe0 # CHECK: lwu $2, 8($4)
 
-# CHECK: lwl $4, 16($5)
-0x85 0x60 0x10 0x00
+0x85 0x60 0x10 0x00 # CHECK: lwl $4, 16($5)
 
-# CHECK: lwr $4, 16($5)
-0x85 0x60 0x10 0x10
+0x85 0x60 0x10 0x10 # CHECK: lwr $4, 16($5)
 
-# CHECK: swl $4, 16($5)
-0x85 0x60 0x10 0x80
+0x85 0x60 0x10 0x80 # CHECK: swl $4, 16($5)
 
-# CHECK: swr $4, 16($5)
-0x85 0x60 0x10 0x90
+0x85 0x60 0x10 0x90 # CHECK: swr $4, 16($5)
 
-# CHECK: movz $9, $6, $7
-0xe6 0x00 0x58 0x48
+0xe6 0x00 0x58 0x48 # CHECK: movz $9, $6, $7
 
-# CHECK: movn $9, $6, $7
-0xe6 0x00 0x18 0x48
+0xe6 0x00 0x18 0x48 # CHECK: movn $9, $6, $7
 
-# CHECK: movt $9, $6, $fcc0
-0x26 0x55 0x7b 0x09
+0x26 0x55 0x7b 0x09 # CHECK: movt $9, $6, $fcc0
 
-# CHECK: movf $9, $6, $fcc0
-0x26 0x55 0x7b 0x01
+0x26 0x55 0x7b 0x01 # CHECK: movf $9, $6, $fcc0
 
-# CHECK: mthi $6
-0x06 0x00 0x7c 0x2d
+0x06 0x00 0x7c 0x2d # CHECK: mthi $6
 
-# CHECK: mfhi $6
-0x06 0x00 0x7c 0x0d
+0x06 0x00 0x7c 0x0d # CHECK: mfhi $6
 
-# CHECK: mtlo $6
-0x06 0x00 0x7c 0x3d
+0x06 0x00 0x7c 0x3d # CHECK: mtlo $6
 
-# CHECK: mflo $6
-0x06 0x00 0x7c 0x1d
+0x06 0x00 0x7c 0x1d # CHECK: mflo $6
 
-# CHECK: madd $4, $5
-0xa4 0x00 0x3c 0xcb
+0xa4 0x00 0x3c 0xcb # CHECK: madd $4, $5
 
-# CHECK: maddu $4, $5
-0xa4 0x00 0x3c 0xdb
+0xa4 0x00 0x3c 0xdb # CHECK: maddu $4, $5
 
-# CHECK: msub $4, $5
-0xa4 0x00 0x3c 0xeb
+0xa4 0x00 0x3c 0xeb # CHECK: msub $4, $5
 
-# CHECK: msubu $4, $5
-0xa4 0x00 0x3c 0xfb
+0xa4 0x00 0x3c 0xfb # CHECK: msubu $4, $5
 
-# CHECK: clz $9, $6
-0x26 0x01 0x3c 0x5b
+0x26 0x01 0x3c 0x5b # CHECK: clz $9, $6
 
-# CHECK: clo $9, $6
-0x26 0x01 0x3c 0x4b
+0x26 0x01 0x3c 0x4b # CHECK: clo $9, $6
 
-# CHECK: seb $9, $6
-0x26 0x01 0x3c 0x2b
+0x26 0x01 0x3c 0x2b # CHECK: seb $9, $6
 
-# CHECK: seh $9, $6
-0x26 0x01 0x3c 0x3b
+0x26 0x01 0x3c 0x3b # CHECK: seh $9, $6
 
-# CHECK: wsbh $9, $6
-0x26 0x01 0x3c 0x7b
+0x26 0x01 0x3c 0x7b # CHECK: wsbh $9, $6
 
-# CHECK: ext $9, $6, 3, 7
-0x26 0x01 0xec 0x30
+0x26 0x01 0xec 0x30 # CHECK: ext $9, $6, 3, 7
 
-# CHECK: ins $9, $6, 3, 7
-0x26 0x01 0xcc 0x48
+0x26 0x01 0xcc 0x48 # CHECK: ins $9, $6, 3, 7
 
-# CHECK: j 1328
-0x00 0xd4 0x98 0x02
+0x00 0xd4 0x98 0x02 # CHECK: j 1328
 
-# CHECK: jal 1328
-0x00 0xf4 0x98 0x02
+0x00 0xf4 0x98 0x02 # CHECK: jal 1328
 
-# CHECK: jalr $ra, $6
-0xe6 0x03 0x3c 0x0f
+0xe6 0x03 0x3c 0x0f # CHECK: jalr $ra, $6
 
-# CHECK: jr $7
-0x07 0x00 0x3c 0x0f
+0x07 0x00 0x3c 0x0f # CHECK: jr $7
 
-# CHECK: jraddiusp 20
-0x05 0x47
+0x05 0x47 # CHECK: jraddiusp 20
 
-# CHECK: beq $9, $6, 1332
-0xc9 0x94 0x9a 0x02
+0xc9 0x94 0x9a 0x02 # CHECK: beq $9, $6, 1332
 
-# CHECK: bgez $6, 1332
-0x46 0x40 0x9a 0x02
+0x46 0x40 0x9a 0x02 # CHECK: bgez $6, 1332
 
-# CHECK: bgezal $6, 1332
-0x66 0x40 0x9a 0x02
+0x66 0x40 0x9a 0x02 # CHECK: bgezal $6, 1332
 
-# CHECK: bltzal $6, 1332
-0x26 0x40 0x9a 0x02
+0x26 0x40 0x9a 0x02 # CHECK: bltzal $6, 1332
 
-# CHECK: bgtz $6, 1332
-0xc6 0x40 0x9a 0x02
+0xc6 0x40 0x9a 0x02 # CHECK: bgtz $6, 1332
 
-# CHECK: blez $6, 1332
-0x86 0x40 0x9a 0x02
+0x86 0x40 0x9a 0x02 # CHECK: blez $6, 1332
 
-# CHECK: bne $9, $6, 1332
-0xc9 0xb4 0x9a 0x02
+0xc9 0xb4 0x9a 0x02 # CHECK: bne $9, $6, 1332
 
-# CHECK: bltz $6, 1332
-0x06 0x40 0x9a 0x02
+0x06 0x40 0x9a 0x02 # CHECK: bltz $6, 1332
 
-# CHECK: teq $8, $9, 0
-0x28 0x01 0x3c 0x00
+0x28 0x01 0x3c 0x00 # CHECK: teq $8, $9, 0
 
-# CHECK: tge $8, $9, 0
-0x28 0x01 0x3c 0x02
+0x28 0x01 0x3c 0x02 # CHECK: tge $8, $9, 0
 
-# CHECK: tgeu $8, $9, 0
-0x28 0x01 0x3c 0x04
+0x28 0x01 0x3c 0x04 # CHECK: tgeu $8, $9, 0
 
-# CHECK: tlt $8, $9, 0
-0x28 0x01 0x3c 0x08
+0x28 0x01 0x3c 0x08 # CHECK: tlt $8, $9, 0
 
-# CHECK: tltu $8, $9, 0
-0x28 0x01 0x3c 0x0a
+0x28 0x01 0x3c 0x0a # CHECK: tltu $8, $9, 0
 
-# CHECK: tne $8, $9, 0
-0x28 0x01 0x3c 0x0c
+0x28 0x01 0x3c 0x0c # CHECK: tne $8, $9, 0
 
-# CHECK: teqi $9, 17767
-0xc9 0x41 0x67 0x45
+0xc9 0x41 0x67 0x45 # CHECK: teqi $9, 17767
 
-# CHECK: tgei $9, 17767
-0x29 0x41 0x67 0x45
+0x29 0x41 0x67 0x45 # CHECK: tgei $9, 17767
 
-# CHECK: tgeiu $9, 17767
-0x69 0x41 0x67 0x45
+0x69 0x41 0x67 0x45 # CHECK: tgeiu $9, 17767
 
-# CHECK: tlti $9, 17767
-0x09 0x41 0x67 0x45
+0x09 0x41 0x67 0x45 # CHECK: tlti $9, 17767
 
-# CHECK: tltiu $9, 17767
-0x49 0x41 0x67 0x45
+0x49 0x41 0x67 0x45 # CHECK: tltiu $9, 17767
 
-# CHECK: tnei $9, 17767
-0x89 0x41 0x67 0x45
+0x89 0x41 0x67 0x45 # CHECK: tnei $9, 17767
 
-# CHECK: cache 1, 8($5)
-0x25 0x20 0x08 0x60
+0x25 0x20 0x08 0x60 # CHECK: cache 1, 8($5)
 
-# CHECK: pref 1, 8($5)
-0x25 0x60 0x08 0x20
+0x25 0x60 0x08 0x20 # CHECK: pref 1, 8($5)
 
-# CHECK: ssnop
-0x00 0x00 0x00 0x08
+0x00 0x00 0x00 0x08 # CHECK: ssnop
 
-# CHECK: ehb
-0x00 0x00 0x00 0x18
+0x00 0x00 0x00 0x18 # CHECK: ehb
 
-# CHECK: pause
-0x00 0x00 0x00 0x28
+0x00 0x00 0x00 0x28 # CHECK: pause
 
-# CHECK: ll $2, 8($4)
-0x44 0x60 0x08 0x30
+0x44 0x60 0x08 0x30 # CHECK: ll $2, 8($4)
 
-# CHECK: sc $2, 8($4)
-0x44 0x60 0x08 0xb0
+0x44 0x60 0x08 0xb0 # CHECK: sc $2, 8($4)
 
-# CHECK: lwxs $2, $3($4)
-0x64 0x00 0x18 0x11
+0x64 0x00 0x18 0x11 # CHECK: lwxs $2, $3($4)
 
-# CHECK: bgezals $6, 1332
-0x66 0x42 0x9a 0x02
+0x66 0x42 0x9a 0x02 # CHECK: bgezals $6, 1332
 
-# CHECK: bltzals $6, 1332
-0x26 0x42 0x9a 0x02
+0x26 0x42 0x9a 0x02 # CHECK: bltzals $6, 1332
 
-# CHECK: beqzc $9, 1332
-0xe9 0x40 0x9a 0x02
+0xe9 0x40 0x9a 0x02 # CHECK: beqzc $9, 1332
 
-# CHECK: bnezc $9, 1332
-0xa9 0x40 0x9a 0x02
+0xa9 0x40 0x9a 0x02 # CHECK: bnezc $9, 1332
 
-# CHECK: jals 1328
-0x00 0x74 0x98 0x02
+0x00 0x74 0x98 0x02 # CHECK: jals 1328
 
-# CHECK: jalrs $ra, $6
-0xe6 0x03 0x3c 0x4f
+0xe6 0x03 0x3c 0x4f # CHECK: jalrs $ra, $6
 
-# CHECK: lwm32 $16, $17, 8($4)
-0x44 0x20 0x08 0x50
+0x44 0x20 0x08 0x50 # CHECK: lwm32 $16, $17, 8($4)
 
-# CHECK: swm32 $16, $17, 8($4)
-0x44 0x20 0x08 0xd0
+0x44 0x20 0x08 0xd0 # CHECK: swm32 $16, $17, 8($4)
 
-# CHECK: swp $16, 8($4)
-0x04 0x22  0x08 0x90
+0x04 0x22 0x08 0x90 # CHECK: swp $16, 8($4)
 
-# CHECK: lwp $16, 8($4)
-0x04 0x22 0x08 0x10
+0x04 0x22 0x08 0x10 # CHECK: lwp $16, 8($4)
 
-# CHECK: nop
-0x00 0x00 0x00 0x00
+0x00 0x00 0x00 0x00 # CHECK: nop
 
-# CHECK: addiupc $2, 20
-0x00 0x79 0x05 0x00
+0x00 0x79 0x05 0x00 # CHECK: addiupc $2, 20
 
-# CHECK: addiupc $7, 16777212
-0xbf 0x7b 0xff 0xff
+0xbf 0x7b 0xff 0xff # CHECK: addiupc $7, 16777212
 
-# CHECK: addiupc $7, -16777216
-0xc0 0x7b 0x00 0x00
+0xc0 0x7b 0x00 0x00 # CHECK: addiupc $7, -16777216
 
-# CHECK: addu16 $6, $17, $4
-0x42 0x07
+0x42 0x07 # CHECK: addu16 $6, $17, $4
 
-# CHECK: subu16 $5, $16, $3
-0xb1 0x06
+0xb1 0x06 # CHECK: subu16 $5, $16, $3
 
-# CHECK: and16 $16, $2
-0x82 0x44
+0x82 0x44 # CHECK: and16 $16, $2
 
-# CHECK: not16 $17, $3
-0x0b 0x44
+0x0b 0x44 # CHECK: not16 $17, $3
 
-# CHECK: or16 $16, $4
-0xc4 0x44
+0xc4 0x44 # CHECK: or16 $16, $4
 
-# CHECK: xor16 $17, $5
-0x4d 0x44
+0x4d 0x44 # CHECK: xor16 $17, $5
 
-# CHECK: sll16 $3, $16, 5
-0x8a 0x25
+0x8a 0x25 # CHECK: sll16 $3, $16, 5
 
-# CHECK: srl16 $4, $17, 6
-0x1d 0x26
+0x1d 0x26 # CHECK: srl16 $4, $17, 6
 
-# CHECK: lbu16 $3, 4($17)
-0x94 0x09
+0x94 0x09 # CHECK: lbu16 $3, 4($17)
 
-# CHECK: lbu16 $3, -1($16)
-0x8f 0x09
+0x8f 0x09 # CHECK: lbu16 $3, -1($16)
 
-# CHECK: lhu16 $3, 4($16)
-0x82 0x29
+0x82 0x29 # CHECK: lhu16 $3, 4($16)
 
-# CHECK: lw16 $4, 8($17)
-0x12 0x6a
+0x12 0x6a # CHECK: lw16 $4, 8($17)
 
-# CHECK: sb16 $3, 4($16)
-0x84 0x89
+0x84 0x89 # CHECK: sb16 $3, 4($16)
 
-# CHECK: sh16 $4, 8($17)
-0x14 0xaa
+0x14 0xaa # CHECK: sh16 $4, 8($17)
 
-# CHECK: sw16 $4, 4($17)
-0x11 0xea
+0x11 0xea # CHECK: sw16 $4, 4($17)
 
-# CHECK: sw16 $zero, 4($17)
-0x11 0xe8
+0x11 0xe8 # CHECK: sw16 $zero, 4($17)
 
-# CHECK: mfhi $9
-0x09 0x46
+0x09 0x46 # CHECK: mfhi $9
 
-# CHECK: mflo $9
-0x49 0x46
+0x49 0x46 # CHECK: mflo $9
 
-# CHECK: move $25, $1
-0x21 0x0f
+0x21 0x0f # CHECK: move $25, $1
 
-# CHECK: jrc $9
-0xa9 0x45
+0xa9 0x45 # CHECK: jrc $9
 
-# CHECK: jalr $9
-0xc9 0x45
+0xc9 0x45 # CHECK: jalr $9
 
-# CHECK: jalrs16 $9
-0xe9 0x45
+0xe9 0x45 # CHECK: jalrs16 $9
 
-# CHECK: jr16 $9
-0x89 0x45
+0x89 0x45 # CHECK: jr16 $9
 
-# CHECK: li16 $3, -1
-0xff 0xed
+0xff 0xed # CHECK: li16 $3, -1
 
-# CHECK: li16 $3, 126
-0xfe 0xed
+0xfe 0xed # CHECK: li16 $3, 126
 
-# CHECK: addiur1sp $7, 4
-0x83 0x6f
+0x83 0x6f # CHECK: addiur1sp $7, 4
 
-# CHECK: addiur2 $6, $7, -1
-0x7e 0x6f
+0x7e 0x6f # CHECK: addiur2 $6, $7, -1
 
-# CHECK: addiur2 $6, $7, 12
-0x76 0x6f
+0x76 0x6f # CHECK: addiur2 $6, $7, 12
 
-# CHECK: addius5 $7, -2
-0xfc 0x4c
+0xfc 0x4c # CHECK: addius5 $7, -2
 
-# CHECK: nop
-0x00 0x0c
+0x00 0x0c # CHECK: nop
 
-# CHECK: lw $3, 32($sp)
-0x68 0x48
+0x68 0x48 # CHECK: lw $3, 32($sp)
 
-# CHECK: sw $4, 124($sp)
-0x9f 0xc8
+0x9f 0xc8 # CHECK: sw $4, 124($sp)
 
-# CHECK: beqz16 $6, 20
-0x0a 0x8f
+0x0a 0x8f # CHECK: beqz16 $6, 20
 
-# CHECK: bnez16 $6, 20
-0x0a 0xaf
+0x0a 0xaf # CHECK: bnez16 $6, 20
 
-# CHECK: b16 132
-0x42 0xcc
+0x42 0xcc # CHECK: b16 132
 
-# CHECK: lw $3, 32($gp)
-0x88 0x65
+0x88 0x65 # CHECK: lw $3, 32($gp)
 
-# CHECK: lwm16 $16, $17, $ra, 8($sp)
-0x12 0x45
+0x12 0x45 # CHECK: lwm16 $16, $17, $ra, 8($sp)
 
-# CHECK: swm16 $16, $17, $ra, 8($sp)
-0x52 0x45
+0x52 0x45 # CHECK: swm16 $16, $17, $ra, 8($sp)
 
-# CHECK: break16 8
-0x88 0x46
+0x88 0x46 # CHECK: break16 8
 
-# CHECK: sdbbp16 14
-0xce 0x46
+0xce 0x46 # CHECK: sdbbp16 14
 
-# CHECK: movep $5, $6, $2, $3
-0x34 0x84
+0x34 0x84 # CHECK: movep $5, $6, $2, $3
+
+0x00 0x00 0x7c 0x57 # CHECK: ei
+
+0x0a 0x00 0x7c 0x57 # CHECK: ei $10
diff --git a/test/MC/Disassembler/Mips/mips1/valid-mips1.txt b/test/MC/Disassembler/Mips/mips1/valid-mips1.txt
index 1a4f94f083e..59e702e17e1 100644
--- a/test/MC/Disassembler/Mips/mips1/valid-mips1.txt
+++ b/test/MC/Disassembler/Mips/mips1/valid-mips1.txt
@@ -1,116 +1,110 @@
 # RUN: llvm-mc %s -triple=mips-unknown-linux -disassemble -mcpu=mips1 | FileCheck %s
 # CHECK: .text
-0x46 0x20 0xc1 0x85 # CHECK: abs.d $f6, $f24
-0x46 0x00 0x82 0x45 # CHECK: abs.s $f9, $f16
+0x00 0x00 0x00 0x00 # CHECK: nop
+0x00 0x00 0x00 0x40 # CHECK: ssnop
+0x00 0x00 0x3c 0x80 # CHECK: sll $7, $zero, 18
+0x00 0x00 0x88 0x12 # CHECK: mflo $17
+0x00 0x00 0x98 0x10 # CHECK: mfhi $19
+0x00 0x00 0xe8 0x10 # CHECK: mfhi $sp
+0x00 0x02 0x10 0x23 # CHECK: negu $2, $2
+0x00 0x02 0x11 0xc2 # CHECK: srl $2, $2, 7
+0x00 0x02 0x11 0xc2 # CHECK: srl $2, $2, 7
+0x00 0x03 0x10 0x23 # CHECK: negu $2, $3
+0x00 0x07 0x38 0x27 # CHECK: nor $7, $zero, $7
+0x00 0x07 0x3c 0x80 # CHECK: sll $7, $7, 18
+0x00 0x11 0x8b 0xc3 # CHECK: sra $17, $17, 15
+0x00 0x17 0x8b 0xc3 # CHECK: sra $17, $23, 15
+0x00 0x4c 0xb8 0x24 # CHECK: and $23, $2, $12
+0x00 0x80 0xf0 0x21 # CHECK: move $fp, $4
+0x00 0x86 0x48 0x21 # CHECK: addu $9, $4, $6
+0x00 0x94 0xc8 0x06 # CHECK: srlv $25, $20, $4
+0x00 0x94 0xc8 0x06 # CHECK: srlv $25, $20, $4
+0x00 0x9e 0x90 0x26 # CHECK: xor $18, $4, $fp
+0x00 0xc0 0xc8 0x21 # CHECK: move $25, $6
+0x01 0x20 0x38 0x04 # CHECK: sllv $7, $zero, $9
+0x01 0x20 0x38 0x04 # CHECK: sllv $7, $zero, $9
+0x01 0x32 0x00 0x19 # CHECK: multu $9, $18
+0x01 0x7b 0xb8 0x2a # CHECK: slt $23, $11, $27
+0x02 0x1d 0x60 0x25 # CHECK: or $12, $16, $sp
+0x02 0x20 0x00 0x11 # CHECK: mthi $17
 0x02 0x45 0xb8 0x20 # CHECK: add $23, $18, $5
-0x46 0x3c 0x30 0x00 # CHECK: add.d $f0, $f6, $f28
-0x46 0x18 0xaa 0x00 # CHECK: add.s $f8, $f21, $f24
-0x21 0x2d 0x66 0xd2 # CHECK: addi $13, $9, 26322
+0x02 0x6c 0xb0 0x22 # CHECK: sub $22, $19, $12
+0x02 0xab 0xa0 0x2b # CHECK: sltu $20, $21, $11
+0x02 0xd6 0xe8 0x23 # CHECK: subu $sp, $22, $22
+0x03 0x20 0x00 0x13 # CHECK: mtlo $25
+0x03 0x2b 0x00 0x1a # CHECK: div $zero, $25, $11
+0x03 0x2f 0x00 0x1b # CHECK: divu $zero, $25, $15
+0x03 0x9a 0x00 0x19 # CHECK: multu $gp, $26
+0x03 0xa0 0x00 0x13 # CHECK: mtlo $sp
+0x03 0xa2 0x00 0x18 # CHECK: mult $sp, $2
+0x03 0xb4 0x00 0x18 # CHECK: mult $sp, $20
+0x03 0xb7 0x88 0x07 # CHECK: srav $17, $23, $sp
+0x03 0xb7 0x88 0x07 # CHECK: srav $17, $23, $sp
+0x04 0x11 0x14 0x9b # CHECK: bal 21104
+0x04 0xd0 0x14 0x9b # CHECK: bltzal $6, 21104
+0x04 0xd1 0x14 0x9b # CHECK: bgezal $6, 21104
 0x21 0x08 0xff 0xfe # CHECK: addi $8, $8, -2
+0x21 0x2d 0x66 0xd2 # CHECK: addi $13, $9, 26322
+0x21 0xad 0xe6 0x90 # CHECK: addi $13, $13, -6512
 0x21 0xc9 0x3b 0x48 # CHECK: addi $9, $14, 15176
+0x22 0x36 0x0c 0x36 # CHECK: addi $22, $17, 3126
 0x23 0x18 0xe3 0xe7 # CHECK: addi $24, $24, -7193
-0x00 0x86 0x48 0x21 # CHECK: addu $9, $4, $6
+0x24 0x00 0x8b 0x3f # CHECK: addiu $zero, $zero, -29889
+0x24 0x01 0x8b 0xb3 # CHECK: addiu $1, $zero, -29773
 0x25 0x29 0x00 0x0a # CHECK: addiu $9, $9, 10
-0x00 0x4c 0xb8 0x24 # CHECK: and $23, $2, $12
-0x45 0x00 0x00 0x01 # CHECK: bc1f 8
+0x29 0x51 0x25 0x11 # CHECK: slti $17, $10, 9489
+0x2f 0x38 0xc3 0x55 # CHECK: sltiu $24, $25, -15531
+0x2f 0x39 0xc3 0x55 # CHECK: sltiu $25, $25, -15531
 0x30 0x42 0x00 0x04 # CHECK: andi $2, $2, 4
-0x45 0x01 0x00 0x01 # CHECK: bc1t 8
-0x00 0x00 0x00 0x00 # CHECK: nop
-0x04 0x11 0x14 0x9b # CHECK: bal 21104
-0x00 0x00 0x00 0x00 # CHECK: nop
-0x04 0x11 0x14 0x9b # CHECK: bal 21104
-0x00 0x00 0x00 0x00 # CHECK: nop
-0x04 0xd1 0x14 0x9b # CHECK: bgezal $6, 21104
-0x00 0x00 0x00 0x00 # CHECK: nop
-0x04 0xd0 0x14 0x9b # CHECK: bltzal $6, 21104
-0x00 0x00 0x00 0x00 # CHECK: nop
-0x46 0x3c 0xe0 0x3b # CHECK: c.ngl.d $f28, $f28
-0x46 0x30 0x00 0x39 # CHECK: c.ngle.d $f0, $f16
-0x46 0x20 0xf0 0x38 # CHECK: c.sf.d $f30, $f0
-0x46 0x16 0x70 0x38 # CHECK: c.sf.s $f14, $f22
+0x34 0x42 0x00 0x04 # CHECK: ori $2, $2, 4
+0x42 0x00 0x00 0x01 # CHECK: tlbr
+0x42 0x00 0x00 0x02 # CHECK: tlbwi
+0x42 0x00 0x00 0x06 # CHECK: tlbwr
+0x42 0x00 0x00 0x08 # CHECK: tlbp
+0x44 0x07 0xd8 0x00 # CHECK: mfc1 $7, $f27
 0x44 0x51 0xa8 0x00 # CHECK: cfc1 $17, $21
+0x44 0x9e 0x48 0x00 # CHECK: mtc1 $fp, $f9
 0x44 0xc6 0xd0 0x00 # CHECK: ctc1 $6, $26
+0x45 0x00 0x00 0x01 # CHECK: bc1f 8
+0x45 0x01 0x00 0x01 # CHECK: bc1t 8
+0x46 0x00 0x78 0x47 # CHECK: neg.s $f1, $f15
+0x46 0x00 0x82 0x45 # CHECK: abs.s $f9, $f16
+0x46 0x00 0xc5 0x24 # CHECK: cvt.w.s $f20, $f24
+0x46 0x00 0xd8 0x86 # CHECK: mov.s $f2, $f27
 0x46 0x00 0xe5 0xa1 # CHECK: cvt.d.s $f22, $f28
-0x46 0x80 0x5e 0xa1 # CHECK: cvt.d.w $f26, $f11
+0x46 0x02 0x57 0x82 # CHECK: mul.s $f30, $f10, $f2
+0x46 0x0f 0x29 0x03 # CHECK: div.s $f4, $f5, $f15
+0x46 0x16 0x70 0x38 # CHECK: c.sf.s $f14, $f22
+0x46 0x16 0xb5 0xc1 # CHECK: sub.s $f23, $f22, $f22
+0x46 0x18 0xaa 0x00 # CHECK: add.s $f8, $f21, $f24
 0x46 0x20 0x46 0xa0 # CHECK: cvt.s.d $f26, $f8
-0x46 0x80 0x7d 0xa0 # CHECK: cvt.s.w $f22, $f15
+0x46 0x20 0x75 0x06 # CHECK: mov.d $f20, $f14
 0x46 0x20 0x75 0x24 # CHECK: cvt.w.d $f20, $f14
-0x46 0x00 0xc5 0x24 # CHECK: cvt.w.s $f20, $f24
-0x03 0x2b 0x00 0x1a # CHECK: div $zero, $25, $11
+0x46 0x20 0x96 0x87 # CHECK: neg.d $f26, $f18
+0x46 0x20 0xc1 0x85 # CHECK: abs.d $f6, $f24
+0x46 0x20 0xf0 0x38 # CHECK: c.sf.d $f30, $f0
+0x46 0x30 0x00 0x39 # CHECK: c.ngle.d $f0, $f16
+0x46 0x30 0x14 0x81 # CHECK: sub.d $f18, $f2, $f16
+0x46 0x30 0xa5 0x02 # CHECK: mul.d $f20, $f20, $f16
 0x46 0x3a 0xa7 0x03 # CHECK: div.d $f28, $f20, $f26
-0x46 0x0f 0x29 0x03 # CHECK: div.s $f4, $f5, $f15
-0x03 0x2f 0x00 0x1b # CHECK: divu $zero, $25, $15
+0x46 0x3c 0x30 0x00 # CHECK: add.d $f0, $f6, $f28
+0x46 0x3c 0xe0 0x3b # CHECK: c.ngl.d $f28, $f28
+0x46 0x80 0x5e 0xa1 # CHECK: cvt.d.w $f26, $f11
+0x46 0x80 0x7d 0xa0 # CHECK: cvt.s.w $f22, $f15
 0x81 0x58 0xc7 0x4d # CHECK: lb $24, -14515($10)
-0x90 0x68 0x75 0xf3 # CHECK: lbu $8, 30195($3)
 0x86 0xab 0xde 0x94 # CHECK: lh $11, -8556($21)
-0x94 0x53 0xa6 0xbd # CHECK: lhu $19, -22851($2)
-0x24 0x01 0x8b 0xb3 # CHECK: addiu $1, $zero, -29773
-0x24 0x00 0x8b 0x3f # CHECK: addiu $zero, $zero, -29889
-0x8c 0xa8 0x16 0x2a # CHECK: lw $8, 5674($5)
-0xc7 0x50 0x27 0xf1 # CHECK: lwc1 $f16, 10225($26)
-0xc8 0xd2 0xfc 0xb7 # CHECK: lwc2 $18, -841($6)
-0xcf 0x4a 0x81 0xf7 # CHECK: lwc3 $10, -32265($26)
 0x89 0xf4 0xef 0x79 # CHECK: lwl $20, -4231($15)
+0x8c 0xa8 0x16 0x2a # CHECK: lw $8, 5674($5)
+0x90 0x68 0x75 0xf3 # CHECK: lbu $8, 30195($3)
+0x94 0x53 0xa6 0xbd # CHECK: lhu $19, -22851($2)
 0x9b 0x80 0xb5 0x35 # CHECK: lwr $zero, -19147($gp)
-0x44 0x07 0xd8 0x00 # CHECK: mfc1 $7, $f27
-0x00 0x00 0x98 0x10 # CHECK: mfhi $19
-0x00 0x00 0xe8 0x10 # CHECK: mfhi $sp
-0x00 0x00 0x88 0x12 # CHECK: mflo $17
-0x46 0x20 0x75 0x06 # CHECK: mov.d $f20, $f14
-0x46 0x00 0xd8 0x86 # CHECK: mov.s $f2, $f27
-0x00 0x80 0xf0 0x21 # CHECK: move $fp, $4
-0x00 0xc0 0xc8 0x21 # CHECK: move $25, $6
-0x44 0x9e 0x48 0x00 # CHECK: mtc1 $fp, $f9
-0x02 0x20 0x00 0x11 # CHECK: mthi $17
-0x03 0xa0 0x00 0x13 # CHECK: mtlo $sp
-0x03 0x20 0x00 0x13 # CHECK: mtlo $25
-0x46 0x30 0xa5 0x02 # CHECK: mul.d $f20, $f20, $f16
-0x46 0x02 0x57 0x82 # CHECK: mul.s $f30, $f10, $f2
-0x03 0xb4 0x00 0x18 # CHECK: mult $sp, $20
-0x03 0xa2 0x00 0x18 # CHECK: mult $sp, $2
-0x03 0x9a 0x00 0x19 # CHECK: multu $gp, $26
-0x01 0x32 0x00 0x19 # CHECK: multu $9, $18
-0x00 0x02 0x10 0x23 # CHECK: negu $2, $2
-0x00 0x03 0x10 0x23 # CHECK: negu $2, $3
-0x46 0x20 0x96 0x87 # CHECK: neg.d $f26, $f18
-0x46 0x00 0x78 0x47 # CHECK: neg.s $f1, $f15
-0x00 0x00 0x00 0x00 # CHECK: nop
-0x00 0x07 0x38 0x27 # CHECK: nor $7, $zero, $7
-0x02 0x1d 0x60 0x25 # CHECK: or $12, $16, $sp
-0x34 0x42 0x00 0x04 # CHECK: ori $2, $2, 4
 0xa1 0xd6 0xb2 0x6f # CHECK: sb $22, -19857($14)
 0xa5 0xee 0xe5 0xd0 # CHECK: sh $14, -6704($15)
-0x00 0x07 0x3c 0x80 # CHECK: sll $7, $7, 18
-0x00 0x00 0x3c 0x80 # CHECK: sll $7, $zero, 18
-0x01 0x20 0x38 0x04 # CHECK: sllv $7, $zero, $9
-0x01 0x20 0x38 0x04 # CHECK: sllv $7, $zero, $9
-0x01 0x7b 0xb8 0x2a # CHECK: slt $23, $11, $27
-0x29 0x51 0x25 0x11 # CHECK: slti $17, $10, 9489
-0x2f 0x39 0xc3 0x55 # CHECK: sltiu $25, $25, -15531
-0x02 0xab 0xa0 0x2b # CHECK: sltu $20, $21, $11
-0x2f 0x38 0xc3 0x55 # CHECK: sltiu $24, $25, -15531
-0x00 0x11 0x8b 0xc3 # CHECK: sra $17, $17, 15
-0x00 0x17 0x8b 0xc3 # CHECK: sra $17, $23, 15
-0x03 0xb7 0x88 0x07 # CHECK: srav $17, $23, $sp
-0x03 0xb7 0x88 0x07 # CHECK: srav $17, $23, $sp
-0x00 0x02 0x11 0xc2 # CHECK: srl $2, $2, 7
-0x00 0x02 0x11 0xc2 # CHECK: srl $2, $2, 7
-0x00 0x94 0xc8 0x06 # CHECK: srlv $25, $20, $4
-0x00 0x94 0xc8 0x06 # CHECK: srlv $25, $20, $4
-0x00 0x00 0x00 0x40 # CHECK: ssnop
-0x02 0x6c 0xb0 0x22 # CHECK: sub $22, $19, $12
-0x22 0x36 0x0c 0x36 # CHECK: addi $22, $17, 3126
-0x21 0xad 0xe6 0x90 # CHECK: addi $13, $13, -6512
-0x46 0x30 0x14 0x81 # CHECK: sub.d $f18, $f2, $f16
-0x46 0x16 0xb5 0xc1 # CHECK: sub.s $f23, $f22, $f22
-0x02 0xd6 0xe8 0x23 # CHECK: subu $sp, $22, $22
+0xaa 0x6f 0x35 0x7e # CHECK: swl $15, 13694($19)
 0xaf 0xbf 0xd8 0x50 # CHECK: sw $ra, -10160($sp)
+0xb9 0xd1 0x98 0x22 # CHECK: swr $17, -26590($14)
+0xc7 0x50 0x27 0xf1 # CHECK: lwc1 $f16, 10225($26)
+0xc8 0xd2 0xfc 0xb7 # CHECK: lwc2 $18, -841($6)
+0xcf 0x4a 0x81 0xf7 # CHECK: lwc3 $10, -32265($26)
 0xe7 0x06 0xde 0xef # CHECK: swc1 $f6, -8465($24)
 0xea 0x19 0x61 0x30 # CHECK: swc2 $25, 24880($16)
-0xaa 0x6f 0x35 0x7e # CHECK: swl $15, 13694($19)
-0xb9 0xd1 0x98 0x22 # CHECK: swr $17, -26590($14)
-0x42 0x00 0x00 0x08 # CHECK: tlbp
-0x42 0x00 0x00 0x01 # CHECK: tlbr
-0x42 0x00 0x00 0x02 # CHECK: tlbwi
-0x42 0x00 0x00 0x06 # CHECK: tlbwr
-0x00 0x9e 0x90 0x26 # CHECK: xor $18, $4, $fp
diff --git a/test/MC/Disassembler/Mips/mips2.txt b/test/MC/Disassembler/Mips/mips2.txt
deleted file mode 100644
index a604055e62e..00000000000
--- a/test/MC/Disassembler/Mips/mips2.txt
+++ /dev/null
@@ -1,13 +0,0 @@
-# RUN: llvm-mc --disassemble %s -triple=mips-unknown-linux -mcpu=mips2 | FileCheck %s
-
-# CHECK: sdc3  $5, 9154($6)
-0xfc 0xc5 0x23 0xc2
-
-# CHECK: swc3  $6, 9158($7)
-0xec 0xe6 0x23 0xc6
-
-# CHECK: ldc3  $7, 9162($8)
-0xdd 0x07 0x23 0xca
-
-# CHECK: lwc3  $8, 9166($9)
-0xcd 0x28 0x23 0xce
diff --git a/test/MC/Disassembler/Mips/mips2/valid-mips2.txt b/test/MC/Disassembler/Mips/mips2/valid-mips2.txt
index 3dc523168ff..268bb290090 100644
--- a/test/MC/Disassembler/Mips/mips2/valid-mips2.txt
+++ b/test/MC/Disassembler/Mips/mips2/valid-mips2.txt
@@ -1,159 +1,161 @@
 # RUN: llvm-mc %s -triple=mips-unknown-linux -disassemble -mcpu=mips2 | FileCheck %s
 # CHECK: .text
-0x46 0x20 0xc1 0x85 # CHECK: abs.d $f6, $f24
-0x46 0x00 0x82 0x45 # CHECK: abs.s $f9, $f16
+0x00 0x00 0x00 0x00 # CHECK: nop
+0x00 0x00 0x00 0x40 # CHECK: ssnop
+0x00 0x00 0x00 0xc0 # CHECK: ehb
+0x00 0x00 0x3c 0x80 # CHECK: sll $7, $zero, 18
+0x00 0x00 0x88 0x12 # CHECK: mflo $17
+0x00 0x00 0x98 0x10 # CHECK: mfhi $19
+0x00 0x00 0xe8 0x10 # CHECK: mfhi $sp
+0x00 0x02 0x10 0x23 # CHECK: negu $2, $2
+0x00 0x02 0x11 0xc2 # CHECK: srl $2, $2, 7
+0x00 0x02 0x11 0xc2 # CHECK: srl $2, $2, 7
+0x00 0x03 0x00 0x34 # CHECK: teq $zero, $3
+0x00 0x03 0x10 0x23 # CHECK: negu $2, $3
+0x00 0x07 0x38 0x27 # CHECK: nor $7, $zero, $7
+0x00 0x07 0x3c 0x80 # CHECK: sll $7, $7, 18
+0x00 0x11 0x8b 0xc3 # CHECK: sra $17, $17, 15
+0x00 0x17 0x8b 0xc3 # CHECK: sra $17, $23, 15
+0x00 0x4c 0xb8 0x24 # CHECK: and $23, $2, $12
+0x00 0x53 0x21 0x72 # CHECK: tlt $2, $19, 133
+0x00 0x80 0xf0 0x21 # CHECK: move $fp, $4
+0x00 0x86 0x48 0x21 # CHECK: addu $9, $4, $6
+0x00 0x94 0xc8 0x06 # CHECK: srlv $25, $20, $4
+0x00 0x94 0xc8 0x06 # CHECK: srlv $25, $20, $4
+0x00 0x9e 0x90 0x26 # CHECK: xor $18, $4, $fp
+0x00 0xa7 0x9b 0x34 # CHECK: teq $5, $7, 620
+0x00 0xb3 0x55 0x30 # CHECK: tge $5, $19, 340
+0x00 0xc0 0xc8 0x21 # CHECK: move $25, $6
+0x00 0xd1 0x00 0x36 # CHECK: tne $6, $17
+0x00 0xe8 0xdd 0x76 # CHECK: tne $7, $8, 885
+0x00 0xea 0x00 0x30 # CHECK: tge $7, $10
+0x01 0x20 0x38 0x04 # CHECK: sllv $7, $zero, $9
+0x01 0x20 0x38 0x04 # CHECK: sllv $7, $zero, $9
+0x01 0x32 0x00 0x19 # CHECK: multu $9, $18
+0x01 0x70 0x00 0x33 # CHECK: tltu $11, $16
+0x01 0x7b 0xb8 0x2a # CHECK: slt $23, $11, $27
+0x01 0xed 0x00 0x32 # CHECK: tlt $15, $13
+0x02 0x1d 0x60 0x25 # CHECK: or $12, $16, $sp
+0x02 0x1d 0xfe 0x33 # CHECK: tltu $16, $sp, 1016
+0x02 0x20 0x00 0x11 # CHECK: mthi $17
 0x02 0x45 0xb8 0x20 # CHECK: add $23, $18, $5
+0x02 0x6c 0xb0 0x22 # CHECK: sub $22, $19, $12
+0x02 0x8e 0x5e 0xf1 # CHECK: tgeu $20, $14, 379
+0x02 0xab 0xa0 0x2b # CHECK: sltu $20, $21, $11
+0x02 0xd6 0xe8 0x23 # CHECK: subu $sp, $22, $22
+0x02 0xdc 0x00 0x31 # CHECK: tgeu $22, $gp
+0x03 0x20 0x00 0x13 # CHECK: mtlo $25
+0x03 0x2b 0x00 0x1a # CHECK: div $zero, $25, $11
+0x03 0x2f 0x00 0x1b # CHECK: divu $zero, $25, $15
+0x03 0x9a 0x00 0x19 # CHECK: multu $gp, $26
+0x03 0xa0 0x00 0x13 # CHECK: mtlo $sp
+0x03 0xa2 0x00 0x18 # CHECK: mult $sp, $2
+0x03 0xb4 0x00 0x18 # CHECK: mult $sp, $20
+0x03 0xb7 0x88 0x07 # CHECK: srav $17, $23, $sp
+0x03 0xb7 0x88 0x07 # CHECK: srav $17, $23, $sp
+0x04 0x11 0x14 0x9b # CHECK: bal 21104
+0x04 0x83 0xf9 0x4d # CHECK: bgezl $4, -6856
+0x04 0xd0 0x14 0x9b # CHECK: bltzal $6, 21104
+0x04 0xd2 0x00 0x7a # CHECK: bltzall $6, 492
+0x05 0x8e 0x8c 0x31 # CHECK: tnei $12, 35889
+0x05 0x93 0x07 0x1f # CHECK: bgezall $12, 7296
+0x05 0xca 0xad 0xbd # CHECK: tlti $14, 44477
+0x06 0x22 0xf6 0x45 # CHECK: bltzl $17, -9960
+0x06 0x28 0x13 0xa1 # CHECK: tgei $17, 5025
+0x06 0xac 0xbb 0xa0 # CHECK: teqi $21, 48032
+0x07 0xa9 0x90 0x33 # CHECK: tgeiu $sp, 36915
+0x07 0xeb 0xec 0x2c # CHECK: tltiu $ra, 60460
+0x21 0x08 0xff 0xfe # CHECK: addi $8, $8, -2
+0x21 0x2d 0x66 0xd2 # CHECK: addi $13, $9, 26322
+0x21 0xad 0xe6 0x90 # CHECK: addi $13, $13, -6512
 0x21 0xc9 0x3b 0x48 # CHECK: addi $9, $14, 15176
+0x22 0x36 0x0c 0x36 # CHECK: addi $22, $17, 3126
 0x23 0x18 0xe3 0xe7 # CHECK: addi $24, $24, -7193
-0x46 0x3c 0x30 0x00 # CHECK: add.d $f0, $f6, $f28
-0x46 0x18 0xaa 0x00 # CHECK: add.s $f8, $f21, $f24
-0x21 0x2d 0x66 0xd2 # CHECK: addi $13, $9, 26322
-0x21 0x08 0xff 0xfe # CHECK: addi $8, $8, -2
-0x00 0x86 0x48 0x21 # CHECK: addu $9, $4, $6
+0x24 0x00 0x8b 0x3f # CHECK: addiu $zero, $zero, -29889
+0x24 0x01 0x8b 0xb3 # CHECK: addiu $1, $zero, -29773
 0x25 0x29 0x00 0x0a # CHECK: addiu $9, $9, 10
-0x00 0x4c 0xb8 0x24 # CHECK: and $23, $2, $12
+0x29 0x51 0x25 0x11 # CHECK: slti $17, $10, 9489
+0x2f 0x38 0xc3 0x55 # CHECK: sltiu $24, $25, -15531
+0x2f 0x39 0xc3 0x55 # CHECK: sltiu $25, $25, -15531
 0x30 0x42 0x00 0x04 # CHECK: andi $2, $2, 4
+0x34 0x42 0x00 0x04 # CHECK: ori $2, $2, 4
+0x42 0x00 0x00 0x01 # CHECK: tlbr
+0x42 0x00 0x00 0x02 # CHECK: tlbwi
+0x42 0x00 0x00 0x06 # CHECK: tlbwr
+0x42 0x00 0x00 0x08 # CHECK: tlbp
+0x44 0x07 0xd8 0x00 # CHECK: mfc1 $7, $f27
+0x44 0x51 0xa8 0x00 # CHECK: cfc1 $17, $21
+0x44 0x9e 0x48 0x00 # CHECK: mtc1 $fp, $f9
+0x44 0xc6 0xd0 0x00 # CHECK: ctc1 $6, $26
 0x45 0x00 0x00 0x01 # CHECK: bc1f 8
-0x45 0x02 0x00 0x0c # CHECK: bc1fl 52
 0x45 0x01 0x00 0x01 # CHECK: bc1t 8
+0x45 0x02 0x00 0x0c # CHECK: bc1fl 52
 0x45 0x03 0xf7 0xf4 # CHECK: bc1tl -8236
-0x04 0x11 0x14 0x9b # CHECK: bal 21104
-0x00 0x00 0x00 0x00 # CHECK: nop
-0x04 0xd0 0x14 0x9b # CHECK: bltzal $6, 21104
-0x00 0x00 0x00 0x00 # CHECK: nop
-0x51 0xd3 0x0c 0x40 # CHECK: beql $14, $19, 12548
-0x05 0x93 0x07 0x1f # CHECK: bgezall $12, 7296
-0x04 0x83 0xf9 0x4d # CHECK: bgezl $4, -6856
-0x5d 0x40 0xfc 0x59 # CHECK: bgtzl $10, -3736
-0x58 0xc0 0x02 0xe7 # CHECK: blezl $6, 2976
-0x04 0xd2 0x00 0x7a # CHECK: bltzall $6, 492
-0x06 0x22 0xf6 0x45 # CHECK: bltzl $17, -9960
-0x57 0x94 0x04 0xfc # CHECK: bnel $gp, $20, 5108
-0x46 0x3c 0xe0 0x3b # CHECK: c.ngl.d $f28, $f28
-0x46 0x30 0x00 0x39 # CHECK: c.ngle.d $f0, $f16
-0x46 0x20 0xf0 0x38 # CHECK: c.sf.d $f30, $f0
-0x46 0x16 0x70 0x38 # CHECK: c.sf.s $f14, $f22
-0x46 0x20 0xc2 0xce # CHECK: ceil.w.d $f11, $f24
+0x46 0x00 0x08 0x04 # CHECK: sqrt.s $f0, $f1
+0x46 0x00 0x4a 0x0f # CHECK: floor.w.s $f8, $f9
+0x46 0x00 0x78 0x47 # CHECK: neg.s $f1, $f15
+0x46 0x00 0x82 0x45 # CHECK: abs.s $f9, $f16
 0x46 0x00 0xa1 0x8e # CHECK: ceil.w.s $f6, $f20
-0x44 0x51 0xa8 0x00 # CHECK: cfc1 $17, $21
-0x44 0xc6 0xd0 0x00 # CHECK: ctc1 $6, $26
+0x46 0x00 0xc5 0x24 # CHECK: cvt.w.s $f20, $f24
+0x46 0x00 0xd8 0x86 # CHECK: mov.s $f2, $f27
 0x46 0x00 0xe5 0xa1 # CHECK: cvt.d.s $f22, $f28
-0x46 0x80 0x5e 0xa1 # CHECK: cvt.d.w $f26, $f11
+0x46 0x00 0xe6 0xcc # CHECK: round.w.s $f27, $f28
+0x46 0x00 0xf7 0x0d # CHECK: trunc.w.s $f28, $f30
+0x46 0x02 0x57 0x82 # CHECK: mul.s $f30, $f10, $f2
+0x46 0x0f 0x29 0x03 # CHECK: div.s $f4, $f5, $f15
+0x46 0x16 0x70 0x38 # CHECK: c.sf.s $f14, $f22
+0x46 0x16 0xb5 0xc1 # CHECK: sub.s $f23, $f22, $f22
+0x46 0x18 0xaa 0x00 # CHECK: add.s $f8, $f21, $f24
+0x46 0x20 0x21 0x8c # CHECK: round.w.d $f6, $f4
 0x46 0x20 0x46 0xa0 # CHECK: cvt.s.d $f26, $f8
-0x46 0x80 0x7d 0xa0 # CHECK: cvt.s.w $f22, $f15
+0x46 0x20 0x53 0x8f # CHECK: floor.w.d $f14, $f10
+0x46 0x20 0x75 0x06 # CHECK: mov.d $f20, $f14
 0x46 0x20 0x75 0x24 # CHECK: cvt.w.d $f20, $f14
-0x46 0x00 0xc5 0x24 # CHECK: cvt.w.s $f20, $f24
-0x03 0x2b 0x00 0x1a # CHECK: div $zero, $25, $11
+0x46 0x20 0x75 0x8d # CHECK: trunc.w.d $f22, $f14
+0x46 0x20 0x96 0x87 # CHECK: neg.d $f26, $f18
+0x46 0x20 0xb4 0x04 # CHECK: sqrt.d $f16, $f22
+0x46 0x20 0xc1 0x85 # CHECK: abs.d $f6, $f24
+0x46 0x20 0xc2 0xce # CHECK: ceil.w.d $f11, $f24
+0x46 0x20 0xf0 0x38 # CHECK: c.sf.d $f30, $f0
+0x46 0x30 0x00 0x39 # CHECK: c.ngle.d $f0, $f16
+0x46 0x30 0x14 0x81 # CHECK: sub.d $f18, $f2, $f16
+0x46 0x30 0xa5 0x02 # CHECK: mul.d $f20, $f20, $f16
 0x46 0x3a 0xa7 0x03 # CHECK: div.d $f28, $f20, $f26
-0x46 0x0f 0x29 0x03 # CHECK: div.s $f4, $f5, $f15
-0x03 0x2f 0x00 0x1b # CHECK: divu $zero, $25, $15
-0x00 0x00 0x00 0xc0 # CHECK: ehb
-0x46 0x20 0x53 0x8f # CHECK: floor.w.d $f14, $f10
-0x46 0x00 0x4a 0x0f # CHECK: floor.w.s $f8, $f9
+0x46 0x3c 0x30 0x00 # CHECK: add.d $f0, $f6, $f28
+0x46 0x3c 0xe0 0x3b # CHECK: c.ngl.d $f28, $f28
+0x46 0x80 0x5e 0xa1 # CHECK: cvt.d.w $f26, $f11
+0x46 0x80 0x7d 0xa0 # CHECK: cvt.s.w $f22, $f15
+0x51 0xd3 0x0c 0x40 # CHECK: beql $14, $19, 12548
+0x57 0x94 0x04 0xfc # CHECK: bnel $gp, $20, 5108
+0x58 0xc0 0x02 0xe7 # CHECK: blezl $6, 2976
+0x5d 0x40 0xfc 0x59 # CHECK: bgtzl $10, -3736
 0x81 0x58 0xc7 0x4d # CHECK: lb $24, -14515($10)
-0x90 0x68 0x75 0xf3 # CHECK: lbu $8, 30195($3)
-0xd6 0x0a 0x40 0x07 # CHECK: ldc1 $f10, 16391($16)
-0xd8 0x28 0xad 0x43 # CHECK: ldc2 $8, -21181($1)
-0xde 0x3d 0x90 0x1b # CHECK: ldc3 $29, -28645($17)
 0x86 0xab 0xde 0x94 # CHECK: lh $11, -8556($21)
+0x89 0xf4 0xef 0x79 # CHECK: lwl $20, -4231($15)
+0x8c 0xa8 0x16 0x2a # CHECK: lw $8, 5674($5)
+0x90 0x68 0x75 0xf3 # CHECK: lbu $8, 30195($3)
 0x94 0x53 0xa6 0xbd # CHECK: lhu $19, -22851($2)
-0x24 0x01 0x8b 0xb3 # CHECK: addiu $1, $zero, -29773
-0x24 0x00 0x8b 0x3f # CHECK: addiu $zero, $zero, -29889
+0x9b 0x80 0xb5 0x35 # CHECK: lwr $zero, -19147($gp)
+0xa1 0xd6 0xb2 0x6f # CHECK: sb $22, -19857($14)
+0xa5 0xee 0xe5 0xd0 # CHECK: sh $14, -6704($15)
+0xaa 0x6f 0x35 0x7e # CHECK: swl $15, 13694($19)
+0xaf 0xbf 0xd8 0x50 # CHECK: sw $ra, -10160($sp)
+0xb9 0xd1 0x98 0x22 # CHECK: swr $17, -26590($14)
 0xc2 0x42 0xe3 0x67 # CHECK: ll $2, -7321($18)
-0x8c 0xa8 0x16 0x2a # CHECK: lw $8, 5674($5)
 0xc7 0x50 0x27 0xf1 # CHECK: lwc1 $f16, 10225($26)
 0xc8 0xd2 0xfc 0xb7 # CHECK: lwc2 $18, -841($6)
+0xcd 0x28 0x23 0xce # CHECK: lwc3  $8, 9166($9)
 0xcf 0x4a 0x81 0xf7 # CHECK: lwc3 $10, -32265($26)
-0x89 0xf4 0xef 0x79 # CHECK: lwl $20, -4231($15)
-0x9b 0x80 0xb5 0x35 # CHECK: lwr $zero, -19147($gp)
-0x44 0x07 0xd8 0x00 # CHECK: mfc1 $7, $f27
-0x00 0x00 0x98 0x10 # CHECK: mfhi $19
-0x00 0x00 0xe8 0x10 # CHECK: mfhi $sp
-0x00 0x00 0x88 0x12 # CHECK: mflo $17
-0x46 0x20 0x75 0x06 # CHECK: mov.d $f20, $f14
-0x46 0x00 0xd8 0x86 # CHECK: mov.s $f2, $f27
-0x00 0x80 0xf0 0x21 # CHECK: move $fp, $4
-0x00 0xc0 0xc8 0x21 # CHECK: move $25, $6
-0x44 0x9e 0x48 0x00 # CHECK: mtc1 $fp, $f9
-0x02 0x20 0x00 0x11 # CHECK: mthi $17
-0x03 0xa0 0x00 0x13 # CHECK: mtlo $sp
-0x03 0x20 0x00 0x13 # CHECK: mtlo $25
-0x46 0x30 0xa5 0x02 # CHECK: mul.d $f20, $f20, $f16
-0x46 0x02 0x57 0x82 # CHECK: mul.s $f30, $f10, $f2
-0x03 0xb4 0x00 0x18 # CHECK: mult $sp, $20
-0x03 0xa2 0x00 0x18 # CHECK: mult $sp, $2
-0x03 0x9a 0x00 0x19 # CHECK: multu $gp, $26
-0x01 0x32 0x00 0x19 # CHECK: multu $9, $18
-0x00 0x02 0x10 0x23 # CHECK: negu $2, $2
-0x00 0x03 0x10 0x23 # CHECK: negu $2, $3
-0x46 0x20 0x96 0x87 # CHECK: neg.d $f26, $f18
-0x46 0x00 0x78 0x47 # CHECK: neg.s $f1, $f15
-0x00 0x00 0x00 0x00 # CHECK: nop
-0x00 0x07 0x38 0x27 # CHECK: nor $7, $zero, $7
-0x02 0x1d 0x60 0x25 # CHECK: or $12, $16, $sp
-0x34 0x42 0x00 0x04 # CHECK: ori $2, $2, 4
-0x46 0x20 0x21 0x8c # CHECK: round.w.d $f6, $f4
-0x46 0x00 0xe6 0xcc # CHECK: round.w.s $f27, $f28
-0xa1 0xd6 0xb2 0x6f # CHECK: sb $22, -19857($14)
+0xd6 0x0a 0x40 0x07 # CHECK: ldc1 $f10, 16391($16)
+0xd8 0x28 0xad 0x43 # CHECK: ldc2 $8, -21181($1)
+0xdd 0x07 0x23 0xca # CHECK: ldc3  $7, 9162($8)
+0xde 0x3d 0x90 0x1b # CHECK: ldc3 $29, -28645($17)
 0xe2 0x6f 0x49 0xd8 # CHECK: sc $15, 18904($19)
-0xf5 0xbe 0x77 0x6e # CHECK: sdc1 $f30, 30574($13)
-0xfa 0x54 0x5a 0x75 # CHECK: sdc2 $20, 23157($18)
-0xfd 0x4c 0x16 0xcb # CHECK: sdc3 $12, 5835($10)
-0xa5 0xee 0xe5 0xd0 # CHECK: sh $14, -6704($15)
-0x00 0x07 0x3c 0x80 # CHECK: sll $7, $7, 18
-0x00 0x00 0x3c 0x80 # CHECK: sll $7, $zero, 18
-0x01 0x20 0x38 0x04 # CHECK: sllv $7, $zero, $9
-0x01 0x20 0x38 0x04 # CHECK: sllv $7, $zero, $9
-0x01 0x7b 0xb8 0x2a # CHECK: slt $23, $11, $27
-0x29 0x51 0x25 0x11 # CHECK: slti $17, $10, 9489
-0x2f 0x39 0xc3 0x55 # CHECK: sltiu $25, $25, -15531
-0x02 0xab 0xa0 0x2b # CHECK: sltu $20, $21, $11
-0x2f 0x38 0xc3 0x55 # CHECK: sltiu $24, $25, -15531
-0x46 0x20 0xb4 0x04 # CHECK: sqrt.d $f16, $f22
-0x46 0x00 0x08 0x04 # CHECK: sqrt.s $f0, $f1
-0x00 0x11 0x8b 0xc3 # CHECK: sra $17, $17, 15
-0x00 0x17 0x8b 0xc3 # CHECK: sra $17, $23, 15
-0x03 0xb7 0x88 0x07 # CHECK: srav $17, $23, $sp
-0x03 0xb7 0x88 0x07 # CHECK: srav $17, $23, $sp
-0x00 0x02 0x11 0xc2 # CHECK: srl $2, $2, 7
-0x00 0x02 0x11 0xc2 # CHECK: srl $2, $2, 7
-0x00 0x94 0xc8 0x06 # CHECK: srlv $25, $20, $4
-0x00 0x94 0xc8 0x06 # CHECK: srlv $25, $20, $4
-0x00 0x00 0x00 0x40 # CHECK: ssnop
-0x02 0x6c 0xb0 0x22 # CHECK: sub $22, $19, $12
-0x22 0x36 0x0c 0x36 # CHECK: addi $22, $17, 3126
-0x21 0xad 0xe6 0x90 # CHECK: addi $13, $13, -6512
-0x46 0x30 0x14 0x81 # CHECK: sub.d $f18, $f2, $f16
-0x46 0x16 0xb5 0xc1 # CHECK: sub.s $f23, $f22, $f22
-0x02 0xd6 0xe8 0x23 # CHECK: subu $sp, $22, $22
-0xaf 0xbf 0xd8 0x50 # CHECK: sw $ra, -10160($sp)
 0xe7 0x06 0xde 0xef # CHECK: swc1 $f6, -8465($24)
 0xea 0x19 0x61 0x30 # CHECK: swc2 $25, 24880($16)
+0xec 0xe6 0x23 0xc6 # CHECK: swc3  $6, 9158($7)
 0xef 0x4a 0x81 0xf7 # CHECK: swc3 $10, -32265($26)
-0xaa 0x6f 0x35 0x7e # CHECK: swl $15, 13694($19)
-0xb9 0xd1 0x98 0x22 # CHECK: swr $17, -26590($14)
-0x00 0x03 0x00 0x34 # CHECK: teq $zero, $3
-0x00 0xa7 0x9b 0x34 # CHECK: teq $5, $7, 620
-0x06 0xac 0xbb 0xa0 # CHECK: teqi $21, 48032
-0x00 0xea 0x00 0x30 # CHECK: tge $7, $10
-0x00 0xb3 0x55 0x30 # CHECK: tge $5, $19, 340
-0x06 0x28 0x13 0xa1 # CHECK: tgei $17, 5025
-0x07 0xa9 0x90 0x33 # CHECK: tgeiu $sp, 36915
-0x02 0xdc 0x00 0x31 # CHECK: tgeu $22, $gp
-0x02 0x8e 0x5e 0xf1 # CHECK: tgeu $20, $14, 379
-0x42 0x00 0x00 0x08 # CHECK: tlbp
-0x42 0x00 0x00 0x01 # CHECK: tlbr
-0x42 0x00 0x00 0x02 # CHECK: tlbwi
-0x42 0x00 0x00 0x06 # CHECK: tlbwr
-0x01 0xed 0x00 0x32 # CHECK: tlt $15, $13
-0x00 0x53 0x21 0x72 # CHECK: tlt $2, $19, 133
-0x05 0xca 0xad 0xbd # CHECK: tlti $14, 44477
-0x07 0xeb 0xec 0x2c # CHECK: tltiu $ra, 60460
-0x01 0x70 0x00 0x33 # CHECK: tltu $11, $16
-0x02 0x1d 0xfe 0x33 # CHECK: tltu $16, $sp, 1016
-0x00 0xd1 0x00 0x36 # CHECK: tne $6, $17
-0x00 0xe8 0xdd 0x76 # CHECK: tne $7, $8, 885
-0x05 0x8e 0x8c 0x31 # CHECK: tnei $12, 35889
-0x46 0x20 0x75 0x8d # CHECK: trunc.w.d $f22, $f14
-0x46 0x00 0xf7 0x0d # CHECK: trunc.w.s $f28, $f30
-0x00 0x9e 0x90 0x26 # CHECK: xor $18, $4, $fp
+0xf5 0xbe 0x77 0x6e # CHECK: sdc1 $f30, 30574($13)
+0xfa 0x54 0x5a 0x75 # CHECK: sdc2 $20, 23157($18)
+0xfc 0xc5 0x23 0xc2 # CHECK: sdc3  $5, 9154($6)
+0xfd 0x4c 0x16 0xcb # CHECK: sdc3 $12, 5835($10)
diff --git a/test/MC/Disassembler/Mips/mips3/valid-mips3.txt b/test/MC/Disassembler/Mips/mips3/valid-mips3.txt
index cb602a35f7e..2a38b19092f 100644
--- a/test/MC/Disassembler/Mips/mips3/valid-mips3.txt
+++ b/test/MC/Disassembler/Mips/mips3/valid-mips3.txt
@@ -1,209 +1,209 @@
 # RUN: llvm-mc %s -triple=mips64-unknown-linux -disassemble -mcpu=mips3 | FileCheck %s
 # CHECK: .text
-0x46 0x20 0xc1 0x85 # CHECK: abs.d $f6, $f24
-0x46 0x00 0x82 0x45 # CHECK: abs.s $f9, $f16
+0x00 0x00 0x00 0x00 # CHECK: nop
+0x00 0x00 0x00 0x40 # CHECK: ssnop
+0x00 0x00 0x00 0xc0 # CHECK: ehb
+0x00 0x00 0x04 0xb8 # CHECK: dsll $zero, $zero, 18
+0x00 0x00 0x04 0xbc # CHECK: dsll32 $zero, $zero, 18
+0x00 0x00 0x04 0xbc # CHECK: dsll32 $zero, $zero, 18
+0x00 0x00 0x3c 0x80 # CHECK: sll $7, $zero, 18
+0x00 0x00 0x88 0x12 # CHECK: mflo $17
+0x00 0x00 0x98 0x10 # CHECK: mfhi $19
+0x00 0x00 0xe8 0x10 # CHECK: mfhi $sp
+0x00 0x02 0x10 0x23 # CHECK: negu $2, $2
+0x00 0x02 0x11 0xc2 # CHECK: srl $2, $2, 7
+0x00 0x02 0x11 0xc2 # CHECK: srl $2, $2, 7
+0x00 0x03 0x00 0x34 # CHECK: teq $zero, $3
+0x00 0x03 0x10 0x23 # CHECK: negu $2, $3
+0x00 0x03 0x10 0x2e # CHECK: dneg $2, $3
+0x00 0x03 0x10 0x2f # CHECK: dnegu $2, $3
+0x00 0x06 0x9d 0xfa # CHECK: dsrl $19, $6, 23
+0x00 0x06 0x9d 0xfe # CHECK: dsrl32 $19, $6, 23
+0x00 0x07 0x38 0x27 # CHECK: nor $7, $zero, $7
+0x00 0x07 0x3c 0x80 # CHECK: sll $7, $7, 18
+0x00 0x11 0x8b 0xc3 # CHECK: sra $17, $17, 15
+0x00 0x12 0xe2 0xbb # CHECK: dsra $gp, $18, 10
+0x00 0x12 0xe2 0xbf # CHECK: dsra32 $gp, $18, 10
+0x00 0x13 0x9d 0xfa # CHECK: dsrl $19, $19, 23
+0x00 0x13 0x9d 0xfe # CHECK: dsrl32 $19, $19, 23
+0x00 0x14 0x04 0xb8 # CHECK: dsll $zero, $20, 18
+0x00 0x17 0x8b 0xc3 # CHECK: sra $17, $23, 15
+0x00 0x1c 0xe2 0xbb # CHECK: dsra $gp, $gp, 10
+0x00 0x1c 0xe2 0xbf # CHECK: dsra32 $gp, $gp, 10
+0x00 0x3f 0x98 0x2c # CHECK: dadd $19, $1, $ra
+0x00 0x3f 0x98 0x2d # CHECK: daddu $19, $1, $ra
+0x00 0x4c 0xb8 0x24 # CHECK: and $23, $2, $12
+0x00 0x53 0x21 0x72 # CHECK: tlt $2, $19, 133
+0x00 0x80 0xf0 0x21 # CHECK: move $fp, $4
+0x00 0x86 0x48 0x21 # CHECK: addu $9, $4, $6
+0x00 0x94 0xc8 0x06 # CHECK: srlv $25, $20, $4
+0x00 0x94 0xc8 0x06 # CHECK: srlv $25, $20, $4
+0x00 0x9e 0x90 0x26 # CHECK: xor $18, $4, $fp
+0x00 0xa6 0x00 0x1d # CHECK: dmultu $5, $6
+0x00 0xa7 0x9b 0x34 # CHECK: teq $5, $7, 620
+0x00 0xb3 0x55 0x30 # CHECK: tge $5, $19, 340
+0x00 0xba 0x28 0x2f # CHECK: dsubu $5, $5, $26
+0x00 0xc0 0xc8 0x21 # CHECK: move $25, $6
+0x00 0xd1 0x00 0x36 # CHECK: tne $6, $17
+0x00 0xe8 0xdd 0x76 # CHECK: tne $7, $8, 885
+0x00 0xea 0x00 0x30 # CHECK: tge $7, $10
+0x01 0x20 0x38 0x04 # CHECK: sllv $7, $zero, $9
+0x01 0x20 0x38 0x04 # CHECK: sllv $7, $zero, $9
+0x01 0x32 0x00 0x19 # CHECK: multu $9, $18
+0x01 0x70 0x00 0x33 # CHECK: tltu $11, $16
+0x01 0x7b 0xb8 0x2a # CHECK: slt $23, $11, $27
+0x01 0x94 0x00 0x14 # CHECK: dsllv $zero, $20, $12
+0x01 0x94 0x00 0x14 # CHECK: dsllv $zero, $20, $12
+0x01 0xed 0x00 0x32 # CHECK: tlt $15, $13
+0x02 0x11 0x00 0x1f # CHECK: ddivu $zero, $16, $17
+0x02 0x1d 0x60 0x25 # CHECK: or $12, $16, $sp
+0x02 0x1d 0xfe 0x33 # CHECK: tltu $16, $sp, 1016
+0x02 0x20 0x00 0x11 # CHECK: mthi $17
 0x02 0x45 0xb8 0x20 # CHECK: add $23, $18, $5
+0x02 0x6c 0xb0 0x22 # CHECK: sub $22, $19, $12
+0x02 0x72 0xe0 0x17 # CHECK: dsrav $gp, $18, $19
+0x02 0x72 0xe0 0x17 # CHECK: dsrav $gp, $18, $19
+0x02 0x86 0x98 0x16 # CHECK: dsrlv $19, $6, $20
+0x02 0x86 0x98 0x16 # CHECK: dsrlv $19, $6, $20
+0x02 0x8e 0x5e 0xf1 # CHECK: tgeu $20, $14, 379
+0x02 0xab 0xa0 0x2b # CHECK: sltu $20, $21, $11
+0x02 0xc8 0x38 0x2e # CHECK: dsub $7, $22, $8
+0x02 0xd6 0xe8 0x23 # CHECK: subu $sp, $22, $22
+0x02 0xdc 0x00 0x31 # CHECK: tgeu $22, $gp
+0x02 0xe9 0x00 0x1c # CHECK: dmult $23, $9
+0x03 0x20 0x00 0x13 # CHECK: mtlo $25
+0x03 0x2b 0x00 0x1a # CHECK: div $zero, $25, $11
+0x03 0x2f 0x00 0x1b # CHECK: divu $zero, $25, $15
+0x03 0x53 0x00 0x1e # CHECK: ddiv $zero, $26, $19
+0x03 0x9a 0x00 0x19 # CHECK: multu $gp, $26
+0x03 0xa0 0x00 0x13 # CHECK: mtlo $sp
+0x03 0xa2 0x00 0x18 # CHECK: mult $sp, $2
+0x03 0xb4 0x00 0x18 # CHECK: mult $sp, $20
+0x03 0xb7 0x88 0x07 # CHECK: srav $17, $23, $sp
+0x03 0xb7 0x88 0x07 # CHECK: srav $17, $23, $sp
+0x04 0x11 0x14 0x9b # CHECK: bal 21104
+0x04 0x83 0xf9 0x4d # CHECK: bgezl $4, -6856
+0x04 0xd0 0x14 0x9b # CHECK: bltzal $6, 21104
+0x04 0xd2 0x00 0x7a # CHECK: bltzall $6, 492
+0x05 0x8e 0x8c 0x31 # CHECK: tnei $12, 35889
+0x05 0x93 0x07 0x1f # CHECK: bgezall $12, 7296
+0x05 0xca 0xad 0xbd # CHECK: tlti $14, 44477
+0x06 0x22 0xf6 0x45 # CHECK: bltzl $17, -9960
+0x06 0x28 0x13 0xa1 # CHECK: tgei $17, 5025
+0x06 0xac 0xbb 0xa0 # CHECK: teqi $21, 48032
+0x07 0xa9 0x90 0x33 # CHECK: tgeiu $sp, 36915
+0x07 0xeb 0xec 0x2c # CHECK: tltiu $ra, 60460
+0x21 0x08 0xff 0xfe # CHECK: addi $8, $8, -2
+0x21 0x2d 0x66 0xd2 # CHECK: addi $13, $9, 26322
+0x21 0xad 0xe6 0x90 # CHECK: addi $13, $13, -6512
 0x21 0xc9 0x3b 0x48 # CHECK: addi $9, $14, 15176
+0x22 0x36 0x0c 0x36 # CHECK: addi $22, $17, 3126
 0x23 0x18 0xe3 0xe7 # CHECK: addi $24, $24, -7193
-0x46 0x3c 0x30 0x00 # CHECK: add.d $f0, $f6, $f28
-0x46 0x18 0xaa 0x00 # CHECK: add.s $f8, $f21, $f24
-0x21 0x2d 0x66 0xd2 # CHECK: addi $13, $9, 26322
-0x21 0x08 0xff 0xfe # CHECK: addi $8, $8, -2
-0x00 0x86 0x48 0x21 # CHECK: addu $9, $4, $6
+0x24 0x00 0x8b 0x3f # CHECK: addiu $zero, $zero, -29889
+0x24 0x01 0x8b 0xb3 # CHECK: addiu $1, $zero, -29773
 0x25 0x29 0x00 0x0a # CHECK: addiu $9, $9, 10
-0x00 0x4c 0xb8 0x24 # CHECK: and $23, $2, $12
+0x29 0x51 0x25 0x11 # CHECK: slti $17, $10, 9489
+0x2f 0x38 0xc3 0x55 # CHECK: sltiu $24, $25, -15531
+0x2f 0x39 0xc3 0x55 # CHECK: sltiu $25, $25, -15531
 0x30 0x42 0x00 0x04 # CHECK: andi $2, $2, 4
+0x34 0x42 0x00 0x04 # CHECK: ori $2, $2, 4
+0x42 0x00 0x00 0x01 # CHECK: tlbr
+0x42 0x00 0x00 0x02 # CHECK: tlbwi
+0x42 0x00 0x00 0x06 # CHECK: tlbwr
+0x42 0x00 0x00 0x08 # CHECK: tlbp
+0x42 0x00 0x00 0x18 # CHECK: eret
+0x44 0x07 0xd8 0x00 # CHECK: mfc1 $7, $f27
+0x44 0x2c 0x68 0x00 # CHECK: dmfc1 $12, $f13
+0x44 0x51 0xa8 0x00 # CHECK: cfc1 $17, $21
+0x44 0x9e 0x48 0x00 # CHECK: mtc1 $fp, $f9
+0x44 0xb0 0x70 0x00 # CHECK: dmtc1 $16, $f14
+0x44 0xc6 0xd0 0x00 # CHECK: ctc1 $6, $26
 0x45 0x00 0x00 0x01 # CHECK: bc1f 8
-0x45 0x02 0x00 0x0c # CHECK: bc1fl 52
 0x45 0x01 0x00 0x01 # CHECK: bc1t 8
+0x45 0x02 0x00 0x0c # CHECK: bc1fl 52
 0x45 0x03 0xf7 0xf4 # CHECK: bc1tl -8236
-0x04 0x11 0x14 0x9b # CHECK: bal 21104
-0x00 0x00 0x00 0x00 # CHECK: nop
-0x04 0xd0 0x14 0x9b # CHECK: bltzal $6, 21104
-0x00 0x00 0x00 0x00 # CHECK: nop
-0x51 0xd3 0x0c 0x40 # CHECK: beql $14, $19, 12548
-0x05 0x93 0x07 0x1f # CHECK: bgezall $12, 7296
-0x04 0x83 0xf9 0x4d # CHECK: bgezl $4, -6856
-0x5d 0x40 0xfc 0x59 # CHECK: bgtzl $10, -3736
-0x58 0xc0 0x02 0xe7 # CHECK: blezl $6, 2976
-0x04 0xd2 0x00 0x7a # CHECK: bltzall $6, 492
-0x06 0x22 0xf6 0x45 # CHECK: bltzl $17, -9960
-0x57 0x94 0x04 0xfc # CHECK: bnel $gp, $20, 5108
-0xbc 0xa1 0x00 0x08 # CHECK: cache 1, 8($5)
-0x46 0x3c 0xe0 0x3b # CHECK: c.ngl.d $f28, $f28
-0x46 0x30 0x00 0x39 # CHECK: c.ngle.d $f0, $f16
-0x46 0x20 0xf0 0x38 # CHECK: c.sf.d $f30, $f0
-0x46 0x16 0x70 0x38 # CHECK: c.sf.s $f14, $f22
-0x46 0x20 0x18 0x4a # CHECK: ceil.l.d $f1, $f3
+0x46 0x00 0x08 0x04 # CHECK: sqrt.s $f0, $f1
+0x46 0x00 0x2b 0x0b # CHECK: floor.l.s $f12, $f5
+0x46 0x00 0x2e 0x48 # CHECK: round.l.s $f25, $f5
+0x46 0x00 0x4a 0x0f # CHECK: floor.w.s $f8, $f9
 0x46 0x00 0x6c 0x8a # CHECK: ceil.l.s $f18, $f13
-0x46 0x20 0xc2 0xce # CHECK: ceil.w.d $f11, $f24
+0x46 0x00 0x78 0x47 # CHECK: neg.s $f1, $f15
+0x46 0x00 0x82 0x45 # CHECK: abs.s $f9, $f16
 0x46 0x00 0xa1 0x8e # CHECK: ceil.w.s $f6, $f20
-0x44 0x51 0xa8 0x00 # CHECK: cfc1 $17, $21
-0x44 0xc6 0xd0 0x00 # CHECK: ctc1 $6, $26
+0x46 0x00 0xc5 0x24 # CHECK: cvt.w.s $f20, $f24
+0x46 0x00 0xd8 0x86 # CHECK: mov.s $f2, $f27
 0x46 0x00 0xe5 0xa1 # CHECK: cvt.d.s $f22, $f28
-0x46 0x80 0x5e 0xa1 # CHECK: cvt.d.w $f26, $f11
-0x46 0xa0 0x81 0x21 # CHECK: cvt.d.l $f4, $f16
-0x46 0x20 0x7e 0x25 # CHECK: cvt.l.d $f24, $f15
+0x46 0x00 0xe6 0xcc # CHECK: round.w.s $f27, $f28
 0x46 0x00 0xea 0xe5 # CHECK: cvt.l.s $f11, $f29
-0x46 0xa0 0xf3 0xe0 # CHECK: cvt.s.l $f15, $f30
+0x46 0x00 0xf7 0x0d # CHECK: trunc.w.s $f28, $f30
+0x46 0x00 0xff 0x09 # CHECK: trunc.l.s $f28, $f31
+0x46 0x02 0x57 0x82 # CHECK: mul.s $f30, $f10, $f2
+0x46 0x0f 0x29 0x03 # CHECK: div.s $f4, $f5, $f15
+0x46 0x16 0x70 0x38 # CHECK: c.sf.s $f14, $f22
+0x46 0x16 0xb5 0xc1 # CHECK: sub.s $f23, $f22, $f22
+0x46 0x18 0xaa 0x00 # CHECK: add.s $f8, $f21, $f24
+0x46 0x20 0x0b 0x08 # CHECK: round.l.d $f12, $f1
+0x46 0x20 0x18 0x4a # CHECK: ceil.l.d $f1, $f3
+0x46 0x20 0x21 0x8c # CHECK: round.w.d $f6, $f4
+0x46 0x20 0x3e 0x8b # CHECK: floor.l.d $f26, $f7
 0x46 0x20 0x46 0xa0 # CHECK: cvt.s.d $f26, $f8
-0x46 0x80 0x7d 0xa0 # CHECK: cvt.s.w $f22, $f15
+0x46 0x20 0x53 0x8f # CHECK: floor.w.d $f14, $f10
+0x46 0x20 0x75 0x06 # CHECK: mov.d $f20, $f14
 0x46 0x20 0x75 0x24 # CHECK: cvt.w.d $f20, $f14
-0x46 0x00 0xc5 0x24 # CHECK: cvt.w.s $f20, $f24
-0x00 0x3f 0x98 0x2c # CHECK: dadd $19, $1, $ra
+0x46 0x20 0x75 0x8d # CHECK: trunc.w.d $f22, $f14
+0x46 0x20 0x7e 0x25 # CHECK: cvt.l.d $f24, $f15
+0x46 0x20 0x96 0x87 # CHECK: neg.d $f26, $f18
+0x46 0x20 0xb4 0x04 # CHECK: sqrt.d $f16, $f22
+0x46 0x20 0xbd 0xc9 # CHECK: trunc.l.d $f23, $f23
+0x46 0x20 0xc1 0x85 # CHECK: abs.d $f6, $f24
+0x46 0x20 0xc2 0xce # CHECK: ceil.w.d $f11, $f24
+0x46 0x20 0xf0 0x38 # CHECK: c.sf.d $f30, $f0
+0x46 0x30 0x00 0x39 # CHECK: c.ngle.d $f0, $f16
+0x46 0x30 0x14 0x81 # CHECK: sub.d $f18, $f2, $f16
+0x46 0x30 0xa5 0x02 # CHECK: mul.d $f20, $f20, $f16
+0x46 0x3a 0xa7 0x03 # CHECK: div.d $f28, $f20, $f26
+0x46 0x3c 0x30 0x00 # CHECK: add.d $f0, $f6, $f28
+0x46 0x3c 0xe0 0x3b # CHECK: c.ngl.d $f28, $f28
+0x46 0x80 0x5e 0xa1 # CHECK: cvt.d.w $f26, $f11
+0x46 0x80 0x7d 0xa0 # CHECK: cvt.s.w $f22, $f15
+0x46 0xa0 0x81 0x21 # CHECK: cvt.d.l $f4, $f16
+0x46 0xa0 0xf3 0xe0 # CHECK: cvt.s.l $f15, $f30
+0x51 0xd3 0x0c 0x40 # CHECK: beql $14, $19, 12548
+0x57 0x94 0x04 0xfc # CHECK: bnel $gp, $20, 5108
+0x58 0xc0 0x02 0xe7 # CHECK: blezl $6, 2976
+0x5d 0x40 0xfc 0x59 # CHECK: bgtzl $10, -3736
 0x62 0x9d 0x93 0xc7 # CHECK: daddi $sp, $20, -27705
-0x63 0xbd 0x93 0xc7 # CHECK: daddi $sp, $sp, -27705
 0x62 0x9d 0x93 0xc7 # CHECK: daddi $sp, $20, -27705
 0x62 0x9d 0x93 0xc7 # CHECK: daddi $sp, $20, -27705
 0x63 0xbd 0x93 0xc7 # CHECK: daddi $sp, $sp, -27705
-0x66 0xda 0xee 0x16 # CHECK: daddiu $26, $22, -4586
-0x00 0x3f 0x98 0x2d # CHECK: daddu $19, $1, $ra
+0x63 0xbd 0x93 0xc7 # CHECK: daddi $sp, $sp, -27705
 0x64 0x58 0x46 0x9f # CHECK: daddiu $24, $2, 18079
 0x66 0x73 0x69 0x3f # CHECK: daddiu $19, $19, 26943
-0x03 0x53 0x00 0x1e # CHECK: ddiv $zero, $26, $19
-0x02 0x11 0x00 0x1f # CHECK: ddivu $zero, $16, $17
-0x03 0x2b 0x00 0x1a # CHECK: div $zero, $25, $11
-0x46 0x3a 0xa7 0x03 # CHECK: div.d $f28, $f20, $f26
-0x46 0x0f 0x29 0x03 # CHECK: div.s $f4, $f5, $f15
-0x03 0x2f 0x00 0x1b # CHECK: divu $zero, $25, $15
-0x44 0x2c 0x68 0x00 # CHECK: dmfc1 $12, $f13
-0x44 0xb0 0x70 0x00 # CHECK: dmtc1 $16, $f14
-0x02 0xe9 0x00 0x1c # CHECK: dmult $23, $9
-0x00 0xa6 0x00 0x1d # CHECK: dmultu $5, $6
-0x00 0x00 0x04 0xb8 # CHECK: dsll $zero, $zero, 18
-0x00 0x14 0x04 0xb8 # CHECK: dsll $zero, $20, 18
-0x01 0x94 0x00 0x14 # CHECK: dsllv $zero, $20, $12
-0x00 0x00 0x04 0xbc # CHECK: dsll32 $zero, $zero, 18
-0x00 0x00 0x04 0xbc # CHECK: dsll32 $zero, $zero, 18
-0x01 0x94 0x00 0x14 # CHECK: dsllv $zero, $20, $12
-0x00 0x1c 0xe2 0xbb # CHECK: dsra $gp, $gp, 10
-0x00 0x12 0xe2 0xbb # CHECK: dsra $gp, $18, 10
-0x02 0x72 0xe0 0x17 # CHECK: dsrav $gp, $18, $19
-0x00 0x1c 0xe2 0xbf # CHECK: dsra32 $gp, $gp, 10
-0x00 0x12 0xe2 0xbf # CHECK: dsra32 $gp, $18, 10
-0x02 0x72 0xe0 0x17 # CHECK: dsrav $gp, $18, $19
-0x00 0x13 0x9d 0xfa # CHECK: dsrl $19, $19, 23
-0x00 0x06 0x9d 0xfa # CHECK: dsrl $19, $6, 23
-0x02 0x86 0x98 0x16 # CHECK: dsrlv $19, $6, $20
-0x00 0x13 0x9d 0xfe # CHECK: dsrl32 $19, $19, 23
-0x00 0x06 0x9d 0xfe # CHECK: dsrl32 $19, $6, 23
-0x02 0x86 0x98 0x16 # CHECK: dsrlv $19, $6, $20
-0x02 0xc8 0x38 0x2e # CHECK: dsub $7, $22, $8
-0x00 0xba 0x28 0x2f # CHECK: dsubu $5, $5, $26
-0x00 0x00 0x00 0xc0 # CHECK: ehb
-0x42 0x00 0x00 0x18 # CHECK: eret
-0x46 0x20 0x53 0x8f # CHECK: floor.w.d $f14, $f10
-0x46 0x00 0x4a 0x0f # CHECK: floor.w.s $f8, $f9
-0x46 0x20 0x3e 0x8b # CHECK: floor.l.d $f26, $f7
-0x46 0x00 0x2b 0x0b # CHECK: floor.l.s $f12, $f5
+0x66 0xda 0xee 0x16 # CHECK: daddiu $26, $22, -4586
 0x81 0x58 0xc7 0x4d # CHECK: lb $24, -14515($10)
-0x90 0x68 0x75 0xf3 # CHECK: lbu $8, 30195($3)
-0xd6 0x0a 0x40 0x07 # CHECK: ldc1 $f10, 16391($16)
-0xd8 0x28 0xad 0x43 # CHECK: ldc2 $8, -21181($1)
 0x86 0xab 0xde 0x94 # CHECK: lh $11, -8556($21)
+0x89 0xf4 0xef 0x79 # CHECK: lwl $20, -4231($15)
+0x8c 0xa8 0x16 0x2a # CHECK: lw $8, 5674($5)
+0x90 0x68 0x75 0xf3 # CHECK: lbu $8, 30195($3)
 0x94 0x53 0xa6 0xbd # CHECK: lhu $19, -22851($2)
-0x24 0x01 0x8b 0xb3 # CHECK: addiu $1, $zero, -29773
-0x24 0x00 0x8b 0x3f # CHECK: addiu $zero, $zero, -29889
+0x9b 0x80 0xb5 0x35 # CHECK: lwr $zero, -19147($gp)
+0xa1 0xd6 0xb2 0x6f # CHECK: sb $22, -19857($14)
+0xa5 0xee 0xe5 0xd0 # CHECK: sh $14, -6704($15)
+0xaa 0x6f 0x35 0x7e # CHECK: swl $15, 13694($19)
+0xaf 0xbf 0xd8 0x50 # CHECK: sw $ra, -10160($sp)
+0xb3 0xc7 0xae 0x1f # CHECK: sdl $7, -20961($fp)
+0xb5 0x8b 0xb0 0x39 # CHECK: sdr $11, -20423($12)
+0xb9 0xd1 0x98 0x22 # CHECK: swr $17, -26590($14)
+0xbc 0xa1 0x00 0x08 # CHECK: cache 1, 8($5)
 0xc2 0x42 0xe3 0x67 # CHECK: ll $2, -7321($18)
-0x8c 0xa8 0x16 0x2a # CHECK: lw $8, 5674($5)
 0xc7 0x50 0x27 0xf1 # CHECK: lwc1 $f16, 10225($26)
 0xc8 0xd2 0xfc 0xb7 # CHECK: lwc2 $18, -841($6)
-0x89 0xf4 0xef 0x79 # CHECK: lwl $20, -4231($15)
-0x9b 0x80 0xb5 0x35 # CHECK: lwr $zero, -19147($gp)
-0x44 0x07 0xd8 0x00 # CHECK: mfc1 $7, $f27
-0x00 0x00 0x98 0x10 # CHECK: mfhi $19
-0x00 0x00 0xe8 0x10 # CHECK: mfhi $sp
-0x00 0x00 0x88 0x12 # CHECK: mflo $17
-0x46 0x20 0x75 0x06 # CHECK: mov.d $f20, $f14
-0x46 0x00 0xd8 0x86 # CHECK: mov.s $f2, $f27
-0x00 0x80 0xf0 0x21 # CHECK: move $fp, $4
-0x00 0xc0 0xc8 0x21 # CHECK: move $25, $6
-0x44 0x9e 0x48 0x00 # CHECK: mtc1 $fp, $f9
-0x02 0x20 0x00 0x11 # CHECK: mthi $17
-0x03 0xa0 0x00 0x13 # CHECK: mtlo $sp
-0x03 0x20 0x00 0x13 # CHECK: mtlo $25
-0x46 0x30 0xa5 0x02 # CHECK: mul.d $f20, $f20, $f16
-0x46 0x02 0x57 0x82 # CHECK: mul.s $f30, $f10, $f2
-0x03 0xb4 0x00 0x18 # CHECK: mult $sp, $20
-0x03 0xa2 0x00 0x18 # CHECK: mult $sp, $2
-0x03 0x9a 0x00 0x19 # CHECK: multu $gp, $26
-0x01 0x32 0x00 0x19 # CHECK: multu $9, $18
-0x00 0x02 0x10 0x23 # CHECK: negu $2, $2
-0x00 0x03 0x10 0x23 # CHECK: negu $2, $3
-0x46 0x20 0x96 0x87 # CHECK: neg.d $f26, $f18
-0x46 0x00 0x78 0x47 # CHECK: neg.s $f1, $f15
-0x00 0x00 0x00 0x00 # CHECK: nop
-0x00 0x07 0x38 0x27 # CHECK: nor $7, $zero, $7
-0x02 0x1d 0x60 0x25 # CHECK: or $12, $16, $sp
-0x34 0x42 0x00 0x04 # CHECK: ori $2, $2, 4
-0x46 0x20 0x0b 0x08 # CHECK: round.l.d $f12, $f1
-0x46 0x00 0x2e 0x48 # CHECK: round.l.s $f25, $f5
-0x46 0x20 0x21 0x8c # CHECK: round.w.d $f6, $f4
-0x46 0x00 0xe6 0xcc # CHECK: round.w.s $f27, $f28
-0xa1 0xd6 0xb2 0x6f # CHECK: sb $22, -19857($14)
+0xd6 0x0a 0x40 0x07 # CHECK: ldc1 $f10, 16391($16)
+0xd8 0x28 0xad 0x43 # CHECK: ldc2 $8, -21181($1)
 0xe2 0x6f 0x49 0xd8 # CHECK: sc $15, 18904($19)
+0xe7 0x06 0xde 0xef # CHECK: swc1 $f6, -8465($24)
+0xea 0x19 0x61 0x30 # CHECK: swc2 $25, 24880($16)
 0xf3 0xaf 0xdf 0xcd # CHECK: scd $15, -8243($sp)
-0xfd 0x4c 0x16 0xcb # CHECK: sd $12, 5835($10)
-0xb3 0xc7 0xae 0x1f # CHECK: sdl $7, -20961($fp)
-0xb5 0x8b 0xb0 0x39 # CHECK: sdr $11, -20423($12)
 0xf5 0xbe 0x77 0x6e # CHECK: sdc1 $f30, 30574($13)
 0xfa 0x54 0x5a 0x75 # CHECK: sdc2 $20, 23157($18)
-0xa5 0xee 0xe5 0xd0 # CHECK: sh $14, -6704($15)
-0x00 0x07 0x3c 0x80 # CHECK: sll $7, $7, 18
-0x00 0x00 0x3c 0x80 # CHECK: sll $7, $zero, 18
-0x01 0x20 0x38 0x04 # CHECK: sllv $7, $zero, $9
-0x01 0x20 0x38 0x04 # CHECK: sllv $7, $zero, $9
-0x01 0x7b 0xb8 0x2a # CHECK: slt $23, $11, $27
-0x29 0x51 0x25 0x11 # CHECK: slti $17, $10, 9489
-0x2f 0x39 0xc3 0x55 # CHECK: sltiu $25, $25, -15531
-0x02 0xab 0xa0 0x2b # CHECK: sltu $20, $21, $11
-0x2f 0x38 0xc3 0x55 # CHECK: sltiu $24, $25, -15531
-0x46 0x20 0xb4 0x04 # CHECK: sqrt.d $f16, $f22
-0x46 0x00 0x08 0x04 # CHECK: sqrt.s $f0, $f1
-0x00 0x11 0x8b 0xc3 # CHECK: sra $17, $17, 15
-0x00 0x17 0x8b 0xc3 # CHECK: sra $17, $23, 15
-0x03 0xb7 0x88 0x07 # CHECK: srav $17, $23, $sp
-0x03 0xb7 0x88 0x07 # CHECK: srav $17, $23, $sp
-0x00 0x02 0x11 0xc2 # CHECK: srl $2, $2, 7
-0x00 0x02 0x11 0xc2 # CHECK: srl $2, $2, 7
-0x00 0x94 0xc8 0x06 # CHECK: srlv $25, $20, $4
-0x00 0x94 0xc8 0x06 # CHECK: srlv $25, $20, $4
-0x00 0x00 0x00 0x40 # CHECK: ssnop
-0x02 0x6c 0xb0 0x22 # CHECK: sub $22, $19, $12
-0x22 0x36 0x0c 0x36 # CHECK: addi $22, $17, 3126
-0x21 0xad 0xe6 0x90 # CHECK: addi $13, $13, -6512
-0x46 0x30 0x14 0x81 # CHECK: sub.d $f18, $f2, $f16
-0x46 0x16 0xb5 0xc1 # CHECK: sub.s $f23, $f22, $f22
-0x02 0xd6 0xe8 0x23 # CHECK: subu $sp, $22, $22
-0xaf 0xbf 0xd8 0x50 # CHECK: sw $ra, -10160($sp)
-0xe7 0x06 0xde 0xef # CHECK: swc1 $f6, -8465($24)
-0xea 0x19 0x61 0x30 # CHECK: swc2 $25, 24880($16)
-0xaa 0x6f 0x35 0x7e # CHECK: swl $15, 13694($19)
-0xb9 0xd1 0x98 0x22 # CHECK: swr $17, -26590($14)
-0x00 0x03 0x00 0x34 # CHECK: teq $zero, $3
-0x00 0xa7 0x9b 0x34 # CHECK: teq $5, $7, 620
-0x06 0xac 0xbb 0xa0 # CHECK: teqi $21, 48032
-0x00 0xea 0x00 0x30 # CHECK: tge $7, $10
-0x00 0xb3 0x55 0x30 # CHECK: tge $5, $19, 340
-0x06 0x28 0x13 0xa1 # CHECK: tgei $17, 5025
-0x07 0xa9 0x90 0x33 # CHECK: tgeiu $sp, 36915
-0x02 0xdc 0x00 0x31 # CHECK: tgeu $22, $gp
-0x02 0x8e 0x5e 0xf1 # CHECK: tgeu $20, $14, 379
-0x42 0x00 0x00 0x08 # CHECK: tlbp
-0x42 0x00 0x00 0x01 # CHECK: tlbr
-0x42 0x00 0x00 0x02 # CHECK: tlbwi
-0x42 0x00 0x00 0x06 # CHECK: tlbwr
-0x01 0xed 0x00 0x32 # CHECK: tlt $15, $13
-0x00 0x53 0x21 0x72 # CHECK: tlt $2, $19, 133
-0x05 0xca 0xad 0xbd # CHECK: tlti $14, 44477
-0x07 0xeb 0xec 0x2c # CHECK: tltiu $ra, 60460
-0x01 0x70 0x00 0x33 # CHECK: tltu $11, $16
-0x02 0x1d 0xfe 0x33 # CHECK: tltu $16, $sp, 1016
-0x00 0xd1 0x00 0x36 # CHECK: tne $6, $17
-0x00 0xe8 0xdd 0x76 # CHECK: tne $7, $8, 885
-0x05 0x8e 0x8c 0x31 # CHECK: tnei $12, 35889
-0x46 0x20 0xbd 0xc9 # CHECK: trunc.l.d $f23, $f23
-0x46 0x00 0xff 0x09 # CHECK: trunc.l.s $f28, $f31
-0x46 0x20 0x75 0x8d # CHECK: trunc.w.d $f22, $f14
-0x46 0x00 0xf7 0x0d # CHECK: trunc.w.s $f28, $f30
-0x00 0x9e 0x90 0x26 # CHECK: xor $18, $4, $fp
+0xfd 0x4c 0x16 0xcb # CHECK: sd $12, 5835($10)
diff --git a/test/MC/Disassembler/Mips/mips32.txt b/test/MC/Disassembler/Mips/mips32.txt
deleted file mode 100644
index bd4ae4daad0..00000000000
--- a/test/MC/Disassembler/Mips/mips32.txt
+++ /dev/null
@@ -1,451 +0,0 @@
-# RUN: llvm-mc --disassemble %s -triple=mips-unknown-linux | FileCheck %s
-
-# CHECK: abs.d $f12, $f14
-0x46 0x20 0x73 0x05
-
-# CHECK: abs.s $f6, $f7
-0x46 0x00 0x39 0x85
-
-# CHECK: add $9, $6, $7
-0x00 0xc7 0x48 0x20
-
-# CHECK: add.d $f8, $f12, $f14
-0x46 0x2e 0x62 0x00
-
-# CHECK: add.s $f9, $f6, $f7
-0x46 0x07 0x32 0x40
-
-# CHECK: addi $9, $6, 17767
-0x20 0xc9 0x45 0x67
-
-# CHECK: addiu $9, $6, -15001
-0x24 0xc9 0xc5 0x67
-
-# CHECK: addu $9, $6, $7
-0x00 0xc7 0x48 0x21
-
-# CHECK: and $9, $6, $7
-0x00 0xc7 0x48 0x24
-
-# CHECK: andi $9, $6, 17767
-0x30 0xc9 0x45 0x67
-
-# CHECK: b 1332
-0x10 0x00 0x01 0x4c
-
-# CHECK: bc1f 1332
-0x45 0x00 0x01 0x4c
-
-# CHECK: bc1f $fcc7, 1332
-0x45 0x1c 0x01 0x4c
-
-# CHECK: bc1t 1332
-0x45 0x01 0x01 0x4c
-
-# CHECK: bc1t $fcc7, 1332
-0x45 0x1d 0x01 0x4c
-
-# CHECK: beq $9, $6, 1332
-0x11 0x26 0x01 0x4c
-
-# CHECK: bgez  $6, 1332
-0x04 0xc1 0x01 0x4c
-
-# CHECK: bgezal  $6, 1332
-0x04 0xd1 0x01 0x4c
-
-# CHECK: bgtz  $6, 1332
-0x1c 0xc0 0x01 0x4c
-
-# CHECK: blez  $6, 1332
-0x18 0xc0 0x01 0x4c
-
-# CHECK: bne $9, $6, 1332
-0x15 0x26 0x01 0x4c
-
-# CHECK: c.eq.d $f12, $f14
-0x46 0x2e 0x60 0x32
-
-# CHECK: c.eq.s $f6, $f7
-0x46 0x07 0x30 0x32
-
-# CHECK: c.f.d $f12, $f14
-0x46 0x2e 0x60 0x30
-
-# CHECK: c.f.s $f6, $f7
-0x46 0x07 0x30 0x30
-
-# CHECK: c.le.d $f12, $f14
-0x46 0x2e 0x60 0x3e
-
-# CHECK: c.le.s $f6, $f7
-0x46 0x07 0x30 0x3e
-
-# CHECK: c.lt.d $f12, $f14
-0x46 0x2e 0x60 0x3c
-
-# CHECK: c.lt.s $f6, $f7
-0x46 0x07 0x30 0x3c
-
-# CHECK: c.nge.d $f12, $f14
-0x46 0x2e 0x60 0x3d
-
-# CHECK: c.nge.s $f6, $f7
-0x46 0x07 0x30 0x3d
-
-# CHECK: c.ngl.d $f12, $f14
-0x46 0x2e 0x60 0x3b
-
-# CHECK: c.ngl.s $f6, $f7
-0x46 0x07 0x30 0x3b
-
-# CHECK: c.ngle.d $f12, $f14
-0x46 0x2e 0x60 0x39
-
-# CHECK: c.ngle.s $f6, $f7
-0x46 0x07 0x30 0x39
-
-# CHECK: c.ngt.d $f12, $f14
-0x46 0x2e 0x60 0x3f
-
-# CHECK: c.ngt.s $f6, $f7
-0x46 0x07 0x30 0x3f
-
-# CHECK: c.ole.d $f12, $f14
-0x46 0x2e 0x60 0x36
-
-# CHECK: c.ole.s $f6, $f7
-0x46 0x07 0x30 0x36
-
-# CHECK: c.olt.d $f12, $f14
-0x46 0x2e 0x60 0x34
-
-# CHECK: c.olt.s $f6, $f7
-0x46 0x07 0x30 0x34
-
-# CHECK: c.seq.d $f12, $f14
-0x46 0x2e 0x60 0x3a
-
-# CHECK: c.seq.s $f6, $f7
-0x46 0x07 0x30 0x3a
-
-# CHECK: c.sf.d $f12, $f14
-0x46 0x2e 0x60 0x38
-
-# CHECK: c.sf.s $f6, $f7
-0x46 0x07 0x30 0x38
-
-# CHECK: c.ueq.d $f12, $f14
-0x46 0x2e 0x60 0x33
-
-# CHECK: c.ueq.s $f28, $f18
-0x46 0x12 0xe0 0x33
-
-# CHECK: c.ule.d $f12, $f14
-0x46 0x2e 0x60 0x37
-
-# CHECK: c.ule.s $f6, $f7
-0x46 0x07 0x30 0x37
-
-# CHECK: c.ult.d $f12, $f14
-0x46 0x2e 0x60 0x35
-
-# CHECK: c.ult.s $f6, $f7
-0x46 0x07 0x30 0x35
-
-# CHECK: c.un.d $f12, $f14
-0x46 0x2e 0x60 0x31
-
-# CHECK: c.un.s $f6, $f7
-0x46 0x07 0x30 0x31
-
-# CHECK: ceil.w.d $f12, $f14
-0x46 0x20 0x73 0x0e
-
-# CHECK: ceil.w.s $f6, $f7
-0x46 0x00 0x39 0x8e
-
-# CHECK: cfc1  $6, $7
-0x44 0x46 0x38 0x00
-
-# CHECK: clo  $6, $7
-0x70 0xe6 0x30 0x21
-
-# CHECK: clz  $6, $7
-0x70 0xe6 0x30 0x20
-
-# CHECK: ctc1  $6, $7
-0x44 0xc6 0x38 0x00
-
-# CHECK: cvt.d.s $f6, $f7
-0x46 0x00 0x39 0xa1
-
-# CHECK: cvt.d.w $f12, $f14
-0x46 0x80 0x73 0x21
-
-# CHECK: cvt.s.d $f12, $f14
-0x46 0x20 0x73 0x20
-
-# CHECK: cvt.s.w $f6, $f7
-0x46 0x80 0x39 0xa0
-
-# CHECK: cvt.w.d $f12, $f14
-0x46 0x20 0x73 0x24
-
-# CHECK: cvt.w.s $f6, $f7
-0x46 0x00 0x39 0xa4
-
-# CHECK: floor.w.d $f12, $f14
-0x46 0x20 0x73 0x0f
-
-# CHECK: floor.w.s $f6, $f7
-0x46 0x00 0x39 0x8f
-
-# CHECK: j 1328
-0x08 0x00 0x01 0x4c
-
-# CHECK: jal 1328
-0x0c 0x00 0x01 0x4c
-
-# CHECK: jalx 1328
-0x74 0x00 0x01 0x4c
-
-# CHECK: jalr  $7
-0x00 0xe0 0xf8 0x09
-
-# CHECK: jr  $7
-0x00 0xe0 0x00 0x08
-
-# CHECK: lb  $4, 9158($5)
-0x80 0xa4 0x23 0xc6
-
-# CHECK: lbu $4, 6($5)
-0x90 0xa4 0x00 0x06
-
-# CHECK: ldc1  $f9, 9158($7)
-0xd4 0xe9 0x23 0xc6
-
-# CHECK: lh  $4, 12($5)
-0x84 0xa4 0x00 0x0c
-
-# CHECK: lh  $4, 12($5)
-0x84 0xa4 0x00 0x0c
-
-# CHECK: ll  $9, 9158($7)
-0xc0 0xe9 0x23 0xc6
-
-# CHECK: lui  $6, 17767
-0x3c 0x06 0x45 0x67
-
-# CHECK: lw  $4, 24($5)
-0x8c 0xa4 0x00 0x18
-
-# CHECK: lwc1  $f9, 9158($7)
-0xc4 0xe9 0x23 0xc6
-
-# CHECK: lwl   $2,  3($4)
-0x88 0x82 0x00 0x03
-
-# CHECK: lwr   $3, 16($5)
-0x98 0xa3 0x00 0x10
-
-# CHECK: madd   $6,  $7
-0x70 0xc7 0x00 0x00
-
-# CHECK: maddu  $6,  $7
-0x70 0xc7 0x00 0x01
-
-# CHECK: mfc1   $6, $f7
-0x44 0x06 0x38 0x00
-
-# CHECK: mfhi  $5
-0x00 0x00 0x28 0x10
-
-# CHECK: mflo  $5
-0x00 0x00 0x28 0x12
-
-# CHECK: mov.d $f6, $f8
-0x46 0x20 0x41 0x86
-
-# CHECK: mov.s $f6, $f7
-0x46 0x00 0x39 0x86
-
-# CHECK: movf $3, $2, $fcc7
-0x00,0x5c,0x18,0x01
-
-# CHECK: movf.d $f4, $f2, $fcc7
-0x46,0x3c,0x11,0x11
-
-# CHECK: movf.s $f4, $f2, $fcc7
-0x46,0x1c,0x11,0x11
-
-# CHECK: movt $3, $2, $fcc7
-0x00,0x5d,0x18,0x01
-
-# CHECK: movt.d $f4, $f2, $fcc7
-0x46,0x3d,0x11,0x11
-
-# CHECK: movt.s $f4, $f2, $fcc7
-0x46,0x1d,0x11,0x11
-
-# CHECK: msub   $6,  $7
-0x70 0xc7 0x00 0x04
-
-# CHECK: msubu  $6,  $7
-0x70 0xc7 0x00 0x05
-
-# CHECK: mtc1   $6, $f7
-0x44 0x86 0x38 0x00
-
-# CHECK: mthi   $7
-0x00 0xe0 0x00 0x11
-
-# CHECK: mtlo   $7
-0x00 0xe0 0x00 0x13
-
-# CHECK: mul.d $f8, $f12, $f14
-0x46 0x2e 0x62 0x02
-
-# CHECK: mul.s $f9, $f6, $f7
-0x46 0x07 0x32 0x42
-
-# CHECK: mul $9,  $6,  $7
-0x70 0xc7 0x48 0x02
-
-# CHECK: mult  $3, $5
-0x00 0x65 0x00 0x18
-
-# CHECK: multu $3, $5
-0x00 0x65 0x00 0x19
-
-# CHECK: neg.d $f12, $f14
-0x46 0x20 0x73 0x07
-
-# CHECK: neg.s $f6, $f7
-0x46 0x00 0x39 0x87
-
-# CHECK: nop
-0x00 0x00 0x00 0x00
-
-# CHECK: nor $9,  $6, $7
-0x00 0xc7 0x48 0x27
-
-# CHECK: or  $3, $3, $5
-0x00 0x65 0x18 0x25
-
-# CHECK: ori $9,  $6, 17767
-0x34 0xc9 0x45 0x67
-
-# CHECK: round.w.d $f12, $f14
-0x46 0x20 0x73 0x0c
-
-# CHECK: round.w.s $f6, $f7
-0x46 0x00 0x39 0x8c
-
-# CHECK: sb  $4, 9158($5)
-0xa0 0xa4 0x23 0xc6
-
-# CHECK: sb  $4, 6($5)
-0xa0 0xa4 0x00 0x06
-
-# CHECK: sc  $9, 9158($7)
-0xe0 0xe9 0x23 0xc6
-
-# CHECK: sdc1  $f9, 9158($7)
-0xf4 0xe9 0x23 0xc6
-
-# CHECK: sh  $4, 9158($5)
-0xa4 0xa4 0x23 0xc6
-
-# CHECK: sll $4, $3, 7
-0x00 0x03 0x21 0xc0
-
-# CHECK: sllv  $2, $3, $5
-0x00 0xa3 0x10 0x04
-
-# CHECK: slt $3, $3, $5
-0x00 0x65 0x18 0x2a
-
-# CHECK: slti  $3, $3, 103
-0x28 0x63 0x00 0x67
-
-# CHECK: sltiu $3, $3, 103
-0x2c 0x63 0x00 0x67
-
-# CHECK: sltu  $3, $3, $5
-0x00 0x65 0x18 0x2b
-
-# CHECK: sqrt.d  $f12, $f14
-0x46 0x20 0x73 0x04
-
-# CHECK: sqrt.s  $f6, $f7
-0x46 0x00 0x39 0x84
-
-# CHECK: sra $4, $3, 7
-0x00 0x03 0x21 0xc3
-
-# CHECK: srav  $2, $3, $5
-0x00 0xa3 0x10 0x07
-
-# CHECK: srl $4, $3, 7
-0x00 0x03 0x21 0xc2
-
-# CHECK: srlv  $2, $3, $5
-0x00 0xa3 0x10 0x06
-
-# CHECK: sub.d $f8, $f12, $f14
-0x46 0x2e 0x62 0x01
-
-# CHECK: sub.s $f9, $f6, $f7
-0x46 0x07 0x32 0x41
-
-# CHECK: sub $9,  $6, $7
-0x00 0xc7 0x48 0x22
-
-# CHECK: subu  $4, $3, $5
-0x00 0x65 0x20 0x23
-
-# CHECK: sw  $4, 24($5)
-0xac 0xa4 0x00 0x18
-
-# CHECK: swc1  $f9, 9158($7)
-0xe4 0xe9 0x23 0xc6
-
-# CHECK: swl $4,  16($5)
-0xa8 0xa4 0x00 0x10
-
-# CHECK: swr $6, 16($7)
-0xb8 0xe6 0x00 0x10
-
-# CHECK: sync  7
-0x00 0x00 0x01 0xcf
-
-# CHECK: trunc.w.d $f12, $f14
-0x46 0x20 0x73 0x0d
-
-# CHECK: trunc.w.s $f6, $f7
-0x46 0x00 0x39 0x8d
-
-# CHECK: xor $3, $3, $5
-0x00 0x65 0x18 0x26
-
-# CHECK: xori  $9,  $6, 17767
-0x38 0xc9 0x45 0x67
-
-# CHECK: .set    push
-# CHECK: .set    mips32r2
-# CHECK: rdhwr   $5, $29
-# CHECK: .set    pop
-0x7c 0x05 0xe8 0x3b
-
-# CHECK: cache 1, 2($3)
-0xbc 0x61 0x00 0x02
-
-# CHECK: pref 3, 4($2)
-0xcc 0x43 0x00 0x04
-
-# CHECK: swc2  $9, 9158($7)
-0xe8 0xe9 0x23 0xc6
-
-# CHECK: lwc2  $8, 9162($6)
-0xc8 0xc8 0x23 0xca
diff --git a/test/MC/Disassembler/Mips/mips32/valid-mips32-el.txt b/test/MC/Disassembler/Mips/mips32/valid-mips32-el.txt
index ea209d1ebab..f2299732a80 100644
--- a/test/MC/Disassembler/Mips/mips32/valid-mips32-el.txt
+++ b/test/MC/Disassembler/Mips/mips32/valid-mips32-el.txt
@@ -86,6 +86,7 @@
 0x10 0x00 0xa3 0x98 # CHECK: lwr $3, 16($5)
 0x00 0x00 0xc7 0x70 # CHECK: madd $6, $7
 0x01 0x00 0xc7 0x70 # CHECK: maddu $6, $7
+0x01 0x78 0x08 0x40 # CHECK: mfc0 $8, $15, 1
 0x00 0x38 0x06 0x44 # CHECK: mfc1 $6, $f7
 0x10 0x28 0x00 0x00 # CHECK: mfhi $5
 0x12 0x28 0x00 0x00 # CHECK: mflo $5
@@ -93,6 +94,7 @@
 0x86 0x39 0x00 0x46 # CHECK: mov.s $f6, $f7
 0x04 0x00 0xc7 0x70 # CHECK: msub $6, $7
 0x05 0x00 0xc7 0x70 # CHECK: msubu $6, $7
+0x01 0x78 0x89 0x40 # CHECK: mtc0 $9, $15, 1
 0x00 0x38 0x86 0x44 # CHECK: mtc1 $6, $f7
 0x11 0x00 0xe0 0x00 # CHECK: mthi $7
 0x13 0x00 0xe0 0x00 # CHECK: mtlo $7
diff --git a/test/MC/Disassembler/Mips/mips32/valid-mips32.txt b/test/MC/Disassembler/Mips/mips32/valid-mips32.txt
index 45b672b2d35..09f1e56fff4 100644
--- a/test/MC/Disassembler/Mips/mips32/valid-mips32.txt
+++ b/test/MC/Disassembler/Mips/mips32/valid-mips32.txt
@@ -1,149 +1,158 @@
 # RUN: llvm-mc --disassemble %s -triple=mips-unknown-linux | FileCheck %s
-0x46 0x20 0x73 0x05 # CHECK: abs.d $f12, $f14
-0x46 0x00 0x39 0x85 # CHECK: abs.s $f6, $f7
+0x00 0x00 0x00 0x00 # CHECK: nop
+0x00 0x00 0x01 0xcf # CHECK: sync 7
+0x00 0x00 0x28 0x10 # CHECK: mfhi $5
+0x00 0x00 0x28 0x12 # CHECK: mflo $5
+0x00 0x03 0x21 0xc0 # CHECK: sll $4, $3, 7
+0x00 0x03 0x21 0xc2 # CHECK: srl $4, $3, 7
+0x00 0x03 0x21 0xc3 # CHECK: sra $4, $3, 7
+0x00 0x5c 0x18 0x01 # CHECK: movf $3, $2, $fcc7
+0x00 0x5d 0x18 0x01 # CHECK: movt $3, $2, $fcc7
+0x00 0x65 0x00 0x18 # CHECK: mult $3, $5
+0x00 0x65 0x00 0x19 # CHECK: multu $3, $5
+0x00 0x65 0x18 0x25 # CHECK: or $3, $3, $5
+0x00 0x65 0x18 0x26 # CHECK: xor $3, $3, $5
+0x00 0x65 0x18 0x2a # CHECK: slt $3, $3, $5
+0x00 0x65 0x18 0x2b # CHECK: sltu $3, $3, $5
+0x00 0x65 0x20 0x23 # CHECK: subu $4, $3, $5
+0x00 0x80 0xfc 0x09 # CHECK: jalr.hb $4
+0x00 0xa0 0x24 0x09 # CHECK: jalr.hb $4, $5
+0x00 0xa3 0x10 0x04 # CHECK: sllv $2, $3, $5
+0x00 0xa3 0x10 0x06 # CHECK: srlv $2, $3, $5
+0x00 0xa3 0x10 0x07 # CHECK: srav $2, $3, $5
 0x00 0xc7 0x48 0x20 # CHECK: add $9, $6, $7
-0x46 0x2e 0x62 0x00 # CHECK: add.d $f8, $f12, $f14
-0x46 0x07 0x32 0x40 # CHECK: add.s $f9, $f6, $f7
-0x20 0xc9 0x45 0x67 # CHECK: addi $9, $6, 17767
-0x24 0xc9 0xc5 0x67 # CHECK: addiu $9, $6, -15001
 0x00 0xc7 0x48 0x21 # CHECK: addu $9, $6, $7
+0x00 0xc7 0x48 0x22 # CHECK: sub $9, $6, $7
 0x00 0xc7 0x48 0x24 # CHECK: and $9, $6, $7
-0x30 0xc9 0x45 0x67 # CHECK: andi $9, $6, 17767
+0x00 0xc7 0x48 0x27 # CHECK: nor $9, $6, $7
+0x00 0xe0 0x00 0x08 # CHECK: jr $7
+0x00 0xe0 0x00 0x11 # CHECK: mthi $7
+0x00 0xe0 0x00 0x13 # CHECK: mtlo $7
+0x00 0xe0 0xf8 0x09 # CHECK: jalr $7
+0x04 0xc1 0x01 0x4c # CHECK: bgez $6, 1332
+0x04 0xd1 0x01 0x4c # CHECK: bgezal $6, 1332
+0x08 0x00 0x01 0x4c # CHECK: j 1328
+0x0c 0x00 0x01 0x4c # CHECK: jal 1328
 0x10 0x00 0x01 0x4c # CHECK: b 1332
+0x11 0x26 0x01 0x4c # CHECK: beq $9, $6, 1332
+0x15 0x26 0x01 0x4c # CHECK: bne $9, $6, 1332
+0x18 0xc0 0x01 0x4c # CHECK: blez $6, 1332
+0x1c 0xc0 0x01 0x4c # CHECK: bgtz $6, 1332
+0x20 0xc9 0x45 0x67 # CHECK: addi $9, $6, 17767
+0x24 0xc9 0xc5 0x67 # CHECK: addiu $9, $6, -15001
+0x28 0x63 0x00 0x67 # CHECK: slti $3, $3, 103
+0x2c 0x63 0x00 0x67 # CHECK: sltiu $3, $3, 103
+0x30 0xc9 0x45 0x67 # CHECK: andi $9, $6, 17767
+0x34 0xc9 0x45 0x67 # CHECK: ori $9, $6, 17767
+0x38 0xc9 0x45 0x67 # CHECK: xori $9, $6, 17767
+0x3c 0x06 0x45 0x67 # CHECK: lui $6, 17767
+0x40 0x08 0x78 0x01 # CHECK: mfc0 $8, $15, 1
+0x40 0x08 0x80 0x04 # CHECK: mfc0 $8, $16, 4
+0x40 0x89 0x78 0x01 # CHECK: mtc0 $9, $15, 1
+0x44 0x06 0x38 0x00 # CHECK: mfc1 $6, $f7
+0x44 0x46 0x38 0x00 # CHECK: cfc1 $6, $7
+0x44 0x86 0x38 0x00 # CHECK: mtc1 $6, $f7
+0x44 0xc6 0x38 0x00 # CHECK: ctc1 $6, $7
 0x45 0x00 0x01 0x4c # CHECK: bc1f 1332
-0x45 0x1c 0x01 0x4c # CHECK: bc1f $fcc7, 1332
 0x45 0x01 0x01 0x4c # CHECK: bc1t 1332
+0x45 0x1c 0x01 0x4c # CHECK: bc1f $fcc7, 1332
 0x45 0x1d 0x01 0x4c # CHECK: bc1t $fcc7, 1332
-0x11 0x26 0x01 0x4c # CHECK: beq $9, $6, 1332
-0x04 0xc1 0x01 0x4c # CHECK: bgez $6, 1332
-0x04 0xd1 0x01 0x4c # CHECK: bgezal $6, 1332
-0x1c 0xc0 0x01 0x4c # CHECK: bgtz $6, 1332
-0x18 0xc0 0x01 0x4c # CHECK: blez $6, 1332
-0x15 0x26 0x01 0x4c # CHECK: bne $9, $6, 1332
-0x46 0x2e 0x60 0x32 # CHECK: c.eq.d $f12, $f14
-0x46 0x07 0x30 0x32 # CHECK: c.eq.s $f6, $f7
-0x46 0x2e 0x60 0x30 # CHECK: c.f.d $f12, $f14
+0x46 0x00 0x39 0x84 # CHECK: sqrt.s $f6, $f7
+0x46 0x00 0x39 0x85 # CHECK: abs.s $f6, $f7
+0x46 0x00 0x39 0x86 # CHECK: mov.s $f6, $f7
+0x46 0x00 0x39 0x87 # CHECK: neg.s $f6, $f7
+0x46 0x00 0x39 0x8c # CHECK: round.w.s $f6, $f7
+0x46 0x00 0x39 0x8d # CHECK: trunc.w.s $f6, $f7
+0x46 0x00 0x39 0x8e # CHECK: ceil.w.s $f6, $f7
+0x46 0x00 0x39 0x8f # CHECK: floor.w.s $f6, $f7
+0x46 0x00 0x39 0xa1 # CHECK: cvt.d.s $f6, $f7
+0x46 0x00 0x39 0xa4 # CHECK: cvt.w.s $f6, $f7
 0x46 0x07 0x30 0x30 # CHECK: c.f.s $f6, $f7
-0x46 0x2e 0x60 0x3e # CHECK: c.le.d $f12, $f14
-0x46 0x07 0x30 0x3e # CHECK: c.le.s $f6, $f7
-0x46 0x2e 0x60 0x3c # CHECK: c.lt.d $f12, $f14
+0x46 0x07 0x30 0x31 # CHECK: c.un.s $f6, $f7
+0x46 0x07 0x30 0x32 # CHECK: c.eq.s $f6, $f7
+0x46 0x07 0x30 0x34 # CHECK: c.olt.s $f6, $f7
+0x46 0x07 0x30 0x35 # CHECK: c.ult.s $f6, $f7
+0x46 0x07 0x30 0x36 # CHECK: c.ole.s $f6, $f7
+0x46 0x07 0x30 0x37 # CHECK: c.ule.s $f6, $f7
+0x46 0x07 0x30 0x38 # CHECK: c.sf.s $f6, $f7
+0x46 0x07 0x30 0x39 # CHECK: c.ngle.s $f6, $f7
+0x46 0x07 0x30 0x3a # CHECK: c.seq.s $f6, $f7
+0x46 0x07 0x30 0x3b # CHECK: c.ngl.s $f6, $f7
 0x46 0x07 0x30 0x3c # CHECK: c.lt.s $f6, $f7
-0x46 0x2e 0x60 0x3d # CHECK: c.nge.d $f12, $f14
 0x46 0x07 0x30 0x3d # CHECK: c.nge.s $f6, $f7
-0x46 0x2e 0x60 0x3b # CHECK: c.ngl.d $f12, $f14
-0x46 0x07 0x30 0x3b # CHECK: c.ngl.s $f6, $f7
-0x46 0x2e 0x60 0x39 # CHECK: c.ngle.d $f12, $f14
-0x46 0x07 0x30 0x39 # CHECK: c.ngle.s $f6, $f7
-0x46 0x2e 0x60 0x3f # CHECK: c.ngt.d $f12, $f14
+0x46 0x07 0x30 0x3e # CHECK: c.le.s $f6, $f7
 0x46 0x07 0x30 0x3f # CHECK: c.ngt.s $f6, $f7
-0x46 0x2e 0x60 0x36 # CHECK: c.ole.d $f12, $f14
-0x46 0x07 0x30 0x36 # CHECK: c.ole.s $f6, $f7
-0x46 0x2e 0x60 0x34 # CHECK: c.olt.d $f12, $f14
-0x46 0x07 0x30 0x34 # CHECK: c.olt.s $f6, $f7
-0x46 0x2e 0x60 0x3a # CHECK: c.seq.d $f12, $f14
-0x46 0x07 0x30 0x3a # CHECK: c.seq.s $f6, $f7
-0x46 0x2e 0x60 0x38 # CHECK: c.sf.d $f12, $f14
-0x46 0x07 0x30 0x38 # CHECK: c.sf.s $f6, $f7
-0x46 0x2e 0x60 0x33 # CHECK: c.ueq.d $f12, $f14
+0x46 0x07 0x32 0x40 # CHECK: add.s $f9, $f6, $f7
+0x46 0x07 0x32 0x41 # CHECK: sub.s $f9, $f6, $f7
+0x46 0x07 0x32 0x42 # CHECK: mul.s $f9, $f6, $f7
 0x46 0x12 0xe0 0x33 # CHECK: c.ueq.s $f28, $f18
-0x46 0x2e 0x60 0x37 # CHECK: c.ule.d $f12, $f14
-0x46 0x07 0x30 0x37 # CHECK: c.ule.s $f6, $f7
-0x46 0x2e 0x60 0x35 # CHECK: c.ult.d $f12, $f14
-0x46 0x07 0x30 0x35 # CHECK: c.ult.s $f6, $f7
-0x46 0x2e 0x60 0x31 # CHECK: c.un.d $f12, $f14
-0x46 0x07 0x30 0x31 # CHECK: c.un.s $f6, $f7
+0x46 0x1c 0x11 0x11 # CHECK: movf.s $f4, $f2, $fcc7
+0x46 0x1d 0x11 0x11 # CHECK: movt.s $f4, $f2, $fcc7
+0x46 0x20 0x41 0x86 # CHECK: mov.d $f6, $f8
+0x46 0x20 0x73 0x04 # CHECK: sqrt.d $f12, $f14
+0x46 0x20 0x73 0x05 # CHECK: abs.d $f12, $f14
+0x46 0x20 0x73 0x07 # CHECK: neg.d $f12, $f14
+0x46 0x20 0x73 0x0c # CHECK: round.w.d $f12, $f14
+0x46 0x20 0x73 0x0d # CHECK: trunc.w.d $f12, $f14
 0x46 0x20 0x73 0x0e # CHECK: ceil.w.d $f12, $f14
-0x46 0x00 0x39 0x8e # CHECK: ceil.w.s $f6, $f7
-0x44 0x46 0x38 0x00 # CHECK: cfc1 $6, $7
-0x70 0xe6 0x30 0x21 # CHECK: clo $6, $7
-0x70 0xe6 0x30 0x20 # CHECK: clz $6, $7
-0x44 0xc6 0x38 0x00 # CHECK: ctc1 $6, $7
-0x46 0x00 0x39 0xa1 # CHECK: cvt.d.s $f6, $f7
-0x46 0x80 0x73 0x21 # CHECK: cvt.d.w $f12, $f14
+0x46 0x20 0x73 0x0f # CHECK: floor.w.d $f12, $f14
 0x46 0x20 0x73 0x20 # CHECK: cvt.s.d $f12, $f14
-0x46 0x80 0x39 0xa0 # CHECK: cvt.s.w $f6, $f7
 0x46 0x20 0x73 0x24 # CHECK: cvt.w.d $f12, $f14
-0x46 0x00 0x39 0xa4 # CHECK: cvt.w.s $f6, $f7
-0x46 0x20 0x73 0x0f # CHECK: floor.w.d $f12, $f14
-0x46 0x00 0x39 0x8f # CHECK: floor.w.s $f6, $f7
-0x08 0x00 0x01 0x4c # CHECK: j 1328
-0x0c 0x00 0x01 0x4c # CHECK: jal 1328
+0x46 0x2e 0x60 0x30 # CHECK: c.f.d $f12, $f14
+0x46 0x2e 0x60 0x31 # CHECK: c.un.d $f12, $f14
+0x46 0x2e 0x60 0x32 # CHECK: c.eq.d $f12, $f14
+0x46 0x2e 0x60 0x33 # CHECK: c.ueq.d $f12, $f14
+0x46 0x2e 0x60 0x34 # CHECK: c.olt.d $f12, $f14
+0x46 0x2e 0x60 0x35 # CHECK: c.ult.d $f12, $f14
+0x46 0x2e 0x60 0x36 # CHECK: c.ole.d $f12, $f14
+0x46 0x2e 0x60 0x37 # CHECK: c.ule.d $f12, $f14
+0x46 0x2e 0x60 0x38 # CHECK: c.sf.d $f12, $f14
+0x46 0x2e 0x60 0x39 # CHECK: c.ngle.d $f12, $f14
+0x46 0x2e 0x60 0x3a # CHECK: c.seq.d $f12, $f14
+0x46 0x2e 0x60 0x3b # CHECK: c.ngl.d $f12, $f14
+0x46 0x2e 0x60 0x3c # CHECK: c.lt.d $f12, $f14
+0x46 0x2e 0x60 0x3d # CHECK: c.nge.d $f12, $f14
+0x46 0x2e 0x60 0x3e # CHECK: c.le.d $f12, $f14
+0x46 0x2e 0x60 0x3f # CHECK: c.ngt.d $f12, $f14
+0x46 0x2e 0x62 0x00 # CHECK: add.d $f8, $f12, $f14
+0x46 0x2e 0x62 0x01 # CHECK: sub.d $f8, $f12, $f14
+0x46 0x2e 0x62 0x02 # CHECK: mul.d $f8, $f12, $f14
+0x46 0x3c 0x11 0x11 # CHECK: movf.d $f4, $f2, $fcc7
+0x46 0x3d 0x11 0x11 # CHECK: movt.d $f4, $f2, $fcc7
+0x46 0x80 0x39 0xa0 # CHECK: cvt.s.w $f6, $f7
+0x46 0x80 0x73 0x21 # CHECK: cvt.d.w $f12, $f14
+0x70 0xc7 0x00 0x00 # CHECK: madd $6, $7
+0x70 0xc7 0x00 0x01 # CHECK: maddu $6, $7
+0x70 0xc7 0x00 0x04 # CHECK: msub $6, $7
+0x70 0xc7 0x00 0x05 # CHECK: msubu $6, $7
+0x70 0xc7 0x48 0x02 # CHECK: mul $9, $6, $7
+0x70 0xe6 0x30 0x20 # CHECK: clz $6, $7
+0x70 0xe6 0x30 0x21 # CHECK: clo $6, $7
 0x74 0x00 0x01 0x4c # CHECK: jalx 1328
-0x00 0xe0 0xf8 0x09 # CHECK: jalr $7
-0x00 0x80 0xfc 0x09 # CHECK: jalr.hb $4
-0x00 0xa0 0x24 0x09 # CHECK: jalr.hb $4, $5
-0x00 0xe0 0x00 0x08 # CHECK: jr $7
+0x7c 0x05 0xe8 0x3b # CHECK: .set push
+                    # CHECK: .set mips32r2
+                    # CHECK: rdhwr $5, $29
+                    # CHECK: .set pop
 0x80 0xa4 0x23 0xc6 # CHECK: lb $4, 9158($5)
-0x90 0xa4 0x00 0x06 # CHECK: lbu $4, 6($5)
-0xd4 0xe9 0x23 0xc6 # CHECK: ldc1 $f9, 9158($7)
 0x84 0xa4 0x00 0x0c # CHECK: lh $4, 12($5)
 0x84 0xa4 0x00 0x0c # CHECK: lh $4, 12($5)
-0xc0 0xe9 0x23 0xc6 # CHECK: ll $9, 9158($7)
-0x3c 0x06 0x45 0x67 # CHECK: lui $6, 17767
-0x8c 0xa4 0x00 0x18 # CHECK: lw $4, 24($5)
-0xc4 0xe9 0x23 0xc6 # CHECK: lwc1 $f9, 9158($7)
 0x88 0x82 0x00 0x03 # CHECK: lwl $2, 3($4)
+0x8c 0xa4 0x00 0x18 # CHECK: lw $4, 24($5)
+0x90 0xa4 0x00 0x06 # CHECK: lbu $4, 6($5)
 0x98 0xa3 0x00 0x10 # CHECK: lwr $3, 16($5)
-0x70 0xc7 0x00 0x00 # CHECK: madd $6, $7
-0x70 0xc7 0x00 0x01 # CHECK: maddu $6, $7
-0x44 0x06 0x38 0x00 # CHECK: mfc1 $6, $f7
-0x00 0x00 0x28 0x10 # CHECK: mfhi $5
-0x00 0x00 0x28 0x12 # CHECK: mflo $5
-0x46 0x20 0x41 0x86 # CHECK: mov.d $f6, $f8
-0x46 0x00 0x39 0x86 # CHECK: mov.s $f6, $f7
-0x70 0xc7 0x00 0x04 # CHECK: msub $6, $7
-0x70 0xc7 0x00 0x05 # CHECK: msubu $6, $7
-0x44 0x86 0x38 0x00 # CHECK: mtc1 $6, $f7
-0x00 0xe0 0x00 0x11 # CHECK: mthi $7
-0x00 0xe0 0x00 0x13 # CHECK: mtlo $7
-0x46 0x2e 0x62 0x02 # CHECK: mul.d $f8, $f12, $f14
-0x46 0x07 0x32 0x42 # CHECK: mul.s $f9, $f6, $f7
-0x70 0xc7 0x48 0x02 # CHECK: mul $9, $6, $7
-0x00 0x65 0x00 0x18 # CHECK: mult $3, $5
-0x00 0x65 0x00 0x19 # CHECK: multu $3, $5
-0x46 0x20 0x73 0x07 # CHECK: neg.d $f12, $f14
-0x46 0x00 0x39 0x87 # CHECK: neg.s $f6, $f7
-0x00 0x00 0x00 0x00 # CHECK: nop
-0x00 0xc7 0x48 0x27 # CHECK: nor $9, $6, $7
-0x00 0x65 0x18 0x25 # CHECK: or $3, $3, $5
-0x34 0xc9 0x45 0x67 # CHECK: ori $9, $6, 17767
-0x46 0x20 0x73 0x0c # CHECK: round.w.d $f12, $f14
-0x46 0x00 0x39 0x8c # CHECK: round.w.s $f6, $f7
-0xa0 0xa4 0x23 0xc6 # CHECK: sb $4, 9158($5)
 0xa0 0xa4 0x00 0x06 # CHECK: sb $4, 6($5)
-0xe0 0xe9 0x23 0xc6 # CHECK: sc $9, 9158($7)
-0xf4 0xe9 0x23 0xc6 # CHECK: sdc1 $f9, 9158($7)
+0xa0 0xa4 0x23 0xc6 # CHECK: sb $4, 9158($5)
 0xa4 0xa4 0x23 0xc6 # CHECK: sh $4, 9158($5)
-0x00 0x03 0x21 0xc0 # CHECK: sll $4, $3, 7
-0x00 0xa3 0x10 0x04 # CHECK: sllv $2, $3, $5
-0x00 0x65 0x18 0x2a # CHECK: slt $3, $3, $5
-0x28 0x63 0x00 0x67 # CHECK: slti $3, $3, 103
-0x2c 0x63 0x00 0x67 # CHECK: sltiu $3, $3, 103
-0x00 0x65 0x18 0x2b # CHECK: sltu $3, $3, $5
-0x46 0x20 0x73 0x04 # CHECK: sqrt.d $f12, $f14
-0x46 0x00 0x39 0x84 # CHECK: sqrt.s $f6, $f7
-0x00 0x03 0x21 0xc3 # CHECK: sra $4, $3, 7
-0x00 0xa3 0x10 0x07 # CHECK: srav $2, $3, $5
-0x00 0x03 0x21 0xc2 # CHECK: srl $4, $3, 7
-0x00 0xa3 0x10 0x06 # CHECK: srlv $2, $3, $5
-0x46 0x2e 0x62 0x01 # CHECK: sub.d $f8, $f12, $f14
-0x46 0x07 0x32 0x41 # CHECK: sub.s $f9, $f6, $f7
-0x00 0xc7 0x48 0x22 # CHECK: sub $9, $6, $7
-0x00 0x65 0x20 0x23 # CHECK: subu $4, $3, $5
-0xac 0xa4 0x00 0x18 # CHECK: sw $4, 24($5)
-0xe4 0xe9 0x23 0xc6 # CHECK: swc1 $f9, 9158($7)
 0xa8 0xa4 0x00 0x10 # CHECK: swl $4, 16($5)
+0xac 0xa4 0x00 0x18 # CHECK: sw $4, 24($5)
 0xb8 0xe6 0x00 0x10 # CHECK: swr $6, 16($7)
-0x00 0x00 0x01 0xcf # CHECK: sync 7
-0x46 0x20 0x73 0x0d # CHECK: trunc.w.d $f12, $f14
-0x46 0x00 0x39 0x8d # CHECK: trunc.w.s $f6, $f7
-0x00 0x65 0x18 0x26 # CHECK: xor $3, $3, $5
-0x38 0xc9 0x45 0x67 # CHECK: xori $9, $6, 17767
-0x7c 0x05 0xe8 0x3b # CHECK: .set push
-                    # CHECK: .set mips32r2
-                    # CHECK: rdhwr $5, $29
-                    # CHECK: .set pop
 0xbc 0x61 0x00 0x02 # CHECK: cache 1, 2($3)
+0xc0 0xe9 0x23 0xc6 # CHECK: ll $9, 9158($7)
+0xc4 0xe9 0x23 0xc6 # CHECK: lwc1 $f9, 9158($7)
+0xc8 0xc8 0x23 0xca # CHECK: lwc2 $8, 9162($6)
 0xcc 0x43 0x00 0x04 # CHECK: pref 3, 4($2)
+0xd4 0xe9 0x23 0xc6 # CHECK: ldc1 $f9, 9158($7)
+0xe0 0xe9 0x23 0xc6 # CHECK: sc $9, 9158($7)
+0xe4 0xe9 0x23 0xc6 # CHECK: swc1 $f9, 9158($7)
 0xe8 0xe9 0x23 0xc6 # CHECK: swc2 $9, 9158($7)
-0xc8 0xc8 0x23 0xca # CHECK: lwc2 $8, 9162($6)
+0xf4 0xe9 0x23 0xc6 # CHECK: sdc1 $f9, 9158($7)
diff --git a/test/MC/Disassembler/Mips/mips32_le.txt b/test/MC/Disassembler/Mips/mips32_le.txt
index 533fc69598c..c019c41bd12 100644
--- a/test/MC/Disassembler/Mips/mips32_le.txt
+++ b/test/MC/Disassembler/Mips/mips32_le.txt
@@ -254,6 +254,9 @@
 # CHECK: maddu  $6,  $7
 0x01 0x00 0xc7 0x70
 
+# CHECK: mfc0 $8, $16, 4
+0x04 0x80 0x08 0x40
+
 # CHECK: mfc1   $6, $f7
 0x00 0x38 0x06 0x44
 
@@ -299,6 +302,9 @@
 # CHECK: msubu  $6,  $7
 0x05 0x00 0xc7 0x70
 
+# CHECK: mtc0 $9, $15, 1
+0x01 0x78 0x89 0x40
+
 # CHECK: mtc1   $6, $f7
 0x00 0x38 0x86 0x44
 
diff --git a/test/MC/Disassembler/Mips/mips32r2.txt b/test/MC/Disassembler/Mips/mips32r2.txt
deleted file mode 100644
index 354ef74a75a..00000000000
--- a/test/MC/Disassembler/Mips/mips32r2.txt
+++ /dev/null
@@ -1,453 +0,0 @@
-# RUN: llvm-mc --disassemble %s -triple=mips-unknown-linux -mcpu=mips32r2 | FileCheck %s
-# CHECK: abs.d $f12, $f14
-0x46 0x20 0x73 0x05
-
-# CHECK: abs.s $f6, $f7
-0x46 0x00 0x39 0x85
-
-# CHECK: add $9, $6, $7
-0x00 0xc7 0x48 0x20
-
-# CHECK: add.d $f8, $f12, $f14
-0x46 0x2e 0x62 0x00
-
-# CHECK: add.s $f9, $f6, $f7
-0x46 0x07 0x32 0x40
-
-# CHECK: addi $9, $6, 17767
-0x20 0xc9 0x45 0x67
-
-# CHECK: addiu $9, $6, -15001
-0x24 0xc9 0xc5 0x67
-
-# CHECK: addu $9, $6, $7
-0x00 0xc7 0x48 0x21
-
-# CHECK: and $9, $6, $7
-0x00 0xc7 0x48 0x24
-
-# CHECK: andi $9, $6, 17767
-0x30 0xc9 0x45 0x67
-
-# CHECK: b 1332
-0x10 0x00 0x01 0x4c
-
-# CHECK: bc1f 1332
-0x45 0x00 0x01 0x4c
-
-# CHECK: bc1f $fcc7, 1332
-0x45 0x1c 0x01 0x4c
-
-# CHECK: bc1t 1332
-0x45 0x01 0x01 0x4c
-
-# CHECK: bc1t $fcc7, 1332
-0x45 0x1d 0x01 0x4c
-
-# CHECK: beq $9, $6, 1332
-0x11 0x26 0x01 0x4c
-
-# CHECK: bgez  $6, 1332
-0x04 0xc1 0x01 0x4c
-
-# CHECK: bgezal  $6, 1332
-0x04 0xd1 0x01 0x4c
-
-# CHECK: bgtz  $6, 1332
-0x1c 0xc0 0x01 0x4c
-
-# CHECK: blez  $6, 1332
-0x18 0xc0 0x01 0x4c
-
-# CHECK: bne $9, $6, 1332
-0x15 0x26 0x01 0x4c
-
-# CHECK: c.eq.d $f12, $f14
-0x46 0x2e 0x60 0x32
-
-# CHECK: c.eq.s $f6, $f7
-0x46 0x07 0x30 0x32
-
-# CHECK: c.f.d $f12, $f14
-0x46 0x2e 0x60 0x30
-
-# CHECK: c.f.s $f6, $f7
-0x46 0x07 0x30 0x30
-
-# CHECK: c.le.d $f12, $f14
-0x46 0x2e 0x60 0x3e
-
-# CHECK: c.le.s $f6, $f7
-0x46 0x07 0x30 0x3e
-
-# CHECK: c.lt.d $f12, $f14
-0x46 0x2e 0x60 0x3c
-
-# CHECK: c.lt.s $f6, $f7
-0x46 0x07 0x30 0x3c
-
-# CHECK: c.nge.d $f12, $f14
-0x46 0x2e 0x60 0x3d
-
-# CHECK: c.nge.s $f6, $f7
-0x46 0x07 0x30 0x3d
-
-# CHECK: c.ngl.d $f12, $f14
-0x46 0x2e 0x60 0x3b
-
-# CHECK: c.ngl.s $f6, $f7
-0x46 0x07 0x30 0x3b
-
-# CHECK: c.ngle.d $f12, $f14
-0x46 0x2e 0x60 0x39
-
-# CHECK: c.ngle.s $f6, $f7
-0x46 0x07 0x30 0x39
-
-# CHECK: c.ngt.d $f12, $f14
-0x46 0x2e 0x60 0x3f
-
-# CHECK: c.ngt.s $f6, $f7
-0x46 0x07 0x30 0x3f
-
-# CHECK: c.ole.d $f12, $f14
-0x46 0x2e 0x60 0x36
-
-# CHECK: c.ole.s $f6, $f7
-0x46 0x07 0x30 0x36
-
-# CHECK: c.olt.d $f12, $f14
-0x46 0x2e 0x60 0x34
-
-# CHECK: c.olt.s $f6, $f7
-0x46 0x07 0x30 0x34
-
-# CHECK: c.seq.d $f12, $f14
-0x46 0x2e 0x60 0x3a
-
-# CHECK: c.seq.s $f6, $f7
-0x46 0x07 0x30 0x3a
-
-# CHECK: c.sf.d $f12, $f14
-0x46 0x2e 0x60 0x38
-
-# CHECK: c.sf.s $f6, $f7
-0x46 0x07 0x30 0x38
-
-# CHECK: c.ueq.d $f12, $f14
-0x46 0x2e 0x60 0x33
-
-# CHECK: c.ueq.s $f28, $f18
-0x46 0x12 0xe0 0x33
-
-# CHECK: c.ule.d $f12, $f14
-0x46 0x2e 0x60 0x37
-
-# CHECK: c.ule.s $f6, $f7
-0x46 0x07 0x30 0x37
-
-# CHECK: c.ult.d $f12, $f14
-0x46 0x2e 0x60 0x35
-
-# CHECK: c.ult.s $f6, $f7
-0x46 0x07 0x30 0x35
-
-# CHECK: c.un.d $f12, $f14
-0x46 0x2e 0x60 0x31
-
-# CHECK: c.un.s $f6, $f7
-0x46 0x07 0x30 0x31
-
-# CHECK: ceil.w.d $f12, $f14
-0x46 0x20 0x73 0x0e
-
-# CHECK: ceil.w.s $f6, $f7
-0x46 0x00 0x39 0x8e
-
-# CHECK: cfc1  $6, $7
-0x44 0x46 0x38 0x00
-
-# CHECK: clo  $6, $7
-0x70 0xe6 0x30 0x21
-
-# CHECK: clz  $6, $7
-0x70 0xe6 0x30 0x20
-
-# CHECK: ctc1  $6, $7
-0x44 0xc6 0x38 0x00
-
-# CHECK: cvt.d.s $f6, $f7
-0x46 0x00 0x39 0xa1
-
-# CHECK: cvt.d.w $f12, $f14
-0x46 0x80 0x73 0x21
-
-# CHECK: cvt.l.d $f12, $f14
-0x46 0x20 0x73 0x25
-
-# CHECK: cvt.l.s $f6, $f7
-0x46 0x00 0x39 0xa5
-
-# CHECK: cvt.s.d $f12, $f14
-0x46 0x20 0x73 0x20
-
-# CHECK: cvt.s.w $f6, $f7
-0x46 0x80 0x39 0xa0
-
-# CHECK: cvt.w.d $f12, $f14
-0x46 0x20 0x73 0x24
-
-# CHECK: cvt.w.s $f6, $f7
-0x46 0x00 0x39 0xa4
-
-# CHECK: floor.w.d $f12, $f14
-0x46 0x20 0x73 0x0f
-
-# CHECK: floor.w.s $f6, $f7
-0x46 0x00 0x39 0x8f
-
-# CHECK: ins $19, $9, 6, 7
-0x7d 0x33 0x61 0x84
-
-# CHECK: j 1328
-0x08 0x00 0x01 0x4c
-
-# CHECK: jal 1328
-0x0c 0x00 0x01 0x4c
-
-# CHECK: jalx 1328
-0x74 0x00 0x01 0x4c
-
-# CHECK: jalr  $7
-0x00 0xe0 0xf8 0x09
-
-# CHECK: jr  $7
-0x00 0xe0 0x00 0x08
-
-# CHECK: lb  $4, 9158($5)
-0x80 0xa4 0x23 0xc6
-
-# CHECK: lbu $4, 6($5)
-0x90 0xa4 0x00 0x06
-
-# CHECK: ldc1  $f9, 9158($7)
-0xd4 0xe9 0x23 0xc6
-
-# CHECK: lh  $4, 12($5)
-0x84 0xa4 0x00 0x0c
-
-# CHECK: lh  $4, 12($5)
-0x84 0xa4 0x00 0x0c
-
-# CHECK: ll  $9, 9158($7)
-0xc0 0xe9 0x23 0xc6
-
-# CHECK: lui  $6, 17767
-0x3c 0x06 0x45 0x67
-
-# CHECK: luxc1 $f0, $6($5)
-0x4c 0xa6 0x00 0x05
-
-# CHECK: lw  $4, 24($5)
-0x8c 0xa4 0x00 0x18
-
-# CHECK: lwc1  $f9, 9158($7)
-0xc4 0xe9 0x23 0xc6
-
-# CHECK: lwl   $2,  3($4)
-0x88 0x82 0x00 0x03
-
-# CHECK: lwr   $3, 16($5)
-0x98 0xa3 0x00 0x10
-
-# CHECK: lwxc1 $f20, $12($14)
-0x4d 0xcc 0x05 0x00
-
-# CHECK: madd   $6,  $7
-0x70 0xc7 0x00 0x00
-
-# CHECK: maddu  $6,  $7
-0x70 0xc7 0x00 0x01
-
-# CHECK: mfc1   $6, $f7
-0x44 0x06 0x38 0x00
-
-# CHECK: mfhi  $5
-0x00 0x00 0x28 0x10
-
-# CHECK: mflo  $5
-0x00 0x00 0x28 0x12
-
-# CHECK: mov.d $f6, $f8
-0x46 0x20 0x41 0x86
-
-# CHECK: mov.s $f6, $f7
-0x46 0x00 0x39 0x86
-
-# CHECK: msub   $6,  $7
-0x70 0xc7 0x00 0x04
-
-# CHECK: msubu  $6,  $7
-0x70 0xc7 0x00 0x05
-
-# CHECK: mtc1   $6, $f7
-0x44 0x86 0x38 0x00
-
-# CHECK: mthi   $7
-0x00 0xe0 0x00 0x11
-
-# CHECK: mtlo   $7
-0x00 0xe0 0x00 0x13
-
-# CHECK: mul.d $f8, $f12, $f14
-0x46 0x2e 0x62 0x02
-
-# CHECK: mul.s $f9, $f6, $f7
-0x46 0x07 0x32 0x42
-
-# CHECK: mul $9,  $6,  $7
-0x70 0xc7 0x48 0x02
-
-# CHECK: mult  $3, $5
-0x00 0x65 0x00 0x18
-
-# CHECK: multu $3, $5
-0x00 0x65 0x00 0x19
-
-# CHECK: neg.d $f12, $f14
-0x46 0x20 0x73 0x07
-
-# CHECK: neg.s $f6, $f7
-0x46 0x00 0x39 0x87
-
-# CHECK: nop
-0x00 0x00 0x00 0x00
-
-# CHECK: nor $9,  $6, $7
-0x00 0xc7 0x48 0x27
-
-# CHECK: or  $3, $3, $5
-0x00 0x65 0x18 0x25
-
-# CHECK: ori $9,  $6, 17767
-0x34 0xc9 0x45 0x67
-
-# CHECK: rotr $9, $6, 7
-0x00 0x26 0x49 0xc2
-
-# CHECK:  rotrv $9, $6, $7
-0x00 0xe6 0x48 0x46
-
-# CHECK: round.w.d $f12, $f14
-0x46 0x20 0x73 0x0c
-
-# CHECK: round.w.s $f6, $f7
-0x46 0x00 0x39 0x8c
-
-# CHECK: sb  $4, 9158($5)
-0xa0 0xa4 0x23 0xc6
-
-# CHECK: sb  $4, 6($5)
-0xa0 0xa4 0x00 0x06
-
-# CHECK: sc  $9, 9158($7)
-0xe0 0xe9 0x23 0xc6
-
-# CHECK: sdc1  $f9, 9158($7)
-0xf4 0xe9 0x23 0xc6
-
-# CHECK: seb $6, $7
-0x7c 0x07 0x34 0x20
-
-# CHECK: seh $6, $7
-0x7c 0x07 0x36 0x20
-
-# CHECK: sh  $4, 9158($5)
-0xa4 0xa4 0x23 0xc6
-
-# CHECK: sll $4, $3, 7
-0x00 0x03 0x21 0xc0
-
-# CHECK: sllv  $2, $3, $5
-0x00 0xa3 0x10 0x04
-
-# CHECK: slt $3, $3, $5
-0x00 0x65 0x18 0x2a
-
-# CHECK: slti  $3, $3, 103
-0x28 0x63 0x00 0x67
-
-# CHECK: sltiu $3, $3, 103
-0x2c 0x63 0x00 0x67
-
-# CHECK: sltu  $3, $3, $5
-0x00 0x65 0x18 0x2b
-
-# CHECK: sqrt.d  $f12, $f14
-0x46 0x20 0x73 0x04
-
-# CHECK: sqrt.s  $f6, $f7
-0x46 0x00 0x39 0x84
-
-# CHECK: sra $4, $3, 7
-0x00 0x03 0x21 0xc3
-
-# CHECK: srav  $2, $3, $5
-0x00 0xa3 0x10 0x07
-
-# CHECK: srl $4, $3, 7
-0x00 0x03 0x21 0xc2
-
-# CHECK: srlv  $2, $3, $5
-0x00 0xa3 0x10 0x06
-
-# CHECK: sub.d $f8, $f12, $f14
-0x46 0x2e 0x62 0x01
-
-# CHECK: sub.s $f9, $f6, $f7
-0x46 0x07 0x32 0x41
-
-# CHECK: sub $9,  $6, $7
-0x00 0xc7 0x48 0x22
-
-# CHECK: subu  $4, $3, $5
-0x00 0x65 0x20 0x23
-
-# CHECK: suxc1 $f4, $24($5)
-0x4c 0xb8 0x20 0x0d
-
-# CHECK: sw  $4, 24($5)
-0xac 0xa4 0x00 0x18
-
-# CHECK: swc1  $f9, 9158($7)
-0xe4 0xe9 0x23 0xc6
-
-# CHECK: swl $4,  16($5)
-0xa8 0xa4 0x00 0x10
-
-# CHECK: swr $6, 16($7)
-0xb8 0xe6 0x00 0x10
-
-# CHECK: swxc1 $f26, $18($22)
-0x4e 0xd2 0xd0 0x08
-
-# CHECK: sync  7
-0x00 0x00 0x01 0xcf
-
-# CHECK: trunc.w.d $f12, $f14
-0x46 0x20 0x73 0x0d
-
-# CHECK: trunc.w.s $f6, $f7
-0x46 0x00 0x39 0x8d
-
-# CHECK: wsbh  $6, $7
-0x7c 0x07 0x30 0xa0
-
-# CHECK: xor $3, $3, $5
-0x00 0x65 0x18 0x26
-
-# CHECK: xori  $9,  $6, 17767
-0x38 0xc9 0x45 0x67
-
-# CHECK: synci -6137($fp)
-0x07 0xdf 0xe8 0x07
diff --git a/test/MC/Disassembler/Mips/mips32r2/valid-mips32r2-el.txt b/test/MC/Disassembler/Mips/mips32r2/valid-mips32r2-el.txt
new file mode 100644
index 00000000000..c487b6d0e96
--- /dev/null
+++ b/test/MC/Disassembler/Mips/mips32r2/valid-mips32r2-el.txt
@@ -0,0 +1,174 @@
+# RUN: llvm-mc --disassemble %s -triple=mipsel-unknown-linux -mcpu=mips32r2 | FileCheck %s
+# Try a mips64* triple to confirm that mips* vs mips64* triples no longer have
+# an effect on the disassembler behaviour.
+# RUN: llvm-mc --disassemble %s -triple=mips64el-unknown-linux -mcpu=mips32r2 | FileCheck %s
+0x05 0x73 0x20 0x46 # CHECK: abs.d $f12, $f14
+0x85 0x39 0x00 0x46 # CHECK: abs.s $f6, $f7
+0x20 0x48 0xc7 0x00 # CHECK: add $9, $6, $7
+0x00 0x62 0x2e 0x46 # CHECK: add.d $f8, $f12, $f14
+0x40 0x32 0x07 0x46 # CHECK: add.s $f9, $f6, $f7
+0x67 0x45 0xc9 0x20 # CHECK: addi $9, $6, 17767
+0x67 0xc5 0xc9 0x24 # CHECK: addiu $9, $6, -15001
+0x21 0x48 0xc7 0x00 # CHECK: addu $9, $6, $7
+0x24 0x48 0xc7 0x00 # CHECK: and $9, $6, $7
+0x67 0x45 0xc9 0x30 # CHECK: andi $9, $6, 17767
+0x4c 0x01 0x00 0x10 # CHECK: b 1332
+0x4c 0x01 0x00 0x45 # CHECK: bc1f 1332
+0x4c 0x01 0x1c 0x45 # CHECK: bc1f $fcc7, 1332
+0x4c 0x01 0x01 0x45 # CHECK: bc1t 1332
+0x4c 0x01 0x1d 0x45 # CHECK: bc1t $fcc7, 1332
+0x4c 0x01 0x26 0x11 # CHECK: beq $9, $6, 1332
+0x4c 0x01 0xc1 0x04 # CHECK: bgez $6, 1332
+0x4c 0x01 0xd1 0x04 # CHECK: bgezal $6, 1332
+0x4c 0x01 0xc0 0x1c # CHECK: bgtz $6, 1332
+0x4c 0x01 0xc0 0x18 # CHECK: blez $6, 1332
+0x4c 0x01 0x26 0x15 # CHECK: bne $9, $6, 1332
+0x32 0x60 0x2e 0x46 # CHECK: c.eq.d $f12, $f14
+0x32 0x30 0x07 0x46 # CHECK: c.eq.s $f6, $f7
+0x30 0x60 0x2e 0x46 # CHECK: c.f.d $f12, $f14
+0x30 0x30 0x07 0x46 # CHECK: c.f.s $f6, $f7
+0x3e 0x60 0x2e 0x46 # CHECK: c.le.d $f12, $f14
+0x3e 0x30 0x07 0x46 # CHECK: c.le.s $f6, $f7
+0x3c 0x60 0x2e 0x46 # CHECK: c.lt.d $f12, $f14
+0x3c 0x30 0x07 0x46 # CHECK: c.lt.s $f6, $f7
+0x3d 0x60 0x2e 0x46 # CHECK: c.nge.d $f12, $f14
+0x3d 0x30 0x07 0x46 # CHECK: c.nge.s $f6, $f7
+0x3b 0x60 0x2e 0x46 # CHECK: c.ngl.d $f12, $f14
+0x3b 0x30 0x07 0x46 # CHECK: c.ngl.s $f6, $f7
+0x39 0x60 0x2e 0x46 # CHECK: c.ngle.d $f12, $f14
+0x39 0x30 0x07 0x46 # CHECK: c.ngle.s $f6, $f7
+0x3f 0x60 0x2e 0x46 # CHECK: c.ngt.d $f12, $f14
+0x3f 0x30 0x07 0x46 # CHECK: c.ngt.s $f6, $f7
+0x36 0x60 0x2e 0x46 # CHECK: c.ole.d $f12, $f14
+0x36 0x30 0x07 0x46 # CHECK: c.ole.s $f6, $f7
+0x34 0x60 0x2e 0x46 # CHECK: c.olt.d $f12, $f14
+0x34 0x30 0x07 0x46 # CHECK: c.olt.s $f6, $f7
+0x3a 0x60 0x2e 0x46 # CHECK: c.seq.d $f12, $f14
+0x3a 0x30 0x07 0x46 # CHECK: c.seq.s $f6, $f7
+0x38 0x60 0x2e 0x46 # CHECK: c.sf.d $f12, $f14
+0x38 0x30 0x07 0x46 # CHECK: c.sf.s $f6, $f7
+0x33 0x60 0x2e 0x46 # CHECK: c.ueq.d $f12, $f14
+0x33 0xe0 0x12 0x46 # CHECK: c.ueq.s $f28, $f18
+0x37 0x60 0x2e 0x46 # CHECK: c.ule.d $f12, $f14
+0x37 0x30 0x07 0x46 # CHECK: c.ule.s $f6, $f7
+0x35 0x60 0x2e 0x46 # CHECK: c.ult.d $f12, $f14
+0x35 0x30 0x07 0x46 # CHECK: c.ult.s $f6, $f7
+0x31 0x60 0x2e 0x46 # CHECK: c.un.d $f12, $f14
+0x31 0x30 0x07 0x46 # CHECK: c.un.s $f6, $f7
+0x0e 0x73 0x20 0x46 # CHECK: ceil.w.d $f12, $f14
+0x8e 0x39 0x00 0x46 # CHECK: ceil.w.s $f6, $f7
+0x00 0x38 0x46 0x44 # CHECK: cfc1 $6, $7
+0x21 0x30 0xe6 0x70 # CHECK: clo $6, $7
+0x20 0x30 0xe6 0x70 # CHECK: clz $6, $7
+0x00 0x38 0xc6 0x44 # CHECK: ctc1 $6, $7
+0xa1 0x39 0x00 0x46 # CHECK: cvt.d.s $f6, $f7
+0x21 0x73 0x80 0x46 # CHECK: cvt.d.w $f12, $f14
+0x25 0x73 0x20 0x46 # CHECK: cvt.l.d $f12, $f14
+0xa5 0x39 0x00 0x46 # CHECK: cvt.l.s $f6, $f7
+0x20 0x73 0x20 0x46 # CHECK: cvt.s.d $f12, $f14
+0xa0 0x39 0x80 0x46 # CHECK: cvt.s.w $f6, $f7
+0x24 0x73 0x20 0x46 # CHECK: cvt.w.d $f12, $f14
+0xa4 0x39 0x00 0x46 # CHECK: cvt.w.s $f6, $f7
+0x00 0x60 0x7e 0x41 # CHECK: di $fp
+0x00 0x60 0x60 0x41 # CHECK: di
+0x20 0x60 0x6e 0x41 # CHECK: ei $14
+0x20 0x60 0x60 0x41 # CHECK: ei
+0x0f 0x73 0x20 0x46 # CHECK: floor.w.d $f12, $f14
+0x8f 0x39 0x00 0x46 # CHECK: floor.w.s $f6, $f7
+0x84 0x61 0x33 0x7d # CHECK: ins $19, $9, 6, 7
+0x4c 0x01 0x00 0x08 # CHECK: j 1328
+0x4c 0x01 0x00 0x0c # CHECK: jal 1328
+0x4c 0x01 0x00 0x74 # CHECK: jalx 1328
+0x09 0xf8 0xe0 0x00 # CHECK: jalr $7
+0x09 0xfc 0x80 0x00 # CHECK: jalr.hb $4
+0x09 0x24 0xa0 0x00 # CHECK: jalr.hb $4, $5
+0x08 0x00 0xe0 0x00 # CHECK: jr $7
+0xc6 0x23 0xa4 0x80 # CHECK: lb $4, 9158($5)
+0x06 0x00 0xa4 0x90 # CHECK: lbu $4, 6($5)
+0xc6 0x23 0xe9 0xd4 # CHECK: ldc1 $f9, 9158($7)
+0x01 0x02 0xf7 0x4d # CHECK: ldxc1 $f8, $23($15)
+0x0c 0x00 0xa4 0x84 # CHECK: lh $4, 12($5)
+0x0c 0x00 0xa4 0x84 # CHECK: lh $4, 12($5)
+0xc6 0x23 0xe9 0xc0 # CHECK: ll $9, 9158($7)
+0x67 0x45 0x06 0x3c # CHECK: lui $6, 17767
+0x05 0x00 0xa6 0x4c # CHECK: luxc1 $f0, $6($5)
+0x18 0x00 0xa4 0x8c # CHECK: lw $4, 24($5)
+0xc6 0x23 0xe9 0xc4 # CHECK: lwc1 $f9, 9158($7)
+0x03 0x00 0x82 0x88 # CHECK: lwl $2, 3($4)
+0x10 0x00 0xa3 0x98 # CHECK: lwr $3, 16($5)
+0x00 0x05 0xcc 0x4d # CHECK: lwxc1 $f20, $12($14)
+0x00 0x00 0xc7 0x70 # CHECK: madd $6, $7
+0xa1 0xd4 0x94 0x4e # CHECK: madd.d $f18, $f20, $f26, $f20
+0x60 0x98 0xf9 0x4f # CHECK: madd.s $f1, $f31, $f19, $f25
+0x01 0x00 0xc7 0x70 # CHECK: maddu $6, $7
+0x01 0x78 0x08 0x40 # CHECK: mfc0 $8, $15, 1
+0x00 0x38 0x06 0x44 # CHECK: mfc1 $6, $f7
+0x10 0x28 0x00 0x00 # CHECK: mfhi $5
+0x00 0xc0 0x7e 0x44 # CHECK: mfhc1 $fp, $f24
+0x12 0x28 0x00 0x00 # CHECK: mflo $5
+0x86 0x41 0x20 0x46 # CHECK: mov.d $f6, $f8
+0x86 0x39 0x00 0x46 # CHECK: mov.s $f6, $f7
+0x04 0x00 0xc7 0x70 # CHECK: msub $6, $7
+0xa9 0xf2 0x52 0x4c # CHECK: msub.d $f10, $f2, $f30, $f18
+0x28 0x53 0x70 0x4e # CHECK: msub.s $f12, $f19, $f10, $f16
+0x05 0x00 0xc7 0x70 # CHECK: msubu $6, $7
+0x01 0x78 0x89 0x40 # CHECK: mtc0 $9, $15, 1
+0x00 0x38 0x86 0x44 # CHECK: mtc1 $6, $f7
+0x11 0x00 0xe0 0x00 # CHECK: mthi $7
+0x00 0x80 0xe0 0x44 # CHECK: mthc1 $zero, $f16
+0x13 0x00 0xe0 0x00 # CHECK: mtlo $7
+0x02 0x62 0x2e 0x46 # CHECK: mul.d $f8, $f12, $f14
+0x42 0x32 0x07 0x46 # CHECK: mul.s $f9, $f6, $f7
+0x02 0x48 0xc7 0x70 # CHECK: mul $9, $6, $7
+0x18 0x00 0x65 0x00 # CHECK: mult $3, $5
+0x19 0x00 0x65 0x00 # CHECK: multu $3, $5
+0x07 0x73 0x20 0x46 # CHECK: neg.d $f12, $f14
+0x87 0x39 0x00 0x46 # CHECK: neg.s $f6, $f7
+0xb1 0x74 0x54 0x4d # CHECK: nmadd.d $f18, $f10, $f14, $f20
+0x30 0xc8 0xac 0x4c # CHECK: nmadd.s $f0, $f5, $f25, $f12
+0x00 0x00 0x00 0x00 # CHECK: nop
+0x27 0x48 0xc7 0x00 # CHECK: nor $9, $6, $7
+0xb9 0x87 0x1e 0x4d # CHECK: nmsub.d $f30, $f8, $f16, $f30
+0x78 0x98 0x04 0x4f # CHECK: nmsub.s $f1, $f24, $f19, $f4
+0x25 0x18 0x65 0x00 # CHECK: or $3, $3, $5
+0x67 0x45 0xc9 0x34 # CHECK: ori $9, $6, 17767
+0xc2 0x49 0x26 0x00 # CHECK: rotr $9, $6, 7
+0x46 0x48 0xe6 0x00 # CHECK: rotrv $9, $6, $7
+0x0c 0x73 0x20 0x46 # CHECK: round.w.d $f12, $f14
+0x8c 0x39 0x00 0x46 # CHECK: round.w.s $f6, $f7
+0xc6 0x23 0xa4 0xa0 # CHECK: sb $4, 9158($5)
+0x06 0x00 0xa4 0xa0 # CHECK: sb $4, 6($5)
+0xc6 0x23 0xe9 0xe0 # CHECK: sc $9, 9158($7)
+0xc6 0x23 0xe9 0xf4 # CHECK: sdc1 $f9, 9158($7)
+0x09 0x40 0x24 0x4f # CHECK: sdxc1 $f8, $4($25)
+0x20 0x34 0x07 0x7c # CHECK: seb $6, $7
+0x20 0x36 0x07 0x7c # CHECK: seh $6, $7
+0xc6 0x23 0xa4 0xa4 # CHECK: sh $4, 9158($5)
+0xc0 0x21 0x03 0x00 # CHECK: sll $4, $3, 7
+0x04 0x10 0xa3 0x00 # CHECK: sllv $2, $3, $5
+0x2a 0x18 0x65 0x00 # CHECK: slt $3, $3, $5
+0x67 0x00 0x63 0x28 # CHECK: slti $3, $3, 103
+0x67 0x00 0x63 0x2c # CHECK: sltiu $3, $3, 103
+0x2b 0x18 0x65 0x00 # CHECK: sltu $3, $3, $5
+0x04 0x73 0x20 0x46 # CHECK: sqrt.d $f12, $f14
+0x84 0x39 0x00 0x46 # CHECK: sqrt.s $f6, $f7
+0xc3 0x21 0x03 0x00 # CHECK: sra $4, $3, 7
+0x07 0x10 0xa3 0x00 # CHECK: srav $2, $3, $5
+0xc2 0x21 0x03 0x00 # CHECK: srl $4, $3, 7
+0x06 0x10 0xa3 0x00 # CHECK: srlv $2, $3, $5
+0x01 0x62 0x2e 0x46 # CHECK: sub.d $f8, $f12, $f14
+0x41 0x32 0x07 0x46 # CHECK: sub.s $f9, $f6, $f7
+0x22 0x48 0xc7 0x00 # CHECK: sub $9, $6, $7
+0x23 0x20 0x65 0x00 # CHECK: subu $4, $3, $5
+0x0d 0x20 0xb8 0x4c # CHECK: suxc1 $f4, $24($5)
+0x18 0x00 0xa4 0xac # CHECK: sw $4, 24($5)
+0xc6 0x23 0xe9 0xe4 # CHECK: swc1 $f9, 9158($7)
+0x10 0x00 0xa4 0xa8 # CHECK: swl $4, 16($5)
+0x10 0x00 0xe6 0xb8 # CHECK: swr $6, 16($7)
+0x08 0xd0 0xd2 0x4e # CHECK: swxc1 $f26, $18($22)
+0xcf 0x01 0x00 0x00 # CHECK: sync 7
+0x0d 0x73 0x20 0x46 # CHECK: trunc.w.d $f12, $f14
+0x8d 0x39 0x00 0x46 # CHECK: trunc.w.s $f6, $f7
+0xa0 0x30 0x07 0x7c # CHECK: wsbh $6, $7
+0x26 0x18 0x65 0x00 # CHECK: xor $3, $3, $5
+0x67 0x45 0xc9 0x38 # CHECK: xori $9, $6, 17767
diff --git a/test/MC/Disassembler/Mips/mips32r2/valid-mips32r2-le.txt b/test/MC/Disassembler/Mips/mips32r2/valid-mips32r2-le.txt
deleted file mode 100644
index d0eb13c5afd..00000000000
--- a/test/MC/Disassembler/Mips/mips32r2/valid-mips32r2-le.txt
+++ /dev/null
@@ -1,172 +0,0 @@
-# RUN: llvm-mc --disassemble %s -triple=mipsel-unknown-linux -mcpu=mips32r2 | FileCheck %s
-# Try a mips64* triple to confirm that mips* vs mips64* triples no longer have
-# an effect on the disassembler behaviour.
-# RUN: llvm-mc --disassemble %s -triple=mips64el-unknown-linux -mcpu=mips32r2 | FileCheck %s
-0x05 0x73 0x20 0x46 # CHECK: abs.d $f12, $f14
-0x85 0x39 0x00 0x46 # CHECK: abs.s $f6, $f7
-0x20 0x48 0xc7 0x00 # CHECK: add $9, $6, $7
-0x00 0x62 0x2e 0x46 # CHECK: add.d $f8, $f12, $f14
-0x40 0x32 0x07 0x46 # CHECK: add.s $f9, $f6, $f7
-0x67 0x45 0xc9 0x20 # CHECK: addi $9, $6, 17767
-0x67 0xc5 0xc9 0x24 # CHECK: addiu $9, $6, -15001
-0x21 0x48 0xc7 0x00 # CHECK: addu $9, $6, $7
-0x24 0x48 0xc7 0x00 # CHECK: and $9, $6, $7
-0x67 0x45 0xc9 0x30 # CHECK: andi $9, $6, 17767
-0x4c 0x01 0x00 0x10 # CHECK: b 1332
-0x4c 0x01 0x00 0x45 # CHECK: bc1f 1332
-0x4c 0x01 0x1c 0x45 # CHECK: bc1f $fcc7, 1332
-0x4c 0x01 0x01 0x45 # CHECK: bc1t 1332
-0x4c 0x01 0x1d 0x45 # CHECK: bc1t $fcc7, 1332
-0x4c 0x01 0x26 0x11 # CHECK: beq $9, $6, 1332
-0x4c 0x01 0xc1 0x04 # CHECK: bgez $6, 1332
-0x4c 0x01 0xd1 0x04 # CHECK: bgezal $6, 1332
-0x4c 0x01 0xc0 0x1c # CHECK: bgtz $6, 1332
-0x4c 0x01 0xc0 0x18 # CHECK: blez $6, 1332
-0x4c 0x01 0x26 0x15 # CHECK: bne $9, $6, 1332
-0x32 0x60 0x2e 0x46 # CHECK: c.eq.d $f12, $f14
-0x32 0x30 0x07 0x46 # CHECK: c.eq.s $f6, $f7
-0x30 0x60 0x2e 0x46 # CHECK: c.f.d $f12, $f14
-0x30 0x30 0x07 0x46 # CHECK: c.f.s $f6, $f7
-0x3e 0x60 0x2e 0x46 # CHECK: c.le.d $f12, $f14
-0x3e 0x30 0x07 0x46 # CHECK: c.le.s $f6, $f7
-0x3c 0x60 0x2e 0x46 # CHECK: c.lt.d $f12, $f14
-0x3c 0x30 0x07 0x46 # CHECK: c.lt.s $f6, $f7
-0x3d 0x60 0x2e 0x46 # CHECK: c.nge.d $f12, $f14
-0x3d 0x30 0x07 0x46 # CHECK: c.nge.s $f6, $f7
-0x3b 0x60 0x2e 0x46 # CHECK: c.ngl.d $f12, $f14
-0x3b 0x30 0x07 0x46 # CHECK: c.ngl.s $f6, $f7
-0x39 0x60 0x2e 0x46 # CHECK: c.ngle.d $f12, $f14
-0x39 0x30 0x07 0x46 # CHECK: c.ngle.s $f6, $f7
-0x3f 0x60 0x2e 0x46 # CHECK: c.ngt.d $f12, $f14
-0x3f 0x30 0x07 0x46 # CHECK: c.ngt.s $f6, $f7
-0x36 0x60 0x2e 0x46 # CHECK: c.ole.d $f12, $f14
-0x36 0x30 0x07 0x46 # CHECK: c.ole.s $f6, $f7
-0x34 0x60 0x2e 0x46 # CHECK: c.olt.d $f12, $f14
-0x34 0x30 0x07 0x46 # CHECK: c.olt.s $f6, $f7
-0x3a 0x60 0x2e 0x46 # CHECK: c.seq.d $f12, $f14
-0x3a 0x30 0x07 0x46 # CHECK: c.seq.s $f6, $f7
-0x38 0x60 0x2e 0x46 # CHECK: c.sf.d $f12, $f14
-0x38 0x30 0x07 0x46 # CHECK: c.sf.s $f6, $f7
-0x33 0x60 0x2e 0x46 # CHECK: c.ueq.d $f12, $f14
-0x33 0xe0 0x12 0x46 # CHECK: c.ueq.s $f28, $f18
-0x37 0x60 0x2e 0x46 # CHECK: c.ule.d $f12, $f14
-0x37 0x30 0x07 0x46 # CHECK: c.ule.s $f6, $f7
-0x35 0x60 0x2e 0x46 # CHECK: c.ult.d $f12, $f14
-0x35 0x30 0x07 0x46 # CHECK: c.ult.s $f6, $f7
-0x31 0x60 0x2e 0x46 # CHECK: c.un.d $f12, $f14
-0x31 0x30 0x07 0x46 # CHECK: c.un.s $f6, $f7
-0x0e 0x73 0x20 0x46 # CHECK: ceil.w.d $f12, $f14
-0x8e 0x39 0x00 0x46 # CHECK: ceil.w.s $f6, $f7
-0x00 0x38 0x46 0x44 # CHECK: cfc1 $6, $7
-0x21 0x30 0xe6 0x70 # CHECK: clo $6, $7
-0x20 0x30 0xe6 0x70 # CHECK: clz $6, $7
-0x00 0x38 0xc6 0x44 # CHECK: ctc1 $6, $7
-0xa1 0x39 0x00 0x46 # CHECK: cvt.d.s $f6, $f7
-0x21 0x73 0x80 0x46 # CHECK: cvt.d.w $f12, $f14
-0x25 0x73 0x20 0x46 # CHECK: cvt.l.d $f12, $f14
-0xa5 0x39 0x00 0x46 # CHECK: cvt.l.s $f6, $f7
-0x20 0x73 0x20 0x46 # CHECK: cvt.s.d $f12, $f14
-0xa0 0x39 0x80 0x46 # CHECK: cvt.s.w $f6, $f7
-0x24 0x73 0x20 0x46 # CHECK: cvt.w.d $f12, $f14
-0xa4 0x39 0x00 0x46 # CHECK: cvt.w.s $f6, $f7
-0x00 0x60 0x7e 0x41 # CHECK: di $fp
-0x00 0x60 0x60 0x41 # CHECK: di
-0x20 0x60 0x6e 0x41 # CHECK: ei $14
-0x20 0x60 0x60 0x41 # CHECK: ei
-0x0f 0x73 0x20 0x46 # CHECK: floor.w.d $f12, $f14
-0x8f 0x39 0x00 0x46 # CHECK: floor.w.s $f6, $f7
-0x84 0x61 0x33 0x7d # CHECK: ins $19, $9, 6, 7
-0x4c 0x01 0x00 0x08 # CHECK: j 1328
-0x4c 0x01 0x00 0x0c # CHECK: jal 1328
-0x4c 0x01 0x00 0x74 # CHECK: jalx 1328
-0x09 0xf8 0xe0 0x00 # CHECK: jalr $7
-0x09 0xfc 0x80 0x00 # CHECK: jalr.hb $4
-0x09 0x24 0xa0 0x00 # CHECK: jalr.hb $4, $5
-0x08 0x00 0xe0 0x00 # CHECK: jr $7
-0xc6 0x23 0xa4 0x80 # CHECK: lb $4, 9158($5)
-0x06 0x00 0xa4 0x90 # CHECK: lbu $4, 6($5)
-0xc6 0x23 0xe9 0xd4 # CHECK: ldc1 $f9, 9158($7)
-0x01 0x02 0xf7 0x4d # CHECK: ldxc1 $f8, $23($15)
-0x0c 0x00 0xa4 0x84 # CHECK: lh $4, 12($5)
-0x0c 0x00 0xa4 0x84 # CHECK: lh $4, 12($5)
-0xc6 0x23 0xe9 0xc0 # CHECK: ll $9, 9158($7)
-0x67 0x45 0x06 0x3c # CHECK: lui $6, 17767
-0x05 0x00 0xa6 0x4c # CHECK: luxc1 $f0, $6($5)
-0x18 0x00 0xa4 0x8c # CHECK: lw $4, 24($5)
-0xc6 0x23 0xe9 0xc4 # CHECK: lwc1 $f9, 9158($7)
-0x03 0x00 0x82 0x88 # CHECK: lwl $2, 3($4)
-0x10 0x00 0xa3 0x98 # CHECK: lwr $3, 16($5)
-0x00 0x05 0xcc 0x4d # CHECK: lwxc1 $f20, $12($14)
-0x00 0x00 0xc7 0x70 # CHECK: madd $6, $7
-0xa1 0xd4 0x94 0x4e # CHECK: madd.d $f18, $f20, $f26, $f20
-0x60 0x98 0xf9 0x4f # CHECK: madd.s $f1, $f31, $f19, $f25
-0x01 0x00 0xc7 0x70 # CHECK: maddu $6, $7
-0x00 0x38 0x06 0x44 # CHECK: mfc1 $6, $f7
-0x10 0x28 0x00 0x00 # CHECK: mfhi $5
-0x00 0xc0 0x7e 0x44 # CHECK: mfhc1 $fp, $f24
-0x12 0x28 0x00 0x00 # CHECK: mflo $5
-0x86 0x41 0x20 0x46 # CHECK: mov.d $f6, $f8
-0x86 0x39 0x00 0x46 # CHECK: mov.s $f6, $f7
-0x04 0x00 0xc7 0x70 # CHECK: msub $6, $7
-0xa9 0xf2 0x52 0x4c # CHECK: msub.d $f10, $f2, $f30, $f18
-0x28 0x53 0x70 0x4e # CHECK: msub.s $f12, $f19, $f10, $f16
-0x05 0x00 0xc7 0x70 # CHECK: msubu $6, $7
-0x00 0x38 0x86 0x44 # CHECK: mtc1 $6, $f7
-0x11 0x00 0xe0 0x00 # CHECK: mthi $7
-0x00 0x80 0xe0 0x44 # CHECK: mthc1 $zero, $f16
-0x13 0x00 0xe0 0x00 # CHECK: mtlo $7
-0x02 0x62 0x2e 0x46 # CHECK: mul.d $f8, $f12, $f14
-0x42 0x32 0x07 0x46 # CHECK: mul.s $f9, $f6, $f7
-0x02 0x48 0xc7 0x70 # CHECK: mul $9, $6, $7
-0x18 0x00 0x65 0x00 # CHECK: mult $3, $5
-0x19 0x00 0x65 0x00 # CHECK: multu $3, $5
-0x07 0x73 0x20 0x46 # CHECK: neg.d $f12, $f14
-0x87 0x39 0x00 0x46 # CHECK: neg.s $f6, $f7
-0xb1 0x74 0x54 0x4d # CHECK: nmadd.d $f18, $f10, $f14, $f20
-0x30 0xc8 0xac 0x4c # CHECK: nmadd.s $f0, $f5, $f25, $f12
-0x00 0x00 0x00 0x00 # CHECK: nop
-0x27 0x48 0xc7 0x00 # CHECK: nor $9, $6, $7
-0xb9 0x87 0x1e 0x4d # CHECK: nmsub.d $f30, $f8, $f16, $f30
-0x78 0x98 0x04 0x4f # CHECK: nmsub.s $f1, $f24, $f19, $f4
-0x25 0x18 0x65 0x00 # CHECK: or $3, $3, $5
-0x67 0x45 0xc9 0x34 # CHECK: ori $9, $6, 17767
-0xc2 0x49 0x26 0x00 # CHECK: rotr $9, $6, 7
-0x46 0x48 0xe6 0x00 # CHECK: rotrv $9, $6, $7
-0x0c 0x73 0x20 0x46 # CHECK: round.w.d $f12, $f14
-0x8c 0x39 0x00 0x46 # CHECK: round.w.s $f6, $f7
-0xc6 0x23 0xa4 0xa0 # CHECK: sb $4, 9158($5)
-0x06 0x00 0xa4 0xa0 # CHECK: sb $4, 6($5)
-0xc6 0x23 0xe9 0xe0 # CHECK: sc $9, 9158($7)
-0xc6 0x23 0xe9 0xf4 # CHECK: sdc1 $f9, 9158($7)
-0x09 0x40 0x24 0x4f # CHECK: sdxc1 $f8, $4($25)
-0x20 0x34 0x07 0x7c # CHECK: seb $6, $7
-0x20 0x36 0x07 0x7c # CHECK: seh $6, $7
-0xc6 0x23 0xa4 0xa4 # CHECK: sh $4, 9158($5)
-0xc0 0x21 0x03 0x00 # CHECK: sll $4, $3, 7
-0x04 0x10 0xa3 0x00 # CHECK: sllv $2, $3, $5
-0x2a 0x18 0x65 0x00 # CHECK: slt $3, $3, $5
-0x67 0x00 0x63 0x28 # CHECK: slti $3, $3, 103
-0x67 0x00 0x63 0x2c # CHECK: sltiu $3, $3, 103
-0x2b 0x18 0x65 0x00 # CHECK: sltu $3, $3, $5
-0x04 0x73 0x20 0x46 # CHECK: sqrt.d $f12, $f14
-0x84 0x39 0x00 0x46 # CHECK: sqrt.s $f6, $f7
-0xc3 0x21 0x03 0x00 # CHECK: sra $4, $3, 7
-0x07 0x10 0xa3 0x00 # CHECK: srav $2, $3, $5
-0xc2 0x21 0x03 0x00 # CHECK: srl $4, $3, 7
-0x06 0x10 0xa3 0x00 # CHECK: srlv $2, $3, $5
-0x01 0x62 0x2e 0x46 # CHECK: sub.d $f8, $f12, $f14
-0x41 0x32 0x07 0x46 # CHECK: sub.s $f9, $f6, $f7
-0x22 0x48 0xc7 0x00 # CHECK: sub $9, $6, $7
-0x23 0x20 0x65 0x00 # CHECK: subu $4, $3, $5
-0x0d 0x20 0xb8 0x4c # CHECK: suxc1 $f4, $24($5)
-0x18 0x00 0xa4 0xac # CHECK: sw $4, 24($5)
-0xc6 0x23 0xe9 0xe4 # CHECK: swc1 $f9, 9158($7)
-0x10 0x00 0xa4 0xa8 # CHECK: swl $4, 16($5)
-0x10 0x00 0xe6 0xb8 # CHECK: swr $6, 16($7)
-0x08 0xd0 0xd2 0x4e # CHECK: swxc1 $f26, $18($22)
-0xcf 0x01 0x00 0x00 # CHECK: sync 7
-0x0d 0x73 0x20 0x46 # CHECK: trunc.w.d $f12, $f14
-0x8d 0x39 0x00 0x46 # CHECK: trunc.w.s $f6, $f7
-0xa0 0x30 0x07 0x7c # CHECK: wsbh $6, $7
-0x26 0x18 0x65 0x00 # CHECK: xor $3, $3, $5
-0x67 0x45 0xc9 0x38 # CHECK: xori $9, $6, 17767
diff --git a/test/MC/Disassembler/Mips/mips32r2/valid-mips32r2.txt b/test/MC/Disassembler/Mips/mips32r2/valid-mips32r2.txt
index 96378357d19..d0138475290 100644
--- a/test/MC/Disassembler/Mips/mips32r2/valid-mips32r2.txt
+++ b/test/MC/Disassembler/Mips/mips32r2/valid-mips32r2.txt
@@ -2,171 +2,175 @@
 # Try a mips64* triple to confirm that mips* vs mips64* triples no longer have
 # an effect on the disassembler behaviour.
 # RUN: llvm-mc --disassemble %s -triple=mips64-unknown-linux -mcpu=mips32r2 | FileCheck %s
-0x46 0x20 0x73 0x05 # CHECK: abs.d $f12, $f14
-0x46 0x00 0x39 0x85 # CHECK: abs.s $f6, $f7
+0x00 0x00 0x00 0x00 # CHECK: nop
+0x00 0x00 0x01 0xcf # CHECK: sync 7
+0x00 0x00 0x28 0x10 # CHECK: mfhi $5
+0x00 0x00 0x28 0x12 # CHECK: mflo $5
+0x00 0x03 0x21 0xc0 # CHECK: sll $4, $3, 7
+0x00 0x03 0x21 0xc2 # CHECK: srl $4, $3, 7
+0x00 0x03 0x21 0xc3 # CHECK: sra $4, $3, 7
+0x00 0x26 0x49 0xc2 # CHECK: rotr $9, $6, 7
+0x00 0x65 0x00 0x18 # CHECK: mult $3, $5
+0x00 0x65 0x00 0x19 # CHECK: multu $3, $5
+0x00 0x65 0x18 0x25 # CHECK: or $3, $3, $5
+0x00 0x65 0x18 0x26 # CHECK: xor $3, $3, $5
+0x00 0x65 0x18 0x2a # CHECK: slt $3, $3, $5
+0x00 0x65 0x18 0x2b # CHECK: sltu $3, $3, $5
+0x00 0x65 0x20 0x23 # CHECK: subu $4, $3, $5
+0x00 0x80 0xfc 0x09 # CHECK: jalr.hb $4
+0x00 0xa0 0x24 0x09 # CHECK: jalr.hb $4, $5
+0x00 0xa3 0x10 0x04 # CHECK: sllv $2, $3, $5
+0x00 0xa3 0x10 0x06 # CHECK: srlv $2, $3, $5
+0x00 0xa3 0x10 0x07 # CHECK: srav $2, $3, $5
 0x00 0xc7 0x48 0x20 # CHECK: add $9, $6, $7
-0x46 0x2e 0x62 0x00 # CHECK: add.d $f8, $f12, $f14
-0x46 0x07 0x32 0x40 # CHECK: add.s $f9, $f6, $f7
-0x20 0xc9 0x45 0x67 # CHECK: addi $9, $6, 17767
-0x24 0xc9 0xc5 0x67 # CHECK: addiu $9, $6, -15001
 0x00 0xc7 0x48 0x21 # CHECK: addu $9, $6, $7
+0x00 0xc7 0x48 0x22 # CHECK: sub $9, $6, $7
 0x00 0xc7 0x48 0x24 # CHECK: and $9, $6, $7
-0x30 0xc9 0x45 0x67 # CHECK: andi $9, $6, 17767
+0x00 0xc7 0x48 0x27 # CHECK: nor $9, $6, $7
+0x00 0xe0 0x00 0x08 # CHECK: jr $7
+0x00 0xe0 0x00 0x11 # CHECK: mthi $7
+0x00 0xe0 0x00 0x13 # CHECK: mtlo $7
+0x00 0xe0 0xf8 0x09 # CHECK: jalr $7
+0x00 0xe6 0x48 0x46 # CHECK: rotrv $9, $6, $7
+0x04 0xc1 0x01 0x4c # CHECK: bgez $6, 1332
+0x04 0xd1 0x01 0x4c # CHECK: bgezal $6, 1332
+0x07 0xdf 0xe8 0x07 # CHECK: synci -6137($fp)
+0x08 0x00 0x01 0x4c # CHECK: j 1328
+0x0c 0x00 0x01 0x4c # CHECK: jal 1328
 0x10 0x00 0x01 0x4c # CHECK: b 1332
+0x11 0x26 0x01 0x4c # CHECK: beq $9, $6, 1332
+0x15 0x26 0x01 0x4c # CHECK: bne $9, $6, 1332
+0x18 0xc0 0x01 0x4c # CHECK: blez $6, 1332
+0x1c 0xc0 0x01 0x4c # CHECK: bgtz $6, 1332
+0x20 0xc9 0x45 0x67 # CHECK: addi $9, $6, 17767
+0x24 0xc9 0xc5 0x67 # CHECK: addiu $9, $6, -15001
+0x28 0x63 0x00 0x67 # CHECK: slti $3, $3, 103
+0x2c 0x63 0x00 0x67 # CHECK: sltiu $3, $3, 103
+0x30 0xc9 0x45 0x67 # CHECK: andi $9, $6, 17767
+0x34 0xc9 0x45 0x67 # CHECK: ori $9, $6, 17767
+0x38 0xc9 0x45 0x67 # CHECK: xori $9, $6, 17767
+0x3c 0x06 0x45 0x67 # CHECK: lui $6, 17767
+0x40 0x08 0x78 0x01 # CHECK: mfc0 $8, $15, 1
+0x40 0x08 0x80 0x04 # CHECK: mfc0 $8, $16, 4
+0x40 0x89 0x78 0x01 # CHECK: mtc0 $9, $15, 1
+0x41 0x60 0x60 0x00 # CHECK: di
+0x41 0x60 0x60 0x20 # CHECK: ei
+0x41 0x6e 0x60 0x20 # CHECK: ei $14
+0x41 0x7e 0x60 0x00 # CHECK: di $fp
+0x44 0x06 0x38 0x00 # CHECK: mfc1 $6, $f7
+0x44 0x46 0x38 0x00 # CHECK: cfc1 $6, $7
+0x44 0x7e 0xc0 0x00 # CHECK: mfhc1 $fp, $f24
+0x44 0x86 0x38 0x00 # CHECK: mtc1 $6, $f7
+0x44 0xc6 0x38 0x00 # CHECK: ctc1 $6, $7
+0x44 0xe0 0x80 0x00 # CHECK: mthc1 $zero, $f16
 0x45 0x00 0x01 0x4c # CHECK: bc1f 1332
-0x45 0x1c 0x01 0x4c # CHECK: bc1f $fcc7, 1332
 0x45 0x01 0x01 0x4c # CHECK: bc1t 1332
+0x45 0x1c 0x01 0x4c # CHECK: bc1f $fcc7, 1332
 0x45 0x1d 0x01 0x4c # CHECK: bc1t $fcc7, 1332
-0x11 0x26 0x01 0x4c # CHECK: beq $9, $6, 1332
-0x04 0xc1 0x01 0x4c # CHECK: bgez $6, 1332
-0x04 0xd1 0x01 0x4c # CHECK: bgezal $6, 1332
-0x1c 0xc0 0x01 0x4c # CHECK: bgtz $6, 1332
-0x18 0xc0 0x01 0x4c # CHECK: blez $6, 1332
-0x15 0x26 0x01 0x4c # CHECK: bne $9, $6, 1332
-0x46 0x2e 0x60 0x32 # CHECK: c.eq.d $f12, $f14
-0x46 0x07 0x30 0x32 # CHECK: c.eq.s $f6, $f7
-0x46 0x2e 0x60 0x30 # CHECK: c.f.d $f12, $f14
+0x46 0x00 0x39 0x84 # CHECK: sqrt.s $f6, $f7
+0x46 0x00 0x39 0x85 # CHECK: abs.s $f6, $f7
+0x46 0x00 0x39 0x86 # CHECK: mov.s $f6, $f7
+0x46 0x00 0x39 0x87 # CHECK: neg.s $f6, $f7
+0x46 0x00 0x39 0x8c # CHECK: round.w.s $f6, $f7
+0x46 0x00 0x39 0x8d # CHECK: trunc.w.s $f6, $f7
+0x46 0x00 0x39 0x8e # CHECK: ceil.w.s $f6, $f7
+0x46 0x00 0x39 0x8f # CHECK: floor.w.s $f6, $f7
+0x46 0x00 0x39 0xa1 # CHECK: cvt.d.s $f6, $f7
+0x46 0x00 0x39 0xa4 # CHECK: cvt.w.s $f6, $f7
+0x46 0x00 0x39 0xa5 # CHECK: cvt.l.s $f6, $f7
 0x46 0x07 0x30 0x30 # CHECK: c.f.s $f6, $f7
-0x46 0x2e 0x60 0x3e # CHECK: c.le.d $f12, $f14
-0x46 0x07 0x30 0x3e # CHECK: c.le.s $f6, $f7
-0x46 0x2e 0x60 0x3c # CHECK: c.lt.d $f12, $f14
+0x46 0x07 0x30 0x31 # CHECK: c.un.s $f6, $f7
+0x46 0x07 0x30 0x32 # CHECK: c.eq.s $f6, $f7
+0x46 0x07 0x30 0x34 # CHECK: c.olt.s $f6, $f7
+0x46 0x07 0x30 0x35 # CHECK: c.ult.s $f6, $f7
+0x46 0x07 0x30 0x36 # CHECK: c.ole.s $f6, $f7
+0x46 0x07 0x30 0x37 # CHECK: c.ule.s $f6, $f7
+0x46 0x07 0x30 0x38 # CHECK: c.sf.s $f6, $f7
+0x46 0x07 0x30 0x39 # CHECK: c.ngle.s $f6, $f7
+0x46 0x07 0x30 0x3a # CHECK: c.seq.s $f6, $f7
+0x46 0x07 0x30 0x3b # CHECK: c.ngl.s $f6, $f7
 0x46 0x07 0x30 0x3c # CHECK: c.lt.s $f6, $f7
-0x46 0x2e 0x60 0x3d # CHECK: c.nge.d $f12, $f14
 0x46 0x07 0x30 0x3d # CHECK: c.nge.s $f6, $f7
-0x46 0x2e 0x60 0x3b # CHECK: c.ngl.d $f12, $f14
-0x46 0x07 0x30 0x3b # CHECK: c.ngl.s $f6, $f7
-0x46 0x2e 0x60 0x39 # CHECK: c.ngle.d $f12, $f14
-0x46 0x07 0x30 0x39 # CHECK: c.ngle.s $f6, $f7
-0x46 0x2e 0x60 0x3f # CHECK: c.ngt.d $f12, $f14
+0x46 0x07 0x30 0x3e # CHECK: c.le.s $f6, $f7
 0x46 0x07 0x30 0x3f # CHECK: c.ngt.s $f6, $f7
-0x46 0x2e 0x60 0x36 # CHECK: c.ole.d $f12, $f14
-0x46 0x07 0x30 0x36 # CHECK: c.ole.s $f6, $f7
-0x46 0x2e 0x60 0x34 # CHECK: c.olt.d $f12, $f14
-0x46 0x07 0x30 0x34 # CHECK: c.olt.s $f6, $f7
-0x46 0x2e 0x60 0x3a # CHECK: c.seq.d $f12, $f14
-0x46 0x07 0x30 0x3a # CHECK: c.seq.s $f6, $f7
-0x46 0x2e 0x60 0x38 # CHECK: c.sf.d $f12, $f14
-0x46 0x07 0x30 0x38 # CHECK: c.sf.s $f6, $f7
-0x46 0x2e 0x60 0x33 # CHECK: c.ueq.d $f12, $f14
+0x46 0x07 0x32 0x40 # CHECK: add.s $f9, $f6, $f7
+0x46 0x07 0x32 0x41 # CHECK: sub.s $f9, $f6, $f7
+0x46 0x07 0x32 0x42 # CHECK: mul.s $f9, $f6, $f7
 0x46 0x12 0xe0 0x33 # CHECK: c.ueq.s $f28, $f18
-0x46 0x2e 0x60 0x37 # CHECK: c.ule.d $f12, $f14
-0x46 0x07 0x30 0x37 # CHECK: c.ule.s $f6, $f7
-0x46 0x2e 0x60 0x35 # CHECK: c.ult.d $f12, $f14
-0x46 0x07 0x30 0x35 # CHECK: c.ult.s $f6, $f7
-0x46 0x2e 0x60 0x31 # CHECK: c.un.d $f12, $f14
-0x46 0x07 0x30 0x31 # CHECK: c.un.s $f6, $f7
+0x46 0x20 0x41 0x86 # CHECK: mov.d $f6, $f8
+0x46 0x20 0x73 0x04 # CHECK: sqrt.d $f12, $f14
+0x46 0x20 0x73 0x05 # CHECK: abs.d $f12, $f14
+0x46 0x20 0x73 0x07 # CHECK: neg.d $f12, $f14
+0x46 0x20 0x73 0x0c # CHECK: round.w.d $f12, $f14
+0x46 0x20 0x73 0x0d # CHECK: trunc.w.d $f12, $f14
 0x46 0x20 0x73 0x0e # CHECK: ceil.w.d $f12, $f14
-0x46 0x00 0x39 0x8e # CHECK: ceil.w.s $f6, $f7
-0x44 0x46 0x38 0x00 # CHECK: cfc1 $6, $7
-0x70 0xe6 0x30 0x21 # CHECK: clo $6, $7
-0x70 0xe6 0x30 0x20 # CHECK: clz $6, $7
-0x44 0xc6 0x38 0x00 # CHECK: ctc1 $6, $7
-0x46 0x00 0x39 0xa1 # CHECK: cvt.d.s $f6, $f7
-0x46 0x80 0x73 0x21 # CHECK: cvt.d.w $f12, $f14
-0x46 0x20 0x73 0x25 # CHECK: cvt.l.d $f12, $f14
-0x46 0x00 0x39 0xa5 # CHECK: cvt.l.s $f6, $f7
+0x46 0x20 0x73 0x0f # CHECK: floor.w.d $f12, $f14
 0x46 0x20 0x73 0x20 # CHECK: cvt.s.d $f12, $f14
-0x46 0x80 0x39 0xa0 # CHECK: cvt.s.w $f6, $f7
 0x46 0x20 0x73 0x24 # CHECK: cvt.w.d $f12, $f14
-0x46 0x00 0x39 0xa4 # CHECK: cvt.w.s $f6, $f7
-0x41 0x7e 0x60 0x00 # CHECK: di $fp
-0x41 0x60 0x60 0x00 # CHECK: di
-0x41 0x6e 0x60 0x20 # CHECK: ei $14
-0x41 0x60 0x60 0x20 # CHECK: ei
-0x46 0x20 0x73 0x0f # CHECK: floor.w.d $f12, $f14
-0x46 0x00 0x39 0x8f # CHECK: floor.w.s $f6, $f7
-0x7d 0x33 0x61 0x84 # CHECK: ins $19, $9, 6, 7
-0x08 0x00 0x01 0x4c # CHECK: j 1328
-0x0c 0x00 0x01 0x4c # CHECK: jal 1328
-0x74 0x00 0x01 0x4c # CHECK: jalx 1328
-0x00 0xe0 0xf8 0x09 # CHECK: jalr $7
-0x00 0x80 0xfc 0x09 # CHECK: jalr.hb $4
-0x00 0xa0 0x24 0x09 # CHECK: jalr.hb $4, $5
-0x00 0xe0 0x00 0x08 # CHECK: jr $7
-0x80 0xa4 0x23 0xc6 # CHECK: lb $4, 9158($5)
-0x90 0xa4 0x00 0x06 # CHECK: lbu $4, 6($5)
-0xd4 0xe9 0x23 0xc6 # CHECK: ldc1 $f9, 9158($7)
-0x4d 0xf7 0x02 0x01 # CHECK: ldxc1 $f8, $23($15)
-0x84 0xa4 0x00 0x0c # CHECK: lh $4, 12($5)
-0x84 0xa4 0x00 0x0c # CHECK: lh $4, 12($5)
-0xc0 0xe9 0x23 0xc6 # CHECK: ll $9, 9158($7)
-0x3c 0x06 0x45 0x67 # CHECK: lui $6, 17767
+0x46 0x20 0x73 0x25 # CHECK: cvt.l.d $f12, $f14
+0x46 0x2e 0x60 0x30 # CHECK: c.f.d $f12, $f14
+0x46 0x2e 0x60 0x31 # CHECK: c.un.d $f12, $f14
+0x46 0x2e 0x60 0x32 # CHECK: c.eq.d $f12, $f14
+0x46 0x2e 0x60 0x33 # CHECK: c.ueq.d $f12, $f14
+0x46 0x2e 0x60 0x34 # CHECK: c.olt.d $f12, $f14
+0x46 0x2e 0x60 0x35 # CHECK: c.ult.d $f12, $f14
+0x46 0x2e 0x60 0x36 # CHECK: c.ole.d $f12, $f14
+0x46 0x2e 0x60 0x37 # CHECK: c.ule.d $f12, $f14
+0x46 0x2e 0x60 0x38 # CHECK: c.sf.d $f12, $f14
+0x46 0x2e 0x60 0x39 # CHECK: c.ngle.d $f12, $f14
+0x46 0x2e 0x60 0x3a # CHECK: c.seq.d $f12, $f14
+0x46 0x2e 0x60 0x3b # CHECK: c.ngl.d $f12, $f14
+0x46 0x2e 0x60 0x3c # CHECK: c.lt.d $f12, $f14
+0x46 0x2e 0x60 0x3d # CHECK: c.nge.d $f12, $f14
+0x46 0x2e 0x60 0x3e # CHECK: c.le.d $f12, $f14
+0x46 0x2e 0x60 0x3f # CHECK: c.ngt.d $f12, $f14
+0x46 0x2e 0x62 0x00 # CHECK: add.d $f8, $f12, $f14
+0x46 0x2e 0x62 0x01 # CHECK: sub.d $f8, $f12, $f14
+0x46 0x2e 0x62 0x02 # CHECK: mul.d $f8, $f12, $f14
+0x46 0x80 0x39 0xa0 # CHECK: cvt.s.w $f6, $f7
+0x46 0x80 0x73 0x21 # CHECK: cvt.d.w $f12, $f14
+0x4c 0x52 0xf2 0xa9 # CHECK: msub.d $f10, $f2, $f30, $f18
 0x4c 0xa6 0x00 0x05 # CHECK: luxc1 $f0, $6($5)
-0x8c 0xa4 0x00 0x18 # CHECK: lw $4, 24($5)
-0xc4 0xe9 0x23 0xc6 # CHECK: lwc1 $f9, 9158($7)
-0x88 0x82 0x00 0x03 # CHECK: lwl $2, 3($4)
-0x98 0xa3 0x00 0x10 # CHECK: lwr $3, 16($5)
+0x4c 0xac 0xc8 0x30 # CHECK: nmadd.s $f0, $f5, $f25, $f12
+0x4c 0xb8 0x20 0x0d # CHECK: suxc1 $f4, $24($5)
+0x4d 0x1e 0x87 0xb9 # CHECK: nmsub.d $f30, $f8, $f16, $f30
+0x4d 0x54 0x74 0xb1 # CHECK: nmadd.d $f18, $f10, $f14, $f20
 0x4d 0xcc 0x05 0x00 # CHECK: lwxc1 $f20, $12($14)
-0x70 0xc7 0x00 0x00 # CHECK: madd $6, $7
+0x4d 0xf7 0x02 0x01 # CHECK: ldxc1 $f8, $23($15)
+0x4e 0x70 0x53 0x28 # CHECK: msub.s $f12, $f19, $f10, $f16
 0x4e 0x94 0xd4 0xa1 # CHECK: madd.d $f18, $f20, $f26, $f20
+0x4e 0xd2 0xd0 0x08 # CHECK: swxc1 $f26, $18($22)
+0x4f 0x04 0x98 0x78 # CHECK: nmsub.s $f1, $f24, $f19, $f4
+0x4f 0x24 0x40 0x09 # CHECK: sdxc1 $f8, $4($25)
 0x4f 0xf9 0x98 0x60 # CHECK: madd.s $f1, $f31, $f19, $f25
+0x70 0xc7 0x00 0x00 # CHECK: madd $6, $7
 0x70 0xc7 0x00 0x01 # CHECK: maddu $6, $7
-0x44 0x06 0x38 0x00 # CHECK: mfc1 $6, $f7
-0x00 0x00 0x28 0x10 # CHECK: mfhi $5
-0x44 0x7e 0xc0 0x00 # CHECK: mfhc1 $fp, $f24
-0x00 0x00 0x28 0x12 # CHECK: mflo $5
-0x46 0x20 0x41 0x86 # CHECK: mov.d $f6, $f8
-0x46 0x00 0x39 0x86 # CHECK: mov.s $f6, $f7
 0x70 0xc7 0x00 0x04 # CHECK: msub $6, $7
-0x4c 0x52 0xf2 0xa9 # CHECK: msub.d $f10, $f2, $f30, $f18
-0x4e 0x70 0x53 0x28 # CHECK: msub.s $f12, $f19, $f10, $f16
 0x70 0xc7 0x00 0x05 # CHECK: msubu $6, $7
-0x44 0x86 0x38 0x00 # CHECK: mtc1 $6, $f7
-0x00 0xe0 0x00 0x11 # CHECK: mthi $7
-0x44 0xe0 0x80 0x00 # CHECK: mthc1 $zero, $f16
-0x00 0xe0 0x00 0x13 # CHECK: mtlo $7
-0x46 0x2e 0x62 0x02 # CHECK: mul.d $f8, $f12, $f14
-0x46 0x07 0x32 0x42 # CHECK: mul.s $f9, $f6, $f7
 0x70 0xc7 0x48 0x02 # CHECK: mul $9, $6, $7
-0x00 0x65 0x00 0x18 # CHECK: mult $3, $5
-0x00 0x65 0x00 0x19 # CHECK: multu $3, $5
-0x46 0x20 0x73 0x07 # CHECK: neg.d $f12, $f14
-0x46 0x00 0x39 0x87 # CHECK: neg.s $f6, $f7
-0x4d 0x54 0x74 0xb1 # CHECK: nmadd.d $f18, $f10, $f14, $f20
-0x4c 0xac 0xc8 0x30 # CHECK: nmadd.s $f0, $f5, $f25, $f12
-0x00 0x00 0x00 0x00 # CHECK: nop
-0x00 0xc7 0x48 0x27 # CHECK: nor $9, $6, $7
-0x4d 0x1e 0x87 0xb9 # CHECK: nmsub.d $f30, $f8, $f16, $f30
-0x4f 0x04 0x98 0x78 # CHECK: nmsub.s $f1, $f24, $f19, $f4
-0x00 0x65 0x18 0x25 # CHECK: or $3, $3, $5
-0x34 0xc9 0x45 0x67 # CHECK: ori $9, $6, 17767
-0x00 0x26 0x49 0xc2 # CHECK: rotr $9, $6, 7
-0x00 0xe6 0x48 0x46 # CHECK: rotrv $9, $6, $7
-0x46 0x20 0x73 0x0c # CHECK: round.w.d $f12, $f14
-0x46 0x00 0x39 0x8c # CHECK: round.w.s $f6, $f7
-0xa0 0xa4 0x23 0xc6 # CHECK: sb $4, 9158($5)
-0xa0 0xa4 0x00 0x06 # CHECK: sb $4, 6($5)
-0xe0 0xe9 0x23 0xc6 # CHECK: sc $9, 9158($7)
-0xf4 0xe9 0x23 0xc6 # CHECK: sdc1 $f9, 9158($7)
-0x4f 0x24 0x40 0x09 # CHECK: sdxc1 $f8, $4($25)
+0x70 0xe6 0x30 0x20 # CHECK: clz $6, $7
+0x70 0xe6 0x30 0x21 # CHECK: clo $6, $7
+0x74 0x00 0x01 0x4c # CHECK: jalx 1328
+0x7c 0x07 0x30 0xa0 # CHECK: wsbh $6, $7
 0x7c 0x07 0x34 0x20 # CHECK: seb $6, $7
 0x7c 0x07 0x36 0x20 # CHECK: seh $6, $7
+0x7d 0x33 0x61 0x84 # CHECK: ins $19, $9, 6, 7
+0x80 0xa4 0x23 0xc6 # CHECK: lb $4, 9158($5)
+0x84 0xa4 0x00 0x0c # CHECK: lh $4, 12($5)
+0x84 0xa4 0x00 0x0c # CHECK: lh $4, 12($5)
+0x88 0x82 0x00 0x03 # CHECK: lwl $2, 3($4)
+0x8c 0xa4 0x00 0x18 # CHECK: lw $4, 24($5)
+0x90 0xa4 0x00 0x06 # CHECK: lbu $4, 6($5)
+0x98 0xa3 0x00 0x10 # CHECK: lwr $3, 16($5)
+0xa0 0xa4 0x00 0x06 # CHECK: sb $4, 6($5)
+0xa0 0xa4 0x23 0xc6 # CHECK: sb $4, 9158($5)
 0xa4 0xa4 0x23 0xc6 # CHECK: sh $4, 9158($5)
-0x00 0x03 0x21 0xc0 # CHECK: sll $4, $3, 7
-0x00 0xa3 0x10 0x04 # CHECK: sllv $2, $3, $5
-0x00 0x65 0x18 0x2a # CHECK: slt $3, $3, $5
-0x28 0x63 0x00 0x67 # CHECK: slti $3, $3, 103
-0x2c 0x63 0x00 0x67 # CHECK: sltiu $3, $3, 103
-0x00 0x65 0x18 0x2b # CHECK: sltu $3, $3, $5
-0x46 0x20 0x73 0x04 # CHECK: sqrt.d $f12, $f14
-0x46 0x00 0x39 0x84 # CHECK: sqrt.s $f6, $f7
-0x00 0x03 0x21 0xc3 # CHECK: sra $4, $3, 7
-0x00 0xa3 0x10 0x07 # CHECK: srav $2, $3, $5
-0x00 0x03 0x21 0xc2 # CHECK: srl $4, $3, 7
-0x00 0xa3 0x10 0x06 # CHECK: srlv $2, $3, $5
-0x46 0x2e 0x62 0x01 # CHECK: sub.d $f8, $f12, $f14
-0x46 0x07 0x32 0x41 # CHECK: sub.s $f9, $f6, $f7
-0x00 0xc7 0x48 0x22 # CHECK: sub $9, $6, $7
-0x00 0x65 0x20 0x23 # CHECK: subu $4, $3, $5
-0x4c 0xb8 0x20 0x0d # CHECK: suxc1 $f4, $24($5)
-0xac 0xa4 0x00 0x18 # CHECK: sw $4, 24($5)
-0xe4 0xe9 0x23 0xc6 # CHECK: swc1 $f9, 9158($7)
 0xa8 0xa4 0x00 0x10 # CHECK: swl $4, 16($5)
+0xac 0xa4 0x00 0x18 # CHECK: sw $4, 24($5)
 0xb8 0xe6 0x00 0x10 # CHECK: swr $6, 16($7)
-0x4e 0xd2 0xd0 0x08 # CHECK: swxc1 $f26, $18($22)
-0x00 0x00 0x01 0xcf # CHECK: sync 7
-0x46 0x20 0x73 0x0d # CHECK: trunc.w.d $f12, $f14
-0x46 0x00 0x39 0x8d # CHECK: trunc.w.s $f6, $f7
-0x7c 0x07 0x30 0xa0 # CHECK: wsbh $6, $7
-0x00 0x65 0x18 0x26 # CHECK: xor $3, $3, $5
-0x38 0xc9 0x45 0x67 # CHECK: xori $9, $6, 17767
+0xc0 0xe9 0x23 0xc6 # CHECK: ll $9, 9158($7)
+0xc4 0xe9 0x23 0xc6 # CHECK: lwc1 $f9, 9158($7)
+0xd4 0xe9 0x23 0xc6 # CHECK: ldc1 $f9, 9158($7)
+0xe0 0xe9 0x23 0xc6 # CHECK: sc $9, 9158($7)
+0xe4 0xe9 0x23 0xc6 # CHECK: swc1 $f9, 9158($7)
+0xf4 0xe9 0x23 0xc6 # CHECK: sdc1 $f9, 9158($7)
diff --git a/test/MC/Disassembler/Mips/mips32r2_le.txt b/test/MC/Disassembler/Mips/mips32r2_le.txt
index 81a05b330fd..faaed7cb345 100644
--- a/test/MC/Disassembler/Mips/mips32r2_le.txt
+++ b/test/MC/Disassembler/Mips/mips32r2_le.txt
@@ -269,6 +269,9 @@
 # CHECK: maddu  $6,  $7
 0x01 0x00 0xc7 0x70
 
+# CHECK: mfc0 $8, $16, 4
+0x04 0x80 0x08 0x40
+
 # CHECK: mfc1   $6, $f7
 0x00 0x38 0x06 0x44
 
@@ -290,6 +293,9 @@
 # CHECK: msubu  $6,  $7
 0x05 0x00 0xc7 0x70
 
+# CHECK: mtc0 $9, $15, 1
+0x01 0x78 0x89 0x40
+
 # CHECK: mtc1   $6, $f7
 0x00 0x38 0x86 0x44
 
diff --git a/test/MC/Disassembler/Mips/mips32r3/valid-mips32r3-el.txt b/test/MC/Disassembler/Mips/mips32r3/valid-mips32r3-el.txt
new file mode 100644
index 00000000000..37c14de4cb6
--- /dev/null
+++ b/test/MC/Disassembler/Mips/mips32r3/valid-mips32r3-el.txt
@@ -0,0 +1,171 @@
+# RUN: llvm-mc --disassemble %s -triple=mipsel-unknown-linux -mcpu=mips32r3 | FileCheck %s
+0x05 0x73 0x20 0x46 # CHECK: abs.d $f12, $f14
+0x85 0x39 0x00 0x46 # CHECK: abs.s $f6, $f7
+0x20 0x48 0xc7 0x00 # CHECK: add $9, $6, $7
+0x00 0x62 0x2e 0x46 # CHECK: add.d $f8, $f12, $f14
+0x40 0x32 0x07 0x46 # CHECK: add.s $f9, $f6, $f7
+0x67 0x45 0xc9 0x20 # CHECK: addi $9, $6, 17767
+0x67 0xc5 0xc9 0x24 # CHECK: addiu $9, $6, -15001
+0x21 0x48 0xc7 0x00 # CHECK: addu $9, $6, $7
+0x24 0x48 0xc7 0x00 # CHECK: and $9, $6, $7
+0x67 0x45 0xc9 0x30 # CHECK: andi $9, $6, 17767
+0x4c 0x01 0x00 0x10 # CHECK: b 1332
+0x4c 0x01 0x00 0x45 # CHECK: bc1f 1332
+0x4c 0x01 0x1c 0x45 # CHECK: bc1f $fcc7, 1332
+0x4c 0x01 0x01 0x45 # CHECK: bc1t 1332
+0x4c 0x01 0x1d 0x45 # CHECK: bc1t $fcc7, 1332
+0x4c 0x01 0x26 0x11 # CHECK: beq $9, $6, 1332
+0x4c 0x01 0xc1 0x04 # CHECK: bgez $6, 1332
+0x4c 0x01 0xd1 0x04 # CHECK: bgezal $6, 1332
+0x4c 0x01 0xc0 0x1c # CHECK: bgtz $6, 1332
+0x4c 0x01 0xc0 0x18 # CHECK: blez $6, 1332
+0x4c 0x01 0x26 0x15 # CHECK: bne $9, $6, 1332
+0x32 0x60 0x2e 0x46 # CHECK: c.eq.d $f12, $f14
+0x32 0x30 0x07 0x46 # CHECK: c.eq.s $f6, $f7
+0x30 0x60 0x2e 0x46 # CHECK: c.f.d $f12, $f14
+0x30 0x30 0x07 0x46 # CHECK: c.f.s $f6, $f7
+0x3e 0x60 0x2e 0x46 # CHECK: c.le.d $f12, $f14
+0x3e 0x30 0x07 0x46 # CHECK: c.le.s $f6, $f7
+0x3c 0x60 0x2e 0x46 # CHECK: c.lt.d $f12, $f14
+0x3c 0x30 0x07 0x46 # CHECK: c.lt.s $f6, $f7
+0x3d 0x60 0x2e 0x46 # CHECK: c.nge.d $f12, $f14
+0x3d 0x30 0x07 0x46 # CHECK: c.nge.s $f6, $f7
+0x3b 0x60 0x2e 0x46 # CHECK: c.ngl.d $f12, $f14
+0x3b 0x30 0x07 0x46 # CHECK: c.ngl.s $f6, $f7
+0x39 0x60 0x2e 0x46 # CHECK: c.ngle.d $f12, $f14
+0x39 0x30 0x07 0x46 # CHECK: c.ngle.s $f6, $f7
+0x3f 0x60 0x2e 0x46 # CHECK: c.ngt.d $f12, $f14
+0x3f 0x30 0x07 0x46 # CHECK: c.ngt.s $f6, $f7
+0x36 0x60 0x2e 0x46 # CHECK: c.ole.d $f12, $f14
+0x36 0x30 0x07 0x46 # CHECK: c.ole.s $f6, $f7
+0x34 0x60 0x2e 0x46 # CHECK: c.olt.d $f12, $f14
+0x34 0x30 0x07 0x46 # CHECK: c.olt.s $f6, $f7
+0x3a 0x60 0x2e 0x46 # CHECK: c.seq.d $f12, $f14
+0x3a 0x30 0x07 0x46 # CHECK: c.seq.s $f6, $f7
+0x38 0x60 0x2e 0x46 # CHECK: c.sf.d $f12, $f14
+0x38 0x30 0x07 0x46 # CHECK: c.sf.s $f6, $f7
+0x33 0x60 0x2e 0x46 # CHECK: c.ueq.d $f12, $f14
+0x33 0xe0 0x12 0x46 # CHECK: c.ueq.s $f28, $f18
+0x37 0x60 0x2e 0x46 # CHECK: c.ule.d $f12, $f14
+0x37 0x30 0x07 0x46 # CHECK: c.ule.s $f6, $f7
+0x35 0x60 0x2e 0x46 # CHECK: c.ult.d $f12, $f14
+0x35 0x30 0x07 0x46 # CHECK: c.ult.s $f6, $f7
+0x31 0x60 0x2e 0x46 # CHECK: c.un.d $f12, $f14
+0x31 0x30 0x07 0x46 # CHECK: c.un.s $f6, $f7
+0x0e 0x73 0x20 0x46 # CHECK: ceil.w.d $f12, $f14
+0x8e 0x39 0x00 0x46 # CHECK: ceil.w.s $f6, $f7
+0x00 0x38 0x46 0x44 # CHECK: cfc1 $6, $7
+0x21 0x30 0xe6 0x70 # CHECK: clo $6, $7
+0x20 0x30 0xe6 0x70 # CHECK: clz $6, $7
+0x00 0x38 0xc6 0x44 # CHECK: ctc1 $6, $7
+0xa1 0x39 0x00 0x46 # CHECK: cvt.d.s $f6, $f7
+0x21 0x73 0x80 0x46 # CHECK: cvt.d.w $f12, $f14
+0x25 0x73 0x20 0x46 # CHECK: cvt.l.d $f12, $f14
+0xa5 0x39 0x00 0x46 # CHECK: cvt.l.s $f6, $f7
+0x20 0x73 0x20 0x46 # CHECK: cvt.s.d $f12, $f14
+0xa0 0x39 0x80 0x46 # CHECK: cvt.s.w $f6, $f7
+0x24 0x73 0x20 0x46 # CHECK: cvt.w.d $f12, $f14
+0xa4 0x39 0x00 0x46 # CHECK: cvt.w.s $f6, $f7
+0x00 0x60 0x7e 0x41 # CHECK: di $fp
+0x00 0x60 0x60 0x41 # CHECK: di
+0x20 0x60 0x6e 0x41 # CHECK: ei $14
+0x20 0x60 0x60 0x41 # CHECK: ei
+0x0f 0x73 0x20 0x46 # CHECK: floor.w.d $f12, $f14
+0x8f 0x39 0x00 0x46 # CHECK: floor.w.s $f6, $f7
+0x84 0x61 0x33 0x7d # CHECK: ins $19, $9, 6, 7
+0x4c 0x01 0x00 0x08 # CHECK: j 1328
+0x4c 0x01 0x00 0x0c # CHECK: jal 1328
+0x4c 0x01 0x00 0x74 # CHECK: jalx 1328
+0x09 0xf8 0xe0 0x00 # CHECK: jalr $7
+0x09 0xfc 0x80 0x00 # CHECK: jalr.hb $4
+0x09 0x24 0xa0 0x00 # CHECK: jalr.hb $4, $5
+0x08 0x00 0xe0 0x00 # CHECK: jr $7
+0xc6 0x23 0xa4 0x80 # CHECK: lb $4, 9158($5)
+0x06 0x00 0xa4 0x90 # CHECK: lbu $4, 6($5)
+0xc6 0x23 0xe9 0xd4 # CHECK: ldc1 $f9, 9158($7)
+0x01 0x02 0xf7 0x4d # CHECK: ldxc1 $f8, $23($15)
+0x0c 0x00 0xa4 0x84 # CHECK: lh $4, 12($5)
+0x0c 0x00 0xa4 0x84 # CHECK: lh $4, 12($5)
+0xc6 0x23 0xe9 0xc0 # CHECK: ll $9, 9158($7)
+0x67 0x45 0x06 0x3c # CHECK: lui $6, 17767
+0x05 0x00 0xa6 0x4c # CHECK: luxc1 $f0, $6($5)
+0x18 0x00 0xa4 0x8c # CHECK: lw $4, 24($5)
+0xc6 0x23 0xe9 0xc4 # CHECK: lwc1 $f9, 9158($7)
+0x03 0x00 0x82 0x88 # CHECK: lwl $2, 3($4)
+0x10 0x00 0xa3 0x98 # CHECK: lwr $3, 16($5)
+0x00 0x05 0xcc 0x4d # CHECK: lwxc1 $f20, $12($14)
+0x00 0x00 0xc7 0x70 # CHECK: madd $6, $7
+0xa1 0xd4 0x94 0x4e # CHECK: madd.d $f18, $f20, $f26, $f20
+0x60 0x98 0xf9 0x4f # CHECK: madd.s $f1, $f31, $f19, $f25
+0x01 0x00 0xc7 0x70 # CHECK: maddu $6, $7
+0x01 0x78 0x08 0x40 # CHECK: mfc0 $8, $15, 1
+0x00 0x38 0x06 0x44 # CHECK: mfc1 $6, $f7
+0x10 0x28 0x00 0x00 # CHECK: mfhi $5
+0x00 0xc0 0x7e 0x44 # CHECK: mfhc1 $fp, $f24
+0x12 0x28 0x00 0x00 # CHECK: mflo $5
+0x86 0x41 0x20 0x46 # CHECK: mov.d $f6, $f8
+0x86 0x39 0x00 0x46 # CHECK: mov.s $f6, $f7
+0x04 0x00 0xc7 0x70 # CHECK: msub $6, $7
+0xa9 0xf2 0x52 0x4c # CHECK: msub.d $f10, $f2, $f30, $f18
+0x28 0x53 0x70 0x4e # CHECK: msub.s $f12, $f19, $f10, $f16
+0x05 0x00 0xc7 0x70 # CHECK: msubu $6, $7
+0x01 0x78 0x89 0x40 # CHECK: mtc0 $9, $15, 1
+0x00 0x38 0x86 0x44 # CHECK: mtc1 $6, $f7
+0x11 0x00 0xe0 0x00 # CHECK: mthi $7
+0x00 0x80 0xe0 0x44 # CHECK: mthc1 $zero, $f16
+0x13 0x00 0xe0 0x00 # CHECK: mtlo $7
+0x02 0x62 0x2e 0x46 # CHECK: mul.d $f8, $f12, $f14
+0x42 0x32 0x07 0x46 # CHECK: mul.s $f9, $f6, $f7
+0x02 0x48 0xc7 0x70 # CHECK: mul $9, $6, $7
+0x18 0x00 0x65 0x00 # CHECK: mult $3, $5
+0x19 0x00 0x65 0x00 # CHECK: multu $3, $5
+0x07 0x73 0x20 0x46 # CHECK: neg.d $f12, $f14
+0x87 0x39 0x00 0x46 # CHECK: neg.s $f6, $f7
+0xb1 0x74 0x54 0x4d # CHECK: nmadd.d $f18, $f10, $f14, $f20
+0x30 0xc8 0xac 0x4c # CHECK: nmadd.s $f0, $f5, $f25, $f12
+0x00 0x00 0x00 0x00 # CHECK: nop
+0x27 0x48 0xc7 0x00 # CHECK: nor $9, $6, $7
+0xb9 0x87 0x1e 0x4d # CHECK: nmsub.d $f30, $f8, $f16, $f30
+0x78 0x98 0x04 0x4f # CHECK: nmsub.s $f1, $f24, $f19, $f4
+0x25 0x18 0x65 0x00 # CHECK: or $3, $3, $5
+0x67 0x45 0xc9 0x34 # CHECK: ori $9, $6, 17767
+0xc2 0x49 0x26 0x00 # CHECK: rotr $9, $6, 7
+0x46 0x48 0xe6 0x00 # CHECK: rotrv $9, $6, $7
+0x0c 0x73 0x20 0x46 # CHECK: round.w.d $f12, $f14
+0x8c 0x39 0x00 0x46 # CHECK: round.w.s $f6, $f7
+0xc6 0x23 0xa4 0xa0 # CHECK: sb $4, 9158($5)
+0x06 0x00 0xa4 0xa0 # CHECK: sb $4, 6($5)
+0xc6 0x23 0xe9 0xe0 # CHECK: sc $9, 9158($7)
+0xc6 0x23 0xe9 0xf4 # CHECK: sdc1 $f9, 9158($7)
+0x09 0x40 0x24 0x4f # CHECK: sdxc1 $f8, $4($25)
+0x20 0x34 0x07 0x7c # CHECK: seb $6, $7
+0x20 0x36 0x07 0x7c # CHECK: seh $6, $7
+0xc6 0x23 0xa4 0xa4 # CHECK: sh $4, 9158($5)
+0xc0 0x21 0x03 0x00 # CHECK: sll $4, $3, 7
+0x04 0x10 0xa3 0x00 # CHECK: sllv $2, $3, $5
+0x2a 0x18 0x65 0x00 # CHECK: slt $3, $3, $5
+0x67 0x00 0x63 0x28 # CHECK: slti $3, $3, 103
+0x67 0x00 0x63 0x2c # CHECK: sltiu $3, $3, 103
+0x2b 0x18 0x65 0x00 # CHECK: sltu $3, $3, $5
+0x04 0x73 0x20 0x46 # CHECK: sqrt.d $f12, $f14
+0x84 0x39 0x00 0x46 # CHECK: sqrt.s $f6, $f7
+0xc3 0x21 0x03 0x00 # CHECK: sra $4, $3, 7
+0x07 0x10 0xa3 0x00 # CHECK: srav $2, $3, $5
+0xc2 0x21 0x03 0x00 # CHECK: srl $4, $3, 7
+0x06 0x10 0xa3 0x00 # CHECK: srlv $2, $3, $5
+0x01 0x62 0x2e 0x46 # CHECK: sub.d $f8, $f12, $f14
+0x41 0x32 0x07 0x46 # CHECK: sub.s $f9, $f6, $f7
+0x22 0x48 0xc7 0x00 # CHECK: sub $9, $6, $7
+0x23 0x20 0x65 0x00 # CHECK: subu $4, $3, $5
+0x0d 0x20 0xb8 0x4c # CHECK: suxc1 $f4, $24($5)
+0x18 0x00 0xa4 0xac # CHECK: sw $4, 24($5)
+0xc6 0x23 0xe9 0xe4 # CHECK: swc1 $f9, 9158($7)
+0x10 0x00 0xa4 0xa8 # CHECK: swl $4, 16($5)
+0x10 0x00 0xe6 0xb8 # CHECK: swr $6, 16($7)
+0x08 0xd0 0xd2 0x4e # CHECK: swxc1 $f26, $18($22)
+0xcf 0x01 0x00 0x00 # CHECK: sync 7
+0x0d 0x73 0x20 0x46 # CHECK: trunc.w.d $f12, $f14
+0x8d 0x39 0x00 0x46 # CHECK: trunc.w.s $f6, $f7
+0xa0 0x30 0x07 0x7c # CHECK: wsbh $6, $7
+0x26 0x18 0x65 0x00 # CHECK: xor $3, $3, $5
+0x67 0x45 0xc9 0x38 # CHECK: xori $9, $6, 17767
diff --git a/test/MC/Disassembler/Mips/mips32r3/valid-mips32r3-le.txt b/test/MC/Disassembler/Mips/mips32r3/valid-mips32r3-le.txt
deleted file mode 100644
index 1909e2a2127..00000000000
--- a/test/MC/Disassembler/Mips/mips32r3/valid-mips32r3-le.txt
+++ /dev/null
@@ -1,169 +0,0 @@
-# RUN: llvm-mc --disassemble %s -triple=mipsel-unknown-linux -mcpu=mips32r3 | FileCheck %s
-0x05 0x73 0x20 0x46 # CHECK: abs.d $f12, $f14
-0x85 0x39 0x00 0x46 # CHECK: abs.s $f6, $f7
-0x20 0x48 0xc7 0x00 # CHECK: add $9, $6, $7
-0x00 0x62 0x2e 0x46 # CHECK: add.d $f8, $f12, $f14
-0x40 0x32 0x07 0x46 # CHECK: add.s $f9, $f6, $f7
-0x67 0x45 0xc9 0x20 # CHECK: addi $9, $6, 17767
-0x67 0xc5 0xc9 0x24 # CHECK: addiu $9, $6, -15001
-0x21 0x48 0xc7 0x00 # CHECK: addu $9, $6, $7
-0x24 0x48 0xc7 0x00 # CHECK: and $9, $6, $7
-0x67 0x45 0xc9 0x30 # CHECK: andi $9, $6, 17767
-0x4c 0x01 0x00 0x10 # CHECK: b 1332
-0x4c 0x01 0x00 0x45 # CHECK: bc1f 1332
-0x4c 0x01 0x1c 0x45 # CHECK: bc1f $fcc7, 1332
-0x4c 0x01 0x01 0x45 # CHECK: bc1t 1332
-0x4c 0x01 0x1d 0x45 # CHECK: bc1t $fcc7, 1332
-0x4c 0x01 0x26 0x11 # CHECK: beq $9, $6, 1332
-0x4c 0x01 0xc1 0x04 # CHECK: bgez $6, 1332
-0x4c 0x01 0xd1 0x04 # CHECK: bgezal $6, 1332
-0x4c 0x01 0xc0 0x1c # CHECK: bgtz $6, 1332
-0x4c 0x01 0xc0 0x18 # CHECK: blez $6, 1332
-0x4c 0x01 0x26 0x15 # CHECK: bne $9, $6, 1332
-0x32 0x60 0x2e 0x46 # CHECK: c.eq.d $f12, $f14
-0x32 0x30 0x07 0x46 # CHECK: c.eq.s $f6, $f7
-0x30 0x60 0x2e 0x46 # CHECK: c.f.d $f12, $f14
-0x30 0x30 0x07 0x46 # CHECK: c.f.s $f6, $f7
-0x3e 0x60 0x2e 0x46 # CHECK: c.le.d $f12, $f14
-0x3e 0x30 0x07 0x46 # CHECK: c.le.s $f6, $f7
-0x3c 0x60 0x2e 0x46 # CHECK: c.lt.d $f12, $f14
-0x3c 0x30 0x07 0x46 # CHECK: c.lt.s $f6, $f7
-0x3d 0x60 0x2e 0x46 # CHECK: c.nge.d $f12, $f14
-0x3d 0x30 0x07 0x46 # CHECK: c.nge.s $f6, $f7
-0x3b 0x60 0x2e 0x46 # CHECK: c.ngl.d $f12, $f14
-0x3b 0x30 0x07 0x46 # CHECK: c.ngl.s $f6, $f7
-0x39 0x60 0x2e 0x46 # CHECK: c.ngle.d $f12, $f14
-0x39 0x30 0x07 0x46 # CHECK: c.ngle.s $f6, $f7
-0x3f 0x60 0x2e 0x46 # CHECK: c.ngt.d $f12, $f14
-0x3f 0x30 0x07 0x46 # CHECK: c.ngt.s $f6, $f7
-0x36 0x60 0x2e 0x46 # CHECK: c.ole.d $f12, $f14
-0x36 0x30 0x07 0x46 # CHECK: c.ole.s $f6, $f7
-0x34 0x60 0x2e 0x46 # CHECK: c.olt.d $f12, $f14
-0x34 0x30 0x07 0x46 # CHECK: c.olt.s $f6, $f7
-0x3a 0x60 0x2e 0x46 # CHECK: c.seq.d $f12, $f14
-0x3a 0x30 0x07 0x46 # CHECK: c.seq.s $f6, $f7
-0x38 0x60 0x2e 0x46 # CHECK: c.sf.d $f12, $f14
-0x38 0x30 0x07 0x46 # CHECK: c.sf.s $f6, $f7
-0x33 0x60 0x2e 0x46 # CHECK: c.ueq.d $f12, $f14
-0x33 0xe0 0x12 0x46 # CHECK: c.ueq.s $f28, $f18
-0x37 0x60 0x2e 0x46 # CHECK: c.ule.d $f12, $f14
-0x37 0x30 0x07 0x46 # CHECK: c.ule.s $f6, $f7
-0x35 0x60 0x2e 0x46 # CHECK: c.ult.d $f12, $f14
-0x35 0x30 0x07 0x46 # CHECK: c.ult.s $f6, $f7
-0x31 0x60 0x2e 0x46 # CHECK: c.un.d $f12, $f14
-0x31 0x30 0x07 0x46 # CHECK: c.un.s $f6, $f7
-0x0e 0x73 0x20 0x46 # CHECK: ceil.w.d $f12, $f14
-0x8e 0x39 0x00 0x46 # CHECK: ceil.w.s $f6, $f7
-0x00 0x38 0x46 0x44 # CHECK: cfc1 $6, $7
-0x21 0x30 0xe6 0x70 # CHECK: clo $6, $7
-0x20 0x30 0xe6 0x70 # CHECK: clz $6, $7
-0x00 0x38 0xc6 0x44 # CHECK: ctc1 $6, $7
-0xa1 0x39 0x00 0x46 # CHECK: cvt.d.s $f6, $f7
-0x21 0x73 0x80 0x46 # CHECK: cvt.d.w $f12, $f14
-0x25 0x73 0x20 0x46 # CHECK: cvt.l.d $f12, $f14
-0xa5 0x39 0x00 0x46 # CHECK: cvt.l.s $f6, $f7
-0x20 0x73 0x20 0x46 # CHECK: cvt.s.d $f12, $f14
-0xa0 0x39 0x80 0x46 # CHECK: cvt.s.w $f6, $f7
-0x24 0x73 0x20 0x46 # CHECK: cvt.w.d $f12, $f14
-0xa4 0x39 0x00 0x46 # CHECK: cvt.w.s $f6, $f7
-0x00 0x60 0x7e 0x41 # CHECK: di $fp
-0x00 0x60 0x60 0x41 # CHECK: di
-0x20 0x60 0x6e 0x41 # CHECK: ei $14
-0x20 0x60 0x60 0x41 # CHECK: ei
-0x0f 0x73 0x20 0x46 # CHECK: floor.w.d $f12, $f14
-0x8f 0x39 0x00 0x46 # CHECK: floor.w.s $f6, $f7
-0x84 0x61 0x33 0x7d # CHECK: ins $19, $9, 6, 7
-0x4c 0x01 0x00 0x08 # CHECK: j 1328
-0x4c 0x01 0x00 0x0c # CHECK: jal 1328
-0x4c 0x01 0x00 0x74 # CHECK: jalx 1328
-0x09 0xf8 0xe0 0x00 # CHECK: jalr $7
-0x09 0xfc 0x80 0x00 # CHECK: jalr.hb $4
-0x09 0x24 0xa0 0x00 # CHECK: jalr.hb $4, $5
-0x08 0x00 0xe0 0x00 # CHECK: jr $7
-0xc6 0x23 0xa4 0x80 # CHECK: lb $4, 9158($5)
-0x06 0x00 0xa4 0x90 # CHECK: lbu $4, 6($5)
-0xc6 0x23 0xe9 0xd4 # CHECK: ldc1 $f9, 9158($7)
-0x01 0x02 0xf7 0x4d # CHECK: ldxc1 $f8, $23($15)
-0x0c 0x00 0xa4 0x84 # CHECK: lh $4, 12($5)
-0x0c 0x00 0xa4 0x84 # CHECK: lh $4, 12($5)
-0xc6 0x23 0xe9 0xc0 # CHECK: ll $9, 9158($7)
-0x67 0x45 0x06 0x3c # CHECK: lui $6, 17767
-0x05 0x00 0xa6 0x4c # CHECK: luxc1 $f0, $6($5)
-0x18 0x00 0xa4 0x8c # CHECK: lw $4, 24($5)
-0xc6 0x23 0xe9 0xc4 # CHECK: lwc1 $f9, 9158($7)
-0x03 0x00 0x82 0x88 # CHECK: lwl $2, 3($4)
-0x10 0x00 0xa3 0x98 # CHECK: lwr $3, 16($5)
-0x00 0x05 0xcc 0x4d # CHECK: lwxc1 $f20, $12($14)
-0x00 0x00 0xc7 0x70 # CHECK: madd $6, $7
-0xa1 0xd4 0x94 0x4e # CHECK: madd.d $f18, $f20, $f26, $f20
-0x60 0x98 0xf9 0x4f # CHECK: madd.s $f1, $f31, $f19, $f25
-0x01 0x00 0xc7 0x70 # CHECK: maddu $6, $7
-0x00 0x38 0x06 0x44 # CHECK: mfc1 $6, $f7
-0x10 0x28 0x00 0x00 # CHECK: mfhi $5
-0x00 0xc0 0x7e 0x44 # CHECK: mfhc1 $fp, $f24
-0x12 0x28 0x00 0x00 # CHECK: mflo $5
-0x86 0x41 0x20 0x46 # CHECK: mov.d $f6, $f8
-0x86 0x39 0x00 0x46 # CHECK: mov.s $f6, $f7
-0x04 0x00 0xc7 0x70 # CHECK: msub $6, $7
-0xa9 0xf2 0x52 0x4c # CHECK: msub.d $f10, $f2, $f30, $f18
-0x28 0x53 0x70 0x4e # CHECK: msub.s $f12, $f19, $f10, $f16
-0x05 0x00 0xc7 0x70 # CHECK: msubu $6, $7
-0x00 0x38 0x86 0x44 # CHECK: mtc1 $6, $f7
-0x11 0x00 0xe0 0x00 # CHECK: mthi $7
-0x00 0x80 0xe0 0x44 # CHECK: mthc1 $zero, $f16
-0x13 0x00 0xe0 0x00 # CHECK: mtlo $7
-0x02 0x62 0x2e 0x46 # CHECK: mul.d $f8, $f12, $f14
-0x42 0x32 0x07 0x46 # CHECK: mul.s $f9, $f6, $f7
-0x02 0x48 0xc7 0x70 # CHECK: mul $9, $6, $7
-0x18 0x00 0x65 0x00 # CHECK: mult $3, $5
-0x19 0x00 0x65 0x00 # CHECK: multu $3, $5
-0x07 0x73 0x20 0x46 # CHECK: neg.d $f12, $f14
-0x87 0x39 0x00 0x46 # CHECK: neg.s $f6, $f7
-0xb1 0x74 0x54 0x4d # CHECK: nmadd.d $f18, $f10, $f14, $f20
-0x30 0xc8 0xac 0x4c # CHECK: nmadd.s $f0, $f5, $f25, $f12
-0x00 0x00 0x00 0x00 # CHECK: nop
-0x27 0x48 0xc7 0x00 # CHECK: nor $9, $6, $7
-0xb9 0x87 0x1e 0x4d # CHECK: nmsub.d $f30, $f8, $f16, $f30
-0x78 0x98 0x04 0x4f # CHECK: nmsub.s $f1, $f24, $f19, $f4
-0x25 0x18 0x65 0x00 # CHECK: or $3, $3, $5
-0x67 0x45 0xc9 0x34 # CHECK: ori $9, $6, 17767
-0xc2 0x49 0x26 0x00 # CHECK: rotr $9, $6, 7
-0x46 0x48 0xe6 0x00 # CHECK: rotrv $9, $6, $7
-0x0c 0x73 0x20 0x46 # CHECK: round.w.d $f12, $f14
-0x8c 0x39 0x00 0x46 # CHECK: round.w.s $f6, $f7
-0xc6 0x23 0xa4 0xa0 # CHECK: sb $4, 9158($5)
-0x06 0x00 0xa4 0xa0 # CHECK: sb $4, 6($5)
-0xc6 0x23 0xe9 0xe0 # CHECK: sc $9, 9158($7)
-0xc6 0x23 0xe9 0xf4 # CHECK: sdc1 $f9, 9158($7)
-0x09 0x40 0x24 0x4f # CHECK: sdxc1 $f8, $4($25)
-0x20 0x34 0x07 0x7c # CHECK: seb $6, $7
-0x20 0x36 0x07 0x7c # CHECK: seh $6, $7
-0xc6 0x23 0xa4 0xa4 # CHECK: sh $4, 9158($5)
-0xc0 0x21 0x03 0x00 # CHECK: sll $4, $3, 7
-0x04 0x10 0xa3 0x00 # CHECK: sllv $2, $3, $5
-0x2a 0x18 0x65 0x00 # CHECK: slt $3, $3, $5
-0x67 0x00 0x63 0x28 # CHECK: slti $3, $3, 103
-0x67 0x00 0x63 0x2c # CHECK: sltiu $3, $3, 103
-0x2b 0x18 0x65 0x00 # CHECK: sltu $3, $3, $5
-0x04 0x73 0x20 0x46 # CHECK: sqrt.d $f12, $f14
-0x84 0x39 0x00 0x46 # CHECK: sqrt.s $f6, $f7
-0xc3 0x21 0x03 0x00 # CHECK: sra $4, $3, 7
-0x07 0x10 0xa3 0x00 # CHECK: srav $2, $3, $5
-0xc2 0x21 0x03 0x00 # CHECK: srl $4, $3, 7
-0x06 0x10 0xa3 0x00 # CHECK: srlv $2, $3, $5
-0x01 0x62 0x2e 0x46 # CHECK: sub.d $f8, $f12, $f14
-0x41 0x32 0x07 0x46 # CHECK: sub.s $f9, $f6, $f7
-0x22 0x48 0xc7 0x00 # CHECK: sub $9, $6, $7
-0x23 0x20 0x65 0x00 # CHECK: subu $4, $3, $5
-0x0d 0x20 0xb8 0x4c # CHECK: suxc1 $f4, $24($5)
-0x18 0x00 0xa4 0xac # CHECK: sw $4, 24($5)
-0xc6 0x23 0xe9 0xe4 # CHECK: swc1 $f9, 9158($7)
-0x10 0x00 0xa4 0xa8 # CHECK: swl $4, 16($5)
-0x10 0x00 0xe6 0xb8 # CHECK: swr $6, 16($7)
-0x08 0xd0 0xd2 0x4e # CHECK: swxc1 $f26, $18($22)
-0xcf 0x01 0x00 0x00 # CHECK: sync 7
-0x0d 0x73 0x20 0x46 # CHECK: trunc.w.d $f12, $f14
-0x8d 0x39 0x00 0x46 # CHECK: trunc.w.s $f6, $f7
-0xa0 0x30 0x07 0x7c # CHECK: wsbh $6, $7
-0x26 0x18 0x65 0x00 # CHECK: xor $3, $3, $5
-0x67 0x45 0xc9 0x38 # CHECK: xori $9, $6, 17767
diff --git a/test/MC/Disassembler/Mips/mips32r3/valid-mips32r3.txt b/test/MC/Disassembler/Mips/mips32r3/valid-mips32r3.txt
index a273c24f9c9..cf9e98675d2 100644
--- a/test/MC/Disassembler/Mips/mips32r3/valid-mips32r3.txt
+++ b/test/MC/Disassembler/Mips/mips32r3/valid-mips32r3.txt
@@ -1,169 +1,171 @@
 # RUN: llvm-mc --disassemble %s -triple=mips-unknown-linux -mcpu=mips32r3 | FileCheck %s
-0x46 0x20 0x73 0x05 # CHECK: abs.d $f12, $f14
-0x46 0x00 0x39 0x85 # CHECK: abs.s $f6, $f7
+0x00 0x00 0x00 0x00 # CHECK: nop
+0x00 0x00 0x01 0xcf # CHECK: sync 7
+0x00 0x00 0x28 0x10 # CHECK: mfhi $5
+0x00 0x00 0x28 0x12 # CHECK: mflo $5
+0x00 0x03 0x21 0xc0 # CHECK: sll $4, $3, 7
+0x00 0x03 0x21 0xc2 # CHECK: srl $4, $3, 7
+0x00 0x03 0x21 0xc3 # CHECK: sra $4, $3, 7
+0x00 0x26 0x49 0xc2 # CHECK: rotr $9, $6, 7
+0x00 0x65 0x00 0x18 # CHECK: mult $3, $5
+0x00 0x65 0x00 0x19 # CHECK: multu $3, $5
+0x00 0x65 0x18 0x25 # CHECK: or $3, $3, $5
+0x00 0x65 0x18 0x26 # CHECK: xor $3, $3, $5
+0x00 0x65 0x18 0x2a # CHECK: slt $3, $3, $5
+0x00 0x65 0x18 0x2b # CHECK: sltu $3, $3, $5
+0x00 0x65 0x20 0x23 # CHECK: subu $4, $3, $5
+0x00 0x80 0xfc 0x09 # CHECK: jalr.hb $4
+0x00 0xa0 0x24 0x09 # CHECK: jalr.hb $4, $5
+0x00 0xa3 0x10 0x04 # CHECK: sllv $2, $3, $5
+0x00 0xa3 0x10 0x06 # CHECK: srlv $2, $3, $5
+0x00 0xa3 0x10 0x07 # CHECK: srav $2, $3, $5
 0x00 0xc7 0x48 0x20 # CHECK: add $9, $6, $7
-0x46 0x2e 0x62 0x00 # CHECK: add.d $f8, $f12, $f14
-0x46 0x07 0x32 0x40 # CHECK: add.s $f9, $f6, $f7
-0x20 0xc9 0x45 0x67 # CHECK: addi $9, $6, 17767
-0x24 0xc9 0xc5 0x67 # CHECK: addiu $9, $6, -15001
 0x00 0xc7 0x48 0x21 # CHECK: addu $9, $6, $7
+0x00 0xc7 0x48 0x22 # CHECK: sub $9, $6, $7
 0x00 0xc7 0x48 0x24 # CHECK: and $9, $6, $7
-0x30 0xc9 0x45 0x67 # CHECK: andi $9, $6, 17767
+0x00 0xc7 0x48 0x27 # CHECK: nor $9, $6, $7
+0x00 0xe0 0x00 0x08 # CHECK: jr $7
+0x00 0xe0 0x00 0x11 # CHECK: mthi $7
+0x00 0xe0 0x00 0x13 # CHECK: mtlo $7
+0x00 0xe0 0xf8 0x09 # CHECK: jalr $7
+0x00 0xe6 0x48 0x46 # CHECK: rotrv $9, $6, $7
+0x04 0xc1 0x01 0x4c # CHECK: bgez $6, 1332
+0x04 0xd1 0x01 0x4c # CHECK: bgezal $6, 1332
+0x08 0x00 0x01 0x4c # CHECK: j 1328
+0x0c 0x00 0x01 0x4c # CHECK: jal 1328
 0x10 0x00 0x01 0x4c # CHECK: b 1332
+0x11 0x26 0x01 0x4c # CHECK: beq $9, $6, 1332
+0x15 0x26 0x01 0x4c # CHECK: bne $9, $6, 1332
+0x18 0xc0 0x01 0x4c # CHECK: blez $6, 1332
+0x1c 0xc0 0x01 0x4c # CHECK: bgtz $6, 1332
+0x20 0xc9 0x45 0x67 # CHECK: addi $9, $6, 17767
+0x24 0xc9 0xc5 0x67 # CHECK: addiu $9, $6, -15001
+0x28 0x63 0x00 0x67 # CHECK: slti $3, $3, 103
+0x2c 0x63 0x00 0x67 # CHECK: sltiu $3, $3, 103
+0x30 0xc9 0x45 0x67 # CHECK: andi $9, $6, 17767
+0x34 0xc9 0x45 0x67 # CHECK: ori $9, $6, 17767
+0x38 0xc9 0x45 0x67 # CHECK: xori $9, $6, 17767
+0x3c 0x06 0x45 0x67 # CHECK: lui $6, 17767
+0x40 0x08 0x78 0x01 # CHECK: mfc0 $8, $15, 1
+0x40 0x89 0x78 0x01 # CHECK: mtc0 $9, $15, 1
+0x41 0x60 0x60 0x00 # CHECK: di
+0x41 0x60 0x60 0x20 # CHECK: ei
+0x41 0x6e 0x60 0x20 # CHECK: ei $14
+0x41 0x7e 0x60 0x00 # CHECK: di $fp
+0x44 0x06 0x38 0x00 # CHECK: mfc1 $6, $f7
+0x44 0x46 0x38 0x00 # CHECK: cfc1 $6, $7
+0x44 0x7e 0xc0 0x00 # CHECK: mfhc1 $fp, $f24
+0x44 0x86 0x38 0x00 # CHECK: mtc1 $6, $f7
+0x44 0xc6 0x38 0x00 # CHECK: ctc1 $6, $7
+0x44 0xe0 0x80 0x00 # CHECK: mthc1 $zero, $f16
 0x45 0x00 0x01 0x4c # CHECK: bc1f 1332
-0x45 0x1c 0x01 0x4c # CHECK: bc1f $fcc7, 1332
 0x45 0x01 0x01 0x4c # CHECK: bc1t 1332
+0x45 0x1c 0x01 0x4c # CHECK: bc1f $fcc7, 1332
 0x45 0x1d 0x01 0x4c # CHECK: bc1t $fcc7, 1332
-0x11 0x26 0x01 0x4c # CHECK: beq $9, $6, 1332
-0x04 0xc1 0x01 0x4c # CHECK: bgez $6, 1332
-0x04 0xd1 0x01 0x4c # CHECK: bgezal $6, 1332
-0x1c 0xc0 0x01 0x4c # CHECK: bgtz $6, 1332
-0x18 0xc0 0x01 0x4c # CHECK: blez $6, 1332
-0x15 0x26 0x01 0x4c # CHECK: bne $9, $6, 1332
-0x46 0x2e 0x60 0x32 # CHECK: c.eq.d $f12, $f14
-0x46 0x07 0x30 0x32 # CHECK: c.eq.s $f6, $f7
-0x46 0x2e 0x60 0x30 # CHECK: c.f.d $f12, $f14
+0x46 0x00 0x39 0x84 # CHECK: sqrt.s $f6, $f7
+0x46 0x00 0x39 0x85 # CHECK: abs.s $f6, $f7
+0x46 0x00 0x39 0x86 # CHECK: mov.s $f6, $f7
+0x46 0x00 0x39 0x87 # CHECK: neg.s $f6, $f7
+0x46 0x00 0x39 0x8c # CHECK: round.w.s $f6, $f7
+0x46 0x00 0x39 0x8d # CHECK: trunc.w.s $f6, $f7
+0x46 0x00 0x39 0x8e # CHECK: ceil.w.s $f6, $f7
+0x46 0x00 0x39 0x8f # CHECK: floor.w.s $f6, $f7
+0x46 0x00 0x39 0xa1 # CHECK: cvt.d.s $f6, $f7
+0x46 0x00 0x39 0xa4 # CHECK: cvt.w.s $f6, $f7
+0x46 0x00 0x39 0xa5 # CHECK: cvt.l.s $f6, $f7
 0x46 0x07 0x30 0x30 # CHECK: c.f.s $f6, $f7
-0x46 0x2e 0x60 0x3e # CHECK: c.le.d $f12, $f14
-0x46 0x07 0x30 0x3e # CHECK: c.le.s $f6, $f7
-0x46 0x2e 0x60 0x3c # CHECK: c.lt.d $f12, $f14
+0x46 0x07 0x30 0x31 # CHECK: c.un.s $f6, $f7
+0x46 0x07 0x30 0x32 # CHECK: c.eq.s $f6, $f7
+0x46 0x07 0x30 0x34 # CHECK: c.olt.s $f6, $f7
+0x46 0x07 0x30 0x35 # CHECK: c.ult.s $f6, $f7
+0x46 0x07 0x30 0x36 # CHECK: c.ole.s $f6, $f7
+0x46 0x07 0x30 0x37 # CHECK: c.ule.s $f6, $f7
+0x46 0x07 0x30 0x38 # CHECK: c.sf.s $f6, $f7
+0x46 0x07 0x30 0x39 # CHECK: c.ngle.s $f6, $f7
+0x46 0x07 0x30 0x3a # CHECK: c.seq.s $f6, $f7
+0x46 0x07 0x30 0x3b # CHECK: c.ngl.s $f6, $f7
 0x46 0x07 0x30 0x3c # CHECK: c.lt.s $f6, $f7
-0x46 0x2e 0x60 0x3d # CHECK: c.nge.d $f12, $f14
 0x46 0x07 0x30 0x3d # CHECK: c.nge.s $f6, $f7
-0x46 0x2e 0x60 0x3b # CHECK: c.ngl.d $f12, $f14
-0x46 0x07 0x30 0x3b # CHECK: c.ngl.s $f6, $f7
-0x46 0x2e 0x60 0x39 # CHECK: c.ngle.d $f12, $f14
-0x46 0x07 0x30 0x39 # CHECK: c.ngle.s $f6, $f7
-0x46 0x2e 0x60 0x3f # CHECK: c.ngt.d $f12, $f14
+0x46 0x07 0x30 0x3e # CHECK: c.le.s $f6, $f7
 0x46 0x07 0x30 0x3f # CHECK: c.ngt.s $f6, $f7
-0x46 0x2e 0x60 0x36 # CHECK: c.ole.d $f12, $f14
-0x46 0x07 0x30 0x36 # CHECK: c.ole.s $f6, $f7
-0x46 0x2e 0x60 0x34 # CHECK: c.olt.d $f12, $f14
-0x46 0x07 0x30 0x34 # CHECK: c.olt.s $f6, $f7
-0x46 0x2e 0x60 0x3a # CHECK: c.seq.d $f12, $f14
-0x46 0x07 0x30 0x3a # CHECK: c.seq.s $f6, $f7
-0x46 0x2e 0x60 0x38 # CHECK: c.sf.d $f12, $f14
-0x46 0x07 0x30 0x38 # CHECK: c.sf.s $f6, $f7
-0x46 0x2e 0x60 0x33 # CHECK: c.ueq.d $f12, $f14
+0x46 0x07 0x32 0x40 # CHECK: add.s $f9, $f6, $f7
+0x46 0x07 0x32 0x41 # CHECK: sub.s $f9, $f6, $f7
+0x46 0x07 0x32 0x42 # CHECK: mul.s $f9, $f6, $f7
 0x46 0x12 0xe0 0x33 # CHECK: c.ueq.s $f28, $f18
-0x46 0x2e 0x60 0x37 # CHECK: c.ule.d $f12, $f14
-0x46 0x07 0x30 0x37 # CHECK: c.ule.s $f6, $f7
-0x46 0x2e 0x60 0x35 # CHECK: c.ult.d $f12, $f14
-0x46 0x07 0x30 0x35 # CHECK: c.ult.s $f6, $f7
-0x46 0x2e 0x60 0x31 # CHECK: c.un.d $f12, $f14
-0x46 0x07 0x30 0x31 # CHECK: c.un.s $f6, $f7
+0x46 0x20 0x41 0x86 # CHECK: mov.d $f6, $f8
+0x46 0x20 0x73 0x04 # CHECK: sqrt.d $f12, $f14
+0x46 0x20 0x73 0x05 # CHECK: abs.d $f12, $f14
+0x46 0x20 0x73 0x07 # CHECK: neg.d $f12, $f14
+0x46 0x20 0x73 0x0c # CHECK: round.w.d $f12, $f14
+0x46 0x20 0x73 0x0d # CHECK: trunc.w.d $f12, $f14
 0x46 0x20 0x73 0x0e # CHECK: ceil.w.d $f12, $f14
-0x46 0x00 0x39 0x8e # CHECK: ceil.w.s $f6, $f7
-0x44 0x46 0x38 0x00 # CHECK: cfc1 $6, $7
-0x70 0xe6 0x30 0x21 # CHECK: clo $6, $7
-0x70 0xe6 0x30 0x20 # CHECK: clz $6, $7
-0x44 0xc6 0x38 0x00 # CHECK: ctc1 $6, $7
-0x46 0x00 0x39 0xa1 # CHECK: cvt.d.s $f6, $f7
-0x46 0x80 0x73 0x21 # CHECK: cvt.d.w $f12, $f14
-0x46 0x20 0x73 0x25 # CHECK: cvt.l.d $f12, $f14
-0x46 0x00 0x39 0xa5 # CHECK: cvt.l.s $f6, $f7
+0x46 0x20 0x73 0x0f # CHECK: floor.w.d $f12, $f14
 0x46 0x20 0x73 0x20 # CHECK: cvt.s.d $f12, $f14
-0x46 0x80 0x39 0xa0 # CHECK: cvt.s.w $f6, $f7
 0x46 0x20 0x73 0x24 # CHECK: cvt.w.d $f12, $f14
-0x46 0x00 0x39 0xa4 # CHECK: cvt.w.s $f6, $f7
-0x41 0x7e 0x60 0x00 # CHECK: di $fp
-0x41 0x60 0x60 0x00 # CHECK: di
-0x41 0x6e 0x60 0x20 # CHECK: ei $14
-0x41 0x60 0x60 0x20 # CHECK: ei
-0x46 0x20 0x73 0x0f # CHECK: floor.w.d $f12, $f14
-0x46 0x00 0x39 0x8f # CHECK: floor.w.s $f6, $f7
-0x7d 0x33 0x61 0x84 # CHECK: ins $19, $9, 6, 7
-0x08 0x00 0x01 0x4c # CHECK: j 1328
-0x0c 0x00 0x01 0x4c # CHECK: jal 1328
-0x74 0x00 0x01 0x4c # CHECK: jalx 1328
-0x00 0xe0 0xf8 0x09 # CHECK: jalr $7
-0x00 0x80 0xfc 0x09 # CHECK: jalr.hb $4
-0x00 0xa0 0x24 0x09 # CHECK: jalr.hb $4, $5
-0x00 0xe0 0x00 0x08 # CHECK: jr $7
-0x80 0xa4 0x23 0xc6 # CHECK: lb $4, 9158($5)
-0x90 0xa4 0x00 0x06 # CHECK: lbu $4, 6($5)
-0xd4 0xe9 0x23 0xc6 # CHECK: ldc1 $f9, 9158($7)
-0x4d 0xf7 0x02 0x01 # CHECK: ldxc1 $f8, $23($15)
-0x84 0xa4 0x00 0x0c # CHECK: lh $4, 12($5)
-0x84 0xa4 0x00 0x0c # CHECK: lh $4, 12($5)
-0xc0 0xe9 0x23 0xc6 # CHECK: ll $9, 9158($7)
-0x3c 0x06 0x45 0x67 # CHECK: lui $6, 17767
+0x46 0x20 0x73 0x25 # CHECK: cvt.l.d $f12, $f14
+0x46 0x2e 0x60 0x30 # CHECK: c.f.d $f12, $f14
+0x46 0x2e 0x60 0x31 # CHECK: c.un.d $f12, $f14
+0x46 0x2e 0x60 0x32 # CHECK: c.eq.d $f12, $f14
+0x46 0x2e 0x60 0x33 # CHECK: c.ueq.d $f12, $f14
+0x46 0x2e 0x60 0x34 # CHECK: c.olt.d $f12, $f14
+0x46 0x2e 0x60 0x35 # CHECK: c.ult.d $f12, $f14
+0x46 0x2e 0x60 0x36 # CHECK: c.ole.d $f12, $f14
+0x46 0x2e 0x60 0x37 # CHECK: c.ule.d $f12, $f14
+0x46 0x2e 0x60 0x38 # CHECK: c.sf.d $f12, $f14
+0x46 0x2e 0x60 0x39 # CHECK: c.ngle.d $f12, $f14
+0x46 0x2e 0x60 0x3a # CHECK: c.seq.d $f12, $f14
+0x46 0x2e 0x60 0x3b # CHECK: c.ngl.d $f12, $f14
+0x46 0x2e 0x60 0x3c # CHECK: c.lt.d $f12, $f14
+0x46 0x2e 0x60 0x3d # CHECK: c.nge.d $f12, $f14
+0x46 0x2e 0x60 0x3e # CHECK: c.le.d $f12, $f14
+0x46 0x2e 0x60 0x3f # CHECK: c.ngt.d $f12, $f14
+0x46 0x2e 0x62 0x00 # CHECK: add.d $f8, $f12, $f14
+0x46 0x2e 0x62 0x01 # CHECK: sub.d $f8, $f12, $f14
+0x46 0x2e 0x62 0x02 # CHECK: mul.d $f8, $f12, $f14
+0x46 0x80 0x39 0xa0 # CHECK: cvt.s.w $f6, $f7
+0x46 0x80 0x73 0x21 # CHECK: cvt.d.w $f12, $f14
+0x4c 0x52 0xf2 0xa9 # CHECK: msub.d $f10, $f2, $f30, $f18
 0x4c 0xa6 0x00 0x05 # CHECK: luxc1 $f0, $6($5)
-0x8c 0xa4 0x00 0x18 # CHECK: lw $4, 24($5)
-0xc4 0xe9 0x23 0xc6 # CHECK: lwc1 $f9, 9158($7)
-0x88 0x82 0x00 0x03 # CHECK: lwl $2, 3($4)
-0x98 0xa3 0x00 0x10 # CHECK: lwr $3, 16($5)
+0x4c 0xac 0xc8 0x30 # CHECK: nmadd.s $f0, $f5, $f25, $f12
+0x4c 0xb8 0x20 0x0d # CHECK: suxc1 $f4, $24($5)
+0x4d 0x1e 0x87 0xb9 # CHECK: nmsub.d $f30, $f8, $f16, $f30
+0x4d 0x54 0x74 0xb1 # CHECK: nmadd.d $f18, $f10, $f14, $f20
 0x4d 0xcc 0x05 0x00 # CHECK: lwxc1 $f20, $12($14)
-0x70 0xc7 0x00 0x00 # CHECK: madd $6, $7
+0x4d 0xf7 0x02 0x01 # CHECK: ldxc1 $f8, $23($15)
+0x4e 0x70 0x53 0x28 # CHECK: msub.s $f12, $f19, $f10, $f16
 0x4e 0x94 0xd4 0xa1 # CHECK: madd.d $f18, $f20, $f26, $f20
+0x4e 0xd2 0xd0 0x08 # CHECK: swxc1 $f26, $18($22)
+0x4f 0x04 0x98 0x78 # CHECK: nmsub.s $f1, $f24, $f19, $f4
+0x4f 0x24 0x40 0x09 # CHECK: sdxc1 $f8, $4($25)
 0x4f 0xf9 0x98 0x60 # CHECK: madd.s $f1, $f31, $f19, $f25
+0x70 0xc7 0x00 0x00 # CHECK: madd $6, $7
 0x70 0xc7 0x00 0x01 # CHECK: maddu $6, $7
-0x44 0x06 0x38 0x00 # CHECK: mfc1 $6, $f7
-0x00 0x00 0x28 0x10 # CHECK: mfhi $5
-0x44 0x7e 0xc0 0x00 # CHECK: mfhc1 $fp, $f24
-0x00 0x00 0x28 0x12 # CHECK: mflo $5
-0x46 0x20 0x41 0x86 # CHECK: mov.d $f6, $f8
-0x46 0x00 0x39 0x86 # CHECK: mov.s $f6, $f7
 0x70 0xc7 0x00 0x04 # CHECK: msub $6, $7
-0x4c 0x52 0xf2 0xa9 # CHECK: msub.d $f10, $f2, $f30, $f18
-0x4e 0x70 0x53 0x28 # CHECK: msub.s $f12, $f19, $f10, $f16
 0x70 0xc7 0x00 0x05 # CHECK: msubu $6, $7
-0x44 0x86 0x38 0x00 # CHECK: mtc1 $6, $f7
-0x00 0xe0 0x00 0x11 # CHECK: mthi $7
-0x44 0xe0 0x80 0x00 # CHECK: mthc1 $zero, $f16
-0x00 0xe0 0x00 0x13 # CHECK: mtlo $7
-0x46 0x2e 0x62 0x02 # CHECK: mul.d $f8, $f12, $f14
-0x46 0x07 0x32 0x42 # CHECK: mul.s $f9, $f6, $f7
 0x70 0xc7 0x48 0x02 # CHECK: mul $9, $6, $7
-0x00 0x65 0x00 0x18 # CHECK: mult $3, $5
-0x00 0x65 0x00 0x19 # CHECK: multu $3, $5
-0x46 0x20 0x73 0x07 # CHECK: neg.d $f12, $f14
-0x46 0x00 0x39 0x87 # CHECK: neg.s $f6, $f7
-0x4d 0x54 0x74 0xb1 # CHECK: nmadd.d $f18, $f10, $f14, $f20
-0x4c 0xac 0xc8 0x30 # CHECK: nmadd.s $f0, $f5, $f25, $f12
-0x00 0x00 0x00 0x00 # CHECK: nop
-0x00 0xc7 0x48 0x27 # CHECK: nor $9, $6, $7
-0x4d 0x1e 0x87 0xb9 # CHECK: nmsub.d $f30, $f8, $f16, $f30
-0x4f 0x04 0x98 0x78 # CHECK: nmsub.s $f1, $f24, $f19, $f4
-0x00 0x65 0x18 0x25 # CHECK: or $3, $3, $5
-0x34 0xc9 0x45 0x67 # CHECK: ori $9, $6, 17767
-0x00 0x26 0x49 0xc2 # CHECK: rotr $9, $6, 7
-0x00 0xe6 0x48 0x46 # CHECK: rotrv $9, $6, $7
-0x46 0x20 0x73 0x0c # CHECK: round.w.d $f12, $f14
-0x46 0x00 0x39 0x8c # CHECK: round.w.s $f6, $f7
-0xa0 0xa4 0x23 0xc6 # CHECK: sb $4, 9158($5)
-0xa0 0xa4 0x00 0x06 # CHECK: sb $4, 6($5)
-0xe0 0xe9 0x23 0xc6 # CHECK: sc $9, 9158($7)
-0xf4 0xe9 0x23 0xc6 # CHECK: sdc1 $f9, 9158($7)
-0x4f 0x24 0x40 0x09 # CHECK: sdxc1 $f8, $4($25)
+0x70 0xe6 0x30 0x20 # CHECK: clz $6, $7
+0x70 0xe6 0x30 0x21 # CHECK: clo $6, $7
+0x74 0x00 0x01 0x4c # CHECK: jalx 1328
+0x7c 0x07 0x30 0xa0 # CHECK: wsbh $6, $7
 0x7c 0x07 0x34 0x20 # CHECK: seb $6, $7
 0x7c 0x07 0x36 0x20 # CHECK: seh $6, $7
+0x7d 0x33 0x61 0x84 # CHECK: ins $19, $9, 6, 7
+0x80 0xa4 0x23 0xc6 # CHECK: lb $4, 9158($5)
+0x84 0xa4 0x00 0x0c # CHECK: lh $4, 12($5)
+0x84 0xa4 0x00 0x0c # CHECK: lh $4, 12($5)
+0x88 0x82 0x00 0x03 # CHECK: lwl $2, 3($4)
+0x8c 0xa4 0x00 0x18 # CHECK: lw $4, 24($5)
+0x90 0xa4 0x00 0x06 # CHECK: lbu $4, 6($5)
+0x98 0xa3 0x00 0x10 # CHECK: lwr $3, 16($5)
+0xa0 0xa4 0x00 0x06 # CHECK: sb $4, 6($5)
+0xa0 0xa4 0x23 0xc6 # CHECK: sb $4, 9158($5)
 0xa4 0xa4 0x23 0xc6 # CHECK: sh $4, 9158($5)
-0x00 0x03 0x21 0xc0 # CHECK: sll $4, $3, 7
-0x00 0xa3 0x10 0x04 # CHECK: sllv $2, $3, $5
-0x00 0x65 0x18 0x2a # CHECK: slt $3, $3, $5
-0x28 0x63 0x00 0x67 # CHECK: slti $3, $3, 103
-0x2c 0x63 0x00 0x67 # CHECK: sltiu $3, $3, 103
-0x00 0x65 0x18 0x2b # CHECK: sltu $3, $3, $5
-0x46 0x20 0x73 0x04 # CHECK: sqrt.d $f12, $f14
-0x46 0x00 0x39 0x84 # CHECK: sqrt.s $f6, $f7
-0x00 0x03 0x21 0xc3 # CHECK: sra $4, $3, 7
-0x00 0xa3 0x10 0x07 # CHECK: srav $2, $3, $5
-0x00 0x03 0x21 0xc2 # CHECK: srl $4, $3, 7
-0x00 0xa3 0x10 0x06 # CHECK: srlv $2, $3, $5
-0x46 0x2e 0x62 0x01 # CHECK: sub.d $f8, $f12, $f14
-0x46 0x07 0x32 0x41 # CHECK: sub.s $f9, $f6, $f7
-0x00 0xc7 0x48 0x22 # CHECK: sub $9, $6, $7
-0x00 0x65 0x20 0x23 # CHECK: subu $4, $3, $5
-0x4c 0xb8 0x20 0x0d # CHECK: suxc1 $f4, $24($5)
-0xac 0xa4 0x00 0x18 # CHECK: sw $4, 24($5)
-0xe4 0xe9 0x23 0xc6 # CHECK: swc1 $f9, 9158($7)
 0xa8 0xa4 0x00 0x10 # CHECK: swl $4, 16($5)
+0xac 0xa4 0x00 0x18 # CHECK: sw $4, 24($5)
 0xb8 0xe6 0x00 0x10 # CHECK: swr $6, 16($7)
-0x4e 0xd2 0xd0 0x08 # CHECK: swxc1 $f26, $18($22)
-0x00 0x00 0x01 0xcf # CHECK: sync 7
-0x46 0x20 0x73 0x0d # CHECK: trunc.w.d $f12, $f14
-0x46 0x00 0x39 0x8d # CHECK: trunc.w.s $f6, $f7
-0x7c 0x07 0x30 0xa0 # CHECK: wsbh $6, $7
-0x00 0x65 0x18 0x26 # CHECK: xor $3, $3, $5
-0x38 0xc9 0x45 0x67 # CHECK: xori $9, $6, 17767
+0xc0 0xe9 0x23 0xc6 # CHECK: ll $9, 9158($7)
+0xc4 0xe9 0x23 0xc6 # CHECK: lwc1 $f9, 9158($7)
+0xd4 0xe9 0x23 0xc6 # CHECK: ldc1 $f9, 9158($7)
+0xe0 0xe9 0x23 0xc6 # CHECK: sc $9, 9158($7)
+0xe4 0xe9 0x23 0xc6 # CHECK: swc1 $f9, 9158($7)
+0xf4 0xe9 0x23 0xc6 # CHECK: sdc1 $f9, 9158($7)
diff --git a/test/MC/Disassembler/Mips/mips32r5/valid-mips32r5-el.txt b/test/MC/Disassembler/Mips/mips32r5/valid-mips32r5-el.txt
new file mode 100644
index 00000000000..b68089b0a07
--- /dev/null
+++ b/test/MC/Disassembler/Mips/mips32r5/valid-mips32r5-el.txt
@@ -0,0 +1,171 @@
+# RUN: llvm-mc --disassemble %s -triple=mipsel-unknown-linux -mcpu=mips32r5 | FileCheck %s
+0x05 0x73 0x20 0x46 # CHECK: abs.d $f12, $f14
+0x85 0x39 0x00 0x46 # CHECK: abs.s $f6, $f7
+0x20 0x48 0xc7 0x00 # CHECK: add $9, $6, $7
+0x00 0x62 0x2e 0x46 # CHECK: add.d $f8, $f12, $f14
+0x40 0x32 0x07 0x46 # CHECK: add.s $f9, $f6, $f7
+0x67 0x45 0xc9 0x20 # CHECK: addi $9, $6, 17767
+0x67 0xc5 0xc9 0x24 # CHECK: addiu $9, $6, -15001
+0x21 0x48 0xc7 0x00 # CHECK: addu $9, $6, $7
+0x24 0x48 0xc7 0x00 # CHECK: and $9, $6, $7
+0x67 0x45 0xc9 0x30 # CHECK: andi $9, $6, 17767
+0x4c 0x01 0x00 0x10 # CHECK: b 1332
+0x4c 0x01 0x00 0x45 # CHECK: bc1f 1332
+0x4c 0x01 0x1c 0x45 # CHECK: bc1f $fcc7, 1332
+0x4c 0x01 0x01 0x45 # CHECK: bc1t 1332
+0x4c 0x01 0x1d 0x45 # CHECK: bc1t $fcc7, 1332
+0x4c 0x01 0x26 0x11 # CHECK: beq $9, $6, 1332
+0x4c 0x01 0xc1 0x04 # CHECK: bgez $6, 1332
+0x4c 0x01 0xd1 0x04 # CHECK: bgezal $6, 1332
+0x4c 0x01 0xc0 0x1c # CHECK: bgtz $6, 1332
+0x4c 0x01 0xc0 0x18 # CHECK: blez $6, 1332
+0x4c 0x01 0x26 0x15 # CHECK: bne $9, $6, 1332
+0x32 0x60 0x2e 0x46 # CHECK: c.eq.d $f12, $f14
+0x32 0x30 0x07 0x46 # CHECK: c.eq.s $f6, $f7
+0x30 0x60 0x2e 0x46 # CHECK: c.f.d $f12, $f14
+0x30 0x30 0x07 0x46 # CHECK: c.f.s $f6, $f7
+0x3e 0x60 0x2e 0x46 # CHECK: c.le.d $f12, $f14
+0x3e 0x30 0x07 0x46 # CHECK: c.le.s $f6, $f7
+0x3c 0x60 0x2e 0x46 # CHECK: c.lt.d $f12, $f14
+0x3c 0x30 0x07 0x46 # CHECK: c.lt.s $f6, $f7
+0x3d 0x60 0x2e 0x46 # CHECK: c.nge.d $f12, $f14
+0x3d 0x30 0x07 0x46 # CHECK: c.nge.s $f6, $f7
+0x3b 0x60 0x2e 0x46 # CHECK: c.ngl.d $f12, $f14
+0x3b 0x30 0x07 0x46 # CHECK: c.ngl.s $f6, $f7
+0x39 0x60 0x2e 0x46 # CHECK: c.ngle.d $f12, $f14
+0x39 0x30 0x07 0x46 # CHECK: c.ngle.s $f6, $f7
+0x3f 0x60 0x2e 0x46 # CHECK: c.ngt.d $f12, $f14
+0x3f 0x30 0x07 0x46 # CHECK: c.ngt.s $f6, $f7
+0x36 0x60 0x2e 0x46 # CHECK: c.ole.d $f12, $f14
+0x36 0x30 0x07 0x46 # CHECK: c.ole.s $f6, $f7
+0x34 0x60 0x2e 0x46 # CHECK: c.olt.d $f12, $f14
+0x34 0x30 0x07 0x46 # CHECK: c.olt.s $f6, $f7
+0x3a 0x60 0x2e 0x46 # CHECK: c.seq.d $f12, $f14
+0x3a 0x30 0x07 0x46 # CHECK: c.seq.s $f6, $f7
+0x38 0x60 0x2e 0x46 # CHECK: c.sf.d $f12, $f14
+0x38 0x30 0x07 0x46 # CHECK: c.sf.s $f6, $f7
+0x33 0x60 0x2e 0x46 # CHECK: c.ueq.d $f12, $f14
+0x33 0xe0 0x12 0x46 # CHECK: c.ueq.s $f28, $f18
+0x37 0x60 0x2e 0x46 # CHECK: c.ule.d $f12, $f14
+0x37 0x30 0x07 0x46 # CHECK: c.ule.s $f6, $f7
+0x35 0x60 0x2e 0x46 # CHECK: c.ult.d $f12, $f14
+0x35 0x30 0x07 0x46 # CHECK: c.ult.s $f6, $f7
+0x31 0x60 0x2e 0x46 # CHECK: c.un.d $f12, $f14
+0x31 0x30 0x07 0x46 # CHECK: c.un.s $f6, $f7
+0x0e 0x73 0x20 0x46 # CHECK: ceil.w.d $f12, $f14
+0x8e 0x39 0x00 0x46 # CHECK: ceil.w.s $f6, $f7
+0x00 0x38 0x46 0x44 # CHECK: cfc1 $6, $7
+0x21 0x30 0xe6 0x70 # CHECK: clo $6, $7
+0x20 0x30 0xe6 0x70 # CHECK: clz $6, $7
+0x00 0x38 0xc6 0x44 # CHECK: ctc1 $6, $7
+0xa1 0x39 0x00 0x46 # CHECK: cvt.d.s $f6, $f7
+0x21 0x73 0x80 0x46 # CHECK: cvt.d.w $f12, $f14
+0x25 0x73 0x20 0x46 # CHECK: cvt.l.d $f12, $f14
+0xa5 0x39 0x00 0x46 # CHECK: cvt.l.s $f6, $f7
+0x20 0x73 0x20 0x46 # CHECK: cvt.s.d $f12, $f14
+0xa0 0x39 0x80 0x46 # CHECK: cvt.s.w $f6, $f7
+0x24 0x73 0x20 0x46 # CHECK: cvt.w.d $f12, $f14
+0xa4 0x39 0x00 0x46 # CHECK: cvt.w.s $f6, $f7
+0x00 0x60 0x7e 0x41 # CHECK: di $fp
+0x00 0x60 0x60 0x41 # CHECK: di
+0x20 0x60 0x6e 0x41 # CHECK: ei $14
+0x20 0x60 0x60 0x41 # CHECK: ei
+0x0f 0x73 0x20 0x46 # CHECK: floor.w.d $f12, $f14
+0x8f 0x39 0x00 0x46 # CHECK: floor.w.s $f6, $f7
+0x84 0x61 0x33 0x7d # CHECK: ins $19, $9, 6, 7
+0x4c 0x01 0x00 0x08 # CHECK: j 1328
+0x4c 0x01 0x00 0x0c # CHECK: jal 1328
+0x4c 0x01 0x00 0x74 # CHECK: jalx 1328
+0x09 0xf8 0xe0 0x00 # CHECK: jalr $7
+0x09 0xfc 0x80 0x00 # CHECK: jalr.hb $4
+0x09 0x24 0xa0 0x00 # CHECK: jalr.hb $4, $5
+0x08 0x00 0xe0 0x00 # CHECK: jr $7
+0xc6 0x23 0xa4 0x80 # CHECK: lb $4, 9158($5)
+0x06 0x00 0xa4 0x90 # CHECK: lbu $4, 6($5)
+0xc6 0x23 0xe9 0xd4 # CHECK: ldc1 $f9, 9158($7)
+0x01 0x02 0xf7 0x4d # CHECK: ldxc1 $f8, $23($15)
+0x0c 0x00 0xa4 0x84 # CHECK: lh $4, 12($5)
+0x0c 0x00 0xa4 0x84 # CHECK: lh $4, 12($5)
+0xc6 0x23 0xe9 0xc0 # CHECK: ll $9, 9158($7)
+0x67 0x45 0x06 0x3c # CHECK: lui $6, 17767
+0x05 0x00 0xa6 0x4c # CHECK: luxc1 $f0, $6($5)
+0x18 0x00 0xa4 0x8c # CHECK: lw $4, 24($5)
+0xc6 0x23 0xe9 0xc4 # CHECK: lwc1 $f9, 9158($7)
+0x03 0x00 0x82 0x88 # CHECK: lwl $2, 3($4)
+0x10 0x00 0xa3 0x98 # CHECK: lwr $3, 16($5)
+0x00 0x05 0xcc 0x4d # CHECK: lwxc1 $f20, $12($14)
+0x00 0x00 0xc7 0x70 # CHECK: madd $6, $7
+0xa1 0xd4 0x94 0x4e # CHECK: madd.d $f18, $f20, $f26, $f20
+0x60 0x98 0xf9 0x4f # CHECK: madd.s $f1, $f31, $f19, $f25
+0x01 0x00 0xc7 0x70 # CHECK: maddu $6, $7
+0x01 0x78 0x08 0x40 # CHECK: mfc0 $8, $15, 1
+0x00 0x38 0x06 0x44 # CHECK: mfc1 $6, $f7
+0x10 0x28 0x00 0x00 # CHECK: mfhi $5
+0x00 0xc0 0x7e 0x44 # CHECK: mfhc1 $fp, $f24
+0x12 0x28 0x00 0x00 # CHECK: mflo $5
+0x86 0x41 0x20 0x46 # CHECK: mov.d $f6, $f8
+0x86 0x39 0x00 0x46 # CHECK: mov.s $f6, $f7
+0x04 0x00 0xc7 0x70 # CHECK: msub $6, $7
+0xa9 0xf2 0x52 0x4c # CHECK: msub.d $f10, $f2, $f30, $f18
+0x28 0x53 0x70 0x4e # CHECK: msub.s $f12, $f19, $f10, $f16
+0x05 0x00 0xc7 0x70 # CHECK: msubu $6, $7
+0x01 0x78 0x89 0x40 # CHECK: mtc0 $9, $15, 1
+0x00 0x38 0x86 0x44 # CHECK: mtc1 $6, $f7
+0x11 0x00 0xe0 0x00 # CHECK: mthi $7
+0x00 0x80 0xe0 0x44 # CHECK: mthc1 $zero, $f16
+0x13 0x00 0xe0 0x00 # CHECK: mtlo $7
+0x02 0x62 0x2e 0x46 # CHECK: mul.d $f8, $f12, $f14
+0x42 0x32 0x07 0x46 # CHECK: mul.s $f9, $f6, $f7
+0x02 0x48 0xc7 0x70 # CHECK: mul $9, $6, $7
+0x18 0x00 0x65 0x00 # CHECK: mult $3, $5
+0x19 0x00 0x65 0x00 # CHECK: multu $3, $5
+0x07 0x73 0x20 0x46 # CHECK: neg.d $f12, $f14
+0x87 0x39 0x00 0x46 # CHECK: neg.s $f6, $f7
+0xb1 0x74 0x54 0x4d # CHECK: nmadd.d $f18, $f10, $f14, $f20
+0x30 0xc8 0xac 0x4c # CHECK: nmadd.s $f0, $f5, $f25, $f12
+0x00 0x00 0x00 0x00 # CHECK: nop
+0x27 0x48 0xc7 0x00 # CHECK: nor $9, $6, $7
+0xb9 0x87 0x1e 0x4d # CHECK: nmsub.d $f30, $f8, $f16, $f30
+0x78 0x98 0x04 0x4f # CHECK: nmsub.s $f1, $f24, $f19, $f4
+0x25 0x18 0x65 0x00 # CHECK: or $3, $3, $5
+0x67 0x45 0xc9 0x34 # CHECK: ori $9, $6, 17767
+0xc2 0x49 0x26 0x00 # CHECK: rotr $9, $6, 7
+0x46 0x48 0xe6 0x00 # CHECK: rotrv $9, $6, $7
+0x0c 0x73 0x20 0x46 # CHECK: round.w.d $f12, $f14
+0x8c 0x39 0x00 0x46 # CHECK: round.w.s $f6, $f7
+0xc6 0x23 0xa4 0xa0 # CHECK: sb $4, 9158($5)
+0x06 0x00 0xa4 0xa0 # CHECK: sb $4, 6($5)
+0xc6 0x23 0xe9 0xe0 # CHECK: sc $9, 9158($7)
+0xc6 0x23 0xe9 0xf4 # CHECK: sdc1 $f9, 9158($7)
+0x09 0x40 0x24 0x4f # CHECK: sdxc1 $f8, $4($25)
+0x20 0x34 0x07 0x7c # CHECK: seb $6, $7
+0x20 0x36 0x07 0x7c # CHECK: seh $6, $7
+0xc6 0x23 0xa4 0xa4 # CHECK: sh $4, 9158($5)
+0xc0 0x21 0x03 0x00 # CHECK: sll $4, $3, 7
+0x04 0x10 0xa3 0x00 # CHECK: sllv $2, $3, $5
+0x2a 0x18 0x65 0x00 # CHECK: slt $3, $3, $5
+0x67 0x00 0x63 0x28 # CHECK: slti $3, $3, 103
+0x67 0x00 0x63 0x2c # CHECK: sltiu $3, $3, 103
+0x2b 0x18 0x65 0x00 # CHECK: sltu $3, $3, $5
+0x04 0x73 0x20 0x46 # CHECK: sqrt.d $f12, $f14
+0x84 0x39 0x00 0x46 # CHECK: sqrt.s $f6, $f7
+0xc3 0x21 0x03 0x00 # CHECK: sra $4, $3, 7
+0x07 0x10 0xa3 0x00 # CHECK: srav $2, $3, $5
+0xc2 0x21 0x03 0x00 # CHECK: srl $4, $3, 7
+0x06 0x10 0xa3 0x00 # CHECK: srlv $2, $3, $5
+0x01 0x62 0x2e 0x46 # CHECK: sub.d $f8, $f12, $f14
+0x41 0x32 0x07 0x46 # CHECK: sub.s $f9, $f6, $f7
+0x22 0x48 0xc7 0x00 # CHECK: sub $9, $6, $7
+0x23 0x20 0x65 0x00 # CHECK: subu $4, $3, $5
+0x0d 0x20 0xb8 0x4c # CHECK: suxc1 $f4, $24($5)
+0x18 0x00 0xa4 0xac # CHECK: sw $4, 24($5)
+0xc6 0x23 0xe9 0xe4 # CHECK: swc1 $f9, 9158($7)
+0x10 0x00 0xa4 0xa8 # CHECK: swl $4, 16($5)
+0x10 0x00 0xe6 0xb8 # CHECK: swr $6, 16($7)
+0x08 0xd0 0xd2 0x4e # CHECK: swxc1 $f26, $18($22)
+0xcf 0x01 0x00 0x00 # CHECK: sync 7
+0x0d 0x73 0x20 0x46 # CHECK: trunc.w.d $f12, $f14
+0x8d 0x39 0x00 0x46 # CHECK: trunc.w.s $f6, $f7
+0xa0 0x30 0x07 0x7c # CHECK: wsbh $6, $7
+0x26 0x18 0x65 0x00 # CHECK: xor $3, $3, $5
+0x67 0x45 0xc9 0x38 # CHECK: xori $9, $6, 17767
diff --git a/test/MC/Disassembler/Mips/mips32r5/valid-mips32r5-le.txt b/test/MC/Disassembler/Mips/mips32r5/valid-mips32r5-le.txt
deleted file mode 100644
index 62977dc3266..00000000000
--- a/test/MC/Disassembler/Mips/mips32r5/valid-mips32r5-le.txt
+++ /dev/null
@@ -1,169 +0,0 @@
-# RUN: llvm-mc --disassemble %s -triple=mipsel-unknown-linux -mcpu=mips32r5 | FileCheck %s
-0x05 0x73 0x20 0x46 # CHECK: abs.d $f12, $f14
-0x85 0x39 0x00 0x46 # CHECK: abs.s $f6, $f7
-0x20 0x48 0xc7 0x00 # CHECK: add $9, $6, $7
-0x00 0x62 0x2e 0x46 # CHECK: add.d $f8, $f12, $f14
-0x40 0x32 0x07 0x46 # CHECK: add.s $f9, $f6, $f7
-0x67 0x45 0xc9 0x20 # CHECK: addi $9, $6, 17767
-0x67 0xc5 0xc9 0x24 # CHECK: addiu $9, $6, -15001
-0x21 0x48 0xc7 0x00 # CHECK: addu $9, $6, $7
-0x24 0x48 0xc7 0x00 # CHECK: and $9, $6, $7
-0x67 0x45 0xc9 0x30 # CHECK: andi $9, $6, 17767
-0x4c 0x01 0x00 0x10 # CHECK: b 1332
-0x4c 0x01 0x00 0x45 # CHECK: bc1f 1332
-0x4c 0x01 0x1c 0x45 # CHECK: bc1f $fcc7, 1332
-0x4c 0x01 0x01 0x45 # CHECK: bc1t 1332
-0x4c 0x01 0x1d 0x45 # CHECK: bc1t $fcc7, 1332
-0x4c 0x01 0x26 0x11 # CHECK: beq $9, $6, 1332
-0x4c 0x01 0xc1 0x04 # CHECK: bgez $6, 1332
-0x4c 0x01 0xd1 0x04 # CHECK: bgezal $6, 1332
-0x4c 0x01 0xc0 0x1c # CHECK: bgtz $6, 1332
-0x4c 0x01 0xc0 0x18 # CHECK: blez $6, 1332
-0x4c 0x01 0x26 0x15 # CHECK: bne $9, $6, 1332
-0x32 0x60 0x2e 0x46 # CHECK: c.eq.d $f12, $f14
-0x32 0x30 0x07 0x46 # CHECK: c.eq.s $f6, $f7
-0x30 0x60 0x2e 0x46 # CHECK: c.f.d $f12, $f14
-0x30 0x30 0x07 0x46 # CHECK: c.f.s $f6, $f7
-0x3e 0x60 0x2e 0x46 # CHECK: c.le.d $f12, $f14
-0x3e 0x30 0x07 0x46 # CHECK: c.le.s $f6, $f7
-0x3c 0x60 0x2e 0x46 # CHECK: c.lt.d $f12, $f14
-0x3c 0x30 0x07 0x46 # CHECK: c.lt.s $f6, $f7
-0x3d 0x60 0x2e 0x46 # CHECK: c.nge.d $f12, $f14
-0x3d 0x30 0x07 0x46 # CHECK: c.nge.s $f6, $f7
-0x3b 0x60 0x2e 0x46 # CHECK: c.ngl.d $f12, $f14
-0x3b 0x30 0x07 0x46 # CHECK: c.ngl.s $f6, $f7
-0x39 0x60 0x2e 0x46 # CHECK: c.ngle.d $f12, $f14
-0x39 0x30 0x07 0x46 # CHECK: c.ngle.s $f6, $f7
-0x3f 0x60 0x2e 0x46 # CHECK: c.ngt.d $f12, $f14
-0x3f 0x30 0x07 0x46 # CHECK: c.ngt.s $f6, $f7
-0x36 0x60 0x2e 0x46 # CHECK: c.ole.d $f12, $f14
-0x36 0x30 0x07 0x46 # CHECK: c.ole.s $f6, $f7
-0x34 0x60 0x2e 0x46 # CHECK: c.olt.d $f12, $f14
-0x34 0x30 0x07 0x46 # CHECK: c.olt.s $f6, $f7
-0x3a 0x60 0x2e 0x46 # CHECK: c.seq.d $f12, $f14
-0x3a 0x30 0x07 0x46 # CHECK: c.seq.s $f6, $f7
-0x38 0x60 0x2e 0x46 # CHECK: c.sf.d $f12, $f14
-0x38 0x30 0x07 0x46 # CHECK: c.sf.s $f6, $f7
-0x33 0x60 0x2e 0x46 # CHECK: c.ueq.d $f12, $f14
-0x33 0xe0 0x12 0x46 # CHECK: c.ueq.s $f28, $f18
-0x37 0x60 0x2e 0x46 # CHECK: c.ule.d $f12, $f14
-0x37 0x30 0x07 0x46 # CHECK: c.ule.s $f6, $f7
-0x35 0x60 0x2e 0x46 # CHECK: c.ult.d $f12, $f14
-0x35 0x30 0x07 0x46 # CHECK: c.ult.s $f6, $f7
-0x31 0x60 0x2e 0x46 # CHECK: c.un.d $f12, $f14
-0x31 0x30 0x07 0x46 # CHECK: c.un.s $f6, $f7
-0x0e 0x73 0x20 0x46 # CHECK: ceil.w.d $f12, $f14
-0x8e 0x39 0x00 0x46 # CHECK: ceil.w.s $f6, $f7
-0x00 0x38 0x46 0x44 # CHECK: cfc1 $6, $7
-0x21 0x30 0xe6 0x70 # CHECK: clo $6, $7
-0x20 0x30 0xe6 0x70 # CHECK: clz $6, $7
-0x00 0x38 0xc6 0x44 # CHECK: ctc1 $6, $7
-0xa1 0x39 0x00 0x46 # CHECK: cvt.d.s $f6, $f7
-0x21 0x73 0x80 0x46 # CHECK: cvt.d.w $f12, $f14
-0x25 0x73 0x20 0x46 # CHECK: cvt.l.d $f12, $f14
-0xa5 0x39 0x00 0x46 # CHECK: cvt.l.s $f6, $f7
-0x20 0x73 0x20 0x46 # CHECK: cvt.s.d $f12, $f14
-0xa0 0x39 0x80 0x46 # CHECK: cvt.s.w $f6, $f7
-0x24 0x73 0x20 0x46 # CHECK: cvt.w.d $f12, $f14
-0xa4 0x39 0x00 0x46 # CHECK: cvt.w.s $f6, $f7
-0x00 0x60 0x7e 0x41 # CHECK: di $fp
-0x00 0x60 0x60 0x41 # CHECK: di
-0x20 0x60 0x6e 0x41 # CHECK: ei $14
-0x20 0x60 0x60 0x41 # CHECK: ei
-0x0f 0x73 0x20 0x46 # CHECK: floor.w.d $f12, $f14
-0x8f 0x39 0x00 0x46 # CHECK: floor.w.s $f6, $f7
-0x84 0x61 0x33 0x7d # CHECK: ins $19, $9, 6, 7
-0x4c 0x01 0x00 0x08 # CHECK: j 1328
-0x4c 0x01 0x00 0x0c # CHECK: jal 1328
-0x4c 0x01 0x00 0x74 # CHECK: jalx 1328
-0x09 0xf8 0xe0 0x00 # CHECK: jalr $7
-0x09 0xfc 0x80 0x00 # CHECK: jalr.hb $4
-0x09 0x24 0xa0 0x00 # CHECK: jalr.hb $4, $5
-0x08 0x00 0xe0 0x00 # CHECK: jr $7
-0xc6 0x23 0xa4 0x80 # CHECK: lb $4, 9158($5)
-0x06 0x00 0xa4 0x90 # CHECK: lbu $4, 6($5)
-0xc6 0x23 0xe9 0xd4 # CHECK: ldc1 $f9, 9158($7)
-0x01 0x02 0xf7 0x4d # CHECK: ldxc1 $f8, $23($15)
-0x0c 0x00 0xa4 0x84 # CHECK: lh $4, 12($5)
-0x0c 0x00 0xa4 0x84 # CHECK: lh $4, 12($5)
-0xc6 0x23 0xe9 0xc0 # CHECK: ll $9, 9158($7)
-0x67 0x45 0x06 0x3c # CHECK: lui $6, 17767
-0x05 0x00 0xa6 0x4c # CHECK: luxc1 $f0, $6($5)
-0x18 0x00 0xa4 0x8c # CHECK: lw $4, 24($5)
-0xc6 0x23 0xe9 0xc4 # CHECK: lwc1 $f9, 9158($7)
-0x03 0x00 0x82 0x88 # CHECK: lwl $2, 3($4)
-0x10 0x00 0xa3 0x98 # CHECK: lwr $3, 16($5)
-0x00 0x05 0xcc 0x4d # CHECK: lwxc1 $f20, $12($14)
-0x00 0x00 0xc7 0x70 # CHECK: madd $6, $7
-0xa1 0xd4 0x94 0x4e # CHECK: madd.d $f18, $f20, $f26, $f20
-0x60 0x98 0xf9 0x4f # CHECK: madd.s $f1, $f31, $f19, $f25
-0x01 0x00 0xc7 0x70 # CHECK: maddu $6, $7
-0x00 0x38 0x06 0x44 # CHECK: mfc1 $6, $f7
-0x10 0x28 0x00 0x00 # CHECK: mfhi $5
-0x00 0xc0 0x7e 0x44 # CHECK: mfhc1 $fp, $f24
-0x12 0x28 0x00 0x00 # CHECK: mflo $5
-0x86 0x41 0x20 0x46 # CHECK: mov.d $f6, $f8
-0x86 0x39 0x00 0x46 # CHECK: mov.s $f6, $f7
-0x04 0x00 0xc7 0x70 # CHECK: msub $6, $7
-0xa9 0xf2 0x52 0x4c # CHECK: msub.d $f10, $f2, $f30, $f18
-0x28 0x53 0x70 0x4e # CHECK: msub.s $f12, $f19, $f10, $f16
-0x05 0x00 0xc7 0x70 # CHECK: msubu $6, $7
-0x00 0x38 0x86 0x44 # CHECK: mtc1 $6, $f7
-0x11 0x00 0xe0 0x00 # CHECK: mthi $7
-0x00 0x80 0xe0 0x44 # CHECK: mthc1 $zero, $f16
-0x13 0x00 0xe0 0x00 # CHECK: mtlo $7
-0x02 0x62 0x2e 0x46 # CHECK: mul.d $f8, $f12, $f14
-0x42 0x32 0x07 0x46 # CHECK: mul.s $f9, $f6, $f7
-0x02 0x48 0xc7 0x70 # CHECK: mul $9, $6, $7
-0x18 0x00 0x65 0x00 # CHECK: mult $3, $5
-0x19 0x00 0x65 0x00 # CHECK: multu $3, $5
-0x07 0x73 0x20 0x46 # CHECK: neg.d $f12, $f14
-0x87 0x39 0x00 0x46 # CHECK: neg.s $f6, $f7
-0xb1 0x74 0x54 0x4d # CHECK: nmadd.d $f18, $f10, $f14, $f20
-0x30 0xc8 0xac 0x4c # CHECK: nmadd.s $f0, $f5, $f25, $f12
-0x00 0x00 0x00 0x00 # CHECK: nop
-0x27 0x48 0xc7 0x00 # CHECK: nor $9, $6, $7
-0xb9 0x87 0x1e 0x4d # CHECK: nmsub.d $f30, $f8, $f16, $f30
-0x78 0x98 0x04 0x4f # CHECK: nmsub.s $f1, $f24, $f19, $f4
-0x25 0x18 0x65 0x00 # CHECK: or $3, $3, $5
-0x67 0x45 0xc9 0x34 # CHECK: ori $9, $6, 17767
-0xc2 0x49 0x26 0x00 # CHECK: rotr $9, $6, 7
-0x46 0x48 0xe6 0x00 # CHECK: rotrv $9, $6, $7
-0x0c 0x73 0x20 0x46 # CHECK: round.w.d $f12, $f14
-0x8c 0x39 0x00 0x46 # CHECK: round.w.s $f6, $f7
-0xc6 0x23 0xa4 0xa0 # CHECK: sb $4, 9158($5)
-0x06 0x00 0xa4 0xa0 # CHECK: sb $4, 6($5)
-0xc6 0x23 0xe9 0xe0 # CHECK: sc $9, 9158($7)
-0xc6 0x23 0xe9 0xf4 # CHECK: sdc1 $f9, 9158($7)
-0x09 0x40 0x24 0x4f # CHECK: sdxc1 $f8, $4($25)
-0x20 0x34 0x07 0x7c # CHECK: seb $6, $7
-0x20 0x36 0x07 0x7c # CHECK: seh $6, $7
-0xc6 0x23 0xa4 0xa4 # CHECK: sh $4, 9158($5)
-0xc0 0x21 0x03 0x00 # CHECK: sll $4, $3, 7
-0x04 0x10 0xa3 0x00 # CHECK: sllv $2, $3, $5
-0x2a 0x18 0x65 0x00 # CHECK: slt $3, $3, $5
-0x67 0x00 0x63 0x28 # CHECK: slti $3, $3, 103
-0x67 0x00 0x63 0x2c # CHECK: sltiu $3, $3, 103
-0x2b 0x18 0x65 0x00 # CHECK: sltu $3, $3, $5
-0x04 0x73 0x20 0x46 # CHECK: sqrt.d $f12, $f14
-0x84 0x39 0x00 0x46 # CHECK: sqrt.s $f6, $f7
-0xc3 0x21 0x03 0x00 # CHECK: sra $4, $3, 7
-0x07 0x10 0xa3 0x00 # CHECK: srav $2, $3, $5
-0xc2 0x21 0x03 0x00 # CHECK: srl $4, $3, 7
-0x06 0x10 0xa3 0x00 # CHECK: srlv $2, $3, $5
-0x01 0x62 0x2e 0x46 # CHECK: sub.d $f8, $f12, $f14
-0x41 0x32 0x07 0x46 # CHECK: sub.s $f9, $f6, $f7
-0x22 0x48 0xc7 0x00 # CHECK: sub $9, $6, $7
-0x23 0x20 0x65 0x00 # CHECK: subu $4, $3, $5
-0x0d 0x20 0xb8 0x4c # CHECK: suxc1 $f4, $24($5)
-0x18 0x00 0xa4 0xac # CHECK: sw $4, 24($5)
-0xc6 0x23 0xe9 0xe4 # CHECK: swc1 $f9, 9158($7)
-0x10 0x00 0xa4 0xa8 # CHECK: swl $4, 16($5)
-0x10 0x00 0xe6 0xb8 # CHECK: swr $6, 16($7)
-0x08 0xd0 0xd2 0x4e # CHECK: swxc1 $f26, $18($22)
-0xcf 0x01 0x00 0x00 # CHECK: sync 7
-0x0d 0x73 0x20 0x46 # CHECK: trunc.w.d $f12, $f14
-0x8d 0x39 0x00 0x46 # CHECK: trunc.w.s $f6, $f7
-0xa0 0x30 0x07 0x7c # CHECK: wsbh $6, $7
-0x26 0x18 0x65 0x00 # CHECK: xor $3, $3, $5
-0x67 0x45 0xc9 0x38 # CHECK: xori $9, $6, 17767
diff --git a/test/MC/Disassembler/Mips/mips32r5/valid-mips32r5.txt b/test/MC/Disassembler/Mips/mips32r5/valid-mips32r5.txt
index 39c46440946..0593fda2111 100644
--- a/test/MC/Disassembler/Mips/mips32r5/valid-mips32r5.txt
+++ b/test/MC/Disassembler/Mips/mips32r5/valid-mips32r5.txt
@@ -1,169 +1,172 @@
 # RUN: llvm-mc --disassemble %s -triple=mips-unknown-linux -mcpu=mips32r5 | FileCheck %s
-0x46 0x20 0x73 0x05 # CHECK: abs.d $f12, $f14
-0x46 0x00 0x39 0x85 # CHECK: abs.s $f6, $f7
+0x00 0x00 0x00 0x00 # CHECK: nop
+0x00 0x00 0x01 0xcf # CHECK: sync 7
+0x00 0x00 0x28 0x10 # CHECK: mfhi $5
+0x00 0x00 0x28 0x12 # CHECK: mflo $5
+0x00 0x03 0x21 0xc0 # CHECK: sll $4, $3, 7
+0x00 0x03 0x21 0xc2 # CHECK: srl $4, $3, 7
+0x00 0x03 0x21 0xc3 # CHECK: sra $4, $3, 7
+0x00 0x26 0x49 0xc2 # CHECK: rotr $9, $6, 7
+0x00 0x65 0x00 0x18 # CHECK: mult $3, $5
+0x00 0x65 0x00 0x19 # CHECK: multu $3, $5
+0x00 0x65 0x18 0x25 # CHECK: or $3, $3, $5
+0x00 0x65 0x18 0x26 # CHECK: xor $3, $3, $5
+0x00 0x65 0x18 0x2a # CHECK: slt $3, $3, $5
+0x00 0x65 0x18 0x2b # CHECK: sltu $3, $3, $5
+0x00 0x65 0x20 0x23 # CHECK: subu $4, $3, $5
+0x00 0x80 0xfc 0x09 # CHECK: jalr.hb $4
+0x00 0xa0 0x24 0x09 # CHECK: jalr.hb $4, $5
+0x00 0xa3 0x10 0x04 # CHECK: sllv $2, $3, $5
+0x00 0xa3 0x10 0x06 # CHECK: srlv $2, $3, $5
+0x00 0xa3 0x10 0x07 # CHECK: srav $2, $3, $5
 0x00 0xc7 0x48 0x20 # CHECK: add $9, $6, $7
-0x46 0x2e 0x62 0x00 # CHECK: add.d $f8, $f12, $f14
-0x46 0x07 0x32 0x40 # CHECK: add.s $f9, $f6, $f7
-0x20 0xc9 0x45 0x67 # CHECK: addi $9, $6, 17767
-0x24 0xc9 0xc5 0x67 # CHECK: addiu $9, $6, -15001
 0x00 0xc7 0x48 0x21 # CHECK: addu $9, $6, $7
+0x00 0xc7 0x48 0x22 # CHECK: sub $9, $6, $7
 0x00 0xc7 0x48 0x24 # CHECK: and $9, $6, $7
-0x30 0xc9 0x45 0x67 # CHECK: andi $9, $6, 17767
+0x00 0xc7 0x48 0x27 # CHECK: nor $9, $6, $7
+0x00 0xe0 0x00 0x08 # CHECK: jr $7
+0x00 0xe0 0x00 0x11 # CHECK: mthi $7
+0x00 0xe0 0x00 0x13 # CHECK: mtlo $7
+0x00 0xe0 0xf8 0x09 # CHECK: jalr $7
+0x00 0xe6 0x48 0x46 # CHECK: rotrv $9, $6, $7
+0x04 0xc1 0x01 0x4c # CHECK: bgez $6, 1332
+0x04 0xd1 0x01 0x4c # CHECK: bgezal $6, 1332
+0x08 0x00 0x01 0x4c # CHECK: j 1328
+0x0c 0x00 0x01 0x4c # CHECK: jal 1328
 0x10 0x00 0x01 0x4c # CHECK: b 1332
+0x11 0x26 0x01 0x4c # CHECK: beq $9, $6, 1332
+0x15 0x26 0x01 0x4c # CHECK: bne $9, $6, 1332
+0x18 0xc0 0x01 0x4c # CHECK: blez $6, 1332
+0x1c 0xc0 0x01 0x4c # CHECK: bgtz $6, 1332
+0x20 0xc9 0x45 0x67 # CHECK: addi $9, $6, 17767
+0x24 0xc9 0xc5 0x67 # CHECK: addiu $9, $6, -15001
+0x28 0x63 0x00 0x67 # CHECK: slti $3, $3, 103
+0x2c 0x63 0x00 0x67 # CHECK: sltiu $3, $3, 103
+0x30 0xc9 0x45 0x67 # CHECK: andi $9, $6, 17767
+0x34 0xc9 0x45 0x67 # CHECK: ori $9, $6, 17767
+0x38 0xc9 0x45 0x67 # CHECK: xori $9, $6, 17767
+0x3c 0x06 0x45 0x67 # CHECK: lui $6, 17767
+0x40 0x08 0x78 0x01 # CHECK: mfc0 $8, $15, 1
+0x40 0x89 0x78 0x01 # CHECK: mtc0 $9, $15, 1
+0x41 0x60 0x60 0x00 # CHECK: di
+0x41 0x60 0x60 0x20 # CHECK: ei
+0x41 0x6e 0x60 0x20 # CHECK: ei $14
+0x41 0x7e 0x60 0x00 # CHECK: di $fp
+0x44 0x06 0x38 0x00 # CHECK: mfc1 $6, $f7
+0x44 0x46 0x38 0x00 # CHECK: cfc1 $6, $7
+0x44 0x7e 0xc0 0x00 # CHECK: mfhc1 $fp, $f24
+0x44 0x86 0x38 0x00 # CHECK: mtc1 $6, $f7
+0x44 0xc6 0x38 0x00 # CHECK: ctc1 $6, $7
+0x44 0xe0 0x80 0x00 # CHECK: mthc1 $zero, $f16
 0x45 0x00 0x01 0x4c # CHECK: bc1f 1332
-0x45 0x1c 0x01 0x4c # CHECK: bc1f $fcc7, 1332
 0x45 0x01 0x01 0x4c # CHECK: bc1t 1332
+0x45 0x1c 0x01 0x4c # CHECK: bc1f $fcc7, 1332
 0x45 0x1d 0x01 0x4c # CHECK: bc1t $fcc7, 1332
-0x11 0x26 0x01 0x4c # CHECK: beq $9, $6, 1332
-0x04 0xc1 0x01 0x4c # CHECK: bgez $6, 1332
-0x04 0xd1 0x01 0x4c # CHECK: bgezal $6, 1332
-0x1c 0xc0 0x01 0x4c # CHECK: bgtz $6, 1332
-0x18 0xc0 0x01 0x4c # CHECK: blez $6, 1332
-0x15 0x26 0x01 0x4c # CHECK: bne $9, $6, 1332
-0x46 0x2e 0x60 0x32 # CHECK: c.eq.d $f12, $f14
-0x46 0x07 0x30 0x32 # CHECK: c.eq.s $f6, $f7
-0x46 0x2e 0x60 0x30 # CHECK: c.f.d $f12, $f14
+0x46 0x00 0x39 0x84 # CHECK: sqrt.s $f6, $f7
+0x46 0x00 0x39 0x85 # CHECK: abs.s $f6, $f7
+0x46 0x00 0x39 0x86 # CHECK: mov.s $f6, $f7
+0x46 0x00 0x39 0x87 # CHECK: neg.s $f6, $f7
+0x46 0x00 0x39 0x8c # CHECK: round.w.s $f6, $f7
+0x46 0x00 0x39 0x8d # CHECK: trunc.w.s $f6, $f7
+0x46 0x00 0x39 0x8e # CHECK: ceil.w.s $f6, $f7
+0x46 0x00 0x39 0x8f # CHECK: floor.w.s $f6, $f7
+0x46 0x00 0x39 0xa1 # CHECK: cvt.d.s $f6, $f7
+0x46 0x00 0x39 0xa4 # CHECK: cvt.w.s $f6, $f7
+0x46 0x00 0x39 0xa5 # CHECK: cvt.l.s $f6, $f7
 0x46 0x07 0x30 0x30 # CHECK: c.f.s $f6, $f7
-0x46 0x2e 0x60 0x3e # CHECK: c.le.d $f12, $f14
-0x46 0x07 0x30 0x3e # CHECK: c.le.s $f6, $f7
-0x46 0x2e 0x60 0x3c # CHECK: c.lt.d $f12, $f14
+0x46 0x07 0x30 0x31 # CHECK: c.un.s $f6, $f7
+0x46 0x07 0x30 0x32 # CHECK: c.eq.s $f6, $f7
+0x46 0x07 0x30 0x34 # CHECK: c.olt.s $f6, $f7
+0x46 0x07 0x30 0x35 # CHECK: c.ult.s $f6, $f7
+0x46 0x07 0x30 0x36 # CHECK: c.ole.s $f6, $f7
+0x46 0x07 0x30 0x37 # CHECK: c.ule.s $f6, $f7
+0x46 0x07 0x30 0x38 # CHECK: c.sf.s $f6, $f7
+0x46 0x07 0x30 0x39 # CHECK: c.ngle.s $f6, $f7
+0x46 0x07 0x30 0x3a # CHECK: c.seq.s $f6, $f7
+0x46 0x07 0x30 0x3b # CHECK: c.ngl.s $f6, $f7
 0x46 0x07 0x30 0x3c # CHECK: c.lt.s $f6, $f7
-0x46 0x2e 0x60 0x3d # CHECK: c.nge.d $f12, $f14
 0x46 0x07 0x30 0x3d # CHECK: c.nge.s $f6, $f7
-0x46 0x2e 0x60 0x3b # CHECK: c.ngl.d $f12, $f14
-0x46 0x07 0x30 0x3b # CHECK: c.ngl.s $f6, $f7
-0x46 0x2e 0x60 0x39 # CHECK: c.ngle.d $f12, $f14
-0x46 0x07 0x30 0x39 # CHECK: c.ngle.s $f6, $f7
-0x46 0x2e 0x60 0x3f # CHECK: c.ngt.d $f12, $f14
+0x46 0x07 0x30 0x3e # CHECK: c.le.s $f6, $f7
 0x46 0x07 0x30 0x3f # CHECK: c.ngt.s $f6, $f7
-0x46 0x2e 0x60 0x36 # CHECK: c.ole.d $f12, $f14
-0x46 0x07 0x30 0x36 # CHECK: c.ole.s $f6, $f7
-0x46 0x2e 0x60 0x34 # CHECK: c.olt.d $f12, $f14
-0x46 0x07 0x30 0x34 # CHECK: c.olt.s $f6, $f7
-0x46 0x2e 0x60 0x3a # CHECK: c.seq.d $f12, $f14
-0x46 0x07 0x30 0x3a # CHECK: c.seq.s $f6, $f7
-0x46 0x2e 0x60 0x38 # CHECK: c.sf.d $f12, $f14
-0x46 0x07 0x30 0x38 # CHECK: c.sf.s $f6, $f7
-0x46 0x2e 0x60 0x33 # CHECK: c.ueq.d $f12, $f14
+0x42 0x00 0x00 0x58 # CHECK: eretnc
+0x46 0x07 0x32 0x40 # CHECK: add.s $f9, $f6, $f7
+0x46 0x07 0x32 0x41 # CHECK: sub.s $f9, $f6, $f7
+0x46 0x07 0x32 0x42 # CHECK: mul.s $f9, $f6, $f7
 0x46 0x12 0xe0 0x33 # CHECK: c.ueq.s $f28, $f18
-0x46 0x2e 0x60 0x37 # CHECK: c.ule.d $f12, $f14
-0x46 0x07 0x30 0x37 # CHECK: c.ule.s $f6, $f7
-0x46 0x2e 0x60 0x35 # CHECK: c.ult.d $f12, $f14
-0x46 0x07 0x30 0x35 # CHECK: c.ult.s $f6, $f7
-0x46 0x2e 0x60 0x31 # CHECK: c.un.d $f12, $f14
-0x46 0x07 0x30 0x31 # CHECK: c.un.s $f6, $f7
+0x46 0x20 0x41 0x86 # CHECK: mov.d $f6, $f8
+0x46 0x20 0x73 0x04 # CHECK: sqrt.d $f12, $f14
+0x46 0x20 0x73 0x05 # CHECK: abs.d $f12, $f14
+0x46 0x20 0x73 0x07 # CHECK: neg.d $f12, $f14
+0x46 0x20 0x73 0x0c # CHECK: round.w.d $f12, $f14
+0x46 0x20 0x73 0x0d # CHECK: trunc.w.d $f12, $f14
 0x46 0x20 0x73 0x0e # CHECK: ceil.w.d $f12, $f14
-0x46 0x00 0x39 0x8e # CHECK: ceil.w.s $f6, $f7
-0x44 0x46 0x38 0x00 # CHECK: cfc1 $6, $7
-0x70 0xe6 0x30 0x21 # CHECK: clo $6, $7
-0x70 0xe6 0x30 0x20 # CHECK: clz $6, $7
-0x44 0xc6 0x38 0x00 # CHECK: ctc1 $6, $7
-0x46 0x00 0x39 0xa1 # CHECK: cvt.d.s $f6, $f7
-0x46 0x80 0x73 0x21 # CHECK: cvt.d.w $f12, $f14
-0x46 0x20 0x73 0x25 # CHECK: cvt.l.d $f12, $f14
-0x46 0x00 0x39 0xa5 # CHECK: cvt.l.s $f6, $f7
+0x46 0x20 0x73 0x0f # CHECK: floor.w.d $f12, $f14
 0x46 0x20 0x73 0x20 # CHECK: cvt.s.d $f12, $f14
-0x46 0x80 0x39 0xa0 # CHECK: cvt.s.w $f6, $f7
 0x46 0x20 0x73 0x24 # CHECK: cvt.w.d $f12, $f14
-0x46 0x00 0x39 0xa4 # CHECK: cvt.w.s $f6, $f7
-0x41 0x7e 0x60 0x00 # CHECK: di $fp
-0x41 0x60 0x60 0x00 # CHECK: di
-0x41 0x6e 0x60 0x20 # CHECK: ei $14
-0x41 0x60 0x60 0x20 # CHECK: ei
-0x46 0x20 0x73 0x0f # CHECK: floor.w.d $f12, $f14
-0x46 0x00 0x39 0x8f # CHECK: floor.w.s $f6, $f7
-0x7d 0x33 0x61 0x84 # CHECK: ins $19, $9, 6, 7
-0x08 0x00 0x01 0x4c # CHECK: j 1328
-0x0c 0x00 0x01 0x4c # CHECK: jal 1328
-0x74 0x00 0x01 0x4c # CHECK: jalx 1328
-0x00 0xe0 0xf8 0x09 # CHECK: jalr $7
-0x00 0x80 0xfc 0x09 # CHECK: jalr.hb $4
-0x00 0xa0 0x24 0x09 # CHECK: jalr.hb $4, $5
-0x00 0xe0 0x00 0x08 # CHECK: jr $7
-0x80 0xa4 0x23 0xc6 # CHECK: lb $4, 9158($5)
-0x90 0xa4 0x00 0x06 # CHECK: lbu $4, 6($5)
-0xd4 0xe9 0x23 0xc6 # CHECK: ldc1 $f9, 9158($7)
-0x4d 0xf7 0x02 0x01 # CHECK: ldxc1 $f8, $23($15)
-0x84 0xa4 0x00 0x0c # CHECK: lh $4, 12($5)
-0x84 0xa4 0x00 0x0c # CHECK: lh $4, 12($5)
-0xc0 0xe9 0x23 0xc6 # CHECK: ll $9, 9158($7)
-0x3c 0x06 0x45 0x67 # CHECK: lui $6, 17767
+0x46 0x20 0x73 0x25 # CHECK: cvt.l.d $f12, $f14
+0x46 0x2e 0x60 0x30 # CHECK: c.f.d $f12, $f14
+0x46 0x2e 0x60 0x31 # CHECK: c.un.d $f12, $f14
+0x46 0x2e 0x60 0x32 # CHECK: c.eq.d $f12, $f14
+0x46 0x2e 0x60 0x33 # CHECK: c.ueq.d $f12, $f14
+0x46 0x2e 0x60 0x34 # CHECK: c.olt.d $f12, $f14
+0x46 0x2e 0x60 0x35 # CHECK: c.ult.d $f12, $f14
+0x46 0x2e 0x60 0x36 # CHECK: c.ole.d $f12, $f14
+0x46 0x2e 0x60 0x37 # CHECK: c.ule.d $f12, $f14
+0x46 0x2e 0x60 0x38 # CHECK: c.sf.d $f12, $f14
+0x46 0x2e 0x60 0x39 # CHECK: c.ngle.d $f12, $f14
+0x46 0x2e 0x60 0x3a # CHECK: c.seq.d $f12, $f14
+0x46 0x2e 0x60 0x3b # CHECK: c.ngl.d $f12, $f14
+0x46 0x2e 0x60 0x3c # CHECK: c.lt.d $f12, $f14
+0x46 0x2e 0x60 0x3d # CHECK: c.nge.d $f12, $f14
+0x46 0x2e 0x60 0x3e # CHECK: c.le.d $f12, $f14
+0x46 0x2e 0x60 0x3f # CHECK: c.ngt.d $f12, $f14
+0x46 0x2e 0x62 0x00 # CHECK: add.d $f8, $f12, $f14
+0x46 0x2e 0x62 0x01 # CHECK: sub.d $f8, $f12, $f14
+0x46 0x2e 0x62 0x02 # CHECK: mul.d $f8, $f12, $f14
+0x46 0x80 0x39 0xa0 # CHECK: cvt.s.w $f6, $f7
+0x46 0x80 0x73 0x21 # CHECK: cvt.d.w $f12, $f14
+0x4c 0x52 0xf2 0xa9 # CHECK: msub.d $f10, $f2, $f30, $f18
 0x4c 0xa6 0x00 0x05 # CHECK: luxc1 $f0, $6($5)
-0x8c 0xa4 0x00 0x18 # CHECK: lw $4, 24($5)
-0xc4 0xe9 0x23 0xc6 # CHECK: lwc1 $f9, 9158($7)
-0x88 0x82 0x00 0x03 # CHECK: lwl $2, 3($4)
-0x98 0xa3 0x00 0x10 # CHECK: lwr $3, 16($5)
+0x4c 0xac 0xc8 0x30 # CHECK: nmadd.s $f0, $f5, $f25, $f12
+0x4c 0xb8 0x20 0x0d # CHECK: suxc1 $f4, $24($5)
+0x4d 0x1e 0x87 0xb9 # CHECK: nmsub.d $f30, $f8, $f16, $f30
+0x4d 0x54 0x74 0xb1 # CHECK: nmadd.d $f18, $f10, $f14, $f20
 0x4d 0xcc 0x05 0x00 # CHECK: lwxc1 $f20, $12($14)
-0x70 0xc7 0x00 0x00 # CHECK: madd $6, $7
+0x4d 0xf7 0x02 0x01 # CHECK: ldxc1 $f8, $23($15)
+0x4e 0x70 0x53 0x28 # CHECK: msub.s $f12, $f19, $f10, $f16
 0x4e 0x94 0xd4 0xa1 # CHECK: madd.d $f18, $f20, $f26, $f20
+0x4e 0xd2 0xd0 0x08 # CHECK: swxc1 $f26, $18($22)
+0x4f 0x04 0x98 0x78 # CHECK: nmsub.s $f1, $f24, $f19, $f4
+0x4f 0x24 0x40 0x09 # CHECK: sdxc1 $f8, $4($25)
 0x4f 0xf9 0x98 0x60 # CHECK: madd.s $f1, $f31, $f19, $f25
+0x70 0xc7 0x00 0x00 # CHECK: madd $6, $7
 0x70 0xc7 0x00 0x01 # CHECK: maddu $6, $7
-0x44 0x06 0x38 0x00 # CHECK: mfc1 $6, $f7
-0x00 0x00 0x28 0x10 # CHECK: mfhi $5
-0x44 0x7e 0xc0 0x00 # CHECK: mfhc1 $fp, $f24
-0x00 0x00 0x28 0x12 # CHECK: mflo $5
-0x46 0x20 0x41 0x86 # CHECK: mov.d $f6, $f8
-0x46 0x00 0x39 0x86 # CHECK: mov.s $f6, $f7
 0x70 0xc7 0x00 0x04 # CHECK: msub $6, $7
-0x4c 0x52 0xf2 0xa9 # CHECK: msub.d $f10, $f2, $f30, $f18
-0x4e 0x70 0x53 0x28 # CHECK: msub.s $f12, $f19, $f10, $f16
 0x70 0xc7 0x00 0x05 # CHECK: msubu $6, $7
-0x44 0x86 0x38 0x00 # CHECK: mtc1 $6, $f7
-0x00 0xe0 0x00 0x11 # CHECK: mthi $7
-0x44 0xe0 0x80 0x00 # CHECK: mthc1 $zero, $f16
-0x00 0xe0 0x00 0x13 # CHECK: mtlo $7
-0x46 0x2e 0x62 0x02 # CHECK: mul.d $f8, $f12, $f14
-0x46 0x07 0x32 0x42 # CHECK: mul.s $f9, $f6, $f7
 0x70 0xc7 0x48 0x02 # CHECK: mul $9, $6, $7
-0x00 0x65 0x00 0x18 # CHECK: mult $3, $5
-0x00 0x65 0x00 0x19 # CHECK: multu $3, $5
-0x46 0x20 0x73 0x07 # CHECK: neg.d $f12, $f14
-0x46 0x00 0x39 0x87 # CHECK: neg.s $f6, $f7
-0x4d 0x54 0x74 0xb1 # CHECK: nmadd.d $f18, $f10, $f14, $f20
-0x4c 0xac 0xc8 0x30 # CHECK: nmadd.s $f0, $f5, $f25, $f12
-0x00 0x00 0x00 0x00 # CHECK: nop
-0x00 0xc7 0x48 0x27 # CHECK: nor $9, $6, $7
-0x4d 0x1e 0x87 0xb9 # CHECK: nmsub.d $f30, $f8, $f16, $f30
-0x4f 0x04 0x98 0x78 # CHECK: nmsub.s $f1, $f24, $f19, $f4
-0x00 0x65 0x18 0x25 # CHECK: or $3, $3, $5
-0x34 0xc9 0x45 0x67 # CHECK: ori $9, $6, 17767
-0x00 0x26 0x49 0xc2 # CHECK: rotr $9, $6, 7
-0x00 0xe6 0x48 0x46 # CHECK: rotrv $9, $6, $7
-0x46 0x20 0x73 0x0c # CHECK: round.w.d $f12, $f14
-0x46 0x00 0x39 0x8c # CHECK: round.w.s $f6, $f7
-0xa0 0xa4 0x23 0xc6 # CHECK: sb $4, 9158($5)
-0xa0 0xa4 0x00 0x06 # CHECK: sb $4, 6($5)
-0xe0 0xe9 0x23 0xc6 # CHECK: sc $9, 9158($7)
-0xf4 0xe9 0x23 0xc6 # CHECK: sdc1 $f9, 9158($7)
-0x4f 0x24 0x40 0x09 # CHECK: sdxc1 $f8, $4($25)
+0x70 0xe6 0x30 0x20 # CHECK: clz $6, $7
+0x70 0xe6 0x30 0x21 # CHECK: clo $6, $7
+0x74 0x00 0x01 0x4c # CHECK: jalx 1328
+0x7c 0x07 0x30 0xa0 # CHECK: wsbh $6, $7
 0x7c 0x07 0x34 0x20 # CHECK: seb $6, $7
 0x7c 0x07 0x36 0x20 # CHECK: seh $6, $7
+0x7d 0x33 0x61 0x84 # CHECK: ins $19, $9, 6, 7
+0x80 0xa4 0x23 0xc6 # CHECK: lb $4, 9158($5)
+0x84 0xa4 0x00 0x0c # CHECK: lh $4, 12($5)
+0x84 0xa4 0x00 0x0c # CHECK: lh $4, 12($5)
+0x88 0x82 0x00 0x03 # CHECK: lwl $2, 3($4)
+0x8c 0xa4 0x00 0x18 # CHECK: lw $4, 24($5)
+0x90 0xa4 0x00 0x06 # CHECK: lbu $4, 6($5)
+0x98 0xa3 0x00 0x10 # CHECK: lwr $3, 16($5)
+0xa0 0xa4 0x00 0x06 # CHECK: sb $4, 6($5)
+0xa0 0xa4 0x23 0xc6 # CHECK: sb $4, 9158($5)
 0xa4 0xa4 0x23 0xc6 # CHECK: sh $4, 9158($5)
-0x00 0x03 0x21 0xc0 # CHECK: sll $4, $3, 7
-0x00 0xa3 0x10 0x04 # CHECK: sllv $2, $3, $5
-0x00 0x65 0x18 0x2a # CHECK: slt $3, $3, $5
-0x28 0x63 0x00 0x67 # CHECK: slti $3, $3, 103
-0x2c 0x63 0x00 0x67 # CHECK: sltiu $3, $3, 103
-0x00 0x65 0x18 0x2b # CHECK: sltu $3, $3, $5
-0x46 0x20 0x73 0x04 # CHECK: sqrt.d $f12, $f14
-0x46 0x00 0x39 0x84 # CHECK: sqrt.s $f6, $f7
-0x00 0x03 0x21 0xc3 # CHECK: sra $4, $3, 7
-0x00 0xa3 0x10 0x07 # CHECK: srav $2, $3, $5
-0x00 0x03 0x21 0xc2 # CHECK: srl $4, $3, 7
-0x00 0xa3 0x10 0x06 # CHECK: srlv $2, $3, $5
-0x46 0x2e 0x62 0x01 # CHECK: sub.d $f8, $f12, $f14
-0x46 0x07 0x32 0x41 # CHECK: sub.s $f9, $f6, $f7
-0x00 0xc7 0x48 0x22 # CHECK: sub $9, $6, $7
-0x00 0x65 0x20 0x23 # CHECK: subu $4, $3, $5
-0x4c 0xb8 0x20 0x0d # CHECK: suxc1 $f4, $24($5)
-0xac 0xa4 0x00 0x18 # CHECK: sw $4, 24($5)
-0xe4 0xe9 0x23 0xc6 # CHECK: swc1 $f9, 9158($7)
 0xa8 0xa4 0x00 0x10 # CHECK: swl $4, 16($5)
+0xac 0xa4 0x00 0x18 # CHECK: sw $4, 24($5)
 0xb8 0xe6 0x00 0x10 # CHECK: swr $6, 16($7)
-0x4e 0xd2 0xd0 0x08 # CHECK: swxc1 $f26, $18($22)
-0x00 0x00 0x01 0xcf # CHECK: sync 7
-0x46 0x20 0x73 0x0d # CHECK: trunc.w.d $f12, $f14
-0x46 0x00 0x39 0x8d # CHECK: trunc.w.s $f6, $f7
-0x7c 0x07 0x30 0xa0 # CHECK: wsbh $6, $7
-0x00 0x65 0x18 0x26 # CHECK: xor $3, $3, $5
-0x38 0xc9 0x45 0x67 # CHECK: xori $9, $6, 17767
+0xc0 0xe9 0x23 0xc6 # CHECK: ll $9, 9158($7)
+0xc4 0xe9 0x23 0xc6 # CHECK: lwc1 $f9, 9158($7)
+0xd4 0xe9 0x23 0xc6 # CHECK: ldc1 $f9, 9158($7)
+0xe0 0xe9 0x23 0xc6 # CHECK: sc $9, 9158($7)
+0xe4 0xe9 0x23 0xc6 # CHECK: swc1 $f9, 9158($7)
+0xf4 0xe9 0x23 0xc6 # CHECK: sdc1 $f9, 9158($7)
diff --git a/test/MC/Disassembler/Mips/mips32r6.txt b/test/MC/Disassembler/Mips/mips32r6.txt
deleted file mode 100644
index afef8ada152..00000000000
--- a/test/MC/Disassembler/Mips/mips32r6.txt
+++ /dev/null
@@ -1,127 +0,0 @@
-# RUN: llvm-mc --disassemble %s -triple=mips-unknown-linux -mcpu=mips32r6 | FileCheck %s
-
-0xec 0x80 0x00 0x19 # CHECK: addiupc $4, 100
-0x7c 0x43 0x22 0xa0 # CHECK: align $4, $2, $3, 2
-0xec 0x7f 0x00 0x38 # CHECK: aluipc $3, 56
-0x3c 0x62 0xff 0xe9 # CHECK: aui $3, $2, -23
-0xec 0x7e 0xff 0xff # CHECK: auipc $3, -1
-0xe8 0x37 0x96 0xb8 # CHECK: balc 14572256
-0xc8 0x37 0x96 0xb8 # CHECK: bc 14572256
-
-# FIXME: Don't check the immediate on these for the moment, the encode/decode
-#        functions are not inverses of eachother.
-#        The immediate should be 4 but the disassembler currently emits 8
-0x45 0x20 0x00 0x01 # CHECK: bc1eqz $f0,
-0x45 0x3f 0x00 0x01 # CHECK: bc1eqz $f31,
-0x45 0xa0 0x00 0x01 # CHECK: bc1nez $f0,
-0x45 0xbf 0x00 0x01 # CHECK: bc1nez $f31,
-# FIXME: Don't check the immediate on these for the moment, the encode/decode
-#        functions are not inverses of eachother.
-#        The immediate should be 8 but the disassembler currently emits 12
-0x49 0x20 0x00 0x02 # CHECK: bc2eqz $0,
-0x49 0x3f 0x00 0x02 # CHECK: bc2eqz $31,
-0x49 0xa0 0x00 0x02 # CHECK: bc2nez $0,
-0x49 0xbf 0x00 0x02 # CHECK: bc2nez $31,
-
-0x20 0xa6 0x00 0x40 # CHECK: beqc $5, $6, 256
-# FIXME: Don't check the immediate on the bcczal's for the moment, the
-#        encode/decode functions are not inverses of eachother.
-0x20 0x02 0x01 0x4d # CHECK: beqzalc $2,
-0x60 0xa6 0x00 0x40 # CHECK: bnec $5, $6, 256
-0x60 0x02 0x01 0x4d # CHECK: bnezalc $2,
-0xd8 0xa0 0x46 0x90 # CHECK: beqzc $5, 72256
-0x58 0x43 0x00 0x40 # CHECK: bgec $2, $3, 256
-0x18 0x43 0x00 0x40 # CHECK: bgeuc $2, $3, 256
-0x18 0x42 0x01 0x4d # CHECK: bgezalc $2,
-0xf8 0xa0 0x46 0x90 # CHECK: bnezc $5, 72256
-0x5c 0xa5 0x00 0x40 # CHECK: bltzc $5, 256
-0x58 0xa5 0x00 0x40 # CHECK: bgezc $5, 256
-0x1c 0x02 0x01 0x4d # CHECK: bgtzalc $2,
-0x58 0x05 0x00 0x40 # CHECK: blezc $5, 256
-0x1c 0x42 0x01 0x4d # CHECK: bltzalc $2,
-0x5c 0x05 0x00 0x40 # CHECK: bgtzc $5, 256
-0x7c 0x02 0x20 0x20 # CHECK: bitswap $4, $2
-0x18 0x02 0x01 0x4d # CHECK: blezalc $2,
-0x5c 0xa6 0x00 0x40 # CHECK: bltc $5, $6, 256
-0x1c 0xa6 0x00 0x40 # CHECK: bltuc $5, $6, 256
-0x60 0x00 0x00 0x01 # CHECK: bnvc $zero, $zero, 4
-0x60 0x40 0x00 0x01 # CHECK: bnvc $2, $zero, 4
-0x60 0x82 0x00 0x01 # CHECK: bnvc $4, $2, 4
-0x20 0x00 0x00 0x01 # CHECK: bovc $zero, $zero, 4
-0x20 0x40 0x00 0x01 # CHECK: bovc $2, $zero, 4
-0x20 0x82 0x00 0x01 # CHECK: bovc $4, $2, 4
-0x46 0x84 0x18 0x80 # CHECK: cmp.af.s $f2, $f3, $f4
-0x46 0xa4 0x18 0x80 # CHECK: cmp.af.d $f2, $f3, $f4
-0x46 0x84 0x18 0x81 # CHECK: cmp.un.s $f2, $f3, $f4
-0x46 0xa4 0x18 0x81 # CHECK: cmp.un.d $f2, $f3, $f4
-0x46 0x84 0x18 0x82 # CHECK: cmp.eq.s $f2, $f3, $f4
-0x46 0xa4 0x18 0x82 # CHECK: cmp.eq.d $f2, $f3, $f4
-0x46 0x84 0x18 0x83 # CHECK: cmp.ueq.s $f2, $f3, $f4
-0x46 0xa4 0x18 0x83 # CHECK: cmp.ueq.d $f2, $f3, $f4
-0x46 0x84 0x18 0x84 # CHECK: cmp.lt.s $f2, $f3, $f4
-0x46 0xa4 0x18 0x84 # CHECK: cmp.lt.d $f2, $f3, $f4
-0x46 0x84 0x18 0x85 # CHECK: cmp.ult.s $f2, $f3, $f4
-0x46 0xa4 0x18 0x85 # CHECK: cmp.ult.d $f2, $f3, $f4
-0x46 0x84 0x18 0x86 # CHECK: cmp.le.s $f2, $f3, $f4
-0x46 0xa4 0x18 0x86 # CHECK: cmp.le.d $f2, $f3, $f4
-0x46 0x84 0x18 0x87 # CHECK: cmp.ule.s $f2, $f3, $f4
-0x46 0xa4 0x18 0x87 # CHECK: cmp.ule.d $f2, $f3, $f4
-0x46 0x84 0x18 0x88 # CHECK: cmp.saf.s $f2, $f3, $f4
-0x46 0xa4 0x18 0x88 # CHECK: cmp.saf.d $f2, $f3, $f4
-0x46 0x84 0x18 0x89 # CHECK: cmp.sun.s $f2, $f3, $f4
-0x46 0xa4 0x18 0x89 # CHECK: cmp.sun.d $f2, $f3, $f4
-0x46 0x84 0x18 0x8a # CHECK: cmp.seq.s $f2, $f3, $f4
-0x46 0xa4 0x18 0x8a # CHECK: cmp.seq.d $f2, $f3, $f4
-0x46 0x84 0x18 0x8b # CHECK: cmp.sueq.s $f2, $f3, $f4
-0x46 0xa4 0x18 0x8b # CHECK: cmp.sueq.d $f2, $f3, $f4
-0x46 0x84 0x18 0x8c # CHECK: cmp.slt.s $f2, $f3, $f4
-0x46 0xa4 0x18 0x8c # CHECK: cmp.slt.d $f2, $f3, $f4
-0x46 0x84 0x18 0x8d # CHECK: cmp.sult.s $f2, $f3, $f4
-0x46 0xa4 0x18 0x8d # CHECK: cmp.sult.d $f2, $f3, $f4
-0x46 0x84 0x18 0x8e # CHECK: cmp.sle.s $f2, $f3, $f4
-0x46 0xa4 0x18 0x8e # CHECK: cmp.sle.d $f2, $f3, $f4
-0x46 0x84 0x18 0x8f # CHECK: cmp.sule.s $f2, $f3, $f4
-0x46 0xa4 0x18 0x8f # CHECK: cmp.sule.d $f2, $f3, $f4
-0x00 0x64 0x10 0x9a # CHECK: div $2, $3, $4
-0x00 0x64 0x10 0x9b # CHECK: divu $2, $3, $4
-# 0xf8 0x05 0x01 0x00 # CHECK-TODO: jialc $5, 256
-# 0xd8 0x05 0x01 0x00 # CHECK-TODO: jic $5, 256
-0xec 0x48 0x00 0x43 # CHECK: lwpc $2, 268
-0xec 0x50 0x00 0x43 # CHECK: lwupc $2, 268
-0x00 0x64 0x10 0xda # CHECK: mod $2, $3, $4
-0x00 0x64 0x10 0xdb # CHECK: modu $2, $3, $4
-0x00 0x64 0x10 0x98 # CHECK: mul $2, $3, $4
-0x00 0x64 0x10 0xd8 # CHECK: muh $2, $3, $4
-0x00 0x64 0x10 0x99 # CHECK: mulu $2, $3, $4
-0x00 0x64 0x10 0xd9 # CHECK: muhu $2, $3, $4
-0x46 0x04 0x18 0x98 # CHECK: maddf.s $f2, $f3, $f4
-0x46 0x24 0x18 0x98 # CHECK: maddf.d $f2, $f3, $f4
-0x46 0x04 0x18 0x99 # CHECK: msubf.s $f2, $f3, $f4
-0x46 0x24 0x18 0x99 # CHECK: msubf.d $f2, $f3, $f4
-0x46 0x22 0x08 0x10 # CHECK: sel.d $f0, $f1, $f2
-0x46 0x02 0x08 0x10 # CHECK: sel.s $f0, $f1, $f2
-0x00 0x64 0x10 0x35 # CHECK: seleqz $2, $3, $4
-0x00 0x64 0x10 0x37 # CHECK: selnez $2, $3, $4
-0x46 0x04 0x10 0x1d # CHECK: max.s $f0, $f2, $f4
-0x46 0x24 0x10 0x1d # CHECK: max.d $f0, $f2, $f4
-0x46 0x04 0x10 0x1c # CHECK: min.s $f0, $f2, $f4
-0x46 0x24 0x10 0x1c # CHECK: min.d $f0, $f2, $f4
-0x46 0x04 0x10 0x1f # CHECK: maxa.s $f0, $f2, $f4
-0x46 0x24 0x10 0x1f # CHECK: maxa.d $f0, $f2, $f4
-0x46 0x04 0x10 0x1e # CHECK: mina.s $f0, $f2, $f4
-0x46 0x24 0x10 0x1e # CHECK: mina.d $f0, $f2, $f4
-0x46 0x04 0x10 0x14 # CHECK: seleqz.s $f0, $f2, $f4
-0x46 0x24 0x10 0x14 # CHECK: seleqz.d $f0, $f2, $f4
-0x46 0x04 0x10 0x17 # CHECK: selnez.s $f0, $f2, $f4
-0x46 0x24 0x10 0x17 # CHECK: selnez.d $f0, $f2, $f4
-0x46 0x00 0x20 0x9a # CHECK: rint.s $f2, $f4
-0x46 0x20 0x20 0x9a # CHECK: rint.d $f2, $f4
-0x46 0x00 0x20 0x9b # CHECK: class.s $f2, $f4
-0x46 0x20 0x20 0x9b # CHECK: class.d $f2, $f4
-0x00 0x80 0x04 0x09 # CHECK: jr.hb $4
-0x00 0x80 0xfc 0x09 # CHECK: jalr.hb $4
-0x00 0xa0 0x24 0x09 # CHECK: jalr.hb $4, $5
-0x7e 0x42 0xb3 0xb6 # CHECK: ll $2, -153($18)
-0x7e 0x6f 0xec 0x26 # CHECK: sc $15, -40($19)
-0x00 0xa0 0x58 0x51 # CHECK: clo $11, $5
-0x03 0x80 0xe8 0x50 # CHECK: clz $sp, $gp
diff --git a/test/MC/Disassembler/Mips/mips32r6/valid-mips32r6-el.txt b/test/MC/Disassembler/Mips/mips32r6/valid-mips32r6-el.txt
index c10d16699b7..94dc3a2645d 100644
--- a/test/MC/Disassembler/Mips/mips32r6/valid-mips32r6-el.txt
+++ b/test/MC/Disassembler/Mips/mips32r6/valid-mips32r6-el.txt
@@ -83,6 +83,7 @@
 0xc5 0x10 0x64 0x00 # CHECK: lsa $2, $3, $4, 3
 0x43 0x00 0x48 0xec # CHECK: lwpc $2, 268
 0x43 0x00 0x50 0xec # CHECK: lwupc $2, 268
+0x01 0x78 0x08 0x40 # CHECK: mfc0 $8, $15, 1
 0xda 0x10 0x64 0x00 # CHECK: mod $2, $3, $4
 0xdb 0x10 0x64 0x00 # CHECK: modu $2, $3, $4
 0x98 0x10 0x64 0x00 # CHECK: mul $2, $3, $4
@@ -93,6 +94,7 @@
 0x98 0x18 0x24 0x46 # CHECK: maddf.d $f2, $f3, $f4
 0x99 0x18 0x04 0x46 # CHECK: msubf.s $f2, $f3, $f4
 0x99 0x18 0x24 0x46 # CHECK: msubf.d $f2, $f3, $f4
+0x01 0x78 0x89 0x40 # CHECK: mtc0 $9, $15, 1
 0x10 0x08 0x22 0x46 # CHECK: sel.d $f0, $f1, $f2
 0x10 0x08 0x02 0x46 # CHECK: sel.s $f0, $f1, $f2
 0x35 0x10 0x64 0x00 # CHECK: seleqz $2, $3, $4
diff --git a/test/MC/Disassembler/Mips/mips32r6/valid-mips32r6.txt b/test/MC/Disassembler/Mips/mips32r6/valid-mips32r6.txt
index 0b78003420b..2791b7af58f 100644
--- a/test/MC/Disassembler/Mips/mips32r6/valid-mips32r6.txt
+++ b/test/MC/Disassembler/Mips/mips32r6/valid-mips32r6.txt
@@ -1,148 +1,174 @@
 # RUN: llvm-mc --disassemble %s -triple=mips-unknown-linux -mcpu=mips32r6 | FileCheck %s
-0x30 0x42 0x00 0x04 # CHECK: andi $2, $2, 4
-0xec 0x80 0x00 0x19 # CHECK: addiupc $4, 100
-0x25 0x29 0x00 0x0a # CHECK: addiu $9, $9, 10
-0x7c 0x43 0x22 0xa0 # CHECK: align $4, $2, $3, 2
-0xec 0x7f 0x00 0x38 # CHECK: aluipc $3, 56
-0x3c 0x62 0xff 0xe9 # CHECK: aui $3, $2, -23
-0xec 0x7e 0xff 0xff # CHECK: auipc $3, -1
+0x00 0x00 0x00 0x0e # CHECK: sdbbp
+0x00 0x00 0x00 0x0f # CHECK: sync
+0x00 0x00 0x00 0x40 # CHECK: ssnop
+0x00 0x00 0x00 0x4f # CHECK: sync 1
+0x00 0x00 0x08 0x8e # CHECK: sdbbp 34
+0x00 0x03 0x00 0x34 # CHECK: teq $zero, $3
+0x00 0x53 0x21 0x72 # CHECK: tlt $2, $19, 133
+0x00 0x64 0x10 0x35 # CHECK: seleqz $2, $3, $4
+0x00 0x64 0x10 0x37 # CHECK: selnez $2, $3, $4
+0x00 0x64 0x10 0x98 # CHECK: mul $2, $3, $4
+0x00 0x64 0x10 0x99 # CHECK: mulu $2, $3, $4
+0x00 0x64 0x10 0x9a # CHECK: div $2, $3, $4
+0x00 0x64 0x10 0x9b # CHECK: divu $2, $3, $4
+0x00 0x64 0x10 0xc5 # CHECK: lsa $2, $3, $4, 3
+0x00 0x64 0x10 0xd8 # CHECK: muh $2, $3, $4
+0x00 0x64 0x10 0xd9 # CHECK: muhu $2, $3, $4
+0x00 0x64 0x10 0xda # CHECK: mod $2, $3, $4
+0x00 0x64 0x10 0xdb # CHECK: modu $2, $3, $4
+0x00 0x80 0x04 0x09 # CHECK: jr.hb $4
+0x00 0x80 0xfc 0x09 # CHECK: jalr.hb $4
+0x00 0xa0 0x24 0x09 # CHECK: jalr.hb $4, $5
+0x00 0xa0 0x58 0x51 # CHECK: clo $11, $5
+0x00 0xa7 0x9b 0x34 # CHECK: teq $5, $7, 620
+0x00 0xb3 0x55 0x30 # CHECK: tge $5, $19, 340
+0x00 0xd1 0x00 0x36 # CHECK: tne $6, $17
+0x00 0xe8 0xdd 0x76 # CHECK: tne $7, $8, 885
+0x00 0xea 0x00 0x30 # CHECK: tge $7, $10
+0x01 0x70 0x00 0x33 # CHECK: tltu $11, $16
+0x01 0xed 0x00 0x32 # CHECK: tlt $15, $13
+0x02 0x1d 0xfe 0x33 # CHECK: tltu $16, $sp, 1016
+0x02 0x8e 0x5e 0xf1 # CHECK: tgeu $20, $14, 379
+0x02 0xdc 0x00 0x31 # CHECK: tgeu $22, $gp
+0x03 0x80 0xe8 0x50 # CHECK: clz $sp, $gp
 0x04 0x11 0x14 0x9b # CHECK: bal 21104
-0xe8 0x37 0x96 0xb8 # CHECK: balc 14572256
-0xc8 0x37 0x96 0xb8 # CHECK: bc 14572256
-0x45 0x20 0x00 0x01 # CHECK: bc1eqz $f0, 8
-0x45 0x3f 0x00 0x01 # CHECK: bc1eqz $f31, 8
-0x45 0xa0 0x00 0x01 # CHECK: bc1nez $f0, 8
-0x45 0xbf 0x00 0x01 # CHECK: bc1nez $f31, 8
-0x49 0x20 0x00 0x02 # CHECK: bc2eqz $0, 12
-0x49 0x3f 0x00 0x02 # CHECK: bc2eqz $31, 12
-0x49 0xa0 0x00 0x02 # CHECK: bc2nez $0, 12
-0x49 0xbf 0x00 0x02 # CHECK: bc2nez $31, 12
-0x20 0xa6 0x00 0x40 # CHECK: beqc $5, $6, 256
-0x20 0x02 0x01 0x4d # CHECK: beqzalc $2, 1332
-0x60 0xa6 0x00 0x40 # CHECK: bnec $5, $6, 256
-0x60 0x02 0x01 0x4d # CHECK: bnezalc $2, 1332
-0xd8 0xa0 0x46 0x90 # CHECK: beqzc $5, 72256
-0x58 0x43 0x00 0x40 # CHECK: bgec $2, $3, 256
-0x18 0x43 0x00 0x40 # CHECK: bgeuc $2, $3, 256
+# FIXME: The encode/decode functions are not inverses of each other.
+0x18 0x02 0x01 0x4d # CHECK: blezalc $2, 1332
+# FIXME: The encode/decode functions are not inverses of each other.
 0x18 0x42 0x01 0x4d # CHECK: bgezalc $2, 1332
-0xf8 0xa0 0x46 0x90 # CHECK: bnezc $5, 72256
-0x5c 0xa5 0x00 0x40 # CHECK: bltzc $5, 256
-0x58 0xa5 0x00 0x40 # CHECK: bgezc $5, 256
+0x18 0x43 0x00 0x40 # CHECK: bgeuc $2, $3, 256
+# FIXME: The encode/decode functions are not inverses of each other.
 0x1c 0x02 0x01 0x4d # CHECK: bgtzalc $2, 1332
-0x58 0x05 0x00 0x40 # CHECK: blezc $5, 256
+# FIXME: The encode/decode functions are not inverses of each other.
 0x1c 0x42 0x01 0x4d # CHECK: bltzalc $2, 1332
-0x5c 0x05 0x00 0x40 # CHECK: bgtzc $5, 256
-0x7c 0x02 0x20 0x20 # CHECK: bitswap $4, $2
-0x18 0x02 0x01 0x4d # CHECK: blezalc $2, 1332
-0x5c 0xa6 0x00 0x40 # CHECK: bltc $5, $6, 256
 0x1c 0xa6 0x00 0x40 # CHECK: bltuc $5, $6, 256
-0x60 0x00 0x00 0x01 # CHECK: bnvc $zero, $zero, 4
-0x60 0x40 0x00 0x01 # CHECK: bnvc $2, $zero, 4
-0x60 0x82 0x00 0x01 # CHECK: bnvc $4, $2, 4
 0x20 0x00 0x00 0x01 # CHECK: bovc $zero, $zero, 4
+# FIXME: The encode/decode functions are not inverses of each other.
+0x20 0x02 0x01 0x4d # CHECK: beqzalc $2, 1332
 0x20 0x40 0x00 0x01 # CHECK: bovc $2, $zero, 4
 0x20 0x82 0x00 0x01 # CHECK: bovc $4, $2, 4
+0x20 0xa6 0x00 0x40 # CHECK: beqc $5, $6, 256
+0x25 0x29 0x00 0x0a # CHECK: addiu $9, $9, 10
+0x30 0x42 0x00 0x04 # CHECK: andi $2, $2, 4
+0x34 0x42 0x00 0x04 # CHECK: ori $2, $2, 4
+0x3c 0x62 0xff 0xe9 # CHECK: aui $3, $2, -23
+0x40 0x08 0x78 0x01 # CHECK: mfc0 $8, $15, 1
+0x40 0x08 0x80 0x03 # CHECK: mfc0 $8, $16, 3
+0x40 0x89 0x78 0x01 # CHECK: mtc0 $9, $15, 1
+0x41 0x60 0x60 0x00 # CHECK: di
+0x41 0x60 0x60 0x20 # CHECK: ei
+0x41 0x6e 0x60 0x20 # CHECK: ei $14
+0x41 0x7e 0x60 0x00 # CHECK: di $fp
+# FIXME: The encode/decode functions are not inverses of each other.
+#        The immediate should be 4 but the disassembler currently emits 8
+0x45 0x20 0x00 0x01 # CHECK: bc1eqz $f0, 8
+# FIXME: The encode/decode functions are not inverses of each other.
+#        The immediate should be 4 but the disassembler currently emits 8
+0x45 0x3f 0x00 0x01 # CHECK: bc1eqz $f31, 8
+# FIXME: The encode/decode functions are not inverses of each other.
+#        The immediate should be 4 but the disassembler currently emits 8
+0x45 0xa0 0x00 0x01 # CHECK: bc1nez $f0, 8
+# FIXME: The encode/decode functions are not inverses of each other.
+#        The immediate should be 4 but the disassembler currently emits 8
+0x45 0xbf 0x00 0x01 # CHECK: bc1nez $f31, 8
+0x46 0x00 0x20 0x9a # CHECK: rint.s $f2, $f4
+0x46 0x00 0x20 0x9b # CHECK: class.s $f2, $f4
+0x46 0x02 0x08 0x10 # CHECK: sel.s $f0, $f1, $f2
+0x46 0x04 0x10 0x14 # CHECK: seleqz.s $f0, $f2, $f4
+0x46 0x04 0x10 0x17 # CHECK: selnez.s $f0, $f2, $f4
+0x46 0x04 0x10 0x1c # CHECK: min.s $f0, $f2, $f4
+0x46 0x04 0x10 0x1d # CHECK: max.s $f0, $f2, $f4
+0x46 0x04 0x10 0x1e # CHECK: mina.s $f0, $f2, $f4
+0x46 0x04 0x10 0x1f # CHECK: maxa.s $f0, $f2, $f4
+0x46 0x04 0x18 0x98 # CHECK: maddf.s $f2, $f3, $f4
+0x46 0x04 0x18 0x99 # CHECK: msubf.s $f2, $f3, $f4
+0x46 0x20 0x20 0x9a # CHECK: rint.d $f2, $f4
+0x46 0x20 0x20 0x9b # CHECK: class.d $f2, $f4
+0x46 0x22 0x08 0x10 # CHECK: sel.d $f0, $f1, $f2
+0x46 0x24 0x10 0x14 # CHECK: seleqz.d $f0, $f2, $f4
+0x46 0x24 0x10 0x17 # CHECK: selnez.d $f0, $f2, $f4
+0x46 0x24 0x10 0x1c # CHECK: min.d $f0, $f2, $f4
+0x46 0x24 0x10 0x1d # CHECK: max.d $f0, $f2, $f4
+0x46 0x24 0x10 0x1e # CHECK: mina.d $f0, $f2, $f4
+0x46 0x24 0x10 0x1f # CHECK: maxa.d $f0, $f2, $f4
+0x46 0x24 0x18 0x98 # CHECK: maddf.d $f2, $f3, $f4
+0x46 0x24 0x18 0x99 # CHECK: msubf.d $f2, $f3, $f4
 0x46 0x84 0x18 0x80 # CHECK: cmp.af.s $f2, $f3, $f4
-0x46 0xa4 0x18 0x80 # CHECK: cmp.af.d $f2, $f3, $f4
 0x46 0x84 0x18 0x81 # CHECK: cmp.un.s $f2, $f3, $f4
-0x46 0xa4 0x18 0x81 # CHECK: cmp.un.d $f2, $f3, $f4
 0x46 0x84 0x18 0x82 # CHECK: cmp.eq.s $f2, $f3, $f4
-0x46 0xa4 0x18 0x82 # CHECK: cmp.eq.d $f2, $f3, $f4
 0x46 0x84 0x18 0x83 # CHECK: cmp.ueq.s $f2, $f3, $f4
-0x46 0xa4 0x18 0x83 # CHECK: cmp.ueq.d $f2, $f3, $f4
 0x46 0x84 0x18 0x84 # CHECK: cmp.lt.s $f2, $f3, $f4
-0x46 0xa4 0x18 0x84 # CHECK: cmp.lt.d $f2, $f3, $f4
 0x46 0x84 0x18 0x85 # CHECK: cmp.ult.s $f2, $f3, $f4
-0x46 0xa4 0x18 0x85 # CHECK: cmp.ult.d $f2, $f3, $f4
 0x46 0x84 0x18 0x86 # CHECK: cmp.le.s $f2, $f3, $f4
-0x46 0xa4 0x18 0x86 # CHECK: cmp.le.d $f2, $f3, $f4
 0x46 0x84 0x18 0x87 # CHECK: cmp.ule.s $f2, $f3, $f4
-0x46 0xa4 0x18 0x87 # CHECK: cmp.ule.d $f2, $f3, $f4
 0x46 0x84 0x18 0x88 # CHECK: cmp.saf.s $f2, $f3, $f4
-0x46 0xa4 0x18 0x88 # CHECK: cmp.saf.d $f2, $f3, $f4
 0x46 0x84 0x18 0x89 # CHECK: cmp.sun.s $f2, $f3, $f4
-0x46 0xa4 0x18 0x89 # CHECK: cmp.sun.d $f2, $f3, $f4
 0x46 0x84 0x18 0x8a # CHECK: cmp.seq.s $f2, $f3, $f4
-0x46 0xa4 0x18 0x8a # CHECK: cmp.seq.d $f2, $f3, $f4
 0x46 0x84 0x18 0x8b # CHECK: cmp.sueq.s $f2, $f3, $f4
-0x46 0xa4 0x18 0x8b # CHECK: cmp.sueq.d $f2, $f3, $f4
 0x46 0x84 0x18 0x8c # CHECK: cmp.slt.s $f2, $f3, $f4
-0x46 0xa4 0x18 0x8c # CHECK: cmp.slt.d $f2, $f3, $f4
 0x46 0x84 0x18 0x8d # CHECK: cmp.sult.s $f2, $f3, $f4
-0x46 0xa4 0x18 0x8d # CHECK: cmp.sult.d $f2, $f3, $f4
 0x46 0x84 0x18 0x8e # CHECK: cmp.sle.s $f2, $f3, $f4
-0x46 0xa4 0x18 0x8e # CHECK: cmp.sle.d $f2, $f3, $f4
 0x46 0x84 0x18 0x8f # CHECK: cmp.sule.s $f2, $f3, $f4
+0x46 0xa4 0x18 0x80 # CHECK: cmp.af.d $f2, $f3, $f4
+0x46 0xa4 0x18 0x81 # CHECK: cmp.un.d $f2, $f3, $f4
+0x46 0xa4 0x18 0x82 # CHECK: cmp.eq.d $f2, $f3, $f4
+0x46 0xa4 0x18 0x83 # CHECK: cmp.ueq.d $f2, $f3, $f4
+0x46 0xa4 0x18 0x84 # CHECK: cmp.lt.d $f2, $f3, $f4
+0x46 0xa4 0x18 0x85 # CHECK: cmp.ult.d $f2, $f3, $f4
+0x46 0xa4 0x18 0x86 # CHECK: cmp.le.d $f2, $f3, $f4
+0x46 0xa4 0x18 0x87 # CHECK: cmp.ule.d $f2, $f3, $f4
+0x46 0xa4 0x18 0x88 # CHECK: cmp.saf.d $f2, $f3, $f4
+0x46 0xa4 0x18 0x89 # CHECK: cmp.sun.d $f2, $f3, $f4
+0x46 0xa4 0x18 0x8a # CHECK: cmp.seq.d $f2, $f3, $f4
+0x46 0xa4 0x18 0x8b # CHECK: cmp.sueq.d $f2, $f3, $f4
+0x46 0xa4 0x18 0x8c # CHECK: cmp.slt.d $f2, $f3, $f4
+0x46 0xa4 0x18 0x8d # CHECK: cmp.sult.d $f2, $f3, $f4
+0x46 0xa4 0x18 0x8e # CHECK: cmp.sle.d $f2, $f3, $f4
 0x46 0xa4 0x18 0x8f # CHECK: cmp.sule.d $f2, $f3, $f4
-0x41 0x7e 0x60 0x00 # CHECK: di $fp
-0x41 0x60 0x60 0x00 # CHECK: di
-0x00 0x64 0x10 0x9a # CHECK: div $2, $3, $4
-0x00 0x64 0x10 0x9b # CHECK: divu $2, $3, $4
-0x41 0x6e 0x60 0x20 # CHECK: ei $14
-0x41 0x60 0x60 0x20 # CHECK: ei
-0x00 0x64 0x10 0xc5 # CHECK: lsa $2, $3, $4, 3
-0xec 0x48 0x00 0x43 # CHECK: lwpc $2, 268
-0xec 0x50 0x00 0x43 # CHECK: lwupc $2, 268
-0x00 0x64 0x10 0xda # CHECK: mod $2, $3, $4
-0x00 0x64 0x10 0xdb # CHECK: modu $2, $3, $4
-0x00 0x64 0x10 0x98 # CHECK: mul $2, $3, $4
-0x00 0x64 0x10 0xd8 # CHECK: muh $2, $3, $4
-0x00 0x64 0x10 0x99 # CHECK: mulu $2, $3, $4
-0x00 0x64 0x10 0xd9 # CHECK: muhu $2, $3, $4
-0x46 0x04 0x18 0x98 # CHECK: maddf.s $f2, $f3, $f4
-0x46 0x24 0x18 0x98 # CHECK: maddf.d $f2, $f3, $f4
-0x46 0x04 0x18 0x99 # CHECK: msubf.s $f2, $f3, $f4
-0x46 0x24 0x18 0x99 # CHECK: msubf.d $f2, $f3, $f4
-0x46 0x22 0x08 0x10 # CHECK: sel.d $f0, $f1, $f2
-0x46 0x02 0x08 0x10 # CHECK: sel.s $f0, $f1, $f2
-0x00 0x64 0x10 0x35 # CHECK: seleqz $2, $3, $4
-0x00 0x64 0x10 0x37 # CHECK: selnez $2, $3, $4
-0x46 0x04 0x10 0x1d # CHECK: max.s $f0, $f2, $f4
-0x46 0x24 0x10 0x1d # CHECK: max.d $f0, $f2, $f4
-0x46 0x04 0x10 0x1c # CHECK: min.s $f0, $f2, $f4
-0x46 0x24 0x10 0x1c # CHECK: min.d $f0, $f2, $f4
-0x46 0x04 0x10 0x1f # CHECK: maxa.s $f0, $f2, $f4
-0x46 0x24 0x10 0x1f # CHECK: maxa.d $f0, $f2, $f4
-0x46 0x04 0x10 0x1e # CHECK: mina.s $f0, $f2, $f4
-0x46 0x24 0x10 0x1e # CHECK: mina.d $f0, $f2, $f4
-0x34 0x42 0x00 0x04 # CHECK: ori $2, $2, 4
-0x46 0x04 0x10 0x14 # CHECK: seleqz.s $f0, $f2, $f4
-0x46 0x24 0x10 0x14 # CHECK: seleqz.d $f0, $f2, $f4
-0x46 0x04 0x10 0x17 # CHECK: selnez.s $f0, $f2, $f4
-0x46 0x24 0x10 0x17 # CHECK: selnez.d $f0, $f2, $f4
-0x46 0x00 0x20 0x9a # CHECK: rint.s $f2, $f4
-0x46 0x20 0x20 0x9a # CHECK: rint.d $f2, $f4
-0x46 0x00 0x20 0x9b # CHECK: class.s $f2, $f4
-0x46 0x20 0x20 0x9b # CHECK: class.d $f2, $f4
-0x00 0x80 0x04 0x09 # CHECK: jr.hb $4
-0x00 0x80 0xfc 0x09 # CHECK: jalr.hb $4
-0x00 0xa0 0x24 0x09 # CHECK: jalr.hb $4, $5
-0x7e 0x42 0xb3 0xb6 # CHECK: ll $2, -153($18)
-0x7e 0x6f 0xec 0x26 # CHECK: sc $15, -40($19)
-0x00 0xa0 0x58 0x51 # CHECK: clo $11, $5
-0x03 0x80 0xe8 0x50 # CHECK: clz $sp, $gp
-0x00 0x00 0x00 0x40 # CHECK: ssnop
-0x00 0x00 0x00 0x0e # CHECK: sdbbp
-0x00 0x00 0x08 0x8e # CHECK: sdbbp 34
-0x00 0x00 0x00 0x0f # CHECK: sync
-0x00 0x00 0x00 0x4f # CHECK: sync 1
-0x00 0x03 0x00 0x34 # CHECK: teq $zero, $3
-0x00 0xa7 0x9b 0x34 # CHECK: teq $5, $7, 620
-0x00 0xea 0x00 0x30 # CHECK: tge $7, $10
-0x00 0xb3 0x55 0x30 # CHECK: tge $5, $19, 340
-0x02 0xdc 0x00 0x31 # CHECK: tgeu $22, $gp
-0x02 0x8e 0x5e 0xf1 # CHECK: tgeu $20, $14, 379
-0x01 0xed 0x00 0x32 # CHECK: tlt $15, $13
-0x00 0x53 0x21 0x72 # CHECK: tlt $2, $19, 133
-0x01 0x70 0x00 0x33 # CHECK: tltu $11, $16
-0x02 0x1d 0xfe 0x33 # CHECK: tltu $16, $sp, 1016
-0x00 0xd1 0x00 0x36 # CHECK: tne $6, $17
-0x00 0xe8 0xdd 0x76 # CHECK: tne $7, $8, 885
-0x49 0xc8 0x0d 0x43 # CHECK: ldc2 $8, -701($1)
+0x42 0x00 0x00 0x58 # CHECK: eretnc
+# FIXME: The encode/decode functions are not inverses of each other.
+#        The immediate should be 8 but the disassembler currently emits 12
+0x49 0x20 0x00 0x02 # CHECK: bc2eqz $0, 12
+# FIXME: The encode/decode functions are not inverses of each other.
+#        The immediate should be 8 but the disassembler currently emits 12
+0x49 0x3f 0x00 0x02 # CHECK: bc2eqz $31, 12
 0x49 0x52 0x34 0xb7 # CHECK: lwc2 $18, -841($6)
-0x49 0xf4 0x92 0x75 # CHECK: sdc2 $20, 629($18)
 0x49 0x79 0x81 0x30 # CHECK: swc2 $25, 304($16)
-0xf8 0x05 0x01 0x00 # CHECK: jialc $5, 256
-0xd8 0x05 0x01 0x00 # CHECK: jic $5, 256
+# FIXME: The encode/decode functions are not inverses of each other.
+#        The immediate should be 8 but the disassembler currently emits 12
+0x49 0xa0 0x00 0x02 # CHECK: bc2nez $0, 12
+# FIXME: The encode/decode functions are not inverses of each other.
+#        The immediate should be 8 but the disassembler currently emits 12
+0x49 0xbf 0x00 0x02 # CHECK: bc2nez $31, 12
+0x49 0xc8 0x0d 0x43 # CHECK: ldc2 $8, -701($1)
+0x49 0xf4 0x92 0x75 # CHECK: sdc2 $20, 629($18)
+0x58 0x05 0x00 0x40 # CHECK: blezc $5, 256
+0x58 0x43 0x00 0x40 # CHECK: bgec $2, $3, 256
+0x58 0xa5 0x00 0x40 # CHECK: bgezc $5, 256
+0x5c 0x05 0x00 0x40 # CHECK: bgtzc $5, 256
+0x5c 0xa5 0x00 0x40 # CHECK: bltzc $5, 256
+0x5c 0xa6 0x00 0x40 # CHECK: bltc $5, $6, 256
+0x60 0x00 0x00 0x01 # CHECK: bnvc $zero, $zero, 4
+# FIXME: The encode/decode functions are not inverses of each other.
+0x60 0x02 0x01 0x4d # CHECK: bnezalc $2, 1332
+0x60 0x40 0x00 0x01 # CHECK: bnvc $2, $zero, 4
+0x60 0x82 0x00 0x01 # CHECK: bnvc $4, $2, 4
+0x60 0xa6 0x00 0x40 # CHECK: bnec $5, $6, 256
+0x7c 0x02 0x20 0x20 # CHECK: bitswap $4, $2
+0x7c 0x43 0x22 0xa0 # CHECK: align $4, $2, $3, 2
 0x7c 0xa1 0x04 0x25 # CHECK: cache 1, 8($5)
 0x7c 0xa1 0x04 0x35 # CHECK: pref 1, 8($5)
+0x7e 0x42 0xb3 0xb6 # CHECK: ll $2, -153($18)
+0x7e 0x6f 0xec 0x26 # CHECK: sc $15, -40($19)
+0xc8 0x37 0x96 0xb8 # CHECK: bc 14572256
+0xd8 0x05 0x01 0x00 # CHECK: jic $5, 256
+0xd8 0xa0 0x46 0x90 # CHECK: beqzc $5, 72256
+0xe8 0x37 0x96 0xb8 # CHECK: balc 14572256
+0xec 0x48 0x00 0x43 # CHECK: lwpc $2, 268
+0xec 0x50 0x00 0x43 # CHECK: lwupc $2, 268
+0xec 0x7e 0xff 0xff # CHECK: auipc $3, -1
+0xec 0x7f 0x00 0x38 # CHECK: aluipc $3, 56
+0xec 0x80 0x00 0x19 # CHECK: addiupc $4, 100
+0xf8 0x05 0x01 0x00 # CHECK: jialc $5, 256
+0xf8 0xa0 0x46 0x90 # CHECK: bnezc $5, 72256
diff --git a/test/MC/Disassembler/Mips/mips4/valid-mips4.txt b/test/MC/Disassembler/Mips/mips4/valid-mips4.txt
index c8c35e9334e..207f4087791 100644
--- a/test/MC/Disassembler/Mips/mips4/valid-mips4.txt
+++ b/test/MC/Disassembler/Mips/mips4/valid-mips4.txt
@@ -1,229 +1,229 @@
 # RUN: llvm-mc %s -triple=mips64-unknown-linux -disassemble -mcpu=mips4 | FileCheck %s
 # CHECK: .text
-0x46 0x20 0xc1 0x85 # CHECK: abs.d $f6, $f24
-0x46 0x00 0x82 0x45 # CHECK: abs.s $f9, $f16
+0x00 0x00 0x00 0x00 # CHECK: nop
+0x00 0x00 0x00 0x40 # CHECK: ssnop
+0x00 0x00 0x00 0xc0 # CHECK: ehb
+0x00 0x00 0x04 0xb8 # CHECK: dsll $zero, $zero, 18
+0x00 0x00 0x04 0xbc # CHECK: dsll32 $zero, $zero, 18
+0x00 0x00 0x04 0xbc # CHECK: dsll32 $zero, $zero, 18
+0x00 0x00 0x3c 0x80 # CHECK: sll $7, $zero, 18
+0x00 0x00 0x88 0x12 # CHECK: mflo $17
+0x00 0x00 0x98 0x10 # CHECK: mfhi $19
+0x00 0x00 0xe8 0x10 # CHECK: mfhi $sp
+0x00 0x02 0x10 0x23 # CHECK: negu $2, $2
+0x00 0x02 0x11 0xc2 # CHECK: srl $2, $2, 7
+0x00 0x02 0x11 0xc2 # CHECK: srl $2, $2, 7
+0x00 0x03 0x00 0x34 # CHECK: teq $zero, $3
+0x00 0x03 0x10 0x23 # CHECK: negu $2, $3
+0x00 0x03 0x10 0x2e # CHECK: dneg $2, $3
+0x00 0x03 0x10 0x2f # CHECK: dnegu $2, $3
+0x00 0x06 0x9d 0xfa # CHECK: dsrl $19, $6, 23
+0x00 0x06 0x9d 0xfe # CHECK: dsrl32 $19, $6, 23
+0x00 0x07 0x38 0x27 # CHECK: nor $7, $zero, $7
+0x00 0x07 0x3c 0x80 # CHECK: sll $7, $7, 18
+0x00 0x11 0x8b 0xc3 # CHECK: sra $17, $17, 15
+0x00 0x12 0xe2 0xbb # CHECK: dsra $gp, $18, 10
+0x00 0x12 0xe2 0xbf # CHECK: dsra32 $gp, $18, 10
+0x00 0x13 0x9d 0xfa # CHECK: dsrl $19, $19, 23
+0x00 0x13 0x9d 0xfe # CHECK: dsrl32 $19, $19, 23
+0x00 0x14 0x04 0xb8 # CHECK: dsll $zero, $20, 18
+0x00 0x17 0x8b 0xc3 # CHECK: sra $17, $23, 15
+0x00 0x1c 0xe2 0xbb # CHECK: dsra $gp, $gp, 10
+0x00 0x1c 0xe2 0xbf # CHECK: dsra32 $gp, $gp, 10
+0x00 0x3f 0x98 0x2c # CHECK: dadd $19, $1, $ra
+0x00 0x3f 0x98 0x2d # CHECK: daddu $19, $1, $ra
+0x00 0x4c 0xb8 0x24 # CHECK: and $23, $2, $12
+0x00 0x53 0x21 0x72 # CHECK: tlt $2, $19, 133
+0x00 0x80 0xf0 0x21 # CHECK: move $fp, $4
+0x00 0x86 0x48 0x21 # CHECK: addu $9, $4, $6
+0x00 0x94 0xc8 0x06 # CHECK: srlv $25, $20, $4
+0x00 0x94 0xc8 0x06 # CHECK: srlv $25, $20, $4
+0x00 0x9e 0x90 0x26 # CHECK: xor $18, $4, $fp
+0x00 0xa6 0x00 0x1d # CHECK: dmultu $5, $6
+0x00 0xa7 0x9b 0x34 # CHECK: teq $5, $7, 620
+0x00 0xb3 0x55 0x30 # CHECK: tge $5, $19, 340
+0x00 0xba 0x28 0x2f # CHECK: dsubu $5, $5, $26
+0x00 0xc0 0xc8 0x21 # CHECK: move $25, $6
+0x00 0xd1 0x00 0x36 # CHECK: tne $6, $17
+0x00 0xe8 0xdd 0x76 # CHECK: tne $7, $8, 885
+0x00 0xea 0x00 0x30 # CHECK: tge $7, $10
+0x01 0x1c 0xe0 0x01 # CHECK: movf $gp, $8, $fcc7
+0x01 0x20 0x38 0x04 # CHECK: sllv $7, $zero, $9
+0x01 0x20 0x38 0x04 # CHECK: sllv $7, $zero, $9
+0x01 0x32 0x00 0x19 # CHECK: multu $9, $18
+0x01 0x70 0x00 0x33 # CHECK: tltu $11, $16
+0x01 0x7b 0xb8 0x2a # CHECK: slt $23, $11, $27
+0x01 0x94 0x00 0x14 # CHECK: dsllv $zero, $20, $12
+0x01 0x94 0x00 0x14 # CHECK: dsllv $zero, $20, $12
+0x01 0xed 0x00 0x32 # CHECK: tlt $15, $13
+0x02 0x11 0x00 0x1f # CHECK: ddivu $zero, $16, $17
+0x02 0x1d 0x60 0x25 # CHECK: or $12, $16, $sp
+0x02 0x1d 0xfe 0x33 # CHECK: tltu $16, $sp, 1016
+0x02 0x20 0x00 0x11 # CHECK: mthi $17
+0x02 0x30 0x18 0x0b # CHECK: movn $3, $17, $16
 0x02 0x45 0xb8 0x20 # CHECK: add $23, $18, $5
+0x02 0x6c 0xb0 0x22 # CHECK: sub $22, $19, $12
+0x02 0x72 0xe0 0x17 # CHECK: dsrav $gp, $18, $19
+0x02 0x72 0xe0 0x17 # CHECK: dsrav $gp, $18, $19
+0x02 0x86 0x98 0x16 # CHECK: dsrlv $19, $6, $20
+0x02 0x86 0x98 0x16 # CHECK: dsrlv $19, $6, $20
+0x02 0x8e 0x5e 0xf1 # CHECK: tgeu $20, $14, 379
+0x02 0x95 0x00 0x01 # CHECK: movt $zero, $20, $fcc5
+0x02 0xab 0xa0 0x2b # CHECK: sltu $20, $21, $11
+0x02 0xc8 0x38 0x2e # CHECK: dsub $7, $22, $8
+0x02 0xc9 0x28 0x0a # CHECK: movz $5, $22, $9
+0x02 0xd6 0xe8 0x23 # CHECK: subu $sp, $22, $22
+0x02 0xdc 0x00 0x31 # CHECK: tgeu $22, $gp
+0x02 0xe9 0x00 0x1c # CHECK: dmult $23, $9
+0x03 0x20 0x00 0x13 # CHECK: mtlo $25
+0x03 0x2b 0x00 0x1a # CHECK: div $zero, $25, $11
+0x03 0x2f 0x00 0x1b # CHECK: divu $zero, $25, $15
+0x03 0x53 0x00 0x1e # CHECK: ddiv $zero, $26, $19
+0x03 0x9a 0x00 0x19 # CHECK: multu $gp, $26
+0x03 0xa0 0x00 0x13 # CHECK: mtlo $sp
+0x03 0xa2 0x00 0x18 # CHECK: mult $sp, $2
+0x03 0xb4 0x00 0x18 # CHECK: mult $sp, $20
+0x03 0xb7 0x88 0x07 # CHECK: srav $17, $23, $sp
+0x03 0xb7 0x88 0x07 # CHECK: srav $17, $23, $sp
+0x04 0x11 0x14 0x9b # CHECK: bal 21104
+0x04 0x83 0xf9 0x4d # CHECK: bgezl $4, -6856
+0x04 0xd0 0x14 0x9b # CHECK: bltzal $6, 21104
+0x04 0xd2 0x00 0x7a # CHECK: bltzall $6, 492
+0x05 0x8e 0x8c 0x31 # CHECK: tnei $12, 35889
+0x05 0x93 0x07 0x1f # CHECK: bgezall $12, 7296
+0x05 0xca 0xad 0xbd # CHECK: tlti $14, 44477
+0x06 0x22 0xf6 0x45 # CHECK: bltzl $17, -9960
+0x06 0x28 0x13 0xa1 # CHECK: tgei $17, 5025
+0x06 0xac 0xbb 0xa0 # CHECK: teqi $21, 48032
+0x07 0xa9 0x90 0x33 # CHECK: tgeiu $sp, 36915
+0x07 0xeb 0xec 0x2c # CHECK: tltiu $ra, 60460
+0x21 0x08 0xff 0xfe # CHECK: addi $8, $8, -2
+0x21 0x2d 0x66 0xd2 # CHECK: addi $13, $9, 26322
+0x21 0xad 0xe6 0x90 # CHECK: addi $13, $13, -6512
 0x21 0xc9 0x3b 0x48 # CHECK: addi $9, $14, 15176
+0x22 0x36 0x0c 0x36 # CHECK: addi $22, $17, 3126
 0x23 0x18 0xe3 0xe7 # CHECK: addi $24, $24, -7193
-0x46 0x3c 0x30 0x00 # CHECK: add.d $f0, $f6, $f28
-0x46 0x18 0xaa 0x00 # CHECK: add.s $f8, $f21, $f24
-0x21 0x2d 0x66 0xd2 # CHECK: addi $13, $9, 26322
-0x21 0x08 0xff 0xfe # CHECK: addi $8, $8, -2
-0x00 0x86 0x48 0x21 # CHECK: addu $9, $4, $6
+0x24 0x00 0x8b 0x3f # CHECK: addiu $zero, $zero, -29889
+0x24 0x01 0x8b 0xb3 # CHECK: addiu $1, $zero, -29773
 0x25 0x29 0x00 0x0a # CHECK: addiu $9, $9, 10
-0x00 0x4c 0xb8 0x24 # CHECK: and $23, $2, $12
+0x29 0x51 0x25 0x11 # CHECK: slti $17, $10, 9489
+0x2f 0x38 0xc3 0x55 # CHECK: sltiu $24, $25, -15531
+0x2f 0x39 0xc3 0x55 # CHECK: sltiu $25, $25, -15531
 0x30 0x42 0x00 0x04 # CHECK: andi $2, $2, 4
+0x34 0x42 0x00 0x04 # CHECK: ori $2, $2, 4
+0x42 0x00 0x00 0x01 # CHECK: tlbr
+0x42 0x00 0x00 0x02 # CHECK: tlbwi
+0x42 0x00 0x00 0x06 # CHECK: tlbwr
+0x42 0x00 0x00 0x08 # CHECK: tlbp
+0x42 0x00 0x00 0x18 # CHECK: eret
+0x44 0x07 0xd8 0x00 # CHECK: mfc1 $7, $f27
+0x44 0x2c 0x68 0x00 # CHECK: dmfc1 $12, $f13
+0x44 0x51 0xa8 0x00 # CHECK: cfc1 $17, $21
+0x44 0x9e 0x48 0x00 # CHECK: mtc1 $fp, $f9
+0x44 0xb0 0x70 0x00 # CHECK: dmtc1 $16, $f14
+0x44 0xc6 0xd0 0x00 # CHECK: ctc1 $6, $26
 0x45 0x00 0x00 0x01 # CHECK: bc1f 8
-0x45 0x04 0x00 0x00 # CHECK: bc1f $fcc1, 4
-0x45 0x1e 0x00 0x06 # CHECK: bc1fl $fcc7, 28
-0x45 0x02 0x00 0x0c # CHECK: bc1fl 52
 0x45 0x01 0x00 0x01 # CHECK: bc1t 8
-0x45 0x05 0x00 0x00 # CHECK: bc1t $fcc1, 4
+0x45 0x02 0x00 0x0c # CHECK: bc1fl 52
 0x45 0x03 0xf7 0xf4 # CHECK: bc1tl -8236
+0x45 0x04 0x00 0x00 # CHECK: bc1f $fcc1, 4
+0x45 0x05 0x00 0x00 # CHECK: bc1t $fcc1, 4
+0x45 0x1e 0x00 0x06 # CHECK: bc1fl $fcc7, 28
 0x45 0x1f 0x00 0x06 # CHECK: bc1tl $fcc7, 28
-0x04 0x11 0x14 0x9b # CHECK: bal 21104
-0x00 0x00 0x00 0x00 # CHECK: nop
-0x04 0xd0 0x14 0x9b # CHECK: bltzal $6, 21104
-0x00 0x00 0x00 0x00 # CHECK: nop
-0x51 0xd3 0x0c 0x40 # CHECK: beql $14, $19, 12548
-0x05 0x93 0x07 0x1f # CHECK: bgezall $12, 7296
-0x04 0x83 0xf9 0x4d # CHECK: bgezl $4, -6856
-0x5d 0x40 0xfc 0x59 # CHECK: bgtzl $10, -3736
-0x58 0xc0 0x02 0xe7 # CHECK: blezl $6, 2976
-0x04 0xd2 0x00 0x7a # CHECK: bltzall $6, 492
-0x06 0x22 0xf6 0x45 # CHECK: bltzl $17, -9960
-0x57 0x94 0x04 0xfc # CHECK: bnel $gp, $20, 5108
-0xbc 0xa1 0x00 0x08 # CHECK: cache 1, 8($5)
-0x46 0x3c 0xe0 0x3b # CHECK: c.ngl.d $f28, $f28
-0x46 0x30 0x00 0x39 # CHECK: c.ngle.d $f0, $f16
-0x46 0x20 0xf0 0x38 # CHECK: c.sf.d $f30, $f0
-0x46 0x16 0x70 0x38 # CHECK: c.sf.s $f14, $f22
-0x46 0x20 0x18 0x4a # CHECK: ceil.l.d $f1, $f3
+0x46 0x00 0x08 0x04 # CHECK: sqrt.s $f0, $f1
+0x46 0x00 0x2b 0x0b # CHECK: floor.l.s $f12, $f5
+0x46 0x00 0x2e 0x48 # CHECK: round.l.s $f25, $f5
+0x46 0x00 0x4a 0x0f # CHECK: floor.w.s $f8, $f9
 0x46 0x00 0x6c 0x8a # CHECK: ceil.l.s $f18, $f13
-0x46 0x20 0xc2 0xce # CHECK: ceil.w.d $f11, $f24
+0x46 0x00 0x78 0x47 # CHECK: neg.s $f1, $f15
+0x46 0x00 0x82 0x45 # CHECK: abs.s $f9, $f16
 0x46 0x00 0xa1 0x8e # CHECK: ceil.w.s $f6, $f20
-0x44 0x51 0xa8 0x00 # CHECK: cfc1 $17, $21
-0x44 0xc6 0xd0 0x00 # CHECK: ctc1 $6, $26
+0x46 0x00 0xc5 0x24 # CHECK: cvt.w.s $f20, $f24
+0x46 0x00 0xd8 0x86 # CHECK: mov.s $f2, $f27
 0x46 0x00 0xe5 0xa1 # CHECK: cvt.d.s $f22, $f28
-0x46 0x80 0x5e 0xa1 # CHECK: cvt.d.w $f26, $f11
-0x46 0xa0 0x81 0x21 # CHECK: cvt.d.l $f4, $f16
-0x46 0x20 0x7e 0x25 # CHECK: cvt.l.d $f24, $f15
+0x46 0x00 0xe6 0xcc # CHECK: round.w.s $f27, $f28
 0x46 0x00 0xea 0xe5 # CHECK: cvt.l.s $f11, $f29
-0x46 0xa0 0xf3 0xe0 # CHECK: cvt.s.l $f15, $f30
+0x46 0x00 0xf7 0x0d # CHECK: trunc.w.s $f28, $f30
+0x46 0x00 0xff 0x09 # CHECK: trunc.l.s $f28, $f31
+0x46 0x02 0x57 0x82 # CHECK: mul.s $f30, $f10, $f2
+0x46 0x03 0x3e 0x52 # CHECK: movz.s $f25, $f7, $3
+0x46 0x05 0x17 0x91 # CHECK: movt.s $f30, $f2, $fcc1
+0x46 0x0f 0x29 0x03 # CHECK: div.s $f4, $f5, $f15
+0x46 0x16 0x70 0x38 # CHECK: c.sf.s $f14, $f22
+0x46 0x16 0xb5 0xc1 # CHECK: sub.s $f23, $f22, $f22
+0x46 0x17 0x03 0x13 # CHECK: movn.s $f12, $f0, $23
+0x46 0x18 0x2d 0xd1 # CHECK: movf.s $f23, $f5, $fcc6
+0x46 0x18 0xaa 0x00 # CHECK: add.s $f8, $f21, $f24
+0x46 0x20 0x0b 0x08 # CHECK: round.l.d $f12, $f1
+0x46 0x20 0x18 0x4a # CHECK: ceil.l.d $f1, $f3
+0x46 0x20 0x21 0x8c # CHECK: round.w.d $f6, $f4
+0x46 0x20 0x3e 0x8b # CHECK: floor.l.d $f26, $f7
 0x46 0x20 0x46 0xa0 # CHECK: cvt.s.d $f26, $f8
-0x46 0x80 0x7d 0xa0 # CHECK: cvt.s.w $f22, $f15
+0x46 0x20 0x53 0x8f # CHECK: floor.w.d $f14, $f10
+0x46 0x20 0x75 0x06 # CHECK: mov.d $f20, $f14
 0x46 0x20 0x75 0x24 # CHECK: cvt.w.d $f20, $f14
-0x46 0x00 0xc5 0x24 # CHECK: cvt.w.s $f20, $f24
-0x00 0x3f 0x98 0x2c # CHECK: dadd $19, $1, $ra
+0x46 0x20 0x75 0x8d # CHECK: trunc.w.d $f22, $f14
+0x46 0x20 0x7e 0x25 # CHECK: cvt.l.d $f24, $f15
+0x46 0x20 0x96 0x87 # CHECK: neg.d $f26, $f18
+0x46 0x20 0xb4 0x04 # CHECK: sqrt.d $f16, $f22
+0x46 0x20 0xbd 0xc9 # CHECK: trunc.l.d $f23, $f23
+0x46 0x20 0xc1 0x85 # CHECK: abs.d $f6, $f24
+0x46 0x20 0xc2 0xce # CHECK: ceil.w.d $f11, $f24
+0x46 0x20 0xf0 0x38 # CHECK: c.sf.d $f30, $f0
+0x46 0x21 0x10 0x11 # CHECK: movt.d $f0, $f2, $fcc0
+0x46 0x29 0xeb 0x12 # CHECK: movz.d $f12, $f29, $9
+0x46 0x30 0x00 0x39 # CHECK: c.ngle.d $f0, $f16
+0x46 0x30 0x14 0x81 # CHECK: sub.d $f18, $f2, $f16
+0x46 0x30 0xa5 0x02 # CHECK: mul.d $f20, $f20, $f16
+0x46 0x34 0x59 0x91 # CHECK: movf.d $f6, $f11, $fcc5
+0x46 0x3a 0xa7 0x03 # CHECK: div.d $f28, $f20, $f26
+0x46 0x3a 0xae 0xd3 # CHECK: movn.d $f27, $f21, $26
+0x46 0x3c 0x30 0x00 # CHECK: add.d $f0, $f6, $f28
+0x46 0x3c 0xe0 0x3b # CHECK: c.ngl.d $f28, $f28
+0x46 0x80 0x5e 0xa1 # CHECK: cvt.d.w $f26, $f11
+0x46 0x80 0x7d 0xa0 # CHECK: cvt.s.w $f22, $f15
+0x46 0xa0 0x81 0x21 # CHECK: cvt.d.l $f4, $f16
+0x46 0xa0 0xf3 0xe0 # CHECK: cvt.s.l $f15, $f30
+0x4d 0xca 0x58 0x09 # CHECK: sdxc1 $f11, $10($14)
+0x4f 0x4c 0x98 0x08 # CHECK: swxc1 $f19, $12($26)
+0x4f 0xd1 0x03 0x00 # CHECK: lwxc1 $f12, $17($fp)
+0x51 0xd3 0x0c 0x40 # CHECK: beql $14, $19, 12548
+0x57 0x94 0x04 0xfc # CHECK: bnel $gp, $20, 5108
+0x58 0xc0 0x02 0xe7 # CHECK: blezl $6, 2976
+0x5d 0x40 0xfc 0x59 # CHECK: bgtzl $10, -3736
 0x62 0x9d 0x93 0xc7 # CHECK: daddi $sp, $20, -27705
-0x63 0xbd 0x93 0xc7 # CHECK: daddi $sp, $sp, -27705
 0x62 0x9d 0x93 0xc7 # CHECK: daddi $sp, $20, -27705
 0x62 0x9d 0x93 0xc7 # CHECK: daddi $sp, $20, -27705
 0x63 0xbd 0x93 0xc7 # CHECK: daddi $sp, $sp, -27705
-0x66 0xda 0xee 0x16 # CHECK: daddiu $26, $22, -4586
-0x00 0x3f 0x98 0x2d # CHECK: daddu $19, $1, $ra
+0x63 0xbd 0x93 0xc7 # CHECK: daddi $sp, $sp, -27705
 0x64 0x58 0x46 0x9f # CHECK: daddiu $24, $2, 18079
 0x66 0x73 0x69 0x3f # CHECK: daddiu $19, $19, 26943
-0x03 0x53 0x00 0x1e # CHECK: ddiv $zero, $26, $19
-0x02 0x11 0x00 0x1f # CHECK: ddivu $zero, $16, $17
-0x03 0x2b 0x00 0x1a # CHECK: div $zero, $25, $11
-0x46 0x3a 0xa7 0x03 # CHECK: div.d $f28, $f20, $f26
-0x46 0x0f 0x29 0x03 # CHECK: div.s $f4, $f5, $f15
-0x03 0x2f 0x00 0x1b # CHECK: divu $zero, $25, $15
-0x44 0x2c 0x68 0x00 # CHECK: dmfc1 $12, $f13
-0x44 0xb0 0x70 0x00 # CHECK: dmtc1 $16, $f14
-0x02 0xe9 0x00 0x1c # CHECK: dmult $23, $9
-0x00 0xa6 0x00 0x1d # CHECK: dmultu $5, $6
-0x00 0x00 0x04 0xb8 # CHECK: dsll $zero, $zero, 18
-0x00 0x14 0x04 0xb8 # CHECK: dsll $zero, $20, 18
-0x01 0x94 0x00 0x14 # CHECK: dsllv $zero, $20, $12
-0x00 0x00 0x04 0xbc # CHECK: dsll32 $zero, $zero, 18
-0x00 0x00 0x04 0xbc # CHECK: dsll32 $zero, $zero, 18
-0x01 0x94 0x00 0x14 # CHECK: dsllv $zero, $20, $12
-0x00 0x1c 0xe2 0xbb # CHECK: dsra $gp, $gp, 10
-0x00 0x12 0xe2 0xbb # CHECK: dsra $gp, $18, 10
-0x02 0x72 0xe0 0x17 # CHECK: dsrav $gp, $18, $19
-0x00 0x1c 0xe2 0xbf # CHECK: dsra32 $gp, $gp, 10
-0x00 0x12 0xe2 0xbf # CHECK: dsra32 $gp, $18, 10
-0x02 0x72 0xe0 0x17 # CHECK: dsrav $gp, $18, $19
-0x00 0x13 0x9d 0xfa # CHECK: dsrl $19, $19, 23
-0x00 0x06 0x9d 0xfa # CHECK: dsrl $19, $6, 23
-0x02 0x86 0x98 0x16 # CHECK: dsrlv $19, $6, $20
-0x00 0x13 0x9d 0xfe # CHECK: dsrl32 $19, $19, 23
-0x00 0x06 0x9d 0xfe # CHECK: dsrl32 $19, $6, 23
-0x02 0x86 0x98 0x16 # CHECK: dsrlv $19, $6, $20
-0x02 0xc8 0x38 0x2e # CHECK: dsub $7, $22, $8
-0x00 0xba 0x28 0x2f # CHECK: dsubu $5, $5, $26
-0x00 0x00 0x00 0xc0 # CHECK: ehb
-0x42 0x00 0x00 0x18 # CHECK: eret
-0x46 0x20 0x53 0x8f # CHECK: floor.w.d $f14, $f10
-0x46 0x00 0x4a 0x0f # CHECK: floor.w.s $f8, $f9
-0x46 0x20 0x3e 0x8b # CHECK: floor.l.d $f26, $f7
-0x46 0x00 0x2b 0x0b # CHECK: floor.l.s $f12, $f5
+0x66 0xda 0xee 0x16 # CHECK: daddiu $26, $22, -4586
 0x81 0x58 0xc7 0x4d # CHECK: lb $24, -14515($10)
-0x90 0x68 0x75 0xf3 # CHECK: lbu $8, 30195($3)
-0xd6 0x0a 0x40 0x07 # CHECK: ldc1 $f10, 16391($16)
-0xd8 0x28 0xad 0x43 # CHECK: ldc2 $8, -21181($1)
 0x86 0xab 0xde 0x94 # CHECK: lh $11, -8556($21)
+0x89 0xf4 0xef 0x79 # CHECK: lwl $20, -4231($15)
+0x8c 0xa8 0x16 0x2a # CHECK: lw $8, 5674($5)
+0x90 0x68 0x75 0xf3 # CHECK: lbu $8, 30195($3)
 0x94 0x53 0xa6 0xbd # CHECK: lhu $19, -22851($2)
-0x24 0x01 0x8b 0xb3 # CHECK: addiu $1, $zero, -29773
-0x24 0x00 0x8b 0x3f # CHECK: addiu $zero, $zero, -29889
+0x9b 0x80 0xb5 0x35 # CHECK: lwr $zero, -19147($gp)
+0xa1 0xd6 0xb2 0x6f # CHECK: sb $22, -19857($14)
+0xa5 0xee 0xe5 0xd0 # CHECK: sh $14, -6704($15)
+0xaa 0x6f 0x35 0x7e # CHECK: swl $15, 13694($19)
+0xaf 0xbf 0xd8 0x50 # CHECK: sw $ra, -10160($sp)
+0xb3 0xc7 0xae 0x1f # CHECK: sdl $7, -20961($fp)
+0xb5 0x8b 0xb0 0x39 # CHECK: sdr $11, -20423($12)
+0xb9 0xd1 0x98 0x22 # CHECK: swr $17, -26590($14)
+0xbc 0xa1 0x00 0x08 # CHECK: cache 1, 8($5)
 0xc2 0x42 0xe3 0x67 # CHECK: ll $2, -7321($18)
-0x8c 0xa8 0x16 0x2a # CHECK: lw $8, 5674($5)
 0xc7 0x50 0x27 0xf1 # CHECK: lwc1 $f16, 10225($26)
 0xc8 0xd2 0xfc 0xb7 # CHECK: lwc2 $18, -841($6)
-0x89 0xf4 0xef 0x79 # CHECK: lwl $20, -4231($15)
-0x9b 0x80 0xb5 0x35 # CHECK: lwr $zero, -19147($gp)
-0x4f 0xd1 0x03 0x00 # CHECK: lwxc1 $f12, $17($fp)
-0x44 0x07 0xd8 0x00 # CHECK: mfc1 $7, $f27
-0x00 0x00 0x98 0x10 # CHECK: mfhi $19
-0x00 0x00 0xe8 0x10 # CHECK: mfhi $sp
-0x00 0x00 0x88 0x12 # CHECK: mflo $17
-0x46 0x20 0x75 0x06 # CHECK: mov.d $f20, $f14
-0x46 0x00 0xd8 0x86 # CHECK: mov.s $f2, $f27
-0x01 0x1c 0xe0 0x01 # CHECK: movf $gp, $8, $fcc7
-0x46 0x34 0x59 0x91 # CHECK: movf.d $f6, $f11, $fcc5
-0x46 0x18 0x2d 0xd1 # CHECK: movf.s $f23, $f5, $fcc6
-0x00 0x80 0xf0 0x21 # CHECK: move $fp, $4
-0x00 0xc0 0xc8 0x21 # CHECK: move $25, $6
-0x02 0x30 0x18 0x0b # CHECK: movn $3, $17, $16
-0x46 0x3a 0xae 0xd3 # CHECK: movn.d $f27, $f21, $26
-0x46 0x17 0x03 0x13 # CHECK: movn.s $f12, $f0, $23
-0x02 0x95 0x00 0x01 # CHECK: movt $zero, $20, $fcc5
-0x46 0x21 0x10 0x11 # CHECK: movt.d $f0, $f2, $fcc0
-0x46 0x05 0x17 0x91 # CHECK: movt.s $f30, $f2, $fcc1
-0x02 0xc9 0x28 0x0a # CHECK: movz $5, $22, $9
-0x46 0x29 0xeb 0x12 # CHECK: movz.d $f12, $f29, $9
-0x46 0x03 0x3e 0x52 # CHECK: movz.s $f25, $f7, $3
-0x44 0x9e 0x48 0x00 # CHECK: mtc1 $fp, $f9
-0x02 0x20 0x00 0x11 # CHECK: mthi $17
-0x03 0xa0 0x00 0x13 # CHECK: mtlo $sp
-0x03 0x20 0x00 0x13 # CHECK: mtlo $25
-0x46 0x30 0xa5 0x02 # CHECK: mul.d $f20, $f20, $f16
-0x46 0x02 0x57 0x82 # CHECK: mul.s $f30, $f10, $f2
-0x03 0xb4 0x00 0x18 # CHECK: mult $sp, $20
-0x03 0xa2 0x00 0x18 # CHECK: mult $sp, $2
-0x03 0x9a 0x00 0x19 # CHECK: multu $gp, $26
-0x01 0x32 0x00 0x19 # CHECK: multu $9, $18
-0x00 0x02 0x10 0x23 # CHECK: negu $2, $2
-0x00 0x03 0x10 0x23 # CHECK: negu $2, $3
-0x46 0x20 0x96 0x87 # CHECK: neg.d $f26, $f18
-0x46 0x00 0x78 0x47 # CHECK: neg.s $f1, $f15
-0x00 0x00 0x00 0x00 # CHECK: nop
-0x00 0x07 0x38 0x27 # CHECK: nor $7, $zero, $7
-0x02 0x1d 0x60 0x25 # CHECK: or $12, $16, $sp
-0x34 0x42 0x00 0x04 # CHECK: ori $2, $2, 4
 0xcc 0xa1 0x00 0x08 # CHECK: pref 1, 8($5) 
-0x46 0x20 0x0b 0x08 # CHECK: round.l.d $f12, $f1
-0x46 0x00 0x2e 0x48 # CHECK: round.l.s $f25, $f5
-0x46 0x20 0x21 0x8c # CHECK: round.w.d $f6, $f4
-0x46 0x00 0xe6 0xcc # CHECK: round.w.s $f27, $f28
-0xa1 0xd6 0xb2 0x6f # CHECK: sb $22, -19857($14)
+0xd6 0x0a 0x40 0x07 # CHECK: ldc1 $f10, 16391($16)
+0xd8 0x28 0xad 0x43 # CHECK: ldc2 $8, -21181($1)
 0xe2 0x6f 0x49 0xd8 # CHECK: sc $15, 18904($19)
+0xe7 0x06 0xde 0xef # CHECK: swc1 $f6, -8465($24)
+0xea 0x19 0x61 0x30 # CHECK: swc2 $25, 24880($16)
 0xf3 0xaf 0xdf 0xcd # CHECK: scd $15, -8243($sp)
-0xfd 0x4c 0x16 0xcb # CHECK: sd $12, 5835($10)
-0xb3 0xc7 0xae 0x1f # CHECK: sdl $7, -20961($fp)
-0xb5 0x8b 0xb0 0x39 # CHECK: sdr $11, -20423($12)
 0xf5 0xbe 0x77 0x6e # CHECK: sdc1 $f30, 30574($13)
 0xfa 0x54 0x5a 0x75 # CHECK: sdc2 $20, 23157($18)
-0x4d 0xca 0x58 0x09 # CHECK: sdxc1 $f11, $10($14)
-0xa5 0xee 0xe5 0xd0 # CHECK: sh $14, -6704($15)
-0x00 0x07 0x3c 0x80 # CHECK: sll $7, $7, 18
-0x00 0x00 0x3c 0x80 # CHECK: sll $7, $zero, 18
-0x01 0x20 0x38 0x04 # CHECK: sllv $7, $zero, $9
-0x01 0x20 0x38 0x04 # CHECK: sllv $7, $zero, $9
-0x01 0x7b 0xb8 0x2a # CHECK: slt $23, $11, $27
-0x29 0x51 0x25 0x11 # CHECK: slti $17, $10, 9489
-0x2f 0x39 0xc3 0x55 # CHECK: sltiu $25, $25, -15531
-0x02 0xab 0xa0 0x2b # CHECK: sltu $20, $21, $11
-0x2f 0x38 0xc3 0x55 # CHECK: sltiu $24, $25, -15531
-0x46 0x20 0xb4 0x04 # CHECK: sqrt.d $f16, $f22
-0x46 0x00 0x08 0x04 # CHECK: sqrt.s $f0, $f1
-0x00 0x11 0x8b 0xc3 # CHECK: sra $17, $17, 15
-0x00 0x17 0x8b 0xc3 # CHECK: sra $17, $23, 15
-0x03 0xb7 0x88 0x07 # CHECK: srav $17, $23, $sp
-0x03 0xb7 0x88 0x07 # CHECK: srav $17, $23, $sp
-0x00 0x02 0x11 0xc2 # CHECK: srl $2, $2, 7
-0x00 0x02 0x11 0xc2 # CHECK: srl $2, $2, 7
-0x00 0x94 0xc8 0x06 # CHECK: srlv $25, $20, $4
-0x00 0x94 0xc8 0x06 # CHECK: srlv $25, $20, $4
-0x00 0x00 0x00 0x40 # CHECK: ssnop
-0x02 0x6c 0xb0 0x22 # CHECK: sub $22, $19, $12
-0x22 0x36 0x0c 0x36 # CHECK: addi $22, $17, 3126
-0x21 0xad 0xe6 0x90 # CHECK: addi $13, $13, -6512
-0x46 0x30 0x14 0x81 # CHECK: sub.d $f18, $f2, $f16
-0x46 0x16 0xb5 0xc1 # CHECK: sub.s $f23, $f22, $f22
-0x02 0xd6 0xe8 0x23 # CHECK: subu $sp, $22, $22
-0xaf 0xbf 0xd8 0x50 # CHECK: sw $ra, -10160($sp)
-0xe7 0x06 0xde 0xef # CHECK: swc1 $f6, -8465($24)
-0xea 0x19 0x61 0x30 # CHECK: swc2 $25, 24880($16)
-0xaa 0x6f 0x35 0x7e # CHECK: swl $15, 13694($19)
-0xb9 0xd1 0x98 0x22 # CHECK: swr $17, -26590($14)
-0x4f 0x4c 0x98 0x08 # CHECK: swxc1 $f19, $12($26)
-0x00 0x03 0x00 0x34 # CHECK: teq $zero, $3
-0x00 0xa7 0x9b 0x34 # CHECK: teq $5, $7, 620
-0x06 0xac 0xbb 0xa0 # CHECK: teqi $21, 48032
-0x00 0xea 0x00 0x30 # CHECK: tge $7, $10
-0x00 0xb3 0x55 0x30 # CHECK: tge $5, $19, 340
-0x06 0x28 0x13 0xa1 # CHECK: tgei $17, 5025
-0x07 0xa9 0x90 0x33 # CHECK: tgeiu $sp, 36915
-0x02 0xdc 0x00 0x31 # CHECK: tgeu $22, $gp
-0x02 0x8e 0x5e 0xf1 # CHECK: tgeu $20, $14, 379
-0x42 0x00 0x00 0x08 # CHECK: tlbp
-0x42 0x00 0x00 0x01 # CHECK: tlbr
-0x42 0x00 0x00 0x02 # CHECK: tlbwi
-0x42 0x00 0x00 0x06 # CHECK: tlbwr
-0x01 0xed 0x00 0x32 # CHECK: tlt $15, $13
-0x00 0x53 0x21 0x72 # CHECK: tlt $2, $19, 133
-0x05 0xca 0xad 0xbd # CHECK: tlti $14, 44477
-0x07 0xeb 0xec 0x2c # CHECK: tltiu $ra, 60460
-0x01 0x70 0x00 0x33 # CHECK: tltu $11, $16
-0x02 0x1d 0xfe 0x33 # CHECK: tltu $16, $sp, 1016
-0x00 0xd1 0x00 0x36 # CHECK: tne $6, $17
-0x00 0xe8 0xdd 0x76 # CHECK: tne $7, $8, 885
-0x05 0x8e 0x8c 0x31 # CHECK: tnei $12, 35889
-0x46 0x20 0xbd 0xc9 # CHECK: trunc.l.d $f23, $f23
-0x46 0x00 0xff 0x09 # CHECK: trunc.l.s $f28, $f31
-0x46 0x20 0x75 0x8d # CHECK: trunc.w.d $f22, $f14
-0x46 0x00 0xf7 0x0d # CHECK: trunc.w.s $f28, $f30
-0x00 0x9e 0x90 0x26 # CHECK: xor $18, $4, $fp
+0xfd 0x4c 0x16 0xcb # CHECK: sd $12, 5835($10)
diff --git a/test/MC/Disassembler/Mips/mips64.txt b/test/MC/Disassembler/Mips/mips64.txt
deleted file mode 100644
index d494df6f9c2..00000000000
--- a/test/MC/Disassembler/Mips/mips64.txt
+++ /dev/null
@@ -1,93 +0,0 @@
-# RUN: llvm-mc --disassemble %s -triple=mips64-unknown-linux | FileCheck %s
-# CHECK: daddiu $11, $26, 31949
-0x67 0x4b 0x7c 0xcd
-
-# CHECK: daddiu $sp, $sp, -32
-0x67 0xbd 0xff 0xe0
-
-# CHECK: daddu $26, $1, $11
-0x00 0x2b 0xd0 0x2d
-
-# CHECK: ddiv $zero, $26, $22
-0x03 0x56 0x00 0x1e
-
-# CHECK: ddivu $zero, $9, $24
-0x01 0x38 0x00 0x1f
-
-# CHECK: dmfc1 $2, $f14
-0x44 0x22 0x70 0x00
-
-# CHECK: dmtc1 $23, $f5
-0x44 0xb7 0x28 0x00
-
-# CHECK: dmult $11, $26
-0x01 0x7a 0x00 0x1c
-
-# CHECK: dmultu $23, $13
-0x02 0xed 0x00 0x1d
-
-# CHECK: dsll $3, $24, 17
-0x00 0x18 0x1c 0x78
-
-# CHECK: dsllv $gp, $27, $24
-0x03 0x1b 0xe0 0x14
-
-# CHECK: dsra $1, $1, 30
-0x00 0x01 0x0f 0xbb
-
-# CHECK: dsrav $1, $1, $fp
-0x03 0xc1 0x08 0x17
-
-# CHECK: dsrl $10, $gp, 24
-0x00 0x1c 0x56 0x3a
-
-# CHECK: dsrlv $gp, $10, $23
-0x02 0xea 0xe0 0x16
-
-# CHECK: dsubu $gp, $27, $24
-0x03 0x78 0xe0 0x2f
-
-# CHECK: lw $27, -15155($1)
-0x8c 0x3b 0xc4 0xcd
-
-# CHECK: lui $1, 1
-0x3c 0x01 0x00 0x01
-
-# CHECK: lwu $3, -1746($3)
-0x9c 0x63 0xf9 0x2e
-
-# CHECK: lui $ra, 1
-0x3c 0x1f 0x00 0x01
-
-# CHECK: sw $26, -15159($1)
-0xac 0x3a 0xc4 0xc9
-
-# CHECK: ld $26, 3958($zero)
-0xdc 0x1a 0x0f 0x76
-
-# CHECK: sd $6, 17767($zero)
-0xfc 0x06 0x45 0x67
-
-# CHECK: luxc1 $f0, $6($5)
-0x4c 0xa6 0x00 0x05
-
-# CHECK: lwxc1 $f20, $12($14)
-0x4d 0xcc 0x05 0x00
-
-# CHECK: suxc1 $f4, $24($5)
-0x4c 0xb8 0x20 0x0d
-
-# CHECK: swxc1 $f26, $18($22)
-0x4e 0xd2 0xd0 0x08
-
-# CHECK: ldxc1 $f2, $2($10)
-0x4d 0x42 0x00 0x81
-
-# CHECK: sdxc1 $f8, $4($25)
-0x4f 0x24 0x40 0x09
-
-# CHECK: sdc2  $9, 9158($7)
-0xf8 0xe9 0x23 0xc6
-
-# CHECK: ldc2  $3, 9162($8)
-0xd9 0x03 0x23 0xca
diff --git a/test/MC/Disassembler/Mips/mips64/valid-mips64-el.txt b/test/MC/Disassembler/Mips/mips64/valid-mips64-el.txt
index 698ebfb3479..2d52216fdda 100644
--- a/test/MC/Disassembler/Mips/mips64/valid-mips64-el.txt
+++ b/test/MC/Disassembler/Mips/mips64/valid-mips64-el.txt
@@ -82,7 +82,9 @@
 0x24 0x80 0x30 0x73 # CHECK: dclz $16, $25
 0x1e 0x00 0x53 0x03 # CHECK: ddiv $zero, $26, $19
 0x1f 0x00 0x11 0x02 # CHECK: ddivu $zero, $16, $17
+0x00 0x50 0x38 0x40 # CHECK: dmfc0 $24, $10, 0
 0x00 0x68 0x2c 0x44 # CHECK: dmfc1 $12, $f13
+0x00 0x50 0xa4 0x40 # CHECK: dmtc0 $4, $10, 0
 0x00 0x70 0xb0 0x44 # CHECK: dmtc1 $16, $f14
 0x1c 0x00 0xe9 0x02 # CHECK: dmult $23, $9
 0x1d 0x00 0xa6 0x00 # CHECK: dmultu $5, $6
@@ -142,6 +144,7 @@
 0x10 0x00 0xa3 0x98 # CHECK: lwr $3, 16($5)
 0x00 0x00 0xc7 0x70 # CHECK: madd $6, $7
 0x01 0x00 0xc7 0x70 # CHECK: maddu $6, $7
+0x01 0x78 0x08 0x40 # CHECK: mfc0 $8, $15, 1
 0x00 0x38 0x06 0x44 # CHECK: mfc1 $6, $f7
 0x10 0x28 0x00 0x00 # CHECK: mfhi $5
 0x12 0x28 0x00 0x00 # CHECK: mflo $5
@@ -149,6 +152,7 @@
 0x86 0x39 0x00 0x46 # CHECK: mov.s $f6, $f7
 0x04 0x00 0xc7 0x70 # CHECK: msub $6, $7
 0x05 0x00 0xc7 0x70 # CHECK: msubu $6, $7
+0x01 0x78 0x89 0x40 # CHECK: mtc0 $9, $15, 1
 0x00 0x38 0x86 0x44 # CHECK: mtc1 $6, $f7
 0x11 0x00 0xe0 0x00 # CHECK: mthi $7
 0x13 0x00 0xe0 0x00 # CHECK: mtlo $7
diff --git a/test/MC/Disassembler/Mips/mips64/valid-mips64.txt b/test/MC/Disassembler/Mips/mips64/valid-mips64.txt
index b0809d8150f..6cbf5d3206b 100644
--- a/test/MC/Disassembler/Mips/mips64/valid-mips64.txt
+++ b/test/MC/Disassembler/Mips/mips64/valid-mips64.txt
@@ -1,216 +1,254 @@
 # RUN: llvm-mc --disassemble %s -triple=mips64-unknown-linux | FileCheck %s
-0x46 0x20 0x73 0x05 # CHECK: abs.d $f12, $f14
-0x46 0x00 0x39 0x85 # CHECK: abs.s $f6, $f7
+0x00 0x00 0x00 0x00 # CHECK: nop
+0x00 0x00 0x01 0xcf # CHECK: sync 7
+0x00 0x00 0x04 0xb8 # CHECK: dsll $zero, $zero, 18
+0x00 0x00 0x04 0xbc # CHECK: dsll32 $zero, $zero, 18
+0x00 0x00 0x04 0xbc # CHECK: dsll32 $zero, $zero, 18
+0x00 0x00 0x28 0x10 # CHECK: mfhi $5
+0x00 0x00 0x28 0x12 # CHECK: mflo $5
+0x00 0x01 0x0f 0xbb # CHECK: dsra $1, $1, 30
+0x00 0x03 0x10 0x2e # CHECK: dneg $2, $3
+0x00 0x03 0x10 0x2f # CHECK: dnegu $2, $3
+0x00 0x03 0x21 0xc0 # CHECK: sll $4, $3, 7
+0x00 0x03 0x21 0xc2 # CHECK: srl $4, $3, 7
+0x00 0x03 0x21 0xc3 # CHECK: sra $4, $3, 7
+0x00 0x06 0x9d 0xfa # CHECK: dsrl $19, $6, 23
+0x00 0x06 0x9d 0xfe # CHECK: dsrl32 $19, $6, 23
+0x00 0x12 0xe2 0xbb # CHECK: dsra $gp, $18, 10
+0x00 0x12 0xe2 0xbf # CHECK: dsra32 $gp, $18, 10
+0x00 0x13 0x9d 0xfa # CHECK: dsrl $19, $19, 23
+0x00 0x13 0x9d 0xfe # CHECK: dsrl32 $19, $19, 23
+0x00 0x14 0x04 0xb8 # CHECK: dsll $zero, $20, 18
+0x00 0x18 0x1c 0x78 # CHECK: dsll $3, $24, 17
+0x00 0x1c 0x56 0x3a # CHECK: dsrl $10, $gp, 24
+0x00 0x1c 0xe2 0xbb # CHECK: dsra $gp, $gp, 10
+0x00 0x1c 0xe2 0xbf # CHECK: dsra32 $gp, $gp, 10
+0x00 0x2b 0xd0 0x2d # CHECK: daddu $26, $1, $11
+0x00 0x3f 0x98 0x2c # CHECK: dadd $19, $1, $ra
+0x00 0x3f 0x98 0x2d # CHECK: daddu $19, $1, $ra
+0x00 0x65 0x00 0x18 # CHECK: mult $3, $5
+0x00 0x65 0x00 0x19 # CHECK: multu $3, $5
+0x00 0x65 0x18 0x25 # CHECK: or $3, $3, $5
+0x00 0x65 0x18 0x26 # CHECK: xor $3, $3, $5
+0x00 0x65 0x18 0x2a # CHECK: slt $3, $3, $5
+0x00 0x65 0x18 0x2b # CHECK: sltu $3, $3, $5
+0x00 0x65 0x20 0x23 # CHECK: subu $4, $3, $5
+0x00 0x80 0xfc 0x09 # CHECK: jalr.hb $4
+0x00 0xa0 0x24 0x09 # CHECK: jalr.hb $4, $5
+0x00 0xa3 0x10 0x04 # CHECK: sllv $2, $3, $5
+0x00 0xa3 0x10 0x06 # CHECK: srlv $2, $3, $5
+0x00 0xa3 0x10 0x07 # CHECK: srav $2, $3, $5
+0x00 0xa6 0x00 0x1d # CHECK: dmultu $5, $6
+0x00 0xba 0x28 0x2f # CHECK: dsubu $5, $5, $26
 0x00 0xc7 0x48 0x20 # CHECK: add $9, $6, $7
-0x46 0x2e 0x62 0x00 # CHECK: add.d $f8, $f12, $f14
-0x46 0x07 0x32 0x40 # CHECK: add.s $f9, $f6, $f7
-0x20 0xc9 0x45 0x67 # CHECK: addi $9, $6, 17767
-0x24 0xc9 0xc5 0x67 # CHECK: addiu $9, $6, -15001
 0x00 0xc7 0x48 0x21 # CHECK: addu $9, $6, $7
+0x00 0xc7 0x48 0x22 # CHECK: sub $9, $6, $7
 0x00 0xc7 0x48 0x24 # CHECK: and $9, $6, $7
-0x30 0xc9 0x45 0x67 # CHECK: andi $9, $6, 17767
+0x00 0xc7 0x48 0x27 # CHECK: nor $9, $6, $7
+0x00 0xe0 0x00 0x08 # CHECK: jr $7
+0x00 0xe0 0x00 0x11 # CHECK: mthi $7
+0x00 0xe0 0x00 0x13 # CHECK: mtlo $7
+0x00 0xe0 0xf8 0x09 # CHECK: jalr $7
+0x01 0x38 0x00 0x1f # CHECK: ddivu $zero, $9, $24
+0x01 0x7a 0x00 0x1c # CHECK: dmult $11, $26
+0x01 0x94 0x00 0x14 # CHECK: dsllv $zero, $20, $12
+0x01 0x94 0x00 0x14 # CHECK: dsllv $zero, $20, $12
+0x02 0x11 0x00 0x1f # CHECK: ddivu $zero, $16, $17
+0x02 0x72 0xe0 0x17 # CHECK: dsrav $gp, $18, $19
+0x02 0x72 0xe0 0x17 # CHECK: dsrav $gp, $18, $19
+0x02 0x86 0x98 0x16 # CHECK: dsrlv $19, $6, $20
+0x02 0x86 0x98 0x16 # CHECK: dsrlv $19, $6, $20
+0x02 0xc8 0x38 0x2e # CHECK: dsub $7, $22, $8
+0x02 0xe9 0x00 0x1c # CHECK: dmult $23, $9
+0x02 0xea 0xe0 0x16 # CHECK: dsrlv $gp, $10, $23
+0x02 0xed 0x00 0x1d # CHECK: dmultu $23, $13
+0x03 0x1b 0xe0 0x14 # CHECK: dsllv $gp, $27, $24
+0x03 0x53 0x00 0x1e # CHECK: ddiv $zero, $26, $19
+0x03 0x56 0x00 0x1e # CHECK: ddiv $zero, $26, $22
+0x03 0x78 0xe0 0x2f # CHECK: dsubu $gp, $27, $24
+0x03 0xc1 0x08 0x17 # CHECK: dsrav $1, $1, $fp
+0x04 0xc1 0x01 0x4c # CHECK: bgez $6, 1332
+0x04 0xd1 0x01 0x4c # CHECK: bgezal $6, 1332
+0x08 0x00 0x01 0x4c # CHECK: j 1328
+0x0c 0x00 0x01 0x4c # CHECK: jal 1328
 0x10 0x00 0x01 0x4c # CHECK: b 1332
+0x11 0x26 0x01 0x4c # CHECK: beq $9, $6, 1332
+0x15 0x26 0x01 0x4c # CHECK: bne $9, $6, 1332
+0x18 0xc0 0x01 0x4c # CHECK: blez $6, 1332
+0x1c 0xc0 0x01 0x4c # CHECK: bgtz $6, 1332
+0x20 0xc9 0x45 0x67 # CHECK: addi $9, $6, 17767
+0x24 0xc9 0xc5 0x67 # CHECK: addiu $9, $6, -15001
+0x28 0x63 0x00 0x67 # CHECK: slti $3, $3, 103
+0x2c 0x63 0x00 0x67 # CHECK: sltiu $3, $3, 103
+0x30 0xc9 0x45 0x67 # CHECK: andi $9, $6, 17767
+0x34 0xc9 0x45 0x67 # CHECK: ori $9, $6, 17767
+0x38 0xc9 0x45 0x67 # CHECK: xori $9, $6, 17767
+0x3c 0x01 0x00 0x01 # CHECK: lui $1, 1
+0x3c 0x06 0x45 0x67 # CHECK: lui $6, 17767
+0x3c 0x1f 0x00 0x01 # CHECK: lui $ra, 1
+0x40 0x08 0x78 0x01 # CHECK: mfc0 $8, $15, 1
+0x40 0x08 0x80 0x04 # CHECK: mfc0 $8, $16, 4
+0x40 0x38 0x50 0x00 # CHECK: dmfc0 $24, $10, 0
+0x40 0x89 0x78 0x01 # CHECK: mtc0 $9, $15, 1
+0x40 0xa4 0x50 0x00 # CHECK: dmtc0 $4, $10, 0
+0x44 0x06 0x38 0x00 # CHECK: mfc1 $6, $f7
+0x44 0x22 0x70 0x00 # CHECK: dmfc1 $2, $f14
+0x44 0x2c 0x68 0x00 # CHECK: dmfc1 $12, $f13
+0x44 0x46 0x38 0x00 # CHECK: cfc1 $6, $7
+0x44 0x86 0x38 0x00 # CHECK: mtc1 $6, $f7
+0x44 0xb0 0x70 0x00 # CHECK: dmtc1 $16, $f14
+0x44 0xb7 0x28 0x00 # CHECK: dmtc1 $23, $f5
+0x44 0xc6 0x38 0x00 # CHECK: ctc1 $6, $7
 0x45 0x00 0x01 0x4c # CHECK: bc1f 1332
-0x45 0x1c 0x01 0x4c # CHECK: bc1f $fcc7, 1332
 0x45 0x01 0x01 0x4c # CHECK: bc1t 1332
+0x45 0x1c 0x01 0x4c # CHECK: bc1f $fcc7, 1332
 0x45 0x1d 0x01 0x4c # CHECK: bc1t $fcc7, 1332
-0x11 0x26 0x01 0x4c # CHECK: beq $9, $6, 1332
-0x04 0xc1 0x01 0x4c # CHECK: bgez $6, 1332
-0x04 0xd1 0x01 0x4c # CHECK: bgezal $6, 1332
-0x1c 0xc0 0x01 0x4c # CHECK: bgtz $6, 1332
-0x18 0xc0 0x01 0x4c # CHECK: blez $6, 1332
-0x15 0x26 0x01 0x4c # CHECK: bne $9, $6, 1332
-0x46 0x2e 0x60 0x32 # CHECK: c.eq.d $f12, $f14
-0x46 0x07 0x30 0x32 # CHECK: c.eq.s $f6, $f7
-0x46 0x2e 0x60 0x30 # CHECK: c.f.d $f12, $f14
+0x46 0x00 0x2b 0x0b # CHECK: floor.l.s $f12, $f5
+0x46 0x00 0x2e 0x48 # CHECK: round.l.s $f25, $f5
+0x46 0x00 0x39 0x84 # CHECK: sqrt.s $f6, $f7
+0x46 0x00 0x39 0x85 # CHECK: abs.s $f6, $f7
+0x46 0x00 0x39 0x86 # CHECK: mov.s $f6, $f7
+0x46 0x00 0x39 0x87 # CHECK: neg.s $f6, $f7
+0x46 0x00 0x39 0x8c # CHECK: round.w.s $f6, $f7
+0x46 0x00 0x39 0x8d # CHECK: trunc.w.s $f6, $f7
+0x46 0x00 0x39 0x8e # CHECK: ceil.w.s $f6, $f7
+0x46 0x00 0x39 0x8f # CHECK: floor.w.s $f6, $f7
+0x46 0x00 0x39 0xa1 # CHECK: cvt.d.s $f6, $f7
+0x46 0x00 0x39 0xa4 # CHECK: cvt.w.s $f6, $f7
+0x46 0x00 0x6c 0x8a # CHECK: ceil.l.s $f18, $f13
+0x46 0x00 0xff 0x09 # CHECK: trunc.l.s $f28, $f31
 0x46 0x07 0x30 0x30 # CHECK: c.f.s $f6, $f7
-0x46 0x2e 0x60 0x3e # CHECK: c.le.d $f12, $f14
-0x46 0x07 0x30 0x3e # CHECK: c.le.s $f6, $f7
-0x46 0x2e 0x60 0x3c # CHECK: c.lt.d $f12, $f14
+0x46 0x07 0x30 0x31 # CHECK: c.un.s $f6, $f7
+0x46 0x07 0x30 0x32 # CHECK: c.eq.s $f6, $f7
+0x46 0x07 0x30 0x34 # CHECK: c.olt.s $f6, $f7
+0x46 0x07 0x30 0x35 # CHECK: c.ult.s $f6, $f7
+0x46 0x07 0x30 0x36 # CHECK: c.ole.s $f6, $f7
+0x46 0x07 0x30 0x37 # CHECK: c.ule.s $f6, $f7
+0x46 0x07 0x30 0x38 # CHECK: c.sf.s $f6, $f7
+0x46 0x07 0x30 0x39 # CHECK: c.ngle.s $f6, $f7
+0x46 0x07 0x30 0x3a # CHECK: c.seq.s $f6, $f7
+0x46 0x07 0x30 0x3b # CHECK: c.ngl.s $f6, $f7
 0x46 0x07 0x30 0x3c # CHECK: c.lt.s $f6, $f7
-0x46 0x2e 0x60 0x3d # CHECK: c.nge.d $f12, $f14
 0x46 0x07 0x30 0x3d # CHECK: c.nge.s $f6, $f7
-0x46 0x2e 0x60 0x3b # CHECK: c.ngl.d $f12, $f14
-0x46 0x07 0x30 0x3b # CHECK: c.ngl.s $f6, $f7
-0x46 0x2e 0x60 0x39 # CHECK: c.ngle.d $f12, $f14
-0x46 0x07 0x30 0x39 # CHECK: c.ngle.s $f6, $f7
-0x46 0x2e 0x60 0x3f # CHECK: c.ngt.d $f12, $f14
+0x46 0x07 0x30 0x3e # CHECK: c.le.s $f6, $f7
 0x46 0x07 0x30 0x3f # CHECK: c.ngt.s $f6, $f7
-0x46 0x2e 0x60 0x36 # CHECK: c.ole.d $f12, $f14
-0x46 0x07 0x30 0x36 # CHECK: c.ole.s $f6, $f7
-0x46 0x2e 0x60 0x34 # CHECK: c.olt.d $f12, $f14
-0x46 0x07 0x30 0x34 # CHECK: c.olt.s $f6, $f7
-0x46 0x2e 0x60 0x3a # CHECK: c.seq.d $f12, $f14
-0x46 0x07 0x30 0x3a # CHECK: c.seq.s $f6, $f7
-0x46 0x2e 0x60 0x38 # CHECK: c.sf.d $f12, $f14
-0x46 0x07 0x30 0x38 # CHECK: c.sf.s $f6, $f7
-0x46 0x2e 0x60 0x33 # CHECK: c.ueq.d $f12, $f14
+0x46 0x07 0x32 0x40 # CHECK: add.s $f9, $f6, $f7
+0x46 0x07 0x32 0x41 # CHECK: sub.s $f9, $f6, $f7
+0x46 0x07 0x32 0x42 # CHECK: mul.s $f9, $f6, $f7
 0x46 0x12 0xe0 0x33 # CHECK: c.ueq.s $f28, $f18
-0x46 0x2e 0x60 0x37 # CHECK: c.ule.d $f12, $f14
-0x46 0x07 0x30 0x37 # CHECK: c.ule.s $f6, $f7
-0x46 0x2e 0x60 0x35 # CHECK: c.ult.d $f12, $f14
-0x46 0x07 0x30 0x35 # CHECK: c.ult.s $f6, $f7
-0x46 0x2e 0x60 0x31 # CHECK: c.un.d $f12, $f14
-0x46 0x07 0x30 0x31 # CHECK: c.un.s $f6, $f7
-0x46 0x20 0x73 0x0e # CHECK: ceil.w.d $f12, $f14
-0x46 0x00 0x39 0x8e # CHECK: ceil.w.s $f6, $f7
+0x46 0x20 0x0b 0x08 # CHECK: round.l.d $f12, $f1
 0x46 0x20 0x18 0x4a # CHECK: ceil.l.d $f1, $f3
-0x46 0x00 0x6c 0x8a # CHECK: ceil.l.s $f18, $f13
-0x44 0x46 0x38 0x00 # CHECK: cfc1 $6, $7
-0x70 0xe6 0x30 0x21 # CHECK: clo $6, $7
-0x70 0xe6 0x30 0x20 # CHECK: clz $6, $7
-0x44 0xc6 0x38 0x00 # CHECK: ctc1 $6, $7
-0x46 0x00 0x39 0xa1 # CHECK: cvt.d.s $f6, $f7
-0x46 0xa0 0x81 0x21 # CHECK: cvt.d.l $f4, $f16
-0x46 0x80 0x73 0x21 # CHECK: cvt.d.w $f12, $f14
+0x46 0x20 0x3e 0x8b # CHECK: floor.l.d $f26, $f7
+0x46 0x20 0x41 0x86 # CHECK: mov.d $f6, $f8
+0x46 0x20 0x73 0x04 # CHECK: sqrt.d $f12, $f14
+0x46 0x20 0x73 0x05 # CHECK: abs.d $f12, $f14
+0x46 0x20 0x73 0x07 # CHECK: neg.d $f12, $f14
+0x46 0x20 0x73 0x0c # CHECK: round.w.d $f12, $f14
+0x46 0x20 0x73 0x0d # CHECK: trunc.w.d $f12, $f14
+0x46 0x20 0x73 0x0e # CHECK: ceil.w.d $f12, $f14
+0x46 0x20 0x73 0x0f # CHECK: floor.w.d $f12, $f14
 0x46 0x20 0x73 0x20 # CHECK: cvt.s.d $f12, $f14
-0x46 0xa0 0xf3 0xe0 # CHECK: cvt.s.l $f15, $f30
-0x46 0x80 0x39 0xa0 # CHECK: cvt.s.w $f6, $f7
 0x46 0x20 0x73 0x24 # CHECK: cvt.w.d $f12, $f14
-0x46 0x00 0x39 0xa4 # CHECK: cvt.w.s $f6, $f7
-0x00 0x3f 0x98 0x2c # CHECK: dadd $19, $1, $ra
+0x46 0x20 0xbd 0xc9 # CHECK: trunc.l.d $f23, $f23
+0x46 0x2e 0x60 0x30 # CHECK: c.f.d $f12, $f14
+0x46 0x2e 0x60 0x31 # CHECK: c.un.d $f12, $f14
+0x46 0x2e 0x60 0x32 # CHECK: c.eq.d $f12, $f14
+0x46 0x2e 0x60 0x33 # CHECK: c.ueq.d $f12, $f14
+0x46 0x2e 0x60 0x34 # CHECK: c.olt.d $f12, $f14
+0x46 0x2e 0x60 0x35 # CHECK: c.ult.d $f12, $f14
+0x46 0x2e 0x60 0x36 # CHECK: c.ole.d $f12, $f14
+0x46 0x2e 0x60 0x37 # CHECK: c.ule.d $f12, $f14
+0x46 0x2e 0x60 0x38 # CHECK: c.sf.d $f12, $f14
+0x46 0x2e 0x60 0x39 # CHECK: c.ngle.d $f12, $f14
+0x46 0x2e 0x60 0x3a # CHECK: c.seq.d $f12, $f14
+0x46 0x2e 0x60 0x3b # CHECK: c.ngl.d $f12, $f14
+0x46 0x2e 0x60 0x3c # CHECK: c.lt.d $f12, $f14
+0x46 0x2e 0x60 0x3d # CHECK: c.nge.d $f12, $f14
+0x46 0x2e 0x60 0x3e # CHECK: c.le.d $f12, $f14
+0x46 0x2e 0x60 0x3f # CHECK: c.ngt.d $f12, $f14
+0x46 0x2e 0x62 0x00 # CHECK: add.d $f8, $f12, $f14
+0x46 0x2e 0x62 0x01 # CHECK: sub.d $f8, $f12, $f14
+0x46 0x2e 0x62 0x02 # CHECK: mul.d $f8, $f12, $f14
+0x46 0x80 0x39 0xa0 # CHECK: cvt.s.w $f6, $f7
+0x46 0x80 0x73 0x21 # CHECK: cvt.d.w $f12, $f14
+0x46 0xa0 0x81 0x21 # CHECK: cvt.d.l $f4, $f16
+0x46 0xa0 0xf3 0xe0 # CHECK: cvt.s.l $f15, $f30
+0x4c 0xa6 0x00 0x05 # CHECK: luxc1 $f0, $6($5)
+0x4c 0xb8 0x20 0x0d # CHECK: suxc1 $f4, $24($5)
+0x4d 0x42 0x00 0x81 # CHECK: ldxc1 $f2, $2($10)
+0x4d 0xbb 0x60 0x0d # CHECK: suxc1 $f12, $27($13)
+0x4d 0xca 0x58 0x09 # CHECK: sdxc1 $f11, $10($14)
+0x4d 0xcc 0x05 0x00 # CHECK: lwxc1 $f20, $12($14)
+0x4d 0xf7 0x02 0x01 # CHECK: ldxc1 $f8, $23($15)
+0x4e 0xb6 0x04 0xc5 # CHECK: luxc1 $f19, $22($21)
+0x4e 0xd2 0xd0 0x08 # CHECK: swxc1 $f26, $18($22)
+0x4f 0x24 0x40 0x09 # CHECK: sdxc1 $f8, $4($25)
+0x4f 0x4c 0x98 0x08 # CHECK: swxc1 $f19, $12($26)
+0x4f 0xd1 0x03 0x00 # CHECK: lwxc1 $f12, $17($fp)
+0x62 0x9d 0x6c 0x39 # CHECK: daddi $sp, $20, 27705
 0x62 0x9d 0x93 0xc7 # CHECK: daddi $sp, $20, -27705
-0x63 0xbd 0x93 0xc7 # CHECK: daddi $sp, $sp, -27705
 0x62 0x9d 0x93 0xc7 # CHECK: daddi $sp, $20, -27705
 0x62 0x9d 0x93 0xc7 # CHECK: daddi $sp, $20, -27705
+0x63 0xbd 0x6c 0x39 # CHECK: daddi $sp, $sp, 27705
+0x63 0xbd 0x93 0xc7 # CHECK: daddi $sp, $sp, -27705
 0x63 0xbd 0x93 0xc7 # CHECK: daddi $sp, $sp, -27705
-0x66 0xda 0xee 0x16 # CHECK: daddiu $26, $22, -4586
-0x00 0x3f 0x98 0x2d # CHECK: daddu $19, $1, $ra
 0x64 0x58 0x46 0x9f # CHECK: daddiu $24, $2, 18079
-0x66 0x73 0x69 0x3f # CHECK: daddiu $19, $19, 26943
-0x70 0xd2 0x90 0x25 # CHECK: dclo $18, $6
-0x73 0x30 0x80 0x24 # CHECK: dclz $16, $25
-0x03 0x53 0x00 0x1e # CHECK: ddiv $zero, $26, $19
-0x02 0x11 0x00 0x1f # CHECK: ddivu $zero, $16, $17
-0x44 0x2c 0x68 0x00 # CHECK: dmfc1 $12, $f13
-0x44 0xb0 0x70 0x00 # CHECK: dmtc1 $16, $f14
-0x02 0xe9 0x00 0x1c # CHECK: dmult $23, $9
-0x00 0xa6 0x00 0x1d # CHECK: dmultu $5, $6
-0x00 0x00 0x04 0xb8 # CHECK: dsll $zero, $zero, 18
-0x00 0x14 0x04 0xb8 # CHECK: dsll $zero, $20, 18
-0x01 0x94 0x00 0x14 # CHECK: dsllv $zero, $20, $12
-0x00 0x00 0x04 0xbc # CHECK: dsll32 $zero, $zero, 18
-0x00 0x00 0x04 0xbc # CHECK: dsll32 $zero, $zero, 18
-0x01 0x94 0x00 0x14 # CHECK: dsllv $zero, $20, $12
-0x00 0x1c 0xe2 0xbb # CHECK: dsra $gp, $gp, 10
-0x00 0x12 0xe2 0xbb # CHECK: dsra $gp, $18, 10
-0x02 0x72 0xe0 0x17 # CHECK: dsrav $gp, $18, $19
-0x00 0x1c 0xe2 0xbf # CHECK: dsra32 $gp, $gp, 10
-0x00 0x12 0xe2 0xbf # CHECK: dsra32 $gp, $18, 10
-0x02 0x72 0xe0 0x17 # CHECK: dsrav $gp, $18, $19
-0x00 0x13 0x9d 0xfa # CHECK: dsrl $19, $19, 23
-0x00 0x06 0x9d 0xfa # CHECK: dsrl $19, $6, 23
-0x02 0x86 0x98 0x16 # CHECK: dsrlv $19, $6, $20
-0x00 0x13 0x9d 0xfe # CHECK: dsrl32 $19, $19, 23
-0x00 0x06 0x9d 0xfe # CHECK: dsrl32 $19, $6, 23
-0x02 0x86 0x98 0x16 # CHECK: dsrlv $19, $6, $20
-0x02 0xc8 0x38 0x2e # CHECK: dsub $7, $22, $8
-0x62 0x9d 0x6c 0x39 # CHECK: daddi $sp, $20, 27705
-0x63 0xbd 0x6c 0x39 # CHECK: daddi $sp, $sp, 27705
-0x00 0xba 0x28 0x2f # CHECK: dsubu $5, $5, $26
 0x65 0x6f 0xec 0x5f # CHECK: daddiu $15, $11, -5025
 0x65 0xce 0x11 0xea # CHECK: daddiu $14, $14, 4586
-0x46 0x20 0x3e 0x8b # CHECK: floor.l.d $f26, $f7
-0x46 0x00 0x2b 0x0b # CHECK: floor.l.s $f12, $f5
-0x46 0x20 0x73 0x0f # CHECK: floor.w.d $f12, $f14
-0x46 0x00 0x39 0x8f # CHECK: floor.w.s $f6, $f7
-0x08 0x00 0x01 0x4c # CHECK: j 1328
-0x0c 0x00 0x01 0x4c # CHECK: jal 1328
-0x74 0x00 0x01 0x4c # CHECK: jalx 1328
-0x00 0xe0 0xf8 0x09 # CHECK: jalr $7
-0x00 0x80 0xfc 0x09 # CHECK: jalr.hb $4
-0x00 0xa0 0x24 0x09 # CHECK: jalr.hb $4, $5
-0x00 0xe0 0x00 0x08 # CHECK: jr $7
-0x80 0xa4 0x23 0xc6 # CHECK: lb $4, 9158($5)
-0x90 0xa4 0x00 0x06 # CHECK: lbu $4, 6($5)
-0xd4 0xe9 0x23 0xc6 # CHECK: ldc1 $f9, 9158($7)
-0xde 0x3d 0x90 0x1b # CHECK: ld $sp, -28645($17)
+0x66 0x73 0x69 0x3f # CHECK: daddiu $19, $19, 26943
+0x66 0xda 0xee 0x16 # CHECK: daddiu $26, $22, -4586
+0x67 0x4b 0x7c 0xcd # CHECK: daddiu $11, $26, 31949
+0x67 0xbd 0xff 0xe0 # CHECK: daddiu $sp, $sp, -32
 0x6b 0x18 0xef 0xb9 # CHECK: ldl $24, -4167($24)
 0x6e 0x8e 0x89 0x6a # CHECK: ldr $14, -30358($20)
-0x4d 0xf7 0x02 0x01 # CHECK: ldxc1 $f8, $23($15)
-0x84 0xa4 0x00 0x0c # CHECK: lh $4, 12($5)
-0x84 0xa4 0x00 0x0c # CHECK: lh $4, 12($5)
-0xc0 0xe9 0x23 0xc6 # CHECK: ll $9, 9158($7)
-0xd3 0xe0 0xc6 0x70 # CHECK: lld $zero, -14736($ra)
-0x3c 0x06 0x45 0x67 # CHECK: lui $6, 17767
-0x4e 0xb6 0x04 0xc5 # CHECK: luxc1 $f19, $22($21)
-0x9c 0x73 0xa1 0xea # CHECK: lwu $19, -24086($3)
-0x4f 0xd1 0x03 0x00 # CHECK: lwxc1 $f12, $17($fp)
-0x8c 0xa4 0x00 0x18 # CHECK: lw $4, 24($5)
-0xc4 0xe9 0x23 0xc6 # CHECK: lwc1 $f9, 9158($7)
-0x88 0x82 0x00 0x03 # CHECK: lwl $2, 3($4)
-0x98 0xa3 0x00 0x10 # CHECK: lwr $3, 16($5)
 0x70 0xc7 0x00 0x00 # CHECK: madd $6, $7
 0x70 0xc7 0x00 0x01 # CHECK: maddu $6, $7
-0x44 0x06 0x38 0x00 # CHECK: mfc1 $6, $f7
-0x00 0x00 0x28 0x10 # CHECK: mfhi $5
-0x00 0x00 0x28 0x12 # CHECK: mflo $5
-0x46 0x20 0x41 0x86 # CHECK: mov.d $f6, $f8
-0x46 0x00 0x39 0x86 # CHECK: mov.s $f6, $f7
 0x70 0xc7 0x00 0x04 # CHECK: msub $6, $7
 0x70 0xc7 0x00 0x05 # CHECK: msubu $6, $7
-0x44 0x86 0x38 0x00 # CHECK: mtc1 $6, $f7
-0x00 0xe0 0x00 0x11 # CHECK: mthi $7
-0x00 0xe0 0x00 0x13 # CHECK: mtlo $7
-0x46 0x2e 0x62 0x02 # CHECK: mul.d $f8, $f12, $f14
-0x46 0x07 0x32 0x42 # CHECK: mul.s $f9, $f6, $f7
 0x70 0xc7 0x48 0x02 # CHECK: mul $9, $6, $7
-0x00 0x65 0x00 0x18 # CHECK: mult $3, $5
-0x00 0x65 0x00 0x19 # CHECK: multu $3, $5
-0x46 0x20 0x73 0x07 # CHECK: neg.d $f12, $f14
-0x46 0x00 0x39 0x87 # CHECK: neg.s $f6, $f7
-0x00 0x00 0x00 0x00 # CHECK: nop
-0x00 0xc7 0x48 0x27 # CHECK: nor $9, $6, $7
-0x00 0x65 0x18 0x25 # CHECK: or $3, $3, $5
-0x34 0xc9 0x45 0x67 # CHECK: ori $9, $6, 17767
-0x46 0x20 0x0b 0x08 # CHECK: round.l.d $f12, $f1
-0x46 0x00 0x2e 0x48 # CHECK: round.l.s $f25, $f5
-0x46 0x20 0x73 0x0c # CHECK: round.w.d $f12, $f14
-0x46 0x00 0x39 0x8c # CHECK: round.w.s $f6, $f7
-0xa0 0xa4 0x23 0xc6 # CHECK: sb $4, 9158($5)
-0xa0 0xa4 0x00 0x06 # CHECK: sb $4, 6($5)
-0xe0 0xe9 0x23 0xc6 # CHECK: sc $9, 9158($7)
-0xf3 0xaf 0xdf 0xcd # CHECK: scd $15, -8243($sp)
-0xfd 0x4c 0x16 0xcb # CHECK: sd $12, 5835($10)
-0xf4 0xe9 0x23 0xc6 # CHECK: sdc1 $f9, 9158($7)
-0xb3 0xc7 0xae 0x1f # CHECK: sdl $7, -20961($fp)
-0xb5 0x8b 0xb0 0x39 # CHECK: sdr $11, -20423($12)
-0x4d 0xca 0x58 0x09 # CHECK: sdxc1 $f11, $10($14)
-0xa4 0xa4 0x23 0xc6 # CHECK: sh $4, 9158($5)
-0x00 0x03 0x21 0xc0 # CHECK: sll $4, $3, 7
-0x00 0xa3 0x10 0x04 # CHECK: sllv $2, $3, $5
-0x00 0x65 0x18 0x2a # CHECK: slt $3, $3, $5
-0x28 0x63 0x00 0x67 # CHECK: slti $3, $3, 103
-0x2c 0x63 0x00 0x67 # CHECK: sltiu $3, $3, 103
-0x00 0x65 0x18 0x2b # CHECK: sltu $3, $3, $5
-0x46 0x20 0x73 0x04 # CHECK: sqrt.d $f12, $f14
-0x46 0x00 0x39 0x84 # CHECK: sqrt.s $f6, $f7
-0x00 0x03 0x21 0xc3 # CHECK: sra $4, $3, 7
-0x00 0xa3 0x10 0x07 # CHECK: srav $2, $3, $5
-0x00 0x03 0x21 0xc2 # CHECK: srl $4, $3, 7
-0x00 0xa3 0x10 0x06 # CHECK: srlv $2, $3, $5
-0x46 0x2e 0x62 0x01 # CHECK: sub.d $f8, $f12, $f14
-0x46 0x07 0x32 0x41 # CHECK: sub.s $f9, $f6, $f7
-0x00 0xc7 0x48 0x22 # CHECK: sub $9, $6, $7
-0x00 0x65 0x20 0x23 # CHECK: subu $4, $3, $5
-0xac 0xa4 0x00 0x18 # CHECK: sw $4, 24($5)
-0xe4 0xe9 0x23 0xc6 # CHECK: swc1 $f9, 9158($7)
-0x4d 0xbb 0x60 0x0d # CHECK: suxc1 $f12, $27($13)
-0x4f 0x4c 0x98 0x08 # CHECK: swxc1 $f19, $12($26)
-0xa8 0xa4 0x00 0x10 # CHECK: swl $4, 16($5)
-0xb8 0xe6 0x00 0x10 # CHECK: swr $6, 16($7)
-0x00 0x00 0x01 0xcf # CHECK: sync 7
-0x46 0x20 0xbd 0xc9 # CHECK: trunc.l.d $f23, $f23
-0x46 0x00 0xff 0x09 # CHECK: trunc.l.s $f28, $f31
-0x46 0x20 0x73 0x0d # CHECK: trunc.w.d $f12, $f14
-0x46 0x00 0x39 0x8d # CHECK: trunc.w.s $f6, $f7
-0x00 0x65 0x18 0x26 # CHECK: xor $3, $3, $5
-0x38 0xc9 0x45 0x67 # CHECK: xori $9, $6, 17767
+0x70 0xd2 0x90 0x25 # CHECK: dclo $18, $6
+0x70 0xe6 0x30 0x20 # CHECK: clz $6, $7
+0x70 0xe6 0x30 0x21 # CHECK: clo $6, $7
+0x73 0x30 0x80 0x24 # CHECK: dclz $16, $25
+0x74 0x00 0x01 0x4c # CHECK: jalx 1328
 0x7c 0x05 0xe8 0x3b # CHECK: .set push
                     # CHECK: .set mips32r2
                     # CHECK: rdhwr $5, $29
                     # CHECK: .set pop
+0x80 0xa4 0x23 0xc6 # CHECK: lb $4, 9158($5)
+0x84 0xa4 0x00 0x0c # CHECK: lh $4, 12($5)
+0x84 0xa4 0x00 0x0c # CHECK: lh $4, 12($5)
+0x88 0x82 0x00 0x03 # CHECK: lwl $2, 3($4)
+0x8c 0x3b 0xc4 0xcd # CHECK: lw $27, -15155($1)
+0x8c 0xa4 0x00 0x18 # CHECK: lw $4, 24($5)
+0x90 0xa4 0x00 0x06 # CHECK: lbu $4, 6($5)
+0x98 0xa3 0x00 0x10 # CHECK: lwr $3, 16($5)
+0x9c 0x63 0xf9 0x2e # CHECK: lwu $3, -1746($3)
+0x9c 0x73 0xa1 0xea # CHECK: lwu $19, -24086($3)
+0xa0 0xa4 0x00 0x06 # CHECK: sb $4, 6($5)
+0xa0 0xa4 0x23 0xc6 # CHECK: sb $4, 9158($5)
+0xa4 0xa4 0x23 0xc6 # CHECK: sh $4, 9158($5)
+0xa8 0xa4 0x00 0x10 # CHECK: swl $4, 16($5)
+0xac 0x3a 0xc4 0xc9 # CHECK: sw $26, -15159($1)
+0xac 0xa4 0x00 0x18 # CHECK: sw $4, 24($5)
+0xb3 0xc7 0xae 0x1f # CHECK: sdl $7, -20961($fp)
+0xb5 0x8b 0xb0 0x39 # CHECK: sdr $11, -20423($12)
+0xb8 0xe6 0x00 0x10 # CHECK: swr $6, 16($7)
 0xbc 0x61 0x00 0x02 # CHECK: cache 1, 2($3)
+0xc0 0xe9 0x23 0xc6 # CHECK: ll $9, 9158($7)
+0xc4 0xe9 0x23 0xc6 # CHECK: lwc1 $f9, 9158($7)
+0xc8 0xc8 0x23 0xca # CHECK: lwc2 $8, 9162($6)
 0xcc 0x43 0x00 0x04 # CHECK: pref 3, 4($2)
+0xd3 0xe0 0xc6 0x70 # CHECK: lld $zero, -14736($ra)
+0xd4 0xe9 0x23 0xc6 # CHECK: ldc1 $f9, 9158($7)
+0xd9 0x03 0x23 0xca # CHECK: ldc2  $3, 9162($8)
+0xdc 0x1a 0x0f 0x76 # CHECK: ld $26, 3958($zero)
+0xde 0x3d 0x90 0x1b # CHECK: ld $sp, -28645($17)
+0xe0 0xe9 0x23 0xc6 # CHECK: sc $9, 9158($7)
+0xe4 0xe9 0x23 0xc6 # CHECK: swc1 $f9, 9158($7)
 0xe8 0xe9 0x23 0xc6 # CHECK: swc2 $9, 9158($7)
-0xc8 0xc8 0x23 0xca # CHECK: lwc2 $8, 9162($6)
+0xf3 0xaf 0xdf 0xcd # CHECK: scd $15, -8243($sp)
+0xf4 0xe9 0x23 0xc6 # CHECK: sdc1 $f9, 9158($7)
+0xf8 0xe9 0x23 0xc6 # CHECK: sdc2  $9, 9158($7)
+0xfc 0x06 0x45 0x67 # CHECK: sd $6, 17767($zero)
+0xfd 0x4c 0x16 0xcb # CHECK: sd $12, 5835($10)
diff --git a/test/MC/Disassembler/Mips/mips64r2.txt b/test/MC/Disassembler/Mips/mips64r2.txt
deleted file mode 100644
index cee6f3c21f7..00000000000
--- a/test/MC/Disassembler/Mips/mips64r2.txt
+++ /dev/null
@@ -1,90 +0,0 @@
-# RUN: llvm-mc --disassemble %s -triple=mips64-unknown-linux -mattr +mips64r2 | FileCheck %s
-# CHECK: daddiu $11, $26, 31949
-0x67 0x4b 0x7c 0xcd
-
-# CHECK: daddu $26, $1, $11
-0x00 0x2b 0xd0 0x2d
-
-# CHECK: ddiv $zero, $26, $22
-0x03 0x56 0x00 0x1e
-
-# CHECK: ddivu $zero, $9, $24
-0x01 0x38 0x00 0x1f
-
-# CHECK: dmfc1 $2, $f14
-0x44 0x22 0x70 0x00
-
-# CHECK: dmtc1 $23, $f5
-0x44 0xb7 0x28 0x00
-
-# CHECK: dmult $11, $26
-0x01 0x7a 0x00 0x1c
-
-# CHECK: dmultu $23, $13
-0x02 0xed 0x00 0x1d
-
-# CHECK: dsll $3, $24, 17
-0x00 0x18 0x1c 0x78
-
-# CHECK: dsllv $gp, $27, $24
-0x03 0x1b 0xe0 0x14
-
-# CHECK: dsra $1, $1, 30
-0x00 0x01 0x0f 0xbb
-
-# CHECK: dsrav $1, $1, $fp
-0x03 0xc1 0x08 0x17
-
-# CHECK: dsrl $10, $gp, 24
-0x00 0x1c 0x56 0x3a
-
-# CHECK: dsrlv $gp, $10, $23
-0x02 0xea 0xe0 0x16
-
-# CHECK: dsubu $gp, $27, $24
-0x03 0x78 0xe0 0x2f
-
-# CHECK: lw $27, -15155($1)
-0x8c 0x3b 0xc4 0xcd
-
-# CHECK: lui $1, 1
-0x3c 0x01 0x00 0x01
-
-# CHECK: lwu $3, -1746($3)
-0x9c 0x63 0xf9 0x2e
-
-# CHECK: lui $ra, 1
-0x3c 0x1f 0x00 0x01
-
-# CHECK: sw $26, -15159($1)
-0xac 0x3a 0xc4 0xc9
-
-# CHECK: ld $26, 3958($zero)
-0xdc 0x1a 0x0f 0x76
-
-# CHECK: sd $6, 17767($zero)
-0xfc 0x06 0x45 0x67
-
-# CHECK: dclo $9, $24
-0x73 0x09 0x48 0x25
-
-# CHECK: dclz $26, $9
-0x71 0x3a 0xd0 0x24
-
-# CHECK: dext $7, $gp, 29, 31
-0x7f 0x87 0xf7 0x43
-
-# CHECK: dins $20, $gp, 15, 1
-0x7f 0x94 0x7b 0xc7
-
-# CHECK: dsbh $7, $gp
-0x7c 0x1c 0x38 0xa4
-
-# CHECK: dshd $3, $14
-0x7c 0x0e 0x19 0x64
-
-# CHECK: drotr $20, $27, 6
-0x00 0x3b 0xa1 0xba
-
-# CHECK: drotrv $24, $23, $5
-0x00 0xb7 0xc0 0x56
diff --git a/test/MC/Disassembler/Mips/mips64r2/valid-mips64r2-el.txt b/test/MC/Disassembler/Mips/mips64r2/valid-mips64r2-el.txt
index 6509456b07e..2c6859f27fa 100644
--- a/test/MC/Disassembler/Mips/mips64r2/valid-mips64r2-el.txt
+++ b/test/MC/Disassembler/Mips/mips64r2/valid-mips64r2-el.txt
@@ -88,7 +88,9 @@
 0x24 0x80 0x30 0x73 # CHECK: dclz $16, $25
 0x1e 0x00 0x53 0x03 # CHECK: ddiv $zero, $26, $19
 0x1f 0x00 0x11 0x02 # CHECK: ddivu $zero, $16, $17
+0x00 0x50 0x38 0x40 # CHECK: dmfc0 $24, $10, 0
 0x00 0x68 0x2c 0x44 # CHECK: dmfc1 $12, $f13
+0x00 0x50 0xa4 0x40 # CHECK: dmtc0 $4, $10, 0
 0x00 0x70 0xb0 0x44 # CHECK: dmtc1 $16, $f14
 0x1c 0x00 0xe9 0x02 # CHECK: dmult $23, $9
 0x1d 0x00 0xa6 0x00 # CHECK: dmultu $5, $6
@@ -161,6 +163,7 @@
 0x00 0x00 0xc7 0x70 # CHECK: madd $6, $7
 0x60 0x98 0xf9 0x4f # CHECK: madd.s $f1, $f31, $f19, $f25
 0x01 0x00 0xc7 0x70 # CHECK: maddu $6, $7
+0x01 0x78 0x08 0x40 # CHECK: mfc0 $8, $15, 1
 0x00 0x38 0x06 0x44 # CHECK: mfc1 $6, $f7
 0x10 0x28 0x00 0x00 # CHECK: mfhi $5
 0x00 0xc0 0x7e 0x44 # CHECK: mfhc1 $fp, $f24
@@ -170,6 +173,7 @@
 0x04 0x00 0xc7 0x70 # CHECK: msub $6, $7
 0x28 0x53 0x70 0x4e # CHECK: msub.s $f12, $f19, $f10, $f16
 0x05 0x00 0xc7 0x70 # CHECK: msubu $6, $7
+0x01 0x78 0x89 0x40 # CHECK: mtc0 $9, $15, 1
 0x00 0x38 0x86 0x44 # CHECK: mtc1 $6, $f7
 0x11 0x00 0xe0 0x00 # CHECK: mthi $7
 0x00 0x80 0xe0 0x44 # CHECK: mthc1 $zero, $f16
diff --git a/test/MC/Disassembler/Mips/mips64r2/valid-mips64r2.txt b/test/MC/Disassembler/Mips/mips64r2/valid-mips64r2.txt
index 9fc9f6e6bb0..0c6e10ee37c 100644
--- a/test/MC/Disassembler/Mips/mips64r2/valid-mips64r2.txt
+++ b/test/MC/Disassembler/Mips/mips64r2/valid-mips64r2.txt
@@ -3,235 +3,271 @@
 # an effect on the disassembler behaviour.
 # RUN: llvm-mc --disassemble %s -triple=mips-unknown-linux -mcpu=mips64r2 | FileCheck %s
 # CHECK: .text
-0x46 0x20 0x73 0x05 # CHECK: abs.d $f12, $f14
-0x46 0x00 0x39 0x85 # CHECK: abs.s $f6, $f7
+0x00 0x00 0x00 0x00 # CHECK: nop
+0x00 0x00 0x01 0xcf # CHECK: sync 7
+0x00 0x00 0x04 0xb8 # CHECK: dsll $zero, $zero, 18
+0x00 0x00 0x04 0xbc # CHECK: dsll32 $zero, $zero, 18
+0x00 0x00 0x04 0xbc # CHECK: dsll32 $zero, $zero, 18
+0x00 0x00 0x28 0x10 # CHECK: mfhi $5
+0x00 0x00 0x28 0x12 # CHECK: mflo $5
+0x00 0x01 0x0f 0xbb # CHECK: dsra $1, $1, 30
+0x00 0x03 0x10 0x2e # CHECK: dneg $2, $3
+0x00 0x03 0x10 0x2f # CHECK: dnegu $2, $3
+0x00 0x03 0x21 0xc0 # CHECK: sll $4, $3, 7
+0x00 0x03 0x21 0xc2 # CHECK: srl $4, $3, 7
+0x00 0x03 0x21 0xc3 # CHECK: sra $4, $3, 7
+0x00 0x06 0x9d 0xfa # CHECK: dsrl $19, $6, 23
+0x00 0x06 0x9d 0xfe # CHECK: dsrl32 $19, $6, 23
+0x00 0x12 0xe2 0xbb # CHECK: dsra $gp, $18, 10
+0x00 0x12 0xe2 0xbf # CHECK: dsra32 $gp, $18, 10
+0x00 0x13 0x9d 0xfa # CHECK: dsrl $19, $19, 23
+0x00 0x13 0x9d 0xfe # CHECK: dsrl32 $19, $19, 23
+0x00 0x14 0x04 0xb8 # CHECK: dsll $zero, $20, 18
+0x00 0x18 0x1c 0x78 # CHECK: dsll $3, $24, 17
+0x00 0x1c 0x56 0x3a # CHECK: dsrl $10, $gp, 24
+0x00 0x1c 0xe2 0xbb # CHECK: dsra $gp, $gp, 10
+0x00 0x1c 0xe2 0xbf # CHECK: dsra32 $gp, $gp, 10
+0x00 0x21 0x0b 0xfa # CHECK: drotr $1, $1, 15
+0x00 0x21 0x0b 0xfe # CHECK: drotr32 $1, $1, 15
+0x00 0x26 0x49 0xc2 # CHECK: rotr $9, $6, 7
+0x00 0x2b 0xd0 0x2d # CHECK: daddu $26, $1, $11
+0x00 0x2e 0x0b 0xfa # CHECK: drotr $1, $14, 15
+0x00 0x2e 0x0b 0xfe # CHECK: drotr32 $1, $14, 15
+0x00 0x3b 0xa1 0xba # CHECK: drotr $20, $27, 6
+0x00 0x3f 0x98 0x2c # CHECK: dadd $19, $1, $ra
+0x00 0x3f 0x98 0x2d # CHECK: daddu $19, $1, $ra
+0x00 0x65 0x00 0x18 # CHECK: mult $3, $5
+0x00 0x65 0x00 0x19 # CHECK: multu $3, $5
+0x00 0x65 0x18 0x25 # CHECK: or $3, $3, $5
+0x00 0x65 0x18 0x26 # CHECK: xor $3, $3, $5
+0x00 0x65 0x18 0x2a # CHECK: slt $3, $3, $5
+0x00 0x65 0x18 0x2b # CHECK: sltu $3, $3, $5
+0x00 0x65 0x20 0x23 # CHECK: subu $4, $3, $5
+0x00 0x80 0xfc 0x09 # CHECK: jalr.hb $4
+0x00 0xa0 0x24 0x09 # CHECK: jalr.hb $4, $5
+0x00 0xa3 0x10 0x04 # CHECK: sllv $2, $3, $5
+0x00 0xa3 0x10 0x06 # CHECK: srlv $2, $3, $5
+0x00 0xa3 0x10 0x07 # CHECK: srav $2, $3, $5
+0x00 0xa6 0x00 0x1d # CHECK: dmultu $5, $6
+0x00 0xb7 0xc0 0x56 # CHECK: drotrv $24, $23, $5
+0x00 0xba 0x28 0x2f # CHECK: dsubu $5, $5, $26
 0x00 0xc7 0x48 0x20 # CHECK: add $9, $6, $7
-0x46 0x2e 0x62 0x00 # CHECK: add.d $f8, $f12, $f14
-0x46 0x07 0x32 0x40 # CHECK: add.s $f9, $f6, $f7
-0x20 0xc9 0x45 0x67 # CHECK: addi $9, $6, 17767
-0x24 0xc9 0xc5 0x67 # CHECK: addiu $9, $6, -15001
 0x00 0xc7 0x48 0x21 # CHECK: addu $9, $6, $7
+0x00 0xc7 0x48 0x22 # CHECK: sub $9, $6, $7
 0x00 0xc7 0x48 0x24 # CHECK: and $9, $6, $7
-0x30 0xc9 0x45 0x67 # CHECK: andi $9, $6, 17767
+0x00 0xc7 0x48 0x27 # CHECK: nor $9, $6, $7
+0x00 0xe0 0x00 0x08 # CHECK: jr $7
+0x00 0xe0 0x00 0x11 # CHECK: mthi $7
+0x00 0xe0 0x00 0x13 # CHECK: mtlo $7
+0x00 0xe0 0xf8 0x09 # CHECK: jalr $7
+0x00 0xe6 0x48 0x46 # CHECK: rotrv $9, $6, $7
+0x01 0x38 0x00 0x1f # CHECK: ddivu $zero, $9, $24
+0x01 0x7a 0x00 0x1c # CHECK: dmult $11, $26
+0x01 0x94 0x00 0x14 # CHECK: dsllv $zero, $20, $12
+0x01 0x94 0x00 0x14 # CHECK: dsllv $zero, $20, $12
+0x01 0xee 0x08 0x56 # CHECK: drotrv $1, $14, $15
+0x02 0x11 0x00 0x1f # CHECK: ddivu $zero, $16, $17
+0x02 0x72 0xe0 0x17 # CHECK: dsrav $gp, $18, $19
+0x02 0x72 0xe0 0x17 # CHECK: dsrav $gp, $18, $19
+0x02 0x86 0x98 0x16 # CHECK: dsrlv $19, $6, $20
+0x02 0x86 0x98 0x16 # CHECK: dsrlv $19, $6, $20
+0x02 0xc8 0x38 0x2e # CHECK: dsub $7, $22, $8
+0x02 0xe9 0x00 0x1c # CHECK: dmult $23, $9
+0x02 0xea 0xe0 0x16 # CHECK: dsrlv $gp, $10, $23
+0x02 0xed 0x00 0x1d # CHECK: dmultu $23, $13
+0x03 0x1b 0xe0 0x14 # CHECK: dsllv $gp, $27, $24
+0x03 0x53 0x00 0x1e # CHECK: ddiv $zero, $26, $19
+0x03 0x56 0x00 0x1e # CHECK: ddiv $zero, $26, $22
+0x03 0x78 0xe0 0x2f # CHECK: dsubu $gp, $27, $24
+0x03 0xc1 0x08 0x17 # CHECK: dsrav $1, $1, $fp
+0x04 0xc1 0x01 0x4c # CHECK: bgez $6, 1332
+0x04 0xd1 0x01 0x4c # CHECK: bgezal $6, 1332
+0x08 0x00 0x01 0x4c # CHECK: j 1328
+0x0c 0x00 0x01 0x4c # CHECK: jal 1328
 0x10 0x00 0x01 0x4c # CHECK: b 1332
+0x11 0x26 0x01 0x4c # CHECK: beq $9, $6, 1332
+0x15 0x26 0x01 0x4c # CHECK: bne $9, $6, 1332
+0x18 0xc0 0x01 0x4c # CHECK: blez $6, 1332
+0x1c 0xc0 0x01 0x4c # CHECK: bgtz $6, 1332
+0x20 0xc9 0x45 0x67 # CHECK: addi $9, $6, 17767
+0x24 0xc9 0xc5 0x67 # CHECK: addiu $9, $6, -15001
+0x28 0x63 0x00 0x67 # CHECK: slti $3, $3, 103
+0x2c 0x63 0x00 0x67 # CHECK: sltiu $3, $3, 103
+0x30 0xc9 0x45 0x67 # CHECK: andi $9, $6, 17767
+0x34 0xc9 0x45 0x67 # CHECK: ori $9, $6, 17767
+0x38 0xc9 0x45 0x67 # CHECK: xori $9, $6, 17767
+0x3c 0x01 0x00 0x01 # CHECK: lui $1, 1
+0x3c 0x06 0x45 0x67 # CHECK: lui $6, 17767
+0x3c 0x1f 0x00 0x01 # CHECK: lui $ra, 1
+0x40 0x08 0x78 0x01 # CHECK: mfc0 $8, $15, 1
+0x40 0x38 0x50 0x00 # CHECK: dmfc0 $24, $10, 0
+0x40 0x89 0x78 0x01 # CHECK: mtc0 $9, $15, 1
+0x40 0xa4 0x50 0x00 # CHECK: dmtc0 $4, $10, 0
+0x41 0x60 0x60 0x00 # CHECK: di
+0x41 0x60 0x60 0x20 # CHECK: ei
+0x41 0x6e 0x60 0x20 # CHECK: ei $14
+0x41 0x7e 0x60 0x00 # CHECK: di $fp
+0x44 0x06 0x38 0x00 # CHECK: mfc1 $6, $f7
+0x44 0x22 0x70 0x00 # CHECK: dmfc1 $2, $f14
+0x44 0x2c 0x68 0x00 # CHECK: dmfc1 $12, $f13
+0x44 0x46 0x38 0x00 # CHECK: cfc1 $6, $7
+0x44 0x7e 0xc0 0x00 # CHECK: mfhc1 $fp, $f24
+0x44 0x86 0x38 0x00 # CHECK: mtc1 $6, $f7
+0x44 0xb0 0x70 0x00 # CHECK: dmtc1 $16, $f14
+0x44 0xb7 0x28 0x00 # CHECK: dmtc1 $23, $f5
+0x44 0xc6 0x38 0x00 # CHECK: ctc1 $6, $7
+0x44 0xe0 0x80 0x00 # CHECK: mthc1 $zero, $f16
 0x45 0x00 0x01 0x4c # CHECK: bc1f 1332
-0x45 0x1c 0x01 0x4c # CHECK: bc1f $fcc7, 1332
 0x45 0x01 0x01 0x4c # CHECK: bc1t 1332
+0x45 0x1c 0x01 0x4c # CHECK: bc1f $fcc7, 1332
 0x45 0x1d 0x01 0x4c # CHECK: bc1t $fcc7, 1332
-0x11 0x26 0x01 0x4c # CHECK: beq $9, $6, 1332
-0x04 0xc1 0x01 0x4c # CHECK: bgez $6, 1332
-0x04 0xd1 0x01 0x4c # CHECK: bgezal $6, 1332
-0x1c 0xc0 0x01 0x4c # CHECK: bgtz $6, 1332
-0x18 0xc0 0x01 0x4c # CHECK: blez $6, 1332
-0x15 0x26 0x01 0x4c # CHECK: bne $9, $6, 1332
-0x46 0x2e 0x60 0x32 # CHECK: c.eq.d $f12, $f14
-0x46 0x07 0x30 0x32 # CHECK: c.eq.s $f6, $f7
-0x46 0x2e 0x60 0x30 # CHECK: c.f.d $f12, $f14
+0x46 0x00 0x2b 0x0b # CHECK: floor.l.s $f12, $f5
+0x46 0x00 0x2e 0x48 # CHECK: round.l.s $f25, $f5
+0x46 0x00 0x39 0x84 # CHECK: sqrt.s $f6, $f7
+0x46 0x00 0x39 0x85 # CHECK: abs.s $f6, $f7
+0x46 0x00 0x39 0x86 # CHECK: mov.s $f6, $f7
+0x46 0x00 0x39 0x87 # CHECK: neg.s $f6, $f7
+0x46 0x00 0x39 0x8c # CHECK: round.w.s $f6, $f7
+0x46 0x00 0x39 0x8d # CHECK: trunc.w.s $f6, $f7
+0x46 0x00 0x39 0x8e # CHECK: ceil.w.s $f6, $f7
+0x46 0x00 0x39 0x8f # CHECK: floor.w.s $f6, $f7
+0x46 0x00 0x39 0xa1 # CHECK: cvt.d.s $f6, $f7
+0x46 0x00 0x39 0xa4 # CHECK: cvt.w.s $f6, $f7
+0x46 0x00 0x39 0xa5 # CHECK: cvt.l.s $f6, $f7
+0x46 0x00 0x6c 0x8a # CHECK: ceil.l.s $f18, $f13
+0x46 0x00 0xff 0x09 # CHECK: trunc.l.s $f28, $f31
 0x46 0x07 0x30 0x30 # CHECK: c.f.s $f6, $f7
-0x46 0x2e 0x60 0x3e # CHECK: c.le.d $f12, $f14
-0x46 0x07 0x30 0x3e # CHECK: c.le.s $f6, $f7
-0x46 0x2e 0x60 0x3c # CHECK: c.lt.d $f12, $f14
+0x46 0x07 0x30 0x31 # CHECK: c.un.s $f6, $f7
+0x46 0x07 0x30 0x32 # CHECK: c.eq.s $f6, $f7
+0x46 0x07 0x30 0x34 # CHECK: c.olt.s $f6, $f7
+0x46 0x07 0x30 0x35 # CHECK: c.ult.s $f6, $f7
+0x46 0x07 0x30 0x36 # CHECK: c.ole.s $f6, $f7
+0x46 0x07 0x30 0x37 # CHECK: c.ule.s $f6, $f7
+0x46 0x07 0x30 0x38 # CHECK: c.sf.s $f6, $f7
+0x46 0x07 0x30 0x39 # CHECK: c.ngle.s $f6, $f7
+0x46 0x07 0x30 0x3a # CHECK: c.seq.s $f6, $f7
+0x46 0x07 0x30 0x3b # CHECK: c.ngl.s $f6, $f7
 0x46 0x07 0x30 0x3c # CHECK: c.lt.s $f6, $f7
-0x46 0x2e 0x60 0x3d # CHECK: c.nge.d $f12, $f14
 0x46 0x07 0x30 0x3d # CHECK: c.nge.s $f6, $f7
-0x46 0x2e 0x60 0x3b # CHECK: c.ngl.d $f12, $f14
-0x46 0x07 0x30 0x3b # CHECK: c.ngl.s $f6, $f7
-0x46 0x2e 0x60 0x39 # CHECK: c.ngle.d $f12, $f14
-0x46 0x07 0x30 0x39 # CHECK: c.ngle.s $f6, $f7
-0x46 0x2e 0x60 0x3f # CHECK: c.ngt.d $f12, $f14
+0x46 0x07 0x30 0x3e # CHECK: c.le.s $f6, $f7
 0x46 0x07 0x30 0x3f # CHECK: c.ngt.s $f6, $f7
-0x46 0x2e 0x60 0x36 # CHECK: c.ole.d $f12, $f14
-0x46 0x07 0x30 0x36 # CHECK: c.ole.s $f6, $f7
-0x46 0x2e 0x60 0x34 # CHECK: c.olt.d $f12, $f14
-0x46 0x07 0x30 0x34 # CHECK: c.olt.s $f6, $f7
-0x46 0x2e 0x60 0x3a # CHECK: c.seq.d $f12, $f14
-0x46 0x07 0x30 0x3a # CHECK: c.seq.s $f6, $f7
-0x46 0x2e 0x60 0x38 # CHECK: c.sf.d $f12, $f14
-0x46 0x07 0x30 0x38 # CHECK: c.sf.s $f6, $f7
-0x46 0x2e 0x60 0x33 # CHECK: c.ueq.d $f12, $f14
+0x46 0x07 0x32 0x40 # CHECK: add.s $f9, $f6, $f7
+0x46 0x07 0x32 0x41 # CHECK: sub.s $f9, $f6, $f7
+0x46 0x07 0x32 0x42 # CHECK: mul.s $f9, $f6, $f7
 0x46 0x12 0xe0 0x33 # CHECK: c.ueq.s $f28, $f18
-0x46 0x2e 0x60 0x37 # CHECK: c.ule.d $f12, $f14
-0x46 0x07 0x30 0x37 # CHECK: c.ule.s $f6, $f7
-0x46 0x2e 0x60 0x35 # CHECK: c.ult.d $f12, $f14
-0x46 0x07 0x30 0x35 # CHECK: c.ult.s $f6, $f7
-0x46 0x2e 0x60 0x31 # CHECK: c.un.d $f12, $f14
-0x46 0x07 0x30 0x31 # CHECK: c.un.s $f6, $f7
+0x46 0x20 0x0b 0x08 # CHECK: round.l.d $f12, $f1
 0x46 0x20 0x18 0x4a # CHECK: ceil.l.d $f1, $f3
-0x46 0x00 0x6c 0x8a # CHECK: ceil.l.s $f18, $f13
+0x46 0x20 0x3e 0x8b # CHECK: floor.l.d $f26, $f7
+0x46 0x20 0x41 0x86 # CHECK: mov.d $f6, $f8
+0x46 0x20 0x73 0x04 # CHECK: sqrt.d $f12, $f14
+0x46 0x20 0x73 0x05 # CHECK: abs.d $f12, $f14
+0x46 0x20 0x73 0x07 # CHECK: neg.d $f12, $f14
+0x46 0x20 0x73 0x0c # CHECK: round.w.d $f12, $f14
+0x46 0x20 0x73 0x0d # CHECK: trunc.w.d $f12, $f14
 0x46 0x20 0x73 0x0e # CHECK: ceil.w.d $f12, $f14
-0x46 0x00 0x39 0x8e # CHECK: ceil.w.s $f6, $f7
-0x44 0x46 0x38 0x00 # CHECK: cfc1 $6, $7
-0x70 0xe6 0x30 0x21 # CHECK: clo $6, $7
-0x70 0xe6 0x30 0x20 # CHECK: clz $6, $7
-0x44 0xc6 0x38 0x00 # CHECK: ctc1 $6, $7
-0x46 0xa0 0x81 0x21 # CHECK: cvt.d.l $f4, $f16
-0x46 0x00 0x39 0xa1 # CHECK: cvt.d.s $f6, $f7
-0x46 0x80 0x73 0x21 # CHECK: cvt.d.w $f12, $f14
-0x46 0x20 0x73 0x25 # CHECK: cvt.l.d $f12, $f14
-0x46 0x00 0x39 0xa5 # CHECK: cvt.l.s $f6, $f7
-0x46 0xa0 0xf3 0xe0 # CHECK: cvt.s.l $f15, $f30
+0x46 0x20 0x73 0x0f # CHECK: floor.w.d $f12, $f14
 0x46 0x20 0x73 0x20 # CHECK: cvt.s.d $f12, $f14
-0x46 0x80 0x39 0xa0 # CHECK: cvt.s.w $f6, $f7
 0x46 0x20 0x73 0x24 # CHECK: cvt.w.d $f12, $f14
-0x46 0x00 0x39 0xa4 # CHECK: cvt.w.s $f6, $f7
-0x00 0x3f 0x98 0x2c # CHECK: dadd $19, $1, $ra
+0x46 0x20 0x73 0x25 # CHECK: cvt.l.d $f12, $f14
+0x46 0x20 0xbd 0xc9 # CHECK: trunc.l.d $f23, $f23
+0x46 0x2e 0x60 0x30 # CHECK: c.f.d $f12, $f14
+0x46 0x2e 0x60 0x31 # CHECK: c.un.d $f12, $f14
+0x46 0x2e 0x60 0x32 # CHECK: c.eq.d $f12, $f14
+0x46 0x2e 0x60 0x33 # CHECK: c.ueq.d $f12, $f14
+0x46 0x2e 0x60 0x34 # CHECK: c.olt.d $f12, $f14
+0x46 0x2e 0x60 0x35 # CHECK: c.ult.d $f12, $f14
+0x46 0x2e 0x60 0x36 # CHECK: c.ole.d $f12, $f14
+0x46 0x2e 0x60 0x37 # CHECK: c.ule.d $f12, $f14
+0x46 0x2e 0x60 0x38 # CHECK: c.sf.d $f12, $f14
+0x46 0x2e 0x60 0x39 # CHECK: c.ngle.d $f12, $f14
+0x46 0x2e 0x60 0x3a # CHECK: c.seq.d $f12, $f14
+0x46 0x2e 0x60 0x3b # CHECK: c.ngl.d $f12, $f14
+0x46 0x2e 0x60 0x3c # CHECK: c.lt.d $f12, $f14
+0x46 0x2e 0x60 0x3d # CHECK: c.nge.d $f12, $f14
+0x46 0x2e 0x60 0x3e # CHECK: c.le.d $f12, $f14
+0x46 0x2e 0x60 0x3f # CHECK: c.ngt.d $f12, $f14
+0x46 0x2e 0x62 0x00 # CHECK: add.d $f8, $f12, $f14
+0x46 0x2e 0x62 0x01 # CHECK: sub.d $f8, $f12, $f14
+0x46 0x2e 0x62 0x02 # CHECK: mul.d $f8, $f12, $f14
+0x46 0x80 0x39 0xa0 # CHECK: cvt.s.w $f6, $f7
+0x46 0x80 0x73 0x21 # CHECK: cvt.d.w $f12, $f14
+0x46 0xa0 0x81 0x21 # CHECK: cvt.d.l $f4, $f16
+0x46 0xa0 0xf3 0xe0 # CHECK: cvt.s.l $f15, $f30
+0x4c 0xa6 0x00 0x05 # CHECK: luxc1 $f0, $6($5)
+0x4c 0xac 0xc8 0x30 # CHECK: nmadd.s $f0, $f5, $f25, $f12
+0x4c 0xb8 0x20 0x0d # CHECK: suxc1 $f4, $24($5)
+0x4d 0xcc 0x05 0x00 # CHECK: lwxc1 $f20, $12($14)
+0x4d 0xf7 0x02 0x01 # CHECK: ldxc1 $f8, $23($15)
+0x4e 0x70 0x53 0x28 # CHECK: msub.s $f12, $f19, $f10, $f16
+0x4e 0xd2 0xd0 0x08 # CHECK: swxc1 $f26, $18($22)
+0x4f 0x04 0x98 0x78 # CHECK: nmsub.s $f1, $f24, $f19, $f4
+0x4f 0x24 0x40 0x09 # CHECK: sdxc1 $f8, $4($25)
+0x4f 0xf9 0x98 0x60 # CHECK: madd.s $f1, $f31, $f19, $f25
+0x62 0x9d 0x6c 0x39 # CHECK: daddi $sp, $20, 27705
 0x62 0x9d 0x93 0xc7 # CHECK: daddi $sp, $20, -27705
-0x63 0xbd 0x93 0xc7 # CHECK: daddi $sp, $sp, -27705
 0x62 0x9d 0x93 0xc7 # CHECK: daddi $sp, $20, -27705
 0x62 0x9d 0x93 0xc7 # CHECK: daddi $sp, $20, -27705
+0x63 0xbd 0x6c 0x39 # CHECK: daddi $sp, $sp, 27705
+0x63 0xbd 0x93 0xc7 # CHECK: daddi $sp, $sp, -27705
 0x63 0xbd 0x93 0xc7 # CHECK: daddi $sp, $sp, -27705
-0x66 0xda 0xee 0x16 # CHECK: daddiu $26, $22, -4586
-0x00 0x3f 0x98 0x2d # CHECK: daddu $19, $1, $ra
 0x64 0x58 0x46 0x9f # CHECK: daddiu $24, $2, 18079
+0x65 0x6f 0xec 0x5f # CHECK: daddiu $15, $11, -5025
+0x65 0xce 0x11 0xea # CHECK: daddiu $14, $14, 4586
 0x66 0x73 0x69 0x3f # CHECK: daddiu $19, $19, 26943
+0x66 0xda 0xee 0x16 # CHECK: daddiu $26, $22, -4586
+0x67 0x4b 0x7c 0xcd # CHECK: daddiu $11, $26, 31949
+0x6b 0x18 0xef 0xb9 # CHECK: ldl $24, -4167($24)
+0x6e 0x8e 0x89 0x6a # CHECK: ldr $14, -30358($20)
+0x70 0xc7 0x00 0x00 # CHECK: madd $6, $7
+0x70 0xc7 0x00 0x01 # CHECK: maddu $6, $7
+0x70 0xc7 0x00 0x04 # CHECK: msub $6, $7
+0x70 0xc7 0x00 0x05 # CHECK: msubu $6, $7
+0x70 0xc7 0x48 0x02 # CHECK: mul $9, $6, $7
 0x70 0xd2 0x90 0x25 # CHECK: dclo $18, $6
+0x70 0xe6 0x30 0x20 # CHECK: clz $6, $7
+0x70 0xe6 0x30 0x21 # CHECK: clo $6, $7
+0x71 0x3a 0xd0 0x24 # CHECK: dclz $26, $9
+0x73 0x09 0x48 0x25 # CHECK: dclo $9, $24
 0x73 0x30 0x80 0x24 # CHECK: dclz $16, $25
-0x03 0x53 0x00 0x1e # CHECK: ddiv $zero, $26, $19
-0x02 0x11 0x00 0x1f # CHECK: ddivu $zero, $16, $17
-0x44 0x2c 0x68 0x00 # CHECK: dmfc1 $12, $f13
-0x44 0xb0 0x70 0x00 # CHECK: dmtc1 $16, $f14
-0x02 0xe9 0x00 0x1c # CHECK: dmult $23, $9
-0x00 0xa6 0x00 0x1d # CHECK: dmultu $5, $6
-0x00 0x00 0x04 0xb8 # CHECK: dsll $zero, $zero, 18
-0x00 0x14 0x04 0xb8 # CHECK: dsll $zero, $20, 18
-0x01 0x94 0x00 0x14 # CHECK: dsllv $zero, $20, $12
-0x00 0x00 0x04 0xbc # CHECK: dsll32 $zero, $zero, 18
-0x00 0x00 0x04 0xbc # CHECK: dsll32 $zero, $zero, 18
-0x01 0x94 0x00 0x14 # CHECK: dsllv $zero, $20, $12
-0x00 0x1c 0xe2 0xbb # CHECK: dsra $gp, $gp, 10
-0x00 0x12 0xe2 0xbb # CHECK: dsra $gp, $18, 10
-0x02 0x72 0xe0 0x17 # CHECK: dsrav $gp, $18, $19
-0x00 0x1c 0xe2 0xbf # CHECK: dsra32 $gp, $gp, 10
-0x00 0x12 0xe2 0xbf # CHECK: dsra32 $gp, $18, 10
-0x02 0x72 0xe0 0x17 # CHECK: dsrav $gp, $18, $19
-0x00 0x13 0x9d 0xfa # CHECK: dsrl $19, $19, 23
-0x00 0x06 0x9d 0xfa # CHECK: dsrl $19, $6, 23
-0x02 0x86 0x98 0x16 # CHECK: dsrlv $19, $6, $20
-0x00 0x13 0x9d 0xfe # CHECK: dsrl32 $19, $19, 23
-0x00 0x06 0x9d 0xfe # CHECK: dsrl32 $19, $6, 23
-0x02 0x86 0x98 0x16 # CHECK: dsrlv $19, $6, $20
-0x02 0xc8 0x38 0x2e # CHECK: dsub $7, $22, $8
+0x74 0x00 0x01 0x4c # CHECK: jalx 1328
+0x7c 0x07 0x30 0xa0 # CHECK: wsbh $6, $7
+0x7c 0x07 0x34 0x20 # CHECK: seb $6, $7
+0x7c 0x07 0x36 0x20 # CHECK: seh $6, $7
 0x7c 0x0e 0x18 0xa4 # CHECK: dsbh $3, $14
+0x7c 0x0e 0x19 0x64 # CHECK: dshd $3, $14
+0x7c 0x1c 0x38 0xa4 # CHECK: dsbh $7, $gp
 0x7c 0x1d 0x11 0x64 # CHECK: dshd $2, $sp
-0x62 0x9d 0x6c 0x39 # CHECK: daddi $sp, $20, 27705
-0x63 0xbd 0x6c 0x39 # CHECK: daddi $sp, $sp, 27705
-0x00 0xba 0x28 0x2f # CHECK: dsubu $5, $5, $26
-0x65 0x6f 0xec 0x5f # CHECK: daddiu $15, $11, -5025
-0x65 0xce 0x11 0xea # CHECK: daddiu $14, $14, 4586
-0x41 0x7e 0x60 0x00 # CHECK: di $fp
-0x41 0x60 0x60 0x00 # CHECK: di
-0x00 0x21 0x0b 0xfa # CHECK: drotr $1, $1, 15
-0x00 0x2e 0x0b 0xfa # CHECK: drotr $1, $14, 15
-0x00 0x21 0x0b 0xfe # CHECK: drotr32 $1, $1, 15
-0x00 0x2e 0x0b 0xfe # CHECK: drotr32 $1, $14, 15
-0x01 0xee 0x08 0x56 # CHECK: drotrv $1, $14, $15
-0x41 0x6e 0x60 0x20 # CHECK: ei $14
-0x41 0x60 0x60 0x20 # CHECK: ei
-0x46 0x20 0x3e 0x8b # CHECK: floor.l.d $f26, $f7
-0x46 0x00 0x2b 0x0b # CHECK: floor.l.s $f12, $f5
-0x46 0x20 0x73 0x0f # CHECK: floor.w.d $f12, $f14
-0x46 0x00 0x39 0x8f # CHECK: floor.w.s $f6, $f7
 0x7d 0x33 0x61 0x84 # CHECK: ins $19, $9, 6, 7
-0x08 0x00 0x01 0x4c # CHECK: j 1328
-0x0c 0x00 0x01 0x4c # CHECK: jal 1328
-0x74 0x00 0x01 0x4c # CHECK: jalx 1328
-0x00 0xe0 0xf8 0x09 # CHECK: jalr $7
-0x00 0x80 0xfc 0x09 # CHECK: jalr.hb $4
-0x00 0xa0 0x24 0x09 # CHECK: jalr.hb $4, $5
-0x00 0xe0 0x00 0x08 # CHECK: jr $7
+0x7f 0x87 0xf7 0x43 # CHECK: dext $7, $gp, 29, 31
+0x7f 0x94 0x7b 0xc7 # CHECK: dins $20, $gp, 15, 1
 0x80 0xa4 0x23 0xc6 # CHECK: lb $4, 9158($5)
-0x90 0xa4 0x00 0x06 # CHECK: lbu $4, 6($5)
-0xde 0x3d 0x90 0x1b # CHECK: ld $sp, -28645($17)
-0x6b 0x18 0xef 0xb9 # CHECK: ldl $24, -4167($24)
-0x6e 0x8e 0x89 0x6a # CHECK: ldr $14, -30358($20)
-0xd4 0xe9 0x23 0xc6 # CHECK: ldc1 $f9, 9158($7)
-0x4d 0xf7 0x02 0x01 # CHECK: ldxc1 $f8, $23($15)
 0x84 0xa4 0x00 0x0c # CHECK: lh $4, 12($5)
 0x84 0xa4 0x00 0x0c # CHECK: lh $4, 12($5)
-0xc0 0xe9 0x23 0xc6 # CHECK: ll $9, 9158($7)
-0xd3 0xe0 0xc6 0x70 # CHECK: lld $zero, -14736($ra)
-0x3c 0x06 0x45 0x67 # CHECK: lui $6, 17767
-0x4c 0xa6 0x00 0x05 # CHECK: luxc1 $f0, $6($5)
-0x8c 0xa4 0x00 0x18 # CHECK: lw $4, 24($5)
-0xc4 0xe9 0x23 0xc6 # CHECK: lwc1 $f9, 9158($7)
 0x88 0x82 0x00 0x03 # CHECK: lwl $2, 3($4)
+0x8c 0x3b 0xc4 0xcd # CHECK: lw $27, -15155($1)
+0x8c 0xa4 0x00 0x18 # CHECK: lw $4, 24($5)
+0x90 0xa4 0x00 0x06 # CHECK: lbu $4, 6($5)
 0x98 0xa3 0x00 0x10 # CHECK: lwr $3, 16($5)
-0x4d 0xcc 0x05 0x00 # CHECK: lwxc1 $f20, $12($14)
+0x9c 0x63 0xf9 0x2e # CHECK: lwu $3, -1746($3)
 0x9c 0x73 0xa1 0xea # CHECK: lwu $19, -24086($3)
-0x70 0xc7 0x00 0x00 # CHECK: madd $6, $7
-0x4f 0xf9 0x98 0x60 # CHECK: madd.s $f1, $f31, $f19, $f25
-0x70 0xc7 0x00 0x01 # CHECK: maddu $6, $7
-0x44 0x06 0x38 0x00 # CHECK: mfc1 $6, $f7
-0x00 0x00 0x28 0x10 # CHECK: mfhi $5
-0x44 0x7e 0xc0 0x00 # CHECK: mfhc1 $fp, $f24
-0x00 0x00 0x28 0x12 # CHECK: mflo $5
-0x46 0x20 0x41 0x86 # CHECK: mov.d $f6, $f8
-0x46 0x00 0x39 0x86 # CHECK: mov.s $f6, $f7
-0x70 0xc7 0x00 0x04 # CHECK: msub $6, $7
-0x4e 0x70 0x53 0x28 # CHECK: msub.s $f12, $f19, $f10, $f16
-0x70 0xc7 0x00 0x05 # CHECK: msubu $6, $7
-0x44 0x86 0x38 0x00 # CHECK: mtc1 $6, $f7
-0x00 0xe0 0x00 0x11 # CHECK: mthi $7
-0x44 0xe0 0x80 0x00 # CHECK: mthc1 $zero, $f16
-0x00 0xe0 0x00 0x13 # CHECK: mtlo $7
-0x46 0x2e 0x62 0x02 # CHECK: mul.d $f8, $f12, $f14
-0x46 0x07 0x32 0x42 # CHECK: mul.s $f9, $f6, $f7
-0x70 0xc7 0x48 0x02 # CHECK: mul $9, $6, $7
-0x00 0x65 0x00 0x18 # CHECK: mult $3, $5
-0x00 0x65 0x00 0x19 # CHECK: multu $3, $5
-0x46 0x20 0x73 0x07 # CHECK: neg.d $f12, $f14
-0x46 0x00 0x39 0x87 # CHECK: neg.s $f6, $f7
-0x4c 0xac 0xc8 0x30 # CHECK: nmadd.s $f0, $f5, $f25, $f12
-0x00 0x00 0x00 0x00 # CHECK: nop
-0x00 0xc7 0x48 0x27 # CHECK: nor $9, $6, $7
-0x4f 0x04 0x98 0x78 # CHECK: nmsub.s $f1, $f24, $f19, $f4
-0x00 0x65 0x18 0x25 # CHECK: or $3, $3, $5
-0x34 0xc9 0x45 0x67 # CHECK: ori $9, $6, 17767
-0x00 0x26 0x49 0xc2 # CHECK: rotr $9, $6, 7
-0x00 0xe6 0x48 0x46 # CHECK: rotrv $9, $6, $7
-0x46 0x20 0x0b 0x08 # CHECK: round.l.d $f12, $f1
-0x46 0x00 0x2e 0x48 # CHECK: round.l.s $f25, $f5
-0x46 0x20 0x73 0x0c # CHECK: round.w.d $f12, $f14
-0x46 0x00 0x39 0x8c # CHECK: round.w.s $f6, $f7
-0xa0 0xa4 0x23 0xc6 # CHECK: sb $4, 9158($5)
 0xa0 0xa4 0x00 0x06 # CHECK: sb $4, 6($5)
-0xe0 0xe9 0x23 0xc6 # CHECK: sc $9, 9158($7)
-0xf3 0xaf 0xdf 0xcd # CHECK: scd $15, -8243($sp)
-0xfd 0x4c 0x16 0xcb # CHECK: sd $12, 5835($10)
-0xf4 0xe9 0x23 0xc6 # CHECK: sdc1 $f9, 9158($7)
-0xb3 0xc7 0xae 0x1f # CHECK: sdl $7, -20961($fp)
-0xb5 0x8b 0xb0 0x39 # CHECK: sdr $11, -20423($12)
-0x4f 0x24 0x40 0x09 # CHECK: sdxc1 $f8, $4($25)
-0x7c 0x07 0x34 0x20 # CHECK: seb $6, $7
-0x7c 0x07 0x36 0x20 # CHECK: seh $6, $7
+0xa0 0xa4 0x23 0xc6 # CHECK: sb $4, 9158($5)
 0xa4 0xa4 0x23 0xc6 # CHECK: sh $4, 9158($5)
-0x00 0x03 0x21 0xc0 # CHECK: sll $4, $3, 7
-0x00 0xa3 0x10 0x04 # CHECK: sllv $2, $3, $5
-0x00 0x65 0x18 0x2a # CHECK: slt $3, $3, $5
-0x28 0x63 0x00 0x67 # CHECK: slti $3, $3, 103
-0x2c 0x63 0x00 0x67 # CHECK: sltiu $3, $3, 103
-0x00 0x65 0x18 0x2b # CHECK: sltu $3, $3, $5
-0x46 0x20 0x73 0x04 # CHECK: sqrt.d $f12, $f14
-0x46 0x00 0x39 0x84 # CHECK: sqrt.s $f6, $f7
-0x00 0x03 0x21 0xc3 # CHECK: sra $4, $3, 7
-0x00 0xa3 0x10 0x07 # CHECK: srav $2, $3, $5
-0x00 0x03 0x21 0xc2 # CHECK: srl $4, $3, 7
-0x00 0xa3 0x10 0x06 # CHECK: srlv $2, $3, $5
-0x46 0x2e 0x62 0x01 # CHECK: sub.d $f8, $f12, $f14
-0x46 0x07 0x32 0x41 # CHECK: sub.s $f9, $f6, $f7
-0x00 0xc7 0x48 0x22 # CHECK: sub $9, $6, $7
-0x00 0x65 0x20 0x23 # CHECK: subu $4, $3, $5
-0x4c 0xb8 0x20 0x0d # CHECK: suxc1 $f4, $24($5)
-0xac 0xa4 0x00 0x18 # CHECK: sw $4, 24($5)
-0xe4 0xe9 0x23 0xc6 # CHECK: swc1 $f9, 9158($7)
 0xa8 0xa4 0x00 0x10 # CHECK: swl $4, 16($5)
+0xac 0x3a 0xc4 0xc9 # CHECK: sw $26, -15159($1)
+0xac 0xa4 0x00 0x18 # CHECK: sw $4, 24($5)
+0xb3 0xc7 0xae 0x1f # CHECK: sdl $7, -20961($fp)
+0xb5 0x8b 0xb0 0x39 # CHECK: sdr $11, -20423($12)
 0xb8 0xe6 0x00 0x10 # CHECK: swr $6, 16($7)
-0x4e 0xd2 0xd0 0x08 # CHECK: swxc1 $f26, $18($22)
-0x00 0x00 0x01 0xcf # CHECK: sync 7
-0x46 0x20 0xbd 0xc9 # CHECK: trunc.l.d $f23, $f23
-0x46 0x00 0xff 0x09 # CHECK: trunc.l.s $f28, $f31
-0x46 0x20 0x73 0x0d # CHECK: trunc.w.d $f12, $f14
-0x46 0x00 0x39 0x8d # CHECK: trunc.w.s $f6, $f7
-0x7c 0x07 0x30 0xa0 # CHECK: wsbh $6, $7
-0x00 0x65 0x18 0x26 # CHECK: xor $3, $3, $5
-0x38 0xc9 0x45 0x67 # CHECK: xori $9, $6, 17767
+0xc0 0xe9 0x23 0xc6 # CHECK: ll $9, 9158($7)
+0xc4 0xe9 0x23 0xc6 # CHECK: lwc1 $f9, 9158($7)
+0xd3 0xe0 0xc6 0x70 # CHECK: lld $zero, -14736($ra)
+0xd4 0xe9 0x23 0xc6 # CHECK: ldc1 $f9, 9158($7)
+0xdc 0x1a 0x0f 0x76 # CHECK: ld $26, 3958($zero)
+0xde 0x3d 0x90 0x1b # CHECK: ld $sp, -28645($17)
+0xe0 0xe9 0x23 0xc6 # CHECK: sc $9, 9158($7)
+0xe4 0xe9 0x23 0xc6 # CHECK: swc1 $f9, 9158($7)
+0xf3 0xaf 0xdf 0xcd # CHECK: scd $15, -8243($sp)
+0xf4 0xe9 0x23 0xc6 # CHECK: sdc1 $f9, 9158($7)
+0xfc 0x06 0x45 0x67 # CHECK: sd $6, 17767($zero)
+0xfd 0x4c 0x16 0xcb # CHECK: sd $12, 5835($10)
diff --git a/test/MC/Disassembler/Mips/mips64r3/valid-mips64r3-el.txt b/test/MC/Disassembler/Mips/mips64r3/valid-mips64r3-el.txt
index 52374af2893..88e9c262a0e 100644
--- a/test/MC/Disassembler/Mips/mips64r3/valid-mips64r3-el.txt
+++ b/test/MC/Disassembler/Mips/mips64r3/valid-mips64r3-el.txt
@@ -85,7 +85,9 @@
 0x24 0x80 0x30 0x73 # CHECK: dclz $16, $25
 0x1e 0x00 0x53 0x03 # CHECK: ddiv $zero, $26, $19
 0x1f 0x00 0x11 0x02 # CHECK: ddivu $zero, $16, $17
+0x00 0x50 0x38 0x40 # CHECK: dmfc0 $24, $10, 0
 0x00 0x68 0x2c 0x44 # CHECK: dmfc1 $12, $f13
+0x00 0x50 0xa4 0x40 # CHECK: dmtc0 $4, $10, 0
 0x00 0x70 0xb0 0x44 # CHECK: dmtc1 $16, $f14
 0x1c 0x00 0xe9 0x02 # CHECK: dmult $23, $9
 0x1d 0x00 0xa6 0x00 # CHECK: dmultu $5, $6
@@ -158,6 +160,7 @@
 0x00 0x00 0xc7 0x70 # CHECK: madd $6, $7
 0x60 0x98 0xf9 0x4f # CHECK: madd.s $f1, $f31, $f19, $f25
 0x01 0x00 0xc7 0x70 # CHECK: maddu $6, $7
+0x01 0x78 0x08 0x40 # CHECK: mfc0 $8, $15, 1
 0x00 0x38 0x06 0x44 # CHECK: mfc1 $6, $f7
 0x10 0x28 0x00 0x00 # CHECK: mfhi $5
 0x00 0xc0 0x7e 0x44 # CHECK: mfhc1 $fp, $f24
@@ -167,6 +170,7 @@
 0x04 0x00 0xc7 0x70 # CHECK: msub $6, $7
 0x28 0x53 0x70 0x4e # CHECK: msub.s $f12, $f19, $f10, $f16
 0x05 0x00 0xc7 0x70 # CHECK: msubu $6, $7
+0x01 0x78 0x89 0x40 # CHECK: mtc0 $9, $15, 1
 0x00 0x38 0x86 0x44 # CHECK: mtc1 $6, $f7
 0x11 0x00 0xe0 0x00 # CHECK: mthi $7
 0x00 0x80 0xe0 0x44 # CHECK: mthc1 $zero, $f16
diff --git a/test/MC/Disassembler/Mips/mips64r3/valid-mips64r3.txt b/test/MC/Disassembler/Mips/mips64r3/valid-mips64r3.txt
index 5a427df4bd7..82405f357bf 100644
--- a/test/MC/Disassembler/Mips/mips64r3/valid-mips64r3.txt
+++ b/test/MC/Disassembler/Mips/mips64r3/valid-mips64r3.txt
@@ -1,234 +1,240 @@
 # RUN: llvm-mc --disassemble %s -triple=mips64-unknown-linux -mcpu=mips64r3 | FileCheck %s
 # CHECK: .text
-0x46 0x20 0x73 0x05 # CHECK: abs.d $f12, $f14
-0x46 0x00 0x39 0x85 # CHECK: abs.s $f6, $f7
+0x00 0x00 0x00 0x00 # CHECK: nop
+0x00 0x00 0x01 0xcf # CHECK: sync 7
+0x00 0x00 0x04 0xb8 # CHECK: dsll $zero, $zero, 18
+0x00 0x00 0x04 0xbc # CHECK: dsll32 $zero, $zero, 18
+0x00 0x00 0x04 0xbc # CHECK: dsll32 $zero, $zero, 18
+0x00 0x00 0x28 0x10 # CHECK: mfhi $5
+0x00 0x00 0x28 0x12 # CHECK: mflo $5
+0x00 0x03 0x10 0x2e # CHECK: dneg $2, $3
+0x00 0x03 0x10 0x2f # CHECK: dnegu $2, $3
+0x00 0x03 0x21 0xc0 # CHECK: sll $4, $3, 7
+0x00 0x03 0x21 0xc2 # CHECK: srl $4, $3, 7
+0x00 0x03 0x21 0xc3 # CHECK: sra $4, $3, 7
+0x00 0x06 0x9d 0xfa # CHECK: dsrl $19, $6, 23
+0x00 0x06 0x9d 0xfe # CHECK: dsrl32 $19, $6, 23
+0x00 0x12 0xe2 0xbb # CHECK: dsra $gp, $18, 10
+0x00 0x12 0xe2 0xbf # CHECK: dsra32 $gp, $18, 10
+0x00 0x13 0x9d 0xfa # CHECK: dsrl $19, $19, 23
+0x00 0x13 0x9d 0xfe # CHECK: dsrl32 $19, $19, 23
+0x00 0x14 0x04 0xb8 # CHECK: dsll $zero, $20, 18
+0x00 0x1c 0xe2 0xbb # CHECK: dsra $gp, $gp, 10
+0x00 0x1c 0xe2 0xbf # CHECK: dsra32 $gp, $gp, 10
+0x00 0x21 0x0b 0xfa # CHECK: drotr $1, $1, 15
+0x00 0x21 0x0b 0xfe # CHECK: drotr32 $1, $1, 15
+0x00 0x26 0x49 0xc2 # CHECK: rotr $9, $6, 7
+0x00 0x2e 0x0b 0xfa # CHECK: drotr $1, $14, 15
+0x00 0x2e 0x0b 0xfe # CHECK: drotr32 $1, $14, 15
+0x00 0x3f 0x98 0x2c # CHECK: dadd $19, $1, $ra
+0x00 0x3f 0x98 0x2d # CHECK: daddu $19, $1, $ra
+0x00 0x65 0x00 0x18 # CHECK: mult $3, $5
+0x00 0x65 0x00 0x19 # CHECK: multu $3, $5
+0x00 0x65 0x18 0x25 # CHECK: or $3, $3, $5
+0x00 0x65 0x18 0x26 # CHECK: xor $3, $3, $5
+0x00 0x65 0x18 0x2a # CHECK: slt $3, $3, $5
+0x00 0x65 0x18 0x2b # CHECK: sltu $3, $3, $5
+0x00 0x65 0x20 0x23 # CHECK: subu $4, $3, $5
+0x00 0x80 0xfc 0x09 # CHECK: jalr.hb $4
+0x00 0xa0 0x24 0x09 # CHECK: jalr.hb $4, $5
+0x00 0xa3 0x10 0x04 # CHECK: sllv $2, $3, $5
+0x00 0xa3 0x10 0x06 # CHECK: srlv $2, $3, $5
+0x00 0xa3 0x10 0x07 # CHECK: srav $2, $3, $5
+0x00 0xa6 0x00 0x1d # CHECK: dmultu $5, $6
+0x00 0xba 0x28 0x2f # CHECK: dsubu $5, $5, $26
 0x00 0xc7 0x48 0x20 # CHECK: add $9, $6, $7
-0x46 0x2e 0x62 0x00 # CHECK: add.d $f8, $f12, $f14
-0x46 0x07 0x32 0x40 # CHECK: add.s $f9, $f6, $f7
-0x20 0xc9 0x45 0x67 # CHECK: addi $9, $6, 17767
-0x24 0xc9 0xc5 0x67 # CHECK: addiu $9, $6, -15001
 0x00 0xc7 0x48 0x21 # CHECK: addu $9, $6, $7
+0x00 0xc7 0x48 0x22 # CHECK: sub $9, $6, $7
 0x00 0xc7 0x48 0x24 # CHECK: and $9, $6, $7
-0x30 0xc9 0x45 0x67 # CHECK: andi $9, $6, 17767
+0x00 0xc7 0x48 0x27 # CHECK: nor $9, $6, $7
+0x00 0xe0 0x00 0x08 # CHECK: jr $7
+0x00 0xe0 0x00 0x11 # CHECK: mthi $7
+0x00 0xe0 0x00 0x13 # CHECK: mtlo $7
+0x00 0xe0 0xf8 0x09 # CHECK: jalr $7
+0x00 0xe6 0x48 0x46 # CHECK: rotrv $9, $6, $7
+0x01 0x94 0x00 0x14 # CHECK: dsllv $zero, $20, $12
+0x01 0x94 0x00 0x14 # CHECK: dsllv $zero, $20, $12
+0x01 0xee 0x08 0x56 # CHECK: drotrv $1, $14, $15
+0x02 0x11 0x00 0x1f # CHECK: ddivu $zero, $16, $17
+0x02 0x72 0xe0 0x17 # CHECK: dsrav $gp, $18, $19
+0x02 0x72 0xe0 0x17 # CHECK: dsrav $gp, $18, $19
+0x02 0x86 0x98 0x16 # CHECK: dsrlv $19, $6, $20
+0x02 0x86 0x98 0x16 # CHECK: dsrlv $19, $6, $20
+0x02 0xc8 0x38 0x2e # CHECK: dsub $7, $22, $8
+0x02 0xe9 0x00 0x1c # CHECK: dmult $23, $9
+0x03 0x53 0x00 0x1e # CHECK: ddiv $zero, $26, $19
+0x04 0xc1 0x01 0x4c # CHECK: bgez $6, 1332
+0x04 0xd1 0x01 0x4c # CHECK: bgezal $6, 1332
+0x08 0x00 0x01 0x4c # CHECK: j 1328
+0x0c 0x00 0x01 0x4c # CHECK: jal 1328
 0x10 0x00 0x01 0x4c # CHECK: b 1332
+0x11 0x26 0x01 0x4c # CHECK: beq $9, $6, 1332
+0x15 0x26 0x01 0x4c # CHECK: bne $9, $6, 1332
+0x18 0xc0 0x01 0x4c # CHECK: blez $6, 1332
+0x1c 0xc0 0x01 0x4c # CHECK: bgtz $6, 1332
+0x20 0xc9 0x45 0x67 # CHECK: addi $9, $6, 17767
+0x24 0xc9 0xc5 0x67 # CHECK: addiu $9, $6, -15001
+0x28 0x63 0x00 0x67 # CHECK: slti $3, $3, 103
+0x2c 0x63 0x00 0x67 # CHECK: sltiu $3, $3, 103
+0x30 0xc9 0x45 0x67 # CHECK: andi $9, $6, 17767
+0x34 0xc9 0x45 0x67 # CHECK: ori $9, $6, 17767
+0x38 0xc9 0x45 0x67 # CHECK: xori $9, $6, 17767
+0x3c 0x06 0x45 0x67 # CHECK: lui $6, 17767
+0x40 0x08 0x78 0x01 # CHECK: mfc0 $8, $15, 1
+0x40 0x38 0x50 0x00 # CHECK: dmfc0 $24, $10, 0
+0x40 0x89 0x78 0x01 # CHECK: mtc0 $9, $15, 1
+0x40 0xa4 0x50 0x00 # CHECK: dmtc0 $4, $10, 0
+0x41 0x60 0x60 0x00 # CHECK: di
+0x41 0x60 0x60 0x20 # CHECK: ei
+0x41 0x6e 0x60 0x20 # CHECK: ei $14
+0x41 0x7e 0x60 0x00 # CHECK: di $fp
+0x44 0x06 0x38 0x00 # CHECK: mfc1 $6, $f7
+0x44 0x2c 0x68 0x00 # CHECK: dmfc1 $12, $f13
+0x44 0x46 0x38 0x00 # CHECK: cfc1 $6, $7
+0x44 0x7e 0xc0 0x00 # CHECK: mfhc1 $fp, $f24
+0x44 0x86 0x38 0x00 # CHECK: mtc1 $6, $f7
+0x44 0xb0 0x70 0x00 # CHECK: dmtc1 $16, $f14
+0x44 0xc6 0x38 0x00 # CHECK: ctc1 $6, $7
+0x44 0xe0 0x80 0x00 # CHECK: mthc1 $zero, $f16
 0x45 0x00 0x01 0x4c # CHECK: bc1f 1332
-0x45 0x1c 0x01 0x4c # CHECK: bc1f $fcc7, 1332
 0x45 0x01 0x01 0x4c # CHECK: bc1t 1332
+0x45 0x1c 0x01 0x4c # CHECK: bc1f $fcc7, 1332
 0x45 0x1d 0x01 0x4c # CHECK: bc1t $fcc7, 1332
-0x11 0x26 0x01 0x4c # CHECK: beq $9, $6, 1332
-0x04 0xc1 0x01 0x4c # CHECK: bgez $6, 1332
-0x04 0xd1 0x01 0x4c # CHECK: bgezal $6, 1332
-0x1c 0xc0 0x01 0x4c # CHECK: bgtz $6, 1332
-0x18 0xc0 0x01 0x4c # CHECK: blez $6, 1332
-0x15 0x26 0x01 0x4c # CHECK: bne $9, $6, 1332
-0x46 0x2e 0x60 0x32 # CHECK: c.eq.d $f12, $f14
-0x46 0x07 0x30 0x32 # CHECK: c.eq.s $f6, $f7
-0x46 0x2e 0x60 0x30 # CHECK: c.f.d $f12, $f14
+0x46 0x00 0x2b 0x0b # CHECK: floor.l.s $f12, $f5
+0x46 0x00 0x2e 0x48 # CHECK: round.l.s $f25, $f5
+0x46 0x00 0x39 0x84 # CHECK: sqrt.s $f6, $f7
+0x46 0x00 0x39 0x85 # CHECK: abs.s $f6, $f7
+0x46 0x00 0x39 0x86 # CHECK: mov.s $f6, $f7
+0x46 0x00 0x39 0x87 # CHECK: neg.s $f6, $f7
+0x46 0x00 0x39 0x8c # CHECK: round.w.s $f6, $f7
+0x46 0x00 0x39 0x8d # CHECK: trunc.w.s $f6, $f7
+0x46 0x00 0x39 0x8e # CHECK: ceil.w.s $f6, $f7
+0x46 0x00 0x39 0x8f # CHECK: floor.w.s $f6, $f7
+0x46 0x00 0x39 0xa1 # CHECK: cvt.d.s $f6, $f7
+0x46 0x00 0x39 0xa4 # CHECK: cvt.w.s $f6, $f7
+0x46 0x00 0x39 0xa5 # CHECK: cvt.l.s $f6, $f7
+0x46 0x00 0x6c 0x8a # CHECK: ceil.l.s $f18, $f13
+0x46 0x00 0xff 0x09 # CHECK: trunc.l.s $f28, $f31
 0x46 0x07 0x30 0x30 # CHECK: c.f.s $f6, $f7
-0x46 0x2e 0x60 0x3e # CHECK: c.le.d $f12, $f14
-0x46 0x07 0x30 0x3e # CHECK: c.le.s $f6, $f7
-0x46 0x2e 0x60 0x3c # CHECK: c.lt.d $f12, $f14
+0x46 0x07 0x30 0x31 # CHECK: c.un.s $f6, $f7
+0x46 0x07 0x30 0x32 # CHECK: c.eq.s $f6, $f7
+0x46 0x07 0x30 0x34 # CHECK: c.olt.s $f6, $f7
+0x46 0x07 0x30 0x35 # CHECK: c.ult.s $f6, $f7
+0x46 0x07 0x30 0x36 # CHECK: c.ole.s $f6, $f7
+0x46 0x07 0x30 0x37 # CHECK: c.ule.s $f6, $f7
+0x46 0x07 0x30 0x38 # CHECK: c.sf.s $f6, $f7
+0x46 0x07 0x30 0x39 # CHECK: c.ngle.s $f6, $f7
+0x46 0x07 0x30 0x3a # CHECK: c.seq.s $f6, $f7
+0x46 0x07 0x30 0x3b # CHECK: c.ngl.s $f6, $f7
 0x46 0x07 0x30 0x3c # CHECK: c.lt.s $f6, $f7
-0x46 0x2e 0x60 0x3d # CHECK: c.nge.d $f12, $f14
 0x46 0x07 0x30 0x3d # CHECK: c.nge.s $f6, $f7
-0x46 0x2e 0x60 0x3b # CHECK: c.ngl.d $f12, $f14
-0x46 0x07 0x30 0x3b # CHECK: c.ngl.s $f6, $f7
-0x46 0x2e 0x60 0x39 # CHECK: c.ngle.d $f12, $f14
-0x46 0x07 0x30 0x39 # CHECK: c.ngle.s $f6, $f7
-0x46 0x2e 0x60 0x3f # CHECK: c.ngt.d $f12, $f14
+0x46 0x07 0x30 0x3e # CHECK: c.le.s $f6, $f7
 0x46 0x07 0x30 0x3f # CHECK: c.ngt.s $f6, $f7
-0x46 0x2e 0x60 0x36 # CHECK: c.ole.d $f12, $f14
-0x46 0x07 0x30 0x36 # CHECK: c.ole.s $f6, $f7
-0x46 0x2e 0x60 0x34 # CHECK: c.olt.d $f12, $f14
-0x46 0x07 0x30 0x34 # CHECK: c.olt.s $f6, $f7
-0x46 0x2e 0x60 0x3a # CHECK: c.seq.d $f12, $f14
-0x46 0x07 0x30 0x3a # CHECK: c.seq.s $f6, $f7
-0x46 0x2e 0x60 0x38 # CHECK: c.sf.d $f12, $f14
-0x46 0x07 0x30 0x38 # CHECK: c.sf.s $f6, $f7
-0x46 0x2e 0x60 0x33 # CHECK: c.ueq.d $f12, $f14
+0x46 0x07 0x32 0x40 # CHECK: add.s $f9, $f6, $f7
+0x46 0x07 0x32 0x41 # CHECK: sub.s $f9, $f6, $f7
+0x46 0x07 0x32 0x42 # CHECK: mul.s $f9, $f6, $f7
 0x46 0x12 0xe0 0x33 # CHECK: c.ueq.s $f28, $f18
-0x46 0x2e 0x60 0x37 # CHECK: c.ule.d $f12, $f14
-0x46 0x07 0x30 0x37 # CHECK: c.ule.s $f6, $f7
-0x46 0x2e 0x60 0x35 # CHECK: c.ult.d $f12, $f14
-0x46 0x07 0x30 0x35 # CHECK: c.ult.s $f6, $f7
-0x46 0x2e 0x60 0x31 # CHECK: c.un.d $f12, $f14
-0x46 0x07 0x30 0x31 # CHECK: c.un.s $f6, $f7
+0x46 0x20 0x0b 0x08 # CHECK: round.l.d $f12, $f1
 0x46 0x20 0x18 0x4a # CHECK: ceil.l.d $f1, $f3
-0x46 0x00 0x6c 0x8a # CHECK: ceil.l.s $f18, $f13
+0x46 0x20 0x3e 0x8b # CHECK: floor.l.d $f26, $f7
+0x46 0x20 0x41 0x86 # CHECK: mov.d $f6, $f8
+0x46 0x20 0x73 0x04 # CHECK: sqrt.d $f12, $f14
+0x46 0x20 0x73 0x05 # CHECK: abs.d $f12, $f14
+0x46 0x20 0x73 0x07 # CHECK: neg.d $f12, $f14
+0x46 0x20 0x73 0x0c # CHECK: round.w.d $f12, $f14
+0x46 0x20 0x73 0x0d # CHECK: trunc.w.d $f12, $f14
 0x46 0x20 0x73 0x0e # CHECK: ceil.w.d $f12, $f14
-0x46 0x00 0x39 0x8e # CHECK: ceil.w.s $f6, $f7
-0x44 0x46 0x38 0x00 # CHECK: cfc1 $6, $7
-0x70 0xe6 0x30 0x21 # CHECK: clo $6, $7
-0x70 0xe6 0x30 0x20 # CHECK: clz $6, $7
-0x44 0xc6 0x38 0x00 # CHECK: ctc1 $6, $7
-0x46 0xa0 0x81 0x21 # CHECK: cvt.d.l $f4, $f16
-0x46 0x00 0x39 0xa1 # CHECK: cvt.d.s $f6, $f7
-0x46 0x80 0x73 0x21 # CHECK: cvt.d.w $f12, $f14
-0x46 0x20 0x73 0x25 # CHECK: cvt.l.d $f12, $f14
-0x46 0x00 0x39 0xa5 # CHECK: cvt.l.s $f6, $f7
-0x46 0xa0 0xf3 0xe0 # CHECK: cvt.s.l $f15, $f30
+0x46 0x20 0x73 0x0f # CHECK: floor.w.d $f12, $f14
 0x46 0x20 0x73 0x20 # CHECK: cvt.s.d $f12, $f14
-0x46 0x80 0x39 0xa0 # CHECK: cvt.s.w $f6, $f7
 0x46 0x20 0x73 0x24 # CHECK: cvt.w.d $f12, $f14
-0x46 0x00 0x39 0xa4 # CHECK: cvt.w.s $f6, $f7
-0x00 0x3f 0x98 0x2c # CHECK: dadd $19, $1, $ra
+0x46 0x20 0x73 0x25 # CHECK: cvt.l.d $f12, $f14
+0x46 0x20 0xbd 0xc9 # CHECK: trunc.l.d $f23, $f23
+0x46 0x2e 0x60 0x30 # CHECK: c.f.d $f12, $f14
+0x46 0x2e 0x60 0x31 # CHECK: c.un.d $f12, $f14
+0x46 0x2e 0x60 0x32 # CHECK: c.eq.d $f12, $f14
+0x46 0x2e 0x60 0x33 # CHECK: c.ueq.d $f12, $f14
+0x46 0x2e 0x60 0x34 # CHECK: c.olt.d $f12, $f14
+0x46 0x2e 0x60 0x35 # CHECK: c.ult.d $f12, $f14
+0x46 0x2e 0x60 0x36 # CHECK: c.ole.d $f12, $f14
+0x46 0x2e 0x60 0x37 # CHECK: c.ule.d $f12, $f14
+0x46 0x2e 0x60 0x38 # CHECK: c.sf.d $f12, $f14
+0x46 0x2e 0x60 0x39 # CHECK: c.ngle.d $f12, $f14
+0x46 0x2e 0x60 0x3a # CHECK: c.seq.d $f12, $f14
+0x46 0x2e 0x60 0x3b # CHECK: c.ngl.d $f12, $f14
+0x46 0x2e 0x60 0x3c # CHECK: c.lt.d $f12, $f14
+0x46 0x2e 0x60 0x3d # CHECK: c.nge.d $f12, $f14
+0x46 0x2e 0x60 0x3e # CHECK: c.le.d $f12, $f14
+0x46 0x2e 0x60 0x3f # CHECK: c.ngt.d $f12, $f14
+0x46 0x2e 0x62 0x00 # CHECK: add.d $f8, $f12, $f14
+0x46 0x2e 0x62 0x01 # CHECK: sub.d $f8, $f12, $f14
+0x46 0x2e 0x62 0x02 # CHECK: mul.d $f8, $f12, $f14
+0x46 0x80 0x39 0xa0 # CHECK: cvt.s.w $f6, $f7
+0x46 0x80 0x73 0x21 # CHECK: cvt.d.w $f12, $f14
+0x46 0xa0 0x81 0x21 # CHECK: cvt.d.l $f4, $f16
+0x46 0xa0 0xf3 0xe0 # CHECK: cvt.s.l $f15, $f30
+0x4c 0xa6 0x00 0x05 # CHECK: luxc1 $f0, $6($5)
+0x4c 0xac 0xc8 0x30 # CHECK: nmadd.s $f0, $f5, $f25, $f12
+0x4c 0xb8 0x20 0x0d # CHECK: suxc1 $f4, $24($5)
+0x4d 0xcc 0x05 0x00 # CHECK: lwxc1 $f20, $12($14)
+0x4d 0xf7 0x02 0x01 # CHECK: ldxc1 $f8, $23($15)
+0x4e 0x70 0x53 0x28 # CHECK: msub.s $f12, $f19, $f10, $f16
+0x4e 0xd2 0xd0 0x08 # CHECK: swxc1 $f26, $18($22)
+0x4f 0x04 0x98 0x78 # CHECK: nmsub.s $f1, $f24, $f19, $f4
+0x4f 0x24 0x40 0x09 # CHECK: sdxc1 $f8, $4($25)
+0x4f 0xf9 0x98 0x60 # CHECK: madd.s $f1, $f31, $f19, $f25
+0x62 0x9d 0x6c 0x39 # CHECK: daddi $sp, $20, 27705
 0x62 0x9d 0x93 0xc7 # CHECK: daddi $sp, $20, -27705
-0x63 0xbd 0x93 0xc7 # CHECK: daddi $sp, $sp, -27705
 0x62 0x9d 0x93 0xc7 # CHECK: daddi $sp, $20, -27705
 0x62 0x9d 0x93 0xc7 # CHECK: daddi $sp, $20, -27705
+0x63 0xbd 0x6c 0x39 # CHECK: daddi $sp, $sp, 27705
+0x63 0xbd 0x93 0xc7 # CHECK: daddi $sp, $sp, -27705
 0x63 0xbd 0x93 0xc7 # CHECK: daddi $sp, $sp, -27705
-0x66 0xda 0xee 0x16 # CHECK: daddiu $26, $22, -4586
-0x00 0x3f 0x98 0x2d # CHECK: daddu $19, $1, $ra
 0x64 0x58 0x46 0x9f # CHECK: daddiu $24, $2, 18079
+0x65 0x6f 0xec 0x5f # CHECK: daddiu $15, $11, -5025
+0x65 0xce 0x11 0xea # CHECK: daddiu $14, $14, 4586
 0x66 0x73 0x69 0x3f # CHECK: daddiu $19, $19, 26943
+0x66 0xda 0xee 0x16 # CHECK: daddiu $26, $22, -4586
+0x6b 0x18 0xef 0xb9 # CHECK: ldl $24, -4167($24)
+0x6e 0x8e 0x89 0x6a # CHECK: ldr $14, -30358($20)
+0x70 0xc7 0x00 0x00 # CHECK: madd $6, $7
+0x70 0xc7 0x00 0x01 # CHECK: maddu $6, $7
+0x70 0xc7 0x00 0x04 # CHECK: msub $6, $7
+0x70 0xc7 0x00 0x05 # CHECK: msubu $6, $7
+0x70 0xc7 0x48 0x02 # CHECK: mul $9, $6, $7
 0x70 0xd2 0x90 0x25 # CHECK: dclo $18, $6
+0x70 0xe6 0x30 0x20 # CHECK: clz $6, $7
+0x70 0xe6 0x30 0x21 # CHECK: clo $6, $7
 0x73 0x30 0x80 0x24 # CHECK: dclz $16, $25
-0x03 0x53 0x00 0x1e # CHECK: ddiv $zero, $26, $19
-0x02 0x11 0x00 0x1f # CHECK: ddivu $zero, $16, $17
-0x44 0x2c 0x68 0x00 # CHECK: dmfc1 $12, $f13
-0x44 0xb0 0x70 0x00 # CHECK: dmtc1 $16, $f14
-0x02 0xe9 0x00 0x1c # CHECK: dmult $23, $9
-0x00 0xa6 0x00 0x1d # CHECK: dmultu $5, $6
-0x00 0x00 0x04 0xb8 # CHECK: dsll $zero, $zero, 18
-0x00 0x14 0x04 0xb8 # CHECK: dsll $zero, $20, 18
-0x01 0x94 0x00 0x14 # CHECK: dsllv $zero, $20, $12
-0x00 0x00 0x04 0xbc # CHECK: dsll32 $zero, $zero, 18
-0x00 0x00 0x04 0xbc # CHECK: dsll32 $zero, $zero, 18
-0x01 0x94 0x00 0x14 # CHECK: dsllv $zero, $20, $12
-0x00 0x1c 0xe2 0xbb # CHECK: dsra $gp, $gp, 10
-0x00 0x12 0xe2 0xbb # CHECK: dsra $gp, $18, 10
-0x02 0x72 0xe0 0x17 # CHECK: dsrav $gp, $18, $19
-0x00 0x1c 0xe2 0xbf # CHECK: dsra32 $gp, $gp, 10
-0x00 0x12 0xe2 0xbf # CHECK: dsra32 $gp, $18, 10
-0x02 0x72 0xe0 0x17 # CHECK: dsrav $gp, $18, $19
-0x00 0x13 0x9d 0xfa # CHECK: dsrl $19, $19, 23
-0x00 0x06 0x9d 0xfa # CHECK: dsrl $19, $6, 23
-0x02 0x86 0x98 0x16 # CHECK: dsrlv $19, $6, $20
-0x00 0x13 0x9d 0xfe # CHECK: dsrl32 $19, $19, 23
-0x00 0x06 0x9d 0xfe # CHECK: dsrl32 $19, $6, 23
-0x02 0x86 0x98 0x16 # CHECK: dsrlv $19, $6, $20
-0x02 0xc8 0x38 0x2e # CHECK: dsub $7, $22, $8
+0x74 0x00 0x01 0x4c # CHECK: jalx 1328
+0x7c 0x07 0x30 0xa0 # CHECK: wsbh $6, $7
+0x7c 0x07 0x34 0x20 # CHECK: seb $6, $7
+0x7c 0x07 0x36 0x20 # CHECK: seh $6, $7
 0x7c 0x0e 0x18 0xa4 # CHECK: dsbh $3, $14
 0x7c 0x1d 0x11 0x64 # CHECK: dshd $2, $sp
-0x62 0x9d 0x6c 0x39 # CHECK: daddi $sp, $20, 27705
-0x63 0xbd 0x6c 0x39 # CHECK: daddi $sp, $sp, 27705
-0x00 0xba 0x28 0x2f # CHECK: dsubu $5, $5, $26
-0x65 0x6f 0xec 0x5f # CHECK: daddiu $15, $11, -5025
-0x65 0xce 0x11 0xea # CHECK: daddiu $14, $14, 4586
-0x41 0x7e 0x60 0x00 # CHECK: di $fp
-0x41 0x60 0x60 0x00 # CHECK: di
-0x00 0x21 0x0b 0xfa # CHECK: drotr $1, $1, 15
-0x00 0x2e 0x0b 0xfa # CHECK: drotr $1, $14, 15
-0x00 0x21 0x0b 0xfe # CHECK: drotr32 $1, $1, 15
-0x00 0x2e 0x0b 0xfe # CHECK: drotr32 $1, $14, 15
-0x01 0xee 0x08 0x56 # CHECK: drotrv $1, $14, $15
-0x41 0x6e 0x60 0x20 # CHECK: ei $14
-0x41 0x60 0x60 0x20 # CHECK: ei
-0x46 0x20 0x3e 0x8b # CHECK: floor.l.d $f26, $f7
-0x46 0x00 0x2b 0x0b # CHECK: floor.l.s $f12, $f5
-0x46 0x20 0x73 0x0f # CHECK: floor.w.d $f12, $f14
-0x46 0x00 0x39 0x8f # CHECK: floor.w.s $f6, $f7
 0x7d 0x33 0x61 0x84 # CHECK: ins $19, $9, 6, 7
-0x08 0x00 0x01 0x4c # CHECK: j 1328
-0x0c 0x00 0x01 0x4c # CHECK: jal 1328
-0x74 0x00 0x01 0x4c # CHECK: jalx 1328
-0x00 0xe0 0xf8 0x09 # CHECK: jalr $7
-0x00 0x80 0xfc 0x09 # CHECK: jalr.hb $4
-0x00 0xa0 0x24 0x09 # CHECK: jalr.hb $4, $5
-0x00 0xe0 0x00 0x08 # CHECK: jr $7
 0x80 0xa4 0x23 0xc6 # CHECK: lb $4, 9158($5)
-0x90 0xa4 0x00 0x06 # CHECK: lbu $4, 6($5)
-0xde 0x3d 0x90 0x1b # CHECK: ld $sp, -28645($17)
-0x6b 0x18 0xef 0xb9 # CHECK: ldl $24, -4167($24)
-0x6e 0x8e 0x89 0x6a # CHECK: ldr $14, -30358($20)
-0xd4 0xe9 0x23 0xc6 # CHECK: ldc1 $f9, 9158($7)
-0x4d 0xf7 0x02 0x01 # CHECK: ldxc1 $f8, $23($15)
 0x84 0xa4 0x00 0x0c # CHECK: lh $4, 12($5)
 0x84 0xa4 0x00 0x0c # CHECK: lh $4, 12($5)
-0xc0 0xe9 0x23 0xc6 # CHECK: ll $9, 9158($7)
-0xd3 0xe0 0xc6 0x70 # CHECK: lld $zero, -14736($ra)
-0x3c 0x06 0x45 0x67 # CHECK: lui $6, 17767
-0x4c 0xa6 0x00 0x05 # CHECK: luxc1 $f0, $6($5)
-0x8c 0xa4 0x00 0x18 # CHECK: lw $4, 24($5)
-0xc4 0xe9 0x23 0xc6 # CHECK: lwc1 $f9, 9158($7)
 0x88 0x82 0x00 0x03 # CHECK: lwl $2, 3($4)
+0x8c 0xa4 0x00 0x18 # CHECK: lw $4, 24($5)
+0x90 0xa4 0x00 0x06 # CHECK: lbu $4, 6($5)
 0x98 0xa3 0x00 0x10 # CHECK: lwr $3, 16($5)
-0x4d 0xcc 0x05 0x00 # CHECK: lwxc1 $f20, $12($14)
 0x9c 0x73 0xa1 0xea # CHECK: lwu $19, -24086($3)
-0x70 0xc7 0x00 0x00 # CHECK: madd $6, $7
-0x4f 0xf9 0x98 0x60 # CHECK: madd.s $f1, $f31, $f19, $f25
-0x70 0xc7 0x00 0x01 # CHECK: maddu $6, $7
-0x44 0x06 0x38 0x00 # CHECK: mfc1 $6, $f7
-0x00 0x00 0x28 0x10 # CHECK: mfhi $5
-0x44 0x7e 0xc0 0x00 # CHECK: mfhc1 $fp, $f24
-0x00 0x00 0x28 0x12 # CHECK: mflo $5
-0x46 0x20 0x41 0x86 # CHECK: mov.d $f6, $f8
-0x46 0x00 0x39 0x86 # CHECK: mov.s $f6, $f7
-0x70 0xc7 0x00 0x04 # CHECK: msub $6, $7
-0x4e 0x70 0x53 0x28 # CHECK: msub.s $f12, $f19, $f10, $f16
-0x70 0xc7 0x00 0x05 # CHECK: msubu $6, $7
-0x44 0x86 0x38 0x00 # CHECK: mtc1 $6, $f7
-0x00 0xe0 0x00 0x11 # CHECK: mthi $7
-0x44 0xe0 0x80 0x00 # CHECK: mthc1 $zero, $f16
-0x00 0xe0 0x00 0x13 # CHECK: mtlo $7
-0x46 0x2e 0x62 0x02 # CHECK: mul.d $f8, $f12, $f14
-0x46 0x07 0x32 0x42 # CHECK: mul.s $f9, $f6, $f7
-0x70 0xc7 0x48 0x02 # CHECK: mul $9, $6, $7
-0x00 0x65 0x00 0x18 # CHECK: mult $3, $5
-0x00 0x65 0x00 0x19 # CHECK: multu $3, $5
-0x46 0x20 0x73 0x07 # CHECK: neg.d $f12, $f14
-0x46 0x00 0x39 0x87 # CHECK: neg.s $f6, $f7
-0x4c 0xac 0xc8 0x30 # CHECK: nmadd.s $f0, $f5, $f25, $f12
-0x00 0x00 0x00 0x00 # CHECK: nop
-0x00 0xc7 0x48 0x27 # CHECK: nor $9, $6, $7
-0x4f 0x04 0x98 0x78 # CHECK: nmsub.s $f1, $f24, $f19, $f4
-0x00 0x65 0x18 0x25 # CHECK: or $3, $3, $5
-0x34 0xc9 0x45 0x67 # CHECK: ori $9, $6, 17767
-0x00 0x26 0x49 0xc2 # CHECK: rotr $9, $6, 7
-0x00 0xe6 0x48 0x46 # CHECK: rotrv $9, $6, $7
-0x46 0x20 0x0b 0x08 # CHECK: round.l.d $f12, $f1
-0x46 0x00 0x2e 0x48 # CHECK: round.l.s $f25, $f5
-0x46 0x20 0x73 0x0c # CHECK: round.w.d $f12, $f14
-0x46 0x00 0x39 0x8c # CHECK: round.w.s $f6, $f7
-0xa0 0xa4 0x23 0xc6 # CHECK: sb $4, 9158($5)
 0xa0 0xa4 0x00 0x06 # CHECK: sb $4, 6($5)
-0xe0 0xe9 0x23 0xc6 # CHECK: sc $9, 9158($7)
-0xf3 0xaf 0xdf 0xcd # CHECK: scd $15, -8243($sp)
-0xfd 0x4c 0x16 0xcb # CHECK: sd $12, 5835($10)
-0xf4 0xe9 0x23 0xc6 # CHECK: sdc1 $f9, 9158($7)
-0xb3 0xc7 0xae 0x1f # CHECK: sdl $7, -20961($fp)
-0xb5 0x8b 0xb0 0x39 # CHECK: sdr $11, -20423($12)
-0x4f 0x24 0x40 0x09 # CHECK: sdxc1 $f8, $4($25)
-0x7c 0x07 0x34 0x20 # CHECK: seb $6, $7
-0x7c 0x07 0x36 0x20 # CHECK: seh $6, $7
+0xa0 0xa4 0x23 0xc6 # CHECK: sb $4, 9158($5)
 0xa4 0xa4 0x23 0xc6 # CHECK: sh $4, 9158($5)
-0x00 0x03 0x21 0xc0 # CHECK: sll $4, $3, 7
-0x00 0xa3 0x10 0x04 # CHECK: sllv $2, $3, $5
-0x00 0x65 0x18 0x2a # CHECK: slt $3, $3, $5
-0x28 0x63 0x00 0x67 # CHECK: slti $3, $3, 103
-0x2c 0x63 0x00 0x67 # CHECK: sltiu $3, $3, 103
-0x00 0x65 0x18 0x2b # CHECK: sltu $3, $3, $5
-0x46 0x20 0x73 0x04 # CHECK: sqrt.d $f12, $f14
-0x46 0x00 0x39 0x84 # CHECK: sqrt.s $f6, $f7
-0x00 0x03 0x21 0xc3 # CHECK: sra $4, $3, 7
-0x00 0xa3 0x10 0x07 # CHECK: srav $2, $3, $5
-0x00 0x03 0x21 0xc2 # CHECK: srl $4, $3, 7
-0x00 0xa3 0x10 0x06 # CHECK: srlv $2, $3, $5
-0x46 0x2e 0x62 0x01 # CHECK: sub.d $f8, $f12, $f14
-0x46 0x07 0x32 0x41 # CHECK: sub.s $f9, $f6, $f7
-0x00 0xc7 0x48 0x22 # CHECK: sub $9, $6, $7
-0x00 0x65 0x20 0x23 # CHECK: subu $4, $3, $5
-0x4c 0xb8 0x20 0x0d # CHECK: suxc1 $f4, $24($5)
-0xac 0xa4 0x00 0x18 # CHECK: sw $4, 24($5)
-0xe4 0xe9 0x23 0xc6 # CHECK: swc1 $f9, 9158($7)
 0xa8 0xa4 0x00 0x10 # CHECK: swl $4, 16($5)
+0xac 0xa4 0x00 0x18 # CHECK: sw $4, 24($5)
+0xb3 0xc7 0xae 0x1f # CHECK: sdl $7, -20961($fp)
+0xb5 0x8b 0xb0 0x39 # CHECK: sdr $11, -20423($12)
 0xb8 0xe6 0x00 0x10 # CHECK: swr $6, 16($7)
-0x4e 0xd2 0xd0 0x08 # CHECK: swxc1 $f26, $18($22)
-0x00 0x00 0x01 0xcf # CHECK: sync 7
-0x46 0x20 0xbd 0xc9 # CHECK: trunc.l.d $f23, $f23
-0x46 0x00 0xff 0x09 # CHECK: trunc.l.s $f28, $f31
-0x46 0x20 0x73 0x0d # CHECK: trunc.w.d $f12, $f14
-0x46 0x00 0x39 0x8d # CHECK: trunc.w.s $f6, $f7
-0x7c 0x07 0x30 0xa0 # CHECK: wsbh $6, $7
-0x00 0x65 0x18 0x26 # CHECK: xor $3, $3, $5
-0x38 0xc9 0x45 0x67 # CHECK: xori $9, $6, 17767
+0xc0 0xe9 0x23 0xc6 # CHECK: ll $9, 9158($7)
+0xc4 0xe9 0x23 0xc6 # CHECK: lwc1 $f9, 9158($7)
+0xd3 0xe0 0xc6 0x70 # CHECK: lld $zero, -14736($ra)
+0xd4 0xe9 0x23 0xc6 # CHECK: ldc1 $f9, 9158($7)
+0xde 0x3d 0x90 0x1b # CHECK: ld $sp, -28645($17)
+0xe0 0xe9 0x23 0xc6 # CHECK: sc $9, 9158($7)
+0xe4 0xe9 0x23 0xc6 # CHECK: swc1 $f9, 9158($7)
+0xf3 0xaf 0xdf 0xcd # CHECK: scd $15, -8243($sp)
+0xf4 0xe9 0x23 0xc6 # CHECK: sdc1 $f9, 9158($7)
+0xfd 0x4c 0x16 0xcb # CHECK: sd $12, 5835($10)
diff --git a/test/MC/Disassembler/Mips/mips64r5/valid-mips64r5-el.txt b/test/MC/Disassembler/Mips/mips64r5/valid-mips64r5-el.txt
index 3d97a2b30ce..bd709d22879 100644
--- a/test/MC/Disassembler/Mips/mips64r5/valid-mips64r5-el.txt
+++ b/test/MC/Disassembler/Mips/mips64r5/valid-mips64r5-el.txt
@@ -85,7 +85,9 @@
 0x24 0x80 0x30 0x73 # CHECK: dclz $16, $25
 0x1e 0x00 0x53 0x03 # CHECK: ddiv $zero, $26, $19
 0x1f 0x00 0x11 0x02 # CHECK: ddivu $zero, $16, $17
+0x00 0x50 0x38 0x40 # CHECK: dmfc0 $24, $10, 0
 0x00 0x68 0x2c 0x44 # CHECK: dmfc1 $12, $f13
+0x00 0x50 0xa4 0x40 # CHECK: dmtc0 $4, $10, 0
 0x00 0x70 0xb0 0x44 # CHECK: dmtc1 $16, $f14
 0x1c 0x00 0xe9 0x02 # CHECK: dmult $23, $9
 0x1d 0x00 0xa6 0x00 # CHECK: dmultu $5, $6
@@ -158,6 +160,7 @@
 0x00 0x00 0xc7 0x70 # CHECK: madd $6, $7
 0x60 0x98 0xf9 0x4f # CHECK: madd.s $f1, $f31, $f19, $f25
 0x01 0x00 0xc7 0x70 # CHECK: maddu $6, $7
+0x01 0x78 0x08 0x40 # CHECK: mfc0 $8, $15, 1
 0x00 0x38 0x06 0x44 # CHECK: mfc1 $6, $f7
 0x10 0x28 0x00 0x00 # CHECK: mfhi $5
 0x00 0xc0 0x7e 0x44 # CHECK: mfhc1 $fp, $f24
@@ -167,6 +170,7 @@
 0x04 0x00 0xc7 0x70 # CHECK: msub $6, $7
 0x28 0x53 0x70 0x4e # CHECK: msub.s $f12, $f19, $f10, $f16
 0x05 0x00 0xc7 0x70 # CHECK: msubu $6, $7
+0x01 0x78 0x89 0x40 # CHECK: mtc0 $9, $15, 1
 0x00 0x38 0x86 0x44 # CHECK: mtc1 $6, $f7
 0x11 0x00 0xe0 0x00 # CHECK: mthi $7
 0x00 0x80 0xe0 0x44 # CHECK: mthc1 $zero, $f16
diff --git a/test/MC/Disassembler/Mips/mips64r5/valid-mips64r5.txt b/test/MC/Disassembler/Mips/mips64r5/valid-mips64r5.txt
index 1f52bd14185..a27fa98c963 100644
--- a/test/MC/Disassembler/Mips/mips64r5/valid-mips64r5.txt
+++ b/test/MC/Disassembler/Mips/mips64r5/valid-mips64r5.txt
@@ -1,234 +1,241 @@
 # RUN: llvm-mc --disassemble %s -triple=mips64-unknown-linux -mcpu=mips64r5 | FileCheck %s
 # CHECK: .text
-0x46 0x20 0x73 0x05 # CHECK: abs.d $f12, $f14
-0x46 0x00 0x39 0x85 # CHECK: abs.s $f6, $f7
+0x00 0x00 0x00 0x00 # CHECK: nop
+0x00 0x00 0x01 0xcf # CHECK: sync 7
+0x00 0x00 0x04 0xb8 # CHECK: dsll $zero, $zero, 18
+0x00 0x00 0x04 0xbc # CHECK: dsll32 $zero, $zero, 18
+0x00 0x00 0x04 0xbc # CHECK: dsll32 $zero, $zero, 18
+0x00 0x00 0x28 0x10 # CHECK: mfhi $5
+0x00 0x00 0x28 0x12 # CHECK: mflo $5
+0x00 0x03 0x10 0x2e # CHECK: dneg $2, $3
+0x00 0x03 0x10 0x2f # CHECK: dnegu $2, $3
+0x00 0x03 0x21 0xc0 # CHECK: sll $4, $3, 7
+0x00 0x03 0x21 0xc2 # CHECK: srl $4, $3, 7
+0x00 0x03 0x21 0xc3 # CHECK: sra $4, $3, 7
+0x00 0x06 0x9d 0xfa # CHECK: dsrl $19, $6, 23
+0x00 0x06 0x9d 0xfe # CHECK: dsrl32 $19, $6, 23
+0x00 0x12 0xe2 0xbb # CHECK: dsra $gp, $18, 10
+0x00 0x12 0xe2 0xbf # CHECK: dsra32 $gp, $18, 10
+0x00 0x13 0x9d 0xfa # CHECK: dsrl $19, $19, 23
+0x00 0x13 0x9d 0xfe # CHECK: dsrl32 $19, $19, 23
+0x00 0x14 0x04 0xb8 # CHECK: dsll $zero, $20, 18
+0x00 0x1c 0xe2 0xbb # CHECK: dsra $gp, $gp, 10
+0x00 0x1c 0xe2 0xbf # CHECK: dsra32 $gp, $gp, 10
+0x00 0x21 0x0b 0xfa # CHECK: drotr $1, $1, 15
+0x00 0x21 0x0b 0xfe # CHECK: drotr32 $1, $1, 15
+0x00 0x26 0x49 0xc2 # CHECK: rotr $9, $6, 7
+0x00 0x2e 0x0b 0xfa # CHECK: drotr $1, $14, 15
+0x00 0x2e 0x0b 0xfe # CHECK: drotr32 $1, $14, 15
+0x00 0x3f 0x98 0x2c # CHECK: dadd $19, $1, $ra
+0x00 0x3f 0x98 0x2d # CHECK: daddu $19, $1, $ra
+0x00 0x65 0x00 0x18 # CHECK: mult $3, $5
+0x00 0x65 0x00 0x19 # CHECK: multu $3, $5
+0x00 0x65 0x18 0x25 # CHECK: or $3, $3, $5
+0x00 0x65 0x18 0x26 # CHECK: xor $3, $3, $5
+0x00 0x65 0x18 0x2a # CHECK: slt $3, $3, $5
+0x00 0x65 0x18 0x2b # CHECK: sltu $3, $3, $5
+0x00 0x65 0x20 0x23 # CHECK: subu $4, $3, $5
+0x00 0x80 0xfc 0x09 # CHECK: jalr.hb $4
+0x00 0xa0 0x24 0x09 # CHECK: jalr.hb $4, $5
+0x00 0xa3 0x10 0x04 # CHECK: sllv $2, $3, $5
+0x00 0xa3 0x10 0x06 # CHECK: srlv $2, $3, $5
+0x00 0xa3 0x10 0x07 # CHECK: srav $2, $3, $5
+0x00 0xa6 0x00 0x1d # CHECK: dmultu $5, $6
+0x00 0xba 0x28 0x2f # CHECK: dsubu $5, $5, $26
 0x00 0xc7 0x48 0x20 # CHECK: add $9, $6, $7
-0x46 0x2e 0x62 0x00 # CHECK: add.d $f8, $f12, $f14
-0x46 0x07 0x32 0x40 # CHECK: add.s $f9, $f6, $f7
-0x20 0xc9 0x45 0x67 # CHECK: addi $9, $6, 17767
-0x24 0xc9 0xc5 0x67 # CHECK: addiu $9, $6, -15001
 0x00 0xc7 0x48 0x21 # CHECK: addu $9, $6, $7
+0x00 0xc7 0x48 0x22 # CHECK: sub $9, $6, $7
 0x00 0xc7 0x48 0x24 # CHECK: and $9, $6, $7
-0x30 0xc9 0x45 0x67 # CHECK: andi $9, $6, 17767
+0x00 0xc7 0x48 0x27 # CHECK: nor $9, $6, $7
+0x00 0xe0 0x00 0x08 # CHECK: jr $7
+0x00 0xe0 0x00 0x11 # CHECK: mthi $7
+0x00 0xe0 0x00 0x13 # CHECK: mtlo $7
+0x00 0xe0 0xf8 0x09 # CHECK: jalr $7
+0x00 0xe6 0x48 0x46 # CHECK: rotrv $9, $6, $7
+0x01 0x94 0x00 0x14 # CHECK: dsllv $zero, $20, $12
+0x01 0x94 0x00 0x14 # CHECK: dsllv $zero, $20, $12
+0x01 0xee 0x08 0x56 # CHECK: drotrv $1, $14, $15
+0x02 0x11 0x00 0x1f # CHECK: ddivu $zero, $16, $17
+0x02 0x72 0xe0 0x17 # CHECK: dsrav $gp, $18, $19
+0x02 0x72 0xe0 0x17 # CHECK: dsrav $gp, $18, $19
+0x02 0x86 0x98 0x16 # CHECK: dsrlv $19, $6, $20
+0x02 0x86 0x98 0x16 # CHECK: dsrlv $19, $6, $20
+0x02 0xc8 0x38 0x2e # CHECK: dsub $7, $22, $8
+0x02 0xe9 0x00 0x1c # CHECK: dmult $23, $9
+0x03 0x53 0x00 0x1e # CHECK: ddiv $zero, $26, $19
+0x04 0xc1 0x01 0x4c # CHECK: bgez $6, 1332
+0x04 0xd1 0x01 0x4c # CHECK: bgezal $6, 1332
+0x08 0x00 0x01 0x4c # CHECK: j 1328
+0x0c 0x00 0x01 0x4c # CHECK: jal 1328
 0x10 0x00 0x01 0x4c # CHECK: b 1332
+0x11 0x26 0x01 0x4c # CHECK: beq $9, $6, 1332
+0x15 0x26 0x01 0x4c # CHECK: bne $9, $6, 1332
+0x18 0xc0 0x01 0x4c # CHECK: blez $6, 1332
+0x1c 0xc0 0x01 0x4c # CHECK: bgtz $6, 1332
+0x20 0xc9 0x45 0x67 # CHECK: addi $9, $6, 17767
+0x24 0xc9 0xc5 0x67 # CHECK: addiu $9, $6, -15001
+0x28 0x63 0x00 0x67 # CHECK: slti $3, $3, 103
+0x2c 0x63 0x00 0x67 # CHECK: sltiu $3, $3, 103
+0x30 0xc9 0x45 0x67 # CHECK: andi $9, $6, 17767
+0x34 0xc9 0x45 0x67 # CHECK: ori $9, $6, 17767
+0x38 0xc9 0x45 0x67 # CHECK: xori $9, $6, 17767
+0x3c 0x06 0x45 0x67 # CHECK: lui $6, 17767
+0x40 0x08 0x78 0x01 # CHECK: mfc0 $8, $15, 1
+0x40 0x38 0x50 0x00 # CHECK: dmfc0 $24, $10, 0
+0x40 0x89 0x78 0x01 # CHECK: mtc0 $9, $15, 1
+0x40 0xa4 0x50 0x00 # CHECK: dmtc0 $4, $10, 0
+0x41 0x60 0x60 0x00 # CHECK: di
+0x41 0x60 0x60 0x20 # CHECK: ei
+0x41 0x6e 0x60 0x20 # CHECK: ei $14
+0x41 0x7e 0x60 0x00 # CHECK: di $fp
+0x44 0x06 0x38 0x00 # CHECK: mfc1 $6, $f7
+0x44 0x2c 0x68 0x00 # CHECK: dmfc1 $12, $f13
+0x44 0x46 0x38 0x00 # CHECK: cfc1 $6, $7
+0x44 0x7e 0xc0 0x00 # CHECK: mfhc1 $fp, $f24
+0x44 0x86 0x38 0x00 # CHECK: mtc1 $6, $f7
+0x44 0xb0 0x70 0x00 # CHECK: dmtc1 $16, $f14
+0x44 0xc6 0x38 0x00 # CHECK: ctc1 $6, $7
+0x44 0xe0 0x80 0x00 # CHECK: mthc1 $zero, $f16
 0x45 0x00 0x01 0x4c # CHECK: bc1f 1332
-0x45 0x1c 0x01 0x4c # CHECK: bc1f $fcc7, 1332
 0x45 0x01 0x01 0x4c # CHECK: bc1t 1332
+0x45 0x1c 0x01 0x4c # CHECK: bc1f $fcc7, 1332
 0x45 0x1d 0x01 0x4c # CHECK: bc1t $fcc7, 1332
-0x11 0x26 0x01 0x4c # CHECK: beq $9, $6, 1332
-0x04 0xc1 0x01 0x4c # CHECK: bgez $6, 1332
-0x04 0xd1 0x01 0x4c # CHECK: bgezal $6, 1332
-0x1c 0xc0 0x01 0x4c # CHECK: bgtz $6, 1332
-0x18 0xc0 0x01 0x4c # CHECK: blez $6, 1332
-0x15 0x26 0x01 0x4c # CHECK: bne $9, $6, 1332
-0x46 0x2e 0x60 0x32 # CHECK: c.eq.d $f12, $f14
-0x46 0x07 0x30 0x32 # CHECK: c.eq.s $f6, $f7
-0x46 0x2e 0x60 0x30 # CHECK: c.f.d $f12, $f14
+0x46 0x00 0x2b 0x0b # CHECK: floor.l.s $f12, $f5
+0x46 0x00 0x2e 0x48 # CHECK: round.l.s $f25, $f5
+0x46 0x00 0x39 0x84 # CHECK: sqrt.s $f6, $f7
+0x46 0x00 0x39 0x85 # CHECK: abs.s $f6, $f7
+0x46 0x00 0x39 0x86 # CHECK: mov.s $f6, $f7
+0x46 0x00 0x39 0x87 # CHECK: neg.s $f6, $f7
+0x46 0x00 0x39 0x8c # CHECK: round.w.s $f6, $f7
+0x46 0x00 0x39 0x8d # CHECK: trunc.w.s $f6, $f7
+0x46 0x00 0x39 0x8e # CHECK: ceil.w.s $f6, $f7
+0x46 0x00 0x39 0x8f # CHECK: floor.w.s $f6, $f7
+0x46 0x00 0x39 0xa1 # CHECK: cvt.d.s $f6, $f7
+0x46 0x00 0x39 0xa4 # CHECK: cvt.w.s $f6, $f7
+0x46 0x00 0x39 0xa5 # CHECK: cvt.l.s $f6, $f7
+0x46 0x00 0x6c 0x8a # CHECK: ceil.l.s $f18, $f13
+0x46 0x00 0xff 0x09 # CHECK: trunc.l.s $f28, $f31
 0x46 0x07 0x30 0x30 # CHECK: c.f.s $f6, $f7
-0x46 0x2e 0x60 0x3e # CHECK: c.le.d $f12, $f14
-0x46 0x07 0x30 0x3e # CHECK: c.le.s $f6, $f7
-0x46 0x2e 0x60 0x3c # CHECK: c.lt.d $f12, $f14
+0x46 0x07 0x30 0x31 # CHECK: c.un.s $f6, $f7
+0x46 0x07 0x30 0x32 # CHECK: c.eq.s $f6, $f7
+0x46 0x07 0x30 0x34 # CHECK: c.olt.s $f6, $f7
+0x46 0x07 0x30 0x35 # CHECK: c.ult.s $f6, $f7
+0x46 0x07 0x30 0x36 # CHECK: c.ole.s $f6, $f7
+0x46 0x07 0x30 0x37 # CHECK: c.ule.s $f6, $f7
+0x46 0x07 0x30 0x38 # CHECK: c.sf.s $f6, $f7
+0x46 0x07 0x30 0x39 # CHECK: c.ngle.s $f6, $f7
+0x46 0x07 0x30 0x3a # CHECK: c.seq.s $f6, $f7
+0x46 0x07 0x30 0x3b # CHECK: c.ngl.s $f6, $f7
 0x46 0x07 0x30 0x3c # CHECK: c.lt.s $f6, $f7
-0x46 0x2e 0x60 0x3d # CHECK: c.nge.d $f12, $f14
 0x46 0x07 0x30 0x3d # CHECK: c.nge.s $f6, $f7
-0x46 0x2e 0x60 0x3b # CHECK: c.ngl.d $f12, $f14
-0x46 0x07 0x30 0x3b # CHECK: c.ngl.s $f6, $f7
-0x46 0x2e 0x60 0x39 # CHECK: c.ngle.d $f12, $f14
-0x46 0x07 0x30 0x39 # CHECK: c.ngle.s $f6, $f7
-0x46 0x2e 0x60 0x3f # CHECK: c.ngt.d $f12, $f14
+0x46 0x07 0x30 0x3e # CHECK: c.le.s $f6, $f7
 0x46 0x07 0x30 0x3f # CHECK: c.ngt.s $f6, $f7
-0x46 0x2e 0x60 0x36 # CHECK: c.ole.d $f12, $f14
-0x46 0x07 0x30 0x36 # CHECK: c.ole.s $f6, $f7
-0x46 0x2e 0x60 0x34 # CHECK: c.olt.d $f12, $f14
-0x46 0x07 0x30 0x34 # CHECK: c.olt.s $f6, $f7
-0x46 0x2e 0x60 0x3a # CHECK: c.seq.d $f12, $f14
-0x46 0x07 0x30 0x3a # CHECK: c.seq.s $f6, $f7
-0x46 0x2e 0x60 0x38 # CHECK: c.sf.d $f12, $f14
-0x46 0x07 0x30 0x38 # CHECK: c.sf.s $f6, $f7
-0x46 0x2e 0x60 0x33 # CHECK: c.ueq.d $f12, $f14
+0x46 0x07 0x32 0x40 # CHECK: add.s $f9, $f6, $f7
+0x46 0x07 0x32 0x41 # CHECK: sub.s $f9, $f6, $f7
+0x46 0x07 0x32 0x42 # CHECK: mul.s $f9, $f6, $f7
 0x46 0x12 0xe0 0x33 # CHECK: c.ueq.s $f28, $f18
-0x46 0x2e 0x60 0x37 # CHECK: c.ule.d $f12, $f14
-0x46 0x07 0x30 0x37 # CHECK: c.ule.s $f6, $f7
-0x46 0x2e 0x60 0x35 # CHECK: c.ult.d $f12, $f14
-0x46 0x07 0x30 0x35 # CHECK: c.ult.s $f6, $f7
-0x46 0x2e 0x60 0x31 # CHECK: c.un.d $f12, $f14
-0x46 0x07 0x30 0x31 # CHECK: c.un.s $f6, $f7
+0x46 0x20 0x0b 0x08 # CHECK: round.l.d $f12, $f1
 0x46 0x20 0x18 0x4a # CHECK: ceil.l.d $f1, $f3
-0x46 0x00 0x6c 0x8a # CHECK: ceil.l.s $f18, $f13
+0x46 0x20 0x3e 0x8b # CHECK: floor.l.d $f26, $f7
+0x46 0x20 0x41 0x86 # CHECK: mov.d $f6, $f8
+0x46 0x20 0x73 0x04 # CHECK: sqrt.d $f12, $f14
+0x46 0x20 0x73 0x05 # CHECK: abs.d $f12, $f14
+0x46 0x20 0x73 0x07 # CHECK: neg.d $f12, $f14
+0x46 0x20 0x73 0x0c # CHECK: round.w.d $f12, $f14
+0x46 0x20 0x73 0x0d # CHECK: trunc.w.d $f12, $f14
 0x46 0x20 0x73 0x0e # CHECK: ceil.w.d $f12, $f14
-0x46 0x00 0x39 0x8e # CHECK: ceil.w.s $f6, $f7
-0x44 0x46 0x38 0x00 # CHECK: cfc1 $6, $7
-0x70 0xe6 0x30 0x21 # CHECK: clo $6, $7
-0x70 0xe6 0x30 0x20 # CHECK: clz $6, $7
-0x44 0xc6 0x38 0x00 # CHECK: ctc1 $6, $7
-0x46 0xa0 0x81 0x21 # CHECK: cvt.d.l $f4, $f16
-0x46 0x00 0x39 0xa1 # CHECK: cvt.d.s $f6, $f7
-0x46 0x80 0x73 0x21 # CHECK: cvt.d.w $f12, $f14
-0x46 0x20 0x73 0x25 # CHECK: cvt.l.d $f12, $f14
-0x46 0x00 0x39 0xa5 # CHECK: cvt.l.s $f6, $f7
-0x46 0xa0 0xf3 0xe0 # CHECK: cvt.s.l $f15, $f30
+0x46 0x20 0x73 0x0f # CHECK: floor.w.d $f12, $f14
 0x46 0x20 0x73 0x20 # CHECK: cvt.s.d $f12, $f14
-0x46 0x80 0x39 0xa0 # CHECK: cvt.s.w $f6, $f7
 0x46 0x20 0x73 0x24 # CHECK: cvt.w.d $f12, $f14
-0x46 0x00 0x39 0xa4 # CHECK: cvt.w.s $f6, $f7
-0x00 0x3f 0x98 0x2c # CHECK: dadd $19, $1, $ra
+0x46 0x20 0x73 0x25 # CHECK: cvt.l.d $f12, $f14
+0x46 0x20 0xbd 0xc9 # CHECK: trunc.l.d $f23, $f23
+0x46 0x2e 0x60 0x30 # CHECK: c.f.d $f12, $f14
+0x46 0x2e 0x60 0x31 # CHECK: c.un.d $f12, $f14
+0x46 0x2e 0x60 0x32 # CHECK: c.eq.d $f12, $f14
+0x46 0x2e 0x60 0x33 # CHECK: c.ueq.d $f12, $f14
+0x46 0x2e 0x60 0x34 # CHECK: c.olt.d $f12, $f14
+0x46 0x2e 0x60 0x35 # CHECK: c.ult.d $f12, $f14
+0x46 0x2e 0x60 0x36 # CHECK: c.ole.d $f12, $f14
+0x46 0x2e 0x60 0x37 # CHECK: c.ule.d $f12, $f14
+0x46 0x2e 0x60 0x38 # CHECK: c.sf.d $f12, $f14
+0x46 0x2e 0x60 0x39 # CHECK: c.ngle.d $f12, $f14
+0x46 0x2e 0x60 0x3a # CHECK: c.seq.d $f12, $f14
+0x46 0x2e 0x60 0x3b # CHECK: c.ngl.d $f12, $f14
+0x46 0x2e 0x60 0x3c # CHECK: c.lt.d $f12, $f14
+0x46 0x2e 0x60 0x3d # CHECK: c.nge.d $f12, $f14
+0x46 0x2e 0x60 0x3e # CHECK: c.le.d $f12, $f14
+0x46 0x2e 0x60 0x3f # CHECK: c.ngt.d $f12, $f14
+0x42 0x00 0x00 0x58 # CHECK: eretnc
+0x46 0x2e 0x62 0x00 # CHECK: add.d $f8, $f12, $f14
+0x46 0x2e 0x62 0x01 # CHECK: sub.d $f8, $f12, $f14
+0x46 0x2e 0x62 0x02 # CHECK: mul.d $f8, $f12, $f14
+0x46 0x80 0x39 0xa0 # CHECK: cvt.s.w $f6, $f7
+0x46 0x80 0x73 0x21 # CHECK: cvt.d.w $f12, $f14
+0x46 0xa0 0x81 0x21 # CHECK: cvt.d.l $f4, $f16
+0x46 0xa0 0xf3 0xe0 # CHECK: cvt.s.l $f15, $f30
+0x4c 0xa6 0x00 0x05 # CHECK: luxc1 $f0, $6($5)
+0x4c 0xac 0xc8 0x30 # CHECK: nmadd.s $f0, $f5, $f25, $f12
+0x4c 0xb8 0x20 0x0d # CHECK: suxc1 $f4, $24($5)
+0x4d 0xcc 0x05 0x00 # CHECK: lwxc1 $f20, $12($14)
+0x4d 0xf7 0x02 0x01 # CHECK: ldxc1 $f8, $23($15)
+0x4e 0x70 0x53 0x28 # CHECK: msub.s $f12, $f19, $f10, $f16
+0x4e 0xd2 0xd0 0x08 # CHECK: swxc1 $f26, $18($22)
+0x4f 0x04 0x98 0x78 # CHECK: nmsub.s $f1, $f24, $f19, $f4
+0x4f 0x24 0x40 0x09 # CHECK: sdxc1 $f8, $4($25)
+0x4f 0xf9 0x98 0x60 # CHECK: madd.s $f1, $f31, $f19, $f25
+0x62 0x9d 0x6c 0x39 # CHECK: daddi $sp, $20, 27705
 0x62 0x9d 0x93 0xc7 # CHECK: daddi $sp, $20, -27705
-0x63 0xbd 0x93 0xc7 # CHECK: daddi $sp, $sp, -27705
 0x62 0x9d 0x93 0xc7 # CHECK: daddi $sp, $20, -27705
 0x62 0x9d 0x93 0xc7 # CHECK: daddi $sp, $20, -27705
+0x63 0xbd 0x6c 0x39 # CHECK: daddi $sp, $sp, 27705
+0x63 0xbd 0x93 0xc7 # CHECK: daddi $sp, $sp, -27705
 0x63 0xbd 0x93 0xc7 # CHECK: daddi $sp, $sp, -27705
-0x66 0xda 0xee 0x16 # CHECK: daddiu $26, $22, -4586
-0x00 0x3f 0x98 0x2d # CHECK: daddu $19, $1, $ra
 0x64 0x58 0x46 0x9f # CHECK: daddiu $24, $2, 18079
+0x65 0x6f 0xec 0x5f # CHECK: daddiu $15, $11, -5025
+0x65 0xce 0x11 0xea # CHECK: daddiu $14, $14, 4586
 0x66 0x73 0x69 0x3f # CHECK: daddiu $19, $19, 26943
+0x66 0xda 0xee 0x16 # CHECK: daddiu $26, $22, -4586
+0x6b 0x18 0xef 0xb9 # CHECK: ldl $24, -4167($24)
+0x6e 0x8e 0x89 0x6a # CHECK: ldr $14, -30358($20)
+0x70 0xc7 0x00 0x00 # CHECK: madd $6, $7
+0x70 0xc7 0x00 0x01 # CHECK: maddu $6, $7
+0x70 0xc7 0x00 0x04 # CHECK: msub $6, $7
+0x70 0xc7 0x00 0x05 # CHECK: msubu $6, $7
+0x70 0xc7 0x48 0x02 # CHECK: mul $9, $6, $7
 0x70 0xd2 0x90 0x25 # CHECK: dclo $18, $6
+0x70 0xe6 0x30 0x20 # CHECK: clz $6, $7
+0x70 0xe6 0x30 0x21 # CHECK: clo $6, $7
 0x73 0x30 0x80 0x24 # CHECK: dclz $16, $25
-0x03 0x53 0x00 0x1e # CHECK: ddiv $zero, $26, $19
-0x02 0x11 0x00 0x1f # CHECK: ddivu $zero, $16, $17
-0x44 0x2c 0x68 0x00 # CHECK: dmfc1 $12, $f13
-0x44 0xb0 0x70 0x00 # CHECK: dmtc1 $16, $f14
-0x02 0xe9 0x00 0x1c # CHECK: dmult $23, $9
-0x00 0xa6 0x00 0x1d # CHECK: dmultu $5, $6
-0x00 0x00 0x04 0xb8 # CHECK: dsll $zero, $zero, 18
-0x00 0x14 0x04 0xb8 # CHECK: dsll $zero, $20, 18
-0x01 0x94 0x00 0x14 # CHECK: dsllv $zero, $20, $12
-0x00 0x00 0x04 0xbc # CHECK: dsll32 $zero, $zero, 18
-0x00 0x00 0x04 0xbc # CHECK: dsll32 $zero, $zero, 18
-0x01 0x94 0x00 0x14 # CHECK: dsllv $zero, $20, $12
-0x00 0x1c 0xe2 0xbb # CHECK: dsra $gp, $gp, 10
-0x00 0x12 0xe2 0xbb # CHECK: dsra $gp, $18, 10
-0x02 0x72 0xe0 0x17 # CHECK: dsrav $gp, $18, $19
-0x00 0x1c 0xe2 0xbf # CHECK: dsra32 $gp, $gp, 10
-0x00 0x12 0xe2 0xbf # CHECK: dsra32 $gp, $18, 10
-0x02 0x72 0xe0 0x17 # CHECK: dsrav $gp, $18, $19
-0x00 0x13 0x9d 0xfa # CHECK: dsrl $19, $19, 23
-0x00 0x06 0x9d 0xfa # CHECK: dsrl $19, $6, 23
-0x02 0x86 0x98 0x16 # CHECK: dsrlv $19, $6, $20
-0x00 0x13 0x9d 0xfe # CHECK: dsrl32 $19, $19, 23
-0x00 0x06 0x9d 0xfe # CHECK: dsrl32 $19, $6, 23
-0x02 0x86 0x98 0x16 # CHECK: dsrlv $19, $6, $20
-0x02 0xc8 0x38 0x2e # CHECK: dsub $7, $22, $8
+0x74 0x00 0x01 0x4c # CHECK: jalx 1328
+0x7c 0x07 0x30 0xa0 # CHECK: wsbh $6, $7
+0x7c 0x07 0x34 0x20 # CHECK: seb $6, $7
+0x7c 0x07 0x36 0x20 # CHECK: seh $6, $7
 0x7c 0x0e 0x18 0xa4 # CHECK: dsbh $3, $14
 0x7c 0x1d 0x11 0x64 # CHECK: dshd $2, $sp
-0x62 0x9d 0x6c 0x39 # CHECK: daddi $sp, $20, 27705
-0x63 0xbd 0x6c 0x39 # CHECK: daddi $sp, $sp, 27705
-0x00 0xba 0x28 0x2f # CHECK: dsubu $5, $5, $26
-0x65 0x6f 0xec 0x5f # CHECK: daddiu $15, $11, -5025
-0x65 0xce 0x11 0xea # CHECK: daddiu $14, $14, 4586
-0x41 0x7e 0x60 0x00 # CHECK: di $fp
-0x41 0x60 0x60 0x00 # CHECK: di
-0x00 0x21 0x0b 0xfa # CHECK: drotr $1, $1, 15
-0x00 0x2e 0x0b 0xfa # CHECK: drotr $1, $14, 15
-0x00 0x21 0x0b 0xfe # CHECK: drotr32 $1, $1, 15
-0x00 0x2e 0x0b 0xfe # CHECK: drotr32 $1, $14, 15
-0x01 0xee 0x08 0x56 # CHECK: drotrv $1, $14, $15
-0x41 0x6e 0x60 0x20 # CHECK: ei $14
-0x41 0x60 0x60 0x20 # CHECK: ei
-0x46 0x20 0x3e 0x8b # CHECK: floor.l.d $f26, $f7
-0x46 0x00 0x2b 0x0b # CHECK: floor.l.s $f12, $f5
-0x46 0x20 0x73 0x0f # CHECK: floor.w.d $f12, $f14
-0x46 0x00 0x39 0x8f # CHECK: floor.w.s $f6, $f7
 0x7d 0x33 0x61 0x84 # CHECK: ins $19, $9, 6, 7
-0x08 0x00 0x01 0x4c # CHECK: j 1328
-0x0c 0x00 0x01 0x4c # CHECK: jal 1328
-0x74 0x00 0x01 0x4c # CHECK: jalx 1328
-0x00 0xe0 0xf8 0x09 # CHECK: jalr $7
-0x00 0x80 0xfc 0x09 # CHECK: jalr.hb $4
-0x00 0xa0 0x24 0x09 # CHECK: jalr.hb $4, $5
-0x00 0xe0 0x00 0x08 # CHECK: jr $7
 0x80 0xa4 0x23 0xc6 # CHECK: lb $4, 9158($5)
-0x90 0xa4 0x00 0x06 # CHECK: lbu $4, 6($5)
-0xde 0x3d 0x90 0x1b # CHECK: ld $sp, -28645($17)
-0x6b 0x18 0xef 0xb9 # CHECK: ldl $24, -4167($24)
-0x6e 0x8e 0x89 0x6a # CHECK: ldr $14, -30358($20)
-0xd4 0xe9 0x23 0xc6 # CHECK: ldc1 $f9, 9158($7)
-0x4d 0xf7 0x02 0x01 # CHECK: ldxc1 $f8, $23($15)
 0x84 0xa4 0x00 0x0c # CHECK: lh $4, 12($5)
 0x84 0xa4 0x00 0x0c # CHECK: lh $4, 12($5)
-0xc0 0xe9 0x23 0xc6 # CHECK: ll $9, 9158($7)
-0xd3 0xe0 0xc6 0x70 # CHECK: lld $zero, -14736($ra)
-0x3c 0x06 0x45 0x67 # CHECK: lui $6, 17767
-0x4c 0xa6 0x00 0x05 # CHECK: luxc1 $f0, $6($5)
-0x8c 0xa4 0x00 0x18 # CHECK: lw $4, 24($5)
-0xc4 0xe9 0x23 0xc6 # CHECK: lwc1 $f9, 9158($7)
 0x88 0x82 0x00 0x03 # CHECK: lwl $2, 3($4)
+0x8c 0xa4 0x00 0x18 # CHECK: lw $4, 24($5)
+0x90 0xa4 0x00 0x06 # CHECK: lbu $4, 6($5)
 0x98 0xa3 0x00 0x10 # CHECK: lwr $3, 16($5)
-0x4d 0xcc 0x05 0x00 # CHECK: lwxc1 $f20, $12($14)
 0x9c 0x73 0xa1 0xea # CHECK: lwu $19, -24086($3)
-0x70 0xc7 0x00 0x00 # CHECK: madd $6, $7
-0x4f 0xf9 0x98 0x60 # CHECK: madd.s $f1, $f31, $f19, $f25
-0x70 0xc7 0x00 0x01 # CHECK: maddu $6, $7
-0x44 0x06 0x38 0x00 # CHECK: mfc1 $6, $f7
-0x00 0x00 0x28 0x10 # CHECK: mfhi $5
-0x44 0x7e 0xc0 0x00 # CHECK: mfhc1 $fp, $f24
-0x00 0x00 0x28 0x12 # CHECK: mflo $5
-0x46 0x20 0x41 0x86 # CHECK: mov.d $f6, $f8
-0x46 0x00 0x39 0x86 # CHECK: mov.s $f6, $f7
-0x70 0xc7 0x00 0x04 # CHECK: msub $6, $7
-0x4e 0x70 0x53 0x28 # CHECK: msub.s $f12, $f19, $f10, $f16
-0x70 0xc7 0x00 0x05 # CHECK: msubu $6, $7
-0x44 0x86 0x38 0x00 # CHECK: mtc1 $6, $f7
-0x00 0xe0 0x00 0x11 # CHECK: mthi $7
-0x44 0xe0 0x80 0x00 # CHECK: mthc1 $zero, $f16
-0x00 0xe0 0x00 0x13 # CHECK: mtlo $7
-0x46 0x2e 0x62 0x02 # CHECK: mul.d $f8, $f12, $f14
-0x46 0x07 0x32 0x42 # CHECK: mul.s $f9, $f6, $f7
-0x70 0xc7 0x48 0x02 # CHECK: mul $9, $6, $7
-0x00 0x65 0x00 0x18 # CHECK: mult $3, $5
-0x00 0x65 0x00 0x19 # CHECK: multu $3, $5
-0x46 0x20 0x73 0x07 # CHECK: neg.d $f12, $f14
-0x46 0x00 0x39 0x87 # CHECK: neg.s $f6, $f7
-0x4c 0xac 0xc8 0x30 # CHECK: nmadd.s $f0, $f5, $f25, $f12
-0x00 0x00 0x00 0x00 # CHECK: nop
-0x00 0xc7 0x48 0x27 # CHECK: nor $9, $6, $7
-0x4f 0x04 0x98 0x78 # CHECK: nmsub.s $f1, $f24, $f19, $f4
-0x00 0x65 0x18 0x25 # CHECK: or $3, $3, $5
-0x34 0xc9 0x45 0x67 # CHECK: ori $9, $6, 17767
-0x00 0x26 0x49 0xc2 # CHECK: rotr $9, $6, 7
-0x00 0xe6 0x48 0x46 # CHECK: rotrv $9, $6, $7
-0x46 0x20 0x0b 0x08 # CHECK: round.l.d $f12, $f1
-0x46 0x00 0x2e 0x48 # CHECK: round.l.s $f25, $f5
-0x46 0x20 0x73 0x0c # CHECK: round.w.d $f12, $f14
-0x46 0x00 0x39 0x8c # CHECK: round.w.s $f6, $f7
-0xa0 0xa4 0x23 0xc6 # CHECK: sb $4, 9158($5)
 0xa0 0xa4 0x00 0x06 # CHECK: sb $4, 6($5)
-0xe0 0xe9 0x23 0xc6 # CHECK: sc $9, 9158($7)
-0xf3 0xaf 0xdf 0xcd # CHECK: scd $15, -8243($sp)
-0xfd 0x4c 0x16 0xcb # CHECK: sd $12, 5835($10)
-0xf4 0xe9 0x23 0xc6 # CHECK: sdc1 $f9, 9158($7)
-0xb3 0xc7 0xae 0x1f # CHECK: sdl $7, -20961($fp)
-0xb5 0x8b 0xb0 0x39 # CHECK: sdr $11, -20423($12)
-0x4f 0x24 0x40 0x09 # CHECK: sdxc1 $f8, $4($25)
-0x7c 0x07 0x34 0x20 # CHECK: seb $6, $7
-0x7c 0x07 0x36 0x20 # CHECK: seh $6, $7
+0xa0 0xa4 0x23 0xc6 # CHECK: sb $4, 9158($5)
 0xa4 0xa4 0x23 0xc6 # CHECK: sh $4, 9158($5)
-0x00 0x03 0x21 0xc0 # CHECK: sll $4, $3, 7
-0x00 0xa3 0x10 0x04 # CHECK: sllv $2, $3, $5
-0x00 0x65 0x18 0x2a # CHECK: slt $3, $3, $5
-0x28 0x63 0x00 0x67 # CHECK: slti $3, $3, 103
-0x2c 0x63 0x00 0x67 # CHECK: sltiu $3, $3, 103
-0x00 0x65 0x18 0x2b # CHECK: sltu $3, $3, $5
-0x46 0x20 0x73 0x04 # CHECK: sqrt.d $f12, $f14
-0x46 0x00 0x39 0x84 # CHECK: sqrt.s $f6, $f7
-0x00 0x03 0x21 0xc3 # CHECK: sra $4, $3, 7
-0x00 0xa3 0x10 0x07 # CHECK: srav $2, $3, $5
-0x00 0x03 0x21 0xc2 # CHECK: srl $4, $3, 7
-0x00 0xa3 0x10 0x06 # CHECK: srlv $2, $3, $5
-0x46 0x2e 0x62 0x01 # CHECK: sub.d $f8, $f12, $f14
-0x46 0x07 0x32 0x41 # CHECK: sub.s $f9, $f6, $f7
-0x00 0xc7 0x48 0x22 # CHECK: sub $9, $6, $7
-0x00 0x65 0x20 0x23 # CHECK: subu $4, $3, $5
-0x4c 0xb8 0x20 0x0d # CHECK: suxc1 $f4, $24($5)
-0xac 0xa4 0x00 0x18 # CHECK: sw $4, 24($5)
-0xe4 0xe9 0x23 0xc6 # CHECK: swc1 $f9, 9158($7)
 0xa8 0xa4 0x00 0x10 # CHECK: swl $4, 16($5)
+0xac 0xa4 0x00 0x18 # CHECK: sw $4, 24($5)
+0xb3 0xc7 0xae 0x1f # CHECK: sdl $7, -20961($fp)
+0xb5 0x8b 0xb0 0x39 # CHECK: sdr $11, -20423($12)
 0xb8 0xe6 0x00 0x10 # CHECK: swr $6, 16($7)
-0x4e 0xd2 0xd0 0x08 # CHECK: swxc1 $f26, $18($22)
-0x00 0x00 0x01 0xcf # CHECK: sync 7
-0x46 0x20 0xbd 0xc9 # CHECK: trunc.l.d $f23, $f23
-0x46 0x00 0xff 0x09 # CHECK: trunc.l.s $f28, $f31
-0x46 0x20 0x73 0x0d # CHECK: trunc.w.d $f12, $f14
-0x46 0x00 0x39 0x8d # CHECK: trunc.w.s $f6, $f7
-0x7c 0x07 0x30 0xa0 # CHECK: wsbh $6, $7
-0x00 0x65 0x18 0x26 # CHECK: xor $3, $3, $5
-0x38 0xc9 0x45 0x67 # CHECK: xori $9, $6, 17767
+0xc0 0xe9 0x23 0xc6 # CHECK: ll $9, 9158($7)
+0xc4 0xe9 0x23 0xc6 # CHECK: lwc1 $f9, 9158($7)
+0xd3 0xe0 0xc6 0x70 # CHECK: lld $zero, -14736($ra)
+0xd4 0xe9 0x23 0xc6 # CHECK: ldc1 $f9, 9158($7)
+0xde 0x3d 0x90 0x1b # CHECK: ld $sp, -28645($17)
+0xe0 0xe9 0x23 0xc6 # CHECK: sc $9, 9158($7)
+0xe4 0xe9 0x23 0xc6 # CHECK: swc1 $f9, 9158($7)
+0xf3 0xaf 0xdf 0xcd # CHECK: scd $15, -8243($sp)
+0xf4 0xe9 0x23 0xc6 # CHECK: sdc1 $f9, 9158($7)
+0xfd 0x4c 0x16 0xcb # CHECK: sd $12, 5835($10)
diff --git a/test/MC/Disassembler/Mips/mips64r6.txt b/test/MC/Disassembler/Mips/mips64r6.txt
deleted file mode 100644
index 3ddef9ab42a..00000000000
--- a/test/MC/Disassembler/Mips/mips64r6.txt
+++ /dev/null
@@ -1,145 +0,0 @@
-# RUN: llvm-mc --disassemble %s -triple=mips-unknown-linux -mcpu=mips64r6 | FileCheck %s
-
-0xec 0x80 0x00 0x19 # CHECK: addiupc $4, 100
-0x7c 0x43 0x22 0xa0 # CHECK: align $4, $2, $3, 2
-0xec 0x7f 0x00 0x38 # CHECK: aluipc $3, 56
-0x3c 0x62 0xff 0xe9 # CHECK: aui $3, $2, -23
-0xec 0x7e 0xff 0xff # CHECK: auipc $3, -1
-0xe8 0x37 0x96 0xb8 # CHECK: balc 14572256
-0xc8 0x37 0x96 0xb8 # CHECK: bc 14572256
-
-# FIXME: Don't check the immediate on these for the moment, the encode/decode
-#        functions are not inverses of eachother.
-#        The immediate should be 4 but the disassembler currently emits 8
-0x45 0x20 0x00 0x01 # CHECK: bc1eqz $f0,
-0x45 0x3f 0x00 0x01 # CHECK: bc1eqz $f31,
-0x45 0xa0 0x00 0x01 # CHECK: bc1nez $f0,
-0x45 0xbf 0x00 0x01 # CHECK: bc1nez $f31,
-# FIXME: Don't check the immediate on these for the moment, the encode/decode
-#        functions are not inverses of eachother.
-#        The immediate should be 8 but the disassembler currently emits 12
-0x49 0x20 0x00 0x02 # CHECK: bc2eqz $0,
-0x49 0x3f 0x00 0x02 # CHECK: bc2eqz $31,
-0x49 0xa0 0x00 0x02 # CHECK: bc2nez $0,
-0x49 0xbf 0x00 0x02 # CHECK: bc2nez $31,
-
-0x20 0xa6 0x00 0x40 # CHECK: beqc $5, $6, 256
-# FIXME: Don't check the immediate on the bcczal's for the moment, the
-#        encode/decode functions are not inverses of eachother.
-0x20 0x02 0x01 0x4d # CHECK: beqzalc $2,
-0x60 0xa6 0x00 0x40 # CHECK: bnec $5, $6, 256
-0x60 0x02 0x01 0x4d # CHECK: bnezalc $2,
-0xd8 0xa0 0x46 0x90 # CHECK: beqzc $5, 72256
-0x58 0x43 0x00 0x40 # CHECK: bgec $2, $3, 256
-0x18 0x43 0x00 0x40 # CHECK: bgeuc $2, $3, 256
-0x18 0x42 0x01 0x4d # CHECK: bgezalc $2,
-0xf8 0xa0 0x46 0x90 # CHECK: bnezc $5, 72256
-0x5c 0xa5 0x00 0x40 # CHECK: bltzc $5, 256
-0x58 0xa5 0x00 0x40 # CHECK: bgezc $5, 256
-0x1c 0x02 0x01 0x4d # CHECK: bgtzalc $2,
-0x58 0x05 0x00 0x40 # CHECK: blezc $5, 256
-0x1c 0x42 0x01 0x4d # CHECK: bltzalc $2,
-0x5c 0x05 0x00 0x40 # CHECK: bgtzc $5, 256
-0x7c 0x02 0x20 0x20 # CHECK: bitswap $4, $2
-0x18 0x02 0x01 0x4d # CHECK: blezalc $2,
-0x5c 0xa6 0x00 0x40 # CHECK: bltc $5, $6, 256
-0x1c 0xa6 0x00 0x40 # CHECK: bltuc $5, $6, 256
-0x60 0x00 0x00 0x01 # CHECK: bnvc $zero, $zero, 4
-0x60 0x40 0x00 0x01 # CHECK: bnvc $2, $zero, 4
-0x60 0x82 0x00 0x01 # CHECK: bnvc $4, $2, 4
-0x20 0x00 0x00 0x01 # CHECK: bovc $zero, $zero, 4
-0x20 0x40 0x00 0x01 # CHECK: bovc $2, $zero, 4
-0x20 0x82 0x00 0x01 # CHECK: bovc $4, $2, 4
-0x46 0x84 0x18 0x80 # CHECK: cmp.af.s $f2, $f3, $f4
-0x46 0xa4 0x18 0x80 # CHECK: cmp.af.d $f2, $f3, $f4
-0x46 0x84 0x18 0x81 # CHECK: cmp.un.s $f2, $f3, $f4
-0x46 0xa4 0x18 0x81 # CHECK: cmp.un.d $f2, $f3, $f4
-0x46 0x84 0x18 0x82 # CHECK: cmp.eq.s $f2, $f3, $f4
-0x46 0xa4 0x18 0x82 # CHECK: cmp.eq.d $f2, $f3, $f4
-0x46 0x84 0x18 0x83 # CHECK: cmp.ueq.s $f2, $f3, $f4
-0x46 0xa4 0x18 0x83 # CHECK: cmp.ueq.d $f2, $f3, $f4
-0x46 0x84 0x18 0x84 # CHECK: cmp.lt.s $f2, $f3, $f4
-0x46 0xa4 0x18 0x84 # CHECK: cmp.lt.d $f2, $f3, $f4
-0x46 0x84 0x18 0x85 # CHECK: cmp.ult.s $f2, $f3, $f4
-0x46 0xa4 0x18 0x85 # CHECK: cmp.ult.d $f2, $f3, $f4
-0x46 0x84 0x18 0x86 # CHECK: cmp.le.s $f2, $f3, $f4
-0x46 0xa4 0x18 0x86 # CHECK: cmp.le.d $f2, $f3, $f4
-0x46 0x84 0x18 0x87 # CHECK: cmp.ule.s $f2, $f3, $f4
-0x46 0xa4 0x18 0x87 # CHECK: cmp.ule.d $f2, $f3, $f4
-0x46 0x84 0x18 0x88 # CHECK: cmp.saf.s $f2, $f3, $f4
-0x46 0xa4 0x18 0x88 # CHECK: cmp.saf.d $f2, $f3, $f4
-0x46 0x84 0x18 0x89 # CHECK: cmp.sun.s $f2, $f3, $f4
-0x46 0xa4 0x18 0x89 # CHECK: cmp.sun.d $f2, $f3, $f4
-0x46 0x84 0x18 0x8a # CHECK: cmp.seq.s $f2, $f3, $f4
-0x46 0xa4 0x18 0x8a # CHECK: cmp.seq.d $f2, $f3, $f4
-0x46 0x84 0x18 0x8b # CHECK: cmp.sueq.s $f2, $f3, $f4
-0x46 0xa4 0x18 0x8b # CHECK: cmp.sueq.d $f2, $f3, $f4
-0x46 0x84 0x18 0x8c # CHECK: cmp.slt.s $f2, $f3, $f4
-0x46 0xa4 0x18 0x8c # CHECK: cmp.slt.d $f2, $f3, $f4
-0x46 0x84 0x18 0x8d # CHECK: cmp.sult.s $f2, $f3, $f4
-0x46 0xa4 0x18 0x8d # CHECK: cmp.sult.d $f2, $f3, $f4
-0x46 0x84 0x18 0x8e # CHECK: cmp.sle.s $f2, $f3, $f4
-0x46 0xa4 0x18 0x8e # CHECK: cmp.sle.d $f2, $f3, $f4
-0x46 0x84 0x18 0x8f # CHECK: cmp.sule.s $f2, $f3, $f4
-0x46 0xa4 0x18 0x8f # CHECK: cmp.sule.d $f2, $f3, $f4
-0x7c 0x43 0x23 0x64 # CHECK: dalign $4, $2, $3, 5
-0x74 0x62 0x12 0x34 # CHECK: daui $3, $2, 4660
-0x04 0x66 0x56 0x78 # CHECK: dahi $3, 22136
-0x04 0x7e 0xab 0xcd # CHECK: dati $3, -21555
-0x7c 0x02 0x20 0x24 # CHECK: dbitswap $4, $2
-0x00 0x64 0x10 0x9a # CHECK: div $2, $3, $4
-0x00 0x64 0x10 0x9b # CHECK: divu $2, $3, $4
-# 0xf8 0x05 0x01 0x00 # CHECK-TODO: jialc $5, 256
-# 0xd8 0x05 0x01 0x00 # CHECK-TODO: jic $5, 256
-0xec 0x48 0x00 0x43 # CHECK: lwpc $2, 268
-0xec 0x50 0x00 0x43 # CHECK: lwupc $2, 268
-0x00 0x64 0x10 0xda # CHECK: mod $2, $3, $4
-0x00 0x64 0x10 0xdb # CHECK: modu $2, $3, $4
-0x00 0x64 0x10 0x9e # CHECK: ddiv $2, $3, $4
-0x00 0x64 0x10 0x9f # CHECK: ddivu $2, $3, $4
-0x00 0x64 0x10 0xde # CHECK: dmod $2, $3, $4
-0x00 0x64 0x10 0xdf # CHECK: dmodu $2, $3, $4
-0x00 0x64 0x10 0x98 # CHECK: mul $2, $3, $4
-0x00 0x64 0x10 0xd8 # CHECK: muh $2, $3, $4
-0x00 0x64 0x10 0x99 # CHECK: mulu $2, $3, $4
-0x00 0x64 0x10 0xd9 # CHECK: muhu $2, $3, $4
-0x00 0x64 0x10 0x9c # CHECK: dmul $2, $3, $4
-0x00 0x64 0x10 0xdc # CHECK: dmuh $2, $3, $4
-0x00 0x64 0x10 0x9d # CHECK: dmulu $2, $3, $4
-0x00 0x64 0x10 0xdd # CHECK: dmuhu $2, $3, $4
-0x46 0x04 0x18 0x98 # CHECK: maddf.s $f2, $f3, $f4
-0x46 0x24 0x18 0x98 # CHECK: maddf.d $f2, $f3, $f4
-0x46 0x04 0x18 0x99 # CHECK: msubf.s $f2, $f3, $f4
-0x46 0x24 0x18 0x99 # CHECK: msubf.d $f2, $f3, $f4
-0x46 0x22 0x08 0x10 # CHECK: sel.d $f0, $f1, $f2
-0x46 0x02 0x08 0x10 # CHECK: sel.s $f0, $f1, $f2
-0x00 0x64 0x10 0x35 # CHECK: seleqz $2, $3, $4
-0x00 0x64 0x10 0x37 # CHECK: selnez $2, $3, $4
-0x46 0x04 0x10 0x1d # CHECK: max.s $f0, $f2, $f4
-0x46 0x24 0x10 0x1d # CHECK: max.d $f0, $f2, $f4
-0x46 0x04 0x10 0x1c # CHECK: min.s $f0, $f2, $f4
-0x46 0x24 0x10 0x1c # CHECK: min.d $f0, $f2, $f4
-0x46 0x04 0x10 0x1f # CHECK: maxa.s $f0, $f2, $f4
-0x46 0x24 0x10 0x1f # CHECK: maxa.d $f0, $f2, $f4
-0x46 0x04 0x10 0x1e # CHECK: mina.s $f0, $f2, $f4
-0x46 0x24 0x10 0x1e # CHECK: mina.d $f0, $f2, $f4
-0x46 0x04 0x10 0x14 # CHECK: seleqz.s $f0, $f2, $f4
-0x46 0x24 0x10 0x14 # CHECK: seleqz.d $f0, $f2, $f4
-0x46 0x04 0x10 0x17 # CHECK: selnez.s $f0, $f2, $f4
-0x46 0x24 0x10 0x17 # CHECK: selnez.d $f0, $f2, $f4
-0x46 0x00 0x20 0x9a # CHECK: rint.s $f2, $f4
-0x46 0x20 0x20 0x9a # CHECK: rint.d $f2, $f4
-0x46 0x00 0x20 0x9b # CHECK: class.s $f2, $f4
-0x46 0x20 0x20 0x9b # CHECK: class.d $f2, $f4
-0xec 0x58 0x3c 0x48 # CHECK: ldpc $2, 123456
-0x00 0x80 0x04 0x09 # CHECK: jr.hb $4
-0x00 0x80 0xfc 0x09 # CHECK: jalr.hb $4
-0x00 0xa0 0x24 0x09 # CHECK: jalr.hb $4, $5
-0x7e 0x42 0xb3 0xb6 # CHECK: ll $2, -153($18)
-0x7f 0xe0 0x38 0x37 # CHECK: lld $zero, 112($ra)
-0x7e 0x6f 0xec 0x26 # CHECK: sc $15, -40($19)
-0x7f 0xaf 0xe6 0xa7 # CHECK: scd $15, -51($sp)
-0x00 0xa0 0x58 0x51 # CHECK: clo $11, $5
-0x03 0x80 0xe8 0x50 # CHECK: clz $sp, $gp
-0x00 0xc0 0x90 0x53 # CHECK: dclo $18, $6
-0x03 0x20 0x80 0x52 # CHECK: dclz $16, $25
diff --git a/test/MC/Disassembler/Mips/mips64r6/valid-mips64r6-el.txt b/test/MC/Disassembler/Mips/mips64r6/valid-mips64r6-el.txt
index b92c79387d6..157e33593e3 100644
--- a/test/MC/Disassembler/Mips/mips64r6/valid-mips64r6-el.txt
+++ b/test/MC/Disassembler/Mips/mips64r6/valid-mips64r6-el.txt
@@ -1,9 +1,9 @@
 # RUN: llvm-mc %s -disassemble -triple=mipsel-unknown-linux -mcpu=mips64r6 | FileCheck %s
-0x04 0x00 0x42 0x30 # CHECK: andi $2, $2, 4
-0x19 0x00 0x80 0xec # CHECK: addiupc $4, 100
 0x0a 0x00 0x29 0x25 # CHECK: addiu $9, $9, 10
+0x19 0x00 0x80 0xec # CHECK: addiupc $4, 100
 0xa0 0x22 0x43 0x7c # CHECK: align $4, $2, $3, 2
 0x38 0x00 0x7f 0xec # CHECK: aluipc $3, 56
+0x04 0x00 0x42 0x30 # CHECK: andi $2, $2, 4
 0xe9 0xff 0x62 0x3c # CHECK: aui $3, $2, -23
 0xff 0xff 0x7e 0xec # CHECK: auipc $3, -1
 0x9b 0x14 0x11 0x04 # CHECK: bal 21104
@@ -19,148 +19,152 @@
 0x02 0x00 0xbf 0x49 # CHECK: bc2nez $31, 12
 0x40 0x00 0xa6 0x20 # CHECK: beqc $5, $6, 256
 0x4d 0x01 0x02 0x20 # CHECK: beqzalc $2, 1332
-0x40 0x00 0xa6 0x60 # CHECK: bnec $5, $6, 256
-0x4d 0x01 0x02 0x60 # CHECK: bnezalc $2, 1332
 0x90 0x46 0xa0 0xd8 # CHECK: beqzc $5, 72256
 0x40 0x00 0x43 0x58 # CHECK: bgec $2, $3, 256
 0x40 0x00 0x43 0x18 # CHECK: bgeuc $2, $3, 256
 0x4d 0x01 0x42 0x18 # CHECK: bgezalc $2, 1332
-0x90 0x46 0xa0 0xf8 # CHECK: bnezc $5, 72256
-0x40 0x00 0xa5 0x5c # CHECK: bltzc $5, 256
 0x40 0x00 0xa5 0x58 # CHECK: bgezc $5, 256
 0x4d 0x01 0x02 0x1c # CHECK: bgtzalc $2, 1332
-0x40 0x00 0x05 0x58 # CHECK: blezc $5, 256
-0x4d 0x01 0x42 0x1c # CHECK: bltzalc $2, 1332
 0x40 0x00 0x05 0x5c # CHECK: bgtzc $5, 256
 0x20 0x20 0x02 0x7c # CHECK: bitswap $4, $2
 0x4d 0x01 0x02 0x18 # CHECK: blezalc $2, 1332
+0x40 0x00 0x05 0x58 # CHECK: blezc $5, 256
 0x40 0x00 0xa6 0x5c # CHECK: bltc $5, $6, 256
 0x40 0x00 0xa6 0x1c # CHECK: bltuc $5, $6, 256
-0x01 0x00 0x00 0x60 # CHECK: bnvc $zero, $zero, 4
+0x4d 0x01 0x42 0x1c # CHECK: bltzalc $2, 1332
+0x40 0x00 0xa5 0x5c # CHECK: bltzc $5, 256
+0x40 0x00 0xa6 0x60 # CHECK: bnec $5, $6, 256
+0x4d 0x01 0x02 0x60 # CHECK: bnezalc $2, 1332
+0x90 0x46 0xa0 0xf8 # CHECK: bnezc $5, 72256
 0x01 0x00 0x40 0x60 # CHECK: bnvc $2, $zero, 4
 0x01 0x00 0x82 0x60 # CHECK: bnvc $4, $2, 4
-0x01 0x00 0x00 0x20 # CHECK: bovc $zero, $zero, 4
+0x01 0x00 0x00 0x60 # CHECK: bnvc $zero, $zero, 4
 0x01 0x00 0x40 0x20 # CHECK: bovc $2, $zero, 4
 0x01 0x00 0x82 0x20 # CHECK: bovc $4, $2, 4
-0x80 0x18 0x84 0x46 # CHECK: cmp.af.s $f2, $f3, $f4
+0x01 0x00 0x00 0x20 # CHECK: bovc $zero, $zero, 4
+0x25 0x04 0xa1 0x7c # CHECK: cache 1, 8($5)
+0x9b 0x20 0x20 0x46 # CHECK: class.d $f2, $f4
+0x9b 0x20 0x00 0x46 # CHECK: class.s $f2, $f4
+0x51 0x58 0xa0 0x00 # CHECK: clo $11, $5
+0x50 0xe8 0x80 0x03 # CHECK: clz $sp, $gp
 0x80 0x18 0xa4 0x46 # CHECK: cmp.af.d $f2, $f3, $f4
-0x81 0x18 0x84 0x46 # CHECK: cmp.un.s $f2, $f3, $f4
-0x81 0x18 0xa4 0x46 # CHECK: cmp.un.d $f2, $f3, $f4
-0x82 0x18 0x84 0x46 # CHECK: cmp.eq.s $f2, $f3, $f4
+0x80 0x18 0x84 0x46 # CHECK: cmp.af.s $f2, $f3, $f4
 0x82 0x18 0xa4 0x46 # CHECK: cmp.eq.d $f2, $f3, $f4
-0x83 0x18 0x84 0x46 # CHECK: cmp.ueq.s $f2, $f3, $f4
-0x83 0x18 0xa4 0x46 # CHECK: cmp.ueq.d $f2, $f3, $f4
-0x84 0x18 0x84 0x46 # CHECK: cmp.lt.s $f2, $f3, $f4
-0x84 0x18 0xa4 0x46 # CHECK: cmp.lt.d $f2, $f3, $f4
-0x85 0x18 0x84 0x46 # CHECK: cmp.ult.s $f2, $f3, $f4
-0x85 0x18 0xa4 0x46 # CHECK: cmp.ult.d $f2, $f3, $f4
-0x86 0x18 0x84 0x46 # CHECK: cmp.le.s $f2, $f3, $f4
+0x82 0x18 0x84 0x46 # CHECK: cmp.eq.s $f2, $f3, $f4
 0x86 0x18 0xa4 0x46 # CHECK: cmp.le.d $f2, $f3, $f4
-0x87 0x18 0x84 0x46 # CHECK: cmp.ule.s $f2, $f3, $f4
-0x87 0x18 0xa4 0x46 # CHECK: cmp.ule.d $f2, $f3, $f4
-0x88 0x18 0x84 0x46 # CHECK: cmp.saf.s $f2, $f3, $f4
+0x86 0x18 0x84 0x46 # CHECK: cmp.le.s $f2, $f3, $f4
+0x84 0x18 0xa4 0x46 # CHECK: cmp.lt.d $f2, $f3, $f4
+0x84 0x18 0x84 0x46 # CHECK: cmp.lt.s $f2, $f3, $f4
 0x88 0x18 0xa4 0x46 # CHECK: cmp.saf.d $f2, $f3, $f4
-0x89 0x18 0x84 0x46 # CHECK: cmp.sun.s $f2, $f3, $f4
-0x89 0x18 0xa4 0x46 # CHECK: cmp.sun.d $f2, $f3, $f4
-0x8a 0x18 0x84 0x46 # CHECK: cmp.seq.s $f2, $f3, $f4
+0x88 0x18 0x84 0x46 # CHECK: cmp.saf.s $f2, $f3, $f4
 0x8a 0x18 0xa4 0x46 # CHECK: cmp.seq.d $f2, $f3, $f4
-0x8b 0x18 0x84 0x46 # CHECK: cmp.sueq.s $f2, $f3, $f4
-0x8b 0x18 0xa4 0x46 # CHECK: cmp.sueq.d $f2, $f3, $f4
-0x8c 0x18 0x84 0x46 # CHECK: cmp.slt.s $f2, $f3, $f4
-0x8c 0x18 0xa4 0x46 # CHECK: cmp.slt.d $f2, $f3, $f4
-0x8d 0x18 0x84 0x46 # CHECK: cmp.sult.s $f2, $f3, $f4
-0x8d 0x18 0xa4 0x46 # CHECK: cmp.sult.d $f2, $f3, $f4
-0x8e 0x18 0x84 0x46 # CHECK: cmp.sle.s $f2, $f3, $f4
+0x8a 0x18 0x84 0x46 # CHECK: cmp.seq.s $f2, $f3, $f4
 0x8e 0x18 0xa4 0x46 # CHECK: cmp.sle.d $f2, $f3, $f4
-0x8f 0x18 0x84 0x46 # CHECK: cmp.sule.s $f2, $f3, $f4
+0x8e 0x18 0x84 0x46 # CHECK: cmp.sle.s $f2, $f3, $f4
+0x8c 0x18 0xa4 0x46 # CHECK: cmp.slt.d $f2, $f3, $f4
+0x8c 0x18 0x84 0x46 # CHECK: cmp.slt.s $f2, $f3, $f4
+0x8b 0x18 0xa4 0x46 # CHECK: cmp.sueq.d $f2, $f3, $f4
+0x8b 0x18 0x84 0x46 # CHECK: cmp.sueq.s $f2, $f3, $f4
 0x8f 0x18 0xa4 0x46 # CHECK: cmp.sule.d $f2, $f3, $f4
+0x8f 0x18 0x84 0x46 # CHECK: cmp.sule.s $f2, $f3, $f4
+0x8d 0x18 0xa4 0x46 # CHECK: cmp.sult.d $f2, $f3, $f4
+0x8d 0x18 0x84 0x46 # CHECK: cmp.sult.s $f2, $f3, $f4
+0x89 0x18 0xa4 0x46 # CHECK: cmp.sun.d $f2, $f3, $f4
+0x89 0x18 0x84 0x46 # CHECK: cmp.sun.s $f2, $f3, $f4
+0x83 0x18 0xa4 0x46 # CHECK: cmp.ueq.d $f2, $f3, $f4
+0x83 0x18 0x84 0x46 # CHECK: cmp.ueq.s $f2, $f3, $f4
+0x87 0x18 0xa4 0x46 # CHECK: cmp.ule.d $f2, $f3, $f4
+0x87 0x18 0x84 0x46 # CHECK: cmp.ule.s $f2, $f3, $f4
+0x85 0x18 0xa4 0x46 # CHECK: cmp.ult.d $f2, $f3, $f4
+0x85 0x18 0x84 0x46 # CHECK: cmp.ult.s $f2, $f3, $f4
+0x81 0x18 0xa4 0x46 # CHECK: cmp.un.d $f2, $f3, $f4
+0x81 0x18 0x84 0x46 # CHECK: cmp.un.s $f2, $f3, $f4
+0x78 0x56 0x66 0x04 # CHECK: dahi $3, 22136
 0x64 0x23 0x43 0x7c # CHECK: dalign $4, $2, $3, 5
 0x34 0x12 0x62 0x74 # CHECK: daui $3, $2, 4660
-0x78 0x56 0x66 0x04 # CHECK: dahi $3, 22136
 0x24 0x20 0x02 0x7c # CHECK: dbitswap $4, $2
-0x00 0x60 0x7e 0x41 # CHECK: di $fp
+0x53 0x90 0xc0 0x00 # CHECK: dclo $18, $6
+0x52 0x80 0x20 0x03 # CHECK: dclz $16, $25
+0x9e 0x10 0x64 0x00 # CHECK: ddiv $2, $3, $4
+0x9f 0x10 0x64 0x00 # CHECK: ddivu $2, $3, $4
 0x00 0x60 0x60 0x41 # CHECK: di
+0x00 0x60 0x7e 0x41 # CHECK: di $fp
 0x9a 0x10 0x64 0x00 # CHECK: div $2, $3, $4
 0x9b 0x10 0x64 0x00 # CHECK: divu $2, $3, $4
-0x20 0x60 0x6e 0x41 # CHECK: ei $14
-0x20 0x60 0x60 0x41 # CHECK: ei
-0xda 0x10 0x64 0x00 # CHECK: mod $2, $3, $4
-0xdb 0x10 0x64 0x00 # CHECK: modu $2, $3, $4
-0x9e 0x10 0x64 0x00 # CHECK: ddiv $2, $3, $4
-0x9f 0x10 0x64 0x00 # CHECK: ddivu $2, $3, $4
+0xd5 0x10 0x64 0x00 # CHECK: dlsa $2, $3, $4, 3
+0x00 0x50 0x38 0x40 # CHECK: dmfc0 $24, $10, 0
 0xde 0x10 0x64 0x00 # CHECK: dmod $2, $3, $4
 0xdf 0x10 0x64 0x00 # CHECK: dmodu $2, $3, $4
-0xc5 0x10 0x64 0x00 # CHECK: lsa $2, $3, $4, 3
-0xd5 0x10 0x64 0x00 # CHECK: dlsa $2, $3, $4, 3
+0x00 0x50 0xa4 0x40 # CHECK: dmtc0 $4, $10, 0
+0xdc 0x10 0x64 0x00 # CHECK: dmuh $2, $3, $4
+0xdd 0x10 0x64 0x00 # CHECK: dmuhu $2, $3, $4
+0x9c 0x10 0x64 0x00 # CHECK: dmul $2, $3, $4
+0x9d 0x10 0x64 0x00 # CHECK: dmulu $2, $3, $4
+0x20 0x60 0x60 0x41 # CHECK: ei
+0x20 0x60 0x6e 0x41 # CHECK: ei $14
+0x09 0xfc 0x80 0x00 # CHECK: jalr.hb $4
+0x09 0x24 0xa0 0x00 # CHECK: jalr.hb $4, $5
+0x00 0x01 0x05 0xf8 # CHECK: jialc $5, 256
+0x00 0x01 0x05 0xd8 # CHECK: jic $5, 256
+0x09 0x04 0x80 0x00 # CHECK: jr.hb $4
+0x43 0x0d 0xc8 0x49 # CHECK: ldc2 $8, -701($1)
 0x48 0x3c 0x58 0xec # CHECK: ldpc $2, 123456
+0xb6 0xb3 0x42 0x7e # CHECK: ll $2, -153($18)
+0x37 0x38 0xe0 0x7f # CHECK: lld $zero, 112($ra)
+0xc5 0x10 0x64 0x00 # CHECK: lsa $2, $3, $4, 3
+0xb7 0x34 0x52 0x49 # CHECK: lwc2 $18, -841($6)
 0x43 0x00 0x48 0xec # CHECK: lwpc $2, 268
 0x43 0x00 0x50 0xec # CHECK: lwupc $2, 268
-0x98 0x10 0x64 0x00 # CHECK: mul $2, $3, $4
-0xd8 0x10 0x64 0x00 # CHECK: muh $2, $3, $4
-0x99 0x10 0x64 0x00 # CHECK: mulu $2, $3, $4
-0xd9 0x10 0x64 0x00 # CHECK: muhu $2, $3, $4
-0x9c 0x10 0x64 0x00 # CHECK: dmul $2, $3, $4
-0xdc 0x10 0x64 0x00 # CHECK: dmuh $2, $3, $4
-0x9d 0x10 0x64 0x00 # CHECK: dmulu $2, $3, $4
-0xdd 0x10 0x64 0x00 # CHECK: dmuhu $2, $3, $4
-0x98 0x18 0x04 0x46 # CHECK: maddf.s $f2, $f3, $f4
 0x98 0x18 0x24 0x46 # CHECK: maddf.d $f2, $f3, $f4
-0x99 0x18 0x04 0x46 # CHECK: msubf.s $f2, $f3, $f4
-0x99 0x18 0x24 0x46 # CHECK: msubf.d $f2, $f3, $f4
-0x10 0x08 0x22 0x46 # CHECK: sel.d $f0, $f1, $f2
-0x10 0x08 0x02 0x46 # CHECK: sel.s $f0, $f1, $f2
-0x35 0x10 0x64 0x00 # CHECK: seleqz $2, $3, $4
-0x37 0x10 0x64 0x00 # CHECK: selnez $2, $3, $4
-0x1d 0x10 0x04 0x46 # CHECK: max.s $f0, $f2, $f4
+0x98 0x18 0x04 0x46 # CHECK: maddf.s $f2, $f3, $f4
 0x1d 0x10 0x24 0x46 # CHECK: max.d $f0, $f2, $f4
-0x1c 0x10 0x04 0x46 # CHECK: min.s $f0, $f2, $f4
-0x1c 0x10 0x24 0x46 # CHECK: min.d $f0, $f2, $f4
-0x1f 0x10 0x04 0x46 # CHECK: maxa.s $f0, $f2, $f4
+0x1d 0x10 0x04 0x46 # CHECK: max.s $f0, $f2, $f4
 0x1f 0x10 0x24 0x46 # CHECK: maxa.d $f0, $f2, $f4
-0x1e 0x10 0x04 0x46 # CHECK: mina.s $f0, $f2, $f4
+0x1f 0x10 0x04 0x46 # CHECK: maxa.s $f0, $f2, $f4
+0x01 0x78 0x08 0x40 # CHECK: mfc0 $8, $15, 1
+0x1c 0x10 0x24 0x46 # CHECK: min.d $f0, $f2, $f4
+0x1c 0x10 0x04 0x46 # CHECK: min.s $f0, $f2, $f4
 0x1e 0x10 0x24 0x46 # CHECK: mina.d $f0, $f2, $f4
+0x1e 0x10 0x04 0x46 # CHECK: mina.s $f0, $f2, $f4
+0xda 0x10 0x64 0x00 # CHECK: mod $2, $3, $4
+0xdb 0x10 0x64 0x00 # CHECK: modu $2, $3, $4
+0x01 0x78 0x89 0x40 # CHECK: mtc0 $9, $15, 1
+0x99 0x18 0x24 0x46 # CHECK: msubf.d $f2, $f3, $f4
+0x99 0x18 0x04 0x46 # CHECK: msubf.s $f2, $f3, $f4
+0xd8 0x10 0x64 0x00 # CHECK: muh $2, $3, $4
+0xd9 0x10 0x64 0x00 # CHECK: muhu $2, $3, $4
+0x98 0x10 0x64 0x00 # CHECK: mul $2, $3, $4
+0x99 0x10 0x64 0x00 # CHECK: mulu $2, $3, $4
 0x04 0x00 0x42 0x34 # CHECK: ori $2, $2, 4
-0x14 0x10 0x04 0x46 # CHECK: seleqz.s $f0, $f2, $f4
-0x14 0x10 0x24 0x46 # CHECK: seleqz.d $f0, $f2, $f4
-0x17 0x10 0x04 0x46 # CHECK: selnez.s $f0, $f2, $f4
-0x17 0x10 0x24 0x46 # CHECK: selnez.d $f0, $f2, $f4
-0x9a 0x20 0x00 0x46 # CHECK: rint.s $f2, $f4
+0x35 0x04 0xa1 0x7c # CHECK: pref 1, 8($5)
 0x9a 0x20 0x20 0x46 # CHECK: rint.d $f2, $f4
-0x9b 0x20 0x00 0x46 # CHECK: class.s $f2, $f4
-0x9b 0x20 0x20 0x46 # CHECK: class.d $f2, $f4
-0x09 0x04 0x80 0x00 # CHECK: jr.hb $4
-0x09 0xfc 0x80 0x00 # CHECK: jalr.hb $4
-0x09 0x24 0xa0 0x00 # CHECK: jalr.hb $4, $5
-0xb6 0xb3 0x42 0x7e # CHECK: ll $2, -153($18)
-0x37 0x38 0xe0 0x7f # CHECK: lld $zero, 112($ra)
+0x9a 0x20 0x00 0x46 # CHECK: rint.s $f2, $f4
 0x26 0xec 0x6f 0x7e # CHECK: sc $15, -40($19)
 0xa7 0xe6 0xaf 0x7f # CHECK: scd $15, -51($sp)
-0x51 0x58 0xa0 0x00 # CHECK: clo $11, $5
-0x50 0xe8 0x80 0x03 # CHECK: clz $sp, $gp
-0x53 0x90 0xc0 0x00 # CHECK: dclo $18, $6
-0x52 0x80 0x20 0x03 # CHECK: dclz $16, $25
-0x40 0x00 0x00 0x00 # CHECK: ssnop
 0x0e 0x00 0x00 0x00 # CHECK: sdbbp
 0x8e 0x08 0x00 0x00 # CHECK: sdbbp 34
+0x75 0x92 0xf4 0x49 # CHECK: sdc2 $20, 629($18)
+0x10 0x08 0x22 0x46 # CHECK: sel.d $f0, $f1, $f2
+0x10 0x08 0x02 0x46 # CHECK: sel.s $f0, $f1, $f2
+0x35 0x10 0x64 0x00 # CHECK: seleqz $2, $3, $4
+0x14 0x10 0x24 0x46 # CHECK: seleqz.d $f0, $f2, $f4
+0x14 0x10 0x04 0x46 # CHECK: seleqz.s $f0, $f2, $f4
+0x37 0x10 0x64 0x00 # CHECK: selnez $2, $3, $4
+0x17 0x10 0x24 0x46 # CHECK: selnez.d $f0, $f2, $f4
+0x17 0x10 0x04 0x46 # CHECK: selnez.s $f0, $f2, $f4
+0x40 0x00 0x00 0x00 # CHECK: ssnop
+0x30 0x81 0x79 0x49 # CHECK: swc2 $25, 304($16)
 0x0f 0x00 0x00 0x00 # CHECK: sync
 0x4f 0x00 0x00 0x00 # CHECK: sync 1
-0x34 0x00 0x03 0x00 # CHECK: teq $zero, $3
 0x34 0x9b 0xa7 0x00 # CHECK: teq $5, $7, 620
-0x30 0x00 0xea 0x00 # CHECK: tge $7, $10
+0x34 0x00 0x03 0x00 # CHECK: teq $zero, $3
 0x30 0x55 0xb3 0x00 # CHECK: tge $5, $19, 340
-0x31 0x00 0xdc 0x02 # CHECK: tgeu $22, $gp
+0x30 0x00 0xea 0x00 # CHECK: tge $7, $10
 0xf1 0x5e 0x8e 0x02 # CHECK: tgeu $20, $14, 379
+0x31 0x00 0xdc 0x02 # CHECK: tgeu $22, $gp
 0x32 0x00 0xed 0x01 # CHECK: tlt $15, $13
 0x72 0x21 0x53 0x00 # CHECK: tlt $2, $19, 133
 0x33 0x00 0x70 0x01 # CHECK: tltu $11, $16
 0x33 0xfe 0x1d 0x02 # CHECK: tltu $16, $sp, 1016
 0x36 0x00 0xd1 0x00 # CHECK: tne $6, $17
 0x76 0xdd 0xe8 0x00 # CHECK: tne $7, $8, 885
-0x43 0x0d 0xc8 0x49 # CHECK: ldc2 $8, -701($1)
-0xb7 0x34 0x52 0x49 # CHECK: lwc2 $18, -841($6)
-0x75 0x92 0xf4 0x49 # CHECK: sdc2 $20, 629($18)
-0x30 0x81 0x79 0x49 # CHECK: swc2 $25, 304($16)
-0x00 0x01 0x05 0xf8 # CHECK: jialc $5, 256
-0x00 0x01 0x05 0xd8 # CHECK: jic $5, 256
-0x25 0x04 0xa1 0x7c # CHECK: cache 1, 8($5)
-0x35 0x04 0xa1 0x7c # CHECK: pref 1, 8($5)
diff --git a/test/MC/Disassembler/Mips/mips64r6/valid-mips64r6.txt b/test/MC/Disassembler/Mips/mips64r6/valid-mips64r6.txt
index debbe5027ff..5be21256d28 100644
--- a/test/MC/Disassembler/Mips/mips64r6/valid-mips64r6.txt
+++ b/test/MC/Disassembler/Mips/mips64r6/valid-mips64r6.txt
@@ -1,166 +1,197 @@
 # RUN: llvm-mc %s -disassemble -triple=mips-unknown-linux -mcpu=mips64r6 | FileCheck %s
-0x30 0x42 0x00 0x04 # CHECK: andi $2, $2, 4
-0xec 0x80 0x00 0x19 # CHECK: addiupc $4, 100
-0x25 0x29 0x00 0x0a # CHECK: addiu $9, $9, 10
-0x7c 0x43 0x22 0xa0 # CHECK: align $4, $2, $3, 2
-0xec 0x7f 0x00 0x38 # CHECK: aluipc $3, 56
-0x3c 0x62 0xff 0xe9 # CHECK: aui $3, $2, -23
-0xec 0x7e 0xff 0xff # CHECK: auipc $3, -1
+0x00 0x00 0x00 0x0e # CHECK: sdbbp
+0x00 0x00 0x00 0x0f # CHECK: sync
+0x00 0x00 0x00 0x40 # CHECK: ssnop
+0x00 0x00 0x00 0x4f # CHECK: sync 1
+0x00 0x00 0x08 0x8e # CHECK: sdbbp 34
+0x00 0x03 0x00 0x34 # CHECK: teq $zero, $3
+0x00 0x03 0x10 0x2e # CHECK: dneg $2, $3
+0x00 0x03 0x10 0x2f # CHECK: dnegu $2, $3
+0x00 0x53 0x21 0x72 # CHECK: tlt $2, $19, 133
+0x00 0x64 0x10 0x35 # CHECK: seleqz $2, $3, $4
+0x00 0x64 0x10 0x37 # CHECK: selnez $2, $3, $4
+0x00 0x64 0x10 0x98 # CHECK: mul $2, $3, $4
+0x00 0x64 0x10 0x99 # CHECK: mulu $2, $3, $4
+0x00 0x64 0x10 0x9a # CHECK: div $2, $3, $4
+0x00 0x64 0x10 0x9b # CHECK: divu $2, $3, $4
+0x00 0x64 0x10 0x9c # CHECK: dmul $2, $3, $4
+0x00 0x64 0x10 0x9d # CHECK: dmulu $2, $3, $4
+0x00 0x64 0x10 0x9e # CHECK: ddiv $2, $3, $4
+0x00 0x64 0x10 0x9f # CHECK: ddivu $2, $3, $4
+0x00 0x64 0x10 0xc5 # CHECK: lsa $2, $3, $4, 3
+0x00 0x64 0x10 0xd5 # CHECK: dlsa $2, $3, $4, 3
+0x00 0x64 0x10 0xd8 # CHECK: muh $2, $3, $4
+0x00 0x64 0x10 0xd9 # CHECK: muhu $2, $3, $4
+0x00 0x64 0x10 0xda # CHECK: mod $2, $3, $4
+0x00 0x64 0x10 0xdb # CHECK: modu $2, $3, $4
+0x00 0x64 0x10 0xdc # CHECK: dmuh $2, $3, $4
+0x00 0x64 0x10 0xdd # CHECK: dmuhu $2, $3, $4
+0x00 0x64 0x10 0xde # CHECK: dmod $2, $3, $4
+0x00 0x64 0x10 0xdf # CHECK: dmodu $2, $3, $4
+0x00 0x80 0x04 0x09 # CHECK: jr.hb $4
+0x00 0x80 0xfc 0x09 # CHECK: jalr.hb $4
+0x00 0xa0 0x24 0x09 # CHECK: jalr.hb $4, $5
+0x00 0xa0 0x58 0x51 # CHECK: clo $11, $5
+0x00 0xa7 0x9b 0x34 # CHECK: teq $5, $7, 620
+0x00 0xb3 0x55 0x30 # CHECK: tge $5, $19, 340
+0x00 0xc0 0x90 0x53 # CHECK: dclo $18, $6
+0x00 0xd1 0x00 0x36 # CHECK: tne $6, $17
+0x00 0xe8 0xdd 0x76 # CHECK: tne $7, $8, 885
+0x00 0xea 0x00 0x30 # CHECK: tge $7, $10
+0x01 0x70 0x00 0x33 # CHECK: tltu $11, $16
+0x01 0xed 0x00 0x32 # CHECK: tlt $15, $13
+0x02 0x1d 0xfe 0x33 # CHECK: tltu $16, $sp, 1016
+0x02 0x8e 0x5e 0xf1 # CHECK: tgeu $20, $14, 379
+0x02 0xdc 0x00 0x31 # CHECK: tgeu $22, $gp
+0x03 0x20 0x80 0x52 # CHECK: dclz $16, $25
+0x03 0x80 0xe8 0x50 # CHECK: clz $sp, $gp
 0x04 0x11 0x14 0x9b # CHECK: bal 21104
-0xe8 0x37 0x96 0xb8 # CHECK: balc 14572256
-0xc8 0x37 0x96 0xb8 # CHECK: bc 14572256
-0x45 0x20 0x00 0x01 # CHECK: bc1eqz $f0, 8
-0x45 0x3f 0x00 0x01 # CHECK: bc1eqz $f31, 8
-0x45 0xa0 0x00 0x01 # CHECK: bc1nez $f0, 8
-0x45 0xbf 0x00 0x01 # CHECK: bc1nez $f31, 8
-0x49 0x20 0x00 0x02 # CHECK: bc2eqz $0, 12
-0x49 0x3f 0x00 0x02 # CHECK: bc2eqz $31, 12
-0x49 0xa0 0x00 0x02 # CHECK: bc2nez $0, 12
-0x49 0xbf 0x00 0x02 # CHECK: bc2nez $31, 12
-0x20 0xa6 0x00 0x40 # CHECK: beqc $5, $6, 256
-0x20 0x02 0x01 0x4d # CHECK: beqzalc $2, 1332
-0x60 0xa6 0x00 0x40 # CHECK: bnec $5, $6, 256
-0x60 0x02 0x01 0x4d # CHECK: bnezalc $2, 1332
-0xd8 0xa0 0x46 0x90 # CHECK: beqzc $5, 72256
-0x58 0x43 0x00 0x40 # CHECK: bgec $2, $3, 256
-0x18 0x43 0x00 0x40 # CHECK: bgeuc $2, $3, 256
+0x04 0x66 0x56 0x78 # CHECK: dahi $3, 22136
+0x04 0x7e 0xab 0xcd # CHECK: dati $3, -21555
+# FIXME: The encode/decode functions are not inverses of each other.
+0x18 0x02 0x01 0x4d # CHECK: blezalc $2, 1332
+# FIXME: The encode/decode functions are not inverses of each other.
 0x18 0x42 0x01 0x4d # CHECK: bgezalc $2, 1332
-0xf8 0xa0 0x46 0x90 # CHECK: bnezc $5, 72256
-0x5c 0xa5 0x00 0x40 # CHECK: bltzc $5, 256
-0x58 0xa5 0x00 0x40 # CHECK: bgezc $5, 256
+0x18 0x43 0x00 0x40 # CHECK: bgeuc $2, $3, 256
+# FIXME: The encode/decode functions are not inverses of each other.
 0x1c 0x02 0x01 0x4d # CHECK: bgtzalc $2, 1332
-0x58 0x05 0x00 0x40 # CHECK: blezc $5, 256
+# FIXME: The encode/decode functions are not inverses of each other.
 0x1c 0x42 0x01 0x4d # CHECK: bltzalc $2, 1332
-0x5c 0x05 0x00 0x40 # CHECK: bgtzc $5, 256
-0x7c 0x02 0x20 0x20 # CHECK: bitswap $4, $2
-0x18 0x02 0x01 0x4d # CHECK: blezalc $2, 1332
-0x5c 0xa6 0x00 0x40 # CHECK: bltc $5, $6, 256
 0x1c 0xa6 0x00 0x40 # CHECK: bltuc $5, $6, 256
-0x60 0x00 0x00 0x01 # CHECK: bnvc $zero, $zero, 4
-0x60 0x40 0x00 0x01 # CHECK: bnvc $2, $zero, 4
-0x60 0x82 0x00 0x01 # CHECK: bnvc $4, $2, 4
 0x20 0x00 0x00 0x01 # CHECK: bovc $zero, $zero, 4
+# FIXME: The encode/decode functions are not inverses of each other.
+0x20 0x02 0x01 0x4d # CHECK: beqzalc $2, 1332
 0x20 0x40 0x00 0x01 # CHECK: bovc $2, $zero, 4
 0x20 0x82 0x00 0x01 # CHECK: bovc $4, $2, 4
+0x20 0xa6 0x00 0x40 # CHECK: beqc $5, $6, 256
+0x25 0x29 0x00 0x0a # CHECK: addiu $9, $9, 10
+0x30 0x42 0x00 0x04 # CHECK: andi $2, $2, 4
+0x34 0x42 0x00 0x04 # CHECK: ori $2, $2, 4
+0x3c 0x62 0xff 0xe9 # CHECK: aui $3, $2, -23
+0x40 0x08 0x78 0x01 # CHECK: mfc0 $8, $15, 1
+0x40 0x08 0x80 0x03 # CHECK: mfc0 $8, $16, 3
+0x40 0x38 0x50 0x00 # CHECK: dmfc0 $24, $10, 0
+0x40 0x89 0x78 0x01 # CHECK: mtc0 $9, $15, 1
+0x40 0xa4 0x50 0x00 # CHECK: dmtc0 $4, $10, 0
+0x41 0x60 0x60 0x00 # CHECK: di
+0x41 0x60 0x60 0x20 # CHECK: ei
+0x41 0x6e 0x60 0x20 # CHECK: ei $14
+0x41 0x7e 0x60 0x00 # CHECK: di $fp
+# FIXME: The encode/decode functions are not inverses of each other.
+#        The immediate should be 4 but the disassembler currently emits 8
+0x45 0x20 0x00 0x01 # CHECK: bc1eqz $f0, 8
+# FIXME: The encode/decode functions are not inverses of each other.
+#        The immediate should be 4 but the disassembler currently emits 8
+0x45 0x3f 0x00 0x01 # CHECK: bc1eqz $f31, 8
+# FIXME: The encode/decode functions are not inverses of each other.
+#        The immediate should be 4 but the disassembler currently emits 8
+0x45 0xa0 0x00 0x01 # CHECK: bc1nez $f0, 8
+# FIXME: The encode/decode functions are not inverses of each other.
+#        The immediate should be 4 but the disassembler currently emits 8
+0x45 0xbf 0x00 0x01 # CHECK: bc1nez $f31, 8
+0x46 0x00 0x20 0x9a # CHECK: rint.s $f2, $f4
+0x46 0x00 0x20 0x9b # CHECK: class.s $f2, $f4
+0x46 0x02 0x08 0x10 # CHECK: sel.s $f0, $f1, $f2
+0x46 0x04 0x10 0x14 # CHECK: seleqz.s $f0, $f2, $f4
+0x46 0x04 0x10 0x17 # CHECK: selnez.s $f0, $f2, $f4
+0x46 0x04 0x10 0x1c # CHECK: min.s $f0, $f2, $f4
+0x46 0x04 0x10 0x1d # CHECK: max.s $f0, $f2, $f4
+0x46 0x04 0x10 0x1e # CHECK: mina.s $f0, $f2, $f4
+0x46 0x04 0x10 0x1f # CHECK: maxa.s $f0, $f2, $f4
+0x46 0x04 0x18 0x98 # CHECK: maddf.s $f2, $f3, $f4
+0x46 0x04 0x18 0x99 # CHECK: msubf.s $f2, $f3, $f4
+0x46 0x20 0x20 0x9a # CHECK: rint.d $f2, $f4
+0x46 0x20 0x20 0x9b # CHECK: class.d $f2, $f4
+0x46 0x22 0x08 0x10 # CHECK: sel.d $f0, $f1, $f2
+0x46 0x24 0x10 0x14 # CHECK: seleqz.d $f0, $f2, $f4
+0x46 0x24 0x10 0x17 # CHECK: selnez.d $f0, $f2, $f4
+0x46 0x24 0x10 0x1c # CHECK: min.d $f0, $f2, $f4
+0x46 0x24 0x10 0x1d # CHECK: max.d $f0, $f2, $f4
+0x46 0x24 0x10 0x1e # CHECK: mina.d $f0, $f2, $f4
+0x46 0x24 0x10 0x1f # CHECK: maxa.d $f0, $f2, $f4
+0x46 0x24 0x18 0x98 # CHECK: maddf.d $f2, $f3, $f4
+0x46 0x24 0x18 0x99 # CHECK: msubf.d $f2, $f3, $f4
 0x46 0x84 0x18 0x80 # CHECK: cmp.af.s $f2, $f3, $f4
-0x46 0xa4 0x18 0x80 # CHECK: cmp.af.d $f2, $f3, $f4
 0x46 0x84 0x18 0x81 # CHECK: cmp.un.s $f2, $f3, $f4
-0x46 0xa4 0x18 0x81 # CHECK: cmp.un.d $f2, $f3, $f4
 0x46 0x84 0x18 0x82 # CHECK: cmp.eq.s $f2, $f3, $f4
-0x46 0xa4 0x18 0x82 # CHECK: cmp.eq.d $f2, $f3, $f4
 0x46 0x84 0x18 0x83 # CHECK: cmp.ueq.s $f2, $f3, $f4
-0x46 0xa4 0x18 0x83 # CHECK: cmp.ueq.d $f2, $f3, $f4
 0x46 0x84 0x18 0x84 # CHECK: cmp.lt.s $f2, $f3, $f4
-0x46 0xa4 0x18 0x84 # CHECK: cmp.lt.d $f2, $f3, $f4
 0x46 0x84 0x18 0x85 # CHECK: cmp.ult.s $f2, $f3, $f4
-0x46 0xa4 0x18 0x85 # CHECK: cmp.ult.d $f2, $f3, $f4
 0x46 0x84 0x18 0x86 # CHECK: cmp.le.s $f2, $f3, $f4
-0x46 0xa4 0x18 0x86 # CHECK: cmp.le.d $f2, $f3, $f4
 0x46 0x84 0x18 0x87 # CHECK: cmp.ule.s $f2, $f3, $f4
-0x46 0xa4 0x18 0x87 # CHECK: cmp.ule.d $f2, $f3, $f4
 0x46 0x84 0x18 0x88 # CHECK: cmp.saf.s $f2, $f3, $f4
-0x46 0xa4 0x18 0x88 # CHECK: cmp.saf.d $f2, $f3, $f4
 0x46 0x84 0x18 0x89 # CHECK: cmp.sun.s $f2, $f3, $f4
-0x46 0xa4 0x18 0x89 # CHECK: cmp.sun.d $f2, $f3, $f4
 0x46 0x84 0x18 0x8a # CHECK: cmp.seq.s $f2, $f3, $f4
-0x46 0xa4 0x18 0x8a # CHECK: cmp.seq.d $f2, $f3, $f4
 0x46 0x84 0x18 0x8b # CHECK: cmp.sueq.s $f2, $f3, $f4
-0x46 0xa4 0x18 0x8b # CHECK: cmp.sueq.d $f2, $f3, $f4
 0x46 0x84 0x18 0x8c # CHECK: cmp.slt.s $f2, $f3, $f4
-0x46 0xa4 0x18 0x8c # CHECK: cmp.slt.d $f2, $f3, $f4
 0x46 0x84 0x18 0x8d # CHECK: cmp.sult.s $f2, $f3, $f4
-0x46 0xa4 0x18 0x8d # CHECK: cmp.sult.d $f2, $f3, $f4
 0x46 0x84 0x18 0x8e # CHECK: cmp.sle.s $f2, $f3, $f4
-0x46 0xa4 0x18 0x8e # CHECK: cmp.sle.d $f2, $f3, $f4
 0x46 0x84 0x18 0x8f # CHECK: cmp.sule.s $f2, $f3, $f4
+0x46 0xa4 0x18 0x80 # CHECK: cmp.af.d $f2, $f3, $f4
+0x46 0xa4 0x18 0x81 # CHECK: cmp.un.d $f2, $f3, $f4
+0x46 0xa4 0x18 0x82 # CHECK: cmp.eq.d $f2, $f3, $f4
+0x46 0xa4 0x18 0x83 # CHECK: cmp.ueq.d $f2, $f3, $f4
+0x46 0xa4 0x18 0x84 # CHECK: cmp.lt.d $f2, $f3, $f4
+0x46 0xa4 0x18 0x85 # CHECK: cmp.ult.d $f2, $f3, $f4
+0x46 0xa4 0x18 0x86 # CHECK: cmp.le.d $f2, $f3, $f4
+0x46 0xa4 0x18 0x87 # CHECK: cmp.ule.d $f2, $f3, $f4
+0x46 0xa4 0x18 0x88 # CHECK: cmp.saf.d $f2, $f3, $f4
+0x46 0xa4 0x18 0x89 # CHECK: cmp.sun.d $f2, $f3, $f4
+0x46 0xa4 0x18 0x8a # CHECK: cmp.seq.d $f2, $f3, $f4
+0x46 0xa4 0x18 0x8b # CHECK: cmp.sueq.d $f2, $f3, $f4
+0x46 0xa4 0x18 0x8c # CHECK: cmp.slt.d $f2, $f3, $f4
+0x46 0xa4 0x18 0x8d # CHECK: cmp.sult.d $f2, $f3, $f4
+0x46 0xa4 0x18 0x8e # CHECK: cmp.sle.d $f2, $f3, $f4
 0x46 0xa4 0x18 0x8f # CHECK: cmp.sule.d $f2, $f3, $f4
-0x7c 0x43 0x23 0x64 # CHECK: dalign $4, $2, $3, 5
+0x42 0x00 0x00 0x58 # CHECK: eretnc
+# FIXME: The encode/decode functions are not inverses of each other.
+#        The immediate should be 8 but the disassembler currently emits 12
+0x49 0x20 0x00 0x02 # CHECK: bc2eqz $0, 12
+# FIXME: The encode/decode functions are not inverses of each other.
+#        The immediate should be 8 but the disassembler currently emits 12
+0x49 0x3f 0x00 0x02 # CHECK: bc2eqz $31, 12
+0x49 0x52 0x34 0xb7 # CHECK: lwc2 $18, -841($6)
+0x49 0x79 0x81 0x30 # CHECK: swc2 $25, 304($16)
+# FIXME: The encode/decode functions are not inverses of each other.
+#        The immediate should be 8 but the disassembler currently emits 12
+0x49 0xa0 0x00 0x02 # CHECK: bc2nez $0, 12
+# FIXME: The encode/decode functions are not inverses of each other.
+#        The immediate should be 8 but the disassembler currently emits 12
+0x49 0xbf 0x00 0x02 # CHECK: bc2nez $31, 12
+0x49 0xc8 0x0d 0x43 # CHECK: ldc2 $8, -701($1)
+0x49 0xf4 0x92 0x75 # CHECK: sdc2 $20, 629($18)
+0x58 0x05 0x00 0x40 # CHECK: blezc $5, 256
+0x58 0x43 0x00 0x40 # CHECK: bgec $2, $3, 256
+0x58 0xa5 0x00 0x40 # CHECK: bgezc $5, 256
+0x5c 0x05 0x00 0x40 # CHECK: bgtzc $5, 256
+0x5c 0xa5 0x00 0x40 # CHECK: bltzc $5, 256
+0x5c 0xa6 0x00 0x40 # CHECK: bltc $5, $6, 256
+0x60 0x00 0x00 0x01 # CHECK: bnvc $zero, $zero, 4
+# FIXME: The encode/decode functions are not inverses of each other.
+0x60 0x02 0x01 0x4d # CHECK: bnezalc $2, 1332
+0x60 0x40 0x00 0x01 # CHECK: bnvc $2, $zero, 4
+0x60 0x82 0x00 0x01 # CHECK: bnvc $4, $2, 4
+0x60 0xa6 0x00 0x40 # CHECK: bnec $5, $6, 256
 0x74 0x62 0x12 0x34 # CHECK: daui $3, $2, 4660
-0x04 0x66 0x56 0x78 # CHECK: dahi $3, 22136
+0x7c 0x02 0x20 0x20 # CHECK: bitswap $4, $2
 0x7c 0x02 0x20 0x24 # CHECK: dbitswap $4, $2
-0x41 0x7e 0x60 0x00 # CHECK: di $fp
-0x41 0x60 0x60 0x00 # CHECK: di
-0x00 0x64 0x10 0x9a # CHECK: div $2, $3, $4
-0x00 0x64 0x10 0x9b # CHECK: divu $2, $3, $4
-0x41 0x6e 0x60 0x20 # CHECK: ei $14
-0x41 0x60 0x60 0x20 # CHECK: ei
-0x00 0x64 0x10 0xda # CHECK: mod $2, $3, $4
-0x00 0x64 0x10 0xdb # CHECK: modu $2, $3, $4
-0x00 0x64 0x10 0x9e # CHECK: ddiv $2, $3, $4
-0x00 0x64 0x10 0x9f # CHECK: ddivu $2, $3, $4
-0x00 0x64 0x10 0xde # CHECK: dmod $2, $3, $4
-0x00 0x64 0x10 0xdf # CHECK: dmodu $2, $3, $4
-0x00 0x64 0x10 0xc5 # CHECK: lsa $2, $3, $4, 3
-0x00 0x64 0x10 0xd5 # CHECK: dlsa $2, $3, $4, 3
-0xec 0x58 0x3c 0x48 # CHECK: ldpc $2, 123456
-0xec 0x48 0x00 0x43 # CHECK: lwpc $2, 268
-0xec 0x50 0x00 0x43 # CHECK: lwupc $2, 268
-0x00 0x64 0x10 0x98 # CHECK: mul $2, $3, $4
-0x00 0x64 0x10 0xd8 # CHECK: muh $2, $3, $4
-0x00 0x64 0x10 0x99 # CHECK: mulu $2, $3, $4
-0x00 0x64 0x10 0xd9 # CHECK: muhu $2, $3, $4
-0x00 0x64 0x10 0x9c # CHECK: dmul $2, $3, $4
-0x00 0x64 0x10 0xdc # CHECK: dmuh $2, $3, $4
-0x00 0x64 0x10 0x9d # CHECK: dmulu $2, $3, $4
-0x00 0x64 0x10 0xdd # CHECK: dmuhu $2, $3, $4
-0x46 0x04 0x18 0x98 # CHECK: maddf.s $f2, $f3, $f4
-0x46 0x24 0x18 0x98 # CHECK: maddf.d $f2, $f3, $f4
-0x46 0x04 0x18 0x99 # CHECK: msubf.s $f2, $f3, $f4
-0x46 0x24 0x18 0x99 # CHECK: msubf.d $f2, $f3, $f4
-0x46 0x22 0x08 0x10 # CHECK: sel.d $f0, $f1, $f2
-0x46 0x02 0x08 0x10 # CHECK: sel.s $f0, $f1, $f2
-0x00 0x64 0x10 0x35 # CHECK: seleqz $2, $3, $4
-0x00 0x64 0x10 0x37 # CHECK: selnez $2, $3, $4
-0x46 0x04 0x10 0x1d # CHECK: max.s $f0, $f2, $f4
-0x46 0x24 0x10 0x1d # CHECK: max.d $f0, $f2, $f4
-0x46 0x04 0x10 0x1c # CHECK: min.s $f0, $f2, $f4
-0x46 0x24 0x10 0x1c # CHECK: min.d $f0, $f2, $f4
-0x46 0x04 0x10 0x1f # CHECK: maxa.s $f0, $f2, $f4
-0x46 0x24 0x10 0x1f # CHECK: maxa.d $f0, $f2, $f4
-0x46 0x04 0x10 0x1e # CHECK: mina.s $f0, $f2, $f4
-0x46 0x24 0x10 0x1e # CHECK: mina.d $f0, $f2, $f4
-0x34 0x42 0x00 0x04 # CHECK: ori $2, $2, 4
-0x46 0x04 0x10 0x14 # CHECK: seleqz.s $f0, $f2, $f4
-0x46 0x24 0x10 0x14 # CHECK: seleqz.d $f0, $f2, $f4
-0x46 0x04 0x10 0x17 # CHECK: selnez.s $f0, $f2, $f4
-0x46 0x24 0x10 0x17 # CHECK: selnez.d $f0, $f2, $f4
-0x46 0x00 0x20 0x9a # CHECK: rint.s $f2, $f4
-0x46 0x20 0x20 0x9a # CHECK: rint.d $f2, $f4
-0x46 0x00 0x20 0x9b # CHECK: class.s $f2, $f4
-0x46 0x20 0x20 0x9b # CHECK: class.d $f2, $f4
-0x00 0x80 0x04 0x09 # CHECK: jr.hb $4
-0x00 0x80 0xfc 0x09 # CHECK: jalr.hb $4
-0x00 0xa0 0x24 0x09 # CHECK: jalr.hb $4, $5
+0x7c 0x43 0x22 0xa0 # CHECK: align $4, $2, $3, 2
+0x7c 0x43 0x23 0x64 # CHECK: dalign $4, $2, $3, 5
+0x7c 0xa1 0x04 0x25 # CHECK: cache 1, 8($5)
+0x7c 0xa1 0x04 0x35 # CHECK: pref 1, 8($5)
 0x7e 0x42 0xb3 0xb6 # CHECK: ll $2, -153($18)
-0x7f 0xe0 0x38 0x37 # CHECK: lld $zero, 112($ra)
 0x7e 0x6f 0xec 0x26 # CHECK: sc $15, -40($19)
 0x7f 0xaf 0xe6 0xa7 # CHECK: scd $15, -51($sp)
-0x00 0xa0 0x58 0x51 # CHECK: clo $11, $5
-0x03 0x80 0xe8 0x50 # CHECK: clz $sp, $gp
-0x00 0xc0 0x90 0x53 # CHECK: dclo $18, $6
-0x03 0x20 0x80 0x52 # CHECK: dclz $16, $25
-0x00 0x00 0x00 0x40 # CHECK: ssnop
-0x00 0x00 0x00 0x0e # CHECK: sdbbp
-0x00 0x00 0x08 0x8e # CHECK: sdbbp 34
-0x00 0x00 0x00 0x0f # CHECK: sync
-0x00 0x00 0x00 0x4f # CHECK: sync 1
-0x00 0x03 0x00 0x34 # CHECK: teq $zero, $3
-0x00 0xa7 0x9b 0x34 # CHECK: teq $5, $7, 620
-0x00 0xea 0x00 0x30 # CHECK: tge $7, $10
-0x00 0xb3 0x55 0x30 # CHECK: tge $5, $19, 340
-0x02 0xdc 0x00 0x31 # CHECK: tgeu $22, $gp
-0x02 0x8e 0x5e 0xf1 # CHECK: tgeu $20, $14, 379
-0x01 0xed 0x00 0x32 # CHECK: tlt $15, $13
-0x00 0x53 0x21 0x72 # CHECK: tlt $2, $19, 133
-0x01 0x70 0x00 0x33 # CHECK: tltu $11, $16
-0x02 0x1d 0xfe 0x33 # CHECK: tltu $16, $sp, 1016
-0x00 0xd1 0x00 0x36 # CHECK: tne $6, $17
-0x00 0xe8 0xdd 0x76 # CHECK: tne $7, $8, 885
-0x49 0xc8 0x0d 0x43 # CHECK: ldc2 $8, -701($1)
-0x49 0x52 0x34 0xb7 # CHECK: lwc2 $18, -841($6)
-0x49 0xf4 0x92 0x75 # CHECK: sdc2 $20, 629($18)
-0x49 0x79 0x81 0x30 # CHECK: swc2 $25, 304($16)
-0xf8 0x05 0x01 0x00 # CHECK: jialc $5, 256
+0x7f 0xe0 0x38 0x37 # CHECK: lld $zero, 112($ra)
+0xc8 0x37 0x96 0xb8 # CHECK: bc 14572256
 0xd8 0x05 0x01 0x00 # CHECK: jic $5, 256
-0x7c 0xa1 0x04 0x25 # CHECK: cache 1, 8($5)
-0x7c 0xa1 0x04 0x35 # CHECK: pref 1, 8($5)
+0xd8 0xa0 0x46 0x90 # CHECK: beqzc $5, 72256
+0xe8 0x37 0x96 0xb8 # CHECK: balc 14572256
+0xec 0x48 0x00 0x43 # CHECK: lwpc $2, 268
+0xec 0x50 0x00 0x43 # CHECK: lwupc $2, 268
+0xec 0x58 0x3c 0x48 # CHECK: ldpc $2, 123456
+0xec 0x7e 0xff 0xff # CHECK: auipc $3, -1
+0xec 0x7f 0x00 0x38 # CHECK: aluipc $3, 56
+0xec 0x80 0x00 0x19 # CHECK: addiupc $4, 100
+0xf8 0x05 0x01 0x00 # CHECK: jialc $5, 256
+0xf8 0xa0 0x46 0x90 # CHECK: bnezc $5, 72256
diff --git a/test/MC/Disassembler/PowerPC/dcbt.txt b/test/MC/Disassembler/PowerPC/dcbt.txt
new file mode 100644
index 00000000000..3289637ca60
--- /dev/null
+++ b/test/MC/Disassembler/PowerPC/dcbt.txt
@@ -0,0 +1,20 @@
+# RUN: llvm-mc --disassemble %s -triple powerpc64-unknown-unknown -mcpu=pwr7 | FileCheck %s -check-prefix=SERVER
+# RUN: llvm-mc --disassemble %s -triple powerpc64-unknown-unknown -mcpu=a2 | FileCheck %s -check-prefix=EMBEDDED
+
+# SERVER: dcbt 2, 3
+# EMBEDDED: dcbt 2, 3
+0x7c 0x02 0x1a 0x2c
+
+# SERVER: dcbtst 2, 3
+# EMBEDDED: dcbtst 2, 3
+0x7c 0x02 0x19 0xec
+
+# SERVER: dcbt 2, 3, 10
+# EMBEDDED: dcbt 10, 2, 3
+0x7d 0x42 0x1a 0x2c
+
+# SERVER: dcbtst 2, 3, 10
+# EMBEDDED: dcbtst 10, 2, 3
+0x7d 0x42 0x19 0xec
+
+
diff --git a/test/MC/Disassembler/PowerPC/ppc64-encoding-4xx.txt b/test/MC/Disassembler/PowerPC/ppc64-encoding-4xx.txt
index 92e88f8e7a5..22eef5669c9 100644
--- a/test/MC/Disassembler/PowerPC/ppc64-encoding-4xx.txt
+++ b/test/MC/Disassembler/PowerPC/ppc64-encoding-4xx.txt
@@ -5,14 +5,14 @@
 # CHECK: mtdcr 178, 3
 0x7c 0x72 0x2b 0x86
 
-# CHECK: tlbre 2, 3, 0
+# CHECK: tlbrehi 2, 3
 0x7c 0x43 0x07 0x64
-# CHECK: tlbre 2, 3, 1
+# CHECK: tlbrelo 2, 3
 0x7c 0x43 0x0f 0x64
 
-# CHECK: tlbwe 2, 3, 0
+# CHECK: tlbwehi 2, 3
 0x7c 0x43 0x07 0xa4
-# CHECK: tlbwe 2, 3, 1
+# CHECK: tlbwelo 2, 3
 0x7c 0x43 0x0f 0xa4
 
 # CHECK: tlbsx 2, 3, 1
diff --git a/test/MC/Disassembler/PowerPC/ppc64-encoding-bookII.txt b/test/MC/Disassembler/PowerPC/ppc64-encoding-bookII.txt
index 9d63bdd6276..480681c67f1 100644
--- a/test/MC/Disassembler/PowerPC/ppc64-encoding-bookII.txt
+++ b/test/MC/Disassembler/PowerPC/ppc64-encoding-bookII.txt
@@ -12,6 +12,12 @@
 # CHECK: dcbtst 2, 3                     
 0x7c 0x02 0x19 0xec
 
+# CHECK: dcbt 2, 3, 10
+0x7d 0x42 0x1a 0x2c
+
+# CHECK: dcbtst 2, 3, 10
+0x7d 0x42 0x19 0xec
+
 # CHECK: dcbz 2, 3                       
 0x7c 0x02 0x1f 0xec
 
@@ -27,13 +33,13 @@
 # CHECK: stdcx. 2, 3, 4                  
 0x7c 0x43 0x21 0xad
 
-# CHECK: sync 2                          
+# CHECK: ptesync
 0x7c 0x40 0x04 0xac
 
 # CHECK: eieio                           
 0x7c 0x00 0x06 0xac
 
-# CHECK: wait 2                          
+# CHECK: waitimpl
 0x7c 0x40 0x00 0x7c
 
 # CHECK: mbar 1
@@ -66,25 +72,25 @@
 # CHECK: ldarx 2, 3, 4, 1
 0x7c 0x43 0x20 0xa9
 
-# CHECK: sync 0                          
+# CHECK: sync
 0x7c 0x00 0x04 0xac
 
-# CHECK: sync 0                          
+# CHECK: sync
 0x7c 0x00 0x04 0xac
 
-# CHECK: sync 1                          
+# CHECK: lwsync
 0x7c 0x20 0x04 0xac
 
-# CHECK: sync 2                          
+# CHECK: ptesync
 0x7c 0x40 0x04 0xac
 
-# CHECK: wait 0                          
+# CHECK: wait
 0x7c 0x00 0x00 0x7c
 
-# CHECK: wait 1                          
+# CHECK: waitrsv
 0x7c 0x20 0x00 0x7c
 
-# CHECK: wait 2                          
+# CHECK: waitimpl
 0x7c 0x40 0x00 0x7c
 
 # CHECK: mftb 2, 123                     
@@ -93,6 +99,6 @@
 # CHECK: mftb 2, 268                     
 0x7c 0x4c 0x42 0xe6
 
-# CHECK: mftb 2, 269                     
+# CHECK: mftbu 2
 0x7c 0x4d 0x42 0xe6
 
diff --git a/test/MC/Disassembler/PowerPC/ppc64-encoding-bookIII.txt b/test/MC/Disassembler/PowerPC/ppc64-encoding-bookIII.txt
index 7996ed178a1..04308f951ec 100644
--- a/test/MC/Disassembler/PowerPC/ppc64-encoding-bookIII.txt
+++ b/test/MC/Disassembler/PowerPC/ppc64-encoding-bookIII.txt
@@ -1,6 +1,6 @@
 # RUN: llvm-mc --disassemble %s -triple powerpc64-unknown-unknown -mcpu=pwr7 | FileCheck %s
 
-# CHECK: mtmsr 4, 0                       
+# CHECK: mtmsr 4
 0x7c 0x80 0x01 0x24
 
 # CHECK: mtmsr 4, 1                       
@@ -9,7 +9,7 @@
 # CHECK: mfmsr 4                         
 0x7c 0x80 0x00 0xa6
 
-# CHECK: mtmsrd 4, 0                     
+# CHECK: mtmsrd 4
 0x7c 0x80 0x01 0x64
 
 # CHECK: mtmsrd 4, 1                     
@@ -60,7 +60,7 @@
 # CHECK: mtspr 22, 4                     
 0x7c 0x96 0x03 0xa6
 
-# CHECK: mfspr 4, 287                    
+# CHECK: mfpvr 4
 0x7c 0x9f 0x42 0xa6
 
 # CHECK: mfspr 4, 25                     
@@ -99,10 +99,10 @@
 # CHECK: tlbiel 4                        
 0x7c 0x00 0x22 0x24
 
-# CHECK: tlbie 4,0                       
+# CHECK: tlbie 4
 0x7c 0x00 0x22 0x64
 
-# CHECK: tlbie 4,0                       
+# CHECK: tlbie 4
 0x7c 0x00 0x22 0x64
 
 # CHECK: rfi
diff --git a/test/MC/Disassembler/PowerPC/ppc64-encoding-ext.txt b/test/MC/Disassembler/PowerPC/ppc64-encoding-ext.txt
index 4f10c742db9..e976aab4966 100644
--- a/test/MC/Disassembler/PowerPC/ppc64-encoding-ext.txt
+++ b/test/MC/Disassembler/PowerPC/ppc64-encoding-ext.txt
@@ -1,155 +1,155 @@
 # RUN: llvm-mc --disassemble %s -triple powerpc64-unknown-unknown -mcpu=pwr7 | FileCheck %s
 
 # FIXME: decode as beqlr 0
-# CHECK: bclr 12, 2, 0
+# CHECK: bclr 12, 2
 0x4d 0x82 0x00 0x20
 
 # FIXME: decode as beqlr 1
-# CHECK: bclr 12, 6, 0
+# CHECK: bclr 12, 6
 0x4d 0x86 0x00 0x20
 
 # FIXME: decode as beqlr 2
-# CHECK: bclr 12, 10, 0
+# CHECK: bclr 12, 10
 0x4d 0x8a 0x00 0x20
 
 # FIXME: decode as beqlr 3
-# CHECK: bclr 12, 14, 0
+# CHECK: bclr 12, 14
 0x4d 0x8e 0x00 0x20
 
 # FIXME: decode as beqlr 4
-# CHECK: bclr 12, 18, 0
+# CHECK: bclr 12, 18
 0x4d 0x92 0x00 0x20
 
 # FIXME: decode as beqlr 5
-# CHECK: bclr 12, 22, 0
+# CHECK: bclr 12, 22
 0x4d 0x96 0x00 0x20
 
 # FIXME: decode as beqlr 6
-# CHECK: bclr 12, 26, 0
+# CHECK: bclr 12, 26
 0x4d 0x9a 0x00 0x20
 
 # FIXME: decode as beqlr 7
-# CHECK: bclr 12, 30, 0
+# CHECK: bclr 12, 30
 0x4d 0x9e 0x00 0x20
 
-# CHECK: bclr 12, 0, 0
+# CHECK: bclr 12, 0
 0x4d 0x80 0x00 0x20
 
-# CHECK: bclr 12, 1, 0
+# CHECK: bclr 12, 1
 0x4d 0x81 0x00 0x20
 
-# CHECK: bclr 12, 2, 0
+# CHECK: bclr 12, 2
 0x4d 0x82 0x00 0x20
 
-# CHECK: bclr 12, 3, 0
+# CHECK: bclr 12, 3
 0x4d 0x83 0x00 0x20
 
-# CHECK: bclr 12, 3, 0
+# CHECK: bclr 12, 3
 0x4d 0x83 0x00 0x20
 
-# CHECK: bclr 12, 4, 0
+# CHECK: bclr 12, 4
 0x4d 0x84 0x00 0x20
 
-# CHECK: bclr 12, 5, 0
+# CHECK: bclr 12, 5
 0x4d 0x85 0x00 0x20
 
-# CHECK: bclr 12, 6, 0
+# CHECK: bclr 12, 6
 0x4d 0x86 0x00 0x20
 
-# CHECK: bclr 12, 7, 0
+# CHECK: bclr 12, 7
 0x4d 0x87 0x00 0x20
 
-# CHECK: bclr 12, 7, 0
+# CHECK: bclr 12, 7
 0x4d 0x87 0x00 0x20
 
-# CHECK: bclr 12, 8, 0
+# CHECK: bclr 12, 8
 0x4d 0x88 0x00 0x20
 
-# CHECK: bclr 12, 9, 0
+# CHECK: bclr 12, 9
 0x4d 0x89 0x00 0x20
 
-# CHECK: bclr 12, 10, 0
+# CHECK: bclr 12, 10
 0x4d 0x8a 0x00 0x20
 
-# CHECK: bclr 12, 11, 0
+# CHECK: bclr 12, 11
 0x4d 0x8b 0x00 0x20
 
-# CHECK: bclr 12, 11, 0
+# CHECK: bclr 12, 11
 0x4d 0x8b 0x00 0x20
 
-# CHECK: bclr 12, 12, 0
+# CHECK: bclr 12, 12
 0x4d 0x8c 0x00 0x20
 
-# CHECK: bclr 12, 13, 0
+# CHECK: bclr 12, 13
 0x4d 0x8d 0x00 0x20
 
-# CHECK: bclr 12, 14, 0
+# CHECK: bclr 12, 14
 0x4d 0x8e 0x00 0x20
 
-# CHECK: bclr 12, 15, 0
+# CHECK: bclr 12, 15
 0x4d 0x8f 0x00 0x20
 
-# CHECK: bclr 12, 15, 0
+# CHECK: bclr 12, 15
 0x4d 0x8f 0x00 0x20
 
-# CHECK: bclr 12, 16, 0
+# CHECK: bclr 12, 16
 0x4d 0x90 0x00 0x20
 
-# CHECK: bclr 12, 17, 0
+# CHECK: bclr 12, 17
 0x4d 0x91 0x00 0x20
 
-# CHECK: bclr 12, 18, 0
+# CHECK: bclr 12, 18
 0x4d 0x92 0x00 0x20
 
-# CHECK: bclr 12, 19, 0
+# CHECK: bclr 12, 19
 0x4d 0x93 0x00 0x20
 
-# CHECK: bclr 12, 19, 0
+# CHECK: bclr 12, 19
 0x4d 0x93 0x00 0x20
 
-# CHECK: bclr 12, 20, 0
+# CHECK: bclr 12, 20
 0x4d 0x94 0x00 0x20
 
-# CHECK: bclr 12, 21, 0
+# CHECK: bclr 12, 21
 0x4d 0x95 0x00 0x20
 
-# CHECK: bclr 12, 22, 0
+# CHECK: bclr 12, 22
 0x4d 0x96 0x00 0x20
 
-# CHECK: bclr 12, 23, 0
+# CHECK: bclr 12, 23
 0x4d 0x97 0x00 0x20
 
-# CHECK: bclr 12, 23, 0
+# CHECK: bclr 12, 23
 0x4d 0x97 0x00 0x20
 
-# CHECK: bclr 12, 24, 0
+# CHECK: bclr 12, 24
 0x4d 0x98 0x00 0x20
 
-# CHECK: bclr 12, 25, 0
+# CHECK: bclr 12, 25
 0x4d 0x99 0x00 0x20
 
-# CHECK: bclr 12, 26, 0
+# CHECK: bclr 12, 26
 0x4d 0x9a 0x00 0x20
 
-# CHECK: bclr 12, 27, 0
+# CHECK: bclr 12, 27
 0x4d 0x9b 0x00 0x20
 
-# CHECK: bclr 12, 27, 0
+# CHECK: bclr 12, 27
 0x4d 0x9b 0x00 0x20
 
-# CHECK: bclr 12, 28, 0
+# CHECK: bclr 12, 28
 0x4d 0x9c 0x00 0x20
 
-# CHECK: bclr 12, 29, 0
+# CHECK: bclr 12, 29
 0x4d 0x9d 0x00 0x20
 
-# CHECK: bclr 12, 30, 0
+# CHECK: bclr 12, 30
 0x4d 0x9e 0x00 0x20
 
-# CHECK: bclr 12, 31, 0
+# CHECK: bclr 12, 31
 0x4d 0x9f 0x00 0x20
 
-# CHECK: bclr 12, 31, 0
+# CHECK: bclr 12, 31
 0x4d 0x9f 0x00 0x20
 
 # CHECK: blr
@@ -164,76 +164,76 @@
 # CHECK: bctrl
 0x4e 0x80 0x04 0x21
 
-# CHECK: bclr 12, 2, 0
+# CHECK: bclr 12, 2
 0x4d 0x82 0x00 0x20
 
-# CHECK: bcctr 12, 2, 0
+# CHECK: bcctr 12, 2
 0x4d 0x82 0x04 0x20
 
-# CHECK: bclrl 12, 2, 0
+# CHECK: bclrl 12, 2
 0x4d 0x82 0x00 0x21
 
-# CHECK: bcctrl 12, 2, 0
+# CHECK: bcctrl 12, 2
 0x4d 0x82 0x04 0x21
 
-# CHECK: bclr 15, 2, 0
+# CHECK: bclr 15, 2
 0x4d 0xe2 0x00 0x20
 
-# CHECK: bcctr 15, 2, 0
+# CHECK: bcctr 15, 2
 0x4d 0xe2 0x04 0x20
 
-# CHECK: bclrl 15, 2, 0
+# CHECK: bclrl 15, 2
 0x4d 0xe2 0x00 0x21
 
-# CHECK: bcctrl 15, 2, 0
+# CHECK: bcctrl 15, 2
 0x4d 0xe2 0x04 0x21
 
-# CHECK: bclr 14, 2, 0
+# CHECK: bclr 14, 2
 0x4d 0xc2 0x00 0x20
 
-# CHECK: bcctr 14, 2, 0
+# CHECK: bcctr 14, 2
 0x4d 0xc2 0x04 0x20
 
-# CHECK: bclrl 14, 2, 0
+# CHECK: bclrl 14, 2
 0x4d 0xc2 0x00 0x21
 
-# CHECK: bcctrl 14, 2, 0
+# CHECK: bcctrl 14, 2
 0x4d 0xc2 0x04 0x21
 
-# CHECK: bclr 4, 2, 0
+# CHECK: bclr 4, 2
 0x4c 0x82 0x00 0x20
 
-# CHECK: bcctr 4, 2, 0
+# CHECK: bcctr 4, 2
 0x4c 0x82 0x04 0x20
 
-# CHECK: bclrl 4, 2, 0
+# CHECK: bclrl 4, 2
 0x4c 0x82 0x00 0x21
 
-# CHECK: bcctrl 4, 2, 0
+# CHECK: bcctrl 4, 2
 0x4c 0x82 0x04 0x21
 
-# CHECK: bclr 7, 2, 0
+# CHECK: bclr 7, 2
 0x4c 0xe2 0x00 0x20
 
-# CHECK: bcctr 7, 2, 0
+# CHECK: bcctr 7, 2
 0x4c 0xe2 0x04 0x20
 
-# CHECK: bclrl 7, 2, 0
+# CHECK: bclrl 7, 2
 0x4c 0xe2 0x00 0x21
 
-# CHECK: bcctrl 7, 2, 0
+# CHECK: bcctrl 7, 2
 0x4c 0xe2 0x04 0x21
 
-# CHECK: bclr 6, 2, 0
+# CHECK: bclr 6, 2
 0x4c 0xc2 0x00 0x20
 
-# CHECK: bcctr 6, 2, 0
+# CHECK: bcctr 6, 2
 0x4c 0xc2 0x04 0x20
 
-# CHECK: bclrl 6, 2, 0
+# CHECK: bclrl 6, 2
 0x4c 0xc2 0x00 0x21
 
-# CHECK: bcctrl 6, 2, 0
+# CHECK: bcctrl 6, 2
 0x4c 0xc2 0x04 0x21
 
 # CHECK: bdnzlr
@@ -254,16 +254,16 @@
 # CHECK: bdnzlrl-
 0x4f 0x00 0x00 0x21
 
-# CHECK: bclr 8, 2, 0
+# CHECK: bclr 8, 2
 0x4d 0x02 0x00 0x20
 
-# CHECK: bclrl 8, 2, 0
+# CHECK: bclrl 8, 2
 0x4d 0x02 0x00 0x21
 
-# CHECK: bclr 0, 2, 0
+# CHECK: bclr 0, 2
 0x4c 0x02 0x00 0x20
 
-# CHECK: bclrl 0, 2, 0
+# CHECK: bclrl 0, 2
 0x4c 0x02 0x00 0x21
 
 # CHECK: bdzlr
@@ -284,1168 +284,1168 @@
 # CHECK: bdzlrl-
 0x4f 0x40 0x00 0x21
 
-# CHECK: bclr 10, 2, 0
+# CHECK: bclr 10, 2
 0x4d 0x42 0x00 0x20
 
-# CHECK: bclrl 10, 2, 0
+# CHECK: bclrl 10, 2
 0x4d 0x42 0x00 0x21
 
-# CHECK: bclr 2, 2, 0
+# CHECK: bclr 2, 2
 0x4c 0x42 0x00 0x20
 
-# CHECK: bclrl 2, 2, 0
+# CHECK: bclrl 2, 2
 0x4c 0x42 0x00 0x21
 
 # FIXME: decode as bltlr 2
-# CHECK: bclr 12, 8, 0
+# CHECK: bclr 12, 8
 0x4d 0x88 0x00 0x20
 
 # FIXME: decode as bltlr 0
-# CHECK: bclr 12, 0, 0
+# CHECK: bclr 12, 0
 0x4d 0x80 0x00 0x20
 
 # FIXME: decode as bltctr 2
-# CHECK: bcctr 12, 8, 0
+# CHECK: bcctr 12, 8
 0x4d 0x88 0x04 0x20
 
 # FIXME: decode as bltctr 0
-# CHECK: bcctr 12, 0, 0
+# CHECK: bcctr 12, 0
 0x4d 0x80 0x04 0x20
 
 # FIXME: decode as bltlrl 2
-# CHECK: bclrl 12, 8, 0
+# CHECK: bclrl 12, 8
 0x4d 0x88 0x00 0x21
 
 # FIXME: decode as bltlrl 0
-# CHECK: bclrl 12, 0, 0
+# CHECK: bclrl 12, 0
 0x4d 0x80 0x00 0x21
 
 # FIXME: decode as bltctrl 2
-# CHECK: bcctrl 12, 8, 0
+# CHECK: bcctrl 12, 8
 0x4d 0x88 0x04 0x21
 
 # FIXME: decode as bltctrl 0
-# CHECK: bcctrl 12, 0, 0
+# CHECK: bcctrl 12, 0
 0x4d 0x80 0x04 0x21
 
 # FIXME: decode as bltlr+ 2
-# CHECK: bclr 15, 8, 0
+# CHECK: bclr 15, 8
 0x4d 0xe8 0x00 0x20
 
 # FIXME: decode as bltlr+ 0
-# CHECK: bclr 15, 0, 0
+# CHECK: bclr 15, 0
 0x4d 0xe0 0x00 0x20
 
 # FIXME: decode as bltctr+ 2
-# CHECK: bcctr 15, 8, 0
+# CHECK: bcctr 15, 8
 0x4d 0xe8 0x04 0x20
 
 # FIXME: decode as bltctr+ 0
-# CHECK: bcctr 15, 0, 0
+# CHECK: bcctr 15, 0
 0x4d 0xe0 0x04 0x20
 
 # FIXME: decode as bltlrl+ 2
-# CHECK: bclrl 15, 8, 0
+# CHECK: bclrl 15, 8
 0x4d 0xe8 0x00 0x21
 
 # FIXME: decode as bltlrl+ 0
-# CHECK: bclrl 15, 0, 0
+# CHECK: bclrl 15, 0
 0x4d 0xe0 0x00 0x21
 
 # FIXME: decode as bltctrl+ 2
-# CHECK: bcctrl 15, 8, 0
+# CHECK: bcctrl 15, 8
 0x4d 0xe8 0x04 0x21
 
 # FIXME: decode as bltctrl+ 0
-# CHECK: bcctrl 15, 0, 0
+# CHECK: bcctrl 15, 0
 0x4d 0xe0 0x04 0x21
 
 # FIXME: decode as bltlr- 2
-# CHECK: bclr 14, 8, 0
+# CHECK: bclr 14, 8
 0x4d 0xc8 0x00 0x20
 
 # FIXME: decode as bltlr- 0
-# CHECK: bclr 14, 0, 0
+# CHECK: bclr 14, 0
 0x4d 0xc0 0x00 0x20
 
 # FIXME: decode as bltctr- 2
-# CHECK: bcctr 14, 8, 0
+# CHECK: bcctr 14, 8
 0x4d 0xc8 0x04 0x20
 
 # FIXME: decode as bltctr- 0
-# CHECK: bcctr 14, 0, 0
+# CHECK: bcctr 14, 0
 0x4d 0xc0 0x04 0x20
 
 # FIXME: decode as bltlrl- 2
-# CHECK: bclrl 14, 8, 0
+# CHECK: bclrl 14, 8
 0x4d 0xc8 0x00 0x21
 
 # FIXME: decode as bltlrl- 0
-# CHECK: bclrl 14, 0, 0
+# CHECK: bclrl 14, 0
 0x4d 0xc0 0x00 0x21
 
 # FIXME: decode as bltctrl- 2
-# CHECK: bcctrl 14, 8, 0
+# CHECK: bcctrl 14, 8
 0x4d 0xc8 0x04 0x21
 
 # FIXME: decode as bltctrl- 0
-# CHECK: bcctrl 14, 0, 0
+# CHECK: bcctrl 14, 0
 0x4d 0xc0 0x04 0x21
 
 # FIXME: decode as blelr 2
-# CHECK: bclr 4, 9, 0
+# CHECK: bclr 4, 9
 0x4c 0x89 0x00 0x20
 
 # FIXME: decode as blelr 0
-# CHECK: bclr 4, 1, 0
+# CHECK: bclr 4, 1
 0x4c 0x81 0x00 0x20
 
 # FIXME: decode as blectr 2
-# CHECK: bcctr 4, 9, 0
+# CHECK: bcctr 4, 9
 0x4c 0x89 0x04 0x20
 
 # FIXME: decode as blectr 0
-# CHECK: bcctr 4, 1, 0
+# CHECK: bcctr 4, 1
 0x4c 0x81 0x04 0x20
 
 # FIXME: decode as blelrl 2
-# CHECK: bclrl 4, 9, 0
+# CHECK: bclrl 4, 9
 0x4c 0x89 0x00 0x21
 
 # FIXME: decode as blelrl 0
-# CHECK: bclrl 4, 1, 0
+# CHECK: bclrl 4, 1
 0x4c 0x81 0x00 0x21
 
 # FIXME: decode as blectrl 2
-# CHECK: bcctrl 4, 9, 0
+# CHECK: bcctrl 4, 9
 0x4c 0x89 0x04 0x21
 
 # FIXME: decode as blectrl 0
-# CHECK: bcctrl 4, 1, 0
+# CHECK: bcctrl 4, 1
 0x4c 0x81 0x04 0x21
 
 # FIXME: decode as blelr+ 2
-# CHECK: bclr 7, 9, 0
+# CHECK: bclr 7, 9
 0x4c 0xe9 0x00 0x20
 
 # FIXME: decode as blelr+ 0
-# CHECK: bclr 7, 1, 0
+# CHECK: bclr 7, 1
 0x4c 0xe1 0x00 0x20
 
 # FIXME: decode as blectr+ 2
-# CHECK: bcctr 7, 9, 0
+# CHECK: bcctr 7, 9
 0x4c 0xe9 0x04 0x20
 
 # FIXME: decode as blectr+ 0
-# CHECK: bcctr 7, 1, 0
+# CHECK: bcctr 7, 1
 0x4c 0xe1 0x04 0x20
 
 # FIXME: decode as blelrl+ 2
-# CHECK: bclrl 7, 9, 0
+# CHECK: bclrl 7, 9
 0x4c 0xe9 0x00 0x21
 
 # FIXME: decode as blelrl+ 0
-# CHECK: bclrl 7, 1, 0
+# CHECK: bclrl 7, 1
 0x4c 0xe1 0x00 0x21
 
 # FIXME: decode as blectrl+ 2
-# CHECK: bcctrl 7, 9, 0
+# CHECK: bcctrl 7, 9
 0x4c 0xe9 0x04 0x21
 
 # FIXME: decode as blectrl+ 0
-# CHECK: bcctrl 7, 1, 0
+# CHECK: bcctrl 7, 1
 0x4c 0xe1 0x04 0x21
 
 # FIXME: decode as blelr- 2
-# CHECK: bclr 6, 9, 0
+# CHECK: bclr 6, 9
 0x4c 0xc9 0x00 0x20
 
 # FIXME: decode as blelr- 0
-# CHECK: bclr 6, 1, 0
+# CHECK: bclr 6, 1
 0x4c 0xc1 0x00 0x20
 
 # FIXME: decode as blectr- 2
-# CHECK: bcctr 6, 9, 0
+# CHECK: bcctr 6, 9
 0x4c 0xc9 0x04 0x20
 
 # FIXME: decode as blectr- 0
-# CHECK: bcctr 6, 1, 0
+# CHECK: bcctr 6, 1
 0x4c 0xc1 0x04 0x20
 
 # FIXME: decode as blelrl- 2
-# CHECK: bclrl 6, 9, 0
+# CHECK: bclrl 6, 9
 0x4c 0xc9 0x00 0x21
 
 # FIXME: decode as blelrl- 0
-# CHECK: bclrl 6, 1, 0
+# CHECK: bclrl 6, 1
 0x4c 0xc1 0x00 0x21
 
 # FIXME: decode as blectrl- 2
-# CHECK: bcctrl 6, 9, 0
+# CHECK: bcctrl 6, 9
 0x4c 0xc9 0x04 0x21
 
 # FIXME: decode as blectrl- 0
-# CHECK: bcctrl 6, 1, 0
+# CHECK: bcctrl 6, 1
 0x4c 0xc1 0x04 0x21
 
 # FIXME: decode as beqlr 2
-# CHECK: bclr 12, 10, 0
+# CHECK: bclr 12, 10
 0x4d 0x8a 0x00 0x20
 
 # FIXME: decode as beqlr 0
-# CHECK: bclr 12, 2, 0
+# CHECK: bclr 12, 2
 0x4d 0x82 0x00 0x20
 
 # FIXME: decode as beqctr 2
-# CHECK: bcctr 12, 10, 0
+# CHECK: bcctr 12, 10
 0x4d 0x8a 0x04 0x20
 
 # FIXME: decode as beqctr 0
-# CHECK: bcctr 12, 2, 0
+# CHECK: bcctr 12, 2
 0x4d 0x82 0x04 0x20
 
 # FIXME: decode as beqlrl 2
-# CHECK: bclrl 12, 10, 0
+# CHECK: bclrl 12, 10
 0x4d 0x8a 0x00 0x21
 
 # FIXME: decode as beqlrl 0
-# CHECK: bclrl 12, 2, 0
+# CHECK: bclrl 12, 2
 0x4d 0x82 0x00 0x21
 
 # FIXME: decode as beqctrl 2
-# CHECK: bcctrl 12, 10, 0
+# CHECK: bcctrl 12, 10
 0x4d 0x8a 0x04 0x21
 
 # FIXME: decode as beqctrl 0
-# CHECK: bcctrl 12, 2, 0
+# CHECK: bcctrl 12, 2
 0x4d 0x82 0x04 0x21
 
 # FIXME: decode as beqlr+ 2
-# CHECK: bclr 15, 10, 0
+# CHECK: bclr 15, 10
 0x4d 0xea 0x00 0x20
 
 # FIXME: decode as beqlr+ 0
-# CHECK: bclr 15, 2, 0
+# CHECK: bclr 15, 2
 0x4d 0xe2 0x00 0x20
 
 # FIXME: decode as beqctr+ 2
-# CHECK: bcctr 15, 10, 0
+# CHECK: bcctr 15, 10
 0x4d 0xea 0x04 0x20
 
 # FIXME: decode as beqctr+ 0
-# CHECK: bcctr 15, 2, 0
+# CHECK: bcctr 15, 2
 0x4d 0xe2 0x04 0x20
 
 # FIXME: decode as beqlrl+ 2
-# CHECK: bclrl 15, 10, 0
+# CHECK: bclrl 15, 10
 0x4d 0xea 0x00 0x21
 
 # FIXME: decode as beqlrl+ 0
-# CHECK: bclrl 15, 2, 0
+# CHECK: bclrl 15, 2
 0x4d 0xe2 0x00 0x21
 
 # FIXME: decode as beqctrl+ 2
-# CHECK: bcctrl 15, 10, 0
+# CHECK: bcctrl 15, 10
 0x4d 0xea 0x04 0x21
 
 # FIXME: decode as beqctrl+ 0
-# CHECK: bcctrl 15, 2, 0
+# CHECK: bcctrl 15, 2
 0x4d 0xe2 0x04 0x21
 
 # FIXME: decode as beqlr- 2
-# CHECK: bclr 14, 10, 0
+# CHECK: bclr 14, 10
 0x4d 0xca 0x00 0x20
 
 # FIXME: decode as beqlr- 0
-# CHECK: bclr 14, 2, 0
+# CHECK: bclr 14, 2
 0x4d 0xc2 0x00 0x20
 
 # FIXME: decode as beqctr- 2
-# CHECK: bcctr 14, 10, 0
+# CHECK: bcctr 14, 10
 0x4d 0xca 0x04 0x20
 
 # FIXME: decode as beqctr- 0
-# CHECK: bcctr 14, 2, 0
+# CHECK: bcctr 14, 2
 0x4d 0xc2 0x04 0x20
 
 # FIXME: decode as beqlrl- 2
-# CHECK: bclrl 14, 10, 0
+# CHECK: bclrl 14, 10
 0x4d 0xca 0x00 0x21
 
 # FIXME: decode as beqlrl- 0
-# CHECK: bclrl 14, 2, 0
+# CHECK: bclrl 14, 2
 0x4d 0xc2 0x00 0x21
 
 # FIXME: decode as beqctrl- 2
-# CHECK: bcctrl 14, 10, 0
+# CHECK: bcctrl 14, 10
 0x4d 0xca 0x04 0x21
 
 # FIXME: decode as beqctrl- 0
-# CHECK: bcctrl 14, 2, 0
+# CHECK: bcctrl 14, 2
 0x4d 0xc2 0x04 0x21
 
 # FIXME: decode as bgelr 2
-# CHECK: bclr 4, 8, 0
+# CHECK: bclr 4, 8
 0x4c 0x88 0x00 0x20
 
 # FIXME: decode as bgelr 0
-# CHECK: bclr 4, 0, 0
+# CHECK: bclr 4, 0
 0x4c 0x80 0x00 0x20
 
 # FIXME: decode as bgectr 2
-# CHECK: bcctr 4, 8, 0
+# CHECK: bcctr 4, 8
 0x4c 0x88 0x04 0x20
 
 # FIXME: decode as bgectr 0
-# CHECK: bcctr 4, 0, 0
+# CHECK: bcctr 4, 0
 0x4c 0x80 0x04 0x20
 
 # FIXME: decode as bgelrl 2
-# CHECK: bclrl 4, 8, 0
+# CHECK: bclrl 4, 8
 0x4c 0x88 0x00 0x21
 
 # FIXME: decode as bgelrl 0
-# CHECK: bclrl 4, 0, 0
+# CHECK: bclrl 4, 0
 0x4c 0x80 0x00 0x21
 
 # FIXME: decode as bgectrl 2
-# CHECK: bcctrl 4, 8, 0
+# CHECK: bcctrl 4, 8
 0x4c 0x88 0x04 0x21
 
 # FIXME: decode as bgectrl 0
-# CHECK: bcctrl 4, 0, 0
+# CHECK: bcctrl 4, 0
 0x4c 0x80 0x04 0x21
 
 # FIXME: decode as bgelr+ 2
-# CHECK: bclr 7, 8, 0
+# CHECK: bclr 7, 8
 0x4c 0xe8 0x00 0x20
 
 # FIXME: decode as bgelr+ 0
-# CHECK: bclr 7, 0, 0
+# CHECK: bclr 7, 0
 0x4c 0xe0 0x00 0x20
 
 # FIXME: decode as bgectr+ 2
-# CHECK: bcctr 7, 8, 0
+# CHECK: bcctr 7, 8
 0x4c 0xe8 0x04 0x20
 
 # FIXME: decode as bgectr+ 0
-# CHECK: bcctr 7, 0, 0
+# CHECK: bcctr 7, 0
 0x4c 0xe0 0x04 0x20
 
 # FIXME: decode as bgelrl+ 2
-# CHECK: bclrl 7, 8, 0
+# CHECK: bclrl 7, 8
 0x4c 0xe8 0x00 0x21
 
 # FIXME: decode as bgelrl+ 0
-# CHECK: bclrl 7, 0, 0
+# CHECK: bclrl 7, 0
 0x4c 0xe0 0x00 0x21
 
 # FIXME: decode as bgectrl+ 2
-# CHECK: bcctrl 7, 8, 0
+# CHECK: bcctrl 7, 8
 0x4c 0xe8 0x04 0x21
 
 # FIXME: decode as bgectrl+ 0
-# CHECK: bcctrl 7, 0, 0
+# CHECK: bcctrl 7, 0
 0x4c 0xe0 0x04 0x21
 
 # FIXME: decode as bgelr- 2
-# CHECK: bclr 6, 8, 0
+# CHECK: bclr 6, 8
 0x4c 0xc8 0x00 0x20
 
 # FIXME: decode as bgelr- 0
-# CHECK: bclr 6, 0, 0
+# CHECK: bclr 6, 0
 0x4c 0xc0 0x00 0x20
 
 # FIXME: decode as bgectr- 2
-# CHECK: bcctr 6, 8, 0
+# CHECK: bcctr 6, 8
 0x4c 0xc8 0x04 0x20
 
 # FIXME: decode as bgectr- 0
-# CHECK: bcctr 6, 0, 0
+# CHECK: bcctr 6, 0
 0x4c 0xc0 0x04 0x20
 
 # FIXME: decode as bgelrl- 2
-# CHECK: bclrl 6, 8, 0
+# CHECK: bclrl 6, 8
 0x4c 0xc8 0x00 0x21
 
 # FIXME: decode as bgelrl- 0
-# CHECK: bclrl 6, 0, 0
+# CHECK: bclrl 6, 0
 0x4c 0xc0 0x00 0x21
 
 # FIXME: decode as bgectrl- 2
-# CHECK: bcctrl 6, 8, 0
+# CHECK: bcctrl 6, 8
 0x4c 0xc8 0x04 0x21
 
 # FIXME: decode as bgectrl- 0
-# CHECK: bcctrl 6, 0, 0
+# CHECK: bcctrl 6, 0
 0x4c 0xc0 0x04 0x21
 
 # FIXME: decode as bgtlr 2
-# CHECK: bclr 12, 9, 0
+# CHECK: bclr 12, 9
 0x4d 0x89 0x00 0x20
 
 # FIXME: decode as bgtlr 0
-# CHECK: bclr 12, 1, 0
+# CHECK: bclr 12, 1
 0x4d 0x81 0x00 0x20
 
 # FIXME: decode as bgtctr 2
-# CHECK: bcctr 12, 9, 0
+# CHECK: bcctr 12, 9
 0x4d 0x89 0x04 0x20
 
 # FIXME: decode as bgtctr 0
-# CHECK: bcctr 12, 1, 0
+# CHECK: bcctr 12, 1
 0x4d 0x81 0x04 0x20
 
 # FIXME: decode as bgtlrl 2
-# CHECK: bclrl 12, 9, 0
+# CHECK: bclrl 12, 9
 0x4d 0x89 0x00 0x21
 
 # FIXME: decode as bgtlrl 0
-# CHECK: bclrl 12, 1, 0
+# CHECK: bclrl 12, 1
 0x4d 0x81 0x00 0x21
 
 # FIXME: decode as bgtctrl 2
-# CHECK: bcctrl 12, 9, 0
+# CHECK: bcctrl 12, 9
 0x4d 0x89 0x04 0x21
 
 # FIXME: decode as bgtctrl 0
-# CHECK: bcctrl 12, 1, 0
+# CHECK: bcctrl 12, 1
 0x4d 0x81 0x04 0x21
 
 # FIXME: decode as bgtlr+ 2
-# CHECK: bclr 15, 9, 0
+# CHECK: bclr 15, 9
 0x4d 0xe9 0x00 0x20
 
 # FIXME: decode as bgtlr+ 0
-# CHECK: bclr 15, 1, 0
+# CHECK: bclr 15, 1
 0x4d 0xe1 0x00 0x20
 
 # FIXME: decode as bgtctr+ 2
-# CHECK: bcctr 15, 9, 0
+# CHECK: bcctr 15, 9
 0x4d 0xe9 0x04 0x20
 
 # FIXME: decode as bgtctr+ 0
-# CHECK: bcctr 15, 1, 0
+# CHECK: bcctr 15, 1
 0x4d 0xe1 0x04 0x20
 
 # FIXME: decode as bgtlrl+ 2
-# CHECK: bclrl 15, 9, 0
+# CHECK: bclrl 15, 9
 0x4d 0xe9 0x00 0x21
 
 # FIXME: decode as bgtlrl+ 0
-# CHECK: bclrl 15, 1, 0
+# CHECK: bclrl 15, 1
 0x4d 0xe1 0x00 0x21
 
 # FIXME: decode as bgtctrl+ 2
-# CHECK: bcctrl 15, 9, 0
+# CHECK: bcctrl 15, 9
 0x4d 0xe9 0x04 0x21
 
 # FIXME: decode as bgtctrl+ 0
-# CHECK: bcctrl 15, 1, 0
+# CHECK: bcctrl 15, 1
 0x4d 0xe1 0x04 0x21
 
 # FIXME: decode as bgtlr- 2
-# CHECK: bclr 14, 9, 0
+# CHECK: bclr 14, 9
 0x4d 0xc9 0x00 0x20
 
 # FIXME: decode as bgtlr- 0
-# CHECK: bclr 14, 1, 0
+# CHECK: bclr 14, 1
 0x4d 0xc1 0x00 0x20
 
 # FIXME: decode as bgtctr- 2
-# CHECK: bcctr 14, 9, 0
+# CHECK: bcctr 14, 9
 0x4d 0xc9 0x04 0x20
 
 # FIXME: decode as bgtctr- 0
-# CHECK: bcctr 14, 1, 0
+# CHECK: bcctr 14, 1
 0x4d 0xc1 0x04 0x20
 
 # FIXME: decode as bgtlrl- 2
-# CHECK: bclrl 14, 9, 0
+# CHECK: bclrl 14, 9
 0x4d 0xc9 0x00 0x21
 
 # FIXME: decode as bgtlrl- 0
-# CHECK: bclrl 14, 1, 0
+# CHECK: bclrl 14, 1
 0x4d 0xc1 0x00 0x21
 
 # FIXME: decode as bgtctrl- 2
-# CHECK: bcctrl 14, 9, 0
+# CHECK: bcctrl 14, 9
 0x4d 0xc9 0x04 0x21
 
 # FIXME: decode as bgtctrl- 0
-# CHECK: bcctrl 14, 1, 0
+# CHECK: bcctrl 14, 1
 0x4d 0xc1 0x04 0x21
 
 # FIXME: decode as bgelr 2
-# CHECK: bclr 4, 8, 0
+# CHECK: bclr 4, 8
 0x4c 0x88 0x00 0x20
 
 # FIXME: decode as bgelr 0
-# CHECK: bclr 4, 0, 0
+# CHECK: bclr 4, 0
 0x4c 0x80 0x00 0x20
 
 # FIXME: decode as bgectr 2
-# CHECK: bcctr 4, 8, 0
+# CHECK: bcctr 4, 8
 0x4c 0x88 0x04 0x20
 
 # FIXME: decode as bgectr 0
-# CHECK: bcctr 4, 0, 0
+# CHECK: bcctr 4, 0
 0x4c 0x80 0x04 0x20
 
 # FIXME: decode as bgelrl 2
-# CHECK: bclrl 4, 8, 0
+# CHECK: bclrl 4, 8
 0x4c 0x88 0x00 0x21
 
 # FIXME: decode as bgelrl 0
-# CHECK: bclrl 4, 0, 0
+# CHECK: bclrl 4, 0
 0x4c 0x80 0x00 0x21
 
 # FIXME: decode as bgectrl 2
-# CHECK: bcctrl 4, 8, 0
+# CHECK: bcctrl 4, 8
 0x4c 0x88 0x04 0x21
 
 # FIXME: decode as bgectrl 0
-# CHECK: bcctrl 4, 0, 0
+# CHECK: bcctrl 4, 0
 0x4c 0x80 0x04 0x21
 
 # FIXME: decode as bgelr+ 2
-# CHECK: bclr 7, 8, 0
+# CHECK: bclr 7, 8
 0x4c 0xe8 0x00 0x20
 
 # FIXME: decode as bgelr+ 0
-# CHECK: bclr 7, 0, 0
+# CHECK: bclr 7, 0
 0x4c 0xe0 0x00 0x20
 
 # FIXME: decode as bgectr+ 2
-# CHECK: bcctr 7, 8, 0
+# CHECK: bcctr 7, 8
 0x4c 0xe8 0x04 0x20
 
 # FIXME: decode as bgectr+ 0
-# CHECK: bcctr 7, 0, 0
+# CHECK: bcctr 7, 0
 0x4c 0xe0 0x04 0x20
 
 # FIXME: decode as bgelrl+ 2
-# CHECK: bclrl 7, 8, 0
+# CHECK: bclrl 7, 8
 0x4c 0xe8 0x00 0x21
 
 # FIXME: decode as bgelrl+ 0
-# CHECK: bclrl 7, 0, 0
+# CHECK: bclrl 7, 0
 0x4c 0xe0 0x00 0x21
 
 # FIXME: decode as bgectrl+ 2
-# CHECK: bcctrl 7, 8, 0
+# CHECK: bcctrl 7, 8
 0x4c 0xe8 0x04 0x21
 
 # FIXME: decode as bgectrl+ 0
-# CHECK: bcctrl 7, 0, 0
+# CHECK: bcctrl 7, 0
 0x4c 0xe0 0x04 0x21
 
 # FIXME: decode as bgelr- 2
-# CHECK: bclr 6, 8, 0
+# CHECK: bclr 6, 8
 0x4c 0xc8 0x00 0x20
 
 # FIXME: decode as bgelr- 0
-# CHECK: bclr 6, 0, 0
+# CHECK: bclr 6, 0
 0x4c 0xc0 0x00 0x20
 
 # FIXME: decode as bgectr- 2
-# CHECK: bcctr 6, 8, 0
+# CHECK: bcctr 6, 8
 0x4c 0xc8 0x04 0x20
 
 # FIXME: decode as bgectr- 0
-# CHECK: bcctr 6, 0, 0
+# CHECK: bcctr 6, 0
 0x4c 0xc0 0x04 0x20
 
 # FIXME: decode as bgelrl- 2
-# CHECK: bclrl 6, 8, 0
+# CHECK: bclrl 6, 8
 0x4c 0xc8 0x00 0x21
 
 # FIXME: decode as bgelrl- 0
-# CHECK: bclrl 6, 0, 0
+# CHECK: bclrl 6, 0
 0x4c 0xc0 0x00 0x21
 
 # FIXME: decode as bgectrl- 2
-# CHECK: bcctrl 6, 8, 0
+# CHECK: bcctrl 6, 8
 0x4c 0xc8 0x04 0x21
 
 # FIXME: decode as bgectrl- 0
-# CHECK: bcctrl 6, 0, 0
+# CHECK: bcctrl 6, 0
 0x4c 0xc0 0x04 0x21
 
 # FIXME: decode as bnelr 2
-# CHECK: bclr 4, 10, 0
+# CHECK: bclr 4, 10
 0x4c 0x8a 0x00 0x20
 
 # FIXME: decode as bnelr 0
-# CHECK: bclr 4, 2, 0
+# CHECK: bclr 4, 2
 0x4c 0x82 0x00 0x20
 
 # FIXME: decode as bnectr 2
-# CHECK: bcctr 4, 10, 0
+# CHECK: bcctr 4, 10
 0x4c 0x8a 0x04 0x20
 
 # FIXME: decode as bnectr 0
-# CHECK: bcctr 4, 2, 0
+# CHECK: bcctr 4, 2
 0x4c 0x82 0x04 0x20
 
 # FIXME: decode as bnelrl 2
-# CHECK: bclrl 4, 10, 0
+# CHECK: bclrl 4, 10
 0x4c 0x8a 0x00 0x21
 
 # FIXME: decode as bnelrl 0
-# CHECK: bclrl 4, 2, 0
+# CHECK: bclrl 4, 2
 0x4c 0x82 0x00 0x21
 
 # FIXME: decode as bnectrl 2
-# CHECK: bcctrl 4, 10, 0
+# CHECK: bcctrl 4, 10
 0x4c 0x8a 0x04 0x21
 
 # FIXME: decode as bnectrl 0
-# CHECK: bcctrl 4, 2, 0
+# CHECK: bcctrl 4, 2
 0x4c 0x82 0x04 0x21
 
 # FIXME: decode as bnelr+ 2
-# CHECK: bclr 7, 10, 0
+# CHECK: bclr 7, 10
 0x4c 0xea 0x00 0x20
 
 # FIXME: decode as bnelr+ 0
-# CHECK: bclr 7, 2, 0
+# CHECK: bclr 7, 2
 0x4c 0xe2 0x00 0x20
 
 # FIXME: decode as bnectr+ 2
-# CHECK: bcctr 7, 10, 0
+# CHECK: bcctr 7, 10
 0x4c 0xea 0x04 0x20
 
 # FIXME: decode as bnectr+ 0
-# CHECK: bcctr 7, 2, 0
+# CHECK: bcctr 7, 2
 0x4c 0xe2 0x04 0x20
 
 # FIXME: decode as bnelrl+ 2
-# CHECK: bclrl 7, 10, 0
+# CHECK: bclrl 7, 10
 0x4c 0xea 0x00 0x21
 
 # FIXME: decode as bnelrl+ 0
-# CHECK: bclrl 7, 2, 0
+# CHECK: bclrl 7, 2
 0x4c 0xe2 0x00 0x21
 
 # FIXME: decode as bnectrl+ 2
-# CHECK: bcctrl 7, 10, 0
+# CHECK: bcctrl 7, 10
 0x4c 0xea 0x04 0x21
 
 # FIXME: decode as bnectrl+ 0
-# CHECK: bcctrl 7, 2, 0
+# CHECK: bcctrl 7, 2
 0x4c 0xe2 0x04 0x21
 
 # FIXME: decode as bnelr- 2
-# CHECK: bclr 6, 10, 0
+# CHECK: bclr 6, 10
 0x4c 0xca 0x00 0x20
 
 # FIXME: decode as bnelr- 0
-# CHECK: bclr 6, 2, 0
+# CHECK: bclr 6, 2
 0x4c 0xc2 0x00 0x20
 
 # FIXME: decode as bnectr- 2
-# CHECK: bcctr 6, 10, 0
+# CHECK: bcctr 6, 10
 0x4c 0xca 0x04 0x20
 
 # FIXME: decode as bnectr- 0
-# CHECK: bcctr 6, 2, 0
+# CHECK: bcctr 6, 2
 0x4c 0xc2 0x04 0x20
 
 # FIXME: decode as bnelrl- 2
-# CHECK: bclrl 6, 10, 0
+# CHECK: bclrl 6, 10
 0x4c 0xca 0x00 0x21
 
 # FIXME: decode as bnelrl- 0
-# CHECK: bclrl 6, 2, 0
+# CHECK: bclrl 6, 2
 0x4c 0xc2 0x00 0x21
 
 # FIXME: decode as bnectrl- 2
-# CHECK: bcctrl 6, 10, 0
+# CHECK: bcctrl 6, 10
 0x4c 0xca 0x04 0x21
 
 # FIXME: decode as bnectrl- 0
-# CHECK: bcctrl 6, 2, 0
+# CHECK: bcctrl 6, 2
 0x4c 0xc2 0x04 0x21
 
 # FIXME: decode as blelr 2
-# CHECK: bclr 4, 9, 0
+# CHECK: bclr 4, 9
 0x4c 0x89 0x00 0x20
 
 # FIXME: decode as blelr 0
-# CHECK: bclr 4, 1, 0
+# CHECK: bclr 4, 1
 0x4c 0x81 0x00 0x20
 
 # FIXME: decode as blectr 2
-# CHECK: bcctr 4, 9, 0
+# CHECK: bcctr 4, 9
 0x4c 0x89 0x04 0x20
 
 # FIXME: decode as blectr 0
-# CHECK: bcctr 4, 1, 0
+# CHECK: bcctr 4, 1
 0x4c 0x81 0x04 0x20
 
 # FIXME: decode as blelrl 2
-# CHECK: bclrl 4, 9, 0
+# CHECK: bclrl 4, 9
 0x4c 0x89 0x00 0x21
 
 # FIXME: decode as blelrl 0
-# CHECK: bclrl 4, 1, 0
+# CHECK: bclrl 4, 1
 0x4c 0x81 0x00 0x21
 
 # FIXME: decode as blectrl 2
-# CHECK: bcctrl 4, 9, 0
+# CHECK: bcctrl 4, 9
 0x4c 0x89 0x04 0x21
 
 # FIXME: decode as blectrl 0
-# CHECK: bcctrl 4, 1, 0
+# CHECK: bcctrl 4, 1
 0x4c 0x81 0x04 0x21
 
 # FIXME: decode as blelr+ 2
-# CHECK: bclr 7, 9, 0
+# CHECK: bclr 7, 9
 0x4c 0xe9 0x00 0x20
 
 # FIXME: decode as blelr+ 0
-# CHECK: bclr 7, 1, 0
+# CHECK: bclr 7, 1
 0x4c 0xe1 0x00 0x20
 
 # FIXME: decode as blectr+ 2
-# CHECK: bcctr 7, 9, 0
+# CHECK: bcctr 7, 9
 0x4c 0xe9 0x04 0x20
 
 # FIXME: decode as blectr+ 0
-# CHECK: bcctr 7, 1, 0
+# CHECK: bcctr 7, 1
 0x4c 0xe1 0x04 0x20
 
 # FIXME: decode as blelrl+ 2
-# CHECK: bclrl 7, 9, 0
+# CHECK: bclrl 7, 9
 0x4c 0xe9 0x00 0x21
 
 # FIXME: decode as blelrl+ 0
-# CHECK: bclrl 7, 1, 0
+# CHECK: bclrl 7, 1
 0x4c 0xe1 0x00 0x21
 
 # FIXME: decode as blectrl+ 2
-# CHECK: bcctrl 7, 9, 0
+# CHECK: bcctrl 7, 9
 0x4c 0xe9 0x04 0x21
 
 # FIXME: decode as blectrl+ 0
-# CHECK: bcctrl 7, 1, 0
+# CHECK: bcctrl 7, 1
 0x4c 0xe1 0x04 0x21
 
 # FIXME: decode as blelr- 2
-# CHECK: bclr 6, 9, 0
+# CHECK: bclr 6, 9
 0x4c 0xc9 0x00 0x20
 
 # FIXME: decode as blelr- 0
-# CHECK: bclr 6, 1, 0
+# CHECK: bclr 6, 1
 0x4c 0xc1 0x00 0x20
 
 # FIXME: decode as blectr- 2
-# CHECK: bcctr 6, 9, 0
+# CHECK: bcctr 6, 9
 0x4c 0xc9 0x04 0x20
 
 # FIXME: decode as blectr- 0
-# CHECK: bcctr 6, 1, 0
+# CHECK: bcctr 6, 1
 0x4c 0xc1 0x04 0x20
 
 # FIXME: decode as blelrl- 2
-# CHECK: bclrl 6, 9, 0
+# CHECK: bclrl 6, 9
 0x4c 0xc9 0x00 0x21
 
 # FIXME: decode as blelrl- 0
-# CHECK: bclrl 6, 1, 0
+# CHECK: bclrl 6, 1
 0x4c 0xc1 0x00 0x21
 
 # FIXME: decode as blectrl- 2
-# CHECK: bcctrl 6, 9, 0
+# CHECK: bcctrl 6, 9
 0x4c 0xc9 0x04 0x21
 
 # FIXME: decode as blectrl- 0
-# CHECK: bcctrl 6, 1, 0
+# CHECK: bcctrl 6, 1
 0x4c 0xc1 0x04 0x21
 
 # FIXME: decode as bunlr 2
-# CHECK: bclr 12, 11, 0
+# CHECK: bclr 12, 11
 0x4d 0x8b 0x00 0x20
 
 # FIXME: decode as bunlr 0
-# CHECK: bclr 12, 3, 0
+# CHECK: bclr 12, 3
 0x4d 0x83 0x00 0x20
 
 # FIXME: decode as bunctr 2
-# CHECK: bcctr 12, 11, 0
+# CHECK: bcctr 12, 11
 0x4d 0x8b 0x04 0x20
 
 # FIXME: decode as bunctr 0
-# CHECK: bcctr 12, 3, 0
+# CHECK: bcctr 12, 3
 0x4d 0x83 0x04 0x20
 
 # FIXME: decode as bunlrl 2
-# CHECK: bclrl 12, 11, 0
+# CHECK: bclrl 12, 11
 0x4d 0x8b 0x00 0x21
 
 # FIXME: decode as bunlrl 0
-# CHECK: bclrl 12, 3, 0
+# CHECK: bclrl 12, 3
 0x4d 0x83 0x00 0x21
 
 # FIXME: decode as bunctrl 2
-# CHECK: bcctrl 12, 11, 0
+# CHECK: bcctrl 12, 11
 0x4d 0x8b 0x04 0x21
 
 # FIXME: decode as bunctrl 0
-# CHECK: bcctrl 12, 3, 0
+# CHECK: bcctrl 12, 3
 0x4d 0x83 0x04 0x21
 
 # FIXME: decode as bunlr+ 2
-# CHECK: bclr 15, 11, 0
+# CHECK: bclr 15, 11
 0x4d 0xeb 0x00 0x20
 
 # FIXME: decode as bunlr+ 0
-# CHECK: bclr 15, 3, 0
+# CHECK: bclr 15, 3
 0x4d 0xe3 0x00 0x20
 
 # FIXME: decode as bunctr+ 2
-# CHECK: bcctr 15, 11, 0
+# CHECK: bcctr 15, 11
 0x4d 0xeb 0x04 0x20
 
 # FIXME: decode as bunctr+ 0
-# CHECK: bcctr 15, 3, 0
+# CHECK: bcctr 15, 3
 0x4d 0xe3 0x04 0x20
 
 # FIXME: decode as bunlrl+ 2
-# CHECK: bclrl 15, 11, 0
+# CHECK: bclrl 15, 11
 0x4d 0xeb 0x00 0x21
 
 # FIXME: decode as bunlrl+ 0
-# CHECK: bclrl 15, 3, 0
+# CHECK: bclrl 15, 3
 0x4d 0xe3 0x00 0x21
 
 # FIXME: decode as bunctrl+ 2
-# CHECK: bcctrl 15, 11, 0
+# CHECK: bcctrl 15, 11
 0x4d 0xeb 0x04 0x21
 
 # FIXME: decode as bunctrl+ 0
-# CHECK: bcctrl 15, 3, 0
+# CHECK: bcctrl 15, 3
 0x4d 0xe3 0x04 0x21
 
 # FIXME: decode as bunlr- 2
-# CHECK: bclr 14, 11, 0
+# CHECK: bclr 14, 11
 0x4d 0xcb 0x00 0x20
 
 # FIXME: decode as bunlr- 0
-# CHECK: bclr 14, 3, 0
+# CHECK: bclr 14, 3
 0x4d 0xc3 0x00 0x20
 
 # FIXME: decode as bunctr- 2
-# CHECK: bcctr 14, 11, 0
+# CHECK: bcctr 14, 11
 0x4d 0xcb 0x04 0x20
 
 # FIXME: decode as bunctr- 0
-# CHECK: bcctr 14, 3, 0
+# CHECK: bcctr 14, 3
 0x4d 0xc3 0x04 0x20
 
 # FIXME: decode as bunlrl- 2
-# CHECK: bclrl 14, 11, 0
+# CHECK: bclrl 14, 11
 0x4d 0xcb 0x00 0x21
 
 # FIXME: decode as bunlrl- 0
-# CHECK: bclrl 14, 3, 0
+# CHECK: bclrl 14, 3
 0x4d 0xc3 0x00 0x21
 
 # FIXME: decode as bunctrl- 2
-# CHECK: bcctrl 14, 11, 0
+# CHECK: bcctrl 14, 11
 0x4d 0xcb 0x04 0x21
 
 # FIXME: decode as bunctrl- 0
-# CHECK: bcctrl 14, 3, 0
+# CHECK: bcctrl 14, 3
 0x4d 0xc3 0x04 0x21
 
 # FIXME: decode as bnulr 2
-# CHECK: bclr 4, 11, 0
+# CHECK: bclr 4, 11
 0x4c 0x8b 0x00 0x20
 
 # FIXME: decode as bnulr 0
-# CHECK: bclr 4, 3, 0
+# CHECK: bclr 4, 3
 0x4c 0x83 0x00 0x20
 
 # FIXME: decode as bnuctr 2
-# CHECK: bcctr 4, 11, 0
+# CHECK: bcctr 4, 11
 0x4c 0x8b 0x04 0x20
 
 # FIXME: decode as bnuctr 0
-# CHECK: bcctr 4, 3, 0
+# CHECK: bcctr 4, 3
 0x4c 0x83 0x04 0x20
 
 # FIXME: decode as bnulrl 2
-# CHECK: bclrl 4, 11, 0
+# CHECK: bclrl 4, 11
 0x4c 0x8b 0x00 0x21
 
 # FIXME: decode as bnulrl 0
-# CHECK: bclrl 4, 3, 0
+# CHECK: bclrl 4, 3
 0x4c 0x83 0x00 0x21
 
 # FIXME: decode as bnuctrl 2
-# CHECK: bcctrl 4, 11, 0
+# CHECK: bcctrl 4, 11
 0x4c 0x8b 0x04 0x21
 
 # FIXME: decode as bnuctrl 0
-# CHECK: bcctrl 4, 3, 0
+# CHECK: bcctrl 4, 3
 0x4c 0x83 0x04 0x21
 
 # FIXME: decode as bnulr+ 2
-# CHECK: bclr 7, 11, 0
+# CHECK: bclr 7, 11
 0x4c 0xeb 0x00 0x20
 
 # FIXME: decode as bnulr+ 0
-# CHECK: bclr 7, 3, 0
+# CHECK: bclr 7, 3
 0x4c 0xe3 0x00 0x20
 
 # FIXME: decode as bnuctr+ 2
-# CHECK: bcctr 7, 11, 0
+# CHECK: bcctr 7, 11
 0x4c 0xeb 0x04 0x20
 
 # FIXME: decode as bnuctr+ 0
-# CHECK: bcctr 7, 3, 0
+# CHECK: bcctr 7, 3
 0x4c 0xe3 0x04 0x20
 
 # FIXME: decode as bnulrl+ 2
-# CHECK: bclrl 7, 11, 0
+# CHECK: bclrl 7, 11
 0x4c 0xeb 0x00 0x21
 
 # FIXME: decode as bnulrl+ 0
-# CHECK: bclrl 7, 3, 0
+# CHECK: bclrl 7, 3
 0x4c 0xe3 0x00 0x21
 
 # FIXME: decode as bnuctrl+ 2
-# CHECK: bcctrl 7, 11, 0
+# CHECK: bcctrl 7, 11
 0x4c 0xeb 0x04 0x21
 
 # FIXME: decode as bnuctrl+ 0
-# CHECK: bcctrl 7, 3, 0
+# CHECK: bcctrl 7, 3
 0x4c 0xe3 0x04 0x21
 
 # FIXME: decode as bnulr- 2
-# CHECK: bclr 6, 11, 0
+# CHECK: bclr 6, 11
 0x4c 0xcb 0x00 0x20
 
 # FIXME: decode as bnulr- 0
-# CHECK: bclr 6, 3, 0
+# CHECK: bclr 6, 3
 0x4c 0xc3 0x00 0x20
 
 # FIXME: decode as bnuctr- 2
-# CHECK: bcctr 6, 11, 0
+# CHECK: bcctr 6, 11
 0x4c 0xcb 0x04 0x20
 
 # FIXME: decode as bnuctr- 0
-# CHECK: bcctr 6, 3, 0
+# CHECK: bcctr 6, 3
 0x4c 0xc3 0x04 0x20
 
 # FIXME: decode as bnulrl- 2
-# CHECK: bclrl 6, 11, 0
+# CHECK: bclrl 6, 11
 0x4c 0xcb 0x00 0x21
 
 # FIXME: decode as bnulrl- 0
-# CHECK: bclrl 6, 3, 0
+# CHECK: bclrl 6, 3
 0x4c 0xc3 0x00 0x21
 
 # FIXME: decode as bnuctrl- 2
-# CHECK: bcctrl 6, 11, 0
+# CHECK: bcctrl 6, 11
 0x4c 0xcb 0x04 0x21
 
 # FIXME: decode as bnuctrl- 0
-# CHECK: bcctrl 6, 3, 0
+# CHECK: bcctrl 6, 3
 0x4c 0xc3 0x04 0x21
 
 # FIXME: decode as bunlr 2
-# CHECK: bclr 12, 11, 0
+# CHECK: bclr 12, 11
 0x4d 0x8b 0x00 0x20
 
 # FIXME: decode as bunlr 0
-# CHECK: bclr 12, 3, 0
+# CHECK: bclr 12, 3
 0x4d 0x83 0x00 0x20
 
 # FIXME: decode as bunctr 2
-# CHECK: bcctr 12, 11, 0
+# CHECK: bcctr 12, 11
 0x4d 0x8b 0x04 0x20
 
 # FIXME: decode as bunctr 0
-# CHECK: bcctr 12, 3, 0
+# CHECK: bcctr 12, 3
 0x4d 0x83 0x04 0x20
 
 # FIXME: decode as bunlrl 2
-# CHECK: bclrl 12, 11, 0
+# CHECK: bclrl 12, 11
 0x4d 0x8b 0x00 0x21
 
 # FIXME: decode as bunlrl 0
-# CHECK: bclrl 12, 3, 0
+# CHECK: bclrl 12, 3
 0x4d 0x83 0x00 0x21
 
 # FIXME: decode as bunctrl 2
-# CHECK: bcctrl 12, 11, 0
+# CHECK: bcctrl 12, 11
 0x4d 0x8b 0x04 0x21
 
 # FIXME: decode as bunctrl 0
-# CHECK: bcctrl 12, 3, 0
+# CHECK: bcctrl 12, 3
 0x4d 0x83 0x04 0x21
 
 # FIXME: decode as bunlr+ 2
-# CHECK: bclr 15, 11, 0
+# CHECK: bclr 15, 11
 0x4d 0xeb 0x00 0x20
 
 # FIXME: decode as bunlr+ 0
-# CHECK: bclr 15, 3, 0
+# CHECK: bclr 15, 3
 0x4d 0xe3 0x00 0x20
 
 # FIXME: decode as bunctr+ 2
-# CHECK: bcctr 15, 11, 0
+# CHECK: bcctr 15, 11
 0x4d 0xeb 0x04 0x20
 
 # FIXME: decode as bunctr+ 0
-# CHECK: bcctr 15, 3, 0
+# CHECK: bcctr 15, 3
 0x4d 0xe3 0x04 0x20
 
 # FIXME: decode as bunlrl+ 2
-# CHECK: bclrl 15, 11, 0
+# CHECK: bclrl 15, 11
 0x4d 0xeb 0x00 0x21
 
 # FIXME: decode as bunlrl+ 0
-# CHECK: bclrl 15, 3, 0
+# CHECK: bclrl 15, 3
 0x4d 0xe3 0x00 0x21
 
 # FIXME: decode as bunctrl+ 2
-# CHECK: bcctrl 15, 11, 0
+# CHECK: bcctrl 15, 11
 0x4d 0xeb 0x04 0x21
 
 # FIXME: decode as bunctrl+ 0
-# CHECK: bcctrl 15, 3, 0
+# CHECK: bcctrl 15, 3
 0x4d 0xe3 0x04 0x21
 
 # FIXME: decode as bunlr- 2
-# CHECK: bclr 14, 11, 0
+# CHECK: bclr 14, 11
 0x4d 0xcb 0x00 0x20
 
 # FIXME: decode as bunlr- 0
-# CHECK: bclr 14, 3, 0
+# CHECK: bclr 14, 3
 0x4d 0xc3 0x00 0x20
 
 # FIXME: decode as bunctr- 2
-# CHECK: bcctr 14, 11, 0
+# CHECK: bcctr 14, 11
 0x4d 0xcb 0x04 0x20
 
 # FIXME: decode as bunctr- 0
-# CHECK: bcctr 14, 3, 0
+# CHECK: bcctr 14, 3
 0x4d 0xc3 0x04 0x20
 
 # FIXME: decode as bunlrl- 2
-# CHECK: bclrl 14, 11, 0
+# CHECK: bclrl 14, 11
 0x4d 0xcb 0x00 0x21
 
 # FIXME: decode as bunlrl- 0
-# CHECK: bclrl 14, 3, 0
+# CHECK: bclrl 14, 3
 0x4d 0xc3 0x00 0x21
 
 # FIXME: decode as bunctrl- 2
-# CHECK: bcctrl 14, 11, 0
+# CHECK: bcctrl 14, 11
 0x4d 0xcb 0x04 0x21
 
 # FIXME: decode as bunctrl- 0
-# CHECK: bcctrl 14, 3, 0
+# CHECK: bcctrl 14, 3
 0x4d 0xc3 0x04 0x21
 
 # FIXME: decode as bnulr 2
-# CHECK: bclr 4, 11, 0
+# CHECK: bclr 4, 11
 0x4c 0x8b 0x00 0x20
 
 # FIXME: decode as bnulr 0
-# CHECK: bclr 4, 3, 0
+# CHECK: bclr 4, 3
 0x4c 0x83 0x00 0x20
 
 # FIXME: decode as bnuctr 2
-# CHECK: bcctr 4, 11, 0
+# CHECK: bcctr 4, 11
 0x4c 0x8b 0x04 0x20
 
 # FIXME: decode as bnuctr 0
-# CHECK: bcctr 4, 3, 0
+# CHECK: bcctr 4, 3
 0x4c 0x83 0x04 0x20
 
 # FIXME: decode as bnulrl 2
-# CHECK: bclrl 4, 11, 0
+# CHECK: bclrl 4, 11
 0x4c 0x8b 0x00 0x21
 
 # FIXME: decode as bnulrl 0
-# CHECK: bclrl 4, 3, 0
+# CHECK: bclrl 4, 3
 0x4c 0x83 0x00 0x21
 
 # FIXME: decode as bnuctrl 2
-# CHECK: bcctrl 4, 11, 0
+# CHECK: bcctrl 4, 11
 0x4c 0x8b 0x04 0x21
 
 # FIXME: decode as bnuctrl 0
-# CHECK: bcctrl 4, 3, 0
+# CHECK: bcctrl 4, 3
 0x4c 0x83 0x04 0x21
 
 # FIXME: decode as bnulr+ 2
-# CHECK: bclr 7, 11, 0
+# CHECK: bclr 7, 11
 0x4c 0xeb 0x00 0x20
 
 # FIXME: decode as bnulr+ 0
-# CHECK: bclr 7, 3, 0
+# CHECK: bclr 7, 3
 0x4c 0xe3 0x00 0x20
 
 # FIXME: decode as bnuctr+ 2
-# CHECK: bcctr 7, 11, 0
+# CHECK: bcctr 7, 11
 0x4c 0xeb 0x04 0x20
 
 # FIXME: decode as bnuctr+ 0
-# CHECK: bcctr 7, 3, 0
+# CHECK: bcctr 7, 3
 0x4c 0xe3 0x04 0x20
 
 # FIXME: decode as bnulrl+ 2
-# CHECK: bclrl 7, 11, 0
+# CHECK: bclrl 7, 11
 0x4c 0xeb 0x00 0x21
 
 # FIXME: decode as bnulrl+ 0
-# CHECK: bclrl 7, 3, 0
+# CHECK: bclrl 7, 3
 0x4c 0xe3 0x00 0x21
 
 # FIXME: decode as bnuctrl+ 2
-# CHECK: bcctrl 7, 11, 0
+# CHECK: bcctrl 7, 11
 0x4c 0xeb 0x04 0x21
 
 # FIXME: decode as bnuctrl+ 0
-# CHECK: bcctrl 7, 3, 0
+# CHECK: bcctrl 7, 3
 0x4c 0xe3 0x04 0x21
 
 # FIXME: decode as bnulr- 2
-# CHECK: bclr 6, 11, 0
+# CHECK: bclr 6, 11
 0x4c 0xcb 0x00 0x20
 
 # FIXME: decode as bnulr- 0
-# CHECK: bclr 6, 3, 0
+# CHECK: bclr 6, 3
 0x4c 0xc3 0x00 0x20
 
 # FIXME: decode as bnuctr- 2
-# CHECK: bcctr 6, 11, 0
+# CHECK: bcctr 6, 11
 0x4c 0xcb 0x04 0x20
 
 # FIXME: decode as bnuctr- 0
-# CHECK: bcctr 6, 3, 0
+# CHECK: bcctr 6, 3
 0x4c 0xc3 0x04 0x20
 
 # FIXME: decode as bnulrl- 2
-# CHECK: bclrl 6, 11, 0
+# CHECK: bclrl 6, 11
 0x4c 0xcb 0x00 0x21
 
 # FIXME: decode as bnulrl- 0
-# CHECK: bclrl 6, 3, 0
+# CHECK: bclrl 6, 3
 0x4c 0xc3 0x00 0x21
 
 # FIXME: decode as bnuctrl- 2
-# CHECK: bcctrl 6, 11, 0
+# CHECK: bcctrl 6, 11
 0x4c 0xcb 0x04 0x21
 
 # FIXME: decode as bnuctrl- 0
-# CHECK: bcctrl 6, 3, 0
+# CHECK: bcctrl 6, 3
 0x4c 0xc3 0x04 0x21
 
 # FIXME: test bc 12, 2, target
@@ -1801,16 +1801,16 @@
 # FIXME: test bnula- 2, target
 # FIXME: test bnula- 0, target
 
-# CHECK: creqv 2, 2, 2
+# CHECK: crset 2
 0x4c 0x42 0x12 0x42
 
-# CHECK: crxor 2, 2, 2
+# CHECK: crclr 2
 0x4c 0x42 0x11 0x82
 
-# CHECK: cror 2, 3, 3
+# CHECK: crmove 2, 3
 0x4c 0x43 0x1b 0x82
 
-# CHECK: crnor 2, 3, 3
+# CHECK: crnot 2, 3
 0x4c 0x43 0x18 0x42
 
 # CHECK: addi 2, 3, -128
@@ -1840,61 +1840,61 @@
 # CHECK: cmpdi 2, 3, 128
 0x2d 0x23 0x00 0x80
 
-# CHECK: cmpdi 0, 3, 128
+# CHECK: cmpdi 3, 128
 0x2c 0x23 0x00 0x80
 
 # CHECK: cmpd 2, 3, 4
 0x7d 0x23 0x20 0x00
 
-# CHECK: cmpd 0, 3, 4
+# CHECK: cmpd 3, 4
 0x7c 0x23 0x20 0x00
 
 # CHECK: cmpldi 2, 3, 128
 0x29 0x23 0x00 0x80
 
-# CHECK: cmpldi 0, 3, 128
+# CHECK: cmpldi 3, 128
 0x28 0x23 0x00 0x80
 
 # CHECK: cmpld 2, 3, 4
 0x7d 0x23 0x20 0x40
 
-# CHECK: cmpld 0, 3, 4
+# CHECK: cmpld 3, 4
 0x7c 0x23 0x20 0x40
 
 # CHECK: cmpwi 2, 3, 128
 0x2d 0x03 0x00 0x80
 
-# CHECK: cmpwi 0, 3, 128
+# CHECK: cmpwi 3, 128
 0x2c 0x03 0x00 0x80
 
 # CHECK: cmpw 2, 3, 4
 0x7d 0x03 0x20 0x00
 
-# CHECK: cmpw 0, 3, 4
+# CHECK: cmpw 3, 4
 0x7c 0x03 0x20 0x00
 
 # CHECK: cmplwi 2, 3, 128
 0x29 0x03 0x00 0x80
 
-# CHECK: cmplwi 0, 3, 128
+# CHECK: cmplwi 3, 128
 0x28 0x03 0x00 0x80
 
 # CHECK: cmplw 2, 3, 4
 0x7d 0x03 0x20 0x40
 
-# CHECK: cmplw 0, 3, 4
+# CHECK: cmplw 3, 4
 0x7c 0x03 0x20 0x40
 
-# CHECK: twi 16, 3, 4
+# CHECK: twlti 3, 4
 0x0e 0x03 0x00 0x04
 
-# CHECK: tw 16, 3, 4
+# CHECK: twlt 3, 4
 0x7e 0x03 0x20 0x08
 
-# CHECK: tdi 16, 3, 4
+# CHECK: tdlti 3, 4
 0x0a 0x03 0x00 0x04
 
-# CHECK: td 16, 3, 4
+# CHECK: tdlt 3, 4
 0x7e 0x03 0x20 0x88
 
 # CHECK: twi 20, 3, 4
@@ -1909,16 +1909,16 @@
 # CHECK: td 20, 3, 4
 0x7e 0x83 0x20 0x88
 
-# CHECK: twi 4, 3, 4
+# CHECK: tweqi 3, 4
 0x0c 0x83 0x00 0x04
 
-# CHECK: tw 4, 3, 4
+# CHECK: tweq 3, 4
 0x7c 0x83 0x20 0x08
 
-# CHECK: tdi 4, 3, 4
+# CHECK: tdeqi 3, 4
 0x08 0x83 0x00 0x04
 
-# CHECK: td 4, 3, 4
+# CHECK: tdeq 3, 4
 0x7c 0x83 0x20 0x88
 
 # CHECK: twi 12, 3, 4
@@ -1933,16 +1933,16 @@
 # CHECK: td 12, 3, 4
 0x7d 0x83 0x20 0x88
 
-# CHECK: twi 8, 3, 4
+# CHECK: twgti 3, 4
 0x0d 0x03 0x00 0x04
 
-# CHECK: tw 8, 3, 4
+# CHECK: twgt 3, 4
 0x7d 0x03 0x20 0x08
 
-# CHECK: tdi 8, 3, 4
+# CHECK: tdgti 3, 4
 0x09 0x03 0x00 0x04
 
-# CHECK: td 8, 3, 4
+# CHECK: tdgt 3, 4
 0x7d 0x03 0x20 0x88
 
 # CHECK: twi 12, 3, 4
@@ -1957,16 +1957,16 @@
 # CHECK: td 12, 3, 4
 0x7d 0x83 0x20 0x88
 
-# CHECK: twi 24, 3, 4
+# CHECK: twnei 3, 4
 0x0f 0x03 0x00 0x04
 
-# CHECK: tw 24, 3, 4
+# CHECK: twne 3, 4
 0x7f 0x03 0x20 0x08
 
-# CHECK: tdi 24, 3, 4
+# CHECK: tdnei 3, 4
 0x0b 0x03 0x00 0x04
 
-# CHECK: td 24, 3, 4
+# CHECK: tdne 3, 4
 0x7f 0x03 0x20 0x88
 
 # CHECK: twi 20, 3, 4
@@ -1981,16 +1981,16 @@
 # CHECK: td 20, 3, 4
 0x7e 0x83 0x20 0x88
 
-# CHECK: twi 2, 3, 4
+# CHECK: twllti 3, 4
 0x0c 0x43 0x00 0x04
 
-# CHECK: tw 2, 3, 4
+# CHECK: twllt 3, 4
 0x7c 0x43 0x20 0x08
 
-# CHECK: tdi 2, 3, 4
+# CHECK: tdllti 3, 4
 0x08 0x43 0x00 0x04
 
-# CHECK: td 2, 3, 4
+# CHECK: tdllt 3, 4
 0x7c 0x43 0x20 0x88
 
 # CHECK: twi 6, 3, 4
@@ -2017,16 +2017,16 @@
 # CHECK: td 5, 3, 4
 0x7c 0xa3 0x20 0x88
 
-# CHECK: twi 1, 3, 4
+# CHECK: twlgti 3, 4
 0x0c 0x23 0x00 0x04
 
-# CHECK: tw 1, 3, 4
+# CHECK: twlgt 3, 4
 0x7c 0x23 0x20 0x08
 
-# CHECK: tdi 1, 3, 4
+# CHECK: tdlgti 3, 4
 0x08 0x23 0x00 0x04
 
-# CHECK: td 1, 3, 4
+# CHECK: tdlgt 3, 4
 0x7c 0x23 0x20 0x88
 
 # CHECK: twi 5, 3, 4
@@ -2053,16 +2053,16 @@
 # CHECK: td 6, 3, 4
 0x7c 0xc3 0x20 0x88
 
-# CHECK: twi 31, 3, 4
+# CHECK: twui 3, 4
 0x0f 0xe3 0x00 0x04
 
-# CHECK: tw 31, 3, 4
+# CHECK: twu 3, 4
 0x7f 0xe3 0x20 0x08
 
-# CHECK: tdi 31, 3, 4
+# CHECK: tdui 3, 4
 0x0b 0xe3 0x00 0x04
 
-# CHECK: td 31, 3, 4
+# CHECK: tdu 3, 4
 0x7f 0xe3 0x20 0x88
 
 # CHECK: trap
@@ -2086,22 +2086,22 @@
 # CHECK: rldimi. 2, 3, 55, 5
 0x78 0x62 0xb9 0x4f
 
-# CHECK: rldicl 2, 3, 4, 0
+# CHECK: rotldi 2, 3, 4
 0x78 0x62 0x20 0x00
 
-# CHECK: rldicl. 2, 3, 4, 0
+# CHECK: rotldi. 2, 3, 4
 0x78 0x62 0x20 0x01
 
-# CHECK: rldicl 2, 3, 60, 0
+# CHECK: rotldi 2, 3, 60
 0x78 0x62 0xe0 0x02
 
-# CHECK: rldicl. 2, 3, 60, 0
+# CHECK: rotldi. 2, 3, 60
 0x78 0x62 0xe0 0x03
 
-# CHECK: rldcl 2, 3, 4, 0
+# CHECK: rotld 2, 3, 4
 0x78 0x62 0x20 0x10
 
-# CHECK: rldcl. 2, 3, 4, 0
+# CHECK: rotld. 2, 3, 4
 0x78 0x62 0x20 0x11
 
 # CHECK: sldi 2, 3, 4
@@ -2116,10 +2116,10 @@
 # CHECK: rldicl. 2, 3, 60, 4
 0x78 0x62 0xe1 0x03
 
-# CHECK: rldicl 2, 3, 0, 4
+# CHECK: clrldi 2, 3, 4
 0x78 0x62 0x01 0x00
 
-# CHECK: rldicl. 2, 3, 0, 4
+# CHECK: clrldi. 2, 3, 4
 0x78 0x62 0x01 0x01
 
 # CHECK: rldicr 2, 3, 0, 59
@@ -2158,22 +2158,22 @@
 # CHECK: rlwimi. 2, 3, 23, 5, 8
 0x50 0x62 0xb9 0x51
 
-# CHECK: rlwinm 2, 3, 4, 0, 31
+# CHECK: rotlwi 2, 3, 4
 0x54 0x62 0x20 0x3e
 
-# CHECK: rlwinm. 2, 3, 4, 0, 31
+# CHECK: rotlwi. 2, 3, 4
 0x54 0x62 0x20 0x3f
 
-# CHECK: rlwinm 2, 3, 28, 0, 31
+# CHECK: rotlwi 2, 3, 28
 0x54 0x62 0xe0 0x3e
 
-# CHECK: rlwinm. 2, 3, 28, 0, 31
+# CHECK: rotlwi. 2, 3, 28
 0x54 0x62 0xe0 0x3f
 
-# CHECK: rlwnm 2, 3, 4, 0, 31
+# CHECK: rotlw 2, 3, 4
 0x5c 0x62 0x20 0x3e
 
-# CHECK: rlwnm. 2, 3, 4, 0, 31
+# CHECK: rotlw. 2, 3, 4
 0x5c 0x62 0x20 0x3f
 
 # CHECK: slwi 2, 3, 4
@@ -2188,10 +2188,10 @@
 # CHECK: rlwinm. 2, 3, 28, 4, 31
 0x54 0x62 0xe1 0x3f
 
-# CHECK: rlwinm 2, 3, 0, 4, 31
+# CHECK: clrlwi 2, 3, 4
 0x54 0x62 0x01 0x3e
 
-# CHECK: rlwinm. 2, 3, 0, 4, 31
+# CHECK: clrlwi. 2, 3, 4
 0x54 0x62 0x01 0x3f
 
 # CHECK: rlwinm 2, 3, 0, 0, 27
@@ -2206,10 +2206,10 @@
 # CHECK: rlwinm. 2, 3, 4, 1, 27
 0x54 0x62 0x20 0x77
 
-# CHECK: mtspr 1, 2
+# CHECK: mtxer 2
 0x7c 0x41 0x03 0xa6
 
-# CHECK: mfspr 2, 1
+# CHECK: mfxer 2
 0x7c 0x41 0x02 0xa6
 
 # CHECK: mtlr 2
@@ -2227,7 +2227,7 @@
 # CHECK: nop
 0x60 0x00 0x00 0x00
 
-# CHECK: xori 0, 0, 0
+# CHECK: xnop
 0x68 0x00 0x00 0x00
 
 # CHECK: li 2, 128
diff --git a/test/MC/Disassembler/PowerPC/ppc64-encoding-p8vector.txt b/test/MC/Disassembler/PowerPC/ppc64-encoding-p8vector.txt
new file mode 100644
index 00000000000..eeea2db673f
--- /dev/null
+++ b/test/MC/Disassembler/PowerPC/ppc64-encoding-p8vector.txt
@@ -0,0 +1,19 @@
+# RUN: llvm-mc --disassemble %s -triple powerpc64-unknown-unknown -mcpu=pwr8 | FileCheck %s
+
+# CHECK: vpksdss 2, 3, 4
+0x10 0x43 0x25 0xce
+
+# CHECK: vpksdus 2, 3, 4
+0x10 0x43 0x25 0x4e
+
+# CHECK: vpkudus 2, 3, 4
+0x10 0x43 0x24 0xce
+
+# CHECK: vpkudum 2, 3, 4
+0x10 0x43 0x24 0x4e
+
+# CHECK: vupkhsw 2, 3
+0x10 0x40 0x1e 0x4e
+
+# CHECK: vupklsw 2, 3
+0x10 0x40 0x1e 0xce
diff --git a/test/MC/Disassembler/PowerPC/ppc64-encoding-vmx.txt b/test/MC/Disassembler/PowerPC/ppc64-encoding-vmx.txt
index 4424d69454c..16ff14c794d 100644
--- a/test/MC/Disassembler/PowerPC/ppc64-encoding-vmx.txt
+++ b/test/MC/Disassembler/PowerPC/ppc64-encoding-vmx.txt
@@ -99,6 +99,12 @@
 # CHECK: vmrglw 2, 3, 4                  
 0x10 0x43 0x21 0x8c
 
+# CHECK: vmrgew 2, 3, 4
+0x10 0x43 0x27 0x8c
+
+# CHECK: vmrgow 2, 3, 4
+0x10 0x43 0x26 0x8c
+
 # CHECK: vspltb 2, 3, 1                  
 0x10 0x41 0x1a 0x0c
 
@@ -207,6 +213,18 @@
 # CHECK: vadduws 2, 3, 4                 
 0x10 0x43 0x22 0x80
 
+# CHECK: vadduqm 2, 3, 4
+0x10 0x43 0x21 0x00
+
+# CHECK: vaddeuqm 2, 3, 4, 5
+0x10 0x43 0x21 0x7c
+
+# CHECK: vaddcuq 2, 3, 4
+0x10 0x43 0x21 0x40
+
+# CHECK: vaddecuq 2, 3, 4, 5
+0x10 0x43 0x21 0x7d
+
 # CHECK: vsubcuw 2, 3, 4                 
 0x10 0x43 0x25 0x80
 
@@ -240,6 +258,18 @@
 # CHECK: vsubuws 2, 3, 4                 
 0x10 0x43 0x26 0x80
 
+# CHECK: vsubuqm 2, 3, 4
+0x10 0x43 0x25 0x00
+
+# CHECK: vsubeuqm 2, 3, 4, 5
+0x10 0x43 0x21 0x7e
+
+# CHECK: vsubcuq 2, 3, 4
+0x10 0x43 0x25 0x40
+
+# CHECK: vsubecuq 2, 3, 4, 5
+0x10 0x43 0x21 0x7f
+
 # CHECK: vmulesb 2, 3, 4                 
 0x10 0x43 0x23 0x08
 
@@ -606,6 +636,12 @@
 # CHECK: vrsqrtefp 2, 3                  
 0x10 0x40 0x19 0x4a
 
+# CHECK: vgbbd 2, 3
+0x10 0x40 0x1d 0x0c
+
+# CHECK: vbpermq 2, 5, 17
+0x10 0x45 0x8d 0x4c
+
 # CHECK: vclzb 2, 3
 0x10 0x40 0x1f 0x02
 
diff --git a/test/MC/Disassembler/PowerPC/ppc64-encoding.txt b/test/MC/Disassembler/PowerPC/ppc64-encoding.txt
index f9cdb51fa2c..f235c242fbc 100644
--- a/test/MC/Disassembler/PowerPC/ppc64-encoding.txt
+++ b/test/MC/Disassembler/PowerPC/ppc64-encoding.txt
@@ -19,25 +19,25 @@
 # CHECK: bclr 4, 10, 3                   
 0x4c 0x8a 0x18 0x20
 
-# CHECK: bclr 4, 10, 0                   
+# CHECK: bclr 4, 10
 0x4c 0x8a 0x00 0x20
 
 # CHECK: bclrl 4, 10, 3                  
 0x4c 0x8a 0x18 0x21
 
-# CHECK: bclrl 4, 10, 0                  
+# CHECK: bclrl 4, 10
 0x4c 0x8a 0x00 0x21
 
 # CHECK: bcctr 4, 10, 3                  
 0x4c 0x8a 0x1c 0x20
 
-# CHECK: bcctr 4, 10, 0                  
+# CHECK: bcctr 4, 10
 0x4c 0x8a 0x04 0x20
 
 # CHECK: bcctrl 4, 10, 3                 
 0x4c 0x8a 0x1c 0x21
 
-# CHECK: bcctrl 4, 10, 0                 
+# CHECK: bcctrl 4, 10
 0x4c 0x8a 0x04 0x21
 
 # CHECK: crand 2, 3, 4                   
@@ -70,9 +70,18 @@
 # CHECK: sc 1                            
 0x44 0x00 0x00 0x22
 
-# CHECK: sc 0                            
+# CHECK: sc
 0x44 0x00 0x00 0x02
 
+# CHECK: clrbhrb
+0x7c 0x00 0x03 0x5c
+
+# CHECK: mfbhrbe 9, 983
+0x7d 0x3e 0xba 0x5c
+
+# CHECK: rfebb 1
+0x4c 0x00 0x09 0x24
+
 # CHECK: lbz 2, 128(4)                   
 0x88 0x44 0x00 0x80
 
@@ -406,16 +415,16 @@
 # CHECK: cmplw 2, 3, 4                   
 0x7d 0x03 0x20 0x40
 
-# CHECK: twi 2, 3, 4                     
+# CHECK: twllti 3, 4
 0x0c 0x43 0x00 0x04
 
-# CHECK: tw 2, 3, 4                      
+# CHECK: twllt 3, 4
 0x7c 0x43 0x20 0x08
 
-# CHECK: tdi 2, 3, 4                     
+# CHECK: tdllti 3, 4
 0x08 0x43 0x00 0x04
 
-# CHECK: td 2, 3, 4                      
+# CHECK: tdllt 3, 4
 0x7c 0x43 0x20 0x88
 
 # CHECK: isel 2, 3, 4, 5                 
@@ -499,10 +508,10 @@
 # CHECK: extsh. 2, 3                     
 0x7c 0x62 0x07 0x35
 
-# CHECK: cntlzw 2, 3                     
+# CHECK: cntlz 2, 3
 0x7c 0x62 0x00 0x34
 
-# CHECK: cntlzw. 2, 3                    
+# CHECK: cntlz. 2, 3
 0x7c 0x62 0x00 0x35
 
 # CHECK: popcntw 2, 3                    
diff --git a/test/MC/Disassembler/PowerPC/ppc64-operands.txt b/test/MC/Disassembler/PowerPC/ppc64-operands.txt
index a2da3227f7c..3fcba735584 100644
--- a/test/MC/Disassembler/PowerPC/ppc64-operands.txt
+++ b/test/MC/Disassembler/PowerPC/ppc64-operands.txt
@@ -85,10 +85,10 @@
 0x48 0x00 0x04 0x02
 
 # FIXME: decode as beq 0, .+1024
-# CHECK: bc 12, 2, .+1024
+# CHECK: bt 2, .+1024
 0x41 0x82 0x04 0x00
 
 # FIXME: decode as beqa 0, 1024
-# CHECK: bca 12, 2, 1024
+# CHECK: bta 2, 1024
 0x41 0x82 0x04 0x02
 
diff --git a/test/MC/Disassembler/PowerPC/ppc64le-encoding.txt b/test/MC/Disassembler/PowerPC/ppc64le-encoding.txt
new file mode 100644
index 00000000000..f154e00ff51
--- /dev/null
+++ b/test/MC/Disassembler/PowerPC/ppc64le-encoding.txt
@@ -0,0 +1,664 @@
+# RUN: llvm-mc --disassemble %s -triple powerpc64le-unknown-unknown -mcpu=pwr7 | FileCheck %s
+
+# FIXME: test b target
+
+# FIXME: test ba target
+
+# FIXME: test bl target
+
+# FIXME: test bla target
+
+# FIXME: test bc 4, 10, target
+
+# FIXME: test bca 4, 10, target
+
+# FIXME: test bcl 4, 10, target
+
+# FIXME: test bcla 4, 10, target
+
+# CHECK: bclr 4, 10, 3
+0x20 0x18 0x8a 0x4c
+
+# CHECK: bclr 4, 10
+0x20 0x00 0x8a 0x4c
+
+# CHECK: bclrl 4, 10, 3
+0x21 0x18 0x8a 0x4c
+
+# CHECK: bclrl 4, 10
+0x21 0x00 0x8a 0x4c
+
+# CHECK: bcctr 4, 10, 3
+0x20 0x1c 0x8a 0x4c
+
+# CHECK: bcctr 4, 10
+0x20 0x04 0x8a 0x4c
+
+# CHECK: bcctrl 4, 10, 3
+0x21 0x1c 0x8a 0x4c
+
+# CHECK: bcctrl 4, 10
+0x21 0x04 0x8a 0x4c
+
+# CHECK: crand 2, 3, 4
+0x02 0x22 0x43 0x4c
+
+# CHECK: crnand 2, 3, 4
+0xc2 0x21 0x43 0x4c
+
+# CHECK: cror 2, 3, 4
+0x82 0x23 0x43 0x4c
+
+# CHECK: crxor 2, 3, 4
+0x82 0x21 0x43 0x4c
+
+# CHECK: crnor 2, 3, 4
+0x42 0x20 0x43 0x4c
+
+# CHECK: creqv 2, 3, 4
+0x42 0x22 0x43 0x4c
+
+# CHECK: crandc 2, 3, 4
+0x02 0x21 0x43 0x4c
+
+# CHECK: crorc 2, 3, 4
+0x42 0x23 0x43 0x4c
+
+# CHECK: mcrf 2, 3
+0x00 0x00 0x0c 0x4d
+
+# CHECK: sc 1
+0x22 0x00 0x00 0x44
+
+# CHECK: sc
+0x02 0x00 0x00 0x44
+
+# CHECK: clrbhrb
+0x5c 0x03 0x00 0x7c
+
+# CHECK: mfbhrbe 9, 983
+0x5c 0xba 0x3e 0x7d
+
+# CHECK: rfebb 1
+0x24 0x09 0x00 0x4c
+
+# CHECK: lbz 2, 128(4)
+0x80 0x00 0x44 0x88
+
+# CHECK: lbzx 2, 3, 4
+0xae 0x20 0x43 0x7c
+
+# CHECK: lbzu 2, 128(4)
+0x80 0x00 0x44 0x8c
+
+# CHECK: lbzux 2, 3, 4
+0xee 0x20 0x43 0x7c
+
+# CHECK: lhz 2, 128(4)
+0x80 0x00 0x44 0xa0
+
+# CHECK: lhzx 2, 3, 4
+0x2e 0x22 0x43 0x7c
+
+# CHECK: lhzu 2, 128(4)
+0x80 0x00 0x44 0xa4
+
+# CHECK: lhzux 2, 3, 4
+0x6e 0x22 0x43 0x7c
+
+# CHECK: lha 2, 128(4)
+0x80 0x00 0x44 0xa8
+
+# CHECK: lhax 2, 3, 4
+0xae 0x22 0x43 0x7c
+
+# CHECK: lhau 2, 128(4)
+0x80 0x00 0x44 0xac
+
+# CHECK: lhaux 2, 3, 4
+0xee 0x22 0x43 0x7c
+
+# CHECK: lwz 2, 128(4)
+0x80 0x00 0x44 0x80
+
+# CHECK: lwzx 2, 3, 4
+0x2e 0x20 0x43 0x7c
+
+# CHECK: lwzu 2, 128(4)
+0x80 0x00 0x44 0x84
+
+# CHECK: lwzux 2, 3, 4
+0x6e 0x20 0x43 0x7c
+
+# CHECK: lwa 2, 128(4)
+0x82 0x00 0x44 0xe8
+
+# CHECK: lwax 2, 3, 4
+0xaa 0x22 0x43 0x7c
+
+# CHECK: lwaux 2, 3, 4
+0xea 0x22 0x43 0x7c
+
+# CHECK: ld 2, 128(4)
+0x80 0x00 0x44 0xe8
+
+# CHECK: ldx 2, 3, 4
+0x2a 0x20 0x43 0x7c
+
+# CHECK: ldu 2, 128(4)
+0x81 0x00 0x44 0xe8
+
+# CHECK: ldux 2, 3, 4
+0x6a 0x20 0x43 0x7c
+
+# CHECK: stb 2, 128(4)
+0x80 0x00 0x44 0x98
+
+# CHECK: stbx 2, 3, 4
+0xae 0x21 0x43 0x7c
+
+# CHECK: stbu 2, 128(4)
+0x80 0x00 0x44 0x9c
+
+# CHECK: stbux 2, 3, 4
+0xee 0x21 0x43 0x7c
+
+# CHECK: sth 2, 128(4)
+0x80 0x00 0x44 0xb0
+
+# CHECK: sthx 2, 3, 4
+0x2e 0x23 0x43 0x7c
+
+# CHECK: sthu 2, 128(4)
+0x80 0x00 0x44 0xb4
+
+# CHECK: sthux 2, 3, 4
+0x6e 0x23 0x43 0x7c
+
+# CHECK: stw 2, 128(4)
+0x80 0x00 0x44 0x90
+
+# CHECK: stwx 2, 3, 4
+0x2e 0x21 0x43 0x7c
+
+# CHECK: stwu 2, 128(4)
+0x80 0x00 0x44 0x94
+
+# CHECK: stwux 2, 3, 4
+0x6e 0x21 0x43 0x7c
+
+# CHECK: std 2, 128(4)
+0x80 0x00 0x44 0xf8
+
+# CHECK: stdx 2, 3, 4
+0x2a 0x21 0x43 0x7c
+
+# CHECK: stdu 2, 128(4)
+0x81 0x00 0x44 0xf8
+
+# CHECK: stdux 2, 3, 4
+0x6a 0x21 0x43 0x7c
+
+# CHECK: lhbrx 2, 3, 4
+0x2c 0x26 0x43 0x7c
+
+# CHECK: sthbrx 2, 3, 4
+0x2c 0x27 0x43 0x7c
+
+# CHECK: lwbrx 2, 3, 4
+0x2c 0x24 0x43 0x7c
+
+# CHECK: stwbrx 2, 3, 4
+0x2c 0x25 0x43 0x7c
+
+# CHECK: ldbrx 2, 3, 4
+0x28 0x24 0x43 0x7c
+
+# CHECK: stdbrx 2, 3, 4
+0x28 0x25 0x43 0x7c
+
+# CHECK: lmw 2, 128(1)
+0x80 0x00 0x41 0xb8
+
+# CHECK: stmw 2, 128(1)
+0x80 0x00 0x41 0xbc
+
+# CHECK: addi 2, 3, 128
+0x80 0x00 0x43 0x38
+
+# CHECK: addis 2, 3, 128
+0x80 0x00 0x43 0x3c
+
+# CHECK: add 2, 3, 4
+0x14 0x22 0x43 0x7c
+
+# CHECK: add. 2, 3, 4
+0x15 0x22 0x43 0x7c
+
+# CHECK: subf 2, 3, 4
+0x50 0x20 0x43 0x7c
+
+# CHECK: subf. 2, 3, 4
+0x51 0x20 0x43 0x7c
+
+# CHECK: addic 2, 3, 128
+0x80 0x00 0x43 0x30
+
+# CHECK: addic. 2, 3, 128
+0x80 0x00 0x43 0x34
+
+# CHECK: subfic 2, 3, 4
+0x04 0x00 0x43 0x20
+
+# CHECK: addc 2, 3, 4
+0x14 0x20 0x43 0x7c
+
+# CHECK: addc. 2, 3, 4
+0x15 0x20 0x43 0x7c
+
+# CHECK: subfc 2, 3, 4
+0x10 0x20 0x43 0x7c
+
+# CHECK: subfc 2, 3, 4
+0x10 0x20 0x43 0x7c
+
+# CHECK: adde 2, 3, 4
+0x14 0x21 0x43 0x7c
+
+# CHECK: adde. 2, 3, 4
+0x15 0x21 0x43 0x7c
+
+# CHECK: subfe 2, 3, 4
+0x10 0x21 0x43 0x7c
+
+# CHECK: subfe. 2, 3, 4
+0x11 0x21 0x43 0x7c
+
+# CHECK: addme 2, 3
+0xd4 0x01 0x43 0x7c
+
+# CHECK: addme. 2, 3
+0xd5 0x01 0x43 0x7c
+
+# CHECK: subfme 2, 3
+0xd0 0x01 0x43 0x7c
+
+# CHECK: subfme. 2, 3
+0xd1 0x01 0x43 0x7c
+
+# CHECK: addze 2, 3
+0x94 0x01 0x43 0x7c
+
+# CHECK: addze. 2, 3
+0x95 0x01 0x43 0x7c
+
+# CHECK: subfze 2, 3
+0x90 0x01 0x43 0x7c
+
+# CHECK: subfze. 2, 3
+0x91 0x01 0x43 0x7c
+
+# CHECK: neg 2, 3
+0xd0 0x00 0x43 0x7c
+
+# CHECK: neg. 2, 3
+0xd1 0x00 0x43 0x7c
+
+# CHECK: mulli 2, 3, 128
+0x80 0x00 0x43 0x1c
+
+# CHECK: mulhw 2, 3, 4
+0x96 0x20 0x43 0x7c
+
+# CHECK: mulhw. 2, 3, 4
+0x97 0x20 0x43 0x7c
+
+# CHECK: mullw 2, 3, 4
+0xd6 0x21 0x43 0x7c
+
+# CHECK: mullw. 2, 3, 4
+0xd7 0x21 0x43 0x7c
+
+# CHECK: mulhwu 2, 3, 4
+0x16 0x20 0x43 0x7c
+
+# CHECK: mulhwu. 2, 3, 4
+0x17 0x20 0x43 0x7c
+
+# CHECK: divw 2, 3, 4
+0xd6 0x23 0x43 0x7c
+
+# CHECK: divw. 2, 3, 4
+0xd7 0x23 0x43 0x7c
+
+# CHECK: divwu 2, 3, 4
+0x96 0x23 0x43 0x7c
+
+# CHECK: divwu. 2, 3, 4
+0x97 0x23 0x43 0x7c
+
+# CHECK: divwe 2, 3, 4
+0x56 0x23 0x43 0x7c
+
+# CHECK: divwe. 2, 3, 4
+0x57 0x23 0x43 0x7c
+
+# CHECK: divweu 2, 3, 4
+0x16 0x23 0x43 0x7c
+
+# CHECK: divweu. 2, 3, 4
+0x17 0x23 0x43 0x7c
+
+# CHECK: mulld 2, 3, 4
+0xd2 0x21 0x43 0x7c
+
+# CHECK: mulld. 2, 3, 4
+0xd3 0x21 0x43 0x7c
+
+# CHECK: mulhd 2, 3, 4
+0x92 0x20 0x43 0x7c
+
+# CHECK: mulhd. 2, 3, 4
+0x93 0x20 0x43 0x7c
+
+# CHECK: mulhdu 2, 3, 4
+0x12 0x20 0x43 0x7c
+
+# CHECK: mulhdu. 2, 3, 4
+0x13 0x20 0x43 0x7c
+
+# CHECK: divd 2, 3, 4
+0xd2 0x23 0x43 0x7c
+
+# CHECK: divd. 2, 3, 4
+0xd3 0x23 0x43 0x7c
+
+# CHECK: divdu 2, 3, 4
+0x92 0x23 0x43 0x7c
+
+# CHECK: divdu. 2, 3, 4
+0x93 0x23 0x43 0x7c
+
+# CHECK: divde 2, 3, 4
+0x52 0x23 0x43 0x7c
+
+# CHECK: divde. 2, 3, 4
+0x53 0x23 0x43 0x7c
+
+# CHECK: divdeu 2, 3, 4
+0x12 0x23 0x43 0x7c
+
+# CHECK: divdeu. 2, 3, 4
+0x13 0x23 0x43 0x7c
+
+# CHECK: cmpdi 2, 3, 128
+0x80 0x00 0x23 0x2d
+
+# CHECK: cmpd 2, 3, 4
+0x00 0x20 0x23 0x7d
+
+# CHECK: cmpldi 2, 3, 128
+0x80 0x00 0x23 0x29
+
+# CHECK: cmpld 2, 3, 4
+0x40 0x20 0x23 0x7d
+
+# CHECK: cmpwi 2, 3, 128
+0x80 0x00 0x03 0x2d
+
+# CHECK: cmpw 2, 3, 4
+0x00 0x20 0x03 0x7d
+
+# CHECK: cmplwi 2, 3, 128
+0x80 0x00 0x03 0x29
+
+# CHECK: cmplw 2, 3, 4
+0x40 0x20 0x03 0x7d
+
+# CHECK: twllti 3, 4
+0x04 0x00 0x43 0x0c
+
+# CHECK: twllt 3, 4
+0x08 0x20 0x43 0x7c
+
+# CHECK: tdllti 3, 4
+0x04 0x00 0x43 0x08
+
+# CHECK: tdllt 3, 4
+0x88 0x20 0x43 0x7c
+
+# CHECK: isel 2, 3, 4, 5
+0x5e 0x21 0x43 0x7c
+
+# CHECK: andi. 2, 3, 128
+0x80 0x00 0x62 0x70
+
+# CHECK: andis. 2, 3, 128
+0x80 0x00 0x62 0x74
+
+# CHECK: ori 2, 3, 128
+0x80 0x00 0x62 0x60
+
+# CHECK: oris 2, 3, 128
+0x80 0x00 0x62 0x64
+
+# CHECK: xori 2, 3, 128
+0x80 0x00 0x62 0x68
+
+# CHECK: xoris 2, 3, 128
+0x80 0x00 0x62 0x6c
+
+# CHECK: and 2, 3, 4
+0x38 0x20 0x62 0x7c
+
+# CHECK: and. 2, 3, 4
+0x39 0x20 0x62 0x7c
+
+# CHECK: xor 2, 3, 4
+0x78 0x22 0x62 0x7c
+
+# CHECK: xor. 2, 3, 4
+0x79 0x22 0x62 0x7c
+
+# CHECK: nand 2, 3, 4
+0xb8 0x23 0x62 0x7c
+
+# CHECK: nand. 2, 3, 4
+0xb9 0x23 0x62 0x7c
+
+# CHECK: or 2, 3, 4
+0x78 0x23 0x62 0x7c
+
+# CHECK: or. 2, 3, 4
+0x79 0x23 0x62 0x7c
+
+# CHECK: nor 2, 3, 4
+0xf8 0x20 0x62 0x7c
+
+# CHECK: nor. 2, 3, 4
+0xf9 0x20 0x62 0x7c
+
+# CHECK: eqv 2, 3, 4
+0x38 0x22 0x62 0x7c
+
+# CHECK: eqv. 2, 3, 4
+0x39 0x22 0x62 0x7c
+
+# CHECK: andc 2, 3, 4
+0x78 0x20 0x62 0x7c
+
+# CHECK: andc. 2, 3, 4
+0x79 0x20 0x62 0x7c
+
+# CHECK: orc 2, 3, 4
+0x38 0x23 0x62 0x7c
+
+# CHECK: orc. 2, 3, 4
+0x39 0x23 0x62 0x7c
+
+# CHECK: extsb 2, 3
+0x74 0x07 0x62 0x7c
+
+# CHECK: extsb. 2, 3
+0x75 0x07 0x62 0x7c
+
+# CHECK: extsh 2, 3
+0x34 0x07 0x62 0x7c
+
+# CHECK: extsh. 2, 3
+0x35 0x07 0x62 0x7c
+
+# CHECK: cntlz 2, 3
+0x34 0x00 0x62 0x7c
+
+# CHECK: cntlz. 2, 3
+0x35 0x00 0x62 0x7c
+
+# CHECK: popcntw 2, 3
+0xf4 0x02 0x62 0x7c
+
+# CHECK: extsw 2, 3
+0xb4 0x07 0x62 0x7c
+
+# CHECK: extsw. 2, 3
+0xb5 0x07 0x62 0x7c
+
+# CHECK: cntlzd 2, 3
+0x74 0x00 0x62 0x7c
+
+# CHECK: cntlzd. 2, 3
+0x75 0x00 0x62 0x7c
+
+# CHECK: popcntd 2, 3
+0xf4 0x03 0x62 0x7c
+
+# CHECK: bpermd 2, 3, 4
+0xf8 0x21 0x62 0x7c
+
+# CHECK: cmpb 7, 21, 4
+0xf8 0x23 0xa7 0x7e
+
+# CHECK: rlwinm 2, 3, 4, 5, 6
+0x4c 0x21 0x62 0x54
+
+# CHECK: rlwinm. 2, 3, 4, 5, 6
+0x4d 0x21 0x62 0x54
+
+# CHECK: rlwnm 2, 3, 4, 5, 6
+0x4c 0x21 0x62 0x5c
+
+# CHECK: rlwnm. 2, 3, 4, 5, 6
+0x4d 0x21 0x62 0x5c
+
+# CHECK: rlwimi 2, 3, 4, 5, 6
+0x4c 0x21 0x62 0x50
+
+# CHECK: rlwimi. 2, 3, 4, 5, 6
+0x4d 0x21 0x62 0x50
+
+# CHECK: rldicl 2, 3, 4, 5
+0x40 0x21 0x62 0x78
+
+# CHECK: rldicl. 2, 3, 4, 5
+0x41 0x21 0x62 0x78
+
+# CHECK: rldicr 2, 3, 4, 5
+0x44 0x21 0x62 0x78
+
+# CHECK: rldicr. 2, 3, 4, 5
+0x45 0x21 0x62 0x78
+
+# CHECK: rldic 2, 3, 4, 5
+0x48 0x21 0x62 0x78
+
+# CHECK: rldic. 2, 3, 4, 5
+0x49 0x21 0x62 0x78
+
+# CHECK: rldcl 2, 3, 4, 5
+0x50 0x21 0x62 0x78
+
+# CHECK: rldcl. 2, 3, 4, 5
+0x51 0x21 0x62 0x78
+
+# CHECK: rldcr 2, 3, 4, 5
+0x52 0x21 0x62 0x78
+
+# CHECK: rldcr. 2, 3, 4, 5
+0x53 0x21 0x62 0x78
+
+# CHECK: rldimi 2, 3, 4, 5
+0x4c 0x21 0x62 0x78
+
+# CHECK: rldimi. 2, 3, 4, 5
+0x4d 0x21 0x62 0x78
+
+# CHECK: slw 2, 3, 4
+0x30 0x20 0x62 0x7c
+
+# CHECK: slw. 2, 3, 4
+0x31 0x20 0x62 0x7c
+
+# CHECK: srw 2, 3, 4
+0x30 0x24 0x62 0x7c
+
+# CHECK: srw. 2, 3, 4
+0x31 0x24 0x62 0x7c
+
+# CHECK: srawi 2, 3, 4
+0x70 0x26 0x62 0x7c
+
+# CHECK: srawi. 2, 3, 4
+0x71 0x26 0x62 0x7c
+
+# CHECK: sraw 2, 3, 4
+0x30 0x26 0x62 0x7c
+
+# CHECK: sraw. 2, 3, 4
+0x31 0x26 0x62 0x7c
+
+# CHECK: sld 2, 3, 4
+0x36 0x20 0x62 0x7c
+
+# CHECK: sld. 2, 3, 4
+0x37 0x20 0x62 0x7c
+
+# CHECK: srd 2, 3, 4
+0x36 0x24 0x62 0x7c
+
+# CHECK: srd. 2, 3, 4
+0x37 0x24 0x62 0x7c
+
+# CHECK: sradi 2, 3, 4
+0x74 0x26 0x62 0x7c
+
+# CHECK: sradi. 2, 3, 4
+0x75 0x26 0x62 0x7c
+
+# CHECK: srad 2, 3, 4
+0x34 0x26 0x62 0x7c
+
+# CHECK: srad. 2, 3, 4
+0x35 0x26 0x62 0x7c
+
+# CHECK: mtspr 600, 2
+0xa6 0x93 0x58 0x7c
+
+# CHECK: mfspr 2, 600
+0xa6 0x92 0x58 0x7c
+
+# CHECK: mtcrf 123, 2
+0x20 0xb1 0x47 0x7c
+
+# CHECK: mfcr 2
+0x26 0x00 0x40 0x7c
+
+# CHECK: mtocrf 16, 2
+0x20 0x01 0x51 0x7c
+
+# CHECK: mfocrf 16, 8
+0x26 0x80 0x10 0x7e
+
+# CHECK: mtsrin 10, 12
+0xe4 0x61 0x40 0x7d
+# CHECK: mfsrin 10, 12
+0x26 0x65 0x40 0x7d
diff --git a/test/MC/Disassembler/PowerPC/qpx.txt b/test/MC/Disassembler/PowerPC/qpx.txt
index b53bb4c4d37..00e598bd435 100644
--- a/test/MC/Disassembler/PowerPC/qpx.txt
+++ b/test/MC/Disassembler/PowerPC/qpx.txt
@@ -9,12 +9,10 @@
 # CHECK: qvfadds 3, 4, 5
 0x00 0x64 0x28 0x2a
 
-# FIXME: decode as qvfandc 3, 4, 5
-# CHECK: qvflogical 3, 4, 5, 4
+# CHECK: qvfandc 3, 4, 5
 0x10 0x64 0x2a 0x08
 
-# FIXME: decode as qvfand 3, 4, 5
-# CHECK: qvflogical 3, 4, 5, 1
+# CHECK: qvfand 3, 4, 5
 0x10 0x64 0x28 0x88
 
 # CHECK: qvfcfid 3, 5
@@ -29,15 +27,13 @@
 # CHECK: qvfcfidus 3, 5
 0x00 0x60 0x2f 0x9c
 
-# FIXME: decode as qvfclr 3
-# CHECK: qvflogical 3, 3, 3, 0
+# CHECK: qvfclr 3
 0x10 0x63 0x18 0x08
 
 # CHECK: qvfcpsgn 3, 4, 5
 0x10 0x64 0x28 0x10
 
-# FIXME: decode as qvfctfb 3, 4
-# CHECK: qvflogical 3, 4, 4, 5
+# CHECK: qvfctfb 3, 4
 0x10 0x64 0x22 0x88
 
 # CHECK: qvfctid 3, 5
@@ -64,8 +60,7 @@
 # CHECK: qvfctiwz 3, 5
 0x10 0x60 0x28 0x1e
 
-# FIXME: decode as qvfequ 3, 4, 5
-# CHECK: qvflogical 3, 4, 5, 9
+# CHECK: qvfequ 3, 4, 5
 0x10 0x64 0x2c 0x88
 
 # CHECK: qvflogical 3, 4, 5, 12
@@ -95,8 +90,7 @@
 # CHECK: qvfnabs 3, 5
 0x10 0x60 0x29 0x10
 
-# FIXME: decode as qvfnand 3, 4, 5
-# CHECK: qvflogical 3, 4, 5, 14
+# CHECK: qvfnand 3, 4, 5
 0x10 0x64 0x2f 0x08
 
 # CHECK: qvfneg 3, 5
@@ -114,20 +108,16 @@
 # CHECK: qvfnmsubs 3, 4, 6, 5
 0x00 0x64 0x29 0xbc
 
-# FIXME: decode as qvfnor 3, 4, 5
-# CHECK: qvflogical 3, 4, 5, 8
+# CHECK: qvfnor 3, 4, 5
 0x10 0x64 0x2c 0x08
 
-# FIXME: decode as qvfnot 3, 4
-# CHECK: qvflogical 3, 4, 4, 10
+# CHECK: qvfnot 3, 4
 0x10 0x64 0x25 0x08
 
-# FIXME: decode as qvforc 3, 4, 5
-# CHECK: qvflogical 3, 4, 5, 13
+# CHECK: qvforc 3, 4, 5
 0x10 0x64 0x2e 0x88
 
-# FIXME: decode as qvfor 3, 4, 5
-# CHECK: qvflogical 3, 4, 5, 7
+# CHECK: qvfor 3, 4, 5
 0x10 0x64 0x2b 0x88
 
 # CHECK: qvfperm 3, 4, 5, 6
@@ -163,8 +153,7 @@
 # CHECK: qvfsel 3, 4, 6, 5
 0x10 0x64 0x29 0xae
 
-# FIXME: decode as qvfset 3
-# CHECK: qvflogical 3, 3, 3, 15
+# CHECK: qvfset 3
 0x10 0x63 0x1f 0x88
 
 # CHECK: qvfsub 3, 4, 5
@@ -185,8 +174,7 @@
 # CHECK: qvfxmuls 3, 4, 6
 0x00 0x64 0x01 0xa2
 
-# FIXME: decode as qvfxor 3, 4, 5
-# CHECK: qvflogical 3, 4, 5, 6
+# CHECK: qvfxor 3, 4, 5
 0x10 0x64 0x2b 0x08
 
 # CHECK: qvfxxcpnmadd 3, 4, 6, 5
diff --git a/test/MC/Disassembler/PowerPC/vsx.txt b/test/MC/Disassembler/PowerPC/vsx.txt
index 1b91b345a57..6f4ba6f6b9a 100644
--- a/test/MC/Disassembler/PowerPC/vsx.txt
+++ b/test/MC/Disassembler/PowerPC/vsx.txt
@@ -3,6 +3,15 @@
 # CHECK: lxsdx 7, 5, 31
 0x7c 0xe5 0xfc 0x98
 
+# CHECK: lxsiwax 7, 5, 31
+0x7c 0xe5 0xf8 0x98
+
+# CHECK: lxsiwzx 7, 5, 31
+0x7c 0xe5 0xf8 0x18
+
+# CHECK: lxsspx 7, 5, 31
+0x7c 0xe5 0xfc 0x18
+
 # CHECK: lxvd2x 7, 5, 31
 0x7c 0xe5 0xfe 0x98
 
@@ -15,6 +24,12 @@
 # CHECK: stxsdx 8, 5, 31
 0x7d 0x05 0xfd 0x98
 
+# CHECK: stxsiwx 8, 5, 31
+0x7d 0x05 0xf9 0x18
+
+# CHECK: stxsspx 8, 5, 31
+0x7d 0x05 0xfd 0x18
+
 # CHECK: stxvd2x 8, 5, 31
 0x7d 0x05 0xff 0x98
 
@@ -24,6 +39,9 @@
 # CHECK: xsabsdp 7, 27
 0xf0 0xe0 0xdd 0x64
 
+# CHECK: xsaddsp 7, 63, 27
+0xf0 0xff 0xd8 0x04
+
 # CHECK: xsadddp 7, 63, 27
 0xf0 0xff 0xd9 0x04
 
@@ -60,6 +78,9 @@
 # CHECK: xscvuxddp 7, 27
 0xf0 0xe0 0xdd 0xa0
 
+# CHECK: xsdivsp 7, 63, 27
+0xf0 0xff 0xd8 0xc4
+
 # CHECK: xsdivdp 7, 63, 27
 0xf0 0xff 0xd9 0xc4
 
@@ -69,6 +90,12 @@
 # CHECK: xsmaddmdp 7, 63, 27
 0xf0 0xff 0xd9 0x4c
 
+# CHECK: xsmaddasp 7, 63, 27
+0xf0 0xff 0xd8 0x0c
+
+# CHECK: xsmaddmsp 7, 63, 27
+0xf0 0xff 0xd8 0x4c
+
 # CHECK: xsmaxdp 7, 63, 27
 0xf0 0xff 0xdd 0x04
 
@@ -81,6 +108,15 @@
 # CHECK: xsmsubmdp 7, 63, 27
 0xf0 0xff 0xd9 0xcc
 
+# CHECK: xsmsubasp 7, 63, 27
+0xf0 0xff 0xd8 0x8c
+
+# CHECK: xsmsubmsp 7, 63, 27
+0xf0 0xff 0xd8 0xcc
+
+# CHECK: xsmulsp 7, 63, 27
+0xf0 0xff 0xd8 0x84
+
 # CHECK: xsmuldp 7, 63, 27
 0xf0 0xff 0xd9 0x84
 
@@ -102,6 +138,18 @@
 # CHECK: xsnmsubmdp 7, 63, 27
 0xf0 0xff 0xdd 0xcc
 
+# CHECK: xsnmaddasp 7, 63, 27
+0xf0 0xff 0xdc 0x0c
+
+# CHECK: xsnmaddmsp 7, 63, 27
+0xf0 0xff 0xdc 0x4c
+
+# CHECK: xsnmsubasp 7, 63, 27
+0xf0 0xff 0xdc 0x8c
+
+# CHECK: xsnmsubmsp 7, 63, 27
+0xf0 0xff 0xdc 0xcc
+
 # CHECK: xsrdpi 7, 27
 0xf0 0xe0 0xd9 0x24
 
@@ -117,15 +165,27 @@
 # CHECK: xsrdpiz 7, 27
 0xf0 0xe0 0xd9 0x64
 
+# CHECK: xsresp 7, 27
+0xf0 0xe0 0xd8 0x68
+
 # CHECK: xsredp 7, 27
 0xf0 0xe0 0xd9 0x68
 
+# CHECK: xsrsqrtesp 7, 27
+0xf0 0xe0 0xd8 0x28
+
 # CHECK: xsrsqrtedp 7, 27
 0xf0 0xe0 0xd9 0x28
 
+# CHECK: xssqrtsp 7, 27
+0xf0 0xe0 0xd8 0x2c
+
 # CHECK: xssqrtdp 7, 27
 0xf0 0xe0 0xd9 0x2c
 
+# CHECK: xssubsp 7, 63, 27
+0xf0 0xff 0xd8 0x44
+
 # CHECK: xssubdp 7, 63, 27
 0xf0 0xff 0xd9 0x44
 
@@ -273,12 +333,10 @@
 # CHECK: xvminsp 7, 63, 27
 0xf0 0xff 0xde 0x44
 
-# FIXME: decode as xvmovdp 7, 63
-# CHECK:  xvcpsgndp 7, 63, 63
+# CHECK: xvmovdp 7, 63
 0xf0 0xff 0xff 0x86
 
-# FIXME: decode as xvmovsp 7, 63
-# CHECK:  xvcpsgnsp 7, 63, 63
+# CHECK: xvmovsp 7, 63
 0xf0 0xff 0xfe 0x86
 
 # CHECK: xvmsubadp 7, 63, 27
@@ -425,15 +483,13 @@
 # CHECK: xxlxor 7, 63, 27
 0xf0 0xff 0xdc 0xd4
 
-# FIXME: decode as xxmrghd 7, 63, 27
-# CHECK: xxpermdi 7, 63, 27, 0
+# CHECK: xxmrghd 7, 63, 27
 0xf0 0xff 0xd8 0x54
 
 # CHECK: xxmrghw 7, 63, 27
 0xf0 0xff 0xd8 0x94
 
-# FIXME: decode as xxmrgld 7, 63, 27
-# CHECK:  xxpermdi 7, 63, 27, 3
+# CHECK: xxmrgld 7, 63, 27
 0xf0 0xff 0xdb 0x54
 
 # CHECK: xxmrglw 7, 63, 27
@@ -448,15 +504,13 @@
 # CHECK: xxsldwi 7, 63, 27, 1
 0xf0 0xff 0xd9 0x14
 
-# FIXME: decode as xxspltd 7, 63, 1
-# CHECK: xxpermdi 7, 63, 63, 3
+# CHECK: xxspltd 7, 63, 1
 0xf0 0xff 0xfb 0x56
 
 # CHECK: xxspltw 7, 27, 3
 0xf0 0xe3 0xda 0x90
 
-# FIXME: decode as xxswapd 7, 63
-# CHECK: xxpermdi 7, 63, 63, 2
+# CHECK: xxswapd 7, 63
 0xf0 0xff 0xfa 0x56
 
 # CHECK: mfvsrd 3, 0
diff --git a/test/MC/Disassembler/Sparc/sparc-mem.txt b/test/MC/Disassembler/Sparc/sparc-mem.txt
index 6ad4be16721..5f8886ef8b7 100644
--- a/test/MC/Disassembler/Sparc/sparc-mem.txt
+++ b/test/MC/Disassembler/Sparc/sparc-mem.txt
@@ -9,6 +9,9 @@
 # CHECK:      ldsb [%g1], %o4
 0xd8 0x48 0x60 0x00
 
+# CHECK:      ldsb [%g1], %o4
+0xd8 0x48 0x40 0x00
+
 # CHECK:      ldsh [%i0+%l6], %o2
 0xd4 0x56 0x00 0x16
 
@@ -18,6 +21,9 @@
 # CHECK:      ldsh [%g1], %o4
 0xd8 0x50 0x60 0x00
 
+# CHECK:      ldsh [%g1], %o4
+0xd8 0x50 0x40 0x00
+
 # CHECK:      ldub [%i0+%l6], %o2
 0xd4 0x0e 0x00 0x16
 
@@ -27,6 +33,9 @@
 # CHECK:      ldub [%g1], %o2
 0xd4 0x08 0x60 0x00
 
+# CHECK:      ldub [%g1], %o2
+0xd4 0x08 0x40 0x00
+
 # CHECK:      lduh [%i0+%l6], %o2
 0xd4 0x16 0x00 0x16
 
@@ -36,6 +45,9 @@
 # CHECK:      lduh [%g1], %o2
 0xd4 0x10 0x60 0x00
 
+# CHECK:      lduh [%g1], %o2
+0xd4 0x10 0x40 0x00
+
 # CHECK:      ld [%i0+%l6], %o2
 0xd4 0x06 0x00 0x16
 
@@ -45,6 +57,9 @@
 # CHECK:      ld [%g1], %o2
 0xd4 0x00 0x60 0x00
 
+# CHECK:      ld [%g1], %o2
+0xd4 0x00 0x40 0x00
+
 # CHECK:     ld [%i0+%l6], %f2
 0xc5 0x06 0x00 0x16
 
@@ -54,6 +69,9 @@
 # CHECK:     ld [%g1], %f2
 0xc5 0x00 0x60 0x00
 
+# CHECK:     ld [%g1], %f2
+0xc5 0x00 0x40 0x00
+
 # CHECK:     ldd [%i0+%l6], %f2
 0xc5 0x1e 0x00 0x16
 
@@ -63,6 +81,9 @@
 # CHECK:     ldd [%g1], %f2
 0xc5 0x18 0x60 0x00
 
+# CHECK:     ldd [%g1], %f2
+0xc5 0x18 0x40 0x00
+
 # CHECK:     ldq [%i0+%l6], %f4
 0xc9 0x16 0x00 0x16
 
@@ -72,6 +93,9 @@
 # CHECK:     ldq [%g1], %f4
 0xc9 0x10 0x60 0x00
 
+# CHECK:     ldq [%g1], %f4
+0xc9 0x10 0x40 0x00
+
 # CHECK:     ldx [%i0+%l6], %o2
 0xd4 0x5e 0x00 0x16
 
@@ -81,6 +105,9 @@
 # CHECK:     ldx [%g1], %o2
 0xd4 0x58 0x60 0x00
 
+# CHECK:     ldx [%g1], %o2
+0xd4 0x58 0x40 0x00
+
 # CHECK:     ldsw [%i0+%l6], %o2
 0xd4 0x46 0x00 0x16
 
@@ -90,6 +117,9 @@
 # CHECK:     ldsw [%g1], %o2
 0xd4 0x40 0x60 0x00
 
+# CHECK:     ldsw [%g1], %o2
+0xd4 0x40 0x40 0x00
+
 # CHECK:      stb %o2, [%i0+%l6]
 0xd4 0x2e 0x00 0x16
 
@@ -99,6 +129,9 @@
 # CHECK:      stb %o2, [%g1]
 0xd4 0x28 0x60 0x00
 
+# CHECK:      stb %o2, [%g1]
+0xd4 0x28 0x40 0x00
+
 # CHECK:      sth %o2, [%i0+%l6]
 0xd4 0x36 0x00 0x16
 
@@ -108,6 +141,9 @@
 # CHECK:      sth %o2, [%g1]
 0xd4 0x30 0x60 0x00
 
+# CHECK:      sth %o2, [%g1]
+0xd4 0x30 0x40 0x00
+
 # CHECK:      st %o2, [%i0+%l6]
 0xd4 0x26 0x00 0x16
 
@@ -117,6 +153,9 @@
 # CHECK:      st %o2, [%g1]
 0xd4 0x20 0x60 0x00
 
+# CHECK:      st %o2, [%g1]
+0xd4 0x20 0x40 0x00
+
 # CHECK:      st %f2, [%i0+%l6]
 0xc5 0x26 0x00 0x16
 
@@ -126,6 +165,9 @@
 # CHECK:      st %f2, [%g1]
 0xc5 0x20 0x60 0x00
 
+# CHECK:      st %f2, [%g1]
+0xc5 0x20 0x40 0x00
+
 # CHECK:      std %f2, [%i0+%l6]
 0xc5 0x3e 0x00 0x16
 
@@ -135,6 +177,9 @@
 # CHECK:      std %f2, [%g1]
 0xc5 0x38 0x60 0x00
 
+# CHECK:      std %f2, [%g1]
+0xc5 0x38 0x40 0x00
+
 # CHECK:      stq %f4, [%i0+%l6]
 0xc9 0x36 0x00 0x16
 
@@ -144,6 +189,9 @@
 # CHECK:      stq %f4, [%g1]
 0xc9 0x30 0x60 0x00
 
+# CHECK:      stq %f4, [%g1]
+0xc9 0x30 0x40 0x00
+
 # CHECK:      stx %o2, [%i0+%l6]
 0xd4 0x76 0x00 0x16
 
@@ -153,6 +201,9 @@
 # CHECK:      stx %o2, [%g1]
 0xd4 0x70 0x60 0x00
 
+# CHECK:      stx %o2, [%g1]
+0xd4 0x70 0x40 0x00
+
 # CHECK:     swap [%i0+%l6], %o2
 0xd4 0x7e 0x00 0x16
 
@@ -161,3 +212,12 @@
 
 # CHECK:     swap [%g1], %o2
 0xd4 0x78 0x60 0x00
+
+# CHECK:     swap [%g1], %o2
+0xd4 0x78 0x40 0x00
+
+# CHECK:     swapa [%i0+%l6] 131, %o2
+0xd4 0xfe 0x10 0x76
+
+# CHECK:     swapa [%g1] 131, %o2
+0xd4 0xf8 0x50 0x60
diff --git a/test/MC/Disassembler/Sparc/sparc-special-registers.txt b/test/MC/Disassembler/Sparc/sparc-special-registers.txt
new file mode 100644
index 00000000000..f653dbda605
--- /dev/null
+++ b/test/MC/Disassembler/Sparc/sparc-special-registers.txt
@@ -0,0 +1,34 @@
+# RUN: llvm-mc --disassemble %s -triple=sparc-unknown-linux | FileCheck %s
+
+# CHECK: wr %g1, -2, %y
+0x81 0x80 0x7f 0xfe
+
+# CHECK: rd %y, %i0
+0xb1 0x40 0x00 0x00
+
+# CHECK: rd %asr1, %i0
+0xb1 0x40 0x40 0x00
+
+# CHECK: wr %i0, 5, %y
+0x81 0x86 0x20 0x05
+
+# CHECK: wr %i0, %i1, %asr15
+0x9f 0x86 0x00 0x19
+
+# CHECK: rd %psr, %i0
+0xb1 0x48 0x00 0x00
+
+# CHECK: rd %wim, %i0
+0xb1 0x50 0x00 0x00
+
+# CHECK: rd %tbr, %i0
+0xb1 0x58 0x00 0x00
+
+# CHECK: wr %i0, 5, %psr
+0x81 0x8e 0x20 0x05
+
+# CHECK: wr %i0, 5, %wim
+0x81 0x96 0x20 0x05
+
+# CHECK: wr %i0, 5, %tbr
+0x81 0x9e 0x20 0x05
diff --git a/test/MC/Disassembler/Sparc/sparc.txt b/test/MC/Disassembler/Sparc/sparc.txt
index a9420246361..6724ebf8bf2 100644
--- a/test/MC/Disassembler/Sparc/sparc.txt
+++ b/test/MC/Disassembler/Sparc/sparc.txt
@@ -183,9 +183,6 @@
 # CHECK: cmp %g1, -2
 0x80 0xa0 0x7f 0xfe
 
-# CHECK: wr %g1, -2, %y
-0x81 0x80 0x7f 0xfe
-
 # CHECK: unimp 12
 0x00 0x00 0x00 0x0c
 
@@ -200,3 +197,6 @@
 
 # CHECK:  rett %i7+8
 0x81 0xcf 0xe0 0x08
+
+# CHECK: stbar
+0x81 0x43 0xc0 0x00
diff --git a/test/MC/Disassembler/SystemZ/insns-z13-bad.txt b/test/MC/Disassembler/SystemZ/insns-z13-bad.txt
new file mode 100644
index 00000000000..e1cf033f4b7
--- /dev/null
+++ b/test/MC/Disassembler/SystemZ/insns-z13-bad.txt
@@ -0,0 +1,39 @@
+# Test z13 instructions that don't have PC-relative operands.
+# RUN: llvm-mc --disassemble %s -triple=s390x-linux-gnu -mcpu=z13 2>&1 \
+# RUN:   | FileCheck %s
+
+# This would be "vlef %v0, 0, 4", but element 4 is invalid.
+#
+#CHECK: warning: invalid instruction encoding
+#CHECK-NEXT: 0xe7 0x00 0x00 0x00 0x40 0x03
+0xe7 0x00 0x00 0x00 0x40 0x03
+
+# ...and again with element 15
+#
+#CHECK: warning: invalid instruction encoding
+#CHECK-NEXT: 0xe7 0x00 0x00 0x00 0xf0 0x03
+0xe7 0x00 0x00 0x00 0xf0 0x03
+
+# This would be "vleg %v0, 0, 2", but element 2 is invalid.
+#
+#CHECK: warning: invalid instruction encoding
+#CHECK-NEXT: 0xe7 0x00 0x00 0x00 0x20 0x02
+0xe7 0x00 0x00 0x00 0x20 0x02
+
+# ...and again with element 15
+#
+#CHECK: warning: invalid instruction encoding
+#CHECK-NEXT: 0xe7 0x00 0x00 0x00 0xf0 0x02
+0xe7 0x00 0x00 0x00 0xf0 0x02
+
+# This would be "vleh %v0, 0, 8", but element 8 is invalid.
+#
+#CHECK: warning: invalid instruction encoding
+#CHECK-NEXT: 0xe7 0x00 0x00 0x00 0x80 0x01
+0xe7 0x00 0x00 0x00 0x80 0x01
+
+# ...and again with element 15
+#
+#CHECK: warning: invalid instruction encoding
+#CHECK-NEXT: 0xe7 0x00 0x00 0x00 0xf0 0x01
+0xe7 0x00 0x00 0x00 0xf0 0x01
diff --git a/test/MC/Disassembler/SystemZ/insns-z13.txt b/test/MC/Disassembler/SystemZ/insns-z13.txt
new file mode 100644
index 00000000000..71e77664abb
--- /dev/null
+++ b/test/MC/Disassembler/SystemZ/insns-z13.txt
@@ -0,0 +1,3315 @@
+# Test z13 instructions that don't have PC-relative operands.
+# RUN: llvm-mc --disassemble %s -triple=s390x-linux-gnu -mcpu=z13 \
+# RUN:   | FileCheck %s
+
+#CHECK: lcbb    %r0, 0, 0
+0xe7 0x00 0x00 0x00 0x00 0x27
+
+#CHECK: lcbb    %r1, 2475(%r7,%r8), 12
+0xe7 0x17 0x89 0xab 0xc0 0x27
+
+#CHECK: lcbb    %r15, 4095(%r15,%r15), 15
+0xe7 0xff 0xff 0xff 0xf0 0x27
+
+#CHECK: vab     %v0, %v0, %v0
+0xe7 0x00 0x00 0x00 0x00 0xf3
+
+#CHECK: vab     %v18, %v3, %v20
+0xe7 0x23 0x40 0x00 0x0a 0xf3
+
+#CHECK: vab     %v31, %v31, %v31
+0xe7 0xff 0xf0 0x00 0x0e 0xf3
+
+#CHECK: vaccb   %v0, %v0, %v0
+0xe7 0x00 0x00 0x00 0x00 0xf1
+
+#CHECK: vaccb   %v18, %v3, %v20
+0xe7 0x23 0x40 0x00 0x0a 0xf1
+
+#CHECK: vaccb   %v31, %v31, %v31
+0xe7 0xff 0xf0 0x00 0x0e 0xf1
+
+#CHECK: vacccq  %v0, %v0, %v0, %v0
+0xe7 0x00 0x04 0x00 0x00 0xb9
+
+#CHECK: vacccq  %v3, %v20, %v5, %v22
+0xe7 0x34 0x54 0x00 0x65 0xb9
+
+#CHECK: vacccq  %v31, %v31, %v31, %v31
+0xe7 0xff 0xf4 0x00 0xff 0xb9
+
+#CHECK: vaccf   %v0, %v0, %v0
+0xe7 0x00 0x00 0x00 0x20 0xf1
+
+#CHECK: vaccf   %v18, %v3, %v20
+0xe7 0x23 0x40 0x00 0x2a 0xf1
+
+#CHECK: vaccf   %v31, %v31, %v31
+0xe7 0xff 0xf0 0x00 0x2e 0xf1
+
+#CHECK: vaccg   %v0, %v0, %v0
+0xe7 0x00 0x00 0x00 0x30 0xf1
+
+#CHECK: vaccg   %v18, %v3, %v20
+0xe7 0x23 0x40 0x00 0x3a 0xf1
+
+#CHECK: vaccg   %v31, %v31, %v31
+0xe7 0xff 0xf0 0x00 0x3e 0xf1
+
+#CHECK: vacch   %v0, %v0, %v0
+0xe7 0x00 0x00 0x00 0x10 0xf1
+
+#CHECK: vacch   %v18, %v3, %v20
+0xe7 0x23 0x40 0x00 0x1a 0xf1
+
+#CHECK: vacch   %v31, %v31, %v31
+0xe7 0xff 0xf0 0x00 0x1e 0xf1
+
+#CHECK: vaccq   %v0, %v0, %v0
+0xe7 0x00 0x00 0x00 0x40 0xf1
+
+#CHECK: vaccq   %v18, %v3, %v20
+0xe7 0x23 0x40 0x00 0x4a 0xf1
+
+#CHECK: vaccq   %v31, %v31, %v31
+0xe7 0xff 0xf0 0x00 0x4e 0xf1
+
+#CHECK: vacq    %v0, %v0, %v0, %v0
+0xe7 0x00 0x04 0x00 0x00 0xbb
+
+#CHECK: vacq    %v3, %v20, %v5, %v22
+0xe7 0x34 0x54 0x00 0x65 0xbb
+
+#CHECK: vacq    %v31, %v31, %v31, %v31
+0xe7 0xff 0xf4 0x00 0xff 0xbb
+
+#CHECK: vaf     %v0, %v0, %v0
+0xe7 0x00 0x00 0x00 0x20 0xf3
+
+#CHECK: vaf     %v18, %v3, %v20
+0xe7 0x23 0x40 0x00 0x2a 0xf3
+
+#CHECK: vaf     %v31, %v31, %v31
+0xe7 0xff 0xf0 0x00 0x2e 0xf3
+
+#CHECK: vag     %v0, %v0, %v0
+0xe7 0x00 0x00 0x00 0x30 0xf3
+
+#CHECK: vag     %v18, %v3, %v20
+0xe7 0x23 0x40 0x00 0x3a 0xf3
+
+#CHECK: vag     %v31, %v31, %v31
+0xe7 0xff 0xf0 0x00 0x3e 0xf3
+
+#CHECK: vah     %v0, %v0, %v0
+0xe7 0x00 0x00 0x00 0x10 0xf3
+
+#CHECK: vah     %v18, %v3, %v20
+0xe7 0x23 0x40 0x00 0x1a 0xf3
+
+#CHECK: vah     %v31, %v31, %v31
+0xe7 0xff 0xf0 0x00 0x1e 0xf3
+
+#CHECK: vaq     %v0, %v0, %v0
+0xe7 0x00 0x00 0x00 0x40 0xf3
+
+#CHECK: vaq     %v18, %v3, %v20
+0xe7 0x23 0x40 0x00 0x4a 0xf3
+
+#CHECK: vaq     %v31, %v31, %v31
+0xe7 0xff 0xf0 0x00 0x4e 0xf3
+
+#CHECK: vavgb   %v0, %v0, %v0
+0xe7 0x00 0x00 0x00 0x00 0xf2
+
+#CHECK: vavgb   %v18, %v3, %v20
+0xe7 0x23 0x40 0x00 0x0a 0xf2
+
+#CHECK: vavgb   %v31, %v31, %v31
+0xe7 0xff 0xf0 0x00 0x0e 0xf2
+
+#CHECK: vavgf   %v0, %v0, %v0
+0xe7 0x00 0x00 0x00 0x20 0xf2
+
+#CHECK: vavgf   %v18, %v3, %v20
+0xe7 0x23 0x40 0x00 0x2a 0xf2
+
+#CHECK: vavgf   %v31, %v31, %v31
+0xe7 0xff 0xf0 0x00 0x2e 0xf2
+
+#CHECK: vavgg   %v0, %v0, %v0
+0xe7 0x00 0x00 0x00 0x30 0xf2
+
+#CHECK: vavgg   %v18, %v3, %v20
+0xe7 0x23 0x40 0x00 0x3a 0xf2
+
+#CHECK: vavgg   %v31, %v31, %v31
+0xe7 0xff 0xf0 0x00 0x3e 0xf2
+
+#CHECK: vavgh   %v0, %v0, %v0
+0xe7 0x00 0x00 0x00 0x10 0xf2
+
+#CHECK: vavgh   %v18, %v3, %v20
+0xe7 0x23 0x40 0x00 0x1a 0xf2
+
+#CHECK: vavgh   %v31, %v31, %v31
+0xe7 0xff 0xf0 0x00 0x1e 0xf2
+
+#CHECK: vavglb  %v0, %v0, %v0
+0xe7 0x00 0x00 0x00 0x00 0xf0
+
+#CHECK: vavglb  %v18, %v3, %v20
+0xe7 0x23 0x40 0x00 0x0a 0xf0
+
+#CHECK: vavglb  %v31, %v31, %v31
+0xe7 0xff 0xf0 0x00 0x0e 0xf0
+
+#CHECK: vavglf  %v0, %v0, %v0
+0xe7 0x00 0x00 0x00 0x20 0xf0
+
+#CHECK: vavglf  %v18, %v3, %v20
+0xe7 0x23 0x40 0x00 0x2a 0xf0
+
+#CHECK: vavglf  %v31, %v31, %v31
+0xe7 0xff 0xf0 0x00 0x2e 0xf0
+
+#CHECK: vavglg  %v0, %v0, %v0
+0xe7 0x00 0x00 0x00 0x30 0xf0
+
+#CHECK: vavglg  %v18, %v3, %v20
+0xe7 0x23 0x40 0x00 0x3a 0xf0
+
+#CHECK: vavglg  %v31, %v31, %v31
+0xe7 0xff 0xf0 0x00 0x3e 0xf0
+
+#CHECK: vavglh  %v0, %v0, %v0
+0xe7 0x00 0x00 0x00 0x10 0xf0
+
+#CHECK: vavglh  %v18, %v3, %v20
+0xe7 0x23 0x40 0x00 0x1a 0xf0
+
+#CHECK: vavglh  %v31, %v31, %v31
+0xe7 0xff 0xf0 0x00 0x1e 0xf0
+
+#CHECK: vcdgb   %v0, %v0, 0, 0
+0xe7 0x00 0x00 0x00 0x30 0xc3
+
+#CHECK: vcdgb   %v19, %v14, 4, 10
+0xe7 0x3e 0x00 0xa4 0x38 0xc3
+
+#CHECK: vcdgb   %v31, %v31, 7, 15
+0xe7 0xff 0x00 0xf7 0x3c 0xc3
+
+#CHECK: vcdlgb  %v0, %v0, 0, 0
+0xe7 0x00 0x00 0x00 0x30 0xc1
+
+#CHECK: vcdlgb  %v19, %v14, 4, 10
+0xe7 0x3e 0x00 0xa4 0x38 0xc1
+
+#CHECK: vcdlgb  %v31, %v31, 7, 15
+0xe7 0xff 0x00 0xf7 0x3c 0xc1
+
+#CHECK: vceqb   %v0, %v0, %v0
+0xe7 0x00 0x00 0x00 0x00 0xf8
+
+#CHECK: vceqb   %v18, %v3, %v20
+0xe7 0x23 0x40 0x00 0x0a 0xf8
+
+#CHECK: vceqbs  %v7, %v24, %v9
+0xe7 0x78 0x90 0x10 0x04 0xf8
+
+#CHECK: vceqb   %v31, %v31, %v31
+0xe7 0xff 0xf0 0x00 0x0e 0xf8
+
+#CHECK: vceqf   %v0, %v0, %v0
+0xe7 0x00 0x00 0x00 0x20 0xf8
+
+#CHECK: vceqf   %v18, %v3, %v20
+0xe7 0x23 0x40 0x00 0x2a 0xf8
+
+#CHECK: vceqfs  %v7, %v24, %v9
+0xe7 0x78 0x90 0x10 0x24 0xf8
+
+#CHECK: vceqf   %v31, %v31, %v31
+0xe7 0xff 0xf0 0x00 0x2e 0xf8
+
+#CHECK: vceqg   %v0, %v0, %v0
+0xe7 0x00 0x00 0x00 0x30 0xf8
+
+#CHECK: vceqg   %v18, %v3, %v20
+0xe7 0x23 0x40 0x00 0x3a 0xf8
+
+#CHECK: vceqgs  %v7, %v24, %v9
+0xe7 0x78 0x90 0x10 0x34 0xf8
+
+#CHECK: vceqg   %v31, %v31, %v31
+0xe7 0xff 0xf0 0x00 0x3e 0xf8
+
+#CHECK: vceqh   %v0, %v0, %v0
+0xe7 0x00 0x00 0x00 0x10 0xf8
+
+#CHECK: vceqh   %v18, %v3, %v20
+0xe7 0x23 0x40 0x00 0x1a 0xf8
+
+#CHECK: vceqhs  %v7, %v24, %v9
+0xe7 0x78 0x90 0x10 0x14 0xf8
+
+#CHECK: vceqh   %v31, %v31, %v31
+0xe7 0xff 0xf0 0x00 0x1e 0xf8
+
+#CHECK: vcgdb   %v0, %v0, 0, 0
+0xe7 0x00 0x00 0x00 0x30 0xc2
+
+#CHECK: vcgdb   %v19, %v14, 4, 10
+0xe7 0x3e 0x00 0xa4 0x38 0xc2
+
+#CHECK: vcgdb   %v31, %v31, 7, 15
+0xe7 0xff 0x00 0xf7 0x3c 0xc2
+
+#CHECK: vchb    %v0, %v0, %v0
+0xe7 0x00 0x00 0x00 0x00 0xfb
+
+#CHECK: vchb    %v18, %v3, %v20
+0xe7 0x23 0x40 0x00 0x0a 0xfb
+
+#CHECK: vchbs   %v7, %v24, %v9
+0xe7 0x78 0x90 0x10 0x04 0xfb
+
+#CHECK: vchb    %v31, %v31, %v31
+0xe7 0xff 0xf0 0x00 0x0e 0xfb
+
+#CHECK: vchf    %v0, %v0, %v0
+0xe7 0x00 0x00 0x00 0x20 0xfb
+
+#CHECK: vchf    %v18, %v3, %v20
+0xe7 0x23 0x40 0x00 0x2a 0xfb
+
+#CHECK: vchfs   %v7, %v24, %v9
+0xe7 0x78 0x90 0x10 0x24 0xfb
+
+#CHECK: vchf    %v31, %v31, %v31
+0xe7 0xff 0xf0 0x00 0x2e 0xfb
+
+#CHECK: vchg    %v0, %v0, %v0
+0xe7 0x00 0x00 0x00 0x30 0xfb
+
+#CHECK: vchg    %v18, %v3, %v20
+0xe7 0x23 0x40 0x00 0x3a 0xfb
+
+#CHECK: vchgs   %v7, %v24, %v9
+0xe7 0x78 0x90 0x10 0x34 0xfb
+
+#CHECK: vchg    %v31, %v31, %v31
+0xe7 0xff 0xf0 0x00 0x3e 0xfb
+
+#CHECK: vchh    %v0, %v0, %v0
+0xe7 0x00 0x00 0x00 0x10 0xfb
+
+#CHECK: vchh    %v18, %v3, %v20
+0xe7 0x23 0x40 0x00 0x1a 0xfb
+
+#CHECK: vchhs   %v7, %v24, %v9
+0xe7 0x78 0x90 0x10 0x14 0xfb
+
+#CHECK: vchh    %v31, %v31, %v31
+0xe7 0xff 0xf0 0x00 0x1e 0xfb
+
+#CHECK: vchlb   %v0, %v0, %v0
+0xe7 0x00 0x00 0x00 0x00 0xf9
+
+#CHECK: vchlb   %v18, %v3, %v20
+0xe7 0x23 0x40 0x00 0x0a 0xf9
+
+#CHECK: vchlbs  %v7, %v24, %v9
+0xe7 0x78 0x90 0x10 0x04 0xf9
+
+#CHECK: vchlb   %v31, %v31, %v31
+0xe7 0xff 0xf0 0x00 0x0e 0xf9
+
+#CHECK: vchlf   %v0, %v0, %v0
+0xe7 0x00 0x00 0x00 0x20 0xf9
+
+#CHECK: vchlf   %v18, %v3, %v20
+0xe7 0x23 0x40 0x00 0x2a 0xf9
+
+#CHECK: vchlfs  %v7, %v24, %v9
+0xe7 0x78 0x90 0x10 0x24 0xf9
+
+#CHECK: vchlf   %v31, %v31, %v31
+0xe7 0xff 0xf0 0x00 0x2e 0xf9
+
+#CHECK: vchlg   %v0, %v0, %v0
+0xe7 0x00 0x00 0x00 0x30 0xf9
+
+#CHECK: vchlg   %v18, %v3, %v20
+0xe7 0x23 0x40 0x00 0x3a 0xf9
+
+#CHECK: vchlgs  %v7, %v24, %v9
+0xe7 0x78 0x90 0x10 0x34 0xf9
+
+#CHECK: vchlg   %v31, %v31, %v31
+0xe7 0xff 0xf0 0x00 0x3e 0xf9
+
+#CHECK: vchlh   %v0, %v0, %v0
+0xe7 0x00 0x00 0x00 0x10 0xf9
+
+#CHECK: vchlh   %v18, %v3, %v20
+0xe7 0x23 0x40 0x00 0x1a 0xf9
+
+#CHECK: vchlhs  %v7, %v24, %v9
+0xe7 0x78 0x90 0x10 0x14 0xf9
+
+#CHECK: vchlh   %v31, %v31, %v31
+0xe7 0xff 0xf0 0x00 0x1e 0xf9
+
+#CHECK: vcksm   %v0, %v0, %v0
+0xe7 0x00 0x00 0x00 0x00 0x66
+
+#CHECK: vcksm   %v18, %v3, %v20
+0xe7 0x23 0x40 0x00 0x0a 0x66
+
+#CHECK: vcksm   %v31, %v31, %v31
+0xe7 0xff 0xf0 0x00 0x0e 0x66
+
+#CHECK: vclgdb  %v0, %v0, 0, 0
+0xe7 0x00 0x00 0x00 0x30 0xc0
+
+#CHECK: vclgdb  %v19, %v14, 4, 10
+0xe7 0x3e 0x00 0xa4 0x38 0xc0
+
+#CHECK: vclgdb  %v31, %v31, 7, 15
+0xe7 0xff 0x00 0xf7 0x3c 0xc0
+
+#CHECK: vclzb   %v0, %v0
+0xe7 0x00 0x00 0x00 0x00 0x53
+
+#CHECK: vclzb   %v19, %v14
+0xe7 0x3e 0x00 0x00 0x08 0x53
+
+#CHECK: vclzb   %v31, %v31
+0xe7 0xff 0x00 0x00 0x0c 0x53
+
+#CHECK: vclzf   %v0, %v0
+0xe7 0x00 0x00 0x00 0x20 0x53
+
+#CHECK: vclzf   %v19, %v14
+0xe7 0x3e 0x00 0x00 0x28 0x53
+
+#CHECK: vclzf   %v31, %v31
+0xe7 0xff 0x00 0x00 0x2c 0x53
+
+#CHECK: vclzg   %v0, %v0
+0xe7 0x00 0x00 0x00 0x30 0x53
+
+#CHECK: vclzg   %v19, %v14
+0xe7 0x3e 0x00 0x00 0x38 0x53
+
+#CHECK: vclzg   %v31, %v31
+0xe7 0xff 0x00 0x00 0x3c 0x53
+
+#CHECK: vclzh   %v0, %v0
+0xe7 0x00 0x00 0x00 0x10 0x53
+
+#CHECK: vclzh   %v19, %v14
+0xe7 0x3e 0x00 0x00 0x18 0x53
+
+#CHECK: vclzh   %v31, %v31
+0xe7 0xff 0x00 0x00 0x1c 0x53
+
+#CHECK: vctzb   %v0, %v0
+0xe7 0x00 0x00 0x00 0x00 0x52
+
+#CHECK: vctzb   %v19, %v14
+0xe7 0x3e 0x00 0x00 0x08 0x52
+
+#CHECK: vctzb   %v31, %v31
+0xe7 0xff 0x00 0x00 0x0c 0x52
+
+#CHECK: vctzf   %v0, %v0
+0xe7 0x00 0x00 0x00 0x20 0x52
+
+#CHECK: vctzf   %v19, %v14
+0xe7 0x3e 0x00 0x00 0x28 0x52
+
+#CHECK: vctzf   %v31, %v31
+0xe7 0xff 0x00 0x00 0x2c 0x52
+
+#CHECK: vctzg   %v0, %v0
+0xe7 0x00 0x00 0x00 0x30 0x52
+
+#CHECK: vctzg   %v19, %v14
+0xe7 0x3e 0x00 0x00 0x38 0x52
+
+#CHECK: vctzg   %v31, %v31
+0xe7 0xff 0x00 0x00 0x3c 0x52
+
+#CHECK: vctzh   %v0, %v0
+0xe7 0x00 0x00 0x00 0x10 0x52
+
+#CHECK: vctzh   %v19, %v14
+0xe7 0x3e 0x00 0x00 0x18 0x52
+
+#CHECK: vctzh   %v31, %v31
+0xe7 0xff 0x00 0x00 0x1c 0x52
+
+#CHECK: vecb    %v0, %v0
+0xe7 0x00 0x00 0x00 0x00 0xdb
+
+#CHECK: vecb    %v19, %v14
+0xe7 0x3e 0x00 0x00 0x08 0xdb
+
+#CHECK: vecb    %v31, %v31
+0xe7 0xff 0x00 0x00 0x0c 0xdb
+
+#CHECK: vecf    %v0, %v0
+0xe7 0x00 0x00 0x00 0x20 0xdb
+
+#CHECK: vecf    %v19, %v14
+0xe7 0x3e 0x00 0x00 0x28 0xdb
+
+#CHECK: vecf    %v31, %v31
+0xe7 0xff 0x00 0x00 0x2c 0xdb
+
+#CHECK: vecg    %v0, %v0
+0xe7 0x00 0x00 0x00 0x30 0xdb
+
+#CHECK: vecg    %v19, %v14
+0xe7 0x3e 0x00 0x00 0x38 0xdb
+
+#CHECK: vecg    %v31, %v31
+0xe7 0xff 0x00 0x00 0x3c 0xdb
+
+#CHECK: vech    %v0, %v0
+0xe7 0x00 0x00 0x00 0x10 0xdb
+
+#CHECK: vech    %v19, %v14
+0xe7 0x3e 0x00 0x00 0x18 0xdb
+
+#CHECK: vech    %v31, %v31
+0xe7 0xff 0x00 0x00 0x1c 0xdb
+
+#CHECK: veclb   %v0, %v0
+0xe7 0x00 0x00 0x00 0x00 0xd9
+
+#CHECK: veclb   %v19, %v14
+0xe7 0x3e 0x00 0x00 0x08 0xd9
+
+#CHECK: veclb   %v31, %v31
+0xe7 0xff 0x00 0x00 0x0c 0xd9
+
+#CHECK: veclf   %v0, %v0
+0xe7 0x00 0x00 0x00 0x20 0xd9
+
+#CHECK: veclf   %v19, %v14
+0xe7 0x3e 0x00 0x00 0x28 0xd9
+
+#CHECK: veclf   %v31, %v31
+0xe7 0xff 0x00 0x00 0x2c 0xd9
+
+#CHECK: veclg   %v0, %v0
+0xe7 0x00 0x00 0x00 0x30 0xd9
+
+#CHECK: veclg   %v19, %v14
+0xe7 0x3e 0x00 0x00 0x38 0xd9
+
+#CHECK: veclg   %v31, %v31
+0xe7 0xff 0x00 0x00 0x3c 0xd9
+
+#CHECK: veclh   %v0, %v0
+0xe7 0x00 0x00 0x00 0x10 0xd9
+
+#CHECK: veclh   %v19, %v14
+0xe7 0x3e 0x00 0x00 0x18 0xd9
+
+#CHECK: veclh   %v31, %v31
+0xe7 0xff 0x00 0x00 0x1c 0xd9
+
+#CHECK: verimb  %v0, %v0, %v0, 0
+0xe7 0x00 0x00 0x00 0x00 0x72
+
+#CHECK: verimb  %v3, %v20, %v5, 103
+0xe7 0x34 0x50 0x67 0x04 0x72
+
+#CHECK: verimb  %v31, %v31, %v31, 255
+0xe7 0xff 0xf0 0xff 0x0e 0x72
+
+#CHECK: verimf  %v0, %v0, %v0, 0
+0xe7 0x00 0x00 0x00 0x20 0x72
+
+#CHECK: verimf  %v3, %v20, %v5, 103
+0xe7 0x34 0x50 0x67 0x24 0x72
+
+#CHECK: verimf  %v31, %v31, %v31, 255
+0xe7 0xff 0xf0 0xff 0x2e 0x72
+
+#CHECK: verimg  %v0, %v0, %v0, 0
+0xe7 0x00 0x00 0x00 0x30 0x72
+
+#CHECK: verimg  %v3, %v20, %v5, 103
+0xe7 0x34 0x50 0x67 0x34 0x72
+
+#CHECK: verimg  %v31, %v31, %v31, 255
+0xe7 0xff 0xf0 0xff 0x3e 0x72
+
+#CHECK: verimh  %v0, %v0, %v0, 0
+0xe7 0x00 0x00 0x00 0x10 0x72
+
+#CHECK: verimh  %v3, %v20, %v5, 103
+0xe7 0x34 0x50 0x67 0x14 0x72
+
+#CHECK: verimh  %v31, %v31, %v31, 255
+0xe7 0xff 0xf0 0xff 0x1e 0x72
+
+#CHECK: verllvb %v0, %v0, %v0
+0xe7 0x00 0x00 0x00 0x00 0x73
+
+#CHECK: verllvb %v18, %v3, %v20
+0xe7 0x23 0x40 0x00 0x0a 0x73
+
+#CHECK: verllvb %v31, %v31, %v31
+0xe7 0xff 0xf0 0x00 0x0e 0x73
+
+#CHECK: verllvf %v0, %v0, %v0
+0xe7 0x00 0x00 0x00 0x20 0x73
+
+#CHECK: verllvf %v18, %v3, %v20
+0xe7 0x23 0x40 0x00 0x2a 0x73
+
+#CHECK: verllvf %v31, %v31, %v31
+0xe7 0xff 0xf0 0x00 0x2e 0x73
+
+#CHECK: verllvg %v0, %v0, %v0
+0xe7 0x00 0x00 0x00 0x30 0x73
+
+#CHECK: verllvg %v18, %v3, %v20
+0xe7 0x23 0x40 0x00 0x3a 0x73
+
+#CHECK: verllvg %v31, %v31, %v31
+0xe7 0xff 0xf0 0x00 0x3e 0x73
+
+#CHECK: verllvh %v0, %v0, %v0
+0xe7 0x00 0x00 0x00 0x10 0x73
+
+#CHECK: verllvh %v18, %v3, %v20
+0xe7 0x23 0x40 0x00 0x1a 0x73
+
+#CHECK: verllvh %v31, %v31, %v31
+0xe7 0xff 0xf0 0x00 0x1e 0x73
+
+#CHECK: verllb  %v0, %v0, 0
+0xe7 0x00 0x00 0x00 0x00 0x33
+
+#CHECK: verllb  %v12, %v18, 1110(%r3)
+0xe7 0xc2 0x34 0x56 0x04 0x33
+
+#CHECK: verllb  %v31, %v31, 4095(%r15)
+0xe7 0xff 0xff 0xff 0x0c 0x33
+
+#CHECK: verllf  %v0, %v0, 0
+0xe7 0x00 0x00 0x00 0x20 0x33
+
+#CHECK: verllf  %v12, %v18, 1110(%r3)
+0xe7 0xc2 0x34 0x56 0x24 0x33
+
+#CHECK: verllf  %v31, %v31, 4095(%r15)
+0xe7 0xff 0xff 0xff 0x2c 0x33
+
+#CHECK: verllg  %v0, %v0, 0
+0xe7 0x00 0x00 0x00 0x30 0x33
+
+#CHECK: verllg  %v12, %v18, 1110(%r3)
+0xe7 0xc2 0x34 0x56 0x34 0x33
+
+#CHECK: verllg  %v31, %v31, 4095(%r15)
+0xe7 0xff 0xff 0xff 0x3c 0x33
+
+#CHECK: verllh  %v0, %v0, 0
+0xe7 0x00 0x00 0x00 0x10 0x33
+
+#CHECK: verllh  %v12, %v18, 1110(%r3)
+0xe7 0xc2 0x34 0x56 0x14 0x33
+
+#CHECK: verllh  %v31, %v31, 4095(%r15)
+0xe7 0xff 0xff 0xff 0x1c 0x33
+
+#CHECK: veslvb  %v0, %v0, %v0
+0xe7 0x00 0x00 0x00 0x00 0x70
+
+#CHECK: veslvb  %v18, %v3, %v20
+0xe7 0x23 0x40 0x00 0x0a 0x70
+
+#CHECK: veslvb  %v31, %v31, %v31
+0xe7 0xff 0xf0 0x00 0x0e 0x70
+
+#CHECK: veslvf  %v0, %v0, %v0
+0xe7 0x00 0x00 0x00 0x20 0x70
+
+#CHECK: veslvf  %v18, %v3, %v20
+0xe7 0x23 0x40 0x00 0x2a 0x70
+
+#CHECK: veslvf  %v31, %v31, %v31
+0xe7 0xff 0xf0 0x00 0x2e 0x70
+
+#CHECK: veslvg  %v0, %v0, %v0
+0xe7 0x00 0x00 0x00 0x30 0x70
+
+#CHECK: veslvg  %v18, %v3, %v20
+0xe7 0x23 0x40 0x00 0x3a 0x70
+
+#CHECK: veslvg  %v31, %v31, %v31
+0xe7 0xff 0xf0 0x00 0x3e 0x70
+
+#CHECK: veslvh  %v0, %v0, %v0
+0xe7 0x00 0x00 0x00 0x10 0x70
+
+#CHECK: veslvh  %v18, %v3, %v20
+0xe7 0x23 0x40 0x00 0x1a 0x70
+
+#CHECK: veslvh  %v31, %v31, %v31
+0xe7 0xff 0xf0 0x00 0x1e 0x70
+
+#CHECK: veslb   %v0, %v0, 0
+0xe7 0x00 0x00 0x00 0x00 0x30
+
+#CHECK: veslb   %v12, %v18, 1110(%r3)
+0xe7 0xc2 0x34 0x56 0x04 0x30
+
+#CHECK: veslb   %v31, %v31, 4095(%r15)
+0xe7 0xff 0xff 0xff 0x0c 0x30
+
+#CHECK: veslf   %v0, %v0, 0
+0xe7 0x00 0x00 0x00 0x20 0x30
+
+#CHECK: veslf   %v12, %v18, 1110(%r3)
+0xe7 0xc2 0x34 0x56 0x24 0x30
+
+#CHECK: veslf   %v31, %v31, 4095(%r15)
+0xe7 0xff 0xff 0xff 0x2c 0x30
+
+#CHECK: veslg   %v0, %v0, 0
+0xe7 0x00 0x00 0x00 0x30 0x30
+
+#CHECK: veslg   %v12, %v18, 1110(%r3)
+0xe7 0xc2 0x34 0x56 0x34 0x30
+
+#CHECK: veslg   %v31, %v31, 4095(%r15)
+0xe7 0xff 0xff 0xff 0x3c 0x30
+
+#CHECK: veslh   %v0, %v0, 0
+0xe7 0x00 0x00 0x00 0x10 0x30
+
+#CHECK: veslh   %v12, %v18, 1110(%r3)
+0xe7 0xc2 0x34 0x56 0x14 0x30
+
+#CHECK: veslh   %v31, %v31, 4095(%r15)
+0xe7 0xff 0xff 0xff 0x1c 0x30
+
+#CHECK: vesravb %v0, %v0, %v0
+0xe7 0x00 0x00 0x00 0x00 0x7a
+
+#CHECK: vesravb %v18, %v3, %v20
+0xe7 0x23 0x40 0x00 0x0a 0x7a
+
+#CHECK: vesravb %v31, %v31, %v31
+0xe7 0xff 0xf0 0x00 0x0e 0x7a
+
+#CHECK: vesravf %v0, %v0, %v0
+0xe7 0x00 0x00 0x00 0x20 0x7a
+
+#CHECK: vesravf %v18, %v3, %v20
+0xe7 0x23 0x40 0x00 0x2a 0x7a
+
+#CHECK: vesravf %v31, %v31, %v31
+0xe7 0xff 0xf0 0x00 0x2e 0x7a
+
+#CHECK: vesravg %v0, %v0, %v0
+0xe7 0x00 0x00 0x00 0x30 0x7a
+
+#CHECK: vesravg %v18, %v3, %v20
+0xe7 0x23 0x40 0x00 0x3a 0x7a
+
+#CHECK: vesravg %v31, %v31, %v31
+0xe7 0xff 0xf0 0x00 0x3e 0x7a
+
+#CHECK: vesravh %v0, %v0, %v0
+0xe7 0x00 0x00 0x00 0x10 0x7a
+
+#CHECK: vesravh %v18, %v3, %v20
+0xe7 0x23 0x40 0x00 0x1a 0x7a
+
+#CHECK: vesravh %v31, %v31, %v31
+0xe7 0xff 0xf0 0x00 0x1e 0x7a
+
+#CHECK: vesrab  %v0, %v0, 0
+0xe7 0x00 0x00 0x00 0x00 0x3a
+
+#CHECK: vesrab  %v12, %v18, 1110(%r3)
+0xe7 0xc2 0x34 0x56 0x04 0x3a
+
+#CHECK: vesrab  %v31, %v31, 4095(%r15)
+0xe7 0xff 0xff 0xff 0x0c 0x3a
+
+#CHECK: vesraf  %v0, %v0, 0
+0xe7 0x00 0x00 0x00 0x20 0x3a
+
+#CHECK: vesraf  %v12, %v18, 1110(%r3)
+0xe7 0xc2 0x34 0x56 0x24 0x3a
+
+#CHECK: vesraf  %v31, %v31, 4095(%r15)
+0xe7 0xff 0xff 0xff 0x2c 0x3a
+
+#CHECK: vesrag  %v0, %v0, 0
+0xe7 0x00 0x00 0x00 0x30 0x3a
+
+#CHECK: vesrag  %v12, %v18, 1110(%r3)
+0xe7 0xc2 0x34 0x56 0x34 0x3a
+
+#CHECK: vesrag  %v31, %v31, 4095(%r15)
+0xe7 0xff 0xff 0xff 0x3c 0x3a
+
+#CHECK: vesrah  %v0, %v0, 0
+0xe7 0x00 0x00 0x00 0x10 0x3a
+
+#CHECK: vesrah  %v12, %v18, 1110(%r3)
+0xe7 0xc2 0x34 0x56 0x14 0x3a
+
+#CHECK: vesrah  %v31, %v31, 4095(%r15)
+0xe7 0xff 0xff 0xff 0x1c 0x3a
+
+#CHECK: vesrlvb %v0, %v0, %v0
+0xe7 0x00 0x00 0x00 0x00 0x78
+
+#CHECK: vesrlvb %v18, %v3, %v20
+0xe7 0x23 0x40 0x00 0x0a 0x78
+
+#CHECK: vesrlvb %v31, %v31, %v31
+0xe7 0xff 0xf0 0x00 0x0e 0x78
+
+#CHECK: vesrlvf %v0, %v0, %v0
+0xe7 0x00 0x00 0x00 0x20 0x78
+
+#CHECK: vesrlvf %v18, %v3, %v20
+0xe7 0x23 0x40 0x00 0x2a 0x78
+
+#CHECK: vesrlvf %v31, %v31, %v31
+0xe7 0xff 0xf0 0x00 0x2e 0x78
+
+#CHECK: vesrlvg %v0, %v0, %v0
+0xe7 0x00 0x00 0x00 0x30 0x78
+
+#CHECK: vesrlvg %v18, %v3, %v20
+0xe7 0x23 0x40 0x00 0x3a 0x78
+
+#CHECK: vesrlvg %v31, %v31, %v31
+0xe7 0xff 0xf0 0x00 0x3e 0x78
+
+#CHECK: vesrlvh %v0, %v0, %v0
+0xe7 0x00 0x00 0x00 0x10 0x78
+
+#CHECK: vesrlvh %v18, %v3, %v20
+0xe7 0x23 0x40 0x00 0x1a 0x78
+
+#CHECK: vesrlvh %v31, %v31, %v31
+0xe7 0xff 0xf0 0x00 0x1e 0x78
+
+#CHECK: vesrlb  %v0, %v0, 0
+0xe7 0x00 0x00 0x00 0x00 0x38
+
+#CHECK: vesrlb  %v12, %v18, 1110(%r3)
+0xe7 0xc2 0x34 0x56 0x04 0x38
+
+#CHECK: vesrlb  %v31, %v31, 4095(%r15)
+0xe7 0xff 0xff 0xff 0x0c 0x38
+
+#CHECK: vesrlf  %v0, %v0, 0
+0xe7 0x00 0x00 0x00 0x20 0x38
+
+#CHECK: vesrlf  %v12, %v18, 1110(%r3)
+0xe7 0xc2 0x34 0x56 0x24 0x38
+
+#CHECK: vesrlf  %v31, %v31, 4095(%r15)
+0xe7 0xff 0xff 0xff 0x2c 0x38
+
+#CHECK: vesrlg  %v0, %v0, 0
+0xe7 0x00 0x00 0x00 0x30 0x38
+
+#CHECK: vesrlg  %v12, %v18, 1110(%r3)
+0xe7 0xc2 0x34 0x56 0x34 0x38
+
+#CHECK: vesrlg  %v31, %v31, 4095(%r15)
+0xe7 0xff 0xff 0xff 0x3c 0x38
+
+#CHECK: vesrlh  %v0, %v0, 0
+0xe7 0x00 0x00 0x00 0x10 0x38
+
+#CHECK: vesrlh  %v12, %v18, 1110(%r3)
+0xe7 0xc2 0x34 0x56 0x14 0x38
+
+#CHECK: vesrlh  %v31, %v31, 4095(%r15)
+0xe7 0xff 0xff 0xff 0x1c 0x38
+
+#CHECK: vfadb   %v0, %v0, %v0
+0xe7 0x00 0x00 0x00 0x30 0xe3
+
+#CHECK: vfadb   %v18, %v3, %v20
+0xe7 0x23 0x40 0x00 0x3a 0xe3
+
+#CHECK: vfadb   %v31, %v31, %v31
+0xe7 0xff 0xf0 0x00 0x3e 0xe3
+
+#CHECK: vfaeb   %v0, %v0, %v0, 0
+0xe7 0x00 0x00 0x00 0x00 0x82
+
+#CHECK: vfaeb   %v0, %v0, %v0, 12
+0xe7 0x00 0x00 0xc0 0x00 0x82
+
+#CHECK: vfaeb   %v18, %v3, %v20, 0
+0xe7 0x23 0x40 0x00 0x0a 0x82
+
+#CHECK: vfaeb   %v31, %v31, %v31, 4
+0xe7 0xff 0xf0 0x40 0x0e 0x82
+
+#CHECK: vfaebs  %v31, %v31, %v31, 8
+0xe7 0xff 0xf0 0x90 0x0e 0x82
+
+#CHECK: vfaezb  %v31, %v31, %v31, 4
+0xe7 0xff 0xf0 0x60 0x0e 0x82
+
+#CHECK: vfaezbs %v31, %v31, %v31, 8
+0xe7 0xff 0xf0 0xb0 0x0e 0x82
+
+#CHECK: vfaef   %v0, %v0, %v0, 0
+0xe7 0x00 0x00 0x00 0x20 0x82
+
+#CHECK: vfaef   %v0, %v0, %v0, 12
+0xe7 0x00 0x00 0xc0 0x20 0x82
+
+#CHECK: vfaef   %v18, %v3, %v20, 0
+0xe7 0x23 0x40 0x00 0x2a 0x82
+
+#CHECK: vfaef   %v31, %v31, %v31, 4
+0xe7 0xff 0xf0 0x40 0x2e 0x82
+
+#CHECK: vfaefs  %v31, %v31, %v31, 8
+0xe7 0xff 0xf0 0x90 0x2e 0x82
+
+#CHECK: vfaezf  %v31, %v31, %v31, 4
+0xe7 0xff 0xf0 0x60 0x2e 0x82
+
+#CHECK: vfaezfs %v31, %v31, %v31, 8
+0xe7 0xff 0xf0 0xb0 0x2e 0x82
+
+#CHECK: vfaeh   %v0, %v0, %v0, 0
+0xe7 0x00 0x00 0x00 0x10 0x82
+
+#CHECK: vfaeh   %v0, %v0, %v0, 12
+0xe7 0x00 0x00 0xc0 0x10 0x82
+
+#CHECK: vfaeh   %v18, %v3, %v20, 0
+0xe7 0x23 0x40 0x00 0x1a 0x82
+
+#CHECK: vfaeh   %v31, %v31, %v31, 4
+0xe7 0xff 0xf0 0x40 0x1e 0x82
+
+#CHECK: vfaehs  %v31, %v31, %v31, 8
+0xe7 0xff 0xf0 0x90 0x1e 0x82
+
+#CHECK: vfaezh  %v31, %v31, %v31, 4
+0xe7 0xff 0xf0 0x60 0x1e 0x82
+
+#CHECK: vfaezhs %v31, %v31, %v31, 8
+0xe7 0xff 0xf0 0xb0 0x1e 0x82
+
+#CHECK: vfcedb  %v0, %v0, %v0
+0xe7 0x00 0x00 0x00 0x30 0xe8
+
+#CHECK: vfcedb  %v18, %v3, %v20
+0xe7 0x23 0x40 0x00 0x3a 0xe8
+
+#CHECK: vfcedb  %v31, %v31, %v31
+0xe7 0xff 0xf0 0x00 0x3e 0xe8
+
+#CHECK: vfcedbs %v0, %v0, %v0
+0xe7 0x00 0x00 0x10 0x30 0xe8
+
+#CHECK: vfcedbs %v18, %v3, %v20
+0xe7 0x23 0x40 0x10 0x3a 0xe8
+
+#CHECK: vfcedbs %v31, %v31, %v31
+0xe7 0xff 0xf0 0x10 0x3e 0xe8
+
+#CHECK: vfchdb  %v0, %v0, %v0
+0xe7 0x00 0x00 0x00 0x30 0xeb
+
+#CHECK: vfchdb  %v18, %v3, %v20
+0xe7 0x23 0x40 0x00 0x3a 0xeb
+
+#CHECK: vfchdb  %v31, %v31, %v31
+0xe7 0xff 0xf0 0x00 0x3e 0xeb
+
+#CHECK: vfchdbs %v0, %v0, %v0
+0xe7 0x00 0x00 0x10 0x30 0xeb
+
+#CHECK: vfchdbs %v18, %v3, %v20
+0xe7 0x23 0x40 0x10 0x3a 0xeb
+
+#CHECK: vfchdbs %v31, %v31, %v31
+0xe7 0xff 0xf0 0x10 0x3e 0xeb
+
+#CHECK: vfchedb %v0, %v0, %v0
+0xe7 0x00 0x00 0x00 0x30 0xea
+
+#CHECK: vfchedb %v18, %v3, %v20
+0xe7 0x23 0x40 0x00 0x3a 0xea
+
+#CHECK: vfchedb %v31, %v31, %v31
+0xe7 0xff 0xf0 0x00 0x3e 0xea
+
+#CHECK: vfchedbs %v0, %v0, %v0
+0xe7 0x00 0x00 0x10 0x30 0xea
+
+#CHECK: vfchedbs %v18, %v3, %v20
+0xe7 0x23 0x40 0x10 0x3a 0xea
+
+#CHECK: vfchedbs %v31, %v31, %v31
+0xe7 0xff 0xf0 0x10 0x3e 0xea
+
+#CHECK: vfddb   %v0, %v0, %v0
+0xe7 0x00 0x00 0x00 0x30 0xe5
+
+#CHECK: vfddb   %v18, %v3, %v20
+0xe7 0x23 0x40 0x00 0x3a 0xe5
+
+#CHECK: vfddb   %v31, %v31, %v31
+0xe7 0xff 0xf0 0x00 0x3e 0xe5
+
+#CHECK: vfeeb   %v0, %v0, %v0
+0xe7 0x00 0x00 0x00 0x00 0x80
+
+#CHECK: vfeeb   %v18, %v3, %v20
+0xe7 0x23 0x40 0x00 0x0a 0x80
+
+#CHECK: vfeebs  %v7, %v24, %v9
+0xe7 0x78 0x90 0x10 0x04 0x80
+
+#CHECK: vfeezb  %v18, %v3, %v20
+0xe7 0x23 0x40 0x20 0x0a 0x80
+
+#CHECK: vfeezbs %v7, %v24, %v9
+0xe7 0x78 0x90 0x30 0x04 0x80
+
+#CHECK: vfeeb   %v31, %v31, %v31
+0xe7 0xff 0xf0 0x00 0x0e 0x80
+
+#CHECK: vfeef   %v0, %v0, %v0
+0xe7 0x00 0x00 0x00 0x20 0x80
+
+#CHECK: vfeef   %v18, %v3, %v20
+0xe7 0x23 0x40 0x00 0x2a 0x80
+
+#CHECK: vfeefs  %v7, %v24, %v9
+0xe7 0x78 0x90 0x10 0x24 0x80
+
+#CHECK: vfeezf  %v18, %v3, %v20
+0xe7 0x23 0x40 0x20 0x2a 0x80
+
+#CHECK: vfeezfs %v7, %v24, %v9
+0xe7 0x78 0x90 0x30 0x24 0x80
+
+#CHECK: vfeef   %v31, %v31, %v31
+0xe7 0xff 0xf0 0x00 0x2e 0x80
+
+#CHECK: vfeeh   %v0, %v0, %v0
+0xe7 0x00 0x00 0x00 0x10 0x80
+
+#CHECK: vfeeh   %v18, %v3, %v20
+0xe7 0x23 0x40 0x00 0x1a 0x80
+
+#CHECK: vfeehs  %v7, %v24, %v9
+0xe7 0x78 0x90 0x10 0x14 0x80
+
+#CHECK: vfeezh  %v18, %v3, %v20
+0xe7 0x23 0x40 0x20 0x1a 0x80
+
+#CHECK: vfeezhs %v7, %v24, %v9
+0xe7 0x78 0x90 0x30 0x14 0x80
+
+#CHECK: vfeeh   %v31, %v31, %v31
+0xe7 0xff 0xf0 0x00 0x1e 0x80
+
+#CHECK: vfeneb   %v0, %v0, %v0
+0xe7 0x00 0x00 0x00 0x00 0x81
+
+#CHECK: vfeneb   %v18, %v3, %v20
+0xe7 0x23 0x40 0x00 0x0a 0x81
+
+#CHECK: vfenebs  %v7, %v24, %v9
+0xe7 0x78 0x90 0x10 0x04 0x81
+
+#CHECK: vfenezb  %v18, %v3, %v20
+0xe7 0x23 0x40 0x20 0x0a 0x81
+
+#CHECK: vfenezbs %v7, %v24, %v9
+0xe7 0x78 0x90 0x30 0x04 0x81
+
+#CHECK: vfeneb   %v31, %v31, %v31
+0xe7 0xff 0xf0 0x00 0x0e 0x81
+
+#CHECK: vfenef   %v0, %v0, %v0
+0xe7 0x00 0x00 0x00 0x20 0x81
+
+#CHECK: vfenef   %v18, %v3, %v20
+0xe7 0x23 0x40 0x00 0x2a 0x81
+
+#CHECK: vfenefs  %v7, %v24, %v9
+0xe7 0x78 0x90 0x10 0x24 0x81
+
+#CHECK: vfenezf  %v18, %v3, %v20
+0xe7 0x23 0x40 0x20 0x2a 0x81
+
+#CHECK: vfenezfs %v7, %v24, %v9
+0xe7 0x78 0x90 0x30 0x24 0x81
+
+#CHECK: vfenef   %v31, %v31, %v31
+0xe7 0xff 0xf0 0x00 0x2e 0x81
+
+#CHECK: vfeneh   %v0, %v0, %v0
+0xe7 0x00 0x00 0x00 0x10 0x81
+
+#CHECK: vfeneh   %v18, %v3, %v20
+0xe7 0x23 0x40 0x00 0x1a 0x81
+
+#CHECK: vfenehs  %v7, %v24, %v9
+0xe7 0x78 0x90 0x10 0x14 0x81
+
+#CHECK: vfenezh  %v18, %v3, %v20
+0xe7 0x23 0x40 0x20 0x1a 0x81
+
+#CHECK: vfenezhs %v7, %v24, %v9
+0xe7 0x78 0x90 0x30 0x14 0x81
+
+#CHECK: vfeneh   %v31, %v31, %v31
+0xe7 0xff 0xf0 0x00 0x1e 0x81
+
+#CHECK: vfidb   %v0, %v0, 0, 0
+0xe7 0x00 0x00 0x00 0x30 0xc7
+
+#CHECK: vfidb   %v19, %v14, 4, 10
+0xe7 0x3e 0x00 0xa4 0x38 0xc7
+
+#CHECK: vfidb   %v31, %v31, 7, 15
+0xe7 0xff 0x00 0xf7 0x3c 0xc7
+
+#CHECK: vistrb  %v0, %v0
+0xe7 0x00 0x00 0x00 0x00 0x5c
+
+#CHECK: vistrb  %v18, %v3
+0xe7 0x23 0x00 0x00 0x08 0x5c
+
+#CHECK: vistrbs %v7, %v24
+0xe7 0x78 0x00 0x10 0x04 0x5c
+
+#CHECK: vistrb  %v31, %v31
+0xe7 0xff 0x00 0x00 0x0c 0x5c
+
+#CHECK: vistrf  %v0, %v0
+0xe7 0x00 0x00 0x00 0x20 0x5c
+
+#CHECK: vistrf  %v18, %v3
+0xe7 0x23 0x00 0x00 0x28 0x5c
+
+#CHECK: vistrfs %v7, %v24
+0xe7 0x78 0x00 0x10 0x24 0x5c
+
+#CHECK: vistrf  %v31, %v31
+0xe7 0xff 0x00 0x00 0x2c 0x5c
+
+#CHECK: vistrh  %v0, %v0
+0xe7 0x00 0x00 0x00 0x10 0x5c
+
+#CHECK: vistrh  %v18, %v3
+0xe7 0x23 0x00 0x00 0x18 0x5c
+
+#CHECK: vistrhs %v7, %v24
+0xe7 0x78 0x00 0x10 0x14 0x5c
+
+#CHECK: vistrh  %v31, %v31
+0xe7 0xff 0x00 0x00 0x1c 0x5c
+
+#CHECK: vfmadb  %v0, %v0, %v0, %v0
+0xe7 0x00 0x03 0x00 0x00 0x8f
+
+#CHECK: vfmadb  %v3, %v20, %v5, %v22
+0xe7 0x34 0x53 0x00 0x65 0x8f
+
+#CHECK: vfmadb  %v31, %v31, %v31, %v31
+0xe7 0xff 0xf3 0x00 0xff 0x8f
+
+#CHECK: vfmdb   %v0, %v0, %v0
+0xe7 0x00 0x00 0x00 0x30 0xe7
+
+#CHECK: vfmdb   %v18, %v3, %v20
+0xe7 0x23 0x40 0x00 0x3a 0xe7
+
+#CHECK: vfmdb   %v31, %v31, %v31
+0xe7 0xff 0xf0 0x00 0x3e 0xe7
+
+#CHECK: vfmsdb  %v0, %v0, %v0, %v0
+0xe7 0x00 0x03 0x00 0x00 0x8e
+
+#CHECK: vfmsdb  %v3, %v20, %v5, %v22
+0xe7 0x34 0x53 0x00 0x65 0x8e
+
+#CHECK: vfmsdb  %v31, %v31, %v31, %v31
+0xe7 0xff 0xf3 0x00 0xff 0x8e
+
+#CHECK: vfsdb   %v0, %v0, %v0
+0xe7 0x00 0x00 0x00 0x30 0xe2
+
+#CHECK: vfsdb   %v18, %v3, %v20
+0xe7 0x23 0x40 0x00 0x3a 0xe2
+
+#CHECK: vfsdb   %v31, %v31, %v31
+0xe7 0xff 0xf0 0x00 0x3e 0xe2
+
+#CHECK: vzero   %v0
+0xe7 0x00 0x00 0x00 0x00 0x44
+
+#CHECK: vgbm    %v0, 1
+0xe7 0x00 0x00 0x01 0x00 0x44
+
+#CHECK: vgbm    %v0, 65534
+0xe7 0x00 0xff 0xfe 0x00 0x44
+
+#CHECK: vone    %v0
+0xe7 0x00 0xff 0xff 0x00 0x44
+
+#CHECK: vgbm    %v17, 4660
+0xe7 0x10 0x12 0x34 0x08 0x44
+
+#CHECK: vone    %v31
+0xe7 0xf0 0xff 0xff 0x08 0x44
+
+#CHECK: vgef    %v0, 0(%v0), 0
+0xe7 0x00 0x00 0x00 0x00 0x13
+
+#CHECK: vgef    %v10, 1000(%v19,%r7), 2
+0xe7 0xa3 0x73 0xe8 0x24 0x13
+
+#CHECK: vgef    %v31, 4095(%v31,%r15), 3
+0xe7 0xff 0xff 0xff 0x3c 0x13
+
+#CHECK: vgeg    %v0, 0(%v0), 0
+0xe7 0x00 0x00 0x00 0x00 0x12
+
+#CHECK: vgeg    %v10, 1000(%v19,%r7), 1
+0xe7 0xa3 0x73 0xe8 0x14 0x12
+
+#CHECK: vgeg    %v31, 4095(%v31,%r15), 1
+0xe7 0xff 0xff 0xff 0x1c 0x12
+
+#CHECK: vgfmab  %v0, %v0, %v0, %v0
+0xe7 0x00 0x00 0x00 0x00 0xbc
+
+#CHECK: vgfmab  %v3, %v20, %v5, %v22
+0xe7 0x34 0x50 0x00 0x65 0xbc
+
+#CHECK: vgfmab  %v31, %v31, %v31, %v31
+0xe7 0xff 0xf0 0x00 0xff 0xbc
+
+#CHECK: vgfmaf  %v0, %v0, %v0, %v0
+0xe7 0x00 0x02 0x00 0x00 0xbc
+
+#CHECK: vgfmaf  %v3, %v20, %v5, %v22
+0xe7 0x34 0x52 0x00 0x65 0xbc
+
+#CHECK: vgfmaf  %v31, %v31, %v31, %v31
+0xe7 0xff 0xf2 0x00 0xff 0xbc
+
+#CHECK: vgfmag  %v0, %v0, %v0, %v0
+0xe7 0x00 0x03 0x00 0x00 0xbc
+
+#CHECK: vgfmag  %v3, %v20, %v5, %v22
+0xe7 0x34 0x53 0x00 0x65 0xbc
+
+#CHECK: vgfmag  %v31, %v31, %v31, %v31
+0xe7 0xff 0xf3 0x00 0xff 0xbc
+
+#CHECK: vgfmah  %v0, %v0, %v0, %v0
+0xe7 0x00 0x01 0x00 0x00 0xbc
+
+#CHECK: vgfmah  %v3, %v20, %v5, %v22
+0xe7 0x34 0x51 0x00 0x65 0xbc
+
+#CHECK: vgfmah  %v31, %v31, %v31, %v31
+0xe7 0xff 0xf1 0x00 0xff 0xbc
+
+#CHECK: vgfmb   %v0, %v0, %v0
+0xe7 0x00 0x00 0x00 0x00 0xb4
+
+#CHECK: vgfmb   %v18, %v3, %v20
+0xe7 0x23 0x40 0x00 0x0a 0xb4
+
+#CHECK: vgfmb   %v31, %v31, %v31
+0xe7 0xff 0xf0 0x00 0x0e 0xb4
+
+#CHECK: vgfmf   %v0, %v0, %v0
+0xe7 0x00 0x00 0x00 0x20 0xb4
+
+#CHECK: vgfmf   %v18, %v3, %v20
+0xe7 0x23 0x40 0x00 0x2a 0xb4
+
+#CHECK: vgfmf   %v31, %v31, %v31
+0xe7 0xff 0xf0 0x00 0x2e 0xb4
+
+#CHECK: vgfmg   %v0, %v0, %v0
+0xe7 0x00 0x00 0x00 0x30 0xb4
+
+#CHECK: vgfmg   %v18, %v3, %v20
+0xe7 0x23 0x40 0x00 0x3a 0xb4
+
+#CHECK: vgfmg   %v31, %v31, %v31
+0xe7 0xff 0xf0 0x00 0x3e 0xb4
+
+#CHECK: vgfmh   %v0, %v0, %v0
+0xe7 0x00 0x00 0x00 0x10 0xb4
+
+#CHECK: vgfmh   %v18, %v3, %v20
+0xe7 0x23 0x40 0x00 0x1a 0xb4
+
+#CHECK: vgfmh   %v31, %v31, %v31
+0xe7 0xff 0xf0 0x00 0x1e 0xb4
+
+#CHECK: vgmb    %v0, 0, 0
+0xe7 0x00 0x00 0x00 0x00 0x46
+
+#CHECK: vgmb    %v22, 55, 66
+0xe7 0x60 0x37 0x42 0x08 0x46
+
+#CHECK: vgmb    %v31, 255, 255
+0xe7 0xf0 0xff 0xff 0x08 0x46
+
+#CHECK: vgmf    %v0, 0, 0
+0xe7 0x00 0x00 0x00 0x20 0x46
+
+#CHECK: vgmf    %v22, 55, 66
+0xe7 0x60 0x37 0x42 0x28 0x46
+
+#CHECK: vgmf    %v31, 255, 255
+0xe7 0xf0 0xff 0xff 0x28 0x46
+
+#CHECK: vgmg    %v0, 0, 0
+0xe7 0x00 0x00 0x00 0x30 0x46
+
+#CHECK: vgmg    %v22, 55, 66
+0xe7 0x60 0x37 0x42 0x38 0x46
+
+#CHECK: vgmg    %v31, 255, 255
+0xe7 0xf0 0xff 0xff 0x38 0x46
+
+#CHECK: vgmh    %v0, 0, 0
+0xe7 0x00 0x00 0x00 0x10 0x46
+
+#CHECK: vgmh    %v22, 55, 66
+0xe7 0x60 0x37 0x42 0x18 0x46
+
+#CHECK: vgmh    %v31, 255, 255
+0xe7 0xf0 0xff 0xff 0x18 0x46
+
+#CHECK: vl      %v0, 0
+0xe7 0x00 0x00 0x00 0x00 0x06
+
+#CHECK: vl      %v17, 2475(%r7,%r8)
+0xe7 0x17 0x89 0xab 0x08 0x06
+
+#CHECK: vl      %v31, 4095(%r15,%r15)
+0xe7 0xff 0xff 0xff 0x08 0x06
+
+#CHECK: vlbb    %v0, 0, 0
+0xe7 0x00 0x00 0x00 0x00 0x07
+
+#CHECK: vlbb    %v17, 2475(%r7,%r8), 12
+0xe7 0x17 0x89 0xab 0xc8 0x07
+
+#CHECK: vlbb    %v31, 4095(%r15,%r15), 15
+0xe7 0xff 0xff 0xff 0xf8 0x07
+
+#CHECK: vlcb    %v0, %v0
+0xe7 0x00 0x00 0x00 0x00 0xde
+
+#CHECK: vlcb    %v19, %v14
+0xe7 0x3e 0x00 0x00 0x08 0xde
+
+#CHECK: vlcb    %v31, %v31
+0xe7 0xff 0x00 0x00 0x0c 0xde
+
+#CHECK: vlcf    %v0, %v0
+0xe7 0x00 0x00 0x00 0x20 0xde
+
+#CHECK: vlcf    %v19, %v14
+0xe7 0x3e 0x00 0x00 0x28 0xde
+
+#CHECK: vlcf    %v31, %v31
+0xe7 0xff 0x00 0x00 0x2c 0xde
+
+#CHECK: vlcg    %v0, %v0
+0xe7 0x00 0x00 0x00 0x30 0xde
+
+#CHECK: vlcg    %v19, %v14
+0xe7 0x3e 0x00 0x00 0x38 0xde
+
+#CHECK: vlcg    %v31, %v31
+0xe7 0xff 0x00 0x00 0x3c 0xde
+
+#CHECK: vlch    %v0, %v0
+0xe7 0x00 0x00 0x00 0x10 0xde
+
+#CHECK: vlch    %v19, %v14
+0xe7 0x3e 0x00 0x00 0x18 0xde
+
+#CHECK: vlch    %v31, %v31
+0xe7 0xff 0x00 0x00 0x1c 0xde
+
+#CHECK: vldeb   %v0, %v0
+0xe7 0x00 0x00 0x00 0x20 0xc4
+
+#CHECK: vldeb   %v19, %v14
+0xe7 0x3e 0x00 0x00 0x28 0xc4
+
+#CHECK: vldeb   %v31, %v31
+0xe7 0xff 0x00 0x00 0x2c 0xc4
+
+#CHECK: vleb    %v0, 0, 0
+0xe7 0x00 0x00 0x00 0x00 0x00
+
+#CHECK: vleb    %v17, 2475(%r7,%r8), 12
+0xe7 0x17 0x89 0xab 0xc8 0x00
+
+#CHECK: vleb    %v31, 4095(%r15,%r15), 15
+0xe7 0xff 0xff 0xff 0xf8 0x00
+
+#CHECK: vledb   %v0, %v0, 0, 0
+0xe7 0x00 0x00 0x00 0x30 0xc5
+
+#CHECK: vledb   %v19, %v14, 4, 10
+0xe7 0x3e 0x00 0xa4 0x38 0xc5
+
+#CHECK: vledb   %v31, %v31, 7, 15
+0xe7 0xff 0x00 0xf7 0x3c 0xc5
+
+#CHECK: vlef    %v0, 0, 0
+0xe7 0x00 0x00 0x00 0x00 0x03
+
+#CHECK: vlef    %v17, 2475(%r7,%r8), 2
+0xe7 0x17 0x89 0xab 0x28 0x03
+
+#CHECK: vlef    %v31, 4095(%r15,%r15), 3
+0xe7 0xff 0xff 0xff 0x38 0x03
+
+#CHECK: vleg    %v0, 0, 0
+0xe7 0x00 0x00 0x00 0x00 0x02
+
+#CHECK: vleg    %v17, 2475(%r7,%r8), 1
+0xe7 0x17 0x89 0xab 0x18 0x02
+
+#CHECK: vleg    %v31, 4095(%r15,%r15), 1
+0xe7 0xff 0xff 0xff 0x18 0x02
+
+#CHECK: vleh    %v0, 0, 0
+0xe7 0x00 0x00 0x00 0x00 0x01
+
+#CHECK: vleh    %v17, 2475(%r7,%r8), 5
+0xe7 0x17 0x89 0xab 0x58 0x01
+
+#CHECK: vleh    %v31, 4095(%r15,%r15), 7
+0xe7 0xff 0xff 0xff 0x78 0x01
+
+#CHECK: vleib   %v0, 0, 0
+0xe7 0x00 0x00 0x00 0x00 0x40
+
+#CHECK: vleib   %v23, -30293, 12
+0xe7 0x70 0x89 0xab 0xc8 0x40
+
+#CHECK: vleib   %v31, -1, 15
+0xe7 0xf0 0xff 0xff 0xf8 0x40
+
+#CHECK: vleif   %v0, 0, 0
+0xe7 0x00 0x00 0x00 0x00 0x43
+
+#CHECK: vleif   %v23, -30293, 2
+0xe7 0x70 0x89 0xab 0x28 0x43
+
+#CHECK: vleif   %v31, -1, 3
+0xe7 0xf0 0xff 0xff 0x38 0x43
+
+#CHECK: vleig   %v0, 0, 0
+0xe7 0x00 0x00 0x00 0x00 0x42
+
+#CHECK: vleig   %v23, -30293, 1
+0xe7 0x70 0x89 0xab 0x18 0x42
+
+#CHECK: vleig   %v31, -1, 1
+0xe7 0xf0 0xff 0xff 0x18 0x42
+
+#CHECK: vleih   %v0, 0, 0
+0xe7 0x00 0x00 0x00 0x00 0x41
+
+#CHECK: vleih   %v23, -30293, 5
+0xe7 0x70 0x89 0xab 0x58 0x41
+
+#CHECK: vleih   %v31, -1, 7
+0xe7 0xf0 0xff 0xff 0x78 0x41
+
+#CHECK: vflcdb  %v0, %v0
+0xe7 0x00 0x00 0x00 0x30 0xcc
+
+#CHECK: vflcdb  %v19, %v14
+0xe7 0x3e 0x00 0x00 0x38 0xcc
+
+#CHECK: vflcdb  %v31, %v31
+0xe7 0xff 0x00 0x00 0x3c 0xcc
+
+#CHECK: vflndb  %v0, %v0
+0xe7 0x00 0x00 0x10 0x30 0xcc
+
+#CHECK: vflndb  %v19, %v14
+0xe7 0x3e 0x00 0x10 0x38 0xcc
+
+#CHECK: vflndb  %v31, %v31
+0xe7 0xff 0x00 0x10 0x3c 0xcc
+
+#CHECK: vflpdb  %v0, %v0
+0xe7 0x00 0x00 0x20 0x30 0xcc
+
+#CHECK: vflpdb  %v19, %v14
+0xe7 0x3e 0x00 0x20 0x38 0xcc
+
+#CHECK: vflpdb  %v31, %v31
+0xe7 0xff 0x00 0x20 0x3c 0xcc
+
+#CHECK: vlgvb   %r0, %v0, 0
+0xe7 0x00 0x00 0x00 0x00 0x21
+
+#CHECK: vlgvb   %r2, %v19, 1383(%r4)
+0xe7 0x23 0x45 0x67 0x04 0x21
+
+#CHECK: vlgvb   %r15, %v31, 4095(%r15)
+0xe7 0xff 0xff 0xff 0x04 0x21
+
+#CHECK: vlgvf   %r0, %v0, 0
+0xe7 0x00 0x00 0x00 0x20 0x21
+
+#CHECK: vlgvf   %r2, %v19, 1383(%r4)
+0xe7 0x23 0x45 0x67 0x24 0x21
+
+#CHECK: vlgvf   %r15, %v31, 4095(%r15)
+0xe7 0xff 0xff 0xff 0x24 0x21
+
+#CHECK: vlgvg   %r0, %v0, 0
+0xe7 0x00 0x00 0x00 0x30 0x21
+
+#CHECK: vlgvg   %r2, %v19, 1383(%r4)
+0xe7 0x23 0x45 0x67 0x34 0x21
+
+#CHECK: vlgvg   %r15, %v31, 4095(%r15)
+0xe7 0xff 0xff 0xff 0x34 0x21
+
+#CHECK: vlgvh   %r0, %v0, 0
+0xe7 0x00 0x00 0x00 0x10 0x21
+
+#CHECK: vlgvh   %r2, %v19, 1383(%r4)
+0xe7 0x23 0x45 0x67 0x14 0x21
+
+#CHECK: vlgvh   %r15, %v31, 4095(%r15)
+0xe7 0xff 0xff 0xff 0x14 0x21
+
+#CHECK: vfsqdb  %v0, %v0
+0xe7 0x00 0x00 0x00 0x30 0xce
+
+#CHECK: vfsqdb  %v19, %v14
+0xe7 0x3e 0x00 0x00 0x38 0xce
+
+#CHECK: vfsqdb  %v31, %v31
+0xe7 0xff 0x00 0x00 0x3c 0xce
+
+#CHECK: vftcidb %v0, %v0, 0
+0xe7 0x00 0x00 0x00 0x30 0x4a
+
+#CHECK: vftcidb %v19, %v4, 1383
+0xe7 0x34 0x56 0x70 0x38 0x4a
+
+#CHECK: vftcidb %v31, %v31, 4095
+0xe7 0xff 0xff 0xf0 0x3c 0x4a
+
+#CHECK: vll     %v0, %r0, 0
+0xe7 0x00 0x00 0x00 0x00 0x37
+
+#CHECK: vll     %v18, %r3, 1383(%r4)
+0xe7 0x23 0x45 0x67 0x08 0x37
+
+#CHECK: vll     %v31, %r15, 4095(%r15)
+0xe7 0xff 0xff 0xff 0x08 0x37
+
+#CHECK: vllezb  %v0, 0
+0xe7 0x00 0x00 0x00 0x00 0x04
+
+#CHECK: vllezb  %v17, 2475(%r7,%r8)
+0xe7 0x17 0x89 0xab 0x08 0x04
+
+#CHECK: vllezb  %v31, 4095(%r15,%r15)
+0xe7 0xff 0xff 0xff 0x08 0x04
+
+#CHECK: vllezf  %v0, 0
+0xe7 0x00 0x00 0x00 0x20 0x04
+
+#CHECK: vllezf  %v17, 2475(%r7,%r8)
+0xe7 0x17 0x89 0xab 0x28 0x04
+
+#CHECK: vllezf  %v31, 4095(%r15,%r15)
+0xe7 0xff 0xff 0xff 0x28 0x04
+
+#CHECK: vllezg  %v0, 0
+0xe7 0x00 0x00 0x00 0x30 0x04
+
+#CHECK: vllezg  %v17, 2475(%r7,%r8)
+0xe7 0x17 0x89 0xab 0x38 0x04
+
+#CHECK: vllezg  %v31, 4095(%r15,%r15)
+0xe7 0xff 0xff 0xff 0x38 0x04
+
+#CHECK: vllezh  %v0, 0
+0xe7 0x00 0x00 0x00 0x10 0x04
+
+#CHECK: vllezh  %v17, 2475(%r7,%r8)
+0xe7 0x17 0x89 0xab 0x18 0x04
+
+#CHECK: vllezh  %v31, 4095(%r15,%r15)
+0xe7 0xff 0xff 0xff 0x18 0x04
+
+#CHECK: vlm     %v0, %v0, 0
+0xe7 0x00 0x00 0x00 0x00 0x36
+
+#CHECK: vlm     %v12, %v18, 1110(%r3)
+0xe7 0xc2 0x34 0x56 0x04 0x36
+
+#CHECK: vlm     %v31, %v31, 4095(%r15)
+0xe7 0xff 0xff 0xff 0x0c 0x36
+
+#CHECK: vlpb    %v0, %v0
+0xe7 0x00 0x00 0x00 0x00 0xdf
+
+#CHECK: vlpb    %v19, %v14
+0xe7 0x3e 0x00 0x00 0x08 0xdf
+
+#CHECK: vlpb    %v31, %v31
+0xe7 0xff 0x00 0x00 0x0c 0xdf
+
+#CHECK: vlpf    %v0, %v0
+0xe7 0x00 0x00 0x00 0x20 0xdf
+
+#CHECK: vlpf    %v19, %v14
+0xe7 0x3e 0x00 0x00 0x28 0xdf
+
+#CHECK: vlpf    %v31, %v31
+0xe7 0xff 0x00 0x00 0x2c 0xdf
+
+#CHECK: vlpg    %v0, %v0
+0xe7 0x00 0x00 0x00 0x30 0xdf
+
+#CHECK: vlpg    %v19, %v14
+0xe7 0x3e 0x00 0x00 0x38 0xdf
+
+#CHECK: vlpg    %v31, %v31
+0xe7 0xff 0x00 0x00 0x3c 0xdf
+
+#CHECK: vlph    %v0, %v0
+0xe7 0x00 0x00 0x00 0x10 0xdf
+
+#CHECK: vlph    %v19, %v14
+0xe7 0x3e 0x00 0x00 0x18 0xdf
+
+#CHECK: vlph    %v31, %v31
+0xe7 0xff 0x00 0x00 0x1c 0xdf
+
+#CHECK: vlr     %v0, %v0
+0xe7 0x00 0x00 0x00 0x00 0x56
+
+#CHECK: vlr     %v19, %v14
+0xe7 0x3e 0x00 0x00 0x08 0x56
+
+#CHECK: vlr     %v31, %v31
+0xe7 0xff 0x00 0x00 0x0c 0x56
+
+#CHECK: vlrepb   %v0, 0
+0xe7 0x00 0x00 0x00 0x00 0x05
+
+#CHECK: vlrepb   %v17, 2475(%r7,%r8)
+0xe7 0x17 0x89 0xab 0x08 0x05
+
+#CHECK: vlrepb   %v31, 4095(%r15,%r15)
+0xe7 0xff 0xff 0xff 0x08 0x05
+
+#CHECK: vlrepf   %v0, 0
+0xe7 0x00 0x00 0x00 0x20 0x05
+
+#CHECK: vlrepf   %v17, 2475(%r7,%r8)
+0xe7 0x17 0x89 0xab 0x28 0x05
+
+#CHECK: vlrepf   %v31, 4095(%r15,%r15)
+0xe7 0xff 0xff 0xff 0x28 0x05
+
+#CHECK: vlrepg   %v0, 0
+0xe7 0x00 0x00 0x00 0x30 0x05
+
+#CHECK: vlrepg   %v17, 2475(%r7,%r8)
+0xe7 0x17 0x89 0xab 0x38 0x05
+
+#CHECK: vlrepg   %v31, 4095(%r15,%r15)
+0xe7 0xff 0xff 0xff 0x38 0x05
+
+#CHECK: vlreph   %v0, 0
+0xe7 0x00 0x00 0x00 0x10 0x05
+
+#CHECK: vlreph   %v17, 2475(%r7,%r8)
+0xe7 0x17 0x89 0xab 0x18 0x05
+
+#CHECK: vlreph   %v31, 4095(%r15,%r15)
+0xe7 0xff 0xff 0xff 0x18 0x05
+
+#CHECK: vlvgb   %v0, %r0, 0
+0xe7 0x00 0x00 0x00 0x00 0x22
+
+#CHECK: vlvgb   %v18, %r3, 1383(%r4)
+0xe7 0x23 0x45 0x67 0x08 0x22
+
+#CHECK: vlvgb   %v31, %r15, 4095(%r15)
+0xe7 0xff 0xff 0xff 0x08 0x22
+
+#CHECK: vlvgf   %v0, %r0, 0
+0xe7 0x00 0x00 0x00 0x20 0x22
+
+#CHECK: vlvgf   %v18, %r3, 1383(%r4)
+0xe7 0x23 0x45 0x67 0x28 0x22
+
+#CHECK: vlvgf   %v31, %r15, 4095(%r15)
+0xe7 0xff 0xff 0xff 0x28 0x22
+
+#CHECK: vlvgg   %v0, %r0, 0
+0xe7 0x00 0x00 0x00 0x30 0x22
+
+#CHECK: vlvgg   %v18, %r3, 1383(%r4)
+0xe7 0x23 0x45 0x67 0x38 0x22
+
+#CHECK: vlvgg   %v31, %r15, 4095(%r15)
+0xe7 0xff 0xff 0xff 0x38 0x22
+
+#CHECK: vlvgh   %v0, %r0, 0
+0xe7 0x00 0x00 0x00 0x10 0x22
+
+#CHECK: vlvgh   %v18, %r3, 1383(%r4)
+0xe7 0x23 0x45 0x67 0x18 0x22
+
+#CHECK: vlvgh   %v31, %r15, 4095(%r15)
+0xe7 0xff 0xff 0xff 0x18 0x22
+
+#CHECK: vlvgp   %v0, %r0, %r0
+0xe7 0x00 0x00 0x00 0x00 0x62
+
+#CHECK: vlvgp   %v18, %r3, %r4
+0xe7 0x23 0x40 0x00 0x08 0x62
+
+#CHECK: vlvgp   %v31, %r15, %r15
+0xe7 0xff 0xf0 0x00 0x08 0x62
+
+#CHECK: vmaeb   %v0, %v0, %v0, %v0
+0xe7 0x00 0x00 0x00 0x00 0xae
+
+#CHECK: vmaeb   %v3, %v20, %v5, %v22
+0xe7 0x34 0x50 0x00 0x65 0xae
+
+#CHECK: vmaeb   %v31, %v31, %v31, %v31
+0xe7 0xff 0xf0 0x00 0xff 0xae
+
+#CHECK: vmaef   %v0, %v0, %v0, %v0
+0xe7 0x00 0x02 0x00 0x00 0xae
+
+#CHECK: vmaef   %v3, %v20, %v5, %v22
+0xe7 0x34 0x52 0x00 0x65 0xae
+
+#CHECK: vmaef   %v31, %v31, %v31, %v31
+0xe7 0xff 0xf2 0x00 0xff 0xae
+
+#CHECK: vmaeh   %v0, %v0, %v0, %v0
+0xe7 0x00 0x01 0x00 0x00 0xae
+
+#CHECK: vmaeh   %v3, %v20, %v5, %v22
+0xe7 0x34 0x51 0x00 0x65 0xae
+
+#CHECK: vmaeh   %v31, %v31, %v31, %v31
+0xe7 0xff 0xf1 0x00 0xff 0xae
+
+#CHECK: vmahb   %v0, %v0, %v0, %v0
+0xe7 0x00 0x00 0x00 0x00 0xab
+
+#CHECK: vmahb   %v3, %v20, %v5, %v22
+0xe7 0x34 0x50 0x00 0x65 0xab
+
+#CHECK: vmahb   %v31, %v31, %v31, %v31
+0xe7 0xff 0xf0 0x00 0xff 0xab
+
+#CHECK: vmahf   %v0, %v0, %v0, %v0
+0xe7 0x00 0x02 0x00 0x00 0xab
+
+#CHECK: vmahf   %v3, %v20, %v5, %v22
+0xe7 0x34 0x52 0x00 0x65 0xab
+
+#CHECK: vmahf   %v31, %v31, %v31, %v31
+0xe7 0xff 0xf2 0x00 0xff 0xab
+
+#CHECK: vmahh   %v0, %v0, %v0, %v0
+0xe7 0x00 0x01 0x00 0x00 0xab
+
+#CHECK: vmahh   %v3, %v20, %v5, %v22
+0xe7 0x34 0x51 0x00 0x65 0xab
+
+#CHECK: vmahh   %v31, %v31, %v31, %v31
+0xe7 0xff 0xf1 0x00 0xff 0xab
+
+#CHECK: vmalb   %v0, %v0, %v0, %v0
+0xe7 0x00 0x00 0x00 0x00 0xaa
+
+#CHECK: vmalb   %v3, %v20, %v5, %v22
+0xe7 0x34 0x50 0x00 0x65 0xaa
+
+#CHECK: vmalb   %v31, %v31, %v31, %v31
+0xe7 0xff 0xf0 0x00 0xff 0xaa
+
+#CHECK: vmaleb  %v0, %v0, %v0, %v0
+0xe7 0x00 0x00 0x00 0x00 0xac
+
+#CHECK: vmaleb  %v3, %v20, %v5, %v22
+0xe7 0x34 0x50 0x00 0x65 0xac
+
+#CHECK: vmaleb  %v31, %v31, %v31, %v31
+0xe7 0xff 0xf0 0x00 0xff 0xac
+
+#CHECK: vmalef  %v0, %v0, %v0, %v0
+0xe7 0x00 0x02 0x00 0x00 0xac
+
+#CHECK: vmalef  %v3, %v20, %v5, %v22
+0xe7 0x34 0x52 0x00 0x65 0xac
+
+#CHECK: vmalef  %v31, %v31, %v31, %v31
+0xe7 0xff 0xf2 0x00 0xff 0xac
+
+#CHECK: vmaleh  %v0, %v0, %v0, %v0
+0xe7 0x00 0x01 0x00 0x00 0xac
+
+#CHECK: vmaleh  %v3, %v20, %v5, %v22
+0xe7 0x34 0x51 0x00 0x65 0xac
+
+#CHECK: vmaleh  %v31, %v31, %v31, %v31
+0xe7 0xff 0xf1 0x00 0xff 0xac
+
+#CHECK: vmalf   %v0, %v0, %v0, %v0
+0xe7 0x00 0x02 0x00 0x00 0xaa
+
+#CHECK: vmalf   %v3, %v20, %v5, %v22
+0xe7 0x34 0x52 0x00 0x65 0xaa
+
+#CHECK: vmalf   %v31, %v31, %v31, %v31
+0xe7 0xff 0xf2 0x00 0xff 0xaa
+
+#CHECK: vmalhb  %v0, %v0, %v0, %v0
+0xe7 0x00 0x00 0x00 0x00 0xa9
+
+#CHECK: vmalhb  %v3, %v20, %v5, %v22
+0xe7 0x34 0x50 0x00 0x65 0xa9
+
+#CHECK: vmalhb  %v31, %v31, %v31, %v31
+0xe7 0xff 0xf0 0x00 0xff 0xa9
+
+#CHECK: vmalhf  %v0, %v0, %v0, %v0
+0xe7 0x00 0x02 0x00 0x00 0xa9
+
+#CHECK: vmalhf  %v3, %v20, %v5, %v22
+0xe7 0x34 0x52 0x00 0x65 0xa9
+
+#CHECK: vmalhf  %v31, %v31, %v31, %v31
+0xe7 0xff 0xf2 0x00 0xff 0xa9
+
+#CHECK: vmalhh  %v0, %v0, %v0, %v0
+0xe7 0x00 0x01 0x00 0x00 0xa9
+
+#CHECK: vmalhh  %v3, %v20, %v5, %v22
+0xe7 0x34 0x51 0x00 0x65 0xa9
+
+#CHECK: vmalhh  %v31, %v31, %v31, %v31
+0xe7 0xff 0xf1 0x00 0xff 0xa9
+
+#CHECK: vmalhw  %v0, %v0, %v0, %v0
+0xe7 0x00 0x01 0x00 0x00 0xaa
+
+#CHECK: vmalhw  %v3, %v20, %v5, %v22
+0xe7 0x34 0x51 0x00 0x65 0xaa
+
+#CHECK: vmalhw  %v31, %v31, %v31, %v31
+0xe7 0xff 0xf1 0x00 0xff 0xaa
+
+#CHECK: vmalob  %v0, %v0, %v0, %v0
+0xe7 0x00 0x00 0x00 0x00 0xad
+
+#CHECK: vmalob  %v3, %v20, %v5, %v22
+0xe7 0x34 0x50 0x00 0x65 0xad
+
+#CHECK: vmalob  %v31, %v31, %v31, %v31
+0xe7 0xff 0xf0 0x00 0xff 0xad
+
+#CHECK: vmalof  %v0, %v0, %v0, %v0
+0xe7 0x00 0x02 0x00 0x00 0xad
+
+#CHECK: vmalof  %v3, %v20, %v5, %v22
+0xe7 0x34 0x52 0x00 0x65 0xad
+
+#CHECK: vmalof  %v31, %v31, %v31, %v31
+0xe7 0xff 0xf2 0x00 0xff 0xad
+
+#CHECK: vmaloh  %v0, %v0, %v0, %v0
+0xe7 0x00 0x01 0x00 0x00 0xad
+
+#CHECK: vmaloh  %v3, %v20, %v5, %v22
+0xe7 0x34 0x51 0x00 0x65 0xad
+
+#CHECK: vmaloh  %v31, %v31, %v31, %v31
+0xe7 0xff 0xf1 0x00 0xff 0xad
+
+#CHECK: vmaob   %v0, %v0, %v0, %v0
+0xe7 0x00 0x00 0x00 0x00 0xaf
+
+#CHECK: vmaob   %v3, %v20, %v5, %v22
+0xe7 0x34 0x50 0x00 0x65 0xaf
+
+#CHECK: vmaob   %v31, %v31, %v31, %v31
+0xe7 0xff 0xf0 0x00 0xff 0xaf
+
+#CHECK: vmaof   %v0, %v0, %v0, %v0
+0xe7 0x00 0x02 0x00 0x00 0xaf
+
+#CHECK: vmaof   %v3, %v20, %v5, %v22
+0xe7 0x34 0x52 0x00 0x65 0xaf
+
+#CHECK: vmaof   %v31, %v31, %v31, %v31
+0xe7 0xff 0xf2 0x00 0xff 0xaf
+
+#CHECK: vmaoh   %v0, %v0, %v0, %v0
+0xe7 0x00 0x01 0x00 0x00 0xaf
+
+#CHECK: vmaoh   %v3, %v20, %v5, %v22
+0xe7 0x34 0x51 0x00 0x65 0xaf
+
+#CHECK: vmaoh   %v31, %v31, %v31, %v31
+0xe7 0xff 0xf1 0x00 0xff 0xaf
+
+#CHECK: vmeb    %v0, %v0, %v0
+0xe7 0x00 0x00 0x00 0x00 0xa6
+
+#CHECK: vmeb    %v18, %v3, %v20
+0xe7 0x23 0x40 0x00 0x0a 0xa6
+
+#CHECK: vmeb    %v31, %v31, %v31
+0xe7 0xff 0xf0 0x00 0x0e 0xa6
+
+#CHECK: vmef    %v0, %v0, %v0
+0xe7 0x00 0x00 0x00 0x20 0xa6
+
+#CHECK: vmef    %v18, %v3, %v20
+0xe7 0x23 0x40 0x00 0x2a 0xa6
+
+#CHECK: vmef    %v31, %v31, %v31
+0xe7 0xff 0xf0 0x00 0x2e 0xa6
+
+#CHECK: vmeh    %v0, %v0, %v0
+0xe7 0x00 0x00 0x00 0x10 0xa6
+
+#CHECK: vmeh    %v18, %v3, %v20
+0xe7 0x23 0x40 0x00 0x1a 0xa6
+
+#CHECK: vmeh    %v31, %v31, %v31
+0xe7 0xff 0xf0 0x00 0x1e 0xa6
+
+#CHECK: vmhb    %v0, %v0, %v0
+0xe7 0x00 0x00 0x00 0x00 0xa3
+
+#CHECK: vmhb    %v18, %v3, %v20
+0xe7 0x23 0x40 0x00 0x0a 0xa3
+
+#CHECK: vmhb    %v31, %v31, %v31
+0xe7 0xff 0xf0 0x00 0x0e 0xa3
+
+#CHECK: vmhf    %v0, %v0, %v0
+0xe7 0x00 0x00 0x00 0x20 0xa3
+
+#CHECK: vmhf    %v18, %v3, %v20
+0xe7 0x23 0x40 0x00 0x2a 0xa3
+
+#CHECK: vmhf    %v31, %v31, %v31
+0xe7 0xff 0xf0 0x00 0x2e 0xa3
+
+#CHECK: vmhh    %v0, %v0, %v0
+0xe7 0x00 0x00 0x00 0x10 0xa3
+
+#CHECK: vmhh    %v18, %v3, %v20
+0xe7 0x23 0x40 0x00 0x1a 0xa3
+
+#CHECK: vmhh    %v31, %v31, %v31
+0xe7 0xff 0xf0 0x00 0x1e 0xa3
+
+#CHECK: vmlb    %v0, %v0, %v0
+0xe7 0x00 0x00 0x00 0x00 0xa2
+
+#CHECK: vmlb    %v18, %v3, %v20
+0xe7 0x23 0x40 0x00 0x0a 0xa2
+
+#CHECK: vmlb    %v31, %v31, %v31
+0xe7 0xff 0xf0 0x00 0x0e 0xa2
+
+#CHECK: vmlf    %v0, %v0, %v0
+0xe7 0x00 0x00 0x00 0x20 0xa2
+
+#CHECK: vmlf    %v18, %v3, %v20
+0xe7 0x23 0x40 0x00 0x2a 0xa2
+
+#CHECK: vmlf    %v31, %v31, %v31
+0xe7 0xff 0xf0 0x00 0x2e 0xa2
+
+#CHECK: vmleb   %v0, %v0, %v0
+0xe7 0x00 0x00 0x00 0x00 0xa4
+
+#CHECK: vmleb   %v18, %v3, %v20
+0xe7 0x23 0x40 0x00 0x0a 0xa4
+
+#CHECK: vmleb   %v31, %v31, %v31
+0xe7 0xff 0xf0 0x00 0x0e 0xa4
+
+#CHECK: vmlef   %v0, %v0, %v0
+0xe7 0x00 0x00 0x00 0x20 0xa4
+
+#CHECK: vmlef   %v18, %v3, %v20
+0xe7 0x23 0x40 0x00 0x2a 0xa4
+
+#CHECK: vmlef   %v31, %v31, %v31
+0xe7 0xff 0xf0 0x00 0x2e 0xa4
+
+#CHECK: vmleh   %v0, %v0, %v0
+0xe7 0x00 0x00 0x00 0x10 0xa4
+
+#CHECK: vmleh   %v18, %v3, %v20
+0xe7 0x23 0x40 0x00 0x1a 0xa4
+
+#CHECK: vmleh   %v31, %v31, %v31
+0xe7 0xff 0xf0 0x00 0x1e 0xa4
+
+#CHECK: vmlhb   %v0, %v0, %v0
+0xe7 0x00 0x00 0x00 0x00 0xa1
+
+#CHECK: vmlhb   %v18, %v3, %v20
+0xe7 0x23 0x40 0x00 0x0a 0xa1
+
+#CHECK: vmlhb   %v31, %v31, %v31
+0xe7 0xff 0xf0 0x00 0x0e 0xa1
+
+#CHECK: vmlhf   %v0, %v0, %v0
+0xe7 0x00 0x00 0x00 0x20 0xa1
+
+#CHECK: vmlhf   %v18, %v3, %v20
+0xe7 0x23 0x40 0x00 0x2a 0xa1
+
+#CHECK: vmlhf   %v31, %v31, %v31
+0xe7 0xff 0xf0 0x00 0x2e 0xa1
+
+#CHECK: vmlhh   %v0, %v0, %v0
+0xe7 0x00 0x00 0x00 0x10 0xa1
+
+#CHECK: vmlhh   %v18, %v3, %v20
+0xe7 0x23 0x40 0x00 0x1a 0xa1
+
+#CHECK: vmlhh   %v31, %v31, %v31
+0xe7 0xff 0xf0 0x00 0x1e 0xa1
+
+#CHECK: vmlhw   %v0, %v0, %v0
+0xe7 0x00 0x00 0x00 0x10 0xa2
+
+#CHECK: vmlhw   %v18, %v3, %v20
+0xe7 0x23 0x40 0x00 0x1a 0xa2
+
+#CHECK: vmlhw   %v31, %v31, %v31
+0xe7 0xff 0xf0 0x00 0x1e 0xa2
+
+#CHECK: vmlob   %v0, %v0, %v0
+0xe7 0x00 0x00 0x00 0x00 0xa5
+
+#CHECK: vmlob   %v18, %v3, %v20
+0xe7 0x23 0x40 0x00 0x0a 0xa5
+
+#CHECK: vmlob   %v31, %v31, %v31
+0xe7 0xff 0xf0 0x00 0x0e 0xa5
+
+#CHECK: vmlof   %v0, %v0, %v0
+0xe7 0x00 0x00 0x00 0x20 0xa5
+
+#CHECK: vmlof   %v18, %v3, %v20
+0xe7 0x23 0x40 0x00 0x2a 0xa5
+
+#CHECK: vmlof   %v31, %v31, %v31
+0xe7 0xff 0xf0 0x00 0x2e 0xa5
+
+#CHECK: vmloh   %v0, %v0, %v0
+0xe7 0x00 0x00 0x00 0x10 0xa5
+
+#CHECK: vmloh   %v18, %v3, %v20
+0xe7 0x23 0x40 0x00 0x1a 0xa5
+
+#CHECK: vmloh   %v31, %v31, %v31
+0xe7 0xff 0xf0 0x00 0x1e 0xa5
+
+#CHECK: vmnb    %v0, %v0, %v0
+0xe7 0x00 0x00 0x00 0x00 0xfe
+
+#CHECK: vmnb    %v18, %v3, %v20
+0xe7 0x23 0x40 0x00 0x0a 0xfe
+
+#CHECK: vmnb    %v31, %v31, %v31
+0xe7 0xff 0xf0 0x00 0x0e 0xfe
+
+#CHECK: vmnf    %v0, %v0, %v0
+0xe7 0x00 0x00 0x00 0x20 0xfe
+
+#CHECK: vmnf    %v18, %v3, %v20
+0xe7 0x23 0x40 0x00 0x2a 0xfe
+
+#CHECK: vmnf    %v31, %v31, %v31
+0xe7 0xff 0xf0 0x00 0x2e 0xfe
+
+#CHECK: vmng    %v0, %v0, %v0
+0xe7 0x00 0x00 0x00 0x30 0xfe
+
+#CHECK: vmng    %v18, %v3, %v20
+0xe7 0x23 0x40 0x00 0x3a 0xfe
+
+#CHECK: vmng    %v31, %v31, %v31
+0xe7 0xff 0xf0 0x00 0x3e 0xfe
+
+#CHECK: vmnh    %v0, %v0, %v0
+0xe7 0x00 0x00 0x00 0x10 0xfe
+
+#CHECK: vmnh    %v18, %v3, %v20
+0xe7 0x23 0x40 0x00 0x1a 0xfe
+
+#CHECK: vmnh    %v31, %v31, %v31
+0xe7 0xff 0xf0 0x00 0x1e 0xfe
+
+#CHECK: vmnlb   %v0, %v0, %v0
+0xe7 0x00 0x00 0x00 0x00 0xfc
+
+#CHECK: vmnlb   %v18, %v3, %v20
+0xe7 0x23 0x40 0x00 0x0a 0xfc
+
+#CHECK: vmnlb   %v31, %v31, %v31
+0xe7 0xff 0xf0 0x00 0x0e 0xfc
+
+#CHECK: vmnlf   %v0, %v0, %v0
+0xe7 0x00 0x00 0x00 0x20 0xfc
+
+#CHECK: vmnlf   %v18, %v3, %v20
+0xe7 0x23 0x40 0x00 0x2a 0xfc
+
+#CHECK: vmnlf   %v31, %v31, %v31
+0xe7 0xff 0xf0 0x00 0x2e 0xfc
+
+#CHECK: vmnlg   %v0, %v0, %v0
+0xe7 0x00 0x00 0x00 0x30 0xfc
+
+#CHECK: vmnlg   %v18, %v3, %v20
+0xe7 0x23 0x40 0x00 0x3a 0xfc
+
+#CHECK: vmnlg   %v31, %v31, %v31
+0xe7 0xff 0xf0 0x00 0x3e 0xfc
+
+#CHECK: vmnlh   %v0, %v0, %v0
+0xe7 0x00 0x00 0x00 0x10 0xfc
+
+#CHECK: vmnlh   %v18, %v3, %v20
+0xe7 0x23 0x40 0x00 0x1a 0xfc
+
+#CHECK: vmnlh   %v31, %v31, %v31
+0xe7 0xff 0xf0 0x00 0x1e 0xfc
+
+#CHECK: vmob    %v0, %v0, %v0
+0xe7 0x00 0x00 0x00 0x00 0xa7
+
+#CHECK: vmob    %v18, %v3, %v20
+0xe7 0x23 0x40 0x00 0x0a 0xa7
+
+#CHECK: vmob    %v31, %v31, %v31
+0xe7 0xff 0xf0 0x00 0x0e 0xa7
+
+#CHECK: vmof    %v0, %v0, %v0
+0xe7 0x00 0x00 0x00 0x20 0xa7
+
+#CHECK: vmof    %v18, %v3, %v20
+0xe7 0x23 0x40 0x00 0x2a 0xa7
+
+#CHECK: vmof    %v31, %v31, %v31
+0xe7 0xff 0xf0 0x00 0x2e 0xa7
+
+#CHECK: vmoh    %v0, %v0, %v0
+0xe7 0x00 0x00 0x00 0x10 0xa7
+
+#CHECK: vmoh    %v18, %v3, %v20
+0xe7 0x23 0x40 0x00 0x1a 0xa7
+
+#CHECK: vmoh    %v31, %v31, %v31
+0xe7 0xff 0xf0 0x00 0x1e 0xa7
+
+#CHECK: vmrhb   %v0, %v0, %v0
+0xe7 0x00 0x00 0x00 0x00 0x61
+
+#CHECK: vmrhb   %v18, %v3, %v20
+0xe7 0x23 0x40 0x00 0x0a 0x61
+
+#CHECK: vmrhb   %v31, %v31, %v31
+0xe7 0xff 0xf0 0x00 0x0e 0x61
+
+#CHECK: vmrhf   %v0, %v0, %v0
+0xe7 0x00 0x00 0x00 0x20 0x61
+
+#CHECK: vmrhf   %v18, %v3, %v20
+0xe7 0x23 0x40 0x00 0x2a 0x61
+
+#CHECK: vmrhf   %v31, %v31, %v31
+0xe7 0xff 0xf0 0x00 0x2e 0x61
+
+#CHECK: vmrhg   %v0, %v0, %v0
+0xe7 0x00 0x00 0x00 0x30 0x61
+
+#CHECK: vmrhg   %v18, %v3, %v20
+0xe7 0x23 0x40 0x00 0x3a 0x61
+
+#CHECK: vmrhg   %v31, %v31, %v31
+0xe7 0xff 0xf0 0x00 0x3e 0x61
+
+#CHECK: vmrhh   %v0, %v0, %v0
+0xe7 0x00 0x00 0x00 0x10 0x61
+
+#CHECK: vmrhh   %v18, %v3, %v20
+0xe7 0x23 0x40 0x00 0x1a 0x61
+
+#CHECK: vmrhh   %v31, %v31, %v31
+0xe7 0xff 0xf0 0x00 0x1e 0x61
+
+#CHECK: vmrlb   %v0, %v0, %v0
+0xe7 0x00 0x00 0x00 0x00 0x60
+
+#CHECK: vmrlb   %v18, %v3, %v20
+0xe7 0x23 0x40 0x00 0x0a 0x60
+
+#CHECK: vmrlb   %v31, %v31, %v31
+0xe7 0xff 0xf0 0x00 0x0e 0x60
+
+#CHECK: vmrlf   %v0, %v0, %v0
+0xe7 0x00 0x00 0x00 0x20 0x60
+
+#CHECK: vmrlf   %v18, %v3, %v20
+0xe7 0x23 0x40 0x00 0x2a 0x60
+
+#CHECK: vmrlf   %v31, %v31, %v31
+0xe7 0xff 0xf0 0x00 0x2e 0x60
+
+#CHECK: vmrlg   %v0, %v0, %v0
+0xe7 0x00 0x00 0x00 0x30 0x60
+
+#CHECK: vmrlg   %v18, %v3, %v20
+0xe7 0x23 0x40 0x00 0x3a 0x60
+
+#CHECK: vmrlg   %v31, %v31, %v31
+0xe7 0xff 0xf0 0x00 0x3e 0x60
+
+#CHECK: vmrlh   %v0, %v0, %v0
+0xe7 0x00 0x00 0x00 0x10 0x60
+
+#CHECK: vmrlh   %v18, %v3, %v20
+0xe7 0x23 0x40 0x00 0x1a 0x60
+
+#CHECK: vmrlh   %v31, %v31, %v31
+0xe7 0xff 0xf0 0x00 0x1e 0x60
+
+#CHECK: vmxb    %v0, %v0, %v0
+0xe7 0x00 0x00 0x00 0x00 0xff
+
+#CHECK: vmxb    %v18, %v3, %v20
+0xe7 0x23 0x40 0x00 0x0a 0xff
+
+#CHECK: vmxb    %v31, %v31, %v31
+0xe7 0xff 0xf0 0x00 0x0e 0xff
+
+#CHECK: vmxf    %v0, %v0, %v0
+0xe7 0x00 0x00 0x00 0x20 0xff
+
+#CHECK: vmxf    %v18, %v3, %v20
+0xe7 0x23 0x40 0x00 0x2a 0xff
+
+#CHECK: vmxf    %v31, %v31, %v31
+0xe7 0xff 0xf0 0x00 0x2e 0xff
+
+#CHECK: vmxg    %v0, %v0, %v0
+0xe7 0x00 0x00 0x00 0x30 0xff
+
+#CHECK: vmxg    %v18, %v3, %v20
+0xe7 0x23 0x40 0x00 0x3a 0xff
+
+#CHECK: vmxg    %v31, %v31, %v31
+0xe7 0xff 0xf0 0x00 0x3e 0xff
+
+#CHECK: vmxh    %v0, %v0, %v0
+0xe7 0x00 0x00 0x00 0x10 0xff
+
+#CHECK: vmxh    %v18, %v3, %v20
+0xe7 0x23 0x40 0x00 0x1a 0xff
+
+#CHECK: vmxh    %v31, %v31, %v31
+0xe7 0xff 0xf0 0x00 0x1e 0xff
+
+#CHECK: vmxlb   %v0, %v0, %v0
+0xe7 0x00 0x00 0x00 0x00 0xfd
+
+#CHECK: vmxlb   %v18, %v3, %v20
+0xe7 0x23 0x40 0x00 0x0a 0xfd
+
+#CHECK: vmxlb   %v31, %v31, %v31
+0xe7 0xff 0xf0 0x00 0x0e 0xfd
+
+#CHECK: vmxlf   %v0, %v0, %v0
+0xe7 0x00 0x00 0x00 0x20 0xfd
+
+#CHECK: vmxlf   %v18, %v3, %v20
+0xe7 0x23 0x40 0x00 0x2a 0xfd
+
+#CHECK: vmxlf   %v31, %v31, %v31
+0xe7 0xff 0xf0 0x00 0x2e 0xfd
+
+#CHECK: vmxlg   %v0, %v0, %v0
+0xe7 0x00 0x00 0x00 0x30 0xfd
+
+#CHECK: vmxlg   %v18, %v3, %v20
+0xe7 0x23 0x40 0x00 0x3a 0xfd
+
+#CHECK: vmxlg   %v31, %v31, %v31
+0xe7 0xff 0xf0 0x00 0x3e 0xfd
+
+#CHECK: vmxlh   %v0, %v0, %v0
+0xe7 0x00 0x00 0x00 0x10 0xfd
+
+#CHECK: vmxlh   %v18, %v3, %v20
+0xe7 0x23 0x40 0x00 0x1a 0xfd
+
+#CHECK: vmxlh   %v31, %v31, %v31
+0xe7 0xff 0xf0 0x00 0x1e 0xfd
+
+#CHECK: vn      %v0, %v0, %v0
+0xe7 0x00 0x00 0x00 0x00 0x68
+
+#CHECK: vn      %v18, %v3, %v20
+0xe7 0x23 0x40 0x00 0x0a 0x68
+
+#CHECK: vn      %v31, %v31, %v31
+0xe7 0xff 0xf0 0x00 0x0e 0x68
+
+#CHECK: vnc     %v0, %v0, %v0
+0xe7 0x00 0x00 0x00 0x00 0x69
+
+#CHECK: vnc     %v18, %v3, %v20
+0xe7 0x23 0x40 0x00 0x0a 0x69
+
+#CHECK: vnc     %v31, %v31, %v31
+0xe7 0xff 0xf0 0x00 0x0e 0x69
+
+#CHECK: vno     %v0, %v0, %v0
+0xe7 0x00 0x00 0x00 0x00 0x6b
+
+#CHECK: vno     %v18, %v3, %v20
+0xe7 0x23 0x40 0x00 0x0a 0x6b
+
+#CHECK: vno     %v31, %v31, %v31
+0xe7 0xff 0xf0 0x00 0x0e 0x6b
+
+#CHECK: vo      %v0, %v0, %v0
+0xe7 0x00 0x00 0x00 0x00 0x6a
+
+#CHECK: vo      %v18, %v3, %v20
+0xe7 0x23 0x40 0x00 0x0a 0x6a
+
+#CHECK: vo      %v31, %v31, %v31
+0xe7 0xff 0xf0 0x00 0x0e 0x6a
+
+#CHECK: vpdi    %v0, %v0, %v0, 0
+0xe7 0x00 0x00 0x00 0x00 0x84
+
+#CHECK: vpdi    %v3, %v20, %v5, 4
+0xe7 0x34 0x50 0x00 0x44 0x84
+
+#CHECK: vpdi    %v31, %v31, %v31, 15
+0xe7 0xff 0xf0 0x00 0xfe 0x84
+
+#CHECK: vperm   %v0, %v0, %v0, %v0
+0xe7 0x00 0x00 0x00 0x00 0x8c
+
+#CHECK: vperm   %v3, %v20, %v5, %v22
+0xe7 0x34 0x50 0x00 0x65 0x8c
+
+#CHECK: vperm   %v31, %v31, %v31, %v31
+0xe7 0xff 0xf0 0x00 0xff 0x8c
+
+#CHECK: vpkf    %v0, %v0, %v0
+0xe7 0x00 0x00 0x00 0x20 0x94
+
+#CHECK: vpkf    %v18, %v3, %v20
+0xe7 0x23 0x40 0x00 0x2a 0x94
+
+#CHECK: vpkf    %v31, %v31, %v31
+0xe7 0xff 0xf0 0x00 0x2e 0x94
+
+#CHECK: vpkg    %v0, %v0, %v0
+0xe7 0x00 0x00 0x00 0x30 0x94
+
+#CHECK: vpkg    %v18, %v3, %v20
+0xe7 0x23 0x40 0x00 0x3a 0x94
+
+#CHECK: vpkg    %v31, %v31, %v31
+0xe7 0xff 0xf0 0x00 0x3e 0x94
+
+#CHECK: vpkh    %v0, %v0, %v0
+0xe7 0x00 0x00 0x00 0x10 0x94
+
+#CHECK: vpkh    %v18, %v3, %v20
+0xe7 0x23 0x40 0x00 0x1a 0x94
+
+#CHECK: vpkh    %v31, %v31, %v31
+0xe7 0xff 0xf0 0x00 0x1e 0x94
+
+#CHECK: vpklsf  %v0, %v0, %v0
+0xe7 0x00 0x00 0x00 0x20 0x95
+
+#CHECK: vpklsf  %v18, %v3, %v20
+0xe7 0x23 0x40 0x00 0x2a 0x95
+
+#CHECK: vpklsfs %v7, %v24, %v9
+0xe7 0x78 0x90 0x10 0x24 0x95
+
+#CHECK: vpklsf  %v31, %v31, %v31
+0xe7 0xff 0xf0 0x00 0x2e 0x95
+
+#CHECK: vpklsg  %v0, %v0, %v0
+0xe7 0x00 0x00 0x00 0x30 0x95
+
+#CHECK: vpklsg  %v18, %v3, %v20
+0xe7 0x23 0x40 0x00 0x3a 0x95
+
+#CHECK: vpklsgs %v7, %v24, %v9
+0xe7 0x78 0x90 0x10 0x34 0x95
+
+#CHECK: vpklsg  %v31, %v31, %v31
+0xe7 0xff 0xf0 0x00 0x3e 0x95
+
+#CHECK: vpklsh  %v0, %v0, %v0
+0xe7 0x00 0x00 0x00 0x10 0x95
+
+#CHECK: vpklsh  %v18, %v3, %v20
+0xe7 0x23 0x40 0x00 0x1a 0x95
+
+#CHECK: vpklshs %v7, %v24, %v9
+0xe7 0x78 0x90 0x10 0x14 0x95
+
+#CHECK: vpklsh  %v31, %v31, %v31
+0xe7 0xff 0xf0 0x00 0x1e 0x95
+
+#CHECK: vpksf   %v0, %v0, %v0
+0xe7 0x00 0x00 0x00 0x20 0x97
+
+#CHECK: vpksf   %v18, %v3, %v20
+0xe7 0x23 0x40 0x00 0x2a 0x97
+
+#CHECK: vpksfs  %v7, %v24, %v9
+0xe7 0x78 0x90 0x10 0x24 0x97
+
+#CHECK: vpksf   %v31, %v31, %v31
+0xe7 0xff 0xf0 0x00 0x2e 0x97
+
+#CHECK: vpksg   %v0, %v0, %v0
+0xe7 0x00 0x00 0x00 0x30 0x97
+
+#CHECK: vpksg   %v18, %v3, %v20
+0xe7 0x23 0x40 0x00 0x3a 0x97
+
+#CHECK: vpksgs  %v7, %v24, %v9
+0xe7 0x78 0x90 0x10 0x34 0x97
+
+#CHECK: vpksg   %v31, %v31, %v31
+0xe7 0xff 0xf0 0x00 0x3e 0x97
+
+#CHECK: vpksh   %v0, %v0, %v0
+0xe7 0x00 0x00 0x00 0x10 0x97
+
+#CHECK: vpksh   %v18, %v3, %v20
+0xe7 0x23 0x40 0x00 0x1a 0x97
+
+#CHECK: vpkshs  %v7, %v24, %v9
+0xe7 0x78 0x90 0x10 0x14 0x97
+
+#CHECK: vpksh   %v31, %v31, %v31
+0xe7 0xff 0xf0 0x00 0x1e 0x97
+
+#CHECK: vpopct  %v0, %v0, 0
+0xe7 0x00 0x00 0x00 0x00 0x50
+
+#CHECK: vpopct  %v19, %v14, 0
+0xe7 0x3e 0x00 0x00 0x08 0x50
+
+#CHECK: vpopct  %v31, %v31
+0xe7 0xff 0x00 0x00 0x0c 0x50
+
+#CHECK: vrepb   %v0, %v0, 0
+0xe7 0x00 0x00 0x00 0x00 0x4d
+
+#CHECK: vrepb   %v19, %v4, 22136
+0xe7 0x34 0x56 0x78 0x08 0x4d
+
+#CHECK: vrepb   %v31, %v31, 65535
+0xe7 0xff 0xff 0xff 0x0c 0x4d
+
+#CHECK: vrepf   %v0, %v0, 0
+0xe7 0x00 0x00 0x00 0x20 0x4d
+
+#CHECK: vrepf   %v19, %v4, 22136
+0xe7 0x34 0x56 0x78 0x28 0x4d
+
+#CHECK: vrepf   %v31, %v31, 65535
+0xe7 0xff 0xff 0xff 0x2c 0x4d
+
+#CHECK: vrepg   %v0, %v0, 0
+0xe7 0x00 0x00 0x00 0x30 0x4d
+
+#CHECK: vrepg   %v19, %v4, 22136
+0xe7 0x34 0x56 0x78 0x38 0x4d
+
+#CHECK: vrepg   %v31, %v31, 65535
+0xe7 0xff 0xff 0xff 0x3c 0x4d
+
+#CHECK: vreph   %v0, %v0, 0
+0xe7 0x00 0x00 0x00 0x10 0x4d
+
+#CHECK: vreph   %v19, %v4, 22136
+0xe7 0x34 0x56 0x78 0x18 0x4d
+
+#CHECK: vreph   %v31, %v31, 65535
+0xe7 0xff 0xff 0xff 0x1c 0x4d
+
+#CHECK: vrepib  %v0, 0
+0xe7 0x00 0x00 0x00 0x00 0x45
+
+#CHECK: vrepib  %v23, -30293
+0xe7 0x70 0x89 0xab 0x08 0x45
+
+#CHECK: vrepib  %v31, -1
+0xe7 0xf0 0xff 0xff 0x08 0x45
+
+#CHECK: vrepif  %v0, 0
+0xe7 0x00 0x00 0x00 0x20 0x45
+
+#CHECK: vrepif  %v23, -30293
+0xe7 0x70 0x89 0xab 0x28 0x45
+
+#CHECK: vrepif  %v31, -1
+0xe7 0xf0 0xff 0xff 0x28 0x45
+
+#CHECK: vrepig  %v0, 0
+0xe7 0x00 0x00 0x00 0x30 0x45
+
+#CHECK: vrepig  %v23, -30293
+0xe7 0x70 0x89 0xab 0x38 0x45
+
+#CHECK: vrepig  %v31, -1
+0xe7 0xf0 0xff 0xff 0x38 0x45
+
+#CHECK: vrepih  %v0, 0
+0xe7 0x00 0x00 0x00 0x10 0x45
+
+#CHECK: vrepih  %v23, -30293
+0xe7 0x70 0x89 0xab 0x18 0x45
+
+#CHECK: vrepih  %v31, -1
+0xe7 0xf0 0xff 0xff 0x18 0x45
+
+#CHECK: vsb     %v0, %v0, %v0
+0xe7 0x00 0x00 0x00 0x00 0xf7
+
+#CHECK: vsb     %v18, %v3, %v20
+0xe7 0x23 0x40 0x00 0x0a 0xf7
+
+#CHECK: vsb     %v31, %v31, %v31
+0xe7 0xff 0xf0 0x00 0x0e 0xf7
+
+#CHECK: vsbiq   %v0, %v0, %v0, %v0
+0xe7 0x00 0x04 0x00 0x00 0xbf
+
+#CHECK: vsbiq   %v3, %v20, %v5, %v22
+0xe7 0x34 0x54 0x00 0x65 0xbf
+
+#CHECK: vsbiq   %v31, %v31, %v31, %v31
+0xe7 0xff 0xf4 0x00 0xff 0xbf
+
+#CHECK: vsbcbiq %v0, %v0, %v0, %v0
+0xe7 0x00 0x04 0x00 0x00 0xbd
+
+#CHECK: vsbcbiq %v3, %v20, %v5, %v22
+0xe7 0x34 0x54 0x00 0x65 0xbd
+
+#CHECK: vsbcbiq %v31, %v31, %v31, %v31
+0xe7 0xff 0xf4 0x00 0xff 0xbd
+
+#CHECK: vscbib  %v0, %v0, %v0
+0xe7 0x00 0x00 0x00 0x00 0xf5
+
+#CHECK: vscbib  %v18, %v3, %v20
+0xe7 0x23 0x40 0x00 0x0a 0xf5
+
+#CHECK: vscbib  %v31, %v31, %v31
+0xe7 0xff 0xf0 0x00 0x0e 0xf5
+
+#CHECK: vscbif  %v0, %v0, %v0
+0xe7 0x00 0x00 0x00 0x20 0xf5
+
+#CHECK: vscbif  %v18, %v3, %v20
+0xe7 0x23 0x40 0x00 0x2a 0xf5
+
+#CHECK: vscbif  %v31, %v31, %v31
+0xe7 0xff 0xf0 0x00 0x2e 0xf5
+
+#CHECK: vscbig  %v0, %v0, %v0
+0xe7 0x00 0x00 0x00 0x30 0xf5
+
+#CHECK: vscbig  %v18, %v3, %v20
+0xe7 0x23 0x40 0x00 0x3a 0xf5
+
+#CHECK: vscbig  %v31, %v31, %v31
+0xe7 0xff 0xf0 0x00 0x3e 0xf5
+
+#CHECK: vscbih  %v0, %v0, %v0
+0xe7 0x00 0x00 0x00 0x10 0xf5
+
+#CHECK: vscbih  %v18, %v3, %v20
+0xe7 0x23 0x40 0x00 0x1a 0xf5
+
+#CHECK: vscbih  %v31, %v31, %v31
+0xe7 0xff 0xf0 0x00 0x1e 0xf5
+
+#CHECK: vscbiq  %v0, %v0, %v0
+0xe7 0x00 0x00 0x00 0x40 0xf5
+
+#CHECK: vscbiq  %v18, %v3, %v20
+0xe7 0x23 0x40 0x00 0x4a 0xf5
+
+#CHECK: vscbiq  %v31, %v31, %v31
+0xe7 0xff 0xf0 0x00 0x4e 0xf5
+
+#CHECK: vscef   %v0, 0(%v0), 0
+0xe7 0x00 0x00 0x00 0x00 0x1b
+
+#CHECK: vscef   %v10, 1000(%v19,%r7), 2
+0xe7 0xa3 0x73 0xe8 0x24 0x1b
+
+#CHECK: vscef   %v31, 4095(%v31,%r15), 3
+0xe7 0xff 0xff 0xff 0x3c 0x1b
+
+#CHECK: vsceg   %v0, 0(%v0), 0
+0xe7 0x00 0x00 0x00 0x00 0x1a
+
+#CHECK: vsceg   %v10, 1000(%v19,%r7), 1
+0xe7 0xa3 0x73 0xe8 0x14 0x1a
+
+#CHECK: vsceg   %v31, 4095(%v31,%r15), 1
+0xe7 0xff 0xff 0xff 0x1c 0x1a
+
+#CHECK: vsegb   %v0, %v0
+0xe7 0x00 0x00 0x00 0x00 0x5f
+
+#CHECK: vsegb   %v19, %v14
+0xe7 0x3e 0x00 0x00 0x08 0x5f
+
+#CHECK: vsegb   %v31, %v31
+0xe7 0xff 0x00 0x00 0x0c 0x5f
+
+#CHECK: vsegf   %v0, %v0
+0xe7 0x00 0x00 0x00 0x20 0x5f
+
+#CHECK: vsegf   %v19, %v14
+0xe7 0x3e 0x00 0x00 0x28 0x5f
+
+#CHECK: vsegf   %v31, %v31
+0xe7 0xff 0x00 0x00 0x2c 0x5f
+
+#CHECK: vsegh   %v0, %v0
+0xe7 0x00 0x00 0x00 0x10 0x5f
+
+#CHECK: vsegh   %v19, %v14
+0xe7 0x3e 0x00 0x00 0x18 0x5f
+
+#CHECK: vsegh   %v31, %v31
+0xe7 0xff 0x00 0x00 0x1c 0x5f
+
+#CHECK: vsel    %v0, %v0, %v0, %v0
+0xe7 0x00 0x00 0x00 0x00 0x8d
+
+#CHECK: vsel    %v3, %v20, %v5, %v22
+0xe7 0x34 0x50 0x00 0x65 0x8d
+
+#CHECK: vsel    %v31, %v31, %v31, %v31
+0xe7 0xff 0xf0 0x00 0xff 0x8d
+
+#CHECK: vsf     %v0, %v0, %v0
+0xe7 0x00 0x00 0x00 0x20 0xf7
+
+#CHECK: vsf     %v18, %v3, %v20
+0xe7 0x23 0x40 0x00 0x2a 0xf7
+
+#CHECK: vsf     %v31, %v31, %v31
+0xe7 0xff 0xf0 0x00 0x2e 0xf7
+
+#CHECK: vsg     %v0, %v0, %v0
+0xe7 0x00 0x00 0x00 0x30 0xf7
+
+#CHECK: vsg     %v18, %v3, %v20
+0xe7 0x23 0x40 0x00 0x3a 0xf7
+
+#CHECK: vsg     %v31, %v31, %v31
+0xe7 0xff 0xf0 0x00 0x3e 0xf7
+
+#CHECK: vsh     %v0, %v0, %v0
+0xe7 0x00 0x00 0x00 0x10 0xf7
+
+#CHECK: vsh     %v18, %v3, %v20
+0xe7 0x23 0x40 0x00 0x1a 0xf7
+
+#CHECK: vsh     %v31, %v31, %v31
+0xe7 0xff 0xf0 0x00 0x1e 0xf7
+
+#CHECK: vsl     %v0, %v0, %v0
+0xe7 0x00 0x00 0x00 0x00 0x74
+
+#CHECK: vsl     %v18, %v3, %v20
+0xe7 0x23 0x40 0x00 0x0a 0x74
+
+#CHECK: vsl     %v31, %v31, %v31
+0xe7 0xff 0xf0 0x00 0x0e 0x74
+
+#CHECK: vslb    %v0, %v0, %v0
+0xe7 0x00 0x00 0x00 0x00 0x75
+
+#CHECK: vslb    %v18, %v3, %v20
+0xe7 0x23 0x40 0x00 0x0a 0x75
+
+#CHECK: vslb    %v31, %v31, %v31
+0xe7 0xff 0xf0 0x00 0x0e 0x75
+
+#CHECK: vsldb   %v0, %v0, %v0, 0
+0xe7 0x00 0x00 0x00 0x00 0x77
+
+#CHECK: vsldb   %v3, %v20, %v5, 103
+0xe7 0x34 0x50 0x67 0x04 0x77
+
+#CHECK: vsldb   %v31, %v31, %v31, 255
+0xe7 0xff 0xf0 0xff 0x0e 0x77
+
+#CHECK: vsq     %v0, %v0, %v0
+0xe7 0x00 0x00 0x00 0x40 0xf7
+
+#CHECK: vsq     %v18, %v3, %v20
+0xe7 0x23 0x40 0x00 0x4a 0xf7
+
+#CHECK: vsq     %v31, %v31, %v31
+0xe7 0xff 0xf0 0x00 0x4e 0xf7
+
+#CHECK: vsra    %v0, %v0, %v0
+0xe7 0x00 0x00 0x00 0x00 0x7e
+
+#CHECK: vsra    %v18, %v3, %v20
+0xe7 0x23 0x40 0x00 0x0a 0x7e
+
+#CHECK: vsra    %v31, %v31, %v31
+0xe7 0xff 0xf0 0x00 0x0e 0x7e
+
+#CHECK: vsrab   %v0, %v0, %v0
+0xe7 0x00 0x00 0x00 0x00 0x7f
+
+#CHECK: vsrab   %v18, %v3, %v20
+0xe7 0x23 0x40 0x00 0x0a 0x7f
+
+#CHECK: vsrab   %v31, %v31, %v31
+0xe7 0xff 0xf0 0x00 0x0e 0x7f
+
+#CHECK: vsrl    %v0, %v0, %v0
+0xe7 0x00 0x00 0x00 0x00 0x7c
+
+#CHECK: vsrl    %v18, %v3, %v20
+0xe7 0x23 0x40 0x00 0x0a 0x7c
+
+#CHECK: vsrl    %v31, %v31, %v31
+0xe7 0xff 0xf0 0x00 0x0e 0x7c
+
+#CHECK: vsrlb   %v0, %v0, %v0
+0xe7 0x00 0x00 0x00 0x00 0x7d
+
+#CHECK: vsrlb   %v18, %v3, %v20
+0xe7 0x23 0x40 0x00 0x0a 0x7d
+
+#CHECK: vsrlb   %v31, %v31, %v31
+0xe7 0xff 0xf0 0x00 0x0e 0x7d
+
+#CHECK: vst     %v0, 0
+0xe7 0x00 0x00 0x00 0x00 0x0E
+
+#CHECK: vst     %v17, 2475(%r7,%r8)
+0xe7 0x17 0x89 0xab 0x08 0x0E
+
+#CHECK: vst     %v31, 4095(%r15,%r15)
+0xe7 0xff 0xff 0xff 0x08 0x0E
+
+#CHECK: vsteb   %v0, 0, 0
+0xe7 0x00 0x00 0x00 0x00 0x08
+
+#CHECK: vsteb   %v17, 2475(%r7,%r8), 12
+0xe7 0x17 0x89 0xab 0xc8 0x08
+
+#CHECK: vsteb   %v31, 4095(%r15,%r15), 15
+0xe7 0xff 0xff 0xff 0xf8 0x08
+
+#CHECK: vstef   %v0, 0, 0
+0xe7 0x00 0x00 0x00 0x00 0x0b
+
+#CHECK: vstef   %v17, 2475(%r7,%r8), 2
+0xe7 0x17 0x89 0xab 0x28 0x0b
+
+#CHECK: vstef   %v31, 4095(%r15,%r15), 3
+0xe7 0xff 0xff 0xff 0x38 0x0b
+
+#CHECK: vsteg   %v0, 0, 0
+0xe7 0x00 0x00 0x00 0x00 0x0a
+
+#CHECK: vsteg   %v17, 2475(%r7,%r8), 1
+0xe7 0x17 0x89 0xab 0x18 0x0a
+
+#CHECK: vsteg   %v31, 4095(%r15,%r15), 1
+0xe7 0xff 0xff 0xff 0x18 0x0a
+
+#CHECK: vsteh   %v0, 0, 0
+0xe7 0x00 0x00 0x00 0x00 0x09
+
+#CHECK: vsteh   %v17, 2475(%r7,%r8), 5
+0xe7 0x17 0x89 0xab 0x58 0x09
+
+#CHECK: vsteh   %v31, 4095(%r15,%r15), 7
+0xe7 0xff 0xff 0xff 0x78 0x09
+
+#CHECK: vstl    %v0, %r0, 0
+0xe7 0x00 0x00 0x00 0x00 0x3f
+
+#CHECK: vstl    %v18, %r3, 1383(%r4)
+0xe7 0x23 0x45 0x67 0x08 0x3f
+
+#CHECK: vstl    %v31, %r15, 4095(%r15)
+0xe7 0xff 0xff 0xff 0x08 0x3f
+
+#CHECK: vstm    %v0, %v0, 0
+0xe7 0x00 0x00 0x00 0x00 0x3e
+
+#CHECK: vstm    %v12, %v18, 1110(%r3)
+0xe7 0xc2 0x34 0x56 0x04 0x3e
+
+#CHECK: vstm    %v31, %v31, 4095(%r15)
+0xe7 0xff 0xff 0xff 0x0c 0x3e
+
+#CHECK: vstrcb   %v0, %v0, %v0, %v0, 0
+0xe7 0x00 0x00 0x00 0x00 0x8a
+
+#CHECK: vstrcb   %v0, %v0, %v0, %v0, 12
+0xe7 0x00 0x00 0xc0 0x00 0x8a
+
+#CHECK: vstrcb   %v18, %v3, %v20, %v5, 0
+0xe7 0x23 0x40 0x00 0x5a 0x8a
+
+#CHECK: vstrcb   %v31, %v31, %v31, %v31, 4
+0xe7 0xff 0xf0 0x40 0xff 0x8a
+
+#CHECK: vstrcbs  %v31, %v31, %v31, %v31, 8
+0xe7 0xff 0xf0 0x90 0xff 0x8a
+
+#CHECK: vstrczb  %v31, %v31, %v31, %v31, 4
+0xe7 0xff 0xf0 0x60 0xff 0x8a
+
+#CHECK: vstrczbs %v31, %v31, %v31, %v31, 8
+0xe7 0xff 0xf0 0xb0 0xff 0x8a
+
+#CHECK: vstrcf   %v0, %v0, %v0, %v0, 0
+0xe7 0x00 0x02 0x00 0x00 0x8a
+
+#CHECK: vstrcf   %v0, %v0, %v0, %v0, 12
+0xe7 0x00 0x02 0xc0 0x00 0x8a
+
+#CHECK: vstrcf   %v18, %v3, %v20, %v5, 0
+0xe7 0x23 0x42 0x00 0x5a 0x8a
+
+#CHECK: vstrcf   %v31, %v31, %v31, %v31, 4
+0xe7 0xff 0xf2 0x40 0xff 0x8a
+
+#CHECK: vstrcfs  %v31, %v31, %v31, %v31, 8
+0xe7 0xff 0xf2 0x90 0xff 0x8a
+
+#CHECK: vstrczf  %v31, %v31, %v31, %v31, 4
+0xe7 0xff 0xf2 0x60 0xff 0x8a
+
+#CHECK: vstrczfs %v31, %v31, %v31, %v31, 8
+0xe7 0xff 0xf2 0xb0 0xff 0x8a
+
+#CHECK: vstrch   %v0, %v0, %v0, %v0, 0
+0xe7 0x00 0x01 0x00 0x00 0x8a
+
+#CHECK: vstrch   %v0, %v0, %v0, %v0, 12
+0xe7 0x00 0x01 0xc0 0x00 0x8a
+
+#CHECK: vstrch   %v18, %v3, %v20, %v5, 0
+0xe7 0x23 0x41 0x00 0x5a 0x8a
+
+#CHECK: vstrch   %v31, %v31, %v31, %v31, 4
+0xe7 0xff 0xf1 0x40 0xff 0x8a
+
+#CHECK: vstrchs  %v31, %v31, %v31, %v31, 8
+0xe7 0xff 0xf1 0x90 0xff 0x8a
+
+#CHECK: vstrczh  %v31, %v31, %v31, %v31, 4
+0xe7 0xff 0xf1 0x60 0xff 0x8a
+
+#CHECK: vstrczhs %v31, %v31, %v31, %v31, 8
+0xe7 0xff 0xf1 0xb0 0xff 0x8a
+
+#CHECK: vsumgh  %v0, %v0, %v0
+0xe7 0x00 0x00 0x00 0x10 0x65
+
+#CHECK: vsumgh  %v18, %v3, %v20
+0xe7 0x23 0x40 0x00 0x1a 0x65
+
+#CHECK: vsumgh  %v31, %v31, %v31
+0xe7 0xff 0xf0 0x00 0x1e 0x65
+
+#CHECK: vsumgf  %v0, %v0, %v0
+0xe7 0x00 0x00 0x00 0x20 0x65
+
+#CHECK: vsumgf  %v18, %v3, %v20
+0xe7 0x23 0x40 0x00 0x2a 0x65
+
+#CHECK: vsumgf  %v31, %v31, %v31
+0xe7 0xff 0xf0 0x00 0x2e 0x65
+
+#CHECK: vsumqf  %v0, %v0, %v0
+0xe7 0x00 0x00 0x00 0x20 0x67
+
+#CHECK: vsumqf  %v18, %v3, %v20
+0xe7 0x23 0x40 0x00 0x2a 0x67
+
+#CHECK: vsumqf  %v31, %v31, %v31
+0xe7 0xff 0xf0 0x00 0x2e 0x67
+
+#CHECK: vsumqg  %v0, %v0, %v0
+0xe7 0x00 0x00 0x00 0x30 0x67
+
+#CHECK: vsumqg  %v18, %v3, %v20
+0xe7 0x23 0x40 0x00 0x3a 0x67
+
+#CHECK: vsumqg  %v31, %v31, %v31
+0xe7 0xff 0xf0 0x00 0x3e 0x67
+
+#CHECK: vsumb   %v0, %v0, %v0
+0xe7 0x00 0x00 0x00 0x00 0x64
+
+#CHECK: vsumb   %v18, %v3, %v20
+0xe7 0x23 0x40 0x00 0x0a 0x64
+
+#CHECK: vsumb   %v31, %v31, %v31
+0xe7 0xff 0xf0 0x00 0x0e 0x64
+
+#CHECK: vsumh   %v0, %v0, %v0
+0xe7 0x00 0x00 0x00 0x10 0x64
+
+#CHECK: vsumh   %v18, %v3, %v20
+0xe7 0x23 0x40 0x00 0x1a 0x64
+
+#CHECK: vsumh   %v31, %v31, %v31
+0xe7 0xff 0xf0 0x00 0x1e 0x64
+
+#CHECK: vtm     %v0, %v0
+0xe7 0x00 0x00 0x00 0x00 0xd8
+
+#CHECK: vtm     %v19, %v14
+0xe7 0x3e 0x00 0x00 0x08 0xd8
+
+#CHECK: vtm     %v31, %v31
+0xe7 0xff 0x00 0x00 0x0c 0xd8
+
+#CHECK: vuphb   %v0, %v0
+0xe7 0x00 0x00 0x00 0x00 0xd7
+
+#CHECK: vuphb   %v19, %v14
+0xe7 0x3e 0x00 0x00 0x08 0xd7
+
+#CHECK: vuphb   %v31, %v31
+0xe7 0xff 0x00 0x00 0x0c 0xd7
+
+#CHECK: vuphf   %v0, %v0
+0xe7 0x00 0x00 0x00 0x20 0xd7
+
+#CHECK: vuphf   %v19, %v14
+0xe7 0x3e 0x00 0x00 0x28 0xd7
+
+#CHECK: vuphf   %v31, %v31
+0xe7 0xff 0x00 0x00 0x2c 0xd7
+
+#CHECK: vuphh   %v0, %v0
+0xe7 0x00 0x00 0x00 0x10 0xd7
+
+#CHECK: vuphh   %v19, %v14
+0xe7 0x3e 0x00 0x00 0x18 0xd7
+
+#CHECK: vuphh   %v31, %v31
+0xe7 0xff 0x00 0x00 0x1c 0xd7
+
+#CHECK: vuplhb  %v0, %v0
+0xe7 0x00 0x00 0x00 0x00 0xd5
+
+#CHECK: vuplhb  %v19, %v14
+0xe7 0x3e 0x00 0x00 0x08 0xd5
+
+#CHECK: vuplhb  %v31, %v31
+0xe7 0xff 0x00 0x00 0x0c 0xd5
+
+#CHECK: vuplhf  %v0, %v0
+0xe7 0x00 0x00 0x00 0x20 0xd5
+
+#CHECK: vuplhf  %v19, %v14
+0xe7 0x3e 0x00 0x00 0x28 0xd5
+
+#CHECK: vuplhf  %v31, %v31
+0xe7 0xff 0x00 0x00 0x2c 0xd5
+
+#CHECK: vuplhh  %v0, %v0
+0xe7 0x00 0x00 0x00 0x10 0xd5
+
+#CHECK: vuplhh  %v19, %v14
+0xe7 0x3e 0x00 0x00 0x18 0xd5
+
+#CHECK: vuplhh  %v31, %v31
+0xe7 0xff 0x00 0x00 0x1c 0xd5
+
+#CHECK: vuplb   %v0, %v0
+0xe7 0x00 0x00 0x00 0x00 0xd6
+
+#CHECK: vuplb   %v19, %v14
+0xe7 0x3e 0x00 0x00 0x08 0xd6
+
+#CHECK: vuplb   %v31, %v31
+0xe7 0xff 0x00 0x00 0x0c 0xd6
+
+#CHECK: vuplf   %v0, %v0
+0xe7 0x00 0x00 0x00 0x20 0xd6
+
+#CHECK: vuplf   %v19, %v14
+0xe7 0x3e 0x00 0x00 0x28 0xd6
+
+#CHECK: vuplf   %v31, %v31
+0xe7 0xff 0x00 0x00 0x2c 0xd6
+
+#CHECK: vuplhw  %v0, %v0
+0xe7 0x00 0x00 0x00 0x10 0xd6
+
+#CHECK: vuplhw  %v19, %v14
+0xe7 0x3e 0x00 0x00 0x18 0xd6
+
+#CHECK: vuplhw  %v31, %v31
+0xe7 0xff 0x00 0x00 0x1c 0xd6
+
+#CHECK: vupllb  %v0, %v0
+0xe7 0x00 0x00 0x00 0x00 0xd4
+
+#CHECK: vupllb  %v19, %v14
+0xe7 0x3e 0x00 0x00 0x08 0xd4
+
+#CHECK: vupllb  %v31, %v31
+0xe7 0xff 0x00 0x00 0x0c 0xd4
+
+#CHECK: vupllf  %v0, %v0
+0xe7 0x00 0x00 0x00 0x20 0xd4
+
+#CHECK: vupllf  %v19, %v14
+0xe7 0x3e 0x00 0x00 0x28 0xd4
+
+#CHECK: vupllf  %v31, %v31
+0xe7 0xff 0x00 0x00 0x2c 0xd4
+
+#CHECK: vupllh  %v0, %v0
+0xe7 0x00 0x00 0x00 0x10 0xd4
+
+#CHECK: vupllh  %v19, %v14
+0xe7 0x3e 0x00 0x00 0x18 0xd4
+
+#CHECK: vupllh  %v31, %v31
+0xe7 0xff 0x00 0x00 0x1c 0xd4
+
+#CHECK: vx      %v0, %v0, %v0
+0xe7 0x00 0x00 0x00 0x00 0x6d
+
+#CHECK: vx      %v18, %v3, %v20
+0xe7 0x23 0x40 0x00 0x0a 0x6d
+
+#CHECK: vx      %v31, %v31, %v31
+0xe7 0xff 0xf0 0x00 0x0e 0x6d
+
+#CHECK: wcdgb   %f0, %f0, 0, 0
+0xe7 0x00 0x00 0x08 0x30 0xc3
+
+#CHECK: wcdgb   %v19, %f14, 4, 10
+0xe7 0x3e 0x00 0xac 0x38 0xc3
+
+#CHECK: wcdgb   %v31, %v31, 7, 15
+0xe7 0xff 0x00 0xff 0x3c 0xc3
+
+#CHECK: wcdlgb  %f0, %f0, 0, 0
+0xe7 0x00 0x00 0x08 0x30 0xc1
+
+#CHECK: wcdlgb  %v19, %f14, 4, 10
+0xe7 0x3e 0x00 0xac 0x38 0xc1
+
+#CHECK: wcdlgb  %v31, %v31, 7, 15
+0xe7 0xff 0x00 0xff 0x3c 0xc1
+
+#CHECK: wcgdb   %f0, %f0, 0, 0
+0xe7 0x00 0x00 0x08 0x30 0xc2
+
+#CHECK: wcgdb   %v19, %f14, 4, 10
+0xe7 0x3e 0x00 0xac 0x38 0xc2
+
+#CHECK: wcgdb   %v31, %v31, 7, 15
+0xe7 0xff 0x00 0xff 0x3c 0xc2
+
+#CHECK: wclgdb  %f0, %f0, 0, 0
+0xe7 0x00 0x00 0x08 0x30 0xc0
+
+#CHECK: wclgdb  %v19, %f14, 4, 10
+0xe7 0x3e 0x00 0xac 0x38 0xc0
+
+#CHECK: wclgdb  %v31, %v31, 7, 15
+0xe7 0xff 0x00 0xff 0x3c 0xc0
+
+#CHECK: wfadb   %f0, %f0, %f0
+0xe7 0x00 0x00 0x08 0x30 0xe3
+
+#CHECK: wfadb   %v18, %f3, %v20
+0xe7 0x23 0x40 0x08 0x3a 0xe3
+
+#CHECK: wfadb   %v31, %v31, %v31
+0xe7 0xff 0xf0 0x08 0x3e 0xe3
+
+#CHECK: wfcdb   %f0, %f0
+0xe7 0x00 0x00 0x00 0x30 0xcb
+
+#CHECK: wfcdb   %v19, %f14
+0xe7 0x3e 0x00 0x00 0x38 0xcb
+
+#CHECK: wfcdb   %v31, %v31
+0xe7 0xff 0x00 0x00 0x3c 0xcb
+
+#CHECK: wfcedb  %f0, %f0, %f0
+0xe7 0x00 0x00 0x08 0x30 0xe8
+
+#CHECK: wfcedb  %v18, %f3, %v20
+0xe7 0x23 0x40 0x08 0x3a 0xe8
+
+#CHECK: wfcedb  %v31, %v31, %v31
+0xe7 0xff 0xf0 0x08 0x3e 0xe8
+
+#CHECK: wfcedbs %f0, %f0, %f0
+0xe7 0x00 0x00 0x18 0x30 0xe8
+
+#CHECK: wfcedbs %v18, %f3, %v20
+0xe7 0x23 0x40 0x18 0x3a 0xe8
+
+#CHECK: wfcedbs %v31, %v31, %v31
+0xe7 0xff 0xf0 0x18 0x3e 0xe8
+
+#CHECK: wfchdb  %f0, %f0, %f0
+0xe7 0x00 0x00 0x08 0x30 0xeb
+
+#CHECK: wfchdb  %v18, %f3, %v20
+0xe7 0x23 0x40 0x08 0x3a 0xeb
+
+#CHECK: wfchdb  %v31, %v31, %v31
+0xe7 0xff 0xf0 0x08 0x3e 0xeb
+
+#CHECK: wfchdbs %f0, %f0, %f0
+0xe7 0x00 0x00 0x18 0x30 0xeb
+
+#CHECK: wfchdbs %v18, %f3, %v20
+0xe7 0x23 0x40 0x18 0x3a 0xeb
+
+#CHECK: wfchdbs %v31, %v31, %v31
+0xe7 0xff 0xf0 0x18 0x3e 0xeb
+
+#CHECK: wfchedb %f0, %f0, %f0
+0xe7 0x00 0x00 0x08 0x30 0xea
+
+#CHECK: wfchedb %v18, %f3, %v20
+0xe7 0x23 0x40 0x08 0x3a 0xea
+
+#CHECK: wfchedb %v31, %v31, %v31
+0xe7 0xff 0xf0 0x08 0x3e 0xea
+
+#CHECK: wfchedbs %f0, %f0, %f0
+0xe7 0x00 0x00 0x18 0x30 0xea
+
+#CHECK: wfchedbs %v18, %f3, %v20
+0xe7 0x23 0x40 0x18 0x3a 0xea
+
+#CHECK: wfchedbs %v31, %v31, %v31
+0xe7 0xff 0xf0 0x18 0x3e 0xea
+
+#CHECK: wfddb   %f0, %f0, %f0
+0xe7 0x00 0x00 0x08 0x30 0xe5
+
+#CHECK: wfddb   %v18, %f3, %v20
+0xe7 0x23 0x40 0x08 0x3a 0xe5
+
+#CHECK: wfddb   %v31, %v31, %v31
+0xe7 0xff 0xf0 0x08 0x3e 0xe5
+
+#CHECK: wfidb   %f0, %f0, 0, 0
+0xe7 0x00 0x00 0x08 0x30 0xc7
+
+#CHECK: wfidb   %v19, %f14, 4, 10
+0xe7 0x3e 0x00 0xac 0x38 0xc7
+
+#CHECK: wfidb   %v31, %v31, 7, 15
+0xe7 0xff 0x00 0xff 0x3c 0xc7
+
+#CHECK: wfkdb   %f0, %f0
+0xe7 0x00 0x00 0x00 0x30 0xca
+
+#CHECK: wfkdb   %v19, %f14
+0xe7 0x3e 0x00 0x00 0x38 0xca
+
+#CHECK: wfkdb   %v31, %v31
+0xe7 0xff 0x00 0x00 0x3c 0xca
+
+#CHECK: wflcdb  %f0, %f0
+0xe7 0x00 0x00 0x08 0x30 0xcc
+
+#CHECK: wflcdb  %v19, %f14
+0xe7 0x3e 0x00 0x08 0x38 0xcc
+
+#CHECK: wflcdb  %v31, %v31
+0xe7 0xff 0x00 0x08 0x3c 0xcc
+
+#CHECK: wflndb  %f0, %f0
+0xe7 0x00 0x00 0x18 0x30 0xcc
+
+#CHECK: wflndb  %v19, %f14
+0xe7 0x3e 0x00 0x18 0x38 0xcc
+
+#CHECK: wflndb  %v31, %v31
+0xe7 0xff 0x00 0x18 0x3c 0xcc
+
+#CHECK: wflpdb  %f0, %f0
+0xe7 0x00 0x00 0x28 0x30 0xcc
+
+#CHECK: wflpdb  %v19, %f14
+0xe7 0x3e 0x00 0x28 0x38 0xcc
+
+#CHECK: wflpdb  %v31, %v31
+0xe7 0xff 0x00 0x28 0x3c 0xcc
+
+#CHECK: wfmadb  %f0, %f0, %f0, %f0
+0xe7 0x00 0x03 0x08 0x00 0x8f
+
+#CHECK: wfmadb  %f3, %v20, %f5, %v22
+0xe7 0x34 0x53 0x08 0x65 0x8f
+
+#CHECK: wfmadb  %v31, %v31, %v31, %v31
+0xe7 0xff 0xf3 0x08 0xff 0x8f
+
+#CHECK: wfmdb   %f0, %f0, %f0
+0xe7 0x00 0x00 0x08 0x30 0xe7
+
+#CHECK: wfmdb   %v18, %f3, %v20
+0xe7 0x23 0x40 0x08 0x3a 0xe7
+
+#CHECK: wfmdb   %v31, %v31, %v31
+0xe7 0xff 0xf0 0x08 0x3e 0xe7
+
+#CHECK: wfmsdb  %f0, %f0, %f0, %f0
+0xe7 0x00 0x03 0x08 0x00 0x8e
+
+#CHECK: wfmsdb  %f3, %v20, %f5, %v22
+0xe7 0x34 0x53 0x08 0x65 0x8e
+
+#CHECK: wfmsdb  %v31, %v31, %v31, %v31
+0xe7 0xff 0xf3 0x08 0xff 0x8e
+
+#CHECK: wfsdb   %f0, %f0, %f0
+0xe7 0x00 0x00 0x08 0x30 0xe2
+
+#CHECK: wfsdb   %v18, %f3, %v20
+0xe7 0x23 0x40 0x08 0x3a 0xe2
+
+#CHECK: wfsdb   %v31, %v31, %v31
+0xe7 0xff 0xf0 0x08 0x3e 0xe2
+
+#CHECK: wfsqdb  %f0, %f0
+0xe7 0x00 0x00 0x08 0x30 0xce
+
+#CHECK: wfsqdb  %v19, %f14
+0xe7 0x3e 0x00 0x08 0x38 0xce
+
+#CHECK: wfsqdb  %v31, %v31
+0xe7 0xff 0x00 0x08 0x3c 0xce
+
+#CHECK: wftcidb %f0, %f0, 0
+0xe7 0x00 0x00 0x08 0x30 0x4a
+
+#CHECK: wftcidb %v19, %f4, 1383
+0xe7 0x34 0x56 0x78 0x38 0x4a
+
+#CHECK: wftcidb %v31, %v31, 4095
+0xe7 0xff 0xff 0xf8 0x3c 0x4a
+
+#CHECK: wldeb   %f0, %f0
+0xe7 0x00 0x00 0x08 0x20 0xc4
+
+#CHECK: wldeb   %v19, %f14
+0xe7 0x3e 0x00 0x08 0x28 0xc4
+
+#CHECK: wldeb   %v31, %v31
+0xe7 0xff 0x00 0x08 0x2c 0xc4
+
+#CHECK: wledb   %f0, %f0, 0, 0
+0xe7 0x00 0x00 0x08 0x30 0xc5
+
+#CHECK: wledb   %v19, %f14, 4, 10
+0xe7 0x3e 0x00 0xac 0x38 0xc5
+
+#CHECK: wledb   %v31, %v31, 7, 15
+0xe7 0xff 0x00 0xff 0x3c 0xc5
diff --git a/test/MC/Disassembler/X86/avx-512.txt b/test/MC/Disassembler/X86/avx-512.txt
index cfe5ffda2db..d618e7e5933 100644
--- a/test/MC/Disassembler/X86/avx-512.txt
+++ b/test/MC/Disassembler/X86/avx-512.txt
@@ -137,5 +137,5 @@
 # CHECK: vpcmpd $8, %zmm10, %zmm25, %k5
 0x62 0xd3 0x35 0x40 0x1f 0xea 0x8
 
-# CHECK: vcmppd	{sae}, $127, %zmm27, %zmm11, %k4
+# CHECK: vcmppd	$127,{sae}, %zmm27, %zmm11, %k4
 0x62 0x91 0xa5 0x58 0xc2 0xe3 0x7f
diff --git a/test/MC/Disassembler/X86/x86-16.txt b/test/MC/Disassembler/X86/x86-16.txt
index c6844cd3cef..021cb237181 100644
--- a/test/MC/Disassembler/X86/x86-16.txt
+++ b/test/MC/Disassembler/X86/x86-16.txt
@@ -786,3 +786,5 @@
 # CHECK: lretl
 0x66 0xcb
 
+# CHECK: callw	-1
+0xe8 0xff 0xff
diff --git a/test/MC/Disassembler/X86/x86-32.txt b/test/MC/Disassembler/X86/x86-32.txt
index 830b8306630..c51e0a3dc37 100644
--- a/test/MC/Disassembler/X86/x86-32.txt
+++ b/test/MC/Disassembler/X86/x86-32.txt
@@ -54,6 +54,9 @@
 # CHECK: calll	-1234
 0xe8 0x2e 0xfb 0xff 0xff
 
+# CHECK: callw	-1
+0x66 0xe8 0xff 0xff
+
 # CHECK: lfence
 0x0f 0xae 0xe8
 
diff --git a/test/MC/Disassembler/X86/x86-64.txt b/test/MC/Disassembler/X86/x86-64.txt
index f000d154984..065b2a57c84 100644
--- a/test/MC/Disassembler/X86/x86-64.txt
+++ b/test/MC/Disassembler/X86/x86-64.txt
@@ -301,3 +301,47 @@
 
 # CHECK: movq %rax, 1515870810
 0x67, 0x48 0xa3 0x5a 0x5a 0x5a 0x5a
+
+# CHECK: addq 255(%rip), %rbx
+0x49, 0x03, 0x1d, 0xff, 0x00, 0x00, 0x00
+
+# The following 4 encodings are equivalent, as confirmed by the 'xed64'
+# decoder tool provided by Intel, which we assume to be canonical even
+# if the real silicon does something different. If that should happen,
+# then we'll all have disassembler bugs to repair.
+
+# Try all combinations of EVEX.x and REX.b:
+# CHECK: vaddps	287453952(%rip), %zmm20, %zmm15
+0x62 0x11 0x5c 0x40 0x58 0x3d 0x00 0x33 0x22 0x11
+# CHECK: vaddps	287453952(%rip), %zmm20, %zmm15
+0x62 0x31 0x5c 0x40 0x58 0x3d 0x00 0x33 0x22 0x11
+# CHECK: vaddps	287453952(%rip), %zmm20, %zmm15
+0x62 0x51 0x5c 0x40 0x58 0x3d 0x00 0x33 0x22 0x11
+# CHECK: vaddps	287453952(%rip), %zmm20, %zmm15
+0x62 0x71 0x5c 0x40 0x58 0x3d 0x00 0x33 0x22 0x11
+
+# Known bugs: these use a SIB byte. The index register is incorrectly
+# printed as an xmm register. Indeed there are "gather" load instructions
+# taking a vector of indices, but ONLY those instructions can do that.
+# The CHECK lines test the current incorrect output; FIXME is desired.
+# CHECK: vaddps (%r10,%xmm9), %zmm20, %zmm15
+# FIXME: vaddps (%r10,%r9), %zmm20, %zmm15
+0x62 0x11 0x5c 0x40 0x58 0x3c 0x0a
+
+# CHECK: vaddps (%rdx,%xmm9), %zmm20, %zmm15
+# FIXME: vaddps (%rdx,%r9), %zmm20, %zmm15
+0x62 0x31 0x5c 0x40 0x58 0x3c 0x0a
+
+# CHECK: vaddps (%r10,%xmm1), %zmm20, %zmm15
+# FIXME: vaddps (%r10,%rcx), %zmm20, %zmm15
+0x62 0x51 0x5c 0x40 0x58 0x3c 0x0a
+
+# CHECK: vaddps (%rdx,%xmm1), %zmm20, %zmm15
+# FIXME: vaddps (%rdx,%rcx), %zmm20, %zmm15
+0x62 0x71 0x5c 0x40 0x58 0x3c 0x0a
+
+# CHECK: callq 32767
+0xe8 0xff 0x7f 0x00 0x00
+
+# CHECK: callq -32769
+0xe8 0xff 0x7f 0xff 0xff
diff --git a/test/MC/ELF/alias-to-local.s b/test/MC/ELF/alias-to-local.s
new file mode 100644
index 00000000000..4983833b33a
--- /dev/null
+++ b/test/MC/ELF/alias-to-local.s
@@ -0,0 +1,18 @@
+// RUN: llvm-mc -filetype=obj -triple x86_64-pc-linux-gnu < %s | llvm-readobj -r | FileCheck %s
+
+// CHECK:      Relocations [
+// CHECK-NEXT:   Section {{.*}} .rela.text {
+// CHECK-NEXT:     0x1 R_X86_64_32 zed 0x0
+// CHECK-NEXT:   }
+// CHECK-NEXT: ]
+
+foo:
+	movl	$zed, %eax
+
+
+	.section	.data.bar,"aGw",@progbits,zed,comdat
+bar:
+	.byte	42
+
+	.globl	zed
+zed = bar
diff --git a/test/MC/ELF/alias.s b/test/MC/ELF/alias.s
index 0ab6dd4b5b8..df39fd4be14 100644
--- a/test/MC/ELF/alias.s
+++ b/test/MC/ELF/alias.s
@@ -107,33 +107,25 @@ leaq .Llocal1(%rip), %rdi
 // CHECK-NEXT:     Section: .text
 // CHECK-NEXT:   }
 // CHECK-NEXT:   Symbol {
-// CHECK-NEXT:     Name: .text (0)
+// CHECK-NEXT:     Name: (0)
 // CHECK-NOT: Symbol {
 // CHECK:        }
 // CHECK-NEXT:   Symbol {
-// CHECK-NEXT:     Name: .data (0)
-// CHECK-NOT: Symbol {
-// CHECK:        }
-// CHECK-NEXT:   Symbol {
-// CHECK-NEXT:     Name: .bss (0)
-// CHECK-NOT: Symbol {
-// CHECK:        }
-// CHECK-NEXT:   Symbol {
-// CHECK-NEXT:     Name: bar3
+// CHECK-NEXT:     Name: bar2
 // CHECK-NEXT:     Value: 0x0
 // CHECK-NEXT:     Size: 0
 // CHECK-NEXT:     Binding: Global
 // CHECK-NEXT:     Type: None
 // CHECK-NEXT:     Other: 0
-// CHECK-NEXT:     Section: .text
+// CHECK-NEXT:     Section: Undefined (0x0)
 // CHECK-NEXT:   }
 // CHECK-NEXT:   Symbol {
-// CHECK-NEXT:     Name: bar2
+// CHECK-NEXT:     Name: bar3
 // CHECK-NEXT:     Value: 0x0
 // CHECK-NEXT:     Size: 0
 // CHECK-NEXT:     Binding: Global
 // CHECK-NEXT:     Type: None
 // CHECK-NEXT:     Other: 0
-// CHECK-NEXT:     Section: Undefined (0x0)
+// CHECK-NEXT:     Section: .text
 // CHECK-NEXT:   }
 // CHECK-NEXT: ]
diff --git a/test/MC/ELF/align.s b/test/MC/ELF/align.s
index 46be3df62f2..43f5b548d9e 100644
--- a/test/MC/ELF/align.s
+++ b/test/MC/ELF/align.s
@@ -23,7 +23,7 @@
 // CHECK-NEXT:     EntrySize: 0
 // CHECK-NEXT:   }
 // CHECK-NEXT:   Section {
-// CHECK-NEXT:     Index: 4
+// CHECK-NEXT:     Index:
 // CHECK-NEXT:     Name: .rodata
 // CHECK-NEXT:     Type: SHT_PROGBITS
 // CHECK-NEXT:     Flags [
diff --git a/test/MC/ELF/basic-elf-32.s b/test/MC/ELF/basic-elf-32.s
index 16266af7f44..1036b04a747 100644
--- a/test/MC/ELF/basic-elf-32.s
+++ b/test/MC/ELF/basic-elf-32.s
@@ -54,20 +54,6 @@ main:                                   # @main
 // CHECK: ]
 
 // CHECK: Symbols [
-// CHECK:   Symbol {
-// CHECK:     Binding: Local
-// CHECK:     Type: Section
-// CHECK:   }
-
-// CHECK:   Symbol {
-// CHECK:     Binding: Local
-// CHECK:     Type: Section
-// CHECK:   }
-
-// CHECK:   Symbol {
-// CHECK:     Binding: Local
-// CHECK:     Type: Section
-// CHECK:   }
 
 // CHECK:   Symbol {
 // CHECK:     Name: main
diff --git a/test/MC/ELF/basic-elf-64.s b/test/MC/ELF/basic-elf-64.s
index d99125e06a9..b93f9aebd3a 100644
--- a/test/MC/ELF/basic-elf-64.s
+++ b/test/MC/ELF/basic-elf-64.s
@@ -57,16 +57,6 @@ main:                                   # @main
 // CHECK:     Binding: Local
 // CHECK:     Type: Section
 
-// CHECK:   Symbol {
-// CHECK:     Binding: Local
-// CHECK:     Type: Section
-// CHECK:   }
-
-// CHECK:   Symbol {
-// CHECK:     Binding: Local
-// CHECK:     Type: Section
-// CHECK:   }
-
 // CHECK:   Symbol {
 // CHECK:     Name: main
 // CHECK:     Binding: Global
diff --git a/test/MC/ELF/bracket-exprs.s b/test/MC/ELF/bracket-exprs.s
index 96f9f9aa450..e7299fde700 100644
--- a/test/MC/ELF/bracket-exprs.s
+++ b/test/MC/ELF/bracket-exprs.s
@@ -1,7 +1,7 @@
 // RUN: llvm-mc -triple i386-unknown-unknown %s | FileCheck %s
 
 // CHECK: .byte 1
-.if [~0 >> 1] == -1
+.if [~0 >> 63] == 1
 .byte 1
 .else
 .byte 2
diff --git a/test/MC/ELF/cfi-adjust-cfa-offset.s b/test/MC/ELF/cfi-adjust-cfa-offset.s
index bbaa7850566..398ad54fe75 100644
--- a/test/MC/ELF/cfi-adjust-cfa-offset.s
+++ b/test/MC/ELF/cfi-adjust-cfa-offset.s
@@ -22,8 +22,8 @@ f:
 	.cfi_endproc
 
 // CHECK:        Section {
-// CHECK:          Index: 4
-// CHECK-NEXT:     Name: .eh_frame
+// CHECK:          Index:
+// CHECK:          Name: .eh_frame
 // CHECK-NEXT:     Type: SHT_PROGBITS
 // CHECK-NEXT:     Flags [
 // CHECK-NEXT:       SHF_ALLOC
@@ -38,7 +38,7 @@ f:
 // CHECK-NEXT:     Relocations [
 // CHECK-NEXT:     ]
 // CHECK-NEXT:     SectionData (
-// CHECK-NEXT:       0000: 14000000 00000000 037A5200 01781001
+// CHECK-NEXT:       0000: 14000000 00000000 017A5200 01781001
 // CHECK-NEXT:       0010: 1B0C0708 90010000 18000000 1C000000
 // CHECK-NEXT:       0020: 00000000 0A000000 00440E10 410E1444
 // CHECK-NEXT:       0030: 0E080000 10000000 38000000 00000000
@@ -47,7 +47,7 @@ f:
 // CHECK-NEXT:     )
 // CHECK-NEXT:   }
 // CHECK-NEXT:   Section {
-// CHECK-NEXT:     Index: 5
+// CHECK-NEXT:     Index:
 // CHECK-NEXT:     Name: .rela.eh_frame
 // CHECK-NEXT:     Type: SHT_RELA
 // CHECK-NEXT:     Flags [
@@ -55,8 +55,8 @@ f:
 // CHECK-NEXT:     Address: 0x0
 // CHECK-NEXT:     Offset:
 // CHECK-NEXT:     Size: 72
-// CHECK-NEXT:     Link: 7
-// CHECK-NEXT:     Info: 4
+// CHECK-NEXT:     Link:
+// CHECK-NEXT:     Info:
 // CHECK-NEXT:     AddressAlignment: 8
 // CHECK-NEXT:     EntrySize: 24
 // CHECK-NEXT:     Relocations [
diff --git a/test/MC/ELF/cfi-advance-loc2.s b/test/MC/ELF/cfi-advance-loc2.s
index be14a431a1d..13397920127 100644
--- a/test/MC/ELF/cfi-advance-loc2.s
+++ b/test/MC/ELF/cfi-advance-loc2.s
@@ -26,7 +26,7 @@ f:
 // CHECK-NEXT:     Relocations [
 // CHECK-NEXT:     ]
 // CHECK-NEXT:     SectionData (
-// CHECK-NEXT:       0000: 14000000 00000000 037A5200 01781001
+// CHECK-NEXT:       0000: 14000000 00000000 017A5200 01781001
 // CHECK-NEXT:       0010: 1B0C0708 90010000 14000000 1C000000
 // CHECK-NEXT:       0020: 00000000 01010000 00030001 0E080000
 // CHECK-NEXT:     )
@@ -40,8 +40,8 @@ f:
 // CHECK-NEXT:     Address: 0x0
 // CHECK-NEXT:     Offset:
 // CHECK-NEXT:     Size: 24
-// CHECK-NEXT:     Link: 7
-// CHECK-NEXT:     Info: 4
+// CHECK-NEXT:     Link:
+// CHECK-NEXT:     Info:
 // CHECK-NEXT:     AddressAlignment: 8
 // CHECK-NEXT:     EntrySize: 24
 // CHECK-NEXT:     Relocations [
diff --git a/test/MC/ELF/cfi-def-cfa-offset.s b/test/MC/ELF/cfi-def-cfa-offset.s
index d84ab829bb3..3e4ca57a516 100644
--- a/test/MC/ELF/cfi-def-cfa-offset.s
+++ b/test/MC/ELF/cfi-def-cfa-offset.s
@@ -11,8 +11,8 @@ f:
 	.cfi_endproc
 
 // CHECK:        Section {
-// CHECK:          Index: 4
-// CHECK-NEXT:     Name: .eh_frame
+// CHECK:          Index:
+// CHECK:          Name: .eh_frame
 // CHECK-NEXT:     Type: SHT_PROGBITS
 // CHECK-NEXT:     Flags [
 // CHECK-NEXT:       SHF_ALLOC
@@ -27,23 +27,23 @@ f:
 // CHECK-NEXT:     Relocations [
 // CHECK-NEXT:     ]
 // CHECK-NEXT:     SectionData (
-// CHECK-NEXT:       0000: 14000000 00000000 037A5200 01781001
+// CHECK-NEXT:       0000: 14000000 00000000 017A5200 01781001
 // CHECK-NEXT:       0010: 1B0C0708 90010000 14000000 1C000000
 // CHECK-NEXT:       0020: 00000000 0A000000 00440E10 450E0800
 // CHECK-NEXT:     )
 // CHECK-NEXT:   }
 
 // CHECK:        Section {
-// CHECK:          Index: 5
-// CHECK-NEXT:     Name: .rela.eh_frame
+// CHECK:          Index:
+// CHECK:          Name: .rela.eh_frame
 // CHECK-NEXT:     Type: SHT_RELA
 // CHECK-NEXT:     Flags [
 // CHECK-NEXT:     ]
 // CHECK-NEXT:     Address: 0x0
 // CHECK-NEXT:     Offset:
 // CHECK-NEXT:     Size: 24
-// CHECK-NEXT:     Link: 7
-// CHECK-NEXT:     Info: 4
+// CHECK-NEXT:     Link:
+// CHECK-NEXT:     Info:
 // CHECK-NEXT:     AddressAlignment: 8
 // CHECK-NEXT:     EntrySize: 24
 // CHECK-NEXT:     Relocations [
diff --git a/test/MC/ELF/cfi-def-cfa-register.s b/test/MC/ELF/cfi-def-cfa-register.s
index 1efe0b29eaa..53174cb2f55 100644
--- a/test/MC/ELF/cfi-def-cfa-register.s
+++ b/test/MC/ELF/cfi-def-cfa-register.s
@@ -23,7 +23,7 @@ f:
 // CHECK-NEXT:     Relocations [
 // CHECK-NEXT:     ]
 // CHECK-NEXT:     SectionData (
-// CHECK-NEXT:       0000: 14000000 00000000 037A5200 01781001
+// CHECK-NEXT:       0000: 14000000 00000000 017A5200 01781001
 // CHECK-NEXT:       0010: 1B0C0708 90010000 14000000 1C000000
 // CHECK-NEXT:       0020: 00000000 02000000 00410D06 00000000
 // CHECK-NEXT:     )
@@ -37,8 +37,8 @@ f:
 // CHECK-NEXT:     Address: 0x0
 // CHECK-NEXT:     Offset:
 // CHECK-NEXT:     Size: 24
-// CHECK-NEXT:     Link: 7
-// CHECK-NEXT:     Info: 4
+// CHECK-NEXT:     Link:
+// CHECK-NEXT:     Info:
 // CHECK-NEXT:     AddressAlignment: 8
 // CHECK-NEXT:     EntrySize: 24
 // CHECK-NEXT:     Relocations [
diff --git a/test/MC/ELF/cfi-def-cfa.s b/test/MC/ELF/cfi-def-cfa.s
index 5e185d5d981..9706c4da097 100644
--- a/test/MC/ELF/cfi-def-cfa.s
+++ b/test/MC/ELF/cfi-def-cfa.s
@@ -23,7 +23,7 @@ f:
 // CHECK-NEXT:     Relocations [
 // CHECK-NEXT:     ]
 // CHECK-NEXT:     SectionData (
-// CHECK-NEXT:       0000: 14000000 00000000 037A5200 01781001
+// CHECK-NEXT:       0000: 14000000 00000000 017A5200 01781001
 // CHECK-NEXT:       0010: 1B0C0708 90010000 14000000 1C000000
 // CHECK-NEXT:       0020: 00000000 02000000 00410C07 08000000
 // CHECK-NEXT:     )
@@ -37,8 +37,8 @@ f:
 // CHECK-NEXT:     Address: 0x0
 // CHECK-NEXT:     Offset:
 // CHECK-NEXT:     Size: 24
-// CHECK-NEXT:     Link: 7
-// CHECK-NEXT:     Info: 4
+// CHECK-NEXT:     Link:
+// CHECK-NEXT:     Info:
 // CHECK-NEXT:     AddressAlignment: 8
 // CHECK-NEXT:     EntrySize: 24
 // CHECK-NEXT:     Relocations [
diff --git a/test/MC/ELF/cfi-escape.s b/test/MC/ELF/cfi-escape.s
index 5c427d62304..e93d5f7be5f 100644
--- a/test/MC/ELF/cfi-escape.s
+++ b/test/MC/ELF/cfi-escape.s
@@ -8,8 +8,8 @@ f:
 	.cfi_endproc
 
 // CHECK:        Section {
-// CHECK:          Index: 4
-// CHECK-NEXT:     Name: .eh_frame
+// CHECK:          Index:
+// CHECK:          Name: .eh_frame
 // CHECK-NEXT:     Type: SHT_PROGBITS
 // CHECK-NEXT:     Flags [
 // CHECK-NEXT:       SHF_ALLOC
@@ -24,13 +24,13 @@ f:
 // CHECK-NEXT:     Relocations [
 // CHECK-NEXT:     ]
 // CHECK-NEXT:     SectionData (
-// CHECK-NEXT:       0000: 14000000 00000000 037A5200 01781001
+// CHECK-NEXT:       0000: 14000000 00000000 017A5200 01781001
 // CHECK-NEXT:       0010: 1B0C0708 90010000 14000000 1C000000
 // CHECK-NEXT:       0020: 00000000 02000000 00411507 7F000000
 // CHECK-NEXT:     )
 // CHECK-NEXT:   }
 // CHECK-NEXT:   Section {
-// CHECK-NEXT:     Index: 5
+// CHECK-NEXT:     Index:
 // CHECK-NEXT:     Name: .rela.eh_frame
 // CHECK-NEXT:     Type: SHT_RELA
 // CHECK-NEXT:     Flags [
@@ -38,8 +38,8 @@ f:
 // CHECK-NEXT:     Address: 0x0
 // CHECK-NEXT:     Offset:
 // CHECK-NEXT:     Size: 24
-// CHECK-NEXT:     Link: 7
-// CHECK-NEXT:     Info: 4
+// CHECK-NEXT:     Link:
+// CHECK-NEXT:     Info:
 // CHECK-NEXT:     AddressAlignment: 8
 // CHECK-NEXT:     EntrySize: 24
 // CHECK-NEXT:     Relocations [
diff --git a/test/MC/ELF/cfi-large-model.s b/test/MC/ELF/cfi-large-model.s
index 16073ad7f0e..f4a20c26267 100644
--- a/test/MC/ELF/cfi-large-model.s
+++ b/test/MC/ELF/cfi-large-model.s
@@ -16,7 +16,7 @@
 // CHECK-NEXT:   AddressAlignment: 8
 // CHECK-NEXT:   EntrySize: 0
 // CHECK-NEXT:   SectionData (
-// CHECK-NEXT:     0000: 14000000 00000000 037A5200 01781001  |.........zR..x..|
+// CHECK-NEXT:     0000: 14000000 00000000 017A5200 01781001  |.........zR..x..|
 // CHECK-NEXT:     0010: 1C0C0708 90010000 1C000000 1C000000  |................|
 // CHECK-NEXT:     0020: 00000000 00000000 00000000 00000000  |................|
 // CHECK-NEXT:     0030: 00000000 00000000                    |........|
diff --git a/test/MC/ELF/cfi-offset.s b/test/MC/ELF/cfi-offset.s
index ad73b502535..9038def03a6 100644
--- a/test/MC/ELF/cfi-offset.s
+++ b/test/MC/ELF/cfi-offset.s
@@ -23,7 +23,7 @@ f:
 // CHECK-NEXT:     Relocations [
 // CHECK-NEXT:     ]
 // CHECK-NEXT:     SectionData (
-// CHECK-NEXT:       0000: 14000000 00000000 037A5200 01781001
+// CHECK-NEXT:       0000: 14000000 00000000 017A5200 01781001
 // CHECK-NEXT:       0010: 1B0C0708 90010000 14000000 1C000000
 // CHECK-NEXT:       0020: 00000000 02000000 00418602 00000000
 // CHECK-NEXT:     )
@@ -37,8 +37,8 @@ f:
 // CHECK-NEXT:     Address: 0x0
 // CHECK-NEXT:     Offset:
 // CHECK-NEXT:     Size: 24
-// CHECK-NEXT:     Link: 7
-// CHECK-NEXT:     Info: 4
+// CHECK-NEXT:     Link:
+// CHECK-NEXT:     Info:
 // CHECK-NEXT:     AddressAlignment: 8
 // CHECK-NEXT:     EntrySize: 24
 // CHECK-NEXT:     Relocations [
diff --git a/test/MC/ELF/cfi-register.s b/test/MC/ELF/cfi-register.s
index 48fd87953b6..be60e2ea2e8 100644
--- a/test/MC/ELF/cfi-register.s
+++ b/test/MC/ELF/cfi-register.s
@@ -8,8 +8,8 @@ f:
 	.cfi_endproc
 
 // CHECK:        Section {
-// CHECK:          Index: 4
-// CHECK-NEXT:     Name: .eh_frame
+// CHECK:          Index:
+// CHECK:          Name: .eh_frame
 // CHECK-NEXT:     Type: SHT_PROGBITS
 // CHECK-NEXT:     Flags [
 // CHECK-NEXT:       SHF_ALLOC
@@ -24,13 +24,13 @@ f:
 // CHECK-NEXT:     Relocations [
 // CHECK-NEXT:     ]
 // CHECK-NEXT:     SectionData (
-// CHECK-NEXT:       0000: 14000000 00000000 037A5200 01781001
+// CHECK-NEXT:       0000: 14000000 00000000 017A5200 01781001
 // CHECK-NEXT:       0010: 1B0C0708 90010000 14000000 1C000000
 // CHECK-NEXT:       0020: 00000000 02000000 00410906 00000000
 // CHECK-NEXT:     )
 // CHECK-NEXT:   }
 // CHECK-NEXT:   Section {
-// CHECK-NEXT:     Index: 5
+// CHECK-NEXT:     Index:
 // CHECK-NEXT:     Name: .rela.eh_frame
 // CHECK-NEXT:     Type: SHT_RELA
 // CHECK-NEXT:     Flags [
@@ -38,8 +38,8 @@ f:
 // CHECK-NEXT:     Address: 0x0
 // CHECK-NEXT:     Offset:
 // CHECK-NEXT:     Size: 24
-// CHECK-NEXT:     Link: 7
-// CHECK-NEXT:     Info: 4
+// CHECK-NEXT:     Link:
+// CHECK-NEXT:     Info:
 // CHECK-NEXT:     AddressAlignment: 8
 // CHECK-NEXT:     EntrySize: 24
 // CHECK-NEXT:     Relocations [
diff --git a/test/MC/ELF/cfi-rel-offset.s b/test/MC/ELF/cfi-rel-offset.s
index 15e1f9eac15..892cda165ba 100644
--- a/test/MC/ELF/cfi-rel-offset.s
+++ b/test/MC/ELF/cfi-rel-offset.s
@@ -15,8 +15,8 @@ f:
 	.cfi_endproc
 
 // CHECK:        Section {
-// CHECK:          Index: 4
-// CHECK-NEXT:     Name: .eh_frame
+// CHECK:          Index:
+// CHECK:          Name: .eh_frame
 // CHECK-NEXT:     Type: SHT_PROGBITS
 // CHECK-NEXT:     Flags [
 // CHECK-NEXT:       SHF_ALLOC
@@ -31,14 +31,14 @@ f:
 // CHECK-NEXT:     Relocations [
 // CHECK-NEXT:     ]
 // CHECK-NEXT:     SectionData (
-// CHECK-NEXT:       0000: 14000000 00000000 037A5200 01781001
+// CHECK-NEXT:       0000: 14000000 00000000 017A5200 01781001
 // CHECK-NEXT:       0010: 1B0C0708 90010000 24000000 1C000000
 // CHECK-NEXT:       0020: 00000000 05000000 00410E08 410D0641
 // CHECK-NEXT:       0030: 11067F41 0E104186 02000000 00000000
 // CHECK-NEXT:     )
 // CHECK-NEXT:   }
 // CHECK-NEXT:   Section {
-// CHECK-NEXT:     Index: 5
+// CHECK-NEXT:     Index:
 // CHECK-NEXT:     Name: .rela.eh_frame
 // CHECK-NEXT:     Type: SHT_RELA
 // CHECK-NEXT:     Flags [
@@ -46,8 +46,8 @@ f:
 // CHECK-NEXT:     Address: 0x0
 // CHECK-NEXT:     Offset:
 // CHECK-NEXT:     Size: 24
-// CHECK-NEXT:     Link: 7
-// CHECK-NEXT:     Info: 4
+// CHECK-NEXT:     Link:
+// CHECK-NEXT:     Info:
 // CHECK-NEXT:     AddressAlignment: 8
 // CHECK-NEXT:     EntrySize: 24
 // CHECK-NEXT:     Relocations [
diff --git a/test/MC/ELF/cfi-rel-offset2.s b/test/MC/ELF/cfi-rel-offset2.s
index 53c207576a3..805b0b290bc 100644
--- a/test/MC/ELF/cfi-rel-offset2.s
+++ b/test/MC/ELF/cfi-rel-offset2.s
@@ -7,8 +7,8 @@ f:
 	.cfi_endproc
 
 // CHECK:        Section {
-// CHECK:          Index: 4
-// CHECK-NEXT:     Name: .eh_frame
+// CHECK:          Index:
+// CHECK:          Name: .eh_frame
 // CHECK-NEXT:     Type: SHT_PROGBITS
 // CHECK-NEXT:     Flags [
 // CHECK-NEXT:       SHF_ALLOC
@@ -23,13 +23,13 @@ f:
 // CHECK-NEXT:     Relocations [
 // CHECK-NEXT:     ]
 // CHECK-NEXT:     SectionData (
-// CHECK-NEXT:       0000: 14000000 00000000 037A5200 01781001
+// CHECK-NEXT:       0000: 14000000 00000000 017A5200 01781001
 // CHECK-NEXT:       0010: 1B0C0708 90010000 14000000 1C000000
 // CHECK-NEXT:       0020: 00000000 01000000 00411106 7F000000
 // CHECK-NEXT:     )
 // CHECK-NEXT:   }
 // CHECK-NEXT:   Section {
-// CHECK-NEXT:     Index: 5
+// CHECK-NEXT:     Index:
 // CHECK-NEXT:     Name: .rela.eh_frame
 // CHECK-NEXT:     Type: SHT_RELA
 // CHECK-NEXT:     Flags [
@@ -37,8 +37,8 @@ f:
 // CHECK-NEXT:     Address: 0x0
 // CHECK-NEXT:     Offset:
 // CHECK-NEXT:     Size: 24
-// CHECK-NEXT:     Link: 7
-// CHECK-NEXT:     Info: 4
+// CHECK-NEXT:     Link:
+// CHECK-NEXT:     Info:
 // CHECK-NEXT:     AddressAlignment: 8
 // CHECK-NEXT:     EntrySize: 24
 // CHECK-NEXT:     Relocations [
diff --git a/test/MC/ELF/cfi-remember.s b/test/MC/ELF/cfi-remember.s
index d9de0ff170b..ae112f59ebe 100644
--- a/test/MC/ELF/cfi-remember.s
+++ b/test/MC/ELF/cfi-remember.s
@@ -10,8 +10,8 @@ f:
 	.cfi_endproc
 
 // CHECK:        Section {
-// CHECK:          Index: 4
-// CHECK-NEXT:     Name: .eh_frame
+// CHECK:          Index:
+// CHECK:          Name: .eh_frame
 // CHECK-NEXT:     Type: SHT_PROGBITS
 // CHECK-NEXT:     Flags [
 // CHECK-NEXT:       SHF_ALLOC
@@ -26,13 +26,13 @@ f:
 // CHECK-NEXT:     Relocations [
 // CHECK-NEXT:     ]
 // CHECK-NEXT:     SectionData (
-// CHECK-NEXT:       0000: 14000000 00000000 037A5200 01781001
+// CHECK-NEXT:       0000: 14000000 00000000 017A5200 01781001
 // CHECK-NEXT:       0010: 1B0C0708 90010000 14000000 1C000000
 // CHECK-NEXT:       0020: 00000000 03000000 00410A41 0B000000
 // CHECK-NEXT:     )
 // CHECK-NEXT:   }
 // CHECK:        Section {
-// CHECK:          Index: 5
+// CHECK:          Index:
 // CHECK-NEXT:     Name: .rela.eh_frame
 // CHECK-NEXT:     Type: SHT_RELA
 // CHECK-NEXT:     Flags [
@@ -40,8 +40,8 @@ f:
 // CHECK-NEXT:     Address: 0x0
 // CHECK-NEXT:     Offset:
 // CHECK-NEXT:     Size: 24
-// CHECK-NEXT:     Link: 7
-// CHECK-NEXT:     Info: 4
+// CHECK-NEXT:     Link:
+// CHECK-NEXT:     Info:
 // CHECK-NEXT:     AddressAlignment: 8
 // CHECK-NEXT:     EntrySize: 24
 // CHECK-NEXT:     Relocations [
diff --git a/test/MC/ELF/cfi-restore.s b/test/MC/ELF/cfi-restore.s
index 19e16247cea..2bc87cf691a 100644
--- a/test/MC/ELF/cfi-restore.s
+++ b/test/MC/ELF/cfi-restore.s
@@ -8,8 +8,8 @@ f:
 	.cfi_endproc
 
 // CHECK:        Section {
-// CHECK:          Index: 4
-// CHECK-NEXT:     Name: .eh_frame
+// CHECK:          Index:
+// CHECK:          Name: .eh_frame
 // CHECK-NEXT:     Type: SHT_PROGBITS
 // CHECK-NEXT:     Flags [
 // CHECK-NEXT:       SHF_ALLOC
@@ -24,13 +24,13 @@ f:
 // CHECK-NEXT:     Relocations [
 // CHECK-NEXT:     ]
 // CHECK-NEXT:     SectionData (
-// CHECK-NEXT:       0000: 14000000 00000000 037A5200 01781001
+// CHECK-NEXT:       0000: 14000000 00000000 017A5200 01781001
 // CHECK-NEXT:       0010: 1B0C0708 90010000 14000000 1C000000
 // CHECK-NEXT:       0020: 00000000 02000000 0041C600 00000000
 // CHECK-NEXT:     )
 // CHECK-NEXT:   }
 // CHECK-NEXT:   Section {
-// CHECK-NEXT:     Index: 5
+// CHECK-NEXT:     Index:
 // CHECK-NEXT:     Name: .rela.eh_frame
 // CHECK-NEXT:     Type: SHT_RELA
 // CHECK-NEXT:     Flags [
@@ -38,8 +38,8 @@ f:
 // CHECK-NEXT:     Address: 0x0
 // CHECK-NEXT:     Offset:
 // CHECK-NEXT:     Size: 24
-// CHECK-NEXT:     Link: 7
-// CHECK-NEXT:     Info: 4
+// CHECK-NEXT:     Link:
+// CHECK-NEXT:     Info:
 // CHECK-NEXT:     AddressAlignment: 8
 // CHECK-NEXT:     EntrySize: 24
 // CHECK-NEXT:     Relocations [
diff --git a/test/MC/ELF/cfi-same-value.s b/test/MC/ELF/cfi-same-value.s
index 53da4dc2494..7df4cbbb6dd 100644
--- a/test/MC/ELF/cfi-same-value.s
+++ b/test/MC/ELF/cfi-same-value.s
@@ -8,8 +8,8 @@ f:
 	.cfi_endproc
 
 // CHECK:        Section {
-// CHECK:          Index: 4
-// CHECK-NEXT:     Name: .eh_frame
+// CHECK:          Index:
+// CHECK:          Name: .eh_frame
 // CHECK-NEXT:     Type: SHT_PROGBITS
 // CHECK-NEXT:     Flags [
 // CHECK-NEXT:       SHF_ALLOC
@@ -24,13 +24,13 @@ f:
 // CHECK-NEXT:     Relocations [
 // CHECK-NEXT:     ]
 // CHECK-NEXT:     SectionData (
-// CHECK-NEXT:       0000: 14000000 00000000 037A5200 01781001
+// CHECK-NEXT:       0000: 14000000 00000000 017A5200 01781001
 // CHECK-NEXT:       0010: 1B0C0708 90010000 14000000 1C000000
 // CHECK-NEXT:       0020: 00000000 02000000 00410806 00000000
 // CHECK-NEXT:     )
 // CHECK-NEXT:   }
 // CHECK-NEXT:   Section {
-// CHECK-NEXT:     Index: 5
+// CHECK-NEXT:     Index:
 // CHECK-NEXT:     Name: .rela.eh_frame
 // CHECK-NEXT:     Type: SHT_RELA
 // CHECK-NEXT:     Flags [
@@ -38,8 +38,8 @@ f:
 // CHECK-NEXT:     Address: 0x0
 // CHECK-NEXT:     Offset:
 // CHECK-NEXT:     Size: 24
-// CHECK-NEXT:     Link: 7
-// CHECK-NEXT:     Info: 4
+// CHECK-NEXT:     Link:
+// CHECK-NEXT:     Info:
 // CHECK-NEXT:     AddressAlignment: 8
 // CHECK-NEXT:     EntrySize: 24
 // CHECK-NEXT:     Relocations [
diff --git a/test/MC/ELF/cfi-sections.s b/test/MC/ELF/cfi-sections.s
index b0ba543e5bd..830fe260ec2 100644
--- a/test/MC/ELF/cfi-sections.s
+++ b/test/MC/ELF/cfi-sections.s
@@ -1,5 +1,11 @@
-// RUN: llvm-mc -filetype=obj -triple x86_64-pc-linux-gnu %s -o - | llvm-readobj -s -sd | FileCheck -check-prefix=ELF_64 %s
-// RUN: llvm-mc -filetype=obj -triple i686-pc-linux-gnu %s -o - | llvm-readobj -s -sd | FileCheck -check-prefix=ELF_32 %s
+// RUN: llvm-mc -filetype=obj -triple x86_64-pc-linux-gnu -dwarf-version 2 %s -o - | llvm-readobj -s -sd | FileCheck -check-prefix=ELF_64 -check-prefix=ELF_64_DWARF_2 %s
+// RUN: llvm-mc -filetype=obj -triple i686-pc-linux-gnu -dwarf-version 2 %s -o - | llvm-readobj -s -sd | FileCheck -check-prefix=ELF_32 -check-prefix=ELF_32_DWARF_2 %s
+
+// RUN: llvm-mc -filetype=obj -triple x86_64-pc-linux-gnu -dwarf-version 3 %s -o - | llvm-readobj -s -sd | FileCheck -check-prefix=ELF_64 -check-prefix=ELF_64_DWARF_3 %s
+// RUN: llvm-mc -filetype=obj -triple i686-pc-linux-gnu -dwarf-version 3 %s -o - | llvm-readobj -s -sd | FileCheck -check-prefix=ELF_32 -check-prefix=ELF_32_DWARF_3 %s
+
+// RUN: llvm-mc -filetype=obj -triple x86_64-pc-linux-gnu -dwarf-version 4 %s -o - | llvm-readobj -s -sd | FileCheck -check-prefix=ELF_64 -check-prefix=ELF_64_DWARF_4 %s
+// RUN: llvm-mc -filetype=obj -triple i686-pc-linux-gnu -dwarf-version 4 %s -o - | llvm-readobj -s -sd | FileCheck -check-prefix=ELF_32 -check-prefix=ELF_32_DWARF_4 %s
 
 .cfi_sections .debug_frame
 
@@ -26,8 +32,12 @@ f2:
 // ELF_64-NEXT:     AddressAlignment: 8
 // ELF_64-NEXT:     EntrySize: 0
 // ELF_64-NEXT:     SectionData (
-// ELF_64-NEXT:       0000: 14000000 FFFFFFFF 03000178 100C0708
-// ELF_64-NEXT:       0010: 90010000 00000000 14000000 00000000
+// ELF_64_DWARF_2-NEXT:       0000: 14000000 FFFFFFFF 01000178 100C0708
+// ELF_64_DWARF_2-NEXT:       0010: 90010000 00000000 14000000 00000000
+// ELF_64_DWARF_3-NEXT:       0000: 14000000 FFFFFFFF 03000178 100C0708
+// ELF_64_DWARF_3-NEXT:       0010: 90010000 00000000 14000000 00000000
+// ELF_64_DWARF_4-NEXT:       0000: 14000000 FFFFFFFF 04000800 0178100C
+// ELF_64_DWARF_4-NEXT:       0010: 07089001 00000000 14000000 00000000
 // ELF_64-NEXT:       0020: 00000000 00000000 01000000 00000000
 // ELF_64-NEXT:       0030: 14000000 00000000 00000000 00000000
 // ELF_64-NEXT:       0040: 01000000 00000000
@@ -47,9 +57,14 @@ f2:
 // ELF_32-NEXT:     AddressAlignment: 4
 // ELF_32-NEXT:     EntrySize: 0
 // ELF_32-NEXT:     SectionData (
-// ELF_32-NEXT:       0000: 10000000 FFFFFFFF 0300017C 080C0404
-// ELF_32-NEXT:       0010: 88010000 0C000000 00000000 00000000
+// ELF_32_DWARF_2-NEXT:       0000: 10000000 FFFFFFFF 0100017C 080C0404
+// ELF_32_DWARF_2-NEXT:       0010: 88010000 0C000000 00000000 00000000
+// ELF_32_DWARF_3-NEXT:       0000: 10000000 FFFFFFFF 0300017C 080C0404
+// ELF_32_DWARF_3-NEXT:       0010: 88010000 0C000000 00000000 00000000
+// ELF_32_DWARF_4-NEXT:       0000: 10000000 FFFFFFFF 04000400 017C080C
+// ELF_32_DWARF_4-NEXT:       0010: 04048801 0C000000 00000000 00000000
 // ELF_32-NEXT:       0020: 01000000 0C000000 00000000 01000000
 // ELF_32-NEXT:       0030: 01000000
+
 // ELF_32-NEXT:     )
 // ELF_32-NEXT:   }
diff --git a/test/MC/ELF/cfi-signal-frame.s b/test/MC/ELF/cfi-signal-frame.s
index 98deb0a1de5..02331196218 100644
--- a/test/MC/ELF/cfi-signal-frame.s
+++ b/test/MC/ELF/cfi-signal-frame.s
@@ -23,10 +23,10 @@ g:
 // CHECK-NEXT:     AddressAlignment: 8
 // CHECK-NEXT:     EntrySize: 0
 // CHECK-NEXT:     SectionData (
-// CHECK-NEXT:       0000: 14000000 00000000 037A5253 00017810
+// CHECK-NEXT:       0000: 14000000 00000000 017A5253 00017810
 // CHECK-NEXT:       0010: 011B0C07 08900100 10000000 1C000000
 // CHECK-NEXT:       0020: 00000000 00000000 00000000 14000000
-// CHECK-NEXT:       0030: 00000000 037A5200 01781001 1B0C0708
+// CHECK-NEXT:       0030: 00000000 017A5200 01781001 1B0C0708
 // CHECK-NEXT:       0040: 90010000 10000000 1C000000 00000000
 // CHECK-NEXT:       0050: 00000000 00000000
 // CHECK-NEXT:     )
diff --git a/test/MC/ELF/cfi-undefined.s b/test/MC/ELF/cfi-undefined.s
index 7afeea68101..9a7012310ad 100644
--- a/test/MC/ELF/cfi-undefined.s
+++ b/test/MC/ELF/cfi-undefined.s
@@ -8,8 +8,8 @@ f:
 	.cfi_endproc
 
 // CHECK:        Section {
-// CHECK:          Index: 4
-// CHECK-NEXT:     Name: .eh_frame
+// CHECK:          Index:
+// CHECK:          Name: .eh_frame
 // CHECK-NEXT:     Type: SHT_PROGBITS
 // CHECK-NEXT:     Flags [
 // CHECK-NEXT:       SHF_ALLOC
@@ -24,13 +24,13 @@ f:
 // CHECK-NEXT:     Relocations [
 // CHECK-NEXT:     ]
 // CHECK-NEXT:     SectionData (
-// CHECK-NEXT:       0000: 14000000 00000000 037A5200 01781001
+// CHECK-NEXT:       0000: 14000000 00000000 017A5200 01781001
 // CHECK-NEXT:       0010: 1B0C0708 90010000 14000000 1C000000
 // CHECK-NEXT:       0020: 00000000 02000000 00410706 00000000
 // CHECK-NEXT:     )
 // CHECK-NEXT:   }
 // CHECK-NEXT:   Section {
-// CHECK-NEXT:     Index: 5
+// CHECK-NEXT:     Index:
 // CHECK-NEXT:     Name: .rela.eh_frame
 // CHECK-NEXT:     Type: SHT_RELA
 // CHECK-NEXT:     Flags [
@@ -38,8 +38,8 @@ f:
 // CHECK-NEXT:     Address: 0x0
 // CHECK-NEXT:     Offset:
 // CHECK-NEXT:     Size: 24
-// CHECK-NEXT:     Link: 7
-// CHECK-NEXT:     Info: 4
+// CHECK-NEXT:     Link:
+// CHECK-NEXT:     Info:
 // CHECK-NEXT:     AddressAlignment: 8
 // CHECK-NEXT:     EntrySize: 24
 // CHECK-NEXT:     Relocations [
diff --git a/test/MC/ELF/cfi-version.ll b/test/MC/ELF/cfi-version.ll
index 2f34b2a1f08..9ea7be4a42b 100644
--- a/test/MC/ELF/cfi-version.ll
+++ b/test/MC/ELF/cfi-version.ll
@@ -1,6 +1,6 @@
 ; RUN: %llc_dwarf %s -o - -dwarf-version 2 -filetype=obj | llvm-dwarfdump - | FileCheck %s --check-prefix=DWARF2
-; RUN: %llc_dwarf %s -o - -dwarf-version 3 -filetype=obj | llvm-dwarfdump - | FileCheck %s --check-prefix=DWARF34
-; RUN: %llc_dwarf %s -o - -dwarf-version 4 -filetype=obj | llvm-dwarfdump - | FileCheck %s --check-prefix=DWARF34
+; RUN: %llc_dwarf %s -o - -dwarf-version 3 -filetype=obj | llvm-dwarfdump - | FileCheck %s --check-prefix=DWARF3
+; RUN: %llc_dwarf %s -o - -dwarf-version 4 -filetype=obj | llvm-dwarfdump - | FileCheck %s --check-prefix=DWARF4
 
 ; .debug_frame is not emitted for targeting Windows x64.
 ; REQUIRES: debug_frame
@@ -22,24 +22,34 @@ attributes #1 = { "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "n
 !llvm.module.flags = !{!9, !10}
 !llvm.ident = !{!11}
 
-!0 = !MDCompileUnit(language: DW_LANG_C99, producer: "clang version 3.5.0 ", isOptimized: false, emissionKind: 1, file: !1, enums: !2, retainedTypes: !2, subprograms: !3, globals: !2, imports: !2)
-!1 = !MDFile(filename: "test.c", directory: "/tmp")
+!0 = !DICompileUnit(language: DW_LANG_C99, producer: "clang version 3.5.0 ", isOptimized: false, emissionKind: 1, file: !1, enums: !2, retainedTypes: !2, subprograms: !3, globals: !2, imports: !2)
+!1 = !DIFile(filename: "test.c", directory: "/tmp")
 !2 = !{}
 !3 = !{!4}
-!4 = !MDSubprogram(name: "foo", line: 2, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 2, file: !1, scope: !5, type: !6, function: i32 ()* @foo, variables: !2)
-!5 = !MDFile(filename: "test.c", directory: "/tmp")
-!6 = !MDSubroutineType(types: !7)
+!4 = !DISubprogram(name: "foo", line: 2, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 2, file: !1, scope: !5, type: !6, function: i32 ()* @foo, variables: !2)
+!5 = !DIFile(filename: "test.c", directory: "/tmp")
+!6 = !DISubroutineType(types: !7)
 !7 = !{!8}
-!8 = !MDBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
+!8 = !DIBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
 !9 = !{i32 2, !"Dwarf Version", i32 4}
 !10 = !{i32 1, !"Debug Info Version", i32 3}
 !11 = !{!"clang version 3.5.0 "}
-!12 = !MDLocation(line: 2, scope: !4)
+!12 = !DILocation(line: 2, scope: !4)
 
 ; DWARF2:      .debug_frame contents:
 ; DWARF2:        Version:               1
 ; DWARF2-NEXT:   Augmentation:
-
-; DWARF34:      .debug_frame contents:
-; DWARF34:        Version:               3
-; DWARF34-NEXT:   Augmentation:
+; DWARF2-NOT:    Address size:
+; DWARF2-NOT:    Segment desc size:
+
+; DWARF3:      .debug_frame contents:
+; DWARF3:        Version:               3
+; DWARF3-NEXT:   Augmentation:
+; DWARF3-NOT:    Address size:
+; DWARF3-NOT:    Segment desc size:
+
+; DWARF4:      .debug_frame contents:
+; DWARF4:        Version:               4
+; DWARF4-NEXT:   Augmentation:
+; DWARF4:        Address size:
+; DWARF4:        Segment desc size:
diff --git a/test/MC/ELF/cfi-window-save.s b/test/MC/ELF/cfi-window-save.s
index 00fd37f7fb1..fb38ecd8560 100644
--- a/test/MC/ELF/cfi-window-save.s
+++ b/test/MC/ELF/cfi-window-save.s
@@ -26,7 +26,7 @@ f:
 // CHECK-NEXT:     Relocations [
 // CHECK-NEXT:     ]
 // CHECK-NEXT:     SectionData (
-// CHECK-NEXT:       0000: 14000000 00000000 037A5200 01781001
+// CHECK-NEXT:       0000: 14000000 00000000 017A5200 01781001
 // CHECK-NEXT:       0010: 1B0C0708 90010000 14000000 1C000000
 // CHECK-NEXT:       0020: 00000000 02000000 00412D00 00000000
 // CHECK-NEXT:     )
@@ -40,8 +40,8 @@ f:
 // CHECK-NEXT:     Address: 0x0
 // CHECK-NEXT:     Offset:
 // CHECK-NEXT:     Size: 24
-// CHECK-NEXT:     Link: 7
-// CHECK-NEXT:     Info: 4
+// CHECK-NEXT:     Link:
+// CHECK-NEXT:     Info:
 // CHECK-NEXT:     AddressAlignment: 8
 // CHECK-NEXT:     EntrySize: 24
 // CHECK-NEXT:     Relocations [
diff --git a/test/MC/ELF/cfi-zero-addr-delta.s b/test/MC/ELF/cfi-zero-addr-delta.s
index 85aa3e126df..1e5c5e7e568 100644
--- a/test/MC/ELF/cfi-zero-addr-delta.s
+++ b/test/MC/ELF/cfi-zero-addr-delta.s
@@ -30,7 +30,7 @@ f:
 // CHECK-NEXT:     Relocations [
 // CHECK-NEXT:     ]
 // CHECK-NEXT:     SectionData (
-// CHECK-NEXT:       0000: 14000000 00000000 037A5200 01781001
+// CHECK-NEXT:       0000: 14000000 00000000 017A5200 01781001
 // CHECK-NEXT:       0010: 1B0C0708 90010000 1C000000 1C000000
 // CHECK-NEXT:       0020: 00000000 04000000 00410E10 410A0E08
 // CHECK-NEXT:       0030: 410B0000 00000000
@@ -45,8 +45,8 @@ f:
 // CHECK-NEXT:     Address: 0x0
 // CHECK-NEXT:     Offset:
 // CHECK-NEXT:     Size: 24
-// CHECK-NEXT:     Link: 7
-// CHECK-NEXT:     Info: 4
+// CHECK-NEXT:     Link:
+// CHECK-NEXT:     Info:
 // CHECK-NEXT:     AddressAlignment: 8
 // CHECK-NEXT:     EntrySize: 24
 // CHECK-NEXT:     Relocations [
diff --git a/test/MC/ELF/cfi.s b/test/MC/ELF/cfi.s
index dc61dca7ca6..10587e03052 100644
--- a/test/MC/ELF/cfi.s
+++ b/test/MC/ELF/cfi.s
@@ -218,8 +218,8 @@ f37:
         .cfi_endproc
 
 // CHECK:        Section {
-// CHECK:          Index: 4
-// CHECK-NEXT:     Name: .eh_frame
+// CHECK:          Index:
+// CHECK:          Name: .eh_frame
 // CHECK-NEXT:     Type: SHT_PROGBITS
 // CHECK-NEXT:     Flags [
 // CHECK-NEXT:       SHF_ALLOC
@@ -234,131 +234,131 @@ f37:
 // CHECK-NEXT:     Relocations [
 // CHECK-NEXT:     ]
 // CHECK-NEXT:     SectionData (
-// CHECK-NEXT:       0000: 14000000 00000000 037A4C52 00017810
+// CHECK-NEXT:       0000: 14000000 00000000 017A4C52 00017810
 // CHECK-NEXT:       0010: 02031B0C 07089001 14000000 1C000000
 // CHECK-NEXT:       0020: 00000000 01000000 04000000 00000000
-// CHECK-NEXT:       0030: 20000000 00000000 037A504C 52000178
+// CHECK-NEXT:       0030: 20000000 00000000 017A504C 52000178
 // CHECK-NEXT:       0040: 100B0000 00000000 00000003 1B0C0708
 // CHECK-NEXT:       0050: 90010000 14000000 28000000 00000000
 // CHECK-NEXT:       0060: 01000000 04000000 00000000 14000000
 // CHECK-NEXT:       0070: 70000000 00000000 01000000 04000000
-// CHECK-NEXT:       0080: 00000000 20000000 00000000 037A504C
+// CHECK-NEXT:       0080: 00000000 20000000 00000000 017A504C
 // CHECK-NEXT:       0090: 52000178 100B0000 00000000 00000002
 // CHECK-NEXT:       00A0: 1B0C0708 90010000 10000000 28000000
 // CHECK-NEXT:       00B0: 00000000 01000000 02000000 18000000
-// CHECK-NEXT:       00C0: 00000000 037A5052 00017810 04020000
+// CHECK-NEXT:       00C0: 00000000 017A5052 00017810 04020000
 // CHECK-NEXT:       00D0: 1B0C0708 90010000 10000000 20000000
 // CHECK-NEXT:       00E0: 00000000 01000000 00000000 18000000
-// CHECK-NEXT:       00F0: 00000000 037A5052 00017810 06030000
+// CHECK-NEXT:       00F0: 00000000 017A5052 00017810 06030000
 // CHECK-NEXT:       0100: 00001B0C 07089001 10000000 20000000
 // CHECK-NEXT:       0110: 00000000 01000000 00000000 1C000000
-// CHECK-NEXT:       0120: 00000000 037A5052 00017810 0A040000
+// CHECK-NEXT:       0120: 00000000 017A5052 00017810 0A040000
 // CHECK-NEXT:       0130: 00000000 00001B0C 07089001 10000000
 // CHECK-NEXT:       0140: 24000000 00000000 01000000 00000000
-// CHECK-NEXT:       0150: 18000000 00000000 037A5052 00017810
+// CHECK-NEXT:       0150: 18000000 00000000 017A5052 00017810
 // CHECK-NEXT:       0160: 040A0000 1B0C0708 90010000 10000000
 // CHECK-NEXT:       0170: 20000000 00000000 01000000 00000000
-// CHECK-NEXT:       0180: 18000000 00000000 037A5052 00017810
+// CHECK-NEXT:       0180: 18000000 00000000 017A5052 00017810
 // CHECK-NEXT:       0190: 060B0000 00001B0C 07089001 10000000
 // CHECK-NEXT:       01A0: 20000000 00000000 01000000 00000000
-// CHECK-NEXT:       01B0: 1C000000 00000000 037A5052 00017810
+// CHECK-NEXT:       01B0: 1C000000 00000000 017A5052 00017810
 // CHECK-NEXT:       01C0: 0A0C0000 00000000 00001B0C 07089001
 // CHECK-NEXT:       01D0: 10000000 24000000 00000000 01000000
-// CHECK-NEXT:       01E0: 00000000 1C000000 00000000 037A5052
+// CHECK-NEXT:       01E0: 00000000 1C000000 00000000 017A5052
 // CHECK-NEXT:       01F0: 00017810 0A080000 00000000 00001B0C
 // CHECK-NEXT:       0200: 07089001 10000000 24000000 00000000
 // CHECK-NEXT:       0210: 01000000 00000000 1C000000 00000000
-// CHECK-NEXT:       0220: 037A5052 00017810 0A100000 00000000
+// CHECK-NEXT:       0220: 017A5052 00017810 0A100000 00000000
 // CHECK-NEXT:       0230: 00001B0C 07089001 10000000 24000000
 // CHECK-NEXT:       0240: 00000000 01000000 00000000 18000000
-// CHECK-NEXT:       0250: 00000000 037A5052 00017810 04120000
+// CHECK-NEXT:       0250: 00000000 017A5052 00017810 04120000
 // CHECK-NEXT:       0260: 1B0C0708 90010000 10000000 20000000
 // CHECK-NEXT:       0270: 00000000 01000000 00000000 18000000
-// CHECK-NEXT:       0280: 00000000 037A5052 00017810 06130000
+// CHECK-NEXT:       0280: 00000000 017A5052 00017810 06130000
 // CHECK-NEXT:       0290: 00001B0C 07089001 10000000 20000000
 // CHECK-NEXT:       02A0: 00000000 01000000 00000000 1C000000
-// CHECK-NEXT:       02B0: 00000000 037A5052 00017810 0A140000
+// CHECK-NEXT:       02B0: 00000000 017A5052 00017810 0A140000
 // CHECK-NEXT:       02C0: 00000000 00001B0C 07089001 10000000
 // CHECK-NEXT:       02D0: 24000000 00000000 01000000 00000000
-// CHECK-NEXT:       02E0: 18000000 00000000 037A5052 00017810
+// CHECK-NEXT:       02E0: 18000000 00000000 017A5052 00017810
 // CHECK-NEXT:       02F0: 041A0000 1B0C0708 90010000 10000000
 // CHECK-NEXT:       0300: 20000000 00000000 01000000 00000000
-// CHECK-NEXT:       0310: 18000000 00000000 037A5052 00017810
+// CHECK-NEXT:       0310: 18000000 00000000 017A5052 00017810
 // CHECK-NEXT:       0320: 061B0000 00001B0C 07089001 10000000
 // CHECK-NEXT:       0330: 20000000 00000000 01000000 00000000
-// CHECK-NEXT:       0340: 1C000000 00000000 037A5052 00017810
+// CHECK-NEXT:       0340: 1C000000 00000000 017A5052 00017810
 // CHECK-NEXT:       0350: 0A1C0000 00000000 00001B0C 07089001
 // CHECK-NEXT:       0360: 10000000 24000000 00000000 01000000
-// CHECK-NEXT:       0370: 00000000 1C000000 00000000 037A5052
+// CHECK-NEXT:       0370: 00000000 1C000000 00000000 017A5052
 // CHECK-NEXT:       0380: 00017810 0A180000 00000000 00001B0C
 // CHECK-NEXT:       0390: 07089001 10000000 24000000 00000000
 // CHECK-NEXT:       03A0: 01000000 00000000 1C000000 00000000
-// CHECK-NEXT:       03B0: 037A5052 00017810 0A800000 00000000
+// CHECK-NEXT:       03B0: 017A5052 00017810 0A800000 00000000
 // CHECK-NEXT:       03C0: 00001B0C 07089001 10000000 24000000
 // CHECK-NEXT:       03D0: 00000000 01000000 00000000 18000000
-// CHECK-NEXT:       03E0: 00000000 037A5052 00017810 04820000
+// CHECK-NEXT:       03E0: 00000000 017A5052 00017810 04820000
 // CHECK-NEXT:       03F0: 1B0C0708 90010000 10000000 20000000
 // CHECK-NEXT:       0400: 00000000 01000000 00000000 18000000
-// CHECK-NEXT:       0410: 00000000 037A5052 00017810 06830000
+// CHECK-NEXT:       0410: 00000000 017A5052 00017810 06830000
 // CHECK-NEXT:       0420: 00001B0C 07089001 10000000 20000000
 // CHECK-NEXT:       0430: 00000000 01000000 00000000 1C000000
-// CHECK-NEXT:       0440: 00000000 037A5052 00017810 0A840000
+// CHECK-NEXT:       0440: 00000000 017A5052 00017810 0A840000
 // CHECK-NEXT:       0450: 00000000 00001B0C 07089001 10000000
 // CHECK-NEXT:       0460: 24000000 00000000 01000000 00000000
-// CHECK-NEXT:       0470: 18000000 00000000 037A5052 00017810
+// CHECK-NEXT:       0470: 18000000 00000000 017A5052 00017810
 // CHECK-NEXT:       0480: 048A0000 1B0C0708 90010000 10000000
 // CHECK-NEXT:       0490: 20000000 00000000 01000000 00000000
-// CHECK-NEXT:       04A0: 18000000 00000000 037A5052 00017810
+// CHECK-NEXT:       04A0: 18000000 00000000 017A5052 00017810
 // CHECK-NEXT:       04B0: 068B0000 00001B0C 07089001 10000000
 // CHECK-NEXT:       04C0: 20000000 00000000 01000000 00000000
-// CHECK-NEXT:       04D0: 1C000000 00000000 037A5052 00017810
+// CHECK-NEXT:       04D0: 1C000000 00000000 017A5052 00017810
 // CHECK-NEXT:       04E0: 0A8C0000 00000000 00001B0C 07089001
 // CHECK-NEXT:       04F0: 10000000 24000000 00000000 01000000
-// CHECK-NEXT:       0500: 00000000 1C000000 00000000 037A5052
+// CHECK-NEXT:       0500: 00000000 1C000000 00000000 017A5052
 // CHECK-NEXT:       0510: 00017810 0A880000 00000000 00001B0C
 // CHECK-NEXT:       0520: 07089001 10000000 24000000 00000000
 // CHECK-NEXT:       0530: 01000000 00000000 1C000000 00000000
-// CHECK-NEXT:       0540: 037A5052 00017810 0A900000 00000000
+// CHECK-NEXT:       0540: 017A5052 00017810 0A900000 00000000
 // CHECK-NEXT:       0550: 00001B0C 07089001 10000000 24000000
 // CHECK-NEXT:       0560: 00000000 01000000 00000000 18000000
-// CHECK-NEXT:       0570: 00000000 037A5052 00017810 04920000
+// CHECK-NEXT:       0570: 00000000 017A5052 00017810 04920000
 // CHECK-NEXT:       0580: 1B0C0708 90010000 10000000 20000000
 // CHECK-NEXT:       0590: 00000000 01000000 00000000 18000000
-// CHECK-NEXT:       05A0: 00000000 037A5052 00017810 06930000
+// CHECK-NEXT:       05A0: 00000000 017A5052 00017810 06930000
 // CHECK-NEXT:       05B0: 00001B0C 07089001 10000000 20000000
 // CHECK-NEXT:       05C0: 00000000 01000000 00000000 1C000000
-// CHECK-NEXT:       05D0: 00000000 037A5052 00017810 0A940000
+// CHECK-NEXT:       05D0: 00000000 017A5052 00017810 0A940000
 // CHECK-NEXT:       05E0: 00000000 00001B0C 07089001 10000000
 // CHECK-NEXT:       05F0: 24000000 00000000 01000000 00000000
-// CHECK-NEXT:       0600: 18000000 00000000 037A5052 00017810
+// CHECK-NEXT:       0600: 18000000 00000000 017A5052 00017810
 // CHECK-NEXT:       0610: 049A0000 1B0C0708 90010000 10000000
 // CHECK-NEXT:       0620: 20000000 00000000 01000000 00000000
-// CHECK-NEXT:       0630: 18000000 00000000 037A5052 00017810
+// CHECK-NEXT:       0630: 18000000 00000000 017A5052 00017810
 // CHECK-NEXT:       0640: 069B0000 00001B0C 07089001 10000000
 // CHECK-NEXT:       0650: 20000000 00000000 01000000 00000000
-// CHECK-NEXT:       0660: 1C000000 00000000 037A5052 00017810
+// CHECK-NEXT:       0660: 1C000000 00000000 017A5052 00017810
 // CHECK-NEXT:       0670: 0A9C0000 00000000 00001B0C 07089001
 // CHECK-NEXT:       0680: 10000000 24000000 00000000 01000000
-// CHECK-NEXT:       0690: 00000000 1C000000 00000000 037A5052
+// CHECK-NEXT:       0690: 00000000 1C000000 00000000 017A5052
 // CHECK-NEXT:       06A0: 00017810 0A980000 00000000 00001B0C
 // CHECK-NEXT:       06B0: 07089001 10000000 24000000 00000000
 // CHECK-NEXT:       06C0: 01000000 00000000 10000000 00000000
-// CHECK-NEXT:       06D0: 037A5200 01781001 1B000000 10000000
+// CHECK-NEXT:       06D0: 017A5200 01781001 1B000000 10000000
 // CHECK-NEXT:       06E0: 18000000 00000000 01000000 00000000
 // CHECK-NEXT:     )
 // CHECK-NEXT:   }
 
 // CHECK:        Section {
-// CHECK:          Index: 5
-// CHECK-NEXT:     Name: .rela.eh_frame
+// CHECK:          Index:
+// CHECK:          Name: .rela.eh_frame
 // CHECK-NEXT:     Type: SHT_RELA
 // CHECK-NEXT:     Flags [
 // CHECK-NEXT:     ]
 // CHECK-NEXT:     Address: 0x0
 // CHECK-NEXT:     Offset:
 // CHECK-NEXT:     Size: 1752
-// CHECK-NEXT:     Link: 7
-// CHECK-NEXT:     Info: 4
+// CHECK-NEXT:     Link:
+// CHECK-NEXT:     Info:
 // CHECK-NEXT:     AddressAlignment: 8
 // CHECK-NEXT:     EntrySize: 24
 // CHECK-NEXT:     Relocations [
diff --git a/test/MC/ELF/comdat-dup-group-name.s b/test/MC/ELF/comdat-dup-group-name.s
index 1181e2eab02..e52f3dcc04d 100644
--- a/test/MC/ELF/comdat-dup-group-name.s
+++ b/test/MC/ELF/comdat-dup-group-name.s
@@ -2,36 +2,27 @@
 
 // Test that we produce two foo sections, each in separate groups
 
-// CHECK: Index: 1
-// CHECK-NEXT: Name: .group
-
-// CHECK: Index: 2
+// CHECK: Index: 5
 // CHECK-NEXT: Name: .group
 
 // CHECK: Index: 6
 // CHECK-NEXT: Name: .foo
 
 // CHECK: Index: 7
+// CHECK-NEXT: Name: .group
+
+// CHECK: Index: 8
 // CHECK-NEXT: Name: .foo
 
 // CHECK: Symbols [
 
 // CHECK: Name: f1
 // CHECK-NOT: }
-// CHECK: Section: .group (0x1)
+// CHECK: Section: .group (0x5)
 
 // CHECK: Name: f2
 // CHECK-NOT: }
-// CHECK: Section: .group (0x2)
-
-// CHECK: Name: .foo
-// CHECK-NOT: }
-// CHECK: Section: .foo (0x6)
-
-// CHECK: Name: .foo
-// CHECK-NOT: }
-// CHECK: Section: .foo (0x7)
-
+// CHECK: Section: .group (0x7)
 
 	.section	.foo,"axG",@progbits,f1,comdat
         nop
diff --git a/test/MC/ELF/comdat-reloc.s b/test/MC/ELF/comdat-reloc.s
index d893a7b149e..1ea3d1e57ef 100644
--- a/test/MC/ELF/comdat-reloc.s
+++ b/test/MC/ELF/comdat-reloc.s
@@ -16,14 +16,14 @@ world:
 // CHECK:  Name: .group
 // CHECK-NOT: SectionData
 // CHECK: SectionData
-// CHECK-NEXT: 0000: 01000000 06000000 07000000
+// CHECK-NEXT: 0000: 01000000 07000000 08000000
 
-// CHECK: Index: 6
+// CHECK: Index: 7
 // CHECK-NEXT: Name: .text.world
 // CHECK-NOT: Section {
 // CHECK: SHF_GROUP
 
-// CHECK: Index: 7
+// CHECK: Index: 8
 // CHECK-NEXT: Name: .rela.text.world
 // CHECK-NOT: Section {
 // CHECK: SHF_GROUP
diff --git a/test/MC/ELF/comdat.s b/test/MC/ELF/comdat.s
index 47966755220..18da17e6118 100644
--- a/test/MC/ELF/comdat.s
+++ b/test/MC/ELF/comdat.s
@@ -1,49 +1,57 @@
-// RUN: llvm-mc -filetype=obj -triple x86_64-pc-linux-gnu %s -o - | llvm-readobj -s -t | FileCheck %s
+// RUN: llvm-mc -filetype=obj -triple x86_64-pc-linux-gnu %s -o - | llvm-readobj -s -t -sd | FileCheck %s
 
-// Test that we produce the group sections and that they are at the beginning
-// of the file.
+// Test that we produce the group sections and that they are before the members
 
 // CHECK:        Section {
-// CHECK:          Index: 1
+// CHECK:          Index: 5
 // CHECK-NEXT:     Name: .group
 // CHECK-NEXT:     Type: SHT_GROUP
 // CHECK-NEXT:     Flags [
 // CHECK-NEXT:     ]
 // CHECK-NEXT:     Address: 0x0
-// CHECK-NEXT:     Offset: 0x40
+// CHECK-NEXT:     Offset:
 // CHECK-NEXT:     Size: 12
-// CHECK-NEXT:     Link: 13
+// CHECK-NEXT:     Link:
 // CHECK-NEXT:     Info: 1
 // CHECK-NEXT:     AddressAlignment: 4
 // CHECK-NEXT:     EntrySize: 4
+// CHECK-NEXT:     SectionData (
+// CHECK-NEXT:       0000:    01000000 06000000 07000000
+// CHECK-NEXT:     )
 // CHECK-NEXT:   }
-// CHECK-NEXT:   Section {
-// CHECK-NEXT:     Index: 2
+// CHECK:        Section {
+// CHECK:          Index: 8
 // CHECK-NEXT:     Name: .group
 // CHECK-NEXT:     Type: SHT_GROUP
 // CHECK-NEXT:     Flags [
 // CHECK-NEXT:     ]
 // CHECK-NEXT:     Address: 0x0
-// CHECK-NEXT:     Offset: 0x4C
+// CHECK-NEXT:     Offset:
 // CHECK-NEXT:     Size: 8
-// CHECK-NEXT:     Link: 13
+// CHECK-NEXT:     Link:
 // CHECK-NEXT:     Info: 2
 // CHECK-NEXT:     AddressAlignment: 4
 // CHECK-NEXT:     EntrySize: 4
+// CHECK-NEXT:     SectionData (
+// CHECK-NEXT:       0000:    01000000 09000000
+// CHECK-NEXT:     )
 // CHECK-NEXT:   }
-// CHECK-NEXT:   Section {
-// CHECK-NEXT:     Index: 3
+// CHECK:        Section {
+// CHECK:          Index: 10
 // CHECK-NEXT:     Name: .group
 // CHECK-NEXT:     Type: SHT_GROUP
 // CHECK-NEXT:     Flags [
 // CHECK-NEXT:     ]
 // CHECK-NEXT:     Address: 0x0
-// CHECK-NEXT:     Offset: 0x54
+// CHECK-NEXT:     Offset:
 // CHECK-NEXT:     Size: 12
-// CHECK-NEXT:     Link: 13
-// CHECK-NEXT:     Info: 10
+// CHECK-NEXT:     Link:
+// CHECK-NEXT:     Info: 3
 // CHECK-NEXT:     AddressAlignment: 4
 // CHECK-NEXT:     EntrySize: 4
+// CHECK-NEXT:     SectionData (
+// CHECK-NEXT:       0000:    01000000 0B000000 0C000000
+// CHECK-NEXT:     )
 // CHECK-NEXT:   }
 
 // Test that g1 and g2 are local, but g3 is an undefined global.
@@ -55,7 +63,7 @@
 // CHECK-NEXT:     Binding: Local
 // CHECK-NEXT:     Type: None
 // CHECK-NEXT:     Other: 0
-// CHECK-NEXT:     Section: .foo (0x7)
+// CHECK-NEXT:     Section: .foo
 // CHECK-NEXT:   }
 // CHECK-NEXT:   Symbol {
 // CHECK-NEXT:     Name: g2
@@ -64,7 +72,7 @@
 // CHECK-NEXT:     Binding: Local
 // CHECK-NEXT:     Type: None
 // CHECK-NEXT:     Other: 0
-// CHECK-NEXT:     Section: .group (0x2)
+// CHECK-NEXT:     Section: .group (0x8)
 // CHECK-NEXT:   }
 
 // CHECK:        Symbol {
diff --git a/test/MC/ELF/common-error3.s b/test/MC/ELF/common-error3.s
new file mode 100644
index 00000000000..a84779e653e
--- /dev/null
+++ b/test/MC/ELF/common-error3.s
@@ -0,0 +1,5 @@
+# RUN: not llvm-mc -filetype=obj -triple x86_64-pc-linux %s 2>&1 | FileCheck %s
+
+# CHECK: Symbol: C redeclared as different type
+        .comm   C,4,4
+        .comm   C,8,4
\ No newline at end of file
diff --git a/test/MC/ELF/common-redeclare.s b/test/MC/ELF/common-redeclare.s
new file mode 100644
index 00000000000..f8ee17d84e2
--- /dev/null
+++ b/test/MC/ELF/common-redeclare.s
@@ -0,0 +1,5 @@
+# RUN: llvm-mc -filetype=obj -triple x86_64-pc-linux %s | llvm-objdump -t - | FileCheck %s
+
+# CHECK: 0000000000000004 g       *COM*  00000004 C
+        .comm   C,4,4
+        .comm   C,4,4
\ No newline at end of file
diff --git a/test/MC/ELF/common2.s b/test/MC/ELF/common2.s
index 526ebc2a95f..26c32a7c840 100644
--- a/test/MC/ELF/common2.s
+++ b/test/MC/ELF/common2.s
@@ -10,7 +10,7 @@
 	.align	8
 
 // CHECK:        Section {
-// CHECK:          Name: .bss (7)
+// CHECK:          Name: .bss
 // CHECK-NEXT:     Type:
 // CHECK-NEXT:     Flags [
 // CHECK:          ]
diff --git a/test/MC/ELF/debug-loc.s b/test/MC/ELF/debug-loc.s
index b24fa169deb..ea8eb3ec70c 100644
--- a/test/MC/ELF/debug-loc.s
+++ b/test/MC/ELF/debug-loc.s
@@ -8,8 +8,8 @@
 // FIXME2: We need a debug_line dumper so that we can test the actual contents.
 
 // CHECK:        Section {
-// CHECK:          Index: 4
-// CHECK-NEXT:     Name: .debug_line
+// CHECK:          Index:
+// CHECK:          Name: .debug_line
 // CHECK-NEXT:     Type: SHT_PROGBITS
 // CHECK-NEXT:     Flags [
 // CHECK-NEXT:     ]
diff --git a/test/MC/ELF/discriminator.s b/test/MC/ELF/discriminator.s
index 8a695b96ede..75e4e86cee0 100644
--- a/test/MC/ELF/discriminator.s
+++ b/test/MC/ELF/discriminator.s
@@ -19,12 +19,12 @@ foo:
         .long   .L.debug_abbrev_begin   # Offset Into Abbrev. Section
         .byte   8                       # Address Size (in bytes)
         .byte   1                       # Abbrev [1] 0xb:0x1b DW_TAG_compile_unit
-        .long   .Linfo_string0          # DW_AT_producer
+        .long   info_string0            # DW_AT_producer
         .short  12                      # DW_AT_language
-        .long   .Linfo_string1          # DW_AT_name
+        .long   info_string1            # DW_AT_name
         .quad   0                       # DW_AT_low_pc
         .long   0                       # DW_AT_stmt_list
-        .long   .Linfo_string2          # DW_AT_comp_dir
+        .long   info_string2            # DW_AT_comp_dir
                                         # DW_AT_APPLE_optimized
         .section        .debug_abbrev,"",@progbits
 .L.debug_abbrev_begin:
diff --git a/test/MC/ELF/empty-dwarf-lines.s b/test/MC/ELF/empty-dwarf-lines.s
index 241580bb3b8..19305709cda 100644
--- a/test/MC/ELF/empty-dwarf-lines.s
+++ b/test/MC/ELF/empty-dwarf-lines.s
@@ -8,8 +8,8 @@ c:
         .asciz   "hi\n"
 
 // CHECK:        Section {
-// CHECK:          Index: 4
-// CHECK-NEXT:     Name: .debug_line
+// CHECK:          Index:
+// CHECK:          Name: .debug_line
 // CHECK-NEXT:     Type: SHT_PROGBITS
 // CHECK-NEXT:     Flags [
 // CHECK-NEXT:     ]
diff --git a/test/MC/ELF/empty.s b/test/MC/ELF/empty.s
index c421fe84488..7b686fef3a2 100644
--- a/test/MC/ELF/empty.s
+++ b/test/MC/ELF/empty.s
@@ -1,8 +1,31 @@
 // RUN: llvm-mc -filetype=obj -triple x86_64-pc-linux-gnu %s -o - | llvm-readobj -s | FileCheck %s
+// RUN: llvm-mc -filetype=obj -triple x86_64-apple-darwin14.0.0-elf %s -o - | llvm-readobj -s | FileCheck %s -check-prefix=DARWIN
+// RUN: llvm-mc -filetype=obj -triple x86_64-pc-win32-elf %s -o - | llvm-readobj -s | FileCheck %s -check-prefix=WINDOWS
+
+// Check that we can create ELF files for darwin/windows, even though
+// it is not the default file format.
+
+// DARWIN:       Format: ELF64-x86-64
+// WINDOWS:      Format: ELF64-x86-64
+// DARWIN-NEXT:  Arch: x86_64
+// WINDOWS-NEXT: Arch: x86_64
 
 // Test that like gnu as we create text, data and bss by default. Also test
-// that shstrtab, symtab and strtab are listed in that order.
+// that symtab and strtab are listed.
 
+// CHECK:        Section {
+// CHECK:          Name: .strtab
+// CHECK-NEXT:     Type: SHT_STRTAB
+// CHECK-NEXT:     Flags [
+// CHECK-NEXT:     ]
+// CHECK-NEXT:     Address: 0x0
+// CHECK-NEXT:     Offset:
+// CHECK-NEXT:     Size: 34
+// CHECK-NEXT:     Link: 0
+// CHECK-NEXT:     Info: 0
+// CHECK-NEXT:     AddressAlignment: 1
+// CHECK-NEXT:     EntrySize: 0
+// CHECK-NEXT:   }
 // CHECK:        Section {
 // CHECK:          Name: .text
 // CHECK-NEXT:     Type: SHT_PROGBITS
@@ -49,41 +72,15 @@
 // CHECK-NEXT:     EntrySize: 0
 // CHECK-NEXT:   }
 // CHECK:        Section {
-// CHECK:          Name: .shstrtab
-// CHECK-NEXT:     Type: SHT_STRTAB
-// CHECK-NEXT:     Flags [
-// CHECK-NEXT:     ]
-// CHECK-NEXT:     Address: 0x0
-// CHECK-NEXT:     Offset: 0x40
-// CHECK-NEXT:     Size: 44
-// CHECK-NEXT:     Link: 0
-// CHECK-NEXT:     Info: 0
-// CHECK-NEXT:     AddressAlignment: 1
-// CHECK-NEXT:     EntrySize: 0
-// CHECK-NEXT:   }
-// CHECK:        Section {
 // CHECK:          Name: .symtab
 // CHECK-NEXT:     Type: SHT_SYMTAB
 // CHECK-NEXT:     Flags [
 // CHECK-NEXT:     ]
 // CHECK-NEXT:     Address: 0x0
 // CHECK-NEXT:     Offset:
-// CHECK-NEXT:     Size: 96
-// CHECK-NEXT:     Link: 6
-// CHECK-NEXT:     Info: 4
+// CHECK-NEXT:     Size: 24
+// CHECK-NEXT:     Link:
+// CHECK-NEXT:     Info: 1
 // CHECK-NEXT:     AddressAlignment: 8
 // CHECK-NEXT:     EntrySize: 24
 // CHECK-NEXT:   }
-// CHECK:        Section {
-// CHECK:          Name: .strtab
-// CHECK-NEXT:     Type: SHT_STRTAB
-// CHECK-NEXT:     Flags [
-// CHECK-NEXT:     ]
-// CHECK-NEXT:     Address: 0x0
-// CHECK-NEXT:     Offset:
-// CHECK-NEXT:     Size: 1
-// CHECK-NEXT:     Link: 0
-// CHECK-NEXT:     Info: 0
-// CHECK-NEXT:     AddressAlignment: 1
-// CHECK-NEXT:     EntrySize: 0
-// CHECK-NEXT:   }
diff --git a/test/MC/ELF/entsize.s b/test/MC/ELF/entsize.s
index 8e084e24b25..1dc6103189d 100644
--- a/test/MC/ELF/entsize.s
+++ b/test/MC/ELF/entsize.s
@@ -33,8 +33,8 @@
     .quad 42
 
 // CHECK:        Section {
-// CHECK:          Index: 4
-// CHECK-NEXT:     Name: .rodata.str1.1
+// CHECK:          Index:
+// CHECK:          Name: .rodata.str1.1
 // CHECK-NEXT:     Type: SHT_PROGBITS
 // CHECK-NEXT:     Flags [
 // CHECK-NEXT:       SHF_ALLOC
@@ -50,8 +50,8 @@
 // CHECK-NEXT:     EntrySize: 1
 // CHECK-NEXT:   }
 // CHECK:        Section {
-// CHECK:          Index: 5
-// CHECK-NEXT:     Name: .rodata.str2.1
+// CHECK:          Index:
+// CHECK:          Name: .rodata.str2.1
 // CHECK-NEXT:     Type: SHT_PROGBITS
 // CHECK-NEXT:     Flags [
 // CHECK-NEXT:       SHF_ALLOC
@@ -67,8 +67,8 @@
 // CHECK-NEXT:     EntrySize: 2
 // CHECK-NEXT:   }
 // CHECK:        Section {
-// CHECK:          Index: 6
-// CHECK-NEXT:     Name: .rodata.cst8
+// CHECK:          Index:
+// CHECK:          Name: .rodata.cst8
 // CHECK-NEXT:     Type: SHT_PROGBITS
 // CHECK-NEXT:     Flags [
 // CHECK-NEXT:       SHF_ALLOC
diff --git a/test/MC/ELF/file-double.s b/test/MC/ELF/file-double.s
index b5da8c5a858..8ea5fc759b3 100644
--- a/test/MC/ELF/file-double.s
+++ b/test/MC/ELF/file-double.s
@@ -34,7 +34,7 @@ bar.c:
 // CHECK-NEXT:     Binding: Global
 // CHECK-NEXT:     Type: None
 // CHECK-NEXT:     Other: 0
-// CHECK-NEXT:     Section: .text (0x1)
+// CHECK-NEXT:     Section: .text
 // CHECK-NEXT:   }
 // CHECK:        Symbol {
 // CHECK:        Name: foo.c
@@ -43,5 +43,5 @@ bar.c:
 // CHECK-NEXT:     Binding: Global
 // CHECK-NEXT:     Type: None
 // CHECK-NEXT:     Other: 0
-// CHECK-NEXT:     Section: .text (0x1)
+// CHECK-NEXT:     Section: .text
 // CHECK-NEXT:   }
diff --git a/test/MC/ELF/file.s b/test/MC/ELF/file.s
index 6b6cb44e2b2..b19bda7c42c 100644
--- a/test/MC/ELF/file.s
+++ b/test/MC/ELF/file.s
@@ -6,7 +6,7 @@
 foa:
 
 // CHECK:        Symbol {
-// CHECK:          Name: foo (1)
+// CHECK:          Name: foo
 // CHECK-NEXT:     Value: 0x0
 // CHECK-NEXT:     Size: 0
 // CHECK-NEXT:     Binding: Local
@@ -15,11 +15,11 @@ foa:
 // CHECK-NEXT:     Section: Absolute (0xFFF1)
 // CHECK-NEXT:   }
 // CHECK-NEXT:   Symbol {
-// CHECK-NEXT:     Name: foa (5)
+// CHECK-NEXT:     Name: foa
 // CHECK-NEXT:     Value: 0x0
 // CHECK-NEXT:     Size: 0
 // CHECK-NEXT:     Binding: Local
 // CHECK-NEXT:     Type: None
 // CHECK-NEXT:     Other: 0
-// CHECK-NEXT:     Section: .text (0x1)
+// CHECK-NEXT:     Section: .text
 // CHECK-NEXT:   }
diff --git a/test/MC/ELF/got.s b/test/MC/ELF/got.s
index 30114b75d84..3fe3e4a62b1 100644
--- a/test/MC/ELF/got.s
+++ b/test/MC/ELF/got.s
@@ -1,7 +1,7 @@
 // RUN: llvm-mc -filetype=obj -triple x86_64-pc-linux-gnu %s -o - | llvm-readobj -r -t | FileCheck %s
 
-// Test that this produces a R_X86_64_GOT32 and that we have an undefined
-// reference to _GLOBAL_OFFSET_TABLE_.
+// Test that this produces the correct relocations R_X86_64_GOT32 and that we,
+// unlike gas, don't create a _GLOBAL_OFFSET_TABLE_ symbol as a side effect.
 
         movl	foo@GOT, %eax
         movl	foo@GOTPCREL(%rip), %eax
@@ -13,8 +13,5 @@
 // CHECK-NEXT:   }
 // CHECK-NEXT: ]
 
-// CHECK:        Symbol {
-// CHECK:          Name: _GLOBAL_OFFSET_TABLE_
-// CHECK-NEXT:     Value:
-// CHECK-NEXT:     Size:
-// CHECK-NEXT:     Binding: Global
+// CHECK:        Symbols [
+// CHECK-NOT:          _GLOBAL_OFFSET_TABLE_
diff --git a/test/MC/ELF/lcomm.s b/test/MC/ELF/lcomm.s
index 7d8ac3fcafa..94bd5ea1997 100644
--- a/test/MC/ELF/lcomm.s
+++ b/test/MC/ELF/lcomm.s
@@ -10,7 +10,7 @@
 // CHECK-NEXT:     Binding: Local
 // CHECK-NEXT:     Type: Object
 // CHECK-NEXT:     Other: 0
-// CHECK-NEXT:     Section: .bss (0x3)
+// CHECK-NEXT:     Section: .bss
 // CHECK-NEXT:   }
 // CHECK:        Symbol {
 // CHECK:          Name: B
@@ -19,5 +19,5 @@
 // CHECK-NEXT:     Binding: Local
 // CHECK-NEXT:     Type: Object
 // CHECK-NEXT:     Other: 0
-// CHECK-NEXT:     Section: .bss (0x3)
+// CHECK-NEXT:     Section: .bss
 // CHECK-NEXT:   }
diff --git a/test/MC/ELF/many-sections-2.s b/test/MC/ELF/many-sections-2.s
index 88a4822c386..0077552ef31 100644
--- a/test/MC/ELF/many-sections-2.s
+++ b/test/MC/ELF/many-sections-2.s
@@ -18,7 +18,7 @@
 // SYMBOLS-NEXT:    Binding: Local (0x0)
 // SYMBOLS-NEXT:    Type: None (0x0)
 // SYMBOLS-NEXT:    Other: 0
-// SYMBOLS-NEXT:    Section: last (0xFF00)
+// SYMBOLS-NEXT:    Section: dm (0xFF00)
 // SYMBOLS-NEXT:  }
 // SYMBOLS-NEXT:  Symbol {
 // SYMBOLS-NEXT:    Name: b
@@ -27,18 +27,18 @@
 // SYMBOLS-NEXT:    Binding: Local (0x0)
 // SYMBOLS-NEXT:    Type: None (0x0)
 // SYMBOLS-NEXT:    Other: 0
-// SYMBOLS-NEXT:    Section: last (0xFF00)
+// SYMBOLS-NEXT:    Section: dm (0xFF00)
 // SYMBOLS-NEXT:  }
 
 
 // Test that this file has one section too many.
-// SYMBOLS:         Name: last
-// SYMBOLS-NEXT:    Value: 0x0
-// SYMBOLS-NEXT:    Size: 0
-// SYMBOLS-NEXT:    Binding: Local (0x0)
-// SYMBOLS-NEXT:    Type: Section (0x3)
-// SYMBOLS-NEXT:    Other: 0
-// SYMBOLS-NEXT:    Section: last (0xFF00)
+// SYMBOLS:         Name: (0)
+// SYMBOLS:         Value: 0x0
+// SYMBOLS:         Size: 0
+// SYMBOLS:         Binding: Local (0x0)
+// SYMBOLS:         Type: Section (0x3)
+// SYMBOLS:         Other: 0
+// SYMBOLS:         Section: dm (0xFF00)
 // SYMBOLS-NEXT:  }
 // SYMBOLS-NEXT:]
 
@@ -128,6 +128,6 @@ gen_sections16 k
 gen_sections8 l
 gen_sections4 m
 
-.section last
 a:
 b = a + 1
+.long dm
diff --git a/test/MC/ELF/many-sections-3.s b/test/MC/ELF/many-sections-3.s
new file mode 100644
index 00000000000..02d30a60523
--- /dev/null
+++ b/test/MC/ELF/many-sections-3.s
@@ -0,0 +1,107 @@
+// RUN: llvm-mc -filetype=obj -triple x86_64-pc-linux-gnu %s -o %t
+// RUN: llvm-readobj -t %t | FileCheck --check-prefix=SYMBOLS %s
+// RUN: llvm-nm %t | FileCheck --check-prefix=NM %s
+
+// Test that symbol a has a section that could be confused with common (0xFFF2)
+// SYMBOLS:         Name: a
+// SYMBOLS-NEXT:    Value: 0x0
+// SYMBOLS-NEXT:    Size: 0
+// SYMBOLS-NEXT:    Binding: Local (0x0)
+// SYMBOLS-NEXT:    Type: None (0x0)
+// SYMBOLS-NEXT:    Other: 0
+// SYMBOLS-NEXT:    Section: bar (0xFFF2)
+// SYMBOLS-NEXT:  }
+
+// Test that we don't get confused
+// NM: 0000000000000000 r a
+
+.macro gen_sections4 x
+        .section a\x
+        .section b\x
+        .section c\x
+        .section d\x
+.endm
+
+.macro gen_sections8 x
+        gen_sections4 a\x
+        gen_sections4 b\x
+.endm
+
+.macro gen_sections16 x
+        gen_sections8 a\x
+        gen_sections8 b\x
+.endm
+
+.macro gen_sections32 x
+        gen_sections16 a\x
+        gen_sections16 b\x
+.endm
+
+.macro gen_sections64 x
+        gen_sections32 a\x
+        gen_sections32 b\x
+.endm
+
+.macro gen_sections128 x
+        gen_sections64 a\x
+        gen_sections64 b\x
+.endm
+
+.macro gen_sections256 x
+        gen_sections128 a\x
+        gen_sections128 b\x
+.endm
+
+.macro gen_sections512 x
+        gen_sections256 a\x
+        gen_sections256 b\x
+.endm
+
+.macro gen_sections1024 x
+        gen_sections512 a\x
+        gen_sections512 b\x
+.endm
+
+.macro gen_sections2048 x
+        gen_sections1024 a\x
+        gen_sections1024 b\x
+.endm
+
+.macro gen_sections4096 x
+        gen_sections2048 a\x
+        gen_sections2048 b\x
+.endm
+
+.macro gen_sections8192 x
+        gen_sections4096 a\x
+        gen_sections4096 b\x
+.endm
+
+.macro gen_sections16384 x
+        gen_sections8192 a\x
+        gen_sections8192 b\x
+.endm
+
+.macro gen_sections32768 x
+        gen_sections16384 a\x
+        gen_sections16384 b\x
+.endm
+
+gen_sections32768 a
+gen_sections16384 b
+gen_sections8192 c
+gen_sections4096 d
+gen_sections2048 e
+gen_sections1024 f
+gen_sections512 g
+gen_sections256 h
+gen_sections128 i
+gen_sections64 j
+gen_sections32 k
+gen_sections8 l
+gen_sections4 m
+
+        .section foo
+        .section bar, "a"
+
+a:
diff --git a/test/MC/ELF/many-sections.s b/test/MC/ELF/many-sections.s
index 93ea8e7821e..2db6abb9321 100644
--- a/test/MC/ELF/many-sections.s
+++ b/test/MC/ELF/many-sections.s
@@ -9,13 +9,13 @@
 
 // Check the last referenced section.
 
-// SYMBOLS:         Name: dm (0)
-// SYMBOLS-NEXT:    Value: 0x0
-// SYMBOLS-NEXT:    Size: 0
-// SYMBOLS-NEXT:    Binding: Local (0x0)
-// SYMBOLS-NEXT:    Type: Section (0x3)
-// SYMBOLS-NEXT:    Other: 0
-// SYMBOLS-NEXT:    Section: dm (0xFEFF)
+// SYMBOLS:         Name: (0)
+// SYMBOLS:         Value: 0x0
+// SYMBOLS:         Size: 0
+// SYMBOLS:         Binding: Local (0x0)
+// SYMBOLS:         Type: Section (0x3)
+// SYMBOLS:         Other: 0
+// SYMBOLS:         Section: zed (0xFEFF)
 // SYMBOLS-NEXT:  }
 // SYMBOLS-NEXT:]
 
@@ -103,4 +103,7 @@ gen_sections64 i
 gen_sections32 j
 gen_sections16 k
 gen_sections8 l
-gen_sections4 m
+        .section foo
+        .section bar
+        .section zed
+.long zed
diff --git a/test/MC/ELF/n_bytes.s b/test/MC/ELF/n_bytes.s
index e658de09a2f..fcbd0e88c2c 100644
--- a/test/MC/ELF/n_bytes.s
+++ b/test/MC/ELF/n_bytes.s
@@ -6,8 +6,8 @@
         .int 42, 1, 2, 3
 
 // CHECK:        Section {
-// CHECK:          Index: 1
-// CHECK-NEXT:     Name: .text
+// CHECK:          Index:
+// CHECK:          Name: .text
 // CHECK-NEXT:     Type: SHT_PROGBITS
 // CHECK-NEXT:     Flags [
 // CHECK-NEXT:       SHF_ALLOC
diff --git a/test/MC/ELF/noexec.s b/test/MC/ELF/noexec.s
index 28f50cb7f69..e8f460e65fa 100644
--- a/test/MC/ELF/noexec.s
+++ b/test/MC/ELF/noexec.s
@@ -1,8 +1,8 @@
-// RUN: llvm-mc -no-exec-stack -filetype=obj -triple x86_64-pc-linux-gnu %s -o - | llvm-readobj -s -t | FileCheck  %s
+// RUN: llvm-mc -no-exec-stack -filetype=obj -triple x86_64-pc-linux-gnu %s -o - | llvm-readobj -s | FileCheck  %s
 
 // CHECK:        Section {
-// CHECK:          Index: 4
-// CHECK-NEXT:     Name: .note.GNU-stack
+// CHECK:          Index:
+// CHECK:          Name: .note.GNU-stack
 // CHECK-NEXT:     Type: SHT_PROGBITS
 // CHECK-NEXT:     Flags [
 // CHECK-NEXT:     ]
@@ -14,13 +14,3 @@
 // CHECK-NEXT:     AddressAlignment: 1
 // CHECK-NEXT:     EntrySize: 0
 // CHECK-NEXT:   }
-
-// CHECK:        Symbol {
-// CHECK:          Name: .note.GNU-stack (0)
-// CHECK-NEXT:     Value: 0x0
-// CHECK-NEXT:     Size: 0
-// CHECK-NEXT:     Binding: Local
-// CHECK-NEXT:     Type: Section
-// CHECK-NEXT:     Other: 0
-// CHECK-NEXT:     Section: .note.GNU-stack (0x4)
-// CHECK-NEXT:   }
diff --git a/test/MC/ELF/norelocation.s b/test/MC/ELF/norelocation.s
index 13703829280..054fc5f95cb 100644
--- a/test/MC/ELF/norelocation.s
+++ b/test/MC/ELF/norelocation.s
@@ -1,26 +1,7 @@
-// RUN: llvm-mc -filetype=obj -triple x86_64-pc-linux-gnu %s -o - | llvm-readobj -s -sd -sr | FileCheck  %s
+// RUN: llvm-mc -filetype=obj -triple x86_64-pc-linux-gnu %s -o - | llvm-readobj -r | FileCheck  %s
 
         call bar
 bar:
 
-// CHECK:        Section {
-// CHECK:          Name: .text
-// CHECK-NEXT:     Type: SHT_PROGBITS
-// CHECK-NEXT:     Flags [ (0x6)
-// CHECK-NEXT:       SHF_ALLOC
-// CHECK-NEXT:       SHF_EXECINSTR
-// CHECK-NEXT:     ]
-// CHECK-NEXT:     Address: 0x0
-// CHECK-NEXT:     Offset: 0x40
-// CHECK-NEXT:     Size: 5
-// CHECK-NEXT:     Link: 0
-// CHECK-NEXT:     Info: 0
-// CHECK-NEXT:     AddressAlignment: 4
-// CHECK-NEXT:     EntrySize: 0
-// CHECK-NEXT:     Relocations [
-// CHECK-NEXT:     ]
-// CHECK-NEXT:     SectionData (
-// CHECK-NEXT:       0000: E8000000 00
-// CHECK-NEXT:     )
-// CHECK-NEXT:   }
-// CHECK: shstrtab
+// CHECK:      Relocations [
+// CHECK-NEXT: ]
diff --git a/test/MC/ELF/popsection.s b/test/MC/ELF/popsection.s
new file mode 100644
index 00000000000..19f55688a1b
--- /dev/null
+++ b/test/MC/ELF/popsection.s
@@ -0,0 +1,21 @@
+// RUN: llvm-mc -filetype=obj %s -o - -triple x86_64-pc-linux | llvm-readobj -s - | FileCheck %s
+
+// This used to crash. Test that it create an empty section instead.
+
+        .pushsection foo
+        .popsection
+
+// CHECK:       Section {
+// CHECK:         Index: 5
+// CHECK-NEXT:    Name: foo
+// CHECK-NEXT:    Type: SHT_PROGBITS
+// CHECK-NEXT:    Flags [ (0x0)
+// CHECK-NEXT:    ]
+// CHECK-NEXT:    Address: 0x0
+// CHECK-NEXT:    Offset:
+// CHECK-NEXT:    Size: 0
+// CHECK-NEXT:    Link: 0
+// CHECK-NEXT:    Info: 0
+// CHECK-NEXT:    AddressAlignment: 1
+// CHECK-NEXT:    EntrySize: 0
+// CHECK-NEXT:  }
diff --git a/test/MC/ELF/pr19430.s b/test/MC/ELF/pr19430.s
index a1e524662a9..3ba6d729b4b 100644
--- a/test/MC/ELF/pr19430.s
+++ b/test/MC/ELF/pr19430.s
@@ -8,7 +8,7 @@
  .cfi_endproc
 
 // CHECK:      Relocations [
-// CHECK-NEXT:   Section (5) .rela.eh_frame {
+// CHECK-NEXT:   Section ({{.*}}) .rela.eh_frame {
 // CHECK-NEXT:     0x20 R_X86_64_PC32 .text 0x3E8
 // CHECK-NEXT:   }
 // CHECK-NEXT: ]
diff --git a/test/MC/ELF/relax-arith.s b/test/MC/ELF/relax-arith.s
index b8145564db0..15e44ebff7e 100644
--- a/test/MC/ELF/relax-arith.s
+++ b/test/MC/ELF/relax-arith.s
@@ -1,17 +1,18 @@
-// RUN: llvm-mc -filetype=obj -triple x86_64-pc-linux-gnu %s -o - | llvm-readobj -s -sd | FileCheck  %s
+// RUN: llvm-mc -filetype=obj -triple x86_64-pc-linux-gnu %s -o - | llvm-objdump -d - | FileCheck  %s
 
 // Test that we correctly relax these instructions into versions that use
 // 16 or 32 bit immediate values.
 
 bar:
-// CHECK:      Name: imul
-// CHECK:      SectionData (
-// CHECK-NEXT:   0000: 6669DB00 0066691C 25000000 00000069
-// CHECK-NEXT:   0010: DB000000 00691C25 00000000 00000000
-// CHECK-NEXT:   0020: 4869DB00 00000048 691C2500 00000000
-// CHECK-NEXT:   0030: 000000
-// CHECK-NEXT: )
-        .section imul
+// CHECK:      Disassembly of section imul:
+// CHECK-NEXT: imul:
+// CHECK-NEXT:   0: 66 69 db 00 00                       imulw $0, %bx, %bx
+// CHECK-NEXT:   5: 66 69 1c 25 00 00 00 00 00 00        imulw $0, 0, %bx
+// CHECK-NEXT:   f: 69 db 00 00 00 00                    imull $0, %ebx, %ebx
+// CHECK-NEXT:  15: 69 1c 25 00 00 00 00 00 00 00 00     imull $0, 0, %ebx
+// CHECK-NEXT:  20: 48 69 db 00 00 00 00                 imulq $0, %rbx, %rbx
+// CHECK-NEXT:  27: 48 69 1c 25 00 00 00 00 00 00 00 00  imulq $0, 0, %rbx
+        .section imul,"x"
         imul $foo, %bx,  %bx
         imul $foo, bar,  %bx
         imul $foo, %ebx, %ebx
@@ -19,15 +20,15 @@ bar:
         imul $foo, %rbx, %rbx
         imul $foo, bar,  %rbx
 
-
-// CHECK:      Name: and
-// CHECK:      SectionData (
-// CHECK-NEXT:   0000: 6681E300 00668124 25000000 00000081
-// CHECK-NEXT:   0010: E3000000 00812425 00000000 00000000
-// CHECK-NEXT:   0020: 4881E300 00000048 81242500 00000000
-// CHECK-NEXT:   0030: 000000
-// CHECK-NEXT: )
-        .section and
+// CHECK:      Disassembly of section and:
+// CHECK-NEXT: and:
+// CHECK-NEXT:   0: 66 81 e3 00 00                       andw $0, %bx
+// CHECK-NEXT:   5: 66 81 24 25 00 00 00 00 00 00        andw $0, 0
+// CHECK-NEXT:   f: 81 e3 00 00 00 00                    andl $0, %ebx
+// CHECK-NEXT:  15: 81 24 25 00 00 00 00 00 00 00 00     andl $0, 0
+// CHECK-NEXT:  20: 48 81 e3 00 00 00 00                 andq $0, %rbx
+// CHECK-NEXT:  27: 48 81 24 25 00 00 00 00 00 00 00 00  andq $0, 0
+        .section and,"x"
         and  $foo, %bx
         andw $foo, bar
         and  $foo, %ebx
@@ -35,14 +36,15 @@ bar:
         and  $foo, %rbx
         andq $foo, bar
 
-// CHECK:      Name: or
-// CHECK:      SectionData (
-// CHECK-NEXT:   0000: 6681CB00 0066810C 25000000 00000081
-// CHECK-NEXT:   0010: CB000000 00810C25 00000000 00000000
-// CHECK-NEXT:   0020: 4881CB00 00000048 810C2500 00000000
-// CHECK-NEXT:   0030: 000000
-// CHECK-NEXT: )
-        .section or
+// CHECK:      Disassembly of section or:
+// CHECK-NEXT: or:
+// CHECK-NEXT:   0: 66 81 cb 00 00                       orw $0, %bx
+// CHECK-NEXT:   5: 66 81 0c 25 00 00 00 00 00 00        orw $0, 0
+// CHECK-NEXT:   f: 81 cb 00 00 00 00                    orl $0, %ebx
+// CHECK-NEXT:  15: 81 0c 25 00 00 00 00 00 00 00 00     orl $0, 0
+// CHECK-NEXT:  20: 48 81 cb 00 00 00 00                 orq $0, %rbx
+// CHECK-NEXT:  27: 48 81 0c 25 00 00 00 00 00 00 00 00  orq $0, 0
+        .section or,"x"
         or  $foo, %bx
         orw $foo, bar
         or  $foo, %ebx
@@ -50,14 +52,15 @@ bar:
         or  $foo, %rbx
         orq $foo, bar
 
-// CHECK:      Name: xor
-// CHECK:      SectionData (
-// CHECK-NEXT:   0000: 6681F300 00668134 25000000 00000081
-// CHECK-NEXT:   0010: F3000000 00813425 00000000 00000000
-// CHECK-NEXT:   0020: 4881F300 00000048 81342500 00000000
-// CHECK-NEXT:   0030: 000000
-// CHECK-NEXT: )
-        .section xor
+// CHECK:      Disassembly of section xor:
+// CHECK-NEXT: xor:
+// CHECK-NEXT:   0: 66 81 f3 00 00                       xorw $0, %bx
+// CHECK-NEXT:   5: 66 81 34 25 00 00 00 00 00 00        xorw $0, 0
+// CHECK-NEXT:   f: 81 f3 00 00 00 00                    xorl $0, %ebx
+// CHECK-NEXT:  15: 81 34 25 00 00 00 00 00 00 00 00     xorl $0, 0
+// CHECK-NEXT:  20: 48 81 f3 00 00 00 00                 xorq $0, %rbx
+// CHECK-NEXT:  27: 48 81 34 25 00 00 00 00 00 00 00 00  xorq $0, 0
+        .section xor,"x"
         xor  $foo, %bx
         xorw $foo, bar
         xor  $foo, %ebx
@@ -65,14 +68,15 @@ bar:
         xor  $foo, %rbx
         xorq $foo, bar
 
-// CHECK:      Name: add
-// CHECK:      SectionData (
-// CHECK-NEXT:   0000: 6681C300 00668104 25000000 00000081
-// CHECK-NEXT:   0010: C3000000 00810425 00000000 00000000
-// CHECK-NEXT:   0020: 4881C300 00000048 81042500 00000000
-// CHECK-NEXT:   0030: 000000
-// CHECK-NEXT: )
-        .section add
+// CHECK:      Disassembly of section add:
+// CHECK-NEXT: add:
+// CHECK-NEXT:   0: 66 81 c3 00 00                       addw $0, %bx
+// CHECK-NEXT:   5: 66 81 04 25 00 00 00 00 00 00        addw $0, 0
+// CHECK-NEXT:   f: 81 c3 00 00 00 00                    addl $0, %ebx
+// CHECK-NEXT:  15: 81 04 25 00 00 00 00 00 00 00 00     addl $0, 0
+// CHECK-NEXT:  20: 48 81 c3 00 00 00 00                 addq $0, %rbx
+// CHECK-NEXT:  27: 48 81 04 25 00 00 00 00 00 00 00 00  addq $0, 0
+        .section add,"x"
         add  $foo, %bx
         addw $foo, bar
         add  $foo, %ebx
@@ -80,14 +84,15 @@ bar:
         add  $foo, %rbx
         addq $foo, bar
 
-// CHECK:      Name: sub
-// CHECK:      SectionData (
-// CHECK-NEXT:   000: 6681EB00 0066812C 25000000 00000081
-// CHECK-NEXT:   010: EB000000 00812C25 00000000 00000000
-// CHECK-NEXT:   020: 4881EB00 00000048 812C2500 00000000
-// CHECK-NEXT:   030: 000000
-// CHECK-NEXT: )
-        .section sub
+// CHECK:      Disassembly of section sub:
+// CHECK-NEXT: sub:
+// CHECK-NEXT:   0: 66 81 eb 00 00                       subw $0, %bx
+// CHECK-NEXT:   5: 66 81 2c 25 00 00 00 00 00 00        subw $0, 0
+// CHECK-NEXT:   f: 81 eb 00 00 00 00                    subl $0, %ebx
+// CHECK-NEXT:  15: 81 2c 25 00 00 00 00 00 00 00 00     subl $0, 0
+// CHECK-NEXT:  20: 48 81 eb 00 00 00 00                 subq $0, %rbx
+// CHECK-NEXT:  27: 48 81 2c 25 00 00 00 00 00 00 00 00  subq $0, 0
+        .section sub,"x"
         sub  $foo, %bx
         subw $foo, bar
         sub  $foo, %ebx
@@ -95,17 +100,26 @@ bar:
         sub  $foo, %rbx
         subq $foo, bar
 
-// CHECK:      Name: cmp
-// CHECK:      SectionData (
-// CHECK-NEXT:   0000: 6681FB00 0066813C 25000000 00000081
-// CHECK-NEXT:   0010: FB000000 00813C25 00000000 00000000
-// CHECK-NEXT:   0020: 4881FB00 00000048 813C2500 00000000
-// CHECK-NEXT:   0030: 000000
-// CHECK-NEXT: )
-        .section cmp
+// CHECK:      Disassembly of section cmp:
+// CHECK-NEXT: cmp:
+// CHECK-NEXT:   0: 66 81 fb 00 00                       cmpw $0, %bx
+// CHECK-NEXT:   5: 66 81 3c 25 00 00 00 00 00 00        cmpw $0, 0
+// CHECK-NEXT:   f: 81 fb 00 00 00 00                    cmpl $0, %ebx
+// CHECK-NEXT:  15: 81 3c 25 00 00 00 00 00 00 00 00     cmpl $0, 0
+// CHECK-NEXT:  20: 48 81 fb 00 00 00 00                 cmpq $0, %rbx
+// CHECK-NEXT:  27: 48 81 3c 25 00 00 00 00 00 00 00 00  cmpq $0, 0
+        .section cmp,"x"
         cmp  $foo, %bx
         cmpw $foo, bar
         cmp  $foo, %ebx
         cmpl $foo, bar
         cmp  $foo, %rbx
         cmpq $foo, bar
+
+// CHECK:      Disassembly of section push:
+// CHECK-NEXT: push:
+// CHECK-NEXT:   0: 66 68 00 00                          pushw $0
+// CHECK-NEXT:   4: 68 00 00 00 00                       pushq $0
+        .section push,"x"
+        pushw $foo
+        push  $foo
diff --git a/test/MC/ELF/relax-arith2.s b/test/MC/ELF/relax-arith2.s
new file mode 100644
index 00000000000..b05418482e3
--- /dev/null
+++ b/test/MC/ELF/relax-arith2.s
@@ -0,0 +1,130 @@
+// RUN: llvm-mc -filetype=obj -triple x86_64-pc-linux-gnu %s -o - | llvm-objdump -d - | FileCheck  %s
+
+// Test that we avoid relaxing these instructions and instead generate versions
+// that use 8-bit immediate values.
+
+bar:
+// CHECK:      Disassembly of section imul:
+// CHECK-NEXT: imul:
+// CHECK-NEXT:   0: 66 6b db 80                   imulw $-128, %bx, %bx
+// CHECK-NEXT:   4: 66 6b 1c 25 00 00 00 00 7f    imulw $127, 0, %bx
+// CHECK-NEXT:   d: 6b db 00                      imull $0, %ebx, %ebx
+// CHECK-NEXT:  10: 6b 1c 25 00 00 00 00 01       imull $1, 0, %ebx
+// CHECK-NEXT:  18: 48 6b db ff                   imulq $-1, %rbx, %rbx
+// CHECK-NEXT:  1c: 48 6b 1c 25 00 00 00 00 2a    imulq $42, 0, %rbx
+        .section imul,"x"
+        imul $-128, %bx,  %bx
+        imul $127, bar,  %bx
+        imul $0, %ebx, %ebx
+        imul $1, bar,  %ebx
+        imul $-1, %rbx, %rbx
+        imul $42, bar,  %rbx
+
+
+// CHECK:      Disassembly of section and:
+// CHECK-NEXT: and:
+// CHECK-NEXT:   0: 66 83 e3 7f                   andw $127, %bx
+// CHECK-NEXT:   4: 66 83 24 25 00 00 00 00 00    andw $0, 0
+// CHECK-NEXT:   d: 83 e3 01                      andl $1, %ebx
+// CHECK-NEXT:  10: 83 24 25 00 00 00 00 ff       andl $-1, 0
+// CHECK-NEXT:  18: 48 83 e3 2a                   andq $42, %rbx
+// CHECK-NEXT:  1c: 48 83 24 25 00 00 00 00 80    andq $-128, 0
+        .section and,"x"
+        and  $127, %bx
+        andw $0, bar
+        and  $1, %ebx
+        andl $-1, bar
+        and  $42, %rbx
+        andq $-128, bar
+
+// CHECK:      Disassembly of section or:
+// CHECK-NEXT: or:
+// CHECK-NEXT:   0: 66 83 cb 00                   orw $0, %bx
+// CHECK-NEXT:   4: 66 83 0c 25 00 00 00 00 01    orw $1, 0
+// CHECK-NEXT:   d: 83 cb ff                      orl $-1, %ebx
+// CHECK-NEXT:  10: 83 0c 25 00 00 00 00 2a       orl $42, 0
+// CHECK-NEXT:  18: 48 83 cb 80                   orq $-128, %rbx
+// CHECK-NEXT:  1c: 48 83 0c 25 00 00 00 00 7f    orq $127, 0
+        .section or,"x"
+        or  $0, %bx
+        orw $1, bar
+        or  $-1, %ebx
+        orl $42, bar
+        or  $-128, %rbx
+        orq $127, bar
+
+// CHECK:      Disassembly of section xor:
+// CHECK-NEXT: xor:
+// CHECK-NEXT:   0: 66 83 f3 01                   xorw $1, %bx
+// CHECK-NEXT:   4: 66 83 34 25 00 00 00 00 ff    xorw $-1, 0
+// CHECK-NEXT:   d: 83 f3 2a                      xorl $42, %ebx
+// CHECK-NEXT:  10: 83 34 25 00 00 00 00 80       xorl $-128, 0
+// CHECK-NEXT:  18: 48 83 f3 7f                   xorq $127, %rbx
+// CHECK-NEXT:  1c: 48 83 34 25 00 00 00 00 00    xorq $0, 0
+        .section xor,"x"
+        xor  $1, %bx
+        xorw $-1, bar
+        xor  $42, %ebx
+        xorl $-128, bar
+        xor  $127, %rbx
+        xorq $0, bar
+
+// CHECK:      Disassembly of section add:
+// CHECK-NEXT: add:
+// CHECK-NEXT:   0: 66 83 c3 ff                   addw $-1, %bx
+// CHECK-NEXT:   4: 66 83 04 25 00 00 00 00 2a    addw $42, 0
+// CHECK-NEXT:   d: 83 c3 80                      addl $-128, %ebx
+// CHECK-NEXT:  10: 83 04 25 00 00 00 00 7f       addl $127, 0
+// CHECK-NEXT:  18: 48 83 c3 00                   addq $0, %rbx
+// CHECK-NEXT:  1c: 48 83 04 25 00 00 00 00 01    addq $1, 0
+        .section add,"x"
+        add  $-1, %bx
+        addw $42, bar
+        add  $-128, %ebx
+        addl $127, bar
+        add  $0, %rbx
+        addq $1, bar
+
+// CHECK:      Disassembly of section sub:
+// CHECK-NEXT: sub:
+// CHECK-NEXT:   0: 66 83 eb 2a                   subw $42, %bx
+// CHECK-NEXT:   4: 66 83 2c 25 00 00 00 00 80    subw $-128, 0
+// CHECK-NEXT:   d: 83 eb 7f                      subl $127, %ebx
+// CHECK-NEXT:  10: 83 2c 25 00 00 00 00 00       subl $0, 0
+// CHECK-NEXT:  18: 48 83 eb 01                   subq $1, %rbx
+// CHECK-NEXT:  1c: 48 83 2c 25 00 00 00 00 ff    subq $-1, 0
+        .section sub,"x"
+        sub  $42, %bx
+        subw $-128, bar
+        sub  $127, %ebx
+        subl $0, bar
+        sub  $1, %rbx
+        subq $-1, bar
+
+// CHECK:      Disassembly of section cmp:
+// CHECK-NEXT: cmp:
+// CHECK-NEXT:   0: 66 83 fb 80                   cmpw $-128, %bx
+// CHECK-NEXT:   4: 66 83 3c 25 00 00 00 00 7f    cmpw $127, 0
+// CHECK-NEXT:   d: 83 fb 00                      cmpl $0, %ebx
+// CHECK-NEXT:  10: 83 3c 25 00 00 00 00 01       cmpl $1, 0
+// CHECK-NEXT:  18: 48 83 fb ff                   cmpq $-1, %rbx
+// CHECK-NEXT:  1c: 48 83 3c 25 00 00 00 00 2a    cmpq $42, 0
+        .section cmp,"x"
+        cmp  $-128, %bx
+        cmpw $127, bar
+        cmp  $0, %ebx
+        cmpl $1, bar
+        cmp  $-1, %rbx
+        cmpq $42, bar
+
+// CHECK:      Disassembly of section push:
+// CHECK-NEXT: push:
+// CHECK-NEXT:   0: 66 6a 80                      pushw $-128
+// CHECK-NEXT:   3: 66 6a 7f                      pushw $127
+// CHECK-NEXT:   6: 6a 80                         pushq $-128
+// CHECK-NEXT:   8: 6a 7f                         pushq $127
+        .section push,"x"
+        pushw $-128
+        pushw $127
+        push  $-128
+        push  $127
diff --git a/test/MC/ELF/relax-arith3.s b/test/MC/ELF/relax-arith3.s
new file mode 100644
index 00000000000..3be8b0ee870
--- /dev/null
+++ b/test/MC/ELF/relax-arith3.s
@@ -0,0 +1,76 @@
+// RUN: llvm-mc -filetype=obj -triple x86_64-pc-linux-gnu %s -o - | llvm-objdump -d - | FileCheck  %s
+
+// Test that we correctly relax these instructions into versions that use
+// 16 or 32 bit immediate values.
+
+bar:
+// CHECK:      Disassembly of section imul:
+// CHECK-NEXT: imul:
+// CHECK-NEXT:   0: 66 69 1d 00 00 00 00 00 00        imulw $0, (%rip), %bx
+// CHECK-NEXT:   9: 69 1d 00 00 00 00 00 00 00 00     imull $0, (%rip), %ebx
+// CHECK-NEXT:  13: 48 69 1d 00 00 00 00 00 00 00 00  imulq $0, (%rip), %rbx
+        .section imul,"x"
+        imul $foo, bar(%rip),  %bx
+        imul $foo, bar(%rip),  %ebx
+        imul $foo, bar(%rip),  %rbx
+
+
+// CHECK:      Disassembly of section and:
+// CHECK-NEXT: and:
+// CHECK-NEXT:   0: 66 81 25 00 00 00 00 00 00        andw $0, (%rip)
+// CHECK-NEXT:   9: 81 25 00 00 00 00 00 00 00 00     andl $0, (%rip)
+// CHECK-NEXT:  13: 48 81 25 00 00 00 00 00 00 00 00  andq $0, (%rip)
+        .section and,"x"
+        andw $foo, bar(%rip)
+        andl $foo, bar(%rip)
+        andq $foo, bar(%rip)
+
+// CHECK:      Disassembly of section or:
+// CHECK-NEXT: or:
+// CHECK-NEXT:   0: 66 81 0d 00 00 00 00 00 00        orw $0, (%rip)
+// CHECK-NEXT:   9: 81 0d 00 00 00 00 00 00 00 00     orl $0, (%rip)
+// CHECK-NEXT:  13: 48 81 0d 00 00 00 00 00 00 00 00  orq $0, (%rip)
+        .section or,"x"
+        orw $foo, bar(%rip)
+        orl $foo, bar(%rip)
+        orq $foo, bar(%rip)
+
+// CHECK:      Disassembly of section xor:
+// CHECK-NEXT: xor:
+// CHECK-NEXT:   0: 66 81 35 00 00 00 00 00 00        xorw $0, (%rip)
+// CHECK-NEXT:   9: 81 35 00 00 00 00 00 00 00 00     xorl $0, (%rip)
+// CHECK-NEXT:  13: 48 81 35 00 00 00 00 00 00 00 00  xorq $0, (%rip)
+        .section xor,"x"
+        xorw $foo, bar(%rip)
+        xorl $foo, bar(%rip)
+        xorq $foo, bar(%rip)
+
+// CHECK:      Disassembly of section add:
+// CHECK-NEXT: add:
+// CHECK-NEXT:   0: 66 81 05 00 00 00 00 00 00        addw $0, (%rip)
+// CHECK-NEXT:   9: 81 05 00 00 00 00 00 00 00 00     addl $0, (%rip)
+// CHECK-NEXT:  13: 48 81 05 00 00 00 00 00 00 00 00  addq $0, (%rip)
+        .section add,"x"
+        addw $foo, bar(%rip)
+        addl $foo, bar(%rip)
+        addq $foo, bar(%rip)
+
+// CHECK:      Disassembly of section sub:
+// CHECK-NEXT: sub:
+// CHECK-NEXT:   0: 66 81 2d 00 00 00 00 00 00        subw $0, (%rip)
+// CHECK-NEXT:   9: 81 2d 00 00 00 00 00 00 00 00     subl $0, (%rip)
+// CHECK-NEXT:  13: 48 81 2d 00 00 00 00 00 00 00 00  subq $0, (%rip)
+        .section sub,"x"
+        subw $foo, bar(%rip)
+        subl $foo, bar(%rip)
+        subq $foo, bar(%rip)
+
+// CHECK:      Disassembly of section cmp:
+// CHECK-NEXT: cmp:
+// CHECK-NEXT:   0: 66 81 3d 00 00 00 00 00 00        cmpw $0, (%rip)
+// CHECK-NEXT:   9: 81 3d 00 00 00 00 00 00 00 00     cmpl $0, (%rip)
+// CHECK-NEXT:  13: 48 81 3d 00 00 00 00 00 00 00 00  cmpq $0, (%rip)
+        .section cmp,"x"
+        cmpw $foo, bar(%rip)
+        cmpl $foo, bar(%rip)
+        cmpq $foo, bar(%rip)
diff --git a/test/MC/ELF/relax-arith4.s b/test/MC/ELF/relax-arith4.s
new file mode 100644
index 00000000000..3fd3cadb76a
--- /dev/null
+++ b/test/MC/ELF/relax-arith4.s
@@ -0,0 +1,25 @@
+// RUN: llvm-mc -filetype=obj -triple i686-pc-linux-gnu %s -o - | llvm-objdump -d - | FileCheck  %s
+
+// Test for proper instruction relaxation behavior for the push $imm
+// instruction forms. This is the 32-bit version of the push $imm tests from
+// relax-arith.s and relax-arith2.s.
+
+// CHECK:      Disassembly of section push8:
+// CHECK-NEXT: push8:
+// CHECK-NEXT:   0: 66 6a 80                      pushw $-128
+// CHECK-NEXT:   3: 66 6a 7f                      pushw $127
+// CHECK-NEXT:   6: 6a 80                         pushl $-128
+// CHECK-NEXT:   8: 6a 7f                         pushl $127
+        .section push8,"x"
+        pushw $-128
+        pushw $127
+        push  $-128
+        push  $127
+
+// CHECK:      Disassembly of section push32:
+// CHECK-NEXT: push32:
+// CHECK-NEXT:   0: 66 68 00 00                   pushw $0
+// CHECK-NEXT:   4: 68 00 00 00 00                pushl $0
+        .section push32,"x"
+        pushw $foo
+        push  $foo
diff --git a/test/MC/ELF/relax.s b/test/MC/ELF/relax.s
index 49ee8e2eefe..93db0fe9b94 100644
--- a/test/MC/ELF/relax.s
+++ b/test/MC/ELF/relax.s
@@ -30,4 +30,4 @@ foo:
 // CHECK-NEXT:     )
 // CHECK-NEXT:   }
 // CHECK:        Symbol {
-// CHECK:          Name: foo (5)
+// CHECK:          Name: foo
diff --git a/test/MC/ELF/reloc-same-name-section.s b/test/MC/ELF/reloc-same-name-section.s
index e63ea548e5a..57463c65f89 100644
--- a/test/MC/ELF/reloc-same-name-section.s
+++ b/test/MC/ELF/reloc-same-name-section.s
@@ -7,13 +7,13 @@
 // CHECK-NEXT:     Relocation {
 // CHECK-NEXT:       Offset:
 // CHECK-NEXT:       Type:
-// CHECK-NEXT:       Symbol:  .foo (7)
+// CHECK-NEXT:       Symbol:  .foo (4)
 // CHECK-NEXT:       Addend:
 // CHECK-NEXT:     }
 // CHECK-NEXT:     Relocation {
 // CHECK-NEXT:       Offset:
 // CHECK-NEXT:       Type:
-// CHECK-NEXT:       Symbol:  .foo (8)
+// CHECK-NEXT:       Symbol:  .foo (5)
 // CHECK-NEXT:       Addend:
 // CHECK-NEXT:     }
 // CHECK-NEXT:   }
diff --git a/test/MC/ELF/relocation-386.s b/test/MC/ELF/relocation-386.s
index 2af6addc3d2..6b7e02f03ea 100644
--- a/test/MC/ELF/relocation-386.s
+++ b/test/MC/ELF/relocation-386.s
@@ -1,4 +1,4 @@
-// RUN: llvm-mc -filetype=obj -triple i386-pc-linux-gnu %s -o - | llvm-readobj -r -t | FileCheck  %s
+// RUN: llvm-mc -filetype=obj -triple i386-pc-linux-gnu %s -o - | llvm-readobj -r  | FileCheck  %s
 
 // Test that we produce the correct relocation types and that the relocations
 // correctly point to the section or the symbol.
@@ -71,27 +71,6 @@
 // CHECK-NEXT:   }
 // CHECK-NEXT: ]
 
-// Symbol 4 is zed
-// CHECK:        Symbol {
-// CHECK:          Name: zed
-// CHECK-NEXT:     Value: 0x0
-// CHECK-NEXT:     Size: 0
-// CHECK-NEXT:     Binding: Local
-// CHECK-NEXT:     Type: TLS
-// CHECK-NEXT:     Other: 0
-// CHECK-NEXT:     Section: zedsec
-// CHECK-NEXT:   }
-// Symbol 7 is section 4
-// CHECK:        Symbol {
-// CHECK:          Name: .bss (0)
-// CHECK-NEXT:     Value: 0x0
-// CHECK-NEXT:     Size: 0
-// CHECK-NEXT:     Binding: Local
-// CHECK-NEXT:     Type: Section
-// CHECK-NEXT:     Other: 0
-// CHECK-NEXT:     Section: .bss
-// CHECK-NEXT:   }
-
         .text
 bar:
 	leal	.Lfoo@GOTOFF(%ebx), %eax
diff --git a/test/MC/ELF/relocation-pc.s b/test/MC/ELF/relocation-pc.s
index a8783a7afb1..ca7addf3a62 100644
--- a/test/MC/ELF/relocation-pc.s
+++ b/test/MC/ELF/relocation-pc.s
@@ -5,13 +5,6 @@
 	loope	0                 # R_X86_64_PC8
 	jmp	-256              # R_X86_64_PC32
 
-// CHECK:        Section {
-// CHECK:          Index: 1
-// CHECK-NEXT:     Name: .text
-// CHECK:          Relocations [
-// CHECK-NEXT:     ]
-// CHECK-NEXT:   }
-
 // CHECK:        Section {
 // CHECK:          Index:
 // CHECK:          Name: .rela.text
@@ -21,8 +14,8 @@
 // CHECK-NEXT:     Address: 0x0
 // CHECK-NEXT:     Offset:
 // CHECK-NEXT:     Size: 48
-// CHECK-NEXT:     Link: 6
-// CHECK-NEXT:     Info: 1
+// CHECK-NEXT:     Link:
+// CHECK-NEXT:     Info:
 // CHECK-NEXT:     AddressAlignment: 8
 // CHECK-NEXT:     EntrySize: 24
 // CHECK-NEXT:     Relocations [
diff --git a/test/MC/ELF/relocation.s b/test/MC/ELF/relocation.s
index de2b43454ac..34f1a403813 100644
--- a/test/MC/ELF/relocation.s
+++ b/test/MC/ELF/relocation.s
@@ -1,7 +1,15 @@
-// RUN: llvm-mc -filetype=obj -triple x86_64-pc-linux-gnu %s -o - | llvm-readobj -s -sr -t | FileCheck  %s
+// RUN: llvm-mc -filetype=obj -triple x86_64-pc-linux-gnu %s -o - | llvm-readobj -s -sr  | FileCheck  %s
 
 // Test that we produce the correct relocation.
 
+
+        .section	.pr23272,"aGw",@progbits,pr23272,comdat
+	.globl pr23272
+pr23272:
+pr23272_2:
+pr23272_3 = pr23272_2
+
+        .text
 bar:
         movl	$bar, %edx        # R_X86_64_32
         movq	$bar, %rdx        # R_X86_64_32S
@@ -43,6 +51,13 @@ bar:
 
         .long   foo@gotpcrel
         .long foo@plt
+
+        .quad	pr23272_2 - pr23272
+        .quad	pr23272_3 - pr23272
+
+        .code16
+        call pr23771
+
 // CHECK:        Section {
 // CHECK:          Name: .rela.text
 // CHECK:          Relocations [
@@ -79,15 +94,6 @@ bar:
 // CHECK-NEXT:       0xD4 R_X86_64_SIZE32 blah 0xFFFFFFFFFFFFFFE0
 // CHECK-NEXT:       0xD8 R_X86_64_GOTPCREL foo 0x0
 // CHECK-NEXT:       0xDC R_X86_64_PLT32 foo 0x0
+// CHECK-NEXT:       0xF1 R_X86_64_PC16 pr23771 0xFFFFFFFFFFFFFFFE
 // CHECK-NEXT:     ]
 // CHECK-NEXT:   }
-
-// CHECK:        Symbol {
-// CHECK:          Name: .text (0)
-// CHECK-NEXT:     Value:
-// CHECK-NEXT:     Size:
-// CHECK-NEXT:     Binding: Local
-// CHECK-NEXT:     Type: Section
-// CHECK-NEXT:     Other: 0
-// CHECK-NEXT:     Section: .text (0x1)
-// CHECK-NEXT:   }
diff --git a/test/MC/ELF/rename.s b/test/MC/ELF/rename.s
index a7f96381cbd..8be06424cf2 100644
--- a/test/MC/ELF/rename.s
+++ b/test/MC/ELF/rename.s
@@ -24,8 +24,8 @@ defined3:
 // CHECK-NEXT:     Address: 0x0
 // CHECK-NEXT:     Offset:
 // CHECK-NEXT:     Size: 24
-// CHECK-NEXT:     Link: 6
-// CHECK-NEXT:     Info: 1
+// CHECK-NEXT:     Link:
+// CHECK-NEXT:     Info:
 // CHECK-NEXT:     AddressAlignment: 8
 // CHECK-NEXT:     EntrySize: 24
 // CHECK-NEXT:     Relocations [
diff --git a/test/MC/ELF/section-sym.s b/test/MC/ELF/section-sym.s
index 3b76d813fe7..4a9484d9b77 100644
--- a/test/MC/ELF/section-sym.s
+++ b/test/MC/ELF/section-sym.s
@@ -17,22 +17,22 @@
 // CHECK-NEXT:     SHF_GROUP (0x200)
 // CHECK-NEXT:   ]
 // CHECK-NEXT:   Address:         0x0
-// CHECK-NEXT:   Offset:  0x50
+// CHECK-NEXT:   Offset:
 // CHECK-NEXT:   Size:    0
 // CHECK-NEXT:   Link:    0
 // CHECK-NEXT:   Info:    0
 // CHECK-NEXT:   AddressAlignment:        1
 // CHECK-NEXT:   EntrySize:       0
 // CHECK-NEXT: }
-// CHECK-NEXT: Section {
-// CHECK-NEXT:   Index:   7
+// CHECK:      Section {
+// CHECK:        Index:   8
 // CHECK-NEXT:   Name:    foo (28)
 // CHECK-NEXT:   Type:    SHT_PROGBITS (0x1)
 // CHECK-NEXT:   Flags [ (0x200)
 // CHECK-NEXT:     SHF_GROUP (0x200)
 // CHECK-NEXT:   ]
 // CHECK-NEXT:   Address:         0x0
-// CHECK-NEXT:   Offset:  0x50
+// CHECK-NEXT:   Offset:
 // CHECK-NEXT:   Size:    0
 // CHECK-NEXT:   Link:    0
 // CHECK-NEXT:   Info:    0
@@ -40,52 +40,54 @@
 // CHECK-NEXT:   EntrySize:       0
 // CHECK-NEXT: }
 
-// The relocation points to symbol 6
+// The relocation points to symbol 3
 // CHECK:      Relocations [
-// CHECK-NEXT:   Section (9) .relabar {
+// CHECK-NEXT:   Section ({{.*}}) .relabar {
 // CHECK-NEXT:     Relocation {
 // CHECK-NEXT:       Offset:  0x0
 // CHECK-NEXT:       Type:    R_X86_64_32 (10)
-// CHECK-NEXT:       Symbol:  foo (6)
+// CHECK-NEXT:       Symbol:  foo (3)
 // CHECK-NEXT:       Addend:  0x0
 // CHECK-NEXT:     }
 // CHECK-NEXT:   }
 // CHECK-NEXT: ]
 
-
-// The symbol 6 corresponds section 6
+// Symbol 3 is section 6
 // CHECK: Symbols [
-
-// symbol 0
-// CHECK-NOT: Name
-// CHECK: Name:
-
-// symbol 1
-// CHECK-NOT: Name
-// CHECK: Name:    f1
-
-// symbol 2
-// CHECK-NOT: Name
-// CHECK: Name:    f2
-
-// symbol 3
-// CHECK-NOT: Name
-// CHECK: Name:    .text
-
-// symbol 4
-// CHECK-NOT: Name
-// CHECK: Name:    .data
-
-// symbol 5
-// CHECK-NOT: Name
-// CHECK: Name:    .bss
-
-// symbol 6
-// CHECK-NOT: Name
-// CHECK: Name:    foo
-// CHECK: Section: foo (0x6)
-
-// symbol 7
-// CHECK-NOT: Name
-// CHECK: Name:    foo
-// CHECK: Section: foo (0x7)
+// CHECK-NEXT:  Symbol {
+// CHECK-NEXT:    Name:  (0)
+// CHECK-NEXT:    Value: 0x0
+// CHECK-NEXT:    Size: 0
+// CHECK-NEXT:    Binding: Local (0x0)
+// CHECK-NEXT:    Type: None (0x0)
+// CHECK-NEXT:    Other: 0
+// CHECK-NEXT:    Section: Undefined (0x0)
+// CHECK-NEXT:  }
+// CHECK-NEXT:  Symbol {
+// CHECK-NEXT:    Name: f1 (57)
+// CHECK-NEXT:    Value: 0x0
+// CHECK-NEXT:    Size: 0
+// CHECK-NEXT:    Binding: Local (0x0)
+// CHECK-NEXT:    Type: None (0x0)
+// CHECK-NEXT:    Other: 0
+// CHECK-NEXT:    Section: .group (0x5)
+// CHECK-NEXT:  }
+// CHECK-NEXT:  Symbol {
+// CHECK-NEXT:    Name: f2 (54)
+// CHECK-NEXT:    Value: 0x0
+// CHECK-NEXT:    Size: 0
+// CHECK-NEXT:    Binding: Local (0x0)
+// CHECK-NEXT:    Type: None (0x0)
+// CHECK-NEXT:    Other: 0
+// CHECK-NEXT:    Section: .group (0x7)
+// CHECK-NEXT:  }
+// CHECK-NEXT:  Symbol {
+// CHECK-NEXT:    Name:  (0)
+// CHECK-NEXT:    Value: 0x0
+// CHECK-NEXT:    Size: 0
+// CHECK-NEXT:    Binding: Local (0x0)
+// CHECK-NEXT:    Type: Section (0x3)
+// CHECK-NEXT:    Other: 0
+// CHECK-NEXT:    Section: foo (0x6)
+// CHECK-NEXT:  }
+// CHECK-NEXT: ]
diff --git a/test/MC/ELF/section-sym2.s b/test/MC/ELF/section-sym2.s
index f62e3f9f8a3..ca3863223db 100644
--- a/test/MC/ELF/section-sym2.s
+++ b/test/MC/ELF/section-sym2.s
@@ -19,10 +19,6 @@ mov .rodata, %rsi
 // There is only one .rodata symbol
 
 // CHECK:Symbols [
-// CHECK-NOT:    Name: .rodata
-// CHECK:        Name: .rodata
-// CHECK-NEXT:   Value: 0x0
-// CHECK-NEXT:   Size: 0
-// CHECK-NEXT:   Binding: Local (0x0)
-// CHECK-NEXT:   Type: Section (0x3)
-// CHECK-NOT:    Name: .rodata
+// CHECK:   Type: Section (0x3)
+// CHECK:   Section: .rodata
+// CHECK-NOT:   Section: .rodata
diff --git a/test/MC/ELF/section-unique.s b/test/MC/ELF/section-unique.s
index 3fe727127af..bd15148f6d8 100644
--- a/test/MC/ELF/section-unique.s
+++ b/test/MC/ELF/section-unique.s
@@ -26,7 +26,7 @@ g:
 // OBJ:   Binding: Global
 // OBJ:   Type:    None
 // OBJ:   Other:   0
-// OBJ:   Section: .text (0x4)
+// OBJ:   Section: .text (0x5)
 // OBJ: }
 // OBJ: Symbol {
 // OBJ:   Name:    g
@@ -35,5 +35,5 @@ g:
 // OBJ:   Binding: Global
 // OBJ:   Type:    None
 // OBJ:   Other:   0
-// OBJ:   Section: .text (0x5)
+// OBJ:   Section: .text (0x6)
 // OBJ: }
diff --git a/test/MC/ELF/section.s b/test/MC/ELF/section.s
index 7dc23c2e596..ab3fe038740 100644
--- a/test/MC/ELF/section.s
+++ b/test/MC/ELF/section.s
@@ -7,10 +7,10 @@
 .section	.note.GNU-,"",@progbits
 .section	-.note.GNU,"","progbits"
 
-// CHECK: Name: .note.GNU-stack (56)
-// CHECK: Name: .note.GNU-stack2 (153)
-// CHECK: Name: .note.GNU- (170)
-// CHECK: Name: -.note.GNU (142)
+// CHECK: Name: .note.GNU-stack
+// CHECK: Name: .note.GNU-stack2
+// CHECK: Name: .note.GNU-
+// CHECK: Name: -.note.GNU
 
 // Test that the defaults are used
 
@@ -27,7 +27,7 @@
 // CHECK-NEXT:       SHF_EXECINSTR
 // CHECK-NEXT:     ]
 // CHECK-NEXT:     Address: 0x0
-// CHECK-NEXT:     Offset: 0x50
+// CHECK-NEXT:     Offset:
 // CHECK-NEXT:     Size: 0
 // CHECK-NEXT:     Link: 0
 // CHECK-NEXT:     Info: 0
@@ -35,7 +35,7 @@
 // CHECK-NEXT:     EntrySize: 0
 // CHECK-NEXT:   }
 // CHECK-NEXT:   Section {
-// CHECK-NEXT:     Index: 11
+// CHECK-NEXT:     Index:
 // CHECK-NEXT:     Name: .fini
 // CHECK-NEXT:     Type: SHT_PROGBITS
 // CHECK-NEXT:     Flags [
@@ -43,7 +43,7 @@
 // CHECK-NEXT:       SHF_EXECINSTR
 // CHECK-NEXT:     ]
 // CHECK-NEXT:     Address: 0x0
-// CHECK-NEXT:     Offset: 0x50
+// CHECK-NEXT:     Offset:
 // CHECK-NEXT:     Size: 0
 // CHECK-NEXT:     Link: 0
 // CHECK-NEXT:     Info: 0
@@ -51,14 +51,14 @@
 // CHECK-NEXT:     EntrySize: 0
 // CHECK-NEXT:   }
 // CHECK-NEXT:   Section {
-// CHECK-NEXT:     Index: 12
+// CHECK-NEXT:     Index:
 // CHECK-NEXT:     Name: .rodata
 // CHECK-NEXT:     Type: SHT_PROGBITS
 // CHECK-NEXT:     Flags [
 // CHECK-NEXT:       SHF_ALLOC
 // CHECK-NEXT:     ]
 // CHECK-NEXT:     Address: 0x0
-// CHECK-NEXT:     Offset: 0x50
+// CHECK-NEXT:     Offset:
 // CHECK-NEXT:     Size: 0
 // CHECK-NEXT:     Link: 0
 // CHECK-NEXT:     Info: 0
@@ -66,13 +66,13 @@
 // CHECK-NEXT:     EntrySize: 0
 // CHECK-NEXT:   }
 // CHECK-NEXT:   Section {
-// CHECK-NEXT:     Index: 13
+// CHECK-NEXT:     Index:
 // CHECK-NEXT:     Name: zed
 // CHECK-NEXT:     Type: SHT_PROGBITS
 // CHECK-NEXT:     Flags [
 // CHECK-NEXT:     ]
 // CHECK-NEXT:     Address: 0x0
-// CHECK-NEXT:     Offset: 0x50
+// CHECK-NEXT:     Offset:
 // CHECK-NEXT:     Size: 0
 // CHECK-NEXT:     Link: 0
 // CHECK-NEXT:     Info: 0
@@ -87,7 +87,7 @@
 // CHECK-NEXT:     Flags [
 // CHECK-NEXT:     ]
 // CHECK-NEXT:     Address: 0x0
-// CHECK-NEXT:     Offset: 0x50
+// CHECK-NEXT:     Offset:
 // CHECK-NEXT:     Size: 0
 // CHECK-NEXT:     Link: 0
 // CHECK-NEXT:     Info: 0
@@ -112,7 +112,7 @@ bar:
 // CHECK-NEXT:       SHF_ALLOC
 // CHECK-NEXT:     ]
 // CHECK-NEXT:     Address: 0x0
-// CHECK-NEXT:     Offset: 0x50
+// CHECK-NEXT:     Offset:
 // CHECK-NEXT:     Size: 0
 // CHECK-NEXT:     Link: 0
 // CHECK-NEXT:     Info: 0
@@ -129,7 +129,7 @@ bar:
 // CHECK-NEXT:     SHF_EXCLUDE (0x80000000)
 // CHECK-NEXT:   ]
 // CHECK-NEXT:   Address: 0x0
-// CHECK-NEXT:   Offset: 0x50
+// CHECK-NEXT:   Offset:
 // CHECK-NEXT:   Size: 0
 // CHECK-NEXT:   Link: 0
 // CHECK-NEXT:   Info: 0
@@ -142,6 +142,6 @@ bar:
 .section "foo"
 
 // CHECK:        Section {
-// CHECK:          Name: bar-"foo" (181)
+// CHECK:          Name: bar-"foo"
 // CHECK:        Section {
-// CHECK:          Name: foo (52)
+// CHECK:          Name: foo
diff --git a/test/MC/ELF/strtab-suffix-opt.s b/test/MC/ELF/strtab-suffix-opt.s
index eb5da8a0155..96d15005c61 100644
--- a/test/MC/ELF/strtab-suffix-opt.s
+++ b/test/MC/ELF/strtab-suffix-opt.s
@@ -16,6 +16,6 @@ foobar:
 .Ltmp3:
 	.size	foobar, .Ltmp3-foobar
 
-// CHECK:     Name: foobar (1)
-// CHECK:     Name: bar (4)
-// CHECK:     Name: foo (8)
+// CHECK:     Name: bar (19)
+// CHECK:     Name: foo (23)
+// CHECK:     Name: foobar (16)
diff --git a/test/MC/ELF/symver-msvc.s b/test/MC/ELF/symver-msvc.s
index a726ff32dd5..354ece140be 100644
--- a/test/MC/ELF/symver-msvc.s
+++ b/test/MC/ELF/symver-msvc.s
@@ -21,7 +21,7 @@
 
 // CHECK:       Symbols [
 // CHECK:         Symbol {
-// CHECK:           Name: ??_R0?AVexception@std@@@8 (102)
+// CHECK:           Name: ??_R0?AVexception@std@@@8
 // CHECK-NEXT:      Value: 0x0
 // CHECK-NEXT:      Size: 0
 // CHECK-NEXT:      Binding: Global (0x1)
@@ -30,7 +30,7 @@
 // CHECK-NEXT:      Section: Undefined (0x0)
 // CHECK-NEXT:    }
 // CHECK-NEXT:    Symbol {
-// CHECK-NEXT:      Name: @??_R0?AVinvalid_argument@std@@@8 (1)
+// CHECK-NEXT:      Name: @??_R0?AVinvalid_argument@std@@@8
 // CHECK-NEXT:      Value: 0x0
 // CHECK-NEXT:      Size: 0
 // CHECK-NEXT:      Binding: Global (0x1)
@@ -39,7 +39,7 @@
 // CHECK-NEXT:      Section: Undefined (0x0)
 // CHECK-NEXT:    }
 // CHECK-NEXT:    Symbol {
-// CHECK-NEXT:      Name: __imp_??_R0?AVlogic_error@std@@@8 (35)
+// CHECK-NEXT:      Name: __imp_??_R0?AVlogic_error@std@@@8
 // CHECK-NEXT:      Value: 0x0
 // CHECK-NEXT:      Size: 0
 // CHECK-NEXT:      Binding: Global (0x1)
@@ -48,7 +48,7 @@
 // CHECK-NEXT:      Section: Undefined (0x0)
 // CHECK-NEXT:    }
 // CHECK-NEXT:    Symbol {
-// CHECK-NEXT:      Name: __imp_@??_R0PAVexception@std@@@8 (69)
+// CHECK-NEXT:      Name: __imp_@??_R0PAVexception@std@@@8
 // CHECK-NEXT:      Value: 0x0
 // CHECK-NEXT:      Size: 0
 // CHECK-NEXT:      Binding: Global (0x1)
diff --git a/test/MC/ELF/symver-pr23914.s b/test/MC/ELF/symver-pr23914.s
new file mode 100644
index 00000000000..e8b43251010
--- /dev/null
+++ b/test/MC/ELF/symver-pr23914.s
@@ -0,0 +1,16 @@
+// Regression test for PR23914.
+// RUN: llvm-mc -filetype=obj -triple x86_64-pc-linux-gnu %s -o - | llvm-readobj -r -t | FileCheck %s
+
+defined:
+        .symver defined, aaaaaaaaaaaaaaaaaa@@@AAAAAAAAAAAAA
+
+// CHECK:      Symbol {
+// CHECK:        Name: aaaaaaaaaaaaaaaaaa@@AAAAAAAAAAAAA
+// CHECK-NEXT:   Value: 0x0
+// CHECK-NEXT:   Size: 0
+// CHECK-NEXT:   Binding: Local
+// CHECK-NEXT:   Type: None
+// CHECK-NEXT:   Other: 0
+// CHECK-NEXT:   Section: .text
+// CHECK-NEXT: }
+
diff --git a/test/MC/ELF/symver.s b/test/MC/ELF/symver.s
index 80d71fd0e06..0f434dd859d 100644
--- a/test/MC/ELF/symver.s
+++ b/test/MC/ELF/symver.s
@@ -77,7 +77,7 @@ global1:
 // CHECK-NEXT:     Section: .text
 // CHECK-NEXT:   }
 // CHECK-NEXT:   Symbol {
-// CHECK-NEXT:     Name: .text
+// CHECK-NEXT:     Name: (0)
 // CHECK-NEXT:     Value: 0x0
 // CHECK-NEXT:     Size: 0
 // CHECK-NEXT:     Binding: Local
@@ -86,22 +86,22 @@ global1:
 // CHECK-NEXT:     Section: .text
 // CHECK-NEXT:   }
 // CHECK-NEXT:   Symbol {
-// CHECK-NEXT:     Name: .data
+// CHECK-NEXT:     Name: bar2@zed
 // CHECK-NEXT:     Value: 0x0
 // CHECK-NEXT:     Size: 0
-// CHECK-NEXT:     Binding: Local
-// CHECK-NEXT:     Type: Section
+// CHECK-NEXT:     Binding: Global
+// CHECK-NEXT:     Type: None
 // CHECK-NEXT:     Other: 0
-// CHECK-NEXT:     Section: .data
+// CHECK-NEXT:     Section: Undefined
 // CHECK-NEXT:   }
 // CHECK-NEXT:   Symbol {
-// CHECK-NEXT:     Name: .bss
+// CHECK-NEXT:     Name: bar6@zed
 // CHECK-NEXT:     Value: 0x0
 // CHECK-NEXT:     Size: 0
-// CHECK-NEXT:     Binding: Local
-// CHECK-NEXT:     Type: Section
+// CHECK-NEXT:     Binding: Global
+// CHECK-NEXT:     Type: None
 // CHECK-NEXT:     Other: 0
-// CHECK-NEXT:     Section: .bss
+// CHECK-NEXT:     Section: Undefined
 // CHECK-NEXT:   }
 // CHECK-NEXT:   Symbol {
 // CHECK-NEXT:     Name: g1@@zed
@@ -121,22 +121,4 @@ global1:
 // CHECK-NEXT:     Other: 0
 // CHECK-NEXT:     Section: .text
 // CHECK-NEXT:   }
-// CHECK-NEXT:   Symbol {
-// CHECK-NEXT:     Name: bar2@zed
-// CHECK-NEXT:     Value: 0x0
-// CHECK-NEXT:     Size: 0
-// CHECK-NEXT:     Binding: Global
-// CHECK-NEXT:     Type: None
-// CHECK-NEXT:     Other: 0
-// CHECK-NEXT:     Section: Undefined
-// CHECK-NEXT:   }
-// CHECK-NEXT:   Symbol {
-// CHECK-NEXT:     Name: bar6@zed
-// CHECK-NEXT:     Value: 0x0
-// CHECK-NEXT:     Size: 0
-// CHECK-NEXT:     Binding: Global
-// CHECK-NEXT:     Type: None
-// CHECK-NEXT:     Other: 0
-// CHECK-NEXT:     Section: Undefined
-// CHECK-NEXT:   }
 // CHECK-NEXT: ]
diff --git a/test/MC/ELF/type-propagate.s b/test/MC/ELF/type-propagate.s
index 15d05af7f2f..ecfab6dbcda 100644
--- a/test/MC/ELF/type-propagate.s
+++ b/test/MC/ELF/type-propagate.s
@@ -36,7 +36,7 @@ sym10:
 // CHECK-NEXT:    Binding: Local (0x0)
 // CHECK-NEXT:    Type: Object (0x1)
 // CHECK-NEXT:    Other: 0
-// CHECK-NEXT:    Section: .text (0x1)
+// CHECK-NEXT:    Section: .text
 // CHECK-NEXT:  }
 // CHECK-NEXT:  Symbol {
 // CHECK-NEXT:    Name: sym02
@@ -45,7 +45,7 @@ sym10:
 // CHECK-NEXT:    Binding: Local (0x0)
 // CHECK-NEXT:    Type: Function (0x2)
 // CHECK-NEXT:    Other: 0
-// CHECK-NEXT:    Section: .text (0x1)
+// CHECK-NEXT:    Section: .text
 // CHECK-NEXT:  }
 // CHECK-NEXT:  Symbol {
 // CHECK-NEXT:    Name: sym03
@@ -54,7 +54,7 @@ sym10:
 // CHECK-NEXT:    Binding: Local (0x0)
 // CHECK-NEXT:    Type: Function (0x2)
 // CHECK-NEXT:    Other: 0
-// CHECK-NEXT:    Section: .text (0x1)
+// CHECK-NEXT:    Section: .text
 // CHECK-NEXT:  }
 // CHECK-NEXT:  Symbol {
 // CHECK-NEXT:    Name: sym04
@@ -63,7 +63,7 @@ sym10:
 // CHECK-NEXT:    Binding: Local (0x0)
 // CHECK-NEXT:    Type: Object (0x1)
 // CHECK-NEXT:    Other: 0
-// CHECK-NEXT:    Section: .text (0x1)
+// CHECK-NEXT:    Section: .text
 // CHECK-NEXT:  }
 // CHECK-NEXT:  Symbol {
 // CHECK-NEXT:    Name: sym05
@@ -75,7 +75,7 @@ sym10:
 // CHECK-NEXT:    Type: Object (0x1)
 
 // CHECK-NEXT:    Other: 0
-// CHECK-NEXT:    Section: .text (0x1)
+// CHECK-NEXT:    Section: .text
 // CHECK-NEXT:  }
 // CHECK-NEXT:  Symbol {
 // CHECK-NEXT:    Name: sym06
@@ -102,7 +102,7 @@ sym10:
 // CHECK-NEXT:    Binding: Local (0x0)
 // CHECK-NEXT:    Type: Function (0x2)
 // CHECK-NEXT:    Other: 0
-// CHECK-NEXT:    Section: .text (0x1)
+// CHECK-NEXT:    Section: .text
 // CHECK-NEXT:  }
 // CHECK-NEXT:  Symbol {
 // CHECK-NEXT:    Name: sym09
@@ -114,7 +114,7 @@ sym10:
 // CHECK-NEXT:    Type: Function (0x2)
 
 // CHECK-NEXT:    Other: 0
-// CHECK-NEXT:    Section: .text (0x1)
+// CHECK-NEXT:    Section: .text
 // CHECK-NEXT:  }
 // CHECK-NEXT:  Symbol {
 // CHECK-NEXT:    Name: sym10
@@ -123,7 +123,7 @@ sym10:
 // CHECK-NEXT:    Binding: Local (0x0)
 // CHECK-NEXT:    Type: Function (0x2)
 // CHECK-NEXT:    Other: 0
-// CHECK-NEXT:    Section: .text (0x1)
+// CHECK-NEXT:    Section: .text
 // CHECK-NEXT:  }
 // CHECK-NEXT:  Symbol {
 // CHECK-NEXT:    Name: sym11
@@ -135,7 +135,7 @@ sym10:
 // CHECK-NEXT:    Type: Function (0x2)
 
 // CHECK-NEXT:    Other: 0
-// CHECK-NEXT:    Section: .text (0x1)
+// CHECK-NEXT:    Section: .text
 // CHECK-NEXT:  }
 // CHECK-NEXT:  Symbol {
 // CHECK-NEXT:    Name: sym12
@@ -147,5 +147,5 @@ sym10:
 // CHECK-NEXT:    Type: Function (0x2)
 
 // CHECK-NEXT:    Other: 0
-// CHECK-NEXT:    Section: .text (0x1)
+// CHECK-NEXT:    Section: .text
 // CHECK-NEXT:  }
diff --git a/test/MC/ELF/type.s b/test/MC/ELF/type.s
index f7745d88ea3..a9cc3ebc777 100644
--- a/test/MC/ELF/type.s
+++ b/test/MC/ELF/type.s
@@ -137,7 +137,7 @@ alias12:
 // CHECK-NEXT:     Binding: Global
 // CHECK-NEXT:     Type: Object
 // CHECK-NEXT:     Other: 0
-// CHECK-NEXT:     Section: .text (0x1)
+// CHECK-NEXT:     Section: .text
 // CHECK-NEXT:   }
 // CHECK-NEXT:   Symbol {
 // CHECK-NEXT:     Name: foo
@@ -146,7 +146,7 @@ alias12:
 // CHECK-NEXT:     Binding: Global
 // CHECK-NEXT:     Type: Function
 // CHECK-NEXT:     Other: 0
-// CHECK-NEXT:     Section: .text (0x1)
+// CHECK-NEXT:     Section: .text
 // CHECK-NEXT:   }
 // CHECK-NEXT:   Symbol {
 // CHECK-NEXT:     Name: func
@@ -155,7 +155,7 @@ alias12:
 // CHECK-NEXT:     Binding: Global
 // CHECK-NEXT:     Type: Function
 // CHECK-NEXT:     Other: 0
-// CHECK-NEXT:     Section: .text (0x1)
+// CHECK-NEXT:     Section: .text
 // CHECK-NEXT:   }
 // CHECK-NEXT:   Symbol {
 // CHECK-NEXT:     Name: ifunc
@@ -164,7 +164,7 @@ alias12:
 // CHECK-NEXT:     Binding: Global
 // CHECK-NEXT:     Type: GNU_IFunc
 // CHECK-NEXT:     Other: 0
-// CHECK-NEXT:     Section: .text (0x1)
+// CHECK-NEXT:     Section: .text
 // CHECK-NEXT:   }
 // CHECK-NEXT:   Symbol {
 // CHECK-NEXT:     Name: obj
@@ -173,7 +173,7 @@ alias12:
 // CHECK-NEXT:     Binding: Global
 // CHECK-NEXT:     Type: Object
 // CHECK-NEXT:     Other: 0
-// CHECK-NEXT:     Section: .text (0x1)
+// CHECK-NEXT:     Section: .text
 // CHECK-NEXT:   }
 // CHECK-NEXT:  Symbol {
 // CHECK-NEXT:    Name: sym1
@@ -182,7 +182,7 @@ alias12:
 // CHECK-NEXT:    Binding: Global (0x1)
 // CHECK-NEXT:    Type: GNU_IFunc (0xA)
 // CHECK-NEXT:    Other: 0
-// CHECK-NEXT:    Section: .text (0x1)
+// CHECK-NEXT:    Section: .text
 // CHECK-NEXT:  }
 // CHECK-NEXT:  Symbol {
 // CHECK-NEXT:    Name: sym10
@@ -191,7 +191,7 @@ alias12:
 // CHECK-NEXT:    Binding: Global (0x1)
 // CHECK-NEXT:    Type: TLS (0x6)
 // CHECK-NEXT:    Other: 0
-// CHECK-NEXT:    Section: .text (0x1)
+// CHECK-NEXT:    Section: .text
 // CHECK-NEXT:  }
 // CHECK-NEXT:  Symbol {
 // CHECK-NEXT:    Name: sym11
@@ -200,7 +200,7 @@ alias12:
 // CHECK-NEXT:    Binding: Global (0x1)
 // CHECK-NEXT:    Type: TLS (0x6)
 // CHECK-NEXT:    Other: 0
-// CHECK-NEXT:    Section: .text (0x1)
+// CHECK-NEXT:    Section: .text
 // CHECK-NEXT:  }
 // CHECK-NEXT:  Symbol {
 // CHECK-NEXT:    Name: sym12
@@ -209,7 +209,7 @@ alias12:
 // CHECK-NEXT:    Binding: Global (0x1)
 // CHECK-NEXT:    Type: TLS (0x6)
 // CHECK-NEXT:    Other: 0
-// CHECK-NEXT:    Section: .text (0x1)
+// CHECK-NEXT:    Section: .text
 // CHECK-NEXT:  }
 // CHECK-NEXT:  Symbol {
 // CHECK-NEXT:    Name: sym2
@@ -218,7 +218,7 @@ alias12:
 // CHECK-NEXT:    Binding: Global (0x1)
 // CHECK-NEXT:    Type: GNU_IFunc (0xA)
 // CHECK-NEXT:    Other: 0
-// CHECK-NEXT:    Section: .text (0x1)
+// CHECK-NEXT:    Section: .text
 // CHECK-NEXT:  }
 // CHECK-NEXT:  Symbol {
 // CHECK-NEXT:    Name: sym3
@@ -227,7 +227,7 @@ alias12:
 // CHECK-NEXT:    Binding: Global (0x1)
 // CHECK-NEXT:    Type: GNU_IFunc (0xA)
 // CHECK-NEXT:    Other: 0
-// CHECK-NEXT:    Section: .text (0x1)
+// CHECK-NEXT:    Section: .text
 // CHECK-NEXT:  }
 // CHECK-NEXT:  Symbol {
 // CHECK-NEXT:    Name: sym4
@@ -236,7 +236,7 @@ alias12:
 // CHECK-NEXT:    Binding: Global (0x1)
 // CHECK-NEXT:    Type: Function (0x2)
 // CHECK-NEXT:    Other: 0
-// CHECK-NEXT:    Section: .text (0x1)
+// CHECK-NEXT:    Section: .text
 // CHECK-NEXT:  }
 // CHECK-NEXT:  Symbol {
 // CHECK-NEXT:    Name: sym5
@@ -245,7 +245,7 @@ alias12:
 // CHECK-NEXT:    Binding: Global (0x1)
 // CHECK-NEXT:    Type: Function (0x2)
 // CHECK-NEXT:    Other: 0
-// CHECK-NEXT:    Section: .text (0x1)
+// CHECK-NEXT:    Section: .text
 // CHECK-NEXT:  }
 // CHECK-NEXT:  Symbol {
 // CHECK-NEXT:    Name: sym6
@@ -254,7 +254,7 @@ alias12:
 // CHECK-NEXT:    Binding: Global (0x1)
 // CHECK-NEXT:    Type: Object (0x1)
 // CHECK-NEXT:    Other: 0
-// CHECK-NEXT:    Section: .text (0x1)
+// CHECK-NEXT:    Section: .text
 // CHECK-NEXT:  }
 // CHECK-NEXT:  Symbol {
 // CHECK-NEXT:    Name: sym7
@@ -263,7 +263,7 @@ alias12:
 // CHECK-NEXT:    Binding: Global (0x1)
 // CHECK-NEXT:    Type: GNU_IFunc (0xA)
 // CHECK-NEXT:    Other: 0
-// CHECK-NEXT:    Section: .text (0x1)
+// CHECK-NEXT:    Section: .text
 // CHECK-NEXT:  }
 // CHECK-NEXT:  Symbol {
 // CHECK-NEXT:    Name: sym8
@@ -272,7 +272,7 @@ alias12:
 // CHECK-NEXT:    Binding: Global (0x1)
 // CHECK-NEXT:    Type: Function (0x2)
 // CHECK-NEXT:    Other: 0
-// CHECK-NEXT:    Section: .text (0x1)
+// CHECK-NEXT:    Section: .text
 // CHECK-NEXT:  }
 // CHECK-NEXT:  Symbol {
 // CHECK-NEXT:    Name: sym9
@@ -281,7 +281,7 @@ alias12:
 // CHECK-NEXT:    Binding: Global (0x1)
 // CHECK-NEXT:    Type: TLS (0x6)
 // CHECK-NEXT:    Other: 0
-// CHECK-NEXT:    Section: .text (0x1)
+// CHECK-NEXT:    Section: .text
 // CHECK-NEXT:  }
 // CHECK-NEXT:   Symbol {
 // CHECK-NEXT:     Name: tls
@@ -290,7 +290,7 @@ alias12:
 // CHECK-NEXT:     Binding: Global
 // CHECK-NEXT:     Type: TLS
 // CHECK-NEXT:     Other: 0
-// CHECK-NEXT:     Section: .text (0x1)
+// CHECK-NEXT:     Section: .text
 // CHECK-NEXT:   }
 // CHECK-NEXT:   Symbol {
 // CHECK-NEXT:     Name: tls_quoted
@@ -299,7 +299,7 @@ alias12:
 // CHECK-NEXT:     Binding: Global
 // CHECK-NEXT:     Type: TLS
 // CHECK-NEXT:     Other: 0
-// CHECK-NEXT:     Section: .text (0x1)
+// CHECK-NEXT:     Section: .text
 // CHECK-NEXT:   }
 // CHECK-NEXT:   Symbol {
 // CHECK-NEXT:     Name: tls_upper_case
@@ -308,15 +308,15 @@ alias12:
 // CHECK-NEXT:     Binding: Global
 // CHECK-NEXT:     Type: TLS
 // CHECK-NEXT:     Other: 0
-// CHECK-NEXT:     Section: .text (0x1)
+// CHECK-NEXT:     Section: .text
 // CHECK-NEXT:   }
 // CHECK-NEXT:   Symbol {
-// CHECK-NEXT:     Name: zed (32)
+// CHECK-NEXT:     Name: zed
 // CHECK-NEXT:     Value: 0x0
 // CHECK-NEXT:     Size: 0
 // CHECK-NEXT:     Binding: Unique (0xA)
 // CHECK-NEXT:     Type: Object (0x1)
 // CHECK-NEXT:     Other: 0
-// CHECK-NEXT:     Section: .text (0x1)
+// CHECK-NEXT:     Section: .text
 // CHECK-NEXT:   }
 // CHECK-NEXT: ]
diff --git a/test/MC/ELF/undef-temp.s b/test/MC/ELF/undef-temp.s
new file mode 100644
index 00000000000..45537a998e7
--- /dev/null
+++ b/test/MC/ELF/undef-temp.s
@@ -0,0 +1,4 @@
+// RUN: not llvm-mc -filetype=obj -triple x86_64-pc-linux %s -o - 2>&1 | FileCheck %s
+
+// CHECK: Undefined temporary
+        .long .Lfoo
diff --git a/test/MC/ELF/undef.s b/test/MC/ELF/undef.s
index 245b5632824..47cd85ab4a4 100644
--- a/test/MC/ELF/undef.s
+++ b/test/MC/ELF/undef.s
@@ -2,7 +2,6 @@
 
 // Test which symbols should be in the symbol table
 
-        .long	.Lsym1
 .Lsym2:
 .Lsym3:
 .Lsym4 = .Lsym2 - .Lsym3
@@ -42,51 +41,6 @@ test2_b = undef + 1
 // CHECK-NEXT:     Section: .rodata.str1.1
 // CHECK-NEXT:   }
 // CHECK-NEXT:   Symbol {
-// CHECK-NEXT:     Name: .text
-// CHECK-NEXT:     Value: 0x0
-// CHECK-NEXT:     Size: 0
-// CHECK-NEXT:     Binding: Local
-// CHECK-NEXT:     Type: Section
-// CHECK-NEXT:     Other: 0
-// CHECK-NEXT:     Section: .text
-// CHECK-NEXT:   }
-// CHECK-NEXT:   Symbol {
-// CHECK-NEXT:     Name: .data
-// CHECK-NEXT:     Value: 0x0
-// CHECK-NEXT:     Size: 0
-// CHECK-NEXT:     Binding: Local
-// CHECK-NEXT:     Type: Section
-// CHECK-NEXT:     Other: 0
-// CHECK-NEXT:     Section: .data
-// CHECK-NEXT:   }
-// CHECK-NEXT:   Symbol {
-// CHECK-NEXT:     Name: .bss
-// CHECK-NEXT:     Value: 0x0
-// CHECK-NEXT:     Size: 0
-// CHECK-NEXT:     Binding: Local
-// CHECK-NEXT:     Type: Section
-// CHECK-NEXT:     Other: 0
-// CHECK-NEXT:     Section: .bss
-// CHECK-NEXT:   }
-// CHECK-NEXT:   Symbol {
-// CHECK-NEXT:     Name: .rodata.str1.1
-// CHECK-NEXT:     Value: 0x0
-// CHECK-NEXT:     Size: 0
-// CHECK-NEXT:     Binding: Local
-// CHECK-NEXT:     Type: Section
-// CHECK-NEXT:     Other: 0
-// CHECK-NEXT:     Section: .rodata.str1.1
-// CHECK-NEXT:   }
-// CHECK-NEXT:   Symbol {
-// CHECK-NEXT:     Name: .Lsym1
-// CHECK-NEXT:     Value: 0x0
-// CHECK-NEXT:     Size: 0
-// CHECK-NEXT:     Binding: Global
-// CHECK-NEXT:     Type: None
-// CHECK-NEXT:     Other: 0
-// CHECK-NEXT:     Section: Undefined
-// CHECK-NEXT:   }
-// CHECK-NEXT:   Symbol {
 // CHECK-NEXT:     Name: sym6
 // CHECK-NEXT:     Value: 0x0
 // CHECK-NEXT:     Size: 0
diff --git a/test/MC/ELF/undef2.s b/test/MC/ELF/undef2.s
deleted file mode 100644
index 6aa66c05c4b..00000000000
--- a/test/MC/ELF/undef2.s
+++ /dev/null
@@ -1,18 +0,0 @@
-// RUN: llvm-mc -filetype=obj -triple x86_64-pc-linux-gnu %s -o - | llvm-readobj -s -t | FileCheck %s
-
-// Test that this produces an undefined reference to .Lfoo
-
-        je	.Lfoo
-
-// CHECK:       Section {
-// CHECK:         Name: .strtab
-
-// CHECK:       Symbol {
-// CHECK:         Name: .Lfoo
-// CHECK-NEXT:    Value:
-// CHECK-NEXT:    Size:
-// CHECK-NEXT:    Binding: Global
-// CHECK-NEXT:    Type:
-// CHECK-NEXT:    Other:
-// CHECK-NEXT:    Section:
-// CHECK-NEXT:  }
diff --git a/test/MC/ELF/weak-diff.s b/test/MC/ELF/weak-diff.s
index d270bbb7334..ded89b8f833 100644
--- a/test/MC/ELF/weak-diff.s
+++ b/test/MC/ELF/weak-diff.s
@@ -1,6 +1,10 @@
-// RUN: not llvm-mc -filetype=obj -triple x86_64-pc-linux-gnu %s -o %t 2>&1 | FileCheck %s
+// RUN: llvm-mc -filetype=obj -triple x86_64-pc-linux-gnu < %s | llvm-readobj -r | FileCheck %s
 
-// CHECK: error: Cannot represent a subtraction with a weak symbol
+// CHECK:      Relocations [
+// CHECK-NEXT:   Section ({{.*}}) .rela.text {
+// CHECK-NEXT:     0x1D R_X86_64_PC32 f2 0xFFFFFFFFFFFFFFFC
+// CHECK-NEXT:   }
+// CHECK-NEXT: ]
 
 .weak f
 .weak g
@@ -10,3 +14,13 @@ g:
     nop
 
 .quad g - f
+
+
+.weak f2
+f2:
+    nop
+g2:
+    nop
+.quad g2 - f2
+.quad f2 - g2
+call f2
diff --git a/test/MC/ELF/weak-diff2.s b/test/MC/ELF/weak-diff2.s
deleted file mode 100644
index daf64a44232..00000000000
--- a/test/MC/ELF/weak-diff2.s
+++ /dev/null
@@ -1,10 +0,0 @@
-// RUN: not llvm-mc -filetype=obj -triple x86_64-pc-linux-gnu %s -o %t 2>&1 | FileCheck %s
-
-// CHECK: error: Cannot represent a subtraction with a weak symbol
-
-.weak f
-f:
-    nop
-g:
-    nop
-.quad g - f
diff --git a/test/MC/ELF/weakref-reloc.s b/test/MC/ELF/weakref-reloc.s
index 582c6946c60..baf80060c43 100644
--- a/test/MC/ELF/weakref-reloc.s
+++ b/test/MC/ELF/weakref-reloc.s
@@ -1,7 +1,6 @@
 // RUN: llvm-mc -filetype=obj -triple x86_64-pc-linux-gnu %s -o - | llvm-readobj -r -t | FileCheck %s
 
-// Test that the relocations point to the correct symbols. We used to get the
-// symbol index wrong for weakrefs when creating _GLOBAL_OFFSET_TABLE_.
+// Test that the relocations point to the correct symbols.
 
 	.weakref bar,foo
         call    zed@PLT
@@ -13,32 +12,3 @@
 // CHECK-NEXT:     0x6 R_X86_64_PC32 foo 0xFFFFFFFFFFFFFFFC
 // CHECK-NEXT:   }
 // CHECK-NEXT: ]
-
-// CHECK:      Symbols [
-// CHECK:        Symbol {
-// CHECK:          Name: _GLOBAL_OFFSET_TABLE_ (9)
-// CHECK-NEXT:     Value: 0x0
-// CHECK-NEXT:     Size: 0
-// CHECK-NEXT:     Binding: Global
-// CHECK-NEXT:     Type: None
-// CHECK-NEXT:     Other: 0
-// CHECK-NEXT:     Section: Undefined (0x0)
-// CHECK-NEXT:   }
-// CHECK-NEXT:   Symbol {
-// CHECK-NEXT:     Name: foo (1)
-// CHECK-NEXT:     Value: 0x0
-// CHECK-NEXT:     Size: 0
-// CHECK-NEXT:     Binding: Weak
-// CHECK-NEXT:     Type: None
-// CHECK-NEXT:     Other: 0
-// CHECK-NEXT:     Section: Undefined (0x0)
-// CHECK-NEXT:   }
-// CHECK-NEXT:   Symbol {
-// CHECK-NEXT:     Name: zed (5)
-// CHECK-NEXT:     Value: 0x0
-// CHECK-NEXT:     Size: 0
-// CHECK-NEXT:     Binding: Global
-// CHECK-NEXT:     Type: None
-// CHECK-NEXT:     Other: 0
-// CHECK-NEXT:     Section: Undefined (0x0)
-// CHECK-NEXT:   }
diff --git a/test/MC/ELF/weakref.s b/test/MC/ELF/weakref.s
index 9485e499c62..d342c61f745 100644
--- a/test/MC/ELF/weakref.s
+++ b/test/MC/ELF/weakref.s
@@ -86,7 +86,7 @@ bar15:
 // CHECK-NEXT:     Binding: Local
 // CHECK-NEXT:     Type: None
 // CHECK-NEXT:     Other: 0
-// CHECK-NEXT:     Section: .text (0x1)
+// CHECK-NEXT:     Section: .text
 // CHECK-NEXT:   }
 // CHECK-NEXT:   Symbol {
 // CHECK-NEXT:     Name: bar7
@@ -95,7 +95,7 @@ bar15:
 // CHECK-NEXT:     Binding: Local
 // CHECK-NEXT:     Type: None
 // CHECK-NEXT:     Other: 0
-// CHECK-NEXT:     Section: .text (0x1)
+// CHECK-NEXT:     Section: .text
 // CHECK-NEXT:   }
 // CHECK-NEXT:   Symbol {
 // CHECK-NEXT:     Name: bar8
@@ -104,7 +104,7 @@ bar15:
 // CHECK-NEXT:     Binding: Local
 // CHECK-NEXT:     Type: None
 // CHECK-NEXT:     Other: 0
-// CHECK-NEXT:     Section: .text (0x1)
+// CHECK-NEXT:     Section: .text
 // CHECK-NEXT:   }
 // CHECK-NEXT:   Symbol {
 // CHECK-NEXT:     Name: bar9
@@ -113,34 +113,16 @@ bar15:
 // CHECK-NEXT:     Binding: Local
 // CHECK-NEXT:     Type: None
 // CHECK-NEXT:     Other: 0
-// CHECK-NEXT:     Section: .text (0x1)
+// CHECK-NEXT:     Section: .text
 // CHECK-NEXT:   }
 // CHECK-NEXT:   Symbol {
-// CHECK-NEXT:     Name: .text
+// CHECK-NEXT:     Name: (0)
 // CHECK-NEXT:     Value: 0x0
 // CHECK-NEXT:     Size: 0
 // CHECK-NEXT:     Binding: Local
 // CHECK-NEXT:     Type: Section
 // CHECK-NEXT:     Other: 0
-// CHECK-NEXT:     Section: .text (0x1)
-// CHECK-NEXT:   }
-// CHECK-NEXT:   Symbol {
-// CHECK-NEXT:     Name: .data
-// CHECK-NEXT:     Value: 0x0
-// CHECK-NEXT:     Size: 0
-// CHECK-NEXT:     Binding: Local
-// CHECK-NEXT:     Type: Section
-// CHECK-NEXT:     Other: 0
-// CHECK-NEXT:     Section: .data
-// CHECK-NEXT:   }
-// CHECK-NEXT:   Symbol {
-// CHECK-NEXT:     Name: .bss
-// CHECK-NEXT:     Value: 0x0
-// CHECK-NEXT:     Size: 0
-// CHECK-NEXT:     Binding: Local
-// CHECK-NEXT:     Type: Section
-// CHECK-NEXT:     Other: 0
-// CHECK-NEXT:     Section: .bss
+// CHECK-NEXT:     Section: .text
 // CHECK-NEXT:   }
 // CHECK-NEXT:   Symbol {
 // CHECK-NEXT:     Name: bar10
@@ -149,7 +131,7 @@ bar15:
 // CHECK-NEXT:     Binding: Global
 // CHECK-NEXT:     Type: None
 // CHECK-NEXT:     Other: 0
-// CHECK-NEXT:     Section: .text (0x1)
+// CHECK-NEXT:     Section: .text
 // CHECK-NEXT:   }
 // CHECK-NEXT:   Symbol {
 // CHECK-NEXT:     Name: bar11
@@ -158,7 +140,7 @@ bar15:
 // CHECK-NEXT:     Binding: Global
 // CHECK-NEXT:     Type: None
 // CHECK-NEXT:     Other: 0
-// CHECK-NEXT:     Section: .text (0x1)
+// CHECK-NEXT:     Section: .text
 // CHECK-NEXT:   }
 // CHECK-NEXT:   Symbol {
 // CHECK-NEXT:     Name: bar12
@@ -167,7 +149,7 @@ bar15:
 // CHECK-NEXT:     Binding: Global
 // CHECK-NEXT:     Type: None
 // CHECK-NEXT:     Other: 0
-// CHECK-NEXT:     Section: .text (0x1)
+// CHECK-NEXT:     Section: .text
 // CHECK-NEXT:   }
 // CHECK-NEXT:   Symbol {
 // CHECK-NEXT:     Name: bar13
@@ -176,7 +158,7 @@ bar15:
 // CHECK-NEXT:     Binding: Global
 // CHECK-NEXT:     Type: None
 // CHECK-NEXT:     Other: 0
-// CHECK-NEXT:     Section: .text (0x1)
+// CHECK-NEXT:     Section: .text
 // CHECK-NEXT:   }
 // CHECK-NEXT:   Symbol {
 // CHECK-NEXT:     Name: bar14
@@ -185,7 +167,7 @@ bar15:
 // CHECK-NEXT:     Binding: Global
 // CHECK-NEXT:     Type: None
 // CHECK-NEXT:     Other: 0
-// CHECK-NEXT:     Section: .text (0x1)
+// CHECK-NEXT:     Section: .text
 // CHECK-NEXT:   }
 // CHECK-NEXT:   Symbol {
 // CHECK-NEXT:     Name: bar15
@@ -194,7 +176,7 @@ bar15:
 // CHECK-NEXT:     Binding: Global
 // CHECK-NEXT:     Type: None
 // CHECK-NEXT:     Other: 0
-// CHECK-NEXT:     Section: .text (0x1)
+// CHECK-NEXT:     Section: .text
 // CHECK-NEXT:   }
 // CHECK-NEXT:   Symbol {
 // CHECK-NEXT:     Name: bar2
diff --git a/test/MC/Hexagon/inst_add.ll b/test/MC/Hexagon/inst_add.ll
index 20a7b312ed6..826776952e0 100644
--- a/test/MC/Hexagon/inst_add.ll
+++ b/test/MC/Hexagon/inst_add.ll
@@ -1,5 +1,5 @@
-;; RUN: llc -mtriple=hexagon-unknown-elf -filetype=obj %s -o - \
-;; RUN: | llvm-objdump -s - | FileCheck %s
+; RUN: llc -march=hexagon -filetype=obj %s -o - \
+; RUN: | llvm-objdump -d - | FileCheck %s
 
 define i32 @foo (i32 %a, i32 %b)
 {
@@ -7,4 +7,4 @@ define i32 @foo (i32 %a, i32 %b)
   ret i32 %1
 }
 
-; CHECK:  0000 004100f3 00c09f52
+; CHECK:  c0 3f 10 58 58103fc0
diff --git a/test/MC/Hexagon/inst_cmp_eq.ll b/test/MC/Hexagon/inst_cmp_eq.ll
index 113db631fa9..98202368aff 100644
--- a/test/MC/Hexagon/inst_cmp_eq.ll
+++ b/test/MC/Hexagon/inst_cmp_eq.ll
@@ -1,5 +1,5 @@
 ;; RUN: llc -mtriple=hexagon-unknown-elf -filetype=obj %s -o - \
-;; RUN: | llvm-objdump -s - | FileCheck %s
+;; RUN: | llvm-objdump -d - | FileCheck %s
 
 define i1 @foo (i32 %a, i32 %b)
 {
@@ -7,4 +7,6 @@ define i1 @foo (i32 %a, i32 %b)
   ret i1 %1
 }
 
-; CHECK:  0000 004100f2 00404089 00c09f52
+; CHECK: p0 = cmp.eq(r0, r1)
+; CHECK: r0 = p0
+; CHECK: jumpr r31
diff --git a/test/MC/Hexagon/inst_cmp_eqi.ll b/test/MC/Hexagon/inst_cmp_eqi.ll
index 70c4c308e1f..612dfdc8f23 100644
--- a/test/MC/Hexagon/inst_cmp_eqi.ll
+++ b/test/MC/Hexagon/inst_cmp_eqi.ll
@@ -1,5 +1,5 @@
 ;; RUN: llc -mtriple=hexagon-unknown-elf -filetype=obj %s -o - \
-;; RUN: | llvm-objdump -s - | FileCheck %s
+;; RUN: | llvm-objdump -d - | FileCheck %s
 
 define i1 @foo (i32 %a)
 {
@@ -7,4 +7,6 @@ define i1 @foo (i32 %a)
   ret i1 %1
 }
 
-; CHECK:  0000 40450075 00404089 00c09f52
+; CHECK: p0 = cmp.eq(r0, #42)
+; CHECK: r0 = p0
+; CHECK: jumpr r31
diff --git a/test/MC/Hexagon/inst_cmp_gt.ll b/test/MC/Hexagon/inst_cmp_gt.ll
index 85fedbfb503..3ce1c0addad 100644
--- a/test/MC/Hexagon/inst_cmp_gt.ll
+++ b/test/MC/Hexagon/inst_cmp_gt.ll
@@ -1,5 +1,5 @@
 ;; RUN: llc -mtriple=hexagon-unknown-elf -filetype=obj %s -o - \
-;; RUN: | llvm-objdump -s - | FileCheck %s
+;; RUN: | llvm-objdump -d - | FileCheck %s
 
 define i1 @foo (i32 %a, i32 %b)
 {
@@ -7,4 +7,6 @@ define i1 @foo (i32 %a, i32 %b)
   ret i1 %1
 }
 
-; CHECK:  0000 004140f2 00404089 00c09f52
+; CHECK: p0 = cmp.gt(r0, r1)
+; CHECK: r0 = p0
+; CHECK: jumpr r31 }
\ No newline at end of file
diff --git a/test/MC/Hexagon/inst_cmp_gti.ll b/test/MC/Hexagon/inst_cmp_gti.ll
index 18ba3e463ef..f3c13a2fb96 100644
--- a/test/MC/Hexagon/inst_cmp_gti.ll
+++ b/test/MC/Hexagon/inst_cmp_gti.ll
@@ -1,5 +1,5 @@
 ;; RUN: llc -mtriple=hexagon-unknown-elf -filetype=obj %s -o - \
-;; RUN: | llvm-objdump -s - | FileCheck %s
+;; RUN: | llvm-objdump -d - | FileCheck %s
 
 define i1 @foo (i32 %a)
 {
@@ -7,4 +7,6 @@ define i1 @foo (i32 %a)
   ret i1 %1
 }
 
-; CHECK:  0000 40454075 00404089 00c09f52
+; CHECK: p0 = cmp.gt(r0, #42)
+; CHECK: r0 = p0
+; CHECK: jumpr r31
diff --git a/test/MC/Hexagon/inst_cmp_lt.ll b/test/MC/Hexagon/inst_cmp_lt.ll
index 3a7618421e3..80ba16f4141 100644
--- a/test/MC/Hexagon/inst_cmp_lt.ll
+++ b/test/MC/Hexagon/inst_cmp_lt.ll
@@ -1,5 +1,5 @@
 ;; RUN: llc -mtriple=hexagon-unknown-elf -filetype=obj %s -o - \
-;; RUN: | llvm-objdump -s - | FileCheck %s
+;; RUN: | llvm-objdump -d - | FileCheck %s
 
 define i1 @foo (i32 %a, i32 %b)
 {
@@ -7,4 +7,6 @@ define i1 @foo (i32 %a, i32 %b)
   ret i1 %1
 }
 
-; CHECK:  0000 004041f2 00404089 00c09f52
+; CHECK: p0 = cmp.gt(r1, r0)
+; CHECK: r0 = p0
+; CHECK: jumpr r31
diff --git a/test/MC/Hexagon/inst_cmp_ugt.ll b/test/MC/Hexagon/inst_cmp_ugt.ll
index 096536f54a7..07fa784dc64 100644
--- a/test/MC/Hexagon/inst_cmp_ugt.ll
+++ b/test/MC/Hexagon/inst_cmp_ugt.ll
@@ -1,5 +1,5 @@
 ;; RUN: llc -mtriple=hexagon-unknown-elf -filetype=obj %s -o - \
-;; RUN: | llvm-objdump -s - | FileCheck %s
+;; RUN: | llvm-objdump -d - | FileCheck %s
 
 define i1 @foo (i32 %a, i32 %b)
 {
@@ -7,4 +7,6 @@ define i1 @foo (i32 %a, i32 %b)
   ret i1 %1
 }
 
-; CHECK:  0000 004160f2 00404089 00c09f52
+; CHECK: p0 = cmp.gtu(r0, r1)
+; CHECK: r0 = p0
+; CHECK: jumpr r31
diff --git a/test/MC/Hexagon/inst_cmp_ugti.ll b/test/MC/Hexagon/inst_cmp_ugti.ll
index a83583457d1..59db552b39f 100644
--- a/test/MC/Hexagon/inst_cmp_ugti.ll
+++ b/test/MC/Hexagon/inst_cmp_ugti.ll
@@ -1,5 +1,5 @@
 ;; RUN: llc -mtriple=hexagon-unknown-elf -filetype=obj %s -o - \
-;; RUN: | llvm-objdump -s - | FileCheck %s
+;; RUN: | llvm-objdump -d - | FileCheck %s
 
 define i1 @foo (i32 %a)
 {
@@ -7,4 +7,6 @@ define i1 @foo (i32 %a)
   ret i1 %1
 }
 
-; CHECK:  0000 40458075 00404089 00c09f52
+; CHECK: p0 = cmp.gtu(r0, #42)
+; CHECK: r0 = p0
+; CHECK: jumpr r31
diff --git a/test/MC/Hexagon/inst_cmp_ult.ll b/test/MC/Hexagon/inst_cmp_ult.ll
index 4323fa0834d..c880ac8a229 100644
--- a/test/MC/Hexagon/inst_cmp_ult.ll
+++ b/test/MC/Hexagon/inst_cmp_ult.ll
@@ -1,5 +1,5 @@
 ;; RUN: llc -mtriple=hexagon-unknown-elf -filetype=obj %s -o - \
-;; RUN: | llvm-objdump -s - | FileCheck %s
+;; RUN: | llvm-objdump -d - | FileCheck %s
 
 define i1 @foo (i32 %a, i32 %b)
 {
@@ -7,4 +7,6 @@ define i1 @foo (i32 %a, i32 %b)
   ret i1 %1
 }
 
-; CHECK:  0000 004061f2 00404089 00c09f52
+; CHECK: p0 = cmp.gtu(r1, r0)
+; CHECK: r0 = p0
+; CHECK: jumpr r31
\ No newline at end of file
diff --git a/test/MC/Hexagon/inst_select.ll b/test/MC/Hexagon/inst_select.ll
index 29a2db0723b..9d12c1de73f 100644
--- a/test/MC/Hexagon/inst_select.ll
+++ b/test/MC/Hexagon/inst_select.ll
@@ -1,5 +1,5 @@
-;; RUN: llc -mtriple=hexagon-unknown-elf -filetype=obj %s -o - \
-;; RUN: | llvm-objdump -s - | FileCheck %s
+; RUN: llc -march=hexagon -filetype=obj %s -o - \
+; RUN: | llvm-objdump -d - | FileCheck %s
 
 define i32 @foo (i1 %a, i32 %b, i32 %c)
 {
@@ -7,4 +7,7 @@ define i32 @foo (i1 %a, i32 %b, i32 %c)
   ret i32 %1
 }
 
-; CHECK:  0000 00400085 00600174 00608274 00c09f52
+; CHECK: 00 40 00 85 85004000
+; CHECK: 00 40 9f 52 529f4000
+; CHECK: 00 60 01 74 74016000
+; CHECK: 00 e0 82 74 7482e000
\ No newline at end of file
diff --git a/test/MC/Hexagon/inst_sxtb.ll b/test/MC/Hexagon/inst_sxtb.ll
index 4a217420a96..34219c72c99 100644
--- a/test/MC/Hexagon/inst_sxtb.ll
+++ b/test/MC/Hexagon/inst_sxtb.ll
@@ -1,5 +1,5 @@
-;; RUN: llc -mtriple=hexagon-unknown-elf -filetype=obj %s -o - \
-;; RUN: | llvm-objdump -s - | FileCheck %s
+; RUN: llc -march=hexagon -filetype=obj %s -o - \
+; RUN: | llvm-objdump -d - | FileCheck %s
 
 define i32 @foo (i8 %a)
 {
@@ -7,4 +7,4 @@ define i32 @foo (i8 %a)
   ret i32 %1
 }
 
-; CHECK:   0000 0040a070 00c09f52
+; CHECK: c0 3f 00 55 55003fc0
diff --git a/test/MC/Hexagon/inst_sxth.ll b/test/MC/Hexagon/inst_sxth.ll
index f0bcf584efa..5d1223da8e0 100644
--- a/test/MC/Hexagon/inst_sxth.ll
+++ b/test/MC/Hexagon/inst_sxth.ll
@@ -1,5 +1,5 @@
-;; RUN: llc -mtriple=hexagon-unknown-elf -filetype=obj %s -o - \
-;; RUN: | llvm-objdump -s - | FileCheck %s
+; RUN: llc -mtriple=hexagon-unknown-elf -filetype=obj %s -o - \
+; RUN: | llvm-objdump -d - | FileCheck %s
 
 define i32 @foo (i16 %a)
 {
@@ -7,4 +7,4 @@ define i32 @foo (i16 %a)
   ret i32 %1
 }
 
-; CHECK:   0000 0040e070 00c09f52
+; CHECK: c0 3f 00 54 54003fc0
\ No newline at end of file
diff --git a/test/MC/Hexagon/inst_zxtb.ll b/test/MC/Hexagon/inst_zxtb.ll
index 622c03692b2..86da5e52cac 100644
--- a/test/MC/Hexagon/inst_zxtb.ll
+++ b/test/MC/Hexagon/inst_zxtb.ll
@@ -1,5 +1,5 @@
-;; RUN: llc -mtriple=hexagon-unknown-elf -filetype=obj %s -o - \
-;; RUN: | llvm-objdump -s - | FileCheck %s
+; RUN: llc -march=hexagon -filetype=obj %s -o - \
+; RUN: | llvm-objdump -d - | FileCheck %s
 
 define i32 @foo (i8 %a)
 {
@@ -7,4 +7,4 @@ define i32 @foo (i8 %a)
   ret i32 %1
 }
 
-; CHECK:   0000 e05f0076 00c09f52
+; CHECK: c0 3f 00 57 57003fc0
diff --git a/test/MC/Hexagon/inst_zxth.ll b/test/MC/Hexagon/inst_zxth.ll
index 962210b1751..7b3d1b30648 100644
--- a/test/MC/Hexagon/inst_zxth.ll
+++ b/test/MC/Hexagon/inst_zxth.ll
@@ -1,5 +1,5 @@
-;; RUN: llc -mtriple=hexagon-unknown-elf -filetype=obj %s -o - \
-;; RUN: | llvm-objdump -s - | FileCheck %s
+; RUN: llc -march=hexagon -filetype=obj %s -o - \
+; RUN: | llvm-objdump -d - | FileCheck %s
 
 define i32 @foo (i16 %a)
 {
@@ -7,4 +7,4 @@ define i32 @foo (i16 %a)
   ret i32 %1
 }
 
-; CHECK:   0000 0040c070 00c09f52
+; CHECK: c0 3f 00 56 56003fc0
diff --git a/test/MC/MachO/AArch64/cfstring.s b/test/MC/MachO/AArch64/cfstring.s
deleted file mode 100644
index 19b50678877..00000000000
--- a/test/MC/MachO/AArch64/cfstring.s
+++ /dev/null
@@ -1,24 +0,0 @@
-; RUN: llvm-mc -triple arm64-apple-darwin10 %s -filetype=obj -o - | llvm-readobj -r --expand-relocs | FileCheck %s
-
-; Test that we produce an external relocation. There is no apparent need for it, but
-; ld64 (241.9) produces a corrupt output if we don't.
-
-// CHECK:      Relocations [
-// CHECK-NEXT:   Section __data {
-// CHECK-NEXT:     Relocation {
-// CHECK-NEXT:       Offset: 0x0
-// CHECK-NEXT:       PCRel: 0
-// CHECK-NEXT:       Length: 3
-// CHECK-NEXT:       Extern: 1
-// CHECK-NEXT:       Type: ARM64_RELOC_UNSIGNED (0)
-// CHECK-NEXT:       Symbol: Lfoo
-// CHECK-NEXT:       Scattered: 0
-// CHECK-NEXT:     }
-// CHECK-NEXT:   }
-// CHECK-NEXT: ]
-
-        .section        __DATA,__cfstring
-Lfoo:
-
-        .section        __DATA,__data
-        .quad   Lfoo
diff --git a/test/MC/MachO/AArch64/classrefs.s b/test/MC/MachO/AArch64/classrefs.s
index 5edc82ca0b1..d92bbb4a57a 100644
--- a/test/MC/MachO/AArch64/classrefs.s
+++ b/test/MC/MachO/AArch64/classrefs.s
@@ -9,10 +9,8 @@
 // CHECK-NEXT:       Offset: 0x0
 // CHECK-NEXT:       PCRel: 0
 // CHECK-NEXT:       Length: 3
-// CHECK-NEXT:       Extern: 1
 // CHECK-NEXT:       Type: ARM64_RELOC_UNSIGNED (0)
 // CHECK-NEXT:       Symbol: Lbar
-// CHECK-NEXT:       Scattered: 0
 // CHECK-NEXT:     }
 // CHECK-NEXT:   }
 // CHECK-NEXT: ]
diff --git a/test/MC/MachO/AArch64/darwin-ARM64-reloc.s b/test/MC/MachO/AArch64/darwin-ARM64-reloc.s
index 07d52528e91..b4d0b082c8a 100644
--- a/test/MC/MachO/AArch64/darwin-ARM64-reloc.s
+++ b/test/MC/MachO/AArch64/darwin-ARM64-reloc.s
@@ -40,127 +40,99 @@ L_.str:
 ; CHECK-NEXT:       Offset: 0x24
 ; CHECK-NEXT:       PCRel: 1
 ; CHECK-NEXT:       Length: 2
-; CHECK-NEXT:       Extern: 1
 ; CHECK-NEXT:       Type: ARM64_RELOC_PAGE21 (3)
 ; CHECK-NEXT:       Symbol: L_.str
-; CHECK-NEXT:       Scattered: 0
 ; CHECK-NEXT:     }
 ; CHECK-NEXT:    Relocation {
 ; CHECK-NEXT:       Offset: 0x20
 ; CHECK-NEXT:       PCRel: 0
 ; CHECK-NEXT:       Length: 2
-; CHECK-NEXT:       Extern: 1
 ; CHECK-NEXT:       Type: ARM64_RELOC_GOT_LOAD_PAGEOFF12 (6)
 ; CHECK-NEXT:       Symbol: _data_ext
-; CHECK-NEXT:       Scattered: 0
 ; CHECK-NEXT:     }
 ; CHECK-NEXT:     Relocation {
 ; CHECK-NEXT:       Offset: 0x1C
 ; CHECK-NEXT:       PCRel: 1
 ; CHECK-NEXT:       Length: 2
-; CHECK-NEXT:       Extern: 1
 ; CHECK-NEXT:       Type: ARM64_RELOC_GOT_LOAD_PAGE21 (5)
 ; CHECK-NEXT:       Symbol: _data_ext
-; CHECK-NEXT:       Scattered: 0
 ; CHECK-NEXT:     }
 ; CHECK-NEXT:     Relocation {
 ; CHECK-NEXT:       Offset: 0x18
 ; CHECK-NEXT:       PCRel: 0
 ; CHECK-NEXT:       Length: 2
-; CHECK-NEXT:       Extern: 0
 ; CHECK-NEXT:       Type: ARM64_RELOC_ADDEND (10)
-; CHECK-NEXT:       Symbol: 0x4
-; CHECK-NEXT:       Scattered: 0
+; CHECK-NEXT:       Section: - (4)
 ; CHECK-NEXT:     }
 ; CHECK-NEXT:     Relocation {
 ; CHECK-NEXT:       Offset: 0x18
 ; CHECK-NEXT:       PCRel: 0
 ; CHECK-NEXT:       Length: 2
-; CHECK-NEXT:       Extern: 1
 ; CHECK-NEXT:       Type: ARM64_RELOC_PAGEOFF12 (4)
 ; CHECK-NEXT:       Symbol: _data
-; CHECK-NEXT:       Scattered: 0
 ; CHECK-NEXT:     }
 ; CHECK-NEXT:     Relocation {
 ; CHECK-NEXT:       Offset: 0x14
 ; CHECK-NEXT:       PCRel: 0
 ; CHECK-NEXT:       Length: 2
-; CHECK-NEXT:       Extern: 0
 ; CHECK-NEXT:       Type: ARM64_RELOC_ADDEND (10)
-; CHECK-NEXT:       Symbol: 0x1
-; CHECK-NEXT:       Scattered: 0
+; CHECK-NEXT:       Section: __text
 ; CHECK-NEXT:     }
 ; CHECK-NEXT:     Relocation {
 ; CHECK-NEXT:       Offset: 0x14
 ; CHECK-NEXT:       PCRel: 1
 ; CHECK-NEXT:       Length: 2
-; CHECK-NEXT:       Extern: 1
 ; CHECK-NEXT:       Type: ARM64_RELOC_PAGE21 (3)
 ; CHECK-NEXT:       Symbol: _data
-; CHECK-NEXT:       Scattered: 0
 ; CHECK-NEXT:     }
 ; CHECK-NEXT:     Relocation {
 ; CHECK-NEXT:       Offset: 0x10
 ; CHECK-NEXT:       PCRel: 0
 ; CHECK-NEXT:       Length: 2
-; CHECK-NEXT:       Extern: 0
 ; CHECK-NEXT:       Type: ARM64_RELOC_ADDEND (10)
-; CHECK-NEXT:       Symbol: 0x4
-; CHECK-NEXT:       Scattered: 0
+; CHECK-NEXT:       Section: - (4)
 ; CHECK-NEXT:     }
 ; CHECK-NEXT:     Relocation {
 ; CHECK-NEXT:       Offset: 0x10
 ; CHECK-NEXT:       PCRel: 0
 ; CHECK-NEXT:       Length: 2
-; CHECK-NEXT:       Extern: 1
 ; CHECK-NEXT:       Type: ARM64_RELOC_PAGEOFF12 (4)
 ; CHECK-NEXT:       Symbol: _data
-; CHECK-NEXT:       Scattered: 0
 ; CHECK-NEXT:     }
 ; CHECK-NEXT:     Relocation {
 ; CHECK-NEXT:       Offset: 0xC
 ; CHECK-NEXT:       PCRel: 0
 ; CHECK-NEXT:       Length: 2
-; CHECK-NEXT:       Extern: 1
 ; CHECK-NEXT:       Type: ARM64_RELOC_PAGEOFF12 (4)
 ; CHECK-NEXT:       Symbol: _data
-; CHECK-NEXT:       Scattered: 0
 ; CHECK-NEXT:     }
 ; CHECK-NEXT:     Relocation {
 ; CHECK-NEXT:       Offset: 0x8
 ; CHECK-NEXT:       PCRel: 1
 ; CHECK-NEXT:       Length: 2
-; CHECK-NEXT:       Extern: 1
 ; CHECK-NEXT:       Type: ARM64_RELOC_PAGE21 (3)
 ; CHECK-NEXT:       Symbol: _data
-; CHECK-NEXT:       Scattered: 0
 ; CHECK-NEXT:     }
 ; CHECK-NEXT:     Relocation {
 ; CHECK-NEXT:       Offset: 0x4
 ; CHECK-NEXT:       PCRel: 0
 ; CHECK-NEXT:       Length: 2
-; CHECK-NEXT:       Extern: 0
 ; CHECK-NEXT:       Type: ARM64_RELOC_ADDEND (10)
-; CHECK-NEXT:       Symbol: 0x14
-; CHECK-NEXT:       Scattered: 0
+; CHECK-NEXT:       Section: - (20)
 ; CHECK-NEXT:     }
 ; CHECK-NEXT:     Relocation {
 ; CHECK-NEXT:       Offset: 0x4
 ; CHECK-NEXT:       PCRel: 1
 ; CHECK-NEXT:       Length: 2
-; CHECK-NEXT:       Extern: 1
 ; CHECK-NEXT:       Type: ARM64_RELOC_BRANCH26 (2)
 ; CHECK-NEXT:       Symbol: _func
-; CHECK-NEXT:       Scattered: 0
 ; CHECK-NEXT:     }
 ; CHECK-NEXT:     Relocation {
 ; CHECK-NEXT:       Offset: 0x0
 ; CHECK-NEXT:       PCRel: 1
 ; CHECK-NEXT:       Length: 2
-; CHECK-NEXT:       Extern: 1
 ; CHECK-NEXT:       Type: ARM64_RELOC_BRANCH26 (2)
 ; CHECK-NEXT:       Symbol: _func
-; CHECK-NEXT:       Scattered: 0
 ; CHECK-NEXT:     }
 ; CHECK-NEXT:   }
 ; CHECK-NEXT:   Section __data {
@@ -168,91 +140,71 @@ L_.str:
 ; CHECK-NEXT:       Offset: 0x2C
 ; CHECK-NEXT:       PCRel: 1
 ; CHECK-NEXT:       Length: 2
-; CHECK-NEXT:       Extern: 1
 ; CHECK-NEXT:       Type: ARM64_RELOC_POINTER_TO_GOT (7)
 ; CHECK-NEXT:       Symbol: _foo
-; CHECK-NEXT:       Scattered: 0
 ; CHECK-NEXT:     }
 ; CHECK-NEXT:     Relocation {
 ; CHECK-NEXT:       Offset: 0x24
 ; CHECK-NEXT:       PCRel: 0
 ; CHECK-NEXT:       Length: 3
-; CHECK-NEXT:       Extern: 1
 ; CHECK-NEXT:       Type: ARM64_RELOC_POINTER_TO_GOT (7)
 ; CHECK-NEXT:       Symbol: _foo
-; CHECK-NEXT:       Scattered: 0
 ; CHECK-NEXT:     }
 ; CHECK-NEXT:     Relocation {
 ; CHECK-NEXT:       Offset: 0x20
 ; CHECK-NEXT:       PCRel: 0
 ; CHECK-NEXT:       Length: 2
-; CHECK-NEXT:       Extern: 1
 ; CHECK-NEXT:       Type: ARM64_RELOC_SUBTRACTOR (1)
 ; CHECK-NEXT:       Symbol: _bar
-; CHECK-NEXT:       Scattered: 0
 ; CHECK-NEXT:     }
 ; CHECK-NEXT:     Relocation {
 ; CHECK-NEXT:       Offset: 0x20
 ; CHECK-NEXT:       PCRel: 0
 ; CHECK-NEXT:       Length: 2
-; CHECK-NEXT:       Extern: 1
 ; CHECK-NEXT:       Type: ARM64_RELOC_UNSIGNED (0)
 ; CHECK-NEXT:       Symbol: _foo
-; CHECK-NEXT:       Scattered: 0
 ; CHECK-NEXT:     }
 ; CHECK-NEXT:     Relocation {
 ; CHECK-NEXT:       Offset: 0x18
 ; CHECK-NEXT:       PCRel: 0
 ; CHECK-NEXT:       Length: 3
-; CHECK-NEXT:       Extern: 1
 ; CHECK-NEXT:       Type: ARM64_RELOC_SUBTRACTOR (1)
 ; CHECK-NEXT:       Symbol: _bar
-; CHECK-NEXT:       Scattered: 0
 ; CHECK-NEXT:     }
 ; CHECK-NEXT:     Relocation {
 ; CHECK-NEXT:       Offset: 0x18
 ; CHECK-NEXT:       PCRel: 0
 ; CHECK-NEXT:       Length: 3
-; CHECK-NEXT:       Extern: 1
 ; CHECK-NEXT:       Type: ARM64_RELOC_UNSIGNED (0)
 ; CHECK-NEXT:       Symbol: _foo
-; CHECK-NEXT:       Scattered: 0
 ; CHECK-NEXT:     }
 ; CHECK-NEXT:     Relocation {
 ; CHECK-NEXT:       Offset: 0x10
 ; CHECK-NEXT:       PCRel: 0
 ; CHECK-NEXT:       Length: 3
-; CHECK-NEXT:       Extern: 1
 ; CHECK-NEXT:       Type: ARM64_RELOC_SUBTRACTOR (1)
 ; CHECK-NEXT:       Symbol: _bar
-; CHECK-NEXT:       Scattered: 0
 ; CHECK-NEXT:     }
 ; CHECK-NEXT:     Relocation {
 ; CHECK-NEXT:       Offset: 0x10
 ; CHECK-NEXT:       PCRel: 0
 ; CHECK-NEXT:       Length: 3
-; CHECK-NEXT:       Extern: 1
 ; CHECK-NEXT:       Type: ARM64_RELOC_UNSIGNED (0)
 ; CHECK-NEXT:       Symbol: _foo
-; CHECK-NEXT:       Scattered: 0
 ; CHECK-NEXT:     }
 ; CHECK-NEXT:     Relocation {
 ; CHECK-NEXT:       Offset: 0x8
 ; CHECK-NEXT:       PCRel: 0
 ; CHECK-NEXT:       Length: 3
-; CHECK-NEXT:       Extern: 1
 ; CHECK-NEXT:       Type: ARM64_RELOC_UNSIGNED (0)
 ; CHECK-NEXT:       Symbol: _foo
-; CHECK-NEXT:       Scattered: 0
 ; CHECK-NEXT:     }
 ; CHECK-NEXT:     Relocation {
 ; CHECK-NEXT:       Offset: 0x0
 ; CHECK-NEXT:       PCRel: 0
 ; CHECK-NEXT:       Length: 3
-; CHECK-NEXT:       Extern: 1
 ; CHECK-NEXT:       Type: ARM64_RELOC_UNSIGNED (0)
 ; CHECK-NEXT:       Symbol: _foo
-; CHECK-NEXT:       Scattered: 0
 ; CHECK-NEXT:     }
 ; CHECK-NEXT:   }
 ; CHECK-NEXT: ]
diff --git a/test/MC/MachO/AArch64/ld64-workaround.s b/test/MC/MachO/AArch64/ld64-workaround.s
new file mode 100644
index 00000000000..63f91705599
--- /dev/null
+++ b/test/MC/MachO/AArch64/ld64-workaround.s
@@ -0,0 +1,60 @@
+; RUN: llvm-mc -triple arm64-apple-darwin10 %s -filetype=obj -o - | llvm-readobj -r --expand-relocs | FileCheck %s
+
+; Test that we produce an external relocation. This is a known and temporary bug
+; in ld64, where it mishandles pointer-sized internal relocations. We should be
+; able to remove this entirely soon.
+
+// CHECK:      Relocations [
+// CHECK-NEXT:   Section __data {
+// CHECK-NEXT:     Relocation {
+// CHECK-NEXT:       Offset: 0x18
+// CHECK-NEXT:       PCRel: 0
+// CHECK-NEXT:       Length: 3
+// CHECK-NEXT:       Type: ARM64_RELOC_UNSIGNED (0)
+// CHECK-NEXT:       Symbol: Llit16
+// CHECK-NEXT:     }
+// CHECK-NEXT:     Relocation {
+// CHECK-NEXT:       Offset: 0x10
+// CHECK-NEXT:       PCRel: 0
+// CHECK-NEXT:       Length: 3
+// CHECK-NEXT:       Type: ARM64_RELOC_UNSIGNED (0)
+// CHECK-NEXT:       Symbol: Llit8
+// CHECK-NEXT:     }
+// CHECK-NEXT:     Relocation {
+// CHECK-NEXT:       Offset: 0x8
+// CHECK-NEXT:       PCRel: 0
+// CHECK-NEXT:       Length: 3
+// CHECK-NEXT:       Type: ARM64_RELOC_UNSIGNED (0)
+// CHECK-NEXT:       Symbol: Llit4
+// CHECK-NEXT:     }
+// CHECK-NEXT:     Relocation {
+// CHECK-NEXT:       Offset: 0x0
+// CHECK-NEXT:       PCRel: 0
+// CHECK-NEXT:       Length: 3
+// CHECK-NEXT:       Type: ARM64_RELOC_UNSIGNED (0)
+// CHECK-NEXT:       Symbol: Lcfstring
+// CHECK-NEXT:     }
+// CHECK-NEXT:   }
+// CHECK-NEXT: ]
+
+        .section        __DATA,__cfstring
+Lcfstring:
+
+        .section        __DATA,__literal4,4byte_literals
+Llit4:
+        .word 42
+
+        .section        __DATA,__literal8,8byte_literals
+Llit8:
+        .quad 42
+
+        .section        __DATA,__literal16,16byte_literals
+Llit16:
+        .quad 42
+        .quad 42
+
+        .section        __DATA,__data
+        .quad   Lcfstring
+        .quad   Llit4
+        .quad   Llit8
+        .quad   Llit16
diff --git a/test/MC/MachO/AArch64/mergeable.s b/test/MC/MachO/AArch64/mergeable.s
index fc6ec04f37b..4fed04c9334 100644
--- a/test/MC/MachO/AArch64/mergeable.s
+++ b/test/MC/MachO/AArch64/mergeable.s
@@ -1,4 +1,6 @@
 // RUN: llvm-mc -triple aarch64-apple-darwin14 %s -filetype=obj -o - | llvm-readobj -r --expand-relocs | FileCheck %s
+// FIXME: the final relocation should be internal, but the linker doesn't
+// currently handle the it correctly.
 
 // Test that we "S + K" produce a relocation with a symbol, but just S produces
 // a relocation with the section.
@@ -23,37 +25,29 @@ L1:
 // CHECK-NEXT:       Offset: 0x18
 // CHECK-NEXT:       PCRel: 0
 // CHECK-NEXT:       Length: 3
-// CHECK-NEXT:       Extern: 1
 // CHECK-NEXT:       Type: ARM64_RELOC_UNSIGNED (0)
 // CHECK-NEXT:       Symbol: L1
-// CHECK-NEXT:       Scattered: 0
 // CHECK-NEXT:     }
 // CHECK-NEXT:     Relocation {
 // CHECK-NEXT:       Offset: 0x10
 // CHECK-NEXT:       PCRel: 0
 // CHECK-NEXT:       Length: 3
-// CHECK-NEXT:       Extern: 1
 // CHECK-NEXT:       Type: ARM64_RELOC_UNSIGNED (0)
 // CHECK-NEXT:       Symbol: L1
-// CHECK-NEXT:       Scattered: 0
 // CHECK-NEXT:     }
 // CHECK-NEXT:     Relocation {
 // CHECK-NEXT:       Offset: 0x8
 // CHECK-NEXT:       PCRel: 0
 // CHECK-NEXT:       Length: 3
-// CHECK-NEXT:       Extern: 1
 // CHECK-NEXT:       Type: ARM64_RELOC_UNSIGNED (0)
 // CHECK-NEXT:       Symbol: L0
-// CHECK-NEXT:       Scattered: 0
 // CHECK-NEXT:     }
 // CHECK-NEXT:     Relocation {
 // CHECK-NEXT:       Offset: 0x0
 // CHECK-NEXT:       PCRel: 0
 // CHECK-NEXT:       Length: 3
-// CHECK-NEXT:       Extern: 0
 // CHECK-NEXT:       Type: ARM64_RELOC_UNSIGNED (0)
-// CHECK-NEXT:       Symbol: 0x2
-// CHECK-NEXT:       Scattered: 0
+// CHECK-NEXT:       Symbol: L0
 // CHECK-NEXT:     }
 // CHECK-NEXT:   }
 // CHECK-NEXT: ]
diff --git a/test/MC/MachO/AArch64/reloc-crash.s b/test/MC/MachO/AArch64/reloc-crash.s
index 4984947f65b..f8ad4c4f751 100644
--- a/test/MC/MachO/AArch64/reloc-crash.s
+++ b/test/MC/MachO/AArch64/reloc-crash.s
@@ -9,10 +9,8 @@
 ; CHECK-NEXT:       Offset: 0x0
 ; CHECK-NEXT:       PCRel: 0
 ; CHECK-NEXT:       Length: 3
-; CHECK-NEXT:       Extern: 1
 ; CHECK-NEXT:       Type: ARM64_RELOC_UNSIGNED (0)
 ; CHECK-NEXT:       Symbol: Lbar
-; CHECK-NEXT:       Scattered: 0
 ; CHECK-NEXT:     }
 ; CHECK-NEXT:   }
 ; CHECK-NEXT: ]
diff --git a/test/MC/MachO/AArch64/reloc-crash2.s b/test/MC/MachO/AArch64/reloc-crash2.s
index 6ae44715c63..3aa26281bc0 100644
--- a/test/MC/MachO/AArch64/reloc-crash2.s
+++ b/test/MC/MachO/AArch64/reloc-crash2.s
@@ -8,10 +8,8 @@
 ; CHECK-NEXT:       Offset: 0x0
 ; CHECK-NEXT:       PCRel: 0
 ; CHECK-NEXT:       Length: 2
-; CHECK-NEXT:       Extern: 1
 ; CHECK-NEXT:       Type: ARM64_RELOC_PAGEOFF12 (4)
 ; CHECK-NEXT:       Symbol: ltmp1
-; CHECK-NEXT:       Scattered: 0
 ; CHECK-NEXT:     }
 ; CHECK-NEXT:   }
 ; CHECK-NEXT: ]
diff --git a/test/MC/MachO/ARM/directive-type-diagnostics.s b/test/MC/MachO/ARM/directive-type-diagnostics.s
new file mode 100644
index 00000000000..f5f9b45b664
--- /dev/null
+++ b/test/MC/MachO/ARM/directive-type-diagnostics.s
@@ -0,0 +1,10 @@
+// RUN: not llvm-mc -triple arm-apple -filetype asm -o /dev/null %s 2>&1 | FileCheck %s
+// RUN: not llvm-mc -triple armeb-apple -filetype asm -o /dev/null %s 2>&1 | FileCheck %s
+// RUN: not llvm-mc -triple thumb-apple -filetype asm -o /dev/null %s 2>&1 | FileCheck %s
+// RUN: not llvm-mc -triple thumbeb-apple -filetype asm -o /dev/null %s 2>&1 | FileCheck %s
+
+        .type symbol 32
+// CHECK: error: expected STT_<TYPE_IN_UPPER_CASE>, '#<type>', '%<type>' or "<type>"
+// CHECK: .type symbol 32
+// CHECK:              ^
+
diff --git a/test/MC/MachO/ARM/nop-armv4-padding.s b/test/MC/MachO/ARM/nop-armv4-padding.s
index f704d801a60..8e03d17a70c 100644
--- a/test/MC/MachO/ARM/nop-armv4-padding.s
+++ b/test/MC/MachO/ARM/nop-armv4-padding.s
@@ -7,5 +7,4 @@ x:
       .align 4
       add r0, r1, r2
 
-; @LOCALMOD - Changed the NOP encoding
 @ CHECK: ('_section_data', '020081e0 0000a0e1 0000a0e1 0000a0e1 020081e0')
diff --git a/test/MC/MachO/ARM/static-movt-relocs.s b/test/MC/MachO/ARM/static-movt-relocs.s
index 4385549035e..d94be2f4f6c 100644
--- a/test/MC/MachO/ARM/static-movt-relocs.s
+++ b/test/MC/MachO/ARM/static-movt-relocs.s
@@ -12,37 +12,29 @@ foo:
 @ CHECK-NEXT:       Offset: 0x4
 @ CHECK-NEXT:       PCRel: 0
 @ CHECK-NEXT:       Length: 3
-@ CHECK-NEXT:       Extern: 1
 @ CHECK-NEXT:       Type: ARM_RELOC_HALF (8)
 @ CHECK-NEXT:       Symbol: bar
-@ CHECK-NEXT:       Scattered: 0
 @ CHECK-NEXT:     }
 @ CHECK-NEXT:     Relocation {
 @ CHECK-NEXT:       Offset: 0x10
 @ CHECK-NEXT:       PCRel: 0
 @ CHECK-NEXT:       Length: 3
-@ CHECK-NEXT:       Extern: 0
 @ CHECK-NEXT:       Type: ARM_RELOC_PAIR (1)
-@ CHECK-NEXT:       Symbol: 0xFFFFFF
-@ CHECK-NEXT:       Scattered: 0
+@ CHECK-NEXT:       Section: -
 @ CHECK-NEXT:     }
 @ CHECK-NEXT:     Relocation {
 @ CHECK-NEXT:       Offset: 0x0
 @ CHECK-NEXT:       PCRel: 0
 @ CHECK-NEXT:       Length: 2
-@ CHECK-NEXT:       Extern: 1
 @ CHECK-NEXT:       Type: ARM_RELOC_HALF (8)
 @ CHECK-NEXT:       Symbol: bar
-@ CHECK-NEXT:       Scattered: 0
 @ CHECK-NEXT:     }
 @ CHECK-NEXT:     Relocation {
 @ CHECK-NEXT:       Offset: 0x0
 @ CHECK-NEXT:       PCRel: 0
 @ CHECK-NEXT:       Length: 2
-@ CHECK-NEXT:       Extern: 0
 @ CHECK-NEXT:       Type: ARM_RELOC_PAIR (1)
-@ CHECK-NEXT:       Symbol: 0xFFFFFF
-@ CHECK-NEXT:       Scattered: 0
+@ CHECK-NEXT:       Section: -
 @ CHECK-NEXT:     }
 @ CHECK-NEXT:   }
 @ CHECK-NEXT: ]
diff --git a/test/MC/MachO/absolutize.s b/test/MC/MachO/absolutize.s
index 19917e3bae1..8947c0f65e5 100644
--- a/test/MC/MachO/absolutize.s
+++ b/test/MC/MachO/absolutize.s
@@ -6,9 +6,9 @@ _text_b:
         xorl %eax,%eax
 Ltext_c:
         xorl %eax,%eax
-Ltext_d:        
+Ltext_d:
         xorl %eax,%eax
-        
+
         movl $(_text_a - _text_b), %eax
 Ltext_expr_0 = _text_a - _text_b
         movl $(Ltext_expr_0), %eax
@@ -30,9 +30,9 @@ _data_b:
         .long 0
 Ldata_c:
         .long 0
-Ldata_d:        
+Ldata_d:
         .long 0
-        
+
         .long _data_a - _data_b
 Ldata_expr_0 = _data_a - _data_b
         .long Ldata_expr_0
@@ -75,7 +75,7 @@ Ldata_expr_2 = Ldata_d - Ldata_c
 // CHECK:     ('offset', 324)
 // CHECK:     ('alignment', 0)
 // CHECK:     ('reloc_offset', 412)
-// CHECK:     ('num_reloc', 7)
+// CHECK:     ('num_reloc', 3)
 // CHECK:     ('flags', 0x80000400)
 // CHECK:     ('reserved1', 0)
 // CHECK:     ('reserved2', 0)
@@ -85,21 +85,9 @@ Ldata_expr_2 = Ldata_d - Ldata_c
 // CHECK:     (('word-0', 0xa0000027),
 // CHECK:      ('word-1', 0x0)),
 // CHECK:     # Relocation 1
-// CHECK:     (('word-0', 0xa400001d),
-// CHECK:      ('word-1', 0x6)),
-// CHECK:     # Relocation 2
-// CHECK:     (('word-0', 0xa1000000),
-// CHECK:      ('word-1', 0x4)),
-// CHECK:     # Relocation 3
-// CHECK:     (('word-0', 0xa4000013),
-// CHECK:      ('word-1', 0x4)),
-// CHECK:     # Relocation 4
-// CHECK:     (('word-0', 0xa1000000),
-// CHECK:      ('word-1', 0x2)),
-// CHECK:     # Relocation 5
 // CHECK:     (('word-0', 0xa4000009),
 // CHECK:      ('word-1', 0x0)),
-// CHECK:     # Relocation 6
+// CHECK:     # Relocation 2
 // CHECK:     (('word-0', 0xa1000000),
 // CHECK:      ('word-1', 0x2)),
 // CHECK:   ])
@@ -111,8 +99,8 @@ Ldata_expr_2 = Ldata_d - Ldata_c
 // CHECK:     ('size', 44)
 // CHECK:     ('offset', 367)
 // CHECK:     ('alignment', 0)
-// CHECK:     ('reloc_offset', 468)
-// CHECK:     ('num_reloc', 7)
+// CHECK:     ('reloc_offset', 436)
+// CHECK:     ('num_reloc', 3)
 // CHECK:     ('flags', 0x0)
 // CHECK:     ('reserved1', 0)
 // CHECK:     ('reserved2', 0)
@@ -122,21 +110,9 @@ Ldata_expr_2 = Ldata_d - Ldata_c
 // CHECK:     (('word-0', 0xa0000028),
 // CHECK:      ('word-1', 0x2b)),
 // CHECK:     # Relocation 1
-// CHECK:     (('word-0', 0xa4000020),
-// CHECK:      ('word-1', 0x37)),
-// CHECK:     # Relocation 2
-// CHECK:     (('word-0', 0xa1000000),
-// CHECK:      ('word-1', 0x33)),
-// CHECK:     # Relocation 3
-// CHECK:     (('word-0', 0xa4000018),
-// CHECK:      ('word-1', 0x33)),
-// CHECK:     # Relocation 4
-// CHECK:     (('word-0', 0xa1000000),
-// CHECK:      ('word-1', 0x2f)),
-// CHECK:     # Relocation 5
 // CHECK:     (('word-0', 0xa4000010),
 // CHECK:      ('word-1', 0x2b)),
-// CHECK:     # Relocation 6
+// CHECK:     # Relocation 2
 // CHECK:     (('word-0', 0xa1000000),
 // CHECK:      ('word-1', 0x2f)),
 // CHECK:   ])
@@ -146,9 +122,9 @@ Ldata_expr_2 = Ldata_d - Ldata_c
 // CHECK:   # Load Command 1
 // CHECK:  (('command', 2)
 // CHECK:   ('size', 24)
-// CHECK:   ('symoff', 524)
+// CHECK:   ('symoff', 460)
 // CHECK:   ('nsyms', 4)
-// CHECK:   ('stroff', 572)
+// CHECK:   ('stroff', 508)
 // CHECK:   ('strsize', 36)
 // CHECK:   ('_string_data', '\x00_text_b\x00_data_b\x00_text_a\x00_data_a\x00\x00\x00\x00')
 // CHECK:   ('_symbols', [
diff --git a/test/MC/MachO/cstexpr-gotpcrel-64.ll b/test/MC/MachO/cstexpr-gotpcrel-64.ll
index bf155647f12..41abeb0179c 100644
--- a/test/MC/MachO/cstexpr-gotpcrel-64.ll
+++ b/test/MC/MachO/cstexpr-gotpcrel-64.ll
@@ -1,6 +1,7 @@
 ; RUN: llc -mtriple=x86_64-apple-darwin %s -o %t
 ; RUN:  FileCheck %s -check-prefix=X86 < %t
 ; RUN:  FileCheck %s -check-prefix=X86-GOT-EQUIV < %t
+; RUN:  FileCheck %s -check-prefix=X86-NOGOT-EQUIV < %t
 
 ; GOT equivalent globals references can be replaced by the GOT entry of the
 ; final symbol instead.
@@ -84,3 +85,17 @@ define i32 @t0(i32 %a) {
 define i32** @t1() {
   ret i32** @bargotequiv
 }
+
+; Do not crash when a pattern cannot be matched as a GOT equivalent
+define void @foo() {
+; X86-NOGOT-EQUIV-LABEL: _foo:
+; X86-NOGOT-EQUIV: leaq  _b(%rip), %rax
+  store i8** @b, i8*** null
+  ret void
+}
+@a = external global i8
+@b = internal unnamed_addr constant i8* @a
+
+; X86-NOGOT-EQUIV-LABEL: _c:
+; X86-NOGOT-EQUIV:   .quad _b
+@c = global i8** @b
diff --git a/test/MC/MachO/darwin-x86_64-diff-reloc-assign-2.s b/test/MC/MachO/darwin-x86_64-diff-reloc-assign-2.s
index b69cd1b1710..e3fa1335a0c 100644
--- a/test/MC/MachO/darwin-x86_64-diff-reloc-assign-2.s
+++ b/test/MC/MachO/darwin-x86_64-diff-reloc-assign-2.s
@@ -1,4 +1,4 @@
-// RUN: llvm-mc -triple x86_64-apple-darwin9 %s -filetype=obj -o - | llvm-readobj -r | FileCheck %s
+// RUN: llvm-mc -triple x86_64-apple-darwin9 %s -filetype=obj -o - | llvm-readobj -r --expand-relocs | FileCheck %s
 
 	.data
 L_var1:
@@ -10,7 +10,19 @@ L_var2:
 
 // CHECK:      Relocations [
 // CHECK-NEXT:   Section __data {
-// CHECK-NEXT:     0x4 0 2 0 X86_64_RELOC_SUBTRACTOR 0 0x2
-// CHECK-NEXT:     0x4 0 2 0 X86_64_RELOC_UNSIGNED 0 0x2
+// CHECK-NEXT:     Relocation {
+// CHECK-NEXT:       Offset: 0x4
+// CHECK-NEXT:       PCRel: 0
+// CHECK-NEXT:       Length: 2
+// CHECK-NEXT:       Type: X86_64_RELOC_SUBTRACTOR (5)
+// CHECK-NEXT:       Section: __data (2)
+// CHECK-NEXT:     }
+// CHECK-NEXT:     Relocation {
+// CHECK-NEXT:       Offset: 0x4
+// CHECK-NEXT:       PCRel: 0
+// CHECK-NEXT:       Length: 2
+// CHECK-NEXT:       Type: X86_64_RELOC_UNSIGNED (0)
+// CHECK-NEXT:       Section: __data (2)
+// CHECK-NEXT:     }
 // CHECK-NEXT:   }
 // CHECK-NEXT: ]
diff --git a/test/MC/MachO/darwin-x86_64-reloc.s b/test/MC/MachO/darwin-x86_64-reloc.s
index 48dd6b4b229..32e079879ea 100644
--- a/test/MC/MachO/darwin-x86_64-reloc.s
+++ b/test/MC/MachO/darwin-x86_64-reloc.s
@@ -93,46 +93,36 @@ L6:
 // CHECK-NEXT:       Offset: 0x20
 // CHECK-NEXT:       PCRel: 0
 // CHECK-NEXT:       Length: 3
-// CHECK-NEXT:       Extern: 0
 // CHECK-NEXT:       Type: X86_64_RELOC_UNSIGNED (0)
-// CHECK-NEXT:       Symbol: 0x4
-// CHECK-NEXT:       Scattered: 0
+// CHECK-NEXT:       Section: __literal8
 // CHECK-NEXT:     }
 // CHECK-NEXT:     Relocation {
 // CHECK-NEXT:       Offset: 0x18
 // CHECK-NEXT:       PCRel: 0
 // CHECK-NEXT:       Length: 3
-// CHECK-NEXT:       Extern: 1
 // CHECK-NEXT:       Type: X86_64_RELOC_UNSIGNED (0)
 // CHECK-NEXT:       Symbol: f6
-// CHECK-NEXT:       Scattered: 0
 // CHECK-NEXT:     }
 // CHECK-NEXT:     Relocation {
 // CHECK-NEXT:       Offset: 0x10
 // CHECK-NEXT:       PCRel: 0
 // CHECK-NEXT:       Length: 3
-// CHECK-NEXT:       Extern: 0
 // CHECK-NEXT:       Type: X86_64_RELOC_UNSIGNED (0)
-// CHECK-NEXT:       Symbol: 0x4
-// CHECK-NEXT:       Scattered: 0
+// CHECK-NEXT:       Section: __literal8
 // CHECK-NEXT:     }
 // CHECK-NEXT:     Relocation {
 // CHECK-NEXT:       Offset: 0x8
 // CHECK-NEXT:       PCRel: 1
 // CHECK-NEXT:       Length: 2
-// CHECK-NEXT:       Extern: 1
 // CHECK-NEXT:       Type: X86_64_RELOC_GOT (4)
 // CHECK-NEXT:       Symbol: _foo
-// CHECK-NEXT:       Scattered: 0
 // CHECK-NEXT:     }
 // CHECK-NEXT:     Relocation {
 // CHECK-NEXT:       Offset: 0x4
 // CHECK-NEXT:       PCRel: 1
 // CHECK-NEXT:       Length: 2
-// CHECK-NEXT:       Extern: 1
 // CHECK-NEXT:       Type: X86_64_RELOC_GOT (4)
 // CHECK-NEXT:       Symbol: _foobar
-// CHECK-NEXT:       Scattered: 0
 // CHECK-NEXT:     }
 // CHECK-NEXT:   }
 // CHECK-NEXT:   Section __text {
@@ -140,289 +130,225 @@ L6:
 // CHECK-NEXT:       Offset: 0xDA
 // CHECK-NEXT:       PCRel: 1
 // CHECK-NEXT:       Length: 2
-// CHECK-NEXT:       Extern: 1
 // CHECK-NEXT:       Type: X86_64_RELOC_GOT (4)
 // CHECK-NEXT:       Symbol: _foo
-// CHECK-NEXT:       Scattered: 0
 // CHECK-NEXT:     }
 // CHECK-NEXT:     Relocation {
 // CHECK-NEXT:       Offset: 0xD3
 // CHECK-NEXT:       PCRel: 1
 // CHECK-NEXT:       Length: 2
-// CHECK-NEXT:       Extern: 0
 // CHECK-NEXT:       Type: X86_64_RELOC_SIGNED (1)
-// CHECK-NEXT:       Symbol: 0x4
-// CHECK-NEXT:       Scattered: 0
+// CHECK-NEXT:       Section: __literal8
 // CHECK-NEXT:     }
 // CHECK-NEXT:     Relocation {
 // CHECK-NEXT:       Offset: 0xCD
 // CHECK-NEXT:       PCRel: 1
 // CHECK-NEXT:       Length: 2
-// CHECK-NEXT:       Extern: 1
 // CHECK-NEXT:       Type: X86_64_RELOC_SIGNED (1)
 // CHECK-NEXT:       Symbol: f6
-// CHECK-NEXT:       Scattered: 0
 // CHECK-NEXT:     }
 // CHECK-NEXT:     Relocation {
 // CHECK-NEXT:       Offset: 0xC7
 // CHECK-NEXT:       PCRel: 1
 // CHECK-NEXT:       Length: 2
-// CHECK-NEXT:       Extern: 0
 // CHECK-NEXT:       Type: X86_64_RELOC_SIGNED (1)
-// CHECK-NEXT:       Symbol: 0x4
-// CHECK-NEXT:       Scattered: 0
+// CHECK-NEXT:       Section: __literal8
 // CHECK-NEXT:     }
 // CHECK-NEXT:     Relocation {
 // CHECK-NEXT:       Offset: 0xC1
 // CHECK-NEXT:       PCRel: 1
 // CHECK-NEXT:       Length: 2
-// CHECK-NEXT:       Extern: 0
 // CHECK-NEXT:       Type: X86_64_RELOC_SIGNED (1)
-// CHECK-NEXT:       Symbol: 0x1
-// CHECK-NEXT:       Scattered: 0
+// CHECK-NEXT:       Section: __data
 // CHECK-NEXT:     }
 // CHECK-NEXT:     Relocation {
 // CHECK-NEXT:       Offset: 0xA5
 // CHECK-NEXT:       PCRel: 0
 // CHECK-NEXT:       Length: 3
-// CHECK-NEXT:       Extern: 1
 // CHECK-NEXT:       Type: X86_64_RELOC_SUBTRACTOR (5)
 // CHECK-NEXT:       Symbol: _prev
-// CHECK-NEXT:       Scattered: 0
 // CHECK-NEXT:     }
 // CHECK-NEXT:     Relocation {
 // CHECK-NEXT:       Offset: 0xA5
 // CHECK-NEXT:       PCRel: 0
 // CHECK-NEXT:       Length: 3
-// CHECK-NEXT:       Extern: 1
 // CHECK-NEXT:       Type: X86_64_RELOC_UNSIGNED (0)
 // CHECK-NEXT:       Symbol: _foo
-// CHECK-NEXT:       Scattered: 0
 // CHECK-NEXT:     }
 // CHECK-NEXT:     Relocation {
 // CHECK-NEXT:       Offset: 0x9D
 // CHECK-NEXT:       PCRel: 0
 // CHECK-NEXT:       Length: 3
-// CHECK-NEXT:       Extern: 1
 // CHECK-NEXT:       Type: X86_64_RELOC_SUBTRACTOR (5)
 // CHECK-NEXT:       Symbol: _prev
-// CHECK-NEXT:       Scattered: 0
 // CHECK-NEXT:     }
 // CHECK-NEXT:     Relocation {
 // CHECK-NEXT:       Offset: 0x9D
 // CHECK-NEXT:       PCRel: 0
 // CHECK-NEXT:       Length: 3
-// CHECK-NEXT:       Extern: 1
 // CHECK-NEXT:       Type: X86_64_RELOC_UNSIGNED (0)
 // CHECK-NEXT:       Symbol: _foo
-// CHECK-NEXT:       Scattered: 0
 // CHECK-NEXT:     }
 // CHECK-NEXT:     Relocation {
 // CHECK-NEXT:       Offset: 0x95
 // CHECK-NEXT:       PCRel: 0
 // CHECK-NEXT:       Length: 3
-// CHECK-NEXT:       Extern: 1
 // CHECK-NEXT:       Type: X86_64_RELOC_UNSIGNED (0)
 // CHECK-NEXT:       Symbol: _prev
-// CHECK-NEXT:       Scattered: 0
 // CHECK-NEXT:     }
 // CHECK-NEXT:     Relocation {
 // CHECK-NEXT:       Offset: 0x8D
 // CHECK-NEXT:       PCRel: 0
 // CHECK-NEXT:       Length: 3
-// CHECK-NEXT:       Extern: 1
 // CHECK-NEXT:       Type: X86_64_RELOC_UNSIGNED (0)
 // CHECK-NEXT:       Symbol: _prev
-// CHECK-NEXT:       Scattered: 0
 // CHECK-NEXT:     }
 // CHECK-NEXT:     Relocation {
 // CHECK-NEXT:       Offset: 0x79
 // CHECK-NEXT:       PCRel: 1
 // CHECK-NEXT:       Length: 2
-// CHECK-NEXT:       Extern: 1
 // CHECK-NEXT:       Type: X86_64_RELOC_SIGNED_4 (8)
 // CHECK-NEXT:       Symbol: _prev
-// CHECK-NEXT:       Scattered: 0
 // CHECK-NEXT:     }
 // CHECK-NEXT:     Relocation {
 // CHECK-NEXT:       Offset: 0x71
 // CHECK-NEXT:       PCRel: 1
 // CHECK-NEXT:       Length: 2
-// CHECK-NEXT:       Extern: 1
 // CHECK-NEXT:       Type: X86_64_RELOC_SIGNED_2 (7)
 // CHECK-NEXT:       Symbol: _prev
-// CHECK-NEXT:       Scattered: 0
 // CHECK-NEXT:     }
 // CHECK-NEXT:     Relocation {
 // CHECK-NEXT:       Offset: 0x69
 // CHECK-NEXT:       PCRel: 1
 // CHECK-NEXT:       Length: 2
-// CHECK-NEXT:       Extern: 1
 // CHECK-NEXT:       Type: X86_64_RELOC_SIGNED_1 (6)
 // CHECK-NEXT:       Symbol: _prev
-// CHECK-NEXT:       Scattered: 0
 // CHECK-NEXT:     }
 // CHECK-NEXT:     Relocation {
 // CHECK-NEXT:       Offset: 0x63
 // CHECK-NEXT:       PCRel: 1
 // CHECK-NEXT:       Length: 2
-// CHECK-NEXT:       Extern: 1
 // CHECK-NEXT:       Type: X86_64_RELOC_SIGNED (1)
 // CHECK-NEXT:       Symbol: _prev
-// CHECK-NEXT:       Scattered: 0
 // CHECK-NEXT:     }
 // CHECK-NEXT:     Relocation {
 // CHECK-NEXT:       Offset: 0x5C
 // CHECK-NEXT:       PCRel: 1
 // CHECK-NEXT:       Length: 2
-// CHECK-NEXT:       Extern: 1
 // CHECK-NEXT:       Type: X86_64_RELOC_SIGNED (1)
 // CHECK-NEXT:       Symbol: _prev
-// CHECK-NEXT:       Scattered: 0
 // CHECK-NEXT:     }
 // CHECK-NEXT:     Relocation {
 // CHECK-NEXT:       Offset: 0x55
 // CHECK-NEXT:       PCRel: 0
 // CHECK-NEXT:       Length: 2
-// CHECK-NEXT:       Extern: 1
 // CHECK-NEXT:       Type: X86_64_RELOC_SUBTRACTOR (5)
 // CHECK-NEXT:       Symbol: _bar
-// CHECK-NEXT:       Scattered: 0
 // CHECK-NEXT:     }
 // CHECK-NEXT:     Relocation {
 // CHECK-NEXT:       Offset: 0x55
 // CHECK-NEXT:       PCRel: 0
 // CHECK-NEXT:       Length: 2
-// CHECK-NEXT:       Extern: 1
 // CHECK-NEXT:       Type: X86_64_RELOC_UNSIGNED (0)
 // CHECK-NEXT:       Symbol: _foo
-// CHECK-NEXT:       Scattered: 0
 // CHECK-NEXT:     }
 // CHECK-NEXT:     Relocation {
 // CHECK-NEXT:       Offset: 0x4D
 // CHECK-NEXT:       PCRel: 0
 // CHECK-NEXT:       Length: 3
-// CHECK-NEXT:       Extern: 1
 // CHECK-NEXT:       Type: X86_64_RELOC_SUBTRACTOR (5)
 // CHECK-NEXT:       Symbol: _bar
-// CHECK-NEXT:       Scattered: 0
 // CHECK-NEXT:     }
 // CHECK-NEXT:     Relocation {
 // CHECK-NEXT:       Offset: 0x4D
 // CHECK-NEXT:       PCRel: 0
 // CHECK-NEXT:       Length: 3
-// CHECK-NEXT:       Extern: 1
 // CHECK-NEXT:       Type: X86_64_RELOC_UNSIGNED (0)
 // CHECK-NEXT:       Symbol: _foo
-// CHECK-NEXT:       Scattered: 0
 // CHECK-NEXT:     }
 // CHECK-NEXT:     Relocation {
 // CHECK-NEXT:       Offset: 0x45
 // CHECK-NEXT:       PCRel: 0
 // CHECK-NEXT:       Length: 3
-// CHECK-NEXT:       Extern: 1
 // CHECK-NEXT:       Type: X86_64_RELOC_SUBTRACTOR (5)
 // CHECK-NEXT:       Symbol: _bar
-// CHECK-NEXT:       Scattered: 0
 // CHECK-NEXT:     }
 // CHECK-NEXT:     Relocation {
 // CHECK-NEXT:       Offset: 0x45
 // CHECK-NEXT:       PCRel: 0
 // CHECK-NEXT:       Length: 3
-// CHECK-NEXT:       Extern: 1
 // CHECK-NEXT:       Type: X86_64_RELOC_UNSIGNED (0)
 // CHECK-NEXT:       Symbol: _foo
-// CHECK-NEXT:       Scattered: 0
 // CHECK-NEXT:     }
 // CHECK-NEXT:     Relocation {
 // CHECK-NEXT:       Offset: 0x3D
 // CHECK-NEXT:       PCRel: 0
 // CHECK-NEXT:       Length: 3
-// CHECK-NEXT:       Extern: 1
 // CHECK-NEXT:       Type: X86_64_RELOC_UNSIGNED (0)
 // CHECK-NEXT:       Symbol: _foo
-// CHECK-NEXT:       Scattered: 0
 // CHECK-NEXT:     }
 // CHECK-NEXT:     Relocation {
 // CHECK-NEXT:       Offset: 0x35
 // CHECK-NEXT:       PCRel: 0
 // CHECK-NEXT:       Length: 3
-// CHECK-NEXT:       Extern: 1
 // CHECK-NEXT:       Type: X86_64_RELOC_UNSIGNED (0)
 // CHECK-NEXT:       Symbol: _foo
-// CHECK-NEXT:       Scattered: 0
 // CHECK-NEXT:     }
 // CHECK-NEXT:     Relocation {
 // CHECK-NEXT:       Offset: 0x2D
 // CHECK-NEXT:       PCRel: 1
 // CHECK-NEXT:       Length: 2
-// CHECK-NEXT:       Extern: 1
 // CHECK-NEXT:       Type: X86_64_RELOC_SIGNED_4 (8)
 // CHECK-NEXT:       Symbol: _foo
-// CHECK-NEXT:       Scattered: 0
 // CHECK-NEXT:     }
 // CHECK-NEXT:     Relocation {
 // CHECK-NEXT:       Offset: 0x26
 // CHECK-NEXT:       PCRel: 1
 // CHECK-NEXT:       Length: 2
-// CHECK-NEXT:       Extern: 1
 // CHECK-NEXT:       Type: X86_64_RELOC_SIGNED_1 (6)
 // CHECK-NEXT:       Symbol: _foo
-// CHECK-NEXT:       Scattered: 0
 // CHECK-NEXT:     }
 // CHECK-NEXT:     Relocation {
 // CHECK-NEXT:       Offset: 0x20
 // CHECK-NEXT:       PCRel: 1
 // CHECK-NEXT:       Length: 2
-// CHECK-NEXT:       Extern: 1
 // CHECK-NEXT:       Type: X86_64_RELOC_SIGNED (1)
 // CHECK-NEXT:       Symbol: _foo
-// CHECK-NEXT:       Scattered: 0
 // CHECK-NEXT:     }
 // CHECK-NEXT:     Relocation {
 // CHECK-NEXT:       Offset: 0x1A
 // CHECK-NEXT:       PCRel: 1
 // CHECK-NEXT:       Length: 2
-// CHECK-NEXT:       Extern: 1
 // CHECK-NEXT:       Type: X86_64_RELOC_SIGNED (1)
 // CHECK-NEXT:       Symbol: _foo
-// CHECK-NEXT:       Scattered: 0
 // CHECK-NEXT:     }
 // CHECK-NEXT:     Relocation {
 // CHECK-NEXT:       Offset: 0x14
 // CHECK-NEXT:       PCRel: 1
 // CHECK-NEXT:       Length: 2
-// CHECK-NEXT:       Extern: 1
 // CHECK-NEXT:       Type: X86_64_RELOC_GOT (4)
 // CHECK-NEXT:       Symbol: _foo
-// CHECK-NEXT:       Scattered: 0
 // CHECK-NEXT:     }
 // CHECK-NEXT:     Relocation {
 // CHECK-NEXT:       Offset: 0xE
 // CHECK-NEXT:       PCRel: 1
 // CHECK-NEXT:       Length: 2
-// CHECK-NEXT:       Extern: 1
 // CHECK-NEXT:       Type: X86_64_RELOC_GOT_LOAD (3)
 // CHECK-NEXT:       Symbol: _foo
-// CHECK-NEXT:       Scattered: 0
 // CHECK-NEXT:     }
 // CHECK-NEXT:     Relocation {
 // CHECK-NEXT:       Offset: 0x7
 // CHECK-NEXT:       PCRel: 1
 // CHECK-NEXT:       Length: 2
-// CHECK-NEXT:       Extern: 1
 // CHECK-NEXT:       Type: X86_64_RELOC_BRANCH (2)
 // CHECK-NEXT:       Symbol: _foo
-// CHECK-NEXT:       Scattered: 0
 // CHECK-NEXT:     }
 // CHECK-NEXT:     Relocation {
 // CHECK-NEXT:       Offset: 0x2
 // CHECK-NEXT:       PCRel: 1
 // CHECK-NEXT:       Length: 2
-// CHECK-NEXT:       Extern: 1
 // CHECK-NEXT:       Type: X86_64_RELOC_BRANCH (2)
 // CHECK-NEXT:       Symbol: _foo
-// CHECK-NEXT:       Scattered: 0
 // CHECK-NEXT:     }
 // CHECK-NEXT:   }
 // CHECK-NEXT:   Section __debug_frame {
@@ -430,19 +356,15 @@ L6:
 // CHECK-NEXT:       Offset: 0x8
 // CHECK-NEXT:       PCRel: 0
 // CHECK-NEXT:       Length: 3
-// CHECK-NEXT:       Extern: 1
 // CHECK-NEXT:       Type: X86_64_RELOC_UNSIGNED (0)
 // CHECK-NEXT:       Symbol: _ext_foo
-// CHECK-NEXT:       Scattered: 0
 // CHECK-NEXT:     }
 // CHECK-NEXT:     Relocation {
 // CHECK-NEXT:       Offset: 0x0
 // CHECK-NEXT:       PCRel: 0
 // CHECK-NEXT:       Length: 3
-// CHECK-NEXT:       Extern: 0
 // CHECK-NEXT:       Type: X86_64_RELOC_UNSIGNED (0)
-// CHECK-NEXT:       Symbol: 0x2
-// CHECK-NEXT:       Scattered: 0
+// CHECK-NEXT:       Section: __text
 // CHECK-NEXT:     }
 // CHECK-NEXT:   }
 // CHECK-NEXT: ]
diff --git a/test/MC/MachO/reloc-diff.s b/test/MC/MachO/reloc-diff.s
index 601edba47b4..a63a413f24b 100644
--- a/test/MC/MachO/reloc-diff.s
+++ b/test/MC/MachO/reloc-diff.s
@@ -1,39 +1,27 @@
 // RUN: llvm-mc -triple i386-apple-darwin9 %s -filetype=obj -o - | macho-dump | FileCheck %s
 
 // CHECK: # Relocation 0
-// CHECK: (('word-0', 0xa2000014),
+// CHECK: (('word-0', 0xa4000010),
 // CHECK:  ('word-1', 0x0)),
 // CHECK: # Relocation 1
 // CHECK: (('word-0', 0xa1000000),
 // CHECK:  ('word-1', 0x0)),
 // CHECK: # Relocation 2
-// CHECK: (('word-0', 0xa4000010),
+// CHECK: (('word-0', 0xa4000008),
 // CHECK:  ('word-1', 0x0)),
 // CHECK: # Relocation 3
 // CHECK: (('word-0', 0xa1000000),
 // CHECK:  ('word-1', 0x0)),
 // CHECK: # Relocation 4
-// CHECK: (('word-0', 0xa400000c),
+// CHECK: (('word-0', 0xa4000004),
 // CHECK:  ('word-1', 0x0)),
 // CHECK: # Relocation 5
 // CHECK: (('word-0', 0xa1000000),
 // CHECK:  ('word-1', 0x0)),
 // CHECK: # Relocation 6
-// CHECK: (('word-0', 0xa4000008),
-// CHECK:  ('word-1', 0x0)),
-// CHECK: # Relocation 7
-// CHECK: (('word-0', 0xa1000000),
-// CHECK:  ('word-1', 0x0)),
-// CHECK: # Relocation 8
-// CHECK: (('word-0', 0xa4000004),
-// CHECK:  ('word-1', 0x0)),
-// CHECK: # Relocation 9
-// CHECK: (('word-0', 0xa1000000),
-// CHECK:  ('word-1', 0x0)),
-// CHECK: # Relocation 10
 // CHECK: (('word-0', 0xa2000000),
 // CHECK:  ('word-1', 0x0)),
-// CHECK: # Relocation 11
+// CHECK: # Relocation 7
 // CHECK: (('word-0', 0xa1000000),
 // CHECK:  ('word-1', 0x0)),
 // CHECK-NEXT: ])
@@ -43,7 +31,7 @@ _local_def:
 _external_def:
 Ltemp:
         ret
-        
+
         .data
         .long _external_def - _local_def
         .long Ltemp - _local_def
diff --git a/test/MC/MachO/reloc.s b/test/MC/MachO/reloc.s
index 55c99402529..6a78d0452ed 100644
--- a/test/MC/MachO/reloc.s
+++ b/test/MC/MachO/reloc.s
@@ -59,19 +59,15 @@ _f1:
 // CHECK-NEXT:      Offset: 0x6
 // CHECK-NEXT:      PCRel: 1
 // CHECK-NEXT:      Length: 2
-// CHECK-NEXT:      Extern: 0
 // CHECK-NEXT:      Type: GENERIC_RELOC_VANILLA (0)
-// CHECK-NEXT:      Symbol: 0x3
-// CHECK-NEXT:      Scattered: 0
+// CHECK-NEXT:      Section: __const
 // CHECK-NEXT:    }
 // CHECK-NEXT:    Relocation {
 // CHECK-NEXT:      Offset: 0x1
 // CHECK-NEXT:      PCRel: 1
 // CHECK-NEXT:      Length: 2
-// CHECK-NEXT:      Extern: 0
 // CHECK-NEXT:      Type: GENERIC_RELOC_VANILLA (0)
-// CHECK-NEXT:      Symbol: 0x0
-// CHECK-NEXT:      Scattered: 0
+// CHECK-NEXT:      Section: - (0)
 // CHECK-NEXT:    }
 // CHECK-NEXT:  }
 // CHECK-NEXT:  Section __data {
@@ -79,100 +75,78 @@ _f1:
 // CHECK-NEXT:      Offset: 0x2F
 // CHECK-NEXT:      PCRel: 0
 // CHECK-NEXT:      Length: 2
-// CHECK-NEXT:      Extern: 1
 // CHECK-NEXT:      Type: GENERIC_RELOC_VANILLA (0)
 // CHECK-NEXT:      Symbol: _f1
-// CHECK-NEXT:      Scattered: 0
 // CHECK-NEXT:    }
 // CHECK-NEXT:    Relocation {
 // CHECK-NEXT:      Offset: 0x2B
 // CHECK-NEXT:      PCRel: 0
 // CHECK-NEXT:      Length: 2
-// CHECK-NEXT:      Extern: 1
 // CHECK-NEXT:      Type: GENERIC_RELOC_VANILLA (0)
 // CHECK-NEXT:      Symbol: _f1
-// CHECK-NEXT:      Scattered: 0
 // CHECK-NEXT:    }
 // CHECK-NEXT:    Relocation {
 // CHECK-NEXT:      Offset: 0x2A
 // CHECK-NEXT:      PCRel: 0
 // CHECK-NEXT:      Length: 0
-// CHECK-NEXT:      Extern: N/A
 // CHECK-NEXT:      Type: GENERIC_RELOC_VANILLA (0)
-// CHECK-NEXT:      Symbol: 0x1D
-// CHECK-NEXT:      Scattered: 1
+// CHECK-NEXT:      Value: 0x1D
 // CHECK-NEXT:    }
 // CHECK-NEXT:    Relocation {
 // CHECK-NEXT:      Offset: 0x28
 // CHECK-NEXT:      PCRel: 0
 // CHECK-NEXT:      Length: 1
-// CHECK-NEXT:      Extern: N/A
 // CHECK-NEXT:      Type: GENERIC_RELOC_VANILLA (0)
-// CHECK-NEXT:      Symbol: 0x1D
-// CHECK-NEXT:      Scattered: 1
+// CHECK-NEXT:      Value: 0x1D
 // CHECK-NEXT:    }
 // CHECK-NEXT:    Relocation {
 // CHECK-NEXT:      Offset: 0x24
 // CHECK-NEXT:      PCRel: 0
 // CHECK-NEXT:      Length: 2
-// CHECK-NEXT:      Extern: N/A
 // CHECK-NEXT:      Type: GENERIC_RELOC_VANILLA (0)
-// CHECK-NEXT:      Symbol: 0x1D
-// CHECK-NEXT:      Scattered: 1
+// CHECK-NEXT:      Value: 0x1D
 // CHECK-NEXT:    }
 // CHECK-NEXT:    Relocation {
 // CHECK-NEXT:      Offset: 0x20
 // CHECK-NEXT:      PCRel: 0
 // CHECK-NEXT:      Length: 2
-// CHECK-NEXT:      Extern: N/A
 // CHECK-NEXT:      Type: GENERIC_RELOC_VANILLA (0)
-// CHECK-NEXT:      Symbol: 0x1D
-// CHECK-NEXT:      Scattered: 1
+// CHECK-NEXT:      Value: 0x1D
 // CHECK-NEXT:    }
 // CHECK-NEXT:    Relocation {
 // CHECK-NEXT:      Offset: 0x14
 // CHECK-NEXT:      PCRel: 0
 // CHECK-NEXT:      Length: 2
-// CHECK-NEXT:      Extern: N/A
 // CHECK-NEXT:      Type: GENERIC_RELOC_LOCAL_SECTDIFF (4)
-// CHECK-NEXT:      Symbol: 0x21
-// CHECK-NEXT:      Scattered: 1
+// CHECK-NEXT:      Value: 0x21
 // CHECK-NEXT:    }
 // CHECK-NEXT:    Relocation {
 // CHECK-NEXT:      Offset: 0x0
 // CHECK-NEXT:      PCRel: 0
 // CHECK-NEXT:      Length: 2
-// CHECK-NEXT:      Extern: N/A
 // CHECK-NEXT:      Type: GENERIC_RELOC_PAIR (1)
-// CHECK-NEXT:      Symbol: 0x29
-// CHECK-NEXT:      Scattered: 1
+// CHECK-NEXT:      Value: 0x29
 // CHECK-NEXT:    }
 // CHECK-NEXT:    Relocation {
 // CHECK-NEXT:      Offset: 0x8
 // CHECK-NEXT:      PCRel: 0
 // CHECK-NEXT:      Length: 2
-// CHECK-NEXT:      Extern: 0
 // CHECK-NEXT:      Type: GENERIC_RELOC_VANILLA (0)
-// CHECK-NEXT:      Symbol: 0x2
-// CHECK-NEXT:      Scattered: 0
+// CHECK-NEXT:      Section: __data
 // CHECK-NEXT:    }
 // CHECK-NEXT:    Relocation {
 // CHECK-NEXT:      Offset: 0x4
 // CHECK-NEXT:      PCRel: 0
 // CHECK-NEXT:      Length: 2
-// CHECK-NEXT:      Extern: 1
 // CHECK-NEXT:      Type: GENERIC_RELOC_VANILLA (0)
 // CHECK-NEXT:      Symbol: undef
-// CHECK-NEXT:      Scattered: 0
 // CHECK-NEXT:    }
 // CHECK-NEXT:    Relocation {
 // CHECK-NEXT:      Offset: 0x0
 // CHECK-NEXT:      PCRel: 0
 // CHECK-NEXT:      Length: 2
-// CHECK-NEXT:      Extern: 1
 // CHECK-NEXT:      Type: GENERIC_RELOC_VANILLA (0)
 // CHECK-NEXT:      Symbol: undef
-// CHECK-NEXT:      Scattered: 0
 // CHECK-NEXT:    }
 // CHECK-NEXT:  }
 // CHECK-NEXT:  Section __const {
@@ -180,37 +154,29 @@ _f1:
 // CHECK-NEXT:      Offset: 0x8
 // CHECK-NEXT:      PCRel: 0
 // CHECK-NEXT:      Length: 2
-// CHECK-NEXT:      Extern: 0
 // CHECK-NEXT:      Type: GENERIC_RELOC_VANILLA (0)
-// CHECK-NEXT:      Symbol: 0x1
-// CHECK-NEXT:      Scattered: 0
+// CHECK-NEXT:      Section: __text
 // CHECK-NEXT:    }
 // CHECK-NEXT:    Relocation {
 // CHECK-NEXT:      Offset: 0x4
 // CHECK-NEXT:      PCRel: 0
 // CHECK-NEXT:      Length: 2
-// CHECK-NEXT:      Extern: 0
 // CHECK-NEXT:      Type: GENERIC_RELOC_VANILLA (0)
-// CHECK-NEXT:      Symbol: 0x3
-// CHECK-NEXT:      Scattered: 0
+// CHECK-NEXT:      Section: __const
 // CHECK-NEXT:    }
 // CHECK-NEXT:    Relocation {
 // CHECK-NEXT:      Offset: 0x0
 // CHECK-NEXT:      PCRel: 0
 // CHECK-NEXT:      Length: 2
-// CHECK-NEXT:      Extern: N/A
 // CHECK-NEXT:      Type: GENERIC_RELOC_LOCAL_SECTDIFF (4)
-// CHECK-NEXT:      Symbol: 0x1D
-// CHECK-NEXT:      Scattered: 1
+// CHECK-NEXT:      Value: 0x1D
 // CHECK-NEXT:    }
 // CHECK-NEXT:    Relocation {
 // CHECK-NEXT:      Offset: 0x0
 // CHECK-NEXT:      PCRel: 0
 // CHECK-NEXT:      Length: 2
-// CHECK-NEXT:      Extern: N/A
 // CHECK-NEXT:      Type: GENERIC_RELOC_PAIR (1)
-// CHECK-NEXT:      Symbol: 0x40
-// CHECK-NEXT:      Scattered: 1
+// CHECK-NEXT:      Value: 0x40
 // CHECK-NEXT:    }
 // CHECK-NEXT:  }
 // CHECK-NEXT:]
diff --git a/test/MC/MachO/x86_64-mergeable.s b/test/MC/MachO/x86_64-mergeable.s
index 972477693ed..b7933f92c0c 100644
--- a/test/MC/MachO/x86_64-mergeable.s
+++ b/test/MC/MachO/x86_64-mergeable.s
@@ -23,37 +23,29 @@ L1:
 // CHECK-NEXT:       Offset: 0x18
 // CHECK-NEXT:       PCRel: 0
 // CHECK-NEXT:       Length: 3
-// CHECK-NEXT:       Extern: 1
 // CHECK-NEXT:       Type: X86_64_RELOC_UNSIGNED (0)
 // CHECK-NEXT:       Symbol: L1
-// CHECK-NEXT:       Scattered: 0
 // CHECK-NEXT:     }
 // CHECK-NEXT:     Relocation {
 // CHECK-NEXT:       Offset: 0x10
 // CHECK-NEXT:       PCRel: 0
 // CHECK-NEXT:       Length: 3
-// CHECK-NEXT:       Extern: 0
 // CHECK-NEXT:       Type: X86_64_RELOC_UNSIGNED (0)
-// CHECK-NEXT:       Symbol: 0x3
-// CHECK-NEXT:       Scattered: 0
+// CHECK-NEXT:       Section: __cstring (3)
 // CHECK-NEXT:     }
 // CHECK-NEXT:     Relocation {
 // CHECK-NEXT:       Offset: 0x8
 // CHECK-NEXT:       PCRel: 0
 // CHECK-NEXT:       Length: 3
-// CHECK-NEXT:       Extern: 1
 // CHECK-NEXT:       Type: X86_64_RELOC_UNSIGNED (0)
 // CHECK-NEXT:       Symbol: L0
-// CHECK-NEXT:       Scattered: 0
 // CHECK-NEXT:     }
 // CHECK-NEXT:     Relocation {
 // CHECK-NEXT:       Offset: 0x0
 // CHECK-NEXT:       PCRel: 0
 // CHECK-NEXT:       Length: 3
-// CHECK-NEXT:       Extern: 0
 // CHECK-NEXT:       Type: X86_64_RELOC_UNSIGNED (0)
-// CHECK-NEXT:       Symbol: 0x2
-// CHECK-NEXT:       Scattered: 0
+// CHECK-NEXT:       Section: __literal4 (2)
 // CHECK-NEXT:     }
 // CHECK-NEXT:   }
 // CHECK-NEXT: ]
diff --git a/test/MC/Mips/asciiz-directive-bad.s b/test/MC/Mips/asciiz-directive-bad.s
new file mode 100644
index 00000000000..0f30cfbee63
--- /dev/null
+++ b/test/MC/Mips/asciiz-directive-bad.s
@@ -0,0 +1,8 @@
+# RUN: not llvm-mc -triple mips-unknown-linux %s 2>&1 | FileCheck %s
+
+  .asciiz 12
+# CHECK: :[[@LINE-1]]:11: error: expected string in '.asciiz' directive
+  .asciiz "a"3
+# CHECK: :[[@LINE-1]]:14: error: unexpected token in '.asciiz' directive
+  .asciiz "a",
+# CHECK: :[[@LINE-1]]:15: error: expected string in '.asciiz' directive
diff --git a/test/MC/Mips/asciiz-directive.s b/test/MC/Mips/asciiz-directive.s
new file mode 100644
index 00000000000..0e582145c71
--- /dev/null
+++ b/test/MC/Mips/asciiz-directive.s
@@ -0,0 +1,28 @@
+# RUN: llvm-mc -triple mips-unknown-linux %s | FileCheck %s
+# .asciiz is exactly the same as .asciz, except it's MIPS-specific.
+
+t1:
+  .asciiz
+# CHECK-LABEL: t1
+
+t2:
+  .asciiz "a"
+# CHECK-LABEL: t2
+# CHECK: .byte 97
+# CHECK: .byte 0
+
+t3:
+  .asciiz "a", "b", "c"
+# CHECK-LABEL: t3
+# CHECK: .byte 97
+# CHECK: .byte 0
+# CHECK: .byte 98
+# CHECK: .byte 0
+# CHECK: .byte 99
+# CHECK: .byte 0
+
+t4:
+  .asciiz "abcdefghijklmnop"
+# CHECK-LABEL: t4
+# CHECK: .ascii "abcdefghijklmnop"
+# CHECK: .byte 0
diff --git a/test/MC/Mips/branch-pseudos-bad.s b/test/MC/Mips/branch-pseudos-bad.s
new file mode 100644
index 00000000000..fcbf84af84d
--- /dev/null
+++ b/test/MC/Mips/branch-pseudos-bad.s
@@ -0,0 +1,21 @@
+# RUN: not llvm-mc %s -arch=mips -mcpu=mips32 2>&1 | FileCheck %s
+
+# Check for errors when using conditional branch pseudos after .set noat.
+  .set noat
+local_label:
+  blt $7, $8, local_label
+# CHECK: :[[@LINE-1]]:3: error: pseudo-instruction requires $at, which is not available
+  bltu $7, $8, local_label
+# CHECK: :[[@LINE-1]]:3: error: pseudo-instruction requires $at, which is not available
+  ble $7, $8, local_label
+# CHECK: :[[@LINE-1]]:3: error: pseudo-instruction requires $at, which is not available
+  bleu $7, $8, local_label
+# CHECK: :[[@LINE-1]]:3: error: pseudo-instruction requires $at, which is not available
+  bge $7, $8, local_label
+# CHECK: :[[@LINE-1]]:3: error: pseudo-instruction requires $at, which is not available
+  bgeu $7, $8, local_label
+# CHECK: :[[@LINE-1]]:3: error: pseudo-instruction requires $at, which is not available
+  bgt $7, $8, local_label
+# CHECK: :[[@LINE-1]]:3: error: pseudo-instruction requires $at, which is not available
+  bgtu $7, $8, local_label
+# CHECK: :[[@LINE-1]]:3: error: pseudo-instruction requires $at, which is not available
diff --git a/test/MC/Mips/branch-pseudos.s b/test/MC/Mips/branch-pseudos.s
new file mode 100644
index 00000000000..d5b06f78d80
--- /dev/null
+++ b/test/MC/Mips/branch-pseudos.s
@@ -0,0 +1,189 @@
+# RUN: llvm-mc %s -arch=mips -mcpu=mips32 -show-encoding | FileCheck %s
+# RUN: llvm-mc %s -arch=mips -mcpu=mips32 2>&1 | \
+# RUN:   FileCheck %s --check-prefix=WARNING
+
+  .text
+local_label:
+  blt $7, $8, local_label
+# CHECK: slt  $1, $7, $8       # encoding: [0x00,0xe8,0x08,0x2a]
+# CHECK: bnez $1, local_label  # encoding: [0x14,0x20,A,A]
+# CHECK:                       #   fixup A - offset: 0, value: local_label-4, kind: fixup_Mips_PC16
+# CHECK: nop
+  blt $7, $8, global_label
+# CHECK: slt  $1, $7, $8       # encoding: [0x00,0xe8,0x08,0x2a]
+# CHECK: bnez $1, global_label # encoding: [0x14,0x20,A,A]
+# CHECK:                       #   fixup A - offset: 0, value: global_label-4, kind: fixup_Mips_PC16
+# CHECK: nop
+  blt $7, $0, local_label
+# CHECK: bltz $7, local_label  # encoding: [0x04,0xe0,A,A]
+# CHECK:                       #   fixup A - offset: 0, value: local_label-4, kind: fixup_Mips_PC16
+# CHECK: nop
+  blt $0, $8, local_label
+# CHECK: bgtz $8, local_label  # encoding: [0x1d,0x00,A,A]
+# CHECK:                       #   fixup A - offset: 0, value: local_label-4, kind: fixup_Mips_PC16
+# CHECK: nop
+  blt $0, $0, local_label
+# CHECK: bltz $zero, local_label # encoding: [0x04,0x00,A,A]
+# CHECK:                         #   fixup A - offset: 0, value: local_label-4, kind: fixup_Mips_PC16
+# CHECK: nop
+
+  bltu $7, $8, local_label
+# CHECK: sltu $1, $7, $8       # encoding: [0x00,0xe8,0x08,0x2b]
+# CHECK: bnez $1, local_label  # encoding: [0x14,0x20,A,A]
+# CHECK:                       #   fixup A - offset: 0, value: local_label-4, kind: fixup_Mips_PC16
+# CHECK: nop
+  bltu $7, $8, global_label
+# CHECK: sltu $1, $7, $8       # encoding: [0x00,0xe8,0x08,0x2b]
+# CHECK: bnez $1, global_label # encoding: [0x14,0x20,A,A]
+# CHECK:                       #   fixup A - offset: 0, value: global_label-4, kind: fixup_Mips_PC16
+# CHECK: nop
+  bltu $7, $0, local_label
+# CHECK: nop
+  bltu $0, $8, local_label
+# CHECK: bnez $8, local_label  # encoding: [0x15,0x00,A,A]
+# CHECK:                       #   fixup A - offset: 0, value: local_label-4, kind: fixup_Mips_PC16
+# CHECK: nop
+  bltu $0, $0, local_label
+# CHECK: nop
+
+  ble $7, $8, local_label
+# CHECK: slt  $1, $8, $7       # encoding: [0x01,0x07,0x08,0x2a]
+# CHECK: beqz $1, local_label  # encoding: [0x10,0x20,A,A]
+# CHECK:                       #   fixup A - offset: 0, value: local_label-4, kind: fixup_Mips_PC16
+# CHECK: nop
+  ble $7, $8, global_label
+# CHECK: slt  $1, $8, $7       # encoding: [0x01,0x07,0x08,0x2a]
+# CHECK: beqz $1, global_label # encoding: [0x10,0x20,A,A]
+# CHECK:                       #   fixup A - offset: 0, value: global_label-4, kind: fixup_Mips_PC16
+# CHECK: nop
+  ble $7, $0, local_label
+# CHECK: blez $7, local_label  # encoding: [0x18,0xe0,A,A]
+# CHECK:                       #   fixup A - offset: 0, value: local_label-4, kind: fixup_Mips_PC16
+# CHECK: nop
+  ble $0, $8, local_label
+# CHECK: bgez $8, local_label  # encoding: [0x05,0x01,A,A]
+# CHECK:                       #   fixup A - offset: 0, value: local_label-4, kind: fixup_Mips_PC16
+# CHECK: nop
+  ble $0, $0, local_label
+# WARNING: :[[@LINE-1]]:3: warning: branch is always taken
+# CHECK: blez $zero, local_label # encoding: [0x18,0x00,A,A]
+# CHECK:                         #   fixup A - offset: 0, value: local_label-4, kind: fixup_Mips_PC16
+# CHECK: nop
+
+  bleu $7, $8, local_label
+# CHECK: sltu $1, $8, $7       # encoding: [0x01,0x07,0x08,0x2b]
+# CHECK: beqz $1, local_label  # encoding: [0x10,0x20,A,A]
+# CHECK:                       #   fixup A - offset: 0, value: local_label-4, kind: fixup_Mips_PC16
+# CHECK: nop
+  bleu $7, $8, global_label
+# CHECK: sltu $1, $8, $7       # encoding: [0x01,0x07,0x08,0x2b]
+# CHECK: beqz $1, global_label # encoding: [0x10,0x20,A,A]
+# CHECK:                       #   fixup A - offset: 0, value: global_label-4, kind: fixup_Mips_PC16
+# CHECK: nop
+  bleu $7, $0, local_label
+# CHECK: beqz $7, local_label  # encoding: [0x10,0xe0,A,A]
+# CHECK:                       #   fixup A - offset: 0, value: local_label-4, kind: fixup_Mips_PC16
+# CHECK: nop
+  bleu $0, $8, local_label
+# WARNING: :[[@LINE-1]]:3: warning: branch is always taken
+# CHECK: b  local_label        # encoding: [0x10,0x00,A,A]
+# CHECK:                       #   fixup A - offset: 0, value: local_label-4, kind: fixup_Mips_PC16
+# CHECK: nop
+  bleu $0, $0, local_label
+# WARNING: :[[@LINE-1]]:3: warning: branch is always taken
+# CHECK: b  local_label        # encoding: [0x10,0x00,A,A]
+# CHECK:                       #   fixup A - offset: 0, value: local_label-4, kind: fixup_Mips_PC16
+# CHECK: nop
+
+  bge $7, $8, local_label
+# CHECK: slt  $1, $7, $8       # encoding: [0x00,0xe8,0x08,0x2a]
+# CHECK: beqz $1, local_label  # encoding: [0x10,0x20,A,A]
+# CHECK:                       #   fixup A - offset: 0, value: local_label-4, kind: fixup_Mips_PC16
+# CHECK: nop
+  bge $7, $8, global_label
+# CHECK: slt  $1, $7, $8       # encoding: [0x00,0xe8,0x08,0x2a]
+# CHECK: beqz $1, global_label # encoding: [0x10,0x20,A,A]
+# CHECK:                       #   fixup A - offset: 0, value: global_label-4, kind: fixup_Mips_PC16
+# CHECK: nop
+  bge $7, $0, local_label
+# CHECK: bgez $7, local_label  # encoding: [0x04,0xe1,A,A]
+# CHECK:                       #   fixup A - offset: 0, value: local_label-4, kind: fixup_Mips_PC16
+# CHECK: nop
+  bge $0, $8, local_label
+# CHECK: blez $8, local_label  # encoding: [0x19,0x00,A,A]
+# CHECK:                       #   fixup A - offset: 0, value: local_label-4, kind: fixup_Mips_PC16
+# CHECK: nop
+  bge $0, $0, local_label
+# WARNING: :[[@LINE-1]]:3: warning: branch is always taken
+# CHECK: bgez $zero, local_label # encoding: [0x04,0x01,A,A]
+# CHECK:                         #   fixup A - offset: 0, value: local_label-4, kind: fixup_Mips_PC16
+# CHECK: nop
+
+  bgeu $7, $8, local_label
+# CHECK: sltu $1, $7, $8       # encoding: [0x00,0xe8,0x08,0x2b]
+# CHECK: beqz $1, local_label  # encoding: [0x10,0x20,A,A]
+# CHECK:                       #   fixup A - offset: 0, value: local_label-4, kind: fixup_Mips_PC16
+# CHECK: nop
+  bgeu $7, $8, global_label
+# CHECK: sltu $1, $7, $8       # encoding: [0x00,0xe8,0x08,0x2b]
+# CHECK: beqz $1, global_label # encoding: [0x10,0x20,A,A]
+# CHECK:                       #   fixup A - offset: 0, value: global_label-4, kind: fixup_Mips_PC16
+# CHECK: nop
+  bgeu $7, $0, local_label
+# WARNING: :[[@LINE-1]]:3: warning: branch is always taken
+# CHECK: b  local_label        # encoding: [0x10,0x00,A,A]
+# CHECK:                       #   fixup A - offset: 0, value: local_label-4, kind: fixup_Mips_PC16
+# CHECK: nop
+  bgeu $0, $8, local_label
+# CHECK: beqz $8, local_label  # encoding: [0x11,0x00,A,A]
+# CHECK:                       #   fixup A - offset: 0, value: local_label-4, kind: fixup_Mips_PC16
+# CHECK: nop
+  bgeu $0, $0, local_label
+# WARNING: :[[@LINE-1]]:3: warning: branch is always taken
+# CHECK: b  local_label        # encoding: [0x10,0x00,A,A]
+# CHECK:                       #   fixup A - offset: 0, value: local_label-4, kind: fixup_Mips_PC16
+# CHECK: nop
+
+  bgt $7, $8, local_label
+# CHECK: slt  $1, $8, $7       # encoding: [0x01,0x07,0x08,0x2a]
+# CHECK: bnez $1, local_label  # encoding: [0x14,0x20,A,A]
+# CHECK:                       #   fixup A - offset: 0, value: local_label-4, kind: fixup_Mips_PC16
+# CHECK: nop
+  bgt $7, $8, global_label
+# CHECK: slt  $1, $8, $7       # encoding: [0x01,0x07,0x08,0x2a]
+# CHECK: bnez $1, global_label # encoding: [0x14,0x20,A,A]
+# CHECK:                       #   fixup A - offset: 0, value: global_label-4, kind: fixup_Mips_PC16
+# CHECK: nop
+  bgt $7, $0, local_label
+# CHECK: bgtz $7, local_label  # encoding: [0x1c,0xe0,A,A]
+# CHECK:                       #   fixup A - offset: 0, value: local_label-4, kind: fixup_Mips_PC16
+# CHECK: nop
+  bgt $0, $8, local_label
+# CHECK: bltz $8, local_label  # encoding: [0x05,0x00,A,A]
+# CHECK:                       #   fixup A - offset: 0, value: local_label-4, kind: fixup_Mips_PC16
+# CHECK: nop
+  bgt $0, $0, local_label
+# CHECK: bgtz  $zero, local_label # encoding: [0x1c,0x00,A,A]
+# CHECK:                          #   fixup A - offset: 0, value: local_label-4, kind: fixup_Mips_PC16
+# CHECK: nop
+
+  bgtu $7, $8, local_label
+# CHECK: sltu $1, $8, $7       # encoding: [0x01,0x07,0x08,0x2b]
+# CHECK: bnez $1, local_label  # encoding: [0x14,0x20,A,A]
+# CHECK:                       #   fixup A - offset: 0, value: local_label-4, kind: fixup_Mips_PC16
+# CHECK: nop
+  bgtu $7, $8, global_label
+# CHECK: sltu $1, $8, $7       # encoding: [0x01,0x07,0x08,0x2b]
+# CHECK: bnez $1, global_label # encoding: [0x14,0x20,A,A]
+# CHECK:                       #   fixup A - offset: 0, value: global_label-4, kind: fixup_Mips_PC16
+# CHECK: nop
+  bgtu $7, $0, local_label
+# CHECK: bnez $7, local_label  # encoding: [0x14,0xe0,A,A]
+# CHECK:                       #   fixup A - offset: 0, value: local_label-4, kind: fixup_Mips_PC16
+# CHECK: nop
+  bgtu $0, $8, local_label
+# CHECK: nop
+  bgtu $0, $0, local_label
+# CHECK: bnez $zero, local_label # encoding: [0x14,0x00,A,A]
+# CHECK:                         #   fixup A - offset: 0, value: local_label-4, kind: fixup_Mips_PC16
+# CHECK: nop
diff --git a/test/MC/Mips/cfi-advance-loc.s b/test/MC/Mips/cfi-advance-loc.s
new file mode 100644
index 00000000000..c84e7e16237
--- /dev/null
+++ b/test/MC/Mips/cfi-advance-loc.s
@@ -0,0 +1,68 @@
+// RUN: llvm-mc -filetype=obj -triple mipsel-pc-Linux-gnu %s -o - | llvm-readobj  -s -sd | FileCheck %s -check-prefix=CHECK-LE
+// RUN: llvm-mc -filetype=obj -triple mips-pc-linux-gnu %s -o - | llvm-readobj -s -sd | FileCheck %s -check-prefix=CHECK-BE
+
+// test that this produces a correctly encoded cfi_advance_loc for both endians.
+
+f:
+	.cfi_startproc
+	nop
+	.zero 252
+	// DW_CFA_advance_loc2: 256 to 00000100
+	.cfi_def_cfa_offset 8
+	nop
+	.cfi_endproc
+
+g:
+	.cfi_startproc
+	nop
+	.zero 65532
+	// DW_CFA_advance_loc4: 65536 to 00010104
+	.cfi_def_cfa_offset 8
+	nop
+	.cfi_endproc
+
+// CHECK-LE:       Section {
+// CHECK-LE:        Index: 7
+// CHECK-LE:        Name: .eh_frame (44)
+// CHECK-LE-NEXT:   Type: SHT_PROGBITS (0x1)
+// CHECK-LE-NEXT:   Flags [ (0x2)
+// CHECK-LE-NEXT:     SHF_ALLOC (0x2)
+// CHECK-LE-NEXT:   ]
+// CHECK-LE-NEXT:   Address: 0x0
+// CHECK-LE-NEXT:   Offset: 0x10180
+// CHECK-LE-NEXT:   Size: 68
+// CHECK-LE-NEXT:   Link: 0
+// CHECK-LE-NEXT:   Info: 0
+// CHECK-LE-NEXT:   AddressAlignment: 4
+// CHECK-LE-NEXT:   EntrySize: 0
+// CHECK-LE-NEXT:   SectionData (
+// CHECK-LE-NEXT:     0000: 10000000 00000000 017A5200 017C1F01
+// CHECK-LE-NEXT:     0010: 0B0C1D00 14000000 18000000 00000000
+// CHECK-LE-NEXT:     0020: 04010000 00030001 0E080000 14000000
+// CHECK-LE-NEXT:     0030: 30000000 04010000 04000100 00040000
+// CHECK-LE-NEXT:     0040: 01000E08
+// CHECK-LE-NEXT:   )
+// CHECK-LE-NEXT: }
+
+// CHECK-BE:      Section {
+// CHECK-BE:        Index: 7
+// CHECK-BE:        Name: .eh_frame (44)
+// CHECK-BE-NEXT:   Type: SHT_PROGBITS (0x1)
+// CHECK-BE-NEXT:   Flags [ (0x2)
+// CHECK-BE-NEXT:     SHF_ALLOC (0x2)
+// CHECK-BE-NEXT:   ]
+// CHECK-BE-NEXT:   Address: 0x0
+// CHECK-BE-NEXT:   Offset: 0x10180
+// CHECK-BE-NEXT:   Size: 68
+// CHECK-BE-NEXT:   Link: 0
+// CHECK-BE-NEXT:   Info: 0
+// CHECK-BE-NEXT:   AddressAlignment: 4
+// CHECK-BE-NEXT:   EntrySize: 0
+// CHECK-BE-NEXT:   SectionData (
+// CHECK-BE-NEXT:     0000: 00000010 00000000 017A5200 017C1F01
+// CHECK-BE-NEXT:     0010: 0B0C1D00 00000014 00000018 00000000
+// CHECK-BE-NEXT:     0020: 00000104 00030100 0E080000 00000014
+// CHECK-BE-NEXT:     0030: 00000030 00000104 00010004 00040001
+// CHECK-BE-NEXT:     0040: 00000E08
+// CHECK-BE-NEXT:   )
+// CHECK-BE-NEXT: }
diff --git a/test/MC/Mips/eh-frame.s b/test/MC/Mips/eh-frame.s
index d6b9cf0a540..167159885d7 100644
--- a/test/MC/Mips/eh-frame.s
+++ b/test/MC/Mips/eh-frame.s
@@ -31,7 +31,7 @@ func:
 // MIPS32: 00000000
 
 // Version
-// MIPS32: 03
+// MIPS32: 01
 
 // Augmentation String
 // MIPS32: 7a5200
@@ -67,7 +67,7 @@ func:
 // MIPS32EL: 00000000
 
 // Version
-// MIPS32EL: 03
+// MIPS32EL: 01
 
 // Augmentation String
 // MIPS32EL: 7a5200
@@ -103,7 +103,7 @@ func:
 // MIPS64: 00000000
 
 // Version
-// MIPS64: 03
+// MIPS64: 01
 
 // Augmentation String
 // MIPS64: 7a5200
@@ -141,7 +141,7 @@ func:
 // MIPS64EL: 00000000
 
 // Version
-// MIPS64EL: 03
+// MIPS64EL: 01
 
 // Augmentation String
 // MIPS64EL: 7a5200
diff --git a/test/MC/Mips/elf-note-nacl.ll b/test/MC/Mips/elf-note-nacl.ll
deleted file mode 100644
index 0361eff25e2..00000000000
--- a/test/MC/Mips/elf-note-nacl.ll
+++ /dev/null
@@ -1,22 +0,0 @@
-; RUN: llc -filetype=obj -mtriple mipsel-none-nacl %s -o - \
-; RUN:   | llvm-objdump -triple mipsel -s - | FileCheck %s
-
-; Tests that NaCl object files contain an ELF note section that identifies them
-; to the binutils gold linker
-
-define void @main() {
-  ret void
-}
-
-; There appears to be no way for llvm-objdump to show flags for sections, or
-; to dump groups like readelf.
-; CHECK: .group
-; CHECK: .note.NaCl.ABI.mipsel
-; The contents of the words in the note section should be:
-;   sizeof "NaCl"
-;   sizeof "mipsel"
-;   1 (NT_VERSION)
-;   "NaCl" with nul termination and padding to align 4
-;   "mipsel" with nul termination and padding to align 4
-; CHECK-NEXT: 0000 05000000 07000000 01000000 4e61436c
-; CHECK-NEXT: 0010 00000000 6d697073 656c0000
diff --git a/test/MC/Mips/expr1.s b/test/MC/Mips/expr1.s
index 67664c1a2ae..4af61636355 100644
--- a/test/MC/Mips/expr1.s
+++ b/test/MC/Mips/expr1.s
@@ -1,18 +1,46 @@
-# RUN: llvm-mc  %s -triple=mipsel-unknown-linux -show-encoding -mcpu=mips32r2 | FileCheck %s
+# RUN: llvm-mc  %s -triple=mipsel-unknown-linux -mcpu=mips32r2 -show-encoding | \
+# RUN:   FileCheck %s --check-prefix=32R2-EL
+# RUN: llvm-mc  %s -triple=mipsel-unknown-linux -mcpu=mips32r2 -mattr=micromips -show-encoding | \
+# RUN:   FileCheck %s --check-prefix=MM-32R2-EL
+
 # Check that the assembler can handle the expressions as operands.
-# CHECK:  .text
-# CHECK:  .globl  foo
-# CHECK:  foo:
-# CHECK:  lw   $4, %lo(foo)($4)    # encoding: [A,A,0x84,0x8c]
-# CHECK:                           #   fixup A - offset: 0, value: foo@ABS_LO, kind: fixup_Mips_LO16
-# CHECK:  lw   $4, 56($4)          # encoding: [0x38,0x00,0x84,0x8c]
-# CHECK:  lw   $4, %lo(foo+8)($4)  # encoding: [0x08'A',A,0x84,0x8c]
-# CHECK:                           #   fixup A - offset: 0, value: foo@ABS_LO, kind: fixup_Mips_LO16
-# CHECK:  lw   $4, %lo(foo+8)($4)  # encoding: [0x08'A',A,0x84,0x8c]
-# CHECK:                           #   fixup A - offset: 0, value: foo@ABS_LO, kind: fixup_Mips_LO16
-# CHECK:  lw   $4, %lo(foo+8)($4)  # encoding: [0x08'A',A,0x84,0x8c]
-# CHECK:                           #   fixup A - offset: 0, value: foo@ABS_LO, kind: fixup_Mips_LO16
-# CHECK:  .space  64
+# 32R2-EL: .text
+# 32R2-EL: .globl  foo
+# 32R2-EL: foo:
+# 32R2-EL: lw   $4, %lo(foo)($4)       # encoding: [A,A,0x84,0x8c]
+# 32R2-EL:                             #   fixup A - offset: 0, value: foo@ABS_LO, kind: fixup_Mips_LO16
+# 32R2-EL: lw   $4, 56($4)             # encoding: [0x38,0x00,0x84,0x8c]
+# 32R2-EL: lw   $4, %lo(foo+8)($4)     # encoding: [0x08'A',A,0x84,0x8c]
+# 32R2-EL:                             #   fixup A - offset: 0, value: foo@ABS_LO, kind: fixup_Mips_LO16
+# 32R2-EL: lw   $4, %lo(foo+8)($4)     # encoding: [0x08'A',A,0x84,0x8c]
+# 32R2-EL:                             #   fixup A - offset: 0, value: foo@ABS_LO, kind: fixup_Mips_LO16
+# 32R2-EL: lw   $4, %lo(foo+8)($4)     # encoding: [0x08'A',A,0x84,0x8c]
+# 32R2-EL:                             #   fixup A - offset: 0, value: foo@ABS_LO, kind: fixup_Mips_LO16
+# 32R2-EL: lw   $4, 10($4)             # encoding: [0x0a,0x00,0x84,0x8c]
+# 32R2-EL: lw   $4, 15($4)             # encoding: [0x0f,0x00,0x84,0x8c]
+# 32R2-EL: lw   $4, 21($4)             # encoding: [0x15,0x00,0x84,0x8c]
+# 32R2-EL: lw   $4, 28($4)             # encoding: [0x1c,0x00,0x84,0x8c]
+# 32R2-EL: lw   $4, 6($4)              # encoding: [0x06,0x00,0x84,0x8c]
+# 32R2-EL: .space  64
+
+# MM-32R2-EL: .text
+# MM-32R2-EL: .globl  foo
+# MM-32R2-EL: foo:
+# MM-32R2-EL: lw   $4, %lo(foo)($4)    # encoding: [0x84'A',0xfc'A',0x00,0x00]
+# MM-32R2-EL:                          #   fixup A - offset: 0, value: foo@ABS_LO, kind: fixup_MICROMIPS_LO16
+# MM-32R2-EL: lw   $4, 56($4)          # encoding: [0x84,0xfc,0x38,0x00]
+# MM-32R2-EL: lw   $4, %lo(foo+8)($4)  # encoding: [0x84'A',0xfc'A',0x08,0x00]
+# MM-32R2-EL:                          #   fixup A - offset: 0, value: foo@ABS_LO, kind: fixup_MICROMIPS_LO16
+# MM-32R2-EL: lw   $4, %lo(foo+8)($4)  # encoding: [0x84'A',0xfc'A',0x08,0x00]
+# MM-32R2-EL:                          #   fixup A - offset: 0, value: foo@ABS_LO, kind: fixup_MICROMIPS_LO16
+# MM-32R2-EL: lw   $4, %lo(foo+8)($4)  # encoding: [0x84'A',0xfc'A',0x08,0x00]
+# MM-32R2-EL:                          #   fixup A - offset: 0, value: foo@ABS_LO, kind: fixup_MICROMIPS_LO16
+# MM-32R2-EL: lw   $4, 10($4)          # encoding: [0x84,0xfc,0x0a,0x00]
+# MM-32R2-EL: lw   $4, 15($4)          # encoding: [0x84,0xfc,0x0f,0x00]
+# MM-32R2-EL: lw   $4, 21($4)          # encoding: [0x84,0xfc,0x15,0x00]
+# MM-32R2-EL: lw   $4, 28($4)          # encoding: [0x84,0xfc,0x1c,0x00]
+# MM-32R2-EL: lw   $4, 6($4)           # encoding: [0x84,0xfc,0x06,0x00]
+# MM-32R2-EL: .space  64
 
   .globl  foo
   .ent  foo
@@ -22,5 +50,10 @@ foo:
   lw  $4,%lo (2 * 4) + foo($4)
   lw  $4,%lo((2 * 4) + foo)($4)
   lw  $4,(((%lo ((2 * 4) + foo))))($4)
+  lw  $4, (((1+2)+3)+4)($4)
+  lw  $4, ((((1+2)+3)+4)+5)($4)
+  lw  $4, (((((1+2)+3)+4)+5)+6)($4)
+  lw  $4, ((((((1+2)+3)+4)+5)+6)+7)($4)
+  lw  $4, (%lo((1+2)+65536)+3)($4)
   .space  64
   .end  foo
diff --git a/test/MC/Mips/macro-la-bad.s b/test/MC/Mips/macro-la-bad.s
new file mode 100644
index 00000000000..89d334030ec
--- /dev/null
+++ b/test/MC/Mips/macro-la-bad.s
@@ -0,0 +1,17 @@
+# RUN: not llvm-mc %s -arch=mips -mcpu=mips32r2 2>%t1
+# RUN: FileCheck %s < %t1 --check-prefix=32-BIT
+# RUN: not llvm-mc %s -arch=mips64 -mcpu=mips64 -target-abi n32 2>&1 | \
+# RUN:   FileCheck %s --check-prefix=64-BIT --check-prefix=N32-ONLY
+# RUN: not llvm-mc %s -arch=mips64 -mcpu=mips64 -target-abi n64 2>&1 | \
+# RUN:   FileCheck %s --check-prefix=64-BIT --check-prefix=N64-ONLY
+
+  .text
+  la $5, 0x100000000
+  # 32-BIT: :[[@LINE-1]]:3: error: instruction requires a 32-bit immediate
+  # 64-BIT: :[[@LINE-2]]:3: error: instruction requires a 32-bit immediate
+  la $5, 0x100000000($6)
+  # 32-BIT: :[[@LINE-1]]:3: error: instruction requires a 32-bit immediate
+  # 64-BIT: :[[@LINE-2]]:3: error: instruction requires a 32-bit immediate
+  la $5, symbol
+  # N64-ONLY: :[[@LINE-1]]:3: warning: instruction loads the 32-bit address of a 64-bit symbol
+  # N32-ONLY-NOT: :[[@LINE-2]]:3: warning: instruction loads the 32-bit address of a 64-bit symbol
diff --git a/test/MC/Mips/macro-la.s b/test/MC/Mips/macro-la.s
new file mode 100644
index 00000000000..8c183a7b23e
--- /dev/null
+++ b/test/MC/Mips/macro-la.s
@@ -0,0 +1,263 @@
+# RUN: llvm-mc %s -triple=mips-unknown-linux -show-encoding -mcpu=mips32r2 | \
+# RUN:   FileCheck %s
+# RUN: llvm-mc %s -triple=mips-unknown-linux -show-encoding -mcpu=mips32r6 | \
+# RUN:   FileCheck %s
+# RUN: llvm-mc %s -triple=mips64-unknown-linux -show-encoding -mcpu=mips64r2 | \
+# RUN:   FileCheck %s
+# RUN: llvm-mc %s -triple=mips64-unknown-linux -show-encoding -mcpu=mips64r6 | \
+# RUN:   FileCheck %s
+
+la $5, 0x00000001 # CHECK: addiu $5, $zero, 1      # encoding: [0x24,0x05,0x00,0x01]
+la $5, 0x00000002 # CHECK: addiu $5, $zero, 2      # encoding: [0x24,0x05,0x00,0x02]
+la $5, 0x00004000 # CHECK: addiu $5, $zero, 16384  # encoding: [0x24,0x05,0x40,0x00]
+la $5, 0x00008000 # CHECK: ori   $5, $zero, 32768  # encoding: [0x34,0x05,0x80,0x00]
+la $5, 0xffffffff # CHECK: addiu $5, $zero, -1     # encoding: [0x24,0x05,0xff,0xff]
+la $5, 0xfffffffe # CHECK: addiu $5, $zero, -2     # encoding: [0x24,0x05,0xff,0xfe]
+la $5, 0xffffc000 # CHECK: addiu $5, $zero, -16384 # encoding: [0x24,0x05,0xc0,0x00]
+la $5, 0xffff8000 # CHECK: addiu $5, $zero, -32768 # encoding: [0x24,0x05,0x80,0x00]
+
+la $5, 0x00010000 # CHECK: lui $5, 1      # encoding: [0x3c,0x05,0x00,0x01]
+la $5, 0x00020000 # CHECK: lui $5, 2      # encoding: [0x3c,0x05,0x00,0x02]
+la $5, 0x40000000 # CHECK: lui $5, 16384  # encoding: [0x3c,0x05,0x40,0x00]
+la $5, 0x80000000 # CHECK: lui $5, 32768  # encoding: [0x3c,0x05,0x80,0x00]
+la $5, 0xffff0000 # CHECK: lui $5, 65535  # encoding: [0x3c,0x05,0xff,0xff]
+la $5, 0xfffe0000 # CHECK: lui $5, 65534  # encoding: [0x3c,0x05,0xff,0xfe]
+la $5, 0xc0000000 # CHECK: lui $5, 49152  # encoding: [0x3c,0x05,0xc0,0x00]
+la $5, 0x80000000 # CHECK: lui $5, 32768  # encoding: [0x3c,0x05,0x80,0x00]
+
+la $5, 0x00010001 # CHECK: lui $5, 1        # encoding: [0x3c,0x05,0x00,0x01]
+                  # CHECK: ori $5, $5, 1    # encoding: [0x34,0xa5,0x00,0x01]
+la $5, 0x00020001 # CHECK: lui $5, 2        # encoding: [0x3c,0x05,0x00,0x02]
+                  # CHECK: ori $5, $5, 1    # encoding: [0x34,0xa5,0x00,0x01]
+la $5, 0x40000001 # CHECK: lui $5, 16384    # encoding: [0x3c,0x05,0x40,0x00]
+                  # CHECK: ori $5, $5, 1    # encoding: [0x34,0xa5,0x00,0x01]
+la $5, 0x80000001 # CHECK: lui $5, 32768    # encoding: [0x3c,0x05,0x80,0x00]
+                  # CHECK: ori $5, $5, 1    # encoding: [0x34,0xa5,0x00,0x01]
+la $5, 0x00010002 # CHECK: lui $5, 1        # encoding: [0x3c,0x05,0x00,0x01]
+                  # CHECK: ori $5, $5, 2    # encoding: [0x34,0xa5,0x00,0x02]
+la $5, 0x00020002 # CHECK: lui $5, 2        # encoding: [0x3c,0x05,0x00,0x02]
+                  # CHECK: ori $5, $5, 2    # encoding: [0x34,0xa5,0x00,0x02]
+la $5, 0x40000002 # CHECK: lui $5, 16384    # encoding: [0x3c,0x05,0x40,0x00]
+                  # CHECK: ori $5, $5, 2    # encoding: [0x34,0xa5,0x00,0x02]
+la $5, 0x80000002 # CHECK: lui $5, 32768    # encoding: [0x3c,0x05,0x80,0x00]
+                  # CHECK: ori $5, $5, 2    # encoding: [0x34,0xa5,0x00,0x02]
+la $5, 0x00014000 # CHECK: lui $5, 1        # encoding: [0x3c,0x05,0x00,0x01]
+                  # CHECK: ori $5, $5, 16384    # encoding: [0x34,0xa5,0x40,0x00]
+la $5, 0x00024000 # CHECK: lui $5, 2            # encoding: [0x3c,0x05,0x00,0x02]
+                  # CHECK: ori $5, $5, 16384    # encoding: [0x34,0xa5,0x40,0x00]
+la $5, 0x40004000 # CHECK: lui $5, 16384        # encoding: [0x3c,0x05,0x40,0x00]
+                  # CHECK: ori $5, $5, 16384    # encoding: [0x34,0xa5,0x40,0x00]
+la $5, 0x80004000 # CHECK: lui $5, 32768        # encoding: [0x3c,0x05,0x80,0x00]
+                  # CHECK: ori $5, $5, 16384    # encoding: [0x34,0xa5,0x40,0x00]
+la $5, 0x00018000 # CHECK: lui $5, 1            # encoding: [0x3c,0x05,0x00,0x01]
+                  # CHECK: ori $5, $5, 32768    # encoding: [0x34,0xa5,0x80,0x00]
+la $5, 0x00028000 # CHECK: lui $5, 2            # encoding: [0x3c,0x05,0x00,0x02]
+                  # CHECK: ori $5, $5, 32768    # encoding: [0x34,0xa5,0x80,0x00]
+la $5, 0x40008000 # CHECK: lui $5, 16384        # encoding: [0x3c,0x05,0x40,0x00]
+                  # CHECK: ori $5, $5, 32768    # encoding: [0x34,0xa5,0x80,0x00]
+la $5, 0x80008000 # CHECK: lui $5, 32768        # encoding: [0x3c,0x05,0x80,0x00]
+                  # CHECK: ori $5, $5, 32768    # encoding: [0x34,0xa5,0x80,0x00]
+la $5, 0xffff4000 # CHECK: lui $5, 65535        # encoding: [0x3c,0x05,0xff,0xff]
+                  # CHECK: ori $5, $5, 16384    # encoding: [0x34,0xa5,0x40,0x00]
+la $5, 0xfffe8000 # CHECK: lui $5, 65534        # encoding: [0x3c,0x05,0xff,0xfe]
+                  # CHECK: ori $5, $5, 32768    # encoding: [0x34,0xa5,0x80,0x00]
+la $5, 0xc0008000 # CHECK: lui $5, 49152        # encoding: [0x3c,0x05,0xc0,0x00]
+                  # CHECK: ori $5, $5, 32768    # encoding: [0x34,0xa5,0x80,0x00]
+la $5, 0x80008000 # CHECK: lui $5, 32768        # encoding: [0x3c,0x05,0x80,0x00]
+                  # CHECK: ori $5, $5, 32768    # encoding: [0x34,0xa5,0x80,0x00]
+
+la $5, 0x00000001($6) # CHECK: addiu $5, $6, 1         # encoding: [0x24,0xc5,0x00,0x01]
+la $5, 0x00000002($6) # CHECK: addiu $5, $6, 2         # encoding: [0x24,0xc5,0x00,0x02]
+la $5, 0x00004000($6) # CHECK: addiu $5, $6, 16384     # encoding: [0x24,0xc5,0x40,0x00]
+la $5, 0x00008000($6) # CHECK: ori   $5, $zero, 32768  # encoding: [0x34,0x05,0x80,0x00]
+                      # CHECK: addu $5, $5, $6         # encoding: [0x00,0xa6,0x28,0x21]
+la $5, 0xffffffff($6) # CHECK: addiu $5, $6, -1        # encoding: [0x24,0xc5,0xff,0xff]
+la $5, 0xfffffffe($6) # CHECK: addiu $5, $6, -2        # encoding: [0x24,0xc5,0xff,0xfe]
+la $5, 0xffffc000($6) # CHECK: addiu $5, $6, -16384    # encoding: [0x24,0xc5,0xc0,0x00]
+la $5, 0xffff8000($6) # CHECK: addiu $5, $6, -32768    # encoding: [0x24,0xc5,0x80,0x00]
+
+la $5, 0x00010000($6) # CHECK: lui $5, 1       # encoding: [0x3c,0x05,0x00,0x01]
+                      # CHECK: addu $5, $5, $6 # encoding: [0x00,0xa6,0x28,0x21]
+la $5, 0x00020000($6) # CHECK: lui $5, 2       # encoding: [0x3c,0x05,0x00,0x02]
+                      # CHECK: addu $5, $5, $6 # encoding: [0x00,0xa6,0x28,0x21]
+la $5, 0x40000000($6) # CHECK: lui $5, 16384   # encoding: [0x3c,0x05,0x40,0x00]
+                      # CHECK: addu $5, $5, $6 # encoding: [0x00,0xa6,0x28,0x21]
+la $5, 0x80000000($6) # CHECK: lui $5, 32768   # encoding: [0x3c,0x05,0x80,0x00]
+                      # CHECK: addu $5, $5, $6 # encoding: [0x00,0xa6,0x28,0x21]
+la $5, 0xffff0000($6) # CHECK: lui $5, 65535   # encoding: [0x3c,0x05,0xff,0xff]
+                      # CHECK: addu $5, $5, $6 # encoding: [0x00,0xa6,0x28,0x21]
+la $5, 0xfffe0000($6) # CHECK: lui $5, 65534   # encoding: [0x3c,0x05,0xff,0xfe]
+                      # CHECK: addu $5, $5, $6 # encoding: [0x00,0xa6,0x28,0x21]
+la $5, 0xc0000000($6) # CHECK: lui $5, 49152   # encoding: [0x3c,0x05,0xc0,0x00]
+                      # CHECK: addu $5, $5, $6 # encoding: [0x00,0xa6,0x28,0x21]
+la $5, 0x80000000($6) # CHECK: lui $5, 32768   # encoding: [0x3c,0x05,0x80,0x00]
+                      # CHECK: addu $5, $5, $6 # encoding: [0x00,0xa6,0x28,0x21]
+
+la $5, 0x00010001($6) # CHECK: lui $5, 1         # encoding: [0x3c,0x05,0x00,0x01]
+                      # CHECK: ori $5, $5, 1     # encoding: [0x34,0xa5,0x00,0x01]
+                      # CHECK: addu $5, $5, $6   # encoding: [0x00,0xa6,0x28,0x21]
+la $5, 0x00020001($6) # CHECK: lui $5, 2         # encoding: [0x3c,0x05,0x00,0x02]
+                      # CHECK: ori $5, $5, 1     # encoding: [0x34,0xa5,0x00,0x01]
+                      # CHECK: addu $5, $5, $6   # encoding: [0x00,0xa6,0x28,0x21]
+la $5, 0x40000001($6) # CHECK: lui $5, 16384     # encoding: [0x3c,0x05,0x40,0x00]
+                      # CHECK: ori $5, $5, 1     # encoding: [0x34,0xa5,0x00,0x01]
+                      # CHECK: addu $5, $5, $6   # encoding: [0x00,0xa6,0x28,0x21]
+la $5, 0x80000001($6) # CHECK: lui $5, 32768     # encoding: [0x3c,0x05,0x80,0x00]
+                      # CHECK: ori $5, $5, 1     # encoding: [0x34,0xa5,0x00,0x01]
+                      # CHECK: addu $5, $5, $6   # encoding: [0x00,0xa6,0x28,0x21]
+la $5, 0x00010002($6) # CHECK: lui $5, 1         # encoding: [0x3c,0x05,0x00,0x01]
+                      # CHECK: ori $5, $5, 2     # encoding: [0x34,0xa5,0x00,0x02]
+                      # CHECK: addu $5, $5, $6   # encoding: [0x00,0xa6,0x28,0x21]
+la $5, 0x00020002($6) # CHECK: lui $5, 2         # encoding: [0x3c,0x05,0x00,0x02]
+                      # CHECK: ori $5, $5, 2     # encoding: [0x34,0xa5,0x00,0x02]
+                      # CHECK: addu $5, $5, $6   # encoding: [0x00,0xa6,0x28,0x21]
+la $5, 0x40000002($6) # CHECK: lui $5, 16384     # encoding: [0x3c,0x05,0x40,0x00]
+                      # CHECK: ori $5, $5, 2     # encoding: [0x34,0xa5,0x00,0x02]
+                      # CHECK: addu $5, $5, $6   # encoding: [0x00,0xa6,0x28,0x21]
+la $5, 0x80000002($6) # CHECK: lui $5, 32768     # encoding: [0x3c,0x05,0x80,0x00]
+                      # CHECK: ori $5, $5, 2     # encoding: [0x34,0xa5,0x00,0x02]
+                      # CHECK: addu $5, $5, $6   # encoding: [0x00,0xa6,0x28,0x21]
+la $5, 0x00014000($6) # CHECK: lui $5, 1         # encoding: [0x3c,0x05,0x00,0x01]
+                      # CHECK: ori $5, $5, 16384 # encoding: [0x34,0xa5,0x40,0x00]
+                      # CHECK: addu $5, $5, $6   # encoding: [0x00,0xa6,0x28,0x21]
+la $5, 0x00024000($6) # CHECK: lui $5, 2         # encoding: [0x3c,0x05,0x00,0x02]
+                      # CHECK: ori $5, $5, 16384 # encoding: [0x34,0xa5,0x40,0x00]
+                      # CHECK: addu $5, $5, $6   # encoding: [0x00,0xa6,0x28,0x21]
+la $5, 0x40004000($6) # CHECK: lui $5, 16384     # encoding: [0x3c,0x05,0x40,0x00]
+                      # CHECK: ori $5, $5, 16384 # encoding: [0x34,0xa5,0x40,0x00]
+                      # CHECK: addu $5, $5, $6   # encoding: [0x00,0xa6,0x28,0x21]
+la $5, 0x80004000($6) # CHECK: lui $5, 32768     # encoding: [0x3c,0x05,0x80,0x00]
+                      # CHECK: ori $5, $5, 16384 # encoding: [0x34,0xa5,0x40,0x00]
+                      # CHECK: addu $5, $5, $6   # encoding: [0x00,0xa6,0x28,0x21]
+la $5, 0x00018000($6) # CHECK: lui $5, 1         # encoding: [0x3c,0x05,0x00,0x01]
+                      # CHECK: ori $5, $5, 32768 # encoding: [0x34,0xa5,0x80,0x00]
+                      # CHECK: addu $5, $5, $6   # encoding: [0x00,0xa6,0x28,0x21]
+la $5, 0x00028000($6) # CHECK: lui $5, 2         # encoding: [0x3c,0x05,0x00,0x02]
+                      # CHECK: ori $5, $5, 32768 # encoding: [0x34,0xa5,0x80,0x00]
+                      # CHECK: addu $5, $5, $6   # encoding: [0x00,0xa6,0x28,0x21]
+la $5, 0x40008000($6) # CHECK: lui $5, 16384     # encoding: [0x3c,0x05,0x40,0x00]
+                      # CHECK: ori $5, $5, 32768 # encoding: [0x34,0xa5,0x80,0x00]
+                      # CHECK: addu $5, $5, $6   # encoding: [0x00,0xa6,0x28,0x21]
+la $5, 0x80008000($6) # CHECK: lui $5, 32768     # encoding: [0x3c,0x05,0x80,0x00]
+                      # CHECK: ori $5, $5, 32768 # encoding: [0x34,0xa5,0x80,0x00]
+                      # CHECK: addu $5, $5, $6   # encoding: [0x00,0xa6,0x28,0x21]
+la $5, 0xffff4000($6) # CHECK: lui $5, 65535     # encoding: [0x3c,0x05,0xff,0xff]
+                      # CHECK: ori $5, $5, 16384 # encoding: [0x34,0xa5,0x40,0x00]
+                      # CHECK: addu $5, $5, $6   # encoding: [0x00,0xa6,0x28,0x21]
+la $5, 0xfffe8000($6) # CHECK: lui $5, 65534     # encoding: [0x3c,0x05,0xff,0xfe]
+                      # CHECK: ori $5, $5, 32768 # encoding: [0x34,0xa5,0x80,0x00]
+                      # CHECK: addu $5, $5, $6   # encoding: [0x00,0xa6,0x28,0x21]
+la $5, 0xc0008000($6) # CHECK: lui $5, 49152     # encoding: [0x3c,0x05,0xc0,0x00]
+                      # CHECK: ori $5, $5, 32768 # encoding: [0x34,0xa5,0x80,0x00]
+                      # CHECK: addu $5, $5, $6   # encoding: [0x00,0xa6,0x28,0x21]
+la $5, 0x80008000($6) # CHECK: lui $5, 32768     # encoding: [0x3c,0x05,0x80,0x00]
+                      # CHECK: ori $5, $5, 32768 # encoding: [0x34,0xa5,0x80,0x00]
+                      # CHECK: addu $5, $5, $6   # encoding: [0x00,0xa6,0x28,0x21]
+
+la $6, 0x00000001($6) # CHECK: addiu $6, $6, 1         # encoding: [0x24,0xc6,0x00,0x01]
+la $6, 0x00000002($6) # CHECK: addiu $6, $6, 2         # encoding: [0x24,0xc6,0x00,0x02]
+la $6, 0x00004000($6) # CHECK: addiu $6, $6, 16384     # encoding: [0x24,0xc6,0x40,0x00]
+la $6, 0x00008000($6) # CHECK: ori   $1, $zero, 32768  # encoding: [0x34,0x01,0x80,0x00]
+                      # CHECK: addu $6, $1, $6         # encoding: [0x00,0x26,0x30,0x21]
+la $6, 0xffffffff($6) # CHECK: addiu $6, $6, -1        # encoding: [0x24,0xc6,0xff,0xff]
+la $6, 0xfffffffe($6) # CHECK: addiu $6, $6, -2        # encoding: [0x24,0xc6,0xff,0xfe]
+la $6, 0xffffc000($6) # CHECK: addiu $6, $6, -16384    # encoding: [0x24,0xc6,0xc0,0x00]
+la $6, 0xffff8000($6) # CHECK: addiu $6, $6, -32768    # encoding: [0x24,0xc6,0x80,0x00]
+
+la $6, 0x00010000($6) # CHECK: lui $1, 1       # encoding: [0x3c,0x01,0x00,0x01]
+                      # CHECK: addu $6, $1, $6 # encoding: [0x00,0x26,0x30,0x21]
+la $6, 0x00020000($6) # CHECK: lui $1, 2       # encoding: [0x3c,0x01,0x00,0x02]
+                      # CHECK: addu $6, $1, $6 # encoding: [0x00,0x26,0x30,0x21]
+la $6, 0x40000000($6) # CHECK: lui $1, 16384   # encoding: [0x3c,0x01,0x40,0x00]
+                      # CHECK: addu $6, $1, $6 # encoding: [0x00,0x26,0x30,0x21]
+la $6, 0x80000000($6) # CHECK: lui $1, 32768   # encoding: [0x3c,0x01,0x80,0x00]
+                      # CHECK: addu $6, $1, $6 # encoding: [0x00,0x26,0x30,0x21]
+la $6, 0xffff0000($6) # CHECK: lui $1, 65535   # encoding: [0x3c,0x01,0xff,0xff]
+                      # CHECK: addu $6, $1, $6 # encoding: [0x00,0x26,0x30,0x21]
+la $6, 0xfffe0000($6) # CHECK: lui $1, 65534   # encoding: [0x3c,0x01,0xff,0xfe]
+                      # CHECK: addu $6, $1, $6 # encoding: [0x00,0x26,0x30,0x21]
+la $6, 0xc0000000($6) # CHECK: lui $1, 49152   # encoding: [0x3c,0x01,0xc0,0x00]
+                      # CHECK: addu $6, $1, $6 # encoding: [0x00,0x26,0x30,0x21]
+la $6, 0x80000000($6) # CHECK: lui $1, 32768   # encoding: [0x3c,0x01,0x80,0x00]
+                      # CHECK: addu $6, $1, $6 # encoding: [0x00,0x26,0x30,0x21]
+
+la $6, 0x00010001($6) # CHECK: lui $1, 1         # encoding: [0x3c,0x01,0x00,0x01]
+                      # CHECK: ori $1, $1, 1     # encoding: [0x34,0x21,0x00,0x01]
+                      # CHECK: addu $6, $1, $6   # encoding: [0x00,0x26,0x30,0x21]
+la $6, 0x00020001($6) # CHECK: lui $1, 2         # encoding: [0x3c,0x01,0x00,0x02]
+                      # CHECK: ori $1, $1, 1     # encoding: [0x34,0x21,0x00,0x01]
+                      # CHECK: addu $6, $1, $6   # encoding: [0x00,0x26,0x30,0x21]
+la $6, 0x40000001($6) # CHECK: lui $1, 16384     # encoding: [0x3c,0x01,0x40,0x00]
+                      # CHECK: ori $1, $1, 1     # encoding: [0x34,0x21,0x00,0x01]
+                      # CHECK: addu $6, $1, $6   # encoding: [0x00,0x26,0x30,0x21]
+la $6, 0x80000001($6) # CHECK: lui $1, 32768     # encoding: [0x3c,0x01,0x80,0x00]
+                      # CHECK: ori $1, $1, 1     # encoding: [0x34,0x21,0x00,0x01]
+                      # CHECK: addu $6, $1, $6   # encoding: [0x00,0x26,0x30,0x21]
+la $6, 0x00010002($6) # CHECK: lui $1, 1         # encoding: [0x3c,0x01,0x00,0x01]
+                      # CHECK: ori $1, $1, 2     # encoding: [0x34,0x21,0x00,0x02]
+                      # CHECK: addu $6, $1, $6   # encoding: [0x00,0x26,0x30,0x21]
+la $6, 0x00020002($6) # CHECK: lui $1, 2         # encoding: [0x3c,0x01,0x00,0x02]
+                      # CHECK: ori $1, $1, 2     # encoding: [0x34,0x21,0x00,0x02]
+                      # CHECK: addu $6, $1, $6   # encoding: [0x00,0x26,0x30,0x21]
+la $6, 0x40000002($6) # CHECK: lui $1, 16384     # encoding: [0x3c,0x01,0x40,0x00]
+                      # CHECK: ori $1, $1, 2     # encoding: [0x34,0x21,0x00,0x02]
+                      # CHECK: addu $6, $1, $6   # encoding: [0x00,0x26,0x30,0x21]
+la $6, 0x80000002($6) # CHECK: lui $1, 32768     # encoding: [0x3c,0x01,0x80,0x00]
+                      # CHECK: ori $1, $1, 2     # encoding: [0x34,0x21,0x00,0x02]
+                      # CHECK: addu $6, $1, $6   # encoding: [0x00,0x26,0x30,0x21]
+la $6, 0x00014000($6) # CHECK: lui $1, 1         # encoding: [0x3c,0x01,0x00,0x01]
+                      # CHECK: ori $1, $1, 16384 # encoding: [0x34,0x21,0x40,0x00]
+                      # CHECK: addu $6, $1, $6   # encoding: [0x00,0x26,0x30,0x21]
+la $6, 0x00024000($6) # CHECK: lui $1, 2         # encoding: [0x3c,0x01,0x00,0x02]
+                      # CHECK: ori $1, $1, 16384 # encoding: [0x34,0x21,0x40,0x00]
+                      # CHECK: addu $6, $1, $6   # encoding: [0x00,0x26,0x30,0x21]
+la $6, 0x40004000($6) # CHECK: lui $1, 16384     # encoding: [0x3c,0x01,0x40,0x00]
+                      # CHECK: ori $1, $1, 16384 # encoding: [0x34,0x21,0x40,0x00]
+                      # CHECK: addu $6, $1, $6   # encoding: [0x00,0x26,0x30,0x21]
+la $6, 0x80004000($6) # CHECK: lui $1, 32768     # encoding: [0x3c,0x01,0x80,0x00]
+                      # CHECK: ori $1, $1, 16384 # encoding: [0x34,0x21,0x40,0x00]
+                      # CHECK: addu $6, $1, $6   # encoding: [0x00,0x26,0x30,0x21]
+la $6, 0x00018000($6) # CHECK: lui $1, 1         # encoding: [0x3c,0x01,0x00,0x01]
+                      # CHECK: ori $1, $1, 32768 # encoding: [0x34,0x21,0x80,0x00]
+                      # CHECK: addu $6, $1, $6   # encoding: [0x00,0x26,0x30,0x21]
+la $6, 0x00028000($6) # CHECK: lui $1, 2         # encoding: [0x3c,0x01,0x00,0x02]
+                      # CHECK: ori $1, $1, 32768 # encoding: [0x34,0x21,0x80,0x00]
+                      # CHECK: addu $6, $1, $6   # encoding: [0x00,0x26,0x30,0x21]
+la $6, 0x40008000($6) # CHECK: lui $1, 16384     # encoding: [0x3c,0x01,0x40,0x00]
+                      # CHECK: ori $1, $1, 32768 # encoding: [0x34,0x21,0x80,0x00]
+                      # CHECK: addu $6, $1, $6   # encoding: [0x00,0x26,0x30,0x21]
+la $6, 0x80008000($6) # CHECK: lui $1, 32768     # encoding: [0x3c,0x01,0x80,0x00]
+                      # CHECK: ori $1, $1, 32768 # encoding: [0x34,0x21,0x80,0x00]
+                      # CHECK: addu $6, $1, $6   # encoding: [0x00,0x26,0x30,0x21]
+la $6, 0xffff4000($6) # CHECK: lui $1, 65535     # encoding: [0x3c,0x01,0xff,0xff]
+                      # CHECK: ori $1, $1, 16384 # encoding: [0x34,0x21,0x40,0x00]
+                      # CHECK: addu $6, $1, $6   # encoding: [0x00,0x26,0x30,0x21]
+la $6, 0xfffe8000($6) # CHECK: lui $1, 65534     # encoding: [0x3c,0x01,0xff,0xfe]
+                      # CHECK: ori $1, $1, 32768 # encoding: [0x34,0x21,0x80,0x00]
+                      # CHECK: addu $6, $1, $6   # encoding: [0x00,0x26,0x30,0x21]
+la $6, 0xc0008000($6) # CHECK: lui $1, 49152     # encoding: [0x3c,0x01,0xc0,0x00]
+                      # CHECK: ori $1, $1, 32768 # encoding: [0x34,0x21,0x80,0x00]
+                      # CHECK: addu $6, $1, $6   # encoding: [0x00,0x26,0x30,0x21]
+la $6, 0x80008000($6) # CHECK: lui $1, 32768     # encoding: [0x3c,0x01,0x80,0x00]
+                      # CHECK: ori $1, $1, 32768 # encoding: [0x34,0x21,0x80,0x00]
+                      # CHECK: addu $6, $1, $6   # encoding: [0x00,0x26,0x30,0x21]
+
+la $5, symbol         # CHECK: lui $5, %hi(symbol)       # encoding: [0x3c,0x05,A,A]
+                      # CHECK:                           #   fixup A - offset: 0, value: symbol@ABS_HI, kind: fixup_Mips_HI16
+                      # CHECK: addiu $5, $5, %lo(symbol) # encoding: [0x24,0xa5,A,A]
+                      # CHECK:                           #   fixup A - offset: 0, value: symbol@ABS_LO, kind: fixup_Mips_LO16
+la $5, symbol($6)     # CHECK: lui $5, %hi(symbol)       # encoding: [0x3c,0x05,A,A]
+                      # CHECK:                           #   fixup A - offset: 0, value: symbol@ABS_HI, kind: fixup_Mips_HI16
+                      # CHECK: addiu $5, $5, %lo(symbol) # encoding: [0x24,0xa5,A,A]
+                      # CHECK:                           #   fixup A - offset: 0, value: symbol@ABS_LO, kind: fixup_Mips_LO16
+                      # CHECK: addu $5, $5, $6           # encoding: [0x00,0xa6,0x28,0x21]
+la $6, symbol($6)     # CHECK: lui $1, %hi(symbol)       # encoding: [0x3c,0x01,A,A]
+                      # CHECK:                           #   fixup A - offset: 0, value: symbol@ABS_HI, kind: fixup_Mips_HI16
+                      # CHECK: addiu $1, $1, %lo(symbol) # encoding: [0x24,0x21,A,A]
+                      # CHECK:                           #   fixup A - offset: 0, value: symbol@ABS_LO, kind: fixup_Mips_LO16
+                      # CHECK: addu $6, $1, $6           # encoding: [0x00,0x26,0x30,0x21]
+la $5, 1f             # CHECK: lui $5, %hi($tmp0)        # encoding: [0x3c,0x05,A,A]
+                      # CHECK:                           #   fixup A - offset: 0, value: ($tmp0)@ABS_HI, kind: fixup_Mips_HI16
+                      # CHECK: addiu $5, $5, %lo($tmp0)  # encoding: [0x24,0xa5,A,A]
+                      # CHECK:                           #   fixup A - offset: 0, value: ($tmp0)@ABS_LO, kind: fixup_Mips_LO16
+1:
diff --git a/test/MC/Mips/macro-li-bad.s b/test/MC/Mips/macro-li-bad.s
new file mode 100644
index 00000000000..8fe622066e9
--- /dev/null
+++ b/test/MC/Mips/macro-li-bad.s
@@ -0,0 +1,11 @@
+# RUN: not llvm-mc %s -arch=mips -mcpu=mips32r2 2>%t1
+# RUN: FileCheck %s < %t1 --check-prefix=32-BIT
+# RUN: not llvm-mc %s -arch=mips64 -mcpu=mips64 -target-abi n32 2>&1 | \
+# RUN:   FileCheck %s --check-prefix=64-BIT
+# RUN: not llvm-mc %s -arch=mips64 -mcpu=mips64 -target-abi n64 2>&1 | \
+# RUN:   FileCheck %s --check-prefix=64-BIT
+
+  .text
+  li $5, 0x100000000
+  # 32-BIT: :[[@LINE-1]]:3: error: instruction requires a 32-bit immediate
+  # 64-BIT: :[[@LINE-2]]:3: error: instruction requires a 32-bit immediate
diff --git a/test/MC/Mips/macro-li.s b/test/MC/Mips/macro-li.s
new file mode 100644
index 00000000000..88e013a854e
--- /dev/null
+++ b/test/MC/Mips/macro-li.s
@@ -0,0 +1,67 @@
+# RUN: llvm-mc %s -triple=mips-unknown-linux -show-encoding -mcpu=mips32r2 | \
+# RUN:   FileCheck %s
+# RUN: llvm-mc %s -triple=mips-unknown-linux -show-encoding -mcpu=mips32r6 | \
+# RUN:   FileCheck %s
+# RUN: llvm-mc %s -triple=mips64-unknown-linux -show-encoding -mcpu=mips64r2 | \
+# RUN:   FileCheck %s
+# RUN: llvm-mc %s -triple=mips64-unknown-linux -show-encoding -mcpu=mips64r6 | \
+# RUN:   FileCheck %s
+
+li $5, 0x00000001 # CHECK: addiu $5, $zero, 1      # encoding: [0x24,0x05,0x00,0x01]
+li $5, 0x00000002 # CHECK: addiu $5, $zero, 2      # encoding: [0x24,0x05,0x00,0x02]
+li $5, 0x00004000 # CHECK: addiu $5, $zero, 16384  # encoding: [0x24,0x05,0x40,0x00]
+li $5, 0x00008000 # CHECK: ori   $5, $zero, 32768  # encoding: [0x34,0x05,0x80,0x00]
+li $5, 0xffffffff # CHECK: addiu $5, $zero, -1     # encoding: [0x24,0x05,0xff,0xff]
+li $5, 0xfffffffe # CHECK: addiu $5, $zero, -2     # encoding: [0x24,0x05,0xff,0xfe]
+li $5, 0xffffc000 # CHECK: addiu $5, $zero, -16384 # encoding: [0x24,0x05,0xc0,0x00]
+li $5, 0xffff8000 # CHECK: addiu $5, $zero, -32768 # encoding: [0x24,0x05,0x80,0x00]
+
+li $5, 0x00010000 # CHECK: lui $5, 1      # encoding: [0x3c,0x05,0x00,0x01]
+li $5, 0x00020000 # CHECK: lui $5, 2      # encoding: [0x3c,0x05,0x00,0x02]
+li $5, 0x40000000 # CHECK: lui $5, 16384  # encoding: [0x3c,0x05,0x40,0x00]
+li $5, 0x80000000 # CHECK: lui $5, 32768  # encoding: [0x3c,0x05,0x80,0x00]
+li $5, 0xffff0000 # CHECK: lui $5, 65535  # encoding: [0x3c,0x05,0xff,0xff]
+li $5, 0xfffe0000 # CHECK: lui $5, 65534  # encoding: [0x3c,0x05,0xff,0xfe]
+li $5, 0xc0000000 # CHECK: lui $5, 49152  # encoding: [0x3c,0x05,0xc0,0x00]
+li $5, 0x80000000 # CHECK: lui $5, 32768  # encoding: [0x3c,0x05,0x80,0x00]
+
+li $5, 0x00010001 # CHECK: lui $5, 1        # encoding: [0x3c,0x05,0x00,0x01]
+                  # CHECK: ori $5, $5, 1    # encoding: [0x34,0xa5,0x00,0x01]
+li $5, 0x00020001 # CHECK: lui $5, 2        # encoding: [0x3c,0x05,0x00,0x02]
+                  # CHECK: ori $5, $5, 1    # encoding: [0x34,0xa5,0x00,0x01]
+li $5, 0x40000001 # CHECK: lui $5, 16384    # encoding: [0x3c,0x05,0x40,0x00]
+                  # CHECK: ori $5, $5, 1    # encoding: [0x34,0xa5,0x00,0x01]
+li $5, 0x80000001 # CHECK: lui $5, 32768    # encoding: [0x3c,0x05,0x80,0x00]
+                  # CHECK: ori $5, $5, 1    # encoding: [0x34,0xa5,0x00,0x01]
+li $5, 0x00010002 # CHECK: lui $5, 1        # encoding: [0x3c,0x05,0x00,0x01]
+                  # CHECK: ori $5, $5, 2    # encoding: [0x34,0xa5,0x00,0x02]
+li $5, 0x00020002 # CHECK: lui $5, 2        # encoding: [0x3c,0x05,0x00,0x02]
+                  # CHECK: ori $5, $5, 2    # encoding: [0x34,0xa5,0x00,0x02]
+li $5, 0x40000002 # CHECK: lui $5, 16384    # encoding: [0x3c,0x05,0x40,0x00]
+                  # CHECK: ori $5, $5, 2    # encoding: [0x34,0xa5,0x00,0x02]
+li $5, 0x80000002 # CHECK: lui $5, 32768    # encoding: [0x3c,0x05,0x80,0x00]
+                  # CHECK: ori $5, $5, 2    # encoding: [0x34,0xa5,0x00,0x02]
+li $5, 0x00014000 # CHECK: lui $5, 1        # encoding: [0x3c,0x05,0x00,0x01]
+                  # CHECK: ori $5, $5, 16384    # encoding: [0x34,0xa5,0x40,0x00]
+li $5, 0x00024000 # CHECK: lui $5, 2            # encoding: [0x3c,0x05,0x00,0x02]
+                  # CHECK: ori $5, $5, 16384    # encoding: [0x34,0xa5,0x40,0x00]
+li $5, 0x40004000 # CHECK: lui $5, 16384        # encoding: [0x3c,0x05,0x40,0x00]
+                  # CHECK: ori $5, $5, 16384    # encoding: [0x34,0xa5,0x40,0x00]
+li $5, 0x80004000 # CHECK: lui $5, 32768        # encoding: [0x3c,0x05,0x80,0x00]
+                  # CHECK: ori $5, $5, 16384    # encoding: [0x34,0xa5,0x40,0x00]
+li $5, 0x00018000 # CHECK: lui $5, 1            # encoding: [0x3c,0x05,0x00,0x01]
+                  # CHECK: ori $5, $5, 32768    # encoding: [0x34,0xa5,0x80,0x00]
+li $5, 0x00028000 # CHECK: lui $5, 2            # encoding: [0x3c,0x05,0x00,0x02]
+                  # CHECK: ori $5, $5, 32768    # encoding: [0x34,0xa5,0x80,0x00]
+li $5, 0x40008000 # CHECK: lui $5, 16384        # encoding: [0x3c,0x05,0x40,0x00]
+                  # CHECK: ori $5, $5, 32768    # encoding: [0x34,0xa5,0x80,0x00]
+li $5, 0x80008000 # CHECK: lui $5, 32768        # encoding: [0x3c,0x05,0x80,0x00]
+                  # CHECK: ori $5, $5, 32768    # encoding: [0x34,0xa5,0x80,0x00]
+li $5, 0xffff4000 # CHECK: lui $5, 65535        # encoding: [0x3c,0x05,0xff,0xff]
+                  # CHECK: ori $5, $5, 16384    # encoding: [0x34,0xa5,0x40,0x00]
+li $5, 0xfffe8000 # CHECK: lui $5, 65534        # encoding: [0x3c,0x05,0xff,0xfe]
+                  # CHECK: ori $5, $5, 32768    # encoding: [0x34,0xa5,0x80,0x00]
+li $5, 0xc0008000 # CHECK: lui $5, 49152        # encoding: [0x3c,0x05,0xc0,0x00]
+                  # CHECK: ori $5, $5, 32768    # encoding: [0x34,0xa5,0x80,0x00]
+li $5, 0x80008000 # CHECK: lui $5, 32768        # encoding: [0x3c,0x05,0x80,0x00]
+                  # CHECK: ori $5, $5, 32768    # encoding: [0x34,0xa5,0x80,0x00]
diff --git a/test/MC/Mips/micromips-expansions.s b/test/MC/Mips/micromips-expansions.s
index af4d3b57900..48c73f34e7e 100644
--- a/test/MC/Mips/micromips-expansions.s
+++ b/test/MC/Mips/micromips-expansions.s
@@ -5,7 +5,7 @@
 #------------------------------------------------------------------------------
 # Load immediate instructions
 #------------------------------------------------------------------------------
-# CHECK: ori   $5, $zero, 123        # encoding: [0xa0,0x50,0x7b,0x00]
+# CHECK: addiu $5, $zero, 123        # encoding: [0xa0,0x30,0x7b,0x00]
 # CHECK: addiu $6, $zero, -2345      # encoding: [0xc0,0x30,0xd7,0xf6]
 # CHECK: lui   $7, 1                 # encoding: [0xa7,0x41,0x01,0x00]
 # CHECK: ori   $7, $7, 2             # encoding: [0xe7,0x50,0x02,0x00]
diff --git a/test/MC/Mips/micromips-expressions.s b/test/MC/Mips/micromips-expressions.s
deleted file mode 100644
index 509e98072bb..00000000000
--- a/test/MC/Mips/micromips-expressions.s
+++ /dev/null
@@ -1,35 +0,0 @@
-# RUN: llvm-mc  %s -triple=mipsel-unknown-linux -show-encoding \
-# RUN: -mcpu=mips32r2 -mattr=micromips | FileCheck %s
-# Check that the assembler can handle the expressions as operands.
-# CHECK:  .text
-# CHECK:  .globl  foo
-# CHECK:  foo:
-# CHECK:  lw   $4, %lo(foo)($4)    # encoding: [0x84'A',0xfc'A',0x00,0x00]
-# CHECK:                           # fixup A - offset: 0,
-# CHECK:                             value: foo@ABS_LO,
-# CHECK:                             kind: fixup_MICROMIPS_LO16
-# CHECK:  lw   $4, 56($4)          # encoding: [0x84,0xfc,0x38,0x00]
-# CHECK:  lw   $4, %lo(foo+8)($4)  # encoding: [0x84'A',0xfc'A',0x08,0x00]
-# CHECK:                           # fixup A - offset: 0,
-# CHECK:                             value: foo@ABS_LO,
-# CHECK:                             kind: fixup_MICROMIPS_LO16
-# CHECK:  lw   $4, %lo(foo+8)($4)  # encoding: [0x84'A',0xfc'A',0x08,0x00]
-# CHECK:                           # fixup A - offset: 0,
-# CHECK:                             value: foo@ABS_LO,
-# CHECK:                             kind: fixup_MICROMIPS_LO16
-# CHECK:  lw   $4, %lo(foo+8)($4)  # encoding: [0x84'A',0xfc'A',0x08,0x00]
-# CHECK:                           # fixup A - offset: 0,
-# CHECK:                             value: foo@ABS_LO,
-# CHECK:                             kind: fixup_MICROMIPS_LO16
-# CHECK:  .space  64
-
-  .globl  foo
-  .ent  foo
-foo:
-  lw  $4,%lo(foo)($4)
-  lw  $4,((10 + 4) * 4)($4)
-  lw  $4,%lo (2 * 4) + foo($4)
-  lw  $4,%lo((2 * 4) + foo)($4)
-  lw  $4,(((%lo ((2 * 4) + foo))))($4)
-  .space  64
-  .end  foo
diff --git a/test/MC/Mips/micromips-invalid.s b/test/MC/Mips/micromips-invalid.s
index 4321574b5bf..74a62ceeba0 100644
--- a/test/MC/Mips/micromips-invalid.s
+++ b/test/MC/Mips/micromips-invalid.s
@@ -73,3 +73,8 @@
   movep   $8, $6, $2, $3  # CHECK: :[[@LINE]]:{{[0-9]+}}: error: invalid operand for instruction
   movep   $5, $6, $5, $3  # CHECK: :[[@LINE]]:{{[0-9]+}}: error: invalid operand for instruction
   movep   $5, $6, $2, $9  # CHECK: :[[@LINE]]:{{[0-9]+}}: error: invalid operand for instruction
+  break 1024        # CHECK: :[[@LINE]]:{{[0-9]+}}: error: invalid operand for instruction
+  break 1024, 5     # CHECK: :[[@LINE]]:{{[0-9]+}}: error: invalid operand for instruction
+  break 7, 1024     # CHECK: :[[@LINE]]:{{[0-9]+}}: error: invalid operand for instruction
+  break 1024, 1024  # CHECK: :[[@LINE]]:{{[0-9]+}}: error: invalid operand for instruction
+  wait 1024         # CHECK: :[[@LINE]]:{{[0-9]+}}: error: invalid operand for instruction
diff --git a/test/MC/Mips/micromips32r6/invalid.s b/test/MC/Mips/micromips32r6/invalid.s
new file mode 100644
index 00000000000..8ba787ae1aa
--- /dev/null
+++ b/test/MC/Mips/micromips32r6/invalid.s
@@ -0,0 +1,6 @@
+# RUN: not llvm-mc %s -triple=mips -show-encoding -mcpu=mips32r6 -mattr=micromips 2>%t1
+# RUN: FileCheck %s < %t1
+
+  break 1024               # CHECK: :[[@LINE]]:{{[0-9]+}}: error: invalid operand for instruction
+  break 1023, 1024         # CHECK: :[[@LINE]]:{{[0-9]+}}: error: invalid operand for instruction
+  ei $32                   # CHECK: :[[@LINE]]:{{[0-9]+}}: error: invalid operand for instruction
diff --git a/test/MC/Mips/micromips32r6/valid.s b/test/MC/Mips/micromips32r6/valid.s
new file mode 100644
index 00000000000..a49622a507d
--- /dev/null
+++ b/test/MC/Mips/micromips32r6/valid.s
@@ -0,0 +1,61 @@
+# RUN: llvm-mc %s -triple=mips-unknown-linux -show-encoding -mcpu=mips32r6 -mattr=micromips | FileCheck %s
+
+  .set noat
+  add $3, $4, $5           # CHECK: add $3, $4, $5      # encoding: [0x00,0xa4,0x19,0x10]
+  addiu $3, $4, 1234       # CHECK: addiu $3, $4, 1234  # encoding: [0x30,0x64,0x04,0xd2]
+  addu $3, $4, $5          # CHECK: addu $3, $4, $5     # encoding: [0x00,0xa4,0x19,0x50]
+  addiupc $4, 100          # CHECK: addiupc $4, 100     # encoding: [0x78,0x80,0x00,0x19]
+  aluipc $3, 56            # CHECK: aluipc $3, 56       # encoding: [0x78,0x7f,0x00,0x38]
+  and $3, $4, $5           # CHECK: and $3, $4, $5      # encoding: [0x00,0xa4,0x1a,0x50]
+  andi $3, $4, 1234        # CHECK: andi $3, $4, 1234   # encoding: [0xd0,0x64,0x04,0xd2]
+  auipc $3, -1             # CHECK: auipc $3, -1        # encoding: [0x78,0x7e,0xff,0xff]
+  align $4, $2, $3, 2      # CHECK: align $4, $2, $3, 2 # encoding: [0x00,0x43,0x24,0x1f]
+  aui $3,$2,-23            # CHECK: aui $3, $2, -23     # encoding: [0x10,0x62,0xff,0xe9]
+  beqzalc $2, 1332         # CHECK: beqzalc $2, 1332    # encoding: [0x74,0x40,0x02,0x9a]
+  bnezalc $2, 1332         # CHECK: bnezalc $2, 1332    # encoding: [0x7c,0x40,0x02,0x9a]
+  bgezalc $2, 1332         # CHECK: bgezalc $2, 1332    # encoding: [0xc0,0x42,0x02,0x9a]
+  bgtzalc $2, 1332         # CHECK: bgtzalc $2, 1332    # encoding: [0xe0,0x40,0x02,0x9a]
+  bltzalc $2, 1332         # CHECK: bltzalc $2, 1332    # encoding: [0xe0,0x42,0x02,0x9a]
+  blezalc $2, 1332         # CHECK: blezalc $2, 1332    # encoding: [0xc0,0x40,0x02,0x9a]
+  balc 14572256            # CHECK: balc 14572256       # encoding: [0xb4,0x37,0x96,0xb8]
+  bc 14572256              # CHECK: bc 14572256         # encoding: [0x94,0x37,0x96,0xb8]
+  bitswap $4, $2           # CHECK: bitswap $4, $2      # encoding: [0x00,0x44,0x0b,0x3c]
+  break                    # CHECK: break               # encoding: [0x00,0x00,0x00,0x07]
+  break 7                  # CHECK: break 7             # encoding: [0x00,0x07,0x00,0x07]
+  break 7, 5               # CHECK: break 7, 5          # encoding: [0x00,0x07,0x01,0x47]
+  cache 1, 8($5)           # CHECK: cache 1, 8($5)      # encoding: [0x20,0x25,0x60,0x08]
+  clo $11, $a1             # CHECK: clo $11, $5         # encoding: [0x01,0x65,0x4b,0x3c]
+  clz $sp, $gp             # CHECK: clz $sp, $gp        # encoding: [0x03,0x80,0xe8,0x50]
+  div $3, $4, $5           # CHECK: div $3, $4, $5      # encoding: [0x00,0xa4,0x19,0x18]
+  divu $3, $4, $5          # CHECK: divu $3, $4, $5     # encoding: [0x00,0xa4,0x19,0x98]
+  ehb                      # CHECK: ehb                 # encoding: [0x00,0x00,0x18,0x00]
+  ei                       # CHECK: ei                  # encoding: [0x00,0x00,0x57,0x7c]
+  ei $0                    # CHECK: ei                  # encoding: [0x00,0x00,0x57,0x7c]
+  ei $10                   # CHECK: ei $10              # encoding: [0x00,0x0a,0x57,0x7c]
+  eret                     # CHECK: eret                # encoding: [0x00,0x00,0xf3,0x7c]
+  eretnc                   # CHECK: eretnc              # encoding: [0x00,0x01,0xf3,0x7c]
+  jialc $5, 256            # CHECK: jialc $5, 256       # encoding: [0x80,0x05,0x01,0x00]
+  jic   $5, 256            # CHECK: jic $5, 256         # encoding: [0xa0,0x05,0x01,0x00]
+  lsa $2, $3, $4, 3        # CHECK: lsa  $2, $3, $4, 3  # encoding: [0x00,0x43,0x26,0x0f]
+  lwpc    $2,268           # CHECK: lwpc $2, 268        # encoding: [0x78,0x48,0x00,0x43]
+  mod $3, $4, $5           # CHECK: mod $3, $4, $5      # encoding: [0x00,0xa4,0x19,0x58]
+  modu $3, $4, $5          # CHECK: modu $3, $4, $5     # encoding: [0x00,0xa4,0x19,0xd8]
+  mul $3, $4, $5           # CHECK mul $3, $4, $5       # encoding: [0x00,0xa4,0x18,0x18]
+  muh $3, $4, $5           # CHECK muh $3, $4, $5       # encoding: [0x00,0xa4,0x18,0x58]
+  mulu $3, $4, $5          # CHECK mulu $3, $4, $5      # encoding: [0x00,0xa4,0x18,0x98]
+  muhu $3, $4, $5          # CHECK muhu $3, $4, $5      # encoding: [0x00,0xa4,0x18,0xd8]
+  nop                      # CHECK: nop                 # encoding: [0x00,0x00,0x00,0x00]
+  nor $3, $4, $5           # CHECK: nor $3, $4, $5      # encoding: [0x00,0xa4,0x1a,0xd0]
+  or $3, $4, $5            # CHECK: or $3, $4, $5       # encoding: [0x00,0xa4,0x1a,0x90]
+  ori $3, $4, 1234         # CHECK: ori $3, $4, 1234    # encoding: [0x50,0x64,0x04,0xd2]
+  pref 1, 8($5)            # CHECK: pref 1, 8($5)       # encoding: [0x60,0x25,0x20,0x08]
+  seb $3, $4               # CHECK: seb $3, $4          # encoding: [0x00,0x64,0x2b,0x3c]
+  seh $3, $4               # CHECK: seh $3, $4          # encoding: [0x00,0x64,0x3b,0x3c]
+  seleqz $2,$3,$4          # CHECK: seleqz $2, $3, $4   # encoding: [0x00,0x83,0x11,0x40]
+  selnez $2,$3,$4          # CHECK: selnez $2, $3, $4   # encoding: [0x00,0x83,0x11,0x80]
+  sll $4, $3, 7            # CHECK: sll $4, $3, 7       # encoding: [0x00,0x83,0x38,0x00]
+  sub $3, $4, $5           # CHECK: sub $3, $4, $5      # encoding: [0x00,0xa4,0x19,0x90]
+  subu $3, $4, $5          # CHECK: subu $3, $4, $5     # encoding: [0x00,0xa4,0x19,0xd0]
+  xor $3, $4, $5           # CHECK: xor $3, $4, $5      # encoding: [0x00,0xa4,0x1b,0x10]
+  xori $3, $4, 1234        # CHECK: xori $3, $4, 1234   # encoding: [0x70,0x64,0x04,0xd2]
+
diff --git a/test/MC/Mips/mips-cop0-reginfo.s b/test/MC/Mips/mips-cop0-reginfo.s
new file mode 100644
index 00000000000..0508a372954
--- /dev/null
+++ b/test/MC/Mips/mips-cop0-reginfo.s
@@ -0,0 +1,28 @@
+# RUN: llvm-mc -arch=mips -mcpu=mips32r2 -filetype=obj %s -o - | \
+# RUN:   llvm-readobj -sections -section-data - | \
+# RUN:     FileCheck %s -check-prefix=CHECK
+	mfc0	$16, $15, 1
+	mfc0	$16, $16, 1
+
+
+# Checking for the coprocessor 0's register usage was recorded
+# and emitted.
+# CHECK:  Section {
+# CHECK:     Index: 5
+# CHECK:     Name: .reginfo (27)
+# CHECK:     Type: SHT_MIPS_REGINFO (0x70000006)
+# CHECK:     Flags [ (0x2)
+# CHECK:       SHF_ALLOC (0x2)
+# CHECK:     ]
+# CHECK:     Address: 0x0
+# CHECK:     Offset: 0x50
+# CHECK:     Size: 24
+# CHECK:     Link: 0
+# CHECK:     Info: 0
+# CHECK:     AddressAlignment: 4
+# CHECK:     EntrySize: 24
+# CHECK:     SectionData (
+# CHECK:       0000: 00010000 00018000 00000000 00000000  |................|
+# CHECK:       0010: 00000000 00000000                    |........|
+# CHECK:     )
+# CHECK:   }
diff --git a/test/MC/Mips/mips-data-directives.s b/test/MC/Mips/mips-data-directives.s
index 8b3e0b3aa19..abff7c21c61 100644
--- a/test/MC/Mips/mips-data-directives.s
+++ b/test/MC/Mips/mips-data-directives.s
@@ -12,7 +12,7 @@
 
 # Checking if the data and reloations were correctly emitted
 # CHECK-OBJ:  Section {
-# CHECK-OBJ:    Name: .data (66)
+# CHECK-OBJ:    Name: .data
 # CHECK-OBJ:    SectionData (
 # CHECK-OBJ:      0000: DEADC0DE DEADC0DE DEADBEEF 00000000
 # CHECK-OBJ:      0010: 00000000 00000000
@@ -20,7 +20,7 @@
 # CHECK-OBJ:  }
 
 # CHECK-OBJ:  Section {
-# CHECK-OBJ:    Name: .rel.data (62)
+# CHECK-OBJ:    Name: .rel.data
 # CHECK-OBJ:    Relocations [
 # CHECK-OBJ:      0xC R_MIPS_32 .data 0x0
 # CHECK-OBJ:      0x10 R_MIPS_64 .data 0x0
diff --git a/test/MC/Mips/mips-expansions-bad.s b/test/MC/Mips/mips-expansions-bad.s
index d9bac20e154..cd74f7d4aa8 100644
--- a/test/MC/Mips/mips-expansions-bad.s
+++ b/test/MC/Mips/mips-expansions-bad.s
@@ -1,8 +1,39 @@
 # RUN: not llvm-mc %s -arch=mips -mcpu=mips32r2 2>%t1
-# RUN: FileCheck %s < %t1
+# RUN: FileCheck %s < %t1 --check-prefix=32-BIT
+# RUN: not llvm-mc %s -arch=mips64 -mcpu=mips64 -target-abi n32 2>&1 | \
+# RUN:   FileCheck %s --check-prefix=64-BIT --check-prefix=N32-ONLY
+# RUN: not llvm-mc %s -arch=mips64 -mcpu=mips64 -target-abi n64 2>&1 | \
+# RUN:   FileCheck %s --check-prefix=64-BIT --check-prefix=N64-ONLY
 
   .text
-  li $5, 0x100000000
-  # CHECK: :[[@LINE-1]]:3: error: instruction requires a 64-bit architecture
   dli $5, 1
-  # CHECK: :[[@LINE-1]]:3: error: instruction requires a 64-bit architecture
+  # 32-BIT: :[[@LINE-1]]:3: error: instruction requires a 64-bit architecture
+  bne $2, 0x100010001, 1332
+  # 32-BIT: :[[@LINE-1]]:3: error: instruction requires a 32-bit immediate
+  beq $2, 0x100010001, 1332
+  # 32-BIT: :[[@LINE-1]]:3: error: instruction requires a 32-bit immediate
+  .set mips32r6
+  ulhu $5, 0
+  # 32-BIT: :[[@LINE-1]]:3: error: instruction not supported on mips32r6 or mips64r6
+  # 64-BIT: :[[@LINE-2]]:3: error: instruction not supported on mips32r6 or mips64r6
+  .set mips32
+  ulhu $5, 1
+  # 32-BIT-NOT: :[[@LINE-1]]:3: error: instruction not supported on mips32r6 or mips64r6
+  # 64-BIT-NOT: :[[@LINE-2]]:3: error: instruction not supported on mips32r6 or mips64r6
+  .set mips64r6
+  ulhu $5, 2
+  # 32-BIT: :[[@LINE-1]]:3: error: instruction not supported on mips32r6 or mips64r6
+  # 64-BIT: :[[@LINE-2]]:3: error: instruction not supported on mips32r6 or mips64r6
+
+  .set mips32r6
+  ulw $5, 0
+  # 32-BIT: :[[@LINE-1]]:3: error: instruction not supported on mips32r6 or mips64r6
+  # 64-BIT: :[[@LINE-2]]:3: error: instruction not supported on mips32r6 or mips64r6
+  .set mips32
+  ulw $5, 1
+  # 32-BIT-NOT: :[[@LINE-1]]:3: error: instruction not supported on mips32r6 or mips64r6
+  # 64-BIT-NOT: :[[@LINE-2]]:3: error: instruction not supported on mips32r6 or mips64r6
+  .set mips64r6
+  ulw $5, 2
+  # 32-BIT: :[[@LINE-1]]:3: error: instruction not supported on mips32r6 or mips64r6
+  # 64-BIT: :[[@LINE-2]]:3: error: instruction not supported on mips32r6 or mips64r6
diff --git a/test/MC/Mips/mips-expansions.s b/test/MC/Mips/mips-expansions.s
index 490b8147413..93c6b7cd75a 100644
--- a/test/MC/Mips/mips-expansions.s
+++ b/test/MC/Mips/mips-expansions.s
@@ -1,95 +1,471 @@
-# RUN: llvm-mc %s -triple=mipsel-unknown-linux -show-encoding -mcpu=mips32r2 | FileCheck %s
-# Check that the assembler can handle the documented syntax
-# for macro instructions
-#------------------------------------------------------------------------------
-# Load immediate instructions
-#------------------------------------------------------------------------------
-# CHECK: ori     $5, $zero, 123      # encoding: [0x7b,0x00,0x05,0x34]
-# CHECK: addiu   $6, $zero, -2345    # encoding: [0xd7,0xf6,0x06,0x24]
-# CHECK: lui     $7, 1               # encoding: [0x01,0x00,0x07,0x3c]
-# CHECK: ori     $7, $7, 2           # encoding: [0x02,0x00,0xe7,0x34]
-# CHECK: addiu   $8, $zero, -8       # encoding: [0xf8,0xff,0x08,0x24]
-
-# CHECK: addiu   $4, $zero, 20       # encoding: [0x14,0x00,0x04,0x24]
-# CHECK: lui     $7, 1               # encoding: [0x01,0x00,0x07,0x3c]
-# CHECK: ori     $7, $7, 2           # encoding: [0x02,0x00,0xe7,0x34]
-# CHECK: addiu   $4, $5, 20          # encoding: [0x14,0x00,0xa4,0x24]
-# CHECK: lui     $7, 1               # encoding: [0x01,0x00,0x07,0x3c]
-# CHECK: ori     $7, $7, 2           # encoding: [0x02,0x00,0xe7,0x34]
-# CHECK: addu    $7, $7, $8          # encoding: [0x21,0x38,0xe8,0x00]
-# CHECK: lui     $8, %hi(symbol)     # encoding: [A,A,0x08,0x3c]
-                                     #   fixup A - offset: 0, value: symbol@ABS_HI, kind: fixup_Mips_HI16
-# CHECK: ori     $8, $8, %lo(symbol) # encoding: [A,A,0x08,0x35]
-                                     #   fixup A - offset: 0, value: symbol@ABS_LO, kind: fixup_Mips_LO16
-# CHECK: .set    mips64
-# CHECK: lui     $8, %highest(symbol)    # encoding: [A,A,0x08,0x3c]
-                                         #   fixup A - offset: 0, value: symbol@HIGHEST, kind: fixup_Mips_HIGHEST
-# CHECK: ori     $8, $8, %higher(symbol) # encoding: [A,A,0x08,0x35]
-                                         #   fixup A - offset: 0, value: symbol@HIGHER, kind: fixup_Mips_HIGHER
-# CHECK: dsll    $8, $8, 16              # encoding: [0x38,0x44,0x08,0x00]
-# CHECK: ori     $8, $8, %hi(symbol)     # encoding: [A,A,0x08,0x35]
-                                         #   fixup A - offset: 0, value: symbol@ABS_HI, kind: fixup_Mips_HI16
-# CHECK: dsll    $8, $8, 16              # encoding: [0x38,0x44,0x08,0x00]
-# CHECK: ori     $8, $8, %lo(symbol)     # encoding: [A,A,0x08,0x35]
-                                         #   fixup A - offset: 0, value: symbol@ABS_LO, kind: fixup_Mips_LO16
-# CHECK: .set    mips32r2
-# CHECK: lui     $10, %hi(symbol)        # encoding: [A,A,0x0a,0x3c]
-# CHECK:                                 #   fixup A - offset: 0, value: symbol@ABS_HI, kind: fixup_Mips_HI16
-# CHECK: addu    $10, $10, $4            # encoding: [0x21,0x50,0x44,0x01]
-# CHECK: lw      $10, %lo(symbol)($10)   # encoding: [A,A,0x4a,0x8d]
-# CHECK:                                 #   fixup A - offset: 0, value: symbol@ABS_LO, kind: fixup_Mips_LO16
-# CHECK: lui     $1, %hi(symbol)         # encoding: [A,A,0x01,0x3c]
-# CHECK:                                 #   fixup A - offset: 0, value: symbol@ABS_HI, kind: fixup_Mips_HI16
-# CHECK: addu    $1, $1, $9              # encoding: [0x21,0x08,0x29,0x00]
-# CHECK: sw      $10, %lo(symbol)($1)    # encoding: [A,A,0x2a,0xac]
-# CHECK:                                 #   fixup A - offset: 0, value: symbol@ABS_LO, kind: fixup_Mips_LO16
-# CHECK: lui     $10, 10                 # encoding: [0x0a,0x00,0x0a,0x3c]
-# CHECK: addu    $10, $10, $4            # encoding: [0x21,0x50,0x44,0x01]
-# CHECK: lw      $10, 123($10)           # encoding: [0x7b,0x00,0x4a,0x8d]
-# CHECK: lui     $1, 2                   # encoding: [0x02,0x00,0x01,0x3c]
-# CHECK: addu    $1, $1, $9              # encoding: [0x21,0x08,0x29,0x00]
-# CHECK: sw      $10, 57920($1)          # encoding: [0x40,0xe2,0x2a,0xac]
-
-# CHECK:     lui     $8, %hi(symbol)     # encoding: [A,A,0x08,0x3c]
-# CHECK:                                 #   fixup A - offset: 0, value: symbol@ABS_HI, kind: fixup_Mips_HI16
-# CHECK-NOT: move    $8, $8              # encoding: [0x21,0x40,0x00,0x01]
-# CHECK:     lw      $8, %lo(symbol)($8) # encoding: [A,A,0x08,0x8d]
-# CHECK:                                 #   fixup A - offset: 0, value: symbol@ABS_LO, kind: fixup_Mips_LO16
-# CHECK:     lui     $1, %hi(symbol)     # encoding: [A,A,0x01,0x3c]
-# CHECK:                                 #   fixup A - offset: 0, value: symbol@ABS_HI, kind: fixup_Mips_HI16
-# CHECK-NOT: move    $1, $1              # encoding: [0x21,0x08,0x20,0x00]
-# CHECK:     sw      $8, %lo(symbol)($1) # encoding: [A,A,0x28,0xac]
-# CHECK:                                 #   fixup A - offset: 0, value: symbol@ABS_LO, kind: fixup_Mips_LO16
-
-# CHECK: lui     $1, %hi(symbol)
-# CHECK: ldc1    $f0, %lo(symbol)($1)
-# CHECK: lui     $1, %hi(symbol)
-# CHECK: sdc1    $f0, %lo(symbol)($1)
-
-    li $5,123
-    li $6,-2345
-    li $7,65538
-    li $8, ~7
-
-    la $a0, 20
-    la $7,65538
-    la $a0, 20($a1)
-    la $7,65538($8)
-    la $t0, symbol
-    .set mips64
-    la $t0, symbol
-    .set mips32r2
-
-    .set noat
-    lw  $t2, symbol($a0)
-    .set at
-    sw  $t2, symbol($t1)
-
-    lw  $t2, 655483($a0)
-    sw  $t2, 123456($t1)
-
-    lw  $8, symbol
-    sw  $8, symbol
-
-    ldc1 $f0, symbol
-    sdc1 $f0, symbol
+# RUN: llvm-mc %s -triple=mipsel-unknown-linux -show-encoding -mcpu=mips32r2 | \
+# RUN:   FileCheck %s --check-prefix=CHECK-LE
+# RUN: llvm-mc %s -triple=mips-unknown-linux -show-encoding -mcpu=mips32r2 | \
+# RUN:   FileCheck %s --check-prefix=CHECK-BE
+
+# Check that the IAS expands macro instructions in the same way as GAS.
+
+# Load address, done by MipsAsmParser::expandLoadAddressReg()
+# and MipsAsmParser::expandLoadAddressImm():
+  la $8, 1f
+# CHECK-LE: lui     $8, %hi($tmp0)      # encoding: [A,A,0x08,0x3c]
+# CHECK-LE:                             #   fixup A - offset: 0, value: ($tmp0)@ABS_HI, kind: fixup_Mips_HI16
+# CHECK-LE: addiu   $8, $8, %lo($tmp0)  # encoding: [A,A,0x08,0x25]
+# CHECK-LE:                             #   fixup A - offset: 0, value: ($tmp0)@ABS_LO, kind: fixup_Mips_LO16
+
+# LW/SW and LDC1/SDC1 of symbol address, done by MipsAsmParser::expandMemInst():
+  .set noat
+  lw $10, symbol($4)
+# CHECK-LE: lui     $10, %hi(symbol)        # encoding: [A,A,0x0a,0x3c]
+# CHECK-LE:                                 #   fixup A - offset: 0, value: symbol@ABS_HI, kind: fixup_Mips_HI16
+# CHECK-LE: addu    $10, $10, $4            # encoding: [0x21,0x50,0x44,0x01]
+# CHECK-LE: lw      $10, %lo(symbol)($10)   # encoding: [A,A,0x4a,0x8d]
+# CHECK-LE:                                 #   fixup A - offset: 0, value: symbol@ABS_LO, kind: fixup_Mips_LO16
+  .set at
+  sw $10, symbol($9)
+# CHECK-LE: lui     $1, %hi(symbol)         # encoding: [A,A,0x01,0x3c]
+# CHECK-LE:                                 #   fixup A - offset: 0, value: symbol@ABS_HI, kind: fixup_Mips_HI16
+# CHECK-LE: addu    $1, $1, $9              # encoding: [0x21,0x08,0x29,0x00]
+# CHECK-LE: sw      $10, %lo(symbol)($1)    # encoding: [A,A,0x2a,0xac]
+# CHECK-LE:                                 #   fixup A - offset: 0, value: symbol@ABS_LO, kind: fixup_Mips_LO16
+
+  lw $8, 1f
+# CHECK-LE: lui $8, %hi($tmp0)              # encoding: [A,A,0x08,0x3c]
+# CHECK-LE:                                 #   fixup A - offset: 0, value: ($tmp0)@ABS_HI, kind: fixup_Mips_HI16
+# CHECK-LE: lw  $8, %lo($tmp0)($8)          # encoding: [A,A,0x08,0x8d]
+# CHECK-LE:                                 #   fixup A - offset: 0, value: ($tmp0)@ABS_LO, kind: fixup_Mips_LO16
+  sw $8, 1f
+# CHECK-LE: lui $1, %hi($tmp0)              # encoding: [A,A,0x01,0x3c]
+# CHECK-LE:                                 #   fixup A - offset: 0, value: ($tmp0)@ABS_HI, kind: fixup_Mips_HI16
+# CHECK-LE: sw  $8, %lo($tmp0)($1)          # encoding: [A,A,0x28,0xac]
+# CHECK-LE:                                 #   fixup A - offset: 0, value: ($tmp0)@ABS_LO, kind: fixup_Mips_LO16
+
+  lw $10, 655483($4)
+# CHECK-LE: lui     $10, 10                 # encoding: [0x0a,0x00,0x0a,0x3c]
+# CHECK-LE: addu    $10, $10, $4            # encoding: [0x21,0x50,0x44,0x01]
+# CHECK-LE: lw      $10, 123($10)           # encoding: [0x7b,0x00,0x4a,0x8d]
+  sw $10, 123456($9)
+# CHECK-LE: lui     $1, 2                   # encoding: [0x02,0x00,0x01,0x3c]
+# CHECK-LE: addu    $1, $1, $9              # encoding: [0x21,0x08,0x29,0x00]
+# CHECK-LE: sw      $10, 57920($1)          # encoding: [0x40,0xe2,0x2a,0xac]
+
+  lw $8, symbol
+# CHECK-LE:     lui     $8, %hi(symbol)     # encoding: [A,A,0x08,0x3c]
+# CHECK-LE:                                 #   fixup A - offset: 0, value: symbol@ABS_HI, kind: fixup_Mips_HI16
+# CHECK-LE-NOT: move    $8, $8              # encoding: [0x21,0x40,0x00,0x01]
+# CHECK-LE:     lw      $8, %lo(symbol)($8) # encoding: [A,A,0x08,0x8d]
+# CHECK-LE:                                 #   fixup A - offset: 0, value: symbol@ABS_LO, kind: fixup_Mips_LO16
+  sw $8, symbol
+# CHECK-LE:     lui     $1, %hi(symbol)     # encoding: [A,A,0x01,0x3c]
+# CHECK-LE:                                 #   fixup A - offset: 0, value: symbol@ABS_HI, kind: fixup_Mips_HI16
+# CHECK-LE-NOT: move    $1, $1              # encoding: [0x21,0x08,0x20,0x00]
+# CHECK-LE:     sw      $8, %lo(symbol)($1) # encoding: [A,A,0x28,0xac]
+# CHECK-LE:                                 #   fixup A - offset: 0, value: symbol@ABS_LO, kind: fixup_Mips_LO16
+
+  ldc1 $f0, symbol
+# CHECK-LE: lui     $1, %hi(symbol)
+# CHECK-LE: ldc1    $f0, %lo(symbol)($1)
+  sdc1 $f0, symbol
+# CHECK-LE: lui     $1, %hi(symbol)
+# CHECK-LE: sdc1    $f0, %lo(symbol)($1)
+
+# Test BNE with an immediate as the 2nd operand.
+  bne $2, 0, 1332
+# CHECK-LE: bnez  $2, 1332          # encoding: [0x4d,0x01,0x40,0x14]
+# CHECK-LE: nop                     # encoding: [0x00,0x00,0x00,0x00]
+
+  bne $2, 123, 1332
+# CHECK-LE: addiu $1, $zero, 123    # encoding: [0x7b,0x00,0x01,0x24]
+# CHECK-LE: bne   $2, $1, 1332      # encoding: [0x4d,0x01,0x41,0x14]
+# CHECK-LE: nop                     # encoding: [0x00,0x00,0x00,0x00]
+
+  bne $2, -2345, 1332
+# CHECK-LE: addiu $1, $zero, -2345  # encoding: [0xd7,0xf6,0x01,0x24]
+# CHECK-LE: bne   $2, $1, 1332      # encoding: [0x4d,0x01,0x41,0x14]
+# CHECK-LE: nop                     # encoding: [0x00,0x00,0x00,0x00]
+
+  bne $2, 65538, 1332
+# CHECK-LE: lui   $1, 1             # encoding: [0x01,0x00,0x01,0x3c]
+# CHECK-LE: ori   $1, $1, 2         # encoding: [0x02,0x00,0x21,0x34]
+# CHECK-LE: bne   $2, $1, 1332      # encoding: [0x4d,0x01,0x41,0x14]
+# CHECK-LE: nop                     # encoding: [0x00,0x00,0x00,0x00]
+
+  bne $2, ~7, 1332
+# CHECK-LE: addiu $1, $zero, -8     # encoding: [0xf8,0xff,0x01,0x24]
+# CHECK-LE: bne   $2, $1, 1332      # encoding: [0x4d,0x01,0x41,0x14]
+# CHECK-LE: nop                     # encoding: [0x00,0x00,0x00,0x00]
+
+  bne $2, 0x10000, 1332
+# CHECK-LE: lui   $1, 1             # encoding: [0x01,0x00,0x01,0x3c]
+# CHECK-LE: bne   $2, $1, 1332      # encoding: [0x4d,0x01,0x41,0x14]
+# CHECK-LE: nop                     # encoding: [0x00,0x00,0x00,0x00]
+
+# Test BEQ with an immediate as the 2nd operand.
+  beq $2, 0, 1332
+# CHECK-LE: beqz  $2, 1332          # encoding: [0x4d,0x01,0x40,0x10]
+# CHECK-LE: nop                     # encoding: [0x00,0x00,0x00,0x00]
+
+  beq $2, 123, 1332
+# CHECK-LE: addiu $1, $zero, 123    # encoding: [0x7b,0x00,0x01,0x24]
+# CHECK-LE: beq   $2, $1, 1332      # encoding: [0x4d,0x01,0x41,0x10]
+# CHECK-LE: nop                     # encoding: [0x00,0x00,0x00,0x00]
+
+  beq $2, -2345, 1332
+# CHECK-LE: addiu $1, $zero, -2345  # encoding: [0xd7,0xf6,0x01,0x24]
+# CHECK-LE: beq   $2, $1, 1332      # encoding: [0x4d,0x01,0x41,0x10]
+# CHECK-LE: nop                     # encoding: [0x00,0x00,0x00,0x00]
+
+  beq $2, 65538, 1332
+# CHECK-LE: lui   $1, 1             # encoding: [0x01,0x00,0x01,0x3c]
+# CHECK-LE: ori   $1, $1, 2         # encoding: [0x02,0x00,0x21,0x34]
+# CHECK-LE: beq   $2, $1, 1332      # encoding: [0x4d,0x01,0x41,0x10]
+# CHECK-LE: nop                     # encoding: [0x00,0x00,0x00,0x00]
+
+  beq $2, ~7, 1332
+# CHECK-LE: addiu $1, $zero, -8     # encoding: [0xf8,0xff,0x01,0x24]
+# CHECK-LE: beq   $2, $1, 1332      # encoding: [0x4d,0x01,0x41,0x10]
+# CHECK-LE: nop                     # encoding: [0x00,0x00,0x00,0x00]
+
+  beq $2, 0x10000, 1332
+# CHECK-LE: lui   $1, 1             # encoding: [0x01,0x00,0x01,0x3c]
+# CHECK-LE: beq   $2, $1, 1332      # encoding: [0x4d,0x01,0x41,0x10]
+# CHECK-LE: nop                     # encoding: [0x00,0x00,0x00,0x00]
+
+# Test ULHU with immediate operand.
+  ulhu $8, 0
+# CHECK-BE: lbu  $1, 0($zero)      # encoding: [0x90,0x01,0x00,0x00]
+# CHECK-BE: lbu  $8, 1($zero)      # encoding: [0x90,0x08,0x00,0x01]
+# CHECK-BE: sll  $1, $1, 8         # encoding: [0x00,0x01,0x0a,0x00]
+# CHECK-BE: or   $8, $8, $1        # encoding: [0x01,0x01,0x40,0x25]
+# CHECK-LE: lbu  $1, 1($zero)      # encoding: [0x01,0x00,0x01,0x90]
+# CHECK-LE: lbu  $8, 0($zero)      # encoding: [0x00,0x00,0x08,0x90]
+# CHECK-LE: sll  $1, $1, 8         # encoding: [0x00,0x0a,0x01,0x00]
+# CHECK-LE: or   $8, $8, $1        # encoding: [0x25,0x40,0x01,0x01]
+
+  ulhu $8, 2
+# CHECK-BE: lbu  $1, 2($zero)      # encoding: [0x90,0x01,0x00,0x02]
+# CHECK-BE: lbu  $8, 3($zero)      # encoding: [0x90,0x08,0x00,0x03]
+# CHECK-BE: sll  $1, $1, 8         # encoding: [0x00,0x01,0x0a,0x00]
+# CHECK-BE: or   $8, $8, $1        # encoding: [0x01,0x01,0x40,0x25]
+# CHECK-LE: lbu  $1, 3($zero)      # encoding: [0x03,0x00,0x01,0x90]
+# CHECK-LE: lbu  $8, 2($zero)      # encoding: [0x02,0x00,0x08,0x90]
+# CHECK-LE: sll  $1, $1, 8         # encoding: [0x00,0x0a,0x01,0x00]
+# CHECK-LE: or   $8, $8, $1        # encoding: [0x25,0x40,0x01,0x01]
+
+  ulhu $8, 0x8000
+# CHECK-BE: ori  $1, $zero, 32768  # encoding: [0x34,0x01,0x80,0x00]
+# CHECK-BE: lbu  $8, 0($1)         # encoding: [0x90,0x28,0x00,0x00]
+# CHECK-BE: lbu  $1, 1($1)         # encoding: [0x90,0x21,0x00,0x01]
+# CHECK-BE: sll  $8, $8, 8         # encoding: [0x00,0x08,0x42,0x00]
+# CHECK-BE: or   $8, $8, $1        # encoding: [0x01,0x01,0x40,0x25]
+# CHECK-LE: ori  $1, $zero, 32768  # encoding: [0x00,0x80,0x01,0x34]
+# CHECK-LE: lbu  $8, 1($1)         # encoding: [0x01,0x00,0x28,0x90]
+# CHECK-LE: lbu  $1, 0($1)         # encoding: [0x00,0x00,0x21,0x90]
+# CHECK-LE: sll  $8, $8, 8         # encoding: [0x00,0x42,0x08,0x00]
+# CHECK-LE: or   $8, $8, $1        # encoding: [0x25,0x40,0x01,0x01]
+
+  ulhu $8, -0x8000
+# CHECK-BE: lbu  $1, -32768($zero) # encoding: [0x90,0x01,0x80,0x00]
+# CHECK-BE: lbu  $8, -32767($zero) # encoding: [0x90,0x08,0x80,0x01]
+# CHECK-BE: sll  $1, $1, 8         # encoding: [0x00,0x01,0x0a,0x00]
+# CHECK-BE: or   $8, $8, $1        # encoding: [0x01,0x01,0x40,0x25]
+# CHECK-LE: lbu  $1, -32767($zero) # encoding: [0x01,0x80,0x01,0x90]
+# CHECK-LE: lbu  $8, -32768($zero) # encoding: [0x00,0x80,0x08,0x90]
+# CHECK-LE: sll  $1, $1, 8         # encoding: [0x00,0x0a,0x01,0x00]
+# CHECK-LE: or   $8, $8, $1        # encoding: [0x25,0x40,0x01,0x01]
+
+  ulhu $8, 0x10000
+# CHECK-BE: lui  $1, 1             # encoding: [0x3c,0x01,0x00,0x01]
+# CHECK-BE: lbu  $8, 0($1)         # encoding: [0x90,0x28,0x00,0x00]
+# CHECK-BE: lbu  $1, 1($1)         # encoding: [0x90,0x21,0x00,0x01]
+# CHECK-BE: sll  $8, $8, 8         # encoding: [0x00,0x08,0x42,0x00]
+# CHECK-BE: or   $8, $8, $1        # encoding: [0x01,0x01,0x40,0x25]
+# CHECK-LE: lui  $1, 1             # encoding: [0x01,0x00,0x01,0x3c]
+# CHECK-LE: lbu  $8, 1($1)         # encoding: [0x01,0x00,0x28,0x90]
+# CHECK-LE: lbu  $1, 0($1)         # encoding: [0x00,0x00,0x21,0x90]
+# CHECK-LE: sll  $8, $8, 8         # encoding: [0x00,0x42,0x08,0x00]
+# CHECK-LE: or   $8, $8, $1        # encoding: [0x25,0x40,0x01,0x01]
+
+  ulhu $8, 0x18888
+# CHECK-BE: lui  $1, 1             # encoding: [0x3c,0x01,0x00,0x01]
+# CHECK-BE: ori  $1, $1, 34952     # encoding: [0x34,0x21,0x88,0x88]
+# CHECK-BE: lbu  $8, 0($1)         # encoding: [0x90,0x28,0x00,0x00]
+# CHECK-BE: lbu  $1, 1($1)         # encoding: [0x90,0x21,0x00,0x01]
+# CHECK-BE: sll  $8, $8, 8         # encoding: [0x00,0x08,0x42,0x00]
+# CHECK-BE: or   $8, $8, $1        # encoding: [0x01,0x01,0x40,0x25]
+# CHECK-LE: lui  $1, 1             # encoding: [0x01,0x00,0x01,0x3c]
+# CHECK-LE: ori  $1, $1, 34952     # encoding: [0x88,0x88,0x21,0x34]
+# CHECK-LE: lbu  $8, 1($1)         # encoding: [0x01,0x00,0x28,0x90]
+# CHECK-LE: lbu  $1, 0($1)         # encoding: [0x00,0x00,0x21,0x90]
+# CHECK-LE: sll  $8, $8, 8         # encoding: [0x00,0x42,0x08,0x00]
+# CHECK-LE: or   $8, $8, $1        # encoding: [0x25,0x40,0x01,0x01]
+
+  ulhu $8, -32769
+# CHECK-BE: lui  $1, 65535         # encoding: [0x3c,0x01,0xff,0xff]
+# CHECK-BE: ori  $1, $1, 32767     # encoding: [0x34,0x21,0x7f,0xff]
+# CHECK-BE: lbu  $8, 0($1)         # encoding: [0x90,0x28,0x00,0x00]
+# CHECK-BE: lbu  $1, 1($1)         # encoding: [0x90,0x21,0x00,0x01]
+# CHECK-BE: sll  $8, $8, 8         # encoding: [0x00,0x08,0x42,0x00]
+# CHECK-BE: or   $8, $8, $1        # encoding: [0x01,0x01,0x40,0x25]
+# CHECK-LE: lui  $1, 65535         # encoding: [0xff,0xff,0x01,0x3c]
+# CHECK-LE: ori  $1, $1, 32767     # encoding: [0xff,0x7f,0x21,0x34]
+# CHECK-LE: lbu  $8, 1($1)         # encoding: [0x01,0x00,0x28,0x90]
+# CHECK-LE: lbu  $1, 0($1)         # encoding: [0x00,0x00,0x21,0x90]
+# CHECK-LE: sll  $8, $8, 8         # encoding: [0x00,0x42,0x08,0x00]
+# CHECK-LE: or   $8, $8, $1        # encoding: [0x25,0x40,0x01,0x01]
+
+  ulhu $8, 32767
+# CHECK-BE: addiu $1, $zero, 32767  # encoding: [0x24,0x01,0x7f,0xff]
+# CHECK-BE: lbu  $8, 0($1)          # encoding: [0x90,0x28,0x00,0x00]
+# CHECK-BE: lbu  $1, 1($1)          # encoding: [0x90,0x21,0x00,0x01]
+# CHECK-BE: sll  $8, $8, 8          # encoding: [0x00,0x08,0x42,0x00]
+# CHECK-BE: or   $8, $8, $1         # encoding: [0x01,0x01,0x40,0x25]
+# CHECK-LE: addiu $1, $zero, 32767  # encoding: [0xff,0x7f,0x01,0x24]
+# CHECK-LE: lbu  $8, 1($1)          # encoding: [0x01,0x00,0x28,0x90]
+# CHECK-LE: lbu  $1, 0($1)          # encoding: [0x00,0x00,0x21,0x90]
+# CHECK-LE: sll  $8, $8, 8          # encoding: [0x00,0x42,0x08,0x00]
+# CHECK-LE: or   $8, $8, $1         # encoding: [0x25,0x40,0x01,0x01]
+
+# Test ULHU with immediate offset and a source register operand.
+  ulhu $8, 0($9)
+# CHECK-BE: lbu  $1, 0($9)         # encoding: [0x91,0x21,0x00,0x00]
+# CHECK-BE: lbu  $8, 1($9)         # encoding: [0x91,0x28,0x00,0x01]
+# CHECK-BE: sll  $1, $1, 8         # encoding: [0x00,0x01,0x0a,0x00]
+# CHECK-BE: or   $8, $8, $1        # encoding: [0x01,0x01,0x40,0x25]
+# CHECK-LE: lbu  $1, 1($9)         # encoding: [0x01,0x00,0x21,0x91]
+# CHECK-LE: lbu  $8, 0($9)         # encoding: [0x00,0x00,0x28,0x91]
+# CHECK-LE: sll  $1, $1, 8         # encoding: [0x00,0x0a,0x01,0x00]
+# CHECK-LE: or   $8, $8, $1        # encoding: [0x25,0x40,0x01,0x01]
+
+  ulhu $8, 2($9)
+# CHECK-BE: lbu  $1, 2($9)         # encoding: [0x91,0x21,0x00,0x02]
+# CHECK-BE: lbu  $8, 3($9)         # encoding: [0x91,0x28,0x00,0x03]
+# CHECK-BE: sll  $1, $1, 8         # encoding: [0x00,0x01,0x0a,0x00]
+# CHECK-BE: or   $8, $8, $1        # encoding: [0x01,0x01,0x40,0x25]
+# CHECK-LE: lbu  $1, 3($9)         # encoding: [0x03,0x00,0x21,0x91]
+# CHECK-LE: lbu  $8, 2($9)         # encoding: [0x02,0x00,0x28,0x91]
+# CHECK-LE: sll  $1, $1, 8         # encoding: [0x00,0x0a,0x01,0x00]
+# CHECK-LE: or   $8, $8, $1        # encoding: [0x25,0x40,0x01,0x01]
+
+  ulhu $8, 0x8000($9)
+# CHECK-BE: ori  $1, $zero, 32768  # encoding: [0x34,0x01,0x80,0x00]
+# CHECK-BE: addu $1, $1, $9        # encoding: [0x00,0x29,0x08,0x21]
+# CHECK-BE: lbu  $8, 0($1)         # encoding: [0x90,0x28,0x00,0x00]
+# CHECK-BE: lbu  $1, 1($1)         # encoding: [0x90,0x21,0x00,0x01]
+# CHECK-BE: sll  $8, $8, 8         # encoding: [0x00,0x08,0x42,0x00]
+# CHECK-BE: or   $8, $8, $1        # encoding: [0x01,0x01,0x40,0x25]
+# CHECK-LE: ori  $1, $zero, 32768  # encoding: [0x00,0x80,0x01,0x34]
+# CHECK-LE: addu $1, $1, $9        # encoding: [0x21,0x08,0x29,0x00]
+# CHECK-LE: lbu  $8, 1($1)         # encoding: [0x01,0x00,0x28,0x90]
+# CHECK-LE: lbu  $1, 0($1)         # encoding: [0x00,0x00,0x21,0x90]
+# CHECK-LE: sll  $8, $8, 8         # encoding: [0x00,0x42,0x08,0x00]
+# CHECK-LE: or   $8, $8, $1        # encoding: [0x25,0x40,0x01,0x01]
+
+  ulhu $8, -0x8000($9)
+# CHECK-BE: lbu  $1, -32768($9)    # encoding: [0x91,0x21,0x80,0x00]
+# CHECK-BE: lbu  $8, -32767($9)    # encoding: [0x91,0x28,0x80,0x01]
+# CHECK-BE: sll  $1, $1, 8         # encoding: [0x00,0x01,0x0a,0x00]
+# CHECK-BE: or   $8, $8, $1        # encoding: [0x01,0x01,0x40,0x25]
+# CHECK-LE: lbu  $1, -32767($9)    # encoding: [0x01,0x80,0x21,0x91]
+# CHECK-LE: lbu  $8, -32768($9)    # encoding: [0x00,0x80,0x28,0x91]
+# CHECK-LE: sll  $1, $1, 8         # encoding: [0x00,0x0a,0x01,0x00]
+# CHECK-LE: or   $8, $8, $1        # encoding: [0x25,0x40,0x01,0x01]
+
+  ulhu $8, 0x10000($9)
+# CHECK-BE: lui  $1, 1             # encoding: [0x3c,0x01,0x00,0x01]
+# CHECK-BE: addu $1, $1, $9        # encoding: [0x00,0x29,0x08,0x21]
+# CHECK-BE: lbu  $8, 0($1)         # encoding: [0x90,0x28,0x00,0x00]
+# CHECK-BE: lbu  $1, 1($1)         # encoding: [0x90,0x21,0x00,0x01]
+# CHECK-BE: sll  $8, $8, 8         # encoding: [0x00,0x08,0x42,0x00]
+# CHECK-BE: or   $8, $8, $1        # encoding: [0x01,0x01,0x40,0x25]
+# CHECK-LE: lui  $1, 1             # encoding: [0x01,0x00,0x01,0x3c]
+# CHECK-LE: addu $1, $1, $9        # encoding: [0x21,0x08,0x29,0x00]
+# CHECK-LE: lbu  $8, 1($1)         # encoding: [0x01,0x00,0x28,0x90]
+# CHECK-LE: lbu  $1, 0($1)         # encoding: [0x00,0x00,0x21,0x90]
+# CHECK-LE: sll  $8, $8, 8         # encoding: [0x00,0x42,0x08,0x00]
+# CHECK-LE: or   $8, $8, $1        # encoding: [0x25,0x40,0x01,0x01]
+
+  ulhu $8, 0x18888($9)
+# CHECK-BE: lui  $1, 1             # encoding: [0x3c,0x01,0x00,0x01]
+# CHECK-BE: ori  $1, $1, 34952     # encoding: [0x34,0x21,0x88,0x88]
+# CHECK-BE: addu $1, $1, $9        # encoding: [0x00,0x29,0x08,0x21]
+# CHECK-BE: lbu  $8, 0($1)         # encoding: [0x90,0x28,0x00,0x00]
+# CHECK-BE: lbu  $1, 1($1)         # encoding: [0x90,0x21,0x00,0x01]
+# CHECK-BE: sll  $8, $8, 8         # encoding: [0x00,0x08,0x42,0x00]
+# CHECK-BE: or   $8, $8, $1        # encoding: [0x01,0x01,0x40,0x25]
+# CHECK-LE: lui  $1, 1             # encoding: [0x01,0x00,0x01,0x3c]
+# CHECK-LE: ori  $1, $1, 34952     # encoding: [0x88,0x88,0x21,0x34]
+# CHECK-LE: addu $1, $1, $9        # encoding: [0x21,0x08,0x29,0x00]
+# CHECK-LE: lbu  $8, 1($1)         # encoding: [0x01,0x00,0x28,0x90]
+# CHECK-LE: lbu  $1, 0($1)         # encoding: [0x00,0x00,0x21,0x90]
+# CHECK-LE: sll  $8, $8, 8         # encoding: [0x00,0x42,0x08,0x00]
+# CHECK-LE: or   $8, $8, $1        # encoding: [0x25,0x40,0x01,0x01]
+
+  ulhu $8, -32769($9)
+# CHECK-BE: lui  $1, 65535         # encoding: [0x3c,0x01,0xff,0xff]
+# CHECK-BE: ori  $1, $1, 32767     # encoding: [0x34,0x21,0x7f,0xff]
+# CHECK-BE: addu $1, $1, $9        # encoding: [0x00,0x29,0x08,0x21]
+# CHECK-BE: lbu  $8, 0($1)         # encoding: [0x90,0x28,0x00,0x00]
+# CHECK-BE: lbu  $1, 1($1)         # encoding: [0x90,0x21,0x00,0x01]
+# CHECK-BE: sll  $8, $8, 8         # encoding: [0x00,0x08,0x42,0x00]
+# CHECK-BE: or   $8, $8, $1        # encoding: [0x01,0x01,0x40,0x25]
+# CHECK-LE: lui  $1, 65535         # encoding: [0xff,0xff,0x01,0x3c]
+# CHECK-LE: ori  $1, $1, 32767     # encoding: [0xff,0x7f,0x21,0x34]
+# CHECK-LE: addu $1, $1, $9        # encoding: [0x21,0x08,0x29,0x00]
+# CHECK-LE: lbu  $8, 1($1)         # encoding: [0x01,0x00,0x28,0x90]
+# CHECK-LE: lbu  $1, 0($1)         # encoding: [0x00,0x00,0x21,0x90]
+# CHECK-LE: sll  $8, $8, 8         # encoding: [0x00,0x42,0x08,0x00]
+# CHECK-LE: or   $8, $8, $1        # encoding: [0x25,0x40,0x01,0x01]
+
+  ulhu $8, 32767($9)
+# CHECK-BE: addiu $1, $zero, 32767 # encoding: [0x24,0x01,0x7f,0xff]
+# CHECK-BE: addu $1, $1, $9        # encoding: [0x00,0x29,0x08,0x21]
+# CHECK-BE: lbu  $8, 0($1)         # encoding: [0x90,0x28,0x00,0x00]
+# CHECK-BE: lbu  $1, 1($1)         # encoding: [0x90,0x21,0x00,0x01]
+# CHECK-BE: sll  $8, $8, 8         # encoding: [0x00,0x08,0x42,0x00]
+# CHECK-BE: or   $8, $8, $1        # encoding: [0x01,0x01,0x40,0x25]
+# CHECK-LE: addiu $1, $zero, 32767 # encoding: [0xff,0x7f,0x01,0x24]
+# CHECK-LE: addu $1, $1, $9        # encoding: [0x21,0x08,0x29,0x00]
+# CHECK-LE: lbu  $8, 1($1)         # encoding: [0x01,0x00,0x28,0x90]
+# CHECK-LE: lbu  $1, 0($1)         # encoding: [0x00,0x00,0x21,0x90]
+# CHECK-LE: sll  $8, $8, 8         # encoding: [0x00,0x42,0x08,0x00]
+# CHECK-LE: or   $8, $8, $1        # encoding: [0x25,0x40,0x01,0x01]
+
+# Test ULW with immediate operand.
+  ulw $8, 0
+# CHECK-BE: lwl  $8, 0($zero)      # encoding: [0x88,0x08,0x00,0x00]
+# CHECK-BE: lwr  $8, 3($zero)      # encoding: [0x98,0x08,0x00,0x03]
+# CHECK-LE: lwl $8, 3($zero)       # encoding: [0x03,0x00,0x08,0x88]
+# CHECK-LE: lwr $8, 0($zero)       # encoding: [0x00,0x00,0x08,0x98]
+
+  ulw $8, 2
+# CHECK-BE: lwl  $8, 2($zero)      # encoding: [0x88,0x08,0x00,0x02]
+# CHECK-BE: lwr  $8, 5($zero)      # encoding: [0x98,0x08,0x00,0x05]
+# CHECK-LE: lwl $8, 5($zero)       # encoding: [0x05,0x00,0x08,0x88]
+# CHECK-LE: lwr $8, 2($zero)       # encoding: [0x02,0x00,0x08,0x98]
+
+  ulw $8, 0x8000
+# CHECK-BE: ori  $1, $zero, 32768  # encoding: [0x34,0x01,0x80,0x00]
+# CHECK-BE: lwl  $8, 0($1)         # encoding: [0x88,0x28,0x00,0x00]
+# CHECK-BE: lwr  $8, 3($1)         # encoding: [0x98,0x28,0x00,0x03]
+# CHECK-LE: ori $1, $zero, 32768   # encoding: [0x00,0x80,0x01,0x34]
+# CHECK-LE: lwl $8, 3($1)          # encoding: [0x03,0x00,0x28,0x88]
+# CHECK-LE: lwr $8, 0($1)          # encoding: [0x00,0x00,0x28,0x98]
+
+  ulw $8, -0x8000
+# CHECK-BE: lwl  $8, -32768($zero) # encoding: [0x88,0x08,0x80,0x00]
+# CHECK-BE: lwr  $8, -32765($zero) # encoding: [0x98,0x08,0x80,0x03]
+# CHECK-LE: lwl $8, -32765($zero)  # encoding: [0x03,0x80,0x08,0x88]
+# CHECK-LE: lwr $8, -32768($zero)  # encoding: [0x00,0x80,0x08,0x98]
+
+  ulw $8, 0x10000
+# CHECK-BE: lui  $1, 1             # encoding: [0x3c,0x01,0x00,0x01]
+# CHECK-BE: lwl  $8, 0($1)         # encoding: [0x88,0x28,0x00,0x00]
+# CHECK-BE: lwr  $8, 3($1)         # encoding: [0x98,0x28,0x00,0x03]
+# CHECK-LE: lui $1, 1              # encoding: [0x01,0x00,0x01,0x3c]
+# CHECK-LE: lwl $8, 3($1)          # encoding: [0x03,0x00,0x28,0x88]
+# CHECK-LE: lwr $8, 0($1)          # encoding: [0x00,0x00,0x28,0x98]
+
+  ulw $8, 0x18888
+# CHECK-BE: lui  $1, 1             # encoding: [0x3c,0x01,0x00,0x01]
+# CHECK-BE: ori  $1, $1, 34952     # encoding: [0x34,0x21,0x88,0x88]
+# CHECK-BE: lwl  $8, 0($1)         # encoding: [0x88,0x28,0x00,0x00]
+# CHECK-BE: lwr  $8, 3($1)         # encoding: [0x98,0x28,0x00,0x03]
+# CHECK-LE: lui $1, 1              # encoding: [0x01,0x00,0x01,0x3c]
+# CHECK-LE: ori $1, $1, 34952      # encoding: [0x88,0x88,0x21,0x34]
+# CHECK-LE: lwl $8, 3($1)          # encoding: [0x03,0x00,0x28,0x88]
+# CHECK-LE: lwr $8, 0($1)          # encoding: [0x00,0x00,0x28,0x98]
+
+  ulw $8, -32771
+# CHECK-BE: lui  $1, 65535         # encoding: [0x3c,0x01,0xff,0xff]
+# CHECK-BE: ori  $1, $1, 32765     # encoding: [0x34,0x21,0x7f,0xfd]
+# CHECK-BE: lwl  $8, 0($1)         # encoding: [0x88,0x28,0x00,0x00]
+# CHECK-BE: lwr  $8, 3($1)         # encoding: [0x98,0x28,0x00,0x03]
+# CHECK-LE: lui $1, 65535          # encoding: [0xff,0xff,0x01,0x3c]
+# CHECK-LE: ori $1, $1, 32765      # encoding: [0xfd,0x7f,0x21,0x34]
+# CHECK-LE: lwl $8, 3($1)          # encoding: [0x03,0x00,0x28,0x88]
+# CHECK-LE: lwr $8, 0($1)          # encoding: [0x00,0x00,0x28,0x98]
+
+  ulw $8, 32765
+# CHECK-BE: addiu $1, $zero, 32765 # encoding: [0x24,0x01,0x7f,0xfd]
+# CHECK-BE: lwl  $8, 0($1)         # encoding: [0x88,0x28,0x00,0x00]
+# CHECK-BE: lwr  $8, 3($1)         # encoding: [0x98,0x28,0x00,0x03]
+# CHECK-LE: addiu $1, $zero, 32765 # encoding: [0xfd,0x7f,0x01,0x24]
+# CHECK-LE: lwl $8, 3($1)          # encoding: [0x03,0x00,0x28,0x88]
+# CHECK-LE: lwr $8, 0($1)          # encoding: [0x00,0x00,0x28,0x98]
+
+# Test ULW with immediate offset and a source register operand.
+  ulw $8, 0($9)
+# CHECK-BE: lwl  $8, 0($9)         # encoding: [0x89,0x28,0x00,0x00]
+# CHECK-BE: lwr  $8, 3($9)         # encoding: [0x99,0x28,0x00,0x03]
+# CHECK-LE: lwl  $8, 3($9)         # encoding: [0x03,0x00,0x28,0x89]
+# CHECK-LE: lwr  $8, 0($9)         # encoding: [0x00,0x00,0x28,0x99]
+
+  ulw $8, 2($9)
+# CHECK-BE: lwl  $8, 2($9)         # encoding: [0x89,0x28,0x00,0x02]
+# CHECK-BE: lwr  $8, 5($9)         # encoding: [0x99,0x28,0x00,0x05]
+# CHECK-LE: lwl  $8, 5($9)         # encoding: [0x05,0x00,0x28,0x89]
+# CHECK-LE: lwr  $8, 2($9)         # encoding: [0x02,0x00,0x28,0x99]
+
+  ulw $8, 0x8000($9)
+# CHECK-BE: ori  $1, $zero, 32768  # encoding: [0x34,0x01,0x80,0x00]
+# CHECK-BE: addu $1, $1, $9        # encoding: [0x00,0x29,0x08,0x21]
+# CHECK-BE: lwl  $8, 0($1)         # encoding: [0x88,0x28,0x00,0x00]
+# CHECK-BE: lwr  $8, 3($1)         # encoding: [0x98,0x28,0x00,0x03]
+# CHECK-LE: ori  $1, $zero, 32768  # encoding: [0x00,0x80,0x01,0x34]
+# CHECK-LE: addu $1, $1, $9        # encoding: [0x21,0x08,0x29,0x00]
+# CHECK-LE: lwl  $8, 3($1)         # encoding: [0x03,0x00,0x28,0x88]
+# CHECK-LE: lwr  $8, 0($1)         # encoding: [0x00,0x00,0x28,0x98]
+
+  ulw $8, -0x8000($9)
+# CHECK-BE: lwl  $8, -32768($9)    # encoding: [0x89,0x28,0x80,0x00]
+# CHECK-BE: lwr  $8, -32765($9)    # encoding: [0x99,0x28,0x80,0x03]
+# CHECK-LE: lwl  $8, -32765($9)    # encoding: [0x03,0x80,0x28,0x89]
+# CHECK-LE: lwr  $8, -32768($9)    # encoding: [0x00,0x80,0x28,0x99]
+
+  ulw $8, 0x10000($9)
+# CHECK-BE: lui  $1, 1             # encoding: [0x3c,0x01,0x00,0x01]
+# CHECK-BE: addu $1, $1, $9        # encoding: [0x00,0x29,0x08,0x21]
+# CHECK-BE: lwl  $8, 0($1)         # encoding: [0x88,0x28,0x00,0x00]
+# CHECK-BE: lwr  $8, 3($1)         # encoding: [0x98,0x28,0x00,0x03]
+# CHECK-LE: lui  $1, 1             # encoding: [0x01,0x00,0x01,0x3c]
+# CHECK-LE: addu $1, $1, $9        # encoding: [0x21,0x08,0x29,0x00]
+# CHECK-LE: lwl  $8, 3($1)         # encoding: [0x03,0x00,0x28,0x88]
+# CHECK-LE: lwr  $8, 0($1)         # encoding: [0x00,0x00,0x28,0x98]
+
+  ulw $8, 0x18888($9)
+# CHECK-BE: lui  $1, 1             # encoding: [0x3c,0x01,0x00,0x01]
+# CHECK-BE: ori  $1, $1, 34952     # encoding: [0x34,0x21,0x88,0x88]
+# CHECK-BE: addu $1, $1, $9        # encoding: [0x00,0x29,0x08,0x21]
+# CHECK-BE: lwl  $8, 0($1)         # encoding: [0x88,0x28,0x00,0x00]
+# CHECK-BE: lwr  $8, 3($1)         # encoding: [0x98,0x28,0x00,0x03]
+# CHECK-LE: lui  $1, 1             # encoding: [0x01,0x00,0x01,0x3c]
+# CHECK-LE: ori  $1, $1, 34952     # encoding: [0x88,0x88,0x21,0x34]
+# CHECK-LE: addu $1, $1, $9        # encoding: [0x21,0x08,0x29,0x00]
+# CHECK-LE: lwl  $8, 3($1)         # encoding: [0x03,0x00,0x28,0x88]
+# CHECK-LE: lwr  $8, 0($1)         # encoding: [0x00,0x00,0x28,0x98]
+
+  ulw $8, -32771($9)
+# CHECK-BE: lui  $1, 65535         # encoding: [0x3c,0x01,0xff,0xff]
+# CHECK-BE: ori  $1, $1, 32765     # encoding: [0x34,0x21,0x7f,0xfd]
+# CHECK-BE: addu $1, $1, $9        # encoding: [0x00,0x29,0x08,0x21]
+# CHECK-BE: lwl  $8, 0($1)         # encoding: [0x88,0x28,0x00,0x00]
+# CHECK-BE: lwr  $8, 3($1)         # encoding: [0x98,0x28,0x00,0x03]
+# CHECK-LE: lui  $1, 65535         # encoding: [0xff,0xff,0x01,0x3c]
+# CHECK-LE: ori  $1, $1, 32765     # encoding: [0xfd,0x7f,0x21,0x34]
+# CHECK-LE: addu $1, $1, $9        # encoding: [0x21,0x08,0x29,0x00]
+# CHECK-LE: lwl  $8, 3($1)         # encoding: [0x03,0x00,0x28,0x88]
+# CHECK-LE: lwr  $8, 0($1)         # encoding: [0x00,0x00,0x28,0x98]
+
+  ulw $8, 32765($9)
+# CHECK-BE: addiu $1, $zero, 32765 # encoding: [0x24,0x01,0x7f,0xfd]
+# CHECK-BE: addu $1, $1, $9        # encoding: [0x00,0x29,0x08,0x21]
+# CHECK-BE: lwl  $8, 0($1)         # encoding: [0x88,0x28,0x00,0x00]
+# CHECK-BE: lwr  $8, 3($1)         # encoding: [0x98,0x28,0x00,0x03]
+# CHECK-LE: addiu $1, $zero, 32765 # encoding: [0xfd,0x7f,0x01,0x24]
+# CHECK-LE: addu $1, $1, $9        # encoding: [0x21,0x08,0x29,0x00]
+# CHECK-LE: lwl  $8, 3($1)         # encoding: [0x03,0x00,0x28,0x88]
+# CHECK-LE: lwr  $8, 0($1)         # encoding: [0x00,0x00,0x28,0x98]
+
+1:
+  add $4, $4, $4
diff --git a/test/MC/Mips/mips-fpu-instructions.s b/test/MC/Mips/mips-fpu-instructions.s
index bfaef9ecacc..7c497171781 100644
--- a/test/MC/Mips/mips-fpu-instructions.s
+++ b/test/MC/Mips/mips-fpu-instructions.s
@@ -139,7 +139,7 @@
 # FP move instructions
 #------------------------------------------------------------------------------
 # CHECK: bc1f    $BB_1                 # encoding: [A,A,0x00,0x45]
-# CHECK: #   fixup A - offset: 0, value: ($BB_1), kind: fixup_Mips_PC16
+# CHECK: #   fixup A - offset: 0, value: ($BB_1)-4, kind: fixup_Mips_PC16
 
 # CHECK:  cfc1    $6, $0               # encoding: [0x00,0x00,0x46,0x44]
 # CHECK:  ctc1    $10, $31             # encoding: [0x00,0xf8,0xca,0x44]
diff --git a/test/MC/Mips/mips-pdr.s b/test/MC/Mips/mips-pdr.s
index 372c259bb09..79e824bda64 100644
--- a/test/MC/Mips/mips-pdr.s
+++ b/test/MC/Mips/mips-pdr.s
@@ -2,7 +2,7 @@
 # RUN:   FileCheck %s -check-prefix=ASMOUT
 
 # RUN: llvm-mc %s -arch=mips -mcpu=mips32r2 -filetype=obj -o - | \
-# RUN:   llvm-readobj -s -section-data | \
+# RUN:   llvm-readobj -s -section-data -r | \
 # RUN:     FileCheck %s -check-prefix=OBJOUT
 
 # ASMOUT: .text
@@ -32,16 +32,12 @@
 # OBJOUT:   }
 
 # We should also check if relocation information was correctly generated.
-# OBJOUT:   Section {
-# OBJOUT:     Name: .rel.pdr
-# OBJOUT:     Type: SHT_REL (0x9)
-# OBJOUT:     Flags [ (0x0)
-# OBJOUT:     ]
-# OBJOUT:     Size: 16
-# OBJOUT:     SectionData (
-# OBJOUT:       0000: 00000000 00000202 00000020 00000802
-# OBJOUT:     )
-# OBJOUT:   }
+# OBJOUT:      Relocations [
+# OBJOUT-NEXT:   Section (6) .rel.pdr {
+# OBJOUT-NEXT:     0x0 R_MIPS_32 .text 0x0
+# OBJOUT-NEXT:     0x20 R_MIPS_32 _global_foo 0x0
+# OBJOUT-NEXT:   }
+# OBJOUT-NEXT: ]
 
 .text
         .type _local_foo,@function
diff --git a/test/MC/Mips/mips-relocations.s b/test/MC/Mips/mips-relocations.s
deleted file mode 100644
index 6f095d1ecdf..00000000000
--- a/test/MC/Mips/mips-relocations.s
+++ /dev/null
@@ -1,40 +0,0 @@
-# RUN: llvm-mc %s -triple=mipsel-unknown-linux -show-encoding -mcpu=mips32r2 | FileCheck %s
-# Check that the assembler can handle the documented syntax
-# for relocations.
-# CHECK:  lui   $2, %hi(_gp_disp)     # encoding: [A,A,0x02,0x3c]
-# CHECK:                              #   fixup A - offset: 0, value: _gp_disp@ABS_HI, kind: fixup_Mips_HI16
-# CHECK:  addiu $2, $2, %lo(_gp_disp) # encoding: [A,A,0x42,0x24]
-# CHECK:                              #   fixup A - offset: 0, value: _gp_disp@ABS_LO, kind: fixup_Mips_LO16
-# CHECK:  lw    $25, %call16(strchr)($gp)   # encoding: [A,A,0x99,0x8f]
-# CHECK:                                    #   fixup A - offset: 0, value: strchr@GOT_CALL, kind: fixup_Mips_CALL16
-# CHECK:  lw      $3, %got(loop_1)($2)    # encoding: [A,A,0x43,0x8c]
-# CHECK:                                  #   fixup A - offset: 0, value: loop_1@GOT, kind: fixup_Mips_GOT_Local
-# CHECK:  lui     $2, %dtprel_hi(_gp_disp) # encoding: [A,A,0x02,0x3c]
-# CHECK:                                        #   fixup A - offset: 0, value: _gp_disp@DTPREL_HI, kind: fixup_Mips_DTPREL_HI
-# CHECK:  addiu   $2, $2, %dtprel_hi(_gp_disp) # encoding: [A,A,0x42,0x24]
-# CHECK:                                  #   fixup A - offset: 0, value: _gp_disp@DTPREL_HI, kind: fixup_Mips_DTPREL_HI
-# CHECK:  lw      $3, %got(loop_1)($2)      # encoding: [A,A,0x43,0x8c]
-# CHECK:                                    #   fixup A - offset: 0, value: loop_1@GOT, kind: fixup_Mips_GOT_Local
-# CHECK:  lw      $4, %got_disp(loop_2)($3) # encoding: [A,A,0x64,0x8c]
-# CHECK:                                    #   fixup A - offset: 0, value: loop_2@GOT_DISP, kind: fixup_Mips_GOT_DISP
-# CHECK:  lw      $5, %got_page(loop_3)($4) # encoding: [A,A,0x85,0x8c]
-# CHECK:                                    #   fixup A - offset: 0, value: loop_3@GOT_PAGE, kind: fixup_Mips_GOT_PAGE
-# CHECK:  lw      $6, %got_ofst(loop_4)($5) # encoding: [A,A,0xa6,0x8c]
-# CHECK:                                    #   fixup A - offset: 0, value: loop_4@GOT_OFST, kind: fixup_Mips_GOT_OFST
-# CHECK:  lui     $2, %tprel_hi(_gp_disp)   # encoding: [A,A,0x02,0x3c]
-# CHECK:                                    #   fixup A - offset: 0, value: _gp_disp@TPREL_HI, kind: fixup_Mips_TPREL_HI
-# CHECK:  addiu   $2, $2, %tprel_lo(_gp_disp) # encoding: [A,A,0x42,0x24]
-# CHECK:                                      #   fixup A - offset: 0, value: _gp_disp@TPREL_LO, kind: fixup_Mips_TPREL_LO
-
-    lui	$2, %hi(_gp_disp)
-	  addiu	$2, $2, %lo(_gp_disp)
-    lw	$25, %call16(strchr)($gp)
-    lw      $3, %got(loop_1)($2)
-    lui	$2, %dtprel_hi(_gp_disp)
-	  addiu	$2, $2, %dtprel_hi(_gp_disp)
-    lw	$3, %got(loop_1)($2)
-    lw	$4, %got_disp(loop_2)($3)
-    lw	$5, %got_page(loop_3)($4)
-    lw	$6, %got_ofst(loop_4)($5)
-    lui	$2, %tprel_hi(_gp_disp)
-	  addiu	$2, $2, %tprel_lo(_gp_disp)
diff --git a/test/MC/Mips/mips1/invalid-mips3.s b/test/MC/Mips/mips1/invalid-mips3.s
index d4be08e48fb..d9400edaa17 100644
--- a/test/MC/Mips/mips1/invalid-mips3.s
+++ b/test/MC/Mips/mips1/invalid-mips3.s
@@ -26,6 +26,9 @@
         dmfc1     $12,$f13          # CHECK: :[[@LINE]]:{{[0-9]+}}: error: instruction requires a CPU feature not currently enabled
         dmtc1     $s0,$f14          # CHECK: :[[@LINE]]:{{[0-9]+}}: error: instruction requires a CPU feature not currently enabled
         dmultu    $a1,$a2           # CHECK: :[[@LINE]]:{{[0-9]+}}: error: instruction requires a CPU feature not currently enabled
+        dneg      $2                # CHECK: :[[@LINE]]:{{[0-9]+}}: error: instruction requires a CPU feature not currently enabled
+        dneg      $2,$3             # CHECK: :[[@LINE]]:{{[0-9]+}}: error: instruction requires a CPU feature not currently enabled
+        dnegu     $2,$3             # CHECK: :[[@LINE]]:{{[0-9]+}}: error: instruction requires a CPU feature not currently enabled
         dsll      $zero,18          # CHECK: :[[@LINE]]:{{[0-9]+}}: error: instruction requires a CPU feature not currently enabled
         dsll      $zero,$s4,18      # CHECK: :[[@LINE]]:{{[0-9]+}}: error: instruction requires a CPU feature not currently enabled
         dsll      $zero,$s4,$12     # CHECK: :[[@LINE]]:{{[0-9]+}}: error: instruction requires a CPU feature not currently enabled
diff --git a/test/MC/Mips/mips2/invalid-mips3.s b/test/MC/Mips/mips2/invalid-mips3.s
index e72b228c67e..67f9d539984 100644
--- a/test/MC/Mips/mips2/invalid-mips3.s
+++ b/test/MC/Mips/mips2/invalid-mips3.s
@@ -22,6 +22,9 @@
         dmfc1      $t0,$f13          # CHECK: :[[@LINE]]:{{[0-9]+}}: error: instruction requires a CPU feature not currently enabled
         dmtc1      $s0,$f14          # CHECK: :[[@LINE]]:{{[0-9]+}}: error: instruction requires a CPU feature not currently enabled
         dmultu     $a1,$a2           # CHECK: :[[@LINE]]:{{[0-9]+}}: error: instruction requires a CPU feature not currently enabled
+        dneg       $2                # CHECK: :[[@LINE]]:{{[0-9]+}}: error: instruction requires a CPU feature not currently enabled
+        dneg       $2,$3             # CHECK: :[[@LINE]]:{{[0-9]+}}: error: instruction requires a CPU feature not currently enabled
+        dnegu      $2,$3             # CHECK: :[[@LINE]]:{{[0-9]+}}: error: instruction requires a CPU feature not currently enabled
         dsll       $zero,18          # CHECK: :[[@LINE]]:{{[0-9]+}}: error: instruction requires a CPU feature not currently enabled
         dsll       $zero,$s4,18      # CHECK: :[[@LINE]]:{{[0-9]+}}: error: instruction requires a CPU feature not currently enabled
         dsll       $zero,$s4,$t0     # CHECK: :[[@LINE]]:{{[0-9]+}}: error: instruction requires a CPU feature not currently enabled
diff --git a/test/MC/Mips/mips3/valid.s b/test/MC/Mips/mips3/valid.s
index 954631d29f2..1c878c97d15 100644
--- a/test/MC/Mips/mips3/valid.s
+++ b/test/MC/Mips/mips3/valid.s
@@ -77,6 +77,9 @@ a:
         dmtc1     $s0,$f14
         dmult     $s7,$9
         dmultu    $a1,$a2
+        dneg      $2                   # CHECK: dneg $2, $2                 # encoding: [0x00,0x02,0x10,0x2e]
+        dneg      $2,$3                # CHECK: dneg $2, $3                 # encoding: [0x00,0x03,0x10,0x2e]
+        dnegu     $2,$3                # CHECK: dnegu $2, $3                # encoding: [0x00,0x03,0x10,0x2f]
         dsll      $zero,18             # CHECK: dsll $zero, $zero, 18       # encoding: [0x00,0x00,0x04,0xb8]
         dsll      $zero,$s4,18         # CHECK: dsll $zero, $20, 18         # encoding: [0x00,0x14,0x04,0xb8]
         dsll      $zero,$s4,$12        # CHECK: dsllv $zero, $20, $12       # encoding: [0x01,0x94,0x00,0x14]
diff --git a/test/MC/Mips/mips32/invalid-mips64.s b/test/MC/Mips/mips32/invalid-mips64.s
index 41040edb7db..0af782060a9 100644
--- a/test/MC/Mips/mips32/invalid-mips64.s
+++ b/test/MC/Mips/mips32/invalid-mips64.s
@@ -7,3 +7,6 @@
 	.set noat
 	dclo	$s2,$a2    # CHECK: :[[@LINE]]:{{[0-9]+}}: error: instruction requires a CPU feature not currently enabled
 	dclz	$s0,$t9    # CHECK: :[[@LINE]]:{{[0-9]+}}: error: instruction requires a CPU feature not currently enabled
+        dneg      $2       # CHECK: :[[@LINE]]:{{[0-9]+}}: error: instruction requires a CPU feature not currently enabled
+        dneg      $2,$3    # CHECK: :[[@LINE]]:{{[0-9]+}}: error: instruction requires a CPU feature not currently enabled
+        dnegu     $2,$3    # CHECK: :[[@LINE]]:{{[0-9]+}}: error: instruction requires a CPU feature not currently enabled
diff --git a/test/MC/Mips/mips32/valid.s b/test/MC/Mips/mips32/valid.s
index 3765044339c..fbe1551f6c2 100644
--- a/test/MC/Mips/mips32/valid.s
+++ b/test/MC/Mips/mips32/valid.s
@@ -89,7 +89,7 @@ a:
         madd      $zero,$9
         maddu     $s3,$gp
         maddu     $24,$s2
-        mfc0      $a2,$14,1
+        mfc0      $8,$15,1             # CHECK: mfc0 $8, $15, 1        # encoding: [0x40,0x08,0x78,0x01]
         mfc1      $a3,$f27
         mfhi      $s3
         mfhi      $sp
@@ -112,7 +112,7 @@ a:
         movz.s    $f25,$f7,$v1
         msub      $s7,$k1
         msubu     $15,$a1
-        mtc0      $9,$29,3
+        mtc0      $9,$15,1             # CHECK: mtc0 $9, $15, 1        # encoding: [0x40,0x89,0x78,0x01]
         mtc1      $s8,$f9
         mthi      $s1
         mtlo      $sp
diff --git a/test/MC/Mips/mips32r2/valid.s b/test/MC/Mips/mips32r2/valid.s
index ee7af3f3ec0..2e4366ab40f 100644
--- a/test/MC/Mips/mips32r2/valid.s
+++ b/test/MC/Mips/mips32r2/valid.s
@@ -103,7 +103,7 @@ a:
         madd.s    $f1,$f31,$f19,$f25
         maddu     $s3,$gp
         maddu     $24,$s2
-        mfc0      $a2,$14,1
+        mfc0      $8,$15,1             # CHECK: mfc0 $8, $15, 1      # encoding: [0x40,0x08,0x78,0x01]
         mfc1      $a3,$f27
         mfhc1     $s8,$f24
         mfhi      $s3
@@ -129,7 +129,7 @@ a:
         msub.d    $f10,$f1,$f31,$f18
         msub.s    $f12,$f19,$f10,$f16
         msubu     $15,$a1
-        mtc0      $9,$29,3
+        mtc0      $9,$15,1             # CHECK: mtc0 $9, $15, 1     # encoding: [0x40,0x89,0x78,0x01]
         mtc1      $s8,$f9
         mthc1     $zero,$f16
         mthi      $s1
diff --git a/test/MC/Mips/mips32r3/valid.s b/test/MC/Mips/mips32r3/valid.s
index 0a4e5b19aef..f6ef1d356c1 100644
--- a/test/MC/Mips/mips32r3/valid.s
+++ b/test/MC/Mips/mips32r3/valid.s
@@ -103,7 +103,7 @@ a:
         madd.s    $f1,$f31,$f19,$f25
         maddu     $s3,$gp
         maddu     $24,$s2
-        mfc0      $a2,$14,1
+        mfc0      $8,$15,1             # CHECK: mfc0 $8, $15, 1        # encoding: [0x40,0x08,0x78,0x01]
         mfc1      $a3,$f27
         mfhc1     $s8,$f24
         mfhi      $s3
@@ -129,7 +129,7 @@ a:
         msub.d    $f10,$f1,$f31,$f18
         msub.s    $f12,$f19,$f10,$f16
         msubu     $15,$a1
-        mtc0      $9,$29,3
+        mtc0      $9,$15,1             # CHECK: mtc0 $9, $15, 1        # encoding: [0x40,0x89,0x78,0x01]
         mtc1      $s8,$f9
         mthc1     $zero,$f16
         mthi      $s1
diff --git a/test/MC/Mips/mips32r5/invalid-mips32.s b/test/MC/Mips/mips32r5/invalid-mips32.s
new file mode 100644
index 00000000000..fc1a516f811
--- /dev/null
+++ b/test/MC/Mips/mips32r5/invalid-mips32.s
@@ -0,0 +1,8 @@
+# Instructions that are invalid
+#
+# RUN: not llvm-mc %s -triple=mips-unknown-linux -show-encoding -mcpu=mips32 \
+# RUN:     2>%t1
+# RUN: FileCheck %s < %t1
+
+	.set noat
+        eretnc                      # CHECK: :[[@LINE]]:{{[0-9]+}}: error: instruction requires a CPU feature not currently enabled
diff --git a/test/MC/Mips/mips32r5/invalid-mips32r2.s b/test/MC/Mips/mips32r5/invalid-mips32r2.s
new file mode 100644
index 00000000000..2e7a2972089
--- /dev/null
+++ b/test/MC/Mips/mips32r5/invalid-mips32r2.s
@@ -0,0 +1,8 @@
+# Instructions that are invalid
+#
+# RUN: not llvm-mc %s -triple=mips-unknown-linux -show-encoding -mcpu=mips32r2 \
+# RUN:     2>%t1
+# RUN: FileCheck %s < %t1
+
+	.set noat
+        eretnc                      # CHECK: :[[@LINE]]:{{[0-9]+}}: error: instruction requires a CPU feature not currently enabled
diff --git a/test/MC/Mips/mips32r5/invalid-mips32r3.s b/test/MC/Mips/mips32r5/invalid-mips32r3.s
new file mode 100644
index 00000000000..3ee188f148d
--- /dev/null
+++ b/test/MC/Mips/mips32r5/invalid-mips32r3.s
@@ -0,0 +1,8 @@
+# Instructions that are invalid
+#
+# RUN: not llvm-mc %s -triple=mips-unknown-linux -show-encoding -mcpu=mips32r3 \
+# RUN:     2>%t1
+# RUN: FileCheck %s < %t1
+
+	.set noat
+        eretnc                      # CHECK: :[[@LINE]]:{{[0-9]+}}: error: instruction requires a CPU feature not currently enabled
diff --git a/test/MC/Mips/mips32r5/valid.s b/test/MC/Mips/mips32r5/valid.s
index 036b908c8ef..3bb8e84ffe0 100644
--- a/test/MC/Mips/mips32r5/valid.s
+++ b/test/MC/Mips/mips32r5/valid.s
@@ -70,6 +70,7 @@ a:
         ei        $14                  # CHECK: ei  $14       # encoding: [0x41,0x6e,0x60,0x20]
         ei                             # CHECK: ei            # encoding: [0x41,0x60,0x60,0x20]
         eret
+        eretnc                         # CHECK: eretnc        # encoding: [0x42,0x00,0x00,0x58]
         floor.w.d $f14,$f11
         floor.w.s $f8,$f9
         j         1f                   # CHECK: j $tmp0 # encoding: [0b000010AA,A,A,A]
@@ -103,7 +104,7 @@ a:
         madd.s    $f1,$f31,$f19,$f25
         maddu     $s3,$gp
         maddu     $24,$s2
-        mfc0      $a2,$14,1
+        mfc0      $8,$15,1             # CHECK: mfc0 $8, $15, 1        # encoding: [0x40,0x08,0x78,0x01]
         mfc1      $a3,$f27
         mfhc1     $s8,$f24
         mfhi      $s3
@@ -129,7 +130,7 @@ a:
         msub.d    $f10,$f1,$f31,$f18
         msub.s    $f12,$f19,$f10,$f16
         msubu     $15,$a1
-        mtc0      $9,$29,3
+        mtc0      $9,$15,1             # CHECK: mtc0 $9, $15, 1        # encoding: [0x40,0x89,0x78,0x01]
         mtc1      $s8,$f9
         mthc1     $zero,$f16
         mthi      $s1
diff --git a/test/MC/Mips/mips32r6/invalid.s b/test/MC/Mips/mips32r6/invalid.s
index 82cb5ab4943..0ce75e6143c 100644
--- a/test/MC/Mips/mips32r6/invalid.s
+++ b/test/MC/Mips/mips32r6/invalid.s
@@ -12,3 +12,7 @@
         ldc2    $8,-21181($at)   # ASM: :[[@LINE]]:{{[0-9]+}}: error: instruction requires a CPU feature not currently enabled
         sdc2    $20,23157($s2)   # ASM: :[[@LINE]]:{{[0-9]+}}: error: instruction requires a CPU feature not currently enabled
         swc2    $25,24880($s0)   # ASM: :[[@LINE]]:{{[0-9]+}}: error: instruction requires a CPU feature not currently enabled
+        break 1024        # CHECK: :[[@LINE]]:{{[0-9]+}}: error: invalid operand for instruction
+        break 1024, 5     # CHECK: :[[@LINE]]:{{[0-9]+}}: error: invalid operand for instruction
+        break 7, 1024     # CHECK: :[[@LINE]]:{{[0-9]+}}: error: invalid operand for instruction
+        break 1024, 1024  # CHECK: :[[@LINE]]:{{[0-9]+}}: error: invalid operand for instruction
diff --git a/test/MC/Mips/mips32r6/relocations.s b/test/MC/Mips/mips32r6/relocations.s
index 13b3387e591..eda74970706 100644
--- a/test/MC/Mips/mips32r6/relocations.s
+++ b/test/MC/Mips/mips32r6/relocations.s
@@ -10,22 +10,22 @@
 # CHECK-FIXUP:                    value: bar, kind: fixup_MIPS_PC19_S2
 # CHECK-FIXUP: beqc $5, $6, bar # encoding: [0x20,0xa6,A,A]
 # CHECK-FIXUP:                  #   fixup A - offset: 0,
-# CHECK-FIXUP:                      value: bar, kind: fixup_Mips_PC16
+# CHECK-FIXUP:                      value: bar-4, kind: fixup_Mips_PC16
 # CHECK-FIXUP: bnec $5, $6, bar # encoding: [0x60,0xa6,A,A]
 # CHECK-FIXUP:                  #   fixup A - offset: 0,
-# CHECK-FIXUP:                      value: bar, kind: fixup_Mips_PC16
+# CHECK-FIXUP:                      value: bar-4, kind: fixup_Mips_PC16
 # CHECK-FIXUP: beqzc $9, bar    # encoding: [0xd9,0b001AAAAA,A,A]
 # CHECK-FIXUP:                  #   fixup A - offset: 0,
-# CHECK-FIXUP:                      value: bar, kind: fixup_MIPS_PC21_S2
+# CHECK-FIXUP:                      value: bar-4, kind: fixup_MIPS_PC21_S2
 # CHECK-FIXUP: bnezc $9, bar    # encoding: [0xf9,0b001AAAAA,A,A]
 # CHECK-FIXUP:                  #   fixup A - offset: 0,
-# CHECK-FIXUP:                      value: bar, kind: fixup_MIPS_PC21_S2
+# CHECK-FIXUP:                      value: bar-4, kind: fixup_MIPS_PC21_S2
 # CHECK-FIXUP: balc  bar        # encoding: [0b111010AA,A,A,A]
 # CHECK-FIXUP:                  #   fixup A - offset: 0,
-# CHECK-FIXUP:                      value: bar, kind: fixup_MIPS_PC26_S2
+# CHECK-FIXUP:                      value: bar-4, kind: fixup_MIPS_PC26_S2
 # CHECK-FIXUP: bc    bar        # encoding: [0b110010AA,A,A,A]
 # CHECK-FIXUP:                  #   fixup A - offset: 0,
-# CHECK-FIXUP:                      value: bar, kind: fixup_MIPS_PC26_S2
+# CHECK-FIXUP:                      value: bar-4, kind: fixup_MIPS_PC26_S2
 # CHECK-FIXUP: aluipc $2, %pcrel_hi(bar)    # encoding: [0xec,0x5f,A,A]
 # CHECK-FIXUP:                              #   fixup A - offset: 0,
 # CHECK-FIXUP:                                  value: bar@PCREL_HI16,
diff --git a/test/MC/Mips/mips32r6/valid.s b/test/MC/Mips/mips32r6/valid.s
index 2c3a5b2a7a4..073da85dfde 100644
--- a/test/MC/Mips/mips32r6/valid.s
+++ b/test/MC/Mips/mips32r6/valid.s
@@ -103,13 +103,17 @@ a:
         divu    $2,$3,$4         # CHECK: divu $2, $3, $4  # encoding: [0x00,0x64,0x10,0x9b]
         ei      $14              # CHECK: ei  $14          # encoding: [0x41,0x6e,0x60,0x20]
         ei                       # CHECK: ei               # encoding: [0x41,0x60,0x60,0x20]
+        eret
+        eretnc                   # CHECK: eretnc           # encoding: [0x42,0x00,0x00,0x58]
         jialc   $5, 256          # CHECK: jialc $5, 256    # encoding: [0xf8,0x05,0x01,0x00]
         jic     $5, 256          # CHECK: jic $5, 256      # encoding: [0xd8,0x05,0x01,0x00]
         lsa     $2, $3, $4, 3    # CHECK: lsa  $2, $3, $4, 3 # encoding: [0x00,0x64,0x10,0xc5]
         lwpc    $2,268           # CHECK: lwpc $2, 268     # encoding: [0xec,0x48,0x00,0x43]
         lwupc   $2,268           # CHECK: lwupc $2, 268    # encoding: [0xec,0x50,0x00,0x43]
+        mfc0    $8,$15,1         # CHECK: mfc0 $8, $15, 1  # encoding: [0x40,0x08,0x78,0x01]
         mod     $2,$3,$4         # CHECK: mod $2, $3, $4   # encoding: [0x00,0x64,0x10,0xda]
         modu    $2,$3,$4         # CHECK: modu $2, $3, $4  # encoding: [0x00,0x64,0x10,0xdb]
+        mtc0    $9,$15,1         # CHECK: mtc0 $9, $15, 1  # encoding: [0x40,0x89,0x78,0x01]
         mul     $2,$3,$4         # CHECK: mul $2, $3, $4   # encoding: [0x00,0x64,0x10,0x98]
         muh     $2,$3,$4         # CHECK: muh $2, $3, $4   # encoding: [0x00,0x64,0x10,0xd8]
         mulu    $2,$3,$4         # CHECK: mulu $2, $3, $4  # encoding: [0x00,0x64,0x10,0x99]
diff --git a/test/MC/Mips/mips4/valid.s b/test/MC/Mips/mips4/valid.s
index a23990c94dd..b89026dd714 100644
--- a/test/MC/Mips/mips4/valid.s
+++ b/test/MC/Mips/mips4/valid.s
@@ -81,6 +81,9 @@ a:
         dmtc1     $s0,$f14
         dmult     $s7,$9
         dmultu    $a1,$a2
+        dneg      $2                   # CHECK: dneg $2, $2                 # encoding: [0x00,0x02,0x10,0x2e]
+        dneg      $2,$3                # CHECK: dneg $2, $3                 # encoding: [0x00,0x03,0x10,0x2e]
+        dnegu     $2,$3                # CHECK: dnegu $2, $3                # encoding: [0x00,0x03,0x10,0x2f]
         dsll      $zero,18             # CHECK: dsll $zero, $zero, 18       # encoding: [0x00,0x00,0x04,0xb8]
         dsll      $zero,$s4,18         # CHECK: dsll $zero, $20, 18         # encoding: [0x00,0x14,0x04,0xb8]
         dsll      $zero,$s4,$12        # CHECK: dsllv $zero, $20, $12       # encoding: [0x01,0x94,0x00,0x14]
diff --git a/test/MC/Mips/mips5/valid.s b/test/MC/Mips/mips5/valid.s
index 094c07fce3b..b444274d78e 100644
--- a/test/MC/Mips/mips5/valid.s
+++ b/test/MC/Mips/mips5/valid.s
@@ -81,6 +81,9 @@ a:
         dmtc1     $s0,$f14
         dmult     $s7,$9
         dmultu    $a1,$a2
+        dneg      $2                   # CHECK: dneg $2, $2                 # encoding: [0x00,0x02,0x10,0x2e]
+        dneg      $2,$3                # CHECK: dneg $2, $3                 # encoding: [0x00,0x03,0x10,0x2e]
+        dnegu     $2,$3                # CHECK: dnegu $2, $3                # encoding: [0x00,0x03,0x10,0x2f]
         dsll      $zero,18             # CHECK: dsll $zero, $zero, 18       # encoding: [0x00,0x00,0x04,0xb8]
         dsll      $zero,$s4,18         # CHECK: dsll $zero, $20, 18         # encoding: [0x00,0x14,0x04,0xb8]
         dsll      $zero,$s4,$12        # CHECK: dsllv $zero, $20, $12       # encoding: [0x01,0x94,0x00,0x14]
diff --git a/test/MC/Mips/mips64-expansions.s b/test/MC/Mips/mips64-expansions.s
index 0efdd2fa5c8..b8f1e7a3e87 100644
--- a/test/MC/Mips/mips64-expansions.s
+++ b/test/MC/Mips/mips64-expansions.s
@@ -1,209 +1,453 @@
 # RUN: llvm-mc %s -triple=mips64el-unknown-linux -show-encoding -mcpu=mips64r2 | FileCheck %s
 #
-# The GNU assembler implements 'dli' and 'dla' variants on 'li' and 'la'
-# supporting double-word lengths.  Test that not only are they present, bu
-# that they also seem to handle 64-bit values.
-#
-# XXXRW: Does using powers of ten make me a bad person?
-#
-# CHECK: ori	$12, $zero, 1           # encoding: [0x01,0x00,0x0c,0x34]
-# CHECK: ori	$12, $zero, 10          # encoding: [0x0a,0x00,0x0c,0x34]
-# CHECK: ori	$12, $zero, 100         # encoding: [0x64,0x00,0x0c,0x34]
-# CHECK: ori	$12, $zero, 1000        # encoding: [0xe8,0x03,0x0c,0x34]
-# CHECK: ori	$12, $zero, 10000       # encoding: [0x10,0x27,0x0c,0x34]
-# CHECK: lui	$12, 1                  # encoding: [0x01,0x00,0x0c,0x3c]
-# CHECK: ori	$12, $12, 34464         # encoding: [0xa0,0x86,0x8c,0x35]
-# CHECK: lui	$12, 15                 # encoding: [0x0f,0x00,0x0c,0x3c]
-# CHECK: ori	$12, $12, 16960         # encoding: [0x40,0x42,0x8c,0x35]
-# CHECK: lui	$12, 152                # encoding: [0x98,0x00,0x0c,0x3c]
-# CHECK: ori	$12, $12, 38528         # encoding: [0x80,0x96,0x8c,0x35]
-# CHECK: lui	$12, 1525               # encoding: [0xf5,0x05,0x0c,0x3c]
-# CHECK: ori	$12, $12, 57600         # encoding: [0x00,0xe1,0x8c,0x35]
-# CHECK: lui	$12, 15258              # encoding: [0x9a,0x3b,0x0c,0x3c]
-# CHECK: ori	$12, $12, 51712         # encoding: [0x00,0xca,0x8c,0x35]
-# CHECK: lui	$12, 2                  # encoding: [0x02,0x00,0x0c,0x3c]
-# CHECK: ori	$12, $12, 21515         # encoding: [0x0b,0x54,0x8c,0x35]
-# CHECK: dsll	$12, $12, 16            # encoding: [0x38,0x64,0x0c,0x00]
-# CHECK: ori	$12, $12, 58368         # encoding: [0x00,0xe4,0x8c,0x35]
-# CHECK: lui	$12, 23                 # encoding: [0x17,0x00,0x0c,0x3c]
-# CHECK: ori	$12, $12, 18550         # encoding: [0x76,0x48,0x8c,0x35]
-# CHECK: dsll	$12, $12, 16            # encoding: [0x38,0x64,0x0c,0x00]
-# CHECK: ori	$12, $12, 59392         # encoding: [0x00,0xe8,0x8c,0x35]
-# CHECK: lui	$12, 232                # encoding: [0xe8,0x00,0x0c,0x3c]
-# CHECK: ori	$12, $12, 54437         # encoding: [0xa5,0xd4,0x8c,0x35]
-# CHECK: dsll	$12, $12, 16            # encoding: [0x38,0x64,0x0c,0x00]
-# CHECK: ori	$12, $12, 4096          # encoding: [0x00,0x10,0x8c,0x35]
-# CHECK: lui	$12, 2328               # encoding: [0x18,0x09,0x0c,0x3c]
-# CHECK: ori	$12, $12, 20082         # encoding: [0x72,0x4e,0x8c,0x35]
-# CHECK: dsll	$12, $12, 16            # encoding: [0x38,0x64,0x0c,0x00]
-# CHECK: ori	$12, $12, 40960         # encoding: [0x00,0xa0,0x8c,0x35]
-# CHECK: lui	$12, 23283              # encoding: [0xf3,0x5a,0x0c,0x3c]
-# CHECK: ori	$12, $12, 4218          # encoding: [0x7a,0x10,0x8c,0x35]
-# CHECK: dsll	$12, $12, 16            # encoding: [0x38,0x64,0x0c,0x00]
-# CHECK: ori	$12, $12, 16384         # encoding: [0x00,0x40,0x8c,0x35]
-# CHECK: lui	$12, 3                  # encoding: [0x03,0x00,0x0c,0x3c]
-# CHECK: ori	$12, $12, 36222         # encoding: [0x7e,0x8d,0x8c,0x35]
-# CHECK: dsll	$12, $12, 16            # encoding: [0x38,0x64,0x0c,0x00]
-# CHECK: ori	$12, $12, 42182         # encoding: [0xc6,0xa4,0x8c,0x35]
-# CHECK: dsll	$12, $12, 16            # encoding: [0x38,0x64,0x0c,0x00]
-# CHECK: ori	$12, $12, 32768         # encoding: [0x00,0x80,0x8c,0x35]
-# CHECK: lui	$12, 35                 # encoding: [0x23,0x00,0x0c,0x3c]
-# CHECK: ori	$12, $12, 34546         # encoding: [0xf2,0x86,0x8c,0x35]
-# CHECK: dsll	$12, $12, 16            # encoding: [0x38,0x64,0x0c,0x00]
-# CHECK: ori	$12, $12, 28609         # encoding: [0xc1,0x6f,0x8c,0x35]
-# CHECK: dsll	$12, $12, 16            # encoding: [0x38,0x64,0x0c,0x00]
-# CHECK: ori	$12, $12, 0             # encoding: [0x00,0x00,0x8c,0x35]
-# CHECK: lui	$12, 355                # encoding: [0x63,0x01,0x0c,0x3c]
-# CHECK: ori	$12, $12, 17784         # encoding: [0x78,0x45,0x8c,0x35]
-# CHECK: dsll	$12, $12, 16            # encoding: [0x38,0x64,0x0c,0x00]
-# CHECK: ori	$12, $12, 23946         # encoding: [0x8a,0x5d,0x8c,0x35]
-# CHECK: dsll	$12, $12, 16            # encoding: [0x38,0x64,0x0c,0x00]
-# CHECK: ori	$12, $12, 0             # encoding: [0x00,0x00,0x8c,0x35]
-# CHECK: lui	$12, 3552               # encoding: [0xe0,0x0d,0x0c,0x3c]
-# CHECK: ori	$12, $12, 46771         # encoding: [0xb3,0xb6,0x8c,0x35]
-# CHECK: dsll	$12, $12, 16            # encoding: [0x38,0x64,0x0c,0x00]
-# CHECK: ori	$12, $12, 42852         # encoding: [0x64,0xa7,0x8c,0x35]
-# CHECK: dsll	$12, $12, 16            # encoding: [0x38,0x64,0x0c,0x00]
-# CHECK: ori	$12, $12, 0             # encoding: [0x00,0x00,0x8c,0x35]
-# CHECK: lui	$12, 35527              # encoding: [0xc7,0x8a,0x0c,0x3c]
-# CHECK: ori	$12, $12, 8964          # encoding: [0x04,0x23,0x8c,0x35]
-# CHECK: dsll	$12, $12, 16            # encoding: [0x38,0x64,0x0c,0x00]
-# CHECK: ori	$12, $12, 35304         # encoding: [0xe8,0x89,0x8c,0x35]
-# CHECK: dsll	$12, $12, 16            # encoding: [0x38,0x64,0x0c,0x00]
-# CHECK: ori	$12, $12, 0             # encoding: [0x00,0x00,0x8c,0x35]
-# CHECK: addiu	$12, $zero, -1          # encoding: [0xff,0xff,0x0c,0x24]
-# CHECK: addiu	$12, $zero, -10         # encoding: [0xf6,0xff,0x0c,0x24]
-# CHECK: addiu	$12, $zero, -100        # encoding: [0x9c,0xff,0x0c,0x24]
-# CHECK: addiu	$12, $zero, -1000       # encoding: [0x18,0xfc,0x0c,0x24]
-# CHECK: addiu	$12, $zero, -10000      # encoding: [0xf0,0xd8,0x0c,0x24]
-# CHECK: lui	$12, 65535              # encoding: [0xff,0xff,0x0c,0x3c]
-# CHECK: ori	$12, $12, 65535         # encoding: [0xff,0xff,0x8c,0x35]
-# CHECK: dsll	$12, $12, 16            # encoding: [0x38,0x64,0x0c,0x00]
-# CHECK: ori	$12, $12, 65534         # encoding: [0xfe,0xff,0x8c,0x35]
-# CHECK: dsll	$12, $12, 16            # encoding: [0x38,0x64,0x0c,0x00]
-# CHECK: ori	$12, $12, 31072         # encoding: [0x60,0x79,0x8c,0x35]
-# CHECK: lui	$12, 65535              # encoding: [0xff,0xff,0x0c,0x3c]
-# CHECK: ori	$12, $12, 65535         # encoding: [0xff,0xff,0x8c,0x35]
-# CHECK: dsll	$12, $12, 16            # encoding: [0x38,0x64,0x0c,0x00]
-# CHECK: ori	$12, $12, 65520         # encoding: [0xf0,0xff,0x8c,0x35]
-# CHECK: dsll	$12, $12, 16            # encoding: [0x38,0x64,0x0c,0x00]
-# CHECK: ori	$12, $12, 48576         # encoding: [0xc0,0xbd,0x8c,0x35]
-# CHECK: lui	$12, 65535              # encoding: [0xff,0xff,0x0c,0x3c]
-# CHECK: ori	$12, $12, 65535         # encoding: [0xff,0xff,0x8c,0x35]
-# CHECK: dsll	$12, $12, 16            # encoding: [0x38,0x64,0x0c,0x00]
-# CHECK: ori	$12, $12, 65383         # encoding: [0x67,0xff,0x8c,0x35]
-# CHECK: dsll	$12, $12, 16            # encoding: [0x38,0x64,0x0c,0x00]
-# CHECK: ori	$12, $12, 27008         # encoding: [0x80,0x69,0x8c,0x35]
-# CHECK: lui	$12, 65535              # encoding: [0xff,0xff,0x0c,0x3c]
-# CHECK: ori	$12, $12, 65535         # encoding: [0xff,0xff,0x8c,0x35]
-# CHECK: dsll	$12, $12, 16            # encoding: [0x38,0x64,0x0c,0x00]
-# CHECK: ori	$12, $12, 64010         # encoding: [0x0a,0xfa,0x8c,0x35]
-# CHECK: dsll	$12, $12, 16            # encoding: [0x38,0x64,0x0c,0x00]
-# CHECK: ori	$12, $12, 7936          # encoding: [0x00,0x1f,0x8c,0x35]
-# CHECK: lui	$12, 65535              # encoding: [0xff,0xff,0x0c,0x3c]
-# CHECK: ori	$12, $12, 65535         # encoding: [0xff,0xff,0x8c,0x35]
-# CHECK: dsll	$12, $12, 16            # encoding: [0x38,0x64,0x0c,0x00]
-# CHECK: ori	$12, $12, 50277         # encoding: [0x65,0xc4,0x8c,0x35]
-# CHECK: dsll	$12, $12, 16            # encoding: [0x38,0x64,0x0c,0x00]
-# CHECK: ori	$12, $12, 13824         # encoding: [0x00,0x36,0x8c,0x35]
-# CHECK: lui	$12, 65535              # encoding: [0xff,0xff,0x0c,0x3c]
-# CHECK: ori	$12, $12, 65533         # encoding: [0xfd,0xff,0x8c,0x35]
-# CHECK: dsll	$12, $12, 16            # encoding: [0x38,0x64,0x0c,0x00]
-# CHECK: ori	$12, $12, 44020         # encoding: [0xf4,0xab,0x8c,0x35]
-# CHECK: dsll	$12, $12, 16            # encoding: [0x38,0x64,0x0c,0x00]
-# CHECK: ori	$12, $12, 7168          # encoding: [0x00,0x1c,0x8c,0x35]
-# CHECK: lui	$12, 65535              # encoding: [0xff,0xff,0x0c,0x3c]
-# CHECK: ori	$12, $12, 65512         # encoding: [0xe8,0xff,0x8c,0x35]
-# CHECK: dsll	$12, $12, 16            # encoding: [0x38,0x64,0x0c,0x00]
-# CHECK: ori	$12, $12, 46985         # encoding: [0x89,0xb7,0x8c,0x35]
-# CHECK: dsll	$12, $12, 16            # encoding: [0x38,0x64,0x0c,0x00]
-# CHECK: ori	$12, $12, 6144          # encoding: [0x00,0x18,0x8c,0x35]
-# CHECK: lui	$12, 65535              # encoding: [0xff,0xff,0x0c,0x3c]
-# CHECK: ori	$12, $12, 65303         # encoding: [0x17,0xff,0x8c,0x35]
-# CHECK: dsll	$12, $12, 16            # encoding: [0x38,0x64,0x0c,0x00]
-# CHECK: ori	$12, $12, 11098         # encoding: [0x5a,0x2b,0x8c,0x35]
-# CHECK: dsll	$12, $12, 16            # encoding: [0x38,0x64,0x0c,0x00]
-# CHECK: ori	$12, $12, 61440         # encoding: [0x00,0xf0,0x8c,0x35]
-# CHECK: lui	$12, 65535              # encoding: [0xff,0xff,0x0c,0x3c]
-# CHECK: ori	$12, $12, 63207         # encoding: [0xe7,0xf6,0x8c,0x35]
-# CHECK: dsll	$12, $12, 16            # encoding: [0x38,0x64,0x0c,0x00]
-# CHECK: ori	$12, $12, 45453         # encoding: [0x8d,0xb1,0x8c,0x35]
-# CHECK: dsll	$12, $12, 16            # encoding: [0x38,0x64,0x0c,0x00]
-# CHECK: ori	$12, $12, 24576         # encoding: [0x00,0x60,0x8c,0x35]
-# CHECK: lui	$12, 65535              # encoding: [0xff,0xff,0x0c,0x3c]
-# CHECK: ori	$12, $12, 42252         # encoding: [0x0c,0xa5,0x8c,0x35]
-# CHECK: dsll	$12, $12, 16            # encoding: [0x38,0x64,0x0c,0x00]
-# CHECK: ori	$12, $12, 61317         # encoding: [0x85,0xef,0x8c,0x35]
-# CHECK: dsll	$12, $12, 16            # encoding: [0x38,0x64,0x0c,0x00]
-# CHECK: ori	$12, $12, 49152         # encoding: [0x00,0xc0,0x8c,0x35]
-# CHECK: lui	$12, 65532              # encoding: [0xfc,0xff,0x0c,0x3c]
-# CHECK: ori	$12, $12, 29313         # encoding: [0x81,0x72,0x8c,0x35]
-# CHECK: dsll	$12, $12, 16            # encoding: [0x38,0x64,0x0c,0x00]
-# CHECK: ori	$12, $12, 23353         # encoding: [0x39,0x5b,0x8c,0x35]
-# CHECK: dsll	$12, $12, 16            # encoding: [0x38,0x64,0x0c,0x00]
-# CHECK: ori	$12, $12, 32768         # encoding: [0x00,0x80,0x8c,0x35]
-# CHECK: lui	$12, 65500              # encoding: [0xdc,0xff,0x0c,0x3c]
-# CHECK: ori	$12, $12, 30989         # encoding: [0x0d,0x79,0x8c,0x35]
-# CHECK: dsll	$12, $12, 16            # encoding: [0x38,0x64,0x0c,0x00]
-# CHECK: ori	$12, $12, 36927         # encoding: [0x3f,0x90,0x8c,0x35]
-# CHECK: dsll	$12, $12, 16            # encoding: [0x38,0x64,0x0c,0x00]
-# CHECK: ori	$12, $12, 0             # encoding: [0x00,0x00,0x8c,0x35]
-# CHECK: lui	$12, 65180              # encoding: [0x9c,0xfe,0x0c,0x3c]
-# CHECK: ori	$12, $12, 47751         # encoding: [0x87,0xba,0x8c,0x35]
-# CHECK: dsll	$12, $12, 16            # encoding: [0x38,0x64,0x0c,0x00]
-# CHECK: ori	$12, $12, 41590         # encoding: [0x76,0xa2,0x8c,0x35]
-# CHECK: dsll	$12, $12, 16            # encoding: [0x38,0x64,0x0c,0x00]
-# CHECK: ori	$12, $12, 0             # encoding: [0x00,0x00,0x8c,0x35]
-# CHECK: lui	$12, 61983              # encoding: [0x1f,0xf2,0x0c,0x3c]
-# CHECK: ori	$12, $12, 18764         # encoding: [0x4c,0x49,0x8c,0x35]
-# CHECK: dsll	$12, $12, 16            # encoding: [0x38,0x64,0x0c,0x00]
-# CHECK: ori	$12, $12, 22684         # encoding: [0x9c,0x58,0x8c,0x35]
-# CHECK: dsll	$12, $12, 16            # encoding: [0x38,0x64,0x0c,0x00]
-# CHECK: ori	$12, $12, 0             # encoding: [0x00,0x00,0x8c,0x35]
-# CHECK: lui	$12, 30008              # encoding: [0x38,0x75,0x0c,0x3c]
-# CHECK: ori	$12, $12, 56571         # encoding: [0xfb,0xdc,0x8c,0x35]
-# CHECK: dsll	$12, $12, 16            # encoding: [0x38,0x64,0x0c,0x00]
-# CHECK: ori	$12, $12, 30232         # encoding: [0x18,0x76,0x8c,0x35]
-# CHECK: dsll	$12, $12, 16            # encoding: [0x38,0x64,0x0c,0x00]
-# CHECK: ori	$12, $12, 0             # encoding: [0x00,0x00,0x8c,0x35]
-
-	dli	$t0, 1
-	dli	$t0, 10
-	dli	$t0, 100
-	dli	$t0, 1000
-	dli	$t0, 10000
-	dli	$t0, 100000
-	dli	$t0, 1000000
-	dli	$t0, 10000000
-	dli	$t0, 100000000
-	dli	$t0, 1000000000
-	dli	$t0, 10000000000
-	dli	$t0, 100000000000
-	dli	$t0, 1000000000000
-	dli	$t0, 10000000000000
-	dli	$t0, 100000000000000
-	dli	$t0, 1000000000000000
-	dli	$t0, 10000000000000000
-	dli	$t0, 100000000000000000
-	dli	$t0, 1000000000000000000
-	dli	$t0, 10000000000000000000
-	dli	$t0, -1
-	dli	$t0, -10
-	dli	$t0, -100
-	dli	$t0, -1000
-	dli	$t0, -10000
-	dli	$t0, -100000
-	dli	$t0, -1000000
-	dli	$t0, -10000000
-	dli	$t0, -100000000
-	dli	$t0, -1000000000
-	dli	$t0, -10000000000
-	dli	$t0, -100000000000
-	dli	$t0, -1000000000000
-	dli	$t0, -10000000000000
-	dli	$t0, -100000000000000
-	dli	$t0, -1000000000000000
-	dli	$t0, -10000000000000000
-	dli	$t0, -100000000000000000
-	dli	$t0, -1000000000000000000
-	dli	$t0, -10000000000000000000
+# Test the 'dli' and 'dla' 64-bit variants of 'li' and 'la'.
+
+# Immediate is <= 32 bits.
+  dli $5, 123
+# CHECK:     addiu $5, $zero, 123   # encoding: [0x7b,0x00,0x05,0x24]
+
+  dli $6, -2345
+# CHECK:     addiu $6, $zero, -2345 # encoding: [0xd7,0xf6,0x06,0x24]
+
+  dli $7, 65538
+# CHECK:     lui   $7, 1            # encoding: [0x01,0x00,0x07,0x3c]
+# CHECK:     ori   $7, $7, 2        # encoding: [0x02,0x00,0xe7,0x34]
+
+  dli $8, ~7
+# CHECK:     addiu $8, $zero, -8    # encoding: [0xf8,0xff,0x08,0x24]
+
+  dli $9, 0x10000
+# CHECK:     lui   $9, 1            # encoding: [0x01,0x00,0x09,0x3c]
+# CHECK-NOT: ori   $9, $9, 0        # encoding: [0x00,0x00,0x29,0x35]
+
+
+# Positive immediate which is => 32 bits and <= 48 bits.
+  dli $8, 0x100000000
+# CHECK: lui  $8, 1                 # encoding: [0x01,0x00,0x08,0x3c]
+# CHECK: dsll $8, $8, 16            # encoding: [0x38,0x44,0x08,0x00]
+
+  dli $8, 0x100000001
+# CHECK: lui  $8, 1                 # encoding: [0x01,0x00,0x08,0x3c]
+# CHECK: dsll $8, $8, 16            # encoding: [0x38,0x44,0x08,0x00]
+# CHECK: ori  $8, $8, 1             # encoding: [0x01,0x00,0x08,0x35]
+
+  dli $8, 0x100010000
+# CHECK: lui  $8, 1                 # encoding: [0x01,0x00,0x08,0x3c]
+# CHECK: ori  $8, $8, 1             # encoding: [0x01,0x00,0x08,0x35]
+# CHECK: dsll $8, $8, 16            # encoding: [0x38,0x44,0x08,0x00]
+
+  dli $8, 0x100010001
+# CHECK: lui  $8, 1                 # encoding: [0x01,0x00,0x08,0x3c]
+# CHECK: ori  $8, $8, 1             # encoding: [0x01,0x00,0x08,0x35]
+# CHECK: dsll $8, $8, 16            # encoding: [0x38,0x44,0x08,0x00]
+# CHECK: ori  $8, $8, 1             # encoding: [0x01,0x00,0x08,0x35]
+
+
+# Positive immediate which is > 48 bits.
+  dli $8, 0x1000000000000
+# CHECK: lui    $8, 1               # encoding: [0x01,0x00,0x08,0x3c]
+# CHECK: dsll32 $8, $8, 0           # encoding: [0x3c,0x40,0x08,0x00]
+
+  dli $8, 0x1000000000001
+# CHECK: lui    $8, 1               # encoding: [0x01,0x00,0x08,0x3c]
+# CHECK: dsll32 $8, $8, 0           # encoding: [0x3c,0x40,0x08,0x00]
+# CHECK: ori    $8, $8, 1           # encoding: [0x01,0x00,0x08,0x35]
+
+  dli $8, 0x1000000010000
+# CHECK: lui    $8, 1               # encoding: [0x01,0x00,0x08,0x3c]
+# CHECK: dsll   $8, $8, 16          # encoding: [0x38,0x44,0x08,0x00]
+# CHECK: ori    $8, $8, 1           # encoding: [0x01,0x00,0x08,0x35]
+# CHECK: dsll   $8, $8, 16          # encoding: [0x38,0x44,0x08,0x00]
+
+  dli $8, 0x1000100000000
+# CHECK: lui    $8, 1               # encoding: [0x01,0x00,0x08,0x3c]
+# CHECK: ori    $8, $8, 1           # encoding: [0x01,0x00,0x08,0x35]
+# CHECK: dsll32 $8, $8, 0           # encoding: [0x3c,0x40,0x08,0x00]
+
+  dli $8, 0x1000000010001
+# CHECK: lui    $8, 1               # encoding: [0x01,0x00,0x08,0x3c]
+# CHECK: dsll   $8, $8, 16          # encoding: [0x38,0x44,0x08,0x00]
+# CHECK: ori    $8, $8, 1           # encoding: [0x01,0x00,0x08,0x35]
+# CHECK: dsll   $8, $8, 16          # encoding: [0x38,0x44,0x08,0x00]
+# CHECK: ori    $8, $8, 1           # encoding: [0x01,0x00,0x08,0x35]
+
+  dli $8, 0x1000100010000
+# CHECK: lui    $8, 1               # encoding: [0x01,0x00,0x08,0x3c]
+# CHECK: ori    $8, $8, 1           # encoding: [0x01,0x00,0x08,0x35]
+# CHECK: dsll   $8, $8, 16          # encoding: [0x38,0x44,0x08,0x00]
+# CHECK: ori    $8, $8, 1           # encoding: [0x01,0x00,0x08,0x35]
+# CHECK: dsll   $8, $8, 16          # encoding: [0x38,0x44,0x08,0x00]
+
+  dli $8, 0x1000100000001
+# CHECK: lui    $8, 1               # encoding: [0x01,0x00,0x08,0x3c]
+# CHECK: ori    $8, $8, 1           # encoding: [0x01,0x00,0x08,0x35]
+# CHECK: dsll32 $8, $8, 0           # encoding: [0x3c,0x40,0x08,0x00]
+# CHECK: ori    $8, $8, 1           # encoding: [0x01,0x00,0x08,0x35]
+
+  dli $8, 0x1000100010001
+# CHECK: lui    $8, 1               # encoding: [0x01,0x00,0x08,0x3c]
+# CHECK: ori    $8, $8, 1           # encoding: [0x01,0x00,0x08,0x35]
+# CHECK: dsll   $8, $8, 16          # encoding: [0x38,0x44,0x08,0x00]
+# CHECK: ori    $8, $8, 1           # encoding: [0x01,0x00,0x08,0x35]
+# CHECK: dsll   $8, $8, 16          # encoding: [0x38,0x44,0x08,0x00]
+# CHECK: ori    $8, $8, 1           # encoding: [0x01,0x00,0x08,0x35]
+
+
+# Negative immediate which is => 32 bits and <= 48 bits.
+  dli $8, -0x100000000
+# CHECK: lui    $8, 65535           # encoding: [0xff,0xff,0x08,0x3c]
+# CHECK: ori    $8, $8, 65535       # encoding: [0xff,0xff,0x08,0x35]
+# CHECK: dsll32 $8, $8, 0           # encoding: [0x3c,0x40,0x08,0x00]
+
+  dli $8, -0x100000001
+# CHECK: lui    $8, 65535           # encoding: [0xff,0xff,0x08,0x3c]
+# CHECK: ori    $8, $8, 65534       # encoding: [0xfe,0xff,0x08,0x35]
+# CHECK: dsll   $8, $8, 16          # encoding: [0x38,0x44,0x08,0x00]
+# CHECK: ori    $8, $8, 65535       # encoding: [0xff,0xff,0x08,0x35]
+# CHECK: dsll   $8, $8, 16          # encoding: [0x38,0x44,0x08,0x00]
+# CHECK: ori    $8, $8, 65535       # encoding: [0xff,0xff,0x08,0x35]
+
+  dli $8, -0x100010000
+# CHECK: lui    $8, 65535           # encoding: [0xff,0xff,0x08,0x3c]
+# CHECK: ori    $8, $8, 65534       # encoding: [0xfe,0xff,0x08,0x35]
+# CHECK: dsll   $8, $8, 16          # encoding: [0x38,0x44,0x08,0x00]
+# CHECK: ori    $8, $8, 65535       # encoding: [0xff,0xff,0x08,0x35]
+# CHECK: dsll   $8, $8, 16          # encoding: [0x38,0x44,0x08,0x00]
+
+  dli $8, -0x100010001
+# CHECK: lui    $8, 65535           # encoding: [0xff,0xff,0x08,0x3c]
+# CHECK: ori    $8, $8, 65534       # encoding: [0xfe,0xff,0x08,0x35]
+# CHECK: dsll   $8, $8, 16          # encoding: [0x38,0x44,0x08,0x00]
+# CHECK: ori    $8, $8, 65534       # encoding: [0xfe,0xff,0x08,0x35]
+# CHECK: dsll   $8, $8, 16          # encoding: [0x38,0x44,0x08,0x00]
+# CHECK: ori    $8, $8, 65535       # encoding: [0xff,0xff,0x08,0x35]
+
+
+# Negative immediate which is > 48 bits.
+  dli $8, -0x1000000000000
+# CHECK: lui    $8, 65535           # encoding: [0xff,0xff,0x08,0x3c]
+# CHECK: dsll32 $8, $8, 0           # encoding: [0x3c,0x40,0x08,0x00]
+
+  dli $8, -0x1000000000001
+# CHECK: lui    $8, 65534           # encoding: [0xfe,0xff,0x08,0x3c]
+# CHECK: ori    $8, $8, 65535       # encoding: [0xff,0xff,0x08,0x35]
+# CHECK: dsll   $8, $8, 16          # encoding: [0x38,0x44,0x08,0x00]
+# CHECK: ori    $8, $8, 65535       # encoding: [0xff,0xff,0x08,0x35]
+# CHECK: dsll   $8, $8, 16          # encoding: [0x38,0x44,0x08,0x00]
+# CHECK: ori    $8, $8, 65535       # encoding: [0xff,0xff,0x08,0x35]
+
+  dli $8, -0x1000000010000
+# CHECK: lui    $8, 65534           # encoding: [0xfe,0xff,0x08,0x3c]
+# CHECK: ori    $8, $8, 65535       # encoding: [0xff,0xff,0x08,0x35]
+# CHECK: dsll   $8, $8, 16          # encoding: [0x38,0x44,0x08,0x00]
+# CHECK: ori    $8, $8, 65535       # encoding: [0xff,0xff,0x08,0x35]
+# CHECK: dsll   $8, $8, 16          # encoding: [0x38,0x44,0x08,0x00]
+
+  dli $8, -0x1000100000000
+# CHECK: lui    $8, 65534           # encoding: [0xfe,0xff,0x08,0x3c]
+# CHECK: ori    $8, $8, 65535       # encoding: [0xff,0xff,0x08,0x35]
+# CHECK: dsll32 $8, $8, 0           # encoding: [0x3c,0x40,0x08,0x00]
+
+  dli $8, -0x1000000010001
+# CHECK: lui    $8, 65534           # encoding: [0xfe,0xff,0x08,0x3c]
+# CHECK: ori    $8, $8, 65535       # encoding: [0xff,0xff,0x08,0x35]
+# CHECK: dsll   $8, $8, 16          # encoding: [0x38,0x44,0x08,0x00]
+# CHECK: ori    $8, $8, 65534       # encoding: [0xfe,0xff,0x08,0x35]
+# CHECK: dsll   $8, $8, 16          # encoding: [0x38,0x44,0x08,0x00]
+# CHECK: ori    $8, $8, 65535       # encoding: [0xff,0xff,0x08,0x35]
+
+  dli $8, -0x1000100010000
+# CHECK: lui    $8, 65534           # encoding: [0xfe,0xff,0x08,0x3c]
+# CHECK: ori    $8, $8, 65534       # encoding: [0xfe,0xff,0x08,0x35]
+# CHECK: dsll   $8, $8, 16          # encoding: [0x38,0x44,0x08,0x00]
+# CHECK: ori    $8, $8, 65535       # encoding: [0xff,0xff,0x08,0x35]
+# CHECK: dsll   $8, $8, 16          # encoding: [0x38,0x44,0x08,0x00]
+
+  dli $8, -0x1000100000001
+# CHECK: lui    $8, 65534           # encoding: [0xfe,0xff,0x08,0x3c]
+# CHECK: ori    $8, $8, 65534       # encoding: [0xfe,0xff,0x08,0x35]
+# CHECK: dsll   $8, $8, 16          # encoding: [0x38,0x44,0x08,0x00]
+# CHECK: ori    $8, $8, 65535       # encoding: [0xff,0xff,0x08,0x35]
+# CHECK: dsll   $8, $8, 16          # encoding: [0x38,0x44,0x08,0x00]
+# CHECK: ori    $8, $8, 65535       # encoding: [0xff,0xff,0x08,0x35]
+
+  dli $8, -0x1000100010001
+# CHECK: lui    $8, 65534           # encoding: [0xfe,0xff,0x08,0x3c]
+# CHECK: ori    $8, $8, 65534       # encoding: [0xfe,0xff,0x08,0x35]
+# CHECK: dsll   $8, $8, 16          # encoding: [0x38,0x44,0x08,0x00]
+# CHECK: ori    $8, $8, 65534       # encoding: [0xfe,0xff,0x08,0x35]
+# CHECK: dsll   $8, $8, 16          # encoding: [0x38,0x44,0x08,0x00]
+# CHECK: ori    $8, $8, 65535       # encoding: [0xff,0xff,0x08,0x35]
+
+# Check that signed negative 32-bit immediates are loaded correctly:
+  li $10, ~(0x101010)
+# CHECK: lui $10, 65519        # encoding: [0xef,0xff,0x0a,0x3c]
+# CHECK: ori $10, $10, 61423   # encoding: [0xef,0xef,0x4a,0x35]
+# CHECK-NOT: dsll
+
+  dli $10, ~(0x202020)
+# CHECK: lui $10, 65503        # encoding: [0xdf,0xff,0x0a,0x3c]
+# CHECK: ori $10, $10, 57311   # encoding: [0xdf,0xdf,0x4a,0x35]
+# CHECK-NOT: dsll
+
+  dli $9, 0x80000000
+# CHECK: ori  $9, $zero, 32768 # encoding: [0x00,0x80,0x09,0x34]
+# CHECK: dsll $9, $9, 16       # encoding: [0x38,0x4c,0x09,0x00]
+
+# Test bne with an immediate as the 2nd operand.
+  bne $2, 0x100010001, 1332
+# CHECK: lui  $1, 1                 # encoding: [0x01,0x00,0x01,0x3c]
+# CHECK: ori  $1, $1, 1             # encoding: [0x01,0x00,0x21,0x34]
+# CHECK: dsll $1, $1, 16            # encoding: [0x38,0x0c,0x01,0x00]
+# CHECK: ori  $1, $1, 1             # encoding: [0x01,0x00,0x21,0x34]
+# CHECK: bne  $2, $1, 1332          # encoding: [0x4d,0x01,0x41,0x14]
+# CHECK: nop                        # encoding: [0x00,0x00,0x00,0x00]
+
+  bne $2, 0x1000100010001, 1332
+# CHECK: lui  $1, 1                 # encoding: [0x01,0x00,0x01,0x3c]
+# CHECK: ori  $1, $1, 1             # encoding: [0x01,0x00,0x21,0x34]
+# CHECK: dsll $1, $1, 16            # encoding: [0x38,0x0c,0x01,0x00]
+# CHECK: ori  $1, $1, 1             # encoding: [0x01,0x00,0x21,0x34]
+# CHECK: dsll $1, $1, 16            # encoding: [0x38,0x0c,0x01,0x00]
+# CHECK: ori  $1, $1, 1             # encoding: [0x01,0x00,0x21,0x34]
+# CHECK: bne  $2, $1, 1332          # encoding: [0x4d,0x01,0x41,0x14]
+# CHECK: nop                        # encoding: [0x00,0x00,0x00,0x00]
+
+  bne $2, -0x100010001, 1332
+# CHECK: lui  $1, 65535             # encoding: [0xff,0xff,0x01,0x3c]
+# CHECK: ori  $1, $1, 65534         # encoding: [0xfe,0xff,0x21,0x34]
+# CHECK: dsll $1, $1, 16            # encoding: [0x38,0x0c,0x01,0x00]
+# CHECK: ori  $1, $1, 65534         # encoding: [0xfe,0xff,0x21,0x34]
+# CHECK: dsll $1, $1, 16            # encoding: [0x38,0x0c,0x01,0x00]
+# CHECK: ori  $1, $1, 65535         # encoding: [0xff,0xff,0x21,0x34]
+# CHECK: bne  $2, $1, 1332          # encoding: [0x4d,0x01,0x41,0x14]
+# CHECK: nop                        # encoding: [0x00,0x00,0x00,0x00]
+
+  bne $2, -0x1000100010001, 1332
+# CHECK: lui  $1, 65534             # encoding: [0xfe,0xff,0x01,0x3c]
+# CHECK: ori  $1, $1, 65534         # encoding: [0xfe,0xff,0x21,0x34]
+# CHECK: dsll $1, $1, 16            # encoding: [0x38,0x0c,0x01,0x00]
+# CHECK: ori  $1, $1, 65534         # encoding: [0xfe,0xff,0x21,0x34]
+# CHECK: dsll $1, $1, 16            # encoding: [0x38,0x0c,0x01,0x00]
+# CHECK: ori  $1, $1, 65535         # encoding: [0xff,0xff,0x21,0x34]
+# CHECK: bne  $2, $1, 1332          # encoding: [0x4d,0x01,0x41,0x14]
+# CHECK: nop                        # encoding: [0x00,0x00,0x00,0x00]
+
+# Test beq with an immediate as the 2nd operand.
+  beq $2, 0x100010001, 1332
+# CHECK: lui  $1, 1                 # encoding: [0x01,0x00,0x01,0x3c]
+# CHECK: ori  $1, $1, 1             # encoding: [0x01,0x00,0x21,0x34]
+# CHECK: dsll $1, $1, 16            # encoding: [0x38,0x0c,0x01,0x00]
+# CHECK: ori  $1, $1, 1             # encoding: [0x01,0x00,0x21,0x34]
+# CHECK: beq  $2, $1, 1332          # encoding: [0x4d,0x01,0x41,0x10]
+# CHECK: nop                        # encoding: [0x00,0x00,0x00,0x00]
+
+  beq $2, 0x1000100010001, 1332
+# CHECK: lui  $1, 1                 # encoding: [0x01,0x00,0x01,0x3c]
+# CHECK: ori  $1, $1, 1             # encoding: [0x01,0x00,0x21,0x34]
+# CHECK: dsll $1, $1, 16            # encoding: [0x38,0x0c,0x01,0x00]
+# CHECK: ori  $1, $1, 1             # encoding: [0x01,0x00,0x21,0x34]
+# CHECK: dsll $1, $1, 16            # encoding: [0x38,0x0c,0x01,0x00]
+# CHECK: ori  $1, $1, 1             # encoding: [0x01,0x00,0x21,0x34]
+# CHECK: beq  $2, $1, 1332          # encoding: [0x4d,0x01,0x41,0x10]
+# CHECK: nop                        # encoding: [0x00,0x00,0x00,0x00]
+
+  beq $2, -0x100010001, 1332
+# CHECK: lui  $1, 65535             # encoding: [0xff,0xff,0x01,0x3c]
+# CHECK: ori  $1, $1, 65534         # encoding: [0xfe,0xff,0x21,0x34]
+# CHECK: dsll $1, $1, 16            # encoding: [0x38,0x0c,0x01,0x00]
+# CHECK: ori  $1, $1, 65534         # encoding: [0xfe,0xff,0x21,0x34]
+# CHECK: dsll $1, $1, 16            # encoding: [0x38,0x0c,0x01,0x00]
+# CHECK: ori  $1, $1, 65535         # encoding: [0xff,0xff,0x21,0x34]
+# CHECK: beq  $2, $1, 1332          # encoding: [0x4d,0x01,0x41,0x10]
+# CHECK: nop                        # encoding: [0x00,0x00,0x00,0x00]
+
+  beq $2, -0x1000100010001, 1332
+# CHECK: lui  $1, 65534             # encoding: [0xfe,0xff,0x01,0x3c]
+# CHECK: ori  $1, $1, 65534         # encoding: [0xfe,0xff,0x21,0x34]
+# CHECK: dsll $1, $1, 16            # encoding: [0x38,0x0c,0x01,0x00]
+# CHECK: ori  $1, $1, 65534         # encoding: [0xfe,0xff,0x21,0x34]
+# CHECK: dsll $1, $1, 16            # encoding: [0x38,0x0c,0x01,0x00]
+# CHECK: ori  $1, $1, 65535         # encoding: [0xff,0xff,0x21,0x34]
+# CHECK: beq  $2, $1, 1332          # encoding: [0x4d,0x01,0x41,0x10]
+# CHECK: nop                        # encoding: [0x00,0x00,0x00,0x00]
+
+# Test ulhu with 64-bit immediate addresses.
+  ulhu $8, 0x100010001
+# CHECK: lui  $1, 1            # encoding: [0x01,0x00,0x01,0x3c]
+# CHECK: ori  $1, $1, 1        # encoding: [0x01,0x00,0x21,0x34]
+# CHECK: dsll $1, $1, 16       # encoding: [0x38,0x0c,0x01,0x00]
+# CHECK: ori  $1, $1, 1        # encoding: [0x01,0x00,0x21,0x34]
+# CHECK: lbu  $8, 1($1)        # encoding: [0x01,0x00,0x28,0x90]
+# CHECK: lbu  $1, 0($1)        # encoding: [0x00,0x00,0x21,0x90]
+# CHECK: sll  $8, $8, 8        # encoding: [0x00,0x42,0x08,0x00]
+# CHECK: or   $8, $8, $1       # encoding: [0x25,0x40,0x01,0x01]
+
+  ulhu $8, 0x1000100010001
+# CHECK: lui  $1, 1            # encoding: [0x01,0x00,0x01,0x3c]
+# CHECK: ori  $1, $1, 1        # encoding: [0x01,0x00,0x21,0x34]
+# CHECK: dsll $1, $1, 16       # encoding: [0x38,0x0c,0x01,0x00]
+# CHECK: ori  $1, $1, 1        # encoding: [0x01,0x00,0x21,0x34]
+# CHECK: dsll $1, $1, 16       # encoding: [0x38,0x0c,0x01,0x00]
+# CHECK: ori  $1, $1, 1        # encoding: [0x01,0x00,0x21,0x34]
+# CHECK: lbu  $8, 1($1)        # encoding: [0x01,0x00,0x28,0x90]
+# CHECK: lbu  $1, 0($1)        # encoding: [0x00,0x00,0x21,0x90]
+# CHECK: sll  $8, $8, 8        # encoding: [0x00,0x42,0x08,0x00]
+# CHECK: or   $8, $8, $1       # encoding: [0x25,0x40,0x01,0x01]
+
+  ulhu $8, -0x100010001
+# CHECK: lui  $1, 65535        # encoding: [0xff,0xff,0x01,0x3c]
+# CHECK: ori  $1, $1, 65534    # encoding: [0xfe,0xff,0x21,0x34]
+# CHECK: dsll $1, $1, 16       # encoding: [0x38,0x0c,0x01,0x00]
+# CHECK: ori  $1, $1, 65534    # encoding: [0xfe,0xff,0x21,0x34]
+# CHECK: dsll $1, $1, 16       # encoding: [0x38,0x0c,0x01,0x00]
+# CHECK: ori  $1, $1, 65535    # encoding: [0xff,0xff,0x21,0x34]
+# CHECK: lbu  $8, 1($1)        # encoding: [0x01,0x00,0x28,0x90]
+# CHECK: lbu  $1, 0($1)        # encoding: [0x00,0x00,0x21,0x90]
+# CHECK: sll  $8, $8, 8        # encoding: [0x00,0x42,0x08,0x00]
+# CHECK: or   $8, $8, $1       # encoding: [0x25,0x40,0x01,0x01]
+
+  ulhu $8, -0x1000100010001
+# CHECK: lui  $1, 65534        # encoding: [0xfe,0xff,0x01,0x3c]
+# CHECK: ori  $1, $1, 65534    # encoding: [0xfe,0xff,0x21,0x34]
+# CHECK: dsll $1, $1, 16       # encoding: [0x38,0x0c,0x01,0x00]
+# CHECK: ori  $1, $1, 65534    # encoding: [0xfe,0xff,0x21,0x34]
+# CHECK: dsll $1, $1, 16       # encoding: [0x38,0x0c,0x01,0x00]
+# CHECK: ori  $1, $1, 65535    # encoding: [0xff,0xff,0x21,0x34]
+# CHECK: lbu  $8, 1($1)        # encoding: [0x01,0x00,0x28,0x90]
+# CHECK: lbu  $1, 0($1)        # encoding: [0x00,0x00,0x21,0x90]
+# CHECK: sll  $8, $8, 8        # encoding: [0x00,0x42,0x08,0x00]
+# CHECK: or   $8, $8, $1       # encoding: [0x25,0x40,0x01,0x01]
+
+# Test ulhu with source register and 64-bit immediate offset.
+  ulhu $8, 0x100010001($9)
+# CHECK: lui   $1, 1           # encoding: [0x01,0x00,0x01,0x3c]
+# CHECK: ori   $1, $1, 1       # encoding: [0x01,0x00,0x21,0x34]
+# CHECK: dsll  $1, $1, 16      # encoding: [0x38,0x0c,0x01,0x00]
+# CHECK: ori   $1, $1, 1       # encoding: [0x01,0x00,0x21,0x34]
+# CHECK: daddu $1, $1, $9      # encoding: [0x2d,0x08,0x29,0x00]
+# CHECK: lbu   $8, 1($1)       # encoding: [0x01,0x00,0x28,0x90]
+# CHECK: lbu   $1, 0($1)       # encoding: [0x00,0x00,0x21,0x90]
+# CHECK: sll   $8, $8, 8       # encoding: [0x00,0x42,0x08,0x00]
+# CHECK: or    $8, $8, $1      # encoding: [0x25,0x40,0x01,0x01]
+
+  ulhu $8, 0x1000100010001($9)
+# CHECK: lui   $1, 1           # encoding: [0x01,0x00,0x01,0x3c]
+# CHECK: ori   $1, $1, 1       # encoding: [0x01,0x00,0x21,0x34]
+# CHECK: dsll  $1, $1, 16      # encoding: [0x38,0x0c,0x01,0x00]
+# CHECK: ori   $1, $1, 1       # encoding: [0x01,0x00,0x21,0x34]
+# CHECK: dsll  $1, $1, 16      # encoding: [0x38,0x0c,0x01,0x00]
+# CHECK: ori   $1, $1, 1       # encoding: [0x01,0x00,0x21,0x34]
+# CHECK: daddu $1, $1, $9      # encoding: [0x2d,0x08,0x29,0x00]
+# CHECK: lbu   $8, 1($1)       # encoding: [0x01,0x00,0x28,0x90]
+# CHECK: lbu   $1, 0($1)       # encoding: [0x00,0x00,0x21,0x90]
+# CHECK: sll   $8, $8, 8       # encoding: [0x00,0x42,0x08,0x00]
+# CHECK: or    $8, $8, $1      # encoding: [0x25,0x40,0x01,0x01]
+
+  ulhu $8, -0x100010001($9)
+# CHECK: lui   $1, 65535       # encoding: [0xff,0xff,0x01,0x3c]
+# CHECK: ori   $1, $1, 65534   # encoding: [0xfe,0xff,0x21,0x34]
+# CHECK: dsll  $1, $1, 16      # encoding: [0x38,0x0c,0x01,0x00]
+# CHECK: ori   $1, $1, 65534   # encoding: [0xfe,0xff,0x21,0x34]
+# CHECK: dsll  $1, $1, 16      # encoding: [0x38,0x0c,0x01,0x00]
+# CHECK: ori   $1, $1, 65535   # encoding: [0xff,0xff,0x21,0x34]
+# CHECK: daddu $1, $1, $9      # encoding: [0x2d,0x08,0x29,0x00]
+# CHECK: lbu   $8, 1($1)       # encoding: [0x01,0x00,0x28,0x90]
+# CHECK: lbu   $1, 0($1)       # encoding: [0x00,0x00,0x21,0x90]
+# CHECK: sll   $8, $8, 8       # encoding: [0x00,0x42,0x08,0x00]
+# CHECK: or    $8, $8, $1      # encoding: [0x25,0x40,0x01,0x01]
+
+  ulhu $8, -0x1000100010001($9)
+# CHECK: lui   $1, 65534       # encoding: [0xfe,0xff,0x01,0x3c]
+# CHECK: ori   $1, $1, 65534   # encoding: [0xfe,0xff,0x21,0x34]
+# CHECK: dsll  $1, $1, 16      # encoding: [0x38,0x0c,0x01,0x00]
+# CHECK: ori   $1, $1, 65534   # encoding: [0xfe,0xff,0x21,0x34]
+# CHECK: dsll  $1, $1, 16      # encoding: [0x38,0x0c,0x01,0x00]
+# CHECK: ori   $1, $1, 65535   # encoding: [0xff,0xff,0x21,0x34]
+# CHECK: daddu $1, $1, $9      # encoding: [0x2d,0x08,0x29,0x00]
+# CHECK: lbu   $8, 1($1)       # encoding: [0x01,0x00,0x28,0x90]
+# CHECK: lbu   $1, 0($1)       # encoding: [0x00,0x00,0x21,0x90]
+# CHECK: sll   $8, $8, 8       # encoding: [0x00,0x42,0x08,0x00]
+# CHECK: or    $8, $8, $1      # encoding: [0x25,0x40,0x01,0x01]
+
+# Test ulw with 64-bit immediate addresses.
+  ulw $8, 0x100010001
+# CHECK: lui  $1, 1           # encoding: [0x01,0x00,0x01,0x3c]
+# CHECK: ori  $1, $1, 1       # encoding: [0x01,0x00,0x21,0x34]
+# CHECK: dsll $1, $1, 16      # encoding: [0x38,0x0c,0x01,0x00]
+# CHECK: ori  $1, $1, 1       # encoding: [0x01,0x00,0x21,0x34]
+# CHECK: lwl  $8, 3($1)       # encoding: [0x03,0x00,0x28,0x88]
+# CHECK: lwr  $8, 0($1)       # encoding: [0x00,0x00,0x28,0x98]
+
+  ulw $8, 0x1000100010001
+# CHECK: lui  $1, 1           # encoding: [0x01,0x00,0x01,0x3c]
+# CHECK: ori  $1, $1, 1       # encoding: [0x01,0x00,0x21,0x34]
+# CHECK: dsll $1, $1, 16      # encoding: [0x38,0x0c,0x01,0x00]
+# CHECK: ori  $1, $1, 1       # encoding: [0x01,0x00,0x21,0x34]
+# CHECK: dsll $1, $1, 16      # encoding: [0x38,0x0c,0x01,0x00]
+# CHECK: ori  $1, $1, 1       # encoding: [0x01,0x00,0x21,0x34]
+# CHECK: lwl  $8, 3($1)       # encoding: [0x03,0x00,0x28,0x88]
+# CHECK: lwr  $8, 0($1)       # encoding: [0x00,0x00,0x28,0x98]
+
+  ulw $8, -0x100010001
+# CHECK: lui  $1, 65535       # encoding: [0xff,0xff,0x01,0x3c]
+# CHECK: ori  $1, $1, 65534   # encoding: [0xfe,0xff,0x21,0x34]
+# CHECK: dsll $1, $1, 16      # encoding: [0x38,0x0c,0x01,0x00]
+# CHECK: ori  $1, $1, 65534   # encoding: [0xfe,0xff,0x21,0x34]
+# CHECK: dsll $1, $1, 16      # encoding: [0x38,0x0c,0x01,0x00]
+# CHECK: ori  $1, $1, 65535   # encoding: [0xff,0xff,0x21,0x34]
+# CHECK: lwl  $8, 3($1)       # encoding: [0x03,0x00,0x28,0x88]
+# CHECK: lwr  $8, 0($1)       # encoding: [0x00,0x00,0x28,0x98]
+
+  ulw $8, -0x1000100010001
+# CHECK: lui  $1, 65534       # encoding: [0xfe,0xff,0x01,0x3c]
+# CHECK: ori  $1, $1, 65534   # encoding: [0xfe,0xff,0x21,0x34]
+# CHECK: dsll $1, $1, 16      # encoding: [0x38,0x0c,0x01,0x00]
+# CHECK: ori  $1, $1, 65534   # encoding: [0xfe,0xff,0x21,0x34]
+# CHECK: dsll $1, $1, 16      # encoding: [0x38,0x0c,0x01,0x00]
+# CHECK: ori  $1, $1, 65535   # encoding: [0xff,0xff,0x21,0x34]
+# CHECK: lwl  $8, 3($1)       # encoding: [0x03,0x00,0x28,0x88]
+# CHECK: lwr  $8, 0($1)       # encoding: [0x00,0x00,0x28,0x98]
+
+# Test ulw with source register and 64-bit immediate offset.
+  ulw $8, 0x100010001($9)
+# CHECK: lui   $1, 1          # encoding: [0x01,0x00,0x01,0x3c]
+# CHECK: ori   $1, $1, 1      # encoding: [0x01,0x00,0x21,0x34]
+# CHECK: dsll  $1, $1, 16     # encoding: [0x38,0x0c,0x01,0x00]
+# CHECK: ori   $1, $1, 1      # encoding: [0x01,0x00,0x21,0x34]
+# CHECK: daddu $1, $1, $9     # encoding: [0x2d,0x08,0x29,0x00]
+# CHECK: lwl   $8, 3($1)      # encoding: [0x03,0x00,0x28,0x88]
+# CHECK: lwr   $8, 0($1)      # encoding: [0x00,0x00,0x28,0x98]
+
+  ulw $8, 0x1000100010001($9)
+# CHECK: lui   $1, 1          # encoding: [0x01,0x00,0x01,0x3c]
+# CHECK: ori   $1, $1, 1      # encoding: [0x01,0x00,0x21,0x34]
+# CHECK: dsll  $1, $1, 16     # encoding: [0x38,0x0c,0x01,0x00]
+# CHECK: ori   $1, $1, 1      # encoding: [0x01,0x00,0x21,0x34]
+# CHECK: dsll  $1, $1, 16     # encoding: [0x38,0x0c,0x01,0x00]
+# CHECK: ori   $1, $1, 1      # encoding: [0x01,0x00,0x21,0x34]
+# CHECK: daddu $1, $1, $9     # encoding: [0x2d,0x08,0x29,0x00]
+# CHECK: lwl   $8, 3($1)      # encoding: [0x03,0x00,0x28,0x88]
+# CHECK: lwr   $8, 0($1)      # encoding: [0x00,0x00,0x28,0x98]
+
+  ulw $8, -0x100010001($9)
+# CHECK: lui   $1, 65535      # encoding: [0xff,0xff,0x01,0x3c]
+# CHECK: ori   $1, $1, 65534  # encoding: [0xfe,0xff,0x21,0x34]
+# CHECK: dsll  $1, $1, 16     # encoding: [0x38,0x0c,0x01,0x00]
+# CHECK: ori   $1, $1, 65534  # encoding: [0xfe,0xff,0x21,0x34]
+# CHECK: dsll  $1, $1, 16     # encoding: [0x38,0x0c,0x01,0x00]
+# CHECK: ori   $1, $1, 65535  # encoding: [0xff,0xff,0x21,0x34]
+# CHECK: daddu $1, $1, $9     # encoding: [0x2d,0x08,0x29,0x00]
+# CHECK: lwl   $8, 3($1)      # encoding: [0x03,0x00,0x28,0x88]
+# CHECK: lwr   $8, 0($1)      # encoding: [0x00,0x00,0x28,0x98]
+
+  ulw $8, -0x1000100010001($9)
+# CHECK: lui   $1, 65534      # encoding: [0xfe,0xff,0x01,0x3c]
+# CHECK: ori   $1, $1, 65534  # encoding: [0xfe,0xff,0x21,0x34]
+# CHECK: dsll  $1, $1, 16     # encoding: [0x38,0x0c,0x01,0x00]
+# CHECK: ori   $1, $1, 65534  # encoding: [0xfe,0xff,0x21,0x34]
+# CHECK: dsll  $1, $1, 16     # encoding: [0x38,0x0c,0x01,0x00]
+# CHECK: ori   $1, $1, 65535  # encoding: [0xff,0xff,0x21,0x34]
+# CHECK: daddu $1, $1, $9     # encoding: [0x2d,0x08,0x29,0x00]
+# CHECK: lwl   $8, 3($1)      # encoding: [0x03,0x00,0x28,0x88]
+# CHECK: lwr   $8, 0($1)      # encoding: [0x00,0x00,0x28,0x98]
diff --git a/test/MC/Mips/mips64/valid.s b/test/MC/Mips/mips64/valid.s
index 1a65152337d..03ea6c15c33 100644
--- a/test/MC/Mips/mips64/valid.s
+++ b/test/MC/Mips/mips64/valid.s
@@ -82,10 +82,15 @@ a:
         div.d     $f29,$f20,$f27
         div.s     $f4,$f5,$f15
         divu      $zero,$25,$15
+        dmfc0     $10, $16, 2          # CHECK: dmfc0 $10, $16, 2           # encoding: [0x40,0x2a,0x80,0x02]
         dmfc1     $12,$f13
+        dmtc0     $4, $10, 0           # CHECK: dmtc0 $4, $10, 0            # encoding: [0x40,0xa4,0x50,0x00]
         dmtc1     $s0,$f14
         dmult     $s7,$9
         dmultu    $a1,$a2
+        dneg      $2                   # CHECK: dneg $2, $2                 # encoding: [0x00,0x02,0x10,0x2e]
+        dneg      $2,$3                # CHECK: dneg $2, $3                 # encoding: [0x00,0x03,0x10,0x2e]
+        dnegu     $2,$3                # CHECK: dnegu $2, $3                # encoding: [0x00,0x03,0x10,0x2f]
         dsll      $zero,18             # CHECK: dsll $zero, $zero, 18       # encoding: [0x00,0x00,0x04,0xb8]
         dsll      $zero,$s4,18         # CHECK: dsll $zero, $20, 18         # encoding: [0x00,0x14,0x04,0xb8]
         dsll      $zero,$s4,$12        # CHECK: dsllv $zero, $20, $12       # encoding: [0x01,0x94,0x00,0x14]
@@ -151,7 +156,7 @@ a:
         maddu     $24,$s2
         madd.d    $f18, $f22, $f26, $f20  # encoding: [0x4e,0xd4,0xd4,0xa1]
         madd.s    $f2, $f30, $f18, $f24   # encoding: [0x4f,0xd8,0x90,0xa0]
-        mfc0      $a2,$14,1
+        mfc0      $8,$15,1             # CHECK: mfc0 $8, $15, 1        # encoding: [0x40,0x08,0x78,0x01]
         mfc1      $a3,$f27
         mfhi      $s3
         mfhi      $sp
@@ -178,7 +183,7 @@ a:
         msubu     $15,$a1
         msub.d    $f10, $f2, $f30, $f18   # encoding: [0x4c,0x52,0xf2,0xa9]
         msub.s    $f12, $f18, $f10, $f16  # encoding: [0x4e,0x50,0x53,0x28]
-        mtc0      $9,$29,3
+        mtc0      $9,$15,1                # CHECK: mtc0 $9, $15, 1     # encoding: [0x40,0x89,0x78,0x01]
         mtc1      $s8,$f9
         mthi      $s1
         mtlo      $sp
diff --git a/test/MC/Mips/mips64r2/valid.s b/test/MC/Mips/mips64r2/valid.s
index 61b1d6dcd83..37753ae0a87 100644
--- a/test/MC/Mips/mips64r2/valid.s
+++ b/test/MC/Mips/mips64r2/valid.s
@@ -84,10 +84,15 @@ a:
         div.d     $f29,$f20,$f27
         div.s     $f4,$f5,$f15
         divu      $zero,$25,$15
+        dmfc0     $10,$16,2            # CHECK: dmfc0 $10, $16, 2           # encoding: [0x40,0x2a,0x80,0x02]
         dmfc1     $12,$f13
+        dmtc0     $4,$10,0             # CHECK: dmtc0 $4, $10, 0            # encoding: [0x40,0xa4,0x50,0x00]
         dmtc1     $s0,$f14
         dmult     $s7,$9
         dmultu    $a1,$a2
+        dneg      $2                   # CHECK: dneg $2, $2                 # encoding: [0x00,0x02,0x10,0x2e]
+        dneg      $2,$3                # CHECK: dneg $2, $3                 # encoding: [0x00,0x03,0x10,0x2e]
+        dnegu     $2,$3                # CHECK: dnegu $2, $3                # encoding: [0x00,0x03,0x10,0x2f]
         drotr     $1,15                # CHECK: drotr $1, $1, 15            # encoding: [0x00,0x21,0x0b,0xfa]
         drotr     $1,$14,15            # CHECK: drotr $1, $14, 15           # encoding: [0x00,0x2e,0x0b,0xfa]
         drotr32   $1,15                # CHECK: drotr32 $1, $1, 15          # encoding: [0x00,0x21,0x0b,0xfe]
@@ -166,7 +171,7 @@ a:
         madd.s    $f1,$f31,$f19,$f25
         maddu     $s3,$gp
         maddu     $24,$s2
-        mfc0      $a2,$14,1
+        mfc0      $8,$15,1             # CHECK: mfc0 $8, $15, 1        # encoding: [0x40,0x08,0x78,0x01]
         mfc1      $a3,$f27
         mfhc1     $s8,$f24
         mfhi      $s3
@@ -193,7 +198,7 @@ a:
         msub      $s7,$k1
         msub.s    $f12,$f19,$f10,$f16
         msubu     $15,$a1
-        mtc0      $9,$29,3
+        mtc0      $9,$15,1             # CHECK: mtc0 $9, $15, 1     # encoding: [0x40,0x89,0x78,0x01]
         mtc1      $s8,$f9
         mthc1     $zero,$f16
         mthi      $s1
diff --git a/test/MC/Mips/mips64r3/valid.s b/test/MC/Mips/mips64r3/valid.s
index bfd16d3faf2..c5d4848458d 100644
--- a/test/MC/Mips/mips64r3/valid.s
+++ b/test/MC/Mips/mips64r3/valid.s
@@ -84,10 +84,15 @@ a:
         div.d     $f29,$f20,$f27
         div.s     $f4,$f5,$f15
         divu      $zero,$25,$15
+        dmfc0     $10, $16, 2          # CHECK: dmfc0 $10, $16, 2           # encoding: [0x40,0x2a,0x80,0x02]
         dmfc1     $12,$f13
+        dmtc0     $4, $10, 0           # CHECK: dmtc0 $4, $10, 0            # encoding: [0x40,0xa4,0x50,0x00]
         dmtc1     $s0,$f14
         dmult     $s7,$9
         dmultu    $a1,$a2
+        dneg      $2                   # CHECK: dneg $2, $2                 # encoding: [0x00,0x02,0x10,0x2e]
+        dneg      $2,$3                # CHECK: dneg $2, $3                 # encoding: [0x00,0x03,0x10,0x2e]
+        dnegu     $2,$3                # CHECK: dnegu $2, $3                # encoding: [0x00,0x03,0x10,0x2f]
         drotr     $1,15                # CHECK: drotr $1, $1, 15            # encoding: [0x00,0x21,0x0b,0xfa]
         drotr     $1,$14,15            # CHECK: drotr $1, $14, 15           # encoding: [0x00,0x2e,0x0b,0xfa]
         drotr32   $1,15                # CHECK: drotr32 $1, $1, 15          # encoding: [0x00,0x21,0x0b,0xfe]
@@ -166,7 +171,7 @@ a:
         madd.s    $f1,$f31,$f19,$f25
         maddu     $s3,$gp
         maddu     $24,$s2
-        mfc0      $a2,$14,1
+        mfc0      $8,$15,1             # CHECK: mfc0 $8, $15, 1        # encoding: [0x40,0x08,0x78,0x01]
         mfc1      $a3,$f27
         mfhc1     $s8,$f24
         mfhi      $s3
@@ -193,7 +198,7 @@ a:
         msub      $s7,$k1
         msub.s    $f12,$f19,$f10,$f16
         msubu     $15,$a1
-        mtc0      $9,$29,3
+        mtc0      $9,$15,1             # CHECK: mtc0 $9, $15, 1        # encoding: [0x40,0x89,0x78,0x01]
         mtc1      $s8,$f9
         mthc1     $zero,$f16
         mthi      $s1
diff --git a/test/MC/Mips/mips64r5/invalid-mips64.s b/test/MC/Mips/mips64r5/invalid-mips64.s
new file mode 100644
index 00000000000..412fb31a104
--- /dev/null
+++ b/test/MC/Mips/mips64r5/invalid-mips64.s
@@ -0,0 +1,8 @@
+# Instructions that are invalid
+#
+# RUN: not llvm-mc %s -triple=mips-unknown-linux -show-encoding -mcpu=mips64 \
+# RUN:     2>%t1
+# RUN: FileCheck %s < %t1
+
+	.set noat
+        eretnc                      # CHECK: :[[@LINE]]:{{[0-9]+}}: error: instruction requires a CPU feature not currently enabled
diff --git a/test/MC/Mips/mips64r5/invalid-mips64r2.s b/test/MC/Mips/mips64r5/invalid-mips64r2.s
new file mode 100644
index 00000000000..1ee159429a7
--- /dev/null
+++ b/test/MC/Mips/mips64r5/invalid-mips64r2.s
@@ -0,0 +1,8 @@
+# Instructions that are invalid
+#
+# RUN: not llvm-mc %s -triple=mips-unknown-linux -show-encoding -mcpu=mips64r2 \
+# RUN:     2>%t1
+# RUN: FileCheck %s < %t1
+
+	.set noat
+        eretnc                      # CHECK: :[[@LINE]]:{{[0-9]+}}: error: instruction requires a CPU feature not currently enabled
diff --git a/test/MC/Mips/mips64r5/invalid-mips64r3.s b/test/MC/Mips/mips64r5/invalid-mips64r3.s
new file mode 100644
index 00000000000..e80d7a1e9b4
--- /dev/null
+++ b/test/MC/Mips/mips64r5/invalid-mips64r3.s
@@ -0,0 +1,8 @@
+# Instructions that are invalid
+#
+# RUN: not llvm-mc %s -triple=mips-unknown-linux -show-encoding -mcpu=mips64r3 \
+# RUN:     2>%t1
+# RUN: FileCheck %s < %t1
+
+	.set noat
+        eretnc                      # CHECK: :[[@LINE]]:{{[0-9]+}}: error: instruction requires a CPU feature not currently enabled
diff --git a/test/MC/Mips/mips64r5/valid.s b/test/MC/Mips/mips64r5/valid.s
index 22c50939cfb..c570b3e24aa 100644
--- a/test/MC/Mips/mips64r5/valid.s
+++ b/test/MC/Mips/mips64r5/valid.s
@@ -84,10 +84,15 @@ a:
         div.d     $f29,$f20,$f27
         div.s     $f4,$f5,$f15
         divu      $zero,$25,$15
+        dmfc0     $10, $16, 2          # CHECK: dmfc0 $10, $16, 2           # encoding: [0x40,0x2a,0x80,0x02]
         dmfc1     $12,$f13
+        dmtc0     $4, $10, 0           # CHECK: dmtc0 $4, $10, 0            # encoding: [0x40,0xa4,0x50,0x00]
         dmtc1     $s0,$f14
         dmult     $s7,$9
         dmultu    $a1,$a2
+        dneg      $2                   # CHECK: dneg $2, $2                 # encoding: [0x00,0x02,0x10,0x2e]
+        dneg      $2,$3                # CHECK: dneg $2, $3                 # encoding: [0x00,0x03,0x10,0x2e]
+        dnegu     $2,$3                # CHECK: dnegu $2, $3                # encoding: [0x00,0x03,0x10,0x2f]
         drotr     $1,15                # CHECK: drotr $1, $1, 15            # encoding: [0x00,0x21,0x0b,0xfa]
         drotr     $1,$14,15            # CHECK: drotr $1, $14, 15           # encoding: [0x00,0x2e,0x0b,0xfa]
         drotr32   $1,15                # CHECK: drotr32 $1, $1, 15          # encoding: [0x00,0x21,0x0b,0xfe]
@@ -127,6 +132,7 @@ a:
         ei        $14                  # CHECK: ei  $14       # encoding: [0x41,0x6e,0x60,0x20]
         ei                             # CHECK: ei            # encoding: [0x41,0x60,0x60,0x20]
         eret
+        eretnc                         # CHECK: eretnc        # encoding: [0x42,0x00,0x00,0x58]
         floor.l.d $f26,$f7
         floor.l.s $f12,$f5
         floor.w.d $f14,$f11
@@ -166,7 +172,7 @@ a:
         madd.s    $f1,$f31,$f19,$f25
         maddu     $s3,$gp
         maddu     $24,$s2
-        mfc0      $a2,$14,1
+        mfc0      $8,$15,1             # CHECK: mfc0 $8, $15, 1        # encoding: [0x40,0x08,0x78,0x01]
         mfc1      $a3,$f27
         mfhc1     $s8,$f24
         mfhi      $s3
@@ -193,7 +199,7 @@ a:
         msub      $s7,$k1
         msub.s    $f12,$f19,$f10,$f16
         msubu     $15,$a1
-        mtc0      $9,$29,3
+        mtc0      $9,$15,1             # CHECK: mtc0 $9, $15, 1        # encoding: [0x40,0x89,0x78,0x01]
         mtc1      $s8,$f9
         mthc1     $zero,$f16
         mthi      $s1
diff --git a/test/MC/Mips/mips64r6/invalid.s b/test/MC/Mips/mips64r6/invalid.s
index 1b01827368a..ae980347f30 100644
--- a/test/MC/Mips/mips64r6/invalid.s
+++ b/test/MC/Mips/mips64r6/invalid.s
@@ -10,3 +10,7 @@
         jalr.hb $31 # ASM: :[[@LINE]]:9: error: source and destination must be different
         jalr.hb $31, $31 # ASM: :[[@LINE]]:9: error: source and destination must be different
         ldc2    $8,-21181($at)   # ASM: :[[@LINE]]:{{[0-9]+}}: error: instruction requires a CPU feature not currently enabled
+        break 1024        # CHECK: :[[@LINE]]:{{[0-9]+}}: error: invalid operand for instruction
+        break 1024, 5     # CHECK: :[[@LINE]]:{{[0-9]+}}: error: invalid operand for instruction
+        break 7, 1024     # CHECK: :[[@LINE]]:{{[0-9]+}}: error: invalid operand for instruction
+        break 1024, 1024  # CHECK: :[[@LINE]]:{{[0-9]+}}: error: invalid operand for instruction
diff --git a/test/MC/Mips/mips64r6/relocations.s b/test/MC/Mips/mips64r6/relocations.s
index 651ebfb6c4c..8374cb8eaa5 100644
--- a/test/MC/Mips/mips64r6/relocations.s
+++ b/test/MC/Mips/mips64r6/relocations.s
@@ -1,6 +1,6 @@
-# RUN: llvm-mc %s -triple=mips-unknown-linux -show-encoding -mcpu=mips64r6 \
+# RUN: llvm-mc %s -triple=mips64-unknown-linux -show-encoding -mcpu=mips64r6 \
 # RUN:   | FileCheck %s -check-prefix=CHECK-FIXUP
-# RUN: llvm-mc %s -filetype=obj -triple=mips-unknown-linux -mcpu=mips64r6 \
+# RUN: llvm-mc %s -filetype=obj -triple=mips64-unknown-linux -mcpu=mips64r6 \
 # RUN:   | llvm-readobj -r | FileCheck %s -check-prefix=CHECK-ELF
 #------------------------------------------------------------------------------
 # Check that the assembler can handle the documented syntax for fixups.
@@ -10,22 +10,22 @@
 # CHECK-FIXUP:                      value: bar, kind: fixup_MIPS_PC19_S2
 # CHECK-FIXUP: beqc     $5, $6, bar # encoding: [0x20,0xa6,A,A]
 # CHECK-FIXUP:                  #   fixup A - offset: 0,
-# CHECK-FIXUP:                      value: bar, kind: fixup_Mips_PC16
+# CHECK-FIXUP:                      value: bar-4, kind: fixup_Mips_PC16
 # CHECK-FIXUP: bnec $5, $6, bar # encoding: [0x60,0xa6,A,A]
 # CHECK-FIXUP:                  #   fixup A - offset: 0,
-# CHECK-FIXUP:                      value: bar, kind: fixup_Mips_PC16
+# CHECK-FIXUP:                      value: bar-4, kind: fixup_Mips_PC16
 # CHECK-FIXUP: beqzc $9, bar    # encoding: [0xd9,0b001AAAAA,A,A]
 # CHECK-FIXUP:                  #   fixup A - offset: 0,
-# CHECK-FIXUP:                      value: bar, kind: fixup_MIPS_PC21_S2
+# CHECK-FIXUP:                      value: bar-4, kind: fixup_MIPS_PC21_S2
 # CHECK-FIXUP: bnezc $9, bar    # encoding: [0xf9,0b001AAAAA,A,A]
 # CHECK-FIXUP:                  #   fixup A - offset: 0,
-# CHECK-FIXUP:                      value: bar, kind: fixup_MIPS_PC21_S2
+# CHECK-FIXUP:                      value: bar-4, kind: fixup_MIPS_PC21_S2
 # CHECK-FIXUP: balc  bar        # encoding: [0b111010AA,A,A,A]
 # CHECK-FIXUP:                  #   fixup A - offset: 0,
-# CHECK-FIXUP:                      value: bar, kind: fixup_MIPS_PC26_S2
+# CHECK-FIXUP:                      value: bar-4, kind: fixup_MIPS_PC26_S2
 # CHECK-FIXUP: bc    bar        # encoding: [0b110010AA,A,A,A]
 # CHECK-FIXUP:                  #   fixup A - offset: 0,
-# CHECK-FIXUP:                      value: bar, kind: fixup_MIPS_PC26_S2
+# CHECK-FIXUP:                      value: bar-4, kind: fixup_MIPS_PC26_S2
 # CHECK-FIXUP: aluipc $2, %pcrel_hi(bar)    # encoding: [0xec,0x5f,A,A]
 # CHECK-FIXUP:                              #   fixup A - offset: 0,
 # CHECK-FIXUP:                                  value: bar@PCREL_HI16,
@@ -48,18 +48,18 @@
 # Check that the appropriate relocations were created.
 #------------------------------------------------------------------------------
 # CHECK-ELF: Relocations [
-# CHECK-ELF:     0x0 R_MIPS_PC19_S2 bar 0x0
-# CHECK-ELF:     0x4 R_MIPS_PC16 bar 0x0
-# CHECK-ELF:     0x8 R_MIPS_PC16 bar 0x0
-# CHECK-ELF:     0xC R_MIPS_PC21_S2 bar 0x0
-# CHECK-ELF:     0x10 R_MIPS_PC21_S2 bar 0x0
-# CHECK-ELF:     0x14 R_MIPS_PC26_S2 bar 0x0
-# CHECK-ELF:     0x18 R_MIPS_PC26_S2 bar 0x0
-# CHECK-ELF:     0x1C R_MIPS_PCHI16 bar 0x0
-# CHECK-ELF:     0x20 R_MIPS_PCLO16 bar 0x0
-# CHECK-ELF:     0x24 R_MIPS_PC18_S3 bar 0x0
-# CHECK-ELF:     0x28 R_MIPS_PC19_S2 bar 0x0
-# CHECK-ELF:     0x2C R_MIPS_PC19_S2 bar 0x0
+# CHECK-ELF:     0x0 R_MIPS_PC19_S2/R_MIPS_NONE/R_MIPS_NONE bar 0x0
+# CHECK-ELF:     0x4 R_MIPS_PC16/R_MIPS_NONE/R_MIPS_NONE bar 0xFFFFFFFFFFFFFFFC
+# CHECK-ELF:     0x8 R_MIPS_PC16/R_MIPS_NONE/R_MIPS_NONE bar 0xFFFFFFFFFFFFFFFC
+# CHECK-ELF:     0xC R_MIPS_PC21_S2/R_MIPS_NONE/R_MIPS_NONE bar 0xFFFFFFFFFFFFFFFC
+# CHECK-ELF:     0x10 R_MIPS_PC21_S2/R_MIPS_NONE/R_MIPS_NONE bar 0xFFFFFFFFFFFFFFFC
+# CHECK-ELF:     0x14 R_MIPS_PC26_S2/R_MIPS_NONE/R_MIPS_NONE bar 0xFFFFFFFFFFFFFFFC
+# CHECK-ELF:     0x18 R_MIPS_PC26_S2/R_MIPS_NONE/R_MIPS_NONE bar 0xFFFFFFFFFFFFFFFC
+# CHECK-ELF:     0x1C R_MIPS_PCHI16/R_MIPS_NONE/R_MIPS_NONE bar 0x0
+# CHECK-ELF:     0x20 R_MIPS_PCLO16/R_MIPS_NONE/R_MIPS_NONE bar 0x0
+# CHECK-ELF:     0x24 R_MIPS_PC18_S3/R_MIPS_NONE/R_MIPS_NONE bar 0x0
+# CHECK-ELF:     0x28 R_MIPS_PC19_S2/R_MIPS_NONE/R_MIPS_NONE bar 0x0
+# CHECK-ELF:     0x2C R_MIPS_PC19_S2/R_MIPS_NONE/R_MIPS_NONE bar 0x0
 # CHECK-ELF: ]
 
   addiupc   $2,bar
diff --git a/test/MC/Mips/mips64r6/valid.s b/test/MC/Mips/mips64r6/valid.s
index 31a0d7f6d36..6249d51781c 100644
--- a/test/MC/Mips/mips64r6/valid.s
+++ b/test/MC/Mips/mips64r6/valid.s
@@ -15,11 +15,11 @@
 a:
         .set noat
         # FIXME: Add the instructions carried forward from older ISA's
-        and     $2,4           # CHECK: andi $2, $2, 4        # encoding: [0x30,0x42,0x00,0x04]
         addiupc $4, 100          # CHECK: addiupc $4, 100     # encoding: [0xec,0x80,0x00,0x19]
         addu    $9,10            # CHECK: addiu $9, $9, 10    # encoding: [0x25,0x29,0x00,0x0a]
         align   $4, $2, $3, 2    # CHECK: align $4, $2, $3, 2 # encoding: [0x7c,0x43,0x22,0xa0]
         aluipc  $3, 56           # CHECK: aluipc $3, 56       # encoding: [0xec,0x7f,0x00,0x38]
+        and     $2,4             # CHECK: andi $2, $2, 4      # encoding: [0x30,0x42,0x00,0x04]
         aui     $3,$2,-23        # CHECK: aui $3, $2, -23     # encoding: [0x3c,0x62,0xff,0xe9]
         auipc   $3, -1           # CHECK: auipc $3, -1        # encoding: [0xec,0x7e,0xff,0xff]
         bal     21100            # CHECK: bal 21100           # encoding: [0x04,0x11,0x14,0x9b]
@@ -39,23 +39,23 @@ a:
         beqzalc $2, 1332         # CHECK: beqzalc $2, 1332    # encoding: [0x20,0x02,0x01,0x4d]
         # bnec requires rs < rt && rs != 0 but we accept when this is not true. See also bnvc
         # FIXME: Testcases are in valid-xfail.s at the moment
-        bnec $5, $6, 256         # CHECK: bnec $5, $6, 256    # encoding: [0x60,0xa6,0x00,0x40]
-        bnezalc $2, 1332         # CHECK: bnezalc $2, 1332    # encoding: [0x60,0x02,0x01,0x4d]
         beqzc $5, 72256          # CHECK: beqzc $5, 72256     # encoding: [0xd8,0xa0,0x46,0x90]
         bgec $2, $3, 256         # CHECK: bgec $2, $3, 256    # encoding: [0x58,0x43,0x00,0x40]
         bgeuc $2, $3, 256        # CHECK: bgeuc $2, $3, 256   # encoding: [0x18,0x43,0x00,0x40]
         bgezalc $2, 1332         # CHECK: bgezalc $2, 1332    # encoding: [0x18,0x42,0x01,0x4d]
-        bnezc $5, 72256          # CHECK: bnezc $5, 72256     # encoding: [0xf8,0xa0,0x46,0x90]
-        bltzc $5, 256            # CHECK: bltzc $5, 256       # encoding: [0x5c,0xa5,0x00,0x40]
         bgezc $5, 256            # CHECK: bgezc $5, 256       # encoding: [0x58,0xa5,0x00,0x40]
         bgtzalc $2, 1332         # CHECK: bgtzalc $2, 1332    # encoding: [0x1c,0x02,0x01,0x4d]
-        blezc $5, 256            # CHECK: blezc $5, 256       # encoding: [0x58,0x05,0x00,0x40]
-        bltzalc $2, 1332         # CHECK: bltzalc $2, 1332    # encoding: [0x1c,0x42,0x01,0x4d]
         bgtzc $5, 256            # CHECK: bgtzc $5, 256       # encoding: [0x5c,0x05,0x00,0x40]
         bitswap $4, $2           # CHECK: bitswap $4, $2      # encoding: [0x7c,0x02,0x20,0x20]
         blezalc $2, 1332         # CHECK: blezalc $2, 1332    # encoding: [0x18,0x02,0x01,0x4d]
+        blezc $5, 256            # CHECK: blezc $5, 256       # encoding: [0x58,0x05,0x00,0x40]
         bltc $5, $6, 256         # CHECK: bltc $5, $6, 256    # encoding: [0x5c,0xa6,0x00,0x40]
         bltuc $5, $6, 256        # CHECK: bltuc $5, $6, 256   # encoding: [0x1c,0xa6,0x00,0x40]
+        bltzalc $2, 1332         # CHECK: bltzalc $2, 1332    # encoding: [0x1c,0x42,0x01,0x4d]
+        bltzc $5, 256            # CHECK: bltzc $5, 256       # encoding: [0x5c,0xa5,0x00,0x40]
+        bnec $5, $6, 256         # CHECK: bnec $5, $6, 256    # encoding: [0x60,0xa6,0x00,0x40]
+        bnezalc $2, 1332         # CHECK: bnezalc $2, 1332    # encoding: [0x60,0x02,0x01,0x4d]
+        bnezc $5, 72256          # CHECK: bnezc $5, 72256     # encoding: [0xf8,0xa0,0x46,0x90]
         # bnvc requires that rs >= rt but we accept both. See also bnec
         bnvc     $0, $0, 4       # CHECK: bnvc $zero, $zero, 4 # encoding: [0x60,0x00,0x00,0x01]
         bnvc     $2, $0, 4       # CHECK: bnvc $2, $zero, 4    # encoding: [0x60,0x40,0x00,0x01]
@@ -65,138 +65,146 @@ a:
         bovc     $2, $0, 4       # CHECK: bovc $2, $zero, 4    # encoding: [0x20,0x40,0x00,0x01]
         bovc     $4, $2, 4       # CHECK: bovc $4, $2, 4      # encoding: [0x20,0x82,0x00,0x01]
         cache      1, 8($5)         # CHECK: cache 1, 8($5)         # encoding: [0x7c,0xa1,0x04,0x25]
-        cmp.af.s   $f2,$f3,$f4      # CHECK: cmp.af.s $f2, $f3, $f4  # encoding: [0x46,0x84,0x18,0x80]
+        class.d $f2, $f4         # CHECK: class.d $f2, $f4       # encoding: [0x46,0x20,0x20,0x9b]
+        class.s $f2, $f4         # CHECK: class.s $f2, $f4       # encoding: [0x46,0x00,0x20,0x9b]
+        clo     $11,$a1          # CHECK: clo $11, $5            # encoding: [0x00,0xa0,0x58,0x51]
+        clz     $sp,$gp          # CHECK: clz $sp, $gp           # encoding: [0x03,0x80,0xe8,0x50]
         cmp.af.d   $f2,$f3,$f4      # CHECK: cmp.af.d $f2, $f3, $f4  # encoding: [0x46,0xa4,0x18,0x80]
-        cmp.un.s   $f2,$f3,$f4      # CHECK: cmp.un.s $f2, $f3, $f4  # encoding: [0x46,0x84,0x18,0x81]
-        cmp.un.d   $f2,$f3,$f4      # CHECK: cmp.un.d $f2, $f3, $f4  # encoding: [0x46,0xa4,0x18,0x81]
-        cmp.eq.s   $f2,$f3,$f4      # CHECK: cmp.eq.s $f2, $f3, $f4  # encoding: [0x46,0x84,0x18,0x82]
+        cmp.af.s   $f2,$f3,$f4      # CHECK: cmp.af.s $f2, $f3, $f4  # encoding: [0x46,0x84,0x18,0x80]
         cmp.eq.d   $f2,$f3,$f4      # CHECK: cmp.eq.d $f2, $f3, $f4  # encoding: [0x46,0xa4,0x18,0x82]
-        cmp.ueq.s  $f2,$f3,$f4      # CHECK: cmp.ueq.s $f2, $f3, $f4  # encoding: [0x46,0x84,0x18,0x83]
-        cmp.ueq.d  $f2,$f3,$f4      # CHECK: cmp.ueq.d $f2, $f3, $f4  # encoding: [0x46,0xa4,0x18,0x83]
-        cmp.lt.s   $f2,$f3,$f4      # CHECK: cmp.lt.s  $f2, $f3, $f4  # encoding: [0x46,0x84,0x18,0x84]
-        cmp.lt.d   $f2,$f3,$f4      # CHECK: cmp.lt.d  $f2, $f3, $f4  # encoding: [0x46,0xa4,0x18,0x84]
-        cmp.ult.s  $f2,$f3,$f4      # CHECK: cmp.ult.s $f2, $f3, $f4  # encoding: [0x46,0x84,0x18,0x85]
-        cmp.ult.d  $f2,$f3,$f4      # CHECK: cmp.ult.d $f2, $f3, $f4  # encoding: [0x46,0xa4,0x18,0x85]
-        cmp.le.s   $f2,$f3,$f4      # CHECK: cmp.le.s  $f2, $f3, $f4  # encoding: [0x46,0x84,0x18,0x86]
+        cmp.eq.s   $f2,$f3,$f4      # CHECK: cmp.eq.s $f2, $f3, $f4  # encoding: [0x46,0x84,0x18,0x82]
         cmp.le.d   $f2,$f3,$f4      # CHECK: cmp.le.d  $f2, $f3, $f4  # encoding: [0x46,0xa4,0x18,0x86]
-        cmp.ule.s  $f2,$f3,$f4      # CHECK: cmp.ule.s $f2, $f3, $f4  # encoding: [0x46,0x84,0x18,0x87]
-        cmp.ule.d  $f2,$f3,$f4      # CHECK: cmp.ule.d $f2, $f3, $f4  # encoding: [0x46,0xa4,0x18,0x87]
-        cmp.saf.s  $f2,$f3,$f4      # CHECK: cmp.saf.s $f2, $f3, $f4  # encoding: [0x46,0x84,0x18,0x88]
+        cmp.le.s   $f2,$f3,$f4      # CHECK: cmp.le.s  $f2, $f3, $f4  # encoding: [0x46,0x84,0x18,0x86]
+        cmp.lt.d   $f2,$f3,$f4      # CHECK: cmp.lt.d  $f2, $f3, $f4  # encoding: [0x46,0xa4,0x18,0x84]
+        cmp.lt.s   $f2,$f3,$f4      # CHECK: cmp.lt.s  $f2, $f3, $f4  # encoding: [0x46,0x84,0x18,0x84]
         cmp.saf.d  $f2,$f3,$f4      # CHECK: cmp.saf.d $f2, $f3, $f4  # encoding: [0x46,0xa4,0x18,0x88]
-        cmp.sun.s  $f2,$f3,$f4      # CHECK: cmp.sun.s $f2, $f3, $f4  # encoding: [0x46,0x84,0x18,0x89]
-        cmp.sun.d  $f2,$f3,$f4      # CHECK: cmp.sun.d $f2, $f3, $f4  # encoding: [0x46,0xa4,0x18,0x89]
-        cmp.seq.s  $f2,$f3,$f4      # CHECK: cmp.seq.s $f2, $f3, $f4  # encoding: [0x46,0x84,0x18,0x8a]
+        cmp.saf.s  $f2,$f3,$f4      # CHECK: cmp.saf.s $f2, $f3, $f4  # encoding: [0x46,0x84,0x18,0x88]
         cmp.seq.d  $f2,$f3,$f4      # CHECK: cmp.seq.d $f2, $f3, $f4  # encoding: [0x46,0xa4,0x18,0x8a]
-        cmp.sueq.s $f2,$f3,$f4      # CHECK: cmp.sueq.s $f2, $f3, $f4 # encoding: [0x46,0x84,0x18,0x8b]
-        cmp.sueq.d $f2,$f3,$f4      # CHECK: cmp.sueq.d $f2, $f3, $f4 # encoding: [0x46,0xa4,0x18,0x8b]
-        cmp.slt.s  $f2,$f3,$f4      # CHECK: cmp.slt.s $f2, $f3, $f4  # encoding: [0x46,0x84,0x18,0x8c]
-        cmp.slt.d  $f2,$f3,$f4      # CHECK: cmp.slt.d $f2, $f3, $f4  # encoding: [0x46,0xa4,0x18,0x8c]
-        cmp.sult.s $f2,$f3,$f4      # CHECK: cmp.sult.s $f2, $f3, $f4 # encoding: [0x46,0x84,0x18,0x8d]
-        cmp.sult.d $f2,$f3,$f4      # CHECK: cmp.sult.d $f2, $f3, $f4 # encoding: [0x46,0xa4,0x18,0x8d]
-        cmp.sle.s  $f2,$f3,$f4      # CHECK: cmp.sle.s $f2, $f3, $f4  # encoding: [0x46,0x84,0x18,0x8e]
+        cmp.seq.s  $f2,$f3,$f4      # CHECK: cmp.seq.s $f2, $f3, $f4  # encoding: [0x46,0x84,0x18,0x8a]
         cmp.sle.d  $f2,$f3,$f4      # CHECK: cmp.sle.d $f2, $f3, $f4  # encoding: [0x46,0xa4,0x18,0x8e]
-        cmp.sule.s $f2,$f3,$f4      # CHECK: cmp.sule.s $f2, $f3, $f4 # encoding: [0x46,0x84,0x18,0x8f]
+        cmp.sle.s  $f2,$f3,$f4      # CHECK: cmp.sle.s $f2, $f3, $f4  # encoding: [0x46,0x84,0x18,0x8e]
+        cmp.slt.d  $f2,$f3,$f4      # CHECK: cmp.slt.d $f2, $f3, $f4  # encoding: [0x46,0xa4,0x18,0x8c]
+        cmp.slt.s  $f2,$f3,$f4      # CHECK: cmp.slt.s $f2, $f3, $f4  # encoding: [0x46,0x84,0x18,0x8c]
+        cmp.sueq.d $f2,$f3,$f4      # CHECK: cmp.sueq.d $f2, $f3, $f4 # encoding: [0x46,0xa4,0x18,0x8b]
+        cmp.sueq.s $f2,$f3,$f4      # CHECK: cmp.sueq.s $f2, $f3, $f4 # encoding: [0x46,0x84,0x18,0x8b]
         cmp.sule.d $f2,$f3,$f4      # CHECK: cmp.sule.d $f2, $f3, $f4 # encoding: [0x46,0xa4,0x18,0x8f]
-        daddu   $24,$2,18079     # CHECK: daddiu $24, $2, 18079  # encoding: [0x64,0x58,0x46,0x9f]
+        cmp.sule.s $f2,$f3,$f4      # CHECK: cmp.sule.s $f2, $f3, $f4 # encoding: [0x46,0x84,0x18,0x8f]
+        cmp.sult.d $f2,$f3,$f4      # CHECK: cmp.sult.d $f2, $f3, $f4 # encoding: [0x46,0xa4,0x18,0x8d]
+        cmp.sult.s $f2,$f3,$f4      # CHECK: cmp.sult.s $f2, $f3, $f4 # encoding: [0x46,0x84,0x18,0x8d]
+        cmp.sun.d  $f2,$f3,$f4      # CHECK: cmp.sun.d $f2, $f3, $f4  # encoding: [0x46,0xa4,0x18,0x89]
+        cmp.sun.s  $f2,$f3,$f4      # CHECK: cmp.sun.s $f2, $f3, $f4  # encoding: [0x46,0x84,0x18,0x89]
+        cmp.ueq.d  $f2,$f3,$f4      # CHECK: cmp.ueq.d $f2, $f3, $f4  # encoding: [0x46,0xa4,0x18,0x83]
+        cmp.ueq.s  $f2,$f3,$f4      # CHECK: cmp.ueq.s $f2, $f3, $f4  # encoding: [0x46,0x84,0x18,0x83]
+        cmp.ule.d  $f2,$f3,$f4      # CHECK: cmp.ule.d $f2, $f3, $f4  # encoding: [0x46,0xa4,0x18,0x87]
+        cmp.ule.s  $f2,$f3,$f4      # CHECK: cmp.ule.s $f2, $f3, $f4  # encoding: [0x46,0x84,0x18,0x87]
+        cmp.ult.d  $f2,$f3,$f4      # CHECK: cmp.ult.d $f2, $f3, $f4  # encoding: [0x46,0xa4,0x18,0x85]
+        cmp.ult.s  $f2,$f3,$f4      # CHECK: cmp.ult.s $f2, $f3, $f4  # encoding: [0x46,0x84,0x18,0x85]
+        cmp.un.d   $f2,$f3,$f4      # CHECK: cmp.un.d $f2, $f3, $f4  # encoding: [0x46,0xa4,0x18,0x81]
+        cmp.un.s   $f2,$f3,$f4      # CHECK: cmp.un.s $f2, $f3, $f4  # encoding: [0x46,0x84,0x18,0x81]
         daddu   $19,26943        # CHECK: daddiu $19, $19, 26943 # encoding: [0x66,0x73,0x69,0x3f]
-        dalign  $4,$2,$3,5       # CHECK: dalign $4, $2, $3, 5 # encoding: [0x7c,0x43,0x23,0x64]
-        daui    $3,$2,0x1234     # CHECK: daui $3, $2, 4660  # encoding: [0x74,0x62,0x12,0x34]
+        daddu   $24,$2,18079     # CHECK: daddiu $24, $2, 18079  # encoding: [0x64,0x58,0x46,0x9f]
         dahi     $3,0x5678       # CHECK: dahi $3, 22136     # encoding: [0x04,0x66,0x56,0x78]
+        dalign  $4,$2,$3,5       # CHECK: dalign $4, $2, $3, 5 # encoding: [0x7c,0x43,0x23,0x64]
         dati     $3,0xabcd       # CHECK: dati $3, 43981     # encoding: [0x04,0x7e,0xab,0xcd]
+        daui    $3,$2,0x1234     # CHECK: daui $3, $2, 4660  # encoding: [0x74,0x62,0x12,0x34]
         dbitswap $4, $2          # CHECK: dbitswap $4, $2    # encoding: [0x7c,0x02,0x20,0x24]
-        di      $s8              # CHECK: di  $fp          # encoding: [0x41,0x7e,0x60,0x00]
+        dclo    $s2,$a2          # CHECK: dclo $18, $6           # encoding: [0x00,0xc0,0x90,0x53]
+        dclz    $s0,$25          # CHECK: dclz $16, $25          # encoding: [0x03,0x20,0x80,0x52]
+        ddiv    $2,$3,$4         # CHECK: ddiv $2, $3, $4  # encoding: [0x00,0x64,0x10,0x9e]
+        ddivu   $2,$3,$4         # CHECK: ddivu $2, $3, $4 # encoding: [0x00,0x64,0x10,0x9f]
         di                       # CHECK: di               # encoding: [0x41,0x60,0x60,0x00]
+        di      $s8              # CHECK: di  $fp          # encoding: [0x41,0x7e,0x60,0x00]
         div     $2,$3,$4         # CHECK: div $2, $3, $4   # encoding: [0x00,0x64,0x10,0x9a]
         divu    $2,$3,$4         # CHECK: divu $2, $3, $4  # encoding: [0x00,0x64,0x10,0x9b]
-        dsubu   $15,$11,5025     # CHECK: daddiu $15, $11, -5025 # encoding: [0x65,0x6f,0xec,0x5f]
+        dlsa    $2, $3, $4, 3    # CHECK: dlsa $2, $3, $4, 3 # encoding: [0x00,0x64,0x10,0xd5]
+        dmfc0   $10, $16, 2      # CHECK: dmfc0 $10, $16, 2  # encoding: [0x40,0x2a,0x80,0x02]
+        dmod    $2,$3,$4         # CHECK: dmod $2, $3, $4  # encoding: [0x00,0x64,0x10,0xde]
+        dmodu   $2,$3,$4         # CHECK: dmodu $2, $3, $4 # encoding: [0x00,0x64,0x10,0xdf]
+        dmtc0   $4, $10, 0       # CHECK: dmtc0 $4, $10, 0 # encoding: [0x40,0xa4,0x50,0x00]
+        dmuh    $2,$3,$4         # CHECK: dmuh $2, $3, $4  # encoding: [0x00,0x64,0x10,0xdc]
+        dmuhu   $2,$3,$4         # CHECK: dmuhu $2, $3, $4 # encoding: [0x00,0x64,0x10,0xdd]
+        dmul    $2,$3,$4         # CHECK: dmul $2, $3, $4  # encoding: [0x00,0x64,0x10,0x9c]
+        dmulu   $2,$3,$4         # CHECK: dmulu $2, $3, $4 # encoding: [0x00,0x64,0x10,0x9d]
+        dneg      $2                   # CHECK: dneg $2, $2                 # encoding: [0x00,0x02,0x10,0x2e]
+        dneg      $2,$3                # CHECK: dneg $2, $3                 # encoding: [0x00,0x03,0x10,0x2e]
+        dnegu     $2,$3                # CHECK: dnegu $2, $3                # encoding: [0x00,0x03,0x10,0x2f]
         dsubu   $14,-4586        # CHECK: daddiu $14, $14, 4586  # encoding: [0x65,0xce,0x11,0xea]
-        ei      $14              # CHECK: ei  $14          # encoding: [0x41,0x6e,0x60,0x20]
+        dsubu   $15,$11,5025     # CHECK: daddiu $15, $11, -5025 # encoding: [0x65,0x6f,0xec,0x5f]
         ei                       # CHECK: ei               # encoding: [0x41,0x60,0x60,0x20]
+        ei      $14              # CHECK: ei  $14          # encoding: [0x41,0x6e,0x60,0x20]
+        eretnc                   # CHECK: eretnc                 # encoding: [0x42,0x00,0x00,0x58]
+        j       1f               # CHECK: j $tmp0                # encoding: [0b000010AA,A,A,A]
+                                 # CHECK:                        #   fixup A - offset: 0, value: ($tmp0), kind: fixup_Mips_26
+        j       a                # CHECK: j a                    # encoding: [0b000010AA,A,A,A]
+                                 # CHECK:                        #   fixup A - offset: 0, value: a, kind: fixup_Mips_26
+        j       1328             # CHECK: j 1328                 # encoding: [0x08,0x00,0x01,0x4c]
+        jr.hb   $4               # CHECK: jr.hb $4               # encoding: [0x00,0x80,0x04,0x09]
+        jalr.hb $4               # CHECK: jalr.hb $4             # encoding: [0x00,0x80,0xfc,0x09]
+        jalr.hb $4, $5           # CHECK: jalr.hb $4, $5         # encoding: [0x00,0xa0,0x24,0x09]
         jialc   $5, 256          # CHECK: jialc $5, 256    # encoding: [0xf8,0x05,0x01,0x00]
         jic     $5, 256          # CHECK: jic $5, 256      # encoding: [0xd8,0x05,0x01,0x00]
-        mod     $2,$3,$4         # CHECK: mod $2, $3, $4   # encoding: [0x00,0x64,0x10,0xda]
-        modu    $2,$3,$4         # CHECK: modu $2, $3, $4  # encoding: [0x00,0x64,0x10,0xdb]
-        ddiv    $2,$3,$4         # CHECK: ddiv $2, $3, $4  # encoding: [0x00,0x64,0x10,0x9e]
-        ddivu   $2,$3,$4         # CHECK: ddivu $2, $3, $4 # encoding: [0x00,0x64,0x10,0x9f]
-        dmod    $2,$3,$4         # CHECK: dmod $2, $3, $4  # encoding: [0x00,0x64,0x10,0xde]
-        dmodu   $2,$3,$4         # CHECK: dmodu $2, $3, $4 # encoding: [0x00,0x64,0x10,0xdf]
-        lsa     $2, $3, $4, 3    # CHECK: lsa  $2, $3, $4, 3 # encoding: [0x00,0x64,0x10,0xc5]
-        dlsa    $2, $3, $4, 3    # CHECK: dlsa $2, $3, $4, 3 # encoding: [0x00,0x64,0x10,0xd5]
+        ldc2    $8, -701($at)    # CHECK: ldc2 $8, -701($1)      # encoding: [0x49,0xc8,0x0d,0x43]
         ldpc    $2,123456        # CHECK: ldpc $2, 123456  # encoding: [0xec,0x58,0x3c,0x48]
+        ll      $v0,-153($s2)    # CHECK: ll $2, -153($18)       # encoding: [0x7e,0x42,0xb3,0xb6]
+        lld     $zero,112($ra)   # CHECK: lld $zero, 112($ra)    # encoding: [0x7f,0xe0,0x38,0x37]
+        lsa     $2, $3, $4, 3    # CHECK: lsa  $2, $3, $4, 3 # encoding: [0x00,0x64,0x10,0xc5]
+        lwc2    $18,-841($a2)    # CHECK: lwc2 $18, -841($6)     # encoding: [0x49,0x52,0x34,0xb7]
         lwpc    $2,268           # CHECK: lwpc $2, 268     # encoding: [0xec,0x48,0x00,0x43]
         lwupc   $2,268           # CHECK: lwupc $2, 268    # encoding: [0xec,0x50,0x00,0x43]
-        mul     $2,$3,$4         # CHECK: mul $2, $3, $4   # encoding: [0x00,0x64,0x10,0x98]
-        muh     $2,$3,$4         # CHECK: muh $2, $3, $4   # encoding: [0x00,0x64,0x10,0xd8]
-        mulu    $2,$3,$4         # CHECK: mulu $2, $3, $4  # encoding: [0x00,0x64,0x10,0x99]
-        muhu    $2,$3,$4         # CHECK: muhu $2, $3, $4  # encoding: [0x00,0x64,0x10,0xd9]
-        dmul    $2,$3,$4         # CHECK: dmul $2, $3, $4  # encoding: [0x00,0x64,0x10,0x9c]
-        dmuh    $2,$3,$4         # CHECK: dmuh $2, $3, $4  # encoding: [0x00,0x64,0x10,0xdc]
-        dmulu   $2,$3,$4         # CHECK: dmulu $2, $3, $4 # encoding: [0x00,0x64,0x10,0x9d]
-        dmuhu   $2,$3,$4         # CHECK: dmuhu $2, $3, $4 # encoding: [0x00,0x64,0x10,0xdd]
-        maddf.s $f2,$f3,$f4      # CHECK: maddf.s $f2, $f3, $f4  # encoding: [0x46,0x04,0x18,0x98]
         maddf.d $f2,$f3,$f4      # CHECK: maddf.d $f2, $f3, $f4  # encoding: [0x46,0x24,0x18,0x98]
-        msubf.s $f2,$f3,$f4      # CHECK: msubf.s $f2, $f3, $f4  # encoding: [0x46,0x04,0x18,0x99]
-        msubf.d $f2,$f3,$f4      # CHECK: msubf.d $f2, $f3, $f4  # encoding: [0x46,0x24,0x18,0x99]
-        pref    1, 8($5)         # CHECK: pref 1, 8($5)          # encoding: [0x7c,0xa1,0x04,0x35]
-        sel.d   $f0,$f1,$f2      # CHECK: sel.d $f0, $f1, $f2 # encoding: [0x46,0x22,0x08,0x10]
-        sel.s   $f0,$f1,$f2      # CHECK: sel.s $f0, $f1, $f2 # encoding: [0x46,0x02,0x08,0x10]
-        seleqz  $2,$3,$4         # CHECK: seleqz $2, $3, $4 # encoding: [0x00,0x64,0x10,0x35]
-        selnez  $2,$3,$4         # CHECK: selnez $2, $3, $4 # encoding: [0x00,0x64,0x10,0x37]
-        max.s   $f0, $f2, $f4    # CHECK: max.s $f0, $f2, $f4 # encoding: [0x46,0x04,0x10,0x1d]
+        maddf.s $f2,$f3,$f4      # CHECK: maddf.s $f2, $f3, $f4  # encoding: [0x46,0x04,0x18,0x98]
         max.d   $f0, $f2, $f4    # CHECK: max.d $f0, $f2, $f4 # encoding: [0x46,0x24,0x10,0x1d]
-        min.s   $f0, $f2, $f4    # CHECK: min.s $f0, $f2, $f4 # encoding: [0x46,0x04,0x10,0x1c]
-        min.d   $f0, $f2, $f4    # CHECK: min.d $f0, $f2, $f4 # encoding: [0x46,0x24,0x10,0x1c]
-        maxa.s  $f0, $f2, $f4    # CHECK: maxa.s $f0, $f2, $f4 # encoding: [0x46,0x04,0x10,0x1f]
+        max.s   $f0, $f2, $f4    # CHECK: max.s $f0, $f2, $f4 # encoding: [0x46,0x04,0x10,0x1d]
         maxa.d  $f0, $f2, $f4    # CHECK: maxa.d $f0, $f2, $f4 # encoding: [0x46,0x24,0x10,0x1f]
-        mina.s  $f0, $f2, $f4    # CHECK: mina.s $f0, $f2, $f4 # encoding: [0x46,0x04,0x10,0x1e]
+        maxa.s  $f0, $f2, $f4    # CHECK: maxa.s $f0, $f2, $f4 # encoding: [0x46,0x04,0x10,0x1f]
+        min.d   $f0, $f2, $f4    # CHECK: min.d $f0, $f2, $f4 # encoding: [0x46,0x24,0x10,0x1c]
+        min.s   $f0, $f2, $f4    # CHECK: min.s $f0, $f2, $f4 # encoding: [0x46,0x04,0x10,0x1c]
         mina.d  $f0, $f2, $f4    # CHECK: mina.d $f0, $f2, $f4 # encoding: [0x46,0x24,0x10,0x1e]
+        mina.s  $f0, $f2, $f4    # CHECK: mina.s $f0, $f2, $f4 # encoding: [0x46,0x04,0x10,0x1e]
+        mfc0    $8,$15,1         # CHECK: mfc0 $8, $15, 1      # encoding: [0x40,0x08,0x78,0x01]
+        mod     $2,$3,$4         # CHECK: mod $2, $3, $4   # encoding: [0x00,0x64,0x10,0xda]
+        modu    $2,$3,$4         # CHECK: modu $2, $3, $4  # encoding: [0x00,0x64,0x10,0xdb]
+        mtc0    $9,$15,1         # CHECK: mtc0 $9, $15, 1        # encoding: [0x40,0x89,0x78,0x01]
+        msubf.d $f2,$f3,$f4      # CHECK: msubf.d $f2, $f3, $f4  # encoding: [0x46,0x24,0x18,0x99]
+        msubf.s $f2,$f3,$f4      # CHECK: msubf.s $f2, $f3, $f4  # encoding: [0x46,0x04,0x18,0x99]
+        muh     $2,$3,$4         # CHECK: muh $2, $3, $4   # encoding: [0x00,0x64,0x10,0xd8]
+        muhu    $2,$3,$4         # CHECK: muhu $2, $3, $4  # encoding: [0x00,0x64,0x10,0xd9]
+        mul     $2,$3,$4         # CHECK: mul $2, $3, $4   # encoding: [0x00,0x64,0x10,0x98]
+        mulu    $2,$3,$4         # CHECK: mulu $2, $3, $4  # encoding: [0x00,0x64,0x10,0x99]
         or      $2, 4            # CHECK: ori $2, $2, 4          # encoding: [0x34,0x42,0x00,0x04]
-        seleqz.s $f0, $f2, $f4   # CHECK: seleqz.s $f0, $f2, $f4 # encoding: [0x46,0x04,0x10,0x14]
-        seleqz.d $f0, $f2, $f4   # CHECK: seleqz.d $f0, $f2, $f4 # encoding: [0x46,0x24,0x10,0x14]
-        selnez.s $f0, $f2, $f4   # CHECK: selnez.s $f0, $f2, $f4 # encoding: [0x46,0x04,0x10,0x17]
-        selnez.d $f0, $f2, $f4   # CHECK: selnez.d $f0, $f2, $f4 # encoding: [0x46,0x24,0x10,0x17]
+        pref    1, 8($5)         # CHECK: pref 1, 8($5)          # encoding: [0x7c,0xa1,0x04,0x35]
         # FIXME: Use the code generator in order to print the .set directives
         #        instead of the instruction printer.
         rdhwr   $sp,$11          # CHECK:      .set  push
                                  # CHECK-NEXT: .set  mips32r2
                                  # CHECK-NEXT: rdhwr $sp, $11
                                  # CHECK-NEXT: .set  pop         # encoding: [0x7c,0x1d,0x58,0x3b]
-        rint.s $f2, $f4          # CHECK: rint.s $f2, $f4        # encoding: [0x46,0x00,0x20,0x9a]
         rint.d $f2, $f4          # CHECK: rint.d $f2, $f4        # encoding: [0x46,0x20,0x20,0x9a]
-        class.s $f2, $f4         # CHECK: class.s $f2, $f4       # encoding: [0x46,0x00,0x20,0x9b]
-        class.d $f2, $f4         # CHECK: class.d $f2, $f4       # encoding: [0x46,0x20,0x20,0x9b]
-        j       1f               # CHECK: j $tmp0                # encoding: [0b000010AA,A,A,A]
-                                 # CHECK:                        #   fixup A - offset: 0, value: ($tmp0), kind: fixup_Mips_26
-        j       a                # CHECK: j a                    # encoding: [0b000010AA,A,A,A]
-                                 # CHECK:                        #   fixup A - offset: 0, value: a, kind: fixup_Mips_26
-        j       1328             # CHECK: j 1328                 # encoding: [0x08,0x00,0x01,0x4c]
-        jr.hb   $4               # CHECK: jr.hb $4               # encoding: [0x00,0x80,0x04,0x09]
-        jalr.hb $4               # CHECK: jalr.hb $4             # encoding: [0x00,0x80,0xfc,0x09]
-        jalr.hb $4, $5           # CHECK: jalr.hb $4, $5         # encoding: [0x00,0xa0,0x24,0x09]
-        ldc2    $8, -701($at)    # CHECK: ldc2 $8, -701($1)      # encoding: [0x49,0xc8,0x0d,0x43]
-        lwc2    $18,-841($a2)    # CHECK: lwc2 $18, -841($6)     # encoding: [0x49,0x52,0x34,0xb7]
-        sdc2    $20,629($s2)     # CHECK: sdc2 $20, 629($18)     # encoding: [0x49,0xf4,0x92,0x75]
-        swc2    $25,304($s0)     # CHECK: swc2 $25, 304($16)     # encoding: [0x49,0x79,0x81,0x30]
-        ll      $v0,-153($s2)    # CHECK: ll $2, -153($18)       # encoding: [0x7e,0x42,0xb3,0xb6]
-        lld     $zero,112($ra)   # CHECK: lld $zero, 112($ra)    # encoding: [0x7f,0xe0,0x38,0x37]
+        rint.s $f2, $f4          # CHECK: rint.s $f2, $f4        # encoding: [0x46,0x00,0x20,0x9a]
         sc      $15,-40($s3)     # CHECK: sc $15, -40($19)       # encoding: [0x7e,0x6f,0xec,0x26]
         scd     $15,-51($sp)     # CHECK: scd $15, -51($sp)      # encoding: [0x7f,0xaf,0xe6,0xa7]
-        clo     $11,$a1          # CHECK: clo $11, $5            # encoding: [0x00,0xa0,0x58,0x51]
-        clz     $sp,$gp          # CHECK: clz $sp, $gp           # encoding: [0x03,0x80,0xe8,0x50]
-        dclo    $s2,$a2          # CHECK: dclo $18, $6           # encoding: [0x00,0xc0,0x90,0x53]
-        dclz    $s0,$25          # CHECK: dclz $16, $25          # encoding: [0x03,0x20,0x80,0x52]
-        ssnop                    # WARNING: [[@LINE]]:9: warning: ssnop is deprecated for MIPS64r6 and is equivalent to a nop instruction
-        ssnop                    # CHECK: ssnop                  # encoding: [0x00,0x00,0x00,0x40]
         sdbbp                    # CHECK: sdbbp                  # encoding: [0x00,0x00,0x00,0x0e]
         sdbbp     34             # CHECK: sdbbp 34               # encoding: [0x00,0x00,0x08,0x8e]
+        sdc2    $20,629($s2)     # CHECK: sdc2 $20, 629($18)     # encoding: [0x49,0xf4,0x92,0x75]
+        sel.d   $f0,$f1,$f2      # CHECK: sel.d $f0, $f1, $f2 # encoding: [0x46,0x22,0x08,0x10]
+        sel.s   $f0,$f1,$f2      # CHECK: sel.s $f0, $f1, $f2 # encoding: [0x46,0x02,0x08,0x10]
+        seleqz  $2,$3,$4         # CHECK: seleqz $2, $3, $4 # encoding: [0x00,0x64,0x10,0x35]
+        seleqz.d $f0, $f2, $f4   # CHECK: seleqz.d $f0, $f2, $f4 # encoding: [0x46,0x24,0x10,0x14]
+        seleqz.s $f0, $f2, $f4   # CHECK: seleqz.s $f0, $f2, $f4 # encoding: [0x46,0x04,0x10,0x14]
+        selnez  $2,$3,$4         # CHECK: selnez $2, $3, $4 # encoding: [0x00,0x64,0x10,0x37]
+        selnez.d $f0, $f2, $f4   # CHECK: selnez.d $f0, $f2, $f4 # encoding: [0x46,0x24,0x10,0x17]
+        selnez.s $f0, $f2, $f4   # CHECK: selnez.s $f0, $f2, $f4 # encoding: [0x46,0x04,0x10,0x17]
+        ssnop                    # CHECK: ssnop                  # encoding: [0x00,0x00,0x00,0x40]
+        ssnop                    # WARNING: [[@LINE]]:9: warning: ssnop is deprecated for MIPS64r6 and is equivalent to a nop instruction
+        swc2    $25,304($s0)     # CHECK: swc2 $25, 304($16)     # encoding: [0x49,0x79,0x81,0x30]
         sync                     # CHECK: sync                   # encoding: [0x00,0x00,0x00,0x0f]
         sync    1                # CHECK: sync 1                 # encoding: [0x00,0x00,0x00,0x4f]
         teq     $0,$3            # CHECK: teq $zero, $3          # encoding: [0x00,0x03,0x00,0x34]
         teq     $5,$7,620        # CHECK: teq $5, $7, 620        # encoding: [0x00,0xa7,0x9b,0x34]
-        tge     $7,$10           # CHECK: tge $7, $10            # encoding: [0x00,0xea,0x00,0x30]
         tge     $5,$19,340       # CHECK: tge $5, $19, 340       # encoding: [0x00,0xb3,0x55,0x30]
-        tgeu    $22,$28          # CHECK: tgeu $22, $gp          # encoding: [0x02,0xdc,0x00,0x31]
+        tge     $7,$10           # CHECK: tge $7, $10            # encoding: [0x00,0xea,0x00,0x30]
         tgeu    $20,$14,379      # CHECK: tgeu $20, $14, 379     # encoding: [0x02,0x8e,0x5e,0xf1]
+        tgeu    $22,$28          # CHECK: tgeu $22, $gp          # encoding: [0x02,0xdc,0x00,0x31]
         tlt     $15,$13          # CHECK: tlt $15, $13           # encoding: [0x01,0xed,0x00,0x32]
         tlt     $2,$19,133       # CHECK: tlt $2, $19, 133       # encoding: [0x00,0x53,0x21,0x72]
         tltu    $11,$16          # CHECK: tltu $11, $16          # encoding: [0x01,0x70,0x00,0x33]
diff --git a/test/MC/Mips/mips_abi_flags_xx.s b/test/MC/Mips/mips_abi_flags_xx.s
index cd6c9de4fac..349b70d609a 100644
--- a/test/MC/Mips/mips_abi_flags_xx.s
+++ b/test/MC/Mips/mips_abi_flags_xx.s
@@ -2,15 +2,15 @@
 # RUN:   FileCheck %s -check-prefix=CHECK-ASM
 #
 # RUN: llvm-mc %s -arch=mips -mcpu=mips32 -filetype=obj -o - | \
-# RUN:   llvm-readobj -sections -section-data -section-relocations - | \
+# RUN:   llvm-readobj -sections -section-data -section-relocations -mips-abi-flags - | \
 # RUN:     FileCheck %s -check-prefix=CHECK-OBJ -check-prefix=CHECK-OBJ-R1
 
 # RUN: llvm-mc /dev/null -arch=mips -mcpu=mips32 -mattr=fpxx -filetype=obj -o - | \
-# RUN:   llvm-readobj -sections -section-data -section-relocations - | \
+# RUN:   llvm-readobj -sections -section-data -section-relocations -mips-abi-flags - | \
 # RUN:     FileCheck %s -check-prefix=CHECK-OBJ -check-prefix=CHECK-OBJ-R1
 
 # RUN: llvm-mc /dev/null -arch=mips -mcpu=mips32r6 -mattr=fpxx -filetype=obj -o - | \
-# RUN:   llvm-readobj -sections -section-data -section-relocations - | \
+# RUN:   llvm-readobj -sections -section-data -section-relocations -mips-abi-flags - | \
 # RUN:     FileCheck %s -check-prefix=CHECK-OBJ -check-prefix=CHECK-OBJ-R6
 
 # CHECK-ASM: .module fp=xx
@@ -31,12 +31,23 @@
 # CHECK-OBJ:         EntrySize: 24
 # CHECK-OBJ:         Relocations [
 # CHECK-OBJ:         ]
-# CHECK-OBJ:         SectionData (
-# CHECK-OBJ-R1:        0000: 00002001 01010005 00000000 00000000  |.. .............|
-# CHECK-OBJ-R6:        0000: 00002006 01010005 00000000 00000000  |.. .............|
-# CHECK-OBJ:           0010: 00000001 00000000                    |........|
-# CHECK-OBJ:         )
 # CHECK-OBJ-LABEL: }
+# CHECK-OBJ:       MIPS ABI Flags {
+# CHECK-OBJ-NEXT:    Version: 0
+# CHECK-OBJ-R1-NEXT: ISA: {{MIPS32$}}
+# CHECK-OBJ-R6-NEXT: ISA: MIPS32r6
+# CHECK-OBJ-NEXT:    ISA Extension: None (0x0)
+# CHECK-OBJ-NEXT:    ASEs [ (0x0)
+# CHECK-OBJ-NEXT:    ]
+# CHECK-OBJ-NEXT:    FP ABI: Hard float (32-bit CPU, Any FPU) (0x5)
+# CHECK-OBJ-NEXT:    GPR size: 32
+# CHECK-OBJ-NEXT:    CPR1 size: 32
+# CHECK-OBJ-NEXT:    CPR2 size: 0
+# CHECK-OBJ-NEXT:    Flags 1 [ (0x1)
+# CHECK-OBJ-NEXT:      ODDSPREG (0x1)
+# CHECK-OBJ-NEXT:    ]
+# CHECK-OBJ-NEXT:    Flags 2: 0x0
+# CHECK-OBJ-NEXT:  }
 
         .module fp=xx
 
diff --git a/test/MC/Mips/mips_abi_flags_xx_set.s b/test/MC/Mips/mips_abi_flags_xx_set.s
index a548972db0d..b31e295f635 100644
--- a/test/MC/Mips/mips_abi_flags_xx_set.s
+++ b/test/MC/Mips/mips_abi_flags_xx_set.s
@@ -2,7 +2,7 @@
 # RUN:   FileCheck %s -check-prefix=CHECK-ASM
 #
 # RUN: llvm-mc %s -arch=mips -mcpu=mips32 -filetype=obj -o - | \
-# RUN:   llvm-readobj -sections -section-data -section-relocations - | \
+# RUN:   llvm-readobj -sections -section-data -section-relocations -mips-abi-flags - | \
 # RUN:     FileCheck %s -check-prefix=CHECK-OBJ
 
 # CHECK-ASM: .module fp=xx
@@ -24,11 +24,22 @@
 # CHECK-OBJ:         EntrySize: 24
 # CHECK-OBJ:         Relocations [
 # CHECK-OBJ:         ]
-# CHECK-OBJ:         SectionData (
-# CHECK-OBJ:           0000: 00002001 01010005 00000000 00000000  |.. .............|
-# CHECK-OBJ:           0010: 00000001 00000000                    |........|
-# CHECK-OBJ:         )
 # CHECK-OBJ-LABEL: }
+# CHECK-OBJ:       MIPS ABI Flags {
+# CHECK-OBJ-NEXT:    Version: 0
+# CHECK-OBJ-NEXT:    ISA: {{MIPS32$}}
+# CHECK-OBJ-NEXT:    ISA Extension: None (0x0)
+# CHECK-OBJ-NEXT:    ASEs [ (0x0)
+# CHECK-OBJ-NEXT:    ]
+# CHECK-OBJ-NEXT:    FP ABI: Hard float (32-bit CPU, Any FPU) (0x5)
+# CHECK-OBJ-NEXT:    GPR size: 32
+# CHECK-OBJ-NEXT:    CPR1 size: 32
+# CHECK-OBJ-NEXT:    CPR2 size: 0
+# CHECK-OBJ-NEXT:    Flags 1 [ (0x1)
+# CHECK-OBJ-NEXT:      ODDSPREG (0x1)
+# CHECK-OBJ-NEXT:    ]
+# CHECK-OBJ-NEXT:    Flags 2: 0x0
+# CHECK-OBJ-NEXT:  }
 
         .module fp=xx
         .set    fp=64
diff --git a/test/MC/Mips/mips_directives.s b/test/MC/Mips/mips_directives.s
index 1a7d61f3ad4..2ea37822f76 100644
--- a/test/MC/Mips/mips_directives.s
+++ b/test/MC/Mips/mips_directives.s
@@ -17,6 +17,7 @@ $BB0_2:
 # CHECK-NOT: nop
 # CHECK:   jal 1328             # encoding: [0x0c,0x00,0x01,0x4c]
 # CHECK-NOT: nop
+# CHECK: .set nomacro
 
     .set    noreorder
      b 1332
@@ -33,6 +34,7 @@ $JTI0_0:
 # CHECK:     .4byte    2013265916
     .set  at=$12
     .set macro
+# CHECK:   .set macro
 # CHECK:   .set reorder
 # CHECK:   b 1332               # encoding: [0x10,0x00,0x01,0x4d]
 # CHECK:   nop                  # encoding: [0x00,0x00,0x00,0x00]
diff --git a/test/MC/Mips/module-directive-bad.s b/test/MC/Mips/module-directive-bad.s
index 963d651afe5..74dc6d22793 100644
--- a/test/MC/Mips/module-directive-bad.s
+++ b/test/MC/Mips/module-directive-bad.s
@@ -203,7 +203,7 @@
 
     .set macro
     .module fp=64
-# FIXME: emitDirectiveSetMacro should call forbidModuleDirective().
+# CHECK: :[[@LINE-1]]:13: error: .module directive must appear before any code
 
     .llvm_internal_mips_reallow_module_directive
     .module fp=32
@@ -211,7 +211,7 @@
 
     .set nomacro
     .module fp=64
-# FIXME: emitDirectiveSetNoMacro should call forbidModuleDirective().
+# CHECK: :[[@LINE-1]]:13: error: .module directive must appear before any code
 
     .llvm_internal_mips_reallow_module_directive
     .module fp=32
diff --git a/test/MC/Mips/module-hardfloat.s b/test/MC/Mips/module-hardfloat.s
new file mode 100644
index 00000000000..51f72487b39
--- /dev/null
+++ b/test/MC/Mips/module-hardfloat.s
@@ -0,0 +1,26 @@
+# RUN: llvm-mc %s -arch=mips -mcpu=mips32 | \
+# RUN:   FileCheck %s -check-prefix=CHECK-ASM
+#
+# RUN: llvm-mc %s -arch=mips -mcpu=mips32 -filetype=obj -o - | \
+# RUN:   llvm-readobj -mips-abi-flags - | \
+# RUN:     FileCheck %s -check-prefix=CHECK-OBJ
+
+# CHECK-ASM: .module hardfloat
+
+# Check if the MIPS.abiflags section was correctly emitted:
+# CHECK-OBJ: MIPS ABI Flags {
+# CHECK-OBJ:   FP ABI: Hard float (32-bit CPU, Any FPU) (0x5)
+# CHECK-OBJ:   CPR1 size: 32
+# CHECK-OBJ:   Flags 1 [ (0x1)
+# CHECK-OBJ:     ODDSPREG (0x1)
+# CHECK-OBJ:   ]
+# CHECK-OBJ: }
+
+  .module fp=xx
+  .module oddspreg
+  .module softfloat
+  .module hardfloat
+
+# FIXME: Test should include gnu_attributes directive when implemented.
+#        An explicit .gnu_attribute must be checked against the effective
+#        command line options and any inconsistencies reported via a warning.
diff --git a/test/MC/Mips/module-softfloat.s b/test/MC/Mips/module-softfloat.s
new file mode 100644
index 00000000000..18559c5aec1
--- /dev/null
+++ b/test/MC/Mips/module-softfloat.s
@@ -0,0 +1,20 @@
+# RUN: llvm-mc %s -arch=mips -mcpu=mips32 | \
+# RUN:   FileCheck %s -check-prefix=CHECK-ASM
+#
+# RUN: llvm-mc %s -arch=mips -mcpu=mips32 -filetype=obj -o - | \
+# RUN:   llvm-readobj -mips-abi-flags - | \
+# RUN:     FileCheck %s -check-prefix=CHECK-OBJ
+
+# CHECK-ASM: .module softfloat
+
+# Check if the MIPS.abiflags section was correctly emitted:
+# CHECK-OBJ: MIPS ABI Flags {
+# CHECK-OBJ:   FP ABI: Soft float (0x3)
+# CHECK-OBJ:   CPR1 size: 0
+# CHECK-OBJ: }
+
+  .module softfloat
+
+# FIXME: Test should include gnu_attributes directive when implemented.
+#        An explicit .gnu_attribute must be checked against the effective
+#        command line options and any inconsistencies reported via a warning.
diff --git a/test/MC/Mips/octeon-instructions.s b/test/MC/Mips/octeon-instructions.s
index 34830c0fc6e..cbb99088ea1 100644
--- a/test/MC/Mips/octeon-instructions.s
+++ b/test/MC/Mips/octeon-instructions.s
@@ -15,6 +15,8 @@
 # CHECK: cins32 $22, $22, 9, 22       # encoding: [0x72,0xd6,0xb2,0x73]
 # CHECK: cins32 $24, $ra, 0, 31       # encoding: [0x73,0xf8,0xf8,0x33]
 # CHECK: cins32 $15, $15, 5, 5        # encoding: [0x71,0xef,0x29,0x73]
+# CHECK: dmtc2 $2, 16455              # encoding: [0x48,0xa2,0x40,0x47]
+# CHECK: dmfc2 $2, 64                 # encoding: [0x48,0x22,0x00,0x40]
 # CHECK: dmul  $9, $6, $7             # encoding: [0x70,0xc7,0x48,0x03]
 # CHECK: dmul  $19, $24, $25          # encoding: [0x73,0x19,0x98,0x03]
 # CHECK: dmul  $9, $9, $6             # encoding: [0x71,0x26,0x48,0x03]
@@ -72,6 +74,8 @@ foo:
   cins32 $22, 9, 22
   cins  $24, $31, 32, 31
   cins  $15, 37, 5
+  dmtc2 $2, 0x4047
+  dmfc2 $2, 0x0040
   dmul  $9, $6, $7
   dmul  $19, $24, $25
   dmul  $9, $6
diff --git a/test/MC/Mips/relocation.s b/test/MC/Mips/relocation.s
new file mode 100644
index 00000000000..f8030d16687
--- /dev/null
+++ b/test/MC/Mips/relocation.s
@@ -0,0 +1,209 @@
+// RUN: llvm-mc -triple mips-unknown-linux < %s -show-encoding \
+// RUN:     | FileCheck -check-prefix=ENCBE -check-prefix=FIXUP %s
+// RUN: llvm-mc -triple mipsel-unknown-linux < %s -show-encoding \
+// RUN:     | FileCheck -check-prefix=ENCLE -check-prefix=FIXUP %s
+// RUN: llvm-mc -filetype=obj -triple mipsel-unknown-linux < %s \
+// RUN:     | llvm-readobj -r | FileCheck -check-prefix=RELOC %s
+
+// Test that we produce the correct relocation.
+// FIXME: move more relocation only tests here.
+
+// Check prefixes:
+// RELOC - Check the relocation in the object.
+// FIXUP - Check the fixup on the instruction.
+// ENCBE - Check the big-endian encoding on the instruction.
+// ENCLE - Check the little-endian encoding on the instruction.
+// ????? - Placeholder. Relocation is defined but the way of generating it is
+//         unknown.
+// FIXME - Placeholder. Generation method is known but doesn't work.
+
+        .short foo                         // RELOC: R_MIPS_16 foo
+
+        .long foo                          // RELOC: R_MIPS_32 foo
+
+                                           // ?????: R_MIPS_REL32 foo
+
+        jal foo                            // RELOC: R_MIPS_26 foo
+                                           // ENCBE: jal foo # encoding: [0b000011AA,A,A,A]
+                                           // ENCLE: jal foo # encoding: [A,A,A,0b000011AA]
+                                           // FIXUP: # fixup A - offset: 0, value: foo, kind: fixup_Mips_26
+
+        addiu $2, $3, %hi(foo)             // RELOC: R_MIPS_HI16 foo
+                                           // ENCBE: addiu $2, $3, %hi(foo) # encoding: [0x24,0x62,A,A]
+                                           // ENCLE: addiu $2, $3, %hi(foo) # encoding: [A,A,0x62,0x24]
+                                           // FIXUP: # fixup A - offset: 0, value: foo@ABS_HI, kind: fixup_Mips_HI16
+
+        addiu $2, $3, %lo(foo)             // RELOC: R_MIPS_LO16 foo
+                                           // ENCBE: addiu $2, $3, %lo(foo) # encoding: [0x24,0x62,A,A]
+                                           // ENCLE: addiu $2, $3, %lo(foo) # encoding: [A,A,0x62,0x24]
+                                           // FIXUP: # fixup A - offset: 0, value: foo@ABS_LO, kind: fixup_Mips_LO16
+
+        addiu $2, $3, %gp_rel(foo)         // RELOC: R_MIPS_GPREL16 foo
+                                           // ENCBE: addiu $2, $3, %gp_rel(foo) # encoding: [0x24,0x62,A,A]
+                                           // ENCLE: addiu $2, $3, %gp_rel(foo) # encoding: [A,A,0x62,0x24]
+                                           // FIXUP: # fixup A - offset: 0, value: foo@GPREL, kind: fixup_Mips_GPREL
+
+                                           // ?????: R_MIPS_LITERAL foo
+
+        addiu $2, $3, %got(foo)            // RELOC: R_MIPS_GOT16 foo
+                                           // ENCBE: addiu $2, $3, %got(foo) # encoding: [0x24,0x62,A,A]
+                                           // ENCLE: addiu $2, $3, %got(foo) # encoding: [A,A,0x62,0x24]
+                                           // FIXUP: # fixup A - offset: 0, value: foo@GOT, kind: fixup_Mips_GOT_Local
+
+        .short foo-.                       // RELOC: R_MIPS_PC16 foo
+
+        addiu $2, $3, %call16(foo)         // RELOC: R_MIPS_CALL16 foo
+                                           // ENCBE: addiu $2, $3, %call16(foo) # encoding: [0x24,0x62,A,A]
+                                           // ENCLE: addiu $2, $3, %call16(foo) # encoding: [A,A,0x62,0x24]
+                                           // FIXUP: # fixup A - offset: 0, value: foo@GOT_CALL, kind: fixup_Mips_CALL16
+
+        .quad foo                          // RELOC: R_MIPS_64 foo
+
+                                           // ?????: R_MIPS_GPREL32 foo
+                                           // ?????: R_MIPS_UNUSED1 foo
+                                           // ?????: R_MIPS_UNUSED2 foo
+                                           // ?????: R_MIPS_UNUSED3 foo
+                                           // ?????: R_MIPS_SHIFT5 foo
+                                           // ?????: R_MIPS_SHIFT6 foo
+
+        addiu $2, $3, %got_disp(foo)       // RELOC: R_MIPS_GOT_DISP foo
+                                           // ENCBE: addiu $2, $3, %got_disp(foo) # encoding: [0x24,0x62,A,A]
+                                           // ENCLE: addiu $2, $3, %got_disp(foo) # encoding: [A,A,0x62,0x24]
+                                           // FIXUP: # fixup A - offset: 0, value: foo@GOT_DISP, kind: fixup_Mips_GOT_DISP
+
+        addiu $2, $3, %got_page(foo)       // RELOC: R_MIPS_GOT_PAGE foo
+                                           // ENCBE: addiu $2, $3, %got_page(foo) # encoding: [0x24,0x62,A,A]
+                                           // ENCLE: addiu $2, $3, %got_page(foo) # encoding: [A,A,0x62,0x24]
+                                           // FIXUP: # fixup A - offset: 0, value: foo@GOT_PAGE, kind: fixup_Mips_GOT_PAGE
+
+        addiu $2, $3, %got_ofst(foo)       // RELOC: R_MIPS_GOT_OFST foo
+                                           // ENCBE: addiu $2, $3, %got_ofst(foo) # encoding: [0x24,0x62,A,A]
+                                           // ENCLE: addiu $2, $3, %got_ofst(foo) # encoding: [A,A,0x62,0x24]
+                                           // FIXUP: # fixup A - offset: 0, value: foo@GOT_OFST, kind: fixup_Mips_GOT_OFST
+
+        addiu $2, $3, %got_hi(foo)         // RELOC: R_MIPS_GOT_HI16 foo
+                                           // ENCBE: addiu $2, $3, %got_hi(foo) # encoding: [0x24,0x62,A,A]
+                                           // ENCLE: addiu $2, $3, %got_hi(foo) # encoding: [A,A,0x62,0x24]
+                                           // FIXUP: # fixup A - offset: 0, value: foo@GOT_HI16, kind: fixup_Mips_GOT_HI16
+
+        addiu $2, $3, %got_lo(foo)         // RELOC: R_MIPS_GOT_LO16 foo
+                                           // ENCBE: addiu $2, $3, %got_lo(foo) # encoding: [0x24,0x62,A,A]
+                                           // ENCLE: addiu $2, $3, %got_lo(foo) # encoding: [A,A,0x62,0x24]
+                                           // FIXUP: # fixup A - offset: 0, value: foo@GOT_LO16, kind: fixup_Mips_GOT_LO16
+
+//      addiu $2, $3, %neg(foo)            // FIXME: R_MIPS_SUB foo
+                                           // ?????: R_MIPS_INSERT_A
+                                           // ?????: R_MIPS_INSERT_B
+                                           // ?????: R_MIPS_DELETE
+
+        .set mips64
+        daddiu $2, $3, %higher(foo)        // RELOC: R_MIPS_HIGHER foo
+                                           // ENCBE: daddiu $2, $3, %higher(foo) # encoding: [0x64,0x62,A,A]
+                                           // ENCLE: daddiu $2, $3, %higher(foo) # encoding: [A,A,0x62,0x64]
+                                           // FIXUP: # fixup A - offset: 0, value: foo@HIGHER, kind: fixup_Mips_HIGHER
+
+        daddiu $2, $3, %highest(foo)       // RELOC: R_MIPS_HIGHEST foo
+                                           // ENCBE: daddiu $2, $3, %highest(foo) # encoding: [0x64,0x62,A,A]
+                                           // ENCLE: daddiu $2, $3, %highest(foo) # encoding: [A,A,0x62,0x64]
+                                           // FIXUP: # fixup A - offset: 0, value: foo@HIGHEST, kind: fixup_Mips_HIGHEST
+
+        .set mips0
+        addiu $2, $3, %call_hi(foo)        // RELOC: R_MIPS_CALL_HI16 foo
+                                           // ENCBE: addiu $2, $3, %call_hi(foo) # encoding: [0x24,0x62,A,A]
+                                           // ENCLE: addiu $2, $3, %call_hi(foo) # encoding: [A,A,0x62,0x24]
+                                           // FIXUP: # fixup A - offset: 0, value: foo@CALL_HI16, kind: fixup_Mips_CALL_HI16
+
+        addiu $2, $3, %call_lo(foo)        // RELOC: R_MIPS_CALL_LO16 foo
+                                           // ENCBE: addiu $2, $3, %call_lo(foo) # encoding: [0x24,0x62,A,A]
+                                           // ENCLE: addiu $2, $3, %call_lo(foo) # encoding: [A,A,0x62,0x24]
+                                           // FIXUP: # fixup A - offset: 0, value: foo@CALL_LO16, kind: fixup_Mips_CALL_LO16
+
+                                           // ?????: R_MIPS_SCN_DISP foo
+                                           // ?????: R_MIPS_REL16 foo
+                                           // ?????: R_MIPS_ADD_IMMEDIATE foo
+                                           // ?????: R_MIPS_PJUMP foo
+                                           // ?????: R_MIPS_RELGOT foo
+//      jalr $25                           // ?????: R_MIPS_JALR foo
+
+                                           // ?????: R_MIPS_TLS_DTPMOD32 foo
+//      .dtprelword foo                    // FIXME: R_MIPS_TLS_DTPREL32 foo
+                                           // ?????: R_MIPS_TLS_DTPMOD64 foo
+//      .dtpreldword foo                   // FIXME: R_MIPS_TLS_DTPREL64 foo
+        addiu $2, $3, %tlsgd(foo)          // RELOC: R_MIPS_TLS_GD foo
+                                           // ENCBE: addiu $2, $3, %tlsgd(foo) # encoding: [0x24,0x62,A,A]
+                                           // ENCLE: addiu $2, $3, %tlsgd(foo) # encoding: [A,A,0x62,0x24]
+                                           // FIXUP: # fixup A - offset: 0, value: foo@TLSGD, kind: fixup_Mips_TLSGD
+
+        addiu $2, $3, %tlsldm(foo)         // RELOC: R_MIPS_TLS_LDM foo
+                                           // ENCBE: addiu $2, $3, %tlsldm(foo) # encoding: [0x24,0x62,A,A]
+                                           // ENCLE: addiu $2, $3, %tlsldm(foo) # encoding: [A,A,0x62,0x24]
+                                           // FIXUP: # fixup A - offset: 0, value: foo@TLSLDM, kind: fixup_Mips_TLSLDM
+
+        addiu $2, $3, %dtprel_hi(foo)      // RELOC: R_MIPS_TLS_DTPREL_HI16 foo
+                                           // ENCBE: addiu $2, $3, %dtprel_hi(foo) # encoding: [0x24,0x62,A,A]
+                                           // ENCLE: addiu $2, $3, %dtprel_hi(foo) # encoding: [A,A,0x62,0x24]
+                                           // FIXUP: # fixup A - offset: 0, value: foo@DTPREL_HI, kind: fixup_Mips_DTPREL_HI
+
+        addiu $2, $3, %dtprel_lo(foo)      // RELOC: R_MIPS_TLS_DTPREL_LO16 foo
+                                           // ENCBE: addiu $2, $3, %dtprel_lo(foo) # encoding: [0x24,0x62,A,A]
+                                           // ENCLE: addiu $2, $3, %dtprel_lo(foo) # encoding: [A,A,0x62,0x24]
+                                           // FIXUP: # fixup A - offset: 0, value: foo@DTPREL_LO, kind: fixup_Mips_DTPREL_LO
+
+        addiu $2, $3, %gottprel(foo)       // RELOC: R_MIPS_TLS_GOTTPREL foo
+                                           // ENCBE: addiu $2, $3, %gottprel(foo) # encoding: [0x24,0x62,A,A]
+                                           // ENCLE: addiu $2, $3, %gottprel(foo) # encoding: [A,A,0x62,0x24]
+                                           // FIXUP: # fixup A - offset: 0, value: foo@GOTTPREL, kind: fixup_Mips_GOTTPREL
+
+//      .tprelword foo                     // FIXME: R_MIPS_TLS_TPREL32 foo
+//      .tpreldword foo                    // FIXME: R_MIPS_TLS_TPREL64 foo
+        addiu $2, $3, %tprel_hi(foo)       // RELOC: R_MIPS_TLS_TPREL_HI16 foo
+                                           // ENCBE: addiu $2, $3, %tprel_hi(foo) # encoding: [0x24,0x62,A,A]
+                                           // ENCLE: addiu $2, $3, %tprel_hi(foo) # encoding: [A,A,0x62,0x24]
+                                           // FIXUP: # fixup A - offset: 0, value: foo@TPREL_HI, kind: fixup_Mips_TPREL_HI
+
+        addiu $2, $3, %tprel_lo(foo)       // RELOC: R_MIPS_TLS_TPREL_LO16 foo
+                                           // ENCBE: addiu $2, $3, %tprel_lo(foo) # encoding: [0x24,0x62,A,A]
+                                           // ENCLE: addiu $2, $3, %tprel_lo(foo) # encoding: [A,A,0x62,0x24]
+                                           // FIXUP: # fixup A - offset: 0, value: foo@TPREL_LO, kind: fixup_Mips_TPREL_LO
+
+                                           // ?????: R_MIPS_GLOB_DAT foo
+        .set mips32r6
+        beqzc $2, foo                      // RELOC: R_MIPS_PC21_S2 foo
+                                           // ENCBE: beqzc $2, foo # encoding: [0xd8,0b010AAAAA,A,A]
+                                           // ENCLE: beqzc $2, foo # encoding: [A,A,0b010AAAAA,0xd8]
+                                           // FIXUP: # fixup A - offset: 0, value: foo-4, kind: fixup_MIPS_PC21_S2
+
+        bc foo                             // RELOC: R_MIPS_PC26_S2 foo
+                                           // ENCBE: bc foo # encoding: [0b110010AA,A,A,A]
+                                           // ENCLE: bc foo # encoding: [A,A,A,0b110010AA]
+                                           // FIXUP: # fixup A - offset: 0, value: foo-4, kind: fixup_MIPS_PC26_S2
+
+        .set mips64r6
+        ldpc $2, foo                       // RELOC: R_MIPS_PC18_S3 foo
+                                           // ENCBE: ldpc $2, foo # encoding: [0xec,0b010110AA,A,A]
+                                           // ENCLE: ldpc $2, foo # encoding: [A,A,0b010110AA,0xec]
+                                           // FIXUP: # fixup A - offset: 0, value: foo, kind: fixup_Mips_PC18_S3
+
+        .set mips32r6
+        lwpc $2, foo                       // RELOC: R_MIPS_PC19_S2 foo
+                                           // ENCBE: lwpc $2, foo # encoding: [0xec,0b01001AAA,A,A]
+                                           // ENCLE: lwpc $2, foo # encoding: [A,A,0b01001AAA,0xec]
+                                           // FIXUP: # fixup A - offset: 0, value: foo, kind: fixup_MIPS_PC19_S2
+
+        addiu $2, $3, %pcrel_hi(foo)       // RELOC: R_MIPS_PCHI16 foo
+                                           // ENCBE: addiu $2, $3, %pcrel_hi(foo) # encoding: [0x24,0x62,A,A]
+                                           // ENCLE: addiu $2, $3, %pcrel_hi(foo) # encoding: [A,A,0x62,0x24]
+                                           // FIXUP: # fixup A - offset: 0, value: foo@PCREL_HI16, kind: fixup_MIPS_PCHI16
+
+        addiu $2, $3, %pcrel_lo(foo)       // RELOC: R_MIPS_PCLO16 foo
+                                           // ENCBE: addiu $2, $3, %pcrel_lo(foo) # encoding: [0x24,0x62,A,A]
+                                           // ENCLE: addiu $2, $3, %pcrel_lo(foo) # encoding: [A,A,0x62,0x24]
+                                           // FIXUP: # fixup A - offset: 0, value: foo@PCREL_LO16, kind: fixup_MIPS_PCLO16
+
+        .set mips0
+                                           // FIXME: R_MIPS16_*
+                                           // ?????: R_MIPS_COPY foo
+                                           // ?????: R_MIPS_JUMP_SLOT foo
+                                           // FIXME: R_MICROMIPS_*
+        .long foo-.                        // RELOC: R_MIPS_PC32 foo
+//      .ehword foo                        // FIXME: R_MIPS_EH foo
diff --git a/test/MC/Mips/set-at-directive-explicit-at.s b/test/MC/Mips/set-at-directive-explicit-at.s
index 28a7091c8d2..d2d1c1d4627 100644
--- a/test/MC/Mips/set-at-directive-explicit-at.s
+++ b/test/MC/Mips/set-at-directive-explicit-at.s
@@ -1,32 +1,34 @@
 # RUN: llvm-mc %s -triple=mipsel-unknown-linux -show-encoding -mcpu=mips32r2 \
 # RUN:     2>%t1 | FileCheck %s
 # RUN: FileCheck -check-prefix=WARNINGS %s < %t1
-# Check that the assembler can handle the documented syntax
-# for ".set at" and set the correct value. The correct value for $at is always
-# $1 when written by the user.
+
+# Check that the assembler can handle the documented syntax for ".set at" and
+# will set the correct value for $at.
+# Note that writing $at is always the same as writing $1.
+
     .text
 foo:
 # CHECK:   jr    $1                      # encoding: [0x08,0x00,0x20,0x00]
-# WARNINGS: :[[@LINE+2]]:11: warning: used $at without ".set noat"
+# WARNINGS: :[[@LINE+2]]:11: warning: used $at (currently $1) without ".set noat"
     .set    at=$1
     jr    $at
 
 # CHECK:   jr    $1                      # encoding: [0x08,0x00,0x20,0x00]
-# WARNINGS: :[[@LINE+2]]:11: warning: used $at without ".set noat"
+# WARNINGS: :[[@LINE+2]]:11: warning: used $at (currently $1) without ".set noat"
     .set    at=$1
     jr    $1
 
 # CHECK:   jr    $1                      # encoding: [0x08,0x00,0x20,0x00]
-# WARNINGS: :[[@LINE+2]]:11: warning: used $at without ".set noat"
+# WARNINGS: :[[@LINE+2]]:11: warning: used $at (currently $1) without ".set noat"
     .set    at=$at
     jr    $at
 
 # CHECK:   jr    $1                      # encoding: [0x08,0x00,0x20,0x00]
-# WARNINGS: :[[@LINE+2]]:11: warning: used $at without ".set noat"
+# WARNINGS: :[[@LINE+2]]:11: warning: used $at (currently $1) without ".set noat"
     .set    at=$at
     jr    $1
-# WARNINGS-NOT: warning: used $at without ".set noat"
 
+# WARNINGS-NOT: warning: used $at (currently ${{[0-9]+}}) without ".set noat"
 # CHECK:   jr    $1                      # encoding: [0x08,0x00,0x20,0x00]
     .set    at=$2
     jr    $at
@@ -41,12 +43,11 @@ foo:
     jr    $at
 
 # CHECK:   jr    $16                     # encoding: [0x08,0x00,0x00,0x02]
-# WARNINGS: :[[@LINE+2]]:11: warning: used $16 with ".set at=$16"
+# WARNINGS: :[[@LINE+2]]:11: warning: used $at (currently $16) without ".set noat"
     .set    at=$16
     jr    $s0
 
 # CHECK:   jr    $16                     # encoding: [0x08,0x00,0x00,0x02]
-# WARNINGS: :[[@LINE+2]]:11: warning: used $16 with ".set at=$16"
+# WARNINGS: :[[@LINE+2]]:11: warning: used $at (currently $16) without ".set noat"
     .set    at=$16
     jr    $16
-# WARNINGS-NOT: warning
diff --git a/test/MC/Mips/set-nomacro-micromips.s b/test/MC/Mips/set-nomacro-micromips.s
new file mode 100644
index 00000000000..c4d8b7c2d3e
--- /dev/null
+++ b/test/MC/Mips/set-nomacro-micromips.s
@@ -0,0 +1,33 @@
+# RUN: llvm-mc %s -arch=mips -mcpu=mips32 -mattr=micromips 2>&1 | FileCheck %s
+
+  .text
+  .type main, @function
+  .set micromips
+main:
+# CHECK-NOT: warning: macro instruction expanded into multiple instructions
+  .set macro
+  b 132
+  b 1332
+  b bar
+
+  lwm $16, $17, 8($sp)
+  swm $16, $17, 8($sp)
+
+  add $4, $5, $6
+
+  .set noreorder
+  .set nomacro
+  b 132
+# CHECK-NOT: [[@LINE-1]]:3: warning: macro instruction expanded into multiple instructions
+  b 1332
+# CHECK-NOT: [[@LINE-1]]:3: warning: macro instruction expanded into multiple instructions
+  b bar
+# CHECK-NOT: [[@LINE-1]]:3: warning: macro instruction expanded into multiple instructions
+
+  lwm $16, $17, 8($sp)
+# CHECK-NOT: [[@LINE-1]]:3: warning: macro instruction expanded into multiple instructions
+  swm $16, $17, 8($sp)
+# CHECK-NOT: [[@LINE-1]]:3: warning: macro instruction expanded into multiple instructions
+
+  add $4, $5, $6
+# CHECK-NOT: [[@LINE-1]]:3: warning: macro instruction expanded into multiple instructions
diff --git a/test/MC/Mips/set-nomacro.s b/test/MC/Mips/set-nomacro.s
new file mode 100644
index 00000000000..3f82f816131
--- /dev/null
+++ b/test/MC/Mips/set-nomacro.s
@@ -0,0 +1,191 @@
+# RUN: llvm-mc %s -arch=mips -mcpu=mips32 2>&1 | FileCheck %s
+
+# CHECK-NOT: warning: macro instruction expanded into multiple instructions
+  .set macro
+  li  $8, -16
+  li  $8, 16
+  li  $8, 161616
+
+  la  $8, 16
+  la  $8, 161616
+  la  $8, 16($9)
+  la  $8, 161616($9)
+  la  $8, symbol
+
+  jal $25
+  jal $4, $25
+
+  bne $2, 0, 1332
+  bne $2, 1, 1332
+  beq $2, 0, 1332
+  beq $2, 1, 1332
+
+  blt $7, $8, local_label
+  blt $7, $0, local_label
+  blt $0, $8, local_label
+  blt $0, $0, local_label
+
+  bltu $7, $8, local_label
+  bltu $7, $0, local_label
+  bltu $0, $8, local_label
+  bltu $0, $0, local_label
+
+  ble $7, $8, local_label
+  ble $7, $0, local_label
+  ble $0, $8, local_label
+  ble $0, $0, local_label
+
+  bleu $7, $8, local_label
+  bleu $7, $0, local_label
+  bleu $0, $8, local_label
+  bleu $0, $0, local_label
+
+  bge $7, $8, local_label
+  bge $7, $0, local_label
+  bge $0, $8, local_label
+  bge $0, $0, local_label
+
+  bgeu $7, $8, local_label
+  bgeu $7, $0, local_label
+  bgeu $0, $8, local_label
+  bgeu $0, $0, local_label
+
+  bgt $7, $8, local_label
+  bgt $7, $0, local_label
+  bgt $0, $8, local_label
+  bgt $0, $0, local_label
+
+  bgtu $7, $8, local_label
+  bgtu $7, $0, local_label
+  bgtu $0, $8, local_label
+  bgtu $0, $0, local_label
+
+  ulhu $5, 0
+
+  ulw $8, 2
+  ulw $8, 0x8000
+  ulw $8, 2($9)
+  ulw $8, 0x8000($9)
+
+  add $4, $5, $6
+
+  .set noreorder
+  .set nomacro
+  li  $8, -16
+# CHECK-NOT: [[@LINE-1]]:3: warning: macro instruction expanded into multiple instructions
+  li  $8, 16
+# CHECK-NOT: [[@LINE-1]]:3: warning: macro instruction expanded into multiple instructions
+  li  $8, 161616
+# CHECK: [[@LINE-1]]:3: warning: macro instruction expanded into multiple instructions
+
+  la  $8, 16
+# CHECK-NOT: [[@LINE-1]]:3: warning: macro instruction expanded into multiple instructions
+  la  $8, 161616
+# CHECK: [[@LINE-1]]:3: warning: macro instruction expanded into multiple instructions
+  la  $8, 16($9)
+# CHECK-NOT: [[@LINE-1]]:3: warning: macro instruction expanded into multiple instructions
+  la  $8, 161616($9)
+# CHECK: [[@LINE-1]]:3: warning: macro instruction expanded into multiple instructions
+  la  $8, symbol
+# CHECK: [[@LINE-1]]:3: warning: macro instruction expanded into multiple instructions
+
+  jal $25
+# CHECK-NOT: [[@LINE-1]]:3: warning: macro instruction expanded into multiple instructions
+  jal $4, $25
+# CHECK-NOT: [[@LINE-1]]:3: warning: macro instruction expanded into multiple instructions
+
+  bne $2, 0, 1332
+# CHECK-NOT: [[@LINE-1]]:3: warning: macro instruction expanded into multiple instructions
+  bne $2, 1, 1332
+# CHECK: [[@LINE-1]]:3: warning: macro instruction expanded into multiple instructions
+  beq $2, 0, 1332
+# CHECK-NOT: [[@LINE-1]]:3: warning: macro instruction expanded into multiple instructions
+  beq $2, 1, 1332
+# CHECK: [[@LINE-1]]:3: warning: macro instruction expanded into multiple instructions
+
+  blt $7, $8, local_label
+# CHECK: [[@LINE-1]]:3: warning: macro instruction expanded into multiple instructions
+  blt $7, $0, local_label
+# CHECK-NOT: [[@LINE-1]]:3: warning: macro instruction expanded into multiple instructions
+  blt $0, $8, local_label
+# CHECK-NOT: [[@LINE-1]]:3: warning: macro instruction expanded into multiple instructions
+  blt $0, $0, local_label
+# CHECK-NOT: [[@LINE-1]]:3: warning: macro instruction expanded into multiple instructions
+
+  bltu $7, $8, local_label
+# CHECK: [[@LINE-1]]:3: warning: macro instruction expanded into multiple instructions
+  bltu $7, $0, local_label
+# CHECK-NOT: [[@LINE-1]]:3: warning: macro instruction expanded into multiple instructions
+  bltu $0, $8, local_label
+# CHECK-NOT: [[@LINE-1]]:3: warning: macro instruction expanded into multiple instructions
+  bltu $0, $0, local_label
+# CHECK-NOT: [[@LINE-1]]:3: warning: macro instruction expanded into multiple instructions
+
+  ble $7, $8, local_label
+# CHECK: [[@LINE-1]]:3: warning: macro instruction expanded into multiple instructions
+  ble $7, $0, local_label
+# CHECK-NOT: [[@LINE-1]]:3: warning: macro instruction expanded into multiple instructions
+  ble $0, $8, local_label
+# CHECK-NOT: [[@LINE-1]]:3: warning: macro instruction expanded into multiple instructions
+  ble $0, $0, local_label
+# CHECK-NOT: [[@LINE-1]]:3: warning: macro instruction expanded into multiple instructions
+
+  bleu $7, $8, local_label
+# CHECK: [[@LINE-1]]:3: warning: macro instruction expanded into multiple instructions
+  bleu $7, $0, local_label
+# CHECK-NOT: [[@LINE-1]]:3: warning: macro instruction expanded into multiple instructions
+  bleu $0, $8, local_label
+# CHECK-NOT: [[@LINE-1]]:3: warning: macro instruction expanded into multiple instructions
+  bleu $0, $0, local_label
+# CHECK-NOT: [[@LINE-1]]:3: warning: macro instruction expanded into multiple instructions
+
+  bge $7, $8, local_label
+# CHECK: [[@LINE-1]]:3: warning: macro instruction expanded into multiple instructions
+  bge $7, $0, local_label
+# CHECK-NOT: [[@LINE-1]]:3: warning: macro instruction expanded into multiple instructions
+  bge $0, $8, local_label
+# CHECK-NOT: [[@LINE-1]]:3: warning: macro instruction expanded into multiple instructions
+  bge $0, $0, local_label
+# CHECK-NOT: [[@LINE-1]]:3: warning: macro instruction expanded into multiple instructions
+
+  bgeu $7, $8, local_label
+# CHECK: [[@LINE-1]]:3: warning: macro instruction expanded into multiple instructions
+  bgeu $7, $0, local_label
+# CHECK-NOT: [[@LINE-1]]:3: warning: macro instruction expanded into multiple instructions
+  bgeu $0, $8, local_label
+# CHECK-NOT: [[@LINE-1]]:3: warning: macro instruction expanded into multiple instructions
+  bgeu $0, $0, local_label
+# CHECK-NOT: [[@LINE-1]]:3: warning: macro instruction expanded into multiple instructions
+
+  bgt $7, $8, local_label
+# CHECK: [[@LINE-1]]:3: warning: macro instruction expanded into multiple instructions
+  bgt $7, $0, local_label
+# CHECK-NOT: [[@LINE-1]]:3: warning: macro instruction expanded into multiple instructions
+  bgt $0, $8, local_label
+# CHECK-NOT: [[@LINE-1]]:3: warning: macro instruction expanded into multiple instructions
+  bgt $0, $0, local_label
+# CHECK-NOT: [[@LINE-1]]:3: warning: macro instruction expanded into multiple instructions
+
+  bgtu $7, $8, local_label
+# CHECK: [[@LINE-1]]:3: warning: macro instruction expanded into multiple instructions
+  bgtu $7, $0, local_label
+# CHECK-NOT: [[@LINE-1]]:3: warning: macro instruction expanded into multiple instructions
+  bgtu $0, $8, local_label
+# CHECK-NOT: [[@LINE-1]]:3: warning: macro instruction expanded into multiple instructions
+  bgtu $0, $0, local_label
+# CHECK-NOT: [[@LINE-1]]:3: warning: macro instruction expanded into multiple instructions
+
+  ulhu $5, 0
+# CHECK: [[@LINE-1]]:3: warning: macro instruction expanded into multiple instructions
+
+  ulw $8, 2
+# CHECK-NOT: [[@LINE-1]]:3: warning: macro instruction expanded into multiple instructions
+  ulw $8, 0x8000
+# CHECK: [[@LINE-1]]:3: warning: macro instruction expanded into multiple instructions
+  ulw $8, 2($9)
+# CHECK-NOT: [[@LINE-1]]:3: warning: macro instruction expanded into multiple instructions
+  ulw $8, 0x8000($9)
+# CHECK: [[@LINE-1]]:3: warning: macro instruction expanded into multiple instructions
+
+  add $4, $5, $6
+# CHECK-NOT: [[@LINE-1]]:3: warning: macro instruction expanded into multiple instructions
diff --git a/test/MC/Mips/set-oddspreg-nooddspreg-error.s b/test/MC/Mips/set-oddspreg-nooddspreg-error.s
new file mode 100644
index 00000000000..5fb1308ceb9
--- /dev/null
+++ b/test/MC/Mips/set-oddspreg-nooddspreg-error.s
@@ -0,0 +1,10 @@
+# RUN: not llvm-mc %s -triple=mips-unknown-linux -mcpu=mips32 -mattr=+nooddspreg 2>%t1
+# RUN: FileCheck %s < %t1
+
+  .set oddspreg
+  sub.s $f1, $f2, $f2
+  # CHECK-NOT: :[[@LINE-1]]:{{[0-9]+}}: error: -mno-odd-spreg prohibits the use of odd FPU registers
+
+  .set nooddspreg
+  sub.s $f1, $f2, $f2
+  # CHECK: :[[@LINE-1]]:9: error: -mno-odd-spreg prohibits the use of odd FPU registers
diff --git a/test/MC/Mips/set-oddspreg-nooddspreg.s b/test/MC/Mips/set-oddspreg-nooddspreg.s
new file mode 100644
index 00000000000..a057c487f79
--- /dev/null
+++ b/test/MC/Mips/set-oddspreg-nooddspreg.s
@@ -0,0 +1,10 @@
+# RUN: llvm-mc %s -triple=mips-unknown-linux -mcpu=mips32r2 -mattr=+nooddspreg | \
+# RUN:   FileCheck %s
+
+  .set oddspreg
+  sub.s $f1, $f2, $f2
+  .set nooddspreg
+
+# CHECK: .set oddspreg
+# CHECK: sub.s $f1, $f2, $f2
+# CHECK: .set nooddspreg
diff --git a/test/MC/Mips/set-push-pop-directives-bad.s b/test/MC/Mips/set-push-pop-directives-bad.s
index 53d8b230815..8994eea1c8b 100644
--- a/test/MC/Mips/set-push-pop-directives-bad.s
+++ b/test/MC/Mips/set-push-pop-directives-bad.s
@@ -12,3 +12,12 @@
 # CHECK: :[[@LINE-1]]:19: error: unexpected token, expected end of statement
         .set pop bar
 # CHECK: :[[@LINE-1]]:18: error: unexpected token, expected end of statement
+
+        .set hardfloat
+        .set push
+        .set softfloat
+        add.s $f2, $f2, $f2
+# CHECK: :[[@LINE-1]]:9: error: instruction requires a CPU feature not currently enabled
+        .set pop
+        add.s $f2, $f2, $f2
+# CHECK-NOT: :[[@LINE-1]]:9: error: instruction requires a CPU feature not currently enabled
diff --git a/test/MC/Mips/set-push-pop-directives.s b/test/MC/Mips/set-push-pop-directives.s
index 5f55b7c7e4d..3a0b2aecc58 100644
--- a/test/MC/Mips/set-push-pop-directives.s
+++ b/test/MC/Mips/set-push-pop-directives.s
@@ -51,3 +51,20 @@
 # CHECK:  b        1336
 # CHECK:  nop
 # CHECK:  addvi.b  $w15, $w13, 18
+
+    .set push
+    .set dsp
+    lbux    $7, $10($11)
+    .set pop
+
+    .set push
+    .set dsp
+    lbux    $7, $10($11)
+# CHECK-NOT: :[[@LINE-1]]:5: error: instruction requires a CPU feature not currently enabled
+    .set pop
+
+    .set push
+    .set dsp
+    lbux    $7, $10($11)
+# CHECK-NOT: :[[@LINE-1]]:5: error: instruction requires a CPU feature not currently enabled
+    .set pop
diff --git a/test/MC/Mips/set-softfloat-hardfloat-bad.s b/test/MC/Mips/set-softfloat-hardfloat-bad.s
new file mode 100644
index 00000000000..14b1e78afe1
--- /dev/null
+++ b/test/MC/Mips/set-softfloat-hardfloat-bad.s
@@ -0,0 +1,14 @@
+# RUN: not llvm-mc %s -triple=mips-unknown-linux -mcpu=mips32 -mattr=+soft-float 2>%t1
+# RUN: FileCheck %s < %t1
+
+  .set hardfloat
+  add.s $f2, $f2, $f2
+  # CHECK-NOT: :[[@LINE-1]]:3: error: instruction requires a CPU feature not currently enabled
+  sub.s $f2, $f2, $f2
+  # CHECK-NOT: :[[@LINE-1]]:3: error: instruction requires a CPU feature not currently enabled
+
+  .set softfloat
+  add.s $f2, $f2, $f2
+  # CHECK: :[[@LINE-1]]:3: error: instruction requires a CPU feature not currently enabled
+  sub.s $f2, $f2, $f2
+  # CHECK: :[[@LINE-1]]:3: error: instruction requires a CPU feature not currently enabled
diff --git a/test/MC/Mips/set-softfloat-hardfloat.s b/test/MC/Mips/set-softfloat-hardfloat.s
new file mode 100644
index 00000000000..ffb10f3ec58
--- /dev/null
+++ b/test/MC/Mips/set-softfloat-hardfloat.s
@@ -0,0 +1,12 @@
+# RUN: llvm-mc %s -triple=mips-unknown-linux -mcpu=mips32r2 -mattr=+soft-float | \
+# RUN:   FileCheck %s
+
+  .set hardfloat
+  add.s $f2, $f2, $f2
+  sub.s $f2, $f2, $f2
+  .set softfloat
+
+# CHECK: .set hardfloat
+# CHECK: add.s $f2, $f2, $f2
+# CHECK: sub.s $f2, $f2, $f2
+# CHECK: .set softfloat
diff --git a/test/MC/Mips/target-soft-float.s b/test/MC/Mips/target-soft-float.s
new file mode 100644
index 00000000000..5865d5ab126
--- /dev/null
+++ b/test/MC/Mips/target-soft-float.s
@@ -0,0 +1,331 @@
+# RUN: not llvm-mc %s -triple=mips-unknown-linux -mcpu=mips32 -mattr=+soft-float 2>&1 |\
+# RUN:   FileCheck %s --check-prefix=32
+# RUN: not llvm-mc %s -triple=mips-unknown-linux -mcpu=mips64 -mattr=+soft-float 2>&1 |\
+# RUN:   FileCheck %s --check-prefix=64
+# RUN: not llvm-mc %s -triple=mips-unknown-linux -mcpu=mips32r2 -mattr=+soft-float 2>&1 |\
+# RUN:   FileCheck %s --check-prefix=R2
+# RUN: not llvm-mc %s -triple=mips-unknown-linux -mcpu=mips32r6 -mattr=+soft-float 2>&1 |\
+# RUN:   FileCheck %s --check-prefix=R6
+
+foo:
+  dmfc1      $7, $f2
+  # 64: :[[@LINE-1]]:3: error: instruction requires a CPU feature not currently enabled
+  dmtc1      $6, $f2
+  # 64: :[[@LINE-1]]:3: error: instruction requires a CPU feature not currently enabled
+
+  ceil.l.d   $f2, $f2
+  # R2: :[[@LINE-1]]:3: error: instruction requires a CPU feature not currently enabled
+  ceil.l.s   $f2, $f2
+  # R2: :[[@LINE-1]]:3: error: instruction requires a CPU feature not currently enabled
+  cvt.d.l    $f2, $f2
+  # R2: :[[@LINE-1]]:3: error: instruction requires a CPU feature not currently enabled
+  cvt.l.d    $f2, $f2
+  # R2: :[[@LINE-1]]:3: error: instruction requires a CPU feature not currently enabled
+  cvt.l.s    $f2, $f2
+  # R2: :[[@LINE-1]]:3: error: instruction requires a CPU feature not currently enabled
+  cvt.s.l    $f2, $f2
+  # R2: :[[@LINE-1]]:3: error: instruction requires a CPU feature not currently enabled
+  floor.l.d  $f2, $f2
+  # R2: :[[@LINE-1]]:3: error: instruction requires a CPU feature not currently enabled
+  floor.l.s  $f2, $f2
+  # R2: :[[@LINE-1]]:3: error: instruction requires a CPU feature not currently enabled
+  ldxc1      $f2, $4($6)
+  # R2: :[[@LINE-1]]:3: error: instruction requires a CPU feature not currently enabled
+  luxc1      $f2, $4($6)
+  # R2: :[[@LINE-1]]:3: error: instruction requires a CPU feature not currently enabled
+  lwxc1      $f2, $4($6)
+  # R2: :[[@LINE-1]]:3: error: instruction requires a CPU feature not currently enabled
+  mfhc1      $7, $f2
+  # R2: :[[@LINE-1]]:3: error: instruction requires a CPU feature not currently enabled
+  msub.s     $f2, $f2, $f2, $f2
+  # R2: :[[@LINE-1]]:3: error: instruction requires a CPU feature not currently enabled
+  mthc1      $7, $f2
+  # R2: :[[@LINE-1]]:3: error: instruction requires a CPU feature not currently enabled
+  nmadd.s    $f2, $f2, $f2, $f2
+  # R2: :[[@LINE-1]]:3: error: instruction requires a CPU feature not currently enabled
+  nmsub.s    $f2, $f2, $f2, $f2
+  # R2: :[[@LINE-1]]:3: error: instruction requires a CPU feature not currently enabled
+  round.l.s  $f2, $f2
+  # R2: :[[@LINE-1]]:3: error: instruction requires a CPU feature not currently enabled
+  sdxc1      $f2, $4($6)
+  # R2: :[[@LINE-1]]:3: error: instruction requires a CPU feature not currently enabled
+  suxc1      $f2, $4($6)
+  # R2: :[[@LINE-1]]:3: error: instruction requires a CPU feature not currently enabled
+  swxc1      $f2, $4($6)
+  # R2: :[[@LINE-1]]:3: error: instruction requires a CPU feature not currently enabled
+  trunc.l.d  $f2, $f2
+  # R2: :[[@LINE-1]]:3: error: instruction requires a CPU feature not currently enabled
+  trunc.l.s  $f2, $f2
+  # R2: :[[@LINE-1]]:3: error: instruction requires a CPU feature not currently enabled
+
+  bc1eqz     $f2, 123
+  # R6: :[[@LINE-1]]:3: error: instruction requires a CPU feature not currently enabled
+  bc1nez     $f2, 456
+  # R6: :[[@LINE-1]]:3: error: instruction requires a CPU feature not currently enabled
+  class.d    $f2, $f2
+  # R6: :[[@LINE-1]]:3: error: instruction requires a CPU feature not currently enabled
+  class.s    $f2, $f2
+  # R6: :[[@LINE-1]]:3: error: instruction requires a CPU feature not currently enabled
+  cmp.af.d   $f2, $f2, $f2
+  # R6: :[[@LINE-1]]:3: error: instruction requires a CPU feature not currently enabled
+  cmp.af.s   $f2, $f2, $f2
+  # R6: :[[@LINE-1]]:3: error: instruction requires a CPU feature not currently enabled
+  cmp.eq.d   $f2, $f2, $f2
+  # R6: :[[@LINE-1]]:3: error: instruction requires a CPU feature not currently enabled
+  cmp.eq.s   $f2, $f2, $f2
+  # R6: :[[@LINE-1]]:3: error: instruction requires a CPU feature not currently enabled
+  cmp.le.d   $f2, $f2, $f2
+  # R6: :[[@LINE-1]]:3: error: instruction requires a CPU feature not currently enabled
+  cmp.le.s   $f2, $f2, $f2
+  # R6: :[[@LINE-1]]:3: error: instruction requires a CPU feature not currently enabled
+  cmp.lt.d   $f2, $f2, $f2
+  # R6: :[[@LINE-1]]:3: error: instruction requires a CPU feature not currently enabled
+  cmp.lt.s   $f2, $f2, $f2
+  # R6: :[[@LINE-1]]:3: error: instruction requires a CPU feature not currently enabled
+  cmp.saf.d  $f2, $f2, $f2
+  # R6: :[[@LINE-1]]:3: error: instruction requires a CPU feature not currently enabled
+  cmp.saf.s  $f2, $f2, $f2
+  # R6: :[[@LINE-1]]:3: error: instruction requires a CPU feature not currently enabled
+  cmp.seq.d  $f2, $f2, $f2
+  # R6: :[[@LINE-1]]:3: error: instruction requires a CPU feature not currently enabled
+  cmp.seq.s  $f2, $f2, $f2
+  # R6: :[[@LINE-1]]:3: error: instruction requires a CPU feature not currently enabled
+  cmp.sle.d  $f2, $f2, $f2
+  # R6: :[[@LINE-1]]:3: error: instruction requires a CPU feature not currently enabled
+  cmp.sle.s  $f2, $f2, $f2
+  # R6: :[[@LINE-1]]:3: error: instruction requires a CPU feature not currently enabled
+  cmp.slt.d  $f2, $f2, $f2
+  # R6: :[[@LINE-1]]:3: error: instruction requires a CPU feature not currently enabled
+  cmp.slt.s  $f2, $f2, $f2
+  # R6: :[[@LINE-1]]:3: error: instruction requires a CPU feature not currently enabled
+  cmp.sueq.d $f2, $f2, $f2
+  # R6: :[[@LINE-1]]:3: error: instruction requires a CPU feature not currently enabled
+  cmp.sueq.s $f2, $f2, $f2
+  # R6: :[[@LINE-1]]:3: error: instruction requires a CPU feature not currently enabled
+  cmp.sule.d $f2, $f2, $f2
+  # R6: :[[@LINE-1]]:3: error: instruction requires a CPU feature not currently enabled
+  cmp.sule.s $f2, $f2, $f2
+  # R6: :[[@LINE-1]]:3: error: instruction requires a CPU feature not currently enabled
+  cmp.sult.d $f2, $f2, $f2
+  # R6: :[[@LINE-1]]:3: error: instruction requires a CPU feature not currently enabled
+  cmp.sult.s $f2, $f2, $f2
+  # R6: :[[@LINE-1]]:3: error: instruction requires a CPU feature not currently enabled
+  cmp.sun.d  $f2, $f2, $f2
+  # R6: :[[@LINE-1]]:3: error: instruction requires a CPU feature not currently enabled
+  cmp.sun.s  $f2, $f2, $f2
+  # R6: :[[@LINE-1]]:3: error: instruction requires a CPU feature not currently enabled
+  cmp.ueq.d  $f2, $f2, $f2
+  # R6: :[[@LINE-1]]:3: error: instruction requires a CPU feature not currently enabled
+  cmp.ueq.s  $f2, $f2, $f2
+  # R6: :[[@LINE-1]]:3: error: instruction requires a CPU feature not currently enabled
+  cmp.ule.d  $f2, $f2, $f2
+  # R6: :[[@LINE-1]]:3: error: instruction requires a CPU feature not currently enabled
+  cmp.ule.s  $f2, $f2, $f2
+  # R6: :[[@LINE-1]]:3: error: instruction requires a CPU feature not currently enabled
+  cmp.ult.d  $f2, $f2, $f2
+  # R6: :[[@LINE-1]]:3: error: instruction requires a CPU feature not currently enabled
+  cmp.ult.s  $f2, $f2, $f2
+  # R6: :[[@LINE-1]]:3: error: instruction requires a CPU feature not currently enabled
+  cmp.un.d   $f2, $f2, $f2
+  # R6: :[[@LINE-1]]:3: error: instruction requires a CPU feature not currently enabled
+  cmp.un.s   $f2, $f2, $f2
+  # R6: :[[@LINE-1]]:3: error: instruction requires a CPU feature not currently enabled
+  maddf.d    $f2, $f2, $f2
+  # R6: :[[@LINE-1]]:3: error: instruction requires a CPU feature not currently enabled
+  maddf.s    $f2, $f2, $f2
+  # R6: :[[@LINE-1]]:3: error: instruction requires a CPU feature not currently enabled
+  max.d      $f2, $f2, $f2
+  # R6: :[[@LINE-1]]:3: error: instruction requires a CPU feature not currently enabled
+  max.s      $f2, $f2, $f2
+  # R6: :[[@LINE-1]]:3: error: instruction requires a CPU feature not currently enabled
+  maxa.d     $f2, $f2, $f2
+  # R6: :[[@LINE-1]]:3: error: instruction requires a CPU feature not currently enabled
+  maxa.s     $f2, $f2, $f2
+  # R6: :[[@LINE-1]]:3: error: instruction requires a CPU feature not currently enabled
+  min.d      $f2, $f2, $f2
+  # R6: :[[@LINE-1]]:3: error: instruction requires a CPU feature not currently enabled
+  min.s      $f2, $f2, $f2
+  # R6: :[[@LINE-1]]:3: error: instruction requires a CPU feature not currently enabled
+  mina.d     $f2, $f2, $f2
+  # R6: :[[@LINE-1]]:3: error: instruction requires a CPU feature not currently enabled
+  mina.s     $f2, $f2, $f2
+  # R6: :[[@LINE-1]]:3: error: instruction requires a CPU feature not currently enabled
+  msubf.d    $f2, $f2, $f2
+  # R6: :[[@LINE-1]]:3: error: instruction requires a CPU feature not currently enabled
+  msubf.s    $f2, $f2, $f2
+  # R6: :[[@LINE-1]]:3: error: instruction requires a CPU feature not currently enabled
+  rint.d     $f2, $f2
+  # R6: :[[@LINE-1]]:3: error: instruction requires a CPU feature not currently enabled
+  rint.s     $f2, $f2
+  # R6: :[[@LINE-1]]:3: error: instruction requires a CPU feature not currently enabled
+  sel.d      $f2, $f2, $f2
+  # R6: :[[@LINE-1]]:3: error: instruction requires a CPU feature not currently enabled
+  sel.s      $f2, $f2, $f2
+  # R6: :[[@LINE-1]]:3: error: instruction requires a CPU feature not currently enabled
+  seleqz.d   $f2, $f2, $f2
+  # R6: :[[@LINE-1]]:3: error: instruction requires a CPU feature not currently enabled
+  seleqz.s   $f2, $f2, $f2
+  # R6: :[[@LINE-1]]:3: error: instruction requires a CPU feature not currently enabled
+  selnez.d   $f2, $f2, $f2
+  # R6: :[[@LINE-1]]:3: error: instruction requires a CPU feature not currently enabled
+  selnez.s   $f2, $f2, $f2
+  # R6: :[[@LINE-1]]:3: error: instruction requires a CPU feature not currently enabled
+
+  abs.d      $f2, $f2
+  # 32: :[[@LINE-1]]:3: error: instruction requires a CPU feature not currently enabled
+  abs.s      $f2, $f2
+  # 32: :[[@LINE-1]]:3: error: instruction requires a CPU feature not currently enabled
+  add.d      $f2, $f2, $f2
+  # 32: :[[@LINE-1]]:3: error: instruction requires a CPU feature not currently enabled
+  add.s      $f2, $f2, $f2
+  # 32: :[[@LINE-1]]:3: error: instruction requires a CPU feature not currently enabled
+  c.eq.d     $f2, $f2
+  # 32: :[[@LINE-1]]:3: error: instruction requires a CPU feature not currently enabled
+  c.eq.s     $f2, $f2
+  # 32: :[[@LINE-1]]:3: error: instruction requires a CPU feature not currently enabled
+  c.f.d      $f2, $f2
+  # 32: :[[@LINE-1]]:3: error: instruction requires a CPU feature not currently enabled
+  c.f.s      $f2, $f2
+  # 32: :[[@LINE-1]]:3: error: instruction requires a CPU feature not currently enabled
+  c.le.d     $f2, $f2
+  # 32: :[[@LINE-1]]:3: error: instruction requires a CPU feature not currently enabled
+  c.le.s     $f2, $f2
+  # 32: :[[@LINE-1]]:3: error: instruction requires a CPU feature not currently enabled
+  c.lt.d     $f2, $f2
+  # 32: :[[@LINE-1]]:3: error: instruction requires a CPU feature not currently enabled
+  c.lt.s     $f2, $f2
+  # 32: :[[@LINE-1]]:3: error: instruction requires a CPU feature not currently enabled
+  c.nge.d    $f2, $f2
+  # 32: :[[@LINE-1]]:3: error: instruction requires a CPU feature not currently enabled
+  c.nge.s    $f2, $f2
+  # 32: :[[@LINE-1]]:3: error: instruction requires a CPU feature not currently enabled
+  c.ngl.d    $f2, $f2
+  # 32: :[[@LINE-1]]:3: error: instruction requires a CPU feature not currently enabled
+  c.ngl.s    $f2, $f2
+  # 32: :[[@LINE-1]]:3: error: instruction requires a CPU feature not currently enabled
+  c.ngle.d   $f2, $f2
+  # 32: :[[@LINE-1]]:3: error: instruction requires a CPU feature not currently enabled
+  c.ngle.s   $f2, $f2
+  # 32: :[[@LINE-1]]:3: error: instruction requires a CPU feature not currently enabled
+  c.ngt.d    $f2, $f2
+  # 32: :[[@LINE-1]]:3: error: instruction requires a CPU feature not currently enabled
+  c.ngt.s    $f2, $f2
+  # 32: :[[@LINE-1]]:3: error: instruction requires a CPU feature not currently enabled
+  c.ole.d    $f2, $f2
+  # 32: :[[@LINE-1]]:3: error: instruction requires a CPU feature not currently enabled
+  c.ole.s    $f2, $f2
+  # 32: :[[@LINE-1]]:3: error: instruction requires a CPU feature not currently enabled
+  c.olt.d    $f2, $f2
+  # 32: :[[@LINE-1]]:3: error: instruction requires a CPU feature not currently enabled
+  c.olt.s    $f2, $f2
+  # 32: :[[@LINE-1]]:3: error: instruction requires a CPU feature not currently enabled
+  c.seq.d    $f2, $f2
+  # 32: :[[@LINE-1]]:3: error: instruction requires a CPU feature not currently enabled
+  c.seq.s    $f2, $f2
+  # 32: :[[@LINE-1]]:3: error: instruction requires a CPU feature not currently enabled
+  c.sf.d     $f2, $f2
+  # 32: :[[@LINE-1]]:3: error: instruction requires a CPU feature not currently enabled
+  c.sf.s     $f2, $f2
+  # 32: :[[@LINE-1]]:3: error: instruction requires a CPU feature not currently enabled
+  c.ueq.d    $f2, $f2
+  # 32: :[[@LINE-1]]:3: error: instruction requires a CPU feature not currently enabled
+  c.ueq.s    $f2, $f2
+  # 32: :[[@LINE-1]]:3: error: instruction requires a CPU feature not currently enabled
+  c.ule.d    $f2, $f2
+  # 32: :[[@LINE-1]]:3: error: instruction requires a CPU feature not currently enabled
+  c.ule.s    $f2, $f2
+  # 32: :[[@LINE-1]]:3: error: instruction requires a CPU feature not currently enabled
+  c.ult.d    $f2, $f2
+  # 32: :[[@LINE-1]]:3: error: instruction requires a CPU feature not currently enabled
+  c.ult.s    $f2, $f2
+  # 32: :[[@LINE-1]]:3: error: instruction requires a CPU feature not currently enabled
+  c.un.d     $f2, $f2
+  # 32: :[[@LINE-1]]:3: error: instruction requires a CPU feature not currently enabled
+  c.un.s     $f2, $f2
+  # 32: :[[@LINE-1]]:3: error: instruction requires a CPU feature not currently enabled
+  ceil.w.d   $f2, $f2
+  # 32: :[[@LINE-1]]:3: error: instruction requires a CPU feature not currently enabled
+  ceil.w.s   $f2, $f2
+  # 32: :[[@LINE-1]]:3: error: instruction requires a CPU feature not currently enabled
+  cvt.d.s    $f2, $f2
+  # 32: :[[@LINE-1]]:3: error: instruction requires a CPU feature not currently enabled
+  cvt.d.w    $f2, $f2
+  # 32: :[[@LINE-1]]:3: error: instruction requires a CPU feature not currently enabled
+  cvt.s.d    $f2, $f2
+  # 32: :[[@LINE-1]]:3: error: instruction requires a CPU feature not currently enabled
+  cvt.s.w    $f2, $f2
+  # 32: :[[@LINE-1]]:3: error: instruction requires a CPU feature not currently enabled
+  cvt.w.d    $f2, $f2
+  # 32: :[[@LINE-1]]:3: error: instruction requires a CPU feature not currently enabled
+  cvt.w.s    $f2, $f2
+  # 32: :[[@LINE-1]]:3: error: instruction requires a CPU feature not currently enabled
+  div.d      $f2, $f2, $f2
+  # 32: :[[@LINE-1]]:3: error: instruction requires a CPU feature not currently enabled
+  div.s      $f2, $f2, $f2
+  # 32: :[[@LINE-1]]:3: error: instruction requires a CPU feature not currently enabled
+  floor.w.d  $f2, $f2
+  # 32: :[[@LINE-1]]:3: error: instruction requires a CPU feature not currently enabled
+  floor.w.s  $f2, $f2
+  # 32: :[[@LINE-1]]:3: error: instruction requires a CPU feature not currently enabled
+  ldc1       $f2, 16($7)
+  # FIXME: LDC1 is correctly rejected but the wrong error message is emitted.
+  # 32: :[[@LINE-2]]:19: error: invalid operand for instruction
+  lwc1       $f2, 16($7)
+  # FIXME: LWC1 is correctly rejected but the wrong error message is emitted.
+  # 32: :[[@LINE-2]]:19: error: invalid operand for instruction
+  madd.s     $f2, $f2, $f2, $f2
+  # 32: :[[@LINE-1]]:3: error: instruction requires a CPU feature not currently enabled
+  mfc1       $7, $f2
+  # 32: :[[@LINE-1]]:3: error: instruction requires a CPU feature not currently enabled
+  mov.d      $f2, $f2
+  # 32: :[[@LINE-1]]:3: error: instruction requires a CPU feature not currently enabled
+  mov.s      $f2, $f2
+  # 32: :[[@LINE-1]]:3: error: instruction requires a CPU feature not currently enabled
+  movf.d     $f2, $f2, $fcc2
+  # 32: :[[@LINE-1]]:3: error: instruction requires a CPU feature not currently enabled
+  movf.s     $f2, $f2, $fcc5
+  # 32: :[[@LINE-1]]:3: error: instruction requires a CPU feature not currently enabled
+  movn.d     $f2, $f2, $6
+  # 32: :[[@LINE-1]]:3: error: instruction requires a CPU feature not currently enabled
+  movn.s     $f2, $f2, $6
+  # 32: :[[@LINE-1]]:3: error: instruction requires a CPU feature not currently enabled
+  movt.d     $f2, $f2, $fcc0
+  # 32: :[[@LINE-1]]:3: error: instruction requires a CPU feature not currently enabled
+  movt.s     $f2, $f2, $fcc1
+  # 32: :[[@LINE-1]]:3: error: instruction requires a CPU feature not currently enabled
+  movz.d     $f2, $f2, $6
+  # 32: :[[@LINE-1]]:3: error: instruction requires a CPU feature not currently enabled
+  movz.s     $f2, $f2, $6
+  # 32: :[[@LINE-1]]:3: error: instruction requires a CPU feature not currently enabled
+  mtc1       $7, $f2
+  # 32: :[[@LINE-1]]:3: error: instruction requires a CPU feature not currently enabled
+  mul.d      $f2, $f2, $f2
+  # 32: :[[@LINE-1]]:3: error: instruction requires a CPU feature not currently enabled
+  mul.s      $f2, $f2, $f2
+  # 32: :[[@LINE-1]]:3: error: instruction requires a CPU feature not currently enabled
+  neg.d      $f2, $f2
+  # 32: :[[@LINE-1]]:3: error: instruction requires a CPU feature not currently enabled
+  neg.s      $f2, $f2
+  # 32: :[[@LINE-1]]:3: error: instruction requires a CPU feature not currently enabled
+  round.w.d  $f2, $f2
+  # 32: :[[@LINE-1]]:3: error: instruction requires a CPU feature not currently enabled
+  round.w.s  $f2, $f2
+  # 32: :[[@LINE-1]]:3: error: instruction requires a CPU feature not currently enabled
+  sdc1       $f2, 16($7)
+  # FIXME: SDC1 is correctly rejected but the wrong error message is emitted.
+  # 32: :[[@LINE-2]]:19: error: invalid operand for instruction
+  sqrt.d     $f2, $f2
+  # 32: :[[@LINE-1]]:3: error: instruction requires a CPU feature not currently enabled
+  sqrt.s     $f2, $f2
+  # 32: :[[@LINE-1]]:3: error: instruction requires a CPU feature not currently enabled
+  sub.d      $f2, $f2, $f2
+  # 32: :[[@LINE-1]]:3: error: instruction requires a CPU feature not currently enabled
+  sub.s      $f2, $f2, $f2
+  # 32: :[[@LINE-1]]:3: error: instruction requires a CPU feature not currently enabled
+  swc1       $f2, 16($7)
+  # FIXME: SWC1 is correctly rejected but the wrong error message is emitted.
+  # 32: :[[@LINE-2]]:19: error: invalid operand for instruction
+  trunc.w.d  $f2, $f2
+  # 32: :[[@LINE-1]]:3: error: instruction requires a CPU feature not currently enabled
+  trunc.w.s  $f2, $f2
+  # 32: :[[@LINE-1]]:3: error: instruction requires a CPU feature not currently enabled
diff --git a/test/MC/Mips/update-module-level-options.s b/test/MC/Mips/update-module-level-options.s
new file mode 100644
index 00000000000..3d6e97cb597
--- /dev/null
+++ b/test/MC/Mips/update-module-level-options.s
@@ -0,0 +1,14 @@
+# RUN: not llvm-mc %s -arch=mips -mcpu=mips32 -mattr=+fp64,-nooddspreg 2>&1 | \
+# RUN:   FileCheck %s
+
+  .module nooddspreg
+  add.s $f1, $f2, $f4
+# CHECK: :[[@LINE-1]]:9: error: -mno-odd-spreg prohibits the use of odd FPU registers
+
+  .set oddspreg
+  add.s $f1, $f2, $f4
+# CHECK-NOT: :[[@LINE-1]]:{{[0-9]+}}: error: -mno-odd-spreg prohibits the use of odd FPU registers
+
+  .set mips0
+  add.s $f1, $f2, $f4
+# CHECK: :[[@LINE-1]]:9: error: -mno-odd-spreg prohibits the use of odd FPU registers
diff --git a/test/MC/PowerPC/dcbt.s b/test/MC/PowerPC/dcbt.s
new file mode 100644
index 00000000000..66f7fa07b03
--- /dev/null
+++ b/test/MC/PowerPC/dcbt.s
@@ -0,0 +1,13 @@
+# RUN: llvm-mc -triple powerpc64-unknown-unknown --show-encoding %s | FileCheck -check-prefix=SERVER %s
+# RUN: llvm-mc -mcpu=a2 -triple powerpc64-unknown-unknown --show-encoding %s | FileCheck -check-prefix=EMBEDDED %s
+
+# SERVER: dcbt 2, 3, 10                   # encoding: [0x7d,0x42,0x1a,0x2c]
+          dcbt 2, 3, 10
+# SERVER: dcbtst 2, 3, 10                 # encoding: [0x7d,0x42,0x19,0xec]
+          dcbtst 2, 3, 10
+
+# EMBEDDED: dcbt 10, 2, 3                 # encoding: [0x7d,0x42,0x1a,0x2c]
+            dcbt 10, 2, 3
+# EMBEDDED: dcbtst 10, 2, 3               # encoding: [0x7d,0x42,0x19,0xec]
+            dcbtst 10, 2, 3
+
diff --git a/test/MC/PowerPC/deprecated-p7.s b/test/MC/PowerPC/deprecated-p7.s
index 21ef6d25a4e..6b5d91255a8 100644
--- a/test/MC/PowerPC/deprecated-p7.s
+++ b/test/MC/PowerPC/deprecated-p7.s
@@ -3,8 +3,8 @@
 # RUN: llvm-mc -triple powerpc-unknown-linux-gnu -mcpu=601 -show-encoding < %s 2>&1 | FileCheck -check-prefix=CHECK-OLD %s
 
          mftb 3
-# CHECK: warning: deprecated
-# CHECK: mftb 3
+# CHECK-NOT: warning: deprecated
+# CHECK: mfspr 3, 268
 
 # CHECK-OLD-NOT: warning: deprecated
 # CHECK-OLD: mftb 3
diff --git a/test/MC/PowerPC/ppc64-encoding-4xx.s b/test/MC/PowerPC/ppc64-encoding-4xx.s
index 5414e1a49bf..46fa5524a8f 100644
--- a/test/MC/PowerPC/ppc64-encoding-4xx.s
+++ b/test/MC/PowerPC/ppc64-encoding-4xx.s
@@ -10,30 +10,30 @@
 # CHECK-LE: mtdcr 178, 3                     # encoding: [0x86,0x2b,0x72,0x7c]
             mtdcr 178,3
 
-# CHECK-BE: tlbre 2, 3, 0                    # encoding: [0x7c,0x43,0x07,0x64]
-# CHECK-LE: tlbre 2, 3, 0                    # encoding: [0x64,0x07,0x43,0x7c]
+# CHECK-BE: tlbrehi 2, 3                     # encoding: [0x7c,0x43,0x07,0x64]
+# CHECK-LE: tlbrehi 2, 3                     # encoding: [0x64,0x07,0x43,0x7c]
             tlbre %r2, %r3, 0
-# CHECK-BE: tlbre 2, 3, 1                    # encoding: [0x7c,0x43,0x0f,0x64]
-# CHECK-LE: tlbre 2, 3, 1                    # encoding: [0x64,0x0f,0x43,0x7c]
+# CHECK-BE: tlbrelo 2, 3                     # encoding: [0x7c,0x43,0x0f,0x64]
+# CHECK-LE: tlbrelo 2, 3                     # encoding: [0x64,0x0f,0x43,0x7c]
             tlbre %r2, %r3, 1
-# CHECK-BE: tlbre 2, 3, 0                    # encoding: [0x7c,0x43,0x07,0x64]
-# CHECK-LE: tlbre 2, 3, 0                    # encoding: [0x64,0x07,0x43,0x7c]
+# CHECK-BE: tlbrehi 2, 3                     # encoding: [0x7c,0x43,0x07,0x64]
+# CHECK-LE: tlbrehi 2, 3                     # encoding: [0x64,0x07,0x43,0x7c]
             tlbrehi %r2, %r3
-# CHECK-BE: tlbre 2, 3, 1                    # encoding: [0x7c,0x43,0x0f,0x64]
-# CHECK-LE: tlbre 2, 3, 1                    # encoding: [0x64,0x0f,0x43,0x7c]
+# CHECK-BE: tlbrelo 2, 3                     # encoding: [0x7c,0x43,0x0f,0x64]
+# CHECK-LE: tlbrelo 2, 3                     # encoding: [0x64,0x0f,0x43,0x7c]
             tlbrelo %r2, %r3
 
-# CHECK-BE: tlbwe 2, 3, 0                    # encoding: [0x7c,0x43,0x07,0xa4]
-# CHECK-LE: tlbwe 2, 3, 0                    # encoding: [0xa4,0x07,0x43,0x7c]
+# CHECK-BE: tlbwehi 2, 3                     # encoding: [0x7c,0x43,0x07,0xa4]
+# CHECK-LE: tlbwehi 2, 3                     # encoding: [0xa4,0x07,0x43,0x7c]
             tlbwe %r2, %r3, 0
-# CHECK-BE: tlbwe 2, 3, 1                    # encoding: [0x7c,0x43,0x0f,0xa4]
-# CHECK-LE: tlbwe 2, 3, 1                    # encoding: [0xa4,0x0f,0x43,0x7c]
+# CHECK-BE: tlbwelo 2, 3                     # encoding: [0x7c,0x43,0x0f,0xa4]
+# CHECK-LE: tlbwelo 2, 3                     # encoding: [0xa4,0x0f,0x43,0x7c]
             tlbwe %r2, %r3, 1
-# CHECK-BE: tlbwe 2, 3, 0                    # encoding: [0x7c,0x43,0x07,0xa4]
-# CHECK-LE: tlbwe 2, 3, 0                    # encoding: [0xa4,0x07,0x43,0x7c]
+# CHECK-BE: tlbwehi 2, 3                     # encoding: [0x7c,0x43,0x07,0xa4]
+# CHECK-LE: tlbwehi 2, 3                     # encoding: [0xa4,0x07,0x43,0x7c]
             tlbwehi %r2, %r3
-# CHECK-BE: tlbwe 2, 3, 1                    # encoding: [0x7c,0x43,0x0f,0xa4]
-# CHECK-LE: tlbwe 2, 3, 1                    # encoding: [0xa4,0x0f,0x43,0x7c]
+# CHECK-BE: tlbwelo 2, 3                     # encoding: [0x7c,0x43,0x0f,0xa4]
+# CHECK-LE: tlbwelo 2, 3                     # encoding: [0xa4,0x0f,0x43,0x7c]
             tlbwelo %r2, %r3
 
 # CHECK-BE: tlbsx 2, 3, 1                    # encoding: [0x7c,0x43,0x0f,0x24]
@@ -43,52 +43,52 @@
 # CHECK-LE: tlbsx. 2, 3, 1                   # encoding: [0x25,0x0f,0x43,0x7c]
             tlbsx. %r2, %r3, %r1
 
-# CHECK-BE: mfspr 2, 1018                    # encoding: [0x7c,0x5a,0xfa,0xa6]
-# CHECK-LE: mfspr 2, 1018                    # encoding: [0xa6,0xfa,0x5a,0x7c]
+# CHECK-BE: mfdccr 2                         # encoding: [0x7c,0x5a,0xfa,0xa6]
+# CHECK-LE: mfdccr 2                         # encoding: [0xa6,0xfa,0x5a,0x7c]
             mfdccr %r2
-# CHECK-BE: mtspr 1018, 2                    # encoding: [0x7c,0x5a,0xfb,0xa6]
-# CHECK-LE: mtspr 1018, 2                    # encoding: [0xa6,0xfb,0x5a,0x7c]
+# CHECK-BE: mtdccr 2                         # encoding: [0x7c,0x5a,0xfb,0xa6]
+# CHECK-LE: mtdccr 2                         # encoding: [0xa6,0xfb,0x5a,0x7c]
             mtdccr %r2
 
-# CHECK-BE: mfspr 2, 1019                    # encoding: [0x7c,0x5b,0xfa,0xa6]
-# CHECK-LE: mfspr 2, 1019                    # encoding: [0xa6,0xfa,0x5b,0x7c]
+# CHECK-BE: mficcr 2                         # encoding: [0x7c,0x5b,0xfa,0xa6]
+# CHECK-LE: mficcr 2                         # encoding: [0xa6,0xfa,0x5b,0x7c]
             mficcr %r2
-# CHECK-BE: mtspr 1019, 2                    # encoding: [0x7c,0x5b,0xfb,0xa6]
-# CHECK-LE: mtspr 1019, 2                    # encoding: [0xa6,0xfb,0x5b,0x7c]
+# CHECK-BE: mticcr 2                         # encoding: [0x7c,0x5b,0xfb,0xa6]
+# CHECK-LE: mticcr 2                         # encoding: [0xa6,0xfb,0x5b,0x7c]
             mticcr %r2
 
-# CHECK-BE: mfspr 2, 981                    # encoding: [0x7c,0x55,0xf2,0xa6]
-# CHECK-LE: mfspr 2, 981                    # encoding: [0xa6,0xf2,0x55,0x7c]
+# CHECK-BE: mfdear 2                        # encoding: [0x7c,0x55,0xf2,0xa6]
+# CHECK-LE: mfdear 2                        # encoding: [0xa6,0xf2,0x55,0x7c]
             mfdear %r2
-# CHECK-BE: mtspr 981, 2                    # encoding: [0x7c,0x55,0xf3,0xa6]
-# CHECK-LE: mtspr 981, 2                    # encoding: [0xa6,0xf3,0x55,0x7c]
+# CHECK-BE: mtdear 2                        # encoding: [0x7c,0x55,0xf3,0xa6]
+# CHECK-LE: mtdear 2                        # encoding: [0xa6,0xf3,0x55,0x7c]
             mtdear %r2
 
-# CHECK-BE: mfspr 2, 980                    # encoding: [0x7c,0x54,0xf2,0xa6]
-# CHECK-LE: mfspr 2, 980                    # encoding: [0xa6,0xf2,0x54,0x7c]
+# CHECK-BE: mfesr 2                         # encoding: [0x7c,0x54,0xf2,0xa6]
+# CHECK-LE: mfesr 2                         # encoding: [0xa6,0xf2,0x54,0x7c]
             mfesr %r2
-# CHECK-BE: mtspr 980, 2                    # encoding: [0x7c,0x54,0xf3,0xa6]
-# CHECK-LE: mtspr 980, 2                    # encoding: [0xa6,0xf3,0x54,0x7c]
+# CHECK-BE: mtesr 2                         # encoding: [0x7c,0x54,0xf3,0xa6]
+# CHECK-LE: mtesr 2                         # encoding: [0xa6,0xf3,0x54,0x7c]
             mtesr %r2
 
-# CHECK-BE: mfspr 2, 986                    # encoding: [0x7c,0x5a,0xf2,0xa6]
-# CHECK-LE: mfspr 2, 986                    # encoding: [0xa6,0xf2,0x5a,0x7c]
+# CHECK-BE: mftcr 2                         # encoding: [0x7c,0x5a,0xf2,0xa6]
+# CHECK-LE: mftcr 2                         # encoding: [0xa6,0xf2,0x5a,0x7c]
             mftcr %r2
-# CHECK-BE: mtspr 986, 2                    # encoding: [0x7c,0x5a,0xf3,0xa6]
-# CHECK-LE: mtspr 986, 2                    # encoding: [0xa6,0xf3,0x5a,0x7c]
+# CHECK-BE: mttcr 2                         # encoding: [0x7c,0x5a,0xf3,0xa6]
+# CHECK-LE: mttcr 2                         # encoding: [0xa6,0xf3,0x5a,0x7c]
             mttcr %r2
 
-# CHECK-BE: mfspr 2, 989                    # encoding: [0x7c,0x5d,0xf2,0xa6]
-# CHECK-LE: mfspr 2, 989                    # encoding: [0xa6,0xf2,0x5d,0x7c]
+# CHECK-BE: mftblo 2                        # encoding: [0x7c,0x5d,0xf2,0xa6]
+# CHECK-LE: mftblo 2                        # encoding: [0xa6,0xf2,0x5d,0x7c]
             mftblo %r2
-# CHECK-BE: mtspr 989, 2                    # encoding: [0x7c,0x5d,0xf3,0xa6]
-# CHECK-LE: mtspr 989, 2                    # encoding: [0xa6,0xf3,0x5d,0x7c]
+# CHECK-BE: mttblo 2                        # encoding: [0x7c,0x5d,0xf3,0xa6]
+# CHECK-LE: mttblo 2                        # encoding: [0xa6,0xf3,0x5d,0x7c]
             mttblo %r2
-# CHECK-BE: mfspr 2, 988                    # encoding: [0x7c,0x5c,0xf2,0xa6]
-# CHECK-LE: mfspr 2, 988                    # encoding: [0xa6,0xf2,0x5c,0x7c]
+# CHECK-BE: mftbhi 2                        # encoding: [0x7c,0x5c,0xf2,0xa6]
+# CHECK-LE: mftbhi 2                        # encoding: [0xa6,0xf2,0x5c,0x7c]
             mftbhi %r2
-# CHECK-BE: mtspr 988, 2                    # encoding: [0x7c,0x5c,0xf3,0xa6]
-# CHECK-LE: mtspr 988, 2                    # encoding: [0xa6,0xf3,0x5c,0x7c]
+# CHECK-BE: mttbhi 2                        # encoding: [0x7c,0x5c,0xf3,0xa6]
+# CHECK-LE: mttbhi 2                        # encoding: [0xa6,0xf3,0x5c,0x7c]
             mttbhi %r2
 
 # CHECK-BE: dccci 5, 6                      # encoding: [0x7c,0x05,0x33,0x8c]
@@ -104,64 +104,64 @@
 # CHECK-LE: iccci 0, 0                      # encoding: [0x8c,0x07,0x00,0x7c]
             ici 0
 
-# CHECK-BE: mfspr 2, 990                    # encoding: [0x7c,0x5e,0xf2,0xa6]
-# CHECK-LE: mfspr 2, 990                    # encoding: [0xa6,0xf2,0x5e,0x7c]
+# CHECK-BE: mfsrr2 2                        # encoding: [0x7c,0x5e,0xf2,0xa6]
+# CHECK-LE: mfsrr2 2                        # encoding: [0xa6,0xf2,0x5e,0x7c]
             mfsrr2 2
-# CHECK-BE: mtspr 990, 2                    # encoding: [0x7c,0x5e,0xf3,0xa6]
-# CHECK-LE: mtspr 990, 2                    # encoding: [0xa6,0xf3,0x5e,0x7c]
+# CHECK-BE: mtsrr2 2                        # encoding: [0x7c,0x5e,0xf3,0xa6]
+# CHECK-LE: mtsrr2 2                        # encoding: [0xa6,0xf3,0x5e,0x7c]
             mtsrr2 2
-# CHECK-BE: mfspr 2, 991                    # encoding: [0x7c,0x5f,0xf2,0xa6]
-# CHECK-LE: mfspr 2, 991                    # encoding: [0xa6,0xf2,0x5f,0x7c]
+# CHECK-BE: mfsrr3 2                        # encoding: [0x7c,0x5f,0xf2,0xa6]
+# CHECK-LE: mfsrr3 2                        # encoding: [0xa6,0xf2,0x5f,0x7c]
             mfsrr3 2
-# CHECK-BE: mtspr 991, 2                    # encoding: [0x7c,0x5f,0xf3,0xa6]
-# CHECK-LE: mtspr 991, 2                    # encoding: [0xa6,0xf3,0x5f,0x7c]
+# CHECK-BE: mtsrr3 2                        # encoding: [0x7c,0x5f,0xf3,0xa6]
+# CHECK-LE: mtsrr3 2                        # encoding: [0xa6,0xf3,0x5f,0x7c]
             mtsrr3 2
 
-# CHECK-BE: mfdcr 5, 128                    # encoding: [0x7c,0xa0,0x22,0x86]
-# CHECK-LE: mfdcr 5, 128                    # encoding: [0x86,0x22,0xa0,0x7c]
+# CHECK-BE: mfbr0 5                         # encoding: [0x7c,0xa0,0x22,0x86]
+# CHECK-LE: mfbr0 5                         # encoding: [0x86,0x22,0xa0,0x7c]
             mfbr0 %r5
-# CHECK-BE: mtdcr 128, 5                    # encoding: [0x7c,0xa0,0x23,0x86]
-# CHECK-LE: mtdcr 128, 5                    # encoding: [0x86,0x23,0xa0,0x7c]
+# CHECK-BE: mtbr0 5                         # encoding: [0x7c,0xa0,0x23,0x86]
+# CHECK-LE: mtbr0 5                         # encoding: [0x86,0x23,0xa0,0x7c]
             mtbr0 %r5
-# CHECK-BE: mfdcr 5, 129                    # encoding: [0x7c,0xa1,0x22,0x86]
-# CHECK-LE: mfdcr 5, 129                    # encoding: [0x86,0x22,0xa1,0x7c]
+# CHECK-BE: mfbr1 5                         # encoding: [0x7c,0xa1,0x22,0x86]
+# CHECK-LE: mfbr1 5                         # encoding: [0x86,0x22,0xa1,0x7c]
             mfbr1 %r5
-# CHECK-BE: mtdcr 129, 5                    # encoding: [0x7c,0xa1,0x23,0x86]
-# CHECK-LE: mtdcr 129, 5                    # encoding: [0x86,0x23,0xa1,0x7c]
+# CHECK-BE: mtbr1 5                         # encoding: [0x7c,0xa1,0x23,0x86]
+# CHECK-LE: mtbr1 5                         # encoding: [0x86,0x23,0xa1,0x7c]
             mtbr1 %r5
-# CHECK-BE: mfdcr 5, 130                    # encoding: [0x7c,0xa2,0x22,0x86]
-# CHECK-LE: mfdcr 5, 130                    # encoding: [0x86,0x22,0xa2,0x7c]
+# CHECK-BE: mfbr2 5                         # encoding: [0x7c,0xa2,0x22,0x86]
+# CHECK-LE: mfbr2 5                         # encoding: [0x86,0x22,0xa2,0x7c]
             mfbr2 %r5
-# CHECK-BE: mtdcr 130, 5                    # encoding: [0x7c,0xa2,0x23,0x86]
-# CHECK-LE: mtdcr 130, 5                    # encoding: [0x86,0x23,0xa2,0x7c]
+# CHECK-BE: mtbr2 5                         # encoding: [0x7c,0xa2,0x23,0x86]
+# CHECK-LE: mtbr2 5                         # encoding: [0x86,0x23,0xa2,0x7c]
             mtbr2 %r5
-# CHECK-BE: mfdcr 5, 131                    # encoding: [0x7c,0xa3,0x22,0x86]
-# CHECK-LE: mfdcr 5, 131                    # encoding: [0x86,0x22,0xa3,0x7c]
+# CHECK-BE: mfbr3 5                         # encoding: [0x7c,0xa3,0x22,0x86]
+# CHECK-LE: mfbr3 5                         # encoding: [0x86,0x22,0xa3,0x7c]
             mfbr3 %r5
-# CHECK-BE: mtdcr 131, 5                    # encoding: [0x7c,0xa3,0x23,0x86]
-# CHECK-LE: mtdcr 131, 5                    # encoding: [0x86,0x23,0xa3,0x7c]
+# CHECK-BE: mtbr3 5                         # encoding: [0x7c,0xa3,0x23,0x86]
+# CHECK-LE: mtbr3 5                         # encoding: [0x86,0x23,0xa3,0x7c]
             mtbr3 %r5
-# CHECK-BE: mfdcr 5, 132                    # encoding: [0x7c,0xa4,0x22,0x86]
-# CHECK-LE: mfdcr 5, 132                    # encoding: [0x86,0x22,0xa4,0x7c]
+# CHECK-BE: mfbr4 5                         # encoding: [0x7c,0xa4,0x22,0x86]
+# CHECK-LE: mfbr4 5                         # encoding: [0x86,0x22,0xa4,0x7c]
             mfbr4 %r5
-# CHECK-BE: mtdcr 132, 5                    # encoding: [0x7c,0xa4,0x23,0x86]
-# CHECK-LE: mtdcr 132, 5                    # encoding: [0x86,0x23,0xa4,0x7c]
+# CHECK-BE: mtbr4 5                         # encoding: [0x7c,0xa4,0x23,0x86]
+# CHECK-LE: mtbr4 5                         # encoding: [0x86,0x23,0xa4,0x7c]
             mtbr4 %r5
-# CHECK-BE: mfdcr 5, 133                    # encoding: [0x7c,0xa5,0x22,0x86]
-# CHECK-LE: mfdcr 5, 133                    # encoding: [0x86,0x22,0xa5,0x7c]
+# CHECK-BE: mfbr5 5                         # encoding: [0x7c,0xa5,0x22,0x86]
+# CHECK-LE: mfbr5 5                         # encoding: [0x86,0x22,0xa5,0x7c]
             mfbr5 %r5
-# CHECK-BE: mtdcr 133, 5                    # encoding: [0x7c,0xa5,0x23,0x86]
-# CHECK-LE: mtdcr 133, 5                    # encoding: [0x86,0x23,0xa5,0x7c]
+# CHECK-BE: mtbr5 5                         # encoding: [0x7c,0xa5,0x23,0x86]
+# CHECK-LE: mtbr5 5                         # encoding: [0x86,0x23,0xa5,0x7c]
             mtbr5 %r5
-# CHECK-BE: mfdcr 5, 134                    # encoding: [0x7c,0xa6,0x22,0x86]
-# CHECK-LE: mfdcr 5, 134                    # encoding: [0x86,0x22,0xa6,0x7c]
+# CHECK-BE: mfbr6 5                         # encoding: [0x7c,0xa6,0x22,0x86]
+# CHECK-LE: mfbr6 5                         # encoding: [0x86,0x22,0xa6,0x7c]
             mfbr6 %r5
-# CHECK-BE: mtdcr 134, 5                    # encoding: [0x7c,0xa6,0x23,0x86]
-# CHECK-LE: mtdcr 134, 5                    # encoding: [0x86,0x23,0xa6,0x7c]
+# CHECK-BE: mtbr6 5                         # encoding: [0x7c,0xa6,0x23,0x86]
+# CHECK-LE: mtbr6 5                         # encoding: [0x86,0x23,0xa6,0x7c]
             mtbr6 %r5
-# CHECK-BE: mfdcr 5, 135                    # encoding: [0x7c,0xa7,0x22,0x86]
-# CHECK-LE: mfdcr 5, 135                    # encoding: [0x86,0x22,0xa7,0x7c]
+# CHECK-BE: mfbr7 5                         # encoding: [0x7c,0xa7,0x22,0x86]
+# CHECK-LE: mfbr7 5                         # encoding: [0x86,0x22,0xa7,0x7c]
             mfbr7 %r5
-# CHECK-BE: mtdcr 135, 5                    # encoding: [0x7c,0xa7,0x23,0x86]
-# CHECK-LE: mtdcr 135, 5                    # encoding: [0x86,0x23,0xa7,0x7c]
+# CHECK-BE: mtbr7 5                         # encoding: [0x7c,0xa7,0x23,0x86]
+# CHECK-LE: mtbr7 5                         # encoding: [0x86,0x23,0xa7,0x7c]
             mtbr7 %r5
diff --git a/test/MC/PowerPC/ppc64-encoding-6xx.s b/test/MC/PowerPC/ppc64-encoding-6xx.s
index 3a5e7a126b8..657eaf3f5da 100644
--- a/test/MC/PowerPC/ppc64-encoding-6xx.s
+++ b/test/MC/PowerPC/ppc64-encoding-6xx.s
@@ -3,102 +3,102 @@
 
 # Instructions specific to the PowerPC 6xx family:
 
-# CHECK-BE: mfspr 12, 528                    # encoding: [0x7d,0x90,0x82,0xa6]
-# CHECK-LE: mfspr 12, 528                    # encoding: [0xa6,0x82,0x90,0x7d]
+# CHECK-BE: mfibatu 12, 0                    # encoding: [0x7d,0x90,0x82,0xa6]
+# CHECK-LE: mfibatu 12, 0                    # encoding: [0xa6,0x82,0x90,0x7d]
 mfibatu %r12, 0
-# CHECK-BE: mfspr 12, 529                    # encoding: [0x7d,0x91,0x82,0xa6]
-# CHECK-LE: mfspr 12, 529                    # encoding: [0xa6,0x82,0x91,0x7d]
+# CHECK-BE: mfibatl 12, 0                    # encoding: [0x7d,0x91,0x82,0xa6]
+# CHECK-LE: mfibatl 12, 0                    # encoding: [0xa6,0x82,0x91,0x7d]
 mfibatl %r12, 0
-# CHECK-BE: mfspr 12, 530                    # encoding: [0x7d,0x92,0x82,0xa6]
-# CHECK-LE: mfspr 12, 530                    # encoding: [0xa6,0x82,0x92,0x7d]
+# CHECK-BE: mfibatu 12, 1                    # encoding: [0x7d,0x92,0x82,0xa6]
+# CHECK-LE: mfibatu 12, 1                    # encoding: [0xa6,0x82,0x92,0x7d]
 mfibatu %r12, 1
-# CHECK-BE: mfspr 12, 531                    # encoding: [0x7d,0x93,0x82,0xa6]
-# CHECK-LE: mfspr 12, 531                    # encoding: [0xa6,0x82,0x93,0x7d]
+# CHECK-BE: mfibatl 12, 1                    # encoding: [0x7d,0x93,0x82,0xa6]
+# CHECK-LE: mfibatl 12, 1                    # encoding: [0xa6,0x82,0x93,0x7d]
 mfibatl %r12, 1
-# CHECK-BE: mfspr 12, 532                    # encoding: [0x7d,0x94,0x82,0xa6]
-# CHECK-LE: mfspr 12, 532                    # encoding: [0xa6,0x82,0x94,0x7d]
+# CHECK-BE: mfibatu 12, 2                    # encoding: [0x7d,0x94,0x82,0xa6]
+# CHECK-LE: mfibatu 12, 2                    # encoding: [0xa6,0x82,0x94,0x7d]
 mfibatu %r12, 2
-# CHECK-BE: mfspr 12, 533                    # encoding: [0x7d,0x95,0x82,0xa6]
-# CHECK-LE: mfspr 12, 533                    # encoding: [0xa6,0x82,0x95,0x7d]
+# CHECK-BE: mfibatl 12, 2                    # encoding: [0x7d,0x95,0x82,0xa6]
+# CHECK-LE: mfibatl 12, 2                    # encoding: [0xa6,0x82,0x95,0x7d]
 mfibatl %r12, 2
-# CHECK-BE: mfspr 12, 534                    # encoding: [0x7d,0x96,0x82,0xa6]
-# CHECK-LE: mfspr 12, 534                    # encoding: [0xa6,0x82,0x96,0x7d]
+# CHECK-BE: mfibatu 12, 3                    # encoding: [0x7d,0x96,0x82,0xa6]
+# CHECK-LE: mfibatu 12, 3                    # encoding: [0xa6,0x82,0x96,0x7d]
 mfibatu %r12, 3
-# CHECK-BE: mfspr 12, 535                    # encoding: [0x7d,0x97,0x82,0xa6]
-# CHECK-LE: mfspr 12, 535                    # encoding: [0xa6,0x82,0x97,0x7d]
+# CHECK-BE: mfibatl 12, 3                    # encoding: [0x7d,0x97,0x82,0xa6]
+# CHECK-LE: mfibatl 12, 3                    # encoding: [0xa6,0x82,0x97,0x7d]
 mfibatl %r12, 3
-# CHECK-BE: mtspr 528, 12                    # encoding: [0x7d,0x90,0x83,0xa6]
-# CHECK-LE: mtspr 528, 12                    # encoding: [0xa6,0x83,0x90,0x7d]
+# CHECK-BE: mtibatu 0, 12                    # encoding: [0x7d,0x90,0x83,0xa6]
+# CHECK-LE: mtibatu 0, 12                    # encoding: [0xa6,0x83,0x90,0x7d]
 mtibatu 0, %r12
-# CHECK-BE: mtspr 529, 12                    # encoding: [0x7d,0x91,0x83,0xa6]
-# CHECK-LE: mtspr 529, 12                    # encoding: [0xa6,0x83,0x91,0x7d]
+# CHECK-BE: mtibatl 0, 12                    # encoding: [0x7d,0x91,0x83,0xa6]
+# CHECK-LE: mtibatl 0, 12                    # encoding: [0xa6,0x83,0x91,0x7d]
 mtibatl 0, %r12
-# CHECK-BE: mtspr 530, 12                    # encoding: [0x7d,0x92,0x83,0xa6]
-# CHECK-LE: mtspr 530, 12                    # encoding: [0xa6,0x83,0x92,0x7d]
+# CHECK-BE: mtibatu 1, 12                    # encoding: [0x7d,0x92,0x83,0xa6]
+# CHECK-LE: mtibatu 1, 12                    # encoding: [0xa6,0x83,0x92,0x7d]
 mtibatu 1, %r12
-# CHECK-BE: mtspr 531, 12                    # encoding: [0x7d,0x93,0x83,0xa6]
-# CHECK-LE: mtspr 531, 12                    # encoding: [0xa6,0x83,0x93,0x7d]
+# CHECK-BE: mtibatl 1, 12                    # encoding: [0x7d,0x93,0x83,0xa6]
+# CHECK-LE: mtibatl 1, 12                    # encoding: [0xa6,0x83,0x93,0x7d]
 mtibatl 1, %r12
-# CHECK-BE: mtspr 532, 12                    # encoding: [0x7d,0x94,0x83,0xa6]
-# CHECK-LE: mtspr 532, 12                    # encoding: [0xa6,0x83,0x94,0x7d]
+# CHECK-BE: mtibatu 2, 12                    # encoding: [0x7d,0x94,0x83,0xa6]
+# CHECK-LE: mtibatu 2, 12                    # encoding: [0xa6,0x83,0x94,0x7d]
 mtibatu 2, %r12
-# CHECK-BE: mtspr 533, 12                    # encoding: [0x7d,0x95,0x83,0xa6]
-# CHECK-LE: mtspr 533, 12                    # encoding: [0xa6,0x83,0x95,0x7d]
+# CHECK-BE: mtibatl 2, 12                    # encoding: [0x7d,0x95,0x83,0xa6]
+# CHECK-LE: mtibatl 2, 12                    # encoding: [0xa6,0x83,0x95,0x7d]
 mtibatl 2, %r12
-# CHECK-BE: mtspr 534, 12                    # encoding: [0x7d,0x96,0x83,0xa6]
-# CHECK-LE: mtspr 534, 12                    # encoding: [0xa6,0x83,0x96,0x7d]
+# CHECK-BE: mtibatu 3, 12                    # encoding: [0x7d,0x96,0x83,0xa6]
+# CHECK-LE: mtibatu 3, 12                    # encoding: [0xa6,0x83,0x96,0x7d]
 mtibatu 3, %r12
-# CHECK-BE: mtspr 535, 12                    # encoding: [0x7d,0x97,0x83,0xa6]
-# CHECK-LE: mtspr 535, 12                    # encoding: [0xa6,0x83,0x97,0x7d]
+# CHECK-BE: mtibatl 3, 12                    # encoding: [0x7d,0x97,0x83,0xa6]
+# CHECK-LE: mtibatl 3, 12                    # encoding: [0xa6,0x83,0x97,0x7d]
 mtibatl 3, %r12
 
-# CHECK-BE: mfspr 12, 536                    # encoding: [0x7d,0x98,0x82,0xa6]
-# CHECK-LE: mfspr 12, 536                    # encoding: [0xa6,0x82,0x98,0x7d]
+# CHECK-BE: mfdbatu 12, 0                    # encoding: [0x7d,0x98,0x82,0xa6]
+# CHECK-LE: mfdbatu 12, 0                    # encoding: [0xa6,0x82,0x98,0x7d]
 mfdbatu %r12, 0
-# CHECK-BE: mfspr 12, 537                    # encoding: [0x7d,0x99,0x82,0xa6]
-# CHECK-LE: mfspr 12, 537                    # encoding: [0xa6,0x82,0x99,0x7d]
+# CHECK-BE: mfdbatl 12, 0                    # encoding: [0x7d,0x99,0x82,0xa6]
+# CHECK-LE: mfdbatl 12, 0                    # encoding: [0xa6,0x82,0x99,0x7d]
 mfdbatl %r12, 0
-# CHECK-BE: mfspr 12, 538                    # encoding: [0x7d,0x9a,0x82,0xa6]
-# CHECK-LE: mfspr 12, 538                    # encoding: [0xa6,0x82,0x9a,0x7d]
+# CHECK-BE: mfdbatu 12, 1                    # encoding: [0x7d,0x9a,0x82,0xa6]
+# CHECK-LE: mfdbatu 12, 1                    # encoding: [0xa6,0x82,0x9a,0x7d]
 mfdbatu %r12, 1
-# CHECK-BE: mfspr 12, 539                    # encoding: [0x7d,0x9b,0x82,0xa6]
-# CHECK-LE: mfspr 12, 539                    # encoding: [0xa6,0x82,0x9b,0x7d]
+# CHECK-BE: mfdbatl 12, 1                    # encoding: [0x7d,0x9b,0x82,0xa6]
+# CHECK-LE: mfdbatl 12, 1                    # encoding: [0xa6,0x82,0x9b,0x7d]
 mfdbatl %r12, 1
-# CHECK-BE: mfspr 12, 540                    # encoding: [0x7d,0x9c,0x82,0xa6]
-# CHECK-LE: mfspr 12, 540                    # encoding: [0xa6,0x82,0x9c,0x7d]
+# CHECK-BE: mfdbatu 12, 2                    # encoding: [0x7d,0x9c,0x82,0xa6]
+# CHECK-LE: mfdbatu 12, 2                    # encoding: [0xa6,0x82,0x9c,0x7d]
 mfdbatu %r12, 2
-# CHECK-BE: mfspr 12, 541                    # encoding: [0x7d,0x9d,0x82,0xa6]
-# CHECK-LE: mfspr 12, 541                    # encoding: [0xa6,0x82,0x9d,0x7d]
+# CHECK-BE: mfdbatl 12, 2                    # encoding: [0x7d,0x9d,0x82,0xa6]
+# CHECK-LE: mfdbatl 12, 2                    # encoding: [0xa6,0x82,0x9d,0x7d]
 mfdbatl %r12, 2
-# CHECK-BE: mfspr 12, 542                    # encoding: [0x7d,0x9e,0x82,0xa6]
-# CHECK-LE: mfspr 12, 542                    # encoding: [0xa6,0x82,0x9e,0x7d]
+# CHECK-BE: mfdbatu 12, 3                    # encoding: [0x7d,0x9e,0x82,0xa6]
+# CHECK-LE: mfdbatu 12, 3                    # encoding: [0xa6,0x82,0x9e,0x7d]
 mfdbatu %r12, 3
-# CHECK-BE: mfspr 12, 543                    # encoding: [0x7d,0x9f,0x82,0xa6]
-# CHECK-LE: mfspr 12, 543                    # encoding: [0xa6,0x82,0x9f,0x7d]
+# CHECK-BE: mfdbatl 12, 3                    # encoding: [0x7d,0x9f,0x82,0xa6]
+# CHECK-LE: mfdbatl 12, 3                    # encoding: [0xa6,0x82,0x9f,0x7d]
 mfdbatl %r12, 3
-# CHECK-BE: mtspr 536, 12                    # encoding: [0x7d,0x98,0x83,0xa6]
-# CHECK-LE: mtspr 536, 12                    # encoding: [0xa6,0x83,0x98,0x7d]
+# CHECK-BE: mtdbatu 0, 12                    # encoding: [0x7d,0x98,0x83,0xa6]
+# CHECK-LE: mtdbatu 0, 12                    # encoding: [0xa6,0x83,0x98,0x7d]
 mtdbatu 0, %r12
-# CHECK-BE: mtspr 537, 12                    # encoding: [0x7d,0x99,0x83,0xa6]
-# CHECK-LE: mtspr 537, 12                    # encoding: [0xa6,0x83,0x99,0x7d]
+# CHECK-BE: mtdbatl 0, 12                    # encoding: [0x7d,0x99,0x83,0xa6]
+# CHECK-LE: mtdbatl 0, 12                    # encoding: [0xa6,0x83,0x99,0x7d]
 mtdbatl 0, %r12
-# CHECK-BE: mtspr 538, 12                    # encoding: [0x7d,0x9a,0x83,0xa6]
-# CHECK-LE: mtspr 538, 12                    # encoding: [0xa6,0x83,0x9a,0x7d]
+# CHECK-BE: mtdbatu 1, 12                    # encoding: [0x7d,0x9a,0x83,0xa6]
+# CHECK-LE: mtdbatu 1, 12                    # encoding: [0xa6,0x83,0x9a,0x7d]
 mtdbatu 1, %r12
-# CHECK-BE: mtspr 539, 12                    # encoding: [0x7d,0x9b,0x83,0xa6]
-# CHECK-LE: mtspr 539, 12                    # encoding: [0xa6,0x83,0x9b,0x7d]
+# CHECK-BE: mtdbatl 1, 12                    # encoding: [0x7d,0x9b,0x83,0xa6]
+# CHECK-LE: mtdbatl 1, 12                    # encoding: [0xa6,0x83,0x9b,0x7d]
 mtdbatl 1, %r12
-# CHECK-BE: mtspr 540, 12                    # encoding: [0x7d,0x9c,0x83,0xa6]
-# CHECK-LE: mtspr 540, 12                    # encoding: [0xa6,0x83,0x9c,0x7d]
+# CHECK-BE: mtdbatu 2, 12                    # encoding: [0x7d,0x9c,0x83,0xa6]
+# CHECK-LE: mtdbatu 2, 12                    # encoding: [0xa6,0x83,0x9c,0x7d]
 mtdbatu 2, %r12
-# CHECK-BE: mtspr 541, 12                    # encoding: [0x7d,0x9d,0x83,0xa6]
-# CHECK-LE: mtspr 541, 12                    # encoding: [0xa6,0x83,0x9d,0x7d]
+# CHECK-BE: mtdbatl 2, 12                    # encoding: [0x7d,0x9d,0x83,0xa6]
+# CHECK-LE: mtdbatl 2, 12                    # encoding: [0xa6,0x83,0x9d,0x7d]
 mtdbatl 2, %r12
-# CHECK-BE: mtspr 542, 12                    # encoding: [0x7d,0x9e,0x83,0xa6]
-# CHECK-LE: mtspr 542, 12                    # encoding: [0xa6,0x83,0x9e,0x7d]
+# CHECK-BE: mtdbatu 3, 12                    # encoding: [0x7d,0x9e,0x83,0xa6]
+# CHECK-LE: mtdbatu 3, 12                    # encoding: [0xa6,0x83,0x9e,0x7d]
 mtdbatu 3, %r12
-# CHECK-BE: mtspr 543, 12                    # encoding: [0x7d,0x9f,0x83,0xa6]
-# CHECK-LE: mtspr 543, 12                    # encoding: [0xa6,0x83,0x9f,0x7d]
+# CHECK-BE: mtdbatl 3, 12                    # encoding: [0x7d,0x9f,0x83,0xa6]
+# CHECK-LE: mtdbatl 3, 12                    # encoding: [0xa6,0x83,0x9f,0x7d]
 mtdbatl 3, %r12
 
 # CHECK-BE: tlbld 4                        # encoding: [0x7c,0x00,0x27,0xa4]
diff --git a/test/MC/PowerPC/ppc64-encoding-bookII.s b/test/MC/PowerPC/ppc64-encoding-bookII.s
index a4e2d0e3b2d..9b68dcc7e8d 100644
--- a/test/MC/PowerPC/ppc64-encoding-bookII.s
+++ b/test/MC/PowerPC/ppc64-encoding-bookII.s
@@ -12,14 +12,48 @@
 # CHECK-LE: icbt 0, 5, 31                   # encoding: [0x2c,0xf8,0x05,0x7c]
             icbt 0, 5, 31
 
-# FIXME:    dcbt 2, 3, 10
+# CHECK-BE: dcbt 2, 3, 10                   # encoding: [0x7d,0x42,0x1a,0x2c]
+# CHECK-LE: dcbt 2, 3, 10                   # encoding: [0x2c,0x1a,0x42,0x7d]
+            dcbt 2, 3, 10
+# CHECK-BE: dcbt 2, 3, 10                   # encoding: [0x7d,0x42,0x1a,0x2c]
+# CHECK-LE: dcbt 2, 3, 10                   # encoding: [0x2c,0x1a,0x42,0x7d]
+            dcbtct 2, 3, 10
+# CHECK-BE: dcbt 2, 3, 10                   # encoding: [0x7d,0x42,0x1a,0x2c]
+# CHECK-LE: dcbt 2, 3, 10                   # encoding: [0x2c,0x1a,0x42,0x7d]
+            dcbtds 2, 3, 10
 # CHECK-BE: dcbt 2, 3                       # encoding: [0x7c,0x02,0x1a,0x2c]
 # CHECK-LE: dcbt 2, 3                       # encoding: [0x2c,0x1a,0x02,0x7c]
             dcbt 2, 3
-# FIXME:    dcbtst 2, 3, 10
+# CHECK-BE: dcbt 2, 3                       # encoding: [0x7c,0x02,0x1a,0x2c]
+# CHECK-LE: dcbt 2, 3                       # encoding: [0x2c,0x1a,0x02,0x7c]
+            dcbt 2, 3, 0
+# CHECK-BE: dcbtt 2, 3                      # encoding: [0x7e,0x02,0x1a,0x2c]
+# CHECK-LE: dcbtt 2, 3                      # encoding: [0x2c,0x1a,0x02,0x7e]
+            dcbtt 2, 3
+# CHECK-BE: dcbtt 2, 3                      # encoding: [0x7e,0x02,0x1a,0x2c]
+# CHECK-LE: dcbtt 2, 3                      # encoding: [0x2c,0x1a,0x02,0x7e]
+            dcbt 2, 3, 16
+# CHECK-BE: dcbtst 2, 3, 10                 # encoding: [0x7d,0x42,0x19,0xec]
+# CHECK-LE: dcbtst 2, 3, 10                 # encoding: [0xec,0x19,0x42,0x7d]
+            dcbtst 2, 3, 10
+# CHECK-BE: dcbtst 2, 3, 10                 # encoding: [0x7d,0x42,0x19,0xec]
+# CHECK-LE: dcbtst 2, 3, 10                 # encoding: [0xec,0x19,0x42,0x7d]
+            dcbtstct 2, 3, 10
+# CHECK-BE: dcbtst 2, 3, 10                 # encoding: [0x7d,0x42,0x19,0xec]
+# CHECK-LE: dcbtst 2, 3, 10                 # encoding: [0xec,0x19,0x42,0x7d]
+            dcbtstds 2, 3, 10
 # CHECK-BE: dcbtst 2, 3                     # encoding: [0x7c,0x02,0x19,0xec]
 # CHECK-LE: dcbtst 2, 3                     # encoding: [0xec,0x19,0x02,0x7c]
             dcbtst 2, 3
+# CHECK-BE: dcbtst 2, 3                     # encoding: [0x7c,0x02,0x19,0xec]
+# CHECK-LE: dcbtst 2, 3                     # encoding: [0xec,0x19,0x02,0x7c]
+            dcbtst 2, 3, 0
+# CHECK-BE: dcbtstt 2, 3                    # encoding: [0x7e,0x02,0x19,0xec]
+# CHECK-LE: dcbtstt 2, 3                    # encoding: [0xec,0x19,0x02,0x7e]
+            dcbtstt 2, 3
+# CHECK-BE: dcbtstt 2, 3                    # encoding: [0x7e,0x02,0x19,0xec]
+# CHECK-LE: dcbtstt 2, 3                    # encoding: [0xec,0x19,0x02,0x7e]
+            dcbtst 2, 3, 16
 # CHECK-BE: dcbz 2, 3                       # encoding: [0x7c,0x02,0x1f,0xec]
 # CHECK-LE: dcbz 2, 3                       # encoding: [0xec,0x1f,0x02,0x7c]
             dcbz 2, 3
@@ -43,19 +77,19 @@
 # CHECK-LE: stdcx. 2, 3, 4                  # encoding: [0xad,0x21,0x43,0x7c]
             stdcx. 2, 3, 4
 
-# CHECK-BE: sync 2                          # encoding: [0x7c,0x40,0x04,0xac]
-# CHECK-LE: sync 2                          # encoding: [0xac,0x04,0x40,0x7c]
+# CHECK-BE: ptesync                         # encoding: [0x7c,0x40,0x04,0xac]
+# CHECK-LE: ptesync                         # encoding: [0xac,0x04,0x40,0x7c]
             sync 2
 # CHECK-BE: eieio                           # encoding: [0x7c,0x00,0x06,0xac]
 # CHECK-LE: eieio                           # encoding: [0xac,0x06,0x00,0x7c]
             eieio
-# CHECK-BE: wait 2                          # encoding: [0x7c,0x40,0x00,0x7c]
-# CHECK-LE: wait 2                          # encoding: [0x7c,0x00,0x40,0x7c]
+# CHECK-BE: waitimpl                        # encoding: [0x7c,0x40,0x00,0x7c]
+# CHECK-LE: waitimpl                        # encoding: [0x7c,0x00,0x40,0x7c]
             wait 2
 # CHECK-BE: mbar 1                          # encoding: [0x7c,0x20,0x06,0xac]
 # CHECK-LE: mbar 1                          # encoding: [0xac,0x06,0x20,0x7c]
             mbar 1
-# CHECK-BE: mbar 0
+# CHECK-BE: mbar                            # encoding: [0x7c,0x00,0x06,0xac]
             mbar
 
 # Extended mnemonics
@@ -97,27 +131,27 @@
 # CHECK-LE: ldarx 2, 3, 4, 1                # encoding: [0xa9,0x20,0x43,0x7c]
             ldarx 2, 3, 4, 1
 
-# CHECK-BE: sync 0                          # encoding: [0x7c,0x00,0x04,0xac]
-# CHECK-LE: sync 0                          # encoding: [0xac,0x04,0x00,0x7c]
+# CHECK-BE: sync                            # encoding: [0x7c,0x00,0x04,0xac]
+# CHECK-LE: sync                            # encoding: [0xac,0x04,0x00,0x7c]
             sync
-# CHECK-BE: sync 0                          # encoding: [0x7c,0x00,0x04,0xac]
-# CHECK-LE: sync 0                          # encoding: [0xac,0x04,0x00,0x7c]
+# CHECK-BE: sync                            # encoding: [0x7c,0x00,0x04,0xac]
+# CHECK-LE: sync                            # encoding: [0xac,0x04,0x00,0x7c]
             msync
-# CHECK-BE: sync 1                          # encoding: [0x7c,0x20,0x04,0xac]
-# CHECK-LE: sync 1                          # encoding: [0xac,0x04,0x20,0x7c]
+# CHECK-BE: lwsync                          # encoding: [0x7c,0x20,0x04,0xac]
+# CHECK-LE: lwsync                          # encoding: [0xac,0x04,0x20,0x7c]
             lwsync
-# CHECK-BE: sync 2                          # encoding: [0x7c,0x40,0x04,0xac]
-# CHECK-LE: sync 2                          # encoding: [0xac,0x04,0x40,0x7c]
+# CHECK-BE: ptesync                         # encoding: [0x7c,0x40,0x04,0xac]
+# CHECK-LE: ptesync                         # encoding: [0xac,0x04,0x40,0x7c]
             ptesync
 
-# CHECK-BE: wait 0                          # encoding: [0x7c,0x00,0x00,0x7c]
-# CHECK-LE: wait 0                          # encoding: [0x7c,0x00,0x00,0x7c]
+# CHECK-BE: wait                            # encoding: [0x7c,0x00,0x00,0x7c]
+# CHECK-LE: wait                            # encoding: [0x7c,0x00,0x00,0x7c]
             wait
-# CHECK-BE: wait 1                          # encoding: [0x7c,0x20,0x00,0x7c]
-# CHECK-LE: wait 1                          # encoding: [0x7c,0x00,0x20,0x7c]
+# CHECK-BE: waitrsv                         # encoding: [0x7c,0x20,0x00,0x7c]
+# CHECK-LE: waitrsv                         # encoding: [0x7c,0x00,0x20,0x7c]
             waitrsv
-# CHECK-BE: wait 2                          # encoding: [0x7c,0x40,0x00,0x7c]
-# CHECK-LE: wait 2                          # encoding: [0x7c,0x00,0x40,0x7c]
+# CHECK-BE: waitimpl                        # encoding: [0x7c,0x40,0x00,0x7c]
+# CHECK-LE: waitimpl                        # encoding: [0x7c,0x00,0x40,0x7c]
             waitimpl
 
 # Time base instructions
@@ -131,13 +165,13 @@
 # CHECK-BE: mftb 2, 268                     # encoding: [0x7c,0x4c,0x42,0xe6]
 # CHECK-LE: mftb 2, 268                     # encoding: [0xe6,0x42,0x4c,0x7c]
             mftbl 2
-# CHECK-BE: mftb 2, 269                     # encoding: [0x7c,0x4d,0x42,0xe6]
-# CHECK-LE: mftb 2, 269                     # encoding: [0xe6,0x42,0x4d,0x7c]
+# CHECK-BE: mftbu 2                         # encoding: [0x7c,0x4d,0x42,0xe6]
+# CHECK-LE: mftbu 2                         # encoding: [0xe6,0x42,0x4d,0x7c]
             mftbu 2
 
-# CHECK-BE: mtspr 284, 3                    # encoding: [0x7c,0x7c,0x43,0xa6]
-# CHECK-LE: mtspr 284, 3                    # encoding: [0xa6,0x43,0x7c,0x7c]
+# CHECK-BE: mttbl 3                         # encoding: [0x7c,0x7c,0x43,0xa6]
+# CHECK-LE: mttbl 3                         # encoding: [0xa6,0x43,0x7c,0x7c]
             mttbl 3
-# CHECK-BE: mtspr 285, 3                    # encoding: [0x7c,0x7d,0x43,0xa6]
-# CHECK-LE: mtspr 285, 3                    # encoding: [0xa6,0x43,0x7d,0x7c]
+# CHECK-BE: mttbu 3                         # encoding: [0x7c,0x7d,0x43,0xa6]
+# CHECK-LE: mttbu 3                         # encoding: [0xa6,0x43,0x7d,0x7c]
             mttbu 3
diff --git a/test/MC/PowerPC/ppc64-encoding-bookIII.s b/test/MC/PowerPC/ppc64-encoding-bookIII.s
index 9e784dbb5e9..d110d4ab6c6 100644
--- a/test/MC/PowerPC/ppc64-encoding-bookIII.s
+++ b/test/MC/PowerPC/ppc64-encoding-bookIII.s
@@ -1,8 +1,8 @@
 # RUN: llvm-mc -triple powerpc64-unknown-unknown --show-encoding %s | FileCheck -check-prefix=CHECK-BE %s
 # RUN: llvm-mc -triple powerpc64le-unknown-unknown --show-encoding %s | FileCheck -check-prefix=CHECK-LE %s
 
-# CHECK-BE: mtmsr 4, 0                       # encoding: [0x7c,0x80,0x01,0x24]
-# CHECK-LE: mtmsr 4, 0                       # encoding: [0x24,0x01,0x80,0x7c]
+# CHECK-BE: mtmsr 4                          # encoding: [0x7c,0x80,0x01,0x24]
+# CHECK-LE: mtmsr 4                          # encoding: [0x24,0x01,0x80,0x7c]
             mtmsr %r4
 
 # CHECK-BE: mtmsr 4, 1                       # encoding: [0x7c,0x81,0x01,0x24]
@@ -13,8 +13,8 @@
 # CHECK-LE: mfmsr 4                         # encoding: [0xa6,0x00,0x80,0x7c]
             mfmsr %r4
 
-# CHECK-BE: mtmsrd 4, 0                     # encoding: [0x7c,0x80,0x01,0x64]
-# CHECK-LE: mtmsrd 4, 0                     # encoding: [0x64,0x01,0x80,0x7c]
+# CHECK-BE: mtmsrd 4                        # encoding: [0x7c,0x80,0x01,0x64]
+# CHECK-LE: mtmsrd 4                        # encoding: [0x64,0x01,0x80,0x7c]
             mtmsrd %r4
 
 # CHECK-BE: mtmsrd 4, 1                     # encoding: [0x7c,0x81,0x01,0x64]
@@ -94,8 +94,8 @@
 # CHECK-LE: mtspr 22, 4                     # encoding: [0xa6,0x03,0x96,0x7c]
             mtdec %r4
 
-# CHECK-BE: mfspr 4, 287                    # encoding: [0x7c,0x9f,0x42,0xa6]
-# CHECK-LE: mfspr 4, 287                    # encoding: [0xa6,0x42,0x9f,0x7c]
+# CHECK-BE: mfpvr 4                         # encoding: [0x7c,0x9f,0x42,0xa6]
+# CHECK-LE: mfpvr 4                         # encoding: [0xa6,0x42,0x9f,0x7c]
             mfpvr %r4
 
 # CHECK-BE: mfspr 4, 25                     # encoding: [0x7c,0x99,0x02,0xa6]
@@ -146,12 +146,12 @@
 # CHECK-LE: tlbiel 4                        # encoding: [0x24,0x22,0x00,0x7c]
             tlbiel %r4
 
-# CHECK-BE: tlbie 4,0                       # encoding: [0x7c,0x00,0x22,0x64]
-# CHECK-LE: tlbie 4,0                       # encoding: [0x64,0x22,0x00,0x7c]
+# CHECK-BE: tlbie 4                         # encoding: [0x7c,0x00,0x22,0x64]
+# CHECK-LE: tlbie 4                         # encoding: [0x64,0x22,0x00,0x7c]
             tlbie %r4, 0
 
-# CHECK-BE: tlbie 4,0                       # encoding: [0x7c,0x00,0x22,0x64]
-# CHECK-LE: tlbie 4,0                       # encoding: [0x64,0x22,0x00,0x7c]
+# CHECK-BE: tlbie 4                         # encoding: [0x7c,0x00,0x22,0x64]
+# CHECK-LE: tlbie 4                         # encoding: [0x64,0x22,0x00,0x7c]
             tlbie %r4
 
 # CHECK-BE: rfi                             # encoding: [0x4c,0x00,0x00,0x64]
diff --git a/test/MC/PowerPC/ppc64-encoding-ext.s b/test/MC/PowerPC/ppc64-encoding-ext.s
index dca8a652674..143a30606f0 100644
--- a/test/MC/PowerPC/ppc64-encoding-ext.s
+++ b/test/MC/PowerPC/ppc64-encoding-ext.s
@@ -29,125 +29,125 @@
 # CHECK-LE: beqlr 7                         # encoding: [0x20,0x00,0x9e,0x4d]
             beqlr cr7
 
-# CHECK-BE: bclr 12, 0, 0                   # encoding: [0x4d,0x80,0x00,0x20]
-# CHECK-LE: bclr 12, 0, 0                   # encoding: [0x20,0x00,0x80,0x4d]
+# CHECK-BE: bclr 12, 0                      # encoding: [0x4d,0x80,0x00,0x20]
+# CHECK-LE: bclr 12, 0                      # encoding: [0x20,0x00,0x80,0x4d]
             btlr 4*cr0+lt
-# CHECK-BE: bclr 12, 1, 0                   # encoding: [0x4d,0x81,0x00,0x20]
-# CHECK-LE: bclr 12, 1, 0                   # encoding: [0x20,0x00,0x81,0x4d]
+# CHECK-BE: bclr 12, 1                      # encoding: [0x4d,0x81,0x00,0x20]
+# CHECK-LE: bclr 12, 1                      # encoding: [0x20,0x00,0x81,0x4d]
             btlr 4*cr0+gt
-# CHECK-BE: bclr 12, 2, 0                   # encoding: [0x4d,0x82,0x00,0x20]
-# CHECK-LE: bclr 12, 2, 0                   # encoding: [0x20,0x00,0x82,0x4d]
+# CHECK-BE: bclr 12, 2                      # encoding: [0x4d,0x82,0x00,0x20]
+# CHECK-LE: bclr 12, 2                      # encoding: [0x20,0x00,0x82,0x4d]
             btlr 4*cr0+eq
-# CHECK-BE: bclr 12, 3, 0                   # encoding: [0x4d,0x83,0x00,0x20]
-# CHECK-LE: bclr 12, 3, 0                   # encoding: [0x20,0x00,0x83,0x4d]
+# CHECK-BE: bclr 12, 3                      # encoding: [0x4d,0x83,0x00,0x20]
+# CHECK-LE: bclr 12, 3                      # encoding: [0x20,0x00,0x83,0x4d]
             btlr 4*cr0+so
-# CHECK-BE: bclr 12, 3, 0                   # encoding: [0x4d,0x83,0x00,0x20]
-# CHECK-LE: bclr 12, 3, 0                   # encoding: [0x20,0x00,0x83,0x4d]
+# CHECK-BE: bclr 12, 3                      # encoding: [0x4d,0x83,0x00,0x20]
+# CHECK-LE: bclr 12, 3                      # encoding: [0x20,0x00,0x83,0x4d]
             btlr 4*cr0+un
-# CHECK-BE: bclr 12, 4, 0                   # encoding: [0x4d,0x84,0x00,0x20]
-# CHECK-LE: bclr 12, 4, 0                   # encoding: [0x20,0x00,0x84,0x4d]
+# CHECK-BE: bclr 12, 4                      # encoding: [0x4d,0x84,0x00,0x20]
+# CHECK-LE: bclr 12, 4                      # encoding: [0x20,0x00,0x84,0x4d]
             btlr 4*cr1+lt
-# CHECK-BE: bclr 12, 5, 0                   # encoding: [0x4d,0x85,0x00,0x20]
-# CHECK-LE: bclr 12, 5, 0                   # encoding: [0x20,0x00,0x85,0x4d]
+# CHECK-BE: bclr 12, 5                      # encoding: [0x4d,0x85,0x00,0x20]
+# CHECK-LE: bclr 12, 5                      # encoding: [0x20,0x00,0x85,0x4d]
             btlr 4*cr1+gt
-# CHECK-BE: bclr 12, 6, 0                   # encoding: [0x4d,0x86,0x00,0x20]
-# CHECK-LE: bclr 12, 6, 0                   # encoding: [0x20,0x00,0x86,0x4d]
+# CHECK-BE: bclr 12, 6                      # encoding: [0x4d,0x86,0x00,0x20]
+# CHECK-LE: bclr 12, 6                      # encoding: [0x20,0x00,0x86,0x4d]
             btlr 4*cr1+eq
-# CHECK-BE: bclr 12, 7, 0                   # encoding: [0x4d,0x87,0x00,0x20]
-# CHECK-LE: bclr 12, 7, 0                   # encoding: [0x20,0x00,0x87,0x4d]
+# CHECK-BE: bclr 12, 7                      # encoding: [0x4d,0x87,0x00,0x20]
+# CHECK-LE: bclr 12, 7                      # encoding: [0x20,0x00,0x87,0x4d]
             btlr 4*cr1+so
-# CHECK-BE: bclr 12, 7, 0                   # encoding: [0x4d,0x87,0x00,0x20]
-# CHECK-LE: bclr 12, 7, 0                   # encoding: [0x20,0x00,0x87,0x4d]
+# CHECK-BE: bclr 12, 7                      # encoding: [0x4d,0x87,0x00,0x20]
+# CHECK-LE: bclr 12, 7                      # encoding: [0x20,0x00,0x87,0x4d]
             btlr 4*cr1+un
-# CHECK-BE: bclr 12, 8, 0                   # encoding: [0x4d,0x88,0x00,0x20]
-# CHECK-LE: bclr 12, 8, 0                   # encoding: [0x20,0x00,0x88,0x4d]
+# CHECK-BE: bclr 12, 8                      # encoding: [0x4d,0x88,0x00,0x20]
+# CHECK-LE: bclr 12, 8                      # encoding: [0x20,0x00,0x88,0x4d]
             btlr 4*cr2+lt
-# CHECK-BE: bclr 12, 9, 0                   # encoding: [0x4d,0x89,0x00,0x20]
-# CHECK-LE: bclr 12, 9, 0                   # encoding: [0x20,0x00,0x89,0x4d]
+# CHECK-BE: bclr 12, 9                      # encoding: [0x4d,0x89,0x00,0x20]
+# CHECK-LE: bclr 12, 9                      # encoding: [0x20,0x00,0x89,0x4d]
             btlr 4*cr2+gt
-# CHECK-BE: bclr 12, 10, 0                  # encoding: [0x4d,0x8a,0x00,0x20]
-# CHECK-LE: bclr 12, 10, 0                  # encoding: [0x20,0x00,0x8a,0x4d]
+# CHECK-BE: bclr 12, 10                     # encoding: [0x4d,0x8a,0x00,0x20]
+# CHECK-LE: bclr 12, 10                     # encoding: [0x20,0x00,0x8a,0x4d]
             btlr 4*cr2+eq
-# CHECK-BE: bclr 12, 11, 0                  # encoding: [0x4d,0x8b,0x00,0x20]
-# CHECK-LE: bclr 12, 11, 0                  # encoding: [0x20,0x00,0x8b,0x4d]
+# CHECK-BE: bclr 12, 11                     # encoding: [0x4d,0x8b,0x00,0x20]
+# CHECK-LE: bclr 12, 11                     # encoding: [0x20,0x00,0x8b,0x4d]
             btlr 4*cr2+so
-# CHECK-BE: bclr 12, 11, 0                  # encoding: [0x4d,0x8b,0x00,0x20]
-# CHECK-LE: bclr 12, 11, 0                  # encoding: [0x20,0x00,0x8b,0x4d]
+# CHECK-BE: bclr 12, 11                     # encoding: [0x4d,0x8b,0x00,0x20]
+# CHECK-LE: bclr 12, 11                     # encoding: [0x20,0x00,0x8b,0x4d]
             btlr 4*cr2+un
-# CHECK-BE: bclr 12, 12, 0                  # encoding: [0x4d,0x8c,0x00,0x20]
-# CHECK-LE: bclr 12, 12, 0                  # encoding: [0x20,0x00,0x8c,0x4d]
+# CHECK-BE: bclr 12, 12                     # encoding: [0x4d,0x8c,0x00,0x20]
+# CHECK-LE: bclr 12, 12                     # encoding: [0x20,0x00,0x8c,0x4d]
             btlr 4*cr3+lt
-# CHECK-BE: bclr 12, 13, 0                  # encoding: [0x4d,0x8d,0x00,0x20]
-# CHECK-LE: bclr 12, 13, 0                  # encoding: [0x20,0x00,0x8d,0x4d]
+# CHECK-BE: bclr 12, 13                     # encoding: [0x4d,0x8d,0x00,0x20]
+# CHECK-LE: bclr 12, 13                     # encoding: [0x20,0x00,0x8d,0x4d]
             btlr 4*cr3+gt
-# CHECK-BE: bclr 12, 14, 0                  # encoding: [0x4d,0x8e,0x00,0x20]
-# CHECK-LE: bclr 12, 14, 0                  # encoding: [0x20,0x00,0x8e,0x4d]
+# CHECK-BE: bclr 12, 14                     # encoding: [0x4d,0x8e,0x00,0x20]
+# CHECK-LE: bclr 12, 14                     # encoding: [0x20,0x00,0x8e,0x4d]
             btlr 4*cr3+eq
-# CHECK-BE: bclr 12, 15, 0                  # encoding: [0x4d,0x8f,0x00,0x20]
-# CHECK-LE: bclr 12, 15, 0                  # encoding: [0x20,0x00,0x8f,0x4d]
+# CHECK-BE: bclr 12, 15                     # encoding: [0x4d,0x8f,0x00,0x20]
+# CHECK-LE: bclr 12, 15                     # encoding: [0x20,0x00,0x8f,0x4d]
             btlr 4*cr3+so
-# CHECK-BE: bclr 12, 15, 0                  # encoding: [0x4d,0x8f,0x00,0x20]
-# CHECK-LE: bclr 12, 15, 0                  # encoding: [0x20,0x00,0x8f,0x4d]
+# CHECK-BE: bclr 12, 15                     # encoding: [0x4d,0x8f,0x00,0x20]
+# CHECK-LE: bclr 12, 15                     # encoding: [0x20,0x00,0x8f,0x4d]
             btlr 4*cr3+un
-# CHECK-BE: bclr 12, 16, 0                  # encoding: [0x4d,0x90,0x00,0x20]
-# CHECK-LE: bclr 12, 16, 0                  # encoding: [0x20,0x00,0x90,0x4d]
+# CHECK-BE: bclr 12, 16                     # encoding: [0x4d,0x90,0x00,0x20]
+# CHECK-LE: bclr 12, 16                     # encoding: [0x20,0x00,0x90,0x4d]
             btlr 4*cr4+lt
-# CHECK-BE: bclr 12, 17, 0                  # encoding: [0x4d,0x91,0x00,0x20]
-# CHECK-LE: bclr 12, 17, 0                  # encoding: [0x20,0x00,0x91,0x4d]
+# CHECK-BE: bclr 12, 17                     # encoding: [0x4d,0x91,0x00,0x20]
+# CHECK-LE: bclr 12, 17                     # encoding: [0x20,0x00,0x91,0x4d]
             btlr 4*cr4+gt
-# CHECK-BE: bclr 12, 18, 0                  # encoding: [0x4d,0x92,0x00,0x20]
-# CHECK-LE: bclr 12, 18, 0                  # encoding: [0x20,0x00,0x92,0x4d]
+# CHECK-BE: bclr 12, 18                     # encoding: [0x4d,0x92,0x00,0x20]
+# CHECK-LE: bclr 12, 18                     # encoding: [0x20,0x00,0x92,0x4d]
             btlr 4*cr4+eq
-# CHECK-BE: bclr 12, 19, 0                  # encoding: [0x4d,0x93,0x00,0x20]
-# CHECK-LE: bclr 12, 19, 0                  # encoding: [0x20,0x00,0x93,0x4d]
+# CHECK-BE: bclr 12, 19                     # encoding: [0x4d,0x93,0x00,0x20]
+# CHECK-LE: bclr 12, 19                     # encoding: [0x20,0x00,0x93,0x4d]
             btlr 4*cr4+so
-# CHECK-BE: bclr 12, 19, 0                  # encoding: [0x4d,0x93,0x00,0x20]
-# CHECK-LE: bclr 12, 19, 0                  # encoding: [0x20,0x00,0x93,0x4d]
+# CHECK-BE: bclr 12, 19                     # encoding: [0x4d,0x93,0x00,0x20]
+# CHECK-LE: bclr 12, 19                     # encoding: [0x20,0x00,0x93,0x4d]
             btlr 4*cr4+un
-# CHECK-BE: bclr 12, 20, 0                  # encoding: [0x4d,0x94,0x00,0x20]
-# CHECK-LE: bclr 12, 20, 0                  # encoding: [0x20,0x00,0x94,0x4d]
+# CHECK-BE: bclr 12, 20                     # encoding: [0x4d,0x94,0x00,0x20]
+# CHECK-LE: bclr 12, 20                     # encoding: [0x20,0x00,0x94,0x4d]
             btlr 4*cr5+lt
-# CHECK-BE: bclr 12, 21, 0                  # encoding: [0x4d,0x95,0x00,0x20]
-# CHECK-LE: bclr 12, 21, 0                  # encoding: [0x20,0x00,0x95,0x4d]
+# CHECK-BE: bclr 12, 21                     # encoding: [0x4d,0x95,0x00,0x20]
+# CHECK-LE: bclr 12, 21                     # encoding: [0x20,0x00,0x95,0x4d]
             btlr 4*cr5+gt
-# CHECK-BE: bclr 12, 22, 0                  # encoding: [0x4d,0x96,0x00,0x20]
-# CHECK-LE: bclr 12, 22, 0                  # encoding: [0x20,0x00,0x96,0x4d]
+# CHECK-BE: bclr 12, 22                     # encoding: [0x4d,0x96,0x00,0x20]
+# CHECK-LE: bclr 12, 22                     # encoding: [0x20,0x00,0x96,0x4d]
             btlr 4*cr5+eq
-# CHECK-BE: bclr 12, 23, 0                  # encoding: [0x4d,0x97,0x00,0x20]
-# CHECK-LE: bclr 12, 23, 0                  # encoding: [0x20,0x00,0x97,0x4d]
+# CHECK-BE: bclr 12, 23                     # encoding: [0x4d,0x97,0x00,0x20]
+# CHECK-LE: bclr 12, 23                     # encoding: [0x20,0x00,0x97,0x4d]
             btlr 4*cr5+so
-# CHECK-BE: bclr 12, 23, 0                  # encoding: [0x4d,0x97,0x00,0x20]
-# CHECK-LE: bclr 12, 23, 0                  # encoding: [0x20,0x00,0x97,0x4d]
+# CHECK-BE: bclr 12, 23                     # encoding: [0x4d,0x97,0x00,0x20]
+# CHECK-LE: bclr 12, 23                     # encoding: [0x20,0x00,0x97,0x4d]
             btlr 4*cr5+un
-# CHECK-BE: bclr 12, 24, 0                  # encoding: [0x4d,0x98,0x00,0x20]
-# CHECK-LE: bclr 12, 24, 0                  # encoding: [0x20,0x00,0x98,0x4d]
+# CHECK-BE: bclr 12, 24                     # encoding: [0x4d,0x98,0x00,0x20]
+# CHECK-LE: bclr 12, 24                     # encoding: [0x20,0x00,0x98,0x4d]
             btlr 4*cr6+lt
-# CHECK-BE: bclr 12, 25, 0                  # encoding: [0x4d,0x99,0x00,0x20]
-# CHECK-LE: bclr 12, 25, 0                  # encoding: [0x20,0x00,0x99,0x4d]
+# CHECK-BE: bclr 12, 25                     # encoding: [0x4d,0x99,0x00,0x20]
+# CHECK-LE: bclr 12, 25                     # encoding: [0x20,0x00,0x99,0x4d]
             btlr 4*cr6+gt
-# CHECK-BE: bclr 12, 26, 0                  # encoding: [0x4d,0x9a,0x00,0x20]
-# CHECK-LE: bclr 12, 26, 0                  # encoding: [0x20,0x00,0x9a,0x4d]
+# CHECK-BE: bclr 12, 26                     # encoding: [0x4d,0x9a,0x00,0x20]
+# CHECK-LE: bclr 12, 26                     # encoding: [0x20,0x00,0x9a,0x4d]
             btlr 4*cr6+eq
-# CHECK-BE: bclr 12, 27, 0                  # encoding: [0x4d,0x9b,0x00,0x20]
-# CHECK-LE: bclr 12, 27, 0                  # encoding: [0x20,0x00,0x9b,0x4d]
+# CHECK-BE: bclr 12, 27                     # encoding: [0x4d,0x9b,0x00,0x20]
+# CHECK-LE: bclr 12, 27                     # encoding: [0x20,0x00,0x9b,0x4d]
             btlr 4*cr6+so
-# CHECK-BE: bclr 12, 27, 0                  # encoding: [0x4d,0x9b,0x00,0x20]
-# CHECK-LE: bclr 12, 27, 0                  # encoding: [0x20,0x00,0x9b,0x4d]
+# CHECK-BE: bclr 12, 27                     # encoding: [0x4d,0x9b,0x00,0x20]
+# CHECK-LE: bclr 12, 27                     # encoding: [0x20,0x00,0x9b,0x4d]
             btlr 4*cr6+un
-# CHECK-BE: bclr 12, 28, 0                  # encoding: [0x4d,0x9c,0x00,0x20]
-# CHECK-LE: bclr 12, 28, 0                  # encoding: [0x20,0x00,0x9c,0x4d]
+# CHECK-BE: bclr 12, 28                     # encoding: [0x4d,0x9c,0x00,0x20]
+# CHECK-LE: bclr 12, 28                     # encoding: [0x20,0x00,0x9c,0x4d]
             btlr 4*cr7+lt
-# CHECK-BE: bclr 12, 29, 0                  # encoding: [0x4d,0x9d,0x00,0x20]
-# CHECK-LE: bclr 12, 29, 0                  # encoding: [0x20,0x00,0x9d,0x4d]
+# CHECK-BE: bclr 12, 29                     # encoding: [0x4d,0x9d,0x00,0x20]
+# CHECK-LE: bclr 12, 29                     # encoding: [0x20,0x00,0x9d,0x4d]
             btlr 4*cr7+gt
-# CHECK-BE: bclr 12, 30, 0                  # encoding: [0x4d,0x9e,0x00,0x20]
-# CHECK-LE: bclr 12, 30, 0                  # encoding: [0x20,0x00,0x9e,0x4d]
+# CHECK-BE: bclr 12, 30                     # encoding: [0x4d,0x9e,0x00,0x20]
+# CHECK-LE: bclr 12, 30                     # encoding: [0x20,0x00,0x9e,0x4d]
             btlr 4*cr7+eq
-# CHECK-BE: bclr 12, 31, 0                  # encoding: [0x4d,0x9f,0x00,0x20]
-# CHECK-LE: bclr 12, 31, 0                  # encoding: [0x20,0x00,0x9f,0x4d]
+# CHECK-BE: bclr 12, 31                     # encoding: [0x4d,0x9f,0x00,0x20]
+# CHECK-LE: bclr 12, 31                     # encoding: [0x20,0x00,0x9f,0x4d]
             btlr 4*cr7+so
-# CHECK-BE: bclr 12, 31, 0                  # encoding: [0x4d,0x9f,0x00,0x20]
-# CHECK-LE: bclr 12, 31, 0                  # encoding: [0x20,0x00,0x9f,0x4d]
+# CHECK-BE: bclr 12, 31                     # encoding: [0x4d,0x9f,0x00,0x20]
+# CHECK-LE: bclr 12, 31                     # encoding: [0x20,0x00,0x9f,0x4d]
             btlr 4*cr7+un
 
 # Branch mnemonics
@@ -165,202 +165,202 @@
 # CHECK-LE: bctrl                           # encoding: [0x21,0x04,0x80,0x4e]
             bctrl
 
-# CHECK-BE: bc 12, 2, target                # encoding: [0x41,0x82,A,0bAAAAAA00]
-# CHECK-LE: bc 12, 2, target                # encoding: [0bAAAAAA00,A,0x82,0x41]
+# CHECK-BE: bt 2, target                    # encoding: [0x41,0x82,A,0bAAAAAA00]
+# CHECK-LE: bt 2, target                    # encoding: [0bAAAAAA00,A,0x82,0x41]
 # CHECK-BE-NEXT:                            #   fixup A - offset: 0, value: target, kind: fixup_ppc_brcond14
 # CHECK-LE-NEXT:                            #   fixup A - offset: 0, value: target, kind: fixup_ppc_brcond14
             bt 2, target
-# CHECK-BE: bca 12, 2, target               # encoding: [0x41,0x82,A,0bAAAAAA10]
-# CHECK-LE: bca 12, 2, target               # encoding: [0bAAAAAA10,A,0x82,0x41]
+# CHECK-BE: bta 2, target                   # encoding: [0x41,0x82,A,0bAAAAAA10]
+# CHECK-LE: bta 2, target                   # encoding: [0bAAAAAA10,A,0x82,0x41]
 # CHECK-BE-NEXT:                            #   fixup A - offset: 0, value: target, kind: fixup_ppc_brcond14abs
 # CHECK-LE-NEXT:                            #   fixup A - offset: 0, value: target, kind: fixup_ppc_brcond14abs
             bta 2, target
-# CHECK-BE: bclr 12, 2, 0                   # encoding: [0x4d,0x82,0x00,0x20]
-# CHECK-LE: bclr 12, 2, 0                   # encoding: [0x20,0x00,0x82,0x4d]
+# CHECK-BE: bclr 12, 2                      # encoding: [0x4d,0x82,0x00,0x20]
+# CHECK-LE: bclr 12, 2                      # encoding: [0x20,0x00,0x82,0x4d]
             btlr 2
-# CHECK-BE: bcctr 12, 2, 0                  # encoding: [0x4d,0x82,0x04,0x20]
-# CHECK-LE: bcctr 12, 2, 0                  # encoding: [0x20,0x04,0x82,0x4d]
+# CHECK-BE: bcctr 12, 2                     # encoding: [0x4d,0x82,0x04,0x20]
+# CHECK-LE: bcctr 12, 2                     # encoding: [0x20,0x04,0x82,0x4d]
             btctr 2
-# CHECK-BE: bcl 12, 2, target               # encoding: [0x41,0x82,A,0bAAAAAA01]
-# CHECK-LE: bcl 12, 2, target               # encoding: [0bAAAAAA01,A,0x82,0x41]
+# CHECK-BE: btl 2, target                   # encoding: [0x41,0x82,A,0bAAAAAA01]
+# CHECK-LE: btl 2, target                   # encoding: [0bAAAAAA01,A,0x82,0x41]
 # CHECK-BE-NEXT:                            #   fixup A - offset: 0, value: target, kind: fixup_ppc_brcond14
 # CHECK-LE-NEXT:                            #   fixup A - offset: 0, value: target, kind: fixup_ppc_brcond14
             btl 2, target
-# CHECK-BE: bcla 12, 2, target              # encoding: [0x41,0x82,A,0bAAAAAA11]
-# CHECK-LE: bcla 12, 2, target              # encoding: [0bAAAAAA11,A,0x82,0x41]
+# CHECK-BE: btla 2, target                  # encoding: [0x41,0x82,A,0bAAAAAA11]
+# CHECK-LE: btla 2, target                  # encoding: [0bAAAAAA11,A,0x82,0x41]
 # CHECK-BE-NEXT:                            #   fixup A - offset: 0, value: target, kind: fixup_ppc_brcond14abs
 # CHECK-LE-NEXT:                            #   fixup A - offset: 0, value: target, kind: fixup_ppc_brcond14abs
             btla 2, target
-# CHECK-BE: bclrl 12, 2, 0                  # encoding: [0x4d,0x82,0x00,0x21]
-# CHECK-LE: bclrl 12, 2, 0                  # encoding: [0x21,0x00,0x82,0x4d]
+# CHECK-BE: bclrl 12, 2                     # encoding: [0x4d,0x82,0x00,0x21]
+# CHECK-LE: bclrl 12, 2                     # encoding: [0x21,0x00,0x82,0x4d]
             btlrl 2
-# CHECK-BE: bcctrl 12, 2, 0                 # encoding: [0x4d,0x82,0x04,0x21]
-# CHECK-LE: bcctrl 12, 2, 0                 # encoding: [0x21,0x04,0x82,0x4d]
+# CHECK-BE: bcctrl 12, 2                    # encoding: [0x4d,0x82,0x04,0x21]
+# CHECK-LE: bcctrl 12, 2                    # encoding: [0x21,0x04,0x82,0x4d]
             btctrl 2
 
-# CHECK-BE: bc 15, 2, target                # encoding: [0x41,0xe2,A,0bAAAAAA00]
-# CHECK-LE: bc 15, 2, target                # encoding: [0bAAAAAA00,A,0xe2,0x41]
+# CHECK-BE: bt+ 2, target                   # encoding: [0x41,0xe2,A,0bAAAAAA00]
+# CHECK-LE: bt+ 2, target                   # encoding: [0bAAAAAA00,A,0xe2,0x41]
 # CHECK-BE-NEXT:                            #   fixup A - offset: 0, value: target, kind: fixup_ppc_brcond14
 # CHECK-LE-NEXT:                            #   fixup A - offset: 0, value: target, kind: fixup_ppc_brcond14
             bt+ 2, target
-# CHECK-BE: bca 15, 2, target               # encoding: [0x41,0xe2,A,0bAAAAAA10]
-# CHECK-LE: bca 15, 2, target               # encoding: [0bAAAAAA10,A,0xe2,0x41]
+# CHECK-BE: bta+ 2, target                  # encoding: [0x41,0xe2,A,0bAAAAAA10]
+# CHECK-LE: bta+ 2, target                  # encoding: [0bAAAAAA10,A,0xe2,0x41]
 # CHECK-BE-NEXT:                            #   fixup A - offset: 0, value: target, kind: fixup_ppc_brcond14abs
 # CHECK-LE-NEXT:                            #   fixup A - offset: 0, value: target, kind: fixup_ppc_brcond14abs
             bta+ 2, target
-# CHECK-BE: bclr 15, 2, 0                   # encoding: [0x4d,0xe2,0x00,0x20]
-# CHECK-LE: bclr 15, 2, 0                   # encoding: [0x20,0x00,0xe2,0x4d]
+# CHECK-BE: bclr 15, 2                      # encoding: [0x4d,0xe2,0x00,0x20]
+# CHECK-LE: bclr 15, 2                      # encoding: [0x20,0x00,0xe2,0x4d]
             btlr+ 2
-# CHECK-BE: bcctr 15, 2, 0                  # encoding: [0x4d,0xe2,0x04,0x20]
-# CHECK-LE: bcctr 15, 2, 0                  # encoding: [0x20,0x04,0xe2,0x4d]
+# CHECK-BE: bcctr 15, 2                     # encoding: [0x4d,0xe2,0x04,0x20]
+# CHECK-LE: bcctr 15, 2                     # encoding: [0x20,0x04,0xe2,0x4d]
             btctr+ 2
-# CHECK-BE: bcl 15, 2, target               # encoding: [0x41,0xe2,A,0bAAAAAA01]
-# CHECK-LE: bcl 15, 2, target               # encoding: [0bAAAAAA01,A,0xe2,0x41]
+# CHECK-BE: btl+ 2, target                  # encoding: [0x41,0xe2,A,0bAAAAAA01]
+# CHECK-LE: btl+ 2, target                  # encoding: [0bAAAAAA01,A,0xe2,0x41]
 # CHECK-BE-NEXT:                            #   fixup A - offset: 0, value: target, kind: fixup_ppc_brcond14
 # CHECK-LE-NEXT:                            #   fixup A - offset: 0, value: target, kind: fixup_ppc_brcond14
             btl+ 2, target
-# CHECK-BE: bcla 15, 2, target              # encoding: [0x41,0xe2,A,0bAAAAAA11]
-# CHECK-LE: bcla 15, 2, target              # encoding: [0bAAAAAA11,A,0xe2,0x41]
+# CHECK-BE: btla+ 2, target                 # encoding: [0x41,0xe2,A,0bAAAAAA11]
+# CHECK-LE: btla+ 2, target                 # encoding: [0bAAAAAA11,A,0xe2,0x41]
 # CHECK-BE-NEXT:                            #   fixup A - offset: 0, value: target, kind: fixup_ppc_brcond14abs
 # CHECK-LE-NEXT:                            #   fixup A - offset: 0, value: target, kind: fixup_ppc_brcond14abs
             btla+ 2, target
-# CHECK-BE: bclrl 15, 2, 0                  # encoding: [0x4d,0xe2,0x00,0x21]
-# CHECK-LE: bclrl 15, 2, 0                  # encoding: [0x21,0x00,0xe2,0x4d]
+# CHECK-BE: bclrl 15, 2                     # encoding: [0x4d,0xe2,0x00,0x21]
+# CHECK-LE: bclrl 15, 2                     # encoding: [0x21,0x00,0xe2,0x4d]
             btlrl+ 2
-# CHECK-BE: bcctrl 15, 2, 0                 # encoding: [0x4d,0xe2,0x04,0x21]
-# CHECK-LE: bcctrl 15, 2, 0                 # encoding: [0x21,0x04,0xe2,0x4d]
+# CHECK-BE: bcctrl 15, 2                    # encoding: [0x4d,0xe2,0x04,0x21]
+# CHECK-LE: bcctrl 15, 2                    # encoding: [0x21,0x04,0xe2,0x4d]
             btctrl+ 2
 
-# CHECK-BE: bc 14, 2, target                # encoding: [0x41,0xc2,A,0bAAAAAA00]
-# CHECK-LE: bc 14, 2, target                # encoding: [0bAAAAAA00,A,0xc2,0x41]
+# CHECK-BE: bt- 2, target                   # encoding: [0x41,0xc2,A,0bAAAAAA00]
+# CHECK-LE: bt- 2, target                   # encoding: [0bAAAAAA00,A,0xc2,0x41]
 # CHECK-BE-NEXT:                            #   fixup A - offset: 0, value: target, kind: fixup_ppc_brcond14
 # CHECK-LE-NEXT:                            #   fixup A - offset: 0, value: target, kind: fixup_ppc_brcond14
             bt- 2, target
-# CHECK-BE: bca 14, 2, target               # encoding: [0x41,0xc2,A,0bAAAAAA10]
-# CHECK-LE: bca 14, 2, target               # encoding: [0bAAAAAA10,A,0xc2,0x41]
+# CHECK-BE: bta- 2, target                  # encoding: [0x41,0xc2,A,0bAAAAAA10]
+# CHECK-LE: bta- 2, target                  # encoding: [0bAAAAAA10,A,0xc2,0x41]
 # CHECK-BE-NEXT:                            #   fixup A - offset: 0, value: target, kind: fixup_ppc_brcond14abs
 # CHECK-LE-NEXT:                            #   fixup A - offset: 0, value: target, kind: fixup_ppc_brcond14abs
             bta- 2, target
-# CHECK-BE: bclr 14, 2, 0                   # encoding: [0x4d,0xc2,0x00,0x20]
-# CHECK-LE: bclr 14, 2, 0                   # encoding: [0x20,0x00,0xc2,0x4d]
+# CHECK-BE: bclr 14, 2                      # encoding: [0x4d,0xc2,0x00,0x20]
+# CHECK-LE: bclr 14, 2                      # encoding: [0x20,0x00,0xc2,0x4d]
             btlr- 2
-# CHECK-BE: bcctr 14, 2, 0                  # encoding: [0x4d,0xc2,0x04,0x20]
-# CHECK-LE: bcctr 14, 2, 0                  # encoding: [0x20,0x04,0xc2,0x4d]
+# CHECK-BE: bcctr 14, 2                     # encoding: [0x4d,0xc2,0x04,0x20]
+# CHECK-LE: bcctr 14, 2                     # encoding: [0x20,0x04,0xc2,0x4d]
             btctr- 2
-# CHECK-BE: bcl 14, 2, target               # encoding: [0x41,0xc2,A,0bAAAAAA01]
-# CHECK-LE: bcl 14, 2, target               # encoding: [0bAAAAAA01,A,0xc2,0x41]
+# CHECK-BE: btl- 2, target                  # encoding: [0x41,0xc2,A,0bAAAAAA01]
+# CHECK-LE: btl- 2, target                  # encoding: [0bAAAAAA01,A,0xc2,0x41]
 # CHECK-BE-NEXT:                            #   fixup A - offset: 0, value: target, kind: fixup_ppc_brcond14
 # CHECK-LE-NEXT:                            #   fixup A - offset: 0, value: target, kind: fixup_ppc_brcond14
             btl- 2, target
-# CHECK-BE: bcla 14, 2, target              # encoding: [0x41,0xc2,A,0bAAAAAA11]
-# CHECK-LE: bcla 14, 2, target              # encoding: [0bAAAAAA11,A,0xc2,0x41]
+# CHECK-BE: btla- 2, target                 # encoding: [0x41,0xc2,A,0bAAAAAA11]
+# CHECK-LE: btla- 2, target                 # encoding: [0bAAAAAA11,A,0xc2,0x41]
 # CHECK-BE-NEXT:                            #   fixup A - offset: 0, value: target, kind: fixup_ppc_brcond14abs
 # CHECK-LE-NEXT:                            #   fixup A - offset: 0, value: target, kind: fixup_ppc_brcond14abs
             btla- 2, target
-# CHECK-BE: bclrl 14, 2, 0                  # encoding: [0x4d,0xc2,0x00,0x21]
-# CHECK-LE: bclrl 14, 2, 0                  # encoding: [0x21,0x00,0xc2,0x4d]
+# CHECK-BE: bclrl 14, 2                     # encoding: [0x4d,0xc2,0x00,0x21]
+# CHECK-LE: bclrl 14, 2                     # encoding: [0x21,0x00,0xc2,0x4d]
             btlrl- 2
-# CHECK-BE: bcctrl 14, 2, 0                 # encoding: [0x4d,0xc2,0x04,0x21]
-# CHECK-LE: bcctrl 14, 2, 0                 # encoding: [0x21,0x04,0xc2,0x4d]
+# CHECK-BE: bcctrl 14, 2                    # encoding: [0x4d,0xc2,0x04,0x21]
+# CHECK-LE: bcctrl 14, 2                    # encoding: [0x21,0x04,0xc2,0x4d]
             btctrl- 2
 
-# CHECK-BE: bc 4, 2, target                 # encoding: [0x40,0x82,A,0bAAAAAA00]
-# CHECK-LE: bc 4, 2, target                 # encoding: [0bAAAAAA00,A,0x82,0x40]
+# CHECK-BE: bf 2, target                    # encoding: [0x40,0x82,A,0bAAAAAA00]
+# CHECK-LE: bf 2, target                    # encoding: [0bAAAAAA00,A,0x82,0x40]
 # CHECK-BE-NEXT:                            #   fixup A - offset: 0, value: target, kind: fixup_ppc_brcond14
 # CHECK-LE-NEXT:                            #   fixup A - offset: 0, value: target, kind: fixup_ppc_brcond14
             bf 2, target
-# CHECK-BE: bca 4, 2, target                # encoding: [0x40,0x82,A,0bAAAAAA10]
-# CHECK-LE: bca 4, 2, target                # encoding: [0bAAAAAA10,A,0x82,0x40]
+# CHECK-BE: bfa 2, target                   # encoding: [0x40,0x82,A,0bAAAAAA10]
+# CHECK-LE: bfa 2, target                   # encoding: [0bAAAAAA10,A,0x82,0x40]
 # CHECK-BE-NEXT:                            #   fixup A - offset: 0, value: target, kind: fixup_ppc_brcond14abs
 # CHECK-LE-NEXT:                            #   fixup A - offset: 0, value: target, kind: fixup_ppc_brcond14abs
             bfa 2, target
-# CHECK-BE: bclr 4, 2, 0                    # encoding: [0x4c,0x82,0x00,0x20]
-# CHECK-LE: bclr 4, 2, 0                    # encoding: [0x20,0x00,0x82,0x4c]
+# CHECK-BE: bclr 4, 2                       # encoding: [0x4c,0x82,0x00,0x20]
+# CHECK-LE: bclr 4, 2                       # encoding: [0x20,0x00,0x82,0x4c]
             bflr 2
-# CHECK-BE: bcctr 4, 2, 0                   # encoding: [0x4c,0x82,0x04,0x20]
-# CHECK-LE: bcctr 4, 2, 0                   # encoding: [0x20,0x04,0x82,0x4c]
+# CHECK-BE: bcctr 4, 2                      # encoding: [0x4c,0x82,0x04,0x20]
+# CHECK-LE: bcctr 4, 2                      # encoding: [0x20,0x04,0x82,0x4c]
             bfctr 2
-# CHECK-BE: bcl 4, 2, target                # encoding: [0x40,0x82,A,0bAAAAAA01]
-# CHECK-LE: bcl 4, 2, target                # encoding: [0bAAAAAA01,A,0x82,0x40]
+# CHECK-BE: bfl 2, target                   # encoding: [0x40,0x82,A,0bAAAAAA01]
+# CHECK-LE: bfl 2, target                   # encoding: [0bAAAAAA01,A,0x82,0x40]
 # CHECK-BE-NEXT:                            #   fixup A - offset: 0, value: target, kind: fixup_ppc_brcond14
 # CHECK-LE-NEXT:                            #   fixup A - offset: 0, value: target, kind: fixup_ppc_brcond14
             bfl 2, target
-# CHECK-BE: bcla 4, 2, target               # encoding: [0x40,0x82,A,0bAAAAAA11]
-# CHECK-LE: bcla 4, 2, target               # encoding: [0bAAAAAA11,A,0x82,0x40]
+# CHECK-BE: bfla 2, target                  # encoding: [0x40,0x82,A,0bAAAAAA11]
+# CHECK-LE: bfla 2, target                  # encoding: [0bAAAAAA11,A,0x82,0x40]
 # CHECK-BE-NEXT:                            #   fixup A - offset: 0, value: target, kind: fixup_ppc_brcond14abs
 # CHECK-LE-NEXT:                            #   fixup A - offset: 0, value: target, kind: fixup_ppc_brcond14abs
             bfla 2, target
-# CHECK-BE: bclrl 4, 2, 0                   # encoding: [0x4c,0x82,0x00,0x21]
-# CHECK-LE: bclrl 4, 2, 0                   # encoding: [0x21,0x00,0x82,0x4c]
+# CHECK-BE: bclrl 4, 2                      # encoding: [0x4c,0x82,0x00,0x21]
+# CHECK-LE: bclrl 4, 2                      # encoding: [0x21,0x00,0x82,0x4c]
             bflrl 2
-# CHECK-BE: bcctrl 4, 2, 0                  # encoding: [0x4c,0x82,0x04,0x21]
-# CHECK-LE: bcctrl 4, 2, 0                  # encoding: [0x21,0x04,0x82,0x4c]
+# CHECK-BE: bcctrl 4, 2                     # encoding: [0x4c,0x82,0x04,0x21]
+# CHECK-LE: bcctrl 4, 2                     # encoding: [0x21,0x04,0x82,0x4c]
             bfctrl 2
 
-# CHECK-BE: bc 7, 2, target                 # encoding: [0x40,0xe2,A,0bAAAAAA00]
-# CHECK-LE: bc 7, 2, target                 # encoding: [0bAAAAAA00,A,0xe2,0x40]
+# CHECK-BE: bf+ 2, target                   # encoding: [0x40,0xe2,A,0bAAAAAA00]
+# CHECK-LE: bf+ 2, target                   # encoding: [0bAAAAAA00,A,0xe2,0x40]
 # CHECK-BE-NEXT:                            #   fixup A - offset: 0, value: target, kind: fixup_ppc_brcond14
 # CHECK-LE-NEXT:                            #   fixup A - offset: 0, value: target, kind: fixup_ppc_brcond14
             bf+ 2, target
-# CHECK-BE: bca 7, 2, target                # encoding: [0x40,0xe2,A,0bAAAAAA10]
-# CHECK-LE: bca 7, 2, target                # encoding: [0bAAAAAA10,A,0xe2,0x40]
+# CHECK-BE: bfa+ 2, target                  # encoding: [0x40,0xe2,A,0bAAAAAA10]
+# CHECK-LE: bfa+ 2, target                  # encoding: [0bAAAAAA10,A,0xe2,0x40]
 # CHECK-BE-NEXT:                            #   fixup A - offset: 0, value: target, kind: fixup_ppc_brcond14abs
 # CHECK-LE-NEXT:                            #   fixup A - offset: 0, value: target, kind: fixup_ppc_brcond14abs
             bfa+ 2, target
-# CHECK-BE: bclr 7, 2, 0                    # encoding: [0x4c,0xe2,0x00,0x20]
-# CHECK-LE: bclr 7, 2, 0                    # encoding: [0x20,0x00,0xe2,0x4c]
+# CHECK-BE: bclr 7, 2                       # encoding: [0x4c,0xe2,0x00,0x20]
+# CHECK-LE: bclr 7, 2                       # encoding: [0x20,0x00,0xe2,0x4c]
             bflr+ 2
-# CHECK-BE: bcctr 7, 2, 0                   # encoding: [0x4c,0xe2,0x04,0x20]
-# CHECK-LE: bcctr 7, 2, 0                   # encoding: [0x20,0x04,0xe2,0x4c]
+# CHECK-BE: bcctr 7, 2                      # encoding: [0x4c,0xe2,0x04,0x20]
+# CHECK-LE: bcctr 7, 2                      # encoding: [0x20,0x04,0xe2,0x4c]
             bfctr+ 2
-# CHECK-BE: bcl 7, 2, target                # encoding: [0x40,0xe2,A,0bAAAAAA01]
-# CHECK-LE: bcl 7, 2, target                # encoding: [0bAAAAAA01,A,0xe2,0x40]
+# CHECK-BE: bfl+ 2, target                  # encoding: [0x40,0xe2,A,0bAAAAAA01]
+# CHECK-LE: bfl+ 2, target                  # encoding: [0bAAAAAA01,A,0xe2,0x40]
 # CHECK-BE-NEXT:                            #   fixup A - offset: 0, value: target, kind: fixup_ppc_brcond14
 # CHECK-LE-NEXT:                            #   fixup A - offset: 0, value: target, kind: fixup_ppc_brcond14
             bfl+ 2, target
-# CHECK-BE: bcla 7, 2, target               # encoding: [0x40,0xe2,A,0bAAAAAA11]
-# CHECK-LE: bcla 7, 2, target               # encoding: [0bAAAAAA11,A,0xe2,0x40]
+# CHECK-BE: bfla+ 2, target                 # encoding: [0x40,0xe2,A,0bAAAAAA11]
+# CHECK-LE: bfla+ 2, target                 # encoding: [0bAAAAAA11,A,0xe2,0x40]
 # CHECK-BE-NEXT:                            #   fixup A - offset: 0, value: target, kind: fixup_ppc_brcond14abs
 # CHECK-LE-NEXT:                            #   fixup A - offset: 0, value: target, kind: fixup_ppc_brcond14abs
             bfla+ 2, target
-# CHECK-BE: bclrl 7, 2, 0                   # encoding: [0x4c,0xe2,0x00,0x21]
-# CHECK-LE: bclrl 7, 2, 0                   # encoding: [0x21,0x00,0xe2,0x4c]
+# CHECK-BE: bclrl 7, 2                      # encoding: [0x4c,0xe2,0x00,0x21]
+# CHECK-LE: bclrl 7, 2                      # encoding: [0x21,0x00,0xe2,0x4c]
             bflrl+ 2
-# CHECK-BE: bcctrl 7, 2, 0                  # encoding: [0x4c,0xe2,0x04,0x21]
-# CHECK-LE: bcctrl 7, 2, 0                  # encoding: [0x21,0x04,0xe2,0x4c]
+# CHECK-BE: bcctrl 7, 2                     # encoding: [0x4c,0xe2,0x04,0x21]
+# CHECK-LE: bcctrl 7, 2                     # encoding: [0x21,0x04,0xe2,0x4c]
             bfctrl+ 2
 
-# CHECK-BE: bc 6, 2, target                 # encoding: [0x40,0xc2,A,0bAAAAAA00]
-# CHECK-LE: bc 6, 2, target                 # encoding: [0bAAAAAA00,A,0xc2,0x40]
+# CHECK-BE: bf- 2, target                   # encoding: [0x40,0xc2,A,0bAAAAAA00]
+# CHECK-LE: bf- 2, target                   # encoding: [0bAAAAAA00,A,0xc2,0x40]
 # CHECK-BE-NEXT:                            #   fixup A - offset: 0, value: target, kind: fixup_ppc_brcond14
 # CHECK-LE-NEXT:                            #   fixup A - offset: 0, value: target, kind: fixup_ppc_brcond14
             bf- 2, target
-# CHECK-BE: bca 6, 2, target                # encoding: [0x40,0xc2,A,0bAAAAAA10]
-# CHECK-LE: bca 6, 2, target                # encoding: [0bAAAAAA10,A,0xc2,0x40]
+# CHECK-BE: bfa- 2, target                  # encoding: [0x40,0xc2,A,0bAAAAAA10]
+# CHECK-LE: bfa- 2, target                  # encoding: [0bAAAAAA10,A,0xc2,0x40]
 # CHECK-BE-NEXT:                            #   fixup A - offset: 0, value: target, kind: fixup_ppc_brcond14abs
 # CHECK-LE-NEXT:                            #   fixup A - offset: 0, value: target, kind: fixup_ppc_brcond14abs
             bfa- 2, target
-# CHECK-BE: bclr 6, 2, 0                    # encoding: [0x4c,0xc2,0x00,0x20]
-# CHECK-LE: bclr 6, 2, 0                    # encoding: [0x20,0x00,0xc2,0x4c]
+# CHECK-BE: bclr 6, 2                       # encoding: [0x4c,0xc2,0x00,0x20]
+# CHECK-LE: bclr 6, 2                       # encoding: [0x20,0x00,0xc2,0x4c]
             bflr- 2
-# CHECK-BE: bcctr 6, 2, 0                   # encoding: [0x4c,0xc2,0x04,0x20]
-# CHECK-LE: bcctr 6, 2, 0                   # encoding: [0x20,0x04,0xc2,0x4c]
+# CHECK-BE: bcctr 6, 2                      # encoding: [0x4c,0xc2,0x04,0x20]
+# CHECK-LE: bcctr 6, 2                      # encoding: [0x20,0x04,0xc2,0x4c]
             bfctr- 2
-# CHECK-BE: bcl 6, 2, target                # encoding: [0x40,0xc2,A,0bAAAAAA01]
-# CHECK-LE: bcl 6, 2, target                # encoding: [0bAAAAAA01,A,0xc2,0x40]
+# CHECK-BE: bfl- 2, target                  # encoding: [0x40,0xc2,A,0bAAAAAA01]
+# CHECK-LE: bfl- 2, target                  # encoding: [0bAAAAAA01,A,0xc2,0x40]
 # CHECK-BE-NEXT:                            #   fixup A - offset: 0, value: target, kind: fixup_ppc_brcond14
 # CHECK-LE-NEXT:                            #   fixup A - offset: 0, value: target, kind: fixup_ppc_brcond14
             bfl- 2, target
-# CHECK-BE: bcla 6, 2, target               # encoding: [0x40,0xc2,A,0bAAAAAA11]
-# CHECK-LE: bcla 6, 2, target               # encoding: [0bAAAAAA11,A,0xc2,0x40]
+# CHECK-BE: bfla- 2, target                 # encoding: [0x40,0xc2,A,0bAAAAAA11]
+# CHECK-LE: bfla- 2, target                 # encoding: [0bAAAAAA11,A,0xc2,0x40]
 # CHECK-BE-NEXT:                            #   fixup A - offset: 0, value: target, kind: fixup_ppc_brcond14abs
 # CHECK-LE-NEXT:                            #   fixup A - offset: 0, value: target, kind: fixup_ppc_brcond14abs
             bfla- 2, target
-# CHECK-BE: bclrl 6, 2, 0                   # encoding: [0x4c,0xc2,0x00,0x21]
-# CHECK-LE: bclrl 6, 2, 0                   # encoding: [0x21,0x00,0xc2,0x4c]
+# CHECK-BE: bclrl 6, 2                      # encoding: [0x4c,0xc2,0x00,0x21]
+# CHECK-LE: bclrl 6, 2                      # encoding: [0x21,0x00,0xc2,0x4c]
             bflrl- 2
-# CHECK-BE: bcctrl 6, 2, 0                  # encoding: [0x4c,0xc2,0x04,0x21]
-# CHECK-LE: bcctrl 6, 2, 0                  # encoding: [0x21,0x04,0xc2,0x4c]
+# CHECK-BE: bcctrl 6, 2                     # encoding: [0x4c,0xc2,0x04,0x21]
+# CHECK-LE: bcctrl 6, 2                     # encoding: [0x21,0x04,0xc2,0x4c]
             bfctrl- 2
 
 # CHECK-BE: bdnz target                     # encoding: [0x42,0x00,A,0bAAAAAA00]
@@ -444,58 +444,58 @@
 # CHECK-LE: bdnzlrl-                        # encoding: [0x21,0x00,0x00,0x4f]
             bdnzlrl-
 
-# CHECK-BE: bc 8, 2, target                 # encoding: [0x41,0x02,A,0bAAAAAA00]
-# CHECK-LE: bc 8, 2, target                 # encoding: [0bAAAAAA00,A,0x02,0x41]
+# CHECK-BE: bdnzt 2, target                 # encoding: [0x41,0x02,A,0bAAAAAA00]
+# CHECK-LE: bdnzt 2, target                 # encoding: [0bAAAAAA00,A,0x02,0x41]
 # CHECK-BE-NEXT:                            #   fixup A - offset: 0, value: target, kind: fixup_ppc_brcond14
 # CHECK-LE-NEXT:                            #   fixup A - offset: 0, value: target, kind: fixup_ppc_brcond14
             bdnzt 2, target
-# CHECK-BE: bca 8, 2, target                # encoding: [0x41,0x02,A,0bAAAAAA10]
-# CHECK-LE: bca 8, 2, target                # encoding: [0bAAAAAA10,A,0x02,0x41]
+# CHECK-BE: bdnzta 2, target                # encoding: [0x41,0x02,A,0bAAAAAA10]
+# CHECK-LE: bdnzta 2, target                # encoding: [0bAAAAAA10,A,0x02,0x41]
 # CHECK-BE-NEXT:                            #   fixup A - offset: 0, value: target, kind: fixup_ppc_brcond14abs
 # CHECK-LE-NEXT:                            #   fixup A - offset: 0, value: target, kind: fixup_ppc_brcond14abs
             bdnzta 2, target
-# CHECK-BE: bclr 8, 2, 0                    # encoding: [0x4d,0x02,0x00,0x20]
-# CHECK-LE: bclr 8, 2, 0                    # encoding: [0x20,0x00,0x02,0x4d]
+# CHECK-BE: bclr 8, 2                       # encoding: [0x4d,0x02,0x00,0x20]
+# CHECK-LE: bclr 8, 2                       # encoding: [0x20,0x00,0x02,0x4d]
             bdnztlr 2
-# CHECK-BE: bcl 8, 2, target                # encoding: [0x41,0x02,A,0bAAAAAA01]
-# CHECK-LE: bcl 8, 2, target                # encoding: [0bAAAAAA01,A,0x02,0x41]
+# CHECK-BE: bdnztl 2, target                # encoding: [0x41,0x02,A,0bAAAAAA01]
+# CHECK-LE: bdnztl 2, target                # encoding: [0bAAAAAA01,A,0x02,0x41]
 # CHECK-BE-NEXT:                            #   fixup A - offset: 0, value: target, kind: fixup_ppc_brcond14
 # CHECK-LE-NEXT:                            #   fixup A - offset: 0, value: target, kind: fixup_ppc_brcond14
             bdnztl 2, target
-# CHECK-BE: bcla 8, 2, target               # encoding: [0x41,0x02,A,0bAAAAAA11]
-# CHECK-LE: bcla 8, 2, target               # encoding: [0bAAAAAA11,A,0x02,0x41]
+# CHECK-BE: bdnztla 2, target               # encoding: [0x41,0x02,A,0bAAAAAA11]
+# CHECK-LE: bdnztla 2, target               # encoding: [0bAAAAAA11,A,0x02,0x41]
 # CHECK-BE-NEXT:                            #   fixup A - offset: 0, value: target, kind: fixup_ppc_brcond14abs
 # CHECK-LE-NEXT:                            #   fixup A - offset: 0, value: target, kind: fixup_ppc_brcond14abs
             bdnztla 2, target
-# CHECK-BE: bclrl 8, 2, 0                   # encoding: [0x4d,0x02,0x00,0x21]
-# CHECK-LE: bclrl 8, 2, 0                   # encoding: [0x21,0x00,0x02,0x4d]
+# CHECK-BE: bclrl 8, 2                      # encoding: [0x4d,0x02,0x00,0x21]
+# CHECK-LE: bclrl 8, 2                      # encoding: [0x21,0x00,0x02,0x4d]
             bdnztlrl 2
 
-# CHECK-BE: bc 0, 2, target                 # encoding: [0x40,0x02,A,0bAAAAAA00]
-# CHECK-LE: bc 0, 2, target                 # encoding: [0bAAAAAA00,A,0x02,0x40]
+# CHECK-BE: bdnzf 2, target                 # encoding: [0x40,0x02,A,0bAAAAAA00]
+# CHECK-LE: bdnzf 2, target                 # encoding: [0bAAAAAA00,A,0x02,0x40]
 # CHECK-BE-NEXT:                            #   fixup A - offset: 0, value: target, kind: fixup_ppc_brcond14
 # CHECK-LE-NEXT:                            #   fixup A - offset: 0, value: target, kind: fixup_ppc_brcond14
             bdnzf 2, target
-# CHECK-BE: bca 0, 2, target                # encoding: [0x40,0x02,A,0bAAAAAA10]
-# CHECK-LE: bca 0, 2, target                # encoding: [0bAAAAAA10,A,0x02,0x40]
+# CHECK-BE: bdnzfa 2, target                # encoding: [0x40,0x02,A,0bAAAAAA10]
+# CHECK-LE: bdnzfa 2, target                # encoding: [0bAAAAAA10,A,0x02,0x40]
 # CHECK-BE-NEXT:                            #   fixup A - offset: 0, value: target, kind: fixup_ppc_brcond14abs
 # CHECK-LE-NEXT:                            #   fixup A - offset: 0, value: target, kind: fixup_ppc_brcond14abs
             bdnzfa 2, target
-# CHECK-BE: bclr 0, 2, 0                    # encoding: [0x4c,0x02,0x00,0x20]
-# CHECK-LE: bclr 0, 2, 0                    # encoding: [0x20,0x00,0x02,0x4c]
+# CHECK-BE: bclr 0, 2                       # encoding: [0x4c,0x02,0x00,0x20]
+# CHECK-LE: bclr 0, 2                       # encoding: [0x20,0x00,0x02,0x4c]
             bdnzflr 2
-# CHECK-BE: bcl 0, 2, target                # encoding: [0x40,0x02,A,0bAAAAAA01]
-# CHECK-LE: bcl 0, 2, target                # encoding: [0bAAAAAA01,A,0x02,0x40]
+# CHECK-BE: bdnzfl 2, target                # encoding: [0x40,0x02,A,0bAAAAAA01]
+# CHECK-LE: bdnzfl 2, target                # encoding: [0bAAAAAA01,A,0x02,0x40]
 # CHECK-BE-NEXT:                            #   fixup A - offset: 0, value: target, kind: fixup_ppc_brcond14
 # CHECK-LE-NEXT:                            #   fixup A - offset: 0, value: target, kind: fixup_ppc_brcond14
             bdnzfl 2, target
-# CHECK-BE: bcla 0, 2, target               # encoding: [0x40,0x02,A,0bAAAAAA11]
-# CHECK-LE: bcla 0, 2, target               # encoding: [0bAAAAAA11,A,0x02,0x40]
+# CHECK-BE: bdnzfla 2, target               # encoding: [0x40,0x02,A,0bAAAAAA11]
+# CHECK-LE: bdnzfla 2, target               # encoding: [0bAAAAAA11,A,0x02,0x40]
 # CHECK-BE-NEXT:                            #   fixup A - offset: 0, value: target, kind: fixup_ppc_brcond14abs
 # CHECK-LE-NEXT:                            #   fixup A - offset: 0, value: target, kind: fixup_ppc_brcond14abs
             bdnzfla 2, target
-# CHECK-BE: bclrl 0, 2, 0                   # encoding: [0x4c,0x02,0x00,0x21]
-# CHECK-LE: bclrl 0, 2, 0                   # encoding: [0x21,0x00,0x02,0x4c]
+# CHECK-BE: bclrl 0, 2                      # encoding: [0x4c,0x02,0x00,0x21]
+# CHECK-LE: bclrl 0, 2                      # encoding: [0x21,0x00,0x02,0x4c]
             bdnzflrl 2
 
 # CHECK-BE: bdz target                      # encoding: [0x42,0x40,A,0bAAAAAA00]
@@ -579,58 +579,58 @@
 # CHECK-LE: bdzlrl-                         # encoding: [0x21,0x00,0x40,0x4f]
             bdzlrl-
 
-# CHECK-BE: bc 10, 2, target                # encoding: [0x41,0x42,A,0bAAAAAA00]
-# CHECK-LE: bc 10, 2, target                # encoding: [0bAAAAAA00,A,0x42,0x41]
+# CHECK-BE: bdzt 2, target                  # encoding: [0x41,0x42,A,0bAAAAAA00]
+# CHECK-LE: bdzt 2, target                  # encoding: [0bAAAAAA00,A,0x42,0x41]
 # CHECK-BE-NEXT:                            #   fixup A - offset: 0, value: target, kind: fixup_ppc_brcond14
 # CHECK-LE-NEXT:                            #   fixup A - offset: 0, value: target, kind: fixup_ppc_brcond14
             bdzt 2, target
-# CHECK-BE: bca 10, 2, target               # encoding: [0x41,0x42,A,0bAAAAAA10]
-# CHECK-LE: bca 10, 2, target               # encoding: [0bAAAAAA10,A,0x42,0x41]
+# CHECK-BE: bdzta 2, target                 # encoding: [0x41,0x42,A,0bAAAAAA10]
+# CHECK-LE: bdzta 2, target                 # encoding: [0bAAAAAA10,A,0x42,0x41]
 # CHECK-BE-NEXT:                            #   fixup A - offset: 0, value: target, kind: fixup_ppc_brcond14abs
 # CHECK-LE-NEXT:                            #   fixup A - offset: 0, value: target, kind: fixup_ppc_brcond14abs
             bdzta 2, target
-# CHECK-BE: bclr 10, 2, 0                   # encoding: [0x4d,0x42,0x00,0x20]
-# CHECK-LE: bclr 10, 2, 0                   # encoding: [0x20,0x00,0x42,0x4d]
+# CHECK-BE: bclr 10, 2                      # encoding: [0x4d,0x42,0x00,0x20]
+# CHECK-LE: bclr 10, 2                      # encoding: [0x20,0x00,0x42,0x4d]
             bdztlr 2
-# CHECK-BE: bcl 10, 2, target               # encoding: [0x41,0x42,A,0bAAAAAA01]
-# CHECK-LE: bcl 10, 2, target               # encoding: [0bAAAAAA01,A,0x42,0x41]
+# CHECK-BE: bdztl 2, target                 # encoding: [0x41,0x42,A,0bAAAAAA01]
+# CHECK-LE: bdztl 2, target                 # encoding: [0bAAAAAA01,A,0x42,0x41]
 # CHECK-BE-NEXT:                            #   fixup A - offset: 0, value: target, kind: fixup_ppc_brcond14
 # CHECK-LE-NEXT:                            #   fixup A - offset: 0, value: target, kind: fixup_ppc_brcond14
             bdztl 2, target
-# CHECK-BE: bcla 10, 2, target              # encoding: [0x41,0x42,A,0bAAAAAA11]
-# CHECK-LE: bcla 10, 2, target              # encoding: [0bAAAAAA11,A,0x42,0x41]
+# CHECK-BE: bdztla 2, target                # encoding: [0x41,0x42,A,0bAAAAAA11]
+# CHECK-LE: bdztla 2, target                # encoding: [0bAAAAAA11,A,0x42,0x41]
 # CHECK-BE-NEXT:                            #   fixup A - offset: 0, value: target, kind: fixup_ppc_brcond14abs
 # CHECK-LE-NEXT:                            #   fixup A - offset: 0, value: target, kind: fixup_ppc_brcond14abs
             bdztla 2, target
-# CHECK-BE: bclrl 10, 2, 0                  # encoding: [0x4d,0x42,0x00,0x21]
-# CHECK-LE: bclrl 10, 2, 0                  # encoding: [0x21,0x00,0x42,0x4d]
+# CHECK-BE: bclrl 10, 2                     # encoding: [0x4d,0x42,0x00,0x21]
+# CHECK-LE: bclrl 10, 2                     # encoding: [0x21,0x00,0x42,0x4d]
             bdztlrl 2
 
-# CHECK-BE: bc 2, 2, target                 # encoding: [0x40,0x42,A,0bAAAAAA00]
-# CHECK-LE: bc 2, 2, target                 # encoding: [0bAAAAAA00,A,0x42,0x40]
+# CHECK-BE: bdzf 2, target                  # encoding: [0x40,0x42,A,0bAAAAAA00]
+# CHECK-LE: bdzf 2, target                  # encoding: [0bAAAAAA00,A,0x42,0x40]
 # CHECK-BE-NEXT:                            #   fixup A - offset: 0, value: target, kind: fixup_ppc_brcond14
 # CHECK-LE-NEXT:                            #   fixup A - offset: 0, value: target, kind: fixup_ppc_brcond14
             bdzf 2, target
-# CHECK-BE: bca 2, 2, target                # encoding: [0x40,0x42,A,0bAAAAAA10]
-# CHECK-LE: bca 2, 2, target                # encoding: [0bAAAAAA10,A,0x42,0x40]
+# CHECK-BE: bdzfa 2, target                 # encoding: [0x40,0x42,A,0bAAAAAA10]
+# CHECK-LE: bdzfa 2, target                 # encoding: [0bAAAAAA10,A,0x42,0x40]
 # CHECK-BE-NEXT:                            #   fixup A - offset: 0, value: target, kind: fixup_ppc_brcond14abs
 # CHECK-LE-NEXT:                            #   fixup A - offset: 0, value: target, kind: fixup_ppc_brcond14abs
             bdzfa 2, target
-# CHECK-BE: bclr 2, 2, 0                    # encoding: [0x4c,0x42,0x00,0x20]
-# CHECK-LE: bclr 2, 2, 0                    # encoding: [0x20,0x00,0x42,0x4c]
+# CHECK-BE: bclr 2, 2                       # encoding: [0x4c,0x42,0x00,0x20]
+# CHECK-LE: bclr 2, 2                       # encoding: [0x20,0x00,0x42,0x4c]
             bdzflr 2
-# CHECK-BE: bcl 2, 2, target                # encoding: [0x40,0x42,A,0bAAAAAA01]
-# CHECK-LE: bcl 2, 2, target                # encoding: [0bAAAAAA01,A,0x42,0x40]
+# CHECK-BE: bdzfl 2, target                 # encoding: [0x40,0x42,A,0bAAAAAA01]
+# CHECK-LE: bdzfl 2, target                 # encoding: [0bAAAAAA01,A,0x42,0x40]
 # CHECK-BE-NEXT:                            #   fixup A - offset: 0, value: target, kind: fixup_ppc_brcond14
 # CHECK-LE-NEXT:                            #   fixup A - offset: 0, value: target, kind: fixup_ppc_brcond14
             bdzfl 2, target
-# CHECK-BE: bcla 2, 2, target               # encoding: [0x40,0x42,A,0bAAAAAA11]
-# CHECK-LE: bcla 2, 2, target               # encoding: [0bAAAAAA11,A,0x42,0x40]
+# CHECK-BE: bdzfla 2, target                # encoding: [0x40,0x42,A,0bAAAAAA11]
+# CHECK-LE: bdzfla 2, target                # encoding: [0bAAAAAA11,A,0x42,0x40]
 # CHECK-BE-NEXT:                            #   fixup A - offset: 0, value: target, kind: fixup_ppc_brcond14abs
 # CHECK-LE-NEXT:                            #   fixup A - offset: 0, value: target, kind: fixup_ppc_brcond14abs
             bdzfla 2, target
-# CHECK-BE: bclrl 2, 2, 0                   # encoding: [0x4c,0x42,0x00,0x21]
-# CHECK-LE: bclrl 2, 2, 0                   # encoding: [0x21,0x00,0x42,0x4c]
+# CHECK-BE: bclrl 2, 2                      # encoding: [0x4c,0x42,0x00,0x21]
+# CHECK-LE: bclrl 2, 2                      # encoding: [0x21,0x00,0x42,0x4c]
             bdzflrl 2
 
 # CHECK-BE: blt 2, target                   # encoding: [0x41,0x88,A,0bAAAAAA00]
@@ -2975,17 +2975,17 @@
 
 # Condition register logical mnemonics
 
-# CHECK-BE: creqv 2, 2, 2                   # encoding: [0x4c,0x42,0x12,0x42]
-# CHECK-LE: creqv 2, 2, 2                   # encoding: [0x42,0x12,0x42,0x4c]
+# CHECK-BE: crset 2                         # encoding: [0x4c,0x42,0x12,0x42]
+# CHECK-LE: crset 2                         # encoding: [0x42,0x12,0x42,0x4c]
             crset 2
-# CHECK-BE: crxor 2, 2, 2                   # encoding: [0x4c,0x42,0x11,0x82]
-# CHECK-LE: crxor 2, 2, 2                   # encoding: [0x82,0x11,0x42,0x4c]
+# CHECK-BE: crclr 2                         # encoding: [0x4c,0x42,0x11,0x82]
+# CHECK-LE: crclr 2                         # encoding: [0x82,0x11,0x42,0x4c]
             crclr 2
-# CHECK-BE: cror 2, 3, 3                    # encoding: [0x4c,0x43,0x1b,0x82]
-# CHECK-LE: cror 2, 3, 3                    # encoding: [0x82,0x1b,0x43,0x4c]
+# CHECK-BE: crmove 2, 3                     # encoding: [0x4c,0x43,0x1b,0x82]
+# CHECK-LE: crmove 2, 3                     # encoding: [0x82,0x1b,0x43,0x4c]
             crmove 2, 3
-# CHECK-BE: crnor 2, 3, 3                   # encoding: [0x4c,0x43,0x18,0x42]
-# CHECK-LE: crnor 2, 3, 3                   # encoding: [0x42,0x18,0x43,0x4c]
+# CHECK-BE: crnot 2, 3                      # encoding: [0x4c,0x43,0x18,0x42]
+# CHECK-LE: crnot 2, 3                      # encoding: [0x42,0x18,0x43,0x4c]
             crnot 2, 3
 
 # Subtract mnemonics
@@ -3003,17 +3003,17 @@
 # CHECK-LE: addic. 2, 3, -128               # encoding: [0x80,0xff,0x43,0x34]
             subic. 2, 3, 128
 
-# CHECK-BE: subf 2, 4, 3                    # encoding: [0x7c,0x44,0x18,0x50]
-# CHECK-LE: subf 2, 4, 3                    # encoding: [0x50,0x18,0x44,0x7c]
+# CHECK-BE: sub 2, 3, 4                     # encoding: [0x7c,0x44,0x18,0x50]
+# CHECK-LE: sub 2, 3, 4                     # encoding: [0x50,0x18,0x44,0x7c]
             sub 2, 3, 4
-# CHECK-BE: subf. 2, 4, 3                   # encoding: [0x7c,0x44,0x18,0x51]
-# CHECK-LE: subf. 2, 4, 3                   # encoding: [0x51,0x18,0x44,0x7c]
+# CHECK-BE: sub. 2, 3, 4                    # encoding: [0x7c,0x44,0x18,0x51]
+# CHECK-LE: sub. 2, 3, 4                    # encoding: [0x51,0x18,0x44,0x7c]
             sub. 2, 3, 4
-# CHECK-BE: subfc 2, 4, 3                   # encoding: [0x7c,0x44,0x18,0x10]
-# CHECK-LE: subfc 2, 4, 3                   # encoding: [0x10,0x18,0x44,0x7c]
+# CHECK-BE: subc 2, 3, 4                    # encoding: [0x7c,0x44,0x18,0x10]
+# CHECK-LE: subc 2, 3, 4                    # encoding: [0x10,0x18,0x44,0x7c]
             subc 2, 3, 4
-# CHECK-BE: subfc. 2, 4, 3                  # encoding: [0x7c,0x44,0x18,0x11]
-# CHECK-LE: subfc. 2, 4, 3                  # encoding: [0x11,0x18,0x44,0x7c]
+# CHECK-BE: subc. 2, 3, 4                   # encoding: [0x7c,0x44,0x18,0x11]
+# CHECK-LE: subc. 2, 3, 4                   # encoding: [0x11,0x18,0x44,0x7c]
             subc. 2, 3, 4
 
 # Compare mnemonics
@@ -3021,66 +3021,66 @@
 # CHECK-BE: cmpdi 2, 3, 128                 # encoding: [0x2d,0x23,0x00,0x80]
 # CHECK-LE: cmpdi 2, 3, 128                 # encoding: [0x80,0x00,0x23,0x2d]
             cmpdi 2, 3, 128
-# CHECK-BE: cmpdi 0, 3, 128                 # encoding: [0x2c,0x23,0x00,0x80]
-# CHECK-LE: cmpdi 0, 3, 128                 # encoding: [0x80,0x00,0x23,0x2c]
+# CHECK-BE: cmpdi 3, 128                    # encoding: [0x2c,0x23,0x00,0x80]
+# CHECK-LE: cmpdi 3, 128                    # encoding: [0x80,0x00,0x23,0x2c]
             cmpdi 3, 128
 # CHECK-BE: cmpd 2, 3, 4                    # encoding: [0x7d,0x23,0x20,0x00]
 # CHECK-LE: cmpd 2, 3, 4                    # encoding: [0x00,0x20,0x23,0x7d]
             cmpd 2, 3, 4
-# CHECK-BE: cmpd 0, 3, 4                    # encoding: [0x7c,0x23,0x20,0x00]
-# CHECK-LE: cmpd 0, 3, 4                    # encoding: [0x00,0x20,0x23,0x7c]
+# CHECK-BE: cmpd 3, 4                       # encoding: [0x7c,0x23,0x20,0x00]
+# CHECK-LE: cmpd 3, 4                       # encoding: [0x00,0x20,0x23,0x7c]
             cmpd 3, 4
 # CHECK-BE: cmpldi 2, 3, 128                # encoding: [0x29,0x23,0x00,0x80]
 # CHECK-LE: cmpldi 2, 3, 128                # encoding: [0x80,0x00,0x23,0x29]
             cmpldi 2, 3, 128
-# CHECK-BE: cmpldi 0, 3, 128                # encoding: [0x28,0x23,0x00,0x80]
-# CHECK-LE: cmpldi 0, 3, 128                # encoding: [0x80,0x00,0x23,0x28]
+# CHECK-BE: cmpldi 3, 128                   # encoding: [0x28,0x23,0x00,0x80]
+# CHECK-LE: cmpldi 3, 128                   # encoding: [0x80,0x00,0x23,0x28]
             cmpldi 3, 128
 # CHECK-BE: cmpld 2, 3, 4                   # encoding: [0x7d,0x23,0x20,0x40]
 # CHECK-LE: cmpld 2, 3, 4                   # encoding: [0x40,0x20,0x23,0x7d]
             cmpld 2, 3, 4
-# CHECK-BE: cmpld 0, 3, 4                   # encoding: [0x7c,0x23,0x20,0x40]
-# CHECK-LE: cmpld 0, 3, 4                   # encoding: [0x40,0x20,0x23,0x7c]
+# CHECK-BE: cmpld 3, 4                      # encoding: [0x7c,0x23,0x20,0x40]
+# CHECK-LE: cmpld 3, 4                      # encoding: [0x40,0x20,0x23,0x7c]
             cmpld 3, 4
 
 # CHECK-BE: cmpwi 2, 3, 128                 # encoding: [0x2d,0x03,0x00,0x80]
 # CHECK-LE: cmpwi 2, 3, 128                 # encoding: [0x80,0x00,0x03,0x2d]
             cmpwi 2, 3, 128
-# CHECK-BE: cmpwi 0, 3, 128                 # encoding: [0x2c,0x03,0x00,0x80]
-# CHECK-LE: cmpwi 0, 3, 128                 # encoding: [0x80,0x00,0x03,0x2c]
+# CHECK-BE: cmpwi 3, 128                    # encoding: [0x2c,0x03,0x00,0x80]
+# CHECK-LE: cmpwi 3, 128                    # encoding: [0x80,0x00,0x03,0x2c]
             cmpwi 3, 128
 # CHECK-BE: cmpw 2, 3, 4                    # encoding: [0x7d,0x03,0x20,0x00]
 # CHECK-LE: cmpw 2, 3, 4                    # encoding: [0x00,0x20,0x03,0x7d]
             cmpw 2, 3, 4
-# CHECK-BE: cmpw 0, 3, 4                    # encoding: [0x7c,0x03,0x20,0x00]
-# CHECK-LE: cmpw 0, 3, 4                    # encoding: [0x00,0x20,0x03,0x7c]
+# CHECK-BE: cmpw 3, 4                       # encoding: [0x7c,0x03,0x20,0x00]
+# CHECK-LE: cmpw 3, 4                       # encoding: [0x00,0x20,0x03,0x7c]
             cmpw 3, 4
 # CHECK-BE: cmplwi 2, 3, 128                # encoding: [0x29,0x03,0x00,0x80]
 # CHECK-LE: cmplwi 2, 3, 128                # encoding: [0x80,0x00,0x03,0x29]
             cmplwi 2, 3, 128
-# CHECK-BE: cmplwi 0, 3, 128                # encoding: [0x28,0x03,0x00,0x80]
-# CHECK-LE: cmplwi 0, 3, 128                # encoding: [0x80,0x00,0x03,0x28]
+# CHECK-BE: cmplwi 3, 128                   # encoding: [0x28,0x03,0x00,0x80]
+# CHECK-LE: cmplwi 3, 128                   # encoding: [0x80,0x00,0x03,0x28]
             cmplwi 3, 128
 # CHECK-BE: cmplw 2, 3, 4                   # encoding: [0x7d,0x03,0x20,0x40]
 # CHECK-LE: cmplw 2, 3, 4                   # encoding: [0x40,0x20,0x03,0x7d]
             cmplw 2, 3, 4
-# CHECK-BE: cmplw 0, 3, 4                   # encoding: [0x7c,0x03,0x20,0x40]
-# CHECK-LE: cmplw 0, 3, 4                   # encoding: [0x40,0x20,0x03,0x7c]
+# CHECK-BE: cmplw 3, 4                      # encoding: [0x7c,0x03,0x20,0x40]
+# CHECK-LE: cmplw 3, 4                      # encoding: [0x40,0x20,0x03,0x7c]
             cmplw 3, 4
 
 # Trap mnemonics
 
-# CHECK-BE: twi 16, 3, 4                    # encoding: [0x0e,0x03,0x00,0x04]
-# CHECK-LE: twi 16, 3, 4                    # encoding: [0x04,0x00,0x03,0x0e]
+# CHECK-BE: twlti 3, 4                      # encoding: [0x0e,0x03,0x00,0x04]
+# CHECK-LE: twlti 3, 4                      # encoding: [0x04,0x00,0x03,0x0e]
             twlti 3, 4
-# CHECK-BE: tw 16, 3, 4                     # encoding: [0x7e,0x03,0x20,0x08]
-# CHECK-LE: tw 16, 3, 4                     # encoding: [0x08,0x20,0x03,0x7e]
+# CHECK-BE: twlt 3, 4                       # encoding: [0x7e,0x03,0x20,0x08]
+# CHECK-LE: twlt 3, 4                       # encoding: [0x08,0x20,0x03,0x7e]
             twlt 3, 4
-# CHECK-BE: tdi 16, 3, 4                    # encoding: [0x0a,0x03,0x00,0x04]
-# CHECK-LE: tdi 16, 3, 4                    # encoding: [0x04,0x00,0x03,0x0a]
+# CHECK-BE: tdlti 3, 4                      # encoding: [0x0a,0x03,0x00,0x04]
+# CHECK-LE: tdlti 3, 4                      # encoding: [0x04,0x00,0x03,0x0a]
             tdlti 3, 4
-# CHECK-BE: td 16, 3, 4                     # encoding: [0x7e,0x03,0x20,0x88]
-# CHECK-LE: td 16, 3, 4                     # encoding: [0x88,0x20,0x03,0x7e]
+# CHECK-BE: tdlt 3, 4                       # encoding: [0x7e,0x03,0x20,0x88]
+# CHECK-LE: tdlt 3, 4                       # encoding: [0x88,0x20,0x03,0x7e]
             tdlt 3, 4
 
 # CHECK-BE: twi 20, 3, 4                    # encoding: [0x0e,0x83,0x00,0x04]
@@ -3096,17 +3096,17 @@
 # CHECK-LE: td 20, 3, 4                     # encoding: [0x88,0x20,0x83,0x7e]
             tdle 3, 4
 
-# CHECK-BE: twi 4, 3, 4                     # encoding: [0x0c,0x83,0x00,0x04]
-# CHECK-LE: twi 4, 3, 4                     # encoding: [0x04,0x00,0x83,0x0c]
+# CHECK-BE: tweqi 3, 4                      # encoding: [0x0c,0x83,0x00,0x04]
+# CHECK-LE: tweqi 3, 4                      # encoding: [0x04,0x00,0x83,0x0c]
             tweqi 3, 4
-# CHECK-BE: tw 4, 3, 4                      # encoding: [0x7c,0x83,0x20,0x08]
-# CHECK-LE: tw 4, 3, 4                      # encoding: [0x08,0x20,0x83,0x7c]
+# CHECK-BE: tweq 3, 4                       # encoding: [0x7c,0x83,0x20,0x08]
+# CHECK-LE: tweq 3, 4                       # encoding: [0x08,0x20,0x83,0x7c]
             tweq 3, 4
-# CHECK-BE: tdi 4, 3, 4                     # encoding: [0x08,0x83,0x00,0x04]
-# CHECK-LE: tdi 4, 3, 4                     # encoding: [0x04,0x00,0x83,0x08]
+# CHECK-BE: tdeqi 3, 4                      # encoding: [0x08,0x83,0x00,0x04]
+# CHECK-LE: tdeqi 3, 4                      # encoding: [0x04,0x00,0x83,0x08]
             tdeqi 3, 4
-# CHECK-BE: td 4, 3, 4                      # encoding: [0x7c,0x83,0x20,0x88]
-# CHECK-LE: td 4, 3, 4                      # encoding: [0x88,0x20,0x83,0x7c]
+# CHECK-BE: tdeq 3, 4                       # encoding: [0x7c,0x83,0x20,0x88]
+# CHECK-LE: tdeq 3, 4                       # encoding: [0x88,0x20,0x83,0x7c]
             tdeq 3, 4
 
 # CHECK-BE: twi 12, 3, 4                    # encoding: [0x0d,0x83,0x00,0x04]
@@ -3122,17 +3122,17 @@
 # CHECK-LE: td 12, 3, 4                     # encoding: [0x88,0x20,0x83,0x7d]
             tdge 3, 4
 
-# CHECK-BE: twi 8, 3, 4                     # encoding: [0x0d,0x03,0x00,0x04]
-# CHECK-LE: twi 8, 3, 4                     # encoding: [0x04,0x00,0x03,0x0d]
+# CHECK-BE: twgti 3, 4                      # encoding: [0x0d,0x03,0x00,0x04]
+# CHECK-LE: twgti 3, 4                      # encoding: [0x04,0x00,0x03,0x0d]
             twgti 3, 4
-# CHECK-BE: tw 8, 3, 4                      # encoding: [0x7d,0x03,0x20,0x08]
-# CHECK-LE: tw 8, 3, 4                      # encoding: [0x08,0x20,0x03,0x7d]
+# CHECK-BE: twgt 3, 4                       # encoding: [0x7d,0x03,0x20,0x08]
+# CHECK-LE: twgt 3, 4                       # encoding: [0x08,0x20,0x03,0x7d]
             twgt 3, 4
-# CHECK-BE: tdi 8, 3, 4                     # encoding: [0x09,0x03,0x00,0x04]
-# CHECK-LE: tdi 8, 3, 4                     # encoding: [0x04,0x00,0x03,0x09]
+# CHECK-BE: tdgti 3, 4                      # encoding: [0x09,0x03,0x00,0x04]
+# CHECK-LE: tdgti 3, 4                      # encoding: [0x04,0x00,0x03,0x09]
             tdgti 3, 4
-# CHECK-BE: td 8, 3, 4                      # encoding: [0x7d,0x03,0x20,0x88]
-# CHECK-LE: td 8, 3, 4                      # encoding: [0x88,0x20,0x03,0x7d]
+# CHECK-BE: tdgt 3, 4                       # encoding: [0x7d,0x03,0x20,0x88]
+# CHECK-LE: tdgt 3, 4                       # encoding: [0x88,0x20,0x03,0x7d]
             tdgt 3, 4
 
 # CHECK-BE: twi 12, 3, 4                    # encoding: [0x0d,0x83,0x00,0x04]
@@ -3148,17 +3148,17 @@
 # CHECK-LE: td 12, 3, 4                     # encoding: [0x88,0x20,0x83,0x7d]
             tdnl 3, 4
 
-# CHECK-BE: twi 24, 3, 4                    # encoding: [0x0f,0x03,0x00,0x04]
-# CHECK-LE: twi 24, 3, 4                    # encoding: [0x04,0x00,0x03,0x0f]
+# CHECK-BE: twnei 3, 4                      # encoding: [0x0f,0x03,0x00,0x04]
+# CHECK-LE: twnei 3, 4                      # encoding: [0x04,0x00,0x03,0x0f]
             twnei 3, 4
-# CHECK-BE: tw 24, 3, 4                     # encoding: [0x7f,0x03,0x20,0x08]
-# CHECK-LE: tw 24, 3, 4                     # encoding: [0x08,0x20,0x03,0x7f]
+# CHECK-BE: twne 3, 4                       # encoding: [0x7f,0x03,0x20,0x08]
+# CHECK-LE: twne 3, 4                       # encoding: [0x08,0x20,0x03,0x7f]
             twne 3, 4
-# CHECK-BE: tdi 24, 3, 4                    # encoding: [0x0b,0x03,0x00,0x04]
-# CHECK-LE: tdi 24, 3, 4                    # encoding: [0x04,0x00,0x03,0x0b]
+# CHECK-BE: tdnei 3, 4                      # encoding: [0x0b,0x03,0x00,0x04]
+# CHECK-LE: tdnei 3, 4                      # encoding: [0x04,0x00,0x03,0x0b]
             tdnei 3, 4
-# CHECK-BE: td 24, 3, 4                     # encoding: [0x7f,0x03,0x20,0x88]
-# CHECK-LE: td 24, 3, 4                     # encoding: [0x88,0x20,0x03,0x7f]
+# CHECK-BE: tdne 3, 4                       # encoding: [0x7f,0x03,0x20,0x88]
+# CHECK-LE: tdne 3, 4                       # encoding: [0x88,0x20,0x03,0x7f]
             tdne 3, 4
 
 # CHECK-BE: twi 20, 3, 4                    # encoding: [0x0e,0x83,0x00,0x04]
@@ -3174,17 +3174,17 @@
 # CHECK-LE: td 20, 3, 4                     # encoding: [0x88,0x20,0x83,0x7e]
             tdng 3, 4
 
-# CHECK-BE: twi 2, 3, 4                     # encoding: [0x0c,0x43,0x00,0x04]
-# CHECK-LE: twi 2, 3, 4                     # encoding: [0x04,0x00,0x43,0x0c]
+# CHECK-BE: twllti 3, 4                     # encoding: [0x0c,0x43,0x00,0x04]
+# CHECK-LE: twllti 3, 4                     # encoding: [0x04,0x00,0x43,0x0c]
             twllti 3, 4
-# CHECK-BE: tw 2, 3, 4                      # encoding: [0x7c,0x43,0x20,0x08]
-# CHECK-LE: tw 2, 3, 4                      # encoding: [0x08,0x20,0x43,0x7c]
+# CHECK-BE: twllt 3, 4                      # encoding: [0x7c,0x43,0x20,0x08]
+# CHECK-LE: twllt 3, 4                      # encoding: [0x08,0x20,0x43,0x7c]
             twllt 3, 4
-# CHECK-BE: tdi 2, 3, 4                     # encoding: [0x08,0x43,0x00,0x04]
-# CHECK-LE: tdi 2, 3, 4                     # encoding: [0x04,0x00,0x43,0x08]
+# CHECK-BE: tdllti 3, 4                     # encoding: [0x08,0x43,0x00,0x04]
+# CHECK-LE: tdllti 3, 4                     # encoding: [0x04,0x00,0x43,0x08]
             tdllti 3, 4
-# CHECK-BE: td 2, 3, 4                      # encoding: [0x7c,0x43,0x20,0x88]
-# CHECK-LE: td 2, 3, 4                      # encoding: [0x88,0x20,0x43,0x7c]
+# CHECK-BE: tdllt 3, 4                      # encoding: [0x7c,0x43,0x20,0x88]
+# CHECK-LE: tdllt 3, 4                      # encoding: [0x88,0x20,0x43,0x7c]
             tdllt 3, 4
 
 # CHECK-BE: twi 6, 3, 4                     # encoding: [0x0c,0xc3,0x00,0x04]
@@ -3213,17 +3213,17 @@
 # CHECK-LE: td 5, 3, 4                      # encoding: [0x88,0x20,0xa3,0x7c]
             tdlge 3, 4
 
-# CHECK-BE: twi 1, 3, 4                     # encoding: [0x0c,0x23,0x00,0x04]
-# CHECK-LE: twi 1, 3, 4                     # encoding: [0x04,0x00,0x23,0x0c]
+# CHECK-BE: twlgti 3, 4                     # encoding: [0x0c,0x23,0x00,0x04]
+# CHECK-LE: twlgti 3, 4                     # encoding: [0x04,0x00,0x23,0x0c]
             twlgti 3, 4
-# CHECK-BE: tw 1, 3, 4                      # encoding: [0x7c,0x23,0x20,0x08]
-# CHECK-LE: tw 1, 3, 4                      # encoding: [0x08,0x20,0x23,0x7c]
+# CHECK-BE: twlgt 3, 4                      # encoding: [0x7c,0x23,0x20,0x08]
+# CHECK-LE: twlgt 3, 4                      # encoding: [0x08,0x20,0x23,0x7c]
             twlgt 3, 4
-# CHECK-BE: tdi 1, 3, 4                     # encoding: [0x08,0x23,0x00,0x04]
-# CHECK-LE: tdi 1, 3, 4                     # encoding: [0x04,0x00,0x23,0x08]
+# CHECK-BE: tdlgti 3, 4                     # encoding: [0x08,0x23,0x00,0x04]
+# CHECK-LE: tdlgti 3, 4                     # encoding: [0x04,0x00,0x23,0x08]
             tdlgti 3, 4
-# CHECK-BE: td 1, 3, 4                      # encoding: [0x7c,0x23,0x20,0x88]
-# CHECK-LE: td 1, 3, 4                      # encoding: [0x88,0x20,0x23,0x7c]
+# CHECK-BE: tdlgt 3, 4                      # encoding: [0x7c,0x23,0x20,0x88]
+# CHECK-LE: tdlgt 3, 4                      # encoding: [0x88,0x20,0x23,0x7c]
             tdlgt 3, 4
 
 # CHECK-BE: twi 5, 3, 4                     # encoding: [0x0c,0xa3,0x00,0x04]
@@ -3252,17 +3252,17 @@
 # CHECK-LE: td 6, 3, 4                      # encoding: [0x88,0x20,0xc3,0x7c]
             tdlng 3, 4
 
-# CHECK-BE: twi 31, 3, 4                    # encoding: [0x0f,0xe3,0x00,0x04]
-# CHECK-LE: twi 31, 3, 4                    # encoding: [0x04,0x00,0xe3,0x0f]
+# CHECK-BE: twui 3, 4                       # encoding: [0x0f,0xe3,0x00,0x04]
+# CHECK-LE: twui 3, 4                       # encoding: [0x04,0x00,0xe3,0x0f]
             twui 3, 4
-# CHECK-BE: tw 31, 3, 4                     # encoding: [0x7f,0xe3,0x20,0x08]
-# CHECK-LE: tw 31, 3, 4                     # encoding: [0x08,0x20,0xe3,0x7f]
+# CHECK-BE: twu 3, 4                        # encoding: [0x7f,0xe3,0x20,0x08]
+# CHECK-LE: twu 3, 4                        # encoding: [0x08,0x20,0xe3,0x7f]
             twu 3, 4
-# CHECK-BE: tdi 31, 3, 4                    # encoding: [0x0b,0xe3,0x00,0x04]
-# CHECK-LE: tdi 31, 3, 4                    # encoding: [0x04,0x00,0xe3,0x0b]
+# CHECK-BE: tdui 3, 4                       # encoding: [0x0b,0xe3,0x00,0x04]
+# CHECK-LE: tdui 3, 4                       # encoding: [0x04,0x00,0xe3,0x0b]
             tdui 3, 4
-# CHECK-BE: td 31, 3, 4                     # encoding: [0x7f,0xe3,0x20,0x88]
-# CHECK-LE: td 31, 3, 4                     # encoding: [0x88,0x20,0xe3,0x7f]
+# CHECK-BE: tdu 3, 4                         # encoding: [0x7f,0xe3,0x20,0x88]
+# CHECK-LE: tdu 3, 4                         # encoding: [0x88,0x20,0xe3,0x7f]
             tdu 3, 4
 
 # CHECK-BE: trap                            # encoding: [0x7f,0xe0,0x00,0x08]
@@ -3289,23 +3289,23 @@
 # CHECK-BE: rldimi. 2, 3, 55, 5             # encoding: [0x78,0x62,0xb9,0x4f]
 # CHECK-LE: rldimi. 2, 3, 55, 5             # encoding: [0x4f,0xb9,0x62,0x78]
             insrdi. 2, 3, 4, 5
-# CHECK-BE: rldicl 2, 3, 4, 0               # encoding: [0x78,0x62,0x20,0x00]
-# CHECK-LE: rldicl 2, 3, 4, 0               # encoding: [0x00,0x20,0x62,0x78]
+# CHECK-BE: rotldi 2, 3, 4                  # encoding: [0x78,0x62,0x20,0x00]
+# CHECK-LE: rotldi 2, 3, 4                  # encoding: [0x00,0x20,0x62,0x78]
             rotldi 2, 3, 4
-# CHECK-BE: rldicl. 2, 3, 4, 0              # encoding: [0x78,0x62,0x20,0x01]
-# CHECK-LE: rldicl. 2, 3, 4, 0              # encoding: [0x01,0x20,0x62,0x78]
+# CHECK-BE: rotldi. 2, 3, 4                 # encoding: [0x78,0x62,0x20,0x01]
+# CHECK-LE: rotldi. 2, 3, 4                 # encoding: [0x01,0x20,0x62,0x78]
             rotldi. 2, 3, 4
-# CHECK-BE: rldicl 2, 3, 60, 0              # encoding: [0x78,0x62,0xe0,0x02]
-# CHECK-LE: rldicl 2, 3, 60, 0              # encoding: [0x02,0xe0,0x62,0x78]
+# CHECK-BE: rotldi 2, 3, 60                 # encoding: [0x78,0x62,0xe0,0x02]
+# CHECK-LE: rotldi 2, 3, 60                 # encoding: [0x02,0xe0,0x62,0x78]
             rotrdi 2, 3, 4
-# CHECK-BE: rldicl. 2, 3, 60, 0             # encoding: [0x78,0x62,0xe0,0x03]
-# CHECK-LE: rldicl. 2, 3, 60, 0             # encoding: [0x03,0xe0,0x62,0x78]
+# CHECK-BE: rotldi. 2, 3, 60                # encoding: [0x78,0x62,0xe0,0x03]
+# CHECK-LE: rotldi. 2, 3, 60                # encoding: [0x03,0xe0,0x62,0x78]
             rotrdi. 2, 3, 4
-# CHECK-BE: rldcl 2, 3, 4, 0                # encoding: [0x78,0x62,0x20,0x10]
-# CHECK-LE: rldcl 2, 3, 4, 0                # encoding: [0x10,0x20,0x62,0x78]
+# CHECK-BE: rotld 2, 3, 4                   # encoding: [0x78,0x62,0x20,0x10]
+# CHECK-LE: rotld 2, 3, 4                   # encoding: [0x10,0x20,0x62,0x78]
             rotld 2, 3, 4
-# CHECK-BE: rldcl. 2, 3, 4, 0               # encoding: [0x78,0x62,0x20,0x11]
-# CHECK-LE: rldcl. 2, 3, 4, 0               # encoding: [0x11,0x20,0x62,0x78]
+# CHECK-BE: rotld. 2, 3, 4                  # encoding: [0x78,0x62,0x20,0x11]
+# CHECK-LE: rotld. 2, 3, 4                  # encoding: [0x11,0x20,0x62,0x78]
             rotld. 2, 3, 4
 # CHECK-BE: sldi 2, 3, 4                    # encoding: [0x78,0x62,0x26,0xe4]
 # CHECK-LE: sldi 2, 3, 4                    # encoding: [0xe4,0x26,0x62,0x78]
@@ -3319,11 +3319,11 @@
 # CHECK-BE: rldicl. 2, 3, 60, 4             # encoding: [0x78,0x62,0xe1,0x03]
 # CHECK-LE: rldicl. 2, 3, 60, 4             # encoding: [0x03,0xe1,0x62,0x78]
             srdi. 2, 3, 4
-# CHECK-BE: rldicl 2, 3, 0, 4               # encoding: [0x78,0x62,0x01,0x00]
-# CHECK-LE: rldicl 2, 3, 0, 4               # encoding: [0x00,0x01,0x62,0x78]
+# CHECK-BE: clrldi 2, 3, 4                  # encoding: [0x78,0x62,0x01,0x00]
+# CHECK-LE: clrldi 2, 3, 4                  # encoding: [0x00,0x01,0x62,0x78]
             clrldi 2, 3, 4
-# CHECK-BE: rldicl. 2, 3, 0, 4              # encoding: [0x78,0x62,0x01,0x01]
-# CHECK-LE: rldicl. 2, 3, 0, 4              # encoding: [0x01,0x01,0x62,0x78]
+# CHECK-BE: clrldi. 2, 3, 4                 # encoding: [0x78,0x62,0x01,0x01]
+# CHECK-LE: clrldi. 2, 3, 4                 # encoding: [0x01,0x01,0x62,0x78]
             clrldi. 2, 3, 4
 # CHECK-BE: rldicr 2, 3, 0, 59              # encoding: [0x78,0x62,0x06,0xe4]
 # CHECK-LE: rldicr 2, 3, 0, 59              # encoding: [0xe4,0x06,0x62,0x78]
@@ -3362,23 +3362,23 @@
 # CHECK-BE: rlwimi. 2, 3, 23, 5, 8          # encoding: [0x50,0x62,0xb9,0x51]
 # CHECK-LE: rlwimi. 2, 3, 23, 5, 8          # encoding: [0x51,0xb9,0x62,0x50]
             insrwi. 2, 3, 4, 5
-# CHECK-BE: rlwinm 2, 3, 4, 0, 31           # encoding: [0x54,0x62,0x20,0x3e]
-# CHECK-LE: rlwinm 2, 3, 4, 0, 31           # encoding: [0x3e,0x20,0x62,0x54]
+# CHECK-BE: rotlwi 2, 3, 4                  # encoding: [0x54,0x62,0x20,0x3e]
+# CHECK-LE: rotlwi 2, 3, 4                  # encoding: [0x3e,0x20,0x62,0x54]
             rotlwi 2, 3, 4
-# CHECK-BE: rlwinm. 2, 3, 4, 0, 31          # encoding: [0x54,0x62,0x20,0x3f]
-# CHECK-LE: rlwinm. 2, 3, 4, 0, 31          # encoding: [0x3f,0x20,0x62,0x54]
+# CHECK-BE: rotlwi. 2, 3, 4                 # encoding: [0x54,0x62,0x20,0x3f]
+# CHECK-LE: rotlwi. 2, 3, 4                 # encoding: [0x3f,0x20,0x62,0x54]
             rotlwi. 2, 3, 4
-# CHECK-BE: rlwinm 2, 3, 28, 0, 31          # encoding: [0x54,0x62,0xe0,0x3e]
-# CHECK-LE: rlwinm 2, 3, 28, 0, 31          # encoding: [0x3e,0xe0,0x62,0x54]
+# CHECK-BE: rotlwi 2, 3, 28                 # encoding: [0x54,0x62,0xe0,0x3e]
+# CHECK-LE: rotlwi 2, 3, 28                 # encoding: [0x3e,0xe0,0x62,0x54]
             rotrwi 2, 3, 4
-# CHECK-BE: rlwinm. 2, 3, 28, 0, 31         # encoding: [0x54,0x62,0xe0,0x3f]
-# CHECK-LE: rlwinm. 2, 3, 28, 0, 31         # encoding: [0x3f,0xe0,0x62,0x54]
+# CHECK-BE: rotlwi. 2, 3, 28                # encoding: [0x54,0x62,0xe0,0x3f]
+# CHECK-LE: rotlwi. 2, 3, 28                # encoding: [0x3f,0xe0,0x62,0x54]
             rotrwi. 2, 3, 4
-# CHECK-BE: rlwnm 2, 3, 4, 0, 31            # encoding: [0x5c,0x62,0x20,0x3e]
-# CHECK-LE: rlwnm 2, 3, 4, 0, 31            # encoding: [0x3e,0x20,0x62,0x5c]
+# CHECK-BE: rotlw 2, 3, 4                   # encoding: [0x5c,0x62,0x20,0x3e]
+# CHECK-LE: rotlw 2, 3, 4                   # encoding: [0x3e,0x20,0x62,0x5c]
             rotlw 2, 3, 4
-# CHECK-BE: rlwnm. 2, 3, 4, 0, 31           # encoding: [0x5c,0x62,0x20,0x3f]
-# CHECK-LE: rlwnm. 2, 3, 4, 0, 31           # encoding: [0x3f,0x20,0x62,0x5c]
+# CHECK-BE: rotlw. 2, 3, 4                  # encoding: [0x5c,0x62,0x20,0x3f]
+# CHECK-LE: rotlw. 2, 3, 4                  # encoding: [0x3f,0x20,0x62,0x5c]
             rotlw. 2, 3, 4
 # CHECK-BE: slwi 2, 3, 4                    # encoding: [0x54,0x62,0x20,0x36]
 # CHECK-LE: slwi 2, 3, 4                    # encoding: [0x36,0x20,0x62,0x54]
@@ -3392,11 +3392,11 @@
 # CHECK-BE: rlwinm. 2, 3, 28, 4, 31         # encoding: [0x54,0x62,0xe1,0x3f]
 # CHECK-LE: rlwinm. 2, 3, 28, 4, 31         # encoding: [0x3f,0xe1,0x62,0x54]
             srwi. 2, 3, 4
-# CHECK-BE: rlwinm 2, 3, 0, 4, 31           # encoding: [0x54,0x62,0x01,0x3e]
-# CHECK-LE: rlwinm 2, 3, 0, 4, 31           # encoding: [0x3e,0x01,0x62,0x54]
+# CHECK-BE: clrlwi 2, 3, 4                  # encoding: [0x54,0x62,0x01,0x3e]
+# CHECK-LE: clrlwi 2, 3, 4                  # encoding: [0x3e,0x01,0x62,0x54]
             clrlwi 2, 3, 4
-# CHECK-BE: rlwinm. 2, 3, 0, 4, 31          # encoding: [0x54,0x62,0x01,0x3f]
-# CHECK-LE: rlwinm. 2, 3, 0, 4, 31          # encoding: [0x3f,0x01,0x62,0x54]
+# CHECK-BE: clrlwi. 2, 3, 4                 # encoding: [0x54,0x62,0x01,0x3f]
+# CHECK-LE: clrlwi. 2, 3, 4                 # encoding: [0x3f,0x01,0x62,0x54]
             clrlwi. 2, 3, 4
 # CHECK-BE: rlwinm 2, 3, 0, 0, 27           # encoding: [0x54,0x62,0x00,0x36]
 # CHECK-LE: rlwinm 2, 3, 0, 0, 27           # encoding: [0x36,0x00,0x62,0x54]
@@ -3413,35 +3413,35 @@
 
 # Move to/from special purpose register mnemonics
 
-# CHECK-BE: mtspr 1, 2                      # encoding: [0x7c,0x41,0x03,0xa6]
-# CHECK-LE: mtspr 1, 2                      # encoding: [0xa6,0x03,0x41,0x7c]
+# CHECK-BE: mtxer 2                         # encoding: [0x7c,0x41,0x03,0xa6]
+# CHECK-LE: mtxer 2                         # encoding: [0xa6,0x03,0x41,0x7c]
             mtxer 2
-# CHECK-BE: mfspr 2, 1                      # encoding: [0x7c,0x41,0x02,0xa6]
-# CHECK-LE: mfspr 2, 1                      # encoding: [0xa6,0x02,0x41,0x7c]
+# CHECK-BE: mfxer 2                         # encoding: [0x7c,0x41,0x02,0xa6]
+# CHECK-LE: mfxer 2                         # encoding: [0xa6,0x02,0x41,0x7c]
             mfxer 2
-# CHECK-BE: mfspr 2, 4                      # encoding: [0x7c,0x44,0x02,0xa6]
-# CHECK-LE: mfspr 2, 4                      # encoding: [0xa6,0x02,0x44,0x7c]
+# CHECK-BE: mfrtcu 2                        # encoding: [0x7c,0x44,0x02,0xa6]
+# CHECK-LE: mfrtcu 2                        # encoding: [0xa6,0x02,0x44,0x7c]
             mfrtcu 2
-# CHECK-BE: mfspr 2, 5                      # encoding: [0x7c,0x45,0x02,0xa6]
-# CHECK-LE: mfspr 2, 5                      # encoding: [0xa6,0x02,0x45,0x7c]
+# CHECK-BE: mfrtcl 2                        # encoding: [0x7c,0x45,0x02,0xa6]
+# CHECK-LE: mfrtcl 2                        # encoding: [0xa6,0x02,0x45,0x7c]
             mfrtcl 2
-# CHECK-BE: mtspr 17, 2                     # encoding: [0x7c,0x51,0x03,0xa6]
-# CHECK-LE: mtspr 17, 2                     # encoding: [0xa6,0x03,0x51,0x7c]
+# CHECK-BE: mtdscr 2                        # encoding: [0x7c,0x51,0x03,0xa6]
+# CHECK-LE: mtdscr 2                        # encoding: [0xa6,0x03,0x51,0x7c]
             mtdscr 2
-# CHECK-BE: mfspr 2, 17                     # encoding: [0x7c,0x51,0x02,0xa6]
-# CHECK-LE: mfspr 2, 17                     # encoding: [0xa6,0x02,0x51,0x7c]
+# CHECK-BE: mfdscr 2                        # encoding: [0x7c,0x51,0x02,0xa6]
+# CHECK-LE: mfdscr 2                        # encoding: [0xa6,0x02,0x51,0x7c]
             mfdscr 2
-# CHECK-BE: mtspr 18, 2                     # encoding: [0x7c,0x52,0x03,0xa6]
-# CHECK-LE: mtspr 18, 2                     # encoding: [0xa6,0x03,0x52,0x7c]
+# CHECK-BE: mtdsisr 2                       # encoding: [0x7c,0x52,0x03,0xa6]
+# CHECK-LE: mtdsisr 2                       # encoding: [0xa6,0x03,0x52,0x7c]
             mtdsisr 2
-# CHECK-BE: mfspr 2, 18                     # encoding: [0x7c,0x52,0x02,0xa6]
-# CHECK-LE: mfspr 2, 18                     # encoding: [0xa6,0x02,0x52,0x7c]
+# CHECK-BE: mfdsisr 2                       # encoding: [0x7c,0x52,0x02,0xa6]
+# CHECK-LE: mfdsisr 2                       # encoding: [0xa6,0x02,0x52,0x7c]
             mfdsisr 2
-# CHECK-BE: mtspr 19, 2                     # encoding: [0x7c,0x53,0x03,0xa6]
-# CHECK-LE: mtspr 19, 2                     # encoding: [0xa6,0x03,0x53,0x7c]
+# CHECK-BE: mtdar 2                         # encoding: [0x7c,0x53,0x03,0xa6]
+# CHECK-LE: mtdar 2                         # encoding: [0xa6,0x03,0x53,0x7c]
             mtdar 2
-# CHECK-BE: mfspr 2, 19                     # encoding: [0x7c,0x53,0x02,0xa6]
-# CHECK-LE: mfspr 2, 19                     # encoding: [0xa6,0x02,0x53,0x7c]
+# CHECK-BE: mfdar 2                         # encoding: [0x7c,0x53,0x02,0xa6]
+# CHECK-LE: mfdar 2                         # encoding: [0xa6,0x02,0x53,0x7c]
             mfdar 2
 # CHECK-BE: mtspr 22, 2                     # encoding: [0x7c,0x56,0x03,0xa6]
 # CHECK-LE: mtspr 22, 2                     # encoding: [0xa6,0x03,0x56,0x7c]
@@ -3467,23 +3467,23 @@
 # CHECK-BE: mfspr 2, 27                     # encoding: [0x7c,0x5b,0x02,0xa6]
 # CHECK-LE: mfspr 2, 27                     # encoding: [0xa6,0x02,0x5b,0x7c]
             mfsrr1 2
-# CHECK-BE: mtspr 28, 2                     # encoding: [0x7c,0x5c,0x03,0xa6]
-# CHECK-LE: mtspr 28, 2                     # encoding: [0xa6,0x03,0x5c,0x7c]
+# CHECK-BE: mtcfar 2                        # encoding: [0x7c,0x5c,0x03,0xa6]
+# CHECK-LE: mtcfar 2                        # encoding: [0xa6,0x03,0x5c,0x7c]
             mtcfar 2
-# CHECK-BE: mfspr 2, 28                     # encoding: [0x7c,0x5c,0x02,0xa6]
-# CHECK-LE: mfspr 2, 28                     # encoding: [0xa6,0x02,0x5c,0x7c]
+# CHECK-BE: mfcfar 2                        # encoding: [0x7c,0x5c,0x02,0xa6]
+# CHECK-LE: mfcfar 2                        # encoding: [0xa6,0x02,0x5c,0x7c]
             mfcfar 2
-# CHECK-BE: mtspr 29, 2                     # encoding: [0x7c,0x5d,0x03,0xa6]
-# CHECK-LE: mtspr 29, 2                     # encoding: [0xa6,0x03,0x5d,0x7c]
+# CHECK-BE: mtamr 2                         # encoding: [0x7c,0x5d,0x03,0xa6]
+# CHECK-LE: mtamr 2                         # encoding: [0xa6,0x03,0x5d,0x7c]
             mtamr 2
-# CHECK-BE: mfspr 2, 29                     # encoding: [0x7c,0x5d,0x02,0xa6]
-# CHECK-LE: mfspr 2, 29                     # encoding: [0xa6,0x02,0x5d,0x7c]
+# CHECK-BE: mfamr 2                         # encoding: [0x7c,0x5d,0x02,0xa6]
+# CHECK-LE: mfamr 2                         # encoding: [0xa6,0x02,0x5d,0x7c]
             mfamr 2
-# CHECK-BE: mtspr 48, 2                     # encoding: [0x7c,0x50,0x0b,0xa6]
-# CHECK-LE: mtspr 48, 2                     # encoding: [0xa6,0x0b,0x50,0x7c]
+# CHECK-BE: mtpid 2                         # encoding: [0x7c,0x50,0x0b,0xa6]
+# CHECK-LE: mtpid 2                         # encoding: [0xa6,0x0b,0x50,0x7c]
             mtpid 2
-# CHECK-BE: mfspr 2, 48                     # encoding: [0x7c,0x50,0x0a,0xa6]
-# CHECK-LE: mfspr 2, 48                     # encoding: [0xa6,0x0a,0x50,0x7c]
+# CHECK-BE: mfpid 2                         # encoding: [0x7c,0x50,0x0a,0xa6]
+# CHECK-LE: mfpid 2                         # encoding: [0xa6,0x0a,0x50,0x7c]
             mfpid 2
 # CHECK-BE: mtlr 2                          # encoding: [0x7c,0x48,0x03,0xa6]
 # CHECK-LE: mtlr 2                          # encoding: [0xa6,0x03,0x48,0x7c]
@@ -3503,8 +3503,8 @@
 # CHECK-BE: nop                             # encoding: [0x60,0x00,0x00,0x00]
 # CHECK-LE: nop                             # encoding: [0x00,0x00,0x00,0x60]
             nop
-# CHECK-BE: xori 0, 0, 0                    # encoding: [0x68,0x00,0x00,0x00]
-# CHECK-LE: xori 0, 0, 0                    # encoding: [0x00,0x00,0x00,0x68]
+# CHECK-BE: xnop                            # encoding: [0x68,0x00,0x00,0x00]
+# CHECK-LE: xnop                            # encoding: [0x00,0x00,0x00,0x68]
             xnop
 # CHECK-BE: li 2, 128                       # encoding: [0x38,0x40,0x00,0x80]
 # CHECK-LE: li 2, 128                       # encoding: [0x80,0x00,0x40,0x38]
@@ -3518,17 +3518,17 @@
 # CHECK-BE: mr 2, 3                         # encoding: [0x7c,0x62,0x1b,0x78]
 # CHECK-LE: mr 2, 3                         # encoding: [0x78,0x1b,0x62,0x7c]
             mr 2, 3
-# CHECK-BE: or. 2, 3, 3                     # encoding: [0x7c,0x62,0x1b,0x79]
-# CHECK-LE: or. 2, 3, 3                     # encoding: [0x79,0x1b,0x62,0x7c]
+# CHECK-BE: mr. 2, 3                        # encoding: [0x7c,0x62,0x1b,0x79]
+# CHECK-LE: mr. 2, 3                        # encoding: [0x79,0x1b,0x62,0x7c]
             mr. 2, 3
-# CHECK-BE: nor 2, 3, 3                     # encoding: [0x7c,0x62,0x18,0xf8]
-# CHECK-LE: nor 2, 3, 3                     # encoding: [0xf8,0x18,0x62,0x7c]
+# CHECK-BE: not 2, 3                        # encoding: [0x7c,0x62,0x18,0xf8]
+# CHECK-LE: not 2, 3                        # encoding: [0xf8,0x18,0x62,0x7c]
             not 2, 3
-# CHECK-BE: nor. 2, 3, 3                    # encoding: [0x7c,0x62,0x18,0xf9]
-# CHECK-LE: nor. 2, 3, 3                    # encoding: [0xf9,0x18,0x62,0x7c]
+# CHECK-BE: not. 2, 3                       # encoding: [0x7c,0x62,0x18,0xf9]
+# CHECK-LE: not. 2, 3                       # encoding: [0xf9,0x18,0x62,0x7c]
             not. 2, 3
-# CHECK-BE: mtcrf 255, 2                    # encoding: [0x7c,0x4f,0xf1,0x20]
-# CHECK-LE: mtcrf 255, 2                    # encoding: [0x20,0xf1,0x4f,0x7c]
+# CHECK-BE: mtcr 2                          # encoding: [0x7c,0x4f,0xf1,0x20]
+# CHECK-LE: mtcr 2                          # encoding: [0x20,0xf1,0x4f,0x7c]
             mtcr 2
 
 # CHECK-BE: mfspr 4, 272                    # encoding: [0x7c,0x90,0x42,0xa6]
@@ -3623,12 +3623,12 @@
 # CHECK-LE: stswi 8, 6, 7                    # encoding: [0xaa,0x3d,0x06,0x7d]
             stswi %r8, %r6, 7
 
-# CHECK-BE: rfid                            # encoding: [0x4c,0x00,0x00,0x24]
-# CHECK-LE: rfid                            # encoding: [0x24,0x00,0x00,0x4c]
+# CHECK-BE: rfid                             # encoding: [0x4c,0x00,0x00,0x24]
+# CHECK-LE: rfid                             # encoding: [0x24,0x00,0x00,0x4c]
             rfid
 
-# CHECK-BE: mfspr 2, 280                     # encoding: [0x7c,0x58,0x42,0xa6]
-# CHECK-LE: mfspr 2, 280                     # encoding: [0xa6,0x42,0x58,0x7c]
+# CHECK-BE: mfasr 2                          # encoding: [0x7c,0x58,0x42,0xa6]
+# CHECK-LE: mfasr 2                          # encoding: [0xa6,0x42,0x58,0x7c]
             mfasr 2
 # CHECK-BE: mtspr 280, 2                     # encoding: [0x7c,0x58,0x43,0xa6]
 # CHECK-LE: mtspr 280, 2                     # encoding: [0xa6,0x43,0x58,0x7c]
diff --git a/test/MC/PowerPC/ppc64-encoding-fp.s b/test/MC/PowerPC/ppc64-encoding-fp.s
index 2f4f8287059..48384845715 100644
--- a/test/MC/PowerPC/ppc64-encoding-fp.s
+++ b/test/MC/PowerPC/ppc64-encoding-fp.s
@@ -371,11 +371,11 @@
 # CHECK-BE: mtfsfi. 5, 2, 1                 # encoding: [0xfe,0x81,0x21,0x0d]
 # CHECK-LE: mtfsfi. 5, 2, 1                 # encoding: [0x0d,0x21,0x81,0xfe]
             mtfsfi. 5, 2, 1
-# CHECK-BE: mtfsfi 6, 2, 0                  # encoding: [0xff,0x00,0x21,0x0c]
-# CHECK-LE: mtfsfi 6, 2, 0                  # encoding: [0x0c,0x21,0x00,0xff]
+# CHECK-BE: mtfsfi 6, 2                     # encoding: [0xff,0x00,0x21,0x0c]
+# CHECK-LE: mtfsfi 6, 2                     # encoding: [0x0c,0x21,0x00,0xff]
             mtfsfi 6, 2
-# CHECK-BE: mtfsfi. 6, 2, 0                 # encoding: [0xff,0x00,0x21,0x0d]
-# CHECK-LE: mtfsfi. 6, 2, 0                 # encoding: [0x0d,0x21,0x00,0xff]
+# CHECK-BE: mtfsfi. 6, 2                    # encoding: [0xff,0x00,0x21,0x0d]
+# CHECK-LE: mtfsfi. 6, 2                    # encoding: [0x0d,0x21,0x00,0xff]
             mtfsfi. 6, 2
 # CHECK-BE: mtfsf 127, 8, 1, 1              # encoding: [0xfe,0xff,0x45,0x8e]
 # CHECK-LE: mtfsf 127, 8, 1, 1              # encoding: [0x8e,0x45,0xff,0xfe]
@@ -383,11 +383,11 @@
 # CHECK-BE: mtfsf. 125, 8, 1, 1             # encoding: [0xfe,0xfb,0x45,0x8f]
 # CHECK-LE: mtfsf. 125, 8, 1, 1             # encoding: [0x8f,0x45,0xfb,0xfe]
             mtfsf. 125, 8, 1, 1
-# CHECK-BE: mtfsf 127, 6, 0, 0              # encoding: [0xfc,0xfe,0x35,0x8e]
-# CHECK-LE: mtfsf 127, 6, 0, 0              # encoding: [0x8e,0x35,0xfe,0xfc]
+# CHECK-BE: mtfsf 127, 6                    # encoding: [0xfc,0xfe,0x35,0x8e]
+# CHECK-LE: mtfsf 127, 6                    # encoding: [0x8e,0x35,0xfe,0xfc]
             mtfsf 127, 6
-# CHECK-BE: mtfsf. 125, 6, 0, 0             # encoding: [0xfc,0xfa,0x35,0x8f]
-# CHECK-LE: mtfsf. 125, 6, 0, 0             # encoding: [0x8f,0x35,0xfa,0xfc]
+# CHECK-BE: mtfsf. 125, 6                   # encoding: [0xfc,0xfa,0x35,0x8f]
+# CHECK-LE: mtfsf. 125, 6                   # encoding: [0x8f,0x35,0xfa,0xfc]
             mtfsf. 125, 6
 # CHECK-BE: mtfsb0 31                       # encoding: [0xff,0xe0,0x00,0x8c]
 # CHECK-LE: mtfsb0 31                       # encoding: [0x8c,0x00,0xe0,0xff]
diff --git a/test/MC/PowerPC/ppc64-encoding-p8vector.s b/test/MC/PowerPC/ppc64-encoding-p8vector.s
new file mode 100644
index 00000000000..657b7378fb4
--- /dev/null
+++ b/test/MC/PowerPC/ppc64-encoding-p8vector.s
@@ -0,0 +1,26 @@
+# RUN: llvm-mc -triple powerpc64-unknown-unknown --show-encoding %s | FileCheck -check-prefix=CHECK-BE %s
+# RUN: llvm-mc -triple powerpc64le-unknown-unknown --show-encoding %s | FileCheck -check-prefix=CHECK-LE %s
+
+# CHECK-BE: vpksdss 2, 3, 4                 # encoding: [0x10,0x43,0x25,0xce]
+# CHECK-LE: vpksdss 2, 3, 4                 # encoding: [0xce,0x25,0x43,0x10]
+            vpksdss 2, 3, 4
+
+# CHECK-BE: vpksdus 2, 3, 4                 # encoding: [0x10,0x43,0x25,0x4e]
+# CHECK-LE: vpksdus 2, 3, 4                 # encoding: [0x4e,0x25,0x43,0x10]
+            vpksdus 2, 3, 4
+
+# CHECK-BE: vpkudus 2, 3, 4                 # encoding: [0x10,0x43,0x24,0xce]
+# CHECK-LE: vpkudus 2, 3, 4                 # encoding: [0xce,0x24,0x43,0x10]
+            vpkudus 2, 3, 4
+
+# CHECK-BE: vpkudum 2, 3, 4                 # encoding: [0x10,0x43,0x24,0x4e]
+# CHECK-LE: vpkudum 2, 3, 4                 # encoding: [0x4e,0x24,0x43,0x10]
+            vpkudum 2, 3, 4
+
+# CHECK-BE: vupkhsw 2, 3                    # encoding: [0x10,0x40,0x1e,0x4e]
+# CHECK-LE: vupkhsw 2, 3                    # encoding: [0x4e,0x1e,0x40,0x10]
+            vupkhsw 2, 3
+
+# CHECK-BE: vupklsw 2, 3                    # encoding: [0x10,0x40,0x1e,0xce]
+# CHECK-LE: vupklsw 2, 3                    # encoding: [0xce,0x1e,0x40,0x10]
+            vupklsw 2, 3
diff --git a/test/MC/PowerPC/ppc64-encoding-vmx.s b/test/MC/PowerPC/ppc64-encoding-vmx.s
index 8f7e91903af..d8825bf3340 100644
--- a/test/MC/PowerPC/ppc64-encoding-vmx.s
+++ b/test/MC/PowerPC/ppc64-encoding-vmx.s
@@ -1,5 +1,5 @@
 
-# RUN: llvm-mc -triple powerpc64-unknown-unknown --show-encoding %s | FileCheck -check-prefix=CHECK-BE %s
+# RUN: llvm-mc -triple powerpc64-unknown-unknown --show-encoding %s | FileCheck -check-prefix=CHECK-BE %s 
 # RUN: llvm-mc -triple powerpc64le-unknown-unknown --show-encoding %s | FileCheck -check-prefix=CHECK-LE %s
 
 # Vector facility
@@ -110,7 +110,13 @@
 # CHECK-BE: vmrglw 2, 3, 4                  # encoding: [0x10,0x43,0x21,0x8c]
 # CHECK-LE: vmrglw 2, 3, 4                  # encoding: [0x8c,0x21,0x43,0x10]
             vmrglw 2, 3, 4
-
+# CHECK-BE: vmrgew 2, 3, 4                  # encoding: [0x10,0x43,0x27,0x8c]
+# CHECK-LE: vmrgew 2, 3, 4                  # encoding: [0x8c,0x27,0x43,0x10]
+            vmrgew 2, 3, 4
+# CHECK-BE: vmrgow 2, 3, 4                  # encoding: [0x10,0x43,0x26,0x8c]
+# CHECK-LE: vmrgow 2, 3, 4                  # encoding: [0x8c,0x26,0x43,0x10]
+            vmrgow 2, 3, 4
+	
 # CHECK-BE: vspltb 2, 3, 1                  # encoding: [0x10,0x41,0x1a,0x0c]
 # CHECK-LE: vspltb 2, 3, 1                  # encoding: [0x0c,0x1a,0x41,0x10]
             vspltb 2, 3, 1
@@ -237,7 +243,19 @@
 # CHECK-BE: vadduws 2, 3, 4                 # encoding: [0x10,0x43,0x22,0x80]
 # CHECK-LE: vadduws 2, 3, 4                 # encoding: [0x80,0x22,0x43,0x10]
             vadduws 2, 3, 4
-
+# CHECK-BE: vadduqm 2, 3, 4                 # encoding: [0x10,0x43,0x21,0x00]
+# CHECK-LE: vadduqm 2, 3, 4                 # encoding: [0x00,0x21,0x43,0x10]
+            vadduqm 2, 3, 4
+# CHECK-BE: vaddeuqm 2, 3, 4, 5             # encoding: [0x10,0x43,0x21,0x7c]
+# CHECK-LE: vaddeuqm 2, 3, 4, 5             # encoding:	[0x7c,0x21,0x43,0x10]
+            vaddeuqm 2, 3, 4, 5
+# CHECK-BE: vaddcuq 2, 3, 4                 # encoding: [0x10,0x43,0x21,0x40]
+# CHECK-LE: vaddcuq 2, 3, 4                 # encoding: [0x40,0x21,0x43,0x10]
+            vaddcuq 2, 3, 4
+# CHECK-BE: vaddecuq 2, 3, 4, 5             # encoding: [0x10,0x43,0x21,0x7d]
+# CHECK-LE: vaddecuq 2, 3, 4, 5             # encoding: [0x7d,0x21,0x43,0x10]
+            vaddecuq 2, 3, 4, 5   
+	
 # CHECK-BE: vsubcuw 2, 3, 4                 # encoding: [0x10,0x43,0x25,0x80]
 # CHECK-LE: vsubcuw 2, 3, 4                 # encoding: [0x80,0x25,0x43,0x10]
             vsubcuw 2, 3, 4
@@ -271,7 +289,19 @@
 # CHECK-BE: vsubuws 2, 3, 4                 # encoding: [0x10,0x43,0x26,0x80]
 # CHECK-LE: vsubuws 2, 3, 4                 # encoding: [0x80,0x26,0x43,0x10]
             vsubuws 2, 3, 4
-
+# CHECK-BE: vsubuqm 2, 3, 4                 # encoding: [0x10,0x43,0x25,0x00]
+# CHECK-LE: vsubuqm 2, 3, 4                 # encoding: [0x00,0x25,0x43,0x10]
+            vsubuqm 2, 3, 4
+# CHECK-BE: vsubeuqm 2, 3, 4, 5             # encoding: [0x10,0x43,0x21,0x7e]
+# CHECK-LE: vsubeuqm 2, 3, 4, 5             # encoding: [0x7e,0x21,0x43,0x10]
+            vsubeuqm 2, 3, 4, 5
+# CHECK-BE: vsubcuq 2, 3, 4                 # encoding: [0x10,0x43,0x25,0x40] 
+# CHECK-LE: vsubcuq 2, 3, 4                 # encoding: [0x40,0x25,0x43,0x10]
+            vsubcuq 2, 3, 4
+# CHECK-BE: vsubecuq 2, 3, 4, 5             # encoding: [0x10,0x43,0x21,0x7f]
+# CHECK-LE: vsubecuq 2, 3, 4, 5             # encoding: [0x7f,0x21,0x43,0x10]
+            vsubecuq 2, 3, 4, 5	
+	
 # CHECK-BE: vmulesb 2, 3, 4                 # encoding: [0x10,0x43,0x23,0x08]
 # CHECK-LE: vmulesb 2, 3, 4                 # encoding: [0x08,0x23,0x43,0x10]
             vmulesb 2, 3, 4
@@ -662,6 +692,12 @@
 # CHECK-BE: vrsqrtefp 2, 3                  # encoding: [0x10,0x40,0x19,0x4a]
 # CHECK-LE: vrsqrtefp 2, 3                  # encoding: [0x4a,0x19,0x40,0x10]
             vrsqrtefp 2, 3
+# CHECK-BE: vgbbd 2, 3                      # encoding: [0x10,0x40,0x1d,0x0c]
+# CHECK-LE: vgbbd 2, 3                      # encoding: [0x0c,0x1d,0x40,0x10]
+            vgbbd 2, 3
+# CHECK-BE: vbpermq 2, 5, 17                # encoding: [0x10,0x45,0x8d,0x4c]
+# CHECK-LE: vbpermq 2, 5, 17                # encoding: [0x4c,0x8d,0x45,0x10]
+            vbpermq 2, 5, 17
 
 # Vector count leading zero instructions
 # CHECK-BE: vclzb 2, 3                      # encoding: [0x10,0x40,0x1f,0x02]
diff --git a/test/MC/PowerPC/ppc64-encoding.s b/test/MC/PowerPC/ppc64-encoding.s
index 05fde620e4a..4a698adadda 100644
--- a/test/MC/PowerPC/ppc64-encoding.s
+++ b/test/MC/PowerPC/ppc64-encoding.s
@@ -27,23 +27,23 @@
 # CHECK-LE-NEXT:                            #   fixup A - offset: 0, value: target, kind: fixup_ppc_br24abs
             bla target
 
-# CHECK-BE: bc 4, 10, target                # encoding: [0x40,0x8a,A,0bAAAAAA00]
-# CHECK-LE: bc 4, 10, target                # encoding: [0bAAAAAA00,A,0x8a,0x40]
+# CHECK-BE: bf 10, target                   # encoding: [0x40,0x8a,A,0bAAAAAA00]
+# CHECK-LE: bf 10, target                   # encoding: [0bAAAAAA00,A,0x8a,0x40]
 # CHECK-BE-NEXT:                            #   fixup A - offset: 0, value: target, kind: fixup_ppc_brcond14
 # CHECK-LE-NEXT:                            #   fixup A - offset: 0, value: target, kind: fixup_ppc_brcond14
             bc 4, 10, target
-# CHECK-BE: bca 4, 10, target               # encoding: [0x40,0x8a,A,0bAAAAAA10]
-# CHECK-LE: bca 4, 10, target               # encoding: [0bAAAAAA10,A,0x8a,0x40]
+# CHECK-BE: bfa 10, target                  # encoding: [0x40,0x8a,A,0bAAAAAA10]
+# CHECK-LE: bfa 10, target                  # encoding: [0bAAAAAA10,A,0x8a,0x40]
 # CHECK-BE-NEXT:                            #   fixup A - offset: 0, value: target, kind: fixup_ppc_brcond14abs
 # CHECK-LE-NEXT:                            #   fixup A - offset: 0, value: target, kind: fixup_ppc_brcond14abs
             bca 4, 10, target
-# CHECK-BE: bcl 4, 10, target               # encoding: [0x40,0x8a,A,0bAAAAAA01]
-# CHECK-LE: bcl 4, 10, target               # encoding: [0bAAAAAA01,A,0x8a,0x40]
+# CHECK-BE: bfl 10, target                  # encoding: [0x40,0x8a,A,0bAAAAAA01]
+# CHECK-LE: bfl 10, target                  # encoding: [0bAAAAAA01,A,0x8a,0x40]
 # CHECK-BE-NEXT:                            #   fixup A - offset: 0, value: target, kind: fixup_ppc_brcond14
 # CHECK-LE-NEXT:                            #   fixup A - offset: 0, value: target, kind: fixup_ppc_brcond14
             bcl 4, 10, target
-# CHECK-BE: bcla 4, 10, target              # encoding: [0x40,0x8a,A,0bAAAAAA11]
-# CHECK-LE: bcla 4, 10, target              # encoding: [0bAAAAAA11,A,0x8a,0x40]
+# CHECK-BE: bfla 10, target                 # encoding: [0x40,0x8a,A,0bAAAAAA11]
+# CHECK-LE: bfla 10, target                 # encoding: [0bAAAAAA11,A,0x8a,0x40]
 # CHECK-BE-NEXT:                            #   fixup A - offset: 0, value: target, kind: fixup_ppc_brcond14abs
 # CHECK-LE-NEXT:                            #   fixup A - offset: 0, value: target, kind: fixup_ppc_brcond14abs
             bcla 4, 10, target
@@ -51,26 +51,26 @@
 # CHECK-BE: bclr 4, 10, 3                   # encoding: [0x4c,0x8a,0x18,0x20]
 # CHECK-LE: bclr 4, 10, 3                   # encoding: [0x20,0x18,0x8a,0x4c]
             bclr 4, 10, 3
-# CHECK-BE: bclr 4, 10, 0                   # encoding: [0x4c,0x8a,0x00,0x20]
-# CHECK-LE: bclr 4, 10, 0                   # encoding: [0x20,0x00,0x8a,0x4c]
+# CHECK-BE: bclr 4, 10                      # encoding: [0x4c,0x8a,0x00,0x20]
+# CHECK-LE: bclr 4, 10                      # encoding: [0x20,0x00,0x8a,0x4c]
             bclr 4, 10
 # CHECK-BE: bclrl 4, 10, 3                  # encoding: [0x4c,0x8a,0x18,0x21]
 # CHECK-LE: bclrl 4, 10, 3                  # encoding: [0x21,0x18,0x8a,0x4c]
             bclrl 4, 10, 3
-# CHECK-BE: bclrl 4, 10, 0                  # encoding: [0x4c,0x8a,0x00,0x21]
-# CHECK-LE: bclrl 4, 10, 0                  # encoding: [0x21,0x00,0x8a,0x4c]
+# CHECK-BE: bclrl 4, 10                     # encoding: [0x4c,0x8a,0x00,0x21]
+# CHECK-LE: bclrl 4, 10                     # encoding: [0x21,0x00,0x8a,0x4c]
             bclrl 4, 10
 # CHECK-BE: bcctr 4, 10, 3                  # encoding: [0x4c,0x8a,0x1c,0x20]
 # CHECK-LE: bcctr 4, 10, 3                  # encoding: [0x20,0x1c,0x8a,0x4c]
             bcctr 4, 10, 3
-# CHECK-BE: bcctr 4, 10, 0                  # encoding: [0x4c,0x8a,0x04,0x20]
-# CHECK-LE: bcctr 4, 10, 0                  # encoding: [0x20,0x04,0x8a,0x4c]
+# CHECK-BE: bcctr 4, 10                     # encoding: [0x4c,0x8a,0x04,0x20]
+# CHECK-LE: bcctr 4, 10                     # encoding: [0x20,0x04,0x8a,0x4c]
             bcctr 4, 10
 # CHECK-BE: bcctrl 4, 10, 3                 # encoding: [0x4c,0x8a,0x1c,0x21]
 # CHECK-LE: bcctrl 4, 10, 3                 # encoding: [0x21,0x1c,0x8a,0x4c]
             bcctrl 4, 10, 3
-# CHECK-BE: bcctrl 4, 10, 0                 # encoding: [0x4c,0x8a,0x04,0x21]
-# CHECK-LE: bcctrl 4, 10, 0                 # encoding: [0x21,0x04,0x8a,0x4c]
+# CHECK-BE: bcctrl 4, 10                    # encoding: [0x4c,0x8a,0x04,0x21]
+# CHECK-LE: bcctrl 4, 10                    # encoding: [0x21,0x04,0x8a,0x4c]
             bcctrl 4, 10
 
 # Condition register instructions
@@ -108,10 +108,22 @@
 # CHECK-BE: sc 1                            # encoding: [0x44,0x00,0x00,0x22]
 # CHECK-LE: sc 1                            # encoding: [0x22,0x00,0x00,0x44]
             sc 1
-# CHECK-BE: sc 0                            # encoding: [0x44,0x00,0x00,0x02]
-# CHECK-LE: sc 0                            # encoding: [0x02,0x00,0x00,0x44]
+# CHECK-BE: sc                              # encoding: [0x44,0x00,0x00,0x02]
+# CHECK-LE: sc                              # encoding: [0x02,0x00,0x00,0x44]
             sc
 
+# Branch history rolling buffer
+
+# CHECK-BE: clrbhrb                         # encoding: [0x7c,0x00,0x03,0x5c]
+# CHECK-LE: clrbhrb                         # encoding: [0x5c,0x03,0x00,0x7c]
+            clrbhrb
+# CHECK-BE: mfbhrbe 9, 983                  # encoding: [0x7d,0x3e,0xba,0x5c]
+# CHECK-LE: mfbhrbe 9, 983                  # encoding: [0x5c,0xba,0x3e,0x7d]
+            mfbhrbe 9, 983
+# CHECK-BE: rfebb 1                         # encoding: [0x4c,0x00,0x09,0x24]
+# CHECK-LE: rfebb 1                         # encoding: [0x24,0x09,0x00,0x4c]
+            rfebb 1
+
 # Fixed-point facility
 
 # Fixed-point load instructions
@@ -521,17 +533,17 @@
 
 # Fixed-point trap instructions
 
-# CHECK-BE: twi 2, 3, 4                     # encoding: [0x0c,0x43,0x00,0x04]
-# CHECK-LE: twi 2, 3, 4                     # encoding: [0x04,0x00,0x43,0x0c]
+# CHECK-BE: twllti 3, 4                     # encoding: [0x0c,0x43,0x00,0x04]
+# CHECK-LE: twllti 3, 4                     # encoding: [0x04,0x00,0x43,0x0c]
             twi 2, 3, 4
-# CHECK-BE: tw 2, 3, 4                      # encoding: [0x7c,0x43,0x20,0x08]
-# CHECK-LE: tw 2, 3, 4                      # encoding: [0x08,0x20,0x43,0x7c]
+# CHECK-BE: twllt 3, 4                      # encoding: [0x7c,0x43,0x20,0x08]
+# CHECK-LE: twllt 3, 4                      # encoding: [0x08,0x20,0x43,0x7c]
             tw 2, 3, 4
-# CHECK-BE: tdi 2, 3, 4                     # encoding: [0x08,0x43,0x00,0x04]
-# CHECK-LE: tdi 2, 3, 4                     # encoding: [0x04,0x00,0x43,0x08]
+# CHECK-BE: tdllti 3, 4                     # encoding: [0x08,0x43,0x00,0x04]
+# CHECK-LE: tdllti 3, 4                     # encoding: [0x04,0x00,0x43,0x08]
             tdi 2, 3, 4
-# CHECK-BE: td 2, 3, 4                      # encoding: [0x7c,0x43,0x20,0x88]
-# CHECK-LE: td 2, 3, 4                      # encoding: [0x88,0x20,0x43,0x7c]
+# CHECK-BE: tdllt 3, 4                      # encoding: [0x7c,0x43,0x20,0x88]
+# CHECK-LE: tdllt 3, 4                      # encoding: [0x88,0x20,0x43,0x7c]
             td 2, 3, 4
 
 # Fixed-point select
@@ -622,17 +634,17 @@
 # CHECK-LE: extsh. 2, 3                     # encoding: [0x35,0x07,0x62,0x7c]
             extsh. 2, 3
 
-# CHECK-BE: cntlzw 2, 3                     # encoding: [0x7c,0x62,0x00,0x34]
-# CHECK-LE: cntlzw 2, 3                     # encoding: [0x34,0x00,0x62,0x7c]
+# CHECK-BE: cntlz 2, 3                     # encoding: [0x7c,0x62,0x00,0x34]
+# CHECK-LE: cntlz 2, 3                     # encoding: [0x34,0x00,0x62,0x7c]
             cntlzw 2, 3
-# CHECK-BE: cntlzw. 2, 3                    # encoding: [0x7c,0x62,0x00,0x35]
-# CHECK-LE: cntlzw. 2, 3                    # encoding: [0x35,0x00,0x62,0x7c]
+# CHECK-BE: cntlz. 2, 3                    # encoding: [0x7c,0x62,0x00,0x35]
+# CHECK-LE: cntlz. 2, 3                    # encoding: [0x35,0x00,0x62,0x7c]
             cntlzw. 2, 3
-# CHECK-BE: cntlzw 2, 3                     # encoding: [0x7c,0x62,0x00,0x34]
-# CHECK-LE: cntlzw 2, 3                     # encoding: [0x34,0x00,0x62,0x7c]
+# CHECK-BE: cntlz 2, 3                     # encoding: [0x7c,0x62,0x00,0x34]
+# CHECK-LE: cntlz 2, 3                     # encoding: [0x34,0x00,0x62,0x7c]
             cntlz  2, 3
-# CHECK-BE: cntlzw. 2, 3                    # encoding: [0x7c,0x62,0x00,0x35]
-# CHECK-LE: cntlzw. 2, 3                    # encoding: [0x35,0x00,0x62,0x7c]
+# CHECK-BE: cntlz. 2, 3                    # encoding: [0x7c,0x62,0x00,0x35]
+# CHECK-LE: cntlz. 2, 3                    # encoding: [0x35,0x00,0x62,0x7c]
             cntlz.  2, 3
             cmpb 7, 21, 4
 # CHECK-BE: cmpb 7, 21, 4                   # encoding: [0x7e,0xa7,0x23,0xf8]
diff --git a/test/MC/PowerPC/ppc64-initial-cfa.s b/test/MC/PowerPC/ppc64-initial-cfa.s
index d0bc6b3ecd7..ca97e1b96b0 100644
--- a/test/MC/PowerPC/ppc64-initial-cfa.s
+++ b/test/MC/PowerPC/ppc64-initial-cfa.s
@@ -28,8 +28,8 @@ _proc:
 # STATIC-NEXT:   Relocations [
 # STATIC-NEXT:   ]
 # STATIC-NEXT:   SectionData (
-# STATIC-BE-NEXT:  0000: 00000010 00000000 037A5200 04784101
-# STATIC-LE-NEXT:  0000: 10000000 00000000 037A5200 04784101
+# STATIC-BE-NEXT:  0000: 00000010 00000000 017A5200 04784101
+# STATIC-LE-NEXT:  0000: 10000000 00000000 017A5200 04784101
 # STATIC-BE-NEXT:  0010: 1B0C0100 00000010 00000018 00000000
 # STATIC-LE-NEXT:  0010: 1B0C0100 10000000 18000000 00000000
 # STATIC-BE-NEXT:  0020: 00000004 00000000
@@ -69,8 +69,8 @@ _proc:
 # PIC-NEXT:   Relocations [
 # PIC-NEXT:   ]
 # PIC-NEXT:   SectionData (
-# PIC-BE-NEXT:  0000: 00000010 00000000 037A5200 04784101
-# PIC-LE-NEXT:  0000: 10000000 00000000 037A5200 04784101
+# PIC-BE-NEXT:  0000: 00000010 00000000 017A5200 04784101
+# PIC-LE-NEXT:  0000: 10000000 00000000 017A5200 04784101
 # PIC-BE-NEXT:  0010: 1B0C0100 00000010 00000018 00000000
 # PIC-LE-NEXT:  0010: 1B0C0100 10000000 18000000 00000000
 # PIC-BE-NEXT:  0020: 00000004 00000000
diff --git a/test/MC/PowerPC/qpx.s b/test/MC/PowerPC/qpx.s
index 6c92d715877..a1fb2090f8f 100644
--- a/test/MC/PowerPC/qpx.s
+++ b/test/MC/PowerPC/qpx.s
@@ -1,6 +1,7 @@
 # RUN: llvm-mc -triple powerpc64-bgq-linux --show-encoding %s | FileCheck %s
 
-# FIXME: print qvflogical aliases.
+# CHECK: qvfabs 3, 5                     # encoding: [0x10,0x60,0x2a,0x10]
+         qvfabs %q3, %q5
 
 # CHECK: qvfabs 3, 5                     # encoding: [0x10,0x60,0x2a,0x10]
          qvfabs 3, 5
@@ -8,9 +9,9 @@
          qvfadd 3, 4, 5
 # CHECK: qvfadds 3, 4, 5                 # encoding: [0x00,0x64,0x28,0x2a]
          qvfadds 3, 4, 5
-# CHECK: qvflogical 3, 4, 5, 4           # encoding: [0x10,0x64,0x2a,0x08]
+# CHECK: qvfandc 3, 4, 5                 # encoding: [0x10,0x64,0x2a,0x08]
          qvfandc 3, 4, 5
-# CHECK: qvflogical 3, 4, 5, 1           # encoding: [0x10,0x64,0x28,0x88]
+# CHECK: qvfand 3, 4, 5                  # encoding: [0x10,0x64,0x28,0x88]
          qvfand 3, 4, 5
 # CHECK: qvfcfid 3, 5                    # encoding: [0x10,0x60,0x2e,0x9c]
          qvfcfid 3, 5
@@ -20,11 +21,11 @@
          qvfcfidu 3, 5
 # CHECK: qvfcfidus 3, 5                  # encoding: [0x00,0x60,0x2f,0x9c]
          qvfcfidus 3, 5
-# CHECK: qvflogical 3, 3, 3, 0           # encoding: [0x10,0x63,0x18,0x08]
+# CHECK: qvfclr 3                        # encoding: [0x10,0x63,0x18,0x08]
          qvfclr 3
 # CHECK: qvfcpsgn 3, 4, 5                # encoding: [0x10,0x64,0x28,0x10]
          qvfcpsgn 3, 4, 5
-# CHECK: qvflogical 3, 4, 4, 5           # encoding: [0x10,0x64,0x22,0x88]
+# CHECK: qvfctfb 3, 4                    # encoding: [0x10,0x64,0x22,0x88]
          qvfctfb 3, 4
 # CHECK: qvfctid 3, 5                    # encoding: [0x10,0x60,0x2e,0x5c]
          qvfctid 3, 5
@@ -42,7 +43,7 @@
          qvfctiwuz 3, 5
 # CHECK: qvfctiwz 3, 5                   # encoding: [0x10,0x60,0x28,0x1e]
          qvfctiwz 3, 5
-# CHECK: qvflogical 3, 4, 5, 9           # encoding: [0x10,0x64,0x2c,0x88]
+# CHECK: qvfequ 3, 4, 5                  # encoding: [0x10,0x64,0x2c,0x88]
          qvfequ 3, 4, 5
 # CHECK: qvflogical 3, 4, 5, 12          # encoding: [0x10,0x64,0x2e,0x08]
          qvflogical 3, 4, 5, 12
@@ -62,7 +63,7 @@
          qvfmuls 3, 4, 6
 # CHECK: qvfnabs 3, 5                    # encoding: [0x10,0x60,0x29,0x10]
          qvfnabs 3, 5
-# CHECK: qvflogical 3, 4, 5, 14          # encoding: [0x10,0x64,0x2f,0x08]
+# CHECK: qvfnand 3, 4, 5                 # encoding: [0x10,0x64,0x2f,0x08]
          qvfnand 3, 4, 5
 # CHECK: qvfneg 3, 5                     # encoding: [0x10,0x60,0x28,0x50]
          qvfneg 3, 5
@@ -74,13 +75,13 @@
          qvfnmsub 3, 4, 6, 5
 # CHECK: qvfnmsubs 3, 4, 6, 5            # encoding: [0x00,0x64,0x29,0xbc]
          qvfnmsubs 3, 4, 6, 5
-# CHECK: qvflogical 3, 4, 5, 8           # encoding: [0x10,0x64,0x2c,0x08]
+# CHECK: qvfnor 3, 4, 5                  # encoding: [0x10,0x64,0x2c,0x08]
          qvfnor 3, 4, 5
-# CHECK: qvflogical 3, 4, 4, 10          # encoding: [0x10,0x64,0x25,0x08]
+# CHECK: qvfnot 3, 4                     # encoding: [0x10,0x64,0x25,0x08]
          qvfnot 3, 4
-# CHECK: qvflogical 3, 4, 5, 13          # encoding: [0x10,0x64,0x2e,0x88]
+# CHECK: qvforc 3, 4, 5                  # encoding: [0x10,0x64,0x2e,0x88]
          qvforc 3, 4, 5
-# CHECK: qvflogical 3, 4, 5, 7           # encoding: [0x10,0x64,0x2b,0x88]
+# CHECK: qvfor 3, 4, 5                   # encoding: [0x10,0x64,0x2b,0x88]
          qvfor 3, 4, 5
 # CHECK: qvfperm 3, 4, 5, 6              # encoding: [0x10,0x64,0x29,0x8c]
          qvfperm 3, 4, 5, 6
@@ -104,7 +105,7 @@
          qvfrsqrtes 3, 5
 # CHECK: qvfsel 3, 4, 6, 5               # encoding: [0x10,0x64,0x29,0xae]
          qvfsel 3, 4, 6, 5
-# CHECK: qvflogical 3, 3, 3, 15          # encoding: [0x10,0x63,0x1f,0x88]
+# CHECK: qvfset 3                        # encoding: [0x10,0x63,0x1f,0x88]
          qvfset 3
 # CHECK: qvfsub 3, 4, 5                  # encoding: [0x10,0x64,0x28,0x28]
          qvfsub 3, 4, 5
@@ -118,7 +119,7 @@
          qvfxmul 3, 4, 6
 # CHECK: qvfxmuls 3, 4, 6                # encoding: [0x00,0x64,0x01,0xa2]
          qvfxmuls 3, 4, 6
-# CHECK: qvflogical 3, 4, 5, 6           # encoding: [0x10,0x64,0x2b,0x08]
+# CHECK: qvfxor 3, 4, 5                  # encoding: [0x10,0x64,0x2b,0x08]
          qvfxor 3, 4, 5
 # CHECK: qvfxxcpnmadd 3, 4, 6, 5         # encoding: [0x10,0x64,0x29,0x86]
          qvfxxcpnmadd 3, 4, 6, 5
diff --git a/test/MC/PowerPC/st-other-crash.s b/test/MC/PowerPC/st-other-crash.s
new file mode 100644
index 00000000000..fcc56ad70c5
--- /dev/null
+++ b/test/MC/PowerPC/st-other-crash.s
@@ -0,0 +1,21 @@
+// RUN: llvm-mc < %s -filetype=obj -triple powerpc64le-pc-linux | \
+// RUN:   llvm-readobj -t | FileCheck %s
+
+// This used to crash. Make sure it produce the correct symbol.
+
+// CHECK:       Symbol {
+// CHECK:         Name: _ZN4llvm11SmallVectorIcLj0EEC2Ev (12)
+// CHECK-NEXT:    Value: 0x0
+// CHECK-NEXT:    Size: 0
+// CHECK-NEXT:    Binding: Local (0x0)
+// CHECK-NEXT:    Type: None (0x0)
+// CHECK-NEXT:    Other: 64
+// CHECK-NEXT:    Section: .group (0x5)
+// CHECK-NEXT:  }
+
+
+	.section	.text._ZN4llvm11SmallVectorIcLj0EEC2Ev,"axG",@progbits,_ZN4llvm11SmallVectorIcLj0EEC2Ev,comdat
+.Ltmp2:
+	addis 2, 12, .TOC.-.Ltmp2@ha
+.Ltmp3:
+	.localentry	_ZN4llvm11SmallVectorIcLj0EEC2Ev, .Ltmp3-.Ltmp2
diff --git a/test/MC/PowerPC/vsx.s b/test/MC/PowerPC/vsx.s
index c317c37de5f..352fc517380 100644
--- a/test/MC/PowerPC/vsx.s
+++ b/test/MC/PowerPC/vsx.s
@@ -1,30 +1,52 @@
 # RUN: llvm-mc -triple powerpc64-unknown-linux-gnu --show-encoding %s | FileCheck -check-prefix=CHECK-BE %s
 # RUN: llvm-mc -triple powerpc64le-unknown-linux-gnu --show-encoding %s | FileCheck -check-prefix=CHECK-LE %s
 
-# CHECK-BE: lxsdx 39, 5, 31                     # encoding: [0x7c,0xe5,0xfc,0x99]
-# CHECK-LE: lxsdx 39, 5, 31                     # encoding: [0x99,0xfc,0xe5,0x7c]
+# CHECK-BE: xxswapd 7, 63                      # encoding: [0xf0,0xff,0xfa,0x56]
+# CHECK-LE: xxswapd 7, 63                      # encoding: [0x56,0xfa,0xff,0xf0]
+            xxswapd %vs7, %vs63
+
+# CHECK-BE: lxsdx 39, 5, 31                    # encoding: [0x7c,0xe5,0xfc,0x99]
+# CHECK-LE: lxsdx 39, 5, 31                    # encoding: [0x99,0xfc,0xe5,0x7c]
             lxsdx 39, 5, 31
-# CHECK-BE: lxvd2x 39, 5, 31                    # encoding: [0x7c,0xe5,0xfe,0x99]
-# CHECK-LE: lxvd2x 39, 5, 31                    # encoding: [0x99,0xfe,0xe5,0x7c]
+# CHECK-BE: lxsiwax 39, 5, 31                  # encoding: [0x7c,0xe5,0xf8,0x99]
+# CHECK-LE: lxsiwax 39, 5, 31                  # encoding: [0x99,0xf8,0xe5,0x7c]
+            lxsiwax 39, 5, 31
+# CHECK-BE: lxsiwzx 39, 5, 31                  # encoding: [0x7c,0xe5,0xf8,0x19]
+# CHECK-LE: lxsiwzx 39, 5, 31                  # encoding: [0x19,0xf8,0xe5,0x7c]
+            lxsiwzx 39, 5, 31
+# CHECK-BE: lxsspx 39, 5, 31                   # encoding: [0x7c,0xe5,0xfc,0x19]
+# CHECK-LE: lxsspx 39, 5, 31                   # encoding: [0x19,0xfc,0xe5,0x7c]
+            lxsspx 39, 5, 31
+# CHECK-BE: lxvd2x 39, 5, 31                   # encoding: [0x7c,0xe5,0xfe,0x99]
+# CHECK-LE: lxvd2x 39, 5, 31                   # encoding: [0x99,0xfe,0xe5,0x7c]
             lxvd2x 39, 5, 31
-# CHECK-BE: lxvdsx 39, 5, 31                    # encoding: [0x7c,0xe5,0xfa,0x99]
-# CHECK-LE: lxvdsx 39, 5, 31                    # encoding: [0x99,0xfa,0xe5,0x7c]
+# CHECK-BE: lxvdsx 39, 5, 31                   # encoding: [0x7c,0xe5,0xfa,0x99]
+# CHECK-LE: lxvdsx 39, 5, 31                   # encoding: [0x99,0xfa,0xe5,0x7c]
             lxvdsx 39, 5, 31
-# CHECK-BE: lxvw4x 39, 5, 31                    # encoding: [0x7c,0xe5,0xfe,0x19]
-# CHECK-LE: lxvw4x 39, 5, 31                    # encoding: [0x19,0xfe,0xe5,0x7c]
+# CHECK-BE: lxvw4x 39, 5, 31                   # encoding: [0x7c,0xe5,0xfe,0x19]
+# CHECK-LE: lxvw4x 39, 5, 31                   # encoding: [0x19,0xfe,0xe5,0x7c]
             lxvw4x 39, 5, 31
-# CHECK-BE: stxsdx 40, 5, 31                    # encoding: [0x7d,0x05,0xfd,0x99]
-# CHECK-LE: stxsdx 40, 5, 31                    # encoding: [0x99,0xfd,0x05,0x7d]
+# CHECK-BE: stxsdx 40, 5, 31                   # encoding: [0x7d,0x05,0xfd,0x99]
+# CHECK-LE: stxsdx 40, 5, 31                   # encoding: [0x99,0xfd,0x05,0x7d]
             stxsdx 40, 5, 31
-# CHECK-BE: stxvd2x 40, 5, 31                   # encoding: [0x7d,0x05,0xff,0x99]
-# CHECK-LE: stxvd2x 40, 5, 31                   # encoding: [0x99,0xff,0x05,0x7d]
+# CHECK-BE: stxsiwx 40, 5, 31                  # encoding: [0x7d,0x05,0xf9,0x19]
+# CHECK-LE: stxsiwx 40, 5, 31                  # encoding: [0x19,0xf9,0x05,0x7d]
+            stxsiwx 40, 5, 31
+# CHECK-BE: stxsspx 40, 5, 31                  # encoding: [0x7d,0x05,0xfd,0x19]
+# CHECK-LE: stxsspx 40, 5, 31                  # encoding: [0x19,0xfd,0x05,0x7d]
+            stxsspx 40, 5, 31
+# CHECK-BE: stxvd2x 40, 5, 31                  # encoding: [0x7d,0x05,0xff,0x99]
+# CHECK-LE: stxvd2x 40, 5, 31                  # encoding: [0x99,0xff,0x05,0x7d]
             stxvd2x 40, 5, 31
-# CHECK-BE: stxvw4x 40, 5, 31                   # encoding: [0x7d,0x05,0xff,0x19]
-# CHECK-LE: stxvw4x 40, 5, 31                   # encoding: [0x19,0xff,0x05,0x7d]
+# CHECK-BE: stxvw4x 40, 5, 31                  # encoding: [0x7d,0x05,0xff,0x19]
+# CHECK-LE: stxvw4x 40, 5, 31                  # encoding: [0x19,0xff,0x05,0x7d]
             stxvw4x 40, 5, 31
 # CHECK-BE: xsabsdp 7, 27                      # encoding: [0xf0,0xe0,0xdd,0x64]
 # CHECK-LE: xsabsdp 7, 27                      # encoding: [0x64,0xdd,0xe0,0xf0]
             xsabsdp 7, 27
+# CHECK-BE: xsaddsp 7, 63, 27                  # encoding: [0xf0,0xff,0xd8,0x04]
+# CHECK-LE: xsaddsp 7, 63, 27                  # encoding: [0x04,0xd8,0xff,0xf0]
+            xsaddsp 7, 63, 27
 # CHECK-BE: xsadddp 7, 63, 27                  # encoding: [0xf0,0xff,0xd9,0x04]
 # CHECK-LE: xsadddp 7, 63, 27                  # encoding: [0x04,0xd9,0xff,0xf0]
             xsadddp 7, 63, 27
@@ -61,6 +83,9 @@
 # CHECK-BE: xscvuxddp 7, 27                    # encoding: [0xf0,0xe0,0xdd,0xa0]
 # CHECK-LE: xscvuxddp 7, 27                    # encoding: [0xa0,0xdd,0xe0,0xf0]
             xscvuxddp 7, 27
+# CHECK-BE: xsdivsp 7, 63, 27                  # encoding: [0xf0,0xff,0xd8,0xc4]
+# CHECK-LE: xsdivsp 7, 63, 27                  # encoding: [0xc4,0xd8,0xff,0xf0]
+            xsdivsp 7, 63, 27
 # CHECK-BE: xsdivdp 7, 63, 27                  # encoding: [0xf0,0xff,0xd9,0xc4]
 # CHECK-LE: xsdivdp 7, 63, 27                  # encoding: [0xc4,0xd9,0xff,0xf0]
             xsdivdp 7, 63, 27
@@ -70,6 +95,12 @@
 # CHECK-BE: xsmaddmdp 7, 63, 27                # encoding: [0xf0,0xff,0xd9,0x4c]
 # CHECK-LE: xsmaddmdp 7, 63, 27                # encoding: [0x4c,0xd9,0xff,0xf0]
             xsmaddmdp 7, 63, 27
+# CHECK-BE: xsmaddasp 7, 63, 27                # encoding: [0xf0,0xff,0xd8,0x0c]
+# CHECK-LE: xsmaddasp 7, 63, 27                # encoding: [0x0c,0xd8,0xff,0xf0]
+            xsmaddasp 7, 63, 27
+# CHECK-BE: xsmaddmsp 7, 63, 27                # encoding: [0xf0,0xff,0xd8,0x4c]
+# CHECK-LE: xsmaddmsp 7, 63, 27                # encoding: [0x4c,0xd8,0xff,0xf0]
+            xsmaddmsp 7, 63, 27
 # CHECK-BE: xsmaxdp 7, 63, 27                  # encoding: [0xf0,0xff,0xdd,0x04]
 # CHECK-LE: xsmaxdp 7, 63, 27                  # encoding: [0x04,0xdd,0xff,0xf0]
             xsmaxdp 7, 63, 27
@@ -82,6 +113,15 @@
 # CHECK-BE: xsmsubmdp 7, 63, 27                # encoding: [0xf0,0xff,0xd9,0xcc]
 # CHECK-LE: xsmsubmdp 7, 63, 27                # encoding: [0xcc,0xd9,0xff,0xf0]
             xsmsubmdp 7, 63, 27
+# CHECK-BE: xsmsubasp 7, 63, 27                # encoding: [0xf0,0xff,0xd8,0x8c]
+# CHECK-LE: xsmsubasp 7, 63, 27                # encoding: [0x8c,0xd8,0xff,0xf0]
+            xsmsubasp 7, 63, 27
+# CHECK-BE: xsmsubmsp 7, 63, 27                # encoding: [0xf0,0xff,0xd8,0xcc]
+# CHECK-LE: xsmsubmsp 7, 63, 27                # encoding: [0xcc,0xd8,0xff,0xf0]
+            xsmsubmsp 7, 63, 27
+# CHECK-BE: xsmulsp 7, 63, 27                  # encoding: [0xf0,0xff,0xd8,0x84]
+# CHECK-LE: xsmulsp 7, 63, 27                  # encoding: [0x84,0xd8,0xff,0xf0]
+            xsmulsp 7, 63, 27
 # CHECK-BE: xsmuldp 7, 63, 27                  # encoding: [0xf0,0xff,0xd9,0x84]
 # CHECK-LE: xsmuldp 7, 63, 27                  # encoding: [0x84,0xd9,0xff,0xf0]
             xsmuldp 7, 63, 27
@@ -103,6 +143,18 @@
 # CHECK-BE: xsnmsubmdp 7, 63, 27               # encoding: [0xf0,0xff,0xdd,0xcc]
 # CHECK-LE: xsnmsubmdp 7, 63, 27               # encoding: [0xcc,0xdd,0xff,0xf0]
             xsnmsubmdp 7, 63, 27
+# CHECK-BE: xsnmaddasp 7, 63, 27               # encoding: [0xf0,0xff,0xdc,0x0c]
+# CHECK-LE: xsnmaddasp 7, 63, 27               # encoding: [0x0c,0xdc,0xff,0xf0]
+            xsnmaddasp 7, 63, 27
+# CHECK-BE: xsnmaddmsp 7, 63, 27               # encoding: [0xf0,0xff,0xdc,0x4c]
+# CHECK-LE: xsnmaddmsp 7, 63, 27               # encoding: [0x4c,0xdc,0xff,0xf0]
+            xsnmaddmsp 7, 63, 27
+# CHECK-BE: xsnmsubasp 7, 63, 27               # encoding: [0xf0,0xff,0xdc,0x8c]
+# CHECK-LE: xsnmsubasp 7, 63, 27               # encoding: [0x8c,0xdc,0xff,0xf0]
+            xsnmsubasp 7, 63, 27
+# CHECK-BE: xsnmsubmsp 7, 63, 27               # encoding: [0xf0,0xff,0xdc,0xcc]
+# CHECK-LE: xsnmsubmsp 7, 63, 27               # encoding: [0xcc,0xdc,0xff,0xf0]
+            xsnmsubmsp 7, 63, 27
 # CHECK-BE: xsrdpi 7, 27                       # encoding: [0xf0,0xe0,0xd9,0x24]
 # CHECK-LE: xsrdpi 7, 27                       # encoding: [0x24,0xd9,0xe0,0xf0]
             xsrdpi 7, 27
@@ -118,15 +170,27 @@
 # CHECK-BE: xsrdpiz 7, 27                      # encoding: [0xf0,0xe0,0xd9,0x64]
 # CHECK-LE: xsrdpiz 7, 27                      # encoding: [0x64,0xd9,0xe0,0xf0]
             xsrdpiz 7, 27
+# CHECK-BE: xsresp 7, 27                       # encoding: [0xf0,0xe0,0xd8,0x68]
+# CHECK-LE: xsresp 7, 27                       # encoding: [0x68,0xd8,0xe0,0xf0]
+            xsresp 7, 27
 # CHECK-BE: xsredp 7, 27                       # encoding: [0xf0,0xe0,0xd9,0x68]
 # CHECK-LE: xsredp 7, 27                       # encoding: [0x68,0xd9,0xe0,0xf0]
             xsredp 7, 27
+# CHECK-BE: xsrsqrtesp 7, 27                   # encoding: [0xf0,0xe0,0xd8,0x28]
+# CHECK-LE: xsrsqrtesp 7, 27                   # encoding: [0x28,0xd8,0xe0,0xf0]
+            xsrsqrtesp 7, 27
 # CHECK-BE: xsrsqrtedp 7, 27                   # encoding: [0xf0,0xe0,0xd9,0x28]
 # CHECK-LE: xsrsqrtedp 7, 27                   # encoding: [0x28,0xd9,0xe0,0xf0]
             xsrsqrtedp 7, 27
+# CHECK-BE: xssqrtsp 7, 27                     # encoding: [0xf0,0xe0,0xd8,0x2c]
+# CHECK-LE: xssqrtsp 7, 27                     # encoding: [0x2c,0xd8,0xe0,0xf0]
+            xssqrtsp 7, 27
 # CHECK-BE: xssqrtdp 7, 27                     # encoding: [0xf0,0xe0,0xd9,0x2c]
 # CHECK-LE: xssqrtdp 7, 27                     # encoding: [0x2c,0xd9,0xe0,0xf0]
             xssqrtdp 7, 27
+# CHECK-BE: xssubsp 7, 63, 27                  # encoding: [0xf0,0xff,0xd8,0x44]
+# CHECK-LE: xssubsp 7, 63, 27                  # encoding: [0x44,0xd8,0xff,0xf0]
+            xssubsp 7, 63, 27
 # CHECK-BE: xssubdp 7, 63, 27                  # encoding: [0xf0,0xff,0xd9,0x44]
 # CHECK-LE: xssubdp 7, 63, 27                  # encoding: [0x44,0xd9,0xff,0xf0]
             xssubdp 7, 63, 27
@@ -274,11 +338,11 @@
 # CHECK-BE: xvminsp 7, 63, 27                  # encoding: [0xf0,0xff,0xde,0x44]
 # CHECK-LE: xvminsp 7, 63, 27                  # encoding: [0x44,0xde,0xff,0xf0]
             xvminsp 7, 63, 27
-# CHECK-BE: xvcpsgndp 7, 63, 63                # encoding: [0xf0,0xff,0xff,0x86]
-# CHECK-LE: xvcpsgndp 7, 63, 63                # encoding: [0x86,0xff,0xff,0xf0]
+# CHECK-BE: xvmovdp 7, 63                      # encoding: [0xf0,0xff,0xff,0x86]
+# CHECK-LE: xvmovdp 7, 63                      # encoding: [0x86,0xff,0xff,0xf0]
             xvmovdp 7, 63
-# CHECK-BE: xvcpsgnsp 7, 63, 63                # encoding: [0xf0,0xff,0xfe,0x86]
-# CHECK-LE: xvcpsgnsp 7, 63, 63                # encoding: [0x86,0xfe,0xff,0xf0]
+# CHECK-BE: xvmovsp 7, 63                      # encoding: [0xf0,0xff,0xfe,0x86]
+# CHECK-LE: xvmovsp 7, 63                      # encoding: [0x86,0xfe,0xff,0xf0]
             xvmovsp 7, 63
 # CHECK-BE: xvmsubadp 7, 63, 27                # encoding: [0xf0,0xff,0xdb,0x8c]
 # CHECK-LE: xvmsubadp 7, 63, 27                # encoding: [0x8c,0xdb,0xff,0xf0]
@@ -424,14 +488,14 @@
 # CHECK-BE: xxlxor 7, 63, 27                   # encoding: [0xf0,0xff,0xdc,0xd4]
 # CHECK-LE: xxlxor 7, 63, 27                   # encoding: [0xd4,0xdc,0xff,0xf0]
             xxlxor 7, 63, 27
-# CHECK-BE: xxpermdi 7, 63, 27, 0              # encoding: [0xf0,0xff,0xd8,0x54]
-# CHECK-LE: xxpermdi 7, 63, 27, 0              # encoding: [0x54,0xd8,0xff,0xf0]
+# CHECK-BE: xxmrghd 7, 63, 27                  # encoding: [0xf0,0xff,0xd8,0x54]
+# CHECK-LE: xxmrghd 7, 63, 27                  # encoding: [0x54,0xd8,0xff,0xf0]
             xxmrghd 7, 63, 27
 # CHECK-BE: xxmrghw 7, 63, 27                  # encoding: [0xf0,0xff,0xd8,0x94]
 # CHECK-LE: xxmrghw 7, 63, 27                  # encoding: [0x94,0xd8,0xff,0xf0]
             xxmrghw 7, 63, 27
-# CHECK-BE: xxpermdi 7, 63, 27, 3              # encoding: [0xf0,0xff,0xdb,0x54]
-# CHECK-LE: xxpermdi 7, 63, 27, 3              # encoding: [0x54,0xdb,0xff,0xf0]
+# CHECK-BE: xxmrgld 7, 63, 27                  # encoding: [0xf0,0xff,0xdb,0x54]
+# CHECK-LE: xxmrgld 7, 63, 27                  # encoding: [0x54,0xdb,0xff,0xf0]
             xxmrgld 7, 63, 27
 # CHECK-BE: xxmrglw 7, 63, 27                  # encoding: [0xf0,0xff,0xd9,0x94]
 # CHECK-LE: xxmrglw 7, 63, 27                  # encoding: [0x94,0xd9,0xff,0xf0]
@@ -445,14 +509,14 @@
 # CHECK-BE: xxsldwi 7, 63, 27, 1               # encoding: [0xf0,0xff,0xd9,0x14]
 # CHECK-LE: xxsldwi 7, 63, 27, 1               # encoding: [0x14,0xd9,0xff,0xf0]
             xxsldwi 7, 63, 27, 1
-# CHECK-BE: xxpermdi 7, 63, 63, 3              # encoding: [0xf0,0xff,0xfb,0x56]
-# CHECK-LE: xxpermdi 7, 63, 63, 3              # encoding: [0x56,0xfb,0xff,0xf0]
+# CHECK-BE: xxspltd 7, 63, 1                   # encoding: [0xf0,0xff,0xfb,0x56]
+# CHECK-LE: xxspltd 7, 63, 1                   # encoding: [0x56,0xfb,0xff,0xf0]
             xxspltd 7, 63, 1
 # CHECK-BE: xxspltw 7, 27, 3                   # encoding: [0xf0,0xe3,0xda,0x90]
 # CHECK-LE: xxspltw 7, 27, 3                   # encoding: [0x90,0xda,0xe3,0xf0]
             xxspltw 7, 27, 3
-# CHECK-BE: xxpermdi 7, 63, 63, 2              # encoding: [0xf0,0xff,0xfa,0x56]
-# CHECK-LE: xxpermdi 7, 63, 63, 2              # encoding: [0x56,0xfa,0xff,0xf0]
+# CHECK-BE: xxswapd 7, 63                      # encoding: [0xf0,0xff,0xfa,0x56]
+# CHECK-LE: xxswapd 7, 63                      # encoding: [0x56,0xfa,0xff,0xf0]
             xxswapd 7, 63
 
 # Move to/from VSR
diff --git a/test/MC/R600/lit.local.cfg b/test/MC/R600/lit.local.cfg
deleted file mode 100644
index ad9ce2541ef..00000000000
--- a/test/MC/R600/lit.local.cfg
+++ /dev/null
@@ -1,2 +0,0 @@
-if not 'R600' in config.root.targets:
-    config.unsupported = True
diff --git a/test/MC/R600/vop1.s b/test/MC/R600/vop1.s
deleted file mode 100644
index 9c9a6b2c0b7..00000000000
--- a/test/MC/R600/vop1.s
+++ /dev/null
@@ -1,182 +0,0 @@
-// RUN: llvm-mc -arch=amdgcn -show-encoding %s | FileCheck %s
-// RUN: llvm-mc -arch=amdgcn -mcpu=SI -show-encoding %s | FileCheck %s
-
-// CHECK: v_nop ; encoding: [0x00,0x00,0x00,0x7e]
-v_nop
-
-// CHECK: v_mov_b32_e32 v1, v2 ; encoding: [0x02,0x03,0x02,0x7e]
-v_mov_b32 v1, v2
-
-// CHECK: v_readfirstlane_b32 s1, v2 ; encoding: [0x02,0x05,0x02,0x7e]
-v_readfirstlane_b32 s1, v2
-
-// CHECK: v_cvt_i32_f64_e32 v1, v[2:3] ; encoding: [0x02,0x07,0x02,0x7e]
-v_cvt_i32_f64 v1, v[2:3]
-
-// CHECK: v_cvt_f64_i32_e32 v[1:2], v2 ; encoding: [0x02,0x09,0x02,0x7e]
-v_cvt_f64_i32 v[1:2], v2
-
-// CHECK: v_cvt_f32_i32_e32 v1, v2 ; encoding: [0x02,0x0b,0x02,0x7e]
-v_cvt_f32_i32 v1, v2
-
-// CHECK: v_cvt_f32_u32_e32 v1, v2 ; encoding: [0x02,0x0d,0x02,0x7e]
-v_cvt_f32_u32 v1, v2
-
-// CHECK: v_cvt_u32_f32_e32 v1, v2 ; encoding: [0x02,0x0f,0x02,0x7e
-v_cvt_u32_f32 v1, v2
-
-// CHECK: v_cvt_i32_f32_e32 v1, v2 ; encoding: [0x02,0x11,0x02,0x7e]
-v_cvt_i32_f32 v1, v2
-
-// CHECK: v_mov_fed_b32_e32 v1, v2 ; encoding: [0x02,0x13,0x02,0x7e]
-v_mov_fed_b32 v1, v2
-
-// CHECK: v_cvt_f16_f32_e32 v1, v2 ; encoding: [0x02,0x15,0x02,0x7e]
-v_cvt_f16_f32 v1, v2
-
-// CHECK: v_cvt_f32_f16_e32 v1, v2 ; encoding: [0x02,0x17,0x02,0x7e]
-v_cvt_f32_f16 v1, v2
-
-// CHECK: v_cvt_rpi_i32_f32_e32 v1, v2 ; encoding: [0x02,0x19,0x02,0x7e]
-v_cvt_rpi_i32_f32 v1, v2
-
-// CHECK: v_cvt_flr_i32_f32_e32 v1, v2 ; encoding: [0x02,0x1b,0x02,0x7e]
-v_cvt_flr_i32_f32 v1, v2
-
-// CHECK: v_cvt_off_f32_i4_e32 v1, v2 ; encoding: [0x02,0x1d,0x02,0x7e]
-v_cvt_off_f32_i4_e32 v1, v2
-
-// CHECK: v_cvt_f32_f64_e32 v1, v[2:3] ; encoding: [0x02,0x1f,0x02,0x7e]
-v_cvt_f32_f64 v1, v[2:3]
-
-// CHECK: v_cvt_f64_f32_e32 v[1:2], v2 ; encoding: [0x02,0x21,0x02,0x7e]
-v_cvt_f64_f32 v[1:2], v2
-
-// CHECK: v_cvt_f32_ubyte0_e32 v1, v2 ; encoding: [0x02,0x23,0x02,0x7e]
-v_cvt_f32_ubyte0 v1, v2
-
-// CHECK: v_cvt_f32_ubyte1_e32 v1, v2 ; encoding: [0x02,0x25,0x02,0x7e]
-v_cvt_f32_ubyte1_e32 v1, v2
-
-// CHECK: v_cvt_f32_ubyte2_e32 v1, v2 ; encoding: [0x02,0x27,0x02,0x7e]
-v_cvt_f32_ubyte2 v1, v2
-
-// CHECK: v_cvt_f32_ubyte3_e32 v1, v2 ; encoding: [0x02,0x29,0x02,0x7e]
-v_cvt_f32_ubyte3 v1, v2
-
-// CHECK: v_cvt_u32_f64_e32 v1, v[2:3] ; encoding: [0x02,0x2b,0x02,0x7e]
-v_cvt_u32_f64 v1, v[2:3]
-
-// CHECK: v_cvt_f64_u32_e32 v[1:2], v2 ; encoding: [0x02,0x2d,0x02,0x7e]
-v_cvt_f64_u32 v[1:2], v2
-
-// CHECK: v_fract_f32_e32 v1, v2 ; encoding: [0x02,0x41,0x02,0x7e]
-v_fract_f32 v1, v2
-
-// CHECK: v_trunc_f32_e32 v1, v2 ; encoding: [0x02,0x43,0x02,0x7e]
-v_trunc_f32 v1, v2
-
-// CHECK: v_ceil_f32_e32 v1, v2 ; encoding: [0x02,0x45,0x02,0x7e]
-v_ceil_f32 v1, v2
-
-// CHECK: v_rndne_f32_e32 v1, v2 ; encoding: [0x02,0x47,0x02,0x7e]
-v_rndne_f32 v1, v2
-
-// CHECK: v_floor_f32_e32 v1, v2 ; encoding: [0x02,0x49,0x02,0x7e]
-v_floor_f32_e32 v1, v2
-
-// CHECK: v_exp_f32_e32 v1, v2 ; encoding: [0x02,0x4b,0x02,0x7e]
-v_exp_f32 v1, v2
-
-// CHECK: v_log_clamp_f32_e32 v1, v2 ; encoding: [0x02,0x4d,0x02,0x7e]
-v_log_clamp_f32 v1, v2
-
-// CHECK: v_log_f32_e32 v1, v2 ; encoding: [0x02,0x4f,0x02,0x7e]
-v_log_f32 v1, v2
-
-// CHECK: v_rcp_clamp_f32_e32 v1, v2 ; encoding: [0x02,0x51,0x02,0x7e]
-v_rcp_clamp_f32 v1, v2
-
-// CHECK: v_rcp_legacy_f32_e32 v1, v2 ; encoding: [0x02,0x53,0x02,0x7e]
-v_rcp_legacy_f32 v1, v2
-
-// CHECK: v_rcp_f32_e32 v1, v2 ; encoding: [0x02,0x55,0x02,0x7e]
-v_rcp_f32 v1, v2
-
-// CHECK: v_rcp_iflag_f32_e32 v1, v2 ; encoding: [0x02,0x57,0x02,0x7e]
-v_rcp_iflag_f32 v1, v2
-
-// CHECK: v_rsq_clamp_f32_e32 v1, v2 ; encoding: [0x02,0x59,0x02,0x7e]
-v_rsq_clamp_f32 v1, v2
-
-// CHECK: v_rsq_legacy_f32_e32 v1, v2 ; encoding: [0x02,0x5b,0x02,0x7e]
-v_rsq_legacy_f32 v1, v2
-
-// CHECK: v_rsq_f32_e32 v1, v2 ; encoding: [0x02,0x5d,0x02,0x7e]
-v_rsq_f32_e32 v1, v2
-
-// CHECK: v_rcp_f64_e32 v[1:2], v[2:3] ; encoding: [0x02,0x5f,0x02,0x7e]
-v_rcp_f64 v[1:2], v[2:3]
-
-// CHECK: v_rcp_clamp_f64_e32 v[1:2], v[2:3] ; encoding: [0x02,0x61,0x02,0x7e]
-v_rcp_clamp_f64 v[1:2], v[2:3]
-
-// CHECK: v_rsq_f64_e32 v[1:2], v[2:3] ; encoding: [0x02,0x63,0x02,0x7e]
-v_rsq_f64 v[1:2], v[2:3]
-
-// CHECK: v_rsq_clamp_f64_e32 v[1:2], v[2:3] ; encoding: [0x02,0x65,0x02,0x7e]
-v_rsq_clamp_f64 v[1:2], v[2:3]
-
-// CHECK: v_sqrt_f32_e32 v1, v2 ; encoding: [0x02,0x67,0x02,0x7e]
-v_sqrt_f32 v1, v2
-
-// CHECK: v_sqrt_f64_e32 v[1:2], v[2:3] ; encoding: [0x02,0x69,0x02,0x7e]
-v_sqrt_f64 v[1:2], v[2:3]
-
-// CHECK: v_sin_f32_e32 v1, v2 ; encoding: [0x02,0x6b,0x02,0x7e]
-v_sin_f32 v1, v2
-
-// CHECK: v_cos_f32_e32 v1, v2 ; encoding: [0x02,0x6d,0x02,0x7e]
-v_cos_f32 v1, v2
-
-// CHECK: v_not_b32_e32 v1, v2 ; encoding: [0x02,0x6f,0x02,0x7e]
-v_not_b32 v1, v2
-
-// CHECK: v_bfrev_b32_e32 v1, v2 ; encoding: [0x02,0x71,0x02,0x7e]
-v_bfrev_b32 v1, v2
-
-// CHECK: v_ffbh_u32_e32 v1, v2 ; encoding: [0x02,0x73,0x02,0x7e]
-v_ffbh_u32 v1, v2
-
-// CHECK: v_ffbl_b32_e32 v1, v2 ; encoding: [0x02,0x75,0x02,0x7e]
-v_ffbl_b32 v1, v2
-
-// CHECK: v_ffbh_i32_e32 v1, v2 ; encoding: [0x02,0x77,0x02,0x7e]
-v_ffbh_i32_e32 v1, v2
-
-// CHECK: v_frexp_exp_i32_f64_e32 v1, v[2:3] ; encoding: [0x02,0x79,0x02,0x7e]
-v_frexp_exp_i32_f64 v1, v[2:3]
-
-// CHECK: v_frexp_mant_f64_e32 v[1:2], v[2:3] ; encoding: [0x02,0x7b,0x02,0x7e]
-v_frexp_mant_f64 v[1:2], v[2:3]
-
-// CHECK: v_fract_f64_e32 v[1:2], v[2:3] ; encoding: [0x02,0x7d,0x02,0x7e]
-v_fract_f64 v[1:2], v[2:3]
-
-// CHECK: v_frexp_exp_i32_f32_e32 v1, v2 ; encoding: [0x02,0x7f,0x02,0x7e]
-v_frexp_exp_i32_f32 v1, v2
-
-// CHECK: v_frexp_mant_f32_e32 v1, v2 ; encoding: [0x02,0x81,0x02,0x7e]
-v_frexp_mant_f32 v1, v2
-
-// CHECK: v_clrexcp ; encoding: [0x00,0x82,0x00,0x7e]
-v_clrexcp
-
-// CHECK: v_movreld_b32_e32 v1, v2 ; encoding: [0x02,0x85,0x02,0x7e]
-v_movreld_b32 v1, v2
-
-// CHECK: v_movrels_b32_e32 v1, v2 ; encoding: [0x02,0x87,0x02,0x7e]
-v_movrels_b32 v1, v2
-
-// CHECK: v_movrelsd_b32_e32 v1, v2 ; encoding: [0x02,0x89,0x02,0x7e]
-v_movrelsd_b32 v1, v2
diff --git a/test/MC/R600/vop2.s b/test/MC/R600/vop2.s
deleted file mode 100644
index 678008852f1..00000000000
--- a/test/MC/R600/vop2.s
+++ /dev/null
@@ -1,242 +0,0 @@
-// RUN: llvm-mc -arch=amdgcn -show-encoding %s | FileCheck %s
-// RUN: llvm-mc -arch=amdgcn -mcpu=SI -show-encoding %s | FileCheck %s
-
-//===----------------------------------------------------------------------===//
-// Generic Checks for floating-point instructions (These have modifiers).
-//===----------------------------------------------------------------------===//
-
-// TODO: 64-bit encoding of instructions with modifiers
-
-// _e32 suffix
-// CHECK: v_add_f32_e32 v1, v2, v3 ; encoding: [0x02,0x07,0x02,0x06]
-v_add_f32_e32 v1, v2, v3
-
-// src0 inline immediate
-// CHECK: v_add_f32_e32 v1, 1.0, v3 ; encoding: [0xf2,0x06,0x02,0x06]
-v_add_f32 v1, 1.0, v3
-
-// src0 negative inline immediate
-// CHECK: v_add_f32_e32 v1, -1.0, v3 ; encoding: [0xf3,0x06,0x02,0x06]
-v_add_f32 v1, -1.0, v3
-
-// src0 literal
-// CHECK: v_add_f32_e32 v1, 0x42c80000, v3 ; encoding: [0xff,0x06,0x02,0x06,0x00,0x00,0xc8,0x42]
-v_add_f32 v1, 100.0, v3
-
-// src0 negative literal
-// CHECK: v_add_f32_e32 v1, 0xc2c80000, v3 ; encoding: [0xff,0x06,0x02,0x06,0x00,0x00,0xc8,0xc2]
-v_add_f32 v1, -100.0, v3
-
-//===----------------------------------------------------------------------===//
-// Generic Checks for integer instructions (These don't have modifiers).
-//===----------------------------------------------------------------------===//
-
-// _e32 suffix
-// CHECK: v_mul_i32_i24_e32 v1, v2, v3 ; encoding: [0x02,0x07,0x02,0x12] 
-v_mul_i32_i24_e32 v1, v2, v3
-
-// _e64 suffix
-// CHECK: v_mul_i32_i24_e64 v1, v2, v3 ; encoding: [0x01,0x00,0x12,0xd2,0x02,0x07,0x02,0x00]
-v_mul_i32_i24_e64 v1, v2, v3
-
-// src0 inline
-// CHECK: v_mul_i32_i24_e32 v1, 3, v3 ; encoding: [0x83,0x06,0x02,0x12]
-v_mul_i32_i24 v1, 3, v3
-
-// src0 negative inline
-// CHECK: v_mul_i32_i24_e32 v1, -3, v3 ; encoding: [0xc3,0x06,0x02,0x12]
-v_mul_i32_i24 v1, -3, v3
-
-// src1 inline
-// CHECK: v_mul_i32_i24_e64 v1, v2, 3 ; encoding: [0x01,0x00,0x12,0xd2,0x02,0x07,0x01,0x00]
-v_mul_i32_i24 v1, v2, 3
-
-// src1 negative inline
-// CHECK: v_mul_i32_i24_e64 v1, v2, -3 ; encoding: [0x01,0x00,0x12,0xd2,0x02,0x87,0x01,0x00]
-v_mul_i32_i24 v1, v2, -3
-
-// src0 literal
-// CHECK: v_mul_i32_i24_e32 v1, 0x64, v3 ; encoding: [0xff,0x06,0x02,0x12,0x64,0x00,0x00,0x00]
-v_mul_i32_i24 v1, 100, v3
-
-// src1 negative literal
-// CHECK: v_mul_i32_i24_e32 v1, 0xffffff9c, v3 ; encoding: [0xff,0x06,0x02,0x12,0x9c,0xff,0xff,0xff]
-v_mul_i32_i24 v1, -100, v3
-
-//===----------------------------------------------------------------------===//
-// Checks for legal operands
-//===----------------------------------------------------------------------===//
-
-// src0 sgpr
-// CHECK: v_mul_i32_i24_e32 v1, s2, v3 ; encoding: [0x02,0x06,0x02,0x12]
-v_mul_i32_i24 v1, s2, v3
-
-// src1 sgpr
-// CHECK: v_mul_i32_i24_e64 v1, v2, s3 ; encoding: [0x01,0x00,0x12,0xd2,0x02,0x07,0x00,0x00]
-v_mul_i32_i24 v1, v2, s3
-
-// src0, src1 same sgpr
-// CHECK: v_mul_i32_i24_e64 v1, s2, s2 ; encoding: [0x01,0x00,0x12,0xd2,0x02,0x04,0x00,0x00]
-v_mul_i32_i24 v1, s2, s2
-
-// src0 sgpr, src1 inline
-// CHECK: v_mul_i32_i24_e64 v1, s2, 3 ; encoding: [0x01,0x00,0x12,0xd2,0x02,0x06,0x01,0x00]
-v_mul_i32_i24 v1, s2, 3
-
-// src0 inline src1 sgpr
-// CHECK: v_mul_i32_i24_e64 v1, 3, s3 ; encoding: [0x01,0x00,0x12,0xd2,0x83,0x06,0x00,0x00]
-v_mul_i32_i24 v1, 3, s3
-
-//===----------------------------------------------------------------------===//
-// Instructions
-//===----------------------------------------------------------------------===//
-
-// CHECK: v_cndmask_b32_e32 v1, v2, v3 ; encoding: [0x02,0x07,0x02,0x00]
-v_cndmask_b32 v1, v2, v3
-
-// CHECK: v_readlane_b32 s1, v2, s3 ; encoding: [0x02,0x07,0x02,0x02]
-v_readlane_b32 s1, v2, s3
-
-// CHECK: v_writelane_b32 v1, s2, s3 ; encoding: [0x02,0x06,0x02,0x04]
-v_writelane_b32 v1, s2, s3
-
-// CHECK: v_add_f32_e32 v1, v2, v3 ; encoding: [0x02,0x07,0x02,0x06]
-v_add_f32 v1, v2, v3
-
-// CHECK: v_sub_f32_e32 v1, v2, v3 ; encoding: [0x02,0x07,0x02,0x08]
-v_sub_f32 v1, v2, v3
-
-// CHECK: v_subrev_f32_e32 v1, v2, v3 ; encoding: [0x02,0x07,0x02,0x0a]
-v_subrev_f32 v1, v2, v3
-
-// CHECK: v_mac_legacy_f32_e32 v1, v2, v3 ; encoding: [0x02,0x07,0x02,0x0c]
-v_mac_legacy_f32 v1, v2, v3
-
-// CHECK: v_mul_legacy_f32_e32 v1, v2, v3 ; encoding: [0x02,0x07,0x02,0x0e]
-v_mul_legacy_f32_e32 v1, v2, v3
-
-// CHECK: v_mul_f32_e32 v1, v2, v3 ; encoding: [0x02,0x07,0x02,0x10]
-v_mul_f32 v1, v2, v3
-
-// CHECK: v_mul_i32_i24_e32 v1, v2, v3 ; encoding: [0x02,0x07,0x02,0x12]
-v_mul_i32_i24 v1, v2, v3
-
-// CHECK: v_mul_hi_i32_i24_e32 v1, v2, v3 ; encoding: [0x02,0x07,0x02,0x14]
-v_mul_hi_i32_i24 v1, v2, v3
-
-// CHECK: v_mul_u32_u24_e32 v1, v2, v3 ; encoding: [0x02,0x07,0x02,0x16]
-v_mul_u32_u24 v1, v2, v3
-
-// CHECK: v_mul_hi_u32_u24_e32 v1, v2, v3 ; encoding: [0x02,0x07,0x02,0x18]
-v_mul_hi_u32_u24 v1, v2, v3
-
-// CHECK: v_min_legacy_f32_e32 v1, v2, v3 ; encoding: [0x02,0x07,0x02,0x1a]
-v_min_legacy_f32_e32 v1, v2, v3
-
-// CHECK: v_max_legacy_f32_e32 v1, v2, v3 ; encoding: [0x02,0x07,0x02,0x1c]
-v_max_legacy_f32 v1, v2, v3
-
-// CHECK: v_min_f32_e32 v1, v2, v3 ; encoding: [0x02,0x07,0x02,0x1e]
-v_min_f32_e32 v1, v2, v3
-
-// CHECK: v_max_f32_e32 v1, v2, v3 ; encoding: [0x02,0x07,0x02,0x20]
-v_max_f32 v1, v2 v3
-
-// CHECK: v_min_i32_e32 v1, v2, v3 ; encoding: [0x02,0x07,0x02,0x22]
-v_min_i32 v1, v2, v3
-
-// CHECK: v_max_i32_e32 v1, v2, v3 ; encoding: [0x02,0x07,0x02,0x24]
-v_max_i32 v1, v2, v3
-
-// CHECK: v_min_u32_e32 v1, v2, v3 ; encoding: [0x02,0x07,0x02,0x26]
-v_min_u32 v1, v2, v3
-
-// CHECK: v_max_u32_e32 v1, v2, v3 ; encoding: [0x02,0x07,0x02,0x28]
-v_max_u32 v1, v2, v3
-
-// CHECK: v_lshr_b32_e32 v1, v2, v3 ; encoding: [0x02,0x07,0x02,0x2a]
-v_lshr_b32 v1, v2, v3
-
-// CHECK: v_lshrrev_b32_e32 v1, v2, v3 ; encoding: [0x02,0x07,0x02,0x2c]
-v_lshrrev_b32 v1, v2, v3
-
-// CHECK: v_ashr_i32_e32 v1, v2, v3 ; encoding: [0x02,0x07,0x02,0x2e]
-v_ashr_i32 v1, v2, v3
-
-// CHECK: v_ashrrev_i32_e32 v1, v2, v3 ; encoding: [0x02,0x07,0x02,0x30]
-v_ashrrev_i32 v1, v2, v3
-
-// CHECK: v_lshl_b32_e32 v1, v2, v3 ; encoding: [0x02,0x07,0x02,0x32]
-v_lshl_b32_e32 v1, v2, v3
-
-// CHECK: v_lshlrev_b32_e32 v1, v2, v3 ; encoding: [0x02,0x07,0x02,0x34]
-v_lshlrev_b32 v1, v2, v3
-
-// CHECK: v_and_b32_e32 v1, v2, v3 ; encoding: [0x02,0x07,0x02,0x36]
-v_and_b32 v1, v2, v3
-
-// CHECK: v_or_b32_e32 v1, v2, v3 ; encoding: [0x02,0x07,0x02,0x38]
-v_or_b32 v1, v2, v3
-
-// CHECK: v_xor_b32_e32 v1, v2, v3 ; encoding: [0x02,0x07,0x02,0x3a]
-v_xor_b32 v1, v2, v3
-
-// CHECK: v_bfm_b32_e32 v1, v2, v3 ; encoding: [0x02,0x07,0x02,0x3c]
-v_bfm_b32 v1, v2, v3
-
-// CHECK: v_mac_f32_e32 v1, v2, v3 ; encoding: [0x02,0x07,0x02,0x3e]
-v_mac_f32 v1, v2, v3
-
-// CHECK: v_madmk_f32_e32 v1, v2, v3, 0x42800000 ; encoding: [0x02,0x07,0x02,0x40,0x00,0x00,0x80,0x42]
-v_madmk_f32 v1, v2, v3, 64.0
-
-// CHECK: v_madak_f32_e32 v1, v2, v3, 0x42800000 ; encoding: [0x02,0x07,0x02,0x42,0x00,0x00,0x80,0x42]
-v_madak_f32 v1, v2, v3, 64.0
-
-// CHECK: v_bcnt_u32_b32_e32 v1, v2, v3 ; encoding: [0x02,0x07,0x02,0x44]
-v_bcnt_u32_b32 v1, v2, v3
-
-// CHECK: v_mbcnt_lo_u32_b32_e32 v1, v2, v3 ; encoding: [0x02,0x07,0x02,0x46]
-v_mbcnt_lo_u32_b32 v1, v2, v3
-
-// CHECK: v_mbcnt_hi_u32_b32_e32 v1, v2, v3 ; encoding: [0x02,0x07,0x02,0x48]
-v_mbcnt_hi_u32_b32_e32 v1, v2, v3
-
-// CHECK: v_add_i32_e32 v1, v2, v3 ; encoding: [0x02,0x07,0x02,0x4a]
-v_add_i32 v1, v2, v3
-
-// CHECK: v_sub_i32_e32 v1, v2, v3 ; encoding: [0x02,0x07,0x02,0x4c]
-v_sub_i32_e32 v1, v2, v3
-
-// CHECK: v_subrev_i32_e32 v1, v2, v3 ; encoding: [0x02,0x07,0x02,0x4e]
-v_subrev_i32 v1, v2, v3
-
-// CHECK : v_addc_u32_e32 v1, v2, v3 ; encoding: [0x02,0x07,0x02,0x50]
-v_addc_u32 v1, v2, v3
-
-// CHECK: v_subb_u32_e32 v1, v2, v3 ; encoding: [0x02,0x07,0x02,0x52]
-v_subb_u32 v1, v2, v3
-
-// CHECK: v_subbrev_u32_e32 v1, v2, v3 ; encoding: [0x02,0x07,0x02,0x54]
-v_subbrev_u32 v1, v2, v3
-
-// CHECK: v_ldexp_f32_e32 v1, v2, v3 ; encoding: [0x02,0x07,0x02,0x56]
-v_ldexp_f32 v1, v2, v3
-
-// CHECK: v_cvt_pkaccum_u8_f32_e32 v1, v2, v3 ; encoding: [0x02,0x07,0x02,0x58]
-v_cvt_pkaccum_u8_f32 v1, v2, v3
-
-// CHECK: v_cvt_pknorm_i16_f32_e32 v1, v2, v3 ; encoding: [0x02,0x07,0x02,0x5a]
-v_cvt_pknorm_i16_f32 v1, v2, v3
-
-// CHECK: v_cvt_pknorm_u16_f32_e32 v1, v2, v3 ; encoding: [0x02,0x07,0x02,0x5c]
-v_cvt_pknorm_u16_f32 v1, v2, v3
-
-// CHECK: v_cvt_pkrtz_f16_f32_e32 v1, v2, v3 ; encoding: [0x02,0x07,0x02,0x5e]
-v_cvt_pkrtz_f16_f32 v1, v2, v3
-
-// CHECK: v_cvt_pk_u16_u32_e32 v1, v2, v3 ; encoding: [0x02,0x07,0x02,0x60]
-v_cvt_pk_u16_u32_e32 v1, v2, v3
-
-// CHECK: v_cvt_pk_i16_i32_e32 v1, v2, v3 ; encoding: [0x02,0x07,0x02,0x62]
-v_cvt_pk_i16_i32 v1, v2, v3
diff --git a/test/MC/Sparc/sparc-alu-instructions.s b/test/MC/Sparc/sparc-alu-instructions.s
index e2e5ef86725..98caf1d6d67 100644
--- a/test/MC/Sparc/sparc-alu-instructions.s
+++ b/test/MC/Sparc/sparc-alu-instructions.s
@@ -76,8 +76,8 @@
         ! CHECK: mov 255, %g3     ! encoding: [0x86,0x10,0x20,0xff]
         mov 0xff, %g3
 
-        ! CHECK: restore              ! encoding: [0x81,0xe8,0x00,0x00]
-        restore %g0, %g0, %g0
+        ! CHECK: restore %g0, %g0, %g1           ! encoding: [0x83,0xe8,0x00,0x00]
+        restore %g0, %g0, %g1
 
         ! CHECK: addx %g2, %g1, %g3              ! encoding: [0x86,0x40,0x80,0x01]
         addx %g2, %g1, %g3
diff --git a/test/MC/Sparc/sparc-assembly-exprs.s b/test/MC/Sparc/sparc-assembly-exprs.s
new file mode 100644
index 00000000000..7fdc5a50345
--- /dev/null
+++ b/test/MC/Sparc/sparc-assembly-exprs.s
@@ -0,0 +1,6 @@
+! RUN: llvm-mc %s -arch=sparc   -show-encoding | FileCheck %s
+
+        ! CHECK: mov 1033, %o1  ! encoding: [0x92,0x10,0x24,0x09]
+        mov      (0x400|9), %o1
+        ! CHECK: mov 60, %o2    ! encoding: [0x94,0x10,0x20,0x3c]
+        mov      (12+3<<2), %o2
diff --git a/test/MC/Sparc/sparc-atomic-instructions.s b/test/MC/Sparc/sparc-atomic-instructions.s
index 5c46067ea3b..17f97d4b535 100644
--- a/test/MC/Sparc/sparc-atomic-instructions.s
+++ b/test/MC/Sparc/sparc-atomic-instructions.s
@@ -1,7 +1,5 @@
 ! RUN: llvm-mc %s -arch=sparcv9 -show-encoding | FileCheck %s
-
-        ! CHECK: membar 15             ! encoding: [0x81,0x43,0xe0,0x0f]
-        membar 15
+! RUN: llvm-mc %s -arch=sparc -show-encoding | FileCheck %s
 
         ! CHECK: stbar                 ! encoding: [0x81,0x43,0xc0,0x00]
         stbar
@@ -12,8 +10,5 @@
         ! CHECK: swap [%i0+32], %o2    ! encoding: [0xd4,0x7e,0x20,0x20]
         swap [%i0+32], %o2
 
-        ! CHECK: cas [%i0], %l6, %o2   ! encoding: [0xd5,0xe6,0x10,0x16]
-        cas [%i0], %l6, %o2
-
-        ! CHECK: casx [%i0], %l6, %o2  ! encoding: [0xd5,0xf6,0x10,0x16]
-        casx [%i0], %l6, %o2
+        ! CHECK: swapa [%i0+%l6] 131, %o2   ! encoding: [0xd4,0xfe,0x10,0x76]
+        swapa [%i0+%l6] 131, %o2
diff --git a/test/MC/Sparc/sparc-ctrl-instructions.s b/test/MC/Sparc/sparc-ctrl-instructions.s
index cf92e7045d4..a2608fdba60 100644
--- a/test/MC/Sparc/sparc-ctrl-instructions.s
+++ b/test/MC/Sparc/sparc-ctrl-instructions.s
@@ -11,7 +11,7 @@
         ! CHECK: call %o1+8   ! encoding: [0x9f,0xc2,0x60,0x08]
         call %o1 + 8
 
-        ! CHECK: call %g1     ! encoding: [0x9f,0xc0,0x60,0x00]
+        ! CHECK: call %g1     ! encoding: [0x9f,0xc0,0x40,0x00]
         call %g1
 
         ! CHECK: call %g1+%lo(sym)   ! encoding: [0x9f,0xc0,0b011000AA,A]
@@ -24,7 +24,7 @@
         ! CHECK: jmp %o1+8    ! encoding: [0x81,0xc2,0x60,0x08]
         jmp %o1 + 8
 
-        ! CHECK: jmp %g1      ! encoding: [0x81,0xc0,0x60,0x00]
+        ! CHECK: jmp %g1      ! encoding: [0x81,0xc0,0x40,0x00]
         jmp %g1
 
         ! CHECK: jmp %g1+%lo(sym)   ! encoding: [0x81,0xc0,0b011000AA,A]
@@ -37,7 +37,7 @@
         ! CHECK: jmpl %o1+8, %g2    ! encoding: [0x85,0xc2,0x60,0x08]
         jmpl %o1 + 8, %g2
 
-        ! CHECK: jmpl %g1, %g2      ! encoding: [0x85,0xc0,0x60,0x00]
+        ! CHECK: jmpl %g1, %g2      ! encoding: [0x85,0xc0,0x40,0x00]
         jmpl %g1, %g2
 
         ! CHECK: jmpl %g1+%lo(sym), %g2   ! encoding: [0x85,0xc0,0b011000AA,A]
@@ -52,10 +52,18 @@
         ! CHECK-NEXT:         ! fixup A - offset: 0, value: .BB0, kind: fixup_sparc_br22
         bne .BB0
 
+        ! CHECK: bne .BB0     ! encoding: [0x12,0b10AAAAAA,A,A]
+        ! CHECK-NEXT:         ! fixup A - offset: 0, value: .BB0, kind: fixup_sparc_br22
+        bnz .BB0
+
         ! CHECK: be .BB0      ! encoding: [0x02,0b10AAAAAA,A,A]
         ! CHECK-NEXT:         ! fixup A - offset: 0, value: .BB0, kind: fixup_sparc_br22
         be .BB0
 
+        ! CHECK: be .BB0      ! encoding: [0x02,0b10AAAAAA,A,A]
+        ! CHECK-NEXT:         ! fixup A - offset: 0, value: .BB0, kind: fixup_sparc_br22
+        bz .BB0
+
         ! CHECK: bg .BB0      ! encoding: [0x14,0b10AAAAAA,A,A]
         ! CHECK-NEXT:         ! fixup A - offset: 0, value: .BB0, kind: fixup_sparc_br22
         bg .BB0
@@ -84,10 +92,18 @@
         ! CHECK-NEXT:         ! fixup A - offset: 0, value: .BB0, kind: fixup_sparc_br22
         bcc .BB0
 
+        ! CHECK: bcc .BB0      ! encoding: [0x1a,0b10AAAAAA,A,A]
+        ! CHECK-NEXT:         ! fixup A - offset: 0, value: .BB0, kind: fixup_sparc_br22
+        bgeu .BB0
+
         ! CHECK: bcs .BB0      ! encoding: [0x0a,0b10AAAAAA,A,A]
         ! CHECK-NEXT:         ! fixup A - offset: 0, value: .BB0, kind: fixup_sparc_br22
         bcs .BB0
 
+        ! CHECK: bcs .BB0      ! encoding: [0x0a,0b10AAAAAA,A,A]
+        ! CHECK-NEXT:         ! fixup A - offset: 0, value: .BB0, kind: fixup_sparc_br22
+        blu .BB0
+
         ! CHECK: bpos .BB0      ! encoding: [0x1c,0b10AAAAAA,A,A]
         ! CHECK-NEXT:         ! fixup A - offset: 0, value: .BB0, kind: fixup_sparc_br22
         bpos .BB0
@@ -131,10 +147,18 @@
         ! CHECK-NEXT:                                        !   fixup A - offset: 0, value: .BB0, kind: fixup_sparc_br22
         fbne .BB0
 
+        ! CHECK:             fbne .BB0                       ! encoding: [0x03,0b10AAAAAA,A,A]
+        ! CHECK-NEXT:                                        !   fixup A - offset: 0, value: .BB0, kind: fixup_sparc_br22
+        fbnz .BB0
+
         ! CHECK:             fbe .BB0                        ! encoding: [0x13,0b10AAAAAA,A,A]
         ! CHECK-NEXT:                                        !   fixup A - offset: 0, value: .BB0, kind: fixup_sparc_br22
         fbe .BB0
 
+        ! CHECK:             fbe .BB0                        ! encoding: [0x13,0b10AAAAAA,A,A]
+        ! CHECK-NEXT:                                        !   fixup A - offset: 0, value: .BB0, kind: fixup_sparc_br22
+        fbz .BB0
+
         ! CHECK:             fbue .BB0                       ! encoding: [0x15,0b10AAAAAA,A,A]
         ! CHECK-NEXT:                                        !   fixup A - offset: 0, value: .BB0, kind: fixup_sparc_br22
         fbue .BB0
diff --git a/test/MC/Sparc/sparc-little-endian.s b/test/MC/Sparc/sparc-little-endian.s
new file mode 100644
index 00000000000..e9a56eb7b96
--- /dev/null
+++ b/test/MC/Sparc/sparc-little-endian.s
@@ -0,0 +1,18 @@
+! RUN: llvm-mc %s -triple=sparcel-linux-gnu -show-encoding | FileCheck %s
+! RUN: llvm-mc -triple=sparcel-linux-gnu -filetype=obj < %s | llvm-objdump -d - | FileCheck %s --check-prefix=CHECK-OBJ
+
+        ! CHECK-OBJ: .text:
+        .BB0:
+
+        ! Ensure instructions are emitted in reversed byte order:
+
+        ! CHECK: call %g1     ! encoding: [0x00,0x40,0xc0,0x9f]
+        ! CHECK-OBJ: 0: 00 40 c0 9f  call %g1
+        call %g1
+
+        ! ...and that fixups are applied to the correct bytes.
+
+        ! CHECK: ba .BB0      ! encoding: [A,A,0b10AAAAAA,0x10]
+        ! CHECK-NEXT:         ! fixup A - offset: 0, value: .BB0, kind: fixup_sparc_br22
+        ! CHECK-OBJ: 4: ff ff bf 10  ba 4194303
+        ba .BB0
diff --git a/test/MC/Sparc/sparc-mem-instructions.s b/test/MC/Sparc/sparc-mem-instructions.s
index e8eb5933c3a..c10c8781fd8 100644
--- a/test/MC/Sparc/sparc-mem-instructions.s
+++ b/test/MC/Sparc/sparc-mem-instructions.s
@@ -5,54 +5,80 @@
         ldsb [%i0 + %l6], %o2
         ! CHECK: ldsb [%i0+32], %o2   ! encoding: [0xd4,0x4e,0x20,0x20]
         ldsb [%i0 + 32], %o2
-        ! CHECK: ldsb [%g1], %o4      ! encoding: [0xd8,0x48,0x60,0x00]
+        ! CHECK: ldsb [%g1], %o4      ! encoding: [0xd8,0x48,0x40,0x00]
         ldsb [%g1], %o4
+        ! CHECK: ldsba [%i0+%l6] 131, %o2  ! encoding: [0xd4,0xce,0x10,0x76]
+        ldsba [%i0 + %l6] 131, %o2
 
         ! CHECK: ldsh [%i0+%l6], %o2  ! encoding: [0xd4,0x56,0x00,0x16]
         ldsh [%i0 + %l6], %o2
         ! CHECK: ldsh [%i0+32], %o2   ! encoding: [0xd4,0x56,0x20,0x20]
         ldsh [%i0 + 32], %o2
-        ! CHECK: ldsh [%g1], %o4      ! encoding: [0xd8,0x50,0x60,0x00]
+        ! CHECK: ldsh [%g1], %o4      ! encoding: [0xd8,0x50,0x40,0x00]
         ldsh [%g1], %o4
+        ! CHECK: ldsha [%i0+%l6] 131, %o2 ! encoding: [0xd4,0xd6,0x10,0x76]
+        ldsha [%i0 + %l6] 131, %o2
 
         ! CHECK: ldub [%i0+%l6], %o2  ! encoding: [0xd4,0x0e,0x00,0x16]
         ldub [%i0 + %l6], %o2
         ! CHECK: ldub [%i0+32], %o2   ! encoding: [0xd4,0x0e,0x20,0x20]
         ldub [%i0 + 32], %o2
-        ! CHECK: ldub [%g1], %o2      ! encoding: [0xd4,0x08,0x60,0x00]
+        ! CHECK: ldub [%g1], %o2      ! encoding: [0xd4,0x08,0x40,0x00]
         ldub [%g1], %o2
+        ! CHECK: lduba [%i0+%l6] 131, %o2 ! encoding: [0xd4,0x8e,0x10,0x76]
+        lduba [%i0 + %l6] 131, %o2
 
         ! CHECK: lduh [%i0+%l6], %o2  ! encoding: [0xd4,0x16,0x00,0x16]
         lduh [%i0 + %l6], %o2
         ! CHECK: lduh [%i0+32], %o2   ! encoding: [0xd4,0x16,0x20,0x20]
         lduh [%i0 + 32], %o2
-        ! CHECK: lduh [%g1], %o2      ! encoding: [0xd4,0x10,0x60,0x00]
+        ! CHECK: lduh [%g1], %o2      ! encoding: [0xd4,0x10,0x40,0x00]
         lduh [%g1], %o2
+        ! CHECK: lduha [%i0+%l6] 131, %o2 ! encoding: [0xd4,0x96,0x10,0x76]
+        lduha [%i0 + %l6] 131, %o2
 
         ! CHECK: ld [%i0+%l6], %o2    ! encoding: [0xd4,0x06,0x00,0x16]
         ld [%i0 + %l6], %o2
         ! CHECK: ld [%i0+32], %o2     ! encoding: [0xd4,0x06,0x20,0x20]
         ld [%i0 + 32], %o2
-        ! CHECK: ld [%g1], %o2        ! encoding: [0xd4,0x00,0x60,0x00]
+        ! CHECK: ld [%g1], %o2        ! encoding: [0xd4,0x00,0x40,0x00]
         ld [%g1], %o2
+        ! CHECK: lda [%i0+%l6] 131, %o2 ! encoding: [0xd4,0x86,0x10,0x76]
+        lda [%i0 + %l6] 131, %o2
 
         ! CHECK: stb %o2, [%i0+%l6]   ! encoding: [0xd4,0x2e,0x00,0x16]
         stb %o2, [%i0 + %l6]
         ! CHECK: stb %o2, [%i0+32]    ! encoding: [0xd4,0x2e,0x20,0x20]
         stb %o2, [%i0 + 32]
-        ! CHECK: stb %o2, [%g1]       ! encoding: [0xd4,0x28,0x60,0x00]
+        ! CHECK: stb %o2, [%g1]       ! encoding: [0xd4,0x28,0x40,0x00]
         stb %o2, [%g1]
+        ! CHECK: stba %o2, [%i0+%l6] 131 ! encoding: [0xd4,0xae,0x10,0x76]
+        stba %o2, [%i0 + %l6] 131
 
         ! CHECK: sth %o2, [%i0+%l6]   ! encoding: [0xd4,0x36,0x00,0x16]
         sth %o2, [%i0 + %l6]
         ! CHECK: sth %o2, [%i0+32]    ! encoding: [0xd4,0x36,0x20,0x20]
         sth %o2, [%i0 + 32]
-        ! CHECK: sth %o2, [%g1]       ! encoding: [0xd4,0x30,0x60,0x00]
+        ! CHECK: sth %o2, [%g1]       ! encoding: [0xd4,0x30,0x40,0x00]
         sth %o2, [%g1]
+        ! CHECK: stha %o2, [%i0+%l6] 131 ! encoding: [0xd4,0xb6,0x10,0x76]
+        stha %o2, [%i0 + %l6] 131
 
         ! CHECK: st %o2, [%i0+%l6]    ! encoding: [0xd4,0x26,0x00,0x16]
         st %o2, [%i0 + %l6]
         ! CHECK: st %o2, [%i0+32]     ! encoding: [0xd4,0x26,0x20,0x20]
         st %o2, [%i0 + 32]
-        ! CHECK: st %o2, [%g1]        ! encoding: [0xd4,0x20,0x60,0x00]
+        ! CHECK: st %o2, [%g1]        ! encoding: [0xd4,0x20,0x40,0x00]
         st %o2, [%g1]
+        ! CHECK: sta %o2, [%i0+%l6] 131 ! encoding: [0xd4,0xa6,0x10,0x76]
+        sta %o2, [%i0 + %l6] 131
+
+        ! CHECK:  flush %g1+%g2         ! encoding: [0x81,0xd8,0x40,0x02]
+        flush %g1 + %g2
+        ! CHECK:  flush %g1+8           ! encoding: [0x81,0xd8,0x60,0x08]
+        flush %g1 + 8
+        ! CHECK:  flush %g1             ! encoding: [0x81,0xd8,0x40,0x00]
+        flush %g1
+        ! Not specified in manual, but accepted by gas.
+        ! CHECK:  flush %g0             ! encoding: [0x81,0xd8,0x00,0x00]
+        flush
diff --git a/test/MC/Sparc/sparc-pic.s b/test/MC/Sparc/sparc-pic.s
index 5a34d309899..5430d1fea10 100644
--- a/test/MC/Sparc/sparc-pic.s
+++ b/test/MC/Sparc/sparc-pic.s
@@ -7,9 +7,16 @@
 ! CHECK-NEXT:   0x{{[0-9,A-F]+}} R_SPARC_PC10 _GLOBAL_OFFSET_TABLE_ 0x8
 ! CHECK-NEXT:   0x{{[0-9,A-F]+}} R_SPARC_GOT22 AGlobalVar 0x0
 ! CHECK-NEXT:   0x{{[0-9,A-F]+}} R_SPARC_GOT10 AGlobalVar 0x0
+! CHECK-NEXT:   0x{{[0-9,A-F]+}} R_SPARC_GOT22 .LC0 0x0
+! CHECK-NEXT:   0x{{[0-9,A-F]+}} R_SPARC_GOT10 .LC0 0x0
 ! CHECK-NEXT:   0x{{[0-9,A-F]+}} R_SPARC_WPLT30 bar 0x0
 ! CHECK:      ]
 
+        .section        ".rodata"
+        .align 8
+.LC0:
+        .asciz   "string"
+        .section ".text"
         .text
         .globl  foo
         .align  4
@@ -29,8 +36,11 @@ foo:
         add %i1, %o7, %i1
         sethi %hi(AGlobalVar), %i2
         add %i2, %lo(AGlobalVar), %i2
-        ldx [%i1+%i2], %i1
-        ldx [%i1], %i1
+        ldx [%i1+%i2], %i3
+        ldx [%i3], %i3
+        sethi %hi(.LC0), %i2
+        add %i2, %lo(.LC0), %i2
+        ldx [%i1+%i2], %i4
         call bar
         add %i0, %i1, %o0
         ret
@@ -46,4 +56,3 @@ foo:
 AGlobalVar:
         .xword  0                       ! 0x0
         .size   AGlobalVar, 8
-
diff --git a/test/MC/Sparc/sparc-special-registers.s b/test/MC/Sparc/sparc-special-registers.s
new file mode 100644
index 00000000000..2cb57d720c4
--- /dev/null
+++ b/test/MC/Sparc/sparc-special-registers.s
@@ -0,0 +1,35 @@
+! RUN: llvm-mc %s -arch=sparc -show-encoding | FileCheck %s
+! RUN: llvm-mc %s -arch=sparcv9 -show-encoding | FileCheck %s
+
+        ! CHECK: rd %y, %i0            ! encoding: [0xb1,0x40,0x00,0x00]
+        rd %y, %i0
+
+        ! CHECK: rd %asr1, %i0         ! encoding: [0xb1,0x40,0x40,0x00]
+        rd %asr1, %i0
+
+        ! CHECK: wr %i0, 5, %y         ! encoding: [0x81,0x86,0x20,0x05]
+        wr %i0, 5, %y
+
+        ! CHECK: wr %i0, %i1, %asr15   ! encoding: [0x9f,0x86,0x00,0x19]
+        wr %i0, %i1, %asr15
+
+        ! CHECK: rd %asr15, %g0        ! encoding: [0x81,0x43,0xc0,0x00]
+        rd %asr15, %g0
+
+        ! CHECK: rd %psr, %i0          ! encoding: [0xb1,0x48,0x00,0x00]
+        rd %psr, %i0
+
+        ! CHECK: rd %wim, %i0          ! encoding: [0xb1,0x50,0x00,0x00]
+        rd %wim, %i0
+
+        ! CHECK: rd %tbr, %i0          ! encoding: [0xb1,0x58,0x00,0x00]
+        rd %tbr, %i0
+
+        ! CHECK: wr %i0, 5, %psr          ! encoding: [0x81,0x8e,0x20,0x05]
+        wr %i0, 5, %psr
+
+        ! CHECK: wr %i0, 5, %wim          ! encoding: [0x81,0x96,0x20,0x05]
+        wr %i0, 5, %wim
+
+        ! CHECK: wr %i0, 5, %tbr          ! encoding: [0x81,0x9e,0x20,0x05]
+        wr %i0, 5, %tbr
diff --git a/test/MC/Sparc/sparc-synthetic-instructions.s b/test/MC/Sparc/sparc-synthetic-instructions.s
new file mode 100644
index 00000000000..09fd30c09e2
--- /dev/null
+++ b/test/MC/Sparc/sparc-synthetic-instructions.s
@@ -0,0 +1,145 @@
+! RUN: llvm-mc %s -arch=sparc   -show-encoding | FileCheck %s
+! RUN: llvm-mc %s -arch=sparcv9 -show-encoding | FileCheck %s
+
+! Section A.3 Synthetic Instructions
+        ! CHECK: cmp %g1, %g2                     ! encoding: [0x80,0xa0,0x40,0x02]
+        cmp %g1, %g2
+        ! CHECK: cmp %g1, 5                       ! encoding: [0x80,0xa0,0x60,0x05]
+        cmp %g1, 5
+
+        ! jmp and call are tested in sparc-ctrl-instructions.
+
+        ! CHECK: tst %g1                          ! encoding: [0x80,0x90,0x40,0x00]
+        tst %g1
+
+        ! CHECK: ret                              ! encoding: [0x81,0xc7,0xe0,0x08]
+        ret
+        ! CHECK: retl                             ! encoding: [0x81,0xc3,0xe0,0x08]
+        retl
+
+        ! CHECK: restore                          ! encoding: [0x81,0xe8,0x00,0x00]
+        restore
+        ! CHECK: save                             ! encoding: [0x81,0xe0,0x00,0x00]
+        save
+
+        ! CHECK: sethi %hi(40000), %g1            ! encoding: [0x03,0b00AAAAAA,A,A]
+        ! CHECK:                                  !   fixup A - offset: 0, value: %hi(40000), kind: fixup_sparc_hi22
+        ! CHECK: or %g1, %lo(40000), %g1          ! encoding: [0x82,0x10,0b011000AA,A]
+        ! CHECK:                                  !   fixup A - offset: 0, value: %lo(40000), kind: fixup_sparc_lo10
+        set 40000, %g1
+        ! CHECK: mov      %lo(1), %g1             ! encoding: [0x82,0x10,0b001000AA,A]
+        ! CHECK:                                  !   fixup A - offset: 0, value: %lo(1), kind: fixup_sparc_lo10
+        set 1, %g1
+        ! CHECK: sethi %hi(32768), %g1            ! encoding: [0x03,0b00AAAAAA,A,A]
+        ! CHECK:                                  !   fixup A - offset: 0, value: %hi(32768), kind: fixup_sparc_hi22
+        set 32768, %g1
+
+        ! CHECK: xnor %g1, %g0, %g2               ! encoding: [0x84,0x38,0x40,0x00]
+        not %g1, %g2
+        ! CHECK: xnor %g1, %g0, %g1               ! encoding: [0x82,0x38,0x40,0x00]
+        not %g1
+
+        ! CHECK: sub %g0, %g1, %g2                ! encoding: [0x84,0x20,0x00,0x01]
+        neg %g1, %g2
+        ! CHECK: sub %g0, %g1, %g1                ! encoding: [0x82,0x20,0x00,0x01]
+        neg %g1
+
+        ! CHECK: add %g1, 1, %g1                  ! encoding: [0x82,0x00,0x60,0x01]
+        inc %g1
+        ! CHECK: add %g1, 55, %g1                 ! encoding: [0x82,0x00,0x60,0x37]
+        inc 55, %g1
+        ! CHECK: addcc %g1, 1, %g1                ! encoding: [0x82,0x80,0x60,0x01]
+        inccc %g1
+        ! CHECK: addcc %g1, 55, %g1               ! encoding: [0x82,0x80,0x60,0x37]
+        inccc 55, %g1
+
+        ! CHECK: sub %g1, 1, %g1                  ! encoding: [0x82,0x20,0x60,0x01]
+        dec %g1
+        ! CHECK: sub %g1, 55, %g1                 ! encoding: [0x82,0x20,0x60,0x37]
+        dec 55, %g1
+        ! CHECK: subcc %g1, 1, %g1                ! encoding: [0x82,0xa0,0x60,0x01]
+        deccc %g1
+        ! CHECK: subcc %g1, 55, %g1               ! encoding: [0x82,0xa0,0x60,0x37]
+        deccc 55, %g1
+
+        ! CHECK: andcc %g2, %g1, %g0              ! encoding: [0x80,0x88,0x80,0x01]
+        btst %g1, %g2
+        ! CHECK: andcc %g2, 4, %g0                ! encoding: [0x80,0x88,0xa0,0x04]
+        btst 4, %g2
+        ! CHECK: or %g2, %g1, %g2                 ! encoding: [0x84,0x10,0x80,0x01]
+        bset %g1, %g2
+        ! CHECK: or %g2, 4, %g2                   ! encoding: [0x84,0x10,0xa0,0x04]
+        bset 4, %g2
+        ! CHECK: andn %g2, %g1, %g2               ! encoding: [0x84,0x28,0x80,0x01]
+        bclr %g1, %g2
+        ! CHECK: andn %g2, 4, %g2                 ! encoding: [0x84,0x28,0xa0,0x04]
+        bclr 4, %g2
+        ! CHECK: xor %g2, %g1, %g2                ! encoding: [0x84,0x18,0x80,0x01]
+        btog %g1, %g2
+        ! CHECK: xor %g2, 4, %g2                  ! encoding: [0x84,0x18,0xa0,0x04]
+        btog 4, %g2
+
+        ! CHECK: mov %g0, %g1                     ! encoding: [0x82,0x10,0x00,0x00]
+        clr %g1
+        ! CHECK: stb %g0, [%g1+%g2]               ! encoding: [0xc0,0x28,0x40,0x02]
+        clrb [%g1+%g2]
+        ! CHECK: sth %g0, [%g1+%g2]               ! encoding: [0xc0,0x30,0x40,0x02]
+        clrh [%g1+%g2]
+        ! CHECK: st %g0, [%g1+%g2]                ! encoding: [0xc0,0x20,0x40,0x02]
+        clr [%g1+%g2]
+
+        ! mov reg_or_imm,reg tested in sparc-alu-instructions.s
+
+        ! CHECK: rd %y, %i0                       ! encoding: [0xb1,0x40,0x00,0x00]
+        mov %y, %i0
+        ! CHECK: rd %asr1, %i0                    ! encoding: [0xb1,0x40,0x40,0x00]
+        mov %asr1, %i0
+        ! CHECK: rd %psr, %i0                     ! encoding: [0xb1,0x48,0x00,0x00]
+        mov %psr, %i0
+        ! CHECK: rd %wim, %i0                     ! encoding: [0xb1,0x50,0x00,0x00]
+        mov %wim, %i0
+        ! CHECK: rd %tbr, %i0                     ! encoding: [0xb1,0x58,0x00,0x00]
+        mov %tbr, %i0
+
+        ! CHECK: wr %g0, %i0, %y                  ! encoding: [0x81,0x80,0x00,0x18]
+        mov %i0, %y
+        ! CHECK: wr %g0, 5, %y                    ! encoding: [0x81,0x80,0x20,0x05]
+        mov 5, %y
+        ! CHECK: wr %g0, %i0, %asr15              ! encoding: [0x9f,0x80,0x00,0x18]
+        mov %i0, %asr15
+        ! CHECK: wr %g0, 5, %asr15                ! encoding: [0x9f,0x80,0x20,0x05]
+        mov 5, %asr15
+        ! CHECK: wr %g0, %i0, %psr                ! encoding: [0x81,0x88,0x00,0x18]
+        mov %i0, %psr
+        ! CHECK: wr %g0, 5, %psr                  ! encoding: [0x81,0x88,0x20,0x05]
+        mov 5, %psr
+        ! CHECK: wr %g0, %i0, %wim                ! encoding: [0x81,0x90,0x00,0x18]
+        mov %i0, %wim
+        ! CHECK: wr %g0, 5, %wim                  ! encoding: [0x81,0x90,0x20,0x05]
+        mov 5, %wim
+        ! CHECK: wr %g0, %i0, %tbr                ! encoding: [0x81,0x98,0x00,0x18]
+        mov %i0, %tbr
+        ! CHECK: wr %g0, 5, %tbr                  ! encoding: [0x81,0x98,0x20,0x05]
+        mov 5, %tbr
+
+! Other aliases
+        ! CHECK: wr %g0, %i0, %y                  ! encoding: [0x81,0x80,0x00,0x18]
+        wr %i0, %y
+        ! CHECK: wr %g0, 5, %y                    ! encoding: [0x81,0x80,0x20,0x05]
+        wr 5, %y
+        ! CHECK: wr %g0, %i0, %asr15              ! encoding: [0x9f,0x80,0x00,0x18]
+        wr %i0, %asr15
+        ! CHECK: wr %g0, 5, %asr15                ! encoding: [0x9f,0x80,0x20,0x05]
+        wr 5, %asr15
+        ! CHECK: wr %g0, %i0, %psr                ! encoding: [0x81,0x88,0x00,0x18]
+        wr %i0, %psr
+        ! CHECK: wr %g0, 5, %psr                  ! encoding: [0x81,0x88,0x20,0x05]
+        wr 5, %psr
+        ! CHECK: wr %g0, %i0, %wim                ! encoding: [0x81,0x90,0x00,0x18]
+        wr %i0, %wim
+        ! CHECK: wr %g0, 5, %wim                  ! encoding: [0x81,0x90,0x20,0x05]
+        wr 5, %wim
+        ! CHECK: wr %g0, %i0, %tbr                ! encoding: [0x81,0x98,0x00,0x18]
+        wr %i0, %tbr
+        ! CHECK: wr %g0, 5, %tbr                  ! encoding: [0x81,0x98,0x20,0x05]
+        wr 5, %tbr
diff --git a/test/MC/Sparc/sparcv9-atomic-instructions.s b/test/MC/Sparc/sparcv9-atomic-instructions.s
new file mode 100644
index 00000000000..40619a75548
--- /dev/null
+++ b/test/MC/Sparc/sparcv9-atomic-instructions.s
@@ -0,0 +1,10 @@
+! RUN: llvm-mc %s -arch=sparcv9 -show-encoding | FileCheck %s
+
+        ! CHECK: membar 15             ! encoding: [0x81,0x43,0xe0,0x0f]
+        membar 15
+
+        ! CHECK: cas [%i0], %l6, %o2   ! encoding: [0xd5,0xe6,0x10,0x16]
+        cas [%i0], %l6, %o2
+
+        ! CHECK: casx [%i0], %l6, %o2  ! encoding: [0xd5,0xf6,0x10,0x16]
+        casx [%i0], %l6, %o2
diff --git a/test/MC/SystemZ/insn-bad-z13.s b/test/MC/SystemZ/insn-bad-z13.s
new file mode 100644
index 00000000000..6468e3ae2ba
--- /dev/null
+++ b/test/MC/SystemZ/insn-bad-z13.s
@@ -0,0 +1,1201 @@
+# For z13 only.
+# RUN: not llvm-mc -triple s390x-linux-gnu -mcpu=z13 < %s 2> %t
+# RUN: FileCheck < %t %s
+
+#CHECK: error: invalid operand
+#CHECK: lcbb	%r0, 0, -1
+#CHECK: error: invalid operand
+#CHECK: lcbb	%r0, 0, 16
+#CHECK: error: invalid operand
+#CHECK: lcbb	%r0, -1, 0
+#CHECK: error: invalid operand
+#CHECK: lcbb	%r0, 4096, 0
+#CHECK: error: invalid use of vector addressing
+#CHECK: lcbb	%r0, 0(%v1,%r2), 0
+
+	lcbb	%r0, 0, -1
+	lcbb	%r0, 0, 16
+	lcbb	%r0, -1, 0
+	lcbb	%r0, 4096, 0
+	lcbb	%r0, 0(%v1,%r2), 0
+
+#CHECK: error: invalid operand
+#CHECK: vcdgb	%v0, %v0, 0, -1
+#CHECK: error: invalid operand
+#CHECK: vcdgb	%v0, %v0, 0, 16
+#CHECK: error: invalid operand
+#CHECK: vcdgb	%v0, %v0, -1, 0
+#CHECK: error: invalid operand
+#CHECK: vcdgb	%v0, %v0, 16, 0
+
+	vcdgb	%v0, %v0, 0, -1
+	vcdgb	%v0, %v0, 0, 16
+	vcdgb	%v0, %v0, -1, 0
+	vcdgb	%v0, %v0, 16, 0
+
+#CHECK: error: invalid operand
+#CHECK: vcdlgb	%v0, %v0, 0, -1
+#CHECK: error: invalid operand
+#CHECK: vcdlgb	%v0, %v0, 0, 16
+#CHECK: error: invalid operand
+#CHECK: vcdlgb	%v0, %v0, -1, 0
+#CHECK: error: invalid operand
+#CHECK: vcdlgb	%v0, %v0, 16, 0
+
+	vcdlgb	%v0, %v0, 0, -1
+	vcdlgb	%v0, %v0, 0, 16
+	vcdlgb	%v0, %v0, -1, 0
+	vcdlgb	%v0, %v0, 16, 0
+
+#CHECK: error: invalid operand
+#CHECK: vcgdb	%v0, %v0, 0, -1
+#CHECK: error: invalid operand
+#CHECK: vcgdb	%v0, %v0, 0, 16
+#CHECK: error: invalid operand
+#CHECK: vcgdb	%v0, %v0, -1, 0
+#CHECK: error: invalid operand
+#CHECK: vcgdb	%v0, %v0, 16, 0
+
+	vcgdb	%v0, %v0, 0, -1
+	vcgdb	%v0, %v0, 0, 16
+	vcgdb	%v0, %v0, -1, 0
+	vcgdb	%v0, %v0, 16, 0
+
+#CHECK: error: invalid operand
+#CHECK: vclgdb	%v0, %v0, 0, -1
+#CHECK: error: invalid operand
+#CHECK: vclgdb	%v0, %v0, 0, 16
+#CHECK: error: invalid operand
+#CHECK: vclgdb	%v0, %v0, -1, 0
+#CHECK: error: invalid operand
+#CHECK: vclgdb	%v0, %v0, 16, 0
+
+	vclgdb	%v0, %v0, 0, -1
+	vclgdb	%v0, %v0, 0, 16
+	vclgdb	%v0, %v0, -1, 0
+	vclgdb	%v0, %v0, 16, 0
+
+#CHECK: error: invalid operand
+#CHECK: verimb	%v0, %v0, %v0, -1
+#CHECK: error: invalid operand
+#CHECK: verimb	%v0, %v0, %v0, 256
+
+	verimb	%v0, %v0, %v0, -1
+	verimb	%v0, %v0, %v0, 256
+
+#CHECK: error: invalid operand
+#CHECK: verimf	%v0, %v0, %v0, -1
+#CHECK: error: invalid operand
+#CHECK: verimf	%v0, %v0, %v0, 256
+
+	verimf	%v0, %v0, %v0, -1
+	verimf	%v0, %v0, %v0, 256
+
+#CHECK: error: invalid operand
+#CHECK: verimg	%v0, %v0, %v0, -1
+#CHECK: error: invalid operand
+#CHECK: verimg	%v0, %v0, %v0, 256
+
+	verimg	%v0, %v0, %v0, -1
+	verimg	%v0, %v0, %v0, 256
+
+#CHECK: error: invalid operand
+#CHECK: verimh	%v0, %v0, %v0, -1
+#CHECK: error: invalid operand
+#CHECK: verimh	%v0, %v0, %v0, 256
+
+	verimh	%v0, %v0, %v0, -1
+	verimh	%v0, %v0, %v0, 256
+
+#CHECK: error: invalid operand
+#CHECK: verllb	%v0, %v0, -1
+#CHECK: error: invalid operand
+#CHECK: verllb	%v0, %v0, 4096
+
+	verllb	%v0, %v0, -1
+	verllb	%v0, %v0, 4096
+
+#CHECK: error: invalid operand
+#CHECK: verllf	%v0, %v0, -1
+#CHECK: error: invalid operand
+#CHECK: verllf	%v0, %v0, 4096
+
+	verllf	%v0, %v0, -1
+	verllf	%v0, %v0, 4096
+
+#CHECK: error: invalid operand
+#CHECK: verllg	%v0, %v0, -1
+#CHECK: error: invalid operand
+#CHECK: verllg	%v0, %v0, 4096
+
+	verllg	%v0, %v0, -1
+	verllg	%v0, %v0, 4096
+
+#CHECK: error: invalid operand
+#CHECK: verllh	%v0, %v0, -1
+#CHECK: error: invalid operand
+#CHECK: verllh	%v0, %v0, 4096
+
+	verllh	%v0, %v0, -1
+	verllh	%v0, %v0, 4096
+
+#CHECK: error: invalid operand
+#CHECK: veslb	%v0, %v0, -1
+#CHECK: error: invalid operand
+#CHECK: veslb	%v0, %v0, 4096
+
+	veslb	%v0, %v0, -1
+	veslb	%v0, %v0, 4096
+
+#CHECK: error: invalid operand
+#CHECK: veslf	%v0, %v0, -1
+#CHECK: error: invalid operand
+#CHECK: veslf	%v0, %v0, 4096
+
+	veslf	%v0, %v0, -1
+	veslf	%v0, %v0, 4096
+
+#CHECK: error: invalid operand
+#CHECK: veslg	%v0, %v0, -1
+#CHECK: error: invalid operand
+#CHECK: veslg	%v0, %v0, 4096
+
+	veslg	%v0, %v0, -1
+	veslg	%v0, %v0, 4096
+
+#CHECK: error: invalid operand
+#CHECK: veslh	%v0, %v0, -1
+#CHECK: error: invalid operand
+#CHECK: veslh	%v0, %v0, 4096
+
+	veslh	%v0, %v0, -1
+	veslh	%v0, %v0, 4096
+
+#CHECK: error: invalid operand
+#CHECK: vesrab	%v0, %v0, -1
+#CHECK: error: invalid operand
+#CHECK: vesrab	%v0, %v0, 4096
+
+	vesrab	%v0, %v0, -1
+	vesrab	%v0, %v0, 4096
+
+#CHECK: error: invalid operand
+#CHECK: vesraf	%v0, %v0, -1
+#CHECK: error: invalid operand
+#CHECK: vesraf	%v0, %v0, 4096
+
+	vesraf	%v0, %v0, -1
+	vesraf	%v0, %v0, 4096
+
+#CHECK: error: invalid operand
+#CHECK: vesrag	%v0, %v0, -1
+#CHECK: error: invalid operand
+#CHECK: vesrag	%v0, %v0, 4096
+
+	vesrag	%v0, %v0, -1
+	vesrag	%v0, %v0, 4096
+
+#CHECK: error: invalid operand
+#CHECK: vesrah	%v0, %v0, -1
+#CHECK: error: invalid operand
+#CHECK: vesrah	%v0, %v0, 4096
+
+	vesrah	%v0, %v0, -1
+	vesrah	%v0, %v0, 4096
+
+#CHECK: error: invalid operand
+#CHECK: vesrlb	%v0, %v0, -1
+#CHECK: error: invalid operand
+#CHECK: vesrlb	%v0, %v0, 4096
+
+	vesrlb	%v0, %v0, -1
+	vesrlb	%v0, %v0, 4096
+
+#CHECK: error: invalid operand
+#CHECK: vesrlf	%v0, %v0, -1
+#CHECK: error: invalid operand
+#CHECK: vesrlf	%v0, %v0, 4096
+
+	vesrlf	%v0, %v0, -1
+	vesrlf	%v0, %v0, 4096
+
+#CHECK: error: invalid operand
+#CHECK: vesrlg	%v0, %v0, -1
+#CHECK: error: invalid operand
+#CHECK: vesrlg	%v0, %v0, 4096
+
+	vesrlg	%v0, %v0, -1
+	vesrlg	%v0, %v0, 4096
+
+#CHECK: error: invalid operand
+#CHECK: vesrlh	%v0, %v0, -1
+#CHECK: error: invalid operand
+#CHECK: vesrlh	%v0, %v0, 4096
+
+	vesrlh	%v0, %v0, -1
+	vesrlh	%v0, %v0, 4096
+
+#CHECK: error: invalid operand
+#CHECK: vfaeb	%v0, %v0, %v0, -1
+#CHECK: error: invalid operand
+#CHECK: vfaeb	%v0, %v0, %v0, 16
+#CHECK: error: too few operands
+#CHECK: vfaeb	%v0, %v0
+#CHECK: error: invalid operand
+#CHECK: vfaeb	%v0, %v0, %v0, 0, 0
+
+	vfaeb	%v0, %v0, %v0, -1
+	vfaeb	%v0, %v0, %v0, 16
+	vfaeb	%v0, %v0
+	vfaeb	%v0, %v0, %v0, 0, 0
+
+#CHECK: error: invalid operand
+#CHECK: vfaebs	%v0, %v0, %v0, -1
+#CHECK: error: invalid operand
+#CHECK: vfaebs	%v0, %v0, %v0, 16
+#CHECK: error: too few operands
+#CHECK: vfaebs	%v0, %v0
+#CHECK: error: invalid operand
+#CHECK: vfaebs	%v0, %v0, %v0, 0, 0
+
+	vfaebs	%v0, %v0, %v0, -1
+	vfaebs	%v0, %v0, %v0, 16
+	vfaebs	%v0, %v0
+	vfaebs	%v0, %v0, %v0, 0, 0
+
+#CHECK: error: invalid operand
+#CHECK: vfaef	%v0, %v0, %v0, -1
+#CHECK: error: invalid operand
+#CHECK: vfaef	%v0, %v0, %v0, 16
+#CHECK: error: too few operands
+#CHECK: vfaef	%v0, %v0
+#CHECK: error: invalid operand
+#CHECK: vfaef	%v0, %v0, %v0, 0, 0
+
+	vfaef	%v0, %v0, %v0, -1
+	vfaef	%v0, %v0, %v0, 16
+	vfaef	%v0, %v0
+	vfaef	%v0, %v0, %v0, 0, 0
+
+#CHECK: error: invalid operand
+#CHECK: vfaeh	%v0, %v0, %v0, -1
+#CHECK: error: invalid operand
+#CHECK: vfaeh	%v0, %v0, %v0, 16
+#CHECK: error: too few operands
+#CHECK: vfaeh	%v0, %v0
+#CHECK: error: invalid operand
+#CHECK: vfaeh	%v0, %v0, %v0, 0, 0
+
+	vfaeh	%v0, %v0, %v0, -1
+	vfaeh	%v0, %v0, %v0, 16
+	vfaeh	%v0, %v0
+	vfaeh	%v0, %v0, %v0, 0, 0
+
+#CHECK: error: invalid operand
+#CHECK: vfaezh	%v0, %v0, %v0, -1
+#CHECK: error: invalid operand
+#CHECK: vfaezh	%v0, %v0, %v0, 16
+#CHECK: error: too few operands
+#CHECK: vfaezh	%v0, %v0
+#CHECK: error: invalid operand
+#CHECK: vfaezh	%v0, %v0, %v0, 0, 0
+
+	vfaezh	%v0, %v0, %v0, -1
+	vfaezh	%v0, %v0, %v0, 16
+	vfaezh	%v0, %v0
+	vfaezh	%v0, %v0, %v0, 0, 0
+
+#CHECK: error: invalid operand
+#CHECK: vfaezfs	%v0, %v0, %v0, -1
+#CHECK: error: invalid operand
+#CHECK: vfaezfs	%v0, %v0, %v0, 16
+#CHECK: error: too few operands
+#CHECK: vfaezfs	%v0, %v0
+#CHECK: error: invalid operand
+#CHECK: vfaezfs	%v0, %v0, %v0, 0, 0
+
+	vfaezfs	%v0, %v0, %v0, -1
+	vfaezfs	%v0, %v0, %v0, 16
+	vfaezfs	%v0, %v0
+	vfaezfs	%v0, %v0, %v0, 0, 0
+
+#CHECK: error: invalid operand
+#CHECK: vfidb	%v0, %v0, 0, -1
+#CHECK: error: invalid operand
+#CHECK: vfidb	%v0, %v0, 0, 16
+#CHECK: error: invalid operand
+#CHECK: vfidb	%v0, %v0, -1, 0
+#CHECK: error: invalid operand
+#CHECK: vfidb	%v0, %v0, 16, 0
+
+	vfidb	%v0, %v0, 0, -1
+	vfidb	%v0, %v0, 0, 16
+	vfidb	%v0, %v0, -1, 0
+	vfidb	%v0, %v0, 16, 0
+
+#CHECK: error: invalid operand
+#CHECK: vftcidb	%v0, %v0, -1
+#CHECK: error: invalid operand
+#CHECK: vftcidb	%v0, %v0, 4096
+
+	vftcidb	%v0, %v0, -1
+	vftcidb	%v0, %v0, 4096
+
+#CHECK: error: invalid operand
+#CHECK: vgbm	%v0, -1
+#CHECK: error: invalid operand
+#CHECK: vgbm	%v0, 0x10000
+
+	vgbm	%v0, -1
+	vgbm	%v0, 0x10000
+
+#CHECK: error: vector index required
+#CHECK: vgef	%v0, 0(%r1), 0
+#CHECK: error: vector index required
+#CHECK: vgef	%v0, 0(%r2,%r1), 0
+#CHECK: error: invalid operand
+#CHECK: vgef	%v0, 0(%v0,%r1), -1
+#CHECK: error: invalid operand
+#CHECK: vgef	%v0, 0(%v0,%r1), 4
+#CHECK: error: invalid operand
+#CHECK: vgef	%v0, -1(%v0,%r1), 0
+#CHECK: error: invalid operand
+#CHECK: vgef	%v0, 4096(%v0,%r1), 0
+
+	vgef	%v0, 0(%r1), 0
+	vgef	%v0, 0(%r2,%r1), 0
+	vgef	%v0, 0(%v0,%r1), -1
+	vgef	%v0, 0(%v0,%r1), 4
+	vgef	%v0, -1(%v0,%r1), 0
+	vgef	%v0, 4096(%v0,%r1), 0
+
+#CHECK: error: vector index required
+#CHECK: vgeg	%v0, 0(%r1), 0
+#CHECK: error: vector index required
+#CHECK: vgeg	%v0, 0(%r2,%r1), 0
+#CHECK: error: invalid operand
+#CHECK: vgeg	%v0, 0(%v0,%r1), -1
+#CHECK: error: invalid operand
+#CHECK: vgeg	%v0, 0(%v0,%r1), 2
+#CHECK: error: invalid operand
+#CHECK: vgeg	%v0, -1(%v0,%r1), 0
+#CHECK: error: invalid operand
+#CHECK: vgeg	%v0, 4096(%v0,%r1), 0
+
+	vgeg	%v0, 0(%r1), 0
+	vgeg	%v0, 0(%r2,%r1), 0
+	vgeg	%v0, 0(%v0,%r1), -1
+	vgeg	%v0, 0(%v0,%r1), 2
+	vgeg	%v0, -1(%v0,%r1), 0
+	vgeg	%v0, 4096(%v0,%r1), 0
+
+#CHECK: error: invalid operand
+#CHECK: vgmb	%v0, 0, -1
+#CHECK: error: invalid operand
+#CHECK: vgmb	%v0, 0, -1
+#CHECK: error: invalid operand
+#CHECK: vgmb	%v0, -1, 0
+#CHECK: error: invalid operand
+#CHECK: vgmb	%v0, 256, 0
+
+	vgmb	%v0, 0, -1
+	vgmb	%v0, 0, -1
+	vgmb	%v0, -1, 0
+	vgmb	%v0, 256, 0
+
+#CHECK: error: invalid operand
+#CHECK: vgmf	%v0, 0, -1
+#CHECK: error: invalid operand
+#CHECK: vgmf	%v0, 0, -1
+#CHECK: error: invalid operand
+#CHECK: vgmf	%v0, -1, 0
+#CHECK: error: invalid operand
+#CHECK: vgmf	%v0, 256, 0
+
+	vgmf	%v0, 0, -1
+	vgmf	%v0, 0, -1
+	vgmf	%v0, -1, 0
+	vgmf	%v0, 256, 0
+
+#CHECK: error: invalid operand
+#CHECK: vgmg	%v0, 0, -1
+#CHECK: error: invalid operand
+#CHECK: vgmg	%v0, 0, -1
+#CHECK: error: invalid operand
+#CHECK: vgmg	%v0, -1, 0
+#CHECK: error: invalid operand
+#CHECK: vgmg	%v0, 256, 0
+
+	vgmg	%v0, 0, -1
+	vgmg	%v0, 0, -1
+	vgmg	%v0, -1, 0
+	vgmg	%v0, 256, 0
+
+#CHECK: error: invalid operand
+#CHECK: vgmh	%v0, 0, -1
+#CHECK: error: invalid operand
+#CHECK: vgmh	%v0, 0, -1
+#CHECK: error: invalid operand
+#CHECK: vgmh	%v0, -1, 0
+#CHECK: error: invalid operand
+#CHECK: vgmh	%v0, 256, 0
+
+	vgmh	%v0, 0, -1
+	vgmh	%v0, 0, -1
+	vgmh	%v0, -1, 0
+	vgmh	%v0, 256, 0
+
+#CHECK: error: invalid operand
+#CHECK: vl	%v0, -1
+#CHECK: error: invalid operand
+#CHECK: vl	%v0, 4096
+#CHECK: error: invalid use of vector addressing
+#CHECK: vl	%v0, 0(%v1,%r2)
+
+	vl	%v0, -1
+	vl	%v0, 4096
+	vl	%v0, 0(%v1,%r2)
+
+#CHECK: error: invalid operand
+#CHECK: vlbb	%v0, 0, -1
+#CHECK: error: invalid operand
+#CHECK: vlbb	%v0, 0, 16
+#CHECK: error: invalid operand
+#CHECK: vlbb	%v0, -1, 0
+#CHECK: error: invalid operand
+#CHECK: vlbb	%v0, 4096, 0
+#CHECK: error: invalid use of vector addressing
+#CHECK: vlbb	%v0, 0(%v1,%r2), 0
+
+	vlbb	%v0, 0, -1
+	vlbb	%v0, 0, 16
+	vlbb	%v0, -1, 0
+	vlbb	%v0, 4096, 0
+	vlbb	%v0, 0(%v1,%r2), 0
+
+#CHECK: error: invalid operand
+#CHECK: vleb	%v0, 0, -1
+#CHECK: error: invalid operand
+#CHECK: vleb	%v0, 0, 16
+#CHECK: error: invalid operand
+#CHECK: vleb	%v0, -1, 0
+#CHECK: error: invalid operand
+#CHECK: vleb	%v0, 4096, 0
+#CHECK: error: invalid use of vector addressing
+#CHECK: vleb	%v0, 0(%v1,%r2), 0
+
+	vleb	%v0, 0, -1
+	vleb	%v0, 0, 16
+	vleb	%v0, -1, 0
+	vleb	%v0, 4096, 0
+	vleb	%v0, 0(%v1,%r2), 0
+
+#CHECK: error: invalid operand
+#CHECK: vledb	%v0, %v0, 0, -1
+#CHECK: error: invalid operand
+#CHECK: vledb	%v0, %v0, 0, 16
+#CHECK: error: invalid operand
+#CHECK: vledb	%v0, %v0, -1, 0
+#CHECK: error: invalid operand
+#CHECK: vledb	%v0, %v0, 16, 0
+
+	vledb	%v0, %v0, 0, -1
+	vledb	%v0, %v0, 0, 16
+	vledb	%v0, %v0, -1, 0
+	vledb	%v0, %v0, 16, 0
+
+#CHECK: error: invalid operand
+#CHECK: vlef	%v0, 0, -1
+#CHECK: error: invalid operand
+#CHECK: vlef	%v0, 0, 4
+#CHECK: error: invalid operand
+#CHECK: vlef	%v0, -1, 0
+#CHECK: error: invalid operand
+#CHECK: vlef	%v0, 4096, 0
+#CHECK: error: invalid use of vector addressing
+#CHECK: vlef	%v0, 0(%v1,%r2), 0
+
+	vlef	%v0, 0, -1
+	vlef	%v0, 0, 4
+	vlef	%v0, -1, 0
+	vlef	%v0, 4096, 0
+	vlef	%v0, 0(%v1,%r2), 0
+
+#CHECK: error: invalid operand
+#CHECK: vleg	%v0, 0, -1
+#CHECK: error: invalid operand
+#CHECK: vleg	%v0, 0, 2
+#CHECK: error: invalid operand
+#CHECK: vleg	%v0, -1, 0
+#CHECK: error: invalid operand
+#CHECK: vleg	%v0, 4096, 0
+#CHECK: error: invalid use of vector addressing
+#CHECK: vleg	%v0, 0(%v1,%r2), 0
+
+	vleg	%v0, 0, -1
+	vleg	%v0, 0, 2
+	vleg	%v0, -1, 0
+	vleg	%v0, 4096, 0
+	vleg	%v0, 0(%v1,%r2), 0
+
+#CHECK: error: invalid operand
+#CHECK: vleh	%v0, 0, -1
+#CHECK: error: invalid operand
+#CHECK: vleh	%v0, 0, 8
+#CHECK: error: invalid operand
+#CHECK: vleh	%v0, -1, 0
+#CHECK: error: invalid operand
+#CHECK: vleh	%v0, 4096, 0
+#CHECK: error: invalid use of vector addressing
+#CHECK: vleh	%v0, 0(%v1,%r2), 0
+
+	vleh	%v0, 0, -1
+	vleh	%v0, 0, 8
+	vleh	%v0, -1, 0
+	vleh	%v0, 4096, 0
+	vleh	%v0, 0(%v1,%r2), 0
+
+#CHECK: error: invalid operand
+#CHECK: vleib	%v0, 0, -1
+#CHECK: error: invalid operand
+#CHECK: vleib	%v0, 0, 16
+#CHECK: error: invalid operand
+#CHECK: vleib	%v0, -32769, 0
+#CHECK: error: invalid operand
+#CHECK: vleib	%v0, 32768, 0
+
+	vleib	%v0, 0, -1
+	vleib	%v0, 0, 16
+	vleib	%v0, -32769, 0
+	vleib	%v0, 32768, 0
+
+#CHECK: error: invalid operand
+#CHECK: vleif	%v0, 0, -1
+#CHECK: error: invalid operand
+#CHECK: vleif	%v0, 0, 4
+#CHECK: error: invalid operand
+#CHECK: vleif	%v0, -32769, 0
+#CHECK: error: invalid operand
+#CHECK: vleif	%v0, 32768, 0
+
+	vleif	%v0, 0, -1
+	vleif	%v0, 0, 4
+	vleif	%v0, -32769, 0
+	vleif	%v0, 32768, 0
+
+#CHECK: error: invalid operand
+#CHECK: vleig	%v0, 0, -1
+#CHECK: error: invalid operand
+#CHECK: vleig	%v0, 0, 2
+#CHECK: error: invalid operand
+#CHECK: vleig	%v0, -32769, 0
+#CHECK: error: invalid operand
+#CHECK: vleig	%v0, 32768, 0
+
+	vleig	%v0, 0, -1
+	vleig	%v0, 0, 2
+	vleig	%v0, -32769, 0
+	vleig	%v0, 32768, 0
+
+#CHECK: error: invalid operand
+#CHECK: vleih	%v0, 0, -1
+#CHECK: error: invalid operand
+#CHECK: vleih	%v0, 0, 8
+#CHECK: error: invalid operand
+#CHECK: vleih	%v0, -32769, 0
+#CHECK: error: invalid operand
+#CHECK: vleih	%v0, 32768, 0
+
+	vleih	%v0, 0, -1
+	vleih	%v0, 0, 8
+	vleih	%v0, -32769, 0
+	vleih	%v0, 32768, 0
+
+#CHECK: error: invalid operand
+#CHECK: vlgvb	%r0, %v0, -1
+#CHECK: error: invalid operand
+#CHECK: vlgvb	%r0, %v0, 4096
+#CHECK: error: %r0 used in an address
+#CHECK: vlgvb	%r0, %v0, 0(%r0)
+
+	vlgvb	%r0, %v0, -1
+	vlgvb	%r0, %v0, 4096
+	vlgvb	%r0, %v0, 0(%r0)
+
+#CHECK: error: invalid operand
+#CHECK: vlgvf	%r0, %v0, -1
+#CHECK: error: invalid operand
+#CHECK: vlgvf	%r0, %v0, 4096
+#CHECK: error: %r0 used in an address
+#CHECK: vlgvf	%r0, %v0, 0(%r0)
+
+	vlgvf	%r0, %v0, -1
+	vlgvf	%r0, %v0, 4096
+	vlgvf	%r0, %v0, 0(%r0)
+
+#CHECK: error: invalid operand
+#CHECK: vlgvg	%r0, %v0, -1
+#CHECK: error: invalid operand
+#CHECK: vlgvg	%r0, %v0, 4096
+#CHECK: error: %r0 used in an address
+#CHECK: vlgvg	%r0, %v0, 0(%r0)
+
+	vlgvg	%r0, %v0, -1
+	vlgvg	%r0, %v0, 4096
+	vlgvg	%r0, %v0, 0(%r0)
+
+#CHECK: error: invalid operand
+#CHECK: vlgvh	%r0, %v0, -1
+#CHECK: error: invalid operand
+#CHECK: vlgvh	%r0, %v0, 4096
+#CHECK: error: %r0 used in an address
+#CHECK: vlgvh	%r0, %v0, 0(%r0)
+
+	vlgvh	%r0, %v0, -1
+	vlgvh	%r0, %v0, 4096
+	vlgvh	%r0, %v0, 0(%r0)
+
+#CHECK: error: invalid operand
+#CHECK: vll	%v0, %r0, -1
+#CHECK: error: invalid operand
+#CHECK: vll	%v0, %r0, 4096
+#CHECK: error: %r0 used in an address
+#CHECK: vll	%v0, %r0, 0(%r0)
+
+	vll	%v0, %r0, -1
+	vll	%v0, %r0, 4096
+	vll	%v0, %r0, 0(%r0)
+
+#CHECK: error: invalid operand
+#CHECK: vllezb	%v0, -1
+#CHECK: error: invalid operand
+#CHECK: vllezb	%v0, 4096
+#CHECK: error: invalid use of vector addressing
+#CHECK: vllezb	%v0, 0(%v1,%r2)
+
+	vllezb	%v0, -1
+	vllezb	%v0, 4096
+	vllezb	%v0, 0(%v1,%r2)
+
+#CHECK: error: invalid operand
+#CHECK: vllezf	%v0, -1
+#CHECK: error: invalid operand
+#CHECK: vllezf	%v0, 4096
+#CHECK: error: invalid use of vector addressing
+#CHECK: vllezf	%v0, 0(%v1,%r2)
+
+	vllezf	%v0, -1
+	vllezf	%v0, 4096
+	vllezf	%v0, 0(%v1,%r2)
+
+#CHECK: error: invalid operand
+#CHECK: vllezg	%v0, -1
+#CHECK: error: invalid operand
+#CHECK: vllezg	%v0, 4096
+#CHECK: error: invalid use of vector addressing
+#CHECK: vllezg	%v0, 0(%v1,%r2)
+
+	vllezg	%v0, -1
+	vllezg	%v0, 4096
+	vllezg	%v0, 0(%v1,%r2)
+
+#CHECK: error: invalid operand
+#CHECK: vllezh	%v0, -1
+#CHECK: error: invalid operand
+#CHECK: vllezh	%v0, 4096
+#CHECK: error: invalid use of vector addressing
+#CHECK: vllezh	%v0, 0(%v1,%r2)
+
+	vllezh	%v0, -1
+	vllezh	%v0, 4096
+	vllezh	%v0, 0(%v1,%r2)
+
+#CHECK: error: invalid operand
+#CHECK: vlm	%v0, %v0, -1
+#CHECK: error: invalid operand
+#CHECK: vlm	%v0, %v0, 4096
+
+	vlm	%v0, %v0, -1
+	vlm	%v0, %v0, 4096
+
+#CHECK: error: invalid operand
+#CHECK: vlrepb	%v0, -1
+#CHECK: error: invalid operand
+#CHECK: vlrepb	%v0, 4096
+#CHECK: error: invalid use of vector addressing
+#CHECK: vlrepb	%v0, 0(%v1,%r2)
+
+	vlrepb	%v0, -1
+	vlrepb	%v0, 4096
+	vlrepb	%v0, 0(%v1,%r2)
+
+#CHECK: error: invalid operand
+#CHECK: vlrepf	%v0, -1
+#CHECK: error: invalid operand
+#CHECK: vlrepf	%v0, 4096
+#CHECK: error: invalid use of vector addressing
+#CHECK: vlrepf	%v0, 0(%v1,%r2)
+
+	vlrepf	%v0, -1
+	vlrepf	%v0, 4096
+	vlrepf	%v0, 0(%v1,%r2)
+
+#CHECK: error: invalid operand
+#CHECK: vlrepg	%v0, -1
+#CHECK: error: invalid operand
+#CHECK: vlrepg	%v0, 4096
+#CHECK: error: invalid use of vector addressing
+#CHECK: vlrepg	%v0, 0(%v1,%r2)
+
+	vlrepg	%v0, -1
+	vlrepg	%v0, 4096
+	vlrepg	%v0, 0(%v1,%r2)
+
+#CHECK: error: invalid operand
+#CHECK: vlreph	%v0, -1
+#CHECK: error: invalid operand
+#CHECK: vlreph	%v0, 4096
+#CHECK: error: invalid use of vector addressing
+#CHECK: vlreph	%v0, 0(%v1,%r2)
+
+	vlreph	%v0, -1
+	vlreph	%v0, 4096
+	vlreph	%v0, 0(%v1,%r2)
+
+#CHECK: error: invalid operand
+#CHECK: vlvgb	%v0, %r0, -1
+#CHECK: error: invalid operand
+#CHECK: vlvgb	%v0, %r0, 4096
+#CHECK: error: %r0 used in an address
+#CHECK: vlvgb	%v0, %r0, 0(%r0)
+
+	vlvgb	%v0, %r0, -1
+	vlvgb	%v0, %r0, 4096
+	vlvgb	%v0, %r0, 0(%r0)
+
+#CHECK: error: invalid operand
+#CHECK: vlvgf	%v0, %r0, -1
+#CHECK: error: invalid operand
+#CHECK: vlvgf	%v0, %r0, 4096
+#CHECK: error: %r0 used in an address
+#CHECK: vlvgf	%v0, %r0, 0(%r0)
+
+	vlvgf	%v0, %r0, -1
+	vlvgf	%v0, %r0, 4096
+	vlvgf	%v0, %r0, 0(%r0)
+
+#CHECK: error: invalid operand
+#CHECK: vlvgg	%v0, %r0, -1
+#CHECK: error: invalid operand
+#CHECK: vlvgg	%v0, %r0, 4096
+#CHECK: error: %r0 used in an address
+#CHECK: vlvgg	%v0, %r0, 0(%r0)
+
+	vlvgg	%v0, %r0, -1
+	vlvgg	%v0, %r0, 4096
+	vlvgg	%v0, %r0, 0(%r0)
+
+#CHECK: error: invalid operand
+#CHECK: vlvgh	%v0, %r0, -1
+#CHECK: error: invalid operand
+#CHECK: vlvgh	%v0, %r0, 4096
+#CHECK: error: %r0 used in an address
+#CHECK: vlvgh	%v0, %r0, 0(%r0)
+
+	vlvgh	%v0, %r0, -1
+	vlvgh	%v0, %r0, 4096
+	vlvgh	%v0, %r0, 0(%r0)
+
+#CHECK: error: invalid operand
+#CHECK: vpdi	%v0, %v0, %v0, -1
+#CHECK: error: invalid operand
+#CHECK: vpdi	%v0, %v0, %v0, 16
+
+	vpdi	%v0, %v0, %v0, -1
+	vpdi	%v0, %v0, %v0, 16
+
+#CHECK: error: invalid operand
+#CHECK: vrepb	%v0, %v0, -1
+#CHECK: error: invalid operand
+#CHECK: vrepb	%v0, %v0, 65536
+
+	vrepb	%v0, %v0, -1
+	vrepb	%v0, %v0, 65536
+
+#CHECK: error: invalid operand
+#CHECK: vrepf	%v0, %v0, -1
+#CHECK: error: invalid operand
+#CHECK: vrepf	%v0, %v0, 65536
+
+	vrepf	%v0, %v0, -1
+	vrepf	%v0, %v0, 65536
+
+#CHECK: error: invalid operand
+#CHECK: vrepg	%v0, %v0, -1
+#CHECK: error: invalid operand
+#CHECK: vrepg	%v0, %v0, 65536
+
+	vrepg	%v0, %v0, -1
+	vrepg	%v0, %v0, 65536
+
+#CHECK: error: invalid operand
+#CHECK: vreph	%v0, %v0, -1
+#CHECK: error: invalid operand
+#CHECK: vreph	%v0, %v0, 65536
+
+	vreph	%v0, %v0, -1
+	vreph	%v0, %v0, 65536
+
+#CHECK: error: invalid operand
+#CHECK: vrepib	%v0, -32769
+#CHECK: error: invalid operand
+#CHECK: vrepib	%v0, 32768
+
+	vrepib	%v0, -32769
+	vrepib	%v0, 32768
+
+#CHECK: error: invalid operand
+#CHECK: vrepif	%v0, -32769
+#CHECK: error: invalid operand
+#CHECK: vrepif	%v0, 32768
+
+	vrepif	%v0, -32769
+	vrepif	%v0, 32768
+
+#CHECK: error: invalid operand
+#CHECK: vrepig	%v0, -32769
+#CHECK: error: invalid operand
+#CHECK: vrepig	%v0, 32768
+
+	vrepig	%v0, -32769
+	vrepig	%v0, 32768
+
+#CHECK: error: invalid operand
+#CHECK: vrepih	%v0, -32769
+#CHECK: error: invalid operand
+#CHECK: vrepih	%v0, 32768
+
+	vrepih	%v0, -32769
+	vrepih	%v0, 32768
+
+#CHECK: error: vector index required
+#CHECK: vscef	%v0, 0(%r1), 0
+#CHECK: error: vector index required
+#CHECK: vscef	%v0, 0(%r2,%r1), 0
+#CHECK: error: invalid operand
+#CHECK: vscef	%v0, 0(%v0,%r1), -1
+#CHECK: error: invalid operand
+#CHECK: vscef	%v0, 0(%v0,%r1), 4
+#CHECK: error: invalid operand
+#CHECK: vscef	%v0, -1(%v0,%r1), 0
+#CHECK: error: invalid operand
+#CHECK: vscef	%v0, 4096(%v0,%r1), 0
+
+	vscef	%v0, 0(%r1), 0
+	vscef	%v0, 0(%r2,%r1), 0
+	vscef	%v0, 0(%v0,%r1), -1
+	vscef	%v0, 0(%v0,%r1), 4
+	vscef	%v0, -1(%v0,%r1), 0
+	vscef	%v0, 4096(%v0,%r1), 0
+
+#CHECK: error: vector index required
+#CHECK: vsceg	%v0, 0(%r1), 0
+#CHECK: error: vector index required
+#CHECK: vsceg	%v0, 0(%r2,%r1), 0
+#CHECK: error: invalid operand
+#CHECK: vsceg	%v0, 0(%v0,%r1), -1
+#CHECK: error: invalid operand
+#CHECK: vsceg	%v0, 0(%v0,%r1), 2
+#CHECK: error: invalid operand
+#CHECK: vsceg	%v0, -1(%v0,%r1), 0
+#CHECK: error: invalid operand
+#CHECK: vsceg	%v0, 4096(%v0,%r1), 0
+
+	vsceg	%v0, 0(%r1), 0
+	vsceg	%v0, 0(%r2,%r1), 0
+	vsceg	%v0, 0(%v0,%r1), -1
+	vsceg	%v0, 0(%v0,%r1), 2
+	vsceg	%v0, -1(%v0,%r1), 0
+	vsceg	%v0, 4096(%v0,%r1), 0
+
+#CHECK: error: invalid operand
+#CHECK: vsldb	%v0, %v0, %v0, -1
+#CHECK: error: invalid operand
+#CHECK: vsldb	%v0, %v0, %v0, 256
+
+	vsldb	%v0, %v0, %v0, -1
+	vsldb	%v0, %v0, %v0, 256
+
+#CHECK: error: invalid operand
+#CHECK: vst	%v0, -1
+#CHECK: error: invalid operand
+#CHECK: vst	%v0, 4096
+#CHECK: error: invalid use of vector addressing
+#CHECK: vst	%v0, 0(%v1,%r2)
+
+	vst	%v0, -1
+	vst	%v0, 4096
+	vst	%v0, 0(%v1,%r2)
+
+#CHECK: error: invalid operand
+#CHECK: vsteb	%v0, 0, -1
+#CHECK: error: invalid operand
+#CHECK: vsteb	%v0, 0, 16
+#CHECK: error: invalid operand
+#CHECK: vsteb	%v0, -1, 0
+#CHECK: error: invalid operand
+#CHECK: vsteb	%v0, 4096, 0
+#CHECK: error: invalid use of vector addressing
+#CHECK: vsteb	%v0, 0(%v1,%r2), 0
+
+	vsteb	%v0, 0, -1
+	vsteb	%v0, 0, 16
+	vsteb	%v0, -1, 0
+	vsteb	%v0, 4096, 0
+	vsteb	%v0, 0(%v1,%r2), 0
+
+#CHECK: error: invalid operand
+#CHECK: vstef	%v0, 0, -1
+#CHECK: error: invalid operand
+#CHECK: vstef	%v0, 0, 4
+#CHECK: error: invalid operand
+#CHECK: vstef	%v0, -1, 0
+#CHECK: error: invalid operand
+#CHECK: vstef	%v0, 4096, 0
+#CHECK: error: invalid use of vector addressing
+#CHECK: vstef	%v0, 0(%v1,%r2), 0
+
+	vstef	%v0, 0, -1
+	vstef	%v0, 0, 4
+	vstef	%v0, -1, 0
+	vstef	%v0, 4096, 0
+	vstef	%v0, 0(%v1,%r2), 0
+
+#CHECK: error: invalid operand
+#CHECK: vsteg	%v0, 0, -1
+#CHECK: error: invalid operand
+#CHECK: vsteg	%v0, 0, 2
+#CHECK: error: invalid operand
+#CHECK: vsteg	%v0, -1, 0
+#CHECK: error: invalid operand
+#CHECK: vsteg	%v0, 4096, 0
+#CHECK: error: invalid use of vector addressing
+#CHECK: vsteg	%v0, 0(%v1,%r2), 0
+
+	vsteg	%v0, 0, -1
+	vsteg	%v0, 0, 2
+	vsteg	%v0, -1, 0
+	vsteg	%v0, 4096, 0
+	vsteg	%v0, 0(%v1,%r2), 0
+
+#CHECK: error: invalid operand
+#CHECK: vsteh	%v0, 0, -1
+#CHECK: error: invalid operand
+#CHECK: vsteh	%v0, 0, 8
+#CHECK: error: invalid operand
+#CHECK: vsteh	%v0, -1, 0
+#CHECK: error: invalid operand
+#CHECK: vsteh	%v0, 4096, 0
+#CHECK: error: invalid use of vector addressing
+#CHECK: vsteh	%v0, 0(%v1,%r2), 0
+
+	vsteh	%v0, 0, -1
+	vsteh	%v0, 0, 8
+	vsteh	%v0, -1, 0
+	vsteh	%v0, 4096, 0
+	vsteh	%v0, 0(%v1,%r2), 0
+
+#CHECK: error: invalid operand
+#CHECK: vstl	%v0, %r0, -1
+#CHECK: error: invalid operand
+#CHECK: vstl	%v0, %r0, 4096
+#CHECK: error: %r0 used in an address
+#CHECK: vstl	%v0, %r0, 0(%r0)
+
+	vstl	%v0, %r0, -1
+	vstl	%v0, %r0, 4096
+	vstl	%v0, %r0, 0(%r0)
+
+#CHECK: error: invalid operand
+#CHECK: vstm	%v0, %v0, -1
+#CHECK: error: invalid operand
+#CHECK: vstm	%v0, %v0, 4096
+
+	vstm	%v0, %v0, -1
+	vstm	%v0, %v0, 4096
+
+#CHECK: error: invalid operand
+#CHECK: vstrcb   %v0, %v0, %v0, %v0, -1
+#CHECK: error: invalid operand
+#CHECK: vstrcb   %v0, %v0, %v0, %v0, 16
+#CHECK: error: too few operands
+#CHECK: vstrcb   %v0, %v0, %v0
+#CHECK: error: invalid operand
+#CHECK: vstrcb   %v0, %v0, %v0, %v0, 0, 0
+
+        vstrcb   %v0, %v0, %v0, %v0, -1
+        vstrcb   %v0, %v0, %v0, %v0, 16
+        vstrcb   %v0, %v0, %v0
+        vstrcb   %v0, %v0, %v0, %v0, 0, 0
+
+#CHECK: error: invalid operand
+#CHECK: vstrcbs  %v0, %v0, %v0, %v0, -1
+#CHECK: error: invalid operand
+#CHECK: vstrcbs  %v0, %v0, %v0, %v0, 16
+#CHECK: error: too few operands
+#CHECK: vstrcbs  %v0, %v0, %v0
+#CHECK: error: invalid operand
+#CHECK: vstrcbs  %v0, %v0, %v0, %v0, 0, 0
+
+        vstrcbs  %v0, %v0, %v0, %v0, -1
+        vstrcbs  %v0, %v0, %v0, %v0, 16
+        vstrcbs  %v0, %v0, %v0
+        vstrcbs  %v0, %v0, %v0, %v0, 0, 0
+
+#CHECK: error: invalid operand
+#CHECK: vstrcf   %v0, %v0, %v0, %v0, -1
+#CHECK: error: invalid operand
+#CHECK: vstrcf   %v0, %v0, %v0, %v0, 16
+#CHECK: error: too few operands
+#CHECK: vstrcf   %v0, %v0, %v0
+#CHECK: error: invalid operand
+#CHECK: vstrcf   %v0, %v0, %v0, %v0, 0, 0
+
+        vstrcf   %v0, %v0, %v0, %v0, -1
+        vstrcf   %v0, %v0, %v0, %v0, 16
+        vstrcf   %v0, %v0, %v0
+        vstrcf   %v0, %v0, %v0, %v0, 0, 0
+
+#CHECK: error: invalid operand
+#CHECK: vstrch   %v0, %v0, %v0, %v0, -1
+#CHECK: error: invalid operand
+#CHECK: vstrch   %v0, %v0, %v0, %v0, 16
+#CHECK: error: too few operands
+#CHECK: vstrch   %v0, %v0, %v0
+#CHECK: error: invalid operand
+#CHECK: vstrch   %v0, %v0, %v0, %v0, 0, 0
+
+        vstrch   %v0, %v0, %v0, %v0, -1
+        vstrch   %v0, %v0, %v0, %v0, 16
+        vstrch   %v0, %v0, %v0
+        vstrch   %v0, %v0, %v0, %v0, 0, 0
+
+#CHECK: error: invalid operand
+#CHECK: vstrczh  %v0, %v0, %v0, %v0, -1
+#CHECK: error: invalid operand
+#CHECK: vstrczh  %v0, %v0, %v0, %v0, 16
+#CHECK: error: too few operands
+#CHECK: vstrczh  %v0, %v0, %v0
+#CHECK: error: invalid operand
+#CHECK: vstrczh  %v0, %v0, %v0, %v0, 0, 0
+
+        vstrczh  %v0, %v0, %v0, %v0, -1
+        vstrczh  %v0, %v0, %v0, %v0, 16
+        vstrczh  %v0, %v0, %v0
+        vstrczh  %v0, %v0, %v0, %v0, 0, 0
+
+#CHECK: error: invalid operand
+#CHECK: vstrczfs %v0, %v0, %v0, %v0, -1
+#CHECK: error: invalid operand
+#CHECK: vstrczfs %v0, %v0, %v0, %v0, 16
+#CHECK: error: too few operands
+#CHECK: vstrczfs %v0, %v0, %v0
+#CHECK: error: invalid operand
+#CHECK: vstrczfs %v0, %v0, %v0, %v0, 0, 0
+
+        vstrczfs %v0, %v0, %v0, %v0, -1
+        vstrczfs %v0, %v0, %v0, %v0, 16
+        vstrczfs %v0, %v0, %v0
+        vstrczfs %v0, %v0, %v0, %v0, 0, 0
+
+#CHECK: error: invalid operand
+#CHECK: wcdgb	%v0, %v0, 0, -1
+#CHECK: error: invalid operand
+#CHECK: wcdgb	%v0, %v0, 0, 16
+#CHECK: error: invalid operand
+#CHECK: wcdgb	%v0, %v0, -1, 0
+#CHECK: error: invalid operand
+#CHECK: wcdgb	%v0, %v0, 16, 0
+
+	wcdgb	%v0, %v0, 0, -1
+	wcdgb	%v0, %v0, 0, 16
+	wcdgb	%v0, %v0, -1, 0
+	wcdgb	%v0, %v0, 16, 0
+
+#CHECK: error: invalid operand
+#CHECK: wcdlgb	%v0, %v0, 0, -1
+#CHECK: error: invalid operand
+#CHECK: wcdlgb	%v0, %v0, 0, 16
+#CHECK: error: invalid operand
+#CHECK: wcdlgb	%v0, %v0, -1, 0
+#CHECK: error: invalid operand
+#CHECK: wcdlgb	%v0, %v0, 16, 0
+
+	wcdlgb	%v0, %v0, 0, -1
+	wcdlgb	%v0, %v0, 0, 16
+	wcdlgb	%v0, %v0, -1, 0
+	wcdlgb	%v0, %v0, 16, 0
+
+#CHECK: error: invalid operand
+#CHECK: wcgdb	%v0, %v0, 0, -1
+#CHECK: error: invalid operand
+#CHECK: wcgdb	%v0, %v0, 0, 16
+#CHECK: error: invalid operand
+#CHECK: wcgdb	%v0, %v0, -1, 0
+#CHECK: error: invalid operand
+#CHECK: wcgdb	%v0, %v0, 16, 0
+
+	wcgdb	%v0, %v0, 0, -1
+	wcgdb	%v0, %v0, 0, 16
+	wcgdb	%v0, %v0, -1, 0
+	wcgdb	%v0, %v0, 16, 0
+
+#CHECK: error: invalid operand
+#CHECK: wclgdb	%v0, %v0, 0, -1
+#CHECK: error: invalid operand
+#CHECK: wclgdb	%v0, %v0, 0, 16
+#CHECK: error: invalid operand
+#CHECK: wclgdb	%v0, %v0, -1, 0
+#CHECK: error: invalid operand
+#CHECK: wclgdb	%v0, %v0, 16, 0
+
+	wclgdb	%v0, %v0, 0, -1
+	wclgdb	%v0, %v0, 0, 16
+	wclgdb	%v0, %v0, -1, 0
+	wclgdb	%v0, %v0, 16, 0
+
+#CHECK: error: invalid operand
+#CHECK: wfidb	%v0, %v0, 0, -1
+#CHECK: error: invalid operand
+#CHECK: wfidb	%v0, %v0, 0, 16
+#CHECK: error: invalid operand
+#CHECK: wfidb	%v0, %v0, -1, 0
+#CHECK: error: invalid operand
+#CHECK: wfidb	%v0, %v0, 16, 0
+
+	wfidb	%v0, %v0, 0, -1
+	wfidb	%v0, %v0, 0, 16
+	wfidb	%v0, %v0, -1, 0
+	wfidb	%v0, %v0, 16, 0
+
+#CHECK: error: invalid operand
+#CHECK: wftcidb	%v0, %v0, -1
+#CHECK: error: invalid operand
+#CHECK: wftcidb	%v0, %v0, 4096
+
+	wftcidb	%v0, %v0, -1
+	wftcidb	%v0, %v0, 4096
+
+#CHECK: error: invalid operand
+#CHECK: wledb	%v0, %v0, 0, -1
+#CHECK: error: invalid operand
+#CHECK: wledb	%v0, %v0, 0, 16
+#CHECK: error: invalid operand
+#CHECK: wledb	%v0, %v0, -1, 0
+#CHECK: error: invalid operand
+#CHECK: wledb	%v0, %v0, 16, 0
+
+	wledb	%v0, %v0, 0, -1
+	wledb	%v0, %v0, 0, 16
+	wledb	%v0, %v0, -1, 0
+	wledb	%v0, %v0, 16, 0
diff --git a/test/MC/SystemZ/insn-bad-zEC12.s b/test/MC/SystemZ/insn-bad-zEC12.s
index d96e35d885b..ee31f5707af 100644
--- a/test/MC/SystemZ/insn-bad-zEC12.s
+++ b/test/MC/SystemZ/insn-bad-zEC12.s
@@ -2,6 +2,11 @@
 # RUN: not llvm-mc -triple s390x-linux-gnu -mcpu=zEC12 < %s 2> %t
 # RUN: FileCheck < %t %s
 
+#CHECK: error: {{(instruction requires: vector)?}}
+#CHECK: lcbb	%r0, 0, 0
+
+	lcbb	%r0, 0, 0
+
 #CHECK: error: invalid operand
 #CHECK: ntstg	%r0, -524289
 #CHECK: error: invalid operand
@@ -82,3 +87,1492 @@
 	tbeginc	0(%r1,%r2), 0
 	tbeginc	0, -1
 	tbeginc	0, 65536
+
+#CHECK: error: {{(instruction requires: vector)?}}
+#CHECK: vab	%v0, %v0, %v0
+#CHECK: error: {{(instruction requires: vector)?}}
+#CHECK: vaf	%v0, %v0, %v0
+#CHECK: error: {{(instruction requires: vector)?}}
+#CHECK: vag	%v0, %v0, %v0
+#CHECK: error: {{(instruction requires: vector)?}}
+#CHECK: vah	%v0, %v0, %v0
+#CHECK: error: {{(instruction requires: vector)?}}
+#CHECK: vaq	%v0, %v0, %v0
+
+	vab	%v0, %v0, %v0
+	vaf	%v0, %v0, %v0
+	vag	%v0, %v0, %v0
+	vah	%v0, %v0, %v0
+	vaq	%v0, %v0, %v0
+
+#CHECK: error: {{(instruction requires: vector)?}}
+#CHECK: vaccb	%v0, %v0, %v0
+#CHECK: error: {{(instruction requires: vector)?}}
+#CHECK: vaccf	%v0, %v0, %v0
+#CHECK: error: {{(instruction requires: vector)?}}
+#CHECK: vaccg	%v0, %v0, %v0
+#CHECK: error: {{(instruction requires: vector)?}}
+#CHECK: vacch	%v0, %v0, %v0
+#CHECK: error: {{(instruction requires: vector)?}}
+#CHECK: vaccq	%v0, %v0, %v0
+
+	vaccb	%v0, %v0, %v0
+	vaccf	%v0, %v0, %v0
+	vaccg	%v0, %v0, %v0
+	vacch	%v0, %v0, %v0
+	vaccq	%v0, %v0, %v0
+
+#CHECK: error: {{(instruction requires: vector)?}}
+#CHECK: vacccq	%v0, %v0, %v0
+
+	vacccq	%v0, %v0, %v0
+
+#CHECK: error: {{(instruction requires: vector)?}}
+#CHECK: vacq	%v0, %v0, %v0
+
+	vacq	%v0, %v0, %v0
+
+#CHECK: error: {{(instruction requires: vector)?}}
+#CHECK: vavgb	%v0, %v0, %v0
+#CHECK: error: {{(instruction requires: vector)?}}
+#CHECK: vavgf	%v0, %v0, %v0
+#CHECK: error: {{(instruction requires: vector)?}}
+#CHECK: vavgg	%v0, %v0, %v0
+#CHECK: error: {{(instruction requires: vector)?}}
+#CHECK: vavgh	%v0, %v0, %v0
+
+	vavgb	%v0, %v0, %v0
+	vavgf	%v0, %v0, %v0
+	vavgg	%v0, %v0, %v0
+	vavgh	%v0, %v0, %v0
+
+#CHECK: error: {{(instruction requires: vector)?}}
+#CHECK: vavglb	%v0, %v0, %v0
+#CHECK: error: {{(instruction requires: vector)?}}
+#CHECK: vavglf	%v0, %v0, %v0
+#CHECK: error: {{(instruction requires: vector)?}}
+#CHECK: vavglg	%v0, %v0, %v0
+#CHECK: error: {{(instruction requires: vector)?}}
+#CHECK: vavglh	%v0, %v0, %v0
+
+	vavglb	%v0, %v0, %v0
+	vavglf	%v0, %v0, %v0
+	vavglg	%v0, %v0, %v0
+	vavglh	%v0, %v0, %v0
+
+#CHECK: error: {{(instruction requires: vector)?}}
+#CHECK: vcdgb	%v0, %v0, 0, 0
+
+	vcdgb	%v0, %v0, 0, 0
+
+#CHECK: error: {{(instruction requires: vector)?}}
+#CHECK: vcdlgb	%v0, %v0, 0, 0
+
+	vcdlgb	%v0, %v0, 0, 0
+
+#CHECK: error: {{(instruction requires: vector)?}}
+#CHECK: vceqb	%v0, %v0, %v0
+#CHECK: error: {{(instruction requires: vector)?}}
+#CHECK: vceqf	%v0, %v0, %v0
+#CHECK: error: {{(instruction requires: vector)?}}
+#CHECK: vceqg	%v0, %v0, %v0
+#CHECK: error: {{(instruction requires: vector)?}}
+#CHECK: vceqh	%v0, %v0, %v0
+#CHECK: error: {{(instruction requires: vector)?}}
+#CHECK: vceqbs	%v0, %v0, %v0
+#CHECK: error: {{(instruction requires: vector)?}}
+#CHECK: vceqhs	%v0, %v0, %v0
+#CHECK: error: {{(instruction requires: vector)?}}
+#CHECK: vceqfs	%v0, %v0, %v0
+#CHECK: error: {{(instruction requires: vector)?}}
+#CHECK: vceqgs	%v0, %v0, %v0
+
+	vceqb	%v0, %v0, %v0
+	vceqf	%v0, %v0, %v0
+	vceqg	%v0, %v0, %v0
+	vceqh	%v0, %v0, %v0
+	vceqbs	%v0, %v0, %v0
+	vceqhs	%v0, %v0, %v0
+	vceqfs	%v0, %v0, %v0
+	vceqgs	%v0, %v0, %v0
+
+#CHECK: error: {{(instruction requires: vector)?}}
+#CHECK: vcgdb	%v0, %v0, 0, 0
+
+	vcgdb	%v0, %v0, 0, 0
+
+#CHECK: error: {{(instruction requires: vector)?}}
+#CHECK: vchb	%v0, %v0, %v0
+#CHECK: error: {{(instruction requires: vector)?}}
+#CHECK: vchf	%v0, %v0, %v0
+#CHECK: error: {{(instruction requires: vector)?}}
+#CHECK: vchg	%v0, %v0, %v0
+#CHECK: error: {{(instruction requires: vector)?}}
+#CHECK: vchh	%v0, %v0, %v0
+#CHECK: error: {{(instruction requires: vector)?}}
+#CHECK: vchbs	%v0, %v0, %v0
+#CHECK: error: {{(instruction requires: vector)?}}
+#CHECK: vchhs	%v0, %v0, %v0
+#CHECK: error: {{(instruction requires: vector)?}}
+#CHECK: vchfs	%v0, %v0, %v0
+#CHECK: error: {{(instruction requires: vector)?}}
+#CHECK: vchgs	%v0, %v0, %v0
+
+	vchb	%v0, %v0, %v0
+	vchf	%v0, %v0, %v0
+	vchg	%v0, %v0, %v0
+	vchh	%v0, %v0, %v0
+	vchbs	%v0, %v0, %v0
+	vchhs	%v0, %v0, %v0
+	vchfs	%v0, %v0, %v0
+	vchgs	%v0, %v0, %v0
+
+#CHECK: error: {{(instruction requires: vector)?}}
+#CHECK: vchlb	%v0, %v0, %v0
+#CHECK: error: {{(instruction requires: vector)?}}
+#CHECK: vchlf	%v0, %v0, %v0
+#CHECK: error: {{(instruction requires: vector)?}}
+#CHECK: vchlg	%v0, %v0, %v0
+#CHECK: error: {{(instruction requires: vector)?}}
+#CHECK: vchlh	%v0, %v0, %v0
+#CHECK: error: {{(instruction requires: vector)?}}
+#CHECK: vchlbs	%v0, %v0, %v0
+#CHECK: error: {{(instruction requires: vector)?}}
+#CHECK: vchlhs	%v0, %v0, %v0
+#CHECK: error: {{(instruction requires: vector)?}}
+#CHECK: vchlfs	%v0, %v0, %v0
+#CHECK: error: {{(instruction requires: vector)?}}
+#CHECK: vchlgs	%v0, %v0, %v0
+
+	vchlb	%v0, %v0, %v0
+	vchlf	%v0, %v0, %v0
+	vchlg	%v0, %v0, %v0
+	vchlh	%v0, %v0, %v0
+	vchlbs	%v0, %v0, %v0
+	vchlhs	%v0, %v0, %v0
+	vchlfs	%v0, %v0, %v0
+	vchlgs	%v0, %v0, %v0
+
+#CHECK: error: {{(instruction requires: vector)?}}
+#CHECK: vcksm	%v0, %v0, %v0
+
+	vcksm	%v0, %v0, %v0
+
+#CHECK: error: {{(instruction requires: vector)?}}
+#CHECK: vclgdb	%v0, %v0, 0, 0
+
+	vclgdb	%v0, %v0, 0, 0
+
+#CHECK: error: {{(instruction requires: vector)?}}
+#CHECK: vclzb	%v0, %v0
+#CHECK: error: {{(instruction requires: vector)?}}
+#CHECK: vclzf	%v0, %v0
+#CHECK: error: {{(instruction requires: vector)?}}
+#CHECK: vclzg	%v0, %v0
+#CHECK: error: {{(instruction requires: vector)?}}
+#CHECK: vclzh	%v0, %v0
+
+	vclzb	%v0, %v0
+	vclzf	%v0, %v0
+	vclzg	%v0, %v0
+	vclzh	%v0, %v0
+
+#CHECK: error: {{(instruction requires: vector)?}}
+#CHECK: vctzb	%v0, %v0
+#CHECK: error: {{(instruction requires: vector)?}}
+#CHECK: vctzf	%v0, %v0
+#CHECK: error: {{(instruction requires: vector)?}}
+#CHECK: vctzg	%v0, %v0
+#CHECK: error: {{(instruction requires: vector)?}}
+#CHECK: vctzh	%v0, %v0
+
+	vctzb	%v0, %v0
+	vctzf	%v0, %v0
+	vctzg	%v0, %v0
+	vctzh	%v0, %v0
+
+#CHECK: error: {{(instruction requires: vector)?}}
+#CHECK: vecb	%v0, %v0
+#CHECK: error: {{(instruction requires: vector)?}}
+#CHECK: vecf	%v0, %v0
+#CHECK: error: {{(instruction requires: vector)?}}
+#CHECK: vecg	%v0, %v0
+#CHECK: error: {{(instruction requires: vector)?}}
+#CHECK: vech	%v0, %v0
+
+	vecb	%v0, %v0
+	vecf	%v0, %v0
+	vecg	%v0, %v0
+	vech	%v0, %v0
+
+#CHECK: error: {{(instruction requires: vector)?}}
+#CHECK: verimb	%v0, %v0, %v0, 0
+#CHECK: error: {{(instruction requires: vector)?}}
+#CHECK: verimf	%v0, %v0, %v0, 0
+#CHECK: error: {{(instruction requires: vector)?}}
+#CHECK: verimg	%v0, %v0, %v0, 0
+#CHECK: error: {{(instruction requires: vector)?}}
+#CHECK: verimh	%v0, %v0, %v0, 0
+
+	verimb	%v0, %v0, %v0, 0
+	verimf	%v0, %v0, %v0, 0
+	verimg	%v0, %v0, %v0, 0
+	verimh	%v0, %v0, %v0, 0
+
+#CHECK: error: {{(instruction requires: vector)?}}
+#CHECK: veclb	%v0, %v0
+#CHECK: error: {{(instruction requires: vector)?}}
+#CHECK: veclf	%v0, %v0
+#CHECK: error: {{(instruction requires: vector)?}}
+#CHECK: veclg	%v0, %v0
+#CHECK: error: {{(instruction requires: vector)?}}
+#CHECK: veclh	%v0, %v0
+
+	veclb	%v0, %v0
+	veclf	%v0, %v0
+	veclg	%v0, %v0
+	veclh	%v0, %v0
+
+#CHECK: error: {{(instruction requires: vector)?}}
+#CHECK: verllvb	%v0, %v0, %v0
+#CHECK: error: {{(instruction requires: vector)?}}
+#CHECK: verllvf	%v0, %v0, %v0
+#CHECK: error: {{(instruction requires: vector)?}}
+#CHECK: verllvg	%v0, %v0, %v0
+#CHECK: error: {{(instruction requires: vector)?}}
+#CHECK: verllvh	%v0, %v0, %v0
+
+	verllvb	%v0, %v0, %v0
+	verllvf	%v0, %v0, %v0
+	verllvg	%v0, %v0, %v0
+	verllvh	%v0, %v0, %v0
+
+#CHECK: error: {{(instruction requires: vector)?}}
+#CHECK: verllb	%v0, %v0, 0
+#CHECK: error: {{(instruction requires: vector)?}}
+#CHECK: verllf	%v0, %v0, 0
+#CHECK: error: {{(instruction requires: vector)?}}
+#CHECK: verllg	%v0, %v0, 0
+#CHECK: error: {{(instruction requires: vector)?}}
+#CHECK: verllh	%v0, %v0, 0
+
+	verllb	%v0, %v0, 0
+	verllf	%v0, %v0, 0
+	verllg	%v0, %v0, 0
+	verllh	%v0, %v0, 0
+
+#CHECK: error: {{(instruction requires: vector)?}}
+#CHECK: veslvb	%v0, %v0, %v0
+#CHECK: error: {{(instruction requires: vector)?}}
+#CHECK: veslvf	%v0, %v0, %v0
+#CHECK: error: {{(instruction requires: vector)?}}
+#CHECK: veslvg	%v0, %v0, %v0
+#CHECK: error: {{(instruction requires: vector)?}}
+#CHECK: veslvh	%v0, %v0, %v0
+
+	veslvb	%v0, %v0, %v0
+	veslvf	%v0, %v0, %v0
+	veslvg	%v0, %v0, %v0
+	veslvh	%v0, %v0, %v0
+
+#CHECK: error: {{(instruction requires: vector)?}}
+#CHECK: veslb	%v0, %v0, 0
+#CHECK: error: {{(instruction requires: vector)?}}
+#CHECK: veslf	%v0, %v0, 0
+#CHECK: error: {{(instruction requires: vector)?}}
+#CHECK: veslg	%v0, %v0, 0
+#CHECK: error: {{(instruction requires: vector)?}}
+#CHECK: veslh	%v0, %v0, 0
+
+	veslb	%v0, %v0, 0
+	veslf	%v0, %v0, 0
+	veslg	%v0, %v0, 0
+	veslh	%v0, %v0, 0
+
+#CHECK: error: {{(instruction requires: vector)?}}
+#CHECK: vesravb	%v0, %v0, %v0
+#CHECK: error: {{(instruction requires: vector)?}}
+#CHECK: vesravf	%v0, %v0, %v0
+#CHECK: error: {{(instruction requires: vector)?}}
+#CHECK: vesravg	%v0, %v0, %v0
+#CHECK: error: {{(instruction requires: vector)?}}
+#CHECK: vesravh	%v0, %v0, %v0
+
+	vesravb	%v0, %v0, %v0
+	vesravf	%v0, %v0, %v0
+	vesravg	%v0, %v0, %v0
+	vesravh	%v0, %v0, %v0
+
+#CHECK: error: {{(instruction requires: vector)?}}
+#CHECK: vesrab	%v0, %v0, 0
+#CHECK: error: {{(instruction requires: vector)?}}
+#CHECK: vesraf	%v0, %v0, 0
+#CHECK: error: {{(instruction requires: vector)?}}
+#CHECK: vesrag	%v0, %v0, 0
+#CHECK: error: {{(instruction requires: vector)?}}
+#CHECK: vesrah	%v0, %v0, 0
+
+	vesrab	%v0, %v0, 0
+	vesraf	%v0, %v0, 0
+	vesrag	%v0, %v0, 0
+	vesrah	%v0, %v0, 0
+
+#CHECK: error: {{(instruction requires: vector)?}}
+#CHECK: vesrlvb	%v0, %v0, %v0
+#CHECK: error: {{(instruction requires: vector)?}}
+#CHECK: vesrlvf	%v0, %v0, %v0
+#CHECK: error: {{(instruction requires: vector)?}}
+#CHECK: vesrlvg	%v0, %v0, %v0
+#CHECK: error: {{(instruction requires: vector)?}}
+#CHECK: vesrlvh	%v0, %v0, %v0
+
+	vesrlvb	%v0, %v0, %v0
+	vesrlvf	%v0, %v0, %v0
+	vesrlvg	%v0, %v0, %v0
+	vesrlvh	%v0, %v0, %v0
+
+#CHECK: error: {{(instruction requires: vector)?}}
+#CHECK: vesrlb	%v0, %v0, 0
+#CHECK: error: {{(instruction requires: vector)?}}
+#CHECK: vesrlf	%v0, %v0, 0
+#CHECK: error: {{(instruction requires: vector)?}}
+#CHECK: vesrlg	%v0, %v0, 0
+#CHECK: error: {{(instruction requires: vector)?}}
+#CHECK: vesrlh	%v0, %v0, 0
+
+	vesrlb	%v0, %v0, 0
+	vesrlf	%v0, %v0, 0
+	vesrlg	%v0, %v0, 0
+	vesrlh	%v0, %v0, 0
+
+#CHECK: error: {{(instruction requires: vector)?}}
+#CHECK: vfadb	%v0, %v0, %v0
+
+	vfadb	%v0, %v0, %v0
+
+#CHECK: error: {{(instruction requires: vector)?}}
+#CHECK: vfcedb	%v0, %v0, %v0
+#CHECK: vfcedbs	%v0, %v0, %v0
+
+	vfcedb	%v0, %v0, %v0
+	vfcedbs	%v0, %v0, %v0
+
+#CHECK: error: {{(instruction requires: vector)?}}
+#CHECK: vfchdb	%v0, %v0, %v0
+#CHECK: vfchdbs	%v0, %v0, %v0
+
+	vfchdb	%v0, %v0, %v0
+	vfchdbs	%v0, %v0, %v0
+
+#CHECK: error: {{(instruction requires: vector)?}}
+#CHECK: vfddb	%v0, %v0, %v0
+
+	vfddb	%v0, %v0, %v0
+
+#CHECK: error: {{(instruction requires: vector)?}}
+#CHECK: vfaeb	%v0, %v0, %v0
+#CHECK: error: {{(instruction requires: vector)?}}
+#CHECK: vfaezb	%v0, %v0, %v0
+#CHECK: error: {{(instruction requires: vector)?}}
+#CHECK: vfaebs	%v0, %v0, %v0
+#CHECK: error: {{(instruction requires: vector)?}}
+#CHECK: vfaezbs	%v0, %v0, %v0
+#CHECK: error: {{(instruction requires: vector)?}}
+#CHECK: vfaeh	%v0, %v0, %v0
+#CHECK: error: {{(instruction requires: vector)?}}
+#CHECK: vfaezh	%v0, %v0, %v0
+#CHECK: error: {{(instruction requires: vector)?}}
+#CHECK: vfaehs	%v0, %v0, %v0
+#CHECK: error: {{(instruction requires: vector)?}}
+#CHECK: vfaezhs	%v0, %v0, %v0
+#CHECK: error: {{(instruction requires: vector)?}}
+#CHECK: vfaef	%v0, %v0, %v0
+#CHECK: error: {{(instruction requires: vector)?}}
+#CHECK: vfaezf	%v0, %v0, %v0
+#CHECK: error: {{(instruction requires: vector)?}}
+#CHECK: vfaefs	%v0, %v0, %v0
+#CHECK: error: {{(instruction requires: vector)?}}
+#CHECK: vfaezfs	%v0, %v0, %v0
+
+	vfaeb	%v0, %v0, %v0
+	vfaezb	%v0, %v0, %v0
+	vfaebs	%v0, %v0, %v0
+	vfaezbs	%v0, %v0, %v0
+	vfaeh	%v0, %v0, %v0
+	vfaezh	%v0, %v0, %v0
+	vfaehs	%v0, %v0, %v0
+	vfaezhs	%v0, %v0, %v0
+	vfaef	%v0, %v0, %v0
+	vfaezf	%v0, %v0, %v0
+	vfaefs	%v0, %v0, %v0
+	vfaezfs	%v0, %v0, %v0
+
+#CHECK: error: {{(instruction requires: vector)?}}
+#CHECK: vfeeb	%v0, %v0, %v0
+#CHECK: error: {{(instruction requires: vector)?}}
+#CHECK: vfeezb	%v0, %v0, %v0
+#CHECK: error: {{(instruction requires: vector)?}}
+#CHECK: vfeebs	%v0, %v0, %v0
+#CHECK: error: {{(instruction requires: vector)?}}
+#CHECK: vfeezbs	%v0, %v0, %v0
+#CHECK: error: {{(instruction requires: vector)?}}
+#CHECK: vfeeh	%v0, %v0, %v0
+#CHECK: error: {{(instruction requires: vector)?}}
+#CHECK: vfeezh	%v0, %v0, %v0
+#CHECK: error: {{(instruction requires: vector)?}}
+#CHECK: vfeehs	%v0, %v0, %v0
+#CHECK: error: {{(instruction requires: vector)?}}
+#CHECK: vfeezhs	%v0, %v0, %v0
+#CHECK: error: {{(instruction requires: vector)?}}
+#CHECK: vfeef	%v0, %v0, %v0
+#CHECK: error: {{(instruction requires: vector)?}}
+#CHECK: vfeezf	%v0, %v0, %v0
+#CHECK: error: {{(instruction requires: vector)?}}
+#CHECK: vfeefs	%v0, %v0, %v0
+#CHECK: error: {{(instruction requires: vector)?}}
+#CHECK: vfeezfs	%v0, %v0, %v0
+
+	vfeeb	%v0, %v0, %v0
+	vfeezb	%v0, %v0, %v0
+	vfeebs	%v0, %v0, %v0
+	vfeezbs	%v0, %v0, %v0
+	vfeeh	%v0, %v0, %v0
+	vfeezh	%v0, %v0, %v0
+	vfeehs	%v0, %v0, %v0
+	vfeezhs	%v0, %v0, %v0
+	vfeef	%v0, %v0, %v0
+	vfeezf	%v0, %v0, %v0
+	vfeefs	%v0, %v0, %v0
+	vfeezfs	%v0, %v0, %v0
+
+#CHECK: error: {{(instruction requires: vector)?}}
+#CHECK: vfeneb   %v0, %v0, %v0
+#CHECK: error: {{(instruction requires: vector)?}}
+#CHECK: vfenezb  %v0, %v0, %v0
+#CHECK: error: {{(instruction requires: vector)?}}
+#CHECK: vfenebs  %v0, %v0, %v0
+#CHECK: error: {{(instruction requires: vector)?}}
+#CHECK: vfenezbs %v0, %v0, %v0
+#CHECK: error: {{(instruction requires: vector)?}}
+#CHECK: vfeneh   %v0, %v0, %v0
+#CHECK: error: {{(instruction requires: vector)?}}
+#CHECK: vfenezh  %v0, %v0, %v0
+#CHECK: error: {{(instruction requires: vector)?}}
+#CHECK: vfenehs  %v0, %v0, %v0
+#CHECK: error: {{(instruction requires: vector)?}}
+#CHECK: vfenezhs %v0, %v0, %v0
+#CHECK: error: {{(instruction requires: vector)?}}
+#CHECK: vfenef   %v0, %v0, %v0
+#CHECK: error: {{(instruction requires: vector)?}}
+#CHECK: vfenezf  %v0, %v0, %v0
+#CHECK: error: {{(instruction requires: vector)?}}
+#CHECK: vfenefs  %v0, %v0, %v0
+#CHECK: error: {{(instruction requires: vector)?}}
+#CHECK: vfenezfs %v0, %v0, %v0
+
+	vfeneb   %v0, %v0, %v0
+	vfenezb  %v0, %v0, %v0
+	vfenebs  %v0, %v0, %v0
+	vfenezbs %v0, %v0, %v0
+	vfeneh   %v0, %v0, %v0
+	vfenezh  %v0, %v0, %v0
+	vfenehs  %v0, %v0, %v0
+	vfenezhs %v0, %v0, %v0
+	vfenef   %v0, %v0, %v0
+	vfenezf  %v0, %v0, %v0
+	vfenefs  %v0, %v0, %v0
+	vfenezfs %v0, %v0, %v0
+
+#CHECK: error: {{(instruction requires: vector)?}}
+#CHECK: vfidb	%v0, %v0, 0, 0
+
+	vfidb	%v0, %v0, 0, 0
+
+#CHECK: error: {{(instruction requires: vector)?}}
+#CHECK: vistrb	%v0, %v0
+#CHECK: error: {{(instruction requires: vector)?}}
+#CHECK: vistrbs	%v0, %v0
+#CHECK: error: {{(instruction requires: vector)?}}
+#CHECK: vistrh	%v0, %v0
+#CHECK: error: {{(instruction requires: vector)?}}
+#CHECK: vistrhs	%v0, %v0
+#CHECK: error: {{(instruction requires: vector)?}}
+#CHECK: vistrf	%v0, %v0
+#CHECK: error: {{(instruction requires: vector)?}}
+#CHECK: vistrfs	%v0, %v0
+
+	vistrb	%v0, %v0
+	vistrbs	%v0, %v0
+	vistrh	%v0, %v0
+	vistrhs	%v0, %v0
+	vistrf	%v0, %v0
+	vistrfs	%v0, %v0
+
+#CHECK: error: {{(instruction requires: vector)?}}
+#CHECK: vflcdb	%v0, %v0
+
+	vflcdb	%v0, %v0
+
+#CHECK: error: {{(instruction requires: vector)?}}
+#CHECK: vflndb	%v0, %v0
+
+	vflndb	%v0, %v0
+
+#CHECK: error: {{(instruction requires: vector)?}}
+#CHECK: vflpdb	%v0, %v0
+
+	vflpdb	%v0, %v0
+
+#CHECK: error: {{(instruction requires: vector)?}}
+#CHECK: vfmadb	%v0, %v0, %v0, %v0
+
+	vfmadb	%v0, %v0, %v0, %v0
+
+#CHECK: error: {{(instruction requires: vector)?}}
+#CHECK: vfmdb	%v0, %v0, %v0
+
+	vfmdb	%v0, %v0, %v0
+
+#CHECK: error: {{(instruction requires: vector)?}}
+#CHECK: vfmsdb	%v0, %v0, %v0, %v0
+
+	vfmsdb	%v0, %v0, %v0, %v0
+
+#CHECK: error: {{(instruction requires: vector)?}}
+#CHECK: vfsdb	%v0, %v0, %v0
+
+	vfsdb	%v0, %v0, %v0
+
+#CHECK: error: {{(instruction requires: vector)?}}
+#CHECK: vfsqdb	%v0, %v0
+
+	vfsqdb	%v0, %v0
+
+#CHECK: error: {{(instruction requires: vector)?}}
+#CHECK: vftcidb	%v0, %v0, 0
+
+	vftcidb	%v0, %v0, 0
+
+#CHECK: error: {{(instruction requires: vector)?}}
+#CHECK: vgbm	%v0, 0
+
+	vgbm	%v0, 0
+
+#CHECK: error: {{(instruction requires: vector)?}}
+#CHECK: vgef	%v0, 0(%v0, %r1), 0
+#CHECK: error: {{(instruction requires: vector)?}}
+#CHECK: vgeg	%v0, 0(%v0, %r1), 0
+
+	vgef	%v0, 0(%v0, %r1), 0
+	vgeg	%v0, 0(%v0, %r1), 0
+
+#CHECK: error: {{(instruction requires: vector)?}}
+#CHECK: vgfmab	%v0, %v0, %v0, %v0
+#CHECK: error: {{(instruction requires: vector)?}}
+#CHECK: vgfmaf	%v0, %v0, %v0, %v0
+#CHECK: error: {{(instruction requires: vector)?}}
+#CHECK: vgfmag	%v0, %v0, %v0, %v0
+#CHECK: error: {{(instruction requires: vector)?}}
+#CHECK: vgfmah	%v0, %v0, %v0, %v0
+
+	vgfmab	%v0, %v0, %v0, %v0
+	vgfmaf	%v0, %v0, %v0, %v0
+	vgfmag	%v0, %v0, %v0, %v0
+	vgfmah	%v0, %v0, %v0, %v0
+
+#CHECK: error: {{(instruction requires: vector)?}}
+#CHECK: vgfmb	%v0, %v0, %v0
+#CHECK: error: {{(instruction requires: vector)?}}
+#CHECK: vgfmf	%v0, %v0, %v0
+#CHECK: error: {{(instruction requires: vector)?}}
+#CHECK: vgfmg	%v0, %v0, %v0
+#CHECK: error: {{(instruction requires: vector)?}}
+#CHECK: vgfmh	%v0, %v0, %v0
+
+	vgfmb	%v0, %v0, %v0
+	vgfmf	%v0, %v0, %v0
+	vgfmg	%v0, %v0, %v0
+	vgfmh	%v0, %v0, %v0
+
+#CHECK: error: {{(instruction requires: vector)?}}
+#CHECK: vgmb	%v0, 0, 0
+#CHECK: error: {{(instruction requires: vector)?}}
+#CHECK: vgmf	%v0, 0, 0
+#CHECK: error: {{(instruction requires: vector)?}}
+#CHECK: vgmg	%v0, 0, 0
+#CHECK: error: {{(instruction requires: vector)?}}
+#CHECK: vgmh	%v0, 0, 0
+
+	vgmb	%v0, 0, 0
+	vgmf	%v0, 0, 0
+	vgmg	%v0, 0, 0
+	vgmh	%v0, 0, 0
+
+#CHECK: error: {{(instruction requires: vector)?}}
+#CHECK: vl	%v0, 0
+
+	vl	%v0, 0
+
+#CHECK: error: {{(instruction requires: vector)?}}
+#CHECK: vlbb	%v0, 0, 0
+
+	vlbb	%v0, 0, 0
+
+#CHECK: error: {{(instruction requires: vector)?}}
+#CHECK: vlcb	%v0, %v0
+#CHECK: error: {{(instruction requires: vector)?}}
+#CHECK: vlcf	%v0, %v0
+#CHECK: error: {{(instruction requires: vector)?}}
+#CHECK: vlcg	%v0, %v0
+#CHECK: error: {{(instruction requires: vector)?}}
+#CHECK: vlch	%v0, %v0
+
+	vlcb	%v0, %v0
+	vlcf	%v0, %v0
+	vlcg	%v0, %v0
+	vlch	%v0, %v0
+
+#CHECK: error: {{(instruction requires: vector)?}}
+#CHECK: vldeb	%v0, %v0
+
+	vldeb	%v0, %v0
+
+#CHECK: error: {{(instruction requires: vector)?}}
+#CHECK: vleb	%v0, 0, 0
+#CHECK: error: {{(instruction requires: vector)?}}
+#CHECK: vlef	%v0, 0, 0
+#CHECK: error: {{(instruction requires: vector)?}}
+#CHECK: vleg	%v0, 0, 0
+#CHECK: error: {{(instruction requires: vector)?}}
+#CHECK: vleh	%v0, 0, 0
+
+	vleb	%v0, 0, 0
+	vlef	%v0, 0, 0
+	vleg	%v0, 0, 0
+	vleh	%v0, 0, 0
+
+#CHECK: error: {{(instruction requires: vector)?}}
+#CHECK: vledb	%v0, %v0, 0, 0
+
+	vledb	%v0, %v0, 0, 0
+
+#CHECK: error: {{(instruction requires: vector)?}}
+#CHECK: vleib	%v0, 0, 0
+#CHECK: error: {{(instruction requires: vector)?}}
+#CHECK: vleif	%v0, 0, 0
+#CHECK: error: {{(instruction requires: vector)?}}
+#CHECK: vleig	%v0, 0, 0
+#CHECK: error: {{(instruction requires: vector)?}}
+#CHECK: vleih	%v0, 0, 0
+
+	vleib	%v0, 0, 0
+	vleif	%v0, 0, 0
+	vleig	%v0, 0, 0
+	vleih	%v0, 0, 0
+
+#CHECK: error: {{(instruction requires: vector)?}}
+#CHECK: vlgvb	%r0, %v0, 0
+#CHECK: error: {{(instruction requires: vector)?}}
+#CHECK: vlgvf	%r0, %v0, 0
+#CHECK: error: {{(instruction requires: vector)?}}
+#CHECK: vlgvg	%r0, %v0, 0
+#CHECK: error: {{(instruction requires: vector)?}}
+#CHECK: vlgvh	%r0, %v0, 0
+
+	vlgvb	%r0, %v0, 0
+	vlgvf	%r0, %v0, 0
+	vlgvg	%r0, %v0, 0
+	vlgvh	%r0, %v0, 0
+
+#CHECK: error: {{(instruction requires: vector)?}}
+#CHECK: vll	%v0, %r0, 0
+
+	vll	%v0, %r0, 0
+
+#CHECK: error: {{(instruction requires: vector)?}}
+#CHECK: vllezb	%v0, 0
+#CHECK: error: {{(instruction requires: vector)?}}
+#CHECK: vllezf	%v0, 0
+#CHECK: error: {{(instruction requires: vector)?}}
+#CHECK: vllezg	%v0, 0
+#CHECK: error: {{(instruction requires: vector)?}}
+#CHECK: vllezh	%v0, 0
+
+	vllezb	%v0, 0
+	vllezf	%v0, 0
+	vllezg	%v0, 0
+	vllezh	%v0, 0
+
+#CHECK: error: {{(instruction requires: vector)?}}
+#CHECK: vlm	%v0, %v0, 0
+
+	vlm	%v0, %v0, 0
+
+#CHECK: error: {{(instruction requires: vector)?}}
+#CHECK: vlpb	%v0, %v0
+#CHECK: error: {{(instruction requires: vector)?}}
+#CHECK: vlpf	%v0, %v0
+#CHECK: error: {{(instruction requires: vector)?}}
+#CHECK: vlpg	%v0, %v0
+#CHECK: error: {{(instruction requires: vector)?}}
+#CHECK: vlph	%v0, %v0
+
+	vlpb	%v0, %v0
+	vlpf	%v0, %v0
+	vlpg	%v0, %v0
+	vlph	%v0, %v0
+
+#CHECK: error: {{(instruction requires: vector)?}}
+#CHECK: vlr	%v0, %v0
+
+	vlr	%v0, %v0
+
+#CHECK: error: {{(instruction requires: vector)?}}
+#CHECK: vlrepb	%v0, 0
+#CHECK: error: {{(instruction requires: vector)?}}
+#CHECK: vlrepf	%v0, 0
+#CHECK: error: {{(instruction requires: vector)?}}
+#CHECK: vlrepg	%v0, 0
+#CHECK: error: {{(instruction requires: vector)?}}
+#CHECK: vlreph	%v0, 0
+
+	vlrepb	%v0, 0
+	vlrepf	%v0, 0
+	vlrepg	%v0, 0
+	vlreph	%v0, 0
+
+#CHECK: error: {{(instruction requires: vector)?}}
+#CHECK: vlvgb	%v0, %r0, 0
+#CHECK: error: {{(instruction requires: vector)?}}
+#CHECK: vlvgf	%v0, %r0, 0
+#CHECK: error: {{(instruction requires: vector)?}}
+#CHECK: vlvgg	%v0, %r0, 0
+#CHECK: error: {{(instruction requires: vector)?}}
+#CHECK: vlvgh	%v0, %r0, 0
+
+	vlvgb	%v0, %r0, 0
+	vlvgf	%v0, %r0, 0
+	vlvgg	%v0, %r0, 0
+	vlvgh	%v0, %r0, 0
+
+#CHECK: error: {{(instruction requires: vector)?}}
+#CHECK: vlvgp	%v0, %r0, %r0
+
+	vlvgp	%v0, %r0, %r0
+
+#CHECK: error: {{(instruction requires: vector)?}}
+#CHECK: vmaeb	%v0, %v0, %v0, %v0
+#CHECK: error: {{(instruction requires: vector)?}}
+#CHECK: vmaef	%v0, %v0, %v0, %v0
+#CHECK: error: {{(instruction requires: vector)?}}
+#CHECK: vmaeh	%v0, %v0, %v0, %v0
+
+	vmaeb	%v0, %v0, %v0, %v0
+	vmaef	%v0, %v0, %v0, %v0
+	vmaeh	%v0, %v0, %v0, %v0
+
+#CHECK: error: {{(instruction requires: vector)?}}
+#CHECK: vmahb	%v0, %v0, %v0, %v0
+#CHECK: error: {{(instruction requires: vector)?}}
+#CHECK: vmahf	%v0, %v0, %v0, %v0
+#CHECK: error: {{(instruction requires: vector)?}}
+#CHECK: vmahh	%v0, %v0, %v0, %v0
+
+	vmahb	%v0, %v0, %v0, %v0
+	vmahf	%v0, %v0, %v0, %v0
+	vmahh	%v0, %v0, %v0, %v0
+
+#CHECK: error: {{(instruction requires: vector)?}}
+#CHECK: vmalb	%v0, %v0, %v0, %v0
+#CHECK: error: {{(instruction requires: vector)?}}
+#CHECK: vmalf	%v0, %v0, %v0, %v0
+#CHECK: error: {{(instruction requires: vector)?}}
+#CHECK: vmalhw	%v0, %v0, %v0, %v0
+
+	vmalb	%v0, %v0, %v0, %v0
+	vmalf	%v0, %v0, %v0, %v0
+	vmalhw	%v0, %v0, %v0, %v0
+
+#CHECK: error: {{(instruction requires: vector)?}}
+#CHECK: vmaleb	%v0, %v0, %v0, %v0
+#CHECK: error: {{(instruction requires: vector)?}}
+#CHECK: vmalef	%v0, %v0, %v0, %v0
+#CHECK: error: {{(instruction requires: vector)?}}
+#CHECK: vmaleh	%v0, %v0, %v0, %v0
+
+	vmaleb	%v0, %v0, %v0, %v0
+	vmalef	%v0, %v0, %v0, %v0
+	vmaleh	%v0, %v0, %v0, %v0
+
+#CHECK: error: {{(instruction requires: vector)?}}
+#CHECK: vmalhb	%v0, %v0, %v0, %v0
+#CHECK: error: {{(instruction requires: vector)?}}
+#CHECK: vmalhf	%v0, %v0, %v0, %v0
+#CHECK: error: {{(instruction requires: vector)?}}
+#CHECK: vmalhh	%v0, %v0, %v0, %v0
+
+	vmalhb	%v0, %v0, %v0, %v0
+	vmalhf	%v0, %v0, %v0, %v0
+	vmalhh	%v0, %v0, %v0, %v0
+
+#CHECK: error: {{(instruction requires: vector)?}}
+#CHECK: vmalob	%v0, %v0, %v0, %v0
+#CHECK: error: {{(instruction requires: vector)?}}
+#CHECK: vmalof	%v0, %v0, %v0, %v0
+#CHECK: error: {{(instruction requires: vector)?}}
+#CHECK: vmaloh	%v0, %v0, %v0, %v0
+
+	vmalob	%v0, %v0, %v0, %v0
+	vmalof	%v0, %v0, %v0, %v0
+	vmaloh	%v0, %v0, %v0, %v0
+
+#CHECK: error: {{(instruction requires: vector)?}}
+#CHECK: vmaob	%v0, %v0, %v0, %v0
+#CHECK: error: {{(instruction requires: vector)?}}
+#CHECK: vmaof	%v0, %v0, %v0, %v0
+#CHECK: error: {{(instruction requires: vector)?}}
+#CHECK: vmaoh	%v0, %v0, %v0, %v0
+
+	vmaob	%v0, %v0, %v0, %v0
+	vmaof	%v0, %v0, %v0, %v0
+	vmaoh	%v0, %v0, %v0, %v0
+
+#CHECK: error: {{(instruction requires: vector)?}}
+#CHECK: vmeb	%v0, %v0, %v0
+#CHECK: error: {{(instruction requires: vector)?}}
+#CHECK: vmef	%v0, %v0, %v0
+#CHECK: error: {{(instruction requires: vector)?}}
+#CHECK: vmeh	%v0, %v0, %v0
+
+	vmeb	%v0, %v0, %v0
+	vmef	%v0, %v0, %v0
+	vmeh	%v0, %v0, %v0
+
+#CHECK: error: {{(instruction requires: vector)?}}
+#CHECK: vmhb	%v0, %v0, %v0
+#CHECK: error: {{(instruction requires: vector)?}}
+#CHECK: vmhf	%v0, %v0, %v0
+#CHECK: error: {{(instruction requires: vector)?}}
+#CHECK: vmhh	%v0, %v0, %v0
+
+	vmhb	%v0, %v0, %v0
+	vmhf	%v0, %v0, %v0
+	vmhh	%v0, %v0, %v0
+
+#CHECK: error: {{(instruction requires: vector)?}}
+#CHECK: vmlb	%v0, %v0, %v0
+#CHECK: error: {{(instruction requires: vector)?}}
+#CHECK: vmlf	%v0, %v0, %v0
+#CHECK: error: {{(instruction requires: vector)?}}
+#CHECK: vmlh	%v0, %v0, %v0
+
+	vmlb	%v0, %v0, %v0
+	vmlf	%v0, %v0, %v0
+	vmlh	%v0, %v0, %v0
+
+#CHECK: error: {{(instruction requires: vector)?}}
+#CHECK: vmleb	%v0, %v0, %v0
+#CHECK: error: {{(instruction requires: vector)?}}
+#CHECK: vmlef	%v0, %v0, %v0
+#CHECK: error: {{(instruction requires: vector)?}}
+#CHECK: vmleh	%v0, %v0, %v0
+
+	vmleb	%v0, %v0, %v0
+	vmlef	%v0, %v0, %v0
+	vmleh	%v0, %v0, %v0
+
+#CHECK: error: {{(instruction requires: vector)?}}
+#CHECK: vmlhb	%v0, %v0, %v0
+#CHECK: error: {{(instruction requires: vector)?}}
+#CHECK: vmlhf	%v0, %v0, %v0
+#CHECK: error: {{(instruction requires: vector)?}}
+#CHECK: vmlhh	%v0, %v0, %v0
+
+	vmlhb	%v0, %v0, %v0
+	vmlhf	%v0, %v0, %v0
+	vmlhh	%v0, %v0, %v0
+
+#CHECK: error: {{(instruction requires: vector)?}}
+#CHECK: vmlob	%v0, %v0, %v0
+#CHECK: error: {{(instruction requires: vector)?}}
+#CHECK: vmlof	%v0, %v0, %v0
+#CHECK: error: {{(instruction requires: vector)?}}
+#CHECK: vmloh	%v0, %v0, %v0
+
+	vmlob	%v0, %v0, %v0
+	vmlof	%v0, %v0, %v0
+	vmloh	%v0, %v0, %v0
+
+#CHECK: error: {{(instruction requires: vector)?}}
+#CHECK: vmnb	%v0, %v0, %v0
+#CHECK: error: {{(instruction requires: vector)?}}
+#CHECK: vmnf	%v0, %v0, %v0
+#CHECK: error: {{(instruction requires: vector)?}}
+#CHECK: vmng	%v0, %v0, %v0
+#CHECK: error: {{(instruction requires: vector)?}}
+#CHECK: vmnh	%v0, %v0, %v0
+
+	vmnb	%v0, %v0, %v0
+	vmnf	%v0, %v0, %v0
+	vmng	%v0, %v0, %v0
+	vmnh	%v0, %v0, %v0
+
+#CHECK: error: {{(instruction requires: vector)?}}
+#CHECK: vmnlb	%v0, %v0, %v0
+#CHECK: error: {{(instruction requires: vector)?}}
+#CHECK: vmnlf	%v0, %v0, %v0
+#CHECK: error: {{(instruction requires: vector)?}}
+#CHECK: vmnlg	%v0, %v0, %v0
+#CHECK: error: {{(instruction requires: vector)?}}
+#CHECK: vmnlh	%v0, %v0, %v0
+
+	vmnlb	%v0, %v0, %v0
+	vmnlf	%v0, %v0, %v0
+	vmnlg	%v0, %v0, %v0
+	vmnlh	%v0, %v0, %v0
+
+#CHECK: error: {{(instruction requires: vector)?}}
+#CHECK: vmob	%v0, %v0, %v0
+#CHECK: error: {{(instruction requires: vector)?}}
+#CHECK: vmof	%v0, %v0, %v0
+#CHECK: error: {{(instruction requires: vector)?}}
+#CHECK: vmoh	%v0, %v0, %v0
+
+	vmob	%v0, %v0, %v0
+	vmof	%v0, %v0, %v0
+	vmoh	%v0, %v0, %v0
+
+#CHECK: error: {{(instruction requires: vector)?}}
+#CHECK: vmrhb	%v0, %v0, %v0
+#CHECK: error: {{(instruction requires: vector)?}}
+#CHECK: vmrhf	%v0, %v0, %v0
+#CHECK: error: {{(instruction requires: vector)?}}
+#CHECK: vmrhg	%v0, %v0, %v0
+#CHECK: error: {{(instruction requires: vector)?}}
+#CHECK: vmrhh	%v0, %v0, %v0
+
+	vmrhb	%v0, %v0, %v0
+	vmrhf	%v0, %v0, %v0
+	vmrhg	%v0, %v0, %v0
+	vmrhh	%v0, %v0, %v0
+
+#CHECK: error: {{(instruction requires: vector)?}}
+#CHECK: vmrlb	%v0, %v0, %v0
+#CHECK: error: {{(instruction requires: vector)?}}
+#CHECK: vmrlf	%v0, %v0, %v0
+#CHECK: error: {{(instruction requires: vector)?}}
+#CHECK: vmrlg	%v0, %v0, %v0
+#CHECK: error: {{(instruction requires: vector)?}}
+#CHECK: vmrlh	%v0, %v0, %v0
+
+	vmrlb	%v0, %v0, %v0
+	vmrlf	%v0, %v0, %v0
+	vmrlg	%v0, %v0, %v0
+	vmrlh	%v0, %v0, %v0
+
+#CHECK: error: {{(instruction requires: vector)?}}
+#CHECK: vmxb	%v0, %v0, %v0
+#CHECK: error: {{(instruction requires: vector)?}}
+#CHECK: vmxf	%v0, %v0, %v0
+#CHECK: error: {{(instruction requires: vector)?}}
+#CHECK: vmxg	%v0, %v0, %v0
+#CHECK: error: {{(instruction requires: vector)?}}
+#CHECK: vmxh	%v0, %v0, %v0
+
+	vmxb	%v0, %v0, %v0
+	vmxf	%v0, %v0, %v0
+	vmxg	%v0, %v0, %v0
+	vmxh	%v0, %v0, %v0
+
+#CHECK: error: {{(instruction requires: vector)?}}
+#CHECK: vmxlb	%v0, %v0, %v0
+#CHECK: error: {{(instruction requires: vector)?}}
+#CHECK: vmxlf	%v0, %v0, %v0
+#CHECK: error: {{(instruction requires: vector)?}}
+#CHECK: vmxlg	%v0, %v0, %v0
+#CHECK: error: {{(instruction requires: vector)?}}
+#CHECK: vmxlh	%v0, %v0, %v0
+
+	vmxlb	%v0, %v0, %v0
+	vmxlf	%v0, %v0, %v0
+	vmxlg	%v0, %v0, %v0
+	vmxlh	%v0, %v0, %v0
+
+#CHECK: error: {{(instruction requires: vector)?}}
+#CHECK: vn	%v0, %v0, %v0
+
+	vn	%v0, %v0, %v0
+
+#CHECK: error: {{(instruction requires: vector)?}}
+#CHECK: vnc	%v0, %v0, %v0
+
+	vnc	%v0, %v0, %v0
+
+#CHECK: error: {{(instruction requires: vector)?}}
+#CHECK: vno	%v0, %v0, %v0
+
+	vno	%v0, %v0, %v0
+
+#CHECK: error: {{(instruction requires: vector)?}}
+#CHECK: vo	%v0, %v0, %v0
+
+	vo	%v0, %v0, %v0
+
+#CHECK: error: {{(instruction requires: vector)?}}
+#CHECK: vone	%v0
+
+	vone	%v0
+
+#CHECK: error: {{(instruction requires: vector)?}}
+#CHECK: vpdi	%v0, %v0, %v0, 0
+
+	vpdi	%v0, %v0, %v0, 0
+
+#CHECK: error: {{(instruction requires: vector)?}}
+#CHECK: vperm	%v0, %v0, %v0, %v0
+
+	vperm	%v0, %v0, %v0, %v0
+
+#CHECK: error: {{(instruction requires: vector)?}}
+#CHECK: vpkf	%v0, %v0, %v0
+#CHECK: error: {{(instruction requires: vector)?}}
+#CHECK: vpkg	%v0, %v0, %v0
+#CHECK: error: {{(instruction requires: vector)?}}
+#CHECK: vpkh	%v0, %v0, %v0
+
+	vpkf	%v0, %v0, %v0
+	vpkg	%v0, %v0, %v0
+	vpkh	%v0, %v0, %v0
+
+#CHECK: error: {{(instruction requires: vector)?}}
+#CHECK: vpksf	%v0, %v0, %v0
+#CHECK: error: {{(instruction requires: vector)?}}
+#CHECK: vpksg	%v0, %v0, %v0
+#CHECK: error: {{(instruction requires: vector)?}}
+#CHECK: vpksh	%v0, %v0, %v0
+#CHECK: error: {{(instruction requires: vector)?}}
+#CHECK: vpksfs	%v0, %v0, %v0
+#CHECK: error: {{(instruction requires: vector)?}}
+#CHECK: vpksgs	%v0, %v0, %v0
+#CHECK: error: {{(instruction requires: vector)?}}
+#CHECK: vpkshs	%v0, %v0, %v0
+
+	vpksf	%v0, %v0, %v0
+	vpksg	%v0, %v0, %v0
+	vpksh	%v0, %v0, %v0
+	vpksfs	%v0, %v0, %v0
+	vpksgs	%v0, %v0, %v0
+	vpkshs	%v0, %v0, %v0
+
+#CHECK: error: {{(instruction requires: vector)?}}
+#CHECK: vpklsf	%v0, %v0, %v0
+#CHECK: error: {{(instruction requires: vector)?}}
+#CHECK: vpklsg	%v0, %v0, %v0
+#CHECK: error: {{(instruction requires: vector)?}}
+#CHECK: vpklsh	%v0, %v0, %v0
+#CHECK: error: {{(instruction requires: vector)?}}
+#CHECK: vpklsfs	%v0, %v0, %v0
+#CHECK: error: {{(instruction requires: vector)?}}
+#CHECK: vpklsgs	%v0, %v0, %v0
+#CHECK: error: {{(instruction requires: vector)?}}
+#CHECK: vpklshs	%v0, %v0, %v0
+
+	vpklsf	%v0, %v0, %v0
+	vpklsg	%v0, %v0, %v0
+	vpklsh	%v0, %v0, %v0
+	vpklsfs	%v0, %v0, %v0
+	vpklsgs	%v0, %v0, %v0
+	vpklshs	%v0, %v0, %v0
+
+#CHECK: error: {{(instruction requires: vector)?}}
+#CHECK: vpopct	%v0, %v0
+
+	vpopct	%v0, %v0
+
+#CHECK: error: {{(instruction requires: vector)?}}
+#CHECK: vrepb	%v0, %v0, 0
+#CHECK: error: {{(instruction requires: vector)?}}
+#CHECK: vrepf	%v0, %v0, 0
+#CHECK: error: {{(instruction requires: vector)?}}
+#CHECK: vrepg	%v0, %v0, 0
+#CHECK: error: {{(instruction requires: vector)?}}
+#CHECK: vreph	%v0, %v0, 0
+
+	vrepb	%v0, %v0, 0
+	vrepf	%v0, %v0, 0
+	vrepg	%v0, %v0, 0
+	vreph	%v0, %v0, 0
+
+#CHECK: error: {{(instruction requires: vector)?}}
+#CHECK: vrepib	%v0, 0
+#CHECK: error: {{(instruction requires: vector)?}}
+#CHECK: vrepif	%v0, 0
+#CHECK: error: {{(instruction requires: vector)?}}
+#CHECK: vrepig	%v0, 0
+#CHECK: error: {{(instruction requires: vector)?}}
+#CHECK: vrepih	%v0, 0
+
+	vrepib	%v0, 0
+	vrepif	%v0, 0
+	vrepig	%v0, 0
+	vrepih	%v0, 0
+
+#CHECK: error: {{(instruction requires: vector)?}}
+#CHECK: vsb	%v0, %v0, %v0
+#CHECK: error: {{(instruction requires: vector)?}}
+#CHECK: vsf	%v0, %v0, %v0
+#CHECK: error: {{(instruction requires: vector)?}}
+#CHECK: vsg	%v0, %v0, %v0
+#CHECK: error: {{(instruction requires: vector)?}}
+#CHECK: vsh	%v0, %v0, %v0
+#CHECK: error: {{(instruction requires: vector)?}}
+#CHECK: vsq	%v0, %v0, %v0
+
+	vsb	%v0, %v0, %v0
+	vsf	%v0, %v0, %v0
+	vsg	%v0, %v0, %v0
+	vsh	%v0, %v0, %v0
+	vsq	%v0, %v0, %v0
+
+#CHECK: error: {{(instruction requires: vector)?}}
+#CHECK: vsbcbiq	%v0, %v0, %v0
+
+	vsbcbiq	%v0, %v0, %v0
+
+#CHECK: error: {{(instruction requires: vector)?}}
+#CHECK: vsbiq	%v0, %v0, %v0
+
+	vsbiq	%v0, %v0, %v0
+
+#CHECK: error: {{(instruction requires: vector)?}}
+#CHECK: vscbib	%v0, %v0, %v0
+#CHECK: error: {{(instruction requires: vector)?}}
+#CHECK: vscbif	%v0, %v0, %v0
+#CHECK: error: {{(instruction requires: vector)?}}
+#CHECK: vscbig	%v0, %v0, %v0
+#CHECK: error: {{(instruction requires: vector)?}}
+#CHECK: vscbih	%v0, %v0, %v0
+#CHECK: error: {{(instruction requires: vector)?}}
+#CHECK: vscbiq	%v0, %v0, %v0
+
+	vscbib	%v0, %v0, %v0
+	vscbif	%v0, %v0, %v0
+	vscbig	%v0, %v0, %v0
+	vscbih	%v0, %v0, %v0
+	vscbiq	%v0, %v0, %v0
+
+#CHECK: error: {{(instruction requires: vector)?}}
+#CHECK: vscef	%v0, 0(%v0, %r1), 0
+#CHECK: error: {{(instruction requires: vector)?}}
+#CHECK: vsceg	%v0, 0(%v0, %r1), 0
+
+	vscef	%v0, 0(%v0, %r1), 0
+	vsceg	%v0, 0(%v0, %r1), 0
+
+#CHECK: error: {{(instruction requires: vector)?}}
+#CHECK: vsegb	%v0, %v0
+#CHECK: error: {{(instruction requires: vector)?}}
+#CHECK: vsegf	%v0, %v0
+#CHECK: error: {{(instruction requires: vector)?}}
+#CHECK: vsegh	%v0, %v0
+
+	vsegb	%v0, %v0
+	vsegf	%v0, %v0
+	vsegh	%v0, %v0
+
+#CHECK: error: {{(instruction requires: vector)?}}
+#CHECK: vsel	%v0, %v0, %v0, %v0
+
+	vsel	%v0, %v0, %v0, %v0
+
+#CHECK: error: {{(instruction requires: vector)?}}
+#CHECK: vsl	%v0, %v0, %v0
+
+	vsl	%v0, %v0, %v0
+
+#CHECK: error: {{(instruction requires: vector)?}}
+#CHECK: vslb	%v0, %v0, %v0
+
+	vslb	%v0, %v0, %v0
+
+#CHECK: error: {{(instruction requires: vector)?}}
+#CHECK: vsldb	%v0, %v0, %v0, 0
+
+	vsldb	%v0, %v0, %v0, 0
+
+#CHECK: error: {{(instruction requires: vector)?}}
+#CHECK: vsra	%v0, %v0, %v0
+
+	vsra	%v0, %v0, %v0
+
+#CHECK: error: {{(instruction requires: vector)?}}
+#CHECK: vsrab	%v0, %v0, %v0
+
+	vsrab	%v0, %v0, %v0
+
+#CHECK: error: {{(instruction requires: vector)?}}
+#CHECK: vsrl	%v0, %v0, %v0
+
+	vsrl	%v0, %v0, %v0
+
+#CHECK: error: {{(instruction requires: vector)?}}
+#CHECK: vsrlb	%v0, %v0, %v0
+
+	vsrlb	%v0, %v0, %v0
+
+#CHECK: error: {{(instruction requires: vector)?}}
+#CHECK: vst	%v0, 0
+
+	vst	%v0, 0
+
+#CHECK: error: {{(instruction requires: vector)?}}
+#CHECK: vstl	%v0, %r0, 0
+
+	vstl	%v0, %r0, 0
+
+#CHECK: error: {{(instruction requires: vector)?}}
+#CHECK: vstm	%v0, %v0, 0
+
+	vstm	%v0, %v0, 0
+
+#CHECK: error: {{(instruction requires: vector)?}}
+#CHECK: vstrcb   %v0, %v0, %v0, %v0
+#CHECK: error: {{(instruction requires: vector)?}}
+#CHECK: vstrczb  %v0, %v0, %v0, %v0
+#CHECK: error: {{(instruction requires: vector)?}}
+#CHECK: vstrcbs  %v0, %v0, %v0, %v0
+#CHECK: error: {{(instruction requires: vector)?}}
+#CHECK: vstrczbs %v0, %v0, %v0, %v0
+#CHECK: error: {{(instruction requires: vector)?}}
+#CHECK: vstrch   %v0, %v0, %v0, %v0
+#CHECK: error: {{(instruction requires: vector)?}}
+#CHECK: vstrczh  %v0, %v0, %v0, %v0
+#CHECK: error: {{(instruction requires: vector)?}}
+#CHECK: vstrchs  %v0, %v0, %v0, %v0
+#CHECK: error: {{(instruction requires: vector)?}}
+#CHECK: vstrczhs %v0, %v0, %v0, %v0
+#CHECK: error: {{(instruction requires: vector)?}}
+#CHECK: vstrcf   %v0, %v0, %v0, %v0
+#CHECK: error: {{(instruction requires: vector)?}}
+#CHECK: vstrczf  %v0, %v0, %v0, %v0
+#CHECK: error: {{(instruction requires: vector)?}}
+#CHECK: vstrcfs  %v0, %v0, %v0, %v0
+#CHECK: error: {{(instruction requires: vector)?}}
+#CHECK: vstrczfs %v0, %v0, %v0, %v0
+
+        vstrcb   %v0, %v0, %v0, %v0
+        vstrczb  %v0, %v0, %v0, %v0
+        vstrcbs  %v0, %v0, %v0, %v0
+        vstrczbs %v0, %v0, %v0, %v0
+        vstrch   %v0, %v0, %v0, %v0
+        vstrczh  %v0, %v0, %v0, %v0
+        vstrchs  %v0, %v0, %v0, %v0
+        vstrczhs %v0, %v0, %v0, %v0
+        vstrcf   %v0, %v0, %v0, %v0
+        vstrczf  %v0, %v0, %v0, %v0
+        vstrcfs  %v0, %v0, %v0, %v0
+        vstrczfs %v0, %v0, %v0, %v0
+
+#CHECK: error: {{(instruction requires: vector)?}}
+#CHECK: vsumgh	%v0, %v0, %v0
+#CHECK: error: {{(instruction requires: vector)?}}
+#CHECK: vsumgf	%v0, %v0, %v0
+
+	vsumgh	%v0, %v0, %v0
+	vsumgf	%v0, %v0, %v0
+
+#CHECK: error: {{(instruction requires: vector)?}}
+#CHECK: vsumqf	%v0, %v0, %v0
+#CHECK: error: {{(instruction requires: vector)?}}
+#CHECK: vsumqg	%v0, %v0, %v0
+
+	vsumqf	%v0, %v0, %v0
+	vsumqg	%v0, %v0, %v0
+
+#CHECK: error: {{(instruction requires: vector)?}}
+#CHECK: vsumb	%v0, %v0, %v0
+#CHECK: error: {{(instruction requires: vector)?}}
+#CHECK: vsumh	%v0, %v0, %v0
+
+	vsumb	%v0, %v0, %v0
+	vsumh	%v0, %v0, %v0
+
+#CHECK: error: {{(instruction requires: vector)?}}
+#CHECK: vtm	%v0, %v0
+
+	vtm	%v0, %v0
+
+#CHECK: error: {{(instruction requires: vector)?}}
+#CHECK: vuphb	%v0, %v0
+#CHECK: error: {{(instruction requires: vector)?}}
+#CHECK: vuphf	%v0, %v0
+#CHECK: error: {{(instruction requires: vector)?}}
+#CHECK: vuphh	%v0, %v0
+
+	vuphb	%v0, %v0
+	vuphf	%v0, %v0
+	vuphh	%v0, %v0
+
+#CHECK: error: {{(instruction requires: vector)?}}
+#CHECK: vuplhb	%v0, %v0
+#CHECK: error: {{(instruction requires: vector)?}}
+#CHECK: vuplhf	%v0, %v0
+#CHECK: error: {{(instruction requires: vector)?}}
+#CHECK: vuplhh	%v0, %v0
+
+	vuplhb	%v0, %v0
+	vuplhf	%v0, %v0
+	vuplhh	%v0, %v0
+
+#CHECK: error: {{(instruction requires: vector)?}}
+#CHECK: vuplb	%v0, %v0
+#CHECK: error: {{(instruction requires: vector)?}}
+#CHECK: vuplf	%v0, %v0
+#CHECK: error: {{(instruction requires: vector)?}}
+#CHECK: vuplhw	%v0, %v0
+
+	vuplb	%v0, %v0
+	vuplf	%v0, %v0
+	vuplhw	%v0, %v0
+
+#CHECK: error: {{(instruction requires: vector)?}}
+#CHECK: vupllb	%v0, %v0
+#CHECK: error: {{(instruction requires: vector)?}}
+#CHECK: vupllf	%v0, %v0
+#CHECK: error: {{(instruction requires: vector)?}}
+#CHECK: vupllh	%v0, %v0
+
+	vupllb	%v0, %v0
+	vupllf	%v0, %v0
+	vupllh	%v0, %v0
+
+#CHECK: error: {{(instruction requires: vector)?}}
+#CHECK: vx	%v0, %v0, %v0
+
+	vx	%v0, %v0, %v0
+
+#CHECK: error: {{(instruction requires: vector)?}}
+#CHECK: vzero	%v0
+
+	vzero	%v0
+
+#CHECK: error: {{(instruction requires: vector)?}}
+#CHECK: wcdgb	%v0, %v0, 0, 0
+
+	wcdgb	%v0, %v0, 0, 0
+
+#CHECK: error: {{(instruction requires: vector)?}}
+#CHECK: wcdlgb	%v0, %v0, 0, 0
+
+	wcdlgb	%v0, %v0, 0, 0
+
+#CHECK: error: {{(instruction requires: vector)?}}
+#CHECK: wcgdb	%v0, %v0, 0, 0
+
+	wcgdb	%v0, %v0, 0, 0
+
+#CHECK: error: {{(instruction requires: vector)?}}
+#CHECK: wclgdb	%v0, %v0, 0, 0
+
+	wclgdb	%v0, %v0, 0, 0
+
+#CHECK: error: {{(instruction requires: vector)?}}
+#CHECK: wfadb	%v0, %v0, %v0
+
+	wfadb	%v0, %v0, %v0
+
+#CHECK: error: {{(instruction requires: vector)?}}
+#CHECK: wfcdb	%v0, %v0
+
+	wfcdb	%v0, %v0
+
+#CHECK: error: {{(instruction requires: vector)?}}
+#CHECK: wfcedb	%v0, %v0, %v0
+#CHECK: wfcedbs	%v0, %v0, %v0
+
+	wfcedb	%v0, %v0, %v0
+	wfcedbs	%v0, %v0, %v0
+
+#CHECK: error: {{(instruction requires: vector)?}}
+#CHECK: wfchdb	%v0, %v0, %v0
+#CHECK: wfchdbs	%v0, %v0, %v0
+
+	wfchdb	%v0, %v0, %v0
+	wfchdbs	%v0, %v0, %v0
+
+#CHECK: error: {{(instruction requires: vector)?}}
+#CHECK: wfchedb	%v0, %v0, %v0
+#CHECK: wfchedbs %v0, %v0, %v0
+
+	wfchedb	%v0, %v0, %v0
+	wfchedbs %v0, %v0, %v0
+
+#CHECK: error: {{(instruction requires: vector)?}}
+#CHECK: wfddb	%v0, %v0, %v0
+
+	wfddb	%v0, %v0, %v0
+
+#CHECK: error: {{(instruction requires: vector)?}}
+#CHECK: wfidb	%v0, %v0, 0, 0
+
+	wfidb	%v0, %v0, 0, 0
+
+#CHECK: error: {{(instruction requires: vector)?}}
+#CHECK: wfkdb	%v0, %v0
+
+	wfkdb	%v0, %v0
+
+#CHECK: error: {{(instruction requires: vector)?}}
+#CHECK: wflcdb	%v0, %v0
+
+	wflcdb	%v0, %v0
+
+#CHECK: error: {{(instruction requires: vector)?}}
+#CHECK: wflndb	%v0, %v0
+
+	wflndb	%v0, %v0
+
+#CHECK: error: {{(instruction requires: vector)?}}
+#CHECK: wflpdb	%v0, %v0
+
+	wflpdb	%v0, %v0
+
+#CHECK: error: {{(instruction requires: vector)?}}
+#CHECK: wfmadb	%v0, %v0, %v0, %v0
+
+	wfmadb	%v0, %v0, %v0, %v0
+
+#CHECK: error: {{(instruction requires: vector)?}}
+#CHECK: wfmdb	%v0, %v0, %v0
+
+	wfmdb	%v0, %v0, %v0
+
+#CHECK: error: {{(instruction requires: vector)?}}
+#CHECK: wfmsdb	%v0, %v0, %v0, %v0
+
+	wfmsdb	%v0, %v0, %v0, %v0
+
+#CHECK: error: {{(instruction requires: vector)?}}
+#CHECK: wfsdb	%v0, %v0, %v0
+
+	wfsdb	%v0, %v0, %v0
+
+#CHECK: error: {{(instruction requires: vector)?}}
+#CHECK: wfsqdb	%v0, %v0
+
+	wfsqdb	%v0, %v0
+
+#CHECK: error: {{(instruction requires: vector)?}}
+#CHECK: wftcidb	%v0, %v0, 0
+
+	wftcidb	%v0, %v0, 0
+
+#CHECK: error: {{(instruction requires: vector)?}}
+#CHECK: wldeb	%v0, %v0
+
+	wldeb	%v0, %v0
+
+#CHECK: error: {{(instruction requires: vector)?}}
+#CHECK: wledb	%v0, %v0, 0, 0
+
+	wledb	%v0, %v0, 0, 0
diff --git a/test/MC/SystemZ/insn-good-z13.s b/test/MC/SystemZ/insn-good-z13.s
new file mode 100644
index 00000000000..621b946d19d
--- /dev/null
+++ b/test/MC/SystemZ/insn-good-z13.s
@@ -0,0 +1,5039 @@
+# For z13 and above.
+# RUN: llvm-mc -triple s390x-linux-gnu -mcpu=z13 -show-encoding %s \
+# RUN:   | FileCheck %s
+
+#CHECK: lcbb    %r0, 0, 0               # encoding: [0xe7,0x00,0x00,0x00,0x00,0x27]
+#CHECK: lcbb    %r0, 0, 15              # encoding: [0xe7,0x00,0x00,0x00,0xf0,0x27]
+#CHECK: lcbb    %r0, 4095, 0            # encoding: [0xe7,0x00,0x0f,0xff,0x00,0x27]
+#CHECK: lcbb    %r0, 0(%r15), 0         # encoding: [0xe7,0x00,0xf0,0x00,0x00,0x27]
+#CHECK: lcbb    %r0, 0(%r15,%r1), 0     # encoding: [0xe7,0x0f,0x10,0x00,0x00,0x27]
+#CHECK: lcbb    %r15, 0, 0              # encoding: [0xe7,0xf0,0x00,0x00,0x00,0x27]
+#CHECK: lcbb    %r2, 1383(%r3,%r4), 8   # encoding: [0xe7,0x23,0x45,0x67,0x80,0x27]
+
+	lcbb	%r0, 0, 0
+	lcbb	%r0, 0, 15
+	lcbb	%r0, 4095, 0
+	lcbb	%r0, 0(%r15), 0
+	lcbb	%r0, 0(%r15,%r1), 0
+	lcbb	%r15, 0, 0
+	lcbb	%r2, 1383(%r3,%r4), 8
+
+#CHECK: vab     %v0, %v0, %v0           # encoding: [0xe7,0x00,0x00,0x00,0x00,0xf3]
+#CHECK: vab     %v0, %v0, %v31          # encoding: [0xe7,0x00,0xf0,0x00,0x02,0xf3]
+#CHECK: vab     %v0, %v31, %v0          # encoding: [0xe7,0x0f,0x00,0x00,0x04,0xf3]
+#CHECK: vab     %v31, %v0, %v0          # encoding: [0xe7,0xf0,0x00,0x00,0x08,0xf3]
+#CHECK: vab     %v18, %v3, %v20         # encoding: [0xe7,0x23,0x40,0x00,0x0a,0xf3]
+
+	vab	%v0, %v0, %v0
+	vab	%v0, %v0, %v31
+	vab	%v0, %v31, %v0
+	vab	%v31, %v0, %v0
+	vab	%v18, %v3, %v20
+
+#CHECK: vaccb   %v0, %v0, %v0           # encoding: [0xe7,0x00,0x00,0x00,0x00,0xf1]
+#CHECK: vaccb   %v0, %v0, %v31          # encoding: [0xe7,0x00,0xf0,0x00,0x02,0xf1]
+#CHECK: vaccb   %v0, %v31, %v0          # encoding: [0xe7,0x0f,0x00,0x00,0x04,0xf1]
+#CHECK: vaccb   %v31, %v0, %v0          # encoding: [0xe7,0xf0,0x00,0x00,0x08,0xf1]
+#CHECK: vaccb   %v18, %v3, %v20         # encoding: [0xe7,0x23,0x40,0x00,0x0a,0xf1]
+
+	vaccb	%v0, %v0, %v0
+	vaccb	%v0, %v0, %v31
+	vaccb	%v0, %v31, %v0
+	vaccb	%v31, %v0, %v0
+	vaccb	%v18, %v3, %v20
+
+#CHECK: vacccq  %v0, %v0, %v0, %v0      # encoding: [0xe7,0x00,0x04,0x00,0x00,0xb9]
+#CHECK: vacccq  %v0, %v0, %v0, %v31     # encoding: [0xe7,0x00,0x04,0x00,0xf1,0xb9]
+#CHECK: vacccq  %v0, %v0, %v31, %v0     # encoding: [0xe7,0x00,0xf4,0x00,0x02,0xb9]
+#CHECK: vacccq  %v0, %v31, %v0, %v0     # encoding: [0xe7,0x0f,0x04,0x00,0x04,0xb9]
+#CHECK: vacccq  %v31, %v0, %v0, %v0     # encoding: [0xe7,0xf0,0x04,0x00,0x08,0xb9]
+#CHECK: vacccq  %v13, %v17, %v21, %v25  # encoding: [0xe7,0xd1,0x54,0x00,0x97,0xb9]
+
+	vacccq	%v0, %v0, %v0, %v0
+	vacccq	%v0, %v0, %v0, %v31
+	vacccq	%v0, %v0, %v31, %v0
+	vacccq	%v0, %v31, %v0, %v0
+	vacccq	%v31, %v0, %v0, %v0
+	vacccq	%v13, %v17, %v21, %v25
+
+#CHECK: vaccf   %v0, %v0, %v0           # encoding: [0xe7,0x00,0x00,0x00,0x20,0xf1]
+#CHECK: vaccf   %v0, %v0, %v31          # encoding: [0xe7,0x00,0xf0,0x00,0x22,0xf1]
+#CHECK: vaccf   %v0, %v31, %v0          # encoding: [0xe7,0x0f,0x00,0x00,0x24,0xf1]
+#CHECK: vaccf   %v31, %v0, %v0          # encoding: [0xe7,0xf0,0x00,0x00,0x28,0xf1]
+#CHECK: vaccf   %v18, %v3, %v20         # encoding: [0xe7,0x23,0x40,0x00,0x2a,0xf1]
+
+	vaccf	%v0, %v0, %v0
+	vaccf	%v0, %v0, %v31
+	vaccf	%v0, %v31, %v0
+	vaccf	%v31, %v0, %v0
+	vaccf	%v18, %v3, %v20
+
+#CHECK: vaccg   %v0, %v0, %v0           # encoding: [0xe7,0x00,0x00,0x00,0x30,0xf1]
+#CHECK: vaccg   %v0, %v0, %v31          # encoding: [0xe7,0x00,0xf0,0x00,0x32,0xf1]
+#CHECK: vaccg   %v0, %v31, %v0          # encoding: [0xe7,0x0f,0x00,0x00,0x34,0xf1]
+#CHECK: vaccg   %v31, %v0, %v0          # encoding: [0xe7,0xf0,0x00,0x00,0x38,0xf1]
+#CHECK: vaccg   %v18, %v3, %v20         # encoding: [0xe7,0x23,0x40,0x00,0x3a,0xf1]
+
+	vaccg	%v0, %v0, %v0
+	vaccg	%v0, %v0, %v31
+	vaccg	%v0, %v31, %v0
+	vaccg	%v31, %v0, %v0
+	vaccg	%v18, %v3, %v20
+
+#CHECK: vacch   %v0, %v0, %v0           # encoding: [0xe7,0x00,0x00,0x00,0x10,0xf1]
+#CHECK: vacch   %v0, %v0, %v31          # encoding: [0xe7,0x00,0xf0,0x00,0x12,0xf1]
+#CHECK: vacch   %v0, %v31, %v0          # encoding: [0xe7,0x0f,0x00,0x00,0x14,0xf1]
+#CHECK: vacch   %v31, %v0, %v0          # encoding: [0xe7,0xf0,0x00,0x00,0x18,0xf1]
+#CHECK: vacch   %v18, %v3, %v20         # encoding: [0xe7,0x23,0x40,0x00,0x1a,0xf1]
+
+	vacch	%v0, %v0, %v0
+	vacch	%v0, %v0, %v31
+	vacch	%v0, %v31, %v0
+	vacch	%v31, %v0, %v0
+	vacch	%v18, %v3, %v20
+
+#CHECK: vaccq   %v0, %v0, %v0           # encoding: [0xe7,0x00,0x00,0x00,0x40,0xf1]
+#CHECK: vaccq   %v0, %v0, %v31          # encoding: [0xe7,0x00,0xf0,0x00,0x42,0xf1]
+#CHECK: vaccq   %v0, %v31, %v0          # encoding: [0xe7,0x0f,0x00,0x00,0x44,0xf1]
+#CHECK: vaccq   %v31, %v0, %v0          # encoding: [0xe7,0xf0,0x00,0x00,0x48,0xf1]
+#CHECK: vaccq   %v18, %v3, %v20         # encoding: [0xe7,0x23,0x40,0x00,0x4a,0xf1]
+
+	vaccq	%v0, %v0, %v0
+	vaccq	%v0, %v0, %v31
+	vaccq	%v0, %v31, %v0
+	vaccq	%v31, %v0, %v0
+	vaccq	%v18, %v3, %v20
+
+#CHECK: vacq    %v0, %v0, %v0, %v0      # encoding: [0xe7,0x00,0x04,0x00,0x00,0xbb]
+#CHECK: vacq    %v0, %v0, %v0, %v31     # encoding: [0xe7,0x00,0x04,0x00,0xf1,0xbb]
+#CHECK: vacq    %v0, %v0, %v31, %v0     # encoding: [0xe7,0x00,0xf4,0x00,0x02,0xbb]
+#CHECK: vacq    %v0, %v31, %v0, %v0     # encoding: [0xe7,0x0f,0x04,0x00,0x04,0xbb]
+#CHECK: vacq    %v31, %v0, %v0, %v0     # encoding: [0xe7,0xf0,0x04,0x00,0x08,0xbb]
+#CHECK: vacq    %v13, %v17, %v21, %v25  # encoding: [0xe7,0xd1,0x54,0x00,0x97,0xbb]
+
+	vacq	%v0, %v0, %v0, %v0
+	vacq	%v0, %v0, %v0, %v31
+	vacq	%v0, %v0, %v31, %v0
+	vacq	%v0, %v31, %v0, %v0
+	vacq	%v31, %v0, %v0, %v0
+	vacq	%v13, %v17, %v21, %v25
+
+#CHECK: vaf     %v0, %v0, %v0           # encoding: [0xe7,0x00,0x00,0x00,0x20,0xf3]
+#CHECK: vaf     %v0, %v0, %v31          # encoding: [0xe7,0x00,0xf0,0x00,0x22,0xf3]
+#CHECK: vaf     %v0, %v31, %v0          # encoding: [0xe7,0x0f,0x00,0x00,0x24,0xf3]
+#CHECK: vaf     %v31, %v0, %v0          # encoding: [0xe7,0xf0,0x00,0x00,0x28,0xf3]
+#CHECK: vaf     %v18, %v3, %v20         # encoding: [0xe7,0x23,0x40,0x00,0x2a,0xf3]
+
+	vaf	%v0, %v0, %v0
+	vaf	%v0, %v0, %v31
+	vaf	%v0, %v31, %v0
+	vaf	%v31, %v0, %v0
+	vaf	%v18, %v3, %v20
+
+#CHECK: vag     %v0, %v0, %v0           # encoding: [0xe7,0x00,0x00,0x00,0x30,0xf3]
+#CHECK: vag     %v0, %v0, %v31          # encoding: [0xe7,0x00,0xf0,0x00,0x32,0xf3]
+#CHECK: vag     %v0, %v31, %v0          # encoding: [0xe7,0x0f,0x00,0x00,0x34,0xf3]
+#CHECK: vag     %v31, %v0, %v0          # encoding: [0xe7,0xf0,0x00,0x00,0x38,0xf3]
+#CHECK: vag     %v18, %v3, %v20         # encoding: [0xe7,0x23,0x40,0x00,0x3a,0xf3]
+
+	vag	%v0, %v0, %v0
+	vag	%v0, %v0, %v31
+	vag	%v0, %v31, %v0
+	vag	%v31, %v0, %v0
+	vag	%v18, %v3, %v20
+
+#CHECK: vah     %v0, %v0, %v0           # encoding: [0xe7,0x00,0x00,0x00,0x10,0xf3]
+#CHECK: vah     %v0, %v0, %v31          # encoding: [0xe7,0x00,0xf0,0x00,0x12,0xf3]
+#CHECK: vah     %v0, %v31, %v0          # encoding: [0xe7,0x0f,0x00,0x00,0x14,0xf3]
+#CHECK: vah     %v31, %v0, %v0          # encoding: [0xe7,0xf0,0x00,0x00,0x18,0xf3]
+#CHECK: vah     %v18, %v3, %v20         # encoding: [0xe7,0x23,0x40,0x00,0x1a,0xf3]
+
+	vah	%v0, %v0, %v0
+	vah	%v0, %v0, %v31
+	vah	%v0, %v31, %v0
+	vah	%v31, %v0, %v0
+	vah	%v18, %v3, %v20
+
+#CHECK: vaq     %v0, %v0, %v0           # encoding: [0xe7,0x00,0x00,0x00,0x40,0xf3]
+#CHECK: vaq     %v0, %v0, %v31          # encoding: [0xe7,0x00,0xf0,0x00,0x42,0xf3]
+#CHECK: vaq     %v0, %v31, %v0          # encoding: [0xe7,0x0f,0x00,0x00,0x44,0xf3]
+#CHECK: vaq     %v31, %v0, %v0          # encoding: [0xe7,0xf0,0x00,0x00,0x48,0xf3]
+#CHECK: vaq     %v18, %v3, %v20         # encoding: [0xe7,0x23,0x40,0x00,0x4a,0xf3]
+
+	vaq	%v0, %v0, %v0
+	vaq	%v0, %v0, %v31
+	vaq	%v0, %v31, %v0
+	vaq	%v31, %v0, %v0
+	vaq	%v18, %v3, %v20
+
+#CHECK: vavgb   %v0, %v0, %v0           # encoding: [0xe7,0x00,0x00,0x00,0x00,0xf2]
+#CHECK: vavgb   %v0, %v0, %v31          # encoding: [0xe7,0x00,0xf0,0x00,0x02,0xf2]
+#CHECK: vavgb   %v0, %v31, %v0          # encoding: [0xe7,0x0f,0x00,0x00,0x04,0xf2]
+#CHECK: vavgb   %v31, %v0, %v0          # encoding: [0xe7,0xf0,0x00,0x00,0x08,0xf2]
+#CHECK: vavgb   %v18, %v3, %v20         # encoding: [0xe7,0x23,0x40,0x00,0x0a,0xf2]
+
+	vavgb	%v0, %v0, %v0
+	vavgb	%v0, %v0, %v31
+	vavgb	%v0, %v31, %v0
+	vavgb	%v31, %v0, %v0
+	vavgb	%v18, %v3, %v20
+
+#CHECK: vavgf   %v0, %v0, %v0           # encoding: [0xe7,0x00,0x00,0x00,0x20,0xf2]
+#CHECK: vavgf   %v0, %v0, %v31          # encoding: [0xe7,0x00,0xf0,0x00,0x22,0xf2]
+#CHECK: vavgf   %v0, %v31, %v0          # encoding: [0xe7,0x0f,0x00,0x00,0x24,0xf2]
+#CHECK: vavgf   %v31, %v0, %v0          # encoding: [0xe7,0xf0,0x00,0x00,0x28,0xf2]
+#CHECK: vavgf   %v18, %v3, %v20         # encoding: [0xe7,0x23,0x40,0x00,0x2a,0xf2]
+
+	vavgf	%v0, %v0, %v0
+	vavgf	%v0, %v0, %v31
+	vavgf	%v0, %v31, %v0
+	vavgf	%v31, %v0, %v0
+	vavgf	%v18, %v3, %v20
+
+#CHECK: vavgg   %v0, %v0, %v0           # encoding: [0xe7,0x00,0x00,0x00,0x30,0xf2]
+#CHECK: vavgg   %v0, %v0, %v31          # encoding: [0xe7,0x00,0xf0,0x00,0x32,0xf2]
+#CHECK: vavgg   %v0, %v31, %v0          # encoding: [0xe7,0x0f,0x00,0x00,0x34,0xf2]
+#CHECK: vavgg   %v31, %v0, %v0          # encoding: [0xe7,0xf0,0x00,0x00,0x38,0xf2]
+#CHECK: vavgg   %v18, %v3, %v20         # encoding: [0xe7,0x23,0x40,0x00,0x3a,0xf2]
+
+	vavgg	%v0, %v0, %v0
+	vavgg	%v0, %v0, %v31
+	vavgg	%v0, %v31, %v0
+	vavgg	%v31, %v0, %v0
+	vavgg	%v18, %v3, %v20
+
+#CHECK: vavgh   %v0, %v0, %v0           # encoding: [0xe7,0x00,0x00,0x00,0x10,0xf2]
+#CHECK: vavgh   %v0, %v0, %v31          # encoding: [0xe7,0x00,0xf0,0x00,0x12,0xf2]
+#CHECK: vavgh   %v0, %v31, %v0          # encoding: [0xe7,0x0f,0x00,0x00,0x14,0xf2]
+#CHECK: vavgh   %v31, %v0, %v0          # encoding: [0xe7,0xf0,0x00,0x00,0x18,0xf2]
+#CHECK: vavgh   %v18, %v3, %v20         # encoding: [0xe7,0x23,0x40,0x00,0x1a,0xf2]
+
+	vavgh	%v0, %v0, %v0
+	vavgh	%v0, %v0, %v31
+	vavgh	%v0, %v31, %v0
+	vavgh	%v31, %v0, %v0
+	vavgh	%v18, %v3, %v20
+
+#CHECK: vavglb  %v0, %v0, %v0           # encoding: [0xe7,0x00,0x00,0x00,0x00,0xf0]
+#CHECK: vavglb  %v0, %v0, %v31          # encoding: [0xe7,0x00,0xf0,0x00,0x02,0xf0]
+#CHECK: vavglb  %v0, %v31, %v0          # encoding: [0xe7,0x0f,0x00,0x00,0x04,0xf0]
+#CHECK: vavglb  %v31, %v0, %v0          # encoding: [0xe7,0xf0,0x00,0x00,0x08,0xf0]
+#CHECK: vavglb  %v18, %v3, %v20         # encoding: [0xe7,0x23,0x40,0x00,0x0a,0xf0]
+
+	vavglb	%v0, %v0, %v0
+	vavglb	%v0, %v0, %v31
+	vavglb	%v0, %v31, %v0
+	vavglb	%v31, %v0, %v0
+	vavglb	%v18, %v3, %v20
+
+#CHECK: vavglf  %v0, %v0, %v0           # encoding: [0xe7,0x00,0x00,0x00,0x20,0xf0]
+#CHECK: vavglf  %v0, %v0, %v31          # encoding: [0xe7,0x00,0xf0,0x00,0x22,0xf0]
+#CHECK: vavglf  %v0, %v31, %v0          # encoding: [0xe7,0x0f,0x00,0x00,0x24,0xf0]
+#CHECK: vavglf  %v31, %v0, %v0          # encoding: [0xe7,0xf0,0x00,0x00,0x28,0xf0]
+#CHECK: vavglf  %v18, %v3, %v20         # encoding: [0xe7,0x23,0x40,0x00,0x2a,0xf0]
+
+	vavglf	%v0, %v0, %v0
+	vavglf	%v0, %v0, %v31
+	vavglf	%v0, %v31, %v0
+	vavglf	%v31, %v0, %v0
+	vavglf	%v18, %v3, %v20
+
+#CHECK: vavglg  %v0, %v0, %v0           # encoding: [0xe7,0x00,0x00,0x00,0x30,0xf0]
+#CHECK: vavglg  %v0, %v0, %v31          # encoding: [0xe7,0x00,0xf0,0x00,0x32,0xf0]
+#CHECK: vavglg  %v0, %v31, %v0          # encoding: [0xe7,0x0f,0x00,0x00,0x34,0xf0]
+#CHECK: vavglg  %v31, %v0, %v0          # encoding: [0xe7,0xf0,0x00,0x00,0x38,0xf0]
+#CHECK: vavglg  %v18, %v3, %v20         # encoding: [0xe7,0x23,0x40,0x00,0x3a,0xf0]
+
+	vavglg	%v0, %v0, %v0
+	vavglg	%v0, %v0, %v31
+	vavglg	%v0, %v31, %v0
+	vavglg	%v31, %v0, %v0
+	vavglg	%v18, %v3, %v20
+
+#CHECK: vavglh  %v0, %v0, %v0           # encoding: [0xe7,0x00,0x00,0x00,0x10,0xf0]
+#CHECK: vavglh  %v0, %v0, %v31          # encoding: [0xe7,0x00,0xf0,0x00,0x12,0xf0]
+#CHECK: vavglh  %v0, %v31, %v0          # encoding: [0xe7,0x0f,0x00,0x00,0x14,0xf0]
+#CHECK: vavglh  %v31, %v0, %v0          # encoding: [0xe7,0xf0,0x00,0x00,0x18,0xf0]
+#CHECK: vavglh  %v18, %v3, %v20         # encoding: [0xe7,0x23,0x40,0x00,0x1a,0xf0]
+
+	vavglh	%v0, %v0, %v0
+	vavglh	%v0, %v0, %v31
+	vavglh	%v0, %v31, %v0
+	vavglh	%v31, %v0, %v0
+	vavglh	%v18, %v3, %v20
+
+#CHECK: vcdgb   %v0, %v0, 0, 0          # encoding: [0xe7,0x00,0x00,0x00,0x30,0xc3]
+#CHECK: vcdgb   %v0, %v0, 0, 15         # encoding: [0xe7,0x00,0x00,0xf0,0x30,0xc3]
+#CHECK: vcdgb   %v0, %v0, 4, 0          # encoding: [0xe7,0x00,0x00,0x04,0x30,0xc3]
+#CHECK: vcdgb   %v0, %v0, 12, 0         # encoding: [0xe7,0x00,0x00,0x0c,0x30,0xc3]
+#CHECK: vcdgb   %v0, %v31, 0, 0         # encoding: [0xe7,0x0f,0x00,0x00,0x34,0xc3]
+#CHECK: vcdgb   %v31, %v0, 0, 0         # encoding: [0xe7,0xf0,0x00,0x00,0x38,0xc3]
+#CHECK: vcdgb   %v14, %v17, 4, 10       # encoding: [0xe7,0xe1,0x00,0xa4,0x34,0xc3]
+
+	vcdgb	%v0, %v0, 0, 0
+	vcdgb	%v0, %v0, 0, 15
+	vcdgb	%v0, %v0, 4, 0
+	vcdgb	%v0, %v0, 12, 0
+	vcdgb	%v0, %v31, 0, 0
+	vcdgb	%v31, %v0, 0, 0
+	vcdgb	%v14, %v17, 4, 10
+
+#CHECK: vcdlgb  %v0, %v0, 0, 0          # encoding: [0xe7,0x00,0x00,0x00,0x30,0xc1]
+#CHECK: vcdlgb  %v0, %v0, 0, 15         # encoding: [0xe7,0x00,0x00,0xf0,0x30,0xc1]
+#CHECK: vcdlgb  %v0, %v0, 4, 0          # encoding: [0xe7,0x00,0x00,0x04,0x30,0xc1]
+#CHECK: vcdlgb  %v0, %v0, 12, 0         # encoding: [0xe7,0x00,0x00,0x0c,0x30,0xc1]
+#CHECK: vcdlgb  %v0, %v31, 0, 0         # encoding: [0xe7,0x0f,0x00,0x00,0x34,0xc1]
+#CHECK: vcdlgb  %v31, %v0, 0, 0         # encoding: [0xe7,0xf0,0x00,0x00,0x38,0xc1]
+#CHECK: vcdlgb  %v14, %v17, 4, 10       # encoding: [0xe7,0xe1,0x00,0xa4,0x34,0xc1]
+
+	vcdlgb	%v0, %v0, 0, 0
+	vcdlgb	%v0, %v0, 0, 15
+	vcdlgb	%v0, %v0, 4, 0
+	vcdlgb	%v0, %v0, 12, 0
+	vcdlgb	%v0, %v31, 0, 0
+	vcdlgb	%v31, %v0, 0, 0
+	vcdlgb	%v14, %v17, 4, 10
+
+#CHECK: vcksm   %v0, %v0, %v0           # encoding: [0xe7,0x00,0x00,0x00,0x00,0x66]
+#CHECK: vcksm   %v0, %v0, %v31          # encoding: [0xe7,0x00,0xf0,0x00,0x02,0x66]
+#CHECK: vcksm   %v0, %v31, %v0          # encoding: [0xe7,0x0f,0x00,0x00,0x04,0x66]
+#CHECK: vcksm   %v31, %v0, %v0          # encoding: [0xe7,0xf0,0x00,0x00,0x08,0x66]
+#CHECK: vcksm   %v18, %v3, %v20         # encoding: [0xe7,0x23,0x40,0x00,0x0a,0x66]
+
+	vcksm	%v0, %v0, %v0
+	vcksm	%v0, %v0, %v31
+	vcksm	%v0, %v31, %v0
+	vcksm	%v31, %v0, %v0
+	vcksm	%v18, %v3, %v20
+
+#CHECK: vceqb   %v0, %v0, %v0           # encoding: [0xe7,0x00,0x00,0x00,0x00,0xf8]
+#CHECK: vceqb   %v0, %v0, %v31          # encoding: [0xe7,0x00,0xf0,0x00,0x02,0xf8]
+#CHECK: vceqb   %v0, %v31, %v0          # encoding: [0xe7,0x0f,0x00,0x00,0x04,0xf8]
+#CHECK: vceqb   %v31, %v0, %v0          # encoding: [0xe7,0xf0,0x00,0x00,0x08,0xf8]
+#CHECK: vceqb   %v18, %v3, %v20         # encoding: [0xe7,0x23,0x40,0x00,0x0a,0xf8]
+#CHECK: vceqbs  %v5, %v22, %v7          # encoding: [0xe7,0x56,0x70,0x10,0x04,0xf8]
+
+	vceqb	%v0, %v0, %v0
+	vceqb	%v0, %v0, %v31
+	vceqb	%v0, %v31, %v0
+	vceqb	%v31, %v0, %v0
+	vceqb	%v18, %v3, %v20
+	vceqbs	%v5, %v22, %v7
+
+#CHECK: vceqf   %v0, %v0, %v0           # encoding: [0xe7,0x00,0x00,0x00,0x20,0xf8]
+#CHECK: vceqf   %v0, %v0, %v31          # encoding: [0xe7,0x00,0xf0,0x00,0x22,0xf8]
+#CHECK: vceqf   %v0, %v31, %v0          # encoding: [0xe7,0x0f,0x00,0x00,0x24,0xf8]
+#CHECK: vceqf   %v31, %v0, %v0          # encoding: [0xe7,0xf0,0x00,0x00,0x28,0xf8]
+#CHECK: vceqf   %v18, %v3, %v20         # encoding: [0xe7,0x23,0x40,0x00,0x2a,0xf8]
+#CHECK: vceqfs  %v5, %v22, %v7          # encoding: [0xe7,0x56,0x70,0x10,0x24,0xf8]
+
+	vceqf	%v0, %v0, %v0
+	vceqf	%v0, %v0, %v31
+	vceqf	%v0, %v31, %v0
+	vceqf	%v31, %v0, %v0
+	vceqf	%v18, %v3, %v20
+	vceqfs	%v5, %v22, %v7
+
+#CHECK: vceqg   %v0, %v0, %v0           # encoding: [0xe7,0x00,0x00,0x00,0x30,0xf8]
+#CHECK: vceqg   %v0, %v0, %v31          # encoding: [0xe7,0x00,0xf0,0x00,0x32,0xf8]
+#CHECK: vceqg   %v0, %v31, %v0          # encoding: [0xe7,0x0f,0x00,0x00,0x34,0xf8]
+#CHECK: vceqg   %v31, %v0, %v0          # encoding: [0xe7,0xf0,0x00,0x00,0x38,0xf8]
+#CHECK: vceqg   %v18, %v3, %v20         # encoding: [0xe7,0x23,0x40,0x00,0x3a,0xf8]
+#CHECK: vceqgs  %v5, %v22, %v7          # encoding: [0xe7,0x56,0x70,0x10,0x34,0xf8]
+
+	vceqg	%v0, %v0, %v0
+	vceqg	%v0, %v0, %v31
+	vceqg	%v0, %v31, %v0
+	vceqg	%v31, %v0, %v0
+	vceqg	%v18, %v3, %v20
+	vceqgs	%v5, %v22, %v7
+
+#CHECK: vceqh   %v0, %v0, %v0           # encoding: [0xe7,0x00,0x00,0x00,0x10,0xf8]
+#CHECK: vceqh   %v0, %v0, %v31          # encoding: [0xe7,0x00,0xf0,0x00,0x12,0xf8]
+#CHECK: vceqh   %v0, %v31, %v0          # encoding: [0xe7,0x0f,0x00,0x00,0x14,0xf8]
+#CHECK: vceqh   %v31, %v0, %v0          # encoding: [0xe7,0xf0,0x00,0x00,0x18,0xf8]
+#CHECK: vceqh   %v18, %v3, %v20         # encoding: [0xe7,0x23,0x40,0x00,0x1a,0xf8]
+#CHECK: vceqhs  %v5, %v22, %v7          # encoding: [0xe7,0x56,0x70,0x10,0x14,0xf8]
+
+	vceqh	%v0, %v0, %v0
+	vceqh	%v0, %v0, %v31
+	vceqh	%v0, %v31, %v0
+	vceqh	%v31, %v0, %v0
+	vceqh	%v18, %v3, %v20
+	vceqhs	%v5, %v22, %v7
+
+#CHECK: vcgdb   %v0, %v0, 0, 0          # encoding: [0xe7,0x00,0x00,0x00,0x30,0xc2]
+#CHECK: vcgdb   %v0, %v0, 0, 15         # encoding: [0xe7,0x00,0x00,0xf0,0x30,0xc2]
+#CHECK: vcgdb   %v0, %v0, 4, 0          # encoding: [0xe7,0x00,0x00,0x04,0x30,0xc2]
+#CHECK: vcgdb   %v0, %v0, 12, 0         # encoding: [0xe7,0x00,0x00,0x0c,0x30,0xc2]
+#CHECK: vcgdb   %v0, %v31, 0, 0         # encoding: [0xe7,0x0f,0x00,0x00,0x34,0xc2]
+#CHECK: vcgdb   %v31, %v0, 0, 0         # encoding: [0xe7,0xf0,0x00,0x00,0x38,0xc2]
+#CHECK: vcgdb   %v14, %v17, 4, 10       # encoding: [0xe7,0xe1,0x00,0xa4,0x34,0xc2]
+
+	vcgdb	%v0, %v0, 0, 0
+	vcgdb	%v0, %v0, 0, 15
+	vcgdb	%v0, %v0, 4, 0
+	vcgdb	%v0, %v0, 12, 0
+	vcgdb	%v0, %v31, 0, 0
+	vcgdb	%v31, %v0, 0, 0
+	vcgdb	%v14, %v17, 4, 10
+
+#CHECK: vchb    %v0, %v0, %v0           # encoding: [0xe7,0x00,0x00,0x00,0x00,0xfb]
+#CHECK: vchb    %v0, %v0, %v31          # encoding: [0xe7,0x00,0xf0,0x00,0x02,0xfb]
+#CHECK: vchb    %v0, %v31, %v0          # encoding: [0xe7,0x0f,0x00,0x00,0x04,0xfb]
+#CHECK: vchb    %v31, %v0, %v0          # encoding: [0xe7,0xf0,0x00,0x00,0x08,0xfb]
+#CHECK: vchb    %v18, %v3, %v20         # encoding: [0xe7,0x23,0x40,0x00,0x0a,0xfb]
+#CHECK: vchbs   %v5, %v22, %v7          # encoding: [0xe7,0x56,0x70,0x10,0x04,0xfb]
+
+	vchb	%v0, %v0, %v0
+	vchb	%v0, %v0, %v31
+	vchb	%v0, %v31, %v0
+	vchb	%v31, %v0, %v0
+	vchb	%v18, %v3, %v20
+	vchbs	%v5, %v22, %v7
+
+#CHECK: vchf    %v0, %v0, %v0           # encoding: [0xe7,0x00,0x00,0x00,0x20,0xfb]
+#CHECK: vchf    %v0, %v0, %v31          # encoding: [0xe7,0x00,0xf0,0x00,0x22,0xfb]
+#CHECK: vchf    %v0, %v31, %v0          # encoding: [0xe7,0x0f,0x00,0x00,0x24,0xfb]
+#CHECK: vchf    %v31, %v0, %v0          # encoding: [0xe7,0xf0,0x00,0x00,0x28,0xfb]
+#CHECK: vchf    %v18, %v3, %v20         # encoding: [0xe7,0x23,0x40,0x00,0x2a,0xfb]
+#CHECK: vchfs   %v5, %v22, %v7          # encoding: [0xe7,0x56,0x70,0x10,0x24,0xfb]
+
+	vchf	%v0, %v0, %v0
+	vchf	%v0, %v0, %v31
+	vchf	%v0, %v31, %v0
+	vchf	%v31, %v0, %v0
+	vchf	%v18, %v3, %v20
+	vchfs	%v5, %v22, %v7
+
+#CHECK: vchg    %v0, %v0, %v0           # encoding: [0xe7,0x00,0x00,0x00,0x30,0xfb]
+#CHECK: vchg    %v0, %v0, %v31          # encoding: [0xe7,0x00,0xf0,0x00,0x32,0xfb]
+#CHECK: vchg    %v0, %v31, %v0          # encoding: [0xe7,0x0f,0x00,0x00,0x34,0xfb]
+#CHECK: vchg    %v31, %v0, %v0          # encoding: [0xe7,0xf0,0x00,0x00,0x38,0xfb]
+#CHECK: vchg    %v18, %v3, %v20         # encoding: [0xe7,0x23,0x40,0x00,0x3a,0xfb]
+#CHECK: vchgs   %v5, %v22, %v7          # encoding: [0xe7,0x56,0x70,0x10,0x34,0xfb]
+
+	vchg	%v0, %v0, %v0
+	vchg	%v0, %v0, %v31
+	vchg	%v0, %v31, %v0
+	vchg	%v31, %v0, %v0
+	vchg	%v18, %v3, %v20
+	vchgs	%v5, %v22, %v7
+
+#CHECK: vchh    %v0, %v0, %v0           # encoding: [0xe7,0x00,0x00,0x00,0x10,0xfb]
+#CHECK: vchh    %v0, %v0, %v31          # encoding: [0xe7,0x00,0xf0,0x00,0x12,0xfb]
+#CHECK: vchh    %v0, %v31, %v0          # encoding: [0xe7,0x0f,0x00,0x00,0x14,0xfb]
+#CHECK: vchh    %v31, %v0, %v0          # encoding: [0xe7,0xf0,0x00,0x00,0x18,0xfb]
+#CHECK: vchh    %v18, %v3, %v20         # encoding: [0xe7,0x23,0x40,0x00,0x1a,0xfb]
+#CHECK: vchhs   %v5, %v22, %v7          # encoding: [0xe7,0x56,0x70,0x10,0x14,0xfb]
+
+	vchh	%v0, %v0, %v0
+	vchh	%v0, %v0, %v31
+	vchh	%v0, %v31, %v0
+	vchh	%v31, %v0, %v0
+	vchh	%v18, %v3, %v20
+	vchhs	%v5, %v22, %v7
+
+#CHECK: vchlb   %v0, %v0, %v0           # encoding: [0xe7,0x00,0x00,0x00,0x00,0xf9]
+#CHECK: vchlb   %v0, %v0, %v31          # encoding: [0xe7,0x00,0xf0,0x00,0x02,0xf9]
+#CHECK: vchlb   %v0, %v31, %v0          # encoding: [0xe7,0x0f,0x00,0x00,0x04,0xf9]
+#CHECK: vchlb   %v31, %v0, %v0          # encoding: [0xe7,0xf0,0x00,0x00,0x08,0xf9]
+#CHECK: vchlb   %v18, %v3, %v20         # encoding: [0xe7,0x23,0x40,0x00,0x0a,0xf9]
+#CHECK: vchlbs  %v5, %v22, %v7          # encoding: [0xe7,0x56,0x70,0x10,0x04,0xf9]
+
+	vchlb	%v0, %v0, %v0
+	vchlb	%v0, %v0, %v31
+	vchlb	%v0, %v31, %v0
+	vchlb	%v31, %v0, %v0
+	vchlb	%v18, %v3, %v20
+	vchlbs	%v5, %v22, %v7
+
+#CHECK: vchlf   %v0, %v0, %v0           # encoding: [0xe7,0x00,0x00,0x00,0x20,0xf9]
+#CHECK: vchlf   %v0, %v0, %v31          # encoding: [0xe7,0x00,0xf0,0x00,0x22,0xf9]
+#CHECK: vchlf   %v0, %v31, %v0          # encoding: [0xe7,0x0f,0x00,0x00,0x24,0xf9]
+#CHECK: vchlf   %v31, %v0, %v0          # encoding: [0xe7,0xf0,0x00,0x00,0x28,0xf9]
+#CHECK: vchlf   %v18, %v3, %v20         # encoding: [0xe7,0x23,0x40,0x00,0x2a,0xf9]
+#CHECK: vchlfs  %v5, %v22, %v7          # encoding: [0xe7,0x56,0x70,0x10,0x24,0xf9]
+
+	vchlf	%v0, %v0, %v0
+	vchlf	%v0, %v0, %v31
+	vchlf	%v0, %v31, %v0
+	vchlf	%v31, %v0, %v0
+	vchlf	%v18, %v3, %v20
+	vchlfs	%v5, %v22, %v7
+
+#CHECK: vchlg   %v0, %v0, %v0           # encoding: [0xe7,0x00,0x00,0x00,0x30,0xf9]
+#CHECK: vchlg   %v0, %v0, %v31          # encoding: [0xe7,0x00,0xf0,0x00,0x32,0xf9]
+#CHECK: vchlg   %v0, %v31, %v0          # encoding: [0xe7,0x0f,0x00,0x00,0x34,0xf9]
+#CHECK: vchlg   %v31, %v0, %v0          # encoding: [0xe7,0xf0,0x00,0x00,0x38,0xf9]
+#CHECK: vchlg   %v18, %v3, %v20         # encoding: [0xe7,0x23,0x40,0x00,0x3a,0xf9]
+#CHECK: vchlgs  %v5, %v22, %v7          # encoding: [0xe7,0x56,0x70,0x10,0x34,0xf9]
+
+	vchlg	%v0, %v0, %v0
+	vchlg	%v0, %v0, %v31
+	vchlg	%v0, %v31, %v0
+	vchlg	%v31, %v0, %v0
+	vchlg	%v18, %v3, %v20
+	vchlgs	%v5, %v22, %v7
+
+#CHECK: vchlh   %v0, %v0, %v0           # encoding: [0xe7,0x00,0x00,0x00,0x10,0xf9]
+#CHECK: vchlh   %v0, %v0, %v31          # encoding: [0xe7,0x00,0xf0,0x00,0x12,0xf9]
+#CHECK: vchlh   %v0, %v31, %v0          # encoding: [0xe7,0x0f,0x00,0x00,0x14,0xf9]
+#CHECK: vchlh   %v31, %v0, %v0          # encoding: [0xe7,0xf0,0x00,0x00,0x18,0xf9]
+#CHECK: vchlh   %v18, %v3, %v20         # encoding: [0xe7,0x23,0x40,0x00,0x1a,0xf9]
+#CHECK: vchlhs  %v5, %v22, %v7          # encoding: [0xe7,0x56,0x70,0x10,0x14,0xf9]
+
+	vchlh	%v0, %v0, %v0
+	vchlh	%v0, %v0, %v31
+	vchlh	%v0, %v31, %v0
+	vchlh	%v31, %v0, %v0
+	vchlh	%v18, %v3, %v20
+	vchlhs	%v5, %v22, %v7
+
+#CHECK: vclgdb  %v0, %v0, 0, 0          # encoding: [0xe7,0x00,0x00,0x00,0x30,0xc0]
+#CHECK: vclgdb  %v0, %v0, 0, 15         # encoding: [0xe7,0x00,0x00,0xf0,0x30,0xc0]
+#CHECK: vclgdb  %v0, %v0, 4, 0          # encoding: [0xe7,0x00,0x00,0x04,0x30,0xc0]
+#CHECK: vclgdb  %v0, %v0, 12, 0         # encoding: [0xe7,0x00,0x00,0x0c,0x30,0xc0]
+#CHECK: vclgdb  %v0, %v31, 0, 0         # encoding: [0xe7,0x0f,0x00,0x00,0x34,0xc0]
+#CHECK: vclgdb  %v31, %v0, 0, 0         # encoding: [0xe7,0xf0,0x00,0x00,0x38,0xc0]
+#CHECK: vclgdb  %v14, %v17, 4, 10       # encoding: [0xe7,0xe1,0x00,0xa4,0x34,0xc0]
+
+	vclgdb	%v0, %v0, 0, 0
+	vclgdb	%v0, %v0, 0, 15
+	vclgdb	%v0, %v0, 4, 0
+	vclgdb	%v0, %v0, 12, 0
+	vclgdb	%v0, %v31, 0, 0
+	vclgdb	%v31, %v0, 0, 0
+	vclgdb	%v14, %v17, 4, 10
+
+#CHECK: vclzb   %v0, %v0                # encoding: [0xe7,0x00,0x00,0x00,0x00,0x53]
+#CHECK: vclzb   %v0, %v15               # encoding: [0xe7,0x0f,0x00,0x00,0x00,0x53]
+#CHECK: vclzb   %v0, %v31               # encoding: [0xe7,0x0f,0x00,0x00,0x04,0x53]
+#CHECK: vclzb   %v15, %v0               # encoding: [0xe7,0xf0,0x00,0x00,0x00,0x53]
+#CHECK: vclzb   %v31, %v0               # encoding: [0xe7,0xf0,0x00,0x00,0x08,0x53]
+#CHECK: vclzb   %v14, %v17              # encoding: [0xe7,0xe1,0x00,0x00,0x04,0x53]
+
+	vclzb	%v0, %v0
+	vclzb	%v0, %v15
+	vclzb	%v0, %v31
+	vclzb	%v15, %v0
+	vclzb	%v31, %v0
+	vclzb	%v14, %v17
+
+#CHECK: vclzf   %v0, %v0                # encoding: [0xe7,0x00,0x00,0x00,0x20,0x53]
+#CHECK: vclzf   %v0, %v15               # encoding: [0xe7,0x0f,0x00,0x00,0x20,0x53]
+#CHECK: vclzf   %v0, %v31               # encoding: [0xe7,0x0f,0x00,0x00,0x24,0x53]
+#CHECK: vclzf   %v15, %v0               # encoding: [0xe7,0xf0,0x00,0x00,0x20,0x53]
+#CHECK: vclzf   %v31, %v0               # encoding: [0xe7,0xf0,0x00,0x00,0x28,0x53]
+#CHECK: vclzf   %v14, %v17              # encoding: [0xe7,0xe1,0x00,0x00,0x24,0x53]
+
+	vclzf	%v0, %v0
+	vclzf	%v0, %v15
+	vclzf	%v0, %v31
+	vclzf	%v15, %v0
+	vclzf	%v31, %v0
+	vclzf	%v14, %v17
+
+#CHECK: vclzg   %v0, %v0                # encoding: [0xe7,0x00,0x00,0x00,0x30,0x53]
+#CHECK: vclzg   %v0, %v15               # encoding: [0xe7,0x0f,0x00,0x00,0x30,0x53]
+#CHECK: vclzg   %v0, %v31               # encoding: [0xe7,0x0f,0x00,0x00,0x34,0x53]
+#CHECK: vclzg   %v15, %v0               # encoding: [0xe7,0xf0,0x00,0x00,0x30,0x53]
+#CHECK: vclzg   %v31, %v0               # encoding: [0xe7,0xf0,0x00,0x00,0x38,0x53]
+#CHECK: vclzg   %v14, %v17              # encoding: [0xe7,0xe1,0x00,0x00,0x34,0x53]
+
+	vclzg	%v0, %v0
+	vclzg	%v0, %v15
+	vclzg	%v0, %v31
+	vclzg	%v15, %v0
+	vclzg	%v31, %v0
+	vclzg	%v14, %v17
+
+#CHECK: vclzh   %v0, %v0                # encoding: [0xe7,0x00,0x00,0x00,0x10,0x53]
+#CHECK: vclzh   %v0, %v15               # encoding: [0xe7,0x0f,0x00,0x00,0x10,0x53]
+#CHECK: vclzh   %v0, %v31               # encoding: [0xe7,0x0f,0x00,0x00,0x14,0x53]
+#CHECK: vclzh   %v15, %v0               # encoding: [0xe7,0xf0,0x00,0x00,0x10,0x53]
+#CHECK: vclzh   %v31, %v0               # encoding: [0xe7,0xf0,0x00,0x00,0x18,0x53]
+#CHECK: vclzh   %v14, %v17              # encoding: [0xe7,0xe1,0x00,0x00,0x14,0x53]
+
+	vclzh	%v0, %v0
+	vclzh	%v0, %v15
+	vclzh	%v0, %v31
+	vclzh	%v15, %v0
+	vclzh	%v31, %v0
+	vclzh	%v14, %v17
+
+#CHECK: vctzb   %v0, %v0                # encoding: [0xe7,0x00,0x00,0x00,0x00,0x52]
+#CHECK: vctzb   %v0, %v15               # encoding: [0xe7,0x0f,0x00,0x00,0x00,0x52]
+#CHECK: vctzb   %v0, %v31               # encoding: [0xe7,0x0f,0x00,0x00,0x04,0x52]
+#CHECK: vctzb   %v15, %v0               # encoding: [0xe7,0xf0,0x00,0x00,0x00,0x52]
+#CHECK: vctzb   %v31, %v0               # encoding: [0xe7,0xf0,0x00,0x00,0x08,0x52]
+#CHECK: vctzb   %v14, %v17              # encoding: [0xe7,0xe1,0x00,0x00,0x04,0x52]
+
+	vctzb	%v0, %v0
+	vctzb	%v0, %v15
+	vctzb	%v0, %v31
+	vctzb	%v15, %v0
+	vctzb	%v31, %v0
+	vctzb	%v14, %v17
+
+#CHECK: vctzf   %v0, %v0                # encoding: [0xe7,0x00,0x00,0x00,0x20,0x52]
+#CHECK: vctzf   %v0, %v15               # encoding: [0xe7,0x0f,0x00,0x00,0x20,0x52]
+#CHECK: vctzf   %v0, %v31               # encoding: [0xe7,0x0f,0x00,0x00,0x24,0x52]
+#CHECK: vctzf   %v15, %v0               # encoding: [0xe7,0xf0,0x00,0x00,0x20,0x52]
+#CHECK: vctzf   %v31, %v0               # encoding: [0xe7,0xf0,0x00,0x00,0x28,0x52]
+#CHECK: vctzf   %v14, %v17              # encoding: [0xe7,0xe1,0x00,0x00,0x24,0x52]
+
+	vctzf	%v0, %v0
+	vctzf	%v0, %v15
+	vctzf	%v0, %v31
+	vctzf	%v15, %v0
+	vctzf	%v31, %v0
+	vctzf	%v14, %v17
+
+#CHECK: vctzg   %v0, %v0                # encoding: [0xe7,0x00,0x00,0x00,0x30,0x52]
+#CHECK: vctzg   %v0, %v15               # encoding: [0xe7,0x0f,0x00,0x00,0x30,0x52]
+#CHECK: vctzg   %v0, %v31               # encoding: [0xe7,0x0f,0x00,0x00,0x34,0x52]
+#CHECK: vctzg   %v15, %v0               # encoding: [0xe7,0xf0,0x00,0x00,0x30,0x52]
+#CHECK: vctzg   %v31, %v0               # encoding: [0xe7,0xf0,0x00,0x00,0x38,0x52]
+#CHECK: vctzg   %v14, %v17              # encoding: [0xe7,0xe1,0x00,0x00,0x34,0x52]
+
+	vctzg	%v0, %v0
+	vctzg	%v0, %v15
+	vctzg	%v0, %v31
+	vctzg	%v15, %v0
+	vctzg	%v31, %v0
+	vctzg	%v14, %v17
+
+#CHECK: vctzh   %v0, %v0                # encoding: [0xe7,0x00,0x00,0x00,0x10,0x52]
+#CHECK: vctzh   %v0, %v15               # encoding: [0xe7,0x0f,0x00,0x00,0x10,0x52]
+#CHECK: vctzh   %v0, %v31               # encoding: [0xe7,0x0f,0x00,0x00,0x14,0x52]
+#CHECK: vctzh   %v15, %v0               # encoding: [0xe7,0xf0,0x00,0x00,0x10,0x52]
+#CHECK: vctzh   %v31, %v0               # encoding: [0xe7,0xf0,0x00,0x00,0x18,0x52]
+#CHECK: vctzh   %v14, %v17              # encoding: [0xe7,0xe1,0x00,0x00,0x14,0x52]
+
+	vctzh	%v0, %v0
+	vctzh	%v0, %v15
+	vctzh	%v0, %v31
+	vctzh	%v15, %v0
+	vctzh	%v31, %v0
+	vctzh	%v14, %v17
+
+#CHECK: vecb    %v0, %v0                # encoding: [0xe7,0x00,0x00,0x00,0x00,0xdb]
+#CHECK: vecb    %v0, %v15               # encoding: [0xe7,0x0f,0x00,0x00,0x00,0xdb]
+#CHECK: vecb    %v0, %v31               # encoding: [0xe7,0x0f,0x00,0x00,0x04,0xdb]
+#CHECK: vecb    %v15, %v0               # encoding: [0xe7,0xf0,0x00,0x00,0x00,0xdb]
+#CHECK: vecb    %v31, %v0               # encoding: [0xe7,0xf0,0x00,0x00,0x08,0xdb]
+#CHECK: vecb    %v14, %v17              # encoding: [0xe7,0xe1,0x00,0x00,0x04,0xdb]
+
+	vecb	%v0, %v0
+	vecb	%v0, %v15
+	vecb	%v0, %v31
+	vecb	%v15, %v0
+	vecb	%v31, %v0
+	vecb	%v14, %v17
+
+#CHECK: vecf    %v0, %v0                # encoding: [0xe7,0x00,0x00,0x00,0x20,0xdb]
+#CHECK: vecf    %v0, %v15               # encoding: [0xe7,0x0f,0x00,0x00,0x20,0xdb]
+#CHECK: vecf    %v0, %v31               # encoding: [0xe7,0x0f,0x00,0x00,0x24,0xdb]
+#CHECK: vecf    %v15, %v0               # encoding: [0xe7,0xf0,0x00,0x00,0x20,0xdb]
+#CHECK: vecf    %v31, %v0               # encoding: [0xe7,0xf0,0x00,0x00,0x28,0xdb]
+#CHECK: vecf    %v14, %v17              # encoding: [0xe7,0xe1,0x00,0x00,0x24,0xdb]
+
+	vecf	%v0, %v0
+	vecf	%v0, %v15
+	vecf	%v0, %v31
+	vecf	%v15, %v0
+	vecf	%v31, %v0
+	vecf	%v14, %v17
+
+#CHECK: vecg    %v0, %v0                # encoding: [0xe7,0x00,0x00,0x00,0x30,0xdb]
+#CHECK: vecg    %v0, %v15               # encoding: [0xe7,0x0f,0x00,0x00,0x30,0xdb]
+#CHECK: vecg    %v0, %v31               # encoding: [0xe7,0x0f,0x00,0x00,0x34,0xdb]
+#CHECK: vecg    %v15, %v0               # encoding: [0xe7,0xf0,0x00,0x00,0x30,0xdb]
+#CHECK: vecg    %v31, %v0               # encoding: [0xe7,0xf0,0x00,0x00,0x38,0xdb]
+#CHECK: vecg    %v14, %v17              # encoding: [0xe7,0xe1,0x00,0x00,0x34,0xdb]
+
+	vecg	%v0, %v0
+	vecg	%v0, %v15
+	vecg	%v0, %v31
+	vecg	%v15, %v0
+	vecg	%v31, %v0
+	vecg	%v14, %v17
+
+#CHECK: vech    %v0, %v0                # encoding: [0xe7,0x00,0x00,0x00,0x10,0xdb]
+#CHECK: vech    %v0, %v15               # encoding: [0xe7,0x0f,0x00,0x00,0x10,0xdb]
+#CHECK: vech    %v0, %v31               # encoding: [0xe7,0x0f,0x00,0x00,0x14,0xdb]
+#CHECK: vech    %v15, %v0               # encoding: [0xe7,0xf0,0x00,0x00,0x10,0xdb]
+#CHECK: vech    %v31, %v0               # encoding: [0xe7,0xf0,0x00,0x00,0x18,0xdb]
+#CHECK: vech    %v14, %v17              # encoding: [0xe7,0xe1,0x00,0x00,0x14,0xdb]
+
+	vech	%v0, %v0
+	vech	%v0, %v15
+	vech	%v0, %v31
+	vech	%v15, %v0
+	vech	%v31, %v0
+	vech	%v14, %v17
+
+#CHECK: veclb   %v0, %v0                # encoding: [0xe7,0x00,0x00,0x00,0x00,0xd9]
+#CHECK: veclb   %v0, %v15               # encoding: [0xe7,0x0f,0x00,0x00,0x00,0xd9]
+#CHECK: veclb   %v0, %v31               # encoding: [0xe7,0x0f,0x00,0x00,0x04,0xd9]
+#CHECK: veclb   %v15, %v0               # encoding: [0xe7,0xf0,0x00,0x00,0x00,0xd9]
+#CHECK: veclb   %v31, %v0               # encoding: [0xe7,0xf0,0x00,0x00,0x08,0xd9]
+#CHECK: veclb   %v14, %v17              # encoding: [0xe7,0xe1,0x00,0x00,0x04,0xd9]
+
+	veclb	%v0, %v0
+	veclb	%v0, %v15
+	veclb	%v0, %v31
+	veclb	%v15, %v0
+	veclb	%v31, %v0
+	veclb	%v14, %v17
+
+#CHECK: veclf   %v0, %v0                # encoding: [0xe7,0x00,0x00,0x00,0x20,0xd9]
+#CHECK: veclf   %v0, %v15               # encoding: [0xe7,0x0f,0x00,0x00,0x20,0xd9]
+#CHECK: veclf   %v0, %v31               # encoding: [0xe7,0x0f,0x00,0x00,0x24,0xd9]
+#CHECK: veclf   %v15, %v0               # encoding: [0xe7,0xf0,0x00,0x00,0x20,0xd9]
+#CHECK: veclf   %v31, %v0               # encoding: [0xe7,0xf0,0x00,0x00,0x28,0xd9]
+#CHECK: veclf   %v14, %v17              # encoding: [0xe7,0xe1,0x00,0x00,0x24,0xd9]
+
+	veclf	%v0, %v0
+	veclf	%v0, %v15
+	veclf	%v0, %v31
+	veclf	%v15, %v0
+	veclf	%v31, %v0
+	veclf	%v14, %v17
+
+#CHECK: veclg   %v0, %v0                # encoding: [0xe7,0x00,0x00,0x00,0x30,0xd9]
+#CHECK: veclg   %v0, %v15               # encoding: [0xe7,0x0f,0x00,0x00,0x30,0xd9]
+#CHECK: veclg   %v0, %v31               # encoding: [0xe7,0x0f,0x00,0x00,0x34,0xd9]
+#CHECK: veclg   %v15, %v0               # encoding: [0xe7,0xf0,0x00,0x00,0x30,0xd9]
+#CHECK: veclg   %v31, %v0               # encoding: [0xe7,0xf0,0x00,0x00,0x38,0xd9]
+#CHECK: veclg   %v14, %v17              # encoding: [0xe7,0xe1,0x00,0x00,0x34,0xd9]
+
+	veclg	%v0, %v0
+	veclg	%v0, %v15
+	veclg	%v0, %v31
+	veclg	%v15, %v0
+	veclg	%v31, %v0
+	veclg	%v14, %v17
+
+#CHECK: veclh   %v0, %v0                # encoding: [0xe7,0x00,0x00,0x00,0x10,0xd9]
+#CHECK: veclh   %v0, %v15               # encoding: [0xe7,0x0f,0x00,0x00,0x10,0xd9]
+#CHECK: veclh   %v0, %v31               # encoding: [0xe7,0x0f,0x00,0x00,0x14,0xd9]
+#CHECK: veclh   %v15, %v0               # encoding: [0xe7,0xf0,0x00,0x00,0x10,0xd9]
+#CHECK: veclh   %v31, %v0               # encoding: [0xe7,0xf0,0x00,0x00,0x18,0xd9]
+#CHECK: veclh   %v14, %v17              # encoding: [0xe7,0xe1,0x00,0x00,0x14,0xd9]
+
+	veclh	%v0, %v0
+	veclh	%v0, %v15
+	veclh	%v0, %v31
+	veclh	%v15, %v0
+	veclh	%v31, %v0
+	veclh	%v14, %v17
+
+#CHECK: verimb  %v0, %v0, %v0, 0        # encoding: [0xe7,0x00,0x00,0x00,0x00,0x72]
+#CHECK: verimb  %v0, %v0, %v0, 255      # encoding: [0xe7,0x00,0x00,0xff,0x00,0x72]
+#CHECK: verimb  %v0, %v0, %v31, 0       # encoding: [0xe7,0x00,0xf0,0x00,0x02,0x72]
+#CHECK: verimb  %v0, %v31, %v0, 0       # encoding: [0xe7,0x0f,0x00,0x00,0x04,0x72]
+#CHECK: verimb  %v31, %v0, %v0, 0       # encoding: [0xe7,0xf0,0x00,0x00,0x08,0x72]
+#CHECK: verimb  %v13, %v17, %v21, 121   # encoding: [0xe7,0xd1,0x50,0x79,0x06,0x72]
+
+	verimb	%v0, %v0, %v0, 0
+	verimb	%v0, %v0, %v0, 255
+	verimb	%v0, %v0, %v31, 0
+	verimb	%v0, %v31, %v0, 0
+	verimb	%v31, %v0, %v0, 0
+	verimb 	%v13, %v17, %v21, 0x79
+
+#CHECK: verimf  %v0, %v0, %v0, 0        # encoding: [0xe7,0x00,0x00,0x00,0x20,0x72]
+#CHECK: verimf  %v0, %v0, %v0, 255      # encoding: [0xe7,0x00,0x00,0xff,0x20,0x72]
+#CHECK: verimf  %v0, %v0, %v31, 0       # encoding: [0xe7,0x00,0xf0,0x00,0x22,0x72]
+#CHECK: verimf  %v0, %v31, %v0, 0       # encoding: [0xe7,0x0f,0x00,0x00,0x24,0x72]
+#CHECK: verimf  %v31, %v0, %v0, 0       # encoding: [0xe7,0xf0,0x00,0x00,0x28,0x72]
+#CHECK: verimf  %v13, %v17, %v21, 121   # encoding: [0xe7,0xd1,0x50,0x79,0x26,0x72]
+
+	verimf	%v0, %v0, %v0, 0
+	verimf	%v0, %v0, %v0, 255
+	verimf	%v0, %v0, %v31, 0
+	verimf	%v0, %v31, %v0, 0
+	verimf	%v31, %v0, %v0, 0
+	verimf 	%v13, %v17, %v21, 0x79
+
+#CHECK: verimg  %v0, %v0, %v0, 0        # encoding: [0xe7,0x00,0x00,0x00,0x30,0x72]
+#CHECK: verimg  %v0, %v0, %v0, 255      # encoding: [0xe7,0x00,0x00,0xff,0x30,0x72]
+#CHECK: verimg  %v0, %v0, %v31, 0       # encoding: [0xe7,0x00,0xf0,0x00,0x32,0x72]
+#CHECK: verimg  %v0, %v31, %v0, 0       # encoding: [0xe7,0x0f,0x00,0x00,0x34,0x72]
+#CHECK: verimg  %v31, %v0, %v0, 0       # encoding: [0xe7,0xf0,0x00,0x00,0x38,0x72]
+#CHECK: verimg  %v13, %v17, %v21, 121   # encoding: [0xe7,0xd1,0x50,0x79,0x36,0x72]
+
+	verimg	%v0, %v0, %v0, 0
+	verimg	%v0, %v0, %v0, 255
+	verimg	%v0, %v0, %v31, 0
+	verimg	%v0, %v31, %v0, 0
+	verimg	%v31, %v0, %v0, 0
+	verimg 	%v13, %v17, %v21, 0x79
+
+#CHECK: verimh  %v0, %v0, %v0, 0        # encoding: [0xe7,0x00,0x00,0x00,0x10,0x72]
+#CHECK: verimh  %v0, %v0, %v0, 255      # encoding: [0xe7,0x00,0x00,0xff,0x10,0x72]
+#CHECK: verimh  %v0, %v0, %v31, 0       # encoding: [0xe7,0x00,0xf0,0x00,0x12,0x72]
+#CHECK: verimh  %v0, %v31, %v0, 0       # encoding: [0xe7,0x0f,0x00,0x00,0x14,0x72]
+#CHECK: verimh  %v31, %v0, %v0, 0       # encoding: [0xe7,0xf0,0x00,0x00,0x18,0x72]
+#CHECK: verimh  %v13, %v17, %v21, 121   # encoding: [0xe7,0xd1,0x50,0x79,0x16,0x72]
+
+	verimh	%v0, %v0, %v0, 0
+	verimh	%v0, %v0, %v0, 255
+	verimh	%v0, %v0, %v31, 0
+	verimh	%v0, %v31, %v0, 0
+	verimh	%v31, %v0, %v0, 0
+	verimh 	%v13, %v17, %v21, 0x79
+
+#CHECK: verllvb %v0, %v0, %v0           # encoding: [0xe7,0x00,0x00,0x00,0x00,0x73]
+#CHECK: verllvb %v0, %v0, %v31          # encoding: [0xe7,0x00,0xf0,0x00,0x02,0x73]
+#CHECK: verllvb %v0, %v31, %v0          # encoding: [0xe7,0x0f,0x00,0x00,0x04,0x73]
+#CHECK: verllvb %v31, %v0, %v0          # encoding: [0xe7,0xf0,0x00,0x00,0x08,0x73]
+#CHECK: verllvb %v18, %v3, %v20         # encoding: [0xe7,0x23,0x40,0x00,0x0a,0x73]
+
+	verllvb	%v0, %v0, %v0
+	verllvb	%v0, %v0, %v31
+	verllvb	%v0, %v31, %v0
+	verllvb	%v31, %v0, %v0
+	verllvb	%v18, %v3, %v20
+
+#CHECK: verllvf %v0, %v0, %v0           # encoding: [0xe7,0x00,0x00,0x00,0x20,0x73]
+#CHECK: verllvf %v0, %v0, %v31          # encoding: [0xe7,0x00,0xf0,0x00,0x22,0x73]
+#CHECK: verllvf %v0, %v31, %v0          # encoding: [0xe7,0x0f,0x00,0x00,0x24,0x73]
+#CHECK: verllvf %v31, %v0, %v0          # encoding: [0xe7,0xf0,0x00,0x00,0x28,0x73]
+#CHECK: verllvf %v18, %v3, %v20         # encoding: [0xe7,0x23,0x40,0x00,0x2a,0x73]
+
+	verllvf	%v0, %v0, %v0
+	verllvf	%v0, %v0, %v31
+	verllvf	%v0, %v31, %v0
+	verllvf	%v31, %v0, %v0
+	verllvf	%v18, %v3, %v20
+
+#CHECK: verllvg %v0, %v0, %v0           # encoding: [0xe7,0x00,0x00,0x00,0x30,0x73]
+#CHECK: verllvg %v0, %v0, %v31          # encoding: [0xe7,0x00,0xf0,0x00,0x32,0x73]
+#CHECK: verllvg %v0, %v31, %v0          # encoding: [0xe7,0x0f,0x00,0x00,0x34,0x73]
+#CHECK: verllvg %v31, %v0, %v0          # encoding: [0xe7,0xf0,0x00,0x00,0x38,0x73]
+#CHECK: verllvg %v18, %v3, %v20         # encoding: [0xe7,0x23,0x40,0x00,0x3a,0x73]
+
+	verllvg	%v0, %v0, %v0
+	verllvg	%v0, %v0, %v31
+	verllvg	%v0, %v31, %v0
+	verllvg	%v31, %v0, %v0
+	verllvg	%v18, %v3, %v20
+
+#CHECK: verllvh %v0, %v0, %v0           # encoding: [0xe7,0x00,0x00,0x00,0x10,0x73]
+#CHECK: verllvh %v0, %v0, %v31          # encoding: [0xe7,0x00,0xf0,0x00,0x12,0x73]
+#CHECK: verllvh %v0, %v31, %v0          # encoding: [0xe7,0x0f,0x00,0x00,0x14,0x73]
+#CHECK: verllvh %v31, %v0, %v0          # encoding: [0xe7,0xf0,0x00,0x00,0x18,0x73]
+#CHECK: verllvh %v18, %v3, %v20         # encoding: [0xe7,0x23,0x40,0x00,0x1a,0x73]
+
+	verllvh	%v0, %v0, %v0
+	verllvh	%v0, %v0, %v31
+	verllvh	%v0, %v31, %v0
+	verllvh	%v31, %v0, %v0
+	verllvh	%v18, %v3, %v20
+
+#CHECK: verllb  %v0, %v0, 0             # encoding: [0xe7,0x00,0x00,0x00,0x00,0x33]
+#CHECK: verllb  %v0, %v0, 4095          # encoding: [0xe7,0x00,0x0f,0xff,0x00,0x33]
+#CHECK: verllb  %v0, %v0, 0(%r15)       # encoding: [0xe7,0x00,0xf0,0x00,0x00,0x33]
+#CHECK: verllb  %v0, %v31, 0            # encoding: [0xe7,0x0f,0x00,0x00,0x04,0x33]
+#CHECK: verllb  %v31, %v0, 0            # encoding: [0xe7,0xf0,0x00,0x00,0x08,0x33]
+#CHECK: verllb  %v14, %v17, 1074(%r5)   # encoding: [0xe7,0xe1,0x54,0x32,0x04,0x33]
+
+	verllb	%v0, %v0, 0
+	verllb	%v0, %v0, 4095
+	verllb	%v0, %v0, 0(%r15)
+	verllb	%v0, %v31, 0
+	verllb	%v31, %v0, 0
+	verllb	%v14, %v17, 1074(%r5)
+
+#CHECK: verllf  %v0, %v0, 0             # encoding: [0xe7,0x00,0x00,0x00,0x20,0x33]
+#CHECK: verllf  %v0, %v0, 4095          # encoding: [0xe7,0x00,0x0f,0xff,0x20,0x33]
+#CHECK: verllf  %v0, %v0, 0(%r15)       # encoding: [0xe7,0x00,0xf0,0x00,0x20,0x33]
+#CHECK: verllf  %v0, %v31, 0            # encoding: [0xe7,0x0f,0x00,0x00,0x24,0x33]
+#CHECK: verllf  %v31, %v0, 0            # encoding: [0xe7,0xf0,0x00,0x00,0x28,0x33]
+#CHECK: verllf  %v14, %v17, 1074(%r5)   # encoding: [0xe7,0xe1,0x54,0x32,0x24,0x33]
+
+	verllf	%v0, %v0, 0
+	verllf	%v0, %v0, 4095
+	verllf	%v0, %v0, 0(%r15)
+	verllf	%v0, %v31, 0
+	verllf	%v31, %v0, 0
+	verllf	%v14, %v17, 1074(%r5)
+
+#CHECK: verllg  %v0, %v0, 0             # encoding: [0xe7,0x00,0x00,0x00,0x30,0x33]
+#CHECK: verllg  %v0, %v0, 4095          # encoding: [0xe7,0x00,0x0f,0xff,0x30,0x33]
+#CHECK: verllg  %v0, %v0, 0(%r15)       # encoding: [0xe7,0x00,0xf0,0x00,0x30,0x33]
+#CHECK: verllg  %v0, %v31, 0            # encoding: [0xe7,0x0f,0x00,0x00,0x34,0x33]
+#CHECK: verllg  %v31, %v0, 0            # encoding: [0xe7,0xf0,0x00,0x00,0x38,0x33]
+#CHECK: verllg  %v14, %v17, 1074(%r5)   # encoding: [0xe7,0xe1,0x54,0x32,0x34,0x33]
+
+	verllg	%v0, %v0, 0
+	verllg	%v0, %v0, 4095
+	verllg	%v0, %v0, 0(%r15)
+	verllg	%v0, %v31, 0
+	verllg	%v31, %v0, 0
+	verllg	%v14, %v17, 1074(%r5)
+
+#CHECK: verllh  %v0, %v0, 0             # encoding: [0xe7,0x00,0x00,0x00,0x10,0x33]
+#CHECK: verllh  %v0, %v0, 4095          # encoding: [0xe7,0x00,0x0f,0xff,0x10,0x33]
+#CHECK: verllh  %v0, %v0, 0(%r15)       # encoding: [0xe7,0x00,0xf0,0x00,0x10,0x33]
+#CHECK: verllh  %v0, %v31, 0            # encoding: [0xe7,0x0f,0x00,0x00,0x14,0x33]
+#CHECK: verllh  %v31, %v0, 0            # encoding: [0xe7,0xf0,0x00,0x00,0x18,0x33]
+#CHECK: verllh  %v14, %v17, 1074(%r5)   # encoding: [0xe7,0xe1,0x54,0x32,0x14,0x33]
+
+	verllh	%v0, %v0, 0
+	verllh	%v0, %v0, 4095
+	verllh	%v0, %v0, 0(%r15)
+	verllh	%v0, %v31, 0
+	verllh	%v31, %v0, 0
+	verllh	%v14, %v17, 1074(%r5)
+
+#CHECK: veslvb  %v0, %v0, %v0           # encoding: [0xe7,0x00,0x00,0x00,0x00,0x70]
+#CHECK: veslvb  %v0, %v0, %v31          # encoding: [0xe7,0x00,0xf0,0x00,0x02,0x70]
+#CHECK: veslvb  %v0, %v31, %v0          # encoding: [0xe7,0x0f,0x00,0x00,0x04,0x70]
+#CHECK: veslvb  %v31, %v0, %v0          # encoding: [0xe7,0xf0,0x00,0x00,0x08,0x70]
+#CHECK: veslvb  %v18, %v3, %v20         # encoding: [0xe7,0x23,0x40,0x00,0x0a,0x70]
+
+	veslvb	%v0, %v0, %v0
+	veslvb	%v0, %v0, %v31
+	veslvb	%v0, %v31, %v0
+	veslvb	%v31, %v0, %v0
+	veslvb	%v18, %v3, %v20
+
+#CHECK: veslvf  %v0, %v0, %v0           # encoding: [0xe7,0x00,0x00,0x00,0x20,0x70]
+#CHECK: veslvf  %v0, %v0, %v31          # encoding: [0xe7,0x00,0xf0,0x00,0x22,0x70]
+#CHECK: veslvf  %v0, %v31, %v0          # encoding: [0xe7,0x0f,0x00,0x00,0x24,0x70]
+#CHECK: veslvf  %v31, %v0, %v0          # encoding: [0xe7,0xf0,0x00,0x00,0x28,0x70]
+#CHECK: veslvf  %v18, %v3, %v20         # encoding: [0xe7,0x23,0x40,0x00,0x2a,0x70]
+
+	veslvf	%v0, %v0, %v0
+	veslvf	%v0, %v0, %v31
+	veslvf	%v0, %v31, %v0
+	veslvf	%v31, %v0, %v0
+	veslvf	%v18, %v3, %v20
+
+#CHECK: veslvg  %v0, %v0, %v0           # encoding: [0xe7,0x00,0x00,0x00,0x30,0x70]
+#CHECK: veslvg  %v0, %v0, %v31          # encoding: [0xe7,0x00,0xf0,0x00,0x32,0x70]
+#CHECK: veslvg  %v0, %v31, %v0          # encoding: [0xe7,0x0f,0x00,0x00,0x34,0x70]
+#CHECK: veslvg  %v31, %v0, %v0          # encoding: [0xe7,0xf0,0x00,0x00,0x38,0x70]
+#CHECK: veslvg  %v18, %v3, %v20         # encoding: [0xe7,0x23,0x40,0x00,0x3a,0x70]
+
+	veslvg	%v0, %v0, %v0
+	veslvg	%v0, %v0, %v31
+	veslvg	%v0, %v31, %v0
+	veslvg	%v31, %v0, %v0
+	veslvg	%v18, %v3, %v20
+
+#CHECK: veslvh  %v0, %v0, %v0           # encoding: [0xe7,0x00,0x00,0x00,0x10,0x70]
+#CHECK: veslvh  %v0, %v0, %v31          # encoding: [0xe7,0x00,0xf0,0x00,0x12,0x70]
+#CHECK: veslvh  %v0, %v31, %v0          # encoding: [0xe7,0x0f,0x00,0x00,0x14,0x70]
+#CHECK: veslvh  %v31, %v0, %v0          # encoding: [0xe7,0xf0,0x00,0x00,0x18,0x70]
+#CHECK: veslvh  %v18, %v3, %v20         # encoding: [0xe7,0x23,0x40,0x00,0x1a,0x70]
+
+	veslvh	%v0, %v0, %v0
+	veslvh	%v0, %v0, %v31
+	veslvh	%v0, %v31, %v0
+	veslvh	%v31, %v0, %v0
+	veslvh	%v18, %v3, %v20
+
+#CHECK: veslb   %v0, %v0, 0             # encoding: [0xe7,0x00,0x00,0x00,0x00,0x30]
+#CHECK: veslb   %v0, %v0, 4095          # encoding: [0xe7,0x00,0x0f,0xff,0x00,0x30]
+#CHECK: veslb   %v0, %v0, 0(%r15)       # encoding: [0xe7,0x00,0xf0,0x00,0x00,0x30]
+#CHECK: veslb   %v0, %v31, 0            # encoding: [0xe7,0x0f,0x00,0x00,0x04,0x30]
+#CHECK: veslb   %v31, %v0, 0            # encoding: [0xe7,0xf0,0x00,0x00,0x08,0x30]
+#CHECK: veslb   %v14, %v17, 1074(%r5)   # encoding: [0xe7,0xe1,0x54,0x32,0x04,0x30]
+
+	veslb	%v0, %v0, 0
+	veslb	%v0, %v0, 4095
+	veslb	%v0, %v0, 0(%r15)
+	veslb	%v0, %v31, 0
+	veslb	%v31, %v0, 0
+	veslb	%v14, %v17, 1074(%r5)
+
+#CHECK: veslf   %v0, %v0, 0             # encoding: [0xe7,0x00,0x00,0x00,0x20,0x30]
+#CHECK: veslf   %v0, %v0, 4095          # encoding: [0xe7,0x00,0x0f,0xff,0x20,0x30]
+#CHECK: veslf   %v0, %v0, 0(%r15)       # encoding: [0xe7,0x00,0xf0,0x00,0x20,0x30]
+#CHECK: veslf   %v0, %v31, 0            # encoding: [0xe7,0x0f,0x00,0x00,0x24,0x30]
+#CHECK: veslf   %v31, %v0, 0            # encoding: [0xe7,0xf0,0x00,0x00,0x28,0x30]
+#CHECK: veslf   %v14, %v17, 1074(%r5)   # encoding: [0xe7,0xe1,0x54,0x32,0x24,0x30]
+
+	veslf	%v0, %v0, 0
+	veslf	%v0, %v0, 4095
+	veslf	%v0, %v0, 0(%r15)
+	veslf	%v0, %v31, 0
+	veslf	%v31, %v0, 0
+	veslf	%v14, %v17, 1074(%r5)
+
+#CHECK: veslg   %v0, %v0, 0             # encoding: [0xe7,0x00,0x00,0x00,0x30,0x30]
+#CHECK: veslg   %v0, %v0, 4095          # encoding: [0xe7,0x00,0x0f,0xff,0x30,0x30]
+#CHECK: veslg   %v0, %v0, 0(%r15)       # encoding: [0xe7,0x00,0xf0,0x00,0x30,0x30]
+#CHECK: veslg   %v0, %v31, 0            # encoding: [0xe7,0x0f,0x00,0x00,0x34,0x30]
+#CHECK: veslg   %v31, %v0, 0            # encoding: [0xe7,0xf0,0x00,0x00,0x38,0x30]
+#CHECK: veslg   %v14, %v17, 1074(%r5)   # encoding: [0xe7,0xe1,0x54,0x32,0x34,0x30]
+
+	veslg	%v0, %v0, 0
+	veslg	%v0, %v0, 4095
+	veslg	%v0, %v0, 0(%r15)
+	veslg	%v0, %v31, 0
+	veslg	%v31, %v0, 0
+	veslg	%v14, %v17, 1074(%r5)
+
+#CHECK: veslh   %v0, %v0, 0             # encoding: [0xe7,0x00,0x00,0x00,0x10,0x30]
+#CHECK: veslh   %v0, %v0, 4095          # encoding: [0xe7,0x00,0x0f,0xff,0x10,0x30]
+#CHECK: veslh   %v0, %v0, 0(%r15)       # encoding: [0xe7,0x00,0xf0,0x00,0x10,0x30]
+#CHECK: veslh   %v0, %v31, 0            # encoding: [0xe7,0x0f,0x00,0x00,0x14,0x30]
+#CHECK: veslh   %v31, %v0, 0            # encoding: [0xe7,0xf0,0x00,0x00,0x18,0x30]
+#CHECK: veslh   %v14, %v17, 1074(%r5)   # encoding: [0xe7,0xe1,0x54,0x32,0x14,0x30]
+
+	veslh	%v0, %v0, 0
+	veslh	%v0, %v0, 4095
+	veslh	%v0, %v0, 0(%r15)
+	veslh	%v0, %v31, 0
+	veslh	%v31, %v0, 0
+	veslh	%v14, %v17, 1074(%r5)
+
+#CHECK: vesravb %v0, %v0, %v0           # encoding: [0xe7,0x00,0x00,0x00,0x00,0x7a]
+#CHECK: vesravb %v0, %v0, %v31          # encoding: [0xe7,0x00,0xf0,0x00,0x02,0x7a]
+#CHECK: vesravb %v0, %v31, %v0          # encoding: [0xe7,0x0f,0x00,0x00,0x04,0x7a]
+#CHECK: vesravb %v31, %v0, %v0          # encoding: [0xe7,0xf0,0x00,0x00,0x08,0x7a]
+#CHECK: vesravb %v18, %v3, %v20         # encoding: [0xe7,0x23,0x40,0x00,0x0a,0x7a]
+
+	vesravb	%v0, %v0, %v0
+	vesravb	%v0, %v0, %v31
+	vesravb	%v0, %v31, %v0
+	vesravb	%v31, %v0, %v0
+	vesravb	%v18, %v3, %v20
+
+#CHECK: vesravf %v0, %v0, %v0           # encoding: [0xe7,0x00,0x00,0x00,0x20,0x7a]
+#CHECK: vesravf %v0, %v0, %v31          # encoding: [0xe7,0x00,0xf0,0x00,0x22,0x7a]
+#CHECK: vesravf %v0, %v31, %v0          # encoding: [0xe7,0x0f,0x00,0x00,0x24,0x7a]
+#CHECK: vesravf %v31, %v0, %v0          # encoding: [0xe7,0xf0,0x00,0x00,0x28,0x7a]
+#CHECK: vesravf %v18, %v3, %v20         # encoding: [0xe7,0x23,0x40,0x00,0x2a,0x7a]
+
+	vesravf	%v0, %v0, %v0
+	vesravf	%v0, %v0, %v31
+	vesravf	%v0, %v31, %v0
+	vesravf	%v31, %v0, %v0
+	vesravf	%v18, %v3, %v20
+
+#CHECK: vesravg %v0, %v0, %v0           # encoding: [0xe7,0x00,0x00,0x00,0x30,0x7a]
+#CHECK: vesravg %v0, %v0, %v31          # encoding: [0xe7,0x00,0xf0,0x00,0x32,0x7a]
+#CHECK: vesravg %v0, %v31, %v0          # encoding: [0xe7,0x0f,0x00,0x00,0x34,0x7a]
+#CHECK: vesravg %v31, %v0, %v0          # encoding: [0xe7,0xf0,0x00,0x00,0x38,0x7a]
+#CHECK: vesravg %v18, %v3, %v20         # encoding: [0xe7,0x23,0x40,0x00,0x3a,0x7a]
+
+	vesravg	%v0, %v0, %v0
+	vesravg	%v0, %v0, %v31
+	vesravg	%v0, %v31, %v0
+	vesravg	%v31, %v0, %v0
+	vesravg	%v18, %v3, %v20
+
+#CHECK: vesravh %v0, %v0, %v0           # encoding: [0xe7,0x00,0x00,0x00,0x10,0x7a]
+#CHECK: vesravh %v0, %v0, %v31          # encoding: [0xe7,0x00,0xf0,0x00,0x12,0x7a]
+#CHECK: vesravh %v0, %v31, %v0          # encoding: [0xe7,0x0f,0x00,0x00,0x14,0x7a]
+#CHECK: vesravh %v31, %v0, %v0          # encoding: [0xe7,0xf0,0x00,0x00,0x18,0x7a]
+#CHECK: vesravh %v18, %v3, %v20         # encoding: [0xe7,0x23,0x40,0x00,0x1a,0x7a]
+
+	vesravh	%v0, %v0, %v0
+	vesravh	%v0, %v0, %v31
+	vesravh	%v0, %v31, %v0
+	vesravh	%v31, %v0, %v0
+	vesravh	%v18, %v3, %v20
+
+#CHECK: vesrab  %v0, %v0, 0             # encoding: [0xe7,0x00,0x00,0x00,0x00,0x3a]
+#CHECK: vesrab  %v0, %v0, 4095          # encoding: [0xe7,0x00,0x0f,0xff,0x00,0x3a]
+#CHECK: vesrab  %v0, %v0, 0(%r15)       # encoding: [0xe7,0x00,0xf0,0x00,0x00,0x3a]
+#CHECK: vesrab  %v0, %v31, 0            # encoding: [0xe7,0x0f,0x00,0x00,0x04,0x3a]
+#CHECK: vesrab  %v31, %v0, 0            # encoding: [0xe7,0xf0,0x00,0x00,0x08,0x3a]
+#CHECK: vesrab  %v14, %v17, 1074(%r5)   # encoding: [0xe7,0xe1,0x54,0x32,0x04,0x3a]
+
+	vesrab	%v0, %v0, 0
+	vesrab	%v0, %v0, 4095
+	vesrab	%v0, %v0, 0(%r15)
+	vesrab	%v0, %v31, 0
+	vesrab	%v31, %v0, 0
+	vesrab	%v14, %v17, 1074(%r5)
+
+#CHECK: vesraf  %v0, %v0, 0             # encoding: [0xe7,0x00,0x00,0x00,0x20,0x3a]
+#CHECK: vesraf  %v0, %v0, 4095          # encoding: [0xe7,0x00,0x0f,0xff,0x20,0x3a]
+#CHECK: vesraf  %v0, %v0, 0(%r15)       # encoding: [0xe7,0x00,0xf0,0x00,0x20,0x3a]
+#CHECK: vesraf  %v0, %v31, 0            # encoding: [0xe7,0x0f,0x00,0x00,0x24,0x3a]
+#CHECK: vesraf  %v31, %v0, 0            # encoding: [0xe7,0xf0,0x00,0x00,0x28,0x3a]
+#CHECK: vesraf  %v14, %v17, 1074(%r5)   # encoding: [0xe7,0xe1,0x54,0x32,0x24,0x3a]
+
+	vesraf	%v0, %v0, 0
+	vesraf	%v0, %v0, 4095
+	vesraf	%v0, %v0, 0(%r15)
+	vesraf	%v0, %v31, 0
+	vesraf	%v31, %v0, 0
+	vesraf	%v14, %v17, 1074(%r5)
+
+#CHECK: vesrag  %v0, %v0, 0             # encoding: [0xe7,0x00,0x00,0x00,0x30,0x3a]
+#CHECK: vesrag  %v0, %v0, 4095          # encoding: [0xe7,0x00,0x0f,0xff,0x30,0x3a]
+#CHECK: vesrag  %v0, %v0, 0(%r15)       # encoding: [0xe7,0x00,0xf0,0x00,0x30,0x3a]
+#CHECK: vesrag  %v0, %v31, 0            # encoding: [0xe7,0x0f,0x00,0x00,0x34,0x3a]
+#CHECK: vesrag  %v31, %v0, 0            # encoding: [0xe7,0xf0,0x00,0x00,0x38,0x3a]
+#CHECK: vesrag  %v14, %v17, 1074(%r5)   # encoding: [0xe7,0xe1,0x54,0x32,0x34,0x3a]
+
+	vesrag	%v0, %v0, 0
+	vesrag	%v0, %v0, 4095
+	vesrag	%v0, %v0, 0(%r15)
+	vesrag	%v0, %v31, 0
+	vesrag	%v31, %v0, 0
+	vesrag	%v14, %v17, 1074(%r5)
+
+#CHECK: vesrah  %v0, %v0, 0             # encoding: [0xe7,0x00,0x00,0x00,0x10,0x3a]
+#CHECK: vesrah  %v0, %v0, 4095          # encoding: [0xe7,0x00,0x0f,0xff,0x10,0x3a]
+#CHECK: vesrah  %v0, %v0, 0(%r15)       # encoding: [0xe7,0x00,0xf0,0x00,0x10,0x3a]
+#CHECK: vesrah  %v0, %v31, 0            # encoding: [0xe7,0x0f,0x00,0x00,0x14,0x3a]
+#CHECK: vesrah  %v31, %v0, 0            # encoding: [0xe7,0xf0,0x00,0x00,0x18,0x3a]
+#CHECK: vesrah  %v14, %v17, 1074(%r5)   # encoding: [0xe7,0xe1,0x54,0x32,0x14,0x3a]
+
+	vesrah	%v0, %v0, 0
+	vesrah	%v0, %v0, 4095
+	vesrah	%v0, %v0, 0(%r15)
+	vesrah	%v0, %v31, 0
+	vesrah	%v31, %v0, 0
+	vesrah	%v14, %v17, 1074(%r5)
+
+#CHECK: vesrlvb %v0, %v0, %v0           # encoding: [0xe7,0x00,0x00,0x00,0x00,0x78]
+#CHECK: vesrlvb %v0, %v0, %v31          # encoding: [0xe7,0x00,0xf0,0x00,0x02,0x78]
+#CHECK: vesrlvb %v0, %v31, %v0          # encoding: [0xe7,0x0f,0x00,0x00,0x04,0x78]
+#CHECK: vesrlvb %v31, %v0, %v0          # encoding: [0xe7,0xf0,0x00,0x00,0x08,0x78]
+#CHECK: vesrlvb %v18, %v3, %v20         # encoding: [0xe7,0x23,0x40,0x00,0x0a,0x78]
+
+	vesrlvb	%v0, %v0, %v0
+	vesrlvb	%v0, %v0, %v31
+	vesrlvb	%v0, %v31, %v0
+	vesrlvb	%v31, %v0, %v0
+	vesrlvb	%v18, %v3, %v20
+
+#CHECK: vesrlvf %v0, %v0, %v0           # encoding: [0xe7,0x00,0x00,0x00,0x20,0x78]
+#CHECK: vesrlvf %v0, %v0, %v31          # encoding: [0xe7,0x00,0xf0,0x00,0x22,0x78]
+#CHECK: vesrlvf %v0, %v31, %v0          # encoding: [0xe7,0x0f,0x00,0x00,0x24,0x78]
+#CHECK: vesrlvf %v31, %v0, %v0          # encoding: [0xe7,0xf0,0x00,0x00,0x28,0x78]
+#CHECK: vesrlvf %v18, %v3, %v20         # encoding: [0xe7,0x23,0x40,0x00,0x2a,0x78]
+
+	vesrlvf	%v0, %v0, %v0
+	vesrlvf	%v0, %v0, %v31
+	vesrlvf	%v0, %v31, %v0
+	vesrlvf	%v31, %v0, %v0
+	vesrlvf	%v18, %v3, %v20
+
+#CHECK: vesrlvg %v0, %v0, %v0           # encoding: [0xe7,0x00,0x00,0x00,0x30,0x78]
+#CHECK: vesrlvg %v0, %v0, %v31          # encoding: [0xe7,0x00,0xf0,0x00,0x32,0x78]
+#CHECK: vesrlvg %v0, %v31, %v0          # encoding: [0xe7,0x0f,0x00,0x00,0x34,0x78]
+#CHECK: vesrlvg %v31, %v0, %v0          # encoding: [0xe7,0xf0,0x00,0x00,0x38,0x78]
+#CHECK: vesrlvg %v18, %v3, %v20         # encoding: [0xe7,0x23,0x40,0x00,0x3a,0x78]
+
+	vesrlvg	%v0, %v0, %v0
+	vesrlvg	%v0, %v0, %v31
+	vesrlvg	%v0, %v31, %v0
+	vesrlvg	%v31, %v0, %v0
+	vesrlvg	%v18, %v3, %v20
+
+#CHECK: vesrlvh %v0, %v0, %v0           # encoding: [0xe7,0x00,0x00,0x00,0x10,0x78]
+#CHECK: vesrlvh %v0, %v0, %v31          # encoding: [0xe7,0x00,0xf0,0x00,0x12,0x78]
+#CHECK: vesrlvh %v0, %v31, %v0          # encoding: [0xe7,0x0f,0x00,0x00,0x14,0x78]
+#CHECK: vesrlvh %v31, %v0, %v0          # encoding: [0xe7,0xf0,0x00,0x00,0x18,0x78]
+#CHECK: vesrlvh %v18, %v3, %v20         # encoding: [0xe7,0x23,0x40,0x00,0x1a,0x78]
+
+	vesrlvh	%v0, %v0, %v0
+	vesrlvh	%v0, %v0, %v31
+	vesrlvh	%v0, %v31, %v0
+	vesrlvh	%v31, %v0, %v0
+	vesrlvh	%v18, %v3, %v20
+
+#CHECK: vesrlb  %v0, %v0, 0             # encoding: [0xe7,0x00,0x00,0x00,0x00,0x38]
+#CHECK: vesrlb  %v0, %v0, 4095          # encoding: [0xe7,0x00,0x0f,0xff,0x00,0x38]
+#CHECK: vesrlb  %v0, %v0, 0(%r15)       # encoding: [0xe7,0x00,0xf0,0x00,0x00,0x38]
+#CHECK: vesrlb  %v0, %v31, 0            # encoding: [0xe7,0x0f,0x00,0x00,0x04,0x38]
+#CHECK: vesrlb  %v31, %v0, 0            # encoding: [0xe7,0xf0,0x00,0x00,0x08,0x38]
+#CHECK: vesrlb  %v14, %v17, 1074(%r5)   # encoding: [0xe7,0xe1,0x54,0x32,0x04,0x38]
+
+	vesrlb	%v0, %v0, 0
+	vesrlb	%v0, %v0, 4095
+	vesrlb	%v0, %v0, 0(%r15)
+	vesrlb	%v0, %v31, 0
+	vesrlb	%v31, %v0, 0
+	vesrlb	%v14, %v17, 1074(%r5)
+
+#CHECK: vesrlf  %v0, %v0, 0             # encoding: [0xe7,0x00,0x00,0x00,0x20,0x38]
+#CHECK: vesrlf  %v0, %v0, 4095          # encoding: [0xe7,0x00,0x0f,0xff,0x20,0x38]
+#CHECK: vesrlf  %v0, %v0, 0(%r15)       # encoding: [0xe7,0x00,0xf0,0x00,0x20,0x38]
+#CHECK: vesrlf  %v0, %v31, 0            # encoding: [0xe7,0x0f,0x00,0x00,0x24,0x38]
+#CHECK: vesrlf  %v31, %v0, 0            # encoding: [0xe7,0xf0,0x00,0x00,0x28,0x38]
+#CHECK: vesrlf  %v14, %v17, 1074(%r5)   # encoding: [0xe7,0xe1,0x54,0x32,0x24,0x38]
+
+	vesrlf	%v0, %v0, 0
+	vesrlf	%v0, %v0, 4095
+	vesrlf	%v0, %v0, 0(%r15)
+	vesrlf	%v0, %v31, 0
+	vesrlf	%v31, %v0, 0
+	vesrlf	%v14, %v17, 1074(%r5)
+
+#CHECK: vesrlg  %v0, %v0, 0             # encoding: [0xe7,0x00,0x00,0x00,0x30,0x38]
+#CHECK: vesrlg  %v0, %v0, 4095          # encoding: [0xe7,0x00,0x0f,0xff,0x30,0x38]
+#CHECK: vesrlg  %v0, %v0, 0(%r15)       # encoding: [0xe7,0x00,0xf0,0x00,0x30,0x38]
+#CHECK: vesrlg  %v0, %v31, 0            # encoding: [0xe7,0x0f,0x00,0x00,0x34,0x38]
+#CHECK: vesrlg  %v31, %v0, 0            # encoding: [0xe7,0xf0,0x00,0x00,0x38,0x38]
+#CHECK: vesrlg  %v14, %v17, 1074(%r5)   # encoding: [0xe7,0xe1,0x54,0x32,0x34,0x38]
+
+	vesrlg	%v0, %v0, 0
+	vesrlg	%v0, %v0, 4095
+	vesrlg	%v0, %v0, 0(%r15)
+	vesrlg	%v0, %v31, 0
+	vesrlg	%v31, %v0, 0
+	vesrlg	%v14, %v17, 1074(%r5)
+
+#CHECK: vesrlh  %v0, %v0, 0             # encoding: [0xe7,0x00,0x00,0x00,0x10,0x38]
+#CHECK: vesrlh  %v0, %v0, 4095          # encoding: [0xe7,0x00,0x0f,0xff,0x10,0x38]
+#CHECK: vesrlh  %v0, %v0, 0(%r15)       # encoding: [0xe7,0x00,0xf0,0x00,0x10,0x38]
+#CHECK: vesrlh  %v0, %v31, 0            # encoding: [0xe7,0x0f,0x00,0x00,0x14,0x38]
+#CHECK: vesrlh  %v31, %v0, 0            # encoding: [0xe7,0xf0,0x00,0x00,0x18,0x38]
+#CHECK: vesrlh  %v14, %v17, 1074(%r5)   # encoding: [0xe7,0xe1,0x54,0x32,0x14,0x38]
+
+	vesrlh	%v0, %v0, 0
+	vesrlh	%v0, %v0, 4095
+	vesrlh	%v0, %v0, 0(%r15)
+	vesrlh	%v0, %v31, 0
+	vesrlh	%v31, %v0, 0
+	vesrlh	%v14, %v17, 1074(%r5)
+
+#CHECK: vfadb   %v0, %v0, %v0           # encoding: [0xe7,0x00,0x00,0x00,0x30,0xe3]
+#CHECK: vfadb   %v0, %v0, %v31          # encoding: [0xe7,0x00,0xf0,0x00,0x32,0xe3]
+#CHECK: vfadb   %v0, %v31, %v0          # encoding: [0xe7,0x0f,0x00,0x00,0x34,0xe3]
+#CHECK: vfadb   %v31, %v0, %v0          # encoding: [0xe7,0xf0,0x00,0x00,0x38,0xe3]
+#CHECK: vfadb   %v18, %v3, %v20         # encoding: [0xe7,0x23,0x40,0x00,0x3a,0xe3]
+
+	vfadb	%v0, %v0, %v0
+	vfadb	%v0, %v0, %v31
+	vfadb	%v0, %v31, %v0
+	vfadb	%v31, %v0, %v0
+	vfadb	%v18, %v3, %v20
+
+#CHECK: vfaeb   %v0, %v0, %v0, 0        # encoding: [0xe7,0x00,0x00,0x00,0x00,0x82]
+#CHECK: vfaeb   %v0, %v0, %v0, 0        # encoding: [0xe7,0x00,0x00,0x00,0x00,0x82]
+#CHECK: vfaeb   %v0, %v0, %v0, 12       # encoding: [0xe7,0x00,0x00,0xc0,0x00,0x82]
+#CHECK: vfaeb   %v0, %v0, %v15, 0       # encoding: [0xe7,0x00,0xf0,0x00,0x00,0x82]
+#CHECK: vfaeb   %v0, %v0, %v31, 0       # encoding: [0xe7,0x00,0xf0,0x00,0x02,0x82]
+#CHECK: vfaeb   %v0, %v15, %v0, 0       # encoding: [0xe7,0x0f,0x00,0x00,0x00,0x82]
+#CHECK: vfaeb   %v0, %v31, %v0, 0       # encoding: [0xe7,0x0f,0x00,0x00,0x04,0x82]
+#CHECK: vfaeb   %v15, %v0, %v0, 0       # encoding: [0xe7,0xf0,0x00,0x00,0x00,0x82]
+#CHECK: vfaeb   %v31, %v0, %v0, 0       # encoding: [0xe7,0xf0,0x00,0x00,0x08,0x82]
+#CHECK: vfaeb   %v18, %v3, %v20, 4      # encoding: [0xe7,0x23,0x40,0x40,0x0a,0x82]
+#CHECK: vfaeb   %v18, %v3, %v20, 15     # encoding: [0xe7,0x23,0x40,0xf0,0x0a,0x82]
+#CHECK: vfaebs  %v18, %v3, %v20, 8      # encoding: [0xe7,0x23,0x40,0x90,0x0a,0x82]
+#CHECK: vfaezb  %v18, %v3, %v20, 4      # encoding: [0xe7,0x23,0x40,0x60,0x0a,0x82]
+#CHECK: vfaezbs %v18, %v3, %v20, 8      # encoding: [0xe7,0x23,0x40,0xb0,0x0a,0x82]
+#CHECK: vfaezbs %v18, %v3, %v20, 15     # encoding: [0xe7,0x23,0x40,0xf0,0x0a,0x82]
+
+	vfaeb	%v0, %v0, %v0
+	vfaeb	%v0, %v0, %v0, 0
+	vfaeb	%v0, %v0, %v0, 12
+	vfaeb	%v0, %v0, %v15
+	vfaeb	%v0, %v0, %v31
+	vfaeb	%v0, %v15, %v0
+	vfaeb	%v0, %v31, %v0
+	vfaeb	%v15, %v0, %v0
+	vfaeb	%v31, %v0, %v0
+	vfaeb	%v18, %v3, %v20, 4
+	vfaeb	%v18, %v3, %v20, 15
+	vfaebs	%v18, %v3, %v20, 8
+	vfaezb	%v18, %v3, %v20, 4
+	vfaezbs	%v18, %v3, %v20, 8
+	vfaezbs	%v18, %v3, %v20, 15
+
+#CHECK: vfaef   %v0, %v0, %v0, 0        # encoding: [0xe7,0x00,0x00,0x00,0x20,0x82]
+#CHECK: vfaef   %v0, %v0, %v0, 0        # encoding: [0xe7,0x00,0x00,0x00,0x20,0x82]
+#CHECK: vfaef   %v0, %v0, %v0, 12       # encoding: [0xe7,0x00,0x00,0xc0,0x20,0x82]
+#CHECK: vfaef   %v0, %v0, %v15, 0       # encoding: [0xe7,0x00,0xf0,0x00,0x20,0x82]
+#CHECK: vfaef   %v0, %v0, %v31, 0       # encoding: [0xe7,0x00,0xf0,0x00,0x22,0x82]
+#CHECK: vfaef   %v0, %v15, %v0, 0       # encoding: [0xe7,0x0f,0x00,0x00,0x20,0x82]
+#CHECK: vfaef   %v0, %v31, %v0, 0       # encoding: [0xe7,0x0f,0x00,0x00,0x24,0x82]
+#CHECK: vfaef   %v15, %v0, %v0, 0       # encoding: [0xe7,0xf0,0x00,0x00,0x20,0x82]
+#CHECK: vfaef   %v31, %v0, %v0, 0       # encoding: [0xe7,0xf0,0x00,0x00,0x28,0x82]
+#CHECK: vfaef   %v18, %v3, %v20, 4      # encoding: [0xe7,0x23,0x40,0x40,0x2a,0x82]
+#CHECK: vfaef   %v18, %v3, %v20, 15     # encoding: [0xe7,0x23,0x40,0xf0,0x2a,0x82]
+#CHECK: vfaefs  %v18, %v3, %v20, 8      # encoding: [0xe7,0x23,0x40,0x90,0x2a,0x82]
+#CHECK: vfaezf  %v18, %v3, %v20, 4      # encoding: [0xe7,0x23,0x40,0x60,0x2a,0x82]
+#CHECK: vfaezfs %v18, %v3, %v20, 8      # encoding: [0xe7,0x23,0x40,0xb0,0x2a,0x82]
+#CHECK: vfaezfs %v18, %v3, %v20, 15     # encoding: [0xe7,0x23,0x40,0xf0,0x2a,0x82]
+
+	vfaef	%v0, %v0, %v0
+	vfaef	%v0, %v0, %v0, 0
+	vfaef	%v0, %v0, %v0, 12
+	vfaef	%v0, %v0, %v15
+	vfaef	%v0, %v0, %v31
+	vfaef	%v0, %v15, %v0
+	vfaef	%v0, %v31, %v0
+	vfaef	%v15, %v0, %v0
+	vfaef	%v31, %v0, %v0
+	vfaef	%v18, %v3, %v20, 4
+	vfaef	%v18, %v3, %v20, 15
+	vfaefs	%v18, %v3, %v20, 8
+	vfaezf	%v18, %v3, %v20, 4
+	vfaezfs	%v18, %v3, %v20, 8
+	vfaezfs	%v18, %v3, %v20, 15
+
+#CHECK: vfaeh   %v0, %v0, %v0, 0        # encoding: [0xe7,0x00,0x00,0x00,0x10,0x82]
+#CHECK: vfaeh   %v0, %v0, %v0, 0        # encoding: [0xe7,0x00,0x00,0x00,0x10,0x82]
+#CHECK: vfaeh   %v0, %v0, %v0, 12       # encoding: [0xe7,0x00,0x00,0xc0,0x10,0x82]
+#CHECK: vfaeh   %v0, %v0, %v15, 0       # encoding: [0xe7,0x00,0xf0,0x00,0x10,0x82]
+#CHECK: vfaeh   %v0, %v0, %v31, 0       # encoding: [0xe7,0x00,0xf0,0x00,0x12,0x82]
+#CHECK: vfaeh   %v0, %v15, %v0, 0       # encoding: [0xe7,0x0f,0x00,0x00,0x10,0x82]
+#CHECK: vfaeh   %v0, %v31, %v0, 0       # encoding: [0xe7,0x0f,0x00,0x00,0x14,0x82]
+#CHECK: vfaeh   %v15, %v0, %v0, 0       # encoding: [0xe7,0xf0,0x00,0x00,0x10,0x82]
+#CHECK: vfaeh   %v31, %v0, %v0, 0       # encoding: [0xe7,0xf0,0x00,0x00,0x18,0x82]
+#CHECK: vfaeh   %v18, %v3, %v20, 4      # encoding: [0xe7,0x23,0x40,0x40,0x1a,0x82]
+#CHECK: vfaeh   %v18, %v3, %v20, 15     # encoding: [0xe7,0x23,0x40,0xf0,0x1a,0x82]
+#CHECK: vfaehs  %v18, %v3, %v20, 8      # encoding: [0xe7,0x23,0x40,0x90,0x1a,0x82]
+#CHECK: vfaezh  %v18, %v3, %v20, 4      # encoding: [0xe7,0x23,0x40,0x60,0x1a,0x82]
+#CHECK: vfaezhs %v18, %v3, %v20, 8      # encoding: [0xe7,0x23,0x40,0xb0,0x1a,0x82]
+#CHECK: vfaezhs %v18, %v3, %v20, 15     # encoding: [0xe7,0x23,0x40,0xf0,0x1a,0x82]
+
+	vfaeh	%v0, %v0, %v0
+	vfaeh	%v0, %v0, %v0, 0
+	vfaeh	%v0, %v0, %v0, 12
+	vfaeh	%v0, %v0, %v15
+	vfaeh	%v0, %v0, %v31
+	vfaeh	%v0, %v15, %v0
+	vfaeh	%v0, %v31, %v0
+	vfaeh	%v15, %v0, %v0
+	vfaeh	%v31, %v0, %v0
+	vfaeh	%v18, %v3, %v20, 4
+	vfaeh	%v18, %v3, %v20, 15
+	vfaehs	%v18, %v3, %v20, 8
+	vfaezh	%v18, %v3, %v20, 4
+	vfaezhs	%v18, %v3, %v20, 8
+	vfaezhs	%v18, %v3, %v20, 15
+
+#CHECK: vfcedb  %v0, %v0, %v0           # encoding: [0xe7,0x00,0x00,0x00,0x30,0xe8]
+#CHECK: vfcedb  %v0, %v0, %v31          # encoding: [0xe7,0x00,0xf0,0x00,0x32,0xe8]
+#CHECK: vfcedb  %v0, %v31, %v0          # encoding: [0xe7,0x0f,0x00,0x00,0x34,0xe8]
+#CHECK: vfcedb  %v31, %v0, %v0          # encoding: [0xe7,0xf0,0x00,0x00,0x38,0xe8]
+#CHECK: vfcedb  %v18, %v3, %v20         # encoding: [0xe7,0x23,0x40,0x00,0x3a,0xe8]
+
+	vfcedb	%v0, %v0, %v0
+	vfcedb	%v0, %v0, %v31
+	vfcedb	%v0, %v31, %v0
+	vfcedb	%v31, %v0, %v0
+	vfcedb	%v18, %v3, %v20
+
+#CHECK: vfcedbs %v0, %v0, %v0           # encoding: [0xe7,0x00,0x00,0x10,0x30,0xe8]
+#CHECK: vfcedbs %v0, %v0, %v31          # encoding: [0xe7,0x00,0xf0,0x10,0x32,0xe8]
+#CHECK: vfcedbs %v0, %v31, %v0          # encoding: [0xe7,0x0f,0x00,0x10,0x34,0xe8]
+#CHECK: vfcedbs %v31, %v0, %v0          # encoding: [0xe7,0xf0,0x00,0x10,0x38,0xe8]
+#CHECK: vfcedbs %v18, %v3, %v20         # encoding: [0xe7,0x23,0x40,0x10,0x3a,0xe8]
+
+	vfcedbs	%v0, %v0, %v0
+	vfcedbs	%v0, %v0, %v31
+	vfcedbs	%v0, %v31, %v0
+	vfcedbs	%v31, %v0, %v0
+	vfcedbs	%v18, %v3, %v20
+
+#CHECK: vfchdb  %v0, %v0, %v0           # encoding: [0xe7,0x00,0x00,0x00,0x30,0xeb]
+#CHECK: vfchdb  %v0, %v0, %v31          # encoding: [0xe7,0x00,0xf0,0x00,0x32,0xeb]
+#CHECK: vfchdb  %v0, %v31, %v0          # encoding: [0xe7,0x0f,0x00,0x00,0x34,0xeb]
+#CHECK: vfchdb  %v31, %v0, %v0          # encoding: [0xe7,0xf0,0x00,0x00,0x38,0xeb]
+#CHECK: vfchdb  %v18, %v3, %v20         # encoding: [0xe7,0x23,0x40,0x00,0x3a,0xeb]
+
+	vfchdb	%v0, %v0, %v0
+	vfchdb	%v0, %v0, %v31
+	vfchdb	%v0, %v31, %v0
+	vfchdb	%v31, %v0, %v0
+	vfchdb	%v18, %v3, %v20
+
+#CHECK: vfchdbs %v0, %v0, %v0           # encoding: [0xe7,0x00,0x00,0x10,0x30,0xeb]
+#CHECK: vfchdbs %v0, %v0, %v31          # encoding: [0xe7,0x00,0xf0,0x10,0x32,0xeb]
+#CHECK: vfchdbs %v0, %v31, %v0          # encoding: [0xe7,0x0f,0x00,0x10,0x34,0xeb]
+#CHECK: vfchdbs %v31, %v0, %v0          # encoding: [0xe7,0xf0,0x00,0x10,0x38,0xeb]
+#CHECK: vfchdbs %v18, %v3, %v20         # encoding: [0xe7,0x23,0x40,0x10,0x3a,0xeb]
+
+	vfchdbs	%v0, %v0, %v0
+	vfchdbs	%v0, %v0, %v31
+	vfchdbs	%v0, %v31, %v0
+	vfchdbs	%v31, %v0, %v0
+	vfchdbs	%v18, %v3, %v20
+
+#CHECK: vfchedb %v0, %v0, %v0           # encoding: [0xe7,0x00,0x00,0x00,0x30,0xea]
+#CHECK: vfchedb %v0, %v0, %v31          # encoding: [0xe7,0x00,0xf0,0x00,0x32,0xea]
+#CHECK: vfchedb %v0, %v31, %v0          # encoding: [0xe7,0x0f,0x00,0x00,0x34,0xea]
+#CHECK: vfchedb %v31, %v0, %v0          # encoding: [0xe7,0xf0,0x00,0x00,0x38,0xea]
+#CHECK: vfchedb %v18, %v3, %v20         # encoding: [0xe7,0x23,0x40,0x00,0x3a,0xea]
+
+	vfchedb	%v0, %v0, %v0
+	vfchedb	%v0, %v0, %v31
+	vfchedb	%v0, %v31, %v0
+	vfchedb	%v31, %v0, %v0
+	vfchedb	%v18, %v3, %v20
+
+#CHECK: vfchedbs %v0, %v0, %v0          # encoding: [0xe7,0x00,0x00,0x10,0x30,0xea]
+#CHECK: vfchedbs %v0, %v0, %v31         # encoding: [0xe7,0x00,0xf0,0x10,0x32,0xea]
+#CHECK: vfchedbs %v0, %v31, %v0         # encoding: [0xe7,0x0f,0x00,0x10,0x34,0xea]
+#CHECK: vfchedbs %v31, %v0, %v0         # encoding: [0xe7,0xf0,0x00,0x10,0x38,0xea]
+#CHECK: vfchedbs %v18, %v3, %v20        # encoding: [0xe7,0x23,0x40,0x10,0x3a,0xea]
+
+	vfchedbs %v0, %v0, %v0
+	vfchedbs %v0, %v0, %v31
+	vfchedbs %v0, %v31, %v0
+	vfchedbs %v31, %v0, %v0
+	vfchedbs %v18, %v3, %v20
+
+#CHECK: vfddb   %v0, %v0, %v0           # encoding: [0xe7,0x00,0x00,0x00,0x30,0xe5]
+#CHECK: vfddb   %v0, %v0, %v31          # encoding: [0xe7,0x00,0xf0,0x00,0x32,0xe5]
+#CHECK: vfddb   %v0, %v31, %v0          # encoding: [0xe7,0x0f,0x00,0x00,0x34,0xe5]
+#CHECK: vfddb   %v31, %v0, %v0          # encoding: [0xe7,0xf0,0x00,0x00,0x38,0xe5]
+#CHECK: vfddb   %v18, %v3, %v20         # encoding: [0xe7,0x23,0x40,0x00,0x3a,0xe5]
+
+	vfddb	%v0, %v0, %v0
+	vfddb	%v0, %v0, %v31
+	vfddb	%v0, %v31, %v0
+	vfddb	%v31, %v0, %v0
+	vfddb	%v18, %v3, %v20
+
+#CHECK: vfeeb   %v0, %v0, %v0           # encoding: [0xe7,0x00,0x00,0x00,0x00,0x80]
+#CHECK: vfeeb   %v0, %v0, %v15          # encoding: [0xe7,0x00,0xf0,0x00,0x00,0x80]
+#CHECK: vfeeb   %v0, %v0, %v31          # encoding: [0xe7,0x00,0xf0,0x00,0x02,0x80]
+#CHECK: vfeeb   %v0, %v15, %v0          # encoding: [0xe7,0x0f,0x00,0x00,0x00,0x80]
+#CHECK: vfeeb   %v0, %v31, %v0          # encoding: [0xe7,0x0f,0x00,0x00,0x04,0x80]
+#CHECK: vfeeb   %v15, %v0, %v0          # encoding: [0xe7,0xf0,0x00,0x00,0x00,0x80]
+#CHECK: vfeeb   %v31, %v0, %v0          # encoding: [0xe7,0xf0,0x00,0x00,0x08,0x80]
+#CHECK: vfeeb   %v18, %v3, %v20         # encoding: [0xe7,0x23,0x40,0x00,0x0a,0x80]
+#CHECK: vfeebs  %v5, %v22, %v7          # encoding: [0xe7,0x56,0x70,0x10,0x04,0x80]
+#CHECK: vfeezb  %v18, %v3, %v20         # encoding: [0xe7,0x23,0x40,0x20,0x0a,0x80]
+#CHECK: vfeezbs %v5, %v22, %v7          # encoding: [0xe7,0x56,0x70,0x30,0x04,0x80]
+
+	vfeeb	%v0, %v0, %v0
+	vfeeb	%v0, %v0, %v15
+	vfeeb	%v0, %v0, %v31
+	vfeeb	%v0, %v15, %v0
+	vfeeb	%v0, %v31, %v0
+	vfeeb	%v15, %v0, %v0
+	vfeeb	%v31, %v0, %v0
+	vfeeb	%v18, %v3, %v20
+	vfeebs	%v5, %v22, %v7
+	vfeezb	%v18, %v3, %v20
+	vfeezbs	%v5, %v22, %v7
+
+#CFECK: vfeef   %v0, %v0, %v0           # encoding: [0xe7,0x00,0x00,0x00,0x20,0x80]
+#CFECK: vfeef   %v0, %v0, %v15          # encoding: [0xe7,0x00,0xf0,0x00,0x20,0x80]
+#CFECK: vfeef   %v0, %v0, %v31          # encoding: [0xe7,0x00,0xf0,0x00,0x22,0x80]
+#CFECK: vfeef   %v0, %v15, %v0          # encoding: [0xe7,0x0f,0x00,0x00,0x20,0x80]
+#CFECK: vfeef   %v0, %v31, %v0          # encoding: [0xe7,0x0f,0x00,0x00,0x24,0x80]
+#CFECK: vfeef   %v15, %v0, %v0          # encoding: [0xe7,0xf0,0x00,0x00,0x20,0x80]
+#CFECK: vfeef   %v31, %v0, %v0          # encoding: [0xe7,0xf0,0x00,0x00,0x28,0x80]
+#CFECK: vfeef   %v18, %v3, %v20         # encoding: [0xe7,0x23,0x40,0x00,0x2a,0x80]
+#CFECK: vfeefs  %v5, %v22, %v7          # encoding: [0xe7,0x56,0x70,0x10,0x24,0x80]
+#CFECK: vfeezf  %v18, %v3, %v20         # encoding: [0xe7,0x23,0x40,0x20,0x2a,0x80]
+#CFECK: vfeezfs %v5, %v22, %v7          # encoding: [0xe7,0x56,0x70,0x30,0x24,0x80]
+
+	vfeef	%v0, %v0, %v0
+	vfeef	%v0, %v0, %v15
+	vfeef	%v0, %v0, %v31
+	vfeef	%v0, %v15, %v0
+	vfeef	%v0, %v31, %v0
+	vfeef	%v15, %v0, %v0
+	vfeef	%v31, %v0, %v0
+	vfeef	%v18, %v3, %v20
+	vfeefs	%v5, %v22, %v7
+	vfeezf	%v18, %v3, %v20
+	vfeezfs	%v5, %v22, %v7
+
+#CHECK: vfeeh   %v0, %v0, %v0           # encoding: [0xe7,0x00,0x00,0x00,0x10,0x80]
+#CHECK: vfeeh   %v0, %v0, %v15          # encoding: [0xe7,0x00,0xf0,0x00,0x10,0x80]
+#CHECK: vfeeh   %v0, %v0, %v31          # encoding: [0xe7,0x00,0xf0,0x00,0x12,0x80]
+#CHECK: vfeeh   %v0, %v15, %v0          # encoding: [0xe7,0x0f,0x00,0x00,0x10,0x80]
+#CHECK: vfeeh   %v0, %v31, %v0          # encoding: [0xe7,0x0f,0x00,0x00,0x14,0x80]
+#CHECK: vfeeh   %v15, %v0, %v0          # encoding: [0xe7,0xf0,0x00,0x00,0x10,0x80]
+#CHECK: vfeeh   %v31, %v0, %v0          # encoding: [0xe7,0xf0,0x00,0x00,0x18,0x80]
+#CHECK: vfeeh   %v18, %v3, %v20         # encoding: [0xe7,0x23,0x40,0x00,0x1a,0x80]
+#CHECK: vfeehs  %v5, %v22, %v7          # encoding: [0xe7,0x56,0x70,0x10,0x14,0x80]
+#CHECK: vfeezh  %v18, %v3, %v20         # encoding: [0xe7,0x23,0x40,0x20,0x1a,0x80]
+#CHECK: vfeezhs %v5, %v22, %v7          # encoding: [0xe7,0x56,0x70,0x30,0x14,0x80]
+
+	vfeeh	%v0, %v0, %v0
+	vfeeh	%v0, %v0, %v15
+	vfeeh	%v0, %v0, %v31
+	vfeeh	%v0, %v15, %v0
+	vfeeh	%v0, %v31, %v0
+	vfeeh	%v15, %v0, %v0
+	vfeeh	%v31, %v0, %v0
+	vfeeh	%v18, %v3, %v20
+	vfeehs	%v5, %v22, %v7
+	vfeezh	%v18, %v3, %v20
+	vfeezhs	%v5, %v22, %v7
+
+#CHECK: vfeneb   %v0, %v0, %v0          # encoding: [0xe7,0x00,0x00,0x00,0x00,0x81]
+#CHECK: vfeneb   %v0, %v0, %v15         # encoding: [0xe7,0x00,0xf0,0x00,0x00,0x81]
+#CHECK: vfeneb   %v0, %v0, %v31         # encoding: [0xe7,0x00,0xf0,0x00,0x02,0x81]
+#CHECK: vfeneb   %v0, %v15, %v0         # encoding: [0xe7,0x0f,0x00,0x00,0x00,0x81]
+#CHECK: vfeneb   %v0, %v31, %v0         # encoding: [0xe7,0x0f,0x00,0x00,0x04,0x81]
+#CHECK: vfeneb   %v15, %v0, %v0         # encoding: [0xe7,0xf0,0x00,0x00,0x00,0x81]
+#CHECK: vfeneb   %v31, %v0, %v0         # encoding: [0xe7,0xf0,0x00,0x00,0x08,0x81]
+#CHECK: vfeneb   %v18, %v3, %v20        # encoding: [0xe7,0x23,0x40,0x00,0x0a,0x81]
+#CHECK: vfenebs  %v5, %v22, %v7         # encoding: [0xe7,0x56,0x70,0x10,0x04,0x81]
+#CHECK: vfenezb  %v18, %v3, %v20        # encoding: [0xe7,0x23,0x40,0x20,0x0a,0x81]
+#CHECK: vfenezbs %v5, %v22, %v7         # encoding: [0xe7,0x56,0x70,0x30,0x04,0x81]
+
+	vfeneb   %v0, %v0, %v0
+	vfeneb   %v0, %v0, %v15
+	vfeneb   %v0, %v0, %v31
+	vfeneb   %v0, %v15, %v0
+	vfeneb   %v0, %v31, %v0
+	vfeneb   %v15, %v0, %v0
+	vfeneb   %v31, %v0, %v0
+	vfeneb   %v18, %v3, %v20
+	vfenebs  %v5, %v22, %v7
+	vfenezb  %v18, %v3, %v20
+	vfenezbs %v5, %v22, %v7
+
+#CFECK: vfenef   %v0, %v0, %v0          # encoding: [0xe7,0x00,0x00,0x00,0x20,0x81]
+#CFECK: vfenef   %v0, %v0, %v15         # encoding: [0xe7,0x00,0xf0,0x00,0x20,0x81]
+#CFECK: vfenef   %v0, %v0, %v31         # encoding: [0xe7,0x00,0xf0,0x00,0x22,0x81]
+#CFECK: vfenef   %v0, %v15, %v0         # encoding: [0xe7,0x0f,0x00,0x00,0x20,0x81]
+#CFECK: vfenef   %v0, %v31, %v0         # encoding: [0xe7,0x0f,0x00,0x00,0x24,0x81]
+#CFECK: vfenef   %v15, %v0, %v0         # encoding: [0xe7,0xf0,0x00,0x00,0x20,0x81]
+#CFECK: vfenef   %v31, %v0, %v0         # encoding: [0xe7,0xf0,0x00,0x00,0x28,0x81]
+#CFECK: vfenef   %v18, %v3, %v20        # encoding: [0xe7,0x23,0x40,0x00,0x2a,0x81]
+#CFECK: vfenefs  %v5, %v22, %v7         # encoding: [0xe7,0x56,0x70,0x10,0x24,0x81]
+#CFECK: vfenezf  %v18, %v3, %v20        # encoding: [0xe7,0x23,0x40,0x20,0x2a,0x81]
+#CFECK: vfenezfs %v5, %v22, %v7         # encoding: [0xe7,0x56,0x70,0x30,0x24,0x81]
+
+	vfenef   %v0, %v0, %v0
+	vfenef   %v0, %v0, %v15
+	vfenef   %v0, %v0, %v31
+	vfenef   %v0, %v15, %v0
+	vfenef   %v0, %v31, %v0
+	vfenef   %v15, %v0, %v0
+	vfenef   %v31, %v0, %v0
+	vfenef   %v18, %v3, %v20
+	vfenefs  %v5, %v22, %v7
+	vfenezf  %v18, %v3, %v20
+	vfenezfs %v5, %v22, %v7
+
+#CHECK: vfeneh   %v0, %v0, %v0          # encoding: [0xe7,0x00,0x00,0x00,0x10,0x81]
+#CHECK: vfeneh   %v0, %v0, %v15         # encoding: [0xe7,0x00,0xf0,0x00,0x10,0x81]
+#CHECK: vfeneh   %v0, %v0, %v31         # encoding: [0xe7,0x00,0xf0,0x00,0x12,0x81]
+#CHECK: vfeneh   %v0, %v15, %v0         # encoding: [0xe7,0x0f,0x00,0x00,0x10,0x81]
+#CHECK: vfeneh   %v0, %v31, %v0         # encoding: [0xe7,0x0f,0x00,0x00,0x14,0x81]
+#CHECK: vfeneh   %v15, %v0, %v0         # encoding: [0xe7,0xf0,0x00,0x00,0x10,0x81]
+#CHECK: vfeneh   %v31, %v0, %v0         # encoding: [0xe7,0xf0,0x00,0x00,0x18,0x81]
+#CHECK: vfeneh   %v18, %v3, %v20        # encoding: [0xe7,0x23,0x40,0x00,0x1a,0x81]
+#CHECK: vfenehs  %v5, %v22, %v7         # encoding: [0xe7,0x56,0x70,0x10,0x14,0x81]
+#CHECK: vfenezh  %v18, %v3, %v20        # encoding: [0xe7,0x23,0x40,0x20,0x1a,0x81]
+#CHECK: vfenezhs %v5, %v22, %v7         # encoding: [0xe7,0x56,0x70,0x30,0x14,0x81]
+
+	vfeneh   %v0, %v0, %v0
+	vfeneh   %v0, %v0, %v15
+	vfeneh   %v0, %v0, %v31
+	vfeneh   %v0, %v15, %v0
+	vfeneh   %v0, %v31, %v0
+	vfeneh   %v15, %v0, %v0
+	vfeneh   %v31, %v0, %v0
+	vfeneh   %v18, %v3, %v20
+	vfenehs  %v5, %v22, %v7
+	vfenezh  %v18, %v3, %v20
+	vfenezhs %v5, %v22, %v7
+
+#CHECK: vfidb   %v0, %v0, 0, 0          # encoding: [0xe7,0x00,0x00,0x00,0x30,0xc7]
+#CHECK: vfidb   %v0, %v0, 0, 15         # encoding: [0xe7,0x00,0x00,0xf0,0x30,0xc7]
+#CHECK: vfidb   %v0, %v0, 4, 0          # encoding: [0xe7,0x00,0x00,0x04,0x30,0xc7]
+#CHECK: vfidb   %v0, %v0, 12, 0         # encoding: [0xe7,0x00,0x00,0x0c,0x30,0xc7]
+#CHECK: vfidb   %v0, %v31, 0, 0         # encoding: [0xe7,0x0f,0x00,0x00,0x34,0xc7]
+#CHECK: vfidb   %v31, %v0, 0, 0         # encoding: [0xe7,0xf0,0x00,0x00,0x38,0xc7]
+#CHECK: vfidb   %v14, %v17, 4, 10       # encoding: [0xe7,0xe1,0x00,0xa4,0x34,0xc7]
+
+	vfidb	%v0, %v0, 0, 0
+	vfidb	%v0, %v0, 0, 15
+	vfidb	%v0, %v0, 4, 0
+	vfidb	%v0, %v0, 12, 0
+	vfidb	%v0, %v31, 0, 0
+	vfidb	%v31, %v0, 0, 0
+	vfidb	%v14, %v17, 4, 10
+
+#CHECK: vistrb   %v0, %v0               # encoding: [0xe7,0x00,0x00,0x00,0x00,0x5c]
+#CHECK: vistrb   %v0, %v15              # encoding: [0xe7,0x0f,0x00,0x00,0x00,0x5c]
+#CHECK: vistrb   %v0, %v31              # encoding: [0xe7,0x0f,0x00,0x00,0x04,0x5c]
+#CHECK: vistrb   %v15, %v0              # encoding: [0xe7,0xf0,0x00,0x00,0x00,0x5c]
+#CHECK: vistrb   %v31, %v0              # encoding: [0xe7,0xf0,0x00,0x00,0x08,0x5c]
+#CHECK: vistrb   %v18, %v3              # encoding: [0xe7,0x23,0x00,0x00,0x08,0x5c]
+#CHECK: vistrbs  %v5, %v22              # encoding: [0xe7,0x56,0x00,0x10,0x04,0x5c]
+
+	vistrb   %v0, %v0
+	vistrb   %v0, %v15
+	vistrb   %v0, %v31
+	vistrb   %v15, %v0
+	vistrb   %v31, %v0
+	vistrb   %v18, %v3
+	vistrbs  %v5, %v22
+
+#CBECK: vistrf   %v0, %v0               # encoding: [0xe7,0x00,0x00,0x00,0x20,0x5c]
+#CBECK: vistrf   %v0, %v15              # encoding: [0xe7,0x0f,0x00,0x00,0x20,0x5c]
+#CBECK: vistrf   %v0, %v31              # encoding: [0xe7,0x0f,0x00,0x00,0x24,0x5c]
+#CBECK: vistrf   %v15, %v0              # encoding: [0xe7,0xf0,0x00,0x00,0x20,0x5c]
+#CBECK: vistrf   %v31, %v0              # encoding: [0xe7,0xf0,0x00,0x00,0x28,0x5c]
+#CBECK: vistrf   %v18, %v3              # encoding: [0xe7,0x23,0x00,0x00,0x28,0x5c]
+#CBECK: vistrfs  %v5, %v22              # encoding: [0xe7,0x56,0x00,0x10,0x24,0x5c]
+
+	vistrf   %v0, %v0
+	vistrf   %v0, %v15
+	vistrf   %v0, %v31
+	vistrf   %v15, %v0
+	vistrf   %v31, %v0
+	vistrf   %v18, %v3
+	vistrfs  %v5, %v22
+
+#CHECK: vistrh   %v0, %v0               # encoding: [0xe7,0x00,0x00,0x00,0x10,0x5c]
+#CHECK: vistrh   %v0, %v15              # encoding: [0xe7,0x0f,0x00,0x00,0x10,0x5c]
+#CHECK: vistrh   %v0, %v31              # encoding: [0xe7,0x0f,0x00,0x00,0x14,0x5c]
+#CHECK: vistrh   %v15, %v0              # encoding: [0xe7,0xf0,0x00,0x00,0x10,0x5c]
+#CHECK: vistrh   %v31, %v0              # encoding: [0xe7,0xf0,0x00,0x00,0x18,0x5c]
+#CHECK: vistrh   %v18, %v3              # encoding: [0xe7,0x23,0x00,0x00,0x18,0x5c]
+#CHECK: vistrhs  %v5, %v22              # encoding: [0xe7,0x56,0x00,0x10,0x14,0x5c]
+
+	vistrh   %v0, %v0
+	vistrh   %v0, %v15
+	vistrh   %v0, %v31
+	vistrh   %v15, %v0
+	vistrh   %v31, %v0
+	vistrh   %v18, %v3
+	vistrhs  %v5, %v22
+
+#CHECK: vflcdb  %v0, %v0                # encoding: [0xe7,0x00,0x00,0x00,0x30,0xcc]
+#CHECK: vflcdb  %v0, %v15               # encoding: [0xe7,0x0f,0x00,0x00,0x30,0xcc]
+#CHECK: vflcdb  %v0, %v31               # encoding: [0xe7,0x0f,0x00,0x00,0x34,0xcc]
+#CHECK: vflcdb  %v15, %v0               # encoding: [0xe7,0xf0,0x00,0x00,0x30,0xcc]
+#CHECK: vflcdb  %v31, %v0               # encoding: [0xe7,0xf0,0x00,0x00,0x38,0xcc]
+#CHECK: vflcdb  %v14, %v17              # encoding: [0xe7,0xe1,0x00,0x00,0x34,0xcc]
+
+	vflcdb	%v0, %v0
+	vflcdb	%v0, %v15
+	vflcdb	%v0, %v31
+	vflcdb	%v15, %v0
+	vflcdb	%v31, %v0
+	vflcdb	%v14, %v17
+
+#CHECK: vflndb  %v0, %v0                # encoding: [0xe7,0x00,0x00,0x10,0x30,0xcc]
+#CHECK: vflndb  %v0, %v15               # encoding: [0xe7,0x0f,0x00,0x10,0x30,0xcc]
+#CHECK: vflndb  %v0, %v31               # encoding: [0xe7,0x0f,0x00,0x10,0x34,0xcc]
+#CHECK: vflndb  %v15, %v0               # encoding: [0xe7,0xf0,0x00,0x10,0x30,0xcc]
+#CHECK: vflndb  %v31, %v0               # encoding: [0xe7,0xf0,0x00,0x10,0x38,0xcc]
+#CHECK: vflndb  %v14, %v17              # encoding: [0xe7,0xe1,0x00,0x10,0x34,0xcc]
+
+	vflndb	%v0, %v0
+	vflndb	%v0, %v15
+	vflndb	%v0, %v31
+	vflndb	%v15, %v0
+	vflndb	%v31, %v0
+	vflndb	%v14, %v17
+
+#CHECK: vflpdb  %v0, %v0                # encoding: [0xe7,0x00,0x00,0x20,0x30,0xcc]
+#CHECK: vflpdb  %v0, %v15               # encoding: [0xe7,0x0f,0x00,0x20,0x30,0xcc]
+#CHECK: vflpdb  %v0, %v31               # encoding: [0xe7,0x0f,0x00,0x20,0x34,0xcc]
+#CHECK: vflpdb  %v15, %v0               # encoding: [0xe7,0xf0,0x00,0x20,0x30,0xcc]
+#CHECK: vflpdb  %v31, %v0               # encoding: [0xe7,0xf0,0x00,0x20,0x38,0xcc]
+#CHECK: vflpdb  %v14, %v17              # encoding: [0xe7,0xe1,0x00,0x20,0x34,0xcc]
+
+	vflpdb	%v0, %v0
+	vflpdb	%v0, %v15
+	vflpdb	%v0, %v31
+	vflpdb	%v15, %v0
+	vflpdb	%v31, %v0
+	vflpdb	%v14, %v17
+
+#CHECK: vfmadb  %v0, %v0, %v0, %v0      # encoding: [0xe7,0x00,0x03,0x00,0x00,0x8f]
+#CHECK: vfmadb  %v0, %v0, %v0, %v31     # encoding: [0xe7,0x00,0x03,0x00,0xf1,0x8f]
+#CHECK: vfmadb  %v0, %v0, %v31, %v0     # encoding: [0xe7,0x00,0xf3,0x00,0x02,0x8f]
+#CHECK: vfmadb  %v0, %v31, %v0, %v0     # encoding: [0xe7,0x0f,0x03,0x00,0x04,0x8f]
+#CHECK: vfmadb  %v31, %v0, %v0, %v0     # encoding: [0xe7,0xf0,0x03,0x00,0x08,0x8f]
+#CHECK: vfmadb  %v13, %v17, %v21, %v25  # encoding: [0xe7,0xd1,0x53,0x00,0x97,0x8f]
+
+	vfmadb	%v0, %v0, %v0, %v0
+	vfmadb	%v0, %v0, %v0, %v31
+	vfmadb	%v0, %v0, %v31, %v0
+	vfmadb	%v0, %v31, %v0, %v0
+	vfmadb	%v31, %v0, %v0, %v0
+	vfmadb	%v13, %v17, %v21, %v25
+
+#CHECK: vfmdb   %v0, %v0, %v0           # encoding: [0xe7,0x00,0x00,0x00,0x30,0xe7]
+#CHECK: vfmdb   %v0, %v0, %v31          # encoding: [0xe7,0x00,0xf0,0x00,0x32,0xe7]
+#CHECK: vfmdb   %v0, %v31, %v0          # encoding: [0xe7,0x0f,0x00,0x00,0x34,0xe7]
+#CHECK: vfmdb   %v31, %v0, %v0          # encoding: [0xe7,0xf0,0x00,0x00,0x38,0xe7]
+#CHECK: vfmdb   %v18, %v3, %v20         # encoding: [0xe7,0x23,0x40,0x00,0x3a,0xe7]
+
+	vfmdb	%v0, %v0, %v0
+	vfmdb	%v0, %v0, %v31
+	vfmdb	%v0, %v31, %v0
+	vfmdb	%v31, %v0, %v0
+	vfmdb	%v18, %v3, %v20
+
+#CHECK: vfmsdb  %v0, %v0, %v0, %v0      # encoding: [0xe7,0x00,0x03,0x00,0x00,0x8e]
+#CHECK: vfmsdb  %v0, %v0, %v0, %v31     # encoding: [0xe7,0x00,0x03,0x00,0xf1,0x8e]
+#CHECK: vfmsdb  %v0, %v0, %v31, %v0     # encoding: [0xe7,0x00,0xf3,0x00,0x02,0x8e]
+#CHECK: vfmsdb  %v0, %v31, %v0, %v0     # encoding: [0xe7,0x0f,0x03,0x00,0x04,0x8e]
+#CHECK: vfmsdb  %v31, %v0, %v0, %v0     # encoding: [0xe7,0xf0,0x03,0x00,0x08,0x8e]
+#CHECK: vfmsdb  %v13, %v17, %v21, %v25  # encoding: [0xe7,0xd1,0x53,0x00,0x97,0x8e]
+
+	vfmsdb	%v0, %v0, %v0, %v0
+	vfmsdb	%v0, %v0, %v0, %v31
+	vfmsdb	%v0, %v0, %v31, %v0
+	vfmsdb	%v0, %v31, %v0, %v0
+	vfmsdb	%v31, %v0, %v0, %v0
+	vfmsdb	%v13, %v17, %v21, %v25
+
+#CHECK: vfsdb   %v0, %v0, %v0           # encoding: [0xe7,0x00,0x00,0x00,0x30,0xe2]
+#CHECK: vfsdb   %v0, %v0, %v31          # encoding: [0xe7,0x00,0xf0,0x00,0x32,0xe2]
+#CHECK: vfsdb   %v0, %v31, %v0          # encoding: [0xe7,0x0f,0x00,0x00,0x34,0xe2]
+#CHECK: vfsdb   %v31, %v0, %v0          # encoding: [0xe7,0xf0,0x00,0x00,0x38,0xe2]
+#CHECK: vfsdb   %v18, %v3, %v20         # encoding: [0xe7,0x23,0x40,0x00,0x3a,0xe2]
+
+	vfsdb	%v0, %v0, %v0
+	vfsdb	%v0, %v0, %v31
+	vfsdb	%v0, %v31, %v0
+	vfsdb	%v31, %v0, %v0
+	vfsdb	%v18, %v3, %v20
+
+#CHECK: vfsqdb  %v0, %v0                # encoding: [0xe7,0x00,0x00,0x00,0x30,0xce]
+#CHECK: vfsqdb  %v0, %v15               # encoding: [0xe7,0x0f,0x00,0x00,0x30,0xce]
+#CHECK: vfsqdb  %v0, %v31               # encoding: [0xe7,0x0f,0x00,0x00,0x34,0xce]
+#CHECK: vfsqdb  %v15, %v0               # encoding: [0xe7,0xf0,0x00,0x00,0x30,0xce]
+#CHECK: vfsqdb  %v31, %v0               # encoding: [0xe7,0xf0,0x00,0x00,0x38,0xce]
+#CHECK: vfsqdb  %v14, %v17              # encoding: [0xe7,0xe1,0x00,0x00,0x34,0xce]
+
+	vfsqdb	%v0, %v0
+	vfsqdb	%v0, %v15
+	vfsqdb	%v0, %v31
+	vfsqdb	%v15, %v0
+	vfsqdb	%v31, %v0
+	vfsqdb	%v14, %v17
+
+#CHECK: vftcidb %v0, %v0, 0             # encoding: [0xe7,0x00,0x00,0x00,0x30,0x4a]
+#CHECK: vftcidb %v0, %v0, 4095          # encoding: [0xe7,0x00,0xff,0xf0,0x30,0x4a]
+#CHECK: vftcidb %v0, %v15, 0            # encoding: [0xe7,0x0f,0x00,0x00,0x30,0x4a]
+#CHECK: vftcidb %v0, %v31, 0            # encoding: [0xe7,0x0f,0x00,0x00,0x34,0x4a]
+#CHECK: vftcidb %v15, %v0, 0            # encoding: [0xe7,0xf0,0x00,0x00,0x30,0x4a]
+#CHECK: vftcidb %v31, %v0, 0            # encoding: [0xe7,0xf0,0x00,0x00,0x38,0x4a]
+#CHECK: vftcidb %v4, %v21, 1656         # encoding: [0xe7,0x45,0x67,0x80,0x34,0x4a]
+
+	vftcidb	%v0, %v0, 0
+	vftcidb	%v0, %v0, 4095
+	vftcidb	%v0, %v15, 0
+	vftcidb	%v0, %v31, 0
+	vftcidb	%v15, %v0, 0
+	vftcidb	%v31, %v0, 0
+	vftcidb	%v4, %v21, 0x678
+
+#CHECK: vgbm    %v0, 0                  # encoding: [0xe7,0x00,0x00,0x00,0x00,0x44]
+#CHECK: vgbm    %v0, 65535              # encoding: [0xe7,0x00,0xff,0xff,0x00,0x44]
+#CHECK: vgbm    %v15, 0                 # encoding: [0xe7,0xf0,0x00,0x00,0x00,0x44]
+#CHECK: vgbm    %v31, 0                 # encoding: [0xe7,0xf0,0x00,0x00,0x08,0x44]
+#CHECK: vgbm    %v17, 4660              # encoding: [0xe7,0x10,0x12,0x34,0x08,0x44]
+
+	vgbm	%v0, 0
+	vgbm	%v0, 0xffff
+	vgbm	%v15, 0
+	vgbm	%v31, 0
+	vgbm	%v17, 0x1234
+
+#CHECK: vgef    %v0, 0(%v0), 0          # encoding: [0xe7,0x00,0x00,0x00,0x00,0x13]
+#CHECK: vgef    %v0, 0(%v0,%r1), 0      # encoding: [0xe7,0x00,0x10,0x00,0x00,0x13]
+#CHECK: vgef    %v0, 0(%v0,%r1), 3      # encoding: [0xe7,0x00,0x10,0x00,0x30,0x13]
+#CHECK: vgef    %v0, 0(%v0,%r15), 0     # encoding: [0xe7,0x00,0xf0,0x00,0x00,0x13]
+#CHECK: vgef    %v0, 0(%v15,%r1), 0     # encoding: [0xe7,0x0f,0x10,0x00,0x00,0x13]
+#CHECK: vgef    %v0, 0(%v31,%r1), 0     # encoding: [0xe7,0x0f,0x10,0x00,0x04,0x13]
+#CHECK: vgef    %v0, 4095(%v0,%r1), 0   # encoding: [0xe7,0x00,0x1f,0xff,0x00,0x13]
+#CHECK: vgef    %v15, 0(%v0,%r1), 0     # encoding: [0xe7,0xf0,0x10,0x00,0x00,0x13]
+#CHECK: vgef    %v31, 0(%v0,%r1), 0     # encoding: [0xe7,0xf0,0x10,0x00,0x08,0x13]
+#CHECK: vgef    %v10, 1000(%v19,%r7), 1 # encoding: [0xe7,0xa3,0x73,0xe8,0x14,0x13]
+
+	vgef	%v0, 0(%v0), 0
+	vgef	%v0, 0(%v0,%r1), 0
+	vgef	%v0, 0(%v0,%r1), 3
+	vgef	%v0, 0(%v0,%r15), 0
+	vgef	%v0, 0(%v15,%r1), 0
+	vgef	%v0, 0(%v31,%r1), 0
+	vgef	%v0, 4095(%v0, %r1), 0
+	vgef	%v15, 0(%v0,%r1), 0
+	vgef	%v31, 0(%v0,%r1), 0
+	vgef	%v10, 1000(%v19,%r7), 1
+
+#CHECK: vgeg    %v0, 0(%v0), 0          # encoding: [0xe7,0x00,0x00,0x00,0x00,0x12]
+#CHECK: vgeg    %v0, 0(%v0,%r1), 0      # encoding: [0xe7,0x00,0x10,0x00,0x00,0x12]
+#CHECK: vgeg    %v0, 0(%v0,%r1), 1      # encoding: [0xe7,0x00,0x10,0x00,0x10,0x12]
+#CHECK: vgeg    %v0, 0(%v0,%r15), 0     # encoding: [0xe7,0x00,0xf0,0x00,0x00,0x12]
+#CHECK: vgeg    %v0, 0(%v15,%r1), 0     # encoding: [0xe7,0x0f,0x10,0x00,0x00,0x12]
+#CHECK: vgeg    %v0, 0(%v31,%r1), 0     # encoding: [0xe7,0x0f,0x10,0x00,0x04,0x12]
+#CHECK: vgeg    %v0, 4095(%v0,%r1), 0   # encoding: [0xe7,0x00,0x1f,0xff,0x00,0x12]
+#CHECK: vgeg    %v15, 0(%v0,%r1), 0     # encoding: [0xe7,0xf0,0x10,0x00,0x00,0x12]
+#CHECK: vgeg    %v31, 0(%v0,%r1), 0     # encoding: [0xe7,0xf0,0x10,0x00,0x08,0x12]
+#CHECK: vgeg    %v10, 1000(%v19,%r7), 1 # encoding: [0xe7,0xa3,0x73,0xe8,0x14,0x12]
+
+	vgeg	%v0, 0(%v0), 0
+	vgeg	%v0, 0(%v0,%r1), 0
+	vgeg	%v0, 0(%v0,%r1), 1
+	vgeg	%v0, 0(%v0,%r15), 0
+	vgeg	%v0, 0(%v15,%r1), 0
+	vgeg	%v0, 0(%v31,%r1), 0
+	vgeg	%v0, 4095(%v0,%r1), 0
+	vgeg	%v15, 0(%v0,%r1), 0
+	vgeg	%v31, 0(%v0,%r1), 0
+	vgeg	%v10, 1000(%v19,%r7), 1
+
+#CHECK: vgfmab  %v0, %v0, %v0, %v0      # encoding: [0xe7,0x00,0x00,0x00,0x00,0xbc]
+#CHECK: vgfmab  %v0, %v0, %v0, %v31     # encoding: [0xe7,0x00,0x00,0x00,0xf1,0xbc]
+#CHECK: vgfmab  %v0, %v0, %v31, %v0     # encoding: [0xe7,0x00,0xf0,0x00,0x02,0xbc]
+#CHECK: vgfmab  %v0, %v31, %v0, %v0     # encoding: [0xe7,0x0f,0x00,0x00,0x04,0xbc]
+#CHECK: vgfmab  %v31, %v0, %v0, %v0     # encoding: [0xe7,0xf0,0x00,0x00,0x08,0xbc]
+#CHECK: vgfmab  %v13, %v17, %v21, %v25  # encoding: [0xe7,0xd1,0x50,0x00,0x97,0xbc]
+
+	vgfmab	%v0, %v0, %v0, %v0
+	vgfmab	%v0, %v0, %v0, %v31
+	vgfmab	%v0, %v0, %v31, %v0
+	vgfmab	%v0, %v31, %v0, %v0
+	vgfmab	%v31, %v0, %v0, %v0
+	vgfmab	%v13, %v17, %v21, %v25
+
+#CHECK: vgfmaf  %v0, %v0, %v0, %v0      # encoding: [0xe7,0x00,0x02,0x00,0x00,0xbc]
+#CHECK: vgfmaf  %v0, %v0, %v0, %v31     # encoding: [0xe7,0x00,0x02,0x00,0xf1,0xbc]
+#CHECK: vgfmaf  %v0, %v0, %v31, %v0     # encoding: [0xe7,0x00,0xf2,0x00,0x02,0xbc]
+#CHECK: vgfmaf  %v0, %v31, %v0, %v0     # encoding: [0xe7,0x0f,0x02,0x00,0x04,0xbc]
+#CHECK: vgfmaf  %v31, %v0, %v0, %v0     # encoding: [0xe7,0xf0,0x02,0x00,0x08,0xbc]
+#CHECK: vgfmaf  %v13, %v17, %v21, %v25  # encoding: [0xe7,0xd1,0x52,0x00,0x97,0xbc]
+
+	vgfmaf	%v0, %v0, %v0, %v0
+	vgfmaf	%v0, %v0, %v0, %v31
+	vgfmaf	%v0, %v0, %v31, %v0
+	vgfmaf	%v0, %v31, %v0, %v0
+	vgfmaf	%v31, %v0, %v0, %v0
+	vgfmaf	%v13, %v17, %v21, %v25
+
+#CHECK: vgfmag  %v0, %v0, %v0, %v0      # encoding: [0xe7,0x00,0x03,0x00,0x00,0xbc]
+#CHECK: vgfmag  %v0, %v0, %v0, %v31     # encoding: [0xe7,0x00,0x03,0x00,0xf1,0xbc]
+#CHECK: vgfmag  %v0, %v0, %v31, %v0     # encoding: [0xe7,0x00,0xf3,0x00,0x02,0xbc]
+#CHECK: vgfmag  %v0, %v31, %v0, %v0     # encoding: [0xe7,0x0f,0x03,0x00,0x04,0xbc]
+#CHECK: vgfmag  %v31, %v0, %v0, %v0     # encoding: [0xe7,0xf0,0x03,0x00,0x08,0xbc]
+#CHECK: vgfmag  %v13, %v17, %v21, %v25  # encoding: [0xe7,0xd1,0x53,0x00,0x97,0xbc]
+
+	vgfmag	%v0, %v0, %v0, %v0
+	vgfmag	%v0, %v0, %v0, %v31
+	vgfmag	%v0, %v0, %v31, %v0
+	vgfmag	%v0, %v31, %v0, %v0
+	vgfmag	%v31, %v0, %v0, %v0
+	vgfmag	%v13, %v17, %v21, %v25
+
+#CHECK: vgfmah  %v0, %v0, %v0, %v0      # encoding: [0xe7,0x00,0x01,0x00,0x00,0xbc]
+#CHECK: vgfmah  %v0, %v0, %v0, %v31     # encoding: [0xe7,0x00,0x01,0x00,0xf1,0xbc]
+#CHECK: vgfmah  %v0, %v0, %v31, %v0     # encoding: [0xe7,0x00,0xf1,0x00,0x02,0xbc]
+#CHECK: vgfmah  %v0, %v31, %v0, %v0     # encoding: [0xe7,0x0f,0x01,0x00,0x04,0xbc]
+#CHECK: vgfmah  %v31, %v0, %v0, %v0     # encoding: [0xe7,0xf0,0x01,0x00,0x08,0xbc]
+#CHECK: vgfmah  %v13, %v17, %v21, %v25  # encoding: [0xe7,0xd1,0x51,0x00,0x97,0xbc]
+
+	vgfmah	%v0, %v0, %v0, %v0
+	vgfmah	%v0, %v0, %v0, %v31
+	vgfmah	%v0, %v0, %v31, %v0
+	vgfmah	%v0, %v31, %v0, %v0
+	vgfmah	%v31, %v0, %v0, %v0
+	vgfmah	%v13, %v17, %v21, %v25
+
+#CHECK: vgfmb   %v0, %v0, %v0           # encoding: [0xe7,0x00,0x00,0x00,0x00,0xb4]
+#CHECK: vgfmb   %v0, %v0, %v31          # encoding: [0xe7,0x00,0xf0,0x00,0x02,0xb4]
+#CHECK: vgfmb   %v0, %v31, %v0          # encoding: [0xe7,0x0f,0x00,0x00,0x04,0xb4]
+#CHECK: vgfmb   %v31, %v0, %v0          # encoding: [0xe7,0xf0,0x00,0x00,0x08,0xb4]
+#CHECK: vgfmb   %v18, %v3, %v20         # encoding: [0xe7,0x23,0x40,0x00,0x0a,0xb4]
+
+	vgfmb	%v0, %v0, %v0
+	vgfmb	%v0, %v0, %v31
+	vgfmb	%v0, %v31, %v0
+	vgfmb	%v31, %v0, %v0
+	vgfmb	%v18, %v3, %v20
+
+#CHECK: vgfmf   %v0, %v0, %v0           # encoding: [0xe7,0x00,0x00,0x00,0x20,0xb4]
+#CHECK: vgfmf   %v0, %v0, %v31          # encoding: [0xe7,0x00,0xf0,0x00,0x22,0xb4]
+#CHECK: vgfmf   %v0, %v31, %v0          # encoding: [0xe7,0x0f,0x00,0x00,0x24,0xb4]
+#CHECK: vgfmf   %v31, %v0, %v0          # encoding: [0xe7,0xf0,0x00,0x00,0x28,0xb4]
+#CHECK: vgfmf   %v18, %v3, %v20         # encoding: [0xe7,0x23,0x40,0x00,0x2a,0xb4]
+
+	vgfmf	%v0, %v0, %v0
+	vgfmf	%v0, %v0, %v31
+	vgfmf	%v0, %v31, %v0
+	vgfmf	%v31, %v0, %v0
+	vgfmf	%v18, %v3, %v20
+
+#CHECK: vgfmg   %v0, %v0, %v0           # encoding: [0xe7,0x00,0x00,0x00,0x30,0xb4]
+#CHECK: vgfmg   %v0, %v0, %v31          # encoding: [0xe7,0x00,0xf0,0x00,0x32,0xb4]
+#CHECK: vgfmg   %v0, %v31, %v0          # encoding: [0xe7,0x0f,0x00,0x00,0x34,0xb4]
+#CHECK: vgfmg   %v31, %v0, %v0          # encoding: [0xe7,0xf0,0x00,0x00,0x38,0xb4]
+#CHECK: vgfmg   %v18, %v3, %v20         # encoding: [0xe7,0x23,0x40,0x00,0x3a,0xb4]
+
+	vgfmg	%v0, %v0, %v0
+	vgfmg	%v0, %v0, %v31
+	vgfmg	%v0, %v31, %v0
+	vgfmg	%v31, %v0, %v0
+	vgfmg	%v18, %v3, %v20
+
+#CHECK: vgfmh   %v0, %v0, %v0           # encoding: [0xe7,0x00,0x00,0x00,0x10,0xb4]
+#CHECK: vgfmh   %v0, %v0, %v31          # encoding: [0xe7,0x00,0xf0,0x00,0x12,0xb4]
+#CHECK: vgfmh   %v0, %v31, %v0          # encoding: [0xe7,0x0f,0x00,0x00,0x14,0xb4]
+#CHECK: vgfmh   %v31, %v0, %v0          # encoding: [0xe7,0xf0,0x00,0x00,0x18,0xb4]
+#CHECK: vgfmh   %v18, %v3, %v20         # encoding: [0xe7,0x23,0x40,0x00,0x1a,0xb4]
+
+	vgfmh	%v0, %v0, %v0
+	vgfmh	%v0, %v0, %v31
+	vgfmh	%v0, %v31, %v0
+	vgfmh	%v31, %v0, %v0
+	vgfmh	%v18, %v3, %v20
+
+#CHECK: vgmb    %v0, 0, 0               # encoding: [0xe7,0x00,0x00,0x00,0x00,0x46]
+#CHECK: vgmb    %v0, 0, 255             # encoding: [0xe7,0x00,0x00,0xff,0x00,0x46]
+#CHECK: vgmb    %v0, 255, 0             # encoding: [0xe7,0x00,0xff,0x00,0x00,0x46]
+#CHECK: vgmb    %v15, 0, 0              # encoding: [0xe7,0xf0,0x00,0x00,0x00,0x46]
+#CHECK: vgmb    %v31, 0, 0              # encoding: [0xe7,0xf0,0x00,0x00,0x08,0x46]
+#CHECK: vgmb    %v21, 2, 3              # encoding: [0xe7,0x50,0x02,0x03,0x08,0x46]
+
+	vgmb	%v0, 0, 0
+	vgmb	%v0, 0, 255
+	vgmb	%v0, 255, 0
+	vgmb	%v15, 0, 0
+	vgmb	%v31, 0, 0
+	vgmb	%v21, 2, 3
+
+#CHECK: vgmf    %v0, 0, 0               # encoding: [0xe7,0x00,0x00,0x00,0x20,0x46]
+#CHECK: vgmf    %v0, 0, 255             # encoding: [0xe7,0x00,0x00,0xff,0x20,0x46]
+#CHECK: vgmf    %v0, 255, 0             # encoding: [0xe7,0x00,0xff,0x00,0x20,0x46]
+#CHECK: vgmf    %v15, 0, 0              # encoding: [0xe7,0xf0,0x00,0x00,0x20,0x46]
+#CHECK: vgmf    %v31, 0, 0              # encoding: [0xe7,0xf0,0x00,0x00,0x28,0x46]
+#CHECK: vgmf    %v21, 2, 3              # encoding: [0xe7,0x50,0x02,0x03,0x28,0x46]
+
+	vgmf	%v0, 0, 0
+	vgmf	%v0, 0, 255
+	vgmf	%v0, 255, 0
+	vgmf	%v15, 0, 0
+	vgmf	%v31, 0, 0
+	vgmf	%v21, 2, 3
+
+#CHECK: vgmg    %v0, 0, 0               # encoding: [0xe7,0x00,0x00,0x00,0x30,0x46]
+#CHECK: vgmg    %v0, 0, 255             # encoding: [0xe7,0x00,0x00,0xff,0x30,0x46]
+#CHECK: vgmg    %v0, 255, 0             # encoding: [0xe7,0x00,0xff,0x00,0x30,0x46]
+#CHECK: vgmg    %v15, 0, 0              # encoding: [0xe7,0xf0,0x00,0x00,0x30,0x46]
+#CHECK: vgmg    %v31, 0, 0              # encoding: [0xe7,0xf0,0x00,0x00,0x38,0x46]
+#CHECK: vgmg    %v21, 2, 3              # encoding: [0xe7,0x50,0x02,0x03,0x38,0x46]
+
+	vgmg	%v0, 0, 0
+	vgmg	%v0, 0, 255
+	vgmg	%v0, 255, 0
+	vgmg	%v15, 0, 0
+	vgmg	%v31, 0, 0
+	vgmg	%v21, 2, 3
+
+#CHECK: vgmh    %v0, 0, 0               # encoding: [0xe7,0x00,0x00,0x00,0x10,0x46]
+#CHECK: vgmh    %v0, 0, 255             # encoding: [0xe7,0x00,0x00,0xff,0x10,0x46]
+#CHECK: vgmh    %v0, 255, 0             # encoding: [0xe7,0x00,0xff,0x00,0x10,0x46]
+#CHECK: vgmh    %v15, 0, 0              # encoding: [0xe7,0xf0,0x00,0x00,0x10,0x46]
+#CHECK: vgmh    %v31, 0, 0              # encoding: [0xe7,0xf0,0x00,0x00,0x18,0x46]
+#CHECK: vgmh    %v21, 2, 3              # encoding: [0xe7,0x50,0x02,0x03,0x18,0x46]
+
+	vgmh	%v0, 0, 0
+	vgmh	%v0, 0, 255
+	vgmh	%v0, 255, 0
+	vgmh	%v15, 0, 0
+	vgmh	%v31, 0, 0
+	vgmh	%v21, 2, 3
+
+#CHECK: vl      %v0, 0                  # encoding: [0xe7,0x00,0x00,0x00,0x00,0x06]
+#CHECK: vl      %v0, 4095               # encoding: [0xe7,0x00,0x0f,0xff,0x00,0x06]
+#CHECK: vl      %v0, 0(%r15)            # encoding: [0xe7,0x00,0xf0,0x00,0x00,0x06]
+#CHECK: vl      %v0, 0(%r15,%r1)        # encoding: [0xe7,0x0f,0x10,0x00,0x00,0x06]
+#CHECK: vl      %v15, 0                 # encoding: [0xe7,0xf0,0x00,0x00,0x00,0x06]
+#CHECK: vl      %v31, 0                 # encoding: [0xe7,0xf0,0x00,0x00,0x08,0x06]
+#CHECK: vl      %v18, 1383(%r3,%r4)     # encoding: [0xe7,0x23,0x45,0x67,0x08,0x06]
+
+	vl	%v0, 0
+	vl	%v0, 4095
+	vl	%v0, 0(%r15)
+	vl	%v0, 0(%r15,%r1)
+	vl	%v15, 0
+	vl	%v31, 0
+	vl	%v18, 0x567(%r3,%r4)
+
+#CHECK: vlbb    %v0, 0, 0               # encoding: [0xe7,0x00,0x00,0x00,0x00,0x07]
+#CHECK: vlbb    %v0, 0, 15              # encoding: [0xe7,0x00,0x00,0x00,0xf0,0x07]
+#CHECK: vlbb    %v0, 4095, 0            # encoding: [0xe7,0x00,0x0f,0xff,0x00,0x07]
+#CHECK: vlbb    %v0, 0(%r15), 0         # encoding: [0xe7,0x00,0xf0,0x00,0x00,0x07]
+#CHECK: vlbb    %v0, 0(%r15,%r1), 0     # encoding: [0xe7,0x0f,0x10,0x00,0x00,0x07]
+#CHECK: vlbb    %v15, 0, 0              # encoding: [0xe7,0xf0,0x00,0x00,0x00,0x07]
+#CHECK: vlbb    %v31, 0, 0              # encoding: [0xe7,0xf0,0x00,0x00,0x08,0x07]
+#CHECK: vlbb    %v18, 1383(%r3,%r4), 8  # encoding: [0xe7,0x23,0x45,0x67,0x88,0x07]
+
+	vlbb	%v0, 0, 0
+	vlbb	%v0, 0, 15
+	vlbb	%v0, 4095, 0
+	vlbb	%v0, 0(%r15), 0
+	vlbb	%v0, 0(%r15,%r1), 0
+	vlbb	%v15, 0, 0
+	vlbb	%v31, 0, 0
+	vlbb	%v18, 1383(%r3,%r4), 8
+
+#CHECK: vlcb    %v0, %v0                # encoding: [0xe7,0x00,0x00,0x00,0x00,0xde]
+#CHECK: vlcb    %v0, %v15               # encoding: [0xe7,0x0f,0x00,0x00,0x00,0xde]
+#CHECK: vlcb    %v0, %v31               # encoding: [0xe7,0x0f,0x00,0x00,0x04,0xde]
+#CHECK: vlcb    %v15, %v0               # encoding: [0xe7,0xf0,0x00,0x00,0x00,0xde]
+#CHECK: vlcb    %v31, %v0               # encoding: [0xe7,0xf0,0x00,0x00,0x08,0xde]
+#CHECK: vlcb    %v14, %v17              # encoding: [0xe7,0xe1,0x00,0x00,0x04,0xde]
+
+	vlcb	%v0, %v0
+	vlcb	%v0, %v15
+	vlcb	%v0, %v31
+	vlcb	%v15, %v0
+	vlcb	%v31, %v0
+	vlcb	%v14, %v17
+
+#CHECK: vlcf    %v0, %v0                # encoding: [0xe7,0x00,0x00,0x00,0x20,0xde]
+#CHECK: vlcf    %v0, %v15               # encoding: [0xe7,0x0f,0x00,0x00,0x20,0xde]
+#CHECK: vlcf    %v0, %v31               # encoding: [0xe7,0x0f,0x00,0x00,0x24,0xde]
+#CHECK: vlcf    %v15, %v0               # encoding: [0xe7,0xf0,0x00,0x00,0x20,0xde]
+#CHECK: vlcf    %v31, %v0               # encoding: [0xe7,0xf0,0x00,0x00,0x28,0xde]
+#CHECK: vlcf    %v14, %v17              # encoding: [0xe7,0xe1,0x00,0x00,0x24,0xde]
+
+	vlcf	%v0, %v0
+	vlcf	%v0, %v15
+	vlcf	%v0, %v31
+	vlcf	%v15, %v0
+	vlcf	%v31, %v0
+	vlcf	%v14, %v17
+
+#CHECK: vlcg    %v0, %v0                # encoding: [0xe7,0x00,0x00,0x00,0x30,0xde]
+#CHECK: vlcg    %v0, %v15               # encoding: [0xe7,0x0f,0x00,0x00,0x30,0xde]
+#CHECK: vlcg    %v0, %v31               # encoding: [0xe7,0x0f,0x00,0x00,0x34,0xde]
+#CHECK: vlcg    %v15, %v0               # encoding: [0xe7,0xf0,0x00,0x00,0x30,0xde]
+#CHECK: vlcg    %v31, %v0               # encoding: [0xe7,0xf0,0x00,0x00,0x38,0xde]
+#CHECK: vlcg    %v14, %v17              # encoding: [0xe7,0xe1,0x00,0x00,0x34,0xde]
+
+	vlcg	%v0, %v0
+	vlcg	%v0, %v15
+	vlcg	%v0, %v31
+	vlcg	%v15, %v0
+	vlcg	%v31, %v0
+	vlcg	%v14, %v17
+
+#CHECK: vlch    %v0, %v0                # encoding: [0xe7,0x00,0x00,0x00,0x10,0xde]
+#CHECK: vlch    %v0, %v15               # encoding: [0xe7,0x0f,0x00,0x00,0x10,0xde]
+#CHECK: vlch    %v0, %v31               # encoding: [0xe7,0x0f,0x00,0x00,0x14,0xde]
+#CHECK: vlch    %v15, %v0               # encoding: [0xe7,0xf0,0x00,0x00,0x10,0xde]
+#CHECK: vlch    %v31, %v0               # encoding: [0xe7,0xf0,0x00,0x00,0x18,0xde]
+#CHECK: vlch    %v14, %v17              # encoding: [0xe7,0xe1,0x00,0x00,0x14,0xde]
+
+	vlch	%v0, %v0
+	vlch	%v0, %v15
+	vlch	%v0, %v31
+	vlch	%v15, %v0
+	vlch	%v31, %v0
+	vlch	%v14, %v17
+
+#CHECK: vldeb   %v0, %v0                # encoding: [0xe7,0x00,0x00,0x00,0x20,0xc4]
+#CHECK: vldeb   %v0, %v15               # encoding: [0xe7,0x0f,0x00,0x00,0x20,0xc4]
+#CHECK: vldeb   %v0, %v31               # encoding: [0xe7,0x0f,0x00,0x00,0x24,0xc4]
+#CHECK: vldeb   %v15, %v0               # encoding: [0xe7,0xf0,0x00,0x00,0x20,0xc4]
+#CHECK: vldeb   %v31, %v0               # encoding: [0xe7,0xf0,0x00,0x00,0x28,0xc4]
+#CHECK: vldeb   %v14, %v17              # encoding: [0xe7,0xe1,0x00,0x00,0x24,0xc4]
+
+	vldeb	%v0, %v0
+	vldeb	%v0, %v15
+	vldeb	%v0, %v31
+	vldeb	%v15, %v0
+	vldeb	%v31, %v0
+	vldeb	%v14, %v17
+
+#CHECK: vleb    %v0, 0, 0               # encoding: [0xe7,0x00,0x00,0x00,0x00,0x00]
+#CHECK: vleb    %v0, 0, 15              # encoding: [0xe7,0x00,0x00,0x00,0xf0,0x00]
+#CHECK: vleb    %v0, 4095, 0            # encoding: [0xe7,0x00,0x0f,0xff,0x00,0x00]
+#CHECK: vleb    %v0, 0(%r15), 0         # encoding: [0xe7,0x00,0xf0,0x00,0x00,0x00]
+#CHECK: vleb    %v0, 0(%r15,%r1), 0     # encoding: [0xe7,0x0f,0x10,0x00,0x00,0x00]
+#CHECK: vleb    %v15, 0, 0              # encoding: [0xe7,0xf0,0x00,0x00,0x00,0x00]
+#CHECK: vleb    %v31, 0, 0              # encoding: [0xe7,0xf0,0x00,0x00,0x08,0x00]
+#CHECK: vleb    %v18, 1383(%r3,%r4), 8  # encoding: [0xe7,0x23,0x45,0x67,0x88,0x00]
+
+	vleb	%v0, 0, 0
+	vleb	%v0, 0, 15
+	vleb	%v0, 4095, 0
+	vleb	%v0, 0(%r15), 0
+	vleb	%v0, 0(%r15,%r1), 0
+	vleb	%v15, 0, 0
+	vleb	%v31, 0, 0
+	vleb	%v18, 1383(%r3,%r4), 8
+
+#CHECK: vledb   %v0, %v0, 0, 0          # encoding: [0xe7,0x00,0x00,0x00,0x30,0xc5]
+#CHECK: vledb   %v0, %v0, 0, 15         # encoding: [0xe7,0x00,0x00,0xf0,0x30,0xc5]
+#CHECK: vledb   %v0, %v0, 4, 0          # encoding: [0xe7,0x00,0x00,0x04,0x30,0xc5]
+#CHECK: vledb   %v0, %v0, 12, 0         # encoding: [0xe7,0x00,0x00,0x0c,0x30,0xc5]
+#CHECK: vledb   %v0, %v31, 0, 0         # encoding: [0xe7,0x0f,0x00,0x00,0x34,0xc5]
+#CHECK: vledb   %v31, %v0, 0, 0         # encoding: [0xe7,0xf0,0x00,0x00,0x38,0xc5]
+#CHECK: vledb   %v14, %v17, 4, 10       # encoding: [0xe7,0xe1,0x00,0xa4,0x34,0xc5]
+
+	vledb	%v0, %v0, 0, 0
+	vledb	%v0, %v0, 0, 15
+	vledb	%v0, %v0, 4, 0
+	vledb	%v0, %v0, 12, 0
+	vledb	%v0, %v31, 0, 0
+	vledb	%v31, %v0, 0, 0
+	vledb	%v14, %v17, 4, 10
+
+#CHECK: vlef    %v0, 0, 0               # encoding: [0xe7,0x00,0x00,0x00,0x00,0x03]
+#CHECK: vlef    %v0, 0, 3               # encoding: [0xe7,0x00,0x00,0x00,0x30,0x03]
+#CHECK: vlef    %v0, 4095, 0            # encoding: [0xe7,0x00,0x0f,0xff,0x00,0x03]
+#CHECK: vlef    %v0, 0(%r15), 0         # encoding: [0xe7,0x00,0xf0,0x00,0x00,0x03]
+#CHECK: vlef    %v0, 0(%r15,%r1), 0     # encoding: [0xe7,0x0f,0x10,0x00,0x00,0x03]
+#CHECK: vlef    %v15, 0, 0              # encoding: [0xe7,0xf0,0x00,0x00,0x00,0x03]
+#CHECK: vlef    %v31, 0, 0              # encoding: [0xe7,0xf0,0x00,0x00,0x08,0x03]
+#CHECK: vlef    %v18, 1383(%r3,%r4), 2  # encoding: [0xe7,0x23,0x45,0x67,0x28,0x03]
+
+	vlef	%v0, 0, 0
+	vlef	%v0, 0, 3
+	vlef	%v0, 4095, 0
+	vlef	%v0, 0(%r15), 0
+	vlef	%v0, 0(%r15,%r1), 0
+	vlef	%v15, 0, 0
+	vlef	%v31, 0, 0
+	vlef	%v18, 1383(%r3,%r4), 2
+
+#CHECK: vleg    %v0, 0, 0               # encoding: [0xe7,0x00,0x00,0x00,0x00,0x02]
+#CHECK: vleg    %v0, 0, 1               # encoding: [0xe7,0x00,0x00,0x00,0x10,0x02]
+#CHECK: vleg    %v0, 4095, 0            # encoding: [0xe7,0x00,0x0f,0xff,0x00,0x02]
+#CHECK: vleg    %v0, 0(%r15), 0         # encoding: [0xe7,0x00,0xf0,0x00,0x00,0x02]
+#CHECK: vleg    %v0, 0(%r15,%r1), 0     # encoding: [0xe7,0x0f,0x10,0x00,0x00,0x02]
+#CHECK: vleg    %v15, 0, 0              # encoding: [0xe7,0xf0,0x00,0x00,0x00,0x02]
+#CHECK: vleg    %v31, 0, 0              # encoding: [0xe7,0xf0,0x00,0x00,0x08,0x02]
+#CHECK: vleg    %v18, 1383(%r3,%r4), 1  # encoding: [0xe7,0x23,0x45,0x67,0x18,0x02]
+
+	vleg	%v0, 0, 0
+	vleg	%v0, 0, 1
+	vleg	%v0, 4095, 0
+	vleg	%v0, 0(%r15), 0
+	vleg	%v0, 0(%r15,%r1), 0
+	vleg	%v15, 0, 0
+	vleg	%v31, 0, 0
+	vleg	%v18, 1383(%r3,%r4), 1
+
+#CHECK: vleh    %v0, 0, 0               # encoding: [0xe7,0x00,0x00,0x00,0x00,0x01]
+#CHECK: vleh    %v0, 0, 7               # encoding: [0xe7,0x00,0x00,0x00,0x70,0x01]
+#CHECK: vleh    %v0, 4095, 0            # encoding: [0xe7,0x00,0x0f,0xff,0x00,0x01]
+#CHECK: vleh    %v0, 0(%r15), 0         # encoding: [0xe7,0x00,0xf0,0x00,0x00,0x01]
+#CHECK: vleh    %v0, 0(%r15,%r1), 0     # encoding: [0xe7,0x0f,0x10,0x00,0x00,0x01]
+#CHECK: vleh    %v15, 0, 0              # encoding: [0xe7,0xf0,0x00,0x00,0x00,0x01]
+#CHECK: vleh    %v31, 0, 0              # encoding: [0xe7,0xf0,0x00,0x00,0x08,0x01]
+#CHECK: vleh    %v18, 1383(%r3,%r4), 4  # encoding: [0xe7,0x23,0x45,0x67,0x48,0x01]
+
+	vleh	%v0, 0, 0
+	vleh	%v0, 0, 7
+	vleh	%v0, 4095, 0
+	vleh	%v0, 0(%r15), 0
+	vleh	%v0, 0(%r15,%r1), 0
+	vleh	%v15, 0, 0
+	vleh	%v31, 0, 0
+	vleh	%v18, 1383(%r3,%r4), 4
+
+#CHECK: vleib   %v0, 0, 0               # encoding: [0xe7,0x00,0x00,0x00,0x00,0x40]
+#CHECK: vleib   %v0, 0, 15              # encoding: [0xe7,0x00,0x00,0x00,0xf0,0x40]
+#CHECK: vleib   %v0, -32768, 0          # encoding: [0xe7,0x00,0x80,0x00,0x00,0x40]
+#CHECK: vleib   %v0, 32767, 0           # encoding: [0xe7,0x00,0x7f,0xff,0x00,0x40]
+#CHECK: vleib   %v15, 0, 0              # encoding: [0xe7,0xf0,0x00,0x00,0x00,0x40]
+#CHECK: vleib   %v31, 0, 0              # encoding: [0xe7,0xf0,0x00,0x00,0x08,0x40]
+#CHECK: vleib   %v18, 13398, 11         # encoding: [0xe7,0x20,0x34,0x56,0xb8,0x40]
+
+	vleib	%v0, 0, 0
+	vleib	%v0, 0, 15
+	vleib	%v0, -32768, 0
+	vleib	%v0, 32767, 0
+	vleib	%v15, 0, 0
+	vleib	%v31, 0, 0
+	vleib	%v18, 0x3456, 11
+
+#CHECK: vleif   %v0, 0, 0               # encoding: [0xe7,0x00,0x00,0x00,0x00,0x43]
+#CHECK: vleif   %v0, 0, 3               # encoding: [0xe7,0x00,0x00,0x00,0x30,0x43]
+#CHECK: vleif   %v0, -32768, 0          # encoding: [0xe7,0x00,0x80,0x00,0x00,0x43]
+#CHECK: vleif   %v0, 32767, 0           # encoding: [0xe7,0x00,0x7f,0xff,0x00,0x43]
+#CHECK: vleif   %v15, 0, 0              # encoding: [0xe7,0xf0,0x00,0x00,0x00,0x43]
+#CHECK: vleif   %v31, 0, 0              # encoding: [0xe7,0xf0,0x00,0x00,0x08,0x43]
+#CHECK: vleif   %v18, 13398, 3          # encoding: [0xe7,0x20,0x34,0x56,0x38,0x43]
+
+	vleif	%v0, 0, 0
+	vleif	%v0, 0, 3
+	vleif	%v0, -32768, 0
+	vleif	%v0, 32767, 0
+	vleif	%v15, 0, 0
+	vleif	%v31, 0, 0
+	vleif	%v18, 0x3456, 3
+
+#CHECK: vleig   %v0, 0, 0               # encoding: [0xe7,0x00,0x00,0x00,0x00,0x42]
+#CHECK: vleig   %v0, 0, 1               # encoding: [0xe7,0x00,0x00,0x00,0x10,0x42]
+#CHECK: vleig   %v0, -32768, 0          # encoding: [0xe7,0x00,0x80,0x00,0x00,0x42]
+#CHECK: vleig   %v0, 32767, 0           # encoding: [0xe7,0x00,0x7f,0xff,0x00,0x42]
+#CHECK: vleig   %v15, 0, 0              # encoding: [0xe7,0xf0,0x00,0x00,0x00,0x42]
+#CHECK: vleig   %v31, 0, 0              # encoding: [0xe7,0xf0,0x00,0x00,0x08,0x42]
+#CHECK: vleig   %v18, 13398, 1          # encoding: [0xe7,0x20,0x34,0x56,0x18,0x42]
+
+	vleig	%v0, 0, 0
+	vleig	%v0, 0, 1
+	vleig	%v0, -32768, 0
+	vleig	%v0, 32767, 0
+	vleig	%v15, 0, 0
+	vleig	%v31, 0, 0
+	vleig	%v18, 0x3456, 1
+
+#CHECK: vleih   %v0, 0, 0               # encoding: [0xe7,0x00,0x00,0x00,0x00,0x41]
+#CHECK: vleih   %v0, 0, 7               # encoding: [0xe7,0x00,0x00,0x00,0x70,0x41]
+#CHECK: vleih   %v0, -32768, 0          # encoding: [0xe7,0x00,0x80,0x00,0x00,0x41]
+#CHECK: vleih   %v0, 32767, 0           # encoding: [0xe7,0x00,0x7f,0xff,0x00,0x41]
+#CHECK: vleih   %v15, 0, 0              # encoding: [0xe7,0xf0,0x00,0x00,0x00,0x41]
+#CHECK: vleih   %v31, 0, 0              # encoding: [0xe7,0xf0,0x00,0x00,0x08,0x41]
+#CHECK: vleih   %v18, 13398, 7          # encoding: [0xe7,0x20,0x34,0x56,0x78,0x41]
+
+	vleih	%v0, 0, 0
+	vleih	%v0, 0, 7
+	vleih	%v0, -32768, 0
+	vleih	%v0, 32767, 0
+	vleih	%v15, 0, 0
+	vleih	%v31, 0, 0
+	vleih	%v18, 0x3456, 7
+
+#CHECK: vlgvb   %r0, %v0, 0             # encoding: [0xe7,0x00,0x00,0x00,0x00,0x21]
+#CHECK: vlgvb   %r0, %v0, 4095          # encoding: [0xe7,0x00,0x0f,0xff,0x00,0x21]
+#CHECK: vlgvb   %r0, %v0, 0(%r15)       # encoding: [0xe7,0x00,0xf0,0x00,0x00,0x21]
+#CHECK: vlgvb   %r0, %v15, 0            # encoding: [0xe7,0x0f,0x00,0x00,0x00,0x21]
+#CHECK: vlgvb   %r0, %v31, 0            # encoding: [0xe7,0x0f,0x00,0x00,0x04,0x21]
+#CHECK: vlgvb   %r15, %v0, 0            # encoding: [0xe7,0xf0,0x00,0x00,0x00,0x21]
+#CHECK: vlgvb   %r2, %v19, 1383(%r4)    # encoding: [0xe7,0x23,0x45,0x67,0x04,0x21]
+
+	vlgvb	%r0, %v0, 0
+	vlgvb	%r0, %v0, 4095
+	vlgvb	%r0, %v0, 0(%r15)
+	vlgvb	%r0, %v15, 0
+	vlgvb	%r0, %v31, 0
+	vlgvb	%r15, %v0, 0
+	vlgvb	%r2, %v19, 1383(%r4)
+
+#CHECK: vlgvf   %r0, %v0, 0             # encoding: [0xe7,0x00,0x00,0x00,0x20,0x21]
+#CHECK: vlgvf   %r0, %v0, 4095          # encoding: [0xe7,0x00,0x0f,0xff,0x20,0x21]
+#CHECK: vlgvf   %r0, %v0, 0(%r15)       # encoding: [0xe7,0x00,0xf0,0x00,0x20,0x21]
+#CHECK: vlgvf   %r0, %v15, 0            # encoding: [0xe7,0x0f,0x00,0x00,0x20,0x21]
+#CHECK: vlgvf   %r0, %v31, 0            # encoding: [0xe7,0x0f,0x00,0x00,0x24,0x21]
+#CHECK: vlgvf   %r15, %v0, 0            # encoding: [0xe7,0xf0,0x00,0x00,0x20,0x21]
+#CHECK: vlgvf   %r2, %v19, 1383(%r4)    # encoding: [0xe7,0x23,0x45,0x67,0x24,0x21]
+
+	vlgvf	%r0, %v0, 0
+	vlgvf	%r0, %v0, 4095
+	vlgvf	%r0, %v0, 0(%r15)
+	vlgvf	%r0, %v15, 0
+	vlgvf	%r0, %v31, 0
+	vlgvf	%r15, %v0, 0
+	vlgvf	%r2, %v19, 1383(%r4)
+
+#CHECK: vlgvg   %r0, %v0, 0             # encoding: [0xe7,0x00,0x00,0x00,0x30,0x21]
+#CHECK: vlgvg   %r0, %v0, 4095          # encoding: [0xe7,0x00,0x0f,0xff,0x30,0x21]
+#CHECK: vlgvg   %r0, %v0, 0(%r15)       # encoding: [0xe7,0x00,0xf0,0x00,0x30,0x21]
+#CHECK: vlgvg   %r0, %v15, 0            # encoding: [0xe7,0x0f,0x00,0x00,0x30,0x21]
+#CHECK: vlgvg   %r0, %v31, 0            # encoding: [0xe7,0x0f,0x00,0x00,0x34,0x21]
+#CHECK: vlgvg   %r15, %v0, 0            # encoding: [0xe7,0xf0,0x00,0x00,0x30,0x21]
+#CHECK: vlgvg   %r2, %v19, 1383(%r4)    # encoding: [0xe7,0x23,0x45,0x67,0x34,0x21]
+
+	vlgvg	%r0, %v0, 0
+	vlgvg	%r0, %v0, 4095
+	vlgvg	%r0, %v0, 0(%r15)
+	vlgvg	%r0, %v15, 0
+	vlgvg	%r0, %v31, 0
+	vlgvg	%r15, %v0, 0
+	vlgvg	%r2, %v19, 1383(%r4)
+
+#CHECK: vlgvh   %r0, %v0, 0             # encoding: [0xe7,0x00,0x00,0x00,0x10,0x21]
+#CHECK: vlgvh   %r0, %v0, 4095          # encoding: [0xe7,0x00,0x0f,0xff,0x10,0x21]
+#CHECK: vlgvh   %r0, %v0, 0(%r15)       # encoding: [0xe7,0x00,0xf0,0x00,0x10,0x21]
+#CHECK: vlgvh   %r0, %v15, 0            # encoding: [0xe7,0x0f,0x00,0x00,0x10,0x21]
+#CHECK: vlgvh   %r0, %v31, 0            # encoding: [0xe7,0x0f,0x00,0x00,0x14,0x21]
+#CHECK: vlgvh   %r15, %v0, 0            # encoding: [0xe7,0xf0,0x00,0x00,0x10,0x21]
+#CHECK: vlgvh   %r2, %v19, 1383(%r4)    # encoding: [0xe7,0x23,0x45,0x67,0x14,0x21]
+
+	vlgvh	%r0, %v0, 0
+	vlgvh	%r0, %v0, 4095
+	vlgvh	%r0, %v0, 0(%r15)
+	vlgvh	%r0, %v15, 0
+	vlgvh	%r0, %v31, 0
+	vlgvh	%r15, %v0, 0
+	vlgvh	%r2, %v19, 1383(%r4)
+
+#CHECK: vll     %v0, %r0, 0             # encoding: [0xe7,0x00,0x00,0x00,0x00,0x37]
+#CHECK: vll     %v0, %r0, 4095          # encoding: [0xe7,0x00,0x0f,0xff,0x00,0x37]
+#CHECK: vll     %v0, %r0, 0(%r15)       # encoding: [0xe7,0x00,0xf0,0x00,0x00,0x37]
+#CHECK: vll     %v0, %r15, 0            # encoding: [0xe7,0x0f,0x00,0x00,0x00,0x37]
+#CHECK: vll     %v15, %r0, 0            # encoding: [0xe7,0xf0,0x00,0x00,0x00,0x37]
+#CHECK: vll     %v31, %r0, 0            # encoding: [0xe7,0xf0,0x00,0x00,0x08,0x37]
+#CHECK: vll     %v18, %r3, 1383(%r4)    # encoding: [0xe7,0x23,0x45,0x67,0x08,0x37]
+
+	vll	%v0, %r0, 0
+	vll	%v0, %r0, 4095
+	vll	%v0, %r0, 0(%r15)
+	vll	%v0, %r15, 0
+	vll	%v15, %r0, 0
+	vll	%v31, %r0, 0
+	vll	%v18, %r3, 1383(%r4)
+
+#CHECK: vllezb  %v0, 0                  # encoding: [0xe7,0x00,0x00,0x00,0x00,0x04]
+#CHECK: vllezb  %v0, 4095               # encoding: [0xe7,0x00,0x0f,0xff,0x00,0x04]
+#CHECK: vllezb  %v0, 0(%r15)            # encoding: [0xe7,0x00,0xf0,0x00,0x00,0x04]
+#CHECK: vllezb  %v0, 0(%r15,%r1)        # encoding: [0xe7,0x0f,0x10,0x00,0x00,0x04]
+#CHECK: vllezb  %v15, 0                 # encoding: [0xe7,0xf0,0x00,0x00,0x00,0x04]
+#CHECK: vllezb  %v31, 0                 # encoding: [0xe7,0xf0,0x00,0x00,0x08,0x04]
+#CHECK: vllezb  %v18, 1383(%r3,%r4)     # encoding: [0xe7,0x23,0x45,0x67,0x08,0x04]
+
+	vllezb	%v0, 0
+	vllezb	%v0, 4095
+	vllezb	%v0, 0(%r15)
+	vllezb	%v0, 0(%r15,%r1)
+	vllezb	%v15, 0
+	vllezb	%v31, 0
+	vllezb	%v18, 0x567(%r3,%r4)
+
+#CHECK: vllezf  %v0, 0                  # encoding: [0xe7,0x00,0x00,0x00,0x20,0x04]
+#CHECK: vllezf  %v0, 4095               # encoding: [0xe7,0x00,0x0f,0xff,0x20,0x04]
+#CHECK: vllezf  %v0, 0(%r15)            # encoding: [0xe7,0x00,0xf0,0x00,0x20,0x04]
+#CHECK: vllezf  %v0, 0(%r15,%r1)        # encoding: [0xe7,0x0f,0x10,0x00,0x20,0x04]
+#CHECK: vllezf  %v15, 0                 # encoding: [0xe7,0xf0,0x00,0x00,0x20,0x04]
+#CHECK: vllezf  %v31, 0                 # encoding: [0xe7,0xf0,0x00,0x00,0x28,0x04]
+#CHECK: vllezf  %v18, 1383(%r3,%r4)     # encoding: [0xe7,0x23,0x45,0x67,0x28,0x04]
+
+	vllezf	%v0, 0
+	vllezf	%v0, 4095
+	vllezf	%v0, 0(%r15)
+	vllezf	%v0, 0(%r15,%r1)
+	vllezf	%v15, 0
+	vllezf	%v31, 0
+	vllezf	%v18, 0x567(%r3,%r4)
+
+#CHECK: vllezg  %v0, 0                  # encoding: [0xe7,0x00,0x00,0x00,0x30,0x04]
+#CHECK: vllezg  %v0, 4095               # encoding: [0xe7,0x00,0x0f,0xff,0x30,0x04]
+#CHECK: vllezg  %v0, 0(%r15)            # encoding: [0xe7,0x00,0xf0,0x00,0x30,0x04]
+#CHECK: vllezg  %v0, 0(%r15,%r1)        # encoding: [0xe7,0x0f,0x10,0x00,0x30,0x04]
+#CHECK: vllezg  %v15, 0                 # encoding: [0xe7,0xf0,0x00,0x00,0x30,0x04]
+#CHECK: vllezg  %v31, 0                 # encoding: [0xe7,0xf0,0x00,0x00,0x38,0x04]
+#CHECK: vllezg  %v18, 1383(%r3,%r4)     # encoding: [0xe7,0x23,0x45,0x67,0x38,0x04]
+
+	vllezg	%v0, 0
+	vllezg	%v0, 4095
+	vllezg	%v0, 0(%r15)
+	vllezg	%v0, 0(%r15,%r1)
+	vllezg	%v15, 0
+	vllezg	%v31, 0
+	vllezg	%v18, 0x567(%r3,%r4)
+
+#CHECK: vllezh  %v0, 0                  # encoding: [0xe7,0x00,0x00,0x00,0x10,0x04]
+#CHECK: vllezh  %v0, 4095               # encoding: [0xe7,0x00,0x0f,0xff,0x10,0x04]
+#CHECK: vllezh  %v0, 0(%r15)            # encoding: [0xe7,0x00,0xf0,0x00,0x10,0x04]
+#CHECK: vllezh  %v0, 0(%r15,%r1)        # encoding: [0xe7,0x0f,0x10,0x00,0x10,0x04]
+#CHECK: vllezh  %v15, 0                 # encoding: [0xe7,0xf0,0x00,0x00,0x10,0x04]
+#CHECK: vllezh  %v31, 0                 # encoding: [0xe7,0xf0,0x00,0x00,0x18,0x04]
+#CHECK: vllezh  %v18, 1383(%r3,%r4)     # encoding: [0xe7,0x23,0x45,0x67,0x18,0x04]
+
+	vllezh	%v0, 0
+	vllezh	%v0, 4095
+	vllezh	%v0, 0(%r15)
+	vllezh	%v0, 0(%r15,%r1)
+	vllezh	%v15, 0
+	vllezh	%v31, 0
+	vllezh	%v18, 0x567(%r3,%r4)
+
+#CHECK: vlm     %v0, %v0, 0             # encoding: [0xe7,0x00,0x00,0x00,0x00,0x36]
+#CHECK: vlm     %v0, %v0, 4095          # encoding: [0xe7,0x00,0x0f,0xff,0x00,0x36]
+#CHECK: vlm     %v0, %v0, 0(%r15)       # encoding: [0xe7,0x00,0xf0,0x00,0x00,0x36]
+#CHECK: vlm     %v0, %v31, 0            # encoding: [0xe7,0x0f,0x00,0x00,0x04,0x36]
+#CHECK: vlm     %v31, %v0, 0            # encoding: [0xe7,0xf0,0x00,0x00,0x08,0x36]
+#CHECK: vlm     %v14, %v17, 1074(%r5)   # encoding: [0xe7,0xe1,0x54,0x32,0x04,0x36]
+
+	vlm	%v0, %v0, 0
+	vlm	%v0, %v0, 4095
+	vlm	%v0, %v0, 0(%r15)
+	vlm	%v0, %v31, 0
+	vlm	%v31, %v0, 0
+	vlm	%v14, %v17, 1074(%r5)
+
+#CHECK: vlpb    %v0, %v0                # encoding: [0xe7,0x00,0x00,0x00,0x00,0xdf]
+#CHECK: vlpb    %v0, %v15               # encoding: [0xe7,0x0f,0x00,0x00,0x00,0xdf]
+#CHECK: vlpb    %v0, %v31               # encoding: [0xe7,0x0f,0x00,0x00,0x04,0xdf]
+#CHECK: vlpb    %v15, %v0               # encoding: [0xe7,0xf0,0x00,0x00,0x00,0xdf]
+#CHECK: vlpb    %v31, %v0               # encoding: [0xe7,0xf0,0x00,0x00,0x08,0xdf]
+#CHECK: vlpb    %v14, %v17              # encoding: [0xe7,0xe1,0x00,0x00,0x04,0xdf]
+
+	vlpb	%v0, %v0
+	vlpb	%v0, %v15
+	vlpb	%v0, %v31
+	vlpb	%v15, %v0
+	vlpb	%v31, %v0
+	vlpb	%v14, %v17
+
+#CHECK: vlpf    %v0, %v0                # encoding: [0xe7,0x00,0x00,0x00,0x20,0xdf]
+#CHECK: vlpf    %v0, %v15               # encoding: [0xe7,0x0f,0x00,0x00,0x20,0xdf]
+#CHECK: vlpf    %v0, %v31               # encoding: [0xe7,0x0f,0x00,0x00,0x24,0xdf]
+#CHECK: vlpf    %v15, %v0               # encoding: [0xe7,0xf0,0x00,0x00,0x20,0xdf]
+#CHECK: vlpf    %v31, %v0               # encoding: [0xe7,0xf0,0x00,0x00,0x28,0xdf]
+#CHECK: vlpf    %v14, %v17              # encoding: [0xe7,0xe1,0x00,0x00,0x24,0xdf]
+
+	vlpf	%v0, %v0
+	vlpf	%v0, %v15
+	vlpf	%v0, %v31
+	vlpf	%v15, %v0
+	vlpf	%v31, %v0
+	vlpf	%v14, %v17
+
+#CHECK: vlpg    %v0, %v0                # encoding: [0xe7,0x00,0x00,0x00,0x30,0xdf]
+#CHECK: vlpg    %v0, %v15               # encoding: [0xe7,0x0f,0x00,0x00,0x30,0xdf]
+#CHECK: vlpg    %v0, %v31               # encoding: [0xe7,0x0f,0x00,0x00,0x34,0xdf]
+#CHECK: vlpg    %v15, %v0               # encoding: [0xe7,0xf0,0x00,0x00,0x30,0xdf]
+#CHECK: vlpg    %v31, %v0               # encoding: [0xe7,0xf0,0x00,0x00,0x38,0xdf]
+#CHECK: vlpg    %v14, %v17              # encoding: [0xe7,0xe1,0x00,0x00,0x34,0xdf]
+
+	vlpg	%v0, %v0
+	vlpg	%v0, %v15
+	vlpg	%v0, %v31
+	vlpg	%v15, %v0
+	vlpg	%v31, %v0
+	vlpg	%v14, %v17
+
+#CHECK: vlph    %v0, %v0                # encoding: [0xe7,0x00,0x00,0x00,0x10,0xdf]
+#CHECK: vlph    %v0, %v15               # encoding: [0xe7,0x0f,0x00,0x00,0x10,0xdf]
+#CHECK: vlph    %v0, %v31               # encoding: [0xe7,0x0f,0x00,0x00,0x14,0xdf]
+#CHECK: vlph    %v15, %v0               # encoding: [0xe7,0xf0,0x00,0x00,0x10,0xdf]
+#CHECK: vlph    %v31, %v0               # encoding: [0xe7,0xf0,0x00,0x00,0x18,0xdf]
+#CHECK: vlph    %v14, %v17              # encoding: [0xe7,0xe1,0x00,0x00,0x14,0xdf]
+
+	vlph	%v0, %v0
+	vlph	%v0, %v15
+	vlph	%v0, %v31
+	vlph	%v15, %v0
+	vlph	%v31, %v0
+	vlph	%v14, %v17
+
+#CHECK: vlr     %v0, %v0                # encoding: [0xe7,0x00,0x00,0x00,0x00,0x56]
+#CHECK: vlr     %v0, %v15               # encoding: [0xe7,0x0f,0x00,0x00,0x00,0x56]
+#CHECK: vlr     %v0, %v31               # encoding: [0xe7,0x0f,0x00,0x00,0x04,0x56]
+#CHECK: vlr     %v15, %v0               # encoding: [0xe7,0xf0,0x00,0x00,0x00,0x56]
+#CHECK: vlr     %v31, %v0               # encoding: [0xe7,0xf0,0x00,0x00,0x08,0x56]
+#CHECK: vlr     %v14, %v17              # encoding: [0xe7,0xe1,0x00,0x00,0x04,0x56]
+
+	vlr	%v0, %v0
+	vlr	%v0, %v15
+	vlr	%v0, %v31
+	vlr	%v15, %v0
+	vlr	%v31, %v0
+	vlr	%v14, %v17
+
+#CHECK: vlrepb  %v0, 0                  # encoding: [0xe7,0x00,0x00,0x00,0x00,0x05]
+#CHECK: vlrepb  %v0, 4095               # encoding: [0xe7,0x00,0x0f,0xff,0x00,0x05]
+#CHECK: vlrepb  %v0, 0(%r15)            # encoding: [0xe7,0x00,0xf0,0x00,0x00,0x05]
+#CHECK: vlrepb  %v0, 0(%r15,%r1)        # encoding: [0xe7,0x0f,0x10,0x00,0x00,0x05]
+#CHECK: vlrepb  %v15, 0                 # encoding: [0xe7,0xf0,0x00,0x00,0x00,0x05]
+#CHECK: vlrepb  %v31, 0                 # encoding: [0xe7,0xf0,0x00,0x00,0x08,0x05]
+#CHECK: vlrepb  %v18, 1383(%r3,%r4)     # encoding: [0xe7,0x23,0x45,0x67,0x08,0x05]
+
+	vlrepb	%v0, 0
+	vlrepb	%v0, 4095
+	vlrepb	%v0, 0(%r15)
+	vlrepb	%v0, 0(%r15,%r1)
+	vlrepb	%v15, 0
+	vlrepb	%v31, 0
+	vlrepb	%v18, 0x567(%r3,%r4)
+
+#CHECK: vlrepf  %v0, 0                  # encoding: [0xe7,0x00,0x00,0x00,0x20,0x05]
+#CHECK: vlrepf  %v0, 4095               # encoding: [0xe7,0x00,0x0f,0xff,0x20,0x05]
+#CHECK: vlrepf  %v0, 0(%r15)            # encoding: [0xe7,0x00,0xf0,0x00,0x20,0x05]
+#CHECK: vlrepf  %v0, 0(%r15,%r1)        # encoding: [0xe7,0x0f,0x10,0x00,0x20,0x05]
+#CHECK: vlrepf  %v15, 0                 # encoding: [0xe7,0xf0,0x00,0x00,0x20,0x05]
+#CHECK: vlrepf  %v31, 0                 # encoding: [0xe7,0xf0,0x00,0x00,0x28,0x05]
+#CHECK: vlrepf  %v18, 1383(%r3,%r4)     # encoding: [0xe7,0x23,0x45,0x67,0x28,0x05]
+
+	vlrepf	%v0, 0
+	vlrepf	%v0, 4095
+	vlrepf	%v0, 0(%r15)
+	vlrepf	%v0, 0(%r15,%r1)
+	vlrepf	%v15, 0
+	vlrepf	%v31, 0
+	vlrepf	%v18, 0x567(%r3,%r4)
+
+#CHECK: vlrepg  %v0, 0                  # encoding: [0xe7,0x00,0x00,0x00,0x30,0x05]
+#CHECK: vlrepg  %v0, 4095               # encoding: [0xe7,0x00,0x0f,0xff,0x30,0x05]
+#CHECK: vlrepg  %v0, 0(%r15)            # encoding: [0xe7,0x00,0xf0,0x00,0x30,0x05]
+#CHECK: vlrepg  %v0, 0(%r15,%r1)        # encoding: [0xe7,0x0f,0x10,0x00,0x30,0x05]
+#CHECK: vlrepg  %v15, 0                 # encoding: [0xe7,0xf0,0x00,0x00,0x30,0x05]
+#CHECK: vlrepg  %v31, 0                 # encoding: [0xe7,0xf0,0x00,0x00,0x38,0x05]
+#CHECK: vlrepg  %v18, 1383(%r3,%r4)     # encoding: [0xe7,0x23,0x45,0x67,0x38,0x05]
+
+	vlrepg	%v0, 0
+	vlrepg	%v0, 4095
+	vlrepg	%v0, 0(%r15)
+	vlrepg	%v0, 0(%r15,%r1)
+	vlrepg	%v15, 0
+	vlrepg	%v31, 0
+	vlrepg	%v18, 0x567(%r3,%r4)
+
+#CHECK: vlreph  %v0, 0                  # encoding: [0xe7,0x00,0x00,0x00,0x10,0x05]
+#CHECK: vlreph  %v0, 4095               # encoding: [0xe7,0x00,0x0f,0xff,0x10,0x05]
+#CHECK: vlreph  %v0, 0(%r15)            # encoding: [0xe7,0x00,0xf0,0x00,0x10,0x05]
+#CHECK: vlreph  %v0, 0(%r15,%r1)        # encoding: [0xe7,0x0f,0x10,0x00,0x10,0x05]
+#CHECK: vlreph  %v15, 0                 # encoding: [0xe7,0xf0,0x00,0x00,0x10,0x05]
+#CHECK: vlreph  %v31, 0                 # encoding: [0xe7,0xf0,0x00,0x00,0x18,0x05]
+#CHECK: vlreph  %v18, 1383(%r3,%r4)     # encoding: [0xe7,0x23,0x45,0x67,0x18,0x05]
+
+	vlreph	%v0, 0
+	vlreph	%v0, 4095
+	vlreph	%v0, 0(%r15)
+	vlreph	%v0, 0(%r15,%r1)
+	vlreph	%v15, 0
+	vlreph	%v31, 0
+	vlreph	%v18, 0x567(%r3,%r4)
+
+#CHECK: vlvgb   %v0, %r0, 0             # encoding: [0xe7,0x00,0x00,0x00,0x00,0x22]
+#CHECK: vlvgb   %v0, %r0, 4095          # encoding: [0xe7,0x00,0x0f,0xff,0x00,0x22]
+#CHECK: vlvgb   %v0, %r0, 0(%r15)       # encoding: [0xe7,0x00,0xf0,0x00,0x00,0x22]
+#CHECK: vlvgb   %v0, %r15, 0            # encoding: [0xe7,0x0f,0x00,0x00,0x00,0x22]
+#CHECK: vlvgb   %v15, %r0, 0            # encoding: [0xe7,0xf0,0x00,0x00,0x00,0x22]
+#CHECK: vlvgb   %v31, %r0, 0            # encoding: [0xe7,0xf0,0x00,0x00,0x08,0x22]
+#CHECK: vlvgb   %v18, %r3, 1383(%r4)    # encoding: [0xe7,0x23,0x45,0x67,0x08,0x22]
+
+	vlvgb	%v0, %r0, 0
+	vlvgb	%v0, %r0, 4095
+	vlvgb	%v0, %r0, 0(%r15)
+	vlvgb	%v0, %r15, 0
+	vlvgb	%v15, %r0, 0
+	vlvgb	%v31, %r0, 0
+	vlvgb	%v18, %r3, 1383(%r4)
+
+#CHECK: vlvgf   %v0, %r0, 0             # encoding: [0xe7,0x00,0x00,0x00,0x20,0x22]
+#CHECK: vlvgf   %v0, %r0, 4095          # encoding: [0xe7,0x00,0x0f,0xff,0x20,0x22]
+#CHECK: vlvgf   %v0, %r0, 0(%r15)       # encoding: [0xe7,0x00,0xf0,0x00,0x20,0x22]
+#CHECK: vlvgf   %v0, %r15, 0            # encoding: [0xe7,0x0f,0x00,0x00,0x20,0x22]
+#CHECK: vlvgf   %v15, %r0, 0            # encoding: [0xe7,0xf0,0x00,0x00,0x20,0x22]
+#CHECK: vlvgf   %v31, %r0, 0            # encoding: [0xe7,0xf0,0x00,0x00,0x28,0x22]
+#CHECK: vlvgf   %v18, %r3, 1383(%r4)    # encoding: [0xe7,0x23,0x45,0x67,0x28,0x22]
+
+	vlvgf	%v0, %r0, 0
+	vlvgf	%v0, %r0, 4095
+	vlvgf	%v0, %r0, 0(%r15)
+	vlvgf	%v0, %r15, 0
+	vlvgf	%v15, %r0, 0
+	vlvgf	%v31, %r0, 0
+	vlvgf	%v18, %r3, 1383(%r4)
+
+#CHECK: vlvgg   %v0, %r0, 0             # encoding: [0xe7,0x00,0x00,0x00,0x30,0x22]
+#CHECK: vlvgg   %v0, %r0, 4095          # encoding: [0xe7,0x00,0x0f,0xff,0x30,0x22]
+#CHECK: vlvgg   %v0, %r0, 0(%r15)       # encoding: [0xe7,0x00,0xf0,0x00,0x30,0x22]
+#CHECK: vlvgg   %v0, %r15, 0            # encoding: [0xe7,0x0f,0x00,0x00,0x30,0x22]
+#CHECK: vlvgg   %v15, %r0, 0            # encoding: [0xe7,0xf0,0x00,0x00,0x30,0x22]
+#CHECK: vlvgg   %v31, %r0, 0            # encoding: [0xe7,0xf0,0x00,0x00,0x38,0x22]
+#CHECK: vlvgg   %v18, %r3, 1383(%r4)    # encoding: [0xe7,0x23,0x45,0x67,0x38,0x22]
+
+	vlvgg	%v0, %r0, 0
+	vlvgg	%v0, %r0, 4095
+	vlvgg	%v0, %r0, 0(%r15)
+	vlvgg	%v0, %r15, 0
+	vlvgg	%v15, %r0, 0
+	vlvgg	%v31, %r0, 0
+	vlvgg	%v18, %r3, 1383(%r4)
+
+#CHECK: vlvgh   %v0, %r0, 0             # encoding: [0xe7,0x00,0x00,0x00,0x10,0x22]
+#CHECK: vlvgh   %v0, %r0, 4095          # encoding: [0xe7,0x00,0x0f,0xff,0x10,0x22]
+#CHECK: vlvgh   %v0, %r0, 0(%r15)       # encoding: [0xe7,0x00,0xf0,0x00,0x10,0x22]
+#CHECK: vlvgh   %v0, %r15, 0            # encoding: [0xe7,0x0f,0x00,0x00,0x10,0x22]
+#CHECK: vlvgh   %v15, %r0, 0            # encoding: [0xe7,0xf0,0x00,0x00,0x10,0x22]
+#CHECK: vlvgh   %v31, %r0, 0            # encoding: [0xe7,0xf0,0x00,0x00,0x18,0x22]
+#CHECK: vlvgh   %v18, %r3, 1383(%r4)    # encoding: [0xe7,0x23,0x45,0x67,0x18,0x22]
+
+	vlvgh	%v0, %r0, 0
+	vlvgh	%v0, %r0, 4095
+	vlvgh	%v0, %r0, 0(%r15)
+	vlvgh	%v0, %r15, 0
+	vlvgh	%v15, %r0, 0
+	vlvgh	%v31, %r0, 0
+	vlvgh	%v18, %r3, 1383(%r4)
+
+#CHECK: vlvgp   %v0, %r0, %r0           # encoding: [0xe7,0x00,0x00,0x00,0x00,0x62]
+#CHECK: vlvgp   %v0, %r0, %r15          # encoding: [0xe7,0x00,0xf0,0x00,0x00,0x62]
+#CHECK: vlvgp   %v0, %r15, %r0          # encoding: [0xe7,0x0f,0x00,0x00,0x00,0x62]
+#CHECK: vlvgp   %v15, %r0, %r0          # encoding: [0xe7,0xf0,0x00,0x00,0x00,0x62]
+#CHECK: vlvgp   %v31, %r0, %r0          # encoding: [0xe7,0xf0,0x00,0x00,0x08,0x62]
+#CHECK: vlvgp   %v18, %r3, %r4          # encoding: [0xe7,0x23,0x40,0x00,0x08,0x62]
+
+	vlvgp	%v0, %r0, %r0
+	vlvgp	%v0, %r0, %r15
+	vlvgp	%v0, %r15, %r0
+	vlvgp	%v15, %r0, %r0
+	vlvgp	%v31, %r0, %r0
+	vlvgp	%v18, %r3, %r4
+
+#CHECK: vmaeb   %v0, %v0, %v0, %v0      # encoding: [0xe7,0x00,0x00,0x00,0x00,0xae]
+#CHECK: vmaeb   %v0, %v0, %v0, %v31     # encoding: [0xe7,0x00,0x00,0x00,0xf1,0xae]
+#CHECK: vmaeb   %v0, %v0, %v31, %v0     # encoding: [0xe7,0x00,0xf0,0x00,0x02,0xae]
+#CHECK: vmaeb   %v0, %v31, %v0, %v0     # encoding: [0xe7,0x0f,0x00,0x00,0x04,0xae]
+#CHECK: vmaeb   %v31, %v0, %v0, %v0     # encoding: [0xe7,0xf0,0x00,0x00,0x08,0xae]
+#CHECK: vmaeb   %v13, %v17, %v21, %v25  # encoding: [0xe7,0xd1,0x50,0x00,0x97,0xae]
+
+	vmaeb	%v0, %v0, %v0, %v0
+	vmaeb	%v0, %v0, %v0, %v31
+	vmaeb	%v0, %v0, %v31, %v0
+	vmaeb	%v0, %v31, %v0, %v0
+	vmaeb	%v31, %v0, %v0, %v0
+	vmaeb	%v13, %v17, %v21, %v25
+
+#CHECK: vmaef   %v0, %v0, %v0, %v0      # encoding: [0xe7,0x00,0x02,0x00,0x00,0xae]
+#CHECK: vmaef   %v0, %v0, %v0, %v31     # encoding: [0xe7,0x00,0x02,0x00,0xf1,0xae]
+#CHECK: vmaef   %v0, %v0, %v31, %v0     # encoding: [0xe7,0x00,0xf2,0x00,0x02,0xae]
+#CHECK: vmaef   %v0, %v31, %v0, %v0     # encoding: [0xe7,0x0f,0x02,0x00,0x04,0xae]
+#CHECK: vmaef   %v31, %v0, %v0, %v0     # encoding: [0xe7,0xf0,0x02,0x00,0x08,0xae]
+#CHECK: vmaef   %v13, %v17, %v21, %v25  # encoding: [0xe7,0xd1,0x52,0x00,0x97,0xae]
+
+	vmaef	%v0, %v0, %v0, %v0
+	vmaef	%v0, %v0, %v0, %v31
+	vmaef	%v0, %v0, %v31, %v0
+	vmaef	%v0, %v31, %v0, %v0
+	vmaef	%v31, %v0, %v0, %v0
+	vmaef	%v13, %v17, %v21, %v25
+
+#CHECK: vmaeh   %v0, %v0, %v0, %v0      # encoding: [0xe7,0x00,0x01,0x00,0x00,0xae]
+#CHECK: vmaeh   %v0, %v0, %v0, %v31     # encoding: [0xe7,0x00,0x01,0x00,0xf1,0xae]
+#CHECK: vmaeh   %v0, %v0, %v31, %v0     # encoding: [0xe7,0x00,0xf1,0x00,0x02,0xae]
+#CHECK: vmaeh   %v0, %v31, %v0, %v0     # encoding: [0xe7,0x0f,0x01,0x00,0x04,0xae]
+#CHECK: vmaeh   %v31, %v0, %v0, %v0     # encoding: [0xe7,0xf0,0x01,0x00,0x08,0xae]
+#CHECK: vmaeh   %v13, %v17, %v21, %v25  # encoding: [0xe7,0xd1,0x51,0x00,0x97,0xae]
+
+	vmaeh	%v0, %v0, %v0, %v0
+	vmaeh	%v0, %v0, %v0, %v31
+	vmaeh	%v0, %v0, %v31, %v0
+	vmaeh	%v0, %v31, %v0, %v0
+	vmaeh	%v31, %v0, %v0, %v0
+	vmaeh	%v13, %v17, %v21, %v25
+
+#CHECK: vmahb   %v0, %v0, %v0, %v0      # encoding: [0xe7,0x00,0x00,0x00,0x00,0xab]
+#CHECK: vmahb   %v0, %v0, %v0, %v31     # encoding: [0xe7,0x00,0x00,0x00,0xf1,0xab]
+#CHECK: vmahb   %v0, %v0, %v31, %v0     # encoding: [0xe7,0x00,0xf0,0x00,0x02,0xab]
+#CHECK: vmahb   %v0, %v31, %v0, %v0     # encoding: [0xe7,0x0f,0x00,0x00,0x04,0xab]
+#CHECK: vmahb   %v31, %v0, %v0, %v0     # encoding: [0xe7,0xf0,0x00,0x00,0x08,0xab]
+#CHECK: vmahb   %v13, %v17, %v21, %v25  # encoding: [0xe7,0xd1,0x50,0x00,0x97,0xab]
+
+	vmahb	%v0, %v0, %v0, %v0
+	vmahb	%v0, %v0, %v0, %v31
+	vmahb	%v0, %v0, %v31, %v0
+	vmahb	%v0, %v31, %v0, %v0
+	vmahb	%v31, %v0, %v0, %v0
+	vmahb	%v13, %v17, %v21, %v25
+
+#CHECK: vmahf   %v0, %v0, %v0, %v0      # encoding: [0xe7,0x00,0x02,0x00,0x00,0xab]
+#CHECK: vmahf   %v0, %v0, %v0, %v31     # encoding: [0xe7,0x00,0x02,0x00,0xf1,0xab]
+#CHECK: vmahf   %v0, %v0, %v31, %v0     # encoding: [0xe7,0x00,0xf2,0x00,0x02,0xab]
+#CHECK: vmahf   %v0, %v31, %v0, %v0     # encoding: [0xe7,0x0f,0x02,0x00,0x04,0xab]
+#CHECK: vmahf   %v31, %v0, %v0, %v0     # encoding: [0xe7,0xf0,0x02,0x00,0x08,0xab]
+#CHECK: vmahf   %v13, %v17, %v21, %v25  # encoding: [0xe7,0xd1,0x52,0x00,0x97,0xab]
+
+	vmahf	%v0, %v0, %v0, %v0
+	vmahf	%v0, %v0, %v0, %v31
+	vmahf	%v0, %v0, %v31, %v0
+	vmahf	%v0, %v31, %v0, %v0
+	vmahf	%v31, %v0, %v0, %v0
+	vmahf	%v13, %v17, %v21, %v25
+
+#CHECK: vmahh   %v0, %v0, %v0, %v0      # encoding: [0xe7,0x00,0x01,0x00,0x00,0xab]
+#CHECK: vmahh   %v0, %v0, %v0, %v31     # encoding: [0xe7,0x00,0x01,0x00,0xf1,0xab]
+#CHECK: vmahh   %v0, %v0, %v31, %v0     # encoding: [0xe7,0x00,0xf1,0x00,0x02,0xab]
+#CHECK: vmahh   %v0, %v31, %v0, %v0     # encoding: [0xe7,0x0f,0x01,0x00,0x04,0xab]
+#CHECK: vmahh   %v31, %v0, %v0, %v0     # encoding: [0xe7,0xf0,0x01,0x00,0x08,0xab]
+#CHECK: vmahh   %v13, %v17, %v21, %v25  # encoding: [0xe7,0xd1,0x51,0x00,0x97,0xab]
+
+	vmahh	%v0, %v0, %v0, %v0
+	vmahh	%v0, %v0, %v0, %v31
+	vmahh	%v0, %v0, %v31, %v0
+	vmahh	%v0, %v31, %v0, %v0
+	vmahh	%v31, %v0, %v0, %v0
+	vmahh	%v13, %v17, %v21, %v25
+
+#CHECK: vmalb   %v0, %v0, %v0, %v0      # encoding: [0xe7,0x00,0x00,0x00,0x00,0xaa]
+#CHECK: vmalb   %v0, %v0, %v0, %v31     # encoding: [0xe7,0x00,0x00,0x00,0xf1,0xaa]
+#CHECK: vmalb   %v0, %v0, %v31, %v0     # encoding: [0xe7,0x00,0xf0,0x00,0x02,0xaa]
+#CHECK: vmalb   %v0, %v31, %v0, %v0     # encoding: [0xe7,0x0f,0x00,0x00,0x04,0xaa]
+#CHECK: vmalb   %v31, %v0, %v0, %v0     # encoding: [0xe7,0xf0,0x00,0x00,0x08,0xaa]
+#CHECK: vmalb   %v13, %v17, %v21, %v25  # encoding: [0xe7,0xd1,0x50,0x00,0x97,0xaa]
+
+	vmalb	%v0, %v0, %v0, %v0
+	vmalb	%v0, %v0, %v0, %v31
+	vmalb	%v0, %v0, %v31, %v0
+	vmalb	%v0, %v31, %v0, %v0
+	vmalb	%v31, %v0, %v0, %v0
+	vmalb	%v13, %v17, %v21, %v25
+
+#CHECK: vmaleb  %v0, %v0, %v0, %v0      # encoding: [0xe7,0x00,0x00,0x00,0x00,0xac]
+#CHECK: vmaleb  %v0, %v0, %v0, %v31     # encoding: [0xe7,0x00,0x00,0x00,0xf1,0xac]
+#CHECK: vmaleb  %v0, %v0, %v31, %v0     # encoding: [0xe7,0x00,0xf0,0x00,0x02,0xac]
+#CHECK: vmaleb  %v0, %v31, %v0, %v0     # encoding: [0xe7,0x0f,0x00,0x00,0x04,0xac]
+#CHECK: vmaleb  %v31, %v0, %v0, %v0     # encoding: [0xe7,0xf0,0x00,0x00,0x08,0xac]
+#CHECK: vmaleb  %v13, %v17, %v21, %v25  # encoding: [0xe7,0xd1,0x50,0x00,0x97,0xac]
+
+	vmaleb	%v0, %v0, %v0, %v0
+	vmaleb	%v0, %v0, %v0, %v31
+	vmaleb	%v0, %v0, %v31, %v0
+	vmaleb	%v0, %v31, %v0, %v0
+	vmaleb	%v31, %v0, %v0, %v0
+	vmaleb	%v13, %v17, %v21, %v25
+
+#CHECK: vmalef  %v0, %v0, %v0, %v0      # encoding: [0xe7,0x00,0x02,0x00,0x00,0xac]
+#CHECK: vmalef  %v0, %v0, %v0, %v31     # encoding: [0xe7,0x00,0x02,0x00,0xf1,0xac]
+#CHECK: vmalef  %v0, %v0, %v31, %v0     # encoding: [0xe7,0x00,0xf2,0x00,0x02,0xac]
+#CHECK: vmalef  %v0, %v31, %v0, %v0     # encoding: [0xe7,0x0f,0x02,0x00,0x04,0xac]
+#CHECK: vmalef  %v31, %v0, %v0, %v0     # encoding: [0xe7,0xf0,0x02,0x00,0x08,0xac]
+#CHECK: vmalef  %v13, %v17, %v21, %v25  # encoding: [0xe7,0xd1,0x52,0x00,0x97,0xac]
+
+	vmalef	%v0, %v0, %v0, %v0
+	vmalef	%v0, %v0, %v0, %v31
+	vmalef	%v0, %v0, %v31, %v0
+	vmalef	%v0, %v31, %v0, %v0
+	vmalef	%v31, %v0, %v0, %v0
+	vmalef	%v13, %v17, %v21, %v25
+
+#CHECK: vmaleh  %v0, %v0, %v0, %v0      # encoding: [0xe7,0x00,0x01,0x00,0x00,0xac]
+#CHECK: vmaleh  %v0, %v0, %v0, %v31     # encoding: [0xe7,0x00,0x01,0x00,0xf1,0xac]
+#CHECK: vmaleh  %v0, %v0, %v31, %v0     # encoding: [0xe7,0x00,0xf1,0x00,0x02,0xac]
+#CHECK: vmaleh  %v0, %v31, %v0, %v0     # encoding: [0xe7,0x0f,0x01,0x00,0x04,0xac]
+#CHECK: vmaleh  %v31, %v0, %v0, %v0     # encoding: [0xe7,0xf0,0x01,0x00,0x08,0xac]
+#CHECK: vmaleh  %v13, %v17, %v21, %v25  # encoding: [0xe7,0xd1,0x51,0x00,0x97,0xac]
+
+	vmaleh	%v0, %v0, %v0, %v0
+	vmaleh	%v0, %v0, %v0, %v31
+	vmaleh	%v0, %v0, %v31, %v0
+	vmaleh	%v0, %v31, %v0, %v0
+	vmaleh	%v31, %v0, %v0, %v0
+	vmaleh	%v13, %v17, %v21, %v25
+
+#CHECK: vmalf   %v0, %v0, %v0, %v0      # encoding: [0xe7,0x00,0x02,0x00,0x00,0xaa]
+#CHECK: vmalf   %v0, %v0, %v0, %v31     # encoding: [0xe7,0x00,0x02,0x00,0xf1,0xaa]
+#CHECK: vmalf   %v0, %v0, %v31, %v0     # encoding: [0xe7,0x00,0xf2,0x00,0x02,0xaa]
+#CHECK: vmalf   %v0, %v31, %v0, %v0     # encoding: [0xe7,0x0f,0x02,0x00,0x04,0xaa]
+#CHECK: vmalf   %v31, %v0, %v0, %v0     # encoding: [0xe7,0xf0,0x02,0x00,0x08,0xaa]
+#CHECK: vmalf   %v13, %v17, %v21, %v25  # encoding: [0xe7,0xd1,0x52,0x00,0x97,0xaa]
+
+	vmalf	%v0, %v0, %v0, %v0
+	vmalf	%v0, %v0, %v0, %v31
+	vmalf	%v0, %v0, %v31, %v0
+	vmalf	%v0, %v31, %v0, %v0
+	vmalf	%v31, %v0, %v0, %v0
+	vmalf	%v13, %v17, %v21, %v25
+
+#CHECK: vmalhb  %v0, %v0, %v0, %v0      # encoding: [0xe7,0x00,0x00,0x00,0x00,0xa9]
+#CHECK: vmalhb  %v0, %v0, %v0, %v31     # encoding: [0xe7,0x00,0x00,0x00,0xf1,0xa9]
+#CHECK: vmalhb  %v0, %v0, %v31, %v0     # encoding: [0xe7,0x00,0xf0,0x00,0x02,0xa9]
+#CHECK: vmalhb  %v0, %v31, %v0, %v0     # encoding: [0xe7,0x0f,0x00,0x00,0x04,0xa9]
+#CHECK: vmalhb  %v31, %v0, %v0, %v0     # encoding: [0xe7,0xf0,0x00,0x00,0x08,0xa9]
+#CHECK: vmalhb  %v13, %v17, %v21, %v25  # encoding: [0xe7,0xd1,0x50,0x00,0x97,0xa9]
+
+	vmalhb	%v0, %v0, %v0, %v0
+	vmalhb	%v0, %v0, %v0, %v31
+	vmalhb	%v0, %v0, %v31, %v0
+	vmalhb	%v0, %v31, %v0, %v0
+	vmalhb	%v31, %v0, %v0, %v0
+	vmalhb	%v13, %v17, %v21, %v25
+
+#CHECK: vmalhf  %v0, %v0, %v0, %v0      # encoding: [0xe7,0x00,0x02,0x00,0x00,0xa9]
+#CHECK: vmalhf  %v0, %v0, %v0, %v31     # encoding: [0xe7,0x00,0x02,0x00,0xf1,0xa9]
+#CHECK: vmalhf  %v0, %v0, %v31, %v0     # encoding: [0xe7,0x00,0xf2,0x00,0x02,0xa9]
+#CHECK: vmalhf  %v0, %v31, %v0, %v0     # encoding: [0xe7,0x0f,0x02,0x00,0x04,0xa9]
+#CHECK: vmalhf  %v31, %v0, %v0, %v0     # encoding: [0xe7,0xf0,0x02,0x00,0x08,0xa9]
+#CHECK: vmalhf  %v13, %v17, %v21, %v25  # encoding: [0xe7,0xd1,0x52,0x00,0x97,0xa9]
+
+	vmalhf	%v0, %v0, %v0, %v0
+	vmalhf	%v0, %v0, %v0, %v31
+	vmalhf	%v0, %v0, %v31, %v0
+	vmalhf	%v0, %v31, %v0, %v0
+	vmalhf	%v31, %v0, %v0, %v0
+	vmalhf	%v13, %v17, %v21, %v25
+
+#CHECK: vmalhh  %v0, %v0, %v0, %v0      # encoding: [0xe7,0x00,0x01,0x00,0x00,0xa9]
+#CHECK: vmalhh  %v0, %v0, %v0, %v31     # encoding: [0xe7,0x00,0x01,0x00,0xf1,0xa9]
+#CHECK: vmalhh  %v0, %v0, %v31, %v0     # encoding: [0xe7,0x00,0xf1,0x00,0x02,0xa9]
+#CHECK: vmalhh  %v0, %v31, %v0, %v0     # encoding: [0xe7,0x0f,0x01,0x00,0x04,0xa9]
+#CHECK: vmalhh  %v31, %v0, %v0, %v0     # encoding: [0xe7,0xf0,0x01,0x00,0x08,0xa9]
+#CHECK: vmalhh  %v13, %v17, %v21, %v25  # encoding: [0xe7,0xd1,0x51,0x00,0x97,0xa9]
+
+	vmalhh	%v0, %v0, %v0, %v0
+	vmalhh	%v0, %v0, %v0, %v31
+	vmalhh	%v0, %v0, %v31, %v0
+	vmalhh	%v0, %v31, %v0, %v0
+	vmalhh	%v31, %v0, %v0, %v0
+	vmalhh	%v13, %v17, %v21, %v25
+
+#CHECK: vmalhw  %v0, %v0, %v0, %v0      # encoding: [0xe7,0x00,0x01,0x00,0x00,0xaa]
+#CHECK: vmalhw  %v0, %v0, %v0, %v31     # encoding: [0xe7,0x00,0x01,0x00,0xf1,0xaa]
+#CHECK: vmalhw  %v0, %v0, %v31, %v0     # encoding: [0xe7,0x00,0xf1,0x00,0x02,0xaa]
+#CHECK: vmalhw  %v0, %v31, %v0, %v0     # encoding: [0xe7,0x0f,0x01,0x00,0x04,0xaa]
+#CHECK: vmalhw  %v31, %v0, %v0, %v0     # encoding: [0xe7,0xf0,0x01,0x00,0x08,0xaa]
+#CHECK: vmalhw  %v13, %v17, %v21, %v25  # encoding: [0xe7,0xd1,0x51,0x00,0x97,0xaa]
+
+	vmalhw	%v0, %v0, %v0, %v0
+	vmalhw	%v0, %v0, %v0, %v31
+	vmalhw	%v0, %v0, %v31, %v0
+	vmalhw	%v0, %v31, %v0, %v0
+	vmalhw	%v31, %v0, %v0, %v0
+	vmalhw	%v13, %v17, %v21, %v25
+
+#CHECK: vmalob  %v0, %v0, %v0, %v0      # encoding: [0xe7,0x00,0x00,0x00,0x00,0xad]
+#CHECK: vmalob  %v0, %v0, %v0, %v31     # encoding: [0xe7,0x00,0x00,0x00,0xf1,0xad]
+#CHECK: vmalob  %v0, %v0, %v31, %v0     # encoding: [0xe7,0x00,0xf0,0x00,0x02,0xad]
+#CHECK: vmalob  %v0, %v31, %v0, %v0     # encoding: [0xe7,0x0f,0x00,0x00,0x04,0xad]
+#CHECK: vmalob  %v31, %v0, %v0, %v0     # encoding: [0xe7,0xf0,0x00,0x00,0x08,0xad]
+#CHECK: vmalob  %v13, %v17, %v21, %v25  # encoding: [0xe7,0xd1,0x50,0x00,0x97,0xad]
+
+	vmalob	%v0, %v0, %v0, %v0
+	vmalob	%v0, %v0, %v0, %v31
+	vmalob	%v0, %v0, %v31, %v0
+	vmalob	%v0, %v31, %v0, %v0
+	vmalob	%v31, %v0, %v0, %v0
+	vmalob	%v13, %v17, %v21, %v25
+
+#CHECK: vmalof  %v0, %v0, %v0, %v0      # encoding: [0xe7,0x00,0x02,0x00,0x00,0xad]
+#CHECK: vmalof  %v0, %v0, %v0, %v31     # encoding: [0xe7,0x00,0x02,0x00,0xf1,0xad]
+#CHECK: vmalof  %v0, %v0, %v31, %v0     # encoding: [0xe7,0x00,0xf2,0x00,0x02,0xad]
+#CHECK: vmalof  %v0, %v31, %v0, %v0     # encoding: [0xe7,0x0f,0x02,0x00,0x04,0xad]
+#CHECK: vmalof  %v31, %v0, %v0, %v0     # encoding: [0xe7,0xf0,0x02,0x00,0x08,0xad]
+#CHECK: vmalof  %v13, %v17, %v21, %v25  # encoding: [0xe7,0xd1,0x52,0x00,0x97,0xad]
+
+	vmalof	%v0, %v0, %v0, %v0
+	vmalof	%v0, %v0, %v0, %v31
+	vmalof	%v0, %v0, %v31, %v0
+	vmalof	%v0, %v31, %v0, %v0
+	vmalof	%v31, %v0, %v0, %v0
+	vmalof	%v13, %v17, %v21, %v25
+
+#CHECK: vmaloh  %v0, %v0, %v0, %v0      # encoding: [0xe7,0x00,0x01,0x00,0x00,0xad]
+#CHECK: vmaloh  %v0, %v0, %v0, %v31     # encoding: [0xe7,0x00,0x01,0x00,0xf1,0xad]
+#CHECK: vmaloh  %v0, %v0, %v31, %v0     # encoding: [0xe7,0x00,0xf1,0x00,0x02,0xad]
+#CHECK: vmaloh  %v0, %v31, %v0, %v0     # encoding: [0xe7,0x0f,0x01,0x00,0x04,0xad]
+#CHECK: vmaloh  %v31, %v0, %v0, %v0     # encoding: [0xe7,0xf0,0x01,0x00,0x08,0xad]
+#CHECK: vmaloh  %v13, %v17, %v21, %v25  # encoding: [0xe7,0xd1,0x51,0x00,0x97,0xad]
+
+	vmaloh	%v0, %v0, %v0, %v0
+	vmaloh	%v0, %v0, %v0, %v31
+	vmaloh	%v0, %v0, %v31, %v0
+	vmaloh	%v0, %v31, %v0, %v0
+	vmaloh	%v31, %v0, %v0, %v0
+	vmaloh	%v13, %v17, %v21, %v25
+
+#CHECK: vmaob   %v0, %v0, %v0, %v0      # encoding: [0xe7,0x00,0x00,0x00,0x00,0xaf]
+#CHECK: vmaob   %v0, %v0, %v0, %v31     # encoding: [0xe7,0x00,0x00,0x00,0xf1,0xaf]
+#CHECK: vmaob   %v0, %v0, %v31, %v0     # encoding: [0xe7,0x00,0xf0,0x00,0x02,0xaf]
+#CHECK: vmaob   %v0, %v31, %v0, %v0     # encoding: [0xe7,0x0f,0x00,0x00,0x04,0xaf]
+#CHECK: vmaob   %v31, %v0, %v0, %v0     # encoding: [0xe7,0xf0,0x00,0x00,0x08,0xaf]
+#CHECK: vmaob   %v13, %v17, %v21, %v25  # encoding: [0xe7,0xd1,0x50,0x00,0x97,0xaf]
+
+	vmaob	%v0, %v0, %v0, %v0
+	vmaob	%v0, %v0, %v0, %v31
+	vmaob	%v0, %v0, %v31, %v0
+	vmaob	%v0, %v31, %v0, %v0
+	vmaob	%v31, %v0, %v0, %v0
+	vmaob	%v13, %v17, %v21, %v25
+
+#CHECK: vmaof   %v0, %v0, %v0, %v0      # encoding: [0xe7,0x00,0x02,0x00,0x00,0xaf]
+#CHECK: vmaof   %v0, %v0, %v0, %v31     # encoding: [0xe7,0x00,0x02,0x00,0xf1,0xaf]
+#CHECK: vmaof   %v0, %v0, %v31, %v0     # encoding: [0xe7,0x00,0xf2,0x00,0x02,0xaf]
+#CHECK: vmaof   %v0, %v31, %v0, %v0     # encoding: [0xe7,0x0f,0x02,0x00,0x04,0xaf]
+#CHECK: vmaof   %v31, %v0, %v0, %v0     # encoding: [0xe7,0xf0,0x02,0x00,0x08,0xaf]
+#CHECK: vmaof   %v13, %v17, %v21, %v25  # encoding: [0xe7,0xd1,0x52,0x00,0x97,0xaf]
+
+	vmaof	%v0, %v0, %v0, %v0
+	vmaof	%v0, %v0, %v0, %v31
+	vmaof	%v0, %v0, %v31, %v0
+	vmaof	%v0, %v31, %v0, %v0
+	vmaof	%v31, %v0, %v0, %v0
+	vmaof	%v13, %v17, %v21, %v25
+
+#CHECK: vmaoh   %v0, %v0, %v0, %v0      # encoding: [0xe7,0x00,0x01,0x00,0x00,0xaf]
+#CHECK: vmaoh   %v0, %v0, %v0, %v31     # encoding: [0xe7,0x00,0x01,0x00,0xf1,0xaf]
+#CHECK: vmaoh   %v0, %v0, %v31, %v0     # encoding: [0xe7,0x00,0xf1,0x00,0x02,0xaf]
+#CHECK: vmaoh   %v0, %v31, %v0, %v0     # encoding: [0xe7,0x0f,0x01,0x00,0x04,0xaf]
+#CHECK: vmaoh   %v31, %v0, %v0, %v0     # encoding: [0xe7,0xf0,0x01,0x00,0x08,0xaf]
+#CHECK: vmaoh   %v13, %v17, %v21, %v25  # encoding: [0xe7,0xd1,0x51,0x00,0x97,0xaf]
+
+	vmaoh	%v0, %v0, %v0, %v0
+	vmaoh	%v0, %v0, %v0, %v31
+	vmaoh	%v0, %v0, %v31, %v0
+	vmaoh	%v0, %v31, %v0, %v0
+	vmaoh	%v31, %v0, %v0, %v0
+	vmaoh	%v13, %v17, %v21, %v25
+
+#CHECK: vmeb    %v0, %v0, %v0           # encoding: [0xe7,0x00,0x00,0x00,0x00,0xa6]
+#CHECK: vmeb    %v0, %v0, %v31          # encoding: [0xe7,0x00,0xf0,0x00,0x02,0xa6]
+#CHECK: vmeb    %v0, %v31, %v0          # encoding: [0xe7,0x0f,0x00,0x00,0x04,0xa6]
+#CHECK: vmeb    %v31, %v0, %v0          # encoding: [0xe7,0xf0,0x00,0x00,0x08,0xa6]
+#CHECK: vmeb    %v18, %v3, %v20         # encoding: [0xe7,0x23,0x40,0x00,0x0a,0xa6]
+
+	vmeb	%v0, %v0, %v0
+	vmeb	%v0, %v0, %v31
+	vmeb	%v0, %v31, %v0
+	vmeb	%v31, %v0, %v0
+	vmeb	%v18, %v3, %v20
+
+#CHECK: vmef    %v0, %v0, %v0           # encoding: [0xe7,0x00,0x00,0x00,0x20,0xa6]
+#CHECK: vmef    %v0, %v0, %v31          # encoding: [0xe7,0x00,0xf0,0x00,0x22,0xa6]
+#CHECK: vmef    %v0, %v31, %v0          # encoding: [0xe7,0x0f,0x00,0x00,0x24,0xa6]
+#CHECK: vmef    %v31, %v0, %v0          # encoding: [0xe7,0xf0,0x00,0x00,0x28,0xa6]
+#CHECK: vmef    %v18, %v3, %v20         # encoding: [0xe7,0x23,0x40,0x00,0x2a,0xa6]
+
+	vmef	%v0, %v0, %v0
+	vmef	%v0, %v0, %v31
+	vmef	%v0, %v31, %v0
+	vmef	%v31, %v0, %v0
+	vmef	%v18, %v3, %v20
+
+#CHECK: vmeh    %v0, %v0, %v0           # encoding: [0xe7,0x00,0x00,0x00,0x10,0xa6]
+#CHECK: vmeh    %v0, %v0, %v31          # encoding: [0xe7,0x00,0xf0,0x00,0x12,0xa6]
+#CHECK: vmeh    %v0, %v31, %v0          # encoding: [0xe7,0x0f,0x00,0x00,0x14,0xa6]
+#CHECK: vmeh    %v31, %v0, %v0          # encoding: [0xe7,0xf0,0x00,0x00,0x18,0xa6]
+#CHECK: vmeh    %v18, %v3, %v20         # encoding: [0xe7,0x23,0x40,0x00,0x1a,0xa6]
+
+	vmeh	%v0, %v0, %v0
+	vmeh	%v0, %v0, %v31
+	vmeh	%v0, %v31, %v0
+	vmeh	%v31, %v0, %v0
+	vmeh	%v18, %v3, %v20
+
+#CHECK: vmhb    %v0, %v0, %v0           # encoding: [0xe7,0x00,0x00,0x00,0x00,0xa3]
+#CHECK: vmhb    %v0, %v0, %v31          # encoding: [0xe7,0x00,0xf0,0x00,0x02,0xa3]
+#CHECK: vmhb    %v0, %v31, %v0          # encoding: [0xe7,0x0f,0x00,0x00,0x04,0xa3]
+#CHECK: vmhb    %v31, %v0, %v0          # encoding: [0xe7,0xf0,0x00,0x00,0x08,0xa3]
+#CHECK: vmhb    %v18, %v3, %v20         # encoding: [0xe7,0x23,0x40,0x00,0x0a,0xa3]
+
+	vmhb	%v0, %v0, %v0
+	vmhb	%v0, %v0, %v31
+	vmhb	%v0, %v31, %v0
+	vmhb	%v31, %v0, %v0
+	vmhb	%v18, %v3, %v20
+
+#CHECK: vmhf    %v0, %v0, %v0           # encoding: [0xe7,0x00,0x00,0x00,0x20,0xa3]
+#CHECK: vmhf    %v0, %v0, %v31          # encoding: [0xe7,0x00,0xf0,0x00,0x22,0xa3]
+#CHECK: vmhf    %v0, %v31, %v0          # encoding: [0xe7,0x0f,0x00,0x00,0x24,0xa3]
+#CHECK: vmhf    %v31, %v0, %v0          # encoding: [0xe7,0xf0,0x00,0x00,0x28,0xa3]
+#CHECK: vmhf    %v18, %v3, %v20         # encoding: [0xe7,0x23,0x40,0x00,0x2a,0xa3]
+
+	vmhf	%v0, %v0, %v0
+	vmhf	%v0, %v0, %v31
+	vmhf	%v0, %v31, %v0
+	vmhf	%v31, %v0, %v0
+	vmhf	%v18, %v3, %v20
+
+#CHECK: vmhh    %v0, %v0, %v0           # encoding: [0xe7,0x00,0x00,0x00,0x10,0xa3]
+#CHECK: vmhh    %v0, %v0, %v31          # encoding: [0xe7,0x00,0xf0,0x00,0x12,0xa3]
+#CHECK: vmhh    %v0, %v31, %v0          # encoding: [0xe7,0x0f,0x00,0x00,0x14,0xa3]
+#CHECK: vmhh    %v31, %v0, %v0          # encoding: [0xe7,0xf0,0x00,0x00,0x18,0xa3]
+#CHECK: vmhh    %v18, %v3, %v20         # encoding: [0xe7,0x23,0x40,0x00,0x1a,0xa3]
+
+	vmhh	%v0, %v0, %v0
+	vmhh	%v0, %v0, %v31
+	vmhh	%v0, %v31, %v0
+	vmhh	%v31, %v0, %v0
+	vmhh	%v18, %v3, %v20
+
+#CHECK: vmlb    %v0, %v0, %v0           # encoding: [0xe7,0x00,0x00,0x00,0x00,0xa2]
+#CHECK: vmlb    %v0, %v0, %v31          # encoding: [0xe7,0x00,0xf0,0x00,0x02,0xa2]
+#CHECK: vmlb    %v0, %v31, %v0          # encoding: [0xe7,0x0f,0x00,0x00,0x04,0xa2]
+#CHECK: vmlb    %v31, %v0, %v0          # encoding: [0xe7,0xf0,0x00,0x00,0x08,0xa2]
+#CHECK: vmlb    %v18, %v3, %v20         # encoding: [0xe7,0x23,0x40,0x00,0x0a,0xa2]
+
+	vmlb	%v0, %v0, %v0
+	vmlb	%v0, %v0, %v31
+	vmlb	%v0, %v31, %v0
+	vmlb	%v31, %v0, %v0
+	vmlb	%v18, %v3, %v20
+
+#CHECK: vmleb   %v0, %v0, %v0           # encoding: [0xe7,0x00,0x00,0x00,0x00,0xa4]
+#CHECK: vmleb   %v0, %v0, %v31          # encoding: [0xe7,0x00,0xf0,0x00,0x02,0xa4]
+#CHECK: vmleb   %v0, %v31, %v0          # encoding: [0xe7,0x0f,0x00,0x00,0x04,0xa4]
+#CHECK: vmleb   %v31, %v0, %v0          # encoding: [0xe7,0xf0,0x00,0x00,0x08,0xa4]
+#CHECK: vmleb   %v18, %v3, %v20         # encoding: [0xe7,0x23,0x40,0x00,0x0a,0xa4]
+
+	vmleb	%v0, %v0, %v0
+	vmleb	%v0, %v0, %v31
+	vmleb	%v0, %v31, %v0
+	vmleb	%v31, %v0, %v0
+	vmleb	%v18, %v3, %v20
+
+#CHECK: vmlef   %v0, %v0, %v0           # encoding: [0xe7,0x00,0x00,0x00,0x20,0xa4]
+#CHECK: vmlef   %v0, %v0, %v31          # encoding: [0xe7,0x00,0xf0,0x00,0x22,0xa4]
+#CHECK: vmlef   %v0, %v31, %v0          # encoding: [0xe7,0x0f,0x00,0x00,0x24,0xa4]
+#CHECK: vmlef   %v31, %v0, %v0          # encoding: [0xe7,0xf0,0x00,0x00,0x28,0xa4]
+#CHECK: vmlef   %v18, %v3, %v20         # encoding: [0xe7,0x23,0x40,0x00,0x2a,0xa4]
+
+	vmlef	%v0, %v0, %v0
+	vmlef	%v0, %v0, %v31
+	vmlef	%v0, %v31, %v0
+	vmlef	%v31, %v0, %v0
+	vmlef	%v18, %v3, %v20
+
+#CHECK: vmleh   %v0, %v0, %v0           # encoding: [0xe7,0x00,0x00,0x00,0x10,0xa4]
+#CHECK: vmleh   %v0, %v0, %v31          # encoding: [0xe7,0x00,0xf0,0x00,0x12,0xa4]
+#CHECK: vmleh   %v0, %v31, %v0          # encoding: [0xe7,0x0f,0x00,0x00,0x14,0xa4]
+#CHECK: vmleh   %v31, %v0, %v0          # encoding: [0xe7,0xf0,0x00,0x00,0x18,0xa4]
+#CHECK: vmleh   %v18, %v3, %v20         # encoding: [0xe7,0x23,0x40,0x00,0x1a,0xa4]
+
+	vmleh	%v0, %v0, %v0
+	vmleh	%v0, %v0, %v31
+	vmleh	%v0, %v31, %v0
+	vmleh	%v31, %v0, %v0
+	vmleh	%v18, %v3, %v20
+
+#CHECK: vmlf    %v0, %v0, %v0           # encoding: [0xe7,0x00,0x00,0x00,0x20,0xa2]
+#CHECK: vmlf    %v0, %v0, %v31          # encoding: [0xe7,0x00,0xf0,0x00,0x22,0xa2]
+#CHECK: vmlf    %v0, %v31, %v0          # encoding: [0xe7,0x0f,0x00,0x00,0x24,0xa2]
+#CHECK: vmlf    %v31, %v0, %v0          # encoding: [0xe7,0xf0,0x00,0x00,0x28,0xa2]
+#CHECK: vmlf    %v18, %v3, %v20         # encoding: [0xe7,0x23,0x40,0x00,0x2a,0xa2]
+
+	vmlf	%v0, %v0, %v0
+	vmlf	%v0, %v0, %v31
+	vmlf	%v0, %v31, %v0
+	vmlf	%v31, %v0, %v0
+	vmlf	%v18, %v3, %v20
+
+#CHECK: vmlhb   %v0, %v0, %v0           # encoding: [0xe7,0x00,0x00,0x00,0x00,0xa1]
+#CHECK: vmlhb   %v0, %v0, %v31          # encoding: [0xe7,0x00,0xf0,0x00,0x02,0xa1]
+#CHECK: vmlhb   %v0, %v31, %v0          # encoding: [0xe7,0x0f,0x00,0x00,0x04,0xa1]
+#CHECK: vmlhb   %v31, %v0, %v0          # encoding: [0xe7,0xf0,0x00,0x00,0x08,0xa1]
+#CHECK: vmlhb   %v18, %v3, %v20         # encoding: [0xe7,0x23,0x40,0x00,0x0a,0xa1]
+
+	vmlhb	%v0, %v0, %v0
+	vmlhb	%v0, %v0, %v31
+	vmlhb	%v0, %v31, %v0
+	vmlhb	%v31, %v0, %v0
+	vmlhb	%v18, %v3, %v20
+
+#CHECK: vmlhf   %v0, %v0, %v0           # encoding: [0xe7,0x00,0x00,0x00,0x20,0xa1]
+#CHECK: vmlhf   %v0, %v0, %v31          # encoding: [0xe7,0x00,0xf0,0x00,0x22,0xa1]
+#CHECK: vmlhf   %v0, %v31, %v0          # encoding: [0xe7,0x0f,0x00,0x00,0x24,0xa1]
+#CHECK: vmlhf   %v31, %v0, %v0          # encoding: [0xe7,0xf0,0x00,0x00,0x28,0xa1]
+#CHECK: vmlhf   %v18, %v3, %v20         # encoding: [0xe7,0x23,0x40,0x00,0x2a,0xa1]
+
+	vmlhf	%v0, %v0, %v0
+	vmlhf	%v0, %v0, %v31
+	vmlhf	%v0, %v31, %v0
+	vmlhf	%v31, %v0, %v0
+	vmlhf	%v18, %v3, %v20
+
+#CHECK: vmlhh   %v0, %v0, %v0           # encoding: [0xe7,0x00,0x00,0x00,0x10,0xa1]
+#CHECK: vmlhh   %v0, %v0, %v31          # encoding: [0xe7,0x00,0xf0,0x00,0x12,0xa1]
+#CHECK: vmlhh   %v0, %v31, %v0          # encoding: [0xe7,0x0f,0x00,0x00,0x14,0xa1]
+#CHECK: vmlhh   %v31, %v0, %v0          # encoding: [0xe7,0xf0,0x00,0x00,0x18,0xa1]
+#CHECK: vmlhh   %v18, %v3, %v20         # encoding: [0xe7,0x23,0x40,0x00,0x1a,0xa1]
+
+	vmlhh	%v0, %v0, %v0
+	vmlhh	%v0, %v0, %v31
+	vmlhh	%v0, %v31, %v0
+	vmlhh	%v31, %v0, %v0
+	vmlhh	%v18, %v3, %v20
+
+#CHECK: vmlhw   %v0, %v0, %v0           # encoding: [0xe7,0x00,0x00,0x00,0x10,0xa2]
+#CHECK: vmlhw   %v0, %v0, %v31          # encoding: [0xe7,0x00,0xf0,0x00,0x12,0xa2]
+#CHECK: vmlhw   %v0, %v31, %v0          # encoding: [0xe7,0x0f,0x00,0x00,0x14,0xa2]
+#CHECK: vmlhw   %v31, %v0, %v0          # encoding: [0xe7,0xf0,0x00,0x00,0x18,0xa2]
+#CHECK: vmlhw   %v18, %v3, %v20         # encoding: [0xe7,0x23,0x40,0x00,0x1a,0xa2]
+
+	vmlhw	%v0, %v0, %v0
+	vmlhw	%v0, %v0, %v31
+	vmlhw	%v0, %v31, %v0
+	vmlhw	%v31, %v0, %v0
+	vmlhw	%v18, %v3, %v20
+
+#CHECK: vmlob   %v0, %v0, %v0           # encoding: [0xe7,0x00,0x00,0x00,0x00,0xa5]
+#CHECK: vmlob   %v0, %v0, %v31          # encoding: [0xe7,0x00,0xf0,0x00,0x02,0xa5]
+#CHECK: vmlob   %v0, %v31, %v0          # encoding: [0xe7,0x0f,0x00,0x00,0x04,0xa5]
+#CHECK: vmlob   %v31, %v0, %v0          # encoding: [0xe7,0xf0,0x00,0x00,0x08,0xa5]
+#CHECK: vmlob   %v18, %v3, %v20         # encoding: [0xe7,0x23,0x40,0x00,0x0a,0xa5]
+
+	vmlob	%v0, %v0, %v0
+	vmlob	%v0, %v0, %v31
+	vmlob	%v0, %v31, %v0
+	vmlob	%v31, %v0, %v0
+	vmlob	%v18, %v3, %v20
+
+#CHECK: vmlof   %v0, %v0, %v0           # encoding: [0xe7,0x00,0x00,0x00,0x20,0xa5]
+#CHECK: vmlof   %v0, %v0, %v31          # encoding: [0xe7,0x00,0xf0,0x00,0x22,0xa5]
+#CHECK: vmlof   %v0, %v31, %v0          # encoding: [0xe7,0x0f,0x00,0x00,0x24,0xa5]
+#CHECK: vmlof   %v31, %v0, %v0          # encoding: [0xe7,0xf0,0x00,0x00,0x28,0xa5]
+#CHECK: vmlof   %v18, %v3, %v20         # encoding: [0xe7,0x23,0x40,0x00,0x2a,0xa5]
+
+	vmlof	%v0, %v0, %v0
+	vmlof	%v0, %v0, %v31
+	vmlof	%v0, %v31, %v0
+	vmlof	%v31, %v0, %v0
+	vmlof	%v18, %v3, %v20
+
+#CHECK: vmloh   %v0, %v0, %v0           # encoding: [0xe7,0x00,0x00,0x00,0x10,0xa5]
+#CHECK: vmloh   %v0, %v0, %v31          # encoding: [0xe7,0x00,0xf0,0x00,0x12,0xa5]
+#CHECK: vmloh   %v0, %v31, %v0          # encoding: [0xe7,0x0f,0x00,0x00,0x14,0xa5]
+#CHECK: vmloh   %v31, %v0, %v0          # encoding: [0xe7,0xf0,0x00,0x00,0x18,0xa5]
+#CHECK: vmloh   %v18, %v3, %v20         # encoding: [0xe7,0x23,0x40,0x00,0x1a,0xa5]
+
+	vmloh	%v0, %v0, %v0
+	vmloh	%v0, %v0, %v31
+	vmloh	%v0, %v31, %v0
+	vmloh	%v31, %v0, %v0
+	vmloh	%v18, %v3, %v20
+
+#CHECK: vmnb    %v0, %v0, %v0           # encoding: [0xe7,0x00,0x00,0x00,0x00,0xfe]
+#CHECK: vmnb    %v0, %v0, %v31          # encoding: [0xe7,0x00,0xf0,0x00,0x02,0xfe]
+#CHECK: vmnb    %v0, %v31, %v0          # encoding: [0xe7,0x0f,0x00,0x00,0x04,0xfe]
+#CHECK: vmnb    %v31, %v0, %v0          # encoding: [0xe7,0xf0,0x00,0x00,0x08,0xfe]
+#CHECK: vmnb    %v18, %v3, %v20         # encoding: [0xe7,0x23,0x40,0x00,0x0a,0xfe]
+
+	vmnb	%v0, %v0, %v0
+	vmnb	%v0, %v0, %v31
+	vmnb	%v0, %v31, %v0
+	vmnb	%v31, %v0, %v0
+	vmnb	%v18, %v3, %v20
+
+#CHECK: vmnf    %v0, %v0, %v0           # encoding: [0xe7,0x00,0x00,0x00,0x20,0xfe]
+#CHECK: vmnf    %v0, %v0, %v31          # encoding: [0xe7,0x00,0xf0,0x00,0x22,0xfe]
+#CHECK: vmnf    %v0, %v31, %v0          # encoding: [0xe7,0x0f,0x00,0x00,0x24,0xfe]
+#CHECK: vmnf    %v31, %v0, %v0          # encoding: [0xe7,0xf0,0x00,0x00,0x28,0xfe]
+#CHECK: vmnf    %v18, %v3, %v20         # encoding: [0xe7,0x23,0x40,0x00,0x2a,0xfe]
+
+	vmnf	%v0, %v0, %v0
+	vmnf	%v0, %v0, %v31
+	vmnf	%v0, %v31, %v0
+	vmnf	%v31, %v0, %v0
+	vmnf	%v18, %v3, %v20
+
+#CHECK: vmng    %v0, %v0, %v0           # encoding: [0xe7,0x00,0x00,0x00,0x30,0xfe]
+#CHECK: vmng    %v0, %v0, %v31          # encoding: [0xe7,0x00,0xf0,0x00,0x32,0xfe]
+#CHECK: vmng    %v0, %v31, %v0          # encoding: [0xe7,0x0f,0x00,0x00,0x34,0xfe]
+#CHECK: vmng    %v31, %v0, %v0          # encoding: [0xe7,0xf0,0x00,0x00,0x38,0xfe]
+#CHECK: vmng    %v18, %v3, %v20         # encoding: [0xe7,0x23,0x40,0x00,0x3a,0xfe]
+
+	vmng	%v0, %v0, %v0
+	vmng	%v0, %v0, %v31
+	vmng	%v0, %v31, %v0
+	vmng	%v31, %v0, %v0
+	vmng	%v18, %v3, %v20
+
+#CHECK: vmnh    %v0, %v0, %v0           # encoding: [0xe7,0x00,0x00,0x00,0x10,0xfe]
+#CHECK: vmnh    %v0, %v0, %v31          # encoding: [0xe7,0x00,0xf0,0x00,0x12,0xfe]
+#CHECK: vmnh    %v0, %v31, %v0          # encoding: [0xe7,0x0f,0x00,0x00,0x14,0xfe]
+#CHECK: vmnh    %v31, %v0, %v0          # encoding: [0xe7,0xf0,0x00,0x00,0x18,0xfe]
+#CHECK: vmnh    %v18, %v3, %v20         # encoding: [0xe7,0x23,0x40,0x00,0x1a,0xfe]
+
+	vmnh	%v0, %v0, %v0
+	vmnh	%v0, %v0, %v31
+	vmnh	%v0, %v31, %v0
+	vmnh	%v31, %v0, %v0
+	vmnh	%v18, %v3, %v20
+
+#CHECK: vmnlb   %v0, %v0, %v0           # encoding: [0xe7,0x00,0x00,0x00,0x00,0xfc]
+#CHECK: vmnlb   %v0, %v0, %v31          # encoding: [0xe7,0x00,0xf0,0x00,0x02,0xfc]
+#CHECK: vmnlb   %v0, %v31, %v0          # encoding: [0xe7,0x0f,0x00,0x00,0x04,0xfc]
+#CHECK: vmnlb   %v31, %v0, %v0          # encoding: [0xe7,0xf0,0x00,0x00,0x08,0xfc]
+#CHECK: vmnlb   %v18, %v3, %v20         # encoding: [0xe7,0x23,0x40,0x00,0x0a,0xfc]
+
+	vmnlb	%v0, %v0, %v0
+	vmnlb	%v0, %v0, %v31
+	vmnlb	%v0, %v31, %v0
+	vmnlb	%v31, %v0, %v0
+	vmnlb	%v18, %v3, %v20
+
+#CHECK: vmnlf   %v0, %v0, %v0           # encoding: [0xe7,0x00,0x00,0x00,0x20,0xfc]
+#CHECK: vmnlf   %v0, %v0, %v31          # encoding: [0xe7,0x00,0xf0,0x00,0x22,0xfc]
+#CHECK: vmnlf   %v0, %v31, %v0          # encoding: [0xe7,0x0f,0x00,0x00,0x24,0xfc]
+#CHECK: vmnlf   %v31, %v0, %v0          # encoding: [0xe7,0xf0,0x00,0x00,0x28,0xfc]
+#CHECK: vmnlf   %v18, %v3, %v20         # encoding: [0xe7,0x23,0x40,0x00,0x2a,0xfc]
+
+	vmnlf	%v0, %v0, %v0
+	vmnlf	%v0, %v0, %v31
+	vmnlf	%v0, %v31, %v0
+	vmnlf	%v31, %v0, %v0
+	vmnlf	%v18, %v3, %v20
+
+#CHECK: vmnlg   %v0, %v0, %v0           # encoding: [0xe7,0x00,0x00,0x00,0x30,0xfc]
+#CHECK: vmnlg   %v0, %v0, %v31          # encoding: [0xe7,0x00,0xf0,0x00,0x32,0xfc]
+#CHECK: vmnlg   %v0, %v31, %v0          # encoding: [0xe7,0x0f,0x00,0x00,0x34,0xfc]
+#CHECK: vmnlg   %v31, %v0, %v0          # encoding: [0xe7,0xf0,0x00,0x00,0x38,0xfc]
+#CHECK: vmnlg   %v18, %v3, %v20         # encoding: [0xe7,0x23,0x40,0x00,0x3a,0xfc]
+
+	vmnlg	%v0, %v0, %v0
+	vmnlg	%v0, %v0, %v31
+	vmnlg	%v0, %v31, %v0
+	vmnlg	%v31, %v0, %v0
+	vmnlg	%v18, %v3, %v20
+
+#CHECK: vmnlh   %v0, %v0, %v0           # encoding: [0xe7,0x00,0x00,0x00,0x10,0xfc]
+#CHECK: vmnlh   %v0, %v0, %v31          # encoding: [0xe7,0x00,0xf0,0x00,0x12,0xfc]
+#CHECK: vmnlh   %v0, %v31, %v0          # encoding: [0xe7,0x0f,0x00,0x00,0x14,0xfc]
+#CHECK: vmnlh   %v31, %v0, %v0          # encoding: [0xe7,0xf0,0x00,0x00,0x18,0xfc]
+#CHECK: vmnlh   %v18, %v3, %v20         # encoding: [0xe7,0x23,0x40,0x00,0x1a,0xfc]
+
+	vmnlh	%v0, %v0, %v0
+	vmnlh	%v0, %v0, %v31
+	vmnlh	%v0, %v31, %v0
+	vmnlh	%v31, %v0, %v0
+	vmnlh	%v18, %v3, %v20
+
+#CHECK: vmob    %v0, %v0, %v0           # encoding: [0xe7,0x00,0x00,0x00,0x00,0xa7]
+#CHECK: vmob    %v0, %v0, %v31          # encoding: [0xe7,0x00,0xf0,0x00,0x02,0xa7]
+#CHECK: vmob    %v0, %v31, %v0          # encoding: [0xe7,0x0f,0x00,0x00,0x04,0xa7]
+#CHECK: vmob    %v31, %v0, %v0          # encoding: [0xe7,0xf0,0x00,0x00,0x08,0xa7]
+#CHECK: vmob    %v18, %v3, %v20         # encoding: [0xe7,0x23,0x40,0x00,0x0a,0xa7]
+
+	vmob	%v0, %v0, %v0
+	vmob	%v0, %v0, %v31
+	vmob	%v0, %v31, %v0
+	vmob	%v31, %v0, %v0
+	vmob	%v18, %v3, %v20
+
+#CHECK: vmof    %v0, %v0, %v0           # encoding: [0xe7,0x00,0x00,0x00,0x20,0xa7]
+#CHECK: vmof    %v0, %v0, %v31          # encoding: [0xe7,0x00,0xf0,0x00,0x22,0xa7]
+#CHECK: vmof    %v0, %v31, %v0          # encoding: [0xe7,0x0f,0x00,0x00,0x24,0xa7]
+#CHECK: vmof    %v31, %v0, %v0          # encoding: [0xe7,0xf0,0x00,0x00,0x28,0xa7]
+#CHECK: vmof    %v18, %v3, %v20         # encoding: [0xe7,0x23,0x40,0x00,0x2a,0xa7]
+
+	vmof	%v0, %v0, %v0
+	vmof	%v0, %v0, %v31
+	vmof	%v0, %v31, %v0
+	vmof	%v31, %v0, %v0
+	vmof	%v18, %v3, %v20
+
+#CHECK: vmoh    %v0, %v0, %v0           # encoding: [0xe7,0x00,0x00,0x00,0x10,0xa7]
+#CHECK: vmoh    %v0, %v0, %v31          # encoding: [0xe7,0x00,0xf0,0x00,0x12,0xa7]
+#CHECK: vmoh    %v0, %v31, %v0          # encoding: [0xe7,0x0f,0x00,0x00,0x14,0xa7]
+#CHECK: vmoh    %v31, %v0, %v0          # encoding: [0xe7,0xf0,0x00,0x00,0x18,0xa7]
+#CHECK: vmoh    %v18, %v3, %v20         # encoding: [0xe7,0x23,0x40,0x00,0x1a,0xa7]
+
+	vmoh	%v0, %v0, %v0
+	vmoh	%v0, %v0, %v31
+	vmoh	%v0, %v31, %v0
+	vmoh	%v31, %v0, %v0
+	vmoh	%v18, %v3, %v20
+
+#CHECK: vmrhb   %v0, %v0, %v0           # encoding: [0xe7,0x00,0x00,0x00,0x00,0x61]
+#CHECK: vmrhb   %v0, %v0, %v15          # encoding: [0xe7,0x00,0xf0,0x00,0x00,0x61]
+#CHECK: vmrhb   %v0, %v0, %v31          # encoding: [0xe7,0x00,0xf0,0x00,0x02,0x61]
+#CHECK: vmrhb   %v0, %v15, %v0          # encoding: [0xe7,0x0f,0x00,0x00,0x00,0x61]
+#CHECK: vmrhb   %v0, %v31, %v0          # encoding: [0xe7,0x0f,0x00,0x00,0x04,0x61]
+#CHECK: vmrhb   %v15, %v0, %v0          # encoding: [0xe7,0xf0,0x00,0x00,0x00,0x61]
+#CHECK: vmrhb   %v31, %v0, %v0          # encoding: [0xe7,0xf0,0x00,0x00,0x08,0x61]
+#CHECK: vmrhb   %v18, %v3, %v20         # encoding: [0xe7,0x23,0x40,0x00,0x0a,0x61]
+
+	vmrhb	%v0, %v0, %v0
+	vmrhb	%v0, %v0, %v15
+	vmrhb	%v0, %v0, %v31
+	vmrhb	%v0, %v15, %v0
+	vmrhb	%v0, %v31, %v0
+	vmrhb	%v15, %v0, %v0
+	vmrhb	%v31, %v0, %v0
+	vmrhb	%v18, %v3, %v20
+
+#CHECK: vmrhf   %v0, %v0, %v0           # encoding: [0xe7,0x00,0x00,0x00,0x20,0x61]
+#CHECK: vmrhf   %v0, %v0, %v15          # encoding: [0xe7,0x00,0xf0,0x00,0x20,0x61]
+#CHECK: vmrhf   %v0, %v0, %v31          # encoding: [0xe7,0x00,0xf0,0x00,0x22,0x61]
+#CHECK: vmrhf   %v0, %v15, %v0          # encoding: [0xe7,0x0f,0x00,0x00,0x20,0x61]
+#CHECK: vmrhf   %v0, %v31, %v0          # encoding: [0xe7,0x0f,0x00,0x00,0x24,0x61]
+#CHECK: vmrhf   %v15, %v0, %v0          # encoding: [0xe7,0xf0,0x00,0x00,0x20,0x61]
+#CHECK: vmrhf   %v31, %v0, %v0          # encoding: [0xe7,0xf0,0x00,0x00,0x28,0x61]
+#CHECK: vmrhf   %v18, %v3, %v20         # encoding: [0xe7,0x23,0x40,0x00,0x2a,0x61]
+
+	vmrhf	%v0, %v0, %v0
+	vmrhf	%v0, %v0, %v15
+	vmrhf	%v0, %v0, %v31
+	vmrhf	%v0, %v15, %v0
+	vmrhf	%v0, %v31, %v0
+	vmrhf	%v15, %v0, %v0
+	vmrhf	%v31, %v0, %v0
+	vmrhf	%v18, %v3, %v20
+
+#CHECK: vmrhg   %v0, %v0, %v0           # encoding: [0xe7,0x00,0x00,0x00,0x30,0x61]
+#CHECK: vmrhg   %v0, %v0, %v15          # encoding: [0xe7,0x00,0xf0,0x00,0x30,0x61]
+#CHECK: vmrhg   %v0, %v0, %v31          # encoding: [0xe7,0x00,0xf0,0x00,0x32,0x61]
+#CHECK: vmrhg   %v0, %v15, %v0          # encoding: [0xe7,0x0f,0x00,0x00,0x30,0x61]
+#CHECK: vmrhg   %v0, %v31, %v0          # encoding: [0xe7,0x0f,0x00,0x00,0x34,0x61]
+#CHECK: vmrhg   %v15, %v0, %v0          # encoding: [0xe7,0xf0,0x00,0x00,0x30,0x61]
+#CHECK: vmrhg   %v31, %v0, %v0          # encoding: [0xe7,0xf0,0x00,0x00,0x38,0x61]
+#CHECK: vmrhg   %v18, %v3, %v20         # encoding: [0xe7,0x23,0x40,0x00,0x3a,0x61]
+
+	vmrhg	%v0, %v0, %v0
+	vmrhg	%v0, %v0, %v15
+	vmrhg	%v0, %v0, %v31
+	vmrhg	%v0, %v15, %v0
+	vmrhg	%v0, %v31, %v0
+	vmrhg	%v15, %v0, %v0
+	vmrhg	%v31, %v0, %v0
+	vmrhg	%v18, %v3, %v20
+
+#CHECK: vmrhh   %v0, %v0, %v0           # encoding: [0xe7,0x00,0x00,0x00,0x10,0x61]
+#CHECK: vmrhh   %v0, %v0, %v15          # encoding: [0xe7,0x00,0xf0,0x00,0x10,0x61]
+#CHECK: vmrhh   %v0, %v0, %v31          # encoding: [0xe7,0x00,0xf0,0x00,0x12,0x61]
+#CHECK: vmrhh   %v0, %v15, %v0          # encoding: [0xe7,0x0f,0x00,0x00,0x10,0x61]
+#CHECK: vmrhh   %v0, %v31, %v0          # encoding: [0xe7,0x0f,0x00,0x00,0x14,0x61]
+#CHECK: vmrhh   %v15, %v0, %v0          # encoding: [0xe7,0xf0,0x00,0x00,0x10,0x61]
+#CHECK: vmrhh   %v31, %v0, %v0          # encoding: [0xe7,0xf0,0x00,0x00,0x18,0x61]
+#CHECK: vmrhh   %v18, %v3, %v20         # encoding: [0xe7,0x23,0x40,0x00,0x1a,0x61]
+
+	vmrhh	%v0, %v0, %v0
+	vmrhh	%v0, %v0, %v15
+	vmrhh	%v0, %v0, %v31
+	vmrhh	%v0, %v15, %v0
+	vmrhh	%v0, %v31, %v0
+	vmrhh	%v15, %v0, %v0
+	vmrhh	%v31, %v0, %v0
+	vmrhh	%v18, %v3, %v20
+
+#CHECK: vmrlb   %v0, %v0, %v0           # encoding: [0xe7,0x00,0x00,0x00,0x00,0x60]
+#CHECK: vmrlb   %v0, %v0, %v15          # encoding: [0xe7,0x00,0xf0,0x00,0x00,0x60]
+#CHECK: vmrlb   %v0, %v0, %v31          # encoding: [0xe7,0x00,0xf0,0x00,0x02,0x60]
+#CHECK: vmrlb   %v0, %v15, %v0          # encoding: [0xe7,0x0f,0x00,0x00,0x00,0x60]
+#CHECK: vmrlb   %v0, %v31, %v0          # encoding: [0xe7,0x0f,0x00,0x00,0x04,0x60]
+#CHECK: vmrlb   %v15, %v0, %v0          # encoding: [0xe7,0xf0,0x00,0x00,0x00,0x60]
+#CHECK: vmrlb   %v31, %v0, %v0          # encoding: [0xe7,0xf0,0x00,0x00,0x08,0x60]
+#CHECK: vmrlb   %v18, %v3, %v20         # encoding: [0xe7,0x23,0x40,0x00,0x0a,0x60]
+
+	vmrlb	%v0, %v0, %v0
+	vmrlb	%v0, %v0, %v15
+	vmrlb	%v0, %v0, %v31
+	vmrlb	%v0, %v15, %v0
+	vmrlb	%v0, %v31, %v0
+	vmrlb	%v15, %v0, %v0
+	vmrlb	%v31, %v0, %v0
+	vmrlb	%v18, %v3, %v20
+
+#CHECK: vmrlf   %v0, %v0, %v0           # encoding: [0xe7,0x00,0x00,0x00,0x20,0x60]
+#CHECK: vmrlf   %v0, %v0, %v15          # encoding: [0xe7,0x00,0xf0,0x00,0x20,0x60]
+#CHECK: vmrlf   %v0, %v0, %v31          # encoding: [0xe7,0x00,0xf0,0x00,0x22,0x60]
+#CHECK: vmrlf   %v0, %v15, %v0          # encoding: [0xe7,0x0f,0x00,0x00,0x20,0x60]
+#CHECK: vmrlf   %v0, %v31, %v0          # encoding: [0xe7,0x0f,0x00,0x00,0x24,0x60]
+#CHECK: vmrlf   %v15, %v0, %v0          # encoding: [0xe7,0xf0,0x00,0x00,0x20,0x60]
+#CHECK: vmrlf   %v31, %v0, %v0          # encoding: [0xe7,0xf0,0x00,0x00,0x28,0x60]
+#CHECK: vmrlf   %v18, %v3, %v20         # encoding: [0xe7,0x23,0x40,0x00,0x2a,0x60]
+
+	vmrlf	%v0, %v0, %v0
+	vmrlf	%v0, %v0, %v15
+	vmrlf	%v0, %v0, %v31
+	vmrlf	%v0, %v15, %v0
+	vmrlf	%v0, %v31, %v0
+	vmrlf	%v15, %v0, %v0
+	vmrlf	%v31, %v0, %v0
+	vmrlf	%v18, %v3, %v20
+
+#CHECK: vmrlg   %v0, %v0, %v0           # encoding: [0xe7,0x00,0x00,0x00,0x30,0x60]
+#CHECK: vmrlg   %v0, %v0, %v15          # encoding: [0xe7,0x00,0xf0,0x00,0x30,0x60]
+#CHECK: vmrlg   %v0, %v0, %v31          # encoding: [0xe7,0x00,0xf0,0x00,0x32,0x60]
+#CHECK: vmrlg   %v0, %v15, %v0          # encoding: [0xe7,0x0f,0x00,0x00,0x30,0x60]
+#CHECK: vmrlg   %v0, %v31, %v0          # encoding: [0xe7,0x0f,0x00,0x00,0x34,0x60]
+#CHECK: vmrlg   %v15, %v0, %v0          # encoding: [0xe7,0xf0,0x00,0x00,0x30,0x60]
+#CHECK: vmrlg   %v31, %v0, %v0          # encoding: [0xe7,0xf0,0x00,0x00,0x38,0x60]
+#CHECK: vmrlg   %v18, %v3, %v20         # encoding: [0xe7,0x23,0x40,0x00,0x3a,0x60]
+
+	vmrlg	%v0, %v0, %v0
+	vmrlg	%v0, %v0, %v15
+	vmrlg	%v0, %v0, %v31
+	vmrlg	%v0, %v15, %v0
+	vmrlg	%v0, %v31, %v0
+	vmrlg	%v15, %v0, %v0
+	vmrlg	%v31, %v0, %v0
+	vmrlg	%v18, %v3, %v20
+
+#CHECK: vmrlh   %v0, %v0, %v0           # encoding: [0xe7,0x00,0x00,0x00,0x10,0x60]
+#CHECK: vmrlh   %v0, %v0, %v15          # encoding: [0xe7,0x00,0xf0,0x00,0x10,0x60]
+#CHECK: vmrlh   %v0, %v0, %v31          # encoding: [0xe7,0x00,0xf0,0x00,0x12,0x60]
+#CHECK: vmrlh   %v0, %v15, %v0          # encoding: [0xe7,0x0f,0x00,0x00,0x10,0x60]
+#CHECK: vmrlh   %v0, %v31, %v0          # encoding: [0xe7,0x0f,0x00,0x00,0x14,0x60]
+#CHECK: vmrlh   %v15, %v0, %v0          # encoding: [0xe7,0xf0,0x00,0x00,0x10,0x60]
+#CHECK: vmrlh   %v31, %v0, %v0          # encoding: [0xe7,0xf0,0x00,0x00,0x18,0x60]
+#CHECK: vmrlh   %v18, %v3, %v20         # encoding: [0xe7,0x23,0x40,0x00,0x1a,0x60]
+
+	vmrlh	%v0, %v0, %v0
+	vmrlh	%v0, %v0, %v15
+	vmrlh	%v0, %v0, %v31
+	vmrlh	%v0, %v15, %v0
+	vmrlh	%v0, %v31, %v0
+	vmrlh	%v15, %v0, %v0
+	vmrlh	%v31, %v0, %v0
+	vmrlh	%v18, %v3, %v20
+
+#CHECK: vmxb    %v0, %v0, %v0           # encoding: [0xe7,0x00,0x00,0x00,0x00,0xff]
+#CHECK: vmxb    %v0, %v0, %v31          # encoding: [0xe7,0x00,0xf0,0x00,0x02,0xff]
+#CHECK: vmxb    %v0, %v31, %v0          # encoding: [0xe7,0x0f,0x00,0x00,0x04,0xff]
+#CHECK: vmxb    %v31, %v0, %v0          # encoding: [0xe7,0xf0,0x00,0x00,0x08,0xff]
+#CHECK: vmxb    %v18, %v3, %v20         # encoding: [0xe7,0x23,0x40,0x00,0x0a,0xff]
+
+	vmxb	%v0, %v0, %v0
+	vmxb	%v0, %v0, %v31
+	vmxb	%v0, %v31, %v0
+	vmxb	%v31, %v0, %v0
+	vmxb	%v18, %v3, %v20
+
+#CHECK: vmxf    %v0, %v0, %v0           # encoding: [0xe7,0x00,0x00,0x00,0x20,0xff]
+#CHECK: vmxf    %v0, %v0, %v31          # encoding: [0xe7,0x00,0xf0,0x00,0x22,0xff]
+#CHECK: vmxf    %v0, %v31, %v0          # encoding: [0xe7,0x0f,0x00,0x00,0x24,0xff]
+#CHECK: vmxf    %v31, %v0, %v0          # encoding: [0xe7,0xf0,0x00,0x00,0x28,0xff]
+#CHECK: vmxf    %v18, %v3, %v20         # encoding: [0xe7,0x23,0x40,0x00,0x2a,0xff]
+
+	vmxf	%v0, %v0, %v0
+	vmxf	%v0, %v0, %v31
+	vmxf	%v0, %v31, %v0
+	vmxf	%v31, %v0, %v0
+	vmxf	%v18, %v3, %v20
+
+#CHECK: vmxg    %v0, %v0, %v0           # encoding: [0xe7,0x00,0x00,0x00,0x30,0xff]
+#CHECK: vmxg    %v0, %v0, %v31          # encoding: [0xe7,0x00,0xf0,0x00,0x32,0xff]
+#CHECK: vmxg    %v0, %v31, %v0          # encoding: [0xe7,0x0f,0x00,0x00,0x34,0xff]
+#CHECK: vmxg    %v31, %v0, %v0          # encoding: [0xe7,0xf0,0x00,0x00,0x38,0xff]
+#CHECK: vmxg    %v18, %v3, %v20         # encoding: [0xe7,0x23,0x40,0x00,0x3a,0xff]
+
+	vmxg	%v0, %v0, %v0
+	vmxg	%v0, %v0, %v31
+	vmxg	%v0, %v31, %v0
+	vmxg	%v31, %v0, %v0
+	vmxg	%v18, %v3, %v20
+
+#CHECK: vmxh    %v0, %v0, %v0           # encoding: [0xe7,0x00,0x00,0x00,0x10,0xff]
+#CHECK: vmxh    %v0, %v0, %v31          # encoding: [0xe7,0x00,0xf0,0x00,0x12,0xff]
+#CHECK: vmxh    %v0, %v31, %v0          # encoding: [0xe7,0x0f,0x00,0x00,0x14,0xff]
+#CHECK: vmxh    %v31, %v0, %v0          # encoding: [0xe7,0xf0,0x00,0x00,0x18,0xff]
+#CHECK: vmxh    %v18, %v3, %v20         # encoding: [0xe7,0x23,0x40,0x00,0x1a,0xff]
+
+	vmxh	%v0, %v0, %v0
+	vmxh	%v0, %v0, %v31
+	vmxh	%v0, %v31, %v0
+	vmxh	%v31, %v0, %v0
+	vmxh	%v18, %v3, %v20
+
+#CHECK: vmxlb   %v0, %v0, %v0           # encoding: [0xe7,0x00,0x00,0x00,0x00,0xfd]
+#CHECK: vmxlb   %v0, %v0, %v31          # encoding: [0xe7,0x00,0xf0,0x00,0x02,0xfd]
+#CHECK: vmxlb   %v0, %v31, %v0          # encoding: [0xe7,0x0f,0x00,0x00,0x04,0xfd]
+#CHECK: vmxlb   %v31, %v0, %v0          # encoding: [0xe7,0xf0,0x00,0x00,0x08,0xfd]
+#CHECK: vmxlb   %v18, %v3, %v20         # encoding: [0xe7,0x23,0x40,0x00,0x0a,0xfd]
+
+	vmxlb	%v0, %v0, %v0
+	vmxlb	%v0, %v0, %v31
+	vmxlb	%v0, %v31, %v0
+	vmxlb	%v31, %v0, %v0
+	vmxlb	%v18, %v3, %v20
+
+#CHECK: vmxlf   %v0, %v0, %v0           # encoding: [0xe7,0x00,0x00,0x00,0x20,0xfd]
+#CHECK: vmxlf   %v0, %v0, %v31          # encoding: [0xe7,0x00,0xf0,0x00,0x22,0xfd]
+#CHECK: vmxlf   %v0, %v31, %v0          # encoding: [0xe7,0x0f,0x00,0x00,0x24,0xfd]
+#CHECK: vmxlf   %v31, %v0, %v0          # encoding: [0xe7,0xf0,0x00,0x00,0x28,0xfd]
+#CHECK: vmxlf   %v18, %v3, %v20         # encoding: [0xe7,0x23,0x40,0x00,0x2a,0xfd]
+
+	vmxlf	%v0, %v0, %v0
+	vmxlf	%v0, %v0, %v31
+	vmxlf	%v0, %v31, %v0
+	vmxlf	%v31, %v0, %v0
+	vmxlf	%v18, %v3, %v20
+
+#CHECK: vmxlg   %v0, %v0, %v0           # encoding: [0xe7,0x00,0x00,0x00,0x30,0xfd]
+#CHECK: vmxlg   %v0, %v0, %v31          # encoding: [0xe7,0x00,0xf0,0x00,0x32,0xfd]
+#CHECK: vmxlg   %v0, %v31, %v0          # encoding: [0xe7,0x0f,0x00,0x00,0x34,0xfd]
+#CHECK: vmxlg   %v31, %v0, %v0          # encoding: [0xe7,0xf0,0x00,0x00,0x38,0xfd]
+#CHECK: vmxlg   %v18, %v3, %v20         # encoding: [0xe7,0x23,0x40,0x00,0x3a,0xfd]
+
+	vmxlg	%v0, %v0, %v0
+	vmxlg	%v0, %v0, %v31
+	vmxlg	%v0, %v31, %v0
+	vmxlg	%v31, %v0, %v0
+	vmxlg	%v18, %v3, %v20
+
+#CHECK: vmxlh   %v0, %v0, %v0           # encoding: [0xe7,0x00,0x00,0x00,0x10,0xfd]
+#CHECK: vmxlh   %v0, %v0, %v31          # encoding: [0xe7,0x00,0xf0,0x00,0x12,0xfd]
+#CHECK: vmxlh   %v0, %v31, %v0          # encoding: [0xe7,0x0f,0x00,0x00,0x14,0xfd]
+#CHECK: vmxlh   %v31, %v0, %v0          # encoding: [0xe7,0xf0,0x00,0x00,0x18,0xfd]
+#CHECK: vmxlh   %v18, %v3, %v20         # encoding: [0xe7,0x23,0x40,0x00,0x1a,0xfd]
+
+	vmxlh	%v0, %v0, %v0
+	vmxlh	%v0, %v0, %v31
+	vmxlh	%v0, %v31, %v0
+	vmxlh	%v31, %v0, %v0
+	vmxlh	%v18, %v3, %v20
+
+#CHECK: vn      %v0, %v0, %v0           # encoding: [0xe7,0x00,0x00,0x00,0x00,0x68]
+#CHECK: vn      %v0, %v0, %v31          # encoding: [0xe7,0x00,0xf0,0x00,0x02,0x68]
+#CHECK: vn      %v0, %v31, %v0          # encoding: [0xe7,0x0f,0x00,0x00,0x04,0x68]
+#CHECK: vn      %v31, %v0, %v0          # encoding: [0xe7,0xf0,0x00,0x00,0x08,0x68]
+#CHECK: vn      %v18, %v3, %v20         # encoding: [0xe7,0x23,0x40,0x00,0x0a,0x68]
+
+	vn	%v0, %v0, %v0
+	vn	%v0, %v0, %v31
+	vn	%v0, %v31, %v0
+	vn	%v31, %v0, %v0
+	vn	%v18, %v3, %v20
+
+#CHECK: vnc     %v0, %v0, %v0           # encoding: [0xe7,0x00,0x00,0x00,0x00,0x69]
+#CHECK: vnc     %v0, %v0, %v31          # encoding: [0xe7,0x00,0xf0,0x00,0x02,0x69]
+#CHECK: vnc     %v0, %v31, %v0          # encoding: [0xe7,0x0f,0x00,0x00,0x04,0x69]
+#CHECK: vnc     %v31, %v0, %v0          # encoding: [0xe7,0xf0,0x00,0x00,0x08,0x69]
+#CHECK: vnc     %v18, %v3, %v20         # encoding: [0xe7,0x23,0x40,0x00,0x0a,0x69]
+
+	vnc	%v0, %v0, %v0
+	vnc	%v0, %v0, %v31
+	vnc	%v0, %v31, %v0
+	vnc	%v31, %v0, %v0
+	vnc	%v18, %v3, %v20
+
+#CHECK: vno     %v0, %v0, %v0           # encoding: [0xe7,0x00,0x00,0x00,0x00,0x6b]
+#CHECK: vno     %v0, %v0, %v31          # encoding: [0xe7,0x00,0xf0,0x00,0x02,0x6b]
+#CHECK: vno     %v0, %v31, %v0          # encoding: [0xe7,0x0f,0x00,0x00,0x04,0x6b]
+#CHECK: vno     %v31, %v0, %v0          # encoding: [0xe7,0xf0,0x00,0x00,0x08,0x6b]
+#CHECK: vno     %v18, %v3, %v20         # encoding: [0xe7,0x23,0x40,0x00,0x0a,0x6b]
+
+	vno	%v0, %v0, %v0
+	vno	%v0, %v0, %v31
+	vno	%v0, %v31, %v0
+	vno	%v31, %v0, %v0
+	vno	%v18, %v3, %v20
+
+#CHECK: vo      %v0, %v0, %v0           # encoding: [0xe7,0x00,0x00,0x00,0x00,0x6a]
+#CHECK: vo      %v0, %v0, %v31          # encoding: [0xe7,0x00,0xf0,0x00,0x02,0x6a]
+#CHECK: vo      %v0, %v31, %v0          # encoding: [0xe7,0x0f,0x00,0x00,0x04,0x6a]
+#CHECK: vo      %v31, %v0, %v0          # encoding: [0xe7,0xf0,0x00,0x00,0x08,0x6a]
+#CHECK: vo      %v18, %v3, %v20         # encoding: [0xe7,0x23,0x40,0x00,0x0a,0x6a]
+
+	vo	%v0, %v0, %v0
+	vo	%v0, %v0, %v31
+	vo	%v0, %v31, %v0
+	vo	%v31, %v0, %v0
+	vo	%v18, %v3, %v20
+
+#CHECK: vone    %v0                     # encoding: [0xe7,0x00,0xff,0xff,0x00,0x44]
+#CHECK: vone    %v15                    # encoding: [0xe7,0xf0,0xff,0xff,0x00,0x44]
+#CHECK: vone    %v22                    # encoding: [0xe7,0x60,0xff,0xff,0x08,0x44]
+#CHECK: vone    %v31                    # encoding: [0xe7,0xf0,0xff,0xff,0x08,0x44]
+
+	vone	%v0
+	vone	%v15
+	vone	%v22
+	vone	%v31
+
+#CHECK: vpdi    %v0, %v0, %v0, 0        # encoding: [0xe7,0x00,0x00,0x00,0x00,0x84]
+#CHECK: vpdi    %v0, %v0, %v0, 5        # encoding: [0xe7,0x00,0x00,0x00,0x50,0x84]
+#CHECK: vpdi    %v0, %v0, %v31, 0       # encoding: [0xe7,0x00,0xf0,0x00,0x02,0x84]
+#CHECK: vpdi    %v0, %v31, %v0, 0       # encoding: [0xe7,0x0f,0x00,0x00,0x04,0x84]
+#CHECK: vpdi    %v31, %v0, %v0, 0       # encoding: [0xe7,0xf0,0x00,0x00,0x08,0x84]
+#CHECK: vpdi    %v13, %v17, %v21, 4     # encoding: [0xe7,0xd1,0x50,0x00,0x46,0x84]
+
+	vpdi	%v0, %v0, %v0, 0
+	vpdi	%v0, %v0, %v0, 5
+	vpdi	%v0, %v0, %v31, 0
+	vpdi	%v0, %v31, %v0, 0
+	vpdi	%v31, %v0, %v0, 0
+	vpdi	%v13, %v17, %v21, 4
+
+#CHECK: vperm   %v0, %v0, %v0, %v0      # encoding: [0xe7,0x00,0x00,0x00,0x00,0x8c]
+#CHECK: vperm   %v0, %v0, %v0, %v31     # encoding: [0xe7,0x00,0x00,0x00,0xf1,0x8c]
+#CHECK: vperm   %v0, %v0, %v31, %v0     # encoding: [0xe7,0x00,0xf0,0x00,0x02,0x8c]
+#CHECK: vperm   %v0, %v31, %v0, %v0     # encoding: [0xe7,0x0f,0x00,0x00,0x04,0x8c]
+#CHECK: vperm   %v31, %v0, %v0, %v0     # encoding: [0xe7,0xf0,0x00,0x00,0x08,0x8c]
+#CHECK: vperm   %v13, %v17, %v21, %v25  # encoding: [0xe7,0xd1,0x50,0x00,0x97,0x8c]
+
+	vperm	%v0, %v0, %v0, %v0
+	vperm	%v0, %v0, %v0, %v31
+	vperm	%v0, %v0, %v31, %v0
+	vperm	%v0, %v31, %v0, %v0
+	vperm	%v31, %v0, %v0, %v0
+	vperm	%v13, %v17, %v21, %v25
+
+#CHECK: vpkf    %v0, %v0, %v0           # encoding: [0xe7,0x00,0x00,0x00,0x20,0x94]
+#CHECK: vpkf    %v0, %v0, %v15          # encoding: [0xe7,0x00,0xf0,0x00,0x20,0x94]
+#CHECK: vpkf    %v0, %v0, %v31          # encoding: [0xe7,0x00,0xf0,0x00,0x22,0x94]
+#CHECK: vpkf    %v0, %v15, %v0          # encoding: [0xe7,0x0f,0x00,0x00,0x20,0x94]
+#CHECK: vpkf    %v0, %v31, %v0          # encoding: [0xe7,0x0f,0x00,0x00,0x24,0x94]
+#CHECK: vpkf    %v15, %v0, %v0          # encoding: [0xe7,0xf0,0x00,0x00,0x20,0x94]
+#CHECK: vpkf    %v31, %v0, %v0          # encoding: [0xe7,0xf0,0x00,0x00,0x28,0x94]
+#CHECK: vpkf    %v18, %v3, %v20         # encoding: [0xe7,0x23,0x40,0x00,0x2a,0x94]
+
+	vpkf	%v0, %v0, %v0
+	vpkf	%v0, %v0, %v15
+	vpkf	%v0, %v0, %v31
+	vpkf	%v0, %v15, %v0
+	vpkf	%v0, %v31, %v0
+	vpkf	%v15, %v0, %v0
+	vpkf	%v31, %v0, %v0
+	vpkf	%v18, %v3, %v20
+
+#CHECK: vpkg    %v0, %v0, %v0           # encoding: [0xe7,0x00,0x00,0x00,0x30,0x94]
+#CHECK: vpkg    %v0, %v0, %v15          # encoding: [0xe7,0x00,0xf0,0x00,0x30,0x94]
+#CHECK: vpkg    %v0, %v0, %v31          # encoding: [0xe7,0x00,0xf0,0x00,0x32,0x94]
+#CHECK: vpkg    %v0, %v15, %v0          # encoding: [0xe7,0x0f,0x00,0x00,0x30,0x94]
+#CHECK: vpkg    %v0, %v31, %v0          # encoding: [0xe7,0x0f,0x00,0x00,0x34,0x94]
+#CHECK: vpkg    %v15, %v0, %v0          # encoding: [0xe7,0xf0,0x00,0x00,0x30,0x94]
+#CHECK: vpkg    %v31, %v0, %v0          # encoding: [0xe7,0xf0,0x00,0x00,0x38,0x94]
+#CHECK: vpkg    %v18, %v3, %v20         # encoding: [0xe7,0x23,0x40,0x00,0x3a,0x94]
+
+	vpkg	%v0, %v0, %v0
+	vpkg	%v0, %v0, %v15
+	vpkg	%v0, %v0, %v31
+	vpkg	%v0, %v15, %v0
+	vpkg	%v0, %v31, %v0
+	vpkg	%v15, %v0, %v0
+	vpkg	%v31, %v0, %v0
+	vpkg	%v18, %v3, %v20
+
+#CHECK: vpkh    %v0, %v0, %v0           # encoding: [0xe7,0x00,0x00,0x00,0x10,0x94]
+#CHECK: vpkh    %v0, %v0, %v15          # encoding: [0xe7,0x00,0xf0,0x00,0x10,0x94]
+#CHECK: vpkh    %v0, %v0, %v31          # encoding: [0xe7,0x00,0xf0,0x00,0x12,0x94]
+#CHECK: vpkh    %v0, %v15, %v0          # encoding: [0xe7,0x0f,0x00,0x00,0x10,0x94]
+#CHECK: vpkh    %v0, %v31, %v0          # encoding: [0xe7,0x0f,0x00,0x00,0x14,0x94]
+#CHECK: vpkh    %v15, %v0, %v0          # encoding: [0xe7,0xf0,0x00,0x00,0x10,0x94]
+#CHECK: vpkh    %v31, %v0, %v0          # encoding: [0xe7,0xf0,0x00,0x00,0x18,0x94]
+#CHECK: vpkh    %v18, %v3, %v20         # encoding: [0xe7,0x23,0x40,0x00,0x1a,0x94]
+
+	vpkh	%v0, %v0, %v0
+	vpkh	%v0, %v0, %v15
+	vpkh	%v0, %v0, %v31
+	vpkh	%v0, %v15, %v0
+	vpkh	%v0, %v31, %v0
+	vpkh	%v15, %v0, %v0
+	vpkh	%v31, %v0, %v0
+	vpkh	%v18, %v3, %v20
+
+#CHECK: vpklsf  %v0, %v0, %v0           # encoding: [0xe7,0x00,0x00,0x00,0x20,0x95]
+#CHECK: vpklsf  %v0, %v0, %v15          # encoding: [0xe7,0x00,0xf0,0x00,0x20,0x95]
+#CHECK: vpklsf  %v0, %v0, %v31          # encoding: [0xe7,0x00,0xf0,0x00,0x22,0x95]
+#CHECK: vpklsf  %v0, %v15, %v0          # encoding: [0xe7,0x0f,0x00,0x00,0x20,0x95]
+#CHECK: vpklsf  %v0, %v31, %v0          # encoding: [0xe7,0x0f,0x00,0x00,0x24,0x95]
+#CHECK: vpklsf  %v15, %v0, %v0          # encoding: [0xe7,0xf0,0x00,0x00,0x20,0x95]
+#CHECK: vpklsf  %v31, %v0, %v0          # encoding: [0xe7,0xf0,0x00,0x00,0x28,0x95]
+#CHECK: vpklsf  %v18, %v3, %v20         # encoding: [0xe7,0x23,0x40,0x00,0x2a,0x95]
+#CHECK: vpklsfs %v5, %v22, %v7          # encoding: [0xe7,0x56,0x70,0x10,0x24,0x95]
+
+	vpklsf	%v0, %v0, %v0
+	vpklsf	%v0, %v0, %v15
+	vpklsf	%v0, %v0, %v31
+	vpklsf	%v0, %v15, %v0
+	vpklsf	%v0, %v31, %v0
+	vpklsf	%v15, %v0, %v0
+	vpklsf	%v31, %v0, %v0
+	vpklsf	%v18, %v3, %v20
+	vpklsfs	%v5, %v22, %v7
+
+#CHECK: vpklsg  %v0, %v0, %v0           # encoding: [0xe7,0x00,0x00,0x00,0x30,0x95]
+#CHECK: vpklsg  %v0, %v0, %v15          # encoding: [0xe7,0x00,0xf0,0x00,0x30,0x95]
+#CHECK: vpklsg  %v0, %v0, %v31          # encoding: [0xe7,0x00,0xf0,0x00,0x32,0x95]
+#CHECK: vpklsg  %v0, %v15, %v0          # encoding: [0xe7,0x0f,0x00,0x00,0x30,0x95]
+#CHECK: vpklsg  %v0, %v31, %v0          # encoding: [0xe7,0x0f,0x00,0x00,0x34,0x95]
+#CHECK: vpklsg  %v15, %v0, %v0          # encoding: [0xe7,0xf0,0x00,0x00,0x30,0x95]
+#CHECK: vpklsg  %v31, %v0, %v0          # encoding: [0xe7,0xf0,0x00,0x00,0x38,0x95]
+#CHECK: vpklsg  %v18, %v3, %v20         # encoding: [0xe7,0x23,0x40,0x00,0x3a,0x95]
+#CHECK: vpklsgs %v5, %v22, %v7          # encoding: [0xe7,0x56,0x70,0x10,0x34,0x95]
+
+	vpklsg	%v0, %v0, %v0
+	vpklsg	%v0, %v0, %v15
+	vpklsg	%v0, %v0, %v31
+	vpklsg	%v0, %v15, %v0
+	vpklsg	%v0, %v31, %v0
+	vpklsg	%v15, %v0, %v0
+	vpklsg	%v31, %v0, %v0
+	vpklsg	%v18, %v3, %v20
+	vpklsgs	%v5, %v22, %v7
+
+#CHECK: vpklsh  %v0, %v0, %v0           # encoding: [0xe7,0x00,0x00,0x00,0x10,0x95]
+#CHECK: vpklsh  %v0, %v0, %v15          # encoding: [0xe7,0x00,0xf0,0x00,0x10,0x95]
+#CHECK: vpklsh  %v0, %v0, %v31          # encoding: [0xe7,0x00,0xf0,0x00,0x12,0x95]
+#CHECK: vpklsh  %v0, %v15, %v0          # encoding: [0xe7,0x0f,0x00,0x00,0x10,0x95]
+#CHECK: vpklsh  %v0, %v31, %v0          # encoding: [0xe7,0x0f,0x00,0x00,0x14,0x95]
+#CHECK: vpklsh  %v15, %v0, %v0          # encoding: [0xe7,0xf0,0x00,0x00,0x10,0x95]
+#CHECK: vpklsh  %v31, %v0, %v0          # encoding: [0xe7,0xf0,0x00,0x00,0x18,0x95]
+#CHECK: vpklsh  %v18, %v3, %v20         # encoding: [0xe7,0x23,0x40,0x00,0x1a,0x95]
+#CHECK: vpklshs %v5, %v22, %v7          # encoding: [0xe7,0x56,0x70,0x10,0x14,0x95]
+
+	vpklsh	%v0, %v0, %v0
+	vpklsh	%v0, %v0, %v15
+	vpklsh	%v0, %v0, %v31
+	vpklsh	%v0, %v15, %v0
+	vpklsh	%v0, %v31, %v0
+	vpklsh	%v15, %v0, %v0
+	vpklsh	%v31, %v0, %v0
+	vpklsh	%v18, %v3, %v20
+	vpklshs	%v5, %v22, %v7
+
+#CHECK: vpksf   %v0, %v0, %v0           # encoding: [0xe7,0x00,0x00,0x00,0x20,0x97]
+#CHECK: vpksf   %v0, %v0, %v15          # encoding: [0xe7,0x00,0xf0,0x00,0x20,0x97]
+#CHECK: vpksf   %v0, %v0, %v31          # encoding: [0xe7,0x00,0xf0,0x00,0x22,0x97]
+#CHECK: vpksf   %v0, %v15, %v0          # encoding: [0xe7,0x0f,0x00,0x00,0x20,0x97]
+#CHECK: vpksf   %v0, %v31, %v0          # encoding: [0xe7,0x0f,0x00,0x00,0x24,0x97]
+#CHECK: vpksf   %v15, %v0, %v0          # encoding: [0xe7,0xf0,0x00,0x00,0x20,0x97]
+#CHECK: vpksf   %v31, %v0, %v0          # encoding: [0xe7,0xf0,0x00,0x00,0x28,0x97]
+#CHECK: vpksf   %v18, %v3, %v20         # encoding: [0xe7,0x23,0x40,0x00,0x2a,0x97]
+#CHECK: vpksfs  %v5, %v22, %v7          # encoding: [0xe7,0x56,0x70,0x10,0x24,0x97]
+
+	vpksf	%v0, %v0, %v0
+	vpksf	%v0, %v0, %v15
+	vpksf	%v0, %v0, %v31
+	vpksf	%v0, %v15, %v0
+	vpksf	%v0, %v31, %v0
+	vpksf	%v15, %v0, %v0
+	vpksf	%v31, %v0, %v0
+	vpksf	%v18, %v3, %v20
+	vpksfs	%v5, %v22, %v7
+
+#CHECK: vpksg   %v0, %v0, %v0           # encoding: [0xe7,0x00,0x00,0x00,0x30,0x97]
+#CHECK: vpksg   %v0, %v0, %v15          # encoding: [0xe7,0x00,0xf0,0x00,0x30,0x97]
+#CHECK: vpksg   %v0, %v0, %v31          # encoding: [0xe7,0x00,0xf0,0x00,0x32,0x97]
+#CHECK: vpksg   %v0, %v15, %v0          # encoding: [0xe7,0x0f,0x00,0x00,0x30,0x97]
+#CHECK: vpksg   %v0, %v31, %v0          # encoding: [0xe7,0x0f,0x00,0x00,0x34,0x97]
+#CHECK: vpksg   %v15, %v0, %v0          # encoding: [0xe7,0xf0,0x00,0x00,0x30,0x97]
+#CHECK: vpksg   %v31, %v0, %v0          # encoding: [0xe7,0xf0,0x00,0x00,0x38,0x97]
+#CHECK: vpksg   %v18, %v3, %v20         # encoding: [0xe7,0x23,0x40,0x00,0x3a,0x97]
+#CHECK: vpksgs  %v5, %v22, %v7          # encoding: [0xe7,0x56,0x70,0x10,0x34,0x97]
+
+	vpksg	%v0, %v0, %v0
+	vpksg	%v0, %v0, %v15
+	vpksg	%v0, %v0, %v31
+	vpksg	%v0, %v15, %v0
+	vpksg	%v0, %v31, %v0
+	vpksg	%v15, %v0, %v0
+	vpksg	%v31, %v0, %v0
+	vpksg	%v18, %v3, %v20
+	vpksgs	%v5, %v22, %v7
+
+#CHECK: vpksh   %v0, %v0, %v0           # encoding: [0xe7,0x00,0x00,0x00,0x10,0x97]
+#CHECK: vpksh   %v0, %v0, %v15          # encoding: [0xe7,0x00,0xf0,0x00,0x10,0x97]
+#CHECK: vpksh   %v0, %v0, %v31          # encoding: [0xe7,0x00,0xf0,0x00,0x12,0x97]
+#CHECK: vpksh   %v0, %v15, %v0          # encoding: [0xe7,0x0f,0x00,0x00,0x10,0x97]
+#CHECK: vpksh   %v0, %v31, %v0          # encoding: [0xe7,0x0f,0x00,0x00,0x14,0x97]
+#CHECK: vpksh   %v15, %v0, %v0          # encoding: [0xe7,0xf0,0x00,0x00,0x10,0x97]
+#CHECK: vpksh   %v31, %v0, %v0          # encoding: [0xe7,0xf0,0x00,0x00,0x18,0x97]
+#CHECK: vpksh   %v18, %v3, %v20         # encoding: [0xe7,0x23,0x40,0x00,0x1a,0x97]
+#CHECK: vpkshs  %v5, %v22, %v7          # encoding: [0xe7,0x56,0x70,0x10,0x14,0x97]
+
+	vpksh	%v0, %v0, %v0
+	vpksh	%v0, %v0, %v15
+	vpksh	%v0, %v0, %v31
+	vpksh	%v0, %v15, %v0
+	vpksh	%v0, %v31, %v0
+	vpksh	%v15, %v0, %v0
+	vpksh	%v31, %v0, %v0
+	vpksh	%v18, %v3, %v20
+	vpkshs	%v5, %v22, %v7
+
+#CHECK: vpopct  %v0, %v0, 0             # encoding: [0xe7,0x00,0x00,0x00,0x00,0x50]
+#CHECK: vpopct  %v0, %v15, 0            # encoding: [0xe7,0x0f,0x00,0x00,0x00,0x50]
+#CHECK: vpopct  %v0, %v31, 0            # encoding: [0xe7,0x0f,0x00,0x00,0x04,0x50]
+#CHECK: vpopct  %v15, %v0, 0            # encoding: [0xe7,0xf0,0x00,0x00,0x00,0x50]
+#CHECK: vpopct  %v31, %v0, 0            # encoding: [0xe7,0xf0,0x00,0x00,0x08,0x50]
+#CHECK: vpopct  %v14, %v17, 0           # encoding: [0xe7,0xe1,0x00,0x00,0x04,0x50]
+
+	vpopct	%v0, %v0, 0
+	vpopct	%v0, %v15, 0
+	vpopct	%v0, %v31, 0
+	vpopct	%v15, %v0, 0
+	vpopct	%v31, %v0, 0
+	vpopct	%v14, %v17, 0
+
+#CHECK: vrepb   %v0, %v0, 0             # encoding: [0xe7,0x00,0x00,0x00,0x00,0x4d]
+#CHECK: vrepb   %v0, %v0, 65535         # encoding: [0xe7,0x00,0xff,0xff,0x00,0x4d]
+#CHECK: vrepb   %v0, %v15, 0            # encoding: [0xe7,0x0f,0x00,0x00,0x00,0x4d]
+#CHECK: vrepb   %v0, %v31, 0            # encoding: [0xe7,0x0f,0x00,0x00,0x04,0x4d]
+#CHECK: vrepb   %v15, %v0, 0            # encoding: [0xe7,0xf0,0x00,0x00,0x00,0x4d]
+#CHECK: vrepb   %v31, %v0, 0            # encoding: [0xe7,0xf0,0x00,0x00,0x08,0x4d]
+#CHECK: vrepb   %v4, %v21, 26505        # encoding: [0xe7,0x45,0x67,0x89,0x04,0x4d]
+
+	vrepb	%v0, %v0, 0
+	vrepb	%v0, %v0, 65535
+	vrepb	%v0, %v15, 0
+	vrepb	%v0, %v31, 0
+	vrepb	%v15, %v0, 0
+	vrepb	%v31, %v0, 0
+	vrepb	%v4, %v21, 0x6789
+
+#CHECK: vrepf   %v0, %v0, 0             # encoding: [0xe7,0x00,0x00,0x00,0x20,0x4d]
+#CHECK: vrepf   %v0, %v0, 65535         # encoding: [0xe7,0x00,0xff,0xff,0x20,0x4d]
+#CHECK: vrepf   %v0, %v15, 0            # encoding: [0xe7,0x0f,0x00,0x00,0x20,0x4d]
+#CHECK: vrepf   %v0, %v31, 0            # encoding: [0xe7,0x0f,0x00,0x00,0x24,0x4d]
+#CHECK: vrepf   %v15, %v0, 0            # encoding: [0xe7,0xf0,0x00,0x00,0x20,0x4d]
+#CHECK: vrepf   %v31, %v0, 0            # encoding: [0xe7,0xf0,0x00,0x00,0x28,0x4d]
+#CHECK: vrepf   %v4, %v21, 26505        # encoding: [0xe7,0x45,0x67,0x89,0x24,0x4d]
+
+	vrepf	%v0, %v0, 0
+	vrepf	%v0, %v0, 65535
+	vrepf	%v0, %v15, 0
+	vrepf	%v0, %v31, 0
+	vrepf	%v15, %v0, 0
+	vrepf	%v31, %v0, 0
+	vrepf	%v4, %v21, 0x6789
+
+#CHECK: vrepg   %v0, %v0, 0             # encoding: [0xe7,0x00,0x00,0x00,0x30,0x4d]
+#CHECK: vrepg   %v0, %v0, 65535         # encoding: [0xe7,0x00,0xff,0xff,0x30,0x4d]
+#CHECK: vrepg   %v0, %v15, 0            # encoding: [0xe7,0x0f,0x00,0x00,0x30,0x4d]
+#CHECK: vrepg   %v0, %v31, 0            # encoding: [0xe7,0x0f,0x00,0x00,0x34,0x4d]
+#CHECK: vrepg   %v15, %v0, 0            # encoding: [0xe7,0xf0,0x00,0x00,0x30,0x4d]
+#CHECK: vrepg   %v31, %v0, 0            # encoding: [0xe7,0xf0,0x00,0x00,0x38,0x4d]
+#CHECK: vrepg   %v4, %v21, 26505        # encoding: [0xe7,0x45,0x67,0x89,0x34,0x4d]
+
+	vrepg	%v0, %v0, 0
+	vrepg	%v0, %v0, 65535
+	vrepg	%v0, %v15, 0
+	vrepg	%v0, %v31, 0
+	vrepg	%v15, %v0, 0
+	vrepg	%v31, %v0, 0
+	vrepg	%v4, %v21, 0x6789
+
+#CHECK: vreph   %v0, %v0, 0             # encoding: [0xe7,0x00,0x00,0x00,0x10,0x4d]
+#CHECK: vreph   %v0, %v0, 65535         # encoding: [0xe7,0x00,0xff,0xff,0x10,0x4d]
+#CHECK: vreph   %v0, %v15, 0            # encoding: [0xe7,0x0f,0x00,0x00,0x10,0x4d]
+#CHECK: vreph   %v0, %v31, 0            # encoding: [0xe7,0x0f,0x00,0x00,0x14,0x4d]
+#CHECK: vreph   %v15, %v0, 0            # encoding: [0xe7,0xf0,0x00,0x00,0x10,0x4d]
+#CHECK: vreph   %v31, %v0, 0            # encoding: [0xe7,0xf0,0x00,0x00,0x18,0x4d]
+#CHECK: vreph   %v4, %v21, 26505        # encoding: [0xe7,0x45,0x67,0x89,0x14,0x4d]
+
+	vreph	%v0, %v0, 0
+	vreph	%v0, %v0, 65535
+	vreph	%v0, %v15, 0
+	vreph	%v0, %v31, 0
+	vreph	%v15, %v0, 0
+	vreph	%v31, %v0, 0
+	vreph	%v4, %v21, 0x6789
+
+#CHECK: vrepib  %v0, 0                  # encoding: [0xe7,0x00,0x00,0x00,0x00,0x45]
+#CHECK: vrepib  %v0, -32768             # encoding: [0xe7,0x00,0x80,0x00,0x00,0x45]
+#CHECK: vrepib  %v0, 32767              # encoding: [0xe7,0x00,0x7f,0xff,0x00,0x45]
+#CHECK: vrepib  %v15, 0                 # encoding: [0xe7,0xf0,0x00,0x00,0x00,0x45]
+#CHECK: vrepib  %v31, 0                 # encoding: [0xe7,0xf0,0x00,0x00,0x08,0x45]
+#CHECK: vrepib  %v18, 13398             # encoding: [0xe7,0x20,0x34,0x56,0x08,0x45]
+
+	vrepib	%v0, 0
+	vrepib	%v0, -32768
+	vrepib	%v0, 32767
+	vrepib	%v15, 0
+	vrepib	%v31, 0
+	vrepib	%v18, 0x3456
+
+#CHECK: vrepif  %v0, 0                  # encoding: [0xe7,0x00,0x00,0x00,0x20,0x45]
+#CHECK: vrepif  %v0, -32768             # encoding: [0xe7,0x00,0x80,0x00,0x20,0x45]
+#CHECK: vrepif  %v0, 32767              # encoding: [0xe7,0x00,0x7f,0xff,0x20,0x45]
+#CHECK: vrepif  %v15, 0                 # encoding: [0xe7,0xf0,0x00,0x00,0x20,0x45]
+#CHECK: vrepif  %v31, 0                 # encoding: [0xe7,0xf0,0x00,0x00,0x28,0x45]
+#CHECK: vrepif  %v18, 13398             # encoding: [0xe7,0x20,0x34,0x56,0x28,0x45]
+
+	vrepif	%v0, 0
+	vrepif	%v0, -32768
+	vrepif	%v0, 32767
+	vrepif	%v15, 0
+	vrepif	%v31, 0
+	vrepif	%v18, 0x3456
+
+#CHECK: vrepig  %v0, 0                  # encoding: [0xe7,0x00,0x00,0x00,0x30,0x45]
+#CHECK: vrepig  %v0, -32768             # encoding: [0xe7,0x00,0x80,0x00,0x30,0x45]
+#CHECK: vrepig  %v0, 32767              # encoding: [0xe7,0x00,0x7f,0xff,0x30,0x45]
+#CHECK: vrepig  %v15, 0                 # encoding: [0xe7,0xf0,0x00,0x00,0x30,0x45]
+#CHECK: vrepig  %v31, 0                 # encoding: [0xe7,0xf0,0x00,0x00,0x38,0x45]
+#CHECK: vrepig  %v18, 13398             # encoding: [0xe7,0x20,0x34,0x56,0x38,0x45]
+
+	vrepig	%v0, 0
+	vrepig	%v0, -32768
+	vrepig	%v0, 32767
+	vrepig	%v15, 0
+	vrepig	%v31, 0
+	vrepig	%v18, 0x3456
+
+#CHECK: vrepih  %v0, 0                  # encoding: [0xe7,0x00,0x00,0x00,0x10,0x45]
+#CHECK: vrepih  %v0, -32768             # encoding: [0xe7,0x00,0x80,0x00,0x10,0x45]
+#CHECK: vrepih  %v0, 32767              # encoding: [0xe7,0x00,0x7f,0xff,0x10,0x45]
+#CHECK: vrepih  %v15, 0                 # encoding: [0xe7,0xf0,0x00,0x00,0x10,0x45]
+#CHECK: vrepih  %v31, 0                 # encoding: [0xe7,0xf0,0x00,0x00,0x18,0x45]
+#CHECK: vrepih  %v18, 13398             # encoding: [0xe7,0x20,0x34,0x56,0x18,0x45]
+
+	vrepih	%v0, 0
+	vrepih	%v0, -32768
+	vrepih	%v0, 32767
+	vrepih	%v15, 0
+	vrepih	%v31, 0
+	vrepih	%v18, 0x3456
+
+#CHECK: vsb     %v0, %v0, %v0           # encoding: [0xe7,0x00,0x00,0x00,0x00,0xf7]
+#CHECK: vsb     %v0, %v0, %v31          # encoding: [0xe7,0x00,0xf0,0x00,0x02,0xf7]
+#CHECK: vsb     %v0, %v31, %v0          # encoding: [0xe7,0x0f,0x00,0x00,0x04,0xf7]
+#CHECK: vsb     %v31, %v0, %v0          # encoding: [0xe7,0xf0,0x00,0x00,0x08,0xf7]
+#CHECK: vsb     %v18, %v3, %v20         # encoding: [0xe7,0x23,0x40,0x00,0x0a,0xf7]
+
+	vsb	%v0, %v0, %v0
+	vsb	%v0, %v0, %v31
+	vsb	%v0, %v31, %v0
+	vsb	%v31, %v0, %v0
+	vsb	%v18, %v3, %v20
+
+#CHECK: vsbcbiq %v0, %v0, %v0, %v0      # encoding: [0xe7,0x00,0x04,0x00,0x00,0xbd]
+#CHECK: vsbcbiq %v0, %v0, %v0, %v31     # encoding: [0xe7,0x00,0x04,0x00,0xf1,0xbd]
+#CHECK: vsbcbiq %v0, %v0, %v31, %v0     # encoding: [0xe7,0x00,0xf4,0x00,0x02,0xbd]
+#CHECK: vsbcbiq %v0, %v31, %v0, %v0     # encoding: [0xe7,0x0f,0x04,0x00,0x04,0xbd]
+#CHECK: vsbcbiq %v31, %v0, %v0, %v0     # encoding: [0xe7,0xf0,0x04,0x00,0x08,0xbd]
+#CHECK: vsbcbiq %v13, %v17, %v21, %v25  # encoding: [0xe7,0xd1,0x54,0x00,0x97,0xbd]
+
+	vsbcbiq	%v0, %v0, %v0, %v0
+	vsbcbiq	%v0, %v0, %v0, %v31
+	vsbcbiq	%v0, %v0, %v31, %v0
+	vsbcbiq	%v0, %v31, %v0, %v0
+	vsbcbiq	%v31, %v0, %v0, %v0
+	vsbcbiq	%v13, %v17, %v21, %v25
+
+#CHECK: vsbiq   %v0, %v0, %v0, %v0      # encoding: [0xe7,0x00,0x04,0x00,0x00,0xbf]
+#CHECK: vsbiq   %v0, %v0, %v0, %v31     # encoding: [0xe7,0x00,0x04,0x00,0xf1,0xbf]
+#CHECK: vsbiq   %v0, %v0, %v31, %v0     # encoding: [0xe7,0x00,0xf4,0x00,0x02,0xbf]
+#CHECK: vsbiq   %v0, %v31, %v0, %v0     # encoding: [0xe7,0x0f,0x04,0x00,0x04,0xbf]
+#CHECK: vsbiq   %v31, %v0, %v0, %v0     # encoding: [0xe7,0xf0,0x04,0x00,0x08,0xbf]
+#CHECK: vsbiq   %v13, %v17, %v21, %v25  # encoding: [0xe7,0xd1,0x54,0x00,0x97,0xbf]
+
+	vsbiq	%v0, %v0, %v0, %v0
+	vsbiq	%v0, %v0, %v0, %v31
+	vsbiq	%v0, %v0, %v31, %v0
+	vsbiq	%v0, %v31, %v0, %v0
+	vsbiq	%v31, %v0, %v0, %v0
+	vsbiq	%v13, %v17, %v21, %v25
+
+#CHECK: vscbib  %v0, %v0, %v0           # encoding: [0xe7,0x00,0x00,0x00,0x00,0xf5]
+#CHECK: vscbib  %v0, %v0, %v31          # encoding: [0xe7,0x00,0xf0,0x00,0x02,0xf5]
+#CHECK: vscbib  %v0, %v31, %v0          # encoding: [0xe7,0x0f,0x00,0x00,0x04,0xf5]
+#CHECK: vscbib  %v31, %v0, %v0          # encoding: [0xe7,0xf0,0x00,0x00,0x08,0xf5]
+#CHECK: vscbib  %v18, %v3, %v20         # encoding: [0xe7,0x23,0x40,0x00,0x0a,0xf5]
+
+	vscbib	%v0, %v0, %v0
+	vscbib	%v0, %v0, %v31
+	vscbib	%v0, %v31, %v0
+	vscbib	%v31, %v0, %v0
+	vscbib	%v18, %v3, %v20
+
+#CHECK: vscbif  %v0, %v0, %v0           # encoding: [0xe7,0x00,0x00,0x00,0x20,0xf5]
+#CHECK: vscbif  %v0, %v0, %v31          # encoding: [0xe7,0x00,0xf0,0x00,0x22,0xf5]
+#CHECK: vscbif  %v0, %v31, %v0          # encoding: [0xe7,0x0f,0x00,0x00,0x24,0xf5]
+#CHECK: vscbif  %v31, %v0, %v0          # encoding: [0xe7,0xf0,0x00,0x00,0x28,0xf5]
+#CHECK: vscbif  %v18, %v3, %v20         # encoding: [0xe7,0x23,0x40,0x00,0x2a,0xf5]
+
+	vscbif	%v0, %v0, %v0
+	vscbif	%v0, %v0, %v31
+	vscbif	%v0, %v31, %v0
+	vscbif	%v31, %v0, %v0
+	vscbif	%v18, %v3, %v20
+
+#CHECK: vscbig  %v0, %v0, %v0           # encoding: [0xe7,0x00,0x00,0x00,0x30,0xf5]
+#CHECK: vscbig  %v0, %v0, %v31          # encoding: [0xe7,0x00,0xf0,0x00,0x32,0xf5]
+#CHECK: vscbig  %v0, %v31, %v0          # encoding: [0xe7,0x0f,0x00,0x00,0x34,0xf5]
+#CHECK: vscbig  %v31, %v0, %v0          # encoding: [0xe7,0xf0,0x00,0x00,0x38,0xf5]
+#CHECK: vscbig  %v18, %v3, %v20         # encoding: [0xe7,0x23,0x40,0x00,0x3a,0xf5]
+
+	vscbig	%v0, %v0, %v0
+	vscbig	%v0, %v0, %v31
+	vscbig	%v0, %v31, %v0
+	vscbig	%v31, %v0, %v0
+	vscbig	%v18, %v3, %v20
+
+#CHECK: vscbih  %v0, %v0, %v0           # encoding: [0xe7,0x00,0x00,0x00,0x10,0xf5]
+#CHECK: vscbih  %v0, %v0, %v31          # encoding: [0xe7,0x00,0xf0,0x00,0x12,0xf5]
+#CHECK: vscbih  %v0, %v31, %v0          # encoding: [0xe7,0x0f,0x00,0x00,0x14,0xf5]
+#CHECK: vscbih  %v31, %v0, %v0          # encoding: [0xe7,0xf0,0x00,0x00,0x18,0xf5]
+#CHECK: vscbih  %v18, %v3, %v20         # encoding: [0xe7,0x23,0x40,0x00,0x1a,0xf5]
+
+	vscbih	%v0, %v0, %v0
+	vscbih	%v0, %v0, %v31
+	vscbih	%v0, %v31, %v0
+	vscbih	%v31, %v0, %v0
+	vscbih	%v18, %v3, %v20
+
+#CHECK: vscbiq  %v0, %v0, %v0           # encoding: [0xe7,0x00,0x00,0x00,0x40,0xf5]
+#CHECK: vscbiq  %v0, %v0, %v31          # encoding: [0xe7,0x00,0xf0,0x00,0x42,0xf5]
+#CHECK: vscbiq  %v0, %v31, %v0          # encoding: [0xe7,0x0f,0x00,0x00,0x44,0xf5]
+#CHECK: vscbiq  %v31, %v0, %v0          # encoding: [0xe7,0xf0,0x00,0x00,0x48,0xf5]
+#CHECK: vscbiq  %v18, %v3, %v20         # encoding: [0xe7,0x23,0x40,0x00,0x4a,0xf5]
+
+	vscbiq	%v0, %v0, %v0
+	vscbiq	%v0, %v0, %v31
+	vscbiq	%v0, %v31, %v0
+	vscbiq	%v31, %v0, %v0
+	vscbiq	%v18, %v3, %v20
+
+#CHECK: vscef   %v0, 0(%v0), 0          # encoding: [0xe7,0x00,0x00,0x00,0x00,0x1b]
+#CHECK: vscef   %v0, 0(%v0,%r1), 0      # encoding: [0xe7,0x00,0x10,0x00,0x00,0x1b]
+#CHECK: vscef   %v0, 0(%v0,%r1), 3      # encoding: [0xe7,0x00,0x10,0x00,0x30,0x1b]
+#CHECK: vscef   %v0, 0(%v0,%r15), 0     # encoding: [0xe7,0x00,0xf0,0x00,0x00,0x1b]
+#CHECK: vscef   %v0, 0(%v15,%r1), 0     # encoding: [0xe7,0x0f,0x10,0x00,0x00,0x1b]
+#CHECK: vscef   %v0, 0(%v31,%r1), 0     # encoding: [0xe7,0x0f,0x10,0x00,0x04,0x1b]
+#CHECK: vscef   %v0, 4095(%v0,%r1), 0   # encoding: [0xe7,0x00,0x1f,0xff,0x00,0x1b]
+#CHECK: vscef   %v15, 0(%v0,%r1), 0     # encoding: [0xe7,0xf0,0x10,0x00,0x00,0x1b]
+#CHECK: vscef   %v31, 0(%v0,%r1), 0     # encoding: [0xe7,0xf0,0x10,0x00,0x08,0x1b]
+#CHECK: vscef   %v10, 1000(%v19,%r7), 1 # encoding: [0xe7,0xa3,0x73,0xe8,0x14,0x1b]
+
+	vscef	%v0, 0(%v0), 0
+	vscef	%v0, 0(%v0,%r1), 0
+	vscef	%v0, 0(%v0,%r1), 3
+	vscef	%v0, 0(%v0,%r15), 0
+	vscef	%v0, 0(%v15,%r1), 0
+	vscef	%v0, 0(%v31,%r1), 0
+	vscef	%v0, 4095(%v0, %r1), 0
+	vscef	%v15, 0(%v0,%r1), 0
+	vscef	%v31, 0(%v0,%r1), 0
+	vscef	%v10, 1000(%v19,%r7), 1
+
+#CHECK: vsceg   %v0, 0(%v0), 0          # encoding: [0xe7,0x00,0x00,0x00,0x00,0x1a]
+#CHECK: vsceg   %v0, 0(%v0,%r1), 0      # encoding: [0xe7,0x00,0x10,0x00,0x00,0x1a]
+#CHECK: vsceg   %v0, 0(%v0,%r1), 1      # encoding: [0xe7,0x00,0x10,0x00,0x10,0x1a]
+#CHECK: vsceg   %v0, 0(%v0,%r15), 0     # encoding: [0xe7,0x00,0xf0,0x00,0x00,0x1a]
+#CHECK: vsceg   %v0, 0(%v15,%r1), 0     # encoding: [0xe7,0x0f,0x10,0x00,0x00,0x1a]
+#CHECK: vsceg   %v0, 0(%v31,%r1), 0     # encoding: [0xe7,0x0f,0x10,0x00,0x04,0x1a]
+#CHECK: vsceg   %v0, 4095(%v0,%r1), 0   # encoding: [0xe7,0x00,0x1f,0xff,0x00,0x1a]
+#CHECK: vsceg   %v15, 0(%v0,%r1), 0     # encoding: [0xe7,0xf0,0x10,0x00,0x00,0x1a]
+#CHECK: vsceg   %v31, 0(%v0,%r1), 0     # encoding: [0xe7,0xf0,0x10,0x00,0x08,0x1a]
+#CHECK: vsceg   %v10, 1000(%v19,%r7), 1 # encoding: [0xe7,0xa3,0x73,0xe8,0x14,0x1a]
+
+	vsceg	%v0, 0(%v0), 0
+	vsceg	%v0, 0(%v0,%r1), 0
+	vsceg	%v0, 0(%v0,%r1), 1
+	vsceg	%v0, 0(%v0,%r15), 0
+	vsceg	%v0, 0(%v15,%r1), 0
+	vsceg	%v0, 0(%v31,%r1), 0
+	vsceg	%v0, 4095(%v0,%r1), 0
+	vsceg	%v15, 0(%v0,%r1), 0
+	vsceg	%v31, 0(%v0,%r1), 0
+	vsceg	%v10, 1000(%v19,%r7), 1
+
+#CHECK: vsel    %v0, %v0, %v0, %v0      # encoding: [0xe7,0x00,0x00,0x00,0x00,0x8d]
+#CHECK: vsel    %v0, %v0, %v0, %v31     # encoding: [0xe7,0x00,0x00,0x00,0xf1,0x8d]
+#CHECK: vsel    %v0, %v0, %v31, %v0     # encoding: [0xe7,0x00,0xf0,0x00,0x02,0x8d]
+#CHECK: vsel    %v0, %v31, %v0, %v0     # encoding: [0xe7,0x0f,0x00,0x00,0x04,0x8d]
+#CHECK: vsel    %v31, %v0, %v0, %v0     # encoding: [0xe7,0xf0,0x00,0x00,0x08,0x8d]
+#CHECK: vsel    %v13, %v17, %v21, %v25  # encoding: [0xe7,0xd1,0x50,0x00,0x97,0x8d]
+
+	vsel	%v0, %v0, %v0, %v0
+	vsel	%v0, %v0, %v0, %v31
+	vsel	%v0, %v0, %v31, %v0
+	vsel	%v0, %v31, %v0, %v0
+	vsel	%v31, %v0, %v0, %v0
+	vsel 	%v13, %v17, %v21, %v25
+
+#CHECK: vsegb   %v0, %v0                # encoding: [0xe7,0x00,0x00,0x00,0x00,0x5f]
+#CHECK: vsegb   %v0, %v15               # encoding: [0xe7,0x0f,0x00,0x00,0x00,0x5f]
+#CHECK: vsegb   %v0, %v31               # encoding: [0xe7,0x0f,0x00,0x00,0x04,0x5f]
+#CHECK: vsegb   %v15, %v0               # encoding: [0xe7,0xf0,0x00,0x00,0x00,0x5f]
+#CHECK: vsegb   %v31, %v0               # encoding: [0xe7,0xf0,0x00,0x00,0x08,0x5f]
+#CHECK: vsegb   %v14, %v17              # encoding: [0xe7,0xe1,0x00,0x00,0x04,0x5f]
+
+	vsegb	%v0, %v0
+	vsegb	%v0, %v15
+	vsegb	%v0, %v31
+	vsegb	%v15, %v0
+	vsegb	%v31, %v0
+	vsegb	%v14, %v17
+
+#CHECK: vsegf   %v0, %v0                # encoding: [0xe7,0x00,0x00,0x00,0x20,0x5f]
+#CHECK: vsegf   %v0, %v15               # encoding: [0xe7,0x0f,0x00,0x00,0x20,0x5f]
+#CHECK: vsegf   %v0, %v31               # encoding: [0xe7,0x0f,0x00,0x00,0x24,0x5f]
+#CHECK: vsegf   %v15, %v0               # encoding: [0xe7,0xf0,0x00,0x00,0x20,0x5f]
+#CHECK: vsegf   %v31, %v0               # encoding: [0xe7,0xf0,0x00,0x00,0x28,0x5f]
+#CHECK: vsegf   %v14, %v17              # encoding: [0xe7,0xe1,0x00,0x00,0x24,0x5f]
+
+	vsegf	%v0, %v0
+	vsegf	%v0, %v15
+	vsegf	%v0, %v31
+	vsegf	%v15, %v0
+	vsegf	%v31, %v0
+	vsegf	%v14, %v17
+
+#CHECK: vsegh   %v0, %v0                # encoding: [0xe7,0x00,0x00,0x00,0x10,0x5f]
+#CHECK: vsegh   %v0, %v15               # encoding: [0xe7,0x0f,0x00,0x00,0x10,0x5f]
+#CHECK: vsegh   %v0, %v31               # encoding: [0xe7,0x0f,0x00,0x00,0x14,0x5f]
+#CHECK: vsegh   %v15, %v0               # encoding: [0xe7,0xf0,0x00,0x00,0x10,0x5f]
+#CHECK: vsegh   %v31, %v0               # encoding: [0xe7,0xf0,0x00,0x00,0x18,0x5f]
+#CHECK: vsegh   %v14, %v17              # encoding: [0xe7,0xe1,0x00,0x00,0x14,0x5f]
+
+	vsegh	%v0, %v0
+	vsegh	%v0, %v15
+	vsegh	%v0, %v31
+	vsegh	%v15, %v0
+	vsegh	%v31, %v0
+	vsegh	%v14, %v17
+
+#CHECK: vsf     %v0, %v0, %v0           # encoding: [0xe7,0x00,0x00,0x00,0x20,0xf7]
+#CHECK: vsf     %v0, %v0, %v31          # encoding: [0xe7,0x00,0xf0,0x00,0x22,0xf7]
+#CHECK: vsf     %v0, %v31, %v0          # encoding: [0xe7,0x0f,0x00,0x00,0x24,0xf7]
+#CHECK: vsf     %v31, %v0, %v0          # encoding: [0xe7,0xf0,0x00,0x00,0x28,0xf7]
+#CHECK: vsf     %v18, %v3, %v20         # encoding: [0xe7,0x23,0x40,0x00,0x2a,0xf7]
+
+	vsf	%v0, %v0, %v0
+	vsf	%v0, %v0, %v31
+	vsf	%v0, %v31, %v0
+	vsf	%v31, %v0, %v0
+	vsf	%v18, %v3, %v20
+
+#CHECK: vsg     %v0, %v0, %v0           # encoding: [0xe7,0x00,0x00,0x00,0x30,0xf7]
+#CHECK: vsg     %v0, %v0, %v31          # encoding: [0xe7,0x00,0xf0,0x00,0x32,0xf7]
+#CHECK: vsg     %v0, %v31, %v0          # encoding: [0xe7,0x0f,0x00,0x00,0x34,0xf7]
+#CHECK: vsg     %v31, %v0, %v0          # encoding: [0xe7,0xf0,0x00,0x00,0x38,0xf7]
+#CHECK: vsg     %v18, %v3, %v20         # encoding: [0xe7,0x23,0x40,0x00,0x3a,0xf7]
+
+	vsg	%v0, %v0, %v0
+	vsg	%v0, %v0, %v31
+	vsg	%v0, %v31, %v0
+	vsg	%v31, %v0, %v0
+	vsg	%v18, %v3, %v20
+
+#CHECK: vsh     %v0, %v0, %v0           # encoding: [0xe7,0x00,0x00,0x00,0x10,0xf7]
+#CHECK: vsh     %v0, %v0, %v31          # encoding: [0xe7,0x00,0xf0,0x00,0x12,0xf7]
+#CHECK: vsh     %v0, %v31, %v0          # encoding: [0xe7,0x0f,0x00,0x00,0x14,0xf7]
+#CHECK: vsh     %v31, %v0, %v0          # encoding: [0xe7,0xf0,0x00,0x00,0x18,0xf7]
+#CHECK: vsh     %v18, %v3, %v20         # encoding: [0xe7,0x23,0x40,0x00,0x1a,0xf7]
+
+	vsh	%v0, %v0, %v0
+	vsh	%v0, %v0, %v31
+	vsh	%v0, %v31, %v0
+	vsh	%v31, %v0, %v0
+	vsh	%v18, %v3, %v20
+
+#CHECK: vsl     %v0, %v0, %v0           # encoding: [0xe7,0x00,0x00,0x00,0x00,0x74]
+#CHECK: vsl     %v0, %v0, %v31          # encoding: [0xe7,0x00,0xf0,0x00,0x02,0x74]
+#CHECK: vsl     %v0, %v31, %v0          # encoding: [0xe7,0x0f,0x00,0x00,0x04,0x74]
+#CHECK: vsl     %v31, %v0, %v0          # encoding: [0xe7,0xf0,0x00,0x00,0x08,0x74]
+#CHECK: vsl     %v18, %v3, %v20         # encoding: [0xe7,0x23,0x40,0x00,0x0a,0x74]
+
+	vsl	%v0, %v0, %v0
+	vsl	%v0, %v0, %v31
+	vsl	%v0, %v31, %v0
+	vsl	%v31, %v0, %v0
+	vsl	%v18, %v3, %v20
+
+#CHECK: vslb    %v0, %v0, %v0           # encoding: [0xe7,0x00,0x00,0x00,0x00,0x75]
+#CHECK: vslb    %v0, %v0, %v31          # encoding: [0xe7,0x00,0xf0,0x00,0x02,0x75]
+#CHECK: vslb    %v0, %v31, %v0          # encoding: [0xe7,0x0f,0x00,0x00,0x04,0x75]
+#CHECK: vslb    %v31, %v0, %v0          # encoding: [0xe7,0xf0,0x00,0x00,0x08,0x75]
+#CHECK: vslb    %v18, %v3, %v20         # encoding: [0xe7,0x23,0x40,0x00,0x0a,0x75]
+
+	vslb	%v0, %v0, %v0
+	vslb	%v0, %v0, %v31
+	vslb	%v0, %v31, %v0
+	vslb	%v31, %v0, %v0
+	vslb	%v18, %v3, %v20
+
+#CHECK: vsldb   %v0, %v0, %v0, 0        # encoding: [0xe7,0x00,0x00,0x00,0x00,0x77]
+#CHECK: vsldb   %v0, %v0, %v0, 255      # encoding: [0xe7,0x00,0x00,0xff,0x00,0x77]
+#CHECK: vsldb   %v0, %v0, %v31, 0       # encoding: [0xe7,0x00,0xf0,0x00,0x02,0x77]
+#CHECK: vsldb   %v0, %v31, %v0, 0       # encoding: [0xe7,0x0f,0x00,0x00,0x04,0x77]
+#CHECK: vsldb   %v31, %v0, %v0, 0       # encoding: [0xe7,0xf0,0x00,0x00,0x08,0x77]
+#CHECK: vsldb   %v13, %v17, %v21, 121   # encoding: [0xe7,0xd1,0x50,0x79,0x06,0x77]
+
+	vsldb	%v0, %v0, %v0, 0
+	vsldb	%v0, %v0, %v0, 255
+	vsldb	%v0, %v0, %v31, 0
+	vsldb	%v0, %v31, %v0, 0
+	vsldb	%v31, %v0, %v0, 0
+	vsldb 	%v13, %v17, %v21, 0x79
+
+#CHECK: vsq     %v0, %v0, %v0           # encoding: [0xe7,0x00,0x00,0x00,0x40,0xf7]
+#CHECK: vsq     %v0, %v0, %v31          # encoding: [0xe7,0x00,0xf0,0x00,0x42,0xf7]
+#CHECK: vsq     %v0, %v31, %v0          # encoding: [0xe7,0x0f,0x00,0x00,0x44,0xf7]
+#CHECK: vsq     %v31, %v0, %v0          # encoding: [0xe7,0xf0,0x00,0x00,0x48,0xf7]
+#CHECK: vsq     %v18, %v3, %v20         # encoding: [0xe7,0x23,0x40,0x00,0x4a,0xf7]
+
+	vsq	%v0, %v0, %v0
+	vsq	%v0, %v0, %v31
+	vsq	%v0, %v31, %v0
+	vsq	%v31, %v0, %v0
+	vsq	%v18, %v3, %v20
+
+#CHECK: vsra    %v0, %v0, %v0           # encoding: [0xe7,0x00,0x00,0x00,0x00,0x7e]
+#CHECK: vsra    %v0, %v0, %v31          # encoding: [0xe7,0x00,0xf0,0x00,0x02,0x7e]
+#CHECK: vsra    %v0, %v31, %v0          # encoding: [0xe7,0x0f,0x00,0x00,0x04,0x7e]
+#CHECK: vsra    %v31, %v0, %v0          # encoding: [0xe7,0xf0,0x00,0x00,0x08,0x7e]
+#CHECK: vsra    %v18, %v3, %v20         # encoding: [0xe7,0x23,0x40,0x00,0x0a,0x7e]
+
+	vsra	%v0, %v0, %v0
+	vsra	%v0, %v0, %v31
+	vsra	%v0, %v31, %v0
+	vsra	%v31, %v0, %v0
+	vsra	%v18, %v3, %v20
+
+#CHECK: vsrab   %v0, %v0, %v0           # encoding: [0xe7,0x00,0x00,0x00,0x00,0x7f]
+#CHECK: vsrab   %v0, %v0, %v31          # encoding: [0xe7,0x00,0xf0,0x00,0x02,0x7f]
+#CHECK: vsrab   %v0, %v31, %v0          # encoding: [0xe7,0x0f,0x00,0x00,0x04,0x7f]
+#CHECK: vsrab   %v31, %v0, %v0          # encoding: [0xe7,0xf0,0x00,0x00,0x08,0x7f]
+#CHECK: vsrab   %v18, %v3, %v20         # encoding: [0xe7,0x23,0x40,0x00,0x0a,0x7f]
+
+	vsrab	%v0, %v0, %v0
+	vsrab	%v0, %v0, %v31
+	vsrab	%v0, %v31, %v0
+	vsrab	%v31, %v0, %v0
+	vsrab	%v18, %v3, %v20
+
+#CHECK: vsrl    %v0, %v0, %v0           # encoding: [0xe7,0x00,0x00,0x00,0x00,0x7c]
+#CHECK: vsrl    %v0, %v0, %v31          # encoding: [0xe7,0x00,0xf0,0x00,0x02,0x7c]
+#CHECK: vsrl    %v0, %v31, %v0          # encoding: [0xe7,0x0f,0x00,0x00,0x04,0x7c]
+#CHECK: vsrl    %v31, %v0, %v0          # encoding: [0xe7,0xf0,0x00,0x00,0x08,0x7c]
+#CHECK: vsrl    %v18, %v3, %v20         # encoding: [0xe7,0x23,0x40,0x00,0x0a,0x7c]
+
+	vsrl	%v0, %v0, %v0
+	vsrl	%v0, %v0, %v31
+	vsrl	%v0, %v31, %v0
+	vsrl	%v31, %v0, %v0
+	vsrl	%v18, %v3, %v20
+
+#CHECK: vsrlb   %v0, %v0, %v0           # encoding: [0xe7,0x00,0x00,0x00,0x00,0x7d]
+#CHECK: vsrlb   %v0, %v0, %v31          # encoding: [0xe7,0x00,0xf0,0x00,0x02,0x7d]
+#CHECK: vsrlb   %v0, %v31, %v0          # encoding: [0xe7,0x0f,0x00,0x00,0x04,0x7d]
+#CHECK: vsrlb   %v31, %v0, %v0          # encoding: [0xe7,0xf0,0x00,0x00,0x08,0x7d]
+#CHECK: vsrlb   %v18, %v3, %v20         # encoding: [0xe7,0x23,0x40,0x00,0x0a,0x7d]
+
+	vsrlb	%v0, %v0, %v0
+	vsrlb	%v0, %v0, %v31
+	vsrlb	%v0, %v31, %v0
+	vsrlb	%v31, %v0, %v0
+	vsrlb	%v18, %v3, %v20
+
+#CHECK: vst     %v0, 0                  # encoding: [0xe7,0x00,0x00,0x00,0x00,0x0e]
+#CHECK: vst     %v0, 4095               # encoding: [0xe7,0x00,0x0f,0xff,0x00,0x0e]
+#CHECK: vst     %v0, 0(%r15)            # encoding: [0xe7,0x00,0xf0,0x00,0x00,0x0e]
+#CHECK: vst     %v0, 0(%r15,%r1)        # encoding: [0xe7,0x0f,0x10,0x00,0x00,0x0e]
+#CHECK: vst     %v15, 0                 # encoding: [0xe7,0xf0,0x00,0x00,0x00,0x0e]
+#CHECK: vst     %v31, 0                 # encoding: [0xe7,0xf0,0x00,0x00,0x08,0x0e]
+#CHECK: vst     %v18, 1383(%r3,%r4)     # encoding: [0xe7,0x23,0x45,0x67,0x08,0x0e]
+
+	vst	%v0, 0
+	vst	%v0, 4095
+	vst	%v0, 0(%r15)
+	vst	%v0, 0(%r15,%r1)
+	vst	%v15, 0
+	vst	%v31, 0
+	vst	%v18, 0x567(%r3,%r4)
+
+#CHECK: vsteb   %v0, 0, 0               # encoding: [0xe7,0x00,0x00,0x00,0x00,0x08]
+#CHECK: vsteb   %v0, 0, 15              # encoding: [0xe7,0x00,0x00,0x00,0xf0,0x08]
+#CHECK: vsteb   %v0, 4095, 0            # encoding: [0xe7,0x00,0x0f,0xff,0x00,0x08]
+#CHECK: vsteb   %v0, 0(%r15), 0         # encoding: [0xe7,0x00,0xf0,0x00,0x00,0x08]
+#CHECK: vsteb   %v0, 0(%r15,%r1), 0     # encoding: [0xe7,0x0f,0x10,0x00,0x00,0x08]
+#CHECK: vsteb   %v15, 0, 0              # encoding: [0xe7,0xf0,0x00,0x00,0x00,0x08]
+#CHECK: vsteb   %v31, 0, 0              # encoding: [0xe7,0xf0,0x00,0x00,0x08,0x08]
+#CHECK: vsteb   %v18, 1383(%r3,%r4), 8  # encoding: [0xe7,0x23,0x45,0x67,0x88,0x08]
+
+	vsteb	%v0, 0, 0
+	vsteb	%v0, 0, 15
+	vsteb	%v0, 4095, 0
+	vsteb	%v0, 0(%r15), 0
+	vsteb	%v0, 0(%r15,%r1), 0
+	vsteb	%v15, 0, 0
+	vsteb	%v31, 0, 0
+	vsteb	%v18, 1383(%r3,%r4), 8
+
+#CHECK: vstef   %v0, 0, 0               # encoding: [0xe7,0x00,0x00,0x00,0x00,0x0b]
+#CHECK: vstef   %v0, 0, 3               # encoding: [0xe7,0x00,0x00,0x00,0x30,0x0b]
+#CHECK: vstef   %v0, 4095, 0            # encoding: [0xe7,0x00,0x0f,0xff,0x00,0x0b]
+#CHECK: vstef   %v0, 0(%r15), 0         # encoding: [0xe7,0x00,0xf0,0x00,0x00,0x0b]
+#CHECK: vstef   %v0, 0(%r15,%r1), 0     # encoding: [0xe7,0x0f,0x10,0x00,0x00,0x0b]
+#CHECK: vstef   %v15, 0, 0              # encoding: [0xe7,0xf0,0x00,0x00,0x00,0x0b]
+#CHECK: vstef   %v31, 0, 0              # encoding: [0xe7,0xf0,0x00,0x00,0x08,0x0b]
+#CHECK: vstef   %v18, 1383(%r3,%r4), 2  # encoding: [0xe7,0x23,0x45,0x67,0x28,0x0b]
+
+	vstef	%v0, 0, 0
+	vstef	%v0, 0, 3
+	vstef	%v0, 4095, 0
+	vstef	%v0, 0(%r15), 0
+	vstef	%v0, 0(%r15,%r1), 0
+	vstef	%v15, 0, 0
+	vstef	%v31, 0, 0
+	vstef	%v18, 1383(%r3,%r4), 2
+
+#CHECK: vsteg   %v0, 0, 0               # encoding: [0xe7,0x00,0x00,0x00,0x00,0x0a]
+#CHECK: vsteg   %v0, 0, 1               # encoding: [0xe7,0x00,0x00,0x00,0x10,0x0a]
+#CHECK: vsteg   %v0, 4095, 0            # encoding: [0xe7,0x00,0x0f,0xff,0x00,0x0a]
+#CHECK: vsteg   %v0, 0(%r15), 0         # encoding: [0xe7,0x00,0xf0,0x00,0x00,0x0a]
+#CHECK: vsteg   %v0, 0(%r15,%r1), 0     # encoding: [0xe7,0x0f,0x10,0x00,0x00,0x0a]
+#CHECK: vsteg   %v15, 0, 0              # encoding: [0xe7,0xf0,0x00,0x00,0x00,0x0a]
+#CHECK: vsteg   %v31, 0, 0              # encoding: [0xe7,0xf0,0x00,0x00,0x08,0x0a]
+#CHECK: vsteg   %v18, 1383(%r3,%r4), 1  # encoding: [0xe7,0x23,0x45,0x67,0x18,0x0a]
+
+	vsteg	%v0, 0, 0
+	vsteg	%v0, 0, 1
+	vsteg	%v0, 4095, 0
+	vsteg	%v0, 0(%r15), 0
+	vsteg	%v0, 0(%r15,%r1), 0
+	vsteg	%v15, 0, 0
+	vsteg	%v31, 0, 0
+	vsteg	%v18, 1383(%r3,%r4), 1
+
+#CHECK: vsteh   %v0, 0, 0               # encoding: [0xe7,0x00,0x00,0x00,0x00,0x09]
+#CHECK: vsteh   %v0, 0, 7               # encoding: [0xe7,0x00,0x00,0x00,0x70,0x09]
+#CHECK: vsteh   %v0, 4095, 0            # encoding: [0xe7,0x00,0x0f,0xff,0x00,0x09]
+#CHECK: vsteh   %v0, 0(%r15), 0         # encoding: [0xe7,0x00,0xf0,0x00,0x00,0x09]
+#CHECK: vsteh   %v0, 0(%r15,%r1), 0     # encoding: [0xe7,0x0f,0x10,0x00,0x00,0x09]
+#CHECK: vsteh   %v15, 0, 0              # encoding: [0xe7,0xf0,0x00,0x00,0x00,0x09]
+#CHECK: vsteh   %v31, 0, 0              # encoding: [0xe7,0xf0,0x00,0x00,0x08,0x09]
+#CHECK: vsteh   %v18, 1383(%r3,%r4), 4  # encoding: [0xe7,0x23,0x45,0x67,0x48,0x09]
+
+	vsteh	%v0, 0, 0
+	vsteh	%v0, 0, 7
+	vsteh	%v0, 4095, 0
+	vsteh	%v0, 0(%r15), 0
+	vsteh	%v0, 0(%r15,%r1), 0
+	vsteh	%v15, 0, 0
+	vsteh	%v31, 0, 0
+	vsteh	%v18, 1383(%r3,%r4), 4
+
+#CHECK: vstl    %v0, %r0, 0             # encoding: [0xe7,0x00,0x00,0x00,0x00,0x3f]
+#CHECK: vstl    %v0, %r0, 4095          # encoding: [0xe7,0x00,0x0f,0xff,0x00,0x3f]
+#CHECK: vstl    %v0, %r0, 0(%r15)       # encoding: [0xe7,0x00,0xf0,0x00,0x00,0x3f]
+#CHECK: vstl    %v0, %r15, 0            # encoding: [0xe7,0x0f,0x00,0x00,0x00,0x3f]
+#CHECK: vstl    %v15, %r0, 0            # encoding: [0xe7,0xf0,0x00,0x00,0x00,0x3f]
+#CHECK: vstl    %v31, %r0, 0            # encoding: [0xe7,0xf0,0x00,0x00,0x08,0x3f]
+#CHECK: vstl    %v18, %r3, 1383(%r4)    # encoding: [0xe7,0x23,0x45,0x67,0x08,0x3f]
+
+	vstl	%v0, %r0, 0
+	vstl	%v0, %r0, 4095
+	vstl	%v0, %r0, 0(%r15)
+	vstl	%v0, %r15, 0
+	vstl	%v15, %r0, 0
+	vstl	%v31, %r0, 0
+	vstl	%v18, %r3, 1383(%r4)
+
+#CHECK: vstm    %v0, %v0, 0             # encoding: [0xe7,0x00,0x00,0x00,0x00,0x3e]
+#CHECK: vstm    %v0, %v0, 4095          # encoding: [0xe7,0x00,0x0f,0xff,0x00,0x3e]
+#CHECK: vstm    %v0, %v0, 0(%r15)       # encoding: [0xe7,0x00,0xf0,0x00,0x00,0x3e]
+#CHECK: vstm    %v0, %v31, 0            # encoding: [0xe7,0x0f,0x00,0x00,0x04,0x3e]
+#CHECK: vstm    %v31, %v0, 0            # encoding: [0xe7,0xf0,0x00,0x00,0x08,0x3e]
+#CHECK: vstm    %v14, %v17, 1074(%r5)   # encoding: [0xe7,0xe1,0x54,0x32,0x04,0x3e]
+
+	vstm	%v0, %v0, 0
+	vstm	%v0, %v0, 4095
+	vstm	%v0, %v0, 0(%r15)
+	vstm	%v0, %v31, 0
+	vstm	%v31, %v0, 0
+	vstm	%v14, %v17, 1074(%r5)
+
+#CHECK: vstrcb   %v0, %v0, %v0, %v0, 0   # encoding: [0xe7,0x00,0x00,0x00,0x00,0x8a]
+#CHECK: vstrcb   %v0, %v0, %v0, %v0, 0   # encoding: [0xe7,0x00,0x00,0x00,0x00,0x8a]
+#CHECK: vstrcb   %v0, %v0, %v0, %v0, 12  # encoding: [0xe7,0x00,0x00,0xc0,0x00,0x8a]
+#CHECK: vstrcb   %v0, %v0, %v0, %v15, 0  # encoding: [0xe7,0x00,0x00,0x00,0xf0,0x8a]
+#CHECK: vstrcb   %v0, %v0, %v0, %v31, 0  # encoding: [0xe7,0x00,0x00,0x00,0xf1,0x8a]
+#CHECK: vstrcb   %v0, %v0, %v15, %v0, 0  # encoding: [0xe7,0x00,0xf0,0x00,0x00,0x8a]
+#CHECK: vstrcb   %v0, %v0, %v31, %v0, 0  # encoding: [0xe7,0x00,0xf0,0x00,0x02,0x8a]
+#CHECK: vstrcb   %v0, %v15, %v0, %v0, 0  # encoding: [0xe7,0x0f,0x00,0x00,0x00,0x8a]
+#CHECK: vstrcb   %v0, %v31, %v0, %v0, 0  # encoding: [0xe7,0x0f,0x00,0x00,0x04,0x8a]
+#CHECK: vstrcb   %v15, %v0, %v0, %v0, 0  # encoding: [0xe7,0xf0,0x00,0x00,0x00,0x8a]
+#CHECK: vstrcb   %v31, %v0, %v0, %v0, 0  # encoding: [0xe7,0xf0,0x00,0x00,0x08,0x8a]
+#CHECK: vstrcb   %v18, %v3, %v20, %v5, 4 # encoding: [0xe7,0x23,0x40,0x40,0x5a,0x8a]
+#CHECK: vstrcb   %v18, %v3, %v20, %v5, 15 # encoding: [0xe7,0x23,0x40,0xf0,0x5a,0x8a]
+#CHECK: vstrcbs  %v18, %v3, %v20, %v5, 8 # encoding: [0xe7,0x23,0x40,0x90,0x5a,0x8a]
+#CHECK: vstrczb  %v18, %v3, %v20, %v5, 4 # encoding: [0xe7,0x23,0x40,0x60,0x5a,0x8a]
+#CHECK: vstrczbs %v18, %v3, %v20, %v5, 8 # encoding: [0xe7,0x23,0x40,0xb0,0x5a,0x8a]
+#CHECK: vstrczbs %v18, %v3, %v20, %v5, 15 # encoding: [0xe7,0x23,0x40,0xf0,0x5a,0x8a]
+
+        vstrcb   %v0, %v0, %v0, %v0
+        vstrcb   %v0, %v0, %v0, %v0, 0
+        vstrcb   %v0, %v0, %v0, %v0, 12
+        vstrcb   %v0, %v0, %v0, %v15
+        vstrcb   %v0, %v0, %v0, %v31
+        vstrcb   %v0, %v0, %v15, %v0
+        vstrcb   %v0, %v0, %v31, %v0
+        vstrcb   %v0, %v15, %v0, %v0
+        vstrcb   %v0, %v31, %v0, %v0
+        vstrcb   %v15, %v0, %v0, %v0
+        vstrcb   %v31, %v0, %v0, %v0
+        vstrcb   %v18, %v3, %v20, %v5, 4
+        vstrcb   %v18, %v3, %v20, %v5, 15
+        vstrcbs  %v18, %v3, %v20, %v5, 8
+        vstrczb  %v18, %v3, %v20, %v5, 4
+        vstrczbs %v18, %v3, %v20, %v5, 8
+        vstrczbs %v18, %v3, %v20, %v5, 15
+
+#CHECK: vstrcf   %v0, %v0, %v0, %v0, 0   # encoding: [0xe7,0x00,0x02,0x00,0x00,0x8a]
+#CHECK: vstrcf   %v0, %v0, %v0, %v0, 0   # encoding: [0xe7,0x00,0x02,0x00,0x00,0x8a]
+#CHECK: vstrcf   %v0, %v0, %v0, %v0, 12  # encoding: [0xe7,0x00,0x02,0xc0,0x00,0x8a]
+#CHECK: vstrcf   %v0, %v0, %v0, %v15, 0  # encoding: [0xe7,0x00,0x02,0x00,0xf0,0x8a]
+#CHECK: vstrcf   %v0, %v0, %v0, %v31, 0  # encoding: [0xe7,0x00,0x02,0x00,0xf1,0x8a]
+#CHECK: vstrcf   %v0, %v0, %v15, %v0, 0  # encoding: [0xe7,0x00,0xf2,0x00,0x00,0x8a]
+#CHECK: vstrcf   %v0, %v0, %v31, %v0, 0  # encoding: [0xe7,0x00,0xf2,0x00,0x02,0x8a]
+#CHECK: vstrcf   %v0, %v15, %v0, %v0, 0  # encoding: [0xe7,0x0f,0x02,0x00,0x00,0x8a]
+#CHECK: vstrcf   %v0, %v31, %v0, %v0, 0  # encoding: [0xe7,0x0f,0x02,0x00,0x04,0x8a]
+#CHECK: vstrcf   %v15, %v0, %v0, %v0, 0  # encoding: [0xe7,0xf0,0x02,0x00,0x00,0x8a]
+#CHECK: vstrcf   %v31, %v0, %v0, %v0, 0  # encoding: [0xe7,0xf0,0x02,0x00,0x08,0x8a]
+#CHECK: vstrcf   %v18, %v3, %v20, %v5, 4 # encoding: [0xe7,0x23,0x42,0x40,0x5a,0x8a]
+#CHECK: vstrcf   %v18, %v3, %v20, %v5, 15 # encoding: [0xe7,0x23,0x42,0xf0,0x5a,0x8a]
+#CHECK: vstrcfs  %v18, %v3, %v20, %v5, 8 # encoding: [0xe7,0x23,0x42,0x90,0x5a,0x8a]
+#CHECK: vstrczf  %v18, %v3, %v20, %v5, 4 # encoding: [0xe7,0x23,0x42,0x60,0x5a,0x8a]
+#CHECK: vstrczfs %v18, %v3, %v20, %v5, 8 # encoding: [0xe7,0x23,0x42,0xb0,0x5a,0x8a]
+#CHECK: vstrczfs %v18, %v3, %v20, %v5, 15 # encoding: [0xe7,0x23,0x42,0xf0,0x5a,0x8a]
+
+        vstrcf   %v0, %v0, %v0, %v0
+        vstrcf   %v0, %v0, %v0, %v0, 0
+        vstrcf   %v0, %v0, %v0, %v0, 12
+        vstrcf   %v0, %v0, %v0, %v15
+        vstrcf   %v0, %v0, %v0, %v31
+        vstrcf   %v0, %v0, %v15, %v0
+        vstrcf   %v0, %v0, %v31, %v0
+        vstrcf   %v0, %v15, %v0, %v0
+        vstrcf   %v0, %v31, %v0, %v0
+        vstrcf   %v15, %v0, %v0, %v0
+        vstrcf   %v31, %v0, %v0, %v0
+        vstrcf   %v18, %v3, %v20, %v5, 4
+        vstrcf   %v18, %v3, %v20, %v5, 15
+        vstrcfs  %v18, %v3, %v20, %v5, 8
+        vstrczf  %v18, %v3, %v20, %v5, 4
+        vstrczfs %v18, %v3, %v20, %v5, 8
+        vstrczfs %v18, %v3, %v20, %v5, 15
+
+#CHECK: vstrch   %v0, %v0, %v0, %v0, 0   # encoding: [0xe7,0x00,0x01,0x00,0x00,0x8a]
+#CHECK: vstrch   %v0, %v0, %v0, %v0, 0   # encoding: [0xe7,0x00,0x01,0x00,0x00,0x8a]
+#CHECK: vstrch   %v0, %v0, %v0, %v0, 12  # encoding: [0xe7,0x00,0x01,0xc0,0x00,0x8a]
+#CHECK: vstrch   %v0, %v0, %v0, %v15, 0  # encoding: [0xe7,0x00,0x01,0x00,0xf0,0x8a]
+#CHECK: vstrch   %v0, %v0, %v0, %v31, 0  # encoding: [0xe7,0x00,0x01,0x00,0xf1,0x8a]
+#CHECK: vstrch   %v0, %v0, %v15, %v0, 0  # encoding: [0xe7,0x00,0xf1,0x00,0x00,0x8a]
+#CHECK: vstrch   %v0, %v0, %v31, %v0, 0  # encoding: [0xe7,0x00,0xf1,0x00,0x02,0x8a]
+#CHECK: vstrch   %v0, %v15, %v0, %v0, 0  # encoding: [0xe7,0x0f,0x01,0x00,0x00,0x8a]
+#CHECK: vstrch   %v0, %v31, %v0, %v0, 0  # encoding: [0xe7,0x0f,0x01,0x00,0x04,0x8a]
+#CHECK: vstrch   %v15, %v0, %v0, %v0, 0  # encoding: [0xe7,0xf0,0x01,0x00,0x00,0x8a]
+#CHECK: vstrch   %v31, %v0, %v0, %v0, 0  # encoding: [0xe7,0xf0,0x01,0x00,0x08,0x8a]
+#CHECK: vstrch   %v18, %v3, %v20, %v5, 4 # encoding: [0xe7,0x23,0x41,0x40,0x5a,0x8a]
+#CHECK: vstrch   %v18, %v3, %v20, %v5, 15 # encoding: [0xe7,0x23,0x41,0xf0,0x5a,0x8a]
+#CHECK: vstrchs  %v18, %v3, %v20, %v5, 8 # encoding: [0xe7,0x23,0x41,0x90,0x5a,0x8a]
+#CHECK: vstrczh  %v18, %v3, %v20, %v5, 4 # encoding: [0xe7,0x23,0x41,0x60,0x5a,0x8a]
+#CHECK: vstrczhs %v18, %v3, %v20, %v5, 8 # encoding: [0xe7,0x23,0x41,0xb0,0x5a,0x8a]
+#CHECK: vstrczhs %v18, %v3, %v20, %v5, 15 # encoding: [0xe7,0x23,0x41,0xf0,0x5a,0x8a]
+
+        vstrch   %v0, %v0, %v0, %v0
+        vstrch   %v0, %v0, %v0, %v0, 0
+        vstrch   %v0, %v0, %v0, %v0, 12
+        vstrch   %v0, %v0, %v0, %v15
+        vstrch   %v0, %v0, %v0, %v31
+        vstrch   %v0, %v0, %v15, %v0
+        vstrch   %v0, %v0, %v31, %v0
+        vstrch   %v0, %v15, %v0, %v0
+        vstrch   %v0, %v31, %v0, %v0
+        vstrch   %v15, %v0, %v0, %v0
+        vstrch   %v31, %v0, %v0, %v0
+        vstrch   %v18, %v3, %v20, %v5, 4
+        vstrch   %v18, %v3, %v20, %v5, 15
+        vstrchs  %v18, %v3, %v20, %v5, 8
+        vstrczh  %v18, %v3, %v20, %v5, 4
+        vstrczhs %v18, %v3, %v20, %v5, 8
+        vstrczhs %v18, %v3, %v20, %v5, 15
+
+#CHECK: vsumgh  %v0, %v0, %v0           # encoding: [0xe7,0x00,0x00,0x00,0x10,0x65]
+#CHECK: vsumgh  %v0, %v0, %v31          # encoding: [0xe7,0x00,0xf0,0x00,0x12,0x65]
+#CHECK: vsumgh  %v0, %v31, %v0          # encoding: [0xe7,0x0f,0x00,0x00,0x14,0x65]
+#CHECK: vsumgh  %v31, %v0, %v0          # encoding: [0xe7,0xf0,0x00,0x00,0x18,0x65]
+#CHECK: vsumgh  %v18, %v3, %v20         # encoding: [0xe7,0x23,0x40,0x00,0x1a,0x65]
+
+	vsumgh	%v0, %v0, %v0
+	vsumgh	%v0, %v0, %v31
+	vsumgh	%v0, %v31, %v0
+	vsumgh	%v31, %v0, %v0
+	vsumgh	%v18, %v3, %v20
+
+#CHECK: vsumgf  %v0, %v0, %v0           # encoding: [0xe7,0x00,0x00,0x00,0x20,0x65]
+#CHECK: vsumgf  %v0, %v0, %v31          # encoding: [0xe7,0x00,0xf0,0x00,0x22,0x65]
+#CHECK: vsumgf  %v0, %v31, %v0          # encoding: [0xe7,0x0f,0x00,0x00,0x24,0x65]
+#CHECK: vsumgf  %v31, %v0, %v0          # encoding: [0xe7,0xf0,0x00,0x00,0x28,0x65]
+#CHECK: vsumgf  %v18, %v3, %v20         # encoding: [0xe7,0x23,0x40,0x00,0x2a,0x65]
+
+	vsumgf	%v0, %v0, %v0
+	vsumgf	%v0, %v0, %v31
+	vsumgf	%v0, %v31, %v0
+	vsumgf	%v31, %v0, %v0
+	vsumgf	%v18, %v3, %v20
+
+#CHECK: vsumqf  %v0, %v0, %v0           # encoding: [0xe7,0x00,0x00,0x00,0x20,0x67]
+#CHECK: vsumqf  %v0, %v0, %v31          # encoding: [0xe7,0x00,0xf0,0x00,0x22,0x67]
+#CHECK: vsumqf  %v0, %v31, %v0          # encoding: [0xe7,0x0f,0x00,0x00,0x24,0x67]
+#CHECK: vsumqf  %v31, %v0, %v0          # encoding: [0xe7,0xf0,0x00,0x00,0x28,0x67]
+#CHECK: vsumqf  %v18, %v3, %v20         # encoding: [0xe7,0x23,0x40,0x00,0x2a,0x67]
+
+	vsumqf	%v0, %v0, %v0
+	vsumqf	%v0, %v0, %v31
+	vsumqf	%v0, %v31, %v0
+	vsumqf	%v31, %v0, %v0
+	vsumqf	%v18, %v3, %v20
+
+#CHECK: vsumqg  %v0, %v0, %v0           # encoding: [0xe7,0x00,0x00,0x00,0x30,0x67]
+#CHECK: vsumqg  %v0, %v0, %v31          # encoding: [0xe7,0x00,0xf0,0x00,0x32,0x67]
+#CHECK: vsumqg  %v0, %v31, %v0          # encoding: [0xe7,0x0f,0x00,0x00,0x34,0x67]
+#CHECK: vsumqg  %v31, %v0, %v0          # encoding: [0xe7,0xf0,0x00,0x00,0x38,0x67]
+#CHECK: vsumqg  %v18, %v3, %v20         # encoding: [0xe7,0x23,0x40,0x00,0x3a,0x67]
+
+	vsumqg	%v0, %v0, %v0
+	vsumqg	%v0, %v0, %v31
+	vsumqg	%v0, %v31, %v0
+	vsumqg	%v31, %v0, %v0
+	vsumqg	%v18, %v3, %v20
+
+#CHECK: vsumb   %v0, %v0, %v0           # encoding: [0xe7,0x00,0x00,0x00,0x00,0x64]
+#CHECK: vsumb   %v0, %v0, %v31          # encoding: [0xe7,0x00,0xf0,0x00,0x02,0x64]
+#CHECK: vsumb   %v0, %v31, %v0          # encoding: [0xe7,0x0f,0x00,0x00,0x04,0x64]
+#CHECK: vsumb   %v31, %v0, %v0          # encoding: [0xe7,0xf0,0x00,0x00,0x08,0x64]
+#CHECK: vsumb   %v18, %v3, %v20         # encoding: [0xe7,0x23,0x40,0x00,0x0a,0x64]
+
+	vsumb	%v0, %v0, %v0
+	vsumb	%v0, %v0, %v31
+	vsumb	%v0, %v31, %v0
+	vsumb	%v31, %v0, %v0
+	vsumb	%v18, %v3, %v20
+
+#CHECK: vsumh   %v0, %v0, %v0           # encoding: [0xe7,0x00,0x00,0x00,0x10,0x64]
+#CHECK: vsumh   %v0, %v0, %v31          # encoding: [0xe7,0x00,0xf0,0x00,0x12,0x64]
+#CHECK: vsumh   %v0, %v31, %v0          # encoding: [0xe7,0x0f,0x00,0x00,0x14,0x64]
+#CHECK: vsumh   %v31, %v0, %v0          # encoding: [0xe7,0xf0,0x00,0x00,0x18,0x64]
+#CHECK: vsumh   %v18, %v3, %v20         # encoding: [0xe7,0x23,0x40,0x00,0x1a,0x64]
+
+	vsumh	%v0, %v0, %v0
+	vsumh	%v0, %v0, %v31
+	vsumh	%v0, %v31, %v0
+	vsumh	%v31, %v0, %v0
+	vsumh	%v18, %v3, %v20
+
+#CHECK: vtm     %v0, %v0                # encoding: [0xe7,0x00,0x00,0x00,0x00,0xd8]
+#CHECK: vtm     %v0, %v15               # encoding: [0xe7,0x0f,0x00,0x00,0x00,0xd8]
+#CHECK: vtm     %v0, %v31               # encoding: [0xe7,0x0f,0x00,0x00,0x04,0xd8]
+#CHECK: vtm     %v15, %v0               # encoding: [0xe7,0xf0,0x00,0x00,0x00,0xd8]
+#CHECK: vtm     %v31, %v0               # encoding: [0xe7,0xf0,0x00,0x00,0x08,0xd8]
+#CHECK: vtm     %v14, %v17              # encoding: [0xe7,0xe1,0x00,0x00,0x04,0xd8]
+
+	vtm	%v0, %v0
+	vtm	%v0, %v15
+	vtm	%v0, %v31
+	vtm	%v15, %v0
+	vtm	%v31, %v0
+	vtm	%v14, %v17
+
+#CHECK: vuphb   %v0, %v0                # encoding: [0xe7,0x00,0x00,0x00,0x00,0xd7]
+#CHECK: vuphb   %v0, %v15               # encoding: [0xe7,0x0f,0x00,0x00,0x00,0xd7]
+#CHECK: vuphb   %v0, %v31               # encoding: [0xe7,0x0f,0x00,0x00,0x04,0xd7]
+#CHECK: vuphb   %v15, %v0               # encoding: [0xe7,0xf0,0x00,0x00,0x00,0xd7]
+#CHECK: vuphb   %v31, %v0               # encoding: [0xe7,0xf0,0x00,0x00,0x08,0xd7]
+#CHECK: vuphb   %v14, %v17              # encoding: [0xe7,0xe1,0x00,0x00,0x04,0xd7]
+
+	vuphb	%v0, %v0
+	vuphb	%v0, %v15
+	vuphb	%v0, %v31
+	vuphb	%v15, %v0
+	vuphb	%v31, %v0
+	vuphb	%v14, %v17
+
+#CHECK: vuphf   %v0, %v0                # encoding: [0xe7,0x00,0x00,0x00,0x20,0xd7]
+#CHECK: vuphf   %v0, %v15               # encoding: [0xe7,0x0f,0x00,0x00,0x20,0xd7]
+#CHECK: vuphf   %v0, %v31               # encoding: [0xe7,0x0f,0x00,0x00,0x24,0xd7]
+#CHECK: vuphf   %v15, %v0               # encoding: [0xe7,0xf0,0x00,0x00,0x20,0xd7]
+#CHECK: vuphf   %v31, %v0               # encoding: [0xe7,0xf0,0x00,0x00,0x28,0xd7]
+#CHECK: vuphf   %v14, %v17              # encoding: [0xe7,0xe1,0x00,0x00,0x24,0xd7]
+
+	vuphf	%v0, %v0
+	vuphf	%v0, %v15
+	vuphf	%v0, %v31
+	vuphf	%v15, %v0
+	vuphf	%v31, %v0
+	vuphf	%v14, %v17
+
+#CHECK: vuphh   %v0, %v0                # encoding: [0xe7,0x00,0x00,0x00,0x10,0xd7]
+#CHECK: vuphh   %v0, %v15               # encoding: [0xe7,0x0f,0x00,0x00,0x10,0xd7]
+#CHECK: vuphh   %v0, %v31               # encoding: [0xe7,0x0f,0x00,0x00,0x14,0xd7]
+#CHECK: vuphh   %v15, %v0               # encoding: [0xe7,0xf0,0x00,0x00,0x10,0xd7]
+#CHECK: vuphh   %v31, %v0               # encoding: [0xe7,0xf0,0x00,0x00,0x18,0xd7]
+#CHECK: vuphh   %v14, %v17              # encoding: [0xe7,0xe1,0x00,0x00,0x14,0xd7]
+
+	vuphh	%v0, %v0
+	vuphh	%v0, %v15
+	vuphh	%v0, %v31
+	vuphh	%v15, %v0
+	vuphh	%v31, %v0
+	vuphh	%v14, %v17
+
+#CHECK: vuplhb  %v0, %v0                # encoding: [0xe7,0x00,0x00,0x00,0x00,0xd5]
+#CHECK: vuplhb  %v0, %v15               # encoding: [0xe7,0x0f,0x00,0x00,0x00,0xd5]
+#CHECK: vuplhb  %v0, %v31               # encoding: [0xe7,0x0f,0x00,0x00,0x04,0xd5]
+#CHECK: vuplhb  %v15, %v0               # encoding: [0xe7,0xf0,0x00,0x00,0x00,0xd5]
+#CHECK: vuplhb  %v31, %v0               # encoding: [0xe7,0xf0,0x00,0x00,0x08,0xd5]
+#CHECK: vuplhb  %v14, %v17              # encoding: [0xe7,0xe1,0x00,0x00,0x04,0xd5]
+
+	vuplhb	%v0, %v0
+	vuplhb	%v0, %v15
+	vuplhb	%v0, %v31
+	vuplhb	%v15, %v0
+	vuplhb	%v31, %v0
+	vuplhb	%v14, %v17
+
+#CHECK: vuplhf  %v0, %v0                # encoding: [0xe7,0x00,0x00,0x00,0x20,0xd5]
+#CHECK: vuplhf  %v0, %v15               # encoding: [0xe7,0x0f,0x00,0x00,0x20,0xd5]
+#CHECK: vuplhf  %v0, %v31               # encoding: [0xe7,0x0f,0x00,0x00,0x24,0xd5]
+#CHECK: vuplhf  %v15, %v0               # encoding: [0xe7,0xf0,0x00,0x00,0x20,0xd5]
+#CHECK: vuplhf  %v31, %v0               # encoding: [0xe7,0xf0,0x00,0x00,0x28,0xd5]
+#CHECK: vuplhf  %v14, %v17              # encoding: [0xe7,0xe1,0x00,0x00,0x24,0xd5]
+
+	vuplhf	%v0, %v0
+	vuplhf	%v0, %v15
+	vuplhf	%v0, %v31
+	vuplhf	%v15, %v0
+	vuplhf	%v31, %v0
+	vuplhf	%v14, %v17
+
+#CHECK: vuplhh  %v0, %v0                # encoding: [0xe7,0x00,0x00,0x00,0x10,0xd5]
+#CHECK: vuplhh  %v0, %v15               # encoding: [0xe7,0x0f,0x00,0x00,0x10,0xd5]
+#CHECK: vuplhh  %v0, %v31               # encoding: [0xe7,0x0f,0x00,0x00,0x14,0xd5]
+#CHECK: vuplhh  %v15, %v0               # encoding: [0xe7,0xf0,0x00,0x00,0x10,0xd5]
+#CHECK: vuplhh  %v31, %v0               # encoding: [0xe7,0xf0,0x00,0x00,0x18,0xd5]
+#CHECK: vuplhh  %v14, %v17              # encoding: [0xe7,0xe1,0x00,0x00,0x14,0xd5]
+
+	vuplhh	%v0, %v0
+	vuplhh	%v0, %v15
+	vuplhh	%v0, %v31
+	vuplhh	%v15, %v0
+	vuplhh	%v31, %v0
+	vuplhh	%v14, %v17
+
+#CHECK: vuplb   %v0, %v0                # encoding: [0xe7,0x00,0x00,0x00,0x00,0xd6]
+#CHECK: vuplb   %v0, %v15               # encoding: [0xe7,0x0f,0x00,0x00,0x00,0xd6]
+#CHECK: vuplb   %v0, %v31               # encoding: [0xe7,0x0f,0x00,0x00,0x04,0xd6]
+#CHECK: vuplb   %v15, %v0               # encoding: [0xe7,0xf0,0x00,0x00,0x00,0xd6]
+#CHECK: vuplb   %v31, %v0               # encoding: [0xe7,0xf0,0x00,0x00,0x08,0xd6]
+#CHECK: vuplb   %v14, %v17              # encoding: [0xe7,0xe1,0x00,0x00,0x04,0xd6]
+
+	vuplb	%v0, %v0
+	vuplb	%v0, %v15
+	vuplb	%v0, %v31
+	vuplb	%v15, %v0
+	vuplb	%v31, %v0
+	vuplb	%v14, %v17
+
+#CHECK: vuplf   %v0, %v0                # encoding: [0xe7,0x00,0x00,0x00,0x20,0xd6]
+#CHECK: vuplf   %v0, %v15               # encoding: [0xe7,0x0f,0x00,0x00,0x20,0xd6]
+#CHECK: vuplf   %v0, %v31               # encoding: [0xe7,0x0f,0x00,0x00,0x24,0xd6]
+#CHECK: vuplf   %v15, %v0               # encoding: [0xe7,0xf0,0x00,0x00,0x20,0xd6]
+#CHECK: vuplf   %v31, %v0               # encoding: [0xe7,0xf0,0x00,0x00,0x28,0xd6]
+#CHECK: vuplf   %v14, %v17              # encoding: [0xe7,0xe1,0x00,0x00,0x24,0xd6]
+
+	vuplf	%v0, %v0
+	vuplf	%v0, %v15
+	vuplf	%v0, %v31
+	vuplf	%v15, %v0
+	vuplf	%v31, %v0
+	vuplf	%v14, %v17
+
+#CHECK: vuplhw  %v0, %v0                # encoding: [0xe7,0x00,0x00,0x00,0x10,0xd6]
+#CHECK: vuplhw  %v0, %v15               # encoding: [0xe7,0x0f,0x00,0x00,0x10,0xd6]
+#CHECK: vuplhw  %v0, %v31               # encoding: [0xe7,0x0f,0x00,0x00,0x14,0xd6]
+#CHECK: vuplhw  %v15, %v0               # encoding: [0xe7,0xf0,0x00,0x00,0x10,0xd6]
+#CHECK: vuplhw  %v31, %v0               # encoding: [0xe7,0xf0,0x00,0x00,0x18,0xd6]
+#CHECK: vuplhw  %v14, %v17              # encoding: [0xe7,0xe1,0x00,0x00,0x14,0xd6]
+
+	vuplhw	%v0, %v0
+	vuplhw	%v0, %v15
+	vuplhw	%v0, %v31
+	vuplhw	%v15, %v0
+	vuplhw	%v31, %v0
+	vuplhw	%v14, %v17
+
+#CHECK: vupllb  %v0, %v0                # encoding: [0xe7,0x00,0x00,0x00,0x00,0xd4]
+#CHECK: vupllb  %v0, %v15               # encoding: [0xe7,0x0f,0x00,0x00,0x00,0xd4]
+#CHECK: vupllb  %v0, %v31               # encoding: [0xe7,0x0f,0x00,0x00,0x04,0xd4]
+#CHECK: vupllb  %v15, %v0               # encoding: [0xe7,0xf0,0x00,0x00,0x00,0xd4]
+#CHECK: vupllb  %v31, %v0               # encoding: [0xe7,0xf0,0x00,0x00,0x08,0xd4]
+#CHECK: vupllb  %v14, %v17              # encoding: [0xe7,0xe1,0x00,0x00,0x04,0xd4]
+
+	vupllb	%v0, %v0
+	vupllb	%v0, %v15
+	vupllb	%v0, %v31
+	vupllb	%v15, %v0
+	vupllb	%v31, %v0
+	vupllb	%v14, %v17
+
+#CHECK: vupllf  %v0, %v0                # encoding: [0xe7,0x00,0x00,0x00,0x20,0xd4]
+#CHECK: vupllf  %v0, %v15               # encoding: [0xe7,0x0f,0x00,0x00,0x20,0xd4]
+#CHECK: vupllf  %v0, %v31               # encoding: [0xe7,0x0f,0x00,0x00,0x24,0xd4]
+#CHECK: vupllf  %v15, %v0               # encoding: [0xe7,0xf0,0x00,0x00,0x20,0xd4]
+#CHECK: vupllf  %v31, %v0               # encoding: [0xe7,0xf0,0x00,0x00,0x28,0xd4]
+#CHECK: vupllf  %v14, %v17              # encoding: [0xe7,0xe1,0x00,0x00,0x24,0xd4]
+
+	vupllf	%v0, %v0
+	vupllf	%v0, %v15
+	vupllf	%v0, %v31
+	vupllf	%v15, %v0
+	vupllf	%v31, %v0
+	vupllf	%v14, %v17
+
+#CHECK: vupllh  %v0, %v0                # encoding: [0xe7,0x00,0x00,0x00,0x10,0xd4]
+#CHECK: vupllh  %v0, %v15               # encoding: [0xe7,0x0f,0x00,0x00,0x10,0xd4]
+#CHECK: vupllh  %v0, %v31               # encoding: [0xe7,0x0f,0x00,0x00,0x14,0xd4]
+#CHECK: vupllh  %v15, %v0               # encoding: [0xe7,0xf0,0x00,0x00,0x10,0xd4]
+#CHECK: vupllh  %v31, %v0               # encoding: [0xe7,0xf0,0x00,0x00,0x18,0xd4]
+#CHECK: vupllh  %v14, %v17              # encoding: [0xe7,0xe1,0x00,0x00,0x14,0xd4]
+
+	vupllh	%v0, %v0
+	vupllh	%v0, %v15
+	vupllh	%v0, %v31
+	vupllh	%v15, %v0
+	vupllh	%v31, %v0
+	vupllh	%v14, %v17
+
+#CHECK: vx      %v0, %v0, %v0           # encoding: [0xe7,0x00,0x00,0x00,0x00,0x6d]
+#CHECK: vx      %v0, %v0, %v31          # encoding: [0xe7,0x00,0xf0,0x00,0x02,0x6d]
+#CHECK: vx      %v0, %v31, %v0          # encoding: [0xe7,0x0f,0x00,0x00,0x04,0x6d]
+#CHECK: vx      %v31, %v0, %v0          # encoding: [0xe7,0xf0,0x00,0x00,0x08,0x6d]
+#CHECK: vx      %v18, %v3, %v20         # encoding: [0xe7,0x23,0x40,0x00,0x0a,0x6d]
+
+	vx	%v0, %v0, %v0
+	vx	%v0, %v0, %v31
+	vx	%v0, %v31, %v0
+	vx	%v31, %v0, %v0
+	vx	%v18, %v3, %v20
+
+#CHECK: vzero   %v0                     # encoding: [0xe7,0x00,0x00,0x00,0x00,0x44]
+#CHECK: vzero   %v11                    # encoding: [0xe7,0xb0,0x00,0x00,0x00,0x44]
+#CHECK: vzero   %v15                    # encoding: [0xe7,0xf0,0x00,0x00,0x00,0x44]
+#CHECK: vzero   %v31                    # encoding: [0xe7,0xf0,0x00,0x00,0x08,0x44]
+
+	vzero	%v0
+	vzero	%v11
+	vzero	%v15
+	vzero	%v31
+
+#CHECK: wcdgb   %f0, %f0, 0, 0          # encoding: [0xe7,0x00,0x00,0x08,0x30,0xc3]
+#CHECK: wcdgb   %f0, %f0, 0, 15         # encoding: [0xe7,0x00,0x00,0xf8,0x30,0xc3]
+#CHECK: wcdgb   %f0, %f0, 4, 0          # encoding: [0xe7,0x00,0x00,0x0c,0x30,0xc3]
+#CHECK: wcdgb   %f0, %f0, 12, 0         # encoding: [0xe7,0x00,0x00,0x0c,0x30,0xc3]
+#CHECK: wcdgb   %f0, %v31, 0, 0         # encoding: [0xe7,0x0f,0x00,0x08,0x34,0xc3]
+#CHECK: wcdgb   %v31, %f0, 0, 0         # encoding: [0xe7,0xf0,0x00,0x08,0x38,0xc3]
+#CHECK: wcdgb   %f14, %v17, 4, 10       # encoding: [0xe7,0xe1,0x00,0xac,0x34,0xc3]
+
+	wcdgb	%v0, %v0, 0, 0
+ 	wcdgb	%v0, %v0, 0, 15
+	wcdgb	%v0, %v0, 4, 0
+	wcdgb	%v0, %v0, 12, 0
+	wcdgb	%v0, %v31, 0, 0
+	wcdgb	%v31, %v0, 0, 0
+	wcdgb	%v14, %v17, 4, 10
+
+#CHECK: wcdlgb  %f0, %f0, 0, 0          # encoding: [0xe7,0x00,0x00,0x08,0x30,0xc1]
+#CHECK: wcdlgb  %f0, %f0, 0, 15         # encoding: [0xe7,0x00,0x00,0xf8,0x30,0xc1]
+#CHECK: wcdlgb  %f0, %f0, 4, 0          # encoding: [0xe7,0x00,0x00,0x0c,0x30,0xc1]
+#CHECK: wcdlgb  %f0, %f0, 12, 0         # encoding: [0xe7,0x00,0x00,0x0c,0x30,0xc1]
+#CHECK: wcdlgb  %f0, %v31, 0, 0         # encoding: [0xe7,0x0f,0x00,0x08,0x34,0xc1]
+#CHECK: wcdlgb  %v31, %f0, 0, 0         # encoding: [0xe7,0xf0,0x00,0x08,0x38,0xc1]
+#CHECK: wcdlgb  %f14, %v17, 4, 10       # encoding: [0xe7,0xe1,0x00,0xac,0x34,0xc1]
+
+	wcdlgb	%v0, %v0, 0, 0
+ 	wcdlgb	%v0, %v0, 0, 15
+	wcdlgb	%v0, %v0, 4, 0
+	wcdlgb	%v0, %v0, 12, 0
+	wcdlgb	%v0, %v31, 0, 0
+	wcdlgb	%v31, %v0, 0, 0
+	wcdlgb	%v14, %v17, 4, 10
+
+#CHECK: wcgdb   %f0, %f0, 0, 0          # encoding: [0xe7,0x00,0x00,0x08,0x30,0xc2]
+#CHECK: wcgdb   %f0, %f0, 0, 15         # encoding: [0xe7,0x00,0x00,0xf8,0x30,0xc2]
+#CHECK: wcgdb   %f0, %f0, 4, 0          # encoding: [0xe7,0x00,0x00,0x0c,0x30,0xc2]
+#CHECK: wcgdb   %f0, %f0, 12, 0         # encoding: [0xe7,0x00,0x00,0x0c,0x30,0xc2]
+#CHECK: wcgdb   %f0, %v31, 0, 0         # encoding: [0xe7,0x0f,0x00,0x08,0x34,0xc2]
+#CHECK: wcgdb   %v31, %f0, 0, 0         # encoding: [0xe7,0xf0,0x00,0x08,0x38,0xc2]
+#CHECK: wcgdb   %f14, %v17, 4, 10       # encoding: [0xe7,0xe1,0x00,0xac,0x34,0xc2]
+
+	wcgdb	%v0, %v0, 0, 0
+ 	wcgdb	%v0, %v0, 0, 15
+	wcgdb	%v0, %v0, 4, 0
+	wcgdb	%v0, %v0, 12, 0
+	wcgdb	%v0, %v31, 0, 0
+	wcgdb	%v31, %v0, 0, 0
+	wcgdb	%v14, %v17, 4, 10
+
+#CHECK: wclgdb  %f0, %f0, 0, 0          # encoding: [0xe7,0x00,0x00,0x08,0x30,0xc0]
+#CHECK: wclgdb  %f0, %f0, 0, 15         # encoding: [0xe7,0x00,0x00,0xf8,0x30,0xc0]
+#CHECK: wclgdb  %f0, %f0, 4, 0          # encoding: [0xe7,0x00,0x00,0x0c,0x30,0xc0]
+#CHECK: wclgdb  %f0, %f0, 12, 0         # encoding: [0xe7,0x00,0x00,0x0c,0x30,0xc0]
+#CHECK: wclgdb  %f0, %v31, 0, 0         # encoding: [0xe7,0x0f,0x00,0x08,0x34,0xc0]
+#CHECK: wclgdb  %v31, %f0, 0, 0         # encoding: [0xe7,0xf0,0x00,0x08,0x38,0xc0]
+#CHECK: wclgdb  %f14, %v17, 4, 10       # encoding: [0xe7,0xe1,0x00,0xac,0x34,0xc0]
+
+	wclgdb	%v0, %v0, 0, 0
+ 	wclgdb	%v0, %v0, 0, 15
+	wclgdb	%v0, %v0, 4, 0
+	wclgdb	%v0, %v0, 12, 0
+	wclgdb	%v0, %v31, 0, 0
+	wclgdb	%v31, %v0, 0, 0
+	wclgdb	%v14, %v17, 4, 10
+
+#CHECK: wfadb   %f0, %f0, %f0           # encoding: [0xe7,0x00,0x00,0x08,0x30,0xe3]
+#CHECK: wfadb   %f0, %f0, %v31          # encoding: [0xe7,0x00,0xf0,0x08,0x32,0xe3]
+#CHECK: wfadb   %f0, %v31, %f0          # encoding: [0xe7,0x0f,0x00,0x08,0x34,0xe3]
+#CHECK: wfadb   %v31, %f0, %f0          # encoding: [0xe7,0xf0,0x00,0x08,0x38,0xe3]
+#CHECK: wfadb   %v18, %f3, %v20         # encoding: [0xe7,0x23,0x40,0x08,0x3a,0xe3]
+
+	wfadb	%v0, %v0, %v0
+	wfadb	%v0, %v0, %v31
+	wfadb	%v0, %v31, %v0
+	wfadb	%v31, %v0, %v0
+	wfadb	%v18, %v3, %v20
+
+#CHECK: wfcdb   %f0, %f0                # encoding: [0xe7,0x00,0x00,0x00,0x30,0xcb]
+#CHECK: wfcdb   %f0, %f15               # encoding: [0xe7,0x0f,0x00,0x00,0x30,0xcb]
+#CHECK: wfcdb   %f0, %v31               # encoding: [0xe7,0x0f,0x00,0x00,0x34,0xcb]
+#CHECK: wfcdb   %f15, %f0               # encoding: [0xe7,0xf0,0x00,0x00,0x30,0xcb]
+#CHECK: wfcdb   %v31, %f0               # encoding: [0xe7,0xf0,0x00,0x00,0x38,0xcb]
+#CHECK: wfcdb   %f14, %v17              # encoding: [0xe7,0xe1,0x00,0x00,0x34,0xcb]
+
+	wfcdb	%v0, %v0
+	wfcdb	%v0, %v15
+	wfcdb	%v0, %v31
+	wfcdb	%v15, %v0
+	wfcdb	%v31, %v0
+	wfcdb	%v14, %v17
+
+#CHECK: wfcedb  %f0, %f0, %f0           # encoding: [0xe7,0x00,0x00,0x08,0x30,0xe8]
+#CHECK: wfcedb  %f0, %f0, %v31          # encoding: [0xe7,0x00,0xf0,0x08,0x32,0xe8]
+#CHECK: wfcedb  %f0, %v31, %f0          # encoding: [0xe7,0x0f,0x00,0x08,0x34,0xe8]
+#CHECK: wfcedb  %v31, %f0, %f0          # encoding: [0xe7,0xf0,0x00,0x08,0x38,0xe8]
+#CHECK: wfcedb  %v18, %f3, %v20         # encoding: [0xe7,0x23,0x40,0x08,0x3a,0xe8]
+
+	wfcedb	%v0, %v0, %v0
+	wfcedb	%v0, %v0, %v31
+	wfcedb	%v0, %v31, %v0
+	wfcedb	%v31, %v0, %v0
+	wfcedb	%v18, %v3, %v20
+
+#CHECK: wfcedbs %f0, %f0, %f0           # encoding: [0xe7,0x00,0x00,0x18,0x30,0xe8]
+#CHECK: wfcedbs %f0, %f0, %v31          # encoding: [0xe7,0x00,0xf0,0x18,0x32,0xe8]
+#CHECK: wfcedbs %f0, %v31, %f0          # encoding: [0xe7,0x0f,0x00,0x18,0x34,0xe8]
+#CHECK: wfcedbs %v31, %f0, %f0          # encoding: [0xe7,0xf0,0x00,0x18,0x38,0xe8]
+#CHECK: wfcedbs %v18, %f3, %v20         # encoding: [0xe7,0x23,0x40,0x18,0x3a,0xe8]
+
+	wfcedbs	%v0, %v0, %v0
+	wfcedbs	%v0, %v0, %v31
+	wfcedbs	%v0, %v31, %v0
+	wfcedbs	%v31, %v0, %v0
+	wfcedbs	%v18, %v3, %v20
+
+#CHECK: wfchdb  %f0, %f0, %f0           # encoding: [0xe7,0x00,0x00,0x08,0x30,0xeb]
+#CHECK: wfchdb  %f0, %f0, %v31          # encoding: [0xe7,0x00,0xf0,0x08,0x32,0xeb]
+#CHECK: wfchdb  %f0, %v31, %f0          # encoding: [0xe7,0x0f,0x00,0x08,0x34,0xeb]
+#CHECK: wfchdb  %v31, %f0, %f0          # encoding: [0xe7,0xf0,0x00,0x08,0x38,0xeb]
+#CHECK: wfchdb  %v18, %f3, %v20         # encoding: [0xe7,0x23,0x40,0x08,0x3a,0xeb]
+
+	wfchdb	%v0, %v0, %v0
+	wfchdb	%v0, %v0, %v31
+	wfchdb	%v0, %v31, %v0
+	wfchdb	%v31, %v0, %v0
+	wfchdb	%v18, %v3, %v20
+
+#CHECK: wfchdbs %f0, %f0, %f0           # encoding: [0xe7,0x00,0x00,0x18,0x30,0xeb]
+#CHECK: wfchdbs %f0, %f0, %v31          # encoding: [0xe7,0x00,0xf0,0x18,0x32,0xeb]
+#CHECK: wfchdbs %f0, %v31, %f0          # encoding: [0xe7,0x0f,0x00,0x18,0x34,0xeb]
+#CHECK: wfchdbs %v31, %f0, %f0          # encoding: [0xe7,0xf0,0x00,0x18,0x38,0xeb]
+#CHECK: wfchdbs %v18, %f3, %v20         # encoding: [0xe7,0x23,0x40,0x18,0x3a,0xeb]
+
+	wfchdbs	%v0, %v0, %v0
+	wfchdbs	%v0, %v0, %v31
+	wfchdbs	%v0, %v31, %v0
+	wfchdbs	%v31, %v0, %v0
+	wfchdbs	%v18, %v3, %v20
+
+#CHECK: wfchedb %f0, %f0, %f0           # encoding: [0xe7,0x00,0x00,0x08,0x30,0xea]
+#CHECK: wfchedb %f0, %f0, %v31          # encoding: [0xe7,0x00,0xf0,0x08,0x32,0xea]
+#CHECK: wfchedb %f0, %v31, %f0          # encoding: [0xe7,0x0f,0x00,0x08,0x34,0xea]
+#CHECK: wfchedb %v31, %f0, %f0          # encoding: [0xe7,0xf0,0x00,0x08,0x38,0xea]
+#CHECK: wfchedb %v18, %f3, %v20         # encoding: [0xe7,0x23,0x40,0x08,0x3a,0xea]
+
+	wfchedb	%v0, %v0, %v0
+	wfchedb	%v0, %v0, %v31
+	wfchedb	%v0, %v31, %v0
+	wfchedb	%v31, %v0, %v0
+	wfchedb	%v18, %v3, %v20
+
+#CHECK: wfchedbs %f0, %f0, %f0          # encoding: [0xe7,0x00,0x00,0x18,0x30,0xea]
+#CHECK: wfchedbs %f0, %f0, %v31         # encoding: [0xe7,0x00,0xf0,0x18,0x32,0xea]
+#CHECK: wfchedbs %f0, %v31, %f0         # encoding: [0xe7,0x0f,0x00,0x18,0x34,0xea]
+#CHECK: wfchedbs %v31, %f0, %f0         # encoding: [0xe7,0xf0,0x00,0x18,0x38,0xea]
+#CHECK: wfchedbs %v18, %f3, %v20        # encoding: [0xe7,0x23,0x40,0x18,0x3a,0xea]
+
+	wfchedbs %v0, %v0, %v0
+	wfchedbs %v0, %v0, %v31
+	wfchedbs %v0, %v31, %v0
+	wfchedbs %v31, %v0, %v0
+	wfchedbs %v18, %v3, %v20
+
+#CHECK: wfddb   %f0, %f0, %f0           # encoding: [0xe7,0x00,0x00,0x08,0x30,0xe5]
+#CHECK: wfddb   %f0, %f0, %v31          # encoding: [0xe7,0x00,0xf0,0x08,0x32,0xe5]
+#CHECK: wfddb   %f0, %v31, %f0          # encoding: [0xe7,0x0f,0x00,0x08,0x34,0xe5]
+#CHECK: wfddb   %v31, %f0, %f0          # encoding: [0xe7,0xf0,0x00,0x08,0x38,0xe5]
+#CHECK: wfddb   %v18, %f3, %v20         # encoding: [0xe7,0x23,0x40,0x08,0x3a,0xe5]
+
+	wfddb	%v0, %v0, %v0
+	wfddb	%v0, %v0, %v31
+	wfddb	%v0, %v31, %v0
+	wfddb	%v31, %v0, %v0
+	wfddb	%v18, %v3, %v20
+
+#CHECK: wfidb   %f0, %f0, 0, 0          # encoding: [0xe7,0x00,0x00,0x08,0x30,0xc7]
+#CHECK: wfidb   %f0, %f0, 0, 15         # encoding: [0xe7,0x00,0x00,0xf8,0x30,0xc7]
+#CHECK: wfidb   %f0, %f0, 4, 0          # encoding: [0xe7,0x00,0x00,0x0c,0x30,0xc7]
+#CHECK: wfidb   %f0, %f0, 12, 0         # encoding: [0xe7,0x00,0x00,0x0c,0x30,0xc7]
+#CHECK: wfidb   %f0, %v31, 0, 0         # encoding: [0xe7,0x0f,0x00,0x08,0x34,0xc7]
+#CHECK: wfidb   %v31, %f0, 0, 0         # encoding: [0xe7,0xf0,0x00,0x08,0x38,0xc7]
+#CHECK: wfidb   %f14, %v17, 4, 10       # encoding: [0xe7,0xe1,0x00,0xac,0x34,0xc7]
+
+	wfidb	%v0, %v0, 0, 0
+ 	wfidb	%v0, %v0, 0, 15
+	wfidb	%v0, %v0, 4, 0
+	wfidb	%v0, %v0, 12, 0
+	wfidb	%v0, %v31, 0, 0
+	wfidb	%v31, %v0, 0, 0
+	wfidb	%v14, %v17, 4, 10
+
+#CHECK: wfkdb   %f0, %f0                # encoding: [0xe7,0x00,0x00,0x00,0x30,0xca]
+#CHECK: wfkdb   %f0, %f15               # encoding: [0xe7,0x0f,0x00,0x00,0x30,0xca]
+#CHECK: wfkdb   %f0, %v31               # encoding: [0xe7,0x0f,0x00,0x00,0x34,0xca]
+#CHECK: wfkdb   %f15, %f0               # encoding: [0xe7,0xf0,0x00,0x00,0x30,0xca]
+#CHECK: wfkdb   %v31, %f0               # encoding: [0xe7,0xf0,0x00,0x00,0x38,0xca]
+#CHECK: wfkdb   %f14, %v17              # encoding: [0xe7,0xe1,0x00,0x00,0x34,0xca]
+
+	wfkdb	%v0, %v0
+	wfkdb	%v0, %v15
+	wfkdb	%v0, %v31
+	wfkdb	%v15, %v0
+	wfkdb	%v31, %v0
+	wfkdb	%v14, %v17
+
+#CHECK: wflcdb  %f0, %f0                # encoding: [0xe7,0x00,0x00,0x08,0x30,0xcc]
+#CHECK: wflcdb  %f0, %f15               # encoding: [0xe7,0x0f,0x00,0x08,0x30,0xcc]
+#CHECK: wflcdb  %f0, %v31               # encoding: [0xe7,0x0f,0x00,0x08,0x34,0xcc]
+#CHECK: wflcdb  %f15, %f0               # encoding: [0xe7,0xf0,0x00,0x08,0x30,0xcc]
+#CHECK: wflcdb  %v31, %f0               # encoding: [0xe7,0xf0,0x00,0x08,0x38,0xcc]
+#CHECK: wflcdb  %f14, %v17              # encoding: [0xe7,0xe1,0x00,0x08,0x34,0xcc]
+
+	wflcdb	%v0, %v0
+	wflcdb	%v0, %v15
+	wflcdb	%v0, %v31
+	wflcdb	%v15, %v0
+	wflcdb	%v31, %v0
+	wflcdb	%v14, %v17
+
+#CHECK: wflndb  %f0, %f0                # encoding: [0xe7,0x00,0x00,0x18,0x30,0xcc]
+#CHECK: wflndb  %f0, %f15               # encoding: [0xe7,0x0f,0x00,0x18,0x30,0xcc]
+#CHECK: wflndb  %f0, %v31               # encoding: [0xe7,0x0f,0x00,0x18,0x34,0xcc]
+#CHECK: wflndb  %f15, %f0               # encoding: [0xe7,0xf0,0x00,0x18,0x30,0xcc]
+#CHECK: wflndb  %v31, %f0               # encoding: [0xe7,0xf0,0x00,0x18,0x38,0xcc]
+#CHECK: wflndb  %f14, %v17              # encoding: [0xe7,0xe1,0x00,0x18,0x34,0xcc]
+
+	wflndb	%v0, %v0
+	wflndb	%v0, %v15
+	wflndb	%v0, %v31
+	wflndb	%v15, %v0
+	wflndb	%v31, %v0
+	wflndb	%v14, %v17
+
+#CHECK: wflpdb  %f0, %f0                # encoding: [0xe7,0x00,0x00,0x28,0x30,0xcc]
+#CHECK: wflpdb  %f0, %f15               # encoding: [0xe7,0x0f,0x00,0x28,0x30,0xcc]
+#CHECK: wflpdb  %f0, %v31               # encoding: [0xe7,0x0f,0x00,0x28,0x34,0xcc]
+#CHECK: wflpdb  %f15, %f0               # encoding: [0xe7,0xf0,0x00,0x28,0x30,0xcc]
+#CHECK: wflpdb  %v31, %f0               # encoding: [0xe7,0xf0,0x00,0x28,0x38,0xcc]
+#CHECK: wflpdb  %f14, %v17              # encoding: [0xe7,0xe1,0x00,0x28,0x34,0xcc]
+
+	wflpdb	%v0, %v0
+	wflpdb	%v0, %v15
+	wflpdb	%v0, %v31
+	wflpdb	%v15, %v0
+	wflpdb	%v31, %v0
+	wflpdb	%v14, %v17
+
+#CHECK: wfmadb  %f0, %f0, %f0, %f0      # encoding: [0xe7,0x00,0x03,0x08,0x00,0x8f]
+#CHECK: wfmadb  %f0, %f0, %f0, %v31     # encoding: [0xe7,0x00,0x03,0x08,0xf1,0x8f]
+#CHECK: wfmadb  %f0, %f0, %v31, %f0     # encoding: [0xe7,0x00,0xf3,0x08,0x02,0x8f]
+#CHECK: wfmadb  %f0, %v31, %f0, %f0     # encoding: [0xe7,0x0f,0x03,0x08,0x04,0x8f]
+#CHECK: wfmadb  %v31, %f0, %f0, %f0     # encoding: [0xe7,0xf0,0x03,0x08,0x08,0x8f]
+#CHECK: wfmadb  %f13, %v17, %v21, %v25  # encoding: [0xe7,0xd1,0x53,0x08,0x97,0x8f]
+
+	wfmadb	%v0, %v0, %v0, %v0
+	wfmadb	%v0, %v0, %v0, %v31
+	wfmadb	%v0, %v0, %v31, %v0
+	wfmadb	%v0, %v31, %v0, %v0
+	wfmadb	%v31, %v0, %v0, %v0
+	wfmadb	%v13, %v17, %v21, %v25
+
+#CHECK: wfmdb   %f0, %f0, %f0           # encoding: [0xe7,0x00,0x00,0x08,0x30,0xe7]
+#CHECK: wfmdb   %f0, %f0, %v31          # encoding: [0xe7,0x00,0xf0,0x08,0x32,0xe7]
+#CHECK: wfmdb   %f0, %v31, %f0          # encoding: [0xe7,0x0f,0x00,0x08,0x34,0xe7]
+#CHECK: wfmdb   %v31, %f0, %f0          # encoding: [0xe7,0xf0,0x00,0x08,0x38,0xe7]
+#CHECK: wfmdb   %v18, %f3, %v20         # encoding: [0xe7,0x23,0x40,0x08,0x3a,0xe7]
+
+	wfmdb	%v0, %v0, %v0
+	wfmdb	%v0, %v0, %v31
+	wfmdb	%v0, %v31, %v0
+	wfmdb	%v31, %v0, %v0
+	wfmdb	%v18, %v3, %v20
+
+#CHECK: wfmsdb  %f0, %f0, %f0, %f0      # encoding: [0xe7,0x00,0x03,0x08,0x00,0x8e]
+#CHECK: wfmsdb  %f0, %f0, %f0, %v31     # encoding: [0xe7,0x00,0x03,0x08,0xf1,0x8e]
+#CHECK: wfmsdb  %f0, %f0, %v31, %f0     # encoding: [0xe7,0x00,0xf3,0x08,0x02,0x8e]
+#CHECK: wfmsdb  %f0, %v31, %f0, %f0     # encoding: [0xe7,0x0f,0x03,0x08,0x04,0x8e]
+#CHECK: wfmsdb  %v31, %f0, %f0, %f0     # encoding: [0xe7,0xf0,0x03,0x08,0x08,0x8e]
+#CHECK: wfmsdb  %f13, %v17, %v21, %v25  # encoding: [0xe7,0xd1,0x53,0x08,0x97,0x8e]
+
+	wfmsdb	%v0, %v0, %v0, %v0
+	wfmsdb	%v0, %v0, %v0, %v31
+	wfmsdb	%v0, %v0, %v31, %v0
+	wfmsdb	%v0, %v31, %v0, %v0
+	wfmsdb	%v31, %v0, %v0, %v0
+	wfmsdb	%v13, %v17, %v21, %v25
+
+#CHECK: wfsdb   %f0, %f0, %f0           # encoding: [0xe7,0x00,0x00,0x08,0x30,0xe2]
+#CHECK: wfsdb   %f0, %f0, %v31          # encoding: [0xe7,0x00,0xf0,0x08,0x32,0xe2]
+#CHECK: wfsdb   %f0, %v31, %f0          # encoding: [0xe7,0x0f,0x00,0x08,0x34,0xe2]
+#CHECK: wfsdb   %v31, %f0, %f0          # encoding: [0xe7,0xf0,0x00,0x08,0x38,0xe2]
+#CHECK: wfsdb   %v18, %f3, %v20         # encoding: [0xe7,0x23,0x40,0x08,0x3a,0xe2]
+
+	wfsdb	%v0, %v0, %v0
+	wfsdb	%v0, %v0, %v31
+	wfsdb	%v0, %v31, %v0
+	wfsdb	%v31, %v0, %v0
+	wfsdb	%v18, %v3, %v20
+
+#CHECK: wfsqdb  %f0, %f0                # encoding: [0xe7,0x00,0x00,0x08,0x30,0xce]
+#CHECK: wfsqdb  %f0, %f15               # encoding: [0xe7,0x0f,0x00,0x08,0x30,0xce]
+#CHECK: wfsqdb  %f0, %v31               # encoding: [0xe7,0x0f,0x00,0x08,0x34,0xce]
+#CHECK: wfsqdb  %f15, %f0               # encoding: [0xe7,0xf0,0x00,0x08,0x30,0xce]
+#CHECK: wfsqdb  %v31, %f0               # encoding: [0xe7,0xf0,0x00,0x08,0x38,0xce]
+#CHECK: wfsqdb  %f14, %v17              # encoding: [0xe7,0xe1,0x00,0x08,0x34,0xce]
+
+	wfsqdb	%v0, %v0
+	wfsqdb	%v0, %v15
+	wfsqdb	%v0, %v31
+	wfsqdb	%v15, %v0
+	wfsqdb	%v31, %v0
+	wfsqdb	%v14, %v17
+
+#CHECK: wftcidb %f0, %f0, 0             # encoding: [0xe7,0x00,0x00,0x08,0x30,0x4a]
+#CHECK: wftcidb %f0, %f0, 4095          # encoding: [0xe7,0x00,0xff,0xf8,0x30,0x4a]
+#CHECK: wftcidb %f0, %f15, 0            # encoding: [0xe7,0x0f,0x00,0x08,0x30,0x4a]
+#CHECK: wftcidb %f0, %v31, 0            # encoding: [0xe7,0x0f,0x00,0x08,0x34,0x4a]
+#CHECK: wftcidb %f15, %f0, 0            # encoding: [0xe7,0xf0,0x00,0x08,0x30,0x4a]
+#CHECK: wftcidb %v31, %f0, 0            # encoding: [0xe7,0xf0,0x00,0x08,0x38,0x4a]
+#CHECK: wftcidb %f4, %v21, 1656         # encoding: [0xe7,0x45,0x67,0x88,0x34,0x4a]
+
+	wftcidb	%v0, %v0, 0
+	wftcidb	%v0, %v0, 4095
+	wftcidb	%v0, %v15, 0
+	wftcidb	%v0, %v31, 0
+	wftcidb	%v15, %v0, 0
+	wftcidb	%v31, %v0, 0
+	wftcidb	%v4, %v21, 0x678
+
+#CHECK: wldeb   %f0, %f0                # encoding: [0xe7,0x00,0x00,0x08,0x20,0xc4]
+#CHECK: wldeb   %f0, %f15               # encoding: [0xe7,0x0f,0x00,0x08,0x20,0xc4]
+#CHECK: wldeb   %f0, %v31               # encoding: [0xe7,0x0f,0x00,0x08,0x24,0xc4]
+#CHECK: wldeb   %f15, %f0               # encoding: [0xe7,0xf0,0x00,0x08,0x20,0xc4]
+#CHECK: wldeb   %v31, %f0               # encoding: [0xe7,0xf0,0x00,0x08,0x28,0xc4]
+#CHECK: wldeb   %f14, %v17              # encoding: [0xe7,0xe1,0x00,0x08,0x24,0xc4]
+
+	wldeb	%v0, %v0
+	wldeb	%v0, %v15
+	wldeb	%v0, %v31
+	wldeb	%v15, %v0
+	wldeb	%v31, %v0
+	wldeb	%v14, %v17
+
+#CHECK: wledb   %f0, %f0, 0, 0          # encoding: [0xe7,0x00,0x00,0x08,0x30,0xc5]
+#CHECK: wledb   %f0, %f0, 0, 15         # encoding: [0xe7,0x00,0x00,0xf8,0x30,0xc5]
+#CHECK: wledb   %f0, %f0, 4, 0          # encoding: [0xe7,0x00,0x00,0x0c,0x30,0xc5]
+#CHECK: wledb   %f0, %f0, 12, 0         # encoding: [0xe7,0x00,0x00,0x0c,0x30,0xc5]
+#CHECK: wledb   %f0, %v31, 0, 0         # encoding: [0xe7,0x0f,0x00,0x08,0x34,0xc5]
+#CHECK: wledb   %v31, %f0, 0, 0         # encoding: [0xe7,0xf0,0x00,0x08,0x38,0xc5]
+#CHECK: wledb   %f14, %v17, 4, 10       # encoding: [0xe7,0xe1,0x00,0xac,0x34,0xc5]
+
+	wledb	%v0, %v0, 0, 0
+ 	wledb	%v0, %v0, 0, 15
+	wledb	%v0, %v0, 4, 0
+	wledb	%v0, %v0, 12, 0
+	wledb	%v0, %v31, 0, 0
+	wledb	%v31, %v0, 0, 0
+	wledb	%v14, %v17, 4, 10
diff --git a/test/MC/SystemZ/tokens.s b/test/MC/SystemZ/tokens.s
index 2719752b736..ceadcdb4b1d 100644
--- a/test/MC/SystemZ/tokens.s
+++ b/test/MC/SystemZ/tokens.s
@@ -13,10 +13,16 @@
 #CHECK: foo	100(200,%r0), 300
 #CHECK: error: invalid instruction
 #CHECK: foo	100(200,%r1), 300
-#CHECK: error: invalid operand
+#CHECK: error: invalid address register
 #CHECK: foo	100(%a0), 200
 #CHECK: error: %r0 used in an address
 #CHECK: foo	100(%r0), 200
+#CHECK: error: %r0 used in an address
+#CHECK: foo	100(%v1,%r0), 200
+#CHECK: error: invalid instruction
+#CHECK: foo	100(%v0,%r1), 200
+#CHECK: error: invalid instruction
+#CHECK: foo	100(%v31), 200
 #CHECK: error: invalid operand
 #CHECK: foo	100(%r1,%a0), 200
 #CHECK: error: %r0 used in an address
@@ -45,6 +51,12 @@
 #CHECK: foo	%a15, 200
 #CHECK: error: invalid register
 #CHECK: foo	%a16, 200
+#CHECK: error: invalid instruction
+#CHECK: foo	%v0, 200
+#CHECK: error: invalid instruction
+#CHECK: foo	%v31, 200
+#CHECK: error: invalid register
+#CHECK: foo	%v32, 200
 #CHECK: error: invalid register
 #CHECK: foo	%c, 200
 #CHECK: error: invalid register
@@ -60,6 +72,9 @@
 	foo	100(200,%r1), 300
 	foo	100(%a0), 200
 	foo	100(%r0), 200
+	foo	100(%v1,%r0), 200
+	foo	100(%v0,%r1), 200
+	foo	100(%v31), 200
 	foo	100(%r1,%a0), 200
 	foo	100(%r1,%r0), 200
 	foo	100(%r1,%r2, 200
@@ -74,6 +89,9 @@
 	foo	%a0, 200
 	foo	%a15, 200
 	foo	%a16, 200
+	foo	%v0, 200
+	foo	%v31, 200
+	foo	%v32, 200
 	foo	%c, 200
 	foo	%, 200
 	foo	{, 200
diff --git a/test/MC/X86/AlignedBundling/bundle_straddle_prefix_inst.s b/test/MC/X86/AlignedBundling/bundle_straddle_prefix_inst.s
deleted file mode 100644
index 94406b19f8c..00000000000
--- a/test/MC/X86/AlignedBundling/bundle_straddle_prefix_inst.s
+++ /dev/null
@@ -1,44 +0,0 @@
-# RUN: llvm-mc -filetype=obj -triple i686-unknown-nacl %s -o - | \
-# RUN:   llvm-objdump -triple i686 -d - | FileCheck %s
-
-    .text
-    .globl test_lock_prefix
-    .type test_lock_prefix,@function
-    .p2align 5
-test_lock_prefix:
-    .fill 29, 1, 0x90
-    lock cmpxchg8b 0x0(%ebp)
-
-# CHECK-LABEL: test_lock_prefix
-# CHECK:       1c:  90          nop
-# CHECK-NEXT:  1d:  0f 1f 00    nopl (%eax)
-# CHECK-NEXT:  20:  f0          lock
-# CHECK-NEXT:  21:  0f c7 4d 00 cmpxchg8b (%ebp)
-
-    .globl  test_rep_prefix
-    .type   test_rep_prefix,@function
-    .p2align 5
-test_rep_prefix:
-    mov 100, %ecx
-    .fill 24, 1, 0x90
-    rep movsw
-
-# CHECK-LABEL: test_rep_prefix
-# CHECK:       5d:  90    nop
-# CHECK-NEXT:  5e:  66 90 nop
-# CHECK-NEXT:  60:  f3    rep
-# CHECK-NEXT:  61:  66 a5 movsw
-
-    .globl  test_repne_prefix
-    .type   test_repne_prefix,@function
-    .p2align 5
-test_repne_prefix:
-    mov 100, %ecx
-    .fill 24, 1, 0x90
-    repne scasw
-
-# CHECK-LABEL: test_repne_prefix
-# CHECK:       9d:  90    nop
-# CHECK-NEXT:  9e:  66 90 nop
-# CHECK-NEXT:  a0:  f2    repne
-# CHECK-NEXT:  a1:  66 af scasw
diff --git a/test/MC/X86/AlignedBundling/nacl-call-auto-align-to-end.s b/test/MC/X86/AlignedBundling/nacl-call-auto-align-to-end.s
deleted file mode 100644
index dd284c33f21..00000000000
--- a/test/MC/X86/AlignedBundling/nacl-call-auto-align-to-end.s
+++ /dev/null
@@ -1,47 +0,0 @@
-# RUN: llvm-mc -filetype=obj -triple i686-unknown-nacl %s -o - \
-# RUN:   | llvm-objdump -disassemble -no-show-raw-insn - | FileCheck %s
-
-# RUN: llvm-mc -filetype=obj -triple x86_64-unknown-nacl \
-# RUN: -sfi-hide-sandbox-base=false %s -o - \
-# RUN:   | llvm-objdump -disassemble -no-show-raw-insn - | FileCheck %s
-
-# RUN: llvm-mc -filetype=obj -triple x86_64-unknown-nacl \
-# RUN: -sfi-hide-sandbox-base=true %s -o - \
-# RUN:   | llvm-objdump -disassemble -no-show-raw-insn - \
-# RUN:   | FileCheck --check-prefix=HIDE %s
-
-# Test that bare call instructions in NaCl are automatically aligned to the end
-# of a bundle without the need for .bundle_align directives, to match gas's
-# behavior.
-  .text
-foo:
-# Each of these mov instructions is 4 bytes long.
-  movsd %xmm1,%xmm2
-  movsd %xmm1,%xmm2
-  movsd %xmm1,%xmm2
-  movsd %xmm1,%xmm2
-# Each of these movs is 5 bytes long.
-  movl $1, %eax
-  movl $1, %eax
-  call   bar
-# To align this call to a bundle end, we need a 1-byte NOP.
-# CHECK:        1a:  nop
-# CHECK-NEXT:   1b: call
-
-  movsd %xmm1,%xmm2
-  movsd %xmm1,%xmm2
-  movsd %xmm1,%xmm2
-  movsd %xmm1,%xmm2
-  movl $1, %eax
-  movl $1, %eax
-  movl $1, %eax
-  call   bar
-# Here we have to pad until the end of the *next* boundary because
-# otherwise the call crosses a boundary.
-# The last byte of the bundle has to be a 1-byte nop so it doesn't
-# cross the boundary itself.
-# CHECK:      3f: nop
-# The remaining nops can be implemented any way the compiler wants.
-# CHECK:      5b: call
-
-# HIDE-NOT: call
diff --git a/test/MC/X86/AlignedBundling/nesting.s b/test/MC/X86/AlignedBundling/nesting.s
index 74b8fe9ff49..16ed5a44da5 100644
--- a/test/MC/X86/AlignedBundling/nesting.s
+++ b/test/MC/X86/AlignedBundling/nesting.s
@@ -6,7 +6,8 @@
 # Will be bundle-aligning to 16 byte boundaries
   .bundle_align_mode 4
   .text
-# CHECK-LABEL: foo
+# CHECK-LABEL: foo:
+.type   foo,@function
 foo:
 # Test that bundle alignment mode can be set more than once.
   .bundle_align_mode 4
@@ -19,11 +20,12 @@ foo:
   callq bar     
   .bundle_unlock
   .bundle_unlock
-# CHECK:      10: callq
-# CHECK-NEXT: 15: callq
+# CHECK:      10: callq {{.*}} <bar>
+# CHECK-NEXT: 15: callq {{.*}} <bar>
 
   .p2align 4
-# CHECK-LABEL: bar
+# CHECK-LABEL: bar:
+.type   bar,@function
 bar:
   callq foo
   callq foo
@@ -35,10 +37,11 @@ bar:
   callq bar
   .bundle_unlock
   .bundle_unlock
-# CHECK:      36: callq
-# CHECK-NEXT: 3b: callq
+# CHECK:      36: callq {{.*}} <bar>
+# CHECK-NEXT: 3b: callq {{.*}} <bar>
 
-# CHECK-LABEL: baz
+# CHECK-LABEL: baz:
+.type   baz,@function
 baz:
   callq foo
   callq foo
@@ -50,10 +53,11 @@ baz:
   callq bar
   .bundle_unlock
   .bundle_unlock
-# CHECK:      56: callq
-# CHECK-NEXT: 5b: callq
+# CHECK:      56: callq {{.*}} <bar>
+# CHECK-NEXT: 5b: callq {{.*}} <bar>
 
 # CHECK-LABEL: quux
+.type   quux,@function
 quux:
   callq bar
   callq bar
@@ -65,5 +69,5 @@ quux:
   .bundle_unlock
 # Check that the calls are bundled together when the second one is after the
 # inner nest is closed.
-# CHECK:      70: callq
-# CHECK-NEXT: 75: callq
+# CHECK:      70: callq {{.*}} <bar>
+# CHECK-NEXT: 75: callq {{.*}} <bar>
diff --git a/test/MC/X86/avx512-encodings.s b/test/MC/X86/avx512-encodings.s
index 9c2b175ffa7..7dd8014213d 100644
--- a/test/MC/X86/avx512-encodings.s
+++ b/test/MC/X86/avx512-encodings.s
@@ -1,5 +1,4 @@
-// RUN: not llvm-mc -triple x86_64-unknown-unknown -mcpu=knl -mattr=+avx512dq --show-encoding %s 2> %t.err | FileCheck %s
-// RUN: FileCheck --check-prefix=ERR < %t.err %s
+// RUN: llvm-mc -triple x86_64-unknown-unknown -mcpu=knl --show-encoding < %s  | FileCheck %s
 
 // CHECK: vaddpd %zmm6, %zmm27, %zmm8
 // CHECK:  encoding: [0x62,0x71,0xa5,0x40,0x58,0xc6]
@@ -4797,6 +4796,38 @@
 // CHECK:  encoding: [0x62,0x61,0x7c,0x58,0x51,0xa2,0xfc,0xfd,0xff,0xff]
           vsqrtps -516(%rdx){1to16}, %zmm28
 
+// CHECK: vsqrtpd {rn-sae}, %zmm19, %zmm19
+// CHECK:  encoding: [0x62,0xa1,0xfd,0x18,0x51,0xdb]
+          vsqrtpd {rn-sae}, %zmm19, %zmm19
+
+// CHECK: vsqrtpd {ru-sae}, %zmm19, %zmm19
+// CHECK:  encoding: [0x62,0xa1,0xfd,0x58,0x51,0xdb]
+          vsqrtpd {ru-sae}, %zmm19, %zmm19
+
+// CHECK: vsqrtpd {rd-sae}, %zmm19, %zmm19
+// CHECK:  encoding: [0x62,0xa1,0xfd,0x38,0x51,0xdb]
+          vsqrtpd {rd-sae}, %zmm19, %zmm19
+
+// CHECK: vsqrtpd {rz-sae}, %zmm19, %zmm19
+// CHECK:  encoding: [0x62,0xa1,0xfd,0x78,0x51,0xdb]
+          vsqrtpd {rz-sae}, %zmm19, %zmm19
+
+// CHECK: vsqrtps {rn-sae}, %zmm29, %zmm28
+// CHECK:  encoding: [0x62,0x01,0x7c,0x18,0x51,0xe5]
+          vsqrtps {rn-sae}, %zmm29, %zmm28
+
+// CHECK: vsqrtps {ru-sae}, %zmm29, %zmm28
+// CHECK:  encoding: [0x62,0x01,0x7c,0x58,0x51,0xe5]
+          vsqrtps {ru-sae}, %zmm29, %zmm28
+
+// CHECK: vsqrtps {rd-sae}, %zmm29, %zmm28
+// CHECK:  encoding: [0x62,0x01,0x7c,0x38,0x51,0xe5]
+          vsqrtps {rd-sae}, %zmm29, %zmm28
+
+// CHECK: vsqrtps {rz-sae}, %zmm29, %zmm28
+// CHECK:  encoding: [0x62,0x01,0x7c,0x78,0x51,0xe5]
+          vsqrtps {rz-sae}, %zmm29, %zmm28
+
 // CHECK: vsubpd %zmm9, %zmm12, %zmm9
 // CHECK:  encoding: [0x62,0x51,0x9d,0x48,0x5c,0xc9]
           vsubpd %zmm9, %zmm12, %zmm9
@@ -5809,14 +5840,6 @@ vinserti32x4  $1, %xmm21, %zmm5, %zmm17
 // CHECK: encoding: [0x62,0xe3,0x1d,0x40,0x38,0x4f,0x10,0x01]
 vinserti32x4  $1, 256(%rdi), %zmm28, %zmm17
 
-// CHECK: vinserti32x8
-// CHECK: encoding: [0x62,0xd3,0x4d,0x40,0x3a,0xdb,0x01]
-vinserti32x8  $1, %ymm11, %zmm22, %zmm3
-
-// CHECK: vinsertf64x2
-// CHECK: encoding: [0x62,0xf3,0xed,0x48,0x18,0x4f,0x10,0x01]
-vinsertf64x2  $1, 256(%rdi), %zmm2, %zmm1
-
 // CHECK: vextracti32x4
 // CHECK: encoding: [0x62,0x33,0x7d,0x48,0x39,0xc9,0x01]
 vextracti32x4  $1, %zmm9, %xmm17
@@ -5849,13 +5872,129 @@ vpsrad 512(%rdi, %rsi, 4), %zmm12, %zmm25
 // CHECK: encoding: [0x62,0xf2,0x7d,0xc9,0x58,0xc8]
 vpbroadcastd  %xmm0, %zmm1 {%k1} {z}
 
-// CHECK: vbroadcasti32x4 {{.*}} {%k7} {z}
-// CHECK: encoding: [0x67,0x62,0xf2,0x7d,0xcf,0x5a,0x52,0x02]
-vbroadcasti32x4 0x20(%edx), %zmm2 {%k7} {z}
+// CHECK: vbroadcastf32x4 (%rcx), %zmm3
+// CHECK:  encoding: [0x62,0xf2,0x7d,0x48,0x1a,0x19]
+          vbroadcastf32x4 (%rcx), %zmm3
+
+// CHECK: vbroadcastf32x4 (%rcx), %zmm3 {%k7}
+// CHECK:  encoding: [0x62,0xf2,0x7d,0x4f,0x1a,0x19]
+          vbroadcastf32x4 (%rcx), %zmm3 {%k7}
+
+// CHECK: vbroadcastf32x4 (%rcx), %zmm3 {%k7} {z}
+// CHECK:  encoding: [0x62,0xf2,0x7d,0xcf,0x1a,0x19]
+          vbroadcastf32x4 (%rcx), %zmm3 {%k7} {z}
+
+// CHECK: vbroadcastf32x4 291(%rax,%r14,8), %zmm3
+// CHECK:  encoding: [0x62,0xb2,0x7d,0x48,0x1a,0x9c,0xf0,0x23,0x01,0x00,0x00]
+          vbroadcastf32x4 291(%rax,%r14,8), %zmm3
+
+// CHECK: vbroadcastf32x4 2032(%rdx), %zmm3
+// CHECK:  encoding: [0x62,0xf2,0x7d,0x48,0x1a,0x5a,0x7f]
+          vbroadcastf32x4 2032(%rdx), %zmm3
+
+// CHECK: vbroadcastf32x4 2048(%rdx), %zmm3
+// CHECK:  encoding: [0x62,0xf2,0x7d,0x48,0x1a,0x9a,0x00,0x08,0x00,0x00]
+          vbroadcastf32x4 2048(%rdx), %zmm3
+
+// CHECK: vbroadcastf32x4 -2048(%rdx), %zmm3
+// CHECK:  encoding: [0x62,0xf2,0x7d,0x48,0x1a,0x5a,0x80]
+          vbroadcastf32x4 -2048(%rdx), %zmm3
+
+// CHECK: vbroadcastf32x4 -2064(%rdx), %zmm3
+// CHECK:  encoding: [0x62,0xf2,0x7d,0x48,0x1a,0x9a,0xf0,0xf7,0xff,0xff]
+          vbroadcastf32x4 -2064(%rdx), %zmm3
+
+// CHECK: vbroadcastf64x4 (%rcx), %zmm25
+// CHECK:  encoding: [0x62,0x62,0xfd,0x48,0x1b,0x09]
+          vbroadcastf64x4 (%rcx), %zmm25
+
+// CHECK: vbroadcastf64x4 (%rcx), %zmm25 {%k6}
+// CHECK:  encoding: [0x62,0x62,0xfd,0x4e,0x1b,0x09]
+          vbroadcastf64x4 (%rcx), %zmm25 {%k6}
+
+// CHECK: vbroadcastf64x4 (%rcx), %zmm25 {%k6} {z}
+// CHECK:  encoding: [0x62,0x62,0xfd,0xce,0x1b,0x09]
+          vbroadcastf64x4 (%rcx), %zmm25 {%k6} {z}
+
+// CHECK: vbroadcastf64x4 291(%rax,%r14,8), %zmm25
+// CHECK:  encoding: [0x62,0x22,0xfd,0x48,0x1b,0x8c,0xf0,0x23,0x01,0x00,0x00]
+          vbroadcastf64x4 291(%rax,%r14,8), %zmm25
+
+// CHECK: vbroadcastf64x4 4064(%rdx), %zmm25
+// CHECK:  encoding: [0x62,0x62,0xfd,0x48,0x1b,0x4a,0x7f]
+          vbroadcastf64x4 4064(%rdx), %zmm25
+
+// CHECK: vbroadcastf64x4 4096(%rdx), %zmm25
+// CHECK:  encoding: [0x62,0x62,0xfd,0x48,0x1b,0x8a,0x00,0x10,0x00,0x00]
+          vbroadcastf64x4 4096(%rdx), %zmm25
+
+// CHECK: vbroadcastf64x4 -4096(%rdx), %zmm25
+// CHECK:  encoding: [0x62,0x62,0xfd,0x48,0x1b,0x4a,0x80]
+          vbroadcastf64x4 -4096(%rdx), %zmm25
+
+// CHECK: vbroadcastf64x4 -4128(%rdx), %zmm25
+// CHECK:  encoding: [0x62,0x62,0xfd,0x48,0x1b,0x8a,0xe0,0xef,0xff,0xff]
+          vbroadcastf64x4 -4128(%rdx), %zmm25
+
+// CHECK: vbroadcasti32x4 (%rcx), %zmm30
+// CHECK:  encoding: [0x62,0x62,0x7d,0x48,0x5a,0x31]
+          vbroadcasti32x4 (%rcx), %zmm30
+
+// CHECK: vbroadcasti32x4 (%rcx), %zmm30 {%k3}
+// CHECK:  encoding: [0x62,0x62,0x7d,0x4b,0x5a,0x31]
+          vbroadcasti32x4 (%rcx), %zmm30 {%k3}
+
+// CHECK: vbroadcasti32x4 (%rcx), %zmm30 {%k3} {z}
+// CHECK:  encoding: [0x62,0x62,0x7d,0xcb,0x5a,0x31]
+          vbroadcasti32x4 (%rcx), %zmm30 {%k3} {z}
+
+// CHECK: vbroadcasti32x4 291(%rax,%r14,8), %zmm30
+// CHECK:  encoding: [0x62,0x22,0x7d,0x48,0x5a,0xb4,0xf0,0x23,0x01,0x00,0x00]
+          vbroadcasti32x4 291(%rax,%r14,8), %zmm30
+
+// CHECK: vbroadcasti32x4 2032(%rdx), %zmm30
+// CHECK:  encoding: [0x62,0x62,0x7d,0x48,0x5a,0x72,0x7f]
+          vbroadcasti32x4 2032(%rdx), %zmm30
+
+// CHECK: vbroadcasti32x4 2048(%rdx), %zmm30
+// CHECK:  encoding: [0x62,0x62,0x7d,0x48,0x5a,0xb2,0x00,0x08,0x00,0x00]
+          vbroadcasti32x4 2048(%rdx), %zmm30
+
+// CHECK: vbroadcasti32x4 -2048(%rdx), %zmm30
+// CHECK:  encoding: [0x62,0x62,0x7d,0x48,0x5a,0x72,0x80]
+          vbroadcasti32x4 -2048(%rdx), %zmm30
 
-// CHECK: vbroadcasti64x4 {{.*}} %zmm22
-// CHECK: encoding: [0x62,0xe2,0xfd,0x48,0x5b,0x72,0x02]
-vbroadcasti64x4 0x40(%rdx), %zmm22
+// CHECK: vbroadcasti32x4 -2064(%rdx), %zmm30
+// CHECK:  encoding: [0x62,0x62,0x7d,0x48,0x5a,0xb2,0xf0,0xf7,0xff,0xff]
+          vbroadcasti32x4 -2064(%rdx), %zmm30
+
+// CHECK: vbroadcasti64x4 (%rcx), %zmm13
+// CHECK:  encoding: [0x62,0x72,0xfd,0x48,0x5b,0x29]
+          vbroadcasti64x4 (%rcx), %zmm13
+
+// CHECK: vbroadcasti64x4 (%rcx), %zmm13 {%k4}
+// CHECK:  encoding: [0x62,0x72,0xfd,0x4c,0x5b,0x29]
+          vbroadcasti64x4 (%rcx), %zmm13 {%k4}
+
+// CHECK: vbroadcasti64x4 (%rcx), %zmm13 {%k4} {z}
+// CHECK:  encoding: [0x62,0x72,0xfd,0xcc,0x5b,0x29]
+          vbroadcasti64x4 (%rcx), %zmm13 {%k4} {z}
+
+// CHECK: vbroadcasti64x4 291(%rax,%r14,8), %zmm13
+// CHECK:  encoding: [0x62,0x32,0xfd,0x48,0x5b,0xac,0xf0,0x23,0x01,0x00,0x00]
+          vbroadcasti64x4 291(%rax,%r14,8), %zmm13
+
+// CHECK: vbroadcasti64x4 4064(%rdx), %zmm13
+// CHECK:  encoding: [0x62,0x72,0xfd,0x48,0x5b,0x6a,0x7f]
+          vbroadcasti64x4 4064(%rdx), %zmm13
+
+// CHECK: vbroadcasti64x4 4096(%rdx), %zmm13
+// CHECK:  encoding: [0x62,0x72,0xfd,0x48,0x5b,0xaa,0x00,0x10,0x00,0x00]
+          vbroadcasti64x4 4096(%rdx), %zmm13
+
+// CHECK: vbroadcasti64x4 -4096(%rdx), %zmm13
+// CHECK:  encoding: [0x62,0x72,0xfd,0x48,0x5b,0x6a,0x80]
+          vbroadcasti64x4 -4096(%rdx), %zmm13
 
 // CHECK: vmovdqu64 {{.*}} {%k3}
 // CHECK: encoding: [0x62,0xf1,0xfe,0x4b,0x7f,0x07]
@@ -5921,25 +6060,6 @@ vpcmpd $1, %zmm24, %zmm7, %k5{%k4}
 // CHECK: encoding: [0x62,0xf3,0xf5,0x47,0x1e,0x72,0x01,0x02]
 vpcmpuq $2, 0x40(%rdx), %zmm17, %k6{%k7}
 
-// ERR: invalid operand for instruction
-vpcmpd $1, %zmm24, %zmm7, %k5{%k0}
-
-// CHECK: vpermi2d
-// CHECK: encoding: [0x62,0x42,0x6d,0x4b,0x76,0xd6]
-vpermi2d %zmm14, %zmm2, %zmm26 {%k3}
-
-// CHECK: vpermt2pd
-// CHECK: encoding: [0x62,0xf2,0xcd,0xc6,0x7f,0xf3]
-vpermt2pd %zmm3, %zmm22, %zmm6 {%k6} {z}
-
-// CHECK: vpermi2q
-// CHECK: encoding: [0x62,0x62,0xed,0x4b,0x76,0x54,0x58,0x02]
-vpermi2q 0x80(%rax,%rbx,2), %zmm2, %zmm26 {%k3}
-
-// CHECK: vpermt2d
-// CHECK: encoding: [0x62,0x32,0x4d,0xc2,0x7e,0x24,0xad,0x05,0x00,0x00,0x00]	
-vpermt2d 5(,%r13,4), %zmm22, %zmm12 {%k2} {z}
-
 // CHECK: valignq $2
 // CHECK: encoding: [0x62,0xf3,0xfd,0x48,0x03,0x4c,0x24,0x04,0x02]
 valignq  $2, 0x100(%rsp), %zmm0, %zmm1
@@ -5948,6 +6068,66 @@ valignq  $2, 0x100(%rsp), %zmm0, %zmm1
 // CHECK: encoding: [0x62,0xf3,0xfd,0x49,0x03,0xcb,0x03]
 valignq  $3, %zmm3, %zmm0, %zmm1 {%k1}
 
+// CHECK: valignq $171, %zmm23, %zmm4, %zmm28
+// CHECK:  encoding: [0x62,0x23,0xdd,0x48,0x03,0xe7,0xab]
+          valignq $0xab, %zmm23, %zmm4, %zmm28
+
+// CHECK: valignq $171, %zmm23, %zmm4, %zmm28 {%k3}
+// CHECK:  encoding: [0x62,0x23,0xdd,0x4b,0x03,0xe7,0xab]
+          valignq $0xab, %zmm23, %zmm4, %zmm28 {%k3}
+
+// CHECK: valignq $171, %zmm23, %zmm4, %zmm28 {%k3} {z}
+// CHECK:  encoding: [0x62,0x23,0xdd,0xcb,0x03,0xe7,0xab]
+          valignq $0xab, %zmm23, %zmm4, %zmm28 {%k3} {z}
+
+// CHECK: valignq $123, %zmm23, %zmm4, %zmm28
+// CHECK:  encoding: [0x62,0x23,0xdd,0x48,0x03,0xe7,0x7b]
+          valignq $0x7b, %zmm23, %zmm4, %zmm28
+
+// CHECK: valignq $123, (%rcx), %zmm4, %zmm28
+// CHECK:  encoding: [0x62,0x63,0xdd,0x48,0x03,0x21,0x7b]
+          valignq $0x7b, (%rcx), %zmm4, %zmm28
+
+// CHECK: valignq $123, 291(%rax,%r14,8), %zmm4, %zmm28
+// CHECK:  encoding: [0x62,0x23,0xdd,0x48,0x03,0xa4,0xf0,0x23,0x01,0x00,0x00,0x7b]
+          valignq $0x7b, 291(%rax,%r14,8), %zmm4, %zmm28
+
+// CHECK: valignq $123, (%rcx){1to8}, %zmm4, %zmm28
+// CHECK:  encoding: [0x62,0x63,0xdd,0x58,0x03,0x21,0x7b]
+          valignq $0x7b, (%rcx){1to8}, %zmm4, %zmm28
+
+// CHECK: valignq $123, 8128(%rdx), %zmm4, %zmm28
+// CHECK:  encoding: [0x62,0x63,0xdd,0x48,0x03,0x62,0x7f,0x7b]
+          valignq $0x7b, 8128(%rdx), %zmm4, %zmm28
+
+// CHECK: valignq $123, 8192(%rdx), %zmm4, %zmm28
+// CHECK:  encoding: [0x62,0x63,0xdd,0x48,0x03,0xa2,0x00,0x20,0x00,0x00,0x7b]
+          valignq $0x7b, 8192(%rdx), %zmm4, %zmm28
+
+// CHECK: valignq $123, -8192(%rdx), %zmm4, %zmm28
+// CHECK:  encoding: [0x62,0x63,0xdd,0x48,0x03,0x62,0x80,0x7b]
+          valignq $0x7b, -8192(%rdx), %zmm4, %zmm28
+
+// CHECK: valignq $123, -8256(%rdx), %zmm4, %zmm28
+// CHECK:  encoding: [0x62,0x63,0xdd,0x48,0x03,0xa2,0xc0,0xdf,0xff,0xff,0x7b]
+          valignq $0x7b, -8256(%rdx), %zmm4, %zmm28
+
+// CHECK: valignq $123, 1016(%rdx){1to8}, %zmm4, %zmm28
+// CHECK:  encoding: [0x62,0x63,0xdd,0x58,0x03,0x62,0x7f,0x7b]
+          valignq $0x7b, 1016(%rdx){1to8}, %zmm4, %zmm28
+
+// CHECK: valignq $123, 1024(%rdx){1to8}, %zmm4, %zmm28
+// CHECK:  encoding: [0x62,0x63,0xdd,0x58,0x03,0xa2,0x00,0x04,0x00,0x00,0x7b]
+          valignq $0x7b, 1024(%rdx){1to8}, %zmm4, %zmm28
+
+// CHECK: valignq $123, -1024(%rdx){1to8}, %zmm4, %zmm28
+// CHECK:  encoding: [0x62,0x63,0xdd,0x58,0x03,0x62,0x80,0x7b]
+          valignq $0x7b, -1024(%rdx){1to8}, %zmm4, %zmm28
+
+// CHECK: valignq $123, -1032(%rdx){1to8}, %zmm4, %zmm28
+// CHECK:  encoding: [0x62,0x63,0xdd,0x58,0x03,0xa2,0xf8,0xfb,0xff,0xff,0x7b]
+          valignq $0x7b, -1032(%rdx){1to8}, %zmm4, %zmm28
+
 // CHECK: vextractf32x4 $3
 // CHECK: encoding: [0x62,0xf3,0x7d,0x49,0x19,0xd9,0x03]
 vextractf32x4  $3, %zmm3, %xmm1 {%k1}
@@ -5995,3 +6175,8786 @@ vpermilps 0x400(%rbx), %zmm2, %zmm3
 // CHECK: vpermilpd
 // CHECK: encoding: [0x62,0xf3,0xfd,0x48,0x05,0x53,0x10,0x23]
 vpermilpd $0x23, 0x400(%rbx), %zmm2
+
+// CHECK: vcmppd $171, %zmm26, %zmm12, %k2
+// CHECK:  encoding: [0x62,0x91,0x9d,0x48,0xc2,0xd2,0xab]
+          vcmppd $0xab, %zmm26, %zmm12, %k2
+
+// CHECK: vcmppd $171, %zmm26, %zmm12, %k2 {%k3}
+// CHECK:  encoding: [0x62,0x91,0x9d,0x4b,0xc2,0xd2,0xab]
+          vcmppd $0xab, %zmm26, %zmm12, %k2 {%k3}
+
+// CHECK: vcmppd $171,{sae}, %zmm26, %zmm12, %k2
+// CHECK:  encoding: [0x62,0x91,0x9d,0x18,0xc2,0xd2,0xab]
+          vcmppd $0xab,{sae}, %zmm26, %zmm12, %k2
+
+// CHECK: vcmppd $123, %zmm26, %zmm12, %k2
+// CHECK:  encoding: [0x62,0x91,0x9d,0x48,0xc2,0xd2,0x7b]
+          vcmppd $0x7b, %zmm26, %zmm12, %k2
+
+// CHECK: vcmppd $123,{sae}, %zmm26, %zmm12, %k2
+// CHECK:  encoding: [0x62,0x91,0x9d,0x18,0xc2,0xd2,0x7b]
+          vcmppd $0x7b,{sae}, %zmm26, %zmm12, %k2
+
+// CHECK: vcmppd $123, (%rcx), %zmm12, %k2
+// CHECK:  encoding: [0x62,0xf1,0x9d,0x48,0xc2,0x11,0x7b]
+          vcmppd $0x7b, (%rcx), %zmm12, %k2
+
+// CHECK: vcmppd $123, 291(%rax,%r14,8), %zmm12, %k2
+// CHECK:  encoding: [0x62,0xb1,0x9d,0x48,0xc2,0x94,0xf0,0x23,0x01,0x00,0x00,0x7b]
+          vcmppd $0x7b, 291(%rax,%r14,8), %zmm12, %k2
+
+// CHECK: vcmppd $123, (%rcx){1to8}, %zmm12, %k2
+// CHECK:  encoding: [0x62,0xf1,0x9d,0x58,0xc2,0x11,0x7b]
+          vcmppd $0x7b, (%rcx){1to8}, %zmm12, %k2
+
+// CHECK: vcmppd $123, 8128(%rdx), %zmm12, %k2
+// CHECK:  encoding: [0x62,0xf1,0x9d,0x48,0xc2,0x52,0x7f,0x7b]
+          vcmppd $0x7b, 8128(%rdx), %zmm12, %k2
+
+// CHECK: vcmppd $123, 8192(%rdx), %zmm12, %k2
+// CHECK:  encoding: [0x62,0xf1,0x9d,0x48,0xc2,0x92,0x00,0x20,0x00,0x00,0x7b]
+          vcmppd $0x7b, 8192(%rdx), %zmm12, %k2
+
+// CHECK: vcmppd $123, -8192(%rdx), %zmm12, %k2
+// CHECK:  encoding: [0x62,0xf1,0x9d,0x48,0xc2,0x52,0x80,0x7b]
+          vcmppd $0x7b, -8192(%rdx), %zmm12, %k2
+
+// CHECK: vcmppd $123, -8256(%rdx), %zmm12, %k2
+// CHECK:  encoding: [0x62,0xf1,0x9d,0x48,0xc2,0x92,0xc0,0xdf,0xff,0xff,0x7b]
+          vcmppd $0x7b, -8256(%rdx), %zmm12, %k2
+
+// CHECK: vcmppd $123, 1016(%rdx){1to8}, %zmm12, %k2
+// CHECK:  encoding: [0x62,0xf1,0x9d,0x58,0xc2,0x52,0x7f,0x7b]
+          vcmppd $0x7b, 1016(%rdx){1to8}, %zmm12, %k2
+
+// CHECK: vcmppd $123, 1024(%rdx){1to8}, %zmm12, %k2
+// CHECK:  encoding: [0x62,0xf1,0x9d,0x58,0xc2,0x92,0x00,0x04,0x00,0x00,0x7b]
+          vcmppd $0x7b, 1024(%rdx){1to8}, %zmm12, %k2
+
+// CHECK: vcmppd $123, -1024(%rdx){1to8}, %zmm12, %k2
+// CHECK:  encoding: [0x62,0xf1,0x9d,0x58,0xc2,0x52,0x80,0x7b]
+          vcmppd $0x7b, -1024(%rdx){1to8}, %zmm12, %k2
+
+// CHECK: vcmppd $123, -1032(%rdx){1to8}, %zmm12, %k2
+// CHECK:  encoding: [0x62,0xf1,0x9d,0x58,0xc2,0x92,0xf8,0xfb,0xff,0xff,0x7b]
+          vcmppd $0x7b, -1032(%rdx){1to8}, %zmm12, %k2
+
+// CHECK: vcmpps $171, %zmm22, %zmm17, %k2
+// CHECK:  encoding: [0x62,0xb1,0x74,0x40,0xc2,0xd6,0xab]
+          vcmpps $0xab, %zmm22, %zmm17, %k2
+
+// CHECK: vcmpps $171, %zmm22, %zmm17, %k2 {%k3}
+// CHECK:  encoding: [0x62,0xb1,0x74,0x43,0xc2,0xd6,0xab]
+          vcmpps $0xab, %zmm22, %zmm17, %k2 {%k3}
+
+// CHECK: vcmpps $171,{sae}, %zmm22, %zmm17, %k2
+// CHECK:  encoding: [0x62,0xb1,0x74,0x10,0xc2,0xd6,0xab]
+          vcmpps $0xab,{sae}, %zmm22, %zmm17, %k2
+
+// CHECK: vcmpps $123, %zmm22, %zmm17, %k2
+// CHECK:  encoding: [0x62,0xb1,0x74,0x40,0xc2,0xd6,0x7b]
+          vcmpps $0x7b, %zmm22, %zmm17, %k2
+
+// CHECK: vcmpps $123,{sae}, %zmm22, %zmm17, %k2
+// CHECK:  encoding: [0x62,0xb1,0x74,0x10,0xc2,0xd6,0x7b]
+          vcmpps $0x7b,{sae}, %zmm22, %zmm17, %k2
+
+// CHECK: vcmpps $123, (%rcx), %zmm17, %k2
+// CHECK:  encoding: [0x62,0xf1,0x74,0x40,0xc2,0x11,0x7b]
+          vcmpps $0x7b, (%rcx), %zmm17, %k2
+
+// CHECK: vcmpps $123, 291(%rax,%r14,8), %zmm17, %k2
+// CHECK:  encoding: [0x62,0xb1,0x74,0x40,0xc2,0x94,0xf0,0x23,0x01,0x00,0x00,0x7b]
+          vcmpps $0x7b, 291(%rax,%r14,8), %zmm17, %k2
+
+// CHECK: vcmpps $123, (%rcx){1to16}, %zmm17, %k2
+// CHECK:  encoding: [0x62,0xf1,0x74,0x50,0xc2,0x11,0x7b]
+          vcmpps $0x7b, (%rcx){1to16}, %zmm17, %k2
+
+// CHECK: vcmpps $123, 8128(%rdx), %zmm17, %k2
+// CHECK:  encoding: [0x62,0xf1,0x74,0x40,0xc2,0x52,0x7f,0x7b]
+          vcmpps $0x7b, 8128(%rdx), %zmm17, %k2
+
+// CHECK: vcmpps $123, 8192(%rdx), %zmm17, %k2
+// CHECK:  encoding: [0x62,0xf1,0x74,0x40,0xc2,0x92,0x00,0x20,0x00,0x00,0x7b]
+          vcmpps $0x7b, 8192(%rdx), %zmm17, %k2
+
+// CHECK: vcmpps $123, -8192(%rdx), %zmm17, %k2
+// CHECK:  encoding: [0x62,0xf1,0x74,0x40,0xc2,0x52,0x80,0x7b]
+          vcmpps $0x7b, -8192(%rdx), %zmm17, %k2
+
+// CHECK: vcmpps $123, -8256(%rdx), %zmm17, %k2
+// CHECK:  encoding: [0x62,0xf1,0x74,0x40,0xc2,0x92,0xc0,0xdf,0xff,0xff,0x7b]
+          vcmpps $0x7b, -8256(%rdx), %zmm17, %k2
+
+// CHECK: vcmpps $123, 508(%rdx){1to16}, %zmm17, %k2
+// CHECK:  encoding: [0x62,0xf1,0x74,0x50,0xc2,0x52,0x7f,0x7b]
+          vcmpps $0x7b, 508(%rdx){1to16}, %zmm17, %k2
+
+// CHECK: vcmpps $123, 512(%rdx){1to16}, %zmm17, %k2
+// CHECK:  encoding: [0x62,0xf1,0x74,0x50,0xc2,0x92,0x00,0x02,0x00,0x00,0x7b]
+          vcmpps $0x7b, 512(%rdx){1to16}, %zmm17, %k2
+
+// CHECK: vcmpps $123, -512(%rdx){1to16}, %zmm17, %k2
+// CHECK:  encoding: [0x62,0xf1,0x74,0x50,0xc2,0x52,0x80,0x7b]
+          vcmpps $0x7b, -512(%rdx){1to16}, %zmm17, %k2
+
+// CHECK: vcmpps $123, -516(%rdx){1to16}, %zmm17, %k2
+// CHECK:  encoding: [0x62,0xf1,0x74,0x50,0xc2,0x92,0xfc,0xfd,0xff,0xff,0x7b]
+          vcmpps $0x7b, -516(%rdx){1to16}, %zmm17, %k2
+
+// CHECK: vaddpd {rn-sae}, %zmm6, %zmm27, %zmm8
+// CHECK:  encoding: [0x62,0x71,0xa5,0x10,0x58,0xc6]
+          vaddpd {rn-sae}, %zmm6, %zmm27, %zmm8
+
+// CHECK: vaddpd {ru-sae}, %zmm6, %zmm27, %zmm8
+// CHECK:  encoding: [0x62,0x71,0xa5,0x50,0x58,0xc6]
+          vaddpd {ru-sae}, %zmm6, %zmm27, %zmm8
+
+// CHECK: vaddpd {rd-sae}, %zmm6, %zmm27, %zmm8
+// CHECK:  encoding: [0x62,0x71,0xa5,0x30,0x58,0xc6]
+          vaddpd {rd-sae}, %zmm6, %zmm27, %zmm8
+
+// CHECK: vaddpd {rz-sae}, %zmm6, %zmm27, %zmm8
+// CHECK:  encoding: [0x62,0x71,0xa5,0x70,0x58,0xc6]
+          vaddpd {rz-sae}, %zmm6, %zmm27, %zmm8
+
+// CHECK: vaddps {rn-sae}, %zmm2, %zmm13, %zmm18
+// CHECK:  encoding: [0x62,0xe1,0x14,0x18,0x58,0xd2]
+          vaddps {rn-sae}, %zmm2, %zmm13, %zmm18
+
+// CHECK: vaddps {ru-sae}, %zmm2, %zmm13, %zmm18
+// CHECK:  encoding: [0x62,0xe1,0x14,0x58,0x58,0xd2]
+          vaddps {ru-sae}, %zmm2, %zmm13, %zmm18
+
+// CHECK: vaddps {rd-sae}, %zmm2, %zmm13, %zmm18
+// CHECK:  encoding: [0x62,0xe1,0x14,0x38,0x58,0xd2]
+          vaddps {rd-sae}, %zmm2, %zmm13, %zmm18
+
+// CHECK: vaddps {rz-sae}, %zmm2, %zmm13, %zmm18
+// CHECK:  encoding: [0x62,0xe1,0x14,0x78,0x58,0xd2]
+          vaddps {rz-sae}, %zmm2, %zmm13, %zmm18
+
+// CHECK: vaddsd %xmm8, %xmm17, %xmm3
+// CHECK:  encoding: [0x62,0xd1,0xf7,0x00,0x58,0xd8]
+          vaddsd %xmm8, %xmm17, %xmm3
+
+// CHECK: vaddsd %xmm8, %xmm17, %xmm3 {%k3}
+// CHECK:  encoding: [0x62,0xd1,0xf7,0x03,0x58,0xd8]
+          vaddsd %xmm8, %xmm17, %xmm3 {%k3}
+
+// CHECK: vaddsd %xmm8, %xmm17, %xmm3 {%k3} {z}
+// CHECK:  encoding: [0x62,0xd1,0xf7,0x83,0x58,0xd8]
+          vaddsd %xmm8, %xmm17, %xmm3 {%k3} {z}
+
+// CHECK: vaddsd {rn-sae}, %xmm8, %xmm17, %xmm3
+// CHECK:  encoding: [0x62,0xd1,0xf7,0x10,0x58,0xd8]
+          vaddsd {rn-sae}, %xmm8, %xmm17, %xmm3
+
+// CHECK: vaddsd {ru-sae}, %xmm8, %xmm17, %xmm3
+// CHECK:  encoding: [0x62,0xd1,0xf7,0x50,0x58,0xd8]
+          vaddsd {ru-sae}, %xmm8, %xmm17, %xmm3
+
+// CHECK: vaddsd {rd-sae}, %xmm8, %xmm17, %xmm3
+// CHECK:  encoding: [0x62,0xd1,0xf7,0x30,0x58,0xd8]
+          vaddsd {rd-sae}, %xmm8, %xmm17, %xmm3
+
+// CHECK: vaddsd {rz-sae}, %xmm8, %xmm17, %xmm3
+// CHECK:  encoding: [0x62,0xd1,0xf7,0x70,0x58,0xd8]
+          vaddsd {rz-sae}, %xmm8, %xmm17, %xmm3
+
+// CHECK: vaddsd (%rcx), %xmm17, %xmm3
+// CHECK:  encoding: [0x62,0xf1,0xf7,0x00,0x58,0x19]
+          vaddsd (%rcx), %xmm17, %xmm3
+
+// CHECK: vaddsd 291(%rax,%r14,8), %xmm17, %xmm3
+// CHECK:  encoding: [0x62,0xb1,0xf7,0x00,0x58,0x9c,0xf0,0x23,0x01,0x00,0x00]
+          vaddsd 291(%rax,%r14,8), %xmm17, %xmm3
+
+// CHECK: vaddsd 1016(%rdx), %xmm17, %xmm3
+// CHECK:  encoding: [0x62,0xf1,0xf7,0x00,0x58,0x5a,0x7f]
+          vaddsd 1016(%rdx), %xmm17, %xmm3
+
+// CHECK: vaddsd 1024(%rdx), %xmm17, %xmm3
+// CHECK:  encoding: [0x62,0xf1,0xf7,0x00,0x58,0x9a,0x00,0x04,0x00,0x00]
+          vaddsd 1024(%rdx), %xmm17, %xmm3
+
+// CHECK: vaddsd -1024(%rdx), %xmm17, %xmm3
+// CHECK:  encoding: [0x62,0xf1,0xf7,0x00,0x58,0x5a,0x80]
+          vaddsd -1024(%rdx), %xmm17, %xmm3
+
+// CHECK: vaddsd -1032(%rdx), %xmm17, %xmm3
+// CHECK:  encoding: [0x62,0xf1,0xf7,0x00,0x58,0x9a,0xf8,0xfb,0xff,0xff]
+          vaddsd -1032(%rdx), %xmm17, %xmm3
+
+// CHECK: vaddss %xmm19, %xmm5, %xmm7
+// CHECK:  encoding: [0x62,0xb1,0x56,0x08,0x58,0xfb]
+          vaddss %xmm19, %xmm5, %xmm7
+
+// CHECK: vaddss %xmm19, %xmm5, %xmm7 {%k2}
+// CHECK:  encoding: [0x62,0xb1,0x56,0x0a,0x58,0xfb]
+          vaddss %xmm19, %xmm5, %xmm7 {%k2}
+
+// CHECK: vaddss %xmm19, %xmm5, %xmm7 {%k2} {z}
+// CHECK:  encoding: [0x62,0xb1,0x56,0x8a,0x58,0xfb]
+          vaddss %xmm19, %xmm5, %xmm7 {%k2} {z}
+
+// CHECK: vaddss {rn-sae}, %xmm19, %xmm5, %xmm7
+// CHECK:  encoding: [0x62,0xb1,0x56,0x18,0x58,0xfb]
+          vaddss {rn-sae}, %xmm19, %xmm5, %xmm7
+
+// CHECK: vaddss {ru-sae}, %xmm19, %xmm5, %xmm7
+// CHECK:  encoding: [0x62,0xb1,0x56,0x58,0x58,0xfb]
+          vaddss {ru-sae}, %xmm19, %xmm5, %xmm7
+
+// CHECK: vaddss {rd-sae}, %xmm19, %xmm5, %xmm7
+// CHECK:  encoding: [0x62,0xb1,0x56,0x38,0x58,0xfb]
+          vaddss {rd-sae}, %xmm19, %xmm5, %xmm7
+
+// CHECK: vaddss {rz-sae}, %xmm19, %xmm5, %xmm7
+// CHECK:  encoding: [0x62,0xb1,0x56,0x78,0x58,0xfb]
+          vaddss {rz-sae}, %xmm19, %xmm5, %xmm7
+
+// CHECK: vaddss (%rcx), %xmm5, %xmm7
+// CHECK:  encoding: [0x62,0xf1,0x56,0x08,0x58,0x39]
+          vaddss (%rcx), %xmm5, %xmm7
+
+// CHECK: vaddss 291(%rax,%r14,8), %xmm5, %xmm7
+// CHECK:  encoding: [0x62,0xb1,0x56,0x08,0x58,0xbc,0xf0,0x23,0x01,0x00,0x00]
+          vaddss 291(%rax,%r14,8), %xmm5, %xmm7
+
+// CHECK: vaddss 508(%rdx), %xmm5, %xmm7
+// CHECK:  encoding: [0x62,0xf1,0x56,0x08,0x58,0x7a,0x7f]
+          vaddss 508(%rdx), %xmm5, %xmm7
+
+// CHECK: vaddss 512(%rdx), %xmm5, %xmm7
+// CHECK:  encoding: [0x62,0xf1,0x56,0x08,0x58,0xba,0x00,0x02,0x00,0x00]
+          vaddss 512(%rdx), %xmm5, %xmm7
+
+// CHECK: vaddss -512(%rdx), %xmm5, %xmm7
+// CHECK:  encoding: [0x62,0xf1,0x56,0x08,0x58,0x7a,0x80]
+          vaddss -512(%rdx), %xmm5, %xmm7
+
+// CHECK: vaddss -516(%rdx), %xmm5, %xmm7
+// CHECK:  encoding: [0x62,0xf1,0x56,0x08,0x58,0xba,0xfc,0xfd,0xff,0xff]
+          vaddss -516(%rdx), %xmm5, %xmm7
+
+// CHECK: vdivpd {rn-sae}, %zmm11, %zmm6, %zmm18
+// CHECK:  encoding: [0x62,0xc1,0xcd,0x18,0x5e,0xd3]
+          vdivpd {rn-sae}, %zmm11, %zmm6, %zmm18
+
+// CHECK: vdivpd {ru-sae}, %zmm11, %zmm6, %zmm18
+// CHECK:  encoding: [0x62,0xc1,0xcd,0x58,0x5e,0xd3]
+          vdivpd {ru-sae}, %zmm11, %zmm6, %zmm18
+
+// CHECK: vdivpd {rd-sae}, %zmm11, %zmm6, %zmm18
+// CHECK:  encoding: [0x62,0xc1,0xcd,0x38,0x5e,0xd3]
+          vdivpd {rd-sae}, %zmm11, %zmm6, %zmm18
+
+// CHECK: vdivpd {rz-sae}, %zmm11, %zmm6, %zmm18
+// CHECK:  encoding: [0x62,0xc1,0xcd,0x78,0x5e,0xd3]
+          vdivpd {rz-sae}, %zmm11, %zmm6, %zmm18
+
+// CHECK: vdivps {rn-sae}, %zmm28, %zmm23, %zmm23
+// CHECK:  encoding: [0x62,0x81,0x44,0x10,0x5e,0xfc]
+          vdivps {rn-sae}, %zmm28, %zmm23, %zmm23
+
+// CHECK: vdivps {ru-sae}, %zmm28, %zmm23, %zmm23
+// CHECK:  encoding: [0x62,0x81,0x44,0x50,0x5e,0xfc]
+          vdivps {ru-sae}, %zmm28, %zmm23, %zmm23
+
+// CHECK: vdivps {rd-sae}, %zmm28, %zmm23, %zmm23
+// CHECK:  encoding: [0x62,0x81,0x44,0x30,0x5e,0xfc]
+          vdivps {rd-sae}, %zmm28, %zmm23, %zmm23
+
+// CHECK: vdivps {rz-sae}, %zmm28, %zmm23, %zmm23
+// CHECK:  encoding: [0x62,0x81,0x44,0x70,0x5e,0xfc]
+          vdivps {rz-sae}, %zmm28, %zmm23, %zmm23
+
+// CHECK: vdivsd %xmm22, %xmm13, %xmm29
+// CHECK:  encoding: [0x62,0x21,0x97,0x08,0x5e,0xee]
+          vdivsd %xmm22, %xmm13, %xmm29
+
+// CHECK: vdivsd %xmm22, %xmm13, %xmm29 {%k3}
+// CHECK:  encoding: [0x62,0x21,0x97,0x0b,0x5e,0xee]
+          vdivsd %xmm22, %xmm13, %xmm29 {%k3}
+
+// CHECK: vdivsd %xmm22, %xmm13, %xmm29 {%k3} {z}
+// CHECK:  encoding: [0x62,0x21,0x97,0x8b,0x5e,0xee]
+          vdivsd %xmm22, %xmm13, %xmm29 {%k3} {z}
+
+// CHECK: vdivsd {rn-sae}, %xmm22, %xmm13, %xmm29
+// CHECK:  encoding: [0x62,0x21,0x97,0x18,0x5e,0xee]
+          vdivsd {rn-sae}, %xmm22, %xmm13, %xmm29
+
+// CHECK: vdivsd {ru-sae}, %xmm22, %xmm13, %xmm29
+// CHECK:  encoding: [0x62,0x21,0x97,0x58,0x5e,0xee]
+          vdivsd {ru-sae}, %xmm22, %xmm13, %xmm29
+
+// CHECK: vdivsd {rd-sae}, %xmm22, %xmm13, %xmm29
+// CHECK:  encoding: [0x62,0x21,0x97,0x38,0x5e,0xee]
+          vdivsd {rd-sae}, %xmm22, %xmm13, %xmm29
+
+// CHECK: vdivsd {rz-sae}, %xmm22, %xmm13, %xmm29
+// CHECK:  encoding: [0x62,0x21,0x97,0x78,0x5e,0xee]
+          vdivsd {rz-sae}, %xmm22, %xmm13, %xmm29
+
+// CHECK: vdivsd (%rcx), %xmm13, %xmm29
+// CHECK:  encoding: [0x62,0x61,0x97,0x08,0x5e,0x29]
+          vdivsd (%rcx), %xmm13, %xmm29
+
+// CHECK: vdivsd 291(%rax,%r14,8), %xmm13, %xmm29
+// CHECK:  encoding: [0x62,0x21,0x97,0x08,0x5e,0xac,0xf0,0x23,0x01,0x00,0x00]
+          vdivsd 291(%rax,%r14,8), %xmm13, %xmm29
+
+// CHECK: vdivsd 1016(%rdx), %xmm13, %xmm29
+// CHECK:  encoding: [0x62,0x61,0x97,0x08,0x5e,0x6a,0x7f]
+          vdivsd 1016(%rdx), %xmm13, %xmm29
+
+// CHECK: vdivsd 1024(%rdx), %xmm13, %xmm29
+// CHECK:  encoding: [0x62,0x61,0x97,0x08,0x5e,0xaa,0x00,0x04,0x00,0x00]
+          vdivsd 1024(%rdx), %xmm13, %xmm29
+
+// CHECK: vdivsd -1024(%rdx), %xmm13, %xmm29
+// CHECK:  encoding: [0x62,0x61,0x97,0x08,0x5e,0x6a,0x80]
+          vdivsd -1024(%rdx), %xmm13, %xmm29
+
+// CHECK: vdivsd -1032(%rdx), %xmm13, %xmm29
+// CHECK:  encoding: [0x62,0x61,0x97,0x08,0x5e,0xaa,0xf8,0xfb,0xff,0xff]
+          vdivsd -1032(%rdx), %xmm13, %xmm29
+
+// CHECK: vdivss %xmm17, %xmm6, %xmm21
+// CHECK:  encoding: [0x62,0xa1,0x4e,0x08,0x5e,0xe9]
+          vdivss %xmm17, %xmm6, %xmm21
+
+// CHECK: vdivss %xmm17, %xmm6, %xmm21 {%k5}
+// CHECK:  encoding: [0x62,0xa1,0x4e,0x0d,0x5e,0xe9]
+          vdivss %xmm17, %xmm6, %xmm21 {%k5}
+
+// CHECK: vdivss %xmm17, %xmm6, %xmm21 {%k5} {z}
+// CHECK:  encoding: [0x62,0xa1,0x4e,0x8d,0x5e,0xe9]
+          vdivss %xmm17, %xmm6, %xmm21 {%k5} {z}
+
+// CHECK: vdivss {rn-sae}, %xmm17, %xmm6, %xmm21
+// CHECK:  encoding: [0x62,0xa1,0x4e,0x18,0x5e,0xe9]
+          vdivss {rn-sae}, %xmm17, %xmm6, %xmm21
+
+// CHECK: vdivss {ru-sae}, %xmm17, %xmm6, %xmm21
+// CHECK:  encoding: [0x62,0xa1,0x4e,0x58,0x5e,0xe9]
+          vdivss {ru-sae}, %xmm17, %xmm6, %xmm21
+
+// CHECK: vdivss {rd-sae}, %xmm17, %xmm6, %xmm21
+// CHECK:  encoding: [0x62,0xa1,0x4e,0x38,0x5e,0xe9]
+          vdivss {rd-sae}, %xmm17, %xmm6, %xmm21
+
+// CHECK: vdivss {rz-sae}, %xmm17, %xmm6, %xmm21
+// CHECK:  encoding: [0x62,0xa1,0x4e,0x78,0x5e,0xe9]
+          vdivss {rz-sae}, %xmm17, %xmm6, %xmm21
+
+// CHECK: vdivss (%rcx), %xmm6, %xmm21
+// CHECK:  encoding: [0x62,0xe1,0x4e,0x08,0x5e,0x29]
+          vdivss (%rcx), %xmm6, %xmm21
+
+// CHECK: vdivss 291(%rax,%r14,8), %xmm6, %xmm21
+// CHECK:  encoding: [0x62,0xa1,0x4e,0x08,0x5e,0xac,0xf0,0x23,0x01,0x00,0x00]
+          vdivss 291(%rax,%r14,8), %xmm6, %xmm21
+
+// CHECK: vdivss 508(%rdx), %xmm6, %xmm21
+// CHECK:  encoding: [0x62,0xe1,0x4e,0x08,0x5e,0x6a,0x7f]
+          vdivss 508(%rdx), %xmm6, %xmm21
+
+// CHECK: vdivss 512(%rdx), %xmm6, %xmm21
+// CHECK:  encoding: [0x62,0xe1,0x4e,0x08,0x5e,0xaa,0x00,0x02,0x00,0x00]
+          vdivss 512(%rdx), %xmm6, %xmm21
+
+// CHECK: vdivss -512(%rdx), %xmm6, %xmm21
+// CHECK:  encoding: [0x62,0xe1,0x4e,0x08,0x5e,0x6a,0x80]
+          vdivss -512(%rdx), %xmm6, %xmm21
+
+// CHECK: vdivss -516(%rdx), %xmm6, %xmm21
+// CHECK:  encoding: [0x62,0xe1,0x4e,0x08,0x5e,0xaa,0xfc,0xfd,0xff,0xff]
+          vdivss -516(%rdx), %xmm6, %xmm21
+
+// CHECK: vmaxpd {sae}, %zmm20, %zmm28, %zmm30
+// CHECK:  encoding: [0x62,0x21,0x9d,0x10,0x5f,0xf4]
+          vmaxpd {sae}, %zmm20, %zmm28, %zmm30
+
+// CHECK: vmaxps {sae}, %zmm20, %zmm6, %zmm25
+// CHECK:  encoding: [0x62,0x21,0x4c,0x18,0x5f,0xcc]
+          vmaxps {sae}, %zmm20, %zmm6, %zmm25
+
+// CHECK: vmaxsd %xmm25, %xmm19, %xmm20
+// CHECK:  encoding: [0x62,0x81,0xe7,0x00,0x5f,0xe1]
+          vmaxsd %xmm25, %xmm19, %xmm20
+
+// CHECK: vmaxsd %xmm25, %xmm19, %xmm20 {%k3}
+// CHECK:  encoding: [0x62,0x81,0xe7,0x03,0x5f,0xe1]
+          vmaxsd %xmm25, %xmm19, %xmm20 {%k3}
+
+// CHECK: vmaxsd %xmm25, %xmm19, %xmm20 {%k3} {z}
+// CHECK:  encoding: [0x62,0x81,0xe7,0x83,0x5f,0xe1]
+          vmaxsd %xmm25, %xmm19, %xmm20 {%k3} {z}
+
+// CHECK: vmaxsd {sae}, %xmm25, %xmm19, %xmm20
+// CHECK:  encoding: [0x62,0x81,0xe7,0x10,0x5f,0xe1]
+          vmaxsd {sae}, %xmm25, %xmm19, %xmm20
+
+// CHECK: vmaxsd (%rcx), %xmm19, %xmm20
+// CHECK:  encoding: [0x62,0xe1,0xe7,0x00,0x5f,0x21]
+          vmaxsd (%rcx), %xmm19, %xmm20
+
+// CHECK: vmaxsd 291(%rax,%r14,8), %xmm19, %xmm20
+// CHECK:  encoding: [0x62,0xa1,0xe7,0x00,0x5f,0xa4,0xf0,0x23,0x01,0x00,0x00]
+          vmaxsd 291(%rax,%r14,8), %xmm19, %xmm20
+
+// CHECK: vmaxsd 1016(%rdx), %xmm19, %xmm20
+// CHECK:  encoding: [0x62,0xe1,0xe7,0x00,0x5f,0x62,0x7f]
+          vmaxsd 1016(%rdx), %xmm19, %xmm20
+
+// CHECK: vmaxsd 1024(%rdx), %xmm19, %xmm20
+// CHECK:  encoding: [0x62,0xe1,0xe7,0x00,0x5f,0xa2,0x00,0x04,0x00,0x00]
+          vmaxsd 1024(%rdx), %xmm19, %xmm20
+
+// CHECK: vmaxsd -1024(%rdx), %xmm19, %xmm20
+// CHECK:  encoding: [0x62,0xe1,0xe7,0x00,0x5f,0x62,0x80]
+          vmaxsd -1024(%rdx), %xmm19, %xmm20
+
+// CHECK: vmaxsd -1032(%rdx), %xmm19, %xmm20
+// CHECK:  encoding: [0x62,0xe1,0xe7,0x00,0x5f,0xa2,0xf8,0xfb,0xff,0xff]
+          vmaxsd -1032(%rdx), %xmm19, %xmm20
+
+// CHECK: vmaxss %xmm6, %xmm4, %xmm8
+// CHECK:  encoding: [0xc5,0x5a,0x5f,0xc6]
+          vmaxss %xmm6, %xmm4, %xmm8
+
+// CHECK: vmaxss %xmm6, %xmm4, %xmm8 {%k4}
+// CHECK:  encoding: [0x62,0x71,0x5e,0x0c,0x5f,0xc6]
+          vmaxss %xmm6, %xmm4, %xmm8 {%k4}
+
+// CHECK: vmaxss %xmm6, %xmm4, %xmm8 {%k4} {z}
+// CHECK:  encoding: [0x62,0x71,0x5e,0x8c,0x5f,0xc6]
+          vmaxss %xmm6, %xmm4, %xmm8 {%k4} {z}
+
+// CHECK: vmaxss {sae}, %xmm6, %xmm4, %xmm8
+// CHECK:  encoding: [0x62,0x71,0x5e,0x18,0x5f,0xc6]
+          vmaxss {sae}, %xmm6, %xmm4, %xmm8
+
+// CHECK: vmaxss (%rcx), %xmm4, %xmm8
+// CHECK:  encoding: [0x62,0x71,0x5e,0x08,0x5f,0x01]
+          vmaxss (%rcx), %xmm4, %xmm8
+
+// CHECK: vmaxss 291(%rax,%r14,8), %xmm4, %xmm8
+// CHECK:  encoding: [0x62,0x31,0x5e,0x08,0x5f,0x84,0xf0,0x23,0x01,0x00,0x00]
+          vmaxss 291(%rax,%r14,8), %xmm4, %xmm8
+
+// CHECK: vmaxss 508(%rdx), %xmm4, %xmm8
+// CHECK:  encoding: [0x62,0x71,0x5e,0x08,0x5f,0x42,0x7f]
+          vmaxss 508(%rdx), %xmm4, %xmm8
+
+// CHECK: vmaxss 512(%rdx), %xmm4, %xmm8
+// CHECK:  encoding: [0x62,0x71,0x5e,0x08,0x5f,0x82,0x00,0x02,0x00,0x00]
+          vmaxss 512(%rdx), %xmm4, %xmm8
+
+// CHECK: vmaxss -512(%rdx), %xmm4, %xmm8
+// CHECK:  encoding: [0x62,0x71,0x5e,0x08,0x5f,0x42,0x80]
+          vmaxss -512(%rdx), %xmm4, %xmm8
+
+// CHECK: vmaxss -516(%rdx), %xmm4, %xmm8
+// CHECK:  encoding: [0x62,0x71,0x5e,0x08,0x5f,0x82,0xfc,0xfd,0xff,0xff]
+          vmaxss -516(%rdx), %xmm4, %xmm8
+
+// CHECK: vminpd {sae}, %zmm22, %zmm6, %zmm6
+// CHECK:  encoding: [0x62,0xb1,0xcd,0x18,0x5d,0xf6]
+          vminpd {sae}, %zmm22, %zmm6, %zmm6
+
+// CHECK: vminps {sae}, %zmm7, %zmm3, %zmm3
+// CHECK:  encoding: [0x62,0xf1,0x64,0x18,0x5d,0xdf]
+          vminps {sae}, %zmm7, %zmm3, %zmm3
+
+// CHECK: vminsd %xmm26, %xmm25, %xmm5
+// CHECK:  encoding: [0x62,0x91,0xb7,0x00,0x5d,0xea]
+          vminsd %xmm26, %xmm25, %xmm5
+
+// CHECK: vminsd %xmm26, %xmm25, %xmm5 {%k3}
+// CHECK:  encoding: [0x62,0x91,0xb7,0x03,0x5d,0xea]
+          vminsd %xmm26, %xmm25, %xmm5 {%k3}
+
+// CHECK: vminsd %xmm26, %xmm25, %xmm5 {%k3} {z}
+// CHECK:  encoding: [0x62,0x91,0xb7,0x83,0x5d,0xea]
+          vminsd %xmm26, %xmm25, %xmm5 {%k3} {z}
+
+// CHECK: vminsd {sae}, %xmm26, %xmm25, %xmm5
+// CHECK:  encoding: [0x62,0x91,0xb7,0x10,0x5d,0xea]
+          vminsd {sae}, %xmm26, %xmm25, %xmm5
+
+// CHECK: vminsd (%rcx), %xmm25, %xmm5
+// CHECK:  encoding: [0x62,0xf1,0xb7,0x00,0x5d,0x29]
+          vminsd (%rcx), %xmm25, %xmm5
+
+// CHECK: vminsd 291(%rax,%r14,8), %xmm25, %xmm5
+// CHECK:  encoding: [0x62,0xb1,0xb7,0x00,0x5d,0xac,0xf0,0x23,0x01,0x00,0x00]
+          vminsd 291(%rax,%r14,8), %xmm25, %xmm5
+
+// CHECK: vminsd 1016(%rdx), %xmm25, %xmm5
+// CHECK:  encoding: [0x62,0xf1,0xb7,0x00,0x5d,0x6a,0x7f]
+          vminsd 1016(%rdx), %xmm25, %xmm5
+
+// CHECK: vminsd 1024(%rdx), %xmm25, %xmm5
+// CHECK:  encoding: [0x62,0xf1,0xb7,0x00,0x5d,0xaa,0x00,0x04,0x00,0x00]
+          vminsd 1024(%rdx), %xmm25, %xmm5
+
+// CHECK: vminsd -1024(%rdx), %xmm25, %xmm5
+// CHECK:  encoding: [0x62,0xf1,0xb7,0x00,0x5d,0x6a,0x80]
+          vminsd -1024(%rdx), %xmm25, %xmm5
+
+// CHECK: vminsd -1032(%rdx), %xmm25, %xmm5
+// CHECK:  encoding: [0x62,0xf1,0xb7,0x00,0x5d,0xaa,0xf8,0xfb,0xff,0xff]
+          vminsd -1032(%rdx), %xmm25, %xmm5
+
+// CHECK: vminss %xmm19, %xmm17, %xmm10
+// CHECK:  encoding: [0x62,0x31,0x76,0x00,0x5d,0xd3]
+          vminss %xmm19, %xmm17, %xmm10
+
+// CHECK: vminss %xmm19, %xmm17, %xmm10 {%k5}
+// CHECK:  encoding: [0x62,0x31,0x76,0x05,0x5d,0xd3]
+          vminss %xmm19, %xmm17, %xmm10 {%k5}
+
+// CHECK: vminss %xmm19, %xmm17, %xmm10 {%k5} {z}
+// CHECK:  encoding: [0x62,0x31,0x76,0x85,0x5d,0xd3]
+          vminss %xmm19, %xmm17, %xmm10 {%k5} {z}
+
+// CHECK: vminss {sae}, %xmm19, %xmm17, %xmm10
+// CHECK:  encoding: [0x62,0x31,0x76,0x10,0x5d,0xd3]
+          vminss {sae}, %xmm19, %xmm17, %xmm10
+
+// CHECK: vminss (%rcx), %xmm17, %xmm10
+// CHECK:  encoding: [0x62,0x71,0x76,0x00,0x5d,0x11]
+          vminss (%rcx), %xmm17, %xmm10
+
+// CHECK: vminss 291(%rax,%r14,8), %xmm17, %xmm10
+// CHECK:  encoding: [0x62,0x31,0x76,0x00,0x5d,0x94,0xf0,0x23,0x01,0x00,0x00]
+          vminss 291(%rax,%r14,8), %xmm17, %xmm10
+
+// CHECK: vminss 508(%rdx), %xmm17, %xmm10
+// CHECK:  encoding: [0x62,0x71,0x76,0x00,0x5d,0x52,0x7f]
+          vminss 508(%rdx), %xmm17, %xmm10
+
+// CHECK: vminss 512(%rdx), %xmm17, %xmm10
+// CHECK:  encoding: [0x62,0x71,0x76,0x00,0x5d,0x92,0x00,0x02,0x00,0x00]
+          vminss 512(%rdx), %xmm17, %xmm10
+
+// CHECK: vminss -512(%rdx), %xmm17, %xmm10
+// CHECK:  encoding: [0x62,0x71,0x76,0x00,0x5d,0x52,0x80]
+          vminss -512(%rdx), %xmm17, %xmm10
+
+// CHECK: vminss -516(%rdx), %xmm17, %xmm10
+// CHECK:  encoding: [0x62,0x71,0x76,0x00,0x5d,0x92,0xfc,0xfd,0xff,0xff]
+          vminss -516(%rdx), %xmm17, %xmm10
+
+// CHECK: vmulpd {rn-sae}, %zmm23, %zmm4, %zmm24
+// CHECK:  encoding: [0x62,0x21,0xdd,0x18,0x59,0xc7]
+          vmulpd {rn-sae}, %zmm23, %zmm4, %zmm24
+
+// CHECK: vmulpd {ru-sae}, %zmm23, %zmm4, %zmm24
+// CHECK:  encoding: [0x62,0x21,0xdd,0x58,0x59,0xc7]
+          vmulpd {ru-sae}, %zmm23, %zmm4, %zmm24
+
+// CHECK: vmulpd {rd-sae}, %zmm23, %zmm4, %zmm24
+// CHECK:  encoding: [0x62,0x21,0xdd,0x38,0x59,0xc7]
+          vmulpd {rd-sae}, %zmm23, %zmm4, %zmm24
+
+// CHECK: vmulpd {rz-sae}, %zmm23, %zmm4, %zmm24
+// CHECK:  encoding: [0x62,0x21,0xdd,0x78,0x59,0xc7]
+          vmulpd {rz-sae}, %zmm23, %zmm4, %zmm24
+
+// CHECK: vmulps {rn-sae}, %zmm24, %zmm6, %zmm3
+// CHECK:  encoding: [0x62,0x91,0x4c,0x18,0x59,0xd8]
+          vmulps {rn-sae}, %zmm24, %zmm6, %zmm3
+
+// CHECK: vmulps {ru-sae}, %zmm24, %zmm6, %zmm3
+// CHECK:  encoding: [0x62,0x91,0x4c,0x58,0x59,0xd8]
+          vmulps {ru-sae}, %zmm24, %zmm6, %zmm3
+
+// CHECK: vmulps {rd-sae}, %zmm24, %zmm6, %zmm3
+// CHECK:  encoding: [0x62,0x91,0x4c,0x38,0x59,0xd8]
+          vmulps {rd-sae}, %zmm24, %zmm6, %zmm3
+
+// CHECK: vmulps {rz-sae}, %zmm24, %zmm6, %zmm3
+// CHECK:  encoding: [0x62,0x91,0x4c,0x78,0x59,0xd8]
+          vmulps {rz-sae}, %zmm24, %zmm6, %zmm3
+
+// CHECK: vmulsd %xmm18, %xmm4, %xmm13
+// CHECK:  encoding: [0x62,0x31,0xdf,0x08,0x59,0xea]
+          vmulsd %xmm18, %xmm4, %xmm13
+
+// CHECK: vmulsd %xmm18, %xmm4, %xmm13 {%k2}
+// CHECK:  encoding: [0x62,0x31,0xdf,0x0a,0x59,0xea]
+          vmulsd %xmm18, %xmm4, %xmm13 {%k2}
+
+// CHECK: vmulsd %xmm18, %xmm4, %xmm13 {%k2} {z}
+// CHECK:  encoding: [0x62,0x31,0xdf,0x8a,0x59,0xea]
+          vmulsd %xmm18, %xmm4, %xmm13 {%k2} {z}
+
+// CHECK: vmulsd {rn-sae}, %xmm18, %xmm4, %xmm13
+// CHECK:  encoding: [0x62,0x31,0xdf,0x18,0x59,0xea]
+          vmulsd {rn-sae}, %xmm18, %xmm4, %xmm13
+
+// CHECK: vmulsd {ru-sae}, %xmm18, %xmm4, %xmm13
+// CHECK:  encoding: [0x62,0x31,0xdf,0x58,0x59,0xea]
+          vmulsd {ru-sae}, %xmm18, %xmm4, %xmm13
+
+// CHECK: vmulsd {rd-sae}, %xmm18, %xmm4, %xmm13
+// CHECK:  encoding: [0x62,0x31,0xdf,0x38,0x59,0xea]
+          vmulsd {rd-sae}, %xmm18, %xmm4, %xmm13
+
+// CHECK: vmulsd {rz-sae}, %xmm18, %xmm4, %xmm13
+// CHECK:  encoding: [0x62,0x31,0xdf,0x78,0x59,0xea]
+          vmulsd {rz-sae}, %xmm18, %xmm4, %xmm13
+
+// CHECK: vmulsd (%rcx), %xmm4, %xmm13
+// CHECK:  encoding: [0x62,0x71,0xdf,0x08,0x59,0x29]
+          vmulsd (%rcx), %xmm4, %xmm13
+
+// CHECK: vmulsd 291(%rax,%r14,8), %xmm4, %xmm13
+// CHECK:  encoding: [0x62,0x31,0xdf,0x08,0x59,0xac,0xf0,0x23,0x01,0x00,0x00]
+          vmulsd 291(%rax,%r14,8), %xmm4, %xmm13
+
+// CHECK: vmulsd 1016(%rdx), %xmm4, %xmm13
+// CHECK:  encoding: [0x62,0x71,0xdf,0x08,0x59,0x6a,0x7f]
+          vmulsd 1016(%rdx), %xmm4, %xmm13
+
+// CHECK: vmulsd 1024(%rdx), %xmm4, %xmm13
+// CHECK:  encoding: [0x62,0x71,0xdf,0x08,0x59,0xaa,0x00,0x04,0x00,0x00]
+          vmulsd 1024(%rdx), %xmm4, %xmm13
+
+// CHECK: vmulsd -1024(%rdx), %xmm4, %xmm13
+// CHECK:  encoding: [0x62,0x71,0xdf,0x08,0x59,0x6a,0x80]
+          vmulsd -1024(%rdx), %xmm4, %xmm13
+
+// CHECK: vmulsd -1032(%rdx), %xmm4, %xmm13
+// CHECK:  encoding: [0x62,0x71,0xdf,0x08,0x59,0xaa,0xf8,0xfb,0xff,0xff]
+          vmulsd -1032(%rdx), %xmm4, %xmm13
+
+// CHECK: vmulss %xmm14, %xmm10, %xmm22
+// CHECK:  encoding: [0x62,0xc1,0x2e,0x08,0x59,0xf6]
+          vmulss %xmm14, %xmm10, %xmm22
+
+// CHECK: vmulss %xmm14, %xmm10, %xmm22 {%k4}
+// CHECK:  encoding: [0x62,0xc1,0x2e,0x0c,0x59,0xf6]
+          vmulss %xmm14, %xmm10, %xmm22 {%k4}
+
+// CHECK: vmulss %xmm14, %xmm10, %xmm22 {%k4} {z}
+// CHECK:  encoding: [0x62,0xc1,0x2e,0x8c,0x59,0xf6]
+          vmulss %xmm14, %xmm10, %xmm22 {%k4} {z}
+
+// CHECK: vmulss {rn-sae}, %xmm14, %xmm10, %xmm22
+// CHECK:  encoding: [0x62,0xc1,0x2e,0x18,0x59,0xf6]
+          vmulss {rn-sae}, %xmm14, %xmm10, %xmm22
+
+// CHECK: vmulss {ru-sae}, %xmm14, %xmm10, %xmm22
+// CHECK:  encoding: [0x62,0xc1,0x2e,0x58,0x59,0xf6]
+          vmulss {ru-sae}, %xmm14, %xmm10, %xmm22
+
+// CHECK: vmulss {rd-sae}, %xmm14, %xmm10, %xmm22
+// CHECK:  encoding: [0x62,0xc1,0x2e,0x38,0x59,0xf6]
+          vmulss {rd-sae}, %xmm14, %xmm10, %xmm22
+
+// CHECK: vmulss {rz-sae}, %xmm14, %xmm10, %xmm22
+// CHECK:  encoding: [0x62,0xc1,0x2e,0x78,0x59,0xf6]
+          vmulss {rz-sae}, %xmm14, %xmm10, %xmm22
+
+// CHECK: vmulss (%rcx), %xmm10, %xmm22
+// CHECK:  encoding: [0x62,0xe1,0x2e,0x08,0x59,0x31]
+          vmulss (%rcx), %xmm10, %xmm22
+
+// CHECK: vmulss 291(%rax,%r14,8), %xmm10, %xmm22
+// CHECK:  encoding: [0x62,0xa1,0x2e,0x08,0x59,0xb4,0xf0,0x23,0x01,0x00,0x00]
+          vmulss 291(%rax,%r14,8), %xmm10, %xmm22
+
+// CHECK: vmulss 508(%rdx), %xmm10, %xmm22
+// CHECK:  encoding: [0x62,0xe1,0x2e,0x08,0x59,0x72,0x7f]
+          vmulss 508(%rdx), %xmm10, %xmm22
+
+// CHECK: vmulss 512(%rdx), %xmm10, %xmm22
+// CHECK:  encoding: [0x62,0xe1,0x2e,0x08,0x59,0xb2,0x00,0x02,0x00,0x00]
+          vmulss 512(%rdx), %xmm10, %xmm22
+
+// CHECK: vmulss -512(%rdx), %xmm10, %xmm22
+// CHECK:  encoding: [0x62,0xe1,0x2e,0x08,0x59,0x72,0x80]
+          vmulss -512(%rdx), %xmm10, %xmm22
+
+// CHECK: vmulss -516(%rdx), %xmm10, %xmm22
+// CHECK:  encoding: [0x62,0xe1,0x2e,0x08,0x59,0xb2,0xfc,0xfd,0xff,0xff]
+          vmulss -516(%rdx), %xmm10, %xmm22
+
+// CHECK: vpslld %xmm3, %zmm6, %zmm17
+// CHECK:  encoding: [0x62,0xe1,0x4d,0x48,0xf2,0xcb]
+          vpslld %xmm3, %zmm6, %zmm17
+
+// CHECK: vpslld %xmm3, %zmm6, %zmm17 {%k3}
+// CHECK:  encoding: [0x62,0xe1,0x4d,0x4b,0xf2,0xcb]
+          vpslld %xmm3, %zmm6, %zmm17 {%k3}
+
+// CHECK: vpslld %xmm3, %zmm6, %zmm17 {%k3} {z}
+// CHECK:  encoding: [0x62,0xe1,0x4d,0xcb,0xf2,0xcb]
+          vpslld %xmm3, %zmm6, %zmm17 {%k3} {z}
+
+// CHECK: vpslld (%rcx), %zmm6, %zmm17
+// CHECK:  encoding: [0x62,0xe1,0x4d,0x48,0xf2,0x09]
+          vpslld (%rcx), %zmm6, %zmm17
+
+// CHECK: vpslld 291(%rax,%r14,8), %zmm6, %zmm17
+// CHECK:  encoding: [0x62,0xa1,0x4d,0x48,0xf2,0x8c,0xf0,0x23,0x01,0x00,0x00]
+          vpslld 291(%rax,%r14,8), %zmm6, %zmm17
+
+// CHECK: vpslld 2032(%rdx), %zmm6, %zmm17
+// CHECK:  encoding: [0x62,0xe1,0x4d,0x48,0xf2,0x4a,0x7f]
+          vpslld 2032(%rdx), %zmm6, %zmm17
+
+// CHECK: vpslld 2048(%rdx), %zmm6, %zmm17
+// CHECK:  encoding: [0x62,0xe1,0x4d,0x48,0xf2,0x8a,0x00,0x08,0x00,0x00]
+          vpslld 2048(%rdx), %zmm6, %zmm17
+
+// CHECK: vpslld -2048(%rdx), %zmm6, %zmm17
+// CHECK:  encoding: [0x62,0xe1,0x4d,0x48,0xf2,0x4a,0x80]
+          vpslld -2048(%rdx), %zmm6, %zmm17
+
+// CHECK: vpslld -2064(%rdx), %zmm6, %zmm17
+// CHECK:  encoding: [0x62,0xe1,0x4d,0x48,0xf2,0x8a,0xf0,0xf7,0xff,0xff]
+          vpslld -2064(%rdx), %zmm6, %zmm17
+
+// CHECK: vpsllq %xmm23, %zmm17, %zmm26
+// CHECK:  encoding: [0x62,0x21,0xf5,0x40,0xf3,0xd7]
+          vpsllq %xmm23, %zmm17, %zmm26
+
+// CHECK: vpsllq %xmm23, %zmm17, %zmm26 {%k6}
+// CHECK:  encoding: [0x62,0x21,0xf5,0x46,0xf3,0xd7]
+          vpsllq %xmm23, %zmm17, %zmm26 {%k6}
+
+// CHECK: vpsllq %xmm23, %zmm17, %zmm26 {%k6} {z}
+// CHECK:  encoding: [0x62,0x21,0xf5,0xc6,0xf3,0xd7]
+          vpsllq %xmm23, %zmm17, %zmm26 {%k6} {z}
+
+// CHECK: vpsllq (%rcx), %zmm17, %zmm26
+// CHECK:  encoding: [0x62,0x61,0xf5,0x40,0xf3,0x11]
+          vpsllq (%rcx), %zmm17, %zmm26
+
+// CHECK: vpsllq 291(%rax,%r14,8), %zmm17, %zmm26
+// CHECK:  encoding: [0x62,0x21,0xf5,0x40,0xf3,0x94,0xf0,0x23,0x01,0x00,0x00]
+          vpsllq 291(%rax,%r14,8), %zmm17, %zmm26
+
+// CHECK: vpsllq 2032(%rdx), %zmm17, %zmm26
+// CHECK:  encoding: [0x62,0x61,0xf5,0x40,0xf3,0x52,0x7f]
+          vpsllq 2032(%rdx), %zmm17, %zmm26
+
+// CHECK: vpsllq 2048(%rdx), %zmm17, %zmm26
+// CHECK:  encoding: [0x62,0x61,0xf5,0x40,0xf3,0x92,0x00,0x08,0x00,0x00]
+          vpsllq 2048(%rdx), %zmm17, %zmm26
+
+// CHECK: vpsllq -2048(%rdx), %zmm17, %zmm26
+// CHECK:  encoding: [0x62,0x61,0xf5,0x40,0xf3,0x52,0x80]
+          vpsllq -2048(%rdx), %zmm17, %zmm26
+
+// CHECK: vpsllq -2064(%rdx), %zmm17, %zmm26
+// CHECK:  encoding: [0x62,0x61,0xf5,0x40,0xf3,0x92,0xf0,0xf7,0xff,0xff]
+          vpsllq -2064(%rdx), %zmm17, %zmm26
+
+// CHECK: vpsllvd %zmm22, %zmm12, %zmm10
+// CHECK:  encoding: [0x62,0x32,0x1d,0x48,0x47,0xd6]
+          vpsllvd %zmm22, %zmm12, %zmm10
+
+// CHECK: vpsllvd %zmm22, %zmm12, %zmm10 {%k6}
+// CHECK:  encoding: [0x62,0x32,0x1d,0x4e,0x47,0xd6]
+          vpsllvd %zmm22, %zmm12, %zmm10 {%k6}
+
+// CHECK: vpsllvd %zmm22, %zmm12, %zmm10 {%k6} {z}
+// CHECK:  encoding: [0x62,0x32,0x1d,0xce,0x47,0xd6]
+          vpsllvd %zmm22, %zmm12, %zmm10 {%k6} {z}
+
+// CHECK: vpsllvd (%rcx), %zmm12, %zmm10
+// CHECK:  encoding: [0x62,0x72,0x1d,0x48,0x47,0x11]
+          vpsllvd (%rcx), %zmm12, %zmm10
+
+// CHECK: vpsllvd 291(%rax,%r14,8), %zmm12, %zmm10
+// CHECK:  encoding: [0x62,0x32,0x1d,0x48,0x47,0x94,0xf0,0x23,0x01,0x00,0x00]
+          vpsllvd 291(%rax,%r14,8), %zmm12, %zmm10
+
+// CHECK: vpsllvd (%rcx){1to16}, %zmm12, %zmm10
+// CHECK:  encoding: [0x62,0x72,0x1d,0x58,0x47,0x11]
+          vpsllvd (%rcx){1to16}, %zmm12, %zmm10
+
+// CHECK: vpsllvd 8128(%rdx), %zmm12, %zmm10
+// CHECK:  encoding: [0x62,0x72,0x1d,0x48,0x47,0x52,0x7f]
+          vpsllvd 8128(%rdx), %zmm12, %zmm10
+
+// CHECK: vpsllvd 8192(%rdx), %zmm12, %zmm10
+// CHECK:  encoding: [0x62,0x72,0x1d,0x48,0x47,0x92,0x00,0x20,0x00,0x00]
+          vpsllvd 8192(%rdx), %zmm12, %zmm10
+
+// CHECK: vpsllvd -8192(%rdx), %zmm12, %zmm10
+// CHECK:  encoding: [0x62,0x72,0x1d,0x48,0x47,0x52,0x80]
+          vpsllvd -8192(%rdx), %zmm12, %zmm10
+
+// CHECK: vpsllvd -8256(%rdx), %zmm12, %zmm10
+// CHECK:  encoding: [0x62,0x72,0x1d,0x48,0x47,0x92,0xc0,0xdf,0xff,0xff]
+          vpsllvd -8256(%rdx), %zmm12, %zmm10
+
+// CHECK: vpsllvd 508(%rdx){1to16}, %zmm12, %zmm10
+// CHECK:  encoding: [0x62,0x72,0x1d,0x58,0x47,0x52,0x7f]
+          vpsllvd 508(%rdx){1to16}, %zmm12, %zmm10
+
+// CHECK: vpsllvd 512(%rdx){1to16}, %zmm12, %zmm10
+// CHECK:  encoding: [0x62,0x72,0x1d,0x58,0x47,0x92,0x00,0x02,0x00,0x00]
+          vpsllvd 512(%rdx){1to16}, %zmm12, %zmm10
+
+// CHECK: vpsllvd -512(%rdx){1to16}, %zmm12, %zmm10
+// CHECK:  encoding: [0x62,0x72,0x1d,0x58,0x47,0x52,0x80]
+          vpsllvd -512(%rdx){1to16}, %zmm12, %zmm10
+
+// CHECK: vpsllvd -516(%rdx){1to16}, %zmm12, %zmm10
+// CHECK:  encoding: [0x62,0x72,0x1d,0x58,0x47,0x92,0xfc,0xfd,0xff,0xff]
+          vpsllvd -516(%rdx){1to16}, %zmm12, %zmm10
+
+// CHECK: vpsllvq %zmm26, %zmm5, %zmm16
+// CHECK:  encoding: [0x62,0x82,0xd5,0x48,0x47,0xc2]
+          vpsllvq %zmm26, %zmm5, %zmm16
+
+// CHECK: vpsllvq %zmm26, %zmm5, %zmm16 {%k5}
+// CHECK:  encoding: [0x62,0x82,0xd5,0x4d,0x47,0xc2]
+          vpsllvq %zmm26, %zmm5, %zmm16 {%k5}
+
+// CHECK: vpsllvq %zmm26, %zmm5, %zmm16 {%k5} {z}
+// CHECK:  encoding: [0x62,0x82,0xd5,0xcd,0x47,0xc2]
+          vpsllvq %zmm26, %zmm5, %zmm16 {%k5} {z}
+
+// CHECK: vpsllvq (%rcx), %zmm5, %zmm16
+// CHECK:  encoding: [0x62,0xe2,0xd5,0x48,0x47,0x01]
+          vpsllvq (%rcx), %zmm5, %zmm16
+
+// CHECK: vpsllvq 291(%rax,%r14,8), %zmm5, %zmm16
+// CHECK:  encoding: [0x62,0xa2,0xd5,0x48,0x47,0x84,0xf0,0x23,0x01,0x00,0x00]
+          vpsllvq 291(%rax,%r14,8), %zmm5, %zmm16
+
+// CHECK: vpsllvq (%rcx){1to8}, %zmm5, %zmm16
+// CHECK:  encoding: [0x62,0xe2,0xd5,0x58,0x47,0x01]
+          vpsllvq (%rcx){1to8}, %zmm5, %zmm16
+
+// CHECK: vpsllvq 8128(%rdx), %zmm5, %zmm16
+// CHECK:  encoding: [0x62,0xe2,0xd5,0x48,0x47,0x42,0x7f]
+          vpsllvq 8128(%rdx), %zmm5, %zmm16
+
+// CHECK: vpsllvq 8192(%rdx), %zmm5, %zmm16
+// CHECK:  encoding: [0x62,0xe2,0xd5,0x48,0x47,0x82,0x00,0x20,0x00,0x00]
+          vpsllvq 8192(%rdx), %zmm5, %zmm16
+
+// CHECK: vpsllvq -8192(%rdx), %zmm5, %zmm16
+// CHECK:  encoding: [0x62,0xe2,0xd5,0x48,0x47,0x42,0x80]
+          vpsllvq -8192(%rdx), %zmm5, %zmm16
+
+// CHECK: vpsllvq -8256(%rdx), %zmm5, %zmm16
+// CHECK:  encoding: [0x62,0xe2,0xd5,0x48,0x47,0x82,0xc0,0xdf,0xff,0xff]
+          vpsllvq -8256(%rdx), %zmm5, %zmm16
+
+// CHECK: vpsllvq 1016(%rdx){1to8}, %zmm5, %zmm16
+// CHECK:  encoding: [0x62,0xe2,0xd5,0x58,0x47,0x42,0x7f]
+          vpsllvq 1016(%rdx){1to8}, %zmm5, %zmm16
+
+// CHECK: vpsllvq 1024(%rdx){1to8}, %zmm5, %zmm16
+// CHECK:  encoding: [0x62,0xe2,0xd5,0x58,0x47,0x82,0x00,0x04,0x00,0x00]
+          vpsllvq 1024(%rdx){1to8}, %zmm5, %zmm16
+
+// CHECK: vpsllvq -1024(%rdx){1to8}, %zmm5, %zmm16
+// CHECK:  encoding: [0x62,0xe2,0xd5,0x58,0x47,0x42,0x80]
+          vpsllvq -1024(%rdx){1to8}, %zmm5, %zmm16
+
+// CHECK: vpsllvq -1032(%rdx){1to8}, %zmm5, %zmm16
+// CHECK:  encoding: [0x62,0xe2,0xd5,0x58,0x47,0x82,0xf8,0xfb,0xff,0xff]
+          vpsllvq -1032(%rdx){1to8}, %zmm5, %zmm16
+
+// CHECK: vpsrad %xmm6, %zmm14, %zmm18
+// CHECK:  encoding: [0x62,0xe1,0x0d,0x48,0xe2,0xd6]
+          vpsrad %xmm6, %zmm14, %zmm18
+
+// CHECK: vpsrad %xmm6, %zmm14, %zmm18 {%k1}
+// CHECK:  encoding: [0x62,0xe1,0x0d,0x49,0xe2,0xd6]
+          vpsrad %xmm6, %zmm14, %zmm18 {%k1}
+
+// CHECK: vpsrad %xmm6, %zmm14, %zmm18 {%k1} {z}
+// CHECK:  encoding: [0x62,0xe1,0x0d,0xc9,0xe2,0xd6]
+          vpsrad %xmm6, %zmm14, %zmm18 {%k1} {z}
+
+// CHECK: vpsrad (%rcx), %zmm14, %zmm18
+// CHECK:  encoding: [0x62,0xe1,0x0d,0x48,0xe2,0x11]
+          vpsrad (%rcx), %zmm14, %zmm18
+
+// CHECK: vpsrad 291(%rax,%r14,8), %zmm14, %zmm18
+// CHECK:  encoding: [0x62,0xa1,0x0d,0x48,0xe2,0x94,0xf0,0x23,0x01,0x00,0x00]
+          vpsrad 291(%rax,%r14,8), %zmm14, %zmm18
+
+// CHECK: vpsrad 2032(%rdx), %zmm14, %zmm18
+// CHECK:  encoding: [0x62,0xe1,0x0d,0x48,0xe2,0x52,0x7f]
+          vpsrad 2032(%rdx), %zmm14, %zmm18
+
+// CHECK: vpsrad 2048(%rdx), %zmm14, %zmm18
+// CHECK:  encoding: [0x62,0xe1,0x0d,0x48,0xe2,0x92,0x00,0x08,0x00,0x00]
+          vpsrad 2048(%rdx), %zmm14, %zmm18
+
+// CHECK: vpsrad -2048(%rdx), %zmm14, %zmm18
+// CHECK:  encoding: [0x62,0xe1,0x0d,0x48,0xe2,0x52,0x80]
+          vpsrad -2048(%rdx), %zmm14, %zmm18
+
+// CHECK: vpsrad -2064(%rdx), %zmm14, %zmm18
+// CHECK:  encoding: [0x62,0xe1,0x0d,0x48,0xe2,0x92,0xf0,0xf7,0xff,0xff]
+          vpsrad -2064(%rdx), %zmm14, %zmm18
+
+// CHECK: vpsraq %xmm22, %zmm17, %zmm21
+// CHECK:  encoding: [0x62,0xa1,0xf5,0x40,0xe2,0xee]
+          vpsraq %xmm22, %zmm17, %zmm21
+
+// CHECK: vpsraq %xmm22, %zmm17, %zmm21 {%k3}
+// CHECK:  encoding: [0x62,0xa1,0xf5,0x43,0xe2,0xee]
+          vpsraq %xmm22, %zmm17, %zmm21 {%k3}
+
+// CHECK: vpsraq %xmm22, %zmm17, %zmm21 {%k3} {z}
+// CHECK:  encoding: [0x62,0xa1,0xf5,0xc3,0xe2,0xee]
+          vpsraq %xmm22, %zmm17, %zmm21 {%k3} {z}
+
+// CHECK: vpsraq (%rcx), %zmm17, %zmm21
+// CHECK:  encoding: [0x62,0xe1,0xf5,0x40,0xe2,0x29]
+          vpsraq (%rcx), %zmm17, %zmm21
+
+// CHECK: vpsraq 291(%rax,%r14,8), %zmm17, %zmm21
+// CHECK:  encoding: [0x62,0xa1,0xf5,0x40,0xe2,0xac,0xf0,0x23,0x01,0x00,0x00]
+          vpsraq 291(%rax,%r14,8), %zmm17, %zmm21
+
+// CHECK: vpsraq 2032(%rdx), %zmm17, %zmm21
+// CHECK:  encoding: [0x62,0xe1,0xf5,0x40,0xe2,0x6a,0x7f]
+          vpsraq 2032(%rdx), %zmm17, %zmm21
+
+// CHECK: vpsraq 2048(%rdx), %zmm17, %zmm21
+// CHECK:  encoding: [0x62,0xe1,0xf5,0x40,0xe2,0xaa,0x00,0x08,0x00,0x00]
+          vpsraq 2048(%rdx), %zmm17, %zmm21
+
+// CHECK: vpsraq -2048(%rdx), %zmm17, %zmm21
+// CHECK:  encoding: [0x62,0xe1,0xf5,0x40,0xe2,0x6a,0x80]
+          vpsraq -2048(%rdx), %zmm17, %zmm21
+
+// CHECK: vpsraq -2064(%rdx), %zmm17, %zmm21
+// CHECK:  encoding: [0x62,0xe1,0xf5,0x40,0xe2,0xaa,0xf0,0xf7,0xff,0xff]
+          vpsraq -2064(%rdx), %zmm17, %zmm21
+
+// CHECK: vpsravd %zmm17, %zmm29, %zmm29
+// CHECK:  encoding: [0x62,0x22,0x15,0x40,0x46,0xe9]
+          vpsravd %zmm17, %zmm29, %zmm29
+
+// CHECK: vpsravd %zmm17, %zmm29, %zmm29 {%k2}
+// CHECK:  encoding: [0x62,0x22,0x15,0x42,0x46,0xe9]
+          vpsravd %zmm17, %zmm29, %zmm29 {%k2}
+
+// CHECK: vpsravd %zmm17, %zmm29, %zmm29 {%k2} {z}
+// CHECK:  encoding: [0x62,0x22,0x15,0xc2,0x46,0xe9]
+          vpsravd %zmm17, %zmm29, %zmm29 {%k2} {z}
+
+// CHECK: vpsravd (%rcx), %zmm29, %zmm29
+// CHECK:  encoding: [0x62,0x62,0x15,0x40,0x46,0x29]
+          vpsravd (%rcx), %zmm29, %zmm29
+
+// CHECK: vpsravd 291(%rax,%r14,8), %zmm29, %zmm29
+// CHECK:  encoding: [0x62,0x22,0x15,0x40,0x46,0xac,0xf0,0x23,0x01,0x00,0x00]
+          vpsravd 291(%rax,%r14,8), %zmm29, %zmm29
+
+// CHECK: vpsravd (%rcx){1to16}, %zmm29, %zmm29
+// CHECK:  encoding: [0x62,0x62,0x15,0x50,0x46,0x29]
+          vpsravd (%rcx){1to16}, %zmm29, %zmm29
+
+// CHECK: vpsravd 8128(%rdx), %zmm29, %zmm29
+// CHECK:  encoding: [0x62,0x62,0x15,0x40,0x46,0x6a,0x7f]
+          vpsravd 8128(%rdx), %zmm29, %zmm29
+
+// CHECK: vpsravd 8192(%rdx), %zmm29, %zmm29
+// CHECK:  encoding: [0x62,0x62,0x15,0x40,0x46,0xaa,0x00,0x20,0x00,0x00]
+          vpsravd 8192(%rdx), %zmm29, %zmm29
+
+// CHECK: vpsravd -8192(%rdx), %zmm29, %zmm29
+// CHECK:  encoding: [0x62,0x62,0x15,0x40,0x46,0x6a,0x80]
+          vpsravd -8192(%rdx), %zmm29, %zmm29
+
+// CHECK: vpsravd -8256(%rdx), %zmm29, %zmm29
+// CHECK:  encoding: [0x62,0x62,0x15,0x40,0x46,0xaa,0xc0,0xdf,0xff,0xff]
+          vpsravd -8256(%rdx), %zmm29, %zmm29
+
+// CHECK: vpsravd 508(%rdx){1to16}, %zmm29, %zmm29
+// CHECK:  encoding: [0x62,0x62,0x15,0x50,0x46,0x6a,0x7f]
+          vpsravd 508(%rdx){1to16}, %zmm29, %zmm29
+
+// CHECK: vpsravd 512(%rdx){1to16}, %zmm29, %zmm29
+// CHECK:  encoding: [0x62,0x62,0x15,0x50,0x46,0xaa,0x00,0x02,0x00,0x00]
+          vpsravd 512(%rdx){1to16}, %zmm29, %zmm29
+
+// CHECK: vpsravd -512(%rdx){1to16}, %zmm29, %zmm29
+// CHECK:  encoding: [0x62,0x62,0x15,0x50,0x46,0x6a,0x80]
+          vpsravd -512(%rdx){1to16}, %zmm29, %zmm29
+
+// CHECK: vpsravd -516(%rdx){1to16}, %zmm29, %zmm29
+// CHECK:  encoding: [0x62,0x62,0x15,0x50,0x46,0xaa,0xfc,0xfd,0xff,0xff]
+          vpsravd -516(%rdx){1to16}, %zmm29, %zmm29
+
+// CHECK: vpsravq %zmm2, %zmm20, %zmm22
+// CHECK:  encoding: [0x62,0xe2,0xdd,0x40,0x46,0xf2]
+          vpsravq %zmm2, %zmm20, %zmm22
+
+// CHECK: vpsravq %zmm2, %zmm20, %zmm22 {%k2}
+// CHECK:  encoding: [0x62,0xe2,0xdd,0x42,0x46,0xf2]
+          vpsravq %zmm2, %zmm20, %zmm22 {%k2}
+
+// CHECK: vpsravq %zmm2, %zmm20, %zmm22 {%k2} {z}
+// CHECK:  encoding: [0x62,0xe2,0xdd,0xc2,0x46,0xf2]
+          vpsravq %zmm2, %zmm20, %zmm22 {%k2} {z}
+
+// CHECK: vpsravq (%rcx), %zmm20, %zmm22
+// CHECK:  encoding: [0x62,0xe2,0xdd,0x40,0x46,0x31]
+          vpsravq (%rcx), %zmm20, %zmm22
+
+// CHECK: vpsravq 291(%rax,%r14,8), %zmm20, %zmm22
+// CHECK:  encoding: [0x62,0xa2,0xdd,0x40,0x46,0xb4,0xf0,0x23,0x01,0x00,0x00]
+          vpsravq 291(%rax,%r14,8), %zmm20, %zmm22
+
+// CHECK: vpsravq (%rcx){1to8}, %zmm20, %zmm22
+// CHECK:  encoding: [0x62,0xe2,0xdd,0x50,0x46,0x31]
+          vpsravq (%rcx){1to8}, %zmm20, %zmm22
+
+// CHECK: vpsravq 8128(%rdx), %zmm20, %zmm22
+// CHECK:  encoding: [0x62,0xe2,0xdd,0x40,0x46,0x72,0x7f]
+          vpsravq 8128(%rdx), %zmm20, %zmm22
+
+// CHECK: vpsravq 8192(%rdx), %zmm20, %zmm22
+// CHECK:  encoding: [0x62,0xe2,0xdd,0x40,0x46,0xb2,0x00,0x20,0x00,0x00]
+          vpsravq 8192(%rdx), %zmm20, %zmm22
+
+// CHECK: vpsravq -8192(%rdx), %zmm20, %zmm22
+// CHECK:  encoding: [0x62,0xe2,0xdd,0x40,0x46,0x72,0x80]
+          vpsravq -8192(%rdx), %zmm20, %zmm22
+
+// CHECK: vpsravq -8256(%rdx), %zmm20, %zmm22
+// CHECK:  encoding: [0x62,0xe2,0xdd,0x40,0x46,0xb2,0xc0,0xdf,0xff,0xff]
+          vpsravq -8256(%rdx), %zmm20, %zmm22
+
+// CHECK: vpsravq 1016(%rdx){1to8}, %zmm20, %zmm22
+// CHECK:  encoding: [0x62,0xe2,0xdd,0x50,0x46,0x72,0x7f]
+          vpsravq 1016(%rdx){1to8}, %zmm20, %zmm22
+
+// CHECK: vpsravq 1024(%rdx){1to8}, %zmm20, %zmm22
+// CHECK:  encoding: [0x62,0xe2,0xdd,0x50,0x46,0xb2,0x00,0x04,0x00,0x00]
+          vpsravq 1024(%rdx){1to8}, %zmm20, %zmm22
+
+// CHECK: vpsravq -1024(%rdx){1to8}, %zmm20, %zmm22
+// CHECK:  encoding: [0x62,0xe2,0xdd,0x50,0x46,0x72,0x80]
+          vpsravq -1024(%rdx){1to8}, %zmm20, %zmm22
+
+// CHECK: vpsravq -1032(%rdx){1to8}, %zmm20, %zmm22
+// CHECK:  encoding: [0x62,0xe2,0xdd,0x50,0x46,0xb2,0xf8,0xfb,0xff,0xff]
+          vpsravq -1032(%rdx){1to8}, %zmm20, %zmm22
+
+// CHECK: vpsrld %xmm25, %zmm17, %zmm1
+// CHECK:  encoding: [0x62,0x91,0x75,0x40,0xd2,0xc9]
+          vpsrld %xmm25, %zmm17, %zmm1
+
+// CHECK: vpsrld %xmm25, %zmm17, %zmm1 {%k4}
+// CHECK:  encoding: [0x62,0x91,0x75,0x44,0xd2,0xc9]
+          vpsrld %xmm25, %zmm17, %zmm1 {%k4}
+
+// CHECK: vpsrld %xmm25, %zmm17, %zmm1 {%k4} {z}
+// CHECK:  encoding: [0x62,0x91,0x75,0xc4,0xd2,0xc9]
+          vpsrld %xmm25, %zmm17, %zmm1 {%k4} {z}
+
+// CHECK: vpsrld (%rcx), %zmm17, %zmm1
+// CHECK:  encoding: [0x62,0xf1,0x75,0x40,0xd2,0x09]
+          vpsrld (%rcx), %zmm17, %zmm1
+
+// CHECK: vpsrld 291(%rax,%r14,8), %zmm17, %zmm1
+// CHECK:  encoding: [0x62,0xb1,0x75,0x40,0xd2,0x8c,0xf0,0x23,0x01,0x00,0x00]
+          vpsrld 291(%rax,%r14,8), %zmm17, %zmm1
+
+// CHECK: vpsrld 2032(%rdx), %zmm17, %zmm1
+// CHECK:  encoding: [0x62,0xf1,0x75,0x40,0xd2,0x4a,0x7f]
+          vpsrld 2032(%rdx), %zmm17, %zmm1
+
+// CHECK: vpsrld 2048(%rdx), %zmm17, %zmm1
+// CHECK:  encoding: [0x62,0xf1,0x75,0x40,0xd2,0x8a,0x00,0x08,0x00,0x00]
+          vpsrld 2048(%rdx), %zmm17, %zmm1
+
+// CHECK: vpsrld -2048(%rdx), %zmm17, %zmm1
+// CHECK:  encoding: [0x62,0xf1,0x75,0x40,0xd2,0x4a,0x80]
+          vpsrld -2048(%rdx), %zmm17, %zmm1
+
+// CHECK: vpsrld -2064(%rdx), %zmm17, %zmm1
+// CHECK:  encoding: [0x62,0xf1,0x75,0x40,0xd2,0x8a,0xf0,0xf7,0xff,0xff]
+          vpsrld -2064(%rdx), %zmm17, %zmm1
+
+// CHECK: vpsrlq %xmm3, %zmm11, %zmm9
+// CHECK:  encoding: [0x62,0x71,0xa5,0x48,0xd3,0xcb]
+          vpsrlq %xmm3, %zmm11, %zmm9
+
+// CHECK: vpsrlq %xmm3, %zmm11, %zmm9 {%k5}
+// CHECK:  encoding: [0x62,0x71,0xa5,0x4d,0xd3,0xcb]
+          vpsrlq %xmm3, %zmm11, %zmm9 {%k5}
+
+// CHECK: vpsrlq %xmm3, %zmm11, %zmm9 {%k5} {z}
+// CHECK:  encoding: [0x62,0x71,0xa5,0xcd,0xd3,0xcb]
+          vpsrlq %xmm3, %zmm11, %zmm9 {%k5} {z}
+
+// CHECK: vpsrlq (%rcx), %zmm11, %zmm9
+// CHECK:  encoding: [0x62,0x71,0xa5,0x48,0xd3,0x09]
+          vpsrlq (%rcx), %zmm11, %zmm9
+
+// CHECK: vpsrlq 291(%rax,%r14,8), %zmm11, %zmm9
+// CHECK:  encoding: [0x62,0x31,0xa5,0x48,0xd3,0x8c,0xf0,0x23,0x01,0x00,0x00]
+          vpsrlq 291(%rax,%r14,8), %zmm11, %zmm9
+
+// CHECK: vpsrlq 2032(%rdx), %zmm11, %zmm9
+// CHECK:  encoding: [0x62,0x71,0xa5,0x48,0xd3,0x4a,0x7f]
+          vpsrlq 2032(%rdx), %zmm11, %zmm9
+
+// CHECK: vpsrlq 2048(%rdx), %zmm11, %zmm9
+// CHECK:  encoding: [0x62,0x71,0xa5,0x48,0xd3,0x8a,0x00,0x08,0x00,0x00]
+          vpsrlq 2048(%rdx), %zmm11, %zmm9
+
+// CHECK: vpsrlq -2048(%rdx), %zmm11, %zmm9
+// CHECK:  encoding: [0x62,0x71,0xa5,0x48,0xd3,0x4a,0x80]
+          vpsrlq -2048(%rdx), %zmm11, %zmm9
+
+// CHECK: vpsrlq -2064(%rdx), %zmm11, %zmm9
+// CHECK:  encoding: [0x62,0x71,0xa5,0x48,0xd3,0x8a,0xf0,0xf7,0xff,0xff]
+          vpsrlq -2064(%rdx), %zmm11, %zmm9
+
+// CHECK: vpsrlvd %zmm28, %zmm14, %zmm8
+// CHECK:  encoding: [0x62,0x12,0x0d,0x48,0x45,0xc4]
+          vpsrlvd %zmm28, %zmm14, %zmm8
+
+// CHECK: vpsrlvd %zmm28, %zmm14, %zmm8 {%k4}
+// CHECK:  encoding: [0x62,0x12,0x0d,0x4c,0x45,0xc4]
+          vpsrlvd %zmm28, %zmm14, %zmm8 {%k4}
+
+// CHECK: vpsrlvd %zmm28, %zmm14, %zmm8 {%k4} {z}
+// CHECK:  encoding: [0x62,0x12,0x0d,0xcc,0x45,0xc4]
+          vpsrlvd %zmm28, %zmm14, %zmm8 {%k4} {z}
+
+// CHECK: vpsrlvd (%rcx), %zmm14, %zmm8
+// CHECK:  encoding: [0x62,0x72,0x0d,0x48,0x45,0x01]
+          vpsrlvd (%rcx), %zmm14, %zmm8
+
+// CHECK: vpsrlvd 291(%rax,%r14,8), %zmm14, %zmm8
+// CHECK:  encoding: [0x62,0x32,0x0d,0x48,0x45,0x84,0xf0,0x23,0x01,0x00,0x00]
+          vpsrlvd 291(%rax,%r14,8), %zmm14, %zmm8
+
+// CHECK: vpsrlvd (%rcx){1to16}, %zmm14, %zmm8
+// CHECK:  encoding: [0x62,0x72,0x0d,0x58,0x45,0x01]
+          vpsrlvd (%rcx){1to16}, %zmm14, %zmm8
+
+// CHECK: vpsrlvd 8128(%rdx), %zmm14, %zmm8
+// CHECK:  encoding: [0x62,0x72,0x0d,0x48,0x45,0x42,0x7f]
+          vpsrlvd 8128(%rdx), %zmm14, %zmm8
+
+// CHECK: vpsrlvd 8192(%rdx), %zmm14, %zmm8
+// CHECK:  encoding: [0x62,0x72,0x0d,0x48,0x45,0x82,0x00,0x20,0x00,0x00]
+          vpsrlvd 8192(%rdx), %zmm14, %zmm8
+
+// CHECK: vpsrlvd -8192(%rdx), %zmm14, %zmm8
+// CHECK:  encoding: [0x62,0x72,0x0d,0x48,0x45,0x42,0x80]
+          vpsrlvd -8192(%rdx), %zmm14, %zmm8
+
+// CHECK: vpsrlvd -8256(%rdx), %zmm14, %zmm8
+// CHECK:  encoding: [0x62,0x72,0x0d,0x48,0x45,0x82,0xc0,0xdf,0xff,0xff]
+          vpsrlvd -8256(%rdx), %zmm14, %zmm8
+
+// CHECK: vpsrlvd 508(%rdx){1to16}, %zmm14, %zmm8
+// CHECK:  encoding: [0x62,0x72,0x0d,0x58,0x45,0x42,0x7f]
+          vpsrlvd 508(%rdx){1to16}, %zmm14, %zmm8
+
+// CHECK: vpsrlvd 512(%rdx){1to16}, %zmm14, %zmm8
+// CHECK:  encoding: [0x62,0x72,0x0d,0x58,0x45,0x82,0x00,0x02,0x00,0x00]
+          vpsrlvd 512(%rdx){1to16}, %zmm14, %zmm8
+
+// CHECK: vpsrlvd -512(%rdx){1to16}, %zmm14, %zmm8
+// CHECK:  encoding: [0x62,0x72,0x0d,0x58,0x45,0x42,0x80]
+          vpsrlvd -512(%rdx){1to16}, %zmm14, %zmm8
+
+// CHECK: vpsrlvd -516(%rdx){1to16}, %zmm14, %zmm8
+// CHECK:  encoding: [0x62,0x72,0x0d,0x58,0x45,0x82,0xfc,0xfd,0xff,0xff]
+          vpsrlvd -516(%rdx){1to16}, %zmm14, %zmm8
+
+// CHECK: vpsrlvq %zmm26, %zmm7, %zmm20
+// CHECK:  encoding: [0x62,0x82,0xc5,0x48,0x45,0xe2]
+          vpsrlvq %zmm26, %zmm7, %zmm20
+
+// CHECK: vpsrlvq %zmm26, %zmm7, %zmm20 {%k5}
+// CHECK:  encoding: [0x62,0x82,0xc5,0x4d,0x45,0xe2]
+          vpsrlvq %zmm26, %zmm7, %zmm20 {%k5}
+
+// CHECK: vpsrlvq %zmm26, %zmm7, %zmm20 {%k5} {z}
+// CHECK:  encoding: [0x62,0x82,0xc5,0xcd,0x45,0xe2]
+          vpsrlvq %zmm26, %zmm7, %zmm20 {%k5} {z}
+
+// CHECK: vpsrlvq (%rcx), %zmm7, %zmm20
+// CHECK:  encoding: [0x62,0xe2,0xc5,0x48,0x45,0x21]
+          vpsrlvq (%rcx), %zmm7, %zmm20
+
+// CHECK: vpsrlvq 291(%rax,%r14,8), %zmm7, %zmm20
+// CHECK:  encoding: [0x62,0xa2,0xc5,0x48,0x45,0xa4,0xf0,0x23,0x01,0x00,0x00]
+          vpsrlvq 291(%rax,%r14,8), %zmm7, %zmm20
+
+// CHECK: vpsrlvq (%rcx){1to8}, %zmm7, %zmm20
+// CHECK:  encoding: [0x62,0xe2,0xc5,0x58,0x45,0x21]
+          vpsrlvq (%rcx){1to8}, %zmm7, %zmm20
+
+// CHECK: vpsrlvq 8128(%rdx), %zmm7, %zmm20
+// CHECK:  encoding: [0x62,0xe2,0xc5,0x48,0x45,0x62,0x7f]
+          vpsrlvq 8128(%rdx), %zmm7, %zmm20
+
+// CHECK: vpsrlvq 8192(%rdx), %zmm7, %zmm20
+// CHECK:  encoding: [0x62,0xe2,0xc5,0x48,0x45,0xa2,0x00,0x20,0x00,0x00]
+          vpsrlvq 8192(%rdx), %zmm7, %zmm20
+
+// CHECK: vpsrlvq -8192(%rdx), %zmm7, %zmm20
+// CHECK:  encoding: [0x62,0xe2,0xc5,0x48,0x45,0x62,0x80]
+          vpsrlvq -8192(%rdx), %zmm7, %zmm20
+
+// CHECK: vpsrlvq -8256(%rdx), %zmm7, %zmm20
+// CHECK:  encoding: [0x62,0xe2,0xc5,0x48,0x45,0xa2,0xc0,0xdf,0xff,0xff]
+          vpsrlvq -8256(%rdx), %zmm7, %zmm20
+
+// CHECK: vpsrlvq 1016(%rdx){1to8}, %zmm7, %zmm20
+// CHECK:  encoding: [0x62,0xe2,0xc5,0x58,0x45,0x62,0x7f]
+          vpsrlvq 1016(%rdx){1to8}, %zmm7, %zmm20
+
+// CHECK: vpsrlvq 1024(%rdx){1to8}, %zmm7, %zmm20
+// CHECK:  encoding: [0x62,0xe2,0xc5,0x58,0x45,0xa2,0x00,0x04,0x00,0x00]
+          vpsrlvq 1024(%rdx){1to8}, %zmm7, %zmm20
+
+// CHECK: vpsrlvq -1024(%rdx){1to8}, %zmm7, %zmm20
+// CHECK:  encoding: [0x62,0xe2,0xc5,0x58,0x45,0x62,0x80]
+          vpsrlvq -1024(%rdx){1to8}, %zmm7, %zmm20
+
+// CHECK: vpsrlvq -1032(%rdx){1to8}, %zmm7, %zmm20
+// CHECK:  encoding: [0x62,0xe2,0xc5,0x58,0x45,0xa2,0xf8,0xfb,0xff,0xff]
+          vpsrlvq -1032(%rdx){1to8}, %zmm7, %zmm20
+
+// CHECK: vpsrld $171, %zmm5, %zmm3
+// CHECK:  encoding: [0x62,0xf1,0x65,0x48,0x72,0xd5,0xab]
+          vpsrld $171, %zmm5, %zmm3
+
+// CHECK: vpsrld $171, %zmm5, %zmm3 {%k4}
+// CHECK:  encoding: [0x62,0xf1,0x65,0x4c,0x72,0xd5,0xab]
+          vpsrld $171, %zmm5, %zmm3 {%k4}
+
+// CHECK: vpsrld $171, %zmm5, %zmm3 {%k4} {z}
+// CHECK:  encoding: [0x62,0xf1,0x65,0xcc,0x72,0xd5,0xab]
+          vpsrld $171, %zmm5, %zmm3 {%k4} {z}
+
+// CHECK: vpsrld $123, %zmm5, %zmm3
+// CHECK:  encoding: [0x62,0xf1,0x65,0x48,0x72,0xd5,0x7b]
+          vpsrld $123, %zmm5, %zmm3
+
+// CHECK: vpsrld $123, (%rcx), %zmm3
+// CHECK:  encoding: [0x62,0xf1,0x65,0x48,0x72,0x11,0x7b]
+          vpsrld $123, (%rcx), %zmm3
+
+// CHECK: vpsrld $123, 291(%rax,%r14,8), %zmm3
+// CHECK:  encoding: [0x62,0xb1,0x65,0x48,0x72,0x94,0xf0,0x23,0x01,0x00,0x00,0x7b]
+          vpsrld $123, 291(%rax,%r14,8), %zmm3
+
+// CHECK: vpsrld $123, (%rcx){1to16}, %zmm3
+// CHECK:  encoding: [0x62,0xf1,0x65,0x58,0x72,0x11,0x7b]
+          vpsrld $123, (%rcx){1to16}, %zmm3
+
+// CHECK: vpsrld $123, 8128(%rdx), %zmm3
+// CHECK:  encoding: [0x62,0xf1,0x65,0x48,0x72,0x52,0x7f,0x7b]
+          vpsrld $123, 8128(%rdx), %zmm3
+
+// CHECK: vpsrld $123, 8192(%rdx), %zmm3
+// CHECK:  encoding: [0x62,0xf1,0x65,0x48,0x72,0x92,0x00,0x20,0x00,0x00,0x7b]
+          vpsrld $123, 8192(%rdx), %zmm3
+
+// CHECK: vpsrld $123, -8192(%rdx), %zmm3
+// CHECK:  encoding: [0x62,0xf1,0x65,0x48,0x72,0x52,0x80,0x7b]
+          vpsrld $123, -8192(%rdx), %zmm3
+
+// CHECK: vpsrld $123, -8256(%rdx), %zmm3
+// CHECK:  encoding: [0x62,0xf1,0x65,0x48,0x72,0x92,0xc0,0xdf,0xff,0xff,0x7b]
+          vpsrld $123, -8256(%rdx), %zmm3
+
+// CHECK: vpsrld $123, 508(%rdx){1to16}, %zmm3
+// CHECK:  encoding: [0x62,0xf1,0x65,0x58,0x72,0x52,0x7f,0x7b]
+          vpsrld $123, 508(%rdx){1to16}, %zmm3
+
+// CHECK: vpsrld $123, 512(%rdx){1to16}, %zmm3
+// CHECK:  encoding: [0x62,0xf1,0x65,0x58,0x72,0x92,0x00,0x02,0x00,0x00,0x7b]
+          vpsrld $123, 512(%rdx){1to16}, %zmm3
+
+// CHECK: vpsrld $123, -512(%rdx){1to16}, %zmm3
+// CHECK:  encoding: [0x62,0xf1,0x65,0x58,0x72,0x52,0x80,0x7b]
+          vpsrld $123, -512(%rdx){1to16}, %zmm3
+
+// CHECK: vpsrld $123, -516(%rdx){1to16}, %zmm3
+// CHECK:  encoding: [0x62,0xf1,0x65,0x58,0x72,0x92,0xfc,0xfd,0xff,0xff,0x7b]
+          vpsrld $123, -516(%rdx){1to16}, %zmm3
+
+// CHECK: vpsrlq $171, %zmm25, %zmm18
+// CHECK:  encoding: [0x62,0x91,0xed,0x40,0x73,0xd1,0xab]
+          vpsrlq $171, %zmm25, %zmm18
+
+// CHECK: vpsrlq $171, %zmm25, %zmm18 {%k1}
+// CHECK:  encoding: [0x62,0x91,0xed,0x41,0x73,0xd1,0xab]
+          vpsrlq $171, %zmm25, %zmm18 {%k1}
+
+// CHECK: vpsrlq $171, %zmm25, %zmm18 {%k1} {z}
+// CHECK:  encoding: [0x62,0x91,0xed,0xc1,0x73,0xd1,0xab]
+          vpsrlq $171, %zmm25, %zmm18 {%k1} {z}
+
+// CHECK: vpsrlq $123, %zmm25, %zmm18
+// CHECK:  encoding: [0x62,0x91,0xed,0x40,0x73,0xd1,0x7b]
+          vpsrlq $123, %zmm25, %zmm18
+
+// CHECK: vpsrlq $123, (%rcx), %zmm18
+// CHECK:  encoding: [0x62,0xf1,0xed,0x40,0x73,0x11,0x7b]
+          vpsrlq $123, (%rcx), %zmm18
+
+// CHECK: vpsrlq $123, 291(%rax,%r14,8), %zmm18
+// CHECK:  encoding: [0x62,0xb1,0xed,0x40,0x73,0x94,0xf0,0x23,0x01,0x00,0x00,0x7b]
+          vpsrlq $123, 291(%rax,%r14,8), %zmm18
+
+// CHECK: vpsrlq $123, (%rcx){1to8}, %zmm18
+// CHECK:  encoding: [0x62,0xf1,0xed,0x50,0x73,0x11,0x7b]
+          vpsrlq $123, (%rcx){1to8}, %zmm18
+
+// CHECK: vpsrlq $123, 8128(%rdx), %zmm18
+// CHECK:  encoding: [0x62,0xf1,0xed,0x40,0x73,0x52,0x7f,0x7b]
+          vpsrlq $123, 8128(%rdx), %zmm18
+
+// CHECK: vpsrlq $123, 8192(%rdx), %zmm18
+// CHECK:  encoding: [0x62,0xf1,0xed,0x40,0x73,0x92,0x00,0x20,0x00,0x00,0x7b]
+          vpsrlq $123, 8192(%rdx), %zmm18
+
+// CHECK: vpsrlq $123, -8192(%rdx), %zmm18
+// CHECK:  encoding: [0x62,0xf1,0xed,0x40,0x73,0x52,0x80,0x7b]
+          vpsrlq $123, -8192(%rdx), %zmm18
+
+// CHECK: vpsrlq $123, -8256(%rdx), %zmm18
+// CHECK:  encoding: [0x62,0xf1,0xed,0x40,0x73,0x92,0xc0,0xdf,0xff,0xff,0x7b]
+          vpsrlq $123, -8256(%rdx), %zmm18
+
+// CHECK: vpsrlq $123, 1016(%rdx){1to8}, %zmm18
+// CHECK:  encoding: [0x62,0xf1,0xed,0x50,0x73,0x52,0x7f,0x7b]
+          vpsrlq $123, 1016(%rdx){1to8}, %zmm18
+
+// CHECK: vpsrlq $123, 1024(%rdx){1to8}, %zmm18
+// CHECK:  encoding: [0x62,0xf1,0xed,0x50,0x73,0x92,0x00,0x04,0x00,0x00,0x7b]
+          vpsrlq $123, 1024(%rdx){1to8}, %zmm18
+
+// CHECK: vpsrlq $123, -1024(%rdx){1to8}, %zmm18
+// CHECK:  encoding: [0x62,0xf1,0xed,0x50,0x73,0x52,0x80,0x7b]
+          vpsrlq $123, -1024(%rdx){1to8}, %zmm18
+
+// CHECK: vpsrlq $123, -1032(%rdx){1to8}, %zmm18
+// CHECK:  encoding: [0x62,0xf1,0xed,0x50,0x73,0x92,0xf8,0xfb,0xff,0xff,0x7b]
+          vpsrlq $123, -1032(%rdx){1to8}, %zmm18
+
+// CHECK: vpslld $171, %zmm24, %zmm29
+// CHECK:  encoding: [0x62,0x91,0x15,0x40,0x72,0xf0,0xab]
+          vpslld $171, %zmm24, %zmm29
+
+// CHECK: vpslld $171, %zmm24, %zmm29 {%k6}
+// CHECK:  encoding: [0x62,0x91,0x15,0x46,0x72,0xf0,0xab]
+          vpslld $171, %zmm24, %zmm29 {%k6}
+
+// CHECK: vpslld $171, %zmm24, %zmm29 {%k6} {z}
+// CHECK:  encoding: [0x62,0x91,0x15,0xc6,0x72,0xf0,0xab]
+          vpslld $171, %zmm24, %zmm29 {%k6} {z}
+
+// CHECK: vpslld $123, %zmm24, %zmm29
+// CHECK:  encoding: [0x62,0x91,0x15,0x40,0x72,0xf0,0x7b]
+          vpslld $123, %zmm24, %zmm29
+
+// CHECK: vpslld $123, (%rcx), %zmm29
+// CHECK:  encoding: [0x62,0xf1,0x15,0x40,0x72,0x31,0x7b]
+          vpslld $123, (%rcx), %zmm29
+
+// CHECK: vpslld $123, 291(%rax,%r14,8), %zmm29
+// CHECK:  encoding: [0x62,0xb1,0x15,0x40,0x72,0xb4,0xf0,0x23,0x01,0x00,0x00,0x7b]
+          vpslld $123, 291(%rax,%r14,8), %zmm29
+
+// CHECK: vpslld $123, (%rcx){1to16}, %zmm29
+// CHECK:  encoding: [0x62,0xf1,0x15,0x50,0x72,0x31,0x7b]
+          vpslld $123, (%rcx){1to16}, %zmm29
+
+// CHECK: vpslld $123, 8128(%rdx), %zmm29
+// CHECK:  encoding: [0x62,0xf1,0x15,0x40,0x72,0x72,0x7f,0x7b]
+          vpslld $123, 8128(%rdx), %zmm29
+
+// CHECK: vpslld $123, 8192(%rdx), %zmm29
+// CHECK:  encoding: [0x62,0xf1,0x15,0x40,0x72,0xb2,0x00,0x20,0x00,0x00,0x7b]
+          vpslld $123, 8192(%rdx), %zmm29
+
+// CHECK: vpslld $123, -8192(%rdx), %zmm29
+// CHECK:  encoding: [0x62,0xf1,0x15,0x40,0x72,0x72,0x80,0x7b]
+          vpslld $123, -8192(%rdx), %zmm29
+
+// CHECK: vpslld $123, -8256(%rdx), %zmm29
+// CHECK:  encoding: [0x62,0xf1,0x15,0x40,0x72,0xb2,0xc0,0xdf,0xff,0xff,0x7b]
+          vpslld $123, -8256(%rdx), %zmm29
+
+// CHECK: vpslld $123, 508(%rdx){1to16}, %zmm29
+// CHECK:  encoding: [0x62,0xf1,0x15,0x50,0x72,0x72,0x7f,0x7b]
+          vpslld $123, 508(%rdx){1to16}, %zmm29
+
+// CHECK: vpslld $123, 512(%rdx){1to16}, %zmm29
+// CHECK:  encoding: [0x62,0xf1,0x15,0x50,0x72,0xb2,0x00,0x02,0x00,0x00,0x7b]
+          vpslld $123, 512(%rdx){1to16}, %zmm29
+
+// CHECK: vpslld $123, -512(%rdx){1to16}, %zmm29
+// CHECK:  encoding: [0x62,0xf1,0x15,0x50,0x72,0x72,0x80,0x7b]
+          vpslld $123, -512(%rdx){1to16}, %zmm29
+
+// CHECK: vpslld $123, -516(%rdx){1to16}, %zmm29
+// CHECK:  encoding: [0x62,0xf1,0x15,0x50,0x72,0xb2,0xfc,0xfd,0xff,0xff,0x7b]
+          vpslld $123, -516(%rdx){1to16}, %zmm29
+
+// CHECK: vpsllq $171, %zmm11, %zmm6
+// CHECK:  encoding: [0x62,0xd1,0xcd,0x48,0x73,0xf3,0xab]
+          vpsllq $171, %zmm11, %zmm6
+
+// CHECK: vpsllq $171, %zmm11, %zmm6 {%k2}
+// CHECK:  encoding: [0x62,0xd1,0xcd,0x4a,0x73,0xf3,0xab]
+          vpsllq $171, %zmm11, %zmm6 {%k2}
+
+// CHECK: vpsllq $171, %zmm11, %zmm6 {%k2} {z}
+// CHECK:  encoding: [0x62,0xd1,0xcd,0xca,0x73,0xf3,0xab]
+          vpsllq $171, %zmm11, %zmm6 {%k2} {z}
+
+// CHECK: vpsllq $123, %zmm11, %zmm6
+// CHECK:  encoding: [0x62,0xd1,0xcd,0x48,0x73,0xf3,0x7b]
+          vpsllq $123, %zmm11, %zmm6
+
+// CHECK: vpsllq $123, (%rcx), %zmm6
+// CHECK:  encoding: [0x62,0xf1,0xcd,0x48,0x73,0x31,0x7b]
+          vpsllq $123, (%rcx), %zmm6
+
+// CHECK: vpsllq $123, 291(%rax,%r14,8), %zmm6
+// CHECK:  encoding: [0x62,0xb1,0xcd,0x48,0x73,0xb4,0xf0,0x23,0x01,0x00,0x00,0x7b]
+          vpsllq $123, 291(%rax,%r14,8), %zmm6
+
+// CHECK: vpsllq $123, (%rcx){1to8}, %zmm6
+// CHECK:  encoding: [0x62,0xf1,0xcd,0x58,0x73,0x31,0x7b]
+          vpsllq $123, (%rcx){1to8}, %zmm6
+
+// CHECK: vpsllq $123, 8128(%rdx), %zmm6
+// CHECK:  encoding: [0x62,0xf1,0xcd,0x48,0x73,0x72,0x7f,0x7b]
+          vpsllq $123, 8128(%rdx), %zmm6
+
+// CHECK: vpsllq $123, 8192(%rdx), %zmm6
+// CHECK:  encoding: [0x62,0xf1,0xcd,0x48,0x73,0xb2,0x00,0x20,0x00,0x00,0x7b]
+          vpsllq $123, 8192(%rdx), %zmm6
+
+// CHECK: vpsllq $123, -8192(%rdx), %zmm6
+// CHECK:  encoding: [0x62,0xf1,0xcd,0x48,0x73,0x72,0x80,0x7b]
+          vpsllq $123, -8192(%rdx), %zmm6
+
+// CHECK: vpsllq $123, -8256(%rdx), %zmm6
+// CHECK:  encoding: [0x62,0xf1,0xcd,0x48,0x73,0xb2,0xc0,0xdf,0xff,0xff,0x7b]
+          vpsllq $123, -8256(%rdx), %zmm6
+
+// CHECK: vpsllq $123, 1016(%rdx){1to8}, %zmm6
+// CHECK:  encoding: [0x62,0xf1,0xcd,0x58,0x73,0x72,0x7f,0x7b]
+          vpsllq $123, 1016(%rdx){1to8}, %zmm6
+
+// CHECK: vpsllq $123, 1024(%rdx){1to8}, %zmm6
+// CHECK:  encoding: [0x62,0xf1,0xcd,0x58,0x73,0xb2,0x00,0x04,0x00,0x00,0x7b]
+          vpsllq $123, 1024(%rdx){1to8}, %zmm6
+
+// CHECK: vpsllq $123, -1024(%rdx){1to8}, %zmm6
+// CHECK:  encoding: [0x62,0xf1,0xcd,0x58,0x73,0x72,0x80,0x7b]
+          vpsllq $123, -1024(%rdx){1to8}, %zmm6
+
+// CHECK: vpsllq $123, -1032(%rdx){1to8}, %zmm6
+// CHECK:  encoding: [0x62,0xf1,0xcd,0x58,0x73,0xb2,0xf8,0xfb,0xff,0xff,0x7b]
+          vpsllq $123, -1032(%rdx){1to8}, %zmm6
+
+// CHECK: vpsrad $171, %zmm5, %zmm27
+// CHECK:  encoding: [0x62,0xf1,0x25,0x40,0x72,0xe5,0xab]
+          vpsrad $171, %zmm5, %zmm27
+
+// CHECK: vpsrad $171, %zmm5, %zmm27 {%k5}
+// CHECK:  encoding: [0x62,0xf1,0x25,0x45,0x72,0xe5,0xab]
+          vpsrad $171, %zmm5, %zmm27 {%k5}
+
+// CHECK: vpsrad $171, %zmm5, %zmm27 {%k5} {z}
+// CHECK:  encoding: [0x62,0xf1,0x25,0xc5,0x72,0xe5,0xab]
+          vpsrad $171, %zmm5, %zmm27 {%k5} {z}
+
+// CHECK: vpsrad $123, %zmm5, %zmm27
+// CHECK:  encoding: [0x62,0xf1,0x25,0x40,0x72,0xe5,0x7b]
+          vpsrad $123, %zmm5, %zmm27
+
+// CHECK: vpsrad $123, (%rcx), %zmm27
+// CHECK:  encoding: [0x62,0xf1,0x25,0x40,0x72,0x21,0x7b]
+          vpsrad $123, (%rcx), %zmm27
+
+// CHECK: vpsrad $123, 291(%rax,%r14,8), %zmm27
+// CHECK:  encoding: [0x62,0xb1,0x25,0x40,0x72,0xa4,0xf0,0x23,0x01,0x00,0x00,0x7b]
+          vpsrad $123, 291(%rax,%r14,8), %zmm27
+
+// CHECK: vpsrad $123, (%rcx){1to16}, %zmm27
+// CHECK:  encoding: [0x62,0xf1,0x25,0x50,0x72,0x21,0x7b]
+          vpsrad $123, (%rcx){1to16}, %zmm27
+
+// CHECK: vpsrad $123, 8128(%rdx), %zmm27
+// CHECK:  encoding: [0x62,0xf1,0x25,0x40,0x72,0x62,0x7f,0x7b]
+          vpsrad $123, 8128(%rdx), %zmm27
+
+// CHECK: vpsrad $123, 8192(%rdx), %zmm27
+// CHECK:  encoding: [0x62,0xf1,0x25,0x40,0x72,0xa2,0x00,0x20,0x00,0x00,0x7b]
+          vpsrad $123, 8192(%rdx), %zmm27
+
+// CHECK: vpsrad $123, -8192(%rdx), %zmm27
+// CHECK:  encoding: [0x62,0xf1,0x25,0x40,0x72,0x62,0x80,0x7b]
+          vpsrad $123, -8192(%rdx), %zmm27
+
+// CHECK: vpsrad $123, -8256(%rdx), %zmm27
+// CHECK:  encoding: [0x62,0xf1,0x25,0x40,0x72,0xa2,0xc0,0xdf,0xff,0xff,0x7b]
+          vpsrad $123, -8256(%rdx), %zmm27
+
+// CHECK: vpsrad $123, 508(%rdx){1to16}, %zmm27
+// CHECK:  encoding: [0x62,0xf1,0x25,0x50,0x72,0x62,0x7f,0x7b]
+          vpsrad $123, 508(%rdx){1to16}, %zmm27
+
+// CHECK: vpsrad $123, 512(%rdx){1to16}, %zmm27
+// CHECK:  encoding: [0x62,0xf1,0x25,0x50,0x72,0xa2,0x00,0x02,0x00,0x00,0x7b]
+          vpsrad $123, 512(%rdx){1to16}, %zmm27
+
+// CHECK: vpsrad $123, -512(%rdx){1to16}, %zmm27
+// CHECK:  encoding: [0x62,0xf1,0x25,0x50,0x72,0x62,0x80,0x7b]
+          vpsrad $123, -512(%rdx){1to16}, %zmm27
+
+// CHECK: vpsrad $123, -516(%rdx){1to16}, %zmm27
+// CHECK:  encoding: [0x62,0xf1,0x25,0x50,0x72,0xa2,0xfc,0xfd,0xff,0xff,0x7b]
+          vpsrad $123, -516(%rdx){1to16}, %zmm27
+
+// CHECK: vpsraq $171, %zmm5, %zmm29
+// CHECK:  encoding: [0x62,0xf1,0x95,0x40,0x72,0xe5,0xab]
+          vpsraq $171, %zmm5, %zmm29
+
+// CHECK: vpsraq $171, %zmm5, %zmm29 {%k4}
+// CHECK:  encoding: [0x62,0xf1,0x95,0x44,0x72,0xe5,0xab]
+          vpsraq $171, %zmm5, %zmm29 {%k4}
+
+// CHECK: vpsraq $171, %zmm5, %zmm29 {%k4} {z}
+// CHECK:  encoding: [0x62,0xf1,0x95,0xc4,0x72,0xe5,0xab]
+          vpsraq $171, %zmm5, %zmm29 {%k4} {z}
+
+// CHECK: vpsraq $123, %zmm5, %zmm29
+// CHECK:  encoding: [0x62,0xf1,0x95,0x40,0x72,0xe5,0x7b]
+          vpsraq $123, %zmm5, %zmm29
+
+// CHECK: vpsraq $123, (%rcx), %zmm29
+// CHECK:  encoding: [0x62,0xf1,0x95,0x40,0x72,0x21,0x7b]
+          vpsraq $123, (%rcx), %zmm29
+
+// CHECK: vpsraq $123, 291(%rax,%r14,8), %zmm29
+// CHECK:  encoding: [0x62,0xb1,0x95,0x40,0x72,0xa4,0xf0,0x23,0x01,0x00,0x00,0x7b]
+          vpsraq $123, 291(%rax,%r14,8), %zmm29
+
+// CHECK: vpsraq $123, (%rcx){1to8}, %zmm29
+// CHECK:  encoding: [0x62,0xf1,0x95,0x50,0x72,0x21,0x7b]
+          vpsraq $123, (%rcx){1to8}, %zmm29
+
+// CHECK: vpsraq $123, 8128(%rdx), %zmm29
+// CHECK:  encoding: [0x62,0xf1,0x95,0x40,0x72,0x62,0x7f,0x7b]
+          vpsraq $123, 8128(%rdx), %zmm29
+
+// CHECK: vpsraq $123, 8192(%rdx), %zmm29
+// CHECK:  encoding: [0x62,0xf1,0x95,0x40,0x72,0xa2,0x00,0x20,0x00,0x00,0x7b]
+          vpsraq $123, 8192(%rdx), %zmm29
+
+// CHECK: vpsraq $123, -8192(%rdx), %zmm29
+// CHECK:  encoding: [0x62,0xf1,0x95,0x40,0x72,0x62,0x80,0x7b]
+          vpsraq $123, -8192(%rdx), %zmm29
+
+// CHECK: vpsraq $123, -8256(%rdx), %zmm29
+// CHECK:  encoding: [0x62,0xf1,0x95,0x40,0x72,0xa2,0xc0,0xdf,0xff,0xff,0x7b]
+          vpsraq $123, -8256(%rdx), %zmm29
+
+// CHECK: vpsraq $123, 1016(%rdx){1to8}, %zmm29
+// CHECK:  encoding: [0x62,0xf1,0x95,0x50,0x72,0x62,0x7f,0x7b]
+          vpsraq $123, 1016(%rdx){1to8}, %zmm29
+
+// CHECK: vpsraq $123, 1024(%rdx){1to8}, %zmm29
+// CHECK:  encoding: [0x62,0xf1,0x95,0x50,0x72,0xa2,0x00,0x04,0x00,0x00,0x7b]
+          vpsraq $123, 1024(%rdx){1to8}, %zmm29
+
+// CHECK: vpsraq $123, -1024(%rdx){1to8}, %zmm29
+// CHECK:  encoding: [0x62,0xf1,0x95,0x50,0x72,0x62,0x80,0x7b]
+          vpsraq $123, -1024(%rdx){1to8}, %zmm29
+
+// CHECK: vpsraq $123, -1032(%rdx){1to8}, %zmm29
+// CHECK:  encoding: [0x62,0xf1,0x95,0x50,0x72,0xa2,0xf8,0xfb,0xff,0xff,0x7b]
+          vpsraq $123, -1032(%rdx){1to8}, %zmm29
+
+// CHECK: vfixupimmps $171, %zmm2, %zmm26, %zmm15
+// CHECK:  encoding: [0x62,0x73,0x2d,0x40,0x54,0xfa,0xab]
+          vfixupimmps $0xab, %zmm2, %zmm26, %zmm15
+
+// CHECK: vfixupimmps $171, %zmm2, %zmm26, %zmm15 {%k4}
+// CHECK:  encoding: [0x62,0x73,0x2d,0x44,0x54,0xfa,0xab]
+          vfixupimmps $0xab, %zmm2, %zmm26, %zmm15 {%k4}
+
+// CHECK: vfixupimmps $171, %zmm2, %zmm26, %zmm15 {%k4} {z}
+// CHECK:  encoding: [0x62,0x73,0x2d,0xc4,0x54,0xfa,0xab]
+          vfixupimmps $0xab, %zmm2, %zmm26, %zmm15 {%k4} {z}
+
+// CHECK: vfixupimmps $171,{sae}, %zmm2, %zmm26, %zmm15
+// CHECK:  encoding: [0x62,0x73,0x2d,0x10,0x54,0xfa,0xab]
+          vfixupimmps $0xab,{sae}, %zmm2, %zmm26, %zmm15
+
+// CHECK: vfixupimmps $123, %zmm2, %zmm26, %zmm15
+// CHECK:  encoding: [0x62,0x73,0x2d,0x40,0x54,0xfa,0x7b]
+          vfixupimmps $0x7b, %zmm2, %zmm26, %zmm15
+
+// CHECK: vfixupimmps $123,{sae}, %zmm2, %zmm26, %zmm15
+// CHECK:  encoding: [0x62,0x73,0x2d,0x10,0x54,0xfa,0x7b]
+          vfixupimmps $0x7b,{sae}, %zmm2, %zmm26, %zmm15
+
+// CHECK: vfixupimmps $123, (%rcx), %zmm26, %zmm15
+// CHECK:  encoding: [0x62,0x73,0x2d,0x40,0x54,0x39,0x7b]
+          vfixupimmps $0x7b, (%rcx), %zmm26, %zmm15
+
+// CHECK: vfixupimmps $123, 291(%rax,%r14,8), %zmm26, %zmm15
+// CHECK:  encoding: [0x62,0x33,0x2d,0x40,0x54,0xbc,0xf0,0x23,0x01,0x00,0x00,0x7b]
+          vfixupimmps $0x7b, 291(%rax,%r14,8), %zmm26, %zmm15
+
+// CHECK: vfixupimmps $123, (%rcx){1to16}, %zmm26, %zmm15
+// CHECK:  encoding: [0x62,0x73,0x2d,0x50,0x54,0x39,0x7b]
+          vfixupimmps $0x7b, (%rcx){1to16}, %zmm26, %zmm15
+
+// CHECK: vfixupimmps $123, 8128(%rdx), %zmm26, %zmm15
+// CHECK:  encoding: [0x62,0x73,0x2d,0x40,0x54,0x7a,0x7f,0x7b]
+          vfixupimmps $0x7b, 8128(%rdx), %zmm26, %zmm15
+
+// CHECK: vfixupimmps $123, 8192(%rdx), %zmm26, %zmm15
+// CHECK:  encoding: [0x62,0x73,0x2d,0x40,0x54,0xba,0x00,0x20,0x00,0x00,0x7b]
+          vfixupimmps $0x7b, 8192(%rdx), %zmm26, %zmm15
+
+// CHECK: vfixupimmps $123, -8192(%rdx), %zmm26, %zmm15
+// CHECK:  encoding: [0x62,0x73,0x2d,0x40,0x54,0x7a,0x80,0x7b]
+          vfixupimmps $0x7b, -8192(%rdx), %zmm26, %zmm15
+
+// CHECK: vfixupimmps $123, -8256(%rdx), %zmm26, %zmm15
+// CHECK:  encoding: [0x62,0x73,0x2d,0x40,0x54,0xba,0xc0,0xdf,0xff,0xff,0x7b]
+          vfixupimmps $0x7b, -8256(%rdx), %zmm26, %zmm15
+
+// CHECK: vfixupimmps $123, 508(%rdx){1to16}, %zmm26, %zmm15
+// CHECK:  encoding: [0x62,0x73,0x2d,0x50,0x54,0x7a,0x7f,0x7b]
+          vfixupimmps $0x7b, 508(%rdx){1to16}, %zmm26, %zmm15
+
+// CHECK: vfixupimmps $123, 512(%rdx){1to16}, %zmm26, %zmm15
+// CHECK:  encoding: [0x62,0x73,0x2d,0x50,0x54,0xba,0x00,0x02,0x00,0x00,0x7b]
+          vfixupimmps $0x7b, 512(%rdx){1to16}, %zmm26, %zmm15
+
+// CHECK: vfixupimmps $123, -512(%rdx){1to16}, %zmm26, %zmm15
+// CHECK:  encoding: [0x62,0x73,0x2d,0x50,0x54,0x7a,0x80,0x7b]
+          vfixupimmps $0x7b, -512(%rdx){1to16}, %zmm26, %zmm15
+
+// CHECK: vfixupimmps $123, -516(%rdx){1to16}, %zmm26, %zmm15
+// CHECK:  encoding: [0x62,0x73,0x2d,0x50,0x54,0xba,0xfc,0xfd,0xff,0xff,0x7b]
+          vfixupimmps $0x7b, -516(%rdx){1to16}, %zmm26, %zmm15
+
+// CHECK: vfixupimmpd $171, %zmm19, %zmm21, %zmm9
+// CHECK:  encoding: [0x62,0x33,0xd5,0x40,0x54,0xcb,0xab]
+          vfixupimmpd $0xab, %zmm19, %zmm21, %zmm9
+
+// CHECK: vfixupimmpd $171, %zmm19, %zmm21, %zmm9 {%k2}
+// CHECK:  encoding: [0x62,0x33,0xd5,0x42,0x54,0xcb,0xab]
+          vfixupimmpd $0xab, %zmm19, %zmm21, %zmm9 {%k2}
+
+// CHECK: vfixupimmpd $171, %zmm19, %zmm21, %zmm9 {%k2} {z}
+// CHECK:  encoding: [0x62,0x33,0xd5,0xc2,0x54,0xcb,0xab]
+          vfixupimmpd $0xab, %zmm19, %zmm21, %zmm9 {%k2} {z}
+
+// CHECK: vfixupimmpd $171,{sae}, %zmm19, %zmm21, %zmm9
+// CHECK:  encoding: [0x62,0x33,0xd5,0x10,0x54,0xcb,0xab]
+          vfixupimmpd $0xab,{sae}, %zmm19, %zmm21, %zmm9
+
+// CHECK: vfixupimmpd $123, %zmm19, %zmm21, %zmm9
+// CHECK:  encoding: [0x62,0x33,0xd5,0x40,0x54,0xcb,0x7b]
+          vfixupimmpd $0x7b, %zmm19, %zmm21, %zmm9
+
+// CHECK: vfixupimmpd $123,{sae}, %zmm19, %zmm21, %zmm9
+// CHECK:  encoding: [0x62,0x33,0xd5,0x10,0x54,0xcb,0x7b]
+          vfixupimmpd $0x7b,{sae}, %zmm19, %zmm21, %zmm9
+
+// CHECK: vfixupimmpd $123, (%rcx), %zmm21, %zmm9
+// CHECK:  encoding: [0x62,0x73,0xd5,0x40,0x54,0x09,0x7b]
+          vfixupimmpd $0x7b, (%rcx), %zmm21, %zmm9
+
+// CHECK: vfixupimmpd $123, 291(%rax,%r14,8), %zmm21, %zmm9
+// CHECK:  encoding: [0x62,0x33,0xd5,0x40,0x54,0x8c,0xf0,0x23,0x01,0x00,0x00,0x7b]
+          vfixupimmpd $0x7b, 291(%rax,%r14,8), %zmm21, %zmm9
+
+// CHECK: vfixupimmpd $123, (%rcx){1to8}, %zmm21, %zmm9
+// CHECK:  encoding: [0x62,0x73,0xd5,0x50,0x54,0x09,0x7b]
+          vfixupimmpd $0x7b, (%rcx){1to8}, %zmm21, %zmm9
+
+// CHECK: vfixupimmpd $123, 8128(%rdx), %zmm21, %zmm9
+// CHECK:  encoding: [0x62,0x73,0xd5,0x40,0x54,0x4a,0x7f,0x7b]
+          vfixupimmpd $0x7b, 8128(%rdx), %zmm21, %zmm9
+
+// CHECK: vfixupimmpd $123, 8192(%rdx), %zmm21, %zmm9
+// CHECK:  encoding: [0x62,0x73,0xd5,0x40,0x54,0x8a,0x00,0x20,0x00,0x00,0x7b]
+          vfixupimmpd $0x7b, 8192(%rdx), %zmm21, %zmm9
+
+// CHECK: vfixupimmpd $123, -8192(%rdx), %zmm21, %zmm9
+// CHECK:  encoding: [0x62,0x73,0xd5,0x40,0x54,0x4a,0x80,0x7b]
+          vfixupimmpd $0x7b, -8192(%rdx), %zmm21, %zmm9
+
+// CHECK: vfixupimmpd $123, -8256(%rdx), %zmm21, %zmm9
+// CHECK:  encoding: [0x62,0x73,0xd5,0x40,0x54,0x8a,0xc0,0xdf,0xff,0xff,0x7b]
+          vfixupimmpd $0x7b, -8256(%rdx), %zmm21, %zmm9
+
+// CHECK: vfixupimmpd $123, 1016(%rdx){1to8}, %zmm21, %zmm9
+// CHECK:  encoding: [0x62,0x73,0xd5,0x50,0x54,0x4a,0x7f,0x7b]
+          vfixupimmpd $0x7b, 1016(%rdx){1to8}, %zmm21, %zmm9
+
+// CHECK: vfixupimmpd $123, 1024(%rdx){1to8}, %zmm21, %zmm9
+// CHECK:  encoding: [0x62,0x73,0xd5,0x50,0x54,0x8a,0x00,0x04,0x00,0x00,0x7b]
+          vfixupimmpd $0x7b, 1024(%rdx){1to8}, %zmm21, %zmm9
+
+// CHECK: vfixupimmpd $123, -1024(%rdx){1to8}, %zmm21, %zmm9
+// CHECK:  encoding: [0x62,0x73,0xd5,0x50,0x54,0x4a,0x80,0x7b]
+          vfixupimmpd $0x7b, -1024(%rdx){1to8}, %zmm21, %zmm9
+
+// CHECK: vfixupimmpd $123, -1032(%rdx){1to8}, %zmm21, %zmm9
+// CHECK:  encoding: [0x62,0x73,0xd5,0x50,0x54,0x8a,0xf8,0xfb,0xff,0xff,0x7b]
+          vfixupimmpd $0x7b, -1032(%rdx){1to8}, %zmm21, %zmm9
+
+// CHECK: vfixupimmss $171, %xmm28, %xmm18, %xmm15
+// CHECK:  encoding: [0x62,0x13,0x6d,0x00,0x55,0xfc,0xab]
+          vfixupimmss $0xab, %xmm28, %xmm18, %xmm15
+
+// CHECK: vfixupimmss $171, %xmm28, %xmm18, %xmm15 {%k5}
+// CHECK:  encoding: [0x62,0x13,0x6d,0x05,0x55,0xfc,0xab]
+          vfixupimmss $0xab, %xmm28, %xmm18, %xmm15 {%k5}
+
+// CHECK: vfixupimmss $171, %xmm28, %xmm18, %xmm15 {%k5} {z}
+// CHECK:  encoding: [0x62,0x13,0x6d,0x85,0x55,0xfc,0xab]
+          vfixupimmss $0xab, %xmm28, %xmm18, %xmm15 {%k5} {z}
+
+// CHECK: vfixupimmss $171,{sae}, %xmm28, %xmm18, %xmm15
+// CHECK:  encoding: [0x62,0x13,0x6d,0x10,0x55,0xfc,0xab]
+          vfixupimmss $0xab,{sae}, %xmm28, %xmm18, %xmm15
+
+// CHECK: vfixupimmss $123, %xmm28, %xmm18, %xmm15
+// CHECK:  encoding: [0x62,0x13,0x6d,0x00,0x55,0xfc,0x7b]
+          vfixupimmss $0x7b, %xmm28, %xmm18, %xmm15
+
+// CHECK: vfixupimmss $123,{sae}, %xmm28, %xmm18, %xmm15
+// CHECK:  encoding: [0x62,0x13,0x6d,0x10,0x55,0xfc,0x7b]
+          vfixupimmss $0x7b,{sae}, %xmm28, %xmm18, %xmm15
+
+// CHECK: vfixupimmss $123, (%rcx), %xmm18, %xmm15
+// CHECK:  encoding: [0x62,0x73,0x6d,0x00,0x55,0x39,0x7b]
+          vfixupimmss $0x7b, (%rcx), %xmm18, %xmm15
+
+// CHECK: vfixupimmss $123, 291(%rax,%r14,8), %xmm18, %xmm15
+// CHECK:  encoding: [0x62,0x33,0x6d,0x00,0x55,0xbc,0xf0,0x23,0x01,0x00,0x00,0x7b]
+          vfixupimmss $0x7b, 291(%rax,%r14,8), %xmm18, %xmm15
+
+// CHECK: vfixupimmss $123, 508(%rdx), %xmm18, %xmm15
+// CHECK:  encoding: [0x62,0x73,0x6d,0x00,0x55,0x7a,0x7f,0x7b]
+          vfixupimmss $0x7b, 508(%rdx), %xmm18, %xmm15
+
+// CHECK: vfixupimmss $123, 512(%rdx), %xmm18, %xmm15
+// CHECK:  encoding: [0x62,0x73,0x6d,0x00,0x55,0xba,0x00,0x02,0x00,0x00,0x7b]
+          vfixupimmss $0x7b, 512(%rdx), %xmm18, %xmm15
+
+// CHECK: vfixupimmss $123, -512(%rdx), %xmm18, %xmm15
+// CHECK:  encoding: [0x62,0x73,0x6d,0x00,0x55,0x7a,0x80,0x7b]
+          vfixupimmss $0x7b, -512(%rdx), %xmm18, %xmm15
+
+// CHECK: vfixupimmss $123, -516(%rdx), %xmm18, %xmm15
+// CHECK:  encoding: [0x62,0x73,0x6d,0x00,0x55,0xba,0xfc,0xfd,0xff,0xff,0x7b]
+          vfixupimmss $0x7b, -516(%rdx), %xmm18, %xmm15
+
+// CHECK: vfixupimmsd $171, %xmm5, %xmm26, %xmm13
+// CHECK:  encoding: [0x62,0x73,0xad,0x00,0x55,0xed,0xab]
+          vfixupimmsd $0xab, %xmm5, %xmm26, %xmm13
+
+// CHECK: vfixupimmsd $171, %xmm5, %xmm26, %xmm13 {%k6}
+// CHECK:  encoding: [0x62,0x73,0xad,0x06,0x55,0xed,0xab]
+          vfixupimmsd $0xab, %xmm5, %xmm26, %xmm13 {%k6}
+
+// CHECK: vfixupimmsd $171, %xmm5, %xmm26, %xmm13 {%k6} {z}
+// CHECK:  encoding: [0x62,0x73,0xad,0x86,0x55,0xed,0xab]
+          vfixupimmsd $0xab, %xmm5, %xmm26, %xmm13 {%k6} {z}
+
+// CHECK: vfixupimmsd $171,{sae}, %xmm5, %xmm26, %xmm13
+// CHECK:  encoding: [0x62,0x73,0xad,0x10,0x55,0xed,0xab]
+          vfixupimmsd $0xab,{sae}, %xmm5, %xmm26, %xmm13
+
+// CHECK: vfixupimmsd $123, %xmm5, %xmm26, %xmm13
+// CHECK:  encoding: [0x62,0x73,0xad,0x00,0x55,0xed,0x7b]
+          vfixupimmsd $0x7b, %xmm5, %xmm26, %xmm13
+
+// CHECK: vfixupimmsd $123,{sae}, %xmm5, %xmm26, %xmm13
+// CHECK:  encoding: [0x62,0x73,0xad,0x10,0x55,0xed,0x7b]
+          vfixupimmsd $0x7b,{sae}, %xmm5, %xmm26, %xmm13
+
+// CHECK: vfixupimmsd $123, (%rcx), %xmm26, %xmm13
+// CHECK:  encoding: [0x62,0x73,0xad,0x00,0x55,0x29,0x7b]
+          vfixupimmsd $0x7b, (%rcx), %xmm26, %xmm13
+
+// CHECK: vfixupimmsd $123, 291(%rax,%r14,8), %xmm26, %xmm13
+// CHECK:  encoding: [0x62,0x33,0xad,0x00,0x55,0xac,0xf0,0x23,0x01,0x00,0x00,0x7b]
+          vfixupimmsd $0x7b, 291(%rax,%r14,8), %xmm26, %xmm13
+
+// CHECK: vfixupimmsd $123, 1016(%rdx), %xmm26, %xmm13
+// CHECK:  encoding: [0x62,0x73,0xad,0x00,0x55,0x6a,0x7f,0x7b]
+          vfixupimmsd $0x7b, 1016(%rdx), %xmm26, %xmm13
+
+// CHECK: vfixupimmsd $123, 1024(%rdx), %xmm26, %xmm13
+// CHECK:  encoding: [0x62,0x73,0xad,0x00,0x55,0xaa,0x00,0x04,0x00,0x00,0x7b]
+          vfixupimmsd $0x7b, 1024(%rdx), %xmm26, %xmm13
+
+// CHECK: vfixupimmsd $123, -1024(%rdx), %xmm26, %xmm13
+// CHECK:  encoding: [0x62,0x73,0xad,0x00,0x55,0x6a,0x80,0x7b]
+          vfixupimmsd $0x7b, -1024(%rdx), %xmm26, %xmm13
+
+// CHECK: vfixupimmsd $123, -1032(%rdx), %xmm26, %xmm13
+// CHECK:  encoding: [0x62,0x73,0xad,0x00,0x55,0xaa,0xf8,0xfb,0xff,0xff,0x7b]
+          vfixupimmsd $0x7b, -1032(%rdx), %xmm26, %xmm13
+
+// CHECK: vpshufd $171, %zmm25, %zmm19
+// CHECK:  encoding: [0x62,0x81,0x7d,0x48,0x70,0xd9,0xab]
+          vpshufd $171, %zmm25, %zmm19
+
+// CHECK: vpshufd $171, %zmm25, %zmm19 {%k6}
+// CHECK:  encoding: [0x62,0x81,0x7d,0x4e,0x70,0xd9,0xab]
+          vpshufd $171, %zmm25, %zmm19 {%k6}
+
+// CHECK: vpshufd $171, %zmm25, %zmm19 {%k6} {z}
+// CHECK:  encoding: [0x62,0x81,0x7d,0xce,0x70,0xd9,0xab]
+          vpshufd $171, %zmm25, %zmm19 {%k6} {z}
+
+// CHECK: vpshufd $123, %zmm25, %zmm19
+// CHECK:  encoding: [0x62,0x81,0x7d,0x48,0x70,0xd9,0x7b]
+          vpshufd $123, %zmm25, %zmm19
+
+// CHECK: vpshufd $123, (%rcx), %zmm19
+// CHECK:  encoding: [0x62,0xe1,0x7d,0x48,0x70,0x19,0x7b]
+          vpshufd $123, (%rcx), %zmm19
+
+// CHECK: vpshufd $123, 291(%rax,%r14,8), %zmm19
+// CHECK:  encoding: [0x62,0xa1,0x7d,0x48,0x70,0x9c,0xf0,0x23,0x01,0x00,0x00,0x7b]
+          vpshufd $123, 291(%rax,%r14,8), %zmm19
+
+// CHECK: vpshufd $123, (%rcx){1to16}, %zmm19
+// CHECK:  encoding: [0x62,0xe1,0x7d,0x58,0x70,0x19,0x7b]
+          vpshufd $123, (%rcx){1to16}, %zmm19
+
+// CHECK: vpshufd $123, 8128(%rdx), %zmm19
+// CHECK:  encoding: [0x62,0xe1,0x7d,0x48,0x70,0x5a,0x7f,0x7b]
+          vpshufd $123, 8128(%rdx), %zmm19
+
+// CHECK: vpshufd $123, 8192(%rdx), %zmm19
+// CHECK:  encoding: [0x62,0xe1,0x7d,0x48,0x70,0x9a,0x00,0x20,0x00,0x00,0x7b]
+          vpshufd $123, 8192(%rdx), %zmm19
+
+// CHECK: vpshufd $123, -8192(%rdx), %zmm19
+// CHECK:  encoding: [0x62,0xe1,0x7d,0x48,0x70,0x5a,0x80,0x7b]
+          vpshufd $123, -8192(%rdx), %zmm19
+
+// CHECK: vpshufd $123, -8256(%rdx), %zmm19
+// CHECK:  encoding: [0x62,0xe1,0x7d,0x48,0x70,0x9a,0xc0,0xdf,0xff,0xff,0x7b]
+          vpshufd $123, -8256(%rdx), %zmm19
+
+// CHECK: vpshufd $123, 508(%rdx){1to16}, %zmm19
+// CHECK:  encoding: [0x62,0xe1,0x7d,0x58,0x70,0x5a,0x7f,0x7b]
+          vpshufd $123, 508(%rdx){1to16}, %zmm19
+
+// CHECK: vpshufd $123, 512(%rdx){1to16}, %zmm19
+// CHECK:  encoding: [0x62,0xe1,0x7d,0x58,0x70,0x9a,0x00,0x02,0x00,0x00,0x7b]
+          vpshufd $123, 512(%rdx){1to16}, %zmm19
+
+// CHECK: vpshufd $123, -512(%rdx){1to16}, %zmm19
+// CHECK:  encoding: [0x62,0xe1,0x7d,0x58,0x70,0x5a,0x80,0x7b]
+          vpshufd $123, -512(%rdx){1to16}, %zmm19
+
+// CHECK: vpshufd $123, -516(%rdx){1to16}, %zmm19
+// CHECK:  encoding: [0x62,0xe1,0x7d,0x58,0x70,0x9a,0xfc,0xfd,0xff,0xff,0x7b]
+          vpshufd $123, -516(%rdx){1to16}, %zmm19
+
+// CHECK: vgetexppd %zmm25, %zmm14
+// CHECK:  encoding: [0x62,0x12,0xfd,0x48,0x42,0xf1]
+          vgetexppd %zmm25, %zmm14
+
+// CHECK: vgetexppd %zmm25, %zmm14 {%k5}
+// CHECK:  encoding: [0x62,0x12,0xfd,0x4d,0x42,0xf1]
+          vgetexppd %zmm25, %zmm14 {%k5}
+
+// CHECK: vgetexppd %zmm25, %zmm14 {%k5} {z}
+// CHECK:  encoding: [0x62,0x12,0xfd,0xcd,0x42,0xf1]
+          vgetexppd %zmm25, %zmm14 {%k5} {z}
+
+// CHECK: vgetexppd {sae}, %zmm25, %zmm14
+// CHECK:  encoding: [0x62,0x12,0xfd,0x18,0x42,0xf1]
+          vgetexppd {sae}, %zmm25, %zmm14
+
+// CHECK: vgetexppd (%rcx), %zmm14
+// CHECK:  encoding: [0x62,0x72,0xfd,0x48,0x42,0x31]
+          vgetexppd (%rcx), %zmm14
+
+// CHECK: vgetexppd 291(%rax,%r14,8), %zmm14
+// CHECK:  encoding: [0x62,0x32,0xfd,0x48,0x42,0xb4,0xf0,0x23,0x01,0x00,0x00]
+          vgetexppd 291(%rax,%r14,8), %zmm14
+
+// CHECK: vgetexppd (%rcx){1to8}, %zmm14
+// CHECK:  encoding: [0x62,0x72,0xfd,0x58,0x42,0x31]
+          vgetexppd (%rcx){1to8}, %zmm14
+
+// CHECK: vgetexppd 8128(%rdx), %zmm14
+// CHECK:  encoding: [0x62,0x72,0xfd,0x48,0x42,0x72,0x7f]
+          vgetexppd 8128(%rdx), %zmm14
+
+// CHECK: vgetexppd 8192(%rdx), %zmm14
+// CHECK:  encoding: [0x62,0x72,0xfd,0x48,0x42,0xb2,0x00,0x20,0x00,0x00]
+          vgetexppd 8192(%rdx), %zmm14
+
+// CHECK: vgetexppd -8192(%rdx), %zmm14
+// CHECK:  encoding: [0x62,0x72,0xfd,0x48,0x42,0x72,0x80]
+          vgetexppd -8192(%rdx), %zmm14
+
+// CHECK: vgetexppd -8256(%rdx), %zmm14
+// CHECK:  encoding: [0x62,0x72,0xfd,0x48,0x42,0xb2,0xc0,0xdf,0xff,0xff]
+          vgetexppd -8256(%rdx), %zmm14
+
+// CHECK: vgetexppd 1016(%rdx){1to8}, %zmm14
+// CHECK:  encoding: [0x62,0x72,0xfd,0x58,0x42,0x72,0x7f]
+          vgetexppd 1016(%rdx){1to8}, %zmm14
+
+// CHECK: vgetexppd 1024(%rdx){1to8}, %zmm14
+// CHECK:  encoding: [0x62,0x72,0xfd,0x58,0x42,0xb2,0x00,0x04,0x00,0x00]
+          vgetexppd 1024(%rdx){1to8}, %zmm14
+
+// CHECK: vgetexppd -1024(%rdx){1to8}, %zmm14
+// CHECK:  encoding: [0x62,0x72,0xfd,0x58,0x42,0x72,0x80]
+          vgetexppd -1024(%rdx){1to8}, %zmm14
+
+// CHECK: vgetexppd -1032(%rdx){1to8}, %zmm14
+// CHECK:  encoding: [0x62,0x72,0xfd,0x58,0x42,0xb2,0xf8,0xfb,0xff,0xff]
+          vgetexppd -1032(%rdx){1to8}, %zmm14
+
+// CHECK: vgetexpps %zmm6, %zmm1
+// CHECK:  encoding: [0x62,0xf2,0x7d,0x48,0x42,0xce]
+          vgetexpps %zmm6, %zmm1
+
+// CHECK: vgetexpps %zmm6, %zmm1 {%k3}
+// CHECK:  encoding: [0x62,0xf2,0x7d,0x4b,0x42,0xce]
+          vgetexpps %zmm6, %zmm1 {%k3}
+
+// CHECK: vgetexpps %zmm6, %zmm1 {%k3} {z}
+// CHECK:  encoding: [0x62,0xf2,0x7d,0xcb,0x42,0xce]
+          vgetexpps %zmm6, %zmm1 {%k3} {z}
+
+// CHECK: vgetexpps {sae}, %zmm6, %zmm1
+// CHECK:  encoding: [0x62,0xf2,0x7d,0x18,0x42,0xce]
+          vgetexpps {sae}, %zmm6, %zmm1
+
+// CHECK: vgetexpps (%rcx), %zmm1
+// CHECK:  encoding: [0x62,0xf2,0x7d,0x48,0x42,0x09]
+          vgetexpps (%rcx), %zmm1
+
+// CHECK: vgetexpps 291(%rax,%r14,8), %zmm1
+// CHECK:  encoding: [0x62,0xb2,0x7d,0x48,0x42,0x8c,0xf0,0x23,0x01,0x00,0x00]
+          vgetexpps 291(%rax,%r14,8), %zmm1
+
+// CHECK: vgetexpps (%rcx){1to16}, %zmm1
+// CHECK:  encoding: [0x62,0xf2,0x7d,0x58,0x42,0x09]
+          vgetexpps (%rcx){1to16}, %zmm1
+
+// CHECK: vgetexpps 8128(%rdx), %zmm1
+// CHECK:  encoding: [0x62,0xf2,0x7d,0x48,0x42,0x4a,0x7f]
+          vgetexpps 8128(%rdx), %zmm1
+
+// CHECK: vgetexpps 8192(%rdx), %zmm1
+// CHECK:  encoding: [0x62,0xf2,0x7d,0x48,0x42,0x8a,0x00,0x20,0x00,0x00]
+          vgetexpps 8192(%rdx), %zmm1
+
+// CHECK: vgetexpps -8192(%rdx), %zmm1
+// CHECK:  encoding: [0x62,0xf2,0x7d,0x48,0x42,0x4a,0x80]
+          vgetexpps -8192(%rdx), %zmm1
+
+// CHECK: vgetexpps -8256(%rdx), %zmm1
+// CHECK:  encoding: [0x62,0xf2,0x7d,0x48,0x42,0x8a,0xc0,0xdf,0xff,0xff]
+          vgetexpps -8256(%rdx), %zmm1
+
+// CHECK: vgetexpps 508(%rdx){1to16}, %zmm1
+// CHECK:  encoding: [0x62,0xf2,0x7d,0x58,0x42,0x4a,0x7f]
+          vgetexpps 508(%rdx){1to16}, %zmm1
+
+// CHECK: vgetexpps 512(%rdx){1to16}, %zmm1
+// CHECK:  encoding: [0x62,0xf2,0x7d,0x58,0x42,0x8a,0x00,0x02,0x00,0x00]
+          vgetexpps 512(%rdx){1to16}, %zmm1
+
+// CHECK: vgetexpps -512(%rdx){1to16}, %zmm1
+// CHECK:  encoding: [0x62,0xf2,0x7d,0x58,0x42,0x4a,0x80]
+          vgetexpps -512(%rdx){1to16}, %zmm1
+
+// CHECK: vgetexpps -516(%rdx){1to16}, %zmm1
+// CHECK:  encoding: [0x62,0xf2,0x7d,0x58,0x42,0x8a,0xfc,0xfd,0xff,0xff]
+          vgetexpps -516(%rdx){1to16}, %zmm1
+
+// CHECK: vshuff32x4 $171, %zmm3, %zmm24, %zmm6
+// CHECK:  encoding: [0x62,0xf3,0x3d,0x40,0x23,0xf3,0xab]
+          vshuff32x4 $171, %zmm3, %zmm24, %zmm6
+
+// CHECK: vshuff32x4 $171, %zmm3, %zmm24, %zmm6 {%k2}
+// CHECK:  encoding: [0x62,0xf3,0x3d,0x42,0x23,0xf3,0xab]
+          vshuff32x4 $171, %zmm3, %zmm24, %zmm6 {%k2}
+
+// CHECK: vshuff32x4 $171, %zmm3, %zmm24, %zmm6 {%k2} {z}
+// CHECK:  encoding: [0x62,0xf3,0x3d,0xc2,0x23,0xf3,0xab]
+          vshuff32x4 $171, %zmm3, %zmm24, %zmm6 {%k2} {z}
+
+// CHECK: vshuff32x4 $123, %zmm3, %zmm24, %zmm6
+// CHECK:  encoding: [0x62,0xf3,0x3d,0x40,0x23,0xf3,0x7b]
+          vshuff32x4 $123, %zmm3, %zmm24, %zmm6
+
+// CHECK: vshuff32x4 $123, (%rcx), %zmm24, %zmm6
+// CHECK:  encoding: [0x62,0xf3,0x3d,0x40,0x23,0x31,0x7b]
+          vshuff32x4 $123, (%rcx), %zmm24, %zmm6
+
+// CHECK: vshuff32x4 $123, 291(%rax,%r14,8), %zmm24, %zmm6
+// CHECK:  encoding: [0x62,0xb3,0x3d,0x40,0x23,0xb4,0xf0,0x23,0x01,0x00,0x00,0x7b]
+          vshuff32x4 $123, 291(%rax,%r14,8), %zmm24, %zmm6
+
+// CHECK: vshuff32x4 $123, (%rcx){1to16}, %zmm24, %zmm6
+// CHECK:  encoding: [0x62,0xf3,0x3d,0x50,0x23,0x31,0x7b]
+          vshuff32x4 $123, (%rcx){1to16}, %zmm24, %zmm6
+
+// CHECK: vshuff32x4 $123, 8128(%rdx), %zmm24, %zmm6
+// CHECK:  encoding: [0x62,0xf3,0x3d,0x40,0x23,0x72,0x7f,0x7b]
+          vshuff32x4 $123, 8128(%rdx), %zmm24, %zmm6
+
+// CHECK: vshuff32x4 $123, 8192(%rdx), %zmm24, %zmm6
+// CHECK:  encoding: [0x62,0xf3,0x3d,0x40,0x23,0xb2,0x00,0x20,0x00,0x00,0x7b]
+          vshuff32x4 $123, 8192(%rdx), %zmm24, %zmm6
+
+// CHECK: vshuff32x4 $123, -8192(%rdx), %zmm24, %zmm6
+// CHECK:  encoding: [0x62,0xf3,0x3d,0x40,0x23,0x72,0x80,0x7b]
+          vshuff32x4 $123, -8192(%rdx), %zmm24, %zmm6
+
+// CHECK: vshuff32x4 $123, -8256(%rdx), %zmm24, %zmm6
+// CHECK:  encoding: [0x62,0xf3,0x3d,0x40,0x23,0xb2,0xc0,0xdf,0xff,0xff,0x7b]
+          vshuff32x4 $123, -8256(%rdx), %zmm24, %zmm6
+
+// CHECK: vshuff32x4 $123, 508(%rdx){1to16}, %zmm24, %zmm6
+// CHECK:  encoding: [0x62,0xf3,0x3d,0x50,0x23,0x72,0x7f,0x7b]
+          vshuff32x4 $123, 508(%rdx){1to16}, %zmm24, %zmm6
+
+// CHECK: vshuff32x4 $123, 512(%rdx){1to16}, %zmm24, %zmm6
+// CHECK:  encoding: [0x62,0xf3,0x3d,0x50,0x23,0xb2,0x00,0x02,0x00,0x00,0x7b]
+          vshuff32x4 $123, 512(%rdx){1to16}, %zmm24, %zmm6
+
+// CHECK: vshuff32x4 $123, -512(%rdx){1to16}, %zmm24, %zmm6
+// CHECK:  encoding: [0x62,0xf3,0x3d,0x50,0x23,0x72,0x80,0x7b]
+          vshuff32x4 $123, -512(%rdx){1to16}, %zmm24, %zmm6
+
+// CHECK: vshuff32x4 $123, -516(%rdx){1to16}, %zmm24, %zmm6
+// CHECK:  encoding: [0x62,0xf3,0x3d,0x50,0x23,0xb2,0xfc,0xfd,0xff,0xff,0x7b]
+          vshuff32x4 $123, -516(%rdx){1to16}, %zmm24, %zmm6
+
+// CHECK: vshuff64x2 $171, %zmm11, %zmm25, %zmm15
+// CHECK:  encoding: [0x62,0x53,0xb5,0x40,0x23,0xfb,0xab]
+          vshuff64x2 $171, %zmm11, %zmm25, %zmm15
+
+// CHECK: vshuff64x2 $171, %zmm11, %zmm25, %zmm15 {%k2}
+// CHECK:  encoding: [0x62,0x53,0xb5,0x42,0x23,0xfb,0xab]
+          vshuff64x2 $171, %zmm11, %zmm25, %zmm15 {%k2}
+
+// CHECK: vshuff64x2 $171, %zmm11, %zmm25, %zmm15 {%k2} {z}
+// CHECK:  encoding: [0x62,0x53,0xb5,0xc2,0x23,0xfb,0xab]
+          vshuff64x2 $171, %zmm11, %zmm25, %zmm15 {%k2} {z}
+
+// CHECK: vshuff64x2 $123, %zmm11, %zmm25, %zmm15
+// CHECK:  encoding: [0x62,0x53,0xb5,0x40,0x23,0xfb,0x7b]
+          vshuff64x2 $123, %zmm11, %zmm25, %zmm15
+
+// CHECK: vshuff64x2 $123, (%rcx), %zmm25, %zmm15
+// CHECK:  encoding: [0x62,0x73,0xb5,0x40,0x23,0x39,0x7b]
+          vshuff64x2 $123, (%rcx), %zmm25, %zmm15
+
+// CHECK: vshuff64x2 $123, 291(%rax,%r14,8), %zmm25, %zmm15
+// CHECK:  encoding: [0x62,0x33,0xb5,0x40,0x23,0xbc,0xf0,0x23,0x01,0x00,0x00,0x7b]
+          vshuff64x2 $123, 291(%rax,%r14,8), %zmm25, %zmm15
+
+// CHECK: vshuff64x2 $123, (%rcx){1to8}, %zmm25, %zmm15
+// CHECK:  encoding: [0x62,0x73,0xb5,0x50,0x23,0x39,0x7b]
+          vshuff64x2 $123, (%rcx){1to8}, %zmm25, %zmm15
+
+// CHECK: vshuff64x2 $123, 8128(%rdx), %zmm25, %zmm15
+// CHECK:  encoding: [0x62,0x73,0xb5,0x40,0x23,0x7a,0x7f,0x7b]
+          vshuff64x2 $123, 8128(%rdx), %zmm25, %zmm15
+
+// CHECK: vshuff64x2 $123, 8192(%rdx), %zmm25, %zmm15
+// CHECK:  encoding: [0x62,0x73,0xb5,0x40,0x23,0xba,0x00,0x20,0x00,0x00,0x7b]
+          vshuff64x2 $123, 8192(%rdx), %zmm25, %zmm15
+
+// CHECK: vshuff64x2 $123, -8192(%rdx), %zmm25, %zmm15
+// CHECK:  encoding: [0x62,0x73,0xb5,0x40,0x23,0x7a,0x80,0x7b]
+          vshuff64x2 $123, -8192(%rdx), %zmm25, %zmm15
+
+// CHECK: vshuff64x2 $123, -8256(%rdx), %zmm25, %zmm15
+// CHECK:  encoding: [0x62,0x73,0xb5,0x40,0x23,0xba,0xc0,0xdf,0xff,0xff,0x7b]
+          vshuff64x2 $123, -8256(%rdx), %zmm25, %zmm15
+
+// CHECK: vshuff64x2 $123, 1016(%rdx){1to8}, %zmm25, %zmm15
+// CHECK:  encoding: [0x62,0x73,0xb5,0x50,0x23,0x7a,0x7f,0x7b]
+          vshuff64x2 $123, 1016(%rdx){1to8}, %zmm25, %zmm15
+
+// CHECK: vshuff64x2 $123, 1024(%rdx){1to8}, %zmm25, %zmm15
+// CHECK:  encoding: [0x62,0x73,0xb5,0x50,0x23,0xba,0x00,0x04,0x00,0x00,0x7b]
+          vshuff64x2 $123, 1024(%rdx){1to8}, %zmm25, %zmm15
+
+// CHECK: vshuff64x2 $123, -1024(%rdx){1to8}, %zmm25, %zmm15
+// CHECK:  encoding: [0x62,0x73,0xb5,0x50,0x23,0x7a,0x80,0x7b]
+          vshuff64x2 $123, -1024(%rdx){1to8}, %zmm25, %zmm15
+
+// CHECK: vshuff64x2 $123, -1032(%rdx){1to8}, %zmm25, %zmm15
+// CHECK:  encoding: [0x62,0x73,0xb5,0x50,0x23,0xba,0xf8,0xfb,0xff,0xff,0x7b]
+          vshuff64x2 $123, -1032(%rdx){1to8}, %zmm25, %zmm15
+
+// CHECK: vshufi32x4 $171, %zmm25, %zmm28, %zmm1
+// CHECK:  encoding: [0x62,0x93,0x1d,0x40,0x43,0xc9,0xab]
+          vshufi32x4 $171, %zmm25, %zmm28, %zmm1
+
+// CHECK: vshufi32x4 $171, %zmm25, %zmm28, %zmm1 {%k4}
+// CHECK:  encoding: [0x62,0x93,0x1d,0x44,0x43,0xc9,0xab]
+          vshufi32x4 $171, %zmm25, %zmm28, %zmm1 {%k4}
+
+// CHECK: vshufi32x4 $171, %zmm25, %zmm28, %zmm1 {%k4} {z}
+// CHECK:  encoding: [0x62,0x93,0x1d,0xc4,0x43,0xc9,0xab]
+          vshufi32x4 $171, %zmm25, %zmm28, %zmm1 {%k4} {z}
+
+// CHECK: vshufi32x4 $123, %zmm25, %zmm28, %zmm1
+// CHECK:  encoding: [0x62,0x93,0x1d,0x40,0x43,0xc9,0x7b]
+          vshufi32x4 $123, %zmm25, %zmm28, %zmm1
+
+// CHECK: vshufi32x4 $123, (%rcx), %zmm28, %zmm1
+// CHECK:  encoding: [0x62,0xf3,0x1d,0x40,0x43,0x09,0x7b]
+          vshufi32x4 $123, (%rcx), %zmm28, %zmm1
+
+// CHECK: vshufi32x4 $123, 291(%rax,%r14,8), %zmm28, %zmm1
+// CHECK:  encoding: [0x62,0xb3,0x1d,0x40,0x43,0x8c,0xf0,0x23,0x01,0x00,0x00,0x7b]
+          vshufi32x4 $123, 291(%rax,%r14,8), %zmm28, %zmm1
+
+// CHECK: vshufi32x4 $123, (%rcx){1to16}, %zmm28, %zmm1
+// CHECK:  encoding: [0x62,0xf3,0x1d,0x50,0x43,0x09,0x7b]
+          vshufi32x4 $123, (%rcx){1to16}, %zmm28, %zmm1
+
+// CHECK: vshufi32x4 $123, 8128(%rdx), %zmm28, %zmm1
+// CHECK:  encoding: [0x62,0xf3,0x1d,0x40,0x43,0x4a,0x7f,0x7b]
+          vshufi32x4 $123, 8128(%rdx), %zmm28, %zmm1
+
+// CHECK: vshufi32x4 $123, 8192(%rdx), %zmm28, %zmm1
+// CHECK:  encoding: [0x62,0xf3,0x1d,0x40,0x43,0x8a,0x00,0x20,0x00,0x00,0x7b]
+          vshufi32x4 $123, 8192(%rdx), %zmm28, %zmm1
+
+// CHECK: vshufi32x4 $123, -8192(%rdx), %zmm28, %zmm1
+// CHECK:  encoding: [0x62,0xf3,0x1d,0x40,0x43,0x4a,0x80,0x7b]
+          vshufi32x4 $123, -8192(%rdx), %zmm28, %zmm1
+
+// CHECK: vshufi32x4 $123, -8256(%rdx), %zmm28, %zmm1
+// CHECK:  encoding: [0x62,0xf3,0x1d,0x40,0x43,0x8a,0xc0,0xdf,0xff,0xff,0x7b]
+          vshufi32x4 $123, -8256(%rdx), %zmm28, %zmm1
+
+// CHECK: vshufi32x4 $123, 508(%rdx){1to16}, %zmm28, %zmm1
+// CHECK:  encoding: [0x62,0xf3,0x1d,0x50,0x43,0x4a,0x7f,0x7b]
+          vshufi32x4 $123, 508(%rdx){1to16}, %zmm28, %zmm1
+
+// CHECK: vshufi32x4 $123, 512(%rdx){1to16}, %zmm28, %zmm1
+// CHECK:  encoding: [0x62,0xf3,0x1d,0x50,0x43,0x8a,0x00,0x02,0x00,0x00,0x7b]
+          vshufi32x4 $123, 512(%rdx){1to16}, %zmm28, %zmm1
+
+// CHECK: vshufi32x4 $123, -512(%rdx){1to16}, %zmm28, %zmm1
+// CHECK:  encoding: [0x62,0xf3,0x1d,0x50,0x43,0x4a,0x80,0x7b]
+          vshufi32x4 $123, -512(%rdx){1to16}, %zmm28, %zmm1
+
+// CHECK: vshufi32x4 $123, -516(%rdx){1to16}, %zmm28, %zmm1
+// CHECK:  encoding: [0x62,0xf3,0x1d,0x50,0x43,0x8a,0xfc,0xfd,0xff,0xff,0x7b]
+          vshufi32x4 $123, -516(%rdx){1to16}, %zmm28, %zmm1
+
+// CHECK: vshufi64x2 $171, %zmm19, %zmm16, %zmm3
+// CHECK:  encoding: [0x62,0xb3,0xfd,0x40,0x43,0xdb,0xab]
+          vshufi64x2 $171, %zmm19, %zmm16, %zmm3
+
+// CHECK: vshufi64x2 $171, %zmm19, %zmm16, %zmm3 {%k7}
+// CHECK:  encoding: [0x62,0xb3,0xfd,0x47,0x43,0xdb,0xab]
+          vshufi64x2 $171, %zmm19, %zmm16, %zmm3 {%k7}
+
+// CHECK: vshufi64x2 $171, %zmm19, %zmm16, %zmm3 {%k7} {z}
+// CHECK:  encoding: [0x62,0xb3,0xfd,0xc7,0x43,0xdb,0xab]
+          vshufi64x2 $171, %zmm19, %zmm16, %zmm3 {%k7} {z}
+
+// CHECK: vshufi64x2 $123, %zmm19, %zmm16, %zmm3
+// CHECK:  encoding: [0x62,0xb3,0xfd,0x40,0x43,0xdb,0x7b]
+          vshufi64x2 $123, %zmm19, %zmm16, %zmm3
+
+// CHECK: vshufi64x2 $123, (%rcx), %zmm16, %zmm3
+// CHECK:  encoding: [0x62,0xf3,0xfd,0x40,0x43,0x19,0x7b]
+          vshufi64x2 $123, (%rcx), %zmm16, %zmm3
+
+// CHECK: vshufi64x2 $123, 291(%rax,%r14,8), %zmm16, %zmm3
+// CHECK:  encoding: [0x62,0xb3,0xfd,0x40,0x43,0x9c,0xf0,0x23,0x01,0x00,0x00,0x7b]
+          vshufi64x2 $123, 291(%rax,%r14,8), %zmm16, %zmm3
+
+// CHECK: vshufi64x2 $123, (%rcx){1to8}, %zmm16, %zmm3
+// CHECK:  encoding: [0x62,0xf3,0xfd,0x50,0x43,0x19,0x7b]
+          vshufi64x2 $123, (%rcx){1to8}, %zmm16, %zmm3
+
+// CHECK: vshufi64x2 $123, 8128(%rdx), %zmm16, %zmm3
+// CHECK:  encoding: [0x62,0xf3,0xfd,0x40,0x43,0x5a,0x7f,0x7b]
+          vshufi64x2 $123, 8128(%rdx), %zmm16, %zmm3
+
+// CHECK: vshufi64x2 $123, 8192(%rdx), %zmm16, %zmm3
+// CHECK:  encoding: [0x62,0xf3,0xfd,0x40,0x43,0x9a,0x00,0x20,0x00,0x00,0x7b]
+          vshufi64x2 $123, 8192(%rdx), %zmm16, %zmm3
+
+// CHECK: vshufi64x2 $123, -8192(%rdx), %zmm16, %zmm3
+// CHECK:  encoding: [0x62,0xf3,0xfd,0x40,0x43,0x5a,0x80,0x7b]
+          vshufi64x2 $123, -8192(%rdx), %zmm16, %zmm3
+
+// CHECK: vshufi64x2 $123, -8256(%rdx), %zmm16, %zmm3
+// CHECK:  encoding: [0x62,0xf3,0xfd,0x40,0x43,0x9a,0xc0,0xdf,0xff,0xff,0x7b]
+          vshufi64x2 $123, -8256(%rdx), %zmm16, %zmm3
+
+// CHECK: vshufi64x2 $123, 1016(%rdx){1to8}, %zmm16, %zmm3
+// CHECK:  encoding: [0x62,0xf3,0xfd,0x50,0x43,0x5a,0x7f,0x7b]
+          vshufi64x2 $123, 1016(%rdx){1to8}, %zmm16, %zmm3
+
+// CHECK: vshufi64x2 $123, 1024(%rdx){1to8}, %zmm16, %zmm3
+// CHECK:  encoding: [0x62,0xf3,0xfd,0x50,0x43,0x9a,0x00,0x04,0x00,0x00,0x7b]
+          vshufi64x2 $123, 1024(%rdx){1to8}, %zmm16, %zmm3
+
+// CHECK: vshufi64x2 $123, -1024(%rdx){1to8}, %zmm16, %zmm3
+// CHECK:  encoding: [0x62,0xf3,0xfd,0x50,0x43,0x5a,0x80,0x7b]
+          vshufi64x2 $123, -1024(%rdx){1to8}, %zmm16, %zmm3
+
+// CHECK: vshufi64x2 $123, -1032(%rdx){1to8}, %zmm16, %zmm3
+// CHECK:  encoding: [0x62,0xf3,0xfd,0x50,0x43,0x9a,0xf8,0xfb,0xff,0xff,0x7b]
+          vshufi64x2 $123, -1032(%rdx){1to8}, %zmm16, %zmm3
+
+// CHECK: vpermps %zmm24, %zmm2, %zmm4
+// CHECK:  encoding: [0x62,0x92,0x6d,0x48,0x16,0xe0]
+          vpermps %zmm24, %zmm2, %zmm4
+
+// CHECK: vpermps %zmm24, %zmm2, %zmm4 {%k4}
+// CHECK:  encoding: [0x62,0x92,0x6d,0x4c,0x16,0xe0]
+          vpermps %zmm24, %zmm2, %zmm4 {%k4}
+
+// CHECK: vpermps %zmm24, %zmm2, %zmm4 {%k4} {z}
+// CHECK:  encoding: [0x62,0x92,0x6d,0xcc,0x16,0xe0]
+          vpermps %zmm24, %zmm2, %zmm4 {%k4} {z}
+
+// CHECK: vpermps (%rcx), %zmm2, %zmm4
+// CHECK:  encoding: [0x62,0xf2,0x6d,0x48,0x16,0x21]
+          vpermps (%rcx), %zmm2, %zmm4
+
+// CHECK: vpermps 291(%rax,%r14,8), %zmm2, %zmm4
+// CHECK:  encoding: [0x62,0xb2,0x6d,0x48,0x16,0xa4,0xf0,0x23,0x01,0x00,0x00]
+          vpermps 291(%rax,%r14,8), %zmm2, %zmm4
+
+// CHECK: vpermps (%rcx){1to16}, %zmm2, %zmm4
+// CHECK:  encoding: [0x62,0xf2,0x6d,0x58,0x16,0x21]
+          vpermps (%rcx){1to16}, %zmm2, %zmm4
+
+// CHECK: vpermps 8128(%rdx), %zmm2, %zmm4
+// CHECK:  encoding: [0x62,0xf2,0x6d,0x48,0x16,0x62,0x7f]
+          vpermps 8128(%rdx), %zmm2, %zmm4
+
+// CHECK: vpermps 8192(%rdx), %zmm2, %zmm4
+// CHECK:  encoding: [0x62,0xf2,0x6d,0x48,0x16,0xa2,0x00,0x20,0x00,0x00]
+          vpermps 8192(%rdx), %zmm2, %zmm4
+
+// CHECK: vpermps -8192(%rdx), %zmm2, %zmm4
+// CHECK:  encoding: [0x62,0xf2,0x6d,0x48,0x16,0x62,0x80]
+          vpermps -8192(%rdx), %zmm2, %zmm4
+
+// CHECK: vpermps -8256(%rdx), %zmm2, %zmm4
+// CHECK:  encoding: [0x62,0xf2,0x6d,0x48,0x16,0xa2,0xc0,0xdf,0xff,0xff]
+          vpermps -8256(%rdx), %zmm2, %zmm4
+
+// CHECK: vpermps 508(%rdx){1to16}, %zmm2, %zmm4
+// CHECK:  encoding: [0x62,0xf2,0x6d,0x58,0x16,0x62,0x7f]
+          vpermps 508(%rdx){1to16}, %zmm2, %zmm4
+
+// CHECK: vpermps 512(%rdx){1to16}, %zmm2, %zmm4
+// CHECK:  encoding: [0x62,0xf2,0x6d,0x58,0x16,0xa2,0x00,0x02,0x00,0x00]
+          vpermps 512(%rdx){1to16}, %zmm2, %zmm4
+
+// CHECK: vpermps -512(%rdx){1to16}, %zmm2, %zmm4
+// CHECK:  encoding: [0x62,0xf2,0x6d,0x58,0x16,0x62,0x80]
+          vpermps -512(%rdx){1to16}, %zmm2, %zmm4
+
+// CHECK: vpermps -516(%rdx){1to16}, %zmm2, %zmm4
+// CHECK:  encoding: [0x62,0xf2,0x6d,0x58,0x16,0xa2,0xfc,0xfd,0xff,0xff]
+          vpermps -516(%rdx){1to16}, %zmm2, %zmm4
+
+// CHECK: vpermq $171, %zmm4, %zmm25
+// CHECK:  encoding: [0x62,0x63,0xfd,0x48,0x00,0xcc,0xab]
+          vpermq $171, %zmm4, %zmm25
+
+// CHECK: vpermq $171, %zmm4, %zmm25 {%k6}
+// CHECK:  encoding: [0x62,0x63,0xfd,0x4e,0x00,0xcc,0xab]
+          vpermq $171, %zmm4, %zmm25 {%k6}
+
+// CHECK: vpermq $171, %zmm4, %zmm25 {%k6} {z}
+// CHECK:  encoding: [0x62,0x63,0xfd,0xce,0x00,0xcc,0xab]
+          vpermq $171, %zmm4, %zmm25 {%k6} {z}
+
+// CHECK: vpermq $123, %zmm4, %zmm25
+// CHECK:  encoding: [0x62,0x63,0xfd,0x48,0x00,0xcc,0x7b]
+          vpermq $123, %zmm4, %zmm25
+
+// CHECK: vpermq $123, (%rcx), %zmm25
+// CHECK:  encoding: [0x62,0x63,0xfd,0x48,0x00,0x09,0x7b]
+          vpermq $123, (%rcx), %zmm25
+
+// CHECK: vpermq $123, 291(%rax,%r14,8), %zmm25
+// CHECK:  encoding: [0x62,0x23,0xfd,0x48,0x00,0x8c,0xf0,0x23,0x01,0x00,0x00,0x7b]
+          vpermq $123, 291(%rax,%r14,8), %zmm25
+
+// CHECK: vpermq $123, (%rcx){1to8}, %zmm25
+// CHECK:  encoding: [0x62,0x63,0xfd,0x58,0x00,0x09,0x7b]
+          vpermq $123, (%rcx){1to8}, %zmm25
+
+// CHECK: vpermq $123, 8128(%rdx), %zmm25
+// CHECK:  encoding: [0x62,0x63,0xfd,0x48,0x00,0x4a,0x7f,0x7b]
+          vpermq $123, 8128(%rdx), %zmm25
+
+// CHECK: vpermq $123, 8192(%rdx), %zmm25
+// CHECK:  encoding: [0x62,0x63,0xfd,0x48,0x00,0x8a,0x00,0x20,0x00,0x00,0x7b]
+          vpermq $123, 8192(%rdx), %zmm25
+
+// CHECK: vpermq $123, -8192(%rdx), %zmm25
+// CHECK:  encoding: [0x62,0x63,0xfd,0x48,0x00,0x4a,0x80,0x7b]
+          vpermq $123, -8192(%rdx), %zmm25
+
+// CHECK: vpermq $123, -8256(%rdx), %zmm25
+// CHECK:  encoding: [0x62,0x63,0xfd,0x48,0x00,0x8a,0xc0,0xdf,0xff,0xff,0x7b]
+          vpermq $123, -8256(%rdx), %zmm25
+
+// CHECK: vpermq $123, 1016(%rdx){1to8}, %zmm25
+// CHECK:  encoding: [0x62,0x63,0xfd,0x58,0x00,0x4a,0x7f,0x7b]
+          vpermq $123, 1016(%rdx){1to8}, %zmm25
+
+// CHECK: vpermq $123, 1024(%rdx){1to8}, %zmm25
+// CHECK:  encoding: [0x62,0x63,0xfd,0x58,0x00,0x8a,0x00,0x04,0x00,0x00,0x7b]
+          vpermq $123, 1024(%rdx){1to8}, %zmm25
+
+// CHECK: vpermq $123, -1024(%rdx){1to8}, %zmm25
+// CHECK:  encoding: [0x62,0x63,0xfd,0x58,0x00,0x4a,0x80,0x7b]
+          vpermq $123, -1024(%rdx){1to8}, %zmm25
+
+// CHECK: vpermq $123, -1032(%rdx){1to8}, %zmm25
+// CHECK:  encoding: [0x62,0x63,0xfd,0x58,0x00,0x8a,0xf8,0xfb,0xff,0xff,0x7b]
+          vpermq $123, -1032(%rdx){1to8}, %zmm25
+
+// CHECK: vpermq %zmm22, %zmm23, %zmm21
+// CHECK:  encoding: [0x62,0xa2,0xc5,0x40,0x36,0xee]
+          vpermq %zmm22, %zmm23, %zmm21
+
+// CHECK: vpermq %zmm22, %zmm23, %zmm21 {%k1}
+// CHECK:  encoding: [0x62,0xa2,0xc5,0x41,0x36,0xee]
+          vpermq %zmm22, %zmm23, %zmm21 {%k1}
+
+// CHECK: vpermq %zmm22, %zmm23, %zmm21 {%k1} {z}
+// CHECK:  encoding: [0x62,0xa2,0xc5,0xc1,0x36,0xee]
+          vpermq %zmm22, %zmm23, %zmm21 {%k1} {z}
+
+// CHECK: vpermq (%rcx), %zmm23, %zmm21
+// CHECK:  encoding: [0x62,0xe2,0xc5,0x40,0x36,0x29]
+          vpermq (%rcx), %zmm23, %zmm21
+
+// CHECK: vpermq 291(%rax,%r14,8), %zmm23, %zmm21
+// CHECK:  encoding: [0x62,0xa2,0xc5,0x40,0x36,0xac,0xf0,0x23,0x01,0x00,0x00]
+          vpermq 291(%rax,%r14,8), %zmm23, %zmm21
+
+// CHECK: vpermq (%rcx){1to8}, %zmm23, %zmm21
+// CHECK:  encoding: [0x62,0xe2,0xc5,0x50,0x36,0x29]
+          vpermq (%rcx){1to8}, %zmm23, %zmm21
+
+// CHECK: vpermq 8128(%rdx), %zmm23, %zmm21
+// CHECK:  encoding: [0x62,0xe2,0xc5,0x40,0x36,0x6a,0x7f]
+          vpermq 8128(%rdx), %zmm23, %zmm21
+
+// CHECK: vpermq 8192(%rdx), %zmm23, %zmm21
+// CHECK:  encoding: [0x62,0xe2,0xc5,0x40,0x36,0xaa,0x00,0x20,0x00,0x00]
+          vpermq 8192(%rdx), %zmm23, %zmm21
+
+// CHECK: vpermq -8192(%rdx), %zmm23, %zmm21
+// CHECK:  encoding: [0x62,0xe2,0xc5,0x40,0x36,0x6a,0x80]
+          vpermq -8192(%rdx), %zmm23, %zmm21
+
+// CHECK: vpermq -8256(%rdx), %zmm23, %zmm21
+// CHECK:  encoding: [0x62,0xe2,0xc5,0x40,0x36,0xaa,0xc0,0xdf,0xff,0xff]
+          vpermq -8256(%rdx), %zmm23, %zmm21
+
+// CHECK: vpermq 1016(%rdx){1to8}, %zmm23, %zmm21
+// CHECK:  encoding: [0x62,0xe2,0xc5,0x50,0x36,0x6a,0x7f]
+          vpermq 1016(%rdx){1to8}, %zmm23, %zmm21
+
+// CHECK: vpermq 1024(%rdx){1to8}, %zmm23, %zmm21
+// CHECK:  encoding: [0x62,0xe2,0xc5,0x50,0x36,0xaa,0x00,0x04,0x00,0x00]
+          vpermq 1024(%rdx){1to8}, %zmm23, %zmm21
+
+// CHECK: vpermq -1024(%rdx){1to8}, %zmm23, %zmm21
+// CHECK:  encoding: [0x62,0xe2,0xc5,0x50,0x36,0x6a,0x80]
+          vpermq -1024(%rdx){1to8}, %zmm23, %zmm21
+
+// CHECK: vpermq -1032(%rdx){1to8}, %zmm23, %zmm21
+// CHECK:  encoding: [0x62,0xe2,0xc5,0x50,0x36,0xaa,0xf8,0xfb,0xff,0xff]
+          vpermq -1032(%rdx){1to8}, %zmm23, %zmm21
+
+// CHECK: vpermpd %zmm18, %zmm29, %zmm26
+// CHECK:  encoding: [0x62,0x22,0x95,0x40,0x16,0xd2]
+          vpermpd %zmm18, %zmm29, %zmm26
+
+// CHECK: vpermpd %zmm18, %zmm29, %zmm26 {%k6}
+// CHECK:  encoding: [0x62,0x22,0x95,0x46,0x16,0xd2]
+          vpermpd %zmm18, %zmm29, %zmm26 {%k6}
+
+// CHECK: vpermpd %zmm18, %zmm29, %zmm26 {%k6} {z}
+// CHECK:  encoding: [0x62,0x22,0x95,0xc6,0x16,0xd2]
+          vpermpd %zmm18, %zmm29, %zmm26 {%k6} {z}
+
+// CHECK: vpermpd (%rcx), %zmm29, %zmm26
+// CHECK:  encoding: [0x62,0x62,0x95,0x40,0x16,0x11]
+          vpermpd (%rcx), %zmm29, %zmm26
+
+// CHECK: vpermpd 291(%rax,%r14,8), %zmm29, %zmm26
+// CHECK:  encoding: [0x62,0x22,0x95,0x40,0x16,0x94,0xf0,0x23,0x01,0x00,0x00]
+          vpermpd 291(%rax,%r14,8), %zmm29, %zmm26
+
+// CHECK: vpermpd (%rcx){1to8}, %zmm29, %zmm26
+// CHECK:  encoding: [0x62,0x62,0x95,0x50,0x16,0x11]
+          vpermpd (%rcx){1to8}, %zmm29, %zmm26
+
+// CHECK: vpermpd 8128(%rdx), %zmm29, %zmm26
+// CHECK:  encoding: [0x62,0x62,0x95,0x40,0x16,0x52,0x7f]
+          vpermpd 8128(%rdx), %zmm29, %zmm26
+
+// CHECK: vpermpd 8192(%rdx), %zmm29, %zmm26
+// CHECK:  encoding: [0x62,0x62,0x95,0x40,0x16,0x92,0x00,0x20,0x00,0x00]
+          vpermpd 8192(%rdx), %zmm29, %zmm26
+
+// CHECK: vpermpd -8192(%rdx), %zmm29, %zmm26
+// CHECK:  encoding: [0x62,0x62,0x95,0x40,0x16,0x52,0x80]
+          vpermpd -8192(%rdx), %zmm29, %zmm26
+
+// CHECK: vpermpd -8256(%rdx), %zmm29, %zmm26
+// CHECK:  encoding: [0x62,0x62,0x95,0x40,0x16,0x92,0xc0,0xdf,0xff,0xff]
+          vpermpd -8256(%rdx), %zmm29, %zmm26
+
+// CHECK: vpermpd 1016(%rdx){1to8}, %zmm29, %zmm26
+// CHECK:  encoding: [0x62,0x62,0x95,0x50,0x16,0x52,0x7f]
+          vpermpd 1016(%rdx){1to8}, %zmm29, %zmm26
+
+// CHECK: vpermpd 1024(%rdx){1to8}, %zmm29, %zmm26
+// CHECK:  encoding: [0x62,0x62,0x95,0x50,0x16,0x92,0x00,0x04,0x00,0x00]
+          vpermpd 1024(%rdx){1to8}, %zmm29, %zmm26
+
+// CHECK: vpermpd -1024(%rdx){1to8}, %zmm29, %zmm26
+// CHECK:  encoding: [0x62,0x62,0x95,0x50,0x16,0x52,0x80]
+          vpermpd -1024(%rdx){1to8}, %zmm29, %zmm26
+
+// CHECK: vpermpd -1032(%rdx){1to8}, %zmm29, %zmm26
+// CHECK:  encoding: [0x62,0x62,0x95,0x50,0x16,0x92,0xf8,0xfb,0xff,0xff]
+          vpermpd -1032(%rdx){1to8}, %zmm29, %zmm26
+
+// CHECK: vpermpd $171, %zmm27, %zmm3
+// CHECK:  encoding: [0x62,0x93,0xfd,0x48,0x01,0xdb,0xab]
+          vpermpd $0xab, %zmm27, %zmm3
+
+// CHECK: vpermpd $171, %zmm27, %zmm3 {%k2}
+// CHECK:  encoding: [0x62,0x93,0xfd,0x4a,0x01,0xdb,0xab]
+          vpermpd $0xab, %zmm27, %zmm3 {%k2}
+
+// CHECK: vpermpd $171, %zmm27, %zmm3 {%k2} {z}
+// CHECK:  encoding: [0x62,0x93,0xfd,0xca,0x01,0xdb,0xab]
+          vpermpd $0xab, %zmm27, %zmm3 {%k2} {z}
+
+// CHECK: vpermpd $123, %zmm27, %zmm3
+// CHECK:  encoding: [0x62,0x93,0xfd,0x48,0x01,0xdb,0x7b]
+          vpermpd $0x7b, %zmm27, %zmm3
+
+// CHECK: vpermpd $123, (%rcx), %zmm3
+// CHECK:  encoding: [0x62,0xf3,0xfd,0x48,0x01,0x19,0x7b]
+          vpermpd $0x7b, (%rcx), %zmm3
+
+// CHECK: vpermpd $123, 291(%rax,%r14,8), %zmm3
+// CHECK:  encoding: [0x62,0xb3,0xfd,0x48,0x01,0x9c,0xf0,0x23,0x01,0x00,0x00,0x7b]
+          vpermpd $0x7b, 291(%rax,%r14,8), %zmm3
+
+// CHECK: vpermpd $123, (%rcx){1to8}, %zmm3
+// CHECK:  encoding: [0x62,0xf3,0xfd,0x58,0x01,0x19,0x7b]
+          vpermpd $0x7b, (%rcx){1to8}, %zmm3
+
+// CHECK: vpermpd $123, 8128(%rdx), %zmm3
+// CHECK:  encoding: [0x62,0xf3,0xfd,0x48,0x01,0x5a,0x7f,0x7b]
+          vpermpd $0x7b, 8128(%rdx), %zmm3
+
+// CHECK: vpermpd $123, 8192(%rdx), %zmm3
+// CHECK:  encoding: [0x62,0xf3,0xfd,0x48,0x01,0x9a,0x00,0x20,0x00,0x00,0x7b]
+          vpermpd $0x7b, 8192(%rdx), %zmm3
+
+// CHECK: vpermpd $123, -8192(%rdx), %zmm3
+// CHECK:  encoding: [0x62,0xf3,0xfd,0x48,0x01,0x5a,0x80,0x7b]
+          vpermpd $0x7b, -8192(%rdx), %zmm3
+
+// CHECK: vpermpd $123, -8256(%rdx), %zmm3
+// CHECK:  encoding: [0x62,0xf3,0xfd,0x48,0x01,0x9a,0xc0,0xdf,0xff,0xff,0x7b]
+          vpermpd $0x7b, -8256(%rdx), %zmm3
+
+// CHECK: vpermpd $123, 1016(%rdx){1to8}, %zmm3
+// CHECK:  encoding: [0x62,0xf3,0xfd,0x58,0x01,0x5a,0x7f,0x7b]
+          vpermpd $0x7b, 1016(%rdx){1to8}, %zmm3
+
+// CHECK: vpermpd $123, 1024(%rdx){1to8}, %zmm3
+// CHECK:  encoding: [0x62,0xf3,0xfd,0x58,0x01,0x9a,0x00,0x04,0x00,0x00,0x7b]
+          vpermpd $0x7b, 1024(%rdx){1to8}, %zmm3
+
+// CHECK: vpermpd $123, -1024(%rdx){1to8}, %zmm3
+// CHECK:  encoding: [0x62,0xf3,0xfd,0x58,0x01,0x5a,0x80,0x7b]
+          vpermpd $0x7b, -1024(%rdx){1to8}, %zmm3
+
+// CHECK: vpermpd $123, -1032(%rdx){1to8}, %zmm3
+// CHECK:  encoding: [0x62,0xf3,0xfd,0x58,0x01,0x9a,0xf8,0xfb,0xff,0xff,0x7b]
+          vpermpd $0x7b, -1032(%rdx){1to8}, %zmm3
+
+// CHECK: vpermd %zmm9, %zmm28, %zmm22
+// CHECK:  encoding: [0x62,0xc2,0x1d,0x40,0x36,0xf1]
+          vpermd %zmm9, %zmm28, %zmm22
+
+// CHECK: vpermd %zmm9, %zmm28, %zmm22 {%k1}
+// CHECK:  encoding: [0x62,0xc2,0x1d,0x41,0x36,0xf1]
+          vpermd %zmm9, %zmm28, %zmm22 {%k1}
+
+// CHECK: vpermd %zmm9, %zmm28, %zmm22 {%k1} {z}
+// CHECK:  encoding: [0x62,0xc2,0x1d,0xc1,0x36,0xf1]
+          vpermd %zmm9, %zmm28, %zmm22 {%k1} {z}
+
+// CHECK: vpermd (%rcx), %zmm28, %zmm22
+// CHECK:  encoding: [0x62,0xe2,0x1d,0x40,0x36,0x31]
+          vpermd (%rcx), %zmm28, %zmm22
+
+// CHECK: vpermd 291(%rax,%r14,8), %zmm28, %zmm22
+// CHECK:  encoding: [0x62,0xa2,0x1d,0x40,0x36,0xb4,0xf0,0x23,0x01,0x00,0x00]
+          vpermd 291(%rax,%r14,8), %zmm28, %zmm22
+
+// CHECK: vpermd (%rcx){1to16}, %zmm28, %zmm22
+// CHECK:  encoding: [0x62,0xe2,0x1d,0x50,0x36,0x31]
+          vpermd (%rcx){1to16}, %zmm28, %zmm22
+
+// CHECK: vpermd 8128(%rdx), %zmm28, %zmm22
+// CHECK:  encoding: [0x62,0xe2,0x1d,0x40,0x36,0x72,0x7f]
+          vpermd 8128(%rdx), %zmm28, %zmm22
+
+// CHECK: vpermd 8192(%rdx), %zmm28, %zmm22
+// CHECK:  encoding: [0x62,0xe2,0x1d,0x40,0x36,0xb2,0x00,0x20,0x00,0x00]
+          vpermd 8192(%rdx), %zmm28, %zmm22
+
+// CHECK: vpermd -8192(%rdx), %zmm28, %zmm22
+// CHECK:  encoding: [0x62,0xe2,0x1d,0x40,0x36,0x72,0x80]
+          vpermd -8192(%rdx), %zmm28, %zmm22
+
+// CHECK: vpermd -8256(%rdx), %zmm28, %zmm22
+// CHECK:  encoding: [0x62,0xe2,0x1d,0x40,0x36,0xb2,0xc0,0xdf,0xff,0xff]
+          vpermd -8256(%rdx), %zmm28, %zmm22
+
+// CHECK: vpermd 508(%rdx){1to16}, %zmm28, %zmm22
+// CHECK:  encoding: [0x62,0xe2,0x1d,0x50,0x36,0x72,0x7f]
+          vpermd 508(%rdx){1to16}, %zmm28, %zmm22
+
+// CHECK: vpermd 512(%rdx){1to16}, %zmm28, %zmm22
+// CHECK:  encoding: [0x62,0xe2,0x1d,0x50,0x36,0xb2,0x00,0x02,0x00,0x00]
+          vpermd 512(%rdx){1to16}, %zmm28, %zmm22
+
+// CHECK: vpermd -512(%rdx){1to16}, %zmm28, %zmm22
+// CHECK:  encoding: [0x62,0xe2,0x1d,0x50,0x36,0x72,0x80]
+          vpermd -512(%rdx){1to16}, %zmm28, %zmm22
+
+// CHECK: vpermd -516(%rdx){1to16}, %zmm28, %zmm22
+// CHECK:  encoding: [0x62,0xe2,0x1d,0x50,0x36,0xb2,0xfc,0xfd,0xff,0xff]
+          vpermd -516(%rdx){1to16}, %zmm28, %zmm22
+
+// CHECK:  vcvtsi2sdl %eax, %xmm10, %xmm7
+// CHECK:  encoding: [0xc5,0xab,0x2a,0xf8]
+          vcvtsi2sd %eax, %xmm10, %xmm7
+
+// CHECK: vcvtsi2sdl %ebp, %xmm10, %xmm7
+// CHECK:  encoding: [0xc5,0xab,0x2a,0xfd]
+          vcvtsi2sd %ebp, %xmm10, %xmm7
+
+// CHECK: vcvtsi2sdl %r13d, %xmm10, %xmm7
+// CHECK:  encoding: [0xc4,0xc1,0x2b,0x2a,0xfd]
+          vcvtsi2sd %r13d, %xmm10, %xmm7
+
+// CHECK: vcvtsi2sdl (%rcx), %xmm10, %xmm7
+// CHECK:  encoding: [0xc5,0xab,0x2a,0x39]
+          vcvtsi2sdl (%rcx), %xmm10, %xmm7
+
+// CHECK: vcvtsi2sdl 291(%rax,%r14,8), %xmm10, %xmm7
+// CHECK:  encoding: [0xc4,0xa1,0x2b,0x2a,0xbc,0xf0,0x23,0x01,0x00,0x00]
+          vcvtsi2sdl 291(%rax,%r14,8), %xmm10, %xmm7
+
+// CHECK: vcvtsi2sdl 508(%rdx), %xmm10, %xmm7
+// CHECK:  encoding: [0xc5,0xab,0x2a,0xba,0xfc,0x01,0x00,0x00]
+          vcvtsi2sdl 508(%rdx), %xmm10, %xmm7
+
+// CHECK: vcvtsi2sdl 512(%rdx), %xmm10, %xmm7
+// CHECK:  encoding: [0xc5,0xab,0x2a,0xba,0x00,0x02,0x00,0x00]
+          vcvtsi2sdl 512(%rdx), %xmm10, %xmm7
+
+// CHECK: vcvtsi2sdl -512(%rdx), %xmm10, %xmm7
+// CHECK:  encoding: [0xc5,0xab,0x2a,0xba,0x00,0xfe,0xff,0xff]
+          vcvtsi2sdl -512(%rdx), %xmm10, %xmm7
+
+// CHECK: vcvtsi2sdl -516(%rdx), %xmm10, %xmm7
+// CHECK:  encoding: [0xc5,0xab,0x2a,0xba,0xfc,0xfd,0xff,0xff]
+          vcvtsi2sdl -516(%rdx), %xmm10, %xmm7
+// CHECK: vcvtsi2sdq %rax, %xmm12, %xmm29
+// CHECK:  encoding: [0x62,0x61,0x9f,0x08,0x2a,0xe8]
+          vcvtsi2sd %rax, %xmm12, %xmm29
+
+// CHECK: vcvtsi2sdq %rax,  {rn-sae}, %xmm12, %xmm29
+// CHECK:  encoding: [0x62,0x61,0x9f,0x18,0x2a,0xe8]
+          vcvtsi2sd %rax,  {rn-sae}, %xmm12, %xmm29
+
+// CHECK: vcvtsi2sdq %rax,  {ru-sae}, %xmm12, %xmm29
+// CHECK:  encoding: [0x62,0x61,0x9f,0x58,0x2a,0xe8]
+          vcvtsi2sd %rax,  {ru-sae}, %xmm12, %xmm29
+
+// CHECK: vcvtsi2sdq %rax,  {rd-sae}, %xmm12, %xmm29
+// CHECK:  encoding: [0x62,0x61,0x9f,0x38,0x2a,0xe8]
+          vcvtsi2sd %rax,  {rd-sae}, %xmm12, %xmm29
+
+// CHECK: vcvtsi2sdq %rax,  {rz-sae}, %xmm12, %xmm29
+// CHECK:  encoding: [0x62,0x61,0x9f,0x78,0x2a,0xe8]
+          vcvtsi2sd %rax,  {rz-sae}, %xmm12, %xmm29
+
+// CHECK: vcvtsi2sdq %r8, %xmm12, %xmm29
+// CHECK:  encoding: [0x62,0x41,0x9f,0x08,0x2a,0xe8]
+          vcvtsi2sd %r8, %xmm12, %xmm29
+
+// CHECK: vcvtsi2sdq %r8,  {rn-sae}, %xmm12, %xmm29
+// CHECK:  encoding: [0x62,0x41,0x9f,0x18,0x2a,0xe8]
+          vcvtsi2sd %r8,  {rn-sae}, %xmm12, %xmm29
+
+// CHECK: vcvtsi2sdq %r8,  {ru-sae}, %xmm12, %xmm29
+// CHECK:  encoding: [0x62,0x41,0x9f,0x58,0x2a,0xe8]
+          vcvtsi2sd %r8,  {ru-sae}, %xmm12, %xmm29
+
+// CHECK: vcvtsi2sdq %r8,  {rd-sae}, %xmm12, %xmm29
+// CHECK:  encoding: [0x62,0x41,0x9f,0x38,0x2a,0xe8]
+          vcvtsi2sd %r8,  {rd-sae}, %xmm12, %xmm29
+
+// CHECK: vcvtsi2sdq %r8,  {rz-sae}, %xmm12, %xmm29
+// CHECK:  encoding: [0x62,0x41,0x9f,0x78,0x2a,0xe8]
+          vcvtsi2sd %r8,  {rz-sae}, %xmm12, %xmm29
+
+// CHECK: vcvtsi2sdq (%rcx), %xmm12, %xmm29
+// CHECK:  encoding: [0x62,0x61,0x9f,0x08,0x2a,0x29]
+          vcvtsi2sdq (%rcx), %xmm12, %xmm29
+
+// CHECK: vcvtsi2sdq 291(%rax,%r14,8), %xmm12, %xmm29
+// CHECK:  encoding: [0x62,0x21,0x9f,0x08,0x2a,0xac,0xf0,0x23,0x01,0x00,0x00]
+          vcvtsi2sdq 291(%rax,%r14,8), %xmm12, %xmm29
+
+// CHECK: vcvtsi2sdq 1016(%rdx), %xmm12, %xmm29
+// CHECK:  encoding: [0x62,0x61,0x9f,0x08,0x2a,0x6a,0x7f]
+          vcvtsi2sdq 1016(%rdx), %xmm12, %xmm29
+
+// CHECK: vcvtsi2sdq 1024(%rdx), %xmm12, %xmm29
+// CHECK:  encoding: [0x62,0x61,0x9f,0x08,0x2a,0xaa,0x00,0x04,0x00,0x00]
+          vcvtsi2sdq 1024(%rdx), %xmm12, %xmm29
+
+// CHECK: vcvtsi2sdq -1024(%rdx), %xmm12, %xmm29
+// CHECK:  encoding: [0x62,0x61,0x9f,0x08,0x2a,0x6a,0x80]
+          vcvtsi2sdq -1024(%rdx), %xmm12, %xmm29
+
+// CHECK: vcvtsi2sdq -1032(%rdx), %xmm12, %xmm29
+// CHECK:  encoding: [0x62,0x61,0x9f,0x08,0x2a,0xaa,0xf8,0xfb,0xff,0xff]
+          vcvtsi2sdq -1032(%rdx), %xmm12, %xmm29
+
+// CHECK: vcvtsi2ssl %eax, %xmm10, %xmm15
+// CHECK:  encoding: [0xc5,0x2a,0x2a,0xf8]
+          vcvtsi2ss %eax, %xmm10, %xmm15
+
+// CHECK: vcvtsi2ssl %eax,  {rn-sae}, %xmm10, %xmm15
+// CHECK:  encoding: [0x62,0x71,0x2e,0x18,0x2a,0xf8]
+          vcvtsi2ss %eax,  {rn-sae}, %xmm10, %xmm15
+
+// CHECK: vcvtsi2ssl %eax,  {ru-sae}, %xmm10, %xmm15
+// CHECK:  encoding: [0x62,0x71,0x2e,0x58,0x2a,0xf8]
+          vcvtsi2ss %eax,  {ru-sae}, %xmm10, %xmm15
+
+// CHECK: vcvtsi2ssl %eax,  {rd-sae}, %xmm10, %xmm15
+// CHECK:  encoding: [0x62,0x71,0x2e,0x38,0x2a,0xf8]
+          vcvtsi2ss %eax,  {rd-sae}, %xmm10, %xmm15
+
+// CHECK: vcvtsi2ssl %eax,  {rz-sae}, %xmm10, %xmm15
+// CHECK:  encoding: [0x62,0x71,0x2e,0x78,0x2a,0xf8]
+          vcvtsi2ss %eax,  {rz-sae}, %xmm10, %xmm15
+
+// CHECK: vcvtsi2ssl %ebp, %xmm10, %xmm15
+// CHECK:  encoding: [0xc5,0x2a,0x2a,0xfd]
+          vcvtsi2ss %ebp, %xmm10, %xmm15
+
+// CHECK: vcvtsi2ssl %ebp,  {rn-sae}, %xmm10, %xmm15
+// CHECK:  encoding: [0x62,0x71,0x2e,0x18,0x2a,0xfd]
+          vcvtsi2ss %ebp,  {rn-sae}, %xmm10, %xmm15
+
+// CHECK: vcvtsi2ssl %ebp,  {ru-sae}, %xmm10, %xmm15
+// CHECK:  encoding: [0x62,0x71,0x2e,0x58,0x2a,0xfd]
+          vcvtsi2ss %ebp,  {ru-sae}, %xmm10, %xmm15
+
+// CHECK: vcvtsi2ssl %ebp,  {rd-sae}, %xmm10, %xmm15
+// CHECK:  encoding: [0x62,0x71,0x2e,0x38,0x2a,0xfd]
+          vcvtsi2ss %ebp,  {rd-sae}, %xmm10, %xmm15
+
+// CHECK: vcvtsi2ssl %ebp,  {rz-sae}, %xmm10, %xmm15
+// CHECK:  encoding: [0x62,0x71,0x2e,0x78,0x2a,0xfd]
+          vcvtsi2ss %ebp,  {rz-sae}, %xmm10, %xmm15
+
+// CHECK: vcvtsi2ssl %r13d, %xmm10, %xmm15
+// CHECK:  encoding: [0xc4,0x41,0x2a,0x2a,0xfd]
+          vcvtsi2ss %r13d, %xmm10, %xmm15
+
+// CHECK: vcvtsi2ssl %r13d,  {rn-sae}, %xmm10, %xmm15
+// CHECK:  encoding: [0x62,0x51,0x2e,0x18,0x2a,0xfd]
+          vcvtsi2ss %r13d,  {rn-sae}, %xmm10, %xmm15
+
+// CHECK: vcvtsi2ssl %r13d,  {ru-sae}, %xmm10, %xmm15
+// CHECK:  encoding: [0x62,0x51,0x2e,0x58,0x2a,0xfd]
+          vcvtsi2ss %r13d,  {ru-sae}, %xmm10, %xmm15
+
+// CHECK: vcvtsi2ssl %r13d,  {rd-sae}, %xmm10, %xmm15
+// CHECK:  encoding: [0x62,0x51,0x2e,0x38,0x2a,0xfd]
+          vcvtsi2ss %r13d,  {rd-sae}, %xmm10, %xmm15
+
+// CHECK: vcvtsi2ssl %r13d,  {rz-sae}, %xmm10, %xmm15
+// CHECK:  encoding: [0x62,0x51,0x2e,0x78,0x2a,0xfd]
+          vcvtsi2ss %r13d,  {rz-sae}, %xmm10, %xmm15
+
+// CHECK: vcvtsi2ssl (%rcx), %xmm10, %xmm15
+// CHECK:  encoding: [0xc5,0x2a,0x2a,0x39]
+          vcvtsi2ssl (%rcx), %xmm10, %xmm15
+
+// CHECK: vcvtsi2ssl 291(%rax,%r14,8), %xmm10, %xmm15
+// CHECK:  encoding: [0xc4,0x21,0x2a,0x2a,0xbc,0xf0,0x23,0x01,0x00,0x00]
+          vcvtsi2ssl 291(%rax,%r14,8), %xmm10, %xmm15
+
+// CHECK: vcvtsi2ssl 508(%rdx), %xmm10, %xmm15
+// CHECK:  encoding: [0xc5,0x2a,0x2a,0xba,0xfc,0x01,0x00,0x00]
+          vcvtsi2ssl 508(%rdx), %xmm10, %xmm15
+
+// CHECK: vcvtsi2ssl 512(%rdx), %xmm10, %xmm15
+// CHECK:  encoding: [0xc5,0x2a,0x2a,0xba,0x00,0x02,0x00,0x00]
+          vcvtsi2ssl 512(%rdx), %xmm10, %xmm15
+
+// CHECK: vcvtsi2ssl -512(%rdx), %xmm10, %xmm15
+// CHECK:  encoding: [0xc5,0x2a,0x2a,0xba,0x00,0xfe,0xff,0xff]
+          vcvtsi2ssl -512(%rdx), %xmm10, %xmm15
+
+// CHECK: vcvtsi2ssl -516(%rdx), %xmm10, %xmm15
+// CHECK:  encoding: [0xc5,0x2a,0x2a,0xba,0xfc,0xfd,0xff,0xff]
+          vcvtsi2ssl -516(%rdx), %xmm10, %xmm15
+// CHECK: vcvtsi2ssq %rax, %xmm10, %xmm16
+// CHECK:  encoding: [0x62,0xe1,0xae,0x08,0x2a,0xc0]
+          vcvtsi2ss %rax, %xmm10, %xmm16
+
+// CHECK: vcvtsi2ssq %rax,  {rn-sae}, %xmm10, %xmm16
+// CHECK:  encoding: [0x62,0xe1,0xae,0x18,0x2a,0xc0]
+          vcvtsi2ss %rax,  {rn-sae}, %xmm10, %xmm16
+
+// CHECK: vcvtsi2ssq %rax,  {ru-sae}, %xmm10, %xmm16
+// CHECK:  encoding: [0x62,0xe1,0xae,0x58,0x2a,0xc0]
+          vcvtsi2ss %rax,  {ru-sae}, %xmm10, %xmm16
+
+// CHECK: vcvtsi2ssq %rax,  {rd-sae}, %xmm10, %xmm16
+// CHECK:  encoding: [0x62,0xe1,0xae,0x38,0x2a,0xc0]
+          vcvtsi2ss %rax,  {rd-sae}, %xmm10, %xmm16
+
+// CHECK: vcvtsi2ssq %rax,  {rz-sae}, %xmm10, %xmm16
+// CHECK:  encoding: [0x62,0xe1,0xae,0x78,0x2a,0xc0]
+          vcvtsi2ss %rax,  {rz-sae}, %xmm10, %xmm16
+
+// CHECK: vcvtsi2ssq %r8, %xmm10, %xmm16
+// CHECK:  encoding: [0x62,0xc1,0xae,0x08,0x2a,0xc0]
+          vcvtsi2ss %r8, %xmm10, %xmm16
+
+// CHECK: vcvtsi2ssq %r8,  {rn-sae}, %xmm10, %xmm16
+// CHECK:  encoding: [0x62,0xc1,0xae,0x18,0x2a,0xc0]
+          vcvtsi2ss %r8,  {rn-sae}, %xmm10, %xmm16
+
+// CHECK: vcvtsi2ssq %r8,  {ru-sae}, %xmm10, %xmm16
+// CHECK:  encoding: [0x62,0xc1,0xae,0x58,0x2a,0xc0]
+          vcvtsi2ss %r8,  {ru-sae}, %xmm10, %xmm16
+
+// CHECK: vcvtsi2ssq %r8,  {rd-sae}, %xmm10, %xmm16
+// CHECK:  encoding: [0x62,0xc1,0xae,0x38,0x2a,0xc0]
+          vcvtsi2ss %r8,  {rd-sae}, %xmm10, %xmm16
+
+// CHECK: vcvtsi2ssq %r8,  {rz-sae}, %xmm10, %xmm16
+// CHECK:  encoding: [0x62,0xc1,0xae,0x78,0x2a,0xc0]
+          vcvtsi2ss %r8,  {rz-sae}, %xmm10, %xmm16
+
+// CHECK: vcvtsi2ssq (%rcx), %xmm10, %xmm16
+// CHECK:  encoding: [0x62,0xe1,0xae,0x08,0x2a,0x01]
+          vcvtsi2ssq (%rcx), %xmm10, %xmm16
+
+// CHECK: vcvtsi2ssq 291(%rax,%r14,8), %xmm10, %xmm16
+// CHECK:  encoding: [0x62,0xa1,0xae,0x08,0x2a,0x84,0xf0,0x23,0x01,0x00,0x00]
+          vcvtsi2ssq 291(%rax,%r14,8), %xmm10, %xmm16
+
+// CHECK: vcvtsi2ssq 1016(%rdx), %xmm10, %xmm16
+// CHECK:  encoding: [0x62,0xe1,0xae,0x08,0x2a,0x42,0x7f]
+          vcvtsi2ssq 1016(%rdx), %xmm10, %xmm16
+
+// CHECK: vcvtsi2ssq 1024(%rdx), %xmm10, %xmm16
+// CHECK:  encoding: [0x62,0xe1,0xae,0x08,0x2a,0x82,0x00,0x04,0x00,0x00]
+          vcvtsi2ssq 1024(%rdx), %xmm10, %xmm16
+
+// CHECK: vcvtsi2ssq -1024(%rdx), %xmm10, %xmm16
+// CHECK:  encoding: [0x62,0xe1,0xae,0x08,0x2a,0x42,0x80]
+          vcvtsi2ssq -1024(%rdx), %xmm10, %xmm16
+
+// CHECK: vcvtsi2ssq -1032(%rdx), %xmm10, %xmm16
+// CHECK:  encoding: [0x62,0xe1,0xae,0x08,0x2a,0x82,0xf8,0xfb,0xff,0xff]
+          vcvtsi2ssq -1032(%rdx), %xmm10, %xmm16
+
+// CHECK:  vcvtusi2sdl  %eax, %xmm1, %xmm19
+// CHECK:  encoding: [0x62,0xe1,0x77,0x08,0x7b,0xd8]
+          vcvtusi2sd %eax, %xmm1, %xmm19
+
+// CHECK:  vcvtusi2sdl  %ebp, %xmm1, %xmm19
+// CHECK:  encoding: [0x62,0xe1,0x77,0x08,0x7b,0xdd]
+          vcvtusi2sd %ebp, %xmm1, %xmm19
+
+// CHECK:  vcvtusi2sdl  %r13d, %xmm1, %xmm19
+// CHECK:  encoding: [0x62,0xc1,0x77,0x08,0x7b,0xdd]
+          vcvtusi2sd %r13d, %xmm1, %xmm19
+
+// CHECK: vcvtusi2sdl (%rcx), %xmm1, %xmm19
+// CHECK:  encoding: [0x62,0xe1,0x77,0x08,0x7b,0x19]
+          vcvtusi2sdl (%rcx), %xmm1, %xmm19
+
+// CHECK: vcvtusi2sdl 291(%rax,%r14,8), %xmm1, %xmm19
+// CHECK:  encoding: [0x62,0xa1,0x77,0x08,0x7b,0x9c,0xf0,0x23,0x01,0x00,0x00]
+          vcvtusi2sdl 291(%rax,%r14,8), %xmm1, %xmm19
+
+// CHECK: vcvtusi2sdl 508(%rdx), %xmm1, %xmm19
+// CHECK:  encoding: [0x62,0xe1,0x77,0x08,0x7b,0x5a,0x7f]
+          vcvtusi2sdl 508(%rdx), %xmm1, %xmm19
+
+// CHECK: vcvtusi2sdl 512(%rdx), %xmm1, %xmm19
+// CHECK:  encoding: [0x62,0xe1,0x77,0x08,0x7b,0x9a,0x00,0x02,0x00,0x00]
+          vcvtusi2sdl 512(%rdx), %xmm1, %xmm19
+
+// CHECK: vcvtusi2sdl -512(%rdx), %xmm1, %xmm19
+// CHECK:  encoding: [0x62,0xe1,0x77,0x08,0x7b,0x5a,0x80]
+          vcvtusi2sdl -512(%rdx), %xmm1, %xmm19
+
+// CHECK: vcvtusi2sdl -516(%rdx), %xmm1, %xmm19
+// CHECK:  encoding: [0x62,0xe1,0x77,0x08,0x7b,0x9a,0xfc,0xfd,0xff,0xff]
+          vcvtusi2sdl -516(%rdx), %xmm1, %xmm19
+
+// CHECK: vcvtusi2sdq %rax, %xmm26, %xmm14
+// CHECK:  encoding: [0x62,0x71,0xaf,0x00,0x7b,0xf0]
+          vcvtusi2sd %rax, %xmm26, %xmm14
+
+// CHECK: vcvtusi2sdq %rax, {rn-sae}, %xmm26, %xmm14
+// CHECK:  encoding: [0x62,0x71,0xaf,0x10,0x7b,0xf0]
+          vcvtusi2sd %rax, {rn-sae}, %xmm26, %xmm14
+
+// CHECK: vcvtusi2sdq %rax, {ru-sae}, %xmm26, %xmm14
+// CHECK:  encoding: [0x62,0x71,0xaf,0x50,0x7b,0xf0]
+          vcvtusi2sd %rax, {ru-sae}, %xmm26, %xmm14
+
+// CHECK: vcvtusi2sdq %rax, {rd-sae}, %xmm26, %xmm14
+// CHECK:  encoding: [0x62,0x71,0xaf,0x30,0x7b,0xf0]
+          vcvtusi2sd %rax, {rd-sae}, %xmm26, %xmm14
+
+// CHECK: vcvtusi2sdq %rax, {rz-sae}, %xmm26, %xmm14
+// CHECK:  encoding: [0x62,0x71,0xaf,0x70,0x7b,0xf0]
+          vcvtusi2sd %rax, {rz-sae}, %xmm26, %xmm14
+
+// CHECK: vcvtusi2sdq %r8, %xmm26, %xmm14
+// CHECK:  encoding: [0x62,0x51,0xaf,0x00,0x7b,0xf0]
+          vcvtusi2sd %r8, %xmm26, %xmm14
+
+// CHECK: vcvtusi2sdq %r8, {rn-sae}, %xmm26, %xmm14
+// CHECK:  encoding: [0x62,0x51,0xaf,0x10,0x7b,0xf0]
+          vcvtusi2sd %r8, {rn-sae}, %xmm26, %xmm14
+
+// CHECK:  vcvtusi2sdq  %r8, {ru-sae}, %xmm26, %xmm14
+// CHECK:  encoding: [0x62,0x51,0xaf,0x50,0x7b,0xf0]
+          vcvtusi2sd %r8, {ru-sae}, %xmm26, %xmm14
+
+// CHECK:  vcvtusi2sdq  %r8, {rd-sae}, %xmm26, %xmm14
+// CHECK:  encoding: [0x62,0x51,0xaf,0x30,0x7b,0xf0]
+          vcvtusi2sd %r8, {rd-sae}, %xmm26, %xmm14
+
+// CHECK:  vcvtusi2sdq  %r8, {rz-sae}, %xmm26, %xmm14
+// CHECK:  encoding: [0x62,0x51,0xaf,0x70,0x7b,0xf0]
+          vcvtusi2sd %r8, {rz-sae}, %xmm26, %xmm14
+
+// CHECK: vcvtusi2sdq (%rcx), %xmm26, %xmm14
+// CHECK:  encoding: [0x62,0x71,0xaf,0x00,0x7b,0x31]
+          vcvtusi2sdq (%rcx), %xmm26, %xmm14
+
+// CHECK: vcvtusi2sdq 291(%rax,%r14,8), %xmm26, %xmm14
+// CHECK:  encoding: [0x62,0x31,0xaf,0x00,0x7b,0xb4,0xf0,0x23,0x01,0x00,0x00]
+          vcvtusi2sdq 291(%rax,%r14,8), %xmm26, %xmm14
+
+// CHECK: vcvtusi2sdq 1016(%rdx), %xmm26, %xmm14
+// CHECK:  encoding: [0x62,0x71,0xaf,0x00,0x7b,0x72,0x7f]
+          vcvtusi2sdq 1016(%rdx), %xmm26, %xmm14
+
+// CHECK: vcvtusi2sdq 1024(%rdx), %xmm26, %xmm14
+// CHECK:  encoding: [0x62,0x71,0xaf,0x00,0x7b,0xb2,0x00,0x04,0x00,0x00]
+          vcvtusi2sdq 1024(%rdx), %xmm26, %xmm14
+
+// CHECK: vcvtusi2sdq -1024(%rdx), %xmm26, %xmm14
+// CHECK:  encoding: [0x62,0x71,0xaf,0x00,0x7b,0x72,0x80]
+          vcvtusi2sdq -1024(%rdx), %xmm26, %xmm14
+
+// CHECK: vcvtusi2sdq -1032(%rdx), %xmm26, %xmm14
+// CHECK:  encoding: [0x62,0x71,0xaf,0x00,0x7b,0xb2,0xf8,0xfb,0xff,0xff]
+          vcvtusi2sdq -1032(%rdx), %xmm26, %xmm14
+
+// CHECK: vcvtusi2ssl %eax, %xmm26, %xmm5
+// CHECK:  encoding: [0x62,0xf1,0x2e,0x00,0x7b,0xe8]
+          vcvtusi2ss %eax, %xmm26, %xmm5
+
+// CHECK:  vcvtusi2ssl  %eax, {rn-sae}, %xmm26, %xmm5
+// CHECK:  encoding: [0x62,0xf1,0x2e,0x10,0x7b,0xe8]
+          vcvtusi2ss %eax, {rn-sae}, %xmm26, %xmm5
+
+// CHECK:  vcvtusi2ssl  %eax, {ru-sae}, %xmm26, %xmm5
+// CHECK:  encoding: [0x62,0xf1,0x2e,0x50,0x7b,0xe8]
+          vcvtusi2ss %eax, {ru-sae}, %xmm26, %xmm5
+
+// CHECK:  vcvtusi2ssl  %eax, {rd-sae}, %xmm26, %xmm5
+// CHECK:  encoding: [0x62,0xf1,0x2e,0x30,0x7b,0xe8]
+          vcvtusi2ss %eax, {rd-sae}, %xmm26, %xmm5
+
+// CHECK:  vcvtusi2ssl  %eax, {rz-sae}, %xmm26, %xmm5
+// CHECK:  encoding: [0x62,0xf1,0x2e,0x70,0x7b,0xe8]
+          vcvtusi2ss %eax, {rz-sae}, %xmm26, %xmm5
+
+// CHECK:  vcvtusi2ssl  %ebp, %xmm26, %xmm5
+// CHECK:  encoding: [0x62,0xf1,0x2e,0x00,0x7b,0xed]
+          vcvtusi2ss %ebp, %xmm26, %xmm5
+
+// CHECK:  vcvtusi2ssl  %ebp, {rn-sae}, %xmm26, %xmm5
+// CHECK:  encoding: [0x62,0xf1,0x2e,0x10,0x7b,0xed]
+          vcvtusi2ss %ebp, {rn-sae}, %xmm26, %xmm5
+
+// CHECK:  vcvtusi2ssl  %ebp, {ru-sae}, %xmm26, %xmm5
+// CHECK:  encoding: [0x62,0xf1,0x2e,0x50,0x7b,0xed]
+          vcvtusi2ss %ebp, {ru-sae}, %xmm26, %xmm5
+
+// CHECK:  vcvtusi2ssl  %ebp, {rd-sae}, %xmm26, %xmm5
+// CHECK:  encoding: [0x62,0xf1,0x2e,0x30,0x7b,0xed]
+          vcvtusi2ss %ebp, {rd-sae}, %xmm26, %xmm5
+
+// CHECK:  vcvtusi2ssl  %ebp, {rz-sae}, %xmm26, %xmm5
+// CHECK:  encoding: [0x62,0xf1,0x2e,0x70,0x7b,0xed]
+          vcvtusi2ss %ebp, {rz-sae}, %xmm26, %xmm5
+
+// CHECK:  vcvtusi2ssl  %r13d, %xmm26, %xmm5
+// CHECK:  encoding: [0x62,0xd1,0x2e,0x00,0x7b,0xed]
+          vcvtusi2ss %r13d, %xmm26, %xmm5
+
+// CHECK:  vcvtusi2ssl  %r13d, {rn-sae}, %xmm26, %xmm5
+// CHECK:  encoding: [0x62,0xd1,0x2e,0x10,0x7b,0xed]
+          vcvtusi2ss %r13d, {rn-sae}, %xmm26, %xmm5
+
+// CHECK:  vcvtusi2ssl  %r13d, {ru-sae}, %xmm26, %xmm5
+// CHECK:  encoding: [0x62,0xd1,0x2e,0x50,0x7b,0xed]
+          vcvtusi2ss %r13d, {ru-sae}, %xmm26, %xmm5
+
+// CHECK:  vcvtusi2ssl  %r13d, {rd-sae}, %xmm26, %xmm5
+// CHECK:  encoding: [0x62,0xd1,0x2e,0x30,0x7b,0xed]
+          vcvtusi2ss %r13d, {rd-sae}, %xmm26, %xmm5
+
+// CHECK:  vcvtusi2ssl  %r13d, {rz-sae}, %xmm26, %xmm5
+// CHECK:  encoding: [0x62,0xd1,0x2e,0x70,0x7b,0xed]
+          vcvtusi2ss %r13d, {rz-sae}, %xmm26, %xmm5
+
+// CHECK: vcvtusi2ssl (%rcx), %xmm26, %xmm5
+// CHECK:  encoding: [0x62,0xf1,0x2e,0x00,0x7b,0x29]
+          vcvtusi2ssl (%rcx), %xmm26, %xmm5
+
+// CHECK: vcvtusi2ssl 291(%rax,%r14,8), %xmm26, %xmm5
+// CHECK:  encoding: [0x62,0xb1,0x2e,0x00,0x7b,0xac,0xf0,0x23,0x01,0x00,0x00]
+          vcvtusi2ssl 291(%rax,%r14,8), %xmm26, %xmm5
+
+// CHECK: vcvtusi2ssl 508(%rdx), %xmm26, %xmm5
+// CHECK:  encoding: [0x62,0xf1,0x2e,0x00,0x7b,0x6a,0x7f]
+          vcvtusi2ssl 508(%rdx), %xmm26, %xmm5
+
+// CHECK: vcvtusi2ssl 512(%rdx), %xmm26, %xmm5
+// CHECK:  encoding: [0x62,0xf1,0x2e,0x00,0x7b,0xaa,0x00,0x02,0x00,0x00]
+          vcvtusi2ssl 512(%rdx), %xmm26, %xmm5
+
+// CHECK: vcvtusi2ssl -512(%rdx), %xmm26, %xmm5
+// CHECK:  encoding: [0x62,0xf1,0x2e,0x00,0x7b,0x6a,0x80]
+          vcvtusi2ssl -512(%rdx), %xmm26, %xmm5
+
+// CHECK: vcvtusi2ssl -516(%rdx), %xmm26, %xmm5
+// CHECK:  encoding: [0x62,0xf1,0x2e,0x00,0x7b,0xaa,0xfc,0xfd,0xff,0xff]
+          vcvtusi2ssl -516(%rdx), %xmm26, %xmm5
+
+// CHECK:  vcvtusi2ssq  %rax, %xmm22, %xmm14
+// CHECK:  encoding: [0x62,0x71,0xce,0x00,0x7b,0xf0]
+          vcvtusi2ss %rax, %xmm22, %xmm14
+
+// CHECK:  vcvtusi2ssq  %rax, {rn-sae}, %xmm22, %xmm14
+// CHECK:  encoding: [0x62,0x71,0xce,0x10,0x7b,0xf0]
+          vcvtusi2ss %rax, {rn-sae}, %xmm22, %xmm14
+
+// CHECK:  vcvtusi2ssq  %rax, {ru-sae}, %xmm22, %xmm14
+// CHECK:  encoding: [0x62,0x71,0xce,0x50,0x7b,0xf0]
+          vcvtusi2ss %rax, {ru-sae}, %xmm22, %xmm14
+
+// CHECK:  vcvtusi2ssq  %rax, {rd-sae}, %xmm22, %xmm14
+// CHECK:  encoding: [0x62,0x71,0xce,0x30,0x7b,0xf0]
+          vcvtusi2ss %rax, {rd-sae}, %xmm22, %xmm14
+
+// CHECK:  vcvtusi2ssq  %rax, {rz-sae}, %xmm22, %xmm14
+// CHECK:  encoding: [0x62,0x71,0xce,0x70,0x7b,0xf0]
+          vcvtusi2ss %rax, {rz-sae}, %xmm22, %xmm14
+
+// CHECK:  vcvtusi2ssq  %r8, %xmm22, %xmm14
+// CHECK:  encoding: [0x62,0x51,0xce,0x00,0x7b,0xf0]
+          vcvtusi2ss %r8, %xmm22, %xmm14
+
+// CHECK:  vcvtusi2ssq  %r8, {rn-sae}, %xmm22, %xmm14
+// CHECK:  encoding: [0x62,0x51,0xce,0x10,0x7b,0xf0]
+          vcvtusi2ss %r8, {rn-sae}, %xmm22, %xmm14
+
+// CHECK:  vcvtusi2ssq  %r8, {ru-sae}, %xmm22, %xmm14
+// CHECK:  encoding: [0x62,0x51,0xce,0x50,0x7b,0xf0]
+          vcvtusi2ss %r8, {ru-sae}, %xmm22, %xmm14
+
+// CHECK:  vcvtusi2ssq  %r8, {rd-sae}, %xmm22, %xmm14
+// CHECK:  encoding: [0x62,0x51,0xce,0x30,0x7b,0xf0]
+          vcvtusi2ss %r8, {rd-sae}, %xmm22, %xmm14
+
+// CHECK: vcvtusi2ssq %r8, {rz-sae}, %xmm22, %xmm14
+// CHECK:  encoding: [0x62,0x51,0xce,0x70,0x7b,0xf0]
+          vcvtusi2ss %r8, {rz-sae}, %xmm22, %xmm14
+
+// CHECK: vcvtusi2ssq (%rcx), %xmm22, %xmm14
+// CHECK:  encoding: [0x62,0x71,0xce,0x00,0x7b,0x31]
+          vcvtusi2ssq (%rcx), %xmm22, %xmm14
+
+// CHECK: vcvtusi2ssq 291(%rax,%r14,8), %xmm22, %xmm14
+// CHECK:  encoding: [0x62,0x31,0xce,0x00,0x7b,0xb4,0xf0,0x23,0x01,0x00,0x00]
+          vcvtusi2ssq 291(%rax,%r14,8), %xmm22, %xmm14
+
+// CHECK: vcvtusi2ssq 1016(%rdx), %xmm22, %xmm14
+// CHECK:  encoding: [0x62,0x71,0xce,0x00,0x7b,0x72,0x7f]
+          vcvtusi2ssq 1016(%rdx), %xmm22, %xmm14
+
+// CHECK: vcvtusi2ssq 1024(%rdx), %xmm22, %xmm14
+// CHECK:  encoding: [0x62,0x71,0xce,0x00,0x7b,0xb2,0x00,0x04,0x00,0x00]
+          vcvtusi2ssq 1024(%rdx), %xmm22, %xmm14
+
+// CHECK: vcvtusi2ssq -1024(%rdx), %xmm22, %xmm14
+// CHECK:  encoding: [0x62,0x71,0xce,0x00,0x7b,0x72,0x80]
+          vcvtusi2ssq -1024(%rdx), %xmm22, %xmm14
+
+// CHECK: vcvtusi2ssq -1032(%rdx), %xmm22, %xmm14
+// CHECK:  encoding: [0x62,0x71,0xce,0x00,0x7b,0xb2,0xf8,0xfb,0xff,0xff]
+          vcvtusi2ssq -1032(%rdx), %xmm22, %xmm14
+
+// CHECK: vfmadd132ps %zmm25, %zmm20, %zmm1
+// CHECK:  encoding: [0x62,0x92,0x5d,0x40,0x98,0xc9]
+          vfmadd132ps %zmm25, %zmm20, %zmm1
+
+// CHECK: vfmadd132ps %zmm25, %zmm20, %zmm1 {%k1}
+// CHECK:  encoding: [0x62,0x92,0x5d,0x41,0x98,0xc9]
+          vfmadd132ps %zmm25, %zmm20, %zmm1 {%k1}
+
+// CHECK: vfmadd132ps %zmm25, %zmm20, %zmm1 {%k1} {z}
+// CHECK:  encoding: [0x62,0x92,0x5d,0xc1,0x98,0xc9]
+          vfmadd132ps %zmm25, %zmm20, %zmm1 {%k1} {z}
+
+// CHECK: vfmadd132ps {rn-sae}, %zmm25, %zmm20, %zmm1
+// CHECK:  encoding: [0x62,0x92,0x5d,0x10,0x98,0xc9]
+          vfmadd132ps {rn-sae}, %zmm25, %zmm20, %zmm1
+
+// CHECK: vfmadd132ps {ru-sae}, %zmm25, %zmm20, %zmm1
+// CHECK:  encoding: [0x62,0x92,0x5d,0x50,0x98,0xc9]
+          vfmadd132ps {ru-sae}, %zmm25, %zmm20, %zmm1
+
+// CHECK: vfmadd132ps {rd-sae}, %zmm25, %zmm20, %zmm1
+// CHECK:  encoding: [0x62,0x92,0x5d,0x30,0x98,0xc9]
+          vfmadd132ps {rd-sae}, %zmm25, %zmm20, %zmm1
+
+// CHECK: vfmadd132ps {rz-sae}, %zmm25, %zmm20, %zmm1
+// CHECK:  encoding: [0x62,0x92,0x5d,0x70,0x98,0xc9]
+          vfmadd132ps {rz-sae}, %zmm25, %zmm20, %zmm1
+
+// CHECK: vfmadd132ps (%rcx), %zmm20, %zmm1
+// CHECK:  encoding: [0x62,0xf2,0x5d,0x40,0x98,0x09]
+          vfmadd132ps (%rcx), %zmm20, %zmm1
+
+// CHECK: vfmadd132ps 291(%rax,%r14,8), %zmm20, %zmm1
+// CHECK:  encoding: [0x62,0xb2,0x5d,0x40,0x98,0x8c,0xf0,0x23,0x01,0x00,0x00]
+          vfmadd132ps 291(%rax,%r14,8), %zmm20, %zmm1
+
+// CHECK: vfmadd132ps (%rcx){1to16}, %zmm20, %zmm1
+// CHECK:  encoding: [0x62,0xf2,0x5d,0x50,0x98,0x09]
+          vfmadd132ps (%rcx){1to16}, %zmm20, %zmm1
+
+// CHECK: vfmadd132ps 8128(%rdx), %zmm20, %zmm1
+// CHECK:  encoding: [0x62,0xf2,0x5d,0x40,0x98,0x4a,0x7f]
+          vfmadd132ps 8128(%rdx), %zmm20, %zmm1
+
+// CHECK: vfmadd132ps 8192(%rdx), %zmm20, %zmm1
+// CHECK:  encoding: [0x62,0xf2,0x5d,0x40,0x98,0x8a,0x00,0x20,0x00,0x00]
+          vfmadd132ps 8192(%rdx), %zmm20, %zmm1
+
+// CHECK: vfmadd132ps -8192(%rdx), %zmm20, %zmm1
+// CHECK:  encoding: [0x62,0xf2,0x5d,0x40,0x98,0x4a,0x80]
+          vfmadd132ps -8192(%rdx), %zmm20, %zmm1
+
+// CHECK: vfmadd132ps -8256(%rdx), %zmm20, %zmm1
+// CHECK:  encoding: [0x62,0xf2,0x5d,0x40,0x98,0x8a,0xc0,0xdf,0xff,0xff]
+          vfmadd132ps -8256(%rdx), %zmm20, %zmm1
+
+// CHECK: vfmadd132ps 508(%rdx){1to16}, %zmm20, %zmm1
+// CHECK:  encoding: [0x62,0xf2,0x5d,0x50,0x98,0x4a,0x7f]
+          vfmadd132ps 508(%rdx){1to16}, %zmm20, %zmm1
+
+// CHECK: vfmadd132ps 512(%rdx){1to16}, %zmm20, %zmm1
+// CHECK:  encoding: [0x62,0xf2,0x5d,0x50,0x98,0x8a,0x00,0x02,0x00,0x00]
+          vfmadd132ps 512(%rdx){1to16}, %zmm20, %zmm1
+
+// CHECK: vfmadd132ps -512(%rdx){1to16}, %zmm20, %zmm1
+// CHECK:  encoding: [0x62,0xf2,0x5d,0x50,0x98,0x4a,0x80]
+          vfmadd132ps -512(%rdx){1to16}, %zmm20, %zmm1
+
+// CHECK: vfmadd132ps -516(%rdx){1to16}, %zmm20, %zmm1
+// CHECK:  encoding: [0x62,0xf2,0x5d,0x50,0x98,0x8a,0xfc,0xfd,0xff,0xff]
+          vfmadd132ps -516(%rdx){1to16}, %zmm20, %zmm1
+
+// CHECK: vfmadd132pd %zmm21, %zmm16, %zmm26
+// CHECK:  encoding: [0x62,0x22,0xfd,0x40,0x98,0xd5]
+          vfmadd132pd %zmm21, %zmm16, %zmm26
+
+// CHECK: vfmadd132pd %zmm21, %zmm16, %zmm26 {%k5}
+// CHECK:  encoding: [0x62,0x22,0xfd,0x45,0x98,0xd5]
+          vfmadd132pd %zmm21, %zmm16, %zmm26 {%k5}
+
+// CHECK: vfmadd132pd %zmm21, %zmm16, %zmm26 {%k5} {z}
+// CHECK:  encoding: [0x62,0x22,0xfd,0xc5,0x98,0xd5]
+          vfmadd132pd %zmm21, %zmm16, %zmm26 {%k5} {z}
+
+// CHECK: vfmadd132pd {rn-sae}, %zmm21, %zmm16, %zmm26
+// CHECK:  encoding: [0x62,0x22,0xfd,0x10,0x98,0xd5]
+          vfmadd132pd {rn-sae}, %zmm21, %zmm16, %zmm26
+
+// CHECK: vfmadd132pd {ru-sae}, %zmm21, %zmm16, %zmm26
+// CHECK:  encoding: [0x62,0x22,0xfd,0x50,0x98,0xd5]
+          vfmadd132pd {ru-sae}, %zmm21, %zmm16, %zmm26
+
+// CHECK: vfmadd132pd {rd-sae}, %zmm21, %zmm16, %zmm26
+// CHECK:  encoding: [0x62,0x22,0xfd,0x30,0x98,0xd5]
+          vfmadd132pd {rd-sae}, %zmm21, %zmm16, %zmm26
+
+// CHECK: vfmadd132pd {rz-sae}, %zmm21, %zmm16, %zmm26
+// CHECK:  encoding: [0x62,0x22,0xfd,0x70,0x98,0xd5]
+          vfmadd132pd {rz-sae}, %zmm21, %zmm16, %zmm26
+
+// CHECK: vfmadd132pd (%rcx), %zmm16, %zmm26
+// CHECK:  encoding: [0x62,0x62,0xfd,0x40,0x98,0x11]
+          vfmadd132pd (%rcx), %zmm16, %zmm26
+
+// CHECK: vfmadd132pd 291(%rax,%r14,8), %zmm16, %zmm26
+// CHECK:  encoding: [0x62,0x22,0xfd,0x40,0x98,0x94,0xf0,0x23,0x01,0x00,0x00]
+          vfmadd132pd 291(%rax,%r14,8), %zmm16, %zmm26
+
+// CHECK: vfmadd132pd (%rcx){1to8}, %zmm16, %zmm26
+// CHECK:  encoding: [0x62,0x62,0xfd,0x50,0x98,0x11]
+          vfmadd132pd (%rcx){1to8}, %zmm16, %zmm26
+
+// CHECK: vfmadd132pd 8128(%rdx), %zmm16, %zmm26
+// CHECK:  encoding: [0x62,0x62,0xfd,0x40,0x98,0x52,0x7f]
+          vfmadd132pd 8128(%rdx), %zmm16, %zmm26
+
+// CHECK: vfmadd132pd 8192(%rdx), %zmm16, %zmm26
+// CHECK:  encoding: [0x62,0x62,0xfd,0x40,0x98,0x92,0x00,0x20,0x00,0x00]
+          vfmadd132pd 8192(%rdx), %zmm16, %zmm26
+
+// CHECK: vfmadd132pd -8192(%rdx), %zmm16, %zmm26
+// CHECK:  encoding: [0x62,0x62,0xfd,0x40,0x98,0x52,0x80]
+          vfmadd132pd -8192(%rdx), %zmm16, %zmm26
+
+// CHECK: vfmadd132pd -8256(%rdx), %zmm16, %zmm26
+// CHECK:  encoding: [0x62,0x62,0xfd,0x40,0x98,0x92,0xc0,0xdf,0xff,0xff]
+          vfmadd132pd -8256(%rdx), %zmm16, %zmm26
+
+// CHECK: vfmadd132pd 1016(%rdx){1to8}, %zmm16, %zmm26
+// CHECK:  encoding: [0x62,0x62,0xfd,0x50,0x98,0x52,0x7f]
+          vfmadd132pd 1016(%rdx){1to8}, %zmm16, %zmm26
+
+// CHECK: vfmadd132pd 1024(%rdx){1to8}, %zmm16, %zmm26
+// CHECK:  encoding: [0x62,0x62,0xfd,0x50,0x98,0x92,0x00,0x04,0x00,0x00]
+          vfmadd132pd 1024(%rdx){1to8}, %zmm16, %zmm26
+
+// CHECK: vfmadd132pd -1024(%rdx){1to8}, %zmm16, %zmm26
+// CHECK:  encoding: [0x62,0x62,0xfd,0x50,0x98,0x52,0x80]
+          vfmadd132pd -1024(%rdx){1to8}, %zmm16, %zmm26
+
+// CHECK: vfmadd132pd -1032(%rdx){1to8}, %zmm16, %zmm26
+// CHECK:  encoding: [0x62,0x62,0xfd,0x50,0x98,0x92,0xf8,0xfb,0xff,0xff]
+          vfmadd132pd -1032(%rdx){1to8}, %zmm16, %zmm26
+
+// CHECK: vfmadd213ps %zmm14, %zmm19, %zmm20
+// CHECK:  encoding: [0x62,0xc2,0x65,0x40,0xa8,0xe6]
+          vfmadd213ps %zmm14, %zmm19, %zmm20
+
+// CHECK: vfmadd213ps %zmm14, %zmm19, %zmm20 {%k4}
+// CHECK:  encoding: [0x62,0xc2,0x65,0x44,0xa8,0xe6]
+          vfmadd213ps %zmm14, %zmm19, %zmm20 {%k4}
+
+// CHECK: vfmadd213ps %zmm14, %zmm19, %zmm20 {%k4} {z}
+// CHECK:  encoding: [0x62,0xc2,0x65,0xc4,0xa8,0xe6]
+          vfmadd213ps %zmm14, %zmm19, %zmm20 {%k4} {z}
+
+// CHECK: vfmadd213ps {rn-sae}, %zmm14, %zmm19, %zmm20
+// CHECK:  encoding: [0x62,0xc2,0x65,0x10,0xa8,0xe6]
+          vfmadd213ps {rn-sae}, %zmm14, %zmm19, %zmm20
+
+// CHECK: vfmadd213ps {ru-sae}, %zmm14, %zmm19, %zmm20
+// CHECK:  encoding: [0x62,0xc2,0x65,0x50,0xa8,0xe6]
+          vfmadd213ps {ru-sae}, %zmm14, %zmm19, %zmm20
+
+// CHECK: vfmadd213ps {rd-sae}, %zmm14, %zmm19, %zmm20
+// CHECK:  encoding: [0x62,0xc2,0x65,0x30,0xa8,0xe6]
+          vfmadd213ps {rd-sae}, %zmm14, %zmm19, %zmm20
+
+// CHECK: vfmadd213ps {rz-sae}, %zmm14, %zmm19, %zmm20
+// CHECK:  encoding: [0x62,0xc2,0x65,0x70,0xa8,0xe6]
+          vfmadd213ps {rz-sae}, %zmm14, %zmm19, %zmm20
+
+// CHECK: vfmadd213ps (%rcx), %zmm19, %zmm20
+// CHECK:  encoding: [0x62,0xe2,0x65,0x40,0xa8,0x21]
+          vfmadd213ps (%rcx), %zmm19, %zmm20
+
+// CHECK: vfmadd213ps 291(%rax,%r14,8), %zmm19, %zmm20
+// CHECK:  encoding: [0x62,0xa2,0x65,0x40,0xa8,0xa4,0xf0,0x23,0x01,0x00,0x00]
+          vfmadd213ps 291(%rax,%r14,8), %zmm19, %zmm20
+
+// CHECK: vfmadd213ps (%rcx){1to16}, %zmm19, %zmm20
+// CHECK:  encoding: [0x62,0xe2,0x65,0x50,0xa8,0x21]
+          vfmadd213ps (%rcx){1to16}, %zmm19, %zmm20
+
+// CHECK: vfmadd213ps 8128(%rdx), %zmm19, %zmm20
+// CHECK:  encoding: [0x62,0xe2,0x65,0x40,0xa8,0x62,0x7f]
+          vfmadd213ps 8128(%rdx), %zmm19, %zmm20
+
+// CHECK: vfmadd213ps 8192(%rdx), %zmm19, %zmm20
+// CHECK:  encoding: [0x62,0xe2,0x65,0x40,0xa8,0xa2,0x00,0x20,0x00,0x00]
+          vfmadd213ps 8192(%rdx), %zmm19, %zmm20
+
+// CHECK: vfmadd213ps -8192(%rdx), %zmm19, %zmm20
+// CHECK:  encoding: [0x62,0xe2,0x65,0x40,0xa8,0x62,0x80]
+          vfmadd213ps -8192(%rdx), %zmm19, %zmm20
+
+// CHECK: vfmadd213ps -8256(%rdx), %zmm19, %zmm20
+// CHECK:  encoding: [0x62,0xe2,0x65,0x40,0xa8,0xa2,0xc0,0xdf,0xff,0xff]
+          vfmadd213ps -8256(%rdx), %zmm19, %zmm20
+
+// CHECK: vfmadd213ps 508(%rdx){1to16}, %zmm19, %zmm20
+// CHECK:  encoding: [0x62,0xe2,0x65,0x50,0xa8,0x62,0x7f]
+          vfmadd213ps 508(%rdx){1to16}, %zmm19, %zmm20
+
+// CHECK: vfmadd213ps 512(%rdx){1to16}, %zmm19, %zmm20
+// CHECK:  encoding: [0x62,0xe2,0x65,0x50,0xa8,0xa2,0x00,0x02,0x00,0x00]
+          vfmadd213ps 512(%rdx){1to16}, %zmm19, %zmm20
+
+// CHECK: vfmadd213ps -512(%rdx){1to16}, %zmm19, %zmm20
+// CHECK:  encoding: [0x62,0xe2,0x65,0x50,0xa8,0x62,0x80]
+          vfmadd213ps -512(%rdx){1to16}, %zmm19, %zmm20
+
+// CHECK: vfmadd213ps -516(%rdx){1to16}, %zmm19, %zmm20
+// CHECK:  encoding: [0x62,0xe2,0x65,0x50,0xa8,0xa2,0xfc,0xfd,0xff,0xff]
+          vfmadd213ps -516(%rdx){1to16}, %zmm19, %zmm20
+
+// CHECK: vfmadd213pd %zmm25, %zmm16, %zmm18
+// CHECK:  encoding: [0x62,0x82,0xfd,0x40,0xa8,0xd1]
+          vfmadd213pd %zmm25, %zmm16, %zmm18
+
+// CHECK: vfmadd213pd %zmm25, %zmm16, %zmm18 {%k3}
+// CHECK:  encoding: [0x62,0x82,0xfd,0x43,0xa8,0xd1]
+          vfmadd213pd %zmm25, %zmm16, %zmm18 {%k3}
+
+// CHECK: vfmadd213pd %zmm25, %zmm16, %zmm18 {%k3} {z}
+// CHECK:  encoding: [0x62,0x82,0xfd,0xc3,0xa8,0xd1]
+          vfmadd213pd %zmm25, %zmm16, %zmm18 {%k3} {z}
+
+// CHECK: vfmadd213pd {rn-sae}, %zmm25, %zmm16, %zmm18
+// CHECK:  encoding: [0x62,0x82,0xfd,0x10,0xa8,0xd1]
+          vfmadd213pd {rn-sae}, %zmm25, %zmm16, %zmm18
+
+// CHECK: vfmadd213pd {ru-sae}, %zmm25, %zmm16, %zmm18
+// CHECK:  encoding: [0x62,0x82,0xfd,0x50,0xa8,0xd1]
+          vfmadd213pd {ru-sae}, %zmm25, %zmm16, %zmm18
+
+// CHECK: vfmadd213pd {rd-sae}, %zmm25, %zmm16, %zmm18
+// CHECK:  encoding: [0x62,0x82,0xfd,0x30,0xa8,0xd1]
+          vfmadd213pd {rd-sae}, %zmm25, %zmm16, %zmm18
+
+// CHECK: vfmadd213pd {rz-sae}, %zmm25, %zmm16, %zmm18
+// CHECK:  encoding: [0x62,0x82,0xfd,0x70,0xa8,0xd1]
+          vfmadd213pd {rz-sae}, %zmm25, %zmm16, %zmm18
+
+// CHECK: vfmadd213pd (%rcx), %zmm16, %zmm18
+// CHECK:  encoding: [0x62,0xe2,0xfd,0x40,0xa8,0x11]
+          vfmadd213pd (%rcx), %zmm16, %zmm18
+
+// CHECK: vfmadd213pd 291(%rax,%r14,8), %zmm16, %zmm18
+// CHECK:  encoding: [0x62,0xa2,0xfd,0x40,0xa8,0x94,0xf0,0x23,0x01,0x00,0x00]
+          vfmadd213pd 291(%rax,%r14,8), %zmm16, %zmm18
+
+// CHECK: vfmadd213pd (%rcx){1to8}, %zmm16, %zmm18
+// CHECK:  encoding: [0x62,0xe2,0xfd,0x50,0xa8,0x11]
+          vfmadd213pd (%rcx){1to8}, %zmm16, %zmm18
+
+// CHECK: vfmadd213pd 8128(%rdx), %zmm16, %zmm18
+// CHECK:  encoding: [0x62,0xe2,0xfd,0x40,0xa8,0x52,0x7f]
+          vfmadd213pd 8128(%rdx), %zmm16, %zmm18
+
+// CHECK: vfmadd213pd 8192(%rdx), %zmm16, %zmm18
+// CHECK:  encoding: [0x62,0xe2,0xfd,0x40,0xa8,0x92,0x00,0x20,0x00,0x00]
+          vfmadd213pd 8192(%rdx), %zmm16, %zmm18
+
+// CHECK: vfmadd213pd -8192(%rdx), %zmm16, %zmm18
+// CHECK:  encoding: [0x62,0xe2,0xfd,0x40,0xa8,0x52,0x80]
+          vfmadd213pd -8192(%rdx), %zmm16, %zmm18
+
+// CHECK: vfmadd213pd -8256(%rdx), %zmm16, %zmm18
+// CHECK:  encoding: [0x62,0xe2,0xfd,0x40,0xa8,0x92,0xc0,0xdf,0xff,0xff]
+          vfmadd213pd -8256(%rdx), %zmm16, %zmm18
+
+// CHECK: vfmadd213pd 1016(%rdx){1to8}, %zmm16, %zmm18
+// CHECK:  encoding: [0x62,0xe2,0xfd,0x50,0xa8,0x52,0x7f]
+          vfmadd213pd 1016(%rdx){1to8}, %zmm16, %zmm18
+
+// CHECK: vfmadd213pd 1024(%rdx){1to8}, %zmm16, %zmm18
+// CHECK:  encoding: [0x62,0xe2,0xfd,0x50,0xa8,0x92,0x00,0x04,0x00,0x00]
+          vfmadd213pd 1024(%rdx){1to8}, %zmm16, %zmm18
+
+// CHECK: vfmadd213pd -1024(%rdx){1to8}, %zmm16, %zmm18
+// CHECK:  encoding: [0x62,0xe2,0xfd,0x50,0xa8,0x52,0x80]
+          vfmadd213pd -1024(%rdx){1to8}, %zmm16, %zmm18
+
+// CHECK: vfmadd213pd -1032(%rdx){1to8}, %zmm16, %zmm18
+// CHECK:  encoding: [0x62,0xe2,0xfd,0x50,0xa8,0x92,0xf8,0xfb,0xff,0xff]
+          vfmadd213pd -1032(%rdx){1to8}, %zmm16, %zmm18
+
+// CHECK: vfmadd231ps %zmm25, %zmm28, %zmm27
+// CHECK:  encoding: [0x62,0x02,0x1d,0x40,0xb8,0xd9]
+          vfmadd231ps %zmm25, %zmm28, %zmm27
+
+// CHECK: vfmadd231ps %zmm25, %zmm28, %zmm27 {%k3}
+// CHECK:  encoding: [0x62,0x02,0x1d,0x43,0xb8,0xd9]
+          vfmadd231ps %zmm25, %zmm28, %zmm27 {%k3}
+
+// CHECK: vfmadd231ps %zmm25, %zmm28, %zmm27 {%k3} {z}
+// CHECK:  encoding: [0x62,0x02,0x1d,0xc3,0xb8,0xd9]
+          vfmadd231ps %zmm25, %zmm28, %zmm27 {%k3} {z}
+
+// CHECK: vfmadd231ps {rn-sae}, %zmm25, %zmm28, %zmm27
+// CHECK:  encoding: [0x62,0x02,0x1d,0x10,0xb8,0xd9]
+          vfmadd231ps {rn-sae}, %zmm25, %zmm28, %zmm27
+
+// CHECK: vfmadd231ps {ru-sae}, %zmm25, %zmm28, %zmm27
+// CHECK:  encoding: [0x62,0x02,0x1d,0x50,0xb8,0xd9]
+          vfmadd231ps {ru-sae}, %zmm25, %zmm28, %zmm27
+
+// CHECK: vfmadd231ps {rd-sae}, %zmm25, %zmm28, %zmm27
+// CHECK:  encoding: [0x62,0x02,0x1d,0x30,0xb8,0xd9]
+          vfmadd231ps {rd-sae}, %zmm25, %zmm28, %zmm27
+
+// CHECK: vfmadd231ps {rz-sae}, %zmm25, %zmm28, %zmm27
+// CHECK:  encoding: [0x62,0x02,0x1d,0x70,0xb8,0xd9]
+          vfmadd231ps {rz-sae}, %zmm25, %zmm28, %zmm27
+
+// CHECK: vfmadd231ps (%rcx), %zmm28, %zmm27
+// CHECK:  encoding: [0x62,0x62,0x1d,0x40,0xb8,0x19]
+          vfmadd231ps (%rcx), %zmm28, %zmm27
+
+// CHECK: vfmadd231ps 291(%rax,%r14,8), %zmm28, %zmm27
+// CHECK:  encoding: [0x62,0x22,0x1d,0x40,0xb8,0x9c,0xf0,0x23,0x01,0x00,0x00]
+          vfmadd231ps 291(%rax,%r14,8), %zmm28, %zmm27
+
+// CHECK: vfmadd231ps (%rcx){1to16}, %zmm28, %zmm27
+// CHECK:  encoding: [0x62,0x62,0x1d,0x50,0xb8,0x19]
+          vfmadd231ps (%rcx){1to16}, %zmm28, %zmm27
+
+// CHECK: vfmadd231ps 8128(%rdx), %zmm28, %zmm27
+// CHECK:  encoding: [0x62,0x62,0x1d,0x40,0xb8,0x5a,0x7f]
+          vfmadd231ps 8128(%rdx), %zmm28, %zmm27
+
+// CHECK: vfmadd231ps 8192(%rdx), %zmm28, %zmm27
+// CHECK:  encoding: [0x62,0x62,0x1d,0x40,0xb8,0x9a,0x00,0x20,0x00,0x00]
+          vfmadd231ps 8192(%rdx), %zmm28, %zmm27
+
+// CHECK: vfmadd231ps -8192(%rdx), %zmm28, %zmm27
+// CHECK:  encoding: [0x62,0x62,0x1d,0x40,0xb8,0x5a,0x80]
+          vfmadd231ps -8192(%rdx), %zmm28, %zmm27
+
+// CHECK: vfmadd231ps -8256(%rdx), %zmm28, %zmm27
+// CHECK:  encoding: [0x62,0x62,0x1d,0x40,0xb8,0x9a,0xc0,0xdf,0xff,0xff]
+          vfmadd231ps -8256(%rdx), %zmm28, %zmm27
+
+// CHECK: vfmadd231ps 508(%rdx){1to16}, %zmm28, %zmm27
+// CHECK:  encoding: [0x62,0x62,0x1d,0x50,0xb8,0x5a,0x7f]
+          vfmadd231ps 508(%rdx){1to16}, %zmm28, %zmm27
+
+// CHECK: vfmadd231ps 512(%rdx){1to16}, %zmm28, %zmm27
+// CHECK:  encoding: [0x62,0x62,0x1d,0x50,0xb8,0x9a,0x00,0x02,0x00,0x00]
+          vfmadd231ps 512(%rdx){1to16}, %zmm28, %zmm27
+
+// CHECK: vfmadd231ps -512(%rdx){1to16}, %zmm28, %zmm27
+// CHECK:  encoding: [0x62,0x62,0x1d,0x50,0xb8,0x5a,0x80]
+          vfmadd231ps -512(%rdx){1to16}, %zmm28, %zmm27
+
+// CHECK: vfmadd231ps -516(%rdx){1to16}, %zmm28, %zmm27
+// CHECK:  encoding: [0x62,0x62,0x1d,0x50,0xb8,0x9a,0xfc,0xfd,0xff,0xff]
+          vfmadd231ps -516(%rdx){1to16}, %zmm28, %zmm27
+
+// CHECK: vfmadd231pd %zmm9, %zmm6, %zmm30
+// CHECK:  encoding: [0x62,0x42,0xcd,0x48,0xb8,0xf1]
+          vfmadd231pd %zmm9, %zmm6, %zmm30
+
+// CHECK: vfmadd231pd %zmm9, %zmm6, %zmm30 {%k4}
+// CHECK:  encoding: [0x62,0x42,0xcd,0x4c,0xb8,0xf1]
+          vfmadd231pd %zmm9, %zmm6, %zmm30 {%k4}
+
+// CHECK: vfmadd231pd %zmm9, %zmm6, %zmm30 {%k4} {z}
+// CHECK:  encoding: [0x62,0x42,0xcd,0xcc,0xb8,0xf1]
+          vfmadd231pd %zmm9, %zmm6, %zmm30 {%k4} {z}
+
+// CHECK: vfmadd231pd {rn-sae}, %zmm9, %zmm6, %zmm30
+// CHECK:  encoding: [0x62,0x42,0xcd,0x18,0xb8,0xf1]
+          vfmadd231pd {rn-sae}, %zmm9, %zmm6, %zmm30
+
+// CHECK: vfmadd231pd {ru-sae}, %zmm9, %zmm6, %zmm30
+// CHECK:  encoding: [0x62,0x42,0xcd,0x58,0xb8,0xf1]
+          vfmadd231pd {ru-sae}, %zmm9, %zmm6, %zmm30
+
+// CHECK: vfmadd231pd {rd-sae}, %zmm9, %zmm6, %zmm30
+// CHECK:  encoding: [0x62,0x42,0xcd,0x38,0xb8,0xf1]
+          vfmadd231pd {rd-sae}, %zmm9, %zmm6, %zmm30
+
+// CHECK: vfmadd231pd {rz-sae}, %zmm9, %zmm6, %zmm30
+// CHECK:  encoding: [0x62,0x42,0xcd,0x78,0xb8,0xf1]
+          vfmadd231pd {rz-sae}, %zmm9, %zmm6, %zmm30
+
+// CHECK: vfmadd231pd (%rcx), %zmm6, %zmm30
+// CHECK:  encoding: [0x62,0x62,0xcd,0x48,0xb8,0x31]
+          vfmadd231pd (%rcx), %zmm6, %zmm30
+
+// CHECK: vfmadd231pd 291(%rax,%r14,8), %zmm6, %zmm30
+// CHECK:  encoding: [0x62,0x22,0xcd,0x48,0xb8,0xb4,0xf0,0x23,0x01,0x00,0x00]
+          vfmadd231pd 291(%rax,%r14,8), %zmm6, %zmm30
+
+// CHECK: vfmadd231pd (%rcx){1to8}, %zmm6, %zmm30
+// CHECK:  encoding: [0x62,0x62,0xcd,0x58,0xb8,0x31]
+          vfmadd231pd (%rcx){1to8}, %zmm6, %zmm30
+
+// CHECK: vfmadd231pd 8128(%rdx), %zmm6, %zmm30
+// CHECK:  encoding: [0x62,0x62,0xcd,0x48,0xb8,0x72,0x7f]
+          vfmadd231pd 8128(%rdx), %zmm6, %zmm30
+
+// CHECK: vfmadd231pd 8192(%rdx), %zmm6, %zmm30
+// CHECK:  encoding: [0x62,0x62,0xcd,0x48,0xb8,0xb2,0x00,0x20,0x00,0x00]
+          vfmadd231pd 8192(%rdx), %zmm6, %zmm30
+
+// CHECK: vfmadd231pd -8192(%rdx), %zmm6, %zmm30
+// CHECK:  encoding: [0x62,0x62,0xcd,0x48,0xb8,0x72,0x80]
+          vfmadd231pd -8192(%rdx), %zmm6, %zmm30
+
+// CHECK: vfmadd231pd -8256(%rdx), %zmm6, %zmm30
+// CHECK:  encoding: [0x62,0x62,0xcd,0x48,0xb8,0xb2,0xc0,0xdf,0xff,0xff]
+          vfmadd231pd -8256(%rdx), %zmm6, %zmm30
+
+// CHECK: vfmadd231pd 1016(%rdx){1to8}, %zmm6, %zmm30
+// CHECK:  encoding: [0x62,0x62,0xcd,0x58,0xb8,0x72,0x7f]
+          vfmadd231pd 1016(%rdx){1to8}, %zmm6, %zmm30
+
+// CHECK: vfmadd231pd 1024(%rdx){1to8}, %zmm6, %zmm30
+// CHECK:  encoding: [0x62,0x62,0xcd,0x58,0xb8,0xb2,0x00,0x04,0x00,0x00]
+          vfmadd231pd 1024(%rdx){1to8}, %zmm6, %zmm30
+
+// CHECK: vfmadd231pd -1024(%rdx){1to8}, %zmm6, %zmm30
+// CHECK:  encoding: [0x62,0x62,0xcd,0x58,0xb8,0x72,0x80]
+          vfmadd231pd -1024(%rdx){1to8}, %zmm6, %zmm30
+
+// CHECK: vfmadd231pd -1032(%rdx){1to8}, %zmm6, %zmm30
+// CHECK:  encoding: [0x62,0x62,0xcd,0x58,0xb8,0xb2,0xf8,0xfb,0xff,0xff]
+          vfmadd231pd -1032(%rdx){1to8}, %zmm6, %zmm30
+
+// CHECK: vfmsub132ps %zmm16, %zmm13, %zmm1
+// CHECK:  encoding: [0x62,0xb2,0x15,0x48,0x9a,0xc8]
+          vfmsub132ps %zmm16, %zmm13, %zmm1
+
+// CHECK: vfmsub132ps %zmm16, %zmm13, %zmm1 {%k4}
+// CHECK:  encoding: [0x62,0xb2,0x15,0x4c,0x9a,0xc8]
+          vfmsub132ps %zmm16, %zmm13, %zmm1 {%k4}
+
+// CHECK: vfmsub132ps %zmm16, %zmm13, %zmm1 {%k4} {z}
+// CHECK:  encoding: [0x62,0xb2,0x15,0xcc,0x9a,0xc8]
+          vfmsub132ps %zmm16, %zmm13, %zmm1 {%k4} {z}
+
+// CHECK: vfmsub132ps {rn-sae}, %zmm16, %zmm13, %zmm1
+// CHECK:  encoding: [0x62,0xb2,0x15,0x18,0x9a,0xc8]
+          vfmsub132ps {rn-sae}, %zmm16, %zmm13, %zmm1
+
+// CHECK: vfmsub132ps {ru-sae}, %zmm16, %zmm13, %zmm1
+// CHECK:  encoding: [0x62,0xb2,0x15,0x58,0x9a,0xc8]
+          vfmsub132ps {ru-sae}, %zmm16, %zmm13, %zmm1
+
+// CHECK: vfmsub132ps {rd-sae}, %zmm16, %zmm13, %zmm1
+// CHECK:  encoding: [0x62,0xb2,0x15,0x38,0x9a,0xc8]
+          vfmsub132ps {rd-sae}, %zmm16, %zmm13, %zmm1
+
+// CHECK: vfmsub132ps {rz-sae}, %zmm16, %zmm13, %zmm1
+// CHECK:  encoding: [0x62,0xb2,0x15,0x78,0x9a,0xc8]
+          vfmsub132ps {rz-sae}, %zmm16, %zmm13, %zmm1
+
+// CHECK: vfmsub132ps (%rcx), %zmm13, %zmm1
+// CHECK:  encoding: [0x62,0xf2,0x15,0x48,0x9a,0x09]
+          vfmsub132ps (%rcx), %zmm13, %zmm1
+
+// CHECK: vfmsub132ps 291(%rax,%r14,8), %zmm13, %zmm1
+// CHECK:  encoding: [0x62,0xb2,0x15,0x48,0x9a,0x8c,0xf0,0x23,0x01,0x00,0x00]
+          vfmsub132ps 291(%rax,%r14,8), %zmm13, %zmm1
+
+// CHECK: vfmsub132ps (%rcx){1to16}, %zmm13, %zmm1
+// CHECK:  encoding: [0x62,0xf2,0x15,0x58,0x9a,0x09]
+          vfmsub132ps (%rcx){1to16}, %zmm13, %zmm1
+
+// CHECK: vfmsub132ps 8128(%rdx), %zmm13, %zmm1
+// CHECK:  encoding: [0x62,0xf2,0x15,0x48,0x9a,0x4a,0x7f]
+          vfmsub132ps 8128(%rdx), %zmm13, %zmm1
+
+// CHECK: vfmsub132ps 8192(%rdx), %zmm13, %zmm1
+// CHECK:  encoding: [0x62,0xf2,0x15,0x48,0x9a,0x8a,0x00,0x20,0x00,0x00]
+          vfmsub132ps 8192(%rdx), %zmm13, %zmm1
+
+// CHECK: vfmsub132ps -8192(%rdx), %zmm13, %zmm1
+// CHECK:  encoding: [0x62,0xf2,0x15,0x48,0x9a,0x4a,0x80]
+          vfmsub132ps -8192(%rdx), %zmm13, %zmm1
+
+// CHECK: vfmsub132ps -8256(%rdx), %zmm13, %zmm1
+// CHECK:  encoding: [0x62,0xf2,0x15,0x48,0x9a,0x8a,0xc0,0xdf,0xff,0xff]
+          vfmsub132ps -8256(%rdx), %zmm13, %zmm1
+
+// CHECK: vfmsub132ps 508(%rdx){1to16}, %zmm13, %zmm1
+// CHECK:  encoding: [0x62,0xf2,0x15,0x58,0x9a,0x4a,0x7f]
+          vfmsub132ps 508(%rdx){1to16}, %zmm13, %zmm1
+
+// CHECK: vfmsub132ps 512(%rdx){1to16}, %zmm13, %zmm1
+// CHECK:  encoding: [0x62,0xf2,0x15,0x58,0x9a,0x8a,0x00,0x02,0x00,0x00]
+          vfmsub132ps 512(%rdx){1to16}, %zmm13, %zmm1
+
+// CHECK: vfmsub132ps -512(%rdx){1to16}, %zmm13, %zmm1
+// CHECK:  encoding: [0x62,0xf2,0x15,0x58,0x9a,0x4a,0x80]
+          vfmsub132ps -512(%rdx){1to16}, %zmm13, %zmm1
+
+// CHECK: vfmsub132ps -516(%rdx){1to16}, %zmm13, %zmm1
+// CHECK:  encoding: [0x62,0xf2,0x15,0x58,0x9a,0x8a,0xfc,0xfd,0xff,0xff]
+          vfmsub132ps -516(%rdx){1to16}, %zmm13, %zmm1
+
+// CHECK: vfmsub132pd %zmm27, %zmm12, %zmm22
+// CHECK:  encoding: [0x62,0x82,0x9d,0x48,0x9a,0xf3]
+          vfmsub132pd %zmm27, %zmm12, %zmm22
+
+// CHECK: vfmsub132pd %zmm27, %zmm12, %zmm22 {%k2}
+// CHECK:  encoding: [0x62,0x82,0x9d,0x4a,0x9a,0xf3]
+          vfmsub132pd %zmm27, %zmm12, %zmm22 {%k2}
+
+// CHECK: vfmsub132pd %zmm27, %zmm12, %zmm22 {%k2} {z}
+// CHECK:  encoding: [0x62,0x82,0x9d,0xca,0x9a,0xf3]
+          vfmsub132pd %zmm27, %zmm12, %zmm22 {%k2} {z}
+
+// CHECK: vfmsub132pd {rn-sae}, %zmm27, %zmm12, %zmm22
+// CHECK:  encoding: [0x62,0x82,0x9d,0x18,0x9a,0xf3]
+          vfmsub132pd {rn-sae}, %zmm27, %zmm12, %zmm22
+
+// CHECK: vfmsub132pd {ru-sae}, %zmm27, %zmm12, %zmm22
+// CHECK:  encoding: [0x62,0x82,0x9d,0x58,0x9a,0xf3]
+          vfmsub132pd {ru-sae}, %zmm27, %zmm12, %zmm22
+
+// CHECK: vfmsub132pd {rd-sae}, %zmm27, %zmm12, %zmm22
+// CHECK:  encoding: [0x62,0x82,0x9d,0x38,0x9a,0xf3]
+          vfmsub132pd {rd-sae}, %zmm27, %zmm12, %zmm22
+
+// CHECK: vfmsub132pd {rz-sae}, %zmm27, %zmm12, %zmm22
+// CHECK:  encoding: [0x62,0x82,0x9d,0x78,0x9a,0xf3]
+          vfmsub132pd {rz-sae}, %zmm27, %zmm12, %zmm22
+
+// CHECK: vfmsub132pd (%rcx), %zmm12, %zmm22
+// CHECK:  encoding: [0x62,0xe2,0x9d,0x48,0x9a,0x31]
+          vfmsub132pd (%rcx), %zmm12, %zmm22
+
+// CHECK: vfmsub132pd 291(%rax,%r14,8), %zmm12, %zmm22
+// CHECK:  encoding: [0x62,0xa2,0x9d,0x48,0x9a,0xb4,0xf0,0x23,0x01,0x00,0x00]
+          vfmsub132pd 291(%rax,%r14,8), %zmm12, %zmm22
+
+// CHECK: vfmsub132pd (%rcx){1to8}, %zmm12, %zmm22
+// CHECK:  encoding: [0x62,0xe2,0x9d,0x58,0x9a,0x31]
+          vfmsub132pd (%rcx){1to8}, %zmm12, %zmm22
+
+// CHECK: vfmsub132pd 8128(%rdx), %zmm12, %zmm22
+// CHECK:  encoding: [0x62,0xe2,0x9d,0x48,0x9a,0x72,0x7f]
+          vfmsub132pd 8128(%rdx), %zmm12, %zmm22
+
+// CHECK: vfmsub132pd 8192(%rdx), %zmm12, %zmm22
+// CHECK:  encoding: [0x62,0xe2,0x9d,0x48,0x9a,0xb2,0x00,0x20,0x00,0x00]
+          vfmsub132pd 8192(%rdx), %zmm12, %zmm22
+
+// CHECK: vfmsub132pd -8192(%rdx), %zmm12, %zmm22
+// CHECK:  encoding: [0x62,0xe2,0x9d,0x48,0x9a,0x72,0x80]
+          vfmsub132pd -8192(%rdx), %zmm12, %zmm22
+
+// CHECK: vfmsub132pd -8256(%rdx), %zmm12, %zmm22
+// CHECK:  encoding: [0x62,0xe2,0x9d,0x48,0x9a,0xb2,0xc0,0xdf,0xff,0xff]
+          vfmsub132pd -8256(%rdx), %zmm12, %zmm22
+
+// CHECK: vfmsub132pd 1016(%rdx){1to8}, %zmm12, %zmm22
+// CHECK:  encoding: [0x62,0xe2,0x9d,0x58,0x9a,0x72,0x7f]
+          vfmsub132pd 1016(%rdx){1to8}, %zmm12, %zmm22
+
+// CHECK: vfmsub132pd 1024(%rdx){1to8}, %zmm12, %zmm22
+// CHECK:  encoding: [0x62,0xe2,0x9d,0x58,0x9a,0xb2,0x00,0x04,0x00,0x00]
+          vfmsub132pd 1024(%rdx){1to8}, %zmm12, %zmm22
+
+// CHECK: vfmsub132pd -1024(%rdx){1to8}, %zmm12, %zmm22
+// CHECK:  encoding: [0x62,0xe2,0x9d,0x58,0x9a,0x72,0x80]
+          vfmsub132pd -1024(%rdx){1to8}, %zmm12, %zmm22
+
+// CHECK: vfmsub132pd -1032(%rdx){1to8}, %zmm12, %zmm22
+// CHECK:  encoding: [0x62,0xe2,0x9d,0x58,0x9a,0xb2,0xf8,0xfb,0xff,0xff]
+          vfmsub132pd -1032(%rdx){1to8}, %zmm12, %zmm22
+
+// CHECK: vfmsub213ps %zmm10, %zmm22, %zmm22
+// CHECK:  encoding: [0x62,0xc2,0x4d,0x40,0xaa,0xf2]
+          vfmsub213ps %zmm10, %zmm22, %zmm22
+
+// CHECK: vfmsub213ps %zmm10, %zmm22, %zmm22 {%k6}
+// CHECK:  encoding: [0x62,0xc2,0x4d,0x46,0xaa,0xf2]
+          vfmsub213ps %zmm10, %zmm22, %zmm22 {%k6}
+
+// CHECK: vfmsub213ps %zmm10, %zmm22, %zmm22 {%k6} {z}
+// CHECK:  encoding: [0x62,0xc2,0x4d,0xc6,0xaa,0xf2]
+          vfmsub213ps %zmm10, %zmm22, %zmm22 {%k6} {z}
+
+// CHECK: vfmsub213ps {rn-sae}, %zmm10, %zmm22, %zmm22
+// CHECK:  encoding: [0x62,0xc2,0x4d,0x10,0xaa,0xf2]
+          vfmsub213ps {rn-sae}, %zmm10, %zmm22, %zmm22
+
+// CHECK: vfmsub213ps {ru-sae}, %zmm10, %zmm22, %zmm22
+// CHECK:  encoding: [0x62,0xc2,0x4d,0x50,0xaa,0xf2]
+          vfmsub213ps {ru-sae}, %zmm10, %zmm22, %zmm22
+
+// CHECK: vfmsub213ps {rd-sae}, %zmm10, %zmm22, %zmm22
+// CHECK:  encoding: [0x62,0xc2,0x4d,0x30,0xaa,0xf2]
+          vfmsub213ps {rd-sae}, %zmm10, %zmm22, %zmm22
+
+// CHECK: vfmsub213ps {rz-sae}, %zmm10, %zmm22, %zmm22
+// CHECK:  encoding: [0x62,0xc2,0x4d,0x70,0xaa,0xf2]
+          vfmsub213ps {rz-sae}, %zmm10, %zmm22, %zmm22
+
+// CHECK: vfmsub213ps (%rcx), %zmm22, %zmm22
+// CHECK:  encoding: [0x62,0xe2,0x4d,0x40,0xaa,0x31]
+          vfmsub213ps (%rcx), %zmm22, %zmm22
+
+// CHECK: vfmsub213ps 291(%rax,%r14,8), %zmm22, %zmm22
+// CHECK:  encoding: [0x62,0xa2,0x4d,0x40,0xaa,0xb4,0xf0,0x23,0x01,0x00,0x00]
+          vfmsub213ps 291(%rax,%r14,8), %zmm22, %zmm22
+
+// CHECK: vfmsub213ps (%rcx){1to16}, %zmm22, %zmm22
+// CHECK:  encoding: [0x62,0xe2,0x4d,0x50,0xaa,0x31]
+          vfmsub213ps (%rcx){1to16}, %zmm22, %zmm22
+
+// CHECK: vfmsub213ps 8128(%rdx), %zmm22, %zmm22
+// CHECK:  encoding: [0x62,0xe2,0x4d,0x40,0xaa,0x72,0x7f]
+          vfmsub213ps 8128(%rdx), %zmm22, %zmm22
+
+// CHECK: vfmsub213ps 8192(%rdx), %zmm22, %zmm22
+// CHECK:  encoding: [0x62,0xe2,0x4d,0x40,0xaa,0xb2,0x00,0x20,0x00,0x00]
+          vfmsub213ps 8192(%rdx), %zmm22, %zmm22
+
+// CHECK: vfmsub213ps -8192(%rdx), %zmm22, %zmm22
+// CHECK:  encoding: [0x62,0xe2,0x4d,0x40,0xaa,0x72,0x80]
+          vfmsub213ps -8192(%rdx), %zmm22, %zmm22
+
+// CHECK: vfmsub213ps -8256(%rdx), %zmm22, %zmm22
+// CHECK:  encoding: [0x62,0xe2,0x4d,0x40,0xaa,0xb2,0xc0,0xdf,0xff,0xff]
+          vfmsub213ps -8256(%rdx), %zmm22, %zmm22
+
+// CHECK: vfmsub213ps 508(%rdx){1to16}, %zmm22, %zmm22
+// CHECK:  encoding: [0x62,0xe2,0x4d,0x50,0xaa,0x72,0x7f]
+          vfmsub213ps 508(%rdx){1to16}, %zmm22, %zmm22
+
+// CHECK: vfmsub213ps 512(%rdx){1to16}, %zmm22, %zmm22
+// CHECK:  encoding: [0x62,0xe2,0x4d,0x50,0xaa,0xb2,0x00,0x02,0x00,0x00]
+          vfmsub213ps 512(%rdx){1to16}, %zmm22, %zmm22
+
+// CHECK: vfmsub213ps -512(%rdx){1to16}, %zmm22, %zmm22
+// CHECK:  encoding: [0x62,0xe2,0x4d,0x50,0xaa,0x72,0x80]
+          vfmsub213ps -512(%rdx){1to16}, %zmm22, %zmm22
+
+// CHECK: vfmsub213ps -516(%rdx){1to16}, %zmm22, %zmm22
+// CHECK:  encoding: [0x62,0xe2,0x4d,0x50,0xaa,0xb2,0xfc,0xfd,0xff,0xff]
+          vfmsub213ps -516(%rdx){1to16}, %zmm22, %zmm22
+
+// CHECK: vfmsub213pd %zmm4, %zmm10, %zmm5
+// CHECK:  encoding: [0x62,0xf2,0xad,0x48,0xaa,0xec]
+          vfmsub213pd %zmm4, %zmm10, %zmm5
+
+// CHECK: vfmsub213pd %zmm4, %zmm10, %zmm5 {%k1}
+// CHECK:  encoding: [0x62,0xf2,0xad,0x49,0xaa,0xec]
+          vfmsub213pd %zmm4, %zmm10, %zmm5 {%k1}
+
+// CHECK: vfmsub213pd %zmm4, %zmm10, %zmm5 {%k1} {z}
+// CHECK:  encoding: [0x62,0xf2,0xad,0xc9,0xaa,0xec]
+          vfmsub213pd %zmm4, %zmm10, %zmm5 {%k1} {z}
+
+// CHECK: vfmsub213pd {rn-sae}, %zmm4, %zmm10, %zmm5
+// CHECK:  encoding: [0x62,0xf2,0xad,0x18,0xaa,0xec]
+          vfmsub213pd {rn-sae}, %zmm4, %zmm10, %zmm5
+
+// CHECK: vfmsub213pd {ru-sae}, %zmm4, %zmm10, %zmm5
+// CHECK:  encoding: [0x62,0xf2,0xad,0x58,0xaa,0xec]
+          vfmsub213pd {ru-sae}, %zmm4, %zmm10, %zmm5
+
+// CHECK: vfmsub213pd {rd-sae}, %zmm4, %zmm10, %zmm5
+// CHECK:  encoding: [0x62,0xf2,0xad,0x38,0xaa,0xec]
+          vfmsub213pd {rd-sae}, %zmm4, %zmm10, %zmm5
+
+// CHECK: vfmsub213pd {rz-sae}, %zmm4, %zmm10, %zmm5
+// CHECK:  encoding: [0x62,0xf2,0xad,0x78,0xaa,0xec]
+          vfmsub213pd {rz-sae}, %zmm4, %zmm10, %zmm5
+
+// CHECK: vfmsub213pd (%rcx), %zmm10, %zmm5
+// CHECK:  encoding: [0x62,0xf2,0xad,0x48,0xaa,0x29]
+          vfmsub213pd (%rcx), %zmm10, %zmm5
+
+// CHECK: vfmsub213pd 291(%rax,%r14,8), %zmm10, %zmm5
+// CHECK:  encoding: [0x62,0xb2,0xad,0x48,0xaa,0xac,0xf0,0x23,0x01,0x00,0x00]
+          vfmsub213pd 291(%rax,%r14,8), %zmm10, %zmm5
+
+// CHECK: vfmsub213pd (%rcx){1to8}, %zmm10, %zmm5
+// CHECK:  encoding: [0x62,0xf2,0xad,0x58,0xaa,0x29]
+          vfmsub213pd (%rcx){1to8}, %zmm10, %zmm5
+
+// CHECK: vfmsub213pd 8128(%rdx), %zmm10, %zmm5
+// CHECK:  encoding: [0x62,0xf2,0xad,0x48,0xaa,0x6a,0x7f]
+          vfmsub213pd 8128(%rdx), %zmm10, %zmm5
+
+// CHECK: vfmsub213pd 8192(%rdx), %zmm10, %zmm5
+// CHECK:  encoding: [0x62,0xf2,0xad,0x48,0xaa,0xaa,0x00,0x20,0x00,0x00]
+          vfmsub213pd 8192(%rdx), %zmm10, %zmm5
+
+// CHECK: vfmsub213pd -8192(%rdx), %zmm10, %zmm5
+// CHECK:  encoding: [0x62,0xf2,0xad,0x48,0xaa,0x6a,0x80]
+          vfmsub213pd -8192(%rdx), %zmm10, %zmm5
+
+// CHECK: vfmsub213pd -8256(%rdx), %zmm10, %zmm5
+// CHECK:  encoding: [0x62,0xf2,0xad,0x48,0xaa,0xaa,0xc0,0xdf,0xff,0xff]
+          vfmsub213pd -8256(%rdx), %zmm10, %zmm5
+
+// CHECK: vfmsub213pd 1016(%rdx){1to8}, %zmm10, %zmm5
+// CHECK:  encoding: [0x62,0xf2,0xad,0x58,0xaa,0x6a,0x7f]
+          vfmsub213pd 1016(%rdx){1to8}, %zmm10, %zmm5
+
+// CHECK: vfmsub213pd 1024(%rdx){1to8}, %zmm10, %zmm5
+// CHECK:  encoding: [0x62,0xf2,0xad,0x58,0xaa,0xaa,0x00,0x04,0x00,0x00]
+          vfmsub213pd 1024(%rdx){1to8}, %zmm10, %zmm5
+
+// CHECK: vfmsub213pd -1024(%rdx){1to8}, %zmm10, %zmm5
+// CHECK:  encoding: [0x62,0xf2,0xad,0x58,0xaa,0x6a,0x80]
+          vfmsub213pd -1024(%rdx){1to8}, %zmm10, %zmm5
+
+// CHECK: vfmsub213pd -1032(%rdx){1to8}, %zmm10, %zmm5
+// CHECK:  encoding: [0x62,0xf2,0xad,0x58,0xaa,0xaa,0xf8,0xfb,0xff,0xff]
+          vfmsub213pd -1032(%rdx){1to8}, %zmm10, %zmm5
+
+// CHECK: vfmsub231ps %zmm27, %zmm21, %zmm6
+// CHECK:  encoding: [0x62,0x92,0x55,0x40,0xba,0xf3]
+          vfmsub231ps %zmm27, %zmm21, %zmm6
+
+// CHECK: vfmsub231ps %zmm27, %zmm21, %zmm6 {%k3}
+// CHECK:  encoding: [0x62,0x92,0x55,0x43,0xba,0xf3]
+          vfmsub231ps %zmm27, %zmm21, %zmm6 {%k3}
+
+// CHECK: vfmsub231ps %zmm27, %zmm21, %zmm6 {%k3} {z}
+// CHECK:  encoding: [0x62,0x92,0x55,0xc3,0xba,0xf3]
+          vfmsub231ps %zmm27, %zmm21, %zmm6 {%k3} {z}
+
+// CHECK: vfmsub231ps {rn-sae}, %zmm27, %zmm21, %zmm6
+// CHECK:  encoding: [0x62,0x92,0x55,0x10,0xba,0xf3]
+          vfmsub231ps {rn-sae}, %zmm27, %zmm21, %zmm6
+
+// CHECK: vfmsub231ps {ru-sae}, %zmm27, %zmm21, %zmm6
+// CHECK:  encoding: [0x62,0x92,0x55,0x50,0xba,0xf3]
+          vfmsub231ps {ru-sae}, %zmm27, %zmm21, %zmm6
+
+// CHECK: vfmsub231ps {rd-sae}, %zmm27, %zmm21, %zmm6
+// CHECK:  encoding: [0x62,0x92,0x55,0x30,0xba,0xf3]
+          vfmsub231ps {rd-sae}, %zmm27, %zmm21, %zmm6
+
+// CHECK: vfmsub231ps {rz-sae}, %zmm27, %zmm21, %zmm6
+// CHECK:  encoding: [0x62,0x92,0x55,0x70,0xba,0xf3]
+          vfmsub231ps {rz-sae}, %zmm27, %zmm21, %zmm6
+
+// CHECK: vfmsub231ps (%rcx), %zmm21, %zmm6
+// CHECK:  encoding: [0x62,0xf2,0x55,0x40,0xba,0x31]
+          vfmsub231ps (%rcx), %zmm21, %zmm6
+
+// CHECK: vfmsub231ps 291(%rax,%r14,8), %zmm21, %zmm6
+// CHECK:  encoding: [0x62,0xb2,0x55,0x40,0xba,0xb4,0xf0,0x23,0x01,0x00,0x00]
+          vfmsub231ps 291(%rax,%r14,8), %zmm21, %zmm6
+
+// CHECK: vfmsub231ps (%rcx){1to16}, %zmm21, %zmm6
+// CHECK:  encoding: [0x62,0xf2,0x55,0x50,0xba,0x31]
+          vfmsub231ps (%rcx){1to16}, %zmm21, %zmm6
+
+// CHECK: vfmsub231ps 8128(%rdx), %zmm21, %zmm6
+// CHECK:  encoding: [0x62,0xf2,0x55,0x40,0xba,0x72,0x7f]
+          vfmsub231ps 8128(%rdx), %zmm21, %zmm6
+
+// CHECK: vfmsub231ps 8192(%rdx), %zmm21, %zmm6
+// CHECK:  encoding: [0x62,0xf2,0x55,0x40,0xba,0xb2,0x00,0x20,0x00,0x00]
+          vfmsub231ps 8192(%rdx), %zmm21, %zmm6
+
+// CHECK: vfmsub231ps -8192(%rdx), %zmm21, %zmm6
+// CHECK:  encoding: [0x62,0xf2,0x55,0x40,0xba,0x72,0x80]
+          vfmsub231ps -8192(%rdx), %zmm21, %zmm6
+
+// CHECK: vfmsub231ps -8256(%rdx), %zmm21, %zmm6
+// CHECK:  encoding: [0x62,0xf2,0x55,0x40,0xba,0xb2,0xc0,0xdf,0xff,0xff]
+          vfmsub231ps -8256(%rdx), %zmm21, %zmm6
+
+// CHECK: vfmsub231ps 508(%rdx){1to16}, %zmm21, %zmm6
+// CHECK:  encoding: [0x62,0xf2,0x55,0x50,0xba,0x72,0x7f]
+          vfmsub231ps 508(%rdx){1to16}, %zmm21, %zmm6
+
+// CHECK: vfmsub231ps 512(%rdx){1to16}, %zmm21, %zmm6
+// CHECK:  encoding: [0x62,0xf2,0x55,0x50,0xba,0xb2,0x00,0x02,0x00,0x00]
+          vfmsub231ps 512(%rdx){1to16}, %zmm21, %zmm6
+
+// CHECK: vfmsub231ps -512(%rdx){1to16}, %zmm21, %zmm6
+// CHECK:  encoding: [0x62,0xf2,0x55,0x50,0xba,0x72,0x80]
+          vfmsub231ps -512(%rdx){1to16}, %zmm21, %zmm6
+
+// CHECK: vfmsub231ps -516(%rdx){1to16}, %zmm21, %zmm6
+// CHECK:  encoding: [0x62,0xf2,0x55,0x50,0xba,0xb2,0xfc,0xfd,0xff,0xff]
+          vfmsub231ps -516(%rdx){1to16}, %zmm21, %zmm6
+
+// CHECK: vfmsub231pd %zmm11, %zmm12, %zmm5
+// CHECK:  encoding: [0x62,0xd2,0x9d,0x48,0xba,0xeb]
+          vfmsub231pd %zmm11, %zmm12, %zmm5
+
+// CHECK: vfmsub231pd %zmm11, %zmm12, %zmm5 {%k2}
+// CHECK:  encoding: [0x62,0xd2,0x9d,0x4a,0xba,0xeb]
+          vfmsub231pd %zmm11, %zmm12, %zmm5 {%k2}
+
+// CHECK: vfmsub231pd %zmm11, %zmm12, %zmm5 {%k2} {z}
+// CHECK:  encoding: [0x62,0xd2,0x9d,0xca,0xba,0xeb]
+          vfmsub231pd %zmm11, %zmm12, %zmm5 {%k2} {z}
+
+// CHECK: vfmsub231pd {rn-sae}, %zmm11, %zmm12, %zmm5
+// CHECK:  encoding: [0x62,0xd2,0x9d,0x18,0xba,0xeb]
+          vfmsub231pd {rn-sae}, %zmm11, %zmm12, %zmm5
+
+// CHECK: vfmsub231pd {ru-sae}, %zmm11, %zmm12, %zmm5
+// CHECK:  encoding: [0x62,0xd2,0x9d,0x58,0xba,0xeb]
+          vfmsub231pd {ru-sae}, %zmm11, %zmm12, %zmm5
+
+// CHECK: vfmsub231pd {rd-sae}, %zmm11, %zmm12, %zmm5
+// CHECK:  encoding: [0x62,0xd2,0x9d,0x38,0xba,0xeb]
+          vfmsub231pd {rd-sae}, %zmm11, %zmm12, %zmm5
+
+// CHECK: vfmsub231pd {rz-sae}, %zmm11, %zmm12, %zmm5
+// CHECK:  encoding: [0x62,0xd2,0x9d,0x78,0xba,0xeb]
+          vfmsub231pd {rz-sae}, %zmm11, %zmm12, %zmm5
+
+// CHECK: vfmsub231pd (%rcx), %zmm12, %zmm5
+// CHECK:  encoding: [0x62,0xf2,0x9d,0x48,0xba,0x29]
+          vfmsub231pd (%rcx), %zmm12, %zmm5
+
+// CHECK: vfmsub231pd 291(%rax,%r14,8), %zmm12, %zmm5
+// CHECK:  encoding: [0x62,0xb2,0x9d,0x48,0xba,0xac,0xf0,0x23,0x01,0x00,0x00]
+          vfmsub231pd 291(%rax,%r14,8), %zmm12, %zmm5
+
+// CHECK: vfmsub231pd (%rcx){1to8}, %zmm12, %zmm5
+// CHECK:  encoding: [0x62,0xf2,0x9d,0x58,0xba,0x29]
+          vfmsub231pd (%rcx){1to8}, %zmm12, %zmm5
+
+// CHECK: vfmsub231pd 8128(%rdx), %zmm12, %zmm5
+// CHECK:  encoding: [0x62,0xf2,0x9d,0x48,0xba,0x6a,0x7f]
+          vfmsub231pd 8128(%rdx), %zmm12, %zmm5
+
+// CHECK: vfmsub231pd 8192(%rdx), %zmm12, %zmm5
+// CHECK:  encoding: [0x62,0xf2,0x9d,0x48,0xba,0xaa,0x00,0x20,0x00,0x00]
+          vfmsub231pd 8192(%rdx), %zmm12, %zmm5
+
+// CHECK: vfmsub231pd -8192(%rdx), %zmm12, %zmm5
+// CHECK:  encoding: [0x62,0xf2,0x9d,0x48,0xba,0x6a,0x80]
+          vfmsub231pd -8192(%rdx), %zmm12, %zmm5
+
+// CHECK: vfmsub231pd -8256(%rdx), %zmm12, %zmm5
+// CHECK:  encoding: [0x62,0xf2,0x9d,0x48,0xba,0xaa,0xc0,0xdf,0xff,0xff]
+          vfmsub231pd -8256(%rdx), %zmm12, %zmm5
+
+// CHECK: vfmsub231pd 1016(%rdx){1to8}, %zmm12, %zmm5
+// CHECK:  encoding: [0x62,0xf2,0x9d,0x58,0xba,0x6a,0x7f]
+          vfmsub231pd 1016(%rdx){1to8}, %zmm12, %zmm5
+
+// CHECK: vfmsub231pd 1024(%rdx){1to8}, %zmm12, %zmm5
+// CHECK:  encoding: [0x62,0xf2,0x9d,0x58,0xba,0xaa,0x00,0x04,0x00,0x00]
+          vfmsub231pd 1024(%rdx){1to8}, %zmm12, %zmm5
+
+// CHECK: vfmsub231pd -1024(%rdx){1to8}, %zmm12, %zmm5
+// CHECK:  encoding: [0x62,0xf2,0x9d,0x58,0xba,0x6a,0x80]
+          vfmsub231pd -1024(%rdx){1to8}, %zmm12, %zmm5
+
+// CHECK: vfmsub231pd -1032(%rdx){1to8}, %zmm12, %zmm5
+// CHECK:  encoding: [0x62,0xf2,0x9d,0x58,0xba,0xaa,0xf8,0xfb,0xff,0xff]
+          vfmsub231pd -1032(%rdx){1to8}, %zmm12, %zmm5
+
+// CHECK: vfmaddsub132ps %zmm20, %zmm9, %zmm10
+// CHECK:  encoding: [0x62,0x32,0x35,0x48,0x96,0xd4]
+          vfmaddsub132ps %zmm20, %zmm9, %zmm10
+
+// CHECK: vfmaddsub132ps %zmm20, %zmm9, %zmm10 {%k3}
+// CHECK:  encoding: [0x62,0x32,0x35,0x4b,0x96,0xd4]
+          vfmaddsub132ps %zmm20, %zmm9, %zmm10 {%k3}
+
+// CHECK: vfmaddsub132ps %zmm20, %zmm9, %zmm10 {%k3} {z}
+// CHECK:  encoding: [0x62,0x32,0x35,0xcb,0x96,0xd4]
+          vfmaddsub132ps %zmm20, %zmm9, %zmm10 {%k3} {z}
+
+// CHECK: vfmaddsub132ps {rn-sae}, %zmm20, %zmm9, %zmm10
+// CHECK:  encoding: [0x62,0x32,0x35,0x18,0x96,0xd4]
+          vfmaddsub132ps {rn-sae}, %zmm20, %zmm9, %zmm10
+
+// CHECK: vfmaddsub132ps {ru-sae}, %zmm20, %zmm9, %zmm10
+// CHECK:  encoding: [0x62,0x32,0x35,0x58,0x96,0xd4]
+          vfmaddsub132ps {ru-sae}, %zmm20, %zmm9, %zmm10
+
+// CHECK: vfmaddsub132ps {rd-sae}, %zmm20, %zmm9, %zmm10
+// CHECK:  encoding: [0x62,0x32,0x35,0x38,0x96,0xd4]
+          vfmaddsub132ps {rd-sae}, %zmm20, %zmm9, %zmm10
+
+// CHECK: vfmaddsub132ps {rz-sae}, %zmm20, %zmm9, %zmm10
+// CHECK:  encoding: [0x62,0x32,0x35,0x78,0x96,0xd4]
+          vfmaddsub132ps {rz-sae}, %zmm20, %zmm9, %zmm10
+
+// CHECK: vfmaddsub132ps (%rcx), %zmm9, %zmm10
+// CHECK:  encoding: [0x62,0x72,0x35,0x48,0x96,0x11]
+          vfmaddsub132ps (%rcx), %zmm9, %zmm10
+
+// CHECK: vfmaddsub132ps 291(%rax,%r14,8), %zmm9, %zmm10
+// CHECK:  encoding: [0x62,0x32,0x35,0x48,0x96,0x94,0xf0,0x23,0x01,0x00,0x00]
+          vfmaddsub132ps 291(%rax,%r14,8), %zmm9, %zmm10
+
+// CHECK: vfmaddsub132ps (%rcx){1to16}, %zmm9, %zmm10
+// CHECK:  encoding: [0x62,0x72,0x35,0x58,0x96,0x11]
+          vfmaddsub132ps (%rcx){1to16}, %zmm9, %zmm10
+
+// CHECK: vfmaddsub132ps 8128(%rdx), %zmm9, %zmm10
+// CHECK:  encoding: [0x62,0x72,0x35,0x48,0x96,0x52,0x7f]
+          vfmaddsub132ps 8128(%rdx), %zmm9, %zmm10
+
+// CHECK: vfmaddsub132ps 8192(%rdx), %zmm9, %zmm10
+// CHECK:  encoding: [0x62,0x72,0x35,0x48,0x96,0x92,0x00,0x20,0x00,0x00]
+          vfmaddsub132ps 8192(%rdx), %zmm9, %zmm10
+
+// CHECK: vfmaddsub132ps -8192(%rdx), %zmm9, %zmm10
+// CHECK:  encoding: [0x62,0x72,0x35,0x48,0x96,0x52,0x80]
+          vfmaddsub132ps -8192(%rdx), %zmm9, %zmm10
+
+// CHECK: vfmaddsub132ps -8256(%rdx), %zmm9, %zmm10
+// CHECK:  encoding: [0x62,0x72,0x35,0x48,0x96,0x92,0xc0,0xdf,0xff,0xff]
+          vfmaddsub132ps -8256(%rdx), %zmm9, %zmm10
+
+// CHECK: vfmaddsub132ps 508(%rdx){1to16}, %zmm9, %zmm10
+// CHECK:  encoding: [0x62,0x72,0x35,0x58,0x96,0x52,0x7f]
+          vfmaddsub132ps 508(%rdx){1to16}, %zmm9, %zmm10
+
+// CHECK: vfmaddsub132ps 512(%rdx){1to16}, %zmm9, %zmm10
+// CHECK:  encoding: [0x62,0x72,0x35,0x58,0x96,0x92,0x00,0x02,0x00,0x00]
+          vfmaddsub132ps 512(%rdx){1to16}, %zmm9, %zmm10
+
+// CHECK: vfmaddsub132ps -512(%rdx){1to16}, %zmm9, %zmm10
+// CHECK:  encoding: [0x62,0x72,0x35,0x58,0x96,0x52,0x80]
+          vfmaddsub132ps -512(%rdx){1to16}, %zmm9, %zmm10
+
+// CHECK: vfmaddsub132ps -516(%rdx){1to16}, %zmm9, %zmm10
+// CHECK:  encoding: [0x62,0x72,0x35,0x58,0x96,0x92,0xfc,0xfd,0xff,0xff]
+          vfmaddsub132ps -516(%rdx){1to16}, %zmm9, %zmm10
+
+// CHECK: vfmaddsub132pd %zmm21, %zmm25, %zmm20
+// CHECK:  encoding: [0x62,0xa2,0xb5,0x40,0x96,0xe5]
+          vfmaddsub132pd %zmm21, %zmm25, %zmm20
+
+// CHECK: vfmaddsub132pd %zmm21, %zmm25, %zmm20 {%k2}
+// CHECK:  encoding: [0x62,0xa2,0xb5,0x42,0x96,0xe5]
+          vfmaddsub132pd %zmm21, %zmm25, %zmm20 {%k2}
+
+// CHECK: vfmaddsub132pd %zmm21, %zmm25, %zmm20 {%k2} {z}
+// CHECK:  encoding: [0x62,0xa2,0xb5,0xc2,0x96,0xe5]
+          vfmaddsub132pd %zmm21, %zmm25, %zmm20 {%k2} {z}
+
+// CHECK: vfmaddsub132pd {rn-sae}, %zmm21, %zmm25, %zmm20
+// CHECK:  encoding: [0x62,0xa2,0xb5,0x10,0x96,0xe5]
+          vfmaddsub132pd {rn-sae}, %zmm21, %zmm25, %zmm20
+
+// CHECK: vfmaddsub132pd {ru-sae}, %zmm21, %zmm25, %zmm20
+// CHECK:  encoding: [0x62,0xa2,0xb5,0x50,0x96,0xe5]
+          vfmaddsub132pd {ru-sae}, %zmm21, %zmm25, %zmm20
+
+// CHECK: vfmaddsub132pd {rd-sae}, %zmm21, %zmm25, %zmm20
+// CHECK:  encoding: [0x62,0xa2,0xb5,0x30,0x96,0xe5]
+          vfmaddsub132pd {rd-sae}, %zmm21, %zmm25, %zmm20
+
+// CHECK: vfmaddsub132pd {rz-sae}, %zmm21, %zmm25, %zmm20
+// CHECK:  encoding: [0x62,0xa2,0xb5,0x70,0x96,0xe5]
+          vfmaddsub132pd {rz-sae}, %zmm21, %zmm25, %zmm20
+
+// CHECK: vfmaddsub132pd (%rcx), %zmm25, %zmm20
+// CHECK:  encoding: [0x62,0xe2,0xb5,0x40,0x96,0x21]
+          vfmaddsub132pd (%rcx), %zmm25, %zmm20
+
+// CHECK: vfmaddsub132pd 291(%rax,%r14,8), %zmm25, %zmm20
+// CHECK:  encoding: [0x62,0xa2,0xb5,0x40,0x96,0xa4,0xf0,0x23,0x01,0x00,0x00]
+          vfmaddsub132pd 291(%rax,%r14,8), %zmm25, %zmm20
+
+// CHECK: vfmaddsub132pd (%rcx){1to8}, %zmm25, %zmm20
+// CHECK:  encoding: [0x62,0xe2,0xb5,0x50,0x96,0x21]
+          vfmaddsub132pd (%rcx){1to8}, %zmm25, %zmm20
+
+// CHECK: vfmaddsub132pd 8128(%rdx), %zmm25, %zmm20
+// CHECK:  encoding: [0x62,0xe2,0xb5,0x40,0x96,0x62,0x7f]
+          vfmaddsub132pd 8128(%rdx), %zmm25, %zmm20
+
+// CHECK: vfmaddsub132pd 8192(%rdx), %zmm25, %zmm20
+// CHECK:  encoding: [0x62,0xe2,0xb5,0x40,0x96,0xa2,0x00,0x20,0x00,0x00]
+          vfmaddsub132pd 8192(%rdx), %zmm25, %zmm20
+
+// CHECK: vfmaddsub132pd -8192(%rdx), %zmm25, %zmm20
+// CHECK:  encoding: [0x62,0xe2,0xb5,0x40,0x96,0x62,0x80]
+          vfmaddsub132pd -8192(%rdx), %zmm25, %zmm20
+
+// CHECK: vfmaddsub132pd -8256(%rdx), %zmm25, %zmm20
+// CHECK:  encoding: [0x62,0xe2,0xb5,0x40,0x96,0xa2,0xc0,0xdf,0xff,0xff]
+          vfmaddsub132pd -8256(%rdx), %zmm25, %zmm20
+
+// CHECK: vfmaddsub132pd 1016(%rdx){1to8}, %zmm25, %zmm20
+// CHECK:  encoding: [0x62,0xe2,0xb5,0x50,0x96,0x62,0x7f]
+          vfmaddsub132pd 1016(%rdx){1to8}, %zmm25, %zmm20
+
+// CHECK: vfmaddsub132pd 1024(%rdx){1to8}, %zmm25, %zmm20
+// CHECK:  encoding: [0x62,0xe2,0xb5,0x50,0x96,0xa2,0x00,0x04,0x00,0x00]
+          vfmaddsub132pd 1024(%rdx){1to8}, %zmm25, %zmm20
+
+// CHECK: vfmaddsub132pd -1024(%rdx){1to8}, %zmm25, %zmm20
+// CHECK:  encoding: [0x62,0xe2,0xb5,0x50,0x96,0x62,0x80]
+          vfmaddsub132pd -1024(%rdx){1to8}, %zmm25, %zmm20
+
+// CHECK: vfmaddsub132pd -1032(%rdx){1to8}, %zmm25, %zmm20
+// CHECK:  encoding: [0x62,0xe2,0xb5,0x50,0x96,0xa2,0xf8,0xfb,0xff,0xff]
+          vfmaddsub132pd -1032(%rdx){1to8}, %zmm25, %zmm20
+
+// CHECK: vfmaddsub213ps %zmm28, %zmm24, %zmm17
+// CHECK:  encoding: [0x62,0x82,0x3d,0x40,0xa6,0xcc]
+          vfmaddsub213ps %zmm28, %zmm24, %zmm17
+
+// CHECK: vfmaddsub213ps %zmm28, %zmm24, %zmm17 {%k6}
+// CHECK:  encoding: [0x62,0x82,0x3d,0x46,0xa6,0xcc]
+          vfmaddsub213ps %zmm28, %zmm24, %zmm17 {%k6}
+
+// CHECK: vfmaddsub213ps %zmm28, %zmm24, %zmm17 {%k6} {z}
+// CHECK:  encoding: [0x62,0x82,0x3d,0xc6,0xa6,0xcc]
+          vfmaddsub213ps %zmm28, %zmm24, %zmm17 {%k6} {z}
+
+// CHECK: vfmaddsub213ps {rn-sae}, %zmm28, %zmm24, %zmm17
+// CHECK:  encoding: [0x62,0x82,0x3d,0x10,0xa6,0xcc]
+          vfmaddsub213ps {rn-sae}, %zmm28, %zmm24, %zmm17
+
+// CHECK: vfmaddsub213ps {ru-sae}, %zmm28, %zmm24, %zmm17
+// CHECK:  encoding: [0x62,0x82,0x3d,0x50,0xa6,0xcc]
+          vfmaddsub213ps {ru-sae}, %zmm28, %zmm24, %zmm17
+
+// CHECK: vfmaddsub213ps {rd-sae}, %zmm28, %zmm24, %zmm17
+// CHECK:  encoding: [0x62,0x82,0x3d,0x30,0xa6,0xcc]
+          vfmaddsub213ps {rd-sae}, %zmm28, %zmm24, %zmm17
+
+// CHECK: vfmaddsub213ps {rz-sae}, %zmm28, %zmm24, %zmm17
+// CHECK:  encoding: [0x62,0x82,0x3d,0x70,0xa6,0xcc]
+          vfmaddsub213ps {rz-sae}, %zmm28, %zmm24, %zmm17
+
+// CHECK: vfmaddsub213ps (%rcx), %zmm24, %zmm17
+// CHECK:  encoding: [0x62,0xe2,0x3d,0x40,0xa6,0x09]
+          vfmaddsub213ps (%rcx), %zmm24, %zmm17
+
+// CHECK: vfmaddsub213ps 291(%rax,%r14,8), %zmm24, %zmm17
+// CHECK:  encoding: [0x62,0xa2,0x3d,0x40,0xa6,0x8c,0xf0,0x23,0x01,0x00,0x00]
+          vfmaddsub213ps 291(%rax,%r14,8), %zmm24, %zmm17
+
+// CHECK: vfmaddsub213ps (%rcx){1to16}, %zmm24, %zmm17
+// CHECK:  encoding: [0x62,0xe2,0x3d,0x50,0xa6,0x09]
+          vfmaddsub213ps (%rcx){1to16}, %zmm24, %zmm17
+
+// CHECK: vfmaddsub213ps 8128(%rdx), %zmm24, %zmm17
+// CHECK:  encoding: [0x62,0xe2,0x3d,0x40,0xa6,0x4a,0x7f]
+          vfmaddsub213ps 8128(%rdx), %zmm24, %zmm17
+
+// CHECK: vfmaddsub213ps 8192(%rdx), %zmm24, %zmm17
+// CHECK:  encoding: [0x62,0xe2,0x3d,0x40,0xa6,0x8a,0x00,0x20,0x00,0x00]
+          vfmaddsub213ps 8192(%rdx), %zmm24, %zmm17
+
+// CHECK: vfmaddsub213ps -8192(%rdx), %zmm24, %zmm17
+// CHECK:  encoding: [0x62,0xe2,0x3d,0x40,0xa6,0x4a,0x80]
+          vfmaddsub213ps -8192(%rdx), %zmm24, %zmm17
+
+// CHECK: vfmaddsub213ps -8256(%rdx), %zmm24, %zmm17
+// CHECK:  encoding: [0x62,0xe2,0x3d,0x40,0xa6,0x8a,0xc0,0xdf,0xff,0xff]
+          vfmaddsub213ps -8256(%rdx), %zmm24, %zmm17
+
+// CHECK: vfmaddsub213ps 508(%rdx){1to16}, %zmm24, %zmm17
+// CHECK:  encoding: [0x62,0xe2,0x3d,0x50,0xa6,0x4a,0x7f]
+          vfmaddsub213ps 508(%rdx){1to16}, %zmm24, %zmm17
+
+// CHECK: vfmaddsub213ps 512(%rdx){1to16}, %zmm24, %zmm17
+// CHECK:  encoding: [0x62,0xe2,0x3d,0x50,0xa6,0x8a,0x00,0x02,0x00,0x00]
+          vfmaddsub213ps 512(%rdx){1to16}, %zmm24, %zmm17
+
+// CHECK: vfmaddsub213ps -512(%rdx){1to16}, %zmm24, %zmm17
+// CHECK:  encoding: [0x62,0xe2,0x3d,0x50,0xa6,0x4a,0x80]
+          vfmaddsub213ps -512(%rdx){1to16}, %zmm24, %zmm17
+
+// CHECK: vfmaddsub213ps -516(%rdx){1to16}, %zmm24, %zmm17
+// CHECK:  encoding: [0x62,0xe2,0x3d,0x50,0xa6,0x8a,0xfc,0xfd,0xff,0xff]
+          vfmaddsub213ps -516(%rdx){1to16}, %zmm24, %zmm17
+
+// CHECK: vfmaddsub213pd %zmm10, %zmm6, %zmm26
+// CHECK:  encoding: [0x62,0x42,0xcd,0x48,0xa6,0xd2]
+          vfmaddsub213pd %zmm10, %zmm6, %zmm26
+
+// CHECK: vfmaddsub213pd %zmm10, %zmm6, %zmm26 {%k6}
+// CHECK:  encoding: [0x62,0x42,0xcd,0x4e,0xa6,0xd2]
+          vfmaddsub213pd %zmm10, %zmm6, %zmm26 {%k6}
+
+// CHECK: vfmaddsub213pd %zmm10, %zmm6, %zmm26 {%k6} {z}
+// CHECK:  encoding: [0x62,0x42,0xcd,0xce,0xa6,0xd2]
+          vfmaddsub213pd %zmm10, %zmm6, %zmm26 {%k6} {z}
+
+// CHECK: vfmaddsub213pd {rn-sae}, %zmm10, %zmm6, %zmm26
+// CHECK:  encoding: [0x62,0x42,0xcd,0x18,0xa6,0xd2]
+          vfmaddsub213pd {rn-sae}, %zmm10, %zmm6, %zmm26
+
+// CHECK: vfmaddsub213pd {ru-sae}, %zmm10, %zmm6, %zmm26
+// CHECK:  encoding: [0x62,0x42,0xcd,0x58,0xa6,0xd2]
+          vfmaddsub213pd {ru-sae}, %zmm10, %zmm6, %zmm26
+
+// CHECK: vfmaddsub213pd {rd-sae}, %zmm10, %zmm6, %zmm26
+// CHECK:  encoding: [0x62,0x42,0xcd,0x38,0xa6,0xd2]
+          vfmaddsub213pd {rd-sae}, %zmm10, %zmm6, %zmm26
+
+// CHECK: vfmaddsub213pd {rz-sae}, %zmm10, %zmm6, %zmm26
+// CHECK:  encoding: [0x62,0x42,0xcd,0x78,0xa6,0xd2]
+          vfmaddsub213pd {rz-sae}, %zmm10, %zmm6, %zmm26
+
+// CHECK: vfmaddsub213pd (%rcx), %zmm6, %zmm26
+// CHECK:  encoding: [0x62,0x62,0xcd,0x48,0xa6,0x11]
+          vfmaddsub213pd (%rcx), %zmm6, %zmm26
+
+// CHECK: vfmaddsub213pd 291(%rax,%r14,8), %zmm6, %zmm26
+// CHECK:  encoding: [0x62,0x22,0xcd,0x48,0xa6,0x94,0xf0,0x23,0x01,0x00,0x00]
+          vfmaddsub213pd 291(%rax,%r14,8), %zmm6, %zmm26
+
+// CHECK: vfmaddsub213pd (%rcx){1to8}, %zmm6, %zmm26
+// CHECK:  encoding: [0x62,0x62,0xcd,0x58,0xa6,0x11]
+          vfmaddsub213pd (%rcx){1to8}, %zmm6, %zmm26
+
+// CHECK: vfmaddsub213pd 8128(%rdx), %zmm6, %zmm26
+// CHECK:  encoding: [0x62,0x62,0xcd,0x48,0xa6,0x52,0x7f]
+          vfmaddsub213pd 8128(%rdx), %zmm6, %zmm26
+
+// CHECK: vfmaddsub213pd 8192(%rdx), %zmm6, %zmm26
+// CHECK:  encoding: [0x62,0x62,0xcd,0x48,0xa6,0x92,0x00,0x20,0x00,0x00]
+          vfmaddsub213pd 8192(%rdx), %zmm6, %zmm26
+
+// CHECK: vfmaddsub213pd -8192(%rdx), %zmm6, %zmm26
+// CHECK:  encoding: [0x62,0x62,0xcd,0x48,0xa6,0x52,0x80]
+          vfmaddsub213pd -8192(%rdx), %zmm6, %zmm26
+
+// CHECK: vfmaddsub213pd -8256(%rdx), %zmm6, %zmm26
+// CHECK:  encoding: [0x62,0x62,0xcd,0x48,0xa6,0x92,0xc0,0xdf,0xff,0xff]
+          vfmaddsub213pd -8256(%rdx), %zmm6, %zmm26
+
+// CHECK: vfmaddsub213pd 1016(%rdx){1to8}, %zmm6, %zmm26
+// CHECK:  encoding: [0x62,0x62,0xcd,0x58,0xa6,0x52,0x7f]
+          vfmaddsub213pd 1016(%rdx){1to8}, %zmm6, %zmm26
+
+// CHECK: vfmaddsub213pd 1024(%rdx){1to8}, %zmm6, %zmm26
+// CHECK:  encoding: [0x62,0x62,0xcd,0x58,0xa6,0x92,0x00,0x04,0x00,0x00]
+          vfmaddsub213pd 1024(%rdx){1to8}, %zmm6, %zmm26
+
+// CHECK: vfmaddsub213pd -1024(%rdx){1to8}, %zmm6, %zmm26
+// CHECK:  encoding: [0x62,0x62,0xcd,0x58,0xa6,0x52,0x80]
+          vfmaddsub213pd -1024(%rdx){1to8}, %zmm6, %zmm26
+
+// CHECK: vfmaddsub213pd -1032(%rdx){1to8}, %zmm6, %zmm26
+// CHECK:  encoding: [0x62,0x62,0xcd,0x58,0xa6,0x92,0xf8,0xfb,0xff,0xff]
+          vfmaddsub213pd -1032(%rdx){1to8}, %zmm6, %zmm26
+
+// CHECK: vfmaddsub231ps %zmm19, %zmm21, %zmm15
+// CHECK:  encoding: [0x62,0x32,0x55,0x40,0xb6,0xfb]
+          vfmaddsub231ps %zmm19, %zmm21, %zmm15
+
+// CHECK: vfmaddsub231ps %zmm19, %zmm21, %zmm15 {%k6}
+// CHECK:  encoding: [0x62,0x32,0x55,0x46,0xb6,0xfb]
+          vfmaddsub231ps %zmm19, %zmm21, %zmm15 {%k6}
+
+// CHECK: vfmaddsub231ps %zmm19, %zmm21, %zmm15 {%k6} {z}
+// CHECK:  encoding: [0x62,0x32,0x55,0xc6,0xb6,0xfb]
+          vfmaddsub231ps %zmm19, %zmm21, %zmm15 {%k6} {z}
+
+// CHECK: vfmaddsub231ps {rn-sae}, %zmm19, %zmm21, %zmm15
+// CHECK:  encoding: [0x62,0x32,0x55,0x10,0xb6,0xfb]
+          vfmaddsub231ps {rn-sae}, %zmm19, %zmm21, %zmm15
+
+// CHECK: vfmaddsub231ps {ru-sae}, %zmm19, %zmm21, %zmm15
+// CHECK:  encoding: [0x62,0x32,0x55,0x50,0xb6,0xfb]
+          vfmaddsub231ps {ru-sae}, %zmm19, %zmm21, %zmm15
+
+// CHECK: vfmaddsub231ps {rd-sae}, %zmm19, %zmm21, %zmm15
+// CHECK:  encoding: [0x62,0x32,0x55,0x30,0xb6,0xfb]
+          vfmaddsub231ps {rd-sae}, %zmm19, %zmm21, %zmm15
+
+// CHECK: vfmaddsub231ps {rz-sae}, %zmm19, %zmm21, %zmm15
+// CHECK:  encoding: [0x62,0x32,0x55,0x70,0xb6,0xfb]
+          vfmaddsub231ps {rz-sae}, %zmm19, %zmm21, %zmm15
+
+// CHECK: vfmaddsub231ps (%rcx), %zmm21, %zmm15
+// CHECK:  encoding: [0x62,0x72,0x55,0x40,0xb6,0x39]
+          vfmaddsub231ps (%rcx), %zmm21, %zmm15
+
+// CHECK: vfmaddsub231ps 291(%rax,%r14,8), %zmm21, %zmm15
+// CHECK:  encoding: [0x62,0x32,0x55,0x40,0xb6,0xbc,0xf0,0x23,0x01,0x00,0x00]
+          vfmaddsub231ps 291(%rax,%r14,8), %zmm21, %zmm15
+
+// CHECK: vfmaddsub231ps (%rcx){1to16}, %zmm21, %zmm15
+// CHECK:  encoding: [0x62,0x72,0x55,0x50,0xb6,0x39]
+          vfmaddsub231ps (%rcx){1to16}, %zmm21, %zmm15
+
+// CHECK: vfmaddsub231ps 8128(%rdx), %zmm21, %zmm15
+// CHECK:  encoding: [0x62,0x72,0x55,0x40,0xb6,0x7a,0x7f]
+          vfmaddsub231ps 8128(%rdx), %zmm21, %zmm15
+
+// CHECK: vfmaddsub231ps 8192(%rdx), %zmm21, %zmm15
+// CHECK:  encoding: [0x62,0x72,0x55,0x40,0xb6,0xba,0x00,0x20,0x00,0x00]
+          vfmaddsub231ps 8192(%rdx), %zmm21, %zmm15
+
+// CHECK: vfmaddsub231ps -8192(%rdx), %zmm21, %zmm15
+// CHECK:  encoding: [0x62,0x72,0x55,0x40,0xb6,0x7a,0x80]
+          vfmaddsub231ps -8192(%rdx), %zmm21, %zmm15
+
+// CHECK: vfmaddsub231ps -8256(%rdx), %zmm21, %zmm15
+// CHECK:  encoding: [0x62,0x72,0x55,0x40,0xb6,0xba,0xc0,0xdf,0xff,0xff]
+          vfmaddsub231ps -8256(%rdx), %zmm21, %zmm15
+
+// CHECK: vfmaddsub231ps 508(%rdx){1to16}, %zmm21, %zmm15
+// CHECK:  encoding: [0x62,0x72,0x55,0x50,0xb6,0x7a,0x7f]
+          vfmaddsub231ps 508(%rdx){1to16}, %zmm21, %zmm15
+
+// CHECK: vfmaddsub231ps 512(%rdx){1to16}, %zmm21, %zmm15
+// CHECK:  encoding: [0x62,0x72,0x55,0x50,0xb6,0xba,0x00,0x02,0x00,0x00]
+          vfmaddsub231ps 512(%rdx){1to16}, %zmm21, %zmm15
+
+// CHECK: vfmaddsub231ps -512(%rdx){1to16}, %zmm21, %zmm15
+// CHECK:  encoding: [0x62,0x72,0x55,0x50,0xb6,0x7a,0x80]
+          vfmaddsub231ps -512(%rdx){1to16}, %zmm21, %zmm15
+
+// CHECK: vfmaddsub231ps -516(%rdx){1to16}, %zmm21, %zmm15
+// CHECK:  encoding: [0x62,0x72,0x55,0x50,0xb6,0xba,0xfc,0xfd,0xff,0xff]
+          vfmaddsub231ps -516(%rdx){1to16}, %zmm21, %zmm15
+
+// CHECK: vfmaddsub231pd %zmm24, %zmm27, %zmm9
+// CHECK:  encoding: [0x62,0x12,0xa5,0x40,0xb6,0xc8]
+          vfmaddsub231pd %zmm24, %zmm27, %zmm9
+
+// CHECK: vfmaddsub231pd %zmm24, %zmm27, %zmm9 {%k7}
+// CHECK:  encoding: [0x62,0x12,0xa5,0x47,0xb6,0xc8]
+          vfmaddsub231pd %zmm24, %zmm27, %zmm9 {%k7}
+
+// CHECK: vfmaddsub231pd %zmm24, %zmm27, %zmm9 {%k7} {z}
+// CHECK:  encoding: [0x62,0x12,0xa5,0xc7,0xb6,0xc8]
+          vfmaddsub231pd %zmm24, %zmm27, %zmm9 {%k7} {z}
+
+// CHECK: vfmaddsub231pd {rn-sae}, %zmm24, %zmm27, %zmm9
+// CHECK:  encoding: [0x62,0x12,0xa5,0x10,0xb6,0xc8]
+          vfmaddsub231pd {rn-sae}, %zmm24, %zmm27, %zmm9
+
+// CHECK: vfmaddsub231pd {ru-sae}, %zmm24, %zmm27, %zmm9
+// CHECK:  encoding: [0x62,0x12,0xa5,0x50,0xb6,0xc8]
+          vfmaddsub231pd {ru-sae}, %zmm24, %zmm27, %zmm9
+
+// CHECK: vfmaddsub231pd {rd-sae}, %zmm24, %zmm27, %zmm9
+// CHECK:  encoding: [0x62,0x12,0xa5,0x30,0xb6,0xc8]
+          vfmaddsub231pd {rd-sae}, %zmm24, %zmm27, %zmm9
+
+// CHECK: vfmaddsub231pd {rz-sae}, %zmm24, %zmm27, %zmm9
+// CHECK:  encoding: [0x62,0x12,0xa5,0x70,0xb6,0xc8]
+          vfmaddsub231pd {rz-sae}, %zmm24, %zmm27, %zmm9
+
+// CHECK: vfmaddsub231pd (%rcx), %zmm27, %zmm9
+// CHECK:  encoding: [0x62,0x72,0xa5,0x40,0xb6,0x09]
+          vfmaddsub231pd (%rcx), %zmm27, %zmm9
+
+// CHECK: vfmaddsub231pd 291(%rax,%r14,8), %zmm27, %zmm9
+// CHECK:  encoding: [0x62,0x32,0xa5,0x40,0xb6,0x8c,0xf0,0x23,0x01,0x00,0x00]
+          vfmaddsub231pd 291(%rax,%r14,8), %zmm27, %zmm9
+
+// CHECK: vfmaddsub231pd (%rcx){1to8}, %zmm27, %zmm9
+// CHECK:  encoding: [0x62,0x72,0xa5,0x50,0xb6,0x09]
+          vfmaddsub231pd (%rcx){1to8}, %zmm27, %zmm9
+
+// CHECK: vfmaddsub231pd 8128(%rdx), %zmm27, %zmm9
+// CHECK:  encoding: [0x62,0x72,0xa5,0x40,0xb6,0x4a,0x7f]
+          vfmaddsub231pd 8128(%rdx), %zmm27, %zmm9
+
+// CHECK: vfmaddsub231pd 8192(%rdx), %zmm27, %zmm9
+// CHECK:  encoding: [0x62,0x72,0xa5,0x40,0xb6,0x8a,0x00,0x20,0x00,0x00]
+          vfmaddsub231pd 8192(%rdx), %zmm27, %zmm9
+
+// CHECK: vfmaddsub231pd -8192(%rdx), %zmm27, %zmm9
+// CHECK:  encoding: [0x62,0x72,0xa5,0x40,0xb6,0x4a,0x80]
+          vfmaddsub231pd -8192(%rdx), %zmm27, %zmm9
+
+// CHECK: vfmaddsub231pd -8256(%rdx), %zmm27, %zmm9
+// CHECK:  encoding: [0x62,0x72,0xa5,0x40,0xb6,0x8a,0xc0,0xdf,0xff,0xff]
+          vfmaddsub231pd -8256(%rdx), %zmm27, %zmm9
+
+// CHECK: vfmaddsub231pd 1016(%rdx){1to8}, %zmm27, %zmm9
+// CHECK:  encoding: [0x62,0x72,0xa5,0x50,0xb6,0x4a,0x7f]
+          vfmaddsub231pd 1016(%rdx){1to8}, %zmm27, %zmm9
+
+// CHECK: vfmaddsub231pd 1024(%rdx){1to8}, %zmm27, %zmm9
+// CHECK:  encoding: [0x62,0x72,0xa5,0x50,0xb6,0x8a,0x00,0x04,0x00,0x00]
+          vfmaddsub231pd 1024(%rdx){1to8}, %zmm27, %zmm9
+
+// CHECK: vfmaddsub231pd -1024(%rdx){1to8}, %zmm27, %zmm9
+// CHECK:  encoding: [0x62,0x72,0xa5,0x50,0xb6,0x4a,0x80]
+          vfmaddsub231pd -1024(%rdx){1to8}, %zmm27, %zmm9
+
+// CHECK: vfmaddsub231pd -1032(%rdx){1to8}, %zmm27, %zmm9
+// CHECK:  encoding: [0x62,0x72,0xa5,0x50,0xb6,0x8a,0xf8,0xfb,0xff,0xff]
+          vfmaddsub231pd -1032(%rdx){1to8}, %zmm27, %zmm9
+
+// CHECK: vfmsubadd132ps %zmm21, %zmm13, %zmm2
+// CHECK:  encoding: [0x62,0xb2,0x15,0x48,0x97,0xd5]
+          vfmsubadd132ps %zmm21, %zmm13, %zmm2
+
+// CHECK: vfmsubadd132ps %zmm21, %zmm13, %zmm2 {%k7}
+// CHECK:  encoding: [0x62,0xb2,0x15,0x4f,0x97,0xd5]
+          vfmsubadd132ps %zmm21, %zmm13, %zmm2 {%k7}
+
+// CHECK: vfmsubadd132ps %zmm21, %zmm13, %zmm2 {%k7} {z}
+// CHECK:  encoding: [0x62,0xb2,0x15,0xcf,0x97,0xd5]
+          vfmsubadd132ps %zmm21, %zmm13, %zmm2 {%k7} {z}
+
+// CHECK: vfmsubadd132ps {rn-sae}, %zmm21, %zmm13, %zmm2
+// CHECK:  encoding: [0x62,0xb2,0x15,0x18,0x97,0xd5]
+          vfmsubadd132ps {rn-sae}, %zmm21, %zmm13, %zmm2
+
+// CHECK: vfmsubadd132ps {ru-sae}, %zmm21, %zmm13, %zmm2
+// CHECK:  encoding: [0x62,0xb2,0x15,0x58,0x97,0xd5]
+          vfmsubadd132ps {ru-sae}, %zmm21, %zmm13, %zmm2
+
+// CHECK: vfmsubadd132ps {rd-sae}, %zmm21, %zmm13, %zmm2
+// CHECK:  encoding: [0x62,0xb2,0x15,0x38,0x97,0xd5]
+          vfmsubadd132ps {rd-sae}, %zmm21, %zmm13, %zmm2
+
+// CHECK: vfmsubadd132ps {rz-sae}, %zmm21, %zmm13, %zmm2
+// CHECK:  encoding: [0x62,0xb2,0x15,0x78,0x97,0xd5]
+          vfmsubadd132ps {rz-sae}, %zmm21, %zmm13, %zmm2
+
+// CHECK: vfmsubadd132ps (%rcx), %zmm13, %zmm2
+// CHECK:  encoding: [0x62,0xf2,0x15,0x48,0x97,0x11]
+          vfmsubadd132ps (%rcx), %zmm13, %zmm2
+
+// CHECK: vfmsubadd132ps 291(%rax,%r14,8), %zmm13, %zmm2
+// CHECK:  encoding: [0x62,0xb2,0x15,0x48,0x97,0x94,0xf0,0x23,0x01,0x00,0x00]
+          vfmsubadd132ps 291(%rax,%r14,8), %zmm13, %zmm2
+
+// CHECK: vfmsubadd132ps (%rcx){1to16}, %zmm13, %zmm2
+// CHECK:  encoding: [0x62,0xf2,0x15,0x58,0x97,0x11]
+          vfmsubadd132ps (%rcx){1to16}, %zmm13, %zmm2
+
+// CHECK: vfmsubadd132ps 8128(%rdx), %zmm13, %zmm2
+// CHECK:  encoding: [0x62,0xf2,0x15,0x48,0x97,0x52,0x7f]
+          vfmsubadd132ps 8128(%rdx), %zmm13, %zmm2
+
+// CHECK: vfmsubadd132ps 8192(%rdx), %zmm13, %zmm2
+// CHECK:  encoding: [0x62,0xf2,0x15,0x48,0x97,0x92,0x00,0x20,0x00,0x00]
+          vfmsubadd132ps 8192(%rdx), %zmm13, %zmm2
+
+// CHECK: vfmsubadd132ps -8192(%rdx), %zmm13, %zmm2
+// CHECK:  encoding: [0x62,0xf2,0x15,0x48,0x97,0x52,0x80]
+          vfmsubadd132ps -8192(%rdx), %zmm13, %zmm2
+
+// CHECK: vfmsubadd132ps -8256(%rdx), %zmm13, %zmm2
+// CHECK:  encoding: [0x62,0xf2,0x15,0x48,0x97,0x92,0xc0,0xdf,0xff,0xff]
+          vfmsubadd132ps -8256(%rdx), %zmm13, %zmm2
+
+// CHECK: vfmsubadd132ps 508(%rdx){1to16}, %zmm13, %zmm2
+// CHECK:  encoding: [0x62,0xf2,0x15,0x58,0x97,0x52,0x7f]
+          vfmsubadd132ps 508(%rdx){1to16}, %zmm13, %zmm2
+
+// CHECK: vfmsubadd132ps 512(%rdx){1to16}, %zmm13, %zmm2
+// CHECK:  encoding: [0x62,0xf2,0x15,0x58,0x97,0x92,0x00,0x02,0x00,0x00]
+          vfmsubadd132ps 512(%rdx){1to16}, %zmm13, %zmm2
+
+// CHECK: vfmsubadd132ps -512(%rdx){1to16}, %zmm13, %zmm2
+// CHECK:  encoding: [0x62,0xf2,0x15,0x58,0x97,0x52,0x80]
+          vfmsubadd132ps -512(%rdx){1to16}, %zmm13, %zmm2
+
+// CHECK: vfmsubadd132ps -516(%rdx){1to16}, %zmm13, %zmm2
+// CHECK:  encoding: [0x62,0xf2,0x15,0x58,0x97,0x92,0xfc,0xfd,0xff,0xff]
+          vfmsubadd132ps -516(%rdx){1to16}, %zmm13, %zmm2
+
+// CHECK: vfmsubadd132pd %zmm18, %zmm28, %zmm21
+// CHECK:  encoding: [0x62,0xa2,0x9d,0x40,0x97,0xea]
+          vfmsubadd132pd %zmm18, %zmm28, %zmm21
+
+// CHECK: vfmsubadd132pd %zmm18, %zmm28, %zmm21 {%k7}
+// CHECK:  encoding: [0x62,0xa2,0x9d,0x47,0x97,0xea]
+          vfmsubadd132pd %zmm18, %zmm28, %zmm21 {%k7}
+
+// CHECK: vfmsubadd132pd %zmm18, %zmm28, %zmm21 {%k7} {z}
+// CHECK:  encoding: [0x62,0xa2,0x9d,0xc7,0x97,0xea]
+          vfmsubadd132pd %zmm18, %zmm28, %zmm21 {%k7} {z}
+
+// CHECK: vfmsubadd132pd {rn-sae}, %zmm18, %zmm28, %zmm21
+// CHECK:  encoding: [0x62,0xa2,0x9d,0x10,0x97,0xea]
+          vfmsubadd132pd {rn-sae}, %zmm18, %zmm28, %zmm21
+
+// CHECK: vfmsubadd132pd {ru-sae}, %zmm18, %zmm28, %zmm21
+// CHECK:  encoding: [0x62,0xa2,0x9d,0x50,0x97,0xea]
+          vfmsubadd132pd {ru-sae}, %zmm18, %zmm28, %zmm21
+
+// CHECK: vfmsubadd132pd {rd-sae}, %zmm18, %zmm28, %zmm21
+// CHECK:  encoding: [0x62,0xa2,0x9d,0x30,0x97,0xea]
+          vfmsubadd132pd {rd-sae}, %zmm18, %zmm28, %zmm21
+
+// CHECK: vfmsubadd132pd {rz-sae}, %zmm18, %zmm28, %zmm21
+// CHECK:  encoding: [0x62,0xa2,0x9d,0x70,0x97,0xea]
+          vfmsubadd132pd {rz-sae}, %zmm18, %zmm28, %zmm21
+
+// CHECK: vfmsubadd132pd (%rcx), %zmm28, %zmm21
+// CHECK:  encoding: [0x62,0xe2,0x9d,0x40,0x97,0x29]
+          vfmsubadd132pd (%rcx), %zmm28, %zmm21
+
+// CHECK: vfmsubadd132pd 291(%rax,%r14,8), %zmm28, %zmm21
+// CHECK:  encoding: [0x62,0xa2,0x9d,0x40,0x97,0xac,0xf0,0x23,0x01,0x00,0x00]
+          vfmsubadd132pd 291(%rax,%r14,8), %zmm28, %zmm21
+
+// CHECK: vfmsubadd132pd (%rcx){1to8}, %zmm28, %zmm21
+// CHECK:  encoding: [0x62,0xe2,0x9d,0x50,0x97,0x29]
+          vfmsubadd132pd (%rcx){1to8}, %zmm28, %zmm21
+
+// CHECK: vfmsubadd132pd 8128(%rdx), %zmm28, %zmm21
+// CHECK:  encoding: [0x62,0xe2,0x9d,0x40,0x97,0x6a,0x7f]
+          vfmsubadd132pd 8128(%rdx), %zmm28, %zmm21
+
+// CHECK: vfmsubadd132pd 8192(%rdx), %zmm28, %zmm21
+// CHECK:  encoding: [0x62,0xe2,0x9d,0x40,0x97,0xaa,0x00,0x20,0x00,0x00]
+          vfmsubadd132pd 8192(%rdx), %zmm28, %zmm21
+
+// CHECK: vfmsubadd132pd -8192(%rdx), %zmm28, %zmm21
+// CHECK:  encoding: [0x62,0xe2,0x9d,0x40,0x97,0x6a,0x80]
+          vfmsubadd132pd -8192(%rdx), %zmm28, %zmm21
+
+// CHECK: vfmsubadd132pd -8256(%rdx), %zmm28, %zmm21
+// CHECK:  encoding: [0x62,0xe2,0x9d,0x40,0x97,0xaa,0xc0,0xdf,0xff,0xff]
+          vfmsubadd132pd -8256(%rdx), %zmm28, %zmm21
+
+// CHECK: vfmsubadd132pd 1016(%rdx){1to8}, %zmm28, %zmm21
+// CHECK:  encoding: [0x62,0xe2,0x9d,0x50,0x97,0x6a,0x7f]
+          vfmsubadd132pd 1016(%rdx){1to8}, %zmm28, %zmm21
+
+// CHECK: vfmsubadd132pd 1024(%rdx){1to8}, %zmm28, %zmm21
+// CHECK:  encoding: [0x62,0xe2,0x9d,0x50,0x97,0xaa,0x00,0x04,0x00,0x00]
+          vfmsubadd132pd 1024(%rdx){1to8}, %zmm28, %zmm21
+
+// CHECK: vfmsubadd132pd -1024(%rdx){1to8}, %zmm28, %zmm21
+// CHECK:  encoding: [0x62,0xe2,0x9d,0x50,0x97,0x6a,0x80]
+          vfmsubadd132pd -1024(%rdx){1to8}, %zmm28, %zmm21
+
+// CHECK: vfmsubadd132pd -1032(%rdx){1to8}, %zmm28, %zmm21
+// CHECK:  encoding: [0x62,0xe2,0x9d,0x50,0x97,0xaa,0xf8,0xfb,0xff,0xff]
+          vfmsubadd132pd -1032(%rdx){1to8}, %zmm28, %zmm21
+
+// CHECK: vfmsubadd213ps %zmm14, %zmm12, %zmm22
+// CHECK:  encoding: [0x62,0xc2,0x1d,0x48,0xa7,0xf6]
+          vfmsubadd213ps %zmm14, %zmm12, %zmm22
+
+// CHECK: vfmsubadd213ps %zmm14, %zmm12, %zmm22 {%k6}
+// CHECK:  encoding: [0x62,0xc2,0x1d,0x4e,0xa7,0xf6]
+          vfmsubadd213ps %zmm14, %zmm12, %zmm22 {%k6}
+
+// CHECK: vfmsubadd213ps %zmm14, %zmm12, %zmm22 {%k6} {z}
+// CHECK:  encoding: [0x62,0xc2,0x1d,0xce,0xa7,0xf6]
+          vfmsubadd213ps %zmm14, %zmm12, %zmm22 {%k6} {z}
+
+// CHECK: vfmsubadd213ps {rn-sae}, %zmm14, %zmm12, %zmm22
+// CHECK:  encoding: [0x62,0xc2,0x1d,0x18,0xa7,0xf6]
+          vfmsubadd213ps {rn-sae}, %zmm14, %zmm12, %zmm22
+
+// CHECK: vfmsubadd213ps {ru-sae}, %zmm14, %zmm12, %zmm22
+// CHECK:  encoding: [0x62,0xc2,0x1d,0x58,0xa7,0xf6]
+          vfmsubadd213ps {ru-sae}, %zmm14, %zmm12, %zmm22
+
+// CHECK: vfmsubadd213ps {rd-sae}, %zmm14, %zmm12, %zmm22
+// CHECK:  encoding: [0x62,0xc2,0x1d,0x38,0xa7,0xf6]
+          vfmsubadd213ps {rd-sae}, %zmm14, %zmm12, %zmm22
+
+// CHECK: vfmsubadd213ps {rz-sae}, %zmm14, %zmm12, %zmm22
+// CHECK:  encoding: [0x62,0xc2,0x1d,0x78,0xa7,0xf6]
+          vfmsubadd213ps {rz-sae}, %zmm14, %zmm12, %zmm22
+
+// CHECK: vfmsubadd213ps (%rcx), %zmm12, %zmm22
+// CHECK:  encoding: [0x62,0xe2,0x1d,0x48,0xa7,0x31]
+          vfmsubadd213ps (%rcx), %zmm12, %zmm22
+
+// CHECK: vfmsubadd213ps 291(%rax,%r14,8), %zmm12, %zmm22
+// CHECK:  encoding: [0x62,0xa2,0x1d,0x48,0xa7,0xb4,0xf0,0x23,0x01,0x00,0x00]
+          vfmsubadd213ps 291(%rax,%r14,8), %zmm12, %zmm22
+
+// CHECK: vfmsubadd213ps (%rcx){1to16}, %zmm12, %zmm22
+// CHECK:  encoding: [0x62,0xe2,0x1d,0x58,0xa7,0x31]
+          vfmsubadd213ps (%rcx){1to16}, %zmm12, %zmm22
+
+// CHECK: vfmsubadd213ps 8128(%rdx), %zmm12, %zmm22
+// CHECK:  encoding: [0x62,0xe2,0x1d,0x48,0xa7,0x72,0x7f]
+          vfmsubadd213ps 8128(%rdx), %zmm12, %zmm22
+
+// CHECK: vfmsubadd213ps 8192(%rdx), %zmm12, %zmm22
+// CHECK:  encoding: [0x62,0xe2,0x1d,0x48,0xa7,0xb2,0x00,0x20,0x00,0x00]
+          vfmsubadd213ps 8192(%rdx), %zmm12, %zmm22
+
+// CHECK: vfmsubadd213ps -8192(%rdx), %zmm12, %zmm22
+// CHECK:  encoding: [0x62,0xe2,0x1d,0x48,0xa7,0x72,0x80]
+          vfmsubadd213ps -8192(%rdx), %zmm12, %zmm22
+
+// CHECK: vfmsubadd213ps -8256(%rdx), %zmm12, %zmm22
+// CHECK:  encoding: [0x62,0xe2,0x1d,0x48,0xa7,0xb2,0xc0,0xdf,0xff,0xff]
+          vfmsubadd213ps -8256(%rdx), %zmm12, %zmm22
+
+// CHECK: vfmsubadd213ps 508(%rdx){1to16}, %zmm12, %zmm22
+// CHECK:  encoding: [0x62,0xe2,0x1d,0x58,0xa7,0x72,0x7f]
+          vfmsubadd213ps 508(%rdx){1to16}, %zmm12, %zmm22
+
+// CHECK: vfmsubadd213ps 512(%rdx){1to16}, %zmm12, %zmm22
+// CHECK:  encoding: [0x62,0xe2,0x1d,0x58,0xa7,0xb2,0x00,0x02,0x00,0x00]
+          vfmsubadd213ps 512(%rdx){1to16}, %zmm12, %zmm22
+
+// CHECK: vfmsubadd213ps -512(%rdx){1to16}, %zmm12, %zmm22
+// CHECK:  encoding: [0x62,0xe2,0x1d,0x58,0xa7,0x72,0x80]
+          vfmsubadd213ps -512(%rdx){1to16}, %zmm12, %zmm22
+
+// CHECK: vfmsubadd213ps -516(%rdx){1to16}, %zmm12, %zmm22
+// CHECK:  encoding: [0x62,0xe2,0x1d,0x58,0xa7,0xb2,0xfc,0xfd,0xff,0xff]
+          vfmsubadd213ps -516(%rdx){1to16}, %zmm12, %zmm22
+
+// CHECK: vfmsubadd213pd %zmm2, %zmm23, %zmm2
+// CHECK:  encoding: [0x62,0xf2,0xc5,0x40,0xa7,0xd2]
+          vfmsubadd213pd %zmm2, %zmm23, %zmm2
+
+// CHECK: vfmsubadd213pd %zmm2, %zmm23, %zmm2 {%k6}
+// CHECK:  encoding: [0x62,0xf2,0xc5,0x46,0xa7,0xd2]
+          vfmsubadd213pd %zmm2, %zmm23, %zmm2 {%k6}
+
+// CHECK: vfmsubadd213pd %zmm2, %zmm23, %zmm2 {%k6} {z}
+// CHECK:  encoding: [0x62,0xf2,0xc5,0xc6,0xa7,0xd2]
+          vfmsubadd213pd %zmm2, %zmm23, %zmm2 {%k6} {z}
+
+// CHECK: vfmsubadd213pd {rn-sae}, %zmm2, %zmm23, %zmm2
+// CHECK:  encoding: [0x62,0xf2,0xc5,0x10,0xa7,0xd2]
+          vfmsubadd213pd {rn-sae}, %zmm2, %zmm23, %zmm2
+
+// CHECK: vfmsubadd213pd {ru-sae}, %zmm2, %zmm23, %zmm2
+// CHECK:  encoding: [0x62,0xf2,0xc5,0x50,0xa7,0xd2]
+          vfmsubadd213pd {ru-sae}, %zmm2, %zmm23, %zmm2
+
+// CHECK: vfmsubadd213pd {rd-sae}, %zmm2, %zmm23, %zmm2
+// CHECK:  encoding: [0x62,0xf2,0xc5,0x30,0xa7,0xd2]
+          vfmsubadd213pd {rd-sae}, %zmm2, %zmm23, %zmm2
+
+// CHECK: vfmsubadd213pd {rz-sae}, %zmm2, %zmm23, %zmm2
+// CHECK:  encoding: [0x62,0xf2,0xc5,0x70,0xa7,0xd2]
+          vfmsubadd213pd {rz-sae}, %zmm2, %zmm23, %zmm2
+
+// CHECK: vfmsubadd213pd (%rcx), %zmm23, %zmm2
+// CHECK:  encoding: [0x62,0xf2,0xc5,0x40,0xa7,0x11]
+          vfmsubadd213pd (%rcx), %zmm23, %zmm2
+
+// CHECK: vfmsubadd213pd 291(%rax,%r14,8), %zmm23, %zmm2
+// CHECK:  encoding: [0x62,0xb2,0xc5,0x40,0xa7,0x94,0xf0,0x23,0x01,0x00,0x00]
+          vfmsubadd213pd 291(%rax,%r14,8), %zmm23, %zmm2
+
+// CHECK: vfmsubadd213pd (%rcx){1to8}, %zmm23, %zmm2
+// CHECK:  encoding: [0x62,0xf2,0xc5,0x50,0xa7,0x11]
+          vfmsubadd213pd (%rcx){1to8}, %zmm23, %zmm2
+
+// CHECK: vfmsubadd213pd 8128(%rdx), %zmm23, %zmm2
+// CHECK:  encoding: [0x62,0xf2,0xc5,0x40,0xa7,0x52,0x7f]
+          vfmsubadd213pd 8128(%rdx), %zmm23, %zmm2
+
+// CHECK: vfmsubadd213pd 8192(%rdx), %zmm23, %zmm2
+// CHECK:  encoding: [0x62,0xf2,0xc5,0x40,0xa7,0x92,0x00,0x20,0x00,0x00]
+          vfmsubadd213pd 8192(%rdx), %zmm23, %zmm2
+
+// CHECK: vfmsubadd213pd -8192(%rdx), %zmm23, %zmm2
+// CHECK:  encoding: [0x62,0xf2,0xc5,0x40,0xa7,0x52,0x80]
+          vfmsubadd213pd -8192(%rdx), %zmm23, %zmm2
+
+// CHECK: vfmsubadd213pd -8256(%rdx), %zmm23, %zmm2
+// CHECK:  encoding: [0x62,0xf2,0xc5,0x40,0xa7,0x92,0xc0,0xdf,0xff,0xff]
+          vfmsubadd213pd -8256(%rdx), %zmm23, %zmm2
+
+// CHECK: vfmsubadd213pd 1016(%rdx){1to8}, %zmm23, %zmm2
+// CHECK:  encoding: [0x62,0xf2,0xc5,0x50,0xa7,0x52,0x7f]
+          vfmsubadd213pd 1016(%rdx){1to8}, %zmm23, %zmm2
+
+// CHECK: vfmsubadd213pd 1024(%rdx){1to8}, %zmm23, %zmm2
+// CHECK:  encoding: [0x62,0xf2,0xc5,0x50,0xa7,0x92,0x00,0x04,0x00,0x00]
+          vfmsubadd213pd 1024(%rdx){1to8}, %zmm23, %zmm2
+
+// CHECK: vfmsubadd213pd -1024(%rdx){1to8}, %zmm23, %zmm2
+// CHECK:  encoding: [0x62,0xf2,0xc5,0x50,0xa7,0x52,0x80]
+          vfmsubadd213pd -1024(%rdx){1to8}, %zmm23, %zmm2
+
+// CHECK: vfmsubadd213pd -1032(%rdx){1to8}, %zmm23, %zmm2
+// CHECK:  encoding: [0x62,0xf2,0xc5,0x50,0xa7,0x92,0xf8,0xfb,0xff,0xff]
+          vfmsubadd213pd -1032(%rdx){1to8}, %zmm23, %zmm2
+
+// CHECK: vfmsubadd231ps %zmm1, %zmm19, %zmm8
+// CHECK:  encoding: [0x62,0x72,0x65,0x40,0xb7,0xc1]
+          vfmsubadd231ps %zmm1, %zmm19, %zmm8
+
+// CHECK: vfmsubadd231ps %zmm1, %zmm19, %zmm8 {%k2}
+// CHECK:  encoding: [0x62,0x72,0x65,0x42,0xb7,0xc1]
+          vfmsubadd231ps %zmm1, %zmm19, %zmm8 {%k2}
+
+// CHECK: vfmsubadd231ps %zmm1, %zmm19, %zmm8 {%k2} {z}
+// CHECK:  encoding: [0x62,0x72,0x65,0xc2,0xb7,0xc1]
+          vfmsubadd231ps %zmm1, %zmm19, %zmm8 {%k2} {z}
+
+// CHECK: vfmsubadd231ps {rn-sae}, %zmm1, %zmm19, %zmm8
+// CHECK:  encoding: [0x62,0x72,0x65,0x10,0xb7,0xc1]
+          vfmsubadd231ps {rn-sae}, %zmm1, %zmm19, %zmm8
+
+// CHECK: vfmsubadd231ps {ru-sae}, %zmm1, %zmm19, %zmm8
+// CHECK:  encoding: [0x62,0x72,0x65,0x50,0xb7,0xc1]
+          vfmsubadd231ps {ru-sae}, %zmm1, %zmm19, %zmm8
+
+// CHECK: vfmsubadd231ps {rd-sae}, %zmm1, %zmm19, %zmm8
+// CHECK:  encoding: [0x62,0x72,0x65,0x30,0xb7,0xc1]
+          vfmsubadd231ps {rd-sae}, %zmm1, %zmm19, %zmm8
+
+// CHECK: vfmsubadd231ps {rz-sae}, %zmm1, %zmm19, %zmm8
+// CHECK:  encoding: [0x62,0x72,0x65,0x70,0xb7,0xc1]
+          vfmsubadd231ps {rz-sae}, %zmm1, %zmm19, %zmm8
+
+// CHECK: vfmsubadd231ps (%rcx), %zmm19, %zmm8
+// CHECK:  encoding: [0x62,0x72,0x65,0x40,0xb7,0x01]
+          vfmsubadd231ps (%rcx), %zmm19, %zmm8
+
+// CHECK: vfmsubadd231ps 291(%rax,%r14,8), %zmm19, %zmm8
+// CHECK:  encoding: [0x62,0x32,0x65,0x40,0xb7,0x84,0xf0,0x23,0x01,0x00,0x00]
+          vfmsubadd231ps 291(%rax,%r14,8), %zmm19, %zmm8
+
+// CHECK: vfmsubadd231ps (%rcx){1to16}, %zmm19, %zmm8
+// CHECK:  encoding: [0x62,0x72,0x65,0x50,0xb7,0x01]
+          vfmsubadd231ps (%rcx){1to16}, %zmm19, %zmm8
+
+// CHECK: vfmsubadd231ps 8128(%rdx), %zmm19, %zmm8
+// CHECK:  encoding: [0x62,0x72,0x65,0x40,0xb7,0x42,0x7f]
+          vfmsubadd231ps 8128(%rdx), %zmm19, %zmm8
+
+// CHECK: vfmsubadd231ps 8192(%rdx), %zmm19, %zmm8
+// CHECK:  encoding: [0x62,0x72,0x65,0x40,0xb7,0x82,0x00,0x20,0x00,0x00]
+          vfmsubadd231ps 8192(%rdx), %zmm19, %zmm8
+
+// CHECK: vfmsubadd231ps -8192(%rdx), %zmm19, %zmm8
+// CHECK:  encoding: [0x62,0x72,0x65,0x40,0xb7,0x42,0x80]
+          vfmsubadd231ps -8192(%rdx), %zmm19, %zmm8
+
+// CHECK: vfmsubadd231ps -8256(%rdx), %zmm19, %zmm8
+// CHECK:  encoding: [0x62,0x72,0x65,0x40,0xb7,0x82,0xc0,0xdf,0xff,0xff]
+          vfmsubadd231ps -8256(%rdx), %zmm19, %zmm8
+
+// CHECK: vfmsubadd231ps 508(%rdx){1to16}, %zmm19, %zmm8
+// CHECK:  encoding: [0x62,0x72,0x65,0x50,0xb7,0x42,0x7f]
+          vfmsubadd231ps 508(%rdx){1to16}, %zmm19, %zmm8
+
+// CHECK: vfmsubadd231ps 512(%rdx){1to16}, %zmm19, %zmm8
+// CHECK:  encoding: [0x62,0x72,0x65,0x50,0xb7,0x82,0x00,0x02,0x00,0x00]
+          vfmsubadd231ps 512(%rdx){1to16}, %zmm19, %zmm8
+
+// CHECK: vfmsubadd231ps -512(%rdx){1to16}, %zmm19, %zmm8
+// CHECK:  encoding: [0x62,0x72,0x65,0x50,0xb7,0x42,0x80]
+          vfmsubadd231ps -512(%rdx){1to16}, %zmm19, %zmm8
+
+// CHECK: vfmsubadd231ps -516(%rdx){1to16}, %zmm19, %zmm8
+// CHECK:  encoding: [0x62,0x72,0x65,0x50,0xb7,0x82,0xfc,0xfd,0xff,0xff]
+          vfmsubadd231ps -516(%rdx){1to16}, %zmm19, %zmm8
+
+// CHECK: vfmsubadd231pd %zmm21, %zmm27, %zmm16
+// CHECK:  encoding: [0x62,0xa2,0xa5,0x40,0xb7,0xc5]
+          vfmsubadd231pd %zmm21, %zmm27, %zmm16
+
+// CHECK: vfmsubadd231pd %zmm21, %zmm27, %zmm16 {%k2}
+// CHECK:  encoding: [0x62,0xa2,0xa5,0x42,0xb7,0xc5]
+          vfmsubadd231pd %zmm21, %zmm27, %zmm16 {%k2}
+
+// CHECK: vfmsubadd231pd %zmm21, %zmm27, %zmm16 {%k2} {z}
+// CHECK:  encoding: [0x62,0xa2,0xa5,0xc2,0xb7,0xc5]
+          vfmsubadd231pd %zmm21, %zmm27, %zmm16 {%k2} {z}
+
+// CHECK: vfmsubadd231pd {rn-sae}, %zmm21, %zmm27, %zmm16
+// CHECK:  encoding: [0x62,0xa2,0xa5,0x10,0xb7,0xc5]
+          vfmsubadd231pd {rn-sae}, %zmm21, %zmm27, %zmm16
+
+// CHECK: vfmsubadd231pd {ru-sae}, %zmm21, %zmm27, %zmm16
+// CHECK:  encoding: [0x62,0xa2,0xa5,0x50,0xb7,0xc5]
+          vfmsubadd231pd {ru-sae}, %zmm21, %zmm27, %zmm16
+
+// CHECK: vfmsubadd231pd {rd-sae}, %zmm21, %zmm27, %zmm16
+// CHECK:  encoding: [0x62,0xa2,0xa5,0x30,0xb7,0xc5]
+          vfmsubadd231pd {rd-sae}, %zmm21, %zmm27, %zmm16
+
+// CHECK: vfmsubadd231pd {rz-sae}, %zmm21, %zmm27, %zmm16
+// CHECK:  encoding: [0x62,0xa2,0xa5,0x70,0xb7,0xc5]
+          vfmsubadd231pd {rz-sae}, %zmm21, %zmm27, %zmm16
+
+// CHECK: vfmsubadd231pd (%rcx), %zmm27, %zmm16
+// CHECK:  encoding: [0x62,0xe2,0xa5,0x40,0xb7,0x01]
+          vfmsubadd231pd (%rcx), %zmm27, %zmm16
+
+// CHECK: vfmsubadd231pd 291(%rax,%r14,8), %zmm27, %zmm16
+// CHECK:  encoding: [0x62,0xa2,0xa5,0x40,0xb7,0x84,0xf0,0x23,0x01,0x00,0x00]
+          vfmsubadd231pd 291(%rax,%r14,8), %zmm27, %zmm16
+
+// CHECK: vfmsubadd231pd (%rcx){1to8}, %zmm27, %zmm16
+// CHECK:  encoding: [0x62,0xe2,0xa5,0x50,0xb7,0x01]
+          vfmsubadd231pd (%rcx){1to8}, %zmm27, %zmm16
+
+// CHECK: vfmsubadd231pd 8128(%rdx), %zmm27, %zmm16
+// CHECK:  encoding: [0x62,0xe2,0xa5,0x40,0xb7,0x42,0x7f]
+          vfmsubadd231pd 8128(%rdx), %zmm27, %zmm16
+
+// CHECK: vfmsubadd231pd 8192(%rdx), %zmm27, %zmm16
+// CHECK:  encoding: [0x62,0xe2,0xa5,0x40,0xb7,0x82,0x00,0x20,0x00,0x00]
+          vfmsubadd231pd 8192(%rdx), %zmm27, %zmm16
+
+// CHECK: vfmsubadd231pd -8192(%rdx), %zmm27, %zmm16
+// CHECK:  encoding: [0x62,0xe2,0xa5,0x40,0xb7,0x42,0x80]
+          vfmsubadd231pd -8192(%rdx), %zmm27, %zmm16
+
+// CHECK: vfmsubadd231pd -8256(%rdx), %zmm27, %zmm16
+// CHECK:  encoding: [0x62,0xe2,0xa5,0x40,0xb7,0x82,0xc0,0xdf,0xff,0xff]
+          vfmsubadd231pd -8256(%rdx), %zmm27, %zmm16
+
+// CHECK: vfmsubadd231pd 1016(%rdx){1to8}, %zmm27, %zmm16
+// CHECK:  encoding: [0x62,0xe2,0xa5,0x50,0xb7,0x42,0x7f]
+          vfmsubadd231pd 1016(%rdx){1to8}, %zmm27, %zmm16
+
+// CHECK: vfmsubadd231pd 1024(%rdx){1to8}, %zmm27, %zmm16
+// CHECK:  encoding: [0x62,0xe2,0xa5,0x50,0xb7,0x82,0x00,0x04,0x00,0x00]
+          vfmsubadd231pd 1024(%rdx){1to8}, %zmm27, %zmm16
+
+// CHECK: vfmsubadd231pd -1024(%rdx){1to8}, %zmm27, %zmm16
+// CHECK:  encoding: [0x62,0xe2,0xa5,0x50,0xb7,0x42,0x80]
+          vfmsubadd231pd -1024(%rdx){1to8}, %zmm27, %zmm16
+
+// CHECK: vfmsubadd231pd -1032(%rdx){1to8}, %zmm27, %zmm16
+// CHECK:  encoding: [0x62,0xe2,0xa5,0x50,0xb7,0x82,0xf8,0xfb,0xff,0xff]
+          vfmsubadd231pd -1032(%rdx){1to8}, %zmm27, %zmm16
+
+// CHECK: vfnmadd132ps %zmm10, %zmm16, %zmm21
+// CHECK:  encoding: [0x62,0xc2,0x7d,0x40,0x9c,0xea]
+          vfnmadd132ps %zmm10, %zmm16, %zmm21
+
+// CHECK: vfnmadd132ps %zmm10, %zmm16, %zmm21 {%k5}
+// CHECK:  encoding: [0x62,0xc2,0x7d,0x45,0x9c,0xea]
+          vfnmadd132ps %zmm10, %zmm16, %zmm21 {%k5}
+
+// CHECK: vfnmadd132ps %zmm10, %zmm16, %zmm21 {%k5} {z}
+// CHECK:  encoding: [0x62,0xc2,0x7d,0xc5,0x9c,0xea]
+          vfnmadd132ps %zmm10, %zmm16, %zmm21 {%k5} {z}
+
+// CHECK: vfnmadd132ps {rn-sae}, %zmm10, %zmm16, %zmm21
+// CHECK:  encoding: [0x62,0xc2,0x7d,0x10,0x9c,0xea]
+          vfnmadd132ps {rn-sae}, %zmm10, %zmm16, %zmm21
+
+// CHECK: vfnmadd132ps {ru-sae}, %zmm10, %zmm16, %zmm21
+// CHECK:  encoding: [0x62,0xc2,0x7d,0x50,0x9c,0xea]
+          vfnmadd132ps {ru-sae}, %zmm10, %zmm16, %zmm21
+
+// CHECK: vfnmadd132ps {rd-sae}, %zmm10, %zmm16, %zmm21
+// CHECK:  encoding: [0x62,0xc2,0x7d,0x30,0x9c,0xea]
+          vfnmadd132ps {rd-sae}, %zmm10, %zmm16, %zmm21
+
+// CHECK: vfnmadd132ps {rz-sae}, %zmm10, %zmm16, %zmm21
+// CHECK:  encoding: [0x62,0xc2,0x7d,0x70,0x9c,0xea]
+          vfnmadd132ps {rz-sae}, %zmm10, %zmm16, %zmm21
+
+// CHECK: vfnmadd132ps (%rcx), %zmm16, %zmm21
+// CHECK:  encoding: [0x62,0xe2,0x7d,0x40,0x9c,0x29]
+          vfnmadd132ps (%rcx), %zmm16, %zmm21
+
+// CHECK: vfnmadd132ps 291(%rax,%r14,8), %zmm16, %zmm21
+// CHECK:  encoding: [0x62,0xa2,0x7d,0x40,0x9c,0xac,0xf0,0x23,0x01,0x00,0x00]
+          vfnmadd132ps 291(%rax,%r14,8), %zmm16, %zmm21
+
+// CHECK: vfnmadd132ps (%rcx){1to16}, %zmm16, %zmm21
+// CHECK:  encoding: [0x62,0xe2,0x7d,0x50,0x9c,0x29]
+          vfnmadd132ps (%rcx){1to16}, %zmm16, %zmm21
+
+// CHECK: vfnmadd132ps 8128(%rdx), %zmm16, %zmm21
+// CHECK:  encoding: [0x62,0xe2,0x7d,0x40,0x9c,0x6a,0x7f]
+          vfnmadd132ps 8128(%rdx), %zmm16, %zmm21
+
+// CHECK: vfnmadd132ps 8192(%rdx), %zmm16, %zmm21
+// CHECK:  encoding: [0x62,0xe2,0x7d,0x40,0x9c,0xaa,0x00,0x20,0x00,0x00]
+          vfnmadd132ps 8192(%rdx), %zmm16, %zmm21
+
+// CHECK: vfnmadd132ps -8192(%rdx), %zmm16, %zmm21
+// CHECK:  encoding: [0x62,0xe2,0x7d,0x40,0x9c,0x6a,0x80]
+          vfnmadd132ps -8192(%rdx), %zmm16, %zmm21
+
+// CHECK: vfnmadd132ps -8256(%rdx), %zmm16, %zmm21
+// CHECK:  encoding: [0x62,0xe2,0x7d,0x40,0x9c,0xaa,0xc0,0xdf,0xff,0xff]
+          vfnmadd132ps -8256(%rdx), %zmm16, %zmm21
+
+// CHECK: vfnmadd132ps 508(%rdx){1to16}, %zmm16, %zmm21
+// CHECK:  encoding: [0x62,0xe2,0x7d,0x50,0x9c,0x6a,0x7f]
+          vfnmadd132ps 508(%rdx){1to16}, %zmm16, %zmm21
+
+// CHECK: vfnmadd132ps 512(%rdx){1to16}, %zmm16, %zmm21
+// CHECK:  encoding: [0x62,0xe2,0x7d,0x50,0x9c,0xaa,0x00,0x02,0x00,0x00]
+          vfnmadd132ps 512(%rdx){1to16}, %zmm16, %zmm21
+
+// CHECK: vfnmadd132ps -512(%rdx){1to16}, %zmm16, %zmm21
+// CHECK:  encoding: [0x62,0xe2,0x7d,0x50,0x9c,0x6a,0x80]
+          vfnmadd132ps -512(%rdx){1to16}, %zmm16, %zmm21
+
+// CHECK: vfnmadd132ps -516(%rdx){1to16}, %zmm16, %zmm21
+// CHECK:  encoding: [0x62,0xe2,0x7d,0x50,0x9c,0xaa,0xfc,0xfd,0xff,0xff]
+          vfnmadd132ps -516(%rdx){1to16}, %zmm16, %zmm21
+
+// CHECK: vfnmadd132pd %zmm1, %zmm14, %zmm12
+// CHECK:  encoding: [0x62,0x72,0x8d,0x48,0x9c,0xe1]
+          vfnmadd132pd %zmm1, %zmm14, %zmm12
+
+// CHECK: vfnmadd132pd %zmm1, %zmm14, %zmm12 {%k7}
+// CHECK:  encoding: [0x62,0x72,0x8d,0x4f,0x9c,0xe1]
+          vfnmadd132pd %zmm1, %zmm14, %zmm12 {%k7}
+
+// CHECK: vfnmadd132pd %zmm1, %zmm14, %zmm12 {%k7} {z}
+// CHECK:  encoding: [0x62,0x72,0x8d,0xcf,0x9c,0xe1]
+          vfnmadd132pd %zmm1, %zmm14, %zmm12 {%k7} {z}
+
+// CHECK: vfnmadd132pd {rn-sae}, %zmm1, %zmm14, %zmm12
+// CHECK:  encoding: [0x62,0x72,0x8d,0x18,0x9c,0xe1]
+          vfnmadd132pd {rn-sae}, %zmm1, %zmm14, %zmm12
+
+// CHECK: vfnmadd132pd {ru-sae}, %zmm1, %zmm14, %zmm12
+// CHECK:  encoding: [0x62,0x72,0x8d,0x58,0x9c,0xe1]
+          vfnmadd132pd {ru-sae}, %zmm1, %zmm14, %zmm12
+
+// CHECK: vfnmadd132pd {rd-sae}, %zmm1, %zmm14, %zmm12
+// CHECK:  encoding: [0x62,0x72,0x8d,0x38,0x9c,0xe1]
+          vfnmadd132pd {rd-sae}, %zmm1, %zmm14, %zmm12
+
+// CHECK: vfnmadd132pd {rz-sae}, %zmm1, %zmm14, %zmm12
+// CHECK:  encoding: [0x62,0x72,0x8d,0x78,0x9c,0xe1]
+          vfnmadd132pd {rz-sae}, %zmm1, %zmm14, %zmm12
+
+// CHECK: vfnmadd132pd (%rcx), %zmm14, %zmm12
+// CHECK:  encoding: [0x62,0x72,0x8d,0x48,0x9c,0x21]
+          vfnmadd132pd (%rcx), %zmm14, %zmm12
+
+// CHECK: vfnmadd132pd 291(%rax,%r14,8), %zmm14, %zmm12
+// CHECK:  encoding: [0x62,0x32,0x8d,0x48,0x9c,0xa4,0xf0,0x23,0x01,0x00,0x00]
+          vfnmadd132pd 291(%rax,%r14,8), %zmm14, %zmm12
+
+// CHECK: vfnmadd132pd (%rcx){1to8}, %zmm14, %zmm12
+// CHECK:  encoding: [0x62,0x72,0x8d,0x58,0x9c,0x21]
+          vfnmadd132pd (%rcx){1to8}, %zmm14, %zmm12
+
+// CHECK: vfnmadd132pd 8128(%rdx), %zmm14, %zmm12
+// CHECK:  encoding: [0x62,0x72,0x8d,0x48,0x9c,0x62,0x7f]
+          vfnmadd132pd 8128(%rdx), %zmm14, %zmm12
+
+// CHECK: vfnmadd132pd 8192(%rdx), %zmm14, %zmm12
+// CHECK:  encoding: [0x62,0x72,0x8d,0x48,0x9c,0xa2,0x00,0x20,0x00,0x00]
+          vfnmadd132pd 8192(%rdx), %zmm14, %zmm12
+
+// CHECK: vfnmadd132pd -8192(%rdx), %zmm14, %zmm12
+// CHECK:  encoding: [0x62,0x72,0x8d,0x48,0x9c,0x62,0x80]
+          vfnmadd132pd -8192(%rdx), %zmm14, %zmm12
+
+// CHECK: vfnmadd132pd -8256(%rdx), %zmm14, %zmm12
+// CHECK:  encoding: [0x62,0x72,0x8d,0x48,0x9c,0xa2,0xc0,0xdf,0xff,0xff]
+          vfnmadd132pd -8256(%rdx), %zmm14, %zmm12
+
+// CHECK: vfnmadd132pd 1016(%rdx){1to8}, %zmm14, %zmm12
+// CHECK:  encoding: [0x62,0x72,0x8d,0x58,0x9c,0x62,0x7f]
+          vfnmadd132pd 1016(%rdx){1to8}, %zmm14, %zmm12
+
+// CHECK: vfnmadd132pd 1024(%rdx){1to8}, %zmm14, %zmm12
+// CHECK:  encoding: [0x62,0x72,0x8d,0x58,0x9c,0xa2,0x00,0x04,0x00,0x00]
+          vfnmadd132pd 1024(%rdx){1to8}, %zmm14, %zmm12
+
+// CHECK: vfnmadd132pd -1024(%rdx){1to8}, %zmm14, %zmm12
+// CHECK:  encoding: [0x62,0x72,0x8d,0x58,0x9c,0x62,0x80]
+          vfnmadd132pd -1024(%rdx){1to8}, %zmm14, %zmm12
+
+// CHECK: vfnmadd132pd -1032(%rdx){1to8}, %zmm14, %zmm12
+// CHECK:  encoding: [0x62,0x72,0x8d,0x58,0x9c,0xa2,0xf8,0xfb,0xff,0xff]
+          vfnmadd132pd -1032(%rdx){1to8}, %zmm14, %zmm12
+
+// CHECK: vfnmadd213ps %zmm6, %zmm10, %zmm26
+// CHECK:  encoding: [0x62,0x62,0x2d,0x48,0xac,0xd6]
+          vfnmadd213ps %zmm6, %zmm10, %zmm26
+
+// CHECK: vfnmadd213ps %zmm6, %zmm10, %zmm26 {%k6}
+// CHECK:  encoding: [0x62,0x62,0x2d,0x4e,0xac,0xd6]
+          vfnmadd213ps %zmm6, %zmm10, %zmm26 {%k6}
+
+// CHECK: vfnmadd213ps %zmm6, %zmm10, %zmm26 {%k6} {z}
+// CHECK:  encoding: [0x62,0x62,0x2d,0xce,0xac,0xd6]
+          vfnmadd213ps %zmm6, %zmm10, %zmm26 {%k6} {z}
+
+// CHECK: vfnmadd213ps {rn-sae}, %zmm6, %zmm10, %zmm26
+// CHECK:  encoding: [0x62,0x62,0x2d,0x18,0xac,0xd6]
+          vfnmadd213ps {rn-sae}, %zmm6, %zmm10, %zmm26
+
+// CHECK: vfnmadd213ps {ru-sae}, %zmm6, %zmm10, %zmm26
+// CHECK:  encoding: [0x62,0x62,0x2d,0x58,0xac,0xd6]
+          vfnmadd213ps {ru-sae}, %zmm6, %zmm10, %zmm26
+
+// CHECK: vfnmadd213ps {rd-sae}, %zmm6, %zmm10, %zmm26
+// CHECK:  encoding: [0x62,0x62,0x2d,0x38,0xac,0xd6]
+          vfnmadd213ps {rd-sae}, %zmm6, %zmm10, %zmm26
+
+// CHECK: vfnmadd213ps {rz-sae}, %zmm6, %zmm10, %zmm26
+// CHECK:  encoding: [0x62,0x62,0x2d,0x78,0xac,0xd6]
+          vfnmadd213ps {rz-sae}, %zmm6, %zmm10, %zmm26
+
+// CHECK: vfnmadd213ps (%rcx), %zmm10, %zmm26
+// CHECK:  encoding: [0x62,0x62,0x2d,0x48,0xac,0x11]
+          vfnmadd213ps (%rcx), %zmm10, %zmm26
+
+// CHECK: vfnmadd213ps 291(%rax,%r14,8), %zmm10, %zmm26
+// CHECK:  encoding: [0x62,0x22,0x2d,0x48,0xac,0x94,0xf0,0x23,0x01,0x00,0x00]
+          vfnmadd213ps 291(%rax,%r14,8), %zmm10, %zmm26
+
+// CHECK: vfnmadd213ps (%rcx){1to16}, %zmm10, %zmm26
+// CHECK:  encoding: [0x62,0x62,0x2d,0x58,0xac,0x11]
+          vfnmadd213ps (%rcx){1to16}, %zmm10, %zmm26
+
+// CHECK: vfnmadd213ps 8128(%rdx), %zmm10, %zmm26
+// CHECK:  encoding: [0x62,0x62,0x2d,0x48,0xac,0x52,0x7f]
+          vfnmadd213ps 8128(%rdx), %zmm10, %zmm26
+
+// CHECK: vfnmadd213ps 8192(%rdx), %zmm10, %zmm26
+// CHECK:  encoding: [0x62,0x62,0x2d,0x48,0xac,0x92,0x00,0x20,0x00,0x00]
+          vfnmadd213ps 8192(%rdx), %zmm10, %zmm26
+
+// CHECK: vfnmadd213ps -8192(%rdx), %zmm10, %zmm26
+// CHECK:  encoding: [0x62,0x62,0x2d,0x48,0xac,0x52,0x80]
+          vfnmadd213ps -8192(%rdx), %zmm10, %zmm26
+
+// CHECK: vfnmadd213ps -8256(%rdx), %zmm10, %zmm26
+// CHECK:  encoding: [0x62,0x62,0x2d,0x48,0xac,0x92,0xc0,0xdf,0xff,0xff]
+          vfnmadd213ps -8256(%rdx), %zmm10, %zmm26
+
+// CHECK: vfnmadd213ps 508(%rdx){1to16}, %zmm10, %zmm26
+// CHECK:  encoding: [0x62,0x62,0x2d,0x58,0xac,0x52,0x7f]
+          vfnmadd213ps 508(%rdx){1to16}, %zmm10, %zmm26
+
+// CHECK: vfnmadd213ps 512(%rdx){1to16}, %zmm10, %zmm26
+// CHECK:  encoding: [0x62,0x62,0x2d,0x58,0xac,0x92,0x00,0x02,0x00,0x00]
+          vfnmadd213ps 512(%rdx){1to16}, %zmm10, %zmm26
+
+// CHECK: vfnmadd213ps -512(%rdx){1to16}, %zmm10, %zmm26
+// CHECK:  encoding: [0x62,0x62,0x2d,0x58,0xac,0x52,0x80]
+          vfnmadd213ps -512(%rdx){1to16}, %zmm10, %zmm26
+
+// CHECK: vfnmadd213ps -516(%rdx){1to16}, %zmm10, %zmm26
+// CHECK:  encoding: [0x62,0x62,0x2d,0x58,0xac,0x92,0xfc,0xfd,0xff,0xff]
+          vfnmadd213ps -516(%rdx){1to16}, %zmm10, %zmm26
+
+// CHECK: vfnmadd213pd %zmm9, %zmm16, %zmm17
+// CHECK:  encoding: [0x62,0xc2,0xfd,0x40,0xac,0xc9]
+          vfnmadd213pd %zmm9, %zmm16, %zmm17
+
+// CHECK: vfnmadd213pd %zmm9, %zmm16, %zmm17 {%k4}
+// CHECK:  encoding: [0x62,0xc2,0xfd,0x44,0xac,0xc9]
+          vfnmadd213pd %zmm9, %zmm16, %zmm17 {%k4}
+
+// CHECK: vfnmadd213pd %zmm9, %zmm16, %zmm17 {%k4} {z}
+// CHECK:  encoding: [0x62,0xc2,0xfd,0xc4,0xac,0xc9]
+          vfnmadd213pd %zmm9, %zmm16, %zmm17 {%k4} {z}
+
+// CHECK: vfnmadd213pd {rn-sae}, %zmm9, %zmm16, %zmm17
+// CHECK:  encoding: [0x62,0xc2,0xfd,0x10,0xac,0xc9]
+          vfnmadd213pd {rn-sae}, %zmm9, %zmm16, %zmm17
+
+// CHECK: vfnmadd213pd {ru-sae}, %zmm9, %zmm16, %zmm17
+// CHECK:  encoding: [0x62,0xc2,0xfd,0x50,0xac,0xc9]
+          vfnmadd213pd {ru-sae}, %zmm9, %zmm16, %zmm17
+
+// CHECK: vfnmadd213pd {rd-sae}, %zmm9, %zmm16, %zmm17
+// CHECK:  encoding: [0x62,0xc2,0xfd,0x30,0xac,0xc9]
+          vfnmadd213pd {rd-sae}, %zmm9, %zmm16, %zmm17
+
+// CHECK: vfnmadd213pd {rz-sae}, %zmm9, %zmm16, %zmm17
+// CHECK:  encoding: [0x62,0xc2,0xfd,0x70,0xac,0xc9]
+          vfnmadd213pd {rz-sae}, %zmm9, %zmm16, %zmm17
+
+// CHECK: vfnmadd213pd (%rcx), %zmm16, %zmm17
+// CHECK:  encoding: [0x62,0xe2,0xfd,0x40,0xac,0x09]
+          vfnmadd213pd (%rcx), %zmm16, %zmm17
+
+// CHECK: vfnmadd213pd 291(%rax,%r14,8), %zmm16, %zmm17
+// CHECK:  encoding: [0x62,0xa2,0xfd,0x40,0xac,0x8c,0xf0,0x23,0x01,0x00,0x00]
+          vfnmadd213pd 291(%rax,%r14,8), %zmm16, %zmm17
+
+// CHECK: vfnmadd213pd (%rcx){1to8}, %zmm16, %zmm17
+// CHECK:  encoding: [0x62,0xe2,0xfd,0x50,0xac,0x09]
+          vfnmadd213pd (%rcx){1to8}, %zmm16, %zmm17
+
+// CHECK: vfnmadd213pd 8128(%rdx), %zmm16, %zmm17
+// CHECK:  encoding: [0x62,0xe2,0xfd,0x40,0xac,0x4a,0x7f]
+          vfnmadd213pd 8128(%rdx), %zmm16, %zmm17
+
+// CHECK: vfnmadd213pd 8192(%rdx), %zmm16, %zmm17
+// CHECK:  encoding: [0x62,0xe2,0xfd,0x40,0xac,0x8a,0x00,0x20,0x00,0x00]
+          vfnmadd213pd 8192(%rdx), %zmm16, %zmm17
+
+// CHECK: vfnmadd213pd -8192(%rdx), %zmm16, %zmm17
+// CHECK:  encoding: [0x62,0xe2,0xfd,0x40,0xac,0x4a,0x80]
+          vfnmadd213pd -8192(%rdx), %zmm16, %zmm17
+
+// CHECK: vfnmadd213pd -8256(%rdx), %zmm16, %zmm17
+// CHECK:  encoding: [0x62,0xe2,0xfd,0x40,0xac,0x8a,0xc0,0xdf,0xff,0xff]
+          vfnmadd213pd -8256(%rdx), %zmm16, %zmm17
+
+// CHECK: vfnmadd213pd 1016(%rdx){1to8}, %zmm16, %zmm17
+// CHECK:  encoding: [0x62,0xe2,0xfd,0x50,0xac,0x4a,0x7f]
+          vfnmadd213pd 1016(%rdx){1to8}, %zmm16, %zmm17
+
+// CHECK: vfnmadd213pd 1024(%rdx){1to8}, %zmm16, %zmm17
+// CHECK:  encoding: [0x62,0xe2,0xfd,0x50,0xac,0x8a,0x00,0x04,0x00,0x00]
+          vfnmadd213pd 1024(%rdx){1to8}, %zmm16, %zmm17
+
+// CHECK: vfnmadd213pd -1024(%rdx){1to8}, %zmm16, %zmm17
+// CHECK:  encoding: [0x62,0xe2,0xfd,0x50,0xac,0x4a,0x80]
+          vfnmadd213pd -1024(%rdx){1to8}, %zmm16, %zmm17
+
+// CHECK: vfnmadd213pd -1032(%rdx){1to8}, %zmm16, %zmm17
+// CHECK:  encoding: [0x62,0xe2,0xfd,0x50,0xac,0x8a,0xf8,0xfb,0xff,0xff]
+          vfnmadd213pd -1032(%rdx){1to8}, %zmm16, %zmm17
+
+// CHECK: vfnmadd231ps %zmm24, %zmm7, %zmm14
+// CHECK:  encoding: [0x62,0x12,0x45,0x48,0xbc,0xf0]
+          vfnmadd231ps %zmm24, %zmm7, %zmm14
+
+// CHECK: vfnmadd231ps %zmm24, %zmm7, %zmm14 {%k5}
+// CHECK:  encoding: [0x62,0x12,0x45,0x4d,0xbc,0xf0]
+          vfnmadd231ps %zmm24, %zmm7, %zmm14 {%k5}
+
+// CHECK: vfnmadd231ps %zmm24, %zmm7, %zmm14 {%k5} {z}
+// CHECK:  encoding: [0x62,0x12,0x45,0xcd,0xbc,0xf0]
+          vfnmadd231ps %zmm24, %zmm7, %zmm14 {%k5} {z}
+
+// CHECK: vfnmadd231ps {rn-sae}, %zmm24, %zmm7, %zmm14
+// CHECK:  encoding: [0x62,0x12,0x45,0x18,0xbc,0xf0]
+          vfnmadd231ps {rn-sae}, %zmm24, %zmm7, %zmm14
+
+// CHECK: vfnmadd231ps {ru-sae}, %zmm24, %zmm7, %zmm14
+// CHECK:  encoding: [0x62,0x12,0x45,0x58,0xbc,0xf0]
+          vfnmadd231ps {ru-sae}, %zmm24, %zmm7, %zmm14
+
+// CHECK: vfnmadd231ps {rd-sae}, %zmm24, %zmm7, %zmm14
+// CHECK:  encoding: [0x62,0x12,0x45,0x38,0xbc,0xf0]
+          vfnmadd231ps {rd-sae}, %zmm24, %zmm7, %zmm14
+
+// CHECK: vfnmadd231ps {rz-sae}, %zmm24, %zmm7, %zmm14
+// CHECK:  encoding: [0x62,0x12,0x45,0x78,0xbc,0xf0]
+          vfnmadd231ps {rz-sae}, %zmm24, %zmm7, %zmm14
+
+// CHECK: vfnmadd231ps (%rcx), %zmm7, %zmm14
+// CHECK:  encoding: [0x62,0x72,0x45,0x48,0xbc,0x31]
+          vfnmadd231ps (%rcx), %zmm7, %zmm14
+
+// CHECK: vfnmadd231ps 291(%rax,%r14,8), %zmm7, %zmm14
+// CHECK:  encoding: [0x62,0x32,0x45,0x48,0xbc,0xb4,0xf0,0x23,0x01,0x00,0x00]
+          vfnmadd231ps 291(%rax,%r14,8), %zmm7, %zmm14
+
+// CHECK: vfnmadd231ps (%rcx){1to16}, %zmm7, %zmm14
+// CHECK:  encoding: [0x62,0x72,0x45,0x58,0xbc,0x31]
+          vfnmadd231ps (%rcx){1to16}, %zmm7, %zmm14
+
+// CHECK: vfnmadd231ps 8128(%rdx), %zmm7, %zmm14
+// CHECK:  encoding: [0x62,0x72,0x45,0x48,0xbc,0x72,0x7f]
+          vfnmadd231ps 8128(%rdx), %zmm7, %zmm14
+
+// CHECK: vfnmadd231ps 8192(%rdx), %zmm7, %zmm14
+// CHECK:  encoding: [0x62,0x72,0x45,0x48,0xbc,0xb2,0x00,0x20,0x00,0x00]
+          vfnmadd231ps 8192(%rdx), %zmm7, %zmm14
+
+// CHECK: vfnmadd231ps -8192(%rdx), %zmm7, %zmm14
+// CHECK:  encoding: [0x62,0x72,0x45,0x48,0xbc,0x72,0x80]
+          vfnmadd231ps -8192(%rdx), %zmm7, %zmm14
+
+// CHECK: vfnmadd231ps -8256(%rdx), %zmm7, %zmm14
+// CHECK:  encoding: [0x62,0x72,0x45,0x48,0xbc,0xb2,0xc0,0xdf,0xff,0xff]
+          vfnmadd231ps -8256(%rdx), %zmm7, %zmm14
+
+// CHECK: vfnmadd231ps 508(%rdx){1to16}, %zmm7, %zmm14
+// CHECK:  encoding: [0x62,0x72,0x45,0x58,0xbc,0x72,0x7f]
+          vfnmadd231ps 508(%rdx){1to16}, %zmm7, %zmm14
+
+// CHECK: vfnmadd231ps 512(%rdx){1to16}, %zmm7, %zmm14
+// CHECK:  encoding: [0x62,0x72,0x45,0x58,0xbc,0xb2,0x00,0x02,0x00,0x00]
+          vfnmadd231ps 512(%rdx){1to16}, %zmm7, %zmm14
+
+// CHECK: vfnmadd231ps -512(%rdx){1to16}, %zmm7, %zmm14
+// CHECK:  encoding: [0x62,0x72,0x45,0x58,0xbc,0x72,0x80]
+          vfnmadd231ps -512(%rdx){1to16}, %zmm7, %zmm14
+
+// CHECK: vfnmadd231ps -516(%rdx){1to16}, %zmm7, %zmm14
+// CHECK:  encoding: [0x62,0x72,0x45,0x58,0xbc,0xb2,0xfc,0xfd,0xff,0xff]
+          vfnmadd231ps -516(%rdx){1to16}, %zmm7, %zmm14
+
+// CHECK: vfnmadd231pd %zmm16, %zmm11, %zmm12
+// CHECK:  encoding: [0x62,0x32,0xa5,0x48,0xbc,0xe0]
+          vfnmadd231pd %zmm16, %zmm11, %zmm12
+
+// CHECK: vfnmadd231pd %zmm16, %zmm11, %zmm12 {%k6}
+// CHECK:  encoding: [0x62,0x32,0xa5,0x4e,0xbc,0xe0]
+          vfnmadd231pd %zmm16, %zmm11, %zmm12 {%k6}
+
+// CHECK: vfnmadd231pd %zmm16, %zmm11, %zmm12 {%k6} {z}
+// CHECK:  encoding: [0x62,0x32,0xa5,0xce,0xbc,0xe0]
+          vfnmadd231pd %zmm16, %zmm11, %zmm12 {%k6} {z}
+
+// CHECK: vfnmadd231pd {rn-sae}, %zmm16, %zmm11, %zmm12
+// CHECK:  encoding: [0x62,0x32,0xa5,0x18,0xbc,0xe0]
+          vfnmadd231pd {rn-sae}, %zmm16, %zmm11, %zmm12
+
+// CHECK: vfnmadd231pd {ru-sae}, %zmm16, %zmm11, %zmm12
+// CHECK:  encoding: [0x62,0x32,0xa5,0x58,0xbc,0xe0]
+          vfnmadd231pd {ru-sae}, %zmm16, %zmm11, %zmm12
+
+// CHECK: vfnmadd231pd {rd-sae}, %zmm16, %zmm11, %zmm12
+// CHECK:  encoding: [0x62,0x32,0xa5,0x38,0xbc,0xe0]
+          vfnmadd231pd {rd-sae}, %zmm16, %zmm11, %zmm12
+
+// CHECK: vfnmadd231pd {rz-sae}, %zmm16, %zmm11, %zmm12
+// CHECK:  encoding: [0x62,0x32,0xa5,0x78,0xbc,0xe0]
+          vfnmadd231pd {rz-sae}, %zmm16, %zmm11, %zmm12
+
+// CHECK: vfnmadd231pd (%rcx), %zmm11, %zmm12
+// CHECK:  encoding: [0x62,0x72,0xa5,0x48,0xbc,0x21]
+          vfnmadd231pd (%rcx), %zmm11, %zmm12
+
+// CHECK: vfnmadd231pd 291(%rax,%r14,8), %zmm11, %zmm12
+// CHECK:  encoding: [0x62,0x32,0xa5,0x48,0xbc,0xa4,0xf0,0x23,0x01,0x00,0x00]
+          vfnmadd231pd 291(%rax,%r14,8), %zmm11, %zmm12
+
+// CHECK: vfnmadd231pd (%rcx){1to8}, %zmm11, %zmm12
+// CHECK:  encoding: [0x62,0x72,0xa5,0x58,0xbc,0x21]
+          vfnmadd231pd (%rcx){1to8}, %zmm11, %zmm12
+
+// CHECK: vfnmadd231pd 8128(%rdx), %zmm11, %zmm12
+// CHECK:  encoding: [0x62,0x72,0xa5,0x48,0xbc,0x62,0x7f]
+          vfnmadd231pd 8128(%rdx), %zmm11, %zmm12
+
+// CHECK: vfnmadd231pd 8192(%rdx), %zmm11, %zmm12
+// CHECK:  encoding: [0x62,0x72,0xa5,0x48,0xbc,0xa2,0x00,0x20,0x00,0x00]
+          vfnmadd231pd 8192(%rdx), %zmm11, %zmm12
+
+// CHECK: vfnmadd231pd -8192(%rdx), %zmm11, %zmm12
+// CHECK:  encoding: [0x62,0x72,0xa5,0x48,0xbc,0x62,0x80]
+          vfnmadd231pd -8192(%rdx), %zmm11, %zmm12
+
+// CHECK: vfnmadd231pd -8256(%rdx), %zmm11, %zmm12
+// CHECK:  encoding: [0x62,0x72,0xa5,0x48,0xbc,0xa2,0xc0,0xdf,0xff,0xff]
+          vfnmadd231pd -8256(%rdx), %zmm11, %zmm12
+
+// CHECK: vfnmadd231pd 1016(%rdx){1to8}, %zmm11, %zmm12
+// CHECK:  encoding: [0x62,0x72,0xa5,0x58,0xbc,0x62,0x7f]
+          vfnmadd231pd 1016(%rdx){1to8}, %zmm11, %zmm12
+
+// CHECK: vfnmadd231pd 1024(%rdx){1to8}, %zmm11, %zmm12
+// CHECK:  encoding: [0x62,0x72,0xa5,0x58,0xbc,0xa2,0x00,0x04,0x00,0x00]
+          vfnmadd231pd 1024(%rdx){1to8}, %zmm11, %zmm12
+
+// CHECK: vfnmadd231pd -1024(%rdx){1to8}, %zmm11, %zmm12
+// CHECK:  encoding: [0x62,0x72,0xa5,0x58,0xbc,0x62,0x80]
+          vfnmadd231pd -1024(%rdx){1to8}, %zmm11, %zmm12
+
+// CHECK: vfnmadd231pd -1032(%rdx){1to8}, %zmm11, %zmm12
+// CHECK:  encoding: [0x62,0x72,0xa5,0x58,0xbc,0xa2,0xf8,0xfb,0xff,0xff]
+          vfnmadd231pd -1032(%rdx){1to8}, %zmm11, %zmm12
+
+// CHECK: vfnmsub132ps %zmm6, %zmm18, %zmm4
+// CHECK:  encoding: [0x62,0xf2,0x6d,0x40,0x9e,0xe6]
+          vfnmsub132ps %zmm6, %zmm18, %zmm4
+
+// CHECK: vfnmsub132ps %zmm6, %zmm18, %zmm4 {%k2}
+// CHECK:  encoding: [0x62,0xf2,0x6d,0x42,0x9e,0xe6]
+          vfnmsub132ps %zmm6, %zmm18, %zmm4 {%k2}
+
+// CHECK: vfnmsub132ps %zmm6, %zmm18, %zmm4 {%k2} {z}
+// CHECK:  encoding: [0x62,0xf2,0x6d,0xc2,0x9e,0xe6]
+          vfnmsub132ps %zmm6, %zmm18, %zmm4 {%k2} {z}
+
+// CHECK: vfnmsub132ps {rn-sae}, %zmm6, %zmm18, %zmm4
+// CHECK:  encoding: [0x62,0xf2,0x6d,0x10,0x9e,0xe6]
+          vfnmsub132ps {rn-sae}, %zmm6, %zmm18, %zmm4
+
+// CHECK: vfnmsub132ps {ru-sae}, %zmm6, %zmm18, %zmm4
+// CHECK:  encoding: [0x62,0xf2,0x6d,0x50,0x9e,0xe6]
+          vfnmsub132ps {ru-sae}, %zmm6, %zmm18, %zmm4
+
+// CHECK: vfnmsub132ps {rd-sae}, %zmm6, %zmm18, %zmm4
+// CHECK:  encoding: [0x62,0xf2,0x6d,0x30,0x9e,0xe6]
+          vfnmsub132ps {rd-sae}, %zmm6, %zmm18, %zmm4
+
+// CHECK: vfnmsub132ps {rz-sae}, %zmm6, %zmm18, %zmm4
+// CHECK:  encoding: [0x62,0xf2,0x6d,0x70,0x9e,0xe6]
+          vfnmsub132ps {rz-sae}, %zmm6, %zmm18, %zmm4
+
+// CHECK: vfnmsub132ps (%rcx), %zmm18, %zmm4
+// CHECK:  encoding: [0x62,0xf2,0x6d,0x40,0x9e,0x21]
+          vfnmsub132ps (%rcx), %zmm18, %zmm4
+
+// CHECK: vfnmsub132ps 291(%rax,%r14,8), %zmm18, %zmm4
+// CHECK:  encoding: [0x62,0xb2,0x6d,0x40,0x9e,0xa4,0xf0,0x23,0x01,0x00,0x00]
+          vfnmsub132ps 291(%rax,%r14,8), %zmm18, %zmm4
+
+// CHECK: vfnmsub132ps (%rcx){1to16}, %zmm18, %zmm4
+// CHECK:  encoding: [0x62,0xf2,0x6d,0x50,0x9e,0x21]
+          vfnmsub132ps (%rcx){1to16}, %zmm18, %zmm4
+
+// CHECK: vfnmsub132ps 8128(%rdx), %zmm18, %zmm4
+// CHECK:  encoding: [0x62,0xf2,0x6d,0x40,0x9e,0x62,0x7f]
+          vfnmsub132ps 8128(%rdx), %zmm18, %zmm4
+
+// CHECK: vfnmsub132ps 8192(%rdx), %zmm18, %zmm4
+// CHECK:  encoding: [0x62,0xf2,0x6d,0x40,0x9e,0xa2,0x00,0x20,0x00,0x00]
+          vfnmsub132ps 8192(%rdx), %zmm18, %zmm4
+
+// CHECK: vfnmsub132ps -8192(%rdx), %zmm18, %zmm4
+// CHECK:  encoding: [0x62,0xf2,0x6d,0x40,0x9e,0x62,0x80]
+          vfnmsub132ps -8192(%rdx), %zmm18, %zmm4
+
+// CHECK: vfnmsub132ps -8256(%rdx), %zmm18, %zmm4
+// CHECK:  encoding: [0x62,0xf2,0x6d,0x40,0x9e,0xa2,0xc0,0xdf,0xff,0xff]
+          vfnmsub132ps -8256(%rdx), %zmm18, %zmm4
+
+// CHECK: vfnmsub132ps 508(%rdx){1to16}, %zmm18, %zmm4
+// CHECK:  encoding: [0x62,0xf2,0x6d,0x50,0x9e,0x62,0x7f]
+          vfnmsub132ps 508(%rdx){1to16}, %zmm18, %zmm4
+
+// CHECK: vfnmsub132ps 512(%rdx){1to16}, %zmm18, %zmm4
+// CHECK:  encoding: [0x62,0xf2,0x6d,0x50,0x9e,0xa2,0x00,0x02,0x00,0x00]
+          vfnmsub132ps 512(%rdx){1to16}, %zmm18, %zmm4
+
+// CHECK: vfnmsub132ps -512(%rdx){1to16}, %zmm18, %zmm4
+// CHECK:  encoding: [0x62,0xf2,0x6d,0x50,0x9e,0x62,0x80]
+          vfnmsub132ps -512(%rdx){1to16}, %zmm18, %zmm4
+
+// CHECK: vfnmsub132ps -516(%rdx){1to16}, %zmm18, %zmm4
+// CHECK:  encoding: [0x62,0xf2,0x6d,0x50,0x9e,0xa2,0xfc,0xfd,0xff,0xff]
+          vfnmsub132ps -516(%rdx){1to16}, %zmm18, %zmm4
+
+// CHECK: vfnmsub132pd %zmm6, %zmm5, %zmm28
+// CHECK:  encoding: [0x62,0x62,0xd5,0x48,0x9e,0xe6]
+          vfnmsub132pd %zmm6, %zmm5, %zmm28
+
+// CHECK: vfnmsub132pd %zmm6, %zmm5, %zmm28 {%k2}
+// CHECK:  encoding: [0x62,0x62,0xd5,0x4a,0x9e,0xe6]
+          vfnmsub132pd %zmm6, %zmm5, %zmm28 {%k2}
+
+// CHECK: vfnmsub132pd %zmm6, %zmm5, %zmm28 {%k2} {z}
+// CHECK:  encoding: [0x62,0x62,0xd5,0xca,0x9e,0xe6]
+          vfnmsub132pd %zmm6, %zmm5, %zmm28 {%k2} {z}
+
+// CHECK: vfnmsub132pd {rn-sae}, %zmm6, %zmm5, %zmm28
+// CHECK:  encoding: [0x62,0x62,0xd5,0x18,0x9e,0xe6]
+          vfnmsub132pd {rn-sae}, %zmm6, %zmm5, %zmm28
+
+// CHECK: vfnmsub132pd {ru-sae}, %zmm6, %zmm5, %zmm28
+// CHECK:  encoding: [0x62,0x62,0xd5,0x58,0x9e,0xe6]
+          vfnmsub132pd {ru-sae}, %zmm6, %zmm5, %zmm28
+
+// CHECK: vfnmsub132pd {rd-sae}, %zmm6, %zmm5, %zmm28
+// CHECK:  encoding: [0x62,0x62,0xd5,0x38,0x9e,0xe6]
+          vfnmsub132pd {rd-sae}, %zmm6, %zmm5, %zmm28
+
+// CHECK: vfnmsub132pd {rz-sae}, %zmm6, %zmm5, %zmm28
+// CHECK:  encoding: [0x62,0x62,0xd5,0x78,0x9e,0xe6]
+          vfnmsub132pd {rz-sae}, %zmm6, %zmm5, %zmm28
+
+// CHECK: vfnmsub132pd (%rcx), %zmm5, %zmm28
+// CHECK:  encoding: [0x62,0x62,0xd5,0x48,0x9e,0x21]
+          vfnmsub132pd (%rcx), %zmm5, %zmm28
+
+// CHECK: vfnmsub132pd 291(%rax,%r14,8), %zmm5, %zmm28
+// CHECK:  encoding: [0x62,0x22,0xd5,0x48,0x9e,0xa4,0xf0,0x23,0x01,0x00,0x00]
+          vfnmsub132pd 291(%rax,%r14,8), %zmm5, %zmm28
+
+// CHECK: vfnmsub132pd (%rcx){1to8}, %zmm5, %zmm28
+// CHECK:  encoding: [0x62,0x62,0xd5,0x58,0x9e,0x21]
+          vfnmsub132pd (%rcx){1to8}, %zmm5, %zmm28
+
+// CHECK: vfnmsub132pd 8128(%rdx), %zmm5, %zmm28
+// CHECK:  encoding: [0x62,0x62,0xd5,0x48,0x9e,0x62,0x7f]
+          vfnmsub132pd 8128(%rdx), %zmm5, %zmm28
+
+// CHECK: vfnmsub132pd 8192(%rdx), %zmm5, %zmm28
+// CHECK:  encoding: [0x62,0x62,0xd5,0x48,0x9e,0xa2,0x00,0x20,0x00,0x00]
+          vfnmsub132pd 8192(%rdx), %zmm5, %zmm28
+
+// CHECK: vfnmsub132pd -8192(%rdx), %zmm5, %zmm28
+// CHECK:  encoding: [0x62,0x62,0xd5,0x48,0x9e,0x62,0x80]
+          vfnmsub132pd -8192(%rdx), %zmm5, %zmm28
+
+// CHECK: vfnmsub132pd -8256(%rdx), %zmm5, %zmm28
+// CHECK:  encoding: [0x62,0x62,0xd5,0x48,0x9e,0xa2,0xc0,0xdf,0xff,0xff]
+          vfnmsub132pd -8256(%rdx), %zmm5, %zmm28
+
+// CHECK: vfnmsub132pd 1016(%rdx){1to8}, %zmm5, %zmm28
+// CHECK:  encoding: [0x62,0x62,0xd5,0x58,0x9e,0x62,0x7f]
+          vfnmsub132pd 1016(%rdx){1to8}, %zmm5, %zmm28
+
+// CHECK: vfnmsub132pd 1024(%rdx){1to8}, %zmm5, %zmm28
+// CHECK:  encoding: [0x62,0x62,0xd5,0x58,0x9e,0xa2,0x00,0x04,0x00,0x00]
+          vfnmsub132pd 1024(%rdx){1to8}, %zmm5, %zmm28
+
+// CHECK: vfnmsub132pd -1024(%rdx){1to8}, %zmm5, %zmm28
+// CHECK:  encoding: [0x62,0x62,0xd5,0x58,0x9e,0x62,0x80]
+          vfnmsub132pd -1024(%rdx){1to8}, %zmm5, %zmm28
+
+// CHECK: vfnmsub132pd -1032(%rdx){1to8}, %zmm5, %zmm28
+// CHECK:  encoding: [0x62,0x62,0xd5,0x58,0x9e,0xa2,0xf8,0xfb,0xff,0xff]
+          vfnmsub132pd -1032(%rdx){1to8}, %zmm5, %zmm28
+
+// CHECK: vfnmsub213ps %zmm2, %zmm13, %zmm21
+// CHECK:  encoding: [0x62,0xe2,0x15,0x48,0xae,0xea]
+          vfnmsub213ps %zmm2, %zmm13, %zmm21
+
+// CHECK: vfnmsub213ps %zmm2, %zmm13, %zmm21 {%k3}
+// CHECK:  encoding: [0x62,0xe2,0x15,0x4b,0xae,0xea]
+          vfnmsub213ps %zmm2, %zmm13, %zmm21 {%k3}
+
+// CHECK: vfnmsub213ps %zmm2, %zmm13, %zmm21 {%k3} {z}
+// CHECK:  encoding: [0x62,0xe2,0x15,0xcb,0xae,0xea]
+          vfnmsub213ps %zmm2, %zmm13, %zmm21 {%k3} {z}
+
+// CHECK: vfnmsub213ps {rn-sae}, %zmm2, %zmm13, %zmm21
+// CHECK:  encoding: [0x62,0xe2,0x15,0x18,0xae,0xea]
+          vfnmsub213ps {rn-sae}, %zmm2, %zmm13, %zmm21
+
+// CHECK: vfnmsub213ps {ru-sae}, %zmm2, %zmm13, %zmm21
+// CHECK:  encoding: [0x62,0xe2,0x15,0x58,0xae,0xea]
+          vfnmsub213ps {ru-sae}, %zmm2, %zmm13, %zmm21
+
+// CHECK: vfnmsub213ps {rd-sae}, %zmm2, %zmm13, %zmm21
+// CHECK:  encoding: [0x62,0xe2,0x15,0x38,0xae,0xea]
+          vfnmsub213ps {rd-sae}, %zmm2, %zmm13, %zmm21
+
+// CHECK: vfnmsub213ps {rz-sae}, %zmm2, %zmm13, %zmm21
+// CHECK:  encoding: [0x62,0xe2,0x15,0x78,0xae,0xea]
+          vfnmsub213ps {rz-sae}, %zmm2, %zmm13, %zmm21
+
+// CHECK: vfnmsub213ps (%rcx), %zmm13, %zmm21
+// CHECK:  encoding: [0x62,0xe2,0x15,0x48,0xae,0x29]
+          vfnmsub213ps (%rcx), %zmm13, %zmm21
+
+// CHECK: vfnmsub213ps 291(%rax,%r14,8), %zmm13, %zmm21
+// CHECK:  encoding: [0x62,0xa2,0x15,0x48,0xae,0xac,0xf0,0x23,0x01,0x00,0x00]
+          vfnmsub213ps 291(%rax,%r14,8), %zmm13, %zmm21
+
+// CHECK: vfnmsub213ps (%rcx){1to16}, %zmm13, %zmm21
+// CHECK:  encoding: [0x62,0xe2,0x15,0x58,0xae,0x29]
+          vfnmsub213ps (%rcx){1to16}, %zmm13, %zmm21
+
+// CHECK: vfnmsub213ps 8128(%rdx), %zmm13, %zmm21
+// CHECK:  encoding: [0x62,0xe2,0x15,0x48,0xae,0x6a,0x7f]
+          vfnmsub213ps 8128(%rdx), %zmm13, %zmm21
+
+// CHECK: vfnmsub213ps 8192(%rdx), %zmm13, %zmm21
+// CHECK:  encoding: [0x62,0xe2,0x15,0x48,0xae,0xaa,0x00,0x20,0x00,0x00]
+          vfnmsub213ps 8192(%rdx), %zmm13, %zmm21
+
+// CHECK: vfnmsub213ps -8192(%rdx), %zmm13, %zmm21
+// CHECK:  encoding: [0x62,0xe2,0x15,0x48,0xae,0x6a,0x80]
+          vfnmsub213ps -8192(%rdx), %zmm13, %zmm21
+
+// CHECK: vfnmsub213ps -8256(%rdx), %zmm13, %zmm21
+// CHECK:  encoding: [0x62,0xe2,0x15,0x48,0xae,0xaa,0xc0,0xdf,0xff,0xff]
+          vfnmsub213ps -8256(%rdx), %zmm13, %zmm21
+
+// CHECK: vfnmsub213ps 508(%rdx){1to16}, %zmm13, %zmm21
+// CHECK:  encoding: [0x62,0xe2,0x15,0x58,0xae,0x6a,0x7f]
+          vfnmsub213ps 508(%rdx){1to16}, %zmm13, %zmm21
+
+// CHECK: vfnmsub213ps 512(%rdx){1to16}, %zmm13, %zmm21
+// CHECK:  encoding: [0x62,0xe2,0x15,0x58,0xae,0xaa,0x00,0x02,0x00,0x00]
+          vfnmsub213ps 512(%rdx){1to16}, %zmm13, %zmm21
+
+// CHECK: vfnmsub213ps -512(%rdx){1to16}, %zmm13, %zmm21
+// CHECK:  encoding: [0x62,0xe2,0x15,0x58,0xae,0x6a,0x80]
+          vfnmsub213ps -512(%rdx){1to16}, %zmm13, %zmm21
+
+// CHECK: vfnmsub213ps -516(%rdx){1to16}, %zmm13, %zmm21
+// CHECK:  encoding: [0x62,0xe2,0x15,0x58,0xae,0xaa,0xfc,0xfd,0xff,0xff]
+          vfnmsub213ps -516(%rdx){1to16}, %zmm13, %zmm21
+
+// CHECK: vfnmsub213pd %zmm11, %zmm18, %zmm23
+// CHECK:  encoding: [0x62,0xc2,0xed,0x40,0xae,0xfb]
+          vfnmsub213pd %zmm11, %zmm18, %zmm23
+
+// CHECK: vfnmsub213pd %zmm11, %zmm18, %zmm23 {%k2}
+// CHECK:  encoding: [0x62,0xc2,0xed,0x42,0xae,0xfb]
+          vfnmsub213pd %zmm11, %zmm18, %zmm23 {%k2}
+
+// CHECK: vfnmsub213pd %zmm11, %zmm18, %zmm23 {%k2} {z}
+// CHECK:  encoding: [0x62,0xc2,0xed,0xc2,0xae,0xfb]
+          vfnmsub213pd %zmm11, %zmm18, %zmm23 {%k2} {z}
+
+// CHECK: vfnmsub213pd {rn-sae}, %zmm11, %zmm18, %zmm23
+// CHECK:  encoding: [0x62,0xc2,0xed,0x10,0xae,0xfb]
+          vfnmsub213pd {rn-sae}, %zmm11, %zmm18, %zmm23
+
+// CHECK: vfnmsub213pd {ru-sae}, %zmm11, %zmm18, %zmm23
+// CHECK:  encoding: [0x62,0xc2,0xed,0x50,0xae,0xfb]
+          vfnmsub213pd {ru-sae}, %zmm11, %zmm18, %zmm23
+
+// CHECK: vfnmsub213pd {rd-sae}, %zmm11, %zmm18, %zmm23
+// CHECK:  encoding: [0x62,0xc2,0xed,0x30,0xae,0xfb]
+          vfnmsub213pd {rd-sae}, %zmm11, %zmm18, %zmm23
+
+// CHECK: vfnmsub213pd {rz-sae}, %zmm11, %zmm18, %zmm23
+// CHECK:  encoding: [0x62,0xc2,0xed,0x70,0xae,0xfb]
+          vfnmsub213pd {rz-sae}, %zmm11, %zmm18, %zmm23
+
+// CHECK: vfnmsub213pd (%rcx), %zmm18, %zmm23
+// CHECK:  encoding: [0x62,0xe2,0xed,0x40,0xae,0x39]
+          vfnmsub213pd (%rcx), %zmm18, %zmm23
+
+// CHECK: vfnmsub213pd 291(%rax,%r14,8), %zmm18, %zmm23
+// CHECK:  encoding: [0x62,0xa2,0xed,0x40,0xae,0xbc,0xf0,0x23,0x01,0x00,0x00]
+          vfnmsub213pd 291(%rax,%r14,8), %zmm18, %zmm23
+
+// CHECK: vfnmsub213pd (%rcx){1to8}, %zmm18, %zmm23
+// CHECK:  encoding: [0x62,0xe2,0xed,0x50,0xae,0x39]
+          vfnmsub213pd (%rcx){1to8}, %zmm18, %zmm23
+
+// CHECK: vfnmsub213pd 8128(%rdx), %zmm18, %zmm23
+// CHECK:  encoding: [0x62,0xe2,0xed,0x40,0xae,0x7a,0x7f]
+          vfnmsub213pd 8128(%rdx), %zmm18, %zmm23
+
+// CHECK: vfnmsub213pd 8192(%rdx), %zmm18, %zmm23
+// CHECK:  encoding: [0x62,0xe2,0xed,0x40,0xae,0xba,0x00,0x20,0x00,0x00]
+          vfnmsub213pd 8192(%rdx), %zmm18, %zmm23
+
+// CHECK: vfnmsub213pd -8192(%rdx), %zmm18, %zmm23
+// CHECK:  encoding: [0x62,0xe2,0xed,0x40,0xae,0x7a,0x80]
+          vfnmsub213pd -8192(%rdx), %zmm18, %zmm23
+
+// CHECK: vfnmsub213pd -8256(%rdx), %zmm18, %zmm23
+// CHECK:  encoding: [0x62,0xe2,0xed,0x40,0xae,0xba,0xc0,0xdf,0xff,0xff]
+          vfnmsub213pd -8256(%rdx), %zmm18, %zmm23
+
+// CHECK: vfnmsub213pd 1016(%rdx){1to8}, %zmm18, %zmm23
+// CHECK:  encoding: [0x62,0xe2,0xed,0x50,0xae,0x7a,0x7f]
+          vfnmsub213pd 1016(%rdx){1to8}, %zmm18, %zmm23
+
+// CHECK: vfnmsub213pd 1024(%rdx){1to8}, %zmm18, %zmm23
+// CHECK:  encoding: [0x62,0xe2,0xed,0x50,0xae,0xba,0x00,0x04,0x00,0x00]
+          vfnmsub213pd 1024(%rdx){1to8}, %zmm18, %zmm23
+
+// CHECK: vfnmsub213pd -1024(%rdx){1to8}, %zmm18, %zmm23
+// CHECK:  encoding: [0x62,0xe2,0xed,0x50,0xae,0x7a,0x80]
+          vfnmsub213pd -1024(%rdx){1to8}, %zmm18, %zmm23
+
+// CHECK: vfnmsub213pd -1032(%rdx){1to8}, %zmm18, %zmm23
+// CHECK:  encoding: [0x62,0xe2,0xed,0x50,0xae,0xba,0xf8,0xfb,0xff,0xff]
+          vfnmsub213pd -1032(%rdx){1to8}, %zmm18, %zmm23
+
+// CHECK: vfnmsub231ps %zmm13, %zmm6, %zmm8
+// CHECK:  encoding: [0x62,0x52,0x4d,0x48,0xbe,0xc5]
+          vfnmsub231ps %zmm13, %zmm6, %zmm8
+
+// CHECK: vfnmsub231ps %zmm13, %zmm6, %zmm8 {%k2}
+// CHECK:  encoding: [0x62,0x52,0x4d,0x4a,0xbe,0xc5]
+          vfnmsub231ps %zmm13, %zmm6, %zmm8 {%k2}
+
+// CHECK: vfnmsub231ps %zmm13, %zmm6, %zmm8 {%k2} {z}
+// CHECK:  encoding: [0x62,0x52,0x4d,0xca,0xbe,0xc5]
+          vfnmsub231ps %zmm13, %zmm6, %zmm8 {%k2} {z}
+
+// CHECK: vfnmsub231ps {rn-sae}, %zmm13, %zmm6, %zmm8
+// CHECK:  encoding: [0x62,0x52,0x4d,0x18,0xbe,0xc5]
+          vfnmsub231ps {rn-sae}, %zmm13, %zmm6, %zmm8
+
+// CHECK: vfnmsub231ps {ru-sae}, %zmm13, %zmm6, %zmm8
+// CHECK:  encoding: [0x62,0x52,0x4d,0x58,0xbe,0xc5]
+          vfnmsub231ps {ru-sae}, %zmm13, %zmm6, %zmm8
+
+// CHECK: vfnmsub231ps {rd-sae}, %zmm13, %zmm6, %zmm8
+// CHECK:  encoding: [0x62,0x52,0x4d,0x38,0xbe,0xc5]
+          vfnmsub231ps {rd-sae}, %zmm13, %zmm6, %zmm8
+
+// CHECK: vfnmsub231ps {rz-sae}, %zmm13, %zmm6, %zmm8
+// CHECK:  encoding: [0x62,0x52,0x4d,0x78,0xbe,0xc5]
+          vfnmsub231ps {rz-sae}, %zmm13, %zmm6, %zmm8
+
+// CHECK: vfnmsub231ps (%rcx), %zmm6, %zmm8
+// CHECK:  encoding: [0x62,0x72,0x4d,0x48,0xbe,0x01]
+          vfnmsub231ps (%rcx), %zmm6, %zmm8
+
+// CHECK: vfnmsub231ps 291(%rax,%r14,8), %zmm6, %zmm8
+// CHECK:  encoding: [0x62,0x32,0x4d,0x48,0xbe,0x84,0xf0,0x23,0x01,0x00,0x00]
+          vfnmsub231ps 291(%rax,%r14,8), %zmm6, %zmm8
+
+// CHECK: vfnmsub231ps (%rcx){1to16}, %zmm6, %zmm8
+// CHECK:  encoding: [0x62,0x72,0x4d,0x58,0xbe,0x01]
+          vfnmsub231ps (%rcx){1to16}, %zmm6, %zmm8
+
+// CHECK: vfnmsub231ps 8128(%rdx), %zmm6, %zmm8
+// CHECK:  encoding: [0x62,0x72,0x4d,0x48,0xbe,0x42,0x7f]
+          vfnmsub231ps 8128(%rdx), %zmm6, %zmm8
+
+// CHECK: vfnmsub231ps 8192(%rdx), %zmm6, %zmm8
+// CHECK:  encoding: [0x62,0x72,0x4d,0x48,0xbe,0x82,0x00,0x20,0x00,0x00]
+          vfnmsub231ps 8192(%rdx), %zmm6, %zmm8
+
+// CHECK: vfnmsub231ps -8192(%rdx), %zmm6, %zmm8
+// CHECK:  encoding: [0x62,0x72,0x4d,0x48,0xbe,0x42,0x80]
+          vfnmsub231ps -8192(%rdx), %zmm6, %zmm8
+
+// CHECK: vfnmsub231ps -8256(%rdx), %zmm6, %zmm8
+// CHECK:  encoding: [0x62,0x72,0x4d,0x48,0xbe,0x82,0xc0,0xdf,0xff,0xff]
+          vfnmsub231ps -8256(%rdx), %zmm6, %zmm8
+
+// CHECK: vfnmsub231ps 508(%rdx){1to16}, %zmm6, %zmm8
+// CHECK:  encoding: [0x62,0x72,0x4d,0x58,0xbe,0x42,0x7f]
+          vfnmsub231ps 508(%rdx){1to16}, %zmm6, %zmm8
+
+// CHECK: vfnmsub231ps 512(%rdx){1to16}, %zmm6, %zmm8
+// CHECK:  encoding: [0x62,0x72,0x4d,0x58,0xbe,0x82,0x00,0x02,0x00,0x00]
+          vfnmsub231ps 512(%rdx){1to16}, %zmm6, %zmm8
+
+// CHECK: vfnmsub231ps -512(%rdx){1to16}, %zmm6, %zmm8
+// CHECK:  encoding: [0x62,0x72,0x4d,0x58,0xbe,0x42,0x80]
+          vfnmsub231ps -512(%rdx){1to16}, %zmm6, %zmm8
+
+// CHECK: vfnmsub231ps -516(%rdx){1to16}, %zmm6, %zmm8
+// CHECK:  encoding: [0x62,0x72,0x4d,0x58,0xbe,0x82,0xfc,0xfd,0xff,0xff]
+          vfnmsub231ps -516(%rdx){1to16}, %zmm6, %zmm8
+
+// CHECK: vfnmsub231pd %zmm24, %zmm4, %zmm29
+// CHECK:  encoding: [0x62,0x02,0xdd,0x48,0xbe,0xe8]
+          vfnmsub231pd %zmm24, %zmm4, %zmm29
+
+// CHECK: vfnmsub231pd %zmm24, %zmm4, %zmm29 {%k7}
+// CHECK:  encoding: [0x62,0x02,0xdd,0x4f,0xbe,0xe8]
+          vfnmsub231pd %zmm24, %zmm4, %zmm29 {%k7}
+
+// CHECK: vfnmsub231pd %zmm24, %zmm4, %zmm29 {%k7} {z}
+// CHECK:  encoding: [0x62,0x02,0xdd,0xcf,0xbe,0xe8]
+          vfnmsub231pd %zmm24, %zmm4, %zmm29 {%k7} {z}
+
+// CHECK: vfnmsub231pd {rn-sae}, %zmm24, %zmm4, %zmm29
+// CHECK:  encoding: [0x62,0x02,0xdd,0x18,0xbe,0xe8]
+          vfnmsub231pd {rn-sae}, %zmm24, %zmm4, %zmm29
+
+// CHECK: vfnmsub231pd {ru-sae}, %zmm24, %zmm4, %zmm29
+// CHECK:  encoding: [0x62,0x02,0xdd,0x58,0xbe,0xe8]
+          vfnmsub231pd {ru-sae}, %zmm24, %zmm4, %zmm29
+
+// CHECK: vfnmsub231pd {rd-sae}, %zmm24, %zmm4, %zmm29
+// CHECK:  encoding: [0x62,0x02,0xdd,0x38,0xbe,0xe8]
+          vfnmsub231pd {rd-sae}, %zmm24, %zmm4, %zmm29
+
+// CHECK: vfnmsub231pd {rz-sae}, %zmm24, %zmm4, %zmm29
+// CHECK:  encoding: [0x62,0x02,0xdd,0x78,0xbe,0xe8]
+          vfnmsub231pd {rz-sae}, %zmm24, %zmm4, %zmm29
+
+// CHECK: vfnmsub231pd (%rcx), %zmm4, %zmm29
+// CHECK:  encoding: [0x62,0x62,0xdd,0x48,0xbe,0x29]
+          vfnmsub231pd (%rcx), %zmm4, %zmm29
+
+// CHECK: vfnmsub231pd 291(%rax,%r14,8), %zmm4, %zmm29
+// CHECK:  encoding: [0x62,0x22,0xdd,0x48,0xbe,0xac,0xf0,0x23,0x01,0x00,0x00]
+          vfnmsub231pd 291(%rax,%r14,8), %zmm4, %zmm29
+
+// CHECK: vfnmsub231pd (%rcx){1to8}, %zmm4, %zmm29
+// CHECK:  encoding: [0x62,0x62,0xdd,0x58,0xbe,0x29]
+          vfnmsub231pd (%rcx){1to8}, %zmm4, %zmm29
+
+// CHECK: vfnmsub231pd 8128(%rdx), %zmm4, %zmm29
+// CHECK:  encoding: [0x62,0x62,0xdd,0x48,0xbe,0x6a,0x7f]
+          vfnmsub231pd 8128(%rdx), %zmm4, %zmm29
+
+// CHECK: vfnmsub231pd 8192(%rdx), %zmm4, %zmm29
+// CHECK:  encoding: [0x62,0x62,0xdd,0x48,0xbe,0xaa,0x00,0x20,0x00,0x00]
+          vfnmsub231pd 8192(%rdx), %zmm4, %zmm29
+
+// CHECK: vfnmsub231pd -8192(%rdx), %zmm4, %zmm29
+// CHECK:  encoding: [0x62,0x62,0xdd,0x48,0xbe,0x6a,0x80]
+          vfnmsub231pd -8192(%rdx), %zmm4, %zmm29
+
+// CHECK: vfnmsub231pd -8256(%rdx), %zmm4, %zmm29
+// CHECK:  encoding: [0x62,0x62,0xdd,0x48,0xbe,0xaa,0xc0,0xdf,0xff,0xff]
+          vfnmsub231pd -8256(%rdx), %zmm4, %zmm29
+
+// CHECK: vfnmsub231pd 1016(%rdx){1to8}, %zmm4, %zmm29
+// CHECK:  encoding: [0x62,0x62,0xdd,0x58,0xbe,0x6a,0x7f]
+          vfnmsub231pd 1016(%rdx){1to8}, %zmm4, %zmm29
+
+// CHECK: vfnmsub231pd 1024(%rdx){1to8}, %zmm4, %zmm29
+// CHECK:  encoding: [0x62,0x62,0xdd,0x58,0xbe,0xaa,0x00,0x04,0x00,0x00]
+          vfnmsub231pd 1024(%rdx){1to8}, %zmm4, %zmm29
+
+// CHECK: vfnmsub231pd -1024(%rdx){1to8}, %zmm4, %zmm29
+// CHECK:  encoding: [0x62,0x62,0xdd,0x58,0xbe,0x6a,0x80]
+          vfnmsub231pd -1024(%rdx){1to8}, %zmm4, %zmm29
+
+// CHECK: vfnmsub231pd -1032(%rdx){1to8}, %zmm4, %zmm29
+// CHECK:  encoding: [0x62,0x62,0xdd,0x58,0xbe,0xaa,0xf8,0xfb,0xff,0xff]
+          vfnmsub231pd -1032(%rdx){1to8}, %zmm4, %zmm29
+
+
+// CHECK: vfmadd231ps 8192(%rdx), %zmm28, %zmm27 {%k3} {z}
+// CHECK: encoding: [0x62,0x62,0x1d,0xc3,0xb8,0x9a,0x00,0x20,0x00,0x00]
+          vfmadd231ps 8192(%rdx), %zmm28, %zmm27 {%k3} {z}
+
+// CHECK: vfmadd231ps {rn-sae}, %zmm25, %zmm28, %zmm27 {%k3} {z}
+// CHECK: encoding: [0x62,0x02,0x1d,0x93,0xb8,0xd9]
+          vfmadd231ps {rn-sae}, %zmm25, %zmm28, %zmm27 {%k3} {z}
+
+// CHECK: vfmadd231ps -1032(%rdx){1to16}, %zmm20, %zmm24 {%k3} {z}
+// CHECK: encoding: [0x62,0x62,0x5d,0xd3,0xb8,0x82,0xf8,0xfb,0xff,0xff]
+          vfmadd231ps -1032(%rdx){1to16}, %zmm20, %zmm24 {%k3} {z}
+
+// CHECK: vfmadd231pd 8192(%rdx), %zmm28, %zmm27 {%k3} {z}
+// CHECK: encoding: [0x62,0x62,0x9d,0xc3,0xb8,0x9a,0x00,0x20,0x00,0x00]
+          vfmadd231pd 8192(%rdx), %zmm28, %zmm27 {%k3} {z}
+
+// CHECK: vfmadd231pd {rn-sae}, %zmm25, %zmm28, %zmm27 {%k3} {z}
+// CHECK: encoding: [0x62,0x02,0x9d,0x93,0xb8,0xd9]
+          vfmadd231pd {rn-sae}, %zmm25, %zmm28, %zmm27 {%k3} {z}
+
+// CHECK: vfmadd231pd -1032(%rdx){1to8}, %zmm20, %zmm24 {%k3} {z}
+// CHECK: encoding: [0x62,0x62,0xdd,0xd3,0xb8,0x82,0xf8,0xfb,0xff,0xff]
+          vfmadd231pd -1032(%rdx){1to8}, %zmm20, %zmm24 {%k3} {z}
+
+// CHECK: vfmadd213ps 8192(%rdx), %zmm28, %zmm27 {%k3} {z}
+// CHECK: encoding: [0x62,0x62,0x1d,0xc3,0xa8,0x9a,0x00,0x20,0x00,0x00]
+          vfmadd213ps 8192(%rdx), %zmm28, %zmm27 {%k3} {z}
+
+// CHECK: vfmadd213ps {rn-sae}, %zmm25, %zmm28, %zmm27 {%k3} {z}
+// CHECK: encoding: [0x62,0x02,0x1d,0x93,0xa8,0xd9]
+          vfmadd213ps {rn-sae}, %zmm25, %zmm28, %zmm27 {%k3} {z}
+
+// CHECK: vfmadd213ps -1032(%rdx){1to16}, %zmm20, %zmm24 {%k3} {z}
+// CHECK: encoding: [0x62,0x62,0x5d,0xd3,0xa8,0x82,0xf8,0xfb,0xff,0xff]
+          vfmadd213ps -1032(%rdx){1to16}, %zmm20, %zmm24 {%k3} {z}
+
+// CHECK: vfmadd213pd 8192(%rdx), %zmm28, %zmm27 {%k3} {z}
+// CHECK: encoding: [0x62,0x62,0x9d,0xc3,0xa8,0x9a,0x00,0x20,0x00,0x00]
+          vfmadd213pd 8192(%rdx), %zmm28, %zmm27 {%k3} {z}
+
+// CHECK: vfmadd213pd {rn-sae}, %zmm25, %zmm28, %zmm27 {%k3} {z}
+// CHECK: encoding: [0x62,0x02,0x9d,0x93,0xa8,0xd9]
+          vfmadd213pd {rn-sae}, %zmm25, %zmm28, %zmm27 {%k3} {z}
+
+// CHECK: vfmadd213pd -1032(%rdx){1to8}, %zmm20, %zmm24 {%k3} {z}
+// CHECK: encoding: [0x62,0x62,0xdd,0xd3,0xa8,0x82,0xf8,0xfb,0xff,0xff]
+          vfmadd213pd -1032(%rdx){1to8}, %zmm20, %zmm24 {%k3} {z}
+
+// CHECK: vfmadd132ps 8192(%rdx), %zmm28, %zmm27 {%k3} {z}
+// CHECK: encoding: [0x62,0x62,0x1d,0xc3,0x98,0x9a,0x00,0x20,0x00,0x00]
+          vfmadd132ps 8192(%rdx), %zmm28, %zmm27 {%k3} {z}
+
+// CHECK: vfmadd132ps {rn-sae}, %zmm25, %zmm28, %zmm27 {%k3} {z}
+// CHECK: encoding: [0x62,0x02,0x1d,0x93,0x98,0xd9]
+          vfmadd132ps {rn-sae}, %zmm25, %zmm28, %zmm27 {%k3} {z}
+
+// CHECK: vfmadd132ps -1032(%rdx){1to16}, %zmm20, %zmm24 {%k3} {z}
+// CHECK: encoding: [0x62,0x62,0x5d,0xd3,0x98,0x82,0xf8,0xfb,0xff,0xff]
+          vfmadd132ps -1032(%rdx){1to16}, %zmm20, %zmm24 {%k3} {z}
+
+// CHECK: vfmadd132pd 8192(%rdx), %zmm28, %zmm27 {%k3} {z}
+// CHECK: encoding: [0x62,0x62,0x9d,0xc3,0x98,0x9a,0x00,0x20,0x00,0x00]
+          vfmadd132pd 8192(%rdx), %zmm28, %zmm27 {%k3} {z}
+
+// CHECK: vfmadd132pd {rn-sae}, %zmm25, %zmm28, %zmm27 {%k3} {z}
+// CHECK: encoding: [0x62,0x02,0x9d,0x93,0x98,0xd9]
+          vfmadd132pd {rn-sae}, %zmm25, %zmm28, %zmm27 {%k3} {z}
+
+// CHECK: vfmadd132pd -1032(%rdx){1to8}, %zmm20, %zmm24 {%k3} {z}
+// CHECK: encoding: [0x62,0x62,0xdd,0xd3,0x98,0x82,0xf8,0xfb,0xff,0xff]
+          vfmadd132pd -1032(%rdx){1to8}, %zmm20, %zmm24 {%k3} {z}
+
+// CHECK: vpermi2d %zmm4, %zmm28, %zmm10
+// CHECK:  encoding: [0x62,0x72,0x1d,0x40,0x76,0xd4]
+          vpermi2d %zmm4, %zmm28, %zmm10
+
+// CHECK: vpermi2d %zmm4, %zmm28, %zmm10 {%k5}
+// CHECK:  encoding: [0x62,0x72,0x1d,0x45,0x76,0xd4]
+          vpermi2d %zmm4, %zmm28, %zmm10 {%k5}
+
+// CHECK: vpermi2d %zmm4, %zmm28, %zmm10 {%k5} {z}
+// CHECK:  encoding: [0x62,0x72,0x1d,0xc5,0x76,0xd4]
+          vpermi2d %zmm4, %zmm28, %zmm10 {%k5} {z}
+
+// CHECK: vpermi2d (%rcx), %zmm28, %zmm10
+// CHECK:  encoding: [0x62,0x72,0x1d,0x40,0x76,0x11]
+          vpermi2d (%rcx), %zmm28, %zmm10
+
+// CHECK: vpermi2d 291(%rax,%r14,8), %zmm28, %zmm10
+// CHECK:  encoding: [0x62,0x32,0x1d,0x40,0x76,0x94,0xf0,0x23,0x01,0x00,0x00]
+          vpermi2d 291(%rax,%r14,8), %zmm28, %zmm10
+
+// CHECK: vpermi2d (%rcx){1to16}, %zmm28, %zmm10
+// CHECK:  encoding: [0x62,0x72,0x1d,0x50,0x76,0x11]
+          vpermi2d (%rcx){1to16}, %zmm28, %zmm10
+
+// CHECK: vpermi2d 8128(%rdx), %zmm28, %zmm10
+// CHECK:  encoding: [0x62,0x72,0x1d,0x40,0x76,0x52,0x7f]
+          vpermi2d 8128(%rdx), %zmm28, %zmm10
+
+// CHECK: vpermi2d 8192(%rdx), %zmm28, %zmm10
+// CHECK:  encoding: [0x62,0x72,0x1d,0x40,0x76,0x92,0x00,0x20,0x00,0x00]
+          vpermi2d 8192(%rdx), %zmm28, %zmm10
+
+// CHECK: vpermi2d -8192(%rdx), %zmm28, %zmm10
+// CHECK:  encoding: [0x62,0x72,0x1d,0x40,0x76,0x52,0x80]
+          vpermi2d -8192(%rdx), %zmm28, %zmm10
+
+// CHECK: vpermi2d -8256(%rdx), %zmm28, %zmm10
+// CHECK:  encoding: [0x62,0x72,0x1d,0x40,0x76,0x92,0xc0,0xdf,0xff,0xff]
+          vpermi2d -8256(%rdx), %zmm28, %zmm10
+
+// CHECK: vpermi2d 508(%rdx){1to16}, %zmm28, %zmm10
+// CHECK:  encoding: [0x62,0x72,0x1d,0x50,0x76,0x52,0x7f]
+          vpermi2d 508(%rdx){1to16}, %zmm28, %zmm10
+
+// CHECK: vpermi2d 512(%rdx){1to16}, %zmm28, %zmm10
+// CHECK:  encoding: [0x62,0x72,0x1d,0x50,0x76,0x92,0x00,0x02,0x00,0x00]
+          vpermi2d 512(%rdx){1to16}, %zmm28, %zmm10
+
+// CHECK: vpermi2d -512(%rdx){1to16}, %zmm28, %zmm10
+// CHECK:  encoding: [0x62,0x72,0x1d,0x50,0x76,0x52,0x80]
+          vpermi2d -512(%rdx){1to16}, %zmm28, %zmm10
+
+// CHECK: vpermi2d -516(%rdx){1to16}, %zmm28, %zmm10
+// CHECK:  encoding: [0x62,0x72,0x1d,0x50,0x76,0x92,0xfc,0xfd,0xff,0xff]
+          vpermi2d -516(%rdx){1to16}, %zmm28, %zmm10
+
+// CHECK: vpermi2q %zmm28, %zmm28, %zmm18
+// CHECK:  encoding: [0x62,0x82,0x9d,0x40,0x76,0xd4]
+          vpermi2q %zmm28, %zmm28, %zmm18
+
+// CHECK: vpermi2q %zmm28, %zmm28, %zmm18 {%k2}
+// CHECK:  encoding: [0x62,0x82,0x9d,0x42,0x76,0xd4]
+          vpermi2q %zmm28, %zmm28, %zmm18 {%k2}
+
+// CHECK: vpermi2q %zmm28, %zmm28, %zmm18 {%k2} {z}
+// CHECK:  encoding: [0x62,0x82,0x9d,0xc2,0x76,0xd4]
+          vpermi2q %zmm28, %zmm28, %zmm18 {%k2} {z}
+
+// CHECK: vpermi2q (%rcx), %zmm28, %zmm18
+// CHECK:  encoding: [0x62,0xe2,0x9d,0x40,0x76,0x11]
+          vpermi2q (%rcx), %zmm28, %zmm18
+
+// CHECK: vpermi2q 291(%rax,%r14,8), %zmm28, %zmm18
+// CHECK:  encoding: [0x62,0xa2,0x9d,0x40,0x76,0x94,0xf0,0x23,0x01,0x00,0x00]
+          vpermi2q 291(%rax,%r14,8), %zmm28, %zmm18
+
+// CHECK: vpermi2q (%rcx){1to8}, %zmm28, %zmm18
+// CHECK:  encoding: [0x62,0xe2,0x9d,0x50,0x76,0x11]
+          vpermi2q (%rcx){1to8}, %zmm28, %zmm18
+
+// CHECK: vpermi2q 8128(%rdx), %zmm28, %zmm18
+// CHECK:  encoding: [0x62,0xe2,0x9d,0x40,0x76,0x52,0x7f]
+          vpermi2q 8128(%rdx), %zmm28, %zmm18
+
+// CHECK: vpermi2q 8192(%rdx), %zmm28, %zmm18
+// CHECK:  encoding: [0x62,0xe2,0x9d,0x40,0x76,0x92,0x00,0x20,0x00,0x00]
+          vpermi2q 8192(%rdx), %zmm28, %zmm18
+
+// CHECK: vpermi2q -8192(%rdx), %zmm28, %zmm18
+// CHECK:  encoding: [0x62,0xe2,0x9d,0x40,0x76,0x52,0x80]
+          vpermi2q -8192(%rdx), %zmm28, %zmm18
+
+// CHECK: vpermi2q -8256(%rdx), %zmm28, %zmm18
+// CHECK:  encoding: [0x62,0xe2,0x9d,0x40,0x76,0x92,0xc0,0xdf,0xff,0xff]
+          vpermi2q -8256(%rdx), %zmm28, %zmm18
+
+// CHECK: vpermi2q 1016(%rdx){1to8}, %zmm28, %zmm18
+// CHECK:  encoding: [0x62,0xe2,0x9d,0x50,0x76,0x52,0x7f]
+          vpermi2q 1016(%rdx){1to8}, %zmm28, %zmm18
+
+// CHECK: vpermi2q 1024(%rdx){1to8}, %zmm28, %zmm18
+// CHECK:  encoding: [0x62,0xe2,0x9d,0x50,0x76,0x92,0x00,0x04,0x00,0x00]
+          vpermi2q 1024(%rdx){1to8}, %zmm28, %zmm18
+
+// CHECK: vpermi2q -1024(%rdx){1to8}, %zmm28, %zmm18
+// CHECK:  encoding: [0x62,0xe2,0x9d,0x50,0x76,0x52,0x80]
+          vpermi2q -1024(%rdx){1to8}, %zmm28, %zmm18
+
+// CHECK: vpermi2q -1032(%rdx){1to8}, %zmm28, %zmm18
+// CHECK:  encoding: [0x62,0xe2,0x9d,0x50,0x76,0x92,0xf8,0xfb,0xff,0xff]
+          vpermi2q -1032(%rdx){1to8}, %zmm28, %zmm18
+
+// CHECK: vpermi2ps %zmm8, %zmm23, %zmm24
+// CHECK:  encoding: [0x62,0x42,0x45,0x40,0x77,0xc0]
+          vpermi2ps %zmm8, %zmm23, %zmm24
+
+// CHECK: vpermi2ps %zmm8, %zmm23, %zmm24 {%k2}
+// CHECK:  encoding: [0x62,0x42,0x45,0x42,0x77,0xc0]
+          vpermi2ps %zmm8, %zmm23, %zmm24 {%k2}
+
+// CHECK: vpermi2ps %zmm8, %zmm23, %zmm24 {%k2} {z}
+// CHECK:  encoding: [0x62,0x42,0x45,0xc2,0x77,0xc0]
+          vpermi2ps %zmm8, %zmm23, %zmm24 {%k2} {z}
+
+// CHECK: vpermi2ps (%rcx), %zmm23, %zmm24
+// CHECK:  encoding: [0x62,0x62,0x45,0x40,0x77,0x01]
+          vpermi2ps (%rcx), %zmm23, %zmm24
+
+// CHECK: vpermi2ps 291(%rax,%r14,8), %zmm23, %zmm24
+// CHECK:  encoding: [0x62,0x22,0x45,0x40,0x77,0x84,0xf0,0x23,0x01,0x00,0x00]
+          vpermi2ps 291(%rax,%r14,8), %zmm23, %zmm24
+
+// CHECK: vpermi2ps (%rcx){1to16}, %zmm23, %zmm24
+// CHECK:  encoding: [0x62,0x62,0x45,0x50,0x77,0x01]
+          vpermi2ps (%rcx){1to16}, %zmm23, %zmm24
+
+// CHECK: vpermi2ps 8128(%rdx), %zmm23, %zmm24
+// CHECK:  encoding: [0x62,0x62,0x45,0x40,0x77,0x42,0x7f]
+          vpermi2ps 8128(%rdx), %zmm23, %zmm24
+
+// CHECK: vpermi2ps 8192(%rdx), %zmm23, %zmm24
+// CHECK:  encoding: [0x62,0x62,0x45,0x40,0x77,0x82,0x00,0x20,0x00,0x00]
+          vpermi2ps 8192(%rdx), %zmm23, %zmm24
+
+// CHECK: vpermi2ps -8192(%rdx), %zmm23, %zmm24
+// CHECK:  encoding: [0x62,0x62,0x45,0x40,0x77,0x42,0x80]
+          vpermi2ps -8192(%rdx), %zmm23, %zmm24
+
+// CHECK: vpermi2ps -8256(%rdx), %zmm23, %zmm24
+// CHECK:  encoding: [0x62,0x62,0x45,0x40,0x77,0x82,0xc0,0xdf,0xff,0xff]
+          vpermi2ps -8256(%rdx), %zmm23, %zmm24
+
+// CHECK: vpermi2ps 508(%rdx){1to16}, %zmm23, %zmm24
+// CHECK:  encoding: [0x62,0x62,0x45,0x50,0x77,0x42,0x7f]
+          vpermi2ps 508(%rdx){1to16}, %zmm23, %zmm24
+
+// CHECK: vpermi2ps 512(%rdx){1to16}, %zmm23, %zmm24
+// CHECK:  encoding: [0x62,0x62,0x45,0x50,0x77,0x82,0x00,0x02,0x00,0x00]
+          vpermi2ps 512(%rdx){1to16}, %zmm23, %zmm24
+
+// CHECK: vpermi2ps -512(%rdx){1to16}, %zmm23, %zmm24
+// CHECK:  encoding: [0x62,0x62,0x45,0x50,0x77,0x42,0x80]
+          vpermi2ps -512(%rdx){1to16}, %zmm23, %zmm24
+
+// CHECK: vpermi2ps -516(%rdx){1to16}, %zmm23, %zmm24
+// CHECK:  encoding: [0x62,0x62,0x45,0x50,0x77,0x82,0xfc,0xfd,0xff,0xff]
+          vpermi2ps -516(%rdx){1to16}, %zmm23, %zmm24
+
+// CHECK: vpermi2pd %zmm20, %zmm5, %zmm20
+// CHECK:  encoding: [0x62,0xa2,0xd5,0x48,0x77,0xe4]
+          vpermi2pd %zmm20, %zmm5, %zmm20
+
+// CHECK: vpermi2pd %zmm20, %zmm5, %zmm20 {%k3}
+// CHECK:  encoding: [0x62,0xa2,0xd5,0x4b,0x77,0xe4]
+          vpermi2pd %zmm20, %zmm5, %zmm20 {%k3}
+
+// CHECK: vpermi2pd %zmm20, %zmm5, %zmm20 {%k3} {z}
+// CHECK:  encoding: [0x62,0xa2,0xd5,0xcb,0x77,0xe4]
+          vpermi2pd %zmm20, %zmm5, %zmm20 {%k3} {z}
+
+// CHECK: vpermi2pd (%rcx), %zmm5, %zmm20
+// CHECK:  encoding: [0x62,0xe2,0xd5,0x48,0x77,0x21]
+          vpermi2pd (%rcx), %zmm5, %zmm20
+
+// CHECK: vpermi2pd 291(%rax,%r14,8), %zmm5, %zmm20
+// CHECK:  encoding: [0x62,0xa2,0xd5,0x48,0x77,0xa4,0xf0,0x23,0x01,0x00,0x00]
+          vpermi2pd 291(%rax,%r14,8), %zmm5, %zmm20
+
+// CHECK: vpermi2pd (%rcx){1to8}, %zmm5, %zmm20
+// CHECK:  encoding: [0x62,0xe2,0xd5,0x58,0x77,0x21]
+          vpermi2pd (%rcx){1to8}, %zmm5, %zmm20
+
+// CHECK: vpermi2pd 8128(%rdx), %zmm5, %zmm20
+// CHECK:  encoding: [0x62,0xe2,0xd5,0x48,0x77,0x62,0x7f]
+          vpermi2pd 8128(%rdx), %zmm5, %zmm20
+
+// CHECK: vpermi2pd 8192(%rdx), %zmm5, %zmm20
+// CHECK:  encoding: [0x62,0xe2,0xd5,0x48,0x77,0xa2,0x00,0x20,0x00,0x00]
+          vpermi2pd 8192(%rdx), %zmm5, %zmm20
+
+// CHECK: vpermi2pd -8192(%rdx), %zmm5, %zmm20
+// CHECK:  encoding: [0x62,0xe2,0xd5,0x48,0x77,0x62,0x80]
+          vpermi2pd -8192(%rdx), %zmm5, %zmm20
+
+// CHECK: vpermi2pd -8256(%rdx), %zmm5, %zmm20
+// CHECK:  encoding: [0x62,0xe2,0xd5,0x48,0x77,0xa2,0xc0,0xdf,0xff,0xff]
+          vpermi2pd -8256(%rdx), %zmm5, %zmm20
+
+// CHECK: vpermi2pd 1016(%rdx){1to8}, %zmm5, %zmm20
+// CHECK:  encoding: [0x62,0xe2,0xd5,0x58,0x77,0x62,0x7f]
+          vpermi2pd 1016(%rdx){1to8}, %zmm5, %zmm20
+
+// CHECK: vpermi2pd 1024(%rdx){1to8}, %zmm5, %zmm20
+// CHECK:  encoding: [0x62,0xe2,0xd5,0x58,0x77,0xa2,0x00,0x04,0x00,0x00]
+          vpermi2pd 1024(%rdx){1to8}, %zmm5, %zmm20
+
+// CHECK: vpermi2pd -1024(%rdx){1to8}, %zmm5, %zmm20
+// CHECK:  encoding: [0x62,0xe2,0xd5,0x58,0x77,0x62,0x80]
+          vpermi2pd -1024(%rdx){1to8}, %zmm5, %zmm20
+
+// CHECK: vpermi2pd -1032(%rdx){1to8}, %zmm5, %zmm20
+// CHECK:  encoding: [0x62,0xe2,0xd5,0x58,0x77,0xa2,0xf8,0xfb,0xff,0xff]
+          vpermi2pd -1032(%rdx){1to8}, %zmm5, %zmm20
+
+// CHECK: vcompresspd %zmm9, (%rcx)
+// CHECK:  encoding: [0x62,0x72,0xfd,0x48,0x8a,0x09]
+          vcompresspd %zmm9, (%rcx)
+
+// CHECK: vcompresspd %zmm9, (%rcx) {%k4}
+// CHECK:  encoding: [0x62,0x72,0xfd,0x4c,0x8a,0x09]
+          vcompresspd %zmm9, (%rcx) {%k4}
+
+// CHECK: vcompresspd %zmm9, 291(%rax,%r14,8)
+// CHECK:  encoding: [0x62,0x32,0xfd,0x48,0x8a,0x8c,0xf0,0x23,0x01,0x00,0x00]
+          vcompresspd %zmm9, 291(%rax,%r14,8)
+
+// CHECK: vcompresspd %zmm9, 1016(%rdx)
+// CHECK:  encoding: [0x62,0x72,0xfd,0x48,0x8a,0x4a,0x7f]
+          vcompresspd %zmm9, 1016(%rdx)
+
+// CHECK: vcompresspd %zmm9, 1024(%rdx)
+// CHECK:  encoding: [0x62,0x72,0xfd,0x48,0x8a,0x8a,0x00,0x04,0x00,0x00]
+          vcompresspd %zmm9, 1024(%rdx)
+
+// CHECK: vcompresspd %zmm9, -1024(%rdx)
+// CHECK:  encoding: [0x62,0x72,0xfd,0x48,0x8a,0x4a,0x80]
+          vcompresspd %zmm9, -1024(%rdx)
+
+// CHECK: vcompresspd %zmm9, -1032(%rdx)
+// CHECK:  encoding: [0x62,0x72,0xfd,0x48,0x8a,0x8a,0xf8,0xfb,0xff,0xff]
+          vcompresspd %zmm9, -1032(%rdx)
+
+// CHECK: vcompresspd %zmm4, %zmm8
+// CHECK:  encoding: [0x62,0xd2,0xfd,0x48,0x8a,0xe0]
+          vcompresspd %zmm4, %zmm8
+
+// CHECK: vcompresspd %zmm4, %zmm8 {%k6}
+// CHECK:  encoding: [0x62,0xd2,0xfd,0x4e,0x8a,0xe0]
+          vcompresspd %zmm4, %zmm8 {%k6}
+
+// CHECK: vcompresspd %zmm4, %zmm8 {%k6} {z}
+// CHECK:  encoding: [0x62,0xd2,0xfd,0xce,0x8a,0xe0]
+          vcompresspd %zmm4, %zmm8 {%k6} {z}
+
+// CHECK: vcompressps %zmm10, (%rcx)
+// CHECK:  encoding: [0x62,0x72,0x7d,0x48,0x8a,0x11]
+          vcompressps %zmm10, (%rcx)
+
+// CHECK: vcompressps %zmm10, (%rcx) {%k7}
+// CHECK:  encoding: [0x62,0x72,0x7d,0x4f,0x8a,0x11]
+          vcompressps %zmm10, (%rcx) {%k7}
+
+// CHECK: vcompressps %zmm10, 291(%rax,%r14,8)
+// CHECK:  encoding: [0x62,0x32,0x7d,0x48,0x8a,0x94,0xf0,0x23,0x01,0x00,0x00]
+          vcompressps %zmm10, 291(%rax,%r14,8)
+
+// CHECK: vcompressps %zmm10, 508(%rdx)
+// CHECK:  encoding: [0x62,0x72,0x7d,0x48,0x8a,0x52,0x7f]
+          vcompressps %zmm10, 508(%rdx)
+
+// CHECK: vcompressps %zmm10, 512(%rdx)
+// CHECK:  encoding: [0x62,0x72,0x7d,0x48,0x8a,0x92,0x00,0x02,0x00,0x00]
+          vcompressps %zmm10, 512(%rdx)
+
+// CHECK: vcompressps %zmm10, -512(%rdx)
+// CHECK:  encoding: [0x62,0x72,0x7d,0x48,0x8a,0x52,0x80]
+          vcompressps %zmm10, -512(%rdx)
+
+// CHECK: vcompressps %zmm10, -516(%rdx)
+// CHECK:  encoding: [0x62,0x72,0x7d,0x48,0x8a,0x92,0xfc,0xfd,0xff,0xff]
+          vcompressps %zmm10, -516(%rdx)
+
+// CHECK: vcompressps %zmm14, %zmm4
+// CHECK:  encoding: [0x62,0x72,0x7d,0x48,0x8a,0xf4]
+          vcompressps %zmm14, %zmm4
+
+// CHECK: vcompressps %zmm14, %zmm4 {%k2}
+// CHECK:  encoding: [0x62,0x72,0x7d,0x4a,0x8a,0xf4]
+          vcompressps %zmm14, %zmm4 {%k2}
+
+// CHECK: vcompressps %zmm14, %zmm4 {%k2} {z}
+// CHECK:  encoding: [0x62,0x72,0x7d,0xca,0x8a,0xf4]
+          vcompressps %zmm14, %zmm4 {%k2} {z}
+
+// CHECK: vexpandpd (%rcx), %zmm24
+// CHECK:  encoding: [0x62,0x62,0xfd,0x48,0x88,0x01]
+          vexpandpd (%rcx), %zmm24
+
+// CHECK: vexpandpd (%rcx), %zmm24 {%k4}
+// CHECK:  encoding: [0x62,0x62,0xfd,0x4c,0x88,0x01]
+          vexpandpd (%rcx), %zmm24 {%k4}
+
+// CHECK: vexpandpd (%rcx), %zmm24 {%k4} {z}
+// CHECK:  encoding: [0x62,0x62,0xfd,0xcc,0x88,0x01]
+          vexpandpd (%rcx), %zmm24 {%k4} {z}
+
+// CHECK: vexpandpd 291(%rax,%r14,8), %zmm24
+// CHECK:  encoding: [0x62,0x22,0xfd,0x48,0x88,0x84,0xf0,0x23,0x01,0x00,0x00]
+          vexpandpd 291(%rax,%r14,8), %zmm24
+
+// CHECK: vexpandpd 1016(%rdx), %zmm24
+// CHECK:  encoding: [0x62,0x62,0xfd,0x48,0x88,0x42,0x7f]
+          vexpandpd 1016(%rdx), %zmm24
+
+// CHECK: vexpandpd 1024(%rdx), %zmm24
+// CHECK:  encoding: [0x62,0x62,0xfd,0x48,0x88,0x82,0x00,0x04,0x00,0x00]
+          vexpandpd 1024(%rdx), %zmm24
+
+// CHECK: vexpandpd -1024(%rdx), %zmm24
+// CHECK:  encoding: [0x62,0x62,0xfd,0x48,0x88,0x42,0x80]
+          vexpandpd -1024(%rdx), %zmm24
+
+// CHECK: vexpandpd -1032(%rdx), %zmm24
+// CHECK:  encoding: [0x62,0x62,0xfd,0x48,0x88,0x82,0xf8,0xfb,0xff,0xff]
+          vexpandpd -1032(%rdx), %zmm24
+
+// CHECK: vexpandpd %zmm15, %zmm23
+// CHECK:  encoding: [0x62,0xc2,0xfd,0x48,0x88,0xff]
+          vexpandpd %zmm15, %zmm23
+
+// CHECK: vexpandpd %zmm15, %zmm23 {%k5}
+// CHECK:  encoding: [0x62,0xc2,0xfd,0x4d,0x88,0xff]
+          vexpandpd %zmm15, %zmm23 {%k5}
+
+// CHECK: vexpandpd %zmm15, %zmm23 {%k5} {z}
+// CHECK:  encoding: [0x62,0xc2,0xfd,0xcd,0x88,0xff]
+          vexpandpd %zmm15, %zmm23 {%k5} {z}
+
+// CHECK: vexpandps (%rcx), %zmm4
+// CHECK:  encoding: [0x62,0xf2,0x7d,0x48,0x88,0x21]
+          vexpandps (%rcx), %zmm4
+
+// CHECK: vexpandps (%rcx), %zmm4 {%k6}
+// CHECK:  encoding: [0x62,0xf2,0x7d,0x4e,0x88,0x21]
+          vexpandps (%rcx), %zmm4 {%k6}
+
+// CHECK: vexpandps (%rcx), %zmm4 {%k6} {z}
+// CHECK:  encoding: [0x62,0xf2,0x7d,0xce,0x88,0x21]
+          vexpandps (%rcx), %zmm4 {%k6} {z}
+
+// CHECK: vexpandps 291(%rax,%r14,8), %zmm4
+// CHECK:  encoding: [0x62,0xb2,0x7d,0x48,0x88,0xa4,0xf0,0x23,0x01,0x00,0x00]
+          vexpandps 291(%rax,%r14,8), %zmm4
+
+// CHECK: vexpandps 508(%rdx), %zmm4
+// CHECK:  encoding: [0x62,0xf2,0x7d,0x48,0x88,0x62,0x7f]
+          vexpandps 508(%rdx), %zmm4
+
+// CHECK: vexpandps 512(%rdx), %zmm4
+// CHECK:  encoding: [0x62,0xf2,0x7d,0x48,0x88,0xa2,0x00,0x02,0x00,0x00]
+          vexpandps 512(%rdx), %zmm4
+
+// CHECK: vexpandps -512(%rdx), %zmm4
+// CHECK:  encoding: [0x62,0xf2,0x7d,0x48,0x88,0x62,0x80]
+          vexpandps -512(%rdx), %zmm4
+
+// CHECK: vexpandps -516(%rdx), %zmm4
+// CHECK:  encoding: [0x62,0xf2,0x7d,0x48,0x88,0xa2,0xfc,0xfd,0xff,0xff]
+          vexpandps -516(%rdx), %zmm4
+
+// CHECK: vexpandps %zmm9, %zmm14
+// CHECK:  encoding: [0x62,0x52,0x7d,0x48,0x88,0xf1]
+          vexpandps %zmm9, %zmm14
+
+// CHECK: vexpandps %zmm9, %zmm14 {%k2}
+// CHECK:  encoding: [0x62,0x52,0x7d,0x4a,0x88,0xf1]
+          vexpandps %zmm9, %zmm14 {%k2}
+
+// CHECK: vexpandps %zmm9, %zmm14 {%k2} {z}
+// CHECK:  encoding: [0x62,0x52,0x7d,0xca,0x88,0xf1]
+          vexpandps %zmm9, %zmm14 {%k2} {z}
+
+// CHECK: vpabsd %zmm14, %zmm15
+// CHECK:  encoding: [0x62,0x52,0x7d,0x48,0x1e,0xfe]
+          vpabsd %zmm14, %zmm15
+
+// CHECK: vpabsd %zmm14, %zmm15 {%k6}
+// CHECK:  encoding: [0x62,0x52,0x7d,0x4e,0x1e,0xfe]
+          vpabsd %zmm14, %zmm15 {%k6}
+
+// CHECK: vpabsd %zmm14, %zmm15 {%k6} {z}
+// CHECK:  encoding: [0x62,0x52,0x7d,0xce,0x1e,0xfe]
+          vpabsd %zmm14, %zmm15 {%k6} {z}
+
+// CHECK: vpabsd (%rcx), %zmm15
+// CHECK:  encoding: [0x62,0x72,0x7d,0x48,0x1e,0x39]
+          vpabsd (%rcx), %zmm15
+
+// CHECK: vpabsd 291(%rax,%r14,8), %zmm15
+// CHECK:  encoding: [0x62,0x32,0x7d,0x48,0x1e,0xbc,0xf0,0x23,0x01,0x00,0x00]
+          vpabsd 291(%rax,%r14,8), %zmm15
+
+// CHECK: vpabsd (%rcx){1to16}, %zmm15
+// CHECK:  encoding: [0x62,0x72,0x7d,0x58,0x1e,0x39]
+          vpabsd (%rcx){1to16}, %zmm15
+
+// CHECK: vpabsd 8128(%rdx), %zmm15
+// CHECK:  encoding: [0x62,0x72,0x7d,0x48,0x1e,0x7a,0x7f]
+          vpabsd 8128(%rdx), %zmm15
+
+// CHECK: vpabsd 8192(%rdx), %zmm15
+// CHECK:  encoding: [0x62,0x72,0x7d,0x48,0x1e,0xba,0x00,0x20,0x00,0x00]
+          vpabsd 8192(%rdx), %zmm15
+
+// CHECK: vpabsd -8192(%rdx), %zmm15
+// CHECK:  encoding: [0x62,0x72,0x7d,0x48,0x1e,0x7a,0x80]
+          vpabsd -8192(%rdx), %zmm15
+
+// CHECK: vpabsd -8256(%rdx), %zmm15
+// CHECK:  encoding: [0x62,0x72,0x7d,0x48,0x1e,0xba,0xc0,0xdf,0xff,0xff]
+          vpabsd -8256(%rdx), %zmm15
+
+// CHECK: vpabsd 508(%rdx){1to16}, %zmm15
+// CHECK:  encoding: [0x62,0x72,0x7d,0x58,0x1e,0x7a,0x7f]
+          vpabsd 508(%rdx){1to16}, %zmm15
+
+// CHECK: vpabsd 512(%rdx){1to16}, %zmm15
+// CHECK:  encoding: [0x62,0x72,0x7d,0x58,0x1e,0xba,0x00,0x02,0x00,0x00]
+          vpabsd 512(%rdx){1to16}, %zmm15
+
+// CHECK: vpabsd -512(%rdx){1to16}, %zmm15
+// CHECK:  encoding: [0x62,0x72,0x7d,0x58,0x1e,0x7a,0x80]
+          vpabsd -512(%rdx){1to16}, %zmm15
+
+// CHECK: vpabsd -516(%rdx){1to16}, %zmm15
+// CHECK:  encoding: [0x62,0x72,0x7d,0x58,0x1e,0xba,0xfc,0xfd,0xff,0xff]
+          vpabsd -516(%rdx){1to16}, %zmm15
+
+// CHECK: vpabsq %zmm24, %zmm5
+// CHECK:  encoding: [0x62,0x92,0xfd,0x48,0x1f,0xe8]
+          vpabsq %zmm24, %zmm5
+
+// CHECK: vpabsq %zmm24, %zmm5 {%k6}
+// CHECK:  encoding: [0x62,0x92,0xfd,0x4e,0x1f,0xe8]
+          vpabsq %zmm24, %zmm5 {%k6}
+
+// CHECK: vpabsq %zmm24, %zmm5 {%k6} {z}
+// CHECK:  encoding: [0x62,0x92,0xfd,0xce,0x1f,0xe8]
+          vpabsq %zmm24, %zmm5 {%k6} {z}
+
+// CHECK: vpabsq (%rcx), %zmm5
+// CHECK:  encoding: [0x62,0xf2,0xfd,0x48,0x1f,0x29]
+          vpabsq (%rcx), %zmm5
+
+// CHECK: vpabsq 291(%rax,%r14,8), %zmm5
+// CHECK:  encoding: [0x62,0xb2,0xfd,0x48,0x1f,0xac,0xf0,0x23,0x01,0x00,0x00]
+          vpabsq 291(%rax,%r14,8), %zmm5
+
+// CHECK: vpabsq (%rcx){1to8}, %zmm5
+// CHECK:  encoding: [0x62,0xf2,0xfd,0x58,0x1f,0x29]
+          vpabsq (%rcx){1to8}, %zmm5
+
+// CHECK: vpabsq 8128(%rdx), %zmm5
+// CHECK:  encoding: [0x62,0xf2,0xfd,0x48,0x1f,0x6a,0x7f]
+          vpabsq 8128(%rdx), %zmm5
+
+// CHECK: vpabsq 8192(%rdx), %zmm5
+// CHECK:  encoding: [0x62,0xf2,0xfd,0x48,0x1f,0xaa,0x00,0x20,0x00,0x00]
+          vpabsq 8192(%rdx), %zmm5
+
+// CHECK: vpabsq -8192(%rdx), %zmm5
+// CHECK:  encoding: [0x62,0xf2,0xfd,0x48,0x1f,0x6a,0x80]
+          vpabsq -8192(%rdx), %zmm5
+
+// CHECK: vpabsq -8256(%rdx), %zmm5
+// CHECK:  encoding: [0x62,0xf2,0xfd,0x48,0x1f,0xaa,0xc0,0xdf,0xff,0xff]
+          vpabsq -8256(%rdx), %zmm5
+
+// CHECK: vpabsq 1016(%rdx){1to8}, %zmm5
+// CHECK:  encoding: [0x62,0xf2,0xfd,0x58,0x1f,0x6a,0x7f]
+          vpabsq 1016(%rdx){1to8}, %zmm5
+
+// CHECK: vpabsq 1024(%rdx){1to8}, %zmm5
+// CHECK:  encoding: [0x62,0xf2,0xfd,0x58,0x1f,0xaa,0x00,0x04,0x00,0x00]
+          vpabsq 1024(%rdx){1to8}, %zmm5
+
+// CHECK: vpabsq -1024(%rdx){1to8}, %zmm5
+// CHECK:  encoding: [0x62,0xf2,0xfd,0x58,0x1f,0x6a,0x80]
+          vpabsq -1024(%rdx){1to8}, %zmm5
+
+// CHECK: vpabsq -1032(%rdx){1to8}, %zmm5
+// CHECK:  encoding: [0x62,0xf2,0xfd,0x58,0x1f,0xaa,0xf8,0xfb,0xff,0xff]
+          vpabsq -1032(%rdx){1to8}, %zmm5
+
+// CHECK: vpgatherdd 123(%r14,%zmm11,8), %zmm17 {%k1}
+// CHECK:  encoding: [0x62,0x82,0x7d,0x49,0x90,0x8c,0xde,0x7b,0x00,0x00,0x00]
+          vpgatherdd 123(%r14, %zmm11,8), %zmm17 {%k1}
+
+// CHECK: vpgatherdd 256(%r9,%zmm11), %zmm17 {%k1}
+// CHECK:  encoding: [0x62,0x82,0x7d,0x49,0x90,0x4c,0x19,0x40]
+          vpgatherdd 256(%r9,%zmm11), %zmm17 {%k1}
+
+// CHECK: vpgatherdd 1024(%rcx,%zmm11,4), %zmm17 {%k1}
+// CHECK:  encoding: [0x62,0xa2,0x7d,0x49,0x90,0x8c,0x99,0x00,0x04,0x00,0x00]
+          vpgatherdd 1024(%rcx, %zmm11,4), %zmm17 {%k1}
+
+// CHECK: vpgatherdq 123(%r14,%ymm14,8), %zmm8 {%k1}
+// CHECK:  encoding: [0x62,0x12,0xfd,0x49,0x90,0x84,0xf6,0x7b,0x00,0x00,0x00]
+          vpgatherdq 123(%r14, %ymm14,8), %zmm8 {%k1}
+
+// CHECK: vpgatherdq 256(%r9,%ymm14), %zmm8 {%k1}
+// CHECK:  encoding: [0x62,0x12,0xfd,0x49,0x90,0x44,0x31,0x20]
+          vpgatherdq 256(%r9, %ymm14), %zmm8 {%k1}
+
+// CHECK: vpgatherdq 1024(%rcx,%ymm14,4), %zmm8 {%k1}
+// CHECK:  encoding: [0x62,0x32,0xfd,0x49,0x90,0x84,0xb1,0x00,0x04,0x00,0x00]
+          vpgatherdq 1024(%rcx, %ymm14,4), %zmm8 {%k1}
+
+// CHECK: vpgatherqd 123(%r14,%zmm17,8), %ymm3 {%k1}
+// CHECK:  encoding: [0x62,0xd2,0x7d,0x41,0x91,0x9c,0xce,0x7b,0x00,0x00,0x00]
+          vpgatherqd 123(%r14, %zmm17,8), %ymm3 {%k1}
+
+// CHECK: vpgatherqd 256(%r9,%zmm17), %ymm3 {%k1}
+// CHECK:  encoding: [0x62,0xd2,0x7d,0x41,0x91,0x5c,0x09,0x40]
+          vpgatherqd 256(%r9,%zmm17), %ymm3 {%k1}
+
+// CHECK: vpgatherqd 1024(%rcx,%zmm17,4), %ymm3 {%k1}
+// CHECK:  encoding: [0x62,0xf2,0x7d,0x41,0x91,0x9c,0x89,0x00,0x04,0x00,0x00]
+          vpgatherqd 1024(%rcx, %zmm17,4), %ymm3 {%k1}
+
+// CHECK: vpgatherqq 123(%r14,%zmm21,8), %zmm17 {%k1}
+// CHECK:  encoding: [0x62,0xc2,0xfd,0x41,0x91,0x8c,0xee,0x7b,0x00,0x00,0x00]
+          vpgatherqq 123(%r14, %zmm21,8), %zmm17 {%k1}
+
+// CHECK: vpgatherqq 256(%r9,%zmm21), %zmm17 {%k1}
+// CHECK:  encoding: [0x62,0xc2,0xfd,0x41,0x91,0x4c,0x29,0x20]
+          vpgatherqq 256(%r9,%zmm21), %zmm17 {%k1}
+
+// CHECK: vpgatherqq 1024(%rcx,%zmm21,4), %zmm17 {%k1}
+// CHECK:  encoding: [0x62,0xe2,0xfd,0x41,0x91,0x8c,0xa9,0x00,0x04,0x00,0x00]
+          vpgatherqq 1024(%rcx, %zmm21,4), %zmm17 {%k1}
+
+// CHECK: vpscatterdd %zmm19, 123(%r14,%zmm16,8) {%k1}
+// CHECK:  encoding: [0x62,0xc2,0x7d,0x41,0xa0,0x9c,0xc6,0x7b,0x00,0x00,0x00]
+          vpscatterdd %zmm19, 123(%r14,%zmm16,8) {%k1}
+
+// CHECK: vpscatterdd %zmm19, 123(%r14,%zmm16,8) {%k1}
+// CHECK:  encoding: [0x62,0xc2,0x7d,0x41,0xa0,0x9c,0xc6,0x7b,0x00,0x00,0x00]
+          vpscatterdd %zmm19, 123(%r14,%zmm16,8) {%k1}
+
+// CHECK: vpscatterdd %zmm19, 256(%r9,%zmm16) {%k1}
+// CHECK:  encoding: [0x62,0xc2,0x7d,0x41,0xa0,0x5c,0x01,0x40]
+          vpscatterdd %zmm19, 256(%r9,%zmm16) {%k1}
+
+// CHECK: vpscatterdd %zmm19, 1024(%rcx,%zmm16,4) {%k1}
+// CHECK:  encoding: [0x62,0xe2,0x7d,0x41,0xa0,0x9c,0x81,0x00,0x04,0x00,0x00]
+          vpscatterdd %zmm19, 1024(%rcx,%zmm16,4) {%k1}
+
+// CHECK: vpscatterdq %zmm5, 123(%r14,%ymm6,8) {%k1}
+// CHECK:  encoding: [0x62,0xd2,0xfd,0x49,0xa0,0xac,0xf6,0x7b,0x00,0x00,0x00]
+          vpscatterdq %zmm5, 123(%r14,%ymm6,8) {%k1}
+
+// CHECK: vpscatterdq %zmm5, 123(%r14,%ymm6,8) {%k1}
+// CHECK:  encoding: [0x62,0xd2,0xfd,0x49,0xa0,0xac,0xf6,0x7b,0x00,0x00,0x00]
+          vpscatterdq %zmm5, 123(%r14,%ymm6,8) {%k1}
+
+// CHECK: vpscatterdq %zmm5, 256(%r9,%ymm6) {%k1}
+// CHECK:  encoding: [0x62,0xd2,0xfd,0x49,0xa0,0x6c,0x31,0x20]
+          vpscatterdq %zmm5, 256(%r9,%ymm6) {%k1}
+
+// CHECK: vpscatterdq %zmm5, 1024(%rcx,%ymm6,4) {%k1}
+// CHECK:  encoding: [0x62,0xf2,0xfd,0x49,0xa0,0xac,0xb1,0x00,0x04,0x00,0x00]
+          vpscatterdq %zmm5, 1024(%rcx,%ymm6,4) {%k1}
+
+// CHECK: vpscatterqd %ymm20, 123(%r14,%zmm2,8) {%k1}
+// CHECK:  encoding: [0x62,0xc2,0x7d,0x49,0xa1,0xa4,0xd6,0x7b,0x00,0x00,0x00]
+          vpscatterqd %ymm20, 123(%r14,%zmm2,8) {%k1}
+
+// CHECK: vpscatterqd %ymm20, 123(%r14,%zmm2,8) {%k1}
+// CHECK:  encoding: [0x62,0xc2,0x7d,0x49,0xa1,0xa4,0xd6,0x7b,0x00,0x00,0x00]
+          vpscatterqd %ymm20, 123(%r14,%zmm2,8) {%k1}
+
+// CHECK: vpscatterqd %ymm20, 256(%r9,%zmm2) {%k1}
+// CHECK:  encoding: [0x62,0xc2,0x7d,0x49,0xa1,0x64,0x11,0x40]
+          vpscatterqd %ymm20, 256(%r9,%zmm2) {%k1}
+
+// CHECK: vpscatterqd %ymm20, 1024(%rcx,%zmm2,4) {%k1}
+// CHECK:  encoding: [0x62,0xe2,0x7d,0x49,0xa1,0xa4,0x91,0x00,0x04,0x00,0x00]
+          vpscatterqd %ymm20, 1024(%rcx,%zmm2,4) {%k1}
+
+// CHECK: vpscatterqq %zmm14, 123(%r14,%zmm20,8) {%k1}
+// CHECK:  encoding: [0x62,0x52,0xfd,0x41,0xa1,0xb4,0xe6,0x7b,0x00,0x00,0x00]
+          vpscatterqq %zmm14, 123(%r14,%zmm20,8) {%k1}
+
+// CHECK: vpscatterqq %zmm14, 123(%r14,%zmm20,8) {%k1}
+// CHECK:  encoding: [0x62,0x52,0xfd,0x41,0xa1,0xb4,0xe6,0x7b,0x00,0x00,0x00]
+          vpscatterqq %zmm14, 123(%r14,%zmm20,8) {%k1}
+
+// CHECK: vpscatterqq %zmm14, 256(%r9,%zmm20) {%k1}
+// CHECK:  encoding: [0x62,0x52,0xfd,0x41,0xa1,0x74,0x21,0x20]
+          vpscatterqq %zmm14, 256(%r9,%zmm20) {%k1}
+
+// CHECK: vpscatterqq %zmm14, 1024(%rcx,%zmm20,4) {%k1}
+// CHECK:  encoding: [0x62,0x72,0xfd,0x41,0xa1,0xb4,0xa1,0x00,0x04,0x00,0x00]
+          vpscatterqq %zmm14, 1024(%rcx,%zmm20,4) {%k1}
+// CHECK: vscalefpd %zmm28, %zmm26, %zmm26
+// CHECK:  encoding: [0x62,0x02,0xad,0x40,0x2c,0xd4]
+          vscalefpd %zmm28, %zmm26, %zmm26
+
+// CHECK: vscalefpd %zmm28, %zmm26, %zmm26 {%k5}
+// CHECK:  encoding: [0x62,0x02,0xad,0x45,0x2c,0xd4]
+          vscalefpd %zmm28, %zmm26, %zmm26 {%k5}
+
+// CHECK: vscalefpd %zmm28, %zmm26, %zmm26 {%k5} {z}
+// CHECK:  encoding: [0x62,0x02,0xad,0xc5,0x2c,0xd4]
+          vscalefpd %zmm28, %zmm26, %zmm26 {%k5} {z}
+
+// CHECK: vscalefpd {rn-sae}, %zmm28, %zmm26, %zmm26
+// CHECK:  encoding: [0x62,0x02,0xad,0x10,0x2c,0xd4]
+          vscalefpd {rn-sae}, %zmm28, %zmm26, %zmm26
+
+// CHECK: vscalefpd {ru-sae}, %zmm28, %zmm26, %zmm26
+// CHECK:  encoding: [0x62,0x02,0xad,0x50,0x2c,0xd4]
+          vscalefpd {ru-sae}, %zmm28, %zmm26, %zmm26
+
+// CHECK: vscalefpd {rd-sae}, %zmm28, %zmm26, %zmm26
+// CHECK:  encoding: [0x62,0x02,0xad,0x30,0x2c,0xd4]
+          vscalefpd {rd-sae}, %zmm28, %zmm26, %zmm26
+
+// CHECK: vscalefpd {rz-sae}, %zmm28, %zmm26, %zmm26
+// CHECK:  encoding: [0x62,0x02,0xad,0x70,0x2c,0xd4]
+          vscalefpd {rz-sae}, %zmm28, %zmm26, %zmm26
+
+// CHECK: vscalefpd (%rcx), %zmm26, %zmm26
+// CHECK:  encoding: [0x62,0x62,0xad,0x40,0x2c,0x11]
+          vscalefpd (%rcx), %zmm26, %zmm26
+
+// CHECK: vscalefpd 291(%rax,%r14,8), %zmm26, %zmm26
+// CHECK:  encoding: [0x62,0x22,0xad,0x40,0x2c,0x94,0xf0,0x23,0x01,0x00,0x00]
+          vscalefpd 291(%rax,%r14,8), %zmm26, %zmm26
+
+// CHECK: vscalefpd (%rcx){1to8}, %zmm26, %zmm26
+// CHECK:  encoding: [0x62,0x62,0xad,0x50,0x2c,0x11]
+          vscalefpd (%rcx){1to8}, %zmm26, %zmm26
+
+// CHECK: vscalefpd 8128(%rdx), %zmm26, %zmm26
+// CHECK:  encoding: [0x62,0x62,0xad,0x40,0x2c,0x52,0x7f]
+          vscalefpd 8128(%rdx), %zmm26, %zmm26
+
+// CHECK: vscalefpd 8192(%rdx), %zmm26, %zmm26
+// CHECK:  encoding: [0x62,0x62,0xad,0x40,0x2c,0x92,0x00,0x20,0x00,0x00]
+          vscalefpd 8192(%rdx), %zmm26, %zmm26
+
+// CHECK: vscalefpd -8192(%rdx), %zmm26, %zmm26
+// CHECK:  encoding: [0x62,0x62,0xad,0x40,0x2c,0x52,0x80]
+          vscalefpd -8192(%rdx), %zmm26, %zmm26
+
+// CHECK: vscalefpd -8256(%rdx), %zmm26, %zmm26
+// CHECK:  encoding: [0x62,0x62,0xad,0x40,0x2c,0x92,0xc0,0xdf,0xff,0xff]
+          vscalefpd -8256(%rdx), %zmm26, %zmm26
+
+// CHECK: vscalefpd 1016(%rdx){1to8}, %zmm26, %zmm26
+// CHECK:  encoding: [0x62,0x62,0xad,0x50,0x2c,0x52,0x7f]
+          vscalefpd 1016(%rdx){1to8}, %zmm26, %zmm26
+
+// CHECK: vscalefpd 1024(%rdx){1to8}, %zmm26, %zmm26
+// CHECK:  encoding: [0x62,0x62,0xad,0x50,0x2c,0x92,0x00,0x04,0x00,0x00]
+          vscalefpd 1024(%rdx){1to8}, %zmm26, %zmm26
+
+// CHECK: vscalefpd -1024(%rdx){1to8}, %zmm26, %zmm26
+// CHECK:  encoding: [0x62,0x62,0xad,0x50,0x2c,0x52,0x80]
+          vscalefpd -1024(%rdx){1to8}, %zmm26, %zmm26
+
+// CHECK: vscalefpd -1032(%rdx){1to8}, %zmm26, %zmm26
+// CHECK:  encoding: [0x62,0x62,0xad,0x50,0x2c,0x92,0xf8,0xfb,0xff,0xff]
+          vscalefpd -1032(%rdx){1to8}, %zmm26, %zmm26
+
+// CHECK: vscalefps %zmm18, %zmm6, %zmm19
+// CHECK:  encoding: [0x62,0xa2,0x4d,0x48,0x2c,0xda]
+          vscalefps %zmm18, %zmm6, %zmm19
+
+// CHECK: vscalefps %zmm18, %zmm6, %zmm19 {%k6}
+// CHECK:  encoding: [0x62,0xa2,0x4d,0x4e,0x2c,0xda]
+          vscalefps %zmm18, %zmm6, %zmm19 {%k6}
+
+// CHECK: vscalefps %zmm18, %zmm6, %zmm19 {%k6} {z}
+// CHECK:  encoding: [0x62,0xa2,0x4d,0xce,0x2c,0xda]
+          vscalefps %zmm18, %zmm6, %zmm19 {%k6} {z}
+
+// CHECK: vscalefps {rn-sae}, %zmm18, %zmm6, %zmm19
+// CHECK:  encoding: [0x62,0xa2,0x4d,0x18,0x2c,0xda]
+          vscalefps {rn-sae}, %zmm18, %zmm6, %zmm19
+
+// CHECK: vscalefps {ru-sae}, %zmm18, %zmm6, %zmm19
+// CHECK:  encoding: [0x62,0xa2,0x4d,0x58,0x2c,0xda]
+          vscalefps {ru-sae}, %zmm18, %zmm6, %zmm19
+
+// CHECK: vscalefps {rd-sae}, %zmm18, %zmm6, %zmm19
+// CHECK:  encoding: [0x62,0xa2,0x4d,0x38,0x2c,0xda]
+          vscalefps {rd-sae}, %zmm18, %zmm6, %zmm19
+
+// CHECK: vscalefps {rz-sae}, %zmm18, %zmm6, %zmm19
+// CHECK:  encoding: [0x62,0xa2,0x4d,0x78,0x2c,0xda]
+          vscalefps {rz-sae}, %zmm18, %zmm6, %zmm19
+
+// CHECK: vscalefps (%rcx), %zmm6, %zmm19
+// CHECK:  encoding: [0x62,0xe2,0x4d,0x48,0x2c,0x19]
+          vscalefps (%rcx), %zmm6, %zmm19
+
+// CHECK: vscalefps 291(%rax,%r14,8), %zmm6, %zmm19
+// CHECK:  encoding: [0x62,0xa2,0x4d,0x48,0x2c,0x9c,0xf0,0x23,0x01,0x00,0x00]
+          vscalefps 291(%rax,%r14,8), %zmm6, %zmm19
+
+// CHECK: vscalefps (%rcx){1to16}, %zmm6, %zmm19
+// CHECK:  encoding: [0x62,0xe2,0x4d,0x58,0x2c,0x19]
+          vscalefps (%rcx){1to16}, %zmm6, %zmm19
+
+// CHECK: vscalefps 8128(%rdx), %zmm6, %zmm19
+// CHECK:  encoding: [0x62,0xe2,0x4d,0x48,0x2c,0x5a,0x7f]
+          vscalefps 8128(%rdx), %zmm6, %zmm19
+
+// CHECK: vscalefps 8192(%rdx), %zmm6, %zmm19
+// CHECK:  encoding: [0x62,0xe2,0x4d,0x48,0x2c,0x9a,0x00,0x20,0x00,0x00]
+          vscalefps 8192(%rdx), %zmm6, %zmm19
+
+// CHECK: vscalefps -8192(%rdx), %zmm6, %zmm19
+// CHECK:  encoding: [0x62,0xe2,0x4d,0x48,0x2c,0x5a,0x80]
+          vscalefps -8192(%rdx), %zmm6, %zmm19
+
+// CHECK: vscalefps -8256(%rdx), %zmm6, %zmm19
+// CHECK:  encoding: [0x62,0xe2,0x4d,0x48,0x2c,0x9a,0xc0,0xdf,0xff,0xff]
+          vscalefps -8256(%rdx), %zmm6, %zmm19
+
+// CHECK: vscalefps 508(%rdx){1to16}, %zmm6, %zmm19
+// CHECK:  encoding: [0x62,0xe2,0x4d,0x58,0x2c,0x5a,0x7f]
+          vscalefps 508(%rdx){1to16}, %zmm6, %zmm19
+
+// CHECK: vscalefps 512(%rdx){1to16}, %zmm6, %zmm19
+// CHECK:  encoding: [0x62,0xe2,0x4d,0x58,0x2c,0x9a,0x00,0x02,0x00,0x00]
+          vscalefps 512(%rdx){1to16}, %zmm6, %zmm19
+
+// CHECK: vscalefps -512(%rdx){1to16}, %zmm6, %zmm19
+// CHECK:  encoding: [0x62,0xe2,0x4d,0x58,0x2c,0x5a,0x80]
+          vscalefps -512(%rdx){1to16}, %zmm6, %zmm19
+
+// CHECK: vscalefps -516(%rdx){1to16}, %zmm6, %zmm19
+// CHECK:  encoding: [0x62,0xe2,0x4d,0x58,0x2c,0x9a,0xfc,0xfd,0xff,0xff]
+          vscalefps -516(%rdx){1to16}, %zmm6, %zmm19
+
+// CHECK: vcvtps2pd %ymm6, %zmm13
+// CHECK:  encoding: [0x62,0x71,0x7c,0x48,0x5a,0xee]
+          vcvtps2pd %ymm6, %zmm13
+
+// CHECK: vcvtps2pd %ymm6, %zmm13 {%k3}
+// CHECK:  encoding: [0x62,0x71,0x7c,0x4b,0x5a,0xee]
+          vcvtps2pd %ymm6, %zmm13 {%k3}
+
+// CHECK: vcvtps2pd %ymm6, %zmm13 {%k3} {z}
+// CHECK:  encoding: [0x62,0x71,0x7c,0xcb,0x5a,0xee]
+          vcvtps2pd %ymm6, %zmm13 {%k3} {z}
+
+// CHECK: vcvtps2pd {sae}, %ymm6, %zmm13
+// CHECK:  encoding: [0x62,0x71,0x7c,0x18,0x5a,0xee]
+          vcvtps2pd {sae}, %ymm6, %zmm13
+
+// CHECK: vcvtps2pd (%rcx), %zmm13
+// CHECK:  encoding: [0x62,0x71,0x7c,0x48,0x5a,0x29]
+          vcvtps2pd (%rcx), %zmm13
+
+// CHECK: vcvtps2pd 291(%rax,%r14,8), %zmm13
+// CHECK:  encoding: [0x62,0x31,0x7c,0x48,0x5a,0xac,0xf0,0x23,0x01,0x00,0x00]
+          vcvtps2pd 291(%rax,%r14,8), %zmm13
+
+// CHECK: vcvtps2pd (%rcx){1to8}, %zmm13
+// CHECK:  encoding: [0x62,0x71,0x7c,0x58,0x5a,0x29]
+          vcvtps2pd (%rcx){1to8}, %zmm13
+
+// CHECK: vcvtps2pd 4064(%rdx), %zmm13
+// CHECK:  encoding: [0x62,0x71,0x7c,0x48,0x5a,0x6a,0x7f]
+          vcvtps2pd 4064(%rdx), %zmm13
+
+// CHECK: vcvtps2pd 4096(%rdx), %zmm13
+// CHECK:  encoding: [0x62,0x71,0x7c,0x48,0x5a,0xaa,0x00,0x10,0x00,0x00]
+          vcvtps2pd 4096(%rdx), %zmm13
+
+// CHECK: vcvtps2pd -4096(%rdx), %zmm13
+// CHECK:  encoding: [0x62,0x71,0x7c,0x48,0x5a,0x6a,0x80]
+          vcvtps2pd -4096(%rdx), %zmm13
+
+// CHECK: vcvtps2pd -4128(%rdx), %zmm13
+// CHECK:  encoding: [0x62,0x71,0x7c,0x48,0x5a,0xaa,0xe0,0xef,0xff,0xff]
+          vcvtps2pd -4128(%rdx), %zmm13
+
+// CHECK: vcvtps2pd 508(%rdx){1to8}, %zmm13
+// CHECK:  encoding: [0x62,0x71,0x7c,0x58,0x5a,0x6a,0x7f]
+          vcvtps2pd 508(%rdx){1to8}, %zmm13
+
+// CHECK: vcvtps2pd 512(%rdx){1to8}, %zmm13
+// CHECK:  encoding: [0x62,0x71,0x7c,0x58,0x5a,0xaa,0x00,0x02,0x00,0x00]
+          vcvtps2pd 512(%rdx){1to8}, %zmm13
+
+// CHECK: vcvtps2pd -512(%rdx){1to8}, %zmm13
+// CHECK:  encoding: [0x62,0x71,0x7c,0x58,0x5a,0x6a,0x80]
+          vcvtps2pd -512(%rdx){1to8}, %zmm13
+
+// CHECK: vcvtps2pd -516(%rdx){1to8}, %zmm13
+// CHECK:  encoding: [0x62,0x71,0x7c,0x58,0x5a,0xaa,0xfc,0xfd,0xff,0xff]
+          vcvtps2pd -516(%rdx){1to8}, %zmm13
+
+// CHECK: vcvtpd2ps %zmm23, %ymm5
+// CHECK:  encoding: [0x62,0xb1,0xfd,0x48,0x5a,0xef]
+          vcvtpd2ps %zmm23, %ymm5
+
+// CHECK: vcvtpd2ps %zmm23, %ymm5 {%k5}
+// CHECK:  encoding: [0x62,0xb1,0xfd,0x4d,0x5a,0xef]
+          vcvtpd2ps %zmm23, %ymm5 {%k5}
+
+// CHECK: vcvtpd2ps %zmm23, %ymm5 {%k5} {z}
+// CHECK:  encoding: [0x62,0xb1,0xfd,0xcd,0x5a,0xef]
+          vcvtpd2ps %zmm23, %ymm5 {%k5} {z}
+
+// CHECK: vcvtpd2ps {rn-sae}, %zmm23, %ymm5
+// CHECK:  encoding: [0x62,0xb1,0xfd,0x18,0x5a,0xef]
+          vcvtpd2ps {rn-sae}, %zmm23, %ymm5
+
+// CHECK: vcvtpd2ps {ru-sae}, %zmm23, %ymm5
+// CHECK:  encoding: [0x62,0xb1,0xfd,0x58,0x5a,0xef]
+          vcvtpd2ps {ru-sae}, %zmm23, %ymm5
+
+// CHECK: vcvtpd2ps {rd-sae}, %zmm23, %ymm5
+// CHECK:  encoding: [0x62,0xb1,0xfd,0x38,0x5a,0xef]
+          vcvtpd2ps {rd-sae}, %zmm23, %ymm5
+
+// CHECK: vcvtpd2ps {rz-sae}, %zmm23, %ymm5
+// CHECK:  encoding: [0x62,0xb1,0xfd,0x78,0x5a,0xef]
+          vcvtpd2ps {rz-sae}, %zmm23, %ymm5
+
+// CHECK: vcvtpd2ps (%rcx), %ymm5
+// CHECK:  encoding: [0x62,0xf1,0xfd,0x48,0x5a,0x29]
+          vcvtpd2ps (%rcx), %ymm5
+
+// CHECK: vcvtpd2ps 291(%rax,%r14,8), %ymm5
+// CHECK:  encoding: [0x62,0xb1,0xfd,0x48,0x5a,0xac,0xf0,0x23,0x01,0x00,0x00]
+          vcvtpd2ps 291(%rax,%r14,8), %ymm5
+
+// CHECK: vcvtpd2ps (%rcx){1to8}, %ymm5
+// CHECK:  encoding: [0x62,0xf1,0xfd,0x58,0x5a,0x29]
+          vcvtpd2ps (%rcx){1to8}, %ymm5
+
+// CHECK: vcvtpd2ps 8128(%rdx), %ymm5
+// CHECK:  encoding: [0x62,0xf1,0xfd,0x48,0x5a,0x6a,0x7f]
+          vcvtpd2ps 8128(%rdx), %ymm5
+
+// CHECK: vcvtpd2ps 8192(%rdx), %ymm5
+// CHECK:  encoding: [0x62,0xf1,0xfd,0x48,0x5a,0xaa,0x00,0x20,0x00,0x00]
+          vcvtpd2ps 8192(%rdx), %ymm5
+
+// CHECK: vcvtpd2ps -8192(%rdx), %ymm5
+// CHECK:  encoding: [0x62,0xf1,0xfd,0x48,0x5a,0x6a,0x80]
+          vcvtpd2ps -8192(%rdx), %ymm5
+
+// CHECK: vcvtpd2ps -8256(%rdx), %ymm5
+// CHECK:  encoding: [0x62,0xf1,0xfd,0x48,0x5a,0xaa,0xc0,0xdf,0xff,0xff]
+          vcvtpd2ps -8256(%rdx), %ymm5
+
+// CHECK: vcvtpd2ps 1016(%rdx){1to8}, %ymm5
+// CHECK:  encoding: [0x62,0xf1,0xfd,0x58,0x5a,0x6a,0x7f]
+          vcvtpd2ps 1016(%rdx){1to8}, %ymm5
+
+// CHECK: vcvtpd2ps 1024(%rdx){1to8}, %ymm5
+// CHECK:  encoding: [0x62,0xf1,0xfd,0x58,0x5a,0xaa,0x00,0x04,0x00,0x00]
+          vcvtpd2ps 1024(%rdx){1to8}, %ymm5
+
+// CHECK: vcvtpd2ps -1024(%rdx){1to8}, %ymm5
+// CHECK:  encoding: [0x62,0xf1,0xfd,0x58,0x5a,0x6a,0x80]
+          vcvtpd2ps -1024(%rdx){1to8}, %ymm5
+
+// CHECK: vcvtpd2ps -1032(%rdx){1to8}, %ymm5
+// CHECK:  encoding: [0x62,0xf1,0xfd,0x58,0x5a,0xaa,0xf8,0xfb,0xff,0xff]
+          vcvtpd2ps -1032(%rdx){1to8}, %ymm5
+
+// CHECK: vscalefsd %xmm21, %xmm22, %xmm21
+// CHECK:  encoding: [0x62,0xa2,0xcd,0x00,0x2d,0xed]
+          vscalefsd %xmm21, %xmm22, %xmm21
+
+// CHECK: vscalefsd %xmm21, %xmm22, %xmm21 {%k2}
+// CHECK:  encoding: [0x62,0xa2,0xcd,0x02,0x2d,0xed]
+          vscalefsd %xmm21, %xmm22, %xmm21 {%k2}
+
+// CHECK: vscalefsd %xmm21, %xmm22, %xmm21 {%k2} {z}
+// CHECK:  encoding: [0x62,0xa2,0xcd,0x82,0x2d,0xed]
+          vscalefsd %xmm21, %xmm22, %xmm21 {%k2} {z}
+
+// CHECK: vscalefsd {rn-sae}, %xmm21, %xmm22, %xmm21
+// CHECK:  encoding: [0x62,0xa2,0xcd,0x10,0x2d,0xed]
+          vscalefsd {rn-sae}, %xmm21, %xmm22, %xmm21
+
+// CHECK: vscalefsd {ru-sae}, %xmm21, %xmm22, %xmm21
+// CHECK:  encoding: [0x62,0xa2,0xcd,0x50,0x2d,0xed]
+          vscalefsd {ru-sae}, %xmm21, %xmm22, %xmm21
+
+// CHECK: vscalefsd {rd-sae}, %xmm21, %xmm22, %xmm21
+// CHECK:  encoding: [0x62,0xa2,0xcd,0x30,0x2d,0xed]
+          vscalefsd {rd-sae}, %xmm21, %xmm22, %xmm21
+
+// CHECK: vscalefsd {rz-sae}, %xmm21, %xmm22, %xmm21
+// CHECK:  encoding: [0x62,0xa2,0xcd,0x70,0x2d,0xed]
+          vscalefsd {rz-sae}, %xmm21, %xmm22, %xmm21
+
+// CHECK: vscalefsd (%rcx), %xmm22, %xmm21
+// CHECK:  encoding: [0x62,0xe2,0xcd,0x00,0x2d,0x29]
+          vscalefsd (%rcx), %xmm22, %xmm21
+
+// CHECK: vscalefsd 291(%rax,%r14,8), %xmm22, %xmm21
+// CHECK:  encoding: [0x62,0xa2,0xcd,0x00,0x2d,0xac,0xf0,0x23,0x01,0x00,0x00]
+          vscalefsd 291(%rax,%r14,8), %xmm22, %xmm21
+
+// CHECK: vscalefsd 1016(%rdx), %xmm22, %xmm21
+// CHECK:  encoding: [0x62,0xe2,0xcd,0x00,0x2d,0x6a,0x7f]
+          vscalefsd 1016(%rdx), %xmm22, %xmm21
+
+// CHECK: vscalefsd 1024(%rdx), %xmm22, %xmm21
+// CHECK:  encoding: [0x62,0xe2,0xcd,0x00,0x2d,0xaa,0x00,0x04,0x00,0x00]
+          vscalefsd 1024(%rdx), %xmm22, %xmm21
+
+// CHECK: vscalefsd -1024(%rdx), %xmm22, %xmm21
+// CHECK:  encoding: [0x62,0xe2,0xcd,0x00,0x2d,0x6a,0x80]
+          vscalefsd -1024(%rdx), %xmm22, %xmm21
+
+// CHECK: vscalefsd -1032(%rdx), %xmm22, %xmm21
+// CHECK:  encoding: [0x62,0xe2,0xcd,0x00,0x2d,0xaa,0xf8,0xfb,0xff,0xff]
+          vscalefsd -1032(%rdx), %xmm22, %xmm21
+
+// CHECK: vscalefss %xmm23, %xmm15, %xmm13
+// CHECK:  encoding: [0x62,0x32,0x05,0x08,0x2d,0xef]
+          vscalefss %xmm23, %xmm15, %xmm13
+
+// CHECK: vscalefss %xmm23, %xmm15, %xmm13 {%k3}
+// CHECK:  encoding: [0x62,0x32,0x05,0x0b,0x2d,0xef]
+          vscalefss %xmm23, %xmm15, %xmm13 {%k3}
+
+// CHECK: vscalefss %xmm23, %xmm15, %xmm13 {%k3} {z}
+// CHECK:  encoding: [0x62,0x32,0x05,0x8b,0x2d,0xef]
+          vscalefss %xmm23, %xmm15, %xmm13 {%k3} {z}
+
+// CHECK: vscalefss {rn-sae}, %xmm23, %xmm15, %xmm13
+// CHECK:  encoding: [0x62,0x32,0x05,0x18,0x2d,0xef]
+          vscalefss {rn-sae}, %xmm23, %xmm15, %xmm13
+
+// CHECK: vscalefss {ru-sae}, %xmm23, %xmm15, %xmm13
+// CHECK:  encoding: [0x62,0x32,0x05,0x58,0x2d,0xef]
+          vscalefss {ru-sae}, %xmm23, %xmm15, %xmm13
+
+// CHECK: vscalefss {rd-sae}, %xmm23, %xmm15, %xmm13
+// CHECK:  encoding: [0x62,0x32,0x05,0x38,0x2d,0xef]
+          vscalefss {rd-sae}, %xmm23, %xmm15, %xmm13
+
+// CHECK: vscalefss {rz-sae}, %xmm23, %xmm15, %xmm13
+// CHECK:  encoding: [0x62,0x32,0x05,0x78,0x2d,0xef]
+          vscalefss {rz-sae}, %xmm23, %xmm15, %xmm13
+
+// CHECK: vscalefss (%rcx), %xmm15, %xmm13
+// CHECK:  encoding: [0x62,0x72,0x05,0x08,0x2d,0x29]
+          vscalefss (%rcx), %xmm15, %xmm13
+
+// CHECK: vscalefss 291(%rax,%r14,8), %xmm15, %xmm13
+// CHECK:  encoding: [0x62,0x32,0x05,0x08,0x2d,0xac,0xf0,0x23,0x01,0x00,0x00]
+          vscalefss 291(%rax,%r14,8), %xmm15, %xmm13
+
+// CHECK: vscalefss 508(%rdx), %xmm15, %xmm13
+// CHECK:  encoding: [0x62,0x72,0x05,0x08,0x2d,0x6a,0x7f]
+          vscalefss 508(%rdx), %xmm15, %xmm13
+
+// CHECK: vscalefss 512(%rdx), %xmm15, %xmm13
+// CHECK:  encoding: [0x62,0x72,0x05,0x08,0x2d,0xaa,0x00,0x02,0x00,0x00]
+          vscalefss 512(%rdx), %xmm15, %xmm13
+
+// CHECK: vscalefss -512(%rdx), %xmm15, %xmm13
+// CHECK:  encoding: [0x62,0x72,0x05,0x08,0x2d,0x6a,0x80]
+          vscalefss -512(%rdx), %xmm15, %xmm13
+
+// CHECK: vscalefss -516(%rdx), %xmm15, %xmm13
+// CHECK:  encoding: [0x62,0x72,0x05,0x08,0x2d,0xaa,0xfc,0xfd,0xff,0xff]
+          vscalefss -516(%rdx), %xmm15, %xmm13
+
+// CHECK: vrndscalepd $171, %zmm7, %zmm22
+// CHECK:  encoding: [0x62,0xe3,0xfd,0x48,0x09,0xf7,0xab]
+          vrndscalepd $0xab, %zmm7, %zmm22
+
+// CHECK: vrndscalepd $171, %zmm7, %zmm22 {%k1}
+// CHECK:  encoding: [0x62,0xe3,0xfd,0x49,0x09,0xf7,0xab]
+          vrndscalepd $0xab, %zmm7, %zmm22 {%k1}
+
+// CHECK: vrndscalepd $171, %zmm7, %zmm22 {%k1} {z}
+// CHECK:  encoding: [0x62,0xe3,0xfd,0xc9,0x09,0xf7,0xab]
+          vrndscalepd $0xab, %zmm7, %zmm22 {%k1} {z}
+
+// CHECK: vrndscalepd $171,{sae}, %zmm7, %zmm22
+// CHECK:  encoding: [0x62,0xe3,0xfd,0x18,0x09,0xf7,0xab]
+          vrndscalepd $0xab,{sae}, %zmm7, %zmm22
+
+// CHECK: vrndscalepd $123, %zmm7, %zmm22
+// CHECK:  encoding: [0x62,0xe3,0xfd,0x48,0x09,0xf7,0x7b]
+          vrndscalepd $0x7b, %zmm7, %zmm22
+
+// CHECK: vrndscalepd $123,{sae}, %zmm7, %zmm22
+// CHECK:  encoding: [0x62,0xe3,0xfd,0x18,0x09,0xf7,0x7b]
+          vrndscalepd $0x7b,{sae}, %zmm7, %zmm22
+
+// CHECK: vrndscalepd $123, (%rcx), %zmm22
+// CHECK:  encoding: [0x62,0xe3,0xfd,0x48,0x09,0x31,0x7b]
+          vrndscalepd $0x7b, (%rcx), %zmm22
+
+// CHECK: vrndscalepd $123, 291(%rax,%r14,8), %zmm22
+// CHECK:  encoding: [0x62,0xa3,0xfd,0x48,0x09,0xb4,0xf0,0x23,0x01,0x00,0x00,0x7b]
+          vrndscalepd $0x7b, 291(%rax,%r14,8), %zmm22
+
+// CHECK: vrndscalepd $123, (%rcx){1to8}, %zmm22
+// CHECK:  encoding: [0x62,0xe3,0xfd,0x58,0x09,0x31,0x7b]
+          vrndscalepd $0x7b, (%rcx){1to8}, %zmm22
+
+// CHECK: vrndscalepd $123, 8128(%rdx), %zmm22
+// CHECK:  encoding: [0x62,0xe3,0xfd,0x48,0x09,0x72,0x7f,0x7b]
+          vrndscalepd $0x7b, 8128(%rdx), %zmm22
+
+// CHECK: vrndscalepd $123, 8192(%rdx), %zmm22
+// CHECK:  encoding: [0x62,0xe3,0xfd,0x48,0x09,0xb2,0x00,0x20,0x00,0x00,0x7b]
+          vrndscalepd $0x7b, 8192(%rdx), %zmm22
+
+// CHECK: vrndscalepd $123, -8192(%rdx), %zmm22
+// CHECK:  encoding: [0x62,0xe3,0xfd,0x48,0x09,0x72,0x80,0x7b]
+          vrndscalepd $0x7b, -8192(%rdx), %zmm22
+
+// CHECK: vrndscalepd $123, -8256(%rdx), %zmm22
+// CHECK:  encoding: [0x62,0xe3,0xfd,0x48,0x09,0xb2,0xc0,0xdf,0xff,0xff,0x7b]
+          vrndscalepd $0x7b, -8256(%rdx), %zmm22
+
+// CHECK: vrndscalepd $123, 1016(%rdx){1to8}, %zmm22
+// CHECK:  encoding: [0x62,0xe3,0xfd,0x58,0x09,0x72,0x7f,0x7b]
+          vrndscalepd $0x7b, 1016(%rdx){1to8}, %zmm22
+
+// CHECK: vrndscalepd $123, 1024(%rdx){1to8}, %zmm22
+// CHECK:  encoding: [0x62,0xe3,0xfd,0x58,0x09,0xb2,0x00,0x04,0x00,0x00,0x7b]
+          vrndscalepd $0x7b, 1024(%rdx){1to8}, %zmm22
+
+// CHECK: vrndscalepd $123, -1024(%rdx){1to8}, %zmm22
+// CHECK:  encoding: [0x62,0xe3,0xfd,0x58,0x09,0x72,0x80,0x7b]
+          vrndscalepd $0x7b, -1024(%rdx){1to8}, %zmm22
+
+// CHECK: vrndscalepd $123, -1032(%rdx){1to8}, %zmm22
+// CHECK:  encoding: [0x62,0xe3,0xfd,0x58,0x09,0xb2,0xf8,0xfb,0xff,0xff,0x7b]
+          vrndscalepd $0x7b, -1032(%rdx){1to8}, %zmm22
+
+// CHECK: vrndscaleps $171, %zmm7, %zmm13
+// CHECK:  encoding: [0x62,0x73,0x7d,0x48,0x08,0xef,0xab]
+          vrndscaleps $0xab, %zmm7, %zmm13
+
+// CHECK: vrndscaleps $171, %zmm7, %zmm13 {%k1}
+// CHECK:  encoding: [0x62,0x73,0x7d,0x49,0x08,0xef,0xab]
+          vrndscaleps $0xab, %zmm7, %zmm13 {%k1}
+
+// CHECK: vrndscaleps $171, %zmm7, %zmm13 {%k1} {z}
+// CHECK:  encoding: [0x62,0x73,0x7d,0xc9,0x08,0xef,0xab]
+          vrndscaleps $0xab, %zmm7, %zmm13 {%k1} {z}
+
+// CHECK: vrndscaleps $171,{sae}, %zmm7, %zmm13
+// CHECK:  encoding: [0x62,0x73,0x7d,0x18,0x08,0xef,0xab]
+          vrndscaleps $0xab,{sae}, %zmm7, %zmm13
+
+// CHECK: vrndscaleps $123, %zmm7, %zmm13
+// CHECK:  encoding: [0x62,0x73,0x7d,0x48,0x08,0xef,0x7b]
+          vrndscaleps $0x7b, %zmm7, %zmm13
+
+// CHECK: vrndscaleps $123,{sae}, %zmm7, %zmm13
+// CHECK:  encoding: [0x62,0x73,0x7d,0x18,0x08,0xef,0x7b]
+          vrndscaleps $0x7b,{sae}, %zmm7, %zmm13
+
+// CHECK: vrndscaleps $123, (%rcx), %zmm13
+// CHECK:  encoding: [0x62,0x73,0x7d,0x48,0x08,0x29,0x7b]
+          vrndscaleps $0x7b, (%rcx), %zmm13
+
+// CHECK: vrndscaleps $123, 291(%rax,%r14,8), %zmm13
+// CHECK:  encoding: [0x62,0x33,0x7d,0x48,0x08,0xac,0xf0,0x23,0x01,0x00,0x00,0x7b]
+          vrndscaleps $0x7b, 291(%rax,%r14,8), %zmm13
+
+// CHECK: vrndscaleps $123, (%rcx){1to16}, %zmm13
+// CHECK:  encoding: [0x62,0x73,0x7d,0x58,0x08,0x29,0x7b]
+          vrndscaleps $0x7b, (%rcx){1to16}, %zmm13
+
+// CHECK: vrndscaleps $123, 8128(%rdx), %zmm13
+// CHECK:  encoding: [0x62,0x73,0x7d,0x48,0x08,0x6a,0x7f,0x7b]
+          vrndscaleps $0x7b, 8128(%rdx), %zmm13
+
+// CHECK: vrndscaleps $123, 8192(%rdx), %zmm13
+// CHECK:  encoding: [0x62,0x73,0x7d,0x48,0x08,0xaa,0x00,0x20,0x00,0x00,0x7b]
+          vrndscaleps $0x7b, 8192(%rdx), %zmm13
+
+// CHECK: vrndscaleps $123, -8192(%rdx), %zmm13
+// CHECK:  encoding: [0x62,0x73,0x7d,0x48,0x08,0x6a,0x80,0x7b]
+          vrndscaleps $0x7b, -8192(%rdx), %zmm13
+
+// CHECK: vrndscaleps $123, -8256(%rdx), %zmm13
+// CHECK:  encoding: [0x62,0x73,0x7d,0x48,0x08,0xaa,0xc0,0xdf,0xff,0xff,0x7b]
+          vrndscaleps $0x7b, -8256(%rdx), %zmm13
+
+// CHECK: vrndscaleps $123, 508(%rdx){1to16}, %zmm13
+// CHECK:  encoding: [0x62,0x73,0x7d,0x58,0x08,0x6a,0x7f,0x7b]
+          vrndscaleps $0x7b, 508(%rdx){1to16}, %zmm13
+
+// CHECK: vrndscaleps $123, 512(%rdx){1to16}, %zmm13
+// CHECK:  encoding: [0x62,0x73,0x7d,0x58,0x08,0xaa,0x00,0x02,0x00,0x00,0x7b]
+          vrndscaleps $0x7b, 512(%rdx){1to16}, %zmm13
+
+// CHECK: vrndscaleps $123, -512(%rdx){1to16}, %zmm13
+// CHECK:  encoding: [0x62,0x73,0x7d,0x58,0x08,0x6a,0x80,0x7b]
+          vrndscaleps $0x7b, -512(%rdx){1to16}, %zmm13
+
+// CHECK: vrndscaleps $123, -516(%rdx){1to16}, %zmm13
+// CHECK:  encoding: [0x62,0x73,0x7d,0x58,0x08,0xaa,0xfc,0xfd,0xff,0xff,0x7b]
+          vrndscaleps $0x7b, -516(%rdx){1to16}, %zmm13
+
+// CHECK: vrndscalesd $171, %xmm15, %xmm12, %xmm25
+// CHECK:  encoding: [0x62,0x43,0x9d,0x08,0x0b,0xcf,0xab]
+          vrndscalesd $0xab, %xmm15, %xmm12, %xmm25
+
+// CHECK: vrndscalesd $171, %xmm15, %xmm12, %xmm25 {%k6}
+// CHECK:  encoding: [0x62,0x43,0x9d,0x0e,0x0b,0xcf,0xab]
+          vrndscalesd $0xab, %xmm15, %xmm12, %xmm25 {%k6}
+
+// CHECK: vrndscalesd $171, %xmm15, %xmm12, %xmm25 {%k6} {z}
+// CHECK:  encoding: [0x62,0x43,0x9d,0x8e,0x0b,0xcf,0xab]
+          vrndscalesd $0xab, %xmm15, %xmm12, %xmm25 {%k6} {z}
+
+// CHECK: vrndscalesd $171, {sae}, %xmm15, %xmm12, %xmm25
+// CHECK:  encoding: [0x62,0x43,0x9d,0x18,0x0b,0xcf,0xab]
+          vrndscalesd $0xab,{sae}, %xmm15, %xmm12, %xmm25
+
+// CHECK: vrndscalesd $123, %xmm15, %xmm12, %xmm25
+// CHECK:  encoding: [0x62,0x43,0x9d,0x08,0x0b,0xcf,0x7b]
+          vrndscalesd $0x7b, %xmm15, %xmm12, %xmm25
+
+// CHECK: vrndscalesd $123, {sae}, %xmm15, %xmm12, %xmm25
+// CHECK:  encoding: [0x62,0x43,0x9d,0x18,0x0b,0xcf,0x7b]
+          vrndscalesd $0x7b,{sae}, %xmm15, %xmm12, %xmm25
+
+// CHECK: vrndscalesd $123, (%rcx), %xmm12, %xmm25
+// CHECK:  encoding: [0x62,0x63,0x9d,0x08,0x0b,0x09,0x7b]
+          vrndscalesd $0x7b, (%rcx), %xmm12, %xmm25
+
+// CHECK: vrndscalesd $123, 291(%rax,%r14,8), %xmm12, %xmm25
+// CHECK:  encoding: [0x62,0x23,0x9d,0x08,0x0b,0x8c,0xf0,0x23,0x01,0x00,0x00,0x7b]
+          vrndscalesd $0x7b, 291(%rax,%r14,8), %xmm12, %xmm25
+
+// CHECK: vrndscalesd $123, 1016(%rdx), %xmm12, %xmm25
+// CHECK:  encoding: [0x62,0x63,0x9d,0x08,0x0b,0x4a,0x7f,0x7b]
+          vrndscalesd $0x7b, 1016(%rdx), %xmm12, %xmm25
+
+// CHECK: vrndscalesd $123, 1024(%rdx), %xmm12, %xmm25
+// CHECK:  encoding: [0x62,0x63,0x9d,0x08,0x0b,0x8a,0x00,0x04,0x00,0x00,0x7b]
+          vrndscalesd $0x7b, 1024(%rdx), %xmm12, %xmm25
+
+// CHECK: vrndscalesd $123, -1024(%rdx), %xmm12, %xmm25
+// CHECK:  encoding: [0x62,0x63,0x9d,0x08,0x0b,0x4a,0x80,0x7b]
+          vrndscalesd $0x7b, -1024(%rdx), %xmm12, %xmm25
+
+// CHECK: vrndscalesd $123, -1032(%rdx), %xmm12, %xmm25
+// CHECK:  encoding: [0x62,0x63,0x9d,0x08,0x0b,0x8a,0xf8,0xfb,0xff,0xff,0x7b]
+          vrndscalesd $0x7b, -1032(%rdx), %xmm12, %xmm25
+
+// CHECK: vrndscaless $171, %xmm17, %xmm11, %xmm11
+// CHECK:  encoding: [0x62,0x33,0x25,0x08,0x0a,0xd9,0xab]
+          vrndscaless $0xab, %xmm17, %xmm11, %xmm11
+
+// CHECK: vrndscaless $171, %xmm17, %xmm11, %xmm11 {%k3}
+// CHECK:  encoding: [0x62,0x33,0x25,0x0b,0x0a,0xd9,0xab]
+          vrndscaless $0xab, %xmm17, %xmm11, %xmm11 {%k3}
+
+// CHECK: vrndscaless $171, %xmm17, %xmm11, %xmm11 {%k3} {z}
+// CHECK:  encoding: [0x62,0x33,0x25,0x8b,0x0a,0xd9,0xab]
+          vrndscaless $0xab, %xmm17, %xmm11, %xmm11 {%k3} {z}
+
+// CHECK: vrndscaless $171, {sae}, %xmm17, %xmm11, %xmm11
+// CHECK:  encoding: [0x62,0x33,0x25,0x18,0x0a,0xd9,0xab]
+          vrndscaless $0xab,{sae}, %xmm17, %xmm11, %xmm11
+
+// CHECK: vrndscaless $123, %xmm17, %xmm11, %xmm11
+// CHECK:  encoding: [0x62,0x33,0x25,0x08,0x0a,0xd9,0x7b]
+          vrndscaless $0x7b, %xmm17, %xmm11, %xmm11
+
+// CHECK: vrndscaless $123, {sae}, %xmm17, %xmm11, %xmm11
+// CHECK:  encoding: [0x62,0x33,0x25,0x18,0x0a,0xd9,0x7b]
+          vrndscaless $0x7b,{sae}, %xmm17, %xmm11, %xmm11
+
+// CHECK: vrndscaless $123, (%rcx), %xmm11, %xmm11
+// CHECK:  encoding: [0x62,0x73,0x25,0x08,0x0a,0x19,0x7b]
+          vrndscaless $0x7b, (%rcx), %xmm11, %xmm11
+
+// CHECK: vrndscaless $123, 291(%rax,%r14,8), %xmm11, %xmm11
+// CHECK:  encoding: [0x62,0x33,0x25,0x08,0x0a,0x9c,0xf0,0x23,0x01,0x00,0x00,0x7b]
+          vrndscaless $0x7b, 291(%rax,%r14,8), %xmm11, %xmm11
+
+// CHECK: vrndscaless $123, 508(%rdx), %xmm11, %xmm11
+// CHECK:  encoding: [0x62,0x73,0x25,0x08,0x0a,0x5a,0x7f,0x7b]
+          vrndscaless $0x7b, 508(%rdx), %xmm11, %xmm11
+
+// CHECK: vrndscaless $123, 512(%rdx), %xmm11, %xmm11
+// CHECK:  encoding: [0x62,0x73,0x25,0x08,0x0a,0x9a,0x00,0x02,0x00,0x00,0x7b]
+          vrndscaless $0x7b, 512(%rdx), %xmm11, %xmm11
+
+// CHECK: vrndscaless $123, -512(%rdx), %xmm11, %xmm11
+// CHECK:  encoding: [0x62,0x73,0x25,0x08,0x0a,0x5a,0x80,0x7b]
+          vrndscaless $0x7b, -512(%rdx), %xmm11, %xmm11
+
+// CHECK: vrndscaless $123, -516(%rdx), %xmm11, %xmm11
+// CHECK:  encoding: [0x62,0x73,0x25,0x08,0x0a,0x9a,0xfc,0xfd,0xff,0xff,0x7b]
+          vrndscaless $0x7b, -516(%rdx), %xmm11, %xmm11
+
+// CHECK: vfmadd132ss %xmm22, %xmm17, %xmm30
+// CHECK:  encoding: [0x62,0x22,0x75,0x00,0x99,0xf6]
+          vfmadd132ss %xmm22, %xmm17, %xmm30
+
+// CHECK: vfmadd132ss %xmm22, %xmm17, %xmm30 {%k3}
+// CHECK:  encoding: [0x62,0x22,0x75,0x03,0x99,0xf6]
+          vfmadd132ss %xmm22, %xmm17, %xmm30 {%k3}
+
+// CHECK: vfmadd132ss %xmm22, %xmm17, %xmm30 {%k3} {z}
+// CHECK:  encoding: [0x62,0x22,0x75,0x83,0x99,0xf6]
+          vfmadd132ss %xmm22, %xmm17, %xmm30 {%k3} {z}
+
+// CHECK: vfmadd132ss {rn-sae}, %xmm22, %xmm17, %xmm30
+// CHECK:  encoding: [0x62,0x22,0x75,0x10,0x99,0xf6]
+          vfmadd132ss {rn-sae}, %xmm22, %xmm17, %xmm30
+
+// CHECK: vfmadd132ss {ru-sae}, %xmm22, %xmm17, %xmm30
+// CHECK:  encoding: [0x62,0x22,0x75,0x50,0x99,0xf6]
+          vfmadd132ss {ru-sae}, %xmm22, %xmm17, %xmm30
+
+// CHECK: vfmadd132ss {rd-sae}, %xmm22, %xmm17, %xmm30
+// CHECK:  encoding: [0x62,0x22,0x75,0x30,0x99,0xf6]
+          vfmadd132ss {rd-sae}, %xmm22, %xmm17, %xmm30
+
+// CHECK: vfmadd132ss {rz-sae}, %xmm22, %xmm17, %xmm30
+// CHECK:  encoding: [0x62,0x22,0x75,0x70,0x99,0xf6]
+          vfmadd132ss {rz-sae}, %xmm22, %xmm17, %xmm30
+
+// CHECK: vfmadd132ss (%rcx), %xmm17, %xmm30
+// CHECK:  encoding: [0x62,0x62,0x75,0x00,0x99,0x31]
+          vfmadd132ss (%rcx), %xmm17, %xmm30
+
+// CHECK: vfmadd132ss 291(%rax,%r14,8), %xmm17, %xmm30
+// CHECK:  encoding: [0x62,0x22,0x75,0x00,0x99,0xb4,0xf0,0x23,0x01,0x00,0x00]
+          vfmadd132ss 291(%rax,%r14,8), %xmm17, %xmm30
+
+// CHECK: vfmadd132ss 508(%rdx), %xmm17, %xmm30
+// CHECK:  encoding: [0x62,0x62,0x75,0x00,0x99,0x72,0x7f]
+          vfmadd132ss 508(%rdx), %xmm17, %xmm30
+
+// CHECK: vfmadd132ss 512(%rdx), %xmm17, %xmm30
+// CHECK:  encoding: [0x62,0x62,0x75,0x00,0x99,0xb2,0x00,0x02,0x00,0x00]
+          vfmadd132ss 512(%rdx), %xmm17, %xmm30
+
+// CHECK: vfmadd132ss -512(%rdx), %xmm17, %xmm30
+// CHECK:  encoding: [0x62,0x62,0x75,0x00,0x99,0x72,0x80]
+          vfmadd132ss -512(%rdx), %xmm17, %xmm30
+
+// CHECK: vfmadd132ss -516(%rdx), %xmm17, %xmm30
+// CHECK:  encoding: [0x62,0x62,0x75,0x00,0x99,0xb2,0xfc,0xfd,0xff,0xff]
+          vfmadd132ss -516(%rdx), %xmm17, %xmm30
+
+// CHECK: vfmadd132sd %xmm3, %xmm17, %xmm28
+// CHECK:  encoding: [0x62,0x62,0xf5,0x00,0x99,0xe3]
+          vfmadd132sd %xmm3, %xmm17, %xmm28
+
+// CHECK: vfmadd132sd %xmm3, %xmm17, %xmm28 {%k2}
+// CHECK:  encoding: [0x62,0x62,0xf5,0x02,0x99,0xe3]
+          vfmadd132sd %xmm3, %xmm17, %xmm28 {%k2}
+
+// CHECK: vfmadd132sd %xmm3, %xmm17, %xmm28 {%k2} {z}
+// CHECK:  encoding: [0x62,0x62,0xf5,0x82,0x99,0xe3]
+          vfmadd132sd %xmm3, %xmm17, %xmm28 {%k2} {z}
+
+// CHECK: vfmadd132sd {rn-sae}, %xmm3, %xmm17, %xmm28
+// CHECK:  encoding: [0x62,0x62,0xf5,0x10,0x99,0xe3]
+          vfmadd132sd {rn-sae}, %xmm3, %xmm17, %xmm28
+
+// CHECK: vfmadd132sd {ru-sae}, %xmm3, %xmm17, %xmm28
+// CHECK:  encoding: [0x62,0x62,0xf5,0x50,0x99,0xe3]
+          vfmadd132sd {ru-sae}, %xmm3, %xmm17, %xmm28
+
+// CHECK: vfmadd132sd {rd-sae}, %xmm3, %xmm17, %xmm28
+// CHECK:  encoding: [0x62,0x62,0xf5,0x30,0x99,0xe3]
+          vfmadd132sd {rd-sae}, %xmm3, %xmm17, %xmm28
+
+// CHECK: vfmadd132sd {rz-sae}, %xmm3, %xmm17, %xmm28
+// CHECK:  encoding: [0x62,0x62,0xf5,0x70,0x99,0xe3]
+          vfmadd132sd {rz-sae}, %xmm3, %xmm17, %xmm28
+
+// CHECK: vfmadd132sd (%rcx), %xmm17, %xmm28
+// CHECK:  encoding: [0x62,0x62,0xf5,0x00,0x99,0x21]
+          vfmadd132sd (%rcx), %xmm17, %xmm28
+
+// CHECK: vfmadd132sd 291(%rax,%r14,8), %xmm17, %xmm28
+// CHECK:  encoding: [0x62,0x22,0xf5,0x00,0x99,0xa4,0xf0,0x23,0x01,0x00,0x00]
+          vfmadd132sd 291(%rax,%r14,8), %xmm17, %xmm28
+
+// CHECK: vfmadd132sd 1016(%rdx), %xmm17, %xmm28
+// CHECK:  encoding: [0x62,0x62,0xf5,0x00,0x99,0x62,0x7f]
+          vfmadd132sd 1016(%rdx), %xmm17, %xmm28
+
+// CHECK: vfmadd132sd 1024(%rdx), %xmm17, %xmm28
+// CHECK:  encoding: [0x62,0x62,0xf5,0x00,0x99,0xa2,0x00,0x04,0x00,0x00]
+          vfmadd132sd 1024(%rdx), %xmm17, %xmm28
+
+// CHECK: vfmadd132sd -1024(%rdx), %xmm17, %xmm28
+// CHECK:  encoding: [0x62,0x62,0xf5,0x00,0x99,0x62,0x80]
+          vfmadd132sd -1024(%rdx), %xmm17, %xmm28
+
+// CHECK: vfmadd132sd -1032(%rdx), %xmm17, %xmm28
+// CHECK:  encoding: [0x62,0x62,0xf5,0x00,0x99,0xa2,0xf8,0xfb,0xff,0xff]
+          vfmadd132sd -1032(%rdx), %xmm17, %xmm28
+
+// CHECK: vfmadd213ss %xmm16, %xmm22, %xmm30
+// CHECK:  encoding: [0x62,0x22,0x4d,0x00,0xa9,0xf0]
+          vfmadd213ss %xmm16, %xmm22, %xmm30
+
+// CHECK: vfmadd213ss %xmm16, %xmm22, %xmm30 {%k1}
+// CHECK:  encoding: [0x62,0x22,0x4d,0x01,0xa9,0xf0]
+          vfmadd213ss %xmm16, %xmm22, %xmm30 {%k1}
+
+// CHECK: vfmadd213ss %xmm16, %xmm22, %xmm30 {%k1} {z}
+// CHECK:  encoding: [0x62,0x22,0x4d,0x81,0xa9,0xf0]
+          vfmadd213ss %xmm16, %xmm22, %xmm30 {%k1} {z}
+
+// CHECK: vfmadd213ss {rn-sae}, %xmm16, %xmm22, %xmm30
+// CHECK:  encoding: [0x62,0x22,0x4d,0x10,0xa9,0xf0]
+          vfmadd213ss {rn-sae}, %xmm16, %xmm22, %xmm30
+
+// CHECK: vfmadd213ss {ru-sae}, %xmm16, %xmm22, %xmm30
+// CHECK:  encoding: [0x62,0x22,0x4d,0x50,0xa9,0xf0]
+          vfmadd213ss {ru-sae}, %xmm16, %xmm22, %xmm30
+
+// CHECK: vfmadd213ss {rd-sae}, %xmm16, %xmm22, %xmm30
+// CHECK:  encoding: [0x62,0x22,0x4d,0x30,0xa9,0xf0]
+          vfmadd213ss {rd-sae}, %xmm16, %xmm22, %xmm30
+
+// CHECK: vfmadd213ss {rz-sae}, %xmm16, %xmm22, %xmm30
+// CHECK:  encoding: [0x62,0x22,0x4d,0x70,0xa9,0xf0]
+          vfmadd213ss {rz-sae}, %xmm16, %xmm22, %xmm30
+
+// CHECK: vfmadd213ss (%rcx), %xmm22, %xmm30
+// CHECK:  encoding: [0x62,0x62,0x4d,0x00,0xa9,0x31]
+          vfmadd213ss (%rcx), %xmm22, %xmm30
+
+// CHECK: vfmadd213ss 291(%rax,%r14,8), %xmm22, %xmm30
+// CHECK:  encoding: [0x62,0x22,0x4d,0x00,0xa9,0xb4,0xf0,0x23,0x01,0x00,0x00]
+          vfmadd213ss 291(%rax,%r14,8), %xmm22, %xmm30
+
+// CHECK: vfmadd213ss 508(%rdx), %xmm22, %xmm30
+// CHECK:  encoding: [0x62,0x62,0x4d,0x00,0xa9,0x72,0x7f]
+          vfmadd213ss 508(%rdx), %xmm22, %xmm30
+
+// CHECK: vfmadd213ss 512(%rdx), %xmm22, %xmm30
+// CHECK:  encoding: [0x62,0x62,0x4d,0x00,0xa9,0xb2,0x00,0x02,0x00,0x00]
+          vfmadd213ss 512(%rdx), %xmm22, %xmm30
+
+// CHECK: vfmadd213ss -512(%rdx), %xmm22, %xmm30
+// CHECK:  encoding: [0x62,0x62,0x4d,0x00,0xa9,0x72,0x80]
+          vfmadd213ss -512(%rdx), %xmm22, %xmm30
+
+// CHECK: vfmadd213ss -516(%rdx), %xmm22, %xmm30
+// CHECK:  encoding: [0x62,0x62,0x4d,0x00,0xa9,0xb2,0xfc,0xfd,0xff,0xff]
+          vfmadd213ss -516(%rdx), %xmm22, %xmm30
+
+// CHECK: vfmadd213sd %xmm13, %xmm26, %xmm24
+// CHECK:  encoding: [0x62,0x42,0xad,0x00,0xa9,0xc5]
+          vfmadd213sd %xmm13, %xmm26, %xmm24
+
+// CHECK: vfmadd213sd %xmm13, %xmm26, %xmm24 {%k3}
+// CHECK:  encoding: [0x62,0x42,0xad,0x03,0xa9,0xc5]
+          vfmadd213sd %xmm13, %xmm26, %xmm24 {%k3}
+
+// CHECK: vfmadd213sd %xmm13, %xmm26, %xmm24 {%k3} {z}
+// CHECK:  encoding: [0x62,0x42,0xad,0x83,0xa9,0xc5]
+          vfmadd213sd %xmm13, %xmm26, %xmm24 {%k3} {z}
+
+// CHECK: vfmadd213sd {rn-sae}, %xmm13, %xmm26, %xmm24
+// CHECK:  encoding: [0x62,0x42,0xad,0x10,0xa9,0xc5]
+          vfmadd213sd {rn-sae}, %xmm13, %xmm26, %xmm24
+
+// CHECK: vfmadd213sd {ru-sae}, %xmm13, %xmm26, %xmm24
+// CHECK:  encoding: [0x62,0x42,0xad,0x50,0xa9,0xc5]
+          vfmadd213sd {ru-sae}, %xmm13, %xmm26, %xmm24
+
+// CHECK: vfmadd213sd {rd-sae}, %xmm13, %xmm26, %xmm24
+// CHECK:  encoding: [0x62,0x42,0xad,0x30,0xa9,0xc5]
+          vfmadd213sd {rd-sae}, %xmm13, %xmm26, %xmm24
+
+// CHECK: vfmadd213sd {rz-sae}, %xmm13, %xmm26, %xmm24
+// CHECK:  encoding: [0x62,0x42,0xad,0x70,0xa9,0xc5]
+          vfmadd213sd {rz-sae}, %xmm13, %xmm26, %xmm24
+
+// CHECK: vfmadd213sd (%rcx), %xmm26, %xmm24
+// CHECK:  encoding: [0x62,0x62,0xad,0x00,0xa9,0x01]
+          vfmadd213sd (%rcx), %xmm26, %xmm24
+
+// CHECK: vfmadd213sd 291(%rax,%r14,8), %xmm26, %xmm24
+// CHECK:  encoding: [0x62,0x22,0xad,0x00,0xa9,0x84,0xf0,0x23,0x01,0x00,0x00]
+          vfmadd213sd 291(%rax,%r14,8), %xmm26, %xmm24
+
+// CHECK: vfmadd213sd 1016(%rdx), %xmm26, %xmm24
+// CHECK:  encoding: [0x62,0x62,0xad,0x00,0xa9,0x42,0x7f]
+          vfmadd213sd 1016(%rdx), %xmm26, %xmm24
+
+// CHECK: vfmadd213sd 1024(%rdx), %xmm26, %xmm24
+// CHECK:  encoding: [0x62,0x62,0xad,0x00,0xa9,0x82,0x00,0x04,0x00,0x00]
+          vfmadd213sd 1024(%rdx), %xmm26, %xmm24
+
+// CHECK: vfmadd213sd -1024(%rdx), %xmm26, %xmm24
+// CHECK:  encoding: [0x62,0x62,0xad,0x00,0xa9,0x42,0x80]
+          vfmadd213sd -1024(%rdx), %xmm26, %xmm24
+
+// CHECK: vfmadd213sd -1032(%rdx), %xmm26, %xmm24
+// CHECK:  encoding: [0x62,0x62,0xad,0x00,0xa9,0x82,0xf8,0xfb,0xff,0xff]
+          vfmadd213sd -1032(%rdx), %xmm26, %xmm24
+
+// CHECK: vfmadd231ss %xmm10, %xmm15, %xmm29
+// CHECK:  encoding: [0x62,0x42,0x05,0x08,0xb9,0xea]
+          vfmadd231ss %xmm10, %xmm15, %xmm29
+
+// CHECK: vfmadd231ss %xmm10, %xmm15, %xmm29 {%k4}
+// CHECK:  encoding: [0x62,0x42,0x05,0x0c,0xb9,0xea]
+          vfmadd231ss %xmm10, %xmm15, %xmm29 {%k4}
+
+// CHECK: vfmadd231ss %xmm10, %xmm15, %xmm29 {%k4} {z}
+// CHECK:  encoding: [0x62,0x42,0x05,0x8c,0xb9,0xea]
+          vfmadd231ss %xmm10, %xmm15, %xmm29 {%k4} {z}
+
+// CHECK: vfmadd231ss {rn-sae}, %xmm10, %xmm15, %xmm29
+// CHECK:  encoding: [0x62,0x42,0x05,0x18,0xb9,0xea]
+          vfmadd231ss {rn-sae}, %xmm10, %xmm15, %xmm29
+
+// CHECK: vfmadd231ss {ru-sae}, %xmm10, %xmm15, %xmm29
+// CHECK:  encoding: [0x62,0x42,0x05,0x58,0xb9,0xea]
+          vfmadd231ss {ru-sae}, %xmm10, %xmm15, %xmm29
+
+// CHECK: vfmadd231ss {rd-sae}, %xmm10, %xmm15, %xmm29
+// CHECK:  encoding: [0x62,0x42,0x05,0x38,0xb9,0xea]
+          vfmadd231ss {rd-sae}, %xmm10, %xmm15, %xmm29
+
+// CHECK: vfmadd231ss {rz-sae}, %xmm10, %xmm15, %xmm29
+// CHECK:  encoding: [0x62,0x42,0x05,0x78,0xb9,0xea]
+          vfmadd231ss {rz-sae}, %xmm10, %xmm15, %xmm29
+
+// CHECK: vfmadd231ss (%rcx), %xmm15, %xmm29
+// CHECK:  encoding: [0x62,0x62,0x05,0x08,0xb9,0x29]
+          vfmadd231ss (%rcx), %xmm15, %xmm29
+
+// CHECK: vfmadd231ss 291(%rax,%r14,8), %xmm15, %xmm29
+// CHECK:  encoding: [0x62,0x22,0x05,0x08,0xb9,0xac,0xf0,0x23,0x01,0x00,0x00]
+          vfmadd231ss 291(%rax,%r14,8), %xmm15, %xmm29
+
+// CHECK: vfmadd231ss 508(%rdx), %xmm15, %xmm29
+// CHECK:  encoding: [0x62,0x62,0x05,0x08,0xb9,0x6a,0x7f]
+          vfmadd231ss 508(%rdx), %xmm15, %xmm29
+
+// CHECK: vfmadd231ss 512(%rdx), %xmm15, %xmm29
+// CHECK:  encoding: [0x62,0x62,0x05,0x08,0xb9,0xaa,0x00,0x02,0x00,0x00]
+          vfmadd231ss 512(%rdx), %xmm15, %xmm29
+
+// CHECK: vfmadd231ss -512(%rdx), %xmm15, %xmm29
+// CHECK:  encoding: [0x62,0x62,0x05,0x08,0xb9,0x6a,0x80]
+          vfmadd231ss -512(%rdx), %xmm15, %xmm29
+
+// CHECK: vfmadd231ss -516(%rdx), %xmm15, %xmm29
+// CHECK:  encoding: [0x62,0x62,0x05,0x08,0xb9,0xaa,0xfc,0xfd,0xff,0xff]
+          vfmadd231ss -516(%rdx), %xmm15, %xmm29
+
+// CHECK: vfmadd231sd %xmm14, %xmm1, %xmm4
+// CHECK:  encoding: [0xc4,0xc2,0xf1,0xb9,0xe6]
+          vfmadd231sd %xmm14, %xmm1, %xmm4
+
+// CHECK: vfmadd231sd %xmm14, %xmm1, %xmm4 {%k1}
+// CHECK:  encoding: [0x62,0xd2,0xf5,0x09,0xb9,0xe6]
+          vfmadd231sd %xmm14, %xmm1, %xmm4 {%k1}
+
+// CHECK: vfmadd231sd %xmm14, %xmm1, %xmm4 {%k1} {z}
+// CHECK:  encoding: [0x62,0xd2,0xf5,0x89,0xb9,0xe6]
+          vfmadd231sd %xmm14, %xmm1, %xmm4 {%k1} {z}
+
+// CHECK: vfmadd231sd {rn-sae}, %xmm14, %xmm1, %xmm4
+// CHECK:  encoding: [0x62,0xd2,0xf5,0x18,0xb9,0xe6]
+          vfmadd231sd {rn-sae}, %xmm14, %xmm1, %xmm4
+
+// CHECK: vfmadd231sd {ru-sae}, %xmm14, %xmm1, %xmm4
+// CHECK:  encoding: [0x62,0xd2,0xf5,0x58,0xb9,0xe6]
+          vfmadd231sd {ru-sae}, %xmm14, %xmm1, %xmm4
+
+// CHECK: vfmadd231sd {rd-sae}, %xmm14, %xmm1, %xmm4
+// CHECK:  encoding: [0x62,0xd2,0xf5,0x38,0xb9,0xe6]
+          vfmadd231sd {rd-sae}, %xmm14, %xmm1, %xmm4
+
+// CHECK: vfmadd231sd {rz-sae}, %xmm14, %xmm1, %xmm4
+// CHECK:  encoding: [0x62,0xd2,0xf5,0x78,0xb9,0xe6]
+          vfmadd231sd {rz-sae}, %xmm14, %xmm1, %xmm4
+
+// CHECK: vfmadd231sd   (%rcx), %xmm1, %xmm24 
+// CHECK: encoding: [0x62,0x62,0xf5,0x08,0xb9,0x01]
+          vfmadd231sd   (%rcx), %xmm1, %xmm24 
+
+// CHECK: vfmadd231sd   291(%rax,%r14,8), %xmm1, %xmm24 
+// CHECK: encoding: [0x62,0x22,0xf5,0x08,0xb9,0x84,0xf0,0x23,0x01,0x00,0x00]
+          vfmadd231sd   291(%rax,%r14,8), %xmm1, %xmm24 
+
+// CHECK: vfmadd231sd   1016(%rdx), %xmm1, %xmm24 
+// CHECK: encoding: [0x62,0x62,0xf5,0x08,0xb9,0x42,0x7f]
+          vfmadd231sd   1016(%rdx), %xmm1, %xmm24 
+
+// CHECK: vfmadd231sd   1024(%rdx), %xmm1, %xmm24 
+// CHECK: encoding: [0x62,0x62,0xf5,0x08,0xb9,0x82,0x00,0x04,0x00,0x00]
+          vfmadd231sd   1024(%rdx), %xmm1, %xmm24 
+
+// CHECK: vfmadd231sd   -1024(%rdx), %xmm1, %xmm24 
+// CHECK: encoding: [0x62,0x62,0xf5,0x08,0xb9,0x42,0x80]
+          vfmadd231sd   -1024(%rdx), %xmm1, %xmm24 
+
+// CHECK: vfmadd231sd   -1032(%rdx), %xmm1, %xmm24 
+// CHECK: encoding: [0x62,0x62,0xf5,0x08,0xb9,0x82,0xf8,0xfb,0xff,0xff]
+          vfmadd231sd   -1032(%rdx), %xmm1, %xmm24 
+
+// CHECK: vfmsub132ss %xmm27, %xmm22, %xmm30
+// CHECK:  encoding: [0x62,0x02,0x4d,0x00,0x9b,0xf3]
+          vfmsub132ss %xmm27, %xmm22, %xmm30
+
+// CHECK: vfmsub132ss %xmm27, %xmm22, %xmm30 {%k3}
+// CHECK:  encoding: [0x62,0x02,0x4d,0x03,0x9b,0xf3]
+          vfmsub132ss %xmm27, %xmm22, %xmm30 {%k3}
+
+// CHECK: vfmsub132ss %xmm27, %xmm22, %xmm30 {%k3} {z}
+// CHECK:  encoding: [0x62,0x02,0x4d,0x83,0x9b,0xf3]
+          vfmsub132ss %xmm27, %xmm22, %xmm30 {%k3} {z}
+
+// CHECK: vfmsub132ss {rn-sae}, %xmm27, %xmm22, %xmm30
+// CHECK:  encoding: [0x62,0x02,0x4d,0x10,0x9b,0xf3]
+          vfmsub132ss {rn-sae}, %xmm27, %xmm22, %xmm30
+
+// CHECK: vfmsub132ss {ru-sae}, %xmm27, %xmm22, %xmm30
+// CHECK:  encoding: [0x62,0x02,0x4d,0x50,0x9b,0xf3]
+          vfmsub132ss {ru-sae}, %xmm27, %xmm22, %xmm30
+
+// CHECK: vfmsub132ss {rd-sae}, %xmm27, %xmm22, %xmm30
+// CHECK:  encoding: [0x62,0x02,0x4d,0x30,0x9b,0xf3]
+          vfmsub132ss {rd-sae}, %xmm27, %xmm22, %xmm30
+
+// CHECK: vfmsub132ss {rz-sae}, %xmm27, %xmm22, %xmm30
+// CHECK:  encoding: [0x62,0x02,0x4d,0x70,0x9b,0xf3]
+          vfmsub132ss {rz-sae}, %xmm27, %xmm22, %xmm30
+
+// CHECK: vfmsub132ss (%rcx), %xmm22, %xmm30
+// CHECK:  encoding: [0x62,0x62,0x4d,0x00,0x9b,0x31]
+          vfmsub132ss (%rcx), %xmm22, %xmm30
+
+// CHECK: vfmsub132ss 291(%rax,%r14,8), %xmm22, %xmm30
+// CHECK:  encoding: [0x62,0x22,0x4d,0x00,0x9b,0xb4,0xf0,0x23,0x01,0x00,0x00]
+          vfmsub132ss 291(%rax,%r14,8), %xmm22, %xmm30
+
+// CHECK: vfmsub132ss 508(%rdx), %xmm22, %xmm30
+// CHECK:  encoding: [0x62,0x62,0x4d,0x00,0x9b,0x72,0x7f]
+          vfmsub132ss 508(%rdx), %xmm22, %xmm30
+
+// CHECK: vfmsub132ss 512(%rdx), %xmm22, %xmm30
+// CHECK:  encoding: [0x62,0x62,0x4d,0x00,0x9b,0xb2,0x00,0x02,0x00,0x00]
+          vfmsub132ss 512(%rdx), %xmm22, %xmm30
+
+// CHECK: vfmsub132ss -512(%rdx), %xmm22, %xmm30
+// CHECK:  encoding: [0x62,0x62,0x4d,0x00,0x9b,0x72,0x80]
+          vfmsub132ss -512(%rdx), %xmm22, %xmm30
+
+// CHECK: vfmsub132ss -516(%rdx), %xmm22, %xmm30
+// CHECK:  encoding: [0x62,0x62,0x4d,0x00,0x9b,0xb2,0xfc,0xfd,0xff,0xff]
+          vfmsub132ss -516(%rdx), %xmm22, %xmm30
+
+// CHECK: vfmsub132sd %xmm27, %xmm8, %xmm12
+// CHECK:  encoding: [0x62,0x12,0xbd,0x08,0x9b,0xe3]
+          vfmsub132sd %xmm27, %xmm8, %xmm12
+
+// CHECK: vfmsub132sd %xmm27, %xmm8, %xmm12 {%k3}
+// CHECK:  encoding: [0x62,0x12,0xbd,0x0b,0x9b,0xe3]
+          vfmsub132sd %xmm27, %xmm8, %xmm12 {%k3}
+
+// CHECK: vfmsub132sd %xmm27, %xmm8, %xmm12 {%k3} {z}
+// CHECK:  encoding: [0x62,0x12,0xbd,0x8b,0x9b,0xe3]
+          vfmsub132sd %xmm27, %xmm8, %xmm12 {%k3} {z}
+
+// CHECK: vfmsub132sd {rn-sae}, %xmm27, %xmm8, %xmm12
+// CHECK:  encoding: [0x62,0x12,0xbd,0x18,0x9b,0xe3]
+          vfmsub132sd {rn-sae}, %xmm27, %xmm8, %xmm12
+
+// CHECK: vfmsub132sd {ru-sae}, %xmm27, %xmm8, %xmm12
+// CHECK:  encoding: [0x62,0x12,0xbd,0x58,0x9b,0xe3]
+          vfmsub132sd {ru-sae}, %xmm27, %xmm8, %xmm12
+
+// CHECK: vfmsub132sd {rd-sae}, %xmm27, %xmm8, %xmm12
+// CHECK:  encoding: [0x62,0x12,0xbd,0x38,0x9b,0xe3]
+          vfmsub132sd {rd-sae}, %xmm27, %xmm8, %xmm12
+
+// CHECK: vfmsub132sd {rz-sae}, %xmm27, %xmm8, %xmm12
+// CHECK:  encoding: [0x62,0x12,0xbd,0x78,0x9b,0xe3]
+          vfmsub132sd {rz-sae}, %xmm27, %xmm8, %xmm12
+
+// CHECK: vfmsub132sd   (%rcx), %xmm8, %xmm22 
+// CHECK: encoding: [0x62,0xe2,0xbd,0x08,0x9b,0x31]
+          vfmsub132sd   (%rcx), %xmm8, %xmm22 
+
+// CHECK: vfmsub132sd   291(%rax,%r14,8), %xmm8, %xmm22 
+// CHECK: encoding: [0x62,0xa2,0xbd,0x08,0x9b,0xb4,0xf0,0x23,0x01,0x00,0x00]
+          vfmsub132sd   291(%rax,%r14,8), %xmm8, %xmm22 
+
+// CHECK: vfmsub132sd   1016(%rdx), %xmm8, %xmm22 
+// CHECK: encoding: [0x62,0xe2,0xbd,0x08,0x9b,0x72,0x7f]
+          vfmsub132sd   1016(%rdx), %xmm8, %xmm22 
+
+// CHECK: vfmsub132sd   1024(%rdx), %xmm8, %xmm22 
+// CHECK: encoding: [0x62,0xe2,0xbd,0x08,0x9b,0xb2,0x00,0x04,0x00,0x00]
+          vfmsub132sd   1024(%rdx), %xmm8, %xmm22 
+
+// CHECK: vfmsub132sd   -1024(%rdx), %xmm8, %xmm22 
+// CHECK: encoding: [0x62,0xe2,0xbd,0x08,0x9b,0x72,0x80]
+          vfmsub132sd   -1024(%rdx), %xmm8, %xmm22 
+
+// CHECK: vfmsub132sd   -1032(%rdx), %xmm8, %xmm22 
+// CHECK: encoding: [0x62,0xe2,0xbd,0x08,0x9b,0xb2,0xf8,0xfb,0xff,0xff]
+          vfmsub132sd   -1032(%rdx), %xmm8, %xmm22 
+
+// CHECK: vfmsub213ss %xmm26, %xmm13, %xmm30
+// CHECK:  encoding: [0x62,0x02,0x15,0x08,0xab,0xf2]
+          vfmsub213ss %xmm26, %xmm13, %xmm30
+
+// CHECK: vfmsub213ss %xmm26, %xmm13, %xmm30 {%k1}
+// CHECK:  encoding: [0x62,0x02,0x15,0x09,0xab,0xf2]
+          vfmsub213ss %xmm26, %xmm13, %xmm30 {%k1}
+
+// CHECK: vfmsub213ss %xmm26, %xmm13, %xmm30 {%k1} {z}
+// CHECK:  encoding: [0x62,0x02,0x15,0x89,0xab,0xf2]
+          vfmsub213ss %xmm26, %xmm13, %xmm30 {%k1} {z}
+
+// CHECK: vfmsub213ss {rn-sae}, %xmm26, %xmm13, %xmm30
+// CHECK:  encoding: [0x62,0x02,0x15,0x18,0xab,0xf2]
+          vfmsub213ss {rn-sae}, %xmm26, %xmm13, %xmm30
+
+// CHECK: vfmsub213ss {ru-sae}, %xmm26, %xmm13, %xmm30
+// CHECK:  encoding: [0x62,0x02,0x15,0x58,0xab,0xf2]
+          vfmsub213ss {ru-sae}, %xmm26, %xmm13, %xmm30
+
+// CHECK: vfmsub213ss {rd-sae}, %xmm26, %xmm13, %xmm30
+// CHECK:  encoding: [0x62,0x02,0x15,0x38,0xab,0xf2]
+          vfmsub213ss {rd-sae}, %xmm26, %xmm13, %xmm30
+
+// CHECK: vfmsub213ss {rz-sae}, %xmm26, %xmm13, %xmm30
+// CHECK:  encoding: [0x62,0x02,0x15,0x78,0xab,0xf2]
+          vfmsub213ss {rz-sae}, %xmm26, %xmm13, %xmm30
+
+// CHECK: vfmsub213ss (%rcx), %xmm13, %xmm30
+// CHECK:  encoding: [0x62,0x62,0x15,0x08,0xab,0x31]
+          vfmsub213ss (%rcx), %xmm13, %xmm30
+
+// CHECK: vfmsub213ss 291(%rax,%r14,8), %xmm13, %xmm30
+// CHECK:  encoding: [0x62,0x22,0x15,0x08,0xab,0xb4,0xf0,0x23,0x01,0x00,0x00]
+          vfmsub213ss 291(%rax,%r14,8), %xmm13, %xmm30
+
+// CHECK: vfmsub213ss 508(%rdx), %xmm13, %xmm30
+// CHECK:  encoding: [0x62,0x62,0x15,0x08,0xab,0x72,0x7f]
+          vfmsub213ss 508(%rdx), %xmm13, %xmm30
+
+// CHECK: vfmsub213ss 512(%rdx), %xmm13, %xmm30
+// CHECK:  encoding: [0x62,0x62,0x15,0x08,0xab,0xb2,0x00,0x02,0x00,0x00]
+          vfmsub213ss 512(%rdx), %xmm13, %xmm30
+
+// CHECK: vfmsub213ss -512(%rdx), %xmm13, %xmm30
+// CHECK:  encoding: [0x62,0x62,0x15,0x08,0xab,0x72,0x80]
+          vfmsub213ss -512(%rdx), %xmm13, %xmm30
+
+// CHECK: vfmsub213ss -516(%rdx), %xmm13, %xmm30
+// CHECK:  encoding: [0x62,0x62,0x15,0x08,0xab,0xb2,0xfc,0xfd,0xff,0xff]
+          vfmsub213ss -516(%rdx), %xmm13, %xmm30
+
+// CHECK: vfmsub213sd   %xmm12, %xmm8, %xmm26 
+// CHECK: encoding: [0x62,0x42,0xbd,0x08,0xab,0xd4]
+          vfmsub213sd   %xmm12, %xmm8, %xmm26 
+
+// CHECK: vfmsub213sd %xmm12, %xmm8, %xmm6 {%k1}
+// CHECK:  encoding: [0x62,0xd2,0xbd,0x09,0xab,0xf4]
+          vfmsub213sd %xmm12, %xmm8, %xmm6 {%k1}
+
+// CHECK: vfmsub213sd %xmm12, %xmm8, %xmm6 {%k1} {z}
+// CHECK:  encoding: [0x62,0xd2,0xbd,0x89,0xab,0xf4]
+          vfmsub213sd %xmm12, %xmm8, %xmm6 {%k1} {z}
+
+// CHECK: vfmsub213sd {rn-sae}, %xmm12, %xmm8, %xmm6
+// CHECK:  encoding: [0x62,0xd2,0xbd,0x18,0xab,0xf4]
+          vfmsub213sd {rn-sae}, %xmm12, %xmm8, %xmm6
+
+// CHECK: vfmsub213sd {ru-sae}, %xmm12, %xmm8, %xmm6
+// CHECK:  encoding: [0x62,0xd2,0xbd,0x58,0xab,0xf4]
+          vfmsub213sd {ru-sae}, %xmm12, %xmm8, %xmm6
+
+// CHECK: vfmsub213sd {rd-sae}, %xmm12, %xmm8, %xmm6
+// CHECK:  encoding: [0x62,0xd2,0xbd,0x38,0xab,0xf4]
+          vfmsub213sd {rd-sae}, %xmm12, %xmm8, %xmm6
+
+// CHECK: vfmsub213sd {rz-sae}, %xmm12, %xmm8, %xmm6
+// CHECK:  encoding: [0x62,0xd2,0xbd,0x78,0xab,0xf4]
+          vfmsub213sd {rz-sae}, %xmm12, %xmm8, %xmm6
+
+// CHECK: vfmsub213sd   (%rcx), %xmm8, %xmm26 
+// CHECK: encoding: [0x62,0x62,0xbd,0x08,0xab,0x11]
+          vfmsub213sd   (%rcx), %xmm8, %xmm26 
+
+// CHECK: vfmsub213sd   291(%rax,%r14,8), %xmm8, %xmm26 
+// CHECK: encoding: [0x62,0x22,0xbd,0x08,0xab,0x94,0xf0,0x23,0x01,0x00,0x00]
+          vfmsub213sd   291(%rax,%r14,8), %xmm8, %xmm26 
+
+// CHECK: vfmsub213sd   1016(%rdx), %xmm8, %xmm26 
+// CHECK: encoding: [0x62,0x62,0xbd,0x08,0xab,0x52,0x7f]
+          vfmsub213sd   1016(%rdx), %xmm8, %xmm26 
+
+// CHECK: vfmsub213sd   1024(%rdx), %xmm8, %xmm26 
+// CHECK: encoding: [0x62,0x62,0xbd,0x08,0xab,0x92,0x00,0x04,0x00,0x00]
+          vfmsub213sd   1024(%rdx), %xmm8, %xmm26 
+
+// CHECK: vfmsub213sd   -1024(%rdx), %xmm8, %xmm26 
+// CHECK: encoding: [0x62,0x62,0xbd,0x08,0xab,0x52,0x80]
+          vfmsub213sd   -1024(%rdx), %xmm8, %xmm26 
+
+// CHECK: vfmsub213sd   -1032(%rdx), %xmm8, %xmm26 
+// CHECK: encoding: [0x62,0x62,0xbd,0x08,0xab,0x92,0xf8,0xfb,0xff,0xff]
+          vfmsub213sd   -1032(%rdx), %xmm8, %xmm26 
+
+// CHECK: vfmsub231ss %xmm5, %xmm3, %xmm29
+// CHECK:  encoding: [0x62,0x62,0x65,0x08,0xbb,0xed]
+          vfmsub231ss %xmm5, %xmm3, %xmm29
+
+// CHECK: vfmsub231ss %xmm5, %xmm3, %xmm29 {%k6}
+// CHECK:  encoding: [0x62,0x62,0x65,0x0e,0xbb,0xed]
+          vfmsub231ss %xmm5, %xmm3, %xmm29 {%k6}
+
+// CHECK: vfmsub231ss %xmm5, %xmm3, %xmm29 {%k6} {z}
+// CHECK:  encoding: [0x62,0x62,0x65,0x8e,0xbb,0xed]
+          vfmsub231ss %xmm5, %xmm3, %xmm29 {%k6} {z}
+
+// CHECK: vfmsub231ss {rn-sae}, %xmm5, %xmm3, %xmm29
+// CHECK:  encoding: [0x62,0x62,0x65,0x18,0xbb,0xed]
+          vfmsub231ss {rn-sae}, %xmm5, %xmm3, %xmm29
+
+// CHECK: vfmsub231ss {ru-sae}, %xmm5, %xmm3, %xmm29
+// CHECK:  encoding: [0x62,0x62,0x65,0x58,0xbb,0xed]
+          vfmsub231ss {ru-sae}, %xmm5, %xmm3, %xmm29
+
+// CHECK: vfmsub231ss {rd-sae}, %xmm5, %xmm3, %xmm29
+// CHECK:  encoding: [0x62,0x62,0x65,0x38,0xbb,0xed]
+          vfmsub231ss {rd-sae}, %xmm5, %xmm3, %xmm29
+
+// CHECK: vfmsub231ss {rz-sae}, %xmm5, %xmm3, %xmm29
+// CHECK:  encoding: [0x62,0x62,0x65,0x78,0xbb,0xed]
+          vfmsub231ss {rz-sae}, %xmm5, %xmm3, %xmm29
+
+// CHECK: vfmsub231ss (%rcx), %xmm3, %xmm29
+// CHECK:  encoding: [0x62,0x62,0x65,0x08,0xbb,0x29]
+          vfmsub231ss (%rcx), %xmm3, %xmm29
+
+// CHECK: vfmsub231ss 291(%rax,%r14,8), %xmm3, %xmm29
+// CHECK:  encoding: [0x62,0x22,0x65,0x08,0xbb,0xac,0xf0,0x23,0x01,0x00,0x00]
+          vfmsub231ss 291(%rax,%r14,8), %xmm3, %xmm29
+
+// CHECK: vfmsub231ss 508(%rdx), %xmm3, %xmm29
+// CHECK:  encoding: [0x62,0x62,0x65,0x08,0xbb,0x6a,0x7f]
+          vfmsub231ss 508(%rdx), %xmm3, %xmm29
+
+// CHECK: vfmsub231ss 512(%rdx), %xmm3, %xmm29
+// CHECK:  encoding: [0x62,0x62,0x65,0x08,0xbb,0xaa,0x00,0x02,0x00,0x00]
+          vfmsub231ss 512(%rdx), %xmm3, %xmm29
+
+// CHECK: vfmsub231ss -512(%rdx), %xmm3, %xmm29
+// CHECK:  encoding: [0x62,0x62,0x65,0x08,0xbb,0x6a,0x80]
+          vfmsub231ss -512(%rdx), %xmm3, %xmm29
+
+// CHECK: vfmsub231ss -516(%rdx), %xmm3, %xmm29
+// CHECK:  encoding: [0x62,0x62,0x65,0x08,0xbb,0xaa,0xfc,0xfd,0xff,0xff]
+          vfmsub231ss -516(%rdx), %xmm3, %xmm29
+
+// CHECK: vfmsub231sd %xmm6, %xmm11, %xmm3
+// CHECK:  encoding: [0xc4,0xe2,0xa1,0xbb,0xde]
+          vfmsub231sd %xmm6, %xmm11, %xmm3
+
+// CHECK: vfmsub231sd %xmm6, %xmm11, %xmm3 {%k7}
+// CHECK:  encoding: [0x62,0xf2,0xa5,0x0f,0xbb,0xde]
+          vfmsub231sd %xmm6, %xmm11, %xmm3 {%k7}
+
+// CHECK: vfmsub231sd %xmm6, %xmm11, %xmm3 {%k7} {z}
+// CHECK:  encoding: [0x62,0xf2,0xa5,0x8f,0xbb,0xde]
+          vfmsub231sd %xmm6, %xmm11, %xmm3 {%k7} {z}
+
+// CHECK: vfmsub231sd {rn-sae}, %xmm6, %xmm11, %xmm3
+// CHECK:  encoding: [0x62,0xf2,0xa5,0x18,0xbb,0xde]
+          vfmsub231sd {rn-sae}, %xmm6, %xmm11, %xmm3
+
+// CHECK: vfmsub231sd {ru-sae}, %xmm6, %xmm11, %xmm3
+// CHECK:  encoding: [0x62,0xf2,0xa5,0x58,0xbb,0xde]
+          vfmsub231sd {ru-sae}, %xmm6, %xmm11, %xmm3
+
+// CHECK: vfmsub231sd {rd-sae}, %xmm6, %xmm11, %xmm3
+// CHECK:  encoding: [0x62,0xf2,0xa5,0x38,0xbb,0xde]
+          vfmsub231sd {rd-sae}, %xmm6, %xmm11, %xmm3
+
+// CHECK: vfmsub231sd {rz-sae}, %xmm6, %xmm11, %xmm3
+// CHECK:  encoding: [0x62,0xf2,0xa5,0x78,0xbb,0xde]
+          vfmsub231sd {rz-sae}, %xmm6, %xmm11, %xmm3
+
+// CHECK: vfmsub231sd   (%rcx), %xmm11, %xmm23 
+// CHECK: encoding: [0x62,0xe2,0xa5,0x08,0xbb,0x39]
+          vfmsub231sd   (%rcx), %xmm11, %xmm23 
+
+// CHECK: vfmsub231sd   291(%rax,%r14,8), %xmm11, %xmm23 
+// CHECK: encoding: [0x62,0xa2,0xa5,0x08,0xbb,0xbc,0xf0,0x23,0x01,0x00,0x00]
+          vfmsub231sd   291(%rax,%r14,8), %xmm11, %xmm23 
+
+// CHECK: vfmsub231sd   1016(%rdx), %xmm11, %xmm23 
+// CHECK: encoding: [0x62,0xe2,0xa5,0x08,0xbb,0x7a,0x7f]
+          vfmsub231sd   1016(%rdx), %xmm11, %xmm23 
+
+// CHECK: vfmsub231sd   1024(%rdx), %xmm11, %xmm23 
+// CHECK: encoding: [0x62,0xe2,0xa5,0x08,0xbb,0xba,0x00,0x04,0x00,0x00]
+          vfmsub231sd   1024(%rdx), %xmm11, %xmm23 
+
+// CHECK: vfmsub231sd   -1024(%rdx), %xmm11, %xmm23 
+// CHECK: encoding: [0x62,0xe2,0xa5,0x08,0xbb,0x7a,0x80]
+          vfmsub231sd   -1024(%rdx), %xmm11, %xmm23 
+
+// CHECK: vfmsub231sd   -1032(%rdx), %xmm11, %xmm23 
+// CHECK: encoding: [0x62,0xe2,0xa5,0x08,0xbb,0xba,0xf8,0xfb,0xff,0xff]
+          vfmsub231sd   -1032(%rdx), %xmm11, %xmm23 
+
+// CHECK: vfnmadd132ss %xmm23, %xmm1, %xmm25
+// CHECK:  encoding: [0x62,0x22,0x75,0x08,0x9d,0xcf]
+          vfnmadd132ss %xmm23, %xmm1, %xmm25
+
+// CHECK: vfnmadd132ss %xmm23, %xmm1, %xmm25 {%k3}
+// CHECK:  encoding: [0x62,0x22,0x75,0x0b,0x9d,0xcf]
+          vfnmadd132ss %xmm23, %xmm1, %xmm25 {%k3}
+
+// CHECK: vfnmadd132ss %xmm23, %xmm1, %xmm25 {%k3} {z}
+// CHECK:  encoding: [0x62,0x22,0x75,0x8b,0x9d,0xcf]
+          vfnmadd132ss %xmm23, %xmm1, %xmm25 {%k3} {z}
+
+// CHECK: vfnmadd132ss {rn-sae}, %xmm23, %xmm1, %xmm25
+// CHECK:  encoding: [0x62,0x22,0x75,0x18,0x9d,0xcf]
+          vfnmadd132ss {rn-sae}, %xmm23, %xmm1, %xmm25
+
+// CHECK: vfnmadd132ss {ru-sae}, %xmm23, %xmm1, %xmm25
+// CHECK:  encoding: [0x62,0x22,0x75,0x58,0x9d,0xcf]
+          vfnmadd132ss {ru-sae}, %xmm23, %xmm1, %xmm25
+
+// CHECK: vfnmadd132ss {rd-sae}, %xmm23, %xmm1, %xmm25
+// CHECK:  encoding: [0x62,0x22,0x75,0x38,0x9d,0xcf]
+          vfnmadd132ss {rd-sae}, %xmm23, %xmm1, %xmm25
+
+// CHECK: vfnmadd132ss {rz-sae}, %xmm23, %xmm1, %xmm25
+// CHECK:  encoding: [0x62,0x22,0x75,0x78,0x9d,0xcf]
+          vfnmadd132ss {rz-sae}, %xmm23, %xmm1, %xmm25
+
+// CHECK: vfnmadd132ss (%rcx), %xmm1, %xmm25
+// CHECK:  encoding: [0x62,0x62,0x75,0x08,0x9d,0x09]
+          vfnmadd132ss (%rcx), %xmm1, %xmm25
+
+// CHECK: vfnmadd132ss 291(%rax,%r14,8), %xmm1, %xmm25
+// CHECK:  encoding: [0x62,0x22,0x75,0x08,0x9d,0x8c,0xf0,0x23,0x01,0x00,0x00]
+          vfnmadd132ss 291(%rax,%r14,8), %xmm1, %xmm25
+
+// CHECK: vfnmadd132ss 508(%rdx), %xmm1, %xmm25
+// CHECK:  encoding: [0x62,0x62,0x75,0x08,0x9d,0x4a,0x7f]
+          vfnmadd132ss 508(%rdx), %xmm1, %xmm25
+
+// CHECK: vfnmadd132ss 512(%rdx), %xmm1, %xmm25
+// CHECK:  encoding: [0x62,0x62,0x75,0x08,0x9d,0x8a,0x00,0x02,0x00,0x00]
+          vfnmadd132ss 512(%rdx), %xmm1, %xmm25
+
+// CHECK: vfnmadd132ss -512(%rdx), %xmm1, %xmm25
+// CHECK:  encoding: [0x62,0x62,0x75,0x08,0x9d,0x4a,0x80]
+          vfnmadd132ss -512(%rdx), %xmm1, %xmm25
+
+// CHECK: vfnmadd132ss -516(%rdx), %xmm1, %xmm25
+// CHECK:  encoding: [0x62,0x62,0x75,0x08,0x9d,0x8a,0xfc,0xfd,0xff,0xff]
+          vfnmadd132ss -516(%rdx), %xmm1, %xmm25
+
+// CHECK: vfnmadd132sd %xmm11, %xmm3, %xmm19
+// CHECK:  encoding: [0x62,0xc2,0xe5,0x08,0x9d,0xdb]
+          vfnmadd132sd %xmm11, %xmm3, %xmm19
+
+// CHECK: vfnmadd132sd %xmm11, %xmm3, %xmm19 {%k2}
+// CHECK:  encoding: [0x62,0xc2,0xe5,0x0a,0x9d,0xdb]
+          vfnmadd132sd %xmm11, %xmm3, %xmm19 {%k2}
+
+// CHECK: vfnmadd132sd %xmm11, %xmm3, %xmm19 {%k2} {z}
+// CHECK:  encoding: [0x62,0xc2,0xe5,0x8a,0x9d,0xdb]
+          vfnmadd132sd %xmm11, %xmm3, %xmm19 {%k2} {z}
+
+// CHECK: vfnmadd132sd {rn-sae}, %xmm11, %xmm3, %xmm19
+// CHECK:  encoding: [0x62,0xc2,0xe5,0x18,0x9d,0xdb]
+          vfnmadd132sd {rn-sae}, %xmm11, %xmm3, %xmm19
+
+// CHECK: vfnmadd132sd {ru-sae}, %xmm11, %xmm3, %xmm19
+// CHECK:  encoding: [0x62,0xc2,0xe5,0x58,0x9d,0xdb]
+          vfnmadd132sd {ru-sae}, %xmm11, %xmm3, %xmm19
+
+// CHECK: vfnmadd132sd {rd-sae}, %xmm11, %xmm3, %xmm19
+// CHECK:  encoding: [0x62,0xc2,0xe5,0x38,0x9d,0xdb]
+          vfnmadd132sd {rd-sae}, %xmm11, %xmm3, %xmm19
+
+// CHECK: vfnmadd132sd {rz-sae}, %xmm11, %xmm3, %xmm19
+// CHECK:  encoding: [0x62,0xc2,0xe5,0x78,0x9d,0xdb]
+          vfnmadd132sd {rz-sae}, %xmm11, %xmm3, %xmm19
+
+// CHECK: vfnmadd132sd (%rcx), %xmm3, %xmm19
+// CHECK:  encoding: [0x62,0xe2,0xe5,0x08,0x9d,0x19]
+          vfnmadd132sd (%rcx), %xmm3, %xmm19
+
+// CHECK: vfnmadd132sd 291(%rax,%r14,8), %xmm3, %xmm19
+// CHECK:  encoding: [0x62,0xa2,0xe5,0x08,0x9d,0x9c,0xf0,0x23,0x01,0x00,0x00]
+          vfnmadd132sd 291(%rax,%r14,8), %xmm3, %xmm19
+
+// CHECK: vfnmadd132sd 1016(%rdx), %xmm3, %xmm19
+// CHECK:  encoding: [0x62,0xe2,0xe5,0x08,0x9d,0x5a,0x7f]
+          vfnmadd132sd 1016(%rdx), %xmm3, %xmm19
+
+// CHECK: vfnmadd132sd 1024(%rdx), %xmm3, %xmm19
+// CHECK:  encoding: [0x62,0xe2,0xe5,0x08,0x9d,0x9a,0x00,0x04,0x00,0x00]
+          vfnmadd132sd 1024(%rdx), %xmm3, %xmm19
+
+// CHECK: vfnmadd132sd -1024(%rdx), %xmm3, %xmm19
+// CHECK:  encoding: [0x62,0xe2,0xe5,0x08,0x9d,0x5a,0x80]
+          vfnmadd132sd -1024(%rdx), %xmm3, %xmm19
+
+// CHECK: vfnmadd132sd -1032(%rdx), %xmm3, %xmm19
+// CHECK:  encoding: [0x62,0xe2,0xe5,0x08,0x9d,0x9a,0xf8,0xfb,0xff,0xff]
+          vfnmadd132sd -1032(%rdx), %xmm3, %xmm19
+
+// CHECK: vfnmadd213ss %xmm28, %xmm7, %xmm29
+// CHECK:  encoding: [0x62,0x02,0x45,0x08,0xad,0xec]
+          vfnmadd213ss %xmm28, %xmm7, %xmm29
+
+// CHECK: vfnmadd213ss %xmm28, %xmm7, %xmm29 {%k2}
+// CHECK:  encoding: [0x62,0x02,0x45,0x0a,0xad,0xec]
+          vfnmadd213ss %xmm28, %xmm7, %xmm29 {%k2}
+
+// CHECK: vfnmadd213ss %xmm28, %xmm7, %xmm29 {%k2} {z}
+// CHECK:  encoding: [0x62,0x02,0x45,0x8a,0xad,0xec]
+          vfnmadd213ss %xmm28, %xmm7, %xmm29 {%k2} {z}
+
+// CHECK: vfnmadd213ss {rn-sae}, %xmm28, %xmm7, %xmm29
+// CHECK:  encoding: [0x62,0x02,0x45,0x18,0xad,0xec]
+          vfnmadd213ss {rn-sae}, %xmm28, %xmm7, %xmm29
+
+// CHECK: vfnmadd213ss {ru-sae}, %xmm28, %xmm7, %xmm29
+// CHECK:  encoding: [0x62,0x02,0x45,0x58,0xad,0xec]
+          vfnmadd213ss {ru-sae}, %xmm28, %xmm7, %xmm29
+
+// CHECK: vfnmadd213ss {rd-sae}, %xmm28, %xmm7, %xmm29
+// CHECK:  encoding: [0x62,0x02,0x45,0x38,0xad,0xec]
+          vfnmadd213ss {rd-sae}, %xmm28, %xmm7, %xmm29
+
+// CHECK: vfnmadd213ss {rz-sae}, %xmm28, %xmm7, %xmm29
+// CHECK:  encoding: [0x62,0x02,0x45,0x78,0xad,0xec]
+          vfnmadd213ss {rz-sae}, %xmm28, %xmm7, %xmm29
+
+// CHECK: vfnmadd213ss (%rcx), %xmm7, %xmm29
+// CHECK:  encoding: [0x62,0x62,0x45,0x08,0xad,0x29]
+          vfnmadd213ss (%rcx), %xmm7, %xmm29
+
+// CHECK: vfnmadd213ss 291(%rax,%r14,8), %xmm7, %xmm29
+// CHECK:  encoding: [0x62,0x22,0x45,0x08,0xad,0xac,0xf0,0x23,0x01,0x00,0x00]
+          vfnmadd213ss 291(%rax,%r14,8), %xmm7, %xmm29
+
+// CHECK: vfnmadd213ss 508(%rdx), %xmm7, %xmm29
+// CHECK:  encoding: [0x62,0x62,0x45,0x08,0xad,0x6a,0x7f]
+          vfnmadd213ss 508(%rdx), %xmm7, %xmm29
+
+// CHECK: vfnmadd213ss 512(%rdx), %xmm7, %xmm29
+// CHECK:  encoding: [0x62,0x62,0x45,0x08,0xad,0xaa,0x00,0x02,0x00,0x00]
+          vfnmadd213ss 512(%rdx), %xmm7, %xmm29
+
+// CHECK: vfnmadd213ss -512(%rdx), %xmm7, %xmm29
+// CHECK:  encoding: [0x62,0x62,0x45,0x08,0xad,0x6a,0x80]
+          vfnmadd213ss -512(%rdx), %xmm7, %xmm29
+
+// CHECK: vfnmadd213ss -516(%rdx), %xmm7, %xmm29
+// CHECK:  encoding: [0x62,0x62,0x45,0x08,0xad,0xaa,0xfc,0xfd,0xff,0xff]
+          vfnmadd213ss -516(%rdx), %xmm7, %xmm29
+
+// CHECK: vfnmadd213sd %xmm2, %xmm27, %xmm13
+// CHECK:  encoding: [0x62,0x72,0xa5,0x00,0xad,0xea]
+          vfnmadd213sd %xmm2, %xmm27, %xmm13
+
+// CHECK: vfnmadd213sd %xmm2, %xmm27, %xmm13 {%k7}
+// CHECK:  encoding: [0x62,0x72,0xa5,0x07,0xad,0xea]
+          vfnmadd213sd %xmm2, %xmm27, %xmm13 {%k7}
+
+// CHECK: vfnmadd213sd %xmm2, %xmm27, %xmm13 {%k7} {z}
+// CHECK:  encoding: [0x62,0x72,0xa5,0x87,0xad,0xea]
+          vfnmadd213sd %xmm2, %xmm27, %xmm13 {%k7} {z}
+
+// CHECK: vfnmadd213sd {rn-sae}, %xmm2, %xmm27, %xmm13
+// CHECK:  encoding: [0x62,0x72,0xa5,0x10,0xad,0xea]
+          vfnmadd213sd {rn-sae}, %xmm2, %xmm27, %xmm13
+
+// CHECK: vfnmadd213sd {ru-sae}, %xmm2, %xmm27, %xmm13
+// CHECK:  encoding: [0x62,0x72,0xa5,0x50,0xad,0xea]
+          vfnmadd213sd {ru-sae}, %xmm2, %xmm27, %xmm13
+
+// CHECK: vfnmadd213sd {rd-sae}, %xmm2, %xmm27, %xmm13
+// CHECK:  encoding: [0x62,0x72,0xa5,0x30,0xad,0xea]
+          vfnmadd213sd {rd-sae}, %xmm2, %xmm27, %xmm13
+
+// CHECK: vfnmadd213sd {rz-sae}, %xmm2, %xmm27, %xmm13
+// CHECK:  encoding: [0x62,0x72,0xa5,0x70,0xad,0xea]
+          vfnmadd213sd {rz-sae}, %xmm2, %xmm27, %xmm13
+
+// CHECK: vfnmadd213sd (%rcx), %xmm27, %xmm13
+// CHECK:  encoding: [0x62,0x72,0xa5,0x00,0xad,0x29]
+          vfnmadd213sd (%rcx), %xmm27, %xmm13
+
+// CHECK: vfnmadd213sd 291(%rax,%r14,8), %xmm27, %xmm13
+// CHECK:  encoding: [0x62,0x32,0xa5,0x00,0xad,0xac,0xf0,0x23,0x01,0x00,0x00]
+          vfnmadd213sd 291(%rax,%r14,8), %xmm27, %xmm13
+
+// CHECK: vfnmadd213sd 1016(%rdx), %xmm27, %xmm13
+// CHECK:  encoding: [0x62,0x72,0xa5,0x00,0xad,0x6a,0x7f]
+          vfnmadd213sd 1016(%rdx), %xmm27, %xmm13
+
+// CHECK: vfnmadd213sd 1024(%rdx), %xmm27, %xmm13
+// CHECK:  encoding: [0x62,0x72,0xa5,0x00,0xad,0xaa,0x00,0x04,0x00,0x00]
+          vfnmadd213sd 1024(%rdx), %xmm27, %xmm13
+
+// CHECK: vfnmadd213sd -1024(%rdx), %xmm27, %xmm13
+// CHECK:  encoding: [0x62,0x72,0xa5,0x00,0xad,0x6a,0x80]
+          vfnmadd213sd -1024(%rdx), %xmm27, %xmm13
+
+// CHECK: vfnmadd213sd -1032(%rdx), %xmm27, %xmm13
+// CHECK:  encoding: [0x62,0x72,0xa5,0x00,0xad,0xaa,0xf8,0xfb,0xff,0xff]
+          vfnmadd213sd -1032(%rdx), %xmm27, %xmm13
+
+// CHECK: vfnmadd231ss %xmm17, %xmm2, %xmm27
+// CHECK:  encoding: [0x62,0x22,0x6d,0x08,0xbd,0xd9]
+          vfnmadd231ss %xmm17, %xmm2, %xmm27
+
+// CHECK: vfnmadd231ss %xmm17, %xmm2, %xmm27 {%k1}
+// CHECK:  encoding: [0x62,0x22,0x6d,0x09,0xbd,0xd9]
+          vfnmadd231ss %xmm17, %xmm2, %xmm27 {%k1}
+
+// CHECK: vfnmadd231ss %xmm17, %xmm2, %xmm27 {%k1} {z}
+// CHECK:  encoding: [0x62,0x22,0x6d,0x89,0xbd,0xd9]
+          vfnmadd231ss %xmm17, %xmm2, %xmm27 {%k1} {z}
+
+// CHECK: vfnmadd231ss {rn-sae}, %xmm17, %xmm2, %xmm27
+// CHECK:  encoding: [0x62,0x22,0x6d,0x18,0xbd,0xd9]
+          vfnmadd231ss {rn-sae}, %xmm17, %xmm2, %xmm27
+
+// CHECK: vfnmadd231ss {ru-sae}, %xmm17, %xmm2, %xmm27
+// CHECK:  encoding: [0x62,0x22,0x6d,0x58,0xbd,0xd9]
+          vfnmadd231ss {ru-sae}, %xmm17, %xmm2, %xmm27
+
+// CHECK: vfnmadd231ss {rd-sae}, %xmm17, %xmm2, %xmm27
+// CHECK:  encoding: [0x62,0x22,0x6d,0x38,0xbd,0xd9]
+          vfnmadd231ss {rd-sae}, %xmm17, %xmm2, %xmm27
+
+// CHECK: vfnmadd231ss {rz-sae}, %xmm17, %xmm2, %xmm27
+// CHECK:  encoding: [0x62,0x22,0x6d,0x78,0xbd,0xd9]
+          vfnmadd231ss {rz-sae}, %xmm17, %xmm2, %xmm27
+
+// CHECK: vfnmadd231ss (%rcx), %xmm2, %xmm27
+// CHECK:  encoding: [0x62,0x62,0x6d,0x08,0xbd,0x19]
+          vfnmadd231ss (%rcx), %xmm2, %xmm27
+
+// CHECK: vfnmadd231ss 291(%rax,%r14,8), %xmm2, %xmm27
+// CHECK:  encoding: [0x62,0x22,0x6d,0x08,0xbd,0x9c,0xf0,0x23,0x01,0x00,0x00]
+          vfnmadd231ss 291(%rax,%r14,8), %xmm2, %xmm27
+
+// CHECK: vfnmadd231ss 508(%rdx), %xmm2, %xmm27
+// CHECK:  encoding: [0x62,0x62,0x6d,0x08,0xbd,0x5a,0x7f]
+          vfnmadd231ss 508(%rdx), %xmm2, %xmm27
+
+// CHECK: vfnmadd231ss 512(%rdx), %xmm2, %xmm27
+// CHECK:  encoding: [0x62,0x62,0x6d,0x08,0xbd,0x9a,0x00,0x02,0x00,0x00]
+          vfnmadd231ss 512(%rdx), %xmm2, %xmm27
+
+// CHECK: vfnmadd231ss -512(%rdx), %xmm2, %xmm27
+// CHECK:  encoding: [0x62,0x62,0x6d,0x08,0xbd,0x5a,0x80]
+          vfnmadd231ss -512(%rdx), %xmm2, %xmm27
+
+// CHECK: vfnmadd231ss -516(%rdx), %xmm2, %xmm27
+// CHECK:  encoding: [0x62,0x62,0x6d,0x08,0xbd,0x9a,0xfc,0xfd,0xff,0xff]
+          vfnmadd231ss -516(%rdx), %xmm2, %xmm27
+
+// CHECK: vfnmadd231sd %xmm18, %xmm11, %xmm6
+// CHECK:  encoding: [0x62,0xb2,0xa5,0x08,0xbd,0xf2]
+          vfnmadd231sd %xmm18, %xmm11, %xmm6
+
+// CHECK: vfnmadd231sd %xmm18, %xmm11, %xmm6 {%k3}
+// CHECK:  encoding: [0x62,0xb2,0xa5,0x0b,0xbd,0xf2]
+          vfnmadd231sd %xmm18, %xmm11, %xmm6 {%k3}
+
+// CHECK: vfnmadd231sd %xmm18, %xmm11, %xmm6 {%k3} {z}
+// CHECK:  encoding: [0x62,0xb2,0xa5,0x8b,0xbd,0xf2]
+          vfnmadd231sd %xmm18, %xmm11, %xmm6 {%k3} {z}
+
+// CHECK: vfnmadd231sd {rn-sae}, %xmm18, %xmm11, %xmm6
+// CHECK:  encoding: [0x62,0xb2,0xa5,0x18,0xbd,0xf2]
+          vfnmadd231sd {rn-sae}, %xmm18, %xmm11, %xmm6
+
+// CHECK: vfnmadd231sd {ru-sae}, %xmm18, %xmm11, %xmm6
+// CHECK:  encoding: [0x62,0xb2,0xa5,0x58,0xbd,0xf2]
+          vfnmadd231sd {ru-sae}, %xmm18, %xmm11, %xmm6
+
+// CHECK: vfnmadd231sd {rd-sae}, %xmm18, %xmm11, %xmm6
+// CHECK:  encoding: [0x62,0xb2,0xa5,0x38,0xbd,0xf2]
+          vfnmadd231sd {rd-sae}, %xmm18, %xmm11, %xmm6
+
+// CHECK: vfnmadd231sd {rz-sae}, %xmm18, %xmm11, %xmm6
+// CHECK:  encoding: [0x62,0xb2,0xa5,0x78,0xbd,0xf2]
+          vfnmadd231sd {rz-sae}, %xmm18, %xmm11, %xmm6
+
+// CHECK: vfnmadd231sd  (%rcx), %xmm11, %xmm26 
+// CHECK: encoding: [0x62,0x62,0xa5,0x08,0xbd,0x11]
+          vfnmadd231sd  (%rcx), %xmm11, %xmm26 
+
+// CHECK: vfnmadd231sd  291(%rax,%r14,8), %xmm11, %xmm26 
+// CHECK: encoding: [0x62,0x22,0xa5,0x08,0xbd,0x94,0xf0,0x23,0x01,0x00,0x00]
+          vfnmadd231sd  291(%rax,%r14,8), %xmm11, %xmm26 
+
+// CHECK: vfnmadd231sd  1016(%rdx), %xmm11, %xmm26 
+// CHECK: encoding: [0x62,0x62,0xa5,0x08,0xbd,0x52,0x7f]
+          vfnmadd231sd  1016(%rdx), %xmm11, %xmm26 
+
+// CHECK: vfnmadd231sd  1024(%rdx), %xmm11, %xmm26 
+// CHECK: encoding: [0x62,0x62,0xa5,0x08,0xbd,0x92,0x00,0x04,0x00,0x00]
+          vfnmadd231sd  1024(%rdx), %xmm11, %xmm26 
+
+// CHECK: vfnmadd231sd  -1024(%rdx), %xmm11, %xmm26 
+// CHECK: encoding: [0x62,0x62,0xa5,0x08,0xbd,0x52,0x80]
+          vfnmadd231sd  -1024(%rdx), %xmm11, %xmm26 
+
+// CHECK: vfnmadd231sd  -1032(%rdx), %xmm11, %xmm26 
+// CHECK: encoding: [0x62,0x62,0xa5,0x08,0xbd,0x92,0xf8,0xfb,0xff,0xff]
+          vfnmadd231sd  -1032(%rdx), %xmm11, %xmm26 
+
+// CHECK: vfnmsub132ss %xmm24, %xmm22, %xmm23
+// CHECK:  encoding: [0x62,0x82,0x4d,0x00,0x9f,0xf8]
+          vfnmsub132ss %xmm24, %xmm22, %xmm23
+
+// CHECK: vfnmsub132ss %xmm24, %xmm22, %xmm23 {%k6}
+// CHECK:  encoding: [0x62,0x82,0x4d,0x06,0x9f,0xf8]
+          vfnmsub132ss %xmm24, %xmm22, %xmm23 {%k6}
+
+// CHECK: vfnmsub132ss %xmm24, %xmm22, %xmm23 {%k6} {z}
+// CHECK:  encoding: [0x62,0x82,0x4d,0x86,0x9f,0xf8]
+          vfnmsub132ss %xmm24, %xmm22, %xmm23 {%k6} {z}
+
+// CHECK: vfnmsub132ss {rn-sae}, %xmm24, %xmm22, %xmm23
+// CHECK:  encoding: [0x62,0x82,0x4d,0x10,0x9f,0xf8]
+          vfnmsub132ss {rn-sae}, %xmm24, %xmm22, %xmm23
+
+// CHECK: vfnmsub132ss {ru-sae}, %xmm24, %xmm22, %xmm23
+// CHECK:  encoding: [0x62,0x82,0x4d,0x50,0x9f,0xf8]
+          vfnmsub132ss {ru-sae}, %xmm24, %xmm22, %xmm23
+
+// CHECK: vfnmsub132ss {rd-sae}, %xmm24, %xmm22, %xmm23
+// CHECK:  encoding: [0x62,0x82,0x4d,0x30,0x9f,0xf8]
+          vfnmsub132ss {rd-sae}, %xmm24, %xmm22, %xmm23
+
+// CHECK: vfnmsub132ss {rz-sae}, %xmm24, %xmm22, %xmm23
+// CHECK:  encoding: [0x62,0x82,0x4d,0x70,0x9f,0xf8]
+          vfnmsub132ss {rz-sae}, %xmm24, %xmm22, %xmm23
+
+// CHECK: vfnmsub132ss (%rcx), %xmm22, %xmm23
+// CHECK:  encoding: [0x62,0xe2,0x4d,0x00,0x9f,0x39]
+          vfnmsub132ss (%rcx), %xmm22, %xmm23
+
+// CHECK: vfnmsub132ss 291(%rax,%r14,8), %xmm22, %xmm23
+// CHECK:  encoding: [0x62,0xa2,0x4d,0x00,0x9f,0xbc,0xf0,0x23,0x01,0x00,0x00]
+          vfnmsub132ss 291(%rax,%r14,8), %xmm22, %xmm23
+
+// CHECK: vfnmsub132ss 508(%rdx), %xmm22, %xmm23
+// CHECK:  encoding: [0x62,0xe2,0x4d,0x00,0x9f,0x7a,0x7f]
+          vfnmsub132ss 508(%rdx), %xmm22, %xmm23
+
+// CHECK: vfnmsub132ss 512(%rdx), %xmm22, %xmm23
+// CHECK:  encoding: [0x62,0xe2,0x4d,0x00,0x9f,0xba,0x00,0x02,0x00,0x00]
+          vfnmsub132ss 512(%rdx), %xmm22, %xmm23
+
+// CHECK: vfnmsub132ss -512(%rdx), %xmm22, %xmm23
+// CHECK:  encoding: [0x62,0xe2,0x4d,0x00,0x9f,0x7a,0x80]
+          vfnmsub132ss -512(%rdx), %xmm22, %xmm23
+
+// CHECK: vfnmsub132ss -516(%rdx), %xmm22, %xmm23
+// CHECK:  encoding: [0x62,0xe2,0x4d,0x00,0x9f,0xba,0xfc,0xfd,0xff,0xff]
+          vfnmsub132ss -516(%rdx), %xmm22, %xmm23
+
+// CHECK: vfnmsub132sd %xmm13, %xmm11, %xmm26
+// CHECK:  encoding: [0x62,0x42,0xa5,0x08,0x9f,0xd5]
+          vfnmsub132sd %xmm13, %xmm11, %xmm26
+
+// CHECK: vfnmsub132sd %xmm13, %xmm11, %xmm26 {%k6}
+// CHECK:  encoding: [0x62,0x42,0xa5,0x0e,0x9f,0xd5]
+          vfnmsub132sd %xmm13, %xmm11, %xmm26 {%k6}
+
+// CHECK: vfnmsub132sd %xmm13, %xmm11, %xmm26 {%k6} {z}
+// CHECK:  encoding: [0x62,0x42,0xa5,0x8e,0x9f,0xd5]
+          vfnmsub132sd %xmm13, %xmm11, %xmm26 {%k6} {z}
+
+// CHECK: vfnmsub132sd {rn-sae}, %xmm13, %xmm11, %xmm26
+// CHECK:  encoding: [0x62,0x42,0xa5,0x18,0x9f,0xd5]
+          vfnmsub132sd {rn-sae}, %xmm13, %xmm11, %xmm26
+
+// CHECK: vfnmsub132sd {ru-sae}, %xmm13, %xmm11, %xmm26
+// CHECK:  encoding: [0x62,0x42,0xa5,0x58,0x9f,0xd5]
+          vfnmsub132sd {ru-sae}, %xmm13, %xmm11, %xmm26
+
+// CHECK: vfnmsub132sd {rd-sae}, %xmm13, %xmm11, %xmm26
+// CHECK:  encoding: [0x62,0x42,0xa5,0x38,0x9f,0xd5]
+          vfnmsub132sd {rd-sae}, %xmm13, %xmm11, %xmm26
+
+// CHECK: vfnmsub132sd {rz-sae}, %xmm13, %xmm11, %xmm26
+// CHECK:  encoding: [0x62,0x42,0xa5,0x78,0x9f,0xd5]
+          vfnmsub132sd {rz-sae}, %xmm13, %xmm11, %xmm26
+
+// CHECK: vfnmsub132sd (%rcx), %xmm11, %xmm26
+// CHECK:  encoding: [0x62,0x62,0xa5,0x08,0x9f,0x11]
+          vfnmsub132sd (%rcx), %xmm11, %xmm26
+
+// CHECK: vfnmsub132sd 291(%rax,%r14,8), %xmm11, %xmm26
+// CHECK:  encoding: [0x62,0x22,0xa5,0x08,0x9f,0x94,0xf0,0x23,0x01,0x00,0x00]
+          vfnmsub132sd 291(%rax,%r14,8), %xmm11, %xmm26
+
+// CHECK: vfnmsub132sd 1016(%rdx), %xmm11, %xmm26
+// CHECK:  encoding: [0x62,0x62,0xa5,0x08,0x9f,0x52,0x7f]
+          vfnmsub132sd 1016(%rdx), %xmm11, %xmm26
+
+// CHECK: vfnmsub132sd 1024(%rdx), %xmm11, %xmm26
+// CHECK:  encoding: [0x62,0x62,0xa5,0x08,0x9f,0x92,0x00,0x04,0x00,0x00]
+          vfnmsub132sd 1024(%rdx), %xmm11, %xmm26
+
+// CHECK: vfnmsub132sd -1024(%rdx), %xmm11, %xmm26
+// CHECK:  encoding: [0x62,0x62,0xa5,0x08,0x9f,0x52,0x80]
+          vfnmsub132sd -1024(%rdx), %xmm11, %xmm26
+
+// CHECK: vfnmsub132sd -1032(%rdx), %xmm11, %xmm26
+// CHECK:  encoding: [0x62,0x62,0xa5,0x08,0x9f,0x92,0xf8,0xfb,0xff,0xff]
+          vfnmsub132sd -1032(%rdx), %xmm11, %xmm26
+
+// CHECK: vfnmsub213ss %xmm12, %xmm28, %xmm14
+// CHECK:  encoding: [0x62,0x52,0x1d,0x00,0xaf,0xf4]
+          vfnmsub213ss %xmm12, %xmm28, %xmm14
+
+// CHECK: vfnmsub213ss %xmm12, %xmm28, %xmm14 {%k4}
+// CHECK:  encoding: [0x62,0x52,0x1d,0x04,0xaf,0xf4]
+          vfnmsub213ss %xmm12, %xmm28, %xmm14 {%k4}
+
+// CHECK: vfnmsub213ss %xmm12, %xmm28, %xmm14 {%k4} {z}
+// CHECK:  encoding: [0x62,0x52,0x1d,0x84,0xaf,0xf4]
+          vfnmsub213ss %xmm12, %xmm28, %xmm14 {%k4} {z}
+
+// CHECK: vfnmsub213ss {rn-sae}, %xmm12, %xmm28, %xmm14
+// CHECK:  encoding: [0x62,0x52,0x1d,0x10,0xaf,0xf4]
+          vfnmsub213ss {rn-sae}, %xmm12, %xmm28, %xmm14
+
+// CHECK: vfnmsub213ss {ru-sae}, %xmm12, %xmm28, %xmm14
+// CHECK:  encoding: [0x62,0x52,0x1d,0x50,0xaf,0xf4]
+          vfnmsub213ss {ru-sae}, %xmm12, %xmm28, %xmm14
+
+// CHECK: vfnmsub213ss {rd-sae}, %xmm12, %xmm28, %xmm14
+// CHECK:  encoding: [0x62,0x52,0x1d,0x30,0xaf,0xf4]
+          vfnmsub213ss {rd-sae}, %xmm12, %xmm28, %xmm14
+
+// CHECK: vfnmsub213ss {rz-sae}, %xmm12, %xmm28, %xmm14
+// CHECK:  encoding: [0x62,0x52,0x1d,0x70,0xaf,0xf4]
+          vfnmsub213ss {rz-sae}, %xmm12, %xmm28, %xmm14
+
+// CHECK: vfnmsub213ss (%rcx), %xmm28, %xmm14
+// CHECK:  encoding: [0x62,0x72,0x1d,0x00,0xaf,0x31]
+          vfnmsub213ss (%rcx), %xmm28, %xmm14
+
+// CHECK: vfnmsub213ss 291(%rax,%r14,8), %xmm28, %xmm14
+// CHECK:  encoding: [0x62,0x32,0x1d,0x00,0xaf,0xb4,0xf0,0x23,0x01,0x00,0x00]
+          vfnmsub213ss 291(%rax,%r14,8), %xmm28, %xmm14
+
+// CHECK: vfnmsub213ss 508(%rdx), %xmm28, %xmm14
+// CHECK:  encoding: [0x62,0x72,0x1d,0x00,0xaf,0x72,0x7f]
+          vfnmsub213ss 508(%rdx), %xmm28, %xmm14
+
+// CHECK: vfnmsub213ss 512(%rdx), %xmm28, %xmm14
+// CHECK:  encoding: [0x62,0x72,0x1d,0x00,0xaf,0xb2,0x00,0x02,0x00,0x00]
+          vfnmsub213ss 512(%rdx), %xmm28, %xmm14
+
+// CHECK: vfnmsub213ss -512(%rdx), %xmm28, %xmm14
+// CHECK:  encoding: [0x62,0x72,0x1d,0x00,0xaf,0x72,0x80]
+          vfnmsub213ss -512(%rdx), %xmm28, %xmm14
+
+// CHECK: vfnmsub213ss -516(%rdx), %xmm28, %xmm14
+// CHECK:  encoding: [0x62,0x72,0x1d,0x00,0xaf,0xb2,0xfc,0xfd,0xff,0xff]
+          vfnmsub213ss -516(%rdx), %xmm28, %xmm14
+
+// CHECK: vfnmsub213sd %xmm28, %xmm23, %xmm29
+// CHECK:  encoding: [0x62,0x02,0xc5,0x00,0xaf,0xec]
+          vfnmsub213sd %xmm28, %xmm23, %xmm29
+
+// CHECK: vfnmsub213sd %xmm28, %xmm23, %xmm29 {%k3}
+// CHECK:  encoding: [0x62,0x02,0xc5,0x03,0xaf,0xec]
+          vfnmsub213sd %xmm28, %xmm23, %xmm29 {%k3}
+
+// CHECK: vfnmsub213sd %xmm28, %xmm23, %xmm29 {%k3} {z}
+// CHECK:  encoding: [0x62,0x02,0xc5,0x83,0xaf,0xec]
+          vfnmsub213sd %xmm28, %xmm23, %xmm29 {%k3} {z}
+
+// CHECK: vfnmsub213sd {rn-sae}, %xmm28, %xmm23, %xmm29
+// CHECK:  encoding: [0x62,0x02,0xc5,0x10,0xaf,0xec]
+          vfnmsub213sd {rn-sae}, %xmm28, %xmm23, %xmm29
+
+// CHECK: vfnmsub213sd {ru-sae}, %xmm28, %xmm23, %xmm29
+// CHECK:  encoding: [0x62,0x02,0xc5,0x50,0xaf,0xec]
+          vfnmsub213sd {ru-sae}, %xmm28, %xmm23, %xmm29
+
+// CHECK: vfnmsub213sd {rd-sae}, %xmm28, %xmm23, %xmm29
+// CHECK:  encoding: [0x62,0x02,0xc5,0x30,0xaf,0xec]
+          vfnmsub213sd {rd-sae}, %xmm28, %xmm23, %xmm29
+
+// CHECK: vfnmsub213sd {rz-sae}, %xmm28, %xmm23, %xmm29
+// CHECK:  encoding: [0x62,0x02,0xc5,0x70,0xaf,0xec]
+          vfnmsub213sd {rz-sae}, %xmm28, %xmm23, %xmm29
+
+// CHECK: vfnmsub213sd (%rcx), %xmm23, %xmm29
+// CHECK:  encoding: [0x62,0x62,0xc5,0x00,0xaf,0x29]
+          vfnmsub213sd (%rcx), %xmm23, %xmm29
+
+// CHECK: vfnmsub213sd 291(%rax,%r14,8), %xmm23, %xmm29
+// CHECK:  encoding: [0x62,0x22,0xc5,0x00,0xaf,0xac,0xf0,0x23,0x01,0x00,0x00]
+          vfnmsub213sd 291(%rax,%r14,8), %xmm23, %xmm29
+
+// CHECK: vfnmsub213sd 1016(%rdx), %xmm23, %xmm29
+// CHECK:  encoding: [0x62,0x62,0xc5,0x00,0xaf,0x6a,0x7f]
+          vfnmsub213sd 1016(%rdx), %xmm23, %xmm29
+
+// CHECK: vfnmsub213sd 1024(%rdx), %xmm23, %xmm29
+// CHECK:  encoding: [0x62,0x62,0xc5,0x00,0xaf,0xaa,0x00,0x04,0x00,0x00]
+          vfnmsub213sd 1024(%rdx), %xmm23, %xmm29
+
+// CHECK: vfnmsub213sd -1024(%rdx), %xmm23, %xmm29
+// CHECK:  encoding: [0x62,0x62,0xc5,0x00,0xaf,0x6a,0x80]
+          vfnmsub213sd -1024(%rdx), %xmm23, %xmm29
+
+// CHECK: vfnmsub213sd -1032(%rdx), %xmm23, %xmm29
+// CHECK:  encoding: [0x62,0x62,0xc5,0x00,0xaf,0xaa,0xf8,0xfb,0xff,0xff]
+          vfnmsub213sd -1032(%rdx), %xmm23, %xmm29
+
+// CHECK: vfnmsub231ss %xmm10, %xmm13, %xmm26
+// CHECK:  encoding: [0x62,0x42,0x15,0x08,0xbf,0xd2]
+          vfnmsub231ss %xmm10, %xmm13, %xmm26
+
+// CHECK: vfnmsub231ss %xmm10, %xmm13, %xmm26 {%k4}
+// CHECK:  encoding: [0x62,0x42,0x15,0x0c,0xbf,0xd2]
+          vfnmsub231ss %xmm10, %xmm13, %xmm26 {%k4}
+
+// CHECK: vfnmsub231ss %xmm10, %xmm13, %xmm26 {%k4} {z}
+// CHECK:  encoding: [0x62,0x42,0x15,0x8c,0xbf,0xd2]
+          vfnmsub231ss %xmm10, %xmm13, %xmm26 {%k4} {z}
+
+// CHECK: vfnmsub231ss {rn-sae}, %xmm10, %xmm13, %xmm26
+// CHECK:  encoding: [0x62,0x42,0x15,0x18,0xbf,0xd2]
+          vfnmsub231ss {rn-sae}, %xmm10, %xmm13, %xmm26
+
+// CHECK: vfnmsub231ss {ru-sae}, %xmm10, %xmm13, %xmm26
+// CHECK:  encoding: [0x62,0x42,0x15,0x58,0xbf,0xd2]
+          vfnmsub231ss {ru-sae}, %xmm10, %xmm13, %xmm26
+
+// CHECK: vfnmsub231ss {rd-sae}, %xmm10, %xmm13, %xmm26
+// CHECK:  encoding: [0x62,0x42,0x15,0x38,0xbf,0xd2]
+          vfnmsub231ss {rd-sae}, %xmm10, %xmm13, %xmm26
+
+// CHECK: vfnmsub231ss {rz-sae}, %xmm10, %xmm13, %xmm26
+// CHECK:  encoding: [0x62,0x42,0x15,0x78,0xbf,0xd2]
+          vfnmsub231ss {rz-sae}, %xmm10, %xmm13, %xmm26
+
+// CHECK: vfnmsub231ss (%rcx), %xmm13, %xmm26
+// CHECK:  encoding: [0x62,0x62,0x15,0x08,0xbf,0x11]
+          vfnmsub231ss (%rcx), %xmm13, %xmm26
+
+// CHECK: vfnmsub231ss 291(%rax,%r14,8), %xmm13, %xmm26
+// CHECK:  encoding: [0x62,0x22,0x15,0x08,0xbf,0x94,0xf0,0x23,0x01,0x00,0x00]
+          vfnmsub231ss 291(%rax,%r14,8), %xmm13, %xmm26
+
+// CHECK: vfnmsub231ss 508(%rdx), %xmm13, %xmm26
+// CHECK:  encoding: [0x62,0x62,0x15,0x08,0xbf,0x52,0x7f]
+          vfnmsub231ss 508(%rdx), %xmm13, %xmm26
+
+// CHECK: vfnmsub231ss 512(%rdx), %xmm13, %xmm26
+// CHECK:  encoding: [0x62,0x62,0x15,0x08,0xbf,0x92,0x00,0x02,0x00,0x00]
+          vfnmsub231ss 512(%rdx), %xmm13, %xmm26
+
+// CHECK: vfnmsub231ss -512(%rdx), %xmm13, %xmm26
+// CHECK:  encoding: [0x62,0x62,0x15,0x08,0xbf,0x52,0x80]
+          vfnmsub231ss -512(%rdx), %xmm13, %xmm26
+
+// CHECK: vfnmsub231ss -516(%rdx), %xmm13, %xmm26
+// CHECK:  encoding: [0x62,0x62,0x15,0x08,0xbf,0x92,0xfc,0xfd,0xff,0xff]
+          vfnmsub231ss -516(%rdx), %xmm13, %xmm26
+
+// CHECK: vfnmsub231sd %xmm14, %xmm20, %xmm6
+// CHECK:  encoding: [0x62,0xd2,0xdd,0x00,0xbf,0xf6]
+          vfnmsub231sd %xmm14, %xmm20, %xmm6
+
+// CHECK: vfnmsub231sd %xmm14, %xmm20, %xmm6 {%k1}
+// CHECK:  encoding: [0x62,0xd2,0xdd,0x01,0xbf,0xf6]
+          vfnmsub231sd %xmm14, %xmm20, %xmm6 {%k1}
+
+// CHECK: vfnmsub231sd %xmm14, %xmm20, %xmm6 {%k1} {z}
+// CHECK:  encoding: [0x62,0xd2,0xdd,0x81,0xbf,0xf6]
+          vfnmsub231sd %xmm14, %xmm20, %xmm6 {%k1} {z}
+
+// CHECK: vfnmsub231sd {rn-sae}, %xmm14, %xmm20, %xmm6
+// CHECK:  encoding: [0x62,0xd2,0xdd,0x10,0xbf,0xf6]
+          vfnmsub231sd {rn-sae}, %xmm14, %xmm20, %xmm6
+
+// CHECK: vfnmsub231sd {ru-sae}, %xmm14, %xmm20, %xmm6
+// CHECK:  encoding: [0x62,0xd2,0xdd,0x50,0xbf,0xf6]
+          vfnmsub231sd {ru-sae}, %xmm14, %xmm20, %xmm6
+
+// CHECK: vfnmsub231sd {rd-sae}, %xmm14, %xmm20, %xmm6
+// CHECK:  encoding: [0x62,0xd2,0xdd,0x30,0xbf,0xf6]
+          vfnmsub231sd {rd-sae}, %xmm14, %xmm20, %xmm6
+
+// CHECK: vfnmsub231sd {rz-sae}, %xmm14, %xmm20, %xmm6
+// CHECK:  encoding: [0x62,0xd2,0xdd,0x70,0xbf,0xf6]
+          vfnmsub231sd {rz-sae}, %xmm14, %xmm20, %xmm6
+
+// CHECK: vfnmsub231sd (%rcx), %xmm20, %xmm6
+// CHECK:  encoding: [0x62,0xf2,0xdd,0x00,0xbf,0x31]
+          vfnmsub231sd (%rcx), %xmm20, %xmm6
+
+// CHECK: vfnmsub231sd 291(%rax,%r14,8), %xmm20, %xmm6
+// CHECK:  encoding: [0x62,0xb2,0xdd,0x00,0xbf,0xb4,0xf0,0x23,0x01,0x00,0x00]
+          vfnmsub231sd 291(%rax,%r14,8), %xmm20, %xmm6
+
+// CHECK: vfnmsub231sd 1016(%rdx), %xmm20, %xmm6
+// CHECK:  encoding: [0x62,0xf2,0xdd,0x00,0xbf,0x72,0x7f]
+          vfnmsub231sd 1016(%rdx), %xmm20, %xmm6
+
+// CHECK: vfnmsub231sd 1024(%rdx), %xmm20, %xmm6
+// CHECK:  encoding: [0x62,0xf2,0xdd,0x00,0xbf,0xb2,0x00,0x04,0x00,0x00]
+          vfnmsub231sd 1024(%rdx), %xmm20, %xmm6
+
+// CHECK: vfnmsub231sd -1024(%rdx), %xmm20, %xmm6
+// CHECK:  encoding: [0x62,0xf2,0xdd,0x00,0xbf,0x72,0x80]
+          vfnmsub231sd -1024(%rdx), %xmm20, %xmm6
+
+// CHECK: vfnmsub231sd -1032(%rdx), %xmm20, %xmm6
+// CHECK:  encoding: [0x62,0xf2,0xdd,0x00,0xbf,0xb2,0xf8,0xfb,0xff,0xff]
+          vfnmsub231sd -1032(%rdx), %xmm20, %xmm6
+
+// CHECK: vunpckhps %zmm16, %zmm14, %zmm5
+// CHECK:  encoding: [0x62,0xb1,0x0c,0x48,0x15,0xe8]
+          vunpckhps %zmm16, %zmm14, %zmm5
+
+// CHECK: vunpckhps %zmm16, %zmm14, %zmm5 {%k6}
+// CHECK:  encoding: [0x62,0xb1,0x0c,0x4e,0x15,0xe8]
+          vunpckhps %zmm16, %zmm14, %zmm5 {%k6}
+
+// CHECK: vunpckhps %zmm16, %zmm14, %zmm5 {%k6} {z}
+// CHECK:  encoding: [0x62,0xb1,0x0c,0xce,0x15,0xe8]
+          vunpckhps %zmm16, %zmm14, %zmm5 {%k6} {z}
+
+// CHECK: vunpckhps (%rcx), %zmm14, %zmm5
+// CHECK:  encoding: [0x62,0xf1,0x0c,0x48,0x15,0x29]
+          vunpckhps (%rcx), %zmm14, %zmm5
+
+// CHECK: vunpckhps 291(%rax,%r14,8), %zmm14, %zmm5
+// CHECK:  encoding: [0x62,0xb1,0x0c,0x48,0x15,0xac,0xf0,0x23,0x01,0x00,0x00]
+          vunpckhps 291(%rax,%r14,8), %zmm14, %zmm5
+
+// CHECK: vunpckhps (%rcx){1to16}, %zmm14, %zmm5
+// CHECK:  encoding: [0x62,0xf1,0x0c,0x58,0x15,0x29]
+          vunpckhps (%rcx){1to16}, %zmm14, %zmm5
+
+// CHECK: vunpckhps 8128(%rdx), %zmm14, %zmm5
+// CHECK:  encoding: [0x62,0xf1,0x0c,0x48,0x15,0x6a,0x7f]
+          vunpckhps 8128(%rdx), %zmm14, %zmm5
+
+// CHECK: vunpckhps 8192(%rdx), %zmm14, %zmm5
+// CHECK:  encoding: [0x62,0xf1,0x0c,0x48,0x15,0xaa,0x00,0x20,0x00,0x00]
+          vunpckhps 8192(%rdx), %zmm14, %zmm5
+
+// CHECK: vunpckhps -8192(%rdx), %zmm14, %zmm5
+// CHECK:  encoding: [0x62,0xf1,0x0c,0x48,0x15,0x6a,0x80]
+          vunpckhps -8192(%rdx), %zmm14, %zmm5
+
+// CHECK: vunpckhps -8256(%rdx), %zmm14, %zmm5
+// CHECK:  encoding: [0x62,0xf1,0x0c,0x48,0x15,0xaa,0xc0,0xdf,0xff,0xff]
+          vunpckhps -8256(%rdx), %zmm14, %zmm5
+
+// CHECK: vunpckhps 508(%rdx){1to16}, %zmm14, %zmm5
+// CHECK:  encoding: [0x62,0xf1,0x0c,0x58,0x15,0x6a,0x7f]
+          vunpckhps 508(%rdx){1to16}, %zmm14, %zmm5
+
+// CHECK: vunpckhps 512(%rdx){1to16}, %zmm14, %zmm5
+// CHECK:  encoding: [0x62,0xf1,0x0c,0x58,0x15,0xaa,0x00,0x02,0x00,0x00]
+          vunpckhps 512(%rdx){1to16}, %zmm14, %zmm5
+
+// CHECK: vunpckhps -512(%rdx){1to16}, %zmm14, %zmm5
+// CHECK:  encoding: [0x62,0xf1,0x0c,0x58,0x15,0x6a,0x80]
+          vunpckhps -512(%rdx){1to16}, %zmm14, %zmm5
+
+// CHECK: vunpckhps -516(%rdx){1to16}, %zmm14, %zmm5
+// CHECK:  encoding: [0x62,0xf1,0x0c,0x58,0x15,0xaa,0xfc,0xfd,0xff,0xff]
+          vunpckhps -516(%rdx){1to16}, %zmm14, %zmm5
+
+// CHECK: vunpcklps %zmm2, %zmm3, %zmm1
+// CHECK:  encoding: [0x62,0xf1,0x64,0x48,0x14,0xca]
+          vunpcklps %zmm2, %zmm3, %zmm1
+
+// CHECK: vunpcklps %zmm2, %zmm3, %zmm1 {%k3}
+// CHECK:  encoding: [0x62,0xf1,0x64,0x4b,0x14,0xca]
+          vunpcklps %zmm2, %zmm3, %zmm1 {%k3}
+
+// CHECK: vunpcklps %zmm2, %zmm3, %zmm1 {%k3} {z}
+// CHECK:  encoding: [0x62,0xf1,0x64,0xcb,0x14,0xca]
+          vunpcklps %zmm2, %zmm3, %zmm1 {%k3} {z}
+
+// CHECK: vunpcklps (%rcx), %zmm3, %zmm1
+// CHECK:  encoding: [0x62,0xf1,0x64,0x48,0x14,0x09]
+          vunpcklps (%rcx), %zmm3, %zmm1
+
+// CHECK: vunpcklps 291(%rax,%r14,8), %zmm3, %zmm1
+// CHECK:  encoding: [0x62,0xb1,0x64,0x48,0x14,0x8c,0xf0,0x23,0x01,0x00,0x00]
+          vunpcklps 291(%rax,%r14,8), %zmm3, %zmm1
+
+// CHECK: vunpcklps (%rcx){1to16}, %zmm3, %zmm1
+// CHECK:  encoding: [0x62,0xf1,0x64,0x58,0x14,0x09]
+          vunpcklps (%rcx){1to16}, %zmm3, %zmm1
+
+// CHECK: vunpcklps 8128(%rdx), %zmm3, %zmm1
+// CHECK:  encoding: [0x62,0xf1,0x64,0x48,0x14,0x4a,0x7f]
+          vunpcklps 8128(%rdx), %zmm3, %zmm1
+
+// CHECK: vunpcklps 8192(%rdx), %zmm3, %zmm1
+// CHECK:  encoding: [0x62,0xf1,0x64,0x48,0x14,0x8a,0x00,0x20,0x00,0x00]
+          vunpcklps 8192(%rdx), %zmm3, %zmm1
+
+// CHECK: vunpcklps -8192(%rdx), %zmm3, %zmm1
+// CHECK:  encoding: [0x62,0xf1,0x64,0x48,0x14,0x4a,0x80]
+          vunpcklps -8192(%rdx), %zmm3, %zmm1
+
+// CHECK: vunpcklps -8256(%rdx), %zmm3, %zmm1
+// CHECK:  encoding: [0x62,0xf1,0x64,0x48,0x14,0x8a,0xc0,0xdf,0xff,0xff]
+          vunpcklps -8256(%rdx), %zmm3, %zmm1
+
+// CHECK: vunpcklps 508(%rdx){1to16}, %zmm3, %zmm1
+// CHECK:  encoding: [0x62,0xf1,0x64,0x58,0x14,0x4a,0x7f]
+          vunpcklps 508(%rdx){1to16}, %zmm3, %zmm1
+
+// CHECK: vunpcklps 512(%rdx){1to16}, %zmm3, %zmm1
+// CHECK:  encoding: [0x62,0xf1,0x64,0x58,0x14,0x8a,0x00,0x02,0x00,0x00]
+          vunpcklps 512(%rdx){1to16}, %zmm3, %zmm1
+
+// CHECK: vunpcklps -512(%rdx){1to16}, %zmm3, %zmm1
+// CHECK:  encoding: [0x62,0xf1,0x64,0x58,0x14,0x4a,0x80]
+          vunpcklps -512(%rdx){1to16}, %zmm3, %zmm1
+
+// CHECK: vunpcklps -516(%rdx){1to16}, %zmm3, %zmm1
+// CHECK:  encoding: [0x62,0xf1,0x64,0x58,0x14,0x8a,0xfc,0xfd,0xff,0xff]
+          vunpcklps -516(%rdx){1to16}, %zmm3, %zmm1
+
+// CHECK: vunpckhpd %zmm26, %zmm19, %zmm25
+// CHECK:  encoding: [0x62,0x01,0xe5,0x40,0x15,0xca]
+          vunpckhpd %zmm26, %zmm19, %zmm25
+
+// CHECK: vunpckhpd %zmm26, %zmm19, %zmm25 {%k5}
+// CHECK:  encoding: [0x62,0x01,0xe5,0x45,0x15,0xca]
+          vunpckhpd %zmm26, %zmm19, %zmm25 {%k5}
+
+// CHECK: vunpckhpd %zmm26, %zmm19, %zmm25 {%k5} {z}
+// CHECK:  encoding: [0x62,0x01,0xe5,0xc5,0x15,0xca]
+          vunpckhpd %zmm26, %zmm19, %zmm25 {%k5} {z}
+
+// CHECK: vunpckhpd (%rcx), %zmm19, %zmm25
+// CHECK:  encoding: [0x62,0x61,0xe5,0x40,0x15,0x09]
+          vunpckhpd (%rcx), %zmm19, %zmm25
+
+// CHECK: vunpckhpd 291(%rax,%r14,8), %zmm19, %zmm25
+// CHECK:  encoding: [0x62,0x21,0xe5,0x40,0x15,0x8c,0xf0,0x23,0x01,0x00,0x00]
+          vunpckhpd 291(%rax,%r14,8), %zmm19, %zmm25
+
+// CHECK: vunpckhpd (%rcx){1to8}, %zmm19, %zmm25
+// CHECK:  encoding: [0x62,0x61,0xe5,0x50,0x15,0x09]
+          vunpckhpd (%rcx){1to8}, %zmm19, %zmm25
+
+// CHECK: vunpckhpd 8128(%rdx), %zmm19, %zmm25
+// CHECK:  encoding: [0x62,0x61,0xe5,0x40,0x15,0x4a,0x7f]
+          vunpckhpd 8128(%rdx), %zmm19, %zmm25
+
+// CHECK: vunpckhpd 8192(%rdx), %zmm19, %zmm25
+// CHECK:  encoding: [0x62,0x61,0xe5,0x40,0x15,0x8a,0x00,0x20,0x00,0x00]
+          vunpckhpd 8192(%rdx), %zmm19, %zmm25
+
+// CHECK: vunpckhpd -8192(%rdx), %zmm19, %zmm25
+// CHECK:  encoding: [0x62,0x61,0xe5,0x40,0x15,0x4a,0x80]
+          vunpckhpd -8192(%rdx), %zmm19, %zmm25
+
+// CHECK: vunpckhpd -8256(%rdx), %zmm19, %zmm25
+// CHECK:  encoding: [0x62,0x61,0xe5,0x40,0x15,0x8a,0xc0,0xdf,0xff,0xff]
+          vunpckhpd -8256(%rdx), %zmm19, %zmm25
+
+// CHECK: vunpckhpd 1016(%rdx){1to8}, %zmm19, %zmm25
+// CHECK:  encoding: [0x62,0x61,0xe5,0x50,0x15,0x4a,0x7f]
+          vunpckhpd 1016(%rdx){1to8}, %zmm19, %zmm25
+
+// CHECK: vunpckhpd 1024(%rdx){1to8}, %zmm19, %zmm25
+// CHECK:  encoding: [0x62,0x61,0xe5,0x50,0x15,0x8a,0x00,0x04,0x00,0x00]
+          vunpckhpd 1024(%rdx){1to8}, %zmm19, %zmm25
+
+// CHECK: vunpckhpd -1024(%rdx){1to8}, %zmm19, %zmm25
+// CHECK:  encoding: [0x62,0x61,0xe5,0x50,0x15,0x4a,0x80]
+          vunpckhpd -1024(%rdx){1to8}, %zmm19, %zmm25
+
+// CHECK: vunpckhpd -1032(%rdx){1to8}, %zmm19, %zmm25
+// CHECK:  encoding: [0x62,0x61,0xe5,0x50,0x15,0x8a,0xf8,0xfb,0xff,0xff]
+          vunpckhpd -1032(%rdx){1to8}, %zmm19, %zmm25
+
+// CHECK: vunpcklpd %zmm21, %zmm29, %zmm18
+// CHECK:  encoding: [0x62,0xa1,0x95,0x40,0x14,0xd5]
+          vunpcklpd %zmm21, %zmm29, %zmm18
+
+// CHECK: vunpcklpd %zmm21, %zmm29, %zmm18 {%k6}
+// CHECK:  encoding: [0x62,0xa1,0x95,0x46,0x14,0xd5]
+          vunpcklpd %zmm21, %zmm29, %zmm18 {%k6}
+
+// CHECK: vunpcklpd %zmm21, %zmm29, %zmm18 {%k6} {z}
+// CHECK:  encoding: [0x62,0xa1,0x95,0xc6,0x14,0xd5]
+          vunpcklpd %zmm21, %zmm29, %zmm18 {%k6} {z}
+
+// CHECK: vunpcklpd (%rcx), %zmm29, %zmm18
+// CHECK:  encoding: [0x62,0xe1,0x95,0x40,0x14,0x11]
+          vunpcklpd (%rcx), %zmm29, %zmm18
+
+// CHECK: vunpcklpd 291(%rax,%r14,8), %zmm29, %zmm18
+// CHECK:  encoding: [0x62,0xa1,0x95,0x40,0x14,0x94,0xf0,0x23,0x01,0x00,0x00]
+          vunpcklpd 291(%rax,%r14,8), %zmm29, %zmm18
+
+// CHECK: vunpcklpd (%rcx){1to8}, %zmm29, %zmm18
+// CHECK:  encoding: [0x62,0xe1,0x95,0x50,0x14,0x11]
+          vunpcklpd (%rcx){1to8}, %zmm29, %zmm18
+
+// CHECK: vunpcklpd 8128(%rdx), %zmm29, %zmm18
+// CHECK:  encoding: [0x62,0xe1,0x95,0x40,0x14,0x52,0x7f]
+          vunpcklpd 8128(%rdx), %zmm29, %zmm18
+
+// CHECK: vunpcklpd 8192(%rdx), %zmm29, %zmm18
+// CHECK:  encoding: [0x62,0xe1,0x95,0x40,0x14,0x92,0x00,0x20,0x00,0x00]
+          vunpcklpd 8192(%rdx), %zmm29, %zmm18
+
+// CHECK: vunpcklpd -8192(%rdx), %zmm29, %zmm18
+// CHECK:  encoding: [0x62,0xe1,0x95,0x40,0x14,0x52,0x80]
+          vunpcklpd -8192(%rdx), %zmm29, %zmm18
+
+// CHECK: vunpcklpd -8256(%rdx), %zmm29, %zmm18
+// CHECK:  encoding: [0x62,0xe1,0x95,0x40,0x14,0x92,0xc0,0xdf,0xff,0xff]
+          vunpcklpd -8256(%rdx), %zmm29, %zmm18
+
+// CHECK: vunpcklpd 1016(%rdx){1to8}, %zmm29, %zmm18
+// CHECK:  encoding: [0x62,0xe1,0x95,0x50,0x14,0x52,0x7f]
+          vunpcklpd 1016(%rdx){1to8}, %zmm29, %zmm18
+
+// CHECK: vunpcklpd 1024(%rdx){1to8}, %zmm29, %zmm18
+// CHECK:  encoding: [0x62,0xe1,0x95,0x50,0x14,0x92,0x00,0x04,0x00,0x00]
+          vunpcklpd 1024(%rdx){1to8}, %zmm29, %zmm18
+
+// CHECK: vunpcklpd -1024(%rdx){1to8}, %zmm29, %zmm18
+// CHECK:  encoding: [0x62,0xe1,0x95,0x50,0x14,0x52,0x80]
+          vunpcklpd -1024(%rdx){1to8}, %zmm29, %zmm18
+
+// CHECK: vunpcklpd -1032(%rdx){1to8}, %zmm29, %zmm18
+// CHECK:  encoding: [0x62,0xe1,0x95,0x50,0x14,0x92,0xf8,0xfb,0xff,0xff]
+          vunpcklpd -1032(%rdx){1to8}, %zmm29, %zmm18
+
+// CHECK: vpunpckldq %zmm17, %zmm3, %zmm24
+// CHECK:  encoding: [0x62,0x21,0x65,0x48,0x62,0xc1]
+          vpunpckldq %zmm17, %zmm3, %zmm24
+
+// CHECK: vpunpckldq %zmm17, %zmm3, %zmm24 {%k3}
+// CHECK:  encoding: [0x62,0x21,0x65,0x4b,0x62,0xc1]
+          vpunpckldq %zmm17, %zmm3, %zmm24 {%k3}
+
+// CHECK: vpunpckldq %zmm17, %zmm3, %zmm24 {%k3} {z}
+// CHECK:  encoding: [0x62,0x21,0x65,0xcb,0x62,0xc1]
+          vpunpckldq %zmm17, %zmm3, %zmm24 {%k3} {z}
+
+// CHECK: vpunpckldq (%rcx), %zmm3, %zmm24
+// CHECK:  encoding: [0x62,0x61,0x65,0x48,0x62,0x01]
+          vpunpckldq (%rcx), %zmm3, %zmm24
+
+// CHECK: vpunpckldq 291(%rax,%r14,8), %zmm3, %zmm24
+// CHECK:  encoding: [0x62,0x21,0x65,0x48,0x62,0x84,0xf0,0x23,0x01,0x00,0x00]
+          vpunpckldq 291(%rax,%r14,8), %zmm3, %zmm24
+
+// CHECK: vpunpckldq (%rcx){1to16}, %zmm3, %zmm24
+// CHECK:  encoding: [0x62,0x61,0x65,0x58,0x62,0x01]
+          vpunpckldq (%rcx){1to16}, %zmm3, %zmm24
+
+// CHECK: vpunpckldq 8128(%rdx), %zmm3, %zmm24
+// CHECK:  encoding: [0x62,0x61,0x65,0x48,0x62,0x42,0x7f]
+          vpunpckldq 8128(%rdx), %zmm3, %zmm24
+
+// CHECK: vpunpckldq 8192(%rdx), %zmm3, %zmm24
+// CHECK:  encoding: [0x62,0x61,0x65,0x48,0x62,0x82,0x00,0x20,0x00,0x00]
+          vpunpckldq 8192(%rdx), %zmm3, %zmm24
+
+// CHECK: vpunpckldq -8192(%rdx), %zmm3, %zmm24
+// CHECK:  encoding: [0x62,0x61,0x65,0x48,0x62,0x42,0x80]
+          vpunpckldq -8192(%rdx), %zmm3, %zmm24
+
+// CHECK: vpunpckldq -8256(%rdx), %zmm3, %zmm24
+// CHECK:  encoding: [0x62,0x61,0x65,0x48,0x62,0x82,0xc0,0xdf,0xff,0xff]
+          vpunpckldq -8256(%rdx), %zmm3, %zmm24
+
+// CHECK: vpunpckldq 508(%rdx){1to16}, %zmm3, %zmm24
+// CHECK:  encoding: [0x62,0x61,0x65,0x58,0x62,0x42,0x7f]
+          vpunpckldq 508(%rdx){1to16}, %zmm3, %zmm24
+
+// CHECK: vpunpckldq 512(%rdx){1to16}, %zmm3, %zmm24
+// CHECK:  encoding: [0x62,0x61,0x65,0x58,0x62,0x82,0x00,0x02,0x00,0x00]
+          vpunpckldq 512(%rdx){1to16}, %zmm3, %zmm24
+
+// CHECK: vpunpckldq -512(%rdx){1to16}, %zmm3, %zmm24
+// CHECK:  encoding: [0x62,0x61,0x65,0x58,0x62,0x42,0x80]
+          vpunpckldq -512(%rdx){1to16}, %zmm3, %zmm24
+
+// CHECK: vpunpckldq -516(%rdx){1to16}, %zmm3, %zmm24
+// CHECK:  encoding: [0x62,0x61,0x65,0x58,0x62,0x82,0xfc,0xfd,0xff,0xff]
+          vpunpckldq -516(%rdx){1to16}, %zmm3, %zmm24
+
+// CHECK: vpunpckhdq %zmm13, %zmm4, %zmm6
+// CHECK:  encoding: [0x62,0xd1,0x5d,0x48,0x6a,0xf5]
+          vpunpckhdq %zmm13, %zmm4, %zmm6
+
+// CHECK: vpunpckhdq %zmm13, %zmm4, %zmm6 {%k5}
+// CHECK:  encoding: [0x62,0xd1,0x5d,0x4d,0x6a,0xf5]
+          vpunpckhdq %zmm13, %zmm4, %zmm6 {%k5}
+
+// CHECK: vpunpckhdq %zmm13, %zmm4, %zmm6 {%k5} {z}
+// CHECK:  encoding: [0x62,0xd1,0x5d,0xcd,0x6a,0xf5]
+          vpunpckhdq %zmm13, %zmm4, %zmm6 {%k5} {z}
+
+// CHECK: vpunpckhdq (%rcx), %zmm4, %zmm6
+// CHECK:  encoding: [0x62,0xf1,0x5d,0x48,0x6a,0x31]
+          vpunpckhdq (%rcx), %zmm4, %zmm6
+
+// CHECK: vpunpckhdq 291(%rax,%r14,8), %zmm4, %zmm6
+// CHECK:  encoding: [0x62,0xb1,0x5d,0x48,0x6a,0xb4,0xf0,0x23,0x01,0x00,0x00]
+          vpunpckhdq 291(%rax,%r14,8), %zmm4, %zmm6
+
+// CHECK: vpunpckhdq (%rcx){1to16}, %zmm4, %zmm6
+// CHECK:  encoding: [0x62,0xf1,0x5d,0x58,0x6a,0x31]
+          vpunpckhdq (%rcx){1to16}, %zmm4, %zmm6
+
+// CHECK: vpunpckhdq 8128(%rdx), %zmm4, %zmm6
+// CHECK:  encoding: [0x62,0xf1,0x5d,0x48,0x6a,0x72,0x7f]
+          vpunpckhdq 8128(%rdx), %zmm4, %zmm6
+
+// CHECK: vpunpckhdq 8192(%rdx), %zmm4, %zmm6
+// CHECK:  encoding: [0x62,0xf1,0x5d,0x48,0x6a,0xb2,0x00,0x20,0x00,0x00]
+          vpunpckhdq 8192(%rdx), %zmm4, %zmm6
+
+// CHECK: vpunpckhdq -8192(%rdx), %zmm4, %zmm6
+// CHECK:  encoding: [0x62,0xf1,0x5d,0x48,0x6a,0x72,0x80]
+          vpunpckhdq -8192(%rdx), %zmm4, %zmm6
+
+// CHECK: vpunpckhdq -8256(%rdx), %zmm4, %zmm6
+// CHECK:  encoding: [0x62,0xf1,0x5d,0x48,0x6a,0xb2,0xc0,0xdf,0xff,0xff]
+          vpunpckhdq -8256(%rdx), %zmm4, %zmm6
+
+// CHECK: vpunpckhdq 508(%rdx){1to16}, %zmm4, %zmm6
+// CHECK:  encoding: [0x62,0xf1,0x5d,0x58,0x6a,0x72,0x7f]
+          vpunpckhdq 508(%rdx){1to16}, %zmm4, %zmm6
+
+// CHECK: vpunpckhdq 512(%rdx){1to16}, %zmm4, %zmm6
+// CHECK:  encoding: [0x62,0xf1,0x5d,0x58,0x6a,0xb2,0x00,0x02,0x00,0x00]
+          vpunpckhdq 512(%rdx){1to16}, %zmm4, %zmm6
+
+// CHECK: vpunpckhdq -512(%rdx){1to16}, %zmm4, %zmm6
+// CHECK:  encoding: [0x62,0xf1,0x5d,0x58,0x6a,0x72,0x80]
+          vpunpckhdq -512(%rdx){1to16}, %zmm4, %zmm6
+
+// CHECK: vpunpckhdq -516(%rdx){1to16}, %zmm4, %zmm6
+// CHECK:  encoding: [0x62,0xf1,0x5d,0x58,0x6a,0xb2,0xfc,0xfd,0xff,0xff]
+          vpunpckhdq -516(%rdx){1to16}, %zmm4, %zmm6
+
+// CHECK: vpunpcklqdq %zmm17, %zmm4, %zmm3
+// CHECK:  encoding: [0x62,0xb1,0xdd,0x48,0x6c,0xd9]
+          vpunpcklqdq %zmm17, %zmm4, %zmm3
+
+// CHECK: vpunpcklqdq %zmm17, %zmm4, %zmm3 {%k1}
+// CHECK:  encoding: [0x62,0xb1,0xdd,0x49,0x6c,0xd9]
+          vpunpcklqdq %zmm17, %zmm4, %zmm3 {%k1}
+
+// CHECK: vpunpcklqdq %zmm17, %zmm4, %zmm3 {%k1} {z}
+// CHECK:  encoding: [0x62,0xb1,0xdd,0xc9,0x6c,0xd9]
+          vpunpcklqdq %zmm17, %zmm4, %zmm3 {%k1} {z}
+
+// CHECK: vpunpcklqdq (%rcx), %zmm4, %zmm3
+// CHECK:  encoding: [0x62,0xf1,0xdd,0x48,0x6c,0x19]
+          vpunpcklqdq (%rcx), %zmm4, %zmm3
+
+// CHECK: vpunpcklqdq 291(%rax,%r14,8), %zmm4, %zmm3
+// CHECK:  encoding: [0x62,0xb1,0xdd,0x48,0x6c,0x9c,0xf0,0x23,0x01,0x00,0x00]
+          vpunpcklqdq 291(%rax,%r14,8), %zmm4, %zmm3
+
+// CHECK: vpunpcklqdq (%rcx){1to8}, %zmm4, %zmm3
+// CHECK:  encoding: [0x62,0xf1,0xdd,0x58,0x6c,0x19]
+          vpunpcklqdq (%rcx){1to8}, %zmm4, %zmm3
+
+// CHECK: vpunpcklqdq 8128(%rdx), %zmm4, %zmm3
+// CHECK:  encoding: [0x62,0xf1,0xdd,0x48,0x6c,0x5a,0x7f]
+          vpunpcklqdq 8128(%rdx), %zmm4, %zmm3
+
+// CHECK: vpunpcklqdq 8192(%rdx), %zmm4, %zmm3
+// CHECK:  encoding: [0x62,0xf1,0xdd,0x48,0x6c,0x9a,0x00,0x20,0x00,0x00]
+          vpunpcklqdq 8192(%rdx), %zmm4, %zmm3
+
+// CHECK: vpunpcklqdq -8192(%rdx), %zmm4, %zmm3
+// CHECK:  encoding: [0x62,0xf1,0xdd,0x48,0x6c,0x5a,0x80]
+          vpunpcklqdq -8192(%rdx), %zmm4, %zmm3
+
+// CHECK: vpunpcklqdq -8256(%rdx), %zmm4, %zmm3
+// CHECK:  encoding: [0x62,0xf1,0xdd,0x48,0x6c,0x9a,0xc0,0xdf,0xff,0xff]
+          vpunpcklqdq -8256(%rdx), %zmm4, %zmm3
+
+// CHECK: vpunpcklqdq 1016(%rdx){1to8}, %zmm4, %zmm3
+// CHECK:  encoding: [0x62,0xf1,0xdd,0x58,0x6c,0x5a,0x7f]
+          vpunpcklqdq 1016(%rdx){1to8}, %zmm4, %zmm3
+
+// CHECK: vpunpcklqdq 1024(%rdx){1to8}, %zmm4, %zmm3
+// CHECK:  encoding: [0x62,0xf1,0xdd,0x58,0x6c,0x9a,0x00,0x04,0x00,0x00]
+          vpunpcklqdq 1024(%rdx){1to8}, %zmm4, %zmm3
+
+// CHECK: vpunpcklqdq -1024(%rdx){1to8}, %zmm4, %zmm3
+// CHECK:  encoding: [0x62,0xf1,0xdd,0x58,0x6c,0x5a,0x80]
+          vpunpcklqdq -1024(%rdx){1to8}, %zmm4, %zmm3
+
+// CHECK: vpunpcklqdq -1032(%rdx){1to8}, %zmm4, %zmm3
+// CHECK:  encoding: [0x62,0xf1,0xdd,0x58,0x6c,0x9a,0xf8,0xfb,0xff,0xff]
+          vpunpcklqdq -1032(%rdx){1to8}, %zmm4, %zmm3
+
+// CHECK: vpunpckhqdq %zmm16, %zmm15, %zmm27
+// CHECK:  encoding: [0x62,0x21,0x85,0x48,0x6d,0xd8]
+          vpunpckhqdq %zmm16, %zmm15, %zmm27
+
+// CHECK: vpunpckhqdq %zmm16, %zmm15, %zmm27 {%k3}
+// CHECK:  encoding: [0x62,0x21,0x85,0x4b,0x6d,0xd8]
+          vpunpckhqdq %zmm16, %zmm15, %zmm27 {%k3}
+
+// CHECK: vpunpckhqdq %zmm16, %zmm15, %zmm27 {%k3} {z}
+// CHECK:  encoding: [0x62,0x21,0x85,0xcb,0x6d,0xd8]
+          vpunpckhqdq %zmm16, %zmm15, %zmm27 {%k3} {z}
+
+// CHECK: vpunpckhqdq (%rcx), %zmm15, %zmm27
+// CHECK:  encoding: [0x62,0x61,0x85,0x48,0x6d,0x19]
+          vpunpckhqdq (%rcx), %zmm15, %zmm27
+
+// CHECK: vpunpckhqdq 291(%rax,%r14,8), %zmm15, %zmm27
+// CHECK:  encoding: [0x62,0x21,0x85,0x48,0x6d,0x9c,0xf0,0x23,0x01,0x00,0x00]
+          vpunpckhqdq 291(%rax,%r14,8), %zmm15, %zmm27
+
+// CHECK: vpunpckhqdq (%rcx){1to8}, %zmm15, %zmm27
+// CHECK:  encoding: [0x62,0x61,0x85,0x58,0x6d,0x19]
+          vpunpckhqdq (%rcx){1to8}, %zmm15, %zmm27
+
+// CHECK: vpunpckhqdq 8128(%rdx), %zmm15, %zmm27
+// CHECK:  encoding: [0x62,0x61,0x85,0x48,0x6d,0x5a,0x7f]
+          vpunpckhqdq 8128(%rdx), %zmm15, %zmm27
+
+// CHECK: vpunpckhqdq 8192(%rdx), %zmm15, %zmm27
+// CHECK:  encoding: [0x62,0x61,0x85,0x48,0x6d,0x9a,0x00,0x20,0x00,0x00]
+          vpunpckhqdq 8192(%rdx), %zmm15, %zmm27
+
+// CHECK: vpunpckhqdq -8192(%rdx), %zmm15, %zmm27
+// CHECK:  encoding: [0x62,0x61,0x85,0x48,0x6d,0x5a,0x80]
+          vpunpckhqdq -8192(%rdx), %zmm15, %zmm27
+
+// CHECK: vpunpckhqdq -8256(%rdx), %zmm15, %zmm27
+// CHECK:  encoding: [0x62,0x61,0x85,0x48,0x6d,0x9a,0xc0,0xdf,0xff,0xff]
+          vpunpckhqdq -8256(%rdx), %zmm15, %zmm27
+
+// CHECK: vpunpckhqdq 1016(%rdx){1to8}, %zmm15, %zmm27
+// CHECK:  encoding: [0x62,0x61,0x85,0x58,0x6d,0x5a,0x7f]
+          vpunpckhqdq 1016(%rdx){1to8}, %zmm15, %zmm27
+
+// CHECK: vpunpckhqdq 1024(%rdx){1to8}, %zmm15, %zmm27
+// CHECK:  encoding: [0x62,0x61,0x85,0x58,0x6d,0x9a,0x00,0x04,0x00,0x00]
+          vpunpckhqdq 1024(%rdx){1to8}, %zmm15, %zmm27
+
+// CHECK: vpunpckhqdq -1024(%rdx){1to8}, %zmm15, %zmm27
+// CHECK:  encoding: [0x62,0x61,0x85,0x58,0x6d,0x5a,0x80]
+          vpunpckhqdq -1024(%rdx){1to8}, %zmm15, %zmm27
+
+// CHECK: vpunpckhqdq -1032(%rdx){1to8}, %zmm15, %zmm27
+// CHECK:  encoding: [0x62,0x61,0x85,0x58,0x6d,0x9a,0xf8,0xfb,0xff,0xff]
+          vpunpckhqdq -1032(%rdx){1to8}, %zmm15, %zmm27
+
+// CHECK: vgetexpss %xmm26, %xmm1, %xmm20
+// CHECK:  encoding: [0x62,0x82,0x75,0x08,0x43,0xe2]
+          vgetexpss %xmm26, %xmm1, %xmm20
+
+// CHECK: vgetexpss %xmm26, %xmm1, %xmm20 {%k7}
+// CHECK:  encoding: [0x62,0x82,0x75,0x0f,0x43,0xe2]
+          vgetexpss %xmm26, %xmm1, %xmm20 {%k7}
+
+// CHECK: vgetexpss %xmm26, %xmm1, %xmm20 {%k7} {z}
+// CHECK:  encoding: [0x62,0x82,0x75,0x8f,0x43,0xe2]
+          vgetexpss %xmm26, %xmm1, %xmm20 {%k7} {z}
+
+// CHECK: vgetexpss {sae}, %xmm26, %xmm1, %xmm20
+// CHECK:  encoding: [0x62,0x82,0x75,0x18,0x43,0xe2]
+          vgetexpss {sae}, %xmm26, %xmm1, %xmm20
+
+// CHECK: vgetexpss (%rcx), %xmm1, %xmm20
+// CHECK:  encoding: [0x62,0xe2,0x75,0x08,0x43,0x21]
+          vgetexpss (%rcx), %xmm1, %xmm20
+
+// CHECK: vgetexpss 291(%rax,%r14,8), %xmm1, %xmm20
+// CHECK:  encoding: [0x62,0xa2,0x75,0x08,0x43,0xa4,0xf0,0x23,0x01,0x00,0x00]
+          vgetexpss 291(%rax,%r14,8), %xmm1, %xmm20
+
+// CHECK: vgetexpss 508(%rdx), %xmm1, %xmm20
+// CHECK:  encoding: [0x62,0xe2,0x75,0x08,0x43,0x62,0x7f]
+          vgetexpss 508(%rdx), %xmm1, %xmm20
+
+// CHECK: vgetexpss 512(%rdx), %xmm1, %xmm20
+// CHECK:  encoding: [0x62,0xe2,0x75,0x08,0x43,0xa2,0x00,0x02,0x00,0x00]
+          vgetexpss 512(%rdx), %xmm1, %xmm20
+
+// CHECK: vgetexpss -512(%rdx), %xmm1, %xmm20
+// CHECK:  encoding: [0x62,0xe2,0x75,0x08,0x43,0x62,0x80]
+          vgetexpss -512(%rdx), %xmm1, %xmm20
+
+// CHECK: vgetexpss -516(%rdx), %xmm1, %xmm20
+// CHECK:  encoding: [0x62,0xe2,0x75,0x08,0x43,0xa2,0xfc,0xfd,0xff,0xff]
+          vgetexpss -516(%rdx), %xmm1, %xmm20
+
+// CHECK: vgetexpsd %xmm2, %xmm7, %xmm2
+// CHECK:  encoding: [0x62,0xf2,0xc5,0x08,0x43,0xd2]
+          vgetexpsd %xmm2, %xmm7, %xmm2
+
+// CHECK: vgetexpsd %xmm2, %xmm7, %xmm2 {%k5}
+// CHECK:  encoding: [0x62,0xf2,0xc5,0x0d,0x43,0xd2]
+          vgetexpsd %xmm2, %xmm7, %xmm2 {%k5}
+
+// CHECK: vgetexpsd %xmm2, %xmm7, %xmm2 {%k5} {z}
+// CHECK:  encoding: [0x62,0xf2,0xc5,0x8d,0x43,0xd2]
+          vgetexpsd %xmm2, %xmm7, %xmm2 {%k5} {z}
+
+// CHECK: vgetexpsd {sae}, %xmm2, %xmm7, %xmm2
+// CHECK:  encoding: [0x62,0xf2,0xc5,0x18,0x43,0xd2]
+          vgetexpsd {sae}, %xmm2, %xmm7, %xmm2
+
+// CHECK: vgetexpsd (%rcx), %xmm7, %xmm2
+// CHECK:  encoding: [0x62,0xf2,0xc5,0x08,0x43,0x11]
+          vgetexpsd (%rcx), %xmm7, %xmm2
+
+// CHECK: vgetexpsd 291(%rax,%r14,8), %xmm7, %xmm2
+// CHECK:  encoding: [0x62,0xb2,0xc5,0x08,0x43,0x94,0xf0,0x23,0x01,0x00,0x00]
+          vgetexpsd 291(%rax,%r14,8), %xmm7, %xmm2
+
+// CHECK: vgetexpsd 1016(%rdx), %xmm7, %xmm2
+// CHECK:  encoding: [0x62,0xf2,0xc5,0x08,0x43,0x52,0x7f]
+          vgetexpsd 1016(%rdx), %xmm7, %xmm2
+
+// CHECK: vgetexpsd 1024(%rdx), %xmm7, %xmm2
+// CHECK:  encoding: [0x62,0xf2,0xc5,0x08,0x43,0x92,0x00,0x04,0x00,0x00]
+          vgetexpsd 1024(%rdx), %xmm7, %xmm2
+
+// CHECK: vgetexpsd -1024(%rdx), %xmm7, %xmm2
+// CHECK:  encoding: [0x62,0xf2,0xc5,0x08,0x43,0x52,0x80]
+          vgetexpsd -1024(%rdx), %xmm7, %xmm2
+
+// CHECK: vgetexpsd -1032(%rdx), %xmm7, %xmm2
+// CHECK:  encoding: [0x62,0xf2,0xc5,0x08,0x43,0x92,0xf8,0xfb,0xff,0xff]
+          vgetexpsd -1032(%rdx), %xmm7, %xmm2
+
diff --git a/test/MC/X86/avx512-err.s b/test/MC/X86/avx512-err.s
new file mode 100644
index 00000000000..bfe6e39ee31
--- /dev/null
+++ b/test/MC/X86/avx512-err.s
@@ -0,0 +1,6 @@
+// RUN: not llvm-mc -triple x86_64-unknown-unknown -mcpu=knl -mattr=+avx512dq --show-encoding %s 2> %t.err
+// RUN: FileCheck --check-prefix=ERR < %t.err %s
+
+// ERR: invalid operand for instruction
+vpcmpd $1, %zmm24, %zmm7, %k5{%k0}
+
diff --git a/test/MC/X86/avx512bw-encoding.s b/test/MC/X86/avx512bw-encoding.s
index 06397fbebdf..954e0bd6e81 100644
--- a/test/MC/X86/avx512bw-encoding.s
+++ b/test/MC/X86/avx512bw-encoding.s
@@ -71,3 +71,212 @@
 // CHECK: vpblendmw -8256(%rdx), %zmm20, %zmm26
 // CHECK:  encoding: [0x62,0x62,0xdd,0x40,0x66,0x92,0xc0,0xdf,0xff,0xff]
           vpblendmw -8256(%rdx), %zmm20, %zmm26
+
+// CHECK: vptestmb %zmm19, %zmm17, %k5
+// CHECK:  encoding: [0x62,0xb2,0x75,0x40,0x26,0xeb]
+          vptestmb %zmm19, %zmm17, %k5
+
+// CHECK: vptestmb %zmm19, %zmm17, %k5 {%k3}
+// CHECK:  encoding: [0x62,0xb2,0x75,0x43,0x26,0xeb]
+          vptestmb %zmm19, %zmm17, %k5 {%k3}
+
+// CHECK: vptestmb (%rcx), %zmm17, %k5
+// CHECK:  encoding: [0x62,0xf2,0x75,0x40,0x26,0x29]
+          vptestmb (%rcx), %zmm17, %k5
+
+// CHECK: vptestmb 291(%rax,%r14,8), %zmm17, %k5
+// CHECK:  encoding: [0x62,0xb2,0x75,0x40,0x26,0xac,0xf0,0x23,0x01,0x00,0x00]
+          vptestmb 291(%rax,%r14,8), %zmm17, %k5
+
+// CHECK: vptestmb 8128(%rdx), %zmm17, %k5
+// CHECK:  encoding: [0x62,0xf2,0x75,0x40,0x26,0x6a,0x7f]
+          vptestmb 8128(%rdx), %zmm17, %k5
+
+// CHECK: vptestmb 8192(%rdx), %zmm17, %k5
+// CHECK:  encoding: [0x62,0xf2,0x75,0x40,0x26,0xaa,0x00,0x20,0x00,0x00]
+          vptestmb 8192(%rdx), %zmm17, %k5
+
+// CHECK: vptestmb -8192(%rdx), %zmm17, %k5
+// CHECK:  encoding: [0x62,0xf2,0x75,0x40,0x26,0x6a,0x80]
+          vptestmb -8192(%rdx), %zmm17, %k5
+
+// CHECK: vptestmb -8256(%rdx), %zmm17, %k5
+// CHECK:  encoding: [0x62,0xf2,0x75,0x40,0x26,0xaa,0xc0,0xdf,0xff,0xff]
+          vptestmb -8256(%rdx), %zmm17, %k5
+
+// CHECK: vptestmw %zmm19, %zmm29, %k4
+// CHECK:  encoding: [0x62,0xb2,0x95,0x40,0x26,0xe3]
+          vptestmw %zmm19, %zmm29, %k4
+
+// CHECK: vptestmw %zmm19, %zmm29, %k4 {%k2}
+// CHECK:  encoding: [0x62,0xb2,0x95,0x42,0x26,0xe3]
+          vptestmw %zmm19, %zmm29, %k4 {%k2}
+
+// CHECK: vptestmw (%rcx), %zmm29, %k4
+// CHECK:  encoding: [0x62,0xf2,0x95,0x40,0x26,0x21]
+          vptestmw (%rcx), %zmm29, %k4
+
+// CHECK: vptestmw 291(%rax,%r14,8), %zmm29, %k4
+// CHECK:  encoding: [0x62,0xb2,0x95,0x40,0x26,0xa4,0xf0,0x23,0x01,0x00,0x00]
+          vptestmw 291(%rax,%r14,8), %zmm29, %k4
+
+// CHECK: vptestmw 8128(%rdx), %zmm29, %k4
+// CHECK:  encoding: [0x62,0xf2,0x95,0x40,0x26,0x62,0x7f]
+          vptestmw 8128(%rdx), %zmm29, %k4
+
+// CHECK: vptestmw 8192(%rdx), %zmm29, %k4
+// CHECK:  encoding: [0x62,0xf2,0x95,0x40,0x26,0xa2,0x00,0x20,0x00,0x00]
+          vptestmw 8192(%rdx), %zmm29, %k4
+
+// CHECK: vptestmw -8192(%rdx), %zmm29, %k4
+// CHECK:  encoding: [0x62,0xf2,0x95,0x40,0x26,0x62,0x80]
+          vptestmw -8192(%rdx), %zmm29, %k4
+
+// CHECK: vptestmw -8256(%rdx), %zmm29, %k4
+// CHECK:  encoding: [0x62,0xf2,0x95,0x40,0x26,0xa2,0xc0,0xdf,0xff,0xff]
+          vptestmw -8256(%rdx), %zmm29, %k4
+
+// CHECK: vptestnmb %zmm23, %zmm24, %k2
+// CHECK:  encoding: [0x62,0xb2,0x3e,0x40,0x26,0xd7]
+          vptestnmb %zmm23, %zmm24, %k2
+
+// CHECK: vptestnmb %zmm23, %zmm24, %k2 {%k7}
+// CHECK:  encoding: [0x62,0xb2,0x3e,0x47,0x26,0xd7]
+          vptestnmb %zmm23, %zmm24, %k2 {%k7}
+
+// CHECK: vptestnmb (%rcx), %zmm24, %k2
+// CHECK:  encoding: [0x62,0xf2,0x3e,0x40,0x26,0x11]
+          vptestnmb (%rcx), %zmm24, %k2
+
+// CHECK: vptestnmb 291(%rax,%r14,8), %zmm24, %k2
+// CHECK:  encoding: [0x62,0xb2,0x3e,0x40,0x26,0x94,0xf0,0x23,0x01,0x00,0x00]
+          vptestnmb 291(%rax,%r14,8), %zmm24, %k2
+
+// CHECK: vptestnmb 8128(%rdx), %zmm24, %k2
+// CHECK:  encoding: [0x62,0xf2,0x3e,0x40,0x26,0x52,0x7f]
+          vptestnmb 8128(%rdx), %zmm24, %k2
+
+// CHECK: vptestnmb 8192(%rdx), %zmm24, %k2
+// CHECK:  encoding: [0x62,0xf2,0x3e,0x40,0x26,0x92,0x00,0x20,0x00,0x00]
+          vptestnmb 8192(%rdx), %zmm24, %k2
+
+// CHECK: vptestnmb -8192(%rdx), %zmm24, %k2
+// CHECK:  encoding: [0x62,0xf2,0x3e,0x40,0x26,0x52,0x80]
+          vptestnmb -8192(%rdx), %zmm24, %k2
+
+// CHECK: vptestnmb -8256(%rdx), %zmm24, %k2
+// CHECK:  encoding: [0x62,0xf2,0x3e,0x40,0x26,0x92,0xc0,0xdf,0xff,0xff]
+          vptestnmb -8256(%rdx), %zmm24, %k2
+
+// CHECK: vptestnmw %zmm27, %zmm18, %k4
+// CHECK:  encoding: [0x62,0x92,0xee,0x40,0x26,0xe3]
+          vptestnmw %zmm27, %zmm18, %k4
+
+// CHECK: vptestnmw %zmm27, %zmm18, %k4 {%k5}
+// CHECK:  encoding: [0x62,0x92,0xee,0x45,0x26,0xe3]
+          vptestnmw %zmm27, %zmm18, %k4 {%k5}
+
+// CHECK: vptestnmw (%rcx), %zmm18, %k4
+// CHECK:  encoding: [0x62,0xf2,0xee,0x40,0x26,0x21]
+          vptestnmw (%rcx), %zmm18, %k4
+
+// CHECK: vptestnmw 291(%rax,%r14,8), %zmm18, %k4
+// CHECK:  encoding: [0x62,0xb2,0xee,0x40,0x26,0xa4,0xf0,0x23,0x01,0x00,0x00]
+          vptestnmw 291(%rax,%r14,8), %zmm18, %k4
+
+// CHECK: vptestnmw 8128(%rdx), %zmm18, %k4
+// CHECK:  encoding: [0x62,0xf2,0xee,0x40,0x26,0x62,0x7f]
+          vptestnmw 8128(%rdx), %zmm18, %k4
+
+// CHECK: vptestnmw 8192(%rdx), %zmm18, %k4
+// CHECK:  encoding: [0x62,0xf2,0xee,0x40,0x26,0xa2,0x00,0x20,0x00,0x00]
+          vptestnmw 8192(%rdx), %zmm18, %k4
+
+// CHECK: vptestnmw -8192(%rdx), %zmm18, %k4
+// CHECK:  encoding: [0x62,0xf2,0xee,0x40,0x26,0x62,0x80]
+          vptestnmw -8192(%rdx), %zmm18, %k4
+
+// CHECK: vptestnmw -8256(%rdx), %zmm18, %k4
+// CHECK:  encoding: [0x62,0xf2,0xee,0x40,0x26,0xa2,0xc0,0xdf,0xff,0xff]
+          vptestnmw -8256(%rdx), %zmm18, %k4
+
+// CHECK: vptestnmb %zmm19, %zmm27, %k3
+// CHECK:  encoding: [0x62,0xb2,0x26,0x40,0x26,0xdb]
+          vptestnmb %zmm19, %zmm27, %k3
+
+// CHECK: vptestnmb %zmm19, %zmm27, %k3 {%k2}
+// CHECK:  encoding: [0x62,0xb2,0x26,0x42,0x26,0xdb]
+          vptestnmb %zmm19, %zmm27, %k3 {%k2}
+
+// CHECK: vptestnmb (%rcx), %zmm27, %k3
+// CHECK:  encoding: [0x62,0xf2,0x26,0x40,0x26,0x19]
+          vptestnmb (%rcx), %zmm27, %k3
+
+// CHECK: vptestnmb 4660(%rax,%r14,8), %zmm27, %k3
+// CHECK:  encoding: [0x62,0xb2,0x26,0x40,0x26,0x9c,0xf0,0x34,0x12,0x00,0x00]
+          vptestnmb 4660(%rax,%r14,8), %zmm27, %k3
+
+// CHECK: vptestnmb 8128(%rdx), %zmm27, %k3
+// CHECK:  encoding: [0x62,0xf2,0x26,0x40,0x26,0x5a,0x7f]
+          vptestnmb 8128(%rdx), %zmm27, %k3
+
+// CHECK: vptestnmb 8192(%rdx), %zmm27, %k3
+// CHECK:  encoding: [0x62,0xf2,0x26,0x40,0x26,0x9a,0x00,0x20,0x00,0x00]
+          vptestnmb 8192(%rdx), %zmm27, %k3
+
+// CHECK: vptestnmb -8192(%rdx), %zmm27, %k3
+// CHECK:  encoding: [0x62,0xf2,0x26,0x40,0x26,0x5a,0x80]
+          vptestnmb -8192(%rdx), %zmm27, %k3
+
+// CHECK: vptestnmb -8256(%rdx), %zmm27, %k3
+// CHECK:  encoding: [0x62,0xf2,0x26,0x40,0x26,0x9a,0xc0,0xdf,0xff,0xff]
+          vptestnmb -8256(%rdx), %zmm27, %k3
+
+// CHECK: vptestnmw %zmm21, %zmm17, %k2
+// CHECK:  encoding: [0x62,0xb2,0xf6,0x40,0x26,0xd5]
+          vptestnmw %zmm21, %zmm17, %k2
+
+// CHECK: vptestnmw %zmm21, %zmm17, %k2 {%k4}
+// CHECK:  encoding: [0x62,0xb2,0xf6,0x44,0x26,0xd5]
+          vptestnmw %zmm21, %zmm17, %k2 {%k4}
+
+// CHECK: vptestnmw (%rcx), %zmm17, %k2
+// CHECK:  encoding: [0x62,0xf2,0xf6,0x40,0x26,0x11]
+          vptestnmw (%rcx), %zmm17, %k2
+
+// CHECK: vptestnmw 4660(%rax,%r14,8), %zmm17, %k2
+// CHECK:  encoding: [0x62,0xb2,0xf6,0x40,0x26,0x94,0xf0,0x34,0x12,0x00,0x00]
+          vptestnmw 4660(%rax,%r14,8), %zmm17, %k2
+
+// CHECK: vptestnmw 8128(%rdx), %zmm17, %k2
+// CHECK:  encoding: [0x62,0xf2,0xf6,0x40,0x26,0x52,0x7f]
+          vptestnmw 8128(%rdx), %zmm17, %k2
+
+// CHECK: vptestnmw 8192(%rdx), %zmm17, %k2
+// CHECK:  encoding: [0x62,0xf2,0xf6,0x40,0x26,0x92,0x00,0x20,0x00,0x00]
+          vptestnmw 8192(%rdx), %zmm17, %k2
+
+// CHECK: vptestnmw -8192(%rdx), %zmm17, %k2
+// CHECK:  encoding: [0x62,0xf2,0xf6,0x40,0x26,0x52,0x80]
+          vptestnmw -8192(%rdx), %zmm17, %k2
+
+// CHECK: vptestnmw -8256(%rdx), %zmm17, %k2
+// CHECK:  encoding: [0x62,0xf2,0xf6,0x40,0x26,0x92,0xc0,0xdf,0xff,0xff]
+          vptestnmw -8256(%rdx), %zmm17, %k2
+
+// CHECK: vpmovb2m %zmm28, %k5
+// CHECK:  encoding: [0x62,0x92,0x7e,0x48,0x29,0xec]
+          vpmovb2m %zmm28, %k5
+
+// CHECK: vpmovw2m %zmm30, %k3
+// CHECK:  encoding: [0x62,0x92,0xfe,0x48,0x29,0xde]
+          vpmovw2m %zmm30, %k3
+
+// CHECK: vpmovm2b %k3, %zmm18
+// CHECK:  encoding: [0x62,0xe2,0x7e,0x48,0x28,0xd3]
+          vpmovm2b %k3, %zmm18
+
+// CHECK: vpmovm2w %k5, %zmm24
+// CHECK:  encoding: [0x62,0x62,0xfe,0x48,0x28,0xc5]
+          vpmovm2w %k5, %zmm24
+
diff --git a/test/MC/X86/avx512vl-encoding.s b/test/MC/X86/avx512vl-encoding.s
index acea70cb12a..2e3eaf2aa4c 100644
--- a/test/MC/X86/avx512vl-encoding.s
+++ b/test/MC/X86/avx512vl-encoding.s
@@ -447,3 +447,1536 @@
 // CHECK: vpblendmq -1032(%rdx){1to4}, %ymm23, %ymm21
 // CHECK:  encoding: [0x62,0xe2,0xc5,0x30,0x64,0xaa,0xf8,0xfb,0xff,0xff]
           vpblendmq -1032(%rdx){1to4}, %ymm23, %ymm21
+
+// CHECK: vptestmd %xmm20, %xmm20, %k2
+// CHECK:  encoding: [0x62,0xb2,0x5d,0x00,0x27,0xd4]
+          vptestmd %xmm20, %xmm20, %k2
+
+// CHECK: vptestmd %xmm20, %xmm20, %k2 {%k7}
+// CHECK:  encoding: [0x62,0xb2,0x5d,0x07,0x27,0xd4]
+          vptestmd %xmm20, %xmm20, %k2 {%k7}
+
+// CHECK: vptestmd (%rcx), %xmm20, %k2
+// CHECK:  encoding: [0x62,0xf2,0x5d,0x00,0x27,0x11]
+          vptestmd (%rcx), %xmm20, %k2
+
+// CHECK: vptestmd 291(%rax,%r14,8), %xmm20, %k2
+// CHECK:  encoding: [0x62,0xb2,0x5d,0x00,0x27,0x94,0xf0,0x23,0x01,0x00,0x00]
+          vptestmd 291(%rax,%r14,8), %xmm20, %k2
+
+// CHECK: vptestmd (%rcx){1to4}, %xmm20, %k2
+// CHECK:  encoding: [0x62,0xf2,0x5d,0x10,0x27,0x11]
+          vptestmd (%rcx){1to4}, %xmm20, %k2
+
+// CHECK: vptestmd 2032(%rdx), %xmm20, %k2
+// CHECK:  encoding: [0x62,0xf2,0x5d,0x00,0x27,0x52,0x7f]
+          vptestmd 2032(%rdx), %xmm20, %k2
+
+// CHECK: vptestmd 2048(%rdx), %xmm20, %k2
+// CHECK:  encoding: [0x62,0xf2,0x5d,0x00,0x27,0x92,0x00,0x08,0x00,0x00]
+          vptestmd 2048(%rdx), %xmm20, %k2
+
+// CHECK: vptestmd -2048(%rdx), %xmm20, %k2
+// CHECK:  encoding: [0x62,0xf2,0x5d,0x00,0x27,0x52,0x80]
+          vptestmd -2048(%rdx), %xmm20, %k2
+
+// CHECK: vptestmd -2064(%rdx), %xmm20, %k2
+// CHECK:  encoding: [0x62,0xf2,0x5d,0x00,0x27,0x92,0xf0,0xf7,0xff,0xff]
+          vptestmd -2064(%rdx), %xmm20, %k2
+
+// CHECK: vptestmd 508(%rdx){1to4}, %xmm20, %k2
+// CHECK:  encoding: [0x62,0xf2,0x5d,0x10,0x27,0x52,0x7f]
+          vptestmd 508(%rdx){1to4}, %xmm20, %k2
+
+// CHECK: vptestmd 512(%rdx){1to4}, %xmm20, %k2
+// CHECK:  encoding: [0x62,0xf2,0x5d,0x10,0x27,0x92,0x00,0x02,0x00,0x00]
+          vptestmd 512(%rdx){1to4}, %xmm20, %k2
+
+// CHECK: vptestmd -512(%rdx){1to4}, %xmm20, %k2
+// CHECK:  encoding: [0x62,0xf2,0x5d,0x10,0x27,0x52,0x80]
+          vptestmd -512(%rdx){1to4}, %xmm20, %k2
+
+// CHECK: vptestmd -516(%rdx){1to4}, %xmm20, %k2
+// CHECK:  encoding: [0x62,0xf2,0x5d,0x10,0x27,0x92,0xfc,0xfd,0xff,0xff]
+          vptestmd -516(%rdx){1to4}, %xmm20, %k2
+
+// CHECK: vptestmd %ymm17, %ymm20, %k3
+// CHECK:  encoding: [0x62,0xb2,0x5d,0x20,0x27,0xd9]
+          vptestmd %ymm17, %ymm20, %k3
+
+// CHECK: vptestmd %ymm17, %ymm20, %k3 {%k5}
+// CHECK:  encoding: [0x62,0xb2,0x5d,0x25,0x27,0xd9]
+          vptestmd %ymm17, %ymm20, %k3 {%k5}
+
+// CHECK: vptestmd (%rcx), %ymm20, %k3
+// CHECK:  encoding: [0x62,0xf2,0x5d,0x20,0x27,0x19]
+          vptestmd (%rcx), %ymm20, %k3
+
+// CHECK: vptestmd 291(%rax,%r14,8), %ymm20, %k3
+// CHECK:  encoding: [0x62,0xb2,0x5d,0x20,0x27,0x9c,0xf0,0x23,0x01,0x00,0x00]
+          vptestmd 291(%rax,%r14,8), %ymm20, %k3
+
+// CHECK: vptestmd (%rcx){1to8}, %ymm20, %k3
+// CHECK:  encoding: [0x62,0xf2,0x5d,0x30,0x27,0x19]
+          vptestmd (%rcx){1to8}, %ymm20, %k3
+
+// CHECK: vptestmd 4064(%rdx), %ymm20, %k3
+// CHECK:  encoding: [0x62,0xf2,0x5d,0x20,0x27,0x5a,0x7f]
+          vptestmd 4064(%rdx), %ymm20, %k3
+
+// CHECK: vptestmd 4096(%rdx), %ymm20, %k3
+// CHECK:  encoding: [0x62,0xf2,0x5d,0x20,0x27,0x9a,0x00,0x10,0x00,0x00]
+          vptestmd 4096(%rdx), %ymm20, %k3
+
+// CHECK: vptestmd -4096(%rdx), %ymm20, %k3
+// CHECK:  encoding: [0x62,0xf2,0x5d,0x20,0x27,0x5a,0x80]
+          vptestmd -4096(%rdx), %ymm20, %k3
+
+// CHECK: vptestmd -4128(%rdx), %ymm20, %k3
+// CHECK:  encoding: [0x62,0xf2,0x5d,0x20,0x27,0x9a,0xe0,0xef,0xff,0xff]
+          vptestmd -4128(%rdx), %ymm20, %k3
+
+// CHECK: vptestmd 508(%rdx){1to8}, %ymm20, %k3
+// CHECK:  encoding: [0x62,0xf2,0x5d,0x30,0x27,0x5a,0x7f]
+          vptestmd 508(%rdx){1to8}, %ymm20, %k3
+
+// CHECK: vptestmd 512(%rdx){1to8}, %ymm20, %k3
+// CHECK:  encoding: [0x62,0xf2,0x5d,0x30,0x27,0x9a,0x00,0x02,0x00,0x00]
+          vptestmd 512(%rdx){1to8}, %ymm20, %k3
+
+// CHECK: vptestmd -512(%rdx){1to8}, %ymm20, %k3
+// CHECK:  encoding: [0x62,0xf2,0x5d,0x30,0x27,0x5a,0x80]
+          vptestmd -512(%rdx){1to8}, %ymm20, %k3
+
+// CHECK: vptestmd -516(%rdx){1to8}, %ymm20, %k3
+// CHECK:  encoding: [0x62,0xf2,0x5d,0x30,0x27,0x9a,0xfc,0xfd,0xff,0xff]
+          vptestmd -516(%rdx){1to8}, %ymm20, %k3
+
+// CHECK: vptestmq %xmm28, %xmm22, %k4
+// CHECK:  encoding: [0x62,0x92,0xcd,0x00,0x27,0xe4]
+          vptestmq %xmm28, %xmm22, %k4
+
+// CHECK: vptestmq %xmm28, %xmm22, %k4 {%k3}
+// CHECK:  encoding: [0x62,0x92,0xcd,0x03,0x27,0xe4]
+          vptestmq %xmm28, %xmm22, %k4 {%k3}
+
+// CHECK: vptestmq (%rcx), %xmm22, %k4
+// CHECK:  encoding: [0x62,0xf2,0xcd,0x00,0x27,0x21]
+          vptestmq (%rcx), %xmm22, %k4
+
+// CHECK: vptestmq 291(%rax,%r14,8), %xmm22, %k4
+// CHECK:  encoding: [0x62,0xb2,0xcd,0x00,0x27,0xa4,0xf0,0x23,0x01,0x00,0x00]
+          vptestmq 291(%rax,%r14,8), %xmm22, %k4
+
+// CHECK: vptestmq (%rcx){1to2}, %xmm22, %k4
+// CHECK:  encoding: [0x62,0xf2,0xcd,0x10,0x27,0x21]
+          vptestmq (%rcx){1to2}, %xmm22, %k4
+
+// CHECK: vptestmq 2032(%rdx), %xmm22, %k4
+// CHECK:  encoding: [0x62,0xf2,0xcd,0x00,0x27,0x62,0x7f]
+          vptestmq 2032(%rdx), %xmm22, %k4
+
+// CHECK: vptestmq 2048(%rdx), %xmm22, %k4
+// CHECK:  encoding: [0x62,0xf2,0xcd,0x00,0x27,0xa2,0x00,0x08,0x00,0x00]
+          vptestmq 2048(%rdx), %xmm22, %k4
+
+// CHECK: vptestmq -2048(%rdx), %xmm22, %k4
+// CHECK:  encoding: [0x62,0xf2,0xcd,0x00,0x27,0x62,0x80]
+          vptestmq -2048(%rdx), %xmm22, %k4
+
+// CHECK: vptestmq -2064(%rdx), %xmm22, %k4
+// CHECK:  encoding: [0x62,0xf2,0xcd,0x00,0x27,0xa2,0xf0,0xf7,0xff,0xff]
+          vptestmq -2064(%rdx), %xmm22, %k4
+
+// CHECK: vptestmq 1016(%rdx){1to2}, %xmm22, %k4
+// CHECK:  encoding: [0x62,0xf2,0xcd,0x10,0x27,0x62,0x7f]
+          vptestmq 1016(%rdx){1to2}, %xmm22, %k4
+
+// CHECK: vptestmq 1024(%rdx){1to2}, %xmm22, %k4
+// CHECK:  encoding: [0x62,0xf2,0xcd,0x10,0x27,0xa2,0x00,0x04,0x00,0x00]
+          vptestmq 1024(%rdx){1to2}, %xmm22, %k4
+
+// CHECK: vptestmq -1024(%rdx){1to2}, %xmm22, %k4
+// CHECK:  encoding: [0x62,0xf2,0xcd,0x10,0x27,0x62,0x80]
+          vptestmq -1024(%rdx){1to2}, %xmm22, %k4
+
+// CHECK: vptestmq -1032(%rdx){1to2}, %xmm22, %k4
+// CHECK:  encoding: [0x62,0xf2,0xcd,0x10,0x27,0xa2,0xf8,0xfb,0xff,0xff]
+          vptestmq -1032(%rdx){1to2}, %xmm22, %k4
+
+// CHECK: vptestmq %ymm20, %ymm21, %k3
+// CHECK:  encoding: [0x62,0xb2,0xd5,0x20,0x27,0xdc]
+          vptestmq %ymm20, %ymm21, %k3
+
+// CHECK: vptestmq %ymm20, %ymm21, %k3 {%k7}
+// CHECK:  encoding: [0x62,0xb2,0xd5,0x27,0x27,0xdc]
+          vptestmq %ymm20, %ymm21, %k3 {%k7}
+
+// CHECK: vptestmq (%rcx), %ymm21, %k3
+// CHECK:  encoding: [0x62,0xf2,0xd5,0x20,0x27,0x19]
+          vptestmq (%rcx), %ymm21, %k3
+
+// CHECK: vptestmq 291(%rax,%r14,8), %ymm21, %k3
+// CHECK:  encoding: [0x62,0xb2,0xd5,0x20,0x27,0x9c,0xf0,0x23,0x01,0x00,0x00]
+          vptestmq 291(%rax,%r14,8), %ymm21, %k3
+
+// CHECK: vptestmq (%rcx){1to4}, %ymm21, %k3
+// CHECK:  encoding: [0x62,0xf2,0xd5,0x30,0x27,0x19]
+          vptestmq (%rcx){1to4}, %ymm21, %k3
+
+// CHECK: vptestmq 4064(%rdx), %ymm21, %k3
+// CHECK:  encoding: [0x62,0xf2,0xd5,0x20,0x27,0x5a,0x7f]
+          vptestmq 4064(%rdx), %ymm21, %k3
+
+// CHECK: vptestmq 4096(%rdx), %ymm21, %k3
+// CHECK:  encoding: [0x62,0xf2,0xd5,0x20,0x27,0x9a,0x00,0x10,0x00,0x00]
+          vptestmq 4096(%rdx), %ymm21, %k3
+
+// CHECK: vptestmq -4096(%rdx), %ymm21, %k3
+// CHECK:  encoding: [0x62,0xf2,0xd5,0x20,0x27,0x5a,0x80]
+          vptestmq -4096(%rdx), %ymm21, %k3
+
+// CHECK: vptestmq -4128(%rdx), %ymm21, %k3
+// CHECK:  encoding: [0x62,0xf2,0xd5,0x20,0x27,0x9a,0xe0,0xef,0xff,0xff]
+          vptestmq -4128(%rdx), %ymm21, %k3
+
+// CHECK: vptestmq 1016(%rdx){1to4}, %ymm21, %k3
+// CHECK:  encoding: [0x62,0xf2,0xd5,0x30,0x27,0x5a,0x7f]
+          vptestmq 1016(%rdx){1to4}, %ymm21, %k3
+
+// CHECK: vptestmq 1024(%rdx){1to4}, %ymm21, %k3
+// CHECK:  encoding: [0x62,0xf2,0xd5,0x30,0x27,0x9a,0x00,0x04,0x00,0x00]
+          vptestmq 1024(%rdx){1to4}, %ymm21, %k3
+
+// CHECK: vptestmq -1024(%rdx){1to4}, %ymm21, %k3
+// CHECK:  encoding: [0x62,0xf2,0xd5,0x30,0x27,0x5a,0x80]
+          vptestmq -1024(%rdx){1to4}, %ymm21, %k3
+
+// CHECK: vptestnmd %xmm22, %xmm20, %k3
+// CHECK:  encoding: [0x62,0xb2,0x5e,0x00,0x27,0xde]
+          vptestnmd %xmm22, %xmm20, %k3
+
+// CHECK: vptestnmd %xmm22, %xmm20, %k3 {%k7}
+// CHECK:  encoding: [0x62,0xb2,0x5e,0x07,0x27,0xde]
+          vptestnmd %xmm22, %xmm20, %k3 {%k7}
+
+// CHECK: vptestnmd (%rcx), %xmm20, %k3
+// CHECK:  encoding: [0x62,0xf2,0x5e,0x00,0x27,0x19]
+          vptestnmd (%rcx), %xmm20, %k3
+
+// CHECK: vptestnmd 291(%rax,%r14,8), %xmm20, %k3
+// CHECK:  encoding: [0x62,0xb2,0x5e,0x00,0x27,0x9c,0xf0,0x23,0x01,0x00,0x00]
+          vptestnmd 291(%rax,%r14,8), %xmm20, %k3
+
+// CHECK: vptestnmd (%rcx){1to4}, %xmm20, %k3
+// CHECK:  encoding: [0x62,0xf2,0x5e,0x10,0x27,0x19]
+          vptestnmd (%rcx){1to4}, %xmm20, %k3
+
+// CHECK: vptestnmd 2032(%rdx), %xmm20, %k3
+// CHECK:  encoding: [0x62,0xf2,0x5e,0x00,0x27,0x5a,0x7f]
+          vptestnmd 2032(%rdx), %xmm20, %k3
+
+// CHECK: vptestnmd 2048(%rdx), %xmm20, %k3
+// CHECK:  encoding: [0x62,0xf2,0x5e,0x00,0x27,0x9a,0x00,0x08,0x00,0x00]
+          vptestnmd 2048(%rdx), %xmm20, %k3
+
+// CHECK: vptestnmd -2048(%rdx), %xmm20, %k3
+// CHECK:  encoding: [0x62,0xf2,0x5e,0x00,0x27,0x5a,0x80]
+          vptestnmd -2048(%rdx), %xmm20, %k3
+
+// CHECK: vptestnmd -2064(%rdx), %xmm20, %k3
+// CHECK:  encoding: [0x62,0xf2,0x5e,0x00,0x27,0x9a,0xf0,0xf7,0xff,0xff]
+          vptestnmd -2064(%rdx), %xmm20, %k3
+
+// CHECK: vptestnmd 508(%rdx){1to4}, %xmm20, %k3
+// CHECK:  encoding: [0x62,0xf2,0x5e,0x10,0x27,0x5a,0x7f]
+          vptestnmd 508(%rdx){1to4}, %xmm20, %k3
+
+// CHECK: vptestnmd 512(%rdx){1to4}, %xmm20, %k3
+// CHECK:  encoding: [0x62,0xf2,0x5e,0x10,0x27,0x9a,0x00,0x02,0x00,0x00]
+          vptestnmd 512(%rdx){1to4}, %xmm20, %k3
+
+// CHECK: vptestnmd -512(%rdx){1to4}, %xmm20, %k3
+// CHECK:  encoding: [0x62,0xf2,0x5e,0x10,0x27,0x5a,0x80]
+          vptestnmd -512(%rdx){1to4}, %xmm20, %k3
+
+// CHECK: vptestnmd -516(%rdx){1to4}, %xmm20, %k3
+// CHECK:  encoding: [0x62,0xf2,0x5e,0x10,0x27,0x9a,0xfc,0xfd,0xff,0xff]
+          vptestnmd -516(%rdx){1to4}, %xmm20, %k3
+
+// CHECK: vptestnmd %ymm23, %ymm21, %k3
+// CHECK:  encoding: [0x62,0xb2,0x56,0x20,0x27,0xdf]
+          vptestnmd %ymm23, %ymm21, %k3
+
+// CHECK: vptestnmd %ymm23, %ymm21, %k3 {%k7}
+// CHECK:  encoding: [0x62,0xb2,0x56,0x27,0x27,0xdf]
+          vptestnmd %ymm23, %ymm21, %k3 {%k7}
+
+// CHECK: vptestnmd (%rcx), %ymm21, %k3
+// CHECK:  encoding: [0x62,0xf2,0x56,0x20,0x27,0x19]
+          vptestnmd (%rcx), %ymm21, %k3
+
+// CHECK: vptestnmd 291(%rax,%r14,8), %ymm21, %k3
+// CHECK:  encoding: [0x62,0xb2,0x56,0x20,0x27,0x9c,0xf0,0x23,0x01,0x00,0x00]
+          vptestnmd 291(%rax,%r14,8), %ymm21, %k3
+
+// CHECK: vptestnmd (%rcx){1to8}, %ymm21, %k3
+// CHECK:  encoding: [0x62,0xf2,0x56,0x30,0x27,0x19]
+          vptestnmd (%rcx){1to8}, %ymm21, %k3
+
+// CHECK: vptestnmd 4064(%rdx), %ymm21, %k3
+// CHECK:  encoding: [0x62,0xf2,0x56,0x20,0x27,0x5a,0x7f]
+          vptestnmd 4064(%rdx), %ymm21, %k3
+
+// CHECK: vptestnmd 4096(%rdx), %ymm21, %k3
+// CHECK:  encoding: [0x62,0xf2,0x56,0x20,0x27,0x9a,0x00,0x10,0x00,0x00]
+          vptestnmd 4096(%rdx), %ymm21, %k3
+
+// CHECK: vptestnmd -4096(%rdx), %ymm21, %k3
+// CHECK:  encoding: [0x62,0xf2,0x56,0x20,0x27,0x5a,0x80]
+          vptestnmd -4096(%rdx), %ymm21, %k3
+
+// CHECK: vptestnmd -4128(%rdx), %ymm21, %k3
+// CHECK:  encoding: [0x62,0xf2,0x56,0x20,0x27,0x9a,0xe0,0xef,0xff,0xff]
+          vptestnmd -4128(%rdx), %ymm21, %k3
+
+// CHECK: vptestnmd 508(%rdx){1to8}, %ymm21, %k3
+// CHECK:  encoding: [0x62,0xf2,0x56,0x30,0x27,0x5a,0x7f]
+          vptestnmd 508(%rdx){1to8}, %ymm21, %k3
+
+// CHECK: vptestnmd 512(%rdx){1to8}, %ymm21, %k3
+// CHECK:  encoding: [0x62,0xf2,0x56,0x30,0x27,0x9a,0x00,0x02,0x00,0x00]
+          vptestnmd 512(%rdx){1to8}, %ymm21, %k3
+
+// CHECK: vptestnmd -512(%rdx){1to8}, %ymm21, %k3
+// CHECK:  encoding: [0x62,0xf2,0x56,0x30,0x27,0x5a,0x80]
+          vptestnmd -512(%rdx){1to8}, %ymm21, %k3
+
+// CHECK: vptestnmd -516(%rdx){1to8}, %ymm21, %k3
+// CHECK:  encoding: [0x62,0xf2,0x56,0x30,0x27,0x9a,0xfc,0xfd,0xff,0xff]
+          vptestnmd -516(%rdx){1to8}, %ymm21, %k3
+
+// CHECK: vptestnmq %xmm21, %xmm20, %k5
+// CHECK:  encoding: [0x62,0xb2,0xde,0x00,0x27,0xed]
+          vptestnmq %xmm21, %xmm20, %k5
+
+// CHECK: vptestnmq %xmm21, %xmm20, %k5 {%k5}
+// CHECK:  encoding: [0x62,0xb2,0xde,0x05,0x27,0xed]
+          vptestnmq %xmm21, %xmm20, %k5 {%k5}
+
+// CHECK: vptestnmq (%rcx), %xmm20, %k5
+// CHECK:  encoding: [0x62,0xf2,0xde,0x00,0x27,0x29]
+          vptestnmq (%rcx), %xmm20, %k5
+
+// CHECK: vptestnmq 291(%rax,%r14,8), %xmm20, %k5
+// CHECK:  encoding: [0x62,0xb2,0xde,0x00,0x27,0xac,0xf0,0x23,0x01,0x00,0x00]
+          vptestnmq 291(%rax,%r14,8), %xmm20, %k5
+
+// CHECK: vptestnmq (%rcx){1to2}, %xmm20, %k5
+// CHECK:  encoding: [0x62,0xf2,0xde,0x10,0x27,0x29]
+          vptestnmq (%rcx){1to2}, %xmm20, %k5
+
+// CHECK: vptestnmq 2032(%rdx), %xmm20, %k5
+// CHECK:  encoding: [0x62,0xf2,0xde,0x00,0x27,0x6a,0x7f]
+          vptestnmq 2032(%rdx), %xmm20, %k5
+
+// CHECK: vptestnmq 2048(%rdx), %xmm20, %k5
+// CHECK:  encoding: [0x62,0xf2,0xde,0x00,0x27,0xaa,0x00,0x08,0x00,0x00]
+          vptestnmq 2048(%rdx), %xmm20, %k5
+
+// CHECK: vptestnmq -2048(%rdx), %xmm20, %k5
+// CHECK:  encoding: [0x62,0xf2,0xde,0x00,0x27,0x6a,0x80]
+          vptestnmq -2048(%rdx), %xmm20, %k5
+
+// CHECK: vptestnmq -2064(%rdx), %xmm20, %k5
+// CHECK:  encoding: [0x62,0xf2,0xde,0x00,0x27,0xaa,0xf0,0xf7,0xff,0xff]
+          vptestnmq -2064(%rdx), %xmm20, %k5
+
+// CHECK: vptestnmq 1016(%rdx){1to2}, %xmm20, %k5
+// CHECK:  encoding: [0x62,0xf2,0xde,0x10,0x27,0x6a,0x7f]
+          vptestnmq 1016(%rdx){1to2}, %xmm20, %k5
+
+// CHECK: vptestnmq 1024(%rdx){1to2}, %xmm20, %k5
+// CHECK:  encoding: [0x62,0xf2,0xde,0x10,0x27,0xaa,0x00,0x04,0x00,0x00]
+          vptestnmq 1024(%rdx){1to2}, %xmm20, %k5
+
+// CHECK: vptestnmq -1024(%rdx){1to2}, %xmm20, %k5
+// CHECK:  encoding: [0x62,0xf2,0xde,0x10,0x27,0x6a,0x80]
+          vptestnmq -1024(%rdx){1to2}, %xmm20, %k5
+
+// CHECK: vptestnmq -1032(%rdx){1to2}, %xmm20, %k5
+// CHECK:  encoding: [0x62,0xf2,0xde,0x10,0x27,0xaa,0xf8,0xfb,0xff,0xff]
+          vptestnmq -1032(%rdx){1to2}, %xmm20, %k5
+
+// CHECK: vptestnmq %ymm21, %ymm24, %k4
+// CHECK:  encoding: [0x62,0xb2,0xbe,0x20,0x27,0xe5]
+          vptestnmq %ymm21, %ymm24, %k4
+
+// CHECK: vptestnmq %ymm21, %ymm24, %k4 {%k3}
+// CHECK:  encoding: [0x62,0xb2,0xbe,0x23,0x27,0xe5]
+          vptestnmq %ymm21, %ymm24, %k4 {%k3}
+
+// CHECK: vptestnmq (%rcx), %ymm24, %k4
+// CHECK:  encoding: [0x62,0xf2,0xbe,0x20,0x27,0x21]
+          vptestnmq (%rcx), %ymm24, %k4
+
+// CHECK: vptestnmq 291(%rax,%r14,8), %ymm24, %k4
+// CHECK:  encoding: [0x62,0xb2,0xbe,0x20,0x27,0xa4,0xf0,0x23,0x01,0x00,0x00]
+          vptestnmq 291(%rax,%r14,8), %ymm24, %k4
+
+// CHECK: vptestnmq (%rcx){1to4}, %ymm24, %k4
+// CHECK:  encoding: [0x62,0xf2,0xbe,0x30,0x27,0x21]
+          vptestnmq (%rcx){1to4}, %ymm24, %k4
+
+// CHECK: vptestnmq 4064(%rdx), %ymm24, %k4
+// CHECK:  encoding: [0x62,0xf2,0xbe,0x20,0x27,0x62,0x7f]
+          vptestnmq 4064(%rdx), %ymm24, %k4
+
+// CHECK: vptestnmq 4096(%rdx), %ymm24, %k4
+// CHECK:  encoding: [0x62,0xf2,0xbe,0x20,0x27,0xa2,0x00,0x10,0x00,0x00]
+          vptestnmq 4096(%rdx), %ymm24, %k4
+
+// CHECK: vptestnmq -4096(%rdx), %ymm24, %k4
+// CHECK:  encoding: [0x62,0xf2,0xbe,0x20,0x27,0x62,0x80]
+          vptestnmq -4096(%rdx), %ymm24, %k4
+
+// CHECK: vptestnmq -4128(%rdx), %ymm24, %k4
+// CHECK:  encoding: [0x62,0xf2,0xbe,0x20,0x27,0xa2,0xe0,0xef,0xff,0xff]
+          vptestnmq -4128(%rdx), %ymm24, %k4
+
+// CHECK: vptestnmq 1016(%rdx){1to4}, %ymm24, %k4
+// CHECK:  encoding: [0x62,0xf2,0xbe,0x30,0x27,0x62,0x7f]
+          vptestnmq 1016(%rdx){1to4}, %ymm24, %k4
+
+// CHECK: vptestnmq 1024(%rdx){1to4}, %ymm24, %k4
+// CHECK:  encoding: [0x62,0xf2,0xbe,0x30,0x27,0xa2,0x00,0x04,0x00,0x00]
+          vptestnmq 1024(%rdx){1to4}, %ymm24, %k4
+
+// CHECK: vptestnmq -1024(%rdx){1to4}, %ymm24, %k4
+// CHECK:  encoding: [0x62,0xf2,0xbe,0x30,0x27,0x62,0x80]
+          vptestnmq -1024(%rdx){1to4}, %ymm24, %k4
+
+// CHECK: vptestnmq -1032(%rdx){1to4}, %ymm24, %k4
+// CHECK:  encoding: [0x62,0xf2,0xbe,0x30,0x27,0xa2,0xf8,0xfb,0xff,0xff]
+          vptestnmq -1032(%rdx){1to4}, %ymm24, %k4
+
+// CHECK: vptestnmq %xmm19, %xmm28, %k2
+// CHECK:  encoding: [0x62,0xb2,0x9e,0x00,0x27,0xd3]
+          vptestnmq %xmm19, %xmm28,%k2
+
+// CHECK: vptestnmq %xmm19, %xmm28, %k2 {%k1}
+// CHECK:  encoding: [0x62,0xb2,0x9e,0x01,0x27,0xd3]
+          vptestnmq %xmm19, %xmm28,%k2 {%k1}
+
+// CHECK: vptestnmq (%rcx), %xmm28, %k2
+// CHECK:  encoding: [0x62,0xf2,0x9e,0x00,0x27,0x11]
+          vptestnmq (%rcx), %xmm28,%k2
+
+// CHECK: vptestnmq 4660(%rax,%r14,8), %xmm28, %k2
+// CHECK:  encoding: [0x62,0xb2,0x9e,0x00,0x27,0x94,0xf0,0x34,0x12,0x00,0x00]
+          vptestnmq 4660(%rax,%r14,8), %xmm28,%k2
+
+// CHECK: vptestnmq (%rcx){1to2}, %xmm28, %k2
+// CHECK:  encoding: [0x62,0xf2,0x9e,0x10,0x27,0x11]
+          vptestnmq (%rcx){1to2}, %xmm28,%k2
+
+// CHECK: vptestnmq 2032(%rdx), %xmm28, %k2
+// CHECK:  encoding: [0x62,0xf2,0x9e,0x00,0x27,0x52,0x7f]
+          vptestnmq 2032(%rdx), %xmm28,%k2
+
+// CHECK: vptestnmq 2048(%rdx), %xmm28, %k2
+// CHECK:  encoding: [0x62,0xf2,0x9e,0x00,0x27,0x92,0x00,0x08,0x00,0x00]
+          vptestnmq 2048(%rdx), %xmm28,%k2
+
+// CHECK: vptestnmq -2048(%rdx), %xmm28, %k2
+// CHECK:  encoding: [0x62,0xf2,0x9e,0x00,0x27,0x52,0x80]
+          vptestnmq -2048(%rdx), %xmm28,%k2
+
+// CHECK: vptestnmq -2064(%rdx), %xmm28, %k2
+// CHECK:  encoding: [0x62,0xf2,0x9e,0x00,0x27,0x92,0xf0,0xf7,0xff,0xff]
+          vptestnmq -2064(%rdx), %xmm28,%k2
+
+// CHECK: vptestnmq 1016(%rdx){1to2}, %xmm28, %k2
+// CHECK:  encoding: [0x62,0xf2,0x9e,0x10,0x27,0x52,0x7f]
+          vptestnmq 1016(%rdx){1to2}, %xmm28,%k2
+
+// CHECK: vptestnmq 1024(%rdx){1to2}, %xmm28, %k2
+// CHECK:  encoding: [0x62,0xf2,0x9e,0x10,0x27,0x92,0x00,0x04,0x00,0x00]
+          vptestnmq 1024(%rdx){1to2}, %xmm28,%k2
+
+// CHECK: vptestnmq -1024(%rdx){1to2}, %xmm28, %k2
+// CHECK:  encoding: [0x62,0xf2,0x9e,0x10,0x27,0x52,0x80]
+          vptestnmq -1024(%rdx){1to2}, %xmm28,%k2
+
+// CHECK: vptestnmq -1032(%rdx){1to2}, %xmm28, %k2
+// CHECK:  encoding: [0x62,0xf2,0x9e,0x10,0x27,0x92,0xf8,0xfb,0xff,0xff]
+          vptestnmq -1032(%rdx){1to2}, %xmm28,%k2
+
+// CHECK: vptestnmq %ymm17, %ymm26, %k4
+// CHECK:  encoding: [0x62,0xb2,0xae,0x20,0x27,0xe1]
+          vptestnmq %ymm17, %ymm26,%k4
+
+// CHECK: vptestnmq %ymm17, %ymm26, %k4 {%k1}
+// CHECK:  encoding: [0x62,0xb2,0xae,0x21,0x27,0xe1]
+          vptestnmq %ymm17, %ymm26,%k4 {%k1}
+
+// CHECK: vptestnmq (%rcx), %ymm26, %k4
+// CHECK:  encoding: [0x62,0xf2,0xae,0x20,0x27,0x21]
+          vptestnmq (%rcx), %ymm26,%k4
+
+// CHECK: vptestnmq 4660(%rax,%r14,8), %ymm26, %k4
+// CHECK:  encoding: [0x62,0xb2,0xae,0x20,0x27,0xa4,0xf0,0x34,0x12,0x00,0x00]
+          vptestnmq 4660(%rax,%r14,8), %ymm26,%k4
+
+// CHECK: vptestnmq (%rcx){1to4}, %ymm26, %k4
+// CHECK:  encoding: [0x62,0xf2,0xae,0x30,0x27,0x21]
+          vptestnmq (%rcx){1to4}, %ymm26,%k4
+
+// CHECK: vptestnmq 4064(%rdx), %ymm26, %k4
+// CHECK:  encoding: [0x62,0xf2,0xae,0x20,0x27,0x62,0x7f]
+          vptestnmq 4064(%rdx), %ymm26,%k4
+
+// CHECK: vptestnmq 4096(%rdx), %ymm26, %k4
+// CHECK:  encoding: [0x62,0xf2,0xae,0x20,0x27,0xa2,0x00,0x10,0x00,0x00]
+          vptestnmq 4096(%rdx), %ymm26,%k4
+
+// CHECK: vptestnmq -4096(%rdx), %ymm26, %k4
+// CHECK:  encoding: [0x62,0xf2,0xae,0x20,0x27,0x62,0x80]
+          vptestnmq -4096(%rdx), %ymm26,%k4
+
+// CHECK: vptestnmq -4128(%rdx), %ymm26, %k4
+// CHECK:  encoding: [0x62,0xf2,0xae,0x20,0x27,0xa2,0xe0,0xef,0xff,0xff]
+          vptestnmq -4128(%rdx), %ymm26,%k4
+
+// CHECK: vptestnmq 1016(%rdx){1to4}, %ymm26, %k4
+// CHECK:  encoding: [0x62,0xf2,0xae,0x30,0x27,0x62,0x7f]
+          vptestnmq 1016(%rdx){1to4}, %ymm26,%k4
+
+// CHECK: vptestnmq 1024(%rdx){1to4}, %ymm26, %k4
+// CHECK:  encoding: [0x62,0xf2,0xae,0x30,0x27,0xa2,0x00,0x04,0x00,0x00]
+          vptestnmq 1024(%rdx){1to4}, %ymm26,%k4
+
+// CHECK: vptestnmq -1024(%rdx){1to4}, %ymm26, %k4
+// CHECK:  encoding: [0x62,0xf2,0xae,0x30,0x27,0x62,0x80]
+          vptestnmq -1024(%rdx){1to4}, %ymm26,%k4
+
+// CHECK: vptestnmq -1032(%rdx){1to4}, %ymm26, %k4
+// CHECK:  encoding: [0x62,0xf2,0xae,0x30,0x27,0xa2,0xf8,0xfb,0xff,0xff]
+          vptestnmq -1032(%rdx){1to4}, %ymm26,%k4
+
+
+// CHECK: vptestnmd %xmm21, %xmm25, %k4
+// CHECK:  encoding: [0x62,0xb2,0x36,0x00,0x27,0xe5]
+          vptestnmd %xmm21, %xmm25,%k4
+
+// CHECK: vptestnmd %xmm21, %xmm25, %k4 {%k5}
+// CHECK:  encoding: [0x62,0xb2,0x36,0x05,0x27,0xe5]
+          vptestnmd %xmm21, %xmm25,%k4 {%k5}
+
+// CHECK: vptestnmd (%rcx), %xmm25, %k4
+// CHECK:  encoding: [0x62,0xf2,0x36,0x00,0x27,0x21]
+          vptestnmd (%rcx), %xmm25,%k4
+
+// CHECK: vptestnmd 4660(%rax,%r14,8), %xmm25, %k4
+// CHECK:  encoding: [0x62,0xb2,0x36,0x00,0x27,0xa4,0xf0,0x34,0x12,0x00,0x00]
+          vptestnmd 4660(%rax,%r14,8), %xmm25,%k4
+
+// CHECK: vptestnmd (%rcx){1to4}, %xmm25, %k4
+// CHECK:  encoding: [0x62,0xf2,0x36,0x10,0x27,0x21]
+          vptestnmd (%rcx){1to4}, %xmm25,%k4
+
+// CHECK: vptestnmd 2032(%rdx), %xmm25, %k4
+// CHECK:  encoding: [0x62,0xf2,0x36,0x00,0x27,0x62,0x7f]
+          vptestnmd 2032(%rdx), %xmm25,%k4
+
+// CHECK: vptestnmd 2048(%rdx), %xmm25, %k4
+// CHECK:  encoding: [0x62,0xf2,0x36,0x00,0x27,0xa2,0x00,0x08,0x00,0x00]
+          vptestnmd 2048(%rdx), %xmm25,%k4
+
+// CHECK: vptestnmd -2048(%rdx), %xmm25, %k4
+// CHECK:  encoding: [0x62,0xf2,0x36,0x00,0x27,0x62,0x80]
+          vptestnmd -2048(%rdx), %xmm25,%k4
+
+// CHECK: vptestnmd -2064(%rdx), %xmm25, %k4
+// CHECK:  encoding: [0x62,0xf2,0x36,0x00,0x27,0xa2,0xf0,0xf7,0xff,0xff]
+          vptestnmd -2064(%rdx), %xmm25,%k4
+
+// CHECK: vptestnmd 508(%rdx){1to4}, %xmm25, %k4
+// CHECK:  encoding: [0x62,0xf2,0x36,0x10,0x27,0x62,0x7f]
+          vptestnmd 508(%rdx){1to4}, %xmm25,%k4
+
+// CHECK: vptestnmd 512(%rdx){1to4}, %xmm25, %k4
+// CHECK:  encoding: [0x62,0xf2,0x36,0x10,0x27,0xa2,0x00,0x02,0x00,0x00]
+          vptestnmd 512(%rdx){1to4}, %xmm25,%k4
+
+// CHECK: vptestnmd -512(%rdx){1to4}, %xmm25, %k4
+// CHECK:  encoding: [0x62,0xf2,0x36,0x10,0x27,0x62,0x80]
+          vptestnmd -512(%rdx){1to4}, %xmm25,%k4
+
+// CHECK: vptestnmd -516(%rdx){1to4}, %xmm25, %k4
+// CHECK:  encoding: [0x62,0xf2,0x36,0x10,0x27,0xa2,0xfc,0xfd,0xff,0xff]
+          vptestnmd -516(%rdx){1to4}, %xmm25,%k4
+
+// CHECK: vptestnmd %ymm26, %ymm29, %k4
+// CHECK:  encoding: [0x62,0x92,0x16,0x20,0x27,0xe2]
+          vptestnmd %ymm26, %ymm29,%k4
+
+// CHECK: vptestnmd %ymm26, %ymm29, %k4 {%k4}
+// CHECK:  encoding: [0x62,0x92,0x16,0x24,0x27,0xe2]
+          vptestnmd %ymm26, %ymm29,%k4 {%k4}
+
+// CHECK: vptestnmd (%rcx), %ymm29, %k4
+// CHECK:  encoding: [0x62,0xf2,0x16,0x20,0x27,0x21]
+          vptestnmd (%rcx), %ymm29,%k4
+
+// CHECK: vptestnmd 4660(%rax,%r14,8), %ymm29, %k4
+// CHECK:  encoding: [0x62,0xb2,0x16,0x20,0x27,0xa4,0xf0,0x34,0x12,0x00,0x00]
+          vptestnmd 4660(%rax,%r14,8), %ymm29,%k4
+
+// CHECK: vptestnmd (%rcx){1to8}, %ymm29, %k4
+// CHECK:  encoding: [0x62,0xf2,0x16,0x30,0x27,0x21]
+          vptestnmd (%rcx){1to8}, %ymm29,%k4
+
+// CHECK: vptestnmd 4064(%rdx), %ymm29, %k4
+// CHECK:  encoding: [0x62,0xf2,0x16,0x20,0x27,0x62,0x7f]
+          vptestnmd 4064(%rdx), %ymm29,%k4
+
+// CHECK: vptestnmd 4096(%rdx), %ymm29, %k4
+// CHECK:  encoding: [0x62,0xf2,0x16,0x20,0x27,0xa2,0x00,0x10,0x00,0x00]
+          vptestnmd 4096(%rdx), %ymm29,%k4
+
+// CHECK: vptestnmd -4096(%rdx), %ymm29, %k4
+// CHECK:  encoding: [0x62,0xf2,0x16,0x20,0x27,0x62,0x80]
+          vptestnmd -4096(%rdx), %ymm29,%k4
+
+// CHECK: vptestnmd -4128(%rdx), %ymm29, %k4
+// CHECK:  encoding: [0x62,0xf2,0x16,0x20,0x27,0xa2,0xe0,0xef,0xff,0xff]
+          vptestnmd -4128(%rdx), %ymm29,%k4
+
+// CHECK: vptestnmd 508(%rdx){1to8}, %ymm29, %k4
+// CHECK:  encoding: [0x62,0xf2,0x16,0x30,0x27,0x62,0x7f]
+          vptestnmd 508(%rdx){1to8}, %ymm29,%k4
+
+// CHECK: vptestnmd 512(%rdx){1to8}, %ymm29, %k4
+// CHECK:  encoding: [0x62,0xf2,0x16,0x30,0x27,0xa2,0x00,0x02,0x00,0x00]
+          vptestnmd 512(%rdx){1to8}, %ymm29,%k4
+
+// CHECK: vptestnmd -512(%rdx){1to8}, %ymm29, %k4
+// CHECK:  encoding: [0x62,0xf2,0x16,0x30,0x27,0x62,0x80]
+          vptestnmd -512(%rdx){1to8}, %ymm29,%k4
+
+// CHECK: vptestnmd -516(%rdx){1to8}, %ymm29, %k4
+// CHECK:  encoding: [0x62,0xf2,0x16,0x30,0x27,0xa2,0xfc,0xfd,0xff,0xff]
+          vptestnmd -516(%rdx){1to8}, %ymm29,%k4
+
+// CHECK: vpmovd2m %xmm27, %k3
+// CHECK:  encoding: [0x62,0x92,0x7e,0x08,0x39,0xdb]
+          vpmovd2m %xmm27, %k3
+
+// CHECK: vpmovd2m %ymm28, %k4
+// CHECK:  encoding: [0x62,0x92,0x7e,0x28,0x39,0xe4]
+          vpmovd2m %ymm28, %k4
+
+// CHECK: vpmovq2m %xmm28, %k5
+// CHECK:  encoding: [0x62,0x92,0xfe,0x08,0x39,0xec]
+          vpmovq2m %xmm28, %k5
+
+// CHECK: vpmovq2m %ymm29, %k4
+// CHECK:  encoding: [0x62,0x92,0xfe,0x28,0x39,0xe5]
+          vpmovq2m %ymm29, %k4
+
+// CHECK: vpmovm2d %k2, %xmm29
+// CHECK:  encoding: [0x62,0x62,0x7e,0x08,0x38,0xea]
+          vpmovm2d %k2, %xmm29
+
+// CHECK: vpmovm2d %k5, %ymm20
+// CHECK:  encoding: [0x62,0xe2,0x7e,0x28,0x38,0xe5]
+          vpmovm2d %k5, %ymm20
+
+// CHECK: vpmovm2q %k5, %xmm17
+// CHECK:  encoding: [0x62,0xe2,0xfe,0x08,0x38,0xcd]
+          vpmovm2q %k5, %xmm17
+
+// CHECK: vpmovm2q %k2, %ymm30
+// CHECK:  encoding: [0x62,0x62,0xfe,0x28,0x38,0xf2]
+          vpmovm2q %k2, %ymm30
+
+// CHECK: vcompresspd %xmm23, (%rcx)
+// CHECK:  encoding: [0x62,0xe2,0xfd,0x08,0x8a,0x39]
+          vcompresspd %xmm23, (%rcx)
+
+// CHECK: vcompresspd %xmm23, (%rcx) {%k6}
+// CHECK:  encoding: [0x62,0xe2,0xfd,0x0e,0x8a,0x39]
+          vcompresspd %xmm23, (%rcx) {%k6}
+
+// CHECK: vcompresspd %xmm23, 291(%rax,%r14,8)
+// CHECK:  encoding: [0x62,0xa2,0xfd,0x08,0x8a,0xbc,0xf0,0x23,0x01,0x00,0x00]
+          vcompresspd %xmm23, 291(%rax,%r14,8)
+
+// CHECK: vcompresspd %xmm23, 1016(%rdx)
+// CHECK:  encoding: [0x62,0xe2,0xfd,0x08,0x8a,0x7a,0x7f]
+          vcompresspd %xmm23, 1016(%rdx)
+
+// CHECK: vcompresspd %xmm23, 1024(%rdx)
+// CHECK:  encoding: [0x62,0xe2,0xfd,0x08,0x8a,0xba,0x00,0x04,0x00,0x00]
+          vcompresspd %xmm23, 1024(%rdx)
+
+// CHECK: vcompresspd %xmm23, -1024(%rdx)
+// CHECK:  encoding: [0x62,0xe2,0xfd,0x08,0x8a,0x7a,0x80]
+          vcompresspd %xmm23, -1024(%rdx)
+
+// CHECK: vcompresspd %xmm23, -1032(%rdx)
+// CHECK:  encoding: [0x62,0xe2,0xfd,0x08,0x8a,0xba,0xf8,0xfb,0xff,0xff]
+          vcompresspd %xmm23, -1032(%rdx)
+
+// CHECK: vcompresspd %ymm29, (%rcx)
+// CHECK:  encoding: [0x62,0x62,0xfd,0x28,0x8a,0x29]
+          vcompresspd %ymm29, (%rcx)
+
+// CHECK: vcompresspd %ymm29, (%rcx) {%k2}
+// CHECK:  encoding: [0x62,0x62,0xfd,0x2a,0x8a,0x29]
+          vcompresspd %ymm29, (%rcx) {%k2}
+
+// CHECK: vcompresspd %ymm29, 291(%rax,%r14,8)
+// CHECK:  encoding: [0x62,0x22,0xfd,0x28,0x8a,0xac,0xf0,0x23,0x01,0x00,0x00]
+          vcompresspd %ymm29, 291(%rax,%r14,8)
+
+// CHECK: vcompresspd %ymm29, 1016(%rdx)
+// CHECK:  encoding: [0x62,0x62,0xfd,0x28,0x8a,0x6a,0x7f]
+          vcompresspd %ymm29, 1016(%rdx)
+
+// CHECK: vcompresspd %ymm29, 1024(%rdx)
+// CHECK:  encoding: [0x62,0x62,0xfd,0x28,0x8a,0xaa,0x00,0x04,0x00,0x00]
+          vcompresspd %ymm29, 1024(%rdx)
+
+// CHECK: vcompresspd %ymm29, -1024(%rdx)
+// CHECK:  encoding: [0x62,0x62,0xfd,0x28,0x8a,0x6a,0x80]
+          vcompresspd %ymm29, -1024(%rdx)
+
+// CHECK: vcompresspd %ymm29, -1032(%rdx)
+// CHECK:  encoding: [0x62,0x62,0xfd,0x28,0x8a,0xaa,0xf8,0xfb,0xff,0xff]
+          vcompresspd %ymm29, -1032(%rdx)
+
+// CHECK: vcompresspd %xmm27, %xmm20
+// CHECK:  encoding: [0x62,0x22,0xfd,0x08,0x8a,0xdc]
+          vcompresspd %xmm27, %xmm20
+
+// CHECK: vcompresspd %xmm27, %xmm20 {%k2}
+// CHECK:  encoding: [0x62,0x22,0xfd,0x0a,0x8a,0xdc]
+          vcompresspd %xmm27, %xmm20 {%k2}
+
+// CHECK: vcompresspd %xmm27, %xmm20 {%k2} {z}
+// CHECK:  encoding: [0x62,0x22,0xfd,0x8a,0x8a,0xdc]
+          vcompresspd %xmm27, %xmm20 {%k2} {z}
+
+// CHECK: vcompresspd %ymm20, %ymm24
+// CHECK:  encoding: [0x62,0x82,0xfd,0x28,0x8a,0xe0]
+          vcompresspd %ymm20, %ymm24
+
+// CHECK: vcompresspd %ymm20, %ymm24 {%k3}
+// CHECK:  encoding: [0x62,0x82,0xfd,0x2b,0x8a,0xe0]
+          vcompresspd %ymm20, %ymm24 {%k3}
+
+// CHECK: vcompresspd %ymm20, %ymm24 {%k3} {z}
+// CHECK:  encoding: [0x62,0x82,0xfd,0xab,0x8a,0xe0]
+          vcompresspd %ymm20, %ymm24 {%k3} {z}
+
+// CHECK: vcompressps %xmm21, (%rcx)
+// CHECK:  encoding: [0x62,0xe2,0x7d,0x08,0x8a,0x29]
+          vcompressps %xmm21, (%rcx)
+
+// CHECK: vcompressps %xmm21, (%rcx) {%k7}
+// CHECK:  encoding: [0x62,0xe2,0x7d,0x0f,0x8a,0x29]
+          vcompressps %xmm21, (%rcx) {%k7}
+
+// CHECK: vcompressps %xmm21, 291(%rax,%r14,8)
+// CHECK:  encoding: [0x62,0xa2,0x7d,0x08,0x8a,0xac,0xf0,0x23,0x01,0x00,0x00]
+          vcompressps %xmm21, 291(%rax,%r14,8)
+
+// CHECK: vcompressps %xmm21, 508(%rdx)
+// CHECK:  encoding: [0x62,0xe2,0x7d,0x08,0x8a,0x6a,0x7f]
+          vcompressps %xmm21, 508(%rdx)
+
+// CHECK: vcompressps %xmm21, 512(%rdx)
+// CHECK:  encoding: [0x62,0xe2,0x7d,0x08,0x8a,0xaa,0x00,0x02,0x00,0x00]
+          vcompressps %xmm21, 512(%rdx)
+
+// CHECK: vcompressps %xmm21, -512(%rdx)
+// CHECK:  encoding: [0x62,0xe2,0x7d,0x08,0x8a,0x6a,0x80]
+          vcompressps %xmm21, -512(%rdx)
+
+// CHECK: vcompressps %xmm21, -516(%rdx)
+// CHECK:  encoding: [0x62,0xe2,0x7d,0x08,0x8a,0xaa,0xfc,0xfd,0xff,0xff]
+          vcompressps %xmm21, -516(%rdx)
+
+// CHECK: vcompressps %ymm24, (%rcx)
+// CHECK:  encoding: [0x62,0x62,0x7d,0x28,0x8a,0x01]
+          vcompressps %ymm24, (%rcx)
+
+// CHECK: vcompressps %ymm24, (%rcx) {%k7}
+// CHECK:  encoding: [0x62,0x62,0x7d,0x2f,0x8a,0x01]
+          vcompressps %ymm24, (%rcx) {%k7}
+
+// CHECK: vcompressps %ymm24, 291(%rax,%r14,8)
+// CHECK:  encoding: [0x62,0x22,0x7d,0x28,0x8a,0x84,0xf0,0x23,0x01,0x00,0x00]
+          vcompressps %ymm24, 291(%rax,%r14,8)
+
+// CHECK: vcompressps %ymm24, 508(%rdx)
+// CHECK:  encoding: [0x62,0x62,0x7d,0x28,0x8a,0x42,0x7f]
+          vcompressps %ymm24, 508(%rdx)
+
+// CHECK: vcompressps %ymm24, 512(%rdx)
+// CHECK:  encoding: [0x62,0x62,0x7d,0x28,0x8a,0x82,0x00,0x02,0x00,0x00]
+          vcompressps %ymm24, 512(%rdx)
+
+// CHECK: vcompressps %ymm24, -512(%rdx)
+// CHECK:  encoding: [0x62,0x62,0x7d,0x28,0x8a,0x42,0x80]
+          vcompressps %ymm24, -512(%rdx)
+
+// CHECK: vcompressps %ymm24, -516(%rdx)
+// CHECK:  encoding: [0x62,0x62,0x7d,0x28,0x8a,0x82,0xfc,0xfd,0xff,0xff]
+          vcompressps %ymm24, -516(%rdx)
+
+// CHECK: vcompressps %xmm29, %xmm28
+// CHECK:  encoding: [0x62,0x02,0x7d,0x08,0x8a,0xec]
+          vcompressps %xmm29, %xmm28
+
+// CHECK: vcompressps %xmm29, %xmm28 {%k3}
+// CHECK:  encoding: [0x62,0x02,0x7d,0x0b,0x8a,0xec]
+          vcompressps %xmm29, %xmm28 {%k3}
+
+// CHECK: vcompressps %xmm29, %xmm28 {%k3} {z}
+// CHECK:  encoding: [0x62,0x02,0x7d,0x8b,0x8a,0xec]
+          vcompressps %xmm29, %xmm28 {%k3} {z}
+
+// CHECK: vcompressps %ymm25, %ymm23
+// CHECK:  encoding: [0x62,0x22,0x7d,0x28,0x8a,0xcf]
+          vcompressps %ymm25, %ymm23
+
+// CHECK: vcompressps %ymm25, %ymm23 {%k6}
+// CHECK:  encoding: [0x62,0x22,0x7d,0x2e,0x8a,0xcf]
+          vcompressps %ymm25, %ymm23 {%k6}
+
+// CHECK: vcompressps %ymm25, %ymm23 {%k6} {z}
+// CHECK:  encoding: [0x62,0x22,0x7d,0xae,0x8a,0xcf]
+          vcompressps %ymm25, %ymm23 {%k6} {z}
+
+// CHECK: vexpandpd (%rcx), %xmm23
+// CHECK:  encoding: [0x62,0xe2,0xfd,0x08,0x88,0x39]
+          vexpandpd (%rcx), %xmm23
+
+// CHECK: vexpandpd (%rcx), %xmm23 {%k3}
+// CHECK:  encoding: [0x62,0xe2,0xfd,0x0b,0x88,0x39]
+          vexpandpd (%rcx), %xmm23 {%k3}
+
+// CHECK: vexpandpd (%rcx), %xmm23 {%k3} {z}
+// CHECK:  encoding: [0x62,0xe2,0xfd,0x8b,0x88,0x39]
+          vexpandpd (%rcx), %xmm23 {%k3} {z}
+
+// CHECK: vexpandpd 291(%rax,%r14,8), %xmm23
+// CHECK:  encoding: [0x62,0xa2,0xfd,0x08,0x88,0xbc,0xf0,0x23,0x01,0x00,0x00]
+          vexpandpd 291(%rax,%r14,8), %xmm23
+
+// CHECK: vexpandpd 1016(%rdx), %xmm23
+// CHECK:  encoding: [0x62,0xe2,0xfd,0x08,0x88,0x7a,0x7f]
+          vexpandpd 1016(%rdx), %xmm23
+
+// CHECK: vexpandpd 1024(%rdx), %xmm23
+// CHECK:  encoding: [0x62,0xe2,0xfd,0x08,0x88,0xba,0x00,0x04,0x00,0x00]
+          vexpandpd 1024(%rdx), %xmm23
+
+// CHECK: vexpandpd -1024(%rdx), %xmm23
+// CHECK:  encoding: [0x62,0xe2,0xfd,0x08,0x88,0x7a,0x80]
+          vexpandpd -1024(%rdx), %xmm23
+
+// CHECK: vexpandpd -1032(%rdx), %xmm23
+// CHECK:  encoding: [0x62,0xe2,0xfd,0x08,0x88,0xba,0xf8,0xfb,0xff,0xff]
+          vexpandpd -1032(%rdx), %xmm23
+
+// CHECK: vexpandpd (%rcx), %ymm22
+// CHECK:  encoding: [0x62,0xe2,0xfd,0x28,0x88,0x31]
+          vexpandpd (%rcx), %ymm22
+
+// CHECK: vexpandpd (%rcx), %ymm22 {%k5}
+// CHECK:  encoding: [0x62,0xe2,0xfd,0x2d,0x88,0x31]
+          vexpandpd (%rcx), %ymm22 {%k5}
+
+// CHECK: vexpandpd (%rcx), %ymm22 {%k5} {z}
+// CHECK:  encoding: [0x62,0xe2,0xfd,0xad,0x88,0x31]
+          vexpandpd (%rcx), %ymm22 {%k5} {z}
+
+// CHECK: vexpandpd 291(%rax,%r14,8), %ymm22
+// CHECK:  encoding: [0x62,0xa2,0xfd,0x28,0x88,0xb4,0xf0,0x23,0x01,0x00,0x00]
+          vexpandpd 291(%rax,%r14,8), %ymm22
+
+// CHECK: vexpandpd 1016(%rdx), %ymm22
+// CHECK:  encoding: [0x62,0xe2,0xfd,0x28,0x88,0x72,0x7f]
+          vexpandpd 1016(%rdx), %ymm22
+
+// CHECK: vexpandpd 1024(%rdx), %ymm22
+// CHECK:  encoding: [0x62,0xe2,0xfd,0x28,0x88,0xb2,0x00,0x04,0x00,0x00]
+          vexpandpd 1024(%rdx), %ymm22
+
+// CHECK: vexpandpd -1024(%rdx), %ymm22
+// CHECK:  encoding: [0x62,0xe2,0xfd,0x28,0x88,0x72,0x80]
+          vexpandpd -1024(%rdx), %ymm22
+
+// CHECK: vexpandpd -1032(%rdx), %ymm22
+// CHECK:  encoding: [0x62,0xe2,0xfd,0x28,0x88,0xb2,0xf8,0xfb,0xff,0xff]
+          vexpandpd -1032(%rdx), %ymm22
+
+// CHECK: vexpandpd %xmm25, %xmm29
+// CHECK:  encoding: [0x62,0x02,0xfd,0x08,0x88,0xe9]
+          vexpandpd %xmm25, %xmm29
+
+// CHECK: vexpandpd %xmm25, %xmm29 {%k7}
+// CHECK:  encoding: [0x62,0x02,0xfd,0x0f,0x88,0xe9]
+          vexpandpd %xmm25, %xmm29 {%k7}
+
+// CHECK: vexpandpd %xmm25, %xmm29 {%k7} {z}
+// CHECK:  encoding: [0x62,0x02,0xfd,0x8f,0x88,0xe9]
+          vexpandpd %xmm25, %xmm29 {%k7} {z}
+
+// CHECK: vexpandpd %ymm27, %ymm21
+// CHECK:  encoding: [0x62,0x82,0xfd,0x28,0x88,0xeb]
+          vexpandpd %ymm27, %ymm21
+
+// CHECK: vexpandpd %ymm27, %ymm21 {%k2}
+// CHECK:  encoding: [0x62,0x82,0xfd,0x2a,0x88,0xeb]
+          vexpandpd %ymm27, %ymm21 {%k2}
+
+// CHECK: vexpandpd %ymm27, %ymm21 {%k2} {z}
+// CHECK:  encoding: [0x62,0x82,0xfd,0xaa,0x88,0xeb]
+          vexpandpd %ymm27, %ymm21 {%k2} {z}
+
+// CHECK: vexpandps (%rcx), %xmm18
+// CHECK:  encoding: [0x62,0xe2,0x7d,0x08,0x88,0x11]
+          vexpandps (%rcx), %xmm18
+
+// CHECK: vexpandps (%rcx), %xmm18 {%k1}
+// CHECK:  encoding: [0x62,0xe2,0x7d,0x09,0x88,0x11]
+          vexpandps (%rcx), %xmm18 {%k1}
+
+// CHECK: vexpandps (%rcx), %xmm18 {%k1} {z}
+// CHECK:  encoding: [0x62,0xe2,0x7d,0x89,0x88,0x11]
+          vexpandps (%rcx), %xmm18 {%k1} {z}
+
+// CHECK: vexpandps 291(%rax,%r14,8), %xmm18
+// CHECK:  encoding: [0x62,0xa2,0x7d,0x08,0x88,0x94,0xf0,0x23,0x01,0x00,0x00]
+          vexpandps 291(%rax,%r14,8), %xmm18
+
+// CHECK: vexpandps 508(%rdx), %xmm18
+// CHECK:  encoding: [0x62,0xe2,0x7d,0x08,0x88,0x52,0x7f]
+          vexpandps 508(%rdx), %xmm18
+
+// CHECK: vexpandps 512(%rdx), %xmm18
+// CHECK:  encoding: [0x62,0xe2,0x7d,0x08,0x88,0x92,0x00,0x02,0x00,0x00]
+          vexpandps 512(%rdx), %xmm18
+
+// CHECK: vexpandps -512(%rdx), %xmm18
+// CHECK:  encoding: [0x62,0xe2,0x7d,0x08,0x88,0x52,0x80]
+          vexpandps -512(%rdx), %xmm18
+
+// CHECK: vexpandps -516(%rdx), %xmm18
+// CHECK:  encoding: [0x62,0xe2,0x7d,0x08,0x88,0x92,0xfc,0xfd,0xff,0xff]
+          vexpandps -516(%rdx), %xmm18
+
+// CHECK: vexpandps (%rcx), %ymm23
+// CHECK:  encoding: [0x62,0xe2,0x7d,0x28,0x88,0x39]
+          vexpandps (%rcx), %ymm23
+
+// CHECK: vexpandps (%rcx), %ymm23 {%k7}
+// CHECK:  encoding: [0x62,0xe2,0x7d,0x2f,0x88,0x39]
+          vexpandps (%rcx), %ymm23 {%k7}
+
+// CHECK: vexpandps (%rcx), %ymm23 {%k7} {z}
+// CHECK:  encoding: [0x62,0xe2,0x7d,0xaf,0x88,0x39]
+          vexpandps (%rcx), %ymm23 {%k7} {z}
+
+// CHECK: vexpandps 291(%rax,%r14,8), %ymm23
+// CHECK:  encoding: [0x62,0xa2,0x7d,0x28,0x88,0xbc,0xf0,0x23,0x01,0x00,0x00]
+          vexpandps 291(%rax,%r14,8), %ymm23
+
+// CHECK: vexpandps 508(%rdx), %ymm23
+// CHECK:  encoding: [0x62,0xe2,0x7d,0x28,0x88,0x7a,0x7f]
+          vexpandps 508(%rdx), %ymm23
+
+// CHECK: vexpandps 512(%rdx), %ymm23
+// CHECK:  encoding: [0x62,0xe2,0x7d,0x28,0x88,0xba,0x00,0x02,0x00,0x00]
+          vexpandps 512(%rdx), %ymm23
+
+// CHECK: vexpandps -512(%rdx), %ymm23
+// CHECK:  encoding: [0x62,0xe2,0x7d,0x28,0x88,0x7a,0x80]
+          vexpandps -512(%rdx), %ymm23
+
+// CHECK: vexpandps -516(%rdx), %ymm23
+// CHECK:  encoding: [0x62,0xe2,0x7d,0x28,0x88,0xba,0xfc,0xfd,0xff,0xff]
+          vexpandps -516(%rdx), %ymm23
+
+// CHECK: vexpandps %xmm19, %xmm29
+// CHECK:  encoding: [0x62,0x22,0x7d,0x08,0x88,0xeb]
+          vexpandps %xmm19, %xmm29
+
+// CHECK: vexpandps %xmm19, %xmm29 {%k5}
+// CHECK:  encoding: [0x62,0x22,0x7d,0x0d,0x88,0xeb]
+          vexpandps %xmm19, %xmm29 {%k5}
+
+// CHECK: vexpandps %xmm19, %xmm29 {%k5} {z}
+// CHECK:  encoding: [0x62,0x22,0x7d,0x8d,0x88,0xeb]
+          vexpandps %xmm19, %xmm29 {%k5} {z}
+
+// CHECK: vexpandps %ymm29, %ymm29
+// CHECK:  encoding: [0x62,0x02,0x7d,0x28,0x88,0xed]
+          vexpandps %ymm29, %ymm29
+
+// CHECK: vexpandps %ymm29, %ymm29 {%k5}
+// CHECK:  encoding: [0x62,0x02,0x7d,0x2d,0x88,0xed]
+          vexpandps %ymm29, %ymm29 {%k5}
+
+// CHECK: vexpandps %ymm29, %ymm29 {%k5} {z}
+// CHECK:  encoding: [0x62,0x02,0x7d,0xad,0x88,0xed]
+          vexpandps %ymm29, %ymm29 {%k5} {z}
+
+// CHECK: vpabsd %xmm19, %xmm28
+// CHECK:  encoding: [0x62,0x22,0x7d,0x08,0x1e,0xe3]
+          vpabsd %xmm19, %xmm28
+
+// CHECK: vpabsd %xmm19, %xmm28 {%k6}
+// CHECK:  encoding: [0x62,0x22,0x7d,0x0e,0x1e,0xe3]
+          vpabsd %xmm19, %xmm28 {%k6}
+
+// CHECK: vpabsd %xmm19, %xmm28 {%k6} {z}
+// CHECK:  encoding: [0x62,0x22,0x7d,0x8e,0x1e,0xe3]
+          vpabsd %xmm19, %xmm28 {%k6} {z}
+
+// CHECK: vpabsd (%rcx), %xmm28
+// CHECK:  encoding: [0x62,0x62,0x7d,0x08,0x1e,0x21]
+          vpabsd (%rcx), %xmm28
+
+// CHECK: vpabsd 291(%rax,%r14,8), %xmm28
+// CHECK:  encoding: [0x62,0x22,0x7d,0x08,0x1e,0xa4,0xf0,0x23,0x01,0x00,0x00]
+          vpabsd 291(%rax,%r14,8), %xmm28
+
+// CHECK: vpabsd (%rcx){1to4}, %xmm28
+// CHECK:  encoding: [0x62,0x62,0x7d,0x18,0x1e,0x21]
+          vpabsd (%rcx){1to4}, %xmm28
+
+// CHECK: vpabsd 2032(%rdx), %xmm28
+// CHECK:  encoding: [0x62,0x62,0x7d,0x08,0x1e,0x62,0x7f]
+          vpabsd 2032(%rdx), %xmm28
+
+// CHECK: vpabsd 2048(%rdx), %xmm28
+// CHECK:  encoding: [0x62,0x62,0x7d,0x08,0x1e,0xa2,0x00,0x08,0x00,0x00]
+          vpabsd 2048(%rdx), %xmm28
+
+// CHECK: vpabsd -2048(%rdx), %xmm28
+// CHECK:  encoding: [0x62,0x62,0x7d,0x08,0x1e,0x62,0x80]
+          vpabsd -2048(%rdx), %xmm28
+
+// CHECK: vpabsd -2064(%rdx), %xmm28
+// CHECK:  encoding: [0x62,0x62,0x7d,0x08,0x1e,0xa2,0xf0,0xf7,0xff,0xff]
+          vpabsd -2064(%rdx), %xmm28
+
+// CHECK: vpabsd 508(%rdx){1to4}, %xmm28
+// CHECK:  encoding: [0x62,0x62,0x7d,0x18,0x1e,0x62,0x7f]
+          vpabsd 508(%rdx){1to4}, %xmm28
+
+// CHECK: vpabsd 512(%rdx){1to4}, %xmm28
+// CHECK:  encoding: [0x62,0x62,0x7d,0x18,0x1e,0xa2,0x00,0x02,0x00,0x00]
+          vpabsd 512(%rdx){1to4}, %xmm28
+
+// CHECK: vpabsd -512(%rdx){1to4}, %xmm28
+// CHECK:  encoding: [0x62,0x62,0x7d,0x18,0x1e,0x62,0x80]
+          vpabsd -512(%rdx){1to4}, %xmm28
+
+// CHECK: vpabsd -516(%rdx){1to4}, %xmm28
+// CHECK:  encoding: [0x62,0x62,0x7d,0x18,0x1e,0xa2,0xfc,0xfd,0xff,0xff]
+          vpabsd -516(%rdx){1to4}, %xmm28
+
+// CHECK: vpabsd %ymm18, %ymm25
+// CHECK:  encoding: [0x62,0x22,0x7d,0x28,0x1e,0xca]
+          vpabsd %ymm18, %ymm25
+
+// CHECK: vpabsd %ymm18, %ymm25 {%k2}
+// CHECK:  encoding: [0x62,0x22,0x7d,0x2a,0x1e,0xca]
+          vpabsd %ymm18, %ymm25 {%k2}
+
+// CHECK: vpabsd %ymm18, %ymm25 {%k2} {z}
+// CHECK:  encoding: [0x62,0x22,0x7d,0xaa,0x1e,0xca]
+          vpabsd %ymm18, %ymm25 {%k2} {z}
+
+// CHECK: vpabsd (%rcx), %ymm25
+// CHECK:  encoding: [0x62,0x62,0x7d,0x28,0x1e,0x09]
+          vpabsd (%rcx), %ymm25
+
+// CHECK: vpabsd 291(%rax,%r14,8), %ymm25
+// CHECK:  encoding: [0x62,0x22,0x7d,0x28,0x1e,0x8c,0xf0,0x23,0x01,0x00,0x00]
+          vpabsd 291(%rax,%r14,8), %ymm25
+
+// CHECK: vpabsd (%rcx){1to8}, %ymm25
+// CHECK:  encoding: [0x62,0x62,0x7d,0x38,0x1e,0x09]
+          vpabsd (%rcx){1to8}, %ymm25
+
+// CHECK: vpabsd 4064(%rdx), %ymm25
+// CHECK:  encoding: [0x62,0x62,0x7d,0x28,0x1e,0x4a,0x7f]
+          vpabsd 4064(%rdx), %ymm25
+
+// CHECK: vpabsd 4096(%rdx), %ymm25
+// CHECK:  encoding: [0x62,0x62,0x7d,0x28,0x1e,0x8a,0x00,0x10,0x00,0x00]
+          vpabsd 4096(%rdx), %ymm25
+
+// CHECK: vpabsd -4096(%rdx), %ymm25
+// CHECK:  encoding: [0x62,0x62,0x7d,0x28,0x1e,0x4a,0x80]
+          vpabsd -4096(%rdx), %ymm25
+
+// CHECK: vpabsd -4128(%rdx), %ymm25
+// CHECK:  encoding: [0x62,0x62,0x7d,0x28,0x1e,0x8a,0xe0,0xef,0xff,0xff]
+          vpabsd -4128(%rdx), %ymm25
+
+// CHECK: vpabsd 508(%rdx){1to8}, %ymm25
+// CHECK:  encoding: [0x62,0x62,0x7d,0x38,0x1e,0x4a,0x7f]
+          vpabsd 508(%rdx){1to8}, %ymm25
+
+// CHECK: vpabsd 512(%rdx){1to8}, %ymm25
+// CHECK:  encoding: [0x62,0x62,0x7d,0x38,0x1e,0x8a,0x00,0x02,0x00,0x00]
+          vpabsd 512(%rdx){1to8}, %ymm25
+
+// CHECK: vpabsd -512(%rdx){1to8}, %ymm25
+// CHECK:  encoding: [0x62,0x62,0x7d,0x38,0x1e,0x4a,0x80]
+          vpabsd -512(%rdx){1to8}, %ymm25
+
+// CHECK: vpabsd -516(%rdx){1to8}, %ymm25
+// CHECK:  encoding: [0x62,0x62,0x7d,0x38,0x1e,0x8a,0xfc,0xfd,0xff,0xff]
+          vpabsd -516(%rdx){1to8}, %ymm25
+
+// CHECK: vpabsq %xmm22, %xmm19
+// CHECK:  encoding: [0x62,0xa2,0xfd,0x08,0x1f,0xde]
+          vpabsq %xmm22, %xmm19
+
+// CHECK: vpabsq %xmm22, %xmm19 {%k2}
+// CHECK:  encoding: [0x62,0xa2,0xfd,0x0a,0x1f,0xde]
+          vpabsq %xmm22, %xmm19 {%k2}
+
+// CHECK: vpabsq %xmm22, %xmm19 {%k2} {z}
+// CHECK:  encoding: [0x62,0xa2,0xfd,0x8a,0x1f,0xde]
+          vpabsq %xmm22, %xmm19 {%k2} {z}
+
+// CHECK: vpabsq (%rcx), %xmm19
+// CHECK:  encoding: [0x62,0xe2,0xfd,0x08,0x1f,0x19]
+          vpabsq (%rcx), %xmm19
+
+// CHECK: vpabsq 291(%rax,%r14,8), %xmm19
+// CHECK:  encoding: [0x62,0xa2,0xfd,0x08,0x1f,0x9c,0xf0,0x23,0x01,0x00,0x00]
+          vpabsq 291(%rax,%r14,8), %xmm19
+
+// CHECK: vpabsq (%rcx){1to2}, %xmm19
+// CHECK:  encoding: [0x62,0xe2,0xfd,0x18,0x1f,0x19]
+          vpabsq (%rcx){1to2}, %xmm19
+
+// CHECK: vpabsq 2032(%rdx), %xmm19
+// CHECK:  encoding: [0x62,0xe2,0xfd,0x08,0x1f,0x5a,0x7f]
+          vpabsq 2032(%rdx), %xmm19
+
+// CHECK: vpabsq 2048(%rdx), %xmm19
+// CHECK:  encoding: [0x62,0xe2,0xfd,0x08,0x1f,0x9a,0x00,0x08,0x00,0x00]
+          vpabsq 2048(%rdx), %xmm19
+
+// CHECK: vpabsq -2048(%rdx), %xmm19
+// CHECK:  encoding: [0x62,0xe2,0xfd,0x08,0x1f,0x5a,0x80]
+          vpabsq -2048(%rdx), %xmm19
+
+// CHECK: vpabsq -2064(%rdx), %xmm19
+// CHECK:  encoding: [0x62,0xe2,0xfd,0x08,0x1f,0x9a,0xf0,0xf7,0xff,0xff]
+          vpabsq -2064(%rdx), %xmm19
+
+// CHECK: vpabsq 1016(%rdx){1to2}, %xmm19
+// CHECK:  encoding: [0x62,0xe2,0xfd,0x18,0x1f,0x5a,0x7f]
+          vpabsq 1016(%rdx){1to2}, %xmm19
+
+// CHECK: vpabsq 1024(%rdx){1to2}, %xmm19
+// CHECK:  encoding: [0x62,0xe2,0xfd,0x18,0x1f,0x9a,0x00,0x04,0x00,0x00]
+          vpabsq 1024(%rdx){1to2}, %xmm19
+
+// CHECK: vpabsq -1024(%rdx){1to2}, %xmm19
+// CHECK:  encoding: [0x62,0xe2,0xfd,0x18,0x1f,0x5a,0x80]
+          vpabsq -1024(%rdx){1to2}, %xmm19
+
+// CHECK: vpabsq -1032(%rdx){1to2}, %xmm19
+// CHECK:  encoding: [0x62,0xe2,0xfd,0x18,0x1f,0x9a,0xf8,0xfb,0xff,0xff]
+          vpabsq -1032(%rdx){1to2}, %xmm19
+
+// CHECK: vpabsq %ymm17, %ymm22
+// CHECK:  encoding: [0x62,0xa2,0xfd,0x28,0x1f,0xf1]
+          vpabsq %ymm17, %ymm22
+
+// CHECK: vpabsq %ymm17, %ymm22 {%k6}
+// CHECK:  encoding: [0x62,0xa2,0xfd,0x2e,0x1f,0xf1]
+          vpabsq %ymm17, %ymm22 {%k6}
+
+// CHECK: vpabsq %ymm17, %ymm22 {%k6} {z}
+// CHECK:  encoding: [0x62,0xa2,0xfd,0xae,0x1f,0xf1]
+          vpabsq %ymm17, %ymm22 {%k6} {z}
+
+// CHECK: vpabsq (%rcx), %ymm22
+// CHECK:  encoding: [0x62,0xe2,0xfd,0x28,0x1f,0x31]
+          vpabsq (%rcx), %ymm22
+
+// CHECK: vpabsq 291(%rax,%r14,8), %ymm22
+// CHECK:  encoding: [0x62,0xa2,0xfd,0x28,0x1f,0xb4,0xf0,0x23,0x01,0x00,0x00]
+          vpabsq 291(%rax,%r14,8), %ymm22
+
+// CHECK: vpabsq (%rcx){1to4}, %ymm22
+// CHECK:  encoding: [0x62,0xe2,0xfd,0x38,0x1f,0x31]
+          vpabsq (%rcx){1to4}, %ymm22
+
+// CHECK: vpabsq 4064(%rdx), %ymm22
+// CHECK:  encoding: [0x62,0xe2,0xfd,0x28,0x1f,0x72,0x7f]
+          vpabsq 4064(%rdx), %ymm22
+
+// CHECK: vpabsq 4096(%rdx), %ymm22
+// CHECK:  encoding: [0x62,0xe2,0xfd,0x28,0x1f,0xb2,0x00,0x10,0x00,0x00]
+          vpabsq 4096(%rdx), %ymm22
+
+// CHECK: vpabsq -4096(%rdx), %ymm22
+// CHECK:  encoding: [0x62,0xe2,0xfd,0x28,0x1f,0x72,0x80]
+          vpabsq -4096(%rdx), %ymm22
+
+// CHECK: vpabsq -4128(%rdx), %ymm22
+// CHECK:  encoding: [0x62,0xe2,0xfd,0x28,0x1f,0xb2,0xe0,0xef,0xff,0xff]
+          vpabsq -4128(%rdx), %ymm22
+
+// CHECK: vpabsq 1016(%rdx){1to4}, %ymm22
+// CHECK:  encoding: [0x62,0xe2,0xfd,0x38,0x1f,0x72,0x7f]
+          vpabsq 1016(%rdx){1to4}, %ymm22
+
+// CHECK: vpabsq 1024(%rdx){1to4}, %ymm22
+// CHECK:  encoding: [0x62,0xe2,0xfd,0x38,0x1f,0xb2,0x00,0x04,0x00,0x00]
+          vpabsq 1024(%rdx){1to4}, %ymm22
+
+// CHECK: vpabsq -1024(%rdx){1to4}, %ymm22
+// CHECK:  encoding: [0x62,0xe2,0xfd,0x38,0x1f,0x72,0x80]
+          vpabsq -1024(%rdx){1to4}, %ymm22
+
+// CHECK: vpabsq -1032(%rdx){1to4}, %ymm22
+// CHECK:  encoding: [0x62,0xe2,0xfd,0x38,0x1f,0xb2,0xf8,0xfb,0xff,0xff]
+          vpabsq -1032(%rdx){1to4}, %ymm22
+
+// CHECK: vpgatherdd 123(%r14,%xmm31,8), %xmm17 {%k1}
+// CHECK:  encoding: [0x62,0x82,0x7d,0x01,0x90,0x8c,0xfe,0x7b,0x00,0x00,0x00]
+          vpgatherdd 123(%r14,%xmm31,8), %xmm17 {%k1}
+
+// CHECK: vpgatherdd 256(%r9,%xmm31), %xmm17 {%k1}
+// CHECK:  encoding: [0x62,0x82,0x7d,0x01,0x90,0x4c,0x39,0x40]
+          vpgatherdd 256(%r9,%xmm31), %xmm17 {%k1}
+
+// CHECK: vpgatherdd 1024(%rcx,%xmm31,4), %xmm17 {%k1}
+// CHECK:  encoding: [0x62,0xa2,0x7d,0x01,0x90,0x8c,0xb9,0x00,0x04,0x00,0x00]
+          vpgatherdd 1024(%rcx,%xmm31,4), %xmm17 {%k1}
+
+// CHECK: vpgatherdd 123(%r14,%ymm31,8), %ymm19 {%k1}
+// CHECK:  encoding: [0x62,0x82,0x7d,0x21,0x90,0x9c,0xfe,0x7b,0x00,0x00,0x00]
+          vpgatherdd 123(%r14,%ymm31,8), %ymm19 {%k1}
+
+// CHECK: vpgatherdd 256(%r9,%ymm31), %ymm19 {%k1}
+// CHECK:  encoding: [0x62,0x82,0x7d,0x21,0x90,0x5c,0x39,0x40]
+          vpgatherdd 256(%r9,%ymm31), %ymm19 {%k1}
+
+// CHECK: vpgatherdd 1024(%rcx,%ymm31,4), %ymm19 {%k1}
+// CHECK:  encoding: [0x62,0xa2,0x7d,0x21,0x90,0x9c,0xb9,0x00,0x04,0x00,0x00]
+          vpgatherdd 1024(%rcx,%ymm31,4), %ymm19 {%k1}
+
+// CHECK: vpgatherdq 123(%r14,%xmm31,8), %xmm17 {%k1}
+// CHECK:  encoding: [0x62,0x82,0xfd,0x01,0x90,0x8c,0xfe,0x7b,0x00,0x00,0x00]
+          vpgatherdq 123(%r14,%xmm31,8), %xmm17 {%k1}
+
+// CHECK: vpgatherdq 256(%r9,%xmm31), %xmm17 {%k1}
+// CHECK:  encoding: [0x62,0x82,0xfd,0x01,0x90,0x4c,0x39,0x20]
+          vpgatherdq 256(%r9,%xmm31), %xmm17 {%k1}
+
+// CHECK: vpgatherdq 1024(%rcx,%xmm31,4), %xmm17 {%k1}
+// CHECK:  encoding: [0x62,0xa2,0xfd,0x01,0x90,0x8c,0xb9,0x00,0x04,0x00,0x00]
+          vpgatherdq 1024(%rcx,%xmm31,4), %xmm17 {%k1}
+
+// CHECK: vpgatherdq 123(%r14,%xmm31,8), %ymm26 {%k1}
+// CHECK:  encoding: [0x62,0x02,0xfd,0x21,0x90,0x94,0xfe,0x7b,0x00,0x00,0x00]
+          vpgatherdq 123(%r14,%xmm31,8), %ymm26 {%k1}
+
+// CHECK: vpgatherdq 256(%r9,%xmm31), %ymm26 {%k1}
+// CHECK:  encoding: [0x62,0x02,0xfd,0x21,0x90,0x54,0x39,0x20]
+          vpgatherdq 256(%r9,%xmm31), %ymm26 {%k1}
+
+// CHECK: vpgatherdq 1024(%rcx,%xmm31,4), %ymm26 {%k1}
+// CHECK:  encoding: [0x62,0x22,0xfd,0x21,0x90,0x94,0xb9,0x00,0x04,0x00,0x00]
+          vpgatherdq 1024(%rcx,%xmm31,4), %ymm26 {%k1}
+
+// CHECK: vpgatherqd 123(%r14,%xmm31,8), %xmm21 {%k1}
+// CHECK:  encoding: [0x62,0x82,0x7d,0x01,0x91,0xac,0xfe,0x7b,0x00,0x00,0x00]
+          vpgatherqd 123(%r14,%xmm31,8), %xmm21 {%k1}
+
+// CHECK: vpgatherqd 256(%r9,%xmm31), %xmm21 {%k1}
+// CHECK:  encoding: [0x62,0x82,0x7d,0x01,0x91,0x6c,0x39,0x40]
+          vpgatherqd 256(%r9,%xmm31), %xmm21 {%k1}
+
+// CHECK: vpgatherqd 1024(%rcx,%xmm31,4), %xmm21 {%k1}
+// CHECK:  encoding: [0x62,0xa2,0x7d,0x01,0x91,0xac,0xb9,0x00,0x04,0x00,0x00]
+          vpgatherqd 1024(%rcx,%xmm31,4), %xmm21 {%k1}
+
+// CHECK: vpgatherqd 123(%r14,%ymm31,8), %xmm25 {%k1}
+// CHECK:  encoding: [0x62,0x02,0x7d,0x21,0x91,0x8c,0xfe,0x7b,0x00,0x00,0x00]
+          vpgatherqd 123(%r14,%ymm31,8), %xmm25 {%k1}
+
+// CHECK: vpgatherqd 256(%r9,%ymm31), %xmm25 {%k1}
+// CHECK:  encoding: [0x62,0x02,0x7d,0x21,0x91,0x4c,0x39,0x40]
+          vpgatherqd 256(%r9,%ymm31), %xmm25 {%k1}
+
+// CHECK: vpgatherqd 1024(%rcx,%ymm31,4), %xmm25 {%k1}
+// CHECK:  encoding: [0x62,0x22,0x7d,0x21,0x91,0x8c,0xb9,0x00,0x04,0x00,0x00]
+          vpgatherqd 1024(%rcx,%ymm31,4), %xmm25 {%k1}
+
+// CHECK: vpgatherqq 123(%r14,%xmm31,8), %xmm18 {%k1}
+// CHECK:  encoding: [0x62,0x82,0xfd,0x01,0x91,0x94,0xfe,0x7b,0x00,0x00,0x00]
+          vpgatherqq 123(%r14,%xmm31,8), %xmm18 {%k1}
+
+// CHECK: vpgatherqq 256(%r9,%xmm31), %xmm18 {%k1}
+// CHECK:  encoding: [0x62,0x82,0xfd,0x01,0x91,0x54,0x39,0x20]
+          vpgatherqq 256(%r9,%xmm31), %xmm18 {%k1}
+
+// CHECK: vpgatherqq 1024(%rcx,%xmm31,4), %xmm18 {%k1}
+// CHECK:  encoding: [0x62,0xa2,0xfd,0x01,0x91,0x94,0xb9,0x00,0x04,0x00,0x00]
+          vpgatherqq 1024(%rcx,%xmm31,4), %xmm18 {%k1}
+
+// CHECK: vpgatherqq 123(%r14,%ymm31,8), %ymm19 {%k1}
+// CHECK:  encoding: [0x62,0x82,0xfd,0x21,0x91,0x9c,0xfe,0x7b,0x00,0x00,0x00]
+          vpgatherqq 123(%r14,%ymm31,8), %ymm19 {%k1}
+
+// CHECK: vpgatherqq 256(%r9,%ymm31), %ymm19 {%k1}
+// CHECK:  encoding: [0x62,0x82,0xfd,0x21,0x91,0x5c,0x39,0x20]
+          vpgatherqq 256(%r9,%ymm31), %ymm19 {%k1}
+
+// CHECK: vpgatherqq 1024(%rcx,%ymm31,4), %ymm19 {%k1}
+// CHECK:  encoding: [0x62,0xa2,0xfd,0x21,0x91,0x9c,0xb9,0x00,0x04,0x00,0x00]
+          vpgatherqq 1024(%rcx,%ymm31,4), %ymm19 {%k1}
+
+// CHECK: vgatherdpd 123(%r14,%xmm31,8), %xmm17 {%k1}
+// CHECK:  encoding: [0x62,0x82,0xfd,0x01,0x92,0x8c,0xfe,0x7b,0x00,0x00,0x00]
+          vgatherdpd 123(%r14,%xmm31,8), %xmm17 {%k1}
+
+// CHECK: vgatherdpd 256(%r9,%xmm31), %xmm17 {%k1}
+// CHECK:  encoding: [0x62,0x82,0xfd,0x01,0x92,0x4c,0x39,0x20]
+          vgatherdpd 256(%r9,%xmm31), %xmm17 {%k1}
+
+// CHECK: vgatherdpd 1024(%rcx,%xmm31,4), %xmm17 {%k1}
+// CHECK:  encoding: [0x62,0xa2,0xfd,0x01,0x92,0x8c,0xb9,0x00,0x04,0x00,0x00]
+          vgatherdpd 1024(%rcx,%xmm31,4), %xmm17 {%k1}
+
+// CHECK: vgatherdpd 123(%r14,%xmm31,8), %ymm23 {%k1}
+// CHECK:  encoding: [0x62,0x82,0xfd,0x21,0x92,0xbc,0xfe,0x7b,0x00,0x00,0x00]
+          vgatherdpd 123(%r14,%xmm31,8), %ymm23 {%k1}
+
+// CHECK: vgatherdpd 256(%r9,%xmm31), %ymm23 {%k1}
+// CHECK:  encoding: [0x62,0x82,0xfd,0x21,0x92,0x7c,0x39,0x20]
+          vgatherdpd 256(%r9,%xmm31), %ymm23 {%k1}
+
+// CHECK: vgatherdpd 1024(%rcx,%xmm31,4), %ymm23 {%k1}
+// CHECK:  encoding: [0x62,0xa2,0xfd,0x21,0x92,0xbc,0xb9,0x00,0x04,0x00,0x00]
+          vgatherdpd 1024(%rcx,%xmm31,4), %ymm23 {%k1}
+
+// CHECK: vgatherdps 123(%r14,%xmm31,8), %xmm18 {%k1}
+// CHECK:  encoding: [0x62,0x82,0x7d,0x01,0x92,0x94,0xfe,0x7b,0x00,0x00,0x00]
+          vgatherdps 123(%r14,%xmm31,8), %xmm18 {%k1}
+
+// CHECK: vgatherdps 256(%r9,%xmm31), %xmm18 {%k1}
+// CHECK:  encoding: [0x62,0x82,0x7d,0x01,0x92,0x54,0x39,0x40]
+          vgatherdps 256(%r9,%xmm31), %xmm18 {%k1}
+
+// CHECK: vgatherdps 1024(%rcx,%xmm31,4), %xmm18 {%k1}
+// CHECK:  encoding: [0x62,0xa2,0x7d,0x01,0x92,0x94,0xb9,0x00,0x04,0x00,0x00]
+          vgatherdps 1024(%rcx,%xmm31,4), %xmm18 {%k1}
+
+// CHECK: vgatherdps 123(%r14,%ymm31,8), %ymm27 {%k1}
+// CHECK:  encoding: [0x62,0x02,0x7d,0x21,0x92,0x9c,0xfe,0x7b,0x00,0x00,0x00]
+          vgatherdps 123(%r14,%ymm31,8), %ymm27 {%k1}
+
+// CHECK: vgatherdps 256(%r9,%ymm31), %ymm27 {%k1}
+// CHECK:  encoding: [0x62,0x02,0x7d,0x21,0x92,0x5c,0x39,0x40]
+          vgatherdps 256(%r9,%ymm31), %ymm27 {%k1}
+
+// CHECK: vgatherdps 1024(%rcx,%ymm31,4), %ymm27 {%k1}
+// CHECK:  encoding: [0x62,0x22,0x7d,0x21,0x92,0x9c,0xb9,0x00,0x04,0x00,0x00]
+          vgatherdps 1024(%rcx,%ymm31,4), %ymm27 {%k1}
+
+// CHECK: vgatherqpd 123(%r14,%xmm31,8), %xmm17 {%k1}
+// CHECK:  encoding: [0x62,0x82,0xfd,0x01,0x93,0x8c,0xfe,0x7b,0x00,0x00,0x00]
+          vgatherqpd 123(%r14,%xmm31,8), %xmm17 {%k1}
+
+// CHECK: vgatherqpd 256(%r9,%xmm31), %xmm17 {%k1}
+// CHECK:  encoding: [0x62,0x82,0xfd,0x01,0x93,0x4c,0x39,0x20]
+          vgatherqpd 256(%r9,%xmm31), %xmm17 {%k1}
+
+// CHECK: vgatherqpd 1024(%rcx,%xmm31,4), %xmm17 {%k1}
+// CHECK:  encoding: [0x62,0xa2,0xfd,0x01,0x93,0x8c,0xb9,0x00,0x04,0x00,0x00]
+          vgatherqpd 1024(%rcx,%xmm31,4), %xmm17 {%k1}
+
+// CHECK: vgatherqpd 123(%r14,%ymm31,8), %ymm29 {%k1}
+// CHECK:  encoding: [0x62,0x02,0xfd,0x21,0x93,0xac,0xfe,0x7b,0x00,0x00,0x00]
+          vgatherqpd 123(%r14,%ymm31,8), %ymm29 {%k1}
+
+// CHECK: vgatherqpd 256(%r9,%ymm31), %ymm29 {%k1}
+// CHECK:  encoding: [0x62,0x02,0xfd,0x21,0x93,0x6c,0x39,0x20]
+          vgatherqpd 256(%r9,%ymm31), %ymm29 {%k1}
+
+// CHECK: vgatherqpd 1024(%rcx,%ymm31,4), %ymm29 {%k1}
+// CHECK:  encoding: [0x62,0x22,0xfd,0x21,0x93,0xac,0xb9,0x00,0x04,0x00,0x00]
+          vgatherqpd 1024(%rcx,%ymm31,4), %ymm29 {%k1}
+
+// CHECK: vgatherqps 123(%r14,%xmm31,8), %xmm21 {%k1}
+// CHECK:  encoding: [0x62,0x82,0x7d,0x01,0x93,0xac,0xfe,0x7b,0x00,0x00,0x00]
+          vgatherqps 123(%r14,%xmm31,8), %xmm21 {%k1}
+
+// CHECK: vgatherqps 256(%r9,%xmm31), %xmm21 {%k1}
+// CHECK:  encoding: [0x62,0x82,0x7d,0x01,0x93,0x6c,0x39,0x40]
+          vgatherqps 256(%r9,%xmm31), %xmm21 {%k1}
+
+// CHECK: vgatherqps 1024(%rcx,%xmm31,4), %xmm21 {%k1}
+// CHECK:  encoding: [0x62,0xa2,0x7d,0x01,0x93,0xac,0xb9,0x00,0x04,0x00,0x00]
+          vgatherqps 1024(%rcx,%xmm31,4), %xmm21 {%k1}
+
+// CHECK: vgatherqps 123(%r14,%ymm31,8), %xmm19 {%k1}
+// CHECK:  encoding: [0x62,0x82,0x7d,0x21,0x93,0x9c,0xfe,0x7b,0x00,0x00,0x00]
+          vgatherqps 123(%r14,%ymm31,8), %xmm19 {%k1}
+
+// CHECK: vgatherqps 256(%r9,%ymm31), %xmm19 {%k1}
+// CHECK:  encoding: [0x62,0x82,0x7d,0x21,0x93,0x5c,0x39,0x40]
+          vgatherqps 256(%r9,%ymm31), %xmm19 {%k1}
+
+// CHECK: vgatherqps 1024(%rcx,%ymm31,4), %xmm19 {%k1}
+// CHECK:  encoding: [0x62,0xa2,0x7d,0x21,0x93,0x9c,0xb9,0x00,0x04,0x00,0x00]
+          vgatherqps 1024(%rcx,%ymm31,4), %xmm19 {%k1}
+
+// CHECK: vpscatterdd %xmm20, 123(%r14,%xmm31,8) {%k1}
+// CHECK:  encoding: [0x62,0x82,0x7d,0x01,0xa0,0xa4,0xfe,0x7b,0x00,0x00,0x00]
+          vpscatterdd %xmm20, 123(%r14,%xmm31,8) {%k1}
+
+// CHECK: vpscatterdd %xmm20, 123(%r14,%xmm31,8) {%k1}
+// CHECK:  encoding: [0x62,0x82,0x7d,0x01,0xa0,0xa4,0xfe,0x7b,0x00,0x00,0x00]
+          vpscatterdd %xmm20, 123(%r14,%xmm31,8) {%k1}
+
+// CHECK: vpscatterdd %xmm20, 256(%r9,%xmm31) {%k1}
+// CHECK:  encoding: [0x62,0x82,0x7d,0x01,0xa0,0x64,0x39,0x40]
+          vpscatterdd %xmm20, 256(%r9,%xmm31) {%k1}
+
+// CHECK: vpscatterdd %xmm20, 1024(%rcx,%xmm31,4) {%k1}
+// CHECK:  encoding: [0x62,0xa2,0x7d,0x01,0xa0,0xa4,0xb9,0x00,0x04,0x00,0x00]
+          vpscatterdd %xmm20, 1024(%rcx,%xmm31,4) {%k1}
+
+// CHECK: vpscatterdd %ymm28, 123(%r14,%ymm31,8) {%k1}
+// CHECK:  encoding: [0x62,0x02,0x7d,0x21,0xa0,0xa4,0xfe,0x7b,0x00,0x00,0x00]
+          vpscatterdd %ymm28, 123(%r14,%ymm31,8) {%k1}
+
+// CHECK: vpscatterdd %ymm28, 123(%r14,%ymm31,8) {%k1}
+// CHECK:  encoding: [0x62,0x02,0x7d,0x21,0xa0,0xa4,0xfe,0x7b,0x00,0x00,0x00]
+          vpscatterdd %ymm28, 123(%r14,%ymm31,8) {%k1}
+
+// CHECK: vpscatterdd %ymm28, 256(%r9,%ymm31) {%k1}
+// CHECK:  encoding: [0x62,0x02,0x7d,0x21,0xa0,0x64,0x39,0x40]
+          vpscatterdd %ymm28, 256(%r9,%ymm31) {%k1}
+
+// CHECK: vpscatterdd %ymm28, 1024(%rcx,%ymm31,4) {%k1}
+// CHECK:  encoding: [0x62,0x22,0x7d,0x21,0xa0,0xa4,0xb9,0x00,0x04,0x00,0x00]
+          vpscatterdd %ymm28, 1024(%rcx,%ymm31,4) {%k1}
+
+// CHECK: vpscatterdq %xmm21, 123(%r14,%xmm31,8) {%k1}
+// CHECK:  encoding: [0x62,0x82,0xfd,0x01,0xa0,0xac,0xfe,0x7b,0x00,0x00,0x00]
+          vpscatterdq %xmm21, 123(%r14,%xmm31,8) {%k1}
+
+// CHECK: vpscatterdq %xmm21, 123(%r14,%xmm31,8) {%k1}
+// CHECK:  encoding: [0x62,0x82,0xfd,0x01,0xa0,0xac,0xfe,0x7b,0x00,0x00,0x00]
+          vpscatterdq %xmm21, 123(%r14,%xmm31,8) {%k1}
+
+// CHECK: vpscatterdq %xmm21, 256(%r9,%xmm31) {%k1}
+// CHECK:  encoding: [0x62,0x82,0xfd,0x01,0xa0,0x6c,0x39,0x20]
+          vpscatterdq %xmm21, 256(%r9,%xmm31) {%k1}
+
+// CHECK: vpscatterdq %xmm21, 1024(%rcx,%xmm31,4) {%k1}
+// CHECK:  encoding: [0x62,0xa2,0xfd,0x01,0xa0,0xac,0xb9,0x00,0x04,0x00,0x00]
+          vpscatterdq %xmm21, 1024(%rcx,%xmm31,4) {%k1}
+
+// CHECK: vpscatterdq %ymm28, 123(%r14,%xmm31,8) {%k1}
+// CHECK:  encoding: [0x62,0x02,0xfd,0x21,0xa0,0xa4,0xfe,0x7b,0x00,0x00,0x00]
+          vpscatterdq %ymm28, 123(%r14,%xmm31,8) {%k1}
+
+// CHECK: vpscatterdq %ymm28, 123(%r14,%xmm31,8) {%k1}
+// CHECK:  encoding: [0x62,0x02,0xfd,0x21,0xa0,0xa4,0xfe,0x7b,0x00,0x00,0x00]
+          vpscatterdq %ymm28, 123(%r14,%xmm31,8) {%k1}
+
+// CHECK: vpscatterdq %ymm28, 256(%r9,%xmm31) {%k1}
+// CHECK:  encoding: [0x62,0x02,0xfd,0x21,0xa0,0x64,0x39,0x20]
+          vpscatterdq %ymm28, 256(%r9,%xmm31) {%k1}
+
+// CHECK: vpscatterdq %ymm28, 1024(%rcx,%xmm31,4) {%k1}
+// CHECK:  encoding: [0x62,0x22,0xfd,0x21,0xa0,0xa4,0xb9,0x00,0x04,0x00,0x00]
+          vpscatterdq %ymm28, 1024(%rcx,%xmm31,4) {%k1}
+
+// CHECK: vpscatterqd %xmm22, 123(%r14,%xmm31,8) {%k1}
+// CHECK:  encoding: [0x62,0x82,0x7d,0x01,0xa1,0xb4,0xfe,0x7b,0x00,0x00,0x00]
+          vpscatterqd %xmm22, 123(%r14,%xmm31,8) {%k1}
+
+// CHECK: vpscatterqd %xmm22, 123(%r14,%xmm31,8) {%k1}
+// CHECK:  encoding: [0x62,0x82,0x7d,0x01,0xa1,0xb4,0xfe,0x7b,0x00,0x00,0x00]
+          vpscatterqd %xmm22, 123(%r14,%xmm31,8) {%k1}
+
+// CHECK: vpscatterqd %xmm22, 256(%r9,%xmm31) {%k1}
+// CHECK:  encoding: [0x62,0x82,0x7d,0x01,0xa1,0x74,0x39,0x40]
+          vpscatterqd %xmm22, 256(%r9,%xmm31) {%k1}
+
+// CHECK: vpscatterqd %xmm22, 1024(%rcx,%xmm31,4) {%k1}
+// CHECK:  encoding: [0x62,0xa2,0x7d,0x01,0xa1,0xb4,0xb9,0x00,0x04,0x00,0x00]
+          vpscatterqd %xmm22, 1024(%rcx,%xmm31,4) {%k1}
+
+// CHECK: vpscatterqd %xmm24, 123(%r14,%ymm31,8) {%k1}
+// CHECK:  encoding: [0x62,0x02,0x7d,0x21,0xa1,0x84,0xfe,0x7b,0x00,0x00,0x00]
+          vpscatterqd %xmm24, 123(%r14,%ymm31,8) {%k1}
+
+// CHECK: vpscatterqd %xmm24, 123(%r14,%ymm31,8) {%k1}
+// CHECK:  encoding: [0x62,0x02,0x7d,0x21,0xa1,0x84,0xfe,0x7b,0x00,0x00,0x00]
+          vpscatterqd %xmm24, 123(%r14,%ymm31,8) {%k1}
+
+// CHECK: vpscatterqd %xmm24, 256(%r9,%ymm31) {%k1}
+// CHECK:  encoding: [0x62,0x02,0x7d,0x21,0xa1,0x44,0x39,0x40]
+          vpscatterqd %xmm24, 256(%r9,%ymm31) {%k1}
+
+// CHECK: vpscatterqd %xmm24, 1024(%rcx,%ymm31,4) {%k1}
+// CHECK:  encoding: [0x62,0x22,0x7d,0x21,0xa1,0x84,0xb9,0x00,0x04,0x00,0x00]
+          vpscatterqd %xmm24, 1024(%rcx,%ymm31,4) {%k1}
+
+// CHECK: vpscatterqq %xmm28, 123(%r14,%xmm31,8) {%k1}
+// CHECK:  encoding: [0x62,0x02,0xfd,0x01,0xa1,0xa4,0xfe,0x7b,0x00,0x00,0x00]
+          vpscatterqq %xmm28, 123(%r14,%xmm31,8) {%k1}
+
+// CHECK: vpscatterqq %xmm28, 123(%r14,%xmm31,8) {%k1}
+// CHECK:  encoding: [0x62,0x02,0xfd,0x01,0xa1,0xa4,0xfe,0x7b,0x00,0x00,0x00]
+          vpscatterqq %xmm28, 123(%r14,%xmm31,8) {%k1}
+
+// CHECK: vpscatterqq %xmm28, 256(%r9,%xmm31) {%k1}
+// CHECK:  encoding: [0x62,0x02,0xfd,0x01,0xa1,0x64,0x39,0x20]
+          vpscatterqq %xmm28, 256(%r9,%xmm31) {%k1}
+
+// CHECK: vpscatterqq %xmm28, 1024(%rcx,%xmm31,4) {%k1}
+// CHECK:  encoding: [0x62,0x22,0xfd,0x01,0xa1,0xa4,0xb9,0x00,0x04,0x00,0x00]
+          vpscatterqq %xmm28, 1024(%rcx,%xmm31,4) {%k1}
+
+// CHECK: vpscatterqq %ymm19, 123(%r14,%ymm31,8) {%k1}
+// CHECK:  encoding: [0x62,0x82,0xfd,0x21,0xa1,0x9c,0xfe,0x7b,0x00,0x00,0x00]
+          vpscatterqq %ymm19, 123(%r14,%ymm31,8) {%k1}
+
+// CHECK: vpscatterqq %ymm19, 123(%r14,%ymm31,8) {%k1}
+// CHECK:  encoding: [0x62,0x82,0xfd,0x21,0xa1,0x9c,0xfe,0x7b,0x00,0x00,0x00]
+          vpscatterqq %ymm19, 123(%r14,%ymm31,8) {%k1}
+
+// CHECK: vpscatterqq %ymm19, 256(%r9,%ymm31) {%k1}
+// CHECK:  encoding: [0x62,0x82,0xfd,0x21,0xa1,0x5c,0x39,0x20]
+          vpscatterqq %ymm19, 256(%r9,%ymm31) {%k1}
+
+// CHECK: vpscatterqq %ymm19, 1024(%rcx,%ymm31,4) {%k1}
+// CHECK:  encoding: [0x62,0xa2,0xfd,0x21,0xa1,0x9c,0xb9,0x00,0x04,0x00,0x00]
+          vpscatterqq %ymm19, 1024(%rcx,%ymm31,4) {%k1}
diff --git a/test/MC/X86/elf-note-nacl.ll b/test/MC/X86/elf-note-nacl.ll
deleted file mode 100644
index ce15455c84f..00000000000
--- a/test/MC/X86/elf-note-nacl.ll
+++ /dev/null
@@ -1,36 +0,0 @@
-; RUN: llc -filetype=obj -mtriple i686-none-nacl %s -o - \
-; RUN:   | llvm-objdump -triple i686 -s - | FileCheck --check-prefix=I386 %s
-
-; RUN: llc -filetype=obj -mtriple x86_64-none-nacl %s -o - \
-; RUN:   | llvm-objdump -triple x86_64 -s - | FileCheck --check-prefix=X8664 %s
-
-; Tests that NaCl object files contain an ELF note section that identifies them
-; to the binutils gold linker
-
-define void @main() {
-  ret void
-}
-
-; There appears to be no way for llvm-objdump to show flags for sections, or
-; to dump groups like readelf.
-; I386: .group
-; I386: .note.NaCl.ABI.x86-32
-; The contents of the words in the note section should be:
-;   sizeof "NaCl"
-;   sizeof "x86-32"
-;   1 (NT_VERSION)
-;   "NaCl" with nul termination and padding to align 4
-;   "x86-32" with nul termination and padding to align 4
-; I386-NEXT: 0000 05000000 07000000 01000000 4e61436c
-; I386-NEXT: 0010 00000000 7838362d 33320000
-
-; X8664: .group
-; X8664: .note.NaCl.ABI.x86-64
-; The contents of the words in the note section should be:
-;   sizeof "NaCl"
-;   sizeof "x86-64"
-;   1 (NT_VERSION)
-;   "NaCl" with nul termination and padding to align 4
-;   q"x86-64" with nul termination and padding to align 4
-; X8664-NEXT: 0000 05000000 07000000 01000000 4e61436c
-; X8664-NEXT: 0010 00000000 7838362d 36340000
diff --git a/test/MC/X86/faultmap-section-parsing.s b/test/MC/X86/faultmap-section-parsing.s
new file mode 100644
index 00000000000..758e70fe685
--- /dev/null
+++ b/test/MC/X86/faultmap-section-parsing.s
@@ -0,0 +1,29 @@
+// RUN: llvm-mc < %s -triple=x86_64-apple-macosx -filetype=obj -o - | llvm-objdump -fault-map-section - | FileCheck %s
+
+	.section	__LLVM_FAULTMAPS,__llvm_faultmaps
+__LLVM_FaultMaps:
+	.byte	1
+	.byte	0
+	.short	0
+	.long	2
+	.quad	0xFFDEAD
+	.long	1
+	.long	0
+	.long	1
+	.long	100
+	.long	200
+
+	.quad	0xFFDAED
+	.long	1
+	.long	0
+	.long	1
+	.long	400
+	.long	500
+
+// CHECK: FaultMap table:
+// CHECK-NEXT: Version: 0x1
+// CHECK-NEXT: NumFunctions: 2
+// CHECK-NEXT: FunctionAddress: 0xffdead, NumFaultingPCs: 1
+// CHECK-NEXT: Fault kind: FaultingLoad, faulting PC offset: 100, handling PC offset: 200
+// CHECK-NEXT: FunctionAddress: 0xffdaed, NumFaultingPCs: 1
+// CHECK-NEXT: Fault kind: FaultingLoad, faulting PC offset: 400, handling PC offset: 500
diff --git a/test/MC/X86/hex-immediates.s b/test/MC/X86/hex-immediates.s
new file mode 100644
index 00000000000..03f30423ae0
--- /dev/null
+++ b/test/MC/X86/hex-immediates.s
@@ -0,0 +1,10 @@
+# RUN: llvm-mc -filetype=obj %s -triple=x86_64-apple-darwin9 | llvm-objdump -d --print-imm-hex - | FileCheck %s
+
+# CHECK: movabsq	$0x7fffffffffffffff, %rcx
+movabsq	$0x7fffffffffffffff, %rcx
+# CHECK: leaq	0x3e2(%rip), %rdi
+leaq	0x3e2(%rip), %rdi
+# CHECK: subq	$0x40, %rsp
+subq	$0x40, %rsp
+# CHECK: leal	(,%r14,4), %eax
+leal	(,%r14,4), %eax
diff --git a/test/MC/X86/i386-darwin-frame-register.ll b/test/MC/X86/i386-darwin-frame-register.ll
index b382a709244..c994a1efd20 100644
--- a/test/MC/X86/i386-darwin-frame-register.ll
+++ b/test/MC/X86/i386-darwin-frame-register.ll
@@ -29,8 +29,8 @@ attributes #0 = { nounwind }
 !llvm.module.flags = !{!3, !4, !5}
 !llvm.ident = !{!6}
 
-!0 = !MDCompileUnit(language: DW_LANG_C99, producer: "clang version 3.7.0 (trunk 230514) (llvm/trunk 230518)", isOptimized: false, emissionKind: 1, file: !1, enums: !2, retainedTypes: !2, subprograms: !2, globals: !2, imports: !2)
-!1 = !MDFile(filename: "foo.c", directory: "/tmp")
+!0 = !DICompileUnit(language: DW_LANG_C99, producer: "clang version 3.7.0 (trunk 230514) (llvm/trunk 230518)", isOptimized: false, emissionKind: 1, file: !1, enums: !2, retainedTypes: !2, subprograms: !2, globals: !2, imports: !2)
+!1 = !DIFile(filename: "foo.c", directory: "/tmp")
 !2 = !{}
 !3 = !{i32 2, !"Dwarf Version", i32 2}
 !4 = !{i32 2, !"Debug Info Version", i32 3}
diff --git a/test/MC/X86/inline-asm-obj.ll b/test/MC/X86/inline-asm-obj.ll
new file mode 100644
index 00000000000..2ee998dbc45
--- /dev/null
+++ b/test/MC/X86/inline-asm-obj.ll
@@ -0,0 +1,13 @@
+; RUN: llc %s -o - | llvm-mc -triple=x86_64-pc-linux -o %t1 -filetype=obj
+; RUN: llc %s -o %t2 -filetype=obj
+; RUN: cmp %t1 %t2
+
+; Test that we can handle inline assembly referring to a temporary label.
+; We crashed when using direct object emission in the past.
+
+target triple = "x86_64-unknown-linux-gnu"
+
+define void @fj()  {
+  call void asm "bsr $0,%eax", "o"(i32 1)
+  ret void
+}
diff --git a/test/MC/X86/intel-syntax-avx512.s b/test/MC/X86/intel-syntax-avx512.s
index af4e98ccba2..86a1af8bc16 100644
--- a/test/MC/X86/intel-syntax-avx512.s
+++ b/test/MC/X86/intel-syntax-avx512.s
@@ -1,34 +1,258 @@
-// RUN: llvm-mc -triple x86_64-unknown-unknown -x86-asm-syntax=intel -mcpu=knl --show-encoding %s | FileCheck %s
+// RUN: llvm-mc -triple x86_64-unknown-unknown -x86-asm-syntax=intel -output-asm-variant=1 -mcpu=knl --show-encoding %s | FileCheck %s
 
-// CHECK: vaddps (%rax), %zmm1, %zmm1
+// CHECK: vaddps  zmm1 , zmm1, zmmword ptr [rax]
 // CHECK: encoding: [0x62,0xf1,0x74,0x48,0x58,0x08]
 vaddps zmm1, zmm1, zmmword ptr [rax]
 
-// CHECK: vaddpd  %zmm2, %zmm1, %zmm1
+// CHECK: vaddpd  zmm1 , zmm1, zmm2
 // CHECK:  encoding: [0x62,0xf1,0xf5,0x48,0x58,0xca]
 vaddpd zmm1,zmm1,zmm2
 
-// CHECK: vaddpd  %zmm2, %zmm1, %zmm1 {%k5}
+// CHECK: vaddpd  zmm1 {k5}, zmm1, zmm2
 // CHECK:  encoding: [0x62,0xf1,0xf5,0x4d,0x58,0xca]
 vaddpd zmm1{k5},zmm1,zmm2
 
-// CHECK: vaddpd  %zmm2, %zmm1, %zmm1 {%k5} {z}
+// CHECK: vaddpd zmm1 {k5} {z}, zmm1, zmm2
 // CHECK:  encoding: [0x62,0xf1,0xf5,0xcd,0x58,0xca]
 vaddpd zmm1{k5} {z},zmm1,zmm2
 
-// CHECK: vaddpd  {rn-sae}, %zmm2, %zmm1, %zmm1
+// CHECK: vaddpd zmm1 , zmm1, zmm2, {rn-sae}
 // CHECK:  encoding: [0x62,0xf1,0xf5,0x18,0x58,0xca]
 vaddpd zmm1,zmm1,zmm2,{rn-sae}
 
-// CHECK: vaddpd  {ru-sae}, %zmm2, %zmm1, %zmm1
+// CHECK: vaddpd zmm1 , zmm1, zmm2, {ru-sae}
 // CHECK:  encoding: [0x62,0xf1,0xf5,0x58,0x58,0xca]
 vaddpd zmm1,zmm1,zmm2,{ru-sae}
 
-// CHECK:  vaddpd  {rd-sae}, %zmm2, %zmm1, %zmm1
+// CHECK:  vaddpd zmm1 , zmm1, zmm2, {rd-sae}
 // CHECK:  encoding: [0x62,0xf1,0xf5,0x38,0x58,0xca]
 vaddpd zmm1,zmm1,zmm2,{rd-sae}
 
-// CHECK: vaddpd  {rz-sae}, %zmm2, %zmm1, %zmm1
+// CHECK: vaddpd zmm1 , zmm1, zmm2, {rz-sae}
 // CHECK:  encoding: [0x62,0xf1,0xf5,0x78,0x58,0xca]
 vaddpd zmm1,zmm1,zmm2,{rz-sae}
 
+// CHECK: vcmppd k2 , zmm12, zmm26, 171
+// CHECK:  encoding: [0x62,0x91,0x9d,0x48,0xc2,0xd2,0xab]
+          vcmppd k2,zmm12,zmm26,0xab
+
+// CHECK: vcmppd k2 {k3}, zmm12, zmm26, 171
+// CHECK:  encoding: [0x62,0x91,0x9d,0x4b,0xc2,0xd2,0xab]
+          vcmppd k2{k3},zmm12,zmm26,0xab
+
+// CHECK: vcmppd k2 , zmm12, zmm26,{sae}, 171
+// CHECK:  encoding: [0x62,0x91,0x9d,0x18,0xc2,0xd2,0xab]
+          vcmppd k2,zmm12,zmm26,{sae},0xab
+
+// CHECK: vcmppd k2 , zmm12, zmm26, 123
+// CHECK:  encoding: [0x62,0x91,0x9d,0x48,0xc2,0xd2,0x7b]
+          vcmppd k2 ,zmm12,zmm26,0x7b
+
+// CHECK: vcmppd k2 , zmm12, zmm26,{sae}, 123
+// CHECK:  encoding: [0x62,0x91,0x9d,0x18,0xc2,0xd2,0x7b]
+          vcmppd k2,zmm12,zmm26,{sae},0x7b
+
+// CHECK: vcmppd k2 , zmm12, zmmword ptr [rcx], 123
+// CHECK:  encoding: [0x62,0xf1,0x9d,0x48,0xc2,0x11,0x7b]
+          vcmppd k2,zmm12,zmmword PTR [rcx],0x7b
+
+// CHECK: vcmppd  k2 , zmm12, zmmword ptr [rax + 8*r14 + 291], 123
+// CHECK:  encoding: [0x62,0xb1,0x9d,0x48,0xc2,0x94,0xf0,0x23,0x01,0x00,0x00,0x7b]
+          vcmppd k2 ,zmm12,zmmword PTR [rax+r14*8+0x123],0x7b
+
+// CHECK: vcmppd  k2 , zmm12, qword ptr [rcx]{1to8}, 123
+// CHECK:  encoding: [0x62,0xf1,0x9d,0x58,0xc2,0x11,0x7b]
+          vcmppd k2,zmm12,QWORD PTR [rcx]{1to8},0x7b
+
+// CHECK: vcmppd  k2 , zmm12, zmmword ptr [rdx + 8128], 123
+// CHECK:  encoding: [0x62,0xf1,0x9d,0x48,0xc2,0x52,0x7f,0x7b]
+          vcmppd k2,zmm12,zmmword PTR [rdx+0x1fc0],0x7b
+
+// CHECK: vcmppd  k2 , zmm12, zmmword ptr [rdx + 8192], 123
+// CHECK:  encoding: [0x62,0xf1,0x9d,0x48,0xc2,0x92,0x00,0x20,0x00,0x00,0x7b]
+          vcmppd k2,zmm12,zmmword PTR [rdx+0x2000],0x7b
+
+// CHECK: vcmppd  k2 , zmm12, zmmword ptr [rdx - 8192], 123
+// CHECK:  encoding: [0x62,0xf1,0x9d,0x48,0xc2,0x52,0x80,0x7b]
+          vcmppd k2,zmm12,zmmword PTR [rdx-0x2000],0x7b
+
+// CHECK: vcmppd  k2 , zmm12, zmmword ptr [rdx - 8256], 123
+// CHECK:  encoding: [0x62,0xf1,0x9d,0x48,0xc2,0x92,0xc0,0xdf,0xff,0xff,0x7b]
+          vcmppd k2,zmm12,zmmword PTR [rdx-0x2040],0x7b
+
+// CHECK: vcmppd  k2 , zmm12, qword ptr [rdx + 1016]{1to8}, 123
+// CHECK:  encoding: [0x62,0xf1,0x9d,0x58,0xc2,0x52,0x7f,0x7b]
+          vcmppd k2,zmm12,QWORD PTR [rdx+0x3f8]{1to8},0x7b
+
+// CHECK: vcmppd  k2 , zmm12, qword ptr [rdx + 1024]{1to8}, 123
+// CHECK:  encoding: [0x62,0xf1,0x9d,0x58,0xc2,0x92,0x00,0x04,0x00,0x00,0x7b]
+          vcmppd k2,zmm12,QWORD PTR [rdx+0x400]{1to8},0x7b
+
+// CHECK: vcmppd  k2 , zmm12, qword ptr [rdx - 1024]{1to8}, 123
+// CHECK:  encoding: [0x62,0xf1,0x9d,0x58,0xc2,0x52,0x80,0x7b]
+          vcmppd k2,zmm12,QWORD PTR [rdx-0x400]{1to8},0x7b
+
+// CHECK: vcmppd  k2 , zmm12, qword ptr [rdx - 1032]{1to8}, 123
+// CHECK:  encoding: [0x62,0xf1,0x9d,0x58,0xc2,0x92,0xf8,0xfb,0xff,0xff,0x7b]
+          vcmppd k2,zmm12,QWORD PTR [rdx-0x408]{1to8},0x7b
+
+// CHECK: vcmpps  k2 , zmm17, zmm22, 171
+// CHECK:  encoding: [0x62,0xb1,0x74,0x40,0xc2,0xd6,0xab]
+          vcmpps k2,zmm17,zmm22,0xab
+
+// CHECK: vcmpps  k2 {k3}, zmm17, zmm22, 171
+// CHECK:  encoding: [0x62,0xb1,0x74,0x43,0xc2,0xd6,0xab]
+          vcmpps k2{k3},zmm17,zmm22,0xab
+
+// CHECK: vcmpps  k2 , zmm17, zmm22,{sae}, 171
+// CHECK:  encoding: [0x62,0xb1,0x74,0x10,0xc2,0xd6,0xab]
+          vcmpps k2,zmm17,zmm22,{sae},0xab
+
+// CHECK: vcmpps  k2 , zmm17, zmm22, 123
+// CHECK:  encoding: [0x62,0xb1,0x74,0x40,0xc2,0xd6,0x7b]
+          vcmpps k2,zmm17,zmm22,0x7b
+
+// CHECK: vcmpps  k2 , zmm17, zmm22,{sae}, 123
+// CHECK:  encoding: [0x62,0xb1,0x74,0x10,0xc2,0xd6,0x7b]
+          vcmpps k2,zmm17,zmm22,{sae},0x7b
+
+// CHECK: vcmpps  k2 , zmm17, zmmword ptr [rcx], 123
+// CHECK:  encoding: [0x62,0xf1,0x74,0x40,0xc2,0x11,0x7b]
+          vcmpps k2,zmm17,zmmword PTR [rcx],0x7b
+
+// CHECK: vcmpps  k2 , zmm17, zmmword ptr [rax + 8*r14 + 291], 123
+// CHECK:  encoding: [0x62,0xb1,0x74,0x40,0xc2,0x94,0xf0,0x23,0x01,0x00,0x00,0x7b]
+          vcmpps k2,zmm17,zmmword PTR [rax+r14*8+0x123],0x7b
+
+// CHECK: vcmpps  k2 , zmm17, dword ptr [rcx]{1to16}, 123
+// CHECK:  encoding: [0x62,0xf1,0x74,0x50,0xc2,0x11,0x7b]
+          vcmpps k2,zmm17,DWORD PTR [rcx]{1to16},0x7b
+
+// CHECK: vcmpps  k2 , zmm17, zmmword ptr [rdx + 8128], 123
+// CHECK:  encoding: [0x62,0xf1,0x74,0x40,0xc2,0x52,0x7f,0x7b]
+          vcmpps k2,zmm17,zmmword PTR [rdx+0x1fc0],0x7b
+
+// CHECK: vcmpps  k2 , zmm17, zmmword ptr [rdx + 8192], 123
+// CHECK:  encoding: [0x62,0xf1,0x74,0x40,0xc2,0x92,0x00,0x20,0x00,0x00,0x7b]
+          vcmpps k2,zmm17,zmmword PTR [rdx+0x2000],0x7b
+
+// CHECK: vcmpps  k2 , zmm17, zmmword ptr [rdx - 8192], 123
+// CHECK:  encoding: [0x62,0xf1,0x74,0x40,0xc2,0x52,0x80,0x7b]
+          vcmpps k2,zmm17,zmmword PTR [rdx-0x2000],0x7b
+
+// CHECK: vcmpps  k2 , zmm17, zmmword ptr [rdx - 8256], 123
+// CHECK:  encoding: [0x62,0xf1,0x74,0x40,0xc2,0x92,0xc0,0xdf,0xff,0xff,0x7b]
+          vcmpps k2,zmm17,zmmword PTR [rdx-0x2040],0x7b
+
+// CHECK: vcmpps  k2 , zmm17, dword ptr [rdx + 508]{1to16}, 123
+// CHECK:  encoding: [0x62,0xf1,0x74,0x50,0xc2,0x52,0x7f,0x7b]
+          vcmpps k2,zmm17,DWORD PTR [rdx+0x1fc]{1to16},0x7b
+
+// CHECK: vcmpps  k2 , zmm17, dword ptr [rdx + 512]{1to16}, 123
+// CHECK:  encoding: [0x62,0xf1,0x74,0x50,0xc2,0x92,0x00,0x02,0x00,0x00,0x7b]
+          vcmpps k2,zmm17,DWORD PTR [rdx+0x200]{1to16},0x7b
+
+// CHECK: vcmpps  k2 , zmm17, dword ptr [rdx - 512]{1to16}, 123
+// CHECK:  encoding: [0x62,0xf1,0x74,0x50,0xc2,0x52,0x80,0x7b]
+          vcmpps k2,zmm17,DWORD PTR [rdx-0x200]{1to16},0x7b
+
+// CHECK: vcmpps  k2 , zmm17, dword ptr [rdx - 516]{1to16}, 123
+// CHECK:  encoding: [0x62,0xf1,0x74,0x50,0xc2,0x92,0xfc,0xfd,0xff,0xff,0x7b]
+          vcmpps k2,zmm17,DWORD PTR [rdx-0x204]{1to16},0x7b
+
+
+// CHECK:  vfixupimmss  xmm15 , xmm18, xmm28, 171
+// CHECK:  encoding: [0x62,0x13,0x6d,0x00,0x55,0xfc,0xab]
+          vfixupimmss xmm15,xmm18,xmm28,0xab
+
+// CHECK:  vfixupimmss  xmm15 {k5}, xmm18, xmm28, 171
+// CHECK:  encoding: [0x62,0x13,0x6d,0x05,0x55,0xfc,0xab]
+          vfixupimmss xmm15{k5},xmm18,xmm28,0xab
+
+// CHECK:  vfixupimmss  xmm15 {k5} {z}, xmm18, xmm28, 171
+// CHECK:  encoding: [0x62,0x13,0x6d,0x85,0x55,0xfc,0xab]
+          vfixupimmss xmm15{k5} {z},xmm18,xmm28,0xab
+
+// CHECK:  vfixupimmss  xmm15 , xmm18, xmm28,{sae}, 171
+// CHECK:  encoding: [0x62,0x13,0x6d,0x10,0x55,0xfc,0xab]
+          vfixupimmss xmm15,xmm18,xmm28,{sae},0xab
+
+// CHECK:  vfixupimmss  xmm15 , xmm18, xmm28, 123
+// CHECK:  encoding: [0x62,0x13,0x6d,0x00,0x55,0xfc,0x7b]
+          vfixupimmss xmm15,xmm18,xmm28,0x7b
+
+// CHECK:  vfixupimmss  xmm15 , xmm18, xmm28,{sae}, 123
+// CHECK:  encoding: [0x62,0x13,0x6d,0x10,0x55,0xfc,0x7b]
+          vfixupimmss xmm15,xmm18,xmm28,{sae},0x7b
+
+// CHECK:  vfixupimmss  xmm15 , xmm18, dword ptr [rcx], 123
+// CHECK:  encoding: [0x62,0x73,0x6d,0x00,0x55,0x39,0x7b]
+          vfixupimmss xmm15,xmm18,DWORD PTR [rcx],0x7b
+
+// CHECK:  vfixupimmss  xmm15 , xmm18, dword ptr [rax + 8*r14 + 291], 123
+// CHECK:  encoding: [0x62,0x33,0x6d,0x00,0x55,0xbc,0xf0,0x23,0x01,0x00,0x00,0x7b]
+          vfixupimmss xmm15,xmm18,DWORD PTR [rax+r14*8+0x123],0x7b
+
+// CHECK:  vfixupimmss  xmm15 , xmm18, dword ptr [rdx + 508], 123
+// CHECK:  encoding: [0x62,0x73,0x6d,0x00,0x55,0x7a,0x7f,0x7b]
+          vfixupimmss xmm15,xmm18,DWORD PTR [rdx+0x1fc],0x7b
+
+// CHECK:  vfixupimmss  xmm15 , xmm18, dword ptr [rdx + 512], 123
+// CHECK:  encoding: [0x62,0x73,0x6d,0x00,0x55,0xba,0x00,0x02,0x00,0x00,0x7b]
+          vfixupimmss xmm15,xmm18,DWORD PTR [rdx+0x200],0x7b
+
+// CHECK:  vfixupimmss  xmm15 , xmm18, dword ptr [rdx - 512], 123
+// CHECK:  encoding: [0x62,0x73,0x6d,0x00,0x55,0x7a,0x80,0x7b]
+          vfixupimmss xmm15,xmm18,DWORD PTR [rdx-0x200],0x7b
+
+// CHECK:  vfixupimmss  xmm15 , xmm18, dword ptr [rdx - 516], 123
+// CHECK:  encoding: [0x62,0x73,0x6d,0x00,0x55,0xba,0xfc,0xfd,0xff,0xff,0x7b]
+          vfixupimmss xmm15,xmm18,DWORD PTR [rdx-0x204],0x7b
+
+// CHECK:  vfixupimmsd  xmm13 , xmm26, xmm5, 171
+// CHECK:  encoding: [0x62,0x73,0xad,0x00,0x55,0xed,0xab]
+          vfixupimmsd xmm13,xmm26,xmm5,0xab
+
+// CHECK:  vfixupimmsd  xmm13 {k6}, xmm26, xmm5, 171
+// CHECK:  encoding: [0x62,0x73,0xad,0x06,0x55,0xed,0xab]
+          vfixupimmsd xmm13{k6},xmm26,xmm5,0xab
+
+// CHECK:  vfixupimmsd  xmm13 {k6} {z}, xmm26, xmm5, 171
+// CHECK:  encoding: [0x62,0x73,0xad,0x86,0x55,0xed,0xab]
+          vfixupimmsd xmm13{k6} {z},xmm26,xmm5,0xab
+
+// CHECK:  vfixupimmsd  xmm13 , xmm26, xmm5,{sae}, 171
+// CHECK:  encoding: [0x62,0x73,0xad,0x10,0x55,0xed,0xab]
+          vfixupimmsd xmm13,xmm26,xmm5,{sae},0xab
+
+// CHECK:  vfixupimmsd  xmm13 , xmm26, xmm5, 123
+// CHECK:  encoding: [0x62,0x73,0xad,0x00,0x55,0xed,0x7b]
+          vfixupimmsd xmm13,xmm26,xmm5,0x7b
+
+// CHECK:  vfixupimmsd  xmm13 , xmm26, xmm5,{sae}, 123
+// CHECK:  encoding: [0x62,0x73,0xad,0x10,0x55,0xed,0x7b]
+          vfixupimmsd xmm13,xmm26,xmm5,{sae},0x7b
+
+// CHECK:  vfixupimmsd  xmm13 , xmm26, qword ptr [rcx], 123
+// CHECK:  encoding: [0x62,0x73,0xad,0x00,0x55,0x29,0x7b]
+          vfixupimmsd xmm13,xmm26,QWORD PTR [rcx],0x7b
+
+// CHECK:  vfixupimmsd  xmm13 , xmm26, qword ptr [rax + 8*r14 + 291], 123
+// CHECK:  encoding: [0x62,0x33,0xad,0x00,0x55,0xac,0xf0,0x23,0x01,0x00,0x00,0x7b]
+          vfixupimmsd xmm13,xmm26,QWORD PTR [rax+r14*8+0x123],0x7b
+
+// CHECK:  vfixupimmsd  xmm13 , xmm26, qword ptr [rdx + 1016], 123
+// CHECK:  encoding: [0x62,0x73,0xad,0x00,0x55,0x6a,0x7f,0x7b]
+          vfixupimmsd xmm13,xmm26,QWORD PTR [rdx+0x3f8],0x7b
+
+// CHECK:  vfixupimmsd  xmm13 , xmm26, qword ptr [rdx + 1024], 123
+// CHECK:  encoding: [0x62,0x73,0xad,0x00,0x55,0xaa,0x00,0x04,0x00,0x00,0x7b]
+          vfixupimmsd xmm13,xmm26,QWORD PTR [rdx+0x400],0x7b
+
+// CHECK:  vfixupimmsd  xmm13 , xmm26, qword ptr [rdx - 1024], 123
+// CHECK:  encoding: [0x62,0x73,0xad,0x00,0x55,0x6a,0x80,0x7b]
+          vfixupimmsd xmm13,xmm26,QWORD PTR [rdx-0x400],0x7b
+
+// CHECK:  vfixupimmsd  xmm13 , xmm26, qword ptr [rdx - 1032], 123
+// CHECK:  encoding: [0x62,0x73,0xad,0x00,0x55,0xaa,0xf8,0xfb,0xff,0xff,0x7b]
+          vfixupimmsd xmm13,xmm26,QWORD PTR [rdx-0x408],0x7b
diff --git a/test/MC/X86/intel-syntax-bitwise-ops.s b/test/MC/X86/intel-syntax-bitwise-ops.s
index c9c9b1d17b2..1f09996fe91 100644
--- a/test/MC/X86/intel-syntax-bitwise-ops.s
+++ b/test/MC/X86/intel-syntax-bitwise-ops.s
@@ -20,3 +20,5 @@
     add eax, 9876 >> 1
 // CHECK: addl	$19752, %eax
     add eax, 9876 << 1
+// CHECK: addl	$5, %eax
+    add eax, 6 ^ 3
diff --git a/test/MC/X86/intel-syntax-print.ll b/test/MC/X86/intel-syntax-print.ll
new file mode 100644
index 00000000000..14ef46705be
--- /dev/null
+++ b/test/MC/X86/intel-syntax-print.ll
@@ -0,0 +1,10 @@
+; RUN: llc -x86-asm-syntax=intel < %s | FileCheck %s -check-prefix=INTEL
+; RUN: llc -x86-asm-syntax=att < %s | FileCheck %s -check-prefix=ATT
+
+; INTEL: .intel_syntax noprefix
+; ATT-NOT: .intel_syntax noprefix
+target triple = "x86_64-unknown-unknown"
+define i32 @test() {
+entry:
+  ret i32 0
+}
diff --git a/test/MC/X86/intel-syntax-x86-64-avx512f_vl.s b/test/MC/X86/intel-syntax-x86-64-avx512f_vl.s
new file mode 100644
index 00000000000..6de59da40b3
--- /dev/null
+++ b/test/MC/X86/intel-syntax-x86-64-avx512f_vl.s
@@ -0,0 +1,225 @@
+// RUN: llvm-mc -triple x86_64-unknown-unknown -mcpu=knl -mattr=+avx512vl -x86-asm-syntax=intel -output-asm-variant=1 --show-encoding %s | FileCheck %s
+
+// CHECK:  vcmppd k3 , xmm27, xmm23, 171
+// CHECK:  encoding: [0x62,0xb1,0xa5,0x00,0xc2,0xdf,0xab]
+          vcmppd k3,xmm27,xmm23,0xab
+
+// CHECK:  vcmppd k3 {k5}, xmm27, xmm23, 171
+// CHECK:  encoding: [0x62,0xb1,0xa5,0x05,0xc2,0xdf,0xab]
+          vcmppd k3{k5},xmm27,xmm23,0xab
+
+// CHECK:  vcmppd k3 , xmm27, xmm23, 123
+// CHECK:  encoding: [0x62,0xb1,0xa5,0x00,0xc2,0xdf,0x7b]
+          vcmppd k3,xmm27,xmm23,0x7b
+
+// CHECK:  vcmppd k3 , xmm27, xmmword ptr [rcx], 123
+// CHECK:  encoding: [0x62,0xf1,0xa5,0x00,0xc2,0x19,0x7b]
+          vcmppd k3,xmm27,XMMWORD PTR [rcx],0x7b
+
+// CHECK:  vcmppd k3 , xmm27, xmmword ptr [rax + 8*r14 + 291], 123
+// CHECK:  encoding: [0x62,0xb1,0xa5,0x00,0xc2,0x9c,0xf0,0x23,0x01,0x00,0x00,0x7b]
+          vcmppd k3,xmm27,XMMWORD PTR [rax+r14*8+0x123],0x7b
+
+// CHECK:  vcmppd k3 , xmm27, qword ptr [rcx]{1to2}, 123
+// CHECK:  encoding: [0x62,0xf1,0xa5,0x10,0xc2,0x19,0x7b]
+          vcmppd k3,xmm27,QWORD PTR [rcx]{1to2},0x7b
+
+// CHECK:  vcmppd k3 , xmm27, xmmword ptr [rdx + 2032], 123
+// CHECK:  encoding: [0x62,0xf1,0xa5,0x00,0xc2,0x5a,0x7f,0x7b]
+          vcmppd k3,xmm27,XMMWORD PTR [rdx+0x7f0],0x7b
+
+// CHECK:  vcmppd k3 , xmm27, xmmword ptr [rdx + 2048], 123
+// CHECK:  encoding: [0x62,0xf1,0xa5,0x00,0xc2,0x9a,0x00,0x08,0x00,0x00,0x7b]
+          vcmppd k3,xmm27,XMMWORD PTR [rdx+0x800],0x7b
+
+// CHECK:  vcmppd k3 , xmm27, xmmword ptr [rdx - 2048], 123
+// CHECK:  encoding: [0x62,0xf1,0xa5,0x00,0xc2,0x5a,0x80,0x7b]
+          vcmppd k3,xmm27,XMMWORD PTR [rdx-0x800],0x7b
+
+// CHECK:  vcmppd k3 , xmm27, xmmword ptr [rdx - 2064], 123
+// CHECK:  encoding: [0x62,0xf1,0xa5,0x00,0xc2,0x9a,0xf0,0xf7,0xff,0xff,0x7b]
+          vcmppd k3,xmm27,XMMWORD PTR [rdx-0x810],0x7b
+
+// CHECK:  vcmppd k3 , xmm27, qword ptr [rdx + 1016]{1to2}, 123
+// CHECK:  encoding: [0x62,0xf1,0xa5,0x10,0xc2,0x5a,0x7f,0x7b]
+          vcmppd k3,xmm27,QWORD PTR [rdx+0x3f8]{1to2},0x7b
+
+// CHECK:  vcmppd k3 , xmm27, qword ptr [rdx + 1024]{1to2}, 123
+// CHECK:  encoding: [0x62,0xf1,0xa5,0x10,0xc2,0x9a,0x00,0x04,0x00,0x00,0x7b]
+          vcmppd k3,xmm27,QWORD PTR [rdx+0x400]{1to2},0x7b
+
+// CHECK:  vcmppd k3 , xmm27, qword ptr [rdx - 1024]{1to2}, 123
+// CHECK:  encoding: [0x62,0xf1,0xa5,0x10,0xc2,0x5a,0x80,0x7b]
+          vcmppd k3,xmm27,QWORD PTR [rdx-0x400]{1to2},0x7b
+
+// CHECK:  vcmppd k3 , xmm27, qword ptr [rdx - 1032]{1to2}, 123
+// CHECK:  encoding: [0x62,0xf1,0xa5,0x10,0xc2,0x9a,0xf8,0xfb,0xff,0xff,0x7b]
+          vcmppd k3,xmm27,QWORD PTR [rdx-0x408]{1to2},0x7b
+
+// CHECK:  vcmppd k4 , ymm17, ymm27, 171
+// CHECK:  encoding: [0x62,0x91,0xf5,0x20,0xc2,0xe3,0xab]
+          vcmppd k4,ymm17,ymm27,0xab
+
+// CHECK:  vcmppd k4 {k7}, ymm17, ymm27, 171
+// CHECK:  encoding: [0x62,0x91,0xf5,0x27,0xc2,0xe3,0xab]
+          vcmppd k4{k7},ymm17,ymm27,0xab
+
+// CHECK:  vcmppd k4 , ymm17, ymm27, 123
+// CHECK:  encoding: [0x62,0x91,0xf5,0x20,0xc2,0xe3,0x7b]
+          vcmppd k4,ymm17,ymm27,0x7b
+
+// CHECK:  vcmppd k4 , ymm17, ymmword ptr [rcx], 123
+// CHECK:  encoding: [0x62,0xf1,0xf5,0x20,0xc2,0x21,0x7b]
+          vcmppd k4,ymm17,YMMWORD PTR [rcx],0x7b
+
+// CHECK:  vcmppd k4 , ymm17, ymmword ptr [rax + 8*r14 + 291], 123
+// CHECK:  encoding: [0x62,0xb1,0xf5,0x20,0xc2,0xa4,0xf0,0x23,0x01,0x00,0x00,0x7b]
+          vcmppd k4,ymm17,YMMWORD PTR [rax+r14*8+0x123],0x7b
+
+// CHECK:  vcmppd k4 , ymm17, qword ptr [rcx]{1to4}, 123
+// CHECK:  encoding: [0x62,0xf1,0xf5,0x30,0xc2,0x21,0x7b]
+          vcmppd k4,ymm17,QWORD PTR [rcx]{1to4},0x7b
+
+// CHECK:  vcmppd k4 , ymm17, ymmword ptr [rdx + 4064], 123
+// CHECK:  encoding: [0x62,0xf1,0xf5,0x20,0xc2,0x62,0x7f,0x7b]
+          vcmppd k4,ymm17,YMMWORD PTR [rdx+0xfe0],0x7b
+
+// CHECK:  vcmppd k4 , ymm17, ymmword ptr [rdx + 4096], 123
+// CHECK:  encoding: [0x62,0xf1,0xf5,0x20,0xc2,0xa2,0x00,0x10,0x00,0x00,0x7b]
+          vcmppd k4,ymm17,YMMWORD PTR [rdx+0x1000],0x7b
+
+// CHECK:  vcmppd k4 , ymm17, ymmword ptr [rdx - 4096], 123
+// CHECK:  encoding: [0x62,0xf1,0xf5,0x20,0xc2,0x62,0x80,0x7b]
+          vcmppd k4,ymm17,YMMWORD PTR [rdx-0x1000],0x7b
+
+// CHECK:  vcmppd k4 , ymm17, ymmword ptr [rdx - 4128], 123
+// CHECK:  encoding: [0x62,0xf1,0xf5,0x20,0xc2,0xa2,0xe0,0xef,0xff,0xff,0x7b]
+          vcmppd k4,ymm17,YMMWORD PTR [rdx-0x1020],0x7b
+
+// CHECK:  vcmppd k4 , ymm17, qword ptr [rdx + 1016]{1to4}, 123
+// CHECK:  encoding: [0x62,0xf1,0xf5,0x30,0xc2,0x62,0x7f,0x7b]
+          vcmppd k4,ymm17,QWORD PTR [rdx+0x3f8]{1to4},0x7b
+
+// CHECK:  vcmppd k4 , ymm17, qword ptr [rdx + 1024]{1to4}, 123
+// CHECK:  encoding: [0x62,0xf1,0xf5,0x30,0xc2,0xa2,0x00,0x04,0x00,0x00,0x7b]
+          vcmppd k4,ymm17,QWORD PTR [rdx+0x400]{1to4},0x7b
+
+// CHECK:  vcmppd k4 , ymm17, qword ptr [rdx - 1024]{1to4}, 123
+// CHECK:  encoding: [0x62,0xf1,0xf5,0x30,0xc2,0x62,0x80,0x7b]
+          vcmppd k4,ymm17,QWORD PTR [rdx-0x400]{1to4},0x7b
+
+// CHECK:  vcmppd k4 , ymm17, qword ptr [rdx - 1032]{1to4}, 123
+// CHECK:  encoding: [0x62,0xf1,0xf5,0x30,0xc2,0xa2,0xf8,0xfb,0xff,0xff,0x7b]
+          vcmppd k4,ymm17,QWORD PTR [rdx-0x408]{1to4},0x7b
+
+// CHECK:  vcmpps k4 , xmm29, xmm28, 171
+// CHECK:  encoding: [0x62,0x91,0x14,0x00,0xc2,0xe4,0xab]
+          vcmpps k4,xmm29,xmm28,0xab
+
+// CHECK:  vcmpps k4 {k2}, xmm29, xmm28, 171
+// CHECK:  encoding: [0x62,0x91,0x14,0x02,0xc2,0xe4,0xab]
+          vcmpps k4{k2},xmm29,xmm28,0xab
+
+// CHECK:  vcmpps k4 , xmm29, xmm28, 123
+// CHECK:  encoding: [0x62,0x91,0x14,0x00,0xc2,0xe4,0x7b]
+          vcmpps k4,xmm29,xmm28,0x7b
+
+// CHECK:  vcmpps k4 , xmm29, xmmword ptr [rcx], 123
+// CHECK:  encoding: [0x62,0xf1,0x14,0x00,0xc2,0x21,0x7b]
+          vcmpps k4,xmm29,XMMWORD PTR [rcx],0x7b
+
+// CHECK:  vcmpps k4 , xmm29, xmmword ptr [rax + 8*r14 + 291], 123
+// CHECK:  encoding: [0x62,0xb1,0x14,0x00,0xc2,0xa4,0xf0,0x23,0x01,0x00,0x00,0x7b]
+          vcmpps k4,xmm29,XMMWORD PTR [rax+r14*8+0x123],0x7b
+
+// CHECK:  vcmpps k4 , xmm29, dword ptr [rcx]{1to4}, 123
+// CHECK:  encoding: [0x62,0xf1,0x14,0x10,0xc2,0x21,0x7b]
+          vcmpps k4,xmm29,DWORD PTR [rcx]{1to4},0x7b
+
+// CHECK:  vcmpps k4 , xmm29, xmmword ptr [rdx + 2032], 123
+// CHECK:  encoding: [0x62,0xf1,0x14,0x00,0xc2,0x62,0x7f,0x7b]
+          vcmpps k4,xmm29,XMMWORD PTR [rdx+0x7f0],0x7b
+
+// CHECK:  vcmpps k4 , xmm29, xmmword ptr [rdx + 2048], 123
+// CHECK:  encoding: [0x62,0xf1,0x14,0x00,0xc2,0xa2,0x00,0x08,0x00,0x00,0x7b]
+          vcmpps k4,xmm29,XMMWORD PTR [rdx+0x800],0x7b
+
+// CHECK:  vcmpps k4 , xmm29, xmmword ptr [rdx - 2048], 123
+// CHECK:  encoding: [0x62,0xf1,0x14,0x00,0xc2,0x62,0x80,0x7b]
+          vcmpps k4,xmm29,XMMWORD PTR [rdx-0x800],0x7b
+
+// CHECK:  vcmpps k4 , xmm29, xmmword ptr [rdx - 2064], 123
+// CHECK:  encoding: [0x62,0xf1,0x14,0x00,0xc2,0xa2,0xf0,0xf7,0xff,0xff,0x7b]
+          vcmpps k4,xmm29,XMMWORD PTR [rdx-0x810],0x7b
+
+// CHECK:  vcmpps k4 , xmm29, dword ptr [rdx + 508]{1to4}, 123
+// CHECK:  encoding: [0x62,0xf1,0x14,0x10,0xc2,0x62,0x7f,0x7b]
+          vcmpps k4,xmm29,DWORD PTR [rdx+0x1fc]{1to4},0x7b
+
+// CHECK:  vcmpps k4 , xmm29, dword ptr [rdx + 512]{1to4}, 123
+// CHECK:  encoding: [0x62,0xf1,0x14,0x10,0xc2,0xa2,0x00,0x02,0x00,0x00,0x7b]
+          vcmpps k4,xmm29,DWORD PTR [rdx+0x200]{1to4},0x7b
+
+// CHECK:  vcmpps k4 , xmm29, dword ptr [rdx - 512]{1to4}, 123
+// CHECK:  encoding: [0x62,0xf1,0x14,0x10,0xc2,0x62,0x80,0x7b]
+          vcmpps k4,xmm29,DWORD PTR [rdx-0x200]{1to4},0x7b
+
+// CHECK:  vcmpps k4 , xmm29, dword ptr [rdx - 516]{1to4}, 123
+// CHECK:  encoding: [0x62,0xf1,0x14,0x10,0xc2,0xa2,0xfc,0xfd,0xff,0xff,0x7b]
+          vcmpps k4,xmm29,DWORD PTR [rdx-0x204]{1to4},0x7b
+
+// CHECK:  vcmpps k4 , ymm19, ymm18, 171
+// CHECK:  encoding: [0x62,0xb1,0x64,0x20,0xc2,0xe2,0xab]
+          vcmpps k4,ymm19,ymm18,0xab
+
+// CHECK:  vcmpps k4 {k1}, ymm19, ymm18, 171
+// CHECK:  encoding: [0x62,0xb1,0x64,0x21,0xc2,0xe2,0xab]
+          vcmpps k4{k1},ymm19,ymm18,0xab
+
+// CHECK:  vcmpps k4 , ymm19, ymm18, 123
+// CHECK:  encoding: [0x62,0xb1,0x64,0x20,0xc2,0xe2,0x7b]
+          vcmpps k4,ymm19,ymm18,0x7b
+
+// CHECK:  vcmpps k4 , ymm19, ymmword ptr [rcx], 123
+// CHECK:  encoding: [0x62,0xf1,0x64,0x20,0xc2,0x21,0x7b]
+          vcmpps k4,ymm19,YMMWORD PTR [rcx],0x7b
+
+// CHECK:  vcmpps k4 , ymm19, ymmword ptr [rax + 8*r14 + 291], 123
+// CHECK:  encoding: [0x62,0xb1,0x64,0x20,0xc2,0xa4,0xf0,0x23,0x01,0x00,0x00,0x7b]
+          vcmpps k4,ymm19,YMMWORD PTR [rax+r14*8+0x123],0x7b
+
+// CHECK: vcmpps k4 , ymm19, dword ptr [rcx]{1to8}, 123
+// CHECK:  encoding: [0x62,0xf1,0x64,0x30,0xc2,0x21,0x7b]
+          vcmpps k4,ymm19,DWORD PTR [rcx]{1to8},0x7b
+
+// CHECK:  vcmpps k4 , ymm19, ymmword ptr [rdx + 4064], 123
+// CHECK:  encoding: [0x62,0xf1,0x64,0x20,0xc2,0x62,0x7f,0x7b]
+          vcmpps k4,ymm19,YMMWORD PTR [rdx+0xfe0],0x7b
+
+// CHECK:  vcmpps k4 , ymm19, ymmword ptr [rdx + 4096], 123
+// CHECK:  encoding: [0x62,0xf1,0x64,0x20,0xc2,0xa2,0x00,0x10,0x00,0x00,0x7b]
+          vcmpps k4,ymm19,YMMWORD PTR [rdx+0x1000],0x7b
+
+// CHECK:  vcmpps k4 , ymm19, ymmword ptr [rdx - 4096], 123
+// CHECK:  encoding: [0x62,0xf1,0x64,0x20,0xc2,0x62,0x80,0x7b]
+          vcmpps k4,ymm19,YMMWORD PTR [rdx-0x1000],0x7b
+
+// CHECK:  vcmpps k4 , ymm19, ymmword ptr [rdx - 4128], 123
+// CHECK:  encoding: [0x62,0xf1,0x64,0x20,0xc2,0xa2,0xe0,0xef,0xff,0xff,0x7b]
+          vcmpps k4,ymm19,YMMWORD PTR [rdx-0x1020],0x7b
+
+// CHECK:  vcmpps k4 , ymm19, dword ptr [rdx + 508]{1to8}, 123
+// CHECK:  encoding: [0x62,0xf1,0x64,0x30,0xc2,0x62,0x7f,0x7b]
+          vcmpps k4,ymm19,DWORD PTR [rdx+0x1fc]{1to8},0x7b
+
+// CHECK:  vcmpps k4 , ymm19, dword ptr [rdx + 512]{1to8}, 123
+// CHECK:  encoding: [0x62,0xf1,0x64,0x30,0xc2,0xa2,0x00,0x02,0x00,0x00,0x7b]
+          vcmpps k4,ymm19,DWORD PTR [rdx+0x200]{1to8},0x7b
+
+// CHECK:  vcmpps k4 , ymm19, dword ptr [rdx - 512]{1to8}, 123
+// CHECK:  encoding: [0x62,0xf1,0x64,0x30,0xc2,0x62,0x80,0x7b]
+          vcmpps k4,ymm19,DWORD PTR [rdx-0x200]{1to8},0x7b
+
+// CHECK:  vcmpps k4 , ymm19, dword ptr [rdx - 516]{1to8}, 123
+// CHECK:  encoding: [0x62,0xf1,0x64,0x30,0xc2,0xa2,0xfc,0xfd,0xff,0xff,0x7b]
+          vcmpps k4,ymm19,DWORD PTR [rdx-0x204]{1to8},0x7b
diff --git a/test/MC/X86/intel-syntax.s b/test/MC/X86/intel-syntax.s
index fce0c65781c..00295589199 100644
--- a/test/MC/X86/intel-syntax.s
+++ b/test/MC/X86/intel-syntax.s
@@ -635,10 +635,12 @@ add byte ptr [rax], 1
 // CHECK: addw $1, (%rax)
 // CHECK: addb $1, (%rax)
 
+fstp tbyte ptr [rax]
 fstp xword ptr [rax]
 fstp qword ptr [rax]
 fstp dword ptr [rax]
 // CHECK: fstpt (%rax)
+// CHECK: fstpt (%rax)
 // CHECK: fstpl (%rax)
 // CHECK: fstps (%rax)
 
@@ -662,3 +664,20 @@ frstor dword ptr [eax]
 // CHECK: fnsave (%eax)
 // CHECK: fxrstor (%eax)
 // CHECK: frstor (%eax)
+
+// CHECK: cmpnless %xmm1, %xmm0
+cmpnless xmm0, xmm1
+
+insb
+insw
+insd
+// CHECK: insb %dx, %es:(%rdi)
+// CHECK: insw %dx, %es:(%rdi)
+// CHECK: insl %dx, %es:(%rdi)
+
+outsb
+outsw
+outsd
+// CHECK: outsb (%rsi), %dx
+// CHECK: outsw (%rsi), %dx
+// CHECK: outsl (%rsi), %dx
diff --git a/test/MC/X86/mpx-encodings.s b/test/MC/X86/mpx-encodings.s
new file mode 100644
index 00000000000..6fe4e0f328e
--- /dev/null
+++ b/test/MC/X86/mpx-encodings.s
@@ -0,0 +1,38 @@
+// RUN: llvm-mc -triple x86_64-unknown-unknown -mcpu=knl --show-encoding %s | FileCheck %s
+
+// CHECK: bndmk (%rax), %bnd0
+// CHECK:  encoding: [0xf3,0x48,0x0f,0x1b,0x00]
+bndmk (%rax), %bnd0
+
+// CHECK: bndmk 1024(%rax), %bnd1
+// CHECK:  encoding: [0xf3,0x48,0x0f,0x1b,0x88,0x00,0x04,0x00,0x00]
+bndmk 1024(%rax), %bnd1
+
+// CHECK: bndmov  %bnd2, %bnd1
+// CHECK: encoding: [0x66,0x0f,0x1b,0xd1]
+bndmov %bnd2, %bnd1
+
+// CHECK: bndmov %bnd1, 1024(%r9)
+// CHECK: encoding: [0x66,0x49,0x0f,0x1b,0x89,0x00,0x04,0x00,0x00]
+bndmov %bnd1, 1024(%r9)
+
+// CHECK: bndstx %bnd1, 1024(%rax)
+// CHECK: encoding: [0x0f,0x1b,0x88,0x00,0x04,0x00,0x00]
+bndstx %bnd1, 1024(%rax)
+
+// CHECK: bndldx 1024(%r8), %bnd1
+// CHECK:  encoding: [0x41,0x0f,0x1a,0x88,0x00,0x04,0x00,0x00]
+bndldx 1024(%r8), %bnd1
+
+// CHECK: bndcl 121(%r10), %bnd1
+// CHECK:  encoding: [0xf3,0x49,0x0f,0x1a,0x4a,0x79]
+bndcl 121(%r10), %bnd1
+
+// CHECK: bndcn 121(%rcx), %bnd3
+// CHECK:  encoding: [0xf2,0x48,0x0f,0x1b,0x59,0x79]
+bndcn 121(%rcx), %bnd3
+
+// CHECK: bndcu %rdx, %bnd3
+// CHECK:  encoding: [0xf2,0x48,0x0f,0x1a,0xda]
+bndcu %rdx, %bnd3
+
diff --git a/test/MC/X86/nacl-test-call.s b/test/MC/X86/nacl-test-call.s
deleted file mode 100644
index 4f800f2cb90..00000000000
--- a/test/MC/X86/nacl-test-call.s
+++ /dev/null
@@ -1,55 +0,0 @@
-// RUN: llvm-mc -filetype asm -triple i386-unknown-nacl %s | FileCheck %s
-.scratch %ecx
-
-// Tests if the call instruction is expanded correctly.
-	
-	call foo
-foo:
-// CHECK:	calll foo
-	
-	call *%eax
-// CHECK: 	.bundle_lock align_to_end
-// CHECK-NEXT:  andl	$-32, %eax
-// CHECK-NEXT:  calll	*%eax
-// CHECK-NEXT:  .bundle_unlock
-
-	call *%ax
-// CHECK: 	.bundle_lock align_to_end
-// CHECK-NEXT:  andl	$-32, %eax
-// CHECK-NEXT:  calll	*%eax
-// CHECK-NEXT:  .bundle_unlock
-	
-	call *%ecx
-// CHECK: 	.bundle_lock align_to_end
-// CHECK-NEXT:  andl	$-32, %ecx
-// CHECK-NEXT:  calll	*%ecx
-// CHECK-NEXT:  .bundle_unlock
-
-	call *(%ecx)
-// CHECK:	movl (%ecx), %ecx
-// CHECK-NEXT: 	.bundle_lock align_to_end
-// CHECK-NEXT:  andl	$-32, %ecx
-// CHECK-NEXT:  calll	*%ecx
-// CHECK-NEXT:  .bundle_unlock
-
-	call *12(%ecx)
-// CHECK:	movl 12(%ecx), %ecx
-// CHECK-NEXT: 	.bundle_lock align_to_end
-// CHECK-NEXT:  andl	$-32, %ecx
-// CHECK-NEXT:  calll	*%ecx
-// CHECK-NEXT:  .bundle_unlock
-
-	call *-12(%ecx,%edx)
-// CHECK:	movl -12(%ecx,%edx), %ecx
-// CHECK-NEXT: 	.bundle_lock align_to_end
-// CHECK-NEXT:  andl	$-32, %ecx
-// CHECK-NEXT:  calll	*%ecx
-// CHECK-NEXT:  .bundle_unlock
-
-	call *-12(%ecx,%edx,4)
-// CHECK:	movl -12(%ecx,%edx,4), %ecx
-// CHECK-NEXT: 	.bundle_lock align_to_end
-// CHECK-NEXT:  andl	$-32, %ecx
-// CHECK-NEXT:  calll	*%ecx
-// CHECK-NEXT:  .bundle_unlock
-
diff --git a/test/MC/X86/nacl-test-call2.s b/test/MC/X86/nacl-test-call2.s
deleted file mode 100644
index e96ee18008a..00000000000
--- a/test/MC/X86/nacl-test-call2.s
+++ /dev/null
@@ -1,7 +0,0 @@
-// RUN: not llvm-mc -filetype asm -triple i386-unknown-nacl %s
-
-// Tests that the assembler fails if not given a scratch register
-
-	call *12(%ecx)
-
-
diff --git a/test/MC/X86/nacl-test-jmp.s b/test/MC/X86/nacl-test-jmp.s
deleted file mode 100644
index 8ffda6e9c84..00000000000
--- a/test/MC/X86/nacl-test-jmp.s
+++ /dev/null
@@ -1,55 +0,0 @@
-// RUN: llvm-mc -filetype asm -triple i386-unknown-nacl %s | FileCheck %s
-.scratch %ecx
-
-// Tests if the jmp instruction is expanded correctly
-	
-	jmp .L1
-// CHECK:	jmp .L1
-.L1:
-
-	jmp *%eax
-// CHECK: 	.bundle_lock
-// CHECK-NEXT:  andl	$-32, %eax
-// CHECK-NEXT:  jmpl	*%eax
-// CHECK-NEXT:  .bundle_unlock
-
-	jmp *%ax
-// CHECK: 	.bundle_lock
-// CHECK-NEXT:  andl	$-32, %eax
-// CHECK-NEXT:  jmpl	*%eax
-// CHECK-NEXT:  .bundle_unlock
-	
-	jmp *%ecx
-// CHECK: 	.bundle_lock
-// CHECK-NEXT:  andl	$-32, %ecx
-// CHECK-NEXT:  jmpl	*%ecx
-// CHECK-NEXT:  .bundle_unlock
-
-	jmp *(%ecx)
-// CHECK:	movl (%ecx), %ecx
-// CHECK-NEXT: 	.bundle_lock
-// CHECK-NEXT:  andl	$-32, %ecx
-// CHECK-NEXT:  jmpl	*%ecx
-// CHECK-NEXT:  .bundle_unlock
-
-	jmp *12(%ecx)
-// CHECK:	movl 12(%ecx), %ecx
-// CHECK-NEXT: 	.bundle_lock
-// CHECK-NEXT:  andl	$-32, %ecx
-// CHECK-NEXT:  jmpl	*%ecx
-// CHECK-NEXT:  .bundle_unlock
-
-	jmp *-12(%ecx,%edx)
-// CHECK:	movl -12(%ecx,%edx), %ecx
-// CHECK-NEXT: 	.bundle_lock
-// CHECK-NEXT:  andl	$-32, %ecx
-// CHECK-NEXT:  jmpl	*%ecx
-// CHECK-NEXT:  .bundle_unlock
-
-	jmp *-12(%ecx,%edx,4)
-// CHECK:	movl -12(%ecx,%edx,4), %ecx
-// CHECK-NEXT: 	.bundle_lock
-// CHECK-NEXT:  andl	$-32, %ecx
-// CHECK-NEXT:  jmpl	*%ecx
-// CHECK-NEXT:  .bundle_unlock
-
diff --git a/test/MC/X86/nacl-test-jmp2.s b/test/MC/X86/nacl-test-jmp2.s
deleted file mode 100644
index 923b07a9dc3..00000000000
--- a/test/MC/X86/nacl-test-jmp2.s
+++ /dev/null
@@ -1,6 +0,0 @@
-// RUN: not llvm-mc -filetype asm -triple i386-unknown-nacl %s
-
-// Tests that the assembler fails if not given a scratch register
-
-	jmp *12(%ecx)
-
diff --git a/test/MC/X86/nacl-test-ret.s b/test/MC/X86/nacl-test-ret.s
deleted file mode 100644
index 49603cbf01c..00000000000
--- a/test/MC/X86/nacl-test-ret.s
+++ /dev/null
@@ -1,20 +0,0 @@
-// RUN: llvm-mc -filetype asm -triple i386-unknown-nacl %s | FileCheck %s
-
-// Tests if the ret instruction is expanded correctly.
-
-.scratch %ecx
-	ret
-// CHECK:  	popl	%ecx
-// CHECK-NEXT: 	.bundle_lock
-// CHECK-NEXT:  andl	$-32, %ecx
-// CHECK-NEXT:  jmpl	*%ecx
-// CHECK-NEXT:  .bundle_unlock
-	
-	ret $12
-// CHECK:  	popl 	%ecx
-// CHECK-NEXT:  addl	$12, %esp
-// CHECK-NEXT:  .bundle_lock
-// CHECK-NEXT:  andl	$-32, %ecx
-// CHECK-NEXT:  jmpl	*%ecx
-// CHECK-NEXT:  .bundle_unlock
-
diff --git a/test/MC/X86/nacl-test-ret2.s b/test/MC/X86/nacl-test-ret2.s
deleted file mode 100644
index 2d67eaf3786..00000000000
--- a/test/MC/X86/nacl-test-ret2.s
+++ /dev/null
@@ -1,5 +0,0 @@
-// RUN: not llvm-mc -filetype asm -triple i386-unknown-nacl %s
-
-// Tests that the assembler fails if not given a scratch register
-	
-	ret
diff --git a/test/MC/X86/nacl-test-scratch.s b/test/MC/X86/nacl-test-scratch.s
deleted file mode 100644
index 57a5b40d1b8..00000000000
--- a/test/MC/X86/nacl-test-scratch.s
+++ /dev/null
@@ -1,7 +0,0 @@
-// RUN: not llvm-mc -filetype asm -triple i386-unknown-nacl %s 2>&1 | FileCheck %s
-
-// Tests that a bare .unscratch fails, since there are no scratch
-// registers specified
-
-// CHECK: No scratch registers specified
-.unscratch
diff --git a/test/MC/X86/nacl-test-scratch2.s b/test/MC/X86/nacl-test-scratch2.s
deleted file mode 100644
index ccf100c6123..00000000000
--- a/test/MC/X86/nacl-test-scratch2.s
+++ /dev/null
@@ -1,7 +0,0 @@
-// RUN: not llvm-mc -filetype asm -triple i386-unknown-nacl %s 2>&1 | FileCheck %s
-
-// Tests that the assembler fails if the argument is not a valid
-// register
-
-// CHECK: invalid register name
-.scratch %notaregister
diff --git a/test/MC/X86/nacl-test-scratch3.s b/test/MC/X86/nacl-test-scratch3.s
deleted file mode 100644
index c526f5e4bc1..00000000000
--- a/test/MC/X86/nacl-test-scratch3.s
+++ /dev/null
@@ -1,6 +0,0 @@
-// RUN: llvm-mc -filetype asm -triple i386-unknown-nacl %s
-
-// Tests a basic .scratch and .unscratch sequence, should succeed
-
-.scratch %ecx
-.unscratch
diff --git a/test/MC/X86/nacl-test-scratch4.s b/test/MC/X86/nacl-test-scratch4.s
deleted file mode 100644
index 4920d20f49c..00000000000
--- a/test/MC/X86/nacl-test-scratch4.s
+++ /dev/null
@@ -1,11 +0,0 @@
-// RUN: llvm-mc -filetype asm -triple i386-unknown-nacl %s
-
-// Tests a more complicated sequence of .scratch/.unscratch
-// should succeed
-
-.scratch %ecx
-.unscratch
-.scratch %ecx
-.scratch %edx
-.unscratch
-.unscratch
diff --git a/test/MC/X86/nacl-test-scratch5.s b/test/MC/X86/nacl-test-scratch5.s
deleted file mode 100644
index 8041bff166a..00000000000
--- a/test/MC/X86/nacl-test-scratch5.s
+++ /dev/null
@@ -1,8 +0,0 @@
-// RUN: not llvm-mc -filetype asm -triple i386-unknown-nacl %s 2>&1 | FileCheck %s
-
-// Extraneous .unscratch directive. The assembler should fail.
-
-.scratch %ecx
-.unscratch
-// CHECK: No scratch registers specified
-.unscratch
diff --git a/test/MC/X86/nacl-test-scratch6.s b/test/MC/X86/nacl-test-scratch6.s
deleted file mode 100644
index a738c47349e..00000000000
--- a/test/MC/X86/nacl-test-scratch6.s
+++ /dev/null
@@ -1,7 +0,0 @@
-// RUN: not llvm-mc -filetype asm -triple i386-unknown-linux %s 2>&1 | FileCheck %s
-
-// Tests if .scratch directive is seen if the target is not NaCl.
-// Should fail.
-
-// CHECK: unknown directive
-.scratch %ecx
diff --git a/test/MC/X86/nacl-test-scratch7.s b/test/MC/X86/nacl-test-scratch7.s
deleted file mode 100644
index 89c657ac070..00000000000
--- a/test/MC/X86/nacl-test-scratch7.s
+++ /dev/null
@@ -1,7 +0,0 @@
-// RUN: not llvm-mc -filetype asm -triple i386-unknown-linux %s 2>&1 | FileCheck %s
-
-// Tests if .unscratch directive is seen if the target is not NaCl.
-// Should fail.
-
-// CHECK: unknown directive
-.unscratch
diff --git a/test/MC/X86/x86-64-avx512bw.s b/test/MC/X86/x86-64-avx512bw.s
index 0d055b1cecd..6d5087b3d89 100644
--- a/test/MC/X86/x86-64-avx512bw.s
+++ b/test/MC/X86/x86-64-avx512bw.s
@@ -511,6 +511,78 @@
 // CHECK:  encoding: [0x62,0xe2,0x15,0x40,0x3a,0x9a,0xc0,0xdf,0xff,0xff]
           vpminuw -8256(%rdx), %zmm29, %zmm19
 
+// CHECK: vpmovsxbw %ymm18, %zmm22
+// CHECK:  encoding: [0x62,0xa2,0x7d,0x48,0x20,0xf2]
+          vpmovsxbw %ymm18, %zmm22
+
+// CHECK: vpmovsxbw %ymm18, %zmm22 {%k5}
+// CHECK:  encoding: [0x62,0xa2,0x7d,0x4d,0x20,0xf2]
+          vpmovsxbw %ymm18, %zmm22 {%k5}
+
+// CHECK: vpmovsxbw %ymm18, %zmm22 {%k5} {z}
+// CHECK:  encoding: [0x62,0xa2,0x7d,0xcd,0x20,0xf2]
+          vpmovsxbw %ymm18, %zmm22 {%k5} {z}
+
+// CHECK: vpmovsxbw (%rcx), %zmm22
+// CHECK:  encoding: [0x62,0xe2,0x7d,0x48,0x20,0x31]
+          vpmovsxbw (%rcx), %zmm22
+
+// CHECK: vpmovsxbw 291(%rax,%r14,8), %zmm22
+// CHECK:  encoding: [0x62,0xa2,0x7d,0x48,0x20,0xb4,0xf0,0x23,0x01,0x00,0x00]
+          vpmovsxbw 291(%rax,%r14,8), %zmm22
+
+// CHECK: vpmovsxbw 4064(%rdx), %zmm22
+// CHECK:  encoding: [0x62,0xe2,0x7d,0x48,0x20,0x72,0x7f]
+          vpmovsxbw 4064(%rdx), %zmm22
+
+// CHECK: vpmovsxbw 4096(%rdx), %zmm22
+// CHECK:  encoding: [0x62,0xe2,0x7d,0x48,0x20,0xb2,0x00,0x10,0x00,0x00]
+          vpmovsxbw 4096(%rdx), %zmm22
+
+// CHECK: vpmovsxbw -4096(%rdx), %zmm22
+// CHECK:  encoding: [0x62,0xe2,0x7d,0x48,0x20,0x72,0x80]
+          vpmovsxbw -4096(%rdx), %zmm22
+
+// CHECK: vpmovsxbw -4128(%rdx), %zmm22
+// CHECK:  encoding: [0x62,0xe2,0x7d,0x48,0x20,0xb2,0xe0,0xef,0xff,0xff]
+          vpmovsxbw -4128(%rdx), %zmm22
+
+// CHECK: vpmovzxbw %ymm26, %zmm24
+// CHECK:  encoding: [0x62,0x02,0x7d,0x48,0x30,0xc2]
+          vpmovzxbw %ymm26, %zmm24
+
+// CHECK: vpmovzxbw %ymm26, %zmm24 {%k4}
+// CHECK:  encoding: [0x62,0x02,0x7d,0x4c,0x30,0xc2]
+          vpmovzxbw %ymm26, %zmm24 {%k4}
+
+// CHECK: vpmovzxbw %ymm26, %zmm24 {%k4} {z}
+// CHECK:  encoding: [0x62,0x02,0x7d,0xcc,0x30,0xc2]
+          vpmovzxbw %ymm26, %zmm24 {%k4} {z}
+
+// CHECK: vpmovzxbw (%rcx), %zmm24
+// CHECK:  encoding: [0x62,0x62,0x7d,0x48,0x30,0x01]
+          vpmovzxbw (%rcx), %zmm24
+
+// CHECK: vpmovzxbw 291(%rax,%r14,8), %zmm24
+// CHECK:  encoding: [0x62,0x22,0x7d,0x48,0x30,0x84,0xf0,0x23,0x01,0x00,0x00]
+          vpmovzxbw 291(%rax,%r14,8), %zmm24
+
+// CHECK: vpmovzxbw 4064(%rdx), %zmm24
+// CHECK:  encoding: [0x62,0x62,0x7d,0x48,0x30,0x42,0x7f]
+          vpmovzxbw 4064(%rdx), %zmm24
+
+// CHECK: vpmovzxbw 4096(%rdx), %zmm24
+// CHECK:  encoding: [0x62,0x62,0x7d,0x48,0x30,0x82,0x00,0x10,0x00,0x00]
+          vpmovzxbw 4096(%rdx), %zmm24
+
+// CHECK: vpmovzxbw -4096(%rdx), %zmm24
+// CHECK:  encoding: [0x62,0x62,0x7d,0x48,0x30,0x42,0x80]
+          vpmovzxbw -4096(%rdx), %zmm24
+
+// CHECK: vpmovzxbw -4128(%rdx), %zmm24
+// CHECK:  encoding: [0x62,0x62,0x7d,0x48,0x30,0x82,0xe0,0xef,0xff,0xff]
+          vpmovzxbw -4128(%rdx), %zmm24
+
 // CHECK: vpmullw %zmm19, %zmm28, %zmm19
 // CHECK:  encoding: [0x62,0xa1,0x1d,0x40,0xd5,0xdb]
           vpmullw %zmm19, %zmm28, %zmm19
@@ -2008,3 +2080,2035 @@
 // CHECK:  encoding: [0x62,0xe2,0x95,0x40,0x11,0x9a,0xc0,0xdf,0xff,0xff]
           vpsravw -8256(%rdx), %zmm29, %zmm19
 
+// CHECK: vpacksswb %zmm18, %zmm29, %zmm21
+// CHECK:  encoding: [0x62,0xa1,0x95,0x40,0x63,0xea]
+          vpacksswb %zmm18, %zmm29, %zmm21
+
+// CHECK: vpacksswb %zmm18, %zmm29, %zmm21 {%k1}
+// CHECK:  encoding: [0x62,0xa1,0x95,0x41,0x63,0xea]
+          vpacksswb %zmm18, %zmm29, %zmm21 {%k1}
+
+// CHECK: vpacksswb %zmm18, %zmm29, %zmm21 {%k1} {z}
+// CHECK:  encoding: [0x62,0xa1,0x95,0xc1,0x63,0xea]
+          vpacksswb %zmm18, %zmm29, %zmm21 {%k1} {z}
+
+// CHECK: vpacksswb (%rcx), %zmm29, %zmm21
+// CHECK:  encoding: [0x62,0xe1,0x95,0x40,0x63,0x29]
+          vpacksswb (%rcx), %zmm29, %zmm21
+
+// CHECK: vpacksswb 291(%rax,%r14,8), %zmm29, %zmm21
+// CHECK:  encoding: [0x62,0xa1,0x95,0x40,0x63,0xac,0xf0,0x23,0x01,0x00,0x00]
+          vpacksswb 291(%rax,%r14,8), %zmm29, %zmm21
+
+// CHECK: vpacksswb 8128(%rdx), %zmm29, %zmm21
+// CHECK:  encoding: [0x62,0xe1,0x95,0x40,0x63,0x6a,0x7f]
+          vpacksswb 8128(%rdx), %zmm29, %zmm21
+
+// CHECK: vpacksswb 8192(%rdx), %zmm29, %zmm21
+// CHECK:  encoding: [0x62,0xe1,0x95,0x40,0x63,0xaa,0x00,0x20,0x00,0x00]
+          vpacksswb 8192(%rdx), %zmm29, %zmm21
+
+// CHECK: vpacksswb -8192(%rdx), %zmm29, %zmm21
+// CHECK:  encoding: [0x62,0xe1,0x95,0x40,0x63,0x6a,0x80]
+          vpacksswb -8192(%rdx), %zmm29, %zmm21
+
+// CHECK: vpacksswb -8256(%rdx), %zmm29, %zmm21
+// CHECK:  encoding: [0x62,0xe1,0x95,0x40,0x63,0xaa,0xc0,0xdf,0xff,0xff]
+          vpacksswb -8256(%rdx), %zmm29, %zmm21
+
+// CHECK: vpackuswb %zmm28, %zmm24, %zmm28
+// CHECK:  encoding: [0x62,0x01,0xbd,0x40,0x67,0xe4]
+          vpackuswb %zmm28, %zmm24, %zmm28
+
+// CHECK: vpackuswb %zmm28, %zmm24, %zmm28 {%k6}
+// CHECK:  encoding: [0x62,0x01,0xbd,0x46,0x67,0xe4]
+          vpackuswb %zmm28, %zmm24, %zmm28 {%k6}
+
+// CHECK: vpackuswb %zmm28, %zmm24, %zmm28 {%k6} {z}
+// CHECK:  encoding: [0x62,0x01,0xbd,0xc6,0x67,0xe4]
+          vpackuswb %zmm28, %zmm24, %zmm28 {%k6} {z}
+
+// CHECK: vpackuswb (%rcx), %zmm24, %zmm28
+// CHECK:  encoding: [0x62,0x61,0xbd,0x40,0x67,0x21]
+          vpackuswb (%rcx), %zmm24, %zmm28
+
+// CHECK: vpackuswb 291(%rax,%r14,8), %zmm24, %zmm28
+// CHECK:  encoding: [0x62,0x21,0xbd,0x40,0x67,0xa4,0xf0,0x23,0x01,0x00,0x00]
+          vpackuswb 291(%rax,%r14,8), %zmm24, %zmm28
+
+// CHECK: vpackuswb 8128(%rdx), %zmm24, %zmm28
+// CHECK:  encoding: [0x62,0x61,0xbd,0x40,0x67,0x62,0x7f]
+          vpackuswb 8128(%rdx), %zmm24, %zmm28
+
+// CHECK: vpackuswb 8192(%rdx), %zmm24, %zmm28
+// CHECK:  encoding: [0x62,0x61,0xbd,0x40,0x67,0xa2,0x00,0x20,0x00,0x00]
+          vpackuswb 8192(%rdx), %zmm24, %zmm28
+
+// CHECK: vpackuswb -8192(%rdx), %zmm24, %zmm28
+// CHECK:  encoding: [0x62,0x61,0xbd,0x40,0x67,0x62,0x80]
+          vpackuswb -8192(%rdx), %zmm24, %zmm28
+
+// CHECK: vpackuswb -8256(%rdx), %zmm24, %zmm28
+// CHECK:  encoding: [0x62,0x61,0xbd,0x40,0x67,0xa2,0xc0,0xdf,0xff,0xff]
+          vpackuswb -8256(%rdx), %zmm24, %zmm28
+
+// CHECK: vpacksswb %zmm24, %zmm29, %zmm26
+// CHECK:  encoding: [0x62,0x01,0x95,0x40,0x63,0xd0]
+          vpacksswb %zmm24, %zmm29, %zmm26
+
+// CHECK: vpacksswb %zmm24, %zmm29, %zmm26 {%k6}
+// CHECK:  encoding: [0x62,0x01,0x95,0x46,0x63,0xd0]
+          vpacksswb %zmm24, %zmm29, %zmm26 {%k6}
+
+// CHECK: vpacksswb %zmm24, %zmm29, %zmm26 {%k6} {z}
+// CHECK:  encoding: [0x62,0x01,0x95,0xc6,0x63,0xd0]
+          vpacksswb %zmm24, %zmm29, %zmm26 {%k6} {z}
+
+// CHECK: vpacksswb (%rcx), %zmm29, %zmm26
+// CHECK:  encoding: [0x62,0x61,0x95,0x40,0x63,0x11]
+          vpacksswb (%rcx), %zmm29, %zmm26
+
+// CHECK: vpacksswb 4660(%rax,%r14,8), %zmm29, %zmm26
+// CHECK:  encoding: [0x62,0x21,0x95,0x40,0x63,0x94,0xf0,0x34,0x12,0x00,0x00]
+          vpacksswb 4660(%rax,%r14,8), %zmm29, %zmm26
+
+// CHECK: vpacksswb 8128(%rdx), %zmm29, %zmm26
+// CHECK:  encoding: [0x62,0x61,0x95,0x40,0x63,0x52,0x7f]
+          vpacksswb 8128(%rdx), %zmm29, %zmm26
+
+// CHECK: vpacksswb 8192(%rdx), %zmm29, %zmm26
+// CHECK:  encoding: [0x62,0x61,0x95,0x40,0x63,0x92,0x00,0x20,0x00,0x00]
+          vpacksswb 8192(%rdx), %zmm29, %zmm26
+
+// CHECK: vpacksswb -8192(%rdx), %zmm29, %zmm26
+// CHECK:  encoding: [0x62,0x61,0x95,0x40,0x63,0x52,0x80]
+          vpacksswb -8192(%rdx), %zmm29, %zmm26
+
+// CHECK: vpacksswb -8256(%rdx), %zmm29, %zmm26
+// CHECK:  encoding: [0x62,0x61,0x95,0x40,0x63,0x92,0xc0,0xdf,0xff,0xff]
+          vpacksswb -8256(%rdx), %zmm29, %zmm26
+
+// CHECK: vpackuswb %zmm23, %zmm28, %zmm18
+// CHECK:  encoding: [0x62,0xa1,0x9d,0x40,0x67,0xd7]
+          vpackuswb %zmm23, %zmm28, %zmm18
+
+// CHECK: vpackuswb %zmm23, %zmm28, %zmm18 {%k4}
+// CHECK:  encoding: [0x62,0xa1,0x9d,0x44,0x67,0xd7]
+          vpackuswb %zmm23, %zmm28, %zmm18 {%k4}
+
+// CHECK: vpackuswb %zmm23, %zmm28, %zmm18 {%k4} {z}
+// CHECK:  encoding: [0x62,0xa1,0x9d,0xc4,0x67,0xd7]
+          vpackuswb %zmm23, %zmm28, %zmm18 {%k4} {z}
+
+// CHECK: vpackuswb (%rcx), %zmm28, %zmm18
+// CHECK:  encoding: [0x62,0xe1,0x9d,0x40,0x67,0x11]
+          vpackuswb (%rcx), %zmm28, %zmm18
+
+// CHECK: vpackuswb 4660(%rax,%r14,8), %zmm28, %zmm18
+// CHECK:  encoding: [0x62,0xa1,0x9d,0x40,0x67,0x94,0xf0,0x34,0x12,0x00,0x00]
+          vpackuswb 4660(%rax,%r14,8), %zmm28, %zmm18
+
+// CHECK: vpackuswb 8128(%rdx), %zmm28, %zmm18
+// CHECK:  encoding: [0x62,0xe1,0x9d,0x40,0x67,0x52,0x7f]
+          vpackuswb 8128(%rdx), %zmm28, %zmm18
+
+// CHECK: vpackuswb 8192(%rdx), %zmm28, %zmm18
+// CHECK:  encoding: [0x62,0xe1,0x9d,0x40,0x67,0x92,0x00,0x20,0x00,0x00]
+          vpackuswb 8192(%rdx), %zmm28, %zmm18
+
+// CHECK: vpackuswb -8192(%rdx), %zmm28, %zmm18
+// CHECK:  encoding: [0x62,0xe1,0x9d,0x40,0x67,0x52,0x80]
+          vpackuswb -8192(%rdx), %zmm28, %zmm18
+
+// CHECK: vpackuswb -8256(%rdx), %zmm28, %zmm18
+// CHECK:  encoding: [0x62,0xe1,0x9d,0x40,0x67,0x92,0xc0,0xdf,0xff,0xff]
+          vpackuswb -8256(%rdx), %zmm28, %zmm18
+
+// CHECK: vpackssdw %zmm22, %zmm27, %zmm24
+// CHECK:  encoding: [0x62,0x21,0x25,0x40,0x6b,0xc6]
+          vpackssdw %zmm22, %zmm27, %zmm24
+
+// CHECK: vpackssdw %zmm22, %zmm27, %zmm24 {%k2}
+// CHECK:  encoding: [0x62,0x21,0x25,0x42,0x6b,0xc6]
+          vpackssdw %zmm22, %zmm27, %zmm24 {%k2}
+
+// CHECK: vpackssdw %zmm22, %zmm27, %zmm24 {%k2} {z}
+// CHECK:  encoding: [0x62,0x21,0x25,0xc2,0x6b,0xc6]
+          vpackssdw %zmm22, %zmm27, %zmm24 {%k2} {z}
+
+// CHECK: vpackssdw (%rcx), %zmm27, %zmm24
+// CHECK:  encoding: [0x62,0x61,0x25,0x40,0x6b,0x01]
+          vpackssdw (%rcx), %zmm27, %zmm24
+
+// CHECK: vpackssdw 291(%rax,%r14,8), %zmm27, %zmm24
+// CHECK:  encoding: [0x62,0x21,0x25,0x40,0x6b,0x84,0xf0,0x23,0x01,0x00,0x00]
+          vpackssdw 291(%rax,%r14,8), %zmm27, %zmm24
+
+// CHECK: vpackssdw (%rcx){1to16}, %zmm27, %zmm24
+// CHECK:  encoding: [0x62,0x61,0x25,0x50,0x6b,0x01]
+          vpackssdw (%rcx){1to16}, %zmm27, %zmm24
+
+// CHECK: vpackssdw 8128(%rdx), %zmm27, %zmm24
+// CHECK:  encoding: [0x62,0x61,0x25,0x40,0x6b,0x42,0x7f]
+          vpackssdw 8128(%rdx), %zmm27, %zmm24
+
+// CHECK: vpackssdw 8192(%rdx), %zmm27, %zmm24
+// CHECK:  encoding: [0x62,0x61,0x25,0x40,0x6b,0x82,0x00,0x20,0x00,0x00]
+          vpackssdw 8192(%rdx), %zmm27, %zmm24
+
+// CHECK: vpackssdw -8192(%rdx), %zmm27, %zmm24
+// CHECK:  encoding: [0x62,0x61,0x25,0x40,0x6b,0x42,0x80]
+          vpackssdw -8192(%rdx), %zmm27, %zmm24
+
+// CHECK: vpackssdw -8256(%rdx), %zmm27, %zmm24
+// CHECK:  encoding: [0x62,0x61,0x25,0x40,0x6b,0x82,0xc0,0xdf,0xff,0xff]
+          vpackssdw -8256(%rdx), %zmm27, %zmm24
+
+// CHECK: vpackssdw 508(%rdx){1to16}, %zmm27, %zmm24
+// CHECK:  encoding: [0x62,0x61,0x25,0x50,0x6b,0x42,0x7f]
+          vpackssdw 508(%rdx){1to16}, %zmm27, %zmm24
+
+// CHECK: vpackssdw 512(%rdx){1to16}, %zmm27, %zmm24
+// CHECK:  encoding: [0x62,0x61,0x25,0x50,0x6b,0x82,0x00,0x02,0x00,0x00]
+          vpackssdw 512(%rdx){1to16}, %zmm27, %zmm24
+
+// CHECK: vpackssdw -512(%rdx){1to16}, %zmm27, %zmm24
+// CHECK:  encoding: [0x62,0x61,0x25,0x50,0x6b,0x42,0x80]
+          vpackssdw -512(%rdx){1to16}, %zmm27, %zmm24
+
+// CHECK: vpackssdw -516(%rdx){1to16}, %zmm27, %zmm24
+// CHECK:  encoding: [0x62,0x61,0x25,0x50,0x6b,0x82,0xfc,0xfd,0xff,0xff]
+          vpackssdw -516(%rdx){1to16}, %zmm27, %zmm24
+
+// CHECK: vpacksswb %zmm23, %zmm18, %zmm28
+// CHECK:  encoding: [0x62,0x21,0xed,0x40,0x63,0xe7]
+          vpacksswb %zmm23, %zmm18, %zmm28
+
+// CHECK: vpacksswb %zmm23, %zmm18, %zmm28 {%k3}
+// CHECK:  encoding: [0x62,0x21,0xed,0x43,0x63,0xe7]
+          vpacksswb %zmm23, %zmm18, %zmm28 {%k3}
+
+// CHECK: vpacksswb %zmm23, %zmm18, %zmm28 {%k3} {z}
+// CHECK:  encoding: [0x62,0x21,0xed,0xc3,0x63,0xe7]
+          vpacksswb %zmm23, %zmm18, %zmm28 {%k3} {z}
+
+// CHECK: vpacksswb (%rcx), %zmm18, %zmm28
+// CHECK:  encoding: [0x62,0x61,0xed,0x40,0x63,0x21]
+          vpacksswb (%rcx), %zmm18, %zmm28
+
+// CHECK: vpacksswb 291(%rax,%r14,8), %zmm18, %zmm28
+// CHECK:  encoding: [0x62,0x21,0xed,0x40,0x63,0xa4,0xf0,0x23,0x01,0x00,0x00]
+          vpacksswb 291(%rax,%r14,8), %zmm18, %zmm28
+
+// CHECK: vpacksswb 8128(%rdx), %zmm18, %zmm28
+// CHECK:  encoding: [0x62,0x61,0xed,0x40,0x63,0x62,0x7f]
+          vpacksswb 8128(%rdx), %zmm18, %zmm28
+
+// CHECK: vpacksswb 8192(%rdx), %zmm18, %zmm28
+// CHECK:  encoding: [0x62,0x61,0xed,0x40,0x63,0xa2,0x00,0x20,0x00,0x00]
+          vpacksswb 8192(%rdx), %zmm18, %zmm28
+
+// CHECK: vpacksswb -8192(%rdx), %zmm18, %zmm28
+// CHECK:  encoding: [0x62,0x61,0xed,0x40,0x63,0x62,0x80]
+          vpacksswb -8192(%rdx), %zmm18, %zmm28
+
+// CHECK: vpacksswb -8256(%rdx), %zmm18, %zmm28
+// CHECK:  encoding: [0x62,0x61,0xed,0x40,0x63,0xa2,0xc0,0xdf,0xff,0xff]
+          vpacksswb -8256(%rdx), %zmm18, %zmm28
+
+// CHECK: vpackusdw %zmm21, %zmm29, %zmm18
+// CHECK:  encoding: [0x62,0xa2,0x15,0x40,0x2b,0xd5]
+          vpackusdw %zmm21, %zmm29, %zmm18
+
+// CHECK: vpackusdw %zmm21, %zmm29, %zmm18 {%k2}
+// CHECK:  encoding: [0x62,0xa2,0x15,0x42,0x2b,0xd5]
+          vpackusdw %zmm21, %zmm29, %zmm18 {%k2}
+
+// CHECK: vpackusdw %zmm21, %zmm29, %zmm18 {%k2} {z}
+// CHECK:  encoding: [0x62,0xa2,0x15,0xc2,0x2b,0xd5]
+          vpackusdw %zmm21, %zmm29, %zmm18 {%k2} {z}
+
+// CHECK: vpackusdw (%rcx), %zmm29, %zmm18
+// CHECK:  encoding: [0x62,0xe2,0x15,0x40,0x2b,0x11]
+          vpackusdw (%rcx), %zmm29, %zmm18
+
+// CHECK: vpackusdw 291(%rax,%r14,8), %zmm29, %zmm18
+// CHECK:  encoding: [0x62,0xa2,0x15,0x40,0x2b,0x94,0xf0,0x23,0x01,0x00,0x00]
+          vpackusdw 291(%rax,%r14,8), %zmm29, %zmm18
+
+// CHECK: vpackusdw (%rcx){1to16}, %zmm29, %zmm18
+// CHECK:  encoding: [0x62,0xe2,0x15,0x50,0x2b,0x11]
+          vpackusdw (%rcx){1to16}, %zmm29, %zmm18
+
+// CHECK: vpackusdw 8128(%rdx), %zmm29, %zmm18
+// CHECK:  encoding: [0x62,0xe2,0x15,0x40,0x2b,0x52,0x7f]
+          vpackusdw 8128(%rdx), %zmm29, %zmm18
+
+// CHECK: vpackusdw 8192(%rdx), %zmm29, %zmm18
+// CHECK:  encoding: [0x62,0xe2,0x15,0x40,0x2b,0x92,0x00,0x20,0x00,0x00]
+          vpackusdw 8192(%rdx), %zmm29, %zmm18
+
+// CHECK: vpackusdw -8192(%rdx), %zmm29, %zmm18
+// CHECK:  encoding: [0x62,0xe2,0x15,0x40,0x2b,0x52,0x80]
+          vpackusdw -8192(%rdx), %zmm29, %zmm18
+
+// CHECK: vpackusdw -8256(%rdx), %zmm29, %zmm18
+// CHECK:  encoding: [0x62,0xe2,0x15,0x40,0x2b,0x92,0xc0,0xdf,0xff,0xff]
+          vpackusdw -8256(%rdx), %zmm29, %zmm18
+
+// CHECK: vpackusdw 508(%rdx){1to16}, %zmm29, %zmm18
+// CHECK:  encoding: [0x62,0xe2,0x15,0x50,0x2b,0x52,0x7f]
+          vpackusdw 508(%rdx){1to16}, %zmm29, %zmm18
+
+// CHECK: vpackusdw 512(%rdx){1to16}, %zmm29, %zmm18
+// CHECK:  encoding: [0x62,0xe2,0x15,0x50,0x2b,0x92,0x00,0x02,0x00,0x00]
+          vpackusdw 512(%rdx){1to16}, %zmm29, %zmm18
+
+// CHECK: vpackusdw -512(%rdx){1to16}, %zmm29, %zmm18
+// CHECK:  encoding: [0x62,0xe2,0x15,0x50,0x2b,0x52,0x80]
+          vpackusdw -512(%rdx){1to16}, %zmm29, %zmm18
+
+// CHECK: vpackusdw -516(%rdx){1to16}, %zmm29, %zmm18
+// CHECK:  encoding: [0x62,0xe2,0x15,0x50,0x2b,0x92,0xfc,0xfd,0xff,0xff]
+          vpackusdw -516(%rdx){1to16}, %zmm29, %zmm18
+
+// CHECK: vpackuswb %zmm27, %zmm18, %zmm23
+// CHECK:  encoding: [0x62,0x81,0xed,0x40,0x67,0xfb]
+          vpackuswb %zmm27, %zmm18, %zmm23
+
+// CHECK: vpackuswb %zmm27, %zmm18, %zmm23 {%k5}
+// CHECK:  encoding: [0x62,0x81,0xed,0x45,0x67,0xfb]
+          vpackuswb %zmm27, %zmm18, %zmm23 {%k5}
+
+// CHECK: vpackuswb %zmm27, %zmm18, %zmm23 {%k5} {z}
+// CHECK:  encoding: [0x62,0x81,0xed,0xc5,0x67,0xfb]
+          vpackuswb %zmm27, %zmm18, %zmm23 {%k5} {z}
+
+// CHECK: vpackuswb (%rcx), %zmm18, %zmm23
+// CHECK:  encoding: [0x62,0xe1,0xed,0x40,0x67,0x39]
+          vpackuswb (%rcx), %zmm18, %zmm23
+
+// CHECK: vpackuswb 291(%rax,%r14,8), %zmm18, %zmm23
+// CHECK:  encoding: [0x62,0xa1,0xed,0x40,0x67,0xbc,0xf0,0x23,0x01,0x00,0x00]
+          vpackuswb 291(%rax,%r14,8), %zmm18, %zmm23
+
+// CHECK: vpackuswb 8128(%rdx), %zmm18, %zmm23
+// CHECK:  encoding: [0x62,0xe1,0xed,0x40,0x67,0x7a,0x7f]
+          vpackuswb 8128(%rdx), %zmm18, %zmm23
+
+// CHECK: vpackuswb 8192(%rdx), %zmm18, %zmm23
+// CHECK:  encoding: [0x62,0xe1,0xed,0x40,0x67,0xba,0x00,0x20,0x00,0x00]
+          vpackuswb 8192(%rdx), %zmm18, %zmm23
+
+// CHECK: vpackuswb -8192(%rdx), %zmm18, %zmm23
+// CHECK:  encoding: [0x62,0xe1,0xed,0x40,0x67,0x7a,0x80]
+          vpackuswb -8192(%rdx), %zmm18, %zmm23
+
+// CHECK: vpackuswb -8256(%rdx), %zmm18, %zmm23
+// CHECK:  encoding: [0x62,0xe1,0xed,0x40,0x67,0xba,0xc0,0xdf,0xff,0xff]
+          vpackuswb -8256(%rdx), %zmm18, %zmm23
+
+// CHECK: vpaddsb %zmm20, %zmm19, %zmm19
+// CHECK:  encoding: [0x62,0xa1,0x65,0x40,0xec,0xdc]
+          vpaddsb %zmm20, %zmm19, %zmm19
+
+// CHECK: vpaddsb %zmm20, %zmm19, %zmm19 {%k6}
+// CHECK:  encoding: [0x62,0xa1,0x65,0x46,0xec,0xdc]
+          vpaddsb %zmm20, %zmm19, %zmm19 {%k6}
+
+// CHECK: vpaddsb %zmm20, %zmm19, %zmm19 {%k6} {z}
+// CHECK:  encoding: [0x62,0xa1,0x65,0xc6,0xec,0xdc]
+          vpaddsb %zmm20, %zmm19, %zmm19 {%k6} {z}
+
+// CHECK: vpaddsb (%rcx), %zmm19, %zmm19
+// CHECK:  encoding: [0x62,0xe1,0x65,0x40,0xec,0x19]
+          vpaddsb (%rcx), %zmm19, %zmm19
+
+// CHECK: vpaddsb 291(%rax,%r14,8), %zmm19, %zmm19
+// CHECK:  encoding: [0x62,0xa1,0x65,0x40,0xec,0x9c,0xf0,0x23,0x01,0x00,0x00]
+          vpaddsb 291(%rax,%r14,8), %zmm19, %zmm19
+
+// CHECK: vpaddsb 8128(%rdx), %zmm19, %zmm19
+// CHECK:  encoding: [0x62,0xe1,0x65,0x40,0xec,0x5a,0x7f]
+          vpaddsb 8128(%rdx), %zmm19, %zmm19
+
+// CHECK: vpaddsb 8192(%rdx), %zmm19, %zmm19
+// CHECK:  encoding: [0x62,0xe1,0x65,0x40,0xec,0x9a,0x00,0x20,0x00,0x00]
+          vpaddsb 8192(%rdx), %zmm19, %zmm19
+
+// CHECK: vpaddsb -8192(%rdx), %zmm19, %zmm19
+// CHECK:  encoding: [0x62,0xe1,0x65,0x40,0xec,0x5a,0x80]
+          vpaddsb -8192(%rdx), %zmm19, %zmm19
+
+// CHECK: vpaddsb -8256(%rdx), %zmm19, %zmm19
+// CHECK:  encoding: [0x62,0xe1,0x65,0x40,0xec,0x9a,0xc0,0xdf,0xff,0xff]
+          vpaddsb -8256(%rdx), %zmm19, %zmm19
+
+// CHECK: vpaddsw %zmm22, %zmm19, %zmm20
+// CHECK:  encoding: [0x62,0xa1,0x65,0x40,0xed,0xe6]
+          vpaddsw %zmm22, %zmm19, %zmm20
+
+// CHECK: vpaddsw %zmm22, %zmm19, %zmm20 {%k2}
+// CHECK:  encoding: [0x62,0xa1,0x65,0x42,0xed,0xe6]
+          vpaddsw %zmm22, %zmm19, %zmm20 {%k2}
+
+// CHECK: vpaddsw %zmm22, %zmm19, %zmm20 {%k2} {z}
+// CHECK:  encoding: [0x62,0xa1,0x65,0xc2,0xed,0xe6]
+          vpaddsw %zmm22, %zmm19, %zmm20 {%k2} {z}
+
+// CHECK: vpaddsw (%rcx), %zmm19, %zmm20
+// CHECK:  encoding: [0x62,0xe1,0x65,0x40,0xed,0x21]
+          vpaddsw (%rcx), %zmm19, %zmm20
+
+// CHECK: vpaddsw 291(%rax,%r14,8), %zmm19, %zmm20
+// CHECK:  encoding: [0x62,0xa1,0x65,0x40,0xed,0xa4,0xf0,0x23,0x01,0x00,0x00]
+          vpaddsw 291(%rax,%r14,8), %zmm19, %zmm20
+
+// CHECK: vpaddsw 8128(%rdx), %zmm19, %zmm20
+// CHECK:  encoding: [0x62,0xe1,0x65,0x40,0xed,0x62,0x7f]
+          vpaddsw 8128(%rdx), %zmm19, %zmm20
+
+// CHECK: vpaddsw 8192(%rdx), %zmm19, %zmm20
+// CHECK:  encoding: [0x62,0xe1,0x65,0x40,0xed,0xa2,0x00,0x20,0x00,0x00]
+          vpaddsw 8192(%rdx), %zmm19, %zmm20
+
+// CHECK: vpaddsw -8192(%rdx), %zmm19, %zmm20
+// CHECK:  encoding: [0x62,0xe1,0x65,0x40,0xed,0x62,0x80]
+          vpaddsw -8192(%rdx), %zmm19, %zmm20
+
+// CHECK: vpaddsw -8256(%rdx), %zmm19, %zmm20
+// CHECK:  encoding: [0x62,0xe1,0x65,0x40,0xed,0xa2,0xc0,0xdf,0xff,0xff]
+          vpaddsw -8256(%rdx), %zmm19, %zmm20
+
+// CHECK: vpaddusb %zmm25, %zmm29, %zmm29
+// CHECK:  encoding: [0x62,0x01,0x15,0x40,0xdc,0xe9]
+          vpaddusb %zmm25, %zmm29, %zmm29
+
+// CHECK: vpaddusb %zmm25, %zmm29, %zmm29 {%k1}
+// CHECK:  encoding: [0x62,0x01,0x15,0x41,0xdc,0xe9]
+          vpaddusb %zmm25, %zmm29, %zmm29 {%k1}
+
+// CHECK: vpaddusb %zmm25, %zmm29, %zmm29 {%k1} {z}
+// CHECK:  encoding: [0x62,0x01,0x15,0xc1,0xdc,0xe9]
+          vpaddusb %zmm25, %zmm29, %zmm29 {%k1} {z}
+
+// CHECK: vpaddusb (%rcx), %zmm29, %zmm29
+// CHECK:  encoding: [0x62,0x61,0x15,0x40,0xdc,0x29]
+          vpaddusb (%rcx), %zmm29, %zmm29
+
+// CHECK: vpaddusb 291(%rax,%r14,8), %zmm29, %zmm29
+// CHECK:  encoding: [0x62,0x21,0x15,0x40,0xdc,0xac,0xf0,0x23,0x01,0x00,0x00]
+          vpaddusb 291(%rax,%r14,8), %zmm29, %zmm29
+
+// CHECK: vpaddusb 8128(%rdx), %zmm29, %zmm29
+// CHECK:  encoding: [0x62,0x61,0x15,0x40,0xdc,0x6a,0x7f]
+          vpaddusb 8128(%rdx), %zmm29, %zmm29
+
+// CHECK: vpaddusb 8192(%rdx), %zmm29, %zmm29
+// CHECK:  encoding: [0x62,0x61,0x15,0x40,0xdc,0xaa,0x00,0x20,0x00,0x00]
+          vpaddusb 8192(%rdx), %zmm29, %zmm29
+
+// CHECK: vpaddusb -8192(%rdx), %zmm29, %zmm29
+// CHECK:  encoding: [0x62,0x61,0x15,0x40,0xdc,0x6a,0x80]
+          vpaddusb -8192(%rdx), %zmm29, %zmm29
+
+// CHECK: vpaddusb -8256(%rdx), %zmm29, %zmm29
+// CHECK:  encoding: [0x62,0x61,0x15,0x40,0xdc,0xaa,0xc0,0xdf,0xff,0xff]
+          vpaddusb -8256(%rdx), %zmm29, %zmm29
+
+// CHECK: vpaddusw %zmm17, %zmm25, %zmm21
+// CHECK:  encoding: [0x62,0xa1,0x35,0x40,0xdd,0xe9]
+          vpaddusw %zmm17, %zmm25, %zmm21
+
+// CHECK: vpaddusw %zmm17, %zmm25, %zmm21 {%k4}
+// CHECK:  encoding: [0x62,0xa1,0x35,0x44,0xdd,0xe9]
+          vpaddusw %zmm17, %zmm25, %zmm21 {%k4}
+
+// CHECK: vpaddusw %zmm17, %zmm25, %zmm21 {%k4} {z}
+// CHECK:  encoding: [0x62,0xa1,0x35,0xc4,0xdd,0xe9]
+          vpaddusw %zmm17, %zmm25, %zmm21 {%k4} {z}
+
+// CHECK: vpaddusw (%rcx), %zmm25, %zmm21
+// CHECK:  encoding: [0x62,0xe1,0x35,0x40,0xdd,0x29]
+          vpaddusw (%rcx), %zmm25, %zmm21
+
+// CHECK: vpaddusw 291(%rax,%r14,8), %zmm25, %zmm21
+// CHECK:  encoding: [0x62,0xa1,0x35,0x40,0xdd,0xac,0xf0,0x23,0x01,0x00,0x00]
+          vpaddusw 291(%rax,%r14,8), %zmm25, %zmm21
+
+// CHECK: vpaddusw 8128(%rdx), %zmm25, %zmm21
+// CHECK:  encoding: [0x62,0xe1,0x35,0x40,0xdd,0x6a,0x7f]
+          vpaddusw 8128(%rdx), %zmm25, %zmm21
+
+// CHECK: vpaddusw 8192(%rdx), %zmm25, %zmm21
+// CHECK:  encoding: [0x62,0xe1,0x35,0x40,0xdd,0xaa,0x00,0x20,0x00,0x00]
+          vpaddusw 8192(%rdx), %zmm25, %zmm21
+
+// CHECK: vpaddusw -8192(%rdx), %zmm25, %zmm21
+// CHECK:  encoding: [0x62,0xe1,0x35,0x40,0xdd,0x6a,0x80]
+          vpaddusw -8192(%rdx), %zmm25, %zmm21
+
+// CHECK: vpaddusw -8256(%rdx), %zmm25, %zmm21
+// CHECK:  encoding: [0x62,0xe1,0x35,0x40,0xdd,0xaa,0xc0,0xdf,0xff,0xff]
+          vpaddusw -8256(%rdx), %zmm25, %zmm21
+
+// CHECK: vpsubsb %zmm20, %zmm24, %zmm25
+// CHECK:  encoding: [0x62,0x21,0x3d,0x40,0xe8,0xcc]
+          vpsubsb %zmm20, %zmm24, %zmm25
+
+// CHECK: vpsubsb %zmm20, %zmm24, %zmm25 {%k2}
+// CHECK:  encoding: [0x62,0x21,0x3d,0x42,0xe8,0xcc]
+          vpsubsb %zmm20, %zmm24, %zmm25 {%k2}
+
+// CHECK: vpsubsb %zmm20, %zmm24, %zmm25 {%k2} {z}
+// CHECK:  encoding: [0x62,0x21,0x3d,0xc2,0xe8,0xcc]
+          vpsubsb %zmm20, %zmm24, %zmm25 {%k2} {z}
+
+// CHECK: vpsubsb (%rcx), %zmm24, %zmm25
+// CHECK:  encoding: [0x62,0x61,0x3d,0x40,0xe8,0x09]
+          vpsubsb (%rcx), %zmm24, %zmm25
+
+// CHECK: vpsubsb 291(%rax,%r14,8), %zmm24, %zmm25
+// CHECK:  encoding: [0x62,0x21,0x3d,0x40,0xe8,0x8c,0xf0,0x23,0x01,0x00,0x00]
+          vpsubsb 291(%rax,%r14,8), %zmm24, %zmm25
+
+// CHECK: vpsubsb 8128(%rdx), %zmm24, %zmm25
+// CHECK:  encoding: [0x62,0x61,0x3d,0x40,0xe8,0x4a,0x7f]
+          vpsubsb 8128(%rdx), %zmm24, %zmm25
+
+// CHECK: vpsubsb 8192(%rdx), %zmm24, %zmm25
+// CHECK:  encoding: [0x62,0x61,0x3d,0x40,0xe8,0x8a,0x00,0x20,0x00,0x00]
+          vpsubsb 8192(%rdx), %zmm24, %zmm25
+
+// CHECK: vpsubsb -8192(%rdx), %zmm24, %zmm25
+// CHECK:  encoding: [0x62,0x61,0x3d,0x40,0xe8,0x4a,0x80]
+          vpsubsb -8192(%rdx), %zmm24, %zmm25
+
+// CHECK: vpsubsb -8256(%rdx), %zmm24, %zmm25
+// CHECK:  encoding: [0x62,0x61,0x3d,0x40,0xe8,0x8a,0xc0,0xdf,0xff,0xff]
+          vpsubsb -8256(%rdx), %zmm24, %zmm25
+
+// CHECK: vpsubsw %zmm23, %zmm22, %zmm23
+// CHECK:  encoding: [0x62,0xa1,0x4d,0x40,0xe9,0xff]
+          vpsubsw %zmm23, %zmm22, %zmm23
+
+// CHECK: vpsubsw %zmm23, %zmm22, %zmm23 {%k3}
+// CHECK:  encoding: [0x62,0xa1,0x4d,0x43,0xe9,0xff]
+          vpsubsw %zmm23, %zmm22, %zmm23 {%k3}
+
+// CHECK: vpsubsw %zmm23, %zmm22, %zmm23 {%k3} {z}
+// CHECK:  encoding: [0x62,0xa1,0x4d,0xc3,0xe9,0xff]
+          vpsubsw %zmm23, %zmm22, %zmm23 {%k3} {z}
+
+// CHECK: vpsubsw (%rcx), %zmm22, %zmm23
+// CHECK:  encoding: [0x62,0xe1,0x4d,0x40,0xe9,0x39]
+          vpsubsw (%rcx), %zmm22, %zmm23
+
+// CHECK: vpsubsw 291(%rax,%r14,8), %zmm22, %zmm23
+// CHECK:  encoding: [0x62,0xa1,0x4d,0x40,0xe9,0xbc,0xf0,0x23,0x01,0x00,0x00]
+          vpsubsw 291(%rax,%r14,8), %zmm22, %zmm23
+
+// CHECK: vpsubsw 8128(%rdx), %zmm22, %zmm23
+// CHECK:  encoding: [0x62,0xe1,0x4d,0x40,0xe9,0x7a,0x7f]
+          vpsubsw 8128(%rdx), %zmm22, %zmm23
+
+// CHECK: vpsubsw 8192(%rdx), %zmm22, %zmm23
+// CHECK:  encoding: [0x62,0xe1,0x4d,0x40,0xe9,0xba,0x00,0x20,0x00,0x00]
+          vpsubsw 8192(%rdx), %zmm22, %zmm23
+
+// CHECK: vpsubsw -8192(%rdx), %zmm22, %zmm23
+// CHECK:  encoding: [0x62,0xe1,0x4d,0x40,0xe9,0x7a,0x80]
+          vpsubsw -8192(%rdx), %zmm22, %zmm23
+
+// CHECK: vpsubsw -8256(%rdx), %zmm22, %zmm23
+// CHECK:  encoding: [0x62,0xe1,0x4d,0x40,0xe9,0xba,0xc0,0xdf,0xff,0xff]
+          vpsubsw -8256(%rdx), %zmm22, %zmm23
+
+// CHECK: vpsubusb %zmm22, %zmm21, %zmm24
+// CHECK:  encoding: [0x62,0x21,0x55,0x40,0xd8,0xc6]
+          vpsubusb %zmm22, %zmm21, %zmm24
+
+// CHECK: vpsubusb %zmm22, %zmm21, %zmm24 {%k4}
+// CHECK:  encoding: [0x62,0x21,0x55,0x44,0xd8,0xc6]
+          vpsubusb %zmm22, %zmm21, %zmm24 {%k4}
+
+// CHECK: vpsubusb %zmm22, %zmm21, %zmm24 {%k4} {z}
+// CHECK:  encoding: [0x62,0x21,0x55,0xc4,0xd8,0xc6]
+          vpsubusb %zmm22, %zmm21, %zmm24 {%k4} {z}
+
+// CHECK: vpsubusb (%rcx), %zmm21, %zmm24
+// CHECK:  encoding: [0x62,0x61,0x55,0x40,0xd8,0x01]
+          vpsubusb (%rcx), %zmm21, %zmm24
+
+// CHECK: vpsubusb 291(%rax,%r14,8), %zmm21, %zmm24
+// CHECK:  encoding: [0x62,0x21,0x55,0x40,0xd8,0x84,0xf0,0x23,0x01,0x00,0x00]
+          vpsubusb 291(%rax,%r14,8), %zmm21, %zmm24
+
+// CHECK: vpsubusb 8128(%rdx), %zmm21, %zmm24
+// CHECK:  encoding: [0x62,0x61,0x55,0x40,0xd8,0x42,0x7f]
+          vpsubusb 8128(%rdx), %zmm21, %zmm24
+
+// CHECK: vpsubusb 8192(%rdx), %zmm21, %zmm24
+// CHECK:  encoding: [0x62,0x61,0x55,0x40,0xd8,0x82,0x00,0x20,0x00,0x00]
+          vpsubusb 8192(%rdx), %zmm21, %zmm24
+
+// CHECK: vpsubusb -8192(%rdx), %zmm21, %zmm24
+// CHECK:  encoding: [0x62,0x61,0x55,0x40,0xd8,0x42,0x80]
+          vpsubusb -8192(%rdx), %zmm21, %zmm24
+
+// CHECK: vpsubusb -8256(%rdx), %zmm21, %zmm24
+// CHECK:  encoding: [0x62,0x61,0x55,0x40,0xd8,0x82,0xc0,0xdf,0xff,0xff]
+          vpsubusb -8256(%rdx), %zmm21, %zmm24
+
+// CHECK: vpsubusw %zmm23, %zmm17, %zmm25
+// CHECK:  encoding: [0x62,0x21,0x75,0x40,0xd9,0xcf]
+          vpsubusw %zmm23, %zmm17, %zmm25
+
+// CHECK: vpsubusw %zmm23, %zmm17, %zmm25 {%k1}
+// CHECK:  encoding: [0x62,0x21,0x75,0x41,0xd9,0xcf]
+          vpsubusw %zmm23, %zmm17, %zmm25 {%k1}
+
+// CHECK: vpsubusw %zmm23, %zmm17, %zmm25 {%k1} {z}
+// CHECK:  encoding: [0x62,0x21,0x75,0xc1,0xd9,0xcf]
+          vpsubusw %zmm23, %zmm17, %zmm25 {%k1} {z}
+
+// CHECK: vpsubusw (%rcx), %zmm17, %zmm25
+// CHECK:  encoding: [0x62,0x61,0x75,0x40,0xd9,0x09]
+          vpsubusw (%rcx), %zmm17, %zmm25
+
+// CHECK: vpsubusw 291(%rax,%r14,8), %zmm17, %zmm25
+// CHECK:  encoding: [0x62,0x21,0x75,0x40,0xd9,0x8c,0xf0,0x23,0x01,0x00,0x00]
+          vpsubusw 291(%rax,%r14,8), %zmm17, %zmm25
+
+// CHECK: vpsubusw 8128(%rdx), %zmm17, %zmm25
+// CHECK:  encoding: [0x62,0x61,0x75,0x40,0xd9,0x4a,0x7f]
+          vpsubusw 8128(%rdx), %zmm17, %zmm25
+
+// CHECK: vpsubusw 8192(%rdx), %zmm17, %zmm25
+// CHECK:  encoding: [0x62,0x61,0x75,0x40,0xd9,0x8a,0x00,0x20,0x00,0x00]
+          vpsubusw 8192(%rdx), %zmm17, %zmm25
+
+// CHECK: vpsubusw -8192(%rdx), %zmm17, %zmm25
+// CHECK:  encoding: [0x62,0x61,0x75,0x40,0xd9,0x4a,0x80]
+          vpsubusw -8192(%rdx), %zmm17, %zmm25
+
+// CHECK: vpsubusw -8256(%rdx), %zmm17, %zmm25
+// CHECK:  encoding: [0x62,0x61,0x75,0x40,0xd9,0x8a,0xc0,0xdf,0xff,0xff]
+          vpsubusw -8256(%rdx), %zmm17, %zmm25
+
+// CHECK: vpaddb %zmm26, %zmm29, %zmm21
+// CHECK:  encoding: [0x62,0x81,0x15,0x40,0xfc,0xea]
+          vpaddb %zmm26, %zmm29, %zmm21
+
+// CHECK: vpaddb %zmm26, %zmm29, %zmm21 {%k5}
+// CHECK:  encoding: [0x62,0x81,0x15,0x45,0xfc,0xea]
+          vpaddb %zmm26, %zmm29, %zmm21 {%k5}
+
+// CHECK: vpaddb %zmm26, %zmm29, %zmm21 {%k5} {z}
+// CHECK:  encoding: [0x62,0x81,0x15,0xc5,0xfc,0xea]
+          vpaddb %zmm26, %zmm29, %zmm21 {%k5} {z}
+
+// CHECK: vpaddb (%rcx), %zmm29, %zmm21
+// CHECK:  encoding: [0x62,0xe1,0x15,0x40,0xfc,0x29]
+          vpaddb (%rcx), %zmm29, %zmm21
+
+// CHECK: vpaddb 4660(%rax,%r14,8), %zmm29, %zmm21
+// CHECK:  encoding: [0x62,0xa1,0x15,0x40,0xfc,0xac,0xf0,0x34,0x12,0x00,0x00]
+          vpaddb 4660(%rax,%r14,8), %zmm29, %zmm21
+
+// CHECK: vpaddb 8128(%rdx), %zmm29, %zmm21
+// CHECK:  encoding: [0x62,0xe1,0x15,0x40,0xfc,0x6a,0x7f]
+          vpaddb 8128(%rdx), %zmm29, %zmm21
+
+// CHECK: vpaddb 8192(%rdx), %zmm29, %zmm21
+// CHECK:  encoding: [0x62,0xe1,0x15,0x40,0xfc,0xaa,0x00,0x20,0x00,0x00]
+          vpaddb 8192(%rdx), %zmm29, %zmm21
+
+// CHECK: vpaddb -8192(%rdx), %zmm29, %zmm21
+// CHECK:  encoding: [0x62,0xe1,0x15,0x40,0xfc,0x6a,0x80]
+          vpaddb -8192(%rdx), %zmm29, %zmm21
+
+// CHECK: vpaddb -8256(%rdx), %zmm29, %zmm21
+// CHECK:  encoding: [0x62,0xe1,0x15,0x40,0xfc,0xaa,0xc0,0xdf,0xff,0xff]
+          vpaddb -8256(%rdx), %zmm29, %zmm21
+
+// CHECK: vpaddsb %zmm19, %zmm24, %zmm18
+// CHECK:  encoding: [0x62,0xa1,0x3d,0x40,0xec,0xd3]
+          vpaddsb %zmm19, %zmm24, %zmm18
+
+// CHECK: vpaddsb %zmm19, %zmm24, %zmm18 {%k1}
+// CHECK:  encoding: [0x62,0xa1,0x3d,0x41,0xec,0xd3]
+          vpaddsb %zmm19, %zmm24, %zmm18 {%k1}
+
+// CHECK: vpaddsb %zmm19, %zmm24, %zmm18 {%k1} {z}
+// CHECK:  encoding: [0x62,0xa1,0x3d,0xc1,0xec,0xd3]
+          vpaddsb %zmm19, %zmm24, %zmm18 {%k1} {z}
+
+// CHECK: vpaddsb (%rcx), %zmm24, %zmm18
+// CHECK:  encoding: [0x62,0xe1,0x3d,0x40,0xec,0x11]
+          vpaddsb (%rcx), %zmm24, %zmm18
+
+// CHECK: vpaddsb 4660(%rax,%r14,8), %zmm24, %zmm18
+// CHECK:  encoding: [0x62,0xa1,0x3d,0x40,0xec,0x94,0xf0,0x34,0x12,0x00,0x00]
+          vpaddsb 4660(%rax,%r14,8), %zmm24, %zmm18
+
+// CHECK: vpaddsb 8128(%rdx), %zmm24, %zmm18
+// CHECK:  encoding: [0x62,0xe1,0x3d,0x40,0xec,0x52,0x7f]
+          vpaddsb 8128(%rdx), %zmm24, %zmm18
+
+// CHECK: vpaddsb 8192(%rdx), %zmm24, %zmm18
+// CHECK:  encoding: [0x62,0xe1,0x3d,0x40,0xec,0x92,0x00,0x20,0x00,0x00]
+          vpaddsb 8192(%rdx), %zmm24, %zmm18
+
+// CHECK: vpaddsb -8192(%rdx), %zmm24, %zmm18
+// CHECK:  encoding: [0x62,0xe1,0x3d,0x40,0xec,0x52,0x80]
+          vpaddsb -8192(%rdx), %zmm24, %zmm18
+
+// CHECK: vpaddsb -8256(%rdx), %zmm24, %zmm18
+// CHECK:  encoding: [0x62,0xe1,0x3d,0x40,0xec,0x92,0xc0,0xdf,0xff,0xff]
+          vpaddsb -8256(%rdx), %zmm24, %zmm18
+
+// CHECK: vpaddsw %zmm28, %zmm17, %zmm20
+// CHECK:  encoding: [0x62,0x81,0x75,0x40,0xed,0xe4]
+          vpaddsw %zmm28, %zmm17, %zmm20
+
+// CHECK: vpaddsw %zmm28, %zmm17, %zmm20 {%k2}
+// CHECK:  encoding: [0x62,0x81,0x75,0x42,0xed,0xe4]
+          vpaddsw %zmm28, %zmm17, %zmm20 {%k2}
+
+// CHECK: vpaddsw %zmm28, %zmm17, %zmm20 {%k2} {z}
+// CHECK:  encoding: [0x62,0x81,0x75,0xc2,0xed,0xe4]
+          vpaddsw %zmm28, %zmm17, %zmm20 {%k2} {z}
+
+// CHECK: vpaddsw (%rcx), %zmm17, %zmm20
+// CHECK:  encoding: [0x62,0xe1,0x75,0x40,0xed,0x21]
+          vpaddsw (%rcx), %zmm17, %zmm20
+
+// CHECK: vpaddsw 4660(%rax,%r14,8), %zmm17, %zmm20
+// CHECK:  encoding: [0x62,0xa1,0x75,0x40,0xed,0xa4,0xf0,0x34,0x12,0x00,0x00]
+          vpaddsw 4660(%rax,%r14,8), %zmm17, %zmm20
+
+// CHECK: vpaddsw 8128(%rdx), %zmm17, %zmm20
+// CHECK:  encoding: [0x62,0xe1,0x75,0x40,0xed,0x62,0x7f]
+          vpaddsw 8128(%rdx), %zmm17, %zmm20
+
+// CHECK: vpaddsw 8192(%rdx), %zmm17, %zmm20
+// CHECK:  encoding: [0x62,0xe1,0x75,0x40,0xed,0xa2,0x00,0x20,0x00,0x00]
+          vpaddsw 8192(%rdx), %zmm17, %zmm20
+
+// CHECK: vpaddsw -8192(%rdx), %zmm17, %zmm20
+// CHECK:  encoding: [0x62,0xe1,0x75,0x40,0xed,0x62,0x80]
+          vpaddsw -8192(%rdx), %zmm17, %zmm20
+
+// CHECK: vpaddsw -8256(%rdx), %zmm17, %zmm20
+// CHECK:  encoding: [0x62,0xe1,0x75,0x40,0xed,0xa2,0xc0,0xdf,0xff,0xff]
+          vpaddsw -8256(%rdx), %zmm17, %zmm20
+
+// CHECK: vpaddusb %zmm22, %zmm27, %zmm19
+// CHECK:  encoding: [0x62,0xa1,0x25,0x40,0xdc,0xde]
+          vpaddusb %zmm22, %zmm27, %zmm19
+
+// CHECK: vpaddusb %zmm22, %zmm27, %zmm19 {%k7}
+// CHECK:  encoding: [0x62,0xa1,0x25,0x47,0xdc,0xde]
+          vpaddusb %zmm22, %zmm27, %zmm19 {%k7}
+
+// CHECK: vpaddusb %zmm22, %zmm27, %zmm19 {%k7} {z}
+// CHECK:  encoding: [0x62,0xa1,0x25,0xc7,0xdc,0xde]
+          vpaddusb %zmm22, %zmm27, %zmm19 {%k7} {z}
+
+// CHECK: vpaddusb (%rcx), %zmm27, %zmm19
+// CHECK:  encoding: [0x62,0xe1,0x25,0x40,0xdc,0x19]
+          vpaddusb (%rcx), %zmm27, %zmm19
+
+// CHECK: vpaddusb 4660(%rax,%r14,8), %zmm27, %zmm19
+// CHECK:  encoding: [0x62,0xa1,0x25,0x40,0xdc,0x9c,0xf0,0x34,0x12,0x00,0x00]
+          vpaddusb 4660(%rax,%r14,8), %zmm27, %zmm19
+
+// CHECK: vpaddusb 8128(%rdx), %zmm27, %zmm19
+// CHECK:  encoding: [0x62,0xe1,0x25,0x40,0xdc,0x5a,0x7f]
+          vpaddusb 8128(%rdx), %zmm27, %zmm19
+
+// CHECK: vpaddusb 8192(%rdx), %zmm27, %zmm19
+// CHECK:  encoding: [0x62,0xe1,0x25,0x40,0xdc,0x9a,0x00,0x20,0x00,0x00]
+          vpaddusb 8192(%rdx), %zmm27, %zmm19
+
+// CHECK: vpaddusb -8192(%rdx), %zmm27, %zmm19
+// CHECK:  encoding: [0x62,0xe1,0x25,0x40,0xdc,0x5a,0x80]
+          vpaddusb -8192(%rdx), %zmm27, %zmm19
+
+// CHECK: vpaddusb -8256(%rdx), %zmm27, %zmm19
+// CHECK:  encoding: [0x62,0xe1,0x25,0x40,0xdc,0x9a,0xc0,0xdf,0xff,0xff]
+          vpaddusb -8256(%rdx), %zmm27, %zmm19
+
+// CHECK: vpaddusw %zmm23, %zmm23, %zmm27
+// CHECK:  encoding: [0x62,0x21,0x45,0x40,0xdd,0xdf]
+          vpaddusw %zmm23, %zmm23, %zmm27
+
+// CHECK: vpaddusw %zmm23, %zmm23, %zmm27 {%k7}
+// CHECK:  encoding: [0x62,0x21,0x45,0x47,0xdd,0xdf]
+          vpaddusw %zmm23, %zmm23, %zmm27 {%k7}
+
+// CHECK: vpaddusw %zmm23, %zmm23, %zmm27 {%k7} {z}
+// CHECK:  encoding: [0x62,0x21,0x45,0xc7,0xdd,0xdf]
+          vpaddusw %zmm23, %zmm23, %zmm27 {%k7} {z}
+
+// CHECK: vpaddusw (%rcx), %zmm23, %zmm27
+// CHECK:  encoding: [0x62,0x61,0x45,0x40,0xdd,0x19]
+          vpaddusw (%rcx), %zmm23, %zmm27
+
+// CHECK: vpaddusw 4660(%rax,%r14,8), %zmm23, %zmm27
+// CHECK:  encoding: [0x62,0x21,0x45,0x40,0xdd,0x9c,0xf0,0x34,0x12,0x00,0x00]
+          vpaddusw 4660(%rax,%r14,8), %zmm23, %zmm27
+
+// CHECK: vpaddusw 8128(%rdx), %zmm23, %zmm27
+// CHECK:  encoding: [0x62,0x61,0x45,0x40,0xdd,0x5a,0x7f]
+          vpaddusw 8128(%rdx), %zmm23, %zmm27
+
+// CHECK: vpaddusw 8192(%rdx), %zmm23, %zmm27
+// CHECK:  encoding: [0x62,0x61,0x45,0x40,0xdd,0x9a,0x00,0x20,0x00,0x00]
+          vpaddusw 8192(%rdx), %zmm23, %zmm27
+
+// CHECK: vpaddusw -8192(%rdx), %zmm23, %zmm27
+// CHECK:  encoding: [0x62,0x61,0x45,0x40,0xdd,0x5a,0x80]
+          vpaddusw -8192(%rdx), %zmm23, %zmm27
+
+// CHECK: vpaddusw -8256(%rdx), %zmm23, %zmm27
+// CHECK:  encoding: [0x62,0x61,0x45,0x40,0xdd,0x9a,0xc0,0xdf,0xff,0xff]
+          vpaddusw -8256(%rdx), %zmm23, %zmm27
+
+// CHECK: vpsubsb %zmm18, %zmm28, %zmm17
+// CHECK:  encoding: [0x62,0xa1,0x1d,0x40,0xe8,0xca]
+          vpsubsb %zmm18, %zmm28, %zmm17
+
+// CHECK: vpsubsb %zmm18, %zmm28, %zmm17 {%k5}
+// CHECK:  encoding: [0x62,0xa1,0x1d,0x45,0xe8,0xca]
+          vpsubsb %zmm18, %zmm28, %zmm17 {%k5}
+
+// CHECK: vpsubsb %zmm18, %zmm28, %zmm17 {%k5} {z}
+// CHECK:  encoding: [0x62,0xa1,0x1d,0xc5,0xe8,0xca]
+          vpsubsb %zmm18, %zmm28, %zmm17 {%k5} {z}
+
+// CHECK: vpsubsb (%rcx), %zmm28, %zmm17
+// CHECK:  encoding: [0x62,0xe1,0x1d,0x40,0xe8,0x09]
+          vpsubsb (%rcx), %zmm28, %zmm17
+
+// CHECK: vpsubsb 4660(%rax,%r14,8), %zmm28, %zmm17
+// CHECK:  encoding: [0x62,0xa1,0x1d,0x40,0xe8,0x8c,0xf0,0x34,0x12,0x00,0x00]
+          vpsubsb 4660(%rax,%r14,8), %zmm28, %zmm17
+
+// CHECK: vpsubsb 8128(%rdx), %zmm28, %zmm17
+// CHECK:  encoding: [0x62,0xe1,0x1d,0x40,0xe8,0x4a,0x7f]
+          vpsubsb 8128(%rdx), %zmm28, %zmm17
+
+// CHECK: vpsubsb 8192(%rdx), %zmm28, %zmm17
+// CHECK:  encoding: [0x62,0xe1,0x1d,0x40,0xe8,0x8a,0x00,0x20,0x00,0x00]
+          vpsubsb 8192(%rdx), %zmm28, %zmm17
+
+// CHECK: vpsubsb -8192(%rdx), %zmm28, %zmm17
+// CHECK:  encoding: [0x62,0xe1,0x1d,0x40,0xe8,0x4a,0x80]
+          vpsubsb -8192(%rdx), %zmm28, %zmm17
+
+// CHECK: vpsubsb -8256(%rdx), %zmm28, %zmm17
+// CHECK:  encoding: [0x62,0xe1,0x1d,0x40,0xe8,0x8a,0xc0,0xdf,0xff,0xff]
+          vpsubsb -8256(%rdx), %zmm28, %zmm17
+
+// CHECK: vpsubsw %zmm26, %zmm24, %zmm30
+// CHECK:  encoding: [0x62,0x01,0x3d,0x40,0xe9,0xf2]
+          vpsubsw %zmm26, %zmm24, %zmm30
+
+// CHECK: vpsubsw %zmm26, %zmm24, %zmm30 {%k3}
+// CHECK:  encoding: [0x62,0x01,0x3d,0x43,0xe9,0xf2]
+          vpsubsw %zmm26, %zmm24, %zmm30 {%k3}
+
+// CHECK: vpsubsw %zmm26, %zmm24, %zmm30 {%k3} {z}
+// CHECK:  encoding: [0x62,0x01,0x3d,0xc3,0xe9,0xf2]
+          vpsubsw %zmm26, %zmm24, %zmm30 {%k3} {z}
+
+// CHECK: vpsubsw (%rcx), %zmm24, %zmm30
+// CHECK:  encoding: [0x62,0x61,0x3d,0x40,0xe9,0x31]
+          vpsubsw (%rcx), %zmm24, %zmm30
+
+// CHECK: vpsubsw 4660(%rax,%r14,8), %zmm24, %zmm30
+// CHECK:  encoding: [0x62,0x21,0x3d,0x40,0xe9,0xb4,0xf0,0x34,0x12,0x00,0x00]
+          vpsubsw 4660(%rax,%r14,8), %zmm24, %zmm30
+
+// CHECK: vpsubsw 8128(%rdx), %zmm24, %zmm30
+// CHECK:  encoding: [0x62,0x61,0x3d,0x40,0xe9,0x72,0x7f]
+          vpsubsw 8128(%rdx), %zmm24, %zmm30
+
+// CHECK: vpsubsw 8192(%rdx), %zmm24, %zmm30
+// CHECK:  encoding: [0x62,0x61,0x3d,0x40,0xe9,0xb2,0x00,0x20,0x00,0x00]
+          vpsubsw 8192(%rdx), %zmm24, %zmm30
+
+// CHECK: vpsubsw -8192(%rdx), %zmm24, %zmm30
+// CHECK:  encoding: [0x62,0x61,0x3d,0x40,0xe9,0x72,0x80]
+          vpsubsw -8192(%rdx), %zmm24, %zmm30
+
+// CHECK: vpsubsw -8256(%rdx), %zmm24, %zmm30
+// CHECK:  encoding: [0x62,0x61,0x3d,0x40,0xe9,0xb2,0xc0,0xdf,0xff,0xff]
+          vpsubsw -8256(%rdx), %zmm24, %zmm30
+
+// CHECK: vpsubusb %zmm28, %zmm29, %zmm28
+// CHECK:  encoding: [0x62,0x01,0x15,0x40,0xd8,0xe4]
+          vpsubusb %zmm28, %zmm29, %zmm28
+
+// CHECK: vpsubusb %zmm28, %zmm29, %zmm28 {%k2}
+// CHECK:  encoding: [0x62,0x01,0x15,0x42,0xd8,0xe4]
+          vpsubusb %zmm28, %zmm29, %zmm28 {%k2}
+
+// CHECK: vpsubusb %zmm28, %zmm29, %zmm28 {%k2} {z}
+// CHECK:  encoding: [0x62,0x01,0x15,0xc2,0xd8,0xe4]
+          vpsubusb %zmm28, %zmm29, %zmm28 {%k2} {z}
+
+// CHECK: vpsubusb (%rcx), %zmm29, %zmm28
+// CHECK:  encoding: [0x62,0x61,0x15,0x40,0xd8,0x21]
+          vpsubusb (%rcx), %zmm29, %zmm28
+
+// CHECK: vpsubusb 4660(%rax,%r14,8), %zmm29, %zmm28
+// CHECK:  encoding: [0x62,0x21,0x15,0x40,0xd8,0xa4,0xf0,0x34,0x12,0x00,0x00]
+          vpsubusb 4660(%rax,%r14,8), %zmm29, %zmm28
+
+// CHECK: vpsubusb 8128(%rdx), %zmm29, %zmm28
+// CHECK:  encoding: [0x62,0x61,0x15,0x40,0xd8,0x62,0x7f]
+          vpsubusb 8128(%rdx), %zmm29, %zmm28
+
+// CHECK: vpsubusb 8192(%rdx), %zmm29, %zmm28
+// CHECK:  encoding: [0x62,0x61,0x15,0x40,0xd8,0xa2,0x00,0x20,0x00,0x00]
+          vpsubusb 8192(%rdx), %zmm29, %zmm28
+
+// CHECK: vpsubusb -8192(%rdx), %zmm29, %zmm28
+// CHECK:  encoding: [0x62,0x61,0x15,0x40,0xd8,0x62,0x80]
+          vpsubusb -8192(%rdx), %zmm29, %zmm28
+
+// CHECK: vpsubusb -8256(%rdx), %zmm29, %zmm28
+// CHECK:  encoding: [0x62,0x61,0x15,0x40,0xd8,0xa2,0xc0,0xdf,0xff,0xff]
+          vpsubusb -8256(%rdx), %zmm29, %zmm28
+
+// CHECK: vpsubusw %zmm19, %zmm28, %zmm22
+// CHECK:  encoding: [0x62,0xa1,0x1d,0x40,0xd9,0xf3]
+          vpsubusw %zmm19, %zmm28, %zmm22
+
+// CHECK: vpsubusw %zmm19, %zmm28, %zmm22 {%k5}
+// CHECK:  encoding: [0x62,0xa1,0x1d,0x45,0xd9,0xf3]
+          vpsubusw %zmm19, %zmm28, %zmm22 {%k5}
+
+// CHECK: vpsubusw %zmm19, %zmm28, %zmm22 {%k5} {z}
+// CHECK:  encoding: [0x62,0xa1,0x1d,0xc5,0xd9,0xf3]
+          vpsubusw %zmm19, %zmm28, %zmm22 {%k5} {z}
+
+// CHECK: vpsubusw (%rcx), %zmm28, %zmm22
+// CHECK:  encoding: [0x62,0xe1,0x1d,0x40,0xd9,0x31]
+          vpsubusw (%rcx), %zmm28, %zmm22
+
+// CHECK: vpsubusw 4660(%rax,%r14,8), %zmm28, %zmm22
+// CHECK:  encoding: [0x62,0xa1,0x1d,0x40,0xd9,0xb4,0xf0,0x34,0x12,0x00,0x00]
+          vpsubusw 4660(%rax,%r14,8), %zmm28, %zmm22
+
+// CHECK: vpsubusw 8128(%rdx), %zmm28, %zmm22
+// CHECK:  encoding: [0x62,0xe1,0x1d,0x40,0xd9,0x72,0x7f]
+          vpsubusw 8128(%rdx), %zmm28, %zmm22
+
+// CHECK: vpsubusw 8192(%rdx), %zmm28, %zmm22
+// CHECK:  encoding: [0x62,0xe1,0x1d,0x40,0xd9,0xb2,0x00,0x20,0x00,0x00]
+          vpsubusw 8192(%rdx), %zmm28, %zmm22
+
+// CHECK: vpsubusw -8192(%rdx), %zmm28, %zmm22
+// CHECK:  encoding: [0x62,0xe1,0x1d,0x40,0xd9,0x72,0x80]
+          vpsubusw -8192(%rdx), %zmm28, %zmm22
+
+// CHECK: vpsubusw -8256(%rdx), %zmm28, %zmm22
+// CHECK:  encoding: [0x62,0xe1,0x1d,0x40,0xd9,0xb2,0xc0,0xdf,0xff,0xff]
+          vpsubusw -8256(%rdx), %zmm28, %zmm22
+
+// CHECK: vpaddsb %zmm25, %zmm19, %zmm28
+// CHECK:  encoding: [0x62,0x01,0x65,0x40,0xec,0xe1]
+          vpaddsb %zmm25, %zmm19, %zmm28
+
+// CHECK: vpaddsb %zmm25, %zmm19, %zmm28 {%k4}
+// CHECK:  encoding: [0x62,0x01,0x65,0x44,0xec,0xe1]
+          vpaddsb %zmm25, %zmm19, %zmm28 {%k4}
+
+// CHECK: vpaddsb %zmm25, %zmm19, %zmm28 {%k4} {z}
+// CHECK:  encoding: [0x62,0x01,0x65,0xc4,0xec,0xe1]
+          vpaddsb %zmm25, %zmm19, %zmm28 {%k4} {z}
+
+// CHECK: vpaddsb (%rcx), %zmm19, %zmm28
+// CHECK:  encoding: [0x62,0x61,0x65,0x40,0xec,0x21]
+          vpaddsb (%rcx), %zmm19, %zmm28
+
+// CHECK: vpaddsb 291(%rax,%r14,8), %zmm19, %zmm28
+// CHECK:  encoding: [0x62,0x21,0x65,0x40,0xec,0xa4,0xf0,0x23,0x01,0x00,0x00]
+          vpaddsb 291(%rax,%r14,8), %zmm19, %zmm28
+
+// CHECK: vpaddsb 8128(%rdx), %zmm19, %zmm28
+// CHECK:  encoding: [0x62,0x61,0x65,0x40,0xec,0x62,0x7f]
+          vpaddsb 8128(%rdx), %zmm19, %zmm28
+
+// CHECK: vpaddsb 8192(%rdx), %zmm19, %zmm28
+// CHECK:  encoding: [0x62,0x61,0x65,0x40,0xec,0xa2,0x00,0x20,0x00,0x00]
+          vpaddsb 8192(%rdx), %zmm19, %zmm28
+
+// CHECK: vpaddsb -8192(%rdx), %zmm19, %zmm28
+// CHECK:  encoding: [0x62,0x61,0x65,0x40,0xec,0x62,0x80]
+          vpaddsb -8192(%rdx), %zmm19, %zmm28
+
+// CHECK: vpaddsb -8256(%rdx), %zmm19, %zmm28
+// CHECK:  encoding: [0x62,0x61,0x65,0x40,0xec,0xa2,0xc0,0xdf,0xff,0xff]
+          vpaddsb -8256(%rdx), %zmm19, %zmm28
+
+// CHECK: vpaddsw %zmm20, %zmm22, %zmm20
+// CHECK:  encoding: [0x62,0xa1,0x4d,0x40,0xed,0xe4]
+          vpaddsw %zmm20, %zmm22, %zmm20
+
+// CHECK: vpaddsw %zmm20, %zmm22, %zmm20 {%k7}
+// CHECK:  encoding: [0x62,0xa1,0x4d,0x47,0xed,0xe4]
+          vpaddsw %zmm20, %zmm22, %zmm20 {%k7}
+
+// CHECK: vpaddsw %zmm20, %zmm22, %zmm20 {%k7} {z}
+// CHECK:  encoding: [0x62,0xa1,0x4d,0xc7,0xed,0xe4]
+          vpaddsw %zmm20, %zmm22, %zmm20 {%k7} {z}
+
+// CHECK: vpaddsw (%rcx), %zmm22, %zmm20
+// CHECK:  encoding: [0x62,0xe1,0x4d,0x40,0xed,0x21]
+          vpaddsw (%rcx), %zmm22, %zmm20
+
+// CHECK: vpaddsw 291(%rax,%r14,8), %zmm22, %zmm20
+// CHECK:  encoding: [0x62,0xa1,0x4d,0x40,0xed,0xa4,0xf0,0x23,0x01,0x00,0x00]
+          vpaddsw 291(%rax,%r14,8), %zmm22, %zmm20
+
+// CHECK: vpaddsw 8128(%rdx), %zmm22, %zmm20
+// CHECK:  encoding: [0x62,0xe1,0x4d,0x40,0xed,0x62,0x7f]
+          vpaddsw 8128(%rdx), %zmm22, %zmm20
+
+// CHECK: vpaddsw 8192(%rdx), %zmm22, %zmm20
+// CHECK:  encoding: [0x62,0xe1,0x4d,0x40,0xed,0xa2,0x00,0x20,0x00,0x00]
+          vpaddsw 8192(%rdx), %zmm22, %zmm20
+
+// CHECK: vpaddsw -8192(%rdx), %zmm22, %zmm20
+// CHECK:  encoding: [0x62,0xe1,0x4d,0x40,0xed,0x62,0x80]
+          vpaddsw -8192(%rdx), %zmm22, %zmm20
+
+// CHECK: vpaddsw -8256(%rdx), %zmm22, %zmm20
+// CHECK:  encoding: [0x62,0xe1,0x4d,0x40,0xed,0xa2,0xc0,0xdf,0xff,0xff]
+          vpaddsw -8256(%rdx), %zmm22, %zmm20
+
+// CHECK: vpaddusb %zmm17, %zmm27, %zmm26
+// CHECK:  encoding: [0x62,0x21,0x25,0x40,0xdc,0xd1]
+          vpaddusb %zmm17, %zmm27, %zmm26
+
+// CHECK: vpaddusb %zmm17, %zmm27, %zmm26 {%k3}
+// CHECK:  encoding: [0x62,0x21,0x25,0x43,0xdc,0xd1]
+          vpaddusb %zmm17, %zmm27, %zmm26 {%k3}
+
+// CHECK: vpaddusb %zmm17, %zmm27, %zmm26 {%k3} {z}
+// CHECK:  encoding: [0x62,0x21,0x25,0xc3,0xdc,0xd1]
+          vpaddusb %zmm17, %zmm27, %zmm26 {%k3} {z}
+
+// CHECK: vpaddusb (%rcx), %zmm27, %zmm26
+// CHECK:  encoding: [0x62,0x61,0x25,0x40,0xdc,0x11]
+          vpaddusb (%rcx), %zmm27, %zmm26
+
+// CHECK: vpaddusb 291(%rax,%r14,8), %zmm27, %zmm26
+// CHECK:  encoding: [0x62,0x21,0x25,0x40,0xdc,0x94,0xf0,0x23,0x01,0x00,0x00]
+          vpaddusb 291(%rax,%r14,8), %zmm27, %zmm26
+
+// CHECK: vpaddusb 8128(%rdx), %zmm27, %zmm26
+// CHECK:  encoding: [0x62,0x61,0x25,0x40,0xdc,0x52,0x7f]
+          vpaddusb 8128(%rdx), %zmm27, %zmm26
+
+// CHECK: vpaddusb 8192(%rdx), %zmm27, %zmm26
+// CHECK:  encoding: [0x62,0x61,0x25,0x40,0xdc,0x92,0x00,0x20,0x00,0x00]
+          vpaddusb 8192(%rdx), %zmm27, %zmm26
+
+// CHECK: vpaddusb -8192(%rdx), %zmm27, %zmm26
+// CHECK:  encoding: [0x62,0x61,0x25,0x40,0xdc,0x52,0x80]
+          vpaddusb -8192(%rdx), %zmm27, %zmm26
+
+// CHECK: vpaddusb -8256(%rdx), %zmm27, %zmm26
+// CHECK:  encoding: [0x62,0x61,0x25,0x40,0xdc,0x92,0xc0,0xdf,0xff,0xff]
+          vpaddusb -8256(%rdx), %zmm27, %zmm26
+
+// CHECK: vpaddusw %zmm20, %zmm22, %zmm21
+// CHECK:  encoding: [0x62,0xa1,0x4d,0x40,0xdd,0xec]
+          vpaddusw %zmm20, %zmm22, %zmm21
+
+// CHECK: vpaddusw %zmm20, %zmm22, %zmm21 {%k7}
+// CHECK:  encoding: [0x62,0xa1,0x4d,0x47,0xdd,0xec]
+          vpaddusw %zmm20, %zmm22, %zmm21 {%k7}
+
+// CHECK: vpaddusw %zmm20, %zmm22, %zmm21 {%k7} {z}
+// CHECK:  encoding: [0x62,0xa1,0x4d,0xc7,0xdd,0xec]
+          vpaddusw %zmm20, %zmm22, %zmm21 {%k7} {z}
+
+// CHECK: vpaddusw (%rcx), %zmm22, %zmm21
+// CHECK:  encoding: [0x62,0xe1,0x4d,0x40,0xdd,0x29]
+          vpaddusw (%rcx), %zmm22, %zmm21
+
+// CHECK: vpaddusw 291(%rax,%r14,8), %zmm22, %zmm21
+// CHECK:  encoding: [0x62,0xa1,0x4d,0x40,0xdd,0xac,0xf0,0x23,0x01,0x00,0x00]
+          vpaddusw 291(%rax,%r14,8), %zmm22, %zmm21
+
+// CHECK: vpaddusw 8128(%rdx), %zmm22, %zmm21
+// CHECK:  encoding: [0x62,0xe1,0x4d,0x40,0xdd,0x6a,0x7f]
+          vpaddusw 8128(%rdx), %zmm22, %zmm21
+
+// CHECK: vpaddusw 8192(%rdx), %zmm22, %zmm21
+// CHECK:  encoding: [0x62,0xe1,0x4d,0x40,0xdd,0xaa,0x00,0x20,0x00,0x00]
+          vpaddusw 8192(%rdx), %zmm22, %zmm21
+
+// CHECK: vpaddusw -8192(%rdx), %zmm22, %zmm21
+// CHECK:  encoding: [0x62,0xe1,0x4d,0x40,0xdd,0x6a,0x80]
+          vpaddusw -8192(%rdx), %zmm22, %zmm21
+
+// CHECK: vpaddusw -8256(%rdx), %zmm22, %zmm21
+// CHECK:  encoding: [0x62,0xe1,0x4d,0x40,0xdd,0xaa,0xc0,0xdf,0xff,0xff]
+          vpaddusw -8256(%rdx), %zmm22, %zmm21
+
+// CHECK: vpsubsb %zmm28, %zmm21, %zmm19
+// CHECK:  encoding: [0x62,0x81,0x55,0x40,0xe8,0xdc]
+          vpsubsb %zmm28, %zmm21, %zmm19
+
+// CHECK: vpsubsb %zmm28, %zmm21, %zmm19 {%k2}
+// CHECK:  encoding: [0x62,0x81,0x55,0x42,0xe8,0xdc]
+          vpsubsb %zmm28, %zmm21, %zmm19 {%k2}
+
+// CHECK: vpsubsb %zmm28, %zmm21, %zmm19 {%k2} {z}
+// CHECK:  encoding: [0x62,0x81,0x55,0xc2,0xe8,0xdc]
+          vpsubsb %zmm28, %zmm21, %zmm19 {%k2} {z}
+
+// CHECK: vpsubsb (%rcx), %zmm21, %zmm19
+// CHECK:  encoding: [0x62,0xe1,0x55,0x40,0xe8,0x19]
+          vpsubsb (%rcx), %zmm21, %zmm19
+
+// CHECK: vpsubsb 291(%rax,%r14,8), %zmm21, %zmm19
+// CHECK:  encoding: [0x62,0xa1,0x55,0x40,0xe8,0x9c,0xf0,0x23,0x01,0x00,0x00]
+          vpsubsb 291(%rax,%r14,8), %zmm21, %zmm19
+
+// CHECK: vpsubsb 8128(%rdx), %zmm21, %zmm19
+// CHECK:  encoding: [0x62,0xe1,0x55,0x40,0xe8,0x5a,0x7f]
+          vpsubsb 8128(%rdx), %zmm21, %zmm19
+
+// CHECK: vpsubsb 8192(%rdx), %zmm21, %zmm19
+// CHECK:  encoding: [0x62,0xe1,0x55,0x40,0xe8,0x9a,0x00,0x20,0x00,0x00]
+          vpsubsb 8192(%rdx), %zmm21, %zmm19
+
+// CHECK: vpsubsb -8192(%rdx), %zmm21, %zmm19
+// CHECK:  encoding: [0x62,0xe1,0x55,0x40,0xe8,0x5a,0x80]
+          vpsubsb -8192(%rdx), %zmm21, %zmm19
+
+// CHECK: vpsubsb -8256(%rdx), %zmm21, %zmm19
+// CHECK:  encoding: [0x62,0xe1,0x55,0x40,0xe8,0x9a,0xc0,0xdf,0xff,0xff]
+          vpsubsb -8256(%rdx), %zmm21, %zmm19
+
+// CHECK: vpsubsw %zmm23, %zmm23, %zmm23
+// CHECK:  encoding: [0x62,0xa1,0x45,0x40,0xe9,0xff]
+          vpsubsw %zmm23, %zmm23, %zmm23
+
+// CHECK: vpsubsw %zmm23, %zmm23, %zmm23 {%k6}
+// CHECK:  encoding: [0x62,0xa1,0x45,0x46,0xe9,0xff]
+          vpsubsw %zmm23, %zmm23, %zmm23 {%k6}
+
+// CHECK: vpsubsw %zmm23, %zmm23, %zmm23 {%k6} {z}
+// CHECK:  encoding: [0x62,0xa1,0x45,0xc6,0xe9,0xff]
+          vpsubsw %zmm23, %zmm23, %zmm23 {%k6} {z}
+
+// CHECK: vpsubsw (%rcx), %zmm23, %zmm23
+// CHECK:  encoding: [0x62,0xe1,0x45,0x40,0xe9,0x39]
+          vpsubsw (%rcx), %zmm23, %zmm23
+
+// CHECK: vpsubsw 291(%rax,%r14,8), %zmm23, %zmm23
+// CHECK:  encoding: [0x62,0xa1,0x45,0x40,0xe9,0xbc,0xf0,0x23,0x01,0x00,0x00]
+          vpsubsw 291(%rax,%r14,8), %zmm23, %zmm23
+
+// CHECK: vpsubsw 8128(%rdx), %zmm23, %zmm23
+// CHECK:  encoding: [0x62,0xe1,0x45,0x40,0xe9,0x7a,0x7f]
+          vpsubsw 8128(%rdx), %zmm23, %zmm23
+
+// CHECK: vpsubsw 8192(%rdx), %zmm23, %zmm23
+// CHECK:  encoding: [0x62,0xe1,0x45,0x40,0xe9,0xba,0x00,0x20,0x00,0x00]
+          vpsubsw 8192(%rdx), %zmm23, %zmm23
+
+// CHECK: vpsubsw -8192(%rdx), %zmm23, %zmm23
+// CHECK:  encoding: [0x62,0xe1,0x45,0x40,0xe9,0x7a,0x80]
+          vpsubsw -8192(%rdx), %zmm23, %zmm23
+
+// CHECK: vpsubsw -8256(%rdx), %zmm23, %zmm23
+// CHECK:  encoding: [0x62,0xe1,0x45,0x40,0xe9,0xba,0xc0,0xdf,0xff,0xff]
+          vpsubsw -8256(%rdx), %zmm23, %zmm23
+
+// CHECK: vpsubusb %zmm25, %zmm29, %zmm27
+// CHECK:  encoding: [0x62,0x01,0x15,0x40,0xd8,0xd9]
+          vpsubusb %zmm25, %zmm29, %zmm27
+
+// CHECK: vpsubusb %zmm25, %zmm29, %zmm27 {%k4}
+// CHECK:  encoding: [0x62,0x01,0x15,0x44,0xd8,0xd9]
+          vpsubusb %zmm25, %zmm29, %zmm27 {%k4}
+
+// CHECK: vpsubusb %zmm25, %zmm29, %zmm27 {%k4} {z}
+// CHECK:  encoding: [0x62,0x01,0x15,0xc4,0xd8,0xd9]
+          vpsubusb %zmm25, %zmm29, %zmm27 {%k4} {z}
+
+// CHECK: vpsubusb (%rcx), %zmm29, %zmm27
+// CHECK:  encoding: [0x62,0x61,0x15,0x40,0xd8,0x19]
+          vpsubusb (%rcx), %zmm29, %zmm27
+
+// CHECK: vpsubusb 291(%rax,%r14,8), %zmm29, %zmm27
+// CHECK:  encoding: [0x62,0x21,0x15,0x40,0xd8,0x9c,0xf0,0x23,0x01,0x00,0x00]
+          vpsubusb 291(%rax,%r14,8), %zmm29, %zmm27
+
+// CHECK: vpsubusb 8128(%rdx), %zmm29, %zmm27
+// CHECK:  encoding: [0x62,0x61,0x15,0x40,0xd8,0x5a,0x7f]
+          vpsubusb 8128(%rdx), %zmm29, %zmm27
+
+// CHECK: vpsubusb 8192(%rdx), %zmm29, %zmm27
+// CHECK:  encoding: [0x62,0x61,0x15,0x40,0xd8,0x9a,0x00,0x20,0x00,0x00]
+          vpsubusb 8192(%rdx), %zmm29, %zmm27
+
+// CHECK: vpsubusb -8192(%rdx), %zmm29, %zmm27
+// CHECK:  encoding: [0x62,0x61,0x15,0x40,0xd8,0x5a,0x80]
+          vpsubusb -8192(%rdx), %zmm29, %zmm27
+
+// CHECK: vpsubusb -8256(%rdx), %zmm29, %zmm27
+// CHECK:  encoding: [0x62,0x61,0x15,0x40,0xd8,0x9a,0xc0,0xdf,0xff,0xff]
+          vpsubusb -8256(%rdx), %zmm29, %zmm27
+
+// CHECK: vpsubusw %zmm25, %zmm20, %zmm20
+// CHECK:  encoding: [0x62,0x81,0x5d,0x40,0xd9,0xe1]
+          vpsubusw %zmm25, %zmm20, %zmm20
+
+// CHECK: vpsubusw %zmm25, %zmm20, %zmm20 {%k6}
+// CHECK:  encoding: [0x62,0x81,0x5d,0x46,0xd9,0xe1]
+          vpsubusw %zmm25, %zmm20, %zmm20 {%k6}
+
+// CHECK: vpsubusw %zmm25, %zmm20, %zmm20 {%k6} {z}
+// CHECK:  encoding: [0x62,0x81,0x5d,0xc6,0xd9,0xe1]
+          vpsubusw %zmm25, %zmm20, %zmm20 {%k6} {z}
+
+// CHECK: vpsubusw (%rcx), %zmm20, %zmm20
+// CHECK:  encoding: [0x62,0xe1,0x5d,0x40,0xd9,0x21]
+          vpsubusw (%rcx), %zmm20, %zmm20
+
+// CHECK: vpsubusw 291(%rax,%r14,8), %zmm20, %zmm20
+// CHECK:  encoding: [0x62,0xa1,0x5d,0x40,0xd9,0xa4,0xf0,0x23,0x01,0x00,0x00]
+          vpsubusw 291(%rax,%r14,8), %zmm20, %zmm20
+
+// CHECK: vpsubusw 8128(%rdx), %zmm20, %zmm20
+// CHECK:  encoding: [0x62,0xe1,0x5d,0x40,0xd9,0x62,0x7f]
+          vpsubusw 8128(%rdx), %zmm20, %zmm20
+
+// CHECK: vpsubusw 8192(%rdx), %zmm20, %zmm20
+// CHECK:  encoding: [0x62,0xe1,0x5d,0x40,0xd9,0xa2,0x00,0x20,0x00,0x00]
+          vpsubusw 8192(%rdx), %zmm20, %zmm20
+
+// CHECK: vpsubusw -8192(%rdx), %zmm20, %zmm20
+// CHECK:  encoding: [0x62,0xe1,0x5d,0x40,0xd9,0x62,0x80]
+          vpsubusw -8192(%rdx), %zmm20, %zmm20
+
+// CHECK: vpsubusw -8256(%rdx), %zmm20, %zmm20
+// CHECK:  encoding: [0x62,0xe1,0x5d,0x40,0xd9,0xa2,0xc0,0xdf,0xff,0xff]
+          vpsubusw -8256(%rdx), %zmm20, %zmm20
+
+// CHECK: vpermw %zmm21, %zmm19, %zmm22
+// CHECK:  encoding: [0x62,0xa2,0xe5,0x40,0x8d,0xf5]
+          vpermw %zmm21, %zmm19, %zmm22
+
+// CHECK: vpermw %zmm21, %zmm19, %zmm22 {%k6}
+// CHECK:  encoding: [0x62,0xa2,0xe5,0x46,0x8d,0xf5]
+          vpermw %zmm21, %zmm19, %zmm22 {%k6}
+
+// CHECK: vpermw %zmm21, %zmm19, %zmm22 {%k6} {z}
+// CHECK:  encoding: [0x62,0xa2,0xe5,0xc6,0x8d,0xf5]
+          vpermw %zmm21, %zmm19, %zmm22 {%k6} {z}
+
+// CHECK: vpermw (%rcx), %zmm19, %zmm22
+// CHECK:  encoding: [0x62,0xe2,0xe5,0x40,0x8d,0x31]
+          vpermw (%rcx), %zmm19, %zmm22
+
+// CHECK: vpermw 291(%rax,%r14,8), %zmm19, %zmm22
+// CHECK:  encoding: [0x62,0xa2,0xe5,0x40,0x8d,0xb4,0xf0,0x23,0x01,0x00,0x00]
+          vpermw 291(%rax,%r14,8), %zmm19, %zmm22
+
+// CHECK: vpermw 8128(%rdx), %zmm19, %zmm22
+// CHECK:  encoding: [0x62,0xe2,0xe5,0x40,0x8d,0x72,0x7f]
+          vpermw 8128(%rdx), %zmm19, %zmm22
+
+// CHECK: vpermw 8192(%rdx), %zmm19, %zmm22
+// CHECK:  encoding: [0x62,0xe2,0xe5,0x40,0x8d,0xb2,0x00,0x20,0x00,0x00]
+          vpermw 8192(%rdx), %zmm19, %zmm22
+
+// CHECK: vpermw -8192(%rdx), %zmm19, %zmm22
+// CHECK:  encoding: [0x62,0xe2,0xe5,0x40,0x8d,0x72,0x80]
+          vpermw -8192(%rdx), %zmm19, %zmm22
+
+// CHECK: vpermw -8256(%rdx), %zmm19, %zmm22
+// CHECK:  encoding: [0x62,0xe2,0xe5,0x40,0x8d,0xb2,0xc0,0xdf,0xff,0xff]
+          vpermw -8256(%rdx), %zmm19, %zmm22
+
+// CHECK: vpermi2w %zmm24, %zmm24, %zmm17
+// CHECK:  encoding: [0x62,0x82,0xbd,0x40,0x75,0xc8]
+          vpermi2w %zmm24, %zmm24, %zmm17
+
+// CHECK: vpermi2w %zmm24, %zmm24, %zmm17 {%k7}
+// CHECK:  encoding: [0x62,0x82,0xbd,0x47,0x75,0xc8]
+          vpermi2w %zmm24, %zmm24, %zmm17 {%k7}
+
+// CHECK: vpermi2w %zmm24, %zmm24, %zmm17 {%k7} {z}
+// CHECK:  encoding: [0x62,0x82,0xbd,0xc7,0x75,0xc8]
+          vpermi2w %zmm24, %zmm24, %zmm17 {%k7} {z}
+
+// CHECK: vpermi2w (%rcx), %zmm24, %zmm17
+// CHECK:  encoding: [0x62,0xe2,0xbd,0x40,0x75,0x09]
+          vpermi2w (%rcx), %zmm24, %zmm17
+
+// CHECK: vpermi2w 291(%rax,%r14,8), %zmm24, %zmm17
+// CHECK:  encoding: [0x62,0xa2,0xbd,0x40,0x75,0x8c,0xf0,0x23,0x01,0x00,0x00]
+          vpermi2w 291(%rax,%r14,8), %zmm24, %zmm17
+
+// CHECK: vpermi2w 8128(%rdx), %zmm24, %zmm17
+// CHECK:  encoding: [0x62,0xe2,0xbd,0x40,0x75,0x4a,0x7f]
+          vpermi2w 8128(%rdx), %zmm24, %zmm17
+
+// CHECK: vpermi2w 8192(%rdx), %zmm24, %zmm17
+// CHECK:  encoding: [0x62,0xe2,0xbd,0x40,0x75,0x8a,0x00,0x20,0x00,0x00]
+          vpermi2w 8192(%rdx), %zmm24, %zmm17
+
+// CHECK: vpermi2w -8192(%rdx), %zmm24, %zmm17
+// CHECK:  encoding: [0x62,0xe2,0xbd,0x40,0x75,0x4a,0x80]
+          vpermi2w -8192(%rdx), %zmm24, %zmm17
+
+// CHECK: vpermi2w -8256(%rdx), %zmm24, %zmm17
+// CHECK:  encoding: [0x62,0xe2,0xbd,0x40,0x75,0x8a,0xc0,0xdf,0xff,0xff]
+          vpermi2w -8256(%rdx), %zmm24, %zmm17
+
+// CHECK: vpermt2w %zmm19, %zmm25, %zmm18
+// CHECK:  encoding: [0x62,0xa2,0xb5,0x40,0x7d,0xd3]
+          vpermt2w %zmm19, %zmm25, %zmm18
+
+// CHECK: vpermt2w %zmm19, %zmm25, %zmm18 {%k2}
+// CHECK:  encoding: [0x62,0xa2,0xb5,0x42,0x7d,0xd3]
+          vpermt2w %zmm19, %zmm25, %zmm18 {%k2}
+
+// CHECK: vpermt2w %zmm19, %zmm25, %zmm18 {%k2} {z}
+// CHECK:  encoding: [0x62,0xa2,0xb5,0xc2,0x7d,0xd3]
+          vpermt2w %zmm19, %zmm25, %zmm18 {%k2} {z}
+
+// CHECK: vpermt2w (%rcx), %zmm25, %zmm18
+// CHECK:  encoding: [0x62,0xe2,0xb5,0x40,0x7d,0x11]
+          vpermt2w (%rcx), %zmm25, %zmm18
+
+// CHECK: vpermt2w 291(%rax,%r14,8), %zmm25, %zmm18
+// CHECK:  encoding: [0x62,0xa2,0xb5,0x40,0x7d,0x94,0xf0,0x23,0x01,0x00,0x00]
+          vpermt2w 291(%rax,%r14,8), %zmm25, %zmm18
+
+// CHECK: vpermt2w 8128(%rdx), %zmm25, %zmm18
+// CHECK:  encoding: [0x62,0xe2,0xb5,0x40,0x7d,0x52,0x7f]
+          vpermt2w 8128(%rdx), %zmm25, %zmm18
+
+// CHECK: vpermt2w 8192(%rdx), %zmm25, %zmm18
+// CHECK:  encoding: [0x62,0xe2,0xb5,0x40,0x7d,0x92,0x00,0x20,0x00,0x00]
+          vpermt2w 8192(%rdx), %zmm25, %zmm18
+
+// CHECK: vpermt2w -8192(%rdx), %zmm25, %zmm18
+// CHECK:  encoding: [0x62,0xe2,0xb5,0x40,0x7d,0x52,0x80]
+          vpermt2w -8192(%rdx), %zmm25, %zmm18
+
+// CHECK: vpermt2w -8256(%rdx), %zmm25, %zmm18
+// CHECK:  encoding: [0x62,0xe2,0xb5,0x40,0x7d,0x92,0xc0,0xdf,0xff,0xff]
+          vpermt2w -8256(%rdx), %zmm25, %zmm18
+
+// CHECK: vpavgb %zmm21, %zmm29, %zmm17
+// CHECK:  encoding: [0x62,0xa1,0x15,0x40,0xe0,0xcd]
+          vpavgb %zmm21, %zmm29, %zmm17
+
+// CHECK: vpavgb %zmm21, %zmm29, %zmm17 {%k2}
+// CHECK:  encoding: [0x62,0xa1,0x15,0x42,0xe0,0xcd]
+          vpavgb %zmm21, %zmm29, %zmm17 {%k2}
+
+// CHECK: vpavgb %zmm21, %zmm29, %zmm17 {%k2} {z}
+// CHECK:  encoding: [0x62,0xa1,0x15,0xc2,0xe0,0xcd]
+          vpavgb %zmm21, %zmm29, %zmm17 {%k2} {z}
+
+// CHECK: vpavgb (%rcx), %zmm29, %zmm17
+// CHECK:  encoding: [0x62,0xe1,0x15,0x40,0xe0,0x09]
+          vpavgb (%rcx), %zmm29, %zmm17
+
+// CHECK: vpavgb 291(%rax,%r14,8), %zmm29, %zmm17
+// CHECK:  encoding: [0x62,0xa1,0x15,0x40,0xe0,0x8c,0xf0,0x23,0x01,0x00,0x00]
+          vpavgb 291(%rax,%r14,8), %zmm29, %zmm17
+
+// CHECK: vpavgb 8128(%rdx), %zmm29, %zmm17
+// CHECK:  encoding: [0x62,0xe1,0x15,0x40,0xe0,0x4a,0x7f]
+          vpavgb 8128(%rdx), %zmm29, %zmm17
+
+// CHECK: vpavgb 8192(%rdx), %zmm29, %zmm17
+// CHECK:  encoding: [0x62,0xe1,0x15,0x40,0xe0,0x8a,0x00,0x20,0x00,0x00]
+          vpavgb 8192(%rdx), %zmm29, %zmm17
+
+// CHECK: vpavgb -8192(%rdx), %zmm29, %zmm17
+// CHECK:  encoding: [0x62,0xe1,0x15,0x40,0xe0,0x4a,0x80]
+          vpavgb -8192(%rdx), %zmm29, %zmm17
+
+// CHECK: vpavgb -8256(%rdx), %zmm29, %zmm17
+// CHECK:  encoding: [0x62,0xe1,0x15,0x40,0xe0,0x8a,0xc0,0xdf,0xff,0xff]
+          vpavgb -8256(%rdx), %zmm29, %zmm17
+
+// CHECK: vpavgw %zmm22, %zmm27, %zmm19
+// CHECK:  encoding: [0x62,0xa1,0x25,0x40,0xe3,0xde]
+          vpavgw %zmm22, %zmm27, %zmm19
+
+// CHECK: vpavgw %zmm22, %zmm27, %zmm19 {%k4}
+// CHECK:  encoding: [0x62,0xa1,0x25,0x44,0xe3,0xde]
+          vpavgw %zmm22, %zmm27, %zmm19 {%k4}
+
+// CHECK: vpavgw %zmm22, %zmm27, %zmm19 {%k4} {z}
+// CHECK:  encoding: [0x62,0xa1,0x25,0xc4,0xe3,0xde]
+          vpavgw %zmm22, %zmm27, %zmm19 {%k4} {z}
+
+// CHECK: vpavgw (%rcx), %zmm27, %zmm19
+// CHECK:  encoding: [0x62,0xe1,0x25,0x40,0xe3,0x19]
+          vpavgw (%rcx), %zmm27, %zmm19
+
+// CHECK: vpavgw 291(%rax,%r14,8), %zmm27, %zmm19
+// CHECK:  encoding: [0x62,0xa1,0x25,0x40,0xe3,0x9c,0xf0,0x23,0x01,0x00,0x00]
+          vpavgw 291(%rax,%r14,8), %zmm27, %zmm19
+
+// CHECK: vpavgw 8128(%rdx), %zmm27, %zmm19
+// CHECK:  encoding: [0x62,0xe1,0x25,0x40,0xe3,0x5a,0x7f]
+          vpavgw 8128(%rdx), %zmm27, %zmm19
+
+// CHECK: vpavgw 8192(%rdx), %zmm27, %zmm19
+// CHECK:  encoding: [0x62,0xe1,0x25,0x40,0xe3,0x9a,0x00,0x20,0x00,0x00]
+          vpavgw 8192(%rdx), %zmm27, %zmm19
+
+// CHECK: vpavgw -8192(%rdx), %zmm27, %zmm19
+// CHECK:  encoding: [0x62,0xe1,0x25,0x40,0xe3,0x5a,0x80]
+          vpavgw -8192(%rdx), %zmm27, %zmm19
+
+// CHECK: vpavgw -8256(%rdx), %zmm27, %zmm19
+// CHECK:  encoding: [0x62,0xe1,0x25,0x40,0xe3,0x9a,0xc0,0xdf,0xff,0xff]
+          vpavgw -8256(%rdx), %zmm27, %zmm19
+
+// CHECK: vpavgb %zmm19, %zmm27, %zmm24
+// CHECK:  encoding: [0x62,0x21,0x25,0x40,0xe0,0xc3]
+          vpavgb %zmm19, %zmm27, %zmm24
+
+// CHECK: vpavgb %zmm19, %zmm27, %zmm24 {%k4}
+// CHECK:  encoding: [0x62,0x21,0x25,0x44,0xe0,0xc3]
+          vpavgb %zmm19, %zmm27, %zmm24 {%k4}
+
+// CHECK: vpavgb %zmm19, %zmm27, %zmm24 {%k4} {z}
+// CHECK:  encoding: [0x62,0x21,0x25,0xc4,0xe0,0xc3]
+          vpavgb %zmm19, %zmm27, %zmm24 {%k4} {z}
+
+// CHECK: vpavgb (%rcx), %zmm27, %zmm24
+// CHECK:  encoding: [0x62,0x61,0x25,0x40,0xe0,0x01]
+          vpavgb (%rcx), %zmm27, %zmm24
+
+// CHECK: vpavgb 291(%rax,%r14,8), %zmm27, %zmm24
+// CHECK:  encoding: [0x62,0x21,0x25,0x40,0xe0,0x84,0xf0,0x23,0x01,0x00,0x00]
+          vpavgb 291(%rax,%r14,8), %zmm27, %zmm24
+
+// CHECK: vpavgb 8128(%rdx), %zmm27, %zmm24
+// CHECK:  encoding: [0x62,0x61,0x25,0x40,0xe0,0x42,0x7f]
+          vpavgb 8128(%rdx), %zmm27, %zmm24
+
+// CHECK: vpavgb 8192(%rdx), %zmm27, %zmm24
+// CHECK:  encoding: [0x62,0x61,0x25,0x40,0xe0,0x82,0x00,0x20,0x00,0x00]
+          vpavgb 8192(%rdx), %zmm27, %zmm24
+
+// CHECK: vpavgb -8192(%rdx), %zmm27, %zmm24
+// CHECK:  encoding: [0x62,0x61,0x25,0x40,0xe0,0x42,0x80]
+          vpavgb -8192(%rdx), %zmm27, %zmm24
+
+// CHECK: vpavgb -8256(%rdx), %zmm27, %zmm24
+// CHECK:  encoding: [0x62,0x61,0x25,0x40,0xe0,0x82,0xc0,0xdf,0xff,0xff]
+          vpavgb -8256(%rdx), %zmm27, %zmm24
+
+// CHECK: vpavgw %zmm26, %zmm29, %zmm29
+// CHECK:  encoding: [0x62,0x01,0x15,0x40,0xe3,0xea]
+          vpavgw %zmm26, %zmm29, %zmm29
+
+// CHECK: vpavgw %zmm26, %zmm29, %zmm29 {%k6}
+// CHECK:  encoding: [0x62,0x01,0x15,0x46,0xe3,0xea]
+          vpavgw %zmm26, %zmm29, %zmm29 {%k6}
+
+// CHECK: vpavgw %zmm26, %zmm29, %zmm29 {%k6} {z}
+// CHECK:  encoding: [0x62,0x01,0x15,0xc6,0xe3,0xea]
+          vpavgw %zmm26, %zmm29, %zmm29 {%k6} {z}
+
+// CHECK: vpavgw (%rcx), %zmm29, %zmm29
+// CHECK:  encoding: [0x62,0x61,0x15,0x40,0xe3,0x29]
+          vpavgw (%rcx), %zmm29, %zmm29
+
+// CHECK: vpavgw 291(%rax,%r14,8), %zmm29, %zmm29
+// CHECK:  encoding: [0x62,0x21,0x15,0x40,0xe3,0xac,0xf0,0x23,0x01,0x00,0x00]
+          vpavgw 291(%rax,%r14,8), %zmm29, %zmm29
+
+// CHECK: vpavgw 8128(%rdx), %zmm29, %zmm29
+// CHECK:  encoding: [0x62,0x61,0x15,0x40,0xe3,0x6a,0x7f]
+          vpavgw 8128(%rdx), %zmm29, %zmm29
+
+// CHECK: vpavgw 8192(%rdx), %zmm29, %zmm29
+// CHECK:  encoding: [0x62,0x61,0x15,0x40,0xe3,0xaa,0x00,0x20,0x00,0x00]
+          vpavgw 8192(%rdx), %zmm29, %zmm29
+
+// CHECK: vpavgw -8192(%rdx), %zmm29, %zmm29
+// CHECK:  encoding: [0x62,0x61,0x15,0x40,0xe3,0x6a,0x80]
+          vpavgw -8192(%rdx), %zmm29, %zmm29
+
+// CHECK: vpavgw -8256(%rdx), %zmm29, %zmm29
+// CHECK:  encoding: [0x62,0x61,0x15,0x40,0xe3,0xaa,0xc0,0xdf,0xff,0xff]
+          vpavgw -8256(%rdx), %zmm29, %zmm29
+
+// CHECK: vpshufb %zmm20, %zmm26, %zmm22
+// CHECK:  encoding: [0x62,0xa2,0x2d,0x40,0x00,0xf4]
+          vpshufb %zmm20, %zmm26, %zmm22
+
+// CHECK: vpshufb %zmm20, %zmm26, %zmm22 {%k7}
+// CHECK:  encoding: [0x62,0xa2,0x2d,0x47,0x00,0xf4]
+          vpshufb %zmm20, %zmm26, %zmm22 {%k7}
+
+// CHECK: vpshufb %zmm20, %zmm26, %zmm22 {%k7} {z}
+// CHECK:  encoding: [0x62,0xa2,0x2d,0xc7,0x00,0xf4]
+          vpshufb %zmm20, %zmm26, %zmm22 {%k7} {z}
+
+// CHECK: vpshufb (%rcx), %zmm26, %zmm22
+// CHECK:  encoding: [0x62,0xe2,0x2d,0x40,0x00,0x31]
+          vpshufb (%rcx), %zmm26, %zmm22
+
+// CHECK: vpshufb 291(%rax,%r14,8), %zmm26, %zmm22
+// CHECK:  encoding: [0x62,0xa2,0x2d,0x40,0x00,0xb4,0xf0,0x23,0x01,0x00,0x00]
+          vpshufb 291(%rax,%r14,8), %zmm26, %zmm22
+
+// CHECK: vpshufb 8128(%rdx), %zmm26, %zmm22
+// CHECK:  encoding: [0x62,0xe2,0x2d,0x40,0x00,0x72,0x7f]
+          vpshufb 8128(%rdx), %zmm26, %zmm22
+
+// CHECK: vpshufb 8192(%rdx), %zmm26, %zmm22
+// CHECK:  encoding: [0x62,0xe2,0x2d,0x40,0x00,0xb2,0x00,0x20,0x00,0x00]
+          vpshufb 8192(%rdx), %zmm26, %zmm22
+
+// CHECK: vpshufb -8192(%rdx), %zmm26, %zmm22
+// CHECK:  encoding: [0x62,0xe2,0x2d,0x40,0x00,0x72,0x80]
+          vpshufb -8192(%rdx), %zmm26, %zmm22
+
+// CHECK: vpshufb -8256(%rdx), %zmm26, %zmm22
+// CHECK:  encoding: [0x62,0xe2,0x2d,0x40,0x00,0xb2,0xc0,0xdf,0xff,0xff]
+          vpshufb -8256(%rdx), %zmm26, %zmm22
+
+// CHECK: vpabsb %zmm27, %zmm17
+// CHECK:  encoding: [0x62,0x82,0x7d,0x48,0x1c,0xcb]
+          vpabsb %zmm27, %zmm17
+
+// CHECK: vpabsb %zmm27, %zmm17 {%k7}
+// CHECK:  encoding: [0x62,0x82,0x7d,0x4f,0x1c,0xcb]
+          vpabsb %zmm27, %zmm17 {%k7}
+
+// CHECK: vpabsb %zmm27, %zmm17 {%k7} {z}
+// CHECK:  encoding: [0x62,0x82,0x7d,0xcf,0x1c,0xcb]
+          vpabsb %zmm27, %zmm17 {%k7} {z}
+
+// CHECK: vpabsb (%rcx), %zmm17
+// CHECK:  encoding: [0x62,0xe2,0x7d,0x48,0x1c,0x09]
+          vpabsb (%rcx), %zmm17
+
+// CHECK: vpabsb 291(%rax,%r14,8), %zmm17
+// CHECK:  encoding: [0x62,0xa2,0x7d,0x48,0x1c,0x8c,0xf0,0x23,0x01,0x00,0x00]
+          vpabsb 291(%rax,%r14,8), %zmm17
+
+// CHECK: vpabsb 8128(%rdx), %zmm17
+// CHECK:  encoding: [0x62,0xe2,0x7d,0x48,0x1c,0x4a,0x7f]
+          vpabsb 8128(%rdx), %zmm17
+
+// CHECK: vpabsb 8192(%rdx), %zmm17
+// CHECK:  encoding: [0x62,0xe2,0x7d,0x48,0x1c,0x8a,0x00,0x20,0x00,0x00]
+          vpabsb 8192(%rdx), %zmm17
+
+// CHECK: vpabsb -8192(%rdx), %zmm17
+// CHECK:  encoding: [0x62,0xe2,0x7d,0x48,0x1c,0x4a,0x80]
+          vpabsb -8192(%rdx), %zmm17
+
+// CHECK: vpabsb -8256(%rdx), %zmm17
+// CHECK:  encoding: [0x62,0xe2,0x7d,0x48,0x1c,0x8a,0xc0,0xdf,0xff,0xff]
+          vpabsb -8256(%rdx), %zmm17
+
+// CHECK: vpabsw %zmm24, %zmm30
+// CHECK:  encoding: [0x62,0x02,0x7d,0x48,0x1d,0xf0]
+          vpabsw %zmm24, %zmm30
+
+// CHECK: vpabsw %zmm24, %zmm30 {%k6}
+// CHECK:  encoding: [0x62,0x02,0x7d,0x4e,0x1d,0xf0]
+          vpabsw %zmm24, %zmm30 {%k6}
+
+// CHECK: vpabsw %zmm24, %zmm30 {%k6} {z}
+// CHECK:  encoding: [0x62,0x02,0x7d,0xce,0x1d,0xf0]
+          vpabsw %zmm24, %zmm30 {%k6} {z}
+
+// CHECK: vpabsw (%rcx), %zmm30
+// CHECK:  encoding: [0x62,0x62,0x7d,0x48,0x1d,0x31]
+          vpabsw (%rcx), %zmm30
+
+// CHECK: vpabsw 291(%rax,%r14,8), %zmm30
+// CHECK:  encoding: [0x62,0x22,0x7d,0x48,0x1d,0xb4,0xf0,0x23,0x01,0x00,0x00]
+          vpabsw 291(%rax,%r14,8), %zmm30
+
+// CHECK: vpabsw 8128(%rdx), %zmm30
+// CHECK:  encoding: [0x62,0x62,0x7d,0x48,0x1d,0x72,0x7f]
+          vpabsw 8128(%rdx), %zmm30
+
+// CHECK: vpabsw 8192(%rdx), %zmm30
+// CHECK:  encoding: [0x62,0x62,0x7d,0x48,0x1d,0xb2,0x00,0x20,0x00,0x00]
+          vpabsw 8192(%rdx), %zmm30
+
+// CHECK: vpabsw -8192(%rdx), %zmm30
+// CHECK:  encoding: [0x62,0x62,0x7d,0x48,0x1d,0x72,0x80]
+          vpabsw -8192(%rdx), %zmm30
+
+// CHECK: vpabsw -8256(%rdx), %zmm30
+// CHECK:  encoding: [0x62,0x62,0x7d,0x48,0x1d,0xb2,0xc0,0xdf,0xff,0xff]
+          vpabsw -8256(%rdx), %zmm30
+
+// CHECK: vpmovwb %zmm27, %ymm22
+// CHECK:  encoding: [0x62,0x22,0x7e,0x48,0x30,0xde]
+          vpmovwb %zmm27, %ymm22
+
+// CHECK: vpmovwb %zmm27, %ymm22 {%k1}
+// CHECK:  encoding: [0x62,0x22,0x7e,0x49,0x30,0xde]
+          vpmovwb %zmm27, %ymm22 {%k1}
+
+// CHECK: vpmovwb %zmm27, %ymm22 {%k1} {z}
+// CHECK:  encoding: [0x62,0x22,0x7e,0xc9,0x30,0xde]
+          vpmovwb %zmm27, %ymm22 {%k1} {z}
+
+// CHECK: vpmovwb %zmm22, (%rcx)
+// CHECK:  encoding: [0x62,0xe2,0x7e,0x48,0x30,0x31]
+          vpmovwb %zmm22, (%rcx)
+
+// CHECK: vpmovwb %zmm22, (%rcx) {%k4}
+// CHECK:  encoding: [0x62,0xe2,0x7e,0x4c,0x30,0x31]
+          vpmovwb %zmm22, (%rcx) {%k4}
+
+// CHECK: vpmovwb %zmm22, 291(%rax,%r14,8)
+// CHECK:  encoding: [0x62,0xa2,0x7e,0x48,0x30,0xb4,0xf0,0x23,0x01,0x00,0x00]
+          vpmovwb %zmm22, 291(%rax,%r14,8)
+
+// CHECK: vpmovwb %zmm22, 4064(%rdx)
+// CHECK:  encoding: [0x62,0xe2,0x7e,0x48,0x30,0x72,0x7f]
+          vpmovwb %zmm22, 4064(%rdx)
+
+// CHECK: vpmovwb %zmm22, 4096(%rdx)
+// CHECK:  encoding: [0x62,0xe2,0x7e,0x48,0x30,0xb2,0x00,0x10,0x00,0x00]
+          vpmovwb %zmm22, 4096(%rdx)
+
+// CHECK: vpmovwb %zmm22, -4096(%rdx)
+// CHECK:  encoding: [0x62,0xe2,0x7e,0x48,0x30,0x72,0x80]
+          vpmovwb %zmm22, -4096(%rdx)
+
+// CHECK: vpmovwb %zmm22, -4128(%rdx)
+// CHECK:  encoding: [0x62,0xe2,0x7e,0x48,0x30,0xb2,0xe0,0xef,0xff,0xff]
+          vpmovwb %zmm22, -4128(%rdx)
+
+// CHECK: vpmovswb %zmm18, %ymm23
+// CHECK:  encoding: [0x62,0xa2,0x7e,0x48,0x20,0xd7]
+          vpmovswb %zmm18, %ymm23
+
+// CHECK: vpmovswb %zmm18, %ymm23 {%k2}
+// CHECK:  encoding: [0x62,0xa2,0x7e,0x4a,0x20,0xd7]
+          vpmovswb %zmm18, %ymm23 {%k2}
+
+// CHECK: vpmovswb %zmm18, %ymm23 {%k2} {z}
+// CHECK:  encoding: [0x62,0xa2,0x7e,0xca,0x20,0xd7]
+          vpmovswb %zmm18, %ymm23 {%k2} {z}
+
+// CHECK: vpmovswb %zmm24, (%rcx)
+// CHECK:  encoding: [0x62,0x62,0x7e,0x48,0x20,0x01]
+          vpmovswb %zmm24, (%rcx)
+
+// CHECK: vpmovswb %zmm24, (%rcx) {%k7}
+// CHECK:  encoding: [0x62,0x62,0x7e,0x4f,0x20,0x01]
+          vpmovswb %zmm24, (%rcx) {%k7}
+
+// CHECK: vpmovswb %zmm24, 291(%rax,%r14,8)
+// CHECK:  encoding: [0x62,0x22,0x7e,0x48,0x20,0x84,0xf0,0x23,0x01,0x00,0x00]
+          vpmovswb %zmm24, 291(%rax,%r14,8)
+
+// CHECK: vpmovswb %zmm24, 4064(%rdx)
+// CHECK:  encoding: [0x62,0x62,0x7e,0x48,0x20,0x42,0x7f]
+          vpmovswb %zmm24, 4064(%rdx)
+
+// CHECK: vpmovswb %zmm24, 4096(%rdx)
+// CHECK:  encoding: [0x62,0x62,0x7e,0x48,0x20,0x82,0x00,0x10,0x00,0x00]
+          vpmovswb %zmm24, 4096(%rdx)
+
+// CHECK: vpmovswb %zmm24, -4096(%rdx)
+// CHECK:  encoding: [0x62,0x62,0x7e,0x48,0x20,0x42,0x80]
+          vpmovswb %zmm24, -4096(%rdx)
+
+// CHECK: vpmovswb %zmm24, -4128(%rdx)
+// CHECK:  encoding: [0x62,0x62,0x7e,0x48,0x20,0x82,0xe0,0xef,0xff,0xff]
+          vpmovswb %zmm24, -4128(%rdx)
+
+// CHECK: vpmovuswb %zmm22, %ymm28
+// CHECK:  encoding: [0x62,0x82,0x7e,0x48,0x10,0xf4]
+          vpmovuswb %zmm22, %ymm28
+
+// CHECK: vpmovuswb %zmm22, %ymm28 {%k3}
+// CHECK:  encoding: [0x62,0x82,0x7e,0x4b,0x10,0xf4]
+          vpmovuswb %zmm22, %ymm28 {%k3}
+
+// CHECK: vpmovuswb %zmm22, %ymm28 {%k3} {z}
+// CHECK:  encoding: [0x62,0x82,0x7e,0xcb,0x10,0xf4]
+          vpmovuswb %zmm22, %ymm28 {%k3} {z}
+
+// CHECK: vpmovuswb %zmm27, (%rcx)
+// CHECK:  encoding: [0x62,0x62,0x7e,0x48,0x10,0x19]
+          vpmovuswb %zmm27, (%rcx)
+
+// CHECK: vpmovuswb %zmm27, (%rcx) {%k2}
+// CHECK:  encoding: [0x62,0x62,0x7e,0x4a,0x10,0x19]
+          vpmovuswb %zmm27, (%rcx) {%k2}
+
+// CHECK: vpmovuswb %zmm27, 291(%rax,%r14,8)
+// CHECK:  encoding: [0x62,0x22,0x7e,0x48,0x10,0x9c,0xf0,0x23,0x01,0x00,0x00]
+          vpmovuswb %zmm27, 291(%rax,%r14,8)
+
+// CHECK: vpmovuswb %zmm27, 4064(%rdx)
+// CHECK:  encoding: [0x62,0x62,0x7e,0x48,0x10,0x5a,0x7f]
+          vpmovuswb %zmm27, 4064(%rdx)
+
+// CHECK: vpmovuswb %zmm27, 4096(%rdx)
+// CHECK:  encoding: [0x62,0x62,0x7e,0x48,0x10,0x9a,0x00,0x10,0x00,0x00]
+          vpmovuswb %zmm27, 4096(%rdx)
+
+// CHECK: vpmovuswb %zmm27, -4096(%rdx)
+// CHECK:  encoding: [0x62,0x62,0x7e,0x48,0x10,0x5a,0x80]
+          vpmovuswb %zmm27, -4096(%rdx)
+
+// CHECK: vpmovuswb %zmm27, -4128(%rdx)
+// CHECK:  encoding: [0x62,0x62,0x7e,0x48,0x10,0x9a,0xe0,0xef,0xff,0xff]
+          vpmovuswb %zmm27, -4128(%rdx)
+
+// CHECK: vpmulhuw %zmm21, %zmm24, %zmm21
+// CHECK:  encoding: [0x62,0xa1,0x3d,0x40,0xe4,0xed]
+          vpmulhuw %zmm21, %zmm24, %zmm21
+
+// CHECK: vpmulhuw %zmm21, %zmm24, %zmm21 {%k3}
+// CHECK:  encoding: [0x62,0xa1,0x3d,0x43,0xe4,0xed]
+          vpmulhuw %zmm21, %zmm24, %zmm21 {%k3}
+
+// CHECK: vpmulhuw %zmm21, %zmm24, %zmm21 {%k3} {z}
+// CHECK:  encoding: [0x62,0xa1,0x3d,0xc3,0xe4,0xed]
+          vpmulhuw %zmm21, %zmm24, %zmm21 {%k3} {z}
+
+// CHECK: vpmulhuw (%rcx), %zmm24, %zmm21
+// CHECK:  encoding: [0x62,0xe1,0x3d,0x40,0xe4,0x29]
+          vpmulhuw (%rcx), %zmm24, %zmm21
+
+// CHECK: vpmulhuw 291(%rax,%r14,8), %zmm24, %zmm21
+// CHECK:  encoding: [0x62,0xa1,0x3d,0x40,0xe4,0xac,0xf0,0x23,0x01,0x00,0x00]
+          vpmulhuw 291(%rax,%r14,8), %zmm24, %zmm21
+
+// CHECK: vpmulhuw 8128(%rdx), %zmm24, %zmm21
+// CHECK:  encoding: [0x62,0xe1,0x3d,0x40,0xe4,0x6a,0x7f]
+          vpmulhuw 8128(%rdx), %zmm24, %zmm21
+
+// CHECK: vpmulhuw 8192(%rdx), %zmm24, %zmm21
+// CHECK:  encoding: [0x62,0xe1,0x3d,0x40,0xe4,0xaa,0x00,0x20,0x00,0x00]
+          vpmulhuw 8192(%rdx), %zmm24, %zmm21
+
+// CHECK: vpmulhuw -8192(%rdx), %zmm24, %zmm21
+// CHECK:  encoding: [0x62,0xe1,0x3d,0x40,0xe4,0x6a,0x80]
+          vpmulhuw -8192(%rdx), %zmm24, %zmm21
+
+// CHECK: vpmulhuw -8256(%rdx), %zmm24, %zmm21
+// CHECK:  encoding: [0x62,0xe1,0x3d,0x40,0xe4,0xaa,0xc0,0xdf,0xff,0xff]
+          vpmulhuw -8256(%rdx), %zmm24, %zmm21
+
+// CHECK: vpmulhw %zmm27, %zmm26, %zmm30
+// CHECK:  encoding: [0x62,0x01,0x2d,0x40,0xe5,0xf3]
+          vpmulhw %zmm27, %zmm26, %zmm30
+
+// CHECK: vpmulhw %zmm27, %zmm26, %zmm30 {%k6}
+// CHECK:  encoding: [0x62,0x01,0x2d,0x46,0xe5,0xf3]
+          vpmulhw %zmm27, %zmm26, %zmm30 {%k6}
+
+// CHECK: vpmulhw %zmm27, %zmm26, %zmm30 {%k6} {z}
+// CHECK:  encoding: [0x62,0x01,0x2d,0xc6,0xe5,0xf3]
+          vpmulhw %zmm27, %zmm26, %zmm30 {%k6} {z}
+
+// CHECK: vpmulhw (%rcx), %zmm26, %zmm30
+// CHECK:  encoding: [0x62,0x61,0x2d,0x40,0xe5,0x31]
+          vpmulhw (%rcx), %zmm26, %zmm30
+
+// CHECK: vpmulhw 291(%rax,%r14,8), %zmm26, %zmm30
+// CHECK:  encoding: [0x62,0x21,0x2d,0x40,0xe5,0xb4,0xf0,0x23,0x01,0x00,0x00]
+          vpmulhw 291(%rax,%r14,8), %zmm26, %zmm30
+
+// CHECK: vpmulhw 8128(%rdx), %zmm26, %zmm30
+// CHECK:  encoding: [0x62,0x61,0x2d,0x40,0xe5,0x72,0x7f]
+          vpmulhw 8128(%rdx), %zmm26, %zmm30
+
+// CHECK: vpmulhw 8192(%rdx), %zmm26, %zmm30
+// CHECK:  encoding: [0x62,0x61,0x2d,0x40,0xe5,0xb2,0x00,0x20,0x00,0x00]
+          vpmulhw 8192(%rdx), %zmm26, %zmm30
+
+// CHECK: vpmulhw -8192(%rdx), %zmm26, %zmm30
+// CHECK:  encoding: [0x62,0x61,0x2d,0x40,0xe5,0x72,0x80]
+          vpmulhw -8192(%rdx), %zmm26, %zmm30
+
+// CHECK: vpmulhw -8256(%rdx), %zmm26, %zmm30
+// CHECK:  encoding: [0x62,0x61,0x2d,0x40,0xe5,0xb2,0xc0,0xdf,0xff,0xff]
+          vpmulhw -8256(%rdx), %zmm26, %zmm30
+
+// CHECK: vpmulhrsw %zmm25, %zmm27, %zmm21
+// CHECK:  encoding: [0x62,0x82,0x25,0x40,0x0b,0xe9]
+          vpmulhrsw %zmm25, %zmm27, %zmm21
+
+// CHECK: vpmulhrsw %zmm25, %zmm27, %zmm21 {%k7}
+// CHECK:  encoding: [0x62,0x82,0x25,0x47,0x0b,0xe9]
+          vpmulhrsw %zmm25, %zmm27, %zmm21 {%k7}
+
+// CHECK: vpmulhrsw %zmm25, %zmm27, %zmm21 {%k7} {z}
+// CHECK:  encoding: [0x62,0x82,0x25,0xc7,0x0b,0xe9]
+          vpmulhrsw %zmm25, %zmm27, %zmm21 {%k7} {z}
+
+// CHECK: vpmulhrsw (%rcx), %zmm27, %zmm21
+// CHECK:  encoding: [0x62,0xe2,0x25,0x40,0x0b,0x29]
+          vpmulhrsw (%rcx), %zmm27, %zmm21
+
+// CHECK: vpmulhrsw 291(%rax,%r14,8), %zmm27, %zmm21
+// CHECK:  encoding: [0x62,0xa2,0x25,0x40,0x0b,0xac,0xf0,0x23,0x01,0x00,0x00]
+          vpmulhrsw 291(%rax,%r14,8), %zmm27, %zmm21
+
+// CHECK: vpmulhrsw 8128(%rdx), %zmm27, %zmm21
+// CHECK:  encoding: [0x62,0xe2,0x25,0x40,0x0b,0x6a,0x7f]
+          vpmulhrsw 8128(%rdx), %zmm27, %zmm21
+
+// CHECK: vpmulhrsw 8192(%rdx), %zmm27, %zmm21
+// CHECK:  encoding: [0x62,0xe2,0x25,0x40,0x0b,0xaa,0x00,0x20,0x00,0x00]
+          vpmulhrsw 8192(%rdx), %zmm27, %zmm21
+
+// CHECK: vpmulhrsw -8192(%rdx), %zmm27, %zmm21
+// CHECK:  encoding: [0x62,0xe2,0x25,0x40,0x0b,0x6a,0x80]
+          vpmulhrsw -8192(%rdx), %zmm27, %zmm21
+
+// CHECK: vpmulhrsw -8256(%rdx), %zmm27, %zmm21
+// CHECK:  encoding: [0x62,0xe2,0x25,0x40,0x0b,0xaa,0xc0,0xdf,0xff,0xff]
+          vpmulhrsw -8256(%rdx), %zmm27, %zmm21
+
+// CHECK: vpmaddubsw %zmm25, %zmm20, %zmm27
+// CHECK:  encoding: [0x62,0x02,0x5d,0x40,0x04,0xd9]
+          vpmaddubsw %zmm25, %zmm20, %zmm27
+
+// CHECK: vpmaddubsw %zmm25, %zmm20, %zmm27 {%k3}
+// CHECK:  encoding: [0x62,0x02,0x5d,0x43,0x04,0xd9]
+          vpmaddubsw %zmm25, %zmm20, %zmm27 {%k3}
+
+// CHECK: vpmaddubsw %zmm25, %zmm20, %zmm27 {%k3} {z}
+// CHECK:  encoding: [0x62,0x02,0x5d,0xc3,0x04,0xd9]
+          vpmaddubsw %zmm25, %zmm20, %zmm27 {%k3} {z}
+
+// CHECK: vpmaddubsw (%rcx), %zmm20, %zmm27
+// CHECK:  encoding: [0x62,0x62,0x5d,0x40,0x04,0x19]
+          vpmaddubsw (%rcx), %zmm20, %zmm27
+
+// CHECK: vpmaddubsw 291(%rax,%r14,8), %zmm20, %zmm27
+// CHECK:  encoding: [0x62,0x22,0x5d,0x40,0x04,0x9c,0xf0,0x23,0x01,0x00,0x00]
+          vpmaddubsw 291(%rax,%r14,8), %zmm20, %zmm27
+
+// CHECK: vpmaddubsw 8128(%rdx), %zmm20, %zmm27
+// CHECK:  encoding: [0x62,0x62,0x5d,0x40,0x04,0x5a,0x7f]
+          vpmaddubsw 8128(%rdx), %zmm20, %zmm27
+
+// CHECK: vpmaddubsw 8192(%rdx), %zmm20, %zmm27
+// CHECK:  encoding: [0x62,0x62,0x5d,0x40,0x04,0x9a,0x00,0x20,0x00,0x00]
+          vpmaddubsw 8192(%rdx), %zmm20, %zmm27
+
+// CHECK: vpmaddubsw -8192(%rdx), %zmm20, %zmm27
+// CHECK:  encoding: [0x62,0x62,0x5d,0x40,0x04,0x5a,0x80]
+          vpmaddubsw -8192(%rdx), %zmm20, %zmm27
+
+// CHECK: vpmaddubsw -8256(%rdx), %zmm20, %zmm27
+// CHECK:  encoding: [0x62,0x62,0x5d,0x40,0x04,0x9a,0xc0,0xdf,0xff,0xff]
+          vpmaddubsw -8256(%rdx), %zmm20, %zmm27
+
+// CHECK: vpmaddwd %zmm25, %zmm22, %zmm26
+// CHECK:  encoding: [0x62,0x01,0x4d,0x40,0xf5,0xd1]
+          vpmaddwd %zmm25, %zmm22, %zmm26
+
+// CHECK: vpmaddwd %zmm25, %zmm22, %zmm26 {%k2}
+// CHECK:  encoding: [0x62,0x01,0x4d,0x42,0xf5,0xd1]
+          vpmaddwd %zmm25, %zmm22, %zmm26 {%k2}
+
+// CHECK: vpmaddwd %zmm25, %zmm22, %zmm26 {%k2} {z}
+// CHECK:  encoding: [0x62,0x01,0x4d,0xc2,0xf5,0xd1]
+          vpmaddwd %zmm25, %zmm22, %zmm26 {%k2} {z}
+
+// CHECK: vpmaddwd (%rcx), %zmm22, %zmm26
+// CHECK:  encoding: [0x62,0x61,0x4d,0x40,0xf5,0x11]
+          vpmaddwd (%rcx), %zmm22, %zmm26
+
+// CHECK: vpmaddwd 291(%rax,%r14,8), %zmm22, %zmm26
+// CHECK:  encoding: [0x62,0x21,0x4d,0x40,0xf5,0x94,0xf0,0x23,0x01,0x00,0x00]
+          vpmaddwd 291(%rax,%r14,8), %zmm22, %zmm26
+
+// CHECK: vpmaddwd 8128(%rdx), %zmm22, %zmm26
+// CHECK:  encoding: [0x62,0x61,0x4d,0x40,0xf5,0x52,0x7f]
+          vpmaddwd 8128(%rdx), %zmm22, %zmm26
+
+// CHECK: vpmaddwd 8192(%rdx), %zmm22, %zmm26
+// CHECK:  encoding: [0x62,0x61,0x4d,0x40,0xf5,0x92,0x00,0x20,0x00,0x00]
+          vpmaddwd 8192(%rdx), %zmm22, %zmm26
+
+// CHECK: vpmaddwd -8192(%rdx), %zmm22, %zmm26
+// CHECK:  encoding: [0x62,0x61,0x4d,0x40,0xf5,0x52,0x80]
+          vpmaddwd -8192(%rdx), %zmm22, %zmm26
+
+// CHECK: vpmaddwd -8256(%rdx), %zmm22, %zmm26
+// CHECK:  encoding: [0x62,0x61,0x4d,0x40,0xf5,0x92,0xc0,0xdf,0xff,0xff]
+          vpmaddwd -8256(%rdx), %zmm22, %zmm26
+
+// CHECK: vpunpcklbw %zmm24, %zmm17, %zmm28
+// CHECK:  encoding: [0x62,0x01,0x75,0x40,0x60,0xe0]
+          vpunpcklbw %zmm24, %zmm17, %zmm28
+
+// CHECK: vpunpcklbw %zmm24, %zmm17, %zmm28 {%k1}
+// CHECK:  encoding: [0x62,0x01,0x75,0x41,0x60,0xe0]
+          vpunpcklbw %zmm24, %zmm17, %zmm28 {%k1}
+
+// CHECK: vpunpcklbw %zmm24, %zmm17, %zmm28 {%k1} {z}
+// CHECK:  encoding: [0x62,0x01,0x75,0xc1,0x60,0xe0]
+          vpunpcklbw %zmm24, %zmm17, %zmm28 {%k1} {z}
+
+// CHECK: vpunpcklbw (%rcx), %zmm17, %zmm28
+// CHECK:  encoding: [0x62,0x61,0x75,0x40,0x60,0x21]
+          vpunpcklbw (%rcx), %zmm17, %zmm28
+
+// CHECK: vpunpcklbw 291(%rax,%r14,8), %zmm17, %zmm28
+// CHECK:  encoding: [0x62,0x21,0x75,0x40,0x60,0xa4,0xf0,0x23,0x01,0x00,0x00]
+          vpunpcklbw 291(%rax,%r14,8), %zmm17, %zmm28
+
+// CHECK: vpunpcklbw 8128(%rdx), %zmm17, %zmm28
+// CHECK:  encoding: [0x62,0x61,0x75,0x40,0x60,0x62,0x7f]
+          vpunpcklbw 8128(%rdx), %zmm17, %zmm28
+
+// CHECK: vpunpcklbw 8192(%rdx), %zmm17, %zmm28
+// CHECK:  encoding: [0x62,0x61,0x75,0x40,0x60,0xa2,0x00,0x20,0x00,0x00]
+          vpunpcklbw 8192(%rdx), %zmm17, %zmm28
+
+// CHECK: vpunpcklbw -8192(%rdx), %zmm17, %zmm28
+// CHECK:  encoding: [0x62,0x61,0x75,0x40,0x60,0x62,0x80]
+          vpunpcklbw -8192(%rdx), %zmm17, %zmm28
+
+// CHECK: vpunpcklbw -8256(%rdx), %zmm17, %zmm28
+// CHECK:  encoding: [0x62,0x61,0x75,0x40,0x60,0xa2,0xc0,0xdf,0xff,0xff]
+          vpunpcklbw -8256(%rdx), %zmm17, %zmm28
+
+// CHECK: vpunpckhbw %zmm23, %zmm19, %zmm30
+// CHECK:  encoding: [0x62,0x21,0x65,0x40,0x68,0xf7]
+          vpunpckhbw %zmm23, %zmm19, %zmm30
+
+// CHECK: vpunpckhbw %zmm23, %zmm19, %zmm30 {%k7}
+// CHECK:  encoding: [0x62,0x21,0x65,0x47,0x68,0xf7]
+          vpunpckhbw %zmm23, %zmm19, %zmm30 {%k7}
+
+// CHECK: vpunpckhbw %zmm23, %zmm19, %zmm30 {%k7} {z}
+// CHECK:  encoding: [0x62,0x21,0x65,0xc7,0x68,0xf7]
+          vpunpckhbw %zmm23, %zmm19, %zmm30 {%k7} {z}
+
+// CHECK: vpunpckhbw (%rcx), %zmm19, %zmm30
+// CHECK:  encoding: [0x62,0x61,0x65,0x40,0x68,0x31]
+          vpunpckhbw (%rcx), %zmm19, %zmm30
+
+// CHECK: vpunpckhbw 291(%rax,%r14,8), %zmm19, %zmm30
+// CHECK:  encoding: [0x62,0x21,0x65,0x40,0x68,0xb4,0xf0,0x23,0x01,0x00,0x00]
+          vpunpckhbw 291(%rax,%r14,8), %zmm19, %zmm30
+
+// CHECK: vpunpckhbw 8128(%rdx), %zmm19, %zmm30
+// CHECK:  encoding: [0x62,0x61,0x65,0x40,0x68,0x72,0x7f]
+          vpunpckhbw 8128(%rdx), %zmm19, %zmm30
+
+// CHECK: vpunpckhbw 8192(%rdx), %zmm19, %zmm30
+// CHECK:  encoding: [0x62,0x61,0x65,0x40,0x68,0xb2,0x00,0x20,0x00,0x00]
+          vpunpckhbw 8192(%rdx), %zmm19, %zmm30
+
+// CHECK: vpunpckhbw -8192(%rdx), %zmm19, %zmm30
+// CHECK:  encoding: [0x62,0x61,0x65,0x40,0x68,0x72,0x80]
+          vpunpckhbw -8192(%rdx), %zmm19, %zmm30
+
+// CHECK: vpunpckhbw -8256(%rdx), %zmm19, %zmm30
+// CHECK:  encoding: [0x62,0x61,0x65,0x40,0x68,0xb2,0xc0,0xdf,0xff,0xff]
+          vpunpckhbw -8256(%rdx), %zmm19, %zmm30
+
+// CHECK: vpunpcklwd %zmm18, %zmm24, %zmm20
+// CHECK:  encoding: [0x62,0xa1,0x3d,0x40,0x61,0xe2]
+          vpunpcklwd %zmm18, %zmm24, %zmm20
+
+// CHECK: vpunpcklwd %zmm18, %zmm24, %zmm20 {%k7}
+// CHECK:  encoding: [0x62,0xa1,0x3d,0x47,0x61,0xe2]
+          vpunpcklwd %zmm18, %zmm24, %zmm20 {%k7}
+
+// CHECK: vpunpcklwd %zmm18, %zmm24, %zmm20 {%k7} {z}
+// CHECK:  encoding: [0x62,0xa1,0x3d,0xc7,0x61,0xe2]
+          vpunpcklwd %zmm18, %zmm24, %zmm20 {%k7} {z}
+
+// CHECK: vpunpcklwd (%rcx), %zmm24, %zmm20
+// CHECK:  encoding: [0x62,0xe1,0x3d,0x40,0x61,0x21]
+          vpunpcklwd (%rcx), %zmm24, %zmm20
+
+// CHECK: vpunpcklwd 291(%rax,%r14,8), %zmm24, %zmm20
+// CHECK:  encoding: [0x62,0xa1,0x3d,0x40,0x61,0xa4,0xf0,0x23,0x01,0x00,0x00]
+          vpunpcklwd 291(%rax,%r14,8), %zmm24, %zmm20
+
+// CHECK: vpunpcklwd 8128(%rdx), %zmm24, %zmm20
+// CHECK:  encoding: [0x62,0xe1,0x3d,0x40,0x61,0x62,0x7f]
+          vpunpcklwd 8128(%rdx), %zmm24, %zmm20
+
+// CHECK: vpunpcklwd 8192(%rdx), %zmm24, %zmm20
+// CHECK:  encoding: [0x62,0xe1,0x3d,0x40,0x61,0xa2,0x00,0x20,0x00,0x00]
+          vpunpcklwd 8192(%rdx), %zmm24, %zmm20
+
+// CHECK: vpunpcklwd -8192(%rdx), %zmm24, %zmm20
+// CHECK:  encoding: [0x62,0xe1,0x3d,0x40,0x61,0x62,0x80]
+          vpunpcklwd -8192(%rdx), %zmm24, %zmm20
+
+// CHECK: vpunpcklwd -8256(%rdx), %zmm24, %zmm20
+// CHECK:  encoding: [0x62,0xe1,0x3d,0x40,0x61,0xa2,0xc0,0xdf,0xff,0xff]
+          vpunpcklwd -8256(%rdx), %zmm24, %zmm20
+
+// CHECK: vpunpckhwd %zmm24, %zmm26, %zmm30
+// CHECK:  encoding: [0x62,0x01,0x2d,0x40,0x69,0xf0]
+          vpunpckhwd %zmm24, %zmm26, %zmm30
+
+// CHECK: vpunpckhwd %zmm24, %zmm26, %zmm30 {%k4}
+// CHECK:  encoding: [0x62,0x01,0x2d,0x44,0x69,0xf0]
+          vpunpckhwd %zmm24, %zmm26, %zmm30 {%k4}
+
+// CHECK: vpunpckhwd %zmm24, %zmm26, %zmm30 {%k4} {z}
+// CHECK:  encoding: [0x62,0x01,0x2d,0xc4,0x69,0xf0]
+          vpunpckhwd %zmm24, %zmm26, %zmm30 {%k4} {z}
+
+// CHECK: vpunpckhwd (%rcx), %zmm26, %zmm30
+// CHECK:  encoding: [0x62,0x61,0x2d,0x40,0x69,0x31]
+          vpunpckhwd (%rcx), %zmm26, %zmm30
+
+// CHECK: vpunpckhwd 291(%rax,%r14,8), %zmm26, %zmm30
+// CHECK:  encoding: [0x62,0x21,0x2d,0x40,0x69,0xb4,0xf0,0x23,0x01,0x00,0x00]
+          vpunpckhwd 291(%rax,%r14,8), %zmm26, %zmm30
+
+// CHECK: vpunpckhwd 8128(%rdx), %zmm26, %zmm30
+// CHECK:  encoding: [0x62,0x61,0x2d,0x40,0x69,0x72,0x7f]
+          vpunpckhwd 8128(%rdx), %zmm26, %zmm30
+
+// CHECK: vpunpckhwd 8192(%rdx), %zmm26, %zmm30
+// CHECK:  encoding: [0x62,0x61,0x2d,0x40,0x69,0xb2,0x00,0x20,0x00,0x00]
+          vpunpckhwd 8192(%rdx), %zmm26, %zmm30
+
+// CHECK: vpunpckhwd -8192(%rdx), %zmm26, %zmm30
+// CHECK:  encoding: [0x62,0x61,0x2d,0x40,0x69,0x72,0x80]
+          vpunpckhwd -8192(%rdx), %zmm26, %zmm30
+
+// CHECK: vpunpckhwd -8256(%rdx), %zmm26, %zmm30
+// CHECK:  encoding: [0x62,0x61,0x2d,0x40,0x69,0xb2,0xc0,0xdf,0xff,0xff]
+          vpunpckhwd -8256(%rdx), %zmm26, %zmm30
+
diff --git a/test/MC/X86/x86-64-avx512bw_vl.s b/test/MC/X86/x86-64-avx512bw_vl.s
index 3f455827166..d617a31561d 100644
--- a/test/MC/X86/x86-64-avx512bw_vl.s
+++ b/test/MC/X86/x86-64-avx512bw_vl.s
@@ -1312,6 +1312,150 @@
 // CHECK:  encoding: [0x62,0xe2,0x25,0x20,0x3a,0xa2,0xe0,0xef,0xff,0xff]
           vpminuw -4128(%rdx), %ymm27, %ymm20
 
+// CHECK: vpmovsxbw %xmm23, %xmm27
+// CHECK:  encoding: [0x62,0x22,0x7d,0x08,0x20,0xdf]
+          vpmovsxbw %xmm23, %xmm27
+
+// CHECK: vpmovsxbw %xmm23, %xmm27 {%k7}
+// CHECK:  encoding: [0x62,0x22,0x7d,0x0f,0x20,0xdf]
+          vpmovsxbw %xmm23, %xmm27 {%k7}
+
+// CHECK: vpmovsxbw %xmm23, %xmm27 {%k7} {z}
+// CHECK:  encoding: [0x62,0x22,0x7d,0x8f,0x20,0xdf]
+          vpmovsxbw %xmm23, %xmm27 {%k7} {z}
+
+// CHECK: vpmovsxbw (%rcx), %xmm27
+// CHECK:  encoding: [0x62,0x62,0x7d,0x08,0x20,0x19]
+          vpmovsxbw (%rcx), %xmm27
+
+// CHECK: vpmovsxbw 291(%rax,%r14,8), %xmm27
+// CHECK:  encoding: [0x62,0x22,0x7d,0x08,0x20,0x9c,0xf0,0x23,0x01,0x00,0x00]
+          vpmovsxbw 291(%rax,%r14,8), %xmm27
+
+// CHECK: vpmovsxbw 1016(%rdx), %xmm27
+// CHECK:  encoding: [0x62,0x62,0x7d,0x08,0x20,0x5a,0x7f]
+          vpmovsxbw 1016(%rdx), %xmm27
+
+// CHECK: vpmovsxbw 1024(%rdx), %xmm27
+// CHECK:  encoding: [0x62,0x62,0x7d,0x08,0x20,0x9a,0x00,0x04,0x00,0x00]
+          vpmovsxbw 1024(%rdx), %xmm27
+
+// CHECK: vpmovsxbw -1024(%rdx), %xmm27
+// CHECK:  encoding: [0x62,0x62,0x7d,0x08,0x20,0x5a,0x80]
+          vpmovsxbw -1024(%rdx), %xmm27
+
+// CHECK: vpmovsxbw -1032(%rdx), %xmm27
+// CHECK:  encoding: [0x62,0x62,0x7d,0x08,0x20,0x9a,0xf8,0xfb,0xff,0xff]
+          vpmovsxbw -1032(%rdx), %xmm27
+
+// CHECK: vpmovsxbw %xmm23, %ymm21
+// CHECK:  encoding: [0x62,0xa2,0x7d,0x28,0x20,0xef]
+          vpmovsxbw %xmm23, %ymm21
+
+// CHECK: vpmovsxbw %xmm23, %ymm21 {%k7}
+// CHECK:  encoding: [0x62,0xa2,0x7d,0x2f,0x20,0xef]
+          vpmovsxbw %xmm23, %ymm21 {%k7}
+
+// CHECK: vpmovsxbw %xmm23, %ymm21 {%k7} {z}
+// CHECK:  encoding: [0x62,0xa2,0x7d,0xaf,0x20,0xef]
+          vpmovsxbw %xmm23, %ymm21 {%k7} {z}
+
+// CHECK: vpmovsxbw (%rcx), %ymm21
+// CHECK:  encoding: [0x62,0xe2,0x7d,0x28,0x20,0x29]
+          vpmovsxbw (%rcx), %ymm21
+
+// CHECK: vpmovsxbw 291(%rax,%r14,8), %ymm21
+// CHECK:  encoding: [0x62,0xa2,0x7d,0x28,0x20,0xac,0xf0,0x23,0x01,0x00,0x00]
+          vpmovsxbw 291(%rax,%r14,8), %ymm21
+
+// CHECK: vpmovsxbw 2032(%rdx), %ymm21
+// CHECK:  encoding: [0x62,0xe2,0x7d,0x28,0x20,0x6a,0x7f]
+          vpmovsxbw 2032(%rdx), %ymm21
+
+// CHECK: vpmovsxbw 2048(%rdx), %ymm21
+// CHECK:  encoding: [0x62,0xe2,0x7d,0x28,0x20,0xaa,0x00,0x08,0x00,0x00]
+          vpmovsxbw 2048(%rdx), %ymm21
+
+// CHECK: vpmovsxbw -2048(%rdx), %ymm21
+// CHECK:  encoding: [0x62,0xe2,0x7d,0x28,0x20,0x6a,0x80]
+          vpmovsxbw -2048(%rdx), %ymm21
+
+// CHECK: vpmovsxbw -2064(%rdx), %ymm21
+// CHECK:  encoding: [0x62,0xe2,0x7d,0x28,0x20,0xaa,0xf0,0xf7,0xff,0xff]
+          vpmovsxbw -2064(%rdx), %ymm21
+
+// CHECK: vpmovzxbw %xmm29, %xmm30
+// CHECK:  encoding: [0x62,0x02,0x7d,0x08,0x30,0xf5]
+          vpmovzxbw %xmm29, %xmm30
+
+// CHECK: vpmovzxbw %xmm29, %xmm30 {%k7}
+// CHECK:  encoding: [0x62,0x02,0x7d,0x0f,0x30,0xf5]
+          vpmovzxbw %xmm29, %xmm30 {%k7}
+
+// CHECK: vpmovzxbw %xmm29, %xmm30 {%k7} {z}
+// CHECK:  encoding: [0x62,0x02,0x7d,0x8f,0x30,0xf5]
+          vpmovzxbw %xmm29, %xmm30 {%k7} {z}
+
+// CHECK: vpmovzxbw (%rcx), %xmm30
+// CHECK:  encoding: [0x62,0x62,0x7d,0x08,0x30,0x31]
+          vpmovzxbw (%rcx), %xmm30
+
+// CHECK: vpmovzxbw 291(%rax,%r14,8), %xmm30
+// CHECK:  encoding: [0x62,0x22,0x7d,0x08,0x30,0xb4,0xf0,0x23,0x01,0x00,0x00]
+          vpmovzxbw 291(%rax,%r14,8), %xmm30
+
+// CHECK: vpmovzxbw 1016(%rdx), %xmm30
+// CHECK:  encoding: [0x62,0x62,0x7d,0x08,0x30,0x72,0x7f]
+          vpmovzxbw 1016(%rdx), %xmm30
+
+// CHECK: vpmovzxbw 1024(%rdx), %xmm30
+// CHECK:  encoding: [0x62,0x62,0x7d,0x08,0x30,0xb2,0x00,0x04,0x00,0x00]
+          vpmovzxbw 1024(%rdx), %xmm30
+
+// CHECK: vpmovzxbw -1024(%rdx), %xmm30
+// CHECK:  encoding: [0x62,0x62,0x7d,0x08,0x30,0x72,0x80]
+          vpmovzxbw -1024(%rdx), %xmm30
+
+// CHECK: vpmovzxbw -1032(%rdx), %xmm30
+// CHECK:  encoding: [0x62,0x62,0x7d,0x08,0x30,0xb2,0xf8,0xfb,0xff,0xff]
+          vpmovzxbw -1032(%rdx), %xmm30
+
+// CHECK: vpmovzxbw %xmm29, %ymm22
+// CHECK:  encoding: [0x62,0x82,0x7d,0x28,0x30,0xf5]
+          vpmovzxbw %xmm29, %ymm22
+
+// CHECK: vpmovzxbw %xmm29, %ymm22 {%k2}
+// CHECK:  encoding: [0x62,0x82,0x7d,0x2a,0x30,0xf5]
+          vpmovzxbw %xmm29, %ymm22 {%k2}
+
+// CHECK: vpmovzxbw %xmm29, %ymm22 {%k2} {z}
+// CHECK:  encoding: [0x62,0x82,0x7d,0xaa,0x30,0xf5]
+          vpmovzxbw %xmm29, %ymm22 {%k2} {z}
+
+// CHECK: vpmovzxbw (%rcx), %ymm22
+// CHECK:  encoding: [0x62,0xe2,0x7d,0x28,0x30,0x31]
+          vpmovzxbw (%rcx), %ymm22
+
+// CHECK: vpmovzxbw 291(%rax,%r14,8), %ymm22
+// CHECK:  encoding: [0x62,0xa2,0x7d,0x28,0x30,0xb4,0xf0,0x23,0x01,0x00,0x00]
+          vpmovzxbw 291(%rax,%r14,8), %ymm22
+
+// CHECK: vpmovzxbw 2032(%rdx), %ymm22
+// CHECK:  encoding: [0x62,0xe2,0x7d,0x28,0x30,0x72,0x7f]
+          vpmovzxbw 2032(%rdx), %ymm22
+
+// CHECK: vpmovzxbw 2048(%rdx), %ymm22
+// CHECK:  encoding: [0x62,0xe2,0x7d,0x28,0x30,0xb2,0x00,0x08,0x00,0x00]
+          vpmovzxbw 2048(%rdx), %ymm22
+
+// CHECK: vpmovzxbw -2048(%rdx), %ymm22
+// CHECK:  encoding: [0x62,0xe2,0x7d,0x28,0x30,0x72,0x80]
+          vpmovzxbw -2048(%rdx), %ymm22
+
+// CHECK: vpmovzxbw -2064(%rdx), %ymm22
+// CHECK:  encoding: [0x62,0xe2,0x7d,0x28,0x30,0xb2,0xf0,0xf7,0xff,0xff]
+          vpmovzxbw -2064(%rdx), %ymm22
+
 // CHECK: vpmullw %xmm26, %xmm19, %xmm29
 // CHECK:  encoding: [0x62,0x01,0x65,0x00,0xd5,0xea]
           vpmullw %xmm26, %xmm19, %xmm29
@@ -2303,3 +2447,5955 @@
 // CHECK: vpsravw -4128(%rdx), %ymm28, %ymm20
 // CHECK:  encoding: [0x62,0xe2,0x9d,0x20,0x11,0xa2,0xe0,0xef,0xff,0xff]
           vpsravw -4128(%rdx), %ymm28, %ymm20
+
+// CHECK: vpacksswb %xmm20, %xmm29, %xmm17
+// CHECK:  encoding: [0x62,0xa1,0x95,0x00,0x63,0xcc]
+          vpacksswb %xmm20, %xmm29, %xmm17
+
+// CHECK: vpacksswb %xmm20, %xmm29, %xmm17 {%k3}
+// CHECK:  encoding: [0x62,0xa1,0x95,0x03,0x63,0xcc]
+          vpacksswb %xmm20, %xmm29, %xmm17 {%k3}
+
+// CHECK: vpacksswb %xmm20, %xmm29, %xmm17 {%k3} {z}
+// CHECK:  encoding: [0x62,0xa1,0x95,0x83,0x63,0xcc]
+          vpacksswb %xmm20, %xmm29, %xmm17 {%k3} {z}
+
+// CHECK: vpacksswb (%rcx), %xmm29, %xmm17
+// CHECK:  encoding: [0x62,0xe1,0x95,0x00,0x63,0x09]
+          vpacksswb (%rcx), %xmm29, %xmm17
+
+// CHECK: vpacksswb 291(%rax,%r14,8), %xmm29, %xmm17
+// CHECK:  encoding: [0x62,0xa1,0x95,0x00,0x63,0x8c,0xf0,0x23,0x01,0x00,0x00]
+          vpacksswb 291(%rax,%r14,8), %xmm29, %xmm17
+
+// CHECK: vpacksswb 2032(%rdx), %xmm29, %xmm17
+// CHECK:  encoding: [0x62,0xe1,0x95,0x00,0x63,0x4a,0x7f]
+          vpacksswb 2032(%rdx), %xmm29, %xmm17
+
+// CHECK: vpacksswb 2048(%rdx), %xmm29, %xmm17
+// CHECK:  encoding: [0x62,0xe1,0x95,0x00,0x63,0x8a,0x00,0x08,0x00,0x00]
+          vpacksswb 2048(%rdx), %xmm29, %xmm17
+
+// CHECK: vpacksswb -2048(%rdx), %xmm29, %xmm17
+// CHECK:  encoding: [0x62,0xe1,0x95,0x00,0x63,0x4a,0x80]
+          vpacksswb -2048(%rdx), %xmm29, %xmm17
+
+// CHECK: vpacksswb -2064(%rdx), %xmm29, %xmm17
+// CHECK:  encoding: [0x62,0xe1,0x95,0x00,0x63,0x8a,0xf0,0xf7,0xff,0xff]
+          vpacksswb -2064(%rdx), %xmm29, %xmm17
+
+// CHECK: vpacksswb %ymm21, %ymm26, %ymm29
+// CHECK:  encoding: [0x62,0x21,0xad,0x20,0x63,0xed]
+          vpacksswb %ymm21, %ymm26, %ymm29
+
+// CHECK: vpacksswb %ymm21, %ymm26, %ymm29 {%k2}
+// CHECK:  encoding: [0x62,0x21,0xad,0x22,0x63,0xed]
+          vpacksswb %ymm21, %ymm26, %ymm29 {%k2}
+
+// CHECK: vpacksswb %ymm21, %ymm26, %ymm29 {%k2} {z}
+// CHECK:  encoding: [0x62,0x21,0xad,0xa2,0x63,0xed]
+          vpacksswb %ymm21, %ymm26, %ymm29 {%k2} {z}
+
+// CHECK: vpacksswb (%rcx), %ymm26, %ymm29
+// CHECK:  encoding: [0x62,0x61,0xad,0x20,0x63,0x29]
+          vpacksswb (%rcx), %ymm26, %ymm29
+
+// CHECK: vpacksswb 291(%rax,%r14,8), %ymm26, %ymm29
+// CHECK:  encoding: [0x62,0x21,0xad,0x20,0x63,0xac,0xf0,0x23,0x01,0x00,0x00]
+          vpacksswb 291(%rax,%r14,8), %ymm26, %ymm29
+
+// CHECK: vpacksswb 4064(%rdx), %ymm26, %ymm29
+// CHECK:  encoding: [0x62,0x61,0xad,0x20,0x63,0x6a,0x7f]
+          vpacksswb 4064(%rdx), %ymm26, %ymm29
+
+// CHECK: vpacksswb 4096(%rdx), %ymm26, %ymm29
+// CHECK:  encoding: [0x62,0x61,0xad,0x20,0x63,0xaa,0x00,0x10,0x00,0x00]
+          vpacksswb 4096(%rdx), %ymm26, %ymm29
+
+// CHECK: vpacksswb -4096(%rdx), %ymm26, %ymm29
+// CHECK:  encoding: [0x62,0x61,0xad,0x20,0x63,0x6a,0x80]
+          vpacksswb -4096(%rdx), %ymm26, %ymm29
+
+// CHECK: vpacksswb -4128(%rdx), %ymm26, %ymm29
+// CHECK:  encoding: [0x62,0x61,0xad,0x20,0x63,0xaa,0xe0,0xef,0xff,0xff]
+          vpacksswb -4128(%rdx), %ymm26, %ymm29
+
+// CHECK: vpackuswb %xmm24, %xmm26, %xmm20
+// CHECK:  encoding: [0x62,0x81,0xad,0x00,0x67,0xe0]
+          vpackuswb %xmm24, %xmm26, %xmm20
+
+// CHECK: vpackuswb %xmm24, %xmm26, %xmm20 {%k4}
+// CHECK:  encoding: [0x62,0x81,0xad,0x04,0x67,0xe0]
+          vpackuswb %xmm24, %xmm26, %xmm20 {%k4}
+
+// CHECK: vpackuswb %xmm24, %xmm26, %xmm20 {%k4} {z}
+// CHECK:  encoding: [0x62,0x81,0xad,0x84,0x67,0xe0]
+          vpackuswb %xmm24, %xmm26, %xmm20 {%k4} {z}
+
+// CHECK: vpackuswb (%rcx), %xmm26, %xmm20
+// CHECK:  encoding: [0x62,0xe1,0xad,0x00,0x67,0x21]
+          vpackuswb (%rcx), %xmm26, %xmm20
+
+// CHECK: vpackuswb 291(%rax,%r14,8), %xmm26, %xmm20
+// CHECK:  encoding: [0x62,0xa1,0xad,0x00,0x67,0xa4,0xf0,0x23,0x01,0x00,0x00]
+          vpackuswb 291(%rax,%r14,8), %xmm26, %xmm20
+
+// CHECK: vpackuswb 2032(%rdx), %xmm26, %xmm20
+// CHECK:  encoding: [0x62,0xe1,0xad,0x00,0x67,0x62,0x7f]
+          vpackuswb 2032(%rdx), %xmm26, %xmm20
+
+// CHECK: vpackuswb 2048(%rdx), %xmm26, %xmm20
+// CHECK:  encoding: [0x62,0xe1,0xad,0x00,0x67,0xa2,0x00,0x08,0x00,0x00]
+          vpackuswb 2048(%rdx), %xmm26, %xmm20
+
+// CHECK: vpackuswb -2048(%rdx), %xmm26, %xmm20
+// CHECK:  encoding: [0x62,0xe1,0xad,0x00,0x67,0x62,0x80]
+          vpackuswb -2048(%rdx), %xmm26, %xmm20
+
+// CHECK: vpackuswb -2064(%rdx), %xmm26, %xmm20
+// CHECK:  encoding: [0x62,0xe1,0xad,0x00,0x67,0xa2,0xf0,0xf7,0xff,0xff]
+          vpackuswb -2064(%rdx), %xmm26, %xmm20
+
+// CHECK: vpackuswb %ymm20, %ymm28, %ymm19
+// CHECK:  encoding: [0x62,0xa1,0x9d,0x20,0x67,0xdc]
+          vpackuswb %ymm20, %ymm28, %ymm19
+
+// CHECK: vpackuswb %ymm20, %ymm28, %ymm19 {%k2}
+// CHECK:  encoding: [0x62,0xa1,0x9d,0x22,0x67,0xdc]
+          vpackuswb %ymm20, %ymm28, %ymm19 {%k2}
+
+// CHECK: vpackuswb %ymm20, %ymm28, %ymm19 {%k2} {z}
+// CHECK:  encoding: [0x62,0xa1,0x9d,0xa2,0x67,0xdc]
+          vpackuswb %ymm20, %ymm28, %ymm19 {%k2} {z}
+
+// CHECK: vpackuswb (%rcx), %ymm28, %ymm19
+// CHECK:  encoding: [0x62,0xe1,0x9d,0x20,0x67,0x19]
+          vpackuswb (%rcx), %ymm28, %ymm19
+
+// CHECK: vpackuswb 291(%rax,%r14,8), %ymm28, %ymm19
+// CHECK:  encoding: [0x62,0xa1,0x9d,0x20,0x67,0x9c,0xf0,0x23,0x01,0x00,0x00]
+          vpackuswb 291(%rax,%r14,8), %ymm28, %ymm19
+
+// CHECK: vpackuswb 4064(%rdx), %ymm28, %ymm19
+// CHECK:  encoding: [0x62,0xe1,0x9d,0x20,0x67,0x5a,0x7f]
+          vpackuswb 4064(%rdx), %ymm28, %ymm19
+
+// CHECK: vpackuswb 4096(%rdx), %ymm28, %ymm19
+// CHECK:  encoding: [0x62,0xe1,0x9d,0x20,0x67,0x9a,0x00,0x10,0x00,0x00]
+          vpackuswb 4096(%rdx), %ymm28, %ymm19
+
+// CHECK: vpackuswb -4096(%rdx), %ymm28, %ymm19
+// CHECK:  encoding: [0x62,0xe1,0x9d,0x20,0x67,0x5a,0x80]
+          vpackuswb -4096(%rdx), %ymm28, %ymm19
+
+// CHECK: vpackuswb -4128(%rdx), %ymm28, %ymm19
+// CHECK:  encoding: [0x62,0xe1,0x9d,0x20,0x67,0x9a,0xe0,0xef,0xff,0xff]
+          vpackuswb -4128(%rdx), %ymm28, %ymm19
+
+// CHECK: vpacksswb %xmm25, %xmm18, %xmm19
+// CHECK:  encoding: [0x62,0x81,0xed,0x00,0x63,0xd9]
+          vpacksswb %xmm25, %xmm18, %xmm19
+
+// CHECK: vpacksswb %xmm25, %xmm18, %xmm19 {%k4}
+// CHECK:  encoding: [0x62,0x81,0xed,0x04,0x63,0xd9]
+          vpacksswb %xmm25, %xmm18, %xmm19 {%k4}
+
+// CHECK: vpacksswb %xmm25, %xmm18, %xmm19 {%k4} {z}
+// CHECK:  encoding: [0x62,0x81,0xed,0x84,0x63,0xd9]
+          vpacksswb %xmm25, %xmm18, %xmm19 {%k4} {z}
+
+// CHECK: vpacksswb (%rcx), %xmm18, %xmm19
+// CHECK:  encoding: [0x62,0xe1,0xed,0x00,0x63,0x19]
+          vpacksswb (%rcx), %xmm18, %xmm19
+
+// CHECK: vpacksswb 4660(%rax,%r14,8), %xmm18, %xmm19
+// CHECK:  encoding: [0x62,0xa1,0xed,0x00,0x63,0x9c,0xf0,0x34,0x12,0x00,0x00]
+          vpacksswb 4660(%rax,%r14,8), %xmm18, %xmm19
+
+// CHECK: vpacksswb 2032(%rdx), %xmm18, %xmm19
+// CHECK:  encoding: [0x62,0xe1,0xed,0x00,0x63,0x5a,0x7f]
+          vpacksswb 2032(%rdx), %xmm18, %xmm19
+
+// CHECK: vpacksswb 2048(%rdx), %xmm18, %xmm19
+// CHECK:  encoding: [0x62,0xe1,0xed,0x00,0x63,0x9a,0x00,0x08,0x00,0x00]
+          vpacksswb 2048(%rdx), %xmm18, %xmm19
+
+// CHECK: vpacksswb -2048(%rdx), %xmm18, %xmm19
+// CHECK:  encoding: [0x62,0xe1,0xed,0x00,0x63,0x5a,0x80]
+          vpacksswb -2048(%rdx), %xmm18, %xmm19
+
+// CHECK: vpacksswb -2064(%rdx), %xmm18, %xmm19
+// CHECK:  encoding: [0x62,0xe1,0xed,0x00,0x63,0x9a,0xf0,0xf7,0xff,0xff]
+          vpacksswb -2064(%rdx), %xmm18, %xmm19
+
+// CHECK: vpacksswb %ymm19, %ymm22, %ymm22
+// CHECK:  encoding: [0x62,0xa1,0xcd,0x20,0x63,0xf3]
+          vpacksswb %ymm19, %ymm22, %ymm22
+
+// CHECK: vpacksswb %ymm19, %ymm22, %ymm22 {%k7}
+// CHECK:  encoding: [0x62,0xa1,0xcd,0x27,0x63,0xf3]
+          vpacksswb %ymm19, %ymm22, %ymm22 {%k7}
+
+// CHECK: vpacksswb %ymm19, %ymm22, %ymm22 {%k7} {z}
+// CHECK:  encoding: [0x62,0xa1,0xcd,0xa7,0x63,0xf3]
+          vpacksswb %ymm19, %ymm22, %ymm22 {%k7} {z}
+
+// CHECK: vpacksswb (%rcx), %ymm22, %ymm22
+// CHECK:  encoding: [0x62,0xe1,0xcd,0x20,0x63,0x31]
+          vpacksswb (%rcx), %ymm22, %ymm22
+
+// CHECK: vpacksswb 4660(%rax,%r14,8), %ymm22, %ymm22
+// CHECK:  encoding: [0x62,0xa1,0xcd,0x20,0x63,0xb4,0xf0,0x34,0x12,0x00,0x00]
+          vpacksswb 4660(%rax,%r14,8), %ymm22, %ymm22
+
+// CHECK: vpacksswb 4064(%rdx), %ymm22, %ymm22
+// CHECK:  encoding: [0x62,0xe1,0xcd,0x20,0x63,0x72,0x7f]
+          vpacksswb 4064(%rdx), %ymm22, %ymm22
+
+// CHECK: vpacksswb 4096(%rdx), %ymm22, %ymm22
+// CHECK:  encoding: [0x62,0xe1,0xcd,0x20,0x63,0xb2,0x00,0x10,0x00,0x00]
+          vpacksswb 4096(%rdx), %ymm22, %ymm22
+
+// CHECK: vpacksswb -4096(%rdx), %ymm22, %ymm22
+// CHECK:  encoding: [0x62,0xe1,0xcd,0x20,0x63,0x72,0x80]
+          vpacksswb -4096(%rdx), %ymm22, %ymm22
+
+// CHECK: vpacksswb -4128(%rdx), %ymm22, %ymm22
+// CHECK:  encoding: [0x62,0xe1,0xcd,0x20,0x63,0xb2,0xe0,0xef,0xff,0xff]
+          vpacksswb -4128(%rdx), %ymm22, %ymm22
+
+// CHECK: vpackuswb %xmm23, %xmm28, %xmm23
+// CHECK:  encoding: [0x62,0xa1,0x9d,0x00,0x67,0xff]
+          vpackuswb %xmm23, %xmm28, %xmm23
+
+// CHECK: vpackuswb %xmm23, %xmm28, %xmm23 {%k3}
+// CHECK:  encoding: [0x62,0xa1,0x9d,0x03,0x67,0xff]
+          vpackuswb %xmm23, %xmm28, %xmm23 {%k3}
+
+// CHECK: vpackuswb %xmm23, %xmm28, %xmm23 {%k3} {z}
+// CHECK:  encoding: [0x62,0xa1,0x9d,0x83,0x67,0xff]
+          vpackuswb %xmm23, %xmm28, %xmm23 {%k3} {z}
+
+// CHECK: vpackuswb (%rcx), %xmm28, %xmm23
+// CHECK:  encoding: [0x62,0xe1,0x9d,0x00,0x67,0x39]
+          vpackuswb (%rcx), %xmm28, %xmm23
+
+// CHECK: vpackuswb 4660(%rax,%r14,8), %xmm28, %xmm23
+// CHECK:  encoding: [0x62,0xa1,0x9d,0x00,0x67,0xbc,0xf0,0x34,0x12,0x00,0x00]
+          vpackuswb 4660(%rax,%r14,8), %xmm28, %xmm23
+
+// CHECK: vpackuswb 2032(%rdx), %xmm28, %xmm23
+// CHECK:  encoding: [0x62,0xe1,0x9d,0x00,0x67,0x7a,0x7f]
+          vpackuswb 2032(%rdx), %xmm28, %xmm23
+
+// CHECK: vpackuswb 2048(%rdx), %xmm28, %xmm23
+// CHECK:  encoding: [0x62,0xe1,0x9d,0x00,0x67,0xba,0x00,0x08,0x00,0x00]
+          vpackuswb 2048(%rdx), %xmm28, %xmm23
+
+// CHECK: vpackuswb -2048(%rdx), %xmm28, %xmm23
+// CHECK:  encoding: [0x62,0xe1,0x9d,0x00,0x67,0x7a,0x80]
+          vpackuswb -2048(%rdx), %xmm28, %xmm23
+
+// CHECK: vpackuswb -2064(%rdx), %xmm28, %xmm23
+// CHECK:  encoding: [0x62,0xe1,0x9d,0x00,0x67,0xba,0xf0,0xf7,0xff,0xff]
+          vpackuswb -2064(%rdx), %xmm28, %xmm23
+
+// CHECK: vpackuswb %ymm27, %ymm26, %ymm20
+// CHECK:  encoding: [0x62,0x81,0xad,0x20,0x67,0xe3]
+          vpackuswb %ymm27, %ymm26, %ymm20
+
+// CHECK: vpackuswb %ymm27, %ymm26, %ymm20 {%k6}
+// CHECK:  encoding: [0x62,0x81,0xad,0x26,0x67,0xe3]
+          vpackuswb %ymm27, %ymm26, %ymm20 {%k6}
+
+// CHECK: vpackuswb %ymm27, %ymm26, %ymm20 {%k6} {z}
+// CHECK:  encoding: [0x62,0x81,0xad,0xa6,0x67,0xe3]
+          vpackuswb %ymm27, %ymm26, %ymm20 {%k6} {z}
+
+// CHECK: vpackuswb (%rcx), %ymm26, %ymm20
+// CHECK:  encoding: [0x62,0xe1,0xad,0x20,0x67,0x21]
+          vpackuswb (%rcx), %ymm26, %ymm20
+
+// CHECK: vpackuswb 4660(%rax,%r14,8), %ymm26, %ymm20
+// CHECK:  encoding: [0x62,0xa1,0xad,0x20,0x67,0xa4,0xf0,0x34,0x12,0x00,0x00]
+          vpackuswb 4660(%rax,%r14,8), %ymm26, %ymm20
+
+// CHECK: vpackuswb 4064(%rdx), %ymm26, %ymm20
+// CHECK:  encoding: [0x62,0xe1,0xad,0x20,0x67,0x62,0x7f]
+          vpackuswb 4064(%rdx), %ymm26, %ymm20
+
+// CHECK: vpackuswb 4096(%rdx), %ymm26, %ymm20
+// CHECK:  encoding: [0x62,0xe1,0xad,0x20,0x67,0xa2,0x00,0x10,0x00,0x00]
+          vpackuswb 4096(%rdx), %ymm26, %ymm20
+
+// CHECK: vpackuswb -4096(%rdx), %ymm26, %ymm20
+// CHECK:  encoding: [0x62,0xe1,0xad,0x20,0x67,0x62,0x80]
+          vpackuswb -4096(%rdx), %ymm26, %ymm20
+
+// CHECK: vpackuswb -4128(%rdx), %ymm26, %ymm20
+// CHECK:  encoding: [0x62,0xe1,0xad,0x20,0x67,0xa2,0xe0,0xef,0xff,0xff]
+          vpackuswb -4128(%rdx), %ymm26, %ymm20
+
+// CHECK: vpackssdw %xmm26, %xmm26, %xmm30
+// CHECK:  encoding: [0x62,0x01,0x2d,0x00,0x6b,0xf2]
+          vpackssdw %xmm26, %xmm26, %xmm30
+
+// CHECK: vpackssdw %xmm26, %xmm26, %xmm30 {%k2}
+// CHECK:  encoding: [0x62,0x01,0x2d,0x02,0x6b,0xf2]
+          vpackssdw %xmm26, %xmm26, %xmm30 {%k2}
+
+// CHECK: vpackssdw %xmm26, %xmm26, %xmm30 {%k2} {z}
+// CHECK:  encoding: [0x62,0x01,0x2d,0x82,0x6b,0xf2]
+          vpackssdw %xmm26, %xmm26, %xmm30 {%k2} {z}
+
+// CHECK: vpackssdw (%rcx), %xmm26, %xmm30
+// CHECK:  encoding: [0x62,0x61,0x2d,0x00,0x6b,0x31]
+          vpackssdw (%rcx), %xmm26, %xmm30
+
+// CHECK: vpackssdw 291(%rax,%r14,8), %xmm26, %xmm30
+// CHECK:  encoding: [0x62,0x21,0x2d,0x00,0x6b,0xb4,0xf0,0x23,0x01,0x00,0x00]
+          vpackssdw 291(%rax,%r14,8), %xmm26, %xmm30
+
+// CHECK: vpackssdw (%rcx){1to4}, %xmm26, %xmm30
+// CHECK:  encoding: [0x62,0x61,0x2d,0x10,0x6b,0x31]
+          vpackssdw (%rcx){1to4}, %xmm26, %xmm30
+
+// CHECK: vpackssdw 2032(%rdx), %xmm26, %xmm30
+// CHECK:  encoding: [0x62,0x61,0x2d,0x00,0x6b,0x72,0x7f]
+          vpackssdw 2032(%rdx), %xmm26, %xmm30
+
+// CHECK: vpackssdw 2048(%rdx), %xmm26, %xmm30
+// CHECK:  encoding: [0x62,0x61,0x2d,0x00,0x6b,0xb2,0x00,0x08,0x00,0x00]
+          vpackssdw 2048(%rdx), %xmm26, %xmm30
+
+// CHECK: vpackssdw -2048(%rdx), %xmm26, %xmm30
+// CHECK:  encoding: [0x62,0x61,0x2d,0x00,0x6b,0x72,0x80]
+          vpackssdw -2048(%rdx), %xmm26, %xmm30
+
+// CHECK: vpackssdw -2064(%rdx), %xmm26, %xmm30
+// CHECK:  encoding: [0x62,0x61,0x2d,0x00,0x6b,0xb2,0xf0,0xf7,0xff,0xff]
+          vpackssdw -2064(%rdx), %xmm26, %xmm30
+
+// CHECK: vpackssdw 508(%rdx){1to4}, %xmm26, %xmm30
+// CHECK:  encoding: [0x62,0x61,0x2d,0x10,0x6b,0x72,0x7f]
+          vpackssdw 508(%rdx){1to4}, %xmm26, %xmm30
+
+// CHECK: vpackssdw 512(%rdx){1to4}, %xmm26, %xmm30
+// CHECK:  encoding: [0x62,0x61,0x2d,0x10,0x6b,0xb2,0x00,0x02,0x00,0x00]
+          vpackssdw 512(%rdx){1to4}, %xmm26, %xmm30
+
+// CHECK: vpackssdw -512(%rdx){1to4}, %xmm26, %xmm30
+// CHECK:  encoding: [0x62,0x61,0x2d,0x10,0x6b,0x72,0x80]
+          vpackssdw -512(%rdx){1to4}, %xmm26, %xmm30
+
+// CHECK: vpackssdw -516(%rdx){1to4}, %xmm26, %xmm30
+// CHECK:  encoding: [0x62,0x61,0x2d,0x10,0x6b,0xb2,0xfc,0xfd,0xff,0xff]
+          vpackssdw -516(%rdx){1to4}, %xmm26, %xmm30
+
+// CHECK: vpackssdw %ymm26, %ymm22, %ymm28
+// CHECK:  encoding: [0x62,0x01,0x4d,0x20,0x6b,0xe2]
+          vpackssdw %ymm26, %ymm22, %ymm28
+
+// CHECK: vpackssdw %ymm26, %ymm22, %ymm28 {%k7}
+// CHECK:  encoding: [0x62,0x01,0x4d,0x27,0x6b,0xe2]
+          vpackssdw %ymm26, %ymm22, %ymm28 {%k7}
+
+// CHECK: vpackssdw %ymm26, %ymm22, %ymm28 {%k7} {z}
+// CHECK:  encoding: [0x62,0x01,0x4d,0xa7,0x6b,0xe2]
+          vpackssdw %ymm26, %ymm22, %ymm28 {%k7} {z}
+
+// CHECK: vpackssdw (%rcx), %ymm22, %ymm28
+// CHECK:  encoding: [0x62,0x61,0x4d,0x20,0x6b,0x21]
+          vpackssdw (%rcx), %ymm22, %ymm28
+
+// CHECK: vpackssdw 291(%rax,%r14,8), %ymm22, %ymm28
+// CHECK:  encoding: [0x62,0x21,0x4d,0x20,0x6b,0xa4,0xf0,0x23,0x01,0x00,0x00]
+          vpackssdw 291(%rax,%r14,8), %ymm22, %ymm28
+
+// CHECK: vpackssdw (%rcx){1to8}, %ymm22, %ymm28
+// CHECK:  encoding: [0x62,0x61,0x4d,0x30,0x6b,0x21]
+          vpackssdw (%rcx){1to8}, %ymm22, %ymm28
+
+// CHECK: vpackssdw 4064(%rdx), %ymm22, %ymm28
+// CHECK:  encoding: [0x62,0x61,0x4d,0x20,0x6b,0x62,0x7f]
+          vpackssdw 4064(%rdx), %ymm22, %ymm28
+
+// CHECK: vpackssdw 4096(%rdx), %ymm22, %ymm28
+// CHECK:  encoding: [0x62,0x61,0x4d,0x20,0x6b,0xa2,0x00,0x10,0x00,0x00]
+          vpackssdw 4096(%rdx), %ymm22, %ymm28
+
+// CHECK: vpackssdw -4096(%rdx), %ymm22, %ymm28
+// CHECK:  encoding: [0x62,0x61,0x4d,0x20,0x6b,0x62,0x80]
+          vpackssdw -4096(%rdx), %ymm22, %ymm28
+
+// CHECK: vpackssdw -4128(%rdx), %ymm22, %ymm28
+// CHECK:  encoding: [0x62,0x61,0x4d,0x20,0x6b,0xa2,0xe0,0xef,0xff,0xff]
+          vpackssdw -4128(%rdx), %ymm22, %ymm28
+
+// CHECK: vpackssdw 508(%rdx){1to8}, %ymm22, %ymm28
+// CHECK:  encoding: [0x62,0x61,0x4d,0x30,0x6b,0x62,0x7f]
+          vpackssdw 508(%rdx){1to8}, %ymm22, %ymm28
+
+// CHECK: vpackssdw 512(%rdx){1to8}, %ymm22, %ymm28
+// CHECK:  encoding: [0x62,0x61,0x4d,0x30,0x6b,0xa2,0x00,0x02,0x00,0x00]
+          vpackssdw 512(%rdx){1to8}, %ymm22, %ymm28
+
+// CHECK: vpackssdw -512(%rdx){1to8}, %ymm22, %ymm28
+// CHECK:  encoding: [0x62,0x61,0x4d,0x30,0x6b,0x62,0x80]
+          vpackssdw -512(%rdx){1to8}, %ymm22, %ymm28
+
+// CHECK: vpackssdw -516(%rdx){1to8}, %ymm22, %ymm28
+// CHECK:  encoding: [0x62,0x61,0x4d,0x30,0x6b,0xa2,0xfc,0xfd,0xff,0xff]
+          vpackssdw -516(%rdx){1to8}, %ymm22, %ymm28
+
+// CHECK: vpacksswb %xmm21, %xmm28, %xmm25
+// CHECK:  encoding: [0x62,0x21,0x9d,0x00,0x63,0xcd]
+          vpacksswb %xmm21, %xmm28, %xmm25
+
+// CHECK: vpacksswb %xmm21, %xmm28, %xmm25 {%k5}
+// CHECK:  encoding: [0x62,0x21,0x9d,0x05,0x63,0xcd]
+          vpacksswb %xmm21, %xmm28, %xmm25 {%k5}
+
+// CHECK: vpacksswb %xmm21, %xmm28, %xmm25 {%k5} {z}
+// CHECK:  encoding: [0x62,0x21,0x9d,0x85,0x63,0xcd]
+          vpacksswb %xmm21, %xmm28, %xmm25 {%k5} {z}
+
+// CHECK: vpacksswb (%rcx), %xmm28, %xmm25
+// CHECK:  encoding: [0x62,0x61,0x9d,0x00,0x63,0x09]
+          vpacksswb (%rcx), %xmm28, %xmm25
+
+// CHECK: vpacksswb 291(%rax,%r14,8), %xmm28, %xmm25
+// CHECK:  encoding: [0x62,0x21,0x9d,0x00,0x63,0x8c,0xf0,0x23,0x01,0x00,0x00]
+          vpacksswb 291(%rax,%r14,8), %xmm28, %xmm25
+
+// CHECK: vpacksswb 2032(%rdx), %xmm28, %xmm25
+// CHECK:  encoding: [0x62,0x61,0x9d,0x00,0x63,0x4a,0x7f]
+          vpacksswb 2032(%rdx), %xmm28, %xmm25
+
+// CHECK: vpacksswb 2048(%rdx), %xmm28, %xmm25
+// CHECK:  encoding: [0x62,0x61,0x9d,0x00,0x63,0x8a,0x00,0x08,0x00,0x00]
+          vpacksswb 2048(%rdx), %xmm28, %xmm25
+
+// CHECK: vpacksswb -2048(%rdx), %xmm28, %xmm25
+// CHECK:  encoding: [0x62,0x61,0x9d,0x00,0x63,0x4a,0x80]
+          vpacksswb -2048(%rdx), %xmm28, %xmm25
+
+// CHECK: vpacksswb -2064(%rdx), %xmm28, %xmm25
+// CHECK:  encoding: [0x62,0x61,0x9d,0x00,0x63,0x8a,0xf0,0xf7,0xff,0xff]
+          vpacksswb -2064(%rdx), %xmm28, %xmm25
+
+// CHECK: vpacksswb %ymm22, %ymm28, %ymm21
+// CHECK:  encoding: [0x62,0xa1,0x9d,0x20,0x63,0xee]
+          vpacksswb %ymm22, %ymm28, %ymm21
+
+// CHECK: vpacksswb %ymm22, %ymm28, %ymm21 {%k5}
+// CHECK:  encoding: [0x62,0xa1,0x9d,0x25,0x63,0xee]
+          vpacksswb %ymm22, %ymm28, %ymm21 {%k5}
+
+// CHECK: vpacksswb %ymm22, %ymm28, %ymm21 {%k5} {z}
+// CHECK:  encoding: [0x62,0xa1,0x9d,0xa5,0x63,0xee]
+          vpacksswb %ymm22, %ymm28, %ymm21 {%k5} {z}
+
+// CHECK: vpacksswb (%rcx), %ymm28, %ymm21
+// CHECK:  encoding: [0x62,0xe1,0x9d,0x20,0x63,0x29]
+          vpacksswb (%rcx), %ymm28, %ymm21
+
+// CHECK: vpacksswb 291(%rax,%r14,8), %ymm28, %ymm21
+// CHECK:  encoding: [0x62,0xa1,0x9d,0x20,0x63,0xac,0xf0,0x23,0x01,0x00,0x00]
+          vpacksswb 291(%rax,%r14,8), %ymm28, %ymm21
+
+// CHECK: vpacksswb 4064(%rdx), %ymm28, %ymm21
+// CHECK:  encoding: [0x62,0xe1,0x9d,0x20,0x63,0x6a,0x7f]
+          vpacksswb 4064(%rdx), %ymm28, %ymm21
+
+// CHECK: vpacksswb 4096(%rdx), %ymm28, %ymm21
+// CHECK:  encoding: [0x62,0xe1,0x9d,0x20,0x63,0xaa,0x00,0x10,0x00,0x00]
+          vpacksswb 4096(%rdx), %ymm28, %ymm21
+
+// CHECK: vpacksswb -4096(%rdx), %ymm28, %ymm21
+// CHECK:  encoding: [0x62,0xe1,0x9d,0x20,0x63,0x6a,0x80]
+          vpacksswb -4096(%rdx), %ymm28, %ymm21
+
+// CHECK: vpacksswb -4128(%rdx), %ymm28, %ymm21
+// CHECK:  encoding: [0x62,0xe1,0x9d,0x20,0x63,0xaa,0xe0,0xef,0xff,0xff]
+          vpacksswb -4128(%rdx), %ymm28, %ymm21
+
+// CHECK: vpackusdw %xmm20, %xmm24, %xmm24
+// CHECK:  encoding: [0x62,0x22,0x3d,0x00,0x2b,0xc4]
+          vpackusdw %xmm20, %xmm24, %xmm24
+
+// CHECK: vpackusdw %xmm20, %xmm24, %xmm24 {%k5}
+// CHECK:  encoding: [0x62,0x22,0x3d,0x05,0x2b,0xc4]
+          vpackusdw %xmm20, %xmm24, %xmm24 {%k5}
+
+// CHECK: vpackusdw %xmm20, %xmm24, %xmm24 {%k5} {z}
+// CHECK:  encoding: [0x62,0x22,0x3d,0x85,0x2b,0xc4]
+          vpackusdw %xmm20, %xmm24, %xmm24 {%k5} {z}
+
+// CHECK: vpackusdw (%rcx), %xmm24, %xmm24
+// CHECK:  encoding: [0x62,0x62,0x3d,0x00,0x2b,0x01]
+          vpackusdw (%rcx), %xmm24, %xmm24
+
+// CHECK: vpackusdw 291(%rax,%r14,8), %xmm24, %xmm24
+// CHECK:  encoding: [0x62,0x22,0x3d,0x00,0x2b,0x84,0xf0,0x23,0x01,0x00,0x00]
+          vpackusdw 291(%rax,%r14,8), %xmm24, %xmm24
+
+// CHECK: vpackusdw (%rcx){1to4}, %xmm24, %xmm24
+// CHECK:  encoding: [0x62,0x62,0x3d,0x10,0x2b,0x01]
+          vpackusdw (%rcx){1to4}, %xmm24, %xmm24
+
+// CHECK: vpackusdw 2032(%rdx), %xmm24, %xmm24
+// CHECK:  encoding: [0x62,0x62,0x3d,0x00,0x2b,0x42,0x7f]
+          vpackusdw 2032(%rdx), %xmm24, %xmm24
+
+// CHECK: vpackusdw 2048(%rdx), %xmm24, %xmm24
+// CHECK:  encoding: [0x62,0x62,0x3d,0x00,0x2b,0x82,0x00,0x08,0x00,0x00]
+          vpackusdw 2048(%rdx), %xmm24, %xmm24
+
+// CHECK: vpackusdw -2048(%rdx), %xmm24, %xmm24
+// CHECK:  encoding: [0x62,0x62,0x3d,0x00,0x2b,0x42,0x80]
+          vpackusdw -2048(%rdx), %xmm24, %xmm24
+
+// CHECK: vpackusdw -2064(%rdx), %xmm24, %xmm24
+// CHECK:  encoding: [0x62,0x62,0x3d,0x00,0x2b,0x82,0xf0,0xf7,0xff,0xff]
+          vpackusdw -2064(%rdx), %xmm24, %xmm24
+
+// CHECK: vpackusdw 508(%rdx){1to4}, %xmm24, %xmm24
+// CHECK:  encoding: [0x62,0x62,0x3d,0x10,0x2b,0x42,0x7f]
+          vpackusdw 508(%rdx){1to4}, %xmm24, %xmm24
+
+// CHECK: vpackusdw 512(%rdx){1to4}, %xmm24, %xmm24
+// CHECK:  encoding: [0x62,0x62,0x3d,0x10,0x2b,0x82,0x00,0x02,0x00,0x00]
+          vpackusdw 512(%rdx){1to4}, %xmm24, %xmm24
+
+// CHECK: vpackusdw -512(%rdx){1to4}, %xmm24, %xmm24
+// CHECK:  encoding: [0x62,0x62,0x3d,0x10,0x2b,0x42,0x80]
+          vpackusdw -512(%rdx){1to4}, %xmm24, %xmm24
+
+// CHECK: vpackusdw -516(%rdx){1to4}, %xmm24, %xmm24
+// CHECK:  encoding: [0x62,0x62,0x3d,0x10,0x2b,0x82,0xfc,0xfd,0xff,0xff]
+          vpackusdw -516(%rdx){1to4}, %xmm24, %xmm24
+
+// CHECK: vpackusdw %ymm21, %ymm26, %ymm19
+// CHECK:  encoding: [0x62,0xa2,0x2d,0x20,0x2b,0xdd]
+          vpackusdw %ymm21, %ymm26, %ymm19
+
+// CHECK: vpackusdw %ymm21, %ymm26, %ymm19 {%k1}
+// CHECK:  encoding: [0x62,0xa2,0x2d,0x21,0x2b,0xdd]
+          vpackusdw %ymm21, %ymm26, %ymm19 {%k1}
+
+// CHECK: vpackusdw %ymm21, %ymm26, %ymm19 {%k1} {z}
+// CHECK:  encoding: [0x62,0xa2,0x2d,0xa1,0x2b,0xdd]
+          vpackusdw %ymm21, %ymm26, %ymm19 {%k1} {z}
+
+// CHECK: vpackusdw (%rcx), %ymm26, %ymm19
+// CHECK:  encoding: [0x62,0xe2,0x2d,0x20,0x2b,0x19]
+          vpackusdw (%rcx), %ymm26, %ymm19
+
+// CHECK: vpackusdw 291(%rax,%r14,8), %ymm26, %ymm19
+// CHECK:  encoding: [0x62,0xa2,0x2d,0x20,0x2b,0x9c,0xf0,0x23,0x01,0x00,0x00]
+          vpackusdw 291(%rax,%r14,8), %ymm26, %ymm19
+
+// CHECK: vpackusdw (%rcx){1to8}, %ymm26, %ymm19
+// CHECK:  encoding: [0x62,0xe2,0x2d,0x30,0x2b,0x19]
+          vpackusdw (%rcx){1to8}, %ymm26, %ymm19
+
+// CHECK: vpackusdw 4064(%rdx), %ymm26, %ymm19
+// CHECK:  encoding: [0x62,0xe2,0x2d,0x20,0x2b,0x5a,0x7f]
+          vpackusdw 4064(%rdx), %ymm26, %ymm19
+
+// CHECK: vpackusdw 4096(%rdx), %ymm26, %ymm19
+// CHECK:  encoding: [0x62,0xe2,0x2d,0x20,0x2b,0x9a,0x00,0x10,0x00,0x00]
+          vpackusdw 4096(%rdx), %ymm26, %ymm19
+
+// CHECK: vpackusdw -4096(%rdx), %ymm26, %ymm19
+// CHECK:  encoding: [0x62,0xe2,0x2d,0x20,0x2b,0x5a,0x80]
+          vpackusdw -4096(%rdx), %ymm26, %ymm19
+
+// CHECK: vpackusdw -4128(%rdx), %ymm26, %ymm19
+// CHECK:  encoding: [0x62,0xe2,0x2d,0x20,0x2b,0x9a,0xe0,0xef,0xff,0xff]
+          vpackusdw -4128(%rdx), %ymm26, %ymm19
+
+// CHECK: vpackusdw 508(%rdx){1to8}, %ymm26, %ymm19
+// CHECK:  encoding: [0x62,0xe2,0x2d,0x30,0x2b,0x5a,0x7f]
+          vpackusdw 508(%rdx){1to8}, %ymm26, %ymm19
+
+// CHECK: vpackusdw 512(%rdx){1to8}, %ymm26, %ymm19
+// CHECK:  encoding: [0x62,0xe2,0x2d,0x30,0x2b,0x9a,0x00,0x02,0x00,0x00]
+          vpackusdw 512(%rdx){1to8}, %ymm26, %ymm19
+
+// CHECK: vpackusdw -512(%rdx){1to8}, %ymm26, %ymm19
+// CHECK:  encoding: [0x62,0xe2,0x2d,0x30,0x2b,0x5a,0x80]
+          vpackusdw -512(%rdx){1to8}, %ymm26, %ymm19
+
+// CHECK: vpackusdw -516(%rdx){1to8}, %ymm26, %ymm19
+// CHECK:  encoding: [0x62,0xe2,0x2d,0x30,0x2b,0x9a,0xfc,0xfd,0xff,0xff]
+          vpackusdw -516(%rdx){1to8}, %ymm26, %ymm19
+
+// CHECK: vpackuswb %xmm17, %xmm27, %xmm21
+// CHECK:  encoding: [0x62,0xa1,0xa5,0x00,0x67,0xe9]
+          vpackuswb %xmm17, %xmm27, %xmm21
+
+// CHECK: vpackuswb %xmm17, %xmm27, %xmm21 {%k4}
+// CHECK:  encoding: [0x62,0xa1,0xa5,0x04,0x67,0xe9]
+          vpackuswb %xmm17, %xmm27, %xmm21 {%k4}
+
+// CHECK: vpackuswb %xmm17, %xmm27, %xmm21 {%k4} {z}
+// CHECK:  encoding: [0x62,0xa1,0xa5,0x84,0x67,0xe9]
+          vpackuswb %xmm17, %xmm27, %xmm21 {%k4} {z}
+
+// CHECK: vpackuswb (%rcx), %xmm27, %xmm21
+// CHECK:  encoding: [0x62,0xe1,0xa5,0x00,0x67,0x29]
+          vpackuswb (%rcx), %xmm27, %xmm21
+
+// CHECK: vpackuswb 291(%rax,%r14,8), %xmm27, %xmm21
+// CHECK:  encoding: [0x62,0xa1,0xa5,0x00,0x67,0xac,0xf0,0x23,0x01,0x00,0x00]
+          vpackuswb 291(%rax,%r14,8), %xmm27, %xmm21
+
+// CHECK: vpackuswb 2032(%rdx), %xmm27, %xmm21
+// CHECK:  encoding: [0x62,0xe1,0xa5,0x00,0x67,0x6a,0x7f]
+          vpackuswb 2032(%rdx), %xmm27, %xmm21
+
+// CHECK: vpackuswb 2048(%rdx), %xmm27, %xmm21
+// CHECK:  encoding: [0x62,0xe1,0xa5,0x00,0x67,0xaa,0x00,0x08,0x00,0x00]
+          vpackuswb 2048(%rdx), %xmm27, %xmm21
+
+// CHECK: vpackuswb -2048(%rdx), %xmm27, %xmm21
+// CHECK:  encoding: [0x62,0xe1,0xa5,0x00,0x67,0x6a,0x80]
+          vpackuswb -2048(%rdx), %xmm27, %xmm21
+
+// CHECK: vpackuswb -2064(%rdx), %xmm27, %xmm21
+// CHECK:  encoding: [0x62,0xe1,0xa5,0x00,0x67,0xaa,0xf0,0xf7,0xff,0xff]
+          vpackuswb -2064(%rdx), %xmm27, %xmm21
+
+// CHECK: vpackuswb %ymm20, %ymm28, %ymm19
+// CHECK:  encoding: [0x62,0xa1,0x9d,0x20,0x67,0xdc]
+          vpackuswb %ymm20, %ymm28, %ymm19
+
+// CHECK: vpackuswb %ymm20, %ymm28, %ymm19 {%k1}
+// CHECK:  encoding: [0x62,0xa1,0x9d,0x21,0x67,0xdc]
+          vpackuswb %ymm20, %ymm28, %ymm19 {%k1}
+
+// CHECK: vpackuswb %ymm20, %ymm28, %ymm19 {%k1} {z}
+// CHECK:  encoding: [0x62,0xa1,0x9d,0xa1,0x67,0xdc]
+          vpackuswb %ymm20, %ymm28, %ymm19 {%k1} {z}
+
+// CHECK: vpackuswb (%rcx), %ymm28, %ymm19
+// CHECK:  encoding: [0x62,0xe1,0x9d,0x20,0x67,0x19]
+          vpackuswb (%rcx), %ymm28, %ymm19
+
+// CHECK: vpackuswb 291(%rax,%r14,8), %ymm28, %ymm19
+// CHECK:  encoding: [0x62,0xa1,0x9d,0x20,0x67,0x9c,0xf0,0x23,0x01,0x00,0x00]
+          vpackuswb 291(%rax,%r14,8), %ymm28, %ymm19
+
+// CHECK: vpackuswb 4064(%rdx), %ymm28, %ymm19
+// CHECK:  encoding: [0x62,0xe1,0x9d,0x20,0x67,0x5a,0x7f]
+          vpackuswb 4064(%rdx), %ymm28, %ymm19
+
+// CHECK: vpackuswb 4096(%rdx), %ymm28, %ymm19
+// CHECK:  encoding: [0x62,0xe1,0x9d,0x20,0x67,0x9a,0x00,0x10,0x00,0x00]
+          vpackuswb 4096(%rdx), %ymm28, %ymm19
+
+// CHECK: vpackuswb -4096(%rdx), %ymm28, %ymm19
+// CHECK:  encoding: [0x62,0xe1,0x9d,0x20,0x67,0x5a,0x80]
+          vpackuswb -4096(%rdx), %ymm28, %ymm19
+
+// CHECK: vpackuswb -4128(%rdx), %ymm28, %ymm19
+// CHECK:  encoding: [0x62,0xe1,0x9d,0x20,0x67,0x9a,0xe0,0xef,0xff,0xff]
+          vpackuswb -4128(%rdx), %ymm28, %ymm19
+
+// CHECK: vpackssdw %xmm19, %xmm17, %xmm24
+// CHECK:  encoding: [0x62,0x21,0x75,0x00,0x6b,0xc3]
+          vpackssdw %xmm19, %xmm17, %xmm24
+
+// CHECK: vpackssdw %xmm19, %xmm17, %xmm24 {%k7}
+// CHECK:  encoding: [0x62,0x21,0x75,0x07,0x6b,0xc3]
+          vpackssdw %xmm19, %xmm17, %xmm24 {%k7}
+
+// CHECK: vpackssdw %xmm19, %xmm17, %xmm24 {%k7} {z}
+// CHECK:  encoding: [0x62,0x21,0x75,0x87,0x6b,0xc3]
+          vpackssdw %xmm19, %xmm17, %xmm24 {%k7} {z}
+
+// CHECK: vpackssdw (%rcx), %xmm17, %xmm24
+// CHECK:  encoding: [0x62,0x61,0x75,0x00,0x6b,0x01]
+          vpackssdw (%rcx), %xmm17, %xmm24
+
+// CHECK: vpackssdw 4660(%rax,%r14,8), %xmm17, %xmm24
+// CHECK:  encoding: [0x62,0x21,0x75,0x00,0x6b,0x84,0xf0,0x34,0x12,0x00,0x00]
+          vpackssdw 4660(%rax,%r14,8), %xmm17, %xmm24
+
+// CHECK: vpackssdw (%rcx){1to4}, %xmm17, %xmm24
+// CHECK:  encoding: [0x62,0x61,0x75,0x10,0x6b,0x01]
+          vpackssdw (%rcx){1to4}, %xmm17, %xmm24
+
+// CHECK: vpackssdw 2032(%rdx), %xmm17, %xmm24
+// CHECK:  encoding: [0x62,0x61,0x75,0x00,0x6b,0x42,0x7f]
+          vpackssdw 2032(%rdx), %xmm17, %xmm24
+
+// CHECK: vpackssdw 2048(%rdx), %xmm17, %xmm24
+// CHECK:  encoding: [0x62,0x61,0x75,0x00,0x6b,0x82,0x00,0x08,0x00,0x00]
+          vpackssdw 2048(%rdx), %xmm17, %xmm24
+
+// CHECK: vpackssdw -2048(%rdx), %xmm17, %xmm24
+// CHECK:  encoding: [0x62,0x61,0x75,0x00,0x6b,0x42,0x80]
+          vpackssdw -2048(%rdx), %xmm17, %xmm24
+
+// CHECK: vpackssdw -2064(%rdx), %xmm17, %xmm24
+// CHECK:  encoding: [0x62,0x61,0x75,0x00,0x6b,0x82,0xf0,0xf7,0xff,0xff]
+          vpackssdw -2064(%rdx), %xmm17, %xmm24
+
+// CHECK: vpackssdw 508(%rdx){1to4}, %xmm17, %xmm24
+// CHECK:  encoding: [0x62,0x61,0x75,0x10,0x6b,0x42,0x7f]
+          vpackssdw 508(%rdx){1to4}, %xmm17, %xmm24
+
+// CHECK: vpackssdw 512(%rdx){1to4}, %xmm17, %xmm24
+// CHECK:  encoding: [0x62,0x61,0x75,0x10,0x6b,0x82,0x00,0x02,0x00,0x00]
+          vpackssdw 512(%rdx){1to4}, %xmm17, %xmm24
+
+// CHECK: vpackssdw -512(%rdx){1to4}, %xmm17, %xmm24
+// CHECK:  encoding: [0x62,0x61,0x75,0x10,0x6b,0x42,0x80]
+          vpackssdw -512(%rdx){1to4}, %xmm17, %xmm24
+
+// CHECK: vpackssdw -516(%rdx){1to4}, %xmm17, %xmm24
+// CHECK:  encoding: [0x62,0x61,0x75,0x10,0x6b,0x82,0xfc,0xfd,0xff,0xff]
+          vpackssdw -516(%rdx){1to4}, %xmm17, %xmm24
+
+// CHECK: vpackssdw %ymm19, %ymm19, %ymm25
+// CHECK:  encoding: [0x62,0x21,0x65,0x20,0x6b,0xcb]
+          vpackssdw %ymm19, %ymm19, %ymm25
+
+// CHECK: vpackssdw %ymm19, %ymm19, %ymm25 {%k2}
+// CHECK:  encoding: [0x62,0x21,0x65,0x22,0x6b,0xcb]
+          vpackssdw %ymm19, %ymm19, %ymm25 {%k2}
+
+// CHECK: vpackssdw %ymm19, %ymm19, %ymm25 {%k2} {z}
+// CHECK:  encoding: [0x62,0x21,0x65,0xa2,0x6b,0xcb]
+          vpackssdw %ymm19, %ymm19, %ymm25 {%k2} {z}
+
+// CHECK: vpackssdw (%rcx), %ymm19, %ymm25
+// CHECK:  encoding: [0x62,0x61,0x65,0x20,0x6b,0x09]
+          vpackssdw (%rcx), %ymm19, %ymm25
+
+// CHECK: vpackssdw 4660(%rax,%r14,8), %ymm19, %ymm25
+// CHECK:  encoding: [0x62,0x21,0x65,0x20,0x6b,0x8c,0xf0,0x34,0x12,0x00,0x00]
+          vpackssdw 4660(%rax,%r14,8), %ymm19, %ymm25
+
+// CHECK: vpackssdw (%rcx){1to8}, %ymm19, %ymm25
+// CHECK:  encoding: [0x62,0x61,0x65,0x30,0x6b,0x09]
+          vpackssdw (%rcx){1to8}, %ymm19, %ymm25
+
+// CHECK: vpackssdw 4064(%rdx), %ymm19, %ymm25
+// CHECK:  encoding: [0x62,0x61,0x65,0x20,0x6b,0x4a,0x7f]
+          vpackssdw 4064(%rdx), %ymm19, %ymm25
+
+// CHECK: vpackssdw 4096(%rdx), %ymm19, %ymm25
+// CHECK:  encoding: [0x62,0x61,0x65,0x20,0x6b,0x8a,0x00,0x10,0x00,0x00]
+          vpackssdw 4096(%rdx), %ymm19, %ymm25
+
+// CHECK: vpackssdw -4096(%rdx), %ymm19, %ymm25
+// CHECK:  encoding: [0x62,0x61,0x65,0x20,0x6b,0x4a,0x80]
+          vpackssdw -4096(%rdx), %ymm19, %ymm25
+
+// CHECK: vpackssdw -4128(%rdx), %ymm19, %ymm25
+// CHECK:  encoding: [0x62,0x61,0x65,0x20,0x6b,0x8a,0xe0,0xef,0xff,0xff]
+          vpackssdw -4128(%rdx), %ymm19, %ymm25
+
+// CHECK: vpackssdw 508(%rdx){1to8}, %ymm19, %ymm25
+// CHECK:  encoding: [0x62,0x61,0x65,0x30,0x6b,0x4a,0x7f]
+          vpackssdw 508(%rdx){1to8}, %ymm19, %ymm25
+
+// CHECK: vpackssdw 512(%rdx){1to8}, %ymm19, %ymm25
+// CHECK:  encoding: [0x62,0x61,0x65,0x30,0x6b,0x8a,0x00,0x02,0x00,0x00]
+          vpackssdw 512(%rdx){1to8}, %ymm19, %ymm25
+
+// CHECK: vpackssdw -512(%rdx){1to8}, %ymm19, %ymm25
+// CHECK:  encoding: [0x62,0x61,0x65,0x30,0x6b,0x4a,0x80]
+          vpackssdw -512(%rdx){1to8}, %ymm19, %ymm25
+
+// CHECK: vpackssdw -516(%rdx){1to8}, %ymm19, %ymm25
+// CHECK:  encoding: [0x62,0x61,0x65,0x30,0x6b,0x8a,0xfc,0xfd,0xff,0xff]
+          vpackssdw -516(%rdx){1to8}, %ymm19, %ymm25
+
+// CHECK: vpacksswb %xmm22, %xmm25, %xmm29
+// CHECK:  encoding: [0x62,0x21,0xb5,0x00,0x63,0xee]
+          vpacksswb %xmm22, %xmm25, %xmm29
+
+// CHECK: vpacksswb %xmm22, %xmm25, %xmm29 {%k5}
+// CHECK:  encoding: [0x62,0x21,0xb5,0x05,0x63,0xee]
+          vpacksswb %xmm22, %xmm25, %xmm29 {%k5}
+
+// CHECK: vpacksswb %xmm22, %xmm25, %xmm29 {%k5} {z}
+// CHECK:  encoding: [0x62,0x21,0xb5,0x85,0x63,0xee]
+          vpacksswb %xmm22, %xmm25, %xmm29 {%k5} {z}
+
+// CHECK: vpacksswb (%rcx), %xmm25, %xmm29
+// CHECK:  encoding: [0x62,0x61,0xb5,0x00,0x63,0x29]
+          vpacksswb (%rcx), %xmm25, %xmm29
+
+// CHECK: vpacksswb 4660(%rax,%r14,8), %xmm25, %xmm29
+// CHECK:  encoding: [0x62,0x21,0xb5,0x00,0x63,0xac,0xf0,0x34,0x12,0x00,0x00]
+          vpacksswb 4660(%rax,%r14,8), %xmm25, %xmm29
+
+// CHECK: vpacksswb 2032(%rdx), %xmm25, %xmm29
+// CHECK:  encoding: [0x62,0x61,0xb5,0x00,0x63,0x6a,0x7f]
+          vpacksswb 2032(%rdx), %xmm25, %xmm29
+
+// CHECK: vpacksswb 2048(%rdx), %xmm25, %xmm29
+// CHECK:  encoding: [0x62,0x61,0xb5,0x00,0x63,0xaa,0x00,0x08,0x00,0x00]
+          vpacksswb 2048(%rdx), %xmm25, %xmm29
+
+// CHECK: vpacksswb -2048(%rdx), %xmm25, %xmm29
+// CHECK:  encoding: [0x62,0x61,0xb5,0x00,0x63,0x6a,0x80]
+          vpacksswb -2048(%rdx), %xmm25, %xmm29
+
+// CHECK: vpacksswb -2064(%rdx), %xmm25, %xmm29
+// CHECK:  encoding: [0x62,0x61,0xb5,0x00,0x63,0xaa,0xf0,0xf7,0xff,0xff]
+          vpacksswb -2064(%rdx), %xmm25, %xmm29
+
+// CHECK: vpacksswb %ymm27, %ymm20, %ymm26
+// CHECK:  encoding: [0x62,0x01,0xdd,0x20,0x63,0xd3]
+          vpacksswb %ymm27, %ymm20, %ymm26
+
+// CHECK: vpacksswb %ymm27, %ymm20, %ymm26 {%k6}
+// CHECK:  encoding: [0x62,0x01,0xdd,0x26,0x63,0xd3]
+          vpacksswb %ymm27, %ymm20, %ymm26 {%k6}
+
+// CHECK: vpacksswb %ymm27, %ymm20, %ymm26 {%k6} {z}
+// CHECK:  encoding: [0x62,0x01,0xdd,0xa6,0x63,0xd3]
+          vpacksswb %ymm27, %ymm20, %ymm26 {%k6} {z}
+
+// CHECK: vpacksswb (%rcx), %ymm20, %ymm26
+// CHECK:  encoding: [0x62,0x61,0xdd,0x20,0x63,0x11]
+          vpacksswb (%rcx), %ymm20, %ymm26
+
+// CHECK: vpacksswb 4660(%rax,%r14,8), %ymm20, %ymm26
+// CHECK:  encoding: [0x62,0x21,0xdd,0x20,0x63,0x94,0xf0,0x34,0x12,0x00,0x00]
+          vpacksswb 4660(%rax,%r14,8), %ymm20, %ymm26
+
+// CHECK: vpacksswb 4064(%rdx), %ymm20, %ymm26
+// CHECK:  encoding: [0x62,0x61,0xdd,0x20,0x63,0x52,0x7f]
+          vpacksswb 4064(%rdx), %ymm20, %ymm26
+
+// CHECK: vpacksswb 4096(%rdx), %ymm20, %ymm26
+// CHECK:  encoding: [0x62,0x61,0xdd,0x20,0x63,0x92,0x00,0x10,0x00,0x00]
+          vpacksswb 4096(%rdx), %ymm20, %ymm26
+
+// CHECK: vpacksswb -4096(%rdx), %ymm20, %ymm26
+// CHECK:  encoding: [0x62,0x61,0xdd,0x20,0x63,0x52,0x80]
+          vpacksswb -4096(%rdx), %ymm20, %ymm26
+
+// CHECK: vpacksswb -4128(%rdx), %ymm20, %ymm26
+// CHECK:  encoding: [0x62,0x61,0xdd,0x20,0x63,0x92,0xe0,0xef,0xff,0xff]
+          vpacksswb -4128(%rdx), %ymm20, %ymm26
+
+// CHECK: vpackusdw %xmm18, %xmm27, %xmm28
+// CHECK:  encoding: [0x62,0x22,0x25,0x00,0x2b,0xe2]
+          vpackusdw %xmm18, %xmm27, %xmm28
+
+// CHECK: vpackusdw %xmm18, %xmm27, %xmm28 {%k4}
+// CHECK:  encoding: [0x62,0x22,0x25,0x04,0x2b,0xe2]
+          vpackusdw %xmm18, %xmm27, %xmm28 {%k4}
+
+// CHECK: vpackusdw %xmm18, %xmm27, %xmm28 {%k4} {z}
+// CHECK:  encoding: [0x62,0x22,0x25,0x84,0x2b,0xe2]
+          vpackusdw %xmm18, %xmm27, %xmm28 {%k4} {z}
+
+// CHECK: vpackusdw (%rcx), %xmm27, %xmm28
+// CHECK:  encoding: [0x62,0x62,0x25,0x00,0x2b,0x21]
+          vpackusdw (%rcx), %xmm27, %xmm28
+
+// CHECK: vpackusdw 4660(%rax,%r14,8), %xmm27, %xmm28
+// CHECK:  encoding: [0x62,0x22,0x25,0x00,0x2b,0xa4,0xf0,0x34,0x12,0x00,0x00]
+          vpackusdw 4660(%rax,%r14,8), %xmm27, %xmm28
+
+// CHECK: vpackusdw (%rcx){1to4}, %xmm27, %xmm28
+// CHECK:  encoding: [0x62,0x62,0x25,0x10,0x2b,0x21]
+          vpackusdw (%rcx){1to4}, %xmm27, %xmm28
+
+// CHECK: vpackusdw 2032(%rdx), %xmm27, %xmm28
+// CHECK:  encoding: [0x62,0x62,0x25,0x00,0x2b,0x62,0x7f]
+          vpackusdw 2032(%rdx), %xmm27, %xmm28
+
+// CHECK: vpackusdw 2048(%rdx), %xmm27, %xmm28
+// CHECK:  encoding: [0x62,0x62,0x25,0x00,0x2b,0xa2,0x00,0x08,0x00,0x00]
+          vpackusdw 2048(%rdx), %xmm27, %xmm28
+
+// CHECK: vpackusdw -2048(%rdx), %xmm27, %xmm28
+// CHECK:  encoding: [0x62,0x62,0x25,0x00,0x2b,0x62,0x80]
+          vpackusdw -2048(%rdx), %xmm27, %xmm28
+
+// CHECK: vpackusdw -2064(%rdx), %xmm27, %xmm28
+// CHECK:  encoding: [0x62,0x62,0x25,0x00,0x2b,0xa2,0xf0,0xf7,0xff,0xff]
+          vpackusdw -2064(%rdx), %xmm27, %xmm28
+
+// CHECK: vpackusdw 508(%rdx){1to4}, %xmm27, %xmm28
+// CHECK:  encoding: [0x62,0x62,0x25,0x10,0x2b,0x62,0x7f]
+          vpackusdw 508(%rdx){1to4}, %xmm27, %xmm28
+
+// CHECK: vpackusdw 512(%rdx){1to4}, %xmm27, %xmm28
+// CHECK:  encoding: [0x62,0x62,0x25,0x10,0x2b,0xa2,0x00,0x02,0x00,0x00]
+          vpackusdw 512(%rdx){1to4}, %xmm27, %xmm28
+
+// CHECK: vpackusdw -512(%rdx){1to4}, %xmm27, %xmm28
+// CHECK:  encoding: [0x62,0x62,0x25,0x10,0x2b,0x62,0x80]
+          vpackusdw -512(%rdx){1to4}, %xmm27, %xmm28
+
+// CHECK: vpackusdw -516(%rdx){1to4}, %xmm27, %xmm28
+// CHECK:  encoding: [0x62,0x62,0x25,0x10,0x2b,0xa2,0xfc,0xfd,0xff,0xff]
+          vpackusdw -516(%rdx){1to4}, %xmm27, %xmm28
+
+// CHECK: vpackusdw %ymm17, %ymm26, %ymm23
+// CHECK:  encoding: [0x62,0xa2,0x2d,0x20,0x2b,0xf9]
+          vpackusdw %ymm17, %ymm26, %ymm23
+
+// CHECK: vpackusdw %ymm17, %ymm26, %ymm23 {%k4}
+// CHECK:  encoding: [0x62,0xa2,0x2d,0x24,0x2b,0xf9]
+          vpackusdw %ymm17, %ymm26, %ymm23 {%k4}
+
+// CHECK: vpackusdw %ymm17, %ymm26, %ymm23 {%k4} {z}
+// CHECK:  encoding: [0x62,0xa2,0x2d,0xa4,0x2b,0xf9]
+          vpackusdw %ymm17, %ymm26, %ymm23 {%k4} {z}
+
+// CHECK: vpackusdw (%rcx), %ymm26, %ymm23
+// CHECK:  encoding: [0x62,0xe2,0x2d,0x20,0x2b,0x39]
+          vpackusdw (%rcx), %ymm26, %ymm23
+
+// CHECK: vpackusdw 4660(%rax,%r14,8), %ymm26, %ymm23
+// CHECK:  encoding: [0x62,0xa2,0x2d,0x20,0x2b,0xbc,0xf0,0x34,0x12,0x00,0x00]
+          vpackusdw 4660(%rax,%r14,8), %ymm26, %ymm23
+
+// CHECK: vpackusdw (%rcx){1to8}, %ymm26, %ymm23
+// CHECK:  encoding: [0x62,0xe2,0x2d,0x30,0x2b,0x39]
+          vpackusdw (%rcx){1to8}, %ymm26, %ymm23
+
+// CHECK: vpackusdw 4064(%rdx), %ymm26, %ymm23
+// CHECK:  encoding: [0x62,0xe2,0x2d,0x20,0x2b,0x7a,0x7f]
+          vpackusdw 4064(%rdx), %ymm26, %ymm23
+
+// CHECK: vpackusdw 4096(%rdx), %ymm26, %ymm23
+// CHECK:  encoding: [0x62,0xe2,0x2d,0x20,0x2b,0xba,0x00,0x10,0x00,0x00]
+          vpackusdw 4096(%rdx), %ymm26, %ymm23
+
+// CHECK: vpackusdw -4096(%rdx), %ymm26, %ymm23
+// CHECK:  encoding: [0x62,0xe2,0x2d,0x20,0x2b,0x7a,0x80]
+          vpackusdw -4096(%rdx), %ymm26, %ymm23
+
+// CHECK: vpackusdw -4128(%rdx), %ymm26, %ymm23
+// CHECK:  encoding: [0x62,0xe2,0x2d,0x20,0x2b,0xba,0xe0,0xef,0xff,0xff]
+          vpackusdw -4128(%rdx), %ymm26, %ymm23
+
+// CHECK: vpackusdw 508(%rdx){1to8}, %ymm26, %ymm23
+// CHECK:  encoding: [0x62,0xe2,0x2d,0x30,0x2b,0x7a,0x7f]
+          vpackusdw 508(%rdx){1to8}, %ymm26, %ymm23
+
+// CHECK: vpackusdw 512(%rdx){1to8}, %ymm26, %ymm23
+// CHECK:  encoding: [0x62,0xe2,0x2d,0x30,0x2b,0xba,0x00,0x02,0x00,0x00]
+          vpackusdw 512(%rdx){1to8}, %ymm26, %ymm23
+
+// CHECK: vpackusdw -512(%rdx){1to8}, %ymm26, %ymm23
+// CHECK:  encoding: [0x62,0xe2,0x2d,0x30,0x2b,0x7a,0x80]
+          vpackusdw -512(%rdx){1to8}, %ymm26, %ymm23
+
+// CHECK: vpackusdw -516(%rdx){1to8}, %ymm26, %ymm23
+// CHECK:  encoding: [0x62,0xe2,0x2d,0x30,0x2b,0xba,0xfc,0xfd,0xff,0xff]
+          vpackusdw -516(%rdx){1to8}, %ymm26, %ymm23
+
+// CHECK: vpackuswb %xmm23, %xmm20, %xmm20
+// CHECK:  encoding: [0x62,0xa1,0xdd,0x00,0x67,0xe7]
+          vpackuswb %xmm23, %xmm20, %xmm20
+
+// CHECK: vpackuswb %xmm23, %xmm20, %xmm20 {%k2}
+// CHECK:  encoding: [0x62,0xa1,0xdd,0x02,0x67,0xe7]
+          vpackuswb %xmm23, %xmm20, %xmm20 {%k2}
+
+// CHECK: vpackuswb %xmm23, %xmm20, %xmm20 {%k2} {z}
+// CHECK:  encoding: [0x62,0xa1,0xdd,0x82,0x67,0xe7]
+          vpackuswb %xmm23, %xmm20, %xmm20 {%k2} {z}
+
+// CHECK: vpackuswb (%rcx), %xmm20, %xmm20
+// CHECK:  encoding: [0x62,0xe1,0xdd,0x00,0x67,0x21]
+          vpackuswb (%rcx), %xmm20, %xmm20
+
+// CHECK: vpackuswb 4660(%rax,%r14,8), %xmm20, %xmm20
+// CHECK:  encoding: [0x62,0xa1,0xdd,0x00,0x67,0xa4,0xf0,0x34,0x12,0x00,0x00]
+          vpackuswb 4660(%rax,%r14,8), %xmm20, %xmm20
+
+// CHECK: vpackuswb 2032(%rdx), %xmm20, %xmm20
+// CHECK:  encoding: [0x62,0xe1,0xdd,0x00,0x67,0x62,0x7f]
+          vpackuswb 2032(%rdx), %xmm20, %xmm20
+
+// CHECK: vpackuswb 2048(%rdx), %xmm20, %xmm20
+// CHECK:  encoding: [0x62,0xe1,0xdd,0x00,0x67,0xa2,0x00,0x08,0x00,0x00]
+          vpackuswb 2048(%rdx), %xmm20, %xmm20
+
+// CHECK: vpackuswb -2048(%rdx), %xmm20, %xmm20
+// CHECK:  encoding: [0x62,0xe1,0xdd,0x00,0x67,0x62,0x80]
+          vpackuswb -2048(%rdx), %xmm20, %xmm20
+
+// CHECK: vpackuswb -2064(%rdx), %xmm20, %xmm20
+// CHECK:  encoding: [0x62,0xe1,0xdd,0x00,0x67,0xa2,0xf0,0xf7,0xff,0xff]
+          vpackuswb -2064(%rdx), %xmm20, %xmm20
+
+// CHECK: vpackuswb %ymm27, %ymm19, %ymm20
+// CHECK:  encoding: [0x62,0x81,0xe5,0x20,0x67,0xe3]
+          vpackuswb %ymm27, %ymm19, %ymm20
+
+// CHECK: vpackuswb %ymm27, %ymm19, %ymm20 {%k5}
+// CHECK:  encoding: [0x62,0x81,0xe5,0x25,0x67,0xe3]
+          vpackuswb %ymm27, %ymm19, %ymm20 {%k5}
+
+// CHECK: vpackuswb %ymm27, %ymm19, %ymm20 {%k5} {z}
+// CHECK:  encoding: [0x62,0x81,0xe5,0xa5,0x67,0xe3]
+          vpackuswb %ymm27, %ymm19, %ymm20 {%k5} {z}
+
+// CHECK: vpackuswb (%rcx), %ymm19, %ymm20
+// CHECK:  encoding: [0x62,0xe1,0xe5,0x20,0x67,0x21]
+          vpackuswb (%rcx), %ymm19, %ymm20
+
+// CHECK: vpackuswb 4660(%rax,%r14,8), %ymm19, %ymm20
+// CHECK:  encoding: [0x62,0xa1,0xe5,0x20,0x67,0xa4,0xf0,0x34,0x12,0x00,0x00]
+          vpackuswb 4660(%rax,%r14,8), %ymm19, %ymm20
+
+// CHECK: vpackuswb 4064(%rdx), %ymm19, %ymm20
+// CHECK:  encoding: [0x62,0xe1,0xe5,0x20,0x67,0x62,0x7f]
+          vpackuswb 4064(%rdx), %ymm19, %ymm20
+
+// CHECK: vpackuswb 4096(%rdx), %ymm19, %ymm20
+// CHECK:  encoding: [0x62,0xe1,0xe5,0x20,0x67,0xa2,0x00,0x10,0x00,0x00]
+          vpackuswb 4096(%rdx), %ymm19, %ymm20
+
+// CHECK: vpackuswb -4096(%rdx), %ymm19, %ymm20
+// CHECK:  encoding: [0x62,0xe1,0xe5,0x20,0x67,0x62,0x80]
+          vpackuswb -4096(%rdx), %ymm19, %ymm20
+
+// CHECK: vpackuswb -4128(%rdx), %ymm19, %ymm20
+// CHECK:  encoding: [0x62,0xe1,0xe5,0x20,0x67,0xa2,0xe0,0xef,0xff,0xff]
+          vpackuswb -4128(%rdx), %ymm19, %ymm20
+
+// CHECK: vpaddsb %xmm27, %xmm19, %xmm19
+// CHECK:  encoding: [0x62,0x81,0x65,0x00,0xec,0xdb]
+          vpaddsb %xmm27, %xmm19, %xmm19
+
+// CHECK: vpaddsb %xmm27, %xmm19, %xmm19 {%k2}
+// CHECK:  encoding: [0x62,0x81,0x65,0x02,0xec,0xdb]
+          vpaddsb %xmm27, %xmm19, %xmm19 {%k2}
+
+// CHECK: vpaddsb %xmm27, %xmm19, %xmm19 {%k2} {z}
+// CHECK:  encoding: [0x62,0x81,0x65,0x82,0xec,0xdb]
+          vpaddsb %xmm27, %xmm19, %xmm19 {%k2} {z}
+
+// CHECK: vpaddsb (%rcx), %xmm19, %xmm19
+// CHECK:  encoding: [0x62,0xe1,0x65,0x00,0xec,0x19]
+          vpaddsb (%rcx), %xmm19, %xmm19
+
+// CHECK: vpaddsb 291(%rax,%r14,8), %xmm19, %xmm19
+// CHECK:  encoding: [0x62,0xa1,0x65,0x00,0xec,0x9c,0xf0,0x23,0x01,0x00,0x00]
+          vpaddsb 291(%rax,%r14,8), %xmm19, %xmm19
+
+// CHECK: vpaddsb 2032(%rdx), %xmm19, %xmm19
+// CHECK:  encoding: [0x62,0xe1,0x65,0x00,0xec,0x5a,0x7f]
+          vpaddsb 2032(%rdx), %xmm19, %xmm19
+
+// CHECK: vpaddsb 2048(%rdx), %xmm19, %xmm19
+// CHECK:  encoding: [0x62,0xe1,0x65,0x00,0xec,0x9a,0x00,0x08,0x00,0x00]
+          vpaddsb 2048(%rdx), %xmm19, %xmm19
+
+// CHECK: vpaddsb -2048(%rdx), %xmm19, %xmm19
+// CHECK:  encoding: [0x62,0xe1,0x65,0x00,0xec,0x5a,0x80]
+          vpaddsb -2048(%rdx), %xmm19, %xmm19
+
+// CHECK: vpaddsb -2064(%rdx), %xmm19, %xmm19
+// CHECK:  encoding: [0x62,0xe1,0x65,0x00,0xec,0x9a,0xf0,0xf7,0xff,0xff]
+          vpaddsb -2064(%rdx), %xmm19, %xmm19
+
+// CHECK: vpaddsb %ymm27, %ymm21, %ymm19
+// CHECK:  encoding: [0x62,0x81,0x55,0x20,0xec,0xdb]
+          vpaddsb %ymm27, %ymm21, %ymm19
+
+// CHECK: vpaddsb %ymm27, %ymm21, %ymm19 {%k4}
+// CHECK:  encoding: [0x62,0x81,0x55,0x24,0xec,0xdb]
+          vpaddsb %ymm27, %ymm21, %ymm19 {%k4}
+
+// CHECK: vpaddsb %ymm27, %ymm21, %ymm19 {%k4} {z}
+// CHECK:  encoding: [0x62,0x81,0x55,0xa4,0xec,0xdb]
+          vpaddsb %ymm27, %ymm21, %ymm19 {%k4} {z}
+
+// CHECK: vpaddsb (%rcx), %ymm21, %ymm19
+// CHECK:  encoding: [0x62,0xe1,0x55,0x20,0xec,0x19]
+          vpaddsb (%rcx), %ymm21, %ymm19
+
+// CHECK: vpaddsb 291(%rax,%r14,8), %ymm21, %ymm19
+// CHECK:  encoding: [0x62,0xa1,0x55,0x20,0xec,0x9c,0xf0,0x23,0x01,0x00,0x00]
+          vpaddsb 291(%rax,%r14,8), %ymm21, %ymm19
+
+// CHECK: vpaddsb 4064(%rdx), %ymm21, %ymm19
+// CHECK:  encoding: [0x62,0xe1,0x55,0x20,0xec,0x5a,0x7f]
+          vpaddsb 4064(%rdx), %ymm21, %ymm19
+
+// CHECK: vpaddsb 4096(%rdx), %ymm21, %ymm19
+// CHECK:  encoding: [0x62,0xe1,0x55,0x20,0xec,0x9a,0x00,0x10,0x00,0x00]
+          vpaddsb 4096(%rdx), %ymm21, %ymm19
+
+// CHECK: vpaddsb -4096(%rdx), %ymm21, %ymm19
+// CHECK:  encoding: [0x62,0xe1,0x55,0x20,0xec,0x5a,0x80]
+          vpaddsb -4096(%rdx), %ymm21, %ymm19
+
+// CHECK: vpaddsb -4128(%rdx), %ymm21, %ymm19
+// CHECK:  encoding: [0x62,0xe1,0x55,0x20,0xec,0x9a,0xe0,0xef,0xff,0xff]
+          vpaddsb -4128(%rdx), %ymm21, %ymm19
+
+// CHECK: vpaddsw %xmm21, %xmm27, %xmm29
+// CHECK:  encoding: [0x62,0x21,0x25,0x00,0xed,0xed]
+          vpaddsw %xmm21, %xmm27, %xmm29
+
+// CHECK: vpaddsw %xmm21, %xmm27, %xmm29 {%k1}
+// CHECK:  encoding: [0x62,0x21,0x25,0x01,0xed,0xed]
+          vpaddsw %xmm21, %xmm27, %xmm29 {%k1}
+
+// CHECK: vpaddsw %xmm21, %xmm27, %xmm29 {%k1} {z}
+// CHECK:  encoding: [0x62,0x21,0x25,0x81,0xed,0xed]
+          vpaddsw %xmm21, %xmm27, %xmm29 {%k1} {z}
+
+// CHECK: vpaddsw (%rcx), %xmm27, %xmm29
+// CHECK:  encoding: [0x62,0x61,0x25,0x00,0xed,0x29]
+          vpaddsw (%rcx), %xmm27, %xmm29
+
+// CHECK: vpaddsw 291(%rax,%r14,8), %xmm27, %xmm29
+// CHECK:  encoding: [0x62,0x21,0x25,0x00,0xed,0xac,0xf0,0x23,0x01,0x00,0x00]
+          vpaddsw 291(%rax,%r14,8), %xmm27, %xmm29
+
+// CHECK: vpaddsw 2032(%rdx), %xmm27, %xmm29
+// CHECK:  encoding: [0x62,0x61,0x25,0x00,0xed,0x6a,0x7f]
+          vpaddsw 2032(%rdx), %xmm27, %xmm29
+
+// CHECK: vpaddsw 2048(%rdx), %xmm27, %xmm29
+// CHECK:  encoding: [0x62,0x61,0x25,0x00,0xed,0xaa,0x00,0x08,0x00,0x00]
+          vpaddsw 2048(%rdx), %xmm27, %xmm29
+
+// CHECK: vpaddsw -2048(%rdx), %xmm27, %xmm29
+// CHECK:  encoding: [0x62,0x61,0x25,0x00,0xed,0x6a,0x80]
+          vpaddsw -2048(%rdx), %xmm27, %xmm29
+
+// CHECK: vpaddsw -2064(%rdx), %xmm27, %xmm29
+// CHECK:  encoding: [0x62,0x61,0x25,0x00,0xed,0xaa,0xf0,0xf7,0xff,0xff]
+          vpaddsw -2064(%rdx), %xmm27, %xmm29
+
+// CHECK: vpaddsw %ymm26, %ymm17, %ymm22
+// CHECK:  encoding: [0x62,0x81,0x75,0x20,0xed,0xf2]
+          vpaddsw %ymm26, %ymm17, %ymm22
+
+// CHECK: vpaddsw %ymm26, %ymm17, %ymm22 {%k3}
+// CHECK:  encoding: [0x62,0x81,0x75,0x23,0xed,0xf2]
+          vpaddsw %ymm26, %ymm17, %ymm22 {%k3}
+
+// CHECK: vpaddsw %ymm26, %ymm17, %ymm22 {%k3} {z}
+// CHECK:  encoding: [0x62,0x81,0x75,0xa3,0xed,0xf2]
+          vpaddsw %ymm26, %ymm17, %ymm22 {%k3} {z}
+
+// CHECK: vpaddsw (%rcx), %ymm17, %ymm22
+// CHECK:  encoding: [0x62,0xe1,0x75,0x20,0xed,0x31]
+          vpaddsw (%rcx), %ymm17, %ymm22
+
+// CHECK: vpaddsw 291(%rax,%r14,8), %ymm17, %ymm22
+// CHECK:  encoding: [0x62,0xa1,0x75,0x20,0xed,0xb4,0xf0,0x23,0x01,0x00,0x00]
+          vpaddsw 291(%rax,%r14,8), %ymm17, %ymm22
+
+// CHECK: vpaddsw 4064(%rdx), %ymm17, %ymm22
+// CHECK:  encoding: [0x62,0xe1,0x75,0x20,0xed,0x72,0x7f]
+          vpaddsw 4064(%rdx), %ymm17, %ymm22
+
+// CHECK: vpaddsw 4096(%rdx), %ymm17, %ymm22
+// CHECK:  encoding: [0x62,0xe1,0x75,0x20,0xed,0xb2,0x00,0x10,0x00,0x00]
+          vpaddsw 4096(%rdx), %ymm17, %ymm22
+
+// CHECK: vpaddsw -4096(%rdx), %ymm17, %ymm22
+// CHECK:  encoding: [0x62,0xe1,0x75,0x20,0xed,0x72,0x80]
+          vpaddsw -4096(%rdx), %ymm17, %ymm22
+
+// CHECK: vpaddsw -4128(%rdx), %ymm17, %ymm22
+// CHECK:  encoding: [0x62,0xe1,0x75,0x20,0xed,0xb2,0xe0,0xef,0xff,0xff]
+          vpaddsw -4128(%rdx), %ymm17, %ymm22
+
+// CHECK: vpaddusb %xmm26, %xmm28, %xmm21
+// CHECK:  encoding: [0x62,0x81,0x1d,0x00,0xdc,0xea]
+          vpaddusb %xmm26, %xmm28, %xmm21
+
+// CHECK: vpaddusb %xmm26, %xmm28, %xmm21 {%k5}
+// CHECK:  encoding: [0x62,0x81,0x1d,0x05,0xdc,0xea]
+          vpaddusb %xmm26, %xmm28, %xmm21 {%k5}
+
+// CHECK: vpaddusb %xmm26, %xmm28, %xmm21 {%k5} {z}
+// CHECK:  encoding: [0x62,0x81,0x1d,0x85,0xdc,0xea]
+          vpaddusb %xmm26, %xmm28, %xmm21 {%k5} {z}
+
+// CHECK: vpaddusb (%rcx), %xmm28, %xmm21
+// CHECK:  encoding: [0x62,0xe1,0x1d,0x00,0xdc,0x29]
+          vpaddusb (%rcx), %xmm28, %xmm21
+
+// CHECK: vpaddusb 291(%rax,%r14,8), %xmm28, %xmm21
+// CHECK:  encoding: [0x62,0xa1,0x1d,0x00,0xdc,0xac,0xf0,0x23,0x01,0x00,0x00]
+          vpaddusb 291(%rax,%r14,8), %xmm28, %xmm21
+
+// CHECK: vpaddusb 2032(%rdx), %xmm28, %xmm21
+// CHECK:  encoding: [0x62,0xe1,0x1d,0x00,0xdc,0x6a,0x7f]
+          vpaddusb 2032(%rdx), %xmm28, %xmm21
+
+// CHECK: vpaddusb 2048(%rdx), %xmm28, %xmm21
+// CHECK:  encoding: [0x62,0xe1,0x1d,0x00,0xdc,0xaa,0x00,0x08,0x00,0x00]
+          vpaddusb 2048(%rdx), %xmm28, %xmm21
+
+// CHECK: vpaddusb -2048(%rdx), %xmm28, %xmm21
+// CHECK:  encoding: [0x62,0xe1,0x1d,0x00,0xdc,0x6a,0x80]
+          vpaddusb -2048(%rdx), %xmm28, %xmm21
+
+// CHECK: vpaddusb -2064(%rdx), %xmm28, %xmm21
+// CHECK:  encoding: [0x62,0xe1,0x1d,0x00,0xdc,0xaa,0xf0,0xf7,0xff,0xff]
+          vpaddusb -2064(%rdx), %xmm28, %xmm21
+
+// CHECK: vpaddusb %ymm17, %ymm23, %ymm20
+// CHECK:  encoding: [0x62,0xa1,0x45,0x20,0xdc,0xe1]
+          vpaddusb %ymm17, %ymm23, %ymm20
+
+// CHECK: vpaddusb %ymm17, %ymm23, %ymm20 {%k1}
+// CHECK:  encoding: [0x62,0xa1,0x45,0x21,0xdc,0xe1]
+          vpaddusb %ymm17, %ymm23, %ymm20 {%k1}
+
+// CHECK: vpaddusb %ymm17, %ymm23, %ymm20 {%k1} {z}
+// CHECK:  encoding: [0x62,0xa1,0x45,0xa1,0xdc,0xe1]
+          vpaddusb %ymm17, %ymm23, %ymm20 {%k1} {z}
+
+// CHECK: vpaddusb (%rcx), %ymm23, %ymm20
+// CHECK:  encoding: [0x62,0xe1,0x45,0x20,0xdc,0x21]
+          vpaddusb (%rcx), %ymm23, %ymm20
+
+// CHECK: vpaddusb 291(%rax,%r14,8), %ymm23, %ymm20
+// CHECK:  encoding: [0x62,0xa1,0x45,0x20,0xdc,0xa4,0xf0,0x23,0x01,0x00,0x00]
+          vpaddusb 291(%rax,%r14,8), %ymm23, %ymm20
+
+// CHECK: vpaddusb 4064(%rdx), %ymm23, %ymm20
+// CHECK:  encoding: [0x62,0xe1,0x45,0x20,0xdc,0x62,0x7f]
+          vpaddusb 4064(%rdx), %ymm23, %ymm20
+
+// CHECK: vpaddusb 4096(%rdx), %ymm23, %ymm20
+// CHECK:  encoding: [0x62,0xe1,0x45,0x20,0xdc,0xa2,0x00,0x10,0x00,0x00]
+          vpaddusb 4096(%rdx), %ymm23, %ymm20
+
+// CHECK: vpaddusb -4096(%rdx), %ymm23, %ymm20
+// CHECK:  encoding: [0x62,0xe1,0x45,0x20,0xdc,0x62,0x80]
+          vpaddusb -4096(%rdx), %ymm23, %ymm20
+
+// CHECK: vpaddusb -4128(%rdx), %ymm23, %ymm20
+// CHECK:  encoding: [0x62,0xe1,0x45,0x20,0xdc,0xa2,0xe0,0xef,0xff,0xff]
+          vpaddusb -4128(%rdx), %ymm23, %ymm20
+
+// CHECK: vpaddusw %xmm24, %xmm27, %xmm23
+// CHECK:  encoding: [0x62,0x81,0x25,0x00,0xdd,0xf8]
+          vpaddusw %xmm24, %xmm27, %xmm23
+
+// CHECK: vpaddusw %xmm24, %xmm27, %xmm23 {%k2}
+// CHECK:  encoding: [0x62,0x81,0x25,0x02,0xdd,0xf8]
+          vpaddusw %xmm24, %xmm27, %xmm23 {%k2}
+
+// CHECK: vpaddusw %xmm24, %xmm27, %xmm23 {%k2} {z}
+// CHECK:  encoding: [0x62,0x81,0x25,0x82,0xdd,0xf8]
+          vpaddusw %xmm24, %xmm27, %xmm23 {%k2} {z}
+
+// CHECK: vpaddusw (%rcx), %xmm27, %xmm23
+// CHECK:  encoding: [0x62,0xe1,0x25,0x00,0xdd,0x39]
+          vpaddusw (%rcx), %xmm27, %xmm23
+
+// CHECK: vpaddusw 291(%rax,%r14,8), %xmm27, %xmm23
+// CHECK:  encoding: [0x62,0xa1,0x25,0x00,0xdd,0xbc,0xf0,0x23,0x01,0x00,0x00]
+          vpaddusw 291(%rax,%r14,8), %xmm27, %xmm23
+
+// CHECK: vpaddusw 2032(%rdx), %xmm27, %xmm23
+// CHECK:  encoding: [0x62,0xe1,0x25,0x00,0xdd,0x7a,0x7f]
+          vpaddusw 2032(%rdx), %xmm27, %xmm23
+
+// CHECK: vpaddusw 2048(%rdx), %xmm27, %xmm23
+// CHECK:  encoding: [0x62,0xe1,0x25,0x00,0xdd,0xba,0x00,0x08,0x00,0x00]
+          vpaddusw 2048(%rdx), %xmm27, %xmm23
+
+// CHECK: vpaddusw -2048(%rdx), %xmm27, %xmm23
+// CHECK:  encoding: [0x62,0xe1,0x25,0x00,0xdd,0x7a,0x80]
+          vpaddusw -2048(%rdx), %xmm27, %xmm23
+
+// CHECK: vpaddusw -2064(%rdx), %xmm27, %xmm23
+// CHECK:  encoding: [0x62,0xe1,0x25,0x00,0xdd,0xba,0xf0,0xf7,0xff,0xff]
+          vpaddusw -2064(%rdx), %xmm27, %xmm23
+
+// CHECK: vpaddusw %ymm17, %ymm22, %ymm23
+// CHECK:  encoding: [0x62,0xa1,0x4d,0x20,0xdd,0xf9]
+          vpaddusw %ymm17, %ymm22, %ymm23
+
+// CHECK: vpaddusw %ymm17, %ymm22, %ymm23 {%k1}
+// CHECK:  encoding: [0x62,0xa1,0x4d,0x21,0xdd,0xf9]
+          vpaddusw %ymm17, %ymm22, %ymm23 {%k1}
+
+// CHECK: vpaddusw %ymm17, %ymm22, %ymm23 {%k1} {z}
+// CHECK:  encoding: [0x62,0xa1,0x4d,0xa1,0xdd,0xf9]
+          vpaddusw %ymm17, %ymm22, %ymm23 {%k1} {z}
+
+// CHECK: vpaddusw (%rcx), %ymm22, %ymm23
+// CHECK:  encoding: [0x62,0xe1,0x4d,0x20,0xdd,0x39]
+          vpaddusw (%rcx), %ymm22, %ymm23
+
+// CHECK: vpaddusw 291(%rax,%r14,8), %ymm22, %ymm23
+// CHECK:  encoding: [0x62,0xa1,0x4d,0x20,0xdd,0xbc,0xf0,0x23,0x01,0x00,0x00]
+          vpaddusw 291(%rax,%r14,8), %ymm22, %ymm23
+
+// CHECK: vpaddusw 4064(%rdx), %ymm22, %ymm23
+// CHECK:  encoding: [0x62,0xe1,0x4d,0x20,0xdd,0x7a,0x7f]
+          vpaddusw 4064(%rdx), %ymm22, %ymm23
+
+// CHECK: vpaddusw 4096(%rdx), %ymm22, %ymm23
+// CHECK:  encoding: [0x62,0xe1,0x4d,0x20,0xdd,0xba,0x00,0x10,0x00,0x00]
+          vpaddusw 4096(%rdx), %ymm22, %ymm23
+
+// CHECK: vpaddusw -4096(%rdx), %ymm22, %ymm23
+// CHECK:  encoding: [0x62,0xe1,0x4d,0x20,0xdd,0x7a,0x80]
+          vpaddusw -4096(%rdx), %ymm22, %ymm23
+
+// CHECK: vpaddusw -4128(%rdx), %ymm22, %ymm23
+// CHECK:  encoding: [0x62,0xe1,0x4d,0x20,0xdd,0xba,0xe0,0xef,0xff,0xff]
+          vpaddusw -4128(%rdx), %ymm22, %ymm23
+
+// CHECK: vpsubsb %xmm27, %xmm17, %xmm17
+// CHECK:  encoding: [0x62,0x81,0x75,0x00,0xe8,0xcb]
+          vpsubsb %xmm27, %xmm17, %xmm17
+
+// CHECK: vpsubsb %xmm27, %xmm17, %xmm17 {%k7}
+// CHECK:  encoding: [0x62,0x81,0x75,0x07,0xe8,0xcb]
+          vpsubsb %xmm27, %xmm17, %xmm17 {%k7}
+
+// CHECK: vpsubsb %xmm27, %xmm17, %xmm17 {%k7} {z}
+// CHECK:  encoding: [0x62,0x81,0x75,0x87,0xe8,0xcb]
+          vpsubsb %xmm27, %xmm17, %xmm17 {%k7} {z}
+
+// CHECK: vpsubsb (%rcx), %xmm17, %xmm17
+// CHECK:  encoding: [0x62,0xe1,0x75,0x00,0xe8,0x09]
+          vpsubsb (%rcx), %xmm17, %xmm17
+
+// CHECK: vpsubsb 291(%rax,%r14,8), %xmm17, %xmm17
+// CHECK:  encoding: [0x62,0xa1,0x75,0x00,0xe8,0x8c,0xf0,0x23,0x01,0x00,0x00]
+          vpsubsb 291(%rax,%r14,8), %xmm17, %xmm17
+
+// CHECK: vpsubsb 2032(%rdx), %xmm17, %xmm17
+// CHECK:  encoding: [0x62,0xe1,0x75,0x00,0xe8,0x4a,0x7f]
+          vpsubsb 2032(%rdx), %xmm17, %xmm17
+
+// CHECK: vpsubsb 2048(%rdx), %xmm17, %xmm17
+// CHECK:  encoding: [0x62,0xe1,0x75,0x00,0xe8,0x8a,0x00,0x08,0x00,0x00]
+          vpsubsb 2048(%rdx), %xmm17, %xmm17
+
+// CHECK: vpsubsb -2048(%rdx), %xmm17, %xmm17
+// CHECK:  encoding: [0x62,0xe1,0x75,0x00,0xe8,0x4a,0x80]
+          vpsubsb -2048(%rdx), %xmm17, %xmm17
+
+// CHECK: vpsubsb -2064(%rdx), %xmm17, %xmm17
+// CHECK:  encoding: [0x62,0xe1,0x75,0x00,0xe8,0x8a,0xf0,0xf7,0xff,0xff]
+          vpsubsb -2064(%rdx), %xmm17, %xmm17
+
+// CHECK: vpsubsb %ymm27, %ymm25, %ymm17
+// CHECK:  encoding: [0x62,0x81,0x35,0x20,0xe8,0xcb]
+          vpsubsb %ymm27, %ymm25, %ymm17
+
+// CHECK: vpsubsb %ymm27, %ymm25, %ymm17 {%k2}
+// CHECK:  encoding: [0x62,0x81,0x35,0x22,0xe8,0xcb]
+          vpsubsb %ymm27, %ymm25, %ymm17 {%k2}
+
+// CHECK: vpsubsb %ymm27, %ymm25, %ymm17 {%k2} {z}
+// CHECK:  encoding: [0x62,0x81,0x35,0xa2,0xe8,0xcb]
+          vpsubsb %ymm27, %ymm25, %ymm17 {%k2} {z}
+
+// CHECK: vpsubsb (%rcx), %ymm25, %ymm17
+// CHECK:  encoding: [0x62,0xe1,0x35,0x20,0xe8,0x09]
+          vpsubsb (%rcx), %ymm25, %ymm17
+
+// CHECK: vpsubsb 291(%rax,%r14,8), %ymm25, %ymm17
+// CHECK:  encoding: [0x62,0xa1,0x35,0x20,0xe8,0x8c,0xf0,0x23,0x01,0x00,0x00]
+          vpsubsb 291(%rax,%r14,8), %ymm25, %ymm17
+
+// CHECK: vpsubsb 4064(%rdx), %ymm25, %ymm17
+// CHECK:  encoding: [0x62,0xe1,0x35,0x20,0xe8,0x4a,0x7f]
+          vpsubsb 4064(%rdx), %ymm25, %ymm17
+
+// CHECK: vpsubsb 4096(%rdx), %ymm25, %ymm17
+// CHECK:  encoding: [0x62,0xe1,0x35,0x20,0xe8,0x8a,0x00,0x10,0x00,0x00]
+          vpsubsb 4096(%rdx), %ymm25, %ymm17
+
+// CHECK: vpsubsb -4096(%rdx), %ymm25, %ymm17
+// CHECK:  encoding: [0x62,0xe1,0x35,0x20,0xe8,0x4a,0x80]
+          vpsubsb -4096(%rdx), %ymm25, %ymm17
+
+// CHECK: vpsubsb -4128(%rdx), %ymm25, %ymm17
+// CHECK:  encoding: [0x62,0xe1,0x35,0x20,0xe8,0x8a,0xe0,0xef,0xff,0xff]
+          vpsubsb -4128(%rdx), %ymm25, %ymm17
+
+// CHECK: vpsubsw %xmm23, %xmm26, %xmm30
+// CHECK:  encoding: [0x62,0x21,0x2d,0x00,0xe9,0xf7]
+          vpsubsw %xmm23, %xmm26, %xmm30
+
+// CHECK: vpsubsw %xmm23, %xmm26, %xmm30 {%k7}
+// CHECK:  encoding: [0x62,0x21,0x2d,0x07,0xe9,0xf7]
+          vpsubsw %xmm23, %xmm26, %xmm30 {%k7}
+
+// CHECK: vpsubsw %xmm23, %xmm26, %xmm30 {%k7} {z}
+// CHECK:  encoding: [0x62,0x21,0x2d,0x87,0xe9,0xf7]
+          vpsubsw %xmm23, %xmm26, %xmm30 {%k7} {z}
+
+// CHECK: vpsubsw (%rcx), %xmm26, %xmm30
+// CHECK:  encoding: [0x62,0x61,0x2d,0x00,0xe9,0x31]
+          vpsubsw (%rcx), %xmm26, %xmm30
+
+// CHECK: vpsubsw 291(%rax,%r14,8), %xmm26, %xmm30
+// CHECK:  encoding: [0x62,0x21,0x2d,0x00,0xe9,0xb4,0xf0,0x23,0x01,0x00,0x00]
+          vpsubsw 291(%rax,%r14,8), %xmm26, %xmm30
+
+// CHECK: vpsubsw 2032(%rdx), %xmm26, %xmm30
+// CHECK:  encoding: [0x62,0x61,0x2d,0x00,0xe9,0x72,0x7f]
+          vpsubsw 2032(%rdx), %xmm26, %xmm30
+
+// CHECK: vpsubsw 2048(%rdx), %xmm26, %xmm30
+// CHECK:  encoding: [0x62,0x61,0x2d,0x00,0xe9,0xb2,0x00,0x08,0x00,0x00]
+          vpsubsw 2048(%rdx), %xmm26, %xmm30
+
+// CHECK: vpsubsw -2048(%rdx), %xmm26, %xmm30
+// CHECK:  encoding: [0x62,0x61,0x2d,0x00,0xe9,0x72,0x80]
+          vpsubsw -2048(%rdx), %xmm26, %xmm30
+
+// CHECK: vpsubsw -2064(%rdx), %xmm26, %xmm30
+// CHECK:  encoding: [0x62,0x61,0x2d,0x00,0xe9,0xb2,0xf0,0xf7,0xff,0xff]
+          vpsubsw -2064(%rdx), %xmm26, %xmm30
+
+// CHECK: vpsubsw %ymm17, %ymm29, %ymm19
+// CHECK:  encoding: [0x62,0xa1,0x15,0x20,0xe9,0xd9]
+          vpsubsw %ymm17, %ymm29, %ymm19
+
+// CHECK: vpsubsw %ymm17, %ymm29, %ymm19 {%k2}
+// CHECK:  encoding: [0x62,0xa1,0x15,0x22,0xe9,0xd9]
+          vpsubsw %ymm17, %ymm29, %ymm19 {%k2}
+
+// CHECK: vpsubsw %ymm17, %ymm29, %ymm19 {%k2} {z}
+// CHECK:  encoding: [0x62,0xa1,0x15,0xa2,0xe9,0xd9]
+          vpsubsw %ymm17, %ymm29, %ymm19 {%k2} {z}
+
+// CHECK: vpsubsw (%rcx), %ymm29, %ymm19
+// CHECK:  encoding: [0x62,0xe1,0x15,0x20,0xe9,0x19]
+          vpsubsw (%rcx), %ymm29, %ymm19
+
+// CHECK: vpsubsw 291(%rax,%r14,8), %ymm29, %ymm19
+// CHECK:  encoding: [0x62,0xa1,0x15,0x20,0xe9,0x9c,0xf0,0x23,0x01,0x00,0x00]
+          vpsubsw 291(%rax,%r14,8), %ymm29, %ymm19
+
+// CHECK: vpsubsw 4064(%rdx), %ymm29, %ymm19
+// CHECK:  encoding: [0x62,0xe1,0x15,0x20,0xe9,0x5a,0x7f]
+          vpsubsw 4064(%rdx), %ymm29, %ymm19
+
+// CHECK: vpsubsw 4096(%rdx), %ymm29, %ymm19
+// CHECK:  encoding: [0x62,0xe1,0x15,0x20,0xe9,0x9a,0x00,0x10,0x00,0x00]
+          vpsubsw 4096(%rdx), %ymm29, %ymm19
+
+// CHECK: vpsubsw -4096(%rdx), %ymm29, %ymm19
+// CHECK:  encoding: [0x62,0xe1,0x15,0x20,0xe9,0x5a,0x80]
+          vpsubsw -4096(%rdx), %ymm29, %ymm19
+
+// CHECK: vpsubsw -4128(%rdx), %ymm29, %ymm19
+// CHECK:  encoding: [0x62,0xe1,0x15,0x20,0xe9,0x9a,0xe0,0xef,0xff,0xff]
+          vpsubsw -4128(%rdx), %ymm29, %ymm19
+
+// CHECK: vpsubusb %xmm23, %xmm27, %xmm17
+// CHECK:  encoding: [0x62,0xa1,0x25,0x00,0xd8,0xcf]
+          vpsubusb %xmm23, %xmm27, %xmm17
+
+// CHECK: vpsubusb %xmm23, %xmm27, %xmm17 {%k6}
+// CHECK:  encoding: [0x62,0xa1,0x25,0x06,0xd8,0xcf]
+          vpsubusb %xmm23, %xmm27, %xmm17 {%k6}
+
+// CHECK: vpsubusb %xmm23, %xmm27, %xmm17 {%k6} {z}
+// CHECK:  encoding: [0x62,0xa1,0x25,0x86,0xd8,0xcf]
+          vpsubusb %xmm23, %xmm27, %xmm17 {%k6} {z}
+
+// CHECK: vpsubusb (%rcx), %xmm27, %xmm17
+// CHECK:  encoding: [0x62,0xe1,0x25,0x00,0xd8,0x09]
+          vpsubusb (%rcx), %xmm27, %xmm17
+
+// CHECK: vpsubusb 291(%rax,%r14,8), %xmm27, %xmm17
+// CHECK:  encoding: [0x62,0xa1,0x25,0x00,0xd8,0x8c,0xf0,0x23,0x01,0x00,0x00]
+          vpsubusb 291(%rax,%r14,8), %xmm27, %xmm17
+
+// CHECK: vpsubusb 2032(%rdx), %xmm27, %xmm17
+// CHECK:  encoding: [0x62,0xe1,0x25,0x00,0xd8,0x4a,0x7f]
+          vpsubusb 2032(%rdx), %xmm27, %xmm17
+
+// CHECK: vpsubusb 2048(%rdx), %xmm27, %xmm17
+// CHECK:  encoding: [0x62,0xe1,0x25,0x00,0xd8,0x8a,0x00,0x08,0x00,0x00]
+          vpsubusb 2048(%rdx), %xmm27, %xmm17
+
+// CHECK: vpsubusb -2048(%rdx), %xmm27, %xmm17
+// CHECK:  encoding: [0x62,0xe1,0x25,0x00,0xd8,0x4a,0x80]
+          vpsubusb -2048(%rdx), %xmm27, %xmm17
+
+// CHECK: vpsubusb -2064(%rdx), %xmm27, %xmm17
+// CHECK:  encoding: [0x62,0xe1,0x25,0x00,0xd8,0x8a,0xf0,0xf7,0xff,0xff]
+          vpsubusb -2064(%rdx), %xmm27, %xmm17
+
+// CHECK: vpsubusb %ymm18, %ymm21, %ymm17
+// CHECK:  encoding: [0x62,0xa1,0x55,0x20,0xd8,0xca]
+          vpsubusb %ymm18, %ymm21, %ymm17
+
+// CHECK: vpsubusb %ymm18, %ymm21, %ymm17 {%k5}
+// CHECK:  encoding: [0x62,0xa1,0x55,0x25,0xd8,0xca]
+          vpsubusb %ymm18, %ymm21, %ymm17 {%k5}
+
+// CHECK: vpsubusb %ymm18, %ymm21, %ymm17 {%k5} {z}
+// CHECK:  encoding: [0x62,0xa1,0x55,0xa5,0xd8,0xca]
+          vpsubusb %ymm18, %ymm21, %ymm17 {%k5} {z}
+
+// CHECK: vpsubusb (%rcx), %ymm21, %ymm17
+// CHECK:  encoding: [0x62,0xe1,0x55,0x20,0xd8,0x09]
+          vpsubusb (%rcx), %ymm21, %ymm17
+
+// CHECK: vpsubusb 291(%rax,%r14,8), %ymm21, %ymm17
+// CHECK:  encoding: [0x62,0xa1,0x55,0x20,0xd8,0x8c,0xf0,0x23,0x01,0x00,0x00]
+          vpsubusb 291(%rax,%r14,8), %ymm21, %ymm17
+
+// CHECK: vpsubusb 4064(%rdx), %ymm21, %ymm17
+// CHECK:  encoding: [0x62,0xe1,0x55,0x20,0xd8,0x4a,0x7f]
+          vpsubusb 4064(%rdx), %ymm21, %ymm17
+
+// CHECK: vpsubusb 4096(%rdx), %ymm21, %ymm17
+// CHECK:  encoding: [0x62,0xe1,0x55,0x20,0xd8,0x8a,0x00,0x10,0x00,0x00]
+          vpsubusb 4096(%rdx), %ymm21, %ymm17
+
+// CHECK: vpsubusb -4096(%rdx), %ymm21, %ymm17
+// CHECK:  encoding: [0x62,0xe1,0x55,0x20,0xd8,0x4a,0x80]
+          vpsubusb -4096(%rdx), %ymm21, %ymm17
+
+// CHECK: vpsubusb -4128(%rdx), %ymm21, %ymm17
+// CHECK:  encoding: [0x62,0xe1,0x55,0x20,0xd8,0x8a,0xe0,0xef,0xff,0xff]
+          vpsubusb -4128(%rdx), %ymm21, %ymm17
+
+// CHECK: vpsubusw %xmm19, %xmm21, %xmm25
+// CHECK:  encoding: [0x62,0x21,0x55,0x00,0xd9,0xcb]
+          vpsubusw %xmm19, %xmm21, %xmm25
+
+// CHECK: vpsubusw %xmm19, %xmm21, %xmm25 {%k6}
+// CHECK:  encoding: [0x62,0x21,0x55,0x06,0xd9,0xcb]
+          vpsubusw %xmm19, %xmm21, %xmm25 {%k6}
+
+// CHECK: vpsubusw %xmm19, %xmm21, %xmm25 {%k6} {z}
+// CHECK:  encoding: [0x62,0x21,0x55,0x86,0xd9,0xcb]
+          vpsubusw %xmm19, %xmm21, %xmm25 {%k6} {z}
+
+// CHECK: vpsubusw (%rcx), %xmm21, %xmm25
+// CHECK:  encoding: [0x62,0x61,0x55,0x00,0xd9,0x09]
+          vpsubusw (%rcx), %xmm21, %xmm25
+
+// CHECK: vpsubusw 291(%rax,%r14,8), %xmm21, %xmm25
+// CHECK:  encoding: [0x62,0x21,0x55,0x00,0xd9,0x8c,0xf0,0x23,0x01,0x00,0x00]
+          vpsubusw 291(%rax,%r14,8), %xmm21, %xmm25
+
+// CHECK: vpsubusw 2032(%rdx), %xmm21, %xmm25
+// CHECK:  encoding: [0x62,0x61,0x55,0x00,0xd9,0x4a,0x7f]
+          vpsubusw 2032(%rdx), %xmm21, %xmm25
+
+// CHECK: vpsubusw 2048(%rdx), %xmm21, %xmm25
+// CHECK:  encoding: [0x62,0x61,0x55,0x00,0xd9,0x8a,0x00,0x08,0x00,0x00]
+          vpsubusw 2048(%rdx), %xmm21, %xmm25
+
+// CHECK: vpsubusw -2048(%rdx), %xmm21, %xmm25
+// CHECK:  encoding: [0x62,0x61,0x55,0x00,0xd9,0x4a,0x80]
+          vpsubusw -2048(%rdx), %xmm21, %xmm25
+
+// CHECK: vpsubusw -2064(%rdx), %xmm21, %xmm25
+// CHECK:  encoding: [0x62,0x61,0x55,0x00,0xd9,0x8a,0xf0,0xf7,0xff,0xff]
+          vpsubusw -2064(%rdx), %xmm21, %xmm25
+
+// CHECK: vpsubusw %ymm17, %ymm17, %ymm26
+// CHECK:  encoding: [0x62,0x21,0x75,0x20,0xd9,0xd1]
+          vpsubusw %ymm17, %ymm17, %ymm26
+
+// CHECK: vpsubusw %ymm17, %ymm17, %ymm26 {%k4}
+// CHECK:  encoding: [0x62,0x21,0x75,0x24,0xd9,0xd1]
+          vpsubusw %ymm17, %ymm17, %ymm26 {%k4}
+
+// CHECK: vpsubusw %ymm17, %ymm17, %ymm26 {%k4} {z}
+// CHECK:  encoding: [0x62,0x21,0x75,0xa4,0xd9,0xd1]
+          vpsubusw %ymm17, %ymm17, %ymm26 {%k4} {z}
+
+// CHECK: vpsubusw (%rcx), %ymm17, %ymm26
+// CHECK:  encoding: [0x62,0x61,0x75,0x20,0xd9,0x11]
+          vpsubusw (%rcx), %ymm17, %ymm26
+
+// CHECK: vpsubusw 291(%rax,%r14,8), %ymm17, %ymm26
+// CHECK:  encoding: [0x62,0x21,0x75,0x20,0xd9,0x94,0xf0,0x23,0x01,0x00,0x00]
+          vpsubusw 291(%rax,%r14,8), %ymm17, %ymm26
+
+// CHECK: vpsubusw 4064(%rdx), %ymm17, %ymm26
+// CHECK:  encoding: [0x62,0x61,0x75,0x20,0xd9,0x52,0x7f]
+          vpsubusw 4064(%rdx), %ymm17, %ymm26
+
+// CHECK: vpsubusw 4096(%rdx), %ymm17, %ymm26
+// CHECK:  encoding: [0x62,0x61,0x75,0x20,0xd9,0x92,0x00,0x10,0x00,0x00]
+          vpsubusw 4096(%rdx), %ymm17, %ymm26
+
+// CHECK: vpsubusw -4096(%rdx), %ymm17, %ymm26
+// CHECK:  encoding: [0x62,0x61,0x75,0x20,0xd9,0x52,0x80]
+          vpsubusw -4096(%rdx), %ymm17, %ymm26
+
+// CHECK: vpsubusw -4128(%rdx), %ymm17, %ymm26
+// CHECK:  encoding: [0x62,0x61,0x75,0x20,0xd9,0x92,0xe0,0xef,0xff,0xff]
+          vpsubusw -4128(%rdx), %ymm17, %ymm26
+
+// CHECK: vpaddsb %xmm23, %xmm24, %xmm20
+// CHECK:  encoding: [0x62,0xa1,0x3d,0x00,0xec,0xe7]
+          vpaddsb %xmm23, %xmm24, %xmm20
+
+// CHECK: vpaddsb %xmm23, %xmm24, %xmm20 {%k1}
+// CHECK:  encoding: [0x62,0xa1,0x3d,0x01,0xec,0xe7]
+          vpaddsb %xmm23, %xmm24, %xmm20 {%k1}
+
+// CHECK: vpaddsb %xmm23, %xmm24, %xmm20 {%k1} {z}
+// CHECK:  encoding: [0x62,0xa1,0x3d,0x81,0xec,0xe7]
+          vpaddsb %xmm23, %xmm24, %xmm20 {%k1} {z}
+
+// CHECK: vpaddsb (%rcx), %xmm24, %xmm20
+// CHECK:  encoding: [0x62,0xe1,0x3d,0x00,0xec,0x21]
+          vpaddsb (%rcx), %xmm24, %xmm20
+
+// CHECK: vpaddsb 4660(%rax,%r14,8), %xmm24, %xmm20
+// CHECK:  encoding: [0x62,0xa1,0x3d,0x00,0xec,0xa4,0xf0,0x34,0x12,0x00,0x00]
+          vpaddsb 4660(%rax,%r14,8), %xmm24, %xmm20
+
+// CHECK: vpaddsb 2032(%rdx), %xmm24, %xmm20
+// CHECK:  encoding: [0x62,0xe1,0x3d,0x00,0xec,0x62,0x7f]
+          vpaddsb 2032(%rdx), %xmm24, %xmm20
+
+// CHECK: vpaddsb 2048(%rdx), %xmm24, %xmm20
+// CHECK:  encoding: [0x62,0xe1,0x3d,0x00,0xec,0xa2,0x00,0x08,0x00,0x00]
+          vpaddsb 2048(%rdx), %xmm24, %xmm20
+
+// CHECK: vpaddsb -2048(%rdx), %xmm24, %xmm20
+// CHECK:  encoding: [0x62,0xe1,0x3d,0x00,0xec,0x62,0x80]
+          vpaddsb -2048(%rdx), %xmm24, %xmm20
+
+// CHECK: vpaddsb -2064(%rdx), %xmm24, %xmm20
+// CHECK:  encoding: [0x62,0xe1,0x3d,0x00,0xec,0xa2,0xf0,0xf7,0xff,0xff]
+          vpaddsb -2064(%rdx), %xmm24, %xmm20
+
+// CHECK: vpaddsb %ymm19, %ymm20, %ymm18
+// CHECK:  encoding: [0x62,0xa1,0x5d,0x20,0xec,0xd3]
+          vpaddsb %ymm19, %ymm20, %ymm18
+
+// CHECK: vpaddsb %ymm19, %ymm20, %ymm18 {%k7}
+// CHECK:  encoding: [0x62,0xa1,0x5d,0x27,0xec,0xd3]
+          vpaddsb %ymm19, %ymm20, %ymm18 {%k7}
+
+// CHECK: vpaddsb %ymm19, %ymm20, %ymm18 {%k7} {z}
+// CHECK:  encoding: [0x62,0xa1,0x5d,0xa7,0xec,0xd3]
+          vpaddsb %ymm19, %ymm20, %ymm18 {%k7} {z}
+
+// CHECK: vpaddsb (%rcx), %ymm20, %ymm18
+// CHECK:  encoding: [0x62,0xe1,0x5d,0x20,0xec,0x11]
+          vpaddsb (%rcx), %ymm20, %ymm18
+
+// CHECK: vpaddsb 4660(%rax,%r14,8), %ymm20, %ymm18
+// CHECK:  encoding: [0x62,0xa1,0x5d,0x20,0xec,0x94,0xf0,0x34,0x12,0x00,0x00]
+          vpaddsb 4660(%rax,%r14,8), %ymm20, %ymm18
+
+// CHECK: vpaddsb 4064(%rdx), %ymm20, %ymm18
+// CHECK:  encoding: [0x62,0xe1,0x5d,0x20,0xec,0x52,0x7f]
+          vpaddsb 4064(%rdx), %ymm20, %ymm18
+
+// CHECK: vpaddsb 4096(%rdx), %ymm20, %ymm18
+// CHECK:  encoding: [0x62,0xe1,0x5d,0x20,0xec,0x92,0x00,0x10,0x00,0x00]
+          vpaddsb 4096(%rdx), %ymm20, %ymm18
+
+// CHECK: vpaddsb -4096(%rdx), %ymm20, %ymm18
+// CHECK:  encoding: [0x62,0xe1,0x5d,0x20,0xec,0x52,0x80]
+          vpaddsb -4096(%rdx), %ymm20, %ymm18
+
+// CHECK: vpaddsb -4128(%rdx), %ymm20, %ymm18
+// CHECK:  encoding: [0x62,0xe1,0x5d,0x20,0xec,0x92,0xe0,0xef,0xff,0xff]
+          vpaddsb -4128(%rdx), %ymm20, %ymm18
+
+// CHECK: vpaddsw %xmm24, %xmm23, %xmm23
+// CHECK:  encoding: [0x62,0x81,0x45,0x00,0xed,0xf8]
+          vpaddsw %xmm24, %xmm23, %xmm23
+
+// CHECK: vpaddsw %xmm24, %xmm23, %xmm23 {%k5}
+// CHECK:  encoding: [0x62,0x81,0x45,0x05,0xed,0xf8]
+          vpaddsw %xmm24, %xmm23, %xmm23 {%k5}
+
+// CHECK: vpaddsw %xmm24, %xmm23, %xmm23 {%k5} {z}
+// CHECK:  encoding: [0x62,0x81,0x45,0x85,0xed,0xf8]
+          vpaddsw %xmm24, %xmm23, %xmm23 {%k5} {z}
+
+// CHECK: vpaddsw (%rcx), %xmm23, %xmm23
+// CHECK:  encoding: [0x62,0xe1,0x45,0x00,0xed,0x39]
+          vpaddsw (%rcx), %xmm23, %xmm23
+
+// CHECK: vpaddsw 4660(%rax,%r14,8), %xmm23, %xmm23
+// CHECK:  encoding: [0x62,0xa1,0x45,0x00,0xed,0xbc,0xf0,0x34,0x12,0x00,0x00]
+          vpaddsw 4660(%rax,%r14,8), %xmm23, %xmm23
+
+// CHECK: vpaddsw 2032(%rdx), %xmm23, %xmm23
+// CHECK:  encoding: [0x62,0xe1,0x45,0x00,0xed,0x7a,0x7f]
+          vpaddsw 2032(%rdx), %xmm23, %xmm23
+
+// CHECK: vpaddsw 2048(%rdx), %xmm23, %xmm23
+// CHECK:  encoding: [0x62,0xe1,0x45,0x00,0xed,0xba,0x00,0x08,0x00,0x00]
+          vpaddsw 2048(%rdx), %xmm23, %xmm23
+
+// CHECK: vpaddsw -2048(%rdx), %xmm23, %xmm23
+// CHECK:  encoding: [0x62,0xe1,0x45,0x00,0xed,0x7a,0x80]
+          vpaddsw -2048(%rdx), %xmm23, %xmm23
+
+// CHECK: vpaddsw -2064(%rdx), %xmm23, %xmm23
+// CHECK:  encoding: [0x62,0xe1,0x45,0x00,0xed,0xba,0xf0,0xf7,0xff,0xff]
+          vpaddsw -2064(%rdx), %xmm23, %xmm23
+
+// CHECK: vpaddsw %ymm19, %ymm22, %ymm29
+// CHECK:  encoding: [0x62,0x21,0x4d,0x20,0xed,0xeb]
+          vpaddsw %ymm19, %ymm22, %ymm29
+
+// CHECK: vpaddsw %ymm19, %ymm22, %ymm29 {%k6}
+// CHECK:  encoding: [0x62,0x21,0x4d,0x26,0xed,0xeb]
+          vpaddsw %ymm19, %ymm22, %ymm29 {%k6}
+
+// CHECK: vpaddsw %ymm19, %ymm22, %ymm29 {%k6} {z}
+// CHECK:  encoding: [0x62,0x21,0x4d,0xa6,0xed,0xeb]
+          vpaddsw %ymm19, %ymm22, %ymm29 {%k6} {z}
+
+// CHECK: vpaddsw (%rcx), %ymm22, %ymm29
+// CHECK:  encoding: [0x62,0x61,0x4d,0x20,0xed,0x29]
+          vpaddsw (%rcx), %ymm22, %ymm29
+
+// CHECK: vpaddsw 4660(%rax,%r14,8), %ymm22, %ymm29
+// CHECK:  encoding: [0x62,0x21,0x4d,0x20,0xed,0xac,0xf0,0x34,0x12,0x00,0x00]
+          vpaddsw 4660(%rax,%r14,8), %ymm22, %ymm29
+
+// CHECK: vpaddsw 4064(%rdx), %ymm22, %ymm29
+// CHECK:  encoding: [0x62,0x61,0x4d,0x20,0xed,0x6a,0x7f]
+          vpaddsw 4064(%rdx), %ymm22, %ymm29
+
+// CHECK: vpaddsw 4096(%rdx), %ymm22, %ymm29
+// CHECK:  encoding: [0x62,0x61,0x4d,0x20,0xed,0xaa,0x00,0x10,0x00,0x00]
+          vpaddsw 4096(%rdx), %ymm22, %ymm29
+
+// CHECK: vpaddsw -4096(%rdx), %ymm22, %ymm29
+// CHECK:  encoding: [0x62,0x61,0x4d,0x20,0xed,0x6a,0x80]
+          vpaddsw -4096(%rdx), %ymm22, %ymm29
+
+// CHECK: vpaddsw -4128(%rdx), %ymm22, %ymm29
+// CHECK:  encoding: [0x62,0x61,0x4d,0x20,0xed,0xaa,0xe0,0xef,0xff,0xff]
+          vpaddsw -4128(%rdx), %ymm22, %ymm29
+
+// CHECK: vpaddusb %xmm26, %xmm18, %xmm20
+// CHECK:  encoding: [0x62,0x81,0x6d,0x00,0xdc,0xe2]
+          vpaddusb %xmm26, %xmm18, %xmm20
+
+// CHECK: vpaddusb %xmm26, %xmm18, %xmm20 {%k6}
+// CHECK:  encoding: [0x62,0x81,0x6d,0x06,0xdc,0xe2]
+          vpaddusb %xmm26, %xmm18, %xmm20 {%k6}
+
+// CHECK: vpaddusb %xmm26, %xmm18, %xmm20 {%k6} {z}
+// CHECK:  encoding: [0x62,0x81,0x6d,0x86,0xdc,0xe2]
+          vpaddusb %xmm26, %xmm18, %xmm20 {%k6} {z}
+
+// CHECK: vpaddusb (%rcx), %xmm18, %xmm20
+// CHECK:  encoding: [0x62,0xe1,0x6d,0x00,0xdc,0x21]
+          vpaddusb (%rcx), %xmm18, %xmm20
+
+// CHECK: vpaddusb 4660(%rax,%r14,8), %xmm18, %xmm20
+// CHECK:  encoding: [0x62,0xa1,0x6d,0x00,0xdc,0xa4,0xf0,0x34,0x12,0x00,0x00]
+          vpaddusb 4660(%rax,%r14,8), %xmm18, %xmm20
+
+// CHECK: vpaddusb 2032(%rdx), %xmm18, %xmm20
+// CHECK:  encoding: [0x62,0xe1,0x6d,0x00,0xdc,0x62,0x7f]
+          vpaddusb 2032(%rdx), %xmm18, %xmm20
+
+// CHECK: vpaddusb 2048(%rdx), %xmm18, %xmm20
+// CHECK:  encoding: [0x62,0xe1,0x6d,0x00,0xdc,0xa2,0x00,0x08,0x00,0x00]
+          vpaddusb 2048(%rdx), %xmm18, %xmm20
+
+// CHECK: vpaddusb -2048(%rdx), %xmm18, %xmm20
+// CHECK:  encoding: [0x62,0xe1,0x6d,0x00,0xdc,0x62,0x80]
+          vpaddusb -2048(%rdx), %xmm18, %xmm20
+
+// CHECK: vpaddusb -2064(%rdx), %xmm18, %xmm20
+// CHECK:  encoding: [0x62,0xe1,0x6d,0x00,0xdc,0xa2,0xf0,0xf7,0xff,0xff]
+          vpaddusb -2064(%rdx), %xmm18, %xmm20
+
+// CHECK: vpaddusb %ymm24, %ymm26, %ymm25
+// CHECK:  encoding: [0x62,0x01,0x2d,0x20,0xdc,0xc8]
+          vpaddusb %ymm24, %ymm26, %ymm25
+
+// CHECK: vpaddusb %ymm24, %ymm26, %ymm25 {%k3}
+// CHECK:  encoding: [0x62,0x01,0x2d,0x23,0xdc,0xc8]
+          vpaddusb %ymm24, %ymm26, %ymm25 {%k3}
+
+// CHECK: vpaddusb %ymm24, %ymm26, %ymm25 {%k3} {z}
+// CHECK:  encoding: [0x62,0x01,0x2d,0xa3,0xdc,0xc8]
+          vpaddusb %ymm24, %ymm26, %ymm25 {%k3} {z}
+
+// CHECK: vpaddusb (%rcx), %ymm26, %ymm25
+// CHECK:  encoding: [0x62,0x61,0x2d,0x20,0xdc,0x09]
+          vpaddusb (%rcx), %ymm26, %ymm25
+
+// CHECK: vpaddusb 4660(%rax,%r14,8), %ymm26, %ymm25
+// CHECK:  encoding: [0x62,0x21,0x2d,0x20,0xdc,0x8c,0xf0,0x34,0x12,0x00,0x00]
+          vpaddusb 4660(%rax,%r14,8), %ymm26, %ymm25
+
+// CHECK: vpaddusb 4064(%rdx), %ymm26, %ymm25
+// CHECK:  encoding: [0x62,0x61,0x2d,0x20,0xdc,0x4a,0x7f]
+          vpaddusb 4064(%rdx), %ymm26, %ymm25
+
+// CHECK: vpaddusb 4096(%rdx), %ymm26, %ymm25
+// CHECK:  encoding: [0x62,0x61,0x2d,0x20,0xdc,0x8a,0x00,0x10,0x00,0x00]
+          vpaddusb 4096(%rdx), %ymm26, %ymm25
+
+// CHECK: vpaddusb -4096(%rdx), %ymm26, %ymm25
+// CHECK:  encoding: [0x62,0x61,0x2d,0x20,0xdc,0x4a,0x80]
+          vpaddusb -4096(%rdx), %ymm26, %ymm25
+
+// CHECK: vpaddusb -4128(%rdx), %ymm26, %ymm25
+// CHECK:  encoding: [0x62,0x61,0x2d,0x20,0xdc,0x8a,0xe0,0xef,0xff,0xff]
+          vpaddusb -4128(%rdx), %ymm26, %ymm25
+
+// CHECK: vpaddusw %xmm24, %xmm29, %xmm28
+// CHECK:  encoding: [0x62,0x01,0x15,0x00,0xdd,0xe0]
+          vpaddusw %xmm24, %xmm29, %xmm28
+
+// CHECK: vpaddusw %xmm24, %xmm29, %xmm28 {%k4}
+// CHECK:  encoding: [0x62,0x01,0x15,0x04,0xdd,0xe0]
+          vpaddusw %xmm24, %xmm29, %xmm28 {%k4}
+
+// CHECK: vpaddusw %xmm24, %xmm29, %xmm28 {%k4} {z}
+// CHECK:  encoding: [0x62,0x01,0x15,0x84,0xdd,0xe0]
+          vpaddusw %xmm24, %xmm29, %xmm28 {%k4} {z}
+
+// CHECK: vpaddusw (%rcx), %xmm29, %xmm28
+// CHECK:  encoding: [0x62,0x61,0x15,0x00,0xdd,0x21]
+          vpaddusw (%rcx), %xmm29, %xmm28
+
+// CHECK: vpaddusw 4660(%rax,%r14,8), %xmm29, %xmm28
+// CHECK:  encoding: [0x62,0x21,0x15,0x00,0xdd,0xa4,0xf0,0x34,0x12,0x00,0x00]
+          vpaddusw 4660(%rax,%r14,8), %xmm29, %xmm28
+
+// CHECK: vpaddusw 2032(%rdx), %xmm29, %xmm28
+// CHECK:  encoding: [0x62,0x61,0x15,0x00,0xdd,0x62,0x7f]
+          vpaddusw 2032(%rdx), %xmm29, %xmm28
+
+// CHECK: vpaddusw 2048(%rdx), %xmm29, %xmm28
+// CHECK:  encoding: [0x62,0x61,0x15,0x00,0xdd,0xa2,0x00,0x08,0x00,0x00]
+          vpaddusw 2048(%rdx), %xmm29, %xmm28
+
+// CHECK: vpaddusw -2048(%rdx), %xmm29, %xmm28
+// CHECK:  encoding: [0x62,0x61,0x15,0x00,0xdd,0x62,0x80]
+          vpaddusw -2048(%rdx), %xmm29, %xmm28
+
+// CHECK: vpaddusw -2064(%rdx), %xmm29, %xmm28
+// CHECK:  encoding: [0x62,0x61,0x15,0x00,0xdd,0xa2,0xf0,0xf7,0xff,0xff]
+          vpaddusw -2064(%rdx), %xmm29, %xmm28
+
+// CHECK: vpaddusw %ymm22, %ymm26, %ymm27
+// CHECK:  encoding: [0x62,0x21,0x2d,0x20,0xdd,0xde]
+          vpaddusw %ymm22, %ymm26, %ymm27
+
+// CHECK: vpaddusw %ymm22, %ymm26, %ymm27 {%k3}
+// CHECK:  encoding: [0x62,0x21,0x2d,0x23,0xdd,0xde]
+          vpaddusw %ymm22, %ymm26, %ymm27 {%k3}
+
+// CHECK: vpaddusw %ymm22, %ymm26, %ymm27 {%k3} {z}
+// CHECK:  encoding: [0x62,0x21,0x2d,0xa3,0xdd,0xde]
+          vpaddusw %ymm22, %ymm26, %ymm27 {%k3} {z}
+
+// CHECK: vpaddusw (%rcx), %ymm26, %ymm27
+// CHECK:  encoding: [0x62,0x61,0x2d,0x20,0xdd,0x19]
+          vpaddusw (%rcx), %ymm26, %ymm27
+
+// CHECK: vpaddusw 4660(%rax,%r14,8), %ymm26, %ymm27
+// CHECK:  encoding: [0x62,0x21,0x2d,0x20,0xdd,0x9c,0xf0,0x34,0x12,0x00,0x00]
+          vpaddusw 4660(%rax,%r14,8), %ymm26, %ymm27
+
+// CHECK: vpaddusw 4064(%rdx), %ymm26, %ymm27
+// CHECK:  encoding: [0x62,0x61,0x2d,0x20,0xdd,0x5a,0x7f]
+          vpaddusw 4064(%rdx), %ymm26, %ymm27
+
+// CHECK: vpaddusw 4096(%rdx), %ymm26, %ymm27
+// CHECK:  encoding: [0x62,0x61,0x2d,0x20,0xdd,0x9a,0x00,0x10,0x00,0x00]
+          vpaddusw 4096(%rdx), %ymm26, %ymm27
+
+// CHECK: vpaddusw -4096(%rdx), %ymm26, %ymm27
+// CHECK:  encoding: [0x62,0x61,0x2d,0x20,0xdd,0x5a,0x80]
+          vpaddusw -4096(%rdx), %ymm26, %ymm27
+
+// CHECK: vpaddusw -4128(%rdx), %ymm26, %ymm27
+// CHECK:  encoding: [0x62,0x61,0x2d,0x20,0xdd,0x9a,0xe0,0xef,0xff,0xff]
+          vpaddusw -4128(%rdx), %ymm26, %ymm27
+
+// CHECK: vpsubsb %xmm18, %xmm22, %xmm23
+// CHECK:  encoding: [0x62,0xa1,0x4d,0x00,0xe8,0xfa]
+          vpsubsb %xmm18, %xmm22, %xmm23
+
+// CHECK: vpsubsb %xmm18, %xmm22, %xmm23 {%k5}
+// CHECK:  encoding: [0x62,0xa1,0x4d,0x05,0xe8,0xfa]
+          vpsubsb %xmm18, %xmm22, %xmm23 {%k5}
+
+// CHECK: vpsubsb %xmm18, %xmm22, %xmm23 {%k5} {z}
+// CHECK:  encoding: [0x62,0xa1,0x4d,0x85,0xe8,0xfa]
+          vpsubsb %xmm18, %xmm22, %xmm23 {%k5} {z}
+
+// CHECK: vpsubsb (%rcx), %xmm22, %xmm23
+// CHECK:  encoding: [0x62,0xe1,0x4d,0x00,0xe8,0x39]
+          vpsubsb (%rcx), %xmm22, %xmm23
+
+// CHECK: vpsubsb 4660(%rax,%r14,8), %xmm22, %xmm23
+// CHECK:  encoding: [0x62,0xa1,0x4d,0x00,0xe8,0xbc,0xf0,0x34,0x12,0x00,0x00]
+          vpsubsb 4660(%rax,%r14,8), %xmm22, %xmm23
+
+// CHECK: vpsubsb 2032(%rdx), %xmm22, %xmm23
+// CHECK:  encoding: [0x62,0xe1,0x4d,0x00,0xe8,0x7a,0x7f]
+          vpsubsb 2032(%rdx), %xmm22, %xmm23
+
+// CHECK: vpsubsb 2048(%rdx), %xmm22, %xmm23
+// CHECK:  encoding: [0x62,0xe1,0x4d,0x00,0xe8,0xba,0x00,0x08,0x00,0x00]
+          vpsubsb 2048(%rdx), %xmm22, %xmm23
+
+// CHECK: vpsubsb -2048(%rdx), %xmm22, %xmm23
+// CHECK:  encoding: [0x62,0xe1,0x4d,0x00,0xe8,0x7a,0x80]
+          vpsubsb -2048(%rdx), %xmm22, %xmm23
+
+// CHECK: vpsubsb -2064(%rdx), %xmm22, %xmm23
+// CHECK:  encoding: [0x62,0xe1,0x4d,0x00,0xe8,0xba,0xf0,0xf7,0xff,0xff]
+          vpsubsb -2064(%rdx), %xmm22, %xmm23
+
+// CHECK: vpsubsb %ymm24, %ymm21, %ymm28
+// CHECK:  encoding: [0x62,0x01,0x55,0x20,0xe8,0xe0]
+          vpsubsb %ymm24, %ymm21, %ymm28
+
+// CHECK: vpsubsb %ymm24, %ymm21, %ymm28 {%k5}
+// CHECK:  encoding: [0x62,0x01,0x55,0x25,0xe8,0xe0]
+          vpsubsb %ymm24, %ymm21, %ymm28 {%k5}
+
+// CHECK: vpsubsb %ymm24, %ymm21, %ymm28 {%k5} {z}
+// CHECK:  encoding: [0x62,0x01,0x55,0xa5,0xe8,0xe0]
+          vpsubsb %ymm24, %ymm21, %ymm28 {%k5} {z}
+
+// CHECK: vpsubsb (%rcx), %ymm21, %ymm28
+// CHECK:  encoding: [0x62,0x61,0x55,0x20,0xe8,0x21]
+          vpsubsb (%rcx), %ymm21, %ymm28
+
+// CHECK: vpsubsb 4660(%rax,%r14,8), %ymm21, %ymm28
+// CHECK:  encoding: [0x62,0x21,0x55,0x20,0xe8,0xa4,0xf0,0x34,0x12,0x00,0x00]
+          vpsubsb 4660(%rax,%r14,8), %ymm21, %ymm28
+
+// CHECK: vpsubsb 4064(%rdx), %ymm21, %ymm28
+// CHECK:  encoding: [0x62,0x61,0x55,0x20,0xe8,0x62,0x7f]
+          vpsubsb 4064(%rdx), %ymm21, %ymm28
+
+// CHECK: vpsubsb 4096(%rdx), %ymm21, %ymm28
+// CHECK:  encoding: [0x62,0x61,0x55,0x20,0xe8,0xa2,0x00,0x10,0x00,0x00]
+          vpsubsb 4096(%rdx), %ymm21, %ymm28
+
+// CHECK: vpsubsb -4096(%rdx), %ymm21, %ymm28
+// CHECK:  encoding: [0x62,0x61,0x55,0x20,0xe8,0x62,0x80]
+          vpsubsb -4096(%rdx), %ymm21, %ymm28
+
+// CHECK: vpsubsb -4128(%rdx), %ymm21, %ymm28
+// CHECK:  encoding: [0x62,0x61,0x55,0x20,0xe8,0xa2,0xe0,0xef,0xff,0xff]
+          vpsubsb -4128(%rdx), %ymm21, %ymm28
+
+// CHECK: vpsubsw %xmm18, %xmm27, %xmm19
+// CHECK:  encoding: [0x62,0xa1,0x25,0x00,0xe9,0xda]
+          vpsubsw %xmm18, %xmm27, %xmm19
+
+// CHECK: vpsubsw %xmm18, %xmm27, %xmm19 {%k4}
+// CHECK:  encoding: [0x62,0xa1,0x25,0x04,0xe9,0xda]
+          vpsubsw %xmm18, %xmm27, %xmm19 {%k4}
+
+// CHECK: vpsubsw %xmm18, %xmm27, %xmm19 {%k4} {z}
+// CHECK:  encoding: [0x62,0xa1,0x25,0x84,0xe9,0xda]
+          vpsubsw %xmm18, %xmm27, %xmm19 {%k4} {z}
+
+// CHECK: vpsubsw (%rcx), %xmm27, %xmm19
+// CHECK:  encoding: [0x62,0xe1,0x25,0x00,0xe9,0x19]
+          vpsubsw (%rcx), %xmm27, %xmm19
+
+// CHECK: vpsubsw 4660(%rax,%r14,8), %xmm27, %xmm19
+// CHECK:  encoding: [0x62,0xa1,0x25,0x00,0xe9,0x9c,0xf0,0x34,0x12,0x00,0x00]
+          vpsubsw 4660(%rax,%r14,8), %xmm27, %xmm19
+
+// CHECK: vpsubsw 2032(%rdx), %xmm27, %xmm19
+// CHECK:  encoding: [0x62,0xe1,0x25,0x00,0xe9,0x5a,0x7f]
+          vpsubsw 2032(%rdx), %xmm27, %xmm19
+
+// CHECK: vpsubsw 2048(%rdx), %xmm27, %xmm19
+// CHECK:  encoding: [0x62,0xe1,0x25,0x00,0xe9,0x9a,0x00,0x08,0x00,0x00]
+          vpsubsw 2048(%rdx), %xmm27, %xmm19
+
+// CHECK: vpsubsw -2048(%rdx), %xmm27, %xmm19
+// CHECK:  encoding: [0x62,0xe1,0x25,0x00,0xe9,0x5a,0x80]
+          vpsubsw -2048(%rdx), %xmm27, %xmm19
+
+// CHECK: vpsubsw -2064(%rdx), %xmm27, %xmm19
+// CHECK:  encoding: [0x62,0xe1,0x25,0x00,0xe9,0x9a,0xf0,0xf7,0xff,0xff]
+          vpsubsw -2064(%rdx), %xmm27, %xmm19
+
+// CHECK: vpsubsw %ymm25, %ymm29, %ymm18
+// CHECK:  encoding: [0x62,0x81,0x15,0x20,0xe9,0xd1]
+          vpsubsw %ymm25, %ymm29, %ymm18
+
+// CHECK: vpsubsw %ymm25, %ymm29, %ymm18 {%k5}
+// CHECK:  encoding: [0x62,0x81,0x15,0x25,0xe9,0xd1]
+          vpsubsw %ymm25, %ymm29, %ymm18 {%k5}
+
+// CHECK: vpsubsw %ymm25, %ymm29, %ymm18 {%k5} {z}
+// CHECK:  encoding: [0x62,0x81,0x15,0xa5,0xe9,0xd1]
+          vpsubsw %ymm25, %ymm29, %ymm18 {%k5} {z}
+
+// CHECK: vpsubsw (%rcx), %ymm29, %ymm18
+// CHECK:  encoding: [0x62,0xe1,0x15,0x20,0xe9,0x11]
+          vpsubsw (%rcx), %ymm29, %ymm18
+
+// CHECK: vpsubsw 4660(%rax,%r14,8), %ymm29, %ymm18
+// CHECK:  encoding: [0x62,0xa1,0x15,0x20,0xe9,0x94,0xf0,0x34,0x12,0x00,0x00]
+          vpsubsw 4660(%rax,%r14,8), %ymm29, %ymm18
+
+// CHECK: vpsubsw 4064(%rdx), %ymm29, %ymm18
+// CHECK:  encoding: [0x62,0xe1,0x15,0x20,0xe9,0x52,0x7f]
+          vpsubsw 4064(%rdx), %ymm29, %ymm18
+
+// CHECK: vpsubsw 4096(%rdx), %ymm29, %ymm18
+// CHECK:  encoding: [0x62,0xe1,0x15,0x20,0xe9,0x92,0x00,0x10,0x00,0x00]
+          vpsubsw 4096(%rdx), %ymm29, %ymm18
+
+// CHECK: vpsubsw -4096(%rdx), %ymm29, %ymm18
+// CHECK:  encoding: [0x62,0xe1,0x15,0x20,0xe9,0x52,0x80]
+          vpsubsw -4096(%rdx), %ymm29, %ymm18
+
+// CHECK: vpsubsw -4128(%rdx), %ymm29, %ymm18
+// CHECK:  encoding: [0x62,0xe1,0x15,0x20,0xe9,0x92,0xe0,0xef,0xff,0xff]
+          vpsubsw -4128(%rdx), %ymm29, %ymm18
+
+// CHECK: vpsubusb %xmm23, %xmm26, %xmm20
+// CHECK:  encoding: [0x62,0xa1,0x2d,0x00,0xd8,0xe7]
+          vpsubusb %xmm23, %xmm26, %xmm20
+
+// CHECK: vpsubusb %xmm23, %xmm26, %xmm20 {%k5}
+// CHECK:  encoding: [0x62,0xa1,0x2d,0x05,0xd8,0xe7]
+          vpsubusb %xmm23, %xmm26, %xmm20 {%k5}
+
+// CHECK: vpsubusb %xmm23, %xmm26, %xmm20 {%k5} {z}
+// CHECK:  encoding: [0x62,0xa1,0x2d,0x85,0xd8,0xe7]
+          vpsubusb %xmm23, %xmm26, %xmm20 {%k5} {z}
+
+// CHECK: vpsubusb (%rcx), %xmm26, %xmm20
+// CHECK:  encoding: [0x62,0xe1,0x2d,0x00,0xd8,0x21]
+          vpsubusb (%rcx), %xmm26, %xmm20
+
+// CHECK: vpsubusb 4660(%rax,%r14,8), %xmm26, %xmm20
+// CHECK:  encoding: [0x62,0xa1,0x2d,0x00,0xd8,0xa4,0xf0,0x34,0x12,0x00,0x00]
+          vpsubusb 4660(%rax,%r14,8), %xmm26, %xmm20
+
+// CHECK: vpsubusb 2032(%rdx), %xmm26, %xmm20
+// CHECK:  encoding: [0x62,0xe1,0x2d,0x00,0xd8,0x62,0x7f]
+          vpsubusb 2032(%rdx), %xmm26, %xmm20
+
+// CHECK: vpsubusb 2048(%rdx), %xmm26, %xmm20
+// CHECK:  encoding: [0x62,0xe1,0x2d,0x00,0xd8,0xa2,0x00,0x08,0x00,0x00]
+          vpsubusb 2048(%rdx), %xmm26, %xmm20
+
+// CHECK: vpsubusb -2048(%rdx), %xmm26, %xmm20
+// CHECK:  encoding: [0x62,0xe1,0x2d,0x00,0xd8,0x62,0x80]
+          vpsubusb -2048(%rdx), %xmm26, %xmm20
+
+// CHECK: vpsubusb -2064(%rdx), %xmm26, %xmm20
+// CHECK:  encoding: [0x62,0xe1,0x2d,0x00,0xd8,0xa2,0xf0,0xf7,0xff,0xff]
+          vpsubusb -2064(%rdx), %xmm26, %xmm20
+
+// CHECK: vpsubusb %ymm19, %ymm28, %ymm30
+// CHECK:  encoding: [0x62,0x21,0x1d,0x20,0xd8,0xf3]
+          vpsubusb %ymm19, %ymm28, %ymm30
+
+// CHECK: vpsubusb %ymm19, %ymm28, %ymm30 {%k3}
+// CHECK:  encoding: [0x62,0x21,0x1d,0x23,0xd8,0xf3]
+          vpsubusb %ymm19, %ymm28, %ymm30 {%k3}
+
+// CHECK: vpsubusb %ymm19, %ymm28, %ymm30 {%k3} {z}
+// CHECK:  encoding: [0x62,0x21,0x1d,0xa3,0xd8,0xf3]
+          vpsubusb %ymm19, %ymm28, %ymm30 {%k3} {z}
+
+// CHECK: vpsubusb (%rcx), %ymm28, %ymm30
+// CHECK:  encoding: [0x62,0x61,0x1d,0x20,0xd8,0x31]
+          vpsubusb (%rcx), %ymm28, %ymm30
+
+// CHECK: vpsubusb 4660(%rax,%r14,8), %ymm28, %ymm30
+// CHECK:  encoding: [0x62,0x21,0x1d,0x20,0xd8,0xb4,0xf0,0x34,0x12,0x00,0x00]
+          vpsubusb 4660(%rax,%r14,8), %ymm28, %ymm30
+
+// CHECK: vpsubusb 4064(%rdx), %ymm28, %ymm30
+// CHECK:  encoding: [0x62,0x61,0x1d,0x20,0xd8,0x72,0x7f]
+          vpsubusb 4064(%rdx), %ymm28, %ymm30
+
+// CHECK: vpsubusb 4096(%rdx), %ymm28, %ymm30
+// CHECK:  encoding: [0x62,0x61,0x1d,0x20,0xd8,0xb2,0x00,0x10,0x00,0x00]
+          vpsubusb 4096(%rdx), %ymm28, %ymm30
+
+// CHECK: vpsubusb -4096(%rdx), %ymm28, %ymm30
+// CHECK:  encoding: [0x62,0x61,0x1d,0x20,0xd8,0x72,0x80]
+          vpsubusb -4096(%rdx), %ymm28, %ymm30
+
+// CHECK: vpsubusb -4128(%rdx), %ymm28, %ymm30
+// CHECK:  encoding: [0x62,0x61,0x1d,0x20,0xd8,0xb2,0xe0,0xef,0xff,0xff]
+          vpsubusb -4128(%rdx), %ymm28, %ymm30
+
+// CHECK: vpsubusw %xmm21, %xmm23, %xmm26
+// CHECK:  encoding: [0x62,0x21,0x45,0x00,0xd9,0xd5]
+          vpsubusw %xmm21, %xmm23, %xmm26
+
+// CHECK: vpsubusw %xmm21, %xmm23, %xmm26 {%k6}
+// CHECK:  encoding: [0x62,0x21,0x45,0x06,0xd9,0xd5]
+          vpsubusw %xmm21, %xmm23, %xmm26 {%k6}
+
+// CHECK: vpsubusw %xmm21, %xmm23, %xmm26 {%k6} {z}
+// CHECK:  encoding: [0x62,0x21,0x45,0x86,0xd9,0xd5]
+          vpsubusw %xmm21, %xmm23, %xmm26 {%k6} {z}
+
+// CHECK: vpsubusw (%rcx), %xmm23, %xmm26
+// CHECK:  encoding: [0x62,0x61,0x45,0x00,0xd9,0x11]
+          vpsubusw (%rcx), %xmm23, %xmm26
+
+// CHECK: vpsubusw 4660(%rax,%r14,8), %xmm23, %xmm26
+// CHECK:  encoding: [0x62,0x21,0x45,0x00,0xd9,0x94,0xf0,0x34,0x12,0x00,0x00]
+          vpsubusw 4660(%rax,%r14,8), %xmm23, %xmm26
+
+// CHECK: vpsubusw 2032(%rdx), %xmm23, %xmm26
+// CHECK:  encoding: [0x62,0x61,0x45,0x00,0xd9,0x52,0x7f]
+          vpsubusw 2032(%rdx), %xmm23, %xmm26
+
+// CHECK: vpsubusw 2048(%rdx), %xmm23, %xmm26
+// CHECK:  encoding: [0x62,0x61,0x45,0x00,0xd9,0x92,0x00,0x08,0x00,0x00]
+          vpsubusw 2048(%rdx), %xmm23, %xmm26
+
+// CHECK: vpsubusw -2048(%rdx), %xmm23, %xmm26
+// CHECK:  encoding: [0x62,0x61,0x45,0x00,0xd9,0x52,0x80]
+          vpsubusw -2048(%rdx), %xmm23, %xmm26
+
+// CHECK: vpsubusw -2064(%rdx), %xmm23, %xmm26
+// CHECK:  encoding: [0x62,0x61,0x45,0x00,0xd9,0x92,0xf0,0xf7,0xff,0xff]
+          vpsubusw -2064(%rdx), %xmm23, %xmm26
+
+// CHECK: vpsubusw %ymm24, %ymm20, %ymm25
+// CHECK:  encoding: [0x62,0x01,0x5d,0x20,0xd9,0xc8]
+          vpsubusw %ymm24, %ymm20, %ymm25
+
+// CHECK: vpsubusw %ymm24, %ymm20, %ymm25 {%k1}
+// CHECK:  encoding: [0x62,0x01,0x5d,0x21,0xd9,0xc8]
+          vpsubusw %ymm24, %ymm20, %ymm25 {%k1}
+
+// CHECK: vpsubusw %ymm24, %ymm20, %ymm25 {%k1} {z}
+// CHECK:  encoding: [0x62,0x01,0x5d,0xa1,0xd9,0xc8]
+          vpsubusw %ymm24, %ymm20, %ymm25 {%k1} {z}
+
+// CHECK: vpsubusw (%rcx), %ymm20, %ymm25
+// CHECK:  encoding: [0x62,0x61,0x5d,0x20,0xd9,0x09]
+          vpsubusw (%rcx), %ymm20, %ymm25
+
+// CHECK: vpsubusw 4660(%rax,%r14,8), %ymm20, %ymm25
+// CHECK:  encoding: [0x62,0x21,0x5d,0x20,0xd9,0x8c,0xf0,0x34,0x12,0x00,0x00]
+          vpsubusw 4660(%rax,%r14,8), %ymm20, %ymm25
+
+// CHECK: vpsubusw 4064(%rdx), %ymm20, %ymm25
+// CHECK:  encoding: [0x62,0x61,0x5d,0x20,0xd9,0x4a,0x7f]
+          vpsubusw 4064(%rdx), %ymm20, %ymm25
+
+// CHECK: vpsubusw 4096(%rdx), %ymm20, %ymm25
+// CHECK:  encoding: [0x62,0x61,0x5d,0x20,0xd9,0x8a,0x00,0x10,0x00,0x00]
+          vpsubusw 4096(%rdx), %ymm20, %ymm25
+
+// CHECK: vpsubusw -4096(%rdx), %ymm20, %ymm25
+// CHECK:  encoding: [0x62,0x61,0x5d,0x20,0xd9,0x4a,0x80]
+          vpsubusw -4096(%rdx), %ymm20, %ymm25
+
+// CHECK: vpsubusw -4128(%rdx), %ymm20, %ymm25
+// CHECK:  encoding: [0x62,0x61,0x5d,0x20,0xd9,0x8a,0xe0,0xef,0xff,0xff]
+          vpsubusw -4128(%rdx), %ymm20, %ymm25
+
+// CHECK: vpaddsb %xmm25, %xmm19, %xmm30
+// CHECK:  encoding: [0x62,0x01,0x65,0x00,0xec,0xf1]
+          vpaddsb %xmm25, %xmm19, %xmm30
+
+// CHECK: vpaddsb %xmm25, %xmm19, %xmm30 {%k6}
+// CHECK:  encoding: [0x62,0x01,0x65,0x06,0xec,0xf1]
+          vpaddsb %xmm25, %xmm19, %xmm30 {%k6}
+
+// CHECK: vpaddsb %xmm25, %xmm19, %xmm30 {%k6} {z}
+// CHECK:  encoding: [0x62,0x01,0x65,0x86,0xec,0xf1]
+          vpaddsb %xmm25, %xmm19, %xmm30 {%k6} {z}
+
+// CHECK: vpaddsb (%rcx), %xmm19, %xmm30
+// CHECK:  encoding: [0x62,0x61,0x65,0x00,0xec,0x31]
+          vpaddsb (%rcx), %xmm19, %xmm30
+
+// CHECK: vpaddsb 291(%rax,%r14,8), %xmm19, %xmm30
+// CHECK:  encoding: [0x62,0x21,0x65,0x00,0xec,0xb4,0xf0,0x23,0x01,0x00,0x00]
+          vpaddsb 291(%rax,%r14,8), %xmm19, %xmm30
+
+// CHECK: vpaddsb 2032(%rdx), %xmm19, %xmm30
+// CHECK:  encoding: [0x62,0x61,0x65,0x00,0xec,0x72,0x7f]
+          vpaddsb 2032(%rdx), %xmm19, %xmm30
+
+// CHECK: vpaddsb 2048(%rdx), %xmm19, %xmm30
+// CHECK:  encoding: [0x62,0x61,0x65,0x00,0xec,0xb2,0x00,0x08,0x00,0x00]
+          vpaddsb 2048(%rdx), %xmm19, %xmm30
+
+// CHECK: vpaddsb -2048(%rdx), %xmm19, %xmm30
+// CHECK:  encoding: [0x62,0x61,0x65,0x00,0xec,0x72,0x80]
+          vpaddsb -2048(%rdx), %xmm19, %xmm30
+
+// CHECK: vpaddsb -2064(%rdx), %xmm19, %xmm30
+// CHECK:  encoding: [0x62,0x61,0x65,0x00,0xec,0xb2,0xf0,0xf7,0xff,0xff]
+          vpaddsb -2064(%rdx), %xmm19, %xmm30
+
+// CHECK: vpaddsb %ymm28, %ymm29, %ymm22
+// CHECK:  encoding: [0x62,0x81,0x15,0x20,0xec,0xf4]
+          vpaddsb %ymm28, %ymm29, %ymm22
+
+// CHECK: vpaddsb %ymm28, %ymm29, %ymm22 {%k4}
+// CHECK:  encoding: [0x62,0x81,0x15,0x24,0xec,0xf4]
+          vpaddsb %ymm28, %ymm29, %ymm22 {%k4}
+
+// CHECK: vpaddsb %ymm28, %ymm29, %ymm22 {%k4} {z}
+// CHECK:  encoding: [0x62,0x81,0x15,0xa4,0xec,0xf4]
+          vpaddsb %ymm28, %ymm29, %ymm22 {%k4} {z}
+
+// CHECK: vpaddsb (%rcx), %ymm29, %ymm22
+// CHECK:  encoding: [0x62,0xe1,0x15,0x20,0xec,0x31]
+          vpaddsb (%rcx), %ymm29, %ymm22
+
+// CHECK: vpaddsb 291(%rax,%r14,8), %ymm29, %ymm22
+// CHECK:  encoding: [0x62,0xa1,0x15,0x20,0xec,0xb4,0xf0,0x23,0x01,0x00,0x00]
+          vpaddsb 291(%rax,%r14,8), %ymm29, %ymm22
+
+// CHECK: vpaddsb 4064(%rdx), %ymm29, %ymm22
+// CHECK:  encoding: [0x62,0xe1,0x15,0x20,0xec,0x72,0x7f]
+          vpaddsb 4064(%rdx), %ymm29, %ymm22
+
+// CHECK: vpaddsb 4096(%rdx), %ymm29, %ymm22
+// CHECK:  encoding: [0x62,0xe1,0x15,0x20,0xec,0xb2,0x00,0x10,0x00,0x00]
+          vpaddsb 4096(%rdx), %ymm29, %ymm22
+
+// CHECK: vpaddsb -4096(%rdx), %ymm29, %ymm22
+// CHECK:  encoding: [0x62,0xe1,0x15,0x20,0xec,0x72,0x80]
+          vpaddsb -4096(%rdx), %ymm29, %ymm22
+
+// CHECK: vpaddsb -4128(%rdx), %ymm29, %ymm22
+// CHECK:  encoding: [0x62,0xe1,0x15,0x20,0xec,0xb2,0xe0,0xef,0xff,0xff]
+          vpaddsb -4128(%rdx), %ymm29, %ymm22
+
+// CHECK: vpaddsw %xmm19, %xmm19, %xmm20
+// CHECK:  encoding: [0x62,0xa1,0x65,0x00,0xed,0xe3]
+          vpaddsw %xmm19, %xmm19, %xmm20
+
+// CHECK: vpaddsw %xmm19, %xmm19, %xmm20 {%k7}
+// CHECK:  encoding: [0x62,0xa1,0x65,0x07,0xed,0xe3]
+          vpaddsw %xmm19, %xmm19, %xmm20 {%k7}
+
+// CHECK: vpaddsw %xmm19, %xmm19, %xmm20 {%k7} {z}
+// CHECK:  encoding: [0x62,0xa1,0x65,0x87,0xed,0xe3]
+          vpaddsw %xmm19, %xmm19, %xmm20 {%k7} {z}
+
+// CHECK: vpaddsw (%rcx), %xmm19, %xmm20
+// CHECK:  encoding: [0x62,0xe1,0x65,0x00,0xed,0x21]
+          vpaddsw (%rcx), %xmm19, %xmm20
+
+// CHECK: vpaddsw 291(%rax,%r14,8), %xmm19, %xmm20
+// CHECK:  encoding: [0x62,0xa1,0x65,0x00,0xed,0xa4,0xf0,0x23,0x01,0x00,0x00]
+          vpaddsw 291(%rax,%r14,8), %xmm19, %xmm20
+
+// CHECK: vpaddsw 2032(%rdx), %xmm19, %xmm20
+// CHECK:  encoding: [0x62,0xe1,0x65,0x00,0xed,0x62,0x7f]
+          vpaddsw 2032(%rdx), %xmm19, %xmm20
+
+// CHECK: vpaddsw 2048(%rdx), %xmm19, %xmm20
+// CHECK:  encoding: [0x62,0xe1,0x65,0x00,0xed,0xa2,0x00,0x08,0x00,0x00]
+          vpaddsw 2048(%rdx), %xmm19, %xmm20
+
+// CHECK: vpaddsw -2048(%rdx), %xmm19, %xmm20
+// CHECK:  encoding: [0x62,0xe1,0x65,0x00,0xed,0x62,0x80]
+          vpaddsw -2048(%rdx), %xmm19, %xmm20
+
+// CHECK: vpaddsw -2064(%rdx), %xmm19, %xmm20
+// CHECK:  encoding: [0x62,0xe1,0x65,0x00,0xed,0xa2,0xf0,0xf7,0xff,0xff]
+          vpaddsw -2064(%rdx), %xmm19, %xmm20
+
+// CHECK: vpaddsw %ymm19, %ymm27, %ymm20
+// CHECK:  encoding: [0x62,0xa1,0x25,0x20,0xed,0xe3]
+          vpaddsw %ymm19, %ymm27, %ymm20
+
+// CHECK: vpaddsw %ymm19, %ymm27, %ymm20 {%k3}
+// CHECK:  encoding: [0x62,0xa1,0x25,0x23,0xed,0xe3]
+          vpaddsw %ymm19, %ymm27, %ymm20 {%k3}
+
+// CHECK: vpaddsw %ymm19, %ymm27, %ymm20 {%k3} {z}
+// CHECK:  encoding: [0x62,0xa1,0x25,0xa3,0xed,0xe3]
+          vpaddsw %ymm19, %ymm27, %ymm20 {%k3} {z}
+
+// CHECK: vpaddsw (%rcx), %ymm27, %ymm20
+// CHECK:  encoding: [0x62,0xe1,0x25,0x20,0xed,0x21]
+          vpaddsw (%rcx), %ymm27, %ymm20
+
+// CHECK: vpaddsw 291(%rax,%r14,8), %ymm27, %ymm20
+// CHECK:  encoding: [0x62,0xa1,0x25,0x20,0xed,0xa4,0xf0,0x23,0x01,0x00,0x00]
+          vpaddsw 291(%rax,%r14,8), %ymm27, %ymm20
+
+// CHECK: vpaddsw 4064(%rdx), %ymm27, %ymm20
+// CHECK:  encoding: [0x62,0xe1,0x25,0x20,0xed,0x62,0x7f]
+          vpaddsw 4064(%rdx), %ymm27, %ymm20
+
+// CHECK: vpaddsw 4096(%rdx), %ymm27, %ymm20
+// CHECK:  encoding: [0x62,0xe1,0x25,0x20,0xed,0xa2,0x00,0x10,0x00,0x00]
+          vpaddsw 4096(%rdx), %ymm27, %ymm20
+
+// CHECK: vpaddsw -4096(%rdx), %ymm27, %ymm20
+// CHECK:  encoding: [0x62,0xe1,0x25,0x20,0xed,0x62,0x80]
+          vpaddsw -4096(%rdx), %ymm27, %ymm20
+
+// CHECK: vpaddsw -4128(%rdx), %ymm27, %ymm20
+// CHECK:  encoding: [0x62,0xe1,0x25,0x20,0xed,0xa2,0xe0,0xef,0xff,0xff]
+          vpaddsw -4128(%rdx), %ymm27, %ymm20
+
+// CHECK: vpaddusb %xmm19, %xmm21, %xmm25
+// CHECK:  encoding: [0x62,0x21,0x55,0x00,0xdc,0xcb]
+          vpaddusb %xmm19, %xmm21, %xmm25
+
+// CHECK: vpaddusb %xmm19, %xmm21, %xmm25 {%k7}
+// CHECK:  encoding: [0x62,0x21,0x55,0x07,0xdc,0xcb]
+          vpaddusb %xmm19, %xmm21, %xmm25 {%k7}
+
+// CHECK: vpaddusb %xmm19, %xmm21, %xmm25 {%k7} {z}
+// CHECK:  encoding: [0x62,0x21,0x55,0x87,0xdc,0xcb]
+          vpaddusb %xmm19, %xmm21, %xmm25 {%k7} {z}
+
+// CHECK: vpaddusb (%rcx), %xmm21, %xmm25
+// CHECK:  encoding: [0x62,0x61,0x55,0x00,0xdc,0x09]
+          vpaddusb (%rcx), %xmm21, %xmm25
+
+// CHECK: vpaddusb 291(%rax,%r14,8), %xmm21, %xmm25
+// CHECK:  encoding: [0x62,0x21,0x55,0x00,0xdc,0x8c,0xf0,0x23,0x01,0x00,0x00]
+          vpaddusb 291(%rax,%r14,8), %xmm21, %xmm25
+
+// CHECK: vpaddusb 2032(%rdx), %xmm21, %xmm25
+// CHECK:  encoding: [0x62,0x61,0x55,0x00,0xdc,0x4a,0x7f]
+          vpaddusb 2032(%rdx), %xmm21, %xmm25
+
+// CHECK: vpaddusb 2048(%rdx), %xmm21, %xmm25
+// CHECK:  encoding: [0x62,0x61,0x55,0x00,0xdc,0x8a,0x00,0x08,0x00,0x00]
+          vpaddusb 2048(%rdx), %xmm21, %xmm25
+
+// CHECK: vpaddusb -2048(%rdx), %xmm21, %xmm25
+// CHECK:  encoding: [0x62,0x61,0x55,0x00,0xdc,0x4a,0x80]
+          vpaddusb -2048(%rdx), %xmm21, %xmm25
+
+// CHECK: vpaddusb -2064(%rdx), %xmm21, %xmm25
+// CHECK:  encoding: [0x62,0x61,0x55,0x00,0xdc,0x8a,0xf0,0xf7,0xff,0xff]
+          vpaddusb -2064(%rdx), %xmm21, %xmm25
+
+// CHECK: vpaddusb %ymm25, %ymm21, %ymm21
+// CHECK:  encoding: [0x62,0x81,0x55,0x20,0xdc,0xe9]
+          vpaddusb %ymm25, %ymm21, %ymm21
+
+// CHECK: vpaddusb %ymm25, %ymm21, %ymm21 {%k1}
+// CHECK:  encoding: [0x62,0x81,0x55,0x21,0xdc,0xe9]
+          vpaddusb %ymm25, %ymm21, %ymm21 {%k1}
+
+// CHECK: vpaddusb %ymm25, %ymm21, %ymm21 {%k1} {z}
+// CHECK:  encoding: [0x62,0x81,0x55,0xa1,0xdc,0xe9]
+          vpaddusb %ymm25, %ymm21, %ymm21 {%k1} {z}
+
+// CHECK: vpaddusb (%rcx), %ymm21, %ymm21
+// CHECK:  encoding: [0x62,0xe1,0x55,0x20,0xdc,0x29]
+          vpaddusb (%rcx), %ymm21, %ymm21
+
+// CHECK: vpaddusb 291(%rax,%r14,8), %ymm21, %ymm21
+// CHECK:  encoding: [0x62,0xa1,0x55,0x20,0xdc,0xac,0xf0,0x23,0x01,0x00,0x00]
+          vpaddusb 291(%rax,%r14,8), %ymm21, %ymm21
+
+// CHECK: vpaddusb 4064(%rdx), %ymm21, %ymm21
+// CHECK:  encoding: [0x62,0xe1,0x55,0x20,0xdc,0x6a,0x7f]
+          vpaddusb 4064(%rdx), %ymm21, %ymm21
+
+// CHECK: vpaddusb 4096(%rdx), %ymm21, %ymm21
+// CHECK:  encoding: [0x62,0xe1,0x55,0x20,0xdc,0xaa,0x00,0x10,0x00,0x00]
+          vpaddusb 4096(%rdx), %ymm21, %ymm21
+
+// CHECK: vpaddusb -4096(%rdx), %ymm21, %ymm21
+// CHECK:  encoding: [0x62,0xe1,0x55,0x20,0xdc,0x6a,0x80]
+          vpaddusb -4096(%rdx), %ymm21, %ymm21
+
+// CHECK: vpaddusb -4128(%rdx), %ymm21, %ymm21
+// CHECK:  encoding: [0x62,0xe1,0x55,0x20,0xdc,0xaa,0xe0,0xef,0xff,0xff]
+          vpaddusb -4128(%rdx), %ymm21, %ymm21
+
+// CHECK: vpaddusw %xmm26, %xmm17, %xmm24
+// CHECK:  encoding: [0x62,0x01,0x75,0x00,0xdd,0xc2]
+          vpaddusw %xmm26, %xmm17, %xmm24
+
+// CHECK: vpaddusw %xmm26, %xmm17, %xmm24 {%k6}
+// CHECK:  encoding: [0x62,0x01,0x75,0x06,0xdd,0xc2]
+          vpaddusw %xmm26, %xmm17, %xmm24 {%k6}
+
+// CHECK: vpaddusw %xmm26, %xmm17, %xmm24 {%k6} {z}
+// CHECK:  encoding: [0x62,0x01,0x75,0x86,0xdd,0xc2]
+          vpaddusw %xmm26, %xmm17, %xmm24 {%k6} {z}
+
+// CHECK: vpaddusw (%rcx), %xmm17, %xmm24
+// CHECK:  encoding: [0x62,0x61,0x75,0x00,0xdd,0x01]
+          vpaddusw (%rcx), %xmm17, %xmm24
+
+// CHECK: vpaddusw 291(%rax,%r14,8), %xmm17, %xmm24
+// CHECK:  encoding: [0x62,0x21,0x75,0x00,0xdd,0x84,0xf0,0x23,0x01,0x00,0x00]
+          vpaddusw 291(%rax,%r14,8), %xmm17, %xmm24
+
+// CHECK: vpaddusw 2032(%rdx), %xmm17, %xmm24
+// CHECK:  encoding: [0x62,0x61,0x75,0x00,0xdd,0x42,0x7f]
+          vpaddusw 2032(%rdx), %xmm17, %xmm24
+
+// CHECK: vpaddusw 2048(%rdx), %xmm17, %xmm24
+// CHECK:  encoding: [0x62,0x61,0x75,0x00,0xdd,0x82,0x00,0x08,0x00,0x00]
+          vpaddusw 2048(%rdx), %xmm17, %xmm24
+
+// CHECK: vpaddusw -2048(%rdx), %xmm17, %xmm24
+// CHECK:  encoding: [0x62,0x61,0x75,0x00,0xdd,0x42,0x80]
+          vpaddusw -2048(%rdx), %xmm17, %xmm24
+
+// CHECK: vpaddusw -2064(%rdx), %xmm17, %xmm24
+// CHECK:  encoding: [0x62,0x61,0x75,0x00,0xdd,0x82,0xf0,0xf7,0xff,0xff]
+          vpaddusw -2064(%rdx), %xmm17, %xmm24
+
+// CHECK: vpaddusw %ymm22, %ymm27, %ymm22
+// CHECK:  encoding: [0x62,0xa1,0x25,0x20,0xdd,0xf6]
+          vpaddusw %ymm22, %ymm27, %ymm22
+
+// CHECK: vpaddusw %ymm22, %ymm27, %ymm22 {%k5}
+// CHECK:  encoding: [0x62,0xa1,0x25,0x25,0xdd,0xf6]
+          vpaddusw %ymm22, %ymm27, %ymm22 {%k5}
+
+// CHECK: vpaddusw %ymm22, %ymm27, %ymm22 {%k5} {z}
+// CHECK:  encoding: [0x62,0xa1,0x25,0xa5,0xdd,0xf6]
+          vpaddusw %ymm22, %ymm27, %ymm22 {%k5} {z}
+
+// CHECK: vpaddusw (%rcx), %ymm27, %ymm22
+// CHECK:  encoding: [0x62,0xe1,0x25,0x20,0xdd,0x31]
+          vpaddusw (%rcx), %ymm27, %ymm22
+
+// CHECK: vpaddusw 291(%rax,%r14,8), %ymm27, %ymm22
+// CHECK:  encoding: [0x62,0xa1,0x25,0x20,0xdd,0xb4,0xf0,0x23,0x01,0x00,0x00]
+          vpaddusw 291(%rax,%r14,8), %ymm27, %ymm22
+
+// CHECK: vpaddusw 4064(%rdx), %ymm27, %ymm22
+// CHECK:  encoding: [0x62,0xe1,0x25,0x20,0xdd,0x72,0x7f]
+          vpaddusw 4064(%rdx), %ymm27, %ymm22
+
+// CHECK: vpaddusw 4096(%rdx), %ymm27, %ymm22
+// CHECK:  encoding: [0x62,0xe1,0x25,0x20,0xdd,0xb2,0x00,0x10,0x00,0x00]
+          vpaddusw 4096(%rdx), %ymm27, %ymm22
+
+// CHECK: vpaddusw -4096(%rdx), %ymm27, %ymm22
+// CHECK:  encoding: [0x62,0xe1,0x25,0x20,0xdd,0x72,0x80]
+          vpaddusw -4096(%rdx), %ymm27, %ymm22
+
+// CHECK: vpaddusw -4128(%rdx), %ymm27, %ymm22
+// CHECK:  encoding: [0x62,0xe1,0x25,0x20,0xdd,0xb2,0xe0,0xef,0xff,0xff]
+          vpaddusw -4128(%rdx), %ymm27, %ymm22
+
+// CHECK: vpsubsb %xmm19, %xmm19, %xmm28
+// CHECK:  encoding: [0x62,0x21,0x65,0x00,0xe8,0xe3]
+          vpsubsb %xmm19, %xmm19, %xmm28
+
+// CHECK: vpsubsb %xmm19, %xmm19, %xmm28 {%k4}
+// CHECK:  encoding: [0x62,0x21,0x65,0x04,0xe8,0xe3]
+          vpsubsb %xmm19, %xmm19, %xmm28 {%k4}
+
+// CHECK: vpsubsb %xmm19, %xmm19, %xmm28 {%k4} {z}
+// CHECK:  encoding: [0x62,0x21,0x65,0x84,0xe8,0xe3]
+          vpsubsb %xmm19, %xmm19, %xmm28 {%k4} {z}
+
+// CHECK: vpsubsb (%rcx), %xmm19, %xmm28
+// CHECK:  encoding: [0x62,0x61,0x65,0x00,0xe8,0x21]
+          vpsubsb (%rcx), %xmm19, %xmm28
+
+// CHECK: vpsubsb 291(%rax,%r14,8), %xmm19, %xmm28
+// CHECK:  encoding: [0x62,0x21,0x65,0x00,0xe8,0xa4,0xf0,0x23,0x01,0x00,0x00]
+          vpsubsb 291(%rax,%r14,8), %xmm19, %xmm28
+
+// CHECK: vpsubsb 2032(%rdx), %xmm19, %xmm28
+// CHECK:  encoding: [0x62,0x61,0x65,0x00,0xe8,0x62,0x7f]
+          vpsubsb 2032(%rdx), %xmm19, %xmm28
+
+// CHECK: vpsubsb 2048(%rdx), %xmm19, %xmm28
+// CHECK:  encoding: [0x62,0x61,0x65,0x00,0xe8,0xa2,0x00,0x08,0x00,0x00]
+          vpsubsb 2048(%rdx), %xmm19, %xmm28
+
+// CHECK: vpsubsb -2048(%rdx), %xmm19, %xmm28
+// CHECK:  encoding: [0x62,0x61,0x65,0x00,0xe8,0x62,0x80]
+          vpsubsb -2048(%rdx), %xmm19, %xmm28
+
+// CHECK: vpsubsb -2064(%rdx), %xmm19, %xmm28
+// CHECK:  encoding: [0x62,0x61,0x65,0x00,0xe8,0xa2,0xf0,0xf7,0xff,0xff]
+          vpsubsb -2064(%rdx), %xmm19, %xmm28
+
+// CHECK: vpsubsb %ymm18, %ymm23, %ymm18
+// CHECK:  encoding: [0x62,0xa1,0x45,0x20,0xe8,0xd2]
+          vpsubsb %ymm18, %ymm23, %ymm18
+
+// CHECK: vpsubsb %ymm18, %ymm23, %ymm18 {%k6}
+// CHECK:  encoding: [0x62,0xa1,0x45,0x26,0xe8,0xd2]
+          vpsubsb %ymm18, %ymm23, %ymm18 {%k6}
+
+// CHECK: vpsubsb %ymm18, %ymm23, %ymm18 {%k6} {z}
+// CHECK:  encoding: [0x62,0xa1,0x45,0xa6,0xe8,0xd2]
+          vpsubsb %ymm18, %ymm23, %ymm18 {%k6} {z}
+
+// CHECK: vpsubsb (%rcx), %ymm23, %ymm18
+// CHECK:  encoding: [0x62,0xe1,0x45,0x20,0xe8,0x11]
+          vpsubsb (%rcx), %ymm23, %ymm18
+
+// CHECK: vpsubsb 291(%rax,%r14,8), %ymm23, %ymm18
+// CHECK:  encoding: [0x62,0xa1,0x45,0x20,0xe8,0x94,0xf0,0x23,0x01,0x00,0x00]
+          vpsubsb 291(%rax,%r14,8), %ymm23, %ymm18
+
+// CHECK: vpsubsb 4064(%rdx), %ymm23, %ymm18
+// CHECK:  encoding: [0x62,0xe1,0x45,0x20,0xe8,0x52,0x7f]
+          vpsubsb 4064(%rdx), %ymm23, %ymm18
+
+// CHECK: vpsubsb 4096(%rdx), %ymm23, %ymm18
+// CHECK:  encoding: [0x62,0xe1,0x45,0x20,0xe8,0x92,0x00,0x10,0x00,0x00]
+          vpsubsb 4096(%rdx), %ymm23, %ymm18
+
+// CHECK: vpsubsb -4096(%rdx), %ymm23, %ymm18
+// CHECK:  encoding: [0x62,0xe1,0x45,0x20,0xe8,0x52,0x80]
+          vpsubsb -4096(%rdx), %ymm23, %ymm18
+
+// CHECK: vpsubsb -4128(%rdx), %ymm23, %ymm18
+// CHECK:  encoding: [0x62,0xe1,0x45,0x20,0xe8,0x92,0xe0,0xef,0xff,0xff]
+          vpsubsb -4128(%rdx), %ymm23, %ymm18
+
+// CHECK: vpsubsw %xmm19, %xmm24, %xmm26
+// CHECK:  encoding: [0x62,0x21,0x3d,0x00,0xe9,0xd3]
+          vpsubsw %xmm19, %xmm24, %xmm26
+
+// CHECK: vpsubsw %xmm19, %xmm24, %xmm26 {%k7}
+// CHECK:  encoding: [0x62,0x21,0x3d,0x07,0xe9,0xd3]
+          vpsubsw %xmm19, %xmm24, %xmm26 {%k7}
+
+// CHECK: vpsubsw %xmm19, %xmm24, %xmm26 {%k7} {z}
+// CHECK:  encoding: [0x62,0x21,0x3d,0x87,0xe9,0xd3]
+          vpsubsw %xmm19, %xmm24, %xmm26 {%k7} {z}
+
+// CHECK: vpsubsw (%rcx), %xmm24, %xmm26
+// CHECK:  encoding: [0x62,0x61,0x3d,0x00,0xe9,0x11]
+          vpsubsw (%rcx), %xmm24, %xmm26
+
+// CHECK: vpsubsw 291(%rax,%r14,8), %xmm24, %xmm26
+// CHECK:  encoding: [0x62,0x21,0x3d,0x00,0xe9,0x94,0xf0,0x23,0x01,0x00,0x00]
+          vpsubsw 291(%rax,%r14,8), %xmm24, %xmm26
+
+// CHECK: vpsubsw 2032(%rdx), %xmm24, %xmm26
+// CHECK:  encoding: [0x62,0x61,0x3d,0x00,0xe9,0x52,0x7f]
+          vpsubsw 2032(%rdx), %xmm24, %xmm26
+
+// CHECK: vpsubsw 2048(%rdx), %xmm24, %xmm26
+// CHECK:  encoding: [0x62,0x61,0x3d,0x00,0xe9,0x92,0x00,0x08,0x00,0x00]
+          vpsubsw 2048(%rdx), %xmm24, %xmm26
+
+// CHECK: vpsubsw -2048(%rdx), %xmm24, %xmm26
+// CHECK:  encoding: [0x62,0x61,0x3d,0x00,0xe9,0x52,0x80]
+          vpsubsw -2048(%rdx), %xmm24, %xmm26
+
+// CHECK: vpsubsw -2064(%rdx), %xmm24, %xmm26
+// CHECK:  encoding: [0x62,0x61,0x3d,0x00,0xe9,0x92,0xf0,0xf7,0xff,0xff]
+          vpsubsw -2064(%rdx), %xmm24, %xmm26
+
+// CHECK: vpsubsw %ymm19, %ymm27, %ymm20
+// CHECK:  encoding: [0x62,0xa1,0x25,0x20,0xe9,0xe3]
+          vpsubsw %ymm19, %ymm27, %ymm20
+
+// CHECK: vpsubsw %ymm19, %ymm27, %ymm20 {%k3}
+// CHECK:  encoding: [0x62,0xa1,0x25,0x23,0xe9,0xe3]
+          vpsubsw %ymm19, %ymm27, %ymm20 {%k3}
+
+// CHECK: vpsubsw %ymm19, %ymm27, %ymm20 {%k3} {z}
+// CHECK:  encoding: [0x62,0xa1,0x25,0xa3,0xe9,0xe3]
+          vpsubsw %ymm19, %ymm27, %ymm20 {%k3} {z}
+
+// CHECK: vpsubsw (%rcx), %ymm27, %ymm20
+// CHECK:  encoding: [0x62,0xe1,0x25,0x20,0xe9,0x21]
+          vpsubsw (%rcx), %ymm27, %ymm20
+
+// CHECK: vpsubsw 291(%rax,%r14,8), %ymm27, %ymm20
+// CHECK:  encoding: [0x62,0xa1,0x25,0x20,0xe9,0xa4,0xf0,0x23,0x01,0x00,0x00]
+          vpsubsw 291(%rax,%r14,8), %ymm27, %ymm20
+
+// CHECK: vpsubsw 4064(%rdx), %ymm27, %ymm20
+// CHECK:  encoding: [0x62,0xe1,0x25,0x20,0xe9,0x62,0x7f]
+          vpsubsw 4064(%rdx), %ymm27, %ymm20
+
+// CHECK: vpsubsw 4096(%rdx), %ymm27, %ymm20
+// CHECK:  encoding: [0x62,0xe1,0x25,0x20,0xe9,0xa2,0x00,0x10,0x00,0x00]
+          vpsubsw 4096(%rdx), %ymm27, %ymm20
+
+// CHECK: vpsubsw -4096(%rdx), %ymm27, %ymm20
+// CHECK:  encoding: [0x62,0xe1,0x25,0x20,0xe9,0x62,0x80]
+          vpsubsw -4096(%rdx), %ymm27, %ymm20
+
+// CHECK: vpsubsw -4128(%rdx), %ymm27, %ymm20
+// CHECK:  encoding: [0x62,0xe1,0x25,0x20,0xe9,0xa2,0xe0,0xef,0xff,0xff]
+          vpsubsw -4128(%rdx), %ymm27, %ymm20
+
+// CHECK: vpsubusb %xmm25, %xmm19, %xmm20
+// CHECK:  encoding: [0x62,0x81,0x65,0x00,0xd8,0xe1]
+          vpsubusb %xmm25, %xmm19, %xmm20
+
+// CHECK: vpsubusb %xmm25, %xmm19, %xmm20 {%k6}
+// CHECK:  encoding: [0x62,0x81,0x65,0x06,0xd8,0xe1]
+          vpsubusb %xmm25, %xmm19, %xmm20 {%k6}
+
+// CHECK: vpsubusb %xmm25, %xmm19, %xmm20 {%k6} {z}
+// CHECK:  encoding: [0x62,0x81,0x65,0x86,0xd8,0xe1]
+          vpsubusb %xmm25, %xmm19, %xmm20 {%k6} {z}
+
+// CHECK: vpsubusb (%rcx), %xmm19, %xmm20
+// CHECK:  encoding: [0x62,0xe1,0x65,0x00,0xd8,0x21]
+          vpsubusb (%rcx), %xmm19, %xmm20
+
+// CHECK: vpsubusb 291(%rax,%r14,8), %xmm19, %xmm20
+// CHECK:  encoding: [0x62,0xa1,0x65,0x00,0xd8,0xa4,0xf0,0x23,0x01,0x00,0x00]
+          vpsubusb 291(%rax,%r14,8), %xmm19, %xmm20
+
+// CHECK: vpsubusb 2032(%rdx), %xmm19, %xmm20
+// CHECK:  encoding: [0x62,0xe1,0x65,0x00,0xd8,0x62,0x7f]
+          vpsubusb 2032(%rdx), %xmm19, %xmm20
+
+// CHECK: vpsubusb 2048(%rdx), %xmm19, %xmm20
+// CHECK:  encoding: [0x62,0xe1,0x65,0x00,0xd8,0xa2,0x00,0x08,0x00,0x00]
+          vpsubusb 2048(%rdx), %xmm19, %xmm20
+
+// CHECK: vpsubusb -2048(%rdx), %xmm19, %xmm20
+// CHECK:  encoding: [0x62,0xe1,0x65,0x00,0xd8,0x62,0x80]
+          vpsubusb -2048(%rdx), %xmm19, %xmm20
+
+// CHECK: vpsubusb -2064(%rdx), %xmm19, %xmm20
+// CHECK:  encoding: [0x62,0xe1,0x65,0x00,0xd8,0xa2,0xf0,0xf7,0xff,0xff]
+          vpsubusb -2064(%rdx), %xmm19, %xmm20
+
+// CHECK: vpsubusb %ymm19, %ymm19, %ymm21
+// CHECK:  encoding: [0x62,0xa1,0x65,0x20,0xd8,0xeb]
+          vpsubusb %ymm19, %ymm19, %ymm21
+
+// CHECK: vpsubusb %ymm19, %ymm19, %ymm21 {%k3}
+// CHECK:  encoding: [0x62,0xa1,0x65,0x23,0xd8,0xeb]
+          vpsubusb %ymm19, %ymm19, %ymm21 {%k3}
+
+// CHECK: vpsubusb %ymm19, %ymm19, %ymm21 {%k3} {z}
+// CHECK:  encoding: [0x62,0xa1,0x65,0xa3,0xd8,0xeb]
+          vpsubusb %ymm19, %ymm19, %ymm21 {%k3} {z}
+
+// CHECK: vpsubusb (%rcx), %ymm19, %ymm21
+// CHECK:  encoding: [0x62,0xe1,0x65,0x20,0xd8,0x29]
+          vpsubusb (%rcx), %ymm19, %ymm21
+
+// CHECK: vpsubusb 291(%rax,%r14,8), %ymm19, %ymm21
+// CHECK:  encoding: [0x62,0xa1,0x65,0x20,0xd8,0xac,0xf0,0x23,0x01,0x00,0x00]
+          vpsubusb 291(%rax,%r14,8), %ymm19, %ymm21
+
+// CHECK: vpsubusb 4064(%rdx), %ymm19, %ymm21
+// CHECK:  encoding: [0x62,0xe1,0x65,0x20,0xd8,0x6a,0x7f]
+          vpsubusb 4064(%rdx), %ymm19, %ymm21
+
+// CHECK: vpsubusb 4096(%rdx), %ymm19, %ymm21
+// CHECK:  encoding: [0x62,0xe1,0x65,0x20,0xd8,0xaa,0x00,0x10,0x00,0x00]
+          vpsubusb 4096(%rdx), %ymm19, %ymm21
+
+// CHECK: vpsubusb -4096(%rdx), %ymm19, %ymm21
+// CHECK:  encoding: [0x62,0xe1,0x65,0x20,0xd8,0x6a,0x80]
+          vpsubusb -4096(%rdx), %ymm19, %ymm21
+
+// CHECK: vpsubusb -4128(%rdx), %ymm19, %ymm21
+// CHECK:  encoding: [0x62,0xe1,0x65,0x20,0xd8,0xaa,0xe0,0xef,0xff,0xff]
+          vpsubusb -4128(%rdx), %ymm19, %ymm21
+
+// CHECK: vpsubusw %xmm22, %xmm20, %xmm23
+// CHECK:  encoding: [0x62,0xa1,0x5d,0x00,0xd9,0xfe]
+          vpsubusw %xmm22, %xmm20, %xmm23
+
+// CHECK: vpsubusw %xmm22, %xmm20, %xmm23 {%k1}
+// CHECK:  encoding: [0x62,0xa1,0x5d,0x01,0xd9,0xfe]
+          vpsubusw %xmm22, %xmm20, %xmm23 {%k1}
+
+// CHECK: vpsubusw %xmm22, %xmm20, %xmm23 {%k1} {z}
+// CHECK:  encoding: [0x62,0xa1,0x5d,0x81,0xd9,0xfe]
+          vpsubusw %xmm22, %xmm20, %xmm23 {%k1} {z}
+
+// CHECK: vpsubusw (%rcx), %xmm20, %xmm23
+// CHECK:  encoding: [0x62,0xe1,0x5d,0x00,0xd9,0x39]
+          vpsubusw (%rcx), %xmm20, %xmm23
+
+// CHECK: vpsubusw 291(%rax,%r14,8), %xmm20, %xmm23
+// CHECK:  encoding: [0x62,0xa1,0x5d,0x00,0xd9,0xbc,0xf0,0x23,0x01,0x00,0x00]
+          vpsubusw 291(%rax,%r14,8), %xmm20, %xmm23
+
+// CHECK: vpsubusw 2032(%rdx), %xmm20, %xmm23
+// CHECK:  encoding: [0x62,0xe1,0x5d,0x00,0xd9,0x7a,0x7f]
+          vpsubusw 2032(%rdx), %xmm20, %xmm23
+
+// CHECK: vpsubusw 2048(%rdx), %xmm20, %xmm23
+// CHECK:  encoding: [0x62,0xe1,0x5d,0x00,0xd9,0xba,0x00,0x08,0x00,0x00]
+          vpsubusw 2048(%rdx), %xmm20, %xmm23
+
+// CHECK: vpsubusw -2048(%rdx), %xmm20, %xmm23
+// CHECK:  encoding: [0x62,0xe1,0x5d,0x00,0xd9,0x7a,0x80]
+          vpsubusw -2048(%rdx), %xmm20, %xmm23
+
+// CHECK: vpsubusw -2064(%rdx), %xmm20, %xmm23
+// CHECK:  encoding: [0x62,0xe1,0x5d,0x00,0xd9,0xba,0xf0,0xf7,0xff,0xff]
+          vpsubusw -2064(%rdx), %xmm20, %xmm23
+
+// CHECK: vpsubusw %ymm28, %ymm17, %ymm27
+// CHECK:  encoding: [0x62,0x01,0x75,0x20,0xd9,0xdc]
+          vpsubusw %ymm28, %ymm17, %ymm27
+
+// CHECK: vpsubusw %ymm28, %ymm17, %ymm27 {%k3}
+// CHECK:  encoding: [0x62,0x01,0x75,0x23,0xd9,0xdc]
+          vpsubusw %ymm28, %ymm17, %ymm27 {%k3}
+
+// CHECK: vpsubusw %ymm28, %ymm17, %ymm27 {%k3} {z}
+// CHECK:  encoding: [0x62,0x01,0x75,0xa3,0xd9,0xdc]
+          vpsubusw %ymm28, %ymm17, %ymm27 {%k3} {z}
+
+// CHECK: vpsubusw (%rcx), %ymm17, %ymm27
+// CHECK:  encoding: [0x62,0x61,0x75,0x20,0xd9,0x19]
+          vpsubusw (%rcx), %ymm17, %ymm27
+
+// CHECK: vpsubusw 291(%rax,%r14,8), %ymm17, %ymm27
+// CHECK:  encoding: [0x62,0x21,0x75,0x20,0xd9,0x9c,0xf0,0x23,0x01,0x00,0x00]
+          vpsubusw 291(%rax,%r14,8), %ymm17, %ymm27
+
+// CHECK: vpsubusw 4064(%rdx), %ymm17, %ymm27
+// CHECK:  encoding: [0x62,0x61,0x75,0x20,0xd9,0x5a,0x7f]
+          vpsubusw 4064(%rdx), %ymm17, %ymm27
+
+// CHECK: vpsubusw 4096(%rdx), %ymm17, %ymm27
+// CHECK:  encoding: [0x62,0x61,0x75,0x20,0xd9,0x9a,0x00,0x10,0x00,0x00]
+          vpsubusw 4096(%rdx), %ymm17, %ymm27
+
+// CHECK: vpsubusw -4096(%rdx), %ymm17, %ymm27
+// CHECK:  encoding: [0x62,0x61,0x75,0x20,0xd9,0x5a,0x80]
+          vpsubusw -4096(%rdx), %ymm17, %ymm27
+
+// CHECK: vpsubusw -4128(%rdx), %ymm17, %ymm27
+// CHECK:  encoding: [0x62,0x61,0x75,0x20,0xd9,0x9a,0xe0,0xef,0xff,0xff]
+          vpsubusw -4128(%rdx), %ymm17, %ymm27
+
+// CHECK: vpaddsb %xmm18, %xmm23, %xmm17
+// CHECK:  encoding: [0x62,0xa1,0x45,0x00,0xec,0xca]
+          vpaddsb %xmm18, %xmm23, %xmm17
+
+// CHECK: vpaddsb %xmm18, %xmm23, %xmm17 {%k2}
+// CHECK:  encoding: [0x62,0xa1,0x45,0x02,0xec,0xca]
+          vpaddsb %xmm18, %xmm23, %xmm17 {%k2}
+
+// CHECK: vpaddsb %xmm18, %xmm23, %xmm17 {%k2} {z}
+// CHECK:  encoding: [0x62,0xa1,0x45,0x82,0xec,0xca]
+          vpaddsb %xmm18, %xmm23, %xmm17 {%k2} {z}
+
+// CHECK: vpaddsb (%rcx), %xmm23, %xmm17
+// CHECK:  encoding: [0x62,0xe1,0x45,0x00,0xec,0x09]
+          vpaddsb (%rcx), %xmm23, %xmm17
+
+// CHECK: vpaddsb 4660(%rax,%r14,8), %xmm23, %xmm17
+// CHECK:  encoding: [0x62,0xa1,0x45,0x00,0xec,0x8c,0xf0,0x34,0x12,0x00,0x00]
+          vpaddsb 4660(%rax,%r14,8), %xmm23, %xmm17
+
+// CHECK: vpaddsb 2032(%rdx), %xmm23, %xmm17
+// CHECK:  encoding: [0x62,0xe1,0x45,0x00,0xec,0x4a,0x7f]
+          vpaddsb 2032(%rdx), %xmm23, %xmm17
+
+// CHECK: vpaddsb 2048(%rdx), %xmm23, %xmm17
+// CHECK:  encoding: [0x62,0xe1,0x45,0x00,0xec,0x8a,0x00,0x08,0x00,0x00]
+          vpaddsb 2048(%rdx), %xmm23, %xmm17
+
+// CHECK: vpaddsb -2048(%rdx), %xmm23, %xmm17
+// CHECK:  encoding: [0x62,0xe1,0x45,0x00,0xec,0x4a,0x80]
+          vpaddsb -2048(%rdx), %xmm23, %xmm17
+
+// CHECK: vpaddsb -2064(%rdx), %xmm23, %xmm17
+// CHECK:  encoding: [0x62,0xe1,0x45,0x00,0xec,0x8a,0xf0,0xf7,0xff,0xff]
+          vpaddsb -2064(%rdx), %xmm23, %xmm17
+
+// CHECK: vpaddsb %ymm19, %ymm26, %ymm20
+// CHECK:  encoding: [0x62,0xa1,0x2d,0x20,0xec,0xe3]
+          vpaddsb %ymm19, %ymm26, %ymm20
+
+// CHECK: vpaddsb %ymm19, %ymm26, %ymm20 {%k2}
+// CHECK:  encoding: [0x62,0xa1,0x2d,0x22,0xec,0xe3]
+          vpaddsb %ymm19, %ymm26, %ymm20 {%k2}
+
+// CHECK: vpaddsb %ymm19, %ymm26, %ymm20 {%k2} {z}
+// CHECK:  encoding: [0x62,0xa1,0x2d,0xa2,0xec,0xe3]
+          vpaddsb %ymm19, %ymm26, %ymm20 {%k2} {z}
+
+// CHECK: vpaddsb (%rcx), %ymm26, %ymm20
+// CHECK:  encoding: [0x62,0xe1,0x2d,0x20,0xec,0x21]
+          vpaddsb (%rcx), %ymm26, %ymm20
+
+// CHECK: vpaddsb 4660(%rax,%r14,8), %ymm26, %ymm20
+// CHECK:  encoding: [0x62,0xa1,0x2d,0x20,0xec,0xa4,0xf0,0x34,0x12,0x00,0x00]
+          vpaddsb 4660(%rax,%r14,8), %ymm26, %ymm20
+
+// CHECK: vpaddsb 4064(%rdx), %ymm26, %ymm20
+// CHECK:  encoding: [0x62,0xe1,0x2d,0x20,0xec,0x62,0x7f]
+          vpaddsb 4064(%rdx), %ymm26, %ymm20
+
+// CHECK: vpaddsb 4096(%rdx), %ymm26, %ymm20
+// CHECK:  encoding: [0x62,0xe1,0x2d,0x20,0xec,0xa2,0x00,0x10,0x00,0x00]
+          vpaddsb 4096(%rdx), %ymm26, %ymm20
+
+// CHECK: vpaddsb -4096(%rdx), %ymm26, %ymm20
+// CHECK:  encoding: [0x62,0xe1,0x2d,0x20,0xec,0x62,0x80]
+          vpaddsb -4096(%rdx), %ymm26, %ymm20
+
+// CHECK: vpaddsb -4128(%rdx), %ymm26, %ymm20
+// CHECK:  encoding: [0x62,0xe1,0x2d,0x20,0xec,0xa2,0xe0,0xef,0xff,0xff]
+          vpaddsb -4128(%rdx), %ymm26, %ymm20
+
+// CHECK: vpaddsw %xmm26, %xmm17, %xmm26
+// CHECK:  encoding: [0x62,0x01,0x75,0x00,0xed,0xd2]
+          vpaddsw %xmm26, %xmm17, %xmm26
+
+// CHECK: vpaddsw %xmm26, %xmm17, %xmm26 {%k6}
+// CHECK:  encoding: [0x62,0x01,0x75,0x06,0xed,0xd2]
+          vpaddsw %xmm26, %xmm17, %xmm26 {%k6}
+
+// CHECK: vpaddsw %xmm26, %xmm17, %xmm26 {%k6} {z}
+// CHECK:  encoding: [0x62,0x01,0x75,0x86,0xed,0xd2]
+          vpaddsw %xmm26, %xmm17, %xmm26 {%k6} {z}
+
+// CHECK: vpaddsw (%rcx), %xmm17, %xmm26
+// CHECK:  encoding: [0x62,0x61,0x75,0x00,0xed,0x11]
+          vpaddsw (%rcx), %xmm17, %xmm26
+
+// CHECK: vpaddsw 4660(%rax,%r14,8), %xmm17, %xmm26
+// CHECK:  encoding: [0x62,0x21,0x75,0x00,0xed,0x94,0xf0,0x34,0x12,0x00,0x00]
+          vpaddsw 4660(%rax,%r14,8), %xmm17, %xmm26
+
+// CHECK: vpaddsw 2032(%rdx), %xmm17, %xmm26
+// CHECK:  encoding: [0x62,0x61,0x75,0x00,0xed,0x52,0x7f]
+          vpaddsw 2032(%rdx), %xmm17, %xmm26
+
+// CHECK: vpaddsw 2048(%rdx), %xmm17, %xmm26
+// CHECK:  encoding: [0x62,0x61,0x75,0x00,0xed,0x92,0x00,0x08,0x00,0x00]
+          vpaddsw 2048(%rdx), %xmm17, %xmm26
+
+// CHECK: vpaddsw -2048(%rdx), %xmm17, %xmm26
+// CHECK:  encoding: [0x62,0x61,0x75,0x00,0xed,0x52,0x80]
+          vpaddsw -2048(%rdx), %xmm17, %xmm26
+
+// CHECK: vpaddsw -2064(%rdx), %xmm17, %xmm26
+// CHECK:  encoding: [0x62,0x61,0x75,0x00,0xed,0x92,0xf0,0xf7,0xff,0xff]
+          vpaddsw -2064(%rdx), %xmm17, %xmm26
+
+// CHECK: vpaddsw %ymm23, %ymm27, %ymm28
+// CHECK:  encoding: [0x62,0x21,0x25,0x20,0xed,0xe7]
+          vpaddsw %ymm23, %ymm27, %ymm28
+
+// CHECK: vpaddsw %ymm23, %ymm27, %ymm28 {%k4}
+// CHECK:  encoding: [0x62,0x21,0x25,0x24,0xed,0xe7]
+          vpaddsw %ymm23, %ymm27, %ymm28 {%k4}
+
+// CHECK: vpaddsw %ymm23, %ymm27, %ymm28 {%k4} {z}
+// CHECK:  encoding: [0x62,0x21,0x25,0xa4,0xed,0xe7]
+          vpaddsw %ymm23, %ymm27, %ymm28 {%k4} {z}
+
+// CHECK: vpaddsw (%rcx), %ymm27, %ymm28
+// CHECK:  encoding: [0x62,0x61,0x25,0x20,0xed,0x21]
+          vpaddsw (%rcx), %ymm27, %ymm28
+
+// CHECK: vpaddsw 4660(%rax,%r14,8), %ymm27, %ymm28
+// CHECK:  encoding: [0x62,0x21,0x25,0x20,0xed,0xa4,0xf0,0x34,0x12,0x00,0x00]
+          vpaddsw 4660(%rax,%r14,8), %ymm27, %ymm28
+
+// CHECK: vpaddsw 4064(%rdx), %ymm27, %ymm28
+// CHECK:  encoding: [0x62,0x61,0x25,0x20,0xed,0x62,0x7f]
+          vpaddsw 4064(%rdx), %ymm27, %ymm28
+
+// CHECK: vpaddsw 4096(%rdx), %ymm27, %ymm28
+// CHECK:  encoding: [0x62,0x61,0x25,0x20,0xed,0xa2,0x00,0x10,0x00,0x00]
+          vpaddsw 4096(%rdx), %ymm27, %ymm28
+
+// CHECK: vpaddsw -4096(%rdx), %ymm27, %ymm28
+// CHECK:  encoding: [0x62,0x61,0x25,0x20,0xed,0x62,0x80]
+          vpaddsw -4096(%rdx), %ymm27, %ymm28
+
+// CHECK: vpaddsw -4128(%rdx), %ymm27, %ymm28
+// CHECK:  encoding: [0x62,0x61,0x25,0x20,0xed,0xa2,0xe0,0xef,0xff,0xff]
+          vpaddsw -4128(%rdx), %ymm27, %ymm28
+
+// CHECK: vpaddusb %xmm18, %xmm21, %xmm17
+// CHECK:  encoding: [0x62,0xa1,0x55,0x00,0xdc,0xca]
+          vpaddusb %xmm18, %xmm21, %xmm17
+
+// CHECK: vpaddusb %xmm18, %xmm21, %xmm17 {%k4}
+// CHECK:  encoding: [0x62,0xa1,0x55,0x04,0xdc,0xca]
+          vpaddusb %xmm18, %xmm21, %xmm17 {%k4}
+
+// CHECK: vpaddusb %xmm18, %xmm21, %xmm17 {%k4} {z}
+// CHECK:  encoding: [0x62,0xa1,0x55,0x84,0xdc,0xca]
+          vpaddusb %xmm18, %xmm21, %xmm17 {%k4} {z}
+
+// CHECK: vpaddusb (%rcx), %xmm21, %xmm17
+// CHECK:  encoding: [0x62,0xe1,0x55,0x00,0xdc,0x09]
+          vpaddusb (%rcx), %xmm21, %xmm17
+
+// CHECK: vpaddusb 4660(%rax,%r14,8), %xmm21, %xmm17
+// CHECK:  encoding: [0x62,0xa1,0x55,0x00,0xdc,0x8c,0xf0,0x34,0x12,0x00,0x00]
+          vpaddusb 4660(%rax,%r14,8), %xmm21, %xmm17
+
+// CHECK: vpaddusb 2032(%rdx), %xmm21, %xmm17
+// CHECK:  encoding: [0x62,0xe1,0x55,0x00,0xdc,0x4a,0x7f]
+          vpaddusb 2032(%rdx), %xmm21, %xmm17
+
+// CHECK: vpaddusb 2048(%rdx), %xmm21, %xmm17
+// CHECK:  encoding: [0x62,0xe1,0x55,0x00,0xdc,0x8a,0x00,0x08,0x00,0x00]
+          vpaddusb 2048(%rdx), %xmm21, %xmm17
+
+// CHECK: vpaddusb -2048(%rdx), %xmm21, %xmm17
+// CHECK:  encoding: [0x62,0xe1,0x55,0x00,0xdc,0x4a,0x80]
+          vpaddusb -2048(%rdx), %xmm21, %xmm17
+
+// CHECK: vpaddusb -2064(%rdx), %xmm21, %xmm17
+// CHECK:  encoding: [0x62,0xe1,0x55,0x00,0xdc,0x8a,0xf0,0xf7,0xff,0xff]
+          vpaddusb -2064(%rdx), %xmm21, %xmm17
+
+// CHECK: vpaddusb %ymm24, %ymm20, %ymm21
+// CHECK:  encoding: [0x62,0x81,0x5d,0x20,0xdc,0xe8]
+          vpaddusb %ymm24, %ymm20, %ymm21
+
+// CHECK: vpaddusb %ymm24, %ymm20, %ymm21 {%k2}
+// CHECK:  encoding: [0x62,0x81,0x5d,0x22,0xdc,0xe8]
+          vpaddusb %ymm24, %ymm20, %ymm21 {%k2}
+
+// CHECK: vpaddusb %ymm24, %ymm20, %ymm21 {%k2} {z}
+// CHECK:  encoding: [0x62,0x81,0x5d,0xa2,0xdc,0xe8]
+          vpaddusb %ymm24, %ymm20, %ymm21 {%k2} {z}
+
+// CHECK: vpaddusb (%rcx), %ymm20, %ymm21
+// CHECK:  encoding: [0x62,0xe1,0x5d,0x20,0xdc,0x29]
+          vpaddusb (%rcx), %ymm20, %ymm21
+
+// CHECK: vpaddusb 4660(%rax,%r14,8), %ymm20, %ymm21
+// CHECK:  encoding: [0x62,0xa1,0x5d,0x20,0xdc,0xac,0xf0,0x34,0x12,0x00,0x00]
+          vpaddusb 4660(%rax,%r14,8), %ymm20, %ymm21
+
+// CHECK: vpaddusb 4064(%rdx), %ymm20, %ymm21
+// CHECK:  encoding: [0x62,0xe1,0x5d,0x20,0xdc,0x6a,0x7f]
+          vpaddusb 4064(%rdx), %ymm20, %ymm21
+
+// CHECK: vpaddusb 4096(%rdx), %ymm20, %ymm21
+// CHECK:  encoding: [0x62,0xe1,0x5d,0x20,0xdc,0xaa,0x00,0x10,0x00,0x00]
+          vpaddusb 4096(%rdx), %ymm20, %ymm21
+
+// CHECK: vpaddusb -4096(%rdx), %ymm20, %ymm21
+// CHECK:  encoding: [0x62,0xe1,0x5d,0x20,0xdc,0x6a,0x80]
+          vpaddusb -4096(%rdx), %ymm20, %ymm21
+
+// CHECK: vpaddusb -4128(%rdx), %ymm20, %ymm21
+// CHECK:  encoding: [0x62,0xe1,0x5d,0x20,0xdc,0xaa,0xe0,0xef,0xff,0xff]
+          vpaddusb -4128(%rdx), %ymm20, %ymm21
+
+// CHECK: vpaddusw %xmm28, %xmm24, %xmm22
+// CHECK:  encoding: [0x62,0x81,0x3d,0x00,0xdd,0xf4]
+          vpaddusw %xmm28, %xmm24, %xmm22
+
+// CHECK: vpaddusw %xmm28, %xmm24, %xmm22 {%k6}
+// CHECK:  encoding: [0x62,0x81,0x3d,0x06,0xdd,0xf4]
+          vpaddusw %xmm28, %xmm24, %xmm22 {%k6}
+
+// CHECK: vpaddusw %xmm28, %xmm24, %xmm22 {%k6} {z}
+// CHECK:  encoding: [0x62,0x81,0x3d,0x86,0xdd,0xf4]
+          vpaddusw %xmm28, %xmm24, %xmm22 {%k6} {z}
+
+// CHECK: vpaddusw (%rcx), %xmm24, %xmm22
+// CHECK:  encoding: [0x62,0xe1,0x3d,0x00,0xdd,0x31]
+          vpaddusw (%rcx), %xmm24, %xmm22
+
+// CHECK: vpaddusw 4660(%rax,%r14,8), %xmm24, %xmm22
+// CHECK:  encoding: [0x62,0xa1,0x3d,0x00,0xdd,0xb4,0xf0,0x34,0x12,0x00,0x00]
+          vpaddusw 4660(%rax,%r14,8), %xmm24, %xmm22
+
+// CHECK: vpaddusw 2032(%rdx), %xmm24, %xmm22
+// CHECK:  encoding: [0x62,0xe1,0x3d,0x00,0xdd,0x72,0x7f]
+          vpaddusw 2032(%rdx), %xmm24, %xmm22
+
+// CHECK: vpaddusw 2048(%rdx), %xmm24, %xmm22
+// CHECK:  encoding: [0x62,0xe1,0x3d,0x00,0xdd,0xb2,0x00,0x08,0x00,0x00]
+          vpaddusw 2048(%rdx), %xmm24, %xmm22
+
+// CHECK: vpaddusw -2048(%rdx), %xmm24, %xmm22
+// CHECK:  encoding: [0x62,0xe1,0x3d,0x00,0xdd,0x72,0x80]
+          vpaddusw -2048(%rdx), %xmm24, %xmm22
+
+// CHECK: vpaddusw -2064(%rdx), %xmm24, %xmm22
+// CHECK:  encoding: [0x62,0xe1,0x3d,0x00,0xdd,0xb2,0xf0,0xf7,0xff,0xff]
+          vpaddusw -2064(%rdx), %xmm24, %xmm22
+
+// CHECK: vpaddusw %ymm25, %ymm19, %ymm21
+// CHECK:  encoding: [0x62,0x81,0x65,0x20,0xdd,0xe9]
+          vpaddusw %ymm25, %ymm19, %ymm21
+
+// CHECK: vpaddusw %ymm25, %ymm19, %ymm21 {%k2}
+// CHECK:  encoding: [0x62,0x81,0x65,0x22,0xdd,0xe9]
+          vpaddusw %ymm25, %ymm19, %ymm21 {%k2}
+
+// CHECK: vpaddusw %ymm25, %ymm19, %ymm21 {%k2} {z}
+// CHECK:  encoding: [0x62,0x81,0x65,0xa2,0xdd,0xe9]
+          vpaddusw %ymm25, %ymm19, %ymm21 {%k2} {z}
+
+// CHECK: vpaddusw (%rcx), %ymm19, %ymm21
+// CHECK:  encoding: [0x62,0xe1,0x65,0x20,0xdd,0x29]
+          vpaddusw (%rcx), %ymm19, %ymm21
+
+// CHECK: vpaddusw 4660(%rax,%r14,8), %ymm19, %ymm21
+// CHECK:  encoding: [0x62,0xa1,0x65,0x20,0xdd,0xac,0xf0,0x34,0x12,0x00,0x00]
+          vpaddusw 4660(%rax,%r14,8), %ymm19, %ymm21
+
+// CHECK: vpaddusw 4064(%rdx), %ymm19, %ymm21
+// CHECK:  encoding: [0x62,0xe1,0x65,0x20,0xdd,0x6a,0x7f]
+          vpaddusw 4064(%rdx), %ymm19, %ymm21
+
+// CHECK: vpaddusw 4096(%rdx), %ymm19, %ymm21
+// CHECK:  encoding: [0x62,0xe1,0x65,0x20,0xdd,0xaa,0x00,0x10,0x00,0x00]
+          vpaddusw 4096(%rdx), %ymm19, %ymm21
+
+// CHECK: vpaddusw -4096(%rdx), %ymm19, %ymm21
+// CHECK:  encoding: [0x62,0xe1,0x65,0x20,0xdd,0x6a,0x80]
+          vpaddusw -4096(%rdx), %ymm19, %ymm21
+
+// CHECK: vpaddusw -4128(%rdx), %ymm19, %ymm21
+// CHECK:  encoding: [0x62,0xe1,0x65,0x20,0xdd,0xaa,0xe0,0xef,0xff,0xff]
+          vpaddusw -4128(%rdx), %ymm19, %ymm21
+
+// CHECK: vpsubsb %xmm22, %xmm26, %xmm20
+// CHECK:  encoding: [0x62,0xa1,0x2d,0x00,0xe8,0xe6]
+          vpsubsb %xmm22, %xmm26, %xmm20
+
+// CHECK: vpsubsb %xmm22, %xmm26, %xmm20 {%k1}
+// CHECK:  encoding: [0x62,0xa1,0x2d,0x01,0xe8,0xe6]
+          vpsubsb %xmm22, %xmm26, %xmm20 {%k1}
+
+// CHECK: vpsubsb %xmm22, %xmm26, %xmm20 {%k1} {z}
+// CHECK:  encoding: [0x62,0xa1,0x2d,0x81,0xe8,0xe6]
+          vpsubsb %xmm22, %xmm26, %xmm20 {%k1} {z}
+
+// CHECK: vpsubsb (%rcx), %xmm26, %xmm20
+// CHECK:  encoding: [0x62,0xe1,0x2d,0x00,0xe8,0x21]
+          vpsubsb (%rcx), %xmm26, %xmm20
+
+// CHECK: vpsubsb 4660(%rax,%r14,8), %xmm26, %xmm20
+// CHECK:  encoding: [0x62,0xa1,0x2d,0x00,0xe8,0xa4,0xf0,0x34,0x12,0x00,0x00]
+          vpsubsb 4660(%rax,%r14,8), %xmm26, %xmm20
+
+// CHECK: vpsubsb 2032(%rdx), %xmm26, %xmm20
+// CHECK:  encoding: [0x62,0xe1,0x2d,0x00,0xe8,0x62,0x7f]
+          vpsubsb 2032(%rdx), %xmm26, %xmm20
+
+// CHECK: vpsubsb 2048(%rdx), %xmm26, %xmm20
+// CHECK:  encoding: [0x62,0xe1,0x2d,0x00,0xe8,0xa2,0x00,0x08,0x00,0x00]
+          vpsubsb 2048(%rdx), %xmm26, %xmm20
+
+// CHECK: vpsubsb -2048(%rdx), %xmm26, %xmm20
+// CHECK:  encoding: [0x62,0xe1,0x2d,0x00,0xe8,0x62,0x80]
+          vpsubsb -2048(%rdx), %xmm26, %xmm20
+
+// CHECK: vpsubsb -2064(%rdx), %xmm26, %xmm20
+// CHECK:  encoding: [0x62,0xe1,0x2d,0x00,0xe8,0xa2,0xf0,0xf7,0xff,0xff]
+          vpsubsb -2064(%rdx), %xmm26, %xmm20
+
+// CHECK: vpsubsb %ymm26, %ymm18, %ymm25
+// CHECK:  encoding: [0x62,0x01,0x6d,0x20,0xe8,0xca]
+          vpsubsb %ymm26, %ymm18, %ymm25
+
+// CHECK: vpsubsb %ymm26, %ymm18, %ymm25 {%k6}
+// CHECK:  encoding: [0x62,0x01,0x6d,0x26,0xe8,0xca]
+          vpsubsb %ymm26, %ymm18, %ymm25 {%k6}
+
+// CHECK: vpsubsb %ymm26, %ymm18, %ymm25 {%k6} {z}
+// CHECK:  encoding: [0x62,0x01,0x6d,0xa6,0xe8,0xca]
+          vpsubsb %ymm26, %ymm18, %ymm25 {%k6} {z}
+
+// CHECK: vpsubsb (%rcx), %ymm18, %ymm25
+// CHECK:  encoding: [0x62,0x61,0x6d,0x20,0xe8,0x09]
+          vpsubsb (%rcx), %ymm18, %ymm25
+
+// CHECK: vpsubsb 4660(%rax,%r14,8), %ymm18, %ymm25
+// CHECK:  encoding: [0x62,0x21,0x6d,0x20,0xe8,0x8c,0xf0,0x34,0x12,0x00,0x00]
+          vpsubsb 4660(%rax,%r14,8), %ymm18, %ymm25
+
+// CHECK: vpsubsb 4064(%rdx), %ymm18, %ymm25
+// CHECK:  encoding: [0x62,0x61,0x6d,0x20,0xe8,0x4a,0x7f]
+          vpsubsb 4064(%rdx), %ymm18, %ymm25
+
+// CHECK: vpsubsb 4096(%rdx), %ymm18, %ymm25
+// CHECK:  encoding: [0x62,0x61,0x6d,0x20,0xe8,0x8a,0x00,0x10,0x00,0x00]
+          vpsubsb 4096(%rdx), %ymm18, %ymm25
+
+// CHECK: vpsubsb -4096(%rdx), %ymm18, %ymm25
+// CHECK:  encoding: [0x62,0x61,0x6d,0x20,0xe8,0x4a,0x80]
+          vpsubsb -4096(%rdx), %ymm18, %ymm25
+
+// CHECK: vpsubsb -4128(%rdx), %ymm18, %ymm25
+// CHECK:  encoding: [0x62,0x61,0x6d,0x20,0xe8,0x8a,0xe0,0xef,0xff,0xff]
+          vpsubsb -4128(%rdx), %ymm18, %ymm25
+
+// CHECK: vpsubsw %xmm28, %xmm27, %xmm26
+// CHECK:  encoding: [0x62,0x01,0x25,0x00,0xe9,0xd4]
+          vpsubsw %xmm28, %xmm27, %xmm26
+
+// CHECK: vpsubsw %xmm28, %xmm27, %xmm26 {%k5}
+// CHECK:  encoding: [0x62,0x01,0x25,0x05,0xe9,0xd4]
+          vpsubsw %xmm28, %xmm27, %xmm26 {%k5}
+
+// CHECK: vpsubsw %xmm28, %xmm27, %xmm26 {%k5} {z}
+// CHECK:  encoding: [0x62,0x01,0x25,0x85,0xe9,0xd4]
+          vpsubsw %xmm28, %xmm27, %xmm26 {%k5} {z}
+
+// CHECK: vpsubsw (%rcx), %xmm27, %xmm26
+// CHECK:  encoding: [0x62,0x61,0x25,0x00,0xe9,0x11]
+          vpsubsw (%rcx), %xmm27, %xmm26
+
+// CHECK: vpsubsw 4660(%rax,%r14,8), %xmm27, %xmm26
+// CHECK:  encoding: [0x62,0x21,0x25,0x00,0xe9,0x94,0xf0,0x34,0x12,0x00,0x00]
+          vpsubsw 4660(%rax,%r14,8), %xmm27, %xmm26
+
+// CHECK: vpsubsw 2032(%rdx), %xmm27, %xmm26
+// CHECK:  encoding: [0x62,0x61,0x25,0x00,0xe9,0x52,0x7f]
+          vpsubsw 2032(%rdx), %xmm27, %xmm26
+
+// CHECK: vpsubsw 2048(%rdx), %xmm27, %xmm26
+// CHECK:  encoding: [0x62,0x61,0x25,0x00,0xe9,0x92,0x00,0x08,0x00,0x00]
+          vpsubsw 2048(%rdx), %xmm27, %xmm26
+
+// CHECK: vpsubsw -2048(%rdx), %xmm27, %xmm26
+// CHECK:  encoding: [0x62,0x61,0x25,0x00,0xe9,0x52,0x80]
+          vpsubsw -2048(%rdx), %xmm27, %xmm26
+
+// CHECK: vpsubsw -2064(%rdx), %xmm27, %xmm26
+// CHECK:  encoding: [0x62,0x61,0x25,0x00,0xe9,0x92,0xf0,0xf7,0xff,0xff]
+          vpsubsw -2064(%rdx), %xmm27, %xmm26
+
+// CHECK: vpsubsw %ymm21, %ymm25, %ymm22
+// CHECK:  encoding: [0x62,0xa1,0x35,0x20,0xe9,0xf5]
+          vpsubsw %ymm21, %ymm25, %ymm22
+
+// CHECK: vpsubsw %ymm21, %ymm25, %ymm22 {%k3}
+// CHECK:  encoding: [0x62,0xa1,0x35,0x23,0xe9,0xf5]
+          vpsubsw %ymm21, %ymm25, %ymm22 {%k3}
+
+// CHECK: vpsubsw %ymm21, %ymm25, %ymm22 {%k3} {z}
+// CHECK:  encoding: [0x62,0xa1,0x35,0xa3,0xe9,0xf5]
+          vpsubsw %ymm21, %ymm25, %ymm22 {%k3} {z}
+
+// CHECK: vpsubsw (%rcx), %ymm25, %ymm22
+// CHECK:  encoding: [0x62,0xe1,0x35,0x20,0xe9,0x31]
+          vpsubsw (%rcx), %ymm25, %ymm22
+
+// CHECK: vpsubsw 4660(%rax,%r14,8), %ymm25, %ymm22
+// CHECK:  encoding: [0x62,0xa1,0x35,0x20,0xe9,0xb4,0xf0,0x34,0x12,0x00,0x00]
+          vpsubsw 4660(%rax,%r14,8), %ymm25, %ymm22
+
+// CHECK: vpsubsw 4064(%rdx), %ymm25, %ymm22
+// CHECK:  encoding: [0x62,0xe1,0x35,0x20,0xe9,0x72,0x7f]
+          vpsubsw 4064(%rdx), %ymm25, %ymm22
+
+// CHECK: vpsubsw 4096(%rdx), %ymm25, %ymm22
+// CHECK:  encoding: [0x62,0xe1,0x35,0x20,0xe9,0xb2,0x00,0x10,0x00,0x00]
+          vpsubsw 4096(%rdx), %ymm25, %ymm22
+
+// CHECK: vpsubsw -4096(%rdx), %ymm25, %ymm22
+// CHECK:  encoding: [0x62,0xe1,0x35,0x20,0xe9,0x72,0x80]
+          vpsubsw -4096(%rdx), %ymm25, %ymm22
+
+// CHECK: vpsubsw -4128(%rdx), %ymm25, %ymm22
+// CHECK:  encoding: [0x62,0xe1,0x35,0x20,0xe9,0xb2,0xe0,0xef,0xff,0xff]
+          vpsubsw -4128(%rdx), %ymm25, %ymm22
+
+// CHECK: vpsubusb %xmm26, %xmm27, %xmm18
+// CHECK:  encoding: [0x62,0x81,0x25,0x00,0xd8,0xd2]
+          vpsubusb %xmm26, %xmm27, %xmm18
+
+// CHECK: vpsubusb %xmm26, %xmm27, %xmm18 {%k2}
+// CHECK:  encoding: [0x62,0x81,0x25,0x02,0xd8,0xd2]
+          vpsubusb %xmm26, %xmm27, %xmm18 {%k2}
+
+// CHECK: vpsubusb %xmm26, %xmm27, %xmm18 {%k2} {z}
+// CHECK:  encoding: [0x62,0x81,0x25,0x82,0xd8,0xd2]
+          vpsubusb %xmm26, %xmm27, %xmm18 {%k2} {z}
+
+// CHECK: vpsubusb (%rcx), %xmm27, %xmm18
+// CHECK:  encoding: [0x62,0xe1,0x25,0x00,0xd8,0x11]
+          vpsubusb (%rcx), %xmm27, %xmm18
+
+// CHECK: vpsubusb 4660(%rax,%r14,8), %xmm27, %xmm18
+// CHECK:  encoding: [0x62,0xa1,0x25,0x00,0xd8,0x94,0xf0,0x34,0x12,0x00,0x00]
+          vpsubusb 4660(%rax,%r14,8), %xmm27, %xmm18
+
+// CHECK: vpsubusb 2032(%rdx), %xmm27, %xmm18
+// CHECK:  encoding: [0x62,0xe1,0x25,0x00,0xd8,0x52,0x7f]
+          vpsubusb 2032(%rdx), %xmm27, %xmm18
+
+// CHECK: vpsubusb 2048(%rdx), %xmm27, %xmm18
+// CHECK:  encoding: [0x62,0xe1,0x25,0x00,0xd8,0x92,0x00,0x08,0x00,0x00]
+          vpsubusb 2048(%rdx), %xmm27, %xmm18
+
+// CHECK: vpsubusb -2048(%rdx), %xmm27, %xmm18
+// CHECK:  encoding: [0x62,0xe1,0x25,0x00,0xd8,0x52,0x80]
+          vpsubusb -2048(%rdx), %xmm27, %xmm18
+
+// CHECK: vpsubusb -2064(%rdx), %xmm27, %xmm18
+// CHECK:  encoding: [0x62,0xe1,0x25,0x00,0xd8,0x92,0xf0,0xf7,0xff,0xff]
+          vpsubusb -2064(%rdx), %xmm27, %xmm18
+
+// CHECK: vpsubusb %ymm19, %ymm19, %ymm25
+// CHECK:  encoding: [0x62,0x21,0x65,0x20,0xd8,0xcb]
+          vpsubusb %ymm19, %ymm19, %ymm25
+
+// CHECK: vpsubusb %ymm19, %ymm19, %ymm25 {%k6}
+// CHECK:  encoding: [0x62,0x21,0x65,0x26,0xd8,0xcb]
+          vpsubusb %ymm19, %ymm19, %ymm25 {%k6}
+
+// CHECK: vpsubusb %ymm19, %ymm19, %ymm25 {%k6} {z}
+// CHECK:  encoding: [0x62,0x21,0x65,0xa6,0xd8,0xcb]
+          vpsubusb %ymm19, %ymm19, %ymm25 {%k6} {z}
+
+// CHECK: vpsubusb (%rcx), %ymm19, %ymm25
+// CHECK:  encoding: [0x62,0x61,0x65,0x20,0xd8,0x09]
+          vpsubusb (%rcx), %ymm19, %ymm25
+
+// CHECK: vpsubusb 4660(%rax,%r14,8), %ymm19, %ymm25
+// CHECK:  encoding: [0x62,0x21,0x65,0x20,0xd8,0x8c,0xf0,0x34,0x12,0x00,0x00]
+          vpsubusb 4660(%rax,%r14,8), %ymm19, %ymm25
+
+// CHECK: vpsubusb 4064(%rdx), %ymm19, %ymm25
+// CHECK:  encoding: [0x62,0x61,0x65,0x20,0xd8,0x4a,0x7f]
+          vpsubusb 4064(%rdx), %ymm19, %ymm25
+
+// CHECK: vpsubusb 4096(%rdx), %ymm19, %ymm25
+// CHECK:  encoding: [0x62,0x61,0x65,0x20,0xd8,0x8a,0x00,0x10,0x00,0x00]
+          vpsubusb 4096(%rdx), %ymm19, %ymm25
+
+// CHECK: vpsubusb -4096(%rdx), %ymm19, %ymm25
+// CHECK:  encoding: [0x62,0x61,0x65,0x20,0xd8,0x4a,0x80]
+          vpsubusb -4096(%rdx), %ymm19, %ymm25
+
+// CHECK: vpsubusb -4128(%rdx), %ymm19, %ymm25
+// CHECK:  encoding: [0x62,0x61,0x65,0x20,0xd8,0x8a,0xe0,0xef,0xff,0xff]
+          vpsubusb -4128(%rdx), %ymm19, %ymm25
+
+// CHECK: vpsubusw %xmm25, %xmm27, %xmm21
+// CHECK:  encoding: [0x62,0x81,0x25,0x00,0xd9,0xe9]
+          vpsubusw %xmm25, %xmm27, %xmm21
+
+// CHECK: vpsubusw %xmm25, %xmm27, %xmm21 {%k5}
+// CHECK:  encoding: [0x62,0x81,0x25,0x05,0xd9,0xe9]
+          vpsubusw %xmm25, %xmm27, %xmm21 {%k5}
+
+// CHECK: vpsubusw %xmm25, %xmm27, %xmm21 {%k5} {z}
+// CHECK:  encoding: [0x62,0x81,0x25,0x85,0xd9,0xe9]
+          vpsubusw %xmm25, %xmm27, %xmm21 {%k5} {z}
+
+// CHECK: vpsubusw (%rcx), %xmm27, %xmm21
+// CHECK:  encoding: [0x62,0xe1,0x25,0x00,0xd9,0x29]
+          vpsubusw (%rcx), %xmm27, %xmm21
+
+// CHECK: vpsubusw 4660(%rax,%r14,8), %xmm27, %xmm21
+// CHECK:  encoding: [0x62,0xa1,0x25,0x00,0xd9,0xac,0xf0,0x34,0x12,0x00,0x00]
+          vpsubusw 4660(%rax,%r14,8), %xmm27, %xmm21
+
+// CHECK: vpsubusw 2032(%rdx), %xmm27, %xmm21
+// CHECK:  encoding: [0x62,0xe1,0x25,0x00,0xd9,0x6a,0x7f]
+          vpsubusw 2032(%rdx), %xmm27, %xmm21
+
+// CHECK: vpsubusw 2048(%rdx), %xmm27, %xmm21
+// CHECK:  encoding: [0x62,0xe1,0x25,0x00,0xd9,0xaa,0x00,0x08,0x00,0x00]
+          vpsubusw 2048(%rdx), %xmm27, %xmm21
+
+// CHECK: vpsubusw -2048(%rdx), %xmm27, %xmm21
+// CHECK:  encoding: [0x62,0xe1,0x25,0x00,0xd9,0x6a,0x80]
+          vpsubusw -2048(%rdx), %xmm27, %xmm21
+
+// CHECK: vpsubusw -2064(%rdx), %xmm27, %xmm21
+// CHECK:  encoding: [0x62,0xe1,0x25,0x00,0xd9,0xaa,0xf0,0xf7,0xff,0xff]
+          vpsubusw -2064(%rdx), %xmm27, %xmm21
+
+// CHECK: vpsubusw %ymm17, %ymm25, %ymm27
+// CHECK:  encoding: [0x62,0x21,0x35,0x20,0xd9,0xd9]
+          vpsubusw %ymm17, %ymm25, %ymm27
+
+// CHECK: vpsubusw %ymm17, %ymm25, %ymm27 {%k4}
+// CHECK:  encoding: [0x62,0x21,0x35,0x24,0xd9,0xd9]
+          vpsubusw %ymm17, %ymm25, %ymm27 {%k4}
+
+// CHECK: vpsubusw %ymm17, %ymm25, %ymm27 {%k4} {z}
+// CHECK:  encoding: [0x62,0x21,0x35,0xa4,0xd9,0xd9]
+          vpsubusw %ymm17, %ymm25, %ymm27 {%k4} {z}
+
+// CHECK: vpsubusw (%rcx), %ymm25, %ymm27
+// CHECK:  encoding: [0x62,0x61,0x35,0x20,0xd9,0x19]
+          vpsubusw (%rcx), %ymm25, %ymm27
+
+// CHECK: vpsubusw 4660(%rax,%r14,8), %ymm25, %ymm27
+// CHECK:  encoding: [0x62,0x21,0x35,0x20,0xd9,0x9c,0xf0,0x34,0x12,0x00,0x00]
+          vpsubusw 4660(%rax,%r14,8), %ymm25, %ymm27
+
+// CHECK: vpsubusw 4064(%rdx), %ymm25, %ymm27
+// CHECK:  encoding: [0x62,0x61,0x35,0x20,0xd9,0x5a,0x7f]
+          vpsubusw 4064(%rdx), %ymm25, %ymm27
+
+// CHECK: vpsubusw 4096(%rdx), %ymm25, %ymm27
+// CHECK:  encoding: [0x62,0x61,0x35,0x20,0xd9,0x9a,0x00,0x10,0x00,0x00]
+          vpsubusw 4096(%rdx), %ymm25, %ymm27
+
+// CHECK: vpsubusw -4096(%rdx), %ymm25, %ymm27
+// CHECK:  encoding: [0x62,0x61,0x35,0x20,0xd9,0x5a,0x80]
+          vpsubusw -4096(%rdx), %ymm25, %ymm27
+
+// CHECK: vpsubusw -4128(%rdx), %ymm25, %ymm27
+// CHECK:  encoding: [0x62,0x61,0x35,0x20,0xd9,0x9a,0xe0,0xef,0xff,0xff]
+          vpsubusw -4128(%rdx), %ymm25, %ymm27
+
+// CHECK: vpshufhw $171, %xmm19, %xmm23
+// CHECK:  encoding: [0x62,0xa1,0xfe,0x08,0x70,0xfb,0xab]
+          vpshufhw $171, %xmm19, %xmm23
+
+// CHECK: vpshufhw $171, %xmm19, %xmm23 {%k7}
+// CHECK:  encoding: [0x62,0xa1,0xfe,0x0f,0x70,0xfb,0xab]
+          vpshufhw $171, %xmm19, %xmm23 {%k7}
+
+// CHECK: vpshufhw $171, %xmm19, %xmm23 {%k7} {z}
+// CHECK:  encoding: [0x62,0xa1,0xfe,0x8f,0x70,0xfb,0xab]
+          vpshufhw $171, %xmm19, %xmm23 {%k7} {z}
+
+// CHECK: vpshufhw $123,  %xmm19, %xmm23
+// CHECK:  encoding: [0x62,0xa1,0xfe,0x08,0x70,0xfb,0x7b]
+          vpshufhw $123,  %xmm19, %xmm23
+
+// CHECK: vpshufhw $123,  (%rcx), %xmm23
+// CHECK:  encoding: [0x62,0xe1,0xfe,0x08,0x70,0x39,0x7b]
+          vpshufhw $123, (%rcx), %xmm23
+
+// CHECK: vpshufhw $123, 291(%rax,%r14,8), %xmm23
+// CHECK:  encoding: [0x62,0xa1,0xfe,0x08,0x70,0xbc,0xf0,0x23,0x01,0x00,0x00,0x7b]
+          vpshufhw $123, 291(%rax,%r14,8), %xmm23
+
+// CHECK: vpshufhw $123, 2032(%rdx), %xmm23
+// CHECK:  encoding: [0x62,0xe1,0xfe,0x08,0x70,0x7a,0x7f,0x7b]
+          vpshufhw $123, 2032(%rdx), %xmm23
+
+// CHECK: vpshufhw $123, 2048(%rdx), %xmm23
+// CHECK:  encoding: [0x62,0xe1,0xfe,0x08,0x70,0xba,0x00,0x08,0x00,0x00,0x7b]
+          vpshufhw $123, 2048(%rdx), %xmm23
+
+// CHECK: vpshufhw $123, -2048(%rdx), %xmm23
+// CHECK:  encoding: [0x62,0xe1,0xfe,0x08,0x70,0x7a,0x80,0x7b]
+          vpshufhw $123, -2048(%rdx), %xmm23
+
+// CHECK: vpshufhw $123, -2064(%rdx), %xmm23
+// CHECK:  encoding: [0x62,0xe1,0xfe,0x08,0x70,0xba,0xf0,0xf7,0xff,0xff,0x7b]
+          vpshufhw $123, -2064(%rdx), %xmm23
+
+// CHECK: vpshufhw $171, %ymm17, %ymm29
+// CHECK:  encoding: [0x62,0x21,0xfe,0x28,0x70,0xe9,0xab]
+          vpshufhw $171, %ymm17, %ymm29
+
+// CHECK: vpshufhw $171, %ymm17, %ymm29 {%k7}
+// CHECK:  encoding: [0x62,0x21,0xfe,0x2f,0x70,0xe9,0xab]
+          vpshufhw $171, %ymm17, %ymm29 {%k7}
+
+// CHECK: vpshufhw $171, %ymm17, %ymm29 {%k7} {z}
+// CHECK:  encoding: [0x62,0x21,0xfe,0xaf,0x70,0xe9,0xab]
+          vpshufhw $171, %ymm17, %ymm29 {%k7} {z}
+
+// CHECK: vpshufhw $123,  %ymm17, %ymm29
+// CHECK:  encoding: [0x62,0x21,0xfe,0x28,0x70,0xe9,0x7b]
+          vpshufhw $123,  %ymm17, %ymm29
+
+// CHECK: vpshufhw $123, (%rcx), %ymm29
+// CHECK:  encoding: [0x62,0x61,0xfe,0x28,0x70,0x29,0x7b]
+          vpshufhw $123, (%rcx), %ymm29
+
+// CHECK: vpshufhw $123, 291(%rax,%r14,8), %ymm29
+// CHECK:  encoding: [0x62,0x21,0xfe,0x28,0x70,0xac,0xf0,0x23,0x01,0x00,0x00,0x7b]
+          vpshufhw $123, 291(%rax,%r14,8), %ymm29
+
+// CHECK: vpshufhw $123, 4064(%rdx), %ymm29
+// CHECK:  encoding: [0x62,0x61,0xfe,0x28,0x70,0x6a,0x7f,0x7b]
+          vpshufhw $123, 4064(%rdx), %ymm29
+
+// CHECK: vpshufhw $123, 4096(%rdx), %ymm29
+// CHECK:  encoding: [0x62,0x61,0xfe,0x28,0x70,0xaa,0x00,0x10,0x00,0x00,0x7b]
+          vpshufhw $123, 4096(%rdx), %ymm29
+
+// CHECK: vpshufhw $123, -4096(%rdx), %ymm29
+// CHECK:  encoding: [0x62,0x61,0xfe,0x28,0x70,0x6a,0x80,0x7b]
+          vpshufhw $123, -4096(%rdx), %ymm29
+
+// CHECK: vpshufhw $123, -4128(%rdx), %ymm29
+// CHECK:  encoding: [0x62,0x61,0xfe,0x28,0x70,0xaa,0xe0,0xef,0xff,0xff,0x7b]
+          vpshufhw $123, -4128(%rdx), %ymm29
+
+// CHECK: vpshuflw $171, %xmm27, %xmm30
+// CHECK:  encoding: [0x62,0x01,0xff,0x08,0x70,0xf3,0xab]
+          vpshuflw $171, %xmm27, %xmm30
+
+// CHECK: vpshuflw $171, %xmm27, %xmm30 {%k6}
+// CHECK:  encoding: [0x62,0x01,0xff,0x0e,0x70,0xf3,0xab]
+          vpshuflw $171, %xmm27, %xmm30 {%k6}
+
+// CHECK: vpshuflw $171, %xmm27, %xmm30 {%k6} {z}
+// CHECK:  encoding: [0x62,0x01,0xff,0x8e,0x70,0xf3,0xab]
+          vpshuflw $171, %xmm27, %xmm30 {%k6} {z}
+
+// CHECK: vpshuflw $123,  %xmm27, %xmm30
+// CHECK:  encoding: [0x62,0x01,0xff,0x08,0x70,0xf3,0x7b]
+          vpshuflw $123,  %xmm27, %xmm30
+
+// CHECK: vpshuflw $123, (%rcx), %xmm30
+// CHECK:  encoding: [0x62,0x61,0xff,0x08,0x70,0x31,0x7b]
+          vpshuflw $123, (%rcx), %xmm30
+
+// CHECK: vpshuflw $123, 291(%rax,%r14,8), %xmm30
+// CHECK:  encoding: [0x62,0x21,0xff,0x08,0x70,0xb4,0xf0,0x23,0x01,0x00,0x00,0x7b]
+          vpshuflw $123, 291(%rax,%r14,8), %xmm30
+
+// CHECK: vpshuflw $123, 2032(%rdx), %xmm30
+// CHECK:  encoding: [0x62,0x61,0xff,0x08,0x70,0x72,0x7f,0x7b]
+          vpshuflw $123, 2032(%rdx), %xmm30
+
+// CHECK: vpshuflw $123, 2048(%rdx), %xmm30
+// CHECK:  encoding: [0x62,0x61,0xff,0x08,0x70,0xb2,0x00,0x08,0x00,0x00,0x7b]
+          vpshuflw $123, 2048(%rdx), %xmm30
+
+// CHECK: vpshuflw $123, -2048(%rdx), %xmm30
+// CHECK:  encoding: [0x62,0x61,0xff,0x08,0x70,0x72,0x80,0x7b]
+          vpshuflw $123, -2048(%rdx), %xmm30
+
+// CHECK: vpshuflw $123, -2064(%rdx), %xmm30
+// CHECK:  encoding: [0x62,0x61,0xff,0x08,0x70,0xb2,0xf0,0xf7,0xff,0xff,0x7b]
+          vpshuflw $123, -2064(%rdx), %xmm30
+
+// CHECK: vpshuflw $171, %ymm25, %ymm25
+// CHECK:  encoding: [0x62,0x01,0xff,0x28,0x70,0xc9,0xab]
+          vpshuflw $171, %ymm25, %ymm25
+
+// CHECK: vpshuflw $171, %ymm25, %ymm25 {%k5}
+// CHECK:  encoding: [0x62,0x01,0xff,0x2d,0x70,0xc9,0xab]
+          vpshuflw $171, %ymm25, %ymm25 {%k5}
+
+// CHECK: vpshuflw $171, %ymm25, %ymm25 {%k5} {z}
+// CHECK:  encoding: [0x62,0x01,0xff,0xad,0x70,0xc9,0xab]
+          vpshuflw $171, %ymm25, %ymm25 {%k5} {z}
+
+// CHECK: vpshuflw $123,  %ymm25, %ymm25
+// CHECK:  encoding: [0x62,0x01,0xff,0x28,0x70,0xc9,0x7b]
+          vpshuflw $123,  %ymm25, %ymm25
+
+// CHECK: vpshuflw $123, (%rcx), %ymm25
+// CHECK:  encoding: [0x62,0x61,0xff,0x28,0x70,0x09,0x7b]
+          vpshuflw $123, (%rcx), %ymm25
+
+// CHECK: vpshuflw $123, 291(%rax,%r14,8), %ymm25
+// CHECK:  encoding: [0x62,0x21,0xff,0x28,0x70,0x8c,0xf0,0x23,0x01,0x00,0x00,0x7b]
+          vpshuflw $123, 291(%rax,%r14,8), %ymm25
+
+// CHECK: vpshuflw $123, 4064(%rdx), %ymm25
+// CHECK:  encoding: [0x62,0x61,0xff,0x28,0x70,0x4a,0x7f,0x7b]
+          vpshuflw $123, 4064(%rdx), %ymm25
+
+// CHECK: vpshuflw $123, 4096(%rdx), %ymm25
+// CHECK:  encoding: [0x62,0x61,0xff,0x28,0x70,0x8a,0x00,0x10,0x00,0x00,0x7b]
+          vpshuflw $123, 4096(%rdx), %ymm25
+
+// CHECK: vpshuflw $123, -4096(%rdx), %ymm25
+// CHECK:  encoding: [0x62,0x61,0xff,0x28,0x70,0x4a,0x80,0x7b]
+          vpshuflw $123, -4096(%rdx), %ymm25
+
+// CHECK: vpshuflw $123, -4128(%rdx), %ymm25
+// CHECK:  encoding: [0x62,0x61,0xff,0x28,0x70,0x8a,0xe0,0xef,0xff,0xff,0x7b]
+          vpshuflw $123, -4128(%rdx), %ymm25
+
+// CHECK: vpermi2w %xmm21, %xmm29, %xmm19
+// CHECK:  encoding: [0x62,0xa2,0x95,0x00,0x75,0xdd]
+          vpermi2w %xmm21, %xmm29, %xmm19
+
+// CHECK: vpermi2w %xmm21, %xmm29, %xmm19 {%k2}
+// CHECK:  encoding: [0x62,0xa2,0x95,0x02,0x75,0xdd]
+          vpermi2w %xmm21, %xmm29, %xmm19 {%k2}
+
+// CHECK: vpermi2w %xmm21, %xmm29, %xmm19 {%k2} {z}
+// CHECK:  encoding: [0x62,0xa2,0x95,0x82,0x75,0xdd]
+          vpermi2w %xmm21, %xmm29, %xmm19 {%k2} {z}
+
+// CHECK: vpermi2w (%rcx), %xmm29, %xmm19
+// CHECK:  encoding: [0x62,0xe2,0x95,0x00,0x75,0x19]
+          vpermi2w (%rcx), %xmm29, %xmm19
+
+// CHECK: vpermi2w 291(%rax,%r14,8), %xmm29, %xmm19
+// CHECK:  encoding: [0x62,0xa2,0x95,0x00,0x75,0x9c,0xf0,0x23,0x01,0x00,0x00]
+          vpermi2w 291(%rax,%r14,8), %xmm29, %xmm19
+
+// CHECK: vpermi2w 2032(%rdx), %xmm29, %xmm19
+// CHECK:  encoding: [0x62,0xe2,0x95,0x00,0x75,0x5a,0x7f]
+          vpermi2w 2032(%rdx), %xmm29, %xmm19
+
+// CHECK: vpermi2w 2048(%rdx), %xmm29, %xmm19
+// CHECK:  encoding: [0x62,0xe2,0x95,0x00,0x75,0x9a,0x00,0x08,0x00,0x00]
+          vpermi2w 2048(%rdx), %xmm29, %xmm19
+
+// CHECK: vpermi2w -2048(%rdx), %xmm29, %xmm19
+// CHECK:  encoding: [0x62,0xe2,0x95,0x00,0x75,0x5a,0x80]
+          vpermi2w -2048(%rdx), %xmm29, %xmm19
+
+// CHECK: vpermi2w -2064(%rdx), %xmm29, %xmm19
+// CHECK:  encoding: [0x62,0xe2,0x95,0x00,0x75,0x9a,0xf0,0xf7,0xff,0xff]
+          vpermi2w -2064(%rdx), %xmm29, %xmm19
+
+// CHECK: vpermi2w %ymm19, %ymm25, %ymm30
+// CHECK:  encoding: [0x62,0x22,0xb5,0x20,0x75,0xf3]
+          vpermi2w %ymm19, %ymm25, %ymm30
+
+// CHECK: vpermi2w %ymm19, %ymm25, %ymm30 {%k3}
+// CHECK:  encoding: [0x62,0x22,0xb5,0x23,0x75,0xf3]
+          vpermi2w %ymm19, %ymm25, %ymm30 {%k3}
+
+// CHECK: vpermi2w %ymm19, %ymm25, %ymm30 {%k3} {z}
+// CHECK:  encoding: [0x62,0x22,0xb5,0xa3,0x75,0xf3]
+          vpermi2w %ymm19, %ymm25, %ymm30 {%k3} {z}
+
+// CHECK: vpermi2w (%rcx), %ymm25, %ymm30
+// CHECK:  encoding: [0x62,0x62,0xb5,0x20,0x75,0x31]
+          vpermi2w (%rcx), %ymm25, %ymm30
+
+// CHECK: vpermi2w 291(%rax,%r14,8), %ymm25, %ymm30
+// CHECK:  encoding: [0x62,0x22,0xb5,0x20,0x75,0xb4,0xf0,0x23,0x01,0x00,0x00]
+          vpermi2w 291(%rax,%r14,8), %ymm25, %ymm30
+
+// CHECK: vpermi2w 4064(%rdx), %ymm25, %ymm30
+// CHECK:  encoding: [0x62,0x62,0xb5,0x20,0x75,0x72,0x7f]
+          vpermi2w 4064(%rdx), %ymm25, %ymm30
+
+// CHECK: vpermi2w 4096(%rdx), %ymm25, %ymm30
+// CHECK:  encoding: [0x62,0x62,0xb5,0x20,0x75,0xb2,0x00,0x10,0x00,0x00]
+          vpermi2w 4096(%rdx), %ymm25, %ymm30
+
+// CHECK: vpermi2w -4096(%rdx), %ymm25, %ymm30
+// CHECK:  encoding: [0x62,0x62,0xb5,0x20,0x75,0x72,0x80]
+          vpermi2w -4096(%rdx), %ymm25, %ymm30
+
+// CHECK: vpermi2w -4128(%rdx), %ymm25, %ymm30
+// CHECK:  encoding: [0x62,0x62,0xb5,0x20,0x75,0xb2,0xe0,0xef,0xff,0xff]
+          vpermi2w -4128(%rdx), %ymm25, %ymm30
+
+// CHECK: vpermt2w %xmm25, %xmm22, %xmm18
+// CHECK:  encoding: [0x62,0x82,0xcd,0x00,0x7d,0xd1]
+          vpermt2w %xmm25, %xmm22, %xmm18
+
+// CHECK: vpermt2w %xmm25, %xmm22, %xmm18 {%k6}
+// CHECK:  encoding: [0x62,0x82,0xcd,0x06,0x7d,0xd1]
+          vpermt2w %xmm25, %xmm22, %xmm18 {%k6}
+
+// CHECK: vpermt2w %xmm25, %xmm22, %xmm18 {%k6} {z}
+// CHECK:  encoding: [0x62,0x82,0xcd,0x86,0x7d,0xd1]
+          vpermt2w %xmm25, %xmm22, %xmm18 {%k6} {z}
+
+// CHECK: vpermt2w (%rcx), %xmm22, %xmm18
+// CHECK:  encoding: [0x62,0xe2,0xcd,0x00,0x7d,0x11]
+          vpermt2w (%rcx), %xmm22, %xmm18
+
+// CHECK: vpermt2w 291(%rax,%r14,8), %xmm22, %xmm18
+// CHECK:  encoding: [0x62,0xa2,0xcd,0x00,0x7d,0x94,0xf0,0x23,0x01,0x00,0x00]
+          vpermt2w 291(%rax,%r14,8), %xmm22, %xmm18
+
+// CHECK: vpermt2w 2032(%rdx), %xmm22, %xmm18
+// CHECK:  encoding: [0x62,0xe2,0xcd,0x00,0x7d,0x52,0x7f]
+          vpermt2w 2032(%rdx), %xmm22, %xmm18
+
+// CHECK: vpermt2w 2048(%rdx), %xmm22, %xmm18
+// CHECK:  encoding: [0x62,0xe2,0xcd,0x00,0x7d,0x92,0x00,0x08,0x00,0x00]
+          vpermt2w 2048(%rdx), %xmm22, %xmm18
+
+// CHECK: vpermt2w -2048(%rdx), %xmm22, %xmm18
+// CHECK:  encoding: [0x62,0xe2,0xcd,0x00,0x7d,0x52,0x80]
+          vpermt2w -2048(%rdx), %xmm22, %xmm18
+
+// CHECK: vpermt2w -2064(%rdx), %xmm22, %xmm18
+// CHECK:  encoding: [0x62,0xe2,0xcd,0x00,0x7d,0x92,0xf0,0xf7,0xff,0xff]
+          vpermt2w -2064(%rdx), %xmm22, %xmm18
+
+// CHECK: vpermt2w %ymm26, %ymm23, %ymm28
+// CHECK:  encoding: [0x62,0x02,0xc5,0x20,0x7d,0xe2]
+          vpermt2w %ymm26, %ymm23, %ymm28
+
+// CHECK: vpermt2w %ymm26, %ymm23, %ymm28 {%k4}
+// CHECK:  encoding: [0x62,0x02,0xc5,0x24,0x7d,0xe2]
+          vpermt2w %ymm26, %ymm23, %ymm28 {%k4}
+
+// CHECK: vpermt2w %ymm26, %ymm23, %ymm28 {%k4} {z}
+// CHECK:  encoding: [0x62,0x02,0xc5,0xa4,0x7d,0xe2]
+          vpermt2w %ymm26, %ymm23, %ymm28 {%k4} {z}
+
+// CHECK: vpermt2w (%rcx), %ymm23, %ymm28
+// CHECK:  encoding: [0x62,0x62,0xc5,0x20,0x7d,0x21]
+          vpermt2w (%rcx), %ymm23, %ymm28
+
+// CHECK: vpermt2w 291(%rax,%r14,8), %ymm23, %ymm28
+// CHECK:  encoding: [0x62,0x22,0xc5,0x20,0x7d,0xa4,0xf0,0x23,0x01,0x00,0x00]
+          vpermt2w 291(%rax,%r14,8), %ymm23, %ymm28
+
+// CHECK: vpermt2w 4064(%rdx), %ymm23, %ymm28
+// CHECK:  encoding: [0x62,0x62,0xc5,0x20,0x7d,0x62,0x7f]
+          vpermt2w 4064(%rdx), %ymm23, %ymm28
+
+// CHECK: vpermt2w 4096(%rdx), %ymm23, %ymm28
+// CHECK:  encoding: [0x62,0x62,0xc5,0x20,0x7d,0xa2,0x00,0x10,0x00,0x00]
+          vpermt2w 4096(%rdx), %ymm23, %ymm28
+
+// CHECK: vpermt2w -4096(%rdx), %ymm23, %ymm28
+// CHECK:  encoding: [0x62,0x62,0xc5,0x20,0x7d,0x62,0x80]
+          vpermt2w -4096(%rdx), %ymm23, %ymm28
+
+// CHECK: vpermt2w -4128(%rdx), %ymm23, %ymm28
+// CHECK:  encoding: [0x62,0x62,0xc5,0x20,0x7d,0xa2,0xe0,0xef,0xff,0xff]
+          vpermt2w -4128(%rdx), %ymm23, %ymm28
+// CHECK: vpavgb %xmm22, %xmm24, %xmm21
+// CHECK:  encoding: [0x62,0xa1,0x3d,0x00,0xe0,0xee]
+          vpavgb %xmm22, %xmm24, %xmm21
+
+// CHECK: vpavgb %xmm22, %xmm24, %xmm21 {%k7}
+// CHECK:  encoding: [0x62,0xa1,0x3d,0x07,0xe0,0xee]
+          vpavgb %xmm22, %xmm24, %xmm21 {%k7}
+
+// CHECK: vpavgb %xmm22, %xmm24, %xmm21 {%k7} {z}
+// CHECK:  encoding: [0x62,0xa1,0x3d,0x87,0xe0,0xee]
+          vpavgb %xmm22, %xmm24, %xmm21 {%k7} {z}
+
+// CHECK: vpavgb (%rcx), %xmm24, %xmm21
+// CHECK:  encoding: [0x62,0xe1,0x3d,0x00,0xe0,0x29]
+          vpavgb (%rcx), %xmm24, %xmm21
+
+// CHECK: vpavgb 4660(%rax,%r14,8), %xmm24, %xmm21
+// CHECK:  encoding: [0x62,0xa1,0x3d,0x00,0xe0,0xac,0xf0,0x34,0x12,0x00,0x00]
+          vpavgb 4660(%rax,%r14,8), %xmm24, %xmm21
+
+// CHECK: vpavgb 2032(%rdx), %xmm24, %xmm21
+// CHECK:  encoding: [0x62,0xe1,0x3d,0x00,0xe0,0x6a,0x7f]
+          vpavgb 2032(%rdx), %xmm24, %xmm21
+
+// CHECK: vpavgb 2048(%rdx), %xmm24, %xmm21
+// CHECK:  encoding: [0x62,0xe1,0x3d,0x00,0xe0,0xaa,0x00,0x08,0x00,0x00]
+          vpavgb 2048(%rdx), %xmm24, %xmm21
+
+// CHECK: vpavgb -2048(%rdx), %xmm24, %xmm21
+// CHECK:  encoding: [0x62,0xe1,0x3d,0x00,0xe0,0x6a,0x80]
+          vpavgb -2048(%rdx), %xmm24, %xmm21
+
+// CHECK: vpavgb -2064(%rdx), %xmm24, %xmm21
+// CHECK:  encoding: [0x62,0xe1,0x3d,0x00,0xe0,0xaa,0xf0,0xf7,0xff,0xff]
+          vpavgb -2064(%rdx), %xmm24, %xmm21
+
+// CHECK: vpavgb %ymm18, %ymm18, %ymm26
+// CHECK:  encoding: [0x62,0x21,0x6d,0x20,0xe0,0xd2]
+          vpavgb %ymm18, %ymm18, %ymm26
+
+// CHECK: vpavgb %ymm18, %ymm18, %ymm26 {%k4}
+// CHECK:  encoding: [0x62,0x21,0x6d,0x24,0xe0,0xd2]
+          vpavgb %ymm18, %ymm18, %ymm26 {%k4}
+
+// CHECK: vpavgb %ymm18, %ymm18, %ymm26 {%k4} {z}
+// CHECK:  encoding: [0x62,0x21,0x6d,0xa4,0xe0,0xd2]
+          vpavgb %ymm18, %ymm18, %ymm26 {%k4} {z}
+
+// CHECK: vpavgb (%rcx), %ymm18, %ymm26
+// CHECK:  encoding: [0x62,0x61,0x6d,0x20,0xe0,0x11]
+          vpavgb (%rcx), %ymm18, %ymm26
+
+// CHECK: vpavgb 4660(%rax,%r14,8), %ymm18, %ymm26
+// CHECK:  encoding: [0x62,0x21,0x6d,0x20,0xe0,0x94,0xf0,0x34,0x12,0x00,0x00]
+          vpavgb 4660(%rax,%r14,8), %ymm18, %ymm26
+
+// CHECK: vpavgb 4064(%rdx), %ymm18, %ymm26
+// CHECK:  encoding: [0x62,0x61,0x6d,0x20,0xe0,0x52,0x7f]
+          vpavgb 4064(%rdx), %ymm18, %ymm26
+
+// CHECK: vpavgb 4096(%rdx), %ymm18, %ymm26
+// CHECK:  encoding: [0x62,0x61,0x6d,0x20,0xe0,0x92,0x00,0x10,0x00,0x00]
+          vpavgb 4096(%rdx), %ymm18, %ymm26
+
+// CHECK: vpavgb -4096(%rdx), %ymm18, %ymm26
+// CHECK:  encoding: [0x62,0x61,0x6d,0x20,0xe0,0x52,0x80]
+          vpavgb -4096(%rdx), %ymm18, %ymm26
+
+// CHECK: vpavgb -4128(%rdx), %ymm18, %ymm26
+// CHECK:  encoding: [0x62,0x61,0x6d,0x20,0xe0,0x92,0xe0,0xef,0xff,0xff]
+          vpavgb -4128(%rdx), %ymm18, %ymm26
+
+// CHECK: vpavgw %xmm23, %xmm28, %xmm29
+// CHECK:  encoding: [0x62,0x21,0x1d,0x00,0xe3,0xef]
+          vpavgw %xmm23, %xmm28, %xmm29
+
+// CHECK: vpavgw %xmm23, %xmm28, %xmm29 {%k7}
+// CHECK:  encoding: [0x62,0x21,0x1d,0x07,0xe3,0xef]
+          vpavgw %xmm23, %xmm28, %xmm29 {%k7}
+
+// CHECK: vpavgw %xmm23, %xmm28, %xmm29 {%k7} {z}
+// CHECK:  encoding: [0x62,0x21,0x1d,0x87,0xe3,0xef]
+          vpavgw %xmm23, %xmm28, %xmm29 {%k7} {z}
+
+// CHECK: vpavgw (%rcx), %xmm28, %xmm29
+// CHECK:  encoding: [0x62,0x61,0x1d,0x00,0xe3,0x29]
+          vpavgw (%rcx), %xmm28, %xmm29
+
+// CHECK: vpavgw 4660(%rax,%r14,8), %xmm28, %xmm29
+// CHECK:  encoding: [0x62,0x21,0x1d,0x00,0xe3,0xac,0xf0,0x34,0x12,0x00,0x00]
+          vpavgw 4660(%rax,%r14,8), %xmm28, %xmm29
+
+// CHECK: vpavgw 2032(%rdx), %xmm28, %xmm29
+// CHECK:  encoding: [0x62,0x61,0x1d,0x00,0xe3,0x6a,0x7f]
+          vpavgw 2032(%rdx), %xmm28, %xmm29
+
+// CHECK: vpavgw 2048(%rdx), %xmm28, %xmm29
+// CHECK:  encoding: [0x62,0x61,0x1d,0x00,0xe3,0xaa,0x00,0x08,0x00,0x00]
+          vpavgw 2048(%rdx), %xmm28, %xmm29
+
+// CHECK: vpavgw -2048(%rdx), %xmm28, %xmm29
+// CHECK:  encoding: [0x62,0x61,0x1d,0x00,0xe3,0x6a,0x80]
+          vpavgw -2048(%rdx), %xmm28, %xmm29
+
+// CHECK: vpavgw -2064(%rdx), %xmm28, %xmm29
+// CHECK:  encoding: [0x62,0x61,0x1d,0x00,0xe3,0xaa,0xf0,0xf7,0xff,0xff]
+          vpavgw -2064(%rdx), %xmm28, %xmm29
+
+// CHECK: vpavgw %ymm17, %ymm18, %ymm27
+// CHECK:  encoding: [0x62,0x21,0x6d,0x20,0xe3,0xd9]
+          vpavgw %ymm17, %ymm18, %ymm27
+
+// CHECK: vpavgw %ymm17, %ymm18, %ymm27 {%k5}
+// CHECK:  encoding: [0x62,0x21,0x6d,0x25,0xe3,0xd9]
+          vpavgw %ymm17, %ymm18, %ymm27 {%k5}
+
+// CHECK: vpavgw %ymm17, %ymm18, %ymm27 {%k5} {z}
+// CHECK:  encoding: [0x62,0x21,0x6d,0xa5,0xe3,0xd9]
+          vpavgw %ymm17, %ymm18, %ymm27 {%k5} {z}
+
+// CHECK: vpavgw (%rcx), %ymm18, %ymm27
+// CHECK:  encoding: [0x62,0x61,0x6d,0x20,0xe3,0x19]
+          vpavgw (%rcx), %ymm18, %ymm27
+
+// CHECK: vpavgw 4660(%rax,%r14,8), %ymm18, %ymm27
+// CHECK:  encoding: [0x62,0x21,0x6d,0x20,0xe3,0x9c,0xf0,0x34,0x12,0x00,0x00]
+          vpavgw 4660(%rax,%r14,8), %ymm18, %ymm27
+
+// CHECK: vpavgw 4064(%rdx), %ymm18, %ymm27
+// CHECK:  encoding: [0x62,0x61,0x6d,0x20,0xe3,0x5a,0x7f]
+          vpavgw 4064(%rdx), %ymm18, %ymm27
+
+// CHECK: vpavgw 4096(%rdx), %ymm18, %ymm27
+// CHECK:  encoding: [0x62,0x61,0x6d,0x20,0xe3,0x9a,0x00,0x10,0x00,0x00]
+          vpavgw 4096(%rdx), %ymm18, %ymm27
+
+// CHECK: vpavgw -4096(%rdx), %ymm18, %ymm27
+// CHECK:  encoding: [0x62,0x61,0x6d,0x20,0xe3,0x5a,0x80]
+          vpavgw -4096(%rdx), %ymm18, %ymm27
+
+// CHECK: vpavgw -4128(%rdx), %ymm18, %ymm27
+// CHECK:  encoding: [0x62,0x61,0x6d,0x20,0xe3,0x9a,0xe0,0xef,0xff,0xff]
+          vpavgw -4128(%rdx), %ymm18, %ymm27
+
+// CHECK: vpavgb %xmm20, %xmm22, %xmm26
+// CHECK:  encoding: [0x62,0x21,0x4d,0x00,0xe0,0xd4]
+          vpavgb %xmm20, %xmm22, %xmm26
+
+// CHECK: vpavgb %xmm20, %xmm22, %xmm26 {%k6}
+// CHECK:  encoding: [0x62,0x21,0x4d,0x06,0xe0,0xd4]
+          vpavgb %xmm20, %xmm22, %xmm26 {%k6}
+
+// CHECK: vpavgb %xmm20, %xmm22, %xmm26 {%k6} {z}
+// CHECK:  encoding: [0x62,0x21,0x4d,0x86,0xe0,0xd4]
+          vpavgb %xmm20, %xmm22, %xmm26 {%k6} {z}
+
+// CHECK: vpavgb (%rcx), %xmm22, %xmm26
+// CHECK:  encoding: [0x62,0x61,0x4d,0x00,0xe0,0x11]
+          vpavgb (%rcx), %xmm22, %xmm26
+
+// CHECK: vpavgb 291(%rax,%r14,8), %xmm22, %xmm26
+// CHECK:  encoding: [0x62,0x21,0x4d,0x00,0xe0,0x94,0xf0,0x23,0x01,0x00,0x00]
+          vpavgb 291(%rax,%r14,8), %xmm22, %xmm26
+
+// CHECK: vpavgb 2032(%rdx), %xmm22, %xmm26
+// CHECK:  encoding: [0x62,0x61,0x4d,0x00,0xe0,0x52,0x7f]
+          vpavgb 2032(%rdx), %xmm22, %xmm26
+
+// CHECK: vpavgb 2048(%rdx), %xmm22, %xmm26
+// CHECK:  encoding: [0x62,0x61,0x4d,0x00,0xe0,0x92,0x00,0x08,0x00,0x00]
+          vpavgb 2048(%rdx), %xmm22, %xmm26
+
+// CHECK: vpavgb -2048(%rdx), %xmm22, %xmm26
+// CHECK:  encoding: [0x62,0x61,0x4d,0x00,0xe0,0x52,0x80]
+          vpavgb -2048(%rdx), %xmm22, %xmm26
+
+// CHECK: vpavgb -2064(%rdx), %xmm22, %xmm26
+// CHECK:  encoding: [0x62,0x61,0x4d,0x00,0xe0,0x92,0xf0,0xf7,0xff,0xff]
+          vpavgb -2064(%rdx), %xmm22, %xmm26
+
+// CHECK: vpavgb %ymm18, %ymm22, %ymm29
+// CHECK:  encoding: [0x62,0x21,0x4d,0x20,0xe0,0xea]
+          vpavgb %ymm18, %ymm22, %ymm29
+
+// CHECK: vpavgb %ymm18, %ymm22, %ymm29 {%k1}
+// CHECK:  encoding: [0x62,0x21,0x4d,0x21,0xe0,0xea]
+          vpavgb %ymm18, %ymm22, %ymm29 {%k1}
+
+// CHECK: vpavgb %ymm18, %ymm22, %ymm29 {%k1} {z}
+// CHECK:  encoding: [0x62,0x21,0x4d,0xa1,0xe0,0xea]
+          vpavgb %ymm18, %ymm22, %ymm29 {%k1} {z}
+
+// CHECK: vpavgb (%rcx), %ymm22, %ymm29
+// CHECK:  encoding: [0x62,0x61,0x4d,0x20,0xe0,0x29]
+          vpavgb (%rcx), %ymm22, %ymm29
+
+// CHECK: vpavgb 291(%rax,%r14,8), %ymm22, %ymm29
+// CHECK:  encoding: [0x62,0x21,0x4d,0x20,0xe0,0xac,0xf0,0x23,0x01,0x00,0x00]
+          vpavgb 291(%rax,%r14,8), %ymm22, %ymm29
+
+// CHECK: vpavgb 4064(%rdx), %ymm22, %ymm29
+// CHECK:  encoding: [0x62,0x61,0x4d,0x20,0xe0,0x6a,0x7f]
+          vpavgb 4064(%rdx), %ymm22, %ymm29
+
+// CHECK: vpavgb 4096(%rdx), %ymm22, %ymm29
+// CHECK:  encoding: [0x62,0x61,0x4d,0x20,0xe0,0xaa,0x00,0x10,0x00,0x00]
+          vpavgb 4096(%rdx), %ymm22, %ymm29
+
+// CHECK: vpavgb -4096(%rdx), %ymm22, %ymm29
+// CHECK:  encoding: [0x62,0x61,0x4d,0x20,0xe0,0x6a,0x80]
+          vpavgb -4096(%rdx), %ymm22, %ymm29
+
+// CHECK: vpavgb -4128(%rdx), %ymm22, %ymm29
+// CHECK:  encoding: [0x62,0x61,0x4d,0x20,0xe0,0xaa,0xe0,0xef,0xff,0xff]
+          vpavgb -4128(%rdx), %ymm22, %ymm29
+
+// CHECK: vpavgw %xmm23, %xmm24, %xmm28
+// CHECK:  encoding: [0x62,0x21,0x3d,0x00,0xe3,0xe7]
+          vpavgw %xmm23, %xmm24, %xmm28
+
+// CHECK: vpavgw %xmm23, %xmm24, %xmm28 {%k7}
+// CHECK:  encoding: [0x62,0x21,0x3d,0x07,0xe3,0xe7]
+          vpavgw %xmm23, %xmm24, %xmm28 {%k7}
+
+// CHECK: vpavgw %xmm23, %xmm24, %xmm28 {%k7} {z}
+// CHECK:  encoding: [0x62,0x21,0x3d,0x87,0xe3,0xe7]
+          vpavgw %xmm23, %xmm24, %xmm28 {%k7} {z}
+
+// CHECK: vpavgw (%rcx), %xmm24, %xmm28
+// CHECK:  encoding: [0x62,0x61,0x3d,0x00,0xe3,0x21]
+          vpavgw (%rcx), %xmm24, %xmm28
+
+// CHECK: vpavgw 291(%rax,%r14,8), %xmm24, %xmm28
+// CHECK:  encoding: [0x62,0x21,0x3d,0x00,0xe3,0xa4,0xf0,0x23,0x01,0x00,0x00]
+          vpavgw 291(%rax,%r14,8), %xmm24, %xmm28
+
+// CHECK: vpavgw 2032(%rdx), %xmm24, %xmm28
+// CHECK:  encoding: [0x62,0x61,0x3d,0x00,0xe3,0x62,0x7f]
+          vpavgw 2032(%rdx), %xmm24, %xmm28
+
+// CHECK: vpavgw 2048(%rdx), %xmm24, %xmm28
+// CHECK:  encoding: [0x62,0x61,0x3d,0x00,0xe3,0xa2,0x00,0x08,0x00,0x00]
+          vpavgw 2048(%rdx), %xmm24, %xmm28
+
+// CHECK: vpavgw -2048(%rdx), %xmm24, %xmm28
+// CHECK:  encoding: [0x62,0x61,0x3d,0x00,0xe3,0x62,0x80]
+          vpavgw -2048(%rdx), %xmm24, %xmm28
+
+// CHECK: vpavgw -2064(%rdx), %xmm24, %xmm28
+// CHECK:  encoding: [0x62,0x61,0x3d,0x00,0xe3,0xa2,0xf0,0xf7,0xff,0xff]
+          vpavgw -2064(%rdx), %xmm24, %xmm28
+
+// CHECK: vpavgw %ymm22, %ymm26, %ymm22
+// CHECK:  encoding: [0x62,0xa1,0x2d,0x20,0xe3,0xf6]
+          vpavgw %ymm22, %ymm26, %ymm22
+
+// CHECK: vpavgw %ymm22, %ymm26, %ymm22 {%k7}
+// CHECK:  encoding: [0x62,0xa1,0x2d,0x27,0xe3,0xf6]
+          vpavgw %ymm22, %ymm26, %ymm22 {%k7}
+
+// CHECK: vpavgw %ymm22, %ymm26, %ymm22 {%k7} {z}
+// CHECK:  encoding: [0x62,0xa1,0x2d,0xa7,0xe3,0xf6]
+          vpavgw %ymm22, %ymm26, %ymm22 {%k7} {z}
+
+// CHECK: vpavgw (%rcx), %ymm26, %ymm22
+// CHECK:  encoding: [0x62,0xe1,0x2d,0x20,0xe3,0x31]
+          vpavgw (%rcx), %ymm26, %ymm22
+
+// CHECK: vpavgw 291(%rax,%r14,8), %ymm26, %ymm22
+// CHECK:  encoding: [0x62,0xa1,0x2d,0x20,0xe3,0xb4,0xf0,0x23,0x01,0x00,0x00]
+          vpavgw 291(%rax,%r14,8), %ymm26, %ymm22
+
+// CHECK: vpavgw 4064(%rdx), %ymm26, %ymm22
+// CHECK:  encoding: [0x62,0xe1,0x2d,0x20,0xe3,0x72,0x7f]
+          vpavgw 4064(%rdx), %ymm26, %ymm22
+
+// CHECK: vpavgw 4096(%rdx), %ymm26, %ymm22
+// CHECK:  encoding: [0x62,0xe1,0x2d,0x20,0xe3,0xb2,0x00,0x10,0x00,0x00]
+          vpavgw 4096(%rdx), %ymm26, %ymm22
+
+// CHECK: vpavgw -4096(%rdx), %ymm26, %ymm22
+// CHECK:  encoding: [0x62,0xe1,0x2d,0x20,0xe3,0x72,0x80]
+          vpavgw -4096(%rdx), %ymm26, %ymm22
+
+// CHECK: vpavgw -4128(%rdx), %ymm26, %ymm22
+// CHECK:  encoding: [0x62,0xe1,0x2d,0x20,0xe3,0xb2,0xe0,0xef,0xff,0xff]
+          vpavgw -4128(%rdx), %ymm26, %ymm22
+
+// CHECK: vpavgb %xmm26, %xmm19, %xmm20
+// CHECK:  encoding: [0x62,0x81,0x65,0x00,0xe0,0xe2]
+          vpavgb %xmm26, %xmm19, %xmm20
+
+// CHECK: vpavgb %xmm26, %xmm19, %xmm20 {%k7}
+// CHECK:  encoding: [0x62,0x81,0x65,0x07,0xe0,0xe2]
+          vpavgb %xmm26, %xmm19, %xmm20 {%k7}
+
+// CHECK: vpavgb %xmm26, %xmm19, %xmm20 {%k7} {z}
+// CHECK:  encoding: [0x62,0x81,0x65,0x87,0xe0,0xe2]
+          vpavgb %xmm26, %xmm19, %xmm20 {%k7} {z}
+
+// CHECK: vpavgb (%rcx), %xmm19, %xmm20
+// CHECK:  encoding: [0x62,0xe1,0x65,0x00,0xe0,0x21]
+          vpavgb (%rcx), %xmm19, %xmm20
+
+// CHECK: vpavgb 291(%rax,%r14,8), %xmm19, %xmm20
+// CHECK:  encoding: [0x62,0xa1,0x65,0x00,0xe0,0xa4,0xf0,0x23,0x01,0x00,0x00]
+          vpavgb 291(%rax,%r14,8), %xmm19, %xmm20
+
+// CHECK: vpavgb 2032(%rdx), %xmm19, %xmm20
+// CHECK:  encoding: [0x62,0xe1,0x65,0x00,0xe0,0x62,0x7f]
+          vpavgb 2032(%rdx), %xmm19, %xmm20
+
+// CHECK: vpavgb 2048(%rdx), %xmm19, %xmm20
+// CHECK:  encoding: [0x62,0xe1,0x65,0x00,0xe0,0xa2,0x00,0x08,0x00,0x00]
+          vpavgb 2048(%rdx), %xmm19, %xmm20
+
+// CHECK: vpavgb -2048(%rdx), %xmm19, %xmm20
+// CHECK:  encoding: [0x62,0xe1,0x65,0x00,0xe0,0x62,0x80]
+          vpavgb -2048(%rdx), %xmm19, %xmm20
+
+// CHECK: vpavgb -2064(%rdx), %xmm19, %xmm20
+// CHECK:  encoding: [0x62,0xe1,0x65,0x00,0xe0,0xa2,0xf0,0xf7,0xff,0xff]
+          vpavgb -2064(%rdx), %xmm19, %xmm20
+
+// CHECK: vpavgb %ymm17, %ymm22, %ymm26
+// CHECK:  encoding: [0x62,0x21,0x4d,0x20,0xe0,0xd1]
+          vpavgb %ymm17, %ymm22, %ymm26
+
+// CHECK: vpavgb %ymm17, %ymm22, %ymm26 {%k5}
+// CHECK:  encoding: [0x62,0x21,0x4d,0x25,0xe0,0xd1]
+          vpavgb %ymm17, %ymm22, %ymm26 {%k5}
+
+// CHECK: vpavgb %ymm17, %ymm22, %ymm26 {%k5} {z}
+// CHECK:  encoding: [0x62,0x21,0x4d,0xa5,0xe0,0xd1]
+          vpavgb %ymm17, %ymm22, %ymm26 {%k5} {z}
+
+// CHECK: vpavgb (%rcx), %ymm22, %ymm26
+// CHECK:  encoding: [0x62,0x61,0x4d,0x20,0xe0,0x11]
+          vpavgb (%rcx), %ymm22, %ymm26
+
+// CHECK: vpavgb 291(%rax,%r14,8), %ymm22, %ymm26
+// CHECK:  encoding: [0x62,0x21,0x4d,0x20,0xe0,0x94,0xf0,0x23,0x01,0x00,0x00]
+          vpavgb 291(%rax,%r14,8), %ymm22, %ymm26
+
+// CHECK: vpavgb 4064(%rdx), %ymm22, %ymm26
+// CHECK:  encoding: [0x62,0x61,0x4d,0x20,0xe0,0x52,0x7f]
+          vpavgb 4064(%rdx), %ymm22, %ymm26
+
+// CHECK: vpavgb 4096(%rdx), %ymm22, %ymm26
+// CHECK:  encoding: [0x62,0x61,0x4d,0x20,0xe0,0x92,0x00,0x10,0x00,0x00]
+          vpavgb 4096(%rdx), %ymm22, %ymm26
+
+// CHECK: vpavgb -4096(%rdx), %ymm22, %ymm26
+// CHECK:  encoding: [0x62,0x61,0x4d,0x20,0xe0,0x52,0x80]
+          vpavgb -4096(%rdx), %ymm22, %ymm26
+
+// CHECK: vpavgb -4128(%rdx), %ymm22, %ymm26
+// CHECK:  encoding: [0x62,0x61,0x4d,0x20,0xe0,0x92,0xe0,0xef,0xff,0xff]
+          vpavgb -4128(%rdx), %ymm22, %ymm26
+
+// CHECK: vpavgw %xmm20, %xmm18, %xmm22
+// CHECK:  encoding: [0x62,0xa1,0x6d,0x00,0xe3,0xf4]
+          vpavgw %xmm20, %xmm18, %xmm22
+
+// CHECK: vpavgw %xmm20, %xmm18, %xmm22 {%k7}
+// CHECK:  encoding: [0x62,0xa1,0x6d,0x07,0xe3,0xf4]
+          vpavgw %xmm20, %xmm18, %xmm22 {%k7}
+
+// CHECK: vpavgw %xmm20, %xmm18, %xmm22 {%k7} {z}
+// CHECK:  encoding: [0x62,0xa1,0x6d,0x87,0xe3,0xf4]
+          vpavgw %xmm20, %xmm18, %xmm22 {%k7} {z}
+
+// CHECK: vpavgw (%rcx), %xmm18, %xmm22
+// CHECK:  encoding: [0x62,0xe1,0x6d,0x00,0xe3,0x31]
+          vpavgw (%rcx), %xmm18, %xmm22
+
+// CHECK: vpavgw 291(%rax,%r14,8), %xmm18, %xmm22
+// CHECK:  encoding: [0x62,0xa1,0x6d,0x00,0xe3,0xb4,0xf0,0x23,0x01,0x00,0x00]
+          vpavgw 291(%rax,%r14,8), %xmm18, %xmm22
+
+// CHECK: vpavgw 2032(%rdx), %xmm18, %xmm22
+// CHECK:  encoding: [0x62,0xe1,0x6d,0x00,0xe3,0x72,0x7f]
+          vpavgw 2032(%rdx), %xmm18, %xmm22
+
+// CHECK: vpavgw 2048(%rdx), %xmm18, %xmm22
+// CHECK:  encoding: [0x62,0xe1,0x6d,0x00,0xe3,0xb2,0x00,0x08,0x00,0x00]
+          vpavgw 2048(%rdx), %xmm18, %xmm22
+
+// CHECK: vpavgw -2048(%rdx), %xmm18, %xmm22
+// CHECK:  encoding: [0x62,0xe1,0x6d,0x00,0xe3,0x72,0x80]
+          vpavgw -2048(%rdx), %xmm18, %xmm22
+
+// CHECK: vpavgw -2064(%rdx), %xmm18, %xmm22
+// CHECK:  encoding: [0x62,0xe1,0x6d,0x00,0xe3,0xb2,0xf0,0xf7,0xff,0xff]
+          vpavgw -2064(%rdx), %xmm18, %xmm22
+
+// CHECK: vpavgw %ymm21, %ymm23, %ymm21
+// CHECK:  encoding: [0x62,0xa1,0x45,0x20,0xe3,0xed]
+          vpavgw %ymm21, %ymm23, %ymm21
+
+// CHECK: vpavgw %ymm21, %ymm23, %ymm21 {%k2}
+// CHECK:  encoding: [0x62,0xa1,0x45,0x22,0xe3,0xed]
+          vpavgw %ymm21, %ymm23, %ymm21 {%k2}
+
+// CHECK: vpavgw %ymm21, %ymm23, %ymm21 {%k2} {z}
+// CHECK:  encoding: [0x62,0xa1,0x45,0xa2,0xe3,0xed]
+          vpavgw %ymm21, %ymm23, %ymm21 {%k2} {z}
+
+// CHECK: vpavgw (%rcx), %ymm23, %ymm21
+// CHECK:  encoding: [0x62,0xe1,0x45,0x20,0xe3,0x29]
+          vpavgw (%rcx), %ymm23, %ymm21
+
+// CHECK: vpavgw 291(%rax,%r14,8), %ymm23, %ymm21
+// CHECK:  encoding: [0x62,0xa1,0x45,0x20,0xe3,0xac,0xf0,0x23,0x01,0x00,0x00]
+          vpavgw 291(%rax,%r14,8), %ymm23, %ymm21
+
+// CHECK: vpavgw 4064(%rdx), %ymm23, %ymm21
+// CHECK:  encoding: [0x62,0xe1,0x45,0x20,0xe3,0x6a,0x7f]
+          vpavgw 4064(%rdx), %ymm23, %ymm21
+
+// CHECK: vpavgw 4096(%rdx), %ymm23, %ymm21
+// CHECK:  encoding: [0x62,0xe1,0x45,0x20,0xe3,0xaa,0x00,0x10,0x00,0x00]
+          vpavgw 4096(%rdx), %ymm23, %ymm21
+
+// CHECK: vpavgw -4096(%rdx), %ymm23, %ymm21
+// CHECK:  encoding: [0x62,0xe1,0x45,0x20,0xe3,0x6a,0x80]
+          vpavgw -4096(%rdx), %ymm23, %ymm21
+
+// CHECK: vpavgw -4128(%rdx), %ymm23, %ymm21
+// CHECK:  encoding: [0x62,0xe1,0x45,0x20,0xe3,0xaa,0xe0,0xef,0xff,0xff]
+          vpavgw -4128(%rdx), %ymm23, %ymm21
+
+// CHECK: vpshufb %xmm27, %xmm24, %xmm23
+// CHECK:  encoding: [0x62,0x82,0x3d,0x00,0x00,0xfb]
+          vpshufb %xmm27, %xmm24, %xmm23
+
+// CHECK: vpshufb %xmm27, %xmm24, %xmm23 {%k4}
+// CHECK:  encoding: [0x62,0x82,0x3d,0x04,0x00,0xfb]
+          vpshufb %xmm27, %xmm24, %xmm23 {%k4}
+
+// CHECK: vpshufb %xmm27, %xmm24, %xmm23 {%k4} {z}
+// CHECK:  encoding: [0x62,0x82,0x3d,0x84,0x00,0xfb]
+          vpshufb %xmm27, %xmm24, %xmm23 {%k4} {z}
+
+// CHECK: vpshufb (%rcx), %xmm24, %xmm23
+// CHECK:  encoding: [0x62,0xe2,0x3d,0x00,0x00,0x39]
+          vpshufb (%rcx), %xmm24, %xmm23
+
+// CHECK: vpshufb 291(%rax,%r14,8), %xmm24, %xmm23
+// CHECK:  encoding: [0x62,0xa2,0x3d,0x00,0x00,0xbc,0xf0,0x23,0x01,0x00,0x00]
+          vpshufb 291(%rax,%r14,8), %xmm24, %xmm23
+
+// CHECK: vpshufb 2032(%rdx), %xmm24, %xmm23
+// CHECK:  encoding: [0x62,0xe2,0x3d,0x00,0x00,0x7a,0x7f]
+          vpshufb 2032(%rdx), %xmm24, %xmm23
+
+// CHECK: vpshufb 2048(%rdx), %xmm24, %xmm23
+// CHECK:  encoding: [0x62,0xe2,0x3d,0x00,0x00,0xba,0x00,0x08,0x00,0x00]
+          vpshufb 2048(%rdx), %xmm24, %xmm23
+
+// CHECK: vpshufb -2048(%rdx), %xmm24, %xmm23
+// CHECK:  encoding: [0x62,0xe2,0x3d,0x00,0x00,0x7a,0x80]
+          vpshufb -2048(%rdx), %xmm24, %xmm23
+
+// CHECK: vpshufb -2064(%rdx), %xmm24, %xmm23
+// CHECK:  encoding: [0x62,0xe2,0x3d,0x00,0x00,0xba,0xf0,0xf7,0xff,0xff]
+          vpshufb -2064(%rdx), %xmm24, %xmm23
+
+// CHECK: vpshufb %ymm17, %ymm18, %ymm19
+// CHECK:  encoding: [0x62,0xa2,0x6d,0x20,0x00,0xd9]
+          vpshufb %ymm17, %ymm18, %ymm19
+
+// CHECK: vpshufb %ymm17, %ymm18, %ymm19 {%k4}
+// CHECK:  encoding: [0x62,0xa2,0x6d,0x24,0x00,0xd9]
+          vpshufb %ymm17, %ymm18, %ymm19 {%k4}
+
+// CHECK: vpshufb %ymm17, %ymm18, %ymm19 {%k4} {z}
+// CHECK:  encoding: [0x62,0xa2,0x6d,0xa4,0x00,0xd9]
+          vpshufb %ymm17, %ymm18, %ymm19 {%k4} {z}
+
+// CHECK: vpshufb (%rcx), %ymm18, %ymm19
+// CHECK:  encoding: [0x62,0xe2,0x6d,0x20,0x00,0x19]
+          vpshufb (%rcx), %ymm18, %ymm19
+
+// CHECK: vpshufb 291(%rax,%r14,8), %ymm18, %ymm19
+// CHECK:  encoding: [0x62,0xa2,0x6d,0x20,0x00,0x9c,0xf0,0x23,0x01,0x00,0x00]
+          vpshufb 291(%rax,%r14,8), %ymm18, %ymm19
+
+// CHECK: vpshufb 4064(%rdx), %ymm18, %ymm19
+// CHECK:  encoding: [0x62,0xe2,0x6d,0x20,0x00,0x5a,0x7f]
+          vpshufb 4064(%rdx), %ymm18, %ymm19
+
+// CHECK: vpshufb 4096(%rdx), %ymm18, %ymm19
+// CHECK:  encoding: [0x62,0xe2,0x6d,0x20,0x00,0x9a,0x00,0x10,0x00,0x00]
+          vpshufb 4096(%rdx), %ymm18, %ymm19
+
+// CHECK: vpshufb -4096(%rdx), %ymm18, %ymm19
+// CHECK:  encoding: [0x62,0xe2,0x6d,0x20,0x00,0x5a,0x80]
+          vpshufb -4096(%rdx), %ymm18, %ymm19
+
+// CHECK: vpshufb -4128(%rdx), %ymm18, %ymm19
+// CHECK:  encoding: [0x62,0xe2,0x6d,0x20,0x00,0x9a,0xe0,0xef,0xff,0xff]
+          vpshufb -4128(%rdx), %ymm18, %ymm19
+
+// CHECK: vpmovwb %xmm28, %xmm27
+// CHECK:  encoding: [0x62,0x02,0x7e,0x08,0x30,0xe3]
+          vpmovwb %xmm28, %xmm27
+
+// CHECK: vpmovwb %xmm28, %xmm27 {%k2}
+// CHECK:  encoding: [0x62,0x02,0x7e,0x0a,0x30,0xe3]
+          vpmovwb %xmm28, %xmm27 {%k2}
+
+// CHECK: vpmovwb %xmm28, %xmm27 {%k2} {z}
+// CHECK:  encoding: [0x62,0x02,0x7e,0x8a,0x30,0xe3]
+          vpmovwb %xmm28, %xmm27 {%k2} {z}
+
+// CHECK: vpmovwb %ymm26, %xmm26
+// CHECK:  encoding: [0x62,0x02,0x7e,0x28,0x30,0xd2]
+          vpmovwb %ymm26, %xmm26
+
+// CHECK: vpmovwb %ymm26, %xmm26 {%k4}
+// CHECK:  encoding: [0x62,0x02,0x7e,0x2c,0x30,0xd2]
+          vpmovwb %ymm26, %xmm26 {%k4}
+
+// CHECK: vpmovwb %ymm26, %xmm26 {%k4} {z}
+// CHECK:  encoding: [0x62,0x02,0x7e,0xac,0x30,0xd2]
+          vpmovwb %ymm26, %xmm26 {%k4} {z}
+
+// CHECK: vpmovwb %xmm23, (%rcx)
+// CHECK:  encoding: [0x62,0xe2,0x7e,0x08,0x30,0x39]
+          vpmovwb %xmm23,(%rcx)
+
+// CHECK: vpmovwb %xmm23, (%rcx) {%k6}
+// CHECK:  encoding: [0x62,0xe2,0x7e,0x0e,0x30,0x39]
+          vpmovwb %xmm23,(%rcx) {%k6}
+
+// CHECK: vpmovwb %xmm23, 4660(%rax,%r14,8)
+// CHECK:  encoding: [0x62,0xa2,0x7e,0x08,0x30,0xbc,0xf0,0x34,0x12,0x00,0x00]
+          vpmovwb %xmm23,4660(%rax,%r14,8)
+
+// CHECK: vpmovwb %xmm23, 1016(%rdx)
+// CHECK:  encoding: [0x62,0xe2,0x7e,0x08,0x30,0x7a,0x7f]
+          vpmovwb %xmm23, 1016(%rdx)
+
+// CHECK: vpmovwb %xmm23, 1024(%rdx)
+// CHECK:  encoding: [0x62,0xe2,0x7e,0x08,0x30,0xba,0x00,0x04,0x00,0x00]
+          vpmovwb %xmm23, 1024(%rdx)
+
+// CHECK: vpmovwb %xmm23, -1024(%rdx)
+// CHECK:  encoding: [0x62,0xe2,0x7e,0x08,0x30,0x7a,0x80]
+          vpmovwb %xmm23,-1024(%rdx)
+
+// CHECK: vpmovwb %xmm23, -1032(%rdx)
+// CHECK:  encoding: [0x62,0xe2,0x7e,0x08,0x30,0xba,0xf8,0xfb,0xff,0xff]
+          vpmovwb %xmm23,-1032(%rdx)
+
+// CHECK: vpmovwb %ymm21, (%rcx)
+// CHECK:  encoding: [0x62,0xe2,0x7e,0x28,0x30,0x29]
+          vpmovwb %ymm21,(%rcx)
+
+// CHECK: vpmovwb %ymm21, (%rcx) {%k5}
+// CHECK:  encoding: [0x62,0xe2,0x7e,0x2d,0x30,0x29]
+          vpmovwb %ymm21,(%rcx) {%k5}
+
+// CHECK: vpmovwb %ymm21, 4660(%rax,%r14,8)
+// CHECK:  encoding: [0x62,0xa2,0x7e,0x28,0x30,0xac,0xf0,0x34,0x12,0x00,0x00]
+          vpmovwb %ymm21, 4660(%rax,%r14,8)
+
+// CHECK: vpmovwb %ymm21, 2032(%rdx)
+// CHECK:  encoding: [0x62,0xe2,0x7e,0x28,0x30,0x6a,0x7f]
+          vpmovwb %ymm21, 2032(%rdx)
+
+// CHECK: vpmovwb %ymm21, 2048(%rdx)
+// CHECK:  encoding: [0x62,0xe2,0x7e,0x28,0x30,0xaa,0x00,0x08,0x00,0x00]
+          vpmovwb %ymm21, 2048(%rdx)
+
+// CHECK: vpmovwb %ymm21, -2048(%rdx)
+// CHECK:  encoding: [0x62,0xe2,0x7e,0x28,0x30,0x6a,0x80]
+          vpmovwb %ymm21,-2048(%rdx)
+
+// CHECK: vpmovwb %ymm21, -2064(%rdx)
+// CHECK:  encoding: [0x62,0xe2,0x7e,0x28,0x30,0xaa,0xf0,0xf7,0xff,0xff]
+          vpmovwb %ymm21, -2064(%rdx)
+
+// CHECK: vpmovswb %xmm19, %xmm17
+// CHECK:  encoding: [0x62,0xa2,0x7e,0x08,0x20,0xd9]
+          vpmovswb %xmm19, %xmm17
+
+// CHECK: vpmovswb %xmm19, %xmm17 {%k1}
+// CHECK:  encoding: [0x62,0xa2,0x7e,0x09,0x20,0xd9]
+          vpmovswb %xmm19, %xmm17 {%k1}
+
+// CHECK: vpmovswb %xmm19, %xmm17 {%k1} {z}
+// CHECK:  encoding: [0x62,0xa2,0x7e,0x89,0x20,0xd9]
+          vpmovswb %xmm19, %xmm17 {%k1} {z}
+
+// CHECK: vpmovswb %ymm19, %xmm21
+// CHECK:  encoding: [0x62,0xa2,0x7e,0x28,0x20,0xdd]
+          vpmovswb %ymm19, %xmm21
+
+// CHECK: vpmovswb %ymm19, %xmm21 {%k4}
+// CHECK:  encoding: [0x62,0xa2,0x7e,0x2c,0x20,0xdd]
+          vpmovswb %ymm19, %xmm21 {%k4}
+
+// CHECK: vpmovswb %ymm19, %xmm21 {%k4} {z}
+// CHECK:  encoding: [0x62,0xa2,0x7e,0xac,0x20,0xdd]
+          vpmovswb %ymm19, %xmm21 {%k4} {z}
+
+// CHECK: vpmovswb %xmm18, (%rcx)
+// CHECK:  encoding: [0x62,0xe2,0x7e,0x08,0x20,0x11]
+          vpmovswb %xmm18,(%rcx)
+
+// CHECK: vpmovswb %xmm18, (%rcx) {%k2}
+// CHECK:  encoding: [0x62,0xe2,0x7e,0x0a,0x20,0x11]
+          vpmovswb %xmm18,(%rcx) {%k2}
+
+// CHECK: vpmovswb %xmm18, 4660(%rax,%r14,8)
+// CHECK:  encoding: [0x62,0xa2,0x7e,0x08,0x20,0x94,0xf0,0x34,0x12,0x00,0x00]
+          vpmovswb %xmm18, 4660(%rax,%r14,8)
+
+// CHECK: vpmovswb %xmm18, 1016(%rdx)
+// CHECK:  encoding: [0x62,0xe2,0x7e,0x08,0x20,0x52,0x7f]
+          vpmovswb %xmm18, 1016(%rdx)
+
+// CHECK: vpmovswb %xmm18, 1024(%rdx)
+// CHECK:  encoding: [0x62,0xe2,0x7e,0x08,0x20,0x92,0x00,0x04,0x00,0x00]
+          vpmovswb %xmm18, 1024(%rdx)
+
+// CHECK: vpmovswb %xmm18, -1024(%rdx)
+// CHECK:  encoding: [0x62,0xe2,0x7e,0x08,0x20,0x52,0x80]
+          vpmovswb %xmm18, -1024(%rdx)
+
+// CHECK: vpmovswb %xmm18, -1032(%rdx)
+// CHECK:  encoding: [0x62,0xe2,0x7e,0x08,0x20,0x92,0xf8,0xfb,0xff,0xff]
+          vpmovswb %xmm18, -1032(%rdx)
+
+// CHECK: vpmovswb %ymm23, (%rcx)
+// CHECK:  encoding: [0x62,0xe2,0x7e,0x28,0x20,0x39]
+          vpmovswb %ymm23,(%rcx)
+
+// CHECK: vpmovswb %ymm23, (%rcx) {%k2}
+// CHECK:  encoding: [0x62,0xe2,0x7e,0x2a,0x20,0x39]
+          vpmovswb %ymm23,(%rcx) {%k2}
+
+// CHECK: vpmovswb %ymm23, 4660(%rax,%r14,8)
+// CHECK:  encoding: [0x62,0xa2,0x7e,0x28,0x20,0xbc,0xf0,0x34,0x12,0x00,0x00]
+          vpmovswb %ymm23, 4660(%rax,%r14,8)
+
+// CHECK: vpmovswb %ymm23, 2032(%rdx)
+// CHECK:  encoding: [0x62,0xe2,0x7e,0x28,0x20,0x7a,0x7f]
+          vpmovswb %ymm23, 2032(%rdx)
+
+// CHECK: vpmovswb %ymm23, 2048(%rdx)
+// CHECK:  encoding: [0x62,0xe2,0x7e,0x28,0x20,0xba,0x00,0x08,0x00,0x00]
+          vpmovswb %ymm23, 2048(%rdx)
+
+// CHECK: vpmovswb %ymm23, -2048(%rdx)
+// CHECK:  encoding: [0x62,0xe2,0x7e,0x28,0x20,0x7a,0x80]
+          vpmovswb %ymm23, -2048(%rdx)
+
+// CHECK: vpmovswb %ymm23, -2064(%rdx)
+// CHECK:  encoding: [0x62,0xe2,0x7e,0x28,0x20,0xba,0xf0,0xf7,0xff,0xff]
+          vpmovswb %ymm23, -2064(%rdx)
+
+// CHECK: vpmovuswb %xmm17, %xmm26
+// CHECK:  encoding: [0x62,0x82,0x7e,0x08,0x10,0xca]
+          vpmovuswb %xmm17, %xmm26
+
+// CHECK: vpmovuswb %xmm17, %xmm26 {%k6}
+// CHECK:  encoding: [0x62,0x82,0x7e,0x0e,0x10,0xca]
+          vpmovuswb %xmm17, %xmm26 {%k6}
+
+// CHECK: vpmovuswb %xmm17, %xmm26 {%k6} {z}
+// CHECK:  encoding: [0x62,0x82,0x7e,0x8e,0x10,0xca]
+          vpmovuswb %xmm17, %xmm26 {%k6} {z}
+
+// CHECK: vpmovuswb %ymm26, %xmm17
+// CHECK:  encoding: [0x62,0x22,0x7e,0x28,0x10,0xd1]
+          vpmovuswb %ymm26, %xmm17
+
+// CHECK: vpmovuswb %ymm26, %xmm17 {%k2}
+// CHECK:  encoding: [0x62,0x22,0x7e,0x2a,0x10,0xd1]
+          vpmovuswb %ymm26, %xmm17 {%k2}
+
+// CHECK: vpmovuswb %ymm26, %xmm17 {%k2} {z}
+// CHECK:  encoding: [0x62,0x22,0x7e,0xaa,0x10,0xd1]
+          vpmovuswb %ymm26, %xmm17 {%k2} {z}
+
+// CHECK: vpmovuswb %xmm19, (%rcx)
+// CHECK:  encoding: [0x62,0xe2,0x7e,0x08,0x10,0x19]
+          vpmovuswb %xmm19,(%rcx)
+
+// CHECK: vpmovuswb %xmm19, (%rcx) {%k1}
+// CHECK:  encoding: [0x62,0xe2,0x7e,0x09,0x10,0x19]
+          vpmovuswb %xmm19,(%rcx) {%k1}
+
+// CHECK: vpmovuswb %xmm19, 4660(%rax,%r14,8)
+// CHECK:  encoding: [0x62,0xa2,0x7e,0x08,0x10,0x9c,0xf0,0x34,0x12,0x00,0x00]
+          vpmovuswb %xmm19, 4660(%rax,%r14,8)
+
+// CHECK: vpmovuswb %xmm19, 1016(%rdx)
+// CHECK:  encoding: [0x62,0xe2,0x7e,0x08,0x10,0x5a,0x7f]
+          vpmovuswb %xmm19, 1016(%rdx)
+
+// CHECK: vpmovuswb %xmm19, 1024(%rdx)
+// CHECK:  encoding: [0x62,0xe2,0x7e,0x08,0x10,0x9a,0x00,0x04,0x00,0x00]
+          vpmovuswb %xmm19, 1024(%rdx)
+
+// CHECK: vpmovuswb %xmm19, -1024(%rdx)
+// CHECK:  encoding: [0x62,0xe2,0x7e,0x08,0x10,0x5a,0x80]
+          vpmovuswb %xmm19, -1024(%rdx)
+
+// CHECK: vpmovuswb %xmm19, -1032(%rdx)
+// CHECK:  encoding: [0x62,0xe2,0x7e,0x08,0x10,0x9a,0xf8,0xfb,0xff,0xff]
+          vpmovuswb %xmm19, -1032(%rdx)
+
+// CHECK: vpmovuswb %ymm23, (%rcx)
+// CHECK:  encoding: [0x62,0xe2,0x7e,0x28,0x10,0x39]
+          vpmovuswb %ymm23,(%rcx)
+
+// CHECK: vpmovuswb %ymm23, (%rcx) {%k6}
+// CHECK:  encoding: [0x62,0xe2,0x7e,0x2e,0x10,0x39]
+          vpmovuswb %ymm23,(%rcx) {%k6}
+
+// CHECK: vpmovuswb %ymm23, 4660(%rax,%r14,8)
+// CHECK:  encoding: [0x62,0xa2,0x7e,0x28,0x10,0xbc,0xf0,0x34,0x12,0x00,0x00]
+          vpmovuswb %ymm23, 4660(%rax,%r14,8)
+
+// CHECK: vpmovuswb %ymm23, 2032(%rdx)
+// CHECK:  encoding: [0x62,0xe2,0x7e,0x28,0x10,0x7a,0x7f]
+          vpmovuswb %ymm23, 2032(%rdx)
+
+// CHECK: vpmovuswb %ymm23, 2048(%rdx)
+// CHECK:  encoding: [0x62,0xe2,0x7e,0x28,0x10,0xba,0x00,0x08,0x00,0x00]
+          vpmovuswb %ymm23, 2048(%rdx)
+
+// CHECK: vpmovuswb %ymm23, -2048(%rdx)
+// CHECK:  encoding: [0x62,0xe2,0x7e,0x28,0x10,0x7a,0x80]
+          vpmovuswb %ymm23, -2048(%rdx)
+
+// CHECK: vpmovuswb %ymm23, -2064(%rdx)
+// CHECK:  encoding: [0x62,0xe2,0x7e,0x28,0x10,0xba,0xf0,0xf7,0xff,0xff]
+          vpmovuswb %ymm23, -2064(%rdx)
+
+// CHECK: vpmovwb %xmm17, %xmm21
+// CHECK:  encoding: [0x62,0xa2,0x7e,0x08,0x30,0xcd]
+          vpmovwb %xmm17, %xmm21
+
+// CHECK: vpmovwb %xmm17, %xmm21 {%k1}
+// CHECK:  encoding: [0x62,0xa2,0x7e,0x09,0x30,0xcd]
+          vpmovwb %xmm17, %xmm21 {%k1}
+
+// CHECK: vpmovwb %xmm17, %xmm21 {%k1} {z}
+// CHECK:  encoding: [0x62,0xa2,0x7e,0x89,0x30,0xcd]
+          vpmovwb %xmm17, %xmm21 {%k1} {z}
+
+// CHECK: vpmovwb %ymm23, %xmm26
+// CHECK:  encoding: [0x62,0x82,0x7e,0x28,0x30,0xfa]
+          vpmovwb %ymm23, %xmm26
+
+// CHECK: vpmovwb %ymm23, %xmm26 {%k7}
+// CHECK:  encoding: [0x62,0x82,0x7e,0x2f,0x30,0xfa]
+          vpmovwb %ymm23, %xmm26 {%k7}
+
+// CHECK: vpmovwb %ymm23, %xmm26 {%k7} {z}
+// CHECK:  encoding: [0x62,0x82,0x7e,0xaf,0x30,0xfa]
+          vpmovwb %ymm23, %xmm26 {%k7} {z}
+
+// CHECK: vpmovwb %xmm21, (%rcx)
+// CHECK:  encoding: [0x62,0xe2,0x7e,0x08,0x30,0x29]
+          vpmovwb %xmm21, (%rcx)
+
+// CHECK: vpmovwb %xmm21, (%rcx) {%k2}
+// CHECK:  encoding: [0x62,0xe2,0x7e,0x0a,0x30,0x29]
+          vpmovwb %xmm21, (%rcx) {%k2}
+
+// CHECK: vpmovwb %xmm21, 291(%rax,%r14,8)
+// CHECK:  encoding: [0x62,0xa2,0x7e,0x08,0x30,0xac,0xf0,0x23,0x01,0x00,0x00]
+          vpmovwb %xmm21, 291(%rax,%r14,8)
+
+// CHECK: vpmovwb %xmm21, 1016(%rdx)
+// CHECK:  encoding: [0x62,0xe2,0x7e,0x08,0x30,0x6a,0x7f]
+          vpmovwb %xmm21, 1016(%rdx)
+
+// CHECK: vpmovwb %xmm21, 1024(%rdx)
+// CHECK:  encoding: [0x62,0xe2,0x7e,0x08,0x30,0xaa,0x00,0x04,0x00,0x00]
+          vpmovwb %xmm21, 1024(%rdx)
+
+// CHECK: vpmovwb %xmm21, -1024(%rdx)
+// CHECK:  encoding: [0x62,0xe2,0x7e,0x08,0x30,0x6a,0x80]
+          vpmovwb %xmm21, -1024(%rdx)
+
+// CHECK: vpmovwb %xmm21, -1032(%rdx)
+// CHECK:  encoding: [0x62,0xe2,0x7e,0x08,0x30,0xaa,0xf8,0xfb,0xff,0xff]
+          vpmovwb %xmm21, -1032(%rdx)
+
+// CHECK: vpmovwb %ymm20, (%rcx)
+// CHECK:  encoding: [0x62,0xe2,0x7e,0x28,0x30,0x21]
+          vpmovwb %ymm20, (%rcx)
+
+// CHECK: vpmovwb %ymm20, (%rcx) {%k4}
+// CHECK:  encoding: [0x62,0xe2,0x7e,0x2c,0x30,0x21]
+          vpmovwb %ymm20, (%rcx) {%k4}
+
+// CHECK: vpmovwb %ymm20, 291(%rax,%r14,8)
+// CHECK:  encoding: [0x62,0xa2,0x7e,0x28,0x30,0xa4,0xf0,0x23,0x01,0x00,0x00]
+          vpmovwb %ymm20, 291(%rax,%r14,8)
+
+// CHECK: vpmovwb %ymm20, 2032(%rdx)
+// CHECK:  encoding: [0x62,0xe2,0x7e,0x28,0x30,0x62,0x7f]
+          vpmovwb %ymm20, 2032(%rdx)
+
+// CHECK: vpmovwb %ymm20, 2048(%rdx)
+// CHECK:  encoding: [0x62,0xe2,0x7e,0x28,0x30,0xa2,0x00,0x08,0x00,0x00]
+          vpmovwb %ymm20, 2048(%rdx)
+
+// CHECK: vpmovwb %ymm20, -2048(%rdx)
+// CHECK:  encoding: [0x62,0xe2,0x7e,0x28,0x30,0x62,0x80]
+          vpmovwb %ymm20, -2048(%rdx)
+
+// CHECK: vpmovwb %ymm20, -2064(%rdx)
+// CHECK:  encoding: [0x62,0xe2,0x7e,0x28,0x30,0xa2,0xf0,0xf7,0xff,0xff]
+          vpmovwb %ymm20, -2064(%rdx)
+
+// CHECK: vpmovswb %xmm20, %xmm24
+// CHECK:  encoding: [0x62,0x82,0x7e,0x08,0x20,0xe0]
+          vpmovswb %xmm20, %xmm24
+
+// CHECK: vpmovswb %xmm20, %xmm24 {%k4}
+// CHECK:  encoding: [0x62,0x82,0x7e,0x0c,0x20,0xe0]
+          vpmovswb %xmm20, %xmm24 {%k4}
+
+// CHECK: vpmovswb %xmm20, %xmm24 {%k4} {z}
+// CHECK:  encoding: [0x62,0x82,0x7e,0x8c,0x20,0xe0]
+          vpmovswb %xmm20, %xmm24 {%k4} {z}
+
+// CHECK: vpmovswb %ymm18, %xmm27
+// CHECK:  encoding: [0x62,0x82,0x7e,0x28,0x20,0xd3]
+          vpmovswb %ymm18, %xmm27
+
+// CHECK: vpmovswb %ymm18, %xmm27 {%k1}
+// CHECK:  encoding: [0x62,0x82,0x7e,0x29,0x20,0xd3]
+          vpmovswb %ymm18, %xmm27 {%k1}
+
+// CHECK: vpmovswb %ymm18, %xmm27 {%k1} {z}
+// CHECK:  encoding: [0x62,0x82,0x7e,0xa9,0x20,0xd3]
+          vpmovswb %ymm18, %xmm27 {%k1} {z}
+
+// CHECK: vpmovswb %xmm24, (%rcx)
+// CHECK:  encoding: [0x62,0x62,0x7e,0x08,0x20,0x01]
+          vpmovswb %xmm24, (%rcx)
+
+// CHECK: vpmovswb %xmm24, (%rcx) {%k3}
+// CHECK:  encoding: [0x62,0x62,0x7e,0x0b,0x20,0x01]
+          vpmovswb %xmm24, (%rcx) {%k3}
+
+// CHECK: vpmovswb %xmm24, 291(%rax,%r14,8)
+// CHECK:  encoding: [0x62,0x22,0x7e,0x08,0x20,0x84,0xf0,0x23,0x01,0x00,0x00]
+          vpmovswb %xmm24, 291(%rax,%r14,8)
+
+// CHECK: vpmovswb %xmm24, 1016(%rdx)
+// CHECK:  encoding: [0x62,0x62,0x7e,0x08,0x20,0x42,0x7f]
+          vpmovswb %xmm24, 1016(%rdx)
+
+// CHECK: vpmovswb %xmm24, 1024(%rdx)
+// CHECK:  encoding: [0x62,0x62,0x7e,0x08,0x20,0x82,0x00,0x04,0x00,0x00]
+          vpmovswb %xmm24, 1024(%rdx)
+
+// CHECK: vpmovswb %xmm24, -1024(%rdx)
+// CHECK:  encoding: [0x62,0x62,0x7e,0x08,0x20,0x42,0x80]
+          vpmovswb %xmm24, -1024(%rdx)
+
+// CHECK: vpmovswb %xmm24, -1032(%rdx)
+// CHECK:  encoding: [0x62,0x62,0x7e,0x08,0x20,0x82,0xf8,0xfb,0xff,0xff]
+          vpmovswb %xmm24, -1032(%rdx)
+
+// CHECK: vpmovswb %ymm27, (%rcx)
+// CHECK:  encoding: [0x62,0x62,0x7e,0x28,0x20,0x19]
+          vpmovswb %ymm27, (%rcx)
+
+// CHECK: vpmovswb %ymm27, (%rcx) {%k7}
+// CHECK:  encoding: [0x62,0x62,0x7e,0x2f,0x20,0x19]
+          vpmovswb %ymm27, (%rcx) {%k7}
+
+// CHECK: vpmovswb %ymm27, 291(%rax,%r14,8)
+// CHECK:  encoding: [0x62,0x22,0x7e,0x28,0x20,0x9c,0xf0,0x23,0x01,0x00,0x00]
+          vpmovswb %ymm27, 291(%rax,%r14,8)
+
+// CHECK: vpmovswb %ymm27, 2032(%rdx)
+// CHECK:  encoding: [0x62,0x62,0x7e,0x28,0x20,0x5a,0x7f]
+          vpmovswb %ymm27, 2032(%rdx)
+
+// CHECK: vpmovswb %ymm27, 2048(%rdx)
+// CHECK:  encoding: [0x62,0x62,0x7e,0x28,0x20,0x9a,0x00,0x08,0x00,0x00]
+          vpmovswb %ymm27, 2048(%rdx)
+
+// CHECK: vpmovswb %ymm27, -2048(%rdx)
+// CHECK:  encoding: [0x62,0x62,0x7e,0x28,0x20,0x5a,0x80]
+          vpmovswb %ymm27, -2048(%rdx)
+
+// CHECK: vpmovswb %ymm27, -2064(%rdx)
+// CHECK:  encoding: [0x62,0x62,0x7e,0x28,0x20,0x9a,0xf0,0xf7,0xff,0xff]
+          vpmovswb %ymm27, -2064(%rdx)
+
+// CHECK: vpmovuswb %xmm19, %xmm23
+// CHECK:  encoding: [0x62,0xa2,0x7e,0x08,0x10,0xdf]
+          vpmovuswb %xmm19, %xmm23
+
+// CHECK: vpmovuswb %xmm19, %xmm23 {%k4}
+// CHECK:  encoding: [0x62,0xa2,0x7e,0x0c,0x10,0xdf]
+          vpmovuswb %xmm19, %xmm23 {%k4}
+
+// CHECK: vpmovuswb %xmm19, %xmm23 {%k4} {z}
+// CHECK:  encoding: [0x62,0xa2,0x7e,0x8c,0x10,0xdf]
+          vpmovuswb %xmm19, %xmm23 {%k4} {z}
+
+// CHECK: vpmovuswb %ymm23, %xmm28
+// CHECK:  encoding: [0x62,0x82,0x7e,0x28,0x10,0xfc]
+          vpmovuswb %ymm23, %xmm28
+
+// CHECK: vpmovuswb %ymm23, %xmm28 {%k6}
+// CHECK:  encoding: [0x62,0x82,0x7e,0x2e,0x10,0xfc]
+          vpmovuswb %ymm23, %xmm28 {%k6}
+
+// CHECK: vpmovuswb %ymm23, %xmm28 {%k6} {z}
+// CHECK:  encoding: [0x62,0x82,0x7e,0xae,0x10,0xfc]
+          vpmovuswb %ymm23, %xmm28 {%k6} {z}
+
+// CHECK: vpmovuswb %xmm25, (%rcx)
+// CHECK:  encoding: [0x62,0x62,0x7e,0x08,0x10,0x09]
+          vpmovuswb %xmm25, (%rcx)
+
+// CHECK: vpmovuswb %xmm25, (%rcx) {%k3}
+// CHECK:  encoding: [0x62,0x62,0x7e,0x0b,0x10,0x09]
+          vpmovuswb %xmm25, (%rcx) {%k3}
+
+// CHECK: vpmovuswb %xmm25, 291(%rax,%r14,8)
+// CHECK:  encoding: [0x62,0x22,0x7e,0x08,0x10,0x8c,0xf0,0x23,0x01,0x00,0x00]
+          vpmovuswb %xmm25, 291(%rax,%r14,8)
+
+// CHECK: vpmovuswb %xmm25, 1016(%rdx)
+// CHECK:  encoding: [0x62,0x62,0x7e,0x08,0x10,0x4a,0x7f]
+          vpmovuswb %xmm25, 1016(%rdx)
+
+// CHECK: vpmovuswb %xmm25, 1024(%rdx)
+// CHECK:  encoding: [0x62,0x62,0x7e,0x08,0x10,0x8a,0x00,0x04,0x00,0x00]
+          vpmovuswb %xmm25, 1024(%rdx)
+
+// CHECK: vpmovuswb %xmm25, -1024(%rdx)
+// CHECK:  encoding: [0x62,0x62,0x7e,0x08,0x10,0x4a,0x80]
+          vpmovuswb %xmm25, -1024(%rdx)
+
+// CHECK: vpmovuswb %xmm25, -1032(%rdx)
+// CHECK:  encoding: [0x62,0x62,0x7e,0x08,0x10,0x8a,0xf8,0xfb,0xff,0xff]
+          vpmovuswb %xmm25, -1032(%rdx)
+
+// CHECK: vpmovuswb %ymm28, (%rcx)
+// CHECK:  encoding: [0x62,0x62,0x7e,0x28,0x10,0x21]
+          vpmovuswb %ymm28, (%rcx)
+
+// CHECK: vpmovuswb %ymm28, (%rcx) {%k2}
+// CHECK:  encoding: [0x62,0x62,0x7e,0x2a,0x10,0x21]
+          vpmovuswb %ymm28, (%rcx) {%k2}
+
+// CHECK: vpmovuswb %ymm28, 291(%rax,%r14,8)
+// CHECK:  encoding: [0x62,0x22,0x7e,0x28,0x10,0xa4,0xf0,0x23,0x01,0x00,0x00]
+          vpmovuswb %ymm28, 291(%rax,%r14,8)
+
+// CHECK: vpmovuswb %ymm28, 2032(%rdx)
+// CHECK:  encoding: [0x62,0x62,0x7e,0x28,0x10,0x62,0x7f]
+          vpmovuswb %ymm28, 2032(%rdx)
+
+// CHECK: vpmovuswb %ymm28, 2048(%rdx)
+// CHECK:  encoding: [0x62,0x62,0x7e,0x28,0x10,0xa2,0x00,0x08,0x00,0x00]
+          vpmovuswb %ymm28, 2048(%rdx)
+
+// CHECK: vpmovuswb %ymm28, -2048(%rdx)
+// CHECK:  encoding: [0x62,0x62,0x7e,0x28,0x10,0x62,0x80]
+          vpmovuswb %ymm28, -2048(%rdx)
+
+// CHECK: vpmovuswb %ymm28, -2064(%rdx)
+// CHECK:  encoding: [0x62,0x62,0x7e,0x28,0x10,0xa2,0xf0,0xf7,0xff,0xff]
+          vpmovuswb %ymm28, -2064(%rdx)
+
+// CHECK: vpmulhuw %xmm18, %xmm21, %xmm24
+// CHECK:  encoding: [0x62,0x21,0x55,0x00,0xe4,0xc2]
+          vpmulhuw %xmm18, %xmm21, %xmm24
+
+// CHECK: vpmulhuw %xmm18, %xmm21, %xmm24 {%k3}
+// CHECK:  encoding: [0x62,0x21,0x55,0x03,0xe4,0xc2]
+          vpmulhuw %xmm18, %xmm21, %xmm24 {%k3}
+
+// CHECK: vpmulhuw %xmm18, %xmm21, %xmm24 {%k3} {z}
+// CHECK:  encoding: [0x62,0x21,0x55,0x83,0xe4,0xc2]
+          vpmulhuw %xmm18, %xmm21, %xmm24 {%k3} {z}
+
+// CHECK: vpmulhuw (%rcx), %xmm21, %xmm24
+// CHECK:  encoding: [0x62,0x61,0x55,0x00,0xe4,0x01]
+          vpmulhuw (%rcx), %xmm21, %xmm24
+
+// CHECK: vpmulhuw 291(%rax,%r14,8), %xmm21, %xmm24
+// CHECK:  encoding: [0x62,0x21,0x55,0x00,0xe4,0x84,0xf0,0x23,0x01,0x00,0x00]
+          vpmulhuw 291(%rax,%r14,8), %xmm21, %xmm24
+
+// CHECK: vpmulhuw 2032(%rdx), %xmm21, %xmm24
+// CHECK:  encoding: [0x62,0x61,0x55,0x00,0xe4,0x42,0x7f]
+          vpmulhuw 2032(%rdx), %xmm21, %xmm24
+
+// CHECK: vpmulhuw 2048(%rdx), %xmm21, %xmm24
+// CHECK:  encoding: [0x62,0x61,0x55,0x00,0xe4,0x82,0x00,0x08,0x00,0x00]
+          vpmulhuw 2048(%rdx), %xmm21, %xmm24
+
+// CHECK: vpmulhuw -2048(%rdx), %xmm21, %xmm24
+// CHECK:  encoding: [0x62,0x61,0x55,0x00,0xe4,0x42,0x80]
+          vpmulhuw -2048(%rdx), %xmm21, %xmm24
+
+// CHECK: vpmulhuw -2064(%rdx), %xmm21, %xmm24
+// CHECK:  encoding: [0x62,0x61,0x55,0x00,0xe4,0x82,0xf0,0xf7,0xff,0xff]
+          vpmulhuw -2064(%rdx), %xmm21, %xmm24
+
+// CHECK: vpmulhuw %ymm19, %ymm28, %ymm19
+// CHECK:  encoding: [0x62,0xa1,0x1d,0x20,0xe4,0xdb]
+          vpmulhuw %ymm19, %ymm28, %ymm19
+
+// CHECK: vpmulhuw %ymm19, %ymm28, %ymm19 {%k2}
+// CHECK:  encoding: [0x62,0xa1,0x1d,0x22,0xe4,0xdb]
+          vpmulhuw %ymm19, %ymm28, %ymm19 {%k2}
+
+// CHECK: vpmulhuw %ymm19, %ymm28, %ymm19 {%k2} {z}
+// CHECK:  encoding: [0x62,0xa1,0x1d,0xa2,0xe4,0xdb]
+          vpmulhuw %ymm19, %ymm28, %ymm19 {%k2} {z}
+
+// CHECK: vpmulhuw (%rcx), %ymm28, %ymm19
+// CHECK:  encoding: [0x62,0xe1,0x1d,0x20,0xe4,0x19]
+          vpmulhuw (%rcx), %ymm28, %ymm19
+
+// CHECK: vpmulhuw 291(%rax,%r14,8), %ymm28, %ymm19
+// CHECK:  encoding: [0x62,0xa1,0x1d,0x20,0xe4,0x9c,0xf0,0x23,0x01,0x00,0x00]
+          vpmulhuw 291(%rax,%r14,8), %ymm28, %ymm19
+
+// CHECK: vpmulhuw 4064(%rdx), %ymm28, %ymm19
+// CHECK:  encoding: [0x62,0xe1,0x1d,0x20,0xe4,0x5a,0x7f]
+          vpmulhuw 4064(%rdx), %ymm28, %ymm19
+
+// CHECK: vpmulhuw 4096(%rdx), %ymm28, %ymm19
+// CHECK:  encoding: [0x62,0xe1,0x1d,0x20,0xe4,0x9a,0x00,0x10,0x00,0x00]
+          vpmulhuw 4096(%rdx), %ymm28, %ymm19
+
+// CHECK: vpmulhuw -4096(%rdx), %ymm28, %ymm19
+// CHECK:  encoding: [0x62,0xe1,0x1d,0x20,0xe4,0x5a,0x80]
+          vpmulhuw -4096(%rdx), %ymm28, %ymm19
+
+// CHECK: vpmulhuw -4128(%rdx), %ymm28, %ymm19
+// CHECK:  encoding: [0x62,0xe1,0x1d,0x20,0xe4,0x9a,0xe0,0xef,0xff,0xff]
+          vpmulhuw -4128(%rdx), %ymm28, %ymm19
+
+// CHECK: vpmulhw %xmm25, %xmm20, %xmm22
+// CHECK:  encoding: [0x62,0x81,0x5d,0x00,0xe5,0xf1]
+          vpmulhw %xmm25, %xmm20, %xmm22
+
+// CHECK: vpmulhw %xmm25, %xmm20, %xmm22 {%k2}
+// CHECK:  encoding: [0x62,0x81,0x5d,0x02,0xe5,0xf1]
+          vpmulhw %xmm25, %xmm20, %xmm22 {%k2}
+
+// CHECK: vpmulhw %xmm25, %xmm20, %xmm22 {%k2} {z}
+// CHECK:  encoding: [0x62,0x81,0x5d,0x82,0xe5,0xf1]
+          vpmulhw %xmm25, %xmm20, %xmm22 {%k2} {z}
+
+// CHECK: vpmulhw (%rcx), %xmm20, %xmm22
+// CHECK:  encoding: [0x62,0xe1,0x5d,0x00,0xe5,0x31]
+          vpmulhw (%rcx), %xmm20, %xmm22
+
+// CHECK: vpmulhw 291(%rax,%r14,8), %xmm20, %xmm22
+// CHECK:  encoding: [0x62,0xa1,0x5d,0x00,0xe5,0xb4,0xf0,0x23,0x01,0x00,0x00]
+          vpmulhw 291(%rax,%r14,8), %xmm20, %xmm22
+
+// CHECK: vpmulhw 2032(%rdx), %xmm20, %xmm22
+// CHECK:  encoding: [0x62,0xe1,0x5d,0x00,0xe5,0x72,0x7f]
+          vpmulhw 2032(%rdx), %xmm20, %xmm22
+
+// CHECK: vpmulhw 2048(%rdx), %xmm20, %xmm22
+// CHECK:  encoding: [0x62,0xe1,0x5d,0x00,0xe5,0xb2,0x00,0x08,0x00,0x00]
+          vpmulhw 2048(%rdx), %xmm20, %xmm22
+
+// CHECK: vpmulhw -2048(%rdx), %xmm20, %xmm22
+// CHECK:  encoding: [0x62,0xe1,0x5d,0x00,0xe5,0x72,0x80]
+          vpmulhw -2048(%rdx), %xmm20, %xmm22
+
+// CHECK: vpmulhw -2064(%rdx), %xmm20, %xmm22
+// CHECK:  encoding: [0x62,0xe1,0x5d,0x00,0xe5,0xb2,0xf0,0xf7,0xff,0xff]
+          vpmulhw -2064(%rdx), %xmm20, %xmm22
+
+// CHECK: vpmulhw %ymm24, %ymm27, %ymm22
+// CHECK:  encoding: [0x62,0x81,0x25,0x20,0xe5,0xf0]
+          vpmulhw %ymm24, %ymm27, %ymm22
+
+// CHECK: vpmulhw %ymm24, %ymm27, %ymm22 {%k1}
+// CHECK:  encoding: [0x62,0x81,0x25,0x21,0xe5,0xf0]
+          vpmulhw %ymm24, %ymm27, %ymm22 {%k1}
+
+// CHECK: vpmulhw %ymm24, %ymm27, %ymm22 {%k1} {z}
+// CHECK:  encoding: [0x62,0x81,0x25,0xa1,0xe5,0xf0]
+          vpmulhw %ymm24, %ymm27, %ymm22 {%k1} {z}
+
+// CHECK: vpmulhw (%rcx), %ymm27, %ymm22
+// CHECK:  encoding: [0x62,0xe1,0x25,0x20,0xe5,0x31]
+          vpmulhw (%rcx), %ymm27, %ymm22
+
+// CHECK: vpmulhw 291(%rax,%r14,8), %ymm27, %ymm22
+// CHECK:  encoding: [0x62,0xa1,0x25,0x20,0xe5,0xb4,0xf0,0x23,0x01,0x00,0x00]
+          vpmulhw 291(%rax,%r14,8), %ymm27, %ymm22
+
+// CHECK: vpmulhw 4064(%rdx), %ymm27, %ymm22
+// CHECK:  encoding: [0x62,0xe1,0x25,0x20,0xe5,0x72,0x7f]
+          vpmulhw 4064(%rdx), %ymm27, %ymm22
+
+// CHECK: vpmulhw 4096(%rdx), %ymm27, %ymm22
+// CHECK:  encoding: [0x62,0xe1,0x25,0x20,0xe5,0xb2,0x00,0x10,0x00,0x00]
+          vpmulhw 4096(%rdx), %ymm27, %ymm22
+
+// CHECK: vpmulhw -4096(%rdx), %ymm27, %ymm22
+// CHECK:  encoding: [0x62,0xe1,0x25,0x20,0xe5,0x72,0x80]
+          vpmulhw -4096(%rdx), %ymm27, %ymm22
+
+// CHECK: vpmulhw -4128(%rdx), %ymm27, %ymm22
+// CHECK:  encoding: [0x62,0xe1,0x25,0x20,0xe5,0xb2,0xe0,0xef,0xff,0xff]
+          vpmulhw -4128(%rdx), %ymm27, %ymm22
+
+// CHECK: vpmulhrsw %xmm26, %xmm19, %xmm28
+// CHECK:  encoding: [0x62,0x02,0x65,0x00,0x0b,0xe2]
+          vpmulhrsw %xmm26, %xmm19, %xmm28
+
+// CHECK: vpmulhrsw %xmm26, %xmm19, %xmm28 {%k6}
+// CHECK:  encoding: [0x62,0x02,0x65,0x06,0x0b,0xe2]
+          vpmulhrsw %xmm26, %xmm19, %xmm28 {%k6}
+
+// CHECK: vpmulhrsw %xmm26, %xmm19, %xmm28 {%k6} {z}
+// CHECK:  encoding: [0x62,0x02,0x65,0x86,0x0b,0xe2]
+          vpmulhrsw %xmm26, %xmm19, %xmm28 {%k6} {z}
+
+// CHECK: vpmulhrsw (%rcx), %xmm19, %xmm28
+// CHECK:  encoding: [0x62,0x62,0x65,0x00,0x0b,0x21]
+          vpmulhrsw (%rcx), %xmm19, %xmm28
+
+// CHECK: vpmulhrsw 291(%rax,%r14,8), %xmm19, %xmm28
+// CHECK:  encoding: [0x62,0x22,0x65,0x00,0x0b,0xa4,0xf0,0x23,0x01,0x00,0x00]
+          vpmulhrsw 291(%rax,%r14,8), %xmm19, %xmm28
+
+// CHECK: vpmulhrsw 2032(%rdx), %xmm19, %xmm28
+// CHECK:  encoding: [0x62,0x62,0x65,0x00,0x0b,0x62,0x7f]
+          vpmulhrsw 2032(%rdx), %xmm19, %xmm28
+
+// CHECK: vpmulhrsw 2048(%rdx), %xmm19, %xmm28
+// CHECK:  encoding: [0x62,0x62,0x65,0x00,0x0b,0xa2,0x00,0x08,0x00,0x00]
+          vpmulhrsw 2048(%rdx), %xmm19, %xmm28
+
+// CHECK: vpmulhrsw -2048(%rdx), %xmm19, %xmm28
+// CHECK:  encoding: [0x62,0x62,0x65,0x00,0x0b,0x62,0x80]
+          vpmulhrsw -2048(%rdx), %xmm19, %xmm28
+
+// CHECK: vpmulhrsw -2064(%rdx), %xmm19, %xmm28
+// CHECK:  encoding: [0x62,0x62,0x65,0x00,0x0b,0xa2,0xf0,0xf7,0xff,0xff]
+          vpmulhrsw -2064(%rdx), %xmm19, %xmm28
+
+// CHECK: vpmulhrsw %ymm26, %ymm20, %ymm28
+// CHECK:  encoding: [0x62,0x02,0x5d,0x20,0x0b,0xe2]
+          vpmulhrsw %ymm26, %ymm20, %ymm28
+
+// CHECK: vpmulhrsw %ymm26, %ymm20, %ymm28 {%k3}
+// CHECK:  encoding: [0x62,0x02,0x5d,0x23,0x0b,0xe2]
+          vpmulhrsw %ymm26, %ymm20, %ymm28 {%k3}
+
+// CHECK: vpmulhrsw %ymm26, %ymm20, %ymm28 {%k3} {z}
+// CHECK:  encoding: [0x62,0x02,0x5d,0xa3,0x0b,0xe2]
+          vpmulhrsw %ymm26, %ymm20, %ymm28 {%k3} {z}
+
+// CHECK: vpmulhrsw (%rcx), %ymm20, %ymm28
+// CHECK:  encoding: [0x62,0x62,0x5d,0x20,0x0b,0x21]
+          vpmulhrsw (%rcx), %ymm20, %ymm28
+
+// CHECK: vpmulhrsw 291(%rax,%r14,8), %ymm20, %ymm28
+// CHECK:  encoding: [0x62,0x22,0x5d,0x20,0x0b,0xa4,0xf0,0x23,0x01,0x00,0x00]
+          vpmulhrsw 291(%rax,%r14,8), %ymm20, %ymm28
+
+// CHECK: vpmulhrsw 4064(%rdx), %ymm20, %ymm28
+// CHECK:  encoding: [0x62,0x62,0x5d,0x20,0x0b,0x62,0x7f]
+          vpmulhrsw 4064(%rdx), %ymm20, %ymm28
+
+// CHECK: vpmulhrsw 4096(%rdx), %ymm20, %ymm28
+// CHECK:  encoding: [0x62,0x62,0x5d,0x20,0x0b,0xa2,0x00,0x10,0x00,0x00]
+          vpmulhrsw 4096(%rdx), %ymm20, %ymm28
+
+// CHECK: vpmulhrsw -4096(%rdx), %ymm20, %ymm28
+// CHECK:  encoding: [0x62,0x62,0x5d,0x20,0x0b,0x62,0x80]
+          vpmulhrsw -4096(%rdx), %ymm20, %ymm28
+
+// CHECK: vpmulhrsw -4128(%rdx), %ymm20, %ymm28
+// CHECK:  encoding: [0x62,0x62,0x5d,0x20,0x0b,0xa2,0xe0,0xef,0xff,0xff]
+          vpmulhrsw -4128(%rdx), %ymm20, %ymm28
+
+// CHECK: vpmaddubsw %xmm20, %xmm21, %xmm28
+// CHECK:  encoding: [0x62,0x22,0x55,0x00,0x04,0xe4]
+          vpmaddubsw %xmm20, %xmm21, %xmm28
+
+// CHECK: vpmaddubsw %xmm20, %xmm21, %xmm28 {%k6}
+// CHECK:  encoding: [0x62,0x22,0x55,0x06,0x04,0xe4]
+          vpmaddubsw %xmm20, %xmm21, %xmm28 {%k6}
+
+// CHECK: vpmaddubsw %xmm20, %xmm21, %xmm28 {%k6} {z}
+// CHECK:  encoding: [0x62,0x22,0x55,0x86,0x04,0xe4]
+          vpmaddubsw %xmm20, %xmm21, %xmm28 {%k6} {z}
+
+// CHECK: vpmaddubsw (%rcx), %xmm21, %xmm28
+// CHECK:  encoding: [0x62,0x62,0x55,0x00,0x04,0x21]
+          vpmaddubsw (%rcx), %xmm21, %xmm28
+
+// CHECK: vpmaddubsw 291(%rax,%r14,8), %xmm21, %xmm28
+// CHECK:  encoding: [0x62,0x22,0x55,0x00,0x04,0xa4,0xf0,0x23,0x01,0x00,0x00]
+          vpmaddubsw 291(%rax,%r14,8), %xmm21, %xmm28
+
+// CHECK: vpmaddubsw 2032(%rdx), %xmm21, %xmm28
+// CHECK:  encoding: [0x62,0x62,0x55,0x00,0x04,0x62,0x7f]
+          vpmaddubsw 2032(%rdx), %xmm21, %xmm28
+
+// CHECK: vpmaddubsw 2048(%rdx), %xmm21, %xmm28
+// CHECK:  encoding: [0x62,0x62,0x55,0x00,0x04,0xa2,0x00,0x08,0x00,0x00]
+          vpmaddubsw 2048(%rdx), %xmm21, %xmm28
+
+// CHECK: vpmaddubsw -2048(%rdx), %xmm21, %xmm28
+// CHECK:  encoding: [0x62,0x62,0x55,0x00,0x04,0x62,0x80]
+          vpmaddubsw -2048(%rdx), %xmm21, %xmm28
+
+// CHECK: vpmaddubsw -2064(%rdx), %xmm21, %xmm28
+// CHECK:  encoding: [0x62,0x62,0x55,0x00,0x04,0xa2,0xf0,0xf7,0xff,0xff]
+          vpmaddubsw -2064(%rdx), %xmm21, %xmm28
+
+// CHECK: vpmaddubsw %ymm26, %ymm26, %ymm30
+// CHECK:  encoding: [0x62,0x02,0x2d,0x20,0x04,0xf2]
+          vpmaddubsw %ymm26, %ymm26, %ymm30
+
+// CHECK: vpmaddubsw %ymm26, %ymm26, %ymm30 {%k5}
+// CHECK:  encoding: [0x62,0x02,0x2d,0x25,0x04,0xf2]
+          vpmaddubsw %ymm26, %ymm26, %ymm30 {%k5}
+
+// CHECK: vpmaddubsw %ymm26, %ymm26, %ymm30 {%k5} {z}
+// CHECK:  encoding: [0x62,0x02,0x2d,0xa5,0x04,0xf2]
+          vpmaddubsw %ymm26, %ymm26, %ymm30 {%k5} {z}
+
+// CHECK: vpmaddubsw (%rcx), %ymm26, %ymm30
+// CHECK:  encoding: [0x62,0x62,0x2d,0x20,0x04,0x31]
+          vpmaddubsw (%rcx), %ymm26, %ymm30
+
+// CHECK: vpmaddubsw 291(%rax,%r14,8), %ymm26, %ymm30
+// CHECK:  encoding: [0x62,0x22,0x2d,0x20,0x04,0xb4,0xf0,0x23,0x01,0x00,0x00]
+          vpmaddubsw 291(%rax,%r14,8), %ymm26, %ymm30
+
+// CHECK: vpmaddubsw 4064(%rdx), %ymm26, %ymm30
+// CHECK:  encoding: [0x62,0x62,0x2d,0x20,0x04,0x72,0x7f]
+          vpmaddubsw 4064(%rdx), %ymm26, %ymm30
+
+// CHECK: vpmaddubsw 4096(%rdx), %ymm26, %ymm30
+// CHECK:  encoding: [0x62,0x62,0x2d,0x20,0x04,0xb2,0x00,0x10,0x00,0x00]
+          vpmaddubsw 4096(%rdx), %ymm26, %ymm30
+
+// CHECK: vpmaddubsw -4096(%rdx), %ymm26, %ymm30
+// CHECK:  encoding: [0x62,0x62,0x2d,0x20,0x04,0x72,0x80]
+          vpmaddubsw -4096(%rdx), %ymm26, %ymm30
+
+// CHECK: vpmaddubsw -4128(%rdx), %ymm26, %ymm30
+// CHECK:  encoding: [0x62,0x62,0x2d,0x20,0x04,0xb2,0xe0,0xef,0xff,0xff]
+          vpmaddubsw -4128(%rdx), %ymm26, %ymm30
+
+// CHECK: vpmaddwd %xmm28, %xmm24, %xmm17
+// CHECK:  encoding: [0x62,0x81,0x3d,0x00,0xf5,0xcc]
+          vpmaddwd %xmm28, %xmm24, %xmm17
+
+// CHECK: vpmaddwd %xmm28, %xmm24, %xmm17 {%k1}
+// CHECK:  encoding: [0x62,0x81,0x3d,0x01,0xf5,0xcc]
+          vpmaddwd %xmm28, %xmm24, %xmm17 {%k1}
+
+// CHECK: vpmaddwd %xmm28, %xmm24, %xmm17 {%k1} {z}
+// CHECK:  encoding: [0x62,0x81,0x3d,0x81,0xf5,0xcc]
+          vpmaddwd %xmm28, %xmm24, %xmm17 {%k1} {z}
+
+// CHECK: vpmaddwd (%rcx), %xmm24, %xmm17
+// CHECK:  encoding: [0x62,0xe1,0x3d,0x00,0xf5,0x09]
+          vpmaddwd (%rcx), %xmm24, %xmm17
+
+// CHECK: vpmaddwd 291(%rax,%r14,8), %xmm24, %xmm17
+// CHECK:  encoding: [0x62,0xa1,0x3d,0x00,0xf5,0x8c,0xf0,0x23,0x01,0x00,0x00]
+          vpmaddwd 291(%rax,%r14,8), %xmm24, %xmm17
+
+// CHECK: vpmaddwd 2032(%rdx), %xmm24, %xmm17
+// CHECK:  encoding: [0x62,0xe1,0x3d,0x00,0xf5,0x4a,0x7f]
+          vpmaddwd 2032(%rdx), %xmm24, %xmm17
+
+// CHECK: vpmaddwd 2048(%rdx), %xmm24, %xmm17
+// CHECK:  encoding: [0x62,0xe1,0x3d,0x00,0xf5,0x8a,0x00,0x08,0x00,0x00]
+          vpmaddwd 2048(%rdx), %xmm24, %xmm17
+
+// CHECK: vpmaddwd -2048(%rdx), %xmm24, %xmm17
+// CHECK:  encoding: [0x62,0xe1,0x3d,0x00,0xf5,0x4a,0x80]
+          vpmaddwd -2048(%rdx), %xmm24, %xmm17
+
+// CHECK: vpmaddwd -2064(%rdx), %xmm24, %xmm17
+// CHECK:  encoding: [0x62,0xe1,0x3d,0x00,0xf5,0x8a,0xf0,0xf7,0xff,0xff]
+          vpmaddwd -2064(%rdx), %xmm24, %xmm17
+
+// CHECK: vpmaddwd %ymm19, %ymm23, %ymm24
+// CHECK:  encoding: [0x62,0x21,0x45,0x20,0xf5,0xc3]
+          vpmaddwd %ymm19, %ymm23, %ymm24
+
+// CHECK: vpmaddwd %ymm19, %ymm23, %ymm24 {%k4}
+// CHECK:  encoding: [0x62,0x21,0x45,0x24,0xf5,0xc3]
+          vpmaddwd %ymm19, %ymm23, %ymm24 {%k4}
+
+// CHECK: vpmaddwd %ymm19, %ymm23, %ymm24 {%k4} {z}
+// CHECK:  encoding: [0x62,0x21,0x45,0xa4,0xf5,0xc3]
+          vpmaddwd %ymm19, %ymm23, %ymm24 {%k4} {z}
+
+// CHECK: vpmaddwd (%rcx), %ymm23, %ymm24
+// CHECK:  encoding: [0x62,0x61,0x45,0x20,0xf5,0x01]
+          vpmaddwd (%rcx), %ymm23, %ymm24
+
+// CHECK: vpmaddwd 291(%rax,%r14,8), %ymm23, %ymm24
+// CHECK:  encoding: [0x62,0x21,0x45,0x20,0xf5,0x84,0xf0,0x23,0x01,0x00,0x00]
+          vpmaddwd 291(%rax,%r14,8), %ymm23, %ymm24
+
+// CHECK: vpmaddwd 4064(%rdx), %ymm23, %ymm24
+// CHECK:  encoding: [0x62,0x61,0x45,0x20,0xf5,0x42,0x7f]
+          vpmaddwd 4064(%rdx), %ymm23, %ymm24
+
+// CHECK: vpmaddwd 4096(%rdx), %ymm23, %ymm24
+// CHECK:  encoding: [0x62,0x61,0x45,0x20,0xf5,0x82,0x00,0x10,0x00,0x00]
+          vpmaddwd 4096(%rdx), %ymm23, %ymm24
+
+// CHECK: vpmaddwd -4096(%rdx), %ymm23, %ymm24
+// CHECK:  encoding: [0x62,0x61,0x45,0x20,0xf5,0x42,0x80]
+          vpmaddwd -4096(%rdx), %ymm23, %ymm24
+
+// CHECK: vpmaddwd -4128(%rdx), %ymm23, %ymm24
+// CHECK:  encoding: [0x62,0x61,0x45,0x20,0xf5,0x82,0xe0,0xef,0xff,0xff]
+          vpmaddwd -4128(%rdx), %ymm23, %ymm24
+
+// CHECK: vpmaddubsw %xmm25, %xmm23, %xmm19
+// CHECK:  encoding: [0x62,0x82,0x45,0x00,0x04,0xd9]
+          vpmaddubsw %xmm25, %xmm23, %xmm19
+
+// CHECK: vpmaddubsw %xmm25, %xmm23, %xmm19 {%k2}
+// CHECK:  encoding: [0x62,0x82,0x45,0x02,0x04,0xd9]
+          vpmaddubsw %xmm25, %xmm23, %xmm19 {%k2}
+
+// CHECK: vpmaddubsw %xmm25, %xmm23, %xmm19 {%k2} {z}
+// CHECK:  encoding: [0x62,0x82,0x45,0x82,0x04,0xd9]
+          vpmaddubsw %xmm25, %xmm23, %xmm19 {%k2} {z}
+
+// CHECK: vpmaddubsw (%rcx), %xmm23, %xmm19
+// CHECK:  encoding: [0x62,0xe2,0x45,0x00,0x04,0x19]
+          vpmaddubsw (%rcx), %xmm23, %xmm19
+
+// CHECK: vpmaddubsw 4660(%rax,%r14,8), %xmm23, %xmm19
+// CHECK:  encoding: [0x62,0xa2,0x45,0x00,0x04,0x9c,0xf0,0x34,0x12,0x00,0x00]
+          vpmaddubsw 4660(%rax,%r14,8), %xmm23, %xmm19
+
+// CHECK: vpmaddubsw 2032(%rdx), %xmm23, %xmm19
+// CHECK:  encoding: [0x62,0xe2,0x45,0x00,0x04,0x5a,0x7f]
+          vpmaddubsw 2032(%rdx), %xmm23, %xmm19
+
+// CHECK: vpmaddubsw 2048(%rdx), %xmm23, %xmm19
+// CHECK:  encoding: [0x62,0xe2,0x45,0x00,0x04,0x9a,0x00,0x08,0x00,0x00]
+          vpmaddubsw 2048(%rdx), %xmm23, %xmm19
+
+// CHECK: vpmaddubsw -2048(%rdx), %xmm23, %xmm19
+// CHECK:  encoding: [0x62,0xe2,0x45,0x00,0x04,0x5a,0x80]
+          vpmaddubsw -2048(%rdx), %xmm23, %xmm19
+
+// CHECK: vpmaddubsw -2064(%rdx), %xmm23, %xmm19
+// CHECK:  encoding: [0x62,0xe2,0x45,0x00,0x04,0x9a,0xf0,0xf7,0xff,0xff]
+          vpmaddubsw -2064(%rdx), %xmm23, %xmm19
+
+// CHECK: vpmaddubsw %ymm22, %ymm19, %ymm17
+// CHECK:  encoding: [0x62,0xa2,0x65,0x20,0x04,0xce]
+          vpmaddubsw %ymm22, %ymm19, %ymm17
+
+// CHECK: vpmaddubsw %ymm22, %ymm19, %ymm17 {%k7}
+// CHECK:  encoding: [0x62,0xa2,0x65,0x27,0x04,0xce]
+          vpmaddubsw %ymm22, %ymm19, %ymm17 {%k7}
+
+// CHECK: vpmaddubsw %ymm22, %ymm19, %ymm17 {%k7} {z}
+// CHECK:  encoding: [0x62,0xa2,0x65,0xa7,0x04,0xce]
+          vpmaddubsw %ymm22, %ymm19, %ymm17 {%k7} {z}
+
+// CHECK: vpmaddubsw (%rcx), %ymm19, %ymm17
+// CHECK:  encoding: [0x62,0xe2,0x65,0x20,0x04,0x09]
+          vpmaddubsw (%rcx), %ymm19, %ymm17
+
+// CHECK: vpmaddubsw 4660(%rax,%r14,8), %ymm19, %ymm17
+// CHECK:  encoding: [0x62,0xa2,0x65,0x20,0x04,0x8c,0xf0,0x34,0x12,0x00,0x00]
+          vpmaddubsw 4660(%rax,%r14,8), %ymm19, %ymm17
+
+// CHECK: vpmaddubsw 4064(%rdx), %ymm19, %ymm17
+// CHECK:  encoding: [0x62,0xe2,0x65,0x20,0x04,0x4a,0x7f]
+          vpmaddubsw 4064(%rdx), %ymm19, %ymm17
+
+// CHECK: vpmaddubsw 4096(%rdx), %ymm19, %ymm17
+// CHECK:  encoding: [0x62,0xe2,0x65,0x20,0x04,0x8a,0x00,0x10,0x00,0x00]
+          vpmaddubsw 4096(%rdx), %ymm19, %ymm17
+
+// CHECK: vpmaddubsw -4096(%rdx), %ymm19, %ymm17
+// CHECK:  encoding: [0x62,0xe2,0x65,0x20,0x04,0x4a,0x80]
+          vpmaddubsw -4096(%rdx), %ymm19, %ymm17
+
+// CHECK: vpmaddubsw -4128(%rdx), %ymm19, %ymm17
+// CHECK:  encoding: [0x62,0xe2,0x65,0x20,0x04,0x8a,0xe0,0xef,0xff,0xff]
+          vpmaddubsw -4128(%rdx), %ymm19, %ymm17
+
+// CHECK: vpmaddwd %xmm20, %xmm22, %xmm23
+// CHECK:  encoding: [0x62,0xa1,0x4d,0x00,0xf5,0xfc]
+          vpmaddwd %xmm20, %xmm22, %xmm23
+
+// CHECK: vpmaddwd %xmm20, %xmm22, %xmm23 {%k3}
+// CHECK:  encoding: [0x62,0xa1,0x4d,0x03,0xf5,0xfc]
+          vpmaddwd %xmm20, %xmm22, %xmm23 {%k3}
+
+// CHECK: vpmaddwd %xmm20, %xmm22, %xmm23 {%k3} {z}
+// CHECK:  encoding: [0x62,0xa1,0x4d,0x83,0xf5,0xfc]
+          vpmaddwd %xmm20, %xmm22, %xmm23 {%k3} {z}
+
+// CHECK: vpmaddwd (%rcx), %xmm22, %xmm23
+// CHECK:  encoding: [0x62,0xe1,0x4d,0x00,0xf5,0x39]
+          vpmaddwd (%rcx), %xmm22, %xmm23
+
+// CHECK: vpmaddwd 4660(%rax,%r14,8), %xmm22, %xmm23
+// CHECK:  encoding: [0x62,0xa1,0x4d,0x00,0xf5,0xbc,0xf0,0x34,0x12,0x00,0x00]
+          vpmaddwd 4660(%rax,%r14,8), %xmm22, %xmm23
+
+// CHECK: vpmaddwd 2032(%rdx), %xmm22, %xmm23
+// CHECK:  encoding: [0x62,0xe1,0x4d,0x00,0xf5,0x7a,0x7f]
+          vpmaddwd 2032(%rdx), %xmm22, %xmm23
+
+// CHECK: vpmaddwd 2048(%rdx), %xmm22, %xmm23
+// CHECK:  encoding: [0x62,0xe1,0x4d,0x00,0xf5,0xba,0x00,0x08,0x00,0x00]
+          vpmaddwd 2048(%rdx), %xmm22, %xmm23
+
+// CHECK: vpmaddwd -2048(%rdx), %xmm22, %xmm23
+// CHECK:  encoding: [0x62,0xe1,0x4d,0x00,0xf5,0x7a,0x80]
+          vpmaddwd -2048(%rdx), %xmm22, %xmm23
+
+// CHECK: vpmaddwd -2064(%rdx), %xmm22, %xmm23
+// CHECK:  encoding: [0x62,0xe1,0x4d,0x00,0xf5,0xba,0xf0,0xf7,0xff,0xff]
+          vpmaddwd -2064(%rdx), %xmm22, %xmm23
+
+// CHECK: vpmaddwd %ymm17, %ymm20, %ymm19
+// CHECK:  encoding: [0x62,0xa1,0x5d,0x20,0xf5,0xd9]
+          vpmaddwd %ymm17, %ymm20, %ymm19
+
+// CHECK: vpmaddwd %ymm17, %ymm20, %ymm19 {%k2}
+// CHECK:  encoding: [0x62,0xa1,0x5d,0x22,0xf5,0xd9]
+          vpmaddwd %ymm17, %ymm20, %ymm19 {%k2}
+
+// CHECK: vpmaddwd %ymm17, %ymm20, %ymm19 {%k2} {z}
+// CHECK:  encoding: [0x62,0xa1,0x5d,0xa2,0xf5,0xd9]
+          vpmaddwd %ymm17, %ymm20, %ymm19 {%k2} {z}
+
+// CHECK: vpmaddwd (%rcx), %ymm20, %ymm19
+// CHECK:  encoding: [0x62,0xe1,0x5d,0x20,0xf5,0x19]
+          vpmaddwd (%rcx), %ymm20, %ymm19
+
+// CHECK: vpmaddwd 4660(%rax,%r14,8), %ymm20, %ymm19
+// CHECK:  encoding: [0x62,0xa1,0x5d,0x20,0xf5,0x9c,0xf0,0x34,0x12,0x00,0x00]
+          vpmaddwd 4660(%rax,%r14,8), %ymm20, %ymm19
+
+// CHECK: vpmaddwd 4064(%rdx), %ymm20, %ymm19
+// CHECK:  encoding: [0x62,0xe1,0x5d,0x20,0xf5,0x5a,0x7f]
+          vpmaddwd 4064(%rdx), %ymm20, %ymm19
+
+// CHECK: vpmaddwd 4096(%rdx), %ymm20, %ymm19
+// CHECK:  encoding: [0x62,0xe1,0x5d,0x20,0xf5,0x9a,0x00,0x10,0x00,0x00]
+          vpmaddwd 4096(%rdx), %ymm20, %ymm19
+
+// CHECK: vpmaddwd -4096(%rdx), %ymm20, %ymm19
+// CHECK:  encoding: [0x62,0xe1,0x5d,0x20,0xf5,0x5a,0x80]
+          vpmaddwd -4096(%rdx), %ymm20, %ymm19
+
+// CHECK: vpmaddwd -4128(%rdx), %ymm20, %ymm19
+// CHECK:  encoding: [0x62,0xe1,0x5d,0x20,0xf5,0x9a,0xe0,0xef,0xff,0xff]
+          vpmaddwd -4128(%rdx), %ymm20, %ymm19
+
+// CHECK: vptestnmw %xmm24, %xmm29, %k4
+// CHECK:  encoding: [0x62,0x92,0x96,0x00,0x26,0xe0]
+          vptestnmw %xmm24, %xmm29, %k4
+
+// CHECK: vptestnmw %xmm24, %xmm29, %k4 {%k5}
+// CHECK:  encoding: [0x62,0x92,0x96,0x05,0x26,0xe0]
+          vptestnmw %xmm24, %xmm29, %k4 {%k5}
+
+// CHECK: vptestnmw (%rcx), %xmm29, %k4
+// CHECK:  encoding: [0x62,0xf2,0x96,0x00,0x26,0x21]
+          vptestnmw (%rcx), %xmm29, %k4
+
+// CHECK: vptestnmw 291(%rax,%r14,8), %xmm29, %k4
+// CHECK:  encoding: [0x62,0xb2,0x96,0x00,0x26,0xa4,0xf0,0x23,0x01,0x00,0x00]
+          vptestnmw 291(%rax,%r14,8), %xmm29, %k4
+
+// CHECK: vptestnmw 2032(%rdx), %xmm29, %k4
+// CHECK:  encoding: [0x62,0xf2,0x96,0x00,0x26,0x62,0x7f]
+          vptestnmw 2032(%rdx), %xmm29, %k4
+
+// CHECK: vptestnmw 2048(%rdx), %xmm29, %k4
+// CHECK:  encoding: [0x62,0xf2,0x96,0x00,0x26,0xa2,0x00,0x08,0x00,0x00]
+          vptestnmw 2048(%rdx), %xmm29, %k4
+
+// CHECK: vptestnmw -2048(%rdx), %xmm29, %k4
+// CHECK:  encoding: [0x62,0xf2,0x96,0x00,0x26,0x62,0x80]
+          vptestnmw -2048(%rdx), %xmm29, %k4
+
+// CHECK: vptestnmw -2064(%rdx), %xmm29, %k4
+// CHECK:  encoding: [0x62,0xf2,0x96,0x00,0x26,0xa2,0xf0,0xf7,0xff,0xff]
+          vptestnmw -2064(%rdx), %xmm29, %k4
+
+// CHECK: vptestnmw %ymm17, %ymm28, %k2
+// CHECK:  encoding: [0x62,0xb2,0x9e,0x20,0x26,0xd1]
+          vptestnmw %ymm17, %ymm28, %k2
+
+// CHECK: vptestnmw %ymm17, %ymm28, %k2 {%k2}
+// CHECK:  encoding: [0x62,0xb2,0x9e,0x22,0x26,0xd1]
+          vptestnmw %ymm17, %ymm28, %k2 {%k2}
+
+// CHECK: vptestnmw (%rcx), %ymm28, %k2
+// CHECK:  encoding: [0x62,0xf2,0x9e,0x20,0x26,0x11]
+          vptestnmw (%rcx), %ymm28, %k2
+
+// CHECK: vptestnmw 291(%rax,%r14,8), %ymm28, %k2
+// CHECK:  encoding: [0x62,0xb2,0x9e,0x20,0x26,0x94,0xf0,0x23,0x01,0x00,0x00]
+          vptestnmw 291(%rax,%r14,8), %ymm28, %k2
+
+// CHECK: vptestnmw 4064(%rdx), %ymm28, %k2
+// CHECK:  encoding: [0x62,0xf2,0x9e,0x20,0x26,0x52,0x7f]
+          vptestnmw 4064(%rdx), %ymm28, %k2
+
+// CHECK: vptestnmw 4096(%rdx), %ymm28, %k2
+// CHECK:  encoding: [0x62,0xf2,0x9e,0x20,0x26,0x92,0x00,0x10,0x00,0x00]
+          vptestnmw 4096(%rdx), %ymm28, %k2
+
+// CHECK: vptestnmw -4096(%rdx), %ymm28, %k2
+// CHECK:  encoding: [0x62,0xf2,0x9e,0x20,0x26,0x52,0x80]
+          vptestnmw -4096(%rdx), %ymm28, %k2
+
+// CHECK: vptestnmw -4128(%rdx), %ymm28, %k2
+// CHECK:  encoding: [0x62,0xf2,0x9e,0x20,0x26,0x92,0xe0,0xef,0xff,0xff]
+          vptestnmw -4128(%rdx), %ymm28, %k2
+
+// CHECK: vptestnmw %xmm21, %xmm27, %k2
+// CHECK:  encoding: [0x62,0xb2,0xa6,0x00,0x26,0xd5]
+          vptestnmw %xmm21, %xmm27, %k2
+
+// CHECK: vptestnmw %xmm21, %xmm27, %k2 {%k3}
+// CHECK:  encoding: [0x62,0xb2,0xa6,0x03,0x26,0xd5]
+          vptestnmw %xmm21, %xmm27, %k2 {%k3}
+
+// CHECK: vptestnmw (%rcx), %xmm27, %k2
+// CHECK:  encoding: [0x62,0xf2,0xa6,0x00,0x26,0x11]
+          vptestnmw (%rcx), %xmm27, %k2
+
+// CHECK: vptestnmw 4660(%rax,%r14,8), %xmm27, %k2
+// CHECK:  encoding: [0x62,0xb2,0xa6,0x00,0x26,0x94,0xf0,0x34,0x12,0x00,0x00]
+          vptestnmw 4660(%rax,%r14,8), %xmm27, %k2
+
+// CHECK: vptestnmw 2032(%rdx), %xmm27, %k2
+// CHECK:  encoding: [0x62,0xf2,0xa6,0x00,0x26,0x52,0x7f]
+          vptestnmw 2032(%rdx), %xmm27, %k2
+
+// CHECK: vptestnmw 2048(%rdx), %xmm27, %k2
+// CHECK:  encoding: [0x62,0xf2,0xa6,0x00,0x26,0x92,0x00,0x08,0x00,0x00]
+          vptestnmw 2048(%rdx), %xmm27, %k2
+
+// CHECK: vptestnmw -2048(%rdx), %xmm27, %k2
+// CHECK:  encoding: [0x62,0xf2,0xa6,0x00,0x26,0x52,0x80]
+          vptestnmw -2048(%rdx), %xmm27, %k2
+
+// CHECK: vptestnmw -2064(%rdx), %xmm27, %k2
+// CHECK:  encoding: [0x62,0xf2,0xa6,0x00,0x26,0x92,0xf0,0xf7,0xff,0xff]
+          vptestnmw -2064(%rdx), %xmm27, %k2
+
+// CHECK: vptestnmw %ymm23, %ymm19, %k4
+// CHECK:  encoding: [0x62,0xb2,0xe6,0x20,0x26,0xe7]
+          vptestnmw %ymm23, %ymm19, %k4
+
+// CHECK: vptestnmw %ymm23, %ymm19, %k4 {%k1}
+// CHECK:  encoding: [0x62,0xb2,0xe6,0x21,0x26,0xe7]
+          vptestnmw %ymm23, %ymm19, %k4 {%k1}
+
+// CHECK: vptestnmw (%rcx), %ymm19, %k4
+// CHECK:  encoding: [0x62,0xf2,0xe6,0x20,0x26,0x21]
+          vptestnmw (%rcx), %ymm19, %k4
+
+// CHECK: vptestnmw 4660(%rax,%r14,8), %ymm19, %k4
+// CHECK:  encoding: [0x62,0xb2,0xe6,0x20,0x26,0xa4,0xf0,0x34,0x12,0x00,0x00]
+          vptestnmw 4660(%rax,%r14,8), %ymm19, %k4
+
+// CHECK: vptestnmw 4064(%rdx), %ymm19, %k4
+// CHECK:  encoding: [0x62,0xf2,0xe6,0x20,0x26,0x62,0x7f]
+          vptestnmw 4064(%rdx), %ymm19, %k4
+
+// CHECK: vptestnmw 4096(%rdx), %ymm19, %k4
+// CHECK:  encoding: [0x62,0xf2,0xe6,0x20,0x26,0xa2,0x00,0x10,0x00,0x00]
+          vptestnmw 4096(%rdx), %ymm19, %k4
+
+// CHECK: vptestnmw -4096(%rdx), %ymm19, %k4
+// CHECK:  encoding: [0x62,0xf2,0xe6,0x20,0x26,0x62,0x80]
+          vptestnmw -4096(%rdx), %ymm19, %k4
+
+// CHECK: vptestnmw -4128(%rdx), %ymm19, %k4
+// CHECK:  encoding: [0x62,0xf2,0xe6,0x20,0x26,0xa2,0xe0,0xef,0xff,0xff]
+          vptestnmw -4128(%rdx), %ymm19, %k4
+
+// CHECK: vptestnmb %xmm22, %xmm27, %k4
+// CHECK:  encoding: [0x62,0xb2,0x26,0x00,0x26,0xe6]
+          vptestnmb %xmm22, %xmm27, %k4
+
+// CHECK: vptestnmb %xmm22, %xmm27, %k4 {%k1}
+// CHECK:  encoding: [0x62,0xb2,0x26,0x01,0x26,0xe6]
+          vptestnmb %xmm22, %xmm27, %k4 {%k1}
+
+// CHECK: vptestnmb (%rcx), %xmm27, %k4
+// CHECK:  encoding: [0x62,0xf2,0x26,0x00,0x26,0x21]
+          vptestnmb (%rcx), %xmm27, %k4
+
+// CHECK: vptestnmb 291(%rax,%r14,8), %xmm27, %k4
+// CHECK:  encoding: [0x62,0xb2,0x26,0x00,0x26,0xa4,0xf0,0x23,0x01,0x00,0x00]
+          vptestnmb 291(%rax,%r14,8), %xmm27, %k4
+
+// CHECK: vptestnmb 2032(%rdx), %xmm27, %k4
+// CHECK:  encoding: [0x62,0xf2,0x26,0x00,0x26,0x62,0x7f]
+          vptestnmb 2032(%rdx), %xmm27, %k4
+
+// CHECK: vptestnmb 2048(%rdx), %xmm27, %k4
+// CHECK:  encoding: [0x62,0xf2,0x26,0x00,0x26,0xa2,0x00,0x08,0x00,0x00]
+          vptestnmb 2048(%rdx), %xmm27, %k4
+
+// CHECK: vptestnmb -2048(%rdx), %xmm27, %k4
+// CHECK:  encoding: [0x62,0xf2,0x26,0x00,0x26,0x62,0x80]
+          vptestnmb -2048(%rdx), %xmm27, %k4
+
+// CHECK: vptestnmb -2064(%rdx), %xmm27, %k4
+// CHECK:  encoding: [0x62,0xf2,0x26,0x00,0x26,0xa2,0xf0,0xf7,0xff,0xff]
+          vptestnmb -2064(%rdx), %xmm27, %k4
+
+// CHECK: vptestnmb %ymm17, %ymm25, %k5
+// CHECK:  encoding: [0x62,0xb2,0x36,0x20,0x26,0xe9]
+          vptestnmb %ymm17, %ymm25, %k5
+
+// CHECK: vptestnmb %ymm17, %ymm25, %k5 {%k6}
+// CHECK:  encoding: [0x62,0xb2,0x36,0x26,0x26,0xe9]
+          vptestnmb %ymm17, %ymm25, %k5 {%k6}
+
+// CHECK: vptestnmb (%rcx), %ymm25, %k5
+// CHECK:  encoding: [0x62,0xf2,0x36,0x20,0x26,0x29]
+          vptestnmb (%rcx), %ymm25, %k5
+
+// CHECK: vptestnmb 291(%rax,%r14,8), %ymm25, %k5
+// CHECK:  encoding: [0x62,0xb2,0x36,0x20,0x26,0xac,0xf0,0x23,0x01,0x00,0x00]
+          vptestnmb 291(%rax,%r14,8), %ymm25, %k5
+
+// CHECK: vptestnmb 4064(%rdx), %ymm25, %k5
+// CHECK:  encoding: [0x62,0xf2,0x36,0x20,0x26,0x6a,0x7f]
+          vptestnmb 4064(%rdx), %ymm25, %k5
+
+// CHECK: vptestnmb 4096(%rdx), %ymm25, %k5
+// CHECK:  encoding: [0x62,0xf2,0x36,0x20,0x26,0xaa,0x00,0x10,0x00,0x00]
+          vptestnmb 4096(%rdx), %ymm25, %k5
+
+// CHECK: vptestnmb -4096(%rdx), %ymm25, %k5
+// CHECK:  encoding: [0x62,0xf2,0x36,0x20,0x26,0x6a,0x80]
+          vptestnmb -4096(%rdx), %ymm25, %k5
+
+// CHECK: vptestnmb -4128(%rdx), %ymm25, %k5
+// CHECK:  encoding: [0x62,0xf2,0x36,0x20,0x26,0xaa,0xe0,0xef,0xff,0xff]
+          vptestnmb -4128(%rdx), %ymm25, %k5
+
+// CHECK: vptestnmb %xmm19, %xmm27, %k4
+// CHECK:  encoding: [0x62,0xb2,0x26,0x00,0x26,0xe3]
+          vptestnmb %xmm19, %xmm27, %k4
+
+// CHECK: vptestnmb %xmm19, %xmm27, %k4 {%k3}
+// CHECK:  encoding: [0x62,0xb2,0x26,0x03,0x26,0xe3]
+          vptestnmb %xmm19, %xmm27, %k4 {%k3}
+
+// CHECK: vptestnmb (%rcx), %xmm27, %k4
+// CHECK:  encoding: [0x62,0xf2,0x26,0x00,0x26,0x21]
+          vptestnmb (%rcx), %xmm27, %k4
+
+// CHECK: vptestnmb 4660(%rax,%r14,8), %xmm27, %k4
+// CHECK:  encoding: [0x62,0xb2,0x26,0x00,0x26,0xa4,0xf0,0x34,0x12,0x00,0x00]
+          vptestnmb 4660(%rax,%r14,8), %xmm27, %k4
+
+// CHECK: vptestnmb 2032(%rdx), %xmm27, %k4
+// CHECK:  encoding: [0x62,0xf2,0x26,0x00,0x26,0x62,0x7f]
+          vptestnmb 2032(%rdx), %xmm27, %k4
+
+// CHECK: vptestnmb 2048(%rdx), %xmm27, %k4
+// CHECK:  encoding: [0x62,0xf2,0x26,0x00,0x26,0xa2,0x00,0x08,0x00,0x00]
+          vptestnmb 2048(%rdx), %xmm27, %k4
+
+// CHECK: vptestnmb -2048(%rdx), %xmm27, %k4
+// CHECK:  encoding: [0x62,0xf2,0x26,0x00,0x26,0x62,0x80]
+          vptestnmb -2048(%rdx), %xmm27, %k4
+
+// CHECK: vptestnmb -2064(%rdx), %xmm27, %k4
+// CHECK:  encoding: [0x62,0xf2,0x26,0x00,0x26,0xa2,0xf0,0xf7,0xff,0xff]
+          vptestnmb -2064(%rdx), %xmm27, %k4
+
+// CHECK: vptestnmb %ymm24, %ymm28, %k4
+// CHECK:  encoding: [0x62,0x92,0x1e,0x20,0x26,0xe0]
+          vptestnmb %ymm24, %ymm28, %k4
+
+// CHECK: vptestnmb %ymm24, %ymm28, %k4 {%k1}
+// CHECK:  encoding: [0x62,0x92,0x1e,0x21,0x26,0xe0]
+          vptestnmb %ymm24, %ymm28, %k4 {%k1}
+
+// CHECK: vptestnmb (%rcx), %ymm28, %k4
+// CHECK:  encoding: [0x62,0xf2,0x1e,0x20,0x26,0x21]
+          vptestnmb (%rcx), %ymm28, %k4
+
+// CHECK: vptestnmb 4660(%rax,%r14,8), %ymm28, %k4
+// CHECK:  encoding: [0x62,0xb2,0x1e,0x20,0x26,0xa4,0xf0,0x34,0x12,0x00,0x00]
+          vptestnmb 4660(%rax,%r14,8), %ymm28, %k4
+
+// CHECK: vptestnmb 4064(%rdx), %ymm28, %k4
+// CHECK:  encoding: [0x62,0xf2,0x1e,0x20,0x26,0x62,0x7f]
+          vptestnmb 4064(%rdx), %ymm28, %k4
+
+// CHECK: vptestnmb 4096(%rdx), %ymm28, %k4
+// CHECK:  encoding: [0x62,0xf2,0x1e,0x20,0x26,0xa2,0x00,0x10,0x00,0x00]
+          vptestnmb 4096(%rdx), %ymm28, %k4
+
+// CHECK: vptestnmb -4096(%rdx), %ymm28, %k4
+// CHECK:  encoding: [0x62,0xf2,0x1e,0x20,0x26,0x62,0x80]
+          vptestnmb -4096(%rdx), %ymm28, %k4
+
+// CHECK: vptestnmb -4128(%rdx), %ymm28, %k4
+// CHECK:  encoding: [0x62,0xf2,0x1e,0x20,0x26,0xa2,0xe0,0xef,0xff,0xff]
+          vptestnmb -4128(%rdx), %ymm28, %k4
+
+// CHECK: vpunpcklbw %xmm20, %xmm19, %xmm30
+// CHECK:  encoding: [0x62,0x21,0x65,0x00,0x60,0xf4]
+          vpunpcklbw %xmm20, %xmm19, %xmm30
+
+// CHECK: vpunpcklbw %xmm20, %xmm19, %xmm30 {%k4}
+// CHECK:  encoding: [0x62,0x21,0x65,0x04,0x60,0xf4]
+          vpunpcklbw %xmm20, %xmm19, %xmm30 {%k4}
+
+// CHECK: vpunpcklbw %xmm20, %xmm19, %xmm30 {%k4} {z}
+// CHECK:  encoding: [0x62,0x21,0x65,0x84,0x60,0xf4]
+          vpunpcklbw %xmm20, %xmm19, %xmm30 {%k4} {z}
+
+// CHECK: vpunpcklbw (%rcx), %xmm19, %xmm30
+// CHECK:  encoding: [0x62,0x61,0x65,0x00,0x60,0x31]
+          vpunpcklbw (%rcx), %xmm19, %xmm30
+
+// CHECK: vpunpcklbw 4660(%rax,%r14,8), %xmm19, %xmm30
+// CHECK:  encoding: [0x62,0x21,0x65,0x00,0x60,0xb4,0xf0,0x34,0x12,0x00,0x00]
+          vpunpcklbw 4660(%rax,%r14,8), %xmm19, %xmm30
+
+// CHECK: vpunpcklbw 2032(%rdx), %xmm19, %xmm30
+// CHECK:  encoding: [0x62,0x61,0x65,0x00,0x60,0x72,0x7f]
+          vpunpcklbw 2032(%rdx), %xmm19, %xmm30
+
+// CHECK: vpunpcklbw 2048(%rdx), %xmm19, %xmm30
+// CHECK:  encoding: [0x62,0x61,0x65,0x00,0x60,0xb2,0x00,0x08,0x00,0x00]
+          vpunpcklbw 2048(%rdx), %xmm19, %xmm30
+
+// CHECK: vpunpcklbw -2048(%rdx), %xmm19, %xmm30
+// CHECK:  encoding: [0x62,0x61,0x65,0x00,0x60,0x72,0x80]
+          vpunpcklbw -2048(%rdx), %xmm19, %xmm30
+
+// CHECK: vpunpcklbw -2064(%rdx), %xmm19, %xmm30
+// CHECK:  encoding: [0x62,0x61,0x65,0x00,0x60,0xb2,0xf0,0xf7,0xff,0xff]
+          vpunpcklbw -2064(%rdx), %xmm19, %xmm30
+
+// CHECK: vpunpcklbw %ymm22, %ymm28, %ymm20
+// CHECK:  encoding: [0x62,0xa1,0x1d,0x20,0x60,0xe6]
+          vpunpcklbw %ymm22, %ymm28, %ymm20
+
+// CHECK: vpunpcklbw %ymm22, %ymm28, %ymm20 {%k1}
+// CHECK:  encoding: [0x62,0xa1,0x1d,0x21,0x60,0xe6]
+          vpunpcklbw %ymm22, %ymm28, %ymm20 {%k1}
+
+// CHECK: vpunpcklbw %ymm22, %ymm28, %ymm20 {%k1} {z}
+// CHECK:  encoding: [0x62,0xa1,0x1d,0xa1,0x60,0xe6]
+          vpunpcklbw %ymm22, %ymm28, %ymm20 {%k1} {z}
+
+// CHECK: vpunpcklbw (%rcx), %ymm28, %ymm20
+// CHECK:  encoding: [0x62,0xe1,0x1d,0x20,0x60,0x21]
+          vpunpcklbw (%rcx), %ymm28, %ymm20
+
+// CHECK: vpunpcklbw 4660(%rax,%r14,8), %ymm28, %ymm20
+// CHECK:  encoding: [0x62,0xa1,0x1d,0x20,0x60,0xa4,0xf0,0x34,0x12,0x00,0x00]
+          vpunpcklbw 4660(%rax,%r14,8), %ymm28, %ymm20
+
+// CHECK: vpunpcklbw 4064(%rdx), %ymm28, %ymm20
+// CHECK:  encoding: [0x62,0xe1,0x1d,0x20,0x60,0x62,0x7f]
+          vpunpcklbw 4064(%rdx), %ymm28, %ymm20
+
+// CHECK: vpunpcklbw 4096(%rdx), %ymm28, %ymm20
+// CHECK:  encoding: [0x62,0xe1,0x1d,0x20,0x60,0xa2,0x00,0x10,0x00,0x00]
+          vpunpcklbw 4096(%rdx), %ymm28, %ymm20
+
+// CHECK: vpunpcklbw -4096(%rdx), %ymm28, %ymm20
+// CHECK:  encoding: [0x62,0xe1,0x1d,0x20,0x60,0x62,0x80]
+          vpunpcklbw -4096(%rdx), %ymm28, %ymm20
+
+// CHECK: vpunpcklbw -4128(%rdx), %ymm28, %ymm20
+// CHECK:  encoding: [0x62,0xe1,0x1d,0x20,0x60,0xa2,0xe0,0xef,0xff,0xff]
+          vpunpcklbw -4128(%rdx), %ymm28, %ymm20
+
+// CHECK: vpunpckhbw %xmm27, %xmm22, %xmm18
+// CHECK:  encoding: [0x62,0x81,0x4d,0x00,0x68,0xd3]
+          vpunpckhbw %xmm27, %xmm22, %xmm18
+
+// CHECK: vpunpckhbw %xmm27, %xmm22, %xmm18 {%k1}
+// CHECK:  encoding: [0x62,0x81,0x4d,0x01,0x68,0xd3]
+          vpunpckhbw %xmm27, %xmm22, %xmm18 {%k1}
+
+// CHECK: vpunpckhbw %xmm27, %xmm22, %xmm18 {%k1} {z}
+// CHECK:  encoding: [0x62,0x81,0x4d,0x81,0x68,0xd3]
+          vpunpckhbw %xmm27, %xmm22, %xmm18 {%k1} {z}
+
+// CHECK: vpunpckhbw (%rcx), %xmm22, %xmm18
+// CHECK:  encoding: [0x62,0xe1,0x4d,0x00,0x68,0x11]
+          vpunpckhbw (%rcx), %xmm22, %xmm18
+
+// CHECK: vpunpckhbw 4660(%rax,%r14,8), %xmm22, %xmm18
+// CHECK:  encoding: [0x62,0xa1,0x4d,0x00,0x68,0x94,0xf0,0x34,0x12,0x00,0x00]
+          vpunpckhbw 4660(%rax,%r14,8), %xmm22, %xmm18
+
+// CHECK: vpunpckhbw 2032(%rdx), %xmm22, %xmm18
+// CHECK:  encoding: [0x62,0xe1,0x4d,0x00,0x68,0x52,0x7f]
+          vpunpckhbw 2032(%rdx), %xmm22, %xmm18
+
+// CHECK: vpunpckhbw 2048(%rdx), %xmm22, %xmm18
+// CHECK:  encoding: [0x62,0xe1,0x4d,0x00,0x68,0x92,0x00,0x08,0x00,0x00]
+          vpunpckhbw 2048(%rdx), %xmm22, %xmm18
+
+// CHECK: vpunpckhbw -2048(%rdx), %xmm22, %xmm18
+// CHECK:  encoding: [0x62,0xe1,0x4d,0x00,0x68,0x52,0x80]
+          vpunpckhbw -2048(%rdx), %xmm22, %xmm18
+
+// CHECK: vpunpckhbw -2064(%rdx), %xmm22, %xmm18
+// CHECK:  encoding: [0x62,0xe1,0x4d,0x00,0x68,0x92,0xf0,0xf7,0xff,0xff]
+          vpunpckhbw -2064(%rdx), %xmm22, %xmm18
+
+// CHECK: vpunpckhbw %ymm24, %ymm20, %ymm17
+// CHECK:  encoding: [0x62,0x81,0x5d,0x20,0x68,0xc8]
+          vpunpckhbw %ymm24, %ymm20, %ymm17
+
+// CHECK: vpunpckhbw %ymm24, %ymm20, %ymm17 {%k5}
+// CHECK:  encoding: [0x62,0x81,0x5d,0x25,0x68,0xc8]
+          vpunpckhbw %ymm24, %ymm20, %ymm17 {%k5}
+
+// CHECK: vpunpckhbw %ymm24, %ymm20, %ymm17 {%k5} {z}
+// CHECK:  encoding: [0x62,0x81,0x5d,0xa5,0x68,0xc8]
+          vpunpckhbw %ymm24, %ymm20, %ymm17 {%k5} {z}
+
+// CHECK: vpunpckhbw (%rcx), %ymm20, %ymm17
+// CHECK:  encoding: [0x62,0xe1,0x5d,0x20,0x68,0x09]
+          vpunpckhbw (%rcx), %ymm20, %ymm17
+
+// CHECK: vpunpckhbw 4660(%rax,%r14,8), %ymm20, %ymm17
+// CHECK:  encoding: [0x62,0xa1,0x5d,0x20,0x68,0x8c,0xf0,0x34,0x12,0x00,0x00]
+          vpunpckhbw 4660(%rax,%r14,8), %ymm20, %ymm17
+
+// CHECK: vpunpckhbw 4064(%rdx), %ymm20, %ymm17
+// CHECK:  encoding: [0x62,0xe1,0x5d,0x20,0x68,0x4a,0x7f]
+          vpunpckhbw 4064(%rdx), %ymm20, %ymm17
+
+// CHECK: vpunpckhbw 4096(%rdx), %ymm20, %ymm17
+// CHECK:  encoding: [0x62,0xe1,0x5d,0x20,0x68,0x8a,0x00,0x10,0x00,0x00]
+          vpunpckhbw 4096(%rdx), %ymm20, %ymm17
+
+// CHECK: vpunpckhbw -4096(%rdx), %ymm20, %ymm17
+// CHECK:  encoding: [0x62,0xe1,0x5d,0x20,0x68,0x4a,0x80]
+          vpunpckhbw -4096(%rdx), %ymm20, %ymm17
+
+// CHECK: vpunpckhbw -4128(%rdx), %ymm20, %ymm17
+// CHECK:  encoding: [0x62,0xe1,0x5d,0x20,0x68,0x8a,0xe0,0xef,0xff,0xff]
+          vpunpckhbw -4128(%rdx), %ymm20, %ymm17
+
+// CHECK: vpunpcklwd %xmm17, %xmm27, %xmm27
+// CHECK:  encoding: [0x62,0x21,0x25,0x00,0x61,0xd9]
+          vpunpcklwd %xmm17, %xmm27, %xmm27
+
+// CHECK: vpunpcklwd %xmm17, %xmm27, %xmm27 {%k5}
+// CHECK:  encoding: [0x62,0x21,0x25,0x05,0x61,0xd9]
+          vpunpcklwd %xmm17, %xmm27, %xmm27 {%k5}
+
+// CHECK: vpunpcklwd %xmm17, %xmm27, %xmm27 {%k5} {z}
+// CHECK:  encoding: [0x62,0x21,0x25,0x85,0x61,0xd9]
+          vpunpcklwd %xmm17, %xmm27, %xmm27 {%k5} {z}
+
+// CHECK: vpunpcklwd (%rcx), %xmm27, %xmm27
+// CHECK:  encoding: [0x62,0x61,0x25,0x00,0x61,0x19]
+          vpunpcklwd (%rcx), %xmm27, %xmm27
+
+// CHECK: vpunpcklwd 4660(%rax,%r14,8), %xmm27, %xmm27
+// CHECK:  encoding: [0x62,0x21,0x25,0x00,0x61,0x9c,0xf0,0x34,0x12,0x00,0x00]
+          vpunpcklwd 4660(%rax,%r14,8), %xmm27, %xmm27
+
+// CHECK: vpunpcklwd 2032(%rdx), %xmm27, %xmm27
+// CHECK:  encoding: [0x62,0x61,0x25,0x00,0x61,0x5a,0x7f]
+          vpunpcklwd 2032(%rdx), %xmm27, %xmm27
+
+// CHECK: vpunpcklwd 2048(%rdx), %xmm27, %xmm27
+// CHECK:  encoding: [0x62,0x61,0x25,0x00,0x61,0x9a,0x00,0x08,0x00,0x00]
+          vpunpcklwd 2048(%rdx), %xmm27, %xmm27
+
+// CHECK: vpunpcklwd -2048(%rdx), %xmm27, %xmm27
+// CHECK:  encoding: [0x62,0x61,0x25,0x00,0x61,0x5a,0x80]
+          vpunpcklwd -2048(%rdx), %xmm27, %xmm27
+
+// CHECK: vpunpcklwd -2064(%rdx), %xmm27, %xmm27
+// CHECK:  encoding: [0x62,0x61,0x25,0x00,0x61,0x9a,0xf0,0xf7,0xff,0xff]
+          vpunpcklwd -2064(%rdx), %xmm27, %xmm27
+
+// CHECK: vpunpcklwd %ymm23, %ymm25, %ymm18
+// CHECK:  encoding: [0x62,0xa1,0x35,0x20,0x61,0xd7]
+          vpunpcklwd %ymm23, %ymm25, %ymm18
+
+// CHECK: vpunpcklwd %ymm23, %ymm25, %ymm18 {%k5}
+// CHECK:  encoding: [0x62,0xa1,0x35,0x25,0x61,0xd7]
+          vpunpcklwd %ymm23, %ymm25, %ymm18 {%k5}
+
+// CHECK: vpunpcklwd %ymm23, %ymm25, %ymm18 {%k5} {z}
+// CHECK:  encoding: [0x62,0xa1,0x35,0xa5,0x61,0xd7]
+          vpunpcklwd %ymm23, %ymm25, %ymm18 {%k5} {z}
+
+// CHECK: vpunpcklwd (%rcx), %ymm25, %ymm18
+// CHECK:  encoding: [0x62,0xe1,0x35,0x20,0x61,0x11]
+          vpunpcklwd (%rcx), %ymm25, %ymm18
+
+// CHECK: vpunpcklwd 4660(%rax,%r14,8), %ymm25, %ymm18
+// CHECK:  encoding: [0x62,0xa1,0x35,0x20,0x61,0x94,0xf0,0x34,0x12,0x00,0x00]
+          vpunpcklwd 4660(%rax,%r14,8), %ymm25, %ymm18
+
+// CHECK: vpunpcklwd 4064(%rdx), %ymm25, %ymm18
+// CHECK:  encoding: [0x62,0xe1,0x35,0x20,0x61,0x52,0x7f]
+          vpunpcklwd 4064(%rdx), %ymm25, %ymm18
+
+// CHECK: vpunpcklwd 4096(%rdx), %ymm25, %ymm18
+// CHECK:  encoding: [0x62,0xe1,0x35,0x20,0x61,0x92,0x00,0x10,0x00,0x00]
+          vpunpcklwd 4096(%rdx), %ymm25, %ymm18
+
+// CHECK: vpunpcklwd -4096(%rdx), %ymm25, %ymm18
+// CHECK:  encoding: [0x62,0xe1,0x35,0x20,0x61,0x52,0x80]
+          vpunpcklwd -4096(%rdx), %ymm25, %ymm18
+
+// CHECK: vpunpcklwd -4128(%rdx), %ymm25, %ymm18
+// CHECK:  encoding: [0x62,0xe1,0x35,0x20,0x61,0x92,0xe0,0xef,0xff,0xff]
+          vpunpcklwd -4128(%rdx), %ymm25, %ymm18
+
+// CHECK: vpunpckhwd %xmm17, %xmm28, %xmm17
+// CHECK:  encoding: [0x62,0xa1,0x1d,0x00,0x69,0xc9]
+          vpunpckhwd %xmm17, %xmm28, %xmm17
+
+// CHECK: vpunpckhwd %xmm17, %xmm28, %xmm17 {%k7}
+// CHECK:  encoding: [0x62,0xa1,0x1d,0x07,0x69,0xc9]
+          vpunpckhwd %xmm17, %xmm28, %xmm17 {%k7}
+
+// CHECK: vpunpckhwd %xmm17, %xmm28, %xmm17 {%k7} {z}
+// CHECK:  encoding: [0x62,0xa1,0x1d,0x87,0x69,0xc9]
+          vpunpckhwd %xmm17, %xmm28, %xmm17 {%k7} {z}
+
+// CHECK: vpunpckhwd (%rcx), %xmm28, %xmm17
+// CHECK:  encoding: [0x62,0xe1,0x1d,0x00,0x69,0x09]
+          vpunpckhwd (%rcx), %xmm28, %xmm17
+
+// CHECK: vpunpckhwd 4660(%rax,%r14,8), %xmm28, %xmm17
+// CHECK:  encoding: [0x62,0xa1,0x1d,0x00,0x69,0x8c,0xf0,0x34,0x12,0x00,0x00]
+          vpunpckhwd 4660(%rax,%r14,8), %xmm28, %xmm17
+
+// CHECK: vpunpckhwd 2032(%rdx), %xmm28, %xmm17
+// CHECK:  encoding: [0x62,0xe1,0x1d,0x00,0x69,0x4a,0x7f]
+          vpunpckhwd 2032(%rdx), %xmm28, %xmm17
+
+// CHECK: vpunpckhwd 2048(%rdx), %xmm28, %xmm17
+// CHECK:  encoding: [0x62,0xe1,0x1d,0x00,0x69,0x8a,0x00,0x08,0x00,0x00]
+          vpunpckhwd 2048(%rdx), %xmm28, %xmm17
+
+// CHECK: vpunpckhwd -2048(%rdx), %xmm28, %xmm17
+// CHECK:  encoding: [0x62,0xe1,0x1d,0x00,0x69,0x4a,0x80]
+          vpunpckhwd -2048(%rdx), %xmm28, %xmm17
+
+// CHECK: vpunpckhwd -2064(%rdx), %xmm28, %xmm17
+// CHECK:  encoding: [0x62,0xe1,0x1d,0x00,0x69,0x8a,0xf0,0xf7,0xff,0xff]
+          vpunpckhwd -2064(%rdx), %xmm28, %xmm17
+
+// CHECK: vpunpckhwd %ymm20, %ymm25, %ymm24
+// CHECK:  encoding: [0x62,0x21,0x35,0x20,0x69,0xc4]
+          vpunpckhwd %ymm20, %ymm25, %ymm24
+
+// CHECK: vpunpckhwd %ymm20, %ymm25, %ymm24 {%k1}
+// CHECK:  encoding: [0x62,0x21,0x35,0x21,0x69,0xc4]
+          vpunpckhwd %ymm20, %ymm25, %ymm24 {%k1}
+
+// CHECK: vpunpckhwd %ymm20, %ymm25, %ymm24 {%k1} {z}
+// CHECK:  encoding: [0x62,0x21,0x35,0xa1,0x69,0xc4]
+          vpunpckhwd %ymm20, %ymm25, %ymm24 {%k1} {z}
+
+// CHECK: vpunpckhwd (%rcx), %ymm25, %ymm24
+// CHECK:  encoding: [0x62,0x61,0x35,0x20,0x69,0x01]
+          vpunpckhwd (%rcx), %ymm25, %ymm24
+
+// CHECK: vpunpckhwd 4660(%rax,%r14,8), %ymm25, %ymm24
+// CHECK:  encoding: [0x62,0x21,0x35,0x20,0x69,0x84,0xf0,0x34,0x12,0x00,0x00]
+          vpunpckhwd 4660(%rax,%r14,8), %ymm25, %ymm24
+
+// CHECK: vpunpckhwd 4064(%rdx), %ymm25, %ymm24
+// CHECK:  encoding: [0x62,0x61,0x35,0x20,0x69,0x42,0x7f]
+          vpunpckhwd 4064(%rdx), %ymm25, %ymm24
+
+// CHECK: vpunpckhwd 4096(%rdx), %ymm25, %ymm24
+// CHECK:  encoding: [0x62,0x61,0x35,0x20,0x69,0x82,0x00,0x10,0x00,0x00]
+          vpunpckhwd 4096(%rdx), %ymm25, %ymm24
+
+// CHECK: vpunpckhwd -4096(%rdx), %ymm25, %ymm24
+// CHECK:  encoding: [0x62,0x61,0x35,0x20,0x69,0x42,0x80]
+          vpunpckhwd -4096(%rdx), %ymm25, %ymm24
+
+// CHECK: vpunpckhwd -4128(%rdx), %ymm25, %ymm24
+// CHECK:  encoding: [0x62,0x61,0x35,0x20,0x69,0x82,0xe0,0xef,0xff,0xff]
+          vpunpckhwd -4128(%rdx), %ymm25, %ymm24
+
+// CHECK: vpunpcklbw %xmm23, %xmm29, %xmm18
+// CHECK:  encoding: [0x62,0xa1,0x15,0x00,0x60,0xd7]
+          vpunpcklbw %xmm23, %xmm29, %xmm18
+
+// CHECK: vpunpcklbw %xmm23, %xmm29, %xmm18 {%k4}
+// CHECK:  encoding: [0x62,0xa1,0x15,0x04,0x60,0xd7]
+          vpunpcklbw %xmm23, %xmm29, %xmm18 {%k4}
+
+// CHECK: vpunpcklbw %xmm23, %xmm29, %xmm18 {%k4} {z}
+// CHECK:  encoding: [0x62,0xa1,0x15,0x84,0x60,0xd7]
+          vpunpcklbw %xmm23, %xmm29, %xmm18 {%k4} {z}
+
+// CHECK: vpunpcklbw (%rcx), %xmm29, %xmm18
+// CHECK:  encoding: [0x62,0xe1,0x15,0x00,0x60,0x11]
+          vpunpcklbw (%rcx), %xmm29, %xmm18
+
+// CHECK: vpunpcklbw 291(%rax,%r14,8), %xmm29, %xmm18
+// CHECK:  encoding: [0x62,0xa1,0x15,0x00,0x60,0x94,0xf0,0x23,0x01,0x00,0x00]
+          vpunpcklbw 291(%rax,%r14,8), %xmm29, %xmm18
+
+// CHECK: vpunpcklbw 2032(%rdx), %xmm29, %xmm18
+// CHECK:  encoding: [0x62,0xe1,0x15,0x00,0x60,0x52,0x7f]
+          vpunpcklbw 2032(%rdx), %xmm29, %xmm18
+
+// CHECK: vpunpcklbw 2048(%rdx), %xmm29, %xmm18
+// CHECK:  encoding: [0x62,0xe1,0x15,0x00,0x60,0x92,0x00,0x08,0x00,0x00]
+          vpunpcklbw 2048(%rdx), %xmm29, %xmm18
+
+// CHECK: vpunpcklbw -2048(%rdx), %xmm29, %xmm18
+// CHECK:  encoding: [0x62,0xe1,0x15,0x00,0x60,0x52,0x80]
+          vpunpcklbw -2048(%rdx), %xmm29, %xmm18
+
+// CHECK: vpunpcklbw -2064(%rdx), %xmm29, %xmm18
+// CHECK:  encoding: [0x62,0xe1,0x15,0x00,0x60,0x92,0xf0,0xf7,0xff,0xff]
+          vpunpcklbw -2064(%rdx), %xmm29, %xmm18
+
+// CHECK: vpunpcklbw %ymm21, %ymm28, %ymm27
+// CHECK:  encoding: [0x62,0x21,0x1d,0x20,0x60,0xdd]
+          vpunpcklbw %ymm21, %ymm28, %ymm27
+
+// CHECK: vpunpcklbw %ymm21, %ymm28, %ymm27 {%k4}
+// CHECK:  encoding: [0x62,0x21,0x1d,0x24,0x60,0xdd]
+          vpunpcklbw %ymm21, %ymm28, %ymm27 {%k4}
+
+// CHECK: vpunpcklbw %ymm21, %ymm28, %ymm27 {%k4} {z}
+// CHECK:  encoding: [0x62,0x21,0x1d,0xa4,0x60,0xdd]
+          vpunpcklbw %ymm21, %ymm28, %ymm27 {%k4} {z}
+
+// CHECK: vpunpcklbw (%rcx), %ymm28, %ymm27
+// CHECK:  encoding: [0x62,0x61,0x1d,0x20,0x60,0x19]
+          vpunpcklbw (%rcx), %ymm28, %ymm27
+
+// CHECK: vpunpcklbw 291(%rax,%r14,8), %ymm28, %ymm27
+// CHECK:  encoding: [0x62,0x21,0x1d,0x20,0x60,0x9c,0xf0,0x23,0x01,0x00,0x00]
+          vpunpcklbw 291(%rax,%r14,8), %ymm28, %ymm27
+
+// CHECK: vpunpcklbw 4064(%rdx), %ymm28, %ymm27
+// CHECK:  encoding: [0x62,0x61,0x1d,0x20,0x60,0x5a,0x7f]
+          vpunpcklbw 4064(%rdx), %ymm28, %ymm27
+
+// CHECK: vpunpcklbw 4096(%rdx), %ymm28, %ymm27
+// CHECK:  encoding: [0x62,0x61,0x1d,0x20,0x60,0x9a,0x00,0x10,0x00,0x00]
+          vpunpcklbw 4096(%rdx), %ymm28, %ymm27
+
+// CHECK: vpunpcklbw -4096(%rdx), %ymm28, %ymm27
+// CHECK:  encoding: [0x62,0x61,0x1d,0x20,0x60,0x5a,0x80]
+          vpunpcklbw -4096(%rdx), %ymm28, %ymm27
+
+// CHECK: vpunpcklbw -4128(%rdx), %ymm28, %ymm27
+// CHECK:  encoding: [0x62,0x61,0x1d,0x20,0x60,0x9a,0xe0,0xef,0xff,0xff]
+          vpunpcklbw -4128(%rdx), %ymm28, %ymm27
+
+// CHECK: vpunpckhbw %xmm24, %xmm21, %xmm18
+// CHECK:  encoding: [0x62,0x81,0x55,0x00,0x68,0xd0]
+          vpunpckhbw %xmm24, %xmm21, %xmm18
+
+// CHECK: vpunpckhbw %xmm24, %xmm21, %xmm18 {%k6}
+// CHECK:  encoding: [0x62,0x81,0x55,0x06,0x68,0xd0]
+          vpunpckhbw %xmm24, %xmm21, %xmm18 {%k6}
+
+// CHECK: vpunpckhbw %xmm24, %xmm21, %xmm18 {%k6} {z}
+// CHECK:  encoding: [0x62,0x81,0x55,0x86,0x68,0xd0]
+          vpunpckhbw %xmm24, %xmm21, %xmm18 {%k6} {z}
+
+// CHECK: vpunpckhbw (%rcx), %xmm21, %xmm18
+// CHECK:  encoding: [0x62,0xe1,0x55,0x00,0x68,0x11]
+          vpunpckhbw (%rcx), %xmm21, %xmm18
+
+// CHECK: vpunpckhbw 291(%rax,%r14,8), %xmm21, %xmm18
+// CHECK:  encoding: [0x62,0xa1,0x55,0x00,0x68,0x94,0xf0,0x23,0x01,0x00,0x00]
+          vpunpckhbw 291(%rax,%r14,8), %xmm21, %xmm18
+
+// CHECK: vpunpckhbw 2032(%rdx), %xmm21, %xmm18
+// CHECK:  encoding: [0x62,0xe1,0x55,0x00,0x68,0x52,0x7f]
+          vpunpckhbw 2032(%rdx), %xmm21, %xmm18
+
+// CHECK: vpunpckhbw 2048(%rdx), %xmm21, %xmm18
+// CHECK:  encoding: [0x62,0xe1,0x55,0x00,0x68,0x92,0x00,0x08,0x00,0x00]
+          vpunpckhbw 2048(%rdx), %xmm21, %xmm18
+
+// CHECK: vpunpckhbw -2048(%rdx), %xmm21, %xmm18
+// CHECK:  encoding: [0x62,0xe1,0x55,0x00,0x68,0x52,0x80]
+          vpunpckhbw -2048(%rdx), %xmm21, %xmm18
+
+// CHECK: vpunpckhbw -2064(%rdx), %xmm21, %xmm18
+// CHECK:  encoding: [0x62,0xe1,0x55,0x00,0x68,0x92,0xf0,0xf7,0xff,0xff]
+          vpunpckhbw -2064(%rdx), %xmm21, %xmm18
+
+// CHECK: vpunpckhbw %ymm23, %ymm26, %ymm20
+// CHECK:  encoding: [0x62,0xa1,0x2d,0x20,0x68,0xe7]
+          vpunpckhbw %ymm23, %ymm26, %ymm20
+
+// CHECK: vpunpckhbw %ymm23, %ymm26, %ymm20 {%k1}
+// CHECK:  encoding: [0x62,0xa1,0x2d,0x21,0x68,0xe7]
+          vpunpckhbw %ymm23, %ymm26, %ymm20 {%k1}
+
+// CHECK: vpunpckhbw %ymm23, %ymm26, %ymm20 {%k1} {z}
+// CHECK:  encoding: [0x62,0xa1,0x2d,0xa1,0x68,0xe7]
+          vpunpckhbw %ymm23, %ymm26, %ymm20 {%k1} {z}
+
+// CHECK: vpunpckhbw (%rcx), %ymm26, %ymm20
+// CHECK:  encoding: [0x62,0xe1,0x2d,0x20,0x68,0x21]
+          vpunpckhbw (%rcx), %ymm26, %ymm20
+
+// CHECK: vpunpckhbw 291(%rax,%r14,8), %ymm26, %ymm20
+// CHECK:  encoding: [0x62,0xa1,0x2d,0x20,0x68,0xa4,0xf0,0x23,0x01,0x00,0x00]
+          vpunpckhbw 291(%rax,%r14,8), %ymm26, %ymm20
+
+// CHECK: vpunpckhbw 4064(%rdx), %ymm26, %ymm20
+// CHECK:  encoding: [0x62,0xe1,0x2d,0x20,0x68,0x62,0x7f]
+          vpunpckhbw 4064(%rdx), %ymm26, %ymm20
+
+// CHECK: vpunpckhbw 4096(%rdx), %ymm26, %ymm20
+// CHECK:  encoding: [0x62,0xe1,0x2d,0x20,0x68,0xa2,0x00,0x10,0x00,0x00]
+          vpunpckhbw 4096(%rdx), %ymm26, %ymm20
+
+// CHECK: vpunpckhbw -4096(%rdx), %ymm26, %ymm20
+// CHECK:  encoding: [0x62,0xe1,0x2d,0x20,0x68,0x62,0x80]
+          vpunpckhbw -4096(%rdx), %ymm26, %ymm20
+
+// CHECK: vpunpckhbw -4128(%rdx), %ymm26, %ymm20
+// CHECK:  encoding: [0x62,0xe1,0x2d,0x20,0x68,0xa2,0xe0,0xef,0xff,0xff]
+          vpunpckhbw -4128(%rdx), %ymm26, %ymm20
+
+// CHECK: vpunpcklwd %xmm21, %xmm25, %xmm21
+// CHECK:  encoding: [0x62,0xa1,0x35,0x00,0x61,0xed]
+          vpunpcklwd %xmm21, %xmm25, %xmm21
+
+// CHECK: vpunpcklwd %xmm21, %xmm25, %xmm21 {%k6}
+// CHECK:  encoding: [0x62,0xa1,0x35,0x06,0x61,0xed]
+          vpunpcklwd %xmm21, %xmm25, %xmm21 {%k6}
+
+// CHECK: vpunpcklwd %xmm21, %xmm25, %xmm21 {%k6} {z}
+// CHECK:  encoding: [0x62,0xa1,0x35,0x86,0x61,0xed]
+          vpunpcklwd %xmm21, %xmm25, %xmm21 {%k6} {z}
+
+// CHECK: vpunpcklwd (%rcx), %xmm25, %xmm21
+// CHECK:  encoding: [0x62,0xe1,0x35,0x00,0x61,0x29]
+          vpunpcklwd (%rcx), %xmm25, %xmm21
+
+// CHECK: vpunpcklwd 291(%rax,%r14,8), %xmm25, %xmm21
+// CHECK:  encoding: [0x62,0xa1,0x35,0x00,0x61,0xac,0xf0,0x23,0x01,0x00,0x00]
+          vpunpcklwd 291(%rax,%r14,8), %xmm25, %xmm21
+
+// CHECK: vpunpcklwd 2032(%rdx), %xmm25, %xmm21
+// CHECK:  encoding: [0x62,0xe1,0x35,0x00,0x61,0x6a,0x7f]
+          vpunpcklwd 2032(%rdx), %xmm25, %xmm21
+
+// CHECK: vpunpcklwd 2048(%rdx), %xmm25, %xmm21
+// CHECK:  encoding: [0x62,0xe1,0x35,0x00,0x61,0xaa,0x00,0x08,0x00,0x00]
+          vpunpcklwd 2048(%rdx), %xmm25, %xmm21
+
+// CHECK: vpunpcklwd -2048(%rdx), %xmm25, %xmm21
+// CHECK:  encoding: [0x62,0xe1,0x35,0x00,0x61,0x6a,0x80]
+          vpunpcklwd -2048(%rdx), %xmm25, %xmm21
+
+// CHECK: vpunpcklwd -2064(%rdx), %xmm25, %xmm21
+// CHECK:  encoding: [0x62,0xe1,0x35,0x00,0x61,0xaa,0xf0,0xf7,0xff,0xff]
+          vpunpcklwd -2064(%rdx), %xmm25, %xmm21
+
+// CHECK: vpunpcklwd %ymm26, %ymm23, %ymm26
+// CHECK:  encoding: [0x62,0x01,0x45,0x20,0x61,0xd2]
+          vpunpcklwd %ymm26, %ymm23, %ymm26
+
+// CHECK: vpunpcklwd %ymm26, %ymm23, %ymm26 {%k2}
+// CHECK:  encoding: [0x62,0x01,0x45,0x22,0x61,0xd2]
+          vpunpcklwd %ymm26, %ymm23, %ymm26 {%k2}
+
+// CHECK: vpunpcklwd %ymm26, %ymm23, %ymm26 {%k2} {z}
+// CHECK:  encoding: [0x62,0x01,0x45,0xa2,0x61,0xd2]
+          vpunpcklwd %ymm26, %ymm23, %ymm26 {%k2} {z}
+
+// CHECK: vpunpcklwd (%rcx), %ymm23, %ymm26
+// CHECK:  encoding: [0x62,0x61,0x45,0x20,0x61,0x11]
+          vpunpcklwd (%rcx), %ymm23, %ymm26
+
+// CHECK: vpunpcklwd 291(%rax,%r14,8), %ymm23, %ymm26
+// CHECK:  encoding: [0x62,0x21,0x45,0x20,0x61,0x94,0xf0,0x23,0x01,0x00,0x00]
+          vpunpcklwd 291(%rax,%r14,8), %ymm23, %ymm26
+
+// CHECK: vpunpcklwd 4064(%rdx), %ymm23, %ymm26
+// CHECK:  encoding: [0x62,0x61,0x45,0x20,0x61,0x52,0x7f]
+          vpunpcklwd 4064(%rdx), %ymm23, %ymm26
+
+// CHECK: vpunpcklwd 4096(%rdx), %ymm23, %ymm26
+// CHECK:  encoding: [0x62,0x61,0x45,0x20,0x61,0x92,0x00,0x10,0x00,0x00]
+          vpunpcklwd 4096(%rdx), %ymm23, %ymm26
+
+// CHECK: vpunpcklwd -4096(%rdx), %ymm23, %ymm26
+// CHECK:  encoding: [0x62,0x61,0x45,0x20,0x61,0x52,0x80]
+          vpunpcklwd -4096(%rdx), %ymm23, %ymm26
+
+// CHECK: vpunpcklwd -4128(%rdx), %ymm23, %ymm26
+// CHECK:  encoding: [0x62,0x61,0x45,0x20,0x61,0x92,0xe0,0xef,0xff,0xff]
+          vpunpcklwd -4128(%rdx), %ymm23, %ymm26
+
+// CHECK: vpunpckhwd %xmm23, %xmm18, %xmm17
+// CHECK:  encoding: [0x62,0xa1,0x6d,0x00,0x69,0xcf]
+          vpunpckhwd %xmm23, %xmm18, %xmm17
+
+// CHECK: vpunpckhwd %xmm23, %xmm18, %xmm17 {%k3}
+// CHECK:  encoding: [0x62,0xa1,0x6d,0x03,0x69,0xcf]
+          vpunpckhwd %xmm23, %xmm18, %xmm17 {%k3}
+
+// CHECK: vpunpckhwd %xmm23, %xmm18, %xmm17 {%k3} {z}
+// CHECK:  encoding: [0x62,0xa1,0x6d,0x83,0x69,0xcf]
+          vpunpckhwd %xmm23, %xmm18, %xmm17 {%k3} {z}
+
+// CHECK: vpunpckhwd (%rcx), %xmm18, %xmm17
+// CHECK:  encoding: [0x62,0xe1,0x6d,0x00,0x69,0x09]
+          vpunpckhwd (%rcx), %xmm18, %xmm17
+
+// CHECK: vpunpckhwd 291(%rax,%r14,8), %xmm18, %xmm17
+// CHECK:  encoding: [0x62,0xa1,0x6d,0x00,0x69,0x8c,0xf0,0x23,0x01,0x00,0x00]
+          vpunpckhwd 291(%rax,%r14,8), %xmm18, %xmm17
+
+// CHECK: vpunpckhwd 2032(%rdx), %xmm18, %xmm17
+// CHECK:  encoding: [0x62,0xe1,0x6d,0x00,0x69,0x4a,0x7f]
+          vpunpckhwd 2032(%rdx), %xmm18, %xmm17
+
+// CHECK: vpunpckhwd 2048(%rdx), %xmm18, %xmm17
+// CHECK:  encoding: [0x62,0xe1,0x6d,0x00,0x69,0x8a,0x00,0x08,0x00,0x00]
+          vpunpckhwd 2048(%rdx), %xmm18, %xmm17
+
+// CHECK: vpunpckhwd -2048(%rdx), %xmm18, %xmm17
+// CHECK:  encoding: [0x62,0xe1,0x6d,0x00,0x69,0x4a,0x80]
+          vpunpckhwd -2048(%rdx), %xmm18, %xmm17
+
+// CHECK: vpunpckhwd -2064(%rdx), %xmm18, %xmm17
+// CHECK:  encoding: [0x62,0xe1,0x6d,0x00,0x69,0x8a,0xf0,0xf7,0xff,0xff]
+          vpunpckhwd -2064(%rdx), %xmm18, %xmm17
+
+// CHECK: vpunpckhwd %ymm26, %ymm25, %ymm28
+// CHECK:  encoding: [0x62,0x01,0x35,0x20,0x69,0xe2]
+          vpunpckhwd %ymm26, %ymm25, %ymm28
+
+// CHECK: vpunpckhwd %ymm26, %ymm25, %ymm28 {%k4}
+// CHECK:  encoding: [0x62,0x01,0x35,0x24,0x69,0xe2]
+          vpunpckhwd %ymm26, %ymm25, %ymm28 {%k4}
+
+// CHECK: vpunpckhwd %ymm26, %ymm25, %ymm28 {%k4} {z}
+// CHECK:  encoding: [0x62,0x01,0x35,0xa4,0x69,0xe2]
+          vpunpckhwd %ymm26, %ymm25, %ymm28 {%k4} {z}
+
+// CHECK: vpunpckhwd (%rcx), %ymm25, %ymm28
+// CHECK:  encoding: [0x62,0x61,0x35,0x20,0x69,0x21]
+          vpunpckhwd (%rcx), %ymm25, %ymm28
+
+// CHECK: vpunpckhwd 291(%rax,%r14,8), %ymm25, %ymm28
+// CHECK:  encoding: [0x62,0x21,0x35,0x20,0x69,0xa4,0xf0,0x23,0x01,0x00,0x00]
+          vpunpckhwd 291(%rax,%r14,8), %ymm25, %ymm28
+
+// CHECK: vpunpckhwd 4064(%rdx), %ymm25, %ymm28
+// CHECK:  encoding: [0x62,0x61,0x35,0x20,0x69,0x62,0x7f]
+          vpunpckhwd 4064(%rdx), %ymm25, %ymm28
+
+// CHECK: vpunpckhwd 4096(%rdx), %ymm25, %ymm28
+// CHECK:  encoding: [0x62,0x61,0x35,0x20,0x69,0xa2,0x00,0x10,0x00,0x00]
+          vpunpckhwd 4096(%rdx), %ymm25, %ymm28
+
+// CHECK: vpunpckhwd -4096(%rdx), %ymm25, %ymm28
+// CHECK:  encoding: [0x62,0x61,0x35,0x20,0x69,0x62,0x80]
+          vpunpckhwd -4096(%rdx), %ymm25, %ymm28
+
+// CHECK: vpunpckhwd -4128(%rdx), %ymm25, %ymm28
+// CHECK:  encoding: [0x62,0x61,0x35,0x20,0x69,0xa2,0xe0,0xef,0xff,0xff]
+          vpunpckhwd -4128(%rdx), %ymm25, %ymm28
+
diff --git a/test/MC/X86/x86-64-avx512dq.s b/test/MC/X86/x86-64-avx512dq.s
index aac176517bf..d4e847557bc 100644
--- a/test/MC/X86/x86-64-avx512dq.s
+++ b/test/MC/X86/x86-64-avx512dq.s
@@ -127,3 +127,2247 @@
 // CHECK: kmovb  %k3, %r13d
 // CHECK:  encoding: [0xc5,0x79,0x93,0xeb]
           kmovb  %k3, %r13d
+
+// CHECK: vandpd %zmm27, %zmm28, %zmm19
+// CHECK:  encoding: [0x62,0x81,0x9d,0x40,0x54,0xdb]
+          vandpd %zmm27, %zmm28, %zmm19
+
+// CHECK: vandpd %zmm27, %zmm28, %zmm19 {%k5}
+// CHECK:  encoding: [0x62,0x81,0x9d,0x45,0x54,0xdb]
+          vandpd %zmm27, %zmm28, %zmm19 {%k5}
+
+// CHECK: vandpd %zmm27, %zmm28, %zmm19 {%k5} {z}
+// CHECK:  encoding: [0x62,0x81,0x9d,0xc5,0x54,0xdb]
+          vandpd %zmm27, %zmm28, %zmm19 {%k5} {z}
+
+// CHECK: vandpd (%rcx), %zmm28, %zmm19
+// CHECK:  encoding: [0x62,0xe1,0x9d,0x40,0x54,0x19]
+          vandpd (%rcx), %zmm28, %zmm19
+
+// CHECK: vandpd 291(%rax,%r14,8), %zmm28, %zmm19
+// CHECK:  encoding: [0x62,0xa1,0x9d,0x40,0x54,0x9c,0xf0,0x23,0x01,0x00,0x00]
+          vandpd 291(%rax,%r14,8), %zmm28, %zmm19
+
+// CHECK: vandpd (%rcx){1to8}, %zmm28, %zmm19
+// CHECK:  encoding: [0x62,0xe1,0x9d,0x50,0x54,0x19]
+          vandpd (%rcx){1to8}, %zmm28, %zmm19
+
+// CHECK: vandpd 8128(%rdx), %zmm28, %zmm19
+// CHECK:  encoding: [0x62,0xe1,0x9d,0x40,0x54,0x5a,0x7f]
+          vandpd 8128(%rdx), %zmm28, %zmm19
+
+// CHECK: vandpd 8192(%rdx), %zmm28, %zmm19
+// CHECK:  encoding: [0x62,0xe1,0x9d,0x40,0x54,0x9a,0x00,0x20,0x00,0x00]
+          vandpd 8192(%rdx), %zmm28, %zmm19
+
+// CHECK: vandpd -8192(%rdx), %zmm28, %zmm19
+// CHECK:  encoding: [0x62,0xe1,0x9d,0x40,0x54,0x5a,0x80]
+          vandpd -8192(%rdx), %zmm28, %zmm19
+
+// CHECK: vandpd -8256(%rdx), %zmm28, %zmm19
+// CHECK:  encoding: [0x62,0xe1,0x9d,0x40,0x54,0x9a,0xc0,0xdf,0xff,0xff]
+          vandpd -8256(%rdx), %zmm28, %zmm19
+
+// CHECK: vandpd 1016(%rdx){1to8}, %zmm28, %zmm19
+// CHECK:  encoding: [0x62,0xe1,0x9d,0x50,0x54,0x5a,0x7f]
+          vandpd 1016(%rdx){1to8}, %zmm28, %zmm19
+
+// CHECK: vandpd 1024(%rdx){1to8}, %zmm28, %zmm19
+// CHECK:  encoding: [0x62,0xe1,0x9d,0x50,0x54,0x9a,0x00,0x04,0x00,0x00]
+          vandpd 1024(%rdx){1to8}, %zmm28, %zmm19
+
+// CHECK: vandpd -1024(%rdx){1to8}, %zmm28, %zmm19
+// CHECK:  encoding: [0x62,0xe1,0x9d,0x50,0x54,0x5a,0x80]
+          vandpd -1024(%rdx){1to8}, %zmm28, %zmm19
+
+// CHECK: vandpd -1032(%rdx){1to8}, %zmm28, %zmm19
+// CHECK:  encoding: [0x62,0xe1,0x9d,0x50,0x54,0x9a,0xf8,0xfb,0xff,0xff]
+          vandpd -1032(%rdx){1to8}, %zmm28, %zmm19
+
+// CHECK: vandps %zmm25, %zmm22, %zmm17
+// CHECK:  encoding: [0x62,0x81,0x4c,0x40,0x54,0xc9]
+          vandps %zmm25, %zmm22, %zmm17
+
+// CHECK: vandps %zmm25, %zmm22, %zmm17 {%k4}
+// CHECK:  encoding: [0x62,0x81,0x4c,0x44,0x54,0xc9]
+          vandps %zmm25, %zmm22, %zmm17 {%k4}
+
+// CHECK: vandps %zmm25, %zmm22, %zmm17 {%k4} {z}
+// CHECK:  encoding: [0x62,0x81,0x4c,0xc4,0x54,0xc9]
+          vandps %zmm25, %zmm22, %zmm17 {%k4} {z}
+
+// CHECK: vandps (%rcx), %zmm22, %zmm17
+// CHECK:  encoding: [0x62,0xe1,0x4c,0x40,0x54,0x09]
+          vandps (%rcx), %zmm22, %zmm17
+
+// CHECK: vandps 291(%rax,%r14,8), %zmm22, %zmm17
+// CHECK:  encoding: [0x62,0xa1,0x4c,0x40,0x54,0x8c,0xf0,0x23,0x01,0x00,0x00]
+          vandps 291(%rax,%r14,8), %zmm22, %zmm17
+
+// CHECK: vandps (%rcx){1to16}, %zmm22, %zmm17
+// CHECK:  encoding: [0x62,0xe1,0x4c,0x50,0x54,0x09]
+          vandps (%rcx){1to16}, %zmm22, %zmm17
+
+// CHECK: vandps 8128(%rdx), %zmm22, %zmm17
+// CHECK:  encoding: [0x62,0xe1,0x4c,0x40,0x54,0x4a,0x7f]
+          vandps 8128(%rdx), %zmm22, %zmm17
+
+// CHECK: vandps 8192(%rdx), %zmm22, %zmm17
+// CHECK:  encoding: [0x62,0xe1,0x4c,0x40,0x54,0x8a,0x00,0x20,0x00,0x00]
+          vandps 8192(%rdx), %zmm22, %zmm17
+
+// CHECK: vandps -8192(%rdx), %zmm22, %zmm17
+// CHECK:  encoding: [0x62,0xe1,0x4c,0x40,0x54,0x4a,0x80]
+          vandps -8192(%rdx), %zmm22, %zmm17
+
+// CHECK: vandps -8256(%rdx), %zmm22, %zmm17
+// CHECK:  encoding: [0x62,0xe1,0x4c,0x40,0x54,0x8a,0xc0,0xdf,0xff,0xff]
+          vandps -8256(%rdx), %zmm22, %zmm17
+
+// CHECK: vandps 508(%rdx){1to16}, %zmm22, %zmm17
+// CHECK:  encoding: [0x62,0xe1,0x4c,0x50,0x54,0x4a,0x7f]
+          vandps 508(%rdx){1to16}, %zmm22, %zmm17
+
+// CHECK: vandps 512(%rdx){1to16}, %zmm22, %zmm17
+// CHECK:  encoding: [0x62,0xe1,0x4c,0x50,0x54,0x8a,0x00,0x02,0x00,0x00]
+          vandps 512(%rdx){1to16}, %zmm22, %zmm17
+
+// CHECK: vandps -512(%rdx){1to16}, %zmm22, %zmm17
+// CHECK:  encoding: [0x62,0xe1,0x4c,0x50,0x54,0x4a,0x80]
+          vandps -512(%rdx){1to16}, %zmm22, %zmm17
+
+// CHECK: vandps -516(%rdx){1to16}, %zmm22, %zmm17
+// CHECK:  encoding: [0x62,0xe1,0x4c,0x50,0x54,0x8a,0xfc,0xfd,0xff,0xff]
+          vandps -516(%rdx){1to16}, %zmm22, %zmm17
+
+// CHECK: vandnpd %zmm22, %zmm17, %zmm20
+// CHECK:  encoding: [0x62,0xa1,0xf5,0x40,0x55,0xe6]
+          vandnpd %zmm22, %zmm17, %zmm20
+
+// CHECK: vandnpd %zmm22, %zmm17, %zmm20 {%k1}
+// CHECK:  encoding: [0x62,0xa1,0xf5,0x41,0x55,0xe6]
+          vandnpd %zmm22, %zmm17, %zmm20 {%k1}
+
+// CHECK: vandnpd %zmm22, %zmm17, %zmm20 {%k1} {z}
+// CHECK:  encoding: [0x62,0xa1,0xf5,0xc1,0x55,0xe6]
+          vandnpd %zmm22, %zmm17, %zmm20 {%k1} {z}
+
+// CHECK: vandnpd (%rcx), %zmm17, %zmm20
+// CHECK:  encoding: [0x62,0xe1,0xf5,0x40,0x55,0x21]
+          vandnpd (%rcx), %zmm17, %zmm20
+
+// CHECK: vandnpd 291(%rax,%r14,8), %zmm17, %zmm20
+// CHECK:  encoding: [0x62,0xa1,0xf5,0x40,0x55,0xa4,0xf0,0x23,0x01,0x00,0x00]
+          vandnpd 291(%rax,%r14,8), %zmm17, %zmm20
+
+// CHECK: vandnpd (%rcx){1to8}, %zmm17, %zmm20
+// CHECK:  encoding: [0x62,0xe1,0xf5,0x50,0x55,0x21]
+          vandnpd (%rcx){1to8}, %zmm17, %zmm20
+
+// CHECK: vandnpd 8128(%rdx), %zmm17, %zmm20
+// CHECK:  encoding: [0x62,0xe1,0xf5,0x40,0x55,0x62,0x7f]
+          vandnpd 8128(%rdx), %zmm17, %zmm20
+
+// CHECK: vandnpd 8192(%rdx), %zmm17, %zmm20
+// CHECK:  encoding: [0x62,0xe1,0xf5,0x40,0x55,0xa2,0x00,0x20,0x00,0x00]
+          vandnpd 8192(%rdx), %zmm17, %zmm20
+
+// CHECK: vandnpd -8192(%rdx), %zmm17, %zmm20
+// CHECK:  encoding: [0x62,0xe1,0xf5,0x40,0x55,0x62,0x80]
+          vandnpd -8192(%rdx), %zmm17, %zmm20
+
+// CHECK: vandnpd -8256(%rdx), %zmm17, %zmm20
+// CHECK:  encoding: [0x62,0xe1,0xf5,0x40,0x55,0xa2,0xc0,0xdf,0xff,0xff]
+          vandnpd -8256(%rdx), %zmm17, %zmm20
+
+// CHECK: vandnpd 1016(%rdx){1to8}, %zmm17, %zmm20
+// CHECK:  encoding: [0x62,0xe1,0xf5,0x50,0x55,0x62,0x7f]
+          vandnpd 1016(%rdx){1to8}, %zmm17, %zmm20
+
+// CHECK: vandnpd 1024(%rdx){1to8}, %zmm17, %zmm20
+// CHECK:  encoding: [0x62,0xe1,0xf5,0x50,0x55,0xa2,0x00,0x04,0x00,0x00]
+          vandnpd 1024(%rdx){1to8}, %zmm17, %zmm20
+
+// CHECK: vandnpd -1024(%rdx){1to8}, %zmm17, %zmm20
+// CHECK:  encoding: [0x62,0xe1,0xf5,0x50,0x55,0x62,0x80]
+          vandnpd -1024(%rdx){1to8}, %zmm17, %zmm20
+
+// CHECK: vandnpd -1032(%rdx){1to8}, %zmm17, %zmm20
+// CHECK:  encoding: [0x62,0xe1,0xf5,0x50,0x55,0xa2,0xf8,0xfb,0xff,0xff]
+          vandnpd -1032(%rdx){1to8}, %zmm17, %zmm20
+
+// CHECK: vandnps %zmm19, %zmm17, %zmm22
+// CHECK:  encoding: [0x62,0xa1,0x74,0x40,0x55,0xf3]
+          vandnps %zmm19, %zmm17, %zmm22
+
+// CHECK: vandnps %zmm19, %zmm17, %zmm22 {%k2}
+// CHECK:  encoding: [0x62,0xa1,0x74,0x42,0x55,0xf3]
+          vandnps %zmm19, %zmm17, %zmm22 {%k2}
+
+// CHECK: vandnps %zmm19, %zmm17, %zmm22 {%k2} {z}
+// CHECK:  encoding: [0x62,0xa1,0x74,0xc2,0x55,0xf3]
+          vandnps %zmm19, %zmm17, %zmm22 {%k2} {z}
+
+// CHECK: vandnps (%rcx), %zmm17, %zmm22
+// CHECK:  encoding: [0x62,0xe1,0x74,0x40,0x55,0x31]
+          vandnps (%rcx), %zmm17, %zmm22
+
+// CHECK: vandnps 291(%rax,%r14,8), %zmm17, %zmm22
+// CHECK:  encoding: [0x62,0xa1,0x74,0x40,0x55,0xb4,0xf0,0x23,0x01,0x00,0x00]
+          vandnps 291(%rax,%r14,8), %zmm17, %zmm22
+
+// CHECK: vandnps (%rcx){1to16}, %zmm17, %zmm22
+// CHECK:  encoding: [0x62,0xe1,0x74,0x50,0x55,0x31]
+          vandnps (%rcx){1to16}, %zmm17, %zmm22
+
+// CHECK: vandnps 8128(%rdx), %zmm17, %zmm22
+// CHECK:  encoding: [0x62,0xe1,0x74,0x40,0x55,0x72,0x7f]
+          vandnps 8128(%rdx), %zmm17, %zmm22
+
+// CHECK: vandnps 8192(%rdx), %zmm17, %zmm22
+// CHECK:  encoding: [0x62,0xe1,0x74,0x40,0x55,0xb2,0x00,0x20,0x00,0x00]
+          vandnps 8192(%rdx), %zmm17, %zmm22
+
+// CHECK: vandnps -8192(%rdx), %zmm17, %zmm22
+// CHECK:  encoding: [0x62,0xe1,0x74,0x40,0x55,0x72,0x80]
+          vandnps -8192(%rdx), %zmm17, %zmm22
+
+// CHECK: vandnps -8256(%rdx), %zmm17, %zmm22
+// CHECK:  encoding: [0x62,0xe1,0x74,0x40,0x55,0xb2,0xc0,0xdf,0xff,0xff]
+          vandnps -8256(%rdx), %zmm17, %zmm22
+
+// CHECK: vandnps 508(%rdx){1to16}, %zmm17, %zmm22
+// CHECK:  encoding: [0x62,0xe1,0x74,0x50,0x55,0x72,0x7f]
+          vandnps 508(%rdx){1to16}, %zmm17, %zmm22
+
+// CHECK: vandnps 512(%rdx){1to16}, %zmm17, %zmm22
+// CHECK:  encoding: [0x62,0xe1,0x74,0x50,0x55,0xb2,0x00,0x02,0x00,0x00]
+          vandnps 512(%rdx){1to16}, %zmm17, %zmm22
+
+// CHECK: vandnps -512(%rdx){1to16}, %zmm17, %zmm22
+// CHECK:  encoding: [0x62,0xe1,0x74,0x50,0x55,0x72,0x80]
+          vandnps -512(%rdx){1to16}, %zmm17, %zmm22
+
+// CHECK: vandnps -516(%rdx){1to16}, %zmm17, %zmm22
+// CHECK:  encoding: [0x62,0xe1,0x74,0x50,0x55,0xb2,0xfc,0xfd,0xff,0xff]
+          vandnps -516(%rdx){1to16}, %zmm17, %zmm22
+
+// CHECK: vorpd  %zmm21, %zmm22, %zmm30
+// CHECK:  encoding: [0x62,0x21,0xcd,0x40,0x56,0xf5]
+          vorpd  %zmm21, %zmm22, %zmm30
+
+// CHECK: vorpd  %zmm21, %zmm22, %zmm30 {%k6}
+// CHECK:  encoding: [0x62,0x21,0xcd,0x46,0x56,0xf5]
+          vorpd  %zmm21, %zmm22, %zmm30 {%k6}
+
+// CHECK: vorpd  %zmm21, %zmm22, %zmm30 {%k6} {z}
+// CHECK:  encoding: [0x62,0x21,0xcd,0xc6,0x56,0xf5]
+          vorpd  %zmm21, %zmm22, %zmm30 {%k6} {z}
+
+// CHECK: vorpd  (%rcx), %zmm22, %zmm30
+// CHECK:  encoding: [0x62,0x61,0xcd,0x40,0x56,0x31]
+          vorpd  (%rcx), %zmm22, %zmm30
+
+// CHECK: vorpd  291(%rax,%r14,8), %zmm22, %zmm30
+// CHECK:  encoding: [0x62,0x21,0xcd,0x40,0x56,0xb4,0xf0,0x23,0x01,0x00,0x00]
+          vorpd  291(%rax,%r14,8), %zmm22, %zmm30
+
+// CHECK: vorpd  (%rcx){1to8}, %zmm22, %zmm30
+// CHECK:  encoding: [0x62,0x61,0xcd,0x50,0x56,0x31]
+          vorpd  (%rcx){1to8}, %zmm22, %zmm30
+
+// CHECK: vorpd  8128(%rdx), %zmm22, %zmm30
+// CHECK:  encoding: [0x62,0x61,0xcd,0x40,0x56,0x72,0x7f]
+          vorpd  8128(%rdx), %zmm22, %zmm30
+
+// CHECK: vorpd  8192(%rdx), %zmm22, %zmm30
+// CHECK:  encoding: [0x62,0x61,0xcd,0x40,0x56,0xb2,0x00,0x20,0x00,0x00]
+          vorpd  8192(%rdx), %zmm22, %zmm30
+
+// CHECK: vorpd  -8192(%rdx), %zmm22, %zmm30
+// CHECK:  encoding: [0x62,0x61,0xcd,0x40,0x56,0x72,0x80]
+          vorpd  -8192(%rdx), %zmm22, %zmm30
+
+// CHECK: vorpd  -8256(%rdx), %zmm22, %zmm30
+// CHECK:  encoding: [0x62,0x61,0xcd,0x40,0x56,0xb2,0xc0,0xdf,0xff,0xff]
+          vorpd  -8256(%rdx), %zmm22, %zmm30
+
+// CHECK: vorpd  1016(%rdx){1to8}, %zmm22, %zmm30
+// CHECK:  encoding: [0x62,0x61,0xcd,0x50,0x56,0x72,0x7f]
+          vorpd  1016(%rdx){1to8}, %zmm22, %zmm30
+
+// CHECK: vorpd  1024(%rdx){1to8}, %zmm22, %zmm30
+// CHECK:  encoding: [0x62,0x61,0xcd,0x50,0x56,0xb2,0x00,0x04,0x00,0x00]
+          vorpd  1024(%rdx){1to8}, %zmm22, %zmm30
+
+// CHECK: vorpd  -1024(%rdx){1to8}, %zmm22, %zmm30
+// CHECK:  encoding: [0x62,0x61,0xcd,0x50,0x56,0x72,0x80]
+          vorpd  -1024(%rdx){1to8}, %zmm22, %zmm30
+
+// CHECK: vorpd  -1032(%rdx){1to8}, %zmm22, %zmm30
+// CHECK:  encoding: [0x62,0x61,0xcd,0x50,0x56,0xb2,0xf8,0xfb,0xff,0xff]
+          vorpd  -1032(%rdx){1to8}, %zmm22, %zmm30
+
+// CHECK: vorps  %zmm26, %zmm21, %zmm22
+// CHECK:  encoding: [0x62,0x81,0x54,0x40,0x56,0xf2]
+          vorps  %zmm26, %zmm21, %zmm22
+
+// CHECK: vorps  %zmm26, %zmm21, %zmm22 {%k7}
+// CHECK:  encoding: [0x62,0x81,0x54,0x47,0x56,0xf2]
+          vorps  %zmm26, %zmm21, %zmm22 {%k7}
+
+// CHECK: vorps  %zmm26, %zmm21, %zmm22 {%k7} {z}
+// CHECK:  encoding: [0x62,0x81,0x54,0xc7,0x56,0xf2]
+          vorps  %zmm26, %zmm21, %zmm22 {%k7} {z}
+
+// CHECK: vorps  (%rcx), %zmm21, %zmm22
+// CHECK:  encoding: [0x62,0xe1,0x54,0x40,0x56,0x31]
+          vorps  (%rcx), %zmm21, %zmm22
+
+// CHECK: vorps  291(%rax,%r14,8), %zmm21, %zmm22
+// CHECK:  encoding: [0x62,0xa1,0x54,0x40,0x56,0xb4,0xf0,0x23,0x01,0x00,0x00]
+          vorps  291(%rax,%r14,8), %zmm21, %zmm22
+
+// CHECK: vorps  (%rcx){1to16}, %zmm21, %zmm22
+// CHECK:  encoding: [0x62,0xe1,0x54,0x50,0x56,0x31]
+          vorps  (%rcx){1to16}, %zmm21, %zmm22
+
+// CHECK: vorps  8128(%rdx), %zmm21, %zmm22
+// CHECK:  encoding: [0x62,0xe1,0x54,0x40,0x56,0x72,0x7f]
+          vorps  8128(%rdx), %zmm21, %zmm22
+
+// CHECK: vorps  8192(%rdx), %zmm21, %zmm22
+// CHECK:  encoding: [0x62,0xe1,0x54,0x40,0x56,0xb2,0x00,0x20,0x00,0x00]
+          vorps  8192(%rdx), %zmm21, %zmm22
+
+// CHECK: vorps  -8192(%rdx), %zmm21, %zmm22
+// CHECK:  encoding: [0x62,0xe1,0x54,0x40,0x56,0x72,0x80]
+          vorps  -8192(%rdx), %zmm21, %zmm22
+
+// CHECK: vorps  -8256(%rdx), %zmm21, %zmm22
+// CHECK:  encoding: [0x62,0xe1,0x54,0x40,0x56,0xb2,0xc0,0xdf,0xff,0xff]
+          vorps  -8256(%rdx), %zmm21, %zmm22
+
+// CHECK: vorps  508(%rdx){1to16}, %zmm21, %zmm22
+// CHECK:  encoding: [0x62,0xe1,0x54,0x50,0x56,0x72,0x7f]
+          vorps  508(%rdx){1to16}, %zmm21, %zmm22
+
+// CHECK: vorps  512(%rdx){1to16}, %zmm21, %zmm22
+// CHECK:  encoding: [0x62,0xe1,0x54,0x50,0x56,0xb2,0x00,0x02,0x00,0x00]
+          vorps  512(%rdx){1to16}, %zmm21, %zmm22
+
+// CHECK: vorps  -512(%rdx){1to16}, %zmm21, %zmm22
+// CHECK:  encoding: [0x62,0xe1,0x54,0x50,0x56,0x72,0x80]
+          vorps  -512(%rdx){1to16}, %zmm21, %zmm22
+
+// CHECK: vorps  -516(%rdx){1to16}, %zmm21, %zmm22
+// CHECK:  encoding: [0x62,0xe1,0x54,0x50,0x56,0xb2,0xfc,0xfd,0xff,0xff]
+          vorps  -516(%rdx){1to16}, %zmm21, %zmm22
+
+// CHECK: vxorpd %zmm24, %zmm24, %zmm27
+// CHECK:  encoding: [0x62,0x01,0xbd,0x40,0x57,0xd8]
+          vxorpd %zmm24, %zmm24, %zmm27
+
+// CHECK: vxorpd %zmm24, %zmm24, %zmm27 {%k5}
+// CHECK:  encoding: [0x62,0x01,0xbd,0x45,0x57,0xd8]
+          vxorpd %zmm24, %zmm24, %zmm27 {%k5}
+
+// CHECK: vxorpd %zmm24, %zmm24, %zmm27 {%k5} {z}
+// CHECK:  encoding: [0x62,0x01,0xbd,0xc5,0x57,0xd8]
+          vxorpd %zmm24, %zmm24, %zmm27 {%k5} {z}
+
+// CHECK: vxorpd (%rcx), %zmm24, %zmm27
+// CHECK:  encoding: [0x62,0x61,0xbd,0x40,0x57,0x19]
+          vxorpd (%rcx), %zmm24, %zmm27
+
+// CHECK: vxorpd 291(%rax,%r14,8), %zmm24, %zmm27
+// CHECK:  encoding: [0x62,0x21,0xbd,0x40,0x57,0x9c,0xf0,0x23,0x01,0x00,0x00]
+          vxorpd 291(%rax,%r14,8), %zmm24, %zmm27
+
+// CHECK: vxorpd (%rcx){1to8}, %zmm24, %zmm27
+// CHECK:  encoding: [0x62,0x61,0xbd,0x50,0x57,0x19]
+          vxorpd (%rcx){1to8}, %zmm24, %zmm27
+
+// CHECK: vxorpd 8128(%rdx), %zmm24, %zmm27
+// CHECK:  encoding: [0x62,0x61,0xbd,0x40,0x57,0x5a,0x7f]
+          vxorpd 8128(%rdx), %zmm24, %zmm27
+
+// CHECK: vxorpd 8192(%rdx), %zmm24, %zmm27
+// CHECK:  encoding: [0x62,0x61,0xbd,0x40,0x57,0x9a,0x00,0x20,0x00,0x00]
+          vxorpd 8192(%rdx), %zmm24, %zmm27
+
+// CHECK: vxorpd -8192(%rdx), %zmm24, %zmm27
+// CHECK:  encoding: [0x62,0x61,0xbd,0x40,0x57,0x5a,0x80]
+          vxorpd -8192(%rdx), %zmm24, %zmm27
+
+// CHECK: vxorpd -8256(%rdx), %zmm24, %zmm27
+// CHECK:  encoding: [0x62,0x61,0xbd,0x40,0x57,0x9a,0xc0,0xdf,0xff,0xff]
+          vxorpd -8256(%rdx), %zmm24, %zmm27
+
+// CHECK: vxorpd 1016(%rdx){1to8}, %zmm24, %zmm27
+// CHECK:  encoding: [0x62,0x61,0xbd,0x50,0x57,0x5a,0x7f]
+          vxorpd 1016(%rdx){1to8}, %zmm24, %zmm27
+
+// CHECK: vxorpd 1024(%rdx){1to8}, %zmm24, %zmm27
+// CHECK:  encoding: [0x62,0x61,0xbd,0x50,0x57,0x9a,0x00,0x04,0x00,0x00]
+          vxorpd 1024(%rdx){1to8}, %zmm24, %zmm27
+
+// CHECK: vxorpd -1024(%rdx){1to8}, %zmm24, %zmm27
+// CHECK:  encoding: [0x62,0x61,0xbd,0x50,0x57,0x5a,0x80]
+          vxorpd -1024(%rdx){1to8}, %zmm24, %zmm27
+
+// CHECK: vxorpd -1032(%rdx){1to8}, %zmm24, %zmm27
+// CHECK:  encoding: [0x62,0x61,0xbd,0x50,0x57,0x9a,0xf8,0xfb,0xff,0xff]
+          vxorpd -1032(%rdx){1to8}, %zmm24, %zmm27
+
+// CHECK: vxorps %zmm19, %zmm18, %zmm18
+// CHECK:  encoding: [0x62,0xa1,0x6c,0x40,0x57,0xd3]
+          vxorps %zmm19, %zmm18, %zmm18
+
+// CHECK: vxorps %zmm19, %zmm18, %zmm18 {%k2}
+// CHECK:  encoding: [0x62,0xa1,0x6c,0x42,0x57,0xd3]
+          vxorps %zmm19, %zmm18, %zmm18 {%k2}
+
+// CHECK: vxorps %zmm19, %zmm18, %zmm18 {%k2} {z}
+// CHECK:  encoding: [0x62,0xa1,0x6c,0xc2,0x57,0xd3]
+          vxorps %zmm19, %zmm18, %zmm18 {%k2} {z}
+
+// CHECK: vxorps (%rcx), %zmm18, %zmm18
+// CHECK:  encoding: [0x62,0xe1,0x6c,0x40,0x57,0x11]
+          vxorps (%rcx), %zmm18, %zmm18
+
+// CHECK: vxorps 291(%rax,%r14,8), %zmm18, %zmm18
+// CHECK:  encoding: [0x62,0xa1,0x6c,0x40,0x57,0x94,0xf0,0x23,0x01,0x00,0x00]
+          vxorps 291(%rax,%r14,8), %zmm18, %zmm18
+
+// CHECK: vxorps (%rcx){1to16}, %zmm18, %zmm18
+// CHECK:  encoding: [0x62,0xe1,0x6c,0x50,0x57,0x11]
+          vxorps (%rcx){1to16}, %zmm18, %zmm18
+
+// CHECK: vxorps 8128(%rdx), %zmm18, %zmm18
+// CHECK:  encoding: [0x62,0xe1,0x6c,0x40,0x57,0x52,0x7f]
+          vxorps 8128(%rdx), %zmm18, %zmm18
+
+// CHECK: vxorps 8192(%rdx), %zmm18, %zmm18
+// CHECK:  encoding: [0x62,0xe1,0x6c,0x40,0x57,0x92,0x00,0x20,0x00,0x00]
+          vxorps 8192(%rdx), %zmm18, %zmm18
+
+// CHECK: vxorps -8192(%rdx), %zmm18, %zmm18
+// CHECK:  encoding: [0x62,0xe1,0x6c,0x40,0x57,0x52,0x80]
+          vxorps -8192(%rdx), %zmm18, %zmm18
+
+// CHECK: vxorps -8256(%rdx), %zmm18, %zmm18
+// CHECK:  encoding: [0x62,0xe1,0x6c,0x40,0x57,0x92,0xc0,0xdf,0xff,0xff]
+          vxorps -8256(%rdx), %zmm18, %zmm18
+
+// CHECK: vxorps 508(%rdx){1to16}, %zmm18, %zmm18
+// CHECK:  encoding: [0x62,0xe1,0x6c,0x50,0x57,0x52,0x7f]
+          vxorps 508(%rdx){1to16}, %zmm18, %zmm18
+
+// CHECK: vxorps 512(%rdx){1to16}, %zmm18, %zmm18
+// CHECK:  encoding: [0x62,0xe1,0x6c,0x50,0x57,0x92,0x00,0x02,0x00,0x00]
+          vxorps 512(%rdx){1to16}, %zmm18, %zmm18
+
+// CHECK: vxorps -512(%rdx){1to16}, %zmm18, %zmm18
+// CHECK:  encoding: [0x62,0xe1,0x6c,0x50,0x57,0x52,0x80]
+          vxorps -512(%rdx){1to16}, %zmm18, %zmm18
+
+// CHECK: vxorps -516(%rdx){1to16}, %zmm18, %zmm18
+// CHECK:  encoding: [0x62,0xe1,0x6c,0x50,0x57,0x92,0xfc,0xfd,0xff,0xff]
+          vxorps -516(%rdx){1to16}, %zmm18, %zmm18
+// CHECK: vandpd %zmm22, %zmm22, %zmm24
+// CHECK:  encoding: [0x62,0x21,0xcd,0x40,0x54,0xc6]
+          vandpd %zmm22, %zmm22, %zmm24
+
+// CHECK: vandpd %zmm22, %zmm22, %zmm24 {%k4}
+// CHECK:  encoding: [0x62,0x21,0xcd,0x44,0x54,0xc6]
+          vandpd %zmm22, %zmm22, %zmm24 {%k4}
+
+// CHECK: vandpd %zmm22, %zmm22, %zmm24 {%k4} {z}
+// CHECK:  encoding: [0x62,0x21,0xcd,0xc4,0x54,0xc6]
+          vandpd %zmm22, %zmm22, %zmm24 {%k4} {z}
+
+// CHECK: vandpd (%rcx), %zmm22, %zmm24
+// CHECK:  encoding: [0x62,0x61,0xcd,0x40,0x54,0x01]
+          vandpd (%rcx), %zmm22, %zmm24
+
+// CHECK: vandpd 4660(%rax,%r14,8), %zmm22, %zmm24
+// CHECK:  encoding: [0x62,0x21,0xcd,0x40,0x54,0x84,0xf0,0x34,0x12,0x00,0x00]
+          vandpd 4660(%rax,%r14,8), %zmm22, %zmm24
+
+// CHECK: vandpd (%rcx){1to8}, %zmm22, %zmm24
+// CHECK:  encoding: [0x62,0x61,0xcd,0x50,0x54,0x01]
+          vandpd (%rcx){1to8}, %zmm22, %zmm24
+
+// CHECK: vandpd 8128(%rdx), %zmm22, %zmm24
+// CHECK:  encoding: [0x62,0x61,0xcd,0x40,0x54,0x42,0x7f]
+          vandpd 8128(%rdx), %zmm22, %zmm24
+
+// CHECK: vandpd 8192(%rdx), %zmm22, %zmm24
+// CHECK:  encoding: [0x62,0x61,0xcd,0x40,0x54,0x82,0x00,0x20,0x00,0x00]
+          vandpd 8192(%rdx), %zmm22, %zmm24
+
+// CHECK: vandpd -8192(%rdx), %zmm22, %zmm24
+// CHECK:  encoding: [0x62,0x61,0xcd,0x40,0x54,0x42,0x80]
+          vandpd -8192(%rdx), %zmm22, %zmm24
+
+// CHECK: vandpd -8256(%rdx), %zmm22, %zmm24
+// CHECK:  encoding: [0x62,0x61,0xcd,0x40,0x54,0x82,0xc0,0xdf,0xff,0xff]
+          vandpd -8256(%rdx), %zmm22, %zmm24
+
+// CHECK: vandpd 1016(%rdx){1to8}, %zmm22, %zmm24
+// CHECK:  encoding: [0x62,0x61,0xcd,0x50,0x54,0x42,0x7f]
+          vandpd 1016(%rdx){1to8}, %zmm22, %zmm24
+
+// CHECK: vandpd 1024(%rdx){1to8}, %zmm22, %zmm24
+// CHECK:  encoding: [0x62,0x61,0xcd,0x50,0x54,0x82,0x00,0x04,0x00,0x00]
+          vandpd 1024(%rdx){1to8}, %zmm22, %zmm24
+
+// CHECK: vandpd -1024(%rdx){1to8}, %zmm22, %zmm24
+// CHECK:  encoding: [0x62,0x61,0xcd,0x50,0x54,0x42,0x80]
+          vandpd -1024(%rdx){1to8}, %zmm22, %zmm24
+
+// CHECK: vandpd -1032(%rdx){1to8}, %zmm22, %zmm24
+// CHECK:  encoding: [0x62,0x61,0xcd,0x50,0x54,0x82,0xf8,0xfb,0xff,0xff]
+          vandpd -1032(%rdx){1to8}, %zmm22, %zmm24
+
+// CHECK: vandps %zmm23, %zmm23, %zmm30
+// CHECK:  encoding: [0x62,0x21,0x44,0x40,0x54,0xf7]
+          vandps %zmm23, %zmm23, %zmm30
+
+// CHECK: vandps %zmm23, %zmm23, %zmm30 {%k5}
+// CHECK:  encoding: [0x62,0x21,0x44,0x45,0x54,0xf7]
+          vandps %zmm23, %zmm23, %zmm30 {%k5}
+
+// CHECK: vandps %zmm23, %zmm23, %zmm30 {%k5} {z}
+// CHECK:  encoding: [0x62,0x21,0x44,0xc5,0x54,0xf7]
+          vandps %zmm23, %zmm23, %zmm30 {%k5} {z}
+
+// CHECK: vandps (%rcx), %zmm23, %zmm30
+// CHECK:  encoding: [0x62,0x61,0x44,0x40,0x54,0x31]
+          vandps (%rcx), %zmm23, %zmm30
+
+// CHECK: vandps 4660(%rax,%r14,8), %zmm23, %zmm30
+// CHECK:  encoding: [0x62,0x21,0x44,0x40,0x54,0xb4,0xf0,0x34,0x12,0x00,0x00]
+          vandps 4660(%rax,%r14,8), %zmm23, %zmm30
+
+// CHECK: vandps (%rcx){1to16}, %zmm23, %zmm30
+// CHECK:  encoding: [0x62,0x61,0x44,0x50,0x54,0x31]
+          vandps (%rcx){1to16}, %zmm23, %zmm30
+
+// CHECK: vandps 8128(%rdx), %zmm23, %zmm30
+// CHECK:  encoding: [0x62,0x61,0x44,0x40,0x54,0x72,0x7f]
+          vandps 8128(%rdx), %zmm23, %zmm30
+
+// CHECK: vandps 8192(%rdx), %zmm23, %zmm30
+// CHECK:  encoding: [0x62,0x61,0x44,0x40,0x54,0xb2,0x00,0x20,0x00,0x00]
+          vandps 8192(%rdx), %zmm23, %zmm30
+
+// CHECK: vandps -8192(%rdx), %zmm23, %zmm30
+// CHECK:  encoding: [0x62,0x61,0x44,0x40,0x54,0x72,0x80]
+          vandps -8192(%rdx), %zmm23, %zmm30
+
+// CHECK: vandps -8256(%rdx), %zmm23, %zmm30
+// CHECK:  encoding: [0x62,0x61,0x44,0x40,0x54,0xb2,0xc0,0xdf,0xff,0xff]
+          vandps -8256(%rdx), %zmm23, %zmm30
+
+// CHECK: vandps 508(%rdx){1to16}, %zmm23, %zmm30
+// CHECK:  encoding: [0x62,0x61,0x44,0x50,0x54,0x72,0x7f]
+          vandps 508(%rdx){1to16}, %zmm23, %zmm30
+
+// CHECK: vandps 512(%rdx){1to16}, %zmm23, %zmm30
+// CHECK:  encoding: [0x62,0x61,0x44,0x50,0x54,0xb2,0x00,0x02,0x00,0x00]
+          vandps 512(%rdx){1to16}, %zmm23, %zmm30
+
+// CHECK: vandps -512(%rdx){1to16}, %zmm23, %zmm30
+// CHECK:  encoding: [0x62,0x61,0x44,0x50,0x54,0x72,0x80]
+          vandps -512(%rdx){1to16}, %zmm23, %zmm30
+
+// CHECK: vandps -516(%rdx){1to16}, %zmm23, %zmm30
+// CHECK:  encoding: [0x62,0x61,0x44,0x50,0x54,0xb2,0xfc,0xfd,0xff,0xff]
+          vandps -516(%rdx){1to16}, %zmm23, %zmm30
+
+// CHECK: vandnpd %zmm21, %zmm21, %zmm25
+// CHECK:  encoding: [0x62,0x21,0xd5,0x40,0x55,0xcd]
+          vandnpd %zmm21, %zmm21, %zmm25
+
+// CHECK: vandnpd %zmm21, %zmm21, %zmm25 {%k2}
+// CHECK:  encoding: [0x62,0x21,0xd5,0x42,0x55,0xcd]
+          vandnpd %zmm21, %zmm21, %zmm25 {%k2}
+
+// CHECK: vandnpd %zmm21, %zmm21, %zmm25 {%k2} {z}
+// CHECK:  encoding: [0x62,0x21,0xd5,0xc2,0x55,0xcd]
+          vandnpd %zmm21, %zmm21, %zmm25 {%k2} {z}
+
+// CHECK: vandnpd (%rcx), %zmm21, %zmm25
+// CHECK:  encoding: [0x62,0x61,0xd5,0x40,0x55,0x09]
+          vandnpd (%rcx), %zmm21, %zmm25
+
+// CHECK: vandnpd 4660(%rax,%r14,8), %zmm21, %zmm25
+// CHECK:  encoding: [0x62,0x21,0xd5,0x40,0x55,0x8c,0xf0,0x34,0x12,0x00,0x00]
+          vandnpd 4660(%rax,%r14,8), %zmm21, %zmm25
+
+// CHECK: vandnpd (%rcx){1to8}, %zmm21, %zmm25
+// CHECK:  encoding: [0x62,0x61,0xd5,0x50,0x55,0x09]
+          vandnpd (%rcx){1to8}, %zmm21, %zmm25
+
+// CHECK: vandnpd 8128(%rdx), %zmm21, %zmm25
+// CHECK:  encoding: [0x62,0x61,0xd5,0x40,0x55,0x4a,0x7f]
+          vandnpd 8128(%rdx), %zmm21, %zmm25
+
+// CHECK: vandnpd 8192(%rdx), %zmm21, %zmm25
+// CHECK:  encoding: [0x62,0x61,0xd5,0x40,0x55,0x8a,0x00,0x20,0x00,0x00]
+          vandnpd 8192(%rdx), %zmm21, %zmm25
+
+// CHECK: vandnpd -8192(%rdx), %zmm21, %zmm25
+// CHECK:  encoding: [0x62,0x61,0xd5,0x40,0x55,0x4a,0x80]
+          vandnpd -8192(%rdx), %zmm21, %zmm25
+
+// CHECK: vandnpd -8256(%rdx), %zmm21, %zmm25
+// CHECK:  encoding: [0x62,0x61,0xd5,0x40,0x55,0x8a,0xc0,0xdf,0xff,0xff]
+          vandnpd -8256(%rdx), %zmm21, %zmm25
+
+// CHECK: vandnpd 1016(%rdx){1to8}, %zmm21, %zmm25
+// CHECK:  encoding: [0x62,0x61,0xd5,0x50,0x55,0x4a,0x7f]
+          vandnpd 1016(%rdx){1to8}, %zmm21, %zmm25
+
+// CHECK: vandnpd 1024(%rdx){1to8}, %zmm21, %zmm25
+// CHECK:  encoding: [0x62,0x61,0xd5,0x50,0x55,0x8a,0x00,0x04,0x00,0x00]
+          vandnpd 1024(%rdx){1to8}, %zmm21, %zmm25
+
+// CHECK: vandnpd -1024(%rdx){1to8}, %zmm21, %zmm25
+// CHECK:  encoding: [0x62,0x61,0xd5,0x50,0x55,0x4a,0x80]
+          vandnpd -1024(%rdx){1to8}, %zmm21, %zmm25
+
+// CHECK: vandnpd -1032(%rdx){1to8}, %zmm21, %zmm25
+// CHECK:  encoding: [0x62,0x61,0xd5,0x50,0x55,0x8a,0xf8,0xfb,0xff,0xff]
+          vandnpd -1032(%rdx){1to8}, %zmm21, %zmm25
+
+// CHECK: vandnps %zmm18, %zmm21, %zmm17
+// CHECK:  encoding: [0x62,0xa1,0x54,0x40,0x55,0xca]
+          vandnps %zmm18, %zmm21, %zmm17
+
+// CHECK: vandnps %zmm18, %zmm21, %zmm17 {%k1}
+// CHECK:  encoding: [0x62,0xa1,0x54,0x41,0x55,0xca]
+          vandnps %zmm18, %zmm21, %zmm17 {%k1}
+
+// CHECK: vandnps %zmm18, %zmm21, %zmm17 {%k1} {z}
+// CHECK:  encoding: [0x62,0xa1,0x54,0xc1,0x55,0xca]
+          vandnps %zmm18, %zmm21, %zmm17 {%k1} {z}
+
+// CHECK: vandnps (%rcx), %zmm21, %zmm17
+// CHECK:  encoding: [0x62,0xe1,0x54,0x40,0x55,0x09]
+          vandnps (%rcx), %zmm21, %zmm17
+
+// CHECK: vandnps 4660(%rax,%r14,8), %zmm21, %zmm17
+// CHECK:  encoding: [0x62,0xa1,0x54,0x40,0x55,0x8c,0xf0,0x34,0x12,0x00,0x00]
+          vandnps 4660(%rax,%r14,8), %zmm21, %zmm17
+
+// CHECK: vandnps (%rcx){1to16}, %zmm21, %zmm17
+// CHECK:  encoding: [0x62,0xe1,0x54,0x50,0x55,0x09]
+          vandnps (%rcx){1to16}, %zmm21, %zmm17
+
+// CHECK: vandnps 8128(%rdx), %zmm21, %zmm17
+// CHECK:  encoding: [0x62,0xe1,0x54,0x40,0x55,0x4a,0x7f]
+          vandnps 8128(%rdx), %zmm21, %zmm17
+
+// CHECK: vandnps 8192(%rdx), %zmm21, %zmm17
+// CHECK:  encoding: [0x62,0xe1,0x54,0x40,0x55,0x8a,0x00,0x20,0x00,0x00]
+          vandnps 8192(%rdx), %zmm21, %zmm17
+
+// CHECK: vandnps -8192(%rdx), %zmm21, %zmm17
+// CHECK:  encoding: [0x62,0xe1,0x54,0x40,0x55,0x4a,0x80]
+          vandnps -8192(%rdx), %zmm21, %zmm17
+
+// CHECK: vandnps -8256(%rdx), %zmm21, %zmm17
+// CHECK:  encoding: [0x62,0xe1,0x54,0x40,0x55,0x8a,0xc0,0xdf,0xff,0xff]
+          vandnps -8256(%rdx), %zmm21, %zmm17
+
+// CHECK: vandnps 508(%rdx){1to16}, %zmm21, %zmm17
+// CHECK:  encoding: [0x62,0xe1,0x54,0x50,0x55,0x4a,0x7f]
+          vandnps 508(%rdx){1to16}, %zmm21, %zmm17
+
+// CHECK: vandnps 512(%rdx){1to16}, %zmm21, %zmm17
+// CHECK:  encoding: [0x62,0xe1,0x54,0x50,0x55,0x8a,0x00,0x02,0x00,0x00]
+          vandnps 512(%rdx){1to16}, %zmm21, %zmm17
+
+// CHECK: vandnps -512(%rdx){1to16}, %zmm21, %zmm17
+// CHECK:  encoding: [0x62,0xe1,0x54,0x50,0x55,0x4a,0x80]
+          vandnps -512(%rdx){1to16}, %zmm21, %zmm17
+
+// CHECK: vandnps -516(%rdx){1to16}, %zmm21, %zmm17
+// CHECK:  encoding: [0x62,0xe1,0x54,0x50,0x55,0x8a,0xfc,0xfd,0xff,0xff]
+          vandnps -516(%rdx){1to16}, %zmm21, %zmm17
+
+// CHECK: vorpd  %zmm24, %zmm28, %zmm18
+// CHECK:  encoding: [0x62,0x81,0x9d,0x40,0x56,0xd0]
+          vorpd  %zmm24, %zmm28, %zmm18
+
+// CHECK: vorpd  %zmm24, %zmm28, %zmm18 {%k1}
+// CHECK:  encoding: [0x62,0x81,0x9d,0x41,0x56,0xd0]
+          vorpd  %zmm24, %zmm28, %zmm18 {%k1}
+
+// CHECK: vorpd  %zmm24, %zmm28, %zmm18 {%k1} {z}
+// CHECK:  encoding: [0x62,0x81,0x9d,0xc1,0x56,0xd0]
+          vorpd  %zmm24, %zmm28, %zmm18 {%k1} {z}
+
+// CHECK: vorpd  (%rcx), %zmm28, %zmm18
+// CHECK:  encoding: [0x62,0xe1,0x9d,0x40,0x56,0x11]
+          vorpd  (%rcx), %zmm28, %zmm18
+
+// CHECK: vorpd  4660(%rax,%r14,8), %zmm28, %zmm18
+// CHECK:  encoding: [0x62,0xa1,0x9d,0x40,0x56,0x94,0xf0,0x34,0x12,0x00,0x00]
+          vorpd  4660(%rax,%r14,8), %zmm28, %zmm18
+
+// CHECK: vorpd  (%rcx){1to8}, %zmm28, %zmm18
+// CHECK:  encoding: [0x62,0xe1,0x9d,0x50,0x56,0x11]
+          vorpd  (%rcx){1to8}, %zmm28, %zmm18
+
+// CHECK: vorpd  8128(%rdx), %zmm28, %zmm18
+// CHECK:  encoding: [0x62,0xe1,0x9d,0x40,0x56,0x52,0x7f]
+          vorpd  8128(%rdx), %zmm28, %zmm18
+
+// CHECK: vorpd  8192(%rdx), %zmm28, %zmm18
+// CHECK:  encoding: [0x62,0xe1,0x9d,0x40,0x56,0x92,0x00,0x20,0x00,0x00]
+          vorpd  8192(%rdx), %zmm28, %zmm18
+
+// CHECK: vorpd  -8192(%rdx), %zmm28, %zmm18
+// CHECK:  encoding: [0x62,0xe1,0x9d,0x40,0x56,0x52,0x80]
+          vorpd  -8192(%rdx), %zmm28, %zmm18
+
+// CHECK: vorpd  -8256(%rdx), %zmm28, %zmm18
+// CHECK:  encoding: [0x62,0xe1,0x9d,0x40,0x56,0x92,0xc0,0xdf,0xff,0xff]
+          vorpd  -8256(%rdx), %zmm28, %zmm18
+
+// CHECK: vorpd  1016(%rdx){1to8}, %zmm28, %zmm18
+// CHECK:  encoding: [0x62,0xe1,0x9d,0x50,0x56,0x52,0x7f]
+          vorpd  1016(%rdx){1to8}, %zmm28, %zmm18
+
+// CHECK: vorpd  1024(%rdx){1to8}, %zmm28, %zmm18
+// CHECK:  encoding: [0x62,0xe1,0x9d,0x50,0x56,0x92,0x00,0x04,0x00,0x00]
+          vorpd  1024(%rdx){1to8}, %zmm28, %zmm18
+
+// CHECK: vorpd  -1024(%rdx){1to8}, %zmm28, %zmm18
+// CHECK:  encoding: [0x62,0xe1,0x9d,0x50,0x56,0x52,0x80]
+          vorpd  -1024(%rdx){1to8}, %zmm28, %zmm18
+
+// CHECK: vorpd  -1032(%rdx){1to8}, %zmm28, %zmm18
+// CHECK:  encoding: [0x62,0xe1,0x9d,0x50,0x56,0x92,0xf8,0xfb,0xff,0xff]
+          vorpd  -1032(%rdx){1to8}, %zmm28, %zmm18
+
+// CHECK: vorps  %zmm23, %zmm17, %zmm28
+// CHECK:  encoding: [0x62,0x21,0x74,0x40,0x56,0xe7]
+          vorps  %zmm23, %zmm17, %zmm28
+
+// CHECK: vorps  %zmm23, %zmm17, %zmm28 {%k7}
+// CHECK:  encoding: [0x62,0x21,0x74,0x47,0x56,0xe7]
+          vorps  %zmm23, %zmm17, %zmm28 {%k7}
+
+// CHECK: vorps  %zmm23, %zmm17, %zmm28 {%k7} {z}
+// CHECK:  encoding: [0x62,0x21,0x74,0xc7,0x56,0xe7]
+          vorps  %zmm23, %zmm17, %zmm28 {%k7} {z}
+
+// CHECK: vorps  (%rcx), %zmm17, %zmm28
+// CHECK:  encoding: [0x62,0x61,0x74,0x40,0x56,0x21]
+          vorps  (%rcx), %zmm17, %zmm28
+
+// CHECK: vorps  4660(%rax,%r14,8), %zmm17, %zmm28
+// CHECK:  encoding: [0x62,0x21,0x74,0x40,0x56,0xa4,0xf0,0x34,0x12,0x00,0x00]
+          vorps  4660(%rax,%r14,8), %zmm17, %zmm28
+
+// CHECK: vorps  (%rcx){1to16}, %zmm17, %zmm28
+// CHECK:  encoding: [0x62,0x61,0x74,0x50,0x56,0x21]
+          vorps  (%rcx){1to16}, %zmm17, %zmm28
+
+// CHECK: vorps  8128(%rdx), %zmm17, %zmm28
+// CHECK:  encoding: [0x62,0x61,0x74,0x40,0x56,0x62,0x7f]
+          vorps  8128(%rdx), %zmm17, %zmm28
+
+// CHECK: vorps  8192(%rdx), %zmm17, %zmm28
+// CHECK:  encoding: [0x62,0x61,0x74,0x40,0x56,0xa2,0x00,0x20,0x00,0x00]
+          vorps  8192(%rdx), %zmm17, %zmm28
+
+// CHECK: vorps  -8192(%rdx), %zmm17, %zmm28
+// CHECK:  encoding: [0x62,0x61,0x74,0x40,0x56,0x62,0x80]
+          vorps  -8192(%rdx), %zmm17, %zmm28
+
+// CHECK: vorps  -8256(%rdx), %zmm17, %zmm28
+// CHECK:  encoding: [0x62,0x61,0x74,0x40,0x56,0xa2,0xc0,0xdf,0xff,0xff]
+          vorps  -8256(%rdx), %zmm17, %zmm28
+
+// CHECK: vorps  508(%rdx){1to16}, %zmm17, %zmm28
+// CHECK:  encoding: [0x62,0x61,0x74,0x50,0x56,0x62,0x7f]
+          vorps  508(%rdx){1to16}, %zmm17, %zmm28
+
+// CHECK: vorps  512(%rdx){1to16}, %zmm17, %zmm28
+// CHECK:  encoding: [0x62,0x61,0x74,0x50,0x56,0xa2,0x00,0x02,0x00,0x00]
+          vorps  512(%rdx){1to16}, %zmm17, %zmm28
+
+// CHECK: vorps  -512(%rdx){1to16}, %zmm17, %zmm28
+// CHECK:  encoding: [0x62,0x61,0x74,0x50,0x56,0x62,0x80]
+          vorps  -512(%rdx){1to16}, %zmm17, %zmm28
+
+// CHECK: vorps  -516(%rdx){1to16}, %zmm17, %zmm28
+// CHECK:  encoding: [0x62,0x61,0x74,0x50,0x56,0xa2,0xfc,0xfd,0xff,0xff]
+          vorps  -516(%rdx){1to16}, %zmm17, %zmm28
+
+// CHECK: vxorpd %zmm27, %zmm18, %zmm28
+// CHECK:  encoding: [0x62,0x01,0xed,0x40,0x57,0xe3]
+          vxorpd %zmm27, %zmm18, %zmm28
+
+// CHECK: vxorpd %zmm27, %zmm18, %zmm28 {%k4}
+// CHECK:  encoding: [0x62,0x01,0xed,0x44,0x57,0xe3]
+          vxorpd %zmm27, %zmm18, %zmm28 {%k4}
+
+// CHECK: vxorpd %zmm27, %zmm18, %zmm28 {%k4} {z}
+// CHECK:  encoding: [0x62,0x01,0xed,0xc4,0x57,0xe3]
+          vxorpd %zmm27, %zmm18, %zmm28 {%k4} {z}
+
+// CHECK: vxorpd (%rcx), %zmm18, %zmm28
+// CHECK:  encoding: [0x62,0x61,0xed,0x40,0x57,0x21]
+          vxorpd (%rcx), %zmm18, %zmm28
+
+// CHECK: vxorpd 4660(%rax,%r14,8), %zmm18, %zmm28
+// CHECK:  encoding: [0x62,0x21,0xed,0x40,0x57,0xa4,0xf0,0x34,0x12,0x00,0x00]
+          vxorpd 4660(%rax,%r14,8), %zmm18, %zmm28
+
+// CHECK: vxorpd (%rcx){1to8}, %zmm18, %zmm28
+// CHECK:  encoding: [0x62,0x61,0xed,0x50,0x57,0x21]
+          vxorpd (%rcx){1to8}, %zmm18, %zmm28
+
+// CHECK: vxorpd 8128(%rdx), %zmm18, %zmm28
+// CHECK:  encoding: [0x62,0x61,0xed,0x40,0x57,0x62,0x7f]
+          vxorpd 8128(%rdx), %zmm18, %zmm28
+
+// CHECK: vxorpd 8192(%rdx), %zmm18, %zmm28
+// CHECK:  encoding: [0x62,0x61,0xed,0x40,0x57,0xa2,0x00,0x20,0x00,0x00]
+          vxorpd 8192(%rdx), %zmm18, %zmm28
+
+// CHECK: vxorpd -8192(%rdx), %zmm18, %zmm28
+// CHECK:  encoding: [0x62,0x61,0xed,0x40,0x57,0x62,0x80]
+          vxorpd -8192(%rdx), %zmm18, %zmm28
+
+// CHECK: vxorpd -8256(%rdx), %zmm18, %zmm28
+// CHECK:  encoding: [0x62,0x61,0xed,0x40,0x57,0xa2,0xc0,0xdf,0xff,0xff]
+          vxorpd -8256(%rdx), %zmm18, %zmm28
+
+// CHECK: vxorpd 1016(%rdx){1to8}, %zmm18, %zmm28
+// CHECK:  encoding: [0x62,0x61,0xed,0x50,0x57,0x62,0x7f]
+          vxorpd 1016(%rdx){1to8}, %zmm18, %zmm28
+
+// CHECK: vxorpd 1024(%rdx){1to8}, %zmm18, %zmm28
+// CHECK:  encoding: [0x62,0x61,0xed,0x50,0x57,0xa2,0x00,0x04,0x00,0x00]
+          vxorpd 1024(%rdx){1to8}, %zmm18, %zmm28
+
+// CHECK: vxorpd -1024(%rdx){1to8}, %zmm18, %zmm28
+// CHECK:  encoding: [0x62,0x61,0xed,0x50,0x57,0x62,0x80]
+          vxorpd -1024(%rdx){1to8}, %zmm18, %zmm28
+
+// CHECK: vxorpd -1032(%rdx){1to8}, %zmm18, %zmm28
+// CHECK:  encoding: [0x62,0x61,0xed,0x50,0x57,0xa2,0xf8,0xfb,0xff,0xff]
+          vxorpd -1032(%rdx){1to8}, %zmm18, %zmm28
+
+// CHECK: vxorps %zmm18, %zmm28, %zmm24
+// CHECK:  encoding: [0x62,0x21,0x1c,0x40,0x57,0xc2]
+          vxorps %zmm18, %zmm28, %zmm24
+
+// CHECK: vxorps %zmm18, %zmm28, %zmm24 {%k4}
+// CHECK:  encoding: [0x62,0x21,0x1c,0x44,0x57,0xc2]
+          vxorps %zmm18, %zmm28, %zmm24 {%k4}
+
+// CHECK: vxorps %zmm18, %zmm28, %zmm24 {%k4} {z}
+// CHECK:  encoding: [0x62,0x21,0x1c,0xc4,0x57,0xc2]
+          vxorps %zmm18, %zmm28, %zmm24 {%k4} {z}
+
+// CHECK: vxorps (%rcx), %zmm28, %zmm24
+// CHECK:  encoding: [0x62,0x61,0x1c,0x40,0x57,0x01]
+          vxorps (%rcx), %zmm28, %zmm24
+
+// CHECK: vxorps 4660(%rax,%r14,8), %zmm28, %zmm24
+// CHECK:  encoding: [0x62,0x21,0x1c,0x40,0x57,0x84,0xf0,0x34,0x12,0x00,0x00]
+          vxorps 4660(%rax,%r14,8), %zmm28, %zmm24
+
+// CHECK: vxorps (%rcx){1to16}, %zmm28, %zmm24
+// CHECK:  encoding: [0x62,0x61,0x1c,0x50,0x57,0x01]
+          vxorps (%rcx){1to16}, %zmm28, %zmm24
+
+// CHECK: vxorps 8128(%rdx), %zmm28, %zmm24
+// CHECK:  encoding: [0x62,0x61,0x1c,0x40,0x57,0x42,0x7f]
+          vxorps 8128(%rdx), %zmm28, %zmm24
+
+// CHECK: vxorps 8192(%rdx), %zmm28, %zmm24
+// CHECK:  encoding: [0x62,0x61,0x1c,0x40,0x57,0x82,0x00,0x20,0x00,0x00]
+          vxorps 8192(%rdx), %zmm28, %zmm24
+
+// CHECK: vxorps -8192(%rdx), %zmm28, %zmm24
+// CHECK:  encoding: [0x62,0x61,0x1c,0x40,0x57,0x42,0x80]
+          vxorps -8192(%rdx), %zmm28, %zmm24
+
+// CHECK: vxorps -8256(%rdx), %zmm28, %zmm24
+// CHECK:  encoding: [0x62,0x61,0x1c,0x40,0x57,0x82,0xc0,0xdf,0xff,0xff]
+          vxorps -8256(%rdx), %zmm28, %zmm24
+
+// CHECK: vxorps 508(%rdx){1to16}, %zmm28, %zmm24
+// CHECK:  encoding: [0x62,0x61,0x1c,0x50,0x57,0x42,0x7f]
+          vxorps 508(%rdx){1to16}, %zmm28, %zmm24
+
+// CHECK: vxorps 512(%rdx){1to16}, %zmm28, %zmm24
+// CHECK:  encoding: [0x62,0x61,0x1c,0x50,0x57,0x82,0x00,0x02,0x00,0x00]
+          vxorps 512(%rdx){1to16}, %zmm28, %zmm24
+
+// CHECK: vxorps -512(%rdx){1to16}, %zmm28, %zmm24
+// CHECK:  encoding: [0x62,0x61,0x1c,0x50,0x57,0x42,0x80]
+          vxorps -512(%rdx){1to16}, %zmm28, %zmm24
+
+// CHECK: vxorps -516(%rdx){1to16}, %zmm28, %zmm24
+// CHECK:  encoding: [0x62,0x61,0x1c,0x50,0x57,0x82,0xfc,0xfd,0xff,0xff]
+          vxorps -516(%rdx){1to16}, %zmm28, %zmm24
+
+// CHECK: vinserti32x8
+// CHECK: encoding: [0x62,0xd3,0x4d,0x40,0x3a,0xdb,0x01]
+          vinserti32x8  $1, %ymm11, %zmm22, %zmm3
+
+// CHECK: vinsertf64x2
+// CHECK: encoding: [0x62,0xf3,0xed,0x48,0x18,0x4f,0x10,0x01]
+          vinsertf64x2  $1, 256(%rdi), %zmm2, %zmm1
+
+// CHECK: vbroadcastf32x8 (%rcx), %zmm30
+// CHECK:  encoding: [0x62,0x62,0x7d,0x48,0x1b,0x31]
+          vbroadcastf32x8 (%rcx), %zmm30
+
+// CHECK: vbroadcastf32x8 (%rcx), %zmm30 {%k3}
+// CHECK:  encoding: [0x62,0x62,0x7d,0x4b,0x1b,0x31]
+          vbroadcastf32x8 (%rcx), %zmm30 {%k3}
+
+// CHECK: vbroadcastf32x8 (%rcx), %zmm30 {%k3} {z}
+// CHECK:  encoding: [0x62,0x62,0x7d,0xcb,0x1b,0x31]
+          vbroadcastf32x8 (%rcx), %zmm30 {%k3} {z}
+
+// CHECK: vbroadcastf32x8 291(%rax,%r14,8), %zmm30
+// CHECK:  encoding: [0x62,0x22,0x7d,0x48,0x1b,0xb4,0xf0,0x23,0x01,0x00,0x00]
+          vbroadcastf32x8 291(%rax,%r14,8), %zmm30
+
+// CHECK: vbroadcastf32x8 4064(%rdx), %zmm30
+// CHECK:  encoding: [0x62,0x62,0x7d,0x48,0x1b,0x72,0x7f]
+          vbroadcastf32x8 4064(%rdx), %zmm30
+
+// CHECK: vbroadcastf32x8 4096(%rdx), %zmm30
+// CHECK:  encoding: [0x62,0x62,0x7d,0x48,0x1b,0xb2,0x00,0x10,0x00,0x00]
+          vbroadcastf32x8 4096(%rdx), %zmm30
+
+// CHECK: vbroadcastf32x8 -4096(%rdx), %zmm30
+// CHECK:  encoding: [0x62,0x62,0x7d,0x48,0x1b,0x72,0x80]
+          vbroadcastf32x8 -4096(%rdx), %zmm30
+
+// CHECK: vbroadcastf32x8 -4128(%rdx), %zmm30
+// CHECK:  encoding: [0x62,0x62,0x7d,0x48,0x1b,0xb2,0xe0,0xef,0xff,0xff]
+          vbroadcastf32x8 -4128(%rdx), %zmm30
+
+// CHECK: vbroadcastf64x2 (%rcx), %zmm28
+// CHECK:  encoding: [0x62,0x62,0xfd,0x48,0x1a,0x21]
+          vbroadcastf64x2 (%rcx), %zmm28
+
+// CHECK: vbroadcastf64x2 (%rcx), %zmm28 {%k4}
+// CHECK:  encoding: [0x62,0x62,0xfd,0x4c,0x1a,0x21]
+          vbroadcastf64x2 (%rcx), %zmm28 {%k4}
+
+// CHECK: vbroadcastf64x2 (%rcx), %zmm28 {%k4} {z}
+// CHECK:  encoding: [0x62,0x62,0xfd,0xcc,0x1a,0x21]
+          vbroadcastf64x2 (%rcx), %zmm28 {%k4} {z}
+
+// CHECK: vbroadcastf64x2 291(%rax,%r14,8), %zmm28
+// CHECK:  encoding: [0x62,0x22,0xfd,0x48,0x1a,0xa4,0xf0,0x23,0x01,0x00,0x00]
+          vbroadcastf64x2 291(%rax,%r14,8), %zmm28
+
+// CHECK: vbroadcastf64x2 2032(%rdx), %zmm28
+// CHECK:  encoding: [0x62,0x62,0xfd,0x48,0x1a,0x62,0x7f]
+          vbroadcastf64x2 2032(%rdx), %zmm28
+
+// CHECK: vbroadcastf64x2 2048(%rdx), %zmm28
+// CHECK:  encoding: [0x62,0x62,0xfd,0x48,0x1a,0xa2,0x00,0x08,0x00,0x00]
+          vbroadcastf64x2 2048(%rdx), %zmm28
+
+// CHECK: vbroadcastf64x2 -2048(%rdx), %zmm28
+// CHECK:  encoding: [0x62,0x62,0xfd,0x48,0x1a,0x62,0x80]
+          vbroadcastf64x2 -2048(%rdx), %zmm28
+
+// CHECK: vbroadcastf64x2 -2064(%rdx), %zmm28
+// CHECK:  encoding: [0x62,0x62,0xfd,0x48,0x1a,0xa2,0xf0,0xf7,0xff,0xff]
+          vbroadcastf64x2 -2064(%rdx), %zmm28
+
+// CHECK: vbroadcasti32x8 (%rcx), %zmm29
+// CHECK:  encoding: [0x62,0x62,0x7d,0x48,0x5b,0x29]
+          vbroadcasti32x8 (%rcx), %zmm29
+
+// CHECK: vbroadcasti32x8 (%rcx), %zmm29 {%k5}
+// CHECK:  encoding: [0x62,0x62,0x7d,0x4d,0x5b,0x29]
+          vbroadcasti32x8 (%rcx), %zmm29 {%k5}
+
+// CHECK: vbroadcasti32x8 (%rcx), %zmm29 {%k5} {z}
+// CHECK:  encoding: [0x62,0x62,0x7d,0xcd,0x5b,0x29]
+          vbroadcasti32x8 (%rcx), %zmm29 {%k5} {z}
+
+// CHECK: vbroadcasti32x8 291(%rax,%r14,8), %zmm29
+// CHECK:  encoding: [0x62,0x22,0x7d,0x48,0x5b,0xac,0xf0,0x23,0x01,0x00,0x00]
+          vbroadcasti32x8 291(%rax,%r14,8), %zmm29
+
+// CHECK: vbroadcasti32x8 4064(%rdx), %zmm29
+// CHECK:  encoding: [0x62,0x62,0x7d,0x48,0x5b,0x6a,0x7f]
+          vbroadcasti32x8 4064(%rdx), %zmm29
+
+// CHECK: vbroadcasti32x8 4096(%rdx), %zmm29
+// CHECK:  encoding: [0x62,0x62,0x7d,0x48,0x5b,0xaa,0x00,0x10,0x00,0x00]
+          vbroadcasti32x8 4096(%rdx), %zmm29
+
+// CHECK: vbroadcasti32x8 -4096(%rdx), %zmm29
+// CHECK:  encoding: [0x62,0x62,0x7d,0x48,0x5b,0x6a,0x80]
+          vbroadcasti32x8 -4096(%rdx), %zmm29
+
+// CHECK: vbroadcasti32x8 -4128(%rdx), %zmm29
+// CHECK:  encoding: [0x62,0x62,0x7d,0x48,0x5b,0xaa,0xe0,0xef,0xff,0xff]
+          vbroadcasti32x8 -4128(%rdx), %zmm29
+
+// CHECK: vbroadcasti64x2 (%rcx), %zmm20
+// CHECK:  encoding: [0x62,0xe2,0xfd,0x48,0x5a,0x21]
+          vbroadcasti64x2 (%rcx), %zmm20
+
+// CHECK: vbroadcasti64x2 (%rcx), %zmm20 {%k3}
+// CHECK:  encoding: [0x62,0xe2,0xfd,0x4b,0x5a,0x21]
+          vbroadcasti64x2 (%rcx), %zmm20 {%k3}
+
+// CHECK: vbroadcasti64x2 (%rcx), %zmm20 {%k3} {z}
+// CHECK:  encoding: [0x62,0xe2,0xfd,0xcb,0x5a,0x21]
+          vbroadcasti64x2 (%rcx), %zmm20 {%k3} {z}
+
+// CHECK: vbroadcasti64x2 291(%rax,%r14,8), %zmm20
+// CHECK:  encoding: [0x62,0xa2,0xfd,0x48,0x5a,0xa4,0xf0,0x23,0x01,0x00,0x00]
+          vbroadcasti64x2 291(%rax,%r14,8), %zmm20
+
+// CHECK: vbroadcasti64x2 2032(%rdx), %zmm20
+// CHECK:  encoding: [0x62,0xe2,0xfd,0x48,0x5a,0x62,0x7f]
+          vbroadcasti64x2 2032(%rdx), %zmm20
+
+// CHECK: vbroadcasti64x2 2048(%rdx), %zmm20
+// CHECK:  encoding: [0x62,0xe2,0xfd,0x48,0x5a,0xa2,0x00,0x08,0x00,0x00]
+          vbroadcasti64x2 2048(%rdx), %zmm20
+
+// CHECK: vbroadcasti64x2 -2048(%rdx), %zmm20
+// CHECK:  encoding: [0x62,0xe2,0xfd,0x48,0x5a,0x62,0x80]
+          vbroadcasti64x2 -2048(%rdx), %zmm20
+
+// CHECK: vbroadcasti64x2 -2064(%rdx), %zmm20
+// CHECK:  encoding: [0x62,0xe2,0xfd,0x48,0x5a,0xa2,0xf0,0xf7,0xff,0xff]
+          vbroadcasti64x2 -2064(%rdx), %zmm20
+
+// CHECK: vrangepd $171, %zmm17, %zmm19, %zmm17
+// CHECK:  encoding: [0x62,0xa3,0xe5,0x40,0x50,0xc9,0xab]
+          vrangepd $0xab, %zmm17, %zmm19, %zmm17
+
+// CHECK: vrangepd $171, %zmm17, %zmm19, %zmm17 {%k6}
+// CHECK:  encoding: [0x62,0xa3,0xe5,0x46,0x50,0xc9,0xab]
+          vrangepd $0xab, %zmm17, %zmm19, %zmm17 {%k6}
+
+// CHECK: vrangepd $171, %zmm17, %zmm19, %zmm17 {%k6} {z}
+// CHECK:  encoding: [0x62,0xa3,0xe5,0xc6,0x50,0xc9,0xab]
+          vrangepd $0xab, %zmm17, %zmm19, %zmm17 {%k6} {z}
+
+// CHECK: vrangepd $171,{sae}, %zmm17, %zmm19, %zmm17
+// CHECK:  encoding: [0x62,0xa3,0xe5,0x10,0x50,0xc9,0xab]
+          vrangepd $0xab,{sae}, %zmm17, %zmm19, %zmm17
+
+// CHECK: vrangepd $123, %zmm17, %zmm19, %zmm17
+// CHECK:  encoding: [0x62,0xa3,0xe5,0x40,0x50,0xc9,0x7b]
+          vrangepd $0x7b, %zmm17, %zmm19, %zmm17
+
+// CHECK: vrangepd $123,{sae}, %zmm17, %zmm19, %zmm17
+// CHECK:  encoding: [0x62,0xa3,0xe5,0x10,0x50,0xc9,0x7b]
+          vrangepd $0x7b,{sae}, %zmm17, %zmm19, %zmm17
+
+// CHECK: vrangepd $123, (%rcx), %zmm19, %zmm17
+// CHECK:  encoding: [0x62,0xe3,0xe5,0x40,0x50,0x09,0x7b]
+          vrangepd $0x7b,(%rcx), %zmm19, %zmm17
+
+// CHECK: vrangepd $123, 291(%rax,%r14,8), %zmm19, %zmm17
+// CHECK:  encoding: [0x62,0xa3,0xe5,0x40,0x50,0x8c,0xf0,0x23,0x01,0x00,0x00,0x7b]
+          vrangepd $0x7b,291(%rax,%r14,8), %zmm19, %zmm17
+
+// CHECK: vrangepd $123, (%rcx){1to8}, %zmm19, %zmm17
+// CHECK:  encoding: [0x62,0xe3,0xe5,0x50,0x50,0x09,0x7b]
+          vrangepd $0x7b,(%rcx){1to8}, %zmm19, %zmm17
+
+// CHECK: vrangepd $123, 8128(%rdx), %zmm19, %zmm17
+// CHECK:  encoding: [0x62,0xe3,0xe5,0x40,0x50,0x4a,0x7f,0x7b]
+          vrangepd $0x7b,8128(%rdx), %zmm19, %zmm17
+
+// CHECK: vrangepd $123, 8192(%rdx), %zmm19, %zmm17
+// CHECK:  encoding: [0x62,0xe3,0xe5,0x40,0x50,0x8a,0x00,0x20,0x00,0x00,0x7b]
+          vrangepd $0x7b,8192(%rdx), %zmm19, %zmm17
+
+// CHECK: vrangepd $123, -8192(%rdx), %zmm19, %zmm17
+// CHECK:  encoding: [0x62,0xe3,0xe5,0x40,0x50,0x4a,0x80,0x7b]
+          vrangepd $0x7b,-8192(%rdx), %zmm19, %zmm17
+
+// CHECK: vrangepd $123, -8256(%rdx), %zmm19, %zmm17
+// CHECK:  encoding: [0x62,0xe3,0xe5,0x40,0x50,0x8a,0xc0,0xdf,0xff,0xff,0x7b]
+          vrangepd $0x7b,-8256(%rdx), %zmm19, %zmm17
+
+// CHECK: vrangepd $123, 1016(%rdx){1to8}, %zmm19, %zmm17
+// CHECK:  encoding: [0x62,0xe3,0xe5,0x50,0x50,0x4a,0x7f,0x7b]
+          vrangepd $0x7b,1016(%rdx){1to8}, %zmm19, %zmm17
+
+// CHECK: vrangepd $123, 1024(%rdx){1to8}, %zmm19, %zmm17
+// CHECK:  encoding: [0x62,0xe3,0xe5,0x50,0x50,0x8a,0x00,0x04,0x00,0x00,0x7b]
+          vrangepd $0x7b,1024(%rdx){1to8}, %zmm19, %zmm17
+
+// CHECK: vrangepd $123, -1024(%rdx){1to8}, %zmm19, %zmm17
+// CHECK:  encoding: [0x62,0xe3,0xe5,0x50,0x50,0x4a,0x80,0x7b]
+          vrangepd $0x7b,-1024(%rdx){1to8}, %zmm19, %zmm17
+
+// CHECK: vrangepd $123, -1032(%rdx){1to8}, %zmm19, %zmm17
+// CHECK:  encoding: [0x62,0xe3,0xe5,0x50,0x50,0x8a,0xf8,0xfb,0xff,0xff,0x7b]
+          vrangepd $0x7b,-1032(%rdx){1to8}, %zmm19, %zmm17
+
+// CHECK: vrangeps $171, %zmm17, %zmm21, %zmm24
+// CHECK:  encoding: [0x62,0x23,0x55,0x40,0x50,0xc1,0xab]
+          vrangeps $0xab, %zmm17, %zmm21, %zmm24
+
+// CHECK: vrangeps $171, %zmm17, %zmm21, %zmm24 {%k6}
+// CHECK:  encoding: [0x62,0x23,0x55,0x46,0x50,0xc1,0xab]
+          vrangeps $0xab, %zmm17, %zmm21, %zmm24 {%k6}
+
+// CHECK: vrangeps $171, %zmm17, %zmm21, %zmm24 {%k6} {z}
+// CHECK:  encoding: [0x62,0x23,0x55,0xc6,0x50,0xc1,0xab]
+          vrangeps $0xab, %zmm17, %zmm21, %zmm24 {%k6} {z}
+
+// CHECK: vrangeps $171,{sae}, %zmm17, %zmm21, %zmm24
+// CHECK:  encoding: [0x62,0x23,0x55,0x10,0x50,0xc1,0xab]
+          vrangeps $0xab,{sae}, %zmm17, %zmm21, %zmm24
+
+// CHECK: vrangeps $123, %zmm17, %zmm21, %zmm24
+// CHECK:  encoding: [0x62,0x23,0x55,0x40,0x50,0xc1,0x7b]
+          vrangeps $0x7b, %zmm17, %zmm21, %zmm24
+
+// CHECK: vrangeps $123,{sae}, %zmm17, %zmm21, %zmm24
+// CHECK:  encoding: [0x62,0x23,0x55,0x10,0x50,0xc1,0x7b]
+          vrangeps $0x7b,{sae}, %zmm17, %zmm21, %zmm24
+
+// CHECK: vrangeps $123, (%rcx), %zmm21, %zmm24
+// CHECK:  encoding: [0x62,0x63,0x55,0x40,0x50,0x01,0x7b]
+          vrangeps $0x7b,(%rcx), %zmm21, %zmm24
+
+// CHECK: vrangeps $123, 291(%rax,%r14,8), %zmm21, %zmm24
+// CHECK:  encoding: [0x62,0x23,0x55,0x40,0x50,0x84,0xf0,0x23,0x01,0x00,0x00,0x7b]
+          vrangeps $0x7b,291(%rax,%r14,8), %zmm21, %zmm24
+
+// CHECK: vrangeps $123, (%rcx){1to16}, %zmm21, %zmm24
+// CHECK:  encoding: [0x62,0x63,0x55,0x50,0x50,0x01,0x7b]
+          vrangeps $0x7b,(%rcx){1to16}, %zmm21, %zmm24
+
+// CHECK: vrangeps $123, 8128(%rdx), %zmm21, %zmm24
+// CHECK:  encoding: [0x62,0x63,0x55,0x40,0x50,0x42,0x7f,0x7b]
+          vrangeps $0x7b,8128(%rdx), %zmm21, %zmm24
+
+// CHECK: vrangeps $123, 8192(%rdx), %zmm21, %zmm24
+// CHECK:  encoding: [0x62,0x63,0x55,0x40,0x50,0x82,0x00,0x20,0x00,0x00,0x7b]
+          vrangeps $0x7b,8192(%rdx), %zmm21, %zmm24
+
+// CHECK: vrangeps $123, -8192(%rdx), %zmm21, %zmm24
+// CHECK:  encoding: [0x62,0x63,0x55,0x40,0x50,0x42,0x80,0x7b]
+          vrangeps $0x7b,-8192(%rdx), %zmm21, %zmm24
+
+// CHECK: vrangeps $123, -8256(%rdx), %zmm21, %zmm24
+// CHECK:  encoding: [0x62,0x63,0x55,0x40,0x50,0x82,0xc0,0xdf,0xff,0xff,0x7b]
+          vrangeps $0x7b,-8256(%rdx), %zmm21, %zmm24
+
+// CHECK: vrangeps $123, 508(%rdx){1to16}, %zmm21, %zmm24
+// CHECK:  encoding: [0x62,0x63,0x55,0x50,0x50,0x42,0x7f,0x7b]
+          vrangeps $0x7b,508(%rdx){1to16}, %zmm21, %zmm24
+
+// CHECK: vrangeps $123, 512(%rdx){1to16}, %zmm21, %zmm24
+// CHECK:  encoding: [0x62,0x63,0x55,0x50,0x50,0x82,0x00,0x02,0x00,0x00,0x7b]
+          vrangeps $0x7b,512(%rdx){1to16}, %zmm21, %zmm24
+
+// CHECK: vrangeps $123, -512(%rdx){1to16}, %zmm21, %zmm24
+// CHECK:  encoding: [0x62,0x63,0x55,0x50,0x50,0x42,0x80,0x7b]
+          vrangeps $0x7b,-512(%rdx){1to16}, %zmm21, %zmm24
+
+// CHECK: vrangeps $123, -516(%rdx){1to16}, %zmm21, %zmm24
+// CHECK:  encoding: [0x62,0x63,0x55,0x50,0x50,0x82,0xfc,0xfd,0xff,0xff,0x7b]
+          vrangeps $0x7b,-516(%rdx){1to16}, %zmm21, %zmm24
+
+// CHECK: vrangesd $171, %xmm21, %xmm17, %xmm17
+// CHECK:  encoding: [0x62,0xa3,0xf5,0x00,0x51,0xcd,0xab]
+          vrangesd $0xab, %xmm21, %xmm17, %xmm17
+
+// CHECK: vrangesd $171, %xmm21, %xmm17, %xmm17 {%k5}
+// CHECK:  encoding: [0x62,0xa3,0xf5,0x05,0x51,0xcd,0xab]
+          vrangesd $0xab, %xmm21, %xmm17, %xmm17 {%k5}
+
+// CHECK: vrangesd $171, %xmm21, %xmm17, %xmm17 {%k5} {z}
+// CHECK:  encoding: [0x62,0xa3,0xf5,0x85,0x51,0xcd,0xab]
+          vrangesd $0xab, %xmm21, %xmm17, %xmm17 {%k5} {z}
+
+// CHECK: vrangesd $171,{sae}, %xmm21, %xmm17, %xmm17
+// CHECK:  encoding: [0x62,0xa3,0xf5,0x10,0x51,0xcd,0xab]
+          vrangesd $0xab,{sae}, %xmm21, %xmm17, %xmm17
+
+// CHECK: vrangesd $123, %xmm21, %xmm17, %xmm17
+// CHECK:  encoding: [0x62,0xa3,0xf5,0x00,0x51,0xcd,0x7b]
+          vrangesd $0x7b, %xmm21, %xmm17, %xmm17
+
+// CHECK: vrangesd $123,{sae}, %xmm21, %xmm17, %xmm17
+// CHECK:  encoding: [0x62,0xa3,0xf5,0x10,0x51,0xcd,0x7b]
+          vrangesd $0x7b,{sae}, %xmm21, %xmm17, %xmm17
+
+// CHECK: vrangesd $123, (%rcx), %xmm17, %xmm17
+// CHECK:  encoding: [0x62,0xe3,0xf5,0x00,0x51,0x09,0x7b]
+          vrangesd $0x7b,(%rcx), %xmm17, %xmm17
+
+// CHECK: vrangesd $123, 291(%rax,%r14,8), %xmm17, %xmm17
+// CHECK:  encoding: [0x62,0xa3,0xf5,0x00,0x51,0x8c,0xf0,0x23,0x01,0x00,0x00,0x7b]
+          vrangesd $0x7b,291(%rax,%r14,8), %xmm17, %xmm17
+
+// CHECK: vrangesd $123, 1016(%rdx), %xmm17, %xmm17
+// CHECK:  encoding: [0x62,0xe3,0xf5,0x00,0x51,0x4a,0x7f,0x7b]
+          vrangesd $0x7b,1016(%rdx), %xmm17, %xmm17
+
+// CHECK: vrangesd $123, 1024(%rdx), %xmm17, %xmm17
+// CHECK:  encoding: [0x62,0xe3,0xf5,0x00,0x51,0x8a,0x00,0x04,0x00,0x00,0x7b]
+          vrangesd $0x7b,1024(%rdx), %xmm17, %xmm17
+
+// CHECK: vrangesd $123, -1024(%rdx), %xmm17, %xmm17
+// CHECK:  encoding: [0x62,0xe3,0xf5,0x00,0x51,0x4a,0x80,0x7b]
+          vrangesd $0x7b,-1024(%rdx), %xmm17, %xmm17
+
+// CHECK: vrangesd $123, -1032(%rdx), %xmm17, %xmm17
+// CHECK:  encoding: [0x62,0xe3,0xf5,0x00,0x51,0x8a,0xf8,0xfb,0xff,0xff,0x7b]
+          vrangesd $0x7b,-1032(%rdx), %xmm17, %xmm17
+
+// CHECK: vrangess $171, %xmm20, %xmm24, %xmm25
+// CHECK:  encoding: [0x62,0x23,0x3d,0x00,0x51,0xcc,0xab]
+          vrangess $0xab, %xmm20, %xmm24, %xmm25
+
+// CHECK: vrangess $171, %xmm20, %xmm24, %xmm25 {%k5}
+// CHECK:  encoding: [0x62,0x23,0x3d,0x05,0x51,0xcc,0xab]
+          vrangess $0xab, %xmm20, %xmm24, %xmm25 {%k5}
+
+// CHECK: vrangess $171, %xmm20, %xmm24, %xmm25 {%k5} {z}
+// CHECK:  encoding: [0x62,0x23,0x3d,0x85,0x51,0xcc,0xab]
+          vrangess $0xab, %xmm20, %xmm24, %xmm25 {%k5} {z}
+
+// CHECK: vrangess $171,{sae}, %xmm20, %xmm24, %xmm25
+// CHECK:  encoding: [0x62,0x23,0x3d,0x10,0x51,0xcc,0xab]
+          vrangess $0xab,{sae}, %xmm20, %xmm24, %xmm25
+
+// CHECK: vrangess $123, %xmm20, %xmm24, %xmm25
+// CHECK:  encoding: [0x62,0x23,0x3d,0x00,0x51,0xcc,0x7b]
+          vrangess $0x7b, %xmm20, %xmm24, %xmm25
+
+// CHECK: vrangess $123,{sae}, %xmm20, %xmm24, %xmm25
+// CHECK:  encoding: [0x62,0x23,0x3d,0x10,0x51,0xcc,0x7b]
+          vrangess $0x7b,{sae}, %xmm20, %xmm24, %xmm25
+
+// CHECK: vrangess $123, (%rcx), %xmm24, %xmm25
+// CHECK:  encoding: [0x62,0x63,0x3d,0x00,0x51,0x09,0x7b]
+          vrangess $0x7b,(%rcx), %xmm24, %xmm25
+
+// CHECK: vrangess $123, 291(%rax,%r14,8), %xmm24, %xmm25
+// CHECK:  encoding: [0x62,0x23,0x3d,0x00,0x51,0x8c,0xf0,0x23,0x01,0x00,0x00,0x7b]
+          vrangess $0x7b,291(%rax,%r14,8), %xmm24, %xmm25
+
+// CHECK: vrangess $123, 508(%rdx), %xmm24, %xmm25
+// CHECK:  encoding: [0x62,0x63,0x3d,0x00,0x51,0x4a,0x7f,0x7b]
+          vrangess $0x7b,508(%rdx), %xmm24, %xmm25
+
+// CHECK: vrangess $123, 512(%rdx), %xmm24, %xmm25
+// CHECK:  encoding: [0x62,0x63,0x3d,0x00,0x51,0x8a,0x00,0x02,0x00,0x00,0x7b]
+          vrangess $0x7b,512(%rdx), %xmm24, %xmm25
+
+// CHECK: vrangess $123, -512(%rdx), %xmm24, %xmm25
+// CHECK:  encoding: [0x62,0x63,0x3d,0x00,0x51,0x4a,0x80,0x7b]
+          vrangess $0x7b,-512(%rdx), %xmm24, %xmm25
+
+// CHECK: vrangess $123, -516(%rdx), %xmm24, %xmm25
+// CHECK:  encoding: [0x62,0x63,0x3d,0x00,0x51,0x8a,0xfc,0xfd,0xff,0xff,0x7b]
+          vrangess $0x7b,-516(%rdx), %xmm24, %xmm25
+
+// CHECK: vreducepd $171, %zmm19, %zmm19
+// CHECK:  encoding: [0x62,0xa3,0xfd,0x48,0x56,0xdb,0xab]
+          vreducepd $0xab, %zmm19, %zmm19
+
+// CHECK: vreducepd $171, %zmm19, %zmm19 {%k6}
+// CHECK:  encoding: [0x62,0xa3,0xfd,0x4e,0x56,0xdb,0xab]
+          vreducepd $0xab, %zmm19, %zmm19 {%k6}
+
+// CHECK: vreducepd $171, %zmm19, %zmm19 {%k6} {z}
+// CHECK:  encoding: [0x62,0xa3,0xfd,0xce,0x56,0xdb,0xab]
+          vreducepd $0xab, %zmm19, %zmm19 {%k6} {z}
+
+// CHECK: vreducepd $171,{sae}, %zmm19, %zmm19
+// CHECK:  encoding: [0x62,0xa3,0xfd,0x18,0x56,0xdb,0xab]
+          vreducepd $0xab,{sae}, %zmm19, %zmm19
+
+// CHECK: vreducepd $123, %zmm19, %zmm19
+// CHECK:  encoding: [0x62,0xa3,0xfd,0x48,0x56,0xdb,0x7b]
+          vreducepd $0x7b, %zmm19, %zmm19
+
+// CHECK: vreducepd $123,{sae}, %zmm19, %zmm19
+// CHECK:  encoding: [0x62,0xa3,0xfd,0x18,0x56,0xdb,0x7b]
+          vreducepd $0x7b,{sae}, %zmm19, %zmm19
+
+// CHECK: vreducepd $123, (%rcx), %zmm19
+// CHECK:  encoding: [0x62,0xe3,0xfd,0x48,0x56,0x19,0x7b]
+          vreducepd $0x7b,(%rcx), %zmm19
+
+// CHECK: vreducepd $123, 291(%rax,%r14,8), %zmm19
+// CHECK:  encoding: [0x62,0xa3,0xfd,0x48,0x56,0x9c,0xf0,0x23,0x01,0x00,0x00,0x7b]
+          vreducepd $0x7b,291(%rax,%r14,8), %zmm19
+
+// CHECK: vreducepd $123, (%rcx){1to8}, %zmm19
+// CHECK:  encoding: [0x62,0xe3,0xfd,0x58,0x56,0x19,0x7b]
+          vreducepd $0x7b,(%rcx){1to8}, %zmm19
+
+// CHECK: vreducepd $123, 8128(%rdx), %zmm19
+// CHECK:  encoding: [0x62,0xe3,0xfd,0x48,0x56,0x5a,0x7f,0x7b]
+          vreducepd $0x7b,8128(%rdx), %zmm19
+
+// CHECK: vreducepd $123, 8192(%rdx), %zmm19
+// CHECK:  encoding: [0x62,0xe3,0xfd,0x48,0x56,0x9a,0x00,0x20,0x00,0x00,0x7b]
+          vreducepd $0x7b,8192(%rdx), %zmm19
+
+// CHECK: vreducepd $123, -8192(%rdx), %zmm19
+// CHECK:  encoding: [0x62,0xe3,0xfd,0x48,0x56,0x5a,0x80,0x7b]
+          vreducepd $0x7b,-8192(%rdx), %zmm19
+
+// CHECK: vreducepd $123, -8256(%rdx), %zmm19
+// CHECK:  encoding: [0x62,0xe3,0xfd,0x48,0x56,0x9a,0xc0,0xdf,0xff,0xff,0x7b]
+          vreducepd $0x7b,-8256(%rdx), %zmm19
+
+// CHECK: vreducepd $123, 1016(%rdx){1to8}, %zmm19
+// CHECK:  encoding: [0x62,0xe3,0xfd,0x58,0x56,0x5a,0x7f,0x7b]
+          vreducepd $0x7b,1016(%rdx){1to8}, %zmm19
+
+// CHECK: vreducepd $123, 1024(%rdx){1to8}, %zmm19
+// CHECK:  encoding: [0x62,0xe3,0xfd,0x58,0x56,0x9a,0x00,0x04,0x00,0x00,0x7b]
+          vreducepd $0x7b,1024(%rdx){1to8}, %zmm19
+
+// CHECK: vreducepd $123, -1024(%rdx){1to8}, %zmm19
+// CHECK:  encoding: [0x62,0xe3,0xfd,0x58,0x56,0x5a,0x80,0x7b]
+          vreducepd $0x7b,-1024(%rdx){1to8}, %zmm19
+
+// CHECK: vreducepd $123, -1032(%rdx){1to8}, %zmm19
+// CHECK:  encoding: [0x62,0xe3,0xfd,0x58,0x56,0x9a,0xf8,0xfb,0xff,0xff,0x7b]
+          vreducepd $0x7b,-1032(%rdx){1to8}, %zmm19
+
+// CHECK: vreduceps $171, %zmm29, %zmm19
+// CHECK:  encoding: [0x62,0x83,0x7d,0x48,0x56,0xdd,0xab]
+          vreduceps $0xab, %zmm29, %zmm19
+
+// CHECK: vreduceps $171, %zmm29, %zmm19 {%k3}
+// CHECK:  encoding: [0x62,0x83,0x7d,0x4b,0x56,0xdd,0xab]
+          vreduceps $0xab, %zmm29, %zmm19 {%k3}
+
+// CHECK: vreduceps $171, %zmm29, %zmm19 {%k3} {z}
+// CHECK:  encoding: [0x62,0x83,0x7d,0xcb,0x56,0xdd,0xab]
+          vreduceps $0xab, %zmm29, %zmm19 {%k3} {z}
+
+// CHECK: vreduceps $171,{sae}, %zmm29, %zmm19
+// CHECK:  encoding: [0x62,0x83,0x7d,0x18,0x56,0xdd,0xab]
+          vreduceps $0xab,{sae}, %zmm29, %zmm19
+
+// CHECK: vreduceps $123, %zmm29, %zmm19
+// CHECK:  encoding: [0x62,0x83,0x7d,0x48,0x56,0xdd,0x7b]
+          vreduceps $0x7b, %zmm29, %zmm19
+
+// CHECK: vreduceps $123,{sae}, %zmm29, %zmm19
+// CHECK:  encoding: [0x62,0x83,0x7d,0x18,0x56,0xdd,0x7b]
+          vreduceps $0x7b,{sae}, %zmm29, %zmm19
+
+// CHECK: vreduceps $123, (%rcx), %zmm19
+// CHECK:  encoding: [0x62,0xe3,0x7d,0x48,0x56,0x19,0x7b]
+          vreduceps $0x7b,(%rcx), %zmm19
+
+// CHECK: vreduceps $123, 291(%rax,%r14,8), %zmm19
+// CHECK:  encoding: [0x62,0xa3,0x7d,0x48,0x56,0x9c,0xf0,0x23,0x01,0x00,0x00,0x7b]
+          vreduceps $0x7b,291(%rax,%r14,8), %zmm19
+
+// CHECK: vreduceps $123, (%rcx){1to16}, %zmm19
+// CHECK:  encoding: [0x62,0xe3,0x7d,0x58,0x56,0x19,0x7b]
+          vreduceps $0x7b,(%rcx){1to16}, %zmm19
+
+// CHECK: vreduceps $123, 8128(%rdx), %zmm19
+// CHECK:  encoding: [0x62,0xe3,0x7d,0x48,0x56,0x5a,0x7f,0x7b]
+          vreduceps $0x7b,8128(%rdx), %zmm19
+
+// CHECK: vreduceps $123, 8192(%rdx), %zmm19
+// CHECK:  encoding: [0x62,0xe3,0x7d,0x48,0x56,0x9a,0x00,0x20,0x00,0x00,0x7b]
+          vreduceps $0x7b,8192(%rdx), %zmm19
+
+// CHECK: vreduceps $123, -8192(%rdx), %zmm19
+// CHECK:  encoding: [0x62,0xe3,0x7d,0x48,0x56,0x5a,0x80,0x7b]
+          vreduceps $0x7b,-8192(%rdx), %zmm19
+
+// CHECK: vreduceps $123, -8256(%rdx), %zmm19
+// CHECK:  encoding: [0x62,0xe3,0x7d,0x48,0x56,0x9a,0xc0,0xdf,0xff,0xff,0x7b]
+          vreduceps $0x7b,-8256(%rdx), %zmm19
+
+// CHECK: vreduceps $123, 508(%rdx){1to16}, %zmm19
+// CHECK:  encoding: [0x62,0xe3,0x7d,0x58,0x56,0x5a,0x7f,0x7b]
+          vreduceps $0x7b,508(%rdx){1to16}, %zmm19
+
+// CHECK: vreduceps $123, 512(%rdx){1to16}, %zmm19
+// CHECK:  encoding: [0x62,0xe3,0x7d,0x58,0x56,0x9a,0x00,0x02,0x00,0x00,0x7b]
+          vreduceps $0x7b,512(%rdx){1to16}, %zmm19
+
+// CHECK: vreduceps $123, -512(%rdx){1to16}, %zmm19
+// CHECK:  encoding: [0x62,0xe3,0x7d,0x58,0x56,0x5a,0x80,0x7b]
+          vreduceps $0x7b,-512(%rdx){1to16}, %zmm19
+
+// CHECK: vreduceps $123, -516(%rdx){1to16}, %zmm19
+// CHECK:  encoding: [0x62,0xe3,0x7d,0x58,0x56,0x9a,0xfc,0xfd,0xff,0xff,0x7b]
+          vreduceps $0x7b,-516(%rdx){1to16}, %zmm19
+
+// CHECK: vreducesd $171, %xmm25, %xmm17, %xmm17
+// CHECK:  encoding: [0x62,0x83,0xf5,0x00,0x57,0xc9,0xab]
+          vreducesd $0xab, %xmm25, %xmm17, %xmm17
+
+// CHECK: vreducesd $171, %xmm25, %xmm17, %xmm17 {%k6}
+// CHECK:  encoding: [0x62,0x83,0xf5,0x06,0x57,0xc9,0xab]
+          vreducesd $0xab, %xmm25, %xmm17, %xmm17 {%k6}
+
+// CHECK: vreducesd $171, %xmm25, %xmm17, %xmm17 {%k6} {z}
+// CHECK:  encoding: [0x62,0x83,0xf5,0x86,0x57,0xc9,0xab]
+          vreducesd $0xab, %xmm25, %xmm17, %xmm17 {%k6} {z}
+
+// CHECK: vreducesd $171,{sae}, %xmm25, %xmm17, %xmm17
+// CHECK:  encoding: [0x62,0x83,0xf5,0x10,0x57,0xc9,0xab]
+          vreducesd $0xab,{sae}, %xmm25, %xmm17, %xmm17
+
+// CHECK: vreducesd $123, %xmm25, %xmm17, %xmm17
+// CHECK:  encoding: [0x62,0x83,0xf5,0x00,0x57,0xc9,0x7b]
+          vreducesd $0x7b, %xmm25, %xmm17, %xmm17
+
+// CHECK: vreducesd $123,{sae}, %xmm25, %xmm17, %xmm17
+// CHECK:  encoding: [0x62,0x83,0xf5,0x10,0x57,0xc9,0x7b]
+          vreducesd $0x7b,{sae}, %xmm25, %xmm17, %xmm17
+
+// CHECK: vreducesd $123, (%rcx), %xmm17, %xmm17
+// CHECK:  encoding: [0x62,0xe3,0xf5,0x00,0x57,0x09,0x7b]
+          vreducesd $0x7b,(%rcx), %xmm17, %xmm17
+
+// CHECK: vreducesd $123, 291(%rax,%r14,8), %xmm17, %xmm17
+// CHECK:  encoding: [0x62,0xa3,0xf5,0x00,0x57,0x8c,0xf0,0x23,0x01,0x00,0x00,0x7b]
+          vreducesd $0x7b,291(%rax,%r14,8), %xmm17, %xmm17
+
+// CHECK: vreducesd $123, 1016(%rdx), %xmm17, %xmm17
+// CHECK:  encoding: [0x62,0xe3,0xf5,0x00,0x57,0x4a,0x7f,0x7b]
+          vreducesd $0x7b,1016(%rdx), %xmm17, %xmm17
+
+// CHECK: vreducesd $123, 1024(%rdx), %xmm17, %xmm17
+// CHECK:  encoding: [0x62,0xe3,0xf5,0x00,0x57,0x8a,0x00,0x04,0x00,0x00,0x7b]
+          vreducesd $0x7b,1024(%rdx), %xmm17, %xmm17
+
+// CHECK: vreducesd $123, -1024(%rdx), %xmm17, %xmm17
+// CHECK:  encoding: [0x62,0xe3,0xf5,0x00,0x57,0x4a,0x80,0x7b]
+          vreducesd $0x7b,-1024(%rdx), %xmm17, %xmm17
+
+// CHECK: vreducesd $123, -1032(%rdx), %xmm17, %xmm17
+// CHECK:  encoding: [0x62,0xe3,0xf5,0x00,0x57,0x8a,0xf8,0xfb,0xff,0xff,0x7b]
+          vreducesd $0x7b,-1032(%rdx), %xmm17, %xmm17
+
+// CHECK: vreducess $171, %xmm21, %xmm29, %xmm30
+// CHECK:  encoding: [0x62,0x23,0x15,0x00,0x57,0xf5,0xab]
+          vreducess $0xab, %xmm21, %xmm29, %xmm30
+
+// CHECK: vreducess $171, %xmm21, %xmm29, %xmm30 {%k1}
+// CHECK:  encoding: [0x62,0x23,0x15,0x01,0x57,0xf5,0xab]
+          vreducess $0xab, %xmm21, %xmm29, %xmm30 {%k1}
+
+// CHECK: vreducess $171, %xmm21, %xmm29, %xmm30 {%k1} {z}
+// CHECK:  encoding: [0x62,0x23,0x15,0x81,0x57,0xf5,0xab]
+          vreducess $0xab, %xmm21, %xmm29, %xmm30 {%k1} {z}
+
+// CHECK: vreducess $171,{sae}, %xmm21, %xmm29, %xmm30
+// CHECK:  encoding: [0x62,0x23,0x15,0x10,0x57,0xf5,0xab]
+          vreducess $0xab,{sae}, %xmm21, %xmm29, %xmm30
+
+// CHECK: vreducess $123, %xmm21, %xmm29, %xmm30
+// CHECK:  encoding: [0x62,0x23,0x15,0x00,0x57,0xf5,0x7b]
+          vreducess $0x7b, %xmm21, %xmm29, %xmm30
+
+// CHECK: vreducess $123,{sae}, %xmm21, %xmm29, %xmm30
+// CHECK:  encoding: [0x62,0x23,0x15,0x10,0x57,0xf5,0x7b]
+          vreducess $0x7b,{sae}, %xmm21, %xmm29, %xmm30
+
+// CHECK: vreducess $123, (%rcx), %xmm29, %xmm30
+// CHECK:  encoding: [0x62,0x63,0x15,0x00,0x57,0x31,0x7b]
+          vreducess $0x7b,(%rcx), %xmm29, %xmm30
+
+// CHECK: vreducess $123, 291(%rax,%r14,8), %xmm29, %xmm30
+// CHECK:  encoding: [0x62,0x23,0x15,0x00,0x57,0xb4,0xf0,0x23,0x01,0x00,0x00,0x7b]
+          vreducess $0x7b,291(%rax,%r14,8), %xmm29, %xmm30
+
+// CHECK: vreducess $123, 508(%rdx), %xmm29, %xmm30
+// CHECK:  encoding: [0x62,0x63,0x15,0x00,0x57,0x72,0x7f,0x7b]
+          vreducess $0x7b,508(%rdx), %xmm29, %xmm30
+
+// CHECK: vreducess $123, 512(%rdx), %xmm29, %xmm30
+// CHECK:  encoding: [0x62,0x63,0x15,0x00,0x57,0xb2,0x00,0x02,0x00,0x00,0x7b]
+          vreducess $0x7b,512(%rdx), %xmm29, %xmm30
+
+// CHECK: vreducess $123, -512(%rdx), %xmm29, %xmm30
+// CHECK:  encoding: [0x62,0x63,0x15,0x00,0x57,0x72,0x80,0x7b]
+          vreducess $0x7b,-512(%rdx), %xmm29, %xmm30
+
+// CHECK: vreducess $123, -516(%rdx), %xmm29, %xmm30
+// CHECK:  encoding: [0x62,0x63,0x15,0x00,0x57,0xb2,0xfc,0xfd,0xff,0xff,0x7b]
+          vreducess $0x7b,-516(%rdx), %xmm29, %xmm30
+
+// CHECK: vreducepd $171, %zmm28, %zmm18
+// CHECK:  encoding: [0x62,0x83,0xfd,0x48,0x56,0xd4,0xab]
+          vreducepd $0xab, %zmm28, %zmm18
+
+// CHECK: vreducepd $171, %zmm28, %zmm18 {%k5}
+// CHECK:  encoding: [0x62,0x83,0xfd,0x4d,0x56,0xd4,0xab]
+          vreducepd $0xab, %zmm28, %zmm18 {%k5}
+
+// CHECK: vreducepd $171, %zmm28, %zmm18 {%k5} {z}
+// CHECK:  encoding: [0x62,0x83,0xfd,0xcd,0x56,0xd4,0xab]
+          vreducepd $0xab, %zmm28, %zmm18 {%k5} {z}
+
+// CHECK: vreducepd $171,{sae}, %zmm28, %zmm18
+// CHECK:  encoding: [0x62,0x83,0xfd,0x18,0x56,0xd4,0xab]
+          vreducepd $0xab,{sae}, %zmm28, %zmm18
+
+// CHECK: vreducepd $123, %zmm28, %zmm18
+// CHECK:  encoding: [0x62,0x83,0xfd,0x48,0x56,0xd4,0x7b]
+          vreducepd $0x7b, %zmm28, %zmm18
+
+// CHECK: vreducepd $123,{sae}, %zmm28, %zmm18
+// CHECK:  encoding: [0x62,0x83,0xfd,0x18,0x56,0xd4,0x7b]
+          vreducepd $0x7b,{sae}, %zmm28, %zmm18
+
+// CHECK: vreducepd $123, (%rcx), %zmm18
+// CHECK:  encoding: [0x62,0xe3,0xfd,0x48,0x56,0x11,0x7b]
+          vreducepd $0x7b,(%rcx), %zmm18
+
+// CHECK: vreducepd $123, 4660(%rax,%r14,8), %zmm18
+// CHECK:  encoding: [0x62,0xa3,0xfd,0x48,0x56,0x94,0xf0,0x34,0x12,0x00,0x00,0x7b]
+          vreducepd $0x7b,4660(%rax,%r14,8), %zmm18
+
+// CHECK: vreducepd $123, (%rcx){1to8}, %zmm18
+// CHECK:  encoding: [0x62,0xe3,0xfd,0x58,0x56,0x11,0x7b]
+          vreducepd $0x7b,(%rcx){1to8}, %zmm18
+
+// CHECK: vreducepd $123, 8128(%rdx), %zmm18
+// CHECK:  encoding: [0x62,0xe3,0xfd,0x48,0x56,0x52,0x7f,0x7b]
+          vreducepd $0x7b,8128(%rdx), %zmm18
+
+// CHECK: vreducepd $123, 8192(%rdx), %zmm18
+// CHECK:  encoding: [0x62,0xe3,0xfd,0x48,0x56,0x92,0x00,0x20,0x00,0x00,0x7b]
+          vreducepd $0x7b,8192(%rdx), %zmm18
+
+// CHECK: vreducepd $123, -8192(%rdx), %zmm18
+// CHECK:  encoding: [0x62,0xe3,0xfd,0x48,0x56,0x52,0x80,0x7b]
+          vreducepd $0x7b,-8192(%rdx), %zmm18
+
+// CHECK: vreducepd $123, -8256(%rdx), %zmm18
+// CHECK:  encoding: [0x62,0xe3,0xfd,0x48,0x56,0x92,0xc0,0xdf,0xff,0xff,0x7b]
+          vreducepd $0x7b,-8256(%rdx), %zmm18
+
+// CHECK: vreducepd $123, 1016(%rdx){1to8}, %zmm18
+// CHECK:  encoding: [0x62,0xe3,0xfd,0x58,0x56,0x52,0x7f,0x7b]
+          vreducepd $0x7b,1016(%rdx){1to8}, %zmm18
+
+// CHECK: vreducepd $123, 1024(%rdx){1to8}, %zmm18
+// CHECK:  encoding: [0x62,0xe3,0xfd,0x58,0x56,0x92,0x00,0x04,0x00,0x00,0x7b]
+          vreducepd $0x7b,1024(%rdx){1to8}, %zmm18
+
+// CHECK: vreducepd $123, -1024(%rdx){1to8}, %zmm18
+// CHECK:  encoding: [0x62,0xe3,0xfd,0x58,0x56,0x52,0x80,0x7b]
+          vreducepd $0x7b,-1024(%rdx){1to8}, %zmm18
+
+// CHECK: vreducepd $123, -1032(%rdx){1to8}, %zmm18
+// CHECK:  encoding: [0x62,0xe3,0xfd,0x58,0x56,0x92,0xf8,0xfb,0xff,0xff,0x7b]
+          vreducepd $0x7b,-1032(%rdx){1to8}, %zmm18
+
+// CHECK: vreduceps $171, %zmm25, %zmm26
+// CHECK:  encoding: [0x62,0x03,0x7d,0x48,0x56,0xd1,0xab]
+          vreduceps $0xab, %zmm25, %zmm26
+
+// CHECK: vreduceps $171, %zmm25, %zmm26 {%k3}
+// CHECK:  encoding: [0x62,0x03,0x7d,0x4b,0x56,0xd1,0xab]
+          vreduceps $0xab, %zmm25, %zmm26 {%k3}
+
+// CHECK: vreduceps $171, %zmm25, %zmm26 {%k3} {z}
+// CHECK:  encoding: [0x62,0x03,0x7d,0xcb,0x56,0xd1,0xab]
+          vreduceps $0xab, %zmm25, %zmm26 {%k3} {z}
+
+// CHECK: vreduceps $171,{sae}, %zmm25, %zmm26
+// CHECK:  encoding: [0x62,0x03,0x7d,0x18,0x56,0xd1,0xab]
+          vreduceps $0xab,{sae}, %zmm25, %zmm26
+
+// CHECK: vreduceps $123, %zmm25, %zmm26
+// CHECK:  encoding: [0x62,0x03,0x7d,0x48,0x56,0xd1,0x7b]
+          vreduceps $0x7b, %zmm25, %zmm26
+
+// CHECK: vreduceps $123,{sae}, %zmm25, %zmm26
+// CHECK:  encoding: [0x62,0x03,0x7d,0x18,0x56,0xd1,0x7b]
+          vreduceps $0x7b,{sae}, %zmm25, %zmm26
+
+// CHECK: vreduceps $123, (%rcx), %zmm26
+// CHECK:  encoding: [0x62,0x63,0x7d,0x48,0x56,0x11,0x7b]
+          vreduceps $0x7b,(%rcx), %zmm26
+
+// CHECK: vreduceps $123, 4660(%rax,%r14,8), %zmm26
+// CHECK:  encoding: [0x62,0x23,0x7d,0x48,0x56,0x94,0xf0,0x34,0x12,0x00,0x00,0x7b]
+          vreduceps $0x7b,4660(%rax,%r14,8), %zmm26
+
+// CHECK: vreduceps $123, (%rcx){1to16}, %zmm26
+// CHECK:  encoding: [0x62,0x63,0x7d,0x58,0x56,0x11,0x7b]
+          vreduceps $0x7b,(%rcx){1to16}, %zmm26
+
+// CHECK: vreduceps $123, 8128(%rdx), %zmm26
+// CHECK:  encoding: [0x62,0x63,0x7d,0x48,0x56,0x52,0x7f,0x7b]
+          vreduceps $0x7b,8128(%rdx), %zmm26
+
+// CHECK: vreduceps $123, 8192(%rdx), %zmm26
+// CHECK:  encoding: [0x62,0x63,0x7d,0x48,0x56,0x92,0x00,0x20,0x00,0x00,0x7b]
+          vreduceps $0x7b,8192(%rdx), %zmm26
+
+// CHECK: vreduceps $123, -8192(%rdx), %zmm26
+// CHECK:  encoding: [0x62,0x63,0x7d,0x48,0x56,0x52,0x80,0x7b]
+          vreduceps $0x7b,-8192(%rdx), %zmm26
+
+// CHECK: vreduceps $123, -8256(%rdx), %zmm26
+// CHECK:  encoding: [0x62,0x63,0x7d,0x48,0x56,0x92,0xc0,0xdf,0xff,0xff,0x7b]
+          vreduceps $0x7b,-8256(%rdx), %zmm26
+
+// CHECK: vreduceps $123, 508(%rdx){1to16}, %zmm26
+// CHECK:  encoding: [0x62,0x63,0x7d,0x58,0x56,0x52,0x7f,0x7b]
+          vreduceps $0x7b,508(%rdx){1to16}, %zmm26
+
+// CHECK: vreduceps $123, 512(%rdx){1to16}, %zmm26
+// CHECK:  encoding: [0x62,0x63,0x7d,0x58,0x56,0x92,0x00,0x02,0x00,0x00,0x7b]
+          vreduceps $0x7b,512(%rdx){1to16}, %zmm26
+
+// CHECK: vreduceps $123, -512(%rdx){1to16}, %zmm26
+// CHECK:  encoding: [0x62,0x63,0x7d,0x58,0x56,0x52,0x80,0x7b]
+          vreduceps $0x7b,-512(%rdx){1to16}, %zmm26
+
+// CHECK: vreduceps $123, -516(%rdx){1to16}, %zmm26
+// CHECK:  encoding: [0x62,0x63,0x7d,0x58,0x56,0x92,0xfc,0xfd,0xff,0xff,0x7b]
+          vreduceps $0x7b,-516(%rdx){1to16}, %zmm26
+
+// CHECK: vreducesd $171, %xmm24, %xmm19, %xmm25
+// CHECK:  encoding: [0x62,0x03,0xe5,0x00,0x57,0xc8,0xab]
+          vreducesd $0xab, %xmm24, %xmm19, %xmm25
+
+// CHECK: vreducesd $171, %xmm24, %xmm19, %xmm25 {%k3}
+// CHECK:  encoding: [0x62,0x03,0xe5,0x03,0x57,0xc8,0xab]
+          vreducesd $0xab, %xmm24, %xmm19, %xmm25 {%k3}
+
+// CHECK: vreducesd $171, %xmm24, %xmm19, %xmm25 {%k3} {z}
+// CHECK:  encoding: [0x62,0x03,0xe5,0x83,0x57,0xc8,0xab]
+          vreducesd $0xab, %xmm24, %xmm19, %xmm25 {%k3} {z}
+
+// CHECK: vreducesd $171,{sae}, %xmm24, %xmm19, %xmm25
+// CHECK:  encoding: [0x62,0x03,0xe5,0x10,0x57,0xc8,0xab]
+          vreducesd $0xab,{sae}, %xmm24, %xmm19, %xmm25
+
+// CHECK: vreducesd $123, %xmm24, %xmm19, %xmm25
+// CHECK:  encoding: [0x62,0x03,0xe5,0x00,0x57,0xc8,0x7b]
+          vreducesd $0x7b, %xmm24, %xmm19, %xmm25
+
+// CHECK: vreducesd $123,{sae}, %xmm24, %xmm19, %xmm25
+// CHECK:  encoding: [0x62,0x03,0xe5,0x10,0x57,0xc8,0x7b]
+          vreducesd $0x7b,{sae}, %xmm24, %xmm19, %xmm25
+
+// CHECK: vreducesd $123, (%rcx), %xmm19, %xmm25
+// CHECK:  encoding: [0x62,0x63,0xe5,0x00,0x57,0x09,0x7b]
+          vreducesd $0x7b,(%rcx), %xmm19, %xmm25
+
+// CHECK: vreducesd $123, 4660(%rax,%r14,8), %xmm19, %xmm25
+// CHECK:  encoding: [0x62,0x23,0xe5,0x00,0x57,0x8c,0xf0,0x34,0x12,0x00,0x00,0x7b]
+          vreducesd $0x7b,4660(%rax,%r14,8), %xmm19, %xmm25
+
+// CHECK: vreducesd $123, 1016(%rdx), %xmm19, %xmm25
+// CHECK:  encoding: [0x62,0x63,0xe5,0x00,0x57,0x4a,0x7f,0x7b]
+          vreducesd $0x7b,1016(%rdx), %xmm19, %xmm25
+
+// CHECK: vreducesd $123, 1024(%rdx), %xmm19, %xmm25
+// CHECK:  encoding: [0x62,0x63,0xe5,0x00,0x57,0x8a,0x00,0x04,0x00,0x00,0x7b]
+          vreducesd $0x7b,1024(%rdx), %xmm19, %xmm25
+
+// CHECK: vreducesd $123, -1024(%rdx), %xmm19, %xmm25
+// CHECK:  encoding: [0x62,0x63,0xe5,0x00,0x57,0x4a,0x80,0x7b]
+          vreducesd $0x7b,-1024(%rdx), %xmm19, %xmm25
+
+// CHECK: vreducesd $123, -1032(%rdx), %xmm19, %xmm25
+// CHECK:  encoding: [0x62,0x63,0xe5,0x00,0x57,0x8a,0xf8,0xfb,0xff,0xff,0x7b]
+          vreducesd $0x7b,-1032(%rdx), %xmm19, %xmm25
+
+// CHECK: vreducess $171, %xmm21, %xmm24, %xmm30
+// CHECK:  encoding: [0x62,0x23,0x3d,0x00,0x57,0xf5,0xab]
+          vreducess $0xab, %xmm21, %xmm24, %xmm30
+
+// CHECK: vreducess $171, %xmm21, %xmm24, %xmm30 {%k2}
+// CHECK:  encoding: [0x62,0x23,0x3d,0x02,0x57,0xf5,0xab]
+          vreducess $0xab, %xmm21, %xmm24, %xmm30 {%k2}
+
+// CHECK: vreducess $171, %xmm21, %xmm24, %xmm30 {%k2} {z}
+// CHECK:  encoding: [0x62,0x23,0x3d,0x82,0x57,0xf5,0xab]
+          vreducess $0xab, %xmm21, %xmm24, %xmm30 {%k2} {z}
+
+// CHECK: vreducess $171,{sae}, %xmm21, %xmm24, %xmm30
+// CHECK:  encoding: [0x62,0x23,0x3d,0x10,0x57,0xf5,0xab]
+          vreducess $0xab,{sae}, %xmm21, %xmm24, %xmm30
+
+// CHECK: vreducess $123, %xmm21, %xmm24, %xmm30
+// CHECK:  encoding: [0x62,0x23,0x3d,0x00,0x57,0xf5,0x7b]
+          vreducess $0x7b, %xmm21, %xmm24, %xmm30
+
+// CHECK: vreducess $123,{sae}, %xmm21, %xmm24, %xmm30
+// CHECK:  encoding: [0x62,0x23,0x3d,0x10,0x57,0xf5,0x7b]
+          vreducess $0x7b,{sae}, %xmm21, %xmm24, %xmm30
+
+// CHECK: vreducess $123, (%rcx), %xmm24, %xmm30
+// CHECK:  encoding: [0x62,0x63,0x3d,0x00,0x57,0x31,0x7b]
+          vreducess $0x7b,(%rcx), %xmm24, %xmm30
+
+// CHECK: vreducess $123, 4660(%rax,%r14,8), %xmm24, %xmm30
+// CHECK:  encoding: [0x62,0x23,0x3d,0x00,0x57,0xb4,0xf0,0x34,0x12,0x00,0x00,0x7b]
+          vreducess $0x7b,4660(%rax,%r14,8), %xmm24, %xmm30
+
+// CHECK: vreducess $123, 508(%rdx), %xmm24, %xmm30
+// CHECK:  encoding: [0x62,0x63,0x3d,0x00,0x57,0x72,0x7f,0x7b]
+          vreducess $0x7b,508(%rdx), %xmm24, %xmm30
+
+// CHECK: vreducess $123, 512(%rdx), %xmm24, %xmm30
+// CHECK:  encoding: [0x62,0x63,0x3d,0x00,0x57,0xb2,0x00,0x02,0x00,0x00,0x7b]
+          vreducess $0x7b,512(%rdx), %xmm24, %xmm30
+
+// CHECK: vreducess $123, -512(%rdx), %xmm24, %xmm30
+// CHECK:  encoding: [0x62,0x63,0x3d,0x00,0x57,0x72,0x80,0x7b]
+          vreducess $0x7b,-512(%rdx), %xmm24, %xmm30
+
+// CHECK: vreducess $123, -516(%rdx), %xmm24, %xmm30
+// CHECK:  encoding: [0x62,0x63,0x3d,0x00,0x57,0xb2,0xfc,0xfd,0xff,0xff,0x7b]
+          vreducess $0x7b,-516(%rdx), %xmm24, %xmm30
+
+// CHECK: vcvtpd2qq %zmm29, %zmm18
+// CHECK:  encoding: [0x62,0x81,0xfd,0x48,0x7b,0xd5]
+          vcvtpd2qq %zmm29, %zmm18
+
+// CHECK: vcvtpd2qq %zmm29, %zmm18 {%k6}
+// CHECK:  encoding: [0x62,0x81,0xfd,0x4e,0x7b,0xd5]
+          vcvtpd2qq %zmm29, %zmm18 {%k6}
+
+// CHECK: vcvtpd2qq %zmm29, %zmm18 {%k6} {z}
+// CHECK:  encoding: [0x62,0x81,0xfd,0xce,0x7b,0xd5]
+          vcvtpd2qq %zmm29, %zmm18 {%k6} {z}
+
+// CHECK: vcvtpd2qq {rn-sae}, %zmm29, %zmm18
+// CHECK:  encoding: [0x62,0x81,0xfd,0x18,0x7b,0xd5]
+          vcvtpd2qq {rn-sae}, %zmm29, %zmm18
+
+// CHECK: vcvtpd2qq {ru-sae}, %zmm29, %zmm18
+// CHECK:  encoding: [0x62,0x81,0xfd,0x58,0x7b,0xd5]
+          vcvtpd2qq {ru-sae}, %zmm29, %zmm18
+
+// CHECK: vcvtpd2qq {rd-sae}, %zmm29, %zmm18
+// CHECK:  encoding: [0x62,0x81,0xfd,0x38,0x7b,0xd5]
+          vcvtpd2qq {rd-sae}, %zmm29, %zmm18
+
+// CHECK: vcvtpd2qq {rz-sae}, %zmm29, %zmm18
+// CHECK:  encoding: [0x62,0x81,0xfd,0x78,0x7b,0xd5]
+          vcvtpd2qq {rz-sae}, %zmm29, %zmm18
+
+// CHECK: vcvtpd2qq (%rcx), %zmm18
+// CHECK:  encoding: [0x62,0xe1,0xfd,0x48,0x7b,0x11]
+          vcvtpd2qq (%rcx), %zmm18
+
+// CHECK: vcvtpd2qq 291(%rax,%r14,8), %zmm18
+// CHECK:  encoding: [0x62,0xa1,0xfd,0x48,0x7b,0x94,0xf0,0x23,0x01,0x00,0x00]
+          vcvtpd2qq 291(%rax,%r14,8), %zmm18
+
+// CHECK: vcvtpd2qq (%rcx){1to8}, %zmm18
+// CHECK:  encoding: [0x62,0xe1,0xfd,0x58,0x7b,0x11]
+          vcvtpd2qq (%rcx){1to8}, %zmm18
+
+// CHECK: vcvtpd2qq 8128(%rdx), %zmm18
+// CHECK:  encoding: [0x62,0xe1,0xfd,0x48,0x7b,0x52,0x7f]
+          vcvtpd2qq 8128(%rdx), %zmm18
+
+// CHECK: vcvtpd2qq 8192(%rdx), %zmm18
+// CHECK:  encoding: [0x62,0xe1,0xfd,0x48,0x7b,0x92,0x00,0x20,0x00,0x00]
+          vcvtpd2qq 8192(%rdx), %zmm18
+
+// CHECK: vcvtpd2qq -8192(%rdx), %zmm18
+// CHECK:  encoding: [0x62,0xe1,0xfd,0x48,0x7b,0x52,0x80]
+          vcvtpd2qq -8192(%rdx), %zmm18
+
+// CHECK: vcvtpd2qq -8256(%rdx), %zmm18
+// CHECK:  encoding: [0x62,0xe1,0xfd,0x48,0x7b,0x92,0xc0,0xdf,0xff,0xff]
+          vcvtpd2qq -8256(%rdx), %zmm18
+
+// CHECK: vcvtpd2qq 1016(%rdx){1to8}, %zmm18
+// CHECK:  encoding: [0x62,0xe1,0xfd,0x58,0x7b,0x52,0x7f]
+          vcvtpd2qq 1016(%rdx){1to8}, %zmm18
+
+// CHECK: vcvtpd2qq 1024(%rdx){1to8}, %zmm18
+// CHECK:  encoding: [0x62,0xe1,0xfd,0x58,0x7b,0x92,0x00,0x04,0x00,0x00]
+          vcvtpd2qq 1024(%rdx){1to8}, %zmm18
+
+// CHECK: vcvtpd2qq -1024(%rdx){1to8}, %zmm18
+// CHECK:  encoding: [0x62,0xe1,0xfd,0x58,0x7b,0x52,0x80]
+          vcvtpd2qq -1024(%rdx){1to8}, %zmm18
+
+// CHECK: vcvtpd2qq -1032(%rdx){1to8}, %zmm18
+// CHECK:  encoding: [0x62,0xe1,0xfd,0x58,0x7b,0x92,0xf8,0xfb,0xff,0xff]
+          vcvtpd2qq -1032(%rdx){1to8}, %zmm18
+
+// CHECK: vcvtpd2uqq %zmm21, %zmm22
+// CHECK:  encoding: [0x62,0xa1,0xfd,0x48,0x79,0xf5]
+          vcvtpd2uqq %zmm21, %zmm22
+
+// CHECK: vcvtpd2uqq %zmm21, %zmm22 {%k5}
+// CHECK:  encoding: [0x62,0xa1,0xfd,0x4d,0x79,0xf5]
+          vcvtpd2uqq %zmm21, %zmm22 {%k5}
+
+// CHECK: vcvtpd2uqq %zmm21, %zmm22 {%k5} {z}
+// CHECK:  encoding: [0x62,0xa1,0xfd,0xcd,0x79,0xf5]
+          vcvtpd2uqq %zmm21, %zmm22 {%k5} {z}
+
+// CHECK: vcvtpd2uqq {rn-sae}, %zmm21, %zmm22
+// CHECK:  encoding: [0x62,0xa1,0xfd,0x18,0x79,0xf5]
+          vcvtpd2uqq {rn-sae}, %zmm21, %zmm22
+
+// CHECK: vcvtpd2uqq {ru-sae}, %zmm21, %zmm22
+// CHECK:  encoding: [0x62,0xa1,0xfd,0x58,0x79,0xf5]
+          vcvtpd2uqq {ru-sae}, %zmm21, %zmm22
+
+// CHECK: vcvtpd2uqq {rd-sae}, %zmm21, %zmm22
+// CHECK:  encoding: [0x62,0xa1,0xfd,0x38,0x79,0xf5]
+          vcvtpd2uqq {rd-sae}, %zmm21, %zmm22
+
+// CHECK: vcvtpd2uqq {rz-sae}, %zmm21, %zmm22
+// CHECK:  encoding: [0x62,0xa1,0xfd,0x78,0x79,0xf5]
+          vcvtpd2uqq {rz-sae}, %zmm21, %zmm22
+
+// CHECK: vcvtpd2uqq (%rcx), %zmm22
+// CHECK:  encoding: [0x62,0xe1,0xfd,0x48,0x79,0x31]
+          vcvtpd2uqq (%rcx), %zmm22
+
+// CHECK: vcvtpd2uqq 291(%rax,%r14,8), %zmm22
+// CHECK:  encoding: [0x62,0xa1,0xfd,0x48,0x79,0xb4,0xf0,0x23,0x01,0x00,0x00]
+          vcvtpd2uqq 291(%rax,%r14,8), %zmm22
+
+// CHECK: vcvtpd2uqq (%rcx){1to8}, %zmm22
+// CHECK:  encoding: [0x62,0xe1,0xfd,0x58,0x79,0x31]
+          vcvtpd2uqq (%rcx){1to8}, %zmm22
+
+// CHECK: vcvtpd2uqq 8128(%rdx), %zmm22
+// CHECK:  encoding: [0x62,0xe1,0xfd,0x48,0x79,0x72,0x7f]
+          vcvtpd2uqq 8128(%rdx), %zmm22
+
+// CHECK: vcvtpd2uqq 8192(%rdx), %zmm22
+// CHECK:  encoding: [0x62,0xe1,0xfd,0x48,0x79,0xb2,0x00,0x20,0x00,0x00]
+          vcvtpd2uqq 8192(%rdx), %zmm22
+
+// CHECK: vcvtpd2uqq -8192(%rdx), %zmm22
+// CHECK:  encoding: [0x62,0xe1,0xfd,0x48,0x79,0x72,0x80]
+          vcvtpd2uqq -8192(%rdx), %zmm22
+
+// CHECK: vcvtpd2uqq -8256(%rdx), %zmm22
+// CHECK:  encoding: [0x62,0xe1,0xfd,0x48,0x79,0xb2,0xc0,0xdf,0xff,0xff]
+          vcvtpd2uqq -8256(%rdx), %zmm22
+
+// CHECK: vcvtpd2uqq 1016(%rdx){1to8}, %zmm22
+// CHECK:  encoding: [0x62,0xe1,0xfd,0x58,0x79,0x72,0x7f]
+          vcvtpd2uqq 1016(%rdx){1to8}, %zmm22
+
+// CHECK: vcvtpd2uqq 1024(%rdx){1to8}, %zmm22
+// CHECK:  encoding: [0x62,0xe1,0xfd,0x58,0x79,0xb2,0x00,0x04,0x00,0x00]
+          vcvtpd2uqq 1024(%rdx){1to8}, %zmm22
+
+// CHECK: vcvtpd2uqq -1024(%rdx){1to8}, %zmm22
+// CHECK:  encoding: [0x62,0xe1,0xfd,0x58,0x79,0x72,0x80]
+          vcvtpd2uqq -1024(%rdx){1to8}, %zmm22
+
+// CHECK: vcvtpd2uqq -1032(%rdx){1to8}, %zmm22
+// CHECK:  encoding: [0x62,0xe1,0xfd,0x58,0x79,0xb2,0xf8,0xfb,0xff,0xff]
+          vcvtpd2uqq -1032(%rdx){1to8}, %zmm22
+
+// CHECK: vcvtps2qq %ymm18, %zmm20
+// CHECK:  encoding: [0x62,0xa1,0x7d,0x48,0x7b,0xe2]
+          vcvtps2qq %ymm18, %zmm20
+
+// CHECK: vcvtps2qq %ymm18, %zmm20 {%k2}
+// CHECK:  encoding: [0x62,0xa1,0x7d,0x4a,0x7b,0xe2]
+          vcvtps2qq %ymm18, %zmm20 {%k2}
+
+// CHECK: vcvtps2qq %ymm18, %zmm20 {%k2} {z}
+// CHECK:  encoding: [0x62,0xa1,0x7d,0xca,0x7b,0xe2]
+          vcvtps2qq %ymm18, %zmm20 {%k2} {z}
+
+// CHECK: vcvtps2qq {rn-sae}, %ymm18, %zmm20
+// CHECK:  encoding: [0x62,0xa1,0x7d,0x18,0x7b,0xe2]
+          vcvtps2qq {rn-sae}, %ymm18, %zmm20
+
+// CHECK: vcvtps2qq {ru-sae}, %ymm18, %zmm20
+// CHECK:  encoding: [0x62,0xa1,0x7d,0x58,0x7b,0xe2]
+          vcvtps2qq {ru-sae}, %ymm18, %zmm20
+
+// CHECK: vcvtps2qq {rd-sae}, %ymm18, %zmm20
+// CHECK:  encoding: [0x62,0xa1,0x7d,0x38,0x7b,0xe2]
+          vcvtps2qq {rd-sae}, %ymm18, %zmm20
+
+// CHECK: vcvtps2qq {rz-sae}, %ymm18, %zmm20
+// CHECK:  encoding: [0x62,0xa1,0x7d,0x78,0x7b,0xe2]
+          vcvtps2qq {rz-sae}, %ymm18, %zmm20
+
+// CHECK: vcvtps2qq (%rcx), %zmm20
+// CHECK:  encoding: [0x62,0xe1,0x7d,0x48,0x7b,0x21]
+          vcvtps2qq (%rcx), %zmm20
+
+// CHECK: vcvtps2qq 291(%rax,%r14,8), %zmm20
+// CHECK:  encoding: [0x62,0xa1,0x7d,0x48,0x7b,0xa4,0xf0,0x23,0x01,0x00,0x00]
+          vcvtps2qq 291(%rax,%r14,8), %zmm20
+
+// CHECK: vcvtps2qq (%rcx){1to8}, %zmm20
+// CHECK:  encoding: [0x62,0xe1,0x7d,0x58,0x7b,0x21]
+          vcvtps2qq (%rcx){1to8}, %zmm20
+
+// CHECK: vcvtps2qq 4064(%rdx), %zmm20
+// CHECK:  encoding: [0x62,0xe1,0x7d,0x48,0x7b,0x62,0x7f]
+          vcvtps2qq 4064(%rdx), %zmm20
+
+// CHECK: vcvtps2qq 4096(%rdx), %zmm20
+// CHECK:  encoding: [0x62,0xe1,0x7d,0x48,0x7b,0xa2,0x00,0x10,0x00,0x00]
+          vcvtps2qq 4096(%rdx), %zmm20
+
+// CHECK: vcvtps2qq -4096(%rdx), %zmm20
+// CHECK:  encoding: [0x62,0xe1,0x7d,0x48,0x7b,0x62,0x80]
+          vcvtps2qq -4096(%rdx), %zmm20
+
+// CHECK: vcvtps2qq -4128(%rdx), %zmm20
+// CHECK:  encoding: [0x62,0xe1,0x7d,0x48,0x7b,0xa2,0xe0,0xef,0xff,0xff]
+          vcvtps2qq -4128(%rdx), %zmm20
+
+// CHECK: vcvtps2qq 508(%rdx){1to8}, %zmm20
+// CHECK:  encoding: [0x62,0xe1,0x7d,0x58,0x7b,0x62,0x7f]
+          vcvtps2qq 508(%rdx){1to8}, %zmm20
+
+// CHECK: vcvtps2qq 512(%rdx){1to8}, %zmm20
+// CHECK:  encoding: [0x62,0xe1,0x7d,0x58,0x7b,0xa2,0x00,0x02,0x00,0x00]
+          vcvtps2qq 512(%rdx){1to8}, %zmm20
+
+// CHECK: vcvtps2qq -512(%rdx){1to8}, %zmm20
+// CHECK:  encoding: [0x62,0xe1,0x7d,0x58,0x7b,0x62,0x80]
+          vcvtps2qq -512(%rdx){1to8}, %zmm20
+
+// CHECK: vcvtps2qq -516(%rdx){1to8}, %zmm20
+// CHECK:  encoding: [0x62,0xe1,0x7d,0x58,0x7b,0xa2,0xfc,0xfd,0xff,0xff]
+          vcvtps2qq -516(%rdx){1to8}, %zmm20
+
+// CHECK: vcvtps2uqq %ymm27, %zmm25
+// CHECK:  encoding: [0x62,0x01,0x7d,0x48,0x79,0xcb]
+          vcvtps2uqq %ymm27, %zmm25
+
+// CHECK: vcvtps2uqq %ymm27, %zmm25 {%k5}
+// CHECK:  encoding: [0x62,0x01,0x7d,0x4d,0x79,0xcb]
+          vcvtps2uqq %ymm27, %zmm25 {%k5}
+
+// CHECK: vcvtps2uqq %ymm27, %zmm25 {%k5} {z}
+// CHECK:  encoding: [0x62,0x01,0x7d,0xcd,0x79,0xcb]
+          vcvtps2uqq %ymm27, %zmm25 {%k5} {z}
+
+// CHECK: vcvtps2uqq {rn-sae}, %ymm27, %zmm25
+// CHECK:  encoding: [0x62,0x01,0x7d,0x18,0x79,0xcb]
+          vcvtps2uqq {rn-sae}, %ymm27, %zmm25
+
+// CHECK: vcvtps2uqq {ru-sae}, %ymm27, %zmm25
+// CHECK:  encoding: [0x62,0x01,0x7d,0x58,0x79,0xcb]
+          vcvtps2uqq {ru-sae}, %ymm27, %zmm25
+
+// CHECK: vcvtps2uqq {rd-sae}, %ymm27, %zmm25
+// CHECK:  encoding: [0x62,0x01,0x7d,0x38,0x79,0xcb]
+          vcvtps2uqq {rd-sae}, %ymm27, %zmm25
+
+// CHECK: vcvtps2uqq {rz-sae}, %ymm27, %zmm25
+// CHECK:  encoding: [0x62,0x01,0x7d,0x78,0x79,0xcb]
+          vcvtps2uqq {rz-sae}, %ymm27, %zmm25
+
+// CHECK: vcvtps2uqq (%rcx), %zmm25
+// CHECK:  encoding: [0x62,0x61,0x7d,0x48,0x79,0x09]
+          vcvtps2uqq (%rcx), %zmm25
+
+// CHECK: vcvtps2uqq 291(%rax,%r14,8), %zmm25
+// CHECK:  encoding: [0x62,0x21,0x7d,0x48,0x79,0x8c,0xf0,0x23,0x01,0x00,0x00]
+          vcvtps2uqq 291(%rax,%r14,8), %zmm25
+
+// CHECK: vcvtps2uqq (%rcx){1to8}, %zmm25
+// CHECK:  encoding: [0x62,0x61,0x7d,0x58,0x79,0x09]
+          vcvtps2uqq (%rcx){1to8}, %zmm25
+
+// CHECK: vcvtps2uqq 4064(%rdx), %zmm25
+// CHECK:  encoding: [0x62,0x61,0x7d,0x48,0x79,0x4a,0x7f]
+          vcvtps2uqq 4064(%rdx), %zmm25
+
+// CHECK: vcvtps2uqq 4096(%rdx), %zmm25
+// CHECK:  encoding: [0x62,0x61,0x7d,0x48,0x79,0x8a,0x00,0x10,0x00,0x00]
+          vcvtps2uqq 4096(%rdx), %zmm25
+
+// CHECK: vcvtps2uqq -4096(%rdx), %zmm25
+// CHECK:  encoding: [0x62,0x61,0x7d,0x48,0x79,0x4a,0x80]
+          vcvtps2uqq -4096(%rdx), %zmm25
+
+// CHECK: vcvtps2uqq -4128(%rdx), %zmm25
+// CHECK:  encoding: [0x62,0x61,0x7d,0x48,0x79,0x8a,0xe0,0xef,0xff,0xff]
+          vcvtps2uqq -4128(%rdx), %zmm25
+
+// CHECK: vcvtps2uqq 508(%rdx){1to8}, %zmm25
+// CHECK:  encoding: [0x62,0x61,0x7d,0x58,0x79,0x4a,0x7f]
+          vcvtps2uqq 508(%rdx){1to8}, %zmm25
+
+// CHECK: vcvtps2uqq 512(%rdx){1to8}, %zmm25
+// CHECK:  encoding: [0x62,0x61,0x7d,0x58,0x79,0x8a,0x00,0x02,0x00,0x00]
+          vcvtps2uqq 512(%rdx){1to8}, %zmm25
+
+// CHECK: vcvtps2uqq -512(%rdx){1to8}, %zmm25
+// CHECK:  encoding: [0x62,0x61,0x7d,0x58,0x79,0x4a,0x80]
+          vcvtps2uqq -512(%rdx){1to8}, %zmm25
+
+// CHECK: vcvtps2uqq -516(%rdx){1to8}, %zmm25
+// CHECK:  encoding: [0x62,0x61,0x7d,0x58,0x79,0x8a,0xfc,0xfd,0xff,0xff]
+          vcvtps2uqq -516(%rdx){1to8}, %zmm25
+
+// CHECK: vcvtqq2pd %zmm25, %zmm17
+// CHECK:  encoding: [0x62,0x81,0xfe,0x48,0xe6,0xc9]
+          vcvtqq2pd %zmm25, %zmm17
+
+// CHECK: vcvtqq2pd %zmm25, %zmm17 {%k4}
+// CHECK:  encoding: [0x62,0x81,0xfe,0x4c,0xe6,0xc9]
+          vcvtqq2pd %zmm25, %zmm17 {%k4}
+
+// CHECK: vcvtqq2pd %zmm25, %zmm17 {%k4} {z}
+// CHECK:  encoding: [0x62,0x81,0xfe,0xcc,0xe6,0xc9]
+          vcvtqq2pd %zmm25, %zmm17 {%k4} {z}
+
+// CHECK: vcvtqq2pd {rn-sae}, %zmm25, %zmm17
+// CHECK:  encoding: [0x62,0x81,0xfe,0x18,0xe6,0xc9]
+          vcvtqq2pd {rn-sae}, %zmm25, %zmm17
+
+// CHECK: vcvtqq2pd {ru-sae}, %zmm25, %zmm17
+// CHECK:  encoding: [0x62,0x81,0xfe,0x58,0xe6,0xc9]
+          vcvtqq2pd {ru-sae}, %zmm25, %zmm17
+
+// CHECK: vcvtqq2pd {rd-sae}, %zmm25, %zmm17
+// CHECK:  encoding: [0x62,0x81,0xfe,0x38,0xe6,0xc9]
+          vcvtqq2pd {rd-sae}, %zmm25, %zmm17
+
+// CHECK: vcvtqq2pd {rz-sae}, %zmm25, %zmm17
+// CHECK:  encoding: [0x62,0x81,0xfe,0x78,0xe6,0xc9]
+          vcvtqq2pd {rz-sae}, %zmm25, %zmm17
+
+// CHECK: vcvtqq2pd (%rcx), %zmm17
+// CHECK:  encoding: [0x62,0xe1,0xfe,0x48,0xe6,0x09]
+          vcvtqq2pd (%rcx), %zmm17
+
+// CHECK: vcvtqq2pd 291(%rax,%r14,8), %zmm17
+// CHECK:  encoding: [0x62,0xa1,0xfe,0x48,0xe6,0x8c,0xf0,0x23,0x01,0x00,0x00]
+          vcvtqq2pd 291(%rax,%r14,8), %zmm17
+
+// CHECK: vcvtqq2pd (%rcx){1to8}, %zmm17
+// CHECK:  encoding: [0x62,0xe1,0xfe,0x58,0xe6,0x09]
+          vcvtqq2pd (%rcx){1to8}, %zmm17
+
+// CHECK: vcvtqq2pd 8128(%rdx), %zmm17
+// CHECK:  encoding: [0x62,0xe1,0xfe,0x48,0xe6,0x4a,0x7f]
+          vcvtqq2pd 8128(%rdx), %zmm17
+
+// CHECK: vcvtqq2pd 8192(%rdx), %zmm17
+// CHECK:  encoding: [0x62,0xe1,0xfe,0x48,0xe6,0x8a,0x00,0x20,0x00,0x00]
+          vcvtqq2pd 8192(%rdx), %zmm17
+
+// CHECK: vcvtqq2pd -8192(%rdx), %zmm17
+// CHECK:  encoding: [0x62,0xe1,0xfe,0x48,0xe6,0x4a,0x80]
+          vcvtqq2pd -8192(%rdx), %zmm17
+
+// CHECK: vcvtqq2pd -8256(%rdx), %zmm17
+// CHECK:  encoding: [0x62,0xe1,0xfe,0x48,0xe6,0x8a,0xc0,0xdf,0xff,0xff]
+          vcvtqq2pd -8256(%rdx), %zmm17
+
+// CHECK: vcvtqq2pd 1016(%rdx){1to8}, %zmm17
+// CHECK:  encoding: [0x62,0xe1,0xfe,0x58,0xe6,0x4a,0x7f]
+          vcvtqq2pd 1016(%rdx){1to8}, %zmm17
+
+// CHECK: vcvtqq2pd 1024(%rdx){1to8}, %zmm17
+// CHECK:  encoding: [0x62,0xe1,0xfe,0x58,0xe6,0x8a,0x00,0x04,0x00,0x00]
+          vcvtqq2pd 1024(%rdx){1to8}, %zmm17
+
+// CHECK: vcvtqq2pd -1024(%rdx){1to8}, %zmm17
+// CHECK:  encoding: [0x62,0xe1,0xfe,0x58,0xe6,0x4a,0x80]
+          vcvtqq2pd -1024(%rdx){1to8}, %zmm17
+
+// CHECK: vcvtqq2pd -1032(%rdx){1to8}, %zmm17
+// CHECK:  encoding: [0x62,0xe1,0xfe,0x58,0xe6,0x8a,0xf8,0xfb,0xff,0xff]
+          vcvtqq2pd -1032(%rdx){1to8}, %zmm17
+
+// CHECK: vcvtqq2ps %zmm27, %ymm20
+// CHECK:  encoding: [0x62,0x81,0xfc,0x48,0x5b,0xe3]
+          vcvtqq2ps %zmm27, %ymm20
+
+// CHECK: vcvtqq2ps %zmm27, %ymm20 {%k5}
+// CHECK:  encoding: [0x62,0x81,0xfc,0x4d,0x5b,0xe3]
+          vcvtqq2ps %zmm27, %ymm20 {%k5}
+
+// CHECK: vcvtqq2ps %zmm27, %ymm20 {%k5} {z}
+// CHECK:  encoding: [0x62,0x81,0xfc,0xcd,0x5b,0xe3]
+          vcvtqq2ps %zmm27, %ymm20 {%k5} {z}
+
+// CHECK: vcvtqq2ps {rn-sae}, %zmm27, %ymm20
+// CHECK:  encoding: [0x62,0x81,0xfc,0x18,0x5b,0xe3]
+          vcvtqq2ps {rn-sae}, %zmm27, %ymm20
+
+// CHECK: vcvtqq2ps {ru-sae}, %zmm27, %ymm20
+// CHECK:  encoding: [0x62,0x81,0xfc,0x58,0x5b,0xe3]
+          vcvtqq2ps {ru-sae}, %zmm27, %ymm20
+
+// CHECK: vcvtqq2ps {rd-sae}, %zmm27, %ymm20
+// CHECK:  encoding: [0x62,0x81,0xfc,0x38,0x5b,0xe3]
+          vcvtqq2ps {rd-sae}, %zmm27, %ymm20
+
+// CHECK: vcvtqq2ps {rz-sae}, %zmm27, %ymm20
+// CHECK:  encoding: [0x62,0x81,0xfc,0x78,0x5b,0xe3]
+          vcvtqq2ps {rz-sae}, %zmm27, %ymm20
+
+// CHECK: vcvtqq2ps (%rcx), %ymm20
+// CHECK:  encoding: [0x62,0xe1,0xfc,0x48,0x5b,0x21]
+          vcvtqq2ps (%rcx), %ymm20
+
+// CHECK: vcvtqq2ps 291(%rax,%r14,8), %ymm20
+// CHECK:  encoding: [0x62,0xa1,0xfc,0x48,0x5b,0xa4,0xf0,0x23,0x01,0x00,0x00]
+          vcvtqq2ps 291(%rax,%r14,8), %ymm20
+
+// CHECK: vcvtqq2ps (%rcx){1to8}, %ymm20
+// CHECK:  encoding: [0x62,0xe1,0xfc,0x58,0x5b,0x21]
+          vcvtqq2ps (%rcx){1to8}, %ymm20
+
+// CHECK: vcvtqq2ps 8128(%rdx), %ymm20
+// CHECK:  encoding: [0x62,0xe1,0xfc,0x48,0x5b,0x62,0x7f]
+          vcvtqq2ps 8128(%rdx), %ymm20
+
+// CHECK: vcvtqq2ps 8192(%rdx), %ymm20
+// CHECK:  encoding: [0x62,0xe1,0xfc,0x48,0x5b,0xa2,0x00,0x20,0x00,0x00]
+          vcvtqq2ps 8192(%rdx), %ymm20
+
+// CHECK: vcvtqq2ps -8192(%rdx), %ymm20
+// CHECK:  encoding: [0x62,0xe1,0xfc,0x48,0x5b,0x62,0x80]
+          vcvtqq2ps -8192(%rdx), %ymm20
+
+// CHECK: vcvtqq2ps -8256(%rdx), %ymm20
+// CHECK:  encoding: [0x62,0xe1,0xfc,0x48,0x5b,0xa2,0xc0,0xdf,0xff,0xff]
+          vcvtqq2ps -8256(%rdx), %ymm20
+
+// CHECK: vcvtqq2ps 1016(%rdx){1to8}, %ymm20
+// CHECK:  encoding: [0x62,0xe1,0xfc,0x58,0x5b,0x62,0x7f]
+          vcvtqq2ps 1016(%rdx){1to8}, %ymm20
+
+// CHECK: vcvtqq2ps 1024(%rdx){1to8}, %ymm20
+// CHECK:  encoding: [0x62,0xe1,0xfc,0x58,0x5b,0xa2,0x00,0x04,0x00,0x00]
+          vcvtqq2ps 1024(%rdx){1to8}, %ymm20
+
+// CHECK: vcvtqq2ps -1024(%rdx){1to8}, %ymm20
+// CHECK:  encoding: [0x62,0xe1,0xfc,0x58,0x5b,0x62,0x80]
+          vcvtqq2ps -1024(%rdx){1to8}, %ymm20
+
+// CHECK: vcvtqq2ps -1032(%rdx){1to8}, %ymm20
+// CHECK:  encoding: [0x62,0xe1,0xfc,0x58,0x5b,0xa2,0xf8,0xfb,0xff,0xff]
+          vcvtqq2ps -1032(%rdx){1to8}, %ymm20
+
+// CHECK: vcvtuqq2pd %zmm29, %zmm21
+// CHECK:  encoding: [0x62,0x81,0xfe,0x48,0x7a,0xed]
+          vcvtuqq2pd %zmm29, %zmm21
+
+// CHECK: vcvtuqq2pd %zmm29, %zmm21 {%k6}
+// CHECK:  encoding: [0x62,0x81,0xfe,0x4e,0x7a,0xed]
+          vcvtuqq2pd %zmm29, %zmm21 {%k6}
+
+// CHECK: vcvtuqq2pd %zmm29, %zmm21 {%k6} {z}
+// CHECK:  encoding: [0x62,0x81,0xfe,0xce,0x7a,0xed]
+          vcvtuqq2pd %zmm29, %zmm21 {%k6} {z}
+
+// CHECK: vcvtuqq2pd {rn-sae}, %zmm29, %zmm21
+// CHECK:  encoding: [0x62,0x81,0xfe,0x18,0x7a,0xed]
+          vcvtuqq2pd {rn-sae}, %zmm29, %zmm21
+
+// CHECK: vcvtuqq2pd {ru-sae}, %zmm29, %zmm21
+// CHECK:  encoding: [0x62,0x81,0xfe,0x58,0x7a,0xed]
+          vcvtuqq2pd {ru-sae}, %zmm29, %zmm21
+
+// CHECK: vcvtuqq2pd {rd-sae}, %zmm29, %zmm21
+// CHECK:  encoding: [0x62,0x81,0xfe,0x38,0x7a,0xed]
+          vcvtuqq2pd {rd-sae}, %zmm29, %zmm21
+
+// CHECK: vcvtuqq2pd {rz-sae}, %zmm29, %zmm21
+// CHECK:  encoding: [0x62,0x81,0xfe,0x78,0x7a,0xed]
+          vcvtuqq2pd {rz-sae}, %zmm29, %zmm21
+
+// CHECK: vcvtuqq2pd (%rcx), %zmm21
+// CHECK:  encoding: [0x62,0xe1,0xfe,0x48,0x7a,0x29]
+          vcvtuqq2pd (%rcx), %zmm21
+
+// CHECK: vcvtuqq2pd 291(%rax,%r14,8), %zmm21
+// CHECK:  encoding: [0x62,0xa1,0xfe,0x48,0x7a,0xac,0xf0,0x23,0x01,0x00,0x00]
+          vcvtuqq2pd 291(%rax,%r14,8), %zmm21
+
+// CHECK: vcvtuqq2pd (%rcx){1to8}, %zmm21
+// CHECK:  encoding: [0x62,0xe1,0xfe,0x58,0x7a,0x29]
+          vcvtuqq2pd (%rcx){1to8}, %zmm21
+
+// CHECK: vcvtuqq2pd 8128(%rdx), %zmm21
+// CHECK:  encoding: [0x62,0xe1,0xfe,0x48,0x7a,0x6a,0x7f]
+          vcvtuqq2pd 8128(%rdx), %zmm21
+
+// CHECK: vcvtuqq2pd 8192(%rdx), %zmm21
+// CHECK:  encoding: [0x62,0xe1,0xfe,0x48,0x7a,0xaa,0x00,0x20,0x00,0x00]
+          vcvtuqq2pd 8192(%rdx), %zmm21
+
+// CHECK: vcvtuqq2pd -8192(%rdx), %zmm21
+// CHECK:  encoding: [0x62,0xe1,0xfe,0x48,0x7a,0x6a,0x80]
+          vcvtuqq2pd -8192(%rdx), %zmm21
+
+// CHECK: vcvtuqq2pd -8256(%rdx), %zmm21
+// CHECK:  encoding: [0x62,0xe1,0xfe,0x48,0x7a,0xaa,0xc0,0xdf,0xff,0xff]
+          vcvtuqq2pd -8256(%rdx), %zmm21
+
+// CHECK: vcvtuqq2pd 1016(%rdx){1to8}, %zmm21
+// CHECK:  encoding: [0x62,0xe1,0xfe,0x58,0x7a,0x6a,0x7f]
+          vcvtuqq2pd 1016(%rdx){1to8}, %zmm21
+
+// CHECK: vcvtuqq2pd 1024(%rdx){1to8}, %zmm21
+// CHECK:  encoding: [0x62,0xe1,0xfe,0x58,0x7a,0xaa,0x00,0x04,0x00,0x00]
+          vcvtuqq2pd 1024(%rdx){1to8}, %zmm21
+
+// CHECK: vcvtuqq2pd -1024(%rdx){1to8}, %zmm21
+// CHECK:  encoding: [0x62,0xe1,0xfe,0x58,0x7a,0x6a,0x80]
+          vcvtuqq2pd -1024(%rdx){1to8}, %zmm21
+
+// CHECK: vcvtuqq2pd -1032(%rdx){1to8}, %zmm21
+// CHECK:  encoding: [0x62,0xe1,0xfe,0x58,0x7a,0xaa,0xf8,0xfb,0xff,0xff]
+          vcvtuqq2pd -1032(%rdx){1to8}, %zmm21
+
+// CHECK: vcvtuqq2ps %zmm21, %ymm18
+// CHECK:  encoding: [0x62,0xa1,0xff,0x48,0x7a,0xd5]
+          vcvtuqq2ps %zmm21, %ymm18
+
+// CHECK: vcvtuqq2ps %zmm21, %ymm18 {%k2}
+// CHECK:  encoding: [0x62,0xa1,0xff,0x4a,0x7a,0xd5]
+          vcvtuqq2ps %zmm21, %ymm18 {%k2}
+
+// CHECK: vcvtuqq2ps %zmm21, %ymm18 {%k2} {z}
+// CHECK:  encoding: [0x62,0xa1,0xff,0xca,0x7a,0xd5]
+          vcvtuqq2ps %zmm21, %ymm18 {%k2} {z}
+
diff --git a/test/MC/X86/x86-64-avx512dq_vl.s b/test/MC/X86/x86-64-avx512dq_vl.s
index 38aab78be2c..e9ffd493fad 100644
--- a/test/MC/X86/x86-64-avx512dq_vl.s
+++ b/test/MC/X86/x86-64-avx512dq_vl.s
@@ -111,3 +111,3476 @@
 // CHECK: vpmullq -1032(%rdx){1to4}, %ymm25, %ymm25
 // CHECK:  encoding: [0x62,0x62,0xb5,0x30,0x40,0x8a,0xf8,0xfb,0xff,0xff]
           vpmullq -1032(%rdx){1to4}, %ymm25, %ymm25
+
+// CHECK: vandpd %xmm20, %xmm29, %xmm21
+// CHECK:  encoding: [0x62,0xa1,0x95,0x00,0x54,0xec]
+          vandpd %xmm20, %xmm29, %xmm21
+
+// CHECK: vandpd %xmm20, %xmm29, %xmm21 {%k6}
+// CHECK:  encoding: [0x62,0xa1,0x95,0x06,0x54,0xec]
+          vandpd %xmm20, %xmm29, %xmm21 {%k6}
+
+// CHECK: vandpd %xmm20, %xmm29, %xmm21 {%k6} {z}
+// CHECK:  encoding: [0x62,0xa1,0x95,0x86,0x54,0xec]
+          vandpd %xmm20, %xmm29, %xmm21 {%k6} {z}
+
+// CHECK: vandpd (%rcx), %xmm29, %xmm21
+// CHECK:  encoding: [0x62,0xe1,0x95,0x00,0x54,0x29]
+          vandpd (%rcx), %xmm29, %xmm21
+
+// CHECK: vandpd 291(%rax,%r14,8), %xmm29, %xmm21
+// CHECK:  encoding: [0x62,0xa1,0x95,0x00,0x54,0xac,0xf0,0x23,0x01,0x00,0x00]
+          vandpd 291(%rax,%r14,8), %xmm29, %xmm21
+
+// CHECK: vandpd (%rcx){1to2}, %xmm29, %xmm21
+// CHECK:  encoding: [0x62,0xe1,0x95,0x10,0x54,0x29]
+          vandpd (%rcx){1to2}, %xmm29, %xmm21
+
+// CHECK: vandpd 2032(%rdx), %xmm29, %xmm21
+// CHECK:  encoding: [0x62,0xe1,0x95,0x00,0x54,0x6a,0x7f]
+          vandpd 2032(%rdx), %xmm29, %xmm21
+
+// CHECK: vandpd 2048(%rdx), %xmm29, %xmm21
+// CHECK:  encoding: [0x62,0xe1,0x95,0x00,0x54,0xaa,0x00,0x08,0x00,0x00]
+          vandpd 2048(%rdx), %xmm29, %xmm21
+
+// CHECK: vandpd -2048(%rdx), %xmm29, %xmm21
+// CHECK:  encoding: [0x62,0xe1,0x95,0x00,0x54,0x6a,0x80]
+          vandpd -2048(%rdx), %xmm29, %xmm21
+
+// CHECK: vandpd -2064(%rdx), %xmm29, %xmm21
+// CHECK:  encoding: [0x62,0xe1,0x95,0x00,0x54,0xaa,0xf0,0xf7,0xff,0xff]
+          vandpd -2064(%rdx), %xmm29, %xmm21
+
+// CHECK: vandpd 1016(%rdx){1to2}, %xmm29, %xmm21
+// CHECK:  encoding: [0x62,0xe1,0x95,0x10,0x54,0x6a,0x7f]
+          vandpd 1016(%rdx){1to2}, %xmm29, %xmm21
+
+// CHECK: vandpd 1024(%rdx){1to2}, %xmm29, %xmm21
+// CHECK:  encoding: [0x62,0xe1,0x95,0x10,0x54,0xaa,0x00,0x04,0x00,0x00]
+          vandpd 1024(%rdx){1to2}, %xmm29, %xmm21
+
+// CHECK: vandpd -1024(%rdx){1to2}, %xmm29, %xmm21
+// CHECK:  encoding: [0x62,0xe1,0x95,0x10,0x54,0x6a,0x80]
+          vandpd -1024(%rdx){1to2}, %xmm29, %xmm21
+
+// CHECK: vandpd -1032(%rdx){1to2}, %xmm29, %xmm21
+// CHECK:  encoding: [0x62,0xe1,0x95,0x10,0x54,0xaa,0xf8,0xfb,0xff,0xff]
+          vandpd -1032(%rdx){1to2}, %xmm29, %xmm21
+
+// CHECK: vandpd %ymm28, %ymm21, %ymm28
+// CHECK:  encoding: [0x62,0x01,0xd5,0x20,0x54,0xe4]
+          vandpd %ymm28, %ymm21, %ymm28
+
+// CHECK: vandpd %ymm28, %ymm21, %ymm28 {%k4}
+// CHECK:  encoding: [0x62,0x01,0xd5,0x24,0x54,0xe4]
+          vandpd %ymm28, %ymm21, %ymm28 {%k4}
+
+// CHECK: vandpd %ymm28, %ymm21, %ymm28 {%k4} {z}
+// CHECK:  encoding: [0x62,0x01,0xd5,0xa4,0x54,0xe4]
+          vandpd %ymm28, %ymm21, %ymm28 {%k4} {z}
+
+// CHECK: vandpd (%rcx), %ymm21, %ymm28
+// CHECK:  encoding: [0x62,0x61,0xd5,0x20,0x54,0x21]
+          vandpd (%rcx), %ymm21, %ymm28
+
+// CHECK: vandpd 291(%rax,%r14,8), %ymm21, %ymm28
+// CHECK:  encoding: [0x62,0x21,0xd5,0x20,0x54,0xa4,0xf0,0x23,0x01,0x00,0x00]
+          vandpd 291(%rax,%r14,8), %ymm21, %ymm28
+
+// CHECK: vandpd (%rcx){1to4}, %ymm21, %ymm28
+// CHECK:  encoding: [0x62,0x61,0xd5,0x30,0x54,0x21]
+          vandpd (%rcx){1to4}, %ymm21, %ymm28
+
+// CHECK: vandpd 4064(%rdx), %ymm21, %ymm28
+// CHECK:  encoding: [0x62,0x61,0xd5,0x20,0x54,0x62,0x7f]
+          vandpd 4064(%rdx), %ymm21, %ymm28
+
+// CHECK: vandpd 4096(%rdx), %ymm21, %ymm28
+// CHECK:  encoding: [0x62,0x61,0xd5,0x20,0x54,0xa2,0x00,0x10,0x00,0x00]
+          vandpd 4096(%rdx), %ymm21, %ymm28
+
+// CHECK: vandpd -4096(%rdx), %ymm21, %ymm28
+// CHECK:  encoding: [0x62,0x61,0xd5,0x20,0x54,0x62,0x80]
+          vandpd -4096(%rdx), %ymm21, %ymm28
+
+// CHECK: vandpd -4128(%rdx), %ymm21, %ymm28
+// CHECK:  encoding: [0x62,0x61,0xd5,0x20,0x54,0xa2,0xe0,0xef,0xff,0xff]
+          vandpd -4128(%rdx), %ymm21, %ymm28
+
+// CHECK: vandpd 1016(%rdx){1to4}, %ymm21, %ymm28
+// CHECK:  encoding: [0x62,0x61,0xd5,0x30,0x54,0x62,0x7f]
+          vandpd 1016(%rdx){1to4}, %ymm21, %ymm28
+
+// CHECK: vandpd 1024(%rdx){1to4}, %ymm21, %ymm28
+// CHECK:  encoding: [0x62,0x61,0xd5,0x30,0x54,0xa2,0x00,0x04,0x00,0x00]
+          vandpd 1024(%rdx){1to4}, %ymm21, %ymm28
+
+// CHECK: vandpd -1024(%rdx){1to4}, %ymm21, %ymm28
+// CHECK:  encoding: [0x62,0x61,0xd5,0x30,0x54,0x62,0x80]
+          vandpd -1024(%rdx){1to4}, %ymm21, %ymm28
+
+// CHECK: vandpd -1032(%rdx){1to4}, %ymm21, %ymm28
+// CHECK:  encoding: [0x62,0x61,0xd5,0x30,0x54,0xa2,0xf8,0xfb,0xff,0xff]
+          vandpd -1032(%rdx){1to4}, %ymm21, %ymm28
+
+// CHECK: vandps %xmm24, %xmm21, %xmm23
+// CHECK:  encoding: [0x62,0x81,0x54,0x00,0x54,0xf8]
+          vandps %xmm24, %xmm21, %xmm23
+
+// CHECK: vandps %xmm24, %xmm21, %xmm23 {%k5}
+// CHECK:  encoding: [0x62,0x81,0x54,0x05,0x54,0xf8]
+          vandps %xmm24, %xmm21, %xmm23 {%k5}
+
+// CHECK: vandps %xmm24, %xmm21, %xmm23 {%k5} {z}
+// CHECK:  encoding: [0x62,0x81,0x54,0x85,0x54,0xf8]
+          vandps %xmm24, %xmm21, %xmm23 {%k5} {z}
+
+// CHECK: vandps (%rcx), %xmm21, %xmm23
+// CHECK:  encoding: [0x62,0xe1,0x54,0x00,0x54,0x39]
+          vandps (%rcx), %xmm21, %xmm23
+
+// CHECK: vandps 291(%rax,%r14,8), %xmm21, %xmm23
+// CHECK:  encoding: [0x62,0xa1,0x54,0x00,0x54,0xbc,0xf0,0x23,0x01,0x00,0x00]
+          vandps 291(%rax,%r14,8), %xmm21, %xmm23
+
+// CHECK: vandps (%rcx){1to4}, %xmm21, %xmm23
+// CHECK:  encoding: [0x62,0xe1,0x54,0x10,0x54,0x39]
+          vandps (%rcx){1to4}, %xmm21, %xmm23
+
+// CHECK: vandps 2032(%rdx), %xmm21, %xmm23
+// CHECK:  encoding: [0x62,0xe1,0x54,0x00,0x54,0x7a,0x7f]
+          vandps 2032(%rdx), %xmm21, %xmm23
+
+// CHECK: vandps 2048(%rdx), %xmm21, %xmm23
+// CHECK:  encoding: [0x62,0xe1,0x54,0x00,0x54,0xba,0x00,0x08,0x00,0x00]
+          vandps 2048(%rdx), %xmm21, %xmm23
+
+// CHECK: vandps -2048(%rdx), %xmm21, %xmm23
+// CHECK:  encoding: [0x62,0xe1,0x54,0x00,0x54,0x7a,0x80]
+          vandps -2048(%rdx), %xmm21, %xmm23
+
+// CHECK: vandps -2064(%rdx), %xmm21, %xmm23
+// CHECK:  encoding: [0x62,0xe1,0x54,0x00,0x54,0xba,0xf0,0xf7,0xff,0xff]
+          vandps -2064(%rdx), %xmm21, %xmm23
+
+// CHECK: vandps 508(%rdx){1to4}, %xmm21, %xmm23
+// CHECK:  encoding: [0x62,0xe1,0x54,0x10,0x54,0x7a,0x7f]
+          vandps 508(%rdx){1to4}, %xmm21, %xmm23
+
+// CHECK: vandps 512(%rdx){1to4}, %xmm21, %xmm23
+// CHECK:  encoding: [0x62,0xe1,0x54,0x10,0x54,0xba,0x00,0x02,0x00,0x00]
+          vandps 512(%rdx){1to4}, %xmm21, %xmm23
+
+// CHECK: vandps -512(%rdx){1to4}, %xmm21, %xmm23
+// CHECK:  encoding: [0x62,0xe1,0x54,0x10,0x54,0x7a,0x80]
+          vandps -512(%rdx){1to4}, %xmm21, %xmm23
+
+// CHECK: vandps -516(%rdx){1to4}, %xmm21, %xmm23
+// CHECK:  encoding: [0x62,0xe1,0x54,0x10,0x54,0xba,0xfc,0xfd,0xff,0xff]
+          vandps -516(%rdx){1to4}, %xmm21, %xmm23
+
+// CHECK: vandps %ymm23, %ymm18, %ymm26
+// CHECK:  encoding: [0x62,0x21,0x6c,0x20,0x54,0xd7]
+          vandps %ymm23, %ymm18, %ymm26
+
+// CHECK: vandps %ymm23, %ymm18, %ymm26 {%k6}
+// CHECK:  encoding: [0x62,0x21,0x6c,0x26,0x54,0xd7]
+          vandps %ymm23, %ymm18, %ymm26 {%k6}
+
+// CHECK: vandps %ymm23, %ymm18, %ymm26 {%k6} {z}
+// CHECK:  encoding: [0x62,0x21,0x6c,0xa6,0x54,0xd7]
+          vandps %ymm23, %ymm18, %ymm26 {%k6} {z}
+
+// CHECK: vandps (%rcx), %ymm18, %ymm26
+// CHECK:  encoding: [0x62,0x61,0x6c,0x20,0x54,0x11]
+          vandps (%rcx), %ymm18, %ymm26
+
+// CHECK: vandps 291(%rax,%r14,8), %ymm18, %ymm26
+// CHECK:  encoding: [0x62,0x21,0x6c,0x20,0x54,0x94,0xf0,0x23,0x01,0x00,0x00]
+          vandps 291(%rax,%r14,8), %ymm18, %ymm26
+
+// CHECK: vandps (%rcx){1to8}, %ymm18, %ymm26
+// CHECK:  encoding: [0x62,0x61,0x6c,0x30,0x54,0x11]
+          vandps (%rcx){1to8}, %ymm18, %ymm26
+
+// CHECK: vandps 4064(%rdx), %ymm18, %ymm26
+// CHECK:  encoding: [0x62,0x61,0x6c,0x20,0x54,0x52,0x7f]
+          vandps 4064(%rdx), %ymm18, %ymm26
+
+// CHECK: vandps 4096(%rdx), %ymm18, %ymm26
+// CHECK:  encoding: [0x62,0x61,0x6c,0x20,0x54,0x92,0x00,0x10,0x00,0x00]
+          vandps 4096(%rdx), %ymm18, %ymm26
+
+// CHECK: vandps -4096(%rdx), %ymm18, %ymm26
+// CHECK:  encoding: [0x62,0x61,0x6c,0x20,0x54,0x52,0x80]
+          vandps -4096(%rdx), %ymm18, %ymm26
+
+// CHECK: vandps -4128(%rdx), %ymm18, %ymm26
+// CHECK:  encoding: [0x62,0x61,0x6c,0x20,0x54,0x92,0xe0,0xef,0xff,0xff]
+          vandps -4128(%rdx), %ymm18, %ymm26
+
+// CHECK: vandps 508(%rdx){1to8}, %ymm18, %ymm26
+// CHECK:  encoding: [0x62,0x61,0x6c,0x30,0x54,0x52,0x7f]
+          vandps 508(%rdx){1to8}, %ymm18, %ymm26
+
+// CHECK: vandps 512(%rdx){1to8}, %ymm18, %ymm26
+// CHECK:  encoding: [0x62,0x61,0x6c,0x30,0x54,0x92,0x00,0x02,0x00,0x00]
+          vandps 512(%rdx){1to8}, %ymm18, %ymm26
+
+// CHECK: vandps -512(%rdx){1to8}, %ymm18, %ymm26
+// CHECK:  encoding: [0x62,0x61,0x6c,0x30,0x54,0x52,0x80]
+          vandps -512(%rdx){1to8}, %ymm18, %ymm26
+
+// CHECK: vandps -516(%rdx){1to8}, %ymm18, %ymm26
+// CHECK:  encoding: [0x62,0x61,0x6c,0x30,0x54,0x92,0xfc,0xfd,0xff,0xff]
+          vandps -516(%rdx){1to8}, %ymm18, %ymm26
+
+// CHECK: vandnpd %xmm25, %xmm27, %xmm25
+// CHECK:  encoding: [0x62,0x01,0xa5,0x00,0x55,0xc9]
+          vandnpd %xmm25, %xmm27, %xmm25
+
+// CHECK: vandnpd %xmm25, %xmm27, %xmm25 {%k5}
+// CHECK:  encoding: [0x62,0x01,0xa5,0x05,0x55,0xc9]
+          vandnpd %xmm25, %xmm27, %xmm25 {%k5}
+
+// CHECK: vandnpd %xmm25, %xmm27, %xmm25 {%k5} {z}
+// CHECK:  encoding: [0x62,0x01,0xa5,0x85,0x55,0xc9]
+          vandnpd %xmm25, %xmm27, %xmm25 {%k5} {z}
+
+// CHECK: vandnpd (%rcx), %xmm27, %xmm25
+// CHECK:  encoding: [0x62,0x61,0xa5,0x00,0x55,0x09]
+          vandnpd (%rcx), %xmm27, %xmm25
+
+// CHECK: vandnpd 291(%rax,%r14,8), %xmm27, %xmm25
+// CHECK:  encoding: [0x62,0x21,0xa5,0x00,0x55,0x8c,0xf0,0x23,0x01,0x00,0x00]
+          vandnpd 291(%rax,%r14,8), %xmm27, %xmm25
+
+// CHECK: vandnpd (%rcx){1to2}, %xmm27, %xmm25
+// CHECK:  encoding: [0x62,0x61,0xa5,0x10,0x55,0x09]
+          vandnpd (%rcx){1to2}, %xmm27, %xmm25
+
+// CHECK: vandnpd 2032(%rdx), %xmm27, %xmm25
+// CHECK:  encoding: [0x62,0x61,0xa5,0x00,0x55,0x4a,0x7f]
+          vandnpd 2032(%rdx), %xmm27, %xmm25
+
+// CHECK: vandnpd 2048(%rdx), %xmm27, %xmm25
+// CHECK:  encoding: [0x62,0x61,0xa5,0x00,0x55,0x8a,0x00,0x08,0x00,0x00]
+          vandnpd 2048(%rdx), %xmm27, %xmm25
+
+// CHECK: vandnpd -2048(%rdx), %xmm27, %xmm25
+// CHECK:  encoding: [0x62,0x61,0xa5,0x00,0x55,0x4a,0x80]
+          vandnpd -2048(%rdx), %xmm27, %xmm25
+
+// CHECK: vandnpd -2064(%rdx), %xmm27, %xmm25
+// CHECK:  encoding: [0x62,0x61,0xa5,0x00,0x55,0x8a,0xf0,0xf7,0xff,0xff]
+          vandnpd -2064(%rdx), %xmm27, %xmm25
+
+// CHECK: vandnpd 1016(%rdx){1to2}, %xmm27, %xmm25
+// CHECK:  encoding: [0x62,0x61,0xa5,0x10,0x55,0x4a,0x7f]
+          vandnpd 1016(%rdx){1to2}, %xmm27, %xmm25
+
+// CHECK: vandnpd 1024(%rdx){1to2}, %xmm27, %xmm25
+// CHECK:  encoding: [0x62,0x61,0xa5,0x10,0x55,0x8a,0x00,0x04,0x00,0x00]
+          vandnpd 1024(%rdx){1to2}, %xmm27, %xmm25
+
+// CHECK: vandnpd -1024(%rdx){1to2}, %xmm27, %xmm25
+// CHECK:  encoding: [0x62,0x61,0xa5,0x10,0x55,0x4a,0x80]
+          vandnpd -1024(%rdx){1to2}, %xmm27, %xmm25
+
+// CHECK: vandnpd -1032(%rdx){1to2}, %xmm27, %xmm25
+// CHECK:  encoding: [0x62,0x61,0xa5,0x10,0x55,0x8a,0xf8,0xfb,0xff,0xff]
+          vandnpd -1032(%rdx){1to2}, %xmm27, %xmm25
+
+// CHECK: vandnpd %ymm22, %ymm18, %ymm22
+// CHECK:  encoding: [0x62,0xa1,0xed,0x20,0x55,0xf6]
+          vandnpd %ymm22, %ymm18, %ymm22
+
+// CHECK: vandnpd %ymm22, %ymm18, %ymm22 {%k7}
+// CHECK:  encoding: [0x62,0xa1,0xed,0x27,0x55,0xf6]
+          vandnpd %ymm22, %ymm18, %ymm22 {%k7}
+
+// CHECK: vandnpd %ymm22, %ymm18, %ymm22 {%k7} {z}
+// CHECK:  encoding: [0x62,0xa1,0xed,0xa7,0x55,0xf6]
+          vandnpd %ymm22, %ymm18, %ymm22 {%k7} {z}
+
+// CHECK: vandnpd (%rcx), %ymm18, %ymm22
+// CHECK:  encoding: [0x62,0xe1,0xed,0x20,0x55,0x31]
+          vandnpd (%rcx), %ymm18, %ymm22
+
+// CHECK: vandnpd 291(%rax,%r14,8), %ymm18, %ymm22
+// CHECK:  encoding: [0x62,0xa1,0xed,0x20,0x55,0xb4,0xf0,0x23,0x01,0x00,0x00]
+          vandnpd 291(%rax,%r14,8), %ymm18, %ymm22
+
+// CHECK: vandnpd (%rcx){1to4}, %ymm18, %ymm22
+// CHECK:  encoding: [0x62,0xe1,0xed,0x30,0x55,0x31]
+          vandnpd (%rcx){1to4}, %ymm18, %ymm22
+
+// CHECK: vandnpd 4064(%rdx), %ymm18, %ymm22
+// CHECK:  encoding: [0x62,0xe1,0xed,0x20,0x55,0x72,0x7f]
+          vandnpd 4064(%rdx), %ymm18, %ymm22
+
+// CHECK: vandnpd 4096(%rdx), %ymm18, %ymm22
+// CHECK:  encoding: [0x62,0xe1,0xed,0x20,0x55,0xb2,0x00,0x10,0x00,0x00]
+          vandnpd 4096(%rdx), %ymm18, %ymm22
+
+// CHECK: vandnpd -4096(%rdx), %ymm18, %ymm22
+// CHECK:  encoding: [0x62,0xe1,0xed,0x20,0x55,0x72,0x80]
+          vandnpd -4096(%rdx), %ymm18, %ymm22
+
+// CHECK: vandnpd -4128(%rdx), %ymm18, %ymm22
+// CHECK:  encoding: [0x62,0xe1,0xed,0x20,0x55,0xb2,0xe0,0xef,0xff,0xff]
+          vandnpd -4128(%rdx), %ymm18, %ymm22
+
+// CHECK: vandnpd 1016(%rdx){1to4}, %ymm18, %ymm22
+// CHECK:  encoding: [0x62,0xe1,0xed,0x30,0x55,0x72,0x7f]
+          vandnpd 1016(%rdx){1to4}, %ymm18, %ymm22
+
+// CHECK: vandnpd 1024(%rdx){1to4}, %ymm18, %ymm22
+// CHECK:  encoding: [0x62,0xe1,0xed,0x30,0x55,0xb2,0x00,0x04,0x00,0x00]
+          vandnpd 1024(%rdx){1to4}, %ymm18, %ymm22
+
+// CHECK: vandnpd -1024(%rdx){1to4}, %ymm18, %ymm22
+// CHECK:  encoding: [0x62,0xe1,0xed,0x30,0x55,0x72,0x80]
+          vandnpd -1024(%rdx){1to4}, %ymm18, %ymm22
+
+// CHECK: vandnpd -1032(%rdx){1to4}, %ymm18, %ymm22
+// CHECK:  encoding: [0x62,0xe1,0xed,0x30,0x55,0xb2,0xf8,0xfb,0xff,0xff]
+          vandnpd -1032(%rdx){1to4}, %ymm18, %ymm22
+
+// CHECK: vandnps %xmm27, %xmm21, %xmm21
+// CHECK:  encoding: [0x62,0x81,0x54,0x00,0x55,0xeb]
+          vandnps %xmm27, %xmm21, %xmm21
+
+// CHECK: vandnps %xmm27, %xmm21, %xmm21 {%k2}
+// CHECK:  encoding: [0x62,0x81,0x54,0x02,0x55,0xeb]
+          vandnps %xmm27, %xmm21, %xmm21 {%k2}
+
+// CHECK: vandnps %xmm27, %xmm21, %xmm21 {%k2} {z}
+// CHECK:  encoding: [0x62,0x81,0x54,0x82,0x55,0xeb]
+          vandnps %xmm27, %xmm21, %xmm21 {%k2} {z}
+
+// CHECK: vandnps (%rcx), %xmm21, %xmm21
+// CHECK:  encoding: [0x62,0xe1,0x54,0x00,0x55,0x29]
+          vandnps (%rcx), %xmm21, %xmm21
+
+// CHECK: vandnps 291(%rax,%r14,8), %xmm21, %xmm21
+// CHECK:  encoding: [0x62,0xa1,0x54,0x00,0x55,0xac,0xf0,0x23,0x01,0x00,0x00]
+          vandnps 291(%rax,%r14,8), %xmm21, %xmm21
+
+// CHECK: vandnps (%rcx){1to4}, %xmm21, %xmm21
+// CHECK:  encoding: [0x62,0xe1,0x54,0x10,0x55,0x29]
+          vandnps (%rcx){1to4}, %xmm21, %xmm21
+
+// CHECK: vandnps 2032(%rdx), %xmm21, %xmm21
+// CHECK:  encoding: [0x62,0xe1,0x54,0x00,0x55,0x6a,0x7f]
+          vandnps 2032(%rdx), %xmm21, %xmm21
+
+// CHECK: vandnps 2048(%rdx), %xmm21, %xmm21
+// CHECK:  encoding: [0x62,0xe1,0x54,0x00,0x55,0xaa,0x00,0x08,0x00,0x00]
+          vandnps 2048(%rdx), %xmm21, %xmm21
+
+// CHECK: vandnps -2048(%rdx), %xmm21, %xmm21
+// CHECK:  encoding: [0x62,0xe1,0x54,0x00,0x55,0x6a,0x80]
+          vandnps -2048(%rdx), %xmm21, %xmm21
+
+// CHECK: vandnps -2064(%rdx), %xmm21, %xmm21
+// CHECK:  encoding: [0x62,0xe1,0x54,0x00,0x55,0xaa,0xf0,0xf7,0xff,0xff]
+          vandnps -2064(%rdx), %xmm21, %xmm21
+
+// CHECK: vandnps 508(%rdx){1to4}, %xmm21, %xmm21
+// CHECK:  encoding: [0x62,0xe1,0x54,0x10,0x55,0x6a,0x7f]
+          vandnps 508(%rdx){1to4}, %xmm21, %xmm21
+
+// CHECK: vandnps 512(%rdx){1to4}, %xmm21, %xmm21
+// CHECK:  encoding: [0x62,0xe1,0x54,0x10,0x55,0xaa,0x00,0x02,0x00,0x00]
+          vandnps 512(%rdx){1to4}, %xmm21, %xmm21
+
+// CHECK: vandnps -512(%rdx){1to4}, %xmm21, %xmm21
+// CHECK:  encoding: [0x62,0xe1,0x54,0x10,0x55,0x6a,0x80]
+          vandnps -512(%rdx){1to4}, %xmm21, %xmm21
+
+// CHECK: vandnps -516(%rdx){1to4}, %xmm21, %xmm21
+// CHECK:  encoding: [0x62,0xe1,0x54,0x10,0x55,0xaa,0xfc,0xfd,0xff,0xff]
+          vandnps -516(%rdx){1to4}, %xmm21, %xmm21
+
+// CHECK: vandnps %ymm25, %ymm23, %ymm19
+// CHECK:  encoding: [0x62,0x81,0x44,0x20,0x55,0xd9]
+          vandnps %ymm25, %ymm23, %ymm19
+
+// CHECK: vandnps %ymm25, %ymm23, %ymm19 {%k1}
+// CHECK:  encoding: [0x62,0x81,0x44,0x21,0x55,0xd9]
+          vandnps %ymm25, %ymm23, %ymm19 {%k1}
+
+// CHECK: vandnps %ymm25, %ymm23, %ymm19 {%k1} {z}
+// CHECK:  encoding: [0x62,0x81,0x44,0xa1,0x55,0xd9]
+          vandnps %ymm25, %ymm23, %ymm19 {%k1} {z}
+
+// CHECK: vandnps (%rcx), %ymm23, %ymm19
+// CHECK:  encoding: [0x62,0xe1,0x44,0x20,0x55,0x19]
+          vandnps (%rcx), %ymm23, %ymm19
+
+// CHECK: vandnps 291(%rax,%r14,8), %ymm23, %ymm19
+// CHECK:  encoding: [0x62,0xa1,0x44,0x20,0x55,0x9c,0xf0,0x23,0x01,0x00,0x00]
+          vandnps 291(%rax,%r14,8), %ymm23, %ymm19
+
+// CHECK: vandnps (%rcx){1to8}, %ymm23, %ymm19
+// CHECK:  encoding: [0x62,0xe1,0x44,0x30,0x55,0x19]
+          vandnps (%rcx){1to8}, %ymm23, %ymm19
+
+// CHECK: vandnps 4064(%rdx), %ymm23, %ymm19
+// CHECK:  encoding: [0x62,0xe1,0x44,0x20,0x55,0x5a,0x7f]
+          vandnps 4064(%rdx), %ymm23, %ymm19
+
+// CHECK: vandnps 4096(%rdx), %ymm23, %ymm19
+// CHECK:  encoding: [0x62,0xe1,0x44,0x20,0x55,0x9a,0x00,0x10,0x00,0x00]
+          vandnps 4096(%rdx), %ymm23, %ymm19
+
+// CHECK: vandnps -4096(%rdx), %ymm23, %ymm19
+// CHECK:  encoding: [0x62,0xe1,0x44,0x20,0x55,0x5a,0x80]
+          vandnps -4096(%rdx), %ymm23, %ymm19
+
+// CHECK: vandnps -4128(%rdx), %ymm23, %ymm19
+// CHECK:  encoding: [0x62,0xe1,0x44,0x20,0x55,0x9a,0xe0,0xef,0xff,0xff]
+          vandnps -4128(%rdx), %ymm23, %ymm19
+
+// CHECK: vandnps 508(%rdx){1to8}, %ymm23, %ymm19
+// CHECK:  encoding: [0x62,0xe1,0x44,0x30,0x55,0x5a,0x7f]
+          vandnps 508(%rdx){1to8}, %ymm23, %ymm19
+
+// CHECK: vandnps 512(%rdx){1to8}, %ymm23, %ymm19
+// CHECK:  encoding: [0x62,0xe1,0x44,0x30,0x55,0x9a,0x00,0x02,0x00,0x00]
+          vandnps 512(%rdx){1to8}, %ymm23, %ymm19
+
+// CHECK: vandnps -512(%rdx){1to8}, %ymm23, %ymm19
+// CHECK:  encoding: [0x62,0xe1,0x44,0x30,0x55,0x5a,0x80]
+          vandnps -512(%rdx){1to8}, %ymm23, %ymm19
+
+// CHECK: vandnps -516(%rdx){1to8}, %ymm23, %ymm19
+// CHECK:  encoding: [0x62,0xe1,0x44,0x30,0x55,0x9a,0xfc,0xfd,0xff,0xff]
+          vandnps -516(%rdx){1to8}, %ymm23, %ymm19
+
+// CHECK: vorpd  %xmm18, %xmm27, %xmm23
+// CHECK:  encoding: [0x62,0xa1,0xa5,0x00,0x56,0xfa]
+          vorpd  %xmm18, %xmm27, %xmm23
+
+// CHECK: vorpd  %xmm18, %xmm27, %xmm23 {%k1}
+// CHECK:  encoding: [0x62,0xa1,0xa5,0x01,0x56,0xfa]
+          vorpd  %xmm18, %xmm27, %xmm23 {%k1}
+
+// CHECK: vorpd  %xmm18, %xmm27, %xmm23 {%k1} {z}
+// CHECK:  encoding: [0x62,0xa1,0xa5,0x81,0x56,0xfa]
+          vorpd  %xmm18, %xmm27, %xmm23 {%k1} {z}
+
+// CHECK: vorpd  (%rcx), %xmm27, %xmm23
+// CHECK:  encoding: [0x62,0xe1,0xa5,0x00,0x56,0x39]
+          vorpd  (%rcx), %xmm27, %xmm23
+
+// CHECK: vorpd  291(%rax,%r14,8), %xmm27, %xmm23
+// CHECK:  encoding: [0x62,0xa1,0xa5,0x00,0x56,0xbc,0xf0,0x23,0x01,0x00,0x00]
+          vorpd  291(%rax,%r14,8), %xmm27, %xmm23
+
+// CHECK: vorpd  (%rcx){1to2}, %xmm27, %xmm23
+// CHECK:  encoding: [0x62,0xe1,0xa5,0x10,0x56,0x39]
+          vorpd  (%rcx){1to2}, %xmm27, %xmm23
+
+// CHECK: vorpd  2032(%rdx), %xmm27, %xmm23
+// CHECK:  encoding: [0x62,0xe1,0xa5,0x00,0x56,0x7a,0x7f]
+          vorpd  2032(%rdx), %xmm27, %xmm23
+
+// CHECK: vorpd  2048(%rdx), %xmm27, %xmm23
+// CHECK:  encoding: [0x62,0xe1,0xa5,0x00,0x56,0xba,0x00,0x08,0x00,0x00]
+          vorpd  2048(%rdx), %xmm27, %xmm23
+
+// CHECK: vorpd  -2048(%rdx), %xmm27, %xmm23
+// CHECK:  encoding: [0x62,0xe1,0xa5,0x00,0x56,0x7a,0x80]
+          vorpd  -2048(%rdx), %xmm27, %xmm23
+
+// CHECK: vorpd  -2064(%rdx), %xmm27, %xmm23
+// CHECK:  encoding: [0x62,0xe1,0xa5,0x00,0x56,0xba,0xf0,0xf7,0xff,0xff]
+          vorpd  -2064(%rdx), %xmm27, %xmm23
+
+// CHECK: vorpd  1016(%rdx){1to2}, %xmm27, %xmm23
+// CHECK:  encoding: [0x62,0xe1,0xa5,0x10,0x56,0x7a,0x7f]
+          vorpd  1016(%rdx){1to2}, %xmm27, %xmm23
+
+// CHECK: vorpd  1024(%rdx){1to2}, %xmm27, %xmm23
+// CHECK:  encoding: [0x62,0xe1,0xa5,0x10,0x56,0xba,0x00,0x04,0x00,0x00]
+          vorpd  1024(%rdx){1to2}, %xmm27, %xmm23
+
+// CHECK: vorpd  -1024(%rdx){1to2}, %xmm27, %xmm23
+// CHECK:  encoding: [0x62,0xe1,0xa5,0x10,0x56,0x7a,0x80]
+          vorpd  -1024(%rdx){1to2}, %xmm27, %xmm23
+
+// CHECK: vorpd  -1032(%rdx){1to2}, %xmm27, %xmm23
+// CHECK:  encoding: [0x62,0xe1,0xa5,0x10,0x56,0xba,0xf8,0xfb,0xff,0xff]
+          vorpd  -1032(%rdx){1to2}, %xmm27, %xmm23
+
+// CHECK: vorpd  %ymm20, %ymm29, %ymm26
+// CHECK:  encoding: [0x62,0x21,0x95,0x20,0x56,0xd4]
+          vorpd  %ymm20, %ymm29, %ymm26
+
+// CHECK: vorpd  %ymm20, %ymm29, %ymm26 {%k5}
+// CHECK:  encoding: [0x62,0x21,0x95,0x25,0x56,0xd4]
+          vorpd  %ymm20, %ymm29, %ymm26 {%k5}
+
+// CHECK: vorpd  %ymm20, %ymm29, %ymm26 {%k5} {z}
+// CHECK:  encoding: [0x62,0x21,0x95,0xa5,0x56,0xd4]
+          vorpd  %ymm20, %ymm29, %ymm26 {%k5} {z}
+
+// CHECK: vorpd  (%rcx), %ymm29, %ymm26
+// CHECK:  encoding: [0x62,0x61,0x95,0x20,0x56,0x11]
+          vorpd  (%rcx), %ymm29, %ymm26
+
+// CHECK: vorpd  291(%rax,%r14,8), %ymm29, %ymm26
+// CHECK:  encoding: [0x62,0x21,0x95,0x20,0x56,0x94,0xf0,0x23,0x01,0x00,0x00]
+          vorpd  291(%rax,%r14,8), %ymm29, %ymm26
+
+// CHECK: vorpd  (%rcx){1to4}, %ymm29, %ymm26
+// CHECK:  encoding: [0x62,0x61,0x95,0x30,0x56,0x11]
+          vorpd  (%rcx){1to4}, %ymm29, %ymm26
+
+// CHECK: vorpd  4064(%rdx), %ymm29, %ymm26
+// CHECK:  encoding: [0x62,0x61,0x95,0x20,0x56,0x52,0x7f]
+          vorpd  4064(%rdx), %ymm29, %ymm26
+
+// CHECK: vorpd  4096(%rdx), %ymm29, %ymm26
+// CHECK:  encoding: [0x62,0x61,0x95,0x20,0x56,0x92,0x00,0x10,0x00,0x00]
+          vorpd  4096(%rdx), %ymm29, %ymm26
+
+// CHECK: vorpd  -4096(%rdx), %ymm29, %ymm26
+// CHECK:  encoding: [0x62,0x61,0x95,0x20,0x56,0x52,0x80]
+          vorpd  -4096(%rdx), %ymm29, %ymm26
+
+// CHECK: vorpd  -4128(%rdx), %ymm29, %ymm26
+// CHECK:  encoding: [0x62,0x61,0x95,0x20,0x56,0x92,0xe0,0xef,0xff,0xff]
+          vorpd  -4128(%rdx), %ymm29, %ymm26
+
+// CHECK: vorpd  1016(%rdx){1to4}, %ymm29, %ymm26
+// CHECK:  encoding: [0x62,0x61,0x95,0x30,0x56,0x52,0x7f]
+          vorpd  1016(%rdx){1to4}, %ymm29, %ymm26
+
+// CHECK: vorpd  1024(%rdx){1to4}, %ymm29, %ymm26
+// CHECK:  encoding: [0x62,0x61,0x95,0x30,0x56,0x92,0x00,0x04,0x00,0x00]
+          vorpd  1024(%rdx){1to4}, %ymm29, %ymm26
+
+// CHECK: vorpd  -1024(%rdx){1to4}, %ymm29, %ymm26
+// CHECK:  encoding: [0x62,0x61,0x95,0x30,0x56,0x52,0x80]
+          vorpd  -1024(%rdx){1to4}, %ymm29, %ymm26
+
+// CHECK: vorpd  -1032(%rdx){1to4}, %ymm29, %ymm26
+// CHECK:  encoding: [0x62,0x61,0x95,0x30,0x56,0x92,0xf8,0xfb,0xff,0xff]
+          vorpd  -1032(%rdx){1to4}, %ymm29, %ymm26
+
+// CHECK: vorps  %xmm27, %xmm28, %xmm19
+// CHECK:  encoding: [0x62,0x81,0x1c,0x00,0x56,0xdb]
+          vorps  %xmm27, %xmm28, %xmm19
+
+// CHECK: vorps  %xmm27, %xmm28, %xmm19 {%k4}
+// CHECK:  encoding: [0x62,0x81,0x1c,0x04,0x56,0xdb]
+          vorps  %xmm27, %xmm28, %xmm19 {%k4}
+
+// CHECK: vorps  %xmm27, %xmm28, %xmm19 {%k4} {z}
+// CHECK:  encoding: [0x62,0x81,0x1c,0x84,0x56,0xdb]
+          vorps  %xmm27, %xmm28, %xmm19 {%k4} {z}
+
+// CHECK: vorps  (%rcx), %xmm28, %xmm19
+// CHECK:  encoding: [0x62,0xe1,0x1c,0x00,0x56,0x19]
+          vorps  (%rcx), %xmm28, %xmm19
+
+// CHECK: vorps  291(%rax,%r14,8), %xmm28, %xmm19
+// CHECK:  encoding: [0x62,0xa1,0x1c,0x00,0x56,0x9c,0xf0,0x23,0x01,0x00,0x00]
+          vorps  291(%rax,%r14,8), %xmm28, %xmm19
+
+// CHECK: vorps  (%rcx){1to4}, %xmm28, %xmm19
+// CHECK:  encoding: [0x62,0xe1,0x1c,0x10,0x56,0x19]
+          vorps  (%rcx){1to4}, %xmm28, %xmm19
+
+// CHECK: vorps  2032(%rdx), %xmm28, %xmm19
+// CHECK:  encoding: [0x62,0xe1,0x1c,0x00,0x56,0x5a,0x7f]
+          vorps  2032(%rdx), %xmm28, %xmm19
+
+// CHECK: vorps  2048(%rdx), %xmm28, %xmm19
+// CHECK:  encoding: [0x62,0xe1,0x1c,0x00,0x56,0x9a,0x00,0x08,0x00,0x00]
+          vorps  2048(%rdx), %xmm28, %xmm19
+
+// CHECK: vorps  -2048(%rdx), %xmm28, %xmm19
+// CHECK:  encoding: [0x62,0xe1,0x1c,0x00,0x56,0x5a,0x80]
+          vorps  -2048(%rdx), %xmm28, %xmm19
+
+// CHECK: vorps  -2064(%rdx), %xmm28, %xmm19
+// CHECK:  encoding: [0x62,0xe1,0x1c,0x00,0x56,0x9a,0xf0,0xf7,0xff,0xff]
+          vorps  -2064(%rdx), %xmm28, %xmm19
+
+// CHECK: vorps  508(%rdx){1to4}, %xmm28, %xmm19
+// CHECK:  encoding: [0x62,0xe1,0x1c,0x10,0x56,0x5a,0x7f]
+          vorps  508(%rdx){1to4}, %xmm28, %xmm19
+
+// CHECK: vorps  512(%rdx){1to4}, %xmm28, %xmm19
+// CHECK:  encoding: [0x62,0xe1,0x1c,0x10,0x56,0x9a,0x00,0x02,0x00,0x00]
+          vorps  512(%rdx){1to4}, %xmm28, %xmm19
+
+// CHECK: vorps  -512(%rdx){1to4}, %xmm28, %xmm19
+// CHECK:  encoding: [0x62,0xe1,0x1c,0x10,0x56,0x5a,0x80]
+          vorps  -512(%rdx){1to4}, %xmm28, %xmm19
+
+// CHECK: vorps  -516(%rdx){1to4}, %xmm28, %xmm19
+// CHECK:  encoding: [0x62,0xe1,0x1c,0x10,0x56,0x9a,0xfc,0xfd,0xff,0xff]
+          vorps  -516(%rdx){1to4}, %xmm28, %xmm19
+
+// CHECK: vorps  %ymm26, %ymm26, %ymm27
+// CHECK:  encoding: [0x62,0x01,0x2c,0x20,0x56,0xda]
+          vorps  %ymm26, %ymm26, %ymm27
+
+// CHECK: vorps  %ymm26, %ymm26, %ymm27 {%k1}
+// CHECK:  encoding: [0x62,0x01,0x2c,0x21,0x56,0xda]
+          vorps  %ymm26, %ymm26, %ymm27 {%k1}
+
+// CHECK: vorps  %ymm26, %ymm26, %ymm27 {%k1} {z}
+// CHECK:  encoding: [0x62,0x01,0x2c,0xa1,0x56,0xda]
+          vorps  %ymm26, %ymm26, %ymm27 {%k1} {z}
+
+// CHECK: vorps  (%rcx), %ymm26, %ymm27
+// CHECK:  encoding: [0x62,0x61,0x2c,0x20,0x56,0x19]
+          vorps  (%rcx), %ymm26, %ymm27
+
+// CHECK: vorps  291(%rax,%r14,8), %ymm26, %ymm27
+// CHECK:  encoding: [0x62,0x21,0x2c,0x20,0x56,0x9c,0xf0,0x23,0x01,0x00,0x00]
+          vorps  291(%rax,%r14,8), %ymm26, %ymm27
+
+// CHECK: vorps  (%rcx){1to8}, %ymm26, %ymm27
+// CHECK:  encoding: [0x62,0x61,0x2c,0x30,0x56,0x19]
+          vorps  (%rcx){1to8}, %ymm26, %ymm27
+
+// CHECK: vorps  4064(%rdx), %ymm26, %ymm27
+// CHECK:  encoding: [0x62,0x61,0x2c,0x20,0x56,0x5a,0x7f]
+          vorps  4064(%rdx), %ymm26, %ymm27
+
+// CHECK: vorps  4096(%rdx), %ymm26, %ymm27
+// CHECK:  encoding: [0x62,0x61,0x2c,0x20,0x56,0x9a,0x00,0x10,0x00,0x00]
+          vorps  4096(%rdx), %ymm26, %ymm27
+
+// CHECK: vorps  -4096(%rdx), %ymm26, %ymm27
+// CHECK:  encoding: [0x62,0x61,0x2c,0x20,0x56,0x5a,0x80]
+          vorps  -4096(%rdx), %ymm26, %ymm27
+
+// CHECK: vorps  -4128(%rdx), %ymm26, %ymm27
+// CHECK:  encoding: [0x62,0x61,0x2c,0x20,0x56,0x9a,0xe0,0xef,0xff,0xff]
+          vorps  -4128(%rdx), %ymm26, %ymm27
+
+// CHECK: vorps  508(%rdx){1to8}, %ymm26, %ymm27
+// CHECK:  encoding: [0x62,0x61,0x2c,0x30,0x56,0x5a,0x7f]
+          vorps  508(%rdx){1to8}, %ymm26, %ymm27
+
+// CHECK: vorps  512(%rdx){1to8}, %ymm26, %ymm27
+// CHECK:  encoding: [0x62,0x61,0x2c,0x30,0x56,0x9a,0x00,0x02,0x00,0x00]
+          vorps  512(%rdx){1to8}, %ymm26, %ymm27
+
+// CHECK: vorps  -512(%rdx){1to8}, %ymm26, %ymm27
+// CHECK:  encoding: [0x62,0x61,0x2c,0x30,0x56,0x5a,0x80]
+          vorps  -512(%rdx){1to8}, %ymm26, %ymm27
+
+// CHECK: vorps  -516(%rdx){1to8}, %ymm26, %ymm27
+// CHECK:  encoding: [0x62,0x61,0x2c,0x30,0x56,0x9a,0xfc,0xfd,0xff,0xff]
+          vorps  -516(%rdx){1to8}, %ymm26, %ymm27
+
+// CHECK: vxorpd %xmm23, %xmm21, %xmm18
+// CHECK:  encoding: [0x62,0xa1,0xd5,0x00,0x57,0xd7]
+          vxorpd %xmm23, %xmm21, %xmm18
+
+// CHECK: vxorpd %xmm23, %xmm21, %xmm18 {%k2}
+// CHECK:  encoding: [0x62,0xa1,0xd5,0x02,0x57,0xd7]
+          vxorpd %xmm23, %xmm21, %xmm18 {%k2}
+
+// CHECK: vxorpd %xmm23, %xmm21, %xmm18 {%k2} {z}
+// CHECK:  encoding: [0x62,0xa1,0xd5,0x82,0x57,0xd7]
+          vxorpd %xmm23, %xmm21, %xmm18 {%k2} {z}
+
+// CHECK: vxorpd (%rcx), %xmm21, %xmm18
+// CHECK:  encoding: [0x62,0xe1,0xd5,0x00,0x57,0x11]
+          vxorpd (%rcx), %xmm21, %xmm18
+
+// CHECK: vxorpd 291(%rax,%r14,8), %xmm21, %xmm18
+// CHECK:  encoding: [0x62,0xa1,0xd5,0x00,0x57,0x94,0xf0,0x23,0x01,0x00,0x00]
+          vxorpd 291(%rax,%r14,8), %xmm21, %xmm18
+
+// CHECK: vxorpd (%rcx){1to2}, %xmm21, %xmm18
+// CHECK:  encoding: [0x62,0xe1,0xd5,0x10,0x57,0x11]
+          vxorpd (%rcx){1to2}, %xmm21, %xmm18
+
+// CHECK: vxorpd 2032(%rdx), %xmm21, %xmm18
+// CHECK:  encoding: [0x62,0xe1,0xd5,0x00,0x57,0x52,0x7f]
+          vxorpd 2032(%rdx), %xmm21, %xmm18
+
+// CHECK: vxorpd 2048(%rdx), %xmm21, %xmm18
+// CHECK:  encoding: [0x62,0xe1,0xd5,0x00,0x57,0x92,0x00,0x08,0x00,0x00]
+          vxorpd 2048(%rdx), %xmm21, %xmm18
+
+// CHECK: vxorpd -2048(%rdx), %xmm21, %xmm18
+// CHECK:  encoding: [0x62,0xe1,0xd5,0x00,0x57,0x52,0x80]
+          vxorpd -2048(%rdx), %xmm21, %xmm18
+
+// CHECK: vxorpd -2064(%rdx), %xmm21, %xmm18
+// CHECK:  encoding: [0x62,0xe1,0xd5,0x00,0x57,0x92,0xf0,0xf7,0xff,0xff]
+          vxorpd -2064(%rdx), %xmm21, %xmm18
+
+// CHECK: vxorpd 1016(%rdx){1to2}, %xmm21, %xmm18
+// CHECK:  encoding: [0x62,0xe1,0xd5,0x10,0x57,0x52,0x7f]
+          vxorpd 1016(%rdx){1to2}, %xmm21, %xmm18
+
+// CHECK: vxorpd 1024(%rdx){1to2}, %xmm21, %xmm18
+// CHECK:  encoding: [0x62,0xe1,0xd5,0x10,0x57,0x92,0x00,0x04,0x00,0x00]
+          vxorpd 1024(%rdx){1to2}, %xmm21, %xmm18
+
+// CHECK: vxorpd -1024(%rdx){1to2}, %xmm21, %xmm18
+// CHECK:  encoding: [0x62,0xe1,0xd5,0x10,0x57,0x52,0x80]
+          vxorpd -1024(%rdx){1to2}, %xmm21, %xmm18
+
+// CHECK: vxorpd -1032(%rdx){1to2}, %xmm21, %xmm18
+// CHECK:  encoding: [0x62,0xe1,0xd5,0x10,0x57,0x92,0xf8,0xfb,0xff,0xff]
+          vxorpd -1032(%rdx){1to2}, %xmm21, %xmm18
+
+// CHECK: vxorpd %ymm19, %ymm29, %ymm24
+// CHECK:  encoding: [0x62,0x21,0x95,0x20,0x57,0xc3]
+          vxorpd %ymm19, %ymm29, %ymm24
+
+// CHECK: vxorpd %ymm19, %ymm29, %ymm24 {%k7}
+// CHECK:  encoding: [0x62,0x21,0x95,0x27,0x57,0xc3]
+          vxorpd %ymm19, %ymm29, %ymm24 {%k7}
+
+// CHECK: vxorpd %ymm19, %ymm29, %ymm24 {%k7} {z}
+// CHECK:  encoding: [0x62,0x21,0x95,0xa7,0x57,0xc3]
+          vxorpd %ymm19, %ymm29, %ymm24 {%k7} {z}
+
+// CHECK: vxorpd (%rcx), %ymm29, %ymm24
+// CHECK:  encoding: [0x62,0x61,0x95,0x20,0x57,0x01]
+          vxorpd (%rcx), %ymm29, %ymm24
+
+// CHECK: vxorpd 291(%rax,%r14,8), %ymm29, %ymm24
+// CHECK:  encoding: [0x62,0x21,0x95,0x20,0x57,0x84,0xf0,0x23,0x01,0x00,0x00]
+          vxorpd 291(%rax,%r14,8), %ymm29, %ymm24
+
+// CHECK: vxorpd (%rcx){1to4}, %ymm29, %ymm24
+// CHECK:  encoding: [0x62,0x61,0x95,0x30,0x57,0x01]
+          vxorpd (%rcx){1to4}, %ymm29, %ymm24
+
+// CHECK: vxorpd 4064(%rdx), %ymm29, %ymm24
+// CHECK:  encoding: [0x62,0x61,0x95,0x20,0x57,0x42,0x7f]
+          vxorpd 4064(%rdx), %ymm29, %ymm24
+
+// CHECK: vxorpd 4096(%rdx), %ymm29, %ymm24
+// CHECK:  encoding: [0x62,0x61,0x95,0x20,0x57,0x82,0x00,0x10,0x00,0x00]
+          vxorpd 4096(%rdx), %ymm29, %ymm24
+
+// CHECK: vxorpd -4096(%rdx), %ymm29, %ymm24
+// CHECK:  encoding: [0x62,0x61,0x95,0x20,0x57,0x42,0x80]
+          vxorpd -4096(%rdx), %ymm29, %ymm24
+
+// CHECK: vxorpd -4128(%rdx), %ymm29, %ymm24
+// CHECK:  encoding: [0x62,0x61,0x95,0x20,0x57,0x82,0xe0,0xef,0xff,0xff]
+          vxorpd -4128(%rdx), %ymm29, %ymm24
+
+// CHECK: vxorpd 1016(%rdx){1to4}, %ymm29, %ymm24
+// CHECK:  encoding: [0x62,0x61,0x95,0x30,0x57,0x42,0x7f]
+          vxorpd 1016(%rdx){1to4}, %ymm29, %ymm24
+
+// CHECK: vxorpd 1024(%rdx){1to4}, %ymm29, %ymm24
+// CHECK:  encoding: [0x62,0x61,0x95,0x30,0x57,0x82,0x00,0x04,0x00,0x00]
+          vxorpd 1024(%rdx){1to4}, %ymm29, %ymm24
+
+// CHECK: vxorpd -1024(%rdx){1to4}, %ymm29, %ymm24
+// CHECK:  encoding: [0x62,0x61,0x95,0x30,0x57,0x42,0x80]
+          vxorpd -1024(%rdx){1to4}, %ymm29, %ymm24
+
+// CHECK: vxorpd -1032(%rdx){1to4}, %ymm29, %ymm24
+// CHECK:  encoding: [0x62,0x61,0x95,0x30,0x57,0x82,0xf8,0xfb,0xff,0xff]
+          vxorpd -1032(%rdx){1to4}, %ymm29, %ymm24
+
+// CHECK: vxorps %xmm19, %xmm18, %xmm20
+// CHECK:  encoding: [0x62,0xa1,0x6c,0x00,0x57,0xe3]
+          vxorps %xmm19, %xmm18, %xmm20
+
+// CHECK: vxorps %xmm19, %xmm18, %xmm20 {%k1}
+// CHECK:  encoding: [0x62,0xa1,0x6c,0x01,0x57,0xe3]
+          vxorps %xmm19, %xmm18, %xmm20 {%k1}
+
+// CHECK: vxorps %xmm19, %xmm18, %xmm20 {%k1} {z}
+// CHECK:  encoding: [0x62,0xa1,0x6c,0x81,0x57,0xe3]
+          vxorps %xmm19, %xmm18, %xmm20 {%k1} {z}
+
+// CHECK: vxorps (%rcx), %xmm18, %xmm20
+// CHECK:  encoding: [0x62,0xe1,0x6c,0x00,0x57,0x21]
+          vxorps (%rcx), %xmm18, %xmm20
+
+// CHECK: vxorps 291(%rax,%r14,8), %xmm18, %xmm20
+// CHECK:  encoding: [0x62,0xa1,0x6c,0x00,0x57,0xa4,0xf0,0x23,0x01,0x00,0x00]
+          vxorps 291(%rax,%r14,8), %xmm18, %xmm20
+
+// CHECK: vxorps (%rcx){1to4}, %xmm18, %xmm20
+// CHECK:  encoding: [0x62,0xe1,0x6c,0x10,0x57,0x21]
+          vxorps (%rcx){1to4}, %xmm18, %xmm20
+
+// CHECK: vxorps 2032(%rdx), %xmm18, %xmm20
+// CHECK:  encoding: [0x62,0xe1,0x6c,0x00,0x57,0x62,0x7f]
+          vxorps 2032(%rdx), %xmm18, %xmm20
+
+// CHECK: vxorps 2048(%rdx), %xmm18, %xmm20
+// CHECK:  encoding: [0x62,0xe1,0x6c,0x00,0x57,0xa2,0x00,0x08,0x00,0x00]
+          vxorps 2048(%rdx), %xmm18, %xmm20
+
+// CHECK: vxorps -2048(%rdx), %xmm18, %xmm20
+// CHECK:  encoding: [0x62,0xe1,0x6c,0x00,0x57,0x62,0x80]
+          vxorps -2048(%rdx), %xmm18, %xmm20
+
+// CHECK: vxorps -2064(%rdx), %xmm18, %xmm20
+// CHECK:  encoding: [0x62,0xe1,0x6c,0x00,0x57,0xa2,0xf0,0xf7,0xff,0xff]
+          vxorps -2064(%rdx), %xmm18, %xmm20
+
+// CHECK: vxorps 508(%rdx){1to4}, %xmm18, %xmm20
+// CHECK:  encoding: [0x62,0xe1,0x6c,0x10,0x57,0x62,0x7f]
+          vxorps 508(%rdx){1to4}, %xmm18, %xmm20
+
+// CHECK: vxorps 512(%rdx){1to4}, %xmm18, %xmm20
+// CHECK:  encoding: [0x62,0xe1,0x6c,0x10,0x57,0xa2,0x00,0x02,0x00,0x00]
+          vxorps 512(%rdx){1to4}, %xmm18, %xmm20
+
+// CHECK: vxorps -512(%rdx){1to4}, %xmm18, %xmm20
+// CHECK:  encoding: [0x62,0xe1,0x6c,0x10,0x57,0x62,0x80]
+          vxorps -512(%rdx){1to4}, %xmm18, %xmm20
+
+// CHECK: vxorps -516(%rdx){1to4}, %xmm18, %xmm20
+// CHECK:  encoding: [0x62,0xe1,0x6c,0x10,0x57,0xa2,0xfc,0xfd,0xff,0xff]
+          vxorps -516(%rdx){1to4}, %xmm18, %xmm20
+
+// CHECK: vxorps %ymm24, %ymm20, %ymm27
+// CHECK:  encoding: [0x62,0x01,0x5c,0x20,0x57,0xd8]
+          vxorps %ymm24, %ymm20, %ymm27
+
+// CHECK: vxorps %ymm24, %ymm20, %ymm27 {%k2}
+// CHECK:  encoding: [0x62,0x01,0x5c,0x22,0x57,0xd8]
+          vxorps %ymm24, %ymm20, %ymm27 {%k2}
+
+// CHECK: vxorps %ymm24, %ymm20, %ymm27 {%k2} {z}
+// CHECK:  encoding: [0x62,0x01,0x5c,0xa2,0x57,0xd8]
+          vxorps %ymm24, %ymm20, %ymm27 {%k2} {z}
+
+// CHECK: vxorps (%rcx), %ymm20, %ymm27
+// CHECK:  encoding: [0x62,0x61,0x5c,0x20,0x57,0x19]
+          vxorps (%rcx), %ymm20, %ymm27
+
+// CHECK: vxorps 291(%rax,%r14,8), %ymm20, %ymm27
+// CHECK:  encoding: [0x62,0x21,0x5c,0x20,0x57,0x9c,0xf0,0x23,0x01,0x00,0x00]
+          vxorps 291(%rax,%r14,8), %ymm20, %ymm27
+
+// CHECK: vxorps (%rcx){1to8}, %ymm20, %ymm27
+// CHECK:  encoding: [0x62,0x61,0x5c,0x30,0x57,0x19]
+          vxorps (%rcx){1to8}, %ymm20, %ymm27
+
+// CHECK: vxorps 4064(%rdx), %ymm20, %ymm27
+// CHECK:  encoding: [0x62,0x61,0x5c,0x20,0x57,0x5a,0x7f]
+          vxorps 4064(%rdx), %ymm20, %ymm27
+
+// CHECK: vxorps 4096(%rdx), %ymm20, %ymm27
+// CHECK:  encoding: [0x62,0x61,0x5c,0x20,0x57,0x9a,0x00,0x10,0x00,0x00]
+          vxorps 4096(%rdx), %ymm20, %ymm27
+
+// CHECK: vxorps -4096(%rdx), %ymm20, %ymm27
+// CHECK:  encoding: [0x62,0x61,0x5c,0x20,0x57,0x5a,0x80]
+          vxorps -4096(%rdx), %ymm20, %ymm27
+
+// CHECK: vxorps -4128(%rdx), %ymm20, %ymm27
+// CHECK:  encoding: [0x62,0x61,0x5c,0x20,0x57,0x9a,0xe0,0xef,0xff,0xff]
+          vxorps -4128(%rdx), %ymm20, %ymm27
+
+// CHECK: vxorps 508(%rdx){1to8}, %ymm20, %ymm27
+// CHECK:  encoding: [0x62,0x61,0x5c,0x30,0x57,0x5a,0x7f]
+          vxorps 508(%rdx){1to8}, %ymm20, %ymm27
+
+// CHECK: vxorps 512(%rdx){1to8}, %ymm20, %ymm27
+// CHECK:  encoding: [0x62,0x61,0x5c,0x30,0x57,0x9a,0x00,0x02,0x00,0x00]
+          vxorps 512(%rdx){1to8}, %ymm20, %ymm27
+
+// CHECK: vxorps -512(%rdx){1to8}, %ymm20, %ymm27
+// CHECK:  encoding: [0x62,0x61,0x5c,0x30,0x57,0x5a,0x80]
+          vxorps -512(%rdx){1to8}, %ymm20, %ymm27
+
+// CHECK: vxorps -516(%rdx){1to8}, %ymm20, %ymm27
+// CHECK:  encoding: [0x62,0x61,0x5c,0x30,0x57,0x9a,0xfc,0xfd,0xff,0xff]
+          vxorps -516(%rdx){1to8}, %ymm20, %ymm27
+
+// CHECK: vandpd %xmm27, %xmm25, %xmm19
+// CHECK:  encoding: [0x62,0x81,0xb5,0x00,0x54,0xdb]
+          vandpd %xmm27, %xmm25, %xmm19
+
+// CHECK: vandpd %xmm27, %xmm25, %xmm19 {%k6}
+// CHECK:  encoding: [0x62,0x81,0xb5,0x06,0x54,0xdb]
+          vandpd %xmm27, %xmm25, %xmm19 {%k6}
+
+// CHECK: vandpd %xmm27, %xmm25, %xmm19 {%k6} {z}
+// CHECK:  encoding: [0x62,0x81,0xb5,0x86,0x54,0xdb]
+          vandpd %xmm27, %xmm25, %xmm19 {%k6} {z}
+
+// CHECK: vandpd (%rcx), %xmm25, %xmm19
+// CHECK:  encoding: [0x62,0xe1,0xb5,0x00,0x54,0x19]
+          vandpd (%rcx), %xmm25, %xmm19
+
+// CHECK: vandpd 4660(%rax,%r14,8), %xmm25, %xmm19
+// CHECK:  encoding: [0x62,0xa1,0xb5,0x00,0x54,0x9c,0xf0,0x34,0x12,0x00,0x00]
+          vandpd 4660(%rax,%r14,8), %xmm25, %xmm19
+
+// CHECK: vandpd (%rcx){1to2}, %xmm25, %xmm19
+// CHECK:  encoding: [0x62,0xe1,0xb5,0x10,0x54,0x19]
+          vandpd (%rcx){1to2}, %xmm25, %xmm19
+
+// CHECK: vandpd 2032(%rdx), %xmm25, %xmm19
+// CHECK:  encoding: [0x62,0xe1,0xb5,0x00,0x54,0x5a,0x7f]
+          vandpd 2032(%rdx), %xmm25, %xmm19
+
+// CHECK: vandpd 2048(%rdx), %xmm25, %xmm19
+// CHECK:  encoding: [0x62,0xe1,0xb5,0x00,0x54,0x9a,0x00,0x08,0x00,0x00]
+          vandpd 2048(%rdx), %xmm25, %xmm19
+
+// CHECK: vandpd -2048(%rdx), %xmm25, %xmm19
+// CHECK:  encoding: [0x62,0xe1,0xb5,0x00,0x54,0x5a,0x80]
+          vandpd -2048(%rdx), %xmm25, %xmm19
+
+// CHECK: vandpd -2064(%rdx), %xmm25, %xmm19
+// CHECK:  encoding: [0x62,0xe1,0xb5,0x00,0x54,0x9a,0xf0,0xf7,0xff,0xff]
+          vandpd -2064(%rdx), %xmm25, %xmm19
+
+// CHECK: vandpd 1016(%rdx){1to2}, %xmm25, %xmm19
+// CHECK:  encoding: [0x62,0xe1,0xb5,0x10,0x54,0x5a,0x7f]
+          vandpd 1016(%rdx){1to2}, %xmm25, %xmm19
+
+// CHECK: vandpd 1024(%rdx){1to2}, %xmm25, %xmm19
+// CHECK:  encoding: [0x62,0xe1,0xb5,0x10,0x54,0x9a,0x00,0x04,0x00,0x00]
+          vandpd 1024(%rdx){1to2}, %xmm25, %xmm19
+
+// CHECK: vandpd -1024(%rdx){1to2}, %xmm25, %xmm19
+// CHECK:  encoding: [0x62,0xe1,0xb5,0x10,0x54,0x5a,0x80]
+          vandpd -1024(%rdx){1to2}, %xmm25, %xmm19
+
+// CHECK: vandpd -1032(%rdx){1to2}, %xmm25, %xmm19
+// CHECK:  encoding: [0x62,0xe1,0xb5,0x10,0x54,0x9a,0xf8,0xfb,0xff,0xff]
+          vandpd -1032(%rdx){1to2}, %xmm25, %xmm19
+
+// CHECK: vandpd %ymm21, %ymm29, %ymm24
+// CHECK:  encoding: [0x62,0x21,0x95,0x20,0x54,0xc5]
+          vandpd %ymm21, %ymm29, %ymm24
+
+// CHECK: vandpd %ymm21, %ymm29, %ymm24 {%k2}
+// CHECK:  encoding: [0x62,0x21,0x95,0x22,0x54,0xc5]
+          vandpd %ymm21, %ymm29, %ymm24 {%k2}
+
+// CHECK: vandpd %ymm21, %ymm29, %ymm24 {%k2} {z}
+// CHECK:  encoding: [0x62,0x21,0x95,0xa2,0x54,0xc5]
+          vandpd %ymm21, %ymm29, %ymm24 {%k2} {z}
+
+// CHECK: vandpd (%rcx), %ymm29, %ymm24
+// CHECK:  encoding: [0x62,0x61,0x95,0x20,0x54,0x01]
+          vandpd (%rcx), %ymm29, %ymm24
+
+// CHECK: vandpd 4660(%rax,%r14,8), %ymm29, %ymm24
+// CHECK:  encoding: [0x62,0x21,0x95,0x20,0x54,0x84,0xf0,0x34,0x12,0x00,0x00]
+          vandpd 4660(%rax,%r14,8), %ymm29, %ymm24
+
+// CHECK: vandpd (%rcx){1to4}, %ymm29, %ymm24
+// CHECK:  encoding: [0x62,0x61,0x95,0x30,0x54,0x01]
+          vandpd (%rcx){1to4}, %ymm29, %ymm24
+
+// CHECK: vandpd 4064(%rdx), %ymm29, %ymm24
+// CHECK:  encoding: [0x62,0x61,0x95,0x20,0x54,0x42,0x7f]
+          vandpd 4064(%rdx), %ymm29, %ymm24
+
+// CHECK: vandpd 4096(%rdx), %ymm29, %ymm24
+// CHECK:  encoding: [0x62,0x61,0x95,0x20,0x54,0x82,0x00,0x10,0x00,0x00]
+          vandpd 4096(%rdx), %ymm29, %ymm24
+
+// CHECK: vandpd -4096(%rdx), %ymm29, %ymm24
+// CHECK:  encoding: [0x62,0x61,0x95,0x20,0x54,0x42,0x80]
+          vandpd -4096(%rdx), %ymm29, %ymm24
+
+// CHECK: vandpd -4128(%rdx), %ymm29, %ymm24
+// CHECK:  encoding: [0x62,0x61,0x95,0x20,0x54,0x82,0xe0,0xef,0xff,0xff]
+          vandpd -4128(%rdx), %ymm29, %ymm24
+
+// CHECK: vandpd 1016(%rdx){1to4}, %ymm29, %ymm24
+// CHECK:  encoding: [0x62,0x61,0x95,0x30,0x54,0x42,0x7f]
+          vandpd 1016(%rdx){1to4}, %ymm29, %ymm24
+
+// CHECK: vandpd 1024(%rdx){1to4}, %ymm29, %ymm24
+// CHECK:  encoding: [0x62,0x61,0x95,0x30,0x54,0x82,0x00,0x04,0x00,0x00]
+          vandpd 1024(%rdx){1to4}, %ymm29, %ymm24
+
+// CHECK: vandpd -1024(%rdx){1to4}, %ymm29, %ymm24
+// CHECK:  encoding: [0x62,0x61,0x95,0x30,0x54,0x42,0x80]
+          vandpd -1024(%rdx){1to4}, %ymm29, %ymm24
+
+// CHECK: vandpd -1032(%rdx){1to4}, %ymm29, %ymm24
+// CHECK:  encoding: [0x62,0x61,0x95,0x30,0x54,0x82,0xf8,0xfb,0xff,0xff]
+          vandpd -1032(%rdx){1to4}, %ymm29, %ymm24
+
+// CHECK: vandps %xmm17, %xmm25, %xmm22
+// CHECK:  encoding: [0x62,0xa1,0x34,0x00,0x54,0xf1]
+          vandps %xmm17, %xmm25, %xmm22
+
+// CHECK: vandps %xmm17, %xmm25, %xmm22 {%k3}
+// CHECK:  encoding: [0x62,0xa1,0x34,0x03,0x54,0xf1]
+          vandps %xmm17, %xmm25, %xmm22 {%k3}
+
+// CHECK: vandps %xmm17, %xmm25, %xmm22 {%k3} {z}
+// CHECK:  encoding: [0x62,0xa1,0x34,0x83,0x54,0xf1]
+          vandps %xmm17, %xmm25, %xmm22 {%k3} {z}
+
+// CHECK: vandps (%rcx), %xmm25, %xmm22
+// CHECK:  encoding: [0x62,0xe1,0x34,0x00,0x54,0x31]
+          vandps (%rcx), %xmm25, %xmm22
+
+// CHECK: vandps 4660(%rax,%r14,8), %xmm25, %xmm22
+// CHECK:  encoding: [0x62,0xa1,0x34,0x00,0x54,0xb4,0xf0,0x34,0x12,0x00,0x00]
+          vandps 4660(%rax,%r14,8), %xmm25, %xmm22
+
+// CHECK: vandps (%rcx){1to4}, %xmm25, %xmm22
+// CHECK:  encoding: [0x62,0xe1,0x34,0x10,0x54,0x31]
+          vandps (%rcx){1to4}, %xmm25, %xmm22
+
+// CHECK: vandps 2032(%rdx), %xmm25, %xmm22
+// CHECK:  encoding: [0x62,0xe1,0x34,0x00,0x54,0x72,0x7f]
+          vandps 2032(%rdx), %xmm25, %xmm22
+
+// CHECK: vandps 2048(%rdx), %xmm25, %xmm22
+// CHECK:  encoding: [0x62,0xe1,0x34,0x00,0x54,0xb2,0x00,0x08,0x00,0x00]
+          vandps 2048(%rdx), %xmm25, %xmm22
+
+// CHECK: vandps -2048(%rdx), %xmm25, %xmm22
+// CHECK:  encoding: [0x62,0xe1,0x34,0x00,0x54,0x72,0x80]
+          vandps -2048(%rdx), %xmm25, %xmm22
+
+// CHECK: vandps -2064(%rdx), %xmm25, %xmm22
+// CHECK:  encoding: [0x62,0xe1,0x34,0x00,0x54,0xb2,0xf0,0xf7,0xff,0xff]
+          vandps -2064(%rdx), %xmm25, %xmm22
+
+// CHECK: vandps 508(%rdx){1to4}, %xmm25, %xmm22
+// CHECK:  encoding: [0x62,0xe1,0x34,0x10,0x54,0x72,0x7f]
+          vandps 508(%rdx){1to4}, %xmm25, %xmm22
+
+// CHECK: vandps 512(%rdx){1to4}, %xmm25, %xmm22
+// CHECK:  encoding: [0x62,0xe1,0x34,0x10,0x54,0xb2,0x00,0x02,0x00,0x00]
+          vandps 512(%rdx){1to4}, %xmm25, %xmm22
+
+// CHECK: vandps -512(%rdx){1to4}, %xmm25, %xmm22
+// CHECK:  encoding: [0x62,0xe1,0x34,0x10,0x54,0x72,0x80]
+          vandps -512(%rdx){1to4}, %xmm25, %xmm22
+
+// CHECK: vandps -516(%rdx){1to4}, %xmm25, %xmm22
+// CHECK:  encoding: [0x62,0xe1,0x34,0x10,0x54,0xb2,0xfc,0xfd,0xff,0xff]
+          vandps -516(%rdx){1to4}, %xmm25, %xmm22
+
+// CHECK: vandps %ymm18, %ymm25, %ymm22
+// CHECK:  encoding: [0x62,0xa1,0x34,0x20,0x54,0xf2]
+          vandps %ymm18, %ymm25, %ymm22
+
+// CHECK: vandps %ymm18, %ymm25, %ymm22 {%k1}
+// CHECK:  encoding: [0x62,0xa1,0x34,0x21,0x54,0xf2]
+          vandps %ymm18, %ymm25, %ymm22 {%k1}
+
+// CHECK: vandps %ymm18, %ymm25, %ymm22 {%k1} {z}
+// CHECK:  encoding: [0x62,0xa1,0x34,0xa1,0x54,0xf2]
+          vandps %ymm18, %ymm25, %ymm22 {%k1} {z}
+
+// CHECK: vandps (%rcx), %ymm25, %ymm22
+// CHECK:  encoding: [0x62,0xe1,0x34,0x20,0x54,0x31]
+          vandps (%rcx), %ymm25, %ymm22
+
+// CHECK: vandps 4660(%rax,%r14,8), %ymm25, %ymm22
+// CHECK:  encoding: [0x62,0xa1,0x34,0x20,0x54,0xb4,0xf0,0x34,0x12,0x00,0x00]
+          vandps 4660(%rax,%r14,8), %ymm25, %ymm22
+
+// CHECK: vandps (%rcx){1to8}, %ymm25, %ymm22
+// CHECK:  encoding: [0x62,0xe1,0x34,0x30,0x54,0x31]
+          vandps (%rcx){1to8}, %ymm25, %ymm22
+
+// CHECK: vandps 4064(%rdx), %ymm25, %ymm22
+// CHECK:  encoding: [0x62,0xe1,0x34,0x20,0x54,0x72,0x7f]
+          vandps 4064(%rdx), %ymm25, %ymm22
+
+// CHECK: vandps 4096(%rdx), %ymm25, %ymm22
+// CHECK:  encoding: [0x62,0xe1,0x34,0x20,0x54,0xb2,0x00,0x10,0x00,0x00]
+          vandps 4096(%rdx), %ymm25, %ymm22
+
+// CHECK: vandps -4096(%rdx), %ymm25, %ymm22
+// CHECK:  encoding: [0x62,0xe1,0x34,0x20,0x54,0x72,0x80]
+          vandps -4096(%rdx), %ymm25, %ymm22
+
+// CHECK: vandps -4128(%rdx), %ymm25, %ymm22
+// CHECK:  encoding: [0x62,0xe1,0x34,0x20,0x54,0xb2,0xe0,0xef,0xff,0xff]
+          vandps -4128(%rdx), %ymm25, %ymm22
+
+// CHECK: vandps 508(%rdx){1to8}, %ymm25, %ymm22
+// CHECK:  encoding: [0x62,0xe1,0x34,0x30,0x54,0x72,0x7f]
+          vandps 508(%rdx){1to8}, %ymm25, %ymm22
+
+// CHECK: vandps 512(%rdx){1to8}, %ymm25, %ymm22
+// CHECK:  encoding: [0x62,0xe1,0x34,0x30,0x54,0xb2,0x00,0x02,0x00,0x00]
+          vandps 512(%rdx){1to8}, %ymm25, %ymm22
+
+// CHECK: vandps -512(%rdx){1to8}, %ymm25, %ymm22
+// CHECK:  encoding: [0x62,0xe1,0x34,0x30,0x54,0x72,0x80]
+          vandps -512(%rdx){1to8}, %ymm25, %ymm22
+
+// CHECK: vandps -516(%rdx){1to8}, %ymm25, %ymm22
+// CHECK:  encoding: [0x62,0xe1,0x34,0x30,0x54,0xb2,0xfc,0xfd,0xff,0xff]
+          vandps -516(%rdx){1to8}, %ymm25, %ymm22
+
+// CHECK: vandnpd %xmm23, %xmm18, %xmm19
+// CHECK:  encoding: [0x62,0xa1,0xed,0x00,0x55,0xdf]
+          vandnpd %xmm23, %xmm18, %xmm19
+
+// CHECK: vandnpd %xmm23, %xmm18, %xmm19 {%k1}
+// CHECK:  encoding: [0x62,0xa1,0xed,0x01,0x55,0xdf]
+          vandnpd %xmm23, %xmm18, %xmm19 {%k1}
+
+// CHECK: vandnpd %xmm23, %xmm18, %xmm19 {%k1} {z}
+// CHECK:  encoding: [0x62,0xa1,0xed,0x81,0x55,0xdf]
+          vandnpd %xmm23, %xmm18, %xmm19 {%k1} {z}
+
+// CHECK: vandnpd (%rcx), %xmm18, %xmm19
+// CHECK:  encoding: [0x62,0xe1,0xed,0x00,0x55,0x19]
+          vandnpd (%rcx), %xmm18, %xmm19
+
+// CHECK: vandnpd 4660(%rax,%r14,8), %xmm18, %xmm19
+// CHECK:  encoding: [0x62,0xa1,0xed,0x00,0x55,0x9c,0xf0,0x34,0x12,0x00,0x00]
+          vandnpd 4660(%rax,%r14,8), %xmm18, %xmm19
+
+// CHECK: vandnpd (%rcx){1to2}, %xmm18, %xmm19
+// CHECK:  encoding: [0x62,0xe1,0xed,0x10,0x55,0x19]
+          vandnpd (%rcx){1to2}, %xmm18, %xmm19
+
+// CHECK: vandnpd 2032(%rdx), %xmm18, %xmm19
+// CHECK:  encoding: [0x62,0xe1,0xed,0x00,0x55,0x5a,0x7f]
+          vandnpd 2032(%rdx), %xmm18, %xmm19
+
+// CHECK: vandnpd 2048(%rdx), %xmm18, %xmm19
+// CHECK:  encoding: [0x62,0xe1,0xed,0x00,0x55,0x9a,0x00,0x08,0x00,0x00]
+          vandnpd 2048(%rdx), %xmm18, %xmm19
+
+// CHECK: vandnpd -2048(%rdx), %xmm18, %xmm19
+// CHECK:  encoding: [0x62,0xe1,0xed,0x00,0x55,0x5a,0x80]
+          vandnpd -2048(%rdx), %xmm18, %xmm19
+
+// CHECK: vandnpd -2064(%rdx), %xmm18, %xmm19
+// CHECK:  encoding: [0x62,0xe1,0xed,0x00,0x55,0x9a,0xf0,0xf7,0xff,0xff]
+          vandnpd -2064(%rdx), %xmm18, %xmm19
+
+// CHECK: vandnpd 1016(%rdx){1to2}, %xmm18, %xmm19
+// CHECK:  encoding: [0x62,0xe1,0xed,0x10,0x55,0x5a,0x7f]
+          vandnpd 1016(%rdx){1to2}, %xmm18, %xmm19
+
+// CHECK: vandnpd 1024(%rdx){1to2}, %xmm18, %xmm19
+// CHECK:  encoding: [0x62,0xe1,0xed,0x10,0x55,0x9a,0x00,0x04,0x00,0x00]
+          vandnpd 1024(%rdx){1to2}, %xmm18, %xmm19
+
+// CHECK: vandnpd -1024(%rdx){1to2}, %xmm18, %xmm19
+// CHECK:  encoding: [0x62,0xe1,0xed,0x10,0x55,0x5a,0x80]
+          vandnpd -1024(%rdx){1to2}, %xmm18, %xmm19
+
+// CHECK: vandnpd -1032(%rdx){1to2}, %xmm18, %xmm19
+// CHECK:  encoding: [0x62,0xe1,0xed,0x10,0x55,0x9a,0xf8,0xfb,0xff,0xff]
+          vandnpd -1032(%rdx){1to2}, %xmm18, %xmm19
+
+// CHECK: vandnpd %ymm28, %ymm29, %ymm25
+// CHECK:  encoding: [0x62,0x01,0x95,0x20,0x55,0xcc]
+          vandnpd %ymm28, %ymm29, %ymm25
+
+// CHECK: vandnpd %ymm28, %ymm29, %ymm25 {%k7}
+// CHECK:  encoding: [0x62,0x01,0x95,0x27,0x55,0xcc]
+          vandnpd %ymm28, %ymm29, %ymm25 {%k7}
+
+// CHECK: vandnpd %ymm28, %ymm29, %ymm25 {%k7} {z}
+// CHECK:  encoding: [0x62,0x01,0x95,0xa7,0x55,0xcc]
+          vandnpd %ymm28, %ymm29, %ymm25 {%k7} {z}
+
+// CHECK: vandnpd (%rcx), %ymm29, %ymm25
+// CHECK:  encoding: [0x62,0x61,0x95,0x20,0x55,0x09]
+          vandnpd (%rcx), %ymm29, %ymm25
+
+// CHECK: vandnpd 4660(%rax,%r14,8), %ymm29, %ymm25
+// CHECK:  encoding: [0x62,0x21,0x95,0x20,0x55,0x8c,0xf0,0x34,0x12,0x00,0x00]
+          vandnpd 4660(%rax,%r14,8), %ymm29, %ymm25
+
+// CHECK: vandnpd (%rcx){1to4}, %ymm29, %ymm25
+// CHECK:  encoding: [0x62,0x61,0x95,0x30,0x55,0x09]
+          vandnpd (%rcx){1to4}, %ymm29, %ymm25
+
+// CHECK: vandnpd 4064(%rdx), %ymm29, %ymm25
+// CHECK:  encoding: [0x62,0x61,0x95,0x20,0x55,0x4a,0x7f]
+          vandnpd 4064(%rdx), %ymm29, %ymm25
+
+// CHECK: vandnpd 4096(%rdx), %ymm29, %ymm25
+// CHECK:  encoding: [0x62,0x61,0x95,0x20,0x55,0x8a,0x00,0x10,0x00,0x00]
+          vandnpd 4096(%rdx), %ymm29, %ymm25
+
+// CHECK: vandnpd -4096(%rdx), %ymm29, %ymm25
+// CHECK:  encoding: [0x62,0x61,0x95,0x20,0x55,0x4a,0x80]
+          vandnpd -4096(%rdx), %ymm29, %ymm25
+
+// CHECK: vandnpd -4128(%rdx), %ymm29, %ymm25
+// CHECK:  encoding: [0x62,0x61,0x95,0x20,0x55,0x8a,0xe0,0xef,0xff,0xff]
+          vandnpd -4128(%rdx), %ymm29, %ymm25
+
+// CHECK: vandnpd 1016(%rdx){1to4}, %ymm29, %ymm25
+// CHECK:  encoding: [0x62,0x61,0x95,0x30,0x55,0x4a,0x7f]
+          vandnpd 1016(%rdx){1to4}, %ymm29, %ymm25
+
+// CHECK: vandnpd 1024(%rdx){1to4}, %ymm29, %ymm25
+// CHECK:  encoding: [0x62,0x61,0x95,0x30,0x55,0x8a,0x00,0x04,0x00,0x00]
+          vandnpd 1024(%rdx){1to4}, %ymm29, %ymm25
+
+// CHECK: vandnpd -1024(%rdx){1to4}, %ymm29, %ymm25
+// CHECK:  encoding: [0x62,0x61,0x95,0x30,0x55,0x4a,0x80]
+          vandnpd -1024(%rdx){1to4}, %ymm29, %ymm25
+
+// CHECK: vandnpd -1032(%rdx){1to4}, %ymm29, %ymm25
+// CHECK:  encoding: [0x62,0x61,0x95,0x30,0x55,0x8a,0xf8,0xfb,0xff,0xff]
+          vandnpd -1032(%rdx){1to4}, %ymm29, %ymm25
+
+// CHECK: vandnps %xmm23, %xmm17, %xmm25
+// CHECK:  encoding: [0x62,0x21,0x74,0x00,0x55,0xcf]
+          vandnps %xmm23, %xmm17, %xmm25
+
+// CHECK: vandnps %xmm23, %xmm17, %xmm25 {%k5}
+// CHECK:  encoding: [0x62,0x21,0x74,0x05,0x55,0xcf]
+          vandnps %xmm23, %xmm17, %xmm25 {%k5}
+
+// CHECK: vandnps %xmm23, %xmm17, %xmm25 {%k5} {z}
+// CHECK:  encoding: [0x62,0x21,0x74,0x85,0x55,0xcf]
+          vandnps %xmm23, %xmm17, %xmm25 {%k5} {z}
+
+// CHECK: vandnps (%rcx), %xmm17, %xmm25
+// CHECK:  encoding: [0x62,0x61,0x74,0x00,0x55,0x09]
+          vandnps (%rcx), %xmm17, %xmm25
+
+// CHECK: vandnps 4660(%rax,%r14,8), %xmm17, %xmm25
+// CHECK:  encoding: [0x62,0x21,0x74,0x00,0x55,0x8c,0xf0,0x34,0x12,0x00,0x00]
+          vandnps 4660(%rax,%r14,8), %xmm17, %xmm25
+
+// CHECK: vandnps (%rcx){1to4}, %xmm17, %xmm25
+// CHECK:  encoding: [0x62,0x61,0x74,0x10,0x55,0x09]
+          vandnps (%rcx){1to4}, %xmm17, %xmm25
+
+// CHECK: vandnps 2032(%rdx), %xmm17, %xmm25
+// CHECK:  encoding: [0x62,0x61,0x74,0x00,0x55,0x4a,0x7f]
+          vandnps 2032(%rdx), %xmm17, %xmm25
+
+// CHECK: vandnps 2048(%rdx), %xmm17, %xmm25
+// CHECK:  encoding: [0x62,0x61,0x74,0x00,0x55,0x8a,0x00,0x08,0x00,0x00]
+          vandnps 2048(%rdx), %xmm17, %xmm25
+
+// CHECK: vandnps -2048(%rdx), %xmm17, %xmm25
+// CHECK:  encoding: [0x62,0x61,0x74,0x00,0x55,0x4a,0x80]
+          vandnps -2048(%rdx), %xmm17, %xmm25
+
+// CHECK: vandnps -2064(%rdx), %xmm17, %xmm25
+// CHECK:  encoding: [0x62,0x61,0x74,0x00,0x55,0x8a,0xf0,0xf7,0xff,0xff]
+          vandnps -2064(%rdx), %xmm17, %xmm25
+
+// CHECK: vandnps 508(%rdx){1to4}, %xmm17, %xmm25
+// CHECK:  encoding: [0x62,0x61,0x74,0x10,0x55,0x4a,0x7f]
+          vandnps 508(%rdx){1to4}, %xmm17, %xmm25
+
+// CHECK: vandnps 512(%rdx){1to4}, %xmm17, %xmm25
+// CHECK:  encoding: [0x62,0x61,0x74,0x10,0x55,0x8a,0x00,0x02,0x00,0x00]
+          vandnps 512(%rdx){1to4}, %xmm17, %xmm25
+
+// CHECK: vandnps -512(%rdx){1to4}, %xmm17, %xmm25
+// CHECK:  encoding: [0x62,0x61,0x74,0x10,0x55,0x4a,0x80]
+          vandnps -512(%rdx){1to4}, %xmm17, %xmm25
+
+// CHECK: vandnps -516(%rdx){1to4}, %xmm17, %xmm25
+// CHECK:  encoding: [0x62,0x61,0x74,0x10,0x55,0x8a,0xfc,0xfd,0xff,0xff]
+          vandnps -516(%rdx){1to4}, %xmm17, %xmm25
+
+// CHECK: vandnps %ymm23, %ymm19, %ymm18
+// CHECK:  encoding: [0x62,0xa1,0x64,0x20,0x55,0xd7]
+          vandnps %ymm23, %ymm19, %ymm18
+
+// CHECK: vandnps %ymm23, %ymm19, %ymm18 {%k6}
+// CHECK:  encoding: [0x62,0xa1,0x64,0x26,0x55,0xd7]
+          vandnps %ymm23, %ymm19, %ymm18 {%k6}
+
+// CHECK: vandnps %ymm23, %ymm19, %ymm18 {%k6} {z}
+// CHECK:  encoding: [0x62,0xa1,0x64,0xa6,0x55,0xd7]
+          vandnps %ymm23, %ymm19, %ymm18 {%k6} {z}
+
+// CHECK: vandnps (%rcx), %ymm19, %ymm18
+// CHECK:  encoding: [0x62,0xe1,0x64,0x20,0x55,0x11]
+          vandnps (%rcx), %ymm19, %ymm18
+
+// CHECK: vandnps 4660(%rax,%r14,8), %ymm19, %ymm18
+// CHECK:  encoding: [0x62,0xa1,0x64,0x20,0x55,0x94,0xf0,0x34,0x12,0x00,0x00]
+          vandnps 4660(%rax,%r14,8), %ymm19, %ymm18
+
+// CHECK: vandnps (%rcx){1to8}, %ymm19, %ymm18
+// CHECK:  encoding: [0x62,0xe1,0x64,0x30,0x55,0x11]
+          vandnps (%rcx){1to8}, %ymm19, %ymm18
+
+// CHECK: vandnps 4064(%rdx), %ymm19, %ymm18
+// CHECK:  encoding: [0x62,0xe1,0x64,0x20,0x55,0x52,0x7f]
+          vandnps 4064(%rdx), %ymm19, %ymm18
+
+// CHECK: vandnps 4096(%rdx), %ymm19, %ymm18
+// CHECK:  encoding: [0x62,0xe1,0x64,0x20,0x55,0x92,0x00,0x10,0x00,0x00]
+          vandnps 4096(%rdx), %ymm19, %ymm18
+
+// CHECK: vandnps -4096(%rdx), %ymm19, %ymm18
+// CHECK:  encoding: [0x62,0xe1,0x64,0x20,0x55,0x52,0x80]
+          vandnps -4096(%rdx), %ymm19, %ymm18
+
+// CHECK: vandnps -4128(%rdx), %ymm19, %ymm18
+// CHECK:  encoding: [0x62,0xe1,0x64,0x20,0x55,0x92,0xe0,0xef,0xff,0xff]
+          vandnps -4128(%rdx), %ymm19, %ymm18
+
+// CHECK: vandnps 508(%rdx){1to8}, %ymm19, %ymm18
+// CHECK:  encoding: [0x62,0xe1,0x64,0x30,0x55,0x52,0x7f]
+          vandnps 508(%rdx){1to8}, %ymm19, %ymm18
+
+// CHECK: vandnps 512(%rdx){1to8}, %ymm19, %ymm18
+// CHECK:  encoding: [0x62,0xe1,0x64,0x30,0x55,0x92,0x00,0x02,0x00,0x00]
+          vandnps 512(%rdx){1to8}, %ymm19, %ymm18
+
+// CHECK: vandnps -512(%rdx){1to8}, %ymm19, %ymm18
+// CHECK:  encoding: [0x62,0xe1,0x64,0x30,0x55,0x52,0x80]
+          vandnps -512(%rdx){1to8}, %ymm19, %ymm18
+
+// CHECK: vandnps -516(%rdx){1to8}, %ymm19, %ymm18
+// CHECK:  encoding: [0x62,0xe1,0x64,0x30,0x55,0x92,0xfc,0xfd,0xff,0xff]
+          vandnps -516(%rdx){1to8}, %ymm19, %ymm18
+
+// CHECK: vorpd  %xmm18, %xmm29, %xmm26
+// CHECK:  encoding: [0x62,0x21,0x95,0x00,0x56,0xd2]
+          vorpd  %xmm18, %xmm29, %xmm26
+
+// CHECK: vorpd  %xmm18, %xmm29, %xmm26 {%k2}
+// CHECK:  encoding: [0x62,0x21,0x95,0x02,0x56,0xd2]
+          vorpd  %xmm18, %xmm29, %xmm26 {%k2}
+
+// CHECK: vorpd  %xmm18, %xmm29, %xmm26 {%k2} {z}
+// CHECK:  encoding: [0x62,0x21,0x95,0x82,0x56,0xd2]
+          vorpd  %xmm18, %xmm29, %xmm26 {%k2} {z}
+
+// CHECK: vorpd  (%rcx), %xmm29, %xmm26
+// CHECK:  encoding: [0x62,0x61,0x95,0x00,0x56,0x11]
+          vorpd  (%rcx), %xmm29, %xmm26
+
+// CHECK: vorpd  4660(%rax,%r14,8), %xmm29, %xmm26
+// CHECK:  encoding: [0x62,0x21,0x95,0x00,0x56,0x94,0xf0,0x34,0x12,0x00,0x00]
+          vorpd  4660(%rax,%r14,8), %xmm29, %xmm26
+
+// CHECK: vorpd  (%rcx){1to2}, %xmm29, %xmm26
+// CHECK:  encoding: [0x62,0x61,0x95,0x10,0x56,0x11]
+          vorpd  (%rcx){1to2}, %xmm29, %xmm26
+
+// CHECK: vorpd  2032(%rdx), %xmm29, %xmm26
+// CHECK:  encoding: [0x62,0x61,0x95,0x00,0x56,0x52,0x7f]
+          vorpd  2032(%rdx), %xmm29, %xmm26
+
+// CHECK: vorpd  2048(%rdx), %xmm29, %xmm26
+// CHECK:  encoding: [0x62,0x61,0x95,0x00,0x56,0x92,0x00,0x08,0x00,0x00]
+          vorpd  2048(%rdx), %xmm29, %xmm26
+
+// CHECK: vorpd  -2048(%rdx), %xmm29, %xmm26
+// CHECK:  encoding: [0x62,0x61,0x95,0x00,0x56,0x52,0x80]
+          vorpd  -2048(%rdx), %xmm29, %xmm26
+
+// CHECK: vorpd  -2064(%rdx), %xmm29, %xmm26
+// CHECK:  encoding: [0x62,0x61,0x95,0x00,0x56,0x92,0xf0,0xf7,0xff,0xff]
+          vorpd  -2064(%rdx), %xmm29, %xmm26
+
+// CHECK: vorpd  1016(%rdx){1to2}, %xmm29, %xmm26
+// CHECK:  encoding: [0x62,0x61,0x95,0x10,0x56,0x52,0x7f]
+          vorpd  1016(%rdx){1to2}, %xmm29, %xmm26
+
+// CHECK: vorpd  1024(%rdx){1to2}, %xmm29, %xmm26
+// CHECK:  encoding: [0x62,0x61,0x95,0x10,0x56,0x92,0x00,0x04,0x00,0x00]
+          vorpd  1024(%rdx){1to2}, %xmm29, %xmm26
+
+// CHECK: vorpd  -1024(%rdx){1to2}, %xmm29, %xmm26
+// CHECK:  encoding: [0x62,0x61,0x95,0x10,0x56,0x52,0x80]
+          vorpd  -1024(%rdx){1to2}, %xmm29, %xmm26
+
+// CHECK: vorpd  -1032(%rdx){1to2}, %xmm29, %xmm26
+// CHECK:  encoding: [0x62,0x61,0x95,0x10,0x56,0x92,0xf8,0xfb,0xff,0xff]
+          vorpd  -1032(%rdx){1to2}, %xmm29, %xmm26
+
+// CHECK: vorpd  %ymm22, %ymm19, %ymm28
+// CHECK:  encoding: [0x62,0x21,0xe5,0x20,0x56,0xe6]
+          vorpd  %ymm22, %ymm19, %ymm28
+
+// CHECK: vorpd  %ymm22, %ymm19, %ymm28 {%k1}
+// CHECK:  encoding: [0x62,0x21,0xe5,0x21,0x56,0xe6]
+          vorpd  %ymm22, %ymm19, %ymm28 {%k1}
+
+// CHECK: vorpd  %ymm22, %ymm19, %ymm28 {%k1} {z}
+// CHECK:  encoding: [0x62,0x21,0xe5,0xa1,0x56,0xe6]
+          vorpd  %ymm22, %ymm19, %ymm28 {%k1} {z}
+
+// CHECK: vorpd  (%rcx), %ymm19, %ymm28
+// CHECK:  encoding: [0x62,0x61,0xe5,0x20,0x56,0x21]
+          vorpd  (%rcx), %ymm19, %ymm28
+
+// CHECK: vorpd  4660(%rax,%r14,8), %ymm19, %ymm28
+// CHECK:  encoding: [0x62,0x21,0xe5,0x20,0x56,0xa4,0xf0,0x34,0x12,0x00,0x00]
+          vorpd  4660(%rax,%r14,8), %ymm19, %ymm28
+
+// CHECK: vorpd  (%rcx){1to4}, %ymm19, %ymm28
+// CHECK:  encoding: [0x62,0x61,0xe5,0x30,0x56,0x21]
+          vorpd  (%rcx){1to4}, %ymm19, %ymm28
+
+// CHECK: vorpd  4064(%rdx), %ymm19, %ymm28
+// CHECK:  encoding: [0x62,0x61,0xe5,0x20,0x56,0x62,0x7f]
+          vorpd  4064(%rdx), %ymm19, %ymm28
+
+// CHECK: vorpd  4096(%rdx), %ymm19, %ymm28
+// CHECK:  encoding: [0x62,0x61,0xe5,0x20,0x56,0xa2,0x00,0x10,0x00,0x00]
+          vorpd  4096(%rdx), %ymm19, %ymm28
+
+// CHECK: vorpd  -4096(%rdx), %ymm19, %ymm28
+// CHECK:  encoding: [0x62,0x61,0xe5,0x20,0x56,0x62,0x80]
+          vorpd  -4096(%rdx), %ymm19, %ymm28
+
+// CHECK: vorpd  -4128(%rdx), %ymm19, %ymm28
+// CHECK:  encoding: [0x62,0x61,0xe5,0x20,0x56,0xa2,0xe0,0xef,0xff,0xff]
+          vorpd  -4128(%rdx), %ymm19, %ymm28
+
+// CHECK: vorpd  1016(%rdx){1to4}, %ymm19, %ymm28
+// CHECK:  encoding: [0x62,0x61,0xe5,0x30,0x56,0x62,0x7f]
+          vorpd  1016(%rdx){1to4}, %ymm19, %ymm28
+
+// CHECK: vorpd  1024(%rdx){1to4}, %ymm19, %ymm28
+// CHECK:  encoding: [0x62,0x61,0xe5,0x30,0x56,0xa2,0x00,0x04,0x00,0x00]
+          vorpd  1024(%rdx){1to4}, %ymm19, %ymm28
+
+// CHECK: vorpd  -1024(%rdx){1to4}, %ymm19, %ymm28
+// CHECK:  encoding: [0x62,0x61,0xe5,0x30,0x56,0x62,0x80]
+          vorpd  -1024(%rdx){1to4}, %ymm19, %ymm28
+
+// CHECK: vorpd  -1032(%rdx){1to4}, %ymm19, %ymm28
+// CHECK:  encoding: [0x62,0x61,0xe5,0x30,0x56,0xa2,0xf8,0xfb,0xff,0xff]
+          vorpd  -1032(%rdx){1to4}, %ymm19, %ymm28
+
+// CHECK: vorps  %xmm24, %xmm22, %xmm28
+// CHECK:  encoding: [0x62,0x01,0x4c,0x00,0x56,0xe0]
+          vorps  %xmm24, %xmm22, %xmm28
+
+// CHECK: vorps  %xmm24, %xmm22, %xmm28 {%k6}
+// CHECK:  encoding: [0x62,0x01,0x4c,0x06,0x56,0xe0]
+          vorps  %xmm24, %xmm22, %xmm28 {%k6}
+
+// CHECK: vorps  %xmm24, %xmm22, %xmm28 {%k6} {z}
+// CHECK:  encoding: [0x62,0x01,0x4c,0x86,0x56,0xe0]
+          vorps  %xmm24, %xmm22, %xmm28 {%k6} {z}
+
+// CHECK: vorps  (%rcx), %xmm22, %xmm28
+// CHECK:  encoding: [0x62,0x61,0x4c,0x00,0x56,0x21]
+          vorps  (%rcx), %xmm22, %xmm28
+
+// CHECK: vorps  4660(%rax,%r14,8), %xmm22, %xmm28
+// CHECK:  encoding: [0x62,0x21,0x4c,0x00,0x56,0xa4,0xf0,0x34,0x12,0x00,0x00]
+          vorps  4660(%rax,%r14,8), %xmm22, %xmm28
+
+// CHECK: vorps  (%rcx){1to4}, %xmm22, %xmm28
+// CHECK:  encoding: [0x62,0x61,0x4c,0x10,0x56,0x21]
+          vorps  (%rcx){1to4}, %xmm22, %xmm28
+
+// CHECK: vorps  2032(%rdx), %xmm22, %xmm28
+// CHECK:  encoding: [0x62,0x61,0x4c,0x00,0x56,0x62,0x7f]
+          vorps  2032(%rdx), %xmm22, %xmm28
+
+// CHECK: vorps  2048(%rdx), %xmm22, %xmm28
+// CHECK:  encoding: [0x62,0x61,0x4c,0x00,0x56,0xa2,0x00,0x08,0x00,0x00]
+          vorps  2048(%rdx), %xmm22, %xmm28
+
+// CHECK: vorps  -2048(%rdx), %xmm22, %xmm28
+// CHECK:  encoding: [0x62,0x61,0x4c,0x00,0x56,0x62,0x80]
+          vorps  -2048(%rdx), %xmm22, %xmm28
+
+// CHECK: vorps  -2064(%rdx), %xmm22, %xmm28
+// CHECK:  encoding: [0x62,0x61,0x4c,0x00,0x56,0xa2,0xf0,0xf7,0xff,0xff]
+          vorps  -2064(%rdx), %xmm22, %xmm28
+
+// CHECK: vorps  508(%rdx){1to4}, %xmm22, %xmm28
+// CHECK:  encoding: [0x62,0x61,0x4c,0x10,0x56,0x62,0x7f]
+          vorps  508(%rdx){1to4}, %xmm22, %xmm28
+
+// CHECK: vorps  512(%rdx){1to4}, %xmm22, %xmm28
+// CHECK:  encoding: [0x62,0x61,0x4c,0x10,0x56,0xa2,0x00,0x02,0x00,0x00]
+          vorps  512(%rdx){1to4}, %xmm22, %xmm28
+
+// CHECK: vorps  -512(%rdx){1to4}, %xmm22, %xmm28
+// CHECK:  encoding: [0x62,0x61,0x4c,0x10,0x56,0x62,0x80]
+          vorps  -512(%rdx){1to4}, %xmm22, %xmm28
+
+// CHECK: vorps  -516(%rdx){1to4}, %xmm22, %xmm28
+// CHECK:  encoding: [0x62,0x61,0x4c,0x10,0x56,0xa2,0xfc,0xfd,0xff,0xff]
+          vorps  -516(%rdx){1to4}, %xmm22, %xmm28
+
+// CHECK: vorps  %ymm25, %ymm24, %ymm20
+// CHECK:  encoding: [0x62,0x81,0x3c,0x20,0x56,0xe1]
+          vorps  %ymm25, %ymm24, %ymm20
+
+// CHECK: vorps  %ymm25, %ymm24, %ymm20 {%k1}
+// CHECK:  encoding: [0x62,0x81,0x3c,0x21,0x56,0xe1]
+          vorps  %ymm25, %ymm24, %ymm20 {%k1}
+
+// CHECK: vorps  %ymm25, %ymm24, %ymm20 {%k1} {z}
+// CHECK:  encoding: [0x62,0x81,0x3c,0xa1,0x56,0xe1]
+          vorps  %ymm25, %ymm24, %ymm20 {%k1} {z}
+
+// CHECK: vorps  (%rcx), %ymm24, %ymm20
+// CHECK:  encoding: [0x62,0xe1,0x3c,0x20,0x56,0x21]
+          vorps  (%rcx), %ymm24, %ymm20
+
+// CHECK: vorps  4660(%rax,%r14,8), %ymm24, %ymm20
+// CHECK:  encoding: [0x62,0xa1,0x3c,0x20,0x56,0xa4,0xf0,0x34,0x12,0x00,0x00]
+          vorps  4660(%rax,%r14,8), %ymm24, %ymm20
+
+// CHECK: vorps  (%rcx){1to8}, %ymm24, %ymm20
+// CHECK:  encoding: [0x62,0xe1,0x3c,0x30,0x56,0x21]
+          vorps  (%rcx){1to8}, %ymm24, %ymm20
+
+// CHECK: vorps  4064(%rdx), %ymm24, %ymm20
+// CHECK:  encoding: [0x62,0xe1,0x3c,0x20,0x56,0x62,0x7f]
+          vorps  4064(%rdx), %ymm24, %ymm20
+
+// CHECK: vorps  4096(%rdx), %ymm24, %ymm20
+// CHECK:  encoding: [0x62,0xe1,0x3c,0x20,0x56,0xa2,0x00,0x10,0x00,0x00]
+          vorps  4096(%rdx), %ymm24, %ymm20
+
+// CHECK: vorps  -4096(%rdx), %ymm24, %ymm20
+// CHECK:  encoding: [0x62,0xe1,0x3c,0x20,0x56,0x62,0x80]
+          vorps  -4096(%rdx), %ymm24, %ymm20
+
+// CHECK: vorps  -4128(%rdx), %ymm24, %ymm20
+// CHECK:  encoding: [0x62,0xe1,0x3c,0x20,0x56,0xa2,0xe0,0xef,0xff,0xff]
+          vorps  -4128(%rdx), %ymm24, %ymm20
+
+// CHECK: vorps  508(%rdx){1to8}, %ymm24, %ymm20
+// CHECK:  encoding: [0x62,0xe1,0x3c,0x30,0x56,0x62,0x7f]
+          vorps  508(%rdx){1to8}, %ymm24, %ymm20
+
+// CHECK: vorps  512(%rdx){1to8}, %ymm24, %ymm20
+// CHECK:  encoding: [0x62,0xe1,0x3c,0x30,0x56,0xa2,0x00,0x02,0x00,0x00]
+          vorps  512(%rdx){1to8}, %ymm24, %ymm20
+
+// CHECK: vorps  -512(%rdx){1to8}, %ymm24, %ymm20
+// CHECK:  encoding: [0x62,0xe1,0x3c,0x30,0x56,0x62,0x80]
+          vorps  -512(%rdx){1to8}, %ymm24, %ymm20
+
+// CHECK: vorps  -516(%rdx){1to8}, %ymm24, %ymm20
+// CHECK:  encoding: [0x62,0xe1,0x3c,0x30,0x56,0xa2,0xfc,0xfd,0xff,0xff]
+          vorps  -516(%rdx){1to8}, %ymm24, %ymm20
+
+// CHECK: vxorpd %xmm18, %xmm21, %xmm22
+// CHECK:  encoding: [0x62,0xa1,0xd5,0x00,0x57,0xf2]
+          vxorpd %xmm18, %xmm21, %xmm22
+
+// CHECK: vxorpd %xmm18, %xmm21, %xmm22 {%k3}
+// CHECK:  encoding: [0x62,0xa1,0xd5,0x03,0x57,0xf2]
+          vxorpd %xmm18, %xmm21, %xmm22 {%k3}
+
+// CHECK: vxorpd %xmm18, %xmm21, %xmm22 {%k3} {z}
+// CHECK:  encoding: [0x62,0xa1,0xd5,0x83,0x57,0xf2]
+          vxorpd %xmm18, %xmm21, %xmm22 {%k3} {z}
+
+// CHECK: vxorpd (%rcx), %xmm21, %xmm22
+// CHECK:  encoding: [0x62,0xe1,0xd5,0x00,0x57,0x31]
+          vxorpd (%rcx), %xmm21, %xmm22
+
+// CHECK: vxorpd 4660(%rax,%r14,8), %xmm21, %xmm22
+// CHECK:  encoding: [0x62,0xa1,0xd5,0x00,0x57,0xb4,0xf0,0x34,0x12,0x00,0x00]
+          vxorpd 4660(%rax,%r14,8), %xmm21, %xmm22
+
+// CHECK: vxorpd (%rcx){1to2}, %xmm21, %xmm22
+// CHECK:  encoding: [0x62,0xe1,0xd5,0x10,0x57,0x31]
+          vxorpd (%rcx){1to2}, %xmm21, %xmm22
+
+// CHECK: vxorpd 2032(%rdx), %xmm21, %xmm22
+// CHECK:  encoding: [0x62,0xe1,0xd5,0x00,0x57,0x72,0x7f]
+          vxorpd 2032(%rdx), %xmm21, %xmm22
+
+// CHECK: vxorpd 2048(%rdx), %xmm21, %xmm22
+// CHECK:  encoding: [0x62,0xe1,0xd5,0x00,0x57,0xb2,0x00,0x08,0x00,0x00]
+          vxorpd 2048(%rdx), %xmm21, %xmm22
+
+// CHECK: vxorpd -2048(%rdx), %xmm21, %xmm22
+// CHECK:  encoding: [0x62,0xe1,0xd5,0x00,0x57,0x72,0x80]
+          vxorpd -2048(%rdx), %xmm21, %xmm22
+
+// CHECK: vxorpd -2064(%rdx), %xmm21, %xmm22
+// CHECK:  encoding: [0x62,0xe1,0xd5,0x00,0x57,0xb2,0xf0,0xf7,0xff,0xff]
+          vxorpd -2064(%rdx), %xmm21, %xmm22
+
+// CHECK: vxorpd 1016(%rdx){1to2}, %xmm21, %xmm22
+// CHECK:  encoding: [0x62,0xe1,0xd5,0x10,0x57,0x72,0x7f]
+          vxorpd 1016(%rdx){1to2}, %xmm21, %xmm22
+
+// CHECK: vxorpd 1024(%rdx){1to2}, %xmm21, %xmm22
+// CHECK:  encoding: [0x62,0xe1,0xd5,0x10,0x57,0xb2,0x00,0x04,0x00,0x00]
+          vxorpd 1024(%rdx){1to2}, %xmm21, %xmm22
+
+// CHECK: vxorpd -1024(%rdx){1to2}, %xmm21, %xmm22
+// CHECK:  encoding: [0x62,0xe1,0xd5,0x10,0x57,0x72,0x80]
+          vxorpd -1024(%rdx){1to2}, %xmm21, %xmm22
+
+// CHECK: vxorpd -1032(%rdx){1to2}, %xmm21, %xmm22
+// CHECK:  encoding: [0x62,0xe1,0xd5,0x10,0x57,0xb2,0xf8,0xfb,0xff,0xff]
+          vxorpd -1032(%rdx){1to2}, %xmm21, %xmm22
+
+// CHECK: vxorpd %ymm27, %ymm21, %ymm25
+// CHECK:  encoding: [0x62,0x01,0xd5,0x20,0x57,0xcb]
+          vxorpd %ymm27, %ymm21, %ymm25
+
+// CHECK: vxorpd %ymm27, %ymm21, %ymm25 {%k7}
+// CHECK:  encoding: [0x62,0x01,0xd5,0x27,0x57,0xcb]
+          vxorpd %ymm27, %ymm21, %ymm25 {%k7}
+
+// CHECK: vxorpd %ymm27, %ymm21, %ymm25 {%k7} {z}
+// CHECK:  encoding: [0x62,0x01,0xd5,0xa7,0x57,0xcb]
+          vxorpd %ymm27, %ymm21, %ymm25 {%k7} {z}
+
+// CHECK: vxorpd (%rcx), %ymm21, %ymm25
+// CHECK:  encoding: [0x62,0x61,0xd5,0x20,0x57,0x09]
+          vxorpd (%rcx), %ymm21, %ymm25
+
+// CHECK: vxorpd 4660(%rax,%r14,8), %ymm21, %ymm25
+// CHECK:  encoding: [0x62,0x21,0xd5,0x20,0x57,0x8c,0xf0,0x34,0x12,0x00,0x00]
+          vxorpd 4660(%rax,%r14,8), %ymm21, %ymm25
+
+// CHECK: vxorpd (%rcx){1to4}, %ymm21, %ymm25
+// CHECK:  encoding: [0x62,0x61,0xd5,0x30,0x57,0x09]
+          vxorpd (%rcx){1to4}, %ymm21, %ymm25
+
+// CHECK: vxorpd 4064(%rdx), %ymm21, %ymm25
+// CHECK:  encoding: [0x62,0x61,0xd5,0x20,0x57,0x4a,0x7f]
+          vxorpd 4064(%rdx), %ymm21, %ymm25
+
+// CHECK: vxorpd 4096(%rdx), %ymm21, %ymm25
+// CHECK:  encoding: [0x62,0x61,0xd5,0x20,0x57,0x8a,0x00,0x10,0x00,0x00]
+          vxorpd 4096(%rdx), %ymm21, %ymm25
+
+// CHECK: vxorpd -4096(%rdx), %ymm21, %ymm25
+// CHECK:  encoding: [0x62,0x61,0xd5,0x20,0x57,0x4a,0x80]
+          vxorpd -4096(%rdx), %ymm21, %ymm25
+
+// CHECK: vxorpd -4128(%rdx), %ymm21, %ymm25
+// CHECK:  encoding: [0x62,0x61,0xd5,0x20,0x57,0x8a,0xe0,0xef,0xff,0xff]
+          vxorpd -4128(%rdx), %ymm21, %ymm25
+
+// CHECK: vxorpd 1016(%rdx){1to4}, %ymm21, %ymm25
+// CHECK:  encoding: [0x62,0x61,0xd5,0x30,0x57,0x4a,0x7f]
+          vxorpd 1016(%rdx){1to4}, %ymm21, %ymm25
+
+// CHECK: vxorpd 1024(%rdx){1to4}, %ymm21, %ymm25
+// CHECK:  encoding: [0x62,0x61,0xd5,0x30,0x57,0x8a,0x00,0x04,0x00,0x00]
+          vxorpd 1024(%rdx){1to4}, %ymm21, %ymm25
+
+// CHECK: vxorpd -1024(%rdx){1to4}, %ymm21, %ymm25
+// CHECK:  encoding: [0x62,0x61,0xd5,0x30,0x57,0x4a,0x80]
+          vxorpd -1024(%rdx){1to4}, %ymm21, %ymm25
+
+// CHECK: vxorpd -1032(%rdx){1to4}, %ymm21, %ymm25
+// CHECK:  encoding: [0x62,0x61,0xd5,0x30,0x57,0x8a,0xf8,0xfb,0xff,0xff]
+          vxorpd -1032(%rdx){1to4}, %ymm21, %ymm25
+
+// CHECK: vxorps %xmm21, %xmm21, %xmm17
+// CHECK:  encoding: [0x62,0xa1,0x54,0x00,0x57,0xcd]
+          vxorps %xmm21, %xmm21, %xmm17
+
+// CHECK: vxorps %xmm21, %xmm21, %xmm17 {%k5}
+// CHECK:  encoding: [0x62,0xa1,0x54,0x05,0x57,0xcd]
+          vxorps %xmm21, %xmm21, %xmm17 {%k5}
+
+// CHECK: vxorps %xmm21, %xmm21, %xmm17 {%k5} {z}
+// CHECK:  encoding: [0x62,0xa1,0x54,0x85,0x57,0xcd]
+          vxorps %xmm21, %xmm21, %xmm17 {%k5} {z}
+
+// CHECK: vxorps (%rcx), %xmm21, %xmm17
+// CHECK:  encoding: [0x62,0xe1,0x54,0x00,0x57,0x09]
+          vxorps (%rcx), %xmm21, %xmm17
+
+// CHECK: vxorps 4660(%rax,%r14,8), %xmm21, %xmm17
+// CHECK:  encoding: [0x62,0xa1,0x54,0x00,0x57,0x8c,0xf0,0x34,0x12,0x00,0x00]
+          vxorps 4660(%rax,%r14,8), %xmm21, %xmm17
+
+// CHECK: vxorps (%rcx){1to4}, %xmm21, %xmm17
+// CHECK:  encoding: [0x62,0xe1,0x54,0x10,0x57,0x09]
+          vxorps (%rcx){1to4}, %xmm21, %xmm17
+
+// CHECK: vxorps 2032(%rdx), %xmm21, %xmm17
+// CHECK:  encoding: [0x62,0xe1,0x54,0x00,0x57,0x4a,0x7f]
+          vxorps 2032(%rdx), %xmm21, %xmm17
+
+// CHECK: vxorps 2048(%rdx), %xmm21, %xmm17
+// CHECK:  encoding: [0x62,0xe1,0x54,0x00,0x57,0x8a,0x00,0x08,0x00,0x00]
+          vxorps 2048(%rdx), %xmm21, %xmm17
+
+// CHECK: vxorps -2048(%rdx), %xmm21, %xmm17
+// CHECK:  encoding: [0x62,0xe1,0x54,0x00,0x57,0x4a,0x80]
+          vxorps -2048(%rdx), %xmm21, %xmm17
+
+// CHECK: vxorps -2064(%rdx), %xmm21, %xmm17
+// CHECK:  encoding: [0x62,0xe1,0x54,0x00,0x57,0x8a,0xf0,0xf7,0xff,0xff]
+          vxorps -2064(%rdx), %xmm21, %xmm17
+
+// CHECK: vxorps 508(%rdx){1to4}, %xmm21, %xmm17
+// CHECK:  encoding: [0x62,0xe1,0x54,0x10,0x57,0x4a,0x7f]
+          vxorps 508(%rdx){1to4}, %xmm21, %xmm17
+
+// CHECK: vxorps 512(%rdx){1to4}, %xmm21, %xmm17
+// CHECK:  encoding: [0x62,0xe1,0x54,0x10,0x57,0x8a,0x00,0x02,0x00,0x00]
+          vxorps 512(%rdx){1to4}, %xmm21, %xmm17
+
+// CHECK: vxorps -512(%rdx){1to4}, %xmm21, %xmm17
+// CHECK:  encoding: [0x62,0xe1,0x54,0x10,0x57,0x4a,0x80]
+          vxorps -512(%rdx){1to4}, %xmm21, %xmm17
+
+// CHECK: vxorps -516(%rdx){1to4}, %xmm21, %xmm17
+// CHECK:  encoding: [0x62,0xe1,0x54,0x10,0x57,0x8a,0xfc,0xfd,0xff,0xff]
+          vxorps -516(%rdx){1to4}, %xmm21, %xmm17
+
+// CHECK: vxorps %ymm22, %ymm25, %ymm28
+// CHECK:  encoding: [0x62,0x21,0x34,0x20,0x57,0xe6]
+          vxorps %ymm22, %ymm25, %ymm28
+
+// CHECK: vxorps %ymm22, %ymm25, %ymm28 {%k3}
+// CHECK:  encoding: [0x62,0x21,0x34,0x23,0x57,0xe6]
+          vxorps %ymm22, %ymm25, %ymm28 {%k3}
+
+// CHECK: vxorps %ymm22, %ymm25, %ymm28 {%k3} {z}
+// CHECK:  encoding: [0x62,0x21,0x34,0xa3,0x57,0xe6]
+          vxorps %ymm22, %ymm25, %ymm28 {%k3} {z}
+
+// CHECK: vxorps (%rcx), %ymm25, %ymm28
+// CHECK:  encoding: [0x62,0x61,0x34,0x20,0x57,0x21]
+          vxorps (%rcx), %ymm25, %ymm28
+
+// CHECK: vxorps 4660(%rax,%r14,8), %ymm25, %ymm28
+// CHECK:  encoding: [0x62,0x21,0x34,0x20,0x57,0xa4,0xf0,0x34,0x12,0x00,0x00]
+          vxorps 4660(%rax,%r14,8), %ymm25, %ymm28
+
+// CHECK: vxorps (%rcx){1to8}, %ymm25, %ymm28
+// CHECK:  encoding: [0x62,0x61,0x34,0x30,0x57,0x21]
+          vxorps (%rcx){1to8}, %ymm25, %ymm28
+
+// CHECK: vxorps 4064(%rdx), %ymm25, %ymm28
+// CHECK:  encoding: [0x62,0x61,0x34,0x20,0x57,0x62,0x7f]
+          vxorps 4064(%rdx), %ymm25, %ymm28
+
+// CHECK: vxorps 4096(%rdx), %ymm25, %ymm28
+// CHECK:  encoding: [0x62,0x61,0x34,0x20,0x57,0xa2,0x00,0x10,0x00,0x00]
+          vxorps 4096(%rdx), %ymm25, %ymm28
+
+// CHECK: vxorps -4096(%rdx), %ymm25, %ymm28
+// CHECK:  encoding: [0x62,0x61,0x34,0x20,0x57,0x62,0x80]
+          vxorps -4096(%rdx), %ymm25, %ymm28
+
+// CHECK: vxorps -4128(%rdx), %ymm25, %ymm28
+// CHECK:  encoding: [0x62,0x61,0x34,0x20,0x57,0xa2,0xe0,0xef,0xff,0xff]
+          vxorps -4128(%rdx), %ymm25, %ymm28
+
+// CHECK: vxorps 508(%rdx){1to8}, %ymm25, %ymm28
+// CHECK:  encoding: [0x62,0x61,0x34,0x30,0x57,0x62,0x7f]
+          vxorps 508(%rdx){1to8}, %ymm25, %ymm28
+
+// CHECK: vxorps 512(%rdx){1to8}, %ymm25, %ymm28
+// CHECK:  encoding: [0x62,0x61,0x34,0x30,0x57,0xa2,0x00,0x02,0x00,0x00]
+          vxorps 512(%rdx){1to8}, %ymm25, %ymm28
+
+// CHECK: vxorps -512(%rdx){1to8}, %ymm25, %ymm28
+// CHECK:  encoding: [0x62,0x61,0x34,0x30,0x57,0x62,0x80]
+          vxorps -512(%rdx){1to8}, %ymm25, %ymm28
+
+// CHECK: vxorps -516(%rdx){1to8}, %ymm25, %ymm28
+// CHECK:  encoding: [0x62,0x61,0x34,0x30,0x57,0xa2,0xfc,0xfd,0xff,0xff]
+          vxorps -516(%rdx){1to8}, %ymm25, %ymm28
+
+// CHECK: vbroadcastf64x2 (%rcx), %ymm27
+// CHECK:  encoding: [0x62,0x62,0xfd,0x28,0x1a,0x19]
+          vbroadcastf64x2 (%rcx), %ymm27
+
+// CHECK: vbroadcastf64x2 (%rcx), %ymm27 {%k5}
+// CHECK:  encoding: [0x62,0x62,0xfd,0x2d,0x1a,0x19]
+          vbroadcastf64x2 (%rcx), %ymm27 {%k5}
+
+// CHECK: vbroadcastf64x2 (%rcx), %ymm27 {%k5} {z}
+// CHECK:  encoding: [0x62,0x62,0xfd,0xad,0x1a,0x19]
+          vbroadcastf64x2 (%rcx), %ymm27 {%k5} {z}
+
+// CHECK: vbroadcastf64x2 291(%rax,%r14,8), %ymm27
+// CHECK:  encoding: [0x62,0x22,0xfd,0x28,0x1a,0x9c,0xf0,0x23,0x01,0x00,0x00]
+          vbroadcastf64x2 291(%rax,%r14,8), %ymm27
+
+// CHECK: vbroadcastf64x2 2032(%rdx), %ymm27
+// CHECK:  encoding: [0x62,0x62,0xfd,0x28,0x1a,0x5a,0x7f]
+          vbroadcastf64x2 2032(%rdx), %ymm27
+
+// CHECK: vbroadcastf64x2 2048(%rdx), %ymm27
+// CHECK:  encoding: [0x62,0x62,0xfd,0x28,0x1a,0x9a,0x00,0x08,0x00,0x00]
+          vbroadcastf64x2 2048(%rdx), %ymm27
+
+// CHECK: vbroadcastf64x2 -2048(%rdx), %ymm27
+// CHECK:  encoding: [0x62,0x62,0xfd,0x28,0x1a,0x5a,0x80]
+          vbroadcastf64x2 -2048(%rdx), %ymm27
+
+// CHECK: vbroadcastf64x2 -2064(%rdx), %ymm27
+// CHECK:  encoding: [0x62,0x62,0xfd,0x28,0x1a,0x9a,0xf0,0xf7,0xff,0xff]
+          vbroadcastf64x2 -2064(%rdx), %ymm27
+
+// CHECK: vbroadcasti64x2 (%rcx), %ymm18
+// CHECK:  encoding: [0x62,0xe2,0xfd,0x28,0x5a,0x11]
+          vbroadcasti64x2 (%rcx), %ymm18
+
+// CHECK: vbroadcasti64x2 (%rcx), %ymm18 {%k1}
+// CHECK:  encoding: [0x62,0xe2,0xfd,0x29,0x5a,0x11]
+          vbroadcasti64x2 (%rcx), %ymm18 {%k1}
+
+// CHECK: vbroadcasti64x2 (%rcx), %ymm18 {%k1} {z}
+// CHECK:  encoding: [0x62,0xe2,0xfd,0xa9,0x5a,0x11]
+          vbroadcasti64x2 (%rcx), %ymm18 {%k1} {z}
+
+// CHECK: vbroadcasti64x2 291(%rax,%r14,8), %ymm18
+// CHECK:  encoding: [0x62,0xa2,0xfd,0x28,0x5a,0x94,0xf0,0x23,0x01,0x00,0x00]
+          vbroadcasti64x2 291(%rax,%r14,8), %ymm18
+
+// CHECK: vbroadcasti64x2 2032(%rdx), %ymm18
+// CHECK:  encoding: [0x62,0xe2,0xfd,0x28,0x5a,0x52,0x7f]
+          vbroadcasti64x2 2032(%rdx), %ymm18
+
+// CHECK: vbroadcasti64x2 2048(%rdx), %ymm18
+// CHECK:  encoding: [0x62,0xe2,0xfd,0x28,0x5a,0x92,0x00,0x08,0x00,0x00]
+          vbroadcasti64x2 2048(%rdx), %ymm18
+
+// CHECK: vbroadcasti64x2 -2048(%rdx), %ymm18
+// CHECK:  encoding: [0x62,0xe2,0xfd,0x28,0x5a,0x52,0x80]
+          vbroadcasti64x2 -2048(%rdx), %ymm18
+
+// CHECK: vbroadcasti64x2 -2064(%rdx), %ymm18
+// CHECK:  encoding: [0x62,0xe2,0xfd,0x28,0x5a,0x92,0xf0,0xf7,0xff,0xff]
+          vbroadcasti64x2 -2064(%rdx), %ymm18
+
+// CHECK: vrangepd $171, %xmm26, %xmm27, %xmm25
+// CHECK:  encoding: [0x62,0x03,0xa5,0x00,0x50,0xca,0xab]
+          vrangepd $0xab, %xmm26, %xmm27, %xmm25
+
+// CHECK: vrangepd $171, %xmm26, %xmm27, %xmm25 {%k6}
+// CHECK:  encoding: [0x62,0x03,0xa5,0x06,0x50,0xca,0xab]
+          vrangepd $0xab, %xmm26, %xmm27, %xmm25 {%k6}
+
+// CHECK: vrangepd $171, %xmm26, %xmm27, %xmm25 {%k6} {z}
+// CHECK:  encoding: [0x62,0x03,0xa5,0x86,0x50,0xca,0xab]
+          vrangepd $0xab, %xmm26, %xmm27, %xmm25 {%k6} {z}
+
+// CHECK: vrangepd $123, %xmm26, %xmm27, %xmm25
+// CHECK:  encoding: [0x62,0x03,0xa5,0x00,0x50,0xca,0x7b]
+          vrangepd $0x7b, %xmm26, %xmm27, %xmm25
+
+// CHECK: vrangepd $123, (%rcx), %xmm27, %xmm25
+// CHECK:  encoding: [0x62,0x63,0xa5,0x00,0x50,0x09,0x7b]
+          vrangepd $0x7b,(%rcx), %xmm27, %xmm25
+
+// CHECK: vrangepd $123, 291(%rax,%r14,8), %xmm27, %xmm25
+// CHECK:  encoding: [0x62,0x23,0xa5,0x00,0x50,0x8c,0xf0,0x23,0x01,0x00,0x00,0x7b]
+          vrangepd $0x7b,291(%rax,%r14,8), %xmm27, %xmm25
+
+// CHECK: vrangepd $123, (%rcx){1to2}, %xmm27, %xmm25
+// CHECK:  encoding: [0x62,0x63,0xa5,0x10,0x50,0x09,0x7b]
+          vrangepd $0x7b,(%rcx){1to2}, %xmm27, %xmm25
+
+// CHECK: vrangepd $123, 2032(%rdx), %xmm27, %xmm25
+// CHECK:  encoding: [0x62,0x63,0xa5,0x00,0x50,0x4a,0x7f,0x7b]
+          vrangepd $0x7b,2032(%rdx), %xmm27, %xmm25
+
+// CHECK: vrangepd $123, 2048(%rdx), %xmm27, %xmm25
+// CHECK:  encoding: [0x62,0x63,0xa5,0x00,0x50,0x8a,0x00,0x08,0x00,0x00,0x7b]
+          vrangepd $0x7b,2048(%rdx), %xmm27, %xmm25
+
+// CHECK: vrangepd $123, -2048(%rdx), %xmm27, %xmm25
+// CHECK:  encoding: [0x62,0x63,0xa5,0x00,0x50,0x4a,0x80,0x7b]
+          vrangepd $0x7b,-2048(%rdx), %xmm27, %xmm25
+
+// CHECK: vrangepd $123, -2064(%rdx), %xmm27, %xmm25
+// CHECK:  encoding: [0x62,0x63,0xa5,0x00,0x50,0x8a,0xf0,0xf7,0xff,0xff,0x7b]
+          vrangepd $0x7b,-2064(%rdx), %xmm27, %xmm25
+
+// CHECK: vrangepd $123, 1016(%rdx){1to2}, %xmm27, %xmm25
+// CHECK:  encoding: [0x62,0x63,0xa5,0x10,0x50,0x4a,0x7f,0x7b]
+          vrangepd $0x7b,1016(%rdx){1to2}, %xmm27, %xmm25
+
+// CHECK: vrangepd $123, 1024(%rdx){1to2}, %xmm27, %xmm25
+// CHECK:  encoding: [0x62,0x63,0xa5,0x10,0x50,0x8a,0x00,0x04,0x00,0x00,0x7b]
+          vrangepd $0x7b,1024(%rdx){1to2}, %xmm27, %xmm25
+
+// CHECK: vrangepd $123, -1024(%rdx){1to2}, %xmm27, %xmm25
+// CHECK:  encoding: [0x62,0x63,0xa5,0x10,0x50,0x4a,0x80,0x7b]
+          vrangepd $0x7b,-1024(%rdx){1to2}, %xmm27, %xmm25
+
+// CHECK: vrangepd $123, -1032(%rdx){1to2}, %xmm27, %xmm25
+// CHECK:  encoding: [0x62,0x63,0xa5,0x10,0x50,0x8a,0xf8,0xfb,0xff,0xff,0x7b]
+          vrangepd $0x7b,-1032(%rdx){1to2}, %xmm27, %xmm25
+
+// CHECK: vrangepd $171, %ymm28, %ymm27, %ymm29
+// CHECK:  encoding: [0x62,0x03,0xa5,0x20,0x50,0xec,0xab]
+          vrangepd $0xab, %ymm28, %ymm27, %ymm29
+
+// CHECK: vrangepd $171, %ymm28, %ymm27, %ymm29 {%k7}
+// CHECK:  encoding: [0x62,0x03,0xa5,0x27,0x50,0xec,0xab]
+          vrangepd $0xab, %ymm28, %ymm27, %ymm29 {%k7}
+
+// CHECK: vrangepd $171, %ymm28, %ymm27, %ymm29 {%k7} {z}
+// CHECK:  encoding: [0x62,0x03,0xa5,0xa7,0x50,0xec,0xab]
+          vrangepd $0xab, %ymm28, %ymm27, %ymm29 {%k7} {z}
+
+// CHECK: vrangepd $123, %ymm28, %ymm27, %ymm29
+// CHECK:  encoding: [0x62,0x03,0xa5,0x20,0x50,0xec,0x7b]
+          vrangepd $0x7b, %ymm28, %ymm27, %ymm29
+
+// CHECK: vrangepd $123, (%rcx), %ymm27, %ymm29
+// CHECK:  encoding: [0x62,0x63,0xa5,0x20,0x50,0x29,0x7b]
+          vrangepd $0x7b,(%rcx), %ymm27, %ymm29
+
+// CHECK: vrangepd $123, 291(%rax,%r14,8), %ymm27, %ymm29
+// CHECK:  encoding: [0x62,0x23,0xa5,0x20,0x50,0xac,0xf0,0x23,0x01,0x00,0x00,0x7b]
+          vrangepd $0x7b,291(%rax,%r14,8), %ymm27, %ymm29
+
+// CHECK: vrangepd $123, (%rcx){1to4}, %ymm27, %ymm29
+// CHECK:  encoding: [0x62,0x63,0xa5,0x30,0x50,0x29,0x7b]
+          vrangepd $0x7b,(%rcx){1to4}, %ymm27, %ymm29
+
+// CHECK: vrangepd $123, 4064(%rdx), %ymm27, %ymm29
+// CHECK:  encoding: [0x62,0x63,0xa5,0x20,0x50,0x6a,0x7f,0x7b]
+          vrangepd $0x7b,4064(%rdx), %ymm27, %ymm29
+
+// CHECK: vrangepd $123, 4096(%rdx), %ymm27, %ymm29
+// CHECK:  encoding: [0x62,0x63,0xa5,0x20,0x50,0xaa,0x00,0x10,0x00,0x00,0x7b]
+          vrangepd $0x7b,4096(%rdx), %ymm27, %ymm29
+
+// CHECK: vrangepd $123, -4096(%rdx), %ymm27, %ymm29
+// CHECK:  encoding: [0x62,0x63,0xa5,0x20,0x50,0x6a,0x80,0x7b]
+          vrangepd $0x7b,-4096(%rdx), %ymm27, %ymm29
+
+// CHECK: vrangepd $123, -4128(%rdx), %ymm27, %ymm29
+// CHECK:  encoding: [0x62,0x63,0xa5,0x20,0x50,0xaa,0xe0,0xef,0xff,0xff,0x7b]
+          vrangepd $0x7b,-4128(%rdx), %ymm27, %ymm29
+
+// CHECK: vrangepd $123, 1016(%rdx){1to4}, %ymm27, %ymm29
+// CHECK:  encoding: [0x62,0x63,0xa5,0x30,0x50,0x6a,0x7f,0x7b]
+          vrangepd $0x7b,1016(%rdx){1to4}, %ymm27, %ymm29
+
+// CHECK: vrangepd $123, 1024(%rdx){1to4}, %ymm27, %ymm29
+// CHECK:  encoding: [0x62,0x63,0xa5,0x30,0x50,0xaa,0x00,0x04,0x00,0x00,0x7b]
+          vrangepd $0x7b,1024(%rdx){1to4}, %ymm27, %ymm29
+
+// CHECK: vrangepd $123, -1024(%rdx){1to4}, %ymm27, %ymm29
+// CHECK:  encoding: [0x62,0x63,0xa5,0x30,0x50,0x6a,0x80,0x7b]
+          vrangepd $0x7b,-1024(%rdx){1to4}, %ymm27, %ymm29
+
+// CHECK: vrangepd $123, -1032(%rdx){1to4}, %ymm27, %ymm29
+// CHECK:  encoding: [0x62,0x63,0xa5,0x30,0x50,0xaa,0xf8,0xfb,0xff,0xff,0x7b]
+          vrangepd $0x7b,-1032(%rdx){1to4}, %ymm27, %ymm29
+
+// CHECK: vrangeps $171, %xmm24, %xmm23, %xmm27
+// CHECK:  encoding: [0x62,0x03,0x45,0x00,0x50,0xd8,0xab]
+          vrangeps $0xab, %xmm24, %xmm23, %xmm27
+
+// CHECK: vrangeps $171, %xmm24, %xmm23, %xmm27 {%k6}
+// CHECK:  encoding: [0x62,0x03,0x45,0x06,0x50,0xd8,0xab]
+          vrangeps $0xab, %xmm24, %xmm23, %xmm27 {%k6}
+
+// CHECK: vrangeps $171, %xmm24, %xmm23, %xmm27 {%k6} {z}
+// CHECK:  encoding: [0x62,0x03,0x45,0x86,0x50,0xd8,0xab]
+          vrangeps $0xab, %xmm24, %xmm23, %xmm27 {%k6} {z}
+
+// CHECK: vrangeps $123, %xmm24, %xmm23, %xmm27
+// CHECK:  encoding: [0x62,0x03,0x45,0x00,0x50,0xd8,0x7b]
+          vrangeps $0x7b, %xmm24, %xmm23, %xmm27
+
+// CHECK: vrangeps $123, (%rcx), %xmm23, %xmm27
+// CHECK:  encoding: [0x62,0x63,0x45,0x00,0x50,0x19,0x7b]
+          vrangeps $0x7b,(%rcx), %xmm23, %xmm27
+
+// CHECK: vrangeps $123, 291(%rax,%r14,8), %xmm23, %xmm27
+// CHECK:  encoding: [0x62,0x23,0x45,0x00,0x50,0x9c,0xf0,0x23,0x01,0x00,0x00,0x7b]
+          vrangeps $0x7b,291(%rax,%r14,8), %xmm23, %xmm27
+
+// CHECK: vrangeps $123, (%rcx){1to4}, %xmm23, %xmm27
+// CHECK:  encoding: [0x62,0x63,0x45,0x10,0x50,0x19,0x7b]
+          vrangeps $0x7b,(%rcx){1to4}, %xmm23, %xmm27
+
+// CHECK: vrangeps $123, 2032(%rdx), %xmm23, %xmm27
+// CHECK:  encoding: [0x62,0x63,0x45,0x00,0x50,0x5a,0x7f,0x7b]
+          vrangeps $0x7b,2032(%rdx), %xmm23, %xmm27
+
+// CHECK: vrangeps $123, 2048(%rdx), %xmm23, %xmm27
+// CHECK:  encoding: [0x62,0x63,0x45,0x00,0x50,0x9a,0x00,0x08,0x00,0x00,0x7b]
+          vrangeps $0x7b,2048(%rdx), %xmm23, %xmm27
+
+// CHECK: vrangeps $123, -2048(%rdx), %xmm23, %xmm27
+// CHECK:  encoding: [0x62,0x63,0x45,0x00,0x50,0x5a,0x80,0x7b]
+          vrangeps $0x7b,-2048(%rdx), %xmm23, %xmm27
+
+// CHECK: vrangeps $123, -2064(%rdx), %xmm23, %xmm27
+// CHECK:  encoding: [0x62,0x63,0x45,0x00,0x50,0x9a,0xf0,0xf7,0xff,0xff,0x7b]
+          vrangeps $0x7b,-2064(%rdx), %xmm23, %xmm27
+
+// CHECK: vrangeps $123, 508(%rdx){1to4}, %xmm23, %xmm27
+// CHECK:  encoding: [0x62,0x63,0x45,0x10,0x50,0x5a,0x7f,0x7b]
+          vrangeps $0x7b,508(%rdx){1to4}, %xmm23, %xmm27
+
+// CHECK: vrangeps $123, 512(%rdx){1to4}, %xmm23, %xmm27
+// CHECK:  encoding: [0x62,0x63,0x45,0x10,0x50,0x9a,0x00,0x02,0x00,0x00,0x7b]
+          vrangeps $0x7b,512(%rdx){1to4}, %xmm23, %xmm27
+
+// CHECK: vrangeps $123, -512(%rdx){1to4}, %xmm23, %xmm27
+// CHECK:  encoding: [0x62,0x63,0x45,0x10,0x50,0x5a,0x80,0x7b]
+          vrangeps $0x7b,-512(%rdx){1to4}, %xmm23, %xmm27
+
+// CHECK: vrangeps $123, -516(%rdx){1to4}, %xmm23, %xmm27
+// CHECK:  encoding: [0x62,0x63,0x45,0x10,0x50,0x9a,0xfc,0xfd,0xff,0xff,0x7b]
+          vrangeps $0x7b,-516(%rdx){1to4}, %xmm23, %xmm27
+
+// CHECK: vrangeps $171, %ymm21, %ymm23, %ymm24
+// CHECK:  encoding: [0x62,0x23,0x45,0x20,0x50,0xc5,0xab]
+          vrangeps $0xab, %ymm21, %ymm23, %ymm24
+
+// CHECK: vrangeps $171, %ymm21, %ymm23, %ymm24 {%k7}
+// CHECK:  encoding: [0x62,0x23,0x45,0x27,0x50,0xc5,0xab]
+          vrangeps $0xab, %ymm21, %ymm23, %ymm24 {%k7}
+
+// CHECK: vrangeps $171, %ymm21, %ymm23, %ymm24 {%k7} {z}
+// CHECK:  encoding: [0x62,0x23,0x45,0xa7,0x50,0xc5,0xab]
+          vrangeps $0xab, %ymm21, %ymm23, %ymm24 {%k7} {z}
+
+// CHECK: vrangeps $123, %ymm21, %ymm23, %ymm24
+// CHECK:  encoding: [0x62,0x23,0x45,0x20,0x50,0xc5,0x7b]
+          vrangeps $0x7b, %ymm21, %ymm23, %ymm24
+
+// CHECK: vrangeps $123, (%rcx), %ymm23, %ymm24
+// CHECK:  encoding: [0x62,0x63,0x45,0x20,0x50,0x01,0x7b]
+          vrangeps $0x7b,(%rcx), %ymm23, %ymm24
+
+// CHECK: vrangeps $123, 291(%rax,%r14,8), %ymm23, %ymm24
+// CHECK:  encoding: [0x62,0x23,0x45,0x20,0x50,0x84,0xf0,0x23,0x01,0x00,0x00,0x7b]
+          vrangeps $0x7b,291(%rax,%r14,8), %ymm23, %ymm24
+
+// CHECK: vrangeps $123, (%rcx){1to8}, %ymm23, %ymm24
+// CHECK:  encoding: [0x62,0x63,0x45,0x30,0x50,0x01,0x7b]
+          vrangeps $0x7b,(%rcx){1to8}, %ymm23, %ymm24
+
+// CHECK: vrangeps $123, 4064(%rdx), %ymm23, %ymm24
+// CHECK:  encoding: [0x62,0x63,0x45,0x20,0x50,0x42,0x7f,0x7b]
+          vrangeps $0x7b,4064(%rdx), %ymm23, %ymm24
+
+// CHECK: vrangeps $123, 4096(%rdx), %ymm23, %ymm24
+// CHECK:  encoding: [0x62,0x63,0x45,0x20,0x50,0x82,0x00,0x10,0x00,0x00,0x7b]
+          vrangeps $0x7b,4096(%rdx), %ymm23, %ymm24
+
+// CHECK: vrangeps $123, -4096(%rdx), %ymm23, %ymm24
+// CHECK:  encoding: [0x62,0x63,0x45,0x20,0x50,0x42,0x80,0x7b]
+          vrangeps $0x7b,-4096(%rdx), %ymm23, %ymm24
+
+// CHECK: vrangeps $123, -4128(%rdx), %ymm23, %ymm24
+// CHECK:  encoding: [0x62,0x63,0x45,0x20,0x50,0x82,0xe0,0xef,0xff,0xff,0x7b]
+          vrangeps $0x7b,-4128(%rdx), %ymm23, %ymm24
+
+// CHECK: vrangeps $123, 508(%rdx){1to8}, %ymm23, %ymm24
+// CHECK:  encoding: [0x62,0x63,0x45,0x30,0x50,0x42,0x7f,0x7b]
+          vrangeps $0x7b,508(%rdx){1to8}, %ymm23, %ymm24
+
+// CHECK: vrangeps $123, 512(%rdx){1to8}, %ymm23, %ymm24
+// CHECK:  encoding: [0x62,0x63,0x45,0x30,0x50,0x82,0x00,0x02,0x00,0x00,0x7b]
+          vrangeps $0x7b,512(%rdx){1to8}, %ymm23, %ymm24
+
+// CHECK: vrangeps $123, -512(%rdx){1to8}, %ymm23, %ymm24
+// CHECK:  encoding: [0x62,0x63,0x45,0x30,0x50,0x42,0x80,0x7b]
+          vrangeps $0x7b,-512(%rdx){1to8}, %ymm23, %ymm24
+
+// CHECK: vrangeps $123, -516(%rdx){1to8}, %ymm23, %ymm24
+// CHECK:  encoding: [0x62,0x63,0x45,0x30,0x50,0x82,0xfc,0xfd,0xff,0xff,0x7b]
+          vrangeps $0x7b,-516(%rdx){1to8}, %ymm23, %ymm24
+
+// CHECK: vreducepd $171, %xmm17, %xmm18
+// CHECK:  encoding: [0x62,0xa3,0xfd,0x08,0x56,0xd1,0xab]
+          vreducepd $0xab, %xmm17, %xmm18
+
+// CHECK: vreducepd $171, %xmm17, %xmm18 {%k3}
+// CHECK:  encoding: [0x62,0xa3,0xfd,0x0b,0x56,0xd1,0xab]
+          vreducepd $0xab, %xmm17, %xmm18 {%k3}
+
+// CHECK: vreducepd $171, %xmm17, %xmm18 {%k3} {z}
+// CHECK:  encoding: [0x62,0xa3,0xfd,0x8b,0x56,0xd1,0xab]
+          vreducepd $0xab, %xmm17, %xmm18 {%k3} {z}
+
+// CHECK: vreducepd $123, %xmm17, %xmm18
+// CHECK:  encoding: [0x62,0xa3,0xfd,0x08,0x56,0xd1,0x7b]
+          vreducepd $0x7b, %xmm17, %xmm18
+
+// CHECK: vreducepd $123, (%rcx), %xmm18
+// CHECK:  encoding: [0x62,0xe3,0xfd,0x08,0x56,0x11,0x7b]
+          vreducepd $0x7b,(%rcx), %xmm18
+
+// CHECK: vreducepd $123, 291(%rax,%r14,8), %xmm18
+// CHECK:  encoding: [0x62,0xa3,0xfd,0x08,0x56,0x94,0xf0,0x23,0x01,0x00,0x00,0x7b]
+          vreducepd $0x7b,291(%rax,%r14,8), %xmm18
+
+// CHECK: vreducepd $171, %xmm28, %xmm25
+// CHECK:  encoding: [0x62,0x03,0xfd,0x08,0x56,0xcc,0xab]
+          vreducepd $0xab, %xmm28, %xmm25
+
+// CHECK: vreducepd $171, %xmm28, %xmm25 {%k4}
+// CHECK:  encoding: [0x62,0x03,0xfd,0x0c,0x56,0xcc,0xab]
+          vreducepd $0xab, %xmm28, %xmm25 {%k4}
+
+// CHECK: vreducepd $171, %xmm28, %xmm25 {%k4} {z}
+// CHECK:  encoding: [0x62,0x03,0xfd,0x8c,0x56,0xcc,0xab]
+          vreducepd $0xab, %xmm28, %xmm25 {%k4} {z}
+
+// CHECK: vreducepd $123, %xmm28, %xmm25
+// CHECK:  encoding: [0x62,0x03,0xfd,0x08,0x56,0xcc,0x7b]
+          vreducepd $0x7b, %xmm28, %xmm25
+
+// CHECK: vreducepd $123, (%rcx), %xmm25
+// CHECK:  encoding: [0x62,0x63,0xfd,0x08,0x56,0x09,0x7b]
+          vreducepd $0x7b,(%rcx), %xmm25
+
+// CHECK: vreducepd $123, 4660(%rax,%r14,8), %xmm25
+// CHECK:  encoding: [0x62,0x23,0xfd,0x08,0x56,0x8c,0xf0,0x34,0x12,0x00,0x00,0x7b]
+          vreducepd $0x7b,4660(%rax,%r14,8), %xmm25
+
+// CHECK: vreducepd $123, (%rcx){1to2}, %xmm25
+// CHECK:  encoding: [0x62,0x63,0xfd,0x18,0x56,0x09,0x7b]
+          vreducepd $0x7b,(%rcx){1to2}, %xmm25
+
+// CHECK: vreducepd $123, 2032(%rdx), %xmm25
+// CHECK:  encoding: [0x62,0x63,0xfd,0x08,0x56,0x4a,0x7f,0x7b]
+          vreducepd $0x7b,2032(%rdx), %xmm25
+
+// CHECK: vreducepd $123, 2048(%rdx), %xmm25
+// CHECK:  encoding: [0x62,0x63,0xfd,0x08,0x56,0x8a,0x00,0x08,0x00,0x00,0x7b]
+          vreducepd $0x7b,2048(%rdx), %xmm25
+
+// CHECK: vreducepd $123, -2048(%rdx), %xmm25
+// CHECK:  encoding: [0x62,0x63,0xfd,0x08,0x56,0x4a,0x80,0x7b]
+          vreducepd $0x7b,-2048(%rdx), %xmm25
+
+// CHECK: vreducepd $123, -2064(%rdx), %xmm25
+// CHECK:  encoding: [0x62,0x63,0xfd,0x08,0x56,0x8a,0xf0,0xf7,0xff,0xff,0x7b]
+          vreducepd $0x7b,-2064(%rdx), %xmm25
+
+// CHECK: vreducepd $123, 1016(%rdx){1to2}, %xmm25
+// CHECK:  encoding: [0x62,0x63,0xfd,0x18,0x56,0x4a,0x7f,0x7b]
+          vreducepd $0x7b,1016(%rdx){1to2}, %xmm25
+
+// CHECK: vreducepd $123, 1024(%rdx){1to2}, %xmm25
+// CHECK:  encoding: [0x62,0x63,0xfd,0x18,0x56,0x8a,0x00,0x04,0x00,0x00,0x7b]
+          vreducepd $0x7b,1024(%rdx){1to2}, %xmm25
+
+// CHECK: vreducepd $123, -1024(%rdx){1to2}, %xmm25
+// CHECK:  encoding: [0x62,0x63,0xfd,0x18,0x56,0x4a,0x80,0x7b]
+          vreducepd $0x7b,-1024(%rdx){1to2}, %xmm25
+
+// CHECK: vreducepd $123, -1032(%rdx){1to2}, %xmm25
+// CHECK:  encoding: [0x62,0x63,0xfd,0x18,0x56,0x8a,0xf8,0xfb,0xff,0xff,0x7b]
+          vreducepd $0x7b,-1032(%rdx){1to2}, %xmm25
+
+// CHECK: vreducepd $171, %ymm17, %ymm28
+// CHECK:  encoding: [0x62,0x23,0xfd,0x28,0x56,0xe1,0xab]
+          vreducepd $0xab, %ymm17, %ymm28
+
+// CHECK: vreducepd $171, %ymm17, %ymm28 {%k4}
+// CHECK:  encoding: [0x62,0x23,0xfd,0x2c,0x56,0xe1,0xab]
+          vreducepd $0xab, %ymm17, %ymm28 {%k4}
+
+// CHECK: vreducepd $171, %ymm17, %ymm28 {%k4} {z}
+// CHECK:  encoding: [0x62,0x23,0xfd,0xac,0x56,0xe1,0xab]
+          vreducepd $0xab, %ymm17, %ymm28 {%k4} {z}
+
+// CHECK: vreducepd $123, %ymm17, %ymm28
+// CHECK:  encoding: [0x62,0x23,0xfd,0x28,0x56,0xe1,0x7b]
+          vreducepd $0x7b, %ymm17, %ymm28
+
+// CHECK: vreducepd $123, (%rcx), %ymm28
+// CHECK:  encoding: [0x62,0x63,0xfd,0x28,0x56,0x21,0x7b]
+          vreducepd $0x7b,(%rcx), %ymm28
+
+// CHECK: vreducepd $123, 4660(%rax,%r14,8), %ymm28
+// CHECK:  encoding: [0x62,0x23,0xfd,0x28,0x56,0xa4,0xf0,0x34,0x12,0x00,0x00,0x7b]
+          vreducepd $0x7b,4660(%rax,%r14,8), %ymm28
+
+// CHECK: vreducepd $123, (%rcx){1to4}, %ymm28
+// CHECK:  encoding: [0x62,0x63,0xfd,0x38,0x56,0x21,0x7b]
+          vreducepd $0x7b,(%rcx){1to4}, %ymm28
+
+// CHECK: vreducepd $123, 4064(%rdx), %ymm28
+// CHECK:  encoding: [0x62,0x63,0xfd,0x28,0x56,0x62,0x7f,0x7b]
+          vreducepd $0x7b,4064(%rdx), %ymm28
+
+// CHECK: vreducepd $123, 4096(%rdx), %ymm28
+// CHECK:  encoding: [0x62,0x63,0xfd,0x28,0x56,0xa2,0x00,0x10,0x00,0x00,0x7b]
+          vreducepd $0x7b,4096(%rdx), %ymm28
+
+// CHECK: vreducepd $123, -4096(%rdx), %ymm28
+// CHECK:  encoding: [0x62,0x63,0xfd,0x28,0x56,0x62,0x80,0x7b]
+          vreducepd $0x7b,-4096(%rdx), %ymm28
+
+// CHECK: vreducepd $123, -4128(%rdx), %ymm28
+// CHECK:  encoding: [0x62,0x63,0xfd,0x28,0x56,0xa2,0xe0,0xef,0xff,0xff,0x7b]
+          vreducepd $0x7b,-4128(%rdx), %ymm28
+
+// CHECK: vreducepd $123, 1016(%rdx){1to4}, %ymm28
+// CHECK:  encoding: [0x62,0x63,0xfd,0x38,0x56,0x62,0x7f,0x7b]
+          vreducepd $0x7b,1016(%rdx){1to4}, %ymm28
+
+// CHECK: vreducepd $123, 1024(%rdx){1to4}, %ymm28
+// CHECK:  encoding: [0x62,0x63,0xfd,0x38,0x56,0xa2,0x00,0x04,0x00,0x00,0x7b]
+          vreducepd $0x7b,1024(%rdx){1to4}, %ymm28
+
+// CHECK: vreducepd $123, -1024(%rdx){1to4}, %ymm28
+// CHECK:  encoding: [0x62,0x63,0xfd,0x38,0x56,0x62,0x80,0x7b]
+          vreducepd $0x7b,-1024(%rdx){1to4}, %ymm28
+
+// CHECK: vreducepd $123, -1032(%rdx){1to4}, %ymm28
+// CHECK:  encoding: [0x62,0x63,0xfd,0x38,0x56,0xa2,0xf8,0xfb,0xff,0xff,0x7b]
+          vreducepd $0x7b,-1032(%rdx){1to4}, %ymm28
+
+// CHECK: vreduceps $171, %xmm21, %xmm29
+// CHECK:  encoding: [0x62,0x23,0x7d,0x08,0x56,0xed,0xab]
+          vreduceps $0xab, %xmm21, %xmm29
+
+// CHECK: vreduceps $171, %xmm21, %xmm29 {%k7}
+// CHECK:  encoding: [0x62,0x23,0x7d,0x0f,0x56,0xed,0xab]
+          vreduceps $0xab, %xmm21, %xmm29 {%k7}
+
+// CHECK: vreduceps $171, %xmm21, %xmm29 {%k7} {z}
+// CHECK:  encoding: [0x62,0x23,0x7d,0x8f,0x56,0xed,0xab]
+          vreduceps $0xab, %xmm21, %xmm29 {%k7} {z}
+
+// CHECK: vreduceps $123, %xmm21, %xmm29
+// CHECK:  encoding: [0x62,0x23,0x7d,0x08,0x56,0xed,0x7b]
+          vreduceps $0x7b, %xmm21, %xmm29
+
+// CHECK: vreduceps $123, (%rcx), %xmm29
+// CHECK:  encoding: [0x62,0x63,0x7d,0x08,0x56,0x29,0x7b]
+          vreduceps $0x7b,(%rcx), %xmm29
+
+// CHECK: vreduceps $123, 4660(%rax,%r14,8), %xmm29
+// CHECK:  encoding: [0x62,0x23,0x7d,0x08,0x56,0xac,0xf0,0x34,0x12,0x00,0x00,0x7b]
+          vreduceps $0x7b,4660(%rax,%r14,8), %xmm29
+
+// CHECK: vreduceps $123, (%rcx){1to4}, %xmm29
+// CHECK:  encoding: [0x62,0x63,0x7d,0x18,0x56,0x29,0x7b]
+          vreduceps $0x7b,(%rcx){1to4}, %xmm29
+
+// CHECK: vreduceps $123, 2032(%rdx), %xmm29
+// CHECK:  encoding: [0x62,0x63,0x7d,0x08,0x56,0x6a,0x7f,0x7b]
+          vreduceps $0x7b,2032(%rdx), %xmm29
+
+// CHECK: vreduceps $123, 2048(%rdx), %xmm29
+// CHECK:  encoding: [0x62,0x63,0x7d,0x08,0x56,0xaa,0x00,0x08,0x00,0x00,0x7b]
+          vreduceps $0x7b,2048(%rdx), %xmm29
+
+// CHECK: vreduceps $123, -2048(%rdx), %xmm29
+// CHECK:  encoding: [0x62,0x63,0x7d,0x08,0x56,0x6a,0x80,0x7b]
+          vreduceps $0x7b,-2048(%rdx), %xmm29
+
+// CHECK: vreduceps $123, -2064(%rdx), %xmm29
+// CHECK:  encoding: [0x62,0x63,0x7d,0x08,0x56,0xaa,0xf0,0xf7,0xff,0xff,0x7b]
+          vreduceps $0x7b,-2064(%rdx), %xmm29
+
+// CHECK: vreduceps $123, 508(%rdx){1to4}, %xmm29
+// CHECK:  encoding: [0x62,0x63,0x7d,0x18,0x56,0x6a,0x7f,0x7b]
+          vreduceps $0x7b,508(%rdx){1to4}, %xmm29
+
+// CHECK: vreduceps $123, 512(%rdx){1to4}, %xmm29
+// CHECK:  encoding: [0x62,0x63,0x7d,0x18,0x56,0xaa,0x00,0x02,0x00,0x00,0x7b]
+          vreduceps $0x7b,512(%rdx){1to4}, %xmm29
+
+// CHECK: vreduceps $123, -512(%rdx){1to4}, %xmm29
+// CHECK:  encoding: [0x62,0x63,0x7d,0x18,0x56,0x6a,0x80,0x7b]
+          vreduceps $0x7b,-512(%rdx){1to4}, %xmm29
+
+// CHECK: vreduceps $123, -516(%rdx){1to4}, %xmm29
+// CHECK:  encoding: [0x62,0x63,0x7d,0x18,0x56,0xaa,0xfc,0xfd,0xff,0xff,0x7b]
+          vreduceps $0x7b,-516(%rdx){1to4}, %xmm29
+
+// CHECK: vreduceps $171, %ymm23, %ymm25
+// CHECK:  encoding: [0x62,0x23,0x7d,0x28,0x56,0xcf,0xab]
+          vreduceps $0xab, %ymm23, %ymm25
+
+// CHECK: vreduceps $171, %ymm23, %ymm25 {%k3}
+// CHECK:  encoding: [0x62,0x23,0x7d,0x2b,0x56,0xcf,0xab]
+          vreduceps $0xab, %ymm23, %ymm25 {%k3}
+
+// CHECK: vreduceps $171, %ymm23, %ymm25 {%k3} {z}
+// CHECK:  encoding: [0x62,0x23,0x7d,0xab,0x56,0xcf,0xab]
+          vreduceps $0xab, %ymm23, %ymm25 {%k3} {z}
+
+// CHECK: vreduceps $123, %ymm23, %ymm25
+// CHECK:  encoding: [0x62,0x23,0x7d,0x28,0x56,0xcf,0x7b]
+          vreduceps $0x7b, %ymm23, %ymm25
+
+// CHECK: vreduceps $123, (%rcx), %ymm25
+// CHECK:  encoding: [0x62,0x63,0x7d,0x28,0x56,0x09,0x7b]
+          vreduceps $0x7b,(%rcx), %ymm25
+
+// CHECK: vreduceps $123, 4660(%rax,%r14,8), %ymm25
+// CHECK:  encoding: [0x62,0x23,0x7d,0x28,0x56,0x8c,0xf0,0x34,0x12,0x00,0x00,0x7b]
+          vreduceps $0x7b,4660(%rax,%r14,8), %ymm25
+
+// CHECK: vreduceps $123, (%rcx){1to8}, %ymm25
+// CHECK:  encoding: [0x62,0x63,0x7d,0x38,0x56,0x09,0x7b]
+          vreduceps $0x7b,(%rcx){1to8}, %ymm25
+
+// CHECK: vreduceps $123, 4064(%rdx), %ymm25
+// CHECK:  encoding: [0x62,0x63,0x7d,0x28,0x56,0x4a,0x7f,0x7b]
+          vreduceps $0x7b,4064(%rdx), %ymm25
+
+// CHECK: vreduceps $123, 4096(%rdx), %ymm25
+// CHECK:  encoding: [0x62,0x63,0x7d,0x28,0x56,0x8a,0x00,0x10,0x00,0x00,0x7b]
+          vreduceps $0x7b,4096(%rdx), %ymm25
+
+// CHECK: vreduceps $123, -4096(%rdx), %ymm25
+// CHECK:  encoding: [0x62,0x63,0x7d,0x28,0x56,0x4a,0x80,0x7b]
+          vreduceps $0x7b,-4096(%rdx), %ymm25
+
+// CHECK: vreduceps $123, -4128(%rdx), %ymm25
+// CHECK:  encoding: [0x62,0x63,0x7d,0x28,0x56,0x8a,0xe0,0xef,0xff,0xff,0x7b]
+          vreduceps $0x7b,-4128(%rdx), %ymm25
+
+// CHECK: vreduceps $123, 508(%rdx){1to8}, %ymm25
+// CHECK:  encoding: [0x62,0x63,0x7d,0x38,0x56,0x4a,0x7f,0x7b]
+          vreduceps $0x7b,508(%rdx){1to8}, %ymm25
+
+// CHECK: vreduceps $123, 512(%rdx){1to8}, %ymm25
+// CHECK:  encoding: [0x62,0x63,0x7d,0x38,0x56,0x8a,0x00,0x02,0x00,0x00,0x7b]
+          vreduceps $0x7b,512(%rdx){1to8}, %ymm25
+
+// CHECK: vreduceps $123, -512(%rdx){1to8}, %ymm25
+// CHECK:  encoding: [0x62,0x63,0x7d,0x38,0x56,0x4a,0x80,0x7b]
+          vreduceps $0x7b,-512(%rdx){1to8}, %ymm25
+
+// CHECK: vreduceps $123, -516(%rdx){1to8}, %ymm25
+// CHECK:  encoding: [0x62,0x63,0x7d,0x38,0x56,0x8a,0xfc,0xfd,0xff,0xff,0x7b]
+          vreduceps $0x7b,-516(%rdx){1to8}, %ymm25
+
+// CHECK: vreducepd $123, (%rcx){1to2}, %xmm18
+// CHECK:  encoding: [0x62,0xe3,0xfd,0x18,0x56,0x11,0x7b]
+          vreducepd $0x7b,(%rcx){1to2}, %xmm18
+
+// CHECK: vreducepd $123, 2032(%rdx), %xmm18
+// CHECK:  encoding: [0x62,0xe3,0xfd,0x08,0x56,0x52,0x7f,0x7b]
+          vreducepd $0x7b,2032(%rdx), %xmm18
+
+// CHECK: vreducepd $123, 2048(%rdx), %xmm18
+// CHECK:  encoding: [0x62,0xe3,0xfd,0x08,0x56,0x92,0x00,0x08,0x00,0x00,0x7b]
+          vreducepd $0x7b,2048(%rdx), %xmm18
+
+// CHECK: vreducepd $123, -2048(%rdx), %xmm18
+// CHECK:  encoding: [0x62,0xe3,0xfd,0x08,0x56,0x52,0x80,0x7b]
+          vreducepd $0x7b,-2048(%rdx), %xmm18
+
+// CHECK: vreducepd $123, -2064(%rdx), %xmm18
+// CHECK:  encoding: [0x62,0xe3,0xfd,0x08,0x56,0x92,0xf0,0xf7,0xff,0xff,0x7b]
+          vreducepd $0x7b,-2064(%rdx), %xmm18
+
+// CHECK: vreducepd $123, 1016(%rdx){1to2}, %xmm18
+// CHECK:  encoding: [0x62,0xe3,0xfd,0x18,0x56,0x52,0x7f,0x7b]
+          vreducepd $0x7b,1016(%rdx){1to2}, %xmm18
+
+// CHECK: vreducepd $123, 1024(%rdx){1to2}, %xmm18
+// CHECK:  encoding: [0x62,0xe3,0xfd,0x18,0x56,0x92,0x00,0x04,0x00,0x00,0x7b]
+          vreducepd $0x7b,1024(%rdx){1to2}, %xmm18
+
+// CHECK: vreducepd $123, -1024(%rdx){1to2}, %xmm18
+// CHECK:  encoding: [0x62,0xe3,0xfd,0x18,0x56,0x52,0x80,0x7b]
+          vreducepd $0x7b,-1024(%rdx){1to2}, %xmm18
+
+// CHECK: vreducepd $123, -1032(%rdx){1to2}, %xmm18
+// CHECK:  encoding: [0x62,0xe3,0xfd,0x18,0x56,0x92,0xf8,0xfb,0xff,0xff,0x7b]
+          vreducepd $0x7b,-1032(%rdx){1to2}, %xmm18
+
+// CHECK: vreducepd $171, %ymm29, %ymm25
+// CHECK:  encoding: [0x62,0x03,0xfd,0x28,0x56,0xcd,0xab]
+          vreducepd $0xab, %ymm29, %ymm25
+
+// CHECK: vreducepd $171, %ymm29, %ymm25 {%k1}
+// CHECK:  encoding: [0x62,0x03,0xfd,0x29,0x56,0xcd,0xab]
+          vreducepd $0xab, %ymm29, %ymm25 {%k1}
+
+// CHECK: vreducepd $171, %ymm29, %ymm25 {%k1} {z}
+// CHECK:  encoding: [0x62,0x03,0xfd,0xa9,0x56,0xcd,0xab]
+          vreducepd $0xab, %ymm29, %ymm25 {%k1} {z}
+
+// CHECK: vreducepd $123, %ymm29, %ymm25
+// CHECK:  encoding: [0x62,0x03,0xfd,0x28,0x56,0xcd,0x7b]
+          vreducepd $0x7b, %ymm29, %ymm25
+
+// CHECK: vreducepd $123, (%rcx), %ymm25
+// CHECK:  encoding: [0x62,0x63,0xfd,0x28,0x56,0x09,0x7b]
+          vreducepd $0x7b,(%rcx), %ymm25
+
+// CHECK: vreducepd $123, 291(%rax,%r14,8), %ymm25
+// CHECK:  encoding: [0x62,0x23,0xfd,0x28,0x56,0x8c,0xf0,0x23,0x01,0x00,0x00,0x7b]
+          vreducepd $0x7b,291(%rax,%r14,8), %ymm25
+
+// CHECK: vreducepd $123, (%rcx){1to4}, %ymm25
+// CHECK:  encoding: [0x62,0x63,0xfd,0x38,0x56,0x09,0x7b]
+          vreducepd $0x7b,(%rcx){1to4}, %ymm25
+
+// CHECK: vreducepd $123, 4064(%rdx), %ymm25
+// CHECK:  encoding: [0x62,0x63,0xfd,0x28,0x56,0x4a,0x7f,0x7b]
+          vreducepd $0x7b,4064(%rdx), %ymm25
+
+// CHECK: vreducepd $123, 4096(%rdx), %ymm25
+// CHECK:  encoding: [0x62,0x63,0xfd,0x28,0x56,0x8a,0x00,0x10,0x00,0x00,0x7b]
+          vreducepd $0x7b,4096(%rdx), %ymm25
+
+// CHECK: vreducepd $123, -4096(%rdx), %ymm25
+// CHECK:  encoding: [0x62,0x63,0xfd,0x28,0x56,0x4a,0x80,0x7b]
+          vreducepd $0x7b,-4096(%rdx), %ymm25
+
+// CHECK: vreducepd $123, -4128(%rdx), %ymm25
+// CHECK:  encoding: [0x62,0x63,0xfd,0x28,0x56,0x8a,0xe0,0xef,0xff,0xff,0x7b]
+          vreducepd $0x7b,-4128(%rdx), %ymm25
+
+// CHECK: vreducepd $123, 1016(%rdx){1to4}, %ymm25
+// CHECK:  encoding: [0x62,0x63,0xfd,0x38,0x56,0x4a,0x7f,0x7b]
+          vreducepd $0x7b,1016(%rdx){1to4}, %ymm25
+
+// CHECK: vreducepd $123, 1024(%rdx){1to4}, %ymm25
+// CHECK:  encoding: [0x62,0x63,0xfd,0x38,0x56,0x8a,0x00,0x04,0x00,0x00,0x7b]
+          vreducepd $0x7b,1024(%rdx){1to4}, %ymm25
+
+// CHECK: vreducepd $123, -1024(%rdx){1to4}, %ymm25
+// CHECK:  encoding: [0x62,0x63,0xfd,0x38,0x56,0x4a,0x80,0x7b]
+          vreducepd $0x7b,-1024(%rdx){1to4}, %ymm25
+
+// CHECK: vreducepd $123, -1032(%rdx){1to4}, %ymm25
+// CHECK:  encoding: [0x62,0x63,0xfd,0x38,0x56,0x8a,0xf8,0xfb,0xff,0xff,0x7b]
+          vreducepd $0x7b,-1032(%rdx){1to4}, %ymm25
+
+// CHECK: vreduceps $171, %xmm23, %xmm20
+// CHECK:  encoding: [0x62,0xa3,0x7d,0x08,0x56,0xe7,0xab]
+          vreduceps $0xab, %xmm23, %xmm20
+
+// CHECK: vreduceps $171, %xmm23, %xmm20 {%k7}
+// CHECK:  encoding: [0x62,0xa3,0x7d,0x0f,0x56,0xe7,0xab]
+          vreduceps $0xab, %xmm23, %xmm20 {%k7}
+
+// CHECK: vreduceps $171, %xmm23, %xmm20 {%k7} {z}
+// CHECK:  encoding: [0x62,0xa3,0x7d,0x8f,0x56,0xe7,0xab]
+          vreduceps $0xab, %xmm23, %xmm20 {%k7} {z}
+
+// CHECK: vreduceps $123, %xmm23, %xmm20
+// CHECK:  encoding: [0x62,0xa3,0x7d,0x08,0x56,0xe7,0x7b]
+          vreduceps $0x7b, %xmm23, %xmm20
+
+// CHECK: vreduceps $123, (%rcx), %xmm20
+// CHECK:  encoding: [0x62,0xe3,0x7d,0x08,0x56,0x21,0x7b]
+          vreduceps $0x7b,(%rcx), %xmm20
+
+// CHECK: vreduceps $123, 291(%rax,%r14,8), %xmm20
+// CHECK:  encoding: [0x62,0xa3,0x7d,0x08,0x56,0xa4,0xf0,0x23,0x01,0x00,0x00,0x7b]
+          vreduceps $0x7b,291(%rax,%r14,8), %xmm20
+
+// CHECK: vreduceps $123, (%rcx){1to4}, %xmm20
+// CHECK:  encoding: [0x62,0xe3,0x7d,0x18,0x56,0x21,0x7b]
+          vreduceps $0x7b,(%rcx){1to4}, %xmm20
+
+// CHECK: vreduceps $123, 2032(%rdx), %xmm20
+// CHECK:  encoding: [0x62,0xe3,0x7d,0x08,0x56,0x62,0x7f,0x7b]
+          vreduceps $0x7b,2032(%rdx), %xmm20
+
+// CHECK: vreduceps $123, 2048(%rdx), %xmm20
+// CHECK:  encoding: [0x62,0xe3,0x7d,0x08,0x56,0xa2,0x00,0x08,0x00,0x00,0x7b]
+          vreduceps $0x7b,2048(%rdx), %xmm20
+
+// CHECK: vreduceps $123, -2048(%rdx), %xmm20
+// CHECK:  encoding: [0x62,0xe3,0x7d,0x08,0x56,0x62,0x80,0x7b]
+          vreduceps $0x7b,-2048(%rdx), %xmm20
+
+// CHECK: vreduceps $123, -2064(%rdx), %xmm20
+// CHECK:  encoding: [0x62,0xe3,0x7d,0x08,0x56,0xa2,0xf0,0xf7,0xff,0xff,0x7b]
+          vreduceps $0x7b,-2064(%rdx), %xmm20
+
+// CHECK: vreduceps $123, 508(%rdx){1to4}, %xmm20
+// CHECK:  encoding: [0x62,0xe3,0x7d,0x18,0x56,0x62,0x7f,0x7b]
+          vreduceps $0x7b,508(%rdx){1to4}, %xmm20
+
+// CHECK: vreduceps $123, 512(%rdx){1to4}, %xmm20
+// CHECK:  encoding: [0x62,0xe3,0x7d,0x18,0x56,0xa2,0x00,0x02,0x00,0x00,0x7b]
+          vreduceps $0x7b,512(%rdx){1to4}, %xmm20
+
+// CHECK: vreduceps $123, -512(%rdx){1to4}, %xmm20
+// CHECK:  encoding: [0x62,0xe3,0x7d,0x18,0x56,0x62,0x80,0x7b]
+          vreduceps $0x7b,-512(%rdx){1to4}, %xmm20
+
+// CHECK: vreduceps $123, -516(%rdx){1to4}, %xmm20
+// CHECK:  encoding: [0x62,0xe3,0x7d,0x18,0x56,0xa2,0xfc,0xfd,0xff,0xff,0x7b]
+          vreduceps $0x7b,-516(%rdx){1to4}, %xmm20
+
+// CHECK: vreduceps $171, %ymm22, %ymm26
+// CHECK:  encoding: [0x62,0x23,0x7d,0x28,0x56,0xd6,0xab]
+          vreduceps $0xab, %ymm22, %ymm26
+
+// CHECK: vreduceps $171, %ymm22, %ymm26 {%k6}
+// CHECK:  encoding: [0x62,0x23,0x7d,0x2e,0x56,0xd6,0xab]
+          vreduceps $0xab, %ymm22, %ymm26 {%k6}
+
+// CHECK: vreduceps $171, %ymm22, %ymm26 {%k6} {z}
+// CHECK:  encoding: [0x62,0x23,0x7d,0xae,0x56,0xd6,0xab]
+          vreduceps $0xab, %ymm22, %ymm26 {%k6} {z}
+
+// CHECK: vreduceps $123, %ymm22, %ymm26
+// CHECK:  encoding: [0x62,0x23,0x7d,0x28,0x56,0xd6,0x7b]
+          vreduceps $0x7b, %ymm22, %ymm26
+
+// CHECK: vreduceps $123, (%rcx), %ymm26
+// CHECK:  encoding: [0x62,0x63,0x7d,0x28,0x56,0x11,0x7b]
+          vreduceps $0x7b,(%rcx), %ymm26
+
+// CHECK: vreduceps $123, 291(%rax,%r14,8), %ymm26
+// CHECK:  encoding: [0x62,0x23,0x7d,0x28,0x56,0x94,0xf0,0x23,0x01,0x00,0x00,0x7b]
+          vreduceps $0x7b,291(%rax,%r14,8), %ymm26
+
+// CHECK: vreduceps $123, (%rcx){1to8}, %ymm26
+// CHECK:  encoding: [0x62,0x63,0x7d,0x38,0x56,0x11,0x7b]
+          vreduceps $0x7b,(%rcx){1to8}, %ymm26
+
+// CHECK: vreduceps $123, 4064(%rdx), %ymm26
+// CHECK:  encoding: [0x62,0x63,0x7d,0x28,0x56,0x52,0x7f,0x7b]
+          vreduceps $0x7b,4064(%rdx), %ymm26
+
+// CHECK: vreduceps $123, 4096(%rdx), %ymm26
+// CHECK:  encoding: [0x62,0x63,0x7d,0x28,0x56,0x92,0x00,0x10,0x00,0x00,0x7b]
+          vreduceps $0x7b,4096(%rdx), %ymm26
+
+// CHECK: vreduceps $123, -4096(%rdx), %ymm26
+// CHECK:  encoding: [0x62,0x63,0x7d,0x28,0x56,0x52,0x80,0x7b]
+          vreduceps $0x7b,-4096(%rdx), %ymm26
+
+// CHECK: vreduceps $123, -4128(%rdx), %ymm26
+// CHECK:  encoding: [0x62,0x63,0x7d,0x28,0x56,0x92,0xe0,0xef,0xff,0xff,0x7b]
+          vreduceps $0x7b,-4128(%rdx), %ymm26
+
+// CHECK: vreduceps $123, 508(%rdx){1to8}, %ymm26
+// CHECK:  encoding: [0x62,0x63,0x7d,0x38,0x56,0x52,0x7f,0x7b]
+          vreduceps $0x7b,508(%rdx){1to8}, %ymm26
+
+// CHECK: vreduceps $123, 512(%rdx){1to8}, %ymm26
+// CHECK:  encoding: [0x62,0x63,0x7d,0x38,0x56,0x92,0x00,0x02,0x00,0x00,0x7b]
+          vreduceps $0x7b,512(%rdx){1to8}, %ymm26
+
+// CHECK: vreduceps $123, -512(%rdx){1to8}, %ymm26
+// CHECK:  encoding: [0x62,0x63,0x7d,0x38,0x56,0x52,0x80,0x7b]
+          vreduceps $0x7b,-512(%rdx){1to8}, %ymm26
+
+// CHECK: vreduceps $123, -516(%rdx){1to8}, %ymm26
+// CHECK:  encoding: [0x62,0x63,0x7d,0x38,0x56,0x92,0xfc,0xfd,0xff,0xff,0x7b]
+          vreduceps $0x7b,-516(%rdx){1to8}, %ymm26
+
+// CHECK: vcvtpd2qq %xmm22, %xmm24
+// CHECK:  encoding: [0x62,0x21,0xfd,0x08,0x7b,0xc6]
+          vcvtpd2qq %xmm22, %xmm24
+
+// CHECK: vcvtpd2qq %xmm22, %xmm24 {%k6}
+// CHECK:  encoding: [0x62,0x21,0xfd,0x0e,0x7b,0xc6]
+          vcvtpd2qq %xmm22, %xmm24 {%k6}
+
+// CHECK: vcvtpd2qq %xmm22, %xmm24 {%k6} {z}
+// CHECK:  encoding: [0x62,0x21,0xfd,0x8e,0x7b,0xc6]
+          vcvtpd2qq %xmm22, %xmm24 {%k6} {z}
+
+// CHECK: vcvtpd2qq (%rcx), %xmm24
+// CHECK:  encoding: [0x62,0x61,0xfd,0x08,0x7b,0x01]
+          vcvtpd2qq (%rcx), %xmm24
+
+// CHECK: vcvtpd2qq 291(%rax,%r14,8), %xmm24
+// CHECK:  encoding: [0x62,0x21,0xfd,0x08,0x7b,0x84,0xf0,0x23,0x01,0x00,0x00]
+          vcvtpd2qq 291(%rax,%r14,8), %xmm24
+
+// CHECK: vcvtpd2qq (%rcx){1to2}, %xmm24
+// CHECK:  encoding: [0x62,0x61,0xfd,0x18,0x7b,0x01]
+          vcvtpd2qq (%rcx){1to2}, %xmm24
+
+// CHECK: vcvtpd2qq 2032(%rdx), %xmm24
+// CHECK:  encoding: [0x62,0x61,0xfd,0x08,0x7b,0x42,0x7f]
+          vcvtpd2qq 2032(%rdx), %xmm24
+
+// CHECK: vcvtpd2qq 2048(%rdx), %xmm24
+// CHECK:  encoding: [0x62,0x61,0xfd,0x08,0x7b,0x82,0x00,0x08,0x00,0x00]
+          vcvtpd2qq 2048(%rdx), %xmm24
+
+// CHECK: vcvtpd2qq -2048(%rdx), %xmm24
+// CHECK:  encoding: [0x62,0x61,0xfd,0x08,0x7b,0x42,0x80]
+          vcvtpd2qq -2048(%rdx), %xmm24
+
+// CHECK: vcvtpd2qq -2064(%rdx), %xmm24
+// CHECK:  encoding: [0x62,0x61,0xfd,0x08,0x7b,0x82,0xf0,0xf7,0xff,0xff]
+          vcvtpd2qq -2064(%rdx), %xmm24
+
+// CHECK: vcvtpd2qq 1016(%rdx){1to2}, %xmm24
+// CHECK:  encoding: [0x62,0x61,0xfd,0x18,0x7b,0x42,0x7f]
+          vcvtpd2qq 1016(%rdx){1to2}, %xmm24
+
+// CHECK: vcvtpd2qq 1024(%rdx){1to2}, %xmm24
+// CHECK:  encoding: [0x62,0x61,0xfd,0x18,0x7b,0x82,0x00,0x04,0x00,0x00]
+          vcvtpd2qq 1024(%rdx){1to2}, %xmm24
+
+// CHECK: vcvtpd2qq -1024(%rdx){1to2}, %xmm24
+// CHECK:  encoding: [0x62,0x61,0xfd,0x18,0x7b,0x42,0x80]
+          vcvtpd2qq -1024(%rdx){1to2}, %xmm24
+
+// CHECK: vcvtpd2qq -1032(%rdx){1to2}, %xmm24
+// CHECK:  encoding: [0x62,0x61,0xfd,0x18,0x7b,0x82,0xf8,0xfb,0xff,0xff]
+          vcvtpd2qq -1032(%rdx){1to2}, %xmm24
+
+// CHECK: vcvtpd2qq %ymm27, %ymm24
+// CHECK:  encoding: [0x62,0x01,0xfd,0x28,0x7b,0xc3]
+          vcvtpd2qq %ymm27, %ymm24
+
+// CHECK: vcvtpd2qq %ymm27, %ymm24 {%k7}
+// CHECK:  encoding: [0x62,0x01,0xfd,0x2f,0x7b,0xc3]
+          vcvtpd2qq %ymm27, %ymm24 {%k7}
+
+// CHECK: vcvtpd2qq %ymm27, %ymm24 {%k7} {z}
+// CHECK:  encoding: [0x62,0x01,0xfd,0xaf,0x7b,0xc3]
+          vcvtpd2qq %ymm27, %ymm24 {%k7} {z}
+
+// CHECK: vcvtpd2qq (%rcx), %ymm24
+// CHECK:  encoding: [0x62,0x61,0xfd,0x28,0x7b,0x01]
+          vcvtpd2qq (%rcx), %ymm24
+
+// CHECK: vcvtpd2qq 291(%rax,%r14,8), %ymm24
+// CHECK:  encoding: [0x62,0x21,0xfd,0x28,0x7b,0x84,0xf0,0x23,0x01,0x00,0x00]
+          vcvtpd2qq 291(%rax,%r14,8), %ymm24
+
+// CHECK: vcvtpd2qq (%rcx){1to4}, %ymm24
+// CHECK:  encoding: [0x62,0x61,0xfd,0x38,0x7b,0x01]
+          vcvtpd2qq (%rcx){1to4}, %ymm24
+
+// CHECK: vcvtpd2qq 4064(%rdx), %ymm24
+// CHECK:  encoding: [0x62,0x61,0xfd,0x28,0x7b,0x42,0x7f]
+          vcvtpd2qq 4064(%rdx), %ymm24
+
+// CHECK: vcvtpd2qq 4096(%rdx), %ymm24
+// CHECK:  encoding: [0x62,0x61,0xfd,0x28,0x7b,0x82,0x00,0x10,0x00,0x00]
+          vcvtpd2qq 4096(%rdx), %ymm24
+
+// CHECK: vcvtpd2qq -4096(%rdx), %ymm24
+// CHECK:  encoding: [0x62,0x61,0xfd,0x28,0x7b,0x42,0x80]
+          vcvtpd2qq -4096(%rdx), %ymm24
+
+// CHECK: vcvtpd2qq -4128(%rdx), %ymm24
+// CHECK:  encoding: [0x62,0x61,0xfd,0x28,0x7b,0x82,0xe0,0xef,0xff,0xff]
+          vcvtpd2qq -4128(%rdx), %ymm24
+
+// CHECK: vcvtpd2qq 1016(%rdx){1to4}, %ymm24
+// CHECK:  encoding: [0x62,0x61,0xfd,0x38,0x7b,0x42,0x7f]
+          vcvtpd2qq 1016(%rdx){1to4}, %ymm24
+
+// CHECK: vcvtpd2qq 1024(%rdx){1to4}, %ymm24
+// CHECK:  encoding: [0x62,0x61,0xfd,0x38,0x7b,0x82,0x00,0x04,0x00,0x00]
+          vcvtpd2qq 1024(%rdx){1to4}, %ymm24
+
+// CHECK: vcvtpd2qq -1024(%rdx){1to4}, %ymm24
+// CHECK:  encoding: [0x62,0x61,0xfd,0x38,0x7b,0x42,0x80]
+          vcvtpd2qq -1024(%rdx){1to4}, %ymm24
+
+// CHECK: vcvtpd2qq -1032(%rdx){1to4}, %ymm24
+// CHECK:  encoding: [0x62,0x61,0xfd,0x38,0x7b,0x82,0xf8,0xfb,0xff,0xff]
+          vcvtpd2qq -1032(%rdx){1to4}, %ymm24
+
+// CHECK: vcvtpd2uqq %xmm20, %xmm22
+// CHECK:  encoding: [0x62,0xa1,0xfd,0x08,0x79,0xf4]
+          vcvtpd2uqq %xmm20, %xmm22
+
+// CHECK: vcvtpd2uqq %xmm20, %xmm22 {%k3}
+// CHECK:  encoding: [0x62,0xa1,0xfd,0x0b,0x79,0xf4]
+          vcvtpd2uqq %xmm20, %xmm22 {%k3}
+
+// CHECK: vcvtpd2uqq %xmm20, %xmm22 {%k3} {z}
+// CHECK:  encoding: [0x62,0xa1,0xfd,0x8b,0x79,0xf4]
+          vcvtpd2uqq %xmm20, %xmm22 {%k3} {z}
+
+// CHECK: vcvtpd2uqq (%rcx), %xmm22
+// CHECK:  encoding: [0x62,0xe1,0xfd,0x08,0x79,0x31]
+          vcvtpd2uqq (%rcx), %xmm22
+
+// CHECK: vcvtpd2uqq 291(%rax,%r14,8), %xmm22
+// CHECK:  encoding: [0x62,0xa1,0xfd,0x08,0x79,0xb4,0xf0,0x23,0x01,0x00,0x00]
+          vcvtpd2uqq 291(%rax,%r14,8), %xmm22
+
+// CHECK: vcvtpd2uqq (%rcx){1to2}, %xmm22
+// CHECK:  encoding: [0x62,0xe1,0xfd,0x18,0x79,0x31]
+          vcvtpd2uqq (%rcx){1to2}, %xmm22
+
+// CHECK: vcvtpd2uqq 2032(%rdx), %xmm22
+// CHECK:  encoding: [0x62,0xe1,0xfd,0x08,0x79,0x72,0x7f]
+          vcvtpd2uqq 2032(%rdx), %xmm22
+
+// CHECK: vcvtpd2uqq 2048(%rdx), %xmm22
+// CHECK:  encoding: [0x62,0xe1,0xfd,0x08,0x79,0xb2,0x00,0x08,0x00,0x00]
+          vcvtpd2uqq 2048(%rdx), %xmm22
+
+// CHECK: vcvtpd2uqq -2048(%rdx), %xmm22
+// CHECK:  encoding: [0x62,0xe1,0xfd,0x08,0x79,0x72,0x80]
+          vcvtpd2uqq -2048(%rdx), %xmm22
+
+// CHECK: vcvtpd2uqq -2064(%rdx), %xmm22
+// CHECK:  encoding: [0x62,0xe1,0xfd,0x08,0x79,0xb2,0xf0,0xf7,0xff,0xff]
+          vcvtpd2uqq -2064(%rdx), %xmm22
+
+// CHECK: vcvtpd2uqq 1016(%rdx){1to2}, %xmm22
+// CHECK:  encoding: [0x62,0xe1,0xfd,0x18,0x79,0x72,0x7f]
+          vcvtpd2uqq 1016(%rdx){1to2}, %xmm22
+
+// CHECK: vcvtpd2uqq 1024(%rdx){1to2}, %xmm22
+// CHECK:  encoding: [0x62,0xe1,0xfd,0x18,0x79,0xb2,0x00,0x04,0x00,0x00]
+          vcvtpd2uqq 1024(%rdx){1to2}, %xmm22
+
+// CHECK: vcvtpd2uqq -1024(%rdx){1to2}, %xmm22
+// CHECK:  encoding: [0x62,0xe1,0xfd,0x18,0x79,0x72,0x80]
+          vcvtpd2uqq -1024(%rdx){1to2}, %xmm22
+
+// CHECK: vcvtpd2uqq -1032(%rdx){1to2}, %xmm22
+// CHECK:  encoding: [0x62,0xe1,0xfd,0x18,0x79,0xb2,0xf8,0xfb,0xff,0xff]
+          vcvtpd2uqq -1032(%rdx){1to2}, %xmm22
+
+// CHECK: vcvtpd2uqq %ymm24, %ymm21
+// CHECK:  encoding: [0x62,0x81,0xfd,0x28,0x79,0xe8]
+          vcvtpd2uqq %ymm24, %ymm21
+
+// CHECK: vcvtpd2uqq %ymm24, %ymm21 {%k6}
+// CHECK:  encoding: [0x62,0x81,0xfd,0x2e,0x79,0xe8]
+          vcvtpd2uqq %ymm24, %ymm21 {%k6}
+
+// CHECK: vcvtpd2uqq %ymm24, %ymm21 {%k6} {z}
+// CHECK:  encoding: [0x62,0x81,0xfd,0xae,0x79,0xe8]
+          vcvtpd2uqq %ymm24, %ymm21 {%k6} {z}
+
+// CHECK: vcvtpd2uqq (%rcx), %ymm21
+// CHECK:  encoding: [0x62,0xe1,0xfd,0x28,0x79,0x29]
+          vcvtpd2uqq (%rcx), %ymm21
+
+// CHECK: vcvtpd2uqq 291(%rax,%r14,8), %ymm21
+// CHECK:  encoding: [0x62,0xa1,0xfd,0x28,0x79,0xac,0xf0,0x23,0x01,0x00,0x00]
+          vcvtpd2uqq 291(%rax,%r14,8), %ymm21
+
+// CHECK: vcvtpd2uqq (%rcx){1to4}, %ymm21
+// CHECK:  encoding: [0x62,0xe1,0xfd,0x38,0x79,0x29]
+          vcvtpd2uqq (%rcx){1to4}, %ymm21
+
+// CHECK: vcvtpd2uqq 4064(%rdx), %ymm21
+// CHECK:  encoding: [0x62,0xe1,0xfd,0x28,0x79,0x6a,0x7f]
+          vcvtpd2uqq 4064(%rdx), %ymm21
+
+// CHECK: vcvtpd2uqq 4096(%rdx), %ymm21
+// CHECK:  encoding: [0x62,0xe1,0xfd,0x28,0x79,0xaa,0x00,0x10,0x00,0x00]
+          vcvtpd2uqq 4096(%rdx), %ymm21
+
+// CHECK: vcvtpd2uqq -4096(%rdx), %ymm21
+// CHECK:  encoding: [0x62,0xe1,0xfd,0x28,0x79,0x6a,0x80]
+          vcvtpd2uqq -4096(%rdx), %ymm21
+
+// CHECK: vcvtpd2uqq -4128(%rdx), %ymm21
+// CHECK:  encoding: [0x62,0xe1,0xfd,0x28,0x79,0xaa,0xe0,0xef,0xff,0xff]
+          vcvtpd2uqq -4128(%rdx), %ymm21
+
+// CHECK: vcvtpd2uqq 1016(%rdx){1to4}, %ymm21
+// CHECK:  encoding: [0x62,0xe1,0xfd,0x38,0x79,0x6a,0x7f]
+          vcvtpd2uqq 1016(%rdx){1to4}, %ymm21
+
+// CHECK: vcvtpd2uqq 1024(%rdx){1to4}, %ymm21
+// CHECK:  encoding: [0x62,0xe1,0xfd,0x38,0x79,0xaa,0x00,0x04,0x00,0x00]
+          vcvtpd2uqq 1024(%rdx){1to4}, %ymm21
+
+// CHECK: vcvtpd2uqq -1024(%rdx){1to4}, %ymm21
+// CHECK:  encoding: [0x62,0xe1,0xfd,0x38,0x79,0x6a,0x80]
+          vcvtpd2uqq -1024(%rdx){1to4}, %ymm21
+
+// CHECK: vcvtpd2uqq -1032(%rdx){1to4}, %ymm21
+// CHECK:  encoding: [0x62,0xe1,0xfd,0x38,0x79,0xaa,0xf8,0xfb,0xff,0xff]
+          vcvtpd2uqq -1032(%rdx){1to4}, %ymm21
+
+// CHECK: vcvtps2qq %xmm28, %xmm17
+// CHECK:  encoding: [0x62,0x81,0x7d,0x08,0x7b,0xcc]
+          vcvtps2qq %xmm28, %xmm17
+
+// CHECK: vcvtps2qq %xmm28, %xmm17 {%k4}
+// CHECK:  encoding: [0x62,0x81,0x7d,0x0c,0x7b,0xcc]
+          vcvtps2qq %xmm28, %xmm17 {%k4}
+
+// CHECK: vcvtps2qq %xmm28, %xmm17 {%k4} {z}
+// CHECK:  encoding: [0x62,0x81,0x7d,0x8c,0x7b,0xcc]
+          vcvtps2qq %xmm28, %xmm17 {%k4} {z}
+
+// CHECK: vcvtps2qq (%rcx), %xmm17
+// CHECK:  encoding: [0x62,0xe1,0x7d,0x08,0x7b,0x09]
+          vcvtps2qq (%rcx), %xmm17
+
+// CHECK: vcvtps2qq 291(%rax,%r14,8), %xmm17
+// CHECK:  encoding: [0x62,0xa1,0x7d,0x08,0x7b,0x8c,0xf0,0x23,0x01,0x00,0x00]
+          vcvtps2qq 291(%rax,%r14,8), %xmm17
+
+// CHECK: vcvtps2qq (%rcx){1to2}, %xmm17
+// CHECK:  encoding: [0x62,0xe1,0x7d,0x18,0x7b,0x09]
+          vcvtps2qq (%rcx){1to2}, %xmm17
+
+// CHECK: vcvtps2qq 1016(%rdx), %xmm17
+// CHECK:  encoding: [0x62,0xe1,0x7d,0x08,0x7b,0x4a,0x7f]
+          vcvtps2qq 1016(%rdx), %xmm17
+
+// CHECK: vcvtps2qq 1024(%rdx), %xmm17
+// CHECK:  encoding: [0x62,0xe1,0x7d,0x08,0x7b,0x8a,0x00,0x04,0x00,0x00]
+          vcvtps2qq 1024(%rdx), %xmm17
+
+// CHECK: vcvtps2qq -1024(%rdx), %xmm17
+// CHECK:  encoding: [0x62,0xe1,0x7d,0x08,0x7b,0x4a,0x80]
+          vcvtps2qq -1024(%rdx), %xmm17
+
+// CHECK: vcvtps2qq -1032(%rdx), %xmm17
+// CHECK:  encoding: [0x62,0xe1,0x7d,0x08,0x7b,0x8a,0xf8,0xfb,0xff,0xff]
+          vcvtps2qq -1032(%rdx), %xmm17
+
+// CHECK: vcvtps2qq 508(%rdx){1to2}, %xmm17
+// CHECK:  encoding: [0x62,0xe1,0x7d,0x18,0x7b,0x4a,0x7f]
+          vcvtps2qq 508(%rdx){1to2}, %xmm17
+
+// CHECK: vcvtps2qq 512(%rdx){1to2}, %xmm17
+// CHECK:  encoding: [0x62,0xe1,0x7d,0x18,0x7b,0x8a,0x00,0x02,0x00,0x00]
+          vcvtps2qq 512(%rdx){1to2}, %xmm17
+
+// CHECK: vcvtps2qq -512(%rdx){1to2}, %xmm17
+// CHECK:  encoding: [0x62,0xe1,0x7d,0x18,0x7b,0x4a,0x80]
+          vcvtps2qq -512(%rdx){1to2}, %xmm17
+
+// CHECK: vcvtps2qq -516(%rdx){1to2}, %xmm17
+// CHECK:  encoding: [0x62,0xe1,0x7d,0x18,0x7b,0x8a,0xfc,0xfd,0xff,0xff]
+          vcvtps2qq -516(%rdx){1to2}, %xmm17
+
+// CHECK: vcvtps2qq %xmm27, %ymm25
+// CHECK:  encoding: [0x62,0x01,0x7d,0x28,0x7b,0xcb]
+          vcvtps2qq %xmm27, %ymm25
+
+// CHECK: vcvtps2qq %xmm27, %ymm25 {%k7}
+// CHECK:  encoding: [0x62,0x01,0x7d,0x2f,0x7b,0xcb]
+          vcvtps2qq %xmm27, %ymm25 {%k7}
+
+// CHECK: vcvtps2qq %xmm27, %ymm25 {%k7} {z}
+// CHECK:  encoding: [0x62,0x01,0x7d,0xaf,0x7b,0xcb]
+          vcvtps2qq %xmm27, %ymm25 {%k7} {z}
+
+// CHECK: vcvtps2qq (%rcx), %ymm25
+// CHECK:  encoding: [0x62,0x61,0x7d,0x28,0x7b,0x09]
+          vcvtps2qq (%rcx), %ymm25
+
+// CHECK: vcvtps2qq 291(%rax,%r14,8), %ymm25
+// CHECK:  encoding: [0x62,0x21,0x7d,0x28,0x7b,0x8c,0xf0,0x23,0x01,0x00,0x00]
+          vcvtps2qq 291(%rax,%r14,8), %ymm25
+
+// CHECK: vcvtps2qq (%rcx){1to4}, %ymm25
+// CHECK:  encoding: [0x62,0x61,0x7d,0x38,0x7b,0x09]
+          vcvtps2qq (%rcx){1to4}, %ymm25
+
+// CHECK: vcvtps2qq 2032(%rdx), %ymm25
+// CHECK:  encoding: [0x62,0x61,0x7d,0x28,0x7b,0x4a,0x7f]
+          vcvtps2qq 2032(%rdx), %ymm25
+
+// CHECK: vcvtps2qq 2048(%rdx), %ymm25
+// CHECK:  encoding: [0x62,0x61,0x7d,0x28,0x7b,0x8a,0x00,0x08,0x00,0x00]
+          vcvtps2qq 2048(%rdx), %ymm25
+
+// CHECK: vcvtps2qq -2048(%rdx), %ymm25
+// CHECK:  encoding: [0x62,0x61,0x7d,0x28,0x7b,0x4a,0x80]
+          vcvtps2qq -2048(%rdx), %ymm25
+
+// CHECK: vcvtps2qq -2064(%rdx), %ymm25
+// CHECK:  encoding: [0x62,0x61,0x7d,0x28,0x7b,0x8a,0xf0,0xf7,0xff,0xff]
+          vcvtps2qq -2064(%rdx), %ymm25
+
+// CHECK: vcvtps2qq 508(%rdx){1to4}, %ymm25
+// CHECK:  encoding: [0x62,0x61,0x7d,0x38,0x7b,0x4a,0x7f]
+          vcvtps2qq 508(%rdx){1to4}, %ymm25
+
+// CHECK: vcvtps2qq 512(%rdx){1to4}, %ymm25
+// CHECK:  encoding: [0x62,0x61,0x7d,0x38,0x7b,0x8a,0x00,0x02,0x00,0x00]
+          vcvtps2qq 512(%rdx){1to4}, %ymm25
+
+// CHECK: vcvtps2qq -512(%rdx){1to4}, %ymm25
+// CHECK:  encoding: [0x62,0x61,0x7d,0x38,0x7b,0x4a,0x80]
+          vcvtps2qq -512(%rdx){1to4}, %ymm25
+
+// CHECK: vcvtps2qq -516(%rdx){1to4}, %ymm25
+// CHECK:  encoding: [0x62,0x61,0x7d,0x38,0x7b,0x8a,0xfc,0xfd,0xff,0xff]
+          vcvtps2qq -516(%rdx){1to4}, %ymm25
+
+// CHECK: vcvtps2uqq %xmm29, %xmm29
+// CHECK:  encoding: [0x62,0x01,0x7d,0x08,0x79,0xed]
+          vcvtps2uqq %xmm29, %xmm29
+
+// CHECK: vcvtps2uqq %xmm29, %xmm29 {%k1}
+// CHECK:  encoding: [0x62,0x01,0x7d,0x09,0x79,0xed]
+          vcvtps2uqq %xmm29, %xmm29 {%k1}
+
+// CHECK: vcvtps2uqq %xmm29, %xmm29 {%k1} {z}
+// CHECK:  encoding: [0x62,0x01,0x7d,0x89,0x79,0xed]
+          vcvtps2uqq %xmm29, %xmm29 {%k1} {z}
+
+// CHECK: vcvtps2uqq (%rcx), %xmm29
+// CHECK:  encoding: [0x62,0x61,0x7d,0x08,0x79,0x29]
+          vcvtps2uqq (%rcx), %xmm29
+
+// CHECK: vcvtps2uqq 291(%rax,%r14,8), %xmm29
+// CHECK:  encoding: [0x62,0x21,0x7d,0x08,0x79,0xac,0xf0,0x23,0x01,0x00,0x00]
+          vcvtps2uqq 291(%rax,%r14,8), %xmm29
+
+// CHECK: vcvtps2uqq (%rcx){1to2}, %xmm29
+// CHECK:  encoding: [0x62,0x61,0x7d,0x18,0x79,0x29]
+          vcvtps2uqq (%rcx){1to2}, %xmm29
+
+// CHECK: vcvtps2uqq 1016(%rdx), %xmm29
+// CHECK:  encoding: [0x62,0x61,0x7d,0x08,0x79,0x6a,0x7f]
+          vcvtps2uqq 1016(%rdx), %xmm29
+
+// CHECK: vcvtps2uqq 1024(%rdx), %xmm29
+// CHECK:  encoding: [0x62,0x61,0x7d,0x08,0x79,0xaa,0x00,0x04,0x00,0x00]
+          vcvtps2uqq 1024(%rdx), %xmm29
+
+// CHECK: vcvtps2uqq -1024(%rdx), %xmm29
+// CHECK:  encoding: [0x62,0x61,0x7d,0x08,0x79,0x6a,0x80]
+          vcvtps2uqq -1024(%rdx), %xmm29
+
+// CHECK: vcvtps2uqq -1032(%rdx), %xmm29
+// CHECK:  encoding: [0x62,0x61,0x7d,0x08,0x79,0xaa,0xf8,0xfb,0xff,0xff]
+          vcvtps2uqq -1032(%rdx), %xmm29
+
+// CHECK: vcvtps2uqq 508(%rdx){1to2}, %xmm29
+// CHECK:  encoding: [0x62,0x61,0x7d,0x18,0x79,0x6a,0x7f]
+          vcvtps2uqq 508(%rdx){1to2}, %xmm29
+
+// CHECK: vcvtps2uqq 512(%rdx){1to2}, %xmm29
+// CHECK:  encoding: [0x62,0x61,0x7d,0x18,0x79,0xaa,0x00,0x02,0x00,0x00]
+          vcvtps2uqq 512(%rdx){1to2}, %xmm29
+
+// CHECK: vcvtps2uqq -512(%rdx){1to2}, %xmm29
+// CHECK:  encoding: [0x62,0x61,0x7d,0x18,0x79,0x6a,0x80]
+          vcvtps2uqq -512(%rdx){1to2}, %xmm29
+
+// CHECK: vcvtps2uqq -516(%rdx){1to2}, %xmm29
+// CHECK:  encoding: [0x62,0x61,0x7d,0x18,0x79,0xaa,0xfc,0xfd,0xff,0xff]
+          vcvtps2uqq -516(%rdx){1to2}, %xmm29
+
+// CHECK: vcvtps2uqq %xmm19, %ymm23
+// CHECK:  encoding: [0x62,0xa1,0x7d,0x28,0x79,0xfb]
+          vcvtps2uqq %xmm19, %ymm23
+
+// CHECK: vcvtps2uqq %xmm19, %ymm23 {%k2}
+// CHECK:  encoding: [0x62,0xa1,0x7d,0x2a,0x79,0xfb]
+          vcvtps2uqq %xmm19, %ymm23 {%k2}
+
+// CHECK: vcvtps2uqq %xmm19, %ymm23 {%k2} {z}
+// CHECK:  encoding: [0x62,0xa1,0x7d,0xaa,0x79,0xfb]
+          vcvtps2uqq %xmm19, %ymm23 {%k2} {z}
+
+// CHECK: vcvtps2uqq (%rcx), %ymm23
+// CHECK:  encoding: [0x62,0xe1,0x7d,0x28,0x79,0x39]
+          vcvtps2uqq (%rcx), %ymm23
+
+// CHECK: vcvtps2uqq 291(%rax,%r14,8), %ymm23
+// CHECK:  encoding: [0x62,0xa1,0x7d,0x28,0x79,0xbc,0xf0,0x23,0x01,0x00,0x00]
+          vcvtps2uqq 291(%rax,%r14,8), %ymm23
+
+// CHECK: vcvtps2uqq (%rcx){1to4}, %ymm23
+// CHECK:  encoding: [0x62,0xe1,0x7d,0x38,0x79,0x39]
+          vcvtps2uqq (%rcx){1to4}, %ymm23
+
+// CHECK: vcvtps2uqq 2032(%rdx), %ymm23
+// CHECK:  encoding: [0x62,0xe1,0x7d,0x28,0x79,0x7a,0x7f]
+          vcvtps2uqq 2032(%rdx), %ymm23
+
+// CHECK: vcvtps2uqq 2048(%rdx), %ymm23
+// CHECK:  encoding: [0x62,0xe1,0x7d,0x28,0x79,0xba,0x00,0x08,0x00,0x00]
+          vcvtps2uqq 2048(%rdx), %ymm23
+
+// CHECK: vcvtps2uqq -2048(%rdx), %ymm23
+// CHECK:  encoding: [0x62,0xe1,0x7d,0x28,0x79,0x7a,0x80]
+          vcvtps2uqq -2048(%rdx), %ymm23
+
+// CHECK: vcvtps2uqq -2064(%rdx), %ymm23
+// CHECK:  encoding: [0x62,0xe1,0x7d,0x28,0x79,0xba,0xf0,0xf7,0xff,0xff]
+          vcvtps2uqq -2064(%rdx), %ymm23
+
+// CHECK: vcvtps2uqq 508(%rdx){1to4}, %ymm23
+// CHECK:  encoding: [0x62,0xe1,0x7d,0x38,0x79,0x7a,0x7f]
+          vcvtps2uqq 508(%rdx){1to4}, %ymm23
+
+// CHECK: vcvtps2uqq 512(%rdx){1to4}, %ymm23
+// CHECK:  encoding: [0x62,0xe1,0x7d,0x38,0x79,0xba,0x00,0x02,0x00,0x00]
+          vcvtps2uqq 512(%rdx){1to4}, %ymm23
+
+// CHECK: vcvtps2uqq -512(%rdx){1to4}, %ymm23
+// CHECK:  encoding: [0x62,0xe1,0x7d,0x38,0x79,0x7a,0x80]
+          vcvtps2uqq -512(%rdx){1to4}, %ymm23
+
+// CHECK: vcvtps2uqq -516(%rdx){1to4}, %ymm23
+// CHECK:  encoding: [0x62,0xe1,0x7d,0x38,0x79,0xba,0xfc,0xfd,0xff,0xff]
+          vcvtps2uqq -516(%rdx){1to4}, %ymm23
+
+// CHECK: vcvtqq2pd %xmm29, %xmm22
+// CHECK:  encoding: [0x62,0x81,0xfe,0x08,0xe6,0xf5]
+          vcvtqq2pd %xmm29, %xmm22
+
+// CHECK: vcvtqq2pd %xmm29, %xmm22 {%k7}
+// CHECK:  encoding: [0x62,0x81,0xfe,0x0f,0xe6,0xf5]
+          vcvtqq2pd %xmm29, %xmm22 {%k7}
+
+// CHECK: vcvtqq2pd %xmm29, %xmm22 {%k7} {z}
+// CHECK:  encoding: [0x62,0x81,0xfe,0x8f,0xe6,0xf5]
+          vcvtqq2pd %xmm29, %xmm22 {%k7} {z}
+
+// CHECK: vcvtqq2pd (%rcx), %xmm22
+// CHECK:  encoding: [0x62,0xe1,0xfe,0x08,0xe6,0x31]
+          vcvtqq2pd (%rcx), %xmm22
+
+// CHECK: vcvtqq2pd 291(%rax,%r14,8), %xmm22
+// CHECK:  encoding: [0x62,0xa1,0xfe,0x08,0xe6,0xb4,0xf0,0x23,0x01,0x00,0x00]
+          vcvtqq2pd 291(%rax,%r14,8), %xmm22
+
+// CHECK: vcvtqq2pd (%rcx){1to2}, %xmm22
+// CHECK:  encoding: [0x62,0xe1,0xfe,0x18,0xe6,0x31]
+          vcvtqq2pd (%rcx){1to2}, %xmm22
+
+// CHECK: vcvtqq2pd 2032(%rdx), %xmm22
+// CHECK:  encoding: [0x62,0xe1,0xfe,0x08,0xe6,0x72,0x7f]
+          vcvtqq2pd 2032(%rdx), %xmm22
+
+// CHECK: vcvtqq2pd 2048(%rdx), %xmm22
+// CHECK:  encoding: [0x62,0xe1,0xfe,0x08,0xe6,0xb2,0x00,0x08,0x00,0x00]
+          vcvtqq2pd 2048(%rdx), %xmm22
+
+// CHECK: vcvtqq2pd -2048(%rdx), %xmm22
+// CHECK:  encoding: [0x62,0xe1,0xfe,0x08,0xe6,0x72,0x80]
+          vcvtqq2pd -2048(%rdx), %xmm22
+
+// CHECK: vcvtqq2pd -2064(%rdx), %xmm22
+// CHECK:  encoding: [0x62,0xe1,0xfe,0x08,0xe6,0xb2,0xf0,0xf7,0xff,0xff]
+          vcvtqq2pd -2064(%rdx), %xmm22
+
+// CHECK: vcvtqq2pd 1016(%rdx){1to2}, %xmm22
+// CHECK:  encoding: [0x62,0xe1,0xfe,0x18,0xe6,0x72,0x7f]
+          vcvtqq2pd 1016(%rdx){1to2}, %xmm22
+
+// CHECK: vcvtqq2pd 1024(%rdx){1to2}, %xmm22
+// CHECK:  encoding: [0x62,0xe1,0xfe,0x18,0xe6,0xb2,0x00,0x04,0x00,0x00]
+          vcvtqq2pd 1024(%rdx){1to2}, %xmm22
+
+// CHECK: vcvtqq2pd -1024(%rdx){1to2}, %xmm22
+// CHECK:  encoding: [0x62,0xe1,0xfe,0x18,0xe6,0x72,0x80]
+          vcvtqq2pd -1024(%rdx){1to2}, %xmm22
+
+// CHECK: vcvtqq2pd -1032(%rdx){1to2}, %xmm22
+// CHECK:  encoding: [0x62,0xe1,0xfe,0x18,0xe6,0xb2,0xf8,0xfb,0xff,0xff]
+          vcvtqq2pd -1032(%rdx){1to2}, %xmm22
+
+// CHECK: vcvtqq2pd %ymm20, %ymm21
+// CHECK:  encoding: [0x62,0xa1,0xfe,0x28,0xe6,0xec]
+          vcvtqq2pd %ymm20, %ymm21
+
+// CHECK: vcvtqq2pd %ymm20, %ymm21 {%k5}
+// CHECK:  encoding: [0x62,0xa1,0xfe,0x2d,0xe6,0xec]
+          vcvtqq2pd %ymm20, %ymm21 {%k5}
+
+// CHECK: vcvtqq2pd %ymm20, %ymm21 {%k5} {z}
+// CHECK:  encoding: [0x62,0xa1,0xfe,0xad,0xe6,0xec]
+          vcvtqq2pd %ymm20, %ymm21 {%k5} {z}
+
+// CHECK: vcvtqq2pd (%rcx), %ymm21
+// CHECK:  encoding: [0x62,0xe1,0xfe,0x28,0xe6,0x29]
+          vcvtqq2pd (%rcx), %ymm21
+
+// CHECK: vcvtqq2pd 291(%rax,%r14,8), %ymm21
+// CHECK:  encoding: [0x62,0xa1,0xfe,0x28,0xe6,0xac,0xf0,0x23,0x01,0x00,0x00]
+          vcvtqq2pd 291(%rax,%r14,8), %ymm21
+
+// CHECK: vcvtqq2pd (%rcx){1to4}, %ymm21
+// CHECK:  encoding: [0x62,0xe1,0xfe,0x38,0xe6,0x29]
+          vcvtqq2pd (%rcx){1to4}, %ymm21
+
+// CHECK: vcvtqq2pd 4064(%rdx), %ymm21
+// CHECK:  encoding: [0x62,0xe1,0xfe,0x28,0xe6,0x6a,0x7f]
+          vcvtqq2pd 4064(%rdx), %ymm21
+
+// CHECK: vcvtqq2pd 4096(%rdx), %ymm21
+// CHECK:  encoding: [0x62,0xe1,0xfe,0x28,0xe6,0xaa,0x00,0x10,0x00,0x00]
+          vcvtqq2pd 4096(%rdx), %ymm21
+
+// CHECK: vcvtqq2pd -4096(%rdx), %ymm21
+// CHECK:  encoding: [0x62,0xe1,0xfe,0x28,0xe6,0x6a,0x80]
+          vcvtqq2pd -4096(%rdx), %ymm21
+
+// CHECK: vcvtqq2pd -4128(%rdx), %ymm21
+// CHECK:  encoding: [0x62,0xe1,0xfe,0x28,0xe6,0xaa,0xe0,0xef,0xff,0xff]
+          vcvtqq2pd -4128(%rdx), %ymm21
+
+// CHECK: vcvtqq2pd 1016(%rdx){1to4}, %ymm21
+// CHECK:  encoding: [0x62,0xe1,0xfe,0x38,0xe6,0x6a,0x7f]
+          vcvtqq2pd 1016(%rdx){1to4}, %ymm21
+
+// CHECK: vcvtqq2pd 1024(%rdx){1to4}, %ymm21
+// CHECK:  encoding: [0x62,0xe1,0xfe,0x38,0xe6,0xaa,0x00,0x04,0x00,0x00]
+          vcvtqq2pd 1024(%rdx){1to4}, %ymm21
+
+// CHECK: vcvtqq2pd -1024(%rdx){1to4}, %ymm21
+// CHECK:  encoding: [0x62,0xe1,0xfe,0x38,0xe6,0x6a,0x80]
+          vcvtqq2pd -1024(%rdx){1to4}, %ymm21
+
+// CHECK: vcvtqq2pd -1032(%rdx){1to4}, %ymm21
+// CHECK:  encoding: [0x62,0xe1,0xfe,0x38,0xe6,0xaa,0xf8,0xfb,0xff,0xff]
+          vcvtqq2pd -1032(%rdx){1to4}, %ymm21
+
+// CHECK: vcvtqq2ps %xmm28, %xmm25
+// CHECK:  encoding: [0x62,0x01,0xfc,0x08,0x5b,0xcc]
+          vcvtqq2ps %xmm28, %xmm25
+
+// CHECK: vcvtqq2ps %xmm28, %xmm25 {%k6}
+// CHECK:  encoding: [0x62,0x01,0xfc,0x0e,0x5b,0xcc]
+          vcvtqq2ps %xmm28, %xmm25 {%k6}
+
+// CHECK: vcvtqq2ps %xmm28, %xmm25 {%k6} {z}
+// CHECK:  encoding: [0x62,0x01,0xfc,0x8e,0x5b,0xcc]
+          vcvtqq2ps %xmm28, %xmm25 {%k6} {z}
+
+// CHECK: vcvtqq2psx (%rcx), %xmm25
+// CHECK:  encoding: [0x62,0x61,0xfc,0x08,0x5b,0x09]
+          vcvtqq2psx (%rcx), %xmm25
+
+// CHECK: vcvtqq2psx 291(%rax,%r14,8), %xmm25
+// CHECK:  encoding: [0x62,0x21,0xfc,0x08,0x5b,0x8c,0xf0,0x23,0x01,0x00,0x00]
+          vcvtqq2psx 291(%rax,%r14,8), %xmm25
+
+// CHECK: vcvtqq2ps (%rcx){1to2}, %xmm25
+// CHECK:  encoding: [0x62,0x61,0xfc,0x18,0x5b,0x09]
+          vcvtqq2ps (%rcx){1to2}, %xmm25
+
+// CHECK: vcvtqq2psx 2032(%rdx), %xmm25
+// CHECK:  encoding: [0x62,0x61,0xfc,0x08,0x5b,0x4a,0x7f]
+          vcvtqq2psx 2032(%rdx), %xmm25
+
+// CHECK: vcvtqq2psx 2048(%rdx), %xmm25
+// CHECK:  encoding: [0x62,0x61,0xfc,0x08,0x5b,0x8a,0x00,0x08,0x00,0x00]
+          vcvtqq2psx 2048(%rdx), %xmm25
+
+// CHECK: vcvtqq2psx -2048(%rdx), %xmm25
+// CHECK:  encoding: [0x62,0x61,0xfc,0x08,0x5b,0x4a,0x80]
+          vcvtqq2psx -2048(%rdx), %xmm25
+
+// CHECK: vcvtqq2psx -2064(%rdx), %xmm25
+// CHECK:  encoding: [0x62,0x61,0xfc,0x08,0x5b,0x8a,0xf0,0xf7,0xff,0xff]
+          vcvtqq2psx -2064(%rdx), %xmm25
+
+// CHECK: vcvtqq2ps 1016(%rdx){1to2}, %xmm25
+// CHECK:  encoding: [0x62,0x61,0xfc,0x18,0x5b,0x4a,0x7f]
+          vcvtqq2ps 1016(%rdx){1to2}, %xmm25
+
+// CHECK: vcvtqq2ps 1024(%rdx){1to2}, %xmm25
+// CHECK:  encoding: [0x62,0x61,0xfc,0x18,0x5b,0x8a,0x00,0x04,0x00,0x00]
+          vcvtqq2ps 1024(%rdx){1to2}, %xmm25
+
+// CHECK: vcvtqq2ps -1024(%rdx){1to2}, %xmm25
+// CHECK:  encoding: [0x62,0x61,0xfc,0x18,0x5b,0x4a,0x80]
+          vcvtqq2ps -1024(%rdx){1to2}, %xmm25
+
+// CHECK: vcvtqq2ps -1032(%rdx){1to2}, %xmm25
+// CHECK:  encoding: [0x62,0x61,0xfc,0x18,0x5b,0x8a,0xf8,0xfb,0xff,0xff]
+          vcvtqq2ps -1032(%rdx){1to2}, %xmm25
+
+// CHECK: vcvtqq2ps %ymm22, %xmm27
+// CHECK:  encoding: [0x62,0x21,0xfc,0x28,0x5b,0xde]
+          vcvtqq2ps %ymm22, %xmm27
+
+// CHECK: vcvtqq2ps %ymm22, %xmm27 {%k3}
+// CHECK:  encoding: [0x62,0x21,0xfc,0x2b,0x5b,0xde]
+          vcvtqq2ps %ymm22, %xmm27 {%k3}
+
+// CHECK: vcvtqq2ps %ymm22, %xmm27 {%k3} {z}
+// CHECK:  encoding: [0x62,0x21,0xfc,0xab,0x5b,0xde]
+          vcvtqq2ps %ymm22, %xmm27 {%k3} {z}
+
+// CHECK: vcvtqq2psy (%rcx), %xmm27
+// CHECK:  encoding: [0x62,0x61,0xfc,0x28,0x5b,0x19]
+          vcvtqq2psy (%rcx), %xmm27
+
+// CHECK: vcvtqq2psy 291(%rax,%r14,8), %xmm27
+// CHECK:  encoding: [0x62,0x21,0xfc,0x28,0x5b,0x9c,0xf0,0x23,0x01,0x00,0x00]
+          vcvtqq2psy 291(%rax,%r14,8), %xmm27
+
+// CHECK: vcvtqq2ps (%rcx){1to4}, %xmm27
+// CHECK:  encoding: [0x62,0x61,0xfc,0x38,0x5b,0x19]
+          vcvtqq2ps (%rcx){1to4}, %xmm27
+
+// CHECK: vcvtqq2psy 4064(%rdx), %xmm27
+// CHECK:  encoding: [0x62,0x61,0xfc,0x28,0x5b,0x5a,0x7f]
+          vcvtqq2psy 4064(%rdx), %xmm27
+
+// CHECK: vcvtqq2psy 4096(%rdx), %xmm27
+// CHECK:  encoding: [0x62,0x61,0xfc,0x28,0x5b,0x9a,0x00,0x10,0x00,0x00]
+          vcvtqq2psy 4096(%rdx), %xmm27
+
+// CHECK: vcvtqq2psy -4096(%rdx), %xmm27
+// CHECK:  encoding: [0x62,0x61,0xfc,0x28,0x5b,0x5a,0x80]
+          vcvtqq2psy -4096(%rdx), %xmm27
+
+// CHECK: vcvtqq2psy -4128(%rdx), %xmm27
+// CHECK:  encoding: [0x62,0x61,0xfc,0x28,0x5b,0x9a,0xe0,0xef,0xff,0xff]
+          vcvtqq2psy -4128(%rdx), %xmm27
+
+// CHECK: vcvtqq2ps 1016(%rdx){1to4}, %xmm27
+// CHECK:  encoding: [0x62,0x61,0xfc,0x38,0x5b,0x5a,0x7f]
+          vcvtqq2ps 1016(%rdx){1to4}, %xmm27
+
+// CHECK: vcvtqq2ps 1024(%rdx){1to4}, %xmm27
+// CHECK:  encoding: [0x62,0x61,0xfc,0x38,0x5b,0x9a,0x00,0x04,0x00,0x00]
+          vcvtqq2ps 1024(%rdx){1to4}, %xmm27
+
+// CHECK: vcvtqq2ps -1024(%rdx){1to4}, %xmm27
+// CHECK:  encoding: [0x62,0x61,0xfc,0x38,0x5b,0x5a,0x80]
+          vcvtqq2ps -1024(%rdx){1to4}, %xmm27
+
+// CHECK: vcvtqq2ps -1032(%rdx){1to4}, %xmm27
+// CHECK:  encoding: [0x62,0x61,0xfc,0x38,0x5b,0x9a,0xf8,0xfb,0xff,0xff]
+          vcvtqq2ps -1032(%rdx){1to4}, %xmm27
+
+// CHECK: vcvtuqq2pd %xmm20, %xmm19
+// CHECK:  encoding: [0x62,0xa1,0xfe,0x08,0x7a,0xdc]
+          vcvtuqq2pd %xmm20, %xmm19
+
+// CHECK: vcvtuqq2pd %xmm20, %xmm19 {%k3}
+// CHECK:  encoding: [0x62,0xa1,0xfe,0x0b,0x7a,0xdc]
+          vcvtuqq2pd %xmm20, %xmm19 {%k3}
+
+// CHECK: vcvtuqq2pd %xmm20, %xmm19 {%k3} {z}
+// CHECK:  encoding: [0x62,0xa1,0xfe,0x8b,0x7a,0xdc]
+          vcvtuqq2pd %xmm20, %xmm19 {%k3} {z}
+
+// CHECK: vcvtuqq2pd (%rcx), %xmm19
+// CHECK:  encoding: [0x62,0xe1,0xfe,0x08,0x7a,0x19]
+          vcvtuqq2pd (%rcx), %xmm19
+
+// CHECK: vcvtuqq2pd 291(%rax,%r14,8), %xmm19
+// CHECK:  encoding: [0x62,0xa1,0xfe,0x08,0x7a,0x9c,0xf0,0x23,0x01,0x00,0x00]
+          vcvtuqq2pd 291(%rax,%r14,8), %xmm19
+
+// CHECK: vcvtuqq2pd (%rcx){1to2}, %xmm19
+// CHECK:  encoding: [0x62,0xe1,0xfe,0x18,0x7a,0x19]
+          vcvtuqq2pd (%rcx){1to2}, %xmm19
+
+// CHECK: vcvtuqq2pd 2032(%rdx), %xmm19
+// CHECK:  encoding: [0x62,0xe1,0xfe,0x08,0x7a,0x5a,0x7f]
+          vcvtuqq2pd 2032(%rdx), %xmm19
+
+// CHECK: vcvtuqq2pd 2048(%rdx), %xmm19
+// CHECK:  encoding: [0x62,0xe1,0xfe,0x08,0x7a,0x9a,0x00,0x08,0x00,0x00]
+          vcvtuqq2pd 2048(%rdx), %xmm19
+
+// CHECK: vcvtuqq2pd -2048(%rdx), %xmm19
+// CHECK:  encoding: [0x62,0xe1,0xfe,0x08,0x7a,0x5a,0x80]
+          vcvtuqq2pd -2048(%rdx), %xmm19
+
+// CHECK: vcvtuqq2pd -2064(%rdx), %xmm19
+// CHECK:  encoding: [0x62,0xe1,0xfe,0x08,0x7a,0x9a,0xf0,0xf7,0xff,0xff]
+          vcvtuqq2pd -2064(%rdx), %xmm19
+
+// CHECK: vcvtuqq2pd 1016(%rdx){1to2}, %xmm19
+// CHECK:  encoding: [0x62,0xe1,0xfe,0x18,0x7a,0x5a,0x7f]
+          vcvtuqq2pd 1016(%rdx){1to2}, %xmm19
+
+// CHECK: vcvtuqq2pd 1024(%rdx){1to2}, %xmm19
+// CHECK:  encoding: [0x62,0xe1,0xfe,0x18,0x7a,0x9a,0x00,0x04,0x00,0x00]
+          vcvtuqq2pd 1024(%rdx){1to2}, %xmm19
+
+// CHECK: vcvtuqq2pd -1024(%rdx){1to2}, %xmm19
+// CHECK:  encoding: [0x62,0xe1,0xfe,0x18,0x7a,0x5a,0x80]
+          vcvtuqq2pd -1024(%rdx){1to2}, %xmm19
+
+// CHECK: vcvtuqq2pd -1032(%rdx){1to2}, %xmm19
+// CHECK:  encoding: [0x62,0xe1,0xfe,0x18,0x7a,0x9a,0xf8,0xfb,0xff,0xff]
+          vcvtuqq2pd -1032(%rdx){1to2}, %xmm19
+
+// CHECK: vcvtuqq2pd %ymm26, %ymm28
+// CHECK:  encoding: [0x62,0x01,0xfe,0x28,0x7a,0xe2]
+          vcvtuqq2pd %ymm26, %ymm28
+
+// CHECK: vcvtuqq2pd %ymm26, %ymm28 {%k4}
+// CHECK:  encoding: [0x62,0x01,0xfe,0x2c,0x7a,0xe2]
+          vcvtuqq2pd %ymm26, %ymm28 {%k4}
+
+// CHECK: vcvtuqq2pd %ymm26, %ymm28 {%k4} {z}
+// CHECK:  encoding: [0x62,0x01,0xfe,0xac,0x7a,0xe2]
+          vcvtuqq2pd %ymm26, %ymm28 {%k4} {z}
+
+// CHECK: vcvtuqq2pd (%rcx), %ymm28
+// CHECK:  encoding: [0x62,0x61,0xfe,0x28,0x7a,0x21]
+          vcvtuqq2pd (%rcx), %ymm28
+
+// CHECK: vcvtuqq2pd 291(%rax,%r14,8), %ymm28
+// CHECK:  encoding: [0x62,0x21,0xfe,0x28,0x7a,0xa4,0xf0,0x23,0x01,0x00,0x00]
+          vcvtuqq2pd 291(%rax,%r14,8), %ymm28
+
+// CHECK: vcvtuqq2pd (%rcx){1to4}, %ymm28
+// CHECK:  encoding: [0x62,0x61,0xfe,0x38,0x7a,0x21]
+          vcvtuqq2pd (%rcx){1to4}, %ymm28
+
+// CHECK: vcvtuqq2pd 4064(%rdx), %ymm28
+// CHECK:  encoding: [0x62,0x61,0xfe,0x28,0x7a,0x62,0x7f]
+          vcvtuqq2pd 4064(%rdx), %ymm28
+
+// CHECK: vcvtuqq2pd 4096(%rdx), %ymm28
+// CHECK:  encoding: [0x62,0x61,0xfe,0x28,0x7a,0xa2,0x00,0x10,0x00,0x00]
+          vcvtuqq2pd 4096(%rdx), %ymm28
+
+// CHECK: vcvtuqq2pd -4096(%rdx), %ymm28
+// CHECK:  encoding: [0x62,0x61,0xfe,0x28,0x7a,0x62,0x80]
+          vcvtuqq2pd -4096(%rdx), %ymm28
+
+// CHECK: vcvtuqq2pd -4128(%rdx), %ymm28
+// CHECK:  encoding: [0x62,0x61,0xfe,0x28,0x7a,0xa2,0xe0,0xef,0xff,0xff]
+          vcvtuqq2pd -4128(%rdx), %ymm28
+
+// CHECK: vcvtuqq2pd 1016(%rdx){1to4}, %ymm28
+// CHECK:  encoding: [0x62,0x61,0xfe,0x38,0x7a,0x62,0x7f]
+          vcvtuqq2pd 1016(%rdx){1to4}, %ymm28
+
+// CHECK: vcvtuqq2pd 1024(%rdx){1to4}, %ymm28
+// CHECK:  encoding: [0x62,0x61,0xfe,0x38,0x7a,0xa2,0x00,0x04,0x00,0x00]
+          vcvtuqq2pd 1024(%rdx){1to4}, %ymm28
+
+// CHECK: vcvtuqq2pd -1024(%rdx){1to4}, %ymm28
+// CHECK:  encoding: [0x62,0x61,0xfe,0x38,0x7a,0x62,0x80]
+          vcvtuqq2pd -1024(%rdx){1to4}, %ymm28
+
+// CHECK: vcvtuqq2pd -1032(%rdx){1to4}, %ymm28
+// CHECK:  encoding: [0x62,0x61,0xfe,0x38,0x7a,0xa2,0xf8,0xfb,0xff,0xff]
+          vcvtuqq2pd -1032(%rdx){1to4}, %ymm28
+
+// CHECK: vcvtuqq2ps %xmm27, %xmm21
+// CHECK:  encoding: [0x62,0x81,0xff,0x08,0x7a,0xeb]
+          vcvtuqq2ps %xmm27, %xmm21
+
+// CHECK: vcvtuqq2ps %xmm27, %xmm21 {%k7}
+// CHECK:  encoding: [0x62,0x81,0xff,0x0f,0x7a,0xeb]
+          vcvtuqq2ps %xmm27, %xmm21 {%k7}
+
+// CHECK: vcvtuqq2ps %xmm27, %xmm21 {%k7} {z}
+// CHECK:  encoding: [0x62,0x81,0xff,0x8f,0x7a,0xeb]
+          vcvtuqq2ps %xmm27, %xmm21 {%k7} {z}
+
+// CHECK: vcvtuqq2psx (%rcx), %xmm21
+// CHECK:  encoding: [0x62,0xe1,0xff,0x08,0x7a,0x29]
+          vcvtuqq2psx (%rcx), %xmm21
+
+// CHECK: vcvtuqq2psx 291(%rax,%r14,8), %xmm21
+// CHECK:  encoding: [0x62,0xa1,0xff,0x08,0x7a,0xac,0xf0,0x23,0x01,0x00,0x00]
+          vcvtuqq2psx 291(%rax,%r14,8), %xmm21
+
+// CHECK: vcvtuqq2ps (%rcx){1to2}, %xmm21
+// CHECK:  encoding: [0x62,0xe1,0xff,0x18,0x7a,0x29]
+          vcvtuqq2ps (%rcx){1to2}, %xmm21
+
+// CHECK: vcvtuqq2psx 2032(%rdx), %xmm21
+// CHECK:  encoding: [0x62,0xe1,0xff,0x08,0x7a,0x6a,0x7f]
+          vcvtuqq2psx 2032(%rdx), %xmm21
+
+// CHECK: vcvtuqq2psx 2048(%rdx), %xmm21
+// CHECK:  encoding: [0x62,0xe1,0xff,0x08,0x7a,0xaa,0x00,0x08,0x00,0x00]
+          vcvtuqq2psx 2048(%rdx), %xmm21
+
+// CHECK: vcvtuqq2psx -2048(%rdx), %xmm21
+// CHECK:  encoding: [0x62,0xe1,0xff,0x08,0x7a,0x6a,0x80]
+          vcvtuqq2psx -2048(%rdx), %xmm21
+
+// CHECK: vcvtuqq2psx -2064(%rdx), %xmm21
+// CHECK:  encoding: [0x62,0xe1,0xff,0x08,0x7a,0xaa,0xf0,0xf7,0xff,0xff]
+          vcvtuqq2psx -2064(%rdx), %xmm21
+
+// CHECK: vcvtuqq2ps 1016(%rdx){1to2}, %xmm21
+// CHECK:  encoding: [0x62,0xe1,0xff,0x18,0x7a,0x6a,0x7f]
+          vcvtuqq2ps 1016(%rdx){1to2}, %xmm21
+
+// CHECK: vcvtuqq2ps 1024(%rdx){1to2}, %xmm21
+// CHECK:  encoding: [0x62,0xe1,0xff,0x18,0x7a,0xaa,0x00,0x04,0x00,0x00]
+          vcvtuqq2ps 1024(%rdx){1to2}, %xmm21
+
+// CHECK: vcvtuqq2ps -1024(%rdx){1to2}, %xmm21
+// CHECK:  encoding: [0x62,0xe1,0xff,0x18,0x7a,0x6a,0x80]
+          vcvtuqq2ps -1024(%rdx){1to2}, %xmm21
+
+// CHECK: vcvtuqq2ps -1032(%rdx){1to2}, %xmm21
+// CHECK:  encoding: [0x62,0xe1,0xff,0x18,0x7a,0xaa,0xf8,0xfb,0xff,0xff]
+          vcvtuqq2ps -1032(%rdx){1to2}, %xmm21
+
+// CHECK: vcvtuqq2ps %ymm24, %xmm28
+// CHECK:  encoding: [0x62,0x01,0xff,0x28,0x7a,0xe0]
+          vcvtuqq2ps %ymm24, %xmm28
+
+// CHECK: vcvtuqq2ps %ymm24, %xmm28 {%k3}
+// CHECK:  encoding: [0x62,0x01,0xff,0x2b,0x7a,0xe0]
+          vcvtuqq2ps %ymm24, %xmm28 {%k3}
+
+// CHECK: vcvtuqq2ps %ymm24, %xmm28 {%k3} {z}
+// CHECK:  encoding: [0x62,0x01,0xff,0xab,0x7a,0xe0]
+          vcvtuqq2ps %ymm24, %xmm28 {%k3} {z}
+
+// CHECK: vcvtuqq2psy (%rcx), %xmm28
+// CHECK:  encoding: [0x62,0x61,0xff,0x28,0x7a,0x21]
+          vcvtuqq2psy (%rcx), %xmm28
+
+// CHECK: vcvtuqq2psy 291(%rax,%r14,8), %xmm28
+// CHECK:  encoding: [0x62,0x21,0xff,0x28,0x7a,0xa4,0xf0,0x23,0x01,0x00,0x00]
+          vcvtuqq2psy 291(%rax,%r14,8), %xmm28
+
+// CHECK: vcvtuqq2ps (%rcx){1to4}, %xmm28
+// CHECK:  encoding: [0x62,0x61,0xff,0x38,0x7a,0x21]
+          vcvtuqq2ps (%rcx){1to4}, %xmm28
+
+// CHECK: vcvtuqq2psy 4064(%rdx), %xmm28
+// CHECK:  encoding: [0x62,0x61,0xff,0x28,0x7a,0x62,0x7f]
+          vcvtuqq2psy 4064(%rdx), %xmm28
+
+// CHECK: vcvtuqq2psy 4096(%rdx), %xmm28
+// CHECK:  encoding: [0x62,0x61,0xff,0x28,0x7a,0xa2,0x00,0x10,0x00,0x00]
+          vcvtuqq2psy 4096(%rdx), %xmm28
+
+// CHECK: vcvtuqq2psy -4096(%rdx), %xmm28
+// CHECK:  encoding: [0x62,0x61,0xff,0x28,0x7a,0x62,0x80]
+          vcvtuqq2psy -4096(%rdx), %xmm28
+
+// CHECK: vcvtuqq2psy -4128(%rdx), %xmm28
+// CHECK:  encoding: [0x62,0x61,0xff,0x28,0x7a,0xa2,0xe0,0xef,0xff,0xff]
+          vcvtuqq2psy -4128(%rdx), %xmm28
+
+// CHECK: vcvtuqq2ps 1016(%rdx){1to4}, %xmm28
+// CHECK:  encoding: [0x62,0x61,0xff,0x38,0x7a,0x62,0x7f]
+          vcvtuqq2ps 1016(%rdx){1to4}, %xmm28
+
+// CHECK: vcvtuqq2ps 1024(%rdx){1to4}, %xmm28
+// CHECK:  encoding: [0x62,0x61,0xff,0x38,0x7a,0xa2,0x00,0x04,0x00,0x00]
+          vcvtuqq2ps 1024(%rdx){1to4}, %xmm28
+
+// CHECK: vcvtuqq2ps -1024(%rdx){1to4}, %xmm28
+// CHECK:  encoding: [0x62,0x61,0xff,0x38,0x7a,0x62,0x80]
+          vcvtuqq2ps -1024(%rdx){1to4}, %xmm28
+
+// CHECK: vcvtuqq2ps -1032(%rdx){1to4}, %xmm28
+// CHECK:  encoding: [0x62,0x61,0xff,0x38,0x7a,0xa2,0xf8,0xfb,0xff,0xff]
+          vcvtuqq2ps -1032(%rdx){1to4}, %xmm28
+
diff --git a/test/MC/X86/x86-64-avx512f_vl.s b/test/MC/X86/x86-64-avx512f_vl.s
index 09386b0bd57..564af897274 100644
--- a/test/MC/X86/x86-64-avx512f_vl.s
+++ b/test/MC/X86/x86-64-avx512f_vl.s
@@ -4524,6 +4524,726 @@
 // CHECK:  encoding: [0x62,0x62,0xd5,0x30,0x3b,0xaa,0xf8,0xfb,0xff,0xff]
           vpminuq -1032(%rdx){1to4}, %ymm21, %ymm29
 
+// CHECK: vpmovsxbd %xmm28, %xmm24
+// CHECK:  encoding: [0x62,0x02,0x7d,0x08,0x21,0xc4]
+          vpmovsxbd %xmm28, %xmm24
+
+// CHECK: vpmovsxbd %xmm28, %xmm24 {%k1}
+// CHECK:  encoding: [0x62,0x02,0x7d,0x09,0x21,0xc4]
+          vpmovsxbd %xmm28, %xmm24 {%k1}
+
+// CHECK: vpmovsxbd %xmm28, %xmm24 {%k1} {z}
+// CHECK:  encoding: [0x62,0x02,0x7d,0x89,0x21,0xc4]
+          vpmovsxbd %xmm28, %xmm24 {%k1} {z}
+
+// CHECK: vpmovsxbd (%rcx), %xmm24
+// CHECK:  encoding: [0x62,0x62,0x7d,0x08,0x21,0x01]
+          vpmovsxbd (%rcx), %xmm24
+
+// CHECK: vpmovsxbd 291(%rax,%r14,8), %xmm24
+// CHECK:  encoding: [0x62,0x22,0x7d,0x08,0x21,0x84,0xf0,0x23,0x01,0x00,0x00]
+          vpmovsxbd 291(%rax,%r14,8), %xmm24
+
+// CHECK: vpmovsxbd 508(%rdx), %xmm24
+// CHECK:  encoding: [0x62,0x62,0x7d,0x08,0x21,0x42,0x7f]
+          vpmovsxbd 508(%rdx), %xmm24
+
+// CHECK: vpmovsxbd 512(%rdx), %xmm24
+// CHECK:  encoding: [0x62,0x62,0x7d,0x08,0x21,0x82,0x00,0x02,0x00,0x00]
+          vpmovsxbd 512(%rdx), %xmm24
+
+// CHECK: vpmovsxbd -512(%rdx), %xmm24
+// CHECK:  encoding: [0x62,0x62,0x7d,0x08,0x21,0x42,0x80]
+          vpmovsxbd -512(%rdx), %xmm24
+
+// CHECK: vpmovsxbd -516(%rdx), %xmm24
+// CHECK:  encoding: [0x62,0x62,0x7d,0x08,0x21,0x82,0xfc,0xfd,0xff,0xff]
+          vpmovsxbd -516(%rdx), %xmm24
+
+// CHECK: vpmovsxbd %xmm20, %ymm24
+// CHECK:  encoding: [0x62,0x22,0x7d,0x28,0x21,0xc4]
+          vpmovsxbd %xmm20, %ymm24
+
+// CHECK: vpmovsxbd %xmm20, %ymm24 {%k3}
+// CHECK:  encoding: [0x62,0x22,0x7d,0x2b,0x21,0xc4]
+          vpmovsxbd %xmm20, %ymm24 {%k3}
+
+// CHECK: vpmovsxbd %xmm20, %ymm24 {%k3} {z}
+// CHECK:  encoding: [0x62,0x22,0x7d,0xab,0x21,0xc4]
+          vpmovsxbd %xmm20, %ymm24 {%k3} {z}
+
+// CHECK: vpmovsxbd (%rcx), %ymm24
+// CHECK:  encoding: [0x62,0x62,0x7d,0x28,0x21,0x01]
+          vpmovsxbd (%rcx), %ymm24
+
+// CHECK: vpmovsxbd 291(%rax,%r14,8), %ymm24
+// CHECK:  encoding: [0x62,0x22,0x7d,0x28,0x21,0x84,0xf0,0x23,0x01,0x00,0x00]
+          vpmovsxbd 291(%rax,%r14,8), %ymm24
+
+// CHECK: vpmovsxbd 1016(%rdx), %ymm24
+// CHECK:  encoding: [0x62,0x62,0x7d,0x28,0x21,0x42,0x7f]
+          vpmovsxbd 1016(%rdx), %ymm24
+
+// CHECK: vpmovsxbd 1024(%rdx), %ymm24
+// CHECK:  encoding: [0x62,0x62,0x7d,0x28,0x21,0x82,0x00,0x04,0x00,0x00]
+          vpmovsxbd 1024(%rdx), %ymm24
+
+// CHECK: vpmovsxbd -1024(%rdx), %ymm24
+// CHECK:  encoding: [0x62,0x62,0x7d,0x28,0x21,0x42,0x80]
+          vpmovsxbd -1024(%rdx), %ymm24
+
+// CHECK: vpmovsxbd -1032(%rdx), %ymm24
+// CHECK:  encoding: [0x62,0x62,0x7d,0x28,0x21,0x82,0xf8,0xfb,0xff,0xff]
+          vpmovsxbd -1032(%rdx), %ymm24
+
+// CHECK: vpmovsxbq %xmm22, %xmm17
+// CHECK:  encoding: [0x62,0xa2,0x7d,0x08,0x22,0xce]
+          vpmovsxbq %xmm22, %xmm17
+
+// CHECK: vpmovsxbq %xmm22, %xmm17 {%k5}
+// CHECK:  encoding: [0x62,0xa2,0x7d,0x0d,0x22,0xce]
+          vpmovsxbq %xmm22, %xmm17 {%k5}
+
+// CHECK: vpmovsxbq %xmm22, %xmm17 {%k5} {z}
+// CHECK:  encoding: [0x62,0xa2,0x7d,0x8d,0x22,0xce]
+          vpmovsxbq %xmm22, %xmm17 {%k5} {z}
+
+// CHECK: vpmovsxbq (%rcx), %xmm17
+// CHECK:  encoding: [0x62,0xe2,0x7d,0x08,0x22,0x09]
+          vpmovsxbq (%rcx), %xmm17
+
+// CHECK: vpmovsxbq 291(%rax,%r14,8), %xmm17
+// CHECK:  encoding: [0x62,0xa2,0x7d,0x08,0x22,0x8c,0xf0,0x23,0x01,0x00,0x00]
+          vpmovsxbq 291(%rax,%r14,8), %xmm17
+
+// CHECK: vpmovsxbq 254(%rdx), %xmm17
+// CHECK:  encoding: [0x62,0xe2,0x7d,0x08,0x22,0x4a,0x7f]
+          vpmovsxbq 254(%rdx), %xmm17
+
+// CHECK: vpmovsxbq 256(%rdx), %xmm17
+// CHECK:  encoding: [0x62,0xe2,0x7d,0x08,0x22,0x8a,0x00,0x01,0x00,0x00]
+          vpmovsxbq 256(%rdx), %xmm17
+
+// CHECK: vpmovsxbq -256(%rdx), %xmm17
+// CHECK:  encoding: [0x62,0xe2,0x7d,0x08,0x22,0x4a,0x80]
+          vpmovsxbq -256(%rdx), %xmm17
+
+// CHECK: vpmovsxbq -258(%rdx), %xmm17
+// CHECK:  encoding: [0x62,0xe2,0x7d,0x08,0x22,0x8a,0xfe,0xfe,0xff,0xff]
+          vpmovsxbq -258(%rdx), %xmm17
+
+// CHECK: vpmovsxbq %xmm26, %ymm28
+// CHECK:  encoding: [0x62,0x02,0x7d,0x28,0x22,0xe2]
+          vpmovsxbq %xmm26, %ymm28
+
+// CHECK: vpmovsxbq %xmm26, %ymm28 {%k5}
+// CHECK:  encoding: [0x62,0x02,0x7d,0x2d,0x22,0xe2]
+          vpmovsxbq %xmm26, %ymm28 {%k5}
+
+// CHECK: vpmovsxbq %xmm26, %ymm28 {%k5} {z}
+// CHECK:  encoding: [0x62,0x02,0x7d,0xad,0x22,0xe2]
+          vpmovsxbq %xmm26, %ymm28 {%k5} {z}
+
+// CHECK: vpmovsxbq (%rcx), %ymm28
+// CHECK:  encoding: [0x62,0x62,0x7d,0x28,0x22,0x21]
+          vpmovsxbq (%rcx), %ymm28
+
+// CHECK: vpmovsxbq 291(%rax,%r14,8), %ymm28
+// CHECK:  encoding: [0x62,0x22,0x7d,0x28,0x22,0xa4,0xf0,0x23,0x01,0x00,0x00]
+          vpmovsxbq 291(%rax,%r14,8), %ymm28
+
+// CHECK: vpmovsxbq 508(%rdx), %ymm28
+// CHECK:  encoding: [0x62,0x62,0x7d,0x28,0x22,0x62,0x7f]
+          vpmovsxbq 508(%rdx), %ymm28
+
+// CHECK: vpmovsxbq 512(%rdx), %ymm28
+// CHECK:  encoding: [0x62,0x62,0x7d,0x28,0x22,0xa2,0x00,0x02,0x00,0x00]
+          vpmovsxbq 512(%rdx), %ymm28
+
+// CHECK: vpmovsxbq -512(%rdx), %ymm28
+// CHECK:  encoding: [0x62,0x62,0x7d,0x28,0x22,0x62,0x80]
+          vpmovsxbq -512(%rdx), %ymm28
+
+// CHECK: vpmovsxbq -516(%rdx), %ymm28
+// CHECK:  encoding: [0x62,0x62,0x7d,0x28,0x22,0xa2,0xfc,0xfd,0xff,0xff]
+          vpmovsxbq -516(%rdx), %ymm28
+
+// CHECK: vpmovsxdq %xmm26, %xmm23
+// CHECK:  encoding: [0x62,0x82,0x7d,0x08,0x25,0xfa]
+          vpmovsxdq %xmm26, %xmm23
+
+// CHECK: vpmovsxdq %xmm26, %xmm23 {%k7}
+// CHECK:  encoding: [0x62,0x82,0x7d,0x0f,0x25,0xfa]
+          vpmovsxdq %xmm26, %xmm23 {%k7}
+
+// CHECK: vpmovsxdq %xmm26, %xmm23 {%k7} {z}
+// CHECK:  encoding: [0x62,0x82,0x7d,0x8f,0x25,0xfa]
+          vpmovsxdq %xmm26, %xmm23 {%k7} {z}
+
+// CHECK: vpmovsxdq (%rcx), %xmm23
+// CHECK:  encoding: [0x62,0xe2,0x7d,0x08,0x25,0x39]
+          vpmovsxdq (%rcx), %xmm23
+
+// CHECK: vpmovsxdq 291(%rax,%r14,8), %xmm23
+// CHECK:  encoding: [0x62,0xa2,0x7d,0x08,0x25,0xbc,0xf0,0x23,0x01,0x00,0x00]
+          vpmovsxdq 291(%rax,%r14,8), %xmm23
+
+// CHECK: vpmovsxdq 1016(%rdx), %xmm23
+// CHECK:  encoding: [0x62,0xe2,0x7d,0x08,0x25,0x7a,0x7f]
+          vpmovsxdq 1016(%rdx), %xmm23
+
+// CHECK: vpmovsxdq 1024(%rdx), %xmm23
+// CHECK:  encoding: [0x62,0xe2,0x7d,0x08,0x25,0xba,0x00,0x04,0x00,0x00]
+          vpmovsxdq 1024(%rdx), %xmm23
+
+// CHECK: vpmovsxdq -1024(%rdx), %xmm23
+// CHECK:  encoding: [0x62,0xe2,0x7d,0x08,0x25,0x7a,0x80]
+          vpmovsxdq -1024(%rdx), %xmm23
+
+// CHECK: vpmovsxdq -1032(%rdx), %xmm23
+// CHECK:  encoding: [0x62,0xe2,0x7d,0x08,0x25,0xba,0xf8,0xfb,0xff,0xff]
+          vpmovsxdq -1032(%rdx), %xmm23
+
+// CHECK: vpmovsxdq %xmm28, %ymm18
+// CHECK:  encoding: [0x62,0x82,0x7d,0x28,0x25,0xd4]
+          vpmovsxdq %xmm28, %ymm18
+
+// CHECK: vpmovsxdq %xmm28, %ymm18 {%k7}
+// CHECK:  encoding: [0x62,0x82,0x7d,0x2f,0x25,0xd4]
+          vpmovsxdq %xmm28, %ymm18 {%k7}
+
+// CHECK: vpmovsxdq %xmm28, %ymm18 {%k7} {z}
+// CHECK:  encoding: [0x62,0x82,0x7d,0xaf,0x25,0xd4]
+          vpmovsxdq %xmm28, %ymm18 {%k7} {z}
+
+// CHECK: vpmovsxdq (%rcx), %ymm18
+// CHECK:  encoding: [0x62,0xe2,0x7d,0x28,0x25,0x11]
+          vpmovsxdq (%rcx), %ymm18
+
+// CHECK: vpmovsxdq 291(%rax,%r14,8), %ymm18
+// CHECK:  encoding: [0x62,0xa2,0x7d,0x28,0x25,0x94,0xf0,0x23,0x01,0x00,0x00]
+          vpmovsxdq 291(%rax,%r14,8), %ymm18
+
+// CHECK: vpmovsxdq 2032(%rdx), %ymm18
+// CHECK:  encoding: [0x62,0xe2,0x7d,0x28,0x25,0x52,0x7f]
+          vpmovsxdq 2032(%rdx), %ymm18
+
+// CHECK: vpmovsxdq 2048(%rdx), %ymm18
+// CHECK:  encoding: [0x62,0xe2,0x7d,0x28,0x25,0x92,0x00,0x08,0x00,0x00]
+          vpmovsxdq 2048(%rdx), %ymm18
+
+// CHECK: vpmovsxdq -2048(%rdx), %ymm18
+// CHECK:  encoding: [0x62,0xe2,0x7d,0x28,0x25,0x52,0x80]
+          vpmovsxdq -2048(%rdx), %ymm18
+
+// CHECK: vpmovsxdq -2064(%rdx), %ymm18
+// CHECK:  encoding: [0x62,0xe2,0x7d,0x28,0x25,0x92,0xf0,0xf7,0xff,0xff]
+          vpmovsxdq -2064(%rdx), %ymm18
+
+// CHECK: vpmovsxwd %xmm18, %xmm17
+// CHECK:  encoding: [0x62,0xa2,0x7d,0x08,0x23,0xca]
+          vpmovsxwd %xmm18, %xmm17
+
+// CHECK: vpmovsxwd %xmm18, %xmm17 {%k4}
+// CHECK:  encoding: [0x62,0xa2,0x7d,0x0c,0x23,0xca]
+          vpmovsxwd %xmm18, %xmm17 {%k4}
+
+// CHECK: vpmovsxwd %xmm18, %xmm17 {%k4} {z}
+// CHECK:  encoding: [0x62,0xa2,0x7d,0x8c,0x23,0xca]
+          vpmovsxwd %xmm18, %xmm17 {%k4} {z}
+
+// CHECK: vpmovsxwd (%rcx), %xmm17
+// CHECK:  encoding: [0x62,0xe2,0x7d,0x08,0x23,0x09]
+          vpmovsxwd (%rcx), %xmm17
+
+// CHECK: vpmovsxwd 291(%rax,%r14,8), %xmm17
+// CHECK:  encoding: [0x62,0xa2,0x7d,0x08,0x23,0x8c,0xf0,0x23,0x01,0x00,0x00]
+          vpmovsxwd 291(%rax,%r14,8), %xmm17
+
+// CHECK: vpmovsxwd 1016(%rdx), %xmm17
+// CHECK:  encoding: [0x62,0xe2,0x7d,0x08,0x23,0x4a,0x7f]
+          vpmovsxwd 1016(%rdx), %xmm17
+
+// CHECK: vpmovsxwd 1024(%rdx), %xmm17
+// CHECK:  encoding: [0x62,0xe2,0x7d,0x08,0x23,0x8a,0x00,0x04,0x00,0x00]
+          vpmovsxwd 1024(%rdx), %xmm17
+
+// CHECK: vpmovsxwd -1024(%rdx), %xmm17
+// CHECK:  encoding: [0x62,0xe2,0x7d,0x08,0x23,0x4a,0x80]
+          vpmovsxwd -1024(%rdx), %xmm17
+
+// CHECK: vpmovsxwd -1032(%rdx), %xmm17
+// CHECK:  encoding: [0x62,0xe2,0x7d,0x08,0x23,0x8a,0xf8,0xfb,0xff,0xff]
+          vpmovsxwd -1032(%rdx), %xmm17
+
+// CHECK: vpmovsxwd %xmm25, %ymm21
+// CHECK:  encoding: [0x62,0x82,0x7d,0x28,0x23,0xe9]
+          vpmovsxwd %xmm25, %ymm21
+
+// CHECK: vpmovsxwd %xmm25, %ymm21 {%k5}
+// CHECK:  encoding: [0x62,0x82,0x7d,0x2d,0x23,0xe9]
+          vpmovsxwd %xmm25, %ymm21 {%k5}
+
+// CHECK: vpmovsxwd %xmm25, %ymm21 {%k5} {z}
+// CHECK:  encoding: [0x62,0x82,0x7d,0xad,0x23,0xe9]
+          vpmovsxwd %xmm25, %ymm21 {%k5} {z}
+
+// CHECK: vpmovsxwd (%rcx), %ymm21
+// CHECK:  encoding: [0x62,0xe2,0x7d,0x28,0x23,0x29]
+          vpmovsxwd (%rcx), %ymm21
+
+// CHECK: vpmovsxwd 291(%rax,%r14,8), %ymm21
+// CHECK:  encoding: [0x62,0xa2,0x7d,0x28,0x23,0xac,0xf0,0x23,0x01,0x00,0x00]
+          vpmovsxwd 291(%rax,%r14,8), %ymm21
+
+// CHECK: vpmovsxwd 2032(%rdx), %ymm21
+// CHECK:  encoding: [0x62,0xe2,0x7d,0x28,0x23,0x6a,0x7f]
+          vpmovsxwd 2032(%rdx), %ymm21
+
+// CHECK: vpmovsxwd 2048(%rdx), %ymm21
+// CHECK:  encoding: [0x62,0xe2,0x7d,0x28,0x23,0xaa,0x00,0x08,0x00,0x00]
+          vpmovsxwd 2048(%rdx), %ymm21
+
+// CHECK: vpmovsxwd -2048(%rdx), %ymm21
+// CHECK:  encoding: [0x62,0xe2,0x7d,0x28,0x23,0x6a,0x80]
+          vpmovsxwd -2048(%rdx), %ymm21
+
+// CHECK: vpmovsxwd -2064(%rdx), %ymm21
+// CHECK:  encoding: [0x62,0xe2,0x7d,0x28,0x23,0xaa,0xf0,0xf7,0xff,0xff]
+          vpmovsxwd -2064(%rdx), %ymm21
+
+// CHECK: vpmovsxwq %xmm20, %xmm29
+// CHECK:  encoding: [0x62,0x22,0x7d,0x08,0x24,0xec]
+          vpmovsxwq %xmm20, %xmm29
+
+// CHECK: vpmovsxwq %xmm20, %xmm29 {%k6}
+// CHECK:  encoding: [0x62,0x22,0x7d,0x0e,0x24,0xec]
+          vpmovsxwq %xmm20, %xmm29 {%k6}
+
+// CHECK: vpmovsxwq %xmm20, %xmm29 {%k6} {z}
+// CHECK:  encoding: [0x62,0x22,0x7d,0x8e,0x24,0xec]
+          vpmovsxwq %xmm20, %xmm29 {%k6} {z}
+
+// CHECK: vpmovsxwq (%rcx), %xmm29
+// CHECK:  encoding: [0x62,0x62,0x7d,0x08,0x24,0x29]
+          vpmovsxwq (%rcx), %xmm29
+
+// CHECK: vpmovsxwq 291(%rax,%r14,8), %xmm29
+// CHECK:  encoding: [0x62,0x22,0x7d,0x08,0x24,0xac,0xf0,0x23,0x01,0x00,0x00]
+          vpmovsxwq 291(%rax,%r14,8), %xmm29
+
+// CHECK: vpmovsxwq 508(%rdx), %xmm29
+// CHECK:  encoding: [0x62,0x62,0x7d,0x08,0x24,0x6a,0x7f]
+          vpmovsxwq 508(%rdx), %xmm29
+
+// CHECK: vpmovsxwq 512(%rdx), %xmm29
+// CHECK:  encoding: [0x62,0x62,0x7d,0x08,0x24,0xaa,0x00,0x02,0x00,0x00]
+          vpmovsxwq 512(%rdx), %xmm29
+
+// CHECK: vpmovsxwq -512(%rdx), %xmm29
+// CHECK:  encoding: [0x62,0x62,0x7d,0x08,0x24,0x6a,0x80]
+          vpmovsxwq -512(%rdx), %xmm29
+
+// CHECK: vpmovsxwq -516(%rdx), %xmm29
+// CHECK:  encoding: [0x62,0x62,0x7d,0x08,0x24,0xaa,0xfc,0xfd,0xff,0xff]
+          vpmovsxwq -516(%rdx), %xmm29
+
+// CHECK: vpmovsxwq %xmm17, %ymm23
+// CHECK:  encoding: [0x62,0xa2,0x7d,0x28,0x24,0xf9]
+          vpmovsxwq %xmm17, %ymm23
+
+// CHECK: vpmovsxwq %xmm17, %ymm23 {%k5}
+// CHECK:  encoding: [0x62,0xa2,0x7d,0x2d,0x24,0xf9]
+          vpmovsxwq %xmm17, %ymm23 {%k5}
+
+// CHECK: vpmovsxwq %xmm17, %ymm23 {%k5} {z}
+// CHECK:  encoding: [0x62,0xa2,0x7d,0xad,0x24,0xf9]
+          vpmovsxwq %xmm17, %ymm23 {%k5} {z}
+
+// CHECK: vpmovsxwq (%rcx), %ymm23
+// CHECK:  encoding: [0x62,0xe2,0x7d,0x28,0x24,0x39]
+          vpmovsxwq (%rcx), %ymm23
+
+// CHECK: vpmovsxwq 291(%rax,%r14,8), %ymm23
+// CHECK:  encoding: [0x62,0xa2,0x7d,0x28,0x24,0xbc,0xf0,0x23,0x01,0x00,0x00]
+          vpmovsxwq 291(%rax,%r14,8), %ymm23
+
+// CHECK: vpmovsxwq 1016(%rdx), %ymm23
+// CHECK:  encoding: [0x62,0xe2,0x7d,0x28,0x24,0x7a,0x7f]
+          vpmovsxwq 1016(%rdx), %ymm23
+
+// CHECK: vpmovsxwq 1024(%rdx), %ymm23
+// CHECK:  encoding: [0x62,0xe2,0x7d,0x28,0x24,0xba,0x00,0x04,0x00,0x00]
+          vpmovsxwq 1024(%rdx), %ymm23
+
+// CHECK: vpmovsxwq -1024(%rdx), %ymm23
+// CHECK:  encoding: [0x62,0xe2,0x7d,0x28,0x24,0x7a,0x80]
+          vpmovsxwq -1024(%rdx), %ymm23
+
+// CHECK: vpmovsxwq -1032(%rdx), %ymm23
+// CHECK:  encoding: [0x62,0xe2,0x7d,0x28,0x24,0xba,0xf8,0xfb,0xff,0xff]
+          vpmovsxwq -1032(%rdx), %ymm23
+
+// CHECK: vpmovzxbd %xmm17, %xmm24
+// CHECK:  encoding: [0x62,0x22,0x7d,0x08,0x31,0xc1]
+          vpmovzxbd %xmm17, %xmm24
+
+// CHECK: vpmovzxbd %xmm17, %xmm24 {%k6}
+// CHECK:  encoding: [0x62,0x22,0x7d,0x0e,0x31,0xc1]
+          vpmovzxbd %xmm17, %xmm24 {%k6}
+
+// CHECK: vpmovzxbd %xmm17, %xmm24 {%k6} {z}
+// CHECK:  encoding: [0x62,0x22,0x7d,0x8e,0x31,0xc1]
+          vpmovzxbd %xmm17, %xmm24 {%k6} {z}
+
+// CHECK: vpmovzxbd (%rcx), %xmm24
+// CHECK:  encoding: [0x62,0x62,0x7d,0x08,0x31,0x01]
+          vpmovzxbd (%rcx), %xmm24
+
+// CHECK: vpmovzxbd 291(%rax,%r14,8), %xmm24
+// CHECK:  encoding: [0x62,0x22,0x7d,0x08,0x31,0x84,0xf0,0x23,0x01,0x00,0x00]
+          vpmovzxbd 291(%rax,%r14,8), %xmm24
+
+// CHECK: vpmovzxbd 508(%rdx), %xmm24
+// CHECK:  encoding: [0x62,0x62,0x7d,0x08,0x31,0x42,0x7f]
+          vpmovzxbd 508(%rdx), %xmm24
+
+// CHECK: vpmovzxbd 512(%rdx), %xmm24
+// CHECK:  encoding: [0x62,0x62,0x7d,0x08,0x31,0x82,0x00,0x02,0x00,0x00]
+          vpmovzxbd 512(%rdx), %xmm24
+
+// CHECK: vpmovzxbd -512(%rdx), %xmm24
+// CHECK:  encoding: [0x62,0x62,0x7d,0x08,0x31,0x42,0x80]
+          vpmovzxbd -512(%rdx), %xmm24
+
+// CHECK: vpmovzxbd -516(%rdx), %xmm24
+// CHECK:  encoding: [0x62,0x62,0x7d,0x08,0x31,0x82,0xfc,0xfd,0xff,0xff]
+          vpmovzxbd -516(%rdx), %xmm24
+
+// CHECK: vpmovzxbd %xmm17, %ymm27
+// CHECK:  encoding: [0x62,0x22,0x7d,0x28,0x31,0xd9]
+          vpmovzxbd %xmm17, %ymm27
+
+// CHECK: vpmovzxbd %xmm17, %ymm27 {%k1}
+// CHECK:  encoding: [0x62,0x22,0x7d,0x29,0x31,0xd9]
+          vpmovzxbd %xmm17, %ymm27 {%k1}
+
+// CHECK: vpmovzxbd %xmm17, %ymm27 {%k1} {z}
+// CHECK:  encoding: [0x62,0x22,0x7d,0xa9,0x31,0xd9]
+          vpmovzxbd %xmm17, %ymm27 {%k1} {z}
+
+// CHECK: vpmovzxbd (%rcx), %ymm27
+// CHECK:  encoding: [0x62,0x62,0x7d,0x28,0x31,0x19]
+          vpmovzxbd (%rcx), %ymm27
+
+// CHECK: vpmovzxbd 291(%rax,%r14,8), %ymm27
+// CHECK:  encoding: [0x62,0x22,0x7d,0x28,0x31,0x9c,0xf0,0x23,0x01,0x00,0x00]
+          vpmovzxbd 291(%rax,%r14,8), %ymm27
+
+// CHECK: vpmovzxbd 1016(%rdx), %ymm27
+// CHECK:  encoding: [0x62,0x62,0x7d,0x28,0x31,0x5a,0x7f]
+          vpmovzxbd 1016(%rdx), %ymm27
+
+// CHECK: vpmovzxbd 1024(%rdx), %ymm27
+// CHECK:  encoding: [0x62,0x62,0x7d,0x28,0x31,0x9a,0x00,0x04,0x00,0x00]
+          vpmovzxbd 1024(%rdx), %ymm27
+
+// CHECK: vpmovzxbd -1024(%rdx), %ymm27
+// CHECK:  encoding: [0x62,0x62,0x7d,0x28,0x31,0x5a,0x80]
+          vpmovzxbd -1024(%rdx), %ymm27
+
+// CHECK: vpmovzxbd -1032(%rdx), %ymm27
+// CHECK:  encoding: [0x62,0x62,0x7d,0x28,0x31,0x9a,0xf8,0xfb,0xff,0xff]
+          vpmovzxbd -1032(%rdx), %ymm27
+
+// CHECK: vpmovzxbq %xmm19, %xmm19
+// CHECK:  encoding: [0x62,0xa2,0x7d,0x08,0x32,0xdb]
+          vpmovzxbq %xmm19, %xmm19
+
+// CHECK: vpmovzxbq %xmm19, %xmm19 {%k1}
+// CHECK:  encoding: [0x62,0xa2,0x7d,0x09,0x32,0xdb]
+          vpmovzxbq %xmm19, %xmm19 {%k1}
+
+// CHECK: vpmovzxbq %xmm19, %xmm19 {%k1} {z}
+// CHECK:  encoding: [0x62,0xa2,0x7d,0x89,0x32,0xdb]
+          vpmovzxbq %xmm19, %xmm19 {%k1} {z}
+
+// CHECK: vpmovzxbq (%rcx), %xmm19
+// CHECK:  encoding: [0x62,0xe2,0x7d,0x08,0x32,0x19]
+          vpmovzxbq (%rcx), %xmm19
+
+// CHECK: vpmovzxbq 291(%rax,%r14,8), %xmm19
+// CHECK:  encoding: [0x62,0xa2,0x7d,0x08,0x32,0x9c,0xf0,0x23,0x01,0x00,0x00]
+          vpmovzxbq 291(%rax,%r14,8), %xmm19
+
+// CHECK: vpmovzxbq 254(%rdx), %xmm19
+// CHECK:  encoding: [0x62,0xe2,0x7d,0x08,0x32,0x5a,0x7f]
+          vpmovzxbq 254(%rdx), %xmm19
+
+// CHECK: vpmovzxbq 256(%rdx), %xmm19
+// CHECK:  encoding: [0x62,0xe2,0x7d,0x08,0x32,0x9a,0x00,0x01,0x00,0x00]
+          vpmovzxbq 256(%rdx), %xmm19
+
+// CHECK: vpmovzxbq -256(%rdx), %xmm19
+// CHECK:  encoding: [0x62,0xe2,0x7d,0x08,0x32,0x5a,0x80]
+          vpmovzxbq -256(%rdx), %xmm19
+
+// CHECK: vpmovzxbq -258(%rdx), %xmm19
+// CHECK:  encoding: [0x62,0xe2,0x7d,0x08,0x32,0x9a,0xfe,0xfe,0xff,0xff]
+          vpmovzxbq -258(%rdx), %xmm19
+
+// CHECK: vpmovzxbq %xmm19, %ymm24
+// CHECK:  encoding: [0x62,0x22,0x7d,0x28,0x32,0xc3]
+          vpmovzxbq %xmm19, %ymm24
+
+// CHECK: vpmovzxbq %xmm19, %ymm24 {%k2}
+// CHECK:  encoding: [0x62,0x22,0x7d,0x2a,0x32,0xc3]
+          vpmovzxbq %xmm19, %ymm24 {%k2}
+
+// CHECK: vpmovzxbq %xmm19, %ymm24 {%k2} {z}
+// CHECK:  encoding: [0x62,0x22,0x7d,0xaa,0x32,0xc3]
+          vpmovzxbq %xmm19, %ymm24 {%k2} {z}
+
+// CHECK: vpmovzxbq (%rcx), %ymm24
+// CHECK:  encoding: [0x62,0x62,0x7d,0x28,0x32,0x01]
+          vpmovzxbq (%rcx), %ymm24
+
+// CHECK: vpmovzxbq 291(%rax,%r14,8), %ymm24
+// CHECK:  encoding: [0x62,0x22,0x7d,0x28,0x32,0x84,0xf0,0x23,0x01,0x00,0x00]
+          vpmovzxbq 291(%rax,%r14,8), %ymm24
+
+// CHECK: vpmovzxbq 508(%rdx), %ymm24
+// CHECK:  encoding: [0x62,0x62,0x7d,0x28,0x32,0x42,0x7f]
+          vpmovzxbq 508(%rdx), %ymm24
+
+// CHECK: vpmovzxbq 512(%rdx), %ymm24
+// CHECK:  encoding: [0x62,0x62,0x7d,0x28,0x32,0x82,0x00,0x02,0x00,0x00]
+          vpmovzxbq 512(%rdx), %ymm24
+
+// CHECK: vpmovzxbq -512(%rdx), %ymm24
+// CHECK:  encoding: [0x62,0x62,0x7d,0x28,0x32,0x42,0x80]
+          vpmovzxbq -512(%rdx), %ymm24
+
+// CHECK: vpmovzxbq -516(%rdx), %ymm24
+// CHECK:  encoding: [0x62,0x62,0x7d,0x28,0x32,0x82,0xfc,0xfd,0xff,0xff]
+          vpmovzxbq -516(%rdx), %ymm24
+
+// CHECK: vpmovzxdq %xmm21, %xmm25
+// CHECK:  encoding: [0x62,0x22,0x7d,0x08,0x35,0xcd]
+          vpmovzxdq %xmm21, %xmm25
+
+// CHECK: vpmovzxdq %xmm21, %xmm25 {%k7}
+// CHECK:  encoding: [0x62,0x22,0x7d,0x0f,0x35,0xcd]
+          vpmovzxdq %xmm21, %xmm25 {%k7}
+
+// CHECK: vpmovzxdq %xmm21, %xmm25 {%k7} {z}
+// CHECK:  encoding: [0x62,0x22,0x7d,0x8f,0x35,0xcd]
+          vpmovzxdq %xmm21, %xmm25 {%k7} {z}
+
+// CHECK: vpmovzxdq (%rcx), %xmm25
+// CHECK:  encoding: [0x62,0x62,0x7d,0x08,0x35,0x09]
+          vpmovzxdq (%rcx), %xmm25
+
+// CHECK: vpmovzxdq 291(%rax,%r14,8), %xmm25
+// CHECK:  encoding: [0x62,0x22,0x7d,0x08,0x35,0x8c,0xf0,0x23,0x01,0x00,0x00]
+          vpmovzxdq 291(%rax,%r14,8), %xmm25
+
+// CHECK: vpmovzxdq 1016(%rdx), %xmm25
+// CHECK:  encoding: [0x62,0x62,0x7d,0x08,0x35,0x4a,0x7f]
+          vpmovzxdq 1016(%rdx), %xmm25
+
+// CHECK: vpmovzxdq 1024(%rdx), %xmm25
+// CHECK:  encoding: [0x62,0x62,0x7d,0x08,0x35,0x8a,0x00,0x04,0x00,0x00]
+          vpmovzxdq 1024(%rdx), %xmm25
+
+// CHECK: vpmovzxdq -1024(%rdx), %xmm25
+// CHECK:  encoding: [0x62,0x62,0x7d,0x08,0x35,0x4a,0x80]
+          vpmovzxdq -1024(%rdx), %xmm25
+
+// CHECK: vpmovzxdq -1032(%rdx), %xmm25
+// CHECK:  encoding: [0x62,0x62,0x7d,0x08,0x35,0x8a,0xf8,0xfb,0xff,0xff]
+          vpmovzxdq -1032(%rdx), %xmm25
+
+// CHECK: vpmovzxdq %xmm22, %ymm28
+// CHECK:  encoding: [0x62,0x22,0x7d,0x28,0x35,0xe6]
+          vpmovzxdq %xmm22, %ymm28
+
+// CHECK: vpmovzxdq %xmm22, %ymm28 {%k7}
+// CHECK:  encoding: [0x62,0x22,0x7d,0x2f,0x35,0xe6]
+          vpmovzxdq %xmm22, %ymm28 {%k7}
+
+// CHECK: vpmovzxdq %xmm22, %ymm28 {%k7} {z}
+// CHECK:  encoding: [0x62,0x22,0x7d,0xaf,0x35,0xe6]
+          vpmovzxdq %xmm22, %ymm28 {%k7} {z}
+
+// CHECK: vpmovzxdq (%rcx), %ymm28
+// CHECK:  encoding: [0x62,0x62,0x7d,0x28,0x35,0x21]
+          vpmovzxdq (%rcx), %ymm28
+
+// CHECK: vpmovzxdq 291(%rax,%r14,8), %ymm28
+// CHECK:  encoding: [0x62,0x22,0x7d,0x28,0x35,0xa4,0xf0,0x23,0x01,0x00,0x00]
+          vpmovzxdq 291(%rax,%r14,8), %ymm28
+
+// CHECK: vpmovzxdq 2032(%rdx), %ymm28
+// CHECK:  encoding: [0x62,0x62,0x7d,0x28,0x35,0x62,0x7f]
+          vpmovzxdq 2032(%rdx), %ymm28
+
+// CHECK: vpmovzxdq 2048(%rdx), %ymm28
+// CHECK:  encoding: [0x62,0x62,0x7d,0x28,0x35,0xa2,0x00,0x08,0x00,0x00]
+          vpmovzxdq 2048(%rdx), %ymm28
+
+// CHECK: vpmovzxdq -2048(%rdx), %ymm28
+// CHECK:  encoding: [0x62,0x62,0x7d,0x28,0x35,0x62,0x80]
+          vpmovzxdq -2048(%rdx), %ymm28
+
+// CHECK: vpmovzxdq -2064(%rdx), %ymm28
+// CHECK:  encoding: [0x62,0x62,0x7d,0x28,0x35,0xa2,0xf0,0xf7,0xff,0xff]
+          vpmovzxdq -2064(%rdx), %ymm28
+
+// CHECK: vpmovzxwd %xmm17, %xmm24
+// CHECK:  encoding: [0x62,0x22,0x7d,0x08,0x33,0xc1]
+          vpmovzxwd %xmm17, %xmm24
+
+// CHECK: vpmovzxwd %xmm17, %xmm24 {%k4}
+// CHECK:  encoding: [0x62,0x22,0x7d,0x0c,0x33,0xc1]
+          vpmovzxwd %xmm17, %xmm24 {%k4}
+
+// CHECK: vpmovzxwd %xmm17, %xmm24 {%k4} {z}
+// CHECK:  encoding: [0x62,0x22,0x7d,0x8c,0x33,0xc1]
+          vpmovzxwd %xmm17, %xmm24 {%k4} {z}
+
+// CHECK: vpmovzxwd (%rcx), %xmm24
+// CHECK:  encoding: [0x62,0x62,0x7d,0x08,0x33,0x01]
+          vpmovzxwd (%rcx), %xmm24
+
+// CHECK: vpmovzxwd 291(%rax,%r14,8), %xmm24
+// CHECK:  encoding: [0x62,0x22,0x7d,0x08,0x33,0x84,0xf0,0x23,0x01,0x00,0x00]
+          vpmovzxwd 291(%rax,%r14,8), %xmm24
+
+// CHECK: vpmovzxwd 1016(%rdx), %xmm24
+// CHECK:  encoding: [0x62,0x62,0x7d,0x08,0x33,0x42,0x7f]
+          vpmovzxwd 1016(%rdx), %xmm24
+
+// CHECK: vpmovzxwd 1024(%rdx), %xmm24
+// CHECK:  encoding: [0x62,0x62,0x7d,0x08,0x33,0x82,0x00,0x04,0x00,0x00]
+          vpmovzxwd 1024(%rdx), %xmm24
+
+// CHECK: vpmovzxwd -1024(%rdx), %xmm24
+// CHECK:  encoding: [0x62,0x62,0x7d,0x08,0x33,0x42,0x80]
+          vpmovzxwd -1024(%rdx), %xmm24
+
+// CHECK: vpmovzxwd -1032(%rdx), %xmm24
+// CHECK:  encoding: [0x62,0x62,0x7d,0x08,0x33,0x82,0xf8,0xfb,0xff,0xff]
+          vpmovzxwd -1032(%rdx), %xmm24
+
+// CHECK: vpmovzxwd %xmm29, %ymm26
+// CHECK:  encoding: [0x62,0x02,0x7d,0x28,0x33,0xd5]
+          vpmovzxwd %xmm29, %ymm26
+
+// CHECK: vpmovzxwd %xmm29, %ymm26 {%k5}
+// CHECK:  encoding: [0x62,0x02,0x7d,0x2d,0x33,0xd5]
+          vpmovzxwd %xmm29, %ymm26 {%k5}
+
+// CHECK: vpmovzxwd %xmm29, %ymm26 {%k5} {z}
+// CHECK:  encoding: [0x62,0x02,0x7d,0xad,0x33,0xd5]
+          vpmovzxwd %xmm29, %ymm26 {%k5} {z}
+
+// CHECK: vpmovzxwd (%rcx), %ymm26
+// CHECK:  encoding: [0x62,0x62,0x7d,0x28,0x33,0x11]
+          vpmovzxwd (%rcx), %ymm26
+
+// CHECK: vpmovzxwd 291(%rax,%r14,8), %ymm26
+// CHECK:  encoding: [0x62,0x22,0x7d,0x28,0x33,0x94,0xf0,0x23,0x01,0x00,0x00]
+          vpmovzxwd 291(%rax,%r14,8), %ymm26
+
+// CHECK: vpmovzxwd 2032(%rdx), %ymm26
+// CHECK:  encoding: [0x62,0x62,0x7d,0x28,0x33,0x52,0x7f]
+          vpmovzxwd 2032(%rdx), %ymm26
+
+// CHECK: vpmovzxwd 2048(%rdx), %ymm26
+// CHECK:  encoding: [0x62,0x62,0x7d,0x28,0x33,0x92,0x00,0x08,0x00,0x00]
+          vpmovzxwd 2048(%rdx), %ymm26
+
+// CHECK: vpmovzxwd -2048(%rdx), %ymm26
+// CHECK:  encoding: [0x62,0x62,0x7d,0x28,0x33,0x52,0x80]
+          vpmovzxwd -2048(%rdx), %ymm26
+
+// CHECK: vpmovzxwd -2064(%rdx), %ymm26
+// CHECK:  encoding: [0x62,0x62,0x7d,0x28,0x33,0x92,0xf0,0xf7,0xff,0xff]
+          vpmovzxwd -2064(%rdx), %ymm26
+
+// CHECK: vpmovzxwq %xmm20, %xmm29
+// CHECK:  encoding: [0x62,0x22,0x7d,0x08,0x34,0xec]
+          vpmovzxwq %xmm20, %xmm29
+
+// CHECK: vpmovzxwq %xmm20, %xmm29 {%k2}
+// CHECK:  encoding: [0x62,0x22,0x7d,0x0a,0x34,0xec]
+          vpmovzxwq %xmm20, %xmm29 {%k2}
+
+// CHECK: vpmovzxwq %xmm20, %xmm29 {%k2} {z}
+// CHECK:  encoding: [0x62,0x22,0x7d,0x8a,0x34,0xec]
+          vpmovzxwq %xmm20, %xmm29 {%k2} {z}
+
+// CHECK: vpmovzxwq (%rcx), %xmm29
+// CHECK:  encoding: [0x62,0x62,0x7d,0x08,0x34,0x29]
+          vpmovzxwq (%rcx), %xmm29
+
+// CHECK: vpmovzxwq 291(%rax,%r14,8), %xmm29
+// CHECK:  encoding: [0x62,0x22,0x7d,0x08,0x34,0xac,0xf0,0x23,0x01,0x00,0x00]
+          vpmovzxwq 291(%rax,%r14,8), %xmm29
+
+// CHECK: vpmovzxwq 508(%rdx), %xmm29
+// CHECK:  encoding: [0x62,0x62,0x7d,0x08,0x34,0x6a,0x7f]
+          vpmovzxwq 508(%rdx), %xmm29
+
+// CHECK: vpmovzxwq 512(%rdx), %xmm29
+// CHECK:  encoding: [0x62,0x62,0x7d,0x08,0x34,0xaa,0x00,0x02,0x00,0x00]
+          vpmovzxwq 512(%rdx), %xmm29
+
+// CHECK: vpmovzxwq -512(%rdx), %xmm29
+// CHECK:  encoding: [0x62,0x62,0x7d,0x08,0x34,0x6a,0x80]
+          vpmovzxwq -512(%rdx), %xmm29
+
+// CHECK: vpmovzxwq -516(%rdx), %xmm29
+// CHECK:  encoding: [0x62,0x62,0x7d,0x08,0x34,0xaa,0xfc,0xfd,0xff,0xff]
+          vpmovzxwq -516(%rdx), %xmm29
+
+// CHECK: vpmovzxwq %xmm25, %ymm18
+// CHECK:  encoding: [0x62,0x82,0x7d,0x28,0x34,0xd1]
+          vpmovzxwq %xmm25, %ymm18
+
+// CHECK: vpmovzxwq %xmm25, %ymm18 {%k1}
+// CHECK:  encoding: [0x62,0x82,0x7d,0x29,0x34,0xd1]
+          vpmovzxwq %xmm25, %ymm18 {%k1}
+
+// CHECK: vpmovzxwq %xmm25, %ymm18 {%k1} {z}
+// CHECK:  encoding: [0x62,0x82,0x7d,0xa9,0x34,0xd1]
+          vpmovzxwq %xmm25, %ymm18 {%k1} {z}
+
+// CHECK: vpmovzxwq (%rcx), %ymm18
+// CHECK:  encoding: [0x62,0xe2,0x7d,0x28,0x34,0x11]
+          vpmovzxwq (%rcx), %ymm18
+
+// CHECK: vpmovzxwq 291(%rax,%r14,8), %ymm18
+// CHECK:  encoding: [0x62,0xa2,0x7d,0x28,0x34,0x94,0xf0,0x23,0x01,0x00,0x00]
+          vpmovzxwq 291(%rax,%r14,8), %ymm18
+
+// CHECK: vpmovzxwq 1016(%rdx), %ymm18
+// CHECK:  encoding: [0x62,0xe2,0x7d,0x28,0x34,0x52,0x7f]
+          vpmovzxwq 1016(%rdx), %ymm18
+
+// CHECK: vpmovzxwq 1024(%rdx), %ymm18
+// CHECK:  encoding: [0x62,0xe2,0x7d,0x28,0x34,0x92,0x00,0x04,0x00,0x00]
+          vpmovzxwq 1024(%rdx), %ymm18
+
+// CHECK: vpmovzxwq -1024(%rdx), %ymm18
+// CHECK:  encoding: [0x62,0xe2,0x7d,0x28,0x34,0x52,0x80]
+          vpmovzxwq -1024(%rdx), %ymm18
+
+// CHECK: vpmovzxwq -1032(%rdx), %ymm18
+// CHECK:  encoding: [0x62,0xe2,0x7d,0x28,0x34,0x92,0xf8,0xfb,0xff,0xff]
+          vpmovzxwq -1032(%rdx), %ymm18
+
 // CHECK: vpmulld %xmm24, %xmm19, %xmm25
 // CHECK:  encoding: [0x62,0x02,0x65,0x00,0x40,0xc8]
           vpmulld %xmm24, %xmm19, %xmm25
@@ -8955,3 +9675,10067 @@ vaddpd  {rz-sae}, %zmm2, %zmm1, %zmm1
 // CHECK: vprorq $123, -1032(%rdx){1to4}, %ymm21
 // CHECK:  encoding: [0x62,0xf1,0xd5,0x30,0x72,0x82,0xf8,0xfb,0xff,0xff,0x7b]
           vprorq $123, -1032(%rdx){1to4}, %ymm21
+
+// CHECK: vpmuludq %xmm18, %xmm22, %xmm27
+// CHECK:  encoding: [0x62,0x21,0xcd,0x00,0xf4,0xda]
+          vpmuludq %xmm18, %xmm22, %xmm27
+
+// CHECK: vpmuludq %xmm18, %xmm22, %xmm27 {%k5}
+// CHECK:  encoding: [0x62,0x21,0xcd,0x05,0xf4,0xda]
+          vpmuludq %xmm18, %xmm22, %xmm27 {%k5}
+
+// CHECK: vpmuludq %xmm18, %xmm22, %xmm27 {%k5} {z}
+// CHECK:  encoding: [0x62,0x21,0xcd,0x85,0xf4,0xda]
+          vpmuludq %xmm18, %xmm22, %xmm27 {%k5} {z}
+
+// CHECK: vpmuludq (%rcx), %xmm22, %xmm27
+// CHECK:  encoding: [0x62,0x61,0xcd,0x00,0xf4,0x19]
+          vpmuludq (%rcx), %xmm22, %xmm27
+
+// CHECK: vpmuludq 291(%rax,%r14,8), %xmm22, %xmm27
+// CHECK:  encoding: [0x62,0x21,0xcd,0x00,0xf4,0x9c,0xf0,0x23,0x01,0x00,0x00]
+          vpmuludq 291(%rax,%r14,8), %xmm22, %xmm27
+
+// CHECK: vpmuludq (%rcx){1to2}, %xmm22, %xmm27
+// CHECK:  encoding: [0x62,0x61,0xcd,0x10,0xf4,0x19]
+          vpmuludq (%rcx){1to2}, %xmm22, %xmm27
+
+// CHECK: vpmuludq 2032(%rdx), %xmm22, %xmm27
+// CHECK:  encoding: [0x62,0x61,0xcd,0x00,0xf4,0x5a,0x7f]
+          vpmuludq 2032(%rdx), %xmm22, %xmm27
+
+// CHECK: vpmuludq 2048(%rdx), %xmm22, %xmm27
+// CHECK:  encoding: [0x62,0x61,0xcd,0x00,0xf4,0x9a,0x00,0x08,0x00,0x00]
+          vpmuludq 2048(%rdx), %xmm22, %xmm27
+
+// CHECK: vpmuludq -2048(%rdx), %xmm22, %xmm27
+// CHECK:  encoding: [0x62,0x61,0xcd,0x00,0xf4,0x5a,0x80]
+          vpmuludq -2048(%rdx), %xmm22, %xmm27
+
+// CHECK: vpmuludq -2064(%rdx), %xmm22, %xmm27
+// CHECK:  encoding: [0x62,0x61,0xcd,0x00,0xf4,0x9a,0xf0,0xf7,0xff,0xff]
+          vpmuludq -2064(%rdx), %xmm22, %xmm27
+
+// CHECK: vpmuludq 1016(%rdx){1to2}, %xmm22, %xmm27
+// CHECK:  encoding: [0x62,0x61,0xcd,0x10,0xf4,0x5a,0x7f]
+          vpmuludq 1016(%rdx){1to2}, %xmm22, %xmm27
+
+// CHECK: vpmuludq 1024(%rdx){1to2}, %xmm22, %xmm27
+// CHECK:  encoding: [0x62,0x61,0xcd,0x10,0xf4,0x9a,0x00,0x04,0x00,0x00]
+          vpmuludq 1024(%rdx){1to2}, %xmm22, %xmm27
+
+// CHECK: vpmuludq -1024(%rdx){1to2}, %xmm22, %xmm27
+// CHECK:  encoding: [0x62,0x61,0xcd,0x10,0xf4,0x5a,0x80]
+          vpmuludq -1024(%rdx){1to2}, %xmm22, %xmm27
+
+// CHECK: vpmuludq -1032(%rdx){1to2}, %xmm22, %xmm27
+// CHECK:  encoding: [0x62,0x61,0xcd,0x10,0xf4,0x9a,0xf8,0xfb,0xff,0xff]
+          vpmuludq -1032(%rdx){1to2}, %xmm22, %xmm27
+
+// CHECK: vpmuludq %ymm18, %ymm27, %ymm30
+// CHECK:  encoding: [0x62,0x21,0xa5,0x20,0xf4,0xf2]
+          vpmuludq %ymm18, %ymm27, %ymm30
+
+// CHECK: vpmuludq %ymm18, %ymm27, %ymm30 {%k7}
+// CHECK:  encoding: [0x62,0x21,0xa5,0x27,0xf4,0xf2]
+          vpmuludq %ymm18, %ymm27, %ymm30 {%k7}
+
+// CHECK: vpmuludq %ymm18, %ymm27, %ymm30 {%k7} {z}
+// CHECK:  encoding: [0x62,0x21,0xa5,0xa7,0xf4,0xf2]
+          vpmuludq %ymm18, %ymm27, %ymm30 {%k7} {z}
+
+// CHECK: vpmuludq (%rcx), %ymm27, %ymm30
+// CHECK:  encoding: [0x62,0x61,0xa5,0x20,0xf4,0x31]
+          vpmuludq (%rcx), %ymm27, %ymm30
+
+// CHECK: vpmuludq 291(%rax,%r14,8), %ymm27, %ymm30
+// CHECK:  encoding: [0x62,0x21,0xa5,0x20,0xf4,0xb4,0xf0,0x23,0x01,0x00,0x00]
+          vpmuludq 291(%rax,%r14,8), %ymm27, %ymm30
+
+// CHECK: vpmuludq (%rcx){1to4}, %ymm27, %ymm30
+// CHECK:  encoding: [0x62,0x61,0xa5,0x30,0xf4,0x31]
+          vpmuludq (%rcx){1to4}, %ymm27, %ymm30
+
+// CHECK: vpmuludq 4064(%rdx), %ymm27, %ymm30
+// CHECK:  encoding: [0x62,0x61,0xa5,0x20,0xf4,0x72,0x7f]
+          vpmuludq 4064(%rdx), %ymm27, %ymm30
+
+// CHECK: vpmuludq 4096(%rdx), %ymm27, %ymm30
+// CHECK:  encoding: [0x62,0x61,0xa5,0x20,0xf4,0xb2,0x00,0x10,0x00,0x00]
+          vpmuludq 4096(%rdx), %ymm27, %ymm30
+
+// CHECK: vpmuludq -4096(%rdx), %ymm27, %ymm30
+// CHECK:  encoding: [0x62,0x61,0xa5,0x20,0xf4,0x72,0x80]
+          vpmuludq -4096(%rdx), %ymm27, %ymm30
+
+// CHECK: vpmuludq -4128(%rdx), %ymm27, %ymm30
+// CHECK:  encoding: [0x62,0x61,0xa5,0x20,0xf4,0xb2,0xe0,0xef,0xff,0xff]
+          vpmuludq -4128(%rdx), %ymm27, %ymm30
+
+// CHECK: vpmuludq 1016(%rdx){1to4}, %ymm27, %ymm30
+// CHECK:  encoding: [0x62,0x61,0xa5,0x30,0xf4,0x72,0x7f]
+          vpmuludq 1016(%rdx){1to4}, %ymm27, %ymm30
+
+// CHECK: vpmuludq 1024(%rdx){1to4}, %ymm27, %ymm30
+// CHECK:  encoding: [0x62,0x61,0xa5,0x30,0xf4,0xb2,0x00,0x04,0x00,0x00]
+          vpmuludq 1024(%rdx){1to4}, %ymm27, %ymm30
+
+// CHECK: vpmuludq -1024(%rdx){1to4}, %ymm27, %ymm30
+// CHECK:  encoding: [0x62,0x61,0xa5,0x30,0xf4,0x72,0x80]
+          vpmuludq -1024(%rdx){1to4}, %ymm27, %ymm30
+
+// CHECK: vpmuludq -1032(%rdx){1to4}, %ymm27, %ymm30
+// CHECK:  encoding: [0x62,0x61,0xa5,0x30,0xf4,0xb2,0xf8,0xfb,0xff,0xff]
+          vpmuludq -1032(%rdx){1to4}, %ymm27, %ymm30
+          
+// CHECK: vpmuldq %xmm26, %xmm24, %xmm25
+// CHECK:  encoding: [0x62,0x02,0xbd,0x00,0x28,0xca]
+          vpmuldq %xmm26, %xmm24, %xmm25
+
+// CHECK: vpmuldq %xmm26, %xmm24, %xmm25 {%k3}
+// CHECK:  encoding: [0x62,0x02,0xbd,0x03,0x28,0xca]
+          vpmuldq %xmm26, %xmm24, %xmm25 {%k3}
+
+// CHECK: vpmuldq %xmm26, %xmm24, %xmm25 {%k3} {z}
+// CHECK:  encoding: [0x62,0x02,0xbd,0x83,0x28,0xca]
+          vpmuldq %xmm26, %xmm24, %xmm25 {%k3} {z}
+
+// CHECK: vpmuldq (%rcx), %xmm24, %xmm25
+// CHECK:  encoding: [0x62,0x62,0xbd,0x00,0x28,0x09]
+          vpmuldq (%rcx), %xmm24, %xmm25
+
+// CHECK: vpmuldq 291(%rax,%r14,8), %xmm24, %xmm25
+// CHECK:  encoding: [0x62,0x22,0xbd,0x00,0x28,0x8c,0xf0,0x23,0x01,0x00,0x00]
+          vpmuldq 291(%rax,%r14,8), %xmm24, %xmm25
+
+// CHECK: vpmuldq (%rcx){1to2}, %xmm24, %xmm25
+// CHECK:  encoding: [0x62,0x62,0xbd,0x10,0x28,0x09]
+          vpmuldq (%rcx){1to2}, %xmm24, %xmm25
+
+// CHECK: vpmuldq 2032(%rdx), %xmm24, %xmm25
+// CHECK:  encoding: [0x62,0x62,0xbd,0x00,0x28,0x4a,0x7f]
+          vpmuldq 2032(%rdx), %xmm24, %xmm25
+
+// CHECK: vpmuldq 2048(%rdx), %xmm24, %xmm25
+// CHECK:  encoding: [0x62,0x62,0xbd,0x00,0x28,0x8a,0x00,0x08,0x00,0x00]
+          vpmuldq 2048(%rdx), %xmm24, %xmm25
+
+// CHECK: vpmuldq -2048(%rdx), %xmm24, %xmm25
+// CHECK:  encoding: [0x62,0x62,0xbd,0x00,0x28,0x4a,0x80]
+          vpmuldq -2048(%rdx), %xmm24, %xmm25
+
+// CHECK: vpmuldq -2064(%rdx), %xmm24, %xmm25
+// CHECK:  encoding: [0x62,0x62,0xbd,0x00,0x28,0x8a,0xf0,0xf7,0xff,0xff]
+          vpmuldq -2064(%rdx), %xmm24, %xmm25
+
+// CHECK: vpmuldq 1016(%rdx){1to2}, %xmm24, %xmm25
+// CHECK:  encoding: [0x62,0x62,0xbd,0x10,0x28,0x4a,0x7f]
+          vpmuldq 1016(%rdx){1to2}, %xmm24, %xmm25
+
+// CHECK: vpmuldq 1024(%rdx){1to2}, %xmm24, %xmm25
+// CHECK:  encoding: [0x62,0x62,0xbd,0x10,0x28,0x8a,0x00,0x04,0x00,0x00]
+          vpmuldq 1024(%rdx){1to2}, %xmm24, %xmm25
+
+// CHECK: vpmuldq -1024(%rdx){1to2}, %xmm24, %xmm25
+// CHECK:  encoding: [0x62,0x62,0xbd,0x10,0x28,0x4a,0x80]
+          vpmuldq -1024(%rdx){1to2}, %xmm24, %xmm25
+
+// CHECK: vpmuldq -1032(%rdx){1to2}, %xmm24, %xmm25
+// CHECK:  encoding: [0x62,0x62,0xbd,0x10,0x28,0x8a,0xf8,0xfb,0xff,0xff]
+          vpmuldq -1032(%rdx){1to2}, %xmm24, %xmm25
+
+// CHECK: vpmuldq %ymm26, %ymm18, %ymm19
+// CHECK:  encoding: [0x62,0x82,0xed,0x20,0x28,0xda]
+          vpmuldq %ymm26, %ymm18, %ymm19
+
+// CHECK: vpmuldq %ymm26, %ymm18, %ymm19 {%k6}
+// CHECK:  encoding: [0x62,0x82,0xed,0x26,0x28,0xda]
+          vpmuldq %ymm26, %ymm18, %ymm19 {%k6}
+
+// CHECK: vpmuldq %ymm26, %ymm18, %ymm19 {%k6} {z}
+// CHECK:  encoding: [0x62,0x82,0xed,0xa6,0x28,0xda]
+          vpmuldq %ymm26, %ymm18, %ymm19 {%k6} {z}
+
+// CHECK: vpmuldq (%rcx), %ymm18, %ymm19
+// CHECK:  encoding: [0x62,0xe2,0xed,0x20,0x28,0x19]
+          vpmuldq (%rcx), %ymm18, %ymm19
+
+// CHECK: vpmuldq 291(%rax,%r14,8), %ymm18, %ymm19
+// CHECK:  encoding: [0x62,0xa2,0xed,0x20,0x28,0x9c,0xf0,0x23,0x01,0x00,0x00]
+          vpmuldq 291(%rax,%r14,8), %ymm18, %ymm19
+
+// CHECK: vpmuldq (%rcx){1to4}, %ymm18, %ymm19
+// CHECK:  encoding: [0x62,0xe2,0xed,0x30,0x28,0x19]
+          vpmuldq (%rcx){1to4}, %ymm18, %ymm19
+
+// CHECK: vpmuldq 4064(%rdx), %ymm18, %ymm19
+// CHECK:  encoding: [0x62,0xe2,0xed,0x20,0x28,0x5a,0x7f]
+          vpmuldq 4064(%rdx), %ymm18, %ymm19
+
+// CHECK: vpmuldq 4096(%rdx), %ymm18, %ymm19
+// CHECK:  encoding: [0x62,0xe2,0xed,0x20,0x28,0x9a,0x00,0x10,0x00,0x00]
+          vpmuldq 4096(%rdx), %ymm18, %ymm19
+
+// CHECK: vpmuldq -4096(%rdx), %ymm18, %ymm19
+// CHECK:  encoding: [0x62,0xe2,0xed,0x20,0x28,0x5a,0x80]
+          vpmuldq -4096(%rdx), %ymm18, %ymm19
+
+// CHECK: vpmuldq -4128(%rdx), %ymm18, %ymm19
+// CHECK:  encoding: [0x62,0xe2,0xed,0x20,0x28,0x9a,0xe0,0xef,0xff,0xff]
+          vpmuldq -4128(%rdx), %ymm18, %ymm19
+
+// CHECK: vpmuldq 1016(%rdx){1to4}, %ymm18, %ymm19
+// CHECK:  encoding: [0x62,0xe2,0xed,0x30,0x28,0x5a,0x7f]
+          vpmuldq 1016(%rdx){1to4}, %ymm18, %ymm19
+
+// CHECK: vpmuldq 1024(%rdx){1to4}, %ymm18, %ymm19
+// CHECK:  encoding: [0x62,0xe2,0xed,0x30,0x28,0x9a,0x00,0x04,0x00,0x00]
+          vpmuldq 1024(%rdx){1to4}, %ymm18, %ymm19
+
+// CHECK: vpmuldq -1024(%rdx){1to4}, %ymm18, %ymm19
+// CHECK:  encoding: [0x62,0xe2,0xed,0x30,0x28,0x5a,0x80]
+          vpmuldq -1024(%rdx){1to4}, %ymm18, %ymm19
+
+// CHECK: vpmuldq -1032(%rdx){1to4}, %ymm18, %ymm19
+// CHECK:  encoding: [0x62,0xe2,0xed,0x30,0x28,0x9a,0xf8,0xfb,0xff,0xff]
+          vpmuldq -1032(%rdx){1to4}, %ymm18, %ymm19
+
+// CHECK: vcmppd $171, %xmm23, %xmm27, %k3
+// CHECK:  encoding: [0x62,0xb1,0xa5,0x00,0xc2,0xdf,0xab]
+          vcmppd $0xab, %xmm23, %xmm27, %k3
+
+// CHECK: vcmppd $171, %xmm23, %xmm27, %k3 {%k5}
+// CHECK:  encoding: [0x62,0xb1,0xa5,0x05,0xc2,0xdf,0xab]
+          vcmppd $0xab, %xmm23, %xmm27, %k3 {%k5}
+
+// CHECK: vcmppd $123, %xmm23, %xmm27, %k3
+// CHECK:  encoding: [0x62,0xb1,0xa5,0x00,0xc2,0xdf,0x7b]
+          vcmppd $0x7b, %xmm23, %xmm27, %k3
+
+// CHECK: vcmppd $123, (%rcx), %xmm27, %k3
+// CHECK:  encoding: [0x62,0xf1,0xa5,0x00,0xc2,0x19,0x7b]
+          vcmppd $0x7b, (%rcx), %xmm27, %k3
+
+// CHECK: vcmppd $123, 291(%rax,%r14,8), %xmm27, %k3
+// CHECK:  encoding: [0x62,0xb1,0xa5,0x00,0xc2,0x9c,0xf0,0x23,0x01,0x00,0x00,0x7b]
+          vcmppd $0x7b, 291(%rax,%r14,8), %xmm27, %k3
+
+// CHECK: vcmppd $123, (%rcx){1to2}, %xmm27, %k3
+// CHECK:  encoding: [0x62,0xf1,0xa5,0x10,0xc2,0x19,0x7b]
+          vcmppd $0x7b, (%rcx){1to2}, %xmm27, %k3
+
+// CHECK: vcmppd $123, 2032(%rdx), %xmm27, %k3
+// CHECK:  encoding: [0x62,0xf1,0xa5,0x00,0xc2,0x5a,0x7f,0x7b]
+          vcmppd $0x7b, 2032(%rdx), %xmm27, %k3
+
+// CHECK: vcmppd $123, 2048(%rdx), %xmm27, %k3
+// CHECK:  encoding: [0x62,0xf1,0xa5,0x00,0xc2,0x9a,0x00,0x08,0x00,0x00,0x7b]
+          vcmppd $0x7b, 2048(%rdx), %xmm27, %k3
+
+// CHECK: vcmppd $123, -2048(%rdx), %xmm27, %k3
+// CHECK:  encoding: [0x62,0xf1,0xa5,0x00,0xc2,0x5a,0x80,0x7b]
+          vcmppd $0x7b, -2048(%rdx), %xmm27, %k3
+
+// CHECK: vcmppd $123, -2064(%rdx), %xmm27, %k3
+// CHECK:  encoding: [0x62,0xf1,0xa5,0x00,0xc2,0x9a,0xf0,0xf7,0xff,0xff,0x7b]
+          vcmppd $0x7b, -2064(%rdx), %xmm27, %k3
+
+// CHECK: vcmppd $123, 1016(%rdx){1to2}, %xmm27, %k3
+// CHECK:  encoding: [0x62,0xf1,0xa5,0x10,0xc2,0x5a,0x7f,0x7b]
+          vcmppd $0x7b, 1016(%rdx){1to2}, %xmm27, %k3
+
+// CHECK: vcmppd $123, 1024(%rdx){1to2}, %xmm27, %k3
+// CHECK:  encoding: [0x62,0xf1,0xa5,0x10,0xc2,0x9a,0x00,0x04,0x00,0x00,0x7b]
+          vcmppd $0x7b, 1024(%rdx){1to2}, %xmm27, %k3
+
+// CHECK: vcmppd $123, -1024(%rdx){1to2}, %xmm27, %k3
+// CHECK:  encoding: [0x62,0xf1,0xa5,0x10,0xc2,0x5a,0x80,0x7b]
+          vcmppd $0x7b, -1024(%rdx){1to2}, %xmm27, %k3
+
+// CHECK: vcmppd $123, -1032(%rdx){1to2}, %xmm27, %k3
+// CHECK:  encoding: [0x62,0xf1,0xa5,0x10,0xc2,0x9a,0xf8,0xfb,0xff,0xff,0x7b]
+          vcmppd $0x7b, -1032(%rdx){1to2}, %xmm27, %k3
+
+// CHECK: vcmppd $171, %ymm27, %ymm17, %k4
+// CHECK:  encoding: [0x62,0x91,0xf5,0x20,0xc2,0xe3,0xab]
+          vcmppd $0xab, %ymm27, %ymm17, %k4
+
+// CHECK: vcmppd $171, %ymm27, %ymm17, %k4 {%k7}
+// CHECK:  encoding: [0x62,0x91,0xf5,0x27,0xc2,0xe3,0xab]
+          vcmppd $0xab, %ymm27, %ymm17, %k4 {%k7}
+
+// CHECK: vcmppd $123, %ymm27, %ymm17, %k4
+// CHECK:  encoding: [0x62,0x91,0xf5,0x20,0xc2,0xe3,0x7b]
+          vcmppd $0x7b, %ymm27, %ymm17, %k4
+
+// CHECK: vcmppd $123, (%rcx), %ymm17, %k4
+// CHECK:  encoding: [0x62,0xf1,0xf5,0x20,0xc2,0x21,0x7b]
+          vcmppd $0x7b, (%rcx), %ymm17, %k4
+
+// CHECK: vcmppd $123, 291(%rax,%r14,8), %ymm17, %k4
+// CHECK:  encoding: [0x62,0xb1,0xf5,0x20,0xc2,0xa4,0xf0,0x23,0x01,0x00,0x00,0x7b]
+          vcmppd $0x7b, 291(%rax,%r14,8), %ymm17, %k4
+
+// CHECK: vcmppd $123, (%rcx){1to4}, %ymm17, %k4
+// CHECK:  encoding: [0x62,0xf1,0xf5,0x30,0xc2,0x21,0x7b]
+          vcmppd $0x7b, (%rcx){1to4}, %ymm17, %k4
+
+// CHECK: vcmppd $123, 4064(%rdx), %ymm17, %k4
+// CHECK:  encoding: [0x62,0xf1,0xf5,0x20,0xc2,0x62,0x7f,0x7b]
+          vcmppd $123, 4064(%rdx), %ymm17, %k4
+
+// CHECK: vcmppd $123, 4096(%rdx), %ymm17, %k4
+// CHECK:  encoding: [0x62,0xf1,0xf5,0x20,0xc2,0xa2,0x00,0x10,0x00,0x00,0x7b]
+          vcmppd $0x7b, 4096(%rdx), %ymm17, %k4
+
+// CHECK: vcmppd $123, -4096(%rdx), %ymm17, %k4
+// CHECK:  encoding: [0x62,0xf1,0xf5,0x20,0xc2,0x62,0x80,0x7b]
+          vcmppd $0x7b, -4096(%rdx), %ymm17, %k4
+
+// CHECK: vcmppd $123, -4128(%rdx), %ymm17, %k4
+// CHECK:  encoding: [0x62,0xf1,0xf5,0x20,0xc2,0xa2,0xe0,0xef,0xff,0xff,0x7b]
+          vcmppd $0x7b, -4128(%rdx), %ymm17, %k4
+
+// CHECK: vcmppd $123, 1016(%rdx){1to4}, %ymm17, %k4
+// CHECK:  encoding: [0x62,0xf1,0xf5,0x30,0xc2,0x62,0x7f,0x7b]
+          vcmppd $0x7b, 1016(%rdx){1to4}, %ymm17, %k4
+
+// CHECK: vcmppd $123, 1024(%rdx){1to4}, %ymm17, %k4
+// CHECK:  encoding: [0x62,0xf1,0xf5,0x30,0xc2,0xa2,0x00,0x04,0x00,0x00,0x7b]
+          vcmppd $0x7b, 1024(%rdx){1to4}, %ymm17, %k4
+
+// CHECK: vcmppd $123, -1024(%rdx){1to4}, %ymm17, %k4
+// CHECK:  encoding: [0x62,0xf1,0xf5,0x30,0xc2,0x62,0x80,0x7b]
+          vcmppd $0x7b, -1024(%rdx){1to4}, %ymm17, %k4
+
+// CHECK: vcmppd $123, -1032(%rdx){1to4}, %ymm17, %k4
+// CHECK:  encoding: [0x62,0xf1,0xf5,0x30,0xc2,0xa2,0xf8,0xfb,0xff,0xff,0x7b]
+          vcmppd $0x7b, -1032(%rdx){1to4}, %ymm17, %k4
+
+// CHECK: vcmpps $171, %xmm28, %xmm29, %k4
+// CHECK:  encoding: [0x62,0x91,0x14,0x00,0xc2,0xe4,0xab]
+          vcmpps $0xab, %xmm28, %xmm29, %k4
+
+// CHECK: vcmpps $171, %xmm28, %xmm29, %k4 {%k2}
+// CHECK:  encoding: [0x62,0x91,0x14,0x02,0xc2,0xe4,0xab]
+          vcmpps $0xab, %xmm28, %xmm29, %k4 {%k2}
+
+// CHECK: vcmpps $123, %xmm28, %xmm29, %k4
+// CHECK:  encoding: [0x62,0x91,0x14,0x00,0xc2,0xe4,0x7b]
+          vcmpps $0x7b, %xmm28, %xmm29, %k4
+
+// CHECK: vcmpps $123, (%rcx), %xmm29, %k4
+// CHECK:  encoding: [0x62,0xf1,0x14,0x00,0xc2,0x21,0x7b]
+          vcmpps $0x7b, (%rcx), %xmm29, %k4
+
+// CHECK: vcmpps $123, 291(%rax,%r14,8), %xmm29, %k4
+// CHECK:  encoding: [0x62,0xb1,0x14,0x00,0xc2,0xa4,0xf0,0x23,0x01,0x00,0x00,0x7b]
+          vcmpps $0x7b, 291(%rax,%r14,8), %xmm29, %k4
+
+// CHECK: vcmpps $123, (%rcx){1to4}, %xmm29, %k4
+// CHECK:  encoding: [0x62,0xf1,0x14,0x10,0xc2,0x21,0x7b]
+          vcmpps $0x7b, (%rcx){1to4}, %xmm29, %k4
+
+// CHECK: vcmpps $123, 2032(%rdx), %xmm29, %k4
+// CHECK:  encoding: [0x62,0xf1,0x14,0x00,0xc2,0x62,0x7f,0x7b]
+          vcmpps $0x7b, 2032(%rdx), %xmm29, %k4
+
+// CHECK: vcmpps $123, 2048(%rdx), %xmm29, %k4
+// CHECK:  encoding: [0x62,0xf1,0x14,0x00,0xc2,0xa2,0x00,0x08,0x00,0x00,0x7b]
+          vcmpps $0x7b, 2048(%rdx), %xmm29, %k4
+
+// CHECK: vcmpps $123, -2048(%rdx), %xmm29, %k4
+// CHECK:  encoding: [0x62,0xf1,0x14,0x00,0xc2,0x62,0x80,0x7b]
+          vcmpps $0x7b, -2048(%rdx), %xmm29, %k4
+
+// CHECK: vcmpps $123, -2064(%rdx), %xmm29, %k4
+// CHECK:  encoding: [0x62,0xf1,0x14,0x00,0xc2,0xa2,0xf0,0xf7,0xff,0xff,0x7b]
+          vcmpps $0x7b, -2064(%rdx), %xmm29, %k4
+
+// CHECK: vcmpps $123, 508(%rdx){1to4}, %xmm29, %k4
+// CHECK:  encoding: [0x62,0xf1,0x14,0x10,0xc2,0x62,0x7f,0x7b]
+          vcmpps $123, 508(%rdx){1to4}, %xmm29, %k4
+
+// CHECK: vcmpps $123, 512(%rdx){1to4}, %xmm29, %k4
+// CHECK:  encoding: [0x62,0xf1,0x14,0x10,0xc2,0xa2,0x00,0x02,0x00,0x00,0x7b]
+          vcmpps $123, 512(%rdx){1to4}, %xmm29, %k4
+
+// CHECK: vcmpps $123, -512(%rdx){1to4}, %xmm29, %k4
+// CHECK:  encoding: [0x62,0xf1,0x14,0x10,0xc2,0x62,0x80,0x7b]
+          vcmpps $0x7b, -512(%rdx){1to4}, %xmm29, %k4
+
+// CHECK: vcmpps $123, -516(%rdx){1to4}, %xmm29, %k4
+// CHECK:  encoding: [0x62,0xf1,0x14,0x10,0xc2,0xa2,0xfc,0xfd,0xff,0xff,0x7b]
+          vcmpps $0x7b, -516(%rdx){1to4}, %xmm29, %k4
+
+// CHECK: vcmpps $171, %ymm18, %ymm19, %k4
+// CHECK:  encoding: [0x62,0xb1,0x64,0x20,0xc2,0xe2,0xab]
+          vcmpps $0xab, %ymm18, %ymm19, %k4
+
+// CHECK: vcmpps $171, %ymm18, %ymm19, %k4 {%k1}
+// CHECK:  encoding: [0x62,0xb1,0x64,0x21,0xc2,0xe2,0xab]
+          vcmpps $0xab, %ymm18, %ymm19, %k4 {%k1}
+
+// CHECK: vcmpps $123, %ymm18, %ymm19, %k4
+// CHECK:  encoding: [0x62,0xb1,0x64,0x20,0xc2,0xe2,0x7b]
+          vcmpps $0x7b, %ymm18, %ymm19, %k4
+
+// CHECK: vcmpps $123, (%rcx), %ymm19, %k4
+// CHECK:  encoding: [0x62,0xf1,0x64,0x20,0xc2,0x21,0x7b]
+          vcmpps $0x7b, (%rcx), %ymm19, %k4
+
+// CHECK: vcmpps $123, 291(%rax,%r14,8), %ymm19, %k4
+// CHECK:  encoding: [0x62,0xb1,0x64,0x20,0xc2,0xa4,0xf0,0x23,0x01,0x00,0x00,0x7b]
+          vcmpps $0x7b, 291(%rax,%r14,8), %ymm19, %k4
+
+// CHECK: vcmpps $123, (%rcx){1to8}, %ymm19, %k4
+// CHECK:  encoding: [0x62,0xf1,0x64,0x30,0xc2,0x21,0x7b]
+          vcmpps $0x7b, (%rcx){1to8}, %ymm19, %k4
+
+// CHECK: vcmpps $123, 4064(%rdx), %ymm19, %k4
+// CHECK:  encoding: [0x62,0xf1,0x64,0x20,0xc2,0x62,0x7f,0x7b]
+          vcmpps $0x7b, 4064(%rdx), %ymm19, %k4
+
+// CHECK: vcmpps $123, 4096(%rdx), %ymm19, %k4
+// CHECK:  encoding: [0x62,0xf1,0x64,0x20,0xc2,0xa2,0x00,0x10,0x00,0x00,0x7b]
+          vcmpps $0x7b, 4096(%rdx), %ymm19, %k4
+
+// CHECK: vcmpps $123, -4096(%rdx), %ymm19, %k4
+// CHECK:  encoding: [0x62,0xf1,0x64,0x20,0xc2,0x62,0x80,0x7b]
+          vcmpps $0x7b, -4096(%rdx), %ymm19, %k4
+
+// CHECK: vcmpps $123, -4128(%rdx), %ymm19, %k4
+// CHECK:  encoding: [0x62,0xf1,0x64,0x20,0xc2,0xa2,0xe0,0xef,0xff,0xff,0x7b]
+          vcmpps $0x7b, -4128(%rdx), %ymm19, %k4
+
+// CHECK: vcmpps $123, 508(%rdx){1to8}, %ymm19, %k4
+// CHECK:  encoding: [0x62,0xf1,0x64,0x30,0xc2,0x62,0x7f,0x7b]
+          vcmpps $0x7b, 508(%rdx){1to8}, %ymm19, %k4
+
+// CHECK: vcmpps $123, 512(%rdx){1to8}, %ymm19, %k4
+// CHECK:  encoding: [0x62,0xf1,0x64,0x30,0xc2,0xa2,0x00,0x02,0x00,0x00,0x7b]
+          vcmpps $0x7b, 512(%rdx){1to8}, %ymm19, %k4
+
+// CHECK: vcmpps $123, -512(%rdx){1to8}, %ymm19, %k4
+// CHECK:  encoding: [0x62,0xf1,0x64,0x30,0xc2,0x62,0x80,0x7b]
+          vcmpps $0x7b, -512(%rdx){1to8}, %ymm19, %k4
+
+// CHECK: vcmpps $123, -516(%rdx){1to8}, %ymm19, %k4
+// CHECK:  encoding: [0x62,0xf1,0x64,0x30,0xc2,0xa2,0xfc,0xfd,0xff,0xff,0x7b]
+          vcmpps $0x7b, -516(%rdx){1to8}, %ymm19, %k4
+
+// CHECK: vbroadcastf32x4 (%rcx), %ymm28
+// CHECK:  encoding: [0x62,0x62,0x7d,0x28,0x1a,0x21]
+          vbroadcastf32x4 (%rcx), %ymm28
+
+// CHECK: vbroadcastf32x4 (%rcx), %ymm28 {%k4}
+// CHECK:  encoding: [0x62,0x62,0x7d,0x2c,0x1a,0x21]
+          vbroadcastf32x4 (%rcx), %ymm28 {%k4}
+
+// CHECK: vbroadcastf32x4 (%rcx), %ymm28 {%k4} {z}
+// CHECK:  encoding: [0x62,0x62,0x7d,0xac,0x1a,0x21]
+          vbroadcastf32x4 (%rcx), %ymm28 {%k4} {z}
+
+// CHECK: vbroadcastf32x4 291(%rax,%r14,8), %ymm28
+// CHECK:  encoding: [0x62,0x22,0x7d,0x28,0x1a,0xa4,0xf0,0x23,0x01,0x00,0x00]
+          vbroadcastf32x4 291(%rax,%r14,8), %ymm28
+
+// CHECK: vbroadcastf32x4 2032(%rdx), %ymm28
+// CHECK:  encoding: [0x62,0x62,0x7d,0x28,0x1a,0x62,0x7f]
+          vbroadcastf32x4 2032(%rdx), %ymm28
+
+// CHECK: vbroadcastf32x4 2048(%rdx), %ymm28
+// CHECK:  encoding: [0x62,0x62,0x7d,0x28,0x1a,0xa2,0x00,0x08,0x00,0x00]
+          vbroadcastf32x4 2048(%rdx), %ymm28
+
+// CHECK: vbroadcastf32x4 -2048(%rdx), %ymm28
+// CHECK:  encoding: [0x62,0x62,0x7d,0x28,0x1a,0x62,0x80]
+          vbroadcastf32x4 -2048(%rdx), %ymm28
+
+// CHECK: vbroadcastf32x4 -2064(%rdx), %ymm28
+// CHECK:  encoding: [0x62,0x62,0x7d,0x28,0x1a,0xa2,0xf0,0xf7,0xff,0xff]
+          vbroadcastf32x4 -2064(%rdx), %ymm28
+
+// CHECK: vbroadcasti32x4 (%rcx), %ymm19
+// CHECK:  encoding: [0x62,0xe2,0x7d,0x28,0x5a,0x19]
+          vbroadcasti32x4 (%rcx), %ymm19
+
+// CHECK: vbroadcasti32x4 (%rcx), %ymm19 {%k5}
+// CHECK:  encoding: [0x62,0xe2,0x7d,0x2d,0x5a,0x19]
+          vbroadcasti32x4 (%rcx), %ymm19 {%k5}
+
+// CHECK: vbroadcasti32x4 (%rcx), %ymm19 {%k5} {z}
+// CHECK:  encoding: [0x62,0xe2,0x7d,0xad,0x5a,0x19]
+          vbroadcasti32x4 (%rcx), %ymm19 {%k5} {z}
+
+// CHECK: vbroadcasti32x4 291(%rax,%r14,8), %ymm19
+// CHECK:  encoding: [0x62,0xa2,0x7d,0x28,0x5a,0x9c,0xf0,0x23,0x01,0x00,0x00]
+          vbroadcasti32x4 291(%rax,%r14,8), %ymm19
+
+// CHECK: vbroadcasti32x4 2032(%rdx), %ymm19
+// CHECK:  encoding: [0x62,0xe2,0x7d,0x28,0x5a,0x5a,0x7f]
+          vbroadcasti32x4 2032(%rdx), %ymm19
+
+// CHECK: vbroadcasti32x4 2048(%rdx), %ymm19
+// CHECK:  encoding: [0x62,0xe2,0x7d,0x28,0x5a,0x9a,0x00,0x08,0x00,0x00]
+          vbroadcasti32x4 2048(%rdx), %ymm19
+
+// CHECK: vbroadcasti32x4 -2048(%rdx), %ymm19
+// CHECK:  encoding: [0x62,0xe2,0x7d,0x28,0x5a,0x5a,0x80]
+          vbroadcasti32x4 -2048(%rdx), %ymm19
+
+// CHECK: vbroadcasti32x4 -2064(%rdx), %ymm19
+// CHECK:  encoding: [0x62,0xe2,0x7d,0x28,0x5a,0x9a,0xf0,0xf7,0xff,0xff]
+          vbroadcasti32x4 -2064(%rdx), %ymm19
+
+// CHECK: vfixupimmps $171, %xmm17, %xmm17, %xmm25
+// CHECK:  encoding: [0x62,0x23,0x75,0x00,0x54,0xc9,0xab]
+          vfixupimmps $0xab, %xmm17, %xmm17, %xmm25
+
+// CHECK: vfixupimmps $171, %xmm17, %xmm17, %xmm25 {%k3}
+// CHECK:  encoding: [0x62,0x23,0x75,0x03,0x54,0xc9,0xab]
+          vfixupimmps $0xab, %xmm17, %xmm17, %xmm25 {%k3}
+
+// CHECK: vfixupimmps $171, %xmm17, %xmm17, %xmm25 {%k3} {z}
+// CHECK:  encoding: [0x62,0x23,0x75,0x83,0x54,0xc9,0xab]
+          vfixupimmps $0xab, %xmm17, %xmm17, %xmm25 {%k3} {z}
+
+// CHECK: vfixupimmps $123, %xmm17, %xmm17, %xmm25
+// CHECK:  encoding: [0x62,0x23,0x75,0x00,0x54,0xc9,0x7b]
+          vfixupimmps $0x7b, %xmm17, %xmm17, %xmm25
+
+// CHECK: vfixupimmps $123, (%rcx), %xmm17, %xmm25
+// CHECK:  encoding: [0x62,0x63,0x75,0x00,0x54,0x09,0x7b]
+          vfixupimmps $0x7b, (%rcx), %xmm17, %xmm25
+
+// CHECK: vfixupimmps $123, 291(%rax,%r14,8), %xmm17, %xmm25
+// CHECK:  encoding: [0x62,0x23,0x75,0x00,0x54,0x8c,0xf0,0x23,0x01,0x00,0x00,0x7b]
+          vfixupimmps $0x7b, 291(%rax,%r14,8), %xmm17, %xmm25
+
+// CHECK: vfixupimmps $123, (%rcx){1to4}, %xmm17, %xmm25
+// CHECK:  encoding: [0x62,0x63,0x75,0x10,0x54,0x09,0x7b]
+          vfixupimmps $0x7b, (%rcx){1to4}, %xmm17, %xmm25
+
+// CHECK: vfixupimmps $123, 2032(%rdx), %xmm17, %xmm25
+// CHECK:  encoding: [0x62,0x63,0x75,0x00,0x54,0x4a,0x7f,0x7b]
+          vfixupimmps $0x7b, 2032(%rdx), %xmm17, %xmm25
+
+// CHECK: vfixupimmps $123, 2048(%rdx), %xmm17, %xmm25
+// CHECK:  encoding: [0x62,0x63,0x75,0x00,0x54,0x8a,0x00,0x08,0x00,0x00,0x7b]
+          vfixupimmps $0x7b, 2048(%rdx), %xmm17, %xmm25
+
+// CHECK: vfixupimmps $123, -2048(%rdx), %xmm17, %xmm25
+// CHECK:  encoding: [0x62,0x63,0x75,0x00,0x54,0x4a,0x80,0x7b]
+          vfixupimmps $0x7b, -2048(%rdx), %xmm17, %xmm25
+
+// CHECK: vfixupimmps $123, -2064(%rdx), %xmm17, %xmm25
+// CHECK:  encoding: [0x62,0x63,0x75,0x00,0x54,0x8a,0xf0,0xf7,0xff,0xff,0x7b]
+          vfixupimmps $0x7b, -2064(%rdx), %xmm17, %xmm25
+
+// CHECK: vfixupimmps $123, 508(%rdx){1to4}, %xmm17, %xmm25
+// CHECK:  encoding: [0x62,0x63,0x75,0x10,0x54,0x4a,0x7f,0x7b]
+          vfixupimmps $0x7b, 508(%rdx){1to4}, %xmm17, %xmm25
+
+// CHECK: vfixupimmps $123, 512(%rdx){1to4}, %xmm17, %xmm25
+// CHECK:  encoding: [0x62,0x63,0x75,0x10,0x54,0x8a,0x00,0x02,0x00,0x00,0x7b]
+          vfixupimmps $0x7b, 512(%rdx){1to4}, %xmm17, %xmm25
+
+// CHECK: vfixupimmps $123, -512(%rdx){1to4}, %xmm17, %xmm25
+// CHECK:  encoding: [0x62,0x63,0x75,0x10,0x54,0x4a,0x80,0x7b]
+          vfixupimmps $0x7b, -512(%rdx){1to4}, %xmm17, %xmm25
+
+// CHECK: vfixupimmps $123, -516(%rdx){1to4}, %xmm17, %xmm25
+// CHECK:  encoding: [0x62,0x63,0x75,0x10,0x54,0x8a,0xfc,0xfd,0xff,0xff,0x7b]
+          vfixupimmps $0x7b, -516(%rdx){1to4}, %xmm17, %xmm25
+
+// CHECK: vfixupimmps $171, %ymm28, %ymm21, %ymm30
+// CHECK:  encoding: [0x62,0x03,0x55,0x20,0x54,0xf4,0xab]
+          vfixupimmps $0xab, %ymm28, %ymm21, %ymm30
+
+// CHECK: vfixupimmps $171, %ymm28, %ymm21, %ymm30 {%k4}
+// CHECK:  encoding: [0x62,0x03,0x55,0x24,0x54,0xf4,0xab]
+          vfixupimmps $0xab, %ymm28, %ymm21, %ymm30 {%k4}
+
+// CHECK: vfixupimmps $171, %ymm28, %ymm21, %ymm30 {%k4} {z}
+// CHECK:  encoding: [0x62,0x03,0x55,0xa4,0x54,0xf4,0xab]
+          vfixupimmps $0xab, %ymm28, %ymm21, %ymm30 {%k4} {z}
+
+// CHECK: vfixupimmps $123, %ymm28, %ymm21, %ymm30
+// CHECK:  encoding: [0x62,0x03,0x55,0x20,0x54,0xf4,0x7b]
+          vfixupimmps $0x7b, %ymm28, %ymm21, %ymm30
+
+// CHECK: vfixupimmps $123, (%rcx), %ymm21, %ymm30
+// CHECK:  encoding: [0x62,0x63,0x55,0x20,0x54,0x31,0x7b]
+          vfixupimmps $0x7b, (%rcx), %ymm21, %ymm30
+
+// CHECK: vfixupimmps $123, 291(%rax,%r14,8), %ymm21, %ymm30
+// CHECK:  encoding: [0x62,0x23,0x55,0x20,0x54,0xb4,0xf0,0x23,0x01,0x00,0x00,0x7b]
+          vfixupimmps $0x7b, 291(%rax,%r14,8), %ymm21, %ymm30
+
+// CHECK: vfixupimmps $123, (%rcx){1to8}, %ymm21, %ymm30
+// CHECK:  encoding: [0x62,0x63,0x55,0x30,0x54,0x31,0x7b]
+          vfixupimmps $0x7b, (%rcx){1to8}, %ymm21, %ymm30
+
+// CHECK: vfixupimmps $123, 4064(%rdx), %ymm21, %ymm30
+// CHECK:  encoding: [0x62,0x63,0x55,0x20,0x54,0x72,0x7f,0x7b]
+          vfixupimmps $0x7b, 4064(%rdx), %ymm21, %ymm30
+
+// CHECK: vfixupimmps $123, 4096(%rdx), %ymm21, %ymm30
+// CHECK:  encoding: [0x62,0x63,0x55,0x20,0x54,0xb2,0x00,0x10,0x00,0x00,0x7b]
+          vfixupimmps $0x7b, 4096(%rdx), %ymm21, %ymm30
+
+// CHECK: vfixupimmps $123, -4096(%rdx), %ymm21, %ymm30
+// CHECK:  encoding: [0x62,0x63,0x55,0x20,0x54,0x72,0x80,0x7b]
+          vfixupimmps $0x7b, -4096(%rdx), %ymm21, %ymm30
+
+// CHECK: vfixupimmps $123, -4128(%rdx), %ymm21, %ymm30
+// CHECK:  encoding: [0x62,0x63,0x55,0x20,0x54,0xb2,0xe0,0xef,0xff,0xff,0x7b]
+          vfixupimmps $0x7b, -4128(%rdx), %ymm21, %ymm30
+
+// CHECK: vfixupimmps $123, 508(%rdx){1to8}, %ymm21, %ymm30
+// CHECK:  encoding: [0x62,0x63,0x55,0x30,0x54,0x72,0x7f,0x7b]
+          vfixupimmps $0x7b, 508(%rdx){1to8}, %ymm21, %ymm30
+
+// CHECK: vfixupimmps $123, 512(%rdx){1to8}, %ymm21, %ymm30
+// CHECK:  encoding: [0x62,0x63,0x55,0x30,0x54,0xb2,0x00,0x02,0x00,0x00,0x7b]
+          vfixupimmps $0x7b, 512(%rdx){1to8}, %ymm21, %ymm30
+
+// CHECK: vfixupimmps $123, -512(%rdx){1to8}, %ymm21, %ymm30
+// CHECK:  encoding: [0x62,0x63,0x55,0x30,0x54,0x72,0x80,0x7b]
+          vfixupimmps $0x7b, -512(%rdx){1to8}, %ymm21, %ymm30
+
+// CHECK: vfixupimmps $123, -516(%rdx){1to8}, %ymm21, %ymm30
+// CHECK:  encoding: [0x62,0x63,0x55,0x30,0x54,0xb2,0xfc,0xfd,0xff,0xff,0x7b]
+          vfixupimmps $0x7b, -516(%rdx){1to8}, %ymm21, %ymm30
+
+// CHECK: vfixupimmpd $171, %xmm25, %xmm18, %xmm24
+// CHECK:  encoding: [0x62,0x03,0xed,0x00,0x54,0xc1,0xab]
+          vfixupimmpd $0xab, %xmm25, %xmm18, %xmm24
+
+// CHECK: vfixupimmpd $171, %xmm25, %xmm18, %xmm24 {%k3}
+// CHECK:  encoding: [0x62,0x03,0xed,0x03,0x54,0xc1,0xab]
+          vfixupimmpd $0xab, %xmm25, %xmm18, %xmm24 {%k3}
+
+// CHECK: vfixupimmpd $171, %xmm25, %xmm18, %xmm24 {%k3} {z}
+// CHECK:  encoding: [0x62,0x03,0xed,0x83,0x54,0xc1,0xab]
+          vfixupimmpd $0xab, %xmm25, %xmm18, %xmm24 {%k3} {z}
+
+// CHECK: vfixupimmpd $123, %xmm25, %xmm18, %xmm24
+// CHECK:  encoding: [0x62,0x03,0xed,0x00,0x54,0xc1,0x7b]
+          vfixupimmpd $0x7b, %xmm25, %xmm18, %xmm24
+
+// CHECK: vfixupimmpd $123, (%rcx), %xmm18, %xmm24
+// CHECK:  encoding: [0x62,0x63,0xed,0x00,0x54,0x01,0x7b]
+          vfixupimmpd $0x7b, (%rcx), %xmm18, %xmm24
+
+// CHECK: vfixupimmpd $123, 291(%rax,%r14,8), %xmm18, %xmm24
+// CHECK:  encoding: [0x62,0x23,0xed,0x00,0x54,0x84,0xf0,0x23,0x01,0x00,0x00,0x7b]
+          vfixupimmpd $0x7b, 291(%rax,%r14,8), %xmm18, %xmm24
+
+// CHECK: vfixupimmpd $123, (%rcx){1to2}, %xmm18, %xmm24
+// CHECK:  encoding: [0x62,0x63,0xed,0x10,0x54,0x01,0x7b]
+          vfixupimmpd $0x7b, (%rcx){1to2}, %xmm18, %xmm24
+
+// CHECK: vfixupimmpd $123, 2032(%rdx), %xmm18, %xmm24
+// CHECK:  encoding: [0x62,0x63,0xed,0x00,0x54,0x42,0x7f,0x7b]
+          vfixupimmpd $0x7b, 2032(%rdx), %xmm18, %xmm24
+
+// CHECK: vfixupimmpd $123, 2048(%rdx), %xmm18, %xmm24
+// CHECK:  encoding: [0x62,0x63,0xed,0x00,0x54,0x82,0x00,0x08,0x00,0x00,0x7b]
+          vfixupimmpd $0x7b, 2048(%rdx), %xmm18, %xmm24
+
+// CHECK: vfixupimmpd $123, -2048(%rdx), %xmm18, %xmm24
+// CHECK:  encoding: [0x62,0x63,0xed,0x00,0x54,0x42,0x80,0x7b]
+          vfixupimmpd $0x7b, -2048(%rdx), %xmm18, %xmm24
+
+// CHECK: vfixupimmpd $123, -2064(%rdx), %xmm18, %xmm24
+// CHECK:  encoding: [0x62,0x63,0xed,0x00,0x54,0x82,0xf0,0xf7,0xff,0xff,0x7b]
+          vfixupimmpd $0x7b, -2064(%rdx), %xmm18, %xmm24
+
+// CHECK: vfixupimmpd $123, 1016(%rdx){1to2}, %xmm18, %xmm24
+// CHECK:  encoding: [0x62,0x63,0xed,0x10,0x54,0x42,0x7f,0x7b]
+          vfixupimmpd $0x7b, 1016(%rdx){1to2}, %xmm18, %xmm24
+
+// CHECK: vfixupimmpd $123, 1024(%rdx){1to2}, %xmm18, %xmm24
+// CHECK:  encoding: [0x62,0x63,0xed,0x10,0x54,0x82,0x00,0x04,0x00,0x00,0x7b]
+          vfixupimmpd $0x7b, 1024(%rdx){1to2}, %xmm18, %xmm24
+
+// CHECK: vfixupimmpd $123, -1024(%rdx){1to2}, %xmm18, %xmm24
+// CHECK:  encoding: [0x62,0x63,0xed,0x10,0x54,0x42,0x80,0x7b]
+          vfixupimmpd $0x7b, -1024(%rdx){1to2}, %xmm18, %xmm24
+
+// CHECK: vfixupimmpd $123, -1032(%rdx){1to2}, %xmm18, %xmm24
+// CHECK:  encoding: [0x62,0x63,0xed,0x10,0x54,0x82,0xf8,0xfb,0xff,0xff,0x7b]
+          vfixupimmpd $0x7b, -1032(%rdx){1to2}, %xmm18, %xmm24
+
+// CHECK: vfixupimmpd $171, %ymm28, %ymm22, %ymm18
+// CHECK:  encoding: [0x62,0x83,0xcd,0x20,0x54,0xd4,0xab]
+          vfixupimmpd $0xab, %ymm28, %ymm22, %ymm18
+
+// CHECK: vfixupimmpd $171, %ymm28, %ymm22, %ymm18 {%k2}
+// CHECK:  encoding: [0x62,0x83,0xcd,0x22,0x54,0xd4,0xab]
+          vfixupimmpd $0xab, %ymm28, %ymm22, %ymm18 {%k2}
+
+// CHECK: vfixupimmpd $171, %ymm28, %ymm22, %ymm18 {%k2} {z}
+// CHECK:  encoding: [0x62,0x83,0xcd,0xa2,0x54,0xd4,0xab]
+          vfixupimmpd $0xab, %ymm28, %ymm22, %ymm18 {%k2} {z}
+
+// CHECK: vfixupimmpd $123, %ymm28, %ymm22, %ymm18
+// CHECK:  encoding: [0x62,0x83,0xcd,0x20,0x54,0xd4,0x7b]
+          vfixupimmpd $0x7b, %ymm28, %ymm22, %ymm18
+
+// CHECK: vfixupimmpd $123, (%rcx), %ymm22, %ymm18
+// CHECK:  encoding: [0x62,0xe3,0xcd,0x20,0x54,0x11,0x7b]
+          vfixupimmpd $0x7b, (%rcx), %ymm22, %ymm18
+
+// CHECK: vfixupimmpd $123, 291(%rax,%r14,8), %ymm22, %ymm18
+// CHECK:  encoding: [0x62,0xa3,0xcd,0x20,0x54,0x94,0xf0,0x23,0x01,0x00,0x00,0x7b]
+          vfixupimmpd $0x7b, 291(%rax,%r14,8), %ymm22, %ymm18
+
+// CHECK: vfixupimmpd $123, (%rcx){1to4}, %ymm22, %ymm18
+// CHECK:  encoding: [0x62,0xe3,0xcd,0x30,0x54,0x11,0x7b]
+          vfixupimmpd $0x7b, (%rcx){1to4}, %ymm22, %ymm18
+
+// CHECK: vfixupimmpd $123, 4064(%rdx), %ymm22, %ymm18
+// CHECK:  encoding: [0x62,0xe3,0xcd,0x20,0x54,0x52,0x7f,0x7b]
+          vfixupimmpd $0x7b, 4064(%rdx), %ymm22, %ymm18
+
+// CHECK: vfixupimmpd $123, 4096(%rdx), %ymm22, %ymm18
+// CHECK:  encoding: [0x62,0xe3,0xcd,0x20,0x54,0x92,0x00,0x10,0x00,0x00,0x7b]
+          vfixupimmpd $0x7b, 4096(%rdx), %ymm22, %ymm18
+
+// CHECK: vfixupimmpd $123, -4096(%rdx), %ymm22, %ymm18
+// CHECK:  encoding: [0x62,0xe3,0xcd,0x20,0x54,0x52,0x80,0x7b]
+          vfixupimmpd $0x7b, -4096(%rdx), %ymm22, %ymm18
+
+// CHECK: vfixupimmpd $123, -4128(%rdx), %ymm22, %ymm18
+// CHECK:  encoding: [0x62,0xe3,0xcd,0x20,0x54,0x92,0xe0,0xef,0xff,0xff,0x7b]
+          vfixupimmpd $0x7b, -4128(%rdx), %ymm22, %ymm18
+
+// CHECK: vfixupimmpd $123, 1016(%rdx){1to4}, %ymm22, %ymm18
+// CHECK:  encoding: [0x62,0xe3,0xcd,0x30,0x54,0x52,0x7f,0x7b]
+          vfixupimmpd $0x7b, 1016(%rdx){1to4}, %ymm22, %ymm18
+
+// CHECK: vfixupimmpd $123, 1024(%rdx){1to4}, %ymm22, %ymm18
+// CHECK:  encoding: [0x62,0xe3,0xcd,0x30,0x54,0x92,0x00,0x04,0x00,0x00,0x7b]
+          vfixupimmpd $0x7b, 1024(%rdx){1to4}, %ymm22, %ymm18
+
+// CHECK: vfixupimmpd $123, -1024(%rdx){1to4}, %ymm22, %ymm18
+// CHECK:  encoding: [0x62,0xe3,0xcd,0x30,0x54,0x52,0x80,0x7b]
+          vfixupimmpd $0x7b, -1024(%rdx){1to4}, %ymm22, %ymm18
+
+// CHECK: vfixupimmpd $123, -1032(%rdx){1to4}, %ymm22, %ymm18
+// CHECK:  encoding: [0x62,0xe3,0xcd,0x30,0x54,0x92,0xf8,0xfb,0xff,0xff,0x7b]
+          vfixupimmpd $0x7b, -1032(%rdx){1to4}, %ymm22, %ymm18
+
+// CHECK: vpshufd $171, %xmm23, %xmm17
+// CHECK:  encoding: [0x62,0xa1,0x7d,0x08,0x70,0xcf,0xab]
+          vpshufd $171, %xmm23, %xmm17
+
+// CHECK: vpshufd $171, %xmm23, %xmm17 {%k1}
+// CHECK:  encoding: [0x62,0xa1,0x7d,0x09,0x70,0xcf,0xab]
+          vpshufd $171, %xmm23, %xmm17 {%k1}
+
+// CHECK: vpshufd $171, %xmm23, %xmm17 {%k1} {z}
+// CHECK:  encoding: [0x62,0xa1,0x7d,0x89,0x70,0xcf,0xab]
+          vpshufd $171, %xmm23, %xmm17 {%k1} {z}
+
+// CHECK: vpshufd $123, %xmm23, %xmm17
+// CHECK:  encoding: [0x62,0xa1,0x7d,0x08,0x70,0xcf,0x7b]
+          vpshufd $123, %xmm23, %xmm17
+
+// CHECK: vpshufd $123, (%rcx), %xmm17
+// CHECK:  encoding: [0x62,0xe1,0x7d,0x08,0x70,0x09,0x7b]
+          vpshufd $123, (%rcx), %xmm17
+
+// CHECK: vpshufd $123, 291(%rax,%r14,8), %xmm17
+// CHECK:  encoding: [0x62,0xa1,0x7d,0x08,0x70,0x8c,0xf0,0x23,0x01,0x00,0x00,0x7b]
+          vpshufd $123, 291(%rax,%r14,8), %xmm17
+
+
+// CHECK: vpshufd $123, 2032(%rdx), %xmm17
+// CHECK:  encoding: [0x62,0xe1,0x7d,0x08,0x70,0x4a,0x7f,0x7b]
+          vpshufd $123, 2032(%rdx), %xmm17
+
+// CHECK: vpshufd $123, (%rcx){1to4}, %xmm17
+// CHECK:  encoding: [0x62,0xe1,0x7d,0x18,0x70,0x09,0x7b]
+          vpshufd $123, (%rcx){1to4}, %xmm17
+
+// CHECK: vpshufd $123, 2048(%rdx), %xmm17
+// CHECK:  encoding: [0x62,0xe1,0x7d,0x08,0x70,0x8a,0x00,0x08,0x00,0x00,0x7b]
+          vpshufd $123, 2048(%rdx), %xmm17
+
+// CHECK: vpshufd $123, -2048(%rdx), %xmm17
+// CHECK:  encoding: [0x62,0xe1,0x7d,0x08,0x70,0x4a,0x80,0x7b]
+          vpshufd $123, -2048(%rdx), %xmm17
+
+// CHECK: vpshufd $123, -2064(%rdx), %xmm17
+// CHECK:  encoding: [0x62,0xe1,0x7d,0x08,0x70,0x8a,0xf0,0xf7,0xff,0xff,0x7b]
+          vpshufd $123, -2064(%rdx), %xmm17
+
+// CHECK: vpshufd $123, 508(%rdx){1to4}, %xmm17
+// CHECK:  encoding: [0x62,0xe1,0x7d,0x18,0x70,0x4a,0x7f,0x7b]
+          vpshufd $123, 508(%rdx){1to4}, %xmm17
+
+// CHECK: vpshufd $123, 512(%rdx){1to4}, %xmm17
+// CHECK:  encoding: [0x62,0xe1,0x7d,0x18,0x70,0x8a,0x00,0x02,0x00,0x00,0x7b]
+          vpshufd $123, 512(%rdx){1to4}, %xmm17
+
+// CHECK: vpshufd $123, -512(%rdx){1to4}, %xmm17
+// CHECK:  encoding: [0x62,0xe1,0x7d,0x18,0x70,0x4a,0x80,0x7b]
+          vpshufd $123, -512(%rdx){1to4}, %xmm17
+
+// CHECK: vpshufd $123, -516(%rdx){1to4}, %xmm17
+// CHECK:  encoding: [0x62,0xe1,0x7d,0x18,0x70,0x8a,0xfc,0xfd,0xff,0xff,0x7b]
+          vpshufd $123, -516(%rdx){1to4}, %xmm17
+
+// CHECK: vpshufd $171, %ymm22, %ymm20
+// CHECK:  encoding: [0x62,0xa1,0x7d,0x28,0x70,0xe6,0xab]
+          vpshufd $171, %ymm22, %ymm20
+
+// CHECK: vpshufd $171, %ymm22, %ymm20 {%k2}
+// CHECK:  encoding: [0x62,0xa1,0x7d,0x2a,0x70,0xe6,0xab]
+          vpshufd $171, %ymm22, %ymm20 {%k2}
+
+// CHECK: vpshufd $171, %ymm22, %ymm20 {%k2} {z}
+// CHECK:  encoding: [0x62,0xa1,0x7d,0xaa,0x70,0xe6,0xab]
+          vpshufd $171, %ymm22, %ymm20 {%k2} {z}
+
+// CHECK: vpshufd $123, %ymm22, %ymm20
+// CHECK:  encoding: [0x62,0xa1,0x7d,0x28,0x70,0xe6,0x7b]
+          vpshufd $123, %ymm22, %ymm20
+
+// CHECK: vpshufd $123, (%rcx), %ymm20
+// CHECK:  encoding: [0x62,0xe1,0x7d,0x28,0x70,0x21,0x7b]
+          vpshufd $123, (%rcx), %ymm20
+
+// CHECK: vpshufd $123, 291(%rax,%r14,8), %ymm20
+// CHECK:  encoding: [0x62,0xa1,0x7d,0x28,0x70,0xa4,0xf0,0x23,0x01,0x00,0x00,0x7b]
+          vpshufd $123, 291(%rax,%r14,8), %ymm20
+
+// CHECK: vpshufd $123, (%rcx){1to8}, %ymm20
+// CHECK:  encoding: [0x62,0xe1,0x7d,0x38,0x70,0x21,0x7b]
+          vpshufd $123, (%rcx){1to8}, %ymm20
+
+// CHECK: vpshufd $123, 4064(%rdx), %ymm20
+// CHECK:  encoding: [0x62,0xe1,0x7d,0x28,0x70,0x62,0x7f,0x7b]
+          vpshufd $123, 4064(%rdx), %ymm20
+
+// CHECK: vpshufd $123, 4096(%rdx), %ymm20
+// CHECK:  encoding: [0x62,0xe1,0x7d,0x28,0x70,0xa2,0x00,0x10,0x00,0x00,0x7b]
+          vpshufd $123, 4096(%rdx), %ymm20
+
+// CHECK: vpshufd $123, -4096(%rdx), %ymm20
+// CHECK:  encoding: [0x62,0xe1,0x7d,0x28,0x70,0x62,0x80,0x7b]
+          vpshufd $123, -4096(%rdx), %ymm20
+
+// CHECK: vpshufd $123, -4128(%rdx), %ymm20
+// CHECK:  encoding: [0x62,0xe1,0x7d,0x28,0x70,0xa2,0xe0,0xef,0xff,0xff,0x7b]
+          vpshufd $123, -4128(%rdx), %ymm20
+
+// CHECK: vpshufd $123, 508(%rdx){1to8}, %ymm20
+// CHECK:  encoding: [0x62,0xe1,0x7d,0x38,0x70,0x62,0x7f,0x7b]
+          vpshufd $123, 508(%rdx){1to8}, %ymm20
+
+// CHECK: vpshufd $123, 512(%rdx){1to8}, %ymm20
+// CHECK:  encoding: [0x62,0xe1,0x7d,0x38,0x70,0xa2,0x00,0x02,0x00,0x00,0x7b]
+          vpshufd $123, 512(%rdx){1to8}, %ymm20
+
+// CHECK: vpshufd $123, -512(%rdx){1to8}, %ymm20
+// CHECK:  encoding: [0x62,0xe1,0x7d,0x38,0x70,0x62,0x80,0x7b]
+          vpshufd $123, -512(%rdx){1to8}, %ymm20
+
+// CHECK: vpshufd $123, -516(%rdx){1to8}, %ymm20
+// CHECK:  encoding: [0x62,0xe1,0x7d,0x38,0x70,0xa2,0xfc,0xfd,0xff,0xff,0x7b]
+          vpshufd $123, -516(%rdx){1to8}, %ymm20
+
+// CHECK: vgetexppd %xmm18, %xmm17
+// CHECK:  encoding: [0x62,0xa2,0xfd,0x08,0x42,0xca]
+          vgetexppd %xmm18, %xmm17
+
+// CHECK: vgetexppd %xmm18, %xmm17 {%k1}
+// CHECK:  encoding: [0x62,0xa2,0xfd,0x09,0x42,0xca]
+          vgetexppd %xmm18, %xmm17 {%k1}
+
+// CHECK: vgetexppd %xmm18, %xmm17 {%k1} {z}
+// CHECK:  encoding: [0x62,0xa2,0xfd,0x89,0x42,0xca]
+          vgetexppd %xmm18, %xmm17 {%k1} {z}
+
+// CHECK: vgetexppd (%rcx), %xmm17
+// CHECK:  encoding: [0x62,0xe2,0xfd,0x08,0x42,0x09]
+          vgetexppd (%rcx), %xmm17
+
+// CHECK: vgetexppd 291(%rax,%r14,8), %xmm17
+// CHECK:  encoding: [0x62,0xa2,0xfd,0x08,0x42,0x8c,0xf0,0x23,0x01,0x00,0x00]
+          vgetexppd 291(%rax,%r14,8), %xmm17
+
+// CHECK: vgetexppd (%rcx){1to2}, %xmm17
+// CHECK:  encoding: [0x62,0xe2,0xfd,0x18,0x42,0x09]
+          vgetexppd (%rcx){1to2}, %xmm17
+
+// CHECK: vgetexppd 2032(%rdx), %xmm17
+// CHECK:  encoding: [0x62,0xe2,0xfd,0x08,0x42,0x4a,0x7f]
+          vgetexppd 2032(%rdx), %xmm17
+
+// CHECK: vgetexppd 2048(%rdx), %xmm17
+// CHECK:  encoding: [0x62,0xe2,0xfd,0x08,0x42,0x8a,0x00,0x08,0x00,0x00]
+          vgetexppd 2048(%rdx), %xmm17
+
+// CHECK: vgetexppd -2048(%rdx), %xmm17
+// CHECK:  encoding: [0x62,0xe2,0xfd,0x08,0x42,0x4a,0x80]
+          vgetexppd -2048(%rdx), %xmm17
+
+// CHECK: vgetexppd -2064(%rdx), %xmm17
+// CHECK:  encoding: [0x62,0xe2,0xfd,0x08,0x42,0x8a,0xf0,0xf7,0xff,0xff]
+          vgetexppd -2064(%rdx), %xmm17
+
+// CHECK: vgetexppd 1016(%rdx){1to2}, %xmm17
+// CHECK:  encoding: [0x62,0xe2,0xfd,0x18,0x42,0x4a,0x7f]
+          vgetexppd 1016(%rdx){1to2}, %xmm17
+
+// CHECK: vgetexppd 1024(%rdx){1to2}, %xmm17
+// CHECK:  encoding: [0x62,0xe2,0xfd,0x18,0x42,0x8a,0x00,0x04,0x00,0x00]
+          vgetexppd 1024(%rdx){1to2}, %xmm17
+
+// CHECK: vgetexppd -1024(%rdx){1to2}, %xmm17
+// CHECK:  encoding: [0x62,0xe2,0xfd,0x18,0x42,0x4a,0x80]
+          vgetexppd -1024(%rdx){1to2}, %xmm17
+
+// CHECK: vgetexppd -1032(%rdx){1to2}, %xmm17
+// CHECK:  encoding: [0x62,0xe2,0xfd,0x18,0x42,0x8a,0xf8,0xfb,0xff,0xff]
+          vgetexppd -1032(%rdx){1to2}, %xmm17
+
+// CHECK: vgetexppd %ymm17, %ymm20
+// CHECK:  encoding: [0x62,0xa2,0xfd,0x28,0x42,0xe1]
+          vgetexppd %ymm17, %ymm20
+
+// CHECK: vgetexppd %ymm17, %ymm20 {%k3}
+// CHECK:  encoding: [0x62,0xa2,0xfd,0x2b,0x42,0xe1]
+          vgetexppd %ymm17, %ymm20 {%k3}
+
+// CHECK: vgetexppd %ymm17, %ymm20 {%k3} {z}
+// CHECK:  encoding: [0x62,0xa2,0xfd,0xab,0x42,0xe1]
+          vgetexppd %ymm17, %ymm20 {%k3} {z}
+
+// CHECK: vgetexppd (%rcx), %ymm20
+// CHECK:  encoding: [0x62,0xe2,0xfd,0x28,0x42,0x21]
+          vgetexppd (%rcx), %ymm20
+
+// CHECK: vgetexppd 291(%rax,%r14,8), %ymm20
+// CHECK:  encoding: [0x62,0xa2,0xfd,0x28,0x42,0xa4,0xf0,0x23,0x01,0x00,0x00]
+          vgetexppd 291(%rax,%r14,8), %ymm20
+
+// CHECK: vgetexppd (%rcx){1to4}, %ymm20
+// CHECK:  encoding: [0x62,0xe2,0xfd,0x38,0x42,0x21]
+          vgetexppd (%rcx){1to4}, %ymm20
+
+// CHECK: vgetexppd 4064(%rdx), %ymm20
+// CHECK:  encoding: [0x62,0xe2,0xfd,0x28,0x42,0x62,0x7f]
+          vgetexppd 4064(%rdx), %ymm20
+
+// CHECK: vgetexppd 4096(%rdx), %ymm20
+// CHECK:  encoding: [0x62,0xe2,0xfd,0x28,0x42,0xa2,0x00,0x10,0x00,0x00]
+          vgetexppd 4096(%rdx), %ymm20
+
+// CHECK: vgetexppd -4096(%rdx), %ymm20
+// CHECK:  encoding: [0x62,0xe2,0xfd,0x28,0x42,0x62,0x80]
+          vgetexppd -4096(%rdx), %ymm20
+
+// CHECK: vgetexppd -4128(%rdx), %ymm20
+// CHECK:  encoding: [0x62,0xe2,0xfd,0x28,0x42,0xa2,0xe0,0xef,0xff,0xff]
+          vgetexppd -4128(%rdx), %ymm20
+
+// CHECK: vgetexppd 1016(%rdx){1to4}, %ymm20
+// CHECK:  encoding: [0x62,0xe2,0xfd,0x38,0x42,0x62,0x7f]
+          vgetexppd 1016(%rdx){1to4}, %ymm20
+
+// CHECK: vgetexppd 1024(%rdx){1to4}, %ymm20
+// CHECK:  encoding: [0x62,0xe2,0xfd,0x38,0x42,0xa2,0x00,0x04,0x00,0x00]
+          vgetexppd 1024(%rdx){1to4}, %ymm20
+
+// CHECK: vgetexppd -1024(%rdx){1to4}, %ymm20
+// CHECK:  encoding: [0x62,0xe2,0xfd,0x38,0x42,0x62,0x80]
+          vgetexppd -1024(%rdx){1to4}, %ymm20
+
+// CHECK: vgetexppd -1032(%rdx){1to4}, %ymm20
+// CHECK:  encoding: [0x62,0xe2,0xfd,0x38,0x42,0xa2,0xf8,0xfb,0xff,0xff]
+          vgetexppd -1032(%rdx){1to4}, %ymm20
+
+// CHECK: vgetexpps %xmm27, %xmm17
+// CHECK:  encoding: [0x62,0x82,0x7d,0x08,0x42,0xcb]
+          vgetexpps %xmm27, %xmm17
+
+// CHECK: vgetexpps %xmm27, %xmm17 {%k2}
+// CHECK:  encoding: [0x62,0x82,0x7d,0x0a,0x42,0xcb]
+          vgetexpps %xmm27, %xmm17 {%k2}
+
+// CHECK: vgetexpps %xmm27, %xmm17 {%k2} {z}
+// CHECK:  encoding: [0x62,0x82,0x7d,0x8a,0x42,0xcb]
+          vgetexpps %xmm27, %xmm17 {%k2} {z}
+
+// CHECK: vgetexpps (%rcx), %xmm17
+// CHECK:  encoding: [0x62,0xe2,0x7d,0x08,0x42,0x09]
+          vgetexpps (%rcx), %xmm17
+
+// CHECK: vgetexpps 291(%rax,%r14,8), %xmm17
+// CHECK:  encoding: [0x62,0xa2,0x7d,0x08,0x42,0x8c,0xf0,0x23,0x01,0x00,0x00]
+          vgetexpps 291(%rax,%r14,8), %xmm17
+
+// CHECK: vgetexpps (%rcx){1to4}, %xmm17
+// CHECK:  encoding: [0x62,0xe2,0x7d,0x18,0x42,0x09]
+          vgetexpps (%rcx){1to4}, %xmm17
+
+// CHECK: vgetexpps 2032(%rdx), %xmm17
+// CHECK:  encoding: [0x62,0xe2,0x7d,0x08,0x42,0x4a,0x7f]
+          vgetexpps 2032(%rdx), %xmm17
+
+// CHECK: vgetexpps 2048(%rdx), %xmm17
+// CHECK:  encoding: [0x62,0xe2,0x7d,0x08,0x42,0x8a,0x00,0x08,0x00,0x00]
+          vgetexpps 2048(%rdx), %xmm17
+
+// CHECK: vgetexpps -2048(%rdx), %xmm17
+// CHECK:  encoding: [0x62,0xe2,0x7d,0x08,0x42,0x4a,0x80]
+          vgetexpps -2048(%rdx), %xmm17
+
+// CHECK: vgetexpps -2064(%rdx), %xmm17
+// CHECK:  encoding: [0x62,0xe2,0x7d,0x08,0x42,0x8a,0xf0,0xf7,0xff,0xff]
+          vgetexpps -2064(%rdx), %xmm17
+
+// CHECK: vgetexpps 508(%rdx){1to4}, %xmm17
+// CHECK:  encoding: [0x62,0xe2,0x7d,0x18,0x42,0x4a,0x7f]
+          vgetexpps 508(%rdx){1to4}, %xmm17
+
+// CHECK: vgetexpps 512(%rdx){1to4}, %xmm17
+// CHECK:  encoding: [0x62,0xe2,0x7d,0x18,0x42,0x8a,0x00,0x02,0x00,0x00]
+          vgetexpps 512(%rdx){1to4}, %xmm17
+
+// CHECK: vgetexpps -512(%rdx){1to4}, %xmm17
+// CHECK:  encoding: [0x62,0xe2,0x7d,0x18,0x42,0x4a,0x80]
+          vgetexpps -512(%rdx){1to4}, %xmm17
+
+// CHECK: vgetexpps -516(%rdx){1to4}, %xmm17
+// CHECK:  encoding: [0x62,0xe2,0x7d,0x18,0x42,0x8a,0xfc,0xfd,0xff,0xff]
+          vgetexpps -516(%rdx){1to4}, %xmm17
+
+// CHECK: vgetexpps %ymm29, %ymm30
+// CHECK:  encoding: [0x62,0x02,0x7d,0x28,0x42,0xf5]
+          vgetexpps %ymm29, %ymm30
+
+// CHECK: vgetexpps %ymm29, %ymm30 {%k6}
+// CHECK:  encoding: [0x62,0x02,0x7d,0x2e,0x42,0xf5]
+          vgetexpps %ymm29, %ymm30 {%k6}
+
+// CHECK: vgetexpps %ymm29, %ymm30 {%k6} {z}
+// CHECK:  encoding: [0x62,0x02,0x7d,0xae,0x42,0xf5]
+          vgetexpps %ymm29, %ymm30 {%k6} {z}
+
+// CHECK: vgetexpps (%rcx), %ymm30
+// CHECK:  encoding: [0x62,0x62,0x7d,0x28,0x42,0x31]
+          vgetexpps (%rcx), %ymm30
+
+// CHECK: vgetexpps 291(%rax,%r14,8), %ymm30
+// CHECK:  encoding: [0x62,0x22,0x7d,0x28,0x42,0xb4,0xf0,0x23,0x01,0x00,0x00]
+          vgetexpps 291(%rax,%r14,8), %ymm30
+
+// CHECK: vgetexpps (%rcx){1to8}, %ymm30
+// CHECK:  encoding: [0x62,0x62,0x7d,0x38,0x42,0x31]
+          vgetexpps (%rcx){1to8}, %ymm30
+
+// CHECK: vgetexpps 4064(%rdx), %ymm30
+// CHECK:  encoding: [0x62,0x62,0x7d,0x28,0x42,0x72,0x7f]
+          vgetexpps 4064(%rdx), %ymm30
+
+// CHECK: vgetexpps 4096(%rdx), %ymm30
+// CHECK:  encoding: [0x62,0x62,0x7d,0x28,0x42,0xb2,0x00,0x10,0x00,0x00]
+          vgetexpps 4096(%rdx), %ymm30
+
+// CHECK: vgetexpps -4096(%rdx), %ymm30
+// CHECK:  encoding: [0x62,0x62,0x7d,0x28,0x42,0x72,0x80]
+          vgetexpps -4096(%rdx), %ymm30
+
+// CHECK: vgetexpps -4128(%rdx), %ymm30
+// CHECK:  encoding: [0x62,0x62,0x7d,0x28,0x42,0xb2,0xe0,0xef,0xff,0xff]
+          vgetexpps -4128(%rdx), %ymm30
+
+// CHECK: vgetexpps 508(%rdx){1to8}, %ymm30
+// CHECK:  encoding: [0x62,0x62,0x7d,0x38,0x42,0x72,0x7f]
+          vgetexpps 508(%rdx){1to8}, %ymm30
+
+// CHECK: vgetexpps 512(%rdx){1to8}, %ymm30
+// CHECK:  encoding: [0x62,0x62,0x7d,0x38,0x42,0xb2,0x00,0x02,0x00,0x00]
+          vgetexpps 512(%rdx){1to8}, %ymm30
+
+// CHECK: vgetexpps -512(%rdx){1to8}, %ymm30
+// CHECK:  encoding: [0x62,0x62,0x7d,0x38,0x42,0x72,0x80]
+          vgetexpps -512(%rdx){1to8}, %ymm30
+
+// CHECK: vgetexpps -516(%rdx){1to8}, %ymm30
+// CHECK:  encoding: [0x62,0x62,0x7d,0x38,0x42,0xb2,0xfc,0xfd,0xff,0xff]
+          vgetexpps -516(%rdx){1to8}, %ymm30
+
+// CHECK: vshuff32x4 $171, %ymm18, %ymm27, %ymm29
+// CHECK:  encoding: [0x62,0x23,0x25,0x20,0x23,0xea,0xab]
+          vshuff32x4 $0xab, %ymm18, %ymm27, %ymm29
+
+// CHECK: vshuff32x4 $171, %ymm18, %ymm27, %ymm29 {%k7}
+// CHECK:  encoding: [0x62,0x23,0x25,0x27,0x23,0xea,0xab]
+          vshuff32x4 $0xab, %ymm18, %ymm27, %ymm29 {%k7}
+
+// CHECK: vshuff32x4 $171, %ymm18, %ymm27, %ymm29 {%k7} {z}
+// CHECK:  encoding: [0x62,0x23,0x25,0xa7,0x23,0xea,0xab]
+          vshuff32x4 $0xab, %ymm18, %ymm27, %ymm29 {%k7} {z}
+
+// CHECK: vshuff32x4 $123, %ymm18, %ymm27, %ymm29
+// CHECK:  encoding: [0x62,0x23,0x25,0x20,0x23,0xea,0x7b]
+          vshuff32x4 $0x7b, %ymm18, %ymm27, %ymm29
+
+// CHECK: vshuff32x4 $123, (%rcx), %ymm27, %ymm29
+// CHECK:  encoding: [0x62,0x63,0x25,0x20,0x23,0x29,0x7b]
+          vshuff32x4 $0x7b, (%rcx), %ymm27, %ymm29
+
+// CHECK: vshuff32x4 $123, 291(%rax,%r14,8), %ymm27, %ymm29
+// CHECK:  encoding: [0x62,0x23,0x25,0x20,0x23,0xac,0xf0,0x23,0x01,0x00,0x00,0x7b]
+          vshuff32x4 $0x7b, 291(%rax,%r14,8), %ymm27, %ymm29
+
+// CHECK: vshuff32x4 $123, (%rcx){1to8}, %ymm27, %ymm29
+// CHECK:  encoding: [0x62,0x63,0x25,0x30,0x23,0x29,0x7b]
+          vshuff32x4 $0x7b, (%rcx){1to8}, %ymm27, %ymm29
+
+// CHECK: vshuff32x4 $123, 4064(%rdx), %ymm27, %ymm29
+// CHECK:  encoding: [0x62,0x63,0x25,0x20,0x23,0x6a,0x7f,0x7b]
+          vshuff32x4 $0x7b, 4064(%rdx), %ymm27, %ymm29
+
+// CHECK: vshuff32x4 $123, 4096(%rdx), %ymm27, %ymm29
+// CHECK:  encoding: [0x62,0x63,0x25,0x20,0x23,0xaa,0x00,0x10,0x00,0x00,0x7b]
+          vshuff32x4 $0x7b, 4096(%rdx), %ymm27, %ymm29
+
+// CHECK: vshuff32x4 $123, -4096(%rdx), %ymm27, %ymm29
+// CHECK:  encoding: [0x62,0x63,0x25,0x20,0x23,0x6a,0x80,0x7b]
+          vshuff32x4 $0x7b, -4096(%rdx), %ymm27, %ymm29
+
+// CHECK: vshuff32x4 $123, -4128(%rdx), %ymm27, %ymm29
+// CHECK:  encoding: [0x62,0x63,0x25,0x20,0x23,0xaa,0xe0,0xef,0xff,0xff,0x7b]
+          vshuff32x4 $0x7b, -4128(%rdx), %ymm27, %ymm29
+
+// CHECK: vshuff32x4 $123, 508(%rdx){1to8}, %ymm27, %ymm29
+// CHECK:  encoding: [0x62,0x63,0x25,0x30,0x23,0x6a,0x7f,0x7b]
+          vshuff32x4 $0x7b, 508(%rdx){1to8}, %ymm27, %ymm29
+
+// CHECK: vshuff32x4 $123, 512(%rdx){1to8}, %ymm27, %ymm29
+// CHECK:  encoding: [0x62,0x63,0x25,0x30,0x23,0xaa,0x00,0x02,0x00,0x00,0x7b]
+          vshuff32x4 $0x7b, 512(%rdx){1to8}, %ymm27, %ymm29
+
+// CHECK: vshuff32x4 $123, -512(%rdx){1to8}, %ymm27, %ymm29
+// CHECK:  encoding: [0x62,0x63,0x25,0x30,0x23,0x6a,0x80,0x7b]
+          vshuff32x4 $0x7b, -512(%rdx){1to8}, %ymm27, %ymm29
+
+// CHECK: vshuff32x4 $123, -516(%rdx){1to8}, %ymm27, %ymm29
+// CHECK:  encoding: [0x62,0x63,0x25,0x30,0x23,0xaa,0xfc,0xfd,0xff,0xff,0x7b]
+          vshuff32x4 $0x7b, -516(%rdx){1to8}, %ymm27, %ymm29
+
+// CHECK: vshuff64x2 $171, %ymm20, %ymm18, %ymm18
+// CHECK:  encoding: [0x62,0xa3,0xed,0x20,0x23,0xd4,0xab]
+          vshuff64x2 $0xab, %ymm20, %ymm18, %ymm18
+
+// CHECK: vshuff64x2 $171, %ymm20, %ymm18, %ymm18 {%k5}
+// CHECK:  encoding: [0x62,0xa3,0xed,0x25,0x23,0xd4,0xab]
+          vshuff64x2 $0xab, %ymm20, %ymm18, %ymm18 {%k5}
+
+// CHECK: vshuff64x2 $171, %ymm20, %ymm18, %ymm18 {%k5} {z}
+// CHECK:  encoding: [0x62,0xa3,0xed,0xa5,0x23,0xd4,0xab]
+          vshuff64x2 $0xab, %ymm20, %ymm18, %ymm18 {%k5} {z}
+
+// CHECK: vshuff64x2 $123, %ymm20, %ymm18, %ymm18
+// CHECK:  encoding: [0x62,0xa3,0xed,0x20,0x23,0xd4,0x7b]
+          vshuff64x2 $0x7b, %ymm20, %ymm18, %ymm18
+
+// CHECK: vshuff64x2 $123, (%rcx), %ymm18, %ymm18
+// CHECK:  encoding: [0x62,0xe3,0xed,0x20,0x23,0x11,0x7b]
+          vshuff64x2 $0x7b, (%rcx), %ymm18, %ymm18
+
+// CHECK: vshuff64x2 $123, 291(%rax,%r14,8), %ymm18, %ymm18
+// CHECK:  encoding: [0x62,0xa3,0xed,0x20,0x23,0x94,0xf0,0x23,0x01,0x00,0x00,0x7b]
+          vshuff64x2 $0x7b, 291(%rax,%r14,8), %ymm18, %ymm18
+
+// CHECK: vshuff64x2 $123, (%rcx){1to4}, %ymm18, %ymm18
+// CHECK:  encoding: [0x62,0xe3,0xed,0x30,0x23,0x11,0x7b]
+          vshuff64x2 $0x7b, (%rcx){1to4}, %ymm18, %ymm18
+
+// CHECK: vshuff64x2 $123, 4064(%rdx), %ymm18, %ymm18
+// CHECK:  encoding: [0x62,0xe3,0xed,0x20,0x23,0x52,0x7f,0x7b]
+          vshuff64x2 $0x7b, 4064(%rdx), %ymm18, %ymm18
+
+// CHECK: vshuff64x2 $123, 4096(%rdx), %ymm18, %ymm18
+// CHECK:  encoding: [0x62,0xe3,0xed,0x20,0x23,0x92,0x00,0x10,0x00,0x00,0x7b]
+          vshuff64x2 $0x7b, 4096(%rdx), %ymm18, %ymm18
+
+// CHECK: vshuff64x2 $123, -4096(%rdx), %ymm18, %ymm18
+// CHECK:  encoding: [0x62,0xe3,0xed,0x20,0x23,0x52,0x80,0x7b]
+          vshuff64x2 $0x7b, -4096(%rdx), %ymm18, %ymm18
+
+// CHECK: vshuff64x2 $123, -4128(%rdx), %ymm18, %ymm18
+// CHECK:  encoding: [0x62,0xe3,0xed,0x20,0x23,0x92,0xe0,0xef,0xff,0xff,0x7b]
+          vshuff64x2 $0x7b, -4128(%rdx), %ymm18, %ymm18
+
+// CHECK: vshuff64x2 $123, 1016(%rdx){1to4}, %ymm18, %ymm18
+// CHECK:  encoding: [0x62,0xe3,0xed,0x30,0x23,0x52,0x7f,0x7b]
+          vshuff64x2 $0x7b, 1016(%rdx){1to4}, %ymm18, %ymm18
+
+// CHECK: vshuff64x2 $123, 1024(%rdx){1to4}, %ymm18, %ymm18
+// CHECK:  encoding: [0x62,0xe3,0xed,0x30,0x23,0x92,0x00,0x04,0x00,0x00,0x7b]
+          vshuff64x2 $0x7b, 1024(%rdx){1to4}, %ymm18, %ymm18
+
+// CHECK: vshuff64x2 $123, -1024(%rdx){1to4}, %ymm18, %ymm18
+// CHECK:  encoding: [0x62,0xe3,0xed,0x30,0x23,0x52,0x80,0x7b]
+          vshuff64x2 $0x7b, -1024(%rdx){1to4}, %ymm18, %ymm18
+
+// CHECK: vshuff64x2 $123, -1032(%rdx){1to4}, %ymm18, %ymm18
+// CHECK:  encoding: [0x62,0xe3,0xed,0x30,0x23,0x92,0xf8,0xfb,0xff,0xff,0x7b]
+          vshuff64x2 $0x7b, -1032(%rdx){1to4}, %ymm18, %ymm18
+
+// CHECK: vshufi32x4 $171, %ymm17, %ymm27, %ymm18
+// CHECK:  encoding: [0x62,0xa3,0x25,0x20,0x43,0xd1,0xab]
+          vshufi32x4 $0xab, %ymm17, %ymm27, %ymm18
+
+// CHECK: vshufi32x4 $171, %ymm17, %ymm27, %ymm18 {%k7}
+// CHECK:  encoding: [0x62,0xa3,0x25,0x27,0x43,0xd1,0xab]
+          vshufi32x4 $0xab, %ymm17, %ymm27, %ymm18 {%k7}
+
+// CHECK: vshufi32x4 $171, %ymm17, %ymm27, %ymm18 {%k7} {z}
+// CHECK:  encoding: [0x62,0xa3,0x25,0xa7,0x43,0xd1,0xab]
+          vshufi32x4 $0xab, %ymm17, %ymm27, %ymm18 {%k7} {z}
+
+// CHECK: vshufi32x4 $123, %ymm17, %ymm27, %ymm18
+// CHECK:  encoding: [0x62,0xa3,0x25,0x20,0x43,0xd1,0x7b]
+          vshufi32x4 $0x7b, %ymm17, %ymm27, %ymm18
+
+// CHECK: vshufi32x4 $123, (%rcx), %ymm27, %ymm18
+// CHECK:  encoding: [0x62,0xe3,0x25,0x20,0x43,0x11,0x7b]
+          vshufi32x4 $0x7b, (%rcx), %ymm27, %ymm18
+
+// CHECK: vshufi32x4 $123, 291(%rax,%r14,8), %ymm27, %ymm18
+// CHECK:  encoding: [0x62,0xa3,0x25,0x20,0x43,0x94,0xf0,0x23,0x01,0x00,0x00,0x7b]
+          vshufi32x4 $0x7b, 291(%rax,%r14,8), %ymm27, %ymm18
+
+// CHECK: vshufi32x4 $123, (%rcx){1to8}, %ymm27, %ymm18
+// CHECK:  encoding: [0x62,0xe3,0x25,0x30,0x43,0x11,0x7b]
+          vshufi32x4 $0x7b, (%rcx){1to8}, %ymm27, %ymm18
+
+// CHECK: vshufi32x4 $123, 4064(%rdx), %ymm27, %ymm18
+// CHECK:  encoding: [0x62,0xe3,0x25,0x20,0x43,0x52,0x7f,0x7b]
+          vshufi32x4 $0x7b, 4064(%rdx), %ymm27, %ymm18
+
+// CHECK: vshufi32x4 $123, 4096(%rdx), %ymm27, %ymm18
+// CHECK:  encoding: [0x62,0xe3,0x25,0x20,0x43,0x92,0x00,0x10,0x00,0x00,0x7b]
+          vshufi32x4 $0x7b, 4096(%rdx), %ymm27, %ymm18
+
+// CHECK: vshufi32x4 $123, -4096(%rdx), %ymm27, %ymm18
+// CHECK:  encoding: [0x62,0xe3,0x25,0x20,0x43,0x52,0x80,0x7b]
+          vshufi32x4 $0x7b, -4096(%rdx), %ymm27, %ymm18
+
+// CHECK: vshufi32x4 $123, -4128(%rdx), %ymm27, %ymm18
+// CHECK:  encoding: [0x62,0xe3,0x25,0x20,0x43,0x92,0xe0,0xef,0xff,0xff,0x7b]
+          vshufi32x4 $0x7b, -4128(%rdx), %ymm27, %ymm18
+
+// CHECK: vshufi32x4 $123, 508(%rdx){1to8}, %ymm27, %ymm18
+// CHECK:  encoding: [0x62,0xe3,0x25,0x30,0x43,0x52,0x7f,0x7b]
+          vshufi32x4 $0x7b, 508(%rdx){1to8}, %ymm27, %ymm18
+
+// CHECK: vshufi32x4 $123, 512(%rdx){1to8}, %ymm27, %ymm18
+// CHECK:  encoding: [0x62,0xe3,0x25,0x30,0x43,0x92,0x00,0x02,0x00,0x00,0x7b]
+          vshufi32x4 $0x7b, 512(%rdx){1to8}, %ymm27, %ymm18
+
+// CHECK: vshufi32x4 $123, -512(%rdx){1to8}, %ymm27, %ymm18
+// CHECK:  encoding: [0x62,0xe3,0x25,0x30,0x43,0x52,0x80,0x7b]
+          vshufi32x4 $0x7b, -512(%rdx){1to8}, %ymm27, %ymm18
+
+// CHECK: vshufi32x4 $123, -516(%rdx){1to8}, %ymm27, %ymm18
+// CHECK:  encoding: [0x62,0xe3,0x25,0x30,0x43,0x92,0xfc,0xfd,0xff,0xff,0x7b]
+          vshufi32x4 $0x7b, -516(%rdx){1to8}, %ymm27, %ymm18
+
+// CHECK: vshufi64x2 $171, %ymm21, %ymm26, %ymm25
+// CHECK:  encoding: [0x62,0x23,0xad,0x20,0x43,0xcd,0xab]
+          vshufi64x2 $0xab, %ymm21, %ymm26, %ymm25
+
+// CHECK: vshufi64x2 $171, %ymm21, %ymm26, %ymm25 {%k3}
+// CHECK:  encoding: [0x62,0x23,0xad,0x23,0x43,0xcd,0xab]
+          vshufi64x2 $0xab, %ymm21, %ymm26, %ymm25 {%k3}
+
+// CHECK: vshufi64x2 $171, %ymm21, %ymm26, %ymm25 {%k3} {z}
+// CHECK:  encoding: [0x62,0x23,0xad,0xa3,0x43,0xcd,0xab]
+          vshufi64x2 $0xab, %ymm21, %ymm26, %ymm25 {%k3} {z}
+
+// CHECK: vshufi64x2 $123, %ymm21, %ymm26, %ymm25
+// CHECK:  encoding: [0x62,0x23,0xad,0x20,0x43,0xcd,0x7b]
+          vshufi64x2 $0x7b, %ymm21, %ymm26, %ymm25
+
+// CHECK: vshufi64x2 $123, (%rcx), %ymm26, %ymm25
+// CHECK:  encoding: [0x62,0x63,0xad,0x20,0x43,0x09,0x7b]
+          vshufi64x2 $0x7b, (%rcx), %ymm26, %ymm25
+
+// CHECK: vshufi64x2 $123, 291(%rax,%r14,8), %ymm26, %ymm25
+// CHECK:  encoding: [0x62,0x23,0xad,0x20,0x43,0x8c,0xf0,0x23,0x01,0x00,0x00,0x7b]
+          vshufi64x2 $0x7b, 291(%rax,%r14,8), %ymm26, %ymm25
+
+// CHECK: vshufi64x2 $123, (%rcx){1to4}, %ymm26, %ymm25
+// CHECK:  encoding: [0x62,0x63,0xad,0x30,0x43,0x09,0x7b]
+          vshufi64x2 $0x7b, (%rcx){1to4}, %ymm26, %ymm25
+
+// CHECK: vshufi64x2 $123, 4064(%rdx), %ymm26, %ymm25
+// CHECK:  encoding: [0x62,0x63,0xad,0x20,0x43,0x4a,0x7f,0x7b]
+          vshufi64x2 $0x7b, 4064(%rdx), %ymm26, %ymm25
+
+// CHECK: vshufi64x2 $123, 4096(%rdx), %ymm26, %ymm25
+// CHECK:  encoding: [0x62,0x63,0xad,0x20,0x43,0x8a,0x00,0x10,0x00,0x00,0x7b]
+          vshufi64x2 $0x7b, 4096(%rdx), %ymm26, %ymm25
+
+// CHECK: vshufi64x2 $123, -4096(%rdx), %ymm26, %ymm25
+// CHECK:  encoding: [0x62,0x63,0xad,0x20,0x43,0x4a,0x80,0x7b]
+          vshufi64x2 $0x7b, -4096(%rdx), %ymm26, %ymm25
+
+// CHECK: vshufi64x2 $123, -4128(%rdx), %ymm26, %ymm25
+// CHECK:  encoding: [0x62,0x63,0xad,0x20,0x43,0x8a,0xe0,0xef,0xff,0xff,0x7b]
+          vshufi64x2 $0x7b, -4128(%rdx), %ymm26, %ymm25
+
+// CHECK: vshufi64x2 $123, 1016(%rdx){1to4}, %ymm26, %ymm25
+// CHECK:  encoding: [0x62,0x63,0xad,0x30,0x43,0x4a,0x7f,0x7b]
+          vshufi64x2 $0x7b, 1016(%rdx){1to4}, %ymm26, %ymm25
+
+// CHECK: vshufi64x2 $123, 1024(%rdx){1to4}, %ymm26, %ymm25
+// CHECK:  encoding: [0x62,0x63,0xad,0x30,0x43,0x8a,0x00,0x04,0x00,0x00,0x7b]
+          vshufi64x2 $0x7b, 1024(%rdx){1to4}, %ymm26, %ymm25
+
+// CHECK: vshufi64x2 $123, -1024(%rdx){1to4}, %ymm26, %ymm25
+// CHECK:  encoding: [0x62,0x63,0xad,0x30,0x43,0x4a,0x80,0x7b]
+          vshufi64x2 $0x7b, -1024(%rdx){1to4}, %ymm26, %ymm25
+
+// CHECK: vshufi64x2 $123, -1032(%rdx){1to4}, %ymm26, %ymm25
+// CHECK:  encoding: [0x62,0x63,0xad,0x30,0x43,0x8a,0xf8,0xfb,0xff,0xff,0x7b]
+          vshufi64x2 $0x7b, -1032(%rdx){1to4}, %ymm26, %ymm25
+
+// CHECK: valignq $171, %xmm24, %xmm18, %xmm19
+// CHECK:  encoding: [0x62,0x83,0xed,0x00,0x03,0xd8,0xab]
+          valignq $0xab, %xmm24, %xmm18, %xmm19
+
+// CHECK: valignq $171, %xmm24, %xmm18, %xmm19 {%k5}
+// CHECK:  encoding: [0x62,0x83,0xed,0x05,0x03,0xd8,0xab]
+          valignq $0xab, %xmm24, %xmm18, %xmm19 {%k5}
+
+// CHECK: valignq $171, %xmm24, %xmm18, %xmm19 {%k5} {z}
+// CHECK:  encoding: [0x62,0x83,0xed,0x85,0x03,0xd8,0xab]
+          valignq $0xab, %xmm24, %xmm18, %xmm19 {%k5} {z}
+
+// CHECK: valignq $123, %xmm24, %xmm18, %xmm19
+// CHECK:  encoding: [0x62,0x83,0xed,0x00,0x03,0xd8,0x7b]
+          valignq $0x7b, %xmm24, %xmm18, %xmm19
+
+// CHECK: valignq $123, (%rcx), %xmm18, %xmm19
+// CHECK:  encoding: [0x62,0xe3,0xed,0x00,0x03,0x19,0x7b]
+          valignq $0x7b, (%rcx), %xmm18, %xmm19
+
+// CHECK: valignq $123, 291(%rax,%r14,8), %xmm18, %xmm19
+// CHECK:  encoding: [0x62,0xa3,0xed,0x00,0x03,0x9c,0xf0,0x23,0x01,0x00,0x00,0x7b]
+          valignq $0x7b, 291(%rax,%r14,8), %xmm18, %xmm19
+
+// CHECK: valignq $123, (%rcx){1to2}, %xmm18, %xmm19
+// CHECK:  encoding: [0x62,0xe3,0xed,0x10,0x03,0x19,0x7b]
+          valignq $0x7b, (%rcx){1to2}, %xmm18, %xmm19
+
+// CHECK: valignq $123, 2032(%rdx), %xmm18, %xmm19
+// CHECK:  encoding: [0x62,0xe3,0xed,0x00,0x03,0x5a,0x7f,0x7b]
+          valignq $0x7b, 2032(%rdx), %xmm18, %xmm19
+
+// CHECK: valignq $123, 2048(%rdx), %xmm18, %xmm19
+// CHECK:  encoding: [0x62,0xe3,0xed,0x00,0x03,0x9a,0x00,0x08,0x00,0x00,0x7b]
+          valignq $0x7b, 2048(%rdx), %xmm18, %xmm19
+
+// CHECK: valignq $123, -2048(%rdx), %xmm18, %xmm19
+// CHECK:  encoding: [0x62,0xe3,0xed,0x00,0x03,0x5a,0x80,0x7b]
+          valignq $0x7b, -2048(%rdx), %xmm18, %xmm19
+
+// CHECK: valignq $123, -2064(%rdx), %xmm18, %xmm19
+// CHECK:  encoding: [0x62,0xe3,0xed,0x00,0x03,0x9a,0xf0,0xf7,0xff,0xff,0x7b]
+          valignq $0x7b, -2064(%rdx), %xmm18, %xmm19
+
+// CHECK: valignq $123, 1016(%rdx){1to2}, %xmm18, %xmm19
+// CHECK:  encoding: [0x62,0xe3,0xed,0x10,0x03,0x5a,0x7f,0x7b]
+          valignq $0x7b, 1016(%rdx){1to2}, %xmm18, %xmm19
+
+// CHECK: valignq $123, 1024(%rdx){1to2}, %xmm18, %xmm19
+// CHECK:  encoding: [0x62,0xe3,0xed,0x10,0x03,0x9a,0x00,0x04,0x00,0x00,0x7b]
+          valignq $0x7b, 1024(%rdx){1to2}, %xmm18, %xmm19
+
+// CHECK: valignq $123, -1024(%rdx){1to2}, %xmm18, %xmm19
+// CHECK:  encoding: [0x62,0xe3,0xed,0x10,0x03,0x5a,0x80,0x7b]
+          valignq $0x7b, -1024(%rdx){1to2}, %xmm18, %xmm19
+
+// CHECK: valignq $123, -1032(%rdx){1to2}, %xmm18, %xmm19
+// CHECK:  encoding: [0x62,0xe3,0xed,0x10,0x03,0x9a,0xf8,0xfb,0xff,0xff,0x7b]
+          valignq $0x7b, -1032(%rdx){1to2}, %xmm18, %xmm19
+
+// CHECK: valignq $171, %ymm26, %ymm24, %ymm25
+// CHECK:  encoding: [0x62,0x03,0xbd,0x20,0x03,0xca,0xab]
+          valignq $0xab, %ymm26, %ymm24, %ymm25
+
+// CHECK: valignq $171, %ymm26, %ymm24, %ymm25 {%k2}
+// CHECK:  encoding: [0x62,0x03,0xbd,0x22,0x03,0xca,0xab]
+          valignq $0xab, %ymm26, %ymm24, %ymm25 {%k2}
+
+// CHECK: valignq $171, %ymm26, %ymm24, %ymm25 {%k2} {z}
+// CHECK:  encoding: [0x62,0x03,0xbd,0xa2,0x03,0xca,0xab]
+          valignq $0xab, %ymm26, %ymm24, %ymm25 {%k2} {z}
+
+// CHECK: valignq $123, %ymm26, %ymm24, %ymm25
+// CHECK:  encoding: [0x62,0x03,0xbd,0x20,0x03,0xca,0x7b]
+          valignq $0x7b, %ymm26, %ymm24, %ymm25
+
+// CHECK: valignq $123, (%rcx), %ymm24, %ymm25
+// CHECK:  encoding: [0x62,0x63,0xbd,0x20,0x03,0x09,0x7b]
+          valignq $0x7b, (%rcx), %ymm24, %ymm25
+
+// CHECK: valignq $123, 291(%rax,%r14,8), %ymm24, %ymm25
+// CHECK:  encoding: [0x62,0x23,0xbd,0x20,0x03,0x8c,0xf0,0x23,0x01,0x00,0x00,0x7b]
+          valignq $0x7b, 291(%rax,%r14,8), %ymm24, %ymm25
+
+// CHECK: valignq $123, (%rcx){1to4}, %ymm24, %ymm25
+// CHECK:  encoding: [0x62,0x63,0xbd,0x30,0x03,0x09,0x7b]
+          valignq $0x7b, (%rcx){1to4}, %ymm24, %ymm25
+
+// CHECK: valignq $123, 4064(%rdx), %ymm24, %ymm25
+// CHECK:  encoding: [0x62,0x63,0xbd,0x20,0x03,0x4a,0x7f,0x7b]
+          valignq $0x7b, 4064(%rdx), %ymm24, %ymm25
+
+// CHECK: valignq $123, 4096(%rdx), %ymm24, %ymm25
+// CHECK:  encoding: [0x62,0x63,0xbd,0x20,0x03,0x8a,0x00,0x10,0x00,0x00,0x7b]
+          valignq $0x7b, 4096(%rdx), %ymm24, %ymm25
+
+// CHECK: valignq $123, -4096(%rdx), %ymm24, %ymm25
+// CHECK:  encoding: [0x62,0x63,0xbd,0x20,0x03,0x4a,0x80,0x7b]
+          valignq $0x7b, -4096(%rdx), %ymm24, %ymm25
+
+// CHECK: valignq $123, -4128(%rdx), %ymm24, %ymm25
+// CHECK:  encoding: [0x62,0x63,0xbd,0x20,0x03,0x8a,0xe0,0xef,0xff,0xff,0x7b]
+          valignq $0x7b, -4128(%rdx), %ymm24, %ymm25
+
+// CHECK: valignq $123, 1016(%rdx){1to4}, %ymm24, %ymm25
+// CHECK:  encoding: [0x62,0x63,0xbd,0x30,0x03,0x4a,0x7f,0x7b]
+          valignq $0x7b, 1016(%rdx){1to4}, %ymm24, %ymm25
+
+// CHECK: valignq $123, 1024(%rdx){1to4}, %ymm24, %ymm25
+// CHECK:  encoding: [0x62,0x63,0xbd,0x30,0x03,0x8a,0x00,0x04,0x00,0x00,0x7b]
+          valignq $0x7b, 1024(%rdx){1to4}, %ymm24, %ymm25
+
+// CHECK: valignq $123, -1024(%rdx){1to4}, %ymm24, %ymm25
+// CHECK:  encoding: [0x62,0x63,0xbd,0x30,0x03,0x4a,0x80,0x7b]
+          valignq $0x7b, -1024(%rdx){1to4}, %ymm24, %ymm25
+
+// CHECK: valignq $123, -1032(%rdx){1to4}, %ymm24, %ymm25
+// CHECK:  encoding: [0x62,0x63,0xbd,0x30,0x03,0x8a,0xf8,0xfb,0xff,0xff,0x7b]
+          valignq $0x7b, -1032(%rdx){1to4}, %ymm24, %ymm25
+
+// CHECK: vfmadd132ps %xmm19, %xmm25, %xmm29
+// CHECK:  encoding: [0x62,0x22,0x35,0x00,0x98,0xeb]
+          vfmadd132ps %xmm19, %xmm25, %xmm29
+
+// CHECK: vfmadd132ps %xmm19, %xmm25, %xmm29 {%k4}
+// CHECK:  encoding: [0x62,0x22,0x35,0x04,0x98,0xeb]
+          vfmadd132ps %xmm19, %xmm25, %xmm29 {%k4}
+
+// CHECK: vfmadd132ps %xmm19, %xmm25, %xmm29 {%k4} {z}
+// CHECK:  encoding: [0x62,0x22,0x35,0x84,0x98,0xeb]
+          vfmadd132ps %xmm19, %xmm25, %xmm29 {%k4} {z}
+
+// CHECK: vfmadd132ps (%rcx), %xmm25, %xmm29
+// CHECK:  encoding: [0x62,0x62,0x35,0x00,0x98,0x29]
+          vfmadd132ps (%rcx), %xmm25, %xmm29
+
+// CHECK: vfmadd132ps 291(%rax,%r14,8), %xmm25, %xmm29
+// CHECK:  encoding: [0x62,0x22,0x35,0x00,0x98,0xac,0xf0,0x23,0x01,0x00,0x00]
+          vfmadd132ps 291(%rax,%r14,8), %xmm25, %xmm29
+
+// CHECK: vfmadd132ps (%rcx){1to4}, %xmm25, %xmm29
+// CHECK:  encoding: [0x62,0x62,0x35,0x10,0x98,0x29]
+          vfmadd132ps (%rcx){1to4}, %xmm25, %xmm29
+
+// CHECK: vfmadd132ps 2032(%rdx), %xmm25, %xmm29
+// CHECK:  encoding: [0x62,0x62,0x35,0x00,0x98,0x6a,0x7f]
+          vfmadd132ps 2032(%rdx), %xmm25, %xmm29
+
+// CHECK: vfmadd132ps 2048(%rdx), %xmm25, %xmm29
+// CHECK:  encoding: [0x62,0x62,0x35,0x00,0x98,0xaa,0x00,0x08,0x00,0x00]
+          vfmadd132ps 2048(%rdx), %xmm25, %xmm29
+
+// CHECK: vfmadd132ps -2048(%rdx), %xmm25, %xmm29
+// CHECK:  encoding: [0x62,0x62,0x35,0x00,0x98,0x6a,0x80]
+          vfmadd132ps -2048(%rdx), %xmm25, %xmm29
+
+// CHECK: vfmadd132ps -2064(%rdx), %xmm25, %xmm29
+// CHECK:  encoding: [0x62,0x62,0x35,0x00,0x98,0xaa,0xf0,0xf7,0xff,0xff]
+          vfmadd132ps -2064(%rdx), %xmm25, %xmm29
+
+// CHECK: vfmadd132ps 508(%rdx){1to4}, %xmm25, %xmm29
+// CHECK:  encoding: [0x62,0x62,0x35,0x10,0x98,0x6a,0x7f]
+          vfmadd132ps 508(%rdx){1to4}, %xmm25, %xmm29
+
+// CHECK: vfmadd132ps 512(%rdx){1to4}, %xmm25, %xmm29
+// CHECK:  encoding: [0x62,0x62,0x35,0x10,0x98,0xaa,0x00,0x02,0x00,0x00]
+          vfmadd132ps 512(%rdx){1to4}, %xmm25, %xmm29
+
+// CHECK: vfmadd132ps -512(%rdx){1to4}, %xmm25, %xmm29
+// CHECK:  encoding: [0x62,0x62,0x35,0x10,0x98,0x6a,0x80]
+          vfmadd132ps -512(%rdx){1to4}, %xmm25, %xmm29
+
+// CHECK: vfmadd132ps -516(%rdx){1to4}, %xmm25, %xmm29
+// CHECK:  encoding: [0x62,0x62,0x35,0x10,0x98,0xaa,0xfc,0xfd,0xff,0xff]
+          vfmadd132ps -516(%rdx){1to4}, %xmm25, %xmm29
+
+// CHECK: vfmadd132ps %ymm26, %ymm22, %ymm23
+// CHECK:  encoding: [0x62,0x82,0x4d,0x20,0x98,0xfa]
+          vfmadd132ps %ymm26, %ymm22, %ymm23
+
+// CHECK: vfmadd132ps %ymm26, %ymm22, %ymm23 {%k5}
+// CHECK:  encoding: [0x62,0x82,0x4d,0x25,0x98,0xfa]
+          vfmadd132ps %ymm26, %ymm22, %ymm23 {%k5}
+
+// CHECK: vfmadd132ps %ymm26, %ymm22, %ymm23 {%k5} {z}
+// CHECK:  encoding: [0x62,0x82,0x4d,0xa5,0x98,0xfa]
+          vfmadd132ps %ymm26, %ymm22, %ymm23 {%k5} {z}
+
+// CHECK: vfmadd132ps (%rcx), %ymm22, %ymm23
+// CHECK:  encoding: [0x62,0xe2,0x4d,0x20,0x98,0x39]
+          vfmadd132ps (%rcx), %ymm22, %ymm23
+
+// CHECK: vfmadd132ps 291(%rax,%r14,8), %ymm22, %ymm23
+// CHECK:  encoding: [0x62,0xa2,0x4d,0x20,0x98,0xbc,0xf0,0x23,0x01,0x00,0x00]
+          vfmadd132ps 291(%rax,%r14,8), %ymm22, %ymm23
+
+// CHECK: vfmadd132ps (%rcx){1to8}, %ymm22, %ymm23
+// CHECK:  encoding: [0x62,0xe2,0x4d,0x30,0x98,0x39]
+          vfmadd132ps (%rcx){1to8}, %ymm22, %ymm23
+
+// CHECK: vfmadd132ps 4064(%rdx), %ymm22, %ymm23
+// CHECK:  encoding: [0x62,0xe2,0x4d,0x20,0x98,0x7a,0x7f]
+          vfmadd132ps 4064(%rdx), %ymm22, %ymm23
+
+// CHECK: vfmadd132ps 4096(%rdx), %ymm22, %ymm23
+// CHECK:  encoding: [0x62,0xe2,0x4d,0x20,0x98,0xba,0x00,0x10,0x00,0x00]
+          vfmadd132ps 4096(%rdx), %ymm22, %ymm23
+
+// CHECK: vfmadd132ps -4096(%rdx), %ymm22, %ymm23
+// CHECK:  encoding: [0x62,0xe2,0x4d,0x20,0x98,0x7a,0x80]
+          vfmadd132ps -4096(%rdx), %ymm22, %ymm23
+
+// CHECK: vfmadd132ps -4128(%rdx), %ymm22, %ymm23
+// CHECK:  encoding: [0x62,0xe2,0x4d,0x20,0x98,0xba,0xe0,0xef,0xff,0xff]
+          vfmadd132ps -4128(%rdx), %ymm22, %ymm23
+
+// CHECK: vfmadd132ps 508(%rdx){1to8}, %ymm22, %ymm23
+// CHECK:  encoding: [0x62,0xe2,0x4d,0x30,0x98,0x7a,0x7f]
+          vfmadd132ps 508(%rdx){1to8}, %ymm22, %ymm23
+
+// CHECK: vfmadd132ps 512(%rdx){1to8}, %ymm22, %ymm23
+// CHECK:  encoding: [0x62,0xe2,0x4d,0x30,0x98,0xba,0x00,0x02,0x00,0x00]
+          vfmadd132ps 512(%rdx){1to8}, %ymm22, %ymm23
+
+// CHECK: vfmadd132ps -512(%rdx){1to8}, %ymm22, %ymm23
+// CHECK:  encoding: [0x62,0xe2,0x4d,0x30,0x98,0x7a,0x80]
+          vfmadd132ps -512(%rdx){1to8}, %ymm22, %ymm23
+
+// CHECK: vfmadd132ps -516(%rdx){1to8}, %ymm22, %ymm23
+// CHECK:  encoding: [0x62,0xe2,0x4d,0x30,0x98,0xba,0xfc,0xfd,0xff,0xff]
+          vfmadd132ps -516(%rdx){1to8}, %ymm22, %ymm23
+
+// CHECK: vfmadd132pd %xmm27, %xmm21, %xmm28
+// CHECK:  encoding: [0x62,0x02,0xd5,0x00,0x98,0xe3]
+          vfmadd132pd %xmm27, %xmm21, %xmm28
+
+// CHECK: vfmadd132pd %xmm27, %xmm21, %xmm28 {%k1}
+// CHECK:  encoding: [0x62,0x02,0xd5,0x01,0x98,0xe3]
+          vfmadd132pd %xmm27, %xmm21, %xmm28 {%k1}
+
+// CHECK: vfmadd132pd %xmm27, %xmm21, %xmm28 {%k1} {z}
+// CHECK:  encoding: [0x62,0x02,0xd5,0x81,0x98,0xe3]
+          vfmadd132pd %xmm27, %xmm21, %xmm28 {%k1} {z}
+
+// CHECK: vfmadd132pd (%rcx), %xmm21, %xmm28
+// CHECK:  encoding: [0x62,0x62,0xd5,0x00,0x98,0x21]
+          vfmadd132pd (%rcx), %xmm21, %xmm28
+
+// CHECK: vfmadd132pd 291(%rax,%r14,8), %xmm21, %xmm28
+// CHECK:  encoding: [0x62,0x22,0xd5,0x00,0x98,0xa4,0xf0,0x23,0x01,0x00,0x00]
+          vfmadd132pd 291(%rax,%r14,8), %xmm21, %xmm28
+
+// CHECK: vfmadd132pd (%rcx){1to2}, %xmm21, %xmm28
+// CHECK:  encoding: [0x62,0x62,0xd5,0x10,0x98,0x21]
+          vfmadd132pd (%rcx){1to2}, %xmm21, %xmm28
+
+// CHECK: vfmadd132pd 2032(%rdx), %xmm21, %xmm28
+// CHECK:  encoding: [0x62,0x62,0xd5,0x00,0x98,0x62,0x7f]
+          vfmadd132pd 2032(%rdx), %xmm21, %xmm28
+
+// CHECK: vfmadd132pd 2048(%rdx), %xmm21, %xmm28
+// CHECK:  encoding: [0x62,0x62,0xd5,0x00,0x98,0xa2,0x00,0x08,0x00,0x00]
+          vfmadd132pd 2048(%rdx), %xmm21, %xmm28
+
+// CHECK: vfmadd132pd -2048(%rdx), %xmm21, %xmm28
+// CHECK:  encoding: [0x62,0x62,0xd5,0x00,0x98,0x62,0x80]
+          vfmadd132pd -2048(%rdx), %xmm21, %xmm28
+
+// CHECK: vfmadd132pd -2064(%rdx), %xmm21, %xmm28
+// CHECK:  encoding: [0x62,0x62,0xd5,0x00,0x98,0xa2,0xf0,0xf7,0xff,0xff]
+          vfmadd132pd -2064(%rdx), %xmm21, %xmm28
+
+// CHECK: vfmadd132pd 1016(%rdx){1to2}, %xmm21, %xmm28
+// CHECK:  encoding: [0x62,0x62,0xd5,0x10,0x98,0x62,0x7f]
+          vfmadd132pd 1016(%rdx){1to2}, %xmm21, %xmm28
+
+// CHECK: vfmadd132pd 1024(%rdx){1to2}, %xmm21, %xmm28
+// CHECK:  encoding: [0x62,0x62,0xd5,0x10,0x98,0xa2,0x00,0x04,0x00,0x00]
+          vfmadd132pd 1024(%rdx){1to2}, %xmm21, %xmm28
+
+// CHECK: vfmadd132pd -1024(%rdx){1to2}, %xmm21, %xmm28
+// CHECK:  encoding: [0x62,0x62,0xd5,0x10,0x98,0x62,0x80]
+          vfmadd132pd -1024(%rdx){1to2}, %xmm21, %xmm28
+
+// CHECK: vfmadd132pd -1032(%rdx){1to2}, %xmm21, %xmm28
+// CHECK:  encoding: [0x62,0x62,0xd5,0x10,0x98,0xa2,0xf8,0xfb,0xff,0xff]
+          vfmadd132pd -1032(%rdx){1to2}, %xmm21, %xmm28
+
+// CHECK: vfmadd132pd %ymm27, %ymm24, %ymm22
+// CHECK:  encoding: [0x62,0x82,0xbd,0x20,0x98,0xf3]
+          vfmadd132pd %ymm27, %ymm24, %ymm22
+
+// CHECK: vfmadd132pd %ymm27, %ymm24, %ymm22 {%k7}
+// CHECK:  encoding: [0x62,0x82,0xbd,0x27,0x98,0xf3]
+          vfmadd132pd %ymm27, %ymm24, %ymm22 {%k7}
+
+// CHECK: vfmadd132pd %ymm27, %ymm24, %ymm22 {%k7} {z}
+// CHECK:  encoding: [0x62,0x82,0xbd,0xa7,0x98,0xf3]
+          vfmadd132pd %ymm27, %ymm24, %ymm22 {%k7} {z}
+
+// CHECK: vfmadd132pd (%rcx), %ymm24, %ymm22
+// CHECK:  encoding: [0x62,0xe2,0xbd,0x20,0x98,0x31]
+          vfmadd132pd (%rcx), %ymm24, %ymm22
+
+// CHECK: vfmadd132pd 291(%rax,%r14,8), %ymm24, %ymm22
+// CHECK:  encoding: [0x62,0xa2,0xbd,0x20,0x98,0xb4,0xf0,0x23,0x01,0x00,0x00]
+          vfmadd132pd 291(%rax,%r14,8), %ymm24, %ymm22
+
+// CHECK: vfmadd132pd (%rcx){1to4}, %ymm24, %ymm22
+// CHECK:  encoding: [0x62,0xe2,0xbd,0x30,0x98,0x31]
+          vfmadd132pd (%rcx){1to4}, %ymm24, %ymm22
+
+// CHECK: vfmadd132pd 4064(%rdx), %ymm24, %ymm22
+// CHECK:  encoding: [0x62,0xe2,0xbd,0x20,0x98,0x72,0x7f]
+          vfmadd132pd 4064(%rdx), %ymm24, %ymm22
+
+// CHECK: vfmadd132pd 4096(%rdx), %ymm24, %ymm22
+// CHECK:  encoding: [0x62,0xe2,0xbd,0x20,0x98,0xb2,0x00,0x10,0x00,0x00]
+          vfmadd132pd 4096(%rdx), %ymm24, %ymm22
+
+// CHECK: vfmadd132pd -4096(%rdx), %ymm24, %ymm22
+// CHECK:  encoding: [0x62,0xe2,0xbd,0x20,0x98,0x72,0x80]
+          vfmadd132pd -4096(%rdx), %ymm24, %ymm22
+
+// CHECK: vfmadd132pd -4128(%rdx), %ymm24, %ymm22
+// CHECK:  encoding: [0x62,0xe2,0xbd,0x20,0x98,0xb2,0xe0,0xef,0xff,0xff]
+          vfmadd132pd -4128(%rdx), %ymm24, %ymm22
+
+// CHECK: vfmadd132pd 1016(%rdx){1to4}, %ymm24, %ymm22
+// CHECK:  encoding: [0x62,0xe2,0xbd,0x30,0x98,0x72,0x7f]
+          vfmadd132pd 1016(%rdx){1to4}, %ymm24, %ymm22
+
+// CHECK: vfmadd132pd 1024(%rdx){1to4}, %ymm24, %ymm22
+// CHECK:  encoding: [0x62,0xe2,0xbd,0x30,0x98,0xb2,0x00,0x04,0x00,0x00]
+          vfmadd132pd 1024(%rdx){1to4}, %ymm24, %ymm22
+
+// CHECK: vfmadd132pd -1024(%rdx){1to4}, %ymm24, %ymm22
+// CHECK:  encoding: [0x62,0xe2,0xbd,0x30,0x98,0x72,0x80]
+          vfmadd132pd -1024(%rdx){1to4}, %ymm24, %ymm22
+
+// CHECK: vfmadd132pd -1032(%rdx){1to4}, %ymm24, %ymm22
+// CHECK:  encoding: [0x62,0xe2,0xbd,0x30,0x98,0xb2,0xf8,0xfb,0xff,0xff]
+          vfmadd132pd -1032(%rdx){1to4}, %ymm24, %ymm22
+
+// CHECK: vfmadd213ps %xmm28, %xmm28, %xmm24
+// CHECK:  encoding: [0x62,0x02,0x1d,0x00,0xa8,0xc4]
+          vfmadd213ps %xmm28, %xmm28, %xmm24
+
+// CHECK: vfmadd213ps %xmm28, %xmm28, %xmm24 {%k1}
+// CHECK:  encoding: [0x62,0x02,0x1d,0x01,0xa8,0xc4]
+          vfmadd213ps %xmm28, %xmm28, %xmm24 {%k1}
+
+// CHECK: vfmadd213ps %xmm28, %xmm28, %xmm24 {%k1} {z}
+// CHECK:  encoding: [0x62,0x02,0x1d,0x81,0xa8,0xc4]
+          vfmadd213ps %xmm28, %xmm28, %xmm24 {%k1} {z}
+
+// CHECK: vfmadd213ps (%rcx), %xmm28, %xmm24
+// CHECK:  encoding: [0x62,0x62,0x1d,0x00,0xa8,0x01]
+          vfmadd213ps (%rcx), %xmm28, %xmm24
+
+// CHECK: vfmadd213ps 291(%rax,%r14,8), %xmm28, %xmm24
+// CHECK:  encoding: [0x62,0x22,0x1d,0x00,0xa8,0x84,0xf0,0x23,0x01,0x00,0x00]
+          vfmadd213ps 291(%rax,%r14,8), %xmm28, %xmm24
+
+// CHECK: vfmadd213ps (%rcx){1to4}, %xmm28, %xmm24
+// CHECK:  encoding: [0x62,0x62,0x1d,0x10,0xa8,0x01]
+          vfmadd213ps (%rcx){1to4}, %xmm28, %xmm24
+
+// CHECK: vfmadd213ps 2032(%rdx), %xmm28, %xmm24
+// CHECK:  encoding: [0x62,0x62,0x1d,0x00,0xa8,0x42,0x7f]
+          vfmadd213ps 2032(%rdx), %xmm28, %xmm24
+
+// CHECK: vfmadd213ps 2048(%rdx), %xmm28, %xmm24
+// CHECK:  encoding: [0x62,0x62,0x1d,0x00,0xa8,0x82,0x00,0x08,0x00,0x00]
+          vfmadd213ps 2048(%rdx), %xmm28, %xmm24
+
+// CHECK: vfmadd213ps -2048(%rdx), %xmm28, %xmm24
+// CHECK:  encoding: [0x62,0x62,0x1d,0x00,0xa8,0x42,0x80]
+          vfmadd213ps -2048(%rdx), %xmm28, %xmm24
+
+// CHECK: vfmadd213ps -2064(%rdx), %xmm28, %xmm24
+// CHECK:  encoding: [0x62,0x62,0x1d,0x00,0xa8,0x82,0xf0,0xf7,0xff,0xff]
+          vfmadd213ps -2064(%rdx), %xmm28, %xmm24
+
+// CHECK: vfmadd213ps 508(%rdx){1to4}, %xmm28, %xmm24
+// CHECK:  encoding: [0x62,0x62,0x1d,0x10,0xa8,0x42,0x7f]
+          vfmadd213ps 508(%rdx){1to4}, %xmm28, %xmm24
+
+// CHECK: vfmadd213ps 512(%rdx){1to4}, %xmm28, %xmm24
+// CHECK:  encoding: [0x62,0x62,0x1d,0x10,0xa8,0x82,0x00,0x02,0x00,0x00]
+          vfmadd213ps 512(%rdx){1to4}, %xmm28, %xmm24
+
+// CHECK: vfmadd213ps -512(%rdx){1to4}, %xmm28, %xmm24
+// CHECK:  encoding: [0x62,0x62,0x1d,0x10,0xa8,0x42,0x80]
+          vfmadd213ps -512(%rdx){1to4}, %xmm28, %xmm24
+
+// CHECK: vfmadd213ps -516(%rdx){1to4}, %xmm28, %xmm24
+// CHECK:  encoding: [0x62,0x62,0x1d,0x10,0xa8,0x82,0xfc,0xfd,0xff,0xff]
+          vfmadd213ps -516(%rdx){1to4}, %xmm28, %xmm24
+
+// CHECK: vfmadd213ps %ymm17, %ymm22, %ymm26
+// CHECK:  encoding: [0x62,0x22,0x4d,0x20,0xa8,0xd1]
+          vfmadd213ps %ymm17, %ymm22, %ymm26
+
+// CHECK: vfmadd213ps %ymm17, %ymm22, %ymm26 {%k3}
+// CHECK:  encoding: [0x62,0x22,0x4d,0x23,0xa8,0xd1]
+          vfmadd213ps %ymm17, %ymm22, %ymm26 {%k3}
+
+// CHECK: vfmadd213ps %ymm17, %ymm22, %ymm26 {%k3} {z}
+// CHECK:  encoding: [0x62,0x22,0x4d,0xa3,0xa8,0xd1]
+          vfmadd213ps %ymm17, %ymm22, %ymm26 {%k3} {z}
+
+// CHECK: vfmadd213ps (%rcx), %ymm22, %ymm26
+// CHECK:  encoding: [0x62,0x62,0x4d,0x20,0xa8,0x11]
+          vfmadd213ps (%rcx), %ymm22, %ymm26
+
+// CHECK: vfmadd213ps 291(%rax,%r14,8), %ymm22, %ymm26
+// CHECK:  encoding: [0x62,0x22,0x4d,0x20,0xa8,0x94,0xf0,0x23,0x01,0x00,0x00]
+          vfmadd213ps 291(%rax,%r14,8), %ymm22, %ymm26
+
+// CHECK: vfmadd213ps (%rcx){1to8}, %ymm22, %ymm26
+// CHECK:  encoding: [0x62,0x62,0x4d,0x30,0xa8,0x11]
+          vfmadd213ps (%rcx){1to8}, %ymm22, %ymm26
+
+// CHECK: vfmadd213ps 4064(%rdx), %ymm22, %ymm26
+// CHECK:  encoding: [0x62,0x62,0x4d,0x20,0xa8,0x52,0x7f]
+          vfmadd213ps 4064(%rdx), %ymm22, %ymm26
+
+// CHECK: vfmadd213ps 4096(%rdx), %ymm22, %ymm26
+// CHECK:  encoding: [0x62,0x62,0x4d,0x20,0xa8,0x92,0x00,0x10,0x00,0x00]
+          vfmadd213ps 4096(%rdx), %ymm22, %ymm26
+
+// CHECK: vfmadd213ps -4096(%rdx), %ymm22, %ymm26
+// CHECK:  encoding: [0x62,0x62,0x4d,0x20,0xa8,0x52,0x80]
+          vfmadd213ps -4096(%rdx), %ymm22, %ymm26
+
+// CHECK: vfmadd213ps -4128(%rdx), %ymm22, %ymm26
+// CHECK:  encoding: [0x62,0x62,0x4d,0x20,0xa8,0x92,0xe0,0xef,0xff,0xff]
+          vfmadd213ps -4128(%rdx), %ymm22, %ymm26
+
+// CHECK: vfmadd213ps 508(%rdx){1to8}, %ymm22, %ymm26
+// CHECK:  encoding: [0x62,0x62,0x4d,0x30,0xa8,0x52,0x7f]
+          vfmadd213ps 508(%rdx){1to8}, %ymm22, %ymm26
+
+// CHECK: vfmadd213ps 512(%rdx){1to8}, %ymm22, %ymm26
+// CHECK:  encoding: [0x62,0x62,0x4d,0x30,0xa8,0x92,0x00,0x02,0x00,0x00]
+          vfmadd213ps 512(%rdx){1to8}, %ymm22, %ymm26
+
+// CHECK: vfmadd213ps -512(%rdx){1to8}, %ymm22, %ymm26
+// CHECK:  encoding: [0x62,0x62,0x4d,0x30,0xa8,0x52,0x80]
+          vfmadd213ps -512(%rdx){1to8}, %ymm22, %ymm26
+
+// CHECK: vfmadd213ps -516(%rdx){1to8}, %ymm22, %ymm26
+// CHECK:  encoding: [0x62,0x62,0x4d,0x30,0xa8,0x92,0xfc,0xfd,0xff,0xff]
+          vfmadd213ps -516(%rdx){1to8}, %ymm22, %ymm26
+
+// CHECK: vfmadd213pd %xmm23, %xmm21, %xmm22
+// CHECK:  encoding: [0x62,0xa2,0xd5,0x00,0xa8,0xf7]
+          vfmadd213pd %xmm23, %xmm21, %xmm22
+
+// CHECK: vfmadd213pd %xmm23, %xmm21, %xmm22 {%k4}
+// CHECK:  encoding: [0x62,0xa2,0xd5,0x04,0xa8,0xf7]
+          vfmadd213pd %xmm23, %xmm21, %xmm22 {%k4}
+
+// CHECK: vfmadd213pd %xmm23, %xmm21, %xmm22 {%k4} {z}
+// CHECK:  encoding: [0x62,0xa2,0xd5,0x84,0xa8,0xf7]
+          vfmadd213pd %xmm23, %xmm21, %xmm22 {%k4} {z}
+
+// CHECK: vfmadd213pd (%rcx), %xmm21, %xmm22
+// CHECK:  encoding: [0x62,0xe2,0xd5,0x00,0xa8,0x31]
+          vfmadd213pd (%rcx), %xmm21, %xmm22
+
+// CHECK: vfmadd213pd 291(%rax,%r14,8), %xmm21, %xmm22
+// CHECK:  encoding: [0x62,0xa2,0xd5,0x00,0xa8,0xb4,0xf0,0x23,0x01,0x00,0x00]
+          vfmadd213pd 291(%rax,%r14,8), %xmm21, %xmm22
+
+// CHECK: vfmadd213pd (%rcx){1to2}, %xmm21, %xmm22
+// CHECK:  encoding: [0x62,0xe2,0xd5,0x10,0xa8,0x31]
+          vfmadd213pd (%rcx){1to2}, %xmm21, %xmm22
+
+// CHECK: vfmadd213pd 2032(%rdx), %xmm21, %xmm22
+// CHECK:  encoding: [0x62,0xe2,0xd5,0x00,0xa8,0x72,0x7f]
+          vfmadd213pd 2032(%rdx), %xmm21, %xmm22
+
+// CHECK: vfmadd213pd 2048(%rdx), %xmm21, %xmm22
+// CHECK:  encoding: [0x62,0xe2,0xd5,0x00,0xa8,0xb2,0x00,0x08,0x00,0x00]
+          vfmadd213pd 2048(%rdx), %xmm21, %xmm22
+
+// CHECK: vfmadd213pd -2048(%rdx), %xmm21, %xmm22
+// CHECK:  encoding: [0x62,0xe2,0xd5,0x00,0xa8,0x72,0x80]
+          vfmadd213pd -2048(%rdx), %xmm21, %xmm22
+
+// CHECK: vfmadd213pd -2064(%rdx), %xmm21, %xmm22
+// CHECK:  encoding: [0x62,0xe2,0xd5,0x00,0xa8,0xb2,0xf0,0xf7,0xff,0xff]
+          vfmadd213pd -2064(%rdx), %xmm21, %xmm22
+
+// CHECK: vfmadd213pd 1016(%rdx){1to2}, %xmm21, %xmm22
+// CHECK:  encoding: [0x62,0xe2,0xd5,0x10,0xa8,0x72,0x7f]
+          vfmadd213pd 1016(%rdx){1to2}, %xmm21, %xmm22
+
+// CHECK: vfmadd213pd 1024(%rdx){1to2}, %xmm21, %xmm22
+// CHECK:  encoding: [0x62,0xe2,0xd5,0x10,0xa8,0xb2,0x00,0x04,0x00,0x00]
+          vfmadd213pd 1024(%rdx){1to2}, %xmm21, %xmm22
+
+// CHECK: vfmadd213pd -1024(%rdx){1to2}, %xmm21, %xmm22
+// CHECK:  encoding: [0x62,0xe2,0xd5,0x10,0xa8,0x72,0x80]
+          vfmadd213pd -1024(%rdx){1to2}, %xmm21, %xmm22
+
+// CHECK: vfmadd213pd -1032(%rdx){1to2}, %xmm21, %xmm22
+// CHECK:  encoding: [0x62,0xe2,0xd5,0x10,0xa8,0xb2,0xf8,0xfb,0xff,0xff]
+          vfmadd213pd -1032(%rdx){1to2}, %xmm21, %xmm22
+
+// CHECK: vfmadd213pd %ymm17, %ymm19, %ymm18
+// CHECK:  encoding: [0x62,0xa2,0xe5,0x20,0xa8,0xd1]
+          vfmadd213pd %ymm17, %ymm19, %ymm18
+
+// CHECK: vfmadd213pd %ymm17, %ymm19, %ymm18 {%k1}
+// CHECK:  encoding: [0x62,0xa2,0xe5,0x21,0xa8,0xd1]
+          vfmadd213pd %ymm17, %ymm19, %ymm18 {%k1}
+
+// CHECK: vfmadd213pd %ymm17, %ymm19, %ymm18 {%k1} {z}
+// CHECK:  encoding: [0x62,0xa2,0xe5,0xa1,0xa8,0xd1]
+          vfmadd213pd %ymm17, %ymm19, %ymm18 {%k1} {z}
+
+// CHECK: vfmadd213pd (%rcx), %ymm19, %ymm18
+// CHECK:  encoding: [0x62,0xe2,0xe5,0x20,0xa8,0x11]
+          vfmadd213pd (%rcx), %ymm19, %ymm18
+
+// CHECK: vfmadd213pd 291(%rax,%r14,8), %ymm19, %ymm18
+// CHECK:  encoding: [0x62,0xa2,0xe5,0x20,0xa8,0x94,0xf0,0x23,0x01,0x00,0x00]
+          vfmadd213pd 291(%rax,%r14,8), %ymm19, %ymm18
+
+// CHECK: vfmadd213pd (%rcx){1to4}, %ymm19, %ymm18
+// CHECK:  encoding: [0x62,0xe2,0xe5,0x30,0xa8,0x11]
+          vfmadd213pd (%rcx){1to4}, %ymm19, %ymm18
+
+// CHECK: vfmadd213pd 4064(%rdx), %ymm19, %ymm18
+// CHECK:  encoding: [0x62,0xe2,0xe5,0x20,0xa8,0x52,0x7f]
+          vfmadd213pd 4064(%rdx), %ymm19, %ymm18
+
+// CHECK: vfmadd213pd 4096(%rdx), %ymm19, %ymm18
+// CHECK:  encoding: [0x62,0xe2,0xe5,0x20,0xa8,0x92,0x00,0x10,0x00,0x00]
+          vfmadd213pd 4096(%rdx), %ymm19, %ymm18
+
+// CHECK: vfmadd213pd -4096(%rdx), %ymm19, %ymm18
+// CHECK:  encoding: [0x62,0xe2,0xe5,0x20,0xa8,0x52,0x80]
+          vfmadd213pd -4096(%rdx), %ymm19, %ymm18
+
+// CHECK: vfmadd213pd -4128(%rdx), %ymm19, %ymm18
+// CHECK:  encoding: [0x62,0xe2,0xe5,0x20,0xa8,0x92,0xe0,0xef,0xff,0xff]
+          vfmadd213pd -4128(%rdx), %ymm19, %ymm18
+
+// CHECK: vfmadd213pd 1016(%rdx){1to4}, %ymm19, %ymm18
+// CHECK:  encoding: [0x62,0xe2,0xe5,0x30,0xa8,0x52,0x7f]
+          vfmadd213pd 1016(%rdx){1to4}, %ymm19, %ymm18
+
+// CHECK: vfmadd213pd 1024(%rdx){1to4}, %ymm19, %ymm18
+// CHECK:  encoding: [0x62,0xe2,0xe5,0x30,0xa8,0x92,0x00,0x04,0x00,0x00]
+          vfmadd213pd 1024(%rdx){1to4}, %ymm19, %ymm18
+
+// CHECK: vfmadd213pd -1024(%rdx){1to4}, %ymm19, %ymm18
+// CHECK:  encoding: [0x62,0xe2,0xe5,0x30,0xa8,0x52,0x80]
+          vfmadd213pd -1024(%rdx){1to4}, %ymm19, %ymm18
+
+// CHECK: vfmadd213pd -1032(%rdx){1to4}, %ymm19, %ymm18
+// CHECK:  encoding: [0x62,0xe2,0xe5,0x30,0xa8,0x92,0xf8,0xfb,0xff,0xff]
+          vfmadd213pd -1032(%rdx){1to4}, %ymm19, %ymm18
+
+// CHECK: vfmadd231ps %xmm27, %xmm20, %xmm30
+// CHECK:  encoding: [0x62,0x02,0x5d,0x00,0xb8,0xf3]
+          vfmadd231ps %xmm27, %xmm20, %xmm30
+
+// CHECK: vfmadd231ps %xmm27, %xmm20, %xmm30 {%k7}
+// CHECK:  encoding: [0x62,0x02,0x5d,0x07,0xb8,0xf3]
+          vfmadd231ps %xmm27, %xmm20, %xmm30 {%k7}
+
+// CHECK: vfmadd231ps %xmm27, %xmm20, %xmm30 {%k7} {z}
+// CHECK:  encoding: [0x62,0x02,0x5d,0x87,0xb8,0xf3]
+          vfmadd231ps %xmm27, %xmm20, %xmm30 {%k7} {z}
+
+// CHECK: vfmadd231ps (%rcx), %xmm20, %xmm30
+// CHECK:  encoding: [0x62,0x62,0x5d,0x00,0xb8,0x31]
+          vfmadd231ps (%rcx), %xmm20, %xmm30
+
+// CHECK: vfmadd231ps 291(%rax,%r14,8), %xmm20, %xmm30
+// CHECK:  encoding: [0x62,0x22,0x5d,0x00,0xb8,0xb4,0xf0,0x23,0x01,0x00,0x00]
+          vfmadd231ps 291(%rax,%r14,8), %xmm20, %xmm30
+
+// CHECK: vfmadd231ps (%rcx){1to4}, %xmm20, %xmm30
+// CHECK:  encoding: [0x62,0x62,0x5d,0x10,0xb8,0x31]
+          vfmadd231ps (%rcx){1to4}, %xmm20, %xmm30
+
+// CHECK: vfmadd231ps 2032(%rdx), %xmm20, %xmm30
+// CHECK:  encoding: [0x62,0x62,0x5d,0x00,0xb8,0x72,0x7f]
+          vfmadd231ps 2032(%rdx), %xmm20, %xmm30
+
+// CHECK: vfmadd231ps 2048(%rdx), %xmm20, %xmm30
+// CHECK:  encoding: [0x62,0x62,0x5d,0x00,0xb8,0xb2,0x00,0x08,0x00,0x00]
+          vfmadd231ps 2048(%rdx), %xmm20, %xmm30
+
+// CHECK: vfmadd231ps -2048(%rdx), %xmm20, %xmm30
+// CHECK:  encoding: [0x62,0x62,0x5d,0x00,0xb8,0x72,0x80]
+          vfmadd231ps -2048(%rdx), %xmm20, %xmm30
+
+// CHECK: vfmadd231ps -2064(%rdx), %xmm20, %xmm30
+// CHECK:  encoding: [0x62,0x62,0x5d,0x00,0xb8,0xb2,0xf0,0xf7,0xff,0xff]
+          vfmadd231ps -2064(%rdx), %xmm20, %xmm30
+
+// CHECK: vfmadd231ps 508(%rdx){1to4}, %xmm20, %xmm30
+// CHECK:  encoding: [0x62,0x62,0x5d,0x10,0xb8,0x72,0x7f]
+          vfmadd231ps 508(%rdx){1to4}, %xmm20, %xmm30
+
+// CHECK: vfmadd231ps 512(%rdx){1to4}, %xmm20, %xmm30
+// CHECK:  encoding: [0x62,0x62,0x5d,0x10,0xb8,0xb2,0x00,0x02,0x00,0x00]
+          vfmadd231ps 512(%rdx){1to4}, %xmm20, %xmm30
+
+// CHECK: vfmadd231ps -512(%rdx){1to4}, %xmm20, %xmm30
+// CHECK:  encoding: [0x62,0x62,0x5d,0x10,0xb8,0x72,0x80]
+          vfmadd231ps -512(%rdx){1to4}, %xmm20, %xmm30
+
+// CHECK: vfmadd231ps -516(%rdx){1to4}, %xmm20, %xmm30
+// CHECK:  encoding: [0x62,0x62,0x5d,0x10,0xb8,0xb2,0xfc,0xfd,0xff,0xff]
+          vfmadd231ps -516(%rdx){1to4}, %xmm20, %xmm30
+
+// CHECK: vfmadd231ps %ymm25, %ymm26, %ymm22
+// CHECK:  encoding: [0x62,0x82,0x2d,0x20,0xb8,0xf1]
+          vfmadd231ps %ymm25, %ymm26, %ymm22
+
+// CHECK: vfmadd231ps %ymm25, %ymm26, %ymm22 {%k7}
+// CHECK:  encoding: [0x62,0x82,0x2d,0x27,0xb8,0xf1]
+          vfmadd231ps %ymm25, %ymm26, %ymm22 {%k7}
+
+// CHECK: vfmadd231ps %ymm25, %ymm26, %ymm22 {%k7} {z}
+// CHECK:  encoding: [0x62,0x82,0x2d,0xa7,0xb8,0xf1]
+          vfmadd231ps %ymm25, %ymm26, %ymm22 {%k7} {z}
+
+// CHECK: vfmadd231ps (%rcx), %ymm26, %ymm22
+// CHECK:  encoding: [0x62,0xe2,0x2d,0x20,0xb8,0x31]
+          vfmadd231ps (%rcx), %ymm26, %ymm22
+
+// CHECK: vfmadd231ps 291(%rax,%r14,8), %ymm26, %ymm22
+// CHECK:  encoding: [0x62,0xa2,0x2d,0x20,0xb8,0xb4,0xf0,0x23,0x01,0x00,0x00]
+          vfmadd231ps 291(%rax,%r14,8), %ymm26, %ymm22
+
+// CHECK: vfmadd231ps (%rcx){1to8}, %ymm26, %ymm22
+// CHECK:  encoding: [0x62,0xe2,0x2d,0x30,0xb8,0x31]
+          vfmadd231ps (%rcx){1to8}, %ymm26, %ymm22
+
+// CHECK: vfmadd231ps 4064(%rdx), %ymm26, %ymm22
+// CHECK:  encoding: [0x62,0xe2,0x2d,0x20,0xb8,0x72,0x7f]
+          vfmadd231ps 4064(%rdx), %ymm26, %ymm22
+
+// CHECK: vfmadd231ps 4096(%rdx), %ymm26, %ymm22
+// CHECK:  encoding: [0x62,0xe2,0x2d,0x20,0xb8,0xb2,0x00,0x10,0x00,0x00]
+          vfmadd231ps 4096(%rdx), %ymm26, %ymm22
+
+// CHECK: vfmadd231ps -4096(%rdx), %ymm26, %ymm22
+// CHECK:  encoding: [0x62,0xe2,0x2d,0x20,0xb8,0x72,0x80]
+          vfmadd231ps -4096(%rdx), %ymm26, %ymm22
+
+// CHECK: vfmadd231ps -4128(%rdx), %ymm26, %ymm22
+// CHECK:  encoding: [0x62,0xe2,0x2d,0x20,0xb8,0xb2,0xe0,0xef,0xff,0xff]
+          vfmadd231ps -4128(%rdx), %ymm26, %ymm22
+
+// CHECK: vfmadd231ps 508(%rdx){1to8}, %ymm26, %ymm22
+// CHECK:  encoding: [0x62,0xe2,0x2d,0x30,0xb8,0x72,0x7f]
+          vfmadd231ps 508(%rdx){1to8}, %ymm26, %ymm22
+
+// CHECK: vfmadd231ps 512(%rdx){1to8}, %ymm26, %ymm22
+// CHECK:  encoding: [0x62,0xe2,0x2d,0x30,0xb8,0xb2,0x00,0x02,0x00,0x00]
+          vfmadd231ps 512(%rdx){1to8}, %ymm26, %ymm22
+
+// CHECK: vfmadd231ps -512(%rdx){1to8}, %ymm26, %ymm22
+// CHECK:  encoding: [0x62,0xe2,0x2d,0x30,0xb8,0x72,0x80]
+          vfmadd231ps -512(%rdx){1to8}, %ymm26, %ymm22
+
+// CHECK: vfmadd231ps -516(%rdx){1to8}, %ymm26, %ymm22
+// CHECK:  encoding: [0x62,0xe2,0x2d,0x30,0xb8,0xb2,0xfc,0xfd,0xff,0xff]
+          vfmadd231ps -516(%rdx){1to8}, %ymm26, %ymm22
+
+// CHECK: vfmadd231pd %xmm24, %xmm20, %xmm29
+// CHECK:  encoding: [0x62,0x02,0xdd,0x00,0xb8,0xe8]
+          vfmadd231pd %xmm24, %xmm20, %xmm29
+
+// CHECK: vfmadd231pd %xmm24, %xmm20, %xmm29 {%k7}
+// CHECK:  encoding: [0x62,0x02,0xdd,0x07,0xb8,0xe8]
+          vfmadd231pd %xmm24, %xmm20, %xmm29 {%k7}
+
+// CHECK: vfmadd231pd %xmm24, %xmm20, %xmm29 {%k7} {z}
+// CHECK:  encoding: [0x62,0x02,0xdd,0x87,0xb8,0xe8]
+          vfmadd231pd %xmm24, %xmm20, %xmm29 {%k7} {z}
+
+// CHECK: vfmadd231pd (%rcx), %xmm20, %xmm29
+// CHECK:  encoding: [0x62,0x62,0xdd,0x00,0xb8,0x29]
+          vfmadd231pd (%rcx), %xmm20, %xmm29
+
+// CHECK: vfmadd231pd 291(%rax,%r14,8), %xmm20, %xmm29
+// CHECK:  encoding: [0x62,0x22,0xdd,0x00,0xb8,0xac,0xf0,0x23,0x01,0x00,0x00]
+          vfmadd231pd 291(%rax,%r14,8), %xmm20, %xmm29
+
+// CHECK: vfmadd231pd (%rcx){1to2}, %xmm20, %xmm29
+// CHECK:  encoding: [0x62,0x62,0xdd,0x10,0xb8,0x29]
+          vfmadd231pd (%rcx){1to2}, %xmm20, %xmm29
+
+// CHECK: vfmadd231pd 2032(%rdx), %xmm20, %xmm29
+// CHECK:  encoding: [0x62,0x62,0xdd,0x00,0xb8,0x6a,0x7f]
+          vfmadd231pd 2032(%rdx), %xmm20, %xmm29
+
+// CHECK: vfmadd231pd 2048(%rdx), %xmm20, %xmm29
+// CHECK:  encoding: [0x62,0x62,0xdd,0x00,0xb8,0xaa,0x00,0x08,0x00,0x00]
+          vfmadd231pd 2048(%rdx), %xmm20, %xmm29
+
+// CHECK: vfmadd231pd -2048(%rdx), %xmm20, %xmm29
+// CHECK:  encoding: [0x62,0x62,0xdd,0x00,0xb8,0x6a,0x80]
+          vfmadd231pd -2048(%rdx), %xmm20, %xmm29
+
+// CHECK: vfmadd231pd -2064(%rdx), %xmm20, %xmm29
+// CHECK:  encoding: [0x62,0x62,0xdd,0x00,0xb8,0xaa,0xf0,0xf7,0xff,0xff]
+          vfmadd231pd -2064(%rdx), %xmm20, %xmm29
+
+// CHECK: vfmadd231pd 1016(%rdx){1to2}, %xmm20, %xmm29
+// CHECK:  encoding: [0x62,0x62,0xdd,0x10,0xb8,0x6a,0x7f]
+          vfmadd231pd 1016(%rdx){1to2}, %xmm20, %xmm29
+
+// CHECK: vfmadd231pd 1024(%rdx){1to2}, %xmm20, %xmm29
+// CHECK:  encoding: [0x62,0x62,0xdd,0x10,0xb8,0xaa,0x00,0x04,0x00,0x00]
+          vfmadd231pd 1024(%rdx){1to2}, %xmm20, %xmm29
+
+// CHECK: vfmadd231pd -1024(%rdx){1to2}, %xmm20, %xmm29
+// CHECK:  encoding: [0x62,0x62,0xdd,0x10,0xb8,0x6a,0x80]
+          vfmadd231pd -1024(%rdx){1to2}, %xmm20, %xmm29
+
+// CHECK: vfmadd231pd -1032(%rdx){1to2}, %xmm20, %xmm29
+// CHECK:  encoding: [0x62,0x62,0xdd,0x10,0xb8,0xaa,0xf8,0xfb,0xff,0xff]
+          vfmadd231pd -1032(%rdx){1to2}, %xmm20, %xmm29
+
+// CHECK: vfmadd231pd %ymm26, %ymm20, %ymm24
+// CHECK:  encoding: [0x62,0x02,0xdd,0x20,0xb8,0xc2]
+          vfmadd231pd %ymm26, %ymm20, %ymm24
+
+// CHECK: vfmadd231pd %ymm26, %ymm20, %ymm24 {%k6}
+// CHECK:  encoding: [0x62,0x02,0xdd,0x26,0xb8,0xc2]
+          vfmadd231pd %ymm26, %ymm20, %ymm24 {%k6}
+
+// CHECK: vfmadd231pd %ymm26, %ymm20, %ymm24 {%k6} {z}
+// CHECK:  encoding: [0x62,0x02,0xdd,0xa6,0xb8,0xc2]
+          vfmadd231pd %ymm26, %ymm20, %ymm24 {%k6} {z}
+
+// CHECK: vfmadd231pd (%rcx), %ymm20, %ymm24
+// CHECK:  encoding: [0x62,0x62,0xdd,0x20,0xb8,0x01]
+          vfmadd231pd (%rcx), %ymm20, %ymm24
+
+// CHECK: vfmadd231pd 291(%rax,%r14,8), %ymm20, %ymm24
+// CHECK:  encoding: [0x62,0x22,0xdd,0x20,0xb8,0x84,0xf0,0x23,0x01,0x00,0x00]
+          vfmadd231pd 291(%rax,%r14,8), %ymm20, %ymm24
+
+// CHECK: vfmadd231pd (%rcx){1to4}, %ymm20, %ymm24
+// CHECK:  encoding: [0x62,0x62,0xdd,0x30,0xb8,0x01]
+          vfmadd231pd (%rcx){1to4}, %ymm20, %ymm24
+
+// CHECK: vfmadd231pd 4064(%rdx), %ymm20, %ymm24
+// CHECK:  encoding: [0x62,0x62,0xdd,0x20,0xb8,0x42,0x7f]
+          vfmadd231pd 4064(%rdx), %ymm20, %ymm24
+
+// CHECK: vfmadd231pd 4096(%rdx), %ymm20, %ymm24
+// CHECK:  encoding: [0x62,0x62,0xdd,0x20,0xb8,0x82,0x00,0x10,0x00,0x00]
+          vfmadd231pd 4096(%rdx), %ymm20, %ymm24
+
+// CHECK: vfmadd231pd -4096(%rdx), %ymm20, %ymm24
+// CHECK:  encoding: [0x62,0x62,0xdd,0x20,0xb8,0x42,0x80]
+          vfmadd231pd -4096(%rdx), %ymm20, %ymm24
+
+// CHECK: vfmadd231pd -4128(%rdx), %ymm20, %ymm24
+// CHECK:  encoding: [0x62,0x62,0xdd,0x20,0xb8,0x82,0xe0,0xef,0xff,0xff]
+          vfmadd231pd -4128(%rdx), %ymm20, %ymm24
+
+// CHECK: vfmadd231pd 1016(%rdx){1to4}, %ymm20, %ymm24
+// CHECK:  encoding: [0x62,0x62,0xdd,0x30,0xb8,0x42,0x7f]
+          vfmadd231pd 1016(%rdx){1to4}, %ymm20, %ymm24
+
+// CHECK: vfmadd231pd 1024(%rdx){1to4}, %ymm20, %ymm24
+// CHECK:  encoding: [0x62,0x62,0xdd,0x30,0xb8,0x82,0x00,0x04,0x00,0x00]
+          vfmadd231pd 1024(%rdx){1to4}, %ymm20, %ymm24
+
+// CHECK: vfmadd231pd -1024(%rdx){1to4}, %ymm20, %ymm24
+// CHECK:  encoding: [0x62,0x62,0xdd,0x30,0xb8,0x42,0x80]
+          vfmadd231pd -1024(%rdx){1to4}, %ymm20, %ymm24
+
+// CHECK: vfmadd231pd -1032(%rdx){1to4}, %ymm20, %ymm24
+// CHECK:  encoding: [0x62,0x62,0xdd,0x30,0xb8,0x82,0xf8,0xfb,0xff,0xff]
+          vfmadd231pd -1032(%rdx){1to4}, %ymm20, %ymm24
+
+// CHECK: vfmsub132ps %xmm21, %xmm18, %xmm17
+// CHECK:  encoding: [0x62,0xa2,0x6d,0x00,0x9a,0xcd]
+          vfmsub132ps %xmm21, %xmm18, %xmm17
+
+// CHECK: vfmsub132ps %xmm21, %xmm18, %xmm17 {%k1}
+// CHECK:  encoding: [0x62,0xa2,0x6d,0x01,0x9a,0xcd]
+          vfmsub132ps %xmm21, %xmm18, %xmm17 {%k1}
+
+// CHECK: vfmsub132ps %xmm21, %xmm18, %xmm17 {%k1} {z}
+// CHECK:  encoding: [0x62,0xa2,0x6d,0x81,0x9a,0xcd]
+          vfmsub132ps %xmm21, %xmm18, %xmm17 {%k1} {z}
+
+// CHECK: vfmsub132ps (%rcx), %xmm18, %xmm17
+// CHECK:  encoding: [0x62,0xe2,0x6d,0x00,0x9a,0x09]
+          vfmsub132ps (%rcx), %xmm18, %xmm17
+
+// CHECK: vfmsub132ps 291(%rax,%r14,8), %xmm18, %xmm17
+// CHECK:  encoding: [0x62,0xa2,0x6d,0x00,0x9a,0x8c,0xf0,0x23,0x01,0x00,0x00]
+          vfmsub132ps 291(%rax,%r14,8), %xmm18, %xmm17
+
+// CHECK: vfmsub132ps (%rcx){1to4}, %xmm18, %xmm17
+// CHECK:  encoding: [0x62,0xe2,0x6d,0x10,0x9a,0x09]
+          vfmsub132ps (%rcx){1to4}, %xmm18, %xmm17
+
+// CHECK: vfmsub132ps 2032(%rdx), %xmm18, %xmm17
+// CHECK:  encoding: [0x62,0xe2,0x6d,0x00,0x9a,0x4a,0x7f]
+          vfmsub132ps 2032(%rdx), %xmm18, %xmm17
+
+// CHECK: vfmsub132ps 2048(%rdx), %xmm18, %xmm17
+// CHECK:  encoding: [0x62,0xe2,0x6d,0x00,0x9a,0x8a,0x00,0x08,0x00,0x00]
+          vfmsub132ps 2048(%rdx), %xmm18, %xmm17
+
+// CHECK: vfmsub132ps -2048(%rdx), %xmm18, %xmm17
+// CHECK:  encoding: [0x62,0xe2,0x6d,0x00,0x9a,0x4a,0x80]
+          vfmsub132ps -2048(%rdx), %xmm18, %xmm17
+
+// CHECK: vfmsub132ps -2064(%rdx), %xmm18, %xmm17
+// CHECK:  encoding: [0x62,0xe2,0x6d,0x00,0x9a,0x8a,0xf0,0xf7,0xff,0xff]
+          vfmsub132ps -2064(%rdx), %xmm18, %xmm17
+
+// CHECK: vfmsub132ps 508(%rdx){1to4}, %xmm18, %xmm17
+// CHECK:  encoding: [0x62,0xe2,0x6d,0x10,0x9a,0x4a,0x7f]
+          vfmsub132ps 508(%rdx){1to4}, %xmm18, %xmm17
+
+// CHECK: vfmsub132ps 512(%rdx){1to4}, %xmm18, %xmm17
+// CHECK:  encoding: [0x62,0xe2,0x6d,0x10,0x9a,0x8a,0x00,0x02,0x00,0x00]
+          vfmsub132ps 512(%rdx){1to4}, %xmm18, %xmm17
+
+// CHECK: vfmsub132ps -512(%rdx){1to4}, %xmm18, %xmm17
+// CHECK:  encoding: [0x62,0xe2,0x6d,0x10,0x9a,0x4a,0x80]
+          vfmsub132ps -512(%rdx){1to4}, %xmm18, %xmm17
+
+// CHECK: vfmsub132ps -516(%rdx){1to4}, %xmm18, %xmm17
+// CHECK:  encoding: [0x62,0xe2,0x6d,0x10,0x9a,0x8a,0xfc,0xfd,0xff,0xff]
+          vfmsub132ps -516(%rdx){1to4}, %xmm18, %xmm17
+
+// CHECK: vfmsub132ps %ymm23, %ymm28, %ymm25
+// CHECK:  encoding: [0x62,0x22,0x1d,0x20,0x9a,0xcf]
+          vfmsub132ps %ymm23, %ymm28, %ymm25
+
+// CHECK: vfmsub132ps %ymm23, %ymm28, %ymm25 {%k5}
+// CHECK:  encoding: [0x62,0x22,0x1d,0x25,0x9a,0xcf]
+          vfmsub132ps %ymm23, %ymm28, %ymm25 {%k5}
+
+// CHECK: vfmsub132ps %ymm23, %ymm28, %ymm25 {%k5} {z}
+// CHECK:  encoding: [0x62,0x22,0x1d,0xa5,0x9a,0xcf]
+          vfmsub132ps %ymm23, %ymm28, %ymm25 {%k5} {z}
+
+// CHECK: vfmsub132ps (%rcx), %ymm28, %ymm25
+// CHECK:  encoding: [0x62,0x62,0x1d,0x20,0x9a,0x09]
+          vfmsub132ps (%rcx), %ymm28, %ymm25
+
+// CHECK: vfmsub132ps 291(%rax,%r14,8), %ymm28, %ymm25
+// CHECK:  encoding: [0x62,0x22,0x1d,0x20,0x9a,0x8c,0xf0,0x23,0x01,0x00,0x00]
+          vfmsub132ps 291(%rax,%r14,8), %ymm28, %ymm25
+
+// CHECK: vfmsub132ps (%rcx){1to8}, %ymm28, %ymm25
+// CHECK:  encoding: [0x62,0x62,0x1d,0x30,0x9a,0x09]
+          vfmsub132ps (%rcx){1to8}, %ymm28, %ymm25
+
+// CHECK: vfmsub132ps 4064(%rdx), %ymm28, %ymm25
+// CHECK:  encoding: [0x62,0x62,0x1d,0x20,0x9a,0x4a,0x7f]
+          vfmsub132ps 4064(%rdx), %ymm28, %ymm25
+
+// CHECK: vfmsub132ps 4096(%rdx), %ymm28, %ymm25
+// CHECK:  encoding: [0x62,0x62,0x1d,0x20,0x9a,0x8a,0x00,0x10,0x00,0x00]
+          vfmsub132ps 4096(%rdx), %ymm28, %ymm25
+
+// CHECK: vfmsub132ps -4096(%rdx), %ymm28, %ymm25
+// CHECK:  encoding: [0x62,0x62,0x1d,0x20,0x9a,0x4a,0x80]
+          vfmsub132ps -4096(%rdx), %ymm28, %ymm25
+
+// CHECK: vfmsub132ps -4128(%rdx), %ymm28, %ymm25
+// CHECK:  encoding: [0x62,0x62,0x1d,0x20,0x9a,0x8a,0xe0,0xef,0xff,0xff]
+          vfmsub132ps -4128(%rdx), %ymm28, %ymm25
+
+// CHECK: vfmsub132ps 508(%rdx){1to8}, %ymm28, %ymm25
+// CHECK:  encoding: [0x62,0x62,0x1d,0x30,0x9a,0x4a,0x7f]
+          vfmsub132ps 508(%rdx){1to8}, %ymm28, %ymm25
+
+// CHECK: vfmsub132ps 512(%rdx){1to8}, %ymm28, %ymm25
+// CHECK:  encoding: [0x62,0x62,0x1d,0x30,0x9a,0x8a,0x00,0x02,0x00,0x00]
+          vfmsub132ps 512(%rdx){1to8}, %ymm28, %ymm25
+
+// CHECK: vfmsub132ps -512(%rdx){1to8}, %ymm28, %ymm25
+// CHECK:  encoding: [0x62,0x62,0x1d,0x30,0x9a,0x4a,0x80]
+          vfmsub132ps -512(%rdx){1to8}, %ymm28, %ymm25
+
+// CHECK: vfmsub132ps -516(%rdx){1to8}, %ymm28, %ymm25
+// CHECK:  encoding: [0x62,0x62,0x1d,0x30,0x9a,0x8a,0xfc,0xfd,0xff,0xff]
+          vfmsub132ps -516(%rdx){1to8}, %ymm28, %ymm25
+
+// CHECK: vfmsub132pd %xmm20, %xmm21, %xmm18
+// CHECK:  encoding: [0x62,0xa2,0xd5,0x00,0x9a,0xd4]
+          vfmsub132pd %xmm20, %xmm21, %xmm18
+
+// CHECK: vfmsub132pd %xmm20, %xmm21, %xmm18 {%k1}
+// CHECK:  encoding: [0x62,0xa2,0xd5,0x01,0x9a,0xd4]
+          vfmsub132pd %xmm20, %xmm21, %xmm18 {%k1}
+
+// CHECK: vfmsub132pd %xmm20, %xmm21, %xmm18 {%k1} {z}
+// CHECK:  encoding: [0x62,0xa2,0xd5,0x81,0x9a,0xd4]
+          vfmsub132pd %xmm20, %xmm21, %xmm18 {%k1} {z}
+
+// CHECK: vfmsub132pd (%rcx), %xmm21, %xmm18
+// CHECK:  encoding: [0x62,0xe2,0xd5,0x00,0x9a,0x11]
+          vfmsub132pd (%rcx), %xmm21, %xmm18
+
+// CHECK: vfmsub132pd 291(%rax,%r14,8), %xmm21, %xmm18
+// CHECK:  encoding: [0x62,0xa2,0xd5,0x00,0x9a,0x94,0xf0,0x23,0x01,0x00,0x00]
+          vfmsub132pd 291(%rax,%r14,8), %xmm21, %xmm18
+
+// CHECK: vfmsub132pd (%rcx){1to2}, %xmm21, %xmm18
+// CHECK:  encoding: [0x62,0xe2,0xd5,0x10,0x9a,0x11]
+          vfmsub132pd (%rcx){1to2}, %xmm21, %xmm18
+
+// CHECK: vfmsub132pd 2032(%rdx), %xmm21, %xmm18
+// CHECK:  encoding: [0x62,0xe2,0xd5,0x00,0x9a,0x52,0x7f]
+          vfmsub132pd 2032(%rdx), %xmm21, %xmm18
+
+// CHECK: vfmsub132pd 2048(%rdx), %xmm21, %xmm18
+// CHECK:  encoding: [0x62,0xe2,0xd5,0x00,0x9a,0x92,0x00,0x08,0x00,0x00]
+          vfmsub132pd 2048(%rdx), %xmm21, %xmm18
+
+// CHECK: vfmsub132pd -2048(%rdx), %xmm21, %xmm18
+// CHECK:  encoding: [0x62,0xe2,0xd5,0x00,0x9a,0x52,0x80]
+          vfmsub132pd -2048(%rdx), %xmm21, %xmm18
+
+// CHECK: vfmsub132pd -2064(%rdx), %xmm21, %xmm18
+// CHECK:  encoding: [0x62,0xe2,0xd5,0x00,0x9a,0x92,0xf0,0xf7,0xff,0xff]
+          vfmsub132pd -2064(%rdx), %xmm21, %xmm18
+
+// CHECK: vfmsub132pd 1016(%rdx){1to2}, %xmm21, %xmm18
+// CHECK:  encoding: [0x62,0xe2,0xd5,0x10,0x9a,0x52,0x7f]
+          vfmsub132pd 1016(%rdx){1to2}, %xmm21, %xmm18
+
+// CHECK: vfmsub132pd 1024(%rdx){1to2}, %xmm21, %xmm18
+// CHECK:  encoding: [0x62,0xe2,0xd5,0x10,0x9a,0x92,0x00,0x04,0x00,0x00]
+          vfmsub132pd 1024(%rdx){1to2}, %xmm21, %xmm18
+
+// CHECK: vfmsub132pd -1024(%rdx){1to2}, %xmm21, %xmm18
+// CHECK:  encoding: [0x62,0xe2,0xd5,0x10,0x9a,0x52,0x80]
+          vfmsub132pd -1024(%rdx){1to2}, %xmm21, %xmm18
+
+// CHECK: vfmsub132pd -1032(%rdx){1to2}, %xmm21, %xmm18
+// CHECK:  encoding: [0x62,0xe2,0xd5,0x10,0x9a,0x92,0xf8,0xfb,0xff,0xff]
+          vfmsub132pd -1032(%rdx){1to2}, %xmm21, %xmm18
+
+// CHECK: vfmsub132pd %ymm17, %ymm28, %ymm22
+// CHECK:  encoding: [0x62,0xa2,0x9d,0x20,0x9a,0xf1]
+          vfmsub132pd %ymm17, %ymm28, %ymm22
+
+// CHECK: vfmsub132pd %ymm17, %ymm28, %ymm22 {%k5}
+// CHECK:  encoding: [0x62,0xa2,0x9d,0x25,0x9a,0xf1]
+          vfmsub132pd %ymm17, %ymm28, %ymm22 {%k5}
+
+// CHECK: vfmsub132pd %ymm17, %ymm28, %ymm22 {%k5} {z}
+// CHECK:  encoding: [0x62,0xa2,0x9d,0xa5,0x9a,0xf1]
+          vfmsub132pd %ymm17, %ymm28, %ymm22 {%k5} {z}
+
+// CHECK: vfmsub132pd (%rcx), %ymm28, %ymm22
+// CHECK:  encoding: [0x62,0xe2,0x9d,0x20,0x9a,0x31]
+          vfmsub132pd (%rcx), %ymm28, %ymm22
+
+// CHECK: vfmsub132pd 291(%rax,%r14,8), %ymm28, %ymm22
+// CHECK:  encoding: [0x62,0xa2,0x9d,0x20,0x9a,0xb4,0xf0,0x23,0x01,0x00,0x00]
+          vfmsub132pd 291(%rax,%r14,8), %ymm28, %ymm22
+
+// CHECK: vfmsub132pd (%rcx){1to4}, %ymm28, %ymm22
+// CHECK:  encoding: [0x62,0xe2,0x9d,0x30,0x9a,0x31]
+          vfmsub132pd (%rcx){1to4}, %ymm28, %ymm22
+
+// CHECK: vfmsub132pd 4064(%rdx), %ymm28, %ymm22
+// CHECK:  encoding: [0x62,0xe2,0x9d,0x20,0x9a,0x72,0x7f]
+          vfmsub132pd 4064(%rdx), %ymm28, %ymm22
+
+// CHECK: vfmsub132pd 4096(%rdx), %ymm28, %ymm22
+// CHECK:  encoding: [0x62,0xe2,0x9d,0x20,0x9a,0xb2,0x00,0x10,0x00,0x00]
+          vfmsub132pd 4096(%rdx), %ymm28, %ymm22
+
+// CHECK: vfmsub132pd -4096(%rdx), %ymm28, %ymm22
+// CHECK:  encoding: [0x62,0xe2,0x9d,0x20,0x9a,0x72,0x80]
+          vfmsub132pd -4096(%rdx), %ymm28, %ymm22
+
+// CHECK: vfmsub132pd -4128(%rdx), %ymm28, %ymm22
+// CHECK:  encoding: [0x62,0xe2,0x9d,0x20,0x9a,0xb2,0xe0,0xef,0xff,0xff]
+          vfmsub132pd -4128(%rdx), %ymm28, %ymm22
+
+// CHECK: vfmsub132pd 1016(%rdx){1to4}, %ymm28, %ymm22
+// CHECK:  encoding: [0x62,0xe2,0x9d,0x30,0x9a,0x72,0x7f]
+          vfmsub132pd 1016(%rdx){1to4}, %ymm28, %ymm22
+
+// CHECK: vfmsub132pd 1024(%rdx){1to4}, %ymm28, %ymm22
+// CHECK:  encoding: [0x62,0xe2,0x9d,0x30,0x9a,0xb2,0x00,0x04,0x00,0x00]
+          vfmsub132pd 1024(%rdx){1to4}, %ymm28, %ymm22
+
+// CHECK: vfmsub132pd -1024(%rdx){1to4}, %ymm28, %ymm22
+// CHECK:  encoding: [0x62,0xe2,0x9d,0x30,0x9a,0x72,0x80]
+          vfmsub132pd -1024(%rdx){1to4}, %ymm28, %ymm22
+
+// CHECK: vfmsub132pd -1032(%rdx){1to4}, %ymm28, %ymm22
+// CHECK:  encoding: [0x62,0xe2,0x9d,0x30,0x9a,0xb2,0xf8,0xfb,0xff,0xff]
+          vfmsub132pd -1032(%rdx){1to4}, %ymm28, %ymm22
+
+// CHECK: vfmsub213ps %xmm28, %xmm27, %xmm22
+// CHECK:  encoding: [0x62,0x82,0x25,0x00,0xaa,0xf4]
+          vfmsub213ps %xmm28, %xmm27, %xmm22
+
+// CHECK: vfmsub213ps %xmm28, %xmm27, %xmm22 {%k2}
+// CHECK:  encoding: [0x62,0x82,0x25,0x02,0xaa,0xf4]
+          vfmsub213ps %xmm28, %xmm27, %xmm22 {%k2}
+
+// CHECK: vfmsub213ps %xmm28, %xmm27, %xmm22 {%k2} {z}
+// CHECK:  encoding: [0x62,0x82,0x25,0x82,0xaa,0xf4]
+          vfmsub213ps %xmm28, %xmm27, %xmm22 {%k2} {z}
+
+// CHECK: vfmsub213ps (%rcx), %xmm27, %xmm22
+// CHECK:  encoding: [0x62,0xe2,0x25,0x00,0xaa,0x31]
+          vfmsub213ps (%rcx), %xmm27, %xmm22
+
+// CHECK: vfmsub213ps 291(%rax,%r14,8), %xmm27, %xmm22
+// CHECK:  encoding: [0x62,0xa2,0x25,0x00,0xaa,0xb4,0xf0,0x23,0x01,0x00,0x00]
+          vfmsub213ps 291(%rax,%r14,8), %xmm27, %xmm22
+
+// CHECK: vfmsub213ps (%rcx){1to4}, %xmm27, %xmm22
+// CHECK:  encoding: [0x62,0xe2,0x25,0x10,0xaa,0x31]
+          vfmsub213ps (%rcx){1to4}, %xmm27, %xmm22
+
+// CHECK: vfmsub213ps 2032(%rdx), %xmm27, %xmm22
+// CHECK:  encoding: [0x62,0xe2,0x25,0x00,0xaa,0x72,0x7f]
+          vfmsub213ps 2032(%rdx), %xmm27, %xmm22
+
+// CHECK: vfmsub213ps 2048(%rdx), %xmm27, %xmm22
+// CHECK:  encoding: [0x62,0xe2,0x25,0x00,0xaa,0xb2,0x00,0x08,0x00,0x00]
+          vfmsub213ps 2048(%rdx), %xmm27, %xmm22
+
+// CHECK: vfmsub213ps -2048(%rdx), %xmm27, %xmm22
+// CHECK:  encoding: [0x62,0xe2,0x25,0x00,0xaa,0x72,0x80]
+          vfmsub213ps -2048(%rdx), %xmm27, %xmm22
+
+// CHECK: vfmsub213ps -2064(%rdx), %xmm27, %xmm22
+// CHECK:  encoding: [0x62,0xe2,0x25,0x00,0xaa,0xb2,0xf0,0xf7,0xff,0xff]
+          vfmsub213ps -2064(%rdx), %xmm27, %xmm22
+
+// CHECK: vfmsub213ps 508(%rdx){1to4}, %xmm27, %xmm22
+// CHECK:  encoding: [0x62,0xe2,0x25,0x10,0xaa,0x72,0x7f]
+          vfmsub213ps 508(%rdx){1to4}, %xmm27, %xmm22
+
+// CHECK: vfmsub213ps 512(%rdx){1to4}, %xmm27, %xmm22
+// CHECK:  encoding: [0x62,0xe2,0x25,0x10,0xaa,0xb2,0x00,0x02,0x00,0x00]
+          vfmsub213ps 512(%rdx){1to4}, %xmm27, %xmm22
+
+// CHECK: vfmsub213ps -512(%rdx){1to4}, %xmm27, %xmm22
+// CHECK:  encoding: [0x62,0xe2,0x25,0x10,0xaa,0x72,0x80]
+          vfmsub213ps -512(%rdx){1to4}, %xmm27, %xmm22
+
+// CHECK: vfmsub213ps -516(%rdx){1to4}, %xmm27, %xmm22
+// CHECK:  encoding: [0x62,0xe2,0x25,0x10,0xaa,0xb2,0xfc,0xfd,0xff,0xff]
+          vfmsub213ps -516(%rdx){1to4}, %xmm27, %xmm22
+
+// CHECK: vfmsub213ps %ymm22, %ymm29, %ymm28
+// CHECK:  encoding: [0x62,0x22,0x15,0x20,0xaa,0xe6]
+          vfmsub213ps %ymm22, %ymm29, %ymm28
+
+// CHECK: vfmsub213ps %ymm22, %ymm29, %ymm28 {%k1}
+// CHECK:  encoding: [0x62,0x22,0x15,0x21,0xaa,0xe6]
+          vfmsub213ps %ymm22, %ymm29, %ymm28 {%k1}
+
+// CHECK: vfmsub213ps %ymm22, %ymm29, %ymm28 {%k1} {z}
+// CHECK:  encoding: [0x62,0x22,0x15,0xa1,0xaa,0xe6]
+          vfmsub213ps %ymm22, %ymm29, %ymm28 {%k1} {z}
+
+// CHECK: vfmsub213ps (%rcx), %ymm29, %ymm28
+// CHECK:  encoding: [0x62,0x62,0x15,0x20,0xaa,0x21]
+          vfmsub213ps (%rcx), %ymm29, %ymm28
+
+// CHECK: vfmsub213ps 291(%rax,%r14,8), %ymm29, %ymm28
+// CHECK:  encoding: [0x62,0x22,0x15,0x20,0xaa,0xa4,0xf0,0x23,0x01,0x00,0x00]
+          vfmsub213ps 291(%rax,%r14,8), %ymm29, %ymm28
+
+// CHECK: vfmsub213ps (%rcx){1to8}, %ymm29, %ymm28
+// CHECK:  encoding: [0x62,0x62,0x15,0x30,0xaa,0x21]
+          vfmsub213ps (%rcx){1to8}, %ymm29, %ymm28
+
+// CHECK: vfmsub213ps 4064(%rdx), %ymm29, %ymm28
+// CHECK:  encoding: [0x62,0x62,0x15,0x20,0xaa,0x62,0x7f]
+          vfmsub213ps 4064(%rdx), %ymm29, %ymm28
+
+// CHECK: vfmsub213ps 4096(%rdx), %ymm29, %ymm28
+// CHECK:  encoding: [0x62,0x62,0x15,0x20,0xaa,0xa2,0x00,0x10,0x00,0x00]
+          vfmsub213ps 4096(%rdx), %ymm29, %ymm28
+
+// CHECK: vfmsub213ps -4096(%rdx), %ymm29, %ymm28
+// CHECK:  encoding: [0x62,0x62,0x15,0x20,0xaa,0x62,0x80]
+          vfmsub213ps -4096(%rdx), %ymm29, %ymm28
+
+// CHECK: vfmsub213ps -4128(%rdx), %ymm29, %ymm28
+// CHECK:  encoding: [0x62,0x62,0x15,0x20,0xaa,0xa2,0xe0,0xef,0xff,0xff]
+          vfmsub213ps -4128(%rdx), %ymm29, %ymm28
+
+// CHECK: vfmsub213ps 508(%rdx){1to8}, %ymm29, %ymm28
+// CHECK:  encoding: [0x62,0x62,0x15,0x30,0xaa,0x62,0x7f]
+          vfmsub213ps 508(%rdx){1to8}, %ymm29, %ymm28
+
+// CHECK: vfmsub213ps 512(%rdx){1to8}, %ymm29, %ymm28
+// CHECK:  encoding: [0x62,0x62,0x15,0x30,0xaa,0xa2,0x00,0x02,0x00,0x00]
+          vfmsub213ps 512(%rdx){1to8}, %ymm29, %ymm28
+
+// CHECK: vfmsub213ps -512(%rdx){1to8}, %ymm29, %ymm28
+// CHECK:  encoding: [0x62,0x62,0x15,0x30,0xaa,0x62,0x80]
+          vfmsub213ps -512(%rdx){1to8}, %ymm29, %ymm28
+
+// CHECK: vfmsub213ps -516(%rdx){1to8}, %ymm29, %ymm28
+// CHECK:  encoding: [0x62,0x62,0x15,0x30,0xaa,0xa2,0xfc,0xfd,0xff,0xff]
+          vfmsub213ps -516(%rdx){1to8}, %ymm29, %ymm28
+
+// CHECK: vfmsub213pd %xmm20, %xmm19, %xmm19
+// CHECK:  encoding: [0x62,0xa2,0xe5,0x00,0xaa,0xdc]
+          vfmsub213pd %xmm20, %xmm19, %xmm19
+
+// CHECK: vfmsub213pd %xmm20, %xmm19, %xmm19 {%k1}
+// CHECK:  encoding: [0x62,0xa2,0xe5,0x01,0xaa,0xdc]
+          vfmsub213pd %xmm20, %xmm19, %xmm19 {%k1}
+
+// CHECK: vfmsub213pd %xmm20, %xmm19, %xmm19 {%k1} {z}
+// CHECK:  encoding: [0x62,0xa2,0xe5,0x81,0xaa,0xdc]
+          vfmsub213pd %xmm20, %xmm19, %xmm19 {%k1} {z}
+
+// CHECK: vfmsub213pd (%rcx), %xmm19, %xmm19
+// CHECK:  encoding: [0x62,0xe2,0xe5,0x00,0xaa,0x19]
+          vfmsub213pd (%rcx), %xmm19, %xmm19
+
+// CHECK: vfmsub213pd 291(%rax,%r14,8), %xmm19, %xmm19
+// CHECK:  encoding: [0x62,0xa2,0xe5,0x00,0xaa,0x9c,0xf0,0x23,0x01,0x00,0x00]
+          vfmsub213pd 291(%rax,%r14,8), %xmm19, %xmm19
+
+// CHECK: vfmsub213pd (%rcx){1to2}, %xmm19, %xmm19
+// CHECK:  encoding: [0x62,0xe2,0xe5,0x10,0xaa,0x19]
+          vfmsub213pd (%rcx){1to2}, %xmm19, %xmm19
+
+// CHECK: vfmsub213pd 2032(%rdx), %xmm19, %xmm19
+// CHECK:  encoding: [0x62,0xe2,0xe5,0x00,0xaa,0x5a,0x7f]
+          vfmsub213pd 2032(%rdx), %xmm19, %xmm19
+
+// CHECK: vfmsub213pd 2048(%rdx), %xmm19, %xmm19
+// CHECK:  encoding: [0x62,0xe2,0xe5,0x00,0xaa,0x9a,0x00,0x08,0x00,0x00]
+          vfmsub213pd 2048(%rdx), %xmm19, %xmm19
+
+// CHECK: vfmsub213pd -2048(%rdx), %xmm19, %xmm19
+// CHECK:  encoding: [0x62,0xe2,0xe5,0x00,0xaa,0x5a,0x80]
+          vfmsub213pd -2048(%rdx), %xmm19, %xmm19
+
+// CHECK: vfmsub213pd -2064(%rdx), %xmm19, %xmm19
+// CHECK:  encoding: [0x62,0xe2,0xe5,0x00,0xaa,0x9a,0xf0,0xf7,0xff,0xff]
+          vfmsub213pd -2064(%rdx), %xmm19, %xmm19
+
+// CHECK: vfmsub213pd 1016(%rdx){1to2}, %xmm19, %xmm19
+// CHECK:  encoding: [0x62,0xe2,0xe5,0x10,0xaa,0x5a,0x7f]
+          vfmsub213pd 1016(%rdx){1to2}, %xmm19, %xmm19
+
+// CHECK: vfmsub213pd 1024(%rdx){1to2}, %xmm19, %xmm19
+// CHECK:  encoding: [0x62,0xe2,0xe5,0x10,0xaa,0x9a,0x00,0x04,0x00,0x00]
+          vfmsub213pd 1024(%rdx){1to2}, %xmm19, %xmm19
+
+// CHECK: vfmsub213pd -1024(%rdx){1to2}, %xmm19, %xmm19
+// CHECK:  encoding: [0x62,0xe2,0xe5,0x10,0xaa,0x5a,0x80]
+          vfmsub213pd -1024(%rdx){1to2}, %xmm19, %xmm19
+
+// CHECK: vfmsub213pd -1032(%rdx){1to2}, %xmm19, %xmm19
+// CHECK:  encoding: [0x62,0xe2,0xe5,0x10,0xaa,0x9a,0xf8,0xfb,0xff,0xff]
+          vfmsub213pd -1032(%rdx){1to2}, %xmm19, %xmm19
+
+// CHECK: vfmsub213pd %ymm28, %ymm25, %ymm27
+// CHECK:  encoding: [0x62,0x02,0xb5,0x20,0xaa,0xdc]
+          vfmsub213pd %ymm28, %ymm25, %ymm27
+
+// CHECK: vfmsub213pd %ymm28, %ymm25, %ymm27 {%k4}
+// CHECK:  encoding: [0x62,0x02,0xb5,0x24,0xaa,0xdc]
+          vfmsub213pd %ymm28, %ymm25, %ymm27 {%k4}
+
+// CHECK: vfmsub213pd %ymm28, %ymm25, %ymm27 {%k4} {z}
+// CHECK:  encoding: [0x62,0x02,0xb5,0xa4,0xaa,0xdc]
+          vfmsub213pd %ymm28, %ymm25, %ymm27 {%k4} {z}
+
+// CHECK: vfmsub213pd (%rcx), %ymm25, %ymm27
+// CHECK:  encoding: [0x62,0x62,0xb5,0x20,0xaa,0x19]
+          vfmsub213pd (%rcx), %ymm25, %ymm27
+
+// CHECK: vfmsub213pd 291(%rax,%r14,8), %ymm25, %ymm27
+// CHECK:  encoding: [0x62,0x22,0xb5,0x20,0xaa,0x9c,0xf0,0x23,0x01,0x00,0x00]
+          vfmsub213pd 291(%rax,%r14,8), %ymm25, %ymm27
+
+// CHECK: vfmsub213pd (%rcx){1to4}, %ymm25, %ymm27
+// CHECK:  encoding: [0x62,0x62,0xb5,0x30,0xaa,0x19]
+          vfmsub213pd (%rcx){1to4}, %ymm25, %ymm27
+
+// CHECK: vfmsub213pd 4064(%rdx), %ymm25, %ymm27
+// CHECK:  encoding: [0x62,0x62,0xb5,0x20,0xaa,0x5a,0x7f]
+          vfmsub213pd 4064(%rdx), %ymm25, %ymm27
+
+// CHECK: vfmsub213pd 4096(%rdx), %ymm25, %ymm27
+// CHECK:  encoding: [0x62,0x62,0xb5,0x20,0xaa,0x9a,0x00,0x10,0x00,0x00]
+          vfmsub213pd 4096(%rdx), %ymm25, %ymm27
+
+// CHECK: vfmsub213pd -4096(%rdx), %ymm25, %ymm27
+// CHECK:  encoding: [0x62,0x62,0xb5,0x20,0xaa,0x5a,0x80]
+          vfmsub213pd -4096(%rdx), %ymm25, %ymm27
+
+// CHECK: vfmsub213pd -4128(%rdx), %ymm25, %ymm27
+// CHECK:  encoding: [0x62,0x62,0xb5,0x20,0xaa,0x9a,0xe0,0xef,0xff,0xff]
+          vfmsub213pd -4128(%rdx), %ymm25, %ymm27
+
+// CHECK: vfmsub213pd 1016(%rdx){1to4}, %ymm25, %ymm27
+// CHECK:  encoding: [0x62,0x62,0xb5,0x30,0xaa,0x5a,0x7f]
+          vfmsub213pd 1016(%rdx){1to4}, %ymm25, %ymm27
+
+// CHECK: vfmsub213pd 1024(%rdx){1to4}, %ymm25, %ymm27
+// CHECK:  encoding: [0x62,0x62,0xb5,0x30,0xaa,0x9a,0x00,0x04,0x00,0x00]
+          vfmsub213pd 1024(%rdx){1to4}, %ymm25, %ymm27
+
+// CHECK: vfmsub213pd -1024(%rdx){1to4}, %ymm25, %ymm27
+// CHECK:  encoding: [0x62,0x62,0xb5,0x30,0xaa,0x5a,0x80]
+          vfmsub213pd -1024(%rdx){1to4}, %ymm25, %ymm27
+
+// CHECK: vfmsub213pd -1032(%rdx){1to4}, %ymm25, %ymm27
+// CHECK:  encoding: [0x62,0x62,0xb5,0x30,0xaa,0x9a,0xf8,0xfb,0xff,0xff]
+          vfmsub213pd -1032(%rdx){1to4}, %ymm25, %ymm27
+
+// CHECK: vfmsub231ps %xmm25, %xmm19, %xmm28
+// CHECK:  encoding: [0x62,0x02,0x65,0x00,0xba,0xe1]
+          vfmsub231ps %xmm25, %xmm19, %xmm28
+
+// CHECK: vfmsub231ps %xmm25, %xmm19, %xmm28 {%k1}
+// CHECK:  encoding: [0x62,0x02,0x65,0x01,0xba,0xe1]
+          vfmsub231ps %xmm25, %xmm19, %xmm28 {%k1}
+
+// CHECK: vfmsub231ps %xmm25, %xmm19, %xmm28 {%k1} {z}
+// CHECK:  encoding: [0x62,0x02,0x65,0x81,0xba,0xe1]
+          vfmsub231ps %xmm25, %xmm19, %xmm28 {%k1} {z}
+
+// CHECK: vfmsub231ps (%rcx), %xmm19, %xmm28
+// CHECK:  encoding: [0x62,0x62,0x65,0x00,0xba,0x21]
+          vfmsub231ps (%rcx), %xmm19, %xmm28
+
+// CHECK: vfmsub231ps 291(%rax,%r14,8), %xmm19, %xmm28
+// CHECK:  encoding: [0x62,0x22,0x65,0x00,0xba,0xa4,0xf0,0x23,0x01,0x00,0x00]
+          vfmsub231ps 291(%rax,%r14,8), %xmm19, %xmm28
+
+// CHECK: vfmsub231ps (%rcx){1to4}, %xmm19, %xmm28
+// CHECK:  encoding: [0x62,0x62,0x65,0x10,0xba,0x21]
+          vfmsub231ps (%rcx){1to4}, %xmm19, %xmm28
+
+// CHECK: vfmsub231ps 2032(%rdx), %xmm19, %xmm28
+// CHECK:  encoding: [0x62,0x62,0x65,0x00,0xba,0x62,0x7f]
+          vfmsub231ps 2032(%rdx), %xmm19, %xmm28
+
+// CHECK: vfmsub231ps 2048(%rdx), %xmm19, %xmm28
+// CHECK:  encoding: [0x62,0x62,0x65,0x00,0xba,0xa2,0x00,0x08,0x00,0x00]
+          vfmsub231ps 2048(%rdx), %xmm19, %xmm28
+
+// CHECK: vfmsub231ps -2048(%rdx), %xmm19, %xmm28
+// CHECK:  encoding: [0x62,0x62,0x65,0x00,0xba,0x62,0x80]
+          vfmsub231ps -2048(%rdx), %xmm19, %xmm28
+
+// CHECK: vfmsub231ps -2064(%rdx), %xmm19, %xmm28
+// CHECK:  encoding: [0x62,0x62,0x65,0x00,0xba,0xa2,0xf0,0xf7,0xff,0xff]
+          vfmsub231ps -2064(%rdx), %xmm19, %xmm28
+
+// CHECK: vfmsub231ps 508(%rdx){1to4}, %xmm19, %xmm28
+// CHECK:  encoding: [0x62,0x62,0x65,0x10,0xba,0x62,0x7f]
+          vfmsub231ps 508(%rdx){1to4}, %xmm19, %xmm28
+
+// CHECK: vfmsub231ps 512(%rdx){1to4}, %xmm19, %xmm28
+// CHECK:  encoding: [0x62,0x62,0x65,0x10,0xba,0xa2,0x00,0x02,0x00,0x00]
+          vfmsub231ps 512(%rdx){1to4}, %xmm19, %xmm28
+
+// CHECK: vfmsub231ps -512(%rdx){1to4}, %xmm19, %xmm28
+// CHECK:  encoding: [0x62,0x62,0x65,0x10,0xba,0x62,0x80]
+          vfmsub231ps -512(%rdx){1to4}, %xmm19, %xmm28
+
+// CHECK: vfmsub231ps -516(%rdx){1to4}, %xmm19, %xmm28
+// CHECK:  encoding: [0x62,0x62,0x65,0x10,0xba,0xa2,0xfc,0xfd,0xff,0xff]
+          vfmsub231ps -516(%rdx){1to4}, %xmm19, %xmm28
+
+// CHECK: vfmsub231ps %ymm26, %ymm26, %ymm23
+// CHECK:  encoding: [0x62,0x82,0x2d,0x20,0xba,0xfa]
+          vfmsub231ps %ymm26, %ymm26, %ymm23
+
+// CHECK: vfmsub231ps %ymm26, %ymm26, %ymm23 {%k1}
+// CHECK:  encoding: [0x62,0x82,0x2d,0x21,0xba,0xfa]
+          vfmsub231ps %ymm26, %ymm26, %ymm23 {%k1}
+
+// CHECK: vfmsub231ps %ymm26, %ymm26, %ymm23 {%k1} {z}
+// CHECK:  encoding: [0x62,0x82,0x2d,0xa1,0xba,0xfa]
+          vfmsub231ps %ymm26, %ymm26, %ymm23 {%k1} {z}
+
+// CHECK: vfmsub231ps (%rcx), %ymm26, %ymm23
+// CHECK:  encoding: [0x62,0xe2,0x2d,0x20,0xba,0x39]
+          vfmsub231ps (%rcx), %ymm26, %ymm23
+
+// CHECK: vfmsub231ps 291(%rax,%r14,8), %ymm26, %ymm23
+// CHECK:  encoding: [0x62,0xa2,0x2d,0x20,0xba,0xbc,0xf0,0x23,0x01,0x00,0x00]
+          vfmsub231ps 291(%rax,%r14,8), %ymm26, %ymm23
+
+// CHECK: vfmsub231ps (%rcx){1to8}, %ymm26, %ymm23
+// CHECK:  encoding: [0x62,0xe2,0x2d,0x30,0xba,0x39]
+          vfmsub231ps (%rcx){1to8}, %ymm26, %ymm23
+
+// CHECK: vfmsub231ps 4064(%rdx), %ymm26, %ymm23
+// CHECK:  encoding: [0x62,0xe2,0x2d,0x20,0xba,0x7a,0x7f]
+          vfmsub231ps 4064(%rdx), %ymm26, %ymm23
+
+// CHECK: vfmsub231ps 4096(%rdx), %ymm26, %ymm23
+// CHECK:  encoding: [0x62,0xe2,0x2d,0x20,0xba,0xba,0x00,0x10,0x00,0x00]
+          vfmsub231ps 4096(%rdx), %ymm26, %ymm23
+
+// CHECK: vfmsub231ps -4096(%rdx), %ymm26, %ymm23
+// CHECK:  encoding: [0x62,0xe2,0x2d,0x20,0xba,0x7a,0x80]
+          vfmsub231ps -4096(%rdx), %ymm26, %ymm23
+
+// CHECK: vfmsub231ps -4128(%rdx), %ymm26, %ymm23
+// CHECK:  encoding: [0x62,0xe2,0x2d,0x20,0xba,0xba,0xe0,0xef,0xff,0xff]
+          vfmsub231ps -4128(%rdx), %ymm26, %ymm23
+
+// CHECK: vfmsub231ps 508(%rdx){1to8}, %ymm26, %ymm23
+// CHECK:  encoding: [0x62,0xe2,0x2d,0x30,0xba,0x7a,0x7f]
+          vfmsub231ps 508(%rdx){1to8}, %ymm26, %ymm23
+
+// CHECK: vfmsub231ps 512(%rdx){1to8}, %ymm26, %ymm23
+// CHECK:  encoding: [0x62,0xe2,0x2d,0x30,0xba,0xba,0x00,0x02,0x00,0x00]
+          vfmsub231ps 512(%rdx){1to8}, %ymm26, %ymm23
+
+// CHECK: vfmsub231ps -512(%rdx){1to8}, %ymm26, %ymm23
+// CHECK:  encoding: [0x62,0xe2,0x2d,0x30,0xba,0x7a,0x80]
+          vfmsub231ps -512(%rdx){1to8}, %ymm26, %ymm23
+
+// CHECK: vfmsub231ps -516(%rdx){1to8}, %ymm26, %ymm23
+// CHECK:  encoding: [0x62,0xe2,0x2d,0x30,0xba,0xba,0xfc,0xfd,0xff,0xff]
+          vfmsub231ps -516(%rdx){1to8}, %ymm26, %ymm23
+
+// CHECK: vfmsub231pd %xmm23, %xmm20, %xmm28
+// CHECK:  encoding: [0x62,0x22,0xdd,0x00,0xba,0xe7]
+          vfmsub231pd %xmm23, %xmm20, %xmm28
+
+// CHECK: vfmsub231pd %xmm23, %xmm20, %xmm28 {%k4}
+// CHECK:  encoding: [0x62,0x22,0xdd,0x04,0xba,0xe7]
+          vfmsub231pd %xmm23, %xmm20, %xmm28 {%k4}
+
+// CHECK: vfmsub231pd %xmm23, %xmm20, %xmm28 {%k4} {z}
+// CHECK:  encoding: [0x62,0x22,0xdd,0x84,0xba,0xe7]
+          vfmsub231pd %xmm23, %xmm20, %xmm28 {%k4} {z}
+
+// CHECK: vfmsub231pd (%rcx), %xmm20, %xmm28
+// CHECK:  encoding: [0x62,0x62,0xdd,0x00,0xba,0x21]
+          vfmsub231pd (%rcx), %xmm20, %xmm28
+
+// CHECK: vfmsub231pd 291(%rax,%r14,8), %xmm20, %xmm28
+// CHECK:  encoding: [0x62,0x22,0xdd,0x00,0xba,0xa4,0xf0,0x23,0x01,0x00,0x00]
+          vfmsub231pd 291(%rax,%r14,8), %xmm20, %xmm28
+
+// CHECK: vfmsub231pd (%rcx){1to2}, %xmm20, %xmm28
+// CHECK:  encoding: [0x62,0x62,0xdd,0x10,0xba,0x21]
+          vfmsub231pd (%rcx){1to2}, %xmm20, %xmm28
+
+// CHECK: vfmsub231pd 2032(%rdx), %xmm20, %xmm28
+// CHECK:  encoding: [0x62,0x62,0xdd,0x00,0xba,0x62,0x7f]
+          vfmsub231pd 2032(%rdx), %xmm20, %xmm28
+
+// CHECK: vfmsub231pd 2048(%rdx), %xmm20, %xmm28
+// CHECK:  encoding: [0x62,0x62,0xdd,0x00,0xba,0xa2,0x00,0x08,0x00,0x00]
+          vfmsub231pd 2048(%rdx), %xmm20, %xmm28
+
+// CHECK: vfmsub231pd -2048(%rdx), %xmm20, %xmm28
+// CHECK:  encoding: [0x62,0x62,0xdd,0x00,0xba,0x62,0x80]
+          vfmsub231pd -2048(%rdx), %xmm20, %xmm28
+
+// CHECK: vfmsub231pd -2064(%rdx), %xmm20, %xmm28
+// CHECK:  encoding: [0x62,0x62,0xdd,0x00,0xba,0xa2,0xf0,0xf7,0xff,0xff]
+          vfmsub231pd -2064(%rdx), %xmm20, %xmm28
+
+// CHECK: vfmsub231pd 1016(%rdx){1to2}, %xmm20, %xmm28
+// CHECK:  encoding: [0x62,0x62,0xdd,0x10,0xba,0x62,0x7f]
+          vfmsub231pd 1016(%rdx){1to2}, %xmm20, %xmm28
+
+// CHECK: vfmsub231pd 1024(%rdx){1to2}, %xmm20, %xmm28
+// CHECK:  encoding: [0x62,0x62,0xdd,0x10,0xba,0xa2,0x00,0x04,0x00,0x00]
+          vfmsub231pd 1024(%rdx){1to2}, %xmm20, %xmm28
+
+// CHECK: vfmsub231pd -1024(%rdx){1to2}, %xmm20, %xmm28
+// CHECK:  encoding: [0x62,0x62,0xdd,0x10,0xba,0x62,0x80]
+          vfmsub231pd -1024(%rdx){1to2}, %xmm20, %xmm28
+
+// CHECK: vfmsub231pd -1032(%rdx){1to2}, %xmm20, %xmm28
+// CHECK:  encoding: [0x62,0x62,0xdd,0x10,0xba,0xa2,0xf8,0xfb,0xff,0xff]
+          vfmsub231pd -1032(%rdx){1to2}, %xmm20, %xmm28
+
+// CHECK: vfmsub231pd %ymm22, %ymm18, %ymm17
+// CHECK:  encoding: [0x62,0xa2,0xed,0x20,0xba,0xce]
+          vfmsub231pd %ymm22, %ymm18, %ymm17
+
+// CHECK: vfmsub231pd %ymm22, %ymm18, %ymm17 {%k2}
+// CHECK:  encoding: [0x62,0xa2,0xed,0x22,0xba,0xce]
+          vfmsub231pd %ymm22, %ymm18, %ymm17 {%k2}
+
+// CHECK: vfmsub231pd %ymm22, %ymm18, %ymm17 {%k2} {z}
+// CHECK:  encoding: [0x62,0xa2,0xed,0xa2,0xba,0xce]
+          vfmsub231pd %ymm22, %ymm18, %ymm17 {%k2} {z}
+
+// CHECK: vfmsub231pd (%rcx), %ymm18, %ymm17
+// CHECK:  encoding: [0x62,0xe2,0xed,0x20,0xba,0x09]
+          vfmsub231pd (%rcx), %ymm18, %ymm17
+
+// CHECK: vfmsub231pd 291(%rax,%r14,8), %ymm18, %ymm17
+// CHECK:  encoding: [0x62,0xa2,0xed,0x20,0xba,0x8c,0xf0,0x23,0x01,0x00,0x00]
+          vfmsub231pd 291(%rax,%r14,8), %ymm18, %ymm17
+
+// CHECK: vfmsub231pd (%rcx){1to4}, %ymm18, %ymm17
+// CHECK:  encoding: [0x62,0xe2,0xed,0x30,0xba,0x09]
+          vfmsub231pd (%rcx){1to4}, %ymm18, %ymm17
+
+// CHECK: vfmsub231pd 4064(%rdx), %ymm18, %ymm17
+// CHECK:  encoding: [0x62,0xe2,0xed,0x20,0xba,0x4a,0x7f]
+          vfmsub231pd 4064(%rdx), %ymm18, %ymm17
+
+// CHECK: vfmsub231pd 4096(%rdx), %ymm18, %ymm17
+// CHECK:  encoding: [0x62,0xe2,0xed,0x20,0xba,0x8a,0x00,0x10,0x00,0x00]
+          vfmsub231pd 4096(%rdx), %ymm18, %ymm17
+
+// CHECK: vfmsub231pd -4096(%rdx), %ymm18, %ymm17
+// CHECK:  encoding: [0x62,0xe2,0xed,0x20,0xba,0x4a,0x80]
+          vfmsub231pd -4096(%rdx), %ymm18, %ymm17
+
+// CHECK: vfmsub231pd -4128(%rdx), %ymm18, %ymm17
+// CHECK:  encoding: [0x62,0xe2,0xed,0x20,0xba,0x8a,0xe0,0xef,0xff,0xff]
+          vfmsub231pd -4128(%rdx), %ymm18, %ymm17
+
+// CHECK: vfmsub231pd 1016(%rdx){1to4}, %ymm18, %ymm17
+// CHECK:  encoding: [0x62,0xe2,0xed,0x30,0xba,0x4a,0x7f]
+          vfmsub231pd 1016(%rdx){1to4}, %ymm18, %ymm17
+
+// CHECK: vfmsub231pd 1024(%rdx){1to4}, %ymm18, %ymm17
+// CHECK:  encoding: [0x62,0xe2,0xed,0x30,0xba,0x8a,0x00,0x04,0x00,0x00]
+          vfmsub231pd 1024(%rdx){1to4}, %ymm18, %ymm17
+
+// CHECK: vfmsub231pd -1024(%rdx){1to4}, %ymm18, %ymm17
+// CHECK:  encoding: [0x62,0xe2,0xed,0x30,0xba,0x4a,0x80]
+          vfmsub231pd -1024(%rdx){1to4}, %ymm18, %ymm17
+
+// CHECK: vfmsub231pd -1032(%rdx){1to4}, %ymm18, %ymm17
+// CHECK:  encoding: [0x62,0xe2,0xed,0x30,0xba,0x8a,0xf8,0xfb,0xff,0xff]
+          vfmsub231pd -1032(%rdx){1to4}, %ymm18, %ymm17
+
+// CHECK: vfmaddsub132ps %xmm18, %xmm27, %xmm24
+// CHECK:  encoding: [0x62,0x22,0x25,0x00,0x96,0xc2]
+          vfmaddsub132ps %xmm18, %xmm27, %xmm24
+
+// CHECK: vfmaddsub132ps %xmm18, %xmm27, %xmm24 {%k2}
+// CHECK:  encoding: [0x62,0x22,0x25,0x02,0x96,0xc2]
+          vfmaddsub132ps %xmm18, %xmm27, %xmm24 {%k2}
+
+// CHECK: vfmaddsub132ps %xmm18, %xmm27, %xmm24 {%k2} {z}
+// CHECK:  encoding: [0x62,0x22,0x25,0x82,0x96,0xc2]
+          vfmaddsub132ps %xmm18, %xmm27, %xmm24 {%k2} {z}
+
+// CHECK: vfmaddsub132ps (%rcx), %xmm27, %xmm24
+// CHECK:  encoding: [0x62,0x62,0x25,0x00,0x96,0x01]
+          vfmaddsub132ps (%rcx), %xmm27, %xmm24
+
+// CHECK: vfmaddsub132ps 291(%rax,%r14,8), %xmm27, %xmm24
+// CHECK:  encoding: [0x62,0x22,0x25,0x00,0x96,0x84,0xf0,0x23,0x01,0x00,0x00]
+          vfmaddsub132ps 291(%rax,%r14,8), %xmm27, %xmm24
+
+// CHECK: vfmaddsub132ps (%rcx){1to4}, %xmm27, %xmm24
+// CHECK:  encoding: [0x62,0x62,0x25,0x10,0x96,0x01]
+          vfmaddsub132ps (%rcx){1to4}, %xmm27, %xmm24
+
+// CHECK: vfmaddsub132ps 2032(%rdx), %xmm27, %xmm24
+// CHECK:  encoding: [0x62,0x62,0x25,0x00,0x96,0x42,0x7f]
+          vfmaddsub132ps 2032(%rdx), %xmm27, %xmm24
+
+// CHECK: vfmaddsub132ps 2048(%rdx), %xmm27, %xmm24
+// CHECK:  encoding: [0x62,0x62,0x25,0x00,0x96,0x82,0x00,0x08,0x00,0x00]
+          vfmaddsub132ps 2048(%rdx), %xmm27, %xmm24
+
+// CHECK: vfmaddsub132ps -2048(%rdx), %xmm27, %xmm24
+// CHECK:  encoding: [0x62,0x62,0x25,0x00,0x96,0x42,0x80]
+          vfmaddsub132ps -2048(%rdx), %xmm27, %xmm24
+
+// CHECK: vfmaddsub132ps -2064(%rdx), %xmm27, %xmm24
+// CHECK:  encoding: [0x62,0x62,0x25,0x00,0x96,0x82,0xf0,0xf7,0xff,0xff]
+          vfmaddsub132ps -2064(%rdx), %xmm27, %xmm24
+
+// CHECK: vfmaddsub132ps 508(%rdx){1to4}, %xmm27, %xmm24
+// CHECK:  encoding: [0x62,0x62,0x25,0x10,0x96,0x42,0x7f]
+          vfmaddsub132ps 508(%rdx){1to4}, %xmm27, %xmm24
+
+// CHECK: vfmaddsub132ps 512(%rdx){1to4}, %xmm27, %xmm24
+// CHECK:  encoding: [0x62,0x62,0x25,0x10,0x96,0x82,0x00,0x02,0x00,0x00]
+          vfmaddsub132ps 512(%rdx){1to4}, %xmm27, %xmm24
+
+// CHECK: vfmaddsub132ps -512(%rdx){1to4}, %xmm27, %xmm24
+// CHECK:  encoding: [0x62,0x62,0x25,0x10,0x96,0x42,0x80]
+          vfmaddsub132ps -512(%rdx){1to4}, %xmm27, %xmm24
+
+// CHECK: vfmaddsub132ps -516(%rdx){1to4}, %xmm27, %xmm24
+// CHECK:  encoding: [0x62,0x62,0x25,0x10,0x96,0x82,0xfc,0xfd,0xff,0xff]
+          vfmaddsub132ps -516(%rdx){1to4}, %xmm27, %xmm24
+
+// CHECK: vfmaddsub132ps %ymm24, %ymm20, %ymm21
+// CHECK:  encoding: [0x62,0x82,0x5d,0x20,0x96,0xe8]
+          vfmaddsub132ps %ymm24, %ymm20, %ymm21
+
+// CHECK: vfmaddsub132ps %ymm24, %ymm20, %ymm21 {%k5}
+// CHECK:  encoding: [0x62,0x82,0x5d,0x25,0x96,0xe8]
+          vfmaddsub132ps %ymm24, %ymm20, %ymm21 {%k5}
+
+// CHECK: vfmaddsub132ps %ymm24, %ymm20, %ymm21 {%k5} {z}
+// CHECK:  encoding: [0x62,0x82,0x5d,0xa5,0x96,0xe8]
+          vfmaddsub132ps %ymm24, %ymm20, %ymm21 {%k5} {z}
+
+// CHECK: vfmaddsub132ps (%rcx), %ymm20, %ymm21
+// CHECK:  encoding: [0x62,0xe2,0x5d,0x20,0x96,0x29]
+          vfmaddsub132ps (%rcx), %ymm20, %ymm21
+
+// CHECK: vfmaddsub132ps 291(%rax,%r14,8), %ymm20, %ymm21
+// CHECK:  encoding: [0x62,0xa2,0x5d,0x20,0x96,0xac,0xf0,0x23,0x01,0x00,0x00]
+          vfmaddsub132ps 291(%rax,%r14,8), %ymm20, %ymm21
+
+// CHECK: vfmaddsub132ps (%rcx){1to8}, %ymm20, %ymm21
+// CHECK:  encoding: [0x62,0xe2,0x5d,0x30,0x96,0x29]
+          vfmaddsub132ps (%rcx){1to8}, %ymm20, %ymm21
+
+// CHECK: vfmaddsub132ps 4064(%rdx), %ymm20, %ymm21
+// CHECK:  encoding: [0x62,0xe2,0x5d,0x20,0x96,0x6a,0x7f]
+          vfmaddsub132ps 4064(%rdx), %ymm20, %ymm21
+
+// CHECK: vfmaddsub132ps 4096(%rdx), %ymm20, %ymm21
+// CHECK:  encoding: [0x62,0xe2,0x5d,0x20,0x96,0xaa,0x00,0x10,0x00,0x00]
+          vfmaddsub132ps 4096(%rdx), %ymm20, %ymm21
+
+// CHECK: vfmaddsub132ps -4096(%rdx), %ymm20, %ymm21
+// CHECK:  encoding: [0x62,0xe2,0x5d,0x20,0x96,0x6a,0x80]
+          vfmaddsub132ps -4096(%rdx), %ymm20, %ymm21
+
+// CHECK: vfmaddsub132ps -4128(%rdx), %ymm20, %ymm21
+// CHECK:  encoding: [0x62,0xe2,0x5d,0x20,0x96,0xaa,0xe0,0xef,0xff,0xff]
+          vfmaddsub132ps -4128(%rdx), %ymm20, %ymm21
+
+// CHECK: vfmaddsub132ps 508(%rdx){1to8}, %ymm20, %ymm21
+// CHECK:  encoding: [0x62,0xe2,0x5d,0x30,0x96,0x6a,0x7f]
+          vfmaddsub132ps 508(%rdx){1to8}, %ymm20, %ymm21
+
+// CHECK: vfmaddsub132ps 512(%rdx){1to8}, %ymm20, %ymm21
+// CHECK:  encoding: [0x62,0xe2,0x5d,0x30,0x96,0xaa,0x00,0x02,0x00,0x00]
+          vfmaddsub132ps 512(%rdx){1to8}, %ymm20, %ymm21
+
+// CHECK: vfmaddsub132ps -512(%rdx){1to8}, %ymm20, %ymm21
+// CHECK:  encoding: [0x62,0xe2,0x5d,0x30,0x96,0x6a,0x80]
+          vfmaddsub132ps -512(%rdx){1to8}, %ymm20, %ymm21
+
+// CHECK: vfmaddsub132ps -516(%rdx){1to8}, %ymm20, %ymm21
+// CHECK:  encoding: [0x62,0xe2,0x5d,0x30,0x96,0xaa,0xfc,0xfd,0xff,0xff]
+          vfmaddsub132ps -516(%rdx){1to8}, %ymm20, %ymm21
+
+// CHECK: vfmaddsub132pd %xmm20, %xmm27, %xmm26
+// CHECK:  encoding: [0x62,0x22,0xa5,0x00,0x96,0xd4]
+          vfmaddsub132pd %xmm20, %xmm27, %xmm26
+
+// CHECK: vfmaddsub132pd %xmm20, %xmm27, %xmm26 {%k6}
+// CHECK:  encoding: [0x62,0x22,0xa5,0x06,0x96,0xd4]
+          vfmaddsub132pd %xmm20, %xmm27, %xmm26 {%k6}
+
+// CHECK: vfmaddsub132pd %xmm20, %xmm27, %xmm26 {%k6} {z}
+// CHECK:  encoding: [0x62,0x22,0xa5,0x86,0x96,0xd4]
+          vfmaddsub132pd %xmm20, %xmm27, %xmm26 {%k6} {z}
+
+// CHECK: vfmaddsub132pd (%rcx), %xmm27, %xmm26
+// CHECK:  encoding: [0x62,0x62,0xa5,0x00,0x96,0x11]
+          vfmaddsub132pd (%rcx), %xmm27, %xmm26
+
+// CHECK: vfmaddsub132pd 291(%rax,%r14,8), %xmm27, %xmm26
+// CHECK:  encoding: [0x62,0x22,0xa5,0x00,0x96,0x94,0xf0,0x23,0x01,0x00,0x00]
+          vfmaddsub132pd 291(%rax,%r14,8), %xmm27, %xmm26
+
+// CHECK: vfmaddsub132pd (%rcx){1to2}, %xmm27, %xmm26
+// CHECK:  encoding: [0x62,0x62,0xa5,0x10,0x96,0x11]
+          vfmaddsub132pd (%rcx){1to2}, %xmm27, %xmm26
+
+// CHECK: vfmaddsub132pd 2032(%rdx), %xmm27, %xmm26
+// CHECK:  encoding: [0x62,0x62,0xa5,0x00,0x96,0x52,0x7f]
+          vfmaddsub132pd 2032(%rdx), %xmm27, %xmm26
+
+// CHECK: vfmaddsub132pd 2048(%rdx), %xmm27, %xmm26
+// CHECK:  encoding: [0x62,0x62,0xa5,0x00,0x96,0x92,0x00,0x08,0x00,0x00]
+          vfmaddsub132pd 2048(%rdx), %xmm27, %xmm26
+
+// CHECK: vfmaddsub132pd -2048(%rdx), %xmm27, %xmm26
+// CHECK:  encoding: [0x62,0x62,0xa5,0x00,0x96,0x52,0x80]
+          vfmaddsub132pd -2048(%rdx), %xmm27, %xmm26
+
+// CHECK: vfmaddsub132pd -2064(%rdx), %xmm27, %xmm26
+// CHECK:  encoding: [0x62,0x62,0xa5,0x00,0x96,0x92,0xf0,0xf7,0xff,0xff]
+          vfmaddsub132pd -2064(%rdx), %xmm27, %xmm26
+
+// CHECK: vfmaddsub132pd 1016(%rdx){1to2}, %xmm27, %xmm26
+// CHECK:  encoding: [0x62,0x62,0xa5,0x10,0x96,0x52,0x7f]
+          vfmaddsub132pd 1016(%rdx){1to2}, %xmm27, %xmm26
+
+// CHECK: vfmaddsub132pd 1024(%rdx){1to2}, %xmm27, %xmm26
+// CHECK:  encoding: [0x62,0x62,0xa5,0x10,0x96,0x92,0x00,0x04,0x00,0x00]
+          vfmaddsub132pd 1024(%rdx){1to2}, %xmm27, %xmm26
+
+// CHECK: vfmaddsub132pd -1024(%rdx){1to2}, %xmm27, %xmm26
+// CHECK:  encoding: [0x62,0x62,0xa5,0x10,0x96,0x52,0x80]
+          vfmaddsub132pd -1024(%rdx){1to2}, %xmm27, %xmm26
+
+// CHECK: vfmaddsub132pd -1032(%rdx){1to2}, %xmm27, %xmm26
+// CHECK:  encoding: [0x62,0x62,0xa5,0x10,0x96,0x92,0xf8,0xfb,0xff,0xff]
+          vfmaddsub132pd -1032(%rdx){1to2}, %xmm27, %xmm26
+
+// CHECK: vfmaddsub132pd %ymm28, %ymm18, %ymm30
+// CHECK:  encoding: [0x62,0x02,0xed,0x20,0x96,0xf4]
+          vfmaddsub132pd %ymm28, %ymm18, %ymm30
+
+// CHECK: vfmaddsub132pd %ymm28, %ymm18, %ymm30 {%k2}
+// CHECK:  encoding: [0x62,0x02,0xed,0x22,0x96,0xf4]
+          vfmaddsub132pd %ymm28, %ymm18, %ymm30 {%k2}
+
+// CHECK: vfmaddsub132pd %ymm28, %ymm18, %ymm30 {%k2} {z}
+// CHECK:  encoding: [0x62,0x02,0xed,0xa2,0x96,0xf4]
+          vfmaddsub132pd %ymm28, %ymm18, %ymm30 {%k2} {z}
+
+// CHECK: vfmaddsub132pd (%rcx), %ymm18, %ymm30
+// CHECK:  encoding: [0x62,0x62,0xed,0x20,0x96,0x31]
+          vfmaddsub132pd (%rcx), %ymm18, %ymm30
+
+// CHECK: vfmaddsub132pd 291(%rax,%r14,8), %ymm18, %ymm30
+// CHECK:  encoding: [0x62,0x22,0xed,0x20,0x96,0xb4,0xf0,0x23,0x01,0x00,0x00]
+          vfmaddsub132pd 291(%rax,%r14,8), %ymm18, %ymm30
+
+// CHECK: vfmaddsub132pd (%rcx){1to4}, %ymm18, %ymm30
+// CHECK:  encoding: [0x62,0x62,0xed,0x30,0x96,0x31]
+          vfmaddsub132pd (%rcx){1to4}, %ymm18, %ymm30
+
+// CHECK: vfmaddsub132pd 4064(%rdx), %ymm18, %ymm30
+// CHECK:  encoding: [0x62,0x62,0xed,0x20,0x96,0x72,0x7f]
+          vfmaddsub132pd 4064(%rdx), %ymm18, %ymm30
+
+// CHECK: vfmaddsub132pd 4096(%rdx), %ymm18, %ymm30
+// CHECK:  encoding: [0x62,0x62,0xed,0x20,0x96,0xb2,0x00,0x10,0x00,0x00]
+          vfmaddsub132pd 4096(%rdx), %ymm18, %ymm30
+
+// CHECK: vfmaddsub132pd -4096(%rdx), %ymm18, %ymm30
+// CHECK:  encoding: [0x62,0x62,0xed,0x20,0x96,0x72,0x80]
+          vfmaddsub132pd -4096(%rdx), %ymm18, %ymm30
+
+// CHECK: vfmaddsub132pd -4128(%rdx), %ymm18, %ymm30
+// CHECK:  encoding: [0x62,0x62,0xed,0x20,0x96,0xb2,0xe0,0xef,0xff,0xff]
+          vfmaddsub132pd -4128(%rdx), %ymm18, %ymm30
+
+// CHECK: vfmaddsub132pd 1016(%rdx){1to4}, %ymm18, %ymm30
+// CHECK:  encoding: [0x62,0x62,0xed,0x30,0x96,0x72,0x7f]
+          vfmaddsub132pd 1016(%rdx){1to4}, %ymm18, %ymm30
+
+// CHECK: vfmaddsub132pd 1024(%rdx){1to4}, %ymm18, %ymm30
+// CHECK:  encoding: [0x62,0x62,0xed,0x30,0x96,0xb2,0x00,0x04,0x00,0x00]
+          vfmaddsub132pd 1024(%rdx){1to4}, %ymm18, %ymm30
+
+// CHECK: vfmaddsub132pd -1024(%rdx){1to4}, %ymm18, %ymm30
+// CHECK:  encoding: [0x62,0x62,0xed,0x30,0x96,0x72,0x80]
+          vfmaddsub132pd -1024(%rdx){1to4}, %ymm18, %ymm30
+
+// CHECK: vfmaddsub132pd -1032(%rdx){1to4}, %ymm18, %ymm30
+// CHECK:  encoding: [0x62,0x62,0xed,0x30,0x96,0xb2,0xf8,0xfb,0xff,0xff]
+          vfmaddsub132pd -1032(%rdx){1to4}, %ymm18, %ymm30
+
+// CHECK: vfmaddsub213ps %xmm17, %xmm29, %xmm20
+// CHECK:  encoding: [0x62,0xa2,0x15,0x00,0xa6,0xe1]
+          vfmaddsub213ps %xmm17, %xmm29, %xmm20
+
+// CHECK: vfmaddsub213ps %xmm17, %xmm29, %xmm20 {%k1}
+// CHECK:  encoding: [0x62,0xa2,0x15,0x01,0xa6,0xe1]
+          vfmaddsub213ps %xmm17, %xmm29, %xmm20 {%k1}
+
+// CHECK: vfmaddsub213ps %xmm17, %xmm29, %xmm20 {%k1} {z}
+// CHECK:  encoding: [0x62,0xa2,0x15,0x81,0xa6,0xe1]
+          vfmaddsub213ps %xmm17, %xmm29, %xmm20 {%k1} {z}
+
+// CHECK: vfmaddsub213ps (%rcx), %xmm29, %xmm20
+// CHECK:  encoding: [0x62,0xe2,0x15,0x00,0xa6,0x21]
+          vfmaddsub213ps (%rcx), %xmm29, %xmm20
+
+// CHECK: vfmaddsub213ps 291(%rax,%r14,8), %xmm29, %xmm20
+// CHECK:  encoding: [0x62,0xa2,0x15,0x00,0xa6,0xa4,0xf0,0x23,0x01,0x00,0x00]
+          vfmaddsub213ps 291(%rax,%r14,8), %xmm29, %xmm20
+
+// CHECK: vfmaddsub213ps (%rcx){1to4}, %xmm29, %xmm20
+// CHECK:  encoding: [0x62,0xe2,0x15,0x10,0xa6,0x21]
+          vfmaddsub213ps (%rcx){1to4}, %xmm29, %xmm20
+
+// CHECK: vfmaddsub213ps 2032(%rdx), %xmm29, %xmm20
+// CHECK:  encoding: [0x62,0xe2,0x15,0x00,0xa6,0x62,0x7f]
+          vfmaddsub213ps 2032(%rdx), %xmm29, %xmm20
+
+// CHECK: vfmaddsub213ps 2048(%rdx), %xmm29, %xmm20
+// CHECK:  encoding: [0x62,0xe2,0x15,0x00,0xa6,0xa2,0x00,0x08,0x00,0x00]
+          vfmaddsub213ps 2048(%rdx), %xmm29, %xmm20
+
+// CHECK: vfmaddsub213ps -2048(%rdx), %xmm29, %xmm20
+// CHECK:  encoding: [0x62,0xe2,0x15,0x00,0xa6,0x62,0x80]
+          vfmaddsub213ps -2048(%rdx), %xmm29, %xmm20
+
+// CHECK: vfmaddsub213ps -2064(%rdx), %xmm29, %xmm20
+// CHECK:  encoding: [0x62,0xe2,0x15,0x00,0xa6,0xa2,0xf0,0xf7,0xff,0xff]
+          vfmaddsub213ps -2064(%rdx), %xmm29, %xmm20
+
+// CHECK: vfmaddsub213ps 508(%rdx){1to4}, %xmm29, %xmm20
+// CHECK:  encoding: [0x62,0xe2,0x15,0x10,0xa6,0x62,0x7f]
+          vfmaddsub213ps 508(%rdx){1to4}, %xmm29, %xmm20
+
+// CHECK: vfmaddsub213ps 512(%rdx){1to4}, %xmm29, %xmm20
+// CHECK:  encoding: [0x62,0xe2,0x15,0x10,0xa6,0xa2,0x00,0x02,0x00,0x00]
+          vfmaddsub213ps 512(%rdx){1to4}, %xmm29, %xmm20
+
+// CHECK: vfmaddsub213ps -512(%rdx){1to4}, %xmm29, %xmm20
+// CHECK:  encoding: [0x62,0xe2,0x15,0x10,0xa6,0x62,0x80]
+          vfmaddsub213ps -512(%rdx){1to4}, %xmm29, %xmm20
+
+// CHECK: vfmaddsub213ps -516(%rdx){1to4}, %xmm29, %xmm20
+// CHECK:  encoding: [0x62,0xe2,0x15,0x10,0xa6,0xa2,0xfc,0xfd,0xff,0xff]
+          vfmaddsub213ps -516(%rdx){1to4}, %xmm29, %xmm20
+
+// CHECK: vfmaddsub213ps %ymm23, %ymm27, %ymm25
+// CHECK:  encoding: [0x62,0x22,0x25,0x20,0xa6,0xcf]
+          vfmaddsub213ps %ymm23, %ymm27, %ymm25
+
+// CHECK: vfmaddsub213ps %ymm23, %ymm27, %ymm25 {%k4}
+// CHECK:  encoding: [0x62,0x22,0x25,0x24,0xa6,0xcf]
+          vfmaddsub213ps %ymm23, %ymm27, %ymm25 {%k4}
+
+// CHECK: vfmaddsub213ps %ymm23, %ymm27, %ymm25 {%k4} {z}
+// CHECK:  encoding: [0x62,0x22,0x25,0xa4,0xa6,0xcf]
+          vfmaddsub213ps %ymm23, %ymm27, %ymm25 {%k4} {z}
+
+// CHECK: vfmaddsub213ps (%rcx), %ymm27, %ymm25
+// CHECK:  encoding: [0x62,0x62,0x25,0x20,0xa6,0x09]
+          vfmaddsub213ps (%rcx), %ymm27, %ymm25
+
+// CHECK: vfmaddsub213ps 291(%rax,%r14,8), %ymm27, %ymm25
+// CHECK:  encoding: [0x62,0x22,0x25,0x20,0xa6,0x8c,0xf0,0x23,0x01,0x00,0x00]
+          vfmaddsub213ps 291(%rax,%r14,8), %ymm27, %ymm25
+
+// CHECK: vfmaddsub213ps (%rcx){1to8}, %ymm27, %ymm25
+// CHECK:  encoding: [0x62,0x62,0x25,0x30,0xa6,0x09]
+          vfmaddsub213ps (%rcx){1to8}, %ymm27, %ymm25
+
+// CHECK: vfmaddsub213ps 4064(%rdx), %ymm27, %ymm25
+// CHECK:  encoding: [0x62,0x62,0x25,0x20,0xa6,0x4a,0x7f]
+          vfmaddsub213ps 4064(%rdx), %ymm27, %ymm25
+
+// CHECK: vfmaddsub213ps 4096(%rdx), %ymm27, %ymm25
+// CHECK:  encoding: [0x62,0x62,0x25,0x20,0xa6,0x8a,0x00,0x10,0x00,0x00]
+          vfmaddsub213ps 4096(%rdx), %ymm27, %ymm25
+
+// CHECK: vfmaddsub213ps -4096(%rdx), %ymm27, %ymm25
+// CHECK:  encoding: [0x62,0x62,0x25,0x20,0xa6,0x4a,0x80]
+          vfmaddsub213ps -4096(%rdx), %ymm27, %ymm25
+
+// CHECK: vfmaddsub213ps -4128(%rdx), %ymm27, %ymm25
+// CHECK:  encoding: [0x62,0x62,0x25,0x20,0xa6,0x8a,0xe0,0xef,0xff,0xff]
+          vfmaddsub213ps -4128(%rdx), %ymm27, %ymm25
+
+// CHECK: vfmaddsub213ps 508(%rdx){1to8}, %ymm27, %ymm25
+// CHECK:  encoding: [0x62,0x62,0x25,0x30,0xa6,0x4a,0x7f]
+          vfmaddsub213ps 508(%rdx){1to8}, %ymm27, %ymm25
+
+// CHECK: vfmaddsub213ps 512(%rdx){1to8}, %ymm27, %ymm25
+// CHECK:  encoding: [0x62,0x62,0x25,0x30,0xa6,0x8a,0x00,0x02,0x00,0x00]
+          vfmaddsub213ps 512(%rdx){1to8}, %ymm27, %ymm25
+
+// CHECK: vfmaddsub213ps -512(%rdx){1to8}, %ymm27, %ymm25
+// CHECK:  encoding: [0x62,0x62,0x25,0x30,0xa6,0x4a,0x80]
+          vfmaddsub213ps -512(%rdx){1to8}, %ymm27, %ymm25
+
+// CHECK: vfmaddsub213ps -516(%rdx){1to8}, %ymm27, %ymm25
+// CHECK:  encoding: [0x62,0x62,0x25,0x30,0xa6,0x8a,0xfc,0xfd,0xff,0xff]
+          vfmaddsub213ps -516(%rdx){1to8}, %ymm27, %ymm25
+
+// CHECK: vfmaddsub213pd %xmm19, %xmm29, %xmm25
+// CHECK:  encoding: [0x62,0x22,0x95,0x00,0xa6,0xcb]
+          vfmaddsub213pd %xmm19, %xmm29, %xmm25
+
+// CHECK: vfmaddsub213pd %xmm19, %xmm29, %xmm25 {%k7}
+// CHECK:  encoding: [0x62,0x22,0x95,0x07,0xa6,0xcb]
+          vfmaddsub213pd %xmm19, %xmm29, %xmm25 {%k7}
+
+// CHECK: vfmaddsub213pd %xmm19, %xmm29, %xmm25 {%k7} {z}
+// CHECK:  encoding: [0x62,0x22,0x95,0x87,0xa6,0xcb]
+          vfmaddsub213pd %xmm19, %xmm29, %xmm25 {%k7} {z}
+
+// CHECK: vfmaddsub213pd (%rcx), %xmm29, %xmm25
+// CHECK:  encoding: [0x62,0x62,0x95,0x00,0xa6,0x09]
+          vfmaddsub213pd (%rcx), %xmm29, %xmm25
+
+// CHECK: vfmaddsub213pd 291(%rax,%r14,8), %xmm29, %xmm25
+// CHECK:  encoding: [0x62,0x22,0x95,0x00,0xa6,0x8c,0xf0,0x23,0x01,0x00,0x00]
+          vfmaddsub213pd 291(%rax,%r14,8), %xmm29, %xmm25
+
+// CHECK: vfmaddsub213pd (%rcx){1to2}, %xmm29, %xmm25
+// CHECK:  encoding: [0x62,0x62,0x95,0x10,0xa6,0x09]
+          vfmaddsub213pd (%rcx){1to2}, %xmm29, %xmm25
+
+// CHECK: vfmaddsub213pd 2032(%rdx), %xmm29, %xmm25
+// CHECK:  encoding: [0x62,0x62,0x95,0x00,0xa6,0x4a,0x7f]
+          vfmaddsub213pd 2032(%rdx), %xmm29, %xmm25
+
+// CHECK: vfmaddsub213pd 2048(%rdx), %xmm29, %xmm25
+// CHECK:  encoding: [0x62,0x62,0x95,0x00,0xa6,0x8a,0x00,0x08,0x00,0x00]
+          vfmaddsub213pd 2048(%rdx), %xmm29, %xmm25
+
+// CHECK: vfmaddsub213pd -2048(%rdx), %xmm29, %xmm25
+// CHECK:  encoding: [0x62,0x62,0x95,0x00,0xa6,0x4a,0x80]
+          vfmaddsub213pd -2048(%rdx), %xmm29, %xmm25
+
+// CHECK: vfmaddsub213pd -2064(%rdx), %xmm29, %xmm25
+// CHECK:  encoding: [0x62,0x62,0x95,0x00,0xa6,0x8a,0xf0,0xf7,0xff,0xff]
+          vfmaddsub213pd -2064(%rdx), %xmm29, %xmm25
+
+// CHECK: vfmaddsub213pd 1016(%rdx){1to2}, %xmm29, %xmm25
+// CHECK:  encoding: [0x62,0x62,0x95,0x10,0xa6,0x4a,0x7f]
+          vfmaddsub213pd 1016(%rdx){1to2}, %xmm29, %xmm25
+
+// CHECK: vfmaddsub213pd 1024(%rdx){1to2}, %xmm29, %xmm25
+// CHECK:  encoding: [0x62,0x62,0x95,0x10,0xa6,0x8a,0x00,0x04,0x00,0x00]
+          vfmaddsub213pd 1024(%rdx){1to2}, %xmm29, %xmm25
+
+// CHECK: vfmaddsub213pd -1024(%rdx){1to2}, %xmm29, %xmm25
+// CHECK:  encoding: [0x62,0x62,0x95,0x10,0xa6,0x4a,0x80]
+          vfmaddsub213pd -1024(%rdx){1to2}, %xmm29, %xmm25
+
+// CHECK: vfmaddsub213pd -1032(%rdx){1to2}, %xmm29, %xmm25
+// CHECK:  encoding: [0x62,0x62,0x95,0x10,0xa6,0x8a,0xf8,0xfb,0xff,0xff]
+          vfmaddsub213pd -1032(%rdx){1to2}, %xmm29, %xmm25
+
+// CHECK: vfmaddsub213pd %ymm20, %ymm17, %ymm21
+// CHECK:  encoding: [0x62,0xa2,0xf5,0x20,0xa6,0xec]
+          vfmaddsub213pd %ymm20, %ymm17, %ymm21
+
+// CHECK: vfmaddsub213pd %ymm20, %ymm17, %ymm21 {%k4}
+// CHECK:  encoding: [0x62,0xa2,0xf5,0x24,0xa6,0xec]
+          vfmaddsub213pd %ymm20, %ymm17, %ymm21 {%k4}
+
+// CHECK: vfmaddsub213pd %ymm20, %ymm17, %ymm21 {%k4} {z}
+// CHECK:  encoding: [0x62,0xa2,0xf5,0xa4,0xa6,0xec]
+          vfmaddsub213pd %ymm20, %ymm17, %ymm21 {%k4} {z}
+
+// CHECK: vfmaddsub213pd (%rcx), %ymm17, %ymm21
+// CHECK:  encoding: [0x62,0xe2,0xf5,0x20,0xa6,0x29]
+          vfmaddsub213pd (%rcx), %ymm17, %ymm21
+
+// CHECK: vfmaddsub213pd 291(%rax,%r14,8), %ymm17, %ymm21
+// CHECK:  encoding: [0x62,0xa2,0xf5,0x20,0xa6,0xac,0xf0,0x23,0x01,0x00,0x00]
+          vfmaddsub213pd 291(%rax,%r14,8), %ymm17, %ymm21
+
+// CHECK: vfmaddsub213pd (%rcx){1to4}, %ymm17, %ymm21
+// CHECK:  encoding: [0x62,0xe2,0xf5,0x30,0xa6,0x29]
+          vfmaddsub213pd (%rcx){1to4}, %ymm17, %ymm21
+
+// CHECK: vfmaddsub213pd 4064(%rdx), %ymm17, %ymm21
+// CHECK:  encoding: [0x62,0xe2,0xf5,0x20,0xa6,0x6a,0x7f]
+          vfmaddsub213pd 4064(%rdx), %ymm17, %ymm21
+
+// CHECK: vfmaddsub213pd 4096(%rdx), %ymm17, %ymm21
+// CHECK:  encoding: [0x62,0xe2,0xf5,0x20,0xa6,0xaa,0x00,0x10,0x00,0x00]
+          vfmaddsub213pd 4096(%rdx), %ymm17, %ymm21
+
+// CHECK: vfmaddsub213pd -4096(%rdx), %ymm17, %ymm21
+// CHECK:  encoding: [0x62,0xe2,0xf5,0x20,0xa6,0x6a,0x80]
+          vfmaddsub213pd -4096(%rdx), %ymm17, %ymm21
+
+// CHECK: vfmaddsub213pd -4128(%rdx), %ymm17, %ymm21
+// CHECK:  encoding: [0x62,0xe2,0xf5,0x20,0xa6,0xaa,0xe0,0xef,0xff,0xff]
+          vfmaddsub213pd -4128(%rdx), %ymm17, %ymm21
+
+// CHECK: vfmaddsub213pd 1016(%rdx){1to4}, %ymm17, %ymm21
+// CHECK:  encoding: [0x62,0xe2,0xf5,0x30,0xa6,0x6a,0x7f]
+          vfmaddsub213pd 1016(%rdx){1to4}, %ymm17, %ymm21
+
+// CHECK: vfmaddsub213pd 1024(%rdx){1to4}, %ymm17, %ymm21
+// CHECK:  encoding: [0x62,0xe2,0xf5,0x30,0xa6,0xaa,0x00,0x04,0x00,0x00]
+          vfmaddsub213pd 1024(%rdx){1to4}, %ymm17, %ymm21
+
+// CHECK: vfmaddsub213pd -1024(%rdx){1to4}, %ymm17, %ymm21
+// CHECK:  encoding: [0x62,0xe2,0xf5,0x30,0xa6,0x6a,0x80]
+          vfmaddsub213pd -1024(%rdx){1to4}, %ymm17, %ymm21
+
+// CHECK: vfmaddsub213pd -1032(%rdx){1to4}, %ymm17, %ymm21
+// CHECK:  encoding: [0x62,0xe2,0xf5,0x30,0xa6,0xaa,0xf8,0xfb,0xff,0xff]
+          vfmaddsub213pd -1032(%rdx){1to4}, %ymm17, %ymm21
+
+// CHECK: vfmaddsub231ps %xmm20, %xmm29, %xmm19
+// CHECK:  encoding: [0x62,0xa2,0x15,0x00,0xb6,0xdc]
+          vfmaddsub231ps %xmm20, %xmm29, %xmm19
+
+// CHECK: vfmaddsub231ps %xmm20, %xmm29, %xmm19 {%k6}
+// CHECK:  encoding: [0x62,0xa2,0x15,0x06,0xb6,0xdc]
+          vfmaddsub231ps %xmm20, %xmm29, %xmm19 {%k6}
+
+// CHECK: vfmaddsub231ps %xmm20, %xmm29, %xmm19 {%k6} {z}
+// CHECK:  encoding: [0x62,0xa2,0x15,0x86,0xb6,0xdc]
+          vfmaddsub231ps %xmm20, %xmm29, %xmm19 {%k6} {z}
+
+// CHECK: vfmaddsub231ps (%rcx), %xmm29, %xmm19
+// CHECK:  encoding: [0x62,0xe2,0x15,0x00,0xb6,0x19]
+          vfmaddsub231ps (%rcx), %xmm29, %xmm19
+
+// CHECK: vfmaddsub231ps 291(%rax,%r14,8), %xmm29, %xmm19
+// CHECK:  encoding: [0x62,0xa2,0x15,0x00,0xb6,0x9c,0xf0,0x23,0x01,0x00,0x00]
+          vfmaddsub231ps 291(%rax,%r14,8), %xmm29, %xmm19
+
+// CHECK: vfmaddsub231ps (%rcx){1to4}, %xmm29, %xmm19
+// CHECK:  encoding: [0x62,0xe2,0x15,0x10,0xb6,0x19]
+          vfmaddsub231ps (%rcx){1to4}, %xmm29, %xmm19
+
+// CHECK: vfmaddsub231ps 2032(%rdx), %xmm29, %xmm19
+// CHECK:  encoding: [0x62,0xe2,0x15,0x00,0xb6,0x5a,0x7f]
+          vfmaddsub231ps 2032(%rdx), %xmm29, %xmm19
+
+// CHECK: vfmaddsub231ps 2048(%rdx), %xmm29, %xmm19
+// CHECK:  encoding: [0x62,0xe2,0x15,0x00,0xb6,0x9a,0x00,0x08,0x00,0x00]
+          vfmaddsub231ps 2048(%rdx), %xmm29, %xmm19
+
+// CHECK: vfmaddsub231ps -2048(%rdx), %xmm29, %xmm19
+// CHECK:  encoding: [0x62,0xe2,0x15,0x00,0xb6,0x5a,0x80]
+          vfmaddsub231ps -2048(%rdx), %xmm29, %xmm19
+
+// CHECK: vfmaddsub231ps -2064(%rdx), %xmm29, %xmm19
+// CHECK:  encoding: [0x62,0xe2,0x15,0x00,0xb6,0x9a,0xf0,0xf7,0xff,0xff]
+          vfmaddsub231ps -2064(%rdx), %xmm29, %xmm19
+
+// CHECK: vfmaddsub231ps 508(%rdx){1to4}, %xmm29, %xmm19
+// CHECK:  encoding: [0x62,0xe2,0x15,0x10,0xb6,0x5a,0x7f]
+          vfmaddsub231ps 508(%rdx){1to4}, %xmm29, %xmm19
+
+// CHECK: vfmaddsub231ps 512(%rdx){1to4}, %xmm29, %xmm19
+// CHECK:  encoding: [0x62,0xe2,0x15,0x10,0xb6,0x9a,0x00,0x02,0x00,0x00]
+          vfmaddsub231ps 512(%rdx){1to4}, %xmm29, %xmm19
+
+// CHECK: vfmaddsub231ps -512(%rdx){1to4}, %xmm29, %xmm19
+// CHECK:  encoding: [0x62,0xe2,0x15,0x10,0xb6,0x5a,0x80]
+          vfmaddsub231ps -512(%rdx){1to4}, %xmm29, %xmm19
+
+// CHECK: vfmaddsub231ps -516(%rdx){1to4}, %xmm29, %xmm19
+// CHECK:  encoding: [0x62,0xe2,0x15,0x10,0xb6,0x9a,0xfc,0xfd,0xff,0xff]
+          vfmaddsub231ps -516(%rdx){1to4}, %xmm29, %xmm19
+
+// CHECK: vfmaddsub231ps %ymm17, %ymm24, %ymm19
+// CHECK:  encoding: [0x62,0xa2,0x3d,0x20,0xb6,0xd9]
+          vfmaddsub231ps %ymm17, %ymm24, %ymm19
+
+// CHECK: vfmaddsub231ps %ymm17, %ymm24, %ymm19 {%k1}
+// CHECK:  encoding: [0x62,0xa2,0x3d,0x21,0xb6,0xd9]
+          vfmaddsub231ps %ymm17, %ymm24, %ymm19 {%k1}
+
+// CHECK: vfmaddsub231ps %ymm17, %ymm24, %ymm19 {%k1} {z}
+// CHECK:  encoding: [0x62,0xa2,0x3d,0xa1,0xb6,0xd9]
+          vfmaddsub231ps %ymm17, %ymm24, %ymm19 {%k1} {z}
+
+// CHECK: vfmaddsub231ps (%rcx), %ymm24, %ymm19
+// CHECK:  encoding: [0x62,0xe2,0x3d,0x20,0xb6,0x19]
+          vfmaddsub231ps (%rcx), %ymm24, %ymm19
+
+// CHECK: vfmaddsub231ps 291(%rax,%r14,8), %ymm24, %ymm19
+// CHECK:  encoding: [0x62,0xa2,0x3d,0x20,0xb6,0x9c,0xf0,0x23,0x01,0x00,0x00]
+          vfmaddsub231ps 291(%rax,%r14,8), %ymm24, %ymm19
+
+// CHECK: vfmaddsub231ps (%rcx){1to8}, %ymm24, %ymm19
+// CHECK:  encoding: [0x62,0xe2,0x3d,0x30,0xb6,0x19]
+          vfmaddsub231ps (%rcx){1to8}, %ymm24, %ymm19
+
+// CHECK: vfmaddsub231ps 4064(%rdx), %ymm24, %ymm19
+// CHECK:  encoding: [0x62,0xe2,0x3d,0x20,0xb6,0x5a,0x7f]
+          vfmaddsub231ps 4064(%rdx), %ymm24, %ymm19
+
+// CHECK: vfmaddsub231ps 4096(%rdx), %ymm24, %ymm19
+// CHECK:  encoding: [0x62,0xe2,0x3d,0x20,0xb6,0x9a,0x00,0x10,0x00,0x00]
+          vfmaddsub231ps 4096(%rdx), %ymm24, %ymm19
+
+// CHECK: vfmaddsub231ps -4096(%rdx), %ymm24, %ymm19
+// CHECK:  encoding: [0x62,0xe2,0x3d,0x20,0xb6,0x5a,0x80]
+          vfmaddsub231ps -4096(%rdx), %ymm24, %ymm19
+
+// CHECK: vfmaddsub231ps -4128(%rdx), %ymm24, %ymm19
+// CHECK:  encoding: [0x62,0xe2,0x3d,0x20,0xb6,0x9a,0xe0,0xef,0xff,0xff]
+          vfmaddsub231ps -4128(%rdx), %ymm24, %ymm19
+
+// CHECK: vfmaddsub231ps 508(%rdx){1to8}, %ymm24, %ymm19
+// CHECK:  encoding: [0x62,0xe2,0x3d,0x30,0xb6,0x5a,0x7f]
+          vfmaddsub231ps 508(%rdx){1to8}, %ymm24, %ymm19
+
+// CHECK: vfmaddsub231ps 512(%rdx){1to8}, %ymm24, %ymm19
+// CHECK:  encoding: [0x62,0xe2,0x3d,0x30,0xb6,0x9a,0x00,0x02,0x00,0x00]
+          vfmaddsub231ps 512(%rdx){1to8}, %ymm24, %ymm19
+
+// CHECK: vfmaddsub231ps -512(%rdx){1to8}, %ymm24, %ymm19
+// CHECK:  encoding: [0x62,0xe2,0x3d,0x30,0xb6,0x5a,0x80]
+          vfmaddsub231ps -512(%rdx){1to8}, %ymm24, %ymm19
+
+// CHECK: vfmaddsub231ps -516(%rdx){1to8}, %ymm24, %ymm19
+// CHECK:  encoding: [0x62,0xe2,0x3d,0x30,0xb6,0x9a,0xfc,0xfd,0xff,0xff]
+          vfmaddsub231ps -516(%rdx){1to8}, %ymm24, %ymm19
+
+// CHECK: vfmaddsub231pd %xmm28, %xmm26, %xmm23
+// CHECK:  encoding: [0x62,0x82,0xad,0x00,0xb6,0xfc]
+          vfmaddsub231pd %xmm28, %xmm26, %xmm23
+
+// CHECK: vfmaddsub231pd %xmm28, %xmm26, %xmm23 {%k7}
+// CHECK:  encoding: [0x62,0x82,0xad,0x07,0xb6,0xfc]
+          vfmaddsub231pd %xmm28, %xmm26, %xmm23 {%k7}
+
+// CHECK: vfmaddsub231pd %xmm28, %xmm26, %xmm23 {%k7} {z}
+// CHECK:  encoding: [0x62,0x82,0xad,0x87,0xb6,0xfc]
+          vfmaddsub231pd %xmm28, %xmm26, %xmm23 {%k7} {z}
+
+// CHECK: vfmaddsub231pd (%rcx), %xmm26, %xmm23
+// CHECK:  encoding: [0x62,0xe2,0xad,0x00,0xb6,0x39]
+          vfmaddsub231pd (%rcx), %xmm26, %xmm23
+
+// CHECK: vfmaddsub231pd 291(%rax,%r14,8), %xmm26, %xmm23
+// CHECK:  encoding: [0x62,0xa2,0xad,0x00,0xb6,0xbc,0xf0,0x23,0x01,0x00,0x00]
+          vfmaddsub231pd 291(%rax,%r14,8), %xmm26, %xmm23
+
+// CHECK: vfmaddsub231pd (%rcx){1to2}, %xmm26, %xmm23
+// CHECK:  encoding: [0x62,0xe2,0xad,0x10,0xb6,0x39]
+          vfmaddsub231pd (%rcx){1to2}, %xmm26, %xmm23
+
+// CHECK: vfmaddsub231pd 2032(%rdx), %xmm26, %xmm23
+// CHECK:  encoding: [0x62,0xe2,0xad,0x00,0xb6,0x7a,0x7f]
+          vfmaddsub231pd 2032(%rdx), %xmm26, %xmm23
+
+// CHECK: vfmaddsub231pd 2048(%rdx), %xmm26, %xmm23
+// CHECK:  encoding: [0x62,0xe2,0xad,0x00,0xb6,0xba,0x00,0x08,0x00,0x00]
+          vfmaddsub231pd 2048(%rdx), %xmm26, %xmm23
+
+// CHECK: vfmaddsub231pd -2048(%rdx), %xmm26, %xmm23
+// CHECK:  encoding: [0x62,0xe2,0xad,0x00,0xb6,0x7a,0x80]
+          vfmaddsub231pd -2048(%rdx), %xmm26, %xmm23
+
+// CHECK: vfmaddsub231pd -2064(%rdx), %xmm26, %xmm23
+// CHECK:  encoding: [0x62,0xe2,0xad,0x00,0xb6,0xba,0xf0,0xf7,0xff,0xff]
+          vfmaddsub231pd -2064(%rdx), %xmm26, %xmm23
+
+// CHECK: vfmaddsub231pd 1016(%rdx){1to2}, %xmm26, %xmm23
+// CHECK:  encoding: [0x62,0xe2,0xad,0x10,0xb6,0x7a,0x7f]
+          vfmaddsub231pd 1016(%rdx){1to2}, %xmm26, %xmm23
+
+// CHECK: vfmaddsub231pd 1024(%rdx){1to2}, %xmm26, %xmm23
+// CHECK:  encoding: [0x62,0xe2,0xad,0x10,0xb6,0xba,0x00,0x04,0x00,0x00]
+          vfmaddsub231pd 1024(%rdx){1to2}, %xmm26, %xmm23
+
+// CHECK: vfmaddsub231pd -1024(%rdx){1to2}, %xmm26, %xmm23
+// CHECK:  encoding: [0x62,0xe2,0xad,0x10,0xb6,0x7a,0x80]
+          vfmaddsub231pd -1024(%rdx){1to2}, %xmm26, %xmm23
+
+// CHECK: vfmaddsub231pd -1032(%rdx){1to2}, %xmm26, %xmm23
+// CHECK:  encoding: [0x62,0xe2,0xad,0x10,0xb6,0xba,0xf8,0xfb,0xff,0xff]
+          vfmaddsub231pd -1032(%rdx){1to2}, %xmm26, %xmm23
+
+// CHECK: vfmaddsub231pd %ymm27, %ymm25, %ymm30
+// CHECK:  encoding: [0x62,0x02,0xb5,0x20,0xb6,0xf3]
+          vfmaddsub231pd %ymm27, %ymm25, %ymm30
+
+// CHECK: vfmaddsub231pd %ymm27, %ymm25, %ymm30 {%k5}
+// CHECK:  encoding: [0x62,0x02,0xb5,0x25,0xb6,0xf3]
+          vfmaddsub231pd %ymm27, %ymm25, %ymm30 {%k5}
+
+// CHECK: vfmaddsub231pd %ymm27, %ymm25, %ymm30 {%k5} {z}
+// CHECK:  encoding: [0x62,0x02,0xb5,0xa5,0xb6,0xf3]
+          vfmaddsub231pd %ymm27, %ymm25, %ymm30 {%k5} {z}
+
+// CHECK: vfmaddsub231pd (%rcx), %ymm25, %ymm30
+// CHECK:  encoding: [0x62,0x62,0xb5,0x20,0xb6,0x31]
+          vfmaddsub231pd (%rcx), %ymm25, %ymm30
+
+// CHECK: vfmaddsub231pd 291(%rax,%r14,8), %ymm25, %ymm30
+// CHECK:  encoding: [0x62,0x22,0xb5,0x20,0xb6,0xb4,0xf0,0x23,0x01,0x00,0x00]
+          vfmaddsub231pd 291(%rax,%r14,8), %ymm25, %ymm30
+
+// CHECK: vfmaddsub231pd (%rcx){1to4}, %ymm25, %ymm30
+// CHECK:  encoding: [0x62,0x62,0xb5,0x30,0xb6,0x31]
+          vfmaddsub231pd (%rcx){1to4}, %ymm25, %ymm30
+
+// CHECK: vfmaddsub231pd 4064(%rdx), %ymm25, %ymm30
+// CHECK:  encoding: [0x62,0x62,0xb5,0x20,0xb6,0x72,0x7f]
+          vfmaddsub231pd 4064(%rdx), %ymm25, %ymm30
+
+// CHECK: vfmaddsub231pd 4096(%rdx), %ymm25, %ymm30
+// CHECK:  encoding: [0x62,0x62,0xb5,0x20,0xb6,0xb2,0x00,0x10,0x00,0x00]
+          vfmaddsub231pd 4096(%rdx), %ymm25, %ymm30
+
+// CHECK: vfmaddsub231pd -4096(%rdx), %ymm25, %ymm30
+// CHECK:  encoding: [0x62,0x62,0xb5,0x20,0xb6,0x72,0x80]
+          vfmaddsub231pd -4096(%rdx), %ymm25, %ymm30
+
+// CHECK: vfmaddsub231pd -4128(%rdx), %ymm25, %ymm30
+// CHECK:  encoding: [0x62,0x62,0xb5,0x20,0xb6,0xb2,0xe0,0xef,0xff,0xff]
+          vfmaddsub231pd -4128(%rdx), %ymm25, %ymm30
+
+// CHECK: vfmaddsub231pd 1016(%rdx){1to4}, %ymm25, %ymm30
+// CHECK:  encoding: [0x62,0x62,0xb5,0x30,0xb6,0x72,0x7f]
+          vfmaddsub231pd 1016(%rdx){1to4}, %ymm25, %ymm30
+
+// CHECK: vfmaddsub231pd 1024(%rdx){1to4}, %ymm25, %ymm30
+// CHECK:  encoding: [0x62,0x62,0xb5,0x30,0xb6,0xb2,0x00,0x04,0x00,0x00]
+          vfmaddsub231pd 1024(%rdx){1to4}, %ymm25, %ymm30
+
+// CHECK: vfmaddsub231pd -1024(%rdx){1to4}, %ymm25, %ymm30
+// CHECK:  encoding: [0x62,0x62,0xb5,0x30,0xb6,0x72,0x80]
+          vfmaddsub231pd -1024(%rdx){1to4}, %ymm25, %ymm30
+
+// CHECK: vfmaddsub231pd -1032(%rdx){1to4}, %ymm25, %ymm30
+// CHECK:  encoding: [0x62,0x62,0xb5,0x30,0xb6,0xb2,0xf8,0xfb,0xff,0xff]
+          vfmaddsub231pd -1032(%rdx){1to4}, %ymm25, %ymm30
+
+// CHECK: vfmsubadd132ps %xmm20, %xmm23, %xmm24
+// CHECK:  encoding: [0x62,0x22,0x45,0x00,0x97,0xc4]
+          vfmsubadd132ps %xmm20, %xmm23, %xmm24
+
+// CHECK: vfmsubadd132ps %xmm20, %xmm23, %xmm24 {%k5}
+// CHECK:  encoding: [0x62,0x22,0x45,0x05,0x97,0xc4]
+          vfmsubadd132ps %xmm20, %xmm23, %xmm24 {%k5}
+
+// CHECK: vfmsubadd132ps %xmm20, %xmm23, %xmm24 {%k5} {z}
+// CHECK:  encoding: [0x62,0x22,0x45,0x85,0x97,0xc4]
+          vfmsubadd132ps %xmm20, %xmm23, %xmm24 {%k5} {z}
+
+// CHECK: vfmsubadd132ps (%rcx), %xmm23, %xmm24
+// CHECK:  encoding: [0x62,0x62,0x45,0x00,0x97,0x01]
+          vfmsubadd132ps (%rcx), %xmm23, %xmm24
+
+// CHECK: vfmsubadd132ps 291(%rax,%r14,8), %xmm23, %xmm24
+// CHECK:  encoding: [0x62,0x22,0x45,0x00,0x97,0x84,0xf0,0x23,0x01,0x00,0x00]
+          vfmsubadd132ps 291(%rax,%r14,8), %xmm23, %xmm24
+
+// CHECK: vfmsubadd132ps (%rcx){1to4}, %xmm23, %xmm24
+// CHECK:  encoding: [0x62,0x62,0x45,0x10,0x97,0x01]
+          vfmsubadd132ps (%rcx){1to4}, %xmm23, %xmm24
+
+// CHECK: vfmsubadd132ps 2032(%rdx), %xmm23, %xmm24
+// CHECK:  encoding: [0x62,0x62,0x45,0x00,0x97,0x42,0x7f]
+          vfmsubadd132ps 2032(%rdx), %xmm23, %xmm24
+
+// CHECK: vfmsubadd132ps 2048(%rdx), %xmm23, %xmm24
+// CHECK:  encoding: [0x62,0x62,0x45,0x00,0x97,0x82,0x00,0x08,0x00,0x00]
+          vfmsubadd132ps 2048(%rdx), %xmm23, %xmm24
+
+// CHECK: vfmsubadd132ps -2048(%rdx), %xmm23, %xmm24
+// CHECK:  encoding: [0x62,0x62,0x45,0x00,0x97,0x42,0x80]
+          vfmsubadd132ps -2048(%rdx), %xmm23, %xmm24
+
+// CHECK: vfmsubadd132ps -2064(%rdx), %xmm23, %xmm24
+// CHECK:  encoding: [0x62,0x62,0x45,0x00,0x97,0x82,0xf0,0xf7,0xff,0xff]
+          vfmsubadd132ps -2064(%rdx), %xmm23, %xmm24
+
+// CHECK: vfmsubadd132ps 508(%rdx){1to4}, %xmm23, %xmm24
+// CHECK:  encoding: [0x62,0x62,0x45,0x10,0x97,0x42,0x7f]
+          vfmsubadd132ps 508(%rdx){1to4}, %xmm23, %xmm24
+
+// CHECK: vfmsubadd132ps 512(%rdx){1to4}, %xmm23, %xmm24
+// CHECK:  encoding: [0x62,0x62,0x45,0x10,0x97,0x82,0x00,0x02,0x00,0x00]
+          vfmsubadd132ps 512(%rdx){1to4}, %xmm23, %xmm24
+
+// CHECK: vfmsubadd132ps -512(%rdx){1to4}, %xmm23, %xmm24
+// CHECK:  encoding: [0x62,0x62,0x45,0x10,0x97,0x42,0x80]
+          vfmsubadd132ps -512(%rdx){1to4}, %xmm23, %xmm24
+
+// CHECK: vfmsubadd132ps -516(%rdx){1to4}, %xmm23, %xmm24
+// CHECK:  encoding: [0x62,0x62,0x45,0x10,0x97,0x82,0xfc,0xfd,0xff,0xff]
+          vfmsubadd132ps -516(%rdx){1to4}, %xmm23, %xmm24
+
+// CHECK: vfmsubadd132ps %ymm23, %ymm28, %ymm23
+// CHECK:  encoding: [0x62,0xa2,0x1d,0x20,0x97,0xff]
+          vfmsubadd132ps %ymm23, %ymm28, %ymm23
+
+// CHECK: vfmsubadd132ps %ymm23, %ymm28, %ymm23 {%k1}
+// CHECK:  encoding: [0x62,0xa2,0x1d,0x21,0x97,0xff]
+          vfmsubadd132ps %ymm23, %ymm28, %ymm23 {%k1}
+
+// CHECK: vfmsubadd132ps %ymm23, %ymm28, %ymm23 {%k1} {z}
+// CHECK:  encoding: [0x62,0xa2,0x1d,0xa1,0x97,0xff]
+          vfmsubadd132ps %ymm23, %ymm28, %ymm23 {%k1} {z}
+
+// CHECK: vfmsubadd132ps (%rcx), %ymm28, %ymm23
+// CHECK:  encoding: [0x62,0xe2,0x1d,0x20,0x97,0x39]
+          vfmsubadd132ps (%rcx), %ymm28, %ymm23
+
+// CHECK: vfmsubadd132ps 291(%rax,%r14,8), %ymm28, %ymm23
+// CHECK:  encoding: [0x62,0xa2,0x1d,0x20,0x97,0xbc,0xf0,0x23,0x01,0x00,0x00]
+          vfmsubadd132ps 291(%rax,%r14,8), %ymm28, %ymm23
+
+// CHECK: vfmsubadd132ps (%rcx){1to8}, %ymm28, %ymm23
+// CHECK:  encoding: [0x62,0xe2,0x1d,0x30,0x97,0x39]
+          vfmsubadd132ps (%rcx){1to8}, %ymm28, %ymm23
+
+// CHECK: vfmsubadd132ps 4064(%rdx), %ymm28, %ymm23
+// CHECK:  encoding: [0x62,0xe2,0x1d,0x20,0x97,0x7a,0x7f]
+          vfmsubadd132ps 4064(%rdx), %ymm28, %ymm23
+
+// CHECK: vfmsubadd132ps 4096(%rdx), %ymm28, %ymm23
+// CHECK:  encoding: [0x62,0xe2,0x1d,0x20,0x97,0xba,0x00,0x10,0x00,0x00]
+          vfmsubadd132ps 4096(%rdx), %ymm28, %ymm23
+
+// CHECK: vfmsubadd132ps -4096(%rdx), %ymm28, %ymm23
+// CHECK:  encoding: [0x62,0xe2,0x1d,0x20,0x97,0x7a,0x80]
+          vfmsubadd132ps -4096(%rdx), %ymm28, %ymm23
+
+// CHECK: vfmsubadd132ps -4128(%rdx), %ymm28, %ymm23
+// CHECK:  encoding: [0x62,0xe2,0x1d,0x20,0x97,0xba,0xe0,0xef,0xff,0xff]
+          vfmsubadd132ps -4128(%rdx), %ymm28, %ymm23
+
+// CHECK: vfmsubadd132ps 508(%rdx){1to8}, %ymm28, %ymm23
+// CHECK:  encoding: [0x62,0xe2,0x1d,0x30,0x97,0x7a,0x7f]
+          vfmsubadd132ps 508(%rdx){1to8}, %ymm28, %ymm23
+
+// CHECK: vfmsubadd132ps 512(%rdx){1to8}, %ymm28, %ymm23
+// CHECK:  encoding: [0x62,0xe2,0x1d,0x30,0x97,0xba,0x00,0x02,0x00,0x00]
+          vfmsubadd132ps 512(%rdx){1to8}, %ymm28, %ymm23
+
+// CHECK: vfmsubadd132ps -512(%rdx){1to8}, %ymm28, %ymm23
+// CHECK:  encoding: [0x62,0xe2,0x1d,0x30,0x97,0x7a,0x80]
+          vfmsubadd132ps -512(%rdx){1to8}, %ymm28, %ymm23
+
+// CHECK: vfmsubadd132ps -516(%rdx){1to8}, %ymm28, %ymm23
+// CHECK:  encoding: [0x62,0xe2,0x1d,0x30,0x97,0xba,0xfc,0xfd,0xff,0xff]
+          vfmsubadd132ps -516(%rdx){1to8}, %ymm28, %ymm23
+
+// CHECK: vfmsubadd132pd %xmm24, %xmm26, %xmm22
+// CHECK:  encoding: [0x62,0x82,0xad,0x00,0x97,0xf0]
+          vfmsubadd132pd %xmm24, %xmm26, %xmm22
+
+// CHECK: vfmsubadd132pd %xmm24, %xmm26, %xmm22 {%k3}
+// CHECK:  encoding: [0x62,0x82,0xad,0x03,0x97,0xf0]
+          vfmsubadd132pd %xmm24, %xmm26, %xmm22 {%k3}
+
+// CHECK: vfmsubadd132pd %xmm24, %xmm26, %xmm22 {%k3} {z}
+// CHECK:  encoding: [0x62,0x82,0xad,0x83,0x97,0xf0]
+          vfmsubadd132pd %xmm24, %xmm26, %xmm22 {%k3} {z}
+
+// CHECK: vfmsubadd132pd (%rcx), %xmm26, %xmm22
+// CHECK:  encoding: [0x62,0xe2,0xad,0x00,0x97,0x31]
+          vfmsubadd132pd (%rcx), %xmm26, %xmm22
+
+// CHECK: vfmsubadd132pd 291(%rax,%r14,8), %xmm26, %xmm22
+// CHECK:  encoding: [0x62,0xa2,0xad,0x00,0x97,0xb4,0xf0,0x23,0x01,0x00,0x00]
+          vfmsubadd132pd 291(%rax,%r14,8), %xmm26, %xmm22
+
+// CHECK: vfmsubadd132pd (%rcx){1to2}, %xmm26, %xmm22
+// CHECK:  encoding: [0x62,0xe2,0xad,0x10,0x97,0x31]
+          vfmsubadd132pd (%rcx){1to2}, %xmm26, %xmm22
+
+// CHECK: vfmsubadd132pd 2032(%rdx), %xmm26, %xmm22
+// CHECK:  encoding: [0x62,0xe2,0xad,0x00,0x97,0x72,0x7f]
+          vfmsubadd132pd 2032(%rdx), %xmm26, %xmm22
+
+// CHECK: vfmsubadd132pd 2048(%rdx), %xmm26, %xmm22
+// CHECK:  encoding: [0x62,0xe2,0xad,0x00,0x97,0xb2,0x00,0x08,0x00,0x00]
+          vfmsubadd132pd 2048(%rdx), %xmm26, %xmm22
+
+// CHECK: vfmsubadd132pd -2048(%rdx), %xmm26, %xmm22
+// CHECK:  encoding: [0x62,0xe2,0xad,0x00,0x97,0x72,0x80]
+          vfmsubadd132pd -2048(%rdx), %xmm26, %xmm22
+
+// CHECK: vfmsubadd132pd -2064(%rdx), %xmm26, %xmm22
+// CHECK:  encoding: [0x62,0xe2,0xad,0x00,0x97,0xb2,0xf0,0xf7,0xff,0xff]
+          vfmsubadd132pd -2064(%rdx), %xmm26, %xmm22
+
+// CHECK: vfmsubadd132pd 1016(%rdx){1to2}, %xmm26, %xmm22
+// CHECK:  encoding: [0x62,0xe2,0xad,0x10,0x97,0x72,0x7f]
+          vfmsubadd132pd 1016(%rdx){1to2}, %xmm26, %xmm22
+
+// CHECK: vfmsubadd132pd 1024(%rdx){1to2}, %xmm26, %xmm22
+// CHECK:  encoding: [0x62,0xe2,0xad,0x10,0x97,0xb2,0x00,0x04,0x00,0x00]
+          vfmsubadd132pd 1024(%rdx){1to2}, %xmm26, %xmm22
+
+// CHECK: vfmsubadd132pd -1024(%rdx){1to2}, %xmm26, %xmm22
+// CHECK:  encoding: [0x62,0xe2,0xad,0x10,0x97,0x72,0x80]
+          vfmsubadd132pd -1024(%rdx){1to2}, %xmm26, %xmm22
+
+// CHECK: vfmsubadd132pd -1032(%rdx){1to2}, %xmm26, %xmm22
+// CHECK:  encoding: [0x62,0xe2,0xad,0x10,0x97,0xb2,0xf8,0xfb,0xff,0xff]
+          vfmsubadd132pd -1032(%rdx){1to2}, %xmm26, %xmm22
+
+// CHECK: vfmsubadd132pd %ymm21, %ymm17, %ymm24
+// CHECK:  encoding: [0x62,0x22,0xf5,0x20,0x97,0xc5]
+          vfmsubadd132pd %ymm21, %ymm17, %ymm24
+
+// CHECK: vfmsubadd132pd %ymm21, %ymm17, %ymm24 {%k7}
+// CHECK:  encoding: [0x62,0x22,0xf5,0x27,0x97,0xc5]
+          vfmsubadd132pd %ymm21, %ymm17, %ymm24 {%k7}
+
+// CHECK: vfmsubadd132pd %ymm21, %ymm17, %ymm24 {%k7} {z}
+// CHECK:  encoding: [0x62,0x22,0xf5,0xa7,0x97,0xc5]
+          vfmsubadd132pd %ymm21, %ymm17, %ymm24 {%k7} {z}
+
+// CHECK: vfmsubadd132pd (%rcx), %ymm17, %ymm24
+// CHECK:  encoding: [0x62,0x62,0xf5,0x20,0x97,0x01]
+          vfmsubadd132pd (%rcx), %ymm17, %ymm24
+
+// CHECK: vfmsubadd132pd 291(%rax,%r14,8), %ymm17, %ymm24
+// CHECK:  encoding: [0x62,0x22,0xf5,0x20,0x97,0x84,0xf0,0x23,0x01,0x00,0x00]
+          vfmsubadd132pd 291(%rax,%r14,8), %ymm17, %ymm24
+
+// CHECK: vfmsubadd132pd (%rcx){1to4}, %ymm17, %ymm24
+// CHECK:  encoding: [0x62,0x62,0xf5,0x30,0x97,0x01]
+          vfmsubadd132pd (%rcx){1to4}, %ymm17, %ymm24
+
+// CHECK: vfmsubadd132pd 4064(%rdx), %ymm17, %ymm24
+// CHECK:  encoding: [0x62,0x62,0xf5,0x20,0x97,0x42,0x7f]
+          vfmsubadd132pd 4064(%rdx), %ymm17, %ymm24
+
+// CHECK: vfmsubadd132pd 4096(%rdx), %ymm17, %ymm24
+// CHECK:  encoding: [0x62,0x62,0xf5,0x20,0x97,0x82,0x00,0x10,0x00,0x00]
+          vfmsubadd132pd 4096(%rdx), %ymm17, %ymm24
+
+// CHECK: vfmsubadd132pd -4096(%rdx), %ymm17, %ymm24
+// CHECK:  encoding: [0x62,0x62,0xf5,0x20,0x97,0x42,0x80]
+          vfmsubadd132pd -4096(%rdx), %ymm17, %ymm24
+
+// CHECK: vfmsubadd132pd -4128(%rdx), %ymm17, %ymm24
+// CHECK:  encoding: [0x62,0x62,0xf5,0x20,0x97,0x82,0xe0,0xef,0xff,0xff]
+          vfmsubadd132pd -4128(%rdx), %ymm17, %ymm24
+
+// CHECK: vfmsubadd132pd 1016(%rdx){1to4}, %ymm17, %ymm24
+// CHECK:  encoding: [0x62,0x62,0xf5,0x30,0x97,0x42,0x7f]
+          vfmsubadd132pd 1016(%rdx){1to4}, %ymm17, %ymm24
+
+// CHECK: vfmsubadd132pd 1024(%rdx){1to4}, %ymm17, %ymm24
+// CHECK:  encoding: [0x62,0x62,0xf5,0x30,0x97,0x82,0x00,0x04,0x00,0x00]
+          vfmsubadd132pd 1024(%rdx){1to4}, %ymm17, %ymm24
+
+// CHECK: vfmsubadd132pd -1024(%rdx){1to4}, %ymm17, %ymm24
+// CHECK:  encoding: [0x62,0x62,0xf5,0x30,0x97,0x42,0x80]
+          vfmsubadd132pd -1024(%rdx){1to4}, %ymm17, %ymm24
+
+// CHECK: vfmsubadd132pd -1032(%rdx){1to4}, %ymm17, %ymm24
+// CHECK:  encoding: [0x62,0x62,0xf5,0x30,0x97,0x82,0xf8,0xfb,0xff,0xff]
+          vfmsubadd132pd -1032(%rdx){1to4}, %ymm17, %ymm24
+
+// CHECK: vfmsubadd213ps %xmm17, %xmm20, %xmm19
+// CHECK:  encoding: [0x62,0xa2,0x5d,0x00,0xa7,0xd9]
+          vfmsubadd213ps %xmm17, %xmm20, %xmm19
+
+// CHECK: vfmsubadd213ps %xmm17, %xmm20, %xmm19 {%k5}
+// CHECK:  encoding: [0x62,0xa2,0x5d,0x05,0xa7,0xd9]
+          vfmsubadd213ps %xmm17, %xmm20, %xmm19 {%k5}
+
+// CHECK: vfmsubadd213ps %xmm17, %xmm20, %xmm19 {%k5} {z}
+// CHECK:  encoding: [0x62,0xa2,0x5d,0x85,0xa7,0xd9]
+          vfmsubadd213ps %xmm17, %xmm20, %xmm19 {%k5} {z}
+
+// CHECK: vfmsubadd213ps (%rcx), %xmm20, %xmm19
+// CHECK:  encoding: [0x62,0xe2,0x5d,0x00,0xa7,0x19]
+          vfmsubadd213ps (%rcx), %xmm20, %xmm19
+
+// CHECK: vfmsubadd213ps 291(%rax,%r14,8), %xmm20, %xmm19
+// CHECK:  encoding: [0x62,0xa2,0x5d,0x00,0xa7,0x9c,0xf0,0x23,0x01,0x00,0x00]
+          vfmsubadd213ps 291(%rax,%r14,8), %xmm20, %xmm19
+
+// CHECK: vfmsubadd213ps (%rcx){1to4}, %xmm20, %xmm19
+// CHECK:  encoding: [0x62,0xe2,0x5d,0x10,0xa7,0x19]
+          vfmsubadd213ps (%rcx){1to4}, %xmm20, %xmm19
+
+// CHECK: vfmsubadd213ps 2032(%rdx), %xmm20, %xmm19
+// CHECK:  encoding: [0x62,0xe2,0x5d,0x00,0xa7,0x5a,0x7f]
+          vfmsubadd213ps 2032(%rdx), %xmm20, %xmm19
+
+// CHECK: vfmsubadd213ps 2048(%rdx), %xmm20, %xmm19
+// CHECK:  encoding: [0x62,0xe2,0x5d,0x00,0xa7,0x9a,0x00,0x08,0x00,0x00]
+          vfmsubadd213ps 2048(%rdx), %xmm20, %xmm19
+
+// CHECK: vfmsubadd213ps -2048(%rdx), %xmm20, %xmm19
+// CHECK:  encoding: [0x62,0xe2,0x5d,0x00,0xa7,0x5a,0x80]
+          vfmsubadd213ps -2048(%rdx), %xmm20, %xmm19
+
+// CHECK: vfmsubadd213ps -2064(%rdx), %xmm20, %xmm19
+// CHECK:  encoding: [0x62,0xe2,0x5d,0x00,0xa7,0x9a,0xf0,0xf7,0xff,0xff]
+          vfmsubadd213ps -2064(%rdx), %xmm20, %xmm19
+
+// CHECK: vfmsubadd213ps 508(%rdx){1to4}, %xmm20, %xmm19
+// CHECK:  encoding: [0x62,0xe2,0x5d,0x10,0xa7,0x5a,0x7f]
+          vfmsubadd213ps 508(%rdx){1to4}, %xmm20, %xmm19
+
+// CHECK: vfmsubadd213ps 512(%rdx){1to4}, %xmm20, %xmm19
+// CHECK:  encoding: [0x62,0xe2,0x5d,0x10,0xa7,0x9a,0x00,0x02,0x00,0x00]
+          vfmsubadd213ps 512(%rdx){1to4}, %xmm20, %xmm19
+
+// CHECK: vfmsubadd213ps -512(%rdx){1to4}, %xmm20, %xmm19
+// CHECK:  encoding: [0x62,0xe2,0x5d,0x10,0xa7,0x5a,0x80]
+          vfmsubadd213ps -512(%rdx){1to4}, %xmm20, %xmm19
+
+// CHECK: vfmsubadd213ps -516(%rdx){1to4}, %xmm20, %xmm19
+// CHECK:  encoding: [0x62,0xe2,0x5d,0x10,0xa7,0x9a,0xfc,0xfd,0xff,0xff]
+          vfmsubadd213ps -516(%rdx){1to4}, %xmm20, %xmm19
+
+// CHECK: vfmsubadd213ps %ymm23, %ymm17, %ymm26
+// CHECK:  encoding: [0x62,0x22,0x75,0x20,0xa7,0xd7]
+          vfmsubadd213ps %ymm23, %ymm17, %ymm26
+
+// CHECK: vfmsubadd213ps %ymm23, %ymm17, %ymm26 {%k7}
+// CHECK:  encoding: [0x62,0x22,0x75,0x27,0xa7,0xd7]
+          vfmsubadd213ps %ymm23, %ymm17, %ymm26 {%k7}
+
+// CHECK: vfmsubadd213ps %ymm23, %ymm17, %ymm26 {%k7} {z}
+// CHECK:  encoding: [0x62,0x22,0x75,0xa7,0xa7,0xd7]
+          vfmsubadd213ps %ymm23, %ymm17, %ymm26 {%k7} {z}
+
+// CHECK: vfmsubadd213ps (%rcx), %ymm17, %ymm26
+// CHECK:  encoding: [0x62,0x62,0x75,0x20,0xa7,0x11]
+          vfmsubadd213ps (%rcx), %ymm17, %ymm26
+
+// CHECK: vfmsubadd213ps 291(%rax,%r14,8), %ymm17, %ymm26
+// CHECK:  encoding: [0x62,0x22,0x75,0x20,0xa7,0x94,0xf0,0x23,0x01,0x00,0x00]
+          vfmsubadd213ps 291(%rax,%r14,8), %ymm17, %ymm26
+
+// CHECK: vfmsubadd213ps (%rcx){1to8}, %ymm17, %ymm26
+// CHECK:  encoding: [0x62,0x62,0x75,0x30,0xa7,0x11]
+          vfmsubadd213ps (%rcx){1to8}, %ymm17, %ymm26
+
+// CHECK: vfmsubadd213ps 4064(%rdx), %ymm17, %ymm26
+// CHECK:  encoding: [0x62,0x62,0x75,0x20,0xa7,0x52,0x7f]
+          vfmsubadd213ps 4064(%rdx), %ymm17, %ymm26
+
+// CHECK: vfmsubadd213ps 4096(%rdx), %ymm17, %ymm26
+// CHECK:  encoding: [0x62,0x62,0x75,0x20,0xa7,0x92,0x00,0x10,0x00,0x00]
+          vfmsubadd213ps 4096(%rdx), %ymm17, %ymm26
+
+// CHECK: vfmsubadd213ps -4096(%rdx), %ymm17, %ymm26
+// CHECK:  encoding: [0x62,0x62,0x75,0x20,0xa7,0x52,0x80]
+          vfmsubadd213ps -4096(%rdx), %ymm17, %ymm26
+
+// CHECK: vfmsubadd213ps -4128(%rdx), %ymm17, %ymm26
+// CHECK:  encoding: [0x62,0x62,0x75,0x20,0xa7,0x92,0xe0,0xef,0xff,0xff]
+          vfmsubadd213ps -4128(%rdx), %ymm17, %ymm26
+
+// CHECK: vfmsubadd213ps 508(%rdx){1to8}, %ymm17, %ymm26
+// CHECK:  encoding: [0x62,0x62,0x75,0x30,0xa7,0x52,0x7f]
+          vfmsubadd213ps 508(%rdx){1to8}, %ymm17, %ymm26
+
+// CHECK: vfmsubadd213ps 512(%rdx){1to8}, %ymm17, %ymm26
+// CHECK:  encoding: [0x62,0x62,0x75,0x30,0xa7,0x92,0x00,0x02,0x00,0x00]
+          vfmsubadd213ps 512(%rdx){1to8}, %ymm17, %ymm26
+
+// CHECK: vfmsubadd213ps -512(%rdx){1to8}, %ymm17, %ymm26
+// CHECK:  encoding: [0x62,0x62,0x75,0x30,0xa7,0x52,0x80]
+          vfmsubadd213ps -512(%rdx){1to8}, %ymm17, %ymm26
+
+// CHECK: vfmsubadd213ps -516(%rdx){1to8}, %ymm17, %ymm26
+// CHECK:  encoding: [0x62,0x62,0x75,0x30,0xa7,0x92,0xfc,0xfd,0xff,0xff]
+          vfmsubadd213ps -516(%rdx){1to8}, %ymm17, %ymm26
+
+// CHECK: vfmsubadd213pd %xmm28, %xmm21, %xmm18
+// CHECK:  encoding: [0x62,0x82,0xd5,0x00,0xa7,0xd4]
+          vfmsubadd213pd %xmm28, %xmm21, %xmm18
+
+// CHECK: vfmsubadd213pd %xmm28, %xmm21, %xmm18 {%k4}
+// CHECK:  encoding: [0x62,0x82,0xd5,0x04,0xa7,0xd4]
+          vfmsubadd213pd %xmm28, %xmm21, %xmm18 {%k4}
+
+// CHECK: vfmsubadd213pd %xmm28, %xmm21, %xmm18 {%k4} {z}
+// CHECK:  encoding: [0x62,0x82,0xd5,0x84,0xa7,0xd4]
+          vfmsubadd213pd %xmm28, %xmm21, %xmm18 {%k4} {z}
+
+// CHECK: vfmsubadd213pd (%rcx), %xmm21, %xmm18
+// CHECK:  encoding: [0x62,0xe2,0xd5,0x00,0xa7,0x11]
+          vfmsubadd213pd (%rcx), %xmm21, %xmm18
+
+// CHECK: vfmsubadd213pd 291(%rax,%r14,8), %xmm21, %xmm18
+// CHECK:  encoding: [0x62,0xa2,0xd5,0x00,0xa7,0x94,0xf0,0x23,0x01,0x00,0x00]
+          vfmsubadd213pd 291(%rax,%r14,8), %xmm21, %xmm18
+
+// CHECK: vfmsubadd213pd (%rcx){1to2}, %xmm21, %xmm18
+// CHECK:  encoding: [0x62,0xe2,0xd5,0x10,0xa7,0x11]
+          vfmsubadd213pd (%rcx){1to2}, %xmm21, %xmm18
+
+// CHECK: vfmsubadd213pd 2032(%rdx), %xmm21, %xmm18
+// CHECK:  encoding: [0x62,0xe2,0xd5,0x00,0xa7,0x52,0x7f]
+          vfmsubadd213pd 2032(%rdx), %xmm21, %xmm18
+
+// CHECK: vfmsubadd213pd 2048(%rdx), %xmm21, %xmm18
+// CHECK:  encoding: [0x62,0xe2,0xd5,0x00,0xa7,0x92,0x00,0x08,0x00,0x00]
+          vfmsubadd213pd 2048(%rdx), %xmm21, %xmm18
+
+// CHECK: vfmsubadd213pd -2048(%rdx), %xmm21, %xmm18
+// CHECK:  encoding: [0x62,0xe2,0xd5,0x00,0xa7,0x52,0x80]
+          vfmsubadd213pd -2048(%rdx), %xmm21, %xmm18
+
+// CHECK: vfmsubadd213pd -2064(%rdx), %xmm21, %xmm18
+// CHECK:  encoding: [0x62,0xe2,0xd5,0x00,0xa7,0x92,0xf0,0xf7,0xff,0xff]
+          vfmsubadd213pd -2064(%rdx), %xmm21, %xmm18
+
+// CHECK: vfmsubadd213pd 1016(%rdx){1to2}, %xmm21, %xmm18
+// CHECK:  encoding: [0x62,0xe2,0xd5,0x10,0xa7,0x52,0x7f]
+          vfmsubadd213pd 1016(%rdx){1to2}, %xmm21, %xmm18
+
+// CHECK: vfmsubadd213pd 1024(%rdx){1to2}, %xmm21, %xmm18
+// CHECK:  encoding: [0x62,0xe2,0xd5,0x10,0xa7,0x92,0x00,0x04,0x00,0x00]
+          vfmsubadd213pd 1024(%rdx){1to2}, %xmm21, %xmm18
+
+// CHECK: vfmsubadd213pd -1024(%rdx){1to2}, %xmm21, %xmm18
+// CHECK:  encoding: [0x62,0xe2,0xd5,0x10,0xa7,0x52,0x80]
+          vfmsubadd213pd -1024(%rdx){1to2}, %xmm21, %xmm18
+
+// CHECK: vfmsubadd213pd -1032(%rdx){1to2}, %xmm21, %xmm18
+// CHECK:  encoding: [0x62,0xe2,0xd5,0x10,0xa7,0x92,0xf8,0xfb,0xff,0xff]
+          vfmsubadd213pd -1032(%rdx){1to2}, %xmm21, %xmm18
+
+// CHECK: vfmsubadd213pd %ymm25, %ymm20, %ymm27
+// CHECK:  encoding: [0x62,0x02,0xdd,0x20,0xa7,0xd9]
+          vfmsubadd213pd %ymm25, %ymm20, %ymm27
+
+// CHECK: vfmsubadd213pd %ymm25, %ymm20, %ymm27 {%k7}
+// CHECK:  encoding: [0x62,0x02,0xdd,0x27,0xa7,0xd9]
+          vfmsubadd213pd %ymm25, %ymm20, %ymm27 {%k7}
+
+// CHECK: vfmsubadd213pd %ymm25, %ymm20, %ymm27 {%k7} {z}
+// CHECK:  encoding: [0x62,0x02,0xdd,0xa7,0xa7,0xd9]
+          vfmsubadd213pd %ymm25, %ymm20, %ymm27 {%k7} {z}
+
+// CHECK: vfmsubadd213pd (%rcx), %ymm20, %ymm27
+// CHECK:  encoding: [0x62,0x62,0xdd,0x20,0xa7,0x19]
+          vfmsubadd213pd (%rcx), %ymm20, %ymm27
+
+// CHECK: vfmsubadd213pd 291(%rax,%r14,8), %ymm20, %ymm27
+// CHECK:  encoding: [0x62,0x22,0xdd,0x20,0xa7,0x9c,0xf0,0x23,0x01,0x00,0x00]
+          vfmsubadd213pd 291(%rax,%r14,8), %ymm20, %ymm27
+
+// CHECK: vfmsubadd213pd (%rcx){1to4}, %ymm20, %ymm27
+// CHECK:  encoding: [0x62,0x62,0xdd,0x30,0xa7,0x19]
+          vfmsubadd213pd (%rcx){1to4}, %ymm20, %ymm27
+
+// CHECK: vfmsubadd213pd 4064(%rdx), %ymm20, %ymm27
+// CHECK:  encoding: [0x62,0x62,0xdd,0x20,0xa7,0x5a,0x7f]
+          vfmsubadd213pd 4064(%rdx), %ymm20, %ymm27
+
+// CHECK: vfmsubadd213pd 4096(%rdx), %ymm20, %ymm27
+// CHECK:  encoding: [0x62,0x62,0xdd,0x20,0xa7,0x9a,0x00,0x10,0x00,0x00]
+          vfmsubadd213pd 4096(%rdx), %ymm20, %ymm27
+
+// CHECK: vfmsubadd213pd -4096(%rdx), %ymm20, %ymm27
+// CHECK:  encoding: [0x62,0x62,0xdd,0x20,0xa7,0x5a,0x80]
+          vfmsubadd213pd -4096(%rdx), %ymm20, %ymm27
+
+// CHECK: vfmsubadd213pd -4128(%rdx), %ymm20, %ymm27
+// CHECK:  encoding: [0x62,0x62,0xdd,0x20,0xa7,0x9a,0xe0,0xef,0xff,0xff]
+          vfmsubadd213pd -4128(%rdx), %ymm20, %ymm27
+
+// CHECK: vfmsubadd213pd 1016(%rdx){1to4}, %ymm20, %ymm27
+// CHECK:  encoding: [0x62,0x62,0xdd,0x30,0xa7,0x5a,0x7f]
+          vfmsubadd213pd 1016(%rdx){1to4}, %ymm20, %ymm27
+
+// CHECK: vfmsubadd213pd 1024(%rdx){1to4}, %ymm20, %ymm27
+// CHECK:  encoding: [0x62,0x62,0xdd,0x30,0xa7,0x9a,0x00,0x04,0x00,0x00]
+          vfmsubadd213pd 1024(%rdx){1to4}, %ymm20, %ymm27
+
+// CHECK: vfmsubadd213pd -1024(%rdx){1to4}, %ymm20, %ymm27
+// CHECK:  encoding: [0x62,0x62,0xdd,0x30,0xa7,0x5a,0x80]
+          vfmsubadd213pd -1024(%rdx){1to4}, %ymm20, %ymm27
+
+// CHECK: vfmsubadd213pd -1032(%rdx){1to4}, %ymm20, %ymm27
+// CHECK:  encoding: [0x62,0x62,0xdd,0x30,0xa7,0x9a,0xf8,0xfb,0xff,0xff]
+          vfmsubadd213pd -1032(%rdx){1to4}, %ymm20, %ymm27
+
+// CHECK: vfmsubadd231ps %xmm23, %xmm25, %xmm21
+// CHECK:  encoding: [0x62,0xa2,0x35,0x00,0xb7,0xef]
+          vfmsubadd231ps %xmm23, %xmm25, %xmm21
+
+// CHECK: vfmsubadd231ps %xmm23, %xmm25, %xmm21 {%k4}
+// CHECK:  encoding: [0x62,0xa2,0x35,0x04,0xb7,0xef]
+          vfmsubadd231ps %xmm23, %xmm25, %xmm21 {%k4}
+
+// CHECK: vfmsubadd231ps %xmm23, %xmm25, %xmm21 {%k4} {z}
+// CHECK:  encoding: [0x62,0xa2,0x35,0x84,0xb7,0xef]
+          vfmsubadd231ps %xmm23, %xmm25, %xmm21 {%k4} {z}
+
+// CHECK: vfmsubadd231ps (%rcx), %xmm25, %xmm21
+// CHECK:  encoding: [0x62,0xe2,0x35,0x00,0xb7,0x29]
+          vfmsubadd231ps (%rcx), %xmm25, %xmm21
+
+// CHECK: vfmsubadd231ps 291(%rax,%r14,8), %xmm25, %xmm21
+// CHECK:  encoding: [0x62,0xa2,0x35,0x00,0xb7,0xac,0xf0,0x23,0x01,0x00,0x00]
+          vfmsubadd231ps 291(%rax,%r14,8), %xmm25, %xmm21
+
+// CHECK: vfmsubadd231ps (%rcx){1to4}, %xmm25, %xmm21
+// CHECK:  encoding: [0x62,0xe2,0x35,0x10,0xb7,0x29]
+          vfmsubadd231ps (%rcx){1to4}, %xmm25, %xmm21
+
+// CHECK: vfmsubadd231ps 2032(%rdx), %xmm25, %xmm21
+// CHECK:  encoding: [0x62,0xe2,0x35,0x00,0xb7,0x6a,0x7f]
+          vfmsubadd231ps 2032(%rdx), %xmm25, %xmm21
+
+// CHECK: vfmsubadd231ps 2048(%rdx), %xmm25, %xmm21
+// CHECK:  encoding: [0x62,0xe2,0x35,0x00,0xb7,0xaa,0x00,0x08,0x00,0x00]
+          vfmsubadd231ps 2048(%rdx), %xmm25, %xmm21
+
+// CHECK: vfmsubadd231ps -2048(%rdx), %xmm25, %xmm21
+// CHECK:  encoding: [0x62,0xe2,0x35,0x00,0xb7,0x6a,0x80]
+          vfmsubadd231ps -2048(%rdx), %xmm25, %xmm21
+
+// CHECK: vfmsubadd231ps -2064(%rdx), %xmm25, %xmm21
+// CHECK:  encoding: [0x62,0xe2,0x35,0x00,0xb7,0xaa,0xf0,0xf7,0xff,0xff]
+          vfmsubadd231ps -2064(%rdx), %xmm25, %xmm21
+
+// CHECK: vfmsubadd231ps 508(%rdx){1to4}, %xmm25, %xmm21
+// CHECK:  encoding: [0x62,0xe2,0x35,0x10,0xb7,0x6a,0x7f]
+          vfmsubadd231ps 508(%rdx){1to4}, %xmm25, %xmm21
+
+// CHECK: vfmsubadd231ps 512(%rdx){1to4}, %xmm25, %xmm21
+// CHECK:  encoding: [0x62,0xe2,0x35,0x10,0xb7,0xaa,0x00,0x02,0x00,0x00]
+          vfmsubadd231ps 512(%rdx){1to4}, %xmm25, %xmm21
+
+// CHECK: vfmsubadd231ps -512(%rdx){1to4}, %xmm25, %xmm21
+// CHECK:  encoding: [0x62,0xe2,0x35,0x10,0xb7,0x6a,0x80]
+          vfmsubadd231ps -512(%rdx){1to4}, %xmm25, %xmm21
+
+// CHECK: vfmsubadd231ps -516(%rdx){1to4}, %xmm25, %xmm21
+// CHECK:  encoding: [0x62,0xe2,0x35,0x10,0xb7,0xaa,0xfc,0xfd,0xff,0xff]
+          vfmsubadd231ps -516(%rdx){1to4}, %xmm25, %xmm21
+
+// CHECK: vfmsubadd231ps %ymm20, %ymm23, %ymm27
+// CHECK:  encoding: [0x62,0x22,0x45,0x20,0xb7,0xdc]
+          vfmsubadd231ps %ymm20, %ymm23, %ymm27
+
+// CHECK: vfmsubadd231ps %ymm20, %ymm23, %ymm27 {%k3}
+// CHECK:  encoding: [0x62,0x22,0x45,0x23,0xb7,0xdc]
+          vfmsubadd231ps %ymm20, %ymm23, %ymm27 {%k3}
+
+// CHECK: vfmsubadd231ps %ymm20, %ymm23, %ymm27 {%k3} {z}
+// CHECK:  encoding: [0x62,0x22,0x45,0xa3,0xb7,0xdc]
+          vfmsubadd231ps %ymm20, %ymm23, %ymm27 {%k3} {z}
+
+// CHECK: vfmsubadd231ps (%rcx), %ymm23, %ymm27
+// CHECK:  encoding: [0x62,0x62,0x45,0x20,0xb7,0x19]
+          vfmsubadd231ps (%rcx), %ymm23, %ymm27
+
+// CHECK: vfmsubadd231ps 291(%rax,%r14,8), %ymm23, %ymm27
+// CHECK:  encoding: [0x62,0x22,0x45,0x20,0xb7,0x9c,0xf0,0x23,0x01,0x00,0x00]
+          vfmsubadd231ps 291(%rax,%r14,8), %ymm23, %ymm27
+
+// CHECK: vfmsubadd231ps (%rcx){1to8}, %ymm23, %ymm27
+// CHECK:  encoding: [0x62,0x62,0x45,0x30,0xb7,0x19]
+          vfmsubadd231ps (%rcx){1to8}, %ymm23, %ymm27
+
+// CHECK: vfmsubadd231ps 4064(%rdx), %ymm23, %ymm27
+// CHECK:  encoding: [0x62,0x62,0x45,0x20,0xb7,0x5a,0x7f]
+          vfmsubadd231ps 4064(%rdx), %ymm23, %ymm27
+
+// CHECK: vfmsubadd231ps 4096(%rdx), %ymm23, %ymm27
+// CHECK:  encoding: [0x62,0x62,0x45,0x20,0xb7,0x9a,0x00,0x10,0x00,0x00]
+          vfmsubadd231ps 4096(%rdx), %ymm23, %ymm27
+
+// CHECK: vfmsubadd231ps -4096(%rdx), %ymm23, %ymm27
+// CHECK:  encoding: [0x62,0x62,0x45,0x20,0xb7,0x5a,0x80]
+          vfmsubadd231ps -4096(%rdx), %ymm23, %ymm27
+
+// CHECK: vfmsubadd231ps -4128(%rdx), %ymm23, %ymm27
+// CHECK:  encoding: [0x62,0x62,0x45,0x20,0xb7,0x9a,0xe0,0xef,0xff,0xff]
+          vfmsubadd231ps -4128(%rdx), %ymm23, %ymm27
+
+// CHECK: vfmsubadd231ps 508(%rdx){1to8}, %ymm23, %ymm27
+// CHECK:  encoding: [0x62,0x62,0x45,0x30,0xb7,0x5a,0x7f]
+          vfmsubadd231ps 508(%rdx){1to8}, %ymm23, %ymm27
+
+// CHECK: vfmsubadd231ps 512(%rdx){1to8}, %ymm23, %ymm27
+// CHECK:  encoding: [0x62,0x62,0x45,0x30,0xb7,0x9a,0x00,0x02,0x00,0x00]
+          vfmsubadd231ps 512(%rdx){1to8}, %ymm23, %ymm27
+
+// CHECK: vfmsubadd231ps -512(%rdx){1to8}, %ymm23, %ymm27
+// CHECK:  encoding: [0x62,0x62,0x45,0x30,0xb7,0x5a,0x80]
+          vfmsubadd231ps -512(%rdx){1to8}, %ymm23, %ymm27
+
+// CHECK: vfmsubadd231ps -516(%rdx){1to8}, %ymm23, %ymm27
+// CHECK:  encoding: [0x62,0x62,0x45,0x30,0xb7,0x9a,0xfc,0xfd,0xff,0xff]
+          vfmsubadd231ps -516(%rdx){1to8}, %ymm23, %ymm27
+
+// CHECK: vfmsubadd231pd %xmm28, %xmm24, %xmm20
+// CHECK:  encoding: [0x62,0x82,0xbd,0x00,0xb7,0xe4]
+          vfmsubadd231pd %xmm28, %xmm24, %xmm20
+
+// CHECK: vfmsubadd231pd %xmm28, %xmm24, %xmm20 {%k3}
+// CHECK:  encoding: [0x62,0x82,0xbd,0x03,0xb7,0xe4]
+          vfmsubadd231pd %xmm28, %xmm24, %xmm20 {%k3}
+
+// CHECK: vfmsubadd231pd %xmm28, %xmm24, %xmm20 {%k3} {z}
+// CHECK:  encoding: [0x62,0x82,0xbd,0x83,0xb7,0xe4]
+          vfmsubadd231pd %xmm28, %xmm24, %xmm20 {%k3} {z}
+
+// CHECK: vfmsubadd231pd (%rcx), %xmm24, %xmm20
+// CHECK:  encoding: [0x62,0xe2,0xbd,0x00,0xb7,0x21]
+          vfmsubadd231pd (%rcx), %xmm24, %xmm20
+
+// CHECK: vfmsubadd231pd 291(%rax,%r14,8), %xmm24, %xmm20
+// CHECK:  encoding: [0x62,0xa2,0xbd,0x00,0xb7,0xa4,0xf0,0x23,0x01,0x00,0x00]
+          vfmsubadd231pd 291(%rax,%r14,8), %xmm24, %xmm20
+
+// CHECK: vfmsubadd231pd (%rcx){1to2}, %xmm24, %xmm20
+// CHECK:  encoding: [0x62,0xe2,0xbd,0x10,0xb7,0x21]
+          vfmsubadd231pd (%rcx){1to2}, %xmm24, %xmm20
+
+// CHECK: vfmsubadd231pd 2032(%rdx), %xmm24, %xmm20
+// CHECK:  encoding: [0x62,0xe2,0xbd,0x00,0xb7,0x62,0x7f]
+          vfmsubadd231pd 2032(%rdx), %xmm24, %xmm20
+
+// CHECK: vfmsubadd231pd 2048(%rdx), %xmm24, %xmm20
+// CHECK:  encoding: [0x62,0xe2,0xbd,0x00,0xb7,0xa2,0x00,0x08,0x00,0x00]
+          vfmsubadd231pd 2048(%rdx), %xmm24, %xmm20
+
+// CHECK: vfmsubadd231pd -2048(%rdx), %xmm24, %xmm20
+// CHECK:  encoding: [0x62,0xe2,0xbd,0x00,0xb7,0x62,0x80]
+          vfmsubadd231pd -2048(%rdx), %xmm24, %xmm20
+
+// CHECK: vfmsubadd231pd -2064(%rdx), %xmm24, %xmm20
+// CHECK:  encoding: [0x62,0xe2,0xbd,0x00,0xb7,0xa2,0xf0,0xf7,0xff,0xff]
+          vfmsubadd231pd -2064(%rdx), %xmm24, %xmm20
+
+// CHECK: vfmsubadd231pd 1016(%rdx){1to2}, %xmm24, %xmm20
+// CHECK:  encoding: [0x62,0xe2,0xbd,0x10,0xb7,0x62,0x7f]
+          vfmsubadd231pd 1016(%rdx){1to2}, %xmm24, %xmm20
+
+// CHECK: vfmsubadd231pd 1024(%rdx){1to2}, %xmm24, %xmm20
+// CHECK:  encoding: [0x62,0xe2,0xbd,0x10,0xb7,0xa2,0x00,0x04,0x00,0x00]
+          vfmsubadd231pd 1024(%rdx){1to2}, %xmm24, %xmm20
+
+// CHECK: vfmsubadd231pd -1024(%rdx){1to2}, %xmm24, %xmm20
+// CHECK:  encoding: [0x62,0xe2,0xbd,0x10,0xb7,0x62,0x80]
+          vfmsubadd231pd -1024(%rdx){1to2}, %xmm24, %xmm20
+
+// CHECK: vfmsubadd231pd -1032(%rdx){1to2}, %xmm24, %xmm20
+// CHECK:  encoding: [0x62,0xe2,0xbd,0x10,0xb7,0xa2,0xf8,0xfb,0xff,0xff]
+          vfmsubadd231pd -1032(%rdx){1to2}, %xmm24, %xmm20
+
+// CHECK: vfmsubadd231pd %ymm21, %ymm29, %ymm30
+// CHECK:  encoding: [0x62,0x22,0x95,0x20,0xb7,0xf5]
+          vfmsubadd231pd %ymm21, %ymm29, %ymm30
+
+// CHECK: vfmsubadd231pd %ymm21, %ymm29, %ymm30 {%k7}
+// CHECK:  encoding: [0x62,0x22,0x95,0x27,0xb7,0xf5]
+          vfmsubadd231pd %ymm21, %ymm29, %ymm30 {%k7}
+
+// CHECK: vfmsubadd231pd %ymm21, %ymm29, %ymm30 {%k7} {z}
+// CHECK:  encoding: [0x62,0x22,0x95,0xa7,0xb7,0xf5]
+          vfmsubadd231pd %ymm21, %ymm29, %ymm30 {%k7} {z}
+
+// CHECK: vfmsubadd231pd (%rcx), %ymm29, %ymm30
+// CHECK:  encoding: [0x62,0x62,0x95,0x20,0xb7,0x31]
+          vfmsubadd231pd (%rcx), %ymm29, %ymm30
+
+// CHECK: vfmsubadd231pd 291(%rax,%r14,8), %ymm29, %ymm30
+// CHECK:  encoding: [0x62,0x22,0x95,0x20,0xb7,0xb4,0xf0,0x23,0x01,0x00,0x00]
+          vfmsubadd231pd 291(%rax,%r14,8), %ymm29, %ymm30
+
+// CHECK: vfmsubadd231pd (%rcx){1to4}, %ymm29, %ymm30
+// CHECK:  encoding: [0x62,0x62,0x95,0x30,0xb7,0x31]
+          vfmsubadd231pd (%rcx){1to4}, %ymm29, %ymm30
+
+// CHECK: vfmsubadd231pd 4064(%rdx), %ymm29, %ymm30
+// CHECK:  encoding: [0x62,0x62,0x95,0x20,0xb7,0x72,0x7f]
+          vfmsubadd231pd 4064(%rdx), %ymm29, %ymm30
+
+// CHECK: vfmsubadd231pd 4096(%rdx), %ymm29, %ymm30
+// CHECK:  encoding: [0x62,0x62,0x95,0x20,0xb7,0xb2,0x00,0x10,0x00,0x00]
+          vfmsubadd231pd 4096(%rdx), %ymm29, %ymm30
+
+// CHECK: vfmsubadd231pd -4096(%rdx), %ymm29, %ymm30
+// CHECK:  encoding: [0x62,0x62,0x95,0x20,0xb7,0x72,0x80]
+          vfmsubadd231pd -4096(%rdx), %ymm29, %ymm30
+
+// CHECK: vfmsubadd231pd -4128(%rdx), %ymm29, %ymm30
+// CHECK:  encoding: [0x62,0x62,0x95,0x20,0xb7,0xb2,0xe0,0xef,0xff,0xff]
+          vfmsubadd231pd -4128(%rdx), %ymm29, %ymm30
+
+// CHECK: vfmsubadd231pd 1016(%rdx){1to4}, %ymm29, %ymm30
+// CHECK:  encoding: [0x62,0x62,0x95,0x30,0xb7,0x72,0x7f]
+          vfmsubadd231pd 1016(%rdx){1to4}, %ymm29, %ymm30
+
+// CHECK: vfmsubadd231pd 1024(%rdx){1to4}, %ymm29, %ymm30
+// CHECK:  encoding: [0x62,0x62,0x95,0x30,0xb7,0xb2,0x00,0x04,0x00,0x00]
+          vfmsubadd231pd 1024(%rdx){1to4}, %ymm29, %ymm30
+
+// CHECK: vfmsubadd231pd -1024(%rdx){1to4}, %ymm29, %ymm30
+// CHECK:  encoding: [0x62,0x62,0x95,0x30,0xb7,0x72,0x80]
+          vfmsubadd231pd -1024(%rdx){1to4}, %ymm29, %ymm30
+
+// CHECK: vfmsubadd231pd -1032(%rdx){1to4}, %ymm29, %ymm30
+// CHECK:  encoding: [0x62,0x62,0x95,0x30,0xb7,0xb2,0xf8,0xfb,0xff,0xff]
+          vfmsubadd231pd -1032(%rdx){1to4}, %ymm29, %ymm30
+
+// CHECK: vfnmadd132ps %xmm18, %xmm26, %xmm20
+// CHECK:  encoding: [0x62,0xa2,0x2d,0x00,0x9c,0xe2]
+          vfnmadd132ps %xmm18, %xmm26, %xmm20
+
+// CHECK: vfnmadd132ps %xmm18, %xmm26, %xmm20 {%k7}
+// CHECK:  encoding: [0x62,0xa2,0x2d,0x07,0x9c,0xe2]
+          vfnmadd132ps %xmm18, %xmm26, %xmm20 {%k7}
+
+// CHECK: vfnmadd132ps %xmm18, %xmm26, %xmm20 {%k7} {z}
+// CHECK:  encoding: [0x62,0xa2,0x2d,0x87,0x9c,0xe2]
+          vfnmadd132ps %xmm18, %xmm26, %xmm20 {%k7} {z}
+
+// CHECK: vfnmadd132ps (%rcx), %xmm26, %xmm20
+// CHECK:  encoding: [0x62,0xe2,0x2d,0x00,0x9c,0x21]
+          vfnmadd132ps (%rcx), %xmm26, %xmm20
+
+// CHECK: vfnmadd132ps 291(%rax,%r14,8), %xmm26, %xmm20
+// CHECK:  encoding: [0x62,0xa2,0x2d,0x00,0x9c,0xa4,0xf0,0x23,0x01,0x00,0x00]
+          vfnmadd132ps 291(%rax,%r14,8), %xmm26, %xmm20
+
+// CHECK: vfnmadd132ps (%rcx){1to4}, %xmm26, %xmm20
+// CHECK:  encoding: [0x62,0xe2,0x2d,0x10,0x9c,0x21]
+          vfnmadd132ps (%rcx){1to4}, %xmm26, %xmm20
+
+// CHECK: vfnmadd132ps 2032(%rdx), %xmm26, %xmm20
+// CHECK:  encoding: [0x62,0xe2,0x2d,0x00,0x9c,0x62,0x7f]
+          vfnmadd132ps 2032(%rdx), %xmm26, %xmm20
+
+// CHECK: vfnmadd132ps 2048(%rdx), %xmm26, %xmm20
+// CHECK:  encoding: [0x62,0xe2,0x2d,0x00,0x9c,0xa2,0x00,0x08,0x00,0x00]
+          vfnmadd132ps 2048(%rdx), %xmm26, %xmm20
+
+// CHECK: vfnmadd132ps -2048(%rdx), %xmm26, %xmm20
+// CHECK:  encoding: [0x62,0xe2,0x2d,0x00,0x9c,0x62,0x80]
+          vfnmadd132ps -2048(%rdx), %xmm26, %xmm20
+
+// CHECK: vfnmadd132ps -2064(%rdx), %xmm26, %xmm20
+// CHECK:  encoding: [0x62,0xe2,0x2d,0x00,0x9c,0xa2,0xf0,0xf7,0xff,0xff]
+          vfnmadd132ps -2064(%rdx), %xmm26, %xmm20
+
+// CHECK: vfnmadd132ps 508(%rdx){1to4}, %xmm26, %xmm20
+// CHECK:  encoding: [0x62,0xe2,0x2d,0x10,0x9c,0x62,0x7f]
+          vfnmadd132ps 508(%rdx){1to4}, %xmm26, %xmm20
+
+// CHECK: vfnmadd132ps 512(%rdx){1to4}, %xmm26, %xmm20
+// CHECK:  encoding: [0x62,0xe2,0x2d,0x10,0x9c,0xa2,0x00,0x02,0x00,0x00]
+          vfnmadd132ps 512(%rdx){1to4}, %xmm26, %xmm20
+
+// CHECK: vfnmadd132ps -512(%rdx){1to4}, %xmm26, %xmm20
+// CHECK:  encoding: [0x62,0xe2,0x2d,0x10,0x9c,0x62,0x80]
+          vfnmadd132ps -512(%rdx){1to4}, %xmm26, %xmm20
+
+// CHECK: vfnmadd132ps -516(%rdx){1to4}, %xmm26, %xmm20
+// CHECK:  encoding: [0x62,0xe2,0x2d,0x10,0x9c,0xa2,0xfc,0xfd,0xff,0xff]
+          vfnmadd132ps -516(%rdx){1to4}, %xmm26, %xmm20
+
+// CHECK: vfnmadd132ps %ymm18, %ymm21, %ymm20
+// CHECK:  encoding: [0x62,0xa2,0x55,0x20,0x9c,0xe2]
+          vfnmadd132ps %ymm18, %ymm21, %ymm20
+
+// CHECK: vfnmadd132ps %ymm18, %ymm21, %ymm20 {%k7}
+// CHECK:  encoding: [0x62,0xa2,0x55,0x27,0x9c,0xe2]
+          vfnmadd132ps %ymm18, %ymm21, %ymm20 {%k7}
+
+// CHECK: vfnmadd132ps %ymm18, %ymm21, %ymm20 {%k7} {z}
+// CHECK:  encoding: [0x62,0xa2,0x55,0xa7,0x9c,0xe2]
+          vfnmadd132ps %ymm18, %ymm21, %ymm20 {%k7} {z}
+
+// CHECK: vfnmadd132ps (%rcx), %ymm21, %ymm20
+// CHECK:  encoding: [0x62,0xe2,0x55,0x20,0x9c,0x21]
+          vfnmadd132ps (%rcx), %ymm21, %ymm20
+
+// CHECK: vfnmadd132ps 291(%rax,%r14,8), %ymm21, %ymm20
+// CHECK:  encoding: [0x62,0xa2,0x55,0x20,0x9c,0xa4,0xf0,0x23,0x01,0x00,0x00]
+          vfnmadd132ps 291(%rax,%r14,8), %ymm21, %ymm20
+
+// CHECK: vfnmadd132ps (%rcx){1to8}, %ymm21, %ymm20
+// CHECK:  encoding: [0x62,0xe2,0x55,0x30,0x9c,0x21]
+          vfnmadd132ps (%rcx){1to8}, %ymm21, %ymm20
+
+// CHECK: vfnmadd132ps 4064(%rdx), %ymm21, %ymm20
+// CHECK:  encoding: [0x62,0xe2,0x55,0x20,0x9c,0x62,0x7f]
+          vfnmadd132ps 4064(%rdx), %ymm21, %ymm20
+
+// CHECK: vfnmadd132ps 4096(%rdx), %ymm21, %ymm20
+// CHECK:  encoding: [0x62,0xe2,0x55,0x20,0x9c,0xa2,0x00,0x10,0x00,0x00]
+          vfnmadd132ps 4096(%rdx), %ymm21, %ymm20
+
+// CHECK: vfnmadd132ps -4096(%rdx), %ymm21, %ymm20
+// CHECK:  encoding: [0x62,0xe2,0x55,0x20,0x9c,0x62,0x80]
+          vfnmadd132ps -4096(%rdx), %ymm21, %ymm20
+
+// CHECK: vfnmadd132ps -4128(%rdx), %ymm21, %ymm20
+// CHECK:  encoding: [0x62,0xe2,0x55,0x20,0x9c,0xa2,0xe0,0xef,0xff,0xff]
+          vfnmadd132ps -4128(%rdx), %ymm21, %ymm20
+
+// CHECK: vfnmadd132ps 508(%rdx){1to8}, %ymm21, %ymm20
+// CHECK:  encoding: [0x62,0xe2,0x55,0x30,0x9c,0x62,0x7f]
+          vfnmadd132ps 508(%rdx){1to8}, %ymm21, %ymm20
+
+// CHECK: vfnmadd132ps 512(%rdx){1to8}, %ymm21, %ymm20
+// CHECK:  encoding: [0x62,0xe2,0x55,0x30,0x9c,0xa2,0x00,0x02,0x00,0x00]
+          vfnmadd132ps 512(%rdx){1to8}, %ymm21, %ymm20
+
+// CHECK: vfnmadd132ps -512(%rdx){1to8}, %ymm21, %ymm20
+// CHECK:  encoding: [0x62,0xe2,0x55,0x30,0x9c,0x62,0x80]
+          vfnmadd132ps -512(%rdx){1to8}, %ymm21, %ymm20
+
+// CHECK: vfnmadd132ps -516(%rdx){1to8}, %ymm21, %ymm20
+// CHECK:  encoding: [0x62,0xe2,0x55,0x30,0x9c,0xa2,0xfc,0xfd,0xff,0xff]
+          vfnmadd132ps -516(%rdx){1to8}, %ymm21, %ymm20
+
+// CHECK: vfnmadd132pd %xmm18, %xmm21, %xmm26
+// CHECK:  encoding: [0x62,0x22,0xd5,0x00,0x9c,0xd2]
+          vfnmadd132pd %xmm18, %xmm21, %xmm26
+
+// CHECK: vfnmadd132pd %xmm18, %xmm21, %xmm26 {%k6}
+// CHECK:  encoding: [0x62,0x22,0xd5,0x06,0x9c,0xd2]
+          vfnmadd132pd %xmm18, %xmm21, %xmm26 {%k6}
+
+// CHECK: vfnmadd132pd %xmm18, %xmm21, %xmm26 {%k6} {z}
+// CHECK:  encoding: [0x62,0x22,0xd5,0x86,0x9c,0xd2]
+          vfnmadd132pd %xmm18, %xmm21, %xmm26 {%k6} {z}
+
+// CHECK: vfnmadd132pd (%rcx), %xmm21, %xmm26
+// CHECK:  encoding: [0x62,0x62,0xd5,0x00,0x9c,0x11]
+          vfnmadd132pd (%rcx), %xmm21, %xmm26
+
+// CHECK: vfnmadd132pd 291(%rax,%r14,8), %xmm21, %xmm26
+// CHECK:  encoding: [0x62,0x22,0xd5,0x00,0x9c,0x94,0xf0,0x23,0x01,0x00,0x00]
+          vfnmadd132pd 291(%rax,%r14,8), %xmm21, %xmm26
+
+// CHECK: vfnmadd132pd (%rcx){1to2}, %xmm21, %xmm26
+// CHECK:  encoding: [0x62,0x62,0xd5,0x10,0x9c,0x11]
+          vfnmadd132pd (%rcx){1to2}, %xmm21, %xmm26
+
+// CHECK: vfnmadd132pd 2032(%rdx), %xmm21, %xmm26
+// CHECK:  encoding: [0x62,0x62,0xd5,0x00,0x9c,0x52,0x7f]
+          vfnmadd132pd 2032(%rdx), %xmm21, %xmm26
+
+// CHECK: vfnmadd132pd 2048(%rdx), %xmm21, %xmm26
+// CHECK:  encoding: [0x62,0x62,0xd5,0x00,0x9c,0x92,0x00,0x08,0x00,0x00]
+          vfnmadd132pd 2048(%rdx), %xmm21, %xmm26
+
+// CHECK: vfnmadd132pd -2048(%rdx), %xmm21, %xmm26
+// CHECK:  encoding: [0x62,0x62,0xd5,0x00,0x9c,0x52,0x80]
+          vfnmadd132pd -2048(%rdx), %xmm21, %xmm26
+
+// CHECK: vfnmadd132pd -2064(%rdx), %xmm21, %xmm26
+// CHECK:  encoding: [0x62,0x62,0xd5,0x00,0x9c,0x92,0xf0,0xf7,0xff,0xff]
+          vfnmadd132pd -2064(%rdx), %xmm21, %xmm26
+
+// CHECK: vfnmadd132pd 1016(%rdx){1to2}, %xmm21, %xmm26
+// CHECK:  encoding: [0x62,0x62,0xd5,0x10,0x9c,0x52,0x7f]
+          vfnmadd132pd 1016(%rdx){1to2}, %xmm21, %xmm26
+
+// CHECK: vfnmadd132pd 1024(%rdx){1to2}, %xmm21, %xmm26
+// CHECK:  encoding: [0x62,0x62,0xd5,0x10,0x9c,0x92,0x00,0x04,0x00,0x00]
+          vfnmadd132pd 1024(%rdx){1to2}, %xmm21, %xmm26
+
+// CHECK: vfnmadd132pd -1024(%rdx){1to2}, %xmm21, %xmm26
+// CHECK:  encoding: [0x62,0x62,0xd5,0x10,0x9c,0x52,0x80]
+          vfnmadd132pd -1024(%rdx){1to2}, %xmm21, %xmm26
+
+// CHECK: vfnmadd132pd -1032(%rdx){1to2}, %xmm21, %xmm26
+// CHECK:  encoding: [0x62,0x62,0xd5,0x10,0x9c,0x92,0xf8,0xfb,0xff,0xff]
+          vfnmadd132pd -1032(%rdx){1to2}, %xmm21, %xmm26
+
+// CHECK: vfnmadd132pd %ymm18, %ymm22, %ymm25
+// CHECK:  encoding: [0x62,0x22,0xcd,0x20,0x9c,0xca]
+          vfnmadd132pd %ymm18, %ymm22, %ymm25
+
+// CHECK: vfnmadd132pd %ymm18, %ymm22, %ymm25 {%k4}
+// CHECK:  encoding: [0x62,0x22,0xcd,0x24,0x9c,0xca]
+          vfnmadd132pd %ymm18, %ymm22, %ymm25 {%k4}
+
+// CHECK: vfnmadd132pd %ymm18, %ymm22, %ymm25 {%k4} {z}
+// CHECK:  encoding: [0x62,0x22,0xcd,0xa4,0x9c,0xca]
+          vfnmadd132pd %ymm18, %ymm22, %ymm25 {%k4} {z}
+
+// CHECK: vfnmadd132pd (%rcx), %ymm22, %ymm25
+// CHECK:  encoding: [0x62,0x62,0xcd,0x20,0x9c,0x09]
+          vfnmadd132pd (%rcx), %ymm22, %ymm25
+
+// CHECK: vfnmadd132pd 291(%rax,%r14,8), %ymm22, %ymm25
+// CHECK:  encoding: [0x62,0x22,0xcd,0x20,0x9c,0x8c,0xf0,0x23,0x01,0x00,0x00]
+          vfnmadd132pd 291(%rax,%r14,8), %ymm22, %ymm25
+
+// CHECK: vfnmadd132pd (%rcx){1to4}, %ymm22, %ymm25
+// CHECK:  encoding: [0x62,0x62,0xcd,0x30,0x9c,0x09]
+          vfnmadd132pd (%rcx){1to4}, %ymm22, %ymm25
+
+// CHECK: vfnmadd132pd 4064(%rdx), %ymm22, %ymm25
+// CHECK:  encoding: [0x62,0x62,0xcd,0x20,0x9c,0x4a,0x7f]
+          vfnmadd132pd 4064(%rdx), %ymm22, %ymm25
+
+// CHECK: vfnmadd132pd 4096(%rdx), %ymm22, %ymm25
+// CHECK:  encoding: [0x62,0x62,0xcd,0x20,0x9c,0x8a,0x00,0x10,0x00,0x00]
+          vfnmadd132pd 4096(%rdx), %ymm22, %ymm25
+
+// CHECK: vfnmadd132pd -4096(%rdx), %ymm22, %ymm25
+// CHECK:  encoding: [0x62,0x62,0xcd,0x20,0x9c,0x4a,0x80]
+          vfnmadd132pd -4096(%rdx), %ymm22, %ymm25
+
+// CHECK: vfnmadd132pd -4128(%rdx), %ymm22, %ymm25
+// CHECK:  encoding: [0x62,0x62,0xcd,0x20,0x9c,0x8a,0xe0,0xef,0xff,0xff]
+          vfnmadd132pd -4128(%rdx), %ymm22, %ymm25
+
+// CHECK: vfnmadd132pd 1016(%rdx){1to4}, %ymm22, %ymm25
+// CHECK:  encoding: [0x62,0x62,0xcd,0x30,0x9c,0x4a,0x7f]
+          vfnmadd132pd 1016(%rdx){1to4}, %ymm22, %ymm25
+
+// CHECK: vfnmadd132pd 1024(%rdx){1to4}, %ymm22, %ymm25
+// CHECK:  encoding: [0x62,0x62,0xcd,0x30,0x9c,0x8a,0x00,0x04,0x00,0x00]
+          vfnmadd132pd 1024(%rdx){1to4}, %ymm22, %ymm25
+
+// CHECK: vfnmadd132pd -1024(%rdx){1to4}, %ymm22, %ymm25
+// CHECK:  encoding: [0x62,0x62,0xcd,0x30,0x9c,0x4a,0x80]
+          vfnmadd132pd -1024(%rdx){1to4}, %ymm22, %ymm25
+
+// CHECK: vfnmadd132pd -1032(%rdx){1to4}, %ymm22, %ymm25
+// CHECK:  encoding: [0x62,0x62,0xcd,0x30,0x9c,0x8a,0xf8,0xfb,0xff,0xff]
+          vfnmadd132pd -1032(%rdx){1to4}, %ymm22, %ymm25
+
+// CHECK: vfnmadd213ps %xmm20, %xmm24, %xmm24
+// CHECK:  encoding: [0x62,0x22,0x3d,0x00,0xac,0xc4]
+          vfnmadd213ps %xmm20, %xmm24, %xmm24
+
+// CHECK: vfnmadd213ps %xmm20, %xmm24, %xmm24 {%k4}
+// CHECK:  encoding: [0x62,0x22,0x3d,0x04,0xac,0xc4]
+          vfnmadd213ps %xmm20, %xmm24, %xmm24 {%k4}
+
+// CHECK: vfnmadd213ps %xmm20, %xmm24, %xmm24 {%k4} {z}
+// CHECK:  encoding: [0x62,0x22,0x3d,0x84,0xac,0xc4]
+          vfnmadd213ps %xmm20, %xmm24, %xmm24 {%k4} {z}
+
+// CHECK: vfnmadd213ps (%rcx), %xmm24, %xmm24
+// CHECK:  encoding: [0x62,0x62,0x3d,0x00,0xac,0x01]
+          vfnmadd213ps (%rcx), %xmm24, %xmm24
+
+// CHECK: vfnmadd213ps 291(%rax,%r14,8), %xmm24, %xmm24
+// CHECK:  encoding: [0x62,0x22,0x3d,0x00,0xac,0x84,0xf0,0x23,0x01,0x00,0x00]
+          vfnmadd213ps 291(%rax,%r14,8), %xmm24, %xmm24
+
+// CHECK: vfnmadd213ps (%rcx){1to4}, %xmm24, %xmm24
+// CHECK:  encoding: [0x62,0x62,0x3d,0x10,0xac,0x01]
+          vfnmadd213ps (%rcx){1to4}, %xmm24, %xmm24
+
+// CHECK: vfnmadd213ps 2032(%rdx), %xmm24, %xmm24
+// CHECK:  encoding: [0x62,0x62,0x3d,0x00,0xac,0x42,0x7f]
+          vfnmadd213ps 2032(%rdx), %xmm24, %xmm24
+
+// CHECK: vfnmadd213ps 2048(%rdx), %xmm24, %xmm24
+// CHECK:  encoding: [0x62,0x62,0x3d,0x00,0xac,0x82,0x00,0x08,0x00,0x00]
+          vfnmadd213ps 2048(%rdx), %xmm24, %xmm24
+
+// CHECK: vfnmadd213ps -2048(%rdx), %xmm24, %xmm24
+// CHECK:  encoding: [0x62,0x62,0x3d,0x00,0xac,0x42,0x80]
+          vfnmadd213ps -2048(%rdx), %xmm24, %xmm24
+
+// CHECK: vfnmadd213ps -2064(%rdx), %xmm24, %xmm24
+// CHECK:  encoding: [0x62,0x62,0x3d,0x00,0xac,0x82,0xf0,0xf7,0xff,0xff]
+          vfnmadd213ps -2064(%rdx), %xmm24, %xmm24
+
+// CHECK: vfnmadd213ps 508(%rdx){1to4}, %xmm24, %xmm24
+// CHECK:  encoding: [0x62,0x62,0x3d,0x10,0xac,0x42,0x7f]
+          vfnmadd213ps 508(%rdx){1to4}, %xmm24, %xmm24
+
+// CHECK: vfnmadd213ps 512(%rdx){1to4}, %xmm24, %xmm24
+// CHECK:  encoding: [0x62,0x62,0x3d,0x10,0xac,0x82,0x00,0x02,0x00,0x00]
+          vfnmadd213ps 512(%rdx){1to4}, %xmm24, %xmm24
+
+// CHECK: vfnmadd213ps -512(%rdx){1to4}, %xmm24, %xmm24
+// CHECK:  encoding: [0x62,0x62,0x3d,0x10,0xac,0x42,0x80]
+          vfnmadd213ps -512(%rdx){1to4}, %xmm24, %xmm24
+
+// CHECK: vfnmadd213ps -516(%rdx){1to4}, %xmm24, %xmm24
+// CHECK:  encoding: [0x62,0x62,0x3d,0x10,0xac,0x82,0xfc,0xfd,0xff,0xff]
+          vfnmadd213ps -516(%rdx){1to4}, %xmm24, %xmm24
+
+// CHECK: vfnmadd213ps %ymm22, %ymm19, %ymm21
+// CHECK:  encoding: [0x62,0xa2,0x65,0x20,0xac,0xee]
+          vfnmadd213ps %ymm22, %ymm19, %ymm21
+
+// CHECK: vfnmadd213ps %ymm22, %ymm19, %ymm21 {%k2}
+// CHECK:  encoding: [0x62,0xa2,0x65,0x22,0xac,0xee]
+          vfnmadd213ps %ymm22, %ymm19, %ymm21 {%k2}
+
+// CHECK: vfnmadd213ps %ymm22, %ymm19, %ymm21 {%k2} {z}
+// CHECK:  encoding: [0x62,0xa2,0x65,0xa2,0xac,0xee]
+          vfnmadd213ps %ymm22, %ymm19, %ymm21 {%k2} {z}
+
+// CHECK: vfnmadd213ps (%rcx), %ymm19, %ymm21
+// CHECK:  encoding: [0x62,0xe2,0x65,0x20,0xac,0x29]
+          vfnmadd213ps (%rcx), %ymm19, %ymm21
+
+// CHECK: vfnmadd213ps 291(%rax,%r14,8), %ymm19, %ymm21
+// CHECK:  encoding: [0x62,0xa2,0x65,0x20,0xac,0xac,0xf0,0x23,0x01,0x00,0x00]
+          vfnmadd213ps 291(%rax,%r14,8), %ymm19, %ymm21
+
+// CHECK: vfnmadd213ps (%rcx){1to8}, %ymm19, %ymm21
+// CHECK:  encoding: [0x62,0xe2,0x65,0x30,0xac,0x29]
+          vfnmadd213ps (%rcx){1to8}, %ymm19, %ymm21
+
+// CHECK: vfnmadd213ps 4064(%rdx), %ymm19, %ymm21
+// CHECK:  encoding: [0x62,0xe2,0x65,0x20,0xac,0x6a,0x7f]
+          vfnmadd213ps 4064(%rdx), %ymm19, %ymm21
+
+// CHECK: vfnmadd213ps 4096(%rdx), %ymm19, %ymm21
+// CHECK:  encoding: [0x62,0xe2,0x65,0x20,0xac,0xaa,0x00,0x10,0x00,0x00]
+          vfnmadd213ps 4096(%rdx), %ymm19, %ymm21
+
+// CHECK: vfnmadd213ps -4096(%rdx), %ymm19, %ymm21
+// CHECK:  encoding: [0x62,0xe2,0x65,0x20,0xac,0x6a,0x80]
+          vfnmadd213ps -4096(%rdx), %ymm19, %ymm21
+
+// CHECK: vfnmadd213ps -4128(%rdx), %ymm19, %ymm21
+// CHECK:  encoding: [0x62,0xe2,0x65,0x20,0xac,0xaa,0xe0,0xef,0xff,0xff]
+          vfnmadd213ps -4128(%rdx), %ymm19, %ymm21
+
+// CHECK: vfnmadd213ps 508(%rdx){1to8}, %ymm19, %ymm21
+// CHECK:  encoding: [0x62,0xe2,0x65,0x30,0xac,0x6a,0x7f]
+          vfnmadd213ps 508(%rdx){1to8}, %ymm19, %ymm21
+
+// CHECK: vfnmadd213ps 512(%rdx){1to8}, %ymm19, %ymm21
+// CHECK:  encoding: [0x62,0xe2,0x65,0x30,0xac,0xaa,0x00,0x02,0x00,0x00]
+          vfnmadd213ps 512(%rdx){1to8}, %ymm19, %ymm21
+
+// CHECK: vfnmadd213ps -512(%rdx){1to8}, %ymm19, %ymm21
+// CHECK:  encoding: [0x62,0xe2,0x65,0x30,0xac,0x6a,0x80]
+          vfnmadd213ps -512(%rdx){1to8}, %ymm19, %ymm21
+
+// CHECK: vfnmadd213ps -516(%rdx){1to8}, %ymm19, %ymm21
+// CHECK:  encoding: [0x62,0xe2,0x65,0x30,0xac,0xaa,0xfc,0xfd,0xff,0xff]
+          vfnmadd213ps -516(%rdx){1to8}, %ymm19, %ymm21
+
+// CHECK: vfnmadd213pd %xmm24, %xmm25, %xmm24
+// CHECK:  encoding: [0x62,0x02,0xb5,0x00,0xac,0xc0]
+          vfnmadd213pd %xmm24, %xmm25, %xmm24
+
+// CHECK: vfnmadd213pd %xmm24, %xmm25, %xmm24 {%k4}
+// CHECK:  encoding: [0x62,0x02,0xb5,0x04,0xac,0xc0]
+          vfnmadd213pd %xmm24, %xmm25, %xmm24 {%k4}
+
+// CHECK: vfnmadd213pd %xmm24, %xmm25, %xmm24 {%k4} {z}
+// CHECK:  encoding: [0x62,0x02,0xb5,0x84,0xac,0xc0]
+          vfnmadd213pd %xmm24, %xmm25, %xmm24 {%k4} {z}
+
+// CHECK: vfnmadd213pd (%rcx), %xmm25, %xmm24
+// CHECK:  encoding: [0x62,0x62,0xb5,0x00,0xac,0x01]
+          vfnmadd213pd (%rcx), %xmm25, %xmm24
+
+// CHECK: vfnmadd213pd 291(%rax,%r14,8), %xmm25, %xmm24
+// CHECK:  encoding: [0x62,0x22,0xb5,0x00,0xac,0x84,0xf0,0x23,0x01,0x00,0x00]
+          vfnmadd213pd 291(%rax,%r14,8), %xmm25, %xmm24
+
+// CHECK: vfnmadd213pd (%rcx){1to2}, %xmm25, %xmm24
+// CHECK:  encoding: [0x62,0x62,0xb5,0x10,0xac,0x01]
+          vfnmadd213pd (%rcx){1to2}, %xmm25, %xmm24
+
+// CHECK: vfnmadd213pd 2032(%rdx), %xmm25, %xmm24
+// CHECK:  encoding: [0x62,0x62,0xb5,0x00,0xac,0x42,0x7f]
+          vfnmadd213pd 2032(%rdx), %xmm25, %xmm24
+
+// CHECK: vfnmadd213pd 2048(%rdx), %xmm25, %xmm24
+// CHECK:  encoding: [0x62,0x62,0xb5,0x00,0xac,0x82,0x00,0x08,0x00,0x00]
+          vfnmadd213pd 2048(%rdx), %xmm25, %xmm24
+
+// CHECK: vfnmadd213pd -2048(%rdx), %xmm25, %xmm24
+// CHECK:  encoding: [0x62,0x62,0xb5,0x00,0xac,0x42,0x80]
+          vfnmadd213pd -2048(%rdx), %xmm25, %xmm24
+
+// CHECK: vfnmadd213pd -2064(%rdx), %xmm25, %xmm24
+// CHECK:  encoding: [0x62,0x62,0xb5,0x00,0xac,0x82,0xf0,0xf7,0xff,0xff]
+          vfnmadd213pd -2064(%rdx), %xmm25, %xmm24
+
+// CHECK: vfnmadd213pd 1016(%rdx){1to2}, %xmm25, %xmm24
+// CHECK:  encoding: [0x62,0x62,0xb5,0x10,0xac,0x42,0x7f]
+          vfnmadd213pd 1016(%rdx){1to2}, %xmm25, %xmm24
+
+// CHECK: vfnmadd213pd 1024(%rdx){1to2}, %xmm25, %xmm24
+// CHECK:  encoding: [0x62,0x62,0xb5,0x10,0xac,0x82,0x00,0x04,0x00,0x00]
+          vfnmadd213pd 1024(%rdx){1to2}, %xmm25, %xmm24
+
+// CHECK: vfnmadd213pd -1024(%rdx){1to2}, %xmm25, %xmm24
+// CHECK:  encoding: [0x62,0x62,0xb5,0x10,0xac,0x42,0x80]
+          vfnmadd213pd -1024(%rdx){1to2}, %xmm25, %xmm24
+
+// CHECK: vfnmadd213pd -1032(%rdx){1to2}, %xmm25, %xmm24
+// CHECK:  encoding: [0x62,0x62,0xb5,0x10,0xac,0x82,0xf8,0xfb,0xff,0xff]
+          vfnmadd213pd -1032(%rdx){1to2}, %xmm25, %xmm24
+
+// CHECK: vfnmadd213pd %ymm24, %ymm27, %ymm20
+// CHECK:  encoding: [0x62,0x82,0xa5,0x20,0xac,0xe0]
+          vfnmadd213pd %ymm24, %ymm27, %ymm20
+
+// CHECK: vfnmadd213pd %ymm24, %ymm27, %ymm20 {%k4}
+// CHECK:  encoding: [0x62,0x82,0xa5,0x24,0xac,0xe0]
+          vfnmadd213pd %ymm24, %ymm27, %ymm20 {%k4}
+
+// CHECK: vfnmadd213pd %ymm24, %ymm27, %ymm20 {%k4} {z}
+// CHECK:  encoding: [0x62,0x82,0xa5,0xa4,0xac,0xe0]
+          vfnmadd213pd %ymm24, %ymm27, %ymm20 {%k4} {z}
+
+// CHECK: vfnmadd213pd (%rcx), %ymm27, %ymm20
+// CHECK:  encoding: [0x62,0xe2,0xa5,0x20,0xac,0x21]
+          vfnmadd213pd (%rcx), %ymm27, %ymm20
+
+// CHECK: vfnmadd213pd 291(%rax,%r14,8), %ymm27, %ymm20
+// CHECK:  encoding: [0x62,0xa2,0xa5,0x20,0xac,0xa4,0xf0,0x23,0x01,0x00,0x00]
+          vfnmadd213pd 291(%rax,%r14,8), %ymm27, %ymm20
+
+// CHECK: vfnmadd213pd (%rcx){1to4}, %ymm27, %ymm20
+// CHECK:  encoding: [0x62,0xe2,0xa5,0x30,0xac,0x21]
+          vfnmadd213pd (%rcx){1to4}, %ymm27, %ymm20
+
+// CHECK: vfnmadd213pd 4064(%rdx), %ymm27, %ymm20
+// CHECK:  encoding: [0x62,0xe2,0xa5,0x20,0xac,0x62,0x7f]
+          vfnmadd213pd 4064(%rdx), %ymm27, %ymm20
+
+// CHECK: vfnmadd213pd 4096(%rdx), %ymm27, %ymm20
+// CHECK:  encoding: [0x62,0xe2,0xa5,0x20,0xac,0xa2,0x00,0x10,0x00,0x00]
+          vfnmadd213pd 4096(%rdx), %ymm27, %ymm20
+
+// CHECK: vfnmadd213pd -4096(%rdx), %ymm27, %ymm20
+// CHECK:  encoding: [0x62,0xe2,0xa5,0x20,0xac,0x62,0x80]
+          vfnmadd213pd -4096(%rdx), %ymm27, %ymm20
+
+// CHECK: vfnmadd213pd -4128(%rdx), %ymm27, %ymm20
+// CHECK:  encoding: [0x62,0xe2,0xa5,0x20,0xac,0xa2,0xe0,0xef,0xff,0xff]
+          vfnmadd213pd -4128(%rdx), %ymm27, %ymm20
+
+// CHECK: vfnmadd213pd 1016(%rdx){1to4}, %ymm27, %ymm20
+// CHECK:  encoding: [0x62,0xe2,0xa5,0x30,0xac,0x62,0x7f]
+          vfnmadd213pd 1016(%rdx){1to4}, %ymm27, %ymm20
+
+// CHECK: vfnmadd213pd 1024(%rdx){1to4}, %ymm27, %ymm20
+// CHECK:  encoding: [0x62,0xe2,0xa5,0x30,0xac,0xa2,0x00,0x04,0x00,0x00]
+          vfnmadd213pd 1024(%rdx){1to4}, %ymm27, %ymm20
+
+// CHECK: vfnmadd213pd -1024(%rdx){1to4}, %ymm27, %ymm20
+// CHECK:  encoding: [0x62,0xe2,0xa5,0x30,0xac,0x62,0x80]
+          vfnmadd213pd -1024(%rdx){1to4}, %ymm27, %ymm20
+
+// CHECK: vfnmadd213pd -1032(%rdx){1to4}, %ymm27, %ymm20
+// CHECK:  encoding: [0x62,0xe2,0xa5,0x30,0xac,0xa2,0xf8,0xfb,0xff,0xff]
+          vfnmadd213pd -1032(%rdx){1to4}, %ymm27, %ymm20
+
+// CHECK: vfnmadd231ps %xmm24, %xmm26, %xmm18
+// CHECK:  encoding: [0x62,0x82,0x2d,0x00,0xbc,0xd0]
+          vfnmadd231ps %xmm24, %xmm26, %xmm18
+
+// CHECK: vfnmadd231ps %xmm24, %xmm26, %xmm18 {%k1}
+// CHECK:  encoding: [0x62,0x82,0x2d,0x01,0xbc,0xd0]
+          vfnmadd231ps %xmm24, %xmm26, %xmm18 {%k1}
+
+// CHECK: vfnmadd231ps %xmm24, %xmm26, %xmm18 {%k1} {z}
+// CHECK:  encoding: [0x62,0x82,0x2d,0x81,0xbc,0xd0]
+          vfnmadd231ps %xmm24, %xmm26, %xmm18 {%k1} {z}
+
+// CHECK: vfnmadd231ps (%rcx), %xmm26, %xmm18
+// CHECK:  encoding: [0x62,0xe2,0x2d,0x00,0xbc,0x11]
+          vfnmadd231ps (%rcx), %xmm26, %xmm18
+
+// CHECK: vfnmadd231ps 291(%rax,%r14,8), %xmm26, %xmm18
+// CHECK:  encoding: [0x62,0xa2,0x2d,0x00,0xbc,0x94,0xf0,0x23,0x01,0x00,0x00]
+          vfnmadd231ps 291(%rax,%r14,8), %xmm26, %xmm18
+
+// CHECK: vfnmadd231ps (%rcx){1to4}, %xmm26, %xmm18
+// CHECK:  encoding: [0x62,0xe2,0x2d,0x10,0xbc,0x11]
+          vfnmadd231ps (%rcx){1to4}, %xmm26, %xmm18
+
+// CHECK: vfnmadd231ps 2032(%rdx), %xmm26, %xmm18
+// CHECK:  encoding: [0x62,0xe2,0x2d,0x00,0xbc,0x52,0x7f]
+          vfnmadd231ps 2032(%rdx), %xmm26, %xmm18
+
+// CHECK: vfnmadd231ps 2048(%rdx), %xmm26, %xmm18
+// CHECK:  encoding: [0x62,0xe2,0x2d,0x00,0xbc,0x92,0x00,0x08,0x00,0x00]
+          vfnmadd231ps 2048(%rdx), %xmm26, %xmm18
+
+// CHECK: vfnmadd231ps -2048(%rdx), %xmm26, %xmm18
+// CHECK:  encoding: [0x62,0xe2,0x2d,0x00,0xbc,0x52,0x80]
+          vfnmadd231ps -2048(%rdx), %xmm26, %xmm18
+
+// CHECK: vfnmadd231ps -2064(%rdx), %xmm26, %xmm18
+// CHECK:  encoding: [0x62,0xe2,0x2d,0x00,0xbc,0x92,0xf0,0xf7,0xff,0xff]
+          vfnmadd231ps -2064(%rdx), %xmm26, %xmm18
+
+// CHECK: vfnmadd231ps 508(%rdx){1to4}, %xmm26, %xmm18
+// CHECK:  encoding: [0x62,0xe2,0x2d,0x10,0xbc,0x52,0x7f]
+          vfnmadd231ps 508(%rdx){1to4}, %xmm26, %xmm18
+
+// CHECK: vfnmadd231ps 512(%rdx){1to4}, %xmm26, %xmm18
+// CHECK:  encoding: [0x62,0xe2,0x2d,0x10,0xbc,0x92,0x00,0x02,0x00,0x00]
+          vfnmadd231ps 512(%rdx){1to4}, %xmm26, %xmm18
+
+// CHECK: vfnmadd231ps -512(%rdx){1to4}, %xmm26, %xmm18
+// CHECK:  encoding: [0x62,0xe2,0x2d,0x10,0xbc,0x52,0x80]
+          vfnmadd231ps -512(%rdx){1to4}, %xmm26, %xmm18
+
+// CHECK: vfnmadd231ps -516(%rdx){1to4}, %xmm26, %xmm18
+// CHECK:  encoding: [0x62,0xe2,0x2d,0x10,0xbc,0x92,0xfc,0xfd,0xff,0xff]
+          vfnmadd231ps -516(%rdx){1to4}, %xmm26, %xmm18
+
+// CHECK: vfnmadd231ps %ymm21, %ymm20, %ymm20
+// CHECK:  encoding: [0x62,0xa2,0x5d,0x20,0xbc,0xe5]
+          vfnmadd231ps %ymm21, %ymm20, %ymm20
+
+// CHECK: vfnmadd231ps %ymm21, %ymm20, %ymm20 {%k4}
+// CHECK:  encoding: [0x62,0xa2,0x5d,0x24,0xbc,0xe5]
+          vfnmadd231ps %ymm21, %ymm20, %ymm20 {%k4}
+
+// CHECK: vfnmadd231ps %ymm21, %ymm20, %ymm20 {%k4} {z}
+// CHECK:  encoding: [0x62,0xa2,0x5d,0xa4,0xbc,0xe5]
+          vfnmadd231ps %ymm21, %ymm20, %ymm20 {%k4} {z}
+
+// CHECK: vfnmadd231ps (%rcx), %ymm20, %ymm20
+// CHECK:  encoding: [0x62,0xe2,0x5d,0x20,0xbc,0x21]
+          vfnmadd231ps (%rcx), %ymm20, %ymm20
+
+// CHECK: vfnmadd231ps 291(%rax,%r14,8), %ymm20, %ymm20
+// CHECK:  encoding: [0x62,0xa2,0x5d,0x20,0xbc,0xa4,0xf0,0x23,0x01,0x00,0x00]
+          vfnmadd231ps 291(%rax,%r14,8), %ymm20, %ymm20
+
+// CHECK: vfnmadd231ps (%rcx){1to8}, %ymm20, %ymm20
+// CHECK:  encoding: [0x62,0xe2,0x5d,0x30,0xbc,0x21]
+          vfnmadd231ps (%rcx){1to8}, %ymm20, %ymm20
+
+// CHECK: vfnmadd231ps 4064(%rdx), %ymm20, %ymm20
+// CHECK:  encoding: [0x62,0xe2,0x5d,0x20,0xbc,0x62,0x7f]
+          vfnmadd231ps 4064(%rdx), %ymm20, %ymm20
+
+// CHECK: vfnmadd231ps 4096(%rdx), %ymm20, %ymm20
+// CHECK:  encoding: [0x62,0xe2,0x5d,0x20,0xbc,0xa2,0x00,0x10,0x00,0x00]
+          vfnmadd231ps 4096(%rdx), %ymm20, %ymm20
+
+// CHECK: vfnmadd231ps -4096(%rdx), %ymm20, %ymm20
+// CHECK:  encoding: [0x62,0xe2,0x5d,0x20,0xbc,0x62,0x80]
+          vfnmadd231ps -4096(%rdx), %ymm20, %ymm20
+
+// CHECK: vfnmadd231ps -4128(%rdx), %ymm20, %ymm20
+// CHECK:  encoding: [0x62,0xe2,0x5d,0x20,0xbc,0xa2,0xe0,0xef,0xff,0xff]
+          vfnmadd231ps -4128(%rdx), %ymm20, %ymm20
+
+// CHECK: vfnmadd231ps 508(%rdx){1to8}, %ymm20, %ymm20
+// CHECK:  encoding: [0x62,0xe2,0x5d,0x30,0xbc,0x62,0x7f]
+          vfnmadd231ps 508(%rdx){1to8}, %ymm20, %ymm20
+
+// CHECK: vfnmadd231ps 512(%rdx){1to8}, %ymm20, %ymm20
+// CHECK:  encoding: [0x62,0xe2,0x5d,0x30,0xbc,0xa2,0x00,0x02,0x00,0x00]
+          vfnmadd231ps 512(%rdx){1to8}, %ymm20, %ymm20
+
+// CHECK: vfnmadd231ps -512(%rdx){1to8}, %ymm20, %ymm20
+// CHECK:  encoding: [0x62,0xe2,0x5d,0x30,0xbc,0x62,0x80]
+          vfnmadd231ps -512(%rdx){1to8}, %ymm20, %ymm20
+
+// CHECK: vfnmadd231ps -516(%rdx){1to8}, %ymm20, %ymm20
+// CHECK:  encoding: [0x62,0xe2,0x5d,0x30,0xbc,0xa2,0xfc,0xfd,0xff,0xff]
+          vfnmadd231ps -516(%rdx){1to8}, %ymm20, %ymm20
+
+// CHECK: vfnmadd231pd %xmm26, %xmm25, %xmm29
+// CHECK:  encoding: [0x62,0x02,0xb5,0x00,0xbc,0xea]
+          vfnmadd231pd %xmm26, %xmm25, %xmm29
+
+// CHECK: vfnmadd231pd %xmm26, %xmm25, %xmm29 {%k3}
+// CHECK:  encoding: [0x62,0x02,0xb5,0x03,0xbc,0xea]
+          vfnmadd231pd %xmm26, %xmm25, %xmm29 {%k3}
+
+// CHECK: vfnmadd231pd %xmm26, %xmm25, %xmm29 {%k3} {z}
+// CHECK:  encoding: [0x62,0x02,0xb5,0x83,0xbc,0xea]
+          vfnmadd231pd %xmm26, %xmm25, %xmm29 {%k3} {z}
+
+// CHECK: vfnmadd231pd (%rcx), %xmm25, %xmm29
+// CHECK:  encoding: [0x62,0x62,0xb5,0x00,0xbc,0x29]
+          vfnmadd231pd (%rcx), %xmm25, %xmm29
+
+// CHECK: vfnmadd231pd 291(%rax,%r14,8), %xmm25, %xmm29
+// CHECK:  encoding: [0x62,0x22,0xb5,0x00,0xbc,0xac,0xf0,0x23,0x01,0x00,0x00]
+          vfnmadd231pd 291(%rax,%r14,8), %xmm25, %xmm29
+
+// CHECK: vfnmadd231pd (%rcx){1to2}, %xmm25, %xmm29
+// CHECK:  encoding: [0x62,0x62,0xb5,0x10,0xbc,0x29]
+          vfnmadd231pd (%rcx){1to2}, %xmm25, %xmm29
+
+// CHECK: vfnmadd231pd 2032(%rdx), %xmm25, %xmm29
+// CHECK:  encoding: [0x62,0x62,0xb5,0x00,0xbc,0x6a,0x7f]
+          vfnmadd231pd 2032(%rdx), %xmm25, %xmm29
+
+// CHECK: vfnmadd231pd 2048(%rdx), %xmm25, %xmm29
+// CHECK:  encoding: [0x62,0x62,0xb5,0x00,0xbc,0xaa,0x00,0x08,0x00,0x00]
+          vfnmadd231pd 2048(%rdx), %xmm25, %xmm29
+
+// CHECK: vfnmadd231pd -2048(%rdx), %xmm25, %xmm29
+// CHECK:  encoding: [0x62,0x62,0xb5,0x00,0xbc,0x6a,0x80]
+          vfnmadd231pd -2048(%rdx), %xmm25, %xmm29
+
+// CHECK: vfnmadd231pd -2064(%rdx), %xmm25, %xmm29
+// CHECK:  encoding: [0x62,0x62,0xb5,0x00,0xbc,0xaa,0xf0,0xf7,0xff,0xff]
+          vfnmadd231pd -2064(%rdx), %xmm25, %xmm29
+
+// CHECK: vfnmadd231pd 1016(%rdx){1to2}, %xmm25, %xmm29
+// CHECK:  encoding: [0x62,0x62,0xb5,0x10,0xbc,0x6a,0x7f]
+          vfnmadd231pd 1016(%rdx){1to2}, %xmm25, %xmm29
+
+// CHECK: vfnmadd231pd 1024(%rdx){1to2}, %xmm25, %xmm29
+// CHECK:  encoding: [0x62,0x62,0xb5,0x10,0xbc,0xaa,0x00,0x04,0x00,0x00]
+          vfnmadd231pd 1024(%rdx){1to2}, %xmm25, %xmm29
+
+// CHECK: vfnmadd231pd -1024(%rdx){1to2}, %xmm25, %xmm29
+// CHECK:  encoding: [0x62,0x62,0xb5,0x10,0xbc,0x6a,0x80]
+          vfnmadd231pd -1024(%rdx){1to2}, %xmm25, %xmm29
+
+// CHECK: vfnmadd231pd -1032(%rdx){1to2}, %xmm25, %xmm29
+// CHECK:  encoding: [0x62,0x62,0xb5,0x10,0xbc,0xaa,0xf8,0xfb,0xff,0xff]
+          vfnmadd231pd -1032(%rdx){1to2}, %xmm25, %xmm29
+
+// CHECK: vfnmadd231pd %ymm23, %ymm29, %ymm22
+// CHECK:  encoding: [0x62,0xa2,0x95,0x20,0xbc,0xf7]
+          vfnmadd231pd %ymm23, %ymm29, %ymm22
+
+// CHECK: vfnmadd231pd %ymm23, %ymm29, %ymm22 {%k1}
+// CHECK:  encoding: [0x62,0xa2,0x95,0x21,0xbc,0xf7]
+          vfnmadd231pd %ymm23, %ymm29, %ymm22 {%k1}
+
+// CHECK: vfnmadd231pd %ymm23, %ymm29, %ymm22 {%k1} {z}
+// CHECK:  encoding: [0x62,0xa2,0x95,0xa1,0xbc,0xf7]
+          vfnmadd231pd %ymm23, %ymm29, %ymm22 {%k1} {z}
+
+// CHECK: vfnmadd231pd (%rcx), %ymm29, %ymm22
+// CHECK:  encoding: [0x62,0xe2,0x95,0x20,0xbc,0x31]
+          vfnmadd231pd (%rcx), %ymm29, %ymm22
+
+// CHECK: vfnmadd231pd 291(%rax,%r14,8), %ymm29, %ymm22
+// CHECK:  encoding: [0x62,0xa2,0x95,0x20,0xbc,0xb4,0xf0,0x23,0x01,0x00,0x00]
+          vfnmadd231pd 291(%rax,%r14,8), %ymm29, %ymm22
+
+// CHECK: vfnmadd231pd (%rcx){1to4}, %ymm29, %ymm22
+// CHECK:  encoding: [0x62,0xe2,0x95,0x30,0xbc,0x31]
+          vfnmadd231pd (%rcx){1to4}, %ymm29, %ymm22
+
+// CHECK: vfnmadd231pd 4064(%rdx), %ymm29, %ymm22
+// CHECK:  encoding: [0x62,0xe2,0x95,0x20,0xbc,0x72,0x7f]
+          vfnmadd231pd 4064(%rdx), %ymm29, %ymm22
+
+// CHECK: vfnmadd231pd 4096(%rdx), %ymm29, %ymm22
+// CHECK:  encoding: [0x62,0xe2,0x95,0x20,0xbc,0xb2,0x00,0x10,0x00,0x00]
+          vfnmadd231pd 4096(%rdx), %ymm29, %ymm22
+
+// CHECK: vfnmadd231pd -4096(%rdx), %ymm29, %ymm22
+// CHECK:  encoding: [0x62,0xe2,0x95,0x20,0xbc,0x72,0x80]
+          vfnmadd231pd -4096(%rdx), %ymm29, %ymm22
+
+// CHECK: vfnmadd231pd -4128(%rdx), %ymm29, %ymm22
+// CHECK:  encoding: [0x62,0xe2,0x95,0x20,0xbc,0xb2,0xe0,0xef,0xff,0xff]
+          vfnmadd231pd -4128(%rdx), %ymm29, %ymm22
+
+// CHECK: vfnmadd231pd 1016(%rdx){1to4}, %ymm29, %ymm22
+// CHECK:  encoding: [0x62,0xe2,0x95,0x30,0xbc,0x72,0x7f]
+          vfnmadd231pd 1016(%rdx){1to4}, %ymm29, %ymm22
+
+// CHECK: vfnmadd231pd 1024(%rdx){1to4}, %ymm29, %ymm22
+// CHECK:  encoding: [0x62,0xe2,0x95,0x30,0xbc,0xb2,0x00,0x04,0x00,0x00]
+          vfnmadd231pd 1024(%rdx){1to4}, %ymm29, %ymm22
+
+// CHECK: vfnmadd231pd -1024(%rdx){1to4}, %ymm29, %ymm22
+// CHECK:  encoding: [0x62,0xe2,0x95,0x30,0xbc,0x72,0x80]
+          vfnmadd231pd -1024(%rdx){1to4}, %ymm29, %ymm22
+
+// CHECK: vfnmadd231pd -1032(%rdx){1to4}, %ymm29, %ymm22
+// CHECK:  encoding: [0x62,0xe2,0x95,0x30,0xbc,0xb2,0xf8,0xfb,0xff,0xff]
+          vfnmadd231pd -1032(%rdx){1to4}, %ymm29, %ymm22
+
+// CHECK: vfnmsub132ps %xmm26, %xmm25, %xmm21
+// CHECK:  encoding: [0x62,0x82,0x35,0x00,0x9e,0xea]
+          vfnmsub132ps %xmm26, %xmm25, %xmm21
+
+// CHECK: vfnmsub132ps %xmm26, %xmm25, %xmm21 {%k3}
+// CHECK:  encoding: [0x62,0x82,0x35,0x03,0x9e,0xea]
+          vfnmsub132ps %xmm26, %xmm25, %xmm21 {%k3}
+
+// CHECK: vfnmsub132ps %xmm26, %xmm25, %xmm21 {%k3} {z}
+// CHECK:  encoding: [0x62,0x82,0x35,0x83,0x9e,0xea]
+          vfnmsub132ps %xmm26, %xmm25, %xmm21 {%k3} {z}
+
+// CHECK: vfnmsub132ps (%rcx), %xmm25, %xmm21
+// CHECK:  encoding: [0x62,0xe2,0x35,0x00,0x9e,0x29]
+          vfnmsub132ps (%rcx), %xmm25, %xmm21
+
+// CHECK: vfnmsub132ps 291(%rax,%r14,8), %xmm25, %xmm21
+// CHECK:  encoding: [0x62,0xa2,0x35,0x00,0x9e,0xac,0xf0,0x23,0x01,0x00,0x00]
+          vfnmsub132ps 291(%rax,%r14,8), %xmm25, %xmm21
+
+// CHECK: vfnmsub132ps (%rcx){1to4}, %xmm25, %xmm21
+// CHECK:  encoding: [0x62,0xe2,0x35,0x10,0x9e,0x29]
+          vfnmsub132ps (%rcx){1to4}, %xmm25, %xmm21
+
+// CHECK: vfnmsub132ps 2032(%rdx), %xmm25, %xmm21
+// CHECK:  encoding: [0x62,0xe2,0x35,0x00,0x9e,0x6a,0x7f]
+          vfnmsub132ps 2032(%rdx), %xmm25, %xmm21
+
+// CHECK: vfnmsub132ps 2048(%rdx), %xmm25, %xmm21
+// CHECK:  encoding: [0x62,0xe2,0x35,0x00,0x9e,0xaa,0x00,0x08,0x00,0x00]
+          vfnmsub132ps 2048(%rdx), %xmm25, %xmm21
+
+// CHECK: vfnmsub132ps -2048(%rdx), %xmm25, %xmm21
+// CHECK:  encoding: [0x62,0xe2,0x35,0x00,0x9e,0x6a,0x80]
+          vfnmsub132ps -2048(%rdx), %xmm25, %xmm21
+
+// CHECK: vfnmsub132ps -2064(%rdx), %xmm25, %xmm21
+// CHECK:  encoding: [0x62,0xe2,0x35,0x00,0x9e,0xaa,0xf0,0xf7,0xff,0xff]
+          vfnmsub132ps -2064(%rdx), %xmm25, %xmm21
+
+// CHECK: vfnmsub132ps 508(%rdx){1to4}, %xmm25, %xmm21
+// CHECK:  encoding: [0x62,0xe2,0x35,0x10,0x9e,0x6a,0x7f]
+          vfnmsub132ps 508(%rdx){1to4}, %xmm25, %xmm21
+
+// CHECK: vfnmsub132ps 512(%rdx){1to4}, %xmm25, %xmm21
+// CHECK:  encoding: [0x62,0xe2,0x35,0x10,0x9e,0xaa,0x00,0x02,0x00,0x00]
+          vfnmsub132ps 512(%rdx){1to4}, %xmm25, %xmm21
+
+// CHECK: vfnmsub132ps -512(%rdx){1to4}, %xmm25, %xmm21
+// CHECK:  encoding: [0x62,0xe2,0x35,0x10,0x9e,0x6a,0x80]
+          vfnmsub132ps -512(%rdx){1to4}, %xmm25, %xmm21
+
+// CHECK: vfnmsub132ps -516(%rdx){1to4}, %xmm25, %xmm21
+// CHECK:  encoding: [0x62,0xe2,0x35,0x10,0x9e,0xaa,0xfc,0xfd,0xff,0xff]
+          vfnmsub132ps -516(%rdx){1to4}, %xmm25, %xmm21
+
+// CHECK: vfnmsub132ps %ymm22, %ymm24, %ymm18
+// CHECK:  encoding: [0x62,0xa2,0x3d,0x20,0x9e,0xd6]
+          vfnmsub132ps %ymm22, %ymm24, %ymm18
+
+// CHECK: vfnmsub132ps %ymm22, %ymm24, %ymm18 {%k5}
+// CHECK:  encoding: [0x62,0xa2,0x3d,0x25,0x9e,0xd6]
+          vfnmsub132ps %ymm22, %ymm24, %ymm18 {%k5}
+
+// CHECK: vfnmsub132ps %ymm22, %ymm24, %ymm18 {%k5} {z}
+// CHECK:  encoding: [0x62,0xa2,0x3d,0xa5,0x9e,0xd6]
+          vfnmsub132ps %ymm22, %ymm24, %ymm18 {%k5} {z}
+
+// CHECK: vfnmsub132ps (%rcx), %ymm24, %ymm18
+// CHECK:  encoding: [0x62,0xe2,0x3d,0x20,0x9e,0x11]
+          vfnmsub132ps (%rcx), %ymm24, %ymm18
+
+// CHECK: vfnmsub132ps 291(%rax,%r14,8), %ymm24, %ymm18
+// CHECK:  encoding: [0x62,0xa2,0x3d,0x20,0x9e,0x94,0xf0,0x23,0x01,0x00,0x00]
+          vfnmsub132ps 291(%rax,%r14,8), %ymm24, %ymm18
+
+// CHECK: vfnmsub132ps (%rcx){1to8}, %ymm24, %ymm18
+// CHECK:  encoding: [0x62,0xe2,0x3d,0x30,0x9e,0x11]
+          vfnmsub132ps (%rcx){1to8}, %ymm24, %ymm18
+
+// CHECK: vfnmsub132ps 4064(%rdx), %ymm24, %ymm18
+// CHECK:  encoding: [0x62,0xe2,0x3d,0x20,0x9e,0x52,0x7f]
+          vfnmsub132ps 4064(%rdx), %ymm24, %ymm18
+
+// CHECK: vfnmsub132ps 4096(%rdx), %ymm24, %ymm18
+// CHECK:  encoding: [0x62,0xe2,0x3d,0x20,0x9e,0x92,0x00,0x10,0x00,0x00]
+          vfnmsub132ps 4096(%rdx), %ymm24, %ymm18
+
+// CHECK: vfnmsub132ps -4096(%rdx), %ymm24, %ymm18
+// CHECK:  encoding: [0x62,0xe2,0x3d,0x20,0x9e,0x52,0x80]
+          vfnmsub132ps -4096(%rdx), %ymm24, %ymm18
+
+// CHECK: vfnmsub132ps -4128(%rdx), %ymm24, %ymm18
+// CHECK:  encoding: [0x62,0xe2,0x3d,0x20,0x9e,0x92,0xe0,0xef,0xff,0xff]
+          vfnmsub132ps -4128(%rdx), %ymm24, %ymm18
+
+// CHECK: vfnmsub132ps 508(%rdx){1to8}, %ymm24, %ymm18
+// CHECK:  encoding: [0x62,0xe2,0x3d,0x30,0x9e,0x52,0x7f]
+          vfnmsub132ps 508(%rdx){1to8}, %ymm24, %ymm18
+
+// CHECK: vfnmsub132ps 512(%rdx){1to8}, %ymm24, %ymm18
+// CHECK:  encoding: [0x62,0xe2,0x3d,0x30,0x9e,0x92,0x00,0x02,0x00,0x00]
+          vfnmsub132ps 512(%rdx){1to8}, %ymm24, %ymm18
+
+// CHECK: vfnmsub132ps -512(%rdx){1to8}, %ymm24, %ymm18
+// CHECK:  encoding: [0x62,0xe2,0x3d,0x30,0x9e,0x52,0x80]
+          vfnmsub132ps -512(%rdx){1to8}, %ymm24, %ymm18
+
+// CHECK: vfnmsub132ps -516(%rdx){1to8}, %ymm24, %ymm18
+// CHECK:  encoding: [0x62,0xe2,0x3d,0x30,0x9e,0x92,0xfc,0xfd,0xff,0xff]
+          vfnmsub132ps -516(%rdx){1to8}, %ymm24, %ymm18
+
+// CHECK: vfnmsub132pd %xmm17, %xmm25, %xmm19
+// CHECK:  encoding: [0x62,0xa2,0xb5,0x00,0x9e,0xd9]
+          vfnmsub132pd %xmm17, %xmm25, %xmm19
+
+// CHECK: vfnmsub132pd %xmm17, %xmm25, %xmm19 {%k4}
+// CHECK:  encoding: [0x62,0xa2,0xb5,0x04,0x9e,0xd9]
+          vfnmsub132pd %xmm17, %xmm25, %xmm19 {%k4}
+
+// CHECK: vfnmsub132pd %xmm17, %xmm25, %xmm19 {%k4} {z}
+// CHECK:  encoding: [0x62,0xa2,0xb5,0x84,0x9e,0xd9]
+          vfnmsub132pd %xmm17, %xmm25, %xmm19 {%k4} {z}
+
+// CHECK: vfnmsub132pd (%rcx), %xmm25, %xmm19
+// CHECK:  encoding: [0x62,0xe2,0xb5,0x00,0x9e,0x19]
+          vfnmsub132pd (%rcx), %xmm25, %xmm19
+
+// CHECK: vfnmsub132pd 291(%rax,%r14,8), %xmm25, %xmm19
+// CHECK:  encoding: [0x62,0xa2,0xb5,0x00,0x9e,0x9c,0xf0,0x23,0x01,0x00,0x00]
+          vfnmsub132pd 291(%rax,%r14,8), %xmm25, %xmm19
+
+// CHECK: vfnmsub132pd (%rcx){1to2}, %xmm25, %xmm19
+// CHECK:  encoding: [0x62,0xe2,0xb5,0x10,0x9e,0x19]
+          vfnmsub132pd (%rcx){1to2}, %xmm25, %xmm19
+
+// CHECK: vfnmsub132pd 2032(%rdx), %xmm25, %xmm19
+// CHECK:  encoding: [0x62,0xe2,0xb5,0x00,0x9e,0x5a,0x7f]
+          vfnmsub132pd 2032(%rdx), %xmm25, %xmm19
+
+// CHECK: vfnmsub132pd 2048(%rdx), %xmm25, %xmm19
+// CHECK:  encoding: [0x62,0xe2,0xb5,0x00,0x9e,0x9a,0x00,0x08,0x00,0x00]
+          vfnmsub132pd 2048(%rdx), %xmm25, %xmm19
+
+// CHECK: vfnmsub132pd -2048(%rdx), %xmm25, %xmm19
+// CHECK:  encoding: [0x62,0xe2,0xb5,0x00,0x9e,0x5a,0x80]
+          vfnmsub132pd -2048(%rdx), %xmm25, %xmm19
+
+// CHECK: vfnmsub132pd -2064(%rdx), %xmm25, %xmm19
+// CHECK:  encoding: [0x62,0xe2,0xb5,0x00,0x9e,0x9a,0xf0,0xf7,0xff,0xff]
+          vfnmsub132pd -2064(%rdx), %xmm25, %xmm19
+
+// CHECK: vfnmsub132pd 1016(%rdx){1to2}, %xmm25, %xmm19
+// CHECK:  encoding: [0x62,0xe2,0xb5,0x10,0x9e,0x5a,0x7f]
+          vfnmsub132pd 1016(%rdx){1to2}, %xmm25, %xmm19
+
+// CHECK: vfnmsub132pd 1024(%rdx){1to2}, %xmm25, %xmm19
+// CHECK:  encoding: [0x62,0xe2,0xb5,0x10,0x9e,0x9a,0x00,0x04,0x00,0x00]
+          vfnmsub132pd 1024(%rdx){1to2}, %xmm25, %xmm19
+
+// CHECK: vfnmsub132pd -1024(%rdx){1to2}, %xmm25, %xmm19
+// CHECK:  encoding: [0x62,0xe2,0xb5,0x10,0x9e,0x5a,0x80]
+          vfnmsub132pd -1024(%rdx){1to2}, %xmm25, %xmm19
+
+// CHECK: vfnmsub132pd -1032(%rdx){1to2}, %xmm25, %xmm19
+// CHECK:  encoding: [0x62,0xe2,0xb5,0x10,0x9e,0x9a,0xf8,0xfb,0xff,0xff]
+          vfnmsub132pd -1032(%rdx){1to2}, %xmm25, %xmm19
+
+// CHECK: vfnmsub132pd %ymm22, %ymm28, %ymm17
+// CHECK:  encoding: [0x62,0xa2,0x9d,0x20,0x9e,0xce]
+          vfnmsub132pd %ymm22, %ymm28, %ymm17
+
+// CHECK: vfnmsub132pd %ymm22, %ymm28, %ymm17 {%k5}
+// CHECK:  encoding: [0x62,0xa2,0x9d,0x25,0x9e,0xce]
+          vfnmsub132pd %ymm22, %ymm28, %ymm17 {%k5}
+
+// CHECK: vfnmsub132pd %ymm22, %ymm28, %ymm17 {%k5} {z}
+// CHECK:  encoding: [0x62,0xa2,0x9d,0xa5,0x9e,0xce]
+          vfnmsub132pd %ymm22, %ymm28, %ymm17 {%k5} {z}
+
+// CHECK: vfnmsub132pd (%rcx), %ymm28, %ymm17
+// CHECK:  encoding: [0x62,0xe2,0x9d,0x20,0x9e,0x09]
+          vfnmsub132pd (%rcx), %ymm28, %ymm17
+
+// CHECK: vfnmsub132pd 291(%rax,%r14,8), %ymm28, %ymm17
+// CHECK:  encoding: [0x62,0xa2,0x9d,0x20,0x9e,0x8c,0xf0,0x23,0x01,0x00,0x00]
+          vfnmsub132pd 291(%rax,%r14,8), %ymm28, %ymm17
+
+// CHECK: vfnmsub132pd (%rcx){1to4}, %ymm28, %ymm17
+// CHECK:  encoding: [0x62,0xe2,0x9d,0x30,0x9e,0x09]
+          vfnmsub132pd (%rcx){1to4}, %ymm28, %ymm17
+
+// CHECK: vfnmsub132pd 4064(%rdx), %ymm28, %ymm17
+// CHECK:  encoding: [0x62,0xe2,0x9d,0x20,0x9e,0x4a,0x7f]
+          vfnmsub132pd 4064(%rdx), %ymm28, %ymm17
+
+// CHECK: vfnmsub132pd 4096(%rdx), %ymm28, %ymm17
+// CHECK:  encoding: [0x62,0xe2,0x9d,0x20,0x9e,0x8a,0x00,0x10,0x00,0x00]
+          vfnmsub132pd 4096(%rdx), %ymm28, %ymm17
+
+// CHECK: vfnmsub132pd -4096(%rdx), %ymm28, %ymm17
+// CHECK:  encoding: [0x62,0xe2,0x9d,0x20,0x9e,0x4a,0x80]
+          vfnmsub132pd -4096(%rdx), %ymm28, %ymm17
+
+// CHECK: vfnmsub132pd -4128(%rdx), %ymm28, %ymm17
+// CHECK:  encoding: [0x62,0xe2,0x9d,0x20,0x9e,0x8a,0xe0,0xef,0xff,0xff]
+          vfnmsub132pd -4128(%rdx), %ymm28, %ymm17
+
+// CHECK: vfnmsub132pd 1016(%rdx){1to4}, %ymm28, %ymm17
+// CHECK:  encoding: [0x62,0xe2,0x9d,0x30,0x9e,0x4a,0x7f]
+          vfnmsub132pd 1016(%rdx){1to4}, %ymm28, %ymm17
+
+// CHECK: vfnmsub132pd 1024(%rdx){1to4}, %ymm28, %ymm17
+// CHECK:  encoding: [0x62,0xe2,0x9d,0x30,0x9e,0x8a,0x00,0x04,0x00,0x00]
+          vfnmsub132pd 1024(%rdx){1to4}, %ymm28, %ymm17
+
+// CHECK: vfnmsub132pd -1024(%rdx){1to4}, %ymm28, %ymm17
+// CHECK:  encoding: [0x62,0xe2,0x9d,0x30,0x9e,0x4a,0x80]
+          vfnmsub132pd -1024(%rdx){1to4}, %ymm28, %ymm17
+
+// CHECK: vfnmsub132pd -1032(%rdx){1to4}, %ymm28, %ymm17
+// CHECK:  encoding: [0x62,0xe2,0x9d,0x30,0x9e,0x8a,0xf8,0xfb,0xff,0xff]
+          vfnmsub132pd -1032(%rdx){1to4}, %ymm28, %ymm17
+
+// CHECK: vfnmsub213ps %xmm18, %xmm28, %xmm28
+// CHECK:  encoding: [0x62,0x22,0x1d,0x00,0xae,0xe2]
+          vfnmsub213ps %xmm18, %xmm28, %xmm28
+
+// CHECK: vfnmsub213ps %xmm18, %xmm28, %xmm28 {%k4}
+// CHECK:  encoding: [0x62,0x22,0x1d,0x04,0xae,0xe2]
+          vfnmsub213ps %xmm18, %xmm28, %xmm28 {%k4}
+
+// CHECK: vfnmsub213ps %xmm18, %xmm28, %xmm28 {%k4} {z}
+// CHECK:  encoding: [0x62,0x22,0x1d,0x84,0xae,0xe2]
+          vfnmsub213ps %xmm18, %xmm28, %xmm28 {%k4} {z}
+
+// CHECK: vfnmsub213ps (%rcx), %xmm28, %xmm28
+// CHECK:  encoding: [0x62,0x62,0x1d,0x00,0xae,0x21]
+          vfnmsub213ps (%rcx), %xmm28, %xmm28
+
+// CHECK: vfnmsub213ps 291(%rax,%r14,8), %xmm28, %xmm28
+// CHECK:  encoding: [0x62,0x22,0x1d,0x00,0xae,0xa4,0xf0,0x23,0x01,0x00,0x00]
+          vfnmsub213ps 291(%rax,%r14,8), %xmm28, %xmm28
+
+// CHECK: vfnmsub213ps (%rcx){1to4}, %xmm28, %xmm28
+// CHECK:  encoding: [0x62,0x62,0x1d,0x10,0xae,0x21]
+          vfnmsub213ps (%rcx){1to4}, %xmm28, %xmm28
+
+// CHECK: vfnmsub213ps 2032(%rdx), %xmm28, %xmm28
+// CHECK:  encoding: [0x62,0x62,0x1d,0x00,0xae,0x62,0x7f]
+          vfnmsub213ps 2032(%rdx), %xmm28, %xmm28
+
+// CHECK: vfnmsub213ps 2048(%rdx), %xmm28, %xmm28
+// CHECK:  encoding: [0x62,0x62,0x1d,0x00,0xae,0xa2,0x00,0x08,0x00,0x00]
+          vfnmsub213ps 2048(%rdx), %xmm28, %xmm28
+
+// CHECK: vfnmsub213ps -2048(%rdx), %xmm28, %xmm28
+// CHECK:  encoding: [0x62,0x62,0x1d,0x00,0xae,0x62,0x80]
+          vfnmsub213ps -2048(%rdx), %xmm28, %xmm28
+
+// CHECK: vfnmsub213ps -2064(%rdx), %xmm28, %xmm28
+// CHECK:  encoding: [0x62,0x62,0x1d,0x00,0xae,0xa2,0xf0,0xf7,0xff,0xff]
+          vfnmsub213ps -2064(%rdx), %xmm28, %xmm28
+
+// CHECK: vfnmsub213ps 508(%rdx){1to4}, %xmm28, %xmm28
+// CHECK:  encoding: [0x62,0x62,0x1d,0x10,0xae,0x62,0x7f]
+          vfnmsub213ps 508(%rdx){1to4}, %xmm28, %xmm28
+
+// CHECK: vfnmsub213ps 512(%rdx){1to4}, %xmm28, %xmm28
+// CHECK:  encoding: [0x62,0x62,0x1d,0x10,0xae,0xa2,0x00,0x02,0x00,0x00]
+          vfnmsub213ps 512(%rdx){1to4}, %xmm28, %xmm28
+
+// CHECK: vfnmsub213ps -512(%rdx){1to4}, %xmm28, %xmm28
+// CHECK:  encoding: [0x62,0x62,0x1d,0x10,0xae,0x62,0x80]
+          vfnmsub213ps -512(%rdx){1to4}, %xmm28, %xmm28
+
+// CHECK: vfnmsub213ps -516(%rdx){1to4}, %xmm28, %xmm28
+// CHECK:  encoding: [0x62,0x62,0x1d,0x10,0xae,0xa2,0xfc,0xfd,0xff,0xff]
+          vfnmsub213ps -516(%rdx){1to4}, %xmm28, %xmm28
+
+// CHECK: vfnmsub213ps %ymm23, %ymm25, %ymm20
+// CHECK:  encoding: [0x62,0xa2,0x35,0x20,0xae,0xe7]
+          vfnmsub213ps %ymm23, %ymm25, %ymm20
+
+// CHECK: vfnmsub213ps %ymm23, %ymm25, %ymm20 {%k1}
+// CHECK:  encoding: [0x62,0xa2,0x35,0x21,0xae,0xe7]
+          vfnmsub213ps %ymm23, %ymm25, %ymm20 {%k1}
+
+// CHECK: vfnmsub213ps %ymm23, %ymm25, %ymm20 {%k1} {z}
+// CHECK:  encoding: [0x62,0xa2,0x35,0xa1,0xae,0xe7]
+          vfnmsub213ps %ymm23, %ymm25, %ymm20 {%k1} {z}
+
+// CHECK: vfnmsub213ps (%rcx), %ymm25, %ymm20
+// CHECK:  encoding: [0x62,0xe2,0x35,0x20,0xae,0x21]
+          vfnmsub213ps (%rcx), %ymm25, %ymm20
+
+// CHECK: vfnmsub213ps 291(%rax,%r14,8), %ymm25, %ymm20
+// CHECK:  encoding: [0x62,0xa2,0x35,0x20,0xae,0xa4,0xf0,0x23,0x01,0x00,0x00]
+          vfnmsub213ps 291(%rax,%r14,8), %ymm25, %ymm20
+
+// CHECK: vfnmsub213ps (%rcx){1to8}, %ymm25, %ymm20
+// CHECK:  encoding: [0x62,0xe2,0x35,0x30,0xae,0x21]
+          vfnmsub213ps (%rcx){1to8}, %ymm25, %ymm20
+
+// CHECK: vfnmsub213ps 4064(%rdx), %ymm25, %ymm20
+// CHECK:  encoding: [0x62,0xe2,0x35,0x20,0xae,0x62,0x7f]
+          vfnmsub213ps 4064(%rdx), %ymm25, %ymm20
+
+// CHECK: vfnmsub213ps 4096(%rdx), %ymm25, %ymm20
+// CHECK:  encoding: [0x62,0xe2,0x35,0x20,0xae,0xa2,0x00,0x10,0x00,0x00]
+          vfnmsub213ps 4096(%rdx), %ymm25, %ymm20
+
+// CHECK: vfnmsub213ps -4096(%rdx), %ymm25, %ymm20
+// CHECK:  encoding: [0x62,0xe2,0x35,0x20,0xae,0x62,0x80]
+          vfnmsub213ps -4096(%rdx), %ymm25, %ymm20
+
+// CHECK: vfnmsub213ps -4128(%rdx), %ymm25, %ymm20
+// CHECK:  encoding: [0x62,0xe2,0x35,0x20,0xae,0xa2,0xe0,0xef,0xff,0xff]
+          vfnmsub213ps -4128(%rdx), %ymm25, %ymm20
+
+// CHECK: vfnmsub213ps 508(%rdx){1to8}, %ymm25, %ymm20
+// CHECK:  encoding: [0x62,0xe2,0x35,0x30,0xae,0x62,0x7f]
+          vfnmsub213ps 508(%rdx){1to8}, %ymm25, %ymm20
+
+// CHECK: vfnmsub213ps 512(%rdx){1to8}, %ymm25, %ymm20
+// CHECK:  encoding: [0x62,0xe2,0x35,0x30,0xae,0xa2,0x00,0x02,0x00,0x00]
+          vfnmsub213ps 512(%rdx){1to8}, %ymm25, %ymm20
+
+// CHECK: vfnmsub213ps -512(%rdx){1to8}, %ymm25, %ymm20
+// CHECK:  encoding: [0x62,0xe2,0x35,0x30,0xae,0x62,0x80]
+          vfnmsub213ps -512(%rdx){1to8}, %ymm25, %ymm20
+
+// CHECK: vfnmsub213ps -516(%rdx){1to8}, %ymm25, %ymm20
+// CHECK:  encoding: [0x62,0xe2,0x35,0x30,0xae,0xa2,0xfc,0xfd,0xff,0xff]
+          vfnmsub213ps -516(%rdx){1to8}, %ymm25, %ymm20
+
+// CHECK: vfnmsub213pd %xmm25, %xmm17, %xmm20
+// CHECK:  encoding: [0x62,0x82,0xf5,0x00,0xae,0xe1]
+          vfnmsub213pd %xmm25, %xmm17, %xmm20
+
+// CHECK: vfnmsub213pd %xmm25, %xmm17, %xmm20 {%k1}
+// CHECK:  encoding: [0x62,0x82,0xf5,0x01,0xae,0xe1]
+          vfnmsub213pd %xmm25, %xmm17, %xmm20 {%k1}
+
+// CHECK: vfnmsub213pd %xmm25, %xmm17, %xmm20 {%k1} {z}
+// CHECK:  encoding: [0x62,0x82,0xf5,0x81,0xae,0xe1]
+          vfnmsub213pd %xmm25, %xmm17, %xmm20 {%k1} {z}
+
+// CHECK: vfnmsub213pd (%rcx), %xmm17, %xmm20
+// CHECK:  encoding: [0x62,0xe2,0xf5,0x00,0xae,0x21]
+          vfnmsub213pd (%rcx), %xmm17, %xmm20
+
+// CHECK: vfnmsub213pd 291(%rax,%r14,8), %xmm17, %xmm20
+// CHECK:  encoding: [0x62,0xa2,0xf5,0x00,0xae,0xa4,0xf0,0x23,0x01,0x00,0x00]
+          vfnmsub213pd 291(%rax,%r14,8), %xmm17, %xmm20
+
+// CHECK: vfnmsub213pd (%rcx){1to2}, %xmm17, %xmm20
+// CHECK:  encoding: [0x62,0xe2,0xf5,0x10,0xae,0x21]
+          vfnmsub213pd (%rcx){1to2}, %xmm17, %xmm20
+
+// CHECK: vfnmsub213pd 2032(%rdx), %xmm17, %xmm20
+// CHECK:  encoding: [0x62,0xe2,0xf5,0x00,0xae,0x62,0x7f]
+          vfnmsub213pd 2032(%rdx), %xmm17, %xmm20
+
+// CHECK: vfnmsub213pd 2048(%rdx), %xmm17, %xmm20
+// CHECK:  encoding: [0x62,0xe2,0xf5,0x00,0xae,0xa2,0x00,0x08,0x00,0x00]
+          vfnmsub213pd 2048(%rdx), %xmm17, %xmm20
+
+// CHECK: vfnmsub213pd -2048(%rdx), %xmm17, %xmm20
+// CHECK:  encoding: [0x62,0xe2,0xf5,0x00,0xae,0x62,0x80]
+          vfnmsub213pd -2048(%rdx), %xmm17, %xmm20
+
+// CHECK: vfnmsub213pd -2064(%rdx), %xmm17, %xmm20
+// CHECK:  encoding: [0x62,0xe2,0xf5,0x00,0xae,0xa2,0xf0,0xf7,0xff,0xff]
+          vfnmsub213pd -2064(%rdx), %xmm17, %xmm20
+
+// CHECK: vfnmsub213pd 1016(%rdx){1to2}, %xmm17, %xmm20
+// CHECK:  encoding: [0x62,0xe2,0xf5,0x10,0xae,0x62,0x7f]
+          vfnmsub213pd 1016(%rdx){1to2}, %xmm17, %xmm20
+
+// CHECK: vfnmsub213pd 1024(%rdx){1to2}, %xmm17, %xmm20
+// CHECK:  encoding: [0x62,0xe2,0xf5,0x10,0xae,0xa2,0x00,0x04,0x00,0x00]
+          vfnmsub213pd 1024(%rdx){1to2}, %xmm17, %xmm20
+
+// CHECK: vfnmsub213pd -1024(%rdx){1to2}, %xmm17, %xmm20
+// CHECK:  encoding: [0x62,0xe2,0xf5,0x10,0xae,0x62,0x80]
+          vfnmsub213pd -1024(%rdx){1to2}, %xmm17, %xmm20
+
+// CHECK: vfnmsub213pd -1032(%rdx){1to2}, %xmm17, %xmm20
+// CHECK:  encoding: [0x62,0xe2,0xf5,0x10,0xae,0xa2,0xf8,0xfb,0xff,0xff]
+          vfnmsub213pd -1032(%rdx){1to2}, %xmm17, %xmm20
+
+// CHECK: vfnmsub213pd %ymm28, %ymm20, %ymm19
+// CHECK:  encoding: [0x62,0x82,0xdd,0x20,0xae,0xdc]
+          vfnmsub213pd %ymm28, %ymm20, %ymm19
+
+// CHECK: vfnmsub213pd %ymm28, %ymm20, %ymm19 {%k7}
+// CHECK:  encoding: [0x62,0x82,0xdd,0x27,0xae,0xdc]
+          vfnmsub213pd %ymm28, %ymm20, %ymm19 {%k7}
+
+// CHECK: vfnmsub213pd %ymm28, %ymm20, %ymm19 {%k7} {z}
+// CHECK:  encoding: [0x62,0x82,0xdd,0xa7,0xae,0xdc]
+          vfnmsub213pd %ymm28, %ymm20, %ymm19 {%k7} {z}
+
+// CHECK: vfnmsub213pd (%rcx), %ymm20, %ymm19
+// CHECK:  encoding: [0x62,0xe2,0xdd,0x20,0xae,0x19]
+          vfnmsub213pd (%rcx), %ymm20, %ymm19
+
+// CHECK: vfnmsub213pd 291(%rax,%r14,8), %ymm20, %ymm19
+// CHECK:  encoding: [0x62,0xa2,0xdd,0x20,0xae,0x9c,0xf0,0x23,0x01,0x00,0x00]
+          vfnmsub213pd 291(%rax,%r14,8), %ymm20, %ymm19
+
+// CHECK: vfnmsub213pd (%rcx){1to4}, %ymm20, %ymm19
+// CHECK:  encoding: [0x62,0xe2,0xdd,0x30,0xae,0x19]
+          vfnmsub213pd (%rcx){1to4}, %ymm20, %ymm19
+
+// CHECK: vfnmsub213pd 4064(%rdx), %ymm20, %ymm19
+// CHECK:  encoding: [0x62,0xe2,0xdd,0x20,0xae,0x5a,0x7f]
+          vfnmsub213pd 4064(%rdx), %ymm20, %ymm19
+
+// CHECK: vfnmsub213pd 4096(%rdx), %ymm20, %ymm19
+// CHECK:  encoding: [0x62,0xe2,0xdd,0x20,0xae,0x9a,0x00,0x10,0x00,0x00]
+          vfnmsub213pd 4096(%rdx), %ymm20, %ymm19
+
+// CHECK: vfnmsub213pd -4096(%rdx), %ymm20, %ymm19
+// CHECK:  encoding: [0x62,0xe2,0xdd,0x20,0xae,0x5a,0x80]
+          vfnmsub213pd -4096(%rdx), %ymm20, %ymm19
+
+// CHECK: vfnmsub213pd -4128(%rdx), %ymm20, %ymm19
+// CHECK:  encoding: [0x62,0xe2,0xdd,0x20,0xae,0x9a,0xe0,0xef,0xff,0xff]
+          vfnmsub213pd -4128(%rdx), %ymm20, %ymm19
+
+// CHECK: vfnmsub213pd 1016(%rdx){1to4}, %ymm20, %ymm19
+// CHECK:  encoding: [0x62,0xe2,0xdd,0x30,0xae,0x5a,0x7f]
+          vfnmsub213pd 1016(%rdx){1to4}, %ymm20, %ymm19
+
+// CHECK: vfnmsub213pd 1024(%rdx){1to4}, %ymm20, %ymm19
+// CHECK:  encoding: [0x62,0xe2,0xdd,0x30,0xae,0x9a,0x00,0x04,0x00,0x00]
+          vfnmsub213pd 1024(%rdx){1to4}, %ymm20, %ymm19
+
+// CHECK: vfnmsub213pd -1024(%rdx){1to4}, %ymm20, %ymm19
+// CHECK:  encoding: [0x62,0xe2,0xdd,0x30,0xae,0x5a,0x80]
+          vfnmsub213pd -1024(%rdx){1to4}, %ymm20, %ymm19
+
+// CHECK: vfnmsub213pd -1032(%rdx){1to4}, %ymm20, %ymm19
+// CHECK:  encoding: [0x62,0xe2,0xdd,0x30,0xae,0x9a,0xf8,0xfb,0xff,0xff]
+          vfnmsub213pd -1032(%rdx){1to4}, %ymm20, %ymm19
+
+// CHECK: vfnmsub231ps %xmm26, %xmm27, %xmm18
+// CHECK:  encoding: [0x62,0x82,0x25,0x00,0xbe,0xd2]
+          vfnmsub231ps %xmm26, %xmm27, %xmm18
+
+// CHECK: vfnmsub231ps %xmm26, %xmm27, %xmm18 {%k2}
+// CHECK:  encoding: [0x62,0x82,0x25,0x02,0xbe,0xd2]
+          vfnmsub231ps %xmm26, %xmm27, %xmm18 {%k2}
+
+// CHECK: vfnmsub231ps %xmm26, %xmm27, %xmm18 {%k2} {z}
+// CHECK:  encoding: [0x62,0x82,0x25,0x82,0xbe,0xd2]
+          vfnmsub231ps %xmm26, %xmm27, %xmm18 {%k2} {z}
+
+// CHECK: vfnmsub231ps (%rcx), %xmm27, %xmm18
+// CHECK:  encoding: [0x62,0xe2,0x25,0x00,0xbe,0x11]
+          vfnmsub231ps (%rcx), %xmm27, %xmm18
+
+// CHECK: vfnmsub231ps 291(%rax,%r14,8), %xmm27, %xmm18
+// CHECK:  encoding: [0x62,0xa2,0x25,0x00,0xbe,0x94,0xf0,0x23,0x01,0x00,0x00]
+          vfnmsub231ps 291(%rax,%r14,8), %xmm27, %xmm18
+
+// CHECK: vfnmsub231ps (%rcx){1to4}, %xmm27, %xmm18
+// CHECK:  encoding: [0x62,0xe2,0x25,0x10,0xbe,0x11]
+          vfnmsub231ps (%rcx){1to4}, %xmm27, %xmm18
+
+// CHECK: vfnmsub231ps 2032(%rdx), %xmm27, %xmm18
+// CHECK:  encoding: [0x62,0xe2,0x25,0x00,0xbe,0x52,0x7f]
+          vfnmsub231ps 2032(%rdx), %xmm27, %xmm18
+
+// CHECK: vfnmsub231ps 2048(%rdx), %xmm27, %xmm18
+// CHECK:  encoding: [0x62,0xe2,0x25,0x00,0xbe,0x92,0x00,0x08,0x00,0x00]
+          vfnmsub231ps 2048(%rdx), %xmm27, %xmm18
+
+// CHECK: vfnmsub231ps -2048(%rdx), %xmm27, %xmm18
+// CHECK:  encoding: [0x62,0xe2,0x25,0x00,0xbe,0x52,0x80]
+          vfnmsub231ps -2048(%rdx), %xmm27, %xmm18
+
+// CHECK: vfnmsub231ps -2064(%rdx), %xmm27, %xmm18
+// CHECK:  encoding: [0x62,0xe2,0x25,0x00,0xbe,0x92,0xf0,0xf7,0xff,0xff]
+          vfnmsub231ps -2064(%rdx), %xmm27, %xmm18
+
+// CHECK: vfnmsub231ps 508(%rdx){1to4}, %xmm27, %xmm18
+// CHECK:  encoding: [0x62,0xe2,0x25,0x10,0xbe,0x52,0x7f]
+          vfnmsub231ps 508(%rdx){1to4}, %xmm27, %xmm18
+
+// CHECK: vfnmsub231ps 512(%rdx){1to4}, %xmm27, %xmm18
+// CHECK:  encoding: [0x62,0xe2,0x25,0x10,0xbe,0x92,0x00,0x02,0x00,0x00]
+          vfnmsub231ps 512(%rdx){1to4}, %xmm27, %xmm18
+
+// CHECK: vfnmsub231ps -512(%rdx){1to4}, %xmm27, %xmm18
+// CHECK:  encoding: [0x62,0xe2,0x25,0x10,0xbe,0x52,0x80]
+          vfnmsub231ps -512(%rdx){1to4}, %xmm27, %xmm18
+
+// CHECK: vfnmsub231ps -516(%rdx){1to4}, %xmm27, %xmm18
+// CHECK:  encoding: [0x62,0xe2,0x25,0x10,0xbe,0x92,0xfc,0xfd,0xff,0xff]
+          vfnmsub231ps -516(%rdx){1to4}, %xmm27, %xmm18
+
+// CHECK: vfnmsub231ps %ymm18, %ymm20, %ymm30
+// CHECK:  encoding: [0x62,0x22,0x5d,0x20,0xbe,0xf2]
+          vfnmsub231ps %ymm18, %ymm20, %ymm30
+
+// CHECK: vfnmsub231ps %ymm18, %ymm20, %ymm30 {%k1}
+// CHECK:  encoding: [0x62,0x22,0x5d,0x21,0xbe,0xf2]
+          vfnmsub231ps %ymm18, %ymm20, %ymm30 {%k1}
+
+// CHECK: vfnmsub231ps %ymm18, %ymm20, %ymm30 {%k1} {z}
+// CHECK:  encoding: [0x62,0x22,0x5d,0xa1,0xbe,0xf2]
+          vfnmsub231ps %ymm18, %ymm20, %ymm30 {%k1} {z}
+
+// CHECK: vfnmsub231ps (%rcx), %ymm20, %ymm30
+// CHECK:  encoding: [0x62,0x62,0x5d,0x20,0xbe,0x31]
+          vfnmsub231ps (%rcx), %ymm20, %ymm30
+
+// CHECK: vfnmsub231ps 291(%rax,%r14,8), %ymm20, %ymm30
+// CHECK:  encoding: [0x62,0x22,0x5d,0x20,0xbe,0xb4,0xf0,0x23,0x01,0x00,0x00]
+          vfnmsub231ps 291(%rax,%r14,8), %ymm20, %ymm30
+
+// CHECK: vfnmsub231ps (%rcx){1to8}, %ymm20, %ymm30
+// CHECK:  encoding: [0x62,0x62,0x5d,0x30,0xbe,0x31]
+          vfnmsub231ps (%rcx){1to8}, %ymm20, %ymm30
+
+// CHECK: vfnmsub231ps 4064(%rdx), %ymm20, %ymm30
+// CHECK:  encoding: [0x62,0x62,0x5d,0x20,0xbe,0x72,0x7f]
+          vfnmsub231ps 4064(%rdx), %ymm20, %ymm30
+
+// CHECK: vfnmsub231ps 4096(%rdx), %ymm20, %ymm30
+// CHECK:  encoding: [0x62,0x62,0x5d,0x20,0xbe,0xb2,0x00,0x10,0x00,0x00]
+          vfnmsub231ps 4096(%rdx), %ymm20, %ymm30
+
+// CHECK: vfnmsub231ps -4096(%rdx), %ymm20, %ymm30
+// CHECK:  encoding: [0x62,0x62,0x5d,0x20,0xbe,0x72,0x80]
+          vfnmsub231ps -4096(%rdx), %ymm20, %ymm30
+
+// CHECK: vfnmsub231ps -4128(%rdx), %ymm20, %ymm30
+// CHECK:  encoding: [0x62,0x62,0x5d,0x20,0xbe,0xb2,0xe0,0xef,0xff,0xff]
+          vfnmsub231ps -4128(%rdx), %ymm20, %ymm30
+
+// CHECK: vfnmsub231ps 508(%rdx){1to8}, %ymm20, %ymm30
+// CHECK:  encoding: [0x62,0x62,0x5d,0x30,0xbe,0x72,0x7f]
+          vfnmsub231ps 508(%rdx){1to8}, %ymm20, %ymm30
+
+// CHECK: vfnmsub231ps 512(%rdx){1to8}, %ymm20, %ymm30
+// CHECK:  encoding: [0x62,0x62,0x5d,0x30,0xbe,0xb2,0x00,0x02,0x00,0x00]
+          vfnmsub231ps 512(%rdx){1to8}, %ymm20, %ymm30
+
+// CHECK: vfnmsub231ps -512(%rdx){1to8}, %ymm20, %ymm30
+// CHECK:  encoding: [0x62,0x62,0x5d,0x30,0xbe,0x72,0x80]
+          vfnmsub231ps -512(%rdx){1to8}, %ymm20, %ymm30
+
+// CHECK: vfnmsub231ps -516(%rdx){1to8}, %ymm20, %ymm30
+// CHECK:  encoding: [0x62,0x62,0x5d,0x30,0xbe,0xb2,0xfc,0xfd,0xff,0xff]
+          vfnmsub231ps -516(%rdx){1to8}, %ymm20, %ymm30
+
+// CHECK: vfnmsub231pd %xmm25, %xmm19, %xmm23
+// CHECK:  encoding: [0x62,0x82,0xe5,0x00,0xbe,0xf9]
+          vfnmsub231pd %xmm25, %xmm19, %xmm23
+
+// CHECK: vfnmsub231pd %xmm25, %xmm19, %xmm23 {%k3}
+// CHECK:  encoding: [0x62,0x82,0xe5,0x03,0xbe,0xf9]
+          vfnmsub231pd %xmm25, %xmm19, %xmm23 {%k3}
+
+// CHECK: vfnmsub231pd %xmm25, %xmm19, %xmm23 {%k3} {z}
+// CHECK:  encoding: [0x62,0x82,0xe5,0x83,0xbe,0xf9]
+          vfnmsub231pd %xmm25, %xmm19, %xmm23 {%k3} {z}
+
+// CHECK: vfnmsub231pd (%rcx), %xmm19, %xmm23
+// CHECK:  encoding: [0x62,0xe2,0xe5,0x00,0xbe,0x39]
+          vfnmsub231pd (%rcx), %xmm19, %xmm23
+
+// CHECK: vfnmsub231pd 291(%rax,%r14,8), %xmm19, %xmm23
+// CHECK:  encoding: [0x62,0xa2,0xe5,0x00,0xbe,0xbc,0xf0,0x23,0x01,0x00,0x00]
+          vfnmsub231pd 291(%rax,%r14,8), %xmm19, %xmm23
+
+// CHECK: vfnmsub231pd (%rcx){1to2}, %xmm19, %xmm23
+// CHECK:  encoding: [0x62,0xe2,0xe5,0x10,0xbe,0x39]
+          vfnmsub231pd (%rcx){1to2}, %xmm19, %xmm23
+
+// CHECK: vfnmsub231pd 2032(%rdx), %xmm19, %xmm23
+// CHECK:  encoding: [0x62,0xe2,0xe5,0x00,0xbe,0x7a,0x7f]
+          vfnmsub231pd 2032(%rdx), %xmm19, %xmm23
+
+// CHECK: vfnmsub231pd 2048(%rdx), %xmm19, %xmm23
+// CHECK:  encoding: [0x62,0xe2,0xe5,0x00,0xbe,0xba,0x00,0x08,0x00,0x00]
+          vfnmsub231pd 2048(%rdx), %xmm19, %xmm23
+
+// CHECK: vfnmsub231pd -2048(%rdx), %xmm19, %xmm23
+// CHECK:  encoding: [0x62,0xe2,0xe5,0x00,0xbe,0x7a,0x80]
+          vfnmsub231pd -2048(%rdx), %xmm19, %xmm23
+
+// CHECK: vfnmsub231pd -2064(%rdx), %xmm19, %xmm23
+// CHECK:  encoding: [0x62,0xe2,0xe5,0x00,0xbe,0xba,0xf0,0xf7,0xff,0xff]
+          vfnmsub231pd -2064(%rdx), %xmm19, %xmm23
+
+// CHECK: vfnmsub231pd 1016(%rdx){1to2}, %xmm19, %xmm23
+// CHECK:  encoding: [0x62,0xe2,0xe5,0x10,0xbe,0x7a,0x7f]
+          vfnmsub231pd 1016(%rdx){1to2}, %xmm19, %xmm23
+
+// CHECK: vfnmsub231pd 1024(%rdx){1to2}, %xmm19, %xmm23
+// CHECK:  encoding: [0x62,0xe2,0xe5,0x10,0xbe,0xba,0x00,0x04,0x00,0x00]
+          vfnmsub231pd 1024(%rdx){1to2}, %xmm19, %xmm23
+
+// CHECK: vfnmsub231pd -1024(%rdx){1to2}, %xmm19, %xmm23
+// CHECK:  encoding: [0x62,0xe2,0xe5,0x10,0xbe,0x7a,0x80]
+          vfnmsub231pd -1024(%rdx){1to2}, %xmm19, %xmm23
+
+// CHECK: vfnmsub231pd -1032(%rdx){1to2}, %xmm19, %xmm23
+// CHECK:  encoding: [0x62,0xe2,0xe5,0x10,0xbe,0xba,0xf8,0xfb,0xff,0xff]
+          vfnmsub231pd -1032(%rdx){1to2}, %xmm19, %xmm23
+
+// CHECK: vfnmsub231pd %ymm20, %ymm18, %ymm22
+// CHECK:  encoding: [0x62,0xa2,0xed,0x20,0xbe,0xf4]
+          vfnmsub231pd %ymm20, %ymm18, %ymm22
+
+// CHECK: vfnmsub231pd %ymm20, %ymm18, %ymm22 {%k1}
+// CHECK:  encoding: [0x62,0xa2,0xed,0x21,0xbe,0xf4]
+          vfnmsub231pd %ymm20, %ymm18, %ymm22 {%k1}
+
+// CHECK: vfnmsub231pd %ymm20, %ymm18, %ymm22 {%k1} {z}
+// CHECK:  encoding: [0x62,0xa2,0xed,0xa1,0xbe,0xf4]
+          vfnmsub231pd %ymm20, %ymm18, %ymm22 {%k1} {z}
+
+// CHECK: vfnmsub231pd (%rcx), %ymm18, %ymm22
+// CHECK:  encoding: [0x62,0xe2,0xed,0x20,0xbe,0x31]
+          vfnmsub231pd (%rcx), %ymm18, %ymm22
+
+// CHECK: vfnmsub231pd 291(%rax,%r14,8), %ymm18, %ymm22
+// CHECK:  encoding: [0x62,0xa2,0xed,0x20,0xbe,0xb4,0xf0,0x23,0x01,0x00,0x00]
+          vfnmsub231pd 291(%rax,%r14,8), %ymm18, %ymm22
+
+// CHECK: vfnmsub231pd (%rcx){1to4}, %ymm18, %ymm22
+// CHECK:  encoding: [0x62,0xe2,0xed,0x30,0xbe,0x31]
+          vfnmsub231pd (%rcx){1to4}, %ymm18, %ymm22
+
+// CHECK: vfnmsub231pd 4064(%rdx), %ymm18, %ymm22
+// CHECK:  encoding: [0x62,0xe2,0xed,0x20,0xbe,0x72,0x7f]
+          vfnmsub231pd 4064(%rdx), %ymm18, %ymm22
+
+// CHECK: vfnmsub231pd 4096(%rdx), %ymm18, %ymm22
+// CHECK:  encoding: [0x62,0xe2,0xed,0x20,0xbe,0xb2,0x00,0x10,0x00,0x00]
+          vfnmsub231pd 4096(%rdx), %ymm18, %ymm22
+
+// CHECK: vfnmsub231pd -4096(%rdx), %ymm18, %ymm22
+// CHECK:  encoding: [0x62,0xe2,0xed,0x20,0xbe,0x72,0x80]
+          vfnmsub231pd -4096(%rdx), %ymm18, %ymm22
+
+// CHECK: vfnmsub231pd -4128(%rdx), %ymm18, %ymm22
+// CHECK:  encoding: [0x62,0xe2,0xed,0x20,0xbe,0xb2,0xe0,0xef,0xff,0xff]
+          vfnmsub231pd -4128(%rdx), %ymm18, %ymm22
+
+// CHECK: vfnmsub231pd 1016(%rdx){1to4}, %ymm18, %ymm22
+// CHECK:  encoding: [0x62,0xe2,0xed,0x30,0xbe,0x72,0x7f]
+          vfnmsub231pd 1016(%rdx){1to4}, %ymm18, %ymm22
+
+// CHECK: vfnmsub231pd 1024(%rdx){1to4}, %ymm18, %ymm22
+// CHECK:  encoding: [0x62,0xe2,0xed,0x30,0xbe,0xb2,0x00,0x04,0x00,0x00]
+          vfnmsub231pd 1024(%rdx){1to4}, %ymm18, %ymm22
+
+// CHECK: vfnmsub231pd -1024(%rdx){1to4}, %ymm18, %ymm22
+// CHECK:  encoding: [0x62,0xe2,0xed,0x30,0xbe,0x72,0x80]
+          vfnmsub231pd -1024(%rdx){1to4}, %ymm18, %ymm22
+
+// CHECK: vfnmsub231pd -1032(%rdx){1to4}, %ymm18, %ymm22
+// CHECK:  encoding: [0x62,0xe2,0xed,0x30,0xbe,0xb2,0xf8,0xfb,0xff,0xff]
+          vfnmsub231pd -1032(%rdx){1to4}, %ymm18, %ymm22
+
+// CHECK: vpermi2d %xmm25, %xmm23, %xmm21
+// CHECK:  encoding: [0x62,0x82,0x45,0x00,0x76,0xe9]
+          vpermi2d %xmm25, %xmm23, %xmm21
+
+// CHECK: vpermi2d %xmm25, %xmm23, %xmm21 {%k6}
+// CHECK:  encoding: [0x62,0x82,0x45,0x06,0x76,0xe9]
+          vpermi2d %xmm25, %xmm23, %xmm21 {%k6}
+
+// CHECK: vpermi2d %xmm25, %xmm23, %xmm21 {%k6} {z}
+// CHECK:  encoding: [0x62,0x82,0x45,0x86,0x76,0xe9]
+          vpermi2d %xmm25, %xmm23, %xmm21 {%k6} {z}
+
+// CHECK: vpermi2d (%rcx), %xmm23, %xmm21
+// CHECK:  encoding: [0x62,0xe2,0x45,0x00,0x76,0x29]
+          vpermi2d (%rcx), %xmm23, %xmm21
+
+// CHECK: vpermi2d 291(%rax,%r14,8), %xmm23, %xmm21
+// CHECK:  encoding: [0x62,0xa2,0x45,0x00,0x76,0xac,0xf0,0x23,0x01,0x00,0x00]
+          vpermi2d 291(%rax,%r14,8), %xmm23, %xmm21
+
+// CHECK: vpermi2d (%rcx){1to4}, %xmm23, %xmm21
+// CHECK:  encoding: [0x62,0xe2,0x45,0x10,0x76,0x29]
+          vpermi2d (%rcx){1to4}, %xmm23, %xmm21
+
+// CHECK: vpermi2d 2032(%rdx), %xmm23, %xmm21
+// CHECK:  encoding: [0x62,0xe2,0x45,0x00,0x76,0x6a,0x7f]
+          vpermi2d 2032(%rdx), %xmm23, %xmm21
+
+// CHECK: vpermi2d 2048(%rdx), %xmm23, %xmm21
+// CHECK:  encoding: [0x62,0xe2,0x45,0x00,0x76,0xaa,0x00,0x08,0x00,0x00]
+          vpermi2d 2048(%rdx), %xmm23, %xmm21
+
+// CHECK: vpermi2d -2048(%rdx), %xmm23, %xmm21
+// CHECK:  encoding: [0x62,0xe2,0x45,0x00,0x76,0x6a,0x80]
+          vpermi2d -2048(%rdx), %xmm23, %xmm21
+
+// CHECK: vpermi2d -2064(%rdx), %xmm23, %xmm21
+// CHECK:  encoding: [0x62,0xe2,0x45,0x00,0x76,0xaa,0xf0,0xf7,0xff,0xff]
+          vpermi2d -2064(%rdx), %xmm23, %xmm21
+
+// CHECK: vpermi2d 508(%rdx){1to4}, %xmm23, %xmm21
+// CHECK:  encoding: [0x62,0xe2,0x45,0x10,0x76,0x6a,0x7f]
+          vpermi2d 508(%rdx){1to4}, %xmm23, %xmm21
+
+// CHECK: vpermi2d 512(%rdx){1to4}, %xmm23, %xmm21
+// CHECK:  encoding: [0x62,0xe2,0x45,0x10,0x76,0xaa,0x00,0x02,0x00,0x00]
+          vpermi2d 512(%rdx){1to4}, %xmm23, %xmm21
+
+// CHECK: vpermi2d -512(%rdx){1to4}, %xmm23, %xmm21
+// CHECK:  encoding: [0x62,0xe2,0x45,0x10,0x76,0x6a,0x80]
+          vpermi2d -512(%rdx){1to4}, %xmm23, %xmm21
+
+// CHECK: vpermi2d -516(%rdx){1to4}, %xmm23, %xmm21
+// CHECK:  encoding: [0x62,0xe2,0x45,0x10,0x76,0xaa,0xfc,0xfd,0xff,0xff]
+          vpermi2d -516(%rdx){1to4}, %xmm23, %xmm21
+
+// CHECK: vpermi2d %ymm22, %ymm24, %ymm18
+// CHECK:  encoding: [0x62,0xa2,0x3d,0x20,0x76,0xd6]
+          vpermi2d %ymm22, %ymm24, %ymm18
+
+// CHECK: vpermi2d %ymm22, %ymm24, %ymm18 {%k1}
+// CHECK:  encoding: [0x62,0xa2,0x3d,0x21,0x76,0xd6]
+          vpermi2d %ymm22, %ymm24, %ymm18 {%k1}
+
+// CHECK: vpermi2d %ymm22, %ymm24, %ymm18 {%k1} {z}
+// CHECK:  encoding: [0x62,0xa2,0x3d,0xa1,0x76,0xd6]
+          vpermi2d %ymm22, %ymm24, %ymm18 {%k1} {z}
+
+// CHECK: vpermi2d (%rcx), %ymm24, %ymm18
+// CHECK:  encoding: [0x62,0xe2,0x3d,0x20,0x76,0x11]
+          vpermi2d (%rcx), %ymm24, %ymm18
+
+// CHECK: vpermi2d 291(%rax,%r14,8), %ymm24, %ymm18
+// CHECK:  encoding: [0x62,0xa2,0x3d,0x20,0x76,0x94,0xf0,0x23,0x01,0x00,0x00]
+          vpermi2d 291(%rax,%r14,8), %ymm24, %ymm18
+
+// CHECK: vpermi2d (%rcx){1to8}, %ymm24, %ymm18
+// CHECK:  encoding: [0x62,0xe2,0x3d,0x30,0x76,0x11]
+          vpermi2d (%rcx){1to8}, %ymm24, %ymm18
+
+// CHECK: vpermi2d 4064(%rdx), %ymm24, %ymm18
+// CHECK:  encoding: [0x62,0xe2,0x3d,0x20,0x76,0x52,0x7f]
+          vpermi2d 4064(%rdx), %ymm24, %ymm18
+
+// CHECK: vpermi2d 4096(%rdx), %ymm24, %ymm18
+// CHECK:  encoding: [0x62,0xe2,0x3d,0x20,0x76,0x92,0x00,0x10,0x00,0x00]
+          vpermi2d 4096(%rdx), %ymm24, %ymm18
+
+// CHECK: vpermi2d -4096(%rdx), %ymm24, %ymm18
+// CHECK:  encoding: [0x62,0xe2,0x3d,0x20,0x76,0x52,0x80]
+          vpermi2d -4096(%rdx), %ymm24, %ymm18
+
+// CHECK: vpermi2d -4128(%rdx), %ymm24, %ymm18
+// CHECK:  encoding: [0x62,0xe2,0x3d,0x20,0x76,0x92,0xe0,0xef,0xff,0xff]
+          vpermi2d -4128(%rdx), %ymm24, %ymm18
+
+// CHECK: vpermi2d 508(%rdx){1to8}, %ymm24, %ymm18
+// CHECK:  encoding: [0x62,0xe2,0x3d,0x30,0x76,0x52,0x7f]
+          vpermi2d 508(%rdx){1to8}, %ymm24, %ymm18
+
+// CHECK: vpermi2d 512(%rdx){1to8}, %ymm24, %ymm18
+// CHECK:  encoding: [0x62,0xe2,0x3d,0x30,0x76,0x92,0x00,0x02,0x00,0x00]
+          vpermi2d 512(%rdx){1to8}, %ymm24, %ymm18
+
+// CHECK: vpermi2d -512(%rdx){1to8}, %ymm24, %ymm18
+// CHECK:  encoding: [0x62,0xe2,0x3d,0x30,0x76,0x52,0x80]
+          vpermi2d -512(%rdx){1to8}, %ymm24, %ymm18
+
+// CHECK: vpermi2d -516(%rdx){1to8}, %ymm24, %ymm18
+// CHECK:  encoding: [0x62,0xe2,0x3d,0x30,0x76,0x92,0xfc,0xfd,0xff,0xff]
+          vpermi2d -516(%rdx){1to8}, %ymm24, %ymm18
+
+// CHECK: vpermi2q %xmm17, %xmm28, %xmm18
+// CHECK:  encoding: [0x62,0xa2,0x9d,0x00,0x76,0xd1]
+          vpermi2q %xmm17, %xmm28, %xmm18
+
+// CHECK: vpermi2q %xmm17, %xmm28, %xmm18 {%k3}
+// CHECK:  encoding: [0x62,0xa2,0x9d,0x03,0x76,0xd1]
+          vpermi2q %xmm17, %xmm28, %xmm18 {%k3}
+
+// CHECK: vpermi2q %xmm17, %xmm28, %xmm18 {%k3} {z}
+// CHECK:  encoding: [0x62,0xa2,0x9d,0x83,0x76,0xd1]
+          vpermi2q %xmm17, %xmm28, %xmm18 {%k3} {z}
+
+// CHECK: vpermi2q (%rcx), %xmm28, %xmm18
+// CHECK:  encoding: [0x62,0xe2,0x9d,0x00,0x76,0x11]
+          vpermi2q (%rcx), %xmm28, %xmm18
+
+// CHECK: vpermi2q 291(%rax,%r14,8), %xmm28, %xmm18
+// CHECK:  encoding: [0x62,0xa2,0x9d,0x00,0x76,0x94,0xf0,0x23,0x01,0x00,0x00]
+          vpermi2q 291(%rax,%r14,8), %xmm28, %xmm18
+
+// CHECK: vpermi2q (%rcx){1to2}, %xmm28, %xmm18
+// CHECK:  encoding: [0x62,0xe2,0x9d,0x10,0x76,0x11]
+          vpermi2q (%rcx){1to2}, %xmm28, %xmm18
+
+// CHECK: vpermi2q 2032(%rdx), %xmm28, %xmm18
+// CHECK:  encoding: [0x62,0xe2,0x9d,0x00,0x76,0x52,0x7f]
+          vpermi2q 2032(%rdx), %xmm28, %xmm18
+
+// CHECK: vpermi2q 2048(%rdx), %xmm28, %xmm18
+// CHECK:  encoding: [0x62,0xe2,0x9d,0x00,0x76,0x92,0x00,0x08,0x00,0x00]
+          vpermi2q 2048(%rdx), %xmm28, %xmm18
+
+// CHECK: vpermi2q -2048(%rdx), %xmm28, %xmm18
+// CHECK:  encoding: [0x62,0xe2,0x9d,0x00,0x76,0x52,0x80]
+          vpermi2q -2048(%rdx), %xmm28, %xmm18
+
+// CHECK: vpermi2q -2064(%rdx), %xmm28, %xmm18
+// CHECK:  encoding: [0x62,0xe2,0x9d,0x00,0x76,0x92,0xf0,0xf7,0xff,0xff]
+          vpermi2q -2064(%rdx), %xmm28, %xmm18
+
+// CHECK: vpermi2q 1016(%rdx){1to2}, %xmm28, %xmm18
+// CHECK:  encoding: [0x62,0xe2,0x9d,0x10,0x76,0x52,0x7f]
+          vpermi2q 1016(%rdx){1to2}, %xmm28, %xmm18
+
+// CHECK: vpermi2q 1024(%rdx){1to2}, %xmm28, %xmm18
+// CHECK:  encoding: [0x62,0xe2,0x9d,0x10,0x76,0x92,0x00,0x04,0x00,0x00]
+          vpermi2q 1024(%rdx){1to2}, %xmm28, %xmm18
+
+// CHECK: vpermi2q -1024(%rdx){1to2}, %xmm28, %xmm18
+// CHECK:  encoding: [0x62,0xe2,0x9d,0x10,0x76,0x52,0x80]
+          vpermi2q -1024(%rdx){1to2}, %xmm28, %xmm18
+
+// CHECK: vpermi2q -1032(%rdx){1to2}, %xmm28, %xmm18
+// CHECK:  encoding: [0x62,0xe2,0x9d,0x10,0x76,0x92,0xf8,0xfb,0xff,0xff]
+          vpermi2q -1032(%rdx){1to2}, %xmm28, %xmm18
+
+// CHECK: vpermi2q %ymm23, %ymm22, %ymm26
+// CHECK:  encoding: [0x62,0x22,0xcd,0x20,0x76,0xd7]
+          vpermi2q %ymm23, %ymm22, %ymm26
+
+// CHECK: vpermi2q %ymm23, %ymm22, %ymm26 {%k2}
+// CHECK:  encoding: [0x62,0x22,0xcd,0x22,0x76,0xd7]
+          vpermi2q %ymm23, %ymm22, %ymm26 {%k2}
+
+// CHECK: vpermi2q %ymm23, %ymm22, %ymm26 {%k2} {z}
+// CHECK:  encoding: [0x62,0x22,0xcd,0xa2,0x76,0xd7]
+          vpermi2q %ymm23, %ymm22, %ymm26 {%k2} {z}
+
+// CHECK: vpermi2q (%rcx), %ymm22, %ymm26
+// CHECK:  encoding: [0x62,0x62,0xcd,0x20,0x76,0x11]
+          vpermi2q (%rcx), %ymm22, %ymm26
+
+// CHECK: vpermi2q 291(%rax,%r14,8), %ymm22, %ymm26
+// CHECK:  encoding: [0x62,0x22,0xcd,0x20,0x76,0x94,0xf0,0x23,0x01,0x00,0x00]
+          vpermi2q 291(%rax,%r14,8), %ymm22, %ymm26
+
+// CHECK: vpermi2q (%rcx){1to4}, %ymm22, %ymm26
+// CHECK:  encoding: [0x62,0x62,0xcd,0x30,0x76,0x11]
+          vpermi2q (%rcx){1to4}, %ymm22, %ymm26
+
+// CHECK: vpermi2q 4064(%rdx), %ymm22, %ymm26
+// CHECK:  encoding: [0x62,0x62,0xcd,0x20,0x76,0x52,0x7f]
+          vpermi2q 4064(%rdx), %ymm22, %ymm26
+
+// CHECK: vpermi2q 4096(%rdx), %ymm22, %ymm26
+// CHECK:  encoding: [0x62,0x62,0xcd,0x20,0x76,0x92,0x00,0x10,0x00,0x00]
+          vpermi2q 4096(%rdx), %ymm22, %ymm26
+
+// CHECK: vpermi2q -4096(%rdx), %ymm22, %ymm26
+// CHECK:  encoding: [0x62,0x62,0xcd,0x20,0x76,0x52,0x80]
+          vpermi2q -4096(%rdx), %ymm22, %ymm26
+
+// CHECK: vpermi2q -4128(%rdx), %ymm22, %ymm26
+// CHECK:  encoding: [0x62,0x62,0xcd,0x20,0x76,0x92,0xe0,0xef,0xff,0xff]
+          vpermi2q -4128(%rdx), %ymm22, %ymm26
+
+// CHECK: vpermi2q 1016(%rdx){1to4}, %ymm22, %ymm26
+// CHECK:  encoding: [0x62,0x62,0xcd,0x30,0x76,0x52,0x7f]
+          vpermi2q 1016(%rdx){1to4}, %ymm22, %ymm26
+
+// CHECK: vpermi2q 1024(%rdx){1to4}, %ymm22, %ymm26
+// CHECK:  encoding: [0x62,0x62,0xcd,0x30,0x76,0x92,0x00,0x04,0x00,0x00]
+          vpermi2q 1024(%rdx){1to4}, %ymm22, %ymm26
+
+// CHECK: vpermi2q -1024(%rdx){1to4}, %ymm22, %ymm26
+// CHECK:  encoding: [0x62,0x62,0xcd,0x30,0x76,0x52,0x80]
+          vpermi2q -1024(%rdx){1to4}, %ymm22, %ymm26
+
+// CHECK: vpermi2q -1032(%rdx){1to4}, %ymm22, %ymm26
+// CHECK:  encoding: [0x62,0x62,0xcd,0x30,0x76,0x92,0xf8,0xfb,0xff,0xff]
+          vpermi2q -1032(%rdx){1to4}, %ymm22, %ymm26
+
+// CHECK: vpermi2ps %xmm23, %xmm24, %xmm23
+// CHECK:  encoding: [0x62,0xa2,0x3d,0x00,0x77,0xff]
+          vpermi2ps %xmm23, %xmm24, %xmm23
+
+// CHECK: vpermi2ps %xmm23, %xmm24, %xmm23 {%k3}
+// CHECK:  encoding: [0x62,0xa2,0x3d,0x03,0x77,0xff]
+          vpermi2ps %xmm23, %xmm24, %xmm23 {%k3}
+
+// CHECK: vpermi2ps %xmm23, %xmm24, %xmm23 {%k3} {z}
+// CHECK:  encoding: [0x62,0xa2,0x3d,0x83,0x77,0xff]
+          vpermi2ps %xmm23, %xmm24, %xmm23 {%k3} {z}
+
+// CHECK: vpermi2ps (%rcx), %xmm24, %xmm23
+// CHECK:  encoding: [0x62,0xe2,0x3d,0x00,0x77,0x39]
+          vpermi2ps (%rcx), %xmm24, %xmm23
+
+// CHECK: vpermi2ps 291(%rax,%r14,8), %xmm24, %xmm23
+// CHECK:  encoding: [0x62,0xa2,0x3d,0x00,0x77,0xbc,0xf0,0x23,0x01,0x00,0x00]
+          vpermi2ps 291(%rax,%r14,8), %xmm24, %xmm23
+
+// CHECK: vpermi2ps (%rcx){1to4}, %xmm24, %xmm23
+// CHECK:  encoding: [0x62,0xe2,0x3d,0x10,0x77,0x39]
+          vpermi2ps (%rcx){1to4}, %xmm24, %xmm23
+
+// CHECK: vpermi2ps 2032(%rdx), %xmm24, %xmm23
+// CHECK:  encoding: [0x62,0xe2,0x3d,0x00,0x77,0x7a,0x7f]
+          vpermi2ps 2032(%rdx), %xmm24, %xmm23
+
+// CHECK: vpermi2ps 2048(%rdx), %xmm24, %xmm23
+// CHECK:  encoding: [0x62,0xe2,0x3d,0x00,0x77,0xba,0x00,0x08,0x00,0x00]
+          vpermi2ps 2048(%rdx), %xmm24, %xmm23
+
+// CHECK: vpermi2ps -2048(%rdx), %xmm24, %xmm23
+// CHECK:  encoding: [0x62,0xe2,0x3d,0x00,0x77,0x7a,0x80]
+          vpermi2ps -2048(%rdx), %xmm24, %xmm23
+
+// CHECK: vpermi2ps -2064(%rdx), %xmm24, %xmm23
+// CHECK:  encoding: [0x62,0xe2,0x3d,0x00,0x77,0xba,0xf0,0xf7,0xff,0xff]
+          vpermi2ps -2064(%rdx), %xmm24, %xmm23
+
+// CHECK: vpermi2ps 508(%rdx){1to4}, %xmm24, %xmm23
+// CHECK:  encoding: [0x62,0xe2,0x3d,0x10,0x77,0x7a,0x7f]
+          vpermi2ps 508(%rdx){1to4}, %xmm24, %xmm23
+
+// CHECK: vpermi2ps 512(%rdx){1to4}, %xmm24, %xmm23
+// CHECK:  encoding: [0x62,0xe2,0x3d,0x10,0x77,0xba,0x00,0x02,0x00,0x00]
+          vpermi2ps 512(%rdx){1to4}, %xmm24, %xmm23
+
+// CHECK: vpermi2ps -512(%rdx){1to4}, %xmm24, %xmm23
+// CHECK:  encoding: [0x62,0xe2,0x3d,0x10,0x77,0x7a,0x80]
+          vpermi2ps -512(%rdx){1to4}, %xmm24, %xmm23
+
+// CHECK: vpermi2ps -516(%rdx){1to4}, %xmm24, %xmm23
+// CHECK:  encoding: [0x62,0xe2,0x3d,0x10,0x77,0xba,0xfc,0xfd,0xff,0xff]
+          vpermi2ps -516(%rdx){1to4}, %xmm24, %xmm23
+
+// CHECK: vpermi2ps %ymm20, %ymm24, %ymm18
+// CHECK:  encoding: [0x62,0xa2,0x3d,0x20,0x77,0xd4]
+          vpermi2ps %ymm20, %ymm24, %ymm18
+
+// CHECK: vpermi2ps %ymm20, %ymm24, %ymm18 {%k5}
+// CHECK:  encoding: [0x62,0xa2,0x3d,0x25,0x77,0xd4]
+          vpermi2ps %ymm20, %ymm24, %ymm18 {%k5}
+
+// CHECK: vpermi2ps %ymm20, %ymm24, %ymm18 {%k5} {z}
+// CHECK:  encoding: [0x62,0xa2,0x3d,0xa5,0x77,0xd4]
+          vpermi2ps %ymm20, %ymm24, %ymm18 {%k5} {z}
+
+// CHECK: vpermi2ps (%rcx), %ymm24, %ymm18
+// CHECK:  encoding: [0x62,0xe2,0x3d,0x20,0x77,0x11]
+          vpermi2ps (%rcx), %ymm24, %ymm18
+
+// CHECK: vpermi2ps 291(%rax,%r14,8), %ymm24, %ymm18
+// CHECK:  encoding: [0x62,0xa2,0x3d,0x20,0x77,0x94,0xf0,0x23,0x01,0x00,0x00]
+          vpermi2ps 291(%rax,%r14,8), %ymm24, %ymm18
+
+// CHECK: vpermi2ps (%rcx){1to8}, %ymm24, %ymm18
+// CHECK:  encoding: [0x62,0xe2,0x3d,0x30,0x77,0x11]
+          vpermi2ps (%rcx){1to8}, %ymm24, %ymm18
+
+// CHECK: vpermi2ps 4064(%rdx), %ymm24, %ymm18
+// CHECK:  encoding: [0x62,0xe2,0x3d,0x20,0x77,0x52,0x7f]
+          vpermi2ps 4064(%rdx), %ymm24, %ymm18
+
+// CHECK: vpermi2ps 4096(%rdx), %ymm24, %ymm18
+// CHECK:  encoding: [0x62,0xe2,0x3d,0x20,0x77,0x92,0x00,0x10,0x00,0x00]
+          vpermi2ps 4096(%rdx), %ymm24, %ymm18
+
+// CHECK: vpermi2ps -4096(%rdx), %ymm24, %ymm18
+// CHECK:  encoding: [0x62,0xe2,0x3d,0x20,0x77,0x52,0x80]
+          vpermi2ps -4096(%rdx), %ymm24, %ymm18
+
+// CHECK: vpermi2ps -4128(%rdx), %ymm24, %ymm18
+// CHECK:  encoding: [0x62,0xe2,0x3d,0x20,0x77,0x92,0xe0,0xef,0xff,0xff]
+          vpermi2ps -4128(%rdx), %ymm24, %ymm18
+
+// CHECK: vpermi2ps 508(%rdx){1to8}, %ymm24, %ymm18
+// CHECK:  encoding: [0x62,0xe2,0x3d,0x30,0x77,0x52,0x7f]
+          vpermi2ps 508(%rdx){1to8}, %ymm24, %ymm18
+
+// CHECK: vpermi2ps 512(%rdx){1to8}, %ymm24, %ymm18
+// CHECK:  encoding: [0x62,0xe2,0x3d,0x30,0x77,0x92,0x00,0x02,0x00,0x00]
+          vpermi2ps 512(%rdx){1to8}, %ymm24, %ymm18
+
+// CHECK: vpermi2ps -512(%rdx){1to8}, %ymm24, %ymm18
+// CHECK:  encoding: [0x62,0xe2,0x3d,0x30,0x77,0x52,0x80]
+          vpermi2ps -512(%rdx){1to8}, %ymm24, %ymm18
+
+// CHECK: vpermi2ps -516(%rdx){1to8}, %ymm24, %ymm18
+// CHECK:  encoding: [0x62,0xe2,0x3d,0x30,0x77,0x92,0xfc,0xfd,0xff,0xff]
+          vpermi2ps -516(%rdx){1to8}, %ymm24, %ymm18
+
+// CHECK: vpermi2pd %xmm27, %xmm17, %xmm28
+// CHECK:  encoding: [0x62,0x02,0xf5,0x00,0x77,0xe3]
+          vpermi2pd %xmm27, %xmm17, %xmm28
+
+// CHECK: vpermi2pd %xmm27, %xmm17, %xmm28 {%k4}
+// CHECK:  encoding: [0x62,0x02,0xf5,0x04,0x77,0xe3]
+          vpermi2pd %xmm27, %xmm17, %xmm28 {%k4}
+
+// CHECK: vpermi2pd %xmm27, %xmm17, %xmm28 {%k4} {z}
+// CHECK:  encoding: [0x62,0x02,0xf5,0x84,0x77,0xe3]
+          vpermi2pd %xmm27, %xmm17, %xmm28 {%k4} {z}
+
+// CHECK: vpermi2pd (%rcx), %xmm17, %xmm28
+// CHECK:  encoding: [0x62,0x62,0xf5,0x00,0x77,0x21]
+          vpermi2pd (%rcx), %xmm17, %xmm28
+
+// CHECK: vpermi2pd 291(%rax,%r14,8), %xmm17, %xmm28
+// CHECK:  encoding: [0x62,0x22,0xf5,0x00,0x77,0xa4,0xf0,0x23,0x01,0x00,0x00]
+          vpermi2pd 291(%rax,%r14,8), %xmm17, %xmm28
+
+// CHECK: vpermi2pd (%rcx){1to2}, %xmm17, %xmm28
+// CHECK:  encoding: [0x62,0x62,0xf5,0x10,0x77,0x21]
+          vpermi2pd (%rcx){1to2}, %xmm17, %xmm28
+
+// CHECK: vpermi2pd 2032(%rdx), %xmm17, %xmm28
+// CHECK:  encoding: [0x62,0x62,0xf5,0x00,0x77,0x62,0x7f]
+          vpermi2pd 2032(%rdx), %xmm17, %xmm28
+
+// CHECK: vpermi2pd 2048(%rdx), %xmm17, %xmm28
+// CHECK:  encoding: [0x62,0x62,0xf5,0x00,0x77,0xa2,0x00,0x08,0x00,0x00]
+          vpermi2pd 2048(%rdx), %xmm17, %xmm28
+
+// CHECK: vpermi2pd -2048(%rdx), %xmm17, %xmm28
+// CHECK:  encoding: [0x62,0x62,0xf5,0x00,0x77,0x62,0x80]
+          vpermi2pd -2048(%rdx), %xmm17, %xmm28
+
+// CHECK: vpermi2pd -2064(%rdx), %xmm17, %xmm28
+// CHECK:  encoding: [0x62,0x62,0xf5,0x00,0x77,0xa2,0xf0,0xf7,0xff,0xff]
+          vpermi2pd -2064(%rdx), %xmm17, %xmm28
+
+// CHECK: vpermi2pd 1016(%rdx){1to2}, %xmm17, %xmm28
+// CHECK:  encoding: [0x62,0x62,0xf5,0x10,0x77,0x62,0x7f]
+          vpermi2pd 1016(%rdx){1to2}, %xmm17, %xmm28
+
+// CHECK: vpermi2pd 1024(%rdx){1to2}, %xmm17, %xmm28
+// CHECK:  encoding: [0x62,0x62,0xf5,0x10,0x77,0xa2,0x00,0x04,0x00,0x00]
+          vpermi2pd 1024(%rdx){1to2}, %xmm17, %xmm28
+
+// CHECK: vpermi2pd -1024(%rdx){1to2}, %xmm17, %xmm28
+// CHECK:  encoding: [0x62,0x62,0xf5,0x10,0x77,0x62,0x80]
+          vpermi2pd -1024(%rdx){1to2}, %xmm17, %xmm28
+
+// CHECK: vpermi2pd -1032(%rdx){1to2}, %xmm17, %xmm28
+// CHECK:  encoding: [0x62,0x62,0xf5,0x10,0x77,0xa2,0xf8,0xfb,0xff,0xff]
+          vpermi2pd -1032(%rdx){1to2}, %xmm17, %xmm28
+
+// CHECK: vpermi2pd %ymm27, %ymm19, %ymm30
+// CHECK:  encoding: [0x62,0x02,0xe5,0x20,0x77,0xf3]
+          vpermi2pd %ymm27, %ymm19, %ymm30
+
+// CHECK: vpermi2pd %ymm27, %ymm19, %ymm30 {%k3}
+// CHECK:  encoding: [0x62,0x02,0xe5,0x23,0x77,0xf3]
+          vpermi2pd %ymm27, %ymm19, %ymm30 {%k3}
+
+// CHECK: vpermi2pd %ymm27, %ymm19, %ymm30 {%k3} {z}
+// CHECK:  encoding: [0x62,0x02,0xe5,0xa3,0x77,0xf3]
+          vpermi2pd %ymm27, %ymm19, %ymm30 {%k3} {z}
+
+// CHECK: vpermi2pd (%rcx), %ymm19, %ymm30
+// CHECK:  encoding: [0x62,0x62,0xe5,0x20,0x77,0x31]
+          vpermi2pd (%rcx), %ymm19, %ymm30
+
+// CHECK: vpermi2pd 291(%rax,%r14,8), %ymm19, %ymm30
+// CHECK:  encoding: [0x62,0x22,0xe5,0x20,0x77,0xb4,0xf0,0x23,0x01,0x00,0x00]
+          vpermi2pd 291(%rax,%r14,8), %ymm19, %ymm30
+
+// CHECK: vpermi2pd (%rcx){1to4}, %ymm19, %ymm30
+// CHECK:  encoding: [0x62,0x62,0xe5,0x30,0x77,0x31]
+          vpermi2pd (%rcx){1to4}, %ymm19, %ymm30
+
+// CHECK: vpermi2pd 4064(%rdx), %ymm19, %ymm30
+// CHECK:  encoding: [0x62,0x62,0xe5,0x20,0x77,0x72,0x7f]
+          vpermi2pd 4064(%rdx), %ymm19, %ymm30
+
+// CHECK: vpermi2pd 4096(%rdx), %ymm19, %ymm30
+// CHECK:  encoding: [0x62,0x62,0xe5,0x20,0x77,0xb2,0x00,0x10,0x00,0x00]
+          vpermi2pd 4096(%rdx), %ymm19, %ymm30
+
+// CHECK: vpermi2pd -4096(%rdx), %ymm19, %ymm30
+// CHECK:  encoding: [0x62,0x62,0xe5,0x20,0x77,0x72,0x80]
+          vpermi2pd -4096(%rdx), %ymm19, %ymm30
+
+// CHECK: vpermi2pd -4128(%rdx), %ymm19, %ymm30
+// CHECK:  encoding: [0x62,0x62,0xe5,0x20,0x77,0xb2,0xe0,0xef,0xff,0xff]
+          vpermi2pd -4128(%rdx), %ymm19, %ymm30
+
+// CHECK: vpermi2pd 1016(%rdx){1to4}, %ymm19, %ymm30
+// CHECK:  encoding: [0x62,0x62,0xe5,0x30,0x77,0x72,0x7f]
+          vpermi2pd 1016(%rdx){1to4}, %ymm19, %ymm30
+
+// CHECK: vpermi2pd 1024(%rdx){1to4}, %ymm19, %ymm30
+// CHECK:  encoding: [0x62,0x62,0xe5,0x30,0x77,0xb2,0x00,0x04,0x00,0x00]
+          vpermi2pd 1024(%rdx){1to4}, %ymm19, %ymm30
+
+// CHECK: vpermi2pd -1024(%rdx){1to4}, %ymm19, %ymm30
+// CHECK:  encoding: [0x62,0x62,0xe5,0x30,0x77,0x72,0x80]
+          vpermi2pd -1024(%rdx){1to4}, %ymm19, %ymm30
+
+// CHECK: vpermi2pd -1032(%rdx){1to4}, %ymm19, %ymm30
+// CHECK:  encoding: [0x62,0x62,0xe5,0x30,0x77,0xb2,0xf8,0xfb,0xff,0xff]
+          vpermi2pd -1032(%rdx){1to4}, %ymm19, %ymm30
+
+// CHECK: vpermt2d %xmm23, %xmm29, %xmm21
+// CHECK:  encoding: [0x62,0xa2,0x15,0x00,0x7e,0xef]
+          vpermt2d %xmm23, %xmm29, %xmm21
+
+// CHECK: vpermt2d %xmm23, %xmm29, %xmm21 {%k4}
+// CHECK:  encoding: [0x62,0xa2,0x15,0x04,0x7e,0xef]
+          vpermt2d %xmm23, %xmm29, %xmm21 {%k4}
+
+// CHECK: vpermt2d %xmm23, %xmm29, %xmm21 {%k4} {z}
+// CHECK:  encoding: [0x62,0xa2,0x15,0x84,0x7e,0xef]
+          vpermt2d %xmm23, %xmm29, %xmm21 {%k4} {z}
+
+// CHECK: vpermt2d (%rcx), %xmm29, %xmm21
+// CHECK:  encoding: [0x62,0xe2,0x15,0x00,0x7e,0x29]
+          vpermt2d (%rcx), %xmm29, %xmm21
+
+// CHECK: vpermt2d 291(%rax,%r14,8), %xmm29, %xmm21
+// CHECK:  encoding: [0x62,0xa2,0x15,0x00,0x7e,0xac,0xf0,0x23,0x01,0x00,0x00]
+          vpermt2d 291(%rax,%r14,8), %xmm29, %xmm21
+
+// CHECK: vpermt2d (%rcx){1to4}, %xmm29, %xmm21
+// CHECK:  encoding: [0x62,0xe2,0x15,0x10,0x7e,0x29]
+          vpermt2d (%rcx){1to4}, %xmm29, %xmm21
+
+// CHECK: vpermt2d 2032(%rdx), %xmm29, %xmm21
+// CHECK:  encoding: [0x62,0xe2,0x15,0x00,0x7e,0x6a,0x7f]
+          vpermt2d 2032(%rdx), %xmm29, %xmm21
+
+// CHECK: vpermt2d 2048(%rdx), %xmm29, %xmm21
+// CHECK:  encoding: [0x62,0xe2,0x15,0x00,0x7e,0xaa,0x00,0x08,0x00,0x00]
+          vpermt2d 2048(%rdx), %xmm29, %xmm21
+
+// CHECK: vpermt2d -2048(%rdx), %xmm29, %xmm21
+// CHECK:  encoding: [0x62,0xe2,0x15,0x00,0x7e,0x6a,0x80]
+          vpermt2d -2048(%rdx), %xmm29, %xmm21
+
+// CHECK: vpermt2d -2064(%rdx), %xmm29, %xmm21
+// CHECK:  encoding: [0x62,0xe2,0x15,0x00,0x7e,0xaa,0xf0,0xf7,0xff,0xff]
+          vpermt2d -2064(%rdx), %xmm29, %xmm21
+
+// CHECK: vpermt2d 508(%rdx){1to4}, %xmm29, %xmm21
+// CHECK:  encoding: [0x62,0xe2,0x15,0x10,0x7e,0x6a,0x7f]
+          vpermt2d 508(%rdx){1to4}, %xmm29, %xmm21
+
+// CHECK: vpermt2d 512(%rdx){1to4}, %xmm29, %xmm21
+// CHECK:  encoding: [0x62,0xe2,0x15,0x10,0x7e,0xaa,0x00,0x02,0x00,0x00]
+          vpermt2d 512(%rdx){1to4}, %xmm29, %xmm21
+
+// CHECK: vpermt2d -512(%rdx){1to4}, %xmm29, %xmm21
+// CHECK:  encoding: [0x62,0xe2,0x15,0x10,0x7e,0x6a,0x80]
+          vpermt2d -512(%rdx){1to4}, %xmm29, %xmm21
+
+// CHECK: vpermt2d -516(%rdx){1to4}, %xmm29, %xmm21
+// CHECK:  encoding: [0x62,0xe2,0x15,0x10,0x7e,0xaa,0xfc,0xfd,0xff,0xff]
+          vpermt2d -516(%rdx){1to4}, %xmm29, %xmm21
+
+// CHECK: vpermt2d %ymm21, %ymm26, %ymm22
+// CHECK:  encoding: [0x62,0xa2,0x2d,0x20,0x7e,0xf5]
+          vpermt2d %ymm21, %ymm26, %ymm22
+
+// CHECK: vpermt2d %ymm21, %ymm26, %ymm22 {%k2}
+// CHECK:  encoding: [0x62,0xa2,0x2d,0x22,0x7e,0xf5]
+          vpermt2d %ymm21, %ymm26, %ymm22 {%k2}
+
+// CHECK: vpermt2d %ymm21, %ymm26, %ymm22 {%k2} {z}
+// CHECK:  encoding: [0x62,0xa2,0x2d,0xa2,0x7e,0xf5]
+          vpermt2d %ymm21, %ymm26, %ymm22 {%k2} {z}
+
+// CHECK: vpermt2d (%rcx), %ymm26, %ymm22
+// CHECK:  encoding: [0x62,0xe2,0x2d,0x20,0x7e,0x31]
+          vpermt2d (%rcx), %ymm26, %ymm22
+
+// CHECK: vpermt2d 291(%rax,%r14,8), %ymm26, %ymm22
+// CHECK:  encoding: [0x62,0xa2,0x2d,0x20,0x7e,0xb4,0xf0,0x23,0x01,0x00,0x00]
+          vpermt2d 291(%rax,%r14,8), %ymm26, %ymm22
+
+// CHECK: vpermt2d (%rcx){1to8}, %ymm26, %ymm22
+// CHECK:  encoding: [0x62,0xe2,0x2d,0x30,0x7e,0x31]
+          vpermt2d (%rcx){1to8}, %ymm26, %ymm22
+
+// CHECK: vpermt2d 4064(%rdx), %ymm26, %ymm22
+// CHECK:  encoding: [0x62,0xe2,0x2d,0x20,0x7e,0x72,0x7f]
+          vpermt2d 4064(%rdx), %ymm26, %ymm22
+
+// CHECK: vpermt2d 4096(%rdx), %ymm26, %ymm22
+// CHECK:  encoding: [0x62,0xe2,0x2d,0x20,0x7e,0xb2,0x00,0x10,0x00,0x00]
+          vpermt2d 4096(%rdx), %ymm26, %ymm22
+
+// CHECK: vpermt2d -4096(%rdx), %ymm26, %ymm22
+// CHECK:  encoding: [0x62,0xe2,0x2d,0x20,0x7e,0x72,0x80]
+          vpermt2d -4096(%rdx), %ymm26, %ymm22
+
+// CHECK: vpermt2d -4128(%rdx), %ymm26, %ymm22
+// CHECK:  encoding: [0x62,0xe2,0x2d,0x20,0x7e,0xb2,0xe0,0xef,0xff,0xff]
+          vpermt2d -4128(%rdx), %ymm26, %ymm22
+
+// CHECK: vpermt2d 508(%rdx){1to8}, %ymm26, %ymm22
+// CHECK:  encoding: [0x62,0xe2,0x2d,0x30,0x7e,0x72,0x7f]
+          vpermt2d 508(%rdx){1to8}, %ymm26, %ymm22
+
+// CHECK: vpermt2d 512(%rdx){1to8}, %ymm26, %ymm22
+// CHECK:  encoding: [0x62,0xe2,0x2d,0x30,0x7e,0xb2,0x00,0x02,0x00,0x00]
+          vpermt2d 512(%rdx){1to8}, %ymm26, %ymm22
+
+// CHECK: vpermt2d -512(%rdx){1to8}, %ymm26, %ymm22
+// CHECK:  encoding: [0x62,0xe2,0x2d,0x30,0x7e,0x72,0x80]
+          vpermt2d -512(%rdx){1to8}, %ymm26, %ymm22
+
+// CHECK: vpermt2d -516(%rdx){1to8}, %ymm26, %ymm22
+// CHECK:  encoding: [0x62,0xe2,0x2d,0x30,0x7e,0xb2,0xfc,0xfd,0xff,0xff]
+          vpermt2d -516(%rdx){1to8}, %ymm26, %ymm22
+
+// CHECK: vpermt2q %xmm18, %xmm25, %xmm22
+// CHECK:  encoding: [0x62,0xa2,0xb5,0x00,0x7e,0xf2]
+          vpermt2q %xmm18, %xmm25, %xmm22
+
+// CHECK: vpermt2q %xmm18, %xmm25, %xmm22 {%k1}
+// CHECK:  encoding: [0x62,0xa2,0xb5,0x01,0x7e,0xf2]
+          vpermt2q %xmm18, %xmm25, %xmm22 {%k1}
+
+// CHECK: vpermt2q %xmm18, %xmm25, %xmm22 {%k1} {z}
+// CHECK:  encoding: [0x62,0xa2,0xb5,0x81,0x7e,0xf2]
+          vpermt2q %xmm18, %xmm25, %xmm22 {%k1} {z}
+
+// CHECK: vpermt2q (%rcx), %xmm25, %xmm22
+// CHECK:  encoding: [0x62,0xe2,0xb5,0x00,0x7e,0x31]
+          vpermt2q (%rcx), %xmm25, %xmm22
+
+// CHECK: vpermt2q 291(%rax,%r14,8), %xmm25, %xmm22
+// CHECK:  encoding: [0x62,0xa2,0xb5,0x00,0x7e,0xb4,0xf0,0x23,0x01,0x00,0x00]
+          vpermt2q 291(%rax,%r14,8), %xmm25, %xmm22
+
+// CHECK: vpermt2q (%rcx){1to2}, %xmm25, %xmm22
+// CHECK:  encoding: [0x62,0xe2,0xb5,0x10,0x7e,0x31]
+          vpermt2q (%rcx){1to2}, %xmm25, %xmm22
+
+// CHECK: vpermt2q 2032(%rdx), %xmm25, %xmm22
+// CHECK:  encoding: [0x62,0xe2,0xb5,0x00,0x7e,0x72,0x7f]
+          vpermt2q 2032(%rdx), %xmm25, %xmm22
+
+// CHECK: vpermt2q 2048(%rdx), %xmm25, %xmm22
+// CHECK:  encoding: [0x62,0xe2,0xb5,0x00,0x7e,0xb2,0x00,0x08,0x00,0x00]
+          vpermt2q 2048(%rdx), %xmm25, %xmm22
+
+// CHECK: vpermt2q -2048(%rdx), %xmm25, %xmm22
+// CHECK:  encoding: [0x62,0xe2,0xb5,0x00,0x7e,0x72,0x80]
+          vpermt2q -2048(%rdx), %xmm25, %xmm22
+
+// CHECK: vpermt2q -2064(%rdx), %xmm25, %xmm22
+// CHECK:  encoding: [0x62,0xe2,0xb5,0x00,0x7e,0xb2,0xf0,0xf7,0xff,0xff]
+          vpermt2q -2064(%rdx), %xmm25, %xmm22
+
+// CHECK: vpermt2q 1016(%rdx){1to2}, %xmm25, %xmm22
+// CHECK:  encoding: [0x62,0xe2,0xb5,0x10,0x7e,0x72,0x7f]
+          vpermt2q 1016(%rdx){1to2}, %xmm25, %xmm22
+
+// CHECK: vpermt2q 1024(%rdx){1to2}, %xmm25, %xmm22
+// CHECK:  encoding: [0x62,0xe2,0xb5,0x10,0x7e,0xb2,0x00,0x04,0x00,0x00]
+          vpermt2q 1024(%rdx){1to2}, %xmm25, %xmm22
+
+// CHECK: vpermt2q -1024(%rdx){1to2}, %xmm25, %xmm22
+// CHECK:  encoding: [0x62,0xe2,0xb5,0x10,0x7e,0x72,0x80]
+          vpermt2q -1024(%rdx){1to2}, %xmm25, %xmm22
+
+// CHECK: vpermt2q -1032(%rdx){1to2}, %xmm25, %xmm22
+// CHECK:  encoding: [0x62,0xe2,0xb5,0x10,0x7e,0xb2,0xf8,0xfb,0xff,0xff]
+          vpermt2q -1032(%rdx){1to2}, %xmm25, %xmm22
+
+// CHECK: vpermt2q %ymm20, %ymm17, %ymm18
+// CHECK:  encoding: [0x62,0xa2,0xf5,0x20,0x7e,0xd4]
+          vpermt2q %ymm20, %ymm17, %ymm18
+
+// CHECK: vpermt2q %ymm20, %ymm17, %ymm18 {%k6}
+// CHECK:  encoding: [0x62,0xa2,0xf5,0x26,0x7e,0xd4]
+          vpermt2q %ymm20, %ymm17, %ymm18 {%k6}
+
+// CHECK: vpermt2q %ymm20, %ymm17, %ymm18 {%k6} {z}
+// CHECK:  encoding: [0x62,0xa2,0xf5,0xa6,0x7e,0xd4]
+          vpermt2q %ymm20, %ymm17, %ymm18 {%k6} {z}
+
+// CHECK: vpermt2q (%rcx), %ymm17, %ymm18
+// CHECK:  encoding: [0x62,0xe2,0xf5,0x20,0x7e,0x11]
+          vpermt2q (%rcx), %ymm17, %ymm18
+
+// CHECK: vpermt2q 291(%rax,%r14,8), %ymm17, %ymm18
+// CHECK:  encoding: [0x62,0xa2,0xf5,0x20,0x7e,0x94,0xf0,0x23,0x01,0x00,0x00]
+          vpermt2q 291(%rax,%r14,8), %ymm17, %ymm18
+
+// CHECK: vpermt2q (%rcx){1to4}, %ymm17, %ymm18
+// CHECK:  encoding: [0x62,0xe2,0xf5,0x30,0x7e,0x11]
+          vpermt2q (%rcx){1to4}, %ymm17, %ymm18
+
+// CHECK: vpermt2q 4064(%rdx), %ymm17, %ymm18
+// CHECK:  encoding: [0x62,0xe2,0xf5,0x20,0x7e,0x52,0x7f]
+          vpermt2q 4064(%rdx), %ymm17, %ymm18
+
+// CHECK: vpermt2q 4096(%rdx), %ymm17, %ymm18
+// CHECK:  encoding: [0x62,0xe2,0xf5,0x20,0x7e,0x92,0x00,0x10,0x00,0x00]
+          vpermt2q 4096(%rdx), %ymm17, %ymm18
+
+// CHECK: vpermt2q -4096(%rdx), %ymm17, %ymm18
+// CHECK:  encoding: [0x62,0xe2,0xf5,0x20,0x7e,0x52,0x80]
+          vpermt2q -4096(%rdx), %ymm17, %ymm18
+
+// CHECK: vpermt2q -4128(%rdx), %ymm17, %ymm18
+// CHECK:  encoding: [0x62,0xe2,0xf5,0x20,0x7e,0x92,0xe0,0xef,0xff,0xff]
+          vpermt2q -4128(%rdx), %ymm17, %ymm18
+
+// CHECK: vpermt2q 1016(%rdx){1to4}, %ymm17, %ymm18
+// CHECK:  encoding: [0x62,0xe2,0xf5,0x30,0x7e,0x52,0x7f]
+          vpermt2q 1016(%rdx){1to4}, %ymm17, %ymm18
+
+// CHECK: vpermt2q 1024(%rdx){1to4}, %ymm17, %ymm18
+// CHECK:  encoding: [0x62,0xe2,0xf5,0x30,0x7e,0x92,0x00,0x04,0x00,0x00]
+          vpermt2q 1024(%rdx){1to4}, %ymm17, %ymm18
+
+// CHECK: vpermt2q -1024(%rdx){1to4}, %ymm17, %ymm18
+// CHECK:  encoding: [0x62,0xe2,0xf5,0x30,0x7e,0x52,0x80]
+          vpermt2q -1024(%rdx){1to4}, %ymm17, %ymm18
+
+// CHECK: vpermt2q -1032(%rdx){1to4}, %ymm17, %ymm18
+// CHECK:  encoding: [0x62,0xe2,0xf5,0x30,0x7e,0x92,0xf8,0xfb,0xff,0xff]
+          vpermt2q -1032(%rdx){1to4}, %ymm17, %ymm18
+
+// CHECK: vpermt2ps %xmm18, %xmm23, %xmm19
+// CHECK:  encoding: [0x62,0xa2,0x45,0x00,0x7f,0xda]
+          vpermt2ps %xmm18, %xmm23, %xmm19
+
+// CHECK: vpermt2ps %xmm18, %xmm23, %xmm19 {%k1}
+// CHECK:  encoding: [0x62,0xa2,0x45,0x01,0x7f,0xda]
+          vpermt2ps %xmm18, %xmm23, %xmm19 {%k1}
+
+// CHECK: vpermt2ps %xmm18, %xmm23, %xmm19 {%k1} {z}
+// CHECK:  encoding: [0x62,0xa2,0x45,0x81,0x7f,0xda]
+          vpermt2ps %xmm18, %xmm23, %xmm19 {%k1} {z}
+
+// CHECK: vpermt2ps (%rcx), %xmm23, %xmm19
+// CHECK:  encoding: [0x62,0xe2,0x45,0x00,0x7f,0x19]
+          vpermt2ps (%rcx), %xmm23, %xmm19
+
+// CHECK: vpermt2ps 291(%rax,%r14,8), %xmm23, %xmm19
+// CHECK:  encoding: [0x62,0xa2,0x45,0x00,0x7f,0x9c,0xf0,0x23,0x01,0x00,0x00]
+          vpermt2ps 291(%rax,%r14,8), %xmm23, %xmm19
+
+// CHECK: vpermt2ps (%rcx){1to4}, %xmm23, %xmm19
+// CHECK:  encoding: [0x62,0xe2,0x45,0x10,0x7f,0x19]
+          vpermt2ps (%rcx){1to4}, %xmm23, %xmm19
+
+// CHECK: vpermt2ps 2032(%rdx), %xmm23, %xmm19
+// CHECK:  encoding: [0x62,0xe2,0x45,0x00,0x7f,0x5a,0x7f]
+          vpermt2ps 2032(%rdx), %xmm23, %xmm19
+
+// CHECK: vpermt2ps 2048(%rdx), %xmm23, %xmm19
+// CHECK:  encoding: [0x62,0xe2,0x45,0x00,0x7f,0x9a,0x00,0x08,0x00,0x00]
+          vpermt2ps 2048(%rdx), %xmm23, %xmm19
+
+// CHECK: vpermt2ps -2048(%rdx), %xmm23, %xmm19
+// CHECK:  encoding: [0x62,0xe2,0x45,0x00,0x7f,0x5a,0x80]
+          vpermt2ps -2048(%rdx), %xmm23, %xmm19
+
+// CHECK: vpermt2ps -2064(%rdx), %xmm23, %xmm19
+// CHECK:  encoding: [0x62,0xe2,0x45,0x00,0x7f,0x9a,0xf0,0xf7,0xff,0xff]
+          vpermt2ps -2064(%rdx), %xmm23, %xmm19
+
+// CHECK: vpermt2ps 508(%rdx){1to4}, %xmm23, %xmm19
+// CHECK:  encoding: [0x62,0xe2,0x45,0x10,0x7f,0x5a,0x7f]
+          vpermt2ps 508(%rdx){1to4}, %xmm23, %xmm19
+
+// CHECK: vpermt2ps 512(%rdx){1to4}, %xmm23, %xmm19
+// CHECK:  encoding: [0x62,0xe2,0x45,0x10,0x7f,0x9a,0x00,0x02,0x00,0x00]
+          vpermt2ps 512(%rdx){1to4}, %xmm23, %xmm19
+
+// CHECK: vpermt2ps -512(%rdx){1to4}, %xmm23, %xmm19
+// CHECK:  encoding: [0x62,0xe2,0x45,0x10,0x7f,0x5a,0x80]
+          vpermt2ps -512(%rdx){1to4}, %xmm23, %xmm19
+
+// CHECK: vpermt2ps -516(%rdx){1to4}, %xmm23, %xmm19
+// CHECK:  encoding: [0x62,0xe2,0x45,0x10,0x7f,0x9a,0xfc,0xfd,0xff,0xff]
+          vpermt2ps -516(%rdx){1to4}, %xmm23, %xmm19
+
+// CHECK: vpermt2ps %ymm21, %ymm27, %ymm26
+// CHECK:  encoding: [0x62,0x22,0x25,0x20,0x7f,0xd5]
+          vpermt2ps %ymm21, %ymm27, %ymm26
+
+// CHECK: vpermt2ps %ymm21, %ymm27, %ymm26 {%k3}
+// CHECK:  encoding: [0x62,0x22,0x25,0x23,0x7f,0xd5]
+          vpermt2ps %ymm21, %ymm27, %ymm26 {%k3}
+
+// CHECK: vpermt2ps %ymm21, %ymm27, %ymm26 {%k3} {z}
+// CHECK:  encoding: [0x62,0x22,0x25,0xa3,0x7f,0xd5]
+          vpermt2ps %ymm21, %ymm27, %ymm26 {%k3} {z}
+
+// CHECK: vpermt2ps (%rcx), %ymm27, %ymm26
+// CHECK:  encoding: [0x62,0x62,0x25,0x20,0x7f,0x11]
+          vpermt2ps (%rcx), %ymm27, %ymm26
+
+// CHECK: vpermt2ps 291(%rax,%r14,8), %ymm27, %ymm26
+// CHECK:  encoding: [0x62,0x22,0x25,0x20,0x7f,0x94,0xf0,0x23,0x01,0x00,0x00]
+          vpermt2ps 291(%rax,%r14,8), %ymm27, %ymm26
+
+// CHECK: vpermt2ps (%rcx){1to8}, %ymm27, %ymm26
+// CHECK:  encoding: [0x62,0x62,0x25,0x30,0x7f,0x11]
+          vpermt2ps (%rcx){1to8}, %ymm27, %ymm26
+
+// CHECK: vpermt2ps 4064(%rdx), %ymm27, %ymm26
+// CHECK:  encoding: [0x62,0x62,0x25,0x20,0x7f,0x52,0x7f]
+          vpermt2ps 4064(%rdx), %ymm27, %ymm26
+
+// CHECK: vpermt2ps 4096(%rdx), %ymm27, %ymm26
+// CHECK:  encoding: [0x62,0x62,0x25,0x20,0x7f,0x92,0x00,0x10,0x00,0x00]
+          vpermt2ps 4096(%rdx), %ymm27, %ymm26
+
+// CHECK: vpermt2ps -4096(%rdx), %ymm27, %ymm26
+// CHECK:  encoding: [0x62,0x62,0x25,0x20,0x7f,0x52,0x80]
+          vpermt2ps -4096(%rdx), %ymm27, %ymm26
+
+// CHECK: vpermt2ps -4128(%rdx), %ymm27, %ymm26
+// CHECK:  encoding: [0x62,0x62,0x25,0x20,0x7f,0x92,0xe0,0xef,0xff,0xff]
+          vpermt2ps -4128(%rdx), %ymm27, %ymm26
+
+// CHECK: vpermt2ps 508(%rdx){1to8}, %ymm27, %ymm26
+// CHECK:  encoding: [0x62,0x62,0x25,0x30,0x7f,0x52,0x7f]
+          vpermt2ps 508(%rdx){1to8}, %ymm27, %ymm26
+
+// CHECK: vpermt2ps 512(%rdx){1to8}, %ymm27, %ymm26
+// CHECK:  encoding: [0x62,0x62,0x25,0x30,0x7f,0x92,0x00,0x02,0x00,0x00]
+          vpermt2ps 512(%rdx){1to8}, %ymm27, %ymm26
+
+// CHECK: vpermt2ps -512(%rdx){1to8}, %ymm27, %ymm26
+// CHECK:  encoding: [0x62,0x62,0x25,0x30,0x7f,0x52,0x80]
+          vpermt2ps -512(%rdx){1to8}, %ymm27, %ymm26
+
+// CHECK: vpermt2ps -516(%rdx){1to8}, %ymm27, %ymm26
+// CHECK:  encoding: [0x62,0x62,0x25,0x30,0x7f,0x92,0xfc,0xfd,0xff,0xff]
+          vpermt2ps -516(%rdx){1to8}, %ymm27, %ymm26
+
+// CHECK: vpermt2pd %xmm17, %xmm21, %xmm26
+// CHECK:  encoding: [0x62,0x22,0xd5,0x00,0x7f,0xd1]
+          vpermt2pd %xmm17, %xmm21, %xmm26
+
+// CHECK: vpermt2pd %xmm17, %xmm21, %xmm26 {%k5}
+// CHECK:  encoding: [0x62,0x22,0xd5,0x05,0x7f,0xd1]
+          vpermt2pd %xmm17, %xmm21, %xmm26 {%k5}
+
+// CHECK: vpermt2pd %xmm17, %xmm21, %xmm26 {%k5} {z}
+// CHECK:  encoding: [0x62,0x22,0xd5,0x85,0x7f,0xd1]
+          vpermt2pd %xmm17, %xmm21, %xmm26 {%k5} {z}
+
+// CHECK: vpermt2pd (%rcx), %xmm21, %xmm26
+// CHECK:  encoding: [0x62,0x62,0xd5,0x00,0x7f,0x11]
+          vpermt2pd (%rcx), %xmm21, %xmm26
+
+// CHECK: vpermt2pd 291(%rax,%r14,8), %xmm21, %xmm26
+// CHECK:  encoding: [0x62,0x22,0xd5,0x00,0x7f,0x94,0xf0,0x23,0x01,0x00,0x00]
+          vpermt2pd 291(%rax,%r14,8), %xmm21, %xmm26
+
+// CHECK: vpermt2pd (%rcx){1to2}, %xmm21, %xmm26
+// CHECK:  encoding: [0x62,0x62,0xd5,0x10,0x7f,0x11]
+          vpermt2pd (%rcx){1to2}, %xmm21, %xmm26
+
+// CHECK: vpermt2pd 2032(%rdx), %xmm21, %xmm26
+// CHECK:  encoding: [0x62,0x62,0xd5,0x00,0x7f,0x52,0x7f]
+          vpermt2pd 2032(%rdx), %xmm21, %xmm26
+
+// CHECK: vpermt2pd 2048(%rdx), %xmm21, %xmm26
+// CHECK:  encoding: [0x62,0x62,0xd5,0x00,0x7f,0x92,0x00,0x08,0x00,0x00]
+          vpermt2pd 2048(%rdx), %xmm21, %xmm26
+
+// CHECK: vpermt2pd -2048(%rdx), %xmm21, %xmm26
+// CHECK:  encoding: [0x62,0x62,0xd5,0x00,0x7f,0x52,0x80]
+          vpermt2pd -2048(%rdx), %xmm21, %xmm26
+
+// CHECK: vpermt2pd -2064(%rdx), %xmm21, %xmm26
+// CHECK:  encoding: [0x62,0x62,0xd5,0x00,0x7f,0x92,0xf0,0xf7,0xff,0xff]
+          vpermt2pd -2064(%rdx), %xmm21, %xmm26
+
+// CHECK: vpermt2pd 1016(%rdx){1to2}, %xmm21, %xmm26
+// CHECK:  encoding: [0x62,0x62,0xd5,0x10,0x7f,0x52,0x7f]
+          vpermt2pd 1016(%rdx){1to2}, %xmm21, %xmm26
+
+// CHECK: vpermt2pd 1024(%rdx){1to2}, %xmm21, %xmm26
+// CHECK:  encoding: [0x62,0x62,0xd5,0x10,0x7f,0x92,0x00,0x04,0x00,0x00]
+          vpermt2pd 1024(%rdx){1to2}, %xmm21, %xmm26
+
+// CHECK: vpermt2pd -1024(%rdx){1to2}, %xmm21, %xmm26
+// CHECK:  encoding: [0x62,0x62,0xd5,0x10,0x7f,0x52,0x80]
+          vpermt2pd -1024(%rdx){1to2}, %xmm21, %xmm26
+
+// CHECK: vpermt2pd -1032(%rdx){1to2}, %xmm21, %xmm26
+// CHECK:  encoding: [0x62,0x62,0xd5,0x10,0x7f,0x92,0xf8,0xfb,0xff,0xff]
+          vpermt2pd -1032(%rdx){1to2}, %xmm21, %xmm26
+
+// CHECK: vpermt2pd %ymm17, %ymm23, %ymm17
+// CHECK:  encoding: [0x62,0xa2,0xc5,0x20,0x7f,0xc9]
+          vpermt2pd %ymm17, %ymm23, %ymm17
+
+// CHECK: vpermt2pd %ymm17, %ymm23, %ymm17 {%k1}
+// CHECK:  encoding: [0x62,0xa2,0xc5,0x21,0x7f,0xc9]
+          vpermt2pd %ymm17, %ymm23, %ymm17 {%k1}
+
+// CHECK: vpermt2pd %ymm17, %ymm23, %ymm17 {%k1} {z}
+// CHECK:  encoding: [0x62,0xa2,0xc5,0xa1,0x7f,0xc9]
+          vpermt2pd %ymm17, %ymm23, %ymm17 {%k1} {z}
+
+// CHECK: vpermt2pd (%rcx), %ymm23, %ymm17
+// CHECK:  encoding: [0x62,0xe2,0xc5,0x20,0x7f,0x09]
+          vpermt2pd (%rcx), %ymm23, %ymm17
+
+// CHECK: vpermt2pd 291(%rax,%r14,8), %ymm23, %ymm17
+// CHECK:  encoding: [0x62,0xa2,0xc5,0x20,0x7f,0x8c,0xf0,0x23,0x01,0x00,0x00]
+          vpermt2pd 291(%rax,%r14,8), %ymm23, %ymm17
+
+// CHECK: vpermt2pd (%rcx){1to4}, %ymm23, %ymm17
+// CHECK:  encoding: [0x62,0xe2,0xc5,0x30,0x7f,0x09]
+          vpermt2pd (%rcx){1to4}, %ymm23, %ymm17
+
+// CHECK: vpermt2pd 4064(%rdx), %ymm23, %ymm17
+// CHECK:  encoding: [0x62,0xe2,0xc5,0x20,0x7f,0x4a,0x7f]
+          vpermt2pd 4064(%rdx), %ymm23, %ymm17
+
+// CHECK: vpermt2pd 4096(%rdx), %ymm23, %ymm17
+// CHECK:  encoding: [0x62,0xe2,0xc5,0x20,0x7f,0x8a,0x00,0x10,0x00,0x00]
+          vpermt2pd 4096(%rdx), %ymm23, %ymm17
+
+// CHECK: vpermt2pd -4096(%rdx), %ymm23, %ymm17
+// CHECK:  encoding: [0x62,0xe2,0xc5,0x20,0x7f,0x4a,0x80]
+          vpermt2pd -4096(%rdx), %ymm23, %ymm17
+
+// CHECK: vpermt2pd -4128(%rdx), %ymm23, %ymm17
+// CHECK:  encoding: [0x62,0xe2,0xc5,0x20,0x7f,0x8a,0xe0,0xef,0xff,0xff]
+          vpermt2pd -4128(%rdx), %ymm23, %ymm17
+
+// CHECK: vpermt2pd 1016(%rdx){1to4}, %ymm23, %ymm17
+// CHECK:  encoding: [0x62,0xe2,0xc5,0x30,0x7f,0x4a,0x7f]
+          vpermt2pd 1016(%rdx){1to4}, %ymm23, %ymm17
+
+// CHECK: vpermt2pd 1024(%rdx){1to4}, %ymm23, %ymm17
+// CHECK:  encoding: [0x62,0xe2,0xc5,0x30,0x7f,0x8a,0x00,0x04,0x00,0x00]
+          vpermt2pd 1024(%rdx){1to4}, %ymm23, %ymm17
+
+// CHECK: vpermt2pd -1024(%rdx){1to4}, %ymm23, %ymm17
+// CHECK:  encoding: [0x62,0xe2,0xc5,0x30,0x7f,0x4a,0x80]
+          vpermt2pd -1024(%rdx){1to4}, %ymm23, %ymm17
+
+// CHECK: vpermt2pd -1032(%rdx){1to4}, %ymm23, %ymm17
+// CHECK:  encoding: [0x62,0xe2,0xc5,0x30,0x7f,0x8a,0xf8,0xfb,0xff,0xff]
+          vpermt2pd -1032(%rdx){1to4}, %ymm23, %ymm17
+
+// CHECK: vscalefpd %xmm17, %xmm17, %xmm24
+// CHECK:  encoding: [0x62,0x22,0xf5,0x00,0x2c,0xc1]
+          vscalefpd %xmm17, %xmm17, %xmm24
+
+// CHECK: vscalefpd %xmm17, %xmm17, %xmm24 {%k2}
+// CHECK:  encoding: [0x62,0x22,0xf5,0x02,0x2c,0xc1]
+          vscalefpd %xmm17, %xmm17, %xmm24 {%k2}
+
+// CHECK: vscalefpd %xmm17, %xmm17, %xmm24 {%k2} {z}
+// CHECK:  encoding: [0x62,0x22,0xf5,0x82,0x2c,0xc1]
+          vscalefpd %xmm17, %xmm17, %xmm24 {%k2} {z}
+
+// CHECK: vscalefpd (%rcx), %xmm17, %xmm24
+// CHECK:  encoding: [0x62,0x62,0xf5,0x00,0x2c,0x01]
+          vscalefpd (%rcx), %xmm17, %xmm24
+
+// CHECK: vscalefpd 291(%rax,%r14,8), %xmm17, %xmm24
+// CHECK:  encoding: [0x62,0x22,0xf5,0x00,0x2c,0x84,0xf0,0x23,0x01,0x00,0x00]
+          vscalefpd 291(%rax,%r14,8), %xmm17, %xmm24
+
+// CHECK: vscalefpd (%rcx){1to2}, %xmm17, %xmm24
+// CHECK:  encoding: [0x62,0x62,0xf5,0x10,0x2c,0x01]
+          vscalefpd (%rcx){1to2}, %xmm17, %xmm24
+
+// CHECK: vscalefpd 2032(%rdx), %xmm17, %xmm24
+// CHECK:  encoding: [0x62,0x62,0xf5,0x00,0x2c,0x42,0x7f]
+          vscalefpd 2032(%rdx), %xmm17, %xmm24
+
+// CHECK: vscalefpd 2048(%rdx), %xmm17, %xmm24
+// CHECK:  encoding: [0x62,0x62,0xf5,0x00,0x2c,0x82,0x00,0x08,0x00,0x00]
+          vscalefpd 2048(%rdx), %xmm17, %xmm24
+
+// CHECK: vscalefpd -2048(%rdx), %xmm17, %xmm24
+// CHECK:  encoding: [0x62,0x62,0xf5,0x00,0x2c,0x42,0x80]
+          vscalefpd -2048(%rdx), %xmm17, %xmm24
+
+// CHECK: vscalefpd -2064(%rdx), %xmm17, %xmm24
+// CHECK:  encoding: [0x62,0x62,0xf5,0x00,0x2c,0x82,0xf0,0xf7,0xff,0xff]
+          vscalefpd -2064(%rdx), %xmm17, %xmm24
+
+// CHECK: vscalefpd 1016(%rdx){1to2}, %xmm17, %xmm24
+// CHECK:  encoding: [0x62,0x62,0xf5,0x10,0x2c,0x42,0x7f]
+          vscalefpd 1016(%rdx){1to2}, %xmm17, %xmm24
+
+// CHECK: vscalefpd 1024(%rdx){1to2}, %xmm17, %xmm24
+// CHECK:  encoding: [0x62,0x62,0xf5,0x10,0x2c,0x82,0x00,0x04,0x00,0x00]
+          vscalefpd 1024(%rdx){1to2}, %xmm17, %xmm24
+
+// CHECK: vscalefpd -1024(%rdx){1to2}, %xmm17, %xmm24
+// CHECK:  encoding: [0x62,0x62,0xf5,0x10,0x2c,0x42,0x80]
+          vscalefpd -1024(%rdx){1to2}, %xmm17, %xmm24
+
+// CHECK: vscalefpd -1032(%rdx){1to2}, %xmm17, %xmm24
+// CHECK:  encoding: [0x62,0x62,0xf5,0x10,0x2c,0x82,0xf8,0xfb,0xff,0xff]
+          vscalefpd -1032(%rdx){1to2}, %xmm17, %xmm24
+
+// CHECK: vscalefpd %ymm26, %ymm20, %ymm29
+// CHECK:  encoding: [0x62,0x02,0xdd,0x20,0x2c,0xea]
+          vscalefpd %ymm26, %ymm20, %ymm29
+
+// CHECK: vscalefpd %ymm26, %ymm20, %ymm29 {%k2}
+// CHECK:  encoding: [0x62,0x02,0xdd,0x22,0x2c,0xea]
+          vscalefpd %ymm26, %ymm20, %ymm29 {%k2}
+
+// CHECK: vscalefpd %ymm26, %ymm20, %ymm29 {%k2} {z}
+// CHECK:  encoding: [0x62,0x02,0xdd,0xa2,0x2c,0xea]
+          vscalefpd %ymm26, %ymm20, %ymm29 {%k2} {z}
+
+// CHECK: vscalefpd (%rcx), %ymm20, %ymm29
+// CHECK:  encoding: [0x62,0x62,0xdd,0x20,0x2c,0x29]
+          vscalefpd (%rcx), %ymm20, %ymm29
+
+// CHECK: vscalefpd 291(%rax,%r14,8), %ymm20, %ymm29
+// CHECK:  encoding: [0x62,0x22,0xdd,0x20,0x2c,0xac,0xf0,0x23,0x01,0x00,0x00]
+          vscalefpd 291(%rax,%r14,8), %ymm20, %ymm29
+
+// CHECK: vscalefpd (%rcx){1to4}, %ymm20, %ymm29
+// CHECK:  encoding: [0x62,0x62,0xdd,0x30,0x2c,0x29]
+          vscalefpd (%rcx){1to4}, %ymm20, %ymm29
+
+// CHECK: vscalefpd 4064(%rdx), %ymm20, %ymm29
+// CHECK:  encoding: [0x62,0x62,0xdd,0x20,0x2c,0x6a,0x7f]
+          vscalefpd 4064(%rdx), %ymm20, %ymm29
+
+// CHECK: vscalefpd 4096(%rdx), %ymm20, %ymm29
+// CHECK:  encoding: [0x62,0x62,0xdd,0x20,0x2c,0xaa,0x00,0x10,0x00,0x00]
+          vscalefpd 4096(%rdx), %ymm20, %ymm29
+
+// CHECK: vscalefpd -4096(%rdx), %ymm20, %ymm29
+// CHECK:  encoding: [0x62,0x62,0xdd,0x20,0x2c,0x6a,0x80]
+          vscalefpd -4096(%rdx), %ymm20, %ymm29
+
+// CHECK: vscalefpd -4128(%rdx), %ymm20, %ymm29
+// CHECK:  encoding: [0x62,0x62,0xdd,0x20,0x2c,0xaa,0xe0,0xef,0xff,0xff]
+          vscalefpd -4128(%rdx), %ymm20, %ymm29
+
+// CHECK: vscalefpd 1016(%rdx){1to4}, %ymm20, %ymm29
+// CHECK:  encoding: [0x62,0x62,0xdd,0x30,0x2c,0x6a,0x7f]
+          vscalefpd 1016(%rdx){1to4}, %ymm20, %ymm29
+
+// CHECK: vscalefpd 1024(%rdx){1to4}, %ymm20, %ymm29
+// CHECK:  encoding: [0x62,0x62,0xdd,0x30,0x2c,0xaa,0x00,0x04,0x00,0x00]
+          vscalefpd 1024(%rdx){1to4}, %ymm20, %ymm29
+
+// CHECK: vscalefpd -1024(%rdx){1to4}, %ymm20, %ymm29
+// CHECK:  encoding: [0x62,0x62,0xdd,0x30,0x2c,0x6a,0x80]
+          vscalefpd -1024(%rdx){1to4}, %ymm20, %ymm29
+
+// CHECK: vscalefpd -1032(%rdx){1to4}, %ymm20, %ymm29
+// CHECK:  encoding: [0x62,0x62,0xdd,0x30,0x2c,0xaa,0xf8,0xfb,0xff,0xff]
+          vscalefpd -1032(%rdx){1to4}, %ymm20, %ymm29
+
+// CHECK: vscalefps %xmm22, %xmm27, %xmm19
+// CHECK:  encoding: [0x62,0xa2,0x25,0x00,0x2c,0xde]
+          vscalefps %xmm22, %xmm27, %xmm19
+
+// CHECK: vscalefps %xmm22, %xmm27, %xmm19 {%k4}
+// CHECK:  encoding: [0x62,0xa2,0x25,0x04,0x2c,0xde]
+          vscalefps %xmm22, %xmm27, %xmm19 {%k4}
+
+// CHECK: vscalefps %xmm22, %xmm27, %xmm19 {%k4} {z}
+// CHECK:  encoding: [0x62,0xa2,0x25,0x84,0x2c,0xde]
+          vscalefps %xmm22, %xmm27, %xmm19 {%k4} {z}
+
+// CHECK: vscalefps (%rcx), %xmm27, %xmm19
+// CHECK:  encoding: [0x62,0xe2,0x25,0x00,0x2c,0x19]
+          vscalefps (%rcx), %xmm27, %xmm19
+
+// CHECK: vscalefps 291(%rax,%r14,8), %xmm27, %xmm19
+// CHECK:  encoding: [0x62,0xa2,0x25,0x00,0x2c,0x9c,0xf0,0x23,0x01,0x00,0x00]
+          vscalefps 291(%rax,%r14,8), %xmm27, %xmm19
+
+// CHECK: vscalefps (%rcx){1to4}, %xmm27, %xmm19
+// CHECK:  encoding: [0x62,0xe2,0x25,0x10,0x2c,0x19]
+          vscalefps (%rcx){1to4}, %xmm27, %xmm19
+
+// CHECK: vscalefps 2032(%rdx), %xmm27, %xmm19
+// CHECK:  encoding: [0x62,0xe2,0x25,0x00,0x2c,0x5a,0x7f]
+          vscalefps 2032(%rdx), %xmm27, %xmm19
+
+// CHECK: vscalefps 2048(%rdx), %xmm27, %xmm19
+// CHECK:  encoding: [0x62,0xe2,0x25,0x00,0x2c,0x9a,0x00,0x08,0x00,0x00]
+          vscalefps 2048(%rdx), %xmm27, %xmm19
+
+// CHECK: vscalefps -2048(%rdx), %xmm27, %xmm19
+// CHECK:  encoding: [0x62,0xe2,0x25,0x00,0x2c,0x5a,0x80]
+          vscalefps -2048(%rdx), %xmm27, %xmm19
+
+// CHECK: vscalefps -2064(%rdx), %xmm27, %xmm19
+// CHECK:  encoding: [0x62,0xe2,0x25,0x00,0x2c,0x9a,0xf0,0xf7,0xff,0xff]
+          vscalefps -2064(%rdx), %xmm27, %xmm19
+
+// CHECK: vscalefps 508(%rdx){1to4}, %xmm27, %xmm19
+// CHECK:  encoding: [0x62,0xe2,0x25,0x10,0x2c,0x5a,0x7f]
+          vscalefps 508(%rdx){1to4}, %xmm27, %xmm19
+
+// CHECK: vscalefps 512(%rdx){1to4}, %xmm27, %xmm19
+// CHECK:  encoding: [0x62,0xe2,0x25,0x10,0x2c,0x9a,0x00,0x02,0x00,0x00]
+          vscalefps 512(%rdx){1to4}, %xmm27, %xmm19
+
+// CHECK: vscalefps -512(%rdx){1to4}, %xmm27, %xmm19
+// CHECK:  encoding: [0x62,0xe2,0x25,0x10,0x2c,0x5a,0x80]
+          vscalefps -512(%rdx){1to4}, %xmm27, %xmm19
+
+// CHECK: vscalefps -516(%rdx){1to4}, %xmm27, %xmm19
+// CHECK:  encoding: [0x62,0xe2,0x25,0x10,0x2c,0x9a,0xfc,0xfd,0xff,0xff]
+          vscalefps -516(%rdx){1to4}, %xmm27, %xmm19
+
+// CHECK: vscalefps %ymm23, %ymm22, %ymm25
+// CHECK:  encoding: [0x62,0x22,0x4d,0x20,0x2c,0xcf]
+          vscalefps %ymm23, %ymm22, %ymm25
+
+// CHECK: vscalefps %ymm23, %ymm22, %ymm25 {%k4}
+// CHECK:  encoding: [0x62,0x22,0x4d,0x24,0x2c,0xcf]
+          vscalefps %ymm23, %ymm22, %ymm25 {%k4}
+
+// CHECK: vscalefps %ymm23, %ymm22, %ymm25 {%k4} {z}
+// CHECK:  encoding: [0x62,0x22,0x4d,0xa4,0x2c,0xcf]
+          vscalefps %ymm23, %ymm22, %ymm25 {%k4} {z}
+
+// CHECK: vscalefps (%rcx), %ymm22, %ymm25
+// CHECK:  encoding: [0x62,0x62,0x4d,0x20,0x2c,0x09]
+          vscalefps (%rcx), %ymm22, %ymm25
+
+// CHECK: vscalefps 291(%rax,%r14,8), %ymm22, %ymm25
+// CHECK:  encoding: [0x62,0x22,0x4d,0x20,0x2c,0x8c,0xf0,0x23,0x01,0x00,0x00]
+          vscalefps 291(%rax,%r14,8), %ymm22, %ymm25
+
+// CHECK: vscalefps (%rcx){1to8}, %ymm22, %ymm25
+// CHECK:  encoding: [0x62,0x62,0x4d,0x30,0x2c,0x09]
+          vscalefps (%rcx){1to8}, %ymm22, %ymm25
+
+// CHECK: vscalefps 4064(%rdx), %ymm22, %ymm25
+// CHECK:  encoding: [0x62,0x62,0x4d,0x20,0x2c,0x4a,0x7f]
+          vscalefps 4064(%rdx), %ymm22, %ymm25
+
+// CHECK: vscalefps 4096(%rdx), %ymm22, %ymm25
+// CHECK:  encoding: [0x62,0x62,0x4d,0x20,0x2c,0x8a,0x00,0x10,0x00,0x00]
+          vscalefps 4096(%rdx), %ymm22, %ymm25
+
+// CHECK: vscalefps -4096(%rdx), %ymm22, %ymm25
+// CHECK:  encoding: [0x62,0x62,0x4d,0x20,0x2c,0x4a,0x80]
+          vscalefps -4096(%rdx), %ymm22, %ymm25
+
+// CHECK: vscalefps -4128(%rdx), %ymm22, %ymm25
+// CHECK:  encoding: [0x62,0x62,0x4d,0x20,0x2c,0x8a,0xe0,0xef,0xff,0xff]
+          vscalefps -4128(%rdx), %ymm22, %ymm25
+
+// CHECK: vscalefps 508(%rdx){1to8}, %ymm22, %ymm25
+// CHECK:  encoding: [0x62,0x62,0x4d,0x30,0x2c,0x4a,0x7f]
+          vscalefps 508(%rdx){1to8}, %ymm22, %ymm25
+
+// CHECK: vscalefps 512(%rdx){1to8}, %ymm22, %ymm25
+// CHECK:  encoding: [0x62,0x62,0x4d,0x30,0x2c,0x8a,0x00,0x02,0x00,0x00]
+          vscalefps 512(%rdx){1to8}, %ymm22, %ymm25
+
+// CHECK: vscalefps -512(%rdx){1to8}, %ymm22, %ymm25
+// CHECK:  encoding: [0x62,0x62,0x4d,0x30,0x2c,0x4a,0x80]
+          vscalefps -512(%rdx){1to8}, %ymm22, %ymm25
+
+// CHECK: vscalefps -516(%rdx){1to8}, %ymm22, %ymm25
+// CHECK:  encoding: [0x62,0x62,0x4d,0x30,0x2c,0x8a,0xfc,0xfd,0xff,0xff]
+          vscalefps -516(%rdx){1to8}, %ymm22, %ymm25
+
+// CHECK: vpmovqb %xmm29, %xmm24
+// CHECK:  encoding: [0x62,0x02,0x7e,0x08,0x32,0xe8]
+          vpmovqb %xmm29, %xmm24
+
+// CHECK: vpmovqb %xmm29, %xmm24 {%k4}
+// CHECK:  encoding: [0x62,0x02,0x7e,0x0c,0x32,0xe8]
+          vpmovqb %xmm29, %xmm24 {%k4}
+
+// CHECK: vpmovqb %xmm29, %xmm24 {%k4} {z}
+// CHECK:  encoding: [0x62,0x02,0x7e,0x8c,0x32,0xe8]
+          vpmovqb %xmm29, %xmm24 {%k4} {z}
+
+// CHECK: vpmovqb %ymm29, %xmm17
+// CHECK:  encoding: [0x62,0x22,0x7e,0x28,0x32,0xe9]
+          vpmovqb %ymm29, %xmm17
+
+// CHECK: vpmovqb %ymm29, %xmm17 {%k3}
+// CHECK:  encoding: [0x62,0x22,0x7e,0x2b,0x32,0xe9]
+          vpmovqb %ymm29, %xmm17 {%k3}
+
+// CHECK: vpmovqb %ymm29, %xmm17 {%k3} {z}
+// CHECK:  encoding: [0x62,0x22,0x7e,0xab,0x32,0xe9]
+          vpmovqb %ymm29, %xmm17 {%k3} {z}
+
+// CHECK: vpmovqb %xmm27, (%rcx)
+// CHECK:  encoding: [0x62,0x62,0x7e,0x08,0x32,0x19]
+          vpmovqb %xmm27, (%rcx)
+
+// CHECK: vpmovqb %xmm27, (%rcx) {%k2}
+// CHECK:  encoding: [0x62,0x62,0x7e,0x0a,0x32,0x19]
+          vpmovqb %xmm27, (%rcx) {%k2}
+
+// CHECK: vpmovqb %xmm27, 291(%rax,%r14,8)
+// CHECK:  encoding: [0x62,0x22,0x7e,0x08,0x32,0x9c,0xf0,0x23,0x01,0x00,0x00]
+          vpmovqb %xmm27, 291(%rax,%r14,8)
+
+// CHECK: vpmovqb %xmm27, 254(%rdx)
+// CHECK:  encoding: [0x62,0x62,0x7e,0x08,0x32,0x5a,0x7f]
+          vpmovqb %xmm27, 254(%rdx)
+
+// CHECK: vpmovqb %xmm27, 256(%rdx)
+// CHECK:  encoding: [0x62,0x62,0x7e,0x08,0x32,0x9a,0x00,0x01,0x00,0x00]
+          vpmovqb %xmm27, 256(%rdx)
+
+// CHECK: vpmovqb %xmm27, -256(%rdx)
+// CHECK:  encoding: [0x62,0x62,0x7e,0x08,0x32,0x5a,0x80]
+          vpmovqb %xmm27, -256(%rdx)
+
+// CHECK: vpmovqb %xmm27, -258(%rdx)
+// CHECK:  encoding: [0x62,0x62,0x7e,0x08,0x32,0x9a,0xfe,0xfe,0xff,0xff]
+          vpmovqb %xmm27, -258(%rdx)
+
+// CHECK: vpmovqb %ymm28, (%rcx)
+// CHECK:  encoding: [0x62,0x62,0x7e,0x28,0x32,0x21]
+          vpmovqb %ymm28, (%rcx)
+
+// CHECK: vpmovqb %ymm28, (%rcx) {%k7}
+// CHECK:  encoding: [0x62,0x62,0x7e,0x2f,0x32,0x21]
+          vpmovqb %ymm28, (%rcx) {%k7}
+
+// CHECK: vpmovqb %ymm28, 291(%rax,%r14,8)
+// CHECK:  encoding: [0x62,0x22,0x7e,0x28,0x32,0xa4,0xf0,0x23,0x01,0x00,0x00]
+          vpmovqb %ymm28, 291(%rax,%r14,8)
+
+// CHECK: vpmovqb %ymm28, 508(%rdx)
+// CHECK:  encoding: [0x62,0x62,0x7e,0x28,0x32,0x62,0x7f]
+          vpmovqb %ymm28, 508(%rdx)
+
+// CHECK: vpmovqb %ymm28, 512(%rdx)
+// CHECK:  encoding: [0x62,0x62,0x7e,0x28,0x32,0xa2,0x00,0x02,0x00,0x00]
+          vpmovqb %ymm28, 512(%rdx)
+
+// CHECK: vpmovqb %ymm28, -512(%rdx)
+// CHECK:  encoding: [0x62,0x62,0x7e,0x28,0x32,0x62,0x80]
+          vpmovqb %ymm28, -512(%rdx)
+
+// CHECK: vpmovqb %ymm28, -516(%rdx)
+// CHECK:  encoding: [0x62,0x62,0x7e,0x28,0x32,0xa2,0xfc,0xfd,0xff,0xff]
+          vpmovqb %ymm28, -516(%rdx)
+
+// CHECK: vpmovsqb %xmm19, %xmm26
+// CHECK:  encoding: [0x62,0x82,0x7e,0x08,0x22,0xda]
+          vpmovsqb %xmm19, %xmm26
+
+// CHECK: vpmovsqb %xmm19, %xmm26 {%k1}
+// CHECK:  encoding: [0x62,0x82,0x7e,0x09,0x22,0xda]
+          vpmovsqb %xmm19, %xmm26 {%k1}
+
+// CHECK: vpmovsqb %xmm19, %xmm26 {%k1} {z}
+// CHECK:  encoding: [0x62,0x82,0x7e,0x89,0x22,0xda]
+          vpmovsqb %xmm19, %xmm26 {%k1} {z}
+
+// CHECK: vpmovsqb %ymm20, %xmm20
+// CHECK:  encoding: [0x62,0xa2,0x7e,0x28,0x22,0xe4]
+          vpmovsqb %ymm20, %xmm20
+
+// CHECK: vpmovsqb %ymm20, %xmm20 {%k6}
+// CHECK:  encoding: [0x62,0xa2,0x7e,0x2e,0x22,0xe4]
+          vpmovsqb %ymm20, %xmm20 {%k6}
+
+// CHECK: vpmovsqb %ymm20, %xmm20 {%k6} {z}
+// CHECK:  encoding: [0x62,0xa2,0x7e,0xae,0x22,0xe4]
+          vpmovsqb %ymm20, %xmm20 {%k6} {z}
+
+// CHECK: vpmovsqb %xmm25, (%rcx)
+// CHECK:  encoding: [0x62,0x62,0x7e,0x08,0x22,0x09]
+          vpmovsqb %xmm25, (%rcx)
+
+// CHECK: vpmovsqb %xmm25, (%rcx) {%k7}
+// CHECK:  encoding: [0x62,0x62,0x7e,0x0f,0x22,0x09]
+          vpmovsqb %xmm25, (%rcx) {%k7}
+
+// CHECK: vpmovsqb %xmm25, 291(%rax,%r14,8)
+// CHECK:  encoding: [0x62,0x22,0x7e,0x08,0x22,0x8c,0xf0,0x23,0x01,0x00,0x00]
+          vpmovsqb %xmm25, 291(%rax,%r14,8)
+
+// CHECK: vpmovsqb %xmm25, 254(%rdx)
+// CHECK:  encoding: [0x62,0x62,0x7e,0x08,0x22,0x4a,0x7f]
+          vpmovsqb %xmm25, 254(%rdx)
+
+// CHECK: vpmovsqb %xmm25, 256(%rdx)
+// CHECK:  encoding: [0x62,0x62,0x7e,0x08,0x22,0x8a,0x00,0x01,0x00,0x00]
+          vpmovsqb %xmm25, 256(%rdx)
+
+// CHECK: vpmovsqb %xmm25, -256(%rdx)
+// CHECK:  encoding: [0x62,0x62,0x7e,0x08,0x22,0x4a,0x80]
+          vpmovsqb %xmm25, -256(%rdx)
+
+// CHECK: vpmovsqb %xmm25, -258(%rdx)
+// CHECK:  encoding: [0x62,0x62,0x7e,0x08,0x22,0x8a,0xfe,0xfe,0xff,0xff]
+          vpmovsqb %xmm25, -258(%rdx)
+
+// CHECK: vpmovsqb %ymm17, (%rcx)
+// CHECK:  encoding: [0x62,0xe2,0x7e,0x28,0x22,0x09]
+          vpmovsqb %ymm17, (%rcx)
+
+// CHECK: vpmovsqb %ymm17, (%rcx) {%k4}
+// CHECK:  encoding: [0x62,0xe2,0x7e,0x2c,0x22,0x09]
+          vpmovsqb %ymm17, (%rcx) {%k4}
+
+// CHECK: vpmovsqb %ymm17, 291(%rax,%r14,8)
+// CHECK:  encoding: [0x62,0xa2,0x7e,0x28,0x22,0x8c,0xf0,0x23,0x01,0x00,0x00]
+          vpmovsqb %ymm17, 291(%rax,%r14,8)
+
+// CHECK: vpmovsqb %ymm17, 508(%rdx)
+// CHECK:  encoding: [0x62,0xe2,0x7e,0x28,0x22,0x4a,0x7f]
+          vpmovsqb %ymm17, 508(%rdx)
+
+// CHECK: vpmovsqb %ymm17, 512(%rdx)
+// CHECK:  encoding: [0x62,0xe2,0x7e,0x28,0x22,0x8a,0x00,0x02,0x00,0x00]
+          vpmovsqb %ymm17, 512(%rdx)
+
+// CHECK: vpmovsqb %ymm17, -512(%rdx)
+// CHECK:  encoding: [0x62,0xe2,0x7e,0x28,0x22,0x4a,0x80]
+          vpmovsqb %ymm17, -512(%rdx)
+
+// CHECK: vpmovsqb %ymm17, -516(%rdx)
+// CHECK:  encoding: [0x62,0xe2,0x7e,0x28,0x22,0x8a,0xfc,0xfd,0xff,0xff]
+          vpmovsqb %ymm17, -516(%rdx)
+
+// CHECK: vpmovusqb %xmm22, %xmm28
+// CHECK:  encoding: [0x62,0x82,0x7e,0x08,0x12,0xf4]
+          vpmovusqb %xmm22, %xmm28
+
+// CHECK: vpmovusqb %xmm22, %xmm28 {%k2}
+// CHECK:  encoding: [0x62,0x82,0x7e,0x0a,0x12,0xf4]
+          vpmovusqb %xmm22, %xmm28 {%k2}
+
+// CHECK: vpmovusqb %xmm22, %xmm28 {%k2} {z}
+// CHECK:  encoding: [0x62,0x82,0x7e,0x8a,0x12,0xf4]
+          vpmovusqb %xmm22, %xmm28 {%k2} {z}
+
+// CHECK: vpmovusqb %ymm23, %xmm22
+// CHECK:  encoding: [0x62,0xa2,0x7e,0x28,0x12,0xfe]
+          vpmovusqb %ymm23, %xmm22
+
+// CHECK: vpmovusqb %ymm23, %xmm22 {%k7}
+// CHECK:  encoding: [0x62,0xa2,0x7e,0x2f,0x12,0xfe]
+          vpmovusqb %ymm23, %xmm22 {%k7}
+
+// CHECK: vpmovusqb %ymm23, %xmm22 {%k7} {z}
+// CHECK:  encoding: [0x62,0xa2,0x7e,0xaf,0x12,0xfe]
+          vpmovusqb %ymm23, %xmm22 {%k7} {z}
+
+// CHECK: vpmovusqb %xmm26, (%rcx)
+// CHECK:  encoding: [0x62,0x62,0x7e,0x08,0x12,0x11]
+          vpmovusqb %xmm26, (%rcx)
+
+// CHECK: vpmovusqb %xmm26, (%rcx) {%k5}
+// CHECK:  encoding: [0x62,0x62,0x7e,0x0d,0x12,0x11]
+          vpmovusqb %xmm26, (%rcx) {%k5}
+
+// CHECK: vpmovusqb %xmm26, 291(%rax,%r14,8)
+// CHECK:  encoding: [0x62,0x22,0x7e,0x08,0x12,0x94,0xf0,0x23,0x01,0x00,0x00]
+          vpmovusqb %xmm26, 291(%rax,%r14,8)
+
+// CHECK: vpmovusqb %xmm26, 254(%rdx)
+// CHECK:  encoding: [0x62,0x62,0x7e,0x08,0x12,0x52,0x7f]
+          vpmovusqb %xmm26, 254(%rdx)
+
+// CHECK: vpmovusqb %xmm26, 256(%rdx)
+// CHECK:  encoding: [0x62,0x62,0x7e,0x08,0x12,0x92,0x00,0x01,0x00,0x00]
+          vpmovusqb %xmm26, 256(%rdx)
+
+// CHECK: vpmovusqb %xmm26, -256(%rdx)
+// CHECK:  encoding: [0x62,0x62,0x7e,0x08,0x12,0x52,0x80]
+          vpmovusqb %xmm26, -256(%rdx)
+
+// CHECK: vpmovusqb %xmm26, -258(%rdx)
+// CHECK:  encoding: [0x62,0x62,0x7e,0x08,0x12,0x92,0xfe,0xfe,0xff,0xff]
+          vpmovusqb %xmm26, -258(%rdx)
+
+// CHECK: vpmovusqb %ymm30, (%rcx)
+// CHECK:  encoding: [0x62,0x62,0x7e,0x28,0x12,0x31]
+          vpmovusqb %ymm30, (%rcx)
+
+// CHECK: vpmovusqb %ymm30, (%rcx) {%k2}
+// CHECK:  encoding: [0x62,0x62,0x7e,0x2a,0x12,0x31]
+          vpmovusqb %ymm30, (%rcx) {%k2}
+
+// CHECK: vpmovusqb %ymm30, 291(%rax,%r14,8)
+// CHECK:  encoding: [0x62,0x22,0x7e,0x28,0x12,0xb4,0xf0,0x23,0x01,0x00,0x00]
+          vpmovusqb %ymm30, 291(%rax,%r14,8)
+
+// CHECK: vpmovusqb %ymm30, 508(%rdx)
+// CHECK:  encoding: [0x62,0x62,0x7e,0x28,0x12,0x72,0x7f]
+          vpmovusqb %ymm30, 508(%rdx)
+
+// CHECK: vpmovusqb %ymm30, 512(%rdx)
+// CHECK:  encoding: [0x62,0x62,0x7e,0x28,0x12,0xb2,0x00,0x02,0x00,0x00]
+          vpmovusqb %ymm30, 512(%rdx)
+
+// CHECK: vpmovusqb %ymm30, -512(%rdx)
+// CHECK:  encoding: [0x62,0x62,0x7e,0x28,0x12,0x72,0x80]
+          vpmovusqb %ymm30, -512(%rdx)
+
+// CHECK: vpmovusqb %ymm30, -516(%rdx)
+// CHECK:  encoding: [0x62,0x62,0x7e,0x28,0x12,0xb2,0xfc,0xfd,0xff,0xff]
+          vpmovusqb %ymm30, -516(%rdx)
+
+// CHECK: vpmovqw %xmm18, %xmm19
+// CHECK:  encoding: [0x62,0xa2,0x7e,0x08,0x34,0xd3]
+          vpmovqw %xmm18, %xmm19
+
+// CHECK: vpmovqw %xmm18, %xmm19 {%k4}
+// CHECK:  encoding: [0x62,0xa2,0x7e,0x0c,0x34,0xd3]
+          vpmovqw %xmm18, %xmm19 {%k4}
+
+// CHECK: vpmovqw %xmm18, %xmm19 {%k4} {z}
+// CHECK:  encoding: [0x62,0xa2,0x7e,0x8c,0x34,0xd3]
+          vpmovqw %xmm18, %xmm19 {%k4} {z}
+
+// CHECK: vpmovqw %ymm22, %xmm19
+// CHECK:  encoding: [0x62,0xa2,0x7e,0x28,0x34,0xf3]
+          vpmovqw %ymm22, %xmm19
+
+// CHECK: vpmovqw %ymm22, %xmm19 {%k5}
+// CHECK:  encoding: [0x62,0xa2,0x7e,0x2d,0x34,0xf3]
+          vpmovqw %ymm22, %xmm19 {%k5}
+
+// CHECK: vpmovqw %ymm22, %xmm19 {%k5} {z}
+// CHECK:  encoding: [0x62,0xa2,0x7e,0xad,0x34,0xf3]
+          vpmovqw %ymm22, %xmm19 {%k5} {z}
+
+// CHECK: vpmovqw %xmm21, (%rcx)
+// CHECK:  encoding: [0x62,0xe2,0x7e,0x08,0x34,0x29]
+          vpmovqw %xmm21, (%rcx)
+
+// CHECK: vpmovqw %xmm21, (%rcx) {%k3}
+// CHECK:  encoding: [0x62,0xe2,0x7e,0x0b,0x34,0x29]
+          vpmovqw %xmm21, (%rcx) {%k3}
+
+// CHECK: vpmovqw %xmm21, 291(%rax,%r14,8)
+// CHECK:  encoding: [0x62,0xa2,0x7e,0x08,0x34,0xac,0xf0,0x23,0x01,0x00,0x00]
+          vpmovqw %xmm21, 291(%rax,%r14,8)
+
+// CHECK: vpmovqw %xmm21, 508(%rdx)
+// CHECK:  encoding: [0x62,0xe2,0x7e,0x08,0x34,0x6a,0x7f]
+          vpmovqw %xmm21, 508(%rdx)
+
+// CHECK: vpmovqw %xmm21, 512(%rdx)
+// CHECK:  encoding: [0x62,0xe2,0x7e,0x08,0x34,0xaa,0x00,0x02,0x00,0x00]
+          vpmovqw %xmm21, 512(%rdx)
+
+// CHECK: vpmovqw %xmm21, -512(%rdx)
+// CHECK:  encoding: [0x62,0xe2,0x7e,0x08,0x34,0x6a,0x80]
+          vpmovqw %xmm21, -512(%rdx)
+
+// CHECK: vpmovqw %xmm21, -516(%rdx)
+// CHECK:  encoding: [0x62,0xe2,0x7e,0x08,0x34,0xaa,0xfc,0xfd,0xff,0xff]
+          vpmovqw %xmm21, -516(%rdx)
+
+// CHECK: vpmovqw %ymm28, (%rcx)
+// CHECK:  encoding: [0x62,0x62,0x7e,0x28,0x34,0x21]
+          vpmovqw %ymm28, (%rcx)
+
+// CHECK: vpmovqw %ymm28, (%rcx) {%k6}
+// CHECK:  encoding: [0x62,0x62,0x7e,0x2e,0x34,0x21]
+          vpmovqw %ymm28, (%rcx) {%k6}
+
+// CHECK: vpmovqw %ymm28, 291(%rax,%r14,8)
+// CHECK:  encoding: [0x62,0x22,0x7e,0x28,0x34,0xa4,0xf0,0x23,0x01,0x00,0x00]
+          vpmovqw %ymm28, 291(%rax,%r14,8)
+
+// CHECK: vpmovqw %ymm28, 1016(%rdx)
+// CHECK:  encoding: [0x62,0x62,0x7e,0x28,0x34,0x62,0x7f]
+          vpmovqw %ymm28, 1016(%rdx)
+
+// CHECK: vpmovqw %ymm28, 1024(%rdx)
+// CHECK:  encoding: [0x62,0x62,0x7e,0x28,0x34,0xa2,0x00,0x04,0x00,0x00]
+          vpmovqw %ymm28, 1024(%rdx)
+
+// CHECK: vpmovqw %ymm28, -1024(%rdx)
+// CHECK:  encoding: [0x62,0x62,0x7e,0x28,0x34,0x62,0x80]
+          vpmovqw %ymm28, -1024(%rdx)
+
+// CHECK: vpmovqw %ymm28, -1032(%rdx)
+// CHECK:  encoding: [0x62,0x62,0x7e,0x28,0x34,0xa2,0xf8,0xfb,0xff,0xff]
+          vpmovqw %ymm28, -1032(%rdx)
+
+// CHECK: vpmovsqw %xmm18, %xmm26
+// CHECK:  encoding: [0x62,0x82,0x7e,0x08,0x24,0xd2]
+          vpmovsqw %xmm18, %xmm26
+
+// CHECK: vpmovsqw %xmm18, %xmm26 {%k7}
+// CHECK:  encoding: [0x62,0x82,0x7e,0x0f,0x24,0xd2]
+          vpmovsqw %xmm18, %xmm26 {%k7}
+
+// CHECK: vpmovsqw %xmm18, %xmm26 {%k7} {z}
+// CHECK:  encoding: [0x62,0x82,0x7e,0x8f,0x24,0xd2]
+          vpmovsqw %xmm18, %xmm26 {%k7} {z}
+
+// CHECK: vpmovsqw %ymm20, %xmm28
+// CHECK:  encoding: [0x62,0x82,0x7e,0x28,0x24,0xe4]
+          vpmovsqw %ymm20, %xmm28
+
+// CHECK: vpmovsqw %ymm20, %xmm28 {%k4}
+// CHECK:  encoding: [0x62,0x82,0x7e,0x2c,0x24,0xe4]
+          vpmovsqw %ymm20, %xmm28 {%k4}
+
+// CHECK: vpmovsqw %ymm20, %xmm28 {%k4} {z}
+// CHECK:  encoding: [0x62,0x82,0x7e,0xac,0x24,0xe4]
+          vpmovsqw %ymm20, %xmm28 {%k4} {z}
+
+// CHECK: vpmovsqw %xmm30, (%rcx)
+// CHECK:  encoding: [0x62,0x62,0x7e,0x08,0x24,0x31]
+          vpmovsqw %xmm30, (%rcx)
+
+// CHECK: vpmovsqw %xmm30, (%rcx) {%k4}
+// CHECK:  encoding: [0x62,0x62,0x7e,0x0c,0x24,0x31]
+          vpmovsqw %xmm30, (%rcx) {%k4}
+
+// CHECK: vpmovsqw %xmm30, 291(%rax,%r14,8)
+// CHECK:  encoding: [0x62,0x22,0x7e,0x08,0x24,0xb4,0xf0,0x23,0x01,0x00,0x00]
+          vpmovsqw %xmm30, 291(%rax,%r14,8)
+
+// CHECK: vpmovsqw %xmm30, 508(%rdx)
+// CHECK:  encoding: [0x62,0x62,0x7e,0x08,0x24,0x72,0x7f]
+          vpmovsqw %xmm30, 508(%rdx)
+
+// CHECK: vpmovsqw %xmm30, 512(%rdx)
+// CHECK:  encoding: [0x62,0x62,0x7e,0x08,0x24,0xb2,0x00,0x02,0x00,0x00]
+          vpmovsqw %xmm30, 512(%rdx)
+
+// CHECK: vpmovsqw %xmm30, -512(%rdx)
+// CHECK:  encoding: [0x62,0x62,0x7e,0x08,0x24,0x72,0x80]
+          vpmovsqw %xmm30, -512(%rdx)
+
+// CHECK: vpmovsqw %xmm30, -516(%rdx)
+// CHECK:  encoding: [0x62,0x62,0x7e,0x08,0x24,0xb2,0xfc,0xfd,0xff,0xff]
+          vpmovsqw %xmm30, -516(%rdx)
+
+// CHECK: vpmovsqw %ymm21, (%rcx)
+// CHECK:  encoding: [0x62,0xe2,0x7e,0x28,0x24,0x29]
+          vpmovsqw %ymm21, (%rcx)
+
+// CHECK: vpmovsqw %ymm21, (%rcx) {%k5}
+// CHECK:  encoding: [0x62,0xe2,0x7e,0x2d,0x24,0x29]
+          vpmovsqw %ymm21, (%rcx) {%k5}
+
+// CHECK: vpmovsqw %ymm21, 291(%rax,%r14,8)
+// CHECK:  encoding: [0x62,0xa2,0x7e,0x28,0x24,0xac,0xf0,0x23,0x01,0x00,0x00]
+          vpmovsqw %ymm21, 291(%rax,%r14,8)
+
+// CHECK: vpmovsqw %ymm21, 1016(%rdx)
+// CHECK:  encoding: [0x62,0xe2,0x7e,0x28,0x24,0x6a,0x7f]
+          vpmovsqw %ymm21, 1016(%rdx)
+
+// CHECK: vpmovsqw %ymm21, 1024(%rdx)
+// CHECK:  encoding: [0x62,0xe2,0x7e,0x28,0x24,0xaa,0x00,0x04,0x00,0x00]
+          vpmovsqw %ymm21, 1024(%rdx)
+
+// CHECK: vpmovsqw %ymm21, -1024(%rdx)
+// CHECK:  encoding: [0x62,0xe2,0x7e,0x28,0x24,0x6a,0x80]
+          vpmovsqw %ymm21, -1024(%rdx)
+
+// CHECK: vpmovsqw %ymm21, -1032(%rdx)
+// CHECK:  encoding: [0x62,0xe2,0x7e,0x28,0x24,0xaa,0xf8,0xfb,0xff,0xff]
+          vpmovsqw %ymm21, -1032(%rdx)
+
+// CHECK: vpmovusqw %xmm20, %xmm29
+// CHECK:  encoding: [0x62,0x82,0x7e,0x08,0x14,0xe5]
+          vpmovusqw %xmm20, %xmm29
+
+// CHECK: vpmovusqw %xmm20, %xmm29 {%k1}
+// CHECK:  encoding: [0x62,0x82,0x7e,0x09,0x14,0xe5]
+          vpmovusqw %xmm20, %xmm29 {%k1}
+
+// CHECK: vpmovusqw %xmm20, %xmm29 {%k1} {z}
+// CHECK:  encoding: [0x62,0x82,0x7e,0x89,0x14,0xe5]
+          vpmovusqw %xmm20, %xmm29 {%k1} {z}
+
+// CHECK: vpmovusqw %ymm21, %xmm20
+// CHECK:  encoding: [0x62,0xa2,0x7e,0x28,0x14,0xec]
+          vpmovusqw %ymm21, %xmm20
+
+// CHECK: vpmovusqw %ymm21, %xmm20 {%k5}
+// CHECK:  encoding: [0x62,0xa2,0x7e,0x2d,0x14,0xec]
+          vpmovusqw %ymm21, %xmm20 {%k5}
+
+// CHECK: vpmovusqw %ymm21, %xmm20 {%k5} {z}
+// CHECK:  encoding: [0x62,0xa2,0x7e,0xad,0x14,0xec]
+          vpmovusqw %ymm21, %xmm20 {%k5} {z}
+
+// CHECK: vpmovusqw %xmm18, (%rcx)
+// CHECK:  encoding: [0x62,0xe2,0x7e,0x08,0x14,0x11]
+          vpmovusqw %xmm18, (%rcx)
+
+// CHECK: vpmovusqw %xmm18, (%rcx) {%k1}
+// CHECK:  encoding: [0x62,0xe2,0x7e,0x09,0x14,0x11]
+          vpmovusqw %xmm18, (%rcx) {%k1}
+
+// CHECK: vpmovusqw %xmm18, 291(%rax,%r14,8)
+// CHECK:  encoding: [0x62,0xa2,0x7e,0x08,0x14,0x94,0xf0,0x23,0x01,0x00,0x00]
+          vpmovusqw %xmm18, 291(%rax,%r14,8)
+
+// CHECK: vpmovusqw %xmm18, 508(%rdx)
+// CHECK:  encoding: [0x62,0xe2,0x7e,0x08,0x14,0x52,0x7f]
+          vpmovusqw %xmm18, 508(%rdx)
+
+// CHECK: vpmovusqw %xmm18, 512(%rdx)
+// CHECK:  encoding: [0x62,0xe2,0x7e,0x08,0x14,0x92,0x00,0x02,0x00,0x00]
+          vpmovusqw %xmm18, 512(%rdx)
+
+// CHECK: vpmovusqw %xmm18, -512(%rdx)
+// CHECK:  encoding: [0x62,0xe2,0x7e,0x08,0x14,0x52,0x80]
+          vpmovusqw %xmm18, -512(%rdx)
+
+// CHECK: vpmovusqw %xmm18, -516(%rdx)
+// CHECK:  encoding: [0x62,0xe2,0x7e,0x08,0x14,0x92,0xfc,0xfd,0xff,0xff]
+          vpmovusqw %xmm18, -516(%rdx)
+
+// CHECK: vpmovusqw %ymm18, (%rcx)
+// CHECK:  encoding: [0x62,0xe2,0x7e,0x28,0x14,0x11]
+          vpmovusqw %ymm18, (%rcx)
+
+// CHECK: vpmovusqw %ymm18, (%rcx) {%k2}
+// CHECK:  encoding: [0x62,0xe2,0x7e,0x2a,0x14,0x11]
+          vpmovusqw %ymm18, (%rcx) {%k2}
+
+// CHECK: vpmovusqw %ymm18, 291(%rax,%r14,8)
+// CHECK:  encoding: [0x62,0xa2,0x7e,0x28,0x14,0x94,0xf0,0x23,0x01,0x00,0x00]
+          vpmovusqw %ymm18, 291(%rax,%r14,8)
+
+// CHECK: vpmovusqw %ymm18, 1016(%rdx)
+// CHECK:  encoding: [0x62,0xe2,0x7e,0x28,0x14,0x52,0x7f]
+          vpmovusqw %ymm18, 1016(%rdx)
+
+// CHECK: vpmovusqw %ymm18, 1024(%rdx)
+// CHECK:  encoding: [0x62,0xe2,0x7e,0x28,0x14,0x92,0x00,0x04,0x00,0x00]
+          vpmovusqw %ymm18, 1024(%rdx)
+
+// CHECK: vpmovusqw %ymm18, -1024(%rdx)
+// CHECK:  encoding: [0x62,0xe2,0x7e,0x28,0x14,0x52,0x80]
+          vpmovusqw %ymm18, -1024(%rdx)
+
+// CHECK: vpmovusqw %ymm18, -1032(%rdx)
+// CHECK:  encoding: [0x62,0xe2,0x7e,0x28,0x14,0x92,0xf8,0xfb,0xff,0xff]
+          vpmovusqw %ymm18, -1032(%rdx)
+
+// CHECK: vpmovqd %xmm25, %xmm21
+// CHECK:  encoding: [0x62,0x22,0x7e,0x08,0x35,0xcd]
+          vpmovqd %xmm25, %xmm21
+
+// CHECK: vpmovqd %xmm25, %xmm21 {%k5}
+// CHECK:  encoding: [0x62,0x22,0x7e,0x0d,0x35,0xcd]
+          vpmovqd %xmm25, %xmm21 {%k5}
+
+// CHECK: vpmovqd %xmm25, %xmm21 {%k5} {z}
+// CHECK:  encoding: [0x62,0x22,0x7e,0x8d,0x35,0xcd]
+          vpmovqd %xmm25, %xmm21 {%k5} {z}
+
+// CHECK: vpmovqd %ymm22, %xmm21
+// CHECK:  encoding: [0x62,0xa2,0x7e,0x28,0x35,0xf5]
+          vpmovqd %ymm22, %xmm21
+
+// CHECK: vpmovqd %ymm22, %xmm21 {%k6}
+// CHECK:  encoding: [0x62,0xa2,0x7e,0x2e,0x35,0xf5]
+          vpmovqd %ymm22, %xmm21 {%k6}
+
+// CHECK: vpmovqd %ymm22, %xmm21 {%k6} {z}
+// CHECK:  encoding: [0x62,0xa2,0x7e,0xae,0x35,0xf5]
+          vpmovqd %ymm22, %xmm21 {%k6} {z}
+
+// CHECK: vpmovqd %xmm29, (%rcx)
+// CHECK:  encoding: [0x62,0x62,0x7e,0x08,0x35,0x29]
+          vpmovqd %xmm29, (%rcx)
+
+// CHECK: vpmovqd %xmm29, (%rcx) {%k6}
+// CHECK:  encoding: [0x62,0x62,0x7e,0x0e,0x35,0x29]
+          vpmovqd %xmm29, (%rcx) {%k6}
+
+// CHECK: vpmovqd %xmm29, 291(%rax,%r14,8)
+// CHECK:  encoding: [0x62,0x22,0x7e,0x08,0x35,0xac,0xf0,0x23,0x01,0x00,0x00]
+          vpmovqd %xmm29, 291(%rax,%r14,8)
+
+// CHECK: vpmovqd %xmm29, 1016(%rdx)
+// CHECK:  encoding: [0x62,0x62,0x7e,0x08,0x35,0x6a,0x7f]
+          vpmovqd %xmm29, 1016(%rdx)
+
+// CHECK: vpmovqd %xmm29, 1024(%rdx)
+// CHECK:  encoding: [0x62,0x62,0x7e,0x08,0x35,0xaa,0x00,0x04,0x00,0x00]
+          vpmovqd %xmm29, 1024(%rdx)
+
+// CHECK: vpmovqd %xmm29, -1024(%rdx)
+// CHECK:  encoding: [0x62,0x62,0x7e,0x08,0x35,0x6a,0x80]
+          vpmovqd %xmm29, -1024(%rdx)
+
+// CHECK: vpmovqd %xmm29, -1032(%rdx)
+// CHECK:  encoding: [0x62,0x62,0x7e,0x08,0x35,0xaa,0xf8,0xfb,0xff,0xff]
+          vpmovqd %xmm29, -1032(%rdx)
+
+// CHECK: vpmovqd %ymm30, (%rcx)
+// CHECK:  encoding: [0x62,0x62,0x7e,0x28,0x35,0x31]
+          vpmovqd %ymm30, (%rcx)
+
+// CHECK: vpmovqd %ymm30, (%rcx) {%k2}
+// CHECK:  encoding: [0x62,0x62,0x7e,0x2a,0x35,0x31]
+          vpmovqd %ymm30, (%rcx) {%k2}
+
+// CHECK: vpmovqd %ymm30, 291(%rax,%r14,8)
+// CHECK:  encoding: [0x62,0x22,0x7e,0x28,0x35,0xb4,0xf0,0x23,0x01,0x00,0x00]
+          vpmovqd %ymm30, 291(%rax,%r14,8)
+
+// CHECK: vpmovqd %ymm30, 2032(%rdx)
+// CHECK:  encoding: [0x62,0x62,0x7e,0x28,0x35,0x72,0x7f]
+          vpmovqd %ymm30, 2032(%rdx)
+
+// CHECK: vpmovqd %ymm30, 2048(%rdx)
+// CHECK:  encoding: [0x62,0x62,0x7e,0x28,0x35,0xb2,0x00,0x08,0x00,0x00]
+          vpmovqd %ymm30, 2048(%rdx)
+
+// CHECK: vpmovqd %ymm30, -2048(%rdx)
+// CHECK:  encoding: [0x62,0x62,0x7e,0x28,0x35,0x72,0x80]
+          vpmovqd %ymm30, -2048(%rdx)
+
+// CHECK: vpmovqd %ymm30, -2064(%rdx)
+// CHECK:  encoding: [0x62,0x62,0x7e,0x28,0x35,0xb2,0xf0,0xf7,0xff,0xff]
+          vpmovqd %ymm30, -2064(%rdx)
+
+// CHECK: vpmovsqd %xmm21, %xmm21
+// CHECK:  encoding: [0x62,0xa2,0x7e,0x08,0x25,0xed]
+          vpmovsqd %xmm21, %xmm21
+
+// CHECK: vpmovsqd %xmm21, %xmm21 {%k2}
+// CHECK:  encoding: [0x62,0xa2,0x7e,0x0a,0x25,0xed]
+          vpmovsqd %xmm21, %xmm21 {%k2}
+
+// CHECK: vpmovsqd %xmm21, %xmm21 {%k2} {z}
+// CHECK:  encoding: [0x62,0xa2,0x7e,0x8a,0x25,0xed]
+          vpmovsqd %xmm21, %xmm21 {%k2} {z}
+
+// CHECK: vpmovsqd %ymm29, %xmm29
+// CHECK:  encoding: [0x62,0x02,0x7e,0x28,0x25,0xed]
+          vpmovsqd %ymm29, %xmm29
+
+// CHECK: vpmovsqd %ymm29, %xmm29 {%k4}
+// CHECK:  encoding: [0x62,0x02,0x7e,0x2c,0x25,0xed]
+          vpmovsqd %ymm29, %xmm29 {%k4}
+
+// CHECK: vpmovsqd %ymm29, %xmm29 {%k4} {z}
+// CHECK:  encoding: [0x62,0x02,0x7e,0xac,0x25,0xed]
+          vpmovsqd %ymm29, %xmm29 {%k4} {z}
+
+// CHECK: vpmovsqd %xmm17, (%rcx)
+// CHECK:  encoding: [0x62,0xe2,0x7e,0x08,0x25,0x09]
+          vpmovsqd %xmm17, (%rcx)
+
+// CHECK: vpmovsqd %xmm17, (%rcx) {%k2}
+// CHECK:  encoding: [0x62,0xe2,0x7e,0x0a,0x25,0x09]
+          vpmovsqd %xmm17, (%rcx) {%k2}
+
+// CHECK: vpmovsqd %xmm17, 291(%rax,%r14,8)
+// CHECK:  encoding: [0x62,0xa2,0x7e,0x08,0x25,0x8c,0xf0,0x23,0x01,0x00,0x00]
+          vpmovsqd %xmm17, 291(%rax,%r14,8)
+
+// CHECK: vpmovsqd %xmm17, 1016(%rdx)
+// CHECK:  encoding: [0x62,0xe2,0x7e,0x08,0x25,0x4a,0x7f]
+          vpmovsqd %xmm17, 1016(%rdx)
+
+// CHECK: vpmovsqd %xmm17, 1024(%rdx)
+// CHECK:  encoding: [0x62,0xe2,0x7e,0x08,0x25,0x8a,0x00,0x04,0x00,0x00]
+          vpmovsqd %xmm17, 1024(%rdx)
+
+// CHECK: vpmovsqd %xmm17, -1024(%rdx)
+// CHECK:  encoding: [0x62,0xe2,0x7e,0x08,0x25,0x4a,0x80]
+          vpmovsqd %xmm17, -1024(%rdx)
+
+// CHECK: vpmovsqd %xmm17, -1032(%rdx)
+// CHECK:  encoding: [0x62,0xe2,0x7e,0x08,0x25,0x8a,0xf8,0xfb,0xff,0xff]
+          vpmovsqd %xmm17, -1032(%rdx)
+
+// CHECK: vpmovsqd %ymm23, (%rcx)
+// CHECK:  encoding: [0x62,0xe2,0x7e,0x28,0x25,0x39]
+          vpmovsqd %ymm23, (%rcx)
+
+// CHECK: vpmovsqd %ymm23, (%rcx) {%k5}
+// CHECK:  encoding: [0x62,0xe2,0x7e,0x2d,0x25,0x39]
+          vpmovsqd %ymm23, (%rcx) {%k5}
+
+// CHECK: vpmovsqd %ymm23, 291(%rax,%r14,8)
+// CHECK:  encoding: [0x62,0xa2,0x7e,0x28,0x25,0xbc,0xf0,0x23,0x01,0x00,0x00]
+          vpmovsqd %ymm23, 291(%rax,%r14,8)
+
+// CHECK: vpmovsqd %ymm23, 2032(%rdx)
+// CHECK:  encoding: [0x62,0xe2,0x7e,0x28,0x25,0x7a,0x7f]
+          vpmovsqd %ymm23, 2032(%rdx)
+
+// CHECK: vpmovsqd %ymm23, 2048(%rdx)
+// CHECK:  encoding: [0x62,0xe2,0x7e,0x28,0x25,0xba,0x00,0x08,0x00,0x00]
+          vpmovsqd %ymm23, 2048(%rdx)
+
+// CHECK: vpmovsqd %ymm23, -2048(%rdx)
+// CHECK:  encoding: [0x62,0xe2,0x7e,0x28,0x25,0x7a,0x80]
+          vpmovsqd %ymm23, -2048(%rdx)
+
+// CHECK: vpmovsqd %ymm23, -2064(%rdx)
+// CHECK:  encoding: [0x62,0xe2,0x7e,0x28,0x25,0xba,0xf0,0xf7,0xff,0xff]
+          vpmovsqd %ymm23, -2064(%rdx)
+
+// CHECK: vpmovusqd %xmm21, %xmm25
+// CHECK:  encoding: [0x62,0x82,0x7e,0x08,0x15,0xe9]
+          vpmovusqd %xmm21, %xmm25
+
+// CHECK: vpmovusqd %xmm21, %xmm25 {%k5}
+// CHECK:  encoding: [0x62,0x82,0x7e,0x0d,0x15,0xe9]
+          vpmovusqd %xmm21, %xmm25 {%k5}
+
+// CHECK: vpmovusqd %xmm21, %xmm25 {%k5} {z}
+// CHECK:  encoding: [0x62,0x82,0x7e,0x8d,0x15,0xe9]
+          vpmovusqd %xmm21, %xmm25 {%k5} {z}
+
+// CHECK: vpmovusqd %ymm21, %xmm20
+// CHECK:  encoding: [0x62,0xa2,0x7e,0x28,0x15,0xec]
+          vpmovusqd %ymm21, %xmm20
+
+// CHECK: vpmovusqd %ymm21, %xmm20 {%k2}
+// CHECK:  encoding: [0x62,0xa2,0x7e,0x2a,0x15,0xec]
+          vpmovusqd %ymm21, %xmm20 {%k2}
+
+// CHECK: vpmovusqd %ymm21, %xmm20 {%k2} {z}
+// CHECK:  encoding: [0x62,0xa2,0x7e,0xaa,0x15,0xec]
+          vpmovusqd %ymm21, %xmm20 {%k2} {z}
+
+// CHECK: vpmovusqd %xmm18, (%rcx)
+// CHECK:  encoding: [0x62,0xe2,0x7e,0x08,0x15,0x11]
+          vpmovusqd %xmm18, (%rcx)
+
+// CHECK: vpmovusqd %xmm18, (%rcx) {%k1}
+// CHECK:  encoding: [0x62,0xe2,0x7e,0x09,0x15,0x11]
+          vpmovusqd %xmm18, (%rcx) {%k1}
+
+// CHECK: vpmovusqd %xmm18, 291(%rax,%r14,8)
+// CHECK:  encoding: [0x62,0xa2,0x7e,0x08,0x15,0x94,0xf0,0x23,0x01,0x00,0x00]
+          vpmovusqd %xmm18, 291(%rax,%r14,8)
+
+// CHECK: vpmovusqd %xmm18, 1016(%rdx)
+// CHECK:  encoding: [0x62,0xe2,0x7e,0x08,0x15,0x52,0x7f]
+          vpmovusqd %xmm18, 1016(%rdx)
+
+// CHECK: vpmovusqd %xmm18, 1024(%rdx)
+// CHECK:  encoding: [0x62,0xe2,0x7e,0x08,0x15,0x92,0x00,0x04,0x00,0x00]
+          vpmovusqd %xmm18, 1024(%rdx)
+
+// CHECK: vpmovusqd %xmm18, -1024(%rdx)
+// CHECK:  encoding: [0x62,0xe2,0x7e,0x08,0x15,0x52,0x80]
+          vpmovusqd %xmm18, -1024(%rdx)
+
+// CHECK: vpmovusqd %xmm18, -1032(%rdx)
+// CHECK:  encoding: [0x62,0xe2,0x7e,0x08,0x15,0x92,0xf8,0xfb,0xff,0xff]
+          vpmovusqd %xmm18, -1032(%rdx)
+
+// CHECK: vpmovusqd %ymm29, (%rcx)
+// CHECK:  encoding: [0x62,0x62,0x7e,0x28,0x15,0x29]
+          vpmovusqd %ymm29, (%rcx)
+
+// CHECK: vpmovusqd %ymm29, (%rcx) {%k6}
+// CHECK:  encoding: [0x62,0x62,0x7e,0x2e,0x15,0x29]
+          vpmovusqd %ymm29, (%rcx) {%k6}
+
+// CHECK: vpmovusqd %ymm29, 291(%rax,%r14,8)
+// CHECK:  encoding: [0x62,0x22,0x7e,0x28,0x15,0xac,0xf0,0x23,0x01,0x00,0x00]
+          vpmovusqd %ymm29, 291(%rax,%r14,8)
+
+// CHECK: vpmovusqd %ymm29, 2032(%rdx)
+// CHECK:  encoding: [0x62,0x62,0x7e,0x28,0x15,0x6a,0x7f]
+          vpmovusqd %ymm29, 2032(%rdx)
+
+// CHECK: vpmovusqd %ymm29, 2048(%rdx)
+// CHECK:  encoding: [0x62,0x62,0x7e,0x28,0x15,0xaa,0x00,0x08,0x00,0x00]
+          vpmovusqd %ymm29, 2048(%rdx)
+
+// CHECK: vpmovusqd %ymm29, -2048(%rdx)
+// CHECK:  encoding: [0x62,0x62,0x7e,0x28,0x15,0x6a,0x80]
+          vpmovusqd %ymm29, -2048(%rdx)
+
+// CHECK: vpmovusqd %ymm29, -2064(%rdx)
+// CHECK:  encoding: [0x62,0x62,0x7e,0x28,0x15,0xaa,0xf0,0xf7,0xff,0xff]
+          vpmovusqd %ymm29, -2064(%rdx)
+
+// CHECK: vpmovdb %xmm21, %xmm30
+// CHECK:  encoding: [0x62,0x82,0x7e,0x08,0x31,0xee]
+          vpmovdb %xmm21, %xmm30
+
+// CHECK: vpmovdb %xmm21, %xmm30 {%k3}
+// CHECK:  encoding: [0x62,0x82,0x7e,0x0b,0x31,0xee]
+          vpmovdb %xmm21, %xmm30 {%k3}
+
+// CHECK: vpmovdb %xmm21, %xmm30 {%k3} {z}
+// CHECK:  encoding: [0x62,0x82,0x7e,0x8b,0x31,0xee]
+          vpmovdb %xmm21, %xmm30 {%k3} {z}
+
+// CHECK: vpmovdb %ymm21, %xmm23
+// CHECK:  encoding: [0x62,0xa2,0x7e,0x28,0x31,0xef]
+          vpmovdb %ymm21, %xmm23
+
+// CHECK: vpmovdb %ymm21, %xmm23 {%k4}
+// CHECK:  encoding: [0x62,0xa2,0x7e,0x2c,0x31,0xef]
+          vpmovdb %ymm21, %xmm23 {%k4}
+
+// CHECK: vpmovdb %ymm21, %xmm23 {%k4} {z}
+// CHECK:  encoding: [0x62,0xa2,0x7e,0xac,0x31,0xef]
+          vpmovdb %ymm21, %xmm23 {%k4} {z}
+
+// CHECK: vpmovdb %xmm29, (%rcx)
+// CHECK:  encoding: [0x62,0x62,0x7e,0x08,0x31,0x29]
+          vpmovdb %xmm29, (%rcx)
+
+// CHECK: vpmovdb %xmm29, (%rcx) {%k3}
+// CHECK:  encoding: [0x62,0x62,0x7e,0x0b,0x31,0x29]
+          vpmovdb %xmm29, (%rcx) {%k3}
+
+// CHECK: vpmovdb %xmm29, 291(%rax,%r14,8)
+// CHECK:  encoding: [0x62,0x22,0x7e,0x08,0x31,0xac,0xf0,0x23,0x01,0x00,0x00]
+          vpmovdb %xmm29, 291(%rax,%r14,8)
+
+// CHECK: vpmovdb %xmm29, 508(%rdx)
+// CHECK:  encoding: [0x62,0x62,0x7e,0x08,0x31,0x6a,0x7f]
+          vpmovdb %xmm29, 508(%rdx)
+
+// CHECK: vpmovdb %xmm29, 512(%rdx)
+// CHECK:  encoding: [0x62,0x62,0x7e,0x08,0x31,0xaa,0x00,0x02,0x00,0x00]
+          vpmovdb %xmm29, 512(%rdx)
+
+// CHECK: vpmovdb %xmm29, -512(%rdx)
+// CHECK:  encoding: [0x62,0x62,0x7e,0x08,0x31,0x6a,0x80]
+          vpmovdb %xmm29, -512(%rdx)
+
+// CHECK: vpmovdb %xmm29, -516(%rdx)
+// CHECK:  encoding: [0x62,0x62,0x7e,0x08,0x31,0xaa,0xfc,0xfd,0xff,0xff]
+          vpmovdb %xmm29, -516(%rdx)
+
+// CHECK: vpmovdb %ymm26, (%rcx)
+// CHECK:  encoding: [0x62,0x62,0x7e,0x28,0x31,0x11]
+          vpmovdb %ymm26, (%rcx)
+
+// CHECK: vpmovdb %ymm26, (%rcx) {%k6}
+// CHECK:  encoding: [0x62,0x62,0x7e,0x2e,0x31,0x11]
+          vpmovdb %ymm26, (%rcx) {%k6}
+
+// CHECK: vpmovdb %ymm26, 291(%rax,%r14,8)
+// CHECK:  encoding: [0x62,0x22,0x7e,0x28,0x31,0x94,0xf0,0x23,0x01,0x00,0x00]
+          vpmovdb %ymm26, 291(%rax,%r14,8)
+
+// CHECK: vpmovdb %ymm26, 1016(%rdx)
+// CHECK:  encoding: [0x62,0x62,0x7e,0x28,0x31,0x52,0x7f]
+          vpmovdb %ymm26, 1016(%rdx)
+
+// CHECK: vpmovdb %ymm26, 1024(%rdx)
+// CHECK:  encoding: [0x62,0x62,0x7e,0x28,0x31,0x92,0x00,0x04,0x00,0x00]
+          vpmovdb %ymm26, 1024(%rdx)
+
+// CHECK: vpmovdb %ymm26, -1024(%rdx)
+// CHECK:  encoding: [0x62,0x62,0x7e,0x28,0x31,0x52,0x80]
+          vpmovdb %ymm26, -1024(%rdx)
+
+// CHECK: vpmovdb %ymm26, -1032(%rdx)
+// CHECK:  encoding: [0x62,0x62,0x7e,0x28,0x31,0x92,0xf8,0xfb,0xff,0xff]
+          vpmovdb %ymm26, -1032(%rdx)
+
+// CHECK: vpmovsdb %xmm27, %xmm30
+// CHECK:  encoding: [0x62,0x02,0x7e,0x08,0x21,0xde]
+          vpmovsdb %xmm27, %xmm30
+
+// CHECK: vpmovsdb %xmm27, %xmm30 {%k1}
+// CHECK:  encoding: [0x62,0x02,0x7e,0x09,0x21,0xde]
+          vpmovsdb %xmm27, %xmm30 {%k1}
+
+// CHECK: vpmovsdb %xmm27, %xmm30 {%k1} {z}
+// CHECK:  encoding: [0x62,0x02,0x7e,0x89,0x21,0xde]
+          vpmovsdb %xmm27, %xmm30 {%k1} {z}
+
+// CHECK: vpmovsdb %ymm27, %xmm26
+// CHECK:  encoding: [0x62,0x02,0x7e,0x28,0x21,0xda]
+          vpmovsdb %ymm27, %xmm26
+
+// CHECK: vpmovsdb %ymm27, %xmm26 {%k3}
+// CHECK:  encoding: [0x62,0x02,0x7e,0x2b,0x21,0xda]
+          vpmovsdb %ymm27, %xmm26 {%k3}
+
+// CHECK: vpmovsdb %ymm27, %xmm26 {%k3} {z}
+// CHECK:  encoding: [0x62,0x02,0x7e,0xab,0x21,0xda]
+          vpmovsdb %ymm27, %xmm26 {%k3} {z}
+
+// CHECK: vpmovsdb %xmm30, (%rcx)
+// CHECK:  encoding: [0x62,0x62,0x7e,0x08,0x21,0x31]
+          vpmovsdb %xmm30, (%rcx)
+
+// CHECK: vpmovsdb %xmm30, (%rcx) {%k3}
+// CHECK:  encoding: [0x62,0x62,0x7e,0x0b,0x21,0x31]
+          vpmovsdb %xmm30, (%rcx) {%k3}
+
+// CHECK: vpmovsdb %xmm30, 291(%rax,%r14,8)
+// CHECK:  encoding: [0x62,0x22,0x7e,0x08,0x21,0xb4,0xf0,0x23,0x01,0x00,0x00]
+          vpmovsdb %xmm30, 291(%rax,%r14,8)
+
+// CHECK: vpmovsdb %xmm30, 508(%rdx)
+// CHECK:  encoding: [0x62,0x62,0x7e,0x08,0x21,0x72,0x7f]
+          vpmovsdb %xmm30, 508(%rdx)
+
+// CHECK: vpmovsdb %xmm30, 512(%rdx)
+// CHECK:  encoding: [0x62,0x62,0x7e,0x08,0x21,0xb2,0x00,0x02,0x00,0x00]
+          vpmovsdb %xmm30, 512(%rdx)
+
+// CHECK: vpmovsdb %xmm30, -512(%rdx)
+// CHECK:  encoding: [0x62,0x62,0x7e,0x08,0x21,0x72,0x80]
+          vpmovsdb %xmm30, -512(%rdx)
+
+// CHECK: vpmovsdb %xmm30, -516(%rdx)
+// CHECK:  encoding: [0x62,0x62,0x7e,0x08,0x21,0xb2,0xfc,0xfd,0xff,0xff]
+          vpmovsdb %xmm30, -516(%rdx)
+
+// CHECK: vpmovsdb %ymm25, (%rcx)
+// CHECK:  encoding: [0x62,0x62,0x7e,0x28,0x21,0x09]
+          vpmovsdb %ymm25, (%rcx)
+
+// CHECK: vpmovsdb %ymm25, (%rcx) {%k5}
+// CHECK:  encoding: [0x62,0x62,0x7e,0x2d,0x21,0x09]
+          vpmovsdb %ymm25, (%rcx) {%k5}
+
+// CHECK: vpmovsdb %ymm25, 291(%rax,%r14,8)
+// CHECK:  encoding: [0x62,0x22,0x7e,0x28,0x21,0x8c,0xf0,0x23,0x01,0x00,0x00]
+          vpmovsdb %ymm25, 291(%rax,%r14,8)
+
+// CHECK: vpmovsdb %ymm25, 1016(%rdx)
+// CHECK:  encoding: [0x62,0x62,0x7e,0x28,0x21,0x4a,0x7f]
+          vpmovsdb %ymm25, 1016(%rdx)
+
+// CHECK: vpmovsdb %ymm25, 1024(%rdx)
+// CHECK:  encoding: [0x62,0x62,0x7e,0x28,0x21,0x8a,0x00,0x04,0x00,0x00]
+          vpmovsdb %ymm25, 1024(%rdx)
+
+// CHECK: vpmovsdb %ymm25, -1024(%rdx)
+// CHECK:  encoding: [0x62,0x62,0x7e,0x28,0x21,0x4a,0x80]
+          vpmovsdb %ymm25, -1024(%rdx)
+
+// CHECK: vpmovsdb %ymm25, -1032(%rdx)
+// CHECK:  encoding: [0x62,0x62,0x7e,0x28,0x21,0x8a,0xf8,0xfb,0xff,0xff]
+          vpmovsdb %ymm25, -1032(%rdx)
+
+// CHECK: vpmovusdb %xmm29, %xmm30
+// CHECK:  encoding: [0x62,0x02,0x7e,0x08,0x11,0xee]
+          vpmovusdb %xmm29, %xmm30
+
+// CHECK: vpmovusdb %xmm29, %xmm30 {%k7}
+// CHECK:  encoding: [0x62,0x02,0x7e,0x0f,0x11,0xee]
+          vpmovusdb %xmm29, %xmm30 {%k7}
+
+// CHECK: vpmovusdb %xmm29, %xmm30 {%k7} {z}
+// CHECK:  encoding: [0x62,0x02,0x7e,0x8f,0x11,0xee]
+          vpmovusdb %xmm29, %xmm30 {%k7} {z}
+
+// CHECK: vpmovusdb %ymm17, %xmm23
+// CHECK:  encoding: [0x62,0xa2,0x7e,0x28,0x11,0xcf]
+          vpmovusdb %ymm17, %xmm23
+
+// CHECK: vpmovusdb %ymm17, %xmm23 {%k6}
+// CHECK:  encoding: [0x62,0xa2,0x7e,0x2e,0x11,0xcf]
+          vpmovusdb %ymm17, %xmm23 {%k6}
+
+// CHECK: vpmovusdb %ymm17, %xmm23 {%k6} {z}
+// CHECK:  encoding: [0x62,0xa2,0x7e,0xae,0x11,0xcf]
+          vpmovusdb %ymm17, %xmm23 {%k6} {z}
+
+// CHECK: vpmovusdb %xmm26, (%rcx)
+// CHECK:  encoding: [0x62,0x62,0x7e,0x08,0x11,0x11]
+          vpmovusdb %xmm26, (%rcx)
+
+// CHECK: vpmovusdb %xmm26, (%rcx) {%k7}
+// CHECK:  encoding: [0x62,0x62,0x7e,0x0f,0x11,0x11]
+          vpmovusdb %xmm26, (%rcx) {%k7}
+
+// CHECK: vpmovusdb %xmm26, 291(%rax,%r14,8)
+// CHECK:  encoding: [0x62,0x22,0x7e,0x08,0x11,0x94,0xf0,0x23,0x01,0x00,0x00]
+          vpmovusdb %xmm26, 291(%rax,%r14,8)
+
+// CHECK: vpmovusdb %xmm26, 508(%rdx)
+// CHECK:  encoding: [0x62,0x62,0x7e,0x08,0x11,0x52,0x7f]
+          vpmovusdb %xmm26, 508(%rdx)
+
+// CHECK: vpmovusdb %xmm26, 512(%rdx)
+// CHECK:  encoding: [0x62,0x62,0x7e,0x08,0x11,0x92,0x00,0x02,0x00,0x00]
+          vpmovusdb %xmm26, 512(%rdx)
+
+// CHECK: vpmovusdb %xmm26, -512(%rdx)
+// CHECK:  encoding: [0x62,0x62,0x7e,0x08,0x11,0x52,0x80]
+          vpmovusdb %xmm26, -512(%rdx)
+
+// CHECK: vpmovusdb %xmm26, -516(%rdx)
+// CHECK:  encoding: [0x62,0x62,0x7e,0x08,0x11,0x92,0xfc,0xfd,0xff,0xff]
+          vpmovusdb %xmm26, -516(%rdx)
+
+// CHECK: vpmovusdb %ymm25, (%rcx)
+// CHECK:  encoding: [0x62,0x62,0x7e,0x28,0x11,0x09]
+          vpmovusdb %ymm25, (%rcx)
+
+// CHECK: vpmovusdb %ymm25, (%rcx) {%k6}
+// CHECK:  encoding: [0x62,0x62,0x7e,0x2e,0x11,0x09]
+          vpmovusdb %ymm25, (%rcx) {%k6}
+
+// CHECK: vpmovusdb %ymm25, 291(%rax,%r14,8)
+// CHECK:  encoding: [0x62,0x22,0x7e,0x28,0x11,0x8c,0xf0,0x23,0x01,0x00,0x00]
+          vpmovusdb %ymm25, 291(%rax,%r14,8)
+
+// CHECK: vpmovusdb %ymm25, 1016(%rdx)
+// CHECK:  encoding: [0x62,0x62,0x7e,0x28,0x11,0x4a,0x7f]
+          vpmovusdb %ymm25, 1016(%rdx)
+
+// CHECK: vpmovusdb %ymm25, 1024(%rdx)
+// CHECK:  encoding: [0x62,0x62,0x7e,0x28,0x11,0x8a,0x00,0x04,0x00,0x00]
+          vpmovusdb %ymm25, 1024(%rdx)
+
+// CHECK: vpmovusdb %ymm25, -1024(%rdx)
+// CHECK:  encoding: [0x62,0x62,0x7e,0x28,0x11,0x4a,0x80]
+          vpmovusdb %ymm25, -1024(%rdx)
+
+// CHECK: vpmovusdb %ymm25, -1032(%rdx)
+// CHECK:  encoding: [0x62,0x62,0x7e,0x28,0x11,0x8a,0xf8,0xfb,0xff,0xff]
+          vpmovusdb %ymm25, -1032(%rdx)
+
+// CHECK: vpmovdw %xmm25, %xmm17
+// CHECK:  encoding: [0x62,0x22,0x7e,0x08,0x33,0xc9]
+          vpmovdw %xmm25, %xmm17
+
+// CHECK: vpmovdw %xmm25, %xmm17 {%k5}
+// CHECK:  encoding: [0x62,0x22,0x7e,0x0d,0x33,0xc9]
+          vpmovdw %xmm25, %xmm17 {%k5}
+
+// CHECK: vpmovdw %xmm25, %xmm17 {%k5} {z}
+// CHECK:  encoding: [0x62,0x22,0x7e,0x8d,0x33,0xc9]
+          vpmovdw %xmm25, %xmm17 {%k5} {z}
+
+// CHECK: vpmovdw %ymm19, %xmm25
+// CHECK:  encoding: [0x62,0x82,0x7e,0x28,0x33,0xd9]
+          vpmovdw %ymm19, %xmm25
+
+// CHECK: vpmovdw %ymm19, %xmm25 {%k4}
+// CHECK:  encoding: [0x62,0x82,0x7e,0x2c,0x33,0xd9]
+          vpmovdw %ymm19, %xmm25 {%k4}
+
+// CHECK: vpmovdw %ymm19, %xmm25 {%k4} {z}
+// CHECK:  encoding: [0x62,0x82,0x7e,0xac,0x33,0xd9]
+          vpmovdw %ymm19, %xmm25 {%k4} {z}
+
+// CHECK: vpmovdw %xmm21, (%rcx)
+// CHECK:  encoding: [0x62,0xe2,0x7e,0x08,0x33,0x29]
+          vpmovdw %xmm21, (%rcx)
+
+// CHECK: vpmovdw %xmm21, (%rcx) {%k2}
+// CHECK:  encoding: [0x62,0xe2,0x7e,0x0a,0x33,0x29]
+          vpmovdw %xmm21, (%rcx) {%k2}
+
+// CHECK: vpmovdw %xmm21, 291(%rax,%r14,8)
+// CHECK:  encoding: [0x62,0xa2,0x7e,0x08,0x33,0xac,0xf0,0x23,0x01,0x00,0x00]
+          vpmovdw %xmm21, 291(%rax,%r14,8)
+
+// CHECK: vpmovdw %xmm21, 1016(%rdx)
+// CHECK:  encoding: [0x62,0xe2,0x7e,0x08,0x33,0x6a,0x7f]
+          vpmovdw %xmm21, 1016(%rdx)
+
+// CHECK: vpmovdw %xmm21, 1024(%rdx)
+// CHECK:  encoding: [0x62,0xe2,0x7e,0x08,0x33,0xaa,0x00,0x04,0x00,0x00]
+          vpmovdw %xmm21, 1024(%rdx)
+
+// CHECK: vpmovdw %xmm21, -1024(%rdx)
+// CHECK:  encoding: [0x62,0xe2,0x7e,0x08,0x33,0x6a,0x80]
+          vpmovdw %xmm21, -1024(%rdx)
+
+// CHECK: vpmovdw %xmm21, -1032(%rdx)
+// CHECK:  encoding: [0x62,0xe2,0x7e,0x08,0x33,0xaa,0xf8,0xfb,0xff,0xff]
+          vpmovdw %xmm21, -1032(%rdx)
+
+// CHECK: vpmovdw %ymm22, (%rcx)
+// CHECK:  encoding: [0x62,0xe2,0x7e,0x28,0x33,0x31]
+          vpmovdw %ymm22, (%rcx)
+
+// CHECK: vpmovdw %ymm22, (%rcx) {%k6}
+// CHECK:  encoding: [0x62,0xe2,0x7e,0x2e,0x33,0x31]
+          vpmovdw %ymm22, (%rcx) {%k6}
+
+// CHECK: vpmovdw %ymm22, 291(%rax,%r14,8)
+// CHECK:  encoding: [0x62,0xa2,0x7e,0x28,0x33,0xb4,0xf0,0x23,0x01,0x00,0x00]
+          vpmovdw %ymm22, 291(%rax,%r14,8)
+
+// CHECK: vpmovdw %ymm22, 2032(%rdx)
+// CHECK:  encoding: [0x62,0xe2,0x7e,0x28,0x33,0x72,0x7f]
+          vpmovdw %ymm22, 2032(%rdx)
+
+// CHECK: vpmovdw %ymm22, 2048(%rdx)
+// CHECK:  encoding: [0x62,0xe2,0x7e,0x28,0x33,0xb2,0x00,0x08,0x00,0x00]
+          vpmovdw %ymm22, 2048(%rdx)
+
+// CHECK: vpmovdw %ymm22, -2048(%rdx)
+// CHECK:  encoding: [0x62,0xe2,0x7e,0x28,0x33,0x72,0x80]
+          vpmovdw %ymm22, -2048(%rdx)
+
+// CHECK: vpmovdw %ymm22, -2064(%rdx)
+// CHECK:  encoding: [0x62,0xe2,0x7e,0x28,0x33,0xb2,0xf0,0xf7,0xff,0xff]
+          vpmovdw %ymm22, -2064(%rdx)
+
+// CHECK: vpmovsdw %xmm18, %xmm18
+// CHECK:  encoding: [0x62,0xa2,0x7e,0x08,0x23,0xd2]
+          vpmovsdw %xmm18, %xmm18
+
+// CHECK: vpmovsdw %xmm18, %xmm18 {%k6}
+// CHECK:  encoding: [0x62,0xa2,0x7e,0x0e,0x23,0xd2]
+          vpmovsdw %xmm18, %xmm18 {%k6}
+
+// CHECK: vpmovsdw %xmm18, %xmm18 {%k6} {z}
+// CHECK:  encoding: [0x62,0xa2,0x7e,0x8e,0x23,0xd2]
+          vpmovsdw %xmm18, %xmm18 {%k6} {z}
+
+// CHECK: vpmovsdw %ymm18, %xmm20
+// CHECK:  encoding: [0x62,0xa2,0x7e,0x28,0x23,0xd4]
+          vpmovsdw %ymm18, %xmm20
+
+// CHECK: vpmovsdw %ymm18, %xmm20 {%k2}
+// CHECK:  encoding: [0x62,0xa2,0x7e,0x2a,0x23,0xd4]
+          vpmovsdw %ymm18, %xmm20 {%k2}
+
+// CHECK: vpmovsdw %ymm18, %xmm20 {%k2} {z}
+// CHECK:  encoding: [0x62,0xa2,0x7e,0xaa,0x23,0xd4]
+          vpmovsdw %ymm18, %xmm20 {%k2} {z}
+
+// CHECK: vpmovsdw %xmm29, (%rcx)
+// CHECK:  encoding: [0x62,0x62,0x7e,0x08,0x23,0x29]
+          vpmovsdw %xmm29, (%rcx)
+
+// CHECK: vpmovsdw %xmm29, (%rcx) {%k1}
+// CHECK:  encoding: [0x62,0x62,0x7e,0x09,0x23,0x29]
+          vpmovsdw %xmm29, (%rcx) {%k1}
+
+// CHECK: vpmovsdw %xmm29, 291(%rax,%r14,8)
+// CHECK:  encoding: [0x62,0x22,0x7e,0x08,0x23,0xac,0xf0,0x23,0x01,0x00,0x00]
+          vpmovsdw %xmm29, 291(%rax,%r14,8)
+
+// CHECK: vpmovsdw %xmm29, 1016(%rdx)
+// CHECK:  encoding: [0x62,0x62,0x7e,0x08,0x23,0x6a,0x7f]
+          vpmovsdw %xmm29, 1016(%rdx)
+
+// CHECK: vpmovsdw %xmm29, 1024(%rdx)
+// CHECK:  encoding: [0x62,0x62,0x7e,0x08,0x23,0xaa,0x00,0x04,0x00,0x00]
+          vpmovsdw %xmm29, 1024(%rdx)
+
+// CHECK: vpmovsdw %xmm29, -1024(%rdx)
+// CHECK:  encoding: [0x62,0x62,0x7e,0x08,0x23,0x6a,0x80]
+          vpmovsdw %xmm29, -1024(%rdx)
+
+// CHECK: vpmovsdw %xmm29, -1032(%rdx)
+// CHECK:  encoding: [0x62,0x62,0x7e,0x08,0x23,0xaa,0xf8,0xfb,0xff,0xff]
+          vpmovsdw %xmm29, -1032(%rdx)
+
+// CHECK: vpmovsdw %ymm19, (%rcx)
+// CHECK:  encoding: [0x62,0xe2,0x7e,0x28,0x23,0x19]
+          vpmovsdw %ymm19, (%rcx)
+
+// CHECK: vpmovsdw %ymm19, (%rcx) {%k6}
+// CHECK:  encoding: [0x62,0xe2,0x7e,0x2e,0x23,0x19]
+          vpmovsdw %ymm19, (%rcx) {%k6}
+
+// CHECK: vpmovsdw %ymm19, 291(%rax,%r14,8)
+// CHECK:  encoding: [0x62,0xa2,0x7e,0x28,0x23,0x9c,0xf0,0x23,0x01,0x00,0x00]
+          vpmovsdw %ymm19, 291(%rax,%r14,8)
+
+// CHECK: vpmovsdw %ymm19, 2032(%rdx)
+// CHECK:  encoding: [0x62,0xe2,0x7e,0x28,0x23,0x5a,0x7f]
+          vpmovsdw %ymm19, 2032(%rdx)
+
+// CHECK: vpmovsdw %ymm19, 2048(%rdx)
+// CHECK:  encoding: [0x62,0xe2,0x7e,0x28,0x23,0x9a,0x00,0x08,0x00,0x00]
+          vpmovsdw %ymm19, 2048(%rdx)
+
+// CHECK: vpmovsdw %ymm19, -2048(%rdx)
+// CHECK:  encoding: [0x62,0xe2,0x7e,0x28,0x23,0x5a,0x80]
+          vpmovsdw %ymm19, -2048(%rdx)
+
+// CHECK: vpmovsdw %ymm19, -2064(%rdx)
+// CHECK:  encoding: [0x62,0xe2,0x7e,0x28,0x23,0x9a,0xf0,0xf7,0xff,0xff]
+          vpmovsdw %ymm19, -2064(%rdx)
+
+// CHECK: vpmovusdw %xmm18, %xmm18
+// CHECK:  encoding: [0x62,0xa2,0x7e,0x08,0x13,0xd2]
+          vpmovusdw %xmm18, %xmm18
+
+// CHECK: vpmovusdw %xmm18, %xmm18 {%k2}
+// CHECK:  encoding: [0x62,0xa2,0x7e,0x0a,0x13,0xd2]
+          vpmovusdw %xmm18, %xmm18 {%k2}
+
+// CHECK: vpmovusdw %xmm18, %xmm18 {%k2} {z}
+// CHECK:  encoding: [0x62,0xa2,0x7e,0x8a,0x13,0xd2]
+          vpmovusdw %xmm18, %xmm18 {%k2} {z}
+
+// CHECK: vpmovusdw %ymm25, %xmm28
+// CHECK:  encoding: [0x62,0x02,0x7e,0x28,0x13,0xcc]
+          vpmovusdw %ymm25, %xmm28
+
+// CHECK: vpmovusdw %ymm25, %xmm28 {%k4}
+// CHECK:  encoding: [0x62,0x02,0x7e,0x2c,0x13,0xcc]
+          vpmovusdw %ymm25, %xmm28 {%k4}
+
+// CHECK: vpmovusdw %ymm25, %xmm28 {%k4} {z}
+// CHECK:  encoding: [0x62,0x02,0x7e,0xac,0x13,0xcc]
+          vpmovusdw %ymm25, %xmm28 {%k4} {z}
+
+// CHECK: vpmovusdw %xmm20, (%rcx)
+// CHECK:  encoding: [0x62,0xe2,0x7e,0x08,0x13,0x21]
+          vpmovusdw %xmm20, (%rcx)
+
+// CHECK: vpmovusdw %xmm20, (%rcx) {%k6}
+// CHECK:  encoding: [0x62,0xe2,0x7e,0x0e,0x13,0x21]
+          vpmovusdw %xmm20, (%rcx) {%k6}
+
+// CHECK: vpmovusdw %xmm20, 291(%rax,%r14,8)
+// CHECK:  encoding: [0x62,0xa2,0x7e,0x08,0x13,0xa4,0xf0,0x23,0x01,0x00,0x00]
+          vpmovusdw %xmm20, 291(%rax,%r14,8)
+
+// CHECK: vpmovusdw %xmm20, 1016(%rdx)
+// CHECK:  encoding: [0x62,0xe2,0x7e,0x08,0x13,0x62,0x7f]
+          vpmovusdw %xmm20, 1016(%rdx)
+
+// CHECK: vpmovusdw %xmm20, 1024(%rdx)
+// CHECK:  encoding: [0x62,0xe2,0x7e,0x08,0x13,0xa2,0x00,0x04,0x00,0x00]
+          vpmovusdw %xmm20, 1024(%rdx)
+
+// CHECK: vpmovusdw %xmm20, -1024(%rdx)
+// CHECK:  encoding: [0x62,0xe2,0x7e,0x08,0x13,0x62,0x80]
+          vpmovusdw %xmm20, -1024(%rdx)
+
+// CHECK: vpmovusdw %xmm20, -1032(%rdx)
+// CHECK:  encoding: [0x62,0xe2,0x7e,0x08,0x13,0xa2,0xf8,0xfb,0xff,0xff]
+          vpmovusdw %xmm20, -1032(%rdx)
+
+// CHECK: vpmovusdw %ymm23, (%rcx)
+// CHECK:  encoding: [0x62,0xe2,0x7e,0x28,0x13,0x39]
+          vpmovusdw %ymm23, (%rcx)
+
+// CHECK: vpmovusdw %ymm23, (%rcx) {%k1}
+// CHECK:  encoding: [0x62,0xe2,0x7e,0x29,0x13,0x39]
+          vpmovusdw %ymm23, (%rcx) {%k1}
+
+// CHECK: vpmovusdw %ymm23, 291(%rax,%r14,8)
+// CHECK:  encoding: [0x62,0xa2,0x7e,0x28,0x13,0xbc,0xf0,0x23,0x01,0x00,0x00]
+          vpmovusdw %ymm23, 291(%rax,%r14,8)
+
+// CHECK: vpmovusdw %ymm23, 2032(%rdx)
+// CHECK:  encoding: [0x62,0xe2,0x7e,0x28,0x13,0x7a,0x7f]
+          vpmovusdw %ymm23, 2032(%rdx)
+
+// CHECK: vpmovusdw %ymm23, 2048(%rdx)
+// CHECK:  encoding: [0x62,0xe2,0x7e,0x28,0x13,0xba,0x00,0x08,0x00,0x00]
+          vpmovusdw %ymm23, 2048(%rdx)
+
+// CHECK: vpmovusdw %ymm23, -2048(%rdx)
+// CHECK:  encoding: [0x62,0xe2,0x7e,0x28,0x13,0x7a,0x80]
+          vpmovusdw %ymm23, -2048(%rdx)
+
+// CHECK: vpmovusdw %ymm23, -2064(%rdx)
+// CHECK:  encoding: [0x62,0xe2,0x7e,0x28,0x13,0xba,0xf0,0xf7,0xff,0xff]
+          vpmovusdw %ymm23, -2064(%rdx)
+
+// CHECK: vrndscalepd $171, %xmm28, %xmm29
+// CHECK:  encoding: [0x62,0x03,0xfd,0x08,0x09,0xec,0xab]
+          vrndscalepd $0xab, %xmm28, %xmm29
+
+// CHECK: vrndscalepd $171, %xmm28, %xmm29 {%k4}
+// CHECK:  encoding: [0x62,0x03,0xfd,0x0c,0x09,0xec,0xab]
+          vrndscalepd $0xab, %xmm28, %xmm29 {%k4}
+
+// CHECK: vrndscalepd $171, %xmm28, %xmm29 {%k4} {z}
+// CHECK:  encoding: [0x62,0x03,0xfd,0x8c,0x09,0xec,0xab]
+          vrndscalepd $0xab, %xmm28, %xmm29 {%k4} {z}
+
+// CHECK: vrndscalepd $123, %xmm28, %xmm29
+// CHECK:  encoding: [0x62,0x03,0xfd,0x08,0x09,0xec,0x7b]
+          vrndscalepd $0x7b, %xmm28, %xmm29
+
+// CHECK: vrndscalepd $123, (%rcx), %xmm29
+// CHECK:  encoding: [0x62,0x63,0xfd,0x08,0x09,0x29,0x7b]
+          vrndscalepd $0x7b, (%rcx), %xmm29
+
+// CHECK: vrndscalepd $123, 291(%rax,%r14,8), %xmm29
+// CHECK:  encoding: [0x62,0x23,0xfd,0x08,0x09,0xac,0xf0,0x23,0x01,0x00,0x00,0x7b]
+          vrndscalepd $0x7b, 291(%rax,%r14,8), %xmm29
+
+// CHECK: vrndscalepd $123, (%rcx){1to2}, %xmm29
+// CHECK:  encoding: [0x62,0x63,0xfd,0x18,0x09,0x29,0x7b]
+          vrndscalepd $0x7b, (%rcx){1to2}, %xmm29
+
+// CHECK: vrndscalepd $123, 2032(%rdx), %xmm29
+// CHECK:  encoding: [0x62,0x63,0xfd,0x08,0x09,0x6a,0x7f,0x7b]
+          vrndscalepd $0x7b, 2032(%rdx), %xmm29
+
+// CHECK: vrndscalepd $123, 2048(%rdx), %xmm29
+// CHECK:  encoding: [0x62,0x63,0xfd,0x08,0x09,0xaa,0x00,0x08,0x00,0x00,0x7b]
+          vrndscalepd $0x7b, 2048(%rdx), %xmm29
+
+// CHECK: vrndscalepd $123, -2048(%rdx), %xmm29
+// CHECK:  encoding: [0x62,0x63,0xfd,0x08,0x09,0x6a,0x80,0x7b]
+          vrndscalepd $0x7b, -2048(%rdx), %xmm29
+
+// CHECK: vrndscalepd $123, -2064(%rdx), %xmm29
+// CHECK:  encoding: [0x62,0x63,0xfd,0x08,0x09,0xaa,0xf0,0xf7,0xff,0xff,0x7b]
+          vrndscalepd $0x7b, -2064(%rdx), %xmm29
+
+// CHECK: vrndscalepd $123, 1016(%rdx){1to2}, %xmm29
+// CHECK:  encoding: [0x62,0x63,0xfd,0x18,0x09,0x6a,0x7f,0x7b]
+          vrndscalepd $0x7b, 1016(%rdx){1to2}, %xmm29
+
+// CHECK: vrndscalepd $123, 1024(%rdx){1to2}, %xmm29
+// CHECK:  encoding: [0x62,0x63,0xfd,0x18,0x09,0xaa,0x00,0x04,0x00,0x00,0x7b]
+          vrndscalepd $0x7b, 1024(%rdx){1to2}, %xmm29
+
+// CHECK: vrndscalepd $123, -1024(%rdx){1to2}, %xmm29
+// CHECK:  encoding: [0x62,0x63,0xfd,0x18,0x09,0x6a,0x80,0x7b]
+          vrndscalepd $0x7b, -1024(%rdx){1to2}, %xmm29
+
+// CHECK: vrndscalepd $123, -1032(%rdx){1to2}, %xmm29
+// CHECK:  encoding: [0x62,0x63,0xfd,0x18,0x09,0xaa,0xf8,0xfb,0xff,0xff,0x7b]
+          vrndscalepd $0x7b, -1032(%rdx){1to2}, %xmm29
+
+// CHECK: vrndscalepd $171, %ymm22, %ymm17
+// CHECK:  encoding: [0x62,0xa3,0xfd,0x28,0x09,0xce,0xab]
+          vrndscalepd $0xab, %ymm22, %ymm17
+
+// CHECK: vrndscalepd $171, %ymm22, %ymm17 {%k7}
+// CHECK:  encoding: [0x62,0xa3,0xfd,0x2f,0x09,0xce,0xab]
+          vrndscalepd $0xab, %ymm22, %ymm17 {%k7}
+
+// CHECK: vrndscalepd $171, %ymm22, %ymm17 {%k7} {z}
+// CHECK:  encoding: [0x62,0xa3,0xfd,0xaf,0x09,0xce,0xab]
+          vrndscalepd $0xab, %ymm22, %ymm17 {%k7} {z}
+
+// CHECK: vrndscalepd $123, %ymm22, %ymm17
+// CHECK:  encoding: [0x62,0xa3,0xfd,0x28,0x09,0xce,0x7b]
+          vrndscalepd $0x7b, %ymm22, %ymm17
+
+// CHECK: vrndscalepd $123, (%rcx), %ymm17
+// CHECK:  encoding: [0x62,0xe3,0xfd,0x28,0x09,0x09,0x7b]
+          vrndscalepd $0x7b, (%rcx), %ymm17
+
+// CHECK: vrndscalepd $123, 291(%rax,%r14,8), %ymm17
+// CHECK:  encoding: [0x62,0xa3,0xfd,0x28,0x09,0x8c,0xf0,0x23,0x01,0x00,0x00,0x7b]
+          vrndscalepd $0x7b, 291(%rax,%r14,8), %ymm17
+
+// CHECK: vrndscalepd $123, (%rcx){1to4}, %ymm17
+// CHECK:  encoding: [0x62,0xe3,0xfd,0x38,0x09,0x09,0x7b]
+          vrndscalepd $0x7b, (%rcx){1to4}, %ymm17
+
+// CHECK: vrndscalepd $123, 4064(%rdx), %ymm17
+// CHECK:  encoding: [0x62,0xe3,0xfd,0x28,0x09,0x4a,0x7f,0x7b]
+          vrndscalepd $0x7b, 4064(%rdx), %ymm17
+
+// CHECK: vrndscalepd $123, 4096(%rdx), %ymm17
+// CHECK:  encoding: [0x62,0xe3,0xfd,0x28,0x09,0x8a,0x00,0x10,0x00,0x00,0x7b]
+          vrndscalepd $0x7b, 4096(%rdx), %ymm17
+
+// CHECK: vrndscalepd $123, -4096(%rdx), %ymm17
+// CHECK:  encoding: [0x62,0xe3,0xfd,0x28,0x09,0x4a,0x80,0x7b]
+          vrndscalepd $0x7b, -4096(%rdx), %ymm17
+
+// CHECK: vrndscalepd $123, -4128(%rdx), %ymm17
+// CHECK:  encoding: [0x62,0xe3,0xfd,0x28,0x09,0x8a,0xe0,0xef,0xff,0xff,0x7b]
+          vrndscalepd $0x7b, -4128(%rdx), %ymm17
+
+// CHECK: vrndscalepd $123, 1016(%rdx){1to4}, %ymm17
+// CHECK:  encoding: [0x62,0xe3,0xfd,0x38,0x09,0x4a,0x7f,0x7b]
+          vrndscalepd $0x7b, 1016(%rdx){1to4}, %ymm17
+
+// CHECK: vrndscalepd $123, 1024(%rdx){1to4}, %ymm17
+// CHECK:  encoding: [0x62,0xe3,0xfd,0x38,0x09,0x8a,0x00,0x04,0x00,0x00,0x7b]
+          vrndscalepd $0x7b, 1024(%rdx){1to4}, %ymm17
+
+// CHECK: vrndscalepd $123, -1024(%rdx){1to4}, %ymm17
+// CHECK:  encoding: [0x62,0xe3,0xfd,0x38,0x09,0x4a,0x80,0x7b]
+          vrndscalepd $0x7b, -1024(%rdx){1to4}, %ymm17
+
+// CHECK: vrndscalepd $123, -1032(%rdx){1to4}, %ymm17
+// CHECK:  encoding: [0x62,0xe3,0xfd,0x38,0x09,0x8a,0xf8,0xfb,0xff,0xff,0x7b]
+          vrndscalepd $0x7b, -1032(%rdx){1to4}, %ymm17
+
+// CHECK: vrndscaleps $171, %xmm26, %xmm22
+// CHECK:  encoding: [0x62,0x83,0x7d,0x08,0x08,0xf2,0xab]
+          vrndscaleps $0xab, %xmm26, %xmm22
+
+// CHECK: vrndscaleps $171, %xmm26, %xmm22 {%k4}
+// CHECK:  encoding: [0x62,0x83,0x7d,0x0c,0x08,0xf2,0xab]
+          vrndscaleps $0xab, %xmm26, %xmm22 {%k4}
+
+// CHECK: vrndscaleps $171, %xmm26, %xmm22 {%k4} {z}
+// CHECK:  encoding: [0x62,0x83,0x7d,0x8c,0x08,0xf2,0xab]
+          vrndscaleps $0xab, %xmm26, %xmm22 {%k4} {z}
+
+// CHECK: vrndscaleps $123, %xmm26, %xmm22
+// CHECK:  encoding: [0x62,0x83,0x7d,0x08,0x08,0xf2,0x7b]
+          vrndscaleps $0x7b, %xmm26, %xmm22
+
+// CHECK: vrndscaleps $123, (%rcx), %xmm22
+// CHECK:  encoding: [0x62,0xe3,0x7d,0x08,0x08,0x31,0x7b]
+          vrndscaleps $0x7b, (%rcx), %xmm22
+
+// CHECK: vrndscaleps $123, 291(%rax,%r14,8), %xmm22
+// CHECK:  encoding: [0x62,0xa3,0x7d,0x08,0x08,0xb4,0xf0,0x23,0x01,0x00,0x00,0x7b]
+          vrndscaleps $0x7b, 291(%rax,%r14,8), %xmm22
+
+// CHECK: vrndscaleps $123, (%rcx){1to4}, %xmm22
+// CHECK:  encoding: [0x62,0xe3,0x7d,0x18,0x08,0x31,0x7b]
+          vrndscaleps $0x7b, (%rcx){1to4}, %xmm22
+
+// CHECK: vrndscaleps $123, 2032(%rdx), %xmm22
+// CHECK:  encoding: [0x62,0xe3,0x7d,0x08,0x08,0x72,0x7f,0x7b]
+          vrndscaleps $0x7b, 2032(%rdx), %xmm22
+
+// CHECK: vrndscaleps $123, 2048(%rdx), %xmm22
+// CHECK:  encoding: [0x62,0xe3,0x7d,0x08,0x08,0xb2,0x00,0x08,0x00,0x00,0x7b]
+          vrndscaleps $0x7b, 2048(%rdx), %xmm22
+
+// CHECK: vrndscaleps $123, -2048(%rdx), %xmm22
+// CHECK:  encoding: [0x62,0xe3,0x7d,0x08,0x08,0x72,0x80,0x7b]
+          vrndscaleps $0x7b, -2048(%rdx), %xmm22
+
+// CHECK: vrndscaleps $123, -2064(%rdx), %xmm22
+// CHECK:  encoding: [0x62,0xe3,0x7d,0x08,0x08,0xb2,0xf0,0xf7,0xff,0xff,0x7b]
+          vrndscaleps $0x7b, -2064(%rdx), %xmm22
+
+// CHECK: vrndscaleps $123, 508(%rdx){1to4}, %xmm22
+// CHECK:  encoding: [0x62,0xe3,0x7d,0x18,0x08,0x72,0x7f,0x7b]
+          vrndscaleps $0x7b, 508(%rdx){1to4}, %xmm22
+
+// CHECK: vrndscaleps $123, 512(%rdx){1to4}, %xmm22
+// CHECK:  encoding: [0x62,0xe3,0x7d,0x18,0x08,0xb2,0x00,0x02,0x00,0x00,0x7b]
+          vrndscaleps $0x7b, 512(%rdx){1to4}, %xmm22
+
+// CHECK: vrndscaleps $123, -512(%rdx){1to4}, %xmm22
+// CHECK:  encoding: [0x62,0xe3,0x7d,0x18,0x08,0x72,0x80,0x7b]
+          vrndscaleps $0x7b, -512(%rdx){1to4}, %xmm22
+
+// CHECK: vrndscaleps $123, -516(%rdx){1to4}, %xmm22
+// CHECK:  encoding: [0x62,0xe3,0x7d,0x18,0x08,0xb2,0xfc,0xfd,0xff,0xff,0x7b]
+          vrndscaleps $0x7b, -516(%rdx){1to4}, %xmm22
+
+// CHECK: vrndscaleps $171, %ymm17, %ymm19
+// CHECK:  encoding: [0x62,0xa3,0x7d,0x28,0x08,0xd9,0xab]
+          vrndscaleps $0xab, %ymm17, %ymm19
+
+// CHECK: vrndscaleps $171, %ymm17, %ymm19 {%k7}
+// CHECK:  encoding: [0x62,0xa3,0x7d,0x2f,0x08,0xd9,0xab]
+          vrndscaleps $0xab, %ymm17, %ymm19 {%k7}
+
+// CHECK: vrndscaleps $171, %ymm17, %ymm19 {%k7} {z}
+// CHECK:  encoding: [0x62,0xa3,0x7d,0xaf,0x08,0xd9,0xab]
+          vrndscaleps $0xab, %ymm17, %ymm19 {%k7} {z}
+
+// CHECK: vrndscaleps $123, %ymm17, %ymm19
+// CHECK:  encoding: [0x62,0xa3,0x7d,0x28,0x08,0xd9,0x7b]
+          vrndscaleps $0x7b, %ymm17, %ymm19
+
+// CHECK: vrndscaleps $123, (%rcx), %ymm19
+// CHECK:  encoding: [0x62,0xe3,0x7d,0x28,0x08,0x19,0x7b]
+          vrndscaleps $0x7b, (%rcx), %ymm19
+
+// CHECK: vrndscaleps $123, 291(%rax,%r14,8), %ymm19
+// CHECK:  encoding: [0x62,0xa3,0x7d,0x28,0x08,0x9c,0xf0,0x23,0x01,0x00,0x00,0x7b]
+          vrndscaleps $0x7b, 291(%rax,%r14,8), %ymm19
+
+// CHECK: vrndscaleps $123, (%rcx){1to8}, %ymm19
+// CHECK:  encoding: [0x62,0xe3,0x7d,0x38,0x08,0x19,0x7b]
+          vrndscaleps $0x7b, (%rcx){1to8}, %ymm19
+
+// CHECK: vrndscaleps $123, 4064(%rdx), %ymm19
+// CHECK:  encoding: [0x62,0xe3,0x7d,0x28,0x08,0x5a,0x7f,0x7b]
+          vrndscaleps $0x7b, 4064(%rdx), %ymm19
+
+// CHECK: vrndscaleps $123, 4096(%rdx), %ymm19
+// CHECK:  encoding: [0x62,0xe3,0x7d,0x28,0x08,0x9a,0x00,0x10,0x00,0x00,0x7b]
+          vrndscaleps $0x7b, 4096(%rdx), %ymm19
+
+// CHECK: vrndscaleps $123, -4096(%rdx), %ymm19
+// CHECK:  encoding: [0x62,0xe3,0x7d,0x28,0x08,0x5a,0x80,0x7b]
+          vrndscaleps $0x7b, -4096(%rdx), %ymm19
+
+// CHECK: vrndscaleps $123, -4128(%rdx), %ymm19
+// CHECK:  encoding: [0x62,0xe3,0x7d,0x28,0x08,0x9a,0xe0,0xef,0xff,0xff,0x7b]
+          vrndscaleps $0x7b, -4128(%rdx), %ymm19
+
+// CHECK: vrndscaleps $123, 508(%rdx){1to8}, %ymm19
+// CHECK:  encoding: [0x62,0xe3,0x7d,0x38,0x08,0x5a,0x7f,0x7b]
+          vrndscaleps $0x7b, 508(%rdx){1to8}, %ymm19
+
+// CHECK: vrndscaleps $123, 512(%rdx){1to8}, %ymm19
+// CHECK:  encoding: [0x62,0xe3,0x7d,0x38,0x08,0x9a,0x00,0x02,0x00,0x00,0x7b]
+          vrndscaleps $0x7b, 512(%rdx){1to8}, %ymm19
+
+// CHECK: vrndscaleps $123, -512(%rdx){1to8}, %ymm19
+// CHECK:  encoding: [0x62,0xe3,0x7d,0x38,0x08,0x5a,0x80,0x7b]
+          vrndscaleps $0x7b, -512(%rdx){1to8}, %ymm19
+
+// CHECK: vrndscaleps $123, -516(%rdx){1to8}, %ymm19
+// CHECK:  encoding: [0x62,0xe3,0x7d,0x38,0x08,0x9a,0xfc,0xfd,0xff,0xff,0x7b]
+          vrndscaleps $0x7b, -516(%rdx){1to8}, %ymm19
+
+// CHECK: vcvtps2pd %xmm27, %xmm20
+// CHECK:  encoding: [0x62,0x81,0x7c,0x08,0x5a,0xe3]
+          vcvtps2pd %xmm27, %xmm20
+
+// CHECK: vcvtps2pd %xmm27, %xmm20 {%k3}
+// CHECK:  encoding: [0x62,0x81,0x7c,0x0b,0x5a,0xe3]
+          vcvtps2pd %xmm27, %xmm20 {%k3}
+
+// CHECK: vcvtps2pd %xmm27, %xmm20 {%k3} {z}
+// CHECK:  encoding: [0x62,0x81,0x7c,0x8b,0x5a,0xe3]
+          vcvtps2pd %xmm27, %xmm20 {%k3} {z}
+
+// CHECK: vcvtps2pd (%rcx), %xmm20
+// CHECK:  encoding: [0x62,0xe1,0x7c,0x08,0x5a,0x21]
+          vcvtps2pd (%rcx), %xmm20
+
+// CHECK: vcvtps2pd 291(%rax,%r14,8), %xmm20
+// CHECK:  encoding: [0x62,0xa1,0x7c,0x08,0x5a,0xa4,0xf0,0x23,0x01,0x00,0x00]
+          vcvtps2pd 291(%rax,%r14,8), %xmm20
+
+// CHECK: vcvtps2pd (%rcx){1to2}, %xmm20
+// CHECK:  encoding: [0x62,0xe1,0x7c,0x18,0x5a,0x21]
+          vcvtps2pd (%rcx){1to2}, %xmm20
+
+// CHECK: vcvtps2pd 1016(%rdx), %xmm20
+// CHECK:  encoding: [0x62,0xe1,0x7c,0x08,0x5a,0x62,0x7f]
+          vcvtps2pd 1016(%rdx), %xmm20
+
+// CHECK: vcvtps2pd 1024(%rdx), %xmm20
+// CHECK:  encoding: [0x62,0xe1,0x7c,0x08,0x5a,0xa2,0x00,0x04,0x00,0x00]
+          vcvtps2pd 1024(%rdx), %xmm20
+
+// CHECK: vcvtps2pd -1024(%rdx), %xmm20
+// CHECK:  encoding: [0x62,0xe1,0x7c,0x08,0x5a,0x62,0x80]
+          vcvtps2pd -1024(%rdx), %xmm20
+
+// CHECK: vcvtps2pd -1032(%rdx), %xmm20
+// CHECK:  encoding: [0x62,0xe1,0x7c,0x08,0x5a,0xa2,0xf8,0xfb,0xff,0xff]
+          vcvtps2pd -1032(%rdx), %xmm20
+
+// CHECK: vcvtps2pd 508(%rdx){1to2}, %xmm20
+// CHECK:  encoding: [0x62,0xe1,0x7c,0x18,0x5a,0x62,0x7f]
+          vcvtps2pd 508(%rdx){1to2}, %xmm20
+
+// CHECK: vcvtps2pd 512(%rdx){1to2}, %xmm20
+// CHECK:  encoding: [0x62,0xe1,0x7c,0x18,0x5a,0xa2,0x00,0x02,0x00,0x00]
+          vcvtps2pd 512(%rdx){1to2}, %xmm20
+
+// CHECK: vcvtps2pd -512(%rdx){1to2}, %xmm20
+// CHECK:  encoding: [0x62,0xe1,0x7c,0x18,0x5a,0x62,0x80]
+          vcvtps2pd -512(%rdx){1to2}, %xmm20
+
+// CHECK: vcvtps2pd -516(%rdx){1to2}, %xmm20
+// CHECK:  encoding: [0x62,0xe1,0x7c,0x18,0x5a,0xa2,0xfc,0xfd,0xff,0xff]
+          vcvtps2pd -516(%rdx){1to2}, %xmm20
+
+// CHECK: vcvtps2pd %xmm18, %ymm22
+// CHECK:  encoding: [0x62,0xa1,0x7c,0x28,0x5a,0xf2]
+          vcvtps2pd %xmm18, %ymm22
+
+// CHECK: vcvtps2pd %xmm18, %ymm22 {%k2}
+// CHECK:  encoding: [0x62,0xa1,0x7c,0x2a,0x5a,0xf2]
+          vcvtps2pd %xmm18, %ymm22 {%k2}
+
+// CHECK: vcvtps2pd %xmm18, %ymm22 {%k2} {z}
+// CHECK:  encoding: [0x62,0xa1,0x7c,0xaa,0x5a,0xf2]
+          vcvtps2pd %xmm18, %ymm22 {%k2} {z}
+
+// CHECK: vcvtps2pd (%rcx), %ymm22
+// CHECK:  encoding: [0x62,0xe1,0x7c,0x28,0x5a,0x31]
+          vcvtps2pd (%rcx), %ymm22
+
+// CHECK: vcvtps2pd 291(%rax,%r14,8), %ymm22
+// CHECK:  encoding: [0x62,0xa1,0x7c,0x28,0x5a,0xb4,0xf0,0x23,0x01,0x00,0x00]
+          vcvtps2pd 291(%rax,%r14,8), %ymm22
+
+// CHECK: vcvtps2pd (%rcx){1to4}, %ymm22
+// CHECK:  encoding: [0x62,0xe1,0x7c,0x38,0x5a,0x31]
+          vcvtps2pd (%rcx){1to4}, %ymm22
+
+// CHECK: vcvtps2pd 2032(%rdx), %ymm22
+// CHECK:  encoding: [0x62,0xe1,0x7c,0x28,0x5a,0x72,0x7f]
+          vcvtps2pd 2032(%rdx), %ymm22
+
+// CHECK: vcvtps2pd 2048(%rdx), %ymm22
+// CHECK:  encoding: [0x62,0xe1,0x7c,0x28,0x5a,0xb2,0x00,0x08,0x00,0x00]
+          vcvtps2pd 2048(%rdx), %ymm22
+
+// CHECK: vcvtps2pd -2048(%rdx), %ymm22
+// CHECK:  encoding: [0x62,0xe1,0x7c,0x28,0x5a,0x72,0x80]
+          vcvtps2pd -2048(%rdx), %ymm22
+
+// CHECK: vcvtps2pd -2064(%rdx), %ymm22
+// CHECK:  encoding: [0x62,0xe1,0x7c,0x28,0x5a,0xb2,0xf0,0xf7,0xff,0xff]
+          vcvtps2pd -2064(%rdx), %ymm22
+
+// CHECK: vcvtps2pd 508(%rdx){1to4}, %ymm22
+// CHECK:  encoding: [0x62,0xe1,0x7c,0x38,0x5a,0x72,0x7f]
+          vcvtps2pd 508(%rdx){1to4}, %ymm22
+
+// CHECK: vcvtps2pd 512(%rdx){1to4}, %ymm22
+// CHECK:  encoding: [0x62,0xe1,0x7c,0x38,0x5a,0xb2,0x00,0x02,0x00,0x00]
+          vcvtps2pd 512(%rdx){1to4}, %ymm22
+
+// CHECK: vcvtps2pd -512(%rdx){1to4}, %ymm22
+// CHECK:  encoding: [0x62,0xe1,0x7c,0x38,0x5a,0x72,0x80]
+          vcvtps2pd -512(%rdx){1to4}, %ymm22
+
+// CHECK: vcvtps2pd -516(%rdx){1to4}, %ymm22
+// CHECK:  encoding: [0x62,0xe1,0x7c,0x38,0x5a,0xb2,0xfc,0xfd,0xff,0xff]
+          vcvtps2pd -516(%rdx){1to4}, %ymm22
+
+// CHECK: vcvtpd2ps %xmm27, %xmm27
+// CHECK:  encoding: [0x62,0x01,0xfd,0x08,0x5a,0xdb]
+          vcvtpd2ps %xmm27, %xmm27
+
+// CHECK: vcvtpd2ps %xmm27, %xmm27 {%k7}
+// CHECK:  encoding: [0x62,0x01,0xfd,0x0f,0x5a,0xdb]
+          vcvtpd2ps %xmm27, %xmm27 {%k7}
+
+// CHECK: vcvtpd2ps %xmm27, %xmm27 {%k7} {z}
+// CHECK:  encoding: [0x62,0x01,0xfd,0x8f,0x5a,0xdb]
+          vcvtpd2ps %xmm27, %xmm27 {%k7} {z}
+
+// CHECK: vcvtpd2psx (%rcx), %xmm27
+// CHECK:  encoding: [0x62,0x61,0xfd,0x08,0x5a,0x19]
+          vcvtpd2psx (%rcx), %xmm27
+
+// CHECK: vcvtpd2psx 291(%rax,%r14,8), %xmm27
+// CHECK:  encoding: [0x62,0x21,0xfd,0x08,0x5a,0x9c,0xf0,0x23,0x01,0x00,0x00]
+          vcvtpd2psx 291(%rax,%r14,8), %xmm27
+
+// CHECK: vcvtpd2ps (%rcx){1to2}, %xmm27
+// CHECK:  encoding: [0x62,0x61,0xfd,0x18,0x5a,0x19]
+          vcvtpd2ps (%rcx){1to2}, %xmm27
+
+// CHECK: vcvtpd2psx 2032(%rdx), %xmm27
+// CHECK:  encoding: [0x62,0x61,0xfd,0x08,0x5a,0x5a,0x7f]
+          vcvtpd2psx 2032(%rdx), %xmm27
+
+// CHECK: vcvtpd2psx 2048(%rdx), %xmm27
+// CHECK:  encoding: [0x62,0x61,0xfd,0x08,0x5a,0x9a,0x00,0x08,0x00,0x00]
+          vcvtpd2psx 2048(%rdx), %xmm27
+
+// CHECK: vcvtpd2psx -2048(%rdx), %xmm27
+// CHECK:  encoding: [0x62,0x61,0xfd,0x08,0x5a,0x5a,0x80]
+          vcvtpd2psx -2048(%rdx), %xmm27
+
+// CHECK: vcvtpd2psx -2064(%rdx), %xmm27
+// CHECK:  encoding: [0x62,0x61,0xfd,0x08,0x5a,0x9a,0xf0,0xf7,0xff,0xff]
+          vcvtpd2psx -2064(%rdx), %xmm27
+
+// CHECK: vcvtpd2ps 1016(%rdx){1to2}, %xmm27
+// CHECK:  encoding: [0x62,0x61,0xfd,0x18,0x5a,0x5a,0x7f]
+          vcvtpd2ps 1016(%rdx){1to2}, %xmm27
+
+// CHECK: vcvtpd2ps 1024(%rdx){1to2}, %xmm27
+// CHECK:  encoding: [0x62,0x61,0xfd,0x18,0x5a,0x9a,0x00,0x04,0x00,0x00]
+          vcvtpd2ps 1024(%rdx){1to2}, %xmm27
+
+// CHECK: vcvtpd2ps -1024(%rdx){1to2}, %xmm27
+// CHECK:  encoding: [0x62,0x61,0xfd,0x18,0x5a,0x5a,0x80]
+          vcvtpd2ps -1024(%rdx){1to2}, %xmm27
+
+// CHECK: vcvtpd2ps -1032(%rdx){1to2}, %xmm27
+// CHECK:  encoding: [0x62,0x61,0xfd,0x18,0x5a,0x9a,0xf8,0xfb,0xff,0xff]
+          vcvtpd2ps -1032(%rdx){1to2}, %xmm27
+
+// CHECK: vcvtpd2ps %ymm20, %xmm20
+// CHECK:  encoding: [0x62,0xa1,0xfd,0x28,0x5a,0xe4]
+          vcvtpd2ps %ymm20, %xmm20
+
+// CHECK: vcvtpd2ps %ymm20, %xmm20 {%k6}
+// CHECK:  encoding: [0x62,0xa1,0xfd,0x2e,0x5a,0xe4]
+          vcvtpd2ps %ymm20, %xmm20 {%k6}
+
+// CHECK: vcvtpd2ps %ymm20, %xmm20 {%k6} {z}
+// CHECK:  encoding: [0x62,0xa1,0xfd,0xae,0x5a,0xe4]
+          vcvtpd2ps %ymm20, %xmm20 {%k6} {z}
+
+// CHECK: vcvtpd2psy (%rcx), %xmm20
+// CHECK:  encoding: [0x62,0xe1,0xfd,0x28,0x5a,0x21]
+          vcvtpd2psy (%rcx), %xmm20
+
+// CHECK: vcvtpd2psy 291(%rax,%r14,8), %xmm20
+// CHECK:  encoding: [0x62,0xa1,0xfd,0x28,0x5a,0xa4,0xf0,0x23,0x01,0x00,0x00]
+          vcvtpd2psy 291(%rax,%r14,8), %xmm20
+
+// CHECK: vcvtpd2ps (%rcx){1to4}, %xmm20
+// CHECK:  encoding: [0x62,0xe1,0xfd,0x38,0x5a,0x21]
+          vcvtpd2ps (%rcx){1to4}, %xmm20
+
+// CHECK: vcvtpd2psy 4064(%rdx), %xmm20
+// CHECK:  encoding: [0x62,0xe1,0xfd,0x28,0x5a,0x62,0x7f]
+          vcvtpd2psy 4064(%rdx), %xmm20
+
+// CHECK: vcvtpd2psy 4096(%rdx), %xmm20
+// CHECK:  encoding: [0x62,0xe1,0xfd,0x28,0x5a,0xa2,0x00,0x10,0x00,0x00]
+          vcvtpd2psy 4096(%rdx), %xmm20
+
+// CHECK: vcvtpd2psy -4096(%rdx), %xmm20
+// CHECK:  encoding: [0x62,0xe1,0xfd,0x28,0x5a,0x62,0x80]
+          vcvtpd2psy -4096(%rdx), %xmm20
+
+// CHECK: vcvtpd2psy -4128(%rdx), %xmm20
+// CHECK:  encoding: [0x62,0xe1,0xfd,0x28,0x5a,0xa2,0xe0,0xef,0xff,0xff]
+          vcvtpd2psy -4128(%rdx), %xmm20
+
+// CHECK: vcvtpd2ps 1016(%rdx){1to4}, %xmm20
+// CHECK:  encoding: [0x62,0xe1,0xfd,0x38,0x5a,0x62,0x7f]
+          vcvtpd2ps 1016(%rdx){1to4}, %xmm20
+
+// CHECK: vcvtpd2ps 1024(%rdx){1to4}, %xmm20
+// CHECK:  encoding: [0x62,0xe1,0xfd,0x38,0x5a,0xa2,0x00,0x04,0x00,0x00]
+          vcvtpd2ps 1024(%rdx){1to4}, %xmm20
+
+// CHECK: vcvtpd2ps -1024(%rdx){1to4}, %xmm20
+// CHECK:  encoding: [0x62,0xe1,0xfd,0x38,0x5a,0x62,0x80]
+          vcvtpd2ps -1024(%rdx){1to4}, %xmm20
+
+// CHECK: vcvtpd2ps -1032(%rdx){1to4}, %xmm20
+// CHECK:  encoding: [0x62,0xe1,0xfd,0x38,0x5a,0xa2,0xf8,0xfb,0xff,0xff]
+          vcvtpd2ps -1032(%rdx){1to4}, %xmm20
+
+// CHECK: vcvtpd2udq %xmm22, %xmm28
+// CHECK:  encoding: [0x62,0x21,0xfc,0x08,0x79,0xe6]
+          vcvtpd2udq %xmm22, %xmm28
+
+// CHECK: vcvtpd2udq %xmm22, %xmm28 {%k3}
+// CHECK:  encoding: [0x62,0x21,0xfc,0x0b,0x79,0xe6]
+          vcvtpd2udq %xmm22, %xmm28 {%k3}
+
+// CHECK: vcvtpd2udq %xmm22, %xmm28 {%k3} {z}
+// CHECK:  encoding: [0x62,0x21,0xfc,0x8b,0x79,0xe6]
+          vcvtpd2udq %xmm22, %xmm28 {%k3} {z}
+
+// CHECK: vcvtpd2udqx (%rcx), %xmm28
+// CHECK:  encoding: [0x62,0x61,0xfc,0x08,0x79,0x21]
+          vcvtpd2udqx (%rcx), %xmm28
+
+// CHECK: vcvtpd2udqx 291(%rax,%r14,8), %xmm28
+// CHECK:  encoding: [0x62,0x21,0xfc,0x08,0x79,0xa4,0xf0,0x23,0x01,0x00,0x00]
+          vcvtpd2udqx 291(%rax,%r14,8), %xmm28
+
+// CHECK: vcvtpd2udq (%rcx){1to2}, %xmm28
+// CHECK:  encoding: [0x62,0x61,0xfc,0x18,0x79,0x21]
+          vcvtpd2udq (%rcx){1to2}, %xmm28
+
+// CHECK: vcvtpd2udqx 2032(%rdx), %xmm28
+// CHECK:  encoding: [0x62,0x61,0xfc,0x08,0x79,0x62,0x7f]
+          vcvtpd2udqx 2032(%rdx), %xmm28
+
+// CHECK: vcvtpd2udqx 2048(%rdx), %xmm28
+// CHECK:  encoding: [0x62,0x61,0xfc,0x08,0x79,0xa2,0x00,0x08,0x00,0x00]
+          vcvtpd2udqx 2048(%rdx), %xmm28
+
+// CHECK: vcvtpd2udqx -2048(%rdx), %xmm28
+// CHECK:  encoding: [0x62,0x61,0xfc,0x08,0x79,0x62,0x80]
+          vcvtpd2udqx -2048(%rdx), %xmm28
+
+// CHECK: vcvtpd2udqx -2064(%rdx), %xmm28
+// CHECK:  encoding: [0x62,0x61,0xfc,0x08,0x79,0xa2,0xf0,0xf7,0xff,0xff]
+          vcvtpd2udqx -2064(%rdx), %xmm28
+
+// CHECK: vcvtpd2udq 1016(%rdx){1to2}, %xmm28
+// CHECK:  encoding: [0x62,0x61,0xfc,0x18,0x79,0x62,0x7f]
+          vcvtpd2udq 1016(%rdx){1to2}, %xmm28
+
+// CHECK: vcvtpd2udq 1024(%rdx){1to2}, %xmm28
+// CHECK:  encoding: [0x62,0x61,0xfc,0x18,0x79,0xa2,0x00,0x04,0x00,0x00]
+          vcvtpd2udq 1024(%rdx){1to2}, %xmm28
+
+// CHECK: vcvtpd2udq -1024(%rdx){1to2}, %xmm28
+// CHECK:  encoding: [0x62,0x61,0xfc,0x18,0x79,0x62,0x80]
+          vcvtpd2udq -1024(%rdx){1to2}, %xmm28
+
+// CHECK: vcvtpd2udq -1032(%rdx){1to2}, %xmm28
+// CHECK:  encoding: [0x62,0x61,0xfc,0x18,0x79,0xa2,0xf8,0xfb,0xff,0xff]
+          vcvtpd2udq -1032(%rdx){1to2}, %xmm28
+
+// CHECK: vcvtpd2udq %ymm20, %xmm21
+// CHECK:  encoding: [0x62,0xa1,0xfc,0x28,0x79,0xec]
+          vcvtpd2udq %ymm20, %xmm21
+
+// CHECK: vcvtpd2udq %ymm20, %xmm21 {%k2}
+// CHECK:  encoding: [0x62,0xa1,0xfc,0x2a,0x79,0xec]
+          vcvtpd2udq %ymm20, %xmm21 {%k2}
+
+// CHECK: vcvtpd2udq %ymm20, %xmm21 {%k2} {z}
+// CHECK:  encoding: [0x62,0xa1,0xfc,0xaa,0x79,0xec]
+          vcvtpd2udq %ymm20, %xmm21 {%k2} {z}
+
+// CHECK: vcvtpd2udqy (%rcx), %xmm21
+// CHECK:  encoding: [0x62,0xe1,0xfc,0x28,0x79,0x29]
+          vcvtpd2udqy (%rcx), %xmm21
+
+// CHECK: vcvtpd2udqy 291(%rax,%r14,8), %xmm21
+// CHECK:  encoding: [0x62,0xa1,0xfc,0x28,0x79,0xac,0xf0,0x23,0x01,0x00,0x00]
+          vcvtpd2udqy 291(%rax,%r14,8), %xmm21
+
+// CHECK: vcvtpd2udq (%rcx){1to4}, %xmm21
+// CHECK:  encoding: [0x62,0xe1,0xfc,0x38,0x79,0x29]
+          vcvtpd2udq (%rcx){1to4}, %xmm21
+
+// CHECK: vcvtpd2udqy 4064(%rdx), %xmm21
+// CHECK:  encoding: [0x62,0xe1,0xfc,0x28,0x79,0x6a,0x7f]
+          vcvtpd2udqy 4064(%rdx), %xmm21
+
+// CHECK: vcvtpd2udqy 4096(%rdx), %xmm21
+// CHECK:  encoding: [0x62,0xe1,0xfc,0x28,0x79,0xaa,0x00,0x10,0x00,0x00]
+          vcvtpd2udqy 4096(%rdx), %xmm21
+
+// CHECK: vcvtpd2udqy -4096(%rdx), %xmm21
+// CHECK:  encoding: [0x62,0xe1,0xfc,0x28,0x79,0x6a,0x80]
+          vcvtpd2udqy -4096(%rdx), %xmm21
+
+// CHECK: vcvtpd2udqy -4128(%rdx), %xmm21
+// CHECK:  encoding: [0x62,0xe1,0xfc,0x28,0x79,0xaa,0xe0,0xef,0xff,0xff]
+          vcvtpd2udqy -4128(%rdx), %xmm21
+
+// CHECK: vcvtpd2udq 1016(%rdx){1to4}, %xmm21
+// CHECK:  encoding: [0x62,0xe1,0xfc,0x38,0x79,0x6a,0x7f]
+          vcvtpd2udq 1016(%rdx){1to4}, %xmm21
+
+// CHECK: vcvtpd2udq 1024(%rdx){1to4}, %xmm21
+// CHECK:  encoding: [0x62,0xe1,0xfc,0x38,0x79,0xaa,0x00,0x04,0x00,0x00]
+          vcvtpd2udq 1024(%rdx){1to4}, %xmm21
+
+// CHECK: vcvtpd2udq -1024(%rdx){1to4}, %xmm21
+// CHECK:  encoding: [0x62,0xe1,0xfc,0x38,0x79,0x6a,0x80]
+          vcvtpd2udq -1024(%rdx){1to4}, %xmm21
+
+// CHECK: vcvtpd2udq -1032(%rdx){1to4}, %xmm21
+// CHECK:  encoding: [0x62,0xe1,0xfc,0x38,0x79,0xaa,0xf8,0xfb,0xff,0xff]
+          vcvtpd2udq -1032(%rdx){1to4}, %xmm21
+// CHECK: vcvtps2udq %xmm19, %xmm30
+// CHECK:  encoding: [0x62,0x21,0x7c,0x08,0x79,0xf3]
+          vcvtps2udq %xmm19, %xmm30
+
+// CHECK: vcvtps2udq %xmm19, %xmm30 {%k3}
+// CHECK:  encoding: [0x62,0x21,0x7c,0x0b,0x79,0xf3]
+          vcvtps2udq %xmm19, %xmm30 {%k3}
+
+// CHECK: vcvtps2udq %xmm19, %xmm30 {%k3} {z}
+// CHECK:  encoding: [0x62,0x21,0x7c,0x8b,0x79,0xf3]
+          vcvtps2udq %xmm19, %xmm30 {%k3} {z}
+
+// CHECK: vcvtps2udq (%rcx), %xmm30
+// CHECK:  encoding: [0x62,0x61,0x7c,0x08,0x79,0x31]
+          vcvtps2udq (%rcx), %xmm30
+
+// CHECK: vcvtps2udq 291(%rax,%r14,8), %xmm30
+// CHECK:  encoding: [0x62,0x21,0x7c,0x08,0x79,0xb4,0xf0,0x23,0x01,0x00,0x00]
+          vcvtps2udq 291(%rax,%r14,8), %xmm30
+
+// CHECK: vcvtps2udq (%rcx){1to4}, %xmm30
+// CHECK:  encoding: [0x62,0x61,0x7c,0x18,0x79,0x31]
+          vcvtps2udq (%rcx){1to4}, %xmm30
+
+// CHECK: vcvtps2udq 2032(%rdx), %xmm30
+// CHECK:  encoding: [0x62,0x61,0x7c,0x08,0x79,0x72,0x7f]
+          vcvtps2udq 2032(%rdx), %xmm30
+
+// CHECK: vcvtps2udq 2048(%rdx), %xmm30
+// CHECK:  encoding: [0x62,0x61,0x7c,0x08,0x79,0xb2,0x00,0x08,0x00,0x00]
+          vcvtps2udq 2048(%rdx), %xmm30
+
+// CHECK: vcvtps2udq -2048(%rdx), %xmm30
+// CHECK:  encoding: [0x62,0x61,0x7c,0x08,0x79,0x72,0x80]
+          vcvtps2udq -2048(%rdx), %xmm30
+
+// CHECK: vcvtps2udq -2064(%rdx), %xmm30
+// CHECK:  encoding: [0x62,0x61,0x7c,0x08,0x79,0xb2,0xf0,0xf7,0xff,0xff]
+          vcvtps2udq -2064(%rdx), %xmm30
+
+// CHECK: vcvtps2udq 508(%rdx){1to4}, %xmm30
+// CHECK:  encoding: [0x62,0x61,0x7c,0x18,0x79,0x72,0x7f]
+          vcvtps2udq 508(%rdx){1to4}, %xmm30
+
+// CHECK: vcvtps2udq 512(%rdx){1to4}, %xmm30
+// CHECK:  encoding: [0x62,0x61,0x7c,0x18,0x79,0xb2,0x00,0x02,0x00,0x00]
+          vcvtps2udq 512(%rdx){1to4}, %xmm30
+
+// CHECK: vcvtps2udq -512(%rdx){1to4}, %xmm30
+// CHECK:  encoding: [0x62,0x61,0x7c,0x18,0x79,0x72,0x80]
+          vcvtps2udq -512(%rdx){1to4}, %xmm30
+
+// CHECK: vcvtps2udq -516(%rdx){1to4}, %xmm30
+// CHECK:  encoding: [0x62,0x61,0x7c,0x18,0x79,0xb2,0xfc,0xfd,0xff,0xff]
+          vcvtps2udq -516(%rdx){1to4}, %xmm30
+
+// CHECK: vcvtps2udq %ymm23, %ymm25
+// CHECK:  encoding: [0x62,0x21,0x7c,0x28,0x79,0xcf]
+          vcvtps2udq %ymm23, %ymm25
+
+// CHECK: vcvtps2udq %ymm23, %ymm25 {%k4}
+// CHECK:  encoding: [0x62,0x21,0x7c,0x2c,0x79,0xcf]
+          vcvtps2udq %ymm23, %ymm25 {%k4}
+
+// CHECK: vcvtps2udq %ymm23, %ymm25 {%k4} {z}
+// CHECK:  encoding: [0x62,0x21,0x7c,0xac,0x79,0xcf]
+          vcvtps2udq %ymm23, %ymm25 {%k4} {z}
+
+// CHECK: vcvtps2udq (%rcx), %ymm25
+// CHECK:  encoding: [0x62,0x61,0x7c,0x28,0x79,0x09]
+          vcvtps2udq (%rcx), %ymm25
+
+// CHECK: vcvtps2udq 291(%rax,%r14,8), %ymm25
+// CHECK:  encoding: [0x62,0x21,0x7c,0x28,0x79,0x8c,0xf0,0x23,0x01,0x00,0x00]
+          vcvtps2udq 291(%rax,%r14,8), %ymm25
+
+// CHECK: vcvtps2udq (%rcx){1to8}, %ymm25
+// CHECK:  encoding: [0x62,0x61,0x7c,0x38,0x79,0x09]
+          vcvtps2udq (%rcx){1to8}, %ymm25
+
+// CHECK: vcvtps2udq 4064(%rdx), %ymm25
+// CHECK:  encoding: [0x62,0x61,0x7c,0x28,0x79,0x4a,0x7f]
+          vcvtps2udq 4064(%rdx), %ymm25
+
+// CHECK: vcvtps2udq 4096(%rdx), %ymm25
+// CHECK:  encoding: [0x62,0x61,0x7c,0x28,0x79,0x8a,0x00,0x10,0x00,0x00]
+          vcvtps2udq 4096(%rdx), %ymm25
+
+// CHECK: vcvtps2udq -4096(%rdx), %ymm25
+// CHECK:  encoding: [0x62,0x61,0x7c,0x28,0x79,0x4a,0x80]
+          vcvtps2udq -4096(%rdx), %ymm25
+
+// CHECK: vcvtps2udq -4128(%rdx), %ymm25
+// CHECK:  encoding: [0x62,0x61,0x7c,0x28,0x79,0x8a,0xe0,0xef,0xff,0xff]
+          vcvtps2udq -4128(%rdx), %ymm25
+
+// CHECK: vcvtps2udq 508(%rdx){1to8}, %ymm25
+// CHECK:  encoding: [0x62,0x61,0x7c,0x38,0x79,0x4a,0x7f]
+          vcvtps2udq 508(%rdx){1to8}, %ymm25
+
+// CHECK: vcvtps2udq 512(%rdx){1to8}, %ymm25
+// CHECK:  encoding: [0x62,0x61,0x7c,0x38,0x79,0x8a,0x00,0x02,0x00,0x00]
+          vcvtps2udq 512(%rdx){1to8}, %ymm25
+
+// CHECK: vcvtps2udq -512(%rdx){1to8}, %ymm25
+// CHECK:  encoding: [0x62,0x61,0x7c,0x38,0x79,0x4a,0x80]
+          vcvtps2udq -512(%rdx){1to8}, %ymm25
+
+// CHECK: vcvtps2udq -516(%rdx){1to8}, %ymm25
+// CHECK:  encoding: [0x62,0x61,0x7c,0x38,0x79,0x8a,0xfc,0xfd,0xff,0xff]
+          vcvtps2udq -516(%rdx){1to8}, %ymm25
+
+// CHECK: vcvttpd2dq %xmm23, %xmm29
+// CHECK:  encoding: [0x62,0x21,0xfd,0x08,0xe6,0xef]
+          vcvttpd2dq %xmm23, %xmm29
+
+// CHECK: vcvttpd2dq %xmm23, %xmm29 {%k6}
+// CHECK:  encoding: [0x62,0x21,0xfd,0x0e,0xe6,0xef]
+          vcvttpd2dq %xmm23, %xmm29 {%k6}
+
+// CHECK: vcvttpd2dq %xmm23, %xmm29 {%k6} {z}
+// CHECK:  encoding: [0x62,0x21,0xfd,0x8e,0xe6,0xef]
+          vcvttpd2dq %xmm23, %xmm29 {%k6} {z}
+
+// CHECK: vcvttpd2dqx (%rcx), %xmm29
+// CHECK:  encoding: [0x62,0x61,0xfd,0x08,0xe6,0x29]
+          vcvttpd2dqx (%rcx), %xmm29
+
+// CHECK: vcvttpd2dqx 291(%rax,%r14,8), %xmm29
+// CHECK:  encoding: [0x62,0x21,0xfd,0x08,0xe6,0xac,0xf0,0x23,0x01,0x00,0x00]
+          vcvttpd2dqx 291(%rax,%r14,8), %xmm29
+
+// CHECK: vcvttpd2dq (%rcx){1to2}, %xmm29
+// CHECK:  encoding: [0x62,0x61,0xfd,0x18,0xe6,0x29]
+          vcvttpd2dq (%rcx){1to2}, %xmm29
+
+// CHECK: vcvttpd2dqx 2032(%rdx), %xmm29
+// CHECK:  encoding: [0x62,0x61,0xfd,0x08,0xe6,0x6a,0x7f]
+          vcvttpd2dqx 2032(%rdx), %xmm29
+
+// CHECK: vcvttpd2dqx 2048(%rdx), %xmm29
+// CHECK:  encoding: [0x62,0x61,0xfd,0x08,0xe6,0xaa,0x00,0x08,0x00,0x00]
+          vcvttpd2dqx 2048(%rdx), %xmm29
+
+// CHECK: vcvttpd2dqx -2048(%rdx), %xmm29
+// CHECK:  encoding: [0x62,0x61,0xfd,0x08,0xe6,0x6a,0x80]
+          vcvttpd2dqx -2048(%rdx), %xmm29
+
+// CHECK: vcvttpd2dqx -2064(%rdx), %xmm29
+// CHECK:  encoding: [0x62,0x61,0xfd,0x08,0xe6,0xaa,0xf0,0xf7,0xff,0xff]
+          vcvttpd2dqx -2064(%rdx), %xmm29
+
+// CHECK: vcvttpd2dq 1016(%rdx){1to2}, %xmm29
+// CHECK:  encoding: [0x62,0x61,0xfd,0x18,0xe6,0x6a,0x7f]
+          vcvttpd2dq 1016(%rdx){1to2}, %xmm29
+
+// CHECK: vcvttpd2dq 1024(%rdx){1to2}, %xmm29
+// CHECK:  encoding: [0x62,0x61,0xfd,0x18,0xe6,0xaa,0x00,0x04,0x00,0x00]
+          vcvttpd2dq 1024(%rdx){1to2}, %xmm29
+
+// CHECK: vcvttpd2dq -1024(%rdx){1to2}, %xmm29
+// CHECK:  encoding: [0x62,0x61,0xfd,0x18,0xe6,0x6a,0x80]
+          vcvttpd2dq -1024(%rdx){1to2}, %xmm29
+
+// CHECK: vcvttpd2dq -1032(%rdx){1to2}, %xmm29
+// CHECK:  encoding: [0x62,0x61,0xfd,0x18,0xe6,0xaa,0xf8,0xfb,0xff,0xff]
+          vcvttpd2dq -1032(%rdx){1to2}, %xmm29
+
+// CHECK: vcvttpd2dq %ymm29, %xmm30
+// CHECK:  encoding: [0x62,0x01,0xfd,0x28,0xe6,0xf5]
+          vcvttpd2dq %ymm29, %xmm30
+
+// CHECK: vcvttpd2dq %ymm29, %xmm30 {%k6}
+// CHECK:  encoding: [0x62,0x01,0xfd,0x2e,0xe6,0xf5]
+          vcvttpd2dq %ymm29, %xmm30 {%k6}
+
+// CHECK: vcvttpd2dq %ymm29, %xmm30 {%k6} {z}
+// CHECK:  encoding: [0x62,0x01,0xfd,0xae,0xe6,0xf5]
+          vcvttpd2dq %ymm29, %xmm30 {%k6} {z}
+
+// CHECK: vcvttpd2dqy (%rcx), %xmm30
+// CHECK:  encoding: [0x62,0x61,0xfd,0x28,0xe6,0x31]
+          vcvttpd2dqy (%rcx), %xmm30
+
+// CHECK: vcvttpd2dqy 291(%rax,%r14,8), %xmm30
+// CHECK:  encoding: [0x62,0x21,0xfd,0x28,0xe6,0xb4,0xf0,0x23,0x01,0x00,0x00]
+          vcvttpd2dqy 291(%rax,%r14,8), %xmm30
+
+// CHECK: vcvttpd2dq (%rcx){1to4}, %xmm30
+// CHECK:  encoding: [0x62,0x61,0xfd,0x38,0xe6,0x31]
+          vcvttpd2dq (%rcx){1to4}, %xmm30
+
+// CHECK: vcvttpd2dqy 4064(%rdx), %xmm30
+// CHECK:  encoding: [0x62,0x61,0xfd,0x28,0xe6,0x72,0x7f]
+          vcvttpd2dqy 4064(%rdx), %xmm30
+
+// CHECK: vcvttpd2dqy 4096(%rdx), %xmm30
+// CHECK:  encoding: [0x62,0x61,0xfd,0x28,0xe6,0xb2,0x00,0x10,0x00,0x00]
+          vcvttpd2dqy 4096(%rdx), %xmm30
+
+// CHECK: vcvttpd2dqy -4096(%rdx), %xmm30
+// CHECK:  encoding: [0x62,0x61,0xfd,0x28,0xe6,0x72,0x80]
+          vcvttpd2dqy -4096(%rdx), %xmm30
+
+// CHECK: vcvttpd2dqy -4128(%rdx), %xmm30
+// CHECK:  encoding: [0x62,0x61,0xfd,0x28,0xe6,0xb2,0xe0,0xef,0xff,0xff]
+          vcvttpd2dqy -4128(%rdx), %xmm30
+
+// CHECK: vcvttpd2dq 1016(%rdx){1to4}, %xmm30
+// CHECK:  encoding: [0x62,0x61,0xfd,0x38,0xe6,0x72,0x7f]
+          vcvttpd2dq 1016(%rdx){1to4}, %xmm30
+
+// CHECK: vcvttpd2dq 1024(%rdx){1to4}, %xmm30
+// CHECK:  encoding: [0x62,0x61,0xfd,0x38,0xe6,0xb2,0x00,0x04,0x00,0x00]
+          vcvttpd2dq 1024(%rdx){1to4}, %xmm30
+
+// CHECK: vcvttpd2dq -1024(%rdx){1to4}, %xmm30
+// CHECK:  encoding: [0x62,0x61,0xfd,0x38,0xe6,0x72,0x80]
+          vcvttpd2dq -1024(%rdx){1to4}, %xmm30
+
+// CHECK: vcvttpd2dq -1032(%rdx){1to4}, %xmm30
+// CHECK:  encoding: [0x62,0x61,0xfd,0x38,0xe6,0xb2,0xf8,0xfb,0xff,0xff]
+          vcvttpd2dq -1032(%rdx){1to4}, %xmm30
+
+// CHECK: vcvttps2dq %xmm22, %xmm17
+// CHECK:  encoding: [0x62,0xa1,0x7e,0x08,0x5b,0xce]
+          vcvttps2dq %xmm22, %xmm17
+
+// CHECK: vcvttps2dq %xmm22, %xmm17 {%k4}
+// CHECK:  encoding: [0x62,0xa1,0x7e,0x0c,0x5b,0xce]
+          vcvttps2dq %xmm22, %xmm17 {%k4}
+
+// CHECK: vcvttps2dq %xmm22, %xmm17 {%k4} {z}
+// CHECK:  encoding: [0x62,0xa1,0x7e,0x8c,0x5b,0xce]
+          vcvttps2dq %xmm22, %xmm17 {%k4} {z}
+
+// CHECK: vcvttps2dq (%rcx), %xmm17
+// CHECK:  encoding: [0x62,0xe1,0x7e,0x08,0x5b,0x09]
+          vcvttps2dq (%rcx), %xmm17
+
+// CHECK: vcvttps2dq 291(%rax,%r14,8), %xmm17
+// CHECK:  encoding: [0x62,0xa1,0x7e,0x08,0x5b,0x8c,0xf0,0x23,0x01,0x00,0x00]
+          vcvttps2dq 291(%rax,%r14,8), %xmm17
+
+// CHECK: vcvttps2dq (%rcx){1to4}, %xmm17
+// CHECK:  encoding: [0x62,0xe1,0x7e,0x18,0x5b,0x09]
+          vcvttps2dq (%rcx){1to4}, %xmm17
+
+// CHECK: vcvttps2dq 2032(%rdx), %xmm17
+// CHECK:  encoding: [0x62,0xe1,0x7e,0x08,0x5b,0x4a,0x7f]
+          vcvttps2dq 2032(%rdx), %xmm17
+
+// CHECK: vcvttps2dq 2048(%rdx), %xmm17
+// CHECK:  encoding: [0x62,0xe1,0x7e,0x08,0x5b,0x8a,0x00,0x08,0x00,0x00]
+          vcvttps2dq 2048(%rdx), %xmm17
+
+// CHECK: vcvttps2dq -2048(%rdx), %xmm17
+// CHECK:  encoding: [0x62,0xe1,0x7e,0x08,0x5b,0x4a,0x80]
+          vcvttps2dq -2048(%rdx), %xmm17
+
+// CHECK: vcvttps2dq -2064(%rdx), %xmm17
+// CHECK:  encoding: [0x62,0xe1,0x7e,0x08,0x5b,0x8a,0xf0,0xf7,0xff,0xff]
+          vcvttps2dq -2064(%rdx), %xmm17
+
+// CHECK: vcvttps2dq 508(%rdx){1to4}, %xmm17
+// CHECK:  encoding: [0x62,0xe1,0x7e,0x18,0x5b,0x4a,0x7f]
+          vcvttps2dq 508(%rdx){1to4}, %xmm17
+
+// CHECK: vcvttps2dq 512(%rdx){1to4}, %xmm17
+// CHECK:  encoding: [0x62,0xe1,0x7e,0x18,0x5b,0x8a,0x00,0x02,0x00,0x00]
+          vcvttps2dq 512(%rdx){1to4}, %xmm17
+
+// CHECK: vcvttps2dq -512(%rdx){1to4}, %xmm17
+// CHECK:  encoding: [0x62,0xe1,0x7e,0x18,0x5b,0x4a,0x80]
+          vcvttps2dq -512(%rdx){1to4}, %xmm17
+
+// CHECK: vcvttps2dq -516(%rdx){1to4}, %xmm17
+// CHECK:  encoding: [0x62,0xe1,0x7e,0x18,0x5b,0x8a,0xfc,0xfd,0xff,0xff]
+          vcvttps2dq -516(%rdx){1to4}, %xmm17
+
+// CHECK: vcvttps2dq %ymm28, %ymm25
+// CHECK:  encoding: [0x62,0x01,0x7e,0x28,0x5b,0xcc]
+          vcvttps2dq %ymm28, %ymm25
+
+// CHECK: vcvttps2dq %ymm28, %ymm25 {%k2}
+// CHECK:  encoding: [0x62,0x01,0x7e,0x2a,0x5b,0xcc]
+          vcvttps2dq %ymm28, %ymm25 {%k2}
+
+// CHECK: vcvttps2dq %ymm28, %ymm25 {%k2} {z}
+// CHECK:  encoding: [0x62,0x01,0x7e,0xaa,0x5b,0xcc]
+          vcvttps2dq %ymm28, %ymm25 {%k2} {z}
+
+// CHECK: vcvttps2dq (%rcx), %ymm25
+// CHECK:  encoding: [0x62,0x61,0x7e,0x28,0x5b,0x09]
+          vcvttps2dq (%rcx), %ymm25
+
+// CHECK: vcvttps2dq 291(%rax,%r14,8), %ymm25
+// CHECK:  encoding: [0x62,0x21,0x7e,0x28,0x5b,0x8c,0xf0,0x23,0x01,0x00,0x00]
+          vcvttps2dq 291(%rax,%r14,8), %ymm25
+
+// CHECK: vcvttps2dq (%rcx){1to8}, %ymm25
+// CHECK:  encoding: [0x62,0x61,0x7e,0x38,0x5b,0x09]
+          vcvttps2dq (%rcx){1to8}, %ymm25
+
+// CHECK: vcvttps2dq 4064(%rdx), %ymm25
+// CHECK:  encoding: [0x62,0x61,0x7e,0x28,0x5b,0x4a,0x7f]
+          vcvttps2dq 4064(%rdx), %ymm25
+
+// CHECK: vcvttps2dq 4096(%rdx), %ymm25
+// CHECK:  encoding: [0x62,0x61,0x7e,0x28,0x5b,0x8a,0x00,0x10,0x00,0x00]
+          vcvttps2dq 4096(%rdx), %ymm25
+
+// CHECK: vcvttps2dq -4096(%rdx), %ymm25
+// CHECK:  encoding: [0x62,0x61,0x7e,0x28,0x5b,0x4a,0x80]
+          vcvttps2dq -4096(%rdx), %ymm25
+
+// CHECK: vcvttps2dq -4128(%rdx), %ymm25
+// CHECK:  encoding: [0x62,0x61,0x7e,0x28,0x5b,0x8a,0xe0,0xef,0xff,0xff]
+          vcvttps2dq -4128(%rdx), %ymm25
+
+// CHECK: vcvttps2dq 508(%rdx){1to8}, %ymm25
+// CHECK:  encoding: [0x62,0x61,0x7e,0x38,0x5b,0x4a,0x7f]
+          vcvttps2dq 508(%rdx){1to8}, %ymm25
+
+// CHECK: vcvttps2dq 512(%rdx){1to8}, %ymm25
+// CHECK:  encoding: [0x62,0x61,0x7e,0x38,0x5b,0x8a,0x00,0x02,0x00,0x00]
+          vcvttps2dq 512(%rdx){1to8}, %ymm25
+
+// CHECK: vcvttps2dq -512(%rdx){1to8}, %ymm25
+// CHECK:  encoding: [0x62,0x61,0x7e,0x38,0x5b,0x4a,0x80]
+          vcvttps2dq -512(%rdx){1to8}, %ymm25
+
+// CHECK: vcvttps2dq -516(%rdx){1to8}, %ymm25
+// CHECK:  encoding: [0x62,0x61,0x7e,0x38,0x5b,0x8a,0xfc,0xfd,0xff,0xff]
+          vcvttps2dq -516(%rdx){1to8}, %ymm25
+
+// CHECK: vcvtudq2pd %xmm19, %xmm25
+// CHECK:  encoding: [0x62,0x21,0x7e,0x08,0x7a,0xcb]
+          vcvtudq2pd %xmm19, %xmm25
+
+// CHECK: vcvtudq2pd %xmm19, %xmm25 {%k4}
+// CHECK:  encoding: [0x62,0x21,0x7e,0x0c,0x7a,0xcb]
+          vcvtudq2pd %xmm19, %xmm25 {%k4}
+
+// CHECK: vcvtudq2pd %xmm19, %xmm25 {%k4} {z}
+// CHECK:  encoding: [0x62,0x21,0x7e,0x8c,0x7a,0xcb]
+          vcvtudq2pd %xmm19, %xmm25 {%k4} {z}
+
+// CHECK: vcvtudq2pd (%rcx), %xmm25
+// CHECK:  encoding: [0x62,0x61,0x7e,0x08,0x7a,0x09]
+          vcvtudq2pd (%rcx), %xmm25
+
+// CHECK: vcvtudq2pd 291(%rax,%r14,8), %xmm25
+// CHECK:  encoding: [0x62,0x21,0x7e,0x08,0x7a,0x8c,0xf0,0x23,0x01,0x00,0x00]
+          vcvtudq2pd 291(%rax,%r14,8), %xmm25
+
+// CHECK: vcvtudq2pd (%rcx){1to2}, %xmm25
+// CHECK:  encoding: [0x62,0x61,0x7e,0x18,0x7a,0x09]
+          vcvtudq2pd (%rcx){1to2}, %xmm25
+
+// CHECK: vcvtudq2pd 1016(%rdx), %xmm25
+// CHECK:  encoding: [0x62,0x61,0x7e,0x08,0x7a,0x4a,0x7f]
+          vcvtudq2pd 1016(%rdx), %xmm25
+
+// CHECK: vcvtudq2pd 1024(%rdx), %xmm25
+// CHECK:  encoding: [0x62,0x61,0x7e,0x08,0x7a,0x8a,0x00,0x04,0x00,0x00]
+          vcvtudq2pd 1024(%rdx), %xmm25
+
+// CHECK: vcvtudq2pd -1024(%rdx), %xmm25
+// CHECK:  encoding: [0x62,0x61,0x7e,0x08,0x7a,0x4a,0x80]
+          vcvtudq2pd -1024(%rdx), %xmm25
+
+// CHECK: vcvtudq2pd -1032(%rdx), %xmm25
+// CHECK:  encoding: [0x62,0x61,0x7e,0x08,0x7a,0x8a,0xf8,0xfb,0xff,0xff]
+          vcvtudq2pd -1032(%rdx), %xmm25
+
+// CHECK: vcvtudq2pd 508(%rdx){1to2}, %xmm25
+// CHECK:  encoding: [0x62,0x61,0x7e,0x18,0x7a,0x4a,0x7f]
+          vcvtudq2pd 508(%rdx){1to2}, %xmm25
+
+// CHECK: vcvtudq2pd 512(%rdx){1to2}, %xmm25
+// CHECK:  encoding: [0x62,0x61,0x7e,0x18,0x7a,0x8a,0x00,0x02,0x00,0x00]
+          vcvtudq2pd 512(%rdx){1to2}, %xmm25
+
+// CHECK: vcvtudq2pd -512(%rdx){1to2}, %xmm25
+// CHECK:  encoding: [0x62,0x61,0x7e,0x18,0x7a,0x4a,0x80]
+          vcvtudq2pd -512(%rdx){1to2}, %xmm25
+
+// CHECK: vcvtudq2pd -516(%rdx){1to2}, %xmm25
+// CHECK:  encoding: [0x62,0x61,0x7e,0x18,0x7a,0x8a,0xfc,0xfd,0xff,0xff]
+          vcvtudq2pd -516(%rdx){1to2}, %xmm25
+
+// CHECK: vcvtudq2pd %xmm20, %ymm25
+// CHECK:  encoding: [0x62,0x21,0x7e,0x28,0x7a,0xcc]
+          vcvtudq2pd %xmm20, %ymm25
+
+// CHECK: vcvtudq2pd %xmm20, %ymm25 {%k6}
+// CHECK:  encoding: [0x62,0x21,0x7e,0x2e,0x7a,0xcc]
+          vcvtudq2pd %xmm20, %ymm25 {%k6}
+
+// CHECK: vcvtudq2pd %xmm20, %ymm25 {%k6} {z}
+// CHECK:  encoding: [0x62,0x21,0x7e,0xae,0x7a,0xcc]
+          vcvtudq2pd %xmm20, %ymm25 {%k6} {z}
+
+// CHECK: vcvtudq2pd (%rcx), %ymm25
+// CHECK:  encoding: [0x62,0x61,0x7e,0x28,0x7a,0x09]
+          vcvtudq2pd (%rcx), %ymm25
+
+// CHECK: vcvtudq2pd 291(%rax,%r14,8), %ymm25
+// CHECK:  encoding: [0x62,0x21,0x7e,0x28,0x7a,0x8c,0xf0,0x23,0x01,0x00,0x00]
+          vcvtudq2pd 291(%rax,%r14,8), %ymm25
+
+// CHECK: vcvtudq2pd (%rcx){1to4}, %ymm25
+// CHECK:  encoding: [0x62,0x61,0x7e,0x38,0x7a,0x09]
+          vcvtudq2pd (%rcx){1to4}, %ymm25
+
+// CHECK: vcvtudq2pd 2032(%rdx), %ymm25
+// CHECK:  encoding: [0x62,0x61,0x7e,0x28,0x7a,0x4a,0x7f]
+          vcvtudq2pd 2032(%rdx), %ymm25
+
+// CHECK: vcvtudq2pd 2048(%rdx), %ymm25
+// CHECK:  encoding: [0x62,0x61,0x7e,0x28,0x7a,0x8a,0x00,0x08,0x00,0x00]
+          vcvtudq2pd 2048(%rdx), %ymm25
+
+// CHECK: vcvtudq2pd -2048(%rdx), %ymm25
+// CHECK:  encoding: [0x62,0x61,0x7e,0x28,0x7a,0x4a,0x80]
+          vcvtudq2pd -2048(%rdx), %ymm25
+
+// CHECK: vcvtudq2pd -2064(%rdx), %ymm25
+// CHECK:  encoding: [0x62,0x61,0x7e,0x28,0x7a,0x8a,0xf0,0xf7,0xff,0xff]
+          vcvtudq2pd -2064(%rdx), %ymm25
+
+// CHECK: vcvtudq2pd 508(%rdx){1to4}, %ymm25
+// CHECK:  encoding: [0x62,0x61,0x7e,0x38,0x7a,0x4a,0x7f]
+          vcvtudq2pd 508(%rdx){1to4}, %ymm25
+
+// CHECK: vcvtudq2pd 512(%rdx){1to4}, %ymm25
+// CHECK:  encoding: [0x62,0x61,0x7e,0x38,0x7a,0x8a,0x00,0x02,0x00,0x00]
+          vcvtudq2pd 512(%rdx){1to4}, %ymm25
+
+// CHECK: vcvtudq2pd -512(%rdx){1to4}, %ymm25
+// CHECK:  encoding: [0x62,0x61,0x7e,0x38,0x7a,0x4a,0x80]
+          vcvtudq2pd -512(%rdx){1to4}, %ymm25
+
+// CHECK: vcvtudq2pd -516(%rdx){1to4}, %ymm25
+// CHECK:  encoding: [0x62,0x61,0x7e,0x38,0x7a,0x8a,0xfc,0xfd,0xff,0xff]
+          vcvtudq2pd -516(%rdx){1to4}, %ymm25
+
+// CHECK: vcvtudq2ps %xmm23, %xmm23
+// CHECK:  encoding: [0x62,0xa1,0x7f,0x08,0x7a,0xff]
+          vcvtudq2ps %xmm23, %xmm23
+
+// CHECK: vcvtudq2ps %xmm23, %xmm23 {%k1}
+// CHECK:  encoding: [0x62,0xa1,0x7f,0x09,0x7a,0xff]
+          vcvtudq2ps %xmm23, %xmm23 {%k1}
+
+// CHECK: vcvtudq2ps %xmm23, %xmm23 {%k1} {z}
+// CHECK:  encoding: [0x62,0xa1,0x7f,0x89,0x7a,0xff]
+          vcvtudq2ps %xmm23, %xmm23 {%k1} {z}
+
+// CHECK: vcvtudq2ps (%rcx), %xmm23
+// CHECK:  encoding: [0x62,0xe1,0x7f,0x08,0x7a,0x39]
+          vcvtudq2ps (%rcx), %xmm23
+
+// CHECK: vcvtudq2ps 291(%rax,%r14,8), %xmm23
+// CHECK:  encoding: [0x62,0xa1,0x7f,0x08,0x7a,0xbc,0xf0,0x23,0x01,0x00,0x00]
+          vcvtudq2ps 291(%rax,%r14,8), %xmm23
+
+// CHECK: vcvtudq2ps (%rcx){1to4}, %xmm23
+// CHECK:  encoding: [0x62,0xe1,0x7f,0x18,0x7a,0x39]
+          vcvtudq2ps (%rcx){1to4}, %xmm23
+
+// CHECK: vcvtudq2ps 2032(%rdx), %xmm23
+// CHECK:  encoding: [0x62,0xe1,0x7f,0x08,0x7a,0x7a,0x7f]
+          vcvtudq2ps 2032(%rdx), %xmm23
+
+// CHECK: vcvtudq2ps 2048(%rdx), %xmm23
+// CHECK:  encoding: [0x62,0xe1,0x7f,0x08,0x7a,0xba,0x00,0x08,0x00,0x00]
+          vcvtudq2ps 2048(%rdx), %xmm23
+
+// CHECK: vcvtudq2ps -2048(%rdx), %xmm23
+// CHECK:  encoding: [0x62,0xe1,0x7f,0x08,0x7a,0x7a,0x80]
+          vcvtudq2ps -2048(%rdx), %xmm23
+
+// CHECK: vcvtudq2ps -2064(%rdx), %xmm23
+// CHECK:  encoding: [0x62,0xe1,0x7f,0x08,0x7a,0xba,0xf0,0xf7,0xff,0xff]
+          vcvtudq2ps -2064(%rdx), %xmm23
+
+// CHECK: vcvtudq2ps 508(%rdx){1to4}, %xmm23
+// CHECK:  encoding: [0x62,0xe1,0x7f,0x18,0x7a,0x7a,0x7f]
+          vcvtudq2ps 508(%rdx){1to4}, %xmm23
+
+// CHECK: vcvtudq2ps 512(%rdx){1to4}, %xmm23
+// CHECK:  encoding: [0x62,0xe1,0x7f,0x18,0x7a,0xba,0x00,0x02,0x00,0x00]
+          vcvtudq2ps 512(%rdx){1to4}, %xmm23
+
+// CHECK: vcvtudq2ps -512(%rdx){1to4}, %xmm23
+// CHECK:  encoding: [0x62,0xe1,0x7f,0x18,0x7a,0x7a,0x80]
+          vcvtudq2ps -512(%rdx){1to4}, %xmm23
+
+// CHECK: vcvtudq2ps -516(%rdx){1to4}, %xmm23
+// CHECK:  encoding: [0x62,0xe1,0x7f,0x18,0x7a,0xba,0xfc,0xfd,0xff,0xff]
+          vcvtudq2ps -516(%rdx){1to4}, %xmm23
+
+// CHECK: vcvtudq2ps %ymm25, %ymm23
+// CHECK:  encoding: [0x62,0x81,0x7f,0x28,0x7a,0xf9]
+          vcvtudq2ps %ymm25, %ymm23
+
+// CHECK: vcvtudq2ps %ymm25, %ymm23 {%k3}
+// CHECK:  encoding: [0x62,0x81,0x7f,0x2b,0x7a,0xf9]
+          vcvtudq2ps %ymm25, %ymm23 {%k3}
+
+// CHECK: vcvtudq2ps %ymm25, %ymm23 {%k3} {z}
+// CHECK:  encoding: [0x62,0x81,0x7f,0xab,0x7a,0xf9]
+          vcvtudq2ps %ymm25, %ymm23 {%k3} {z}
+
+// CHECK: vcvtudq2ps (%rcx), %ymm23
+// CHECK:  encoding: [0x62,0xe1,0x7f,0x28,0x7a,0x39]
+          vcvtudq2ps (%rcx), %ymm23
+
+// CHECK: vcvtudq2ps 291(%rax,%r14,8), %ymm23
+// CHECK:  encoding: [0x62,0xa1,0x7f,0x28,0x7a,0xbc,0xf0,0x23,0x01,0x00,0x00]
+          vcvtudq2ps 291(%rax,%r14,8), %ymm23
+
+// CHECK: vcvtudq2ps (%rcx){1to8}, %ymm23
+// CHECK:  encoding: [0x62,0xe1,0x7f,0x38,0x7a,0x39]
+          vcvtudq2ps (%rcx){1to8}, %ymm23
+
+// CHECK: vcvtudq2ps 4064(%rdx), %ymm23
+// CHECK:  encoding: [0x62,0xe1,0x7f,0x28,0x7a,0x7a,0x7f]
+          vcvtudq2ps 4064(%rdx), %ymm23
+
+// CHECK: vcvtudq2ps 4096(%rdx), %ymm23
+// CHECK:  encoding: [0x62,0xe1,0x7f,0x28,0x7a,0xba,0x00,0x10,0x00,0x00]
+          vcvtudq2ps 4096(%rdx), %ymm23
+
+// CHECK: vcvtudq2ps -4096(%rdx), %ymm23
+// CHECK:  encoding: [0x62,0xe1,0x7f,0x28,0x7a,0x7a,0x80]
+          vcvtudq2ps -4096(%rdx), %ymm23
+
+// CHECK: vcvtudq2ps -4128(%rdx), %ymm23
+// CHECK:  encoding: [0x62,0xe1,0x7f,0x28,0x7a,0xba,0xe0,0xef,0xff,0xff]
+          vcvtudq2ps -4128(%rdx), %ymm23
+
+// CHECK: vcvtudq2ps 508(%rdx){1to8}, %ymm23
+// CHECK:  encoding: [0x62,0xe1,0x7f,0x38,0x7a,0x7a,0x7f]
+          vcvtudq2ps 508(%rdx){1to8}, %ymm23
+
+// CHECK: vcvtudq2ps 512(%rdx){1to8}, %ymm23
+// CHECK:  encoding: [0x62,0xe1,0x7f,0x38,0x7a,0xba,0x00,0x02,0x00,0x00]
+          vcvtudq2ps 512(%rdx){1to8}, %ymm23
+
+// CHECK: vcvtudq2ps -512(%rdx){1to8}, %ymm23
+// CHECK:  encoding: [0x62,0xe1,0x7f,0x38,0x7a,0x7a,0x80]
+          vcvtudq2ps -512(%rdx){1to8}, %ymm23
+
+// CHECK: vcvtudq2ps -516(%rdx){1to8}, %ymm23
+// CHECK:  encoding: [0x62,0xe1,0x7f,0x38,0x7a,0xba,0xfc,0xfd,0xff,0xff]
+          vcvtudq2ps -516(%rdx){1to8}, %ymm23
+
+// CHECK: vcvtdq2pd %xmm21, %xmm20
+// CHECK:  encoding: [0x62,0xa1,0x7e,0x08,0xe6,0xe5]
+          vcvtdq2pd %xmm21, %xmm20
+
+// CHECK: vcvtdq2pd %xmm21, %xmm20 {%k5}
+// CHECK:  encoding: [0x62,0xa1,0x7e,0x0d,0xe6,0xe5]
+          vcvtdq2pd %xmm21, %xmm20 {%k5}
+
+// CHECK: vcvtdq2pd %xmm21, %xmm20 {%k5} {z}
+// CHECK:  encoding: [0x62,0xa1,0x7e,0x8d,0xe6,0xe5]
+          vcvtdq2pd %xmm21, %xmm20 {%k5} {z}
+
+// CHECK: vcvtdq2pd (%rcx), %xmm20
+// CHECK:  encoding: [0x62,0xe1,0x7e,0x08,0xe6,0x21]
+          vcvtdq2pd (%rcx), %xmm20
+
+// CHECK: vcvtdq2pd 291(%rax,%r14,8), %xmm20
+// CHECK:  encoding: [0x62,0xa1,0x7e,0x08,0xe6,0xa4,0xf0,0x23,0x01,0x00,0x00]
+          vcvtdq2pd 291(%rax,%r14,8), %xmm20
+
+// CHECK: vcvtdq2pd (%rcx){1to2}, %xmm20
+// CHECK:  encoding: [0x62,0xe1,0x7e,0x18,0xe6,0x21]
+          vcvtdq2pd (%rcx){1to2}, %xmm20
+
+// CHECK: vcvtdq2pd 1016(%rdx), %xmm20
+// CHECK:  encoding: [0x62,0xe1,0x7e,0x08,0xe6,0x62,0x7f]
+          vcvtdq2pd 1016(%rdx), %xmm20
+
+// CHECK: vcvtdq2pd 1024(%rdx), %xmm20
+// CHECK:  encoding: [0x62,0xe1,0x7e,0x08,0xe6,0xa2,0x00,0x04,0x00,0x00]
+          vcvtdq2pd 1024(%rdx), %xmm20
+
+// CHECK: vcvtdq2pd -1024(%rdx), %xmm20
+// CHECK:  encoding: [0x62,0xe1,0x7e,0x08,0xe6,0x62,0x80]
+          vcvtdq2pd -1024(%rdx), %xmm20
+
+// CHECK: vcvtdq2pd -1032(%rdx), %xmm20
+// CHECK:  encoding: [0x62,0xe1,0x7e,0x08,0xe6,0xa2,0xf8,0xfb,0xff,0xff]
+          vcvtdq2pd -1032(%rdx), %xmm20
+
+// CHECK: vcvtdq2pd 508(%rdx){1to2}, %xmm20
+// CHECK:  encoding: [0x62,0xe1,0x7e,0x18,0xe6,0x62,0x7f]
+          vcvtdq2pd 508(%rdx){1to2}, %xmm20
+
+// CHECK: vcvtdq2pd 512(%rdx){1to2}, %xmm20
+// CHECK:  encoding: [0x62,0xe1,0x7e,0x18,0xe6,0xa2,0x00,0x02,0x00,0x00]
+          vcvtdq2pd 512(%rdx){1to2}, %xmm20
+
+// CHECK: vcvtdq2pd -512(%rdx){1to2}, %xmm20
+// CHECK:  encoding: [0x62,0xe1,0x7e,0x18,0xe6,0x62,0x80]
+          vcvtdq2pd -512(%rdx){1to2}, %xmm20
+
+// CHECK: vcvtdq2pd -516(%rdx){1to2}, %xmm20
+// CHECK:  encoding: [0x62,0xe1,0x7e,0x18,0xe6,0xa2,0xfc,0xfd,0xff,0xff]
+          vcvtdq2pd -516(%rdx){1to2}, %xmm20
+
+// CHECK: vcvtdq2pd %xmm23, %ymm28
+// CHECK:  encoding: [0x62,0x21,0x7e,0x28,0xe6,0xe7]
+          vcvtdq2pd %xmm23, %ymm28
+
+// CHECK: vcvtdq2pd %xmm23, %ymm28 {%k2}
+// CHECK:  encoding: [0x62,0x21,0x7e,0x2a,0xe6,0xe7]
+          vcvtdq2pd %xmm23, %ymm28 {%k2}
+
+// CHECK: vcvtdq2pd %xmm23, %ymm28 {%k2} {z}
+// CHECK:  encoding: [0x62,0x21,0x7e,0xaa,0xe6,0xe7]
+          vcvtdq2pd %xmm23, %ymm28 {%k2} {z}
+
+// CHECK: vcvtdq2pd (%rcx), %ymm28
+// CHECK:  encoding: [0x62,0x61,0x7e,0x28,0xe6,0x21]
+          vcvtdq2pd (%rcx), %ymm28
+
+// CHECK: vcvtdq2pd 291(%rax,%r14,8), %ymm28
+// CHECK:  encoding: [0x62,0x21,0x7e,0x28,0xe6,0xa4,0xf0,0x23,0x01,0x00,0x00]
+          vcvtdq2pd 291(%rax,%r14,8), %ymm28
+
+// CHECK: vcvtdq2pd (%rcx){1to4}, %ymm28
+// CHECK:  encoding: [0x62,0x61,0x7e,0x38,0xe6,0x21]
+          vcvtdq2pd (%rcx){1to4}, %ymm28
+
+// CHECK: vcvtdq2pd 2032(%rdx), %ymm28
+// CHECK:  encoding: [0x62,0x61,0x7e,0x28,0xe6,0x62,0x7f]
+          vcvtdq2pd 2032(%rdx), %ymm28
+
+// CHECK: vcvtdq2pd 2048(%rdx), %ymm28
+// CHECK:  encoding: [0x62,0x61,0x7e,0x28,0xe6,0xa2,0x00,0x08,0x00,0x00]
+          vcvtdq2pd 2048(%rdx), %ymm28
+
+// CHECK: vcvtdq2pd -2048(%rdx), %ymm28
+// CHECK:  encoding: [0x62,0x61,0x7e,0x28,0xe6,0x62,0x80]
+          vcvtdq2pd -2048(%rdx), %ymm28
+
+// CHECK: vcvtdq2pd -2064(%rdx), %ymm28
+// CHECK:  encoding: [0x62,0x61,0x7e,0x28,0xe6,0xa2,0xf0,0xf7,0xff,0xff]
+          vcvtdq2pd -2064(%rdx), %ymm28
+
+// CHECK: vcvtdq2pd 508(%rdx){1to4}, %ymm28
+// CHECK:  encoding: [0x62,0x61,0x7e,0x38,0xe6,0x62,0x7f]
+          vcvtdq2pd 508(%rdx){1to4}, %ymm28
+
+// CHECK: vcvtdq2pd 512(%rdx){1to4}, %ymm28
+// CHECK:  encoding: [0x62,0x61,0x7e,0x38,0xe6,0xa2,0x00,0x02,0x00,0x00]
+          vcvtdq2pd 512(%rdx){1to4}, %ymm28
+
+// CHECK: vcvtdq2pd -512(%rdx){1to4}, %ymm28
+// CHECK:  encoding: [0x62,0x61,0x7e,0x38,0xe6,0x62,0x80]
+          vcvtdq2pd -512(%rdx){1to4}, %ymm28
+
+// CHECK: vcvtdq2pd -516(%rdx){1to4}, %ymm28
+// CHECK:  encoding: [0x62,0x61,0x7e,0x38,0xe6,0xa2,0xfc,0xfd,0xff,0xff]
+          vcvtdq2pd -516(%rdx){1to4}, %ymm28
+
+// CHECK: vcvtdq2ps %xmm22, %xmm28
+// CHECK:  encoding: [0x62,0x21,0x7c,0x08,0x5b,0xe6]
+          vcvtdq2ps %xmm22, %xmm28
+
+// CHECK: vcvtdq2ps %xmm22, %xmm28 {%k1}
+// CHECK:  encoding: [0x62,0x21,0x7c,0x09,0x5b,0xe6]
+          vcvtdq2ps %xmm22, %xmm28 {%k1}
+
+// CHECK: vcvtdq2ps %xmm22, %xmm28 {%k1} {z}
+// CHECK:  encoding: [0x62,0x21,0x7c,0x89,0x5b,0xe6]
+          vcvtdq2ps %xmm22, %xmm28 {%k1} {z}
+
+// CHECK: vcvtdq2ps (%rcx), %xmm28
+// CHECK:  encoding: [0x62,0x61,0x7c,0x08,0x5b,0x21]
+          vcvtdq2ps (%rcx), %xmm28
+
+// CHECK: vcvtdq2ps 291(%rax,%r14,8), %xmm28
+// CHECK:  encoding: [0x62,0x21,0x7c,0x08,0x5b,0xa4,0xf0,0x23,0x01,0x00,0x00]
+          vcvtdq2ps 291(%rax,%r14,8), %xmm28
+
+// CHECK: vcvtdq2ps (%rcx){1to4}, %xmm28
+// CHECK:  encoding: [0x62,0x61,0x7c,0x18,0x5b,0x21]
+          vcvtdq2ps (%rcx){1to4}, %xmm28
+
+// CHECK: vcvtdq2ps 2032(%rdx), %xmm28
+// CHECK:  encoding: [0x62,0x61,0x7c,0x08,0x5b,0x62,0x7f]
+          vcvtdq2ps 2032(%rdx), %xmm28
+
+// CHECK: vcvtdq2ps 2048(%rdx), %xmm28
+// CHECK:  encoding: [0x62,0x61,0x7c,0x08,0x5b,0xa2,0x00,0x08,0x00,0x00]
+          vcvtdq2ps 2048(%rdx), %xmm28
+
+// CHECK: vcvtdq2ps -2048(%rdx), %xmm28
+// CHECK:  encoding: [0x62,0x61,0x7c,0x08,0x5b,0x62,0x80]
+          vcvtdq2ps -2048(%rdx), %xmm28
+
+// CHECK: vcvtdq2ps -2064(%rdx), %xmm28
+// CHECK:  encoding: [0x62,0x61,0x7c,0x08,0x5b,0xa2,0xf0,0xf7,0xff,0xff]
+          vcvtdq2ps -2064(%rdx), %xmm28
+
+// CHECK: vcvtdq2ps 508(%rdx){1to4}, %xmm28
+// CHECK:  encoding: [0x62,0x61,0x7c,0x18,0x5b,0x62,0x7f]
+          vcvtdq2ps 508(%rdx){1to4}, %xmm28
+
+// CHECK: vcvtdq2ps 512(%rdx){1to4}, %xmm28
+// CHECK:  encoding: [0x62,0x61,0x7c,0x18,0x5b,0xa2,0x00,0x02,0x00,0x00]
+          vcvtdq2ps 512(%rdx){1to4}, %xmm28
+
+// CHECK: vcvtdq2ps -512(%rdx){1to4}, %xmm28
+// CHECK:  encoding: [0x62,0x61,0x7c,0x18,0x5b,0x62,0x80]
+          vcvtdq2ps -512(%rdx){1to4}, %xmm28
+
+// CHECK: vcvtdq2ps -516(%rdx){1to4}, %xmm28
+// CHECK:  encoding: [0x62,0x61,0x7c,0x18,0x5b,0xa2,0xfc,0xfd,0xff,0xff]
+          vcvtdq2ps -516(%rdx){1to4}, %xmm28
+
+// CHECK: vcvtdq2ps %ymm23, %ymm24
+// CHECK:  encoding: [0x62,0x21,0x7c,0x28,0x5b,0xc7]
+          vcvtdq2ps %ymm23, %ymm24
+
+// CHECK: vcvtdq2ps %ymm23, %ymm24 {%k3}
+// CHECK:  encoding: [0x62,0x21,0x7c,0x2b,0x5b,0xc7]
+          vcvtdq2ps %ymm23, %ymm24 {%k3}
+
+// CHECK: vcvtdq2ps %ymm23, %ymm24 {%k3} {z}
+// CHECK:  encoding: [0x62,0x21,0x7c,0xab,0x5b,0xc7]
+          vcvtdq2ps %ymm23, %ymm24 {%k3} {z}
+
+// CHECK: vcvtdq2ps (%rcx), %ymm24
+// CHECK:  encoding: [0x62,0x61,0x7c,0x28,0x5b,0x01]
+          vcvtdq2ps (%rcx), %ymm24
+
+// CHECK: vcvtdq2ps 291(%rax,%r14,8), %ymm24
+// CHECK:  encoding: [0x62,0x21,0x7c,0x28,0x5b,0x84,0xf0,0x23,0x01,0x00,0x00]
+          vcvtdq2ps 291(%rax,%r14,8), %ymm24
+
+// CHECK: vcvtdq2ps (%rcx){1to8}, %ymm24
+// CHECK:  encoding: [0x62,0x61,0x7c,0x38,0x5b,0x01]
+          vcvtdq2ps (%rcx){1to8}, %ymm24
+
+// CHECK: vcvtdq2ps 4064(%rdx), %ymm24
+// CHECK:  encoding: [0x62,0x61,0x7c,0x28,0x5b,0x42,0x7f]
+          vcvtdq2ps 4064(%rdx), %ymm24
+
+// CHECK: vcvtdq2ps 4096(%rdx), %ymm24
+// CHECK:  encoding: [0x62,0x61,0x7c,0x28,0x5b,0x82,0x00,0x10,0x00,0x00]
+          vcvtdq2ps 4096(%rdx), %ymm24
+
+// CHECK: vcvtdq2ps -4096(%rdx), %ymm24
+// CHECK:  encoding: [0x62,0x61,0x7c,0x28,0x5b,0x42,0x80]
+          vcvtdq2ps -4096(%rdx), %ymm24
+
+// CHECK: vcvtdq2ps -4128(%rdx), %ymm24
+// CHECK:  encoding: [0x62,0x61,0x7c,0x28,0x5b,0x82,0xe0,0xef,0xff,0xff]
+          vcvtdq2ps -4128(%rdx), %ymm24
+
+// CHECK: vcvtdq2ps 508(%rdx){1to8}, %ymm24
+// CHECK:  encoding: [0x62,0x61,0x7c,0x38,0x5b,0x42,0x7f]
+          vcvtdq2ps 508(%rdx){1to8}, %ymm24
+
+// CHECK: vcvtdq2ps 512(%rdx){1to8}, %ymm24
+// CHECK:  encoding: [0x62,0x61,0x7c,0x38,0x5b,0x82,0x00,0x02,0x00,0x00]
+          vcvtdq2ps 512(%rdx){1to8}, %ymm24
+
+// CHECK: vcvtdq2ps -512(%rdx){1to8}, %ymm24
+// CHECK:  encoding: [0x62,0x61,0x7c,0x38,0x5b,0x42,0x80]
+          vcvtdq2ps -512(%rdx){1to8}, %ymm24
+
+// CHECK: vcvtdq2ps -516(%rdx){1to8}, %ymm24
+// CHECK:  encoding: [0x62,0x61,0x7c,0x38,0x5b,0x82,0xfc,0xfd,0xff,0xff]
+          vcvtdq2ps -516(%rdx){1to8}, %ymm24
+// CHECK: vunpckhps %xmm27, %xmm19, %xmm30
+// CHECK:  encoding: [0x62,0x01,0x64,0x00,0x15,0xf3]
+          vunpckhps %xmm27, %xmm19, %xmm30
+
+// CHECK: vunpckhps %xmm27, %xmm19, %xmm30 {%k2}
+// CHECK:  encoding: [0x62,0x01,0x64,0x02,0x15,0xf3]
+          vunpckhps %xmm27, %xmm19, %xmm30 {%k2}
+
+// CHECK: vunpckhps %xmm27, %xmm19, %xmm30 {%k2} {z}
+// CHECK:  encoding: [0x62,0x01,0x64,0x82,0x15,0xf3]
+          vunpckhps %xmm27, %xmm19, %xmm30 {%k2} {z}
+
+// CHECK: vunpckhps (%rcx), %xmm19, %xmm30
+// CHECK:  encoding: [0x62,0x61,0x64,0x00,0x15,0x31]
+          vunpckhps (%rcx), %xmm19, %xmm30
+
+// CHECK: vunpckhps 291(%rax,%r14,8), %xmm19, %xmm30
+// CHECK:  encoding: [0x62,0x21,0x64,0x00,0x15,0xb4,0xf0,0x23,0x01,0x00,0x00]
+          vunpckhps 291(%rax,%r14,8), %xmm19, %xmm30
+
+// CHECK: vunpckhps (%rcx){1to4}, %xmm19, %xmm30
+// CHECK:  encoding: [0x62,0x61,0x64,0x10,0x15,0x31]
+          vunpckhps (%rcx){1to4}, %xmm19, %xmm30
+
+// CHECK: vunpckhps 2032(%rdx), %xmm19, %xmm30
+// CHECK:  encoding: [0x62,0x61,0x64,0x00,0x15,0x72,0x7f]
+          vunpckhps 2032(%rdx), %xmm19, %xmm30
+
+// CHECK: vunpckhps 2048(%rdx), %xmm19, %xmm30
+// CHECK:  encoding: [0x62,0x61,0x64,0x00,0x15,0xb2,0x00,0x08,0x00,0x00]
+          vunpckhps 2048(%rdx), %xmm19, %xmm30
+
+// CHECK: vunpckhps -2048(%rdx), %xmm19, %xmm30
+// CHECK:  encoding: [0x62,0x61,0x64,0x00,0x15,0x72,0x80]
+          vunpckhps -2048(%rdx), %xmm19, %xmm30
+
+// CHECK: vunpckhps -2064(%rdx), %xmm19, %xmm30
+// CHECK:  encoding: [0x62,0x61,0x64,0x00,0x15,0xb2,0xf0,0xf7,0xff,0xff]
+          vunpckhps -2064(%rdx), %xmm19, %xmm30
+
+// CHECK: vunpckhps 508(%rdx){1to4}, %xmm19, %xmm30
+// CHECK:  encoding: [0x62,0x61,0x64,0x10,0x15,0x72,0x7f]
+          vunpckhps 508(%rdx){1to4}, %xmm19, %xmm30
+
+// CHECK: vunpckhps 512(%rdx){1to4}, %xmm19, %xmm30
+// CHECK:  encoding: [0x62,0x61,0x64,0x10,0x15,0xb2,0x00,0x02,0x00,0x00]
+          vunpckhps 512(%rdx){1to4}, %xmm19, %xmm30
+
+// CHECK: vunpckhps -512(%rdx){1to4}, %xmm19, %xmm30
+// CHECK:  encoding: [0x62,0x61,0x64,0x10,0x15,0x72,0x80]
+          vunpckhps -512(%rdx){1to4}, %xmm19, %xmm30
+
+// CHECK: vunpckhps -516(%rdx){1to4}, %xmm19, %xmm30
+// CHECK:  encoding: [0x62,0x61,0x64,0x10,0x15,0xb2,0xfc,0xfd,0xff,0xff]
+          vunpckhps -516(%rdx){1to4}, %xmm19, %xmm30
+
+// CHECK: vunpckhps %ymm25, %ymm25, %ymm21
+// CHECK:  encoding: [0x62,0x81,0x34,0x20,0x15,0xe9]
+          vunpckhps %ymm25, %ymm25, %ymm21
+
+// CHECK: vunpckhps %ymm25, %ymm25, %ymm21 {%k5}
+// CHECK:  encoding: [0x62,0x81,0x34,0x25,0x15,0xe9]
+          vunpckhps %ymm25, %ymm25, %ymm21 {%k5}
+
+// CHECK: vunpckhps %ymm25, %ymm25, %ymm21 {%k5} {z}
+// CHECK:  encoding: [0x62,0x81,0x34,0xa5,0x15,0xe9]
+          vunpckhps %ymm25, %ymm25, %ymm21 {%k5} {z}
+
+// CHECK: vunpckhps (%rcx), %ymm25, %ymm21
+// CHECK:  encoding: [0x62,0xe1,0x34,0x20,0x15,0x29]
+          vunpckhps (%rcx), %ymm25, %ymm21
+
+// CHECK: vunpckhps 291(%rax,%r14,8), %ymm25, %ymm21
+// CHECK:  encoding: [0x62,0xa1,0x34,0x20,0x15,0xac,0xf0,0x23,0x01,0x00,0x00]
+          vunpckhps 291(%rax,%r14,8), %ymm25, %ymm21
+
+// CHECK: vunpckhps (%rcx){1to8}, %ymm25, %ymm21
+// CHECK:  encoding: [0x62,0xe1,0x34,0x30,0x15,0x29]
+          vunpckhps (%rcx){1to8}, %ymm25, %ymm21
+
+// CHECK: vunpckhps 4064(%rdx), %ymm25, %ymm21
+// CHECK:  encoding: [0x62,0xe1,0x34,0x20,0x15,0x6a,0x7f]
+          vunpckhps 4064(%rdx), %ymm25, %ymm21
+
+// CHECK: vunpckhps 4096(%rdx), %ymm25, %ymm21
+// CHECK:  encoding: [0x62,0xe1,0x34,0x20,0x15,0xaa,0x00,0x10,0x00,0x00]
+          vunpckhps 4096(%rdx), %ymm25, %ymm21
+
+// CHECK: vunpckhps -4096(%rdx), %ymm25, %ymm21
+// CHECK:  encoding: [0x62,0xe1,0x34,0x20,0x15,0x6a,0x80]
+          vunpckhps -4096(%rdx), %ymm25, %ymm21
+
+// CHECK: vunpckhps -4128(%rdx), %ymm25, %ymm21
+// CHECK:  encoding: [0x62,0xe1,0x34,0x20,0x15,0xaa,0xe0,0xef,0xff,0xff]
+          vunpckhps -4128(%rdx), %ymm25, %ymm21
+
+// CHECK: vunpckhps 508(%rdx){1to8}, %ymm25, %ymm21
+// CHECK:  encoding: [0x62,0xe1,0x34,0x30,0x15,0x6a,0x7f]
+          vunpckhps 508(%rdx){1to8}, %ymm25, %ymm21
+
+// CHECK: vunpckhps 512(%rdx){1to8}, %ymm25, %ymm21
+// CHECK:  encoding: [0x62,0xe1,0x34,0x30,0x15,0xaa,0x00,0x02,0x00,0x00]
+          vunpckhps 512(%rdx){1to8}, %ymm25, %ymm21
+
+// CHECK: vunpckhps -512(%rdx){1to8}, %ymm25, %ymm21
+// CHECK:  encoding: [0x62,0xe1,0x34,0x30,0x15,0x6a,0x80]
+          vunpckhps -512(%rdx){1to8}, %ymm25, %ymm21
+
+// CHECK: vunpckhps -516(%rdx){1to8}, %ymm25, %ymm21
+// CHECK:  encoding: [0x62,0xe1,0x34,0x30,0x15,0xaa,0xfc,0xfd,0xff,0xff]
+          vunpckhps -516(%rdx){1to8}, %ymm25, %ymm21
+
+// CHECK: vunpcklps %xmm26, %xmm24, %xmm29
+// CHECK:  encoding: [0x62,0x01,0x3c,0x00,0x14,0xea]
+          vunpcklps %xmm26, %xmm24, %xmm29
+
+// CHECK: vunpcklps %xmm26, %xmm24, %xmm29 {%k6}
+// CHECK:  encoding: [0x62,0x01,0x3c,0x06,0x14,0xea]
+          vunpcklps %xmm26, %xmm24, %xmm29 {%k6}
+
+// CHECK: vunpcklps %xmm26, %xmm24, %xmm29 {%k6} {z}
+// CHECK:  encoding: [0x62,0x01,0x3c,0x86,0x14,0xea]
+          vunpcklps %xmm26, %xmm24, %xmm29 {%k6} {z}
+
+// CHECK: vunpcklps (%rcx), %xmm24, %xmm29
+// CHECK:  encoding: [0x62,0x61,0x3c,0x00,0x14,0x29]
+          vunpcklps (%rcx), %xmm24, %xmm29
+
+// CHECK: vunpcklps 291(%rax,%r14,8), %xmm24, %xmm29
+// CHECK:  encoding: [0x62,0x21,0x3c,0x00,0x14,0xac,0xf0,0x23,0x01,0x00,0x00]
+          vunpcklps 291(%rax,%r14,8), %xmm24, %xmm29
+
+// CHECK: vunpcklps (%rcx){1to4}, %xmm24, %xmm29
+// CHECK:  encoding: [0x62,0x61,0x3c,0x10,0x14,0x29]
+          vunpcklps (%rcx){1to4}, %xmm24, %xmm29
+
+// CHECK: vunpcklps 2032(%rdx), %xmm24, %xmm29
+// CHECK:  encoding: [0x62,0x61,0x3c,0x00,0x14,0x6a,0x7f]
+          vunpcklps 2032(%rdx), %xmm24, %xmm29
+
+// CHECK: vunpcklps 2048(%rdx), %xmm24, %xmm29
+// CHECK:  encoding: [0x62,0x61,0x3c,0x00,0x14,0xaa,0x00,0x08,0x00,0x00]
+          vunpcklps 2048(%rdx), %xmm24, %xmm29
+
+// CHECK: vunpcklps -2048(%rdx), %xmm24, %xmm29
+// CHECK:  encoding: [0x62,0x61,0x3c,0x00,0x14,0x6a,0x80]
+          vunpcklps -2048(%rdx), %xmm24, %xmm29
+
+// CHECK: vunpcklps -2064(%rdx), %xmm24, %xmm29
+// CHECK:  encoding: [0x62,0x61,0x3c,0x00,0x14,0xaa,0xf0,0xf7,0xff,0xff]
+          vunpcklps -2064(%rdx), %xmm24, %xmm29
+
+// CHECK: vunpcklps 508(%rdx){1to4}, %xmm24, %xmm29
+// CHECK:  encoding: [0x62,0x61,0x3c,0x10,0x14,0x6a,0x7f]
+          vunpcklps 508(%rdx){1to4}, %xmm24, %xmm29
+
+// CHECK: vunpcklps 512(%rdx){1to4}, %xmm24, %xmm29
+// CHECK:  encoding: [0x62,0x61,0x3c,0x10,0x14,0xaa,0x00,0x02,0x00,0x00]
+          vunpcklps 512(%rdx){1to4}, %xmm24, %xmm29
+
+// CHECK: vunpcklps -512(%rdx){1to4}, %xmm24, %xmm29
+// CHECK:  encoding: [0x62,0x61,0x3c,0x10,0x14,0x6a,0x80]
+          vunpcklps -512(%rdx){1to4}, %xmm24, %xmm29
+
+// CHECK: vunpcklps -516(%rdx){1to4}, %xmm24, %xmm29
+// CHECK:  encoding: [0x62,0x61,0x3c,0x10,0x14,0xaa,0xfc,0xfd,0xff,0xff]
+          vunpcklps -516(%rdx){1to4}, %xmm24, %xmm29
+
+// CHECK: vunpcklps %ymm18, %ymm18, %ymm26
+// CHECK:  encoding: [0x62,0x21,0x6c,0x20,0x14,0xd2]
+          vunpcklps %ymm18, %ymm18, %ymm26
+
+// CHECK: vunpcklps %ymm18, %ymm18, %ymm26 {%k1}
+// CHECK:  encoding: [0x62,0x21,0x6c,0x21,0x14,0xd2]
+          vunpcklps %ymm18, %ymm18, %ymm26 {%k1}
+
+// CHECK: vunpcklps %ymm18, %ymm18, %ymm26 {%k1} {z}
+// CHECK:  encoding: [0x62,0x21,0x6c,0xa1,0x14,0xd2]
+          vunpcklps %ymm18, %ymm18, %ymm26 {%k1} {z}
+
+// CHECK: vunpcklps (%rcx), %ymm18, %ymm26
+// CHECK:  encoding: [0x62,0x61,0x6c,0x20,0x14,0x11]
+          vunpcklps (%rcx), %ymm18, %ymm26
+
+// CHECK: vunpcklps 291(%rax,%r14,8), %ymm18, %ymm26
+// CHECK:  encoding: [0x62,0x21,0x6c,0x20,0x14,0x94,0xf0,0x23,0x01,0x00,0x00]
+          vunpcklps 291(%rax,%r14,8), %ymm18, %ymm26
+
+// CHECK: vunpcklps (%rcx){1to8}, %ymm18, %ymm26
+// CHECK:  encoding: [0x62,0x61,0x6c,0x30,0x14,0x11]
+          vunpcklps (%rcx){1to8}, %ymm18, %ymm26
+
+// CHECK: vunpcklps 4064(%rdx), %ymm18, %ymm26
+// CHECK:  encoding: [0x62,0x61,0x6c,0x20,0x14,0x52,0x7f]
+          vunpcklps 4064(%rdx), %ymm18, %ymm26
+
+// CHECK: vunpcklps 4096(%rdx), %ymm18, %ymm26
+// CHECK:  encoding: [0x62,0x61,0x6c,0x20,0x14,0x92,0x00,0x10,0x00,0x00]
+          vunpcklps 4096(%rdx), %ymm18, %ymm26
+
+// CHECK: vunpcklps -4096(%rdx), %ymm18, %ymm26
+// CHECK:  encoding: [0x62,0x61,0x6c,0x20,0x14,0x52,0x80]
+          vunpcklps -4096(%rdx), %ymm18, %ymm26
+
+// CHECK: vunpcklps -4128(%rdx), %ymm18, %ymm26
+// CHECK:  encoding: [0x62,0x61,0x6c,0x20,0x14,0x92,0xe0,0xef,0xff,0xff]
+          vunpcklps -4128(%rdx), %ymm18, %ymm26
+
+// CHECK: vunpcklps 508(%rdx){1to8}, %ymm18, %ymm26
+// CHECK:  encoding: [0x62,0x61,0x6c,0x30,0x14,0x52,0x7f]
+          vunpcklps 508(%rdx){1to8}, %ymm18, %ymm26
+
+// CHECK: vunpcklps 512(%rdx){1to8}, %ymm18, %ymm26
+// CHECK:  encoding: [0x62,0x61,0x6c,0x30,0x14,0x92,0x00,0x02,0x00,0x00]
+          vunpcklps 512(%rdx){1to8}, %ymm18, %ymm26
+
+// CHECK: vunpcklps -512(%rdx){1to8}, %ymm18, %ymm26
+// CHECK:  encoding: [0x62,0x61,0x6c,0x30,0x14,0x52,0x80]
+          vunpcklps -512(%rdx){1to8}, %ymm18, %ymm26
+
+// CHECK: vunpcklps -516(%rdx){1to8}, %ymm18, %ymm26
+// CHECK:  encoding: [0x62,0x61,0x6c,0x30,0x14,0x92,0xfc,0xfd,0xff,0xff]
+          vunpcklps -516(%rdx){1to8}, %ymm18, %ymm26
+
+// CHECK: vunpckhpd %xmm26, %xmm25, %xmm19
+// CHECK:  encoding: [0x62,0x81,0xb5,0x00,0x15,0xda]
+          vunpckhpd %xmm26, %xmm25, %xmm19
+
+// CHECK: vunpckhpd %xmm26, %xmm25, %xmm19 {%k3}
+// CHECK:  encoding: [0x62,0x81,0xb5,0x03,0x15,0xda]
+          vunpckhpd %xmm26, %xmm25, %xmm19 {%k3}
+
+// CHECK: vunpckhpd %xmm26, %xmm25, %xmm19 {%k3} {z}
+// CHECK:  encoding: [0x62,0x81,0xb5,0x83,0x15,0xda]
+          vunpckhpd %xmm26, %xmm25, %xmm19 {%k3} {z}
+
+// CHECK: vunpckhpd (%rcx), %xmm25, %xmm19
+// CHECK:  encoding: [0x62,0xe1,0xb5,0x00,0x15,0x19]
+          vunpckhpd (%rcx), %xmm25, %xmm19
+
+// CHECK: vunpckhpd 291(%rax,%r14,8), %xmm25, %xmm19
+// CHECK:  encoding: [0x62,0xa1,0xb5,0x00,0x15,0x9c,0xf0,0x23,0x01,0x00,0x00]
+          vunpckhpd 291(%rax,%r14,8), %xmm25, %xmm19
+
+// CHECK: vunpckhpd (%rcx){1to2}, %xmm25, %xmm19
+// CHECK:  encoding: [0x62,0xe1,0xb5,0x10,0x15,0x19]
+          vunpckhpd (%rcx){1to2}, %xmm25, %xmm19
+
+// CHECK: vunpckhpd 2032(%rdx), %xmm25, %xmm19
+// CHECK:  encoding: [0x62,0xe1,0xb5,0x00,0x15,0x5a,0x7f]
+          vunpckhpd 2032(%rdx), %xmm25, %xmm19
+
+// CHECK: vunpckhpd 2048(%rdx), %xmm25, %xmm19
+// CHECK:  encoding: [0x62,0xe1,0xb5,0x00,0x15,0x9a,0x00,0x08,0x00,0x00]
+          vunpckhpd 2048(%rdx), %xmm25, %xmm19
+
+// CHECK: vunpckhpd -2048(%rdx), %xmm25, %xmm19
+// CHECK:  encoding: [0x62,0xe1,0xb5,0x00,0x15,0x5a,0x80]
+          vunpckhpd -2048(%rdx), %xmm25, %xmm19
+
+// CHECK: vunpckhpd -2064(%rdx), %xmm25, %xmm19
+// CHECK:  encoding: [0x62,0xe1,0xb5,0x00,0x15,0x9a,0xf0,0xf7,0xff,0xff]
+          vunpckhpd -2064(%rdx), %xmm25, %xmm19
+
+// CHECK: vunpckhpd 1016(%rdx){1to2}, %xmm25, %xmm19
+// CHECK:  encoding: [0x62,0xe1,0xb5,0x10,0x15,0x5a,0x7f]
+          vunpckhpd 1016(%rdx){1to2}, %xmm25, %xmm19
+
+// CHECK: vunpckhpd 1024(%rdx){1to2}, %xmm25, %xmm19
+// CHECK:  encoding: [0x62,0xe1,0xb5,0x10,0x15,0x9a,0x00,0x04,0x00,0x00]
+          vunpckhpd 1024(%rdx){1to2}, %xmm25, %xmm19
+
+// CHECK: vunpckhpd -1024(%rdx){1to2}, %xmm25, %xmm19
+// CHECK:  encoding: [0x62,0xe1,0xb5,0x10,0x15,0x5a,0x80]
+          vunpckhpd -1024(%rdx){1to2}, %xmm25, %xmm19
+
+// CHECK: vunpckhpd -1032(%rdx){1to2}, %xmm25, %xmm19
+// CHECK:  encoding: [0x62,0xe1,0xb5,0x10,0x15,0x9a,0xf8,0xfb,0xff,0xff]
+          vunpckhpd -1032(%rdx){1to2}, %xmm25, %xmm19
+
+// CHECK: vunpckhpd %ymm24, %ymm22, %ymm25
+// CHECK:  encoding: [0x62,0x01,0xcd,0x20,0x15,0xc8]
+          vunpckhpd %ymm24, %ymm22, %ymm25
+
+// CHECK: vunpckhpd %ymm24, %ymm22, %ymm25 {%k7}
+// CHECK:  encoding: [0x62,0x01,0xcd,0x27,0x15,0xc8]
+          vunpckhpd %ymm24, %ymm22, %ymm25 {%k7}
+
+// CHECK: vunpckhpd %ymm24, %ymm22, %ymm25 {%k7} {z}
+// CHECK:  encoding: [0x62,0x01,0xcd,0xa7,0x15,0xc8]
+          vunpckhpd %ymm24, %ymm22, %ymm25 {%k7} {z}
+
+// CHECK: vunpckhpd (%rcx), %ymm22, %ymm25
+// CHECK:  encoding: [0x62,0x61,0xcd,0x20,0x15,0x09]
+          vunpckhpd (%rcx), %ymm22, %ymm25
+
+// CHECK: vunpckhpd 291(%rax,%r14,8), %ymm22, %ymm25
+// CHECK:  encoding: [0x62,0x21,0xcd,0x20,0x15,0x8c,0xf0,0x23,0x01,0x00,0x00]
+          vunpckhpd 291(%rax,%r14,8), %ymm22, %ymm25
+
+// CHECK: vunpckhpd (%rcx){1to4}, %ymm22, %ymm25
+// CHECK:  encoding: [0x62,0x61,0xcd,0x30,0x15,0x09]
+          vunpckhpd (%rcx){1to4}, %ymm22, %ymm25
+
+// CHECK: vunpckhpd 4064(%rdx), %ymm22, %ymm25
+// CHECK:  encoding: [0x62,0x61,0xcd,0x20,0x15,0x4a,0x7f]
+          vunpckhpd 4064(%rdx), %ymm22, %ymm25
+
+// CHECK: vunpckhpd 4096(%rdx), %ymm22, %ymm25
+// CHECK:  encoding: [0x62,0x61,0xcd,0x20,0x15,0x8a,0x00,0x10,0x00,0x00]
+          vunpckhpd 4096(%rdx), %ymm22, %ymm25
+
+// CHECK: vunpckhpd -4096(%rdx), %ymm22, %ymm25
+// CHECK:  encoding: [0x62,0x61,0xcd,0x20,0x15,0x4a,0x80]
+          vunpckhpd -4096(%rdx), %ymm22, %ymm25
+
+// CHECK: vunpckhpd -4128(%rdx), %ymm22, %ymm25
+// CHECK:  encoding: [0x62,0x61,0xcd,0x20,0x15,0x8a,0xe0,0xef,0xff,0xff]
+          vunpckhpd -4128(%rdx), %ymm22, %ymm25
+
+// CHECK: vunpckhpd 1016(%rdx){1to4}, %ymm22, %ymm25
+// CHECK:  encoding: [0x62,0x61,0xcd,0x30,0x15,0x4a,0x7f]
+          vunpckhpd 1016(%rdx){1to4}, %ymm22, %ymm25
+
+// CHECK: vunpckhpd 1024(%rdx){1to4}, %ymm22, %ymm25
+// CHECK:  encoding: [0x62,0x61,0xcd,0x30,0x15,0x8a,0x00,0x04,0x00,0x00]
+          vunpckhpd 1024(%rdx){1to4}, %ymm22, %ymm25
+
+// CHECK: vunpckhpd -1024(%rdx){1to4}, %ymm22, %ymm25
+// CHECK:  encoding: [0x62,0x61,0xcd,0x30,0x15,0x4a,0x80]
+          vunpckhpd -1024(%rdx){1to4}, %ymm22, %ymm25
+
+// CHECK: vunpckhpd -1032(%rdx){1to4}, %ymm22, %ymm25
+// CHECK:  encoding: [0x62,0x61,0xcd,0x30,0x15,0x8a,0xf8,0xfb,0xff,0xff]
+          vunpckhpd -1032(%rdx){1to4}, %ymm22, %ymm25
+
+// CHECK: vunpcklpd %xmm18, %xmm26, %xmm30
+// CHECK:  encoding: [0x62,0x21,0xad,0x00,0x14,0xf2]
+          vunpcklpd %xmm18, %xmm26, %xmm30
+
+// CHECK: vunpcklpd %xmm18, %xmm26, %xmm30 {%k5}
+// CHECK:  encoding: [0x62,0x21,0xad,0x05,0x14,0xf2]
+          vunpcklpd %xmm18, %xmm26, %xmm30 {%k5}
+
+// CHECK: vunpcklpd %xmm18, %xmm26, %xmm30 {%k5} {z}
+// CHECK:  encoding: [0x62,0x21,0xad,0x85,0x14,0xf2]
+          vunpcklpd %xmm18, %xmm26, %xmm30 {%k5} {z}
+
+// CHECK: vunpcklpd (%rcx), %xmm26, %xmm30
+// CHECK:  encoding: [0x62,0x61,0xad,0x00,0x14,0x31]
+          vunpcklpd (%rcx), %xmm26, %xmm30
+
+// CHECK: vunpcklpd 291(%rax,%r14,8), %xmm26, %xmm30
+// CHECK:  encoding: [0x62,0x21,0xad,0x00,0x14,0xb4,0xf0,0x23,0x01,0x00,0x00]
+          vunpcklpd 291(%rax,%r14,8), %xmm26, %xmm30
+
+// CHECK: vunpcklpd (%rcx){1to2}, %xmm26, %xmm30
+// CHECK:  encoding: [0x62,0x61,0xad,0x10,0x14,0x31]
+          vunpcklpd (%rcx){1to2}, %xmm26, %xmm30
+
+// CHECK: vunpcklpd 2032(%rdx), %xmm26, %xmm30
+// CHECK:  encoding: [0x62,0x61,0xad,0x00,0x14,0x72,0x7f]
+          vunpcklpd 2032(%rdx), %xmm26, %xmm30
+
+// CHECK: vunpcklpd 2048(%rdx), %xmm26, %xmm30
+// CHECK:  encoding: [0x62,0x61,0xad,0x00,0x14,0xb2,0x00,0x08,0x00,0x00]
+          vunpcklpd 2048(%rdx), %xmm26, %xmm30
+
+// CHECK: vunpcklpd -2048(%rdx), %xmm26, %xmm30
+// CHECK:  encoding: [0x62,0x61,0xad,0x00,0x14,0x72,0x80]
+          vunpcklpd -2048(%rdx), %xmm26, %xmm30
+
+// CHECK: vunpcklpd -2064(%rdx), %xmm26, %xmm30
+// CHECK:  encoding: [0x62,0x61,0xad,0x00,0x14,0xb2,0xf0,0xf7,0xff,0xff]
+          vunpcklpd -2064(%rdx), %xmm26, %xmm30
+
+// CHECK: vunpcklpd 1016(%rdx){1to2}, %xmm26, %xmm30
+// CHECK:  encoding: [0x62,0x61,0xad,0x10,0x14,0x72,0x7f]
+          vunpcklpd 1016(%rdx){1to2}, %xmm26, %xmm30
+
+// CHECK: vunpcklpd 1024(%rdx){1to2}, %xmm26, %xmm30
+// CHECK:  encoding: [0x62,0x61,0xad,0x10,0x14,0xb2,0x00,0x04,0x00,0x00]
+          vunpcklpd 1024(%rdx){1to2}, %xmm26, %xmm30
+
+// CHECK: vunpcklpd -1024(%rdx){1to2}, %xmm26, %xmm30
+// CHECK:  encoding: [0x62,0x61,0xad,0x10,0x14,0x72,0x80]
+          vunpcklpd -1024(%rdx){1to2}, %xmm26, %xmm30
+
+// CHECK: vunpcklpd -1032(%rdx){1to2}, %xmm26, %xmm30
+// CHECK:  encoding: [0x62,0x61,0xad,0x10,0x14,0xb2,0xf8,0xfb,0xff,0xff]
+          vunpcklpd -1032(%rdx){1to2}, %xmm26, %xmm30
+
+// CHECK: vunpcklpd %ymm17, %ymm26, %ymm20
+// CHECK:  encoding: [0x62,0xa1,0xad,0x20,0x14,0xe1]
+          vunpcklpd %ymm17, %ymm26, %ymm20
+
+// CHECK: vunpcklpd %ymm17, %ymm26, %ymm20 {%k2}
+// CHECK:  encoding: [0x62,0xa1,0xad,0x22,0x14,0xe1]
+          vunpcklpd %ymm17, %ymm26, %ymm20 {%k2}
+
+// CHECK: vunpcklpd %ymm17, %ymm26, %ymm20 {%k2} {z}
+// CHECK:  encoding: [0x62,0xa1,0xad,0xa2,0x14,0xe1]
+          vunpcklpd %ymm17, %ymm26, %ymm20 {%k2} {z}
+
+// CHECK: vunpcklpd (%rcx), %ymm26, %ymm20
+// CHECK:  encoding: [0x62,0xe1,0xad,0x20,0x14,0x21]
+          vunpcklpd (%rcx), %ymm26, %ymm20
+
+// CHECK: vunpcklpd 291(%rax,%r14,8), %ymm26, %ymm20
+// CHECK:  encoding: [0x62,0xa1,0xad,0x20,0x14,0xa4,0xf0,0x23,0x01,0x00,0x00]
+          vunpcklpd 291(%rax,%r14,8), %ymm26, %ymm20
+
+// CHECK: vunpcklpd (%rcx){1to4}, %ymm26, %ymm20
+// CHECK:  encoding: [0x62,0xe1,0xad,0x30,0x14,0x21]
+          vunpcklpd (%rcx){1to4}, %ymm26, %ymm20
+
+// CHECK: vunpcklpd 4064(%rdx), %ymm26, %ymm20
+// CHECK:  encoding: [0x62,0xe1,0xad,0x20,0x14,0x62,0x7f]
+          vunpcklpd 4064(%rdx), %ymm26, %ymm20
+
+// CHECK: vunpcklpd 4096(%rdx), %ymm26, %ymm20
+// CHECK:  encoding: [0x62,0xe1,0xad,0x20,0x14,0xa2,0x00,0x10,0x00,0x00]
+          vunpcklpd 4096(%rdx), %ymm26, %ymm20
+
+// CHECK: vunpcklpd -4096(%rdx), %ymm26, %ymm20
+// CHECK:  encoding: [0x62,0xe1,0xad,0x20,0x14,0x62,0x80]
+          vunpcklpd -4096(%rdx), %ymm26, %ymm20
+
+// CHECK: vunpcklpd -4128(%rdx), %ymm26, %ymm20
+// CHECK:  encoding: [0x62,0xe1,0xad,0x20,0x14,0xa2,0xe0,0xef,0xff,0xff]
+          vunpcklpd -4128(%rdx), %ymm26, %ymm20
+
+// CHECK: vunpcklpd 1016(%rdx){1to4}, %ymm26, %ymm20
+// CHECK:  encoding: [0x62,0xe1,0xad,0x30,0x14,0x62,0x7f]
+          vunpcklpd 1016(%rdx){1to4}, %ymm26, %ymm20
+
+// CHECK: vunpcklpd 1024(%rdx){1to4}, %ymm26, %ymm20
+// CHECK:  encoding: [0x62,0xe1,0xad,0x30,0x14,0xa2,0x00,0x04,0x00,0x00]
+          vunpcklpd 1024(%rdx){1to4}, %ymm26, %ymm20
+
+// CHECK: vunpcklpd -1024(%rdx){1to4}, %ymm26, %ymm20
+// CHECK:  encoding: [0x62,0xe1,0xad,0x30,0x14,0x62,0x80]
+          vunpcklpd -1024(%rdx){1to4}, %ymm26, %ymm20
+
+// CHECK: vunpcklpd -1032(%rdx){1to4}, %ymm26, %ymm20
+// CHECK:  encoding: [0x62,0xe1,0xad,0x30,0x14,0xa2,0xf8,0xfb,0xff,0xff]
+          vunpcklpd -1032(%rdx){1to4}, %ymm26, %ymm20
+
+// CHECK: vpunpckldq %xmm17, %xmm21, %xmm25
+// CHECK:  encoding: [0x62,0x21,0x55,0x00,0x62,0xc9]
+          vpunpckldq %xmm17, %xmm21, %xmm25
+
+// CHECK: vpunpckldq %xmm17, %xmm21, %xmm25 {%k7}
+// CHECK:  encoding: [0x62,0x21,0x55,0x07,0x62,0xc9]
+          vpunpckldq %xmm17, %xmm21, %xmm25 {%k7}
+
+// CHECK: vpunpckldq %xmm17, %xmm21, %xmm25 {%k7} {z}
+// CHECK:  encoding: [0x62,0x21,0x55,0x87,0x62,0xc9]
+          vpunpckldq %xmm17, %xmm21, %xmm25 {%k7} {z}
+
+// CHECK: vpunpckldq (%rcx), %xmm21, %xmm25
+// CHECK:  encoding: [0x62,0x61,0x55,0x00,0x62,0x09]
+          vpunpckldq (%rcx), %xmm21, %xmm25
+
+// CHECK: vpunpckldq 291(%rax,%r14,8), %xmm21, %xmm25
+// CHECK:  encoding: [0x62,0x21,0x55,0x00,0x62,0x8c,0xf0,0x23,0x01,0x00,0x00]
+          vpunpckldq 291(%rax,%r14,8), %xmm21, %xmm25
+
+// CHECK: vpunpckldq (%rcx){1to4}, %xmm21, %xmm25
+// CHECK:  encoding: [0x62,0x61,0x55,0x10,0x62,0x09]
+          vpunpckldq (%rcx){1to4}, %xmm21, %xmm25
+
+// CHECK: vpunpckldq 2032(%rdx), %xmm21, %xmm25
+// CHECK:  encoding: [0x62,0x61,0x55,0x00,0x62,0x4a,0x7f]
+          vpunpckldq 2032(%rdx), %xmm21, %xmm25
+
+// CHECK: vpunpckldq 2048(%rdx), %xmm21, %xmm25
+// CHECK:  encoding: [0x62,0x61,0x55,0x00,0x62,0x8a,0x00,0x08,0x00,0x00]
+          vpunpckldq 2048(%rdx), %xmm21, %xmm25
+
+// CHECK: vpunpckldq -2048(%rdx), %xmm21, %xmm25
+// CHECK:  encoding: [0x62,0x61,0x55,0x00,0x62,0x4a,0x80]
+          vpunpckldq -2048(%rdx), %xmm21, %xmm25
+
+// CHECK: vpunpckldq -2064(%rdx), %xmm21, %xmm25
+// CHECK:  encoding: [0x62,0x61,0x55,0x00,0x62,0x8a,0xf0,0xf7,0xff,0xff]
+          vpunpckldq -2064(%rdx), %xmm21, %xmm25
+
+// CHECK: vpunpckldq 508(%rdx){1to4}, %xmm21, %xmm25
+// CHECK:  encoding: [0x62,0x61,0x55,0x10,0x62,0x4a,0x7f]
+          vpunpckldq 508(%rdx){1to4}, %xmm21, %xmm25
+
+// CHECK: vpunpckldq 512(%rdx){1to4}, %xmm21, %xmm25
+// CHECK:  encoding: [0x62,0x61,0x55,0x10,0x62,0x8a,0x00,0x02,0x00,0x00]
+          vpunpckldq 512(%rdx){1to4}, %xmm21, %xmm25
+
+// CHECK: vpunpckldq -512(%rdx){1to4}, %xmm21, %xmm25
+// CHECK:  encoding: [0x62,0x61,0x55,0x10,0x62,0x4a,0x80]
+          vpunpckldq -512(%rdx){1to4}, %xmm21, %xmm25
+
+// CHECK: vpunpckldq -516(%rdx){1to4}, %xmm21, %xmm25
+// CHECK:  encoding: [0x62,0x61,0x55,0x10,0x62,0x8a,0xfc,0xfd,0xff,0xff]
+          vpunpckldq -516(%rdx){1to4}, %xmm21, %xmm25
+
+// CHECK: vpunpckldq %ymm26, %ymm19, %ymm20
+// CHECK:  encoding: [0x62,0x81,0x65,0x20,0x62,0xe2]
+          vpunpckldq %ymm26, %ymm19, %ymm20
+
+// CHECK: vpunpckldq %ymm26, %ymm19, %ymm20 {%k7}
+// CHECK:  encoding: [0x62,0x81,0x65,0x27,0x62,0xe2]
+          vpunpckldq %ymm26, %ymm19, %ymm20 {%k7}
+
+// CHECK: vpunpckldq %ymm26, %ymm19, %ymm20 {%k7} {z}
+// CHECK:  encoding: [0x62,0x81,0x65,0xa7,0x62,0xe2]
+          vpunpckldq %ymm26, %ymm19, %ymm20 {%k7} {z}
+
+// CHECK: vpunpckldq (%rcx), %ymm19, %ymm20
+// CHECK:  encoding: [0x62,0xe1,0x65,0x20,0x62,0x21]
+          vpunpckldq (%rcx), %ymm19, %ymm20
+
+// CHECK: vpunpckldq 291(%rax,%r14,8), %ymm19, %ymm20
+// CHECK:  encoding: [0x62,0xa1,0x65,0x20,0x62,0xa4,0xf0,0x23,0x01,0x00,0x00]
+          vpunpckldq 291(%rax,%r14,8), %ymm19, %ymm20
+
+// CHECK: vpunpckldq (%rcx){1to8}, %ymm19, %ymm20
+// CHECK:  encoding: [0x62,0xe1,0x65,0x30,0x62,0x21]
+          vpunpckldq (%rcx){1to8}, %ymm19, %ymm20
+
+// CHECK: vpunpckldq 4064(%rdx), %ymm19, %ymm20
+// CHECK:  encoding: [0x62,0xe1,0x65,0x20,0x62,0x62,0x7f]
+          vpunpckldq 4064(%rdx), %ymm19, %ymm20
+
+// CHECK: vpunpckldq 4096(%rdx), %ymm19, %ymm20
+// CHECK:  encoding: [0x62,0xe1,0x65,0x20,0x62,0xa2,0x00,0x10,0x00,0x00]
+          vpunpckldq 4096(%rdx), %ymm19, %ymm20
+
+// CHECK: vpunpckldq -4096(%rdx), %ymm19, %ymm20
+// CHECK:  encoding: [0x62,0xe1,0x65,0x20,0x62,0x62,0x80]
+          vpunpckldq -4096(%rdx), %ymm19, %ymm20
+
+// CHECK: vpunpckldq -4128(%rdx), %ymm19, %ymm20
+// CHECK:  encoding: [0x62,0xe1,0x65,0x20,0x62,0xa2,0xe0,0xef,0xff,0xff]
+          vpunpckldq -4128(%rdx), %ymm19, %ymm20
+
+// CHECK: vpunpckldq 508(%rdx){1to8}, %ymm19, %ymm20
+// CHECK:  encoding: [0x62,0xe1,0x65,0x30,0x62,0x62,0x7f]
+          vpunpckldq 508(%rdx){1to8}, %ymm19, %ymm20
+
+// CHECK: vpunpckldq 512(%rdx){1to8}, %ymm19, %ymm20
+// CHECK:  encoding: [0x62,0xe1,0x65,0x30,0x62,0xa2,0x00,0x02,0x00,0x00]
+          vpunpckldq 512(%rdx){1to8}, %ymm19, %ymm20
+
+// CHECK: vpunpckldq -512(%rdx){1to8}, %ymm19, %ymm20
+// CHECK:  encoding: [0x62,0xe1,0x65,0x30,0x62,0x62,0x80]
+          vpunpckldq -512(%rdx){1to8}, %ymm19, %ymm20
+
+// CHECK: vpunpckldq -516(%rdx){1to8}, %ymm19, %ymm20
+// CHECK:  encoding: [0x62,0xe1,0x65,0x30,0x62,0xa2,0xfc,0xfd,0xff,0xff]
+          vpunpckldq -516(%rdx){1to8}, %ymm19, %ymm20
+
+// CHECK: vpunpckhdq %xmm27, %xmm24, %xmm28
+// CHECK:  encoding: [0x62,0x01,0x3d,0x00,0x6a,0xe3]
+          vpunpckhdq %xmm27, %xmm24, %xmm28
+
+// CHECK: vpunpckhdq %xmm27, %xmm24, %xmm28 {%k7}
+// CHECK:  encoding: [0x62,0x01,0x3d,0x07,0x6a,0xe3]
+          vpunpckhdq %xmm27, %xmm24, %xmm28 {%k7}
+
+// CHECK: vpunpckhdq %xmm27, %xmm24, %xmm28 {%k7} {z}
+// CHECK:  encoding: [0x62,0x01,0x3d,0x87,0x6a,0xe3]
+          vpunpckhdq %xmm27, %xmm24, %xmm28 {%k7} {z}
+
+// CHECK: vpunpckhdq (%rcx), %xmm24, %xmm28
+// CHECK:  encoding: [0x62,0x61,0x3d,0x00,0x6a,0x21]
+          vpunpckhdq (%rcx), %xmm24, %xmm28
+
+// CHECK: vpunpckhdq 291(%rax,%r14,8), %xmm24, %xmm28
+// CHECK:  encoding: [0x62,0x21,0x3d,0x00,0x6a,0xa4,0xf0,0x23,0x01,0x00,0x00]
+          vpunpckhdq 291(%rax,%r14,8), %xmm24, %xmm28
+
+// CHECK: vpunpckhdq (%rcx){1to4}, %xmm24, %xmm28
+// CHECK:  encoding: [0x62,0x61,0x3d,0x10,0x6a,0x21]
+          vpunpckhdq (%rcx){1to4}, %xmm24, %xmm28
+
+// CHECK: vpunpckhdq 2032(%rdx), %xmm24, %xmm28
+// CHECK:  encoding: [0x62,0x61,0x3d,0x00,0x6a,0x62,0x7f]
+          vpunpckhdq 2032(%rdx), %xmm24, %xmm28
+
+// CHECK: vpunpckhdq 2048(%rdx), %xmm24, %xmm28
+// CHECK:  encoding: [0x62,0x61,0x3d,0x00,0x6a,0xa2,0x00,0x08,0x00,0x00]
+          vpunpckhdq 2048(%rdx), %xmm24, %xmm28
+
+// CHECK: vpunpckhdq -2048(%rdx), %xmm24, %xmm28
+// CHECK:  encoding: [0x62,0x61,0x3d,0x00,0x6a,0x62,0x80]
+          vpunpckhdq -2048(%rdx), %xmm24, %xmm28
+
+// CHECK: vpunpckhdq -2064(%rdx), %xmm24, %xmm28
+// CHECK:  encoding: [0x62,0x61,0x3d,0x00,0x6a,0xa2,0xf0,0xf7,0xff,0xff]
+          vpunpckhdq -2064(%rdx), %xmm24, %xmm28
+
+// CHECK: vpunpckhdq 508(%rdx){1to4}, %xmm24, %xmm28
+// CHECK:  encoding: [0x62,0x61,0x3d,0x10,0x6a,0x62,0x7f]
+          vpunpckhdq 508(%rdx){1to4}, %xmm24, %xmm28
+
+// CHECK: vpunpckhdq 512(%rdx){1to4}, %xmm24, %xmm28
+// CHECK:  encoding: [0x62,0x61,0x3d,0x10,0x6a,0xa2,0x00,0x02,0x00,0x00]
+          vpunpckhdq 512(%rdx){1to4}, %xmm24, %xmm28
+
+// CHECK: vpunpckhdq -512(%rdx){1to4}, %xmm24, %xmm28
+// CHECK:  encoding: [0x62,0x61,0x3d,0x10,0x6a,0x62,0x80]
+          vpunpckhdq -512(%rdx){1to4}, %xmm24, %xmm28
+
+// CHECK: vpunpckhdq -516(%rdx){1to4}, %xmm24, %xmm28
+// CHECK:  encoding: [0x62,0x61,0x3d,0x10,0x6a,0xa2,0xfc,0xfd,0xff,0xff]
+          vpunpckhdq -516(%rdx){1to4}, %xmm24, %xmm28
+
+// CHECK: vpunpckhdq %ymm28, %ymm24, %ymm26
+// CHECK:  encoding: [0x62,0x01,0x3d,0x20,0x6a,0xd4]
+          vpunpckhdq %ymm28, %ymm24, %ymm26
+
+// CHECK: vpunpckhdq %ymm28, %ymm24, %ymm26 {%k2}
+// CHECK:  encoding: [0x62,0x01,0x3d,0x22,0x6a,0xd4]
+          vpunpckhdq %ymm28, %ymm24, %ymm26 {%k2}
+
+// CHECK: vpunpckhdq %ymm28, %ymm24, %ymm26 {%k2} {z}
+// CHECK:  encoding: [0x62,0x01,0x3d,0xa2,0x6a,0xd4]
+          vpunpckhdq %ymm28, %ymm24, %ymm26 {%k2} {z}
+
+// CHECK: vpunpckhdq (%rcx), %ymm24, %ymm26
+// CHECK:  encoding: [0x62,0x61,0x3d,0x20,0x6a,0x11]
+          vpunpckhdq (%rcx), %ymm24, %ymm26
+
+// CHECK: vpunpckhdq 291(%rax,%r14,8), %ymm24, %ymm26
+// CHECK:  encoding: [0x62,0x21,0x3d,0x20,0x6a,0x94,0xf0,0x23,0x01,0x00,0x00]
+          vpunpckhdq 291(%rax,%r14,8), %ymm24, %ymm26
+
+// CHECK: vpunpckhdq (%rcx){1to8}, %ymm24, %ymm26
+// CHECK:  encoding: [0x62,0x61,0x3d,0x30,0x6a,0x11]
+          vpunpckhdq (%rcx){1to8}, %ymm24, %ymm26
+
+// CHECK: vpunpckhdq 4064(%rdx), %ymm24, %ymm26
+// CHECK:  encoding: [0x62,0x61,0x3d,0x20,0x6a,0x52,0x7f]
+          vpunpckhdq 4064(%rdx), %ymm24, %ymm26
+
+// CHECK: vpunpckhdq 4096(%rdx), %ymm24, %ymm26
+// CHECK:  encoding: [0x62,0x61,0x3d,0x20,0x6a,0x92,0x00,0x10,0x00,0x00]
+          vpunpckhdq 4096(%rdx), %ymm24, %ymm26
+
+// CHECK: vpunpckhdq -4096(%rdx), %ymm24, %ymm26
+// CHECK:  encoding: [0x62,0x61,0x3d,0x20,0x6a,0x52,0x80]
+          vpunpckhdq -4096(%rdx), %ymm24, %ymm26
+
+// CHECK: vpunpckhdq -4128(%rdx), %ymm24, %ymm26
+// CHECK:  encoding: [0x62,0x61,0x3d,0x20,0x6a,0x92,0xe0,0xef,0xff,0xff]
+          vpunpckhdq -4128(%rdx), %ymm24, %ymm26
+
+// CHECK: vpunpckhdq 508(%rdx){1to8}, %ymm24, %ymm26
+// CHECK:  encoding: [0x62,0x61,0x3d,0x30,0x6a,0x52,0x7f]
+          vpunpckhdq 508(%rdx){1to8}, %ymm24, %ymm26
+
+// CHECK: vpunpckhdq 512(%rdx){1to8}, %ymm24, %ymm26
+// CHECK:  encoding: [0x62,0x61,0x3d,0x30,0x6a,0x92,0x00,0x02,0x00,0x00]
+          vpunpckhdq 512(%rdx){1to8}, %ymm24, %ymm26
+
+// CHECK: vpunpckhdq -512(%rdx){1to8}, %ymm24, %ymm26
+// CHECK:  encoding: [0x62,0x61,0x3d,0x30,0x6a,0x52,0x80]
+          vpunpckhdq -512(%rdx){1to8}, %ymm24, %ymm26
+
+// CHECK: vpunpckhdq -516(%rdx){1to8}, %ymm24, %ymm26
+// CHECK:  encoding: [0x62,0x61,0x3d,0x30,0x6a,0x92,0xfc,0xfd,0xff,0xff]
+          vpunpckhdq -516(%rdx){1to8}, %ymm24, %ymm26
+
+// CHECK: vpunpcklqdq %xmm23, %xmm27, %xmm17
+// CHECK:  encoding: [0x62,0xa1,0xa5,0x00,0x6c,0xcf]
+          vpunpcklqdq %xmm23, %xmm27, %xmm17
+
+// CHECK: vpunpcklqdq %xmm23, %xmm27, %xmm17 {%k6}
+// CHECK:  encoding: [0x62,0xa1,0xa5,0x06,0x6c,0xcf]
+          vpunpcklqdq %xmm23, %xmm27, %xmm17 {%k6}
+
+// CHECK: vpunpcklqdq %xmm23, %xmm27, %xmm17 {%k6} {z}
+// CHECK:  encoding: [0x62,0xa1,0xa5,0x86,0x6c,0xcf]
+          vpunpcklqdq %xmm23, %xmm27, %xmm17 {%k6} {z}
+
+// CHECK: vpunpcklqdq (%rcx), %xmm27, %xmm17
+// CHECK:  encoding: [0x62,0xe1,0xa5,0x00,0x6c,0x09]
+          vpunpcklqdq (%rcx), %xmm27, %xmm17
+
+// CHECK: vpunpcklqdq 291(%rax,%r14,8), %xmm27, %xmm17
+// CHECK:  encoding: [0x62,0xa1,0xa5,0x00,0x6c,0x8c,0xf0,0x23,0x01,0x00,0x00]
+          vpunpcklqdq 291(%rax,%r14,8), %xmm27, %xmm17
+
+// CHECK: vpunpcklqdq (%rcx){1to2}, %xmm27, %xmm17
+// CHECK:  encoding: [0x62,0xe1,0xa5,0x10,0x6c,0x09]
+          vpunpcklqdq (%rcx){1to2}, %xmm27, %xmm17
+
+// CHECK: vpunpcklqdq 2032(%rdx), %xmm27, %xmm17
+// CHECK:  encoding: [0x62,0xe1,0xa5,0x00,0x6c,0x4a,0x7f]
+          vpunpcklqdq 2032(%rdx), %xmm27, %xmm17
+
+// CHECK: vpunpcklqdq 2048(%rdx), %xmm27, %xmm17
+// CHECK:  encoding: [0x62,0xe1,0xa5,0x00,0x6c,0x8a,0x00,0x08,0x00,0x00]
+          vpunpcklqdq 2048(%rdx), %xmm27, %xmm17
+
+// CHECK: vpunpcklqdq -2048(%rdx), %xmm27, %xmm17
+// CHECK:  encoding: [0x62,0xe1,0xa5,0x00,0x6c,0x4a,0x80]
+          vpunpcklqdq -2048(%rdx), %xmm27, %xmm17
+
+// CHECK: vpunpcklqdq -2064(%rdx), %xmm27, %xmm17
+// CHECK:  encoding: [0x62,0xe1,0xa5,0x00,0x6c,0x8a,0xf0,0xf7,0xff,0xff]
+          vpunpcklqdq -2064(%rdx), %xmm27, %xmm17
+
+// CHECK: vpunpcklqdq 1016(%rdx){1to2}, %xmm27, %xmm17
+// CHECK:  encoding: [0x62,0xe1,0xa5,0x10,0x6c,0x4a,0x7f]
+          vpunpcklqdq 1016(%rdx){1to2}, %xmm27, %xmm17
+
+// CHECK: vpunpcklqdq 1024(%rdx){1to2}, %xmm27, %xmm17
+// CHECK:  encoding: [0x62,0xe1,0xa5,0x10,0x6c,0x8a,0x00,0x04,0x00,0x00]
+          vpunpcklqdq 1024(%rdx){1to2}, %xmm27, %xmm17
+
+// CHECK: vpunpcklqdq -1024(%rdx){1to2}, %xmm27, %xmm17
+// CHECK:  encoding: [0x62,0xe1,0xa5,0x10,0x6c,0x4a,0x80]
+          vpunpcklqdq -1024(%rdx){1to2}, %xmm27, %xmm17
+
+// CHECK: vpunpcklqdq -1032(%rdx){1to2}, %xmm27, %xmm17
+// CHECK:  encoding: [0x62,0xe1,0xa5,0x10,0x6c,0x8a,0xf8,0xfb,0xff,0xff]
+          vpunpcklqdq -1032(%rdx){1to2}, %xmm27, %xmm17
+
+// CHECK: vpunpcklqdq %ymm28, %ymm20, %ymm29
+// CHECK:  encoding: [0x62,0x01,0xdd,0x20,0x6c,0xec]
+          vpunpcklqdq %ymm28, %ymm20, %ymm29
+
+// CHECK: vpunpcklqdq %ymm28, %ymm20, %ymm29 {%k7}
+// CHECK:  encoding: [0x62,0x01,0xdd,0x27,0x6c,0xec]
+          vpunpcklqdq %ymm28, %ymm20, %ymm29 {%k7}
+
+// CHECK: vpunpcklqdq %ymm28, %ymm20, %ymm29 {%k7} {z}
+// CHECK:  encoding: [0x62,0x01,0xdd,0xa7,0x6c,0xec]
+          vpunpcklqdq %ymm28, %ymm20, %ymm29 {%k7} {z}
+
+// CHECK: vpunpcklqdq (%rcx), %ymm20, %ymm29
+// CHECK:  encoding: [0x62,0x61,0xdd,0x20,0x6c,0x29]
+          vpunpcklqdq (%rcx), %ymm20, %ymm29
+
+// CHECK: vpunpcklqdq 291(%rax,%r14,8), %ymm20, %ymm29
+// CHECK:  encoding: [0x62,0x21,0xdd,0x20,0x6c,0xac,0xf0,0x23,0x01,0x00,0x00]
+          vpunpcklqdq 291(%rax,%r14,8), %ymm20, %ymm29
+
+// CHECK: vpunpcklqdq (%rcx){1to4}, %ymm20, %ymm29
+// CHECK:  encoding: [0x62,0x61,0xdd,0x30,0x6c,0x29]
+          vpunpcklqdq (%rcx){1to4}, %ymm20, %ymm29
+
+// CHECK: vpunpcklqdq 4064(%rdx), %ymm20, %ymm29
+// CHECK:  encoding: [0x62,0x61,0xdd,0x20,0x6c,0x6a,0x7f]
+          vpunpcklqdq 4064(%rdx), %ymm20, %ymm29
+
+// CHECK: vpunpcklqdq 4096(%rdx), %ymm20, %ymm29
+// CHECK:  encoding: [0x62,0x61,0xdd,0x20,0x6c,0xaa,0x00,0x10,0x00,0x00]
+          vpunpcklqdq 4096(%rdx), %ymm20, %ymm29
+
+// CHECK: vpunpcklqdq -4096(%rdx), %ymm20, %ymm29
+// CHECK:  encoding: [0x62,0x61,0xdd,0x20,0x6c,0x6a,0x80]
+          vpunpcklqdq -4096(%rdx), %ymm20, %ymm29
+
+// CHECK: vpunpcklqdq -4128(%rdx), %ymm20, %ymm29
+// CHECK:  encoding: [0x62,0x61,0xdd,0x20,0x6c,0xaa,0xe0,0xef,0xff,0xff]
+          vpunpcklqdq -4128(%rdx), %ymm20, %ymm29
+
+// CHECK: vpunpcklqdq 1016(%rdx){1to4}, %ymm20, %ymm29
+// CHECK:  encoding: [0x62,0x61,0xdd,0x30,0x6c,0x6a,0x7f]
+          vpunpcklqdq 1016(%rdx){1to4}, %ymm20, %ymm29
+
+// CHECK: vpunpcklqdq 1024(%rdx){1to4}, %ymm20, %ymm29
+// CHECK:  encoding: [0x62,0x61,0xdd,0x30,0x6c,0xaa,0x00,0x04,0x00,0x00]
+          vpunpcklqdq 1024(%rdx){1to4}, %ymm20, %ymm29
+
+// CHECK: vpunpcklqdq -1024(%rdx){1to4}, %ymm20, %ymm29
+// CHECK:  encoding: [0x62,0x61,0xdd,0x30,0x6c,0x6a,0x80]
+          vpunpcklqdq -1024(%rdx){1to4}, %ymm20, %ymm29
+
+// CHECK: vpunpcklqdq -1032(%rdx){1to4}, %ymm20, %ymm29
+// CHECK:  encoding: [0x62,0x61,0xdd,0x30,0x6c,0xaa,0xf8,0xfb,0xff,0xff]
+          vpunpcklqdq -1032(%rdx){1to4}, %ymm20, %ymm29
+
+// CHECK: vpunpckhqdq %xmm24, %xmm19, %xmm19
+// CHECK:  encoding: [0x62,0x81,0xe5,0x00,0x6d,0xd8]
+          vpunpckhqdq %xmm24, %xmm19, %xmm19
+
+// CHECK: vpunpckhqdq %xmm24, %xmm19, %xmm19 {%k6}
+// CHECK:  encoding: [0x62,0x81,0xe5,0x06,0x6d,0xd8]
+          vpunpckhqdq %xmm24, %xmm19, %xmm19 {%k6}
+
+// CHECK: vpunpckhqdq %xmm24, %xmm19, %xmm19 {%k6} {z}
+// CHECK:  encoding: [0x62,0x81,0xe5,0x86,0x6d,0xd8]
+          vpunpckhqdq %xmm24, %xmm19, %xmm19 {%k6} {z}
+
+// CHECK: vpunpckhqdq (%rcx), %xmm19, %xmm19
+// CHECK:  encoding: [0x62,0xe1,0xe5,0x00,0x6d,0x19]
+          vpunpckhqdq (%rcx), %xmm19, %xmm19
+
+// CHECK: vpunpckhqdq 291(%rax,%r14,8), %xmm19, %xmm19
+// CHECK:  encoding: [0x62,0xa1,0xe5,0x00,0x6d,0x9c,0xf0,0x23,0x01,0x00,0x00]
+          vpunpckhqdq 291(%rax,%r14,8), %xmm19, %xmm19
+
+// CHECK: vpunpckhqdq (%rcx){1to2}, %xmm19, %xmm19
+// CHECK:  encoding: [0x62,0xe1,0xe5,0x10,0x6d,0x19]
+          vpunpckhqdq (%rcx){1to2}, %xmm19, %xmm19
+
+// CHECK: vpunpckhqdq 2032(%rdx), %xmm19, %xmm19
+// CHECK:  encoding: [0x62,0xe1,0xe5,0x00,0x6d,0x5a,0x7f]
+          vpunpckhqdq 2032(%rdx), %xmm19, %xmm19
+
+// CHECK: vpunpckhqdq 2048(%rdx), %xmm19, %xmm19
+// CHECK:  encoding: [0x62,0xe1,0xe5,0x00,0x6d,0x9a,0x00,0x08,0x00,0x00]
+          vpunpckhqdq 2048(%rdx), %xmm19, %xmm19
+
+// CHECK: vpunpckhqdq -2048(%rdx), %xmm19, %xmm19
+// CHECK:  encoding: [0x62,0xe1,0xe5,0x00,0x6d,0x5a,0x80]
+          vpunpckhqdq -2048(%rdx), %xmm19, %xmm19
+
+// CHECK: vpunpckhqdq -2064(%rdx), %xmm19, %xmm19
+// CHECK:  encoding: [0x62,0xe1,0xe5,0x00,0x6d,0x9a,0xf0,0xf7,0xff,0xff]
+          vpunpckhqdq -2064(%rdx), %xmm19, %xmm19
+
+// CHECK: vpunpckhqdq 1016(%rdx){1to2}, %xmm19, %xmm19
+// CHECK:  encoding: [0x62,0xe1,0xe5,0x10,0x6d,0x5a,0x7f]
+          vpunpckhqdq 1016(%rdx){1to2}, %xmm19, %xmm19
+
+// CHECK: vpunpckhqdq 1024(%rdx){1to2}, %xmm19, %xmm19
+// CHECK:  encoding: [0x62,0xe1,0xe5,0x10,0x6d,0x9a,0x00,0x04,0x00,0x00]
+          vpunpckhqdq 1024(%rdx){1to2}, %xmm19, %xmm19
+
+// CHECK: vpunpckhqdq -1024(%rdx){1to2}, %xmm19, %xmm19
+// CHECK:  encoding: [0x62,0xe1,0xe5,0x10,0x6d,0x5a,0x80]
+          vpunpckhqdq -1024(%rdx){1to2}, %xmm19, %xmm19
+
+// CHECK: vpunpckhqdq -1032(%rdx){1to2}, %xmm19, %xmm19
+// CHECK:  encoding: [0x62,0xe1,0xe5,0x10,0x6d,0x9a,0xf8,0xfb,0xff,0xff]
+          vpunpckhqdq -1032(%rdx){1to2}, %xmm19, %xmm19
+
+// CHECK: vpunpckhqdq %ymm28, %ymm19, %ymm20
+// CHECK:  encoding: [0x62,0x81,0xe5,0x20,0x6d,0xe4]
+          vpunpckhqdq %ymm28, %ymm19, %ymm20
+
+// CHECK: vpunpckhqdq %ymm28, %ymm19, %ymm20 {%k6}
+// CHECK:  encoding: [0x62,0x81,0xe5,0x26,0x6d,0xe4]
+          vpunpckhqdq %ymm28, %ymm19, %ymm20 {%k6}
+
+// CHECK: vpunpckhqdq %ymm28, %ymm19, %ymm20 {%k6} {z}
+// CHECK:  encoding: [0x62,0x81,0xe5,0xa6,0x6d,0xe4]
+          vpunpckhqdq %ymm28, %ymm19, %ymm20 {%k6} {z}
+
+// CHECK: vpunpckhqdq (%rcx), %ymm19, %ymm20
+// CHECK:  encoding: [0x62,0xe1,0xe5,0x20,0x6d,0x21]
+          vpunpckhqdq (%rcx), %ymm19, %ymm20
+
+// CHECK: vpunpckhqdq 291(%rax,%r14,8), %ymm19, %ymm20
+// CHECK:  encoding: [0x62,0xa1,0xe5,0x20,0x6d,0xa4,0xf0,0x23,0x01,0x00,0x00]
+          vpunpckhqdq 291(%rax,%r14,8), %ymm19, %ymm20
+
+// CHECK: vpunpckhqdq (%rcx){1to4}, %ymm19, %ymm20
+// CHECK:  encoding: [0x62,0xe1,0xe5,0x30,0x6d,0x21]
+          vpunpckhqdq (%rcx){1to4}, %ymm19, %ymm20
+
+// CHECK: vpunpckhqdq 4064(%rdx), %ymm19, %ymm20
+// CHECK:  encoding: [0x62,0xe1,0xe5,0x20,0x6d,0x62,0x7f]
+          vpunpckhqdq 4064(%rdx), %ymm19, %ymm20
+
+// CHECK: vpunpckhqdq 4096(%rdx), %ymm19, %ymm20
+// CHECK:  encoding: [0x62,0xe1,0xe5,0x20,0x6d,0xa2,0x00,0x10,0x00,0x00]
+          vpunpckhqdq 4096(%rdx), %ymm19, %ymm20
+
+// CHECK: vpunpckhqdq -4096(%rdx), %ymm19, %ymm20
+// CHECK:  encoding: [0x62,0xe1,0xe5,0x20,0x6d,0x62,0x80]
+          vpunpckhqdq -4096(%rdx), %ymm19, %ymm20
+
+// CHECK: vpunpckhqdq -4128(%rdx), %ymm19, %ymm20
+// CHECK:  encoding: [0x62,0xe1,0xe5,0x20,0x6d,0xa2,0xe0,0xef,0xff,0xff]
+          vpunpckhqdq -4128(%rdx), %ymm19, %ymm20
+
+// CHECK: vpunpckhqdq 1016(%rdx){1to4}, %ymm19, %ymm20
+// CHECK:  encoding: [0x62,0xe1,0xe5,0x30,0x6d,0x62,0x7f]
+          vpunpckhqdq 1016(%rdx){1to4}, %ymm19, %ymm20
+
+// CHECK: vpunpckhqdq 1024(%rdx){1to4}, %ymm19, %ymm20
+// CHECK:  encoding: [0x62,0xe1,0xe5,0x30,0x6d,0xa2,0x00,0x04,0x00,0x00]
+          vpunpckhqdq 1024(%rdx){1to4}, %ymm19, %ymm20
+
+// CHECK: vpunpckhqdq -1024(%rdx){1to4}, %ymm19, %ymm20
+// CHECK:  encoding: [0x62,0xe1,0xe5,0x30,0x6d,0x62,0x80]
+          vpunpckhqdq -1024(%rdx){1to4}, %ymm19, %ymm20
+
+// CHECK: vpunpckhqdq -1032(%rdx){1to4}, %ymm19, %ymm20
+// CHECK:  encoding: [0x62,0xe1,0xe5,0x30,0x6d,0xa2,0xf8,0xfb,0xff,0xff]
+          vpunpckhqdq -1032(%rdx){1to4}, %ymm19, %ymm20
+
diff --git a/test/MC/X86/x86_errors.s b/test/MC/X86/x86_errors.s
index fa87ef676f3..4a5bd977d9f 100644
--- a/test/MC/X86/x86_errors.s
+++ b/test/MC/X86/x86_errors.s
@@ -58,3 +58,9 @@ cmpps $-129, %xmm0, %xmm0
 // 32: error: invalid operand for instruction
 // 64: error: invalid operand for instruction
 cmppd $256, %xmm0, %xmm0
+
+// 32: error: instruction requires: 64-bit mode
+jrcxz 1
+
+// 64: error: instruction requires: Not 64-bit mode
+jcxz 1
diff --git a/test/NaCl/ARM/GOT-with-disabled-constpools.ll b/test/NaCl/ARM/GOT-with-disabled-constpools.ll
deleted file mode 100644
index 490d3d103a7..00000000000
--- a/test/NaCl/ARM/GOT-with-disabled-constpools.ll
+++ /dev/null
@@ -1,49 +0,0 @@
-; RUN: pnacl-llc -mtriple=armv7a-none-nacl-gnueabi %s -filetype=obj \
-; RUN:  -relocation-model=pic -mattr=+neon \
-; RUN:  -O0 -mcpu=cortex-a9 -o - \
-; RUN:  | llvm-objdump -disassemble -r -triple armv7 - | FileCheck %s
-
-; This test exercises NaCl (which doesn't use constant islands)
-; together with -relocation-model=pic, to see that a movw/movt
-; is actually generated as expected. Use -O0 so that movw/movt
-; are scheduled to be adjacent, instead of having other independent
-; instructions interleaved.
-
-%struct.object = type { [16 x i8*] }
-
-@__do_eh_ctor.object = internal global %struct.object zeroinitializer, align 8
-@__EH_FRAME_BEGIN__ = internal global [0 x i8] zeroinitializer, section ".eh_frame", align 4
-@llvm.global_ctors = appending global [1 x { i32, void ()* }] [{ i32, void ()* } { i32 65535, void ()* @__do_eh_ctor }]
-
-declare void @__register_frame_info(i8* %begin, %struct.object* %ob)
-
-define internal void @__do_eh_ctor() {
-entry:
-  call void @__register_frame_info(i8* getelementptr inbounds ([0 x i8], [0 x i8]* @__EH_FRAME_BEGIN__, i32 0, i32 0), %struct.object* @__do_eh_ctor.object)
-; CHECK-LABEL: __do_eh_ctor
-
-; CHECK: movw r[[REG0:[0-9]+]]
-; CHECK-NEXT: R_ARM_MOVW_PREL_NC .LCPI
-; CHECK-NEXT: movt r[[REG0]]
-; CHECK-NEXT: R_ARM_MOVT_PREL .LCPI
-; CHECK-NEXT: add r[[REG0]], pc, r[[REG0]]
-; CHECK: ldr r[[REG0]], {{\[}}r[[REG0]]{{\]}}
-
-; CHECK: movw r[[REG1:[0-9]+]]
-; CHECK-NEXT: R_ARM_MOVW_PREL_NC _GLOBAL_OFFSET_TABLE_
-; CHECK-NEXT: movt r[[REG1]]
-; CHECK-NEXT: R_ARM_MOVT_PREL _GLOBAL_OFFSET_TABLE_
-; CHECK-NEXT: add r[[REG1]], pc, r[[REG1]]
-; CHECK: add r[[REG0]], r[[REG0]], r[[REG1]]
-
-; CHECK: movw r[[REG2:[0-9]+]]
-; CHECK-NEXT: R_ARM_MOVW_PREL_NC .LCPI
-; CHECK-NEXT: movt r[[REG2]]
-; CHECK-NEXT: R_ARM_MOVT_PREL .LCPI
-; CHECK-NEXT: add r[[REG2]], pc, r[[REG2]]
-; CHECK: ldr r[[REG2]], {{\[}}r[[REG2]]{{\]}}
-; CHECK: add r[[REG1]], r[[REG2]], r[[REG1]]
-
-  ret void
-}
-
diff --git a/test/NaCl/ARM/blx-sandboxing.ll b/test/NaCl/ARM/blx-sandboxing.ll
deleted file mode 100644
index 7c78cd7fdb2..00000000000
--- a/test/NaCl/ARM/blx-sandboxing.ll
+++ /dev/null
@@ -1,17 +0,0 @@
-; RUN: pnacl-llc -mtriple=armv7-unknown-nacl -filetype=obj %s -o - \
-; RUN:  | llvm-objdump -disassemble -triple armv7 - | FileCheck %s
-
-define i32 @foobar(i32 %aa, i32 %bb, i32 (i32)* %f) nounwind {
-entry:
-  %aa.addr = alloca i32, align 4
-  %bb.addr = alloca i32, align 4
-  %f.addr = alloca i32 (i32)*, align 8
-  %0 = load i32 (i32)*, i32 (i32)** %f.addr, align 8
-  %1 = load i32, i32* %aa.addr, align 4
-  %call1 = call i32 %0(i32 %1)
-; CHECK: bic	r1, r1, #-1073741809
-; CHECK-NEXT: blx r1
-  ret i32 %call1
-}
-
-
diff --git a/test/NaCl/ARM/bx-sandboxing.ll b/test/NaCl/ARM/bx-sandboxing.ll
deleted file mode 100644
index 641cc11f858..00000000000
--- a/test/NaCl/ARM/bx-sandboxing.ll
+++ /dev/null
@@ -1,10 +0,0 @@
-; RUN: pnacl-llc -mtriple=armv7-unknown-nacl -filetype=obj %s -o - \
-; RUN:  | llvm-objdump -disassemble -triple armv7 - | FileCheck %s
-
-define i32 @foo() nounwind {
-entry:
-  ret i32 42
-; CHECK: bic	lr, lr, #-1073741809
-; CHECK-NEXT: bx lr
-}
-
diff --git a/test/NaCl/ARM/call-return-sandboxing1.ll b/test/NaCl/ARM/call-return-sandboxing1.ll
deleted file mode 100644
index a26d965d292..00000000000
--- a/test/NaCl/ARM/call-return-sandboxing1.ll
+++ /dev/null
@@ -1,65 +0,0 @@
-; RUN: pnacl-llc -mtriple=armv7-unknown-nacl -filetype=obj %s -o - \
-; RUN:  | llvm-objdump -disassemble -triple armv7 - | FileCheck %s
-
-define i32 @foo(i32 %aa, i32 %bb) nounwind {
-entry:
-  %aa.addr = alloca i32, align 4
-  %bb.addr = alloca i32, align 4
-  %cc = alloca i32, align 4
-  %dd = alloca i32, align 4
-  store i32 %aa, i32* %aa.addr, align 4
-  store i32 %bb, i32* %bb.addr, align 4
-  %0 = load i32, i32* %aa.addr, align 4
-  %1 = load i32, i32* %bb.addr, align 4
-  %mul = mul nsw i32 %0, %1
-  store i32 %mul, i32* %cc, align 4
-  %sub = sub nsw i32 %mul, %1
-  store i32 %sub, i32* %dd, align 4
-  %2 = load i32, i32* %dd, align 4
-  ret i32 %2
-
-; This checks two things:
-; 1. bx lr is sandboxed by prepending a bic
-; 2. The bic/bx pair don't straddle a 16-byte bundle boundary, hence the nop
-; CHECK:                    nop
-; CHECK-NEXT: {{.*}}0:{{.*}}bic lr, lr, #-1073741809
-; CHECK-NEXT:               bx lr
-
-}
-
-define i32 @bar(i32 %aa, i32 %bb) nounwind {
-entry:
-
-; Check that the function start is padded with nops to start at a bundle
-; boundary
-; CHECK:                    nop
-; CHECK-LABEL: bar:
-; CHECK-NEXT: {{.*}}0:{{.*}}push
-
-  %aa.addr = alloca i32, align 4
-  %bb.addr = alloca i32, align 4
-  store i32 %aa, i32* %aa.addr, align 4
-  store i32 %bb, i32* %bb.addr, align 4
-  %0 = load i32, i32* %aa.addr, align 4
-  %mul = mul nsw i32 %0, 19
-  %call = call i32 @foo(i32 %mul, i32 7)
-
-; Check that the call is padded to be at the end of a bundle
-; CHECK:      {{.*}}8:{{.*}}nop
-; CHECK-NEXT: {{.*}}c:{{.*}}bl
-
-  %1 = load i32, i32* %bb.addr, align 4
-  %mul1 = mul nsw i32 %1, 31
-  %2 = load i32, i32* %bb.addr, align 4
-  %div = sdiv i32 %2, 7
-  %add = add nsw i32 %div, 191
-  %call2 = call i32 @foo(i32 %mul1, i32 %add)
-
-; Check that the call is padded to be at the end of a bundle
-; CHECK:      {{.*}}8:{{.*}}nop
-; CHECK-NEXT: {{.*}}c:{{.*}}bl
-
-  %add3 = add nsw i32 %call, %call2
-  ret i32 %add3
-}
-
diff --git a/test/NaCl/ARM/divrem-guards-complex.ll b/test/NaCl/ARM/divrem-guards-complex.ll
deleted file mode 100644
index ba66ab4b319..00000000000
--- a/test/NaCl/ARM/divrem-guards-complex.ll
+++ /dev/null
@@ -1,85 +0,0 @@
-; RUN: opt < %s -insert-divide-check -S | FileCheck -check-prefix=OPT %s
-
-declare void @foo()
-
-; Check for multiple divs that occur one block.
-define i32 @twodivs_one_block(i32 %x, i32 %y) #0 {
-entry:
-  call void @foo()
-  br label %divblock
-divblock:
-  %div1 = sdiv i32 %x, %y
-  %div2 = sdiv i32 %x, %y
-; OPT: %0 = icmp eq i32 %y, 0
-; OPT-NEXT: br i1 %0, label %divrem.by.zero, label %guarded.divrem
-; OPT: guarded.divrem:
-; OPT-NEXT: sdiv
-; OPT: %1 = icmp eq i32 %y, 0
-; OPT-NEXT: br i1 %1, label %divrem.by.zero1, label %guarded.divrem2
-; OPT: guarded.divrem2:
-; OPT-NEXT: sdiv
-; OPT-NEXT: add
-; OPT: divrem.by.zero:
-; OPT-NEXT: call void @llvm.trap()
-; OPT-NEXT: unreachable
-; OPT: divrem.by.zero1:
-; OPT-NEXT: call void @llvm.trap()
-; OPT-NEXT: unreachable
-  %sum = add i32 %div1, %div2
-  ret i32 %sum
-}
-
-define i32 @twodivs_three_blocks(i32 %x, i32 %y) #0 {
-entry:
-  call void @foo()
-  br label %divblock
-divblock:
-  %div1 = sdiv i32 %x, %y
-; OPT: %0 = icmp eq i32 %y, 0
-; OPT-NEXT: br i1 %0, label %divrem.by.zero, label %guarded.divrem
-; OPT: guarded.divrem:
-; OPT-NEXT: sdiv
-; OPT-NEXT: br label %exitblock
-  br label %exitblock
-exitblock:
-  call void @foo()
-  %div2 = sdiv i32 %x, %y
-; OPT: %1 = icmp eq i32 %y, 0
-; OPT-NEXT: br i1 %1, label %divrem.by.zero1, label %guarded.divrem2
-; OPT: guarded.divrem2:
-; OPT-NEXT: sdiv
-; OPT-NEXT: add
-; OPT: divrem.by.zero:
-; OPT-NEXT: call void @llvm.trap()
-; OPT-NEXT: unreachable
-; OPT: divrem.by.zero1:
-; OPT-NEXT: call void @llvm.trap()
-; OPT-NEXT: unreachable
-  %sum = add i32 %div1, %div2
-  ret i32 %sum
-}
-
-; Check for divs that occur in blocks with multiple predecessors.
-define i32 @onediv_two_predecessors(i32 %x, i32 %y) #0 {
-entry:
-  call void @foo()
-  br label %divblock
-divblock:
-  %x1 = phi i32 [%x, %entry], [%x2, %divblock]
-  %div1 = sdiv i32 %x, %y
-; OPT: %0 = icmp eq i32 %y, 0
-; OPT-NEXT: br i1 %0, label %divrem.by.zero, label %guarded.divrem
-; OPT: guarded.divrem:
-; OPT-NEXT: sdiv
-; OPT-NEXT: sub
-; OPT: divrem.by.zero:
-; OPT-NEXT: call void @llvm.trap()
-; OPT-NEXT: unreachable
-  %x2 = sub i32 %x1, 1
-  %p = icmp ne i32 %x2, 0
-  br i1 %p, label %divblock, label %exitblock
-exitblock:
-  call void @foo()
-  ret i32 %div1
-}
-
diff --git a/test/NaCl/ARM/divrem-guards.ll b/test/NaCl/ARM/divrem-guards.ll
deleted file mode 100644
index ec0f304afff..00000000000
--- a/test/NaCl/ARM/divrem-guards.ll
+++ /dev/null
@@ -1,104 +0,0 @@
-; RUN: opt < %s -insert-divide-check -S | FileCheck -check-prefix=OPT %s
-; RUN: pnacl-llc -mtriple=armv7-unknown-nacl -filetype=obj %s -o - \
-; RUN:  | llvm-objdump -disassemble -triple armv7 -mattr=+nacl-trap - \
-; RUN:  | FileCheck -check-prefix=ARM %s
-
-
-; Check for all four operators that need guards.
-define i32 @mysdiv(i32 %x, i32 %y) #0 {
-entry:
-  %div1 = sdiv i32 %x, %y
-; OPT: %0 = icmp eq i32 %y, 0
-; OPT-NEXT: br i1 %0, label %divrem.by.zero, label %guarded.divrem
-; OPT: guarded.divrem:
-; OPT-NEXT: sdiv
-; OPT-NEXT: ret
-; OPT: divrem.by.zero:
-; OPT-NEXT: call void @llvm.trap()
-; OPT-NEXT: unreachable
-; ARM: cmp r1, #0
-; ARM-NEXT: beq
-  ret i32 %div1
-; ARM: f0 de fe e7
-}
-
-define i32 @myudiv(i32 %x, i32 %y) #0 {
-entry:
-  %div1 = udiv i32 %x, %y
-; OPT: %0 = icmp eq i32 %y, 0
-; OPT-NEXT: br i1 %0, label %divrem.by.zero, label %guarded.divrem
-; OPT: guarded.divrem:
-; OPT-NEXT: udiv
-; OPT-NEXT: ret
-; OPT: divrem.by.zero:
-; OPT-NEXT: call void @llvm.trap()
-; OPT-NEXT: unreachable
-; ARM: cmp r1, #0
-; ARM-NEXT: beq
-  ret i32 %div1
-; ARM: f0 de fe e7
-}
-
-define i32 @mysrem(i32 %x, i32 %y) #0 {
-entry:
-  %rem1 = srem i32 %x, %y
-; OPT: %0 = icmp eq i32 %y, 0
-; OPT-NEXT: br i1 %0, label %divrem.by.zero, label %guarded.divrem
-; OPT: guarded.divrem:
-; OPT-NEXT: srem
-; OPT-NEXT: ret
-; OPT: divrem.by.zero:
-; OPT-NEXT: call void @llvm.trap()
-; OPT-NEXT: unreachable
-; ARM: cmp r1, #0
-; ARM-NEXT: beq
-  ret i32 %rem1
-; ARM: f0 de fe e7
-}
-
-define i32 @myurem(i32 %x, i32 %y) #0 {
-entry:
-  %rem1 = urem i32 %x, %y
-; OPT: %0 = icmp eq i32 %y, 0
-; OPT-NEXT: br i1 %0, label %divrem.by.zero, label %guarded.divrem
-; OPT: guarded.divrem:
-; OPT-NEXT: urem
-; OPT-NEXT: ret
-; OPT: divrem.by.zero:
-; OPT-NEXT: call void @llvm.trap()
-; OPT-NEXT: unreachable
-; ARM: cmp r1, #0
-; ARM-NEXT: beq
-  ret i32 %rem1
-; ARM: f0 de fe e7
-}
-
-; Divides by non-zero constants should not be guarded.
-define i32 @mysdiv_const(i32 %x) #0 {
-entry:
-  %div1 = sdiv i32 %x, 10
-; OPT-NOT: icmp
-; OPT-NOT: br
-; OPT-NOT: guarded.divrem:
-; OPT-NOT: divrem.by.zero:
-; OPT-NOT: call void @llvm.trap()
-; OPT-NOT: unreachable
-; ARM-NOT: cmp r1, #0
-; ARM-NOT: f0 de fe e7
-  ret i32 %div1
-}
-
-; Divides by explicit zero should prefixed by a trap.
-define i32 @mysdiv_zero(i32 %x) #0 {
-entry:
-  %div1 = sdiv i32 %x, 0
-; OPT-NOT: guarded.divrem:
-; OPT-NOT: divrem.by.zero:
-; OPT: call void @llvm.trap()
-; OPT-NEXT: sdiv
-; ARM-NOT: cmp r1, #0
-; ARM: f0 de fe e7
-  ret i32 %div1
-}
-
-attributes #0 = { nounwind }
diff --git a/test/NaCl/ARM/intrinsics-bitmanip.ll b/test/NaCl/ARM/intrinsics-bitmanip.ll
deleted file mode 100644
index 2bbe0c68308..00000000000
--- a/test/NaCl/ARM/intrinsics-bitmanip.ll
+++ /dev/null
@@ -1,65 +0,0 @@
-; RUN: pnacl-llc -mtriple=armv7-unknown-nacl -filetype=asm %s -o - | FileCheck %s
-; RUN: pnacl-llc -mtriple=armv7-unknown-nacl -O0 -filetype=asm %s -o - | FileCheck %s
-
-; Test that various bit manipulation intrinsics are supported by the
-; NaCl ARM backend.
-
-declare i16 @llvm.bswap.i16(i16)
-declare i32 @llvm.bswap.i32(i32)
-declare i64 @llvm.bswap.i64(i64)
-
-; CHECK: test_bswap_16
-; CHECK: rev [[REG:r[0-9]+]], {{r[0-9]+}}
-; CHECK-NEXT: lsr {{.*}}[[REG]], {{.*}}[[REG]], #16
-define i16 @test_bswap_16(i16 %a) {
-  %b = call i16 @llvm.bswap.i16(i16 %a)
-  ret i16 %b
-}
-
-; CHECK: test_bswap_const_16
-; 0xcdab
-; CHECK: movw r0, #52651
-define i16 @test_bswap_const_16() {
-  ; 0xabcd
-  %a = call i16 @llvm.bswap.i16(i16 43981)
-  ret i16 %a
-}
-
-; CHECK: test_bswap_32
-; CHECK: rev [[REG:r[0-9]+]], {{r[0-9]+}}
-define i32 @test_bswap_32(i32 %a) {
-  %b = call i32 @llvm.bswap.i32(i32 %a)
-  ret i32 %b
-}
-
-; CHECK: test_bswap_const_32
-; 0x01ef cdab
-; CHECK: movw r0, #52651
-; CHECK: movt r0, #495
-define i32 @test_bswap_const_32() {
-  ; 0xabcdef01
-  %a = call i32 @llvm.bswap.i32(i32 2882400001)
-  ret i32 %a
-}
-
-; CHECK: test_bswap_64
-; O0 spills when swapping r0/r1, whereas O2 uses r2 as a temporary.
-; CHECK: rev {{r[0-9]+}}, {{r[0-9]+}}
-; CHECK: rev r1, {{r[0-9]+}}
-; CHECK: mov r0, {{r[0-9]+}}
-define i64 @test_bswap_64(i64 %a) {
-  %b = call i64 @llvm.bswap.i64(i64 %a)
-  ret i64 %b
-}
-
-; CHECK: test_bswap_const_64
-; 0x8967 4523 01ef cdab
-; Just checking movw, since O0 and O2 have different schedules for the
-; movw/movt of r0/r1.
-; CHECK: movw r0, #52651
-; CHECK: movw r1, #17699
-define i64 @test_bswap_const_64() {
-  ; 0xabcdef01 23456789
-  %a = call i64 @llvm.bswap.i64(i64 12379813738877118345)
-  ret i64 %a
-}
diff --git a/test/NaCl/ARM/lit.local.cfg b/test/NaCl/ARM/lit.local.cfg
deleted file mode 100644
index bac2ffab31d..00000000000
--- a/test/NaCl/ARM/lit.local.cfg
+++ /dev/null
@@ -1,6 +0,0 @@
-config.suffixes = ['.ll']
-
-targets = set(config.root.targets_to_build.split())
-if not 'ARM' in targets:
-    config.unsupported = True
-
diff --git a/test/NaCl/ARM/nacl-read-tp-intrinsic.ll b/test/NaCl/ARM/nacl-read-tp-intrinsic.ll
deleted file mode 100644
index 90a9f787b59..00000000000
--- a/test/NaCl/ARM/nacl-read-tp-intrinsic.ll
+++ /dev/null
@@ -1,20 +0,0 @@
-
-; RUN: pnacl-llc -mtriple=armv7-unknown-nacl -filetype=asm %s -o - \
-; RUN:   | FileCheck -check-prefix=ARM %s
-
-; RUN: pnacl-llc -mtriple=armv7-unknown-nacl -filetype=asm -mtls-use-call %s -o - \
-; RUN:   | FileCheck -check-prefix=ARM_IRT %s
-
-
-declare i8* @llvm.nacl.read.tp()
-
-define i8* @get_thread_pointer() {
-  %tp = call i8* @llvm.nacl.read.tp()
-  ret i8* %tp
-}
-
-; ARM: get_thread_pointer:
-; ARM: ldr r0, [r9]
-
-; ARM_IRT: get_thread_pointer:
-; ARM_IRT: bl __aeabi_read_tp
diff --git a/test/NaCl/ARM/nacl-setlongjmp-intrinsics.ll b/test/NaCl/ARM/nacl-setlongjmp-intrinsics.ll
deleted file mode 100644
index 9d240375608..00000000000
--- a/test/NaCl/ARM/nacl-setlongjmp-intrinsics.ll
+++ /dev/null
@@ -1,18 +0,0 @@
-; RUN: pnacl-llc -mtriple=arm-unknown-nacl -filetype=asm %s -o - \
-; RUN:  | FileCheck %s --check-prefix=ARM
-; Test that @llvm.nacl.{set|long}jmp intrinsics calls get translated to library
-; calls as expected.
-
-declare i32 @llvm.nacl.setjmp(i8*)
-declare void @llvm.nacl.longjmp(i8*, i32)
-
-define void @foo(i8* %arg) {
-  %num = call i32 @llvm.nacl.setjmp(i8* %arg)
-; ARM: bl setjmp
-
-  call void @llvm.nacl.longjmp(i8* %arg, i32 %num)
-; ARM: bl longjmp
-
-  ret void
-}
-
diff --git a/test/NaCl/ARM/negative-addend.ll b/test/NaCl/ARM/negative-addend.ll
deleted file mode 100644
index 314461aef89..00000000000
--- a/test/NaCl/ARM/negative-addend.ll
+++ /dev/null
@@ -1,19 +0,0 @@
-; RUN: pnacl-llc -mtriple=arm-unknown-nacl -filetype=obj %s -o - \
-; RUN:   | llvm-objdump -r - | FileCheck %s -check-prefix=ARM
-
-; Check that "add" works for negative values when used as a
-; ConstantExpr in a global variable initializer.
-; See: https://code.google.com/p/nativeclient/issues/detail?id=3548
-
-
-; @spacer and @var end up in the BSS section.
-; @spacer is at offset 0.  @var is at offset 4096 = 0x1000.
-
-@spacer = internal global [4096 x i8] zeroinitializer
-@var = internal global i32 zeroinitializer
-
-@negative_offset = internal global i32 add
-    (i32 ptrtoint (i32* @var to i32), i32 -8)
-
-; ARM: RELOCATION RECORDS
-; ARM-NEXT: 0 R_ARM_ABS32 .bss
diff --git a/test/NaCl/ARM/neon-vld1-sandboxing.ll b/test/NaCl/ARM/neon-vld1-sandboxing.ll
deleted file mode 100644
index 7a63d972d08..00000000000
--- a/test/NaCl/ARM/neon-vld1-sandboxing.ll
+++ /dev/null
@@ -1,113 +0,0 @@
-; RUN: pnacl-llc -mtriple=armv7-unknown-nacl -mattr=+neon -filetype=obj %s -o - \
-; RUN:  | llvm-objdump -disassemble -triple armv7 - | FileCheck %s
-
-define <8 x i8> @vld1i8(i8* %A) nounwind {
-  %tmp1 = call <8 x i8> @llvm.arm.neon.vld1.v8i8(i8* %A, i32 16)
-; CHECK:         bic r0, r0, #-1073741824
-; CHECK-NEXT:    vld1.8 {{{d[0-9]+}}}, [r0:64]
-  ret <8 x i8> %tmp1
-}
-
-define <4 x i16> @vld1i16(i16* %A) nounwind {
-  %tmp0 = bitcast i16* %A to i8*
-  %tmp1 = call <4 x i16> @llvm.arm.neon.vld1.v4i16(i8* %tmp0, i32 1)
-; CHECK:         bic r0, r0, #-1073741824
-; CHECK-NEXT:    vld1.16 {{{d[0-9]+}}}, [r0]
-  ret <4 x i16> %tmp1
-}
-
-define <2 x i32> @vld1i32(i32* %A) nounwind {
-  %tmp0 = bitcast i32* %A to i8*
-  %tmp1 = call <2 x i32> @llvm.arm.neon.vld1.v2i32(i8* %tmp0, i32 1)
-; CHECK:         bic r0, r0, #-1073741824
-; CHECK-NEXT:    vld1.32 {{{d[0-9]+}}}, [r0]
-  ret <2 x i32> %tmp1
-}
-
-; Insert useless arguments here just for the sake of moving
-; %A further down the rN chain (testing how sandboxing detects
-; the correct register and not just the default r0)
-define <1 x i64> @vld1i64(i32 %foo, i32 %bar, i32 %baz,
-                          i64* %A) nounwind {
-  %tmp0 = bitcast i64* %A to i8*
-  %tmp1 = call <1 x i64> @llvm.arm.neon.vld1.v1i64(i8* %tmp0, i32 1)
-; CHECK:         bic r3, r3, #-1073741824
-; CHECK-NEXT:    vld1.64 {{{d[0-9]+}}}, [r3]
-  ret <1 x i64> %tmp1
-}
-
-define <16 x i8> @vld1Qi8(i8* %A) nounwind {
-  %tmp1 = call <16 x i8> @llvm.arm.neon.vld1.v16i8(i8* %A, i32 8)
-; CHECK:         bic r0, r0, #-1073741824
-; CHECK-NEXT:    vld1.8 {{{d[0-9]+}}, {{d[0-9]+}}}, [r0:64]
-  ret <16 x i8> %tmp1
-}
-
-define <8 x i16> @vld1Qi16(i16* %A) nounwind {
-  %tmp0 = bitcast i16* %A to i8*
-  %tmp1 = call <8 x i16> @llvm.arm.neon.vld1.v8i16(i8* %tmp0, i32 32)
-; CHECK:         bic r0, r0, #-1073741824
-; CHECK-NEXT:    vld1.16 {{{d[0-9]+}}, {{d[0-9]+}}}, [r0:128]
-  ret <8 x i16> %tmp1
-}
-
-define <4 x i32> @vld1Qi32(i32* %A) nounwind {
-  %tmp0 = bitcast i32* %A to i8*
-  %tmp1 = call <4 x i32> @llvm.arm.neon.vld1.v4i32(i8* %tmp0, i32 1)
-; CHECK:         bic r0, r0, #-1073741824
-; CHECK-NEXT:    vld1.32 {{{d[0-9]+}}, {{d[0-9]+}}}, [r0]
-  ret <4 x i32> %tmp1
-}
-
-define <2 x i64> @vld1Qi64(i64* %A) nounwind {
-  %tmp0 = bitcast i64* %A to i8*
-  %tmp1 = call <2 x i64> @llvm.arm.neon.vld1.v2i64(i8* %tmp0, i32 1)
-; CHECK:         bic r0, r0, #-1073741824
-; CHECK-NEXT:    vld1.64 {{{d[0-9]+}}, {{d[0-9]+}}}, [r0]
-  ret <2 x i64> %tmp1
-}
-
-declare <8 x i8>  @llvm.arm.neon.vld1.v8i8(i8*, i32) nounwind readonly
-declare <4 x i16> @llvm.arm.neon.vld1.v4i16(i8*, i32) nounwind readonly
-declare <2 x i32> @llvm.arm.neon.vld1.v2i32(i8*, i32) nounwind readonly
-declare <2 x float> @llvm.arm.neon.vld1.v2f32(i8*, i32) nounwind readonly
-declare <1 x i64> @llvm.arm.neon.vld1.v1i64(i8*, i32) nounwind readonly
-
-declare <16 x i8> @llvm.arm.neon.vld1.v16i8(i8*, i32) nounwind readonly
-declare <8 x i16> @llvm.arm.neon.vld1.v8i16(i8*, i32) nounwind readonly
-declare <4 x i32> @llvm.arm.neon.vld1.v4i32(i8*, i32) nounwind readonly
-declare <4 x float> @llvm.arm.neon.vld1.v4f32(i8*, i32) nounwind readonly
-declare <2 x i64> @llvm.arm.neon.vld1.v2i64(i8*, i32) nounwind readonly
-
-define <16 x i8> @vld1Qi8_update(i8** %ptr) nounwind {
-  %A = load i8*, i8** %ptr
-  %tmp1 = call <16 x i8> @llvm.arm.neon.vld1.v16i8(i8* %A, i32 8)
-; CHECK:         bic r1, r1, #-1073741824
-; CHECK-NEXT:    vld1.8 {{{d[0-9]+}}, {{d[0-9]+}}}, [r1:64]!
-  %tmp2 = getelementptr i8, i8* %A, i32 16
-  store i8* %tmp2, i8** %ptr
-  ret <16 x i8> %tmp1
-}
-
-define <4 x i16> @vld1i16_update(i16** %ptr) nounwind {
-  %A = load i16*, i16** %ptr
-  %tmp0 = bitcast i16* %A to i8*
-  %tmp1 = call <4 x i16> @llvm.arm.neon.vld1.v4i16(i8* %tmp0, i32 1)
-; CHECK:         bic r1, r1, #-1073741824
-; CHECK-NEXT:    vld1.16 {{{d[0-9]+}}}, [r1]!
-  %tmp2 = getelementptr i16, i16* %A, i32 4
-         store i16* %tmp2, i16** %ptr
-  ret <4 x i16> %tmp1
-}
-
-define <2 x i32> @vld1i32_update(i32** %ptr, i32 %inc) nounwind {
-  %A = load i32*, i32** %ptr
-  %tmp0 = bitcast i32* %A to i8*
-  %tmp1 = call <2 x i32> @llvm.arm.neon.vld1.v2i32(i8* %tmp0, i32 1)
-; CHECK:         bic r2, r2, #-1073741824
-; CHECK-NEXT:    vld1.32 {{{d[0-9]+}}}, [r2], r1
-  %tmp2 = getelementptr i32, i32* %A, i32 %inc
-  store i32* %tmp2, i32** %ptr
-  ret <2 x i32> %tmp1
-}
-
diff --git a/test/NaCl/ARM/neon-vld2-sandboxing.ll b/test/NaCl/ARM/neon-vld2-sandboxing.ll
deleted file mode 100644
index de36d2db713..00000000000
--- a/test/NaCl/ARM/neon-vld2-sandboxing.ll
+++ /dev/null
@@ -1,116 +0,0 @@
-; RUN: pnacl-llc -mtriple=armv7-unknown-nacl -mattr=+neon -filetype=obj %s -o - \
-; RUN:  | llvm-objdump -disassemble -triple armv7 - | FileCheck %s
-
-%struct.__neon_int8x8x2_t = type { <8 x i8>,  <8 x i8> }
-%struct.__neon_int16x4x2_t = type { <4 x i16>, <4 x i16> }
-%struct.__neon_int32x2x2_t = type { <2 x i32>, <2 x i32> }
-%struct.__neon_float32x2x2_t = type { <2 x float>, <2 x float> }
-%struct.__neon_int64x1x2_t = type { <1 x i64>, <1 x i64> }
-
-%struct.__neon_int8x16x2_t = type { <16 x i8>,  <16 x i8> }
-%struct.__neon_int16x8x2_t = type { <8 x i16>, <8 x i16> }
-%struct.__neon_int32x4x2_t = type { <4 x i32>, <4 x i32> }
-%struct.__neon_float32x4x2_t = type { <4 x float>, <4 x float> }
-
-declare %struct.__neon_int8x8x2_t @llvm.arm.neon.vld2.v8i8(i8*, i32) nounwind readonly
-declare %struct.__neon_int16x4x2_t @llvm.arm.neon.vld2.v4i16(i8*, i32) nounwind readonly
-declare %struct.__neon_int32x2x2_t @llvm.arm.neon.vld2.v2i32(i8*, i32) nounwind readonly
-declare %struct.__neon_float32x2x2_t @llvm.arm.neon.vld2.v2f32(i8*, i32) nounwind readonly
-declare %struct.__neon_int64x1x2_t @llvm.arm.neon.vld2.v1i64(i8*, i32) nounwind readonly
-
-declare %struct.__neon_int8x16x2_t @llvm.arm.neon.vld2.v16i8(i8*, i32) nounwind readonly
-declare %struct.__neon_int16x8x2_t @llvm.arm.neon.vld2.v8i16(i8*, i32) nounwind readonly
-declare %struct.__neon_int32x4x2_t @llvm.arm.neon.vld2.v4i32(i8*, i32) nounwind readonly
-declare %struct.__neon_float32x4x2_t @llvm.arm.neon.vld2.v4f32(i8*, i32) nounwind readonly
-
-define <8 x i8> @vld2i8(i8* %A) nounwind {
-  %tmp1 = call %struct.__neon_int8x8x2_t @llvm.arm.neon.vld2.v8i8(i8* %A, i32 8)
-  %tmp2 = extractvalue %struct.__neon_int8x8x2_t %tmp1, 0
-  %tmp3 = extractvalue %struct.__neon_int8x8x2_t %tmp1, 1
-  %tmp4 = add <8 x i8> %tmp2, %tmp3
-; CHECK: bic      r0, r0, #-1073741824
-; CHECK: vld2.8   {d{{[0-9]+}}, d{{[0-9]+}}}, [r0:64]
-  ret <8 x i8> %tmp4
-}
-
-define <4 x i16> @vld2i16(i16* %A) nounwind {
-  %tmp0 = bitcast i16* %A to i8*
-  %tmp1 = call %struct.__neon_int16x4x2_t @llvm.arm.neon.vld2.v4i16(i8* %tmp0, i32 32)
-  %tmp2 = extractvalue %struct.__neon_int16x4x2_t %tmp1, 0
-  %tmp3 = extractvalue %struct.__neon_int16x4x2_t %tmp1, 1
-  %tmp4 = add <4 x i16> %tmp2, %tmp3
-; CHECK: bic      r0, r0, #-1073741824
-; CHECK: vld2.16   {d{{[0-9]+}}, d{{[0-9]+}}}, [r0:128]
-  ret <4 x i16> %tmp4
-}
-
-define <2 x i32> @vld2i32(i32* %A) nounwind {
-  %tmp0 = bitcast i32* %A to i8*
-  %tmp1 = call %struct.__neon_int32x2x2_t @llvm.arm.neon.vld2.v2i32(i8* %tmp0, i32 1)
-  %tmp2 = extractvalue %struct.__neon_int32x2x2_t %tmp1, 0
-  %tmp3 = extractvalue %struct.__neon_int32x2x2_t %tmp1, 1
-  %tmp4 = add <2 x i32> %tmp2, %tmp3
-; CHECK: bic      r0, r0, #-1073741824
-; CHECK: vld2.32   {d{{[0-9]+}}, d{{[0-9]+}}}, [r0]
-  ret <2 x i32> %tmp4
-}
-
-define <16 x i8> @vld2Qi8(i8* %A) nounwind {
-  %tmp1 = call %struct.__neon_int8x16x2_t @llvm.arm.neon.vld2.v16i8(i8* %A, i32 8)
-  %tmp2 = extractvalue %struct.__neon_int8x16x2_t %tmp1, 0
-  %tmp3 = extractvalue %struct.__neon_int8x16x2_t %tmp1, 1
-  %tmp4 = add <16 x i8> %tmp2, %tmp3
-; CHECK: bic      r0, r0, #-1073741824
-; CHECK: vld2.8   {d{{[0-9]+}}, d{{[0-9]+}}, d{{[0-9]+}}, d{{[0-9]+}}}, [r0:64]
-  ret <16 x i8> %tmp4
-}
-
-define <8 x i16> @vld2Qi16(i16* %A) nounwind {
-  %tmp0 = bitcast i16* %A to i8*
-  %tmp1 = call %struct.__neon_int16x8x2_t @llvm.arm.neon.vld2.v8i16(i8* %tmp0, i32 16)
-  %tmp2 = extractvalue %struct.__neon_int16x8x2_t %tmp1, 0
-  %tmp3 = extractvalue %struct.__neon_int16x8x2_t %tmp1, 1
-  %tmp4 = add <8 x i16> %tmp2, %tmp3
-; CHECK: bic      r0, r0, #-1073741824
-; CHECK: vld2.16   {d{{[0-9]+}}, d{{[0-9]+}}, d{{[0-9]+}}, d{{[0-9]+}}}, [r0:128]
-  ret <8 x i16> %tmp4
-}
-
-define <4 x i32> @vld2Qi32(i32* %A) nounwind {
-  %tmp0 = bitcast i32* %A to i8*
-  %tmp1 = call %struct.__neon_int32x4x2_t @llvm.arm.neon.vld2.v4i32(i8* %tmp0, i32 64)
-  %tmp2 = extractvalue %struct.__neon_int32x4x2_t %tmp1, 0
-  %tmp3 = extractvalue %struct.__neon_int32x4x2_t %tmp1, 1
-  %tmp4 = add <4 x i32> %tmp2, %tmp3
-; CHECK: bic      r0, r0, #-1073741824
-; CHECK: vld2.32   {d{{[0-9]+}}, d{{[0-9]+}}, d{{[0-9]+}}, d{{[0-9]+}}}, [r0:256]
-  ret <4 x i32> %tmp4
-}
-
-define <16 x i8> @vld2Qi8_update(i8** %ptr, i32 %inc) nounwind {
-  %A = load i8*, i8** %ptr
-  %tmp1 = call %struct.__neon_int8x16x2_t @llvm.arm.neon.vld2.v16i8(i8* %A, i32 16)
-  %tmp2 = extractvalue %struct.__neon_int8x16x2_t %tmp1, 0
-  %tmp3 = extractvalue %struct.__neon_int8x16x2_t %tmp1, 1
-  %tmp4 = add <16 x i8> %tmp2, %tmp3
-; CHECK: bic      r2, r2, #-1073741824
-; CHECK: vld2.8   {d{{[0-9]+}}, d{{[0-9]+}}, d{{[0-9]+}}, d{{[0-9]+}}}, [r2:128], r1
-  %tmp5 = getelementptr i8, i8* %A, i32 %inc
-  store i8* %tmp5, i8** %ptr
-  ret <16 x i8> %tmp4
-}
-
-define <2 x float> @vld2f_update(float** %ptr) nounwind {
-  %A = load float*, float** %ptr
-  %tmp0 = bitcast float* %A to i8*
-  %tmp1 = call %struct.__neon_float32x2x2_t @llvm.arm.neon.vld2.v2f32(i8* %tmp0, i32 1)
-; CHECK: bic       r1, r1, #-1073741824
-; CHECK: vld2.32   {d{{[0-9]+}}, d{{[0-9]+}}}, [r1]!
-  %tmp2 = extractvalue %struct.__neon_float32x2x2_t %tmp1, 0
-  %tmp3 = extractvalue %struct.__neon_float32x2x2_t %tmp1, 1
-  %tmp4 = fadd <2 x float> %tmp2, %tmp3
-  %tmp5 = getelementptr float, float* %A, i32 4
-  store float* %tmp5, float** %ptr
-  ret <2 x float> %tmp4
-}
-
diff --git a/test/NaCl/ARM/neon-vld3-sandboxing.ll b/test/NaCl/ARM/neon-vld3-sandboxing.ll
deleted file mode 100644
index 200b04b8c1f..00000000000
--- a/test/NaCl/ARM/neon-vld3-sandboxing.ll
+++ /dev/null
@@ -1,106 +0,0 @@
-; RUN: pnacl-llc -mtriple=armv7-unknown-nacl -mattr=+neon -filetype=obj %s -o - \
-; RUN:  | llvm-objdump -disassemble -triple armv7 - | FileCheck %s
-
-%struct.__neon_int8x8x3_t = type { <8 x i8>,  <8 x i8>,  <8 x i8> }
-%struct.__neon_int16x4x3_t = type { <4 x i16>, <4 x i16>, <4 x i16> }
-%struct.__neon_int32x2x3_t = type { <2 x i32>, <2 x i32>, <2 x i32> }
-%struct.__neon_float32x2x3_t = type { <2 x float>, <2 x float>, <2 x float> }
-%struct.__neon_int64x1x3_t = type { <1 x i64>, <1 x i64>, <1 x i64> }
-
-%struct.__neon_int8x16x3_t = type { <16 x i8>,  <16 x i8>,  <16 x i8> }
-%struct.__neon_int16x8x3_t = type { <8 x i16>, <8 x i16>, <8 x i16> }
-%struct.__neon_int32x4x3_t = type { <4 x i32>, <4 x i32>, <4 x i32> }
-%struct.__neon_float32x4x3_t = type { <4 x float>, <4 x float>, <4 x float> }
-
-declare %struct.__neon_int8x8x3_t @llvm.arm.neon.vld3.v8i8(i8*, i32) nounwind readonly
-declare %struct.__neon_int16x4x3_t @llvm.arm.neon.vld3.v4i16(i8*, i32) nounwind readonly
-declare %struct.__neon_int32x2x3_t @llvm.arm.neon.vld3.v2i32(i8*, i32) nounwind readonly
-declare %struct.__neon_float32x2x3_t @llvm.arm.neon.vld3.v2f32(i8*, i32) nounwind readonly
-declare %struct.__neon_int64x1x3_t @llvm.arm.neon.vld3.v1i64(i8*, i32) nounwind readonly
-
-declare %struct.__neon_int8x16x3_t @llvm.arm.neon.vld3.v16i8(i8*, i32) nounwind readonly
-declare %struct.__neon_int16x8x3_t @llvm.arm.neon.vld3.v8i16(i8*, i32) nounwind readonly
-declare %struct.__neon_int32x4x3_t @llvm.arm.neon.vld3.v4i32(i8*, i32) nounwind readonly
-declare %struct.__neon_float32x4x3_t @llvm.arm.neon.vld3.v4f32(i8*, i32) nounwind readonly
-
-define <8 x i8> @vld3i8(i32 %foobar, i32 %ba, i8* %A) nounwind {
-  %tmp1 = call %struct.__neon_int8x8x3_t @llvm.arm.neon.vld3.v8i8(i8* %A, i32 32)
-  %tmp2 = extractvalue %struct.__neon_int8x8x3_t %tmp1, 0
-  %tmp3 = extractvalue %struct.__neon_int8x8x3_t %tmp1, 2
-  %tmp4 = add <8 x i8> %tmp2, %tmp3
-; CHECK:         bic r2, r2, #-1073741824
-; CHECK-NEXT:    vld3.8 {{{d[0-9]+}}, {{d[0-9]+}}, {{d[0-9]+}}}, [r2:64]
-  ret <8 x i8> %tmp4
-}
-
-define <4 x i16> @vld3i16(i16* %A) nounwind {
-  %tmp0 = bitcast i16* %A to i8*
-  %tmp1 = call %struct.__neon_int16x4x3_t @llvm.arm.neon.vld3.v4i16(i8* %tmp0, i32 1)
-  %tmp2 = extractvalue %struct.__neon_int16x4x3_t %tmp1, 0
-  %tmp3 = extractvalue %struct.__neon_int16x4x3_t %tmp1, 2
-  %tmp4 = add <4 x i16> %tmp2, %tmp3
-; CHECK:         bic r0, r0, #-1073741824
-; CHECK-NEXT:    vld3.16 {{{d[0-9]+}}, {{d[0-9]+}}, {{d[0-9]+}}}, [r0]
-  ret <4 x i16> %tmp4
-}
-
-define <2 x i32> @vld3i32(i32* %A) nounwind {
-  %tmp0 = bitcast i32* %A to i8*
-  %tmp1 = call %struct.__neon_int32x2x3_t @llvm.arm.neon.vld3.v2i32(i8* %tmp0, i32 1)
-  %tmp2 = extractvalue %struct.__neon_int32x2x3_t %tmp1, 0
-  %tmp3 = extractvalue %struct.__neon_int32x2x3_t %tmp1, 2
-  %tmp4 = add <2 x i32> %tmp2, %tmp3
-; CHECK:         bic r0, r0, #-1073741824
-; CHECK-NEXT:    vld3.32 {{{d[0-9]+}}, {{d[0-9]+}}, {{d[0-9]+}}}, [r0]
-  ret <2 x i32> %tmp4
-}
-
-define <1 x i64> @vld3i64(i64* %A) nounwind {
-  %tmp0 = bitcast i64* %A to i8*
-  %tmp1 = call %struct.__neon_int64x1x3_t @llvm.arm.neon.vld3.v1i64(i8* %tmp0, i32 16)
-  %tmp2 = extractvalue %struct.__neon_int64x1x3_t %tmp1, 0
-  %tmp3 = extractvalue %struct.__neon_int64x1x3_t %tmp1, 2
-  %tmp4 = add <1 x i64> %tmp2, %tmp3
-; CHECK:         bic r0, r0, #-1073741824
-; CHECK-NEXT:    vld1.64 {{{d[0-9]+}}, {{d[0-9]+}}, {{d[0-9]+}}}, [r0:64]
-  ret <1 x i64> %tmp4
-}
-
-define <16 x i8> @vld3Qi8(i8* %A) nounwind {
-  %tmp1 = call %struct.__neon_int8x16x3_t @llvm.arm.neon.vld3.v16i8(i8* %A, i32 32)
-  %tmp2 = extractvalue %struct.__neon_int8x16x3_t %tmp1, 0
-  %tmp3 = extractvalue %struct.__neon_int8x16x3_t %tmp1, 2
-  %tmp4 = add <16 x i8> %tmp2, %tmp3
-; CHECK:         bic r0, r0, #-1073741824
-; CHECK-NEXT:    vld3.8 {{{d[0-9]+}}, {{d[0-9]+}}, {{d[0-9]+}}}, [r0:64]!
-  ret <16 x i8> %tmp4
-}
-
-define <4 x i16> @vld3i16_update(i16** %ptr, i32 %inc) nounwind {
-  %A = load i16*, i16** %ptr
-  %tmp0 = bitcast i16* %A to i8*
-  %tmp1 = call %struct.__neon_int16x4x3_t @llvm.arm.neon.vld3.v4i16(i8* %tmp0, i32 1)
-; CHECK:         bic r2, r2, #-1073741824
-; CHECK-NEXT:    vld3.16  {{{d[0-9]+}}, {{d[0-9]+}}, {{d[0-9]+}}}, [r2], r1
-  %tmp2 = extractvalue %struct.__neon_int16x4x3_t %tmp1, 0
-  %tmp3 = extractvalue %struct.__neon_int16x4x3_t %tmp1, 2
-  %tmp4 = add <4 x i16> %tmp2, %tmp3
-  %tmp5 = getelementptr i16, i16* %A, i32 %inc
-  store i16* %tmp5, i16** %ptr
-  ret <4 x i16> %tmp4
-}
-
-define <4 x i32> @vld3Qi32_update(i32** %ptr) nounwind {
-  %A = load i32*, i32** %ptr
-  %tmp0 = bitcast i32* %A to i8*
-  %tmp1 = call %struct.__neon_int32x4x3_t @llvm.arm.neon.vld3.v4i32(i8* %tmp0, i32 1)
-; CHECK:         bic r1, r1, #-1073741824
-; CHECK-NEXT:    vld3.32  {{{d[0-9]+}}, {{d[0-9]+}}, {{d[0-9]+}}}, [r1]!
-  %tmp2 = extractvalue %struct.__neon_int32x4x3_t %tmp1, 0
-  %tmp3 = extractvalue %struct.__neon_int32x4x3_t %tmp1, 2
-  %tmp4 = add <4 x i32> %tmp2, %tmp3
-  %tmp5 = getelementptr i32, i32* %A, i32 12
-  store i32* %tmp5, i32** %ptr
-  ret <4 x i32> %tmp4
-}
-
diff --git a/test/NaCl/ARM/neon-vld4-sandboxing.ll b/test/NaCl/ARM/neon-vld4-sandboxing.ll
deleted file mode 100644
index 50576a6bf3a..00000000000
--- a/test/NaCl/ARM/neon-vld4-sandboxing.ll
+++ /dev/null
@@ -1,107 +0,0 @@
-; RUN: pnacl-llc -mtriple=armv7-unknown-nacl -mattr=+neon -filetype=obj %s -o - \
-; RUN:  | llvm-objdump -disassemble -triple armv7 - | FileCheck %s
-
-%struct.__neon_int8x8x4_t = type { <8 x i8>,  <8 x i8>,  <8 x i8>, <8 x i8> }
-%struct.__neon_int16x4x4_t = type { <4 x i16>, <4 x i16>, <4 x i16>, <4 x i16> }
-%struct.__neon_int32x2x4_t = type { <2 x i32>, <2 x i32>, <2 x i32>, <2 x i32> }
-%struct.__neon_float32x2x4_t = type { <2 x float>, <2 x float>, <2 x float>, <2 x float> }
-%struct.__neon_int64x1x4_t = type { <1 x i64>, <1 x i64>, <1 x i64>, <1 x i64> }
-
-%struct.__neon_int8x16x4_t = type { <16 x i8>,  <16 x i8>,  <16 x i8>, <16 x i8> }
-%struct.__neon_int16x8x4_t = type { <8 x i16>, <8 x i16>, <8 x i16>, <8 x i16> }
-%struct.__neon_int32x4x4_t = type { <4 x i32>, <4 x i32>, <4 x i32>, <4 x i32> }
-%struct.__neon_float32x4x4_t = type { <4 x float>, <4 x float>, <4 x float>, <4 x float> }
-
-declare %struct.__neon_int8x8x4_t @llvm.arm.neon.vld4.v8i8(i8*, i32) nounwind readonly
-declare %struct.__neon_int16x4x4_t @llvm.arm.neon.vld4.v4i16(i8*, i32) nounwind readonly
-declare %struct.__neon_int32x2x4_t @llvm.arm.neon.vld4.v2i32(i8*, i32) nounwind readonly
-declare %struct.__neon_float32x2x4_t @llvm.arm.neon.vld4.v2f32(i8*, i32) nounwind readonly
-declare %struct.__neon_int64x1x4_t @llvm.arm.neon.vld4.v1i64(i8*, i32) nounwind readonly
-
-declare %struct.__neon_int8x16x4_t @llvm.arm.neon.vld4.v16i8(i8*, i32) nounwind readonly
-declare %struct.__neon_int16x8x4_t @llvm.arm.neon.vld4.v8i16(i8*, i32) nounwind readonly
-declare %struct.__neon_int32x4x4_t @llvm.arm.neon.vld4.v4i32(i8*, i32) nounwind readonly
-declare %struct.__neon_float32x4x4_t @llvm.arm.neon.vld4.v4f32(i8*, i32) nounwind readonly
-
-define <8 x i8> @vld4i8(i8* %A) nounwind {
-  %tmp1 = call %struct.__neon_int8x8x4_t @llvm.arm.neon.vld4.v8i8(i8* %A, i32 8)
-  %tmp2 = extractvalue %struct.__neon_int8x8x4_t %tmp1, 0
-  %tmp3 = extractvalue %struct.__neon_int8x8x4_t %tmp1, 2
-  %tmp4 = add <8 x i8> %tmp2, %tmp3
-; CHECK:         bic r0, r0, #-1073741824
-; CHECK-NEXT:    vld4.8 {{{d[0-9]+}}, {{d[0-9]+}}, {{d[0-9]+}}, {{d[0-9]+}}}, [r0:64]
-  ret <8 x i8> %tmp4
-}
-
-define <4 x i16> @vld4i16(i16* %A) nounwind {
-  %tmp0 = bitcast i16* %A to i8*
-  %tmp1 = call %struct.__neon_int16x4x4_t @llvm.arm.neon.vld4.v4i16(i8* %tmp0, i32 16)
-  %tmp2 = extractvalue %struct.__neon_int16x4x4_t %tmp1, 0
-  %tmp3 = extractvalue %struct.__neon_int16x4x4_t %tmp1, 2
-  %tmp4 = add <4 x i16> %tmp2, %tmp3
-; CHECK:         bic r0, r0, #-1073741824
-; CHECK-NEXT:    vld4.16 {{{d[0-9]+}}, {{d[0-9]+}}, {{d[0-9]+}}, {{d[0-9]+}}}, [r0:128]
-  ret <4 x i16> %tmp4
-}
-
-define <2 x i32> @vld4i32(i32* %A) nounwind {
-  %tmp0 = bitcast i32* %A to i8*
-  %tmp1 = call %struct.__neon_int32x2x4_t @llvm.arm.neon.vld4.v2i32(i8* %tmp0, i32 32)
-  %tmp2 = extractvalue %struct.__neon_int32x2x4_t %tmp1, 0
-  %tmp3 = extractvalue %struct.__neon_int32x2x4_t %tmp1, 2
-  %tmp4 = add <2 x i32> %tmp2, %tmp3
-; CHECK:         bic r0, r0, #-1073741824
-; CHECK-NEXT:    vld4.32 {{{d[0-9]+}}, {{d[0-9]+}}, {{d[0-9]+}}, {{d[0-9]+}}}, [r0:256]
-  ret <2 x i32> %tmp4
-}
-
-define <1 x i64> @vld4i64(i64* %A) nounwind {
-  %tmp0 = bitcast i64* %A to i8*
-  %tmp1 = call %struct.__neon_int64x1x4_t @llvm.arm.neon.vld4.v1i64(i8* %tmp0, i32 64)
-  %tmp2 = extractvalue %struct.__neon_int64x1x4_t %tmp1, 0
-  %tmp3 = extractvalue %struct.__neon_int64x1x4_t %tmp1, 2
-  %tmp4 = add <1 x i64> %tmp2, %tmp3
-; CHECK:         bic r0, r0, #-1073741824
-; CHECK-NEXT:    vld1.64 {{{d[0-9]+}}, {{d[0-9]+}}, {{d[0-9]+}}, {{d[0-9]+}}}, [r0:256]
-  ret <1 x i64> %tmp4
-}
-
-define <16 x i8> @vld4Qi8(i8* %A) nounwind {
-  %tmp1 = call %struct.__neon_int8x16x4_t @llvm.arm.neon.vld4.v16i8(i8* %A, i32 64)
-  %tmp2 = extractvalue %struct.__neon_int8x16x4_t %tmp1, 0
-  %tmp3 = extractvalue %struct.__neon_int8x16x4_t %tmp1, 2
-  %tmp4 = add <16 x i8> %tmp2, %tmp3
-; CHECK:         bic r0, r0, #-1073741824
-; CHECK-NEXT:    vld4.8 {{{d[0-9]+}}, {{d[0-9]+}}, {{d[0-9]+}}, {{d[0-9]+}}}, [r0:256]!
-; CHECK:         bic r0, r0, #-1073741824
-; CHECK-NEXT:    vld4.8 {{{d[0-9]+}}, {{d[0-9]+}}, {{d[0-9]+}}, {{d[0-9]+}}}, [r0:256]
-  ret <16 x i8> %tmp4
-}
-
-define <8 x i8> @vld4i8_update(i8** %ptr, i32 %inc) nounwind {
-  %A = load i8*, i8** %ptr
-  %tmp1 = call %struct.__neon_int8x8x4_t @llvm.arm.neon.vld4.v8i8(i8* %A, i32 16)
-; CHECK:         bic r2, r2, #-1073741824
-; CHECK-NEXT:    vld4.8 {{{d[0-9]+}}, {{d[0-9]+}}, {{d[0-9]+}}, {{d[0-9]+}}}, [r2:128], r1
-  %tmp2 = extractvalue %struct.__neon_int8x8x4_t %tmp1, 0
-  %tmp3 = extractvalue %struct.__neon_int8x8x4_t %tmp1, 2
-  %tmp4 = add <8 x i8> %tmp2, %tmp3
-  %tmp5 = getelementptr i8, i8* %A, i32 %inc
-  store i8* %tmp5, i8** %ptr
-  ret <8 x i8> %tmp4
-}
-
-define <8 x i16> @vld4Qi16_update(i16** %ptr) nounwind {
-  %A = load i16*, i16** %ptr
-  %tmp0 = bitcast i16* %A to i8*
-  %tmp1 = call %struct.__neon_int16x8x4_t @llvm.arm.neon.vld4.v8i16(i8* %tmp0, i32 8)
-; CHECK:         bic r1, r1, #-1073741824
-; CHECK-NEXT:    vld4.16 {{{d[0-9]+}}, {{d[0-9]+}}, {{d[0-9]+}}, {{d[0-9]+}}}, [r1:64]!
-  %tmp2 = extractvalue %struct.__neon_int16x8x4_t %tmp1, 0
-  %tmp3 = extractvalue %struct.__neon_int16x8x4_t %tmp1, 2
-  %tmp4 = add <8 x i16> %tmp2, %tmp3
-  %tmp5 = getelementptr i16, i16* %A, i32 32
-  store i16* %tmp5, i16** %ptr
-  ret <8 x i16> %tmp4
-}
-
diff --git a/test/NaCl/ARM/neon-vlddup-sandboxing.ll b/test/NaCl/ARM/neon-vlddup-sandboxing.ll
deleted file mode 100644
index 7744215a02a..00000000000
--- a/test/NaCl/ARM/neon-vlddup-sandboxing.ll
+++ /dev/null
@@ -1,151 +0,0 @@
-; RUN: pnacl-llc -mtriple=armv7-unknown-nacl -mattr=+neon -filetype=obj %s -o - \
-; RUN:  | llvm-objdump -disassemble -triple armv7 - | FileCheck %s
-
-%struct.__neon_int8x8x2_t = type { <8 x i8>, <8 x i8> }
-%struct.__neon_int4x16x2_t = type { <4 x i16>, <4 x i16> }
-%struct.__neon_int2x32x2_t = type { <2 x i32>, <2 x i32> }
-
-declare %struct.__neon_int8x8x2_t @llvm.arm.neon.vld2lane.v8i8(i8*, <8 x i8>, <8 x i8>, i32, i32) nounwind readonly
-declare %struct.__neon_int4x16x2_t @llvm.arm.neon.vld2lane.v4i16(i8*, <4 x i16>, <4 x i16>, i32, i32) nounwind readonly
-declare %struct.__neon_int2x32x2_t @llvm.arm.neon.vld2lane.v2i32(i8*, <2 x i32>, <2 x i32>, i32, i32) nounwind readonly
-
-%struct.__neon_int8x8x3_t = type { <8 x i8>, <8 x i8>, <8 x i8> }
-%struct.__neon_int16x4x3_t = type { <4 x i16>, <4 x i16>, <4 x i16> }
-
-declare %struct.__neon_int8x8x3_t @llvm.arm.neon.vld3lane.v8i8(i8*, <8 x i8>, <8 x i8>, <8 x i8>, i32, i32) nounwind readonly
-declare %struct.__neon_int16x4x3_t @llvm.arm.neon.vld3lane.v4i16(i8*, <4 x i16>, <4 x i16>, <4 x i16>, i32, i32) nounwind readonly
-
-%struct.__neon_int16x4x4_t = type { <4 x i16>, <4 x i16>, <4 x i16>, <4 x i16> }
-%struct.__neon_int32x2x4_t = type { <2 x i32>, <2 x i32>, <2 x i32>, <2 x i32> }
-
-declare %struct.__neon_int16x4x4_t @llvm.arm.neon.vld4lane.v4i16(i8*, <4 x i16>, <4 x i16>, <4 x i16>, <4 x i16>, i32, i32) nounwind readonly
-declare %struct.__neon_int32x2x4_t @llvm.arm.neon.vld4lane.v2i32(i8*, <2 x i32>, <2 x i32>, <2 x i32>, <2 x i32>, i32, i32) nounwind readonly
-
-define <8 x i8> @vld1dupi8(i32 %foo, i32 %bar,
-                           i8* %A) nounwind {
-  %tmp1 = load i8, i8* %A, align 8
-  %tmp2 = insertelement <8 x i8> undef, i8 %tmp1, i32 0
-  %tmp3 = shufflevector <8 x i8> %tmp2, <8 x i8> undef, <8 x i32> zeroinitializer
-; CHECK:         bic r2, r2, #-1073741824
-; CHECK-NEXT:    vld1.8 {{{d[0-9]+\[\]}}}, [r2]
-  ret <8 x i8> %tmp3
-}
-
-define <4 x i16> @vld1dupi16(i16* %A) nounwind {
-  %tmp1 = load i16, i16* %A, align 8
-  %tmp2 = insertelement <4 x i16> undef, i16 %tmp1, i32 0
-  %tmp3 = shufflevector <4 x i16> %tmp2, <4 x i16> undef, <4 x i32> zeroinitializer
-; CHECK:         bic r0, r0, #-1073741824
-; CHECK-NEXT:    vld1.16 {{{d[0-9]+\[\]}}}, [r0:16]
-  ret <4 x i16> %tmp3
-}
-
-define <2 x i32> @vld1dupi32(i32* %A) nounwind {
-  %tmp1 = load i32, i32* %A, align 8
-  %tmp2 = insertelement <2 x i32> undef, i32 %tmp1, i32 0
-  %tmp3 = shufflevector <2 x i32> %tmp2, <2 x i32> undef, <2 x i32> zeroinitializer
-; CHECK:         bic r0, r0, #-1073741824
-; CHECK-NEXT:    vld1.32 {{{d[0-9]+\[\]}}}, [r0:32]
-  ret <2 x i32> %tmp3
-}
-
-define <16 x i8> @vld1dupQi8(i8* %A) nounwind {
-  %tmp1 = load i8, i8* %A, align 8
-  %tmp2 = insertelement <16 x i8> undef, i8 %tmp1, i32 0
-  %tmp3 = shufflevector <16 x i8> %tmp2, <16 x i8> undef, <16 x i32> zeroinitializer
-; CHECK:         bic r0, r0, #-1073741824
-; CHECK-NEXT:    vld1.8 {{{d[0-9]+\[\]}}, {{d[0-9]+\[\]}}}, [r0]
-  ret <16 x i8> %tmp3
-}
-
-define <8 x i8> @vld2dupi8(i8* %A) nounwind {
-  %tmp0 = tail call %struct.__neon_int8x8x2_t @llvm.arm.neon.vld2lane.v8i8(i8* %A, <8 x i8> undef, <8 x i8> undef, i32 0, i32 1)
-; CHECK:         bic r0, r0, #-1073741824
-; CHECK-NEXT:    vld2.8 {{{d[0-9]+\[\]}}, {{d[0-9]+\[\]}}}, [r0]
-  %tmp1 = extractvalue %struct.__neon_int8x8x2_t %tmp0, 0
-  %tmp2 = shufflevector <8 x i8> %tmp1, <8 x i8> undef, <8 x i32> zeroinitializer
-  %tmp3 = extractvalue %struct.__neon_int8x8x2_t %tmp0, 1
-  %tmp4 = shufflevector <8 x i8> %tmp3, <8 x i8> undef, <8 x i32> zeroinitializer
-  %tmp5 = add <8 x i8> %tmp2, %tmp4
-  ret <8 x i8> %tmp5
-}
-
-define <4 x i16> @vld2dupi16(i8* %A) nounwind {
-  %tmp0 = tail call %struct.__neon_int4x16x2_t @llvm.arm.neon.vld2lane.v4i16(i8* %A, <4 x i16> undef, <4 x i16> undef, i32 0, i32 2)
-; CHECK:         bic r0, r0, #-1073741824
-; CHECK-NEXT:    vld2.16 {{{d[0-9]+\[\]}}, {{d[0-9]+\[\]}}}, [r0]
-  %tmp1 = extractvalue %struct.__neon_int4x16x2_t %tmp0, 0
-  %tmp2 = shufflevector <4 x i16> %tmp1, <4 x i16> undef, <4 x i32> zeroinitializer
-  %tmp3 = extractvalue %struct.__neon_int4x16x2_t %tmp0, 1
-  %tmp4 = shufflevector <4 x i16> %tmp3, <4 x i16> undef, <4 x i32> zeroinitializer
-  %tmp5 = add <4 x i16> %tmp2, %tmp4
-  ret <4 x i16> %tmp5
-}
-
-define <2 x i32> @vld2dupi32(i8* %A) nounwind {
-  %tmp0 = tail call %struct.__neon_int2x32x2_t @llvm.arm.neon.vld2lane.v2i32(i8* %A, <2 x i32> undef, <2 x i32> undef, i32 0, i32 16)
-; CHECK:         bic r0, r0, #-1073741824
-; CHECK-NEXT:    vld2.32 {{{d[0-9]+\[\]}}, {{d[0-9]+\[\]}}}, [r0:64]
-  %tmp1 = extractvalue %struct.__neon_int2x32x2_t %tmp0, 0
-  %tmp2 = shufflevector <2 x i32> %tmp1, <2 x i32> undef, <2 x i32> zeroinitializer
-  %tmp3 = extractvalue %struct.__neon_int2x32x2_t %tmp0, 1
-  %tmp4 = shufflevector <2 x i32> %tmp3, <2 x i32> undef, <2 x i32> zeroinitializer
-  %tmp5 = add <2 x i32> %tmp2, %tmp4
-  ret <2 x i32> %tmp5
-}
-
-define <4 x i16> @vld3dupi16(i8* %A) nounwind {
-  %tmp0 = tail call %struct.__neon_int16x4x3_t @llvm.arm.neon.vld3lane.v4i16(i8* %A, <4 x i16> undef, <4 x i16> undef, <4 x i16> undef, i32 0, i32 8)
-; CHECK:         bic r0, r0, #-1073741824
-; CHECK-NEXT:    vld3.16 {{{d[0-9]+\[\]}}, {{d[0-9]+\[\]}}, {{d[0-9]+\[\]}}}, [r0]
-  %tmp1 = extractvalue %struct.__neon_int16x4x3_t %tmp0, 0
-  %tmp2 = shufflevector <4 x i16> %tmp1, <4 x i16> undef, <4 x i32> zeroinitializer
-  %tmp3 = extractvalue %struct.__neon_int16x4x3_t %tmp0, 1
-  %tmp4 = shufflevector <4 x i16> %tmp3, <4 x i16> undef, <4 x i32> zeroinitializer
-  %tmp5 = extractvalue %struct.__neon_int16x4x3_t %tmp0, 2
-  %tmp6 = shufflevector <4 x i16> %tmp5, <4 x i16> undef, <4 x i32> zeroinitializer
-  %tmp7 = add <4 x i16> %tmp2, %tmp4
-  %tmp8 = add <4 x i16> %tmp7, %tmp6
-  ret <4 x i16> %tmp8
-}
-
-define <2 x i32> @vld4dupi32(i8* %A) nounwind {
-  %tmp0 = tail call %struct.__neon_int32x2x4_t @llvm.arm.neon.vld4lane.v2i32(i8* %A, <2 x i32> undef, <2 x i32> undef, <2 x i32> undef, <2 x i32> undef, i32 0, i32 8)
-; CHECK:         bic r0, r0, #-1073741824
-; CHECK-NEXT:    vld4.32 {{{d[0-9]+\[\]}}, {{d[0-9]+\[\]}}, {{d[0-9]+\[\]}}, {{d[0-9]+\[\]}}}, [r0:64]
-  %tmp1 = extractvalue %struct.__neon_int32x2x4_t %tmp0, 0
-  %tmp2 = shufflevector <2 x i32> %tmp1, <2 x i32> undef, <2 x i32> zeroinitializer
-  %tmp3 = extractvalue %struct.__neon_int32x2x4_t %tmp0, 1
-  %tmp4 = shufflevector <2 x i32> %tmp3, <2 x i32> undef, <2 x i32> zeroinitializer
-  %tmp5 = extractvalue %struct.__neon_int32x2x4_t %tmp0, 2
-  %tmp6 = shufflevector <2 x i32> %tmp5, <2 x i32> undef, <2 x i32> zeroinitializer
-  %tmp7 = extractvalue %struct.__neon_int32x2x4_t %tmp0, 3
-  %tmp8 = shufflevector <2 x i32> %tmp7, <2 x i32> undef, <2 x i32> zeroinitializer
-  %tmp9 = add <2 x i32> %tmp2, %tmp4
-  %tmp10 = add <2 x i32> %tmp6, %tmp8
-  %tmp11 = add <2 x i32> %tmp9, %tmp10
-  ret <2 x i32> %tmp11
-}
-
-;Check for a post-increment updating load.
-define <4 x i16> @vld4dupi16_update(i16** %ptr) nounwind {
-  %A = load i16*, i16** %ptr
-  %A2 = bitcast i16* %A to i8*
-  %tmp0 = tail call %struct.__neon_int16x4x4_t @llvm.arm.neon.vld4lane.v4i16(i8* %A2, <4 x i16> undef, <4 x i16> undef, <4 x i16> undef, <4 x i16> undef, i32 0, i32 1)
-; CHECK:         bic r1, r1, #-1073741824
-; CHECK-NEXT:    vld4.16 {{{d[0-9]+\[\]}}, {{d[0-9]+\[\]}}, {{d[0-9]+\[\]}}, {{d[0-9]+\[\]}}}, [r1]!
-  %tmp1 = extractvalue %struct.__neon_int16x4x4_t %tmp0, 0
-  %tmp2 = shufflevector <4 x i16> %tmp1, <4 x i16> undef, <4 x i32> zeroinitializer
-  %tmp3 = extractvalue %struct.__neon_int16x4x4_t %tmp0, 1
-  %tmp4 = shufflevector <4 x i16> %tmp3, <4 x i16> undef, <4 x i32> zeroinitializer
-  %tmp5 = extractvalue %struct.__neon_int16x4x4_t %tmp0, 2
-  %tmp6 = shufflevector <4 x i16> %tmp5, <4 x i16> undef, <4 x i32> zeroinitializer
-  %tmp7 = extractvalue %struct.__neon_int16x4x4_t %tmp0, 3
-  %tmp8 = shufflevector <4 x i16> %tmp7, <4 x i16> undef, <4 x i32> zeroinitializer
-  %tmp9 = add <4 x i16> %tmp2, %tmp4
-  %tmp10 = add <4 x i16> %tmp6, %tmp8
-  %tmp11 = add <4 x i16> %tmp9, %tmp10
-  %tmp12 = getelementptr i16, i16* %A, i32 4
-  store i16* %tmp12, i16** %ptr
-  ret <4 x i16> %tmp11
-}
diff --git a/test/NaCl/ARM/neon-vldlane-sandboxing.ll b/test/NaCl/ARM/neon-vldlane-sandboxing.ll
deleted file mode 100644
index 0f2237c70b8..00000000000
--- a/test/NaCl/ARM/neon-vldlane-sandboxing.ll
+++ /dev/null
@@ -1,319 +0,0 @@
-; RUN: pnacl-llc -mtriple=armv7-unknown-nacl -mattr=+neon -filetype=obj %s -o - \
-; RUN:  | llvm-objdump -disassemble -triple armv7 - | FileCheck %s
-
-%struct.__neon_int8x8x2_t = type { <8 x i8>,  <8 x i8> }
-%struct.__neon_int16x4x2_t = type { <4 x i16>, <4 x i16> }
-%struct.__neon_int32x2x2_t = type { <2 x i32>, <2 x i32> }
-%struct.__neon_float32x2x2_t = type { <2 x float>, <2 x float> }
-
-%struct.__neon_int16x8x2_t = type { <8 x i16>, <8 x i16> }
-%struct.__neon_int32x4x2_t = type { <4 x i32>, <4 x i32> }
-%struct.__neon_float32x4x2_t = type { <4 x float>, <4 x float> }
-
-declare %struct.__neon_int8x8x2_t @llvm.arm.neon.vld2lane.v8i8(i8*, <8 x i8>, <8 x i8>, i32, i32) nounwind readonly
-declare %struct.__neon_int16x4x2_t @llvm.arm.neon.vld2lane.v4i16(i8*, <4 x i16>, <4 x i16>, i32, i32) nounwind readonly
-declare %struct.__neon_int32x2x2_t @llvm.arm.neon.vld2lane.v2i32(i8*, <2 x i32>, <2 x i32>, i32, i32) nounwind readonly
-declare %struct.__neon_float32x2x2_t @llvm.arm.neon.vld2lane.v2f32(i8*, <2 x float>, <2 x float>, i32, i32) nounwind readonly
-
-declare %struct.__neon_int16x8x2_t @llvm.arm.neon.vld2lane.v8i16(i8*, <8 x i16>, <8 x i16>, i32, i32) nounwind readonly
-declare %struct.__neon_int32x4x2_t @llvm.arm.neon.vld2lane.v4i32(i8*, <4 x i32>, <4 x i32>, i32, i32) nounwind readonly
-declare %struct.__neon_float32x4x2_t @llvm.arm.neon.vld2lane.v4f32(i8*, <4 x float>, <4 x float>, i32, i32) nounwind readonly
-
-%struct.__neon_int8x8x3_t = type { <8 x i8>,  <8 x i8>,  <8 x i8> }
-%struct.__neon_int16x4x3_t = type { <4 x i16>, <4 x i16>, <4 x i16> }
-%struct.__neon_int32x2x3_t = type { <2 x i32>, <2 x i32>, <2 x i32> }
-%struct.__neon_float32x2x3_t = type { <2 x float>, <2 x float>, <2 x float> }
-
-%struct.__neon_int16x8x3_t = type { <8 x i16>, <8 x i16>, <8 x i16> }
-%struct.__neon_int32x4x3_t = type { <4 x i32>, <4 x i32>, <4 x i32> }
-%struct.__neon_float32x4x3_t = type { <4 x float>, <4 x float>, <4 x float> }
-
-declare %struct.__neon_int8x8x3_t @llvm.arm.neon.vld3lane.v8i8(i8*, <8 x i8>, <8 x i8>, <8 x i8>, i32, i32) nounwind readonly
-declare %struct.__neon_int16x4x3_t @llvm.arm.neon.vld3lane.v4i16(i8*, <4 x i16>, <4 x i16>, <4 x i16>, i32, i32) nounwind readonly
-declare %struct.__neon_int32x2x3_t @llvm.arm.neon.vld3lane.v2i32(i8*, <2 x i32>, <2 x i32>, <2 x i32>, i32, i32) nounwind readonly
-declare %struct.__neon_float32x2x3_t @llvm.arm.neon.vld3lane.v2f32(i8*, <2 x float>, <2 x float>, <2 x float>, i32, i32) nounwind readonly
-
-declare %struct.__neon_int16x8x3_t @llvm.arm.neon.vld3lane.v8i16(i8*, <8 x i16>, <8 x i16>, <8 x i16>, i32, i32) nounwind readonly
-declare %struct.__neon_int32x4x3_t @llvm.arm.neon.vld3lane.v4i32(i8*, <4 x i32>, <4 x i32>, <4 x i32>, i32, i32) nounwind readonly
-declare %struct.__neon_float32x4x3_t @llvm.arm.neon.vld3lane.v4f32(i8*, <4 x float>, <4 x float>, <4 x float>, i32, i32) nounwind readonly
-
-%struct.__neon_int8x8x4_t = type { <8 x i8>,  <8 x i8>,  <8 x i8>,  <8 x i8> }
-%struct.__neon_int16x4x4_t = type { <4 x i16>, <4 x i16>, <4 x i16>, <4 x i16> }
-%struct.__neon_int32x2x4_t = type { <2 x i32>, <2 x i32>, <2 x i32>, <2 x i32> }
-%struct.__neon_float32x2x4_t = type { <2 x float>, <2 x float>, <2 x float>, <2 x float> }
-
-%struct.__neon_int16x8x4_t = type { <8 x i16>, <8 x i16>, <8 x i16>, <8 x i16> }
-%struct.__neon_int32x4x4_t = type { <4 x i32>, <4 x i32>, <4 x i32>, <4 x i32> }
-%struct.__neon_float32x4x4_t = type { <4 x float>, <4 x float>, <4 x float>, <4 x float> }
-
-declare %struct.__neon_int8x8x4_t @llvm.arm.neon.vld4lane.v8i8(i8*, <8 x i8>, <8 x i8>, <8 x i8>, <8 x i8>, i32, i32) nounwind readonly
-declare %struct.__neon_int16x4x4_t @llvm.arm.neon.vld4lane.v4i16(i8*, <4 x i16>, <4 x i16>, <4 x i16>, <4 x i16>, i32, i32) nounwind readonly
-declare %struct.__neon_int32x2x4_t @llvm.arm.neon.vld4lane.v2i32(i8*, <2 x i32>, <2 x i32>, <2 x i32>, <2 x i32>, i32, i32) nounwind readonly
-declare %struct.__neon_float32x2x4_t @llvm.arm.neon.vld4lane.v2f32(i8*, <2 x float>, <2 x float>, <2 x float>, <2 x float>, i32, i32) nounwind readonly
-
-declare %struct.__neon_int16x8x4_t @llvm.arm.neon.vld4lane.v8i16(i8*, <8 x i16>, <8 x i16>, <8 x i16>, <8 x i16>, i32, i32) nounwind readonly
-declare %struct.__neon_int32x4x4_t @llvm.arm.neon.vld4lane.v4i32(i8*, <4 x i32>, <4 x i32>, <4 x i32>, <4 x i32>, i32, i32) nounwind readonly
-declare %struct.__neon_float32x4x4_t @llvm.arm.neon.vld4lane.v4f32(i8*, <4 x float>, <4 x float>, <4 x float>, <4 x float>, i32, i32) nounwind readonly
-
-define <8 x i8> @vld1lanei8(i8* %A, <8 x i8>* %B) nounwind {
-  %tmp1 = load <8 x i8>, <8 x i8>* %B
-  %tmp2 = load i8, i8* %A, align 8
-  %tmp3 = insertelement <8 x i8> %tmp1, i8 %tmp2, i32 3
-; CHECK:         bic r0, r0, #-1073741824
-; CHECK-NEXT:    vld1.8 {{{d[0-9]+\[[0-9]\]}}}, [r0]
-  ret <8 x i8> %tmp3
-}
-
-define <4 x i16> @vld1lanei16(i16* %A, <4 x i16>* %B) nounwind {
-  %tmp1 = load <4 x i16>, <4 x i16>* %B
-  %tmp2 = load i16, i16* %A, align 8
-  %tmp3 = insertelement <4 x i16> %tmp1, i16 %tmp2, i32 2
-; CHECK:         bic r0, r0, #-1073741824
-; CHECK-NEXT:    vld1.16 {{{d[0-9]+\[[0-9]\]}}}, [r0:16]
-  ret <4 x i16> %tmp3
-}
-
-define <2 x i32> @vld1lanei32(i32* %A, <2 x i32>* %B) nounwind {
-  %tmp1 = load <2 x i32>, <2 x i32>* %B
-  %tmp2 = load i32, i32* %A, align 8
-  %tmp3 = insertelement <2 x i32> %tmp1, i32 %tmp2, i32 1
-; CHECK:         bic r0, r0, #-1073741824
-; CHECK-NEXT:    vld1.32 {{{d[0-9]+\[[0-9]\]}}}, [r0:32]
-  ret <2 x i32> %tmp3
-}
-
-define <16 x i8> @vld1laneQi8(i8* %A, <16 x i8>* %B) nounwind {
-  %tmp1 = load <16 x i8>, <16 x i8>* %B
-  %tmp2 = load i8, i8* %A, align 8
-  %tmp3 = insertelement <16 x i8> %tmp1, i8 %tmp2, i32 9
-; CHECK:         bic r0, r0, #-1073741824
-; CHECK-NEXT:    vld1.8 {{{d[0-9]+\[[0-9]\]}}}, [r0]
-  ret <16 x i8> %tmp3
-}
-
-define <8 x i16> @vld1laneQi16(i16* %A, <8 x i16>* %B) nounwind {
-  %tmp1 = load <8 x i16>, <8 x i16>* %B
-  %tmp2 = load i16, i16* %A, align 8
-  %tmp3 = insertelement <8 x i16> %tmp1, i16 %tmp2, i32 5
-; CHECK:         bic r0, r0, #-1073741824
-; CHECK-NEXT:    vld1.16 {{{d[0-9]+\[[0-9]\]}}}, [r0:16]
-  ret <8 x i16> %tmp3
-}
-
-define <4 x i32> @vld1laneQi32(i32* %A, <4 x i32>* %B) nounwind {
-  %tmp1 = load <4 x i32>, <4 x i32>* %B
-  %tmp2 = load i32, i32* %A, align 8
-  %tmp3 = insertelement <4 x i32> %tmp1, i32 %tmp2, i32 3
-; CHECK:         bic r0, r0, #-1073741824
-; CHECK-NEXT:    vld1.32 {{{d[0-9]+\[[0-9]\]}}}, [r0:32]
-  ret <4 x i32> %tmp3
-}
-
-define <8 x i8> @vld2lanei8(i8* %A, <8 x i8>* %B) nounwind {
-  %tmp1 = load <8 x i8>, <8 x i8>* %B
-  %tmp2 = call %struct.__neon_int8x8x2_t @llvm.arm.neon.vld2lane.v8i8(i8* %A, <8 x i8> %tmp1, <8 x i8> %tmp1, i32 1, i32 4)
-  %tmp3 = extractvalue %struct.__neon_int8x8x2_t %tmp2, 0
-  %tmp4 = extractvalue %struct.__neon_int8x8x2_t %tmp2, 1
-  %tmp5 = add <8 x i8> %tmp3, %tmp4
-; CHECK:         bic r0, r0, #-1073741824
-; CHECK-NEXT:    vld2.8 {{{d[0-9]+\[[0-9]\]}}, {{d[0-9]+\[[0-9]\]}}}, [r0:16]
-  ret <8 x i8> %tmp5
-}
-
-define <4 x i16> @vld2lanei16(i16* %A, <4 x i16>* %B) nounwind {
-  %tmp0 = bitcast i16* %A to i8*
-  %tmp1 = load <4 x i16>, <4 x i16>* %B
-  %tmp2 = call %struct.__neon_int16x4x2_t @llvm.arm.neon.vld2lane.v4i16(i8* %tmp0, <4 x i16> %tmp1, <4 x i16> %tmp1, i32 1, i32 8)
-  %tmp3 = extractvalue %struct.__neon_int16x4x2_t %tmp2, 0
-  %tmp4 = extractvalue %struct.__neon_int16x4x2_t %tmp2, 1
-  %tmp5 = add <4 x i16> %tmp3, %tmp4
-; CHECK:         bic r0, r0, #-1073741824
-; CHECK-NEXT:    vld2.16 {{{d[0-9]+\[[0-9]\]}}, {{d[0-9]+\[[0-9]\]}}}, [r0:32]
-  ret <4 x i16> %tmp5
-}
-
-define <2 x i32> @vld2lanei32(i32 %foo, i32 %bar, i32 %baz,
-                              i32* %A, <2 x i32>* %B) nounwind {
-  %tmp0 = bitcast i32* %A to i8*
-  %tmp1 = load <2 x i32>, <2 x i32>* %B
-  %tmp2 = call %struct.__neon_int32x2x2_t @llvm.arm.neon.vld2lane.v2i32(i8* %tmp0, <2 x i32> %tmp1, <2 x i32> %tmp1, i32 1, i32 1)
-  %tmp3 = extractvalue %struct.__neon_int32x2x2_t %tmp2, 0
-  %tmp4 = extractvalue %struct.__neon_int32x2x2_t %tmp2, 1
-  %tmp5 = add <2 x i32> %tmp3, %tmp4
-; CHECK:         bic r3, r3, #-1073741824
-; CHECK-NEXT:    vld2.32 {{{d[0-9]+\[[0-9]\]}}, {{d[0-9]+\[[0-9]\]}}}, [r3]
-  ret <2 x i32> %tmp5
-}
-
-define <8 x i16> @vld2laneQi16(i16* %A, <8 x i16>* %B) nounwind {
-  %tmp0 = bitcast i16* %A to i8*
-  %tmp1 = load <8 x i16>, <8 x i16>* %B
-  %tmp2 = call %struct.__neon_int16x8x2_t @llvm.arm.neon.vld2lane.v8i16(i8* %tmp0, <8 x i16> %tmp1, <8 x i16> %tmp1, i32 5, i32 1)
-  %tmp3 = extractvalue %struct.__neon_int16x8x2_t %tmp2, 0
-  %tmp4 = extractvalue %struct.__neon_int16x8x2_t %tmp2, 1
-  %tmp5 = add <8 x i16> %tmp3, %tmp4
-; CHECK:         bic r0, r0, #-1073741824
-; CHECK-NEXT:    vld2.16 {{{d[0-9]+\[[0-9]\]}}, {{d[0-9]+\[[0-9]\]}}}, [r0]
-  ret <8 x i16> %tmp5
-}
-
-define <4 x i32> @vld2laneQi32(i32* %A, <4 x i32>* %B) nounwind {
-  %tmp0 = bitcast i32* %A to i8*
-  %tmp1 = load <4 x i32>, <4 x i32>* %B
-  %tmp2 = call %struct.__neon_int32x4x2_t @llvm.arm.neon.vld2lane.v4i32(i8* %tmp0, <4 x i32> %tmp1, <4 x i32> %tmp1, i32 2, i32 16)
-  %tmp3 = extractvalue %struct.__neon_int32x4x2_t %tmp2, 0
-  %tmp4 = extractvalue %struct.__neon_int32x4x2_t %tmp2, 1
-  %tmp5 = add <4 x i32> %tmp3, %tmp4
-; CHECK:         bic r0, r0, #-1073741824
-; CHECK-NEXT:    vld2.32 {{{d[0-9]+\[[0-9]\]}}, {{d[0-9]+\[[0-9]\]}}}, [r0:64]
-  ret <4 x i32> %tmp5
-}
-
-define <8 x i8> @vld3lanei8(i8* %A, <8 x i8>* %B) nounwind {
-  %tmp1 = load <8 x i8>, <8 x i8>* %B
-  %tmp2 = call %struct.__neon_int8x8x3_t @llvm.arm.neon.vld3lane.v8i8(i8* %A, <8 x i8> %tmp1, <8 x i8> %tmp1, <8 x i8> %tmp1, i32 1, i32 1)
-  %tmp3 = extractvalue %struct.__neon_int8x8x3_t %tmp2, 0
-  %tmp4 = extractvalue %struct.__neon_int8x8x3_t %tmp2, 1
-  %tmp5 = extractvalue %struct.__neon_int8x8x3_t %tmp2, 2
-  %tmp6 = add <8 x i8> %tmp3, %tmp4
-  %tmp7 = add <8 x i8> %tmp5, %tmp6
-; CHECK:         bic r0, r0, #-1073741824
-; CHECK-NEXT:    vld3.8 {{{d[0-9]+\[[0-9]\]}}, {{d[0-9]+\[[0-9]\]}}, {{d[0-9]+\[[0-9]\]}}}, [r0]
-  ret <8 x i8> %tmp7
-}
-
-define <4 x i16> @vld3lanei16(i16* %A, <4 x i16>* %B) nounwind {
-  %tmp0 = bitcast i16* %A to i8*
-  %tmp1 = load <4 x i16>, <4 x i16>* %B
-  %tmp2 = call %struct.__neon_int16x4x3_t @llvm.arm.neon.vld3lane.v4i16(i8* %tmp0, <4 x i16> %tmp1, <4 x i16> %tmp1, <4 x i16> %tmp1, i32 1, i32 8)
-  %tmp3 = extractvalue %struct.__neon_int16x4x3_t %tmp2, 0
-  %tmp4 = extractvalue %struct.__neon_int16x4x3_t %tmp2, 1
-  %tmp5 = extractvalue %struct.__neon_int16x4x3_t %tmp2, 2
-  %tmp6 = add <4 x i16> %tmp3, %tmp4
-  %tmp7 = add <4 x i16> %tmp5, %tmp6
-; CHECK:         bic r0, r0, #-1073741824
-; CHECK-NEXT:    vld3.16 {{{d[0-9]+\[[0-9]\]}}, {{d[0-9]+\[[0-9]\]}}, {{d[0-9]+\[[0-9]\]}}}, [r0]
-  ret <4 x i16> %tmp7
-}
-
-define <2 x i32> @vld3lanei32(i32* %A, <2 x i32>* %B) nounwind {
-  %tmp0 = bitcast i32* %A to i8*
-  %tmp1 = load <2 x i32>, <2 x i32>* %B
-  %tmp2 = call %struct.__neon_int32x2x3_t @llvm.arm.neon.vld3lane.v2i32(i8* %tmp0, <2 x i32> %tmp1, <2 x i32> %tmp1, <2 x i32> %tmp1, i32 1, i32 1)
-  %tmp3 = extractvalue %struct.__neon_int32x2x3_t %tmp2, 0
-  %tmp4 = extractvalue %struct.__neon_int32x2x3_t %tmp2, 1
-  %tmp5 = extractvalue %struct.__neon_int32x2x3_t %tmp2, 2
-  %tmp6 = add <2 x i32> %tmp3, %tmp4
-  %tmp7 = add <2 x i32> %tmp5, %tmp6
-; CHECK:         bic r0, r0, #-1073741824
-; CHECK-NEXT:    vld3.32 {{{d[0-9]+\[[0-9]\]}}, {{d[0-9]+\[[0-9]\]}}, {{d[0-9]+\[[0-9]\]}}}, [r0]
-  ret <2 x i32> %tmp7
-}
-
-define <8 x i16> @vld3laneQi16(i16* %A, <8 x i16>* %B) nounwind {
-  %tmp0 = bitcast i16* %A to i8*
-  %tmp1 = load <8 x i16>, <8 x i16>* %B
-  %tmp2 = call %struct.__neon_int16x8x3_t @llvm.arm.neon.vld3lane.v8i16(i8* %tmp0, <8 x i16> %tmp1, <8 x i16> %tmp1, <8 x i16> %tmp1, i32 1, i32 8)
-  %tmp3 = extractvalue %struct.__neon_int16x8x3_t %tmp2, 0
-  %tmp4 = extractvalue %struct.__neon_int16x8x3_t %tmp2, 1
-  %tmp5 = extractvalue %struct.__neon_int16x8x3_t %tmp2, 2
-  %tmp6 = add <8 x i16> %tmp3, %tmp4
-  %tmp7 = add <8 x i16> %tmp5, %tmp6
-; CHECK:         bic r0, r0, #-1073741824
-; CHECK-NEXT:    vld3.16 {{{d[0-9]+\[[0-9]\]}}, {{d[0-9]+\[[0-9]\]}}, {{d[0-9]+\[[0-9]\]}}}, [r0]
-  ret <8 x i16> %tmp7
-}
-
-define <4 x i32> @vld3laneQi32(i32* %A, <4 x i32>* %B) nounwind {
-  %tmp0 = bitcast i32* %A to i8*
-  %tmp1 = load <4 x i32>, <4 x i32>* %B
-  %tmp2 = call %struct.__neon_int32x4x3_t @llvm.arm.neon.vld3lane.v4i32(i8* %tmp0, <4 x i32> %tmp1, <4 x i32> %tmp1, <4 x i32> %tmp1, i32 3, i32 1)
-  %tmp3 = extractvalue %struct.__neon_int32x4x3_t %tmp2, 0
-  %tmp4 = extractvalue %struct.__neon_int32x4x3_t %tmp2, 1
-  %tmp5 = extractvalue %struct.__neon_int32x4x3_t %tmp2, 2
-  %tmp6 = add <4 x i32> %tmp3, %tmp4
-  %tmp7 = add <4 x i32> %tmp5, %tmp6
-; CHECK:         bic r0, r0, #-1073741824
-; CHECK-NEXT:    vld3.32 {{{d[0-9]+\[[0-9]\]}}, {{d[0-9]+\[[0-9]\]}}, {{d[0-9]+\[[0-9]\]}}}, [r0]
-  ret <4 x i32> %tmp7
-}
-
-define <8 x i8> @vld4lanei8(i8* %A, <8 x i8>* %B) nounwind {
-  %tmp1 = load <8 x i8>, <8 x i8>* %B
-  %tmp2 = call %struct.__neon_int8x8x4_t @llvm.arm.neon.vld4lane.v8i8(i8* %A, <8 x i8> %tmp1, <8 x i8> %tmp1, <8 x i8> %tmp1, <8 x i8> %tmp1, i32 1, i32 8)
-  %tmp3 = extractvalue %struct.__neon_int8x8x4_t %tmp2, 0
-  %tmp4 = extractvalue %struct.__neon_int8x8x4_t %tmp2, 1
-  %tmp5 = extractvalue %struct.__neon_int8x8x4_t %tmp2, 2
-  %tmp6 = extractvalue %struct.__neon_int8x8x4_t %tmp2, 3
-  %tmp7 = add <8 x i8> %tmp3, %tmp4
-  %tmp8 = add <8 x i8> %tmp5, %tmp6
-  %tmp9 = add <8 x i8> %tmp7, %tmp8
-; CHECK:         bic r0, r0, #-1073741824
-; CHECK-NEXT:    vld4.8 {{{d[0-9]+\[[0-9]\]}}, {{d[0-9]+\[[0-9]\]}}, {{d[0-9]+\[[0-9]\]}}, {{d[0-9]+\[[0-9]\]}}}, [r0:32]
-  ret <8 x i8> %tmp9
-}
-
-define <4 x i16> @vld4lanei16(i16* %A, <4 x i16>* %B) nounwind {
-  %tmp0 = bitcast i16* %A to i8*
-  %tmp1 = load <4 x i16>, <4 x i16>* %B
-  %tmp2 = call %struct.__neon_int16x4x4_t @llvm.arm.neon.vld4lane.v4i16(i8* %tmp0, <4 x i16> %tmp1, <4 x i16> %tmp1, <4 x i16> %tmp1, <4 x i16> %tmp1, i32 1, i32 4)
-  %tmp3 = extractvalue %struct.__neon_int16x4x4_t %tmp2, 0
-  %tmp4 = extractvalue %struct.__neon_int16x4x4_t %tmp2, 1
-  %tmp5 = extractvalue %struct.__neon_int16x4x4_t %tmp2, 2
-  %tmp6 = extractvalue %struct.__neon_int16x4x4_t %tmp2, 3
-  %tmp7 = add <4 x i16> %tmp3, %tmp4
-  %tmp8 = add <4 x i16> %tmp5, %tmp6
-  %tmp9 = add <4 x i16> %tmp7, %tmp8
-; CHECK:         bic r0, r0, #-1073741824
-; CHECK-NEXT:    vld4.16 {{{d[0-9]+\[[0-9]\]}}, {{d[0-9]+\[[0-9]\]}}, {{d[0-9]+\[[0-9]\]}}, {{d[0-9]+\[[0-9]\]}}}, [r0]
-  ret <4 x i16> %tmp9
-}
-
-define <2 x i32> @vld4lanei32(i32* %A, <2 x i32>* %B) nounwind {
-  %tmp0 = bitcast i32* %A to i8*
-  %tmp1 = load <2 x i32>, <2 x i32>* %B
-  %tmp2 = call %struct.__neon_int32x2x4_t @llvm.arm.neon.vld4lane.v2i32(i8* %tmp0, <2 x i32> %tmp1, <2 x i32> %tmp1, <2 x i32> %tmp1, <2 x i32> %tmp1, i32 1, i32 8)
-  %tmp3 = extractvalue %struct.__neon_int32x2x4_t %tmp2, 0
-  %tmp4 = extractvalue %struct.__neon_int32x2x4_t %tmp2, 1
-  %tmp5 = extractvalue %struct.__neon_int32x2x4_t %tmp2, 2
-  %tmp6 = extractvalue %struct.__neon_int32x2x4_t %tmp2, 3
-  %tmp7 = add <2 x i32> %tmp3, %tmp4
-  %tmp8 = add <2 x i32> %tmp5, %tmp6
-  %tmp9 = add <2 x i32> %tmp7, %tmp8
-; CHECK:         bic r0, r0, #-1073741824
-; CHECK-NEXT:    vld4.32 {{{d[0-9]+\[[0-9]\]}}, {{d[0-9]+\[[0-9]\]}}, {{d[0-9]+\[[0-9]\]}}, {{d[0-9]+\[[0-9]\]}}}, [r0:64]
-  ret <2 x i32> %tmp9
-}
-
-define <8 x i16> @vld4laneQi16(i16* %A, <8 x i16>* %B) nounwind {
-  %tmp0 = bitcast i16* %A to i8*
-  %tmp1 = load <8 x i16>, <8 x i16>* %B
-  %tmp2 = call %struct.__neon_int16x8x4_t @llvm.arm.neon.vld4lane.v8i16(i8* %tmp0, <8 x i16> %tmp1, <8 x i16> %tmp1, <8 x i16> %tmp1, <8 x i16> %tmp1, i32 1, i32 16)
-  %tmp3 = extractvalue %struct.__neon_int16x8x4_t %tmp2, 0
-  %tmp4 = extractvalue %struct.__neon_int16x8x4_t %tmp2, 1
-  %tmp5 = extractvalue %struct.__neon_int16x8x4_t %tmp2, 2
-  %tmp6 = extractvalue %struct.__neon_int16x8x4_t %tmp2, 3
-  %tmp7 = add <8 x i16> %tmp3, %tmp4
-  %tmp8 = add <8 x i16> %tmp5, %tmp6
-  %tmp9 = add <8 x i16> %tmp7, %tmp8
-; CHECK:         bic r0, r0, #-1073741824
-; CHECK-NEXT:    vld4.16 {{{d[0-9]+\[[0-9]\]}}, {{d[0-9]+\[[0-9]\]}}, {{d[0-9]+\[[0-9]\]}}, {{d[0-9]+\[[0-9]\]}}}, [r0:64]
-  ret <8 x i16> %tmp9
-}
-
-define <4 x i32> @vld4laneQi32(i32* %A, <4 x i32>* %B) nounwind {
-  %tmp0 = bitcast i32* %A to i8*
-  %tmp1 = load <4 x i32>, <4 x i32>* %B
-  %tmp2 = call %struct.__neon_int32x4x4_t @llvm.arm.neon.vld4lane.v4i32(i8* %tmp0, <4 x i32> %tmp1, <4 x i32> %tmp1, <4 x i32> %tmp1, <4 x i32> %tmp1, i32 2, i32 1)
-  %tmp3 = extractvalue %struct.__neon_int32x4x4_t %tmp2, 0
-  %tmp4 = extractvalue %struct.__neon_int32x4x4_t %tmp2, 1
-  %tmp5 = extractvalue %struct.__neon_int32x4x4_t %tmp2, 2
-  %tmp6 = extractvalue %struct.__neon_int32x4x4_t %tmp2, 3
-  %tmp7 = add <4 x i32> %tmp3, %tmp4
-  %tmp8 = add <4 x i32> %tmp5, %tmp6
-  %tmp9 = add <4 x i32> %tmp7, %tmp8
-; CHECK:         bic r0, r0, #-1073741824
-; CHECK-NEXT:    vld4.32 {{{d[0-9]+\[[0-9]\]}}, {{d[0-9]+\[[0-9]\]}}, {{d[0-9]+\[[0-9]\]}}, {{d[0-9]+\[[0-9]\]}}}, [r0]
-  ret <4 x i32> %tmp9
-}
-
diff --git a/test/NaCl/ARM/neon-vst1-sandboxing.ll b/test/NaCl/ARM/neon-vst1-sandboxing.ll
deleted file mode 100644
index 539e40d0bc0..00000000000
--- a/test/NaCl/ARM/neon-vst1-sandboxing.ll
+++ /dev/null
@@ -1,127 +0,0 @@
-; RUN: pnacl-llc -mtriple=armv7-unknown-nacl -mattr=+neon -filetype=obj %s -o - \
-; RUN:  | llvm-objdump -disassemble -triple armv7 - | FileCheck %s
-
-define void @vst1i8(i8* %A, <8 x i8>* %B) nounwind {
-  %tmp1 = load <8 x i8>, <8 x i8>* %B
-  call void @llvm.arm.neon.vst1.v8i8(i8* %A, <8 x i8> %tmp1, i32 16)
-; CHECK:         bic r0, r0, #-1073741824
-; CHECK-NEXT:    vst1.{{[0-9]+}} {{{d[0-9]+}}}, [r0:64]
-  ret void
-}
-
-define void @vst1i16(i16* %A, <4 x i16>* %B) nounwind {
-  %tmp0 = bitcast i16* %A to i8*
-  %tmp1 = load <4 x i16>, <4 x i16>* %B
-  call void @llvm.arm.neon.vst1.v4i16(i8* %tmp0, <4 x i16> %tmp1, i32 1)
-; CHECK:         bic r0, r0, #-1073741824
-; CHECK-NEXT:    vst1.{{[0-9]+}} {{{d[0-9]+}}}, [r0]
-  ret void
-}
-
-define void @vst1i32(i32* %A, <2 x i32>* %B) nounwind {
-  %tmp0 = bitcast i32* %A to i8*
-  %tmp1 = load <2 x i32>, <2 x i32>* %B
-  call void @llvm.arm.neon.vst1.v2i32(i8* %tmp0, <2 x i32> %tmp1, i32 1)
-; CHECK:         bic r0, r0, #-1073741824
-; CHECK-NEXT:    vst1.{{[0-9]+}} {{{d[0-9]+}}}, [r0]
-  ret void
-}
-
-define void @vst1f(float* %A, <2 x float>* %B) nounwind {
-  %tmp0 = bitcast float* %A to i8*
-  %tmp1 = load <2 x float>, <2 x float>* %B
-  call void @llvm.arm.neon.vst1.v2f32(i8* %tmp0, <2 x float> %tmp1, i32 1)
-; CHECK:         bic r0, r0, #-1073741824
-; CHECK-NEXT:    vst1.{{[0-9]+}} {{{d[0-9]+}}}, [r0]
-  ret void
-}
-
-define void @vst1i64(i64* %A, <1 x i64>* %B) nounwind {
-  %tmp0 = bitcast i64* %A to i8*
-  %tmp1 = load <1 x i64>, <1 x i64>* %B
-  call void @llvm.arm.neon.vst1.v1i64(i8* %tmp0, <1 x i64> %tmp1, i32 1)
-; CHECK:         bic r0, r0, #-1073741824
-; CHECK-NEXT:    vst1.{{[0-9]+}} {{{d[0-9]+}}}, [r0]
-  ret void
-}
-
-define void @vst1Qi8(i8* %A, <16 x i8>* %B) nounwind {
-  %tmp1 = load <16 x i8>, <16 x i8>* %B
-; CHECK:         bic r1, r1, #-1073741824
-; CHECK-NEXT:    vld1.{{[0-9]+}} {d{{[0-9]+}}, d{{[0-9]+}}}, [r1]
-  call void @llvm.arm.neon.vst1.v16i8(i8* %A, <16 x i8> %tmp1, i32 8)
-; CHECK:         bic r0, r0, #-1073741824
-; CHECK-NEXT:    vst1.{{[0-9]+}} {d{{[0-9]+}}, d{{[0-9]+}}}, [r0:64]
-  ret void
-}
-
-define void @vst1Qi16(i16* %A, <8 x i16>* %B) nounwind {
-  %tmp0 = bitcast i16* %A to i8*
-  %tmp1 = load <8 x i16>, <8 x i16>* %B
-; CHECK:         bic r1, r1, #-1073741824
-; CHECK-NEXT:    vld1.{{[0-9]+}} {d{{[0-9]+}}, d{{[0-9]+}}}, [r1]
-  call void @llvm.arm.neon.vst1.v8i16(i8* %tmp0, <8 x i16> %tmp1, i32 32)
-; CHECK:         bic r0, r0, #-1073741824
-; CHECK-NEXT:    vst1.{{[0-9]+}} {{{d[0-9]+, d[0-9]+}}}, [r0:128]
-  ret void
-}
-
-define void @vst1Qi32(i32* %A, <4 x i32>* %B) nounwind {
-  %tmp0 = bitcast i32* %A to i8*
-  %tmp1 = load <4 x i32>, <4 x i32>* %B
-; CHECK:         bic r1, r1, #-1073741824
-; CHECK-NEXT:    vld1.{{[0-9]+}} {d{{[0-9]+}}, d{{[0-9]+}}}, [r1]
-  call void @llvm.arm.neon.vst1.v4i32(i8* %tmp0, <4 x i32> %tmp1, i32 1)
-; CHECK:         bic r0, r0, #-1073741824
-; CHECK-NEXT:    vst1.{{[0-9]+}} {{{d[0-9]+, d[0-9]+}}}, [r0]
-  ret void
-}
-
-define void @vst1Qf(float* %A, <4 x float>* %B) nounwind {
-  %tmp0 = bitcast float* %A to i8*
-  %tmp1 = load <4 x float>, <4 x float>* %B
-; CHECK:         bic r1, r1, #-1073741824
-; CHECK-NEXT:    vld1.{{[0-9]+}} {d{{[0-9]+}}, d{{[0-9]+}}}, [r1]
-  call void @llvm.arm.neon.vst1.v4f32(i8* %tmp0, <4 x float> %tmp1, i32 1)
-; CHECK:         bic r0, r0, #-1073741824
-; CHECK-NEXT:    vst1.{{[0-9]+}} {{{d[0-9]+, d[0-9]+}}}, [r0]
-  ret void
-}
-
-define void @vst1Qi64(i64* %A, <2 x i64>* %B) nounwind {
-  %tmp0 = bitcast i64* %A to i8*
-  %tmp1 = load <2 x i64>, <2 x i64>* %B
-; CHECK:         bic r1, r1, #-1073741824
-; CHECK-NEXT:    vld1.{{[0-9]+}} {d{{[0-9]+}}, d{{[0-9]+}}}, [r1]
-  call void @llvm.arm.neon.vst1.v2i64(i8* %tmp0, <2 x i64> %tmp1, i32 1)
-; CHECK:         bic r0, r0, #-1073741824
-; CHECK-NEXT:    vst1.{{[0-9]+}} {{{d[0-9]+, d[0-9]+}}}, [r0]
-  ret void
-}
-
-;Check for a post-increment updating store.
-define void @vst1f_update(float** %ptr, <2 x float>* %B) nounwind {
-; CHECK:         bic r1, r1, #-1073741824
-  %A = load float*, float** %ptr
-  %tmp0 = bitcast float* %A to i8*
-  %tmp1 = load <2 x float>, <2 x float>* %B
-  call void @llvm.arm.neon.vst1.v2f32(i8* %tmp0, <2 x float> %tmp1, i32 1)
-; CHECK:         bic r1, r1, #-1073741824
-; CHECK-NEXT:    vst1.{{[0-9]+}} {{{d[0-9]+}}}, [r1]!
-  %tmp2 = getelementptr float, float* %A, i32 2
-  store float* %tmp2, float** %ptr
-  ret void
-}
-
-declare void @llvm.arm.neon.vst1.v8i8(i8*, <8 x i8>, i32) nounwind
-declare void @llvm.arm.neon.vst1.v4i16(i8*, <4 x i16>, i32) nounwind
-declare void @llvm.arm.neon.vst1.v2i32(i8*, <2 x i32>, i32) nounwind
-declare void @llvm.arm.neon.vst1.v2f32(i8*, <2 x float>, i32) nounwind
-declare void @llvm.arm.neon.vst1.v1i64(i8*, <1 x i64>, i32) nounwind
-
-declare void @llvm.arm.neon.vst1.v16i8(i8*, <16 x i8>, i32) nounwind
-declare void @llvm.arm.neon.vst1.v8i16(i8*, <8 x i16>, i32) nounwind
-declare void @llvm.arm.neon.vst1.v4i32(i8*, <4 x i32>, i32) nounwind
-declare void @llvm.arm.neon.vst1.v4f32(i8*, <4 x float>, i32) nounwind
-declare void @llvm.arm.neon.vst1.v2i64(i8*, <2 x i64>, i32) nounwind
-
diff --git a/test/NaCl/ARM/neon-vst2-sandboxing.ll b/test/NaCl/ARM/neon-vst2-sandboxing.ll
deleted file mode 100644
index 777e5c71a9a..00000000000
--- a/test/NaCl/ARM/neon-vst2-sandboxing.ll
+++ /dev/null
@@ -1,104 +0,0 @@
-; RUN: pnacl-llc -mtriple=armv7-unknown-nacl -mattr=+neon -filetype=obj %s -o - \
-; RUN:  | llvm-objdump -disassemble -triple armv7 - | FileCheck %s
-
-define void @vst2i8(i8* %A, <8 x i8>* %B) nounwind {
-  %tmp1 = load <8 x i8>, <8 x i8>* %B
-  call void @llvm.arm.neon.vst2.v8i8(i8* %A, <8 x i8> %tmp1, <8 x i8> %tmp1, i32 8)
-; CHECK:         bic r0, r0, #-1073741824
-; CHECK-NEXT:    vst2.{{[0-9]+}} {{{d[0-9]+, d[0-9]+}}}, [r0:64]
-  ret void
-}
-
-define void @vst2i16(i16* %A, <4 x i16>* %B) nounwind {
-  %tmp0 = bitcast i16* %A to i8*
-  %tmp1 = load <4 x i16>, <4 x i16>* %B
-  call void @llvm.arm.neon.vst2.v4i16(i8* %tmp0, <4 x i16> %tmp1, <4 x i16> %tmp1, i32 32)
-; CHECK:         bic r0, r0, #-1073741824
-; CHECK-NEXT:    vst2.{{[0-9]+}} {{{d[0-9]+, d[0-9]+}}}, [r0:128]
-  ret void
-}
-
-define void @vst2i32(i32* %A, <2 x i32>* %B) nounwind {
-  %tmp0 = bitcast i32* %A to i8*
-  %tmp1 = load <2 x i32>, <2 x i32>* %B
-  call void @llvm.arm.neon.vst2.v2i32(i8* %tmp0, <2 x i32> %tmp1, <2 x i32> %tmp1, i32 1)
-; CHECK:         bic r0, r0, #-1073741824
-; CHECK-NEXT:    vst2.{{[0-9]+}} {{{d[0-9]+, d[0-9]+}}}, [r0]
-  ret void
-}
-
-define void @vst2f(float* %A, <2 x float>* %B) nounwind {
-  %tmp0 = bitcast float* %A to i8*
-  %tmp1 = load <2 x float>, <2 x float>* %B
-  call void @llvm.arm.neon.vst2.v2f32(i8* %tmp0, <2 x float> %tmp1, <2 x float> %tmp1, i32 1)
-; CHECK:         bic r0, r0, #-1073741824
-; CHECK-NEXT:    vst2.{{[0-9]+}} {{{d[0-9]+, d[0-9]+}}}, [r0]
-  ret void
-}
-
-define void @vst2Qi8(i8* %A, <16 x i8>* %B) nounwind {
-  %tmp1 = load <16 x i8>, <16 x i8>* %B
-; CHECK:         bic r1, r1, #-1073741824
-; CHECK-NEXT:    vld1.{{[0-9]+}} {d{{[0-9]+}}, d{{[0-9]+}}}, [r1]
-  call void @llvm.arm.neon.vst2.v16i8(i8* %A, <16 x i8> %tmp1, <16 x i8> %tmp1, i32 8)
-; CHECK:         bic r0, r0, #-1073741824
-; CHECK-NEXT:    vst2.{{[0-9]+}} {{{d[0-9]+, d[0-9]+, d[0-9]+, d[0-9]+}}}, [r0:64]
-  ret void
-}
-
-define void @vst2Qi16(i16* %A, <8 x i16>* %B) nounwind {
-  %tmp0 = bitcast i16* %A to i8*
-  %tmp1 = load <8 x i16>, <8 x i16>* %B
-; CHECK:         bic r1, r1, #-1073741824
-; CHECK-NEXT:    vld1.{{[0-9]+}} {d{{[0-9]+}}, d{{[0-9]+}}}, [r1]
-  call void @llvm.arm.neon.vst2.v8i16(i8* %tmp0, <8 x i16> %tmp1, <8 x i16> %tmp1, i32 16)
-; CHECK:         bic r0, r0, #-1073741824
-; CHECK-NEXT:    vst2.{{[0-9]+}} {{{d[0-9]+, d[0-9]+, d[0-9]+, d[0-9]+}}}, [r0:128]
-  ret void
-}
-
-define void @vst2Qi32(i32* %A, <4 x i32>* %B) nounwind {
-  %tmp0 = bitcast i32* %A to i8*
-  %tmp1 = load <4 x i32>, <4 x i32>* %B
-; CHECK:         bic r1, r1, #-1073741824
-; CHECK-NEXT:    vld1.{{[0-9]+}} {d{{[0-9]+}}, d{{[0-9]+}}}, [r1]
-  call void @llvm.arm.neon.vst2.v4i32(i8* %tmp0, <4 x i32> %tmp1, <4 x i32> %tmp1, i32 64)
-; CHECK:         bic r0, r0, #-1073741824
-; CHECK-NEXT:    vst2.{{[0-9]+}} {{{d[0-9]+, d[0-9]+, d[0-9]+, d[0-9]+}}}, [r0:256]
-  ret void
-}
-
-define void @vst2Qf(float* %A, <4 x float>* %B) nounwind {
-  %tmp0 = bitcast float* %A to i8*
-  %tmp1 = load <4 x float>, <4 x float>* %B
-; CHECK:         bic r1, r1, #-1073741824
-; CHECK-NEXT:    vld1.{{[0-9]+}} {d{{[0-9]+}}, d{{[0-9]+}}}, [r1]
-  call void @llvm.arm.neon.vst2.v4f32(i8* %tmp0, <4 x float> %tmp1, <4 x float> %tmp1, i32 1)
-; CHECK:         bic r0, r0, #-1073741824
-; CHECK-NEXT:    vst2.{{[0-9]+}} {{{d[0-9]+, d[0-9]+, d[0-9]+, d[0-9]+}}}, [r0]
-  ret void
-}
-
-;Check for a post-increment updating store with register increment.
-define void @vst2i8_update(i8** %ptr, <8 x i8>* %B, i32 %inc) nounwind {
-; CHECK:         bic r1, r1, #-1073741824
-  %A = load i8*, i8** %ptr
-  %tmp1 = load <8 x i8>, <8 x i8>* %B
-  call void @llvm.arm.neon.vst2.v8i8(i8* %A, <8 x i8> %tmp1, <8 x i8> %tmp1, i32 4)
-; CHECK:         bic r1, r1, #-1073741824
-; CHECK-NEXT:    vst2.{{[0-9]+}} {{{d[0-9]+, d[0-9]+}}}, [r1], r2
-  %tmp2 = getelementptr i8, i8* %A, i32 %inc
-  store i8* %tmp2, i8** %ptr
-  ret void
-}
-
-declare void @llvm.arm.neon.vst2.v8i8(i8*, <8 x i8>, <8 x i8>, i32) nounwind
-declare void @llvm.arm.neon.vst2.v4i16(i8*, <4 x i16>, <4 x i16>, i32) nounwind
-declare void @llvm.arm.neon.vst2.v2i32(i8*, <2 x i32>, <2 x i32>, i32) nounwind
-declare void @llvm.arm.neon.vst2.v2f32(i8*, <2 x float>, <2 x float>, i32) nounwind
-declare void @llvm.arm.neon.vst2.v1i64(i8*, <1 x i64>, <1 x i64>, i32) nounwind
-
-declare void @llvm.arm.neon.vst2.v16i8(i8*, <16 x i8>, <16 x i8>, i32) nounwind
-declare void @llvm.arm.neon.vst2.v8i16(i8*, <8 x i16>, <8 x i16>, i32) nounwind
-declare void @llvm.arm.neon.vst2.v4i32(i8*, <4 x i32>, <4 x i32>, i32) nounwind
-declare void @llvm.arm.neon.vst2.v4f32(i8*, <4 x float>, <4 x float>, i32) nounwind
diff --git a/test/NaCl/ARM/neon-vst3-sandboxing.ll b/test/NaCl/ARM/neon-vst3-sandboxing.ll
deleted file mode 100644
index d726bdb71ff..00000000000
--- a/test/NaCl/ARM/neon-vst3-sandboxing.ll
+++ /dev/null
@@ -1,50 +0,0 @@
-; RUN: pnacl-llc -mtriple=armv7-unknown-nacl -mattr=+neon -filetype=obj %s -o - \
-; RUN:  | llvm-objdump -disassemble -triple armv7 - | FileCheck %s
-
-define void @vst3i8(i8* %A, <8 x i8>* %B) nounwind {
-  %tmp1 = load <8 x i8>, <8 x i8>* %B
-  call void @llvm.arm.neon.vst3.v8i8(i8* %A, <8 x i8> %tmp1, <8 x i8> %tmp1, <8 x i8> %tmp1, i32 32)
-; CHECK:         bic r0, r0, #-1073741824
-; CHECK-NEXT:    vst3.{{[0-9]+}} {d{{[0-9]+}}, d{{[0-9]+}}, d{{[0-9]+}}}, [r0:64]
-  ret void
-}
-
-define void @vst3i16(i16* %A, <4 x i16>* %B) nounwind {
-  %tmp0 = bitcast i16* %A to i8*
-  %tmp1 = load <4 x i16>, <4 x i16>* %B
-  call void @llvm.arm.neon.vst3.v4i16(i8* %tmp0, <4 x i16> %tmp1, <4 x i16> %tmp1, <4 x i16> %tmp1, i32 1)
-; CHECK:         bic r0, r0, #-1073741824
-; CHECK-NEXT:    vst3.{{[0-9]+}} {d{{[0-9]+}}, d{{[0-9]+}}, d{{[0-9]+}}}, [r0]
-  ret void
-}
-
-define void @vst3i32(i32* %A, <2 x i32>* %B) nounwind {
-  %tmp0 = bitcast i32* %A to i8*
-  %tmp1 = load <2 x i32>, <2 x i32>* %B
-  call void @llvm.arm.neon.vst3.v2i32(i8* %tmp0, <2 x i32> %tmp1, <2 x i32> %tmp1, <2 x i32> %tmp1, i32 1)
-; CHECK:         bic r0, r0, #-1073741824
-; CHECK-NEXT:    vst3.{{[0-9]+}} {d{{[0-9]+}}, d{{[0-9]+}}, d{{[0-9]+}}}, [r0]
-  ret void
-}
-
-;Check for a post-increment updating store.
-define void @vst3Qi16_update(i16** %ptr, <8 x i16>* %B) nounwind {
-  %A = load i16*, i16** %ptr
-  %tmp0 = bitcast i16* %A to i8*
-  %tmp1 = load <8 x i16>, <8 x i16>* %B
-; CHECK:         bic r1, r1, #-1073741824
-; CHECK-NEXT:    vld1.{{[0-9]+}} {d{{[0-9]+}}, d{{[0-9]+}}}, [r1]
-  call void @llvm.arm.neon.vst3.v8i16(i8* %tmp0, <8 x i16> %tmp1, <8 x i16> %tmp1, <8 x i16> %tmp1, i32 1)
-; CHECK:         bic r1, r1, #-1073741824
-; CHECK-NEXT:    vst3.{{[0-9]+}} {d{{[0-9]+}}, d{{[0-9]+}}, d{{[0-9]+}}}, [r1]!
-  %tmp2 = getelementptr i16, i16* %A, i32 24
-  store i16* %tmp2, i16** %ptr
-  ret void
-}
-
-declare void @llvm.arm.neon.vst3.v8i8(i8*, <8 x i8>, <8 x i8>, <8 x i8>, i32) nounwind
-declare void @llvm.arm.neon.vst3.v4i16(i8*, <4 x i16>, <4 x i16>, <4 x i16>, i32) nounwind
-declare void @llvm.arm.neon.vst3.v2i32(i8*, <2 x i32>, <2 x i32>, <2 x i32>, i32) nounwind
-declare void @llvm.arm.neon.vst3.v2f32(i8*, <2 x float>, <2 x float>, <2 x float>, i32) nounwind
-
-declare void @llvm.arm.neon.vst3.v8i16(i8*, <8 x i16>, <8 x i16>, <8 x i16>, i32) nounwind
diff --git a/test/NaCl/ARM/neon-vst4-sandboxing.ll b/test/NaCl/ARM/neon-vst4-sandboxing.ll
deleted file mode 100644
index fdbd6192791..00000000000
--- a/test/NaCl/ARM/neon-vst4-sandboxing.ll
+++ /dev/null
@@ -1,55 +0,0 @@
-; RUN: pnacl-llc -mtriple=armv7-unknown-nacl -mattr=+neon -filetype=obj %s -o - \
-; RUN:  | llvm-objdump -disassemble -triple armv7 - | FileCheck %s
-
-define void @vst4i8(i8* %A, <8 x i8>* %B) nounwind {
-  %tmp1 = load <8 x i8>, <8 x i8>* %B
-  call void @llvm.arm.neon.vst4.v8i8(i8* %A, <8 x i8> %tmp1, <8 x i8> %tmp1, <8 x i8> %tmp1, <8 x i8> %tmp1, i32 8)
-; CHECK:         bic r0, r0, #-1073741824
-; CHECK-NEXT:    vst4.{{[0-9]+}} {d{{[0-9]+}}, d{{[0-9]+}}, d{{[0-9]+}}, d{{[0-9]+}}}, [r0:64]
-  ret void
-}
-
-define void @vst4i16(i16* %A, <4 x i16>* %B) nounwind {
-  %tmp0 = bitcast i16* %A to i8*
-  %tmp1 = load <4 x i16>, <4 x i16>* %B
-  call void @llvm.arm.neon.vst4.v4i16(i8* %tmp0, <4 x i16> %tmp1, <4 x i16> %tmp1, <4 x i16> %tmp1, <4 x i16> %tmp1, i32 16)
-; CHECK:         bic r0, r0, #-1073741824
-; CHECK-NEXT:    vst4.{{[0-9]+}} {d{{[0-9]+}}, d{{[0-9]+}}, d{{[0-9]+}}, d{{[0-9]+}}}, [r0:128]
-  ret void
-}
-
-define void @vst4i32(i32* %A, <2 x i32>* %B) nounwind {
-  %tmp0 = bitcast i32* %A to i8*
-  %tmp1 = load <2 x i32>, <2 x i32>* %B
-  call void @llvm.arm.neon.vst4.v2i32(i8* %tmp0, <2 x i32> %tmp1, <2 x i32> %tmp1, <2 x i32> %tmp1, <2 x i32> %tmp1, i32 32)
-; CHECK:         bic r0, r0, #-1073741824
-; CHECK-NEXT:    vst4.{{[0-9]+}} {d{{[0-9]+}}, d{{[0-9]+}}, d{{[0-9]+}}, d{{[0-9]+}}}, [r0:256]
-  ret void
-}
-
-;Check for a post-increment updating store.
-define void @vst4Qf_update(float** %ptr, <4 x float>* %B) nounwind {
-  %A = load float*, float** %ptr
-  %tmp0 = bitcast float* %A to i8*
-  %tmp1 = load <4 x float>, <4 x float>* %B
-; CHECK:         bic r1, r1, #-1073741824
-; CHECK-NEXT:    vld1.{{[0-9]+}} {d{{[0-9]+}}, d{{[0-9]+}}}, [r1]
-  call void @llvm.arm.neon.vst4.v4f32(i8* %tmp0, <4 x float> %tmp1, <4 x float> %tmp1, <4 x float> %tmp1, <4 x float> %tmp1, i32 1)
-; CHECK:         bic r1, r1, #-1073741824
-; CHECK-NEXT:    vst4.{{[0-9]+}} {d{{[0-9]+}}, d{{[0-9]+}}, d{{[0-9]+}}, d{{[0-9]+}}}, [r1]!
-  %tmp2 = getelementptr float, float* %A, i32 16
-  store float* %tmp2, float** %ptr
-  ret void
-}
-
-declare void @llvm.arm.neon.vst4.v8i8(i8*, <8 x i8>, <8 x i8>, <8 x i8>, <8 x i8>, i32) nounwind
-declare void @llvm.arm.neon.vst4.v4i16(i8*, <4 x i16>, <4 x i16>, <4 x i16>, <4 x i16>, i32) nounwind
-declare void @llvm.arm.neon.vst4.v2i32(i8*, <2 x i32>, <2 x i32>, <2 x i32>, <2 x i32>, i32) nounwind
-declare void @llvm.arm.neon.vst4.v2f32(i8*, <2 x float>, <2 x float>, <2 x float>, <2 x float>, i32) nounwind
-declare void @llvm.arm.neon.vst4.v1i64(i8*, <1 x i64>, <1 x i64>, <1 x i64>, <1 x i64>, i32) nounwind
-
-declare void @llvm.arm.neon.vst4.v16i8(i8*, <16 x i8>, <16 x i8>, <16 x i8>, <16 x i8>, i32) nounwind
-declare void @llvm.arm.neon.vst4.v8i16(i8*, <8 x i16>, <8 x i16>, <8 x i16>, <8 x i16>, i32) nounwind
-declare void @llvm.arm.neon.vst4.v4i32(i8*, <4 x i32>, <4 x i32>, <4 x i32>, <4 x i32>, i32) nounwind
-declare void @llvm.arm.neon.vst4.v4f32(i8*, <4 x float>, <4 x float>, <4 x float>, <4 x float>, i32) nounwind
-
diff --git a/test/NaCl/ARM/neon-vstlane-sandboxing.ll b/test/NaCl/ARM/neon-vstlane-sandboxing.ll
deleted file mode 100644
index ca1713cdc9f..00000000000
--- a/test/NaCl/ARM/neon-vstlane-sandboxing.ll
+++ /dev/null
@@ -1,209 +0,0 @@
-; RUN: pnacl-llc -mtriple=armv7-unknown-nacl -mattr=+neon -filetype=obj %s -o - \
-; RUN:  | llvm-objdump -disassemble -triple armv7 - | FileCheck %s
-
-define void @vst1lanei8(i8* %A, <8 x i8>* %B) nounwind {
-  %tmp1 = load <8 x i8>, <8 x i8>* %B
-  %tmp2 = extractelement <8 x i8> %tmp1, i32 3
-  store i8 %tmp2, i8* %A, align 8
-; CHECK:         bic r0, r0, #-1073741824
-; CHECK-NEXT:    vst1.{{[0-9]+}} {d{{[0-9]+}}[3]}, [r0]
-  ret void
-}
-
-define void @vst1lanei16(i16* %A, <4 x i16>* %B) nounwind {
-  %tmp1 = load <4 x i16>, <4 x i16>* %B
-  %tmp2 = extractelement <4 x i16> %tmp1, i32 2
-  store i16 %tmp2, i16* %A, align 8
-; CHECK:         bic r0, r0, #-1073741824
-; CHECK-NEXT:    vst1.{{[0-9]+}} {d{{[0-9]+}}[2]}, [r0:16]
-  ret void
-}
-
-define void @vst1lanei32(i32* %A, <2 x i32>* %B) nounwind {
-  %tmp1 = load <2 x i32>, <2 x i32>* %B
-  %tmp2 = extractelement <2 x i32> %tmp1, i32 1
-  store i32 %tmp2, i32* %A, align 8
-; CHECK:         bic r0, r0, #-1073741824
-; CHECK-NEXT:    vst1.{{[0-9]+}} {d{{[0-9]+}}[1]}, [r0:32]
-  ret void
-}
-
-define void @vst1laneQi8(i8* %A, <16 x i8>* %B) nounwind {
-  %tmp1 = load <16 x i8>, <16 x i8>* %B
-; CHECK:         bic r1, r1, #-1073741824
-; CHECK-NEXT:    vld1.{{[0-9]+}} {d{{[0-9]+}}, d{{[0-9]+}}}, [r1]
-  %tmp2 = extractelement <16 x i8> %tmp1, i32 9
-  store i8 %tmp2, i8* %A, align 8
-; CHECK:         bic r0, r0, #-1073741824
-; CHECK-NEXT:    vst1.{{[0-9]+}} {d{{[0-9]+}}[1]}, [r0]
-  ret void
-}
-
-define void @vst1laneQi16(i16* %A, <8 x i16>* %B) nounwind {
-  %tmp1 = load <8 x i16>, <8 x i16>* %B
-; CHECK:         bic r1, r1, #-1073741824
-; CHECK-NEXT:    vld1.{{[0-9]+}} {d{{[0-9]+}}, d{{[0-9]+}}}, [r1]
-  %tmp2 = extractelement <8 x i16> %tmp1, i32 5
-  store i16 %tmp2, i16* %A, align 8
-; CHECK:         bic r0, r0, #-1073741824
-; CHECK-NEXT:    vst1.16 {d{{[0-9]+}}[1]}, [r0:16]
-  ret void
-}
-
-define void @vst2lanei8(i8* %A, <8 x i8>* %B) nounwind {
-  %tmp1 = load <8 x i8>, <8 x i8>* %B
-  call void @llvm.arm.neon.vst2lane.v8i8(i8* %A, <8 x i8> %tmp1, <8 x i8> %tmp1, i32 1, i32 4)
-; CHECK:         bic r0, r0, #-1073741824
-; CHECK-NEXT:    vst2.{{[0-9]+}} {d{{[0-9]+}}[1], d{{[0-9]+}}[1]}, [r0:16]
-  ret void
-}
-
-define void @vst2lanei16(i16* %A, <4 x i16>* %B) nounwind {
-  %tmp0 = bitcast i16* %A to i8*
-  %tmp1 = load <4 x i16>, <4 x i16>* %B
-  call void @llvm.arm.neon.vst2lane.v4i16(i8* %tmp0, <4 x i16> %tmp1, <4 x i16> %tmp1, i32 1, i32 8)
-; CHECK:         bic r0, r0, #-1073741824
-; CHECK-NEXT:    vst2.{{[0-9]+}} {d{{[0-9]+}}[1], d{{[0-9]+}}[1]}, [r0:32]
-  ret void
-}
-
-define void @vst2lanei32(i32* %A, <2 x i32>* %B) nounwind {
-  %tmp0 = bitcast i32* %A to i8*
-  %tmp1 = load <2 x i32>, <2 x i32>* %B
-  call void @llvm.arm.neon.vst2lane.v2i32(i8* %tmp0, <2 x i32> %tmp1, <2 x i32> %tmp1, i32 1, i32 1)
-; CHECK:         bic r0, r0, #-1073741824
-; CHECK-NEXT:    vst2.{{[0-9]+}} {d{{[0-9]+}}[1], d{{[0-9]+}}[1]}, [r0]
-  ret void
-}
-
-define void @vst2laneQi16(i16* %A, <8 x i16>* %B) nounwind {
-  %tmp0 = bitcast i16* %A to i8*
-  %tmp1 = load <8 x i16>, <8 x i16>* %B
-; CHECK:         bic r1, r1, #-1073741824
-; CHECK-NEXT:    vld1.{{[0-9]+}} {d{{[0-9]+}}, d{{[0-9]+}}}, [r1]
-  call void @llvm.arm.neon.vst2lane.v8i16(i8* %tmp0, <8 x i16> %tmp1, <8 x i16> %tmp1, i32 5, i32 1)
-; CHECK:         bic r0, r0, #-1073741824
-; CHECK-NEXT:    vst2.{{[0-9]+}} {d{{[0-9]+}}[1], d{{[0-9]+}}[1]}, [r0]
-  ret void
-}
-
-define void @vst2laneQi32(i32* %A, <4 x i32>* %B) nounwind {
-  %tmp0 = bitcast i32* %A to i8*
-  %tmp1 = load <4 x i32>, <4 x i32>* %B
-; CHECK:         bic r1, r1, #-1073741824
-; CHECK-NEXT:    vld1.{{[0-9]+}} {d{{[0-9]+}}, d{{[0-9]+}}}, [r1]
-  call void @llvm.arm.neon.vst2lane.v4i32(i8* %tmp0, <4 x i32> %tmp1, <4 x i32> %tmp1, i32 2, i32 16)
-; CHECK:         bic r0, r0, #-1073741824
-; CHECK-NEXT:    vst2.{{[0-9]+}} {d{{[0-9]+}}[0], d{{[0-9]+}}[0]}, [r0:64]
-  ret void
-}
-
-define void @vst3lanei8(i8* %A, <8 x i8>* %B) nounwind {
-  %tmp1 = load <8 x i8>, <8 x i8>* %B
-  call void @llvm.arm.neon.vst3lane.v8i8(i8* %A, <8 x i8> %tmp1, <8 x i8> %tmp1, <8 x i8> %tmp1, i32 1, i32 1)
-; CHECK:         bic r0, r0, #-1073741824
-; CHECK-NEXT:    vst3.{{[0-9]+}} {d{{[0-9]+}}[1], d{{[0-9]+}}[1], d{{[0-9]+}}[1]}, [r0]
-  ret void
-}
-
-define void @vst3lanei16(i16* %A, <4 x i16>* %B) nounwind {
-  %tmp0 = bitcast i16* %A to i8*
-  %tmp1 = load <4 x i16>, <4 x i16>* %B
-  call void @llvm.arm.neon.vst3lane.v4i16(i8* %tmp0, <4 x i16> %tmp1, <4 x i16> %tmp1, <4 x i16> %tmp1, i32 1, i32 8)
-; CHECK:         bic r0, r0, #-1073741824
-; CHECK-NEXT:    vst3.{{[0-9]+}} {d{{[0-9]+}}[1], d{{[0-9]+}}[1], d{{[0-9]+}}[1]}, [r0]
-  ret void
-}
-
-define void @vst3lanei32(i32* %A, <2 x i32>* %B) nounwind {
-  %tmp0 = bitcast i32* %A to i8*
-  %tmp1 = load <2 x i32>, <2 x i32>* %B
-  call void @llvm.arm.neon.vst3lane.v2i32(i8* %tmp0, <2 x i32> %tmp1, <2 x i32> %tmp1, <2 x i32> %tmp1, i32 1, i32 1)
-; CHECK:         bic r0, r0, #-1073741824
-; CHECK-NEXT:    vst3.{{[0-9]+}} {d{{[0-9]+}}[1], d{{[0-9]+}}[1], d{{[0-9]+}}[1]}, [r0]
-  ret void
-}
-
-define void @vst4lanei8(i8* %A, <8 x i8>* %B) nounwind {
-  %tmp1 = load <8 x i8>, <8 x i8>* %B
-  call void @llvm.arm.neon.vst4lane.v8i8(i8* %A, <8 x i8> %tmp1, <8 x i8> %tmp1, <8 x i8> %tmp1, <8 x i8> %tmp1, i32 1, i32 8)
-; CHECK:         bic r0, r0, #-1073741824
-; CHECK-NEXT:    vst4.{{[0-9]+}} {d{{[0-9]+}}[1], d{{[0-9]+}}[1], d{{[0-9]+}}[1], d{{[0-9]+}}[1]}, [r0:32]
-  ret void
-}
-
-define void @vst4lanei16(i16* %A, <4 x i16>* %B) nounwind {
-  %tmp0 = bitcast i16* %A to i8*
-  %tmp1 = load <4 x i16>, <4 x i16>* %B
-  call void @llvm.arm.neon.vst4lane.v4i16(i8* %tmp0, <4 x i16> %tmp1, <4 x i16> %tmp1, <4 x i16> %tmp1, <4 x i16> %tmp1, i32 1, i32 1)
-; CHECK:         bic r0, r0, #-1073741824
-; CHECK-NEXT:    vst4.{{[0-9]+}} {d{{[0-9]+}}[1], d{{[0-9]+}}[1], d{{[0-9]+}}[1], d{{[0-9]+}}[1]}, [r0]
-  ret void
-}
-
-define void @vst4lanei32(i32* %A, <2 x i32>* %B) nounwind {
-  %tmp0 = bitcast i32* %A to i8*
-  %tmp1 = load <2 x i32>, <2 x i32>* %B
-  call void @llvm.arm.neon.vst4lane.v2i32(i8* %tmp0, <2 x i32> %tmp1, <2 x i32> %tmp1, <2 x i32> %tmp1, <2 x i32> %tmp1, i32 1, i32 16)
-; CHECK:         bic r0, r0, #-1073741824
-; CHECK-NEXT:    vst4.{{[0-9]+}} {d{{[0-9]+}}[1], d{{[0-9]+}}[1], d{{[0-9]+}}[1], d{{[0-9]+}}[1]}, [r0:128]
-  ret void
-}
-
-define void @vst4laneQi16(i16* %A, <8 x i16>* %B) nounwind {
-  %tmp0 = bitcast i16* %A to i8*
-  %tmp1 = load <8 x i16>, <8 x i16>* %B
-; CHECK:         bic r1, r1, #-1073741824
-; CHECK-NEXT:    vld1.{{[0-9]+}} {d{{[0-9]+}}, d{{[0-9]+}}}, [r1]
-  call void @llvm.arm.neon.vst4lane.v8i16(i8* %tmp0, <8 x i16> %tmp1, <8 x i16> %tmp1, <8 x i16> %tmp1, <8 x i16> %tmp1, i32 7, i32 16)
-; CHECK:         bic r0, r0, #-1073741824
-; CHECK-NEXT:    vst4.{{[0-9]+}} {d{{[0-9]+}}[3], d{{[0-9]+}}[3], d{{[0-9]+}}[3], d{{[0-9]+}}[3]}, [r0:64]
-  ret void
-}
-
-define void @vst4laneQi32(i32* %A, <4 x i32>* %B) nounwind {
-  %tmp0 = bitcast i32* %A to i8*
-  %tmp1 = load <4 x i32>, <4 x i32>* %B
-; CHECK:         bic r1, r1, #-1073741824
-; CHECK-NEXT:    vld1.{{[0-9]+}} {d{{[0-9]+}}, d{{[0-9]+}}}, [r1]
-  call void @llvm.arm.neon.vst4lane.v4i32(i8* %tmp0, <4 x i32> %tmp1, <4 x i32> %tmp1, <4 x i32> %tmp1, <4 x i32> %tmp1, i32 2, i32 1)
-; CHECK:         bic r0, r0, #-1073741824
-; CHECK-NEXT:    vst4.{{[0-9]+}} {d{{[0-9]+}}[0], d{{[0-9]+}}[0], d{{[0-9]+}}[0], d{{[0-9]+}}[0]}, [r0]
-  ret void
-}
-
-declare void @llvm.arm.neon.vst2lane.v8i8(i8*, <8 x i8>, <8 x i8>, i32, i32) nounwind
-declare void @llvm.arm.neon.vst2lane.v4i16(i8*, <4 x i16>, <4 x i16>, i32, i32) nounwind
-declare void @llvm.arm.neon.vst2lane.v2i32(i8*, <2 x i32>, <2 x i32>, i32, i32) nounwind
-declare void @llvm.arm.neon.vst2lane.v2f32(i8*, <2 x float>, <2 x float>, i32, i32) nounwind
-declare void @llvm.arm.neon.vst2lane.v8i16(i8*, <8 x i16>, <8 x i16>, i32, i32) nounwind
-declare void @llvm.arm.neon.vst2lane.v4i32(i8*, <4 x i32>, <4 x i32>, i32, i32) nounwind
-declare void @llvm.arm.neon.vst2lane.v4f32(i8*, <4 x float>, <4 x float>, i32, i32) nounwind
-declare void @llvm.arm.neon.vst3lane.v8i8(i8*, <8 x i8>, <8 x i8>, <8 x i8>, i32, i32) nounwind
-declare void @llvm.arm.neon.vst3lane.v4i16(i8*, <4 x i16>, <4 x i16>, <4 x i16>, i32, i32) nounwind
-declare void @llvm.arm.neon.vst3lane.v2i32(i8*, <2 x i32>, <2 x i32>, <2 x i32>, i32, i32) nounwind
-declare void @llvm.arm.neon.vst3lane.v2f32(i8*, <2 x float>, <2 x float>, <2 x float>, i32, i32) nounwind
-declare void @llvm.arm.neon.vst3lane.v8i16(i8*, <8 x i16>, <8 x i16>, <8 x i16>, i32, i32) nounwind
-declare void @llvm.arm.neon.vst3lane.v4i32(i8*, <4 x i32>, <4 x i32>, <4 x i32>, i32, i32) nounwind
-declare void @llvm.arm.neon.vst3lane.v4f32(i8*, <4 x float>, <4 x float>, <4 x float>, i32, i32) nounwind
-declare void @llvm.arm.neon.vst4lane.v8i8(i8*, <8 x i8>, <8 x i8>, <8 x i8>, <8 x i8>, i32, i32) nounwind
-declare void @llvm.arm.neon.vst4lane.v4i16(i8*, <4 x i16>, <4 x i16>, <4 x i16>, <4 x i16>, i32, i32) nounwind
-declare void @llvm.arm.neon.vst4lane.v2i32(i8*, <2 x i32>, <2 x i32>, <2 x i32>, <2 x i32>, i32, i32) nounwind
-declare void @llvm.arm.neon.vst4lane.v2f32(i8*, <2 x float>, <2 x float>, <2 x float>, <2 x float>, i32, i32) nounwind
-declare void @llvm.arm.neon.vst4lane.v8i16(i8*, <8 x i16>, <8 x i16>, <8 x i16>, <8 x i16>, i32, i32) nounwind
-declare void @llvm.arm.neon.vst4lane.v4i32(i8*, <4 x i32>, <4 x i32>, <4 x i32>, <4 x i32>, i32, i32) nounwind
-declare void @llvm.arm.neon.vst4lane.v4f32(i8*, <4 x float>, <4 x float>, <4 x float>, <4 x float>, i32, i32) nounwind
-
-;Check for a post-increment updating store with register increment.
-define void @vst2lanei16_update(i16** %ptr, <4 x i16>* %B, i32 %inc) nounwind {
-; CHECK:         bic r1, r1, #-1073741824
-  %A = load i16*, i16** %ptr
-  %tmp0 = bitcast i16* %A to i8*
-  %tmp1 = load <4 x i16>, <4 x i16>* %B
-  call void @llvm.arm.neon.vst2lane.v4i16(i8* %tmp0, <4 x i16> %tmp1, <4 x i16> %tmp1, i32 1, i32 2)
-; CHECK:         bic r1, r1, #-1073741824
-; CHECK-NEXT:    vst2.{{[0-9]+}} {d{{[0-9]+}}[1], d{{[0-9]+}}[1]}, [r1], r2
-  %tmp2 = getelementptr i16, i16* %A, i32 %inc
-  store i16* %tmp2, i16** %ptr
-  ret void
-}
diff --git a/test/NaCl/ARM/simple-load-store_sandboxing1.ll b/test/NaCl/ARM/simple-load-store_sandboxing1.ll
deleted file mode 100644
index 91c1b83fac3..00000000000
--- a/test/NaCl/ARM/simple-load-store_sandboxing1.ll
+++ /dev/null
@@ -1,27 +0,0 @@
-; RUN: pnacl-llc -mtriple=armv7-unknown-nacl -filetype=obj %s -o - \
-; RUN:  | llvm-objdump -disassemble -triple armv7 - | FileCheck %s
-
-define void @foo(i32* %input, i32* %output) nounwind {
-entry:
-  %input.addr = alloca i32*, align 4
-  %output.addr = alloca i32*, align 4
-  store i32* %input, i32** %input.addr, align 4
-  store i32* %output, i32** %output.addr, align 4
-  %0 = load i32*, i32** %input.addr, align 4
-  %1 = load i32, i32* %0, align 4
-
-; CHECK:          bic r0, r0, #-1073741824
-; CHECK-NEXT:     ldr r0, [r0]
-
-  %add = add nsw i32 %1, 4
-  %2 = load i32*, i32** %output.addr, align 4
-  store i32 %add, i32* %2, align 4
-
-; CHECK:          bic r1, r1, #-1073741824
-; CHECK-NEXT:     str r0, [r1]
-
-  ret void
-}
-
-
-
diff --git a/test/NaCl/ARM/sp-arithmetic-sandboxing1.ll b/test/NaCl/ARM/sp-arithmetic-sandboxing1.ll
deleted file mode 100644
index 92b890cf3bb..00000000000
--- a/test/NaCl/ARM/sp-arithmetic-sandboxing1.ll
+++ /dev/null
@@ -1,28 +0,0 @@
-; RUN: pnacl-llc -mtriple=armv7-unknown-nacl -filetype=obj %s -o - \
-; RUN:  | llvm-objdump -disassemble -triple armv7 - | FileCheck %s
-
-define void @foo(i32* %input, i32* %output) nounwind {
-entry:
-  %input.addr = alloca i32*, align 4
-  %output.addr = alloca i32*, align 4
-  %temp = alloca i32, align 4
-
-; CHECK:        sub   sp, sp
-; CHECK-NEXT:   bic   sp, sp, #-1073741824
-
-  store i32* %input, i32** %input.addr, align 4
-  store i32* %output, i32** %output.addr, align 4
-  %0 = load i32*, i32** %input.addr, align 4
-  %arrayidx = getelementptr inbounds i32, i32* %0, i32 1
-  %1 = load i32, i32* %arrayidx, align 4
-  store i32 %1, i32* %temp, align 4
-  %2 = load i32, i32* %temp, align 4
-  %3 = load i32*, i32** %output.addr, align 4
-  %arrayidx1 = getelementptr inbounds i32, i32* %3, i32 0
-  store i32 %2, i32* %arrayidx1, align 4
-
-; CHECK:        add   sp, sp
-; CHECK-NEXT:   bic   sp, sp, #-1073741824
-
-  ret void
-}
diff --git a/test/NaCl/ARM/stack-change-sandboxing.ll b/test/NaCl/ARM/stack-change-sandboxing.ll
deleted file mode 100644
index 0b52eb0ca8d..00000000000
--- a/test/NaCl/ARM/stack-change-sandboxing.ll
+++ /dev/null
@@ -1,34 +0,0 @@
-; RUN: pnacl-llc -mtriple=armv7-unknown-nacl -filetype=obj %s -o - \
-; RUN:  | llvm-objdump -disassemble -triple armv7 - | FileCheck %s
-
-define i32 @foo(i32 %aa, i32 %bb) nounwind {
-entry:
-
-; CHECK:      sub sp, sp, #16
-; CHECK-NEXT: bic	sp, sp, #-1073741824
-
-  %aa.addr = alloca i32, align 4
-  %bb.addr = alloca i32, align 4
-  %cc = alloca i32, align 4
-  %dd = alloca i32, align 4
-  store i32 %aa, i32* %aa.addr, align 4
-  store i32 %bb, i32* %bb.addr, align 4
-  %0 = load i32, i32* %aa.addr, align 4
-  %1 = load i32, i32* %bb.addr, align 4
-  %mul = mul nsw i32 %0, %1
-  store i32 %mul, i32* %cc, align 4
-  %2 = load i32, i32* %aa.addr, align 4
-  %mul1 = mul nsw i32 %2, 17
-  %3 = load i32, i32* %cc, align 4
-  %sub = sub nsw i32 %mul1, %3
-  store i32 %sub, i32* %dd, align 4
-  %4 = load i32, i32* %dd, align 4
-  ret i32 %4
-
-; The nop here is to prevent add/bic to straddle a bundle boundary
-; CHECK:      nop
-; CHECK-NEXT: add sp, sp, #16
-; CHECK-NEXT: bic	sp, sp, #-1073741824
-
-}
-
diff --git a/test/NaCl/ARM/vstr-sandboxing1.ll b/test/NaCl/ARM/vstr-sandboxing1.ll
deleted file mode 100644
index b33f553c79d..00000000000
--- a/test/NaCl/ARM/vstr-sandboxing1.ll
+++ /dev/null
@@ -1,13 +0,0 @@
-; RUN: pnacl-llc -mtriple=armv7-unknown-nacl -filetype=obj %s -o - \
-; RUN:  | llvm-objdump -disassemble -triple armv7 - | FileCheck %s
-
-define void @test_vstr_sandbox(<8 x i8>* %ptr) nounwind {
-  %1 = insertelement <8 x i8> undef, i8 -128, i32 0
-  %2 = shufflevector <8 x i8> %1, <8 x i8> undef, <8 x i32> zeroinitializer
-  store <8 x i8> %2, <8 x i8>* %ptr, align 8
-; CHECK:         bic r0, r0, #-1073741824
-; CHECK-NEXT:    vstr {{[0-9a-z]+}}, [r0]
-
-  ret void
-}
-
diff --git a/test/NaCl/Bitcode/Inputs/array-of-lit.bc b/test/NaCl/Bitcode/Inputs/array-of-lit.bc
deleted file mode 100644
index c25f619af6d..00000000000
Binary files a/test/NaCl/Bitcode/Inputs/array-of-lit.bc and /dev/null differ
diff --git a/test/NaCl/Bitcode/Inputs/bad-blockinfo-numbits.tbc b/test/NaCl/Bitcode/Inputs/bad-blockinfo-numbits.tbc
deleted file mode 100644
index c70c39dcf45..00000000000
--- a/test/NaCl/Bitcode/Inputs/bad-blockinfo-numbits.tbc
+++ /dev/null
@@ -1,31 +0,0 @@
-65535,0,6;
-1,1;
-65535,17,3;
-1,4;
-7,32;
-2;
-21,0,0,0;
-7,1;
-65534;
-8,2,0,0,0;
-65535,19,4;
-5,0;
-65534;
-65535,14,3;
-1,0,102,105,98;
-65534;
-65535,12,4;
-1,3;
-65535,11,3;
-1,0;
-4,2;
-65534;
-28,2,1,36;
-11,1,2,1;
-10,2;
-2,3,2,1;
-34,0,5,1;
-2,1,5,0;
-10,1;
-65534;
-65534;
diff --git a/test/NaCl/Bitcode/Inputs/bad-branch-compare.tbc b/test/NaCl/Bitcode/Inputs/bad-branch-compare.tbc
deleted file mode 100644
index 39e267bdad1..00000000000
--- a/test/NaCl/Bitcode/Inputs/bad-branch-compare.tbc
+++ /dev/null
@@ -1,31 +0,0 @@
-65535,8,2;
-1,1;
-65535,17,2;
-1,4;
-7,32;
-2;
-21,0,0,0;
-7,1;
-65534;
-8,2,0,0,0;
-65535,19,2;
-5,0;
-65534;
-65535,14,2;
-1,0,102,105,98;
-65534;
-65535,12,2;
-1,3;
-65535,11,2;
-1,0;
-4,2;
-65534;
-28,2,1,36;
-11,1,2,3;
-10,2;
-2,3,2,1;
-34,0,5,1;
-2,1,5,0;
-10,1;
-65534;
-65534;
diff --git a/test/NaCl/Bitcode/Inputs/bad-code.tbc b/test/NaCl/Bitcode/Inputs/bad-code.tbc
deleted file mode 100644
index 2bb72e58aa2..00000000000
--- a/test/NaCl/Bitcode/Inputs/bad-code.tbc
+++ /dev/null
@@ -1,2 +0,0 @@
-+,0,2;
-65534;
diff --git a/test/NaCl/Bitcode/Inputs/bad-separator.tbc b/test/NaCl/Bitcode/Inputs/bad-separator.tbc
deleted file mode 100644
index c0e7de8a324..00000000000
--- a/test/NaCl/Bitcode/Inputs/bad-separator.tbc
+++ /dev/null
@@ -1 +0,0 @@
-65535,¸,2;
diff --git a/test/NaCl/Bitcode/alloca-operand.ll b/test/NaCl/Bitcode/alloca-operand.ll
deleted file mode 100644
index fcabead6c7a..00000000000
--- a/test/NaCl/Bitcode/alloca-operand.ll
+++ /dev/null
@@ -1,29 +0,0 @@
-; RUN: llvm-as < %s | pnacl-freeze | pnacl-bcanalyzer -dump-records \
-; RUN:              | FileCheck %s
-
-; Test that alloca's size operand is represented with a relative value
-; ID, the same as other instructions' operands.
-
-define external void @_start(i32 %arg) {
-; CHECK: <FUNCTION_BLOCK
-; CHECK: </CONSTANTS_BLOCK>
-
-  %size = mul i32 %arg, 4
-; CHECK-NEXT: <INST_BINOP
-  alloca i8, i32 %size
-; CHECK-NEXT: <INST_ALLOCA op0=1
-
-  ; Since the operand reference is a relative ID, references to %size
-  ; go up by 1 with each instruction.
-  alloca i8, i32 %size
-; CHECK-NEXT: <INST_ALLOCA op0=2
-  alloca i8, i32 %size
-; CHECK-NEXT: <INST_ALLOCA op0=3
-
-  ; Reference to a Constant operand.
-  alloca i8, i32 256
-; CHECK-NEXT: <INST_ALLOCA op0=5
-
-  ret void
-; CHECK-NEXT: <INST_RET/>
-}
diff --git a/test/NaCl/Bitcode/array-of-lit.test b/test/NaCl/Bitcode/array-of-lit.test
deleted file mode 100644
index d9ce25c22c4..00000000000
--- a/test/NaCl/Bitcode/array-of-lit.test
+++ /dev/null
@@ -1,14 +0,0 @@
-; Show that PNaCl bitcode files allow arrays of literals in abbreviations.
-
-; Show that bitcode can contain an array of literals.
-RUN: pnacl-bcdis %p/Inputs/array-of-lit.bc -no-records \
-RUN:     | FileCheck --check-prefix=ARRAY-LIT %s
-
-ARRAY-LIT: %a0 = abbrev <21, fixed(1), array(0)>;
-
-; Show that we can create the corresponding module.
-RUN: pnacl-thaw %p/Inputs/array-of-lit.bc 2>&1 \
-RUN:     | llvm-dis | FileCheck --check-prefix=CODE %s
-
-CODE: define i32 @fib(i32) {
-CODE: }
diff --git a/test/NaCl/Bitcode/bcanalyzer-width.ll b/test/NaCl/Bitcode/bcanalyzer-width.ll
deleted file mode 100644
index e9c7c289cc1..00000000000
--- a/test/NaCl/Bitcode/bcanalyzer-width.ll
+++ /dev/null
@@ -1,23 +0,0 @@
-; RUN: llvm-as < %s | pnacl-freeze | pnacl-bcanalyzer -dump-records \
-; RUN:              | FileCheck %s -check-prefix=BC
-; RUN: llvm-as < %s | pnacl-freeze \
-; RUN:              | pnacl-bcanalyzer -dump-records -operands-per-line=2 \
-; RUN:              | FileCheck %s -check-prefix=BC2
-; RUN: llvm-as < %s | pnacl-freeze \
-; RUN:              | pnacl-bcanalyzer -dump-records -operands-per-line=8 \
-; RUN:              | FileCheck %s -check-prefix=BC8
-
-; Test that the command-line option -operands-per-line works as expected.
-
-@bytes = internal global [10 x i8] c"abcdefghij"
-
-; BC: <DATA op0=97 op1=98 op2=99 op3=100 op4=101 op5=102 op6=103 op7=104 op8=105 op9=106/>
-
-; BC2: <DATA op0=97 op1=98
-; BC2:       op2=99 op3=100
-; BC2:       op4=101 op5=102
-; BC2:       op6=103 op7=104
-; BC2:       op8=105 op9=106/>
-
-; BC8: <DATA op0=97 op1=98 op2=99 op3=100 op4=101 op5=102 op6=103 op7=104
-; BC8:       op8=105 op9=106/>
diff --git a/test/NaCl/Bitcode/bccompress.ll b/test/NaCl/Bitcode/bccompress.ll
deleted file mode 100644
index aa7d318c726..00000000000
--- a/test/NaCl/Bitcode/bccompress.ll
+++ /dev/null
@@ -1,998 +0,0 @@
-; Simple test to see if pnacl-bccompress maintains bitcode.
-
-; Test 1: Show that we generate the same disassembled code.
-; RUN: llvm-as < %s | pnacl-freeze -allow-local-symbol-tables \
-; RUN:              | pnacl-bccompress \
-; RUN:              | pnacl-thaw -allow-local-symbol-tables \
-; RUN:              | llvm-dis - | FileCheck %s
-
-; Test 2: Show that both the precompressed, and the compressed versions
-; of the bitcode contain the same records.
-; RUN: llvm-as < %s | pnacl-freeze \
-; RUN:              | pnacl-bcanalyzer -operands-per-line=6 -dump-records \
-; RUN:              | FileCheck %s --check-prefix DUMP
-; RUN: llvm-as < %s | pnacl-freeze | pnacl-bccompress \
-; RUN:              | pnacl-bcanalyzer -operands-per-line=6 -dump-records \
-; RUN:              | FileCheck %s --check-prefix DUMP
-
-@bytes7 = internal global [7 x i8] c"abcdefg"
-; CHECK: @bytes7 = internal global [7 x i8] c"abcdefg"
-
-@ptr_to_ptr = internal global i32 ptrtoint (i32* @ptr to i32)
-; CHECK-NEXT: @ptr_to_ptr = internal global i32 ptrtoint (i32* @ptr to i32)
-
-@ptr_to_func = internal global i32 ptrtoint (void ()* @func to i32)
-; CHECK-NEXT: @ptr_to_func = internal global i32 ptrtoint (void ()* @func to i32)
-
-@compound = internal global <{ [3 x i8], i32 }> <{ [3 x i8] c"foo", i32 ptrtoint (void ()* @func to i32) }>
-; CHECK-NEXT: @compound = internal global <{ [3 x i8], i32 }> <{ [3 x i8] c"foo", i32 ptrtoint (void ()* @func to i32) }>
-
-@ptr = internal global i32 ptrtoint ([7 x i8]* @bytes7 to i32)
-; CHECK-NEXT: @ptr = internal global i32 ptrtoint ([7 x i8]* @bytes7 to i32)
-
-@addend_ptr = internal global i32 add (i32 ptrtoint (i32* @ptr to i32), i32 1)
-; CHECK-NEXT: @addend_ptr = internal global i32 add (i32 ptrtoint (i32* @ptr to i32), i32 1)
-
-@addend_negative = internal global i32 add (i32 ptrtoint (i32* @ptr to i32), i32 -1)
-; CHECK-NEXT: @addend_negative = internal global i32 add (i32 ptrtoint (i32* @ptr to i32), i32 -1)
-
-@addend_array1 = internal global i32 add (i32 ptrtoint ([7 x i8]* @bytes7 to i32), i32 1)
-; CHECK-NEXT: @addend_array1 = internal global i32 add (i32 ptrtoint ([7 x i8]* @bytes7 to i32), i32 1)
-
-@addend_array2 = internal global i32 add (i32 ptrtoint ([7 x i8]* @bytes7 to i32), i32 7)
-; CHECK-NEXT: @addend_array2 = internal global i32 add (i32 ptrtoint ([7 x i8]* @bytes7 to i32), i32 7)
-
-@addend_array3 = internal global i32 add (i32 ptrtoint ([7 x i8]* @bytes7 to i32), i32 9)
-; CHECK-NEXT: @addend_array3 = internal global i32 add (i32 ptrtoint ([7 x i8]* @bytes7 to i32), i32 9)
-
-@addend_struct1 = internal global i32 add (i32 ptrtoint (<{ [3 x i8], i32 }>* @compound to i32), i32 1)
-; CHECK-NEXT: @addend_struct1 = internal global i32 add (i32 ptrtoint (<{ [3 x i8], i32 }>* @compound to i32), i32 1)
-
-@addend_struct2 = internal global i32 add (i32 ptrtoint (<{ [3 x i8], i32 }>* @compound to i32), i32 4)
-; CHECK-NEXT: @addend_struct2 = internal global i32 add (i32 ptrtoint (<{ [3 x i8], i32 }>* @compound to i32), i32 4)
-
-@ptr_to_func_align = internal global i32 ptrtoint (void ()* @func to i32), align 8
-; CHECK-NEXT: @ptr_to_func_align = internal global i32 ptrtoint (void ()* @func to i32), align 8
-
-@char = internal constant [1 x i8] c"0"
-; CHECK-NEXT: @char = internal constant [1 x i8] c"0"
-
-@short = internal constant [2 x i8] zeroinitializer
-; CHECK-NEXT: @short = internal constant [2 x i8] zeroinitializer
-
-@bytes = internal global [4 x i8] c"abcd"
-; CHECK-NEXT: @bytes = internal global [4 x i8] c"abcd"
-
-declare i32 @bar(i32)
-; CHECK: declare i32 @bar(i32)
-
-define void @func() {
-  ret void
-}
-
-; CHECK:      define void @func() {
-; CHECK-NEXT:   ret void
-; CHECK-NEXT: }
-
-define void @AllocCastSimple() {
-  %1 = alloca i8, i32 4, align 8
-  %2 = ptrtoint i8* %1 to i32
-  %3 = bitcast [4 x i8]* @bytes to i32*
-  store i32 %2, i32* %3, align 1
-  ret void
-}
-
-; CHECK:      define void @AllocCastSimple() {
-; CHECK-NEXT:   %1 = alloca i8, i32 4, align 8
-; CHECK-NEXT:   %2 = ptrtoint i8* %1 to i32
-; CHECK-NEXT:   %3 = bitcast [4 x i8]* @bytes to i32*
-; CHECK-NEXT:   store i32 %2, i32* %3, align 1
-; CHECK-NEXT:   ret void
-; CHECK-NEXT: }
-
-define void @AllocCastSimpleReversed() {
-  %1 = alloca i8, i32 4, align 8
-  %2 = ptrtoint i8* %1 to i32
-  %3 = bitcast [4 x i8]* @bytes to i32*
-  store i32 %2, i32* %3, align 1
-  ret void
-}
-
-; CHECK:      define void @AllocCastSimpleReversed() {
-; CHECK-NEXT:   %1 = alloca i8, i32 4, align 8
-; CHECK-NEXT:   %2 = ptrtoint i8* %1 to i32
-; CHECK-NEXT:   %3 = bitcast [4 x i8]* @bytes to i32*
-; CHECK-NEXT:   store i32 %2, i32* %3, align 1
-; CHECK-NEXT:   ret void
-; CHECK-NEXT: }
-
-define void @AllocCastDelete() {
-  %1 = alloca i8, i32 4, align 8
-  %2 = alloca i8, i32 4, align 8
-  ret void
-}
-
-; CHECK:      define void @AllocCastDelete() {
-; CHECK-NEXT:   %1 = alloca i8, i32 4, align 8
-; CHECK-NEXT:   %2 = alloca i8, i32 4, align 8
-; CHECK-NEXT:   ret void
-; CHECK-NEXT: }
-
-define void @AllocCastOpt() {
-  %1 = alloca i8, i32 4, align 8
-  %2 = ptrtoint i8* %1 to i32
-  %3 = bitcast [4 x i8]* @bytes to i32*
-  store i32 %2, i32* %3, align 1
-  store i32 %2, i32* %3, align 1
-  ret void
-}
-
-; CHECK:      define void @AllocCastOpt() {
-; CHECK-NEXT:   %1 = alloca i8, i32 4, align 8
-; CHECK-NEXT:   %2 = ptrtoint i8* %1 to i32
-; CHECK-NEXT:   %3 = bitcast [4 x i8]* @bytes to i32*
-; CHECK-NEXT:   store i32 %2, i32* %3, align 1
-; CHECK-NEXT:   store i32 %2, i32* %3, align 1
-; CHECK-NEXT:   ret void
-; CHECK-NEXT: }
-
-define void @AllocBitcast(i32) {
-  %2 = alloca i8, i32 4, align 8
-  %3 = add i32 %0, 1
-  %4 = ptrtoint i8* %2 to i32
-  %5 = bitcast [4 x i8]* @bytes to i32*
-  store i32 %4, i32* %5, align 1
-  ret void
-}
-
-; CHECK:      define void @AllocBitcast(i32) {
-; CHECK-NEXT:   %2 = alloca i8, i32 4, align 8
-; CHECK-NEXT:   %3 = add i32 %0, 1
-; CHECK-NEXT:   %4 = ptrtoint i8* %2 to i32
-; CHECK-NEXT:   %5 = bitcast [4 x i8]* @bytes to i32*
-; CHECK-NEXT:   store i32 %4, i32* %5, align 1
-; CHECK-NEXT:   ret void
-; CHECK-NEXT: }
-
-define void @StoreGlobal() {
-  %1 = alloca i8, i32 4, align 8
-  %2 = ptrtoint [4 x i8]* @bytes to i32
-  %3 = bitcast i8* %1 to i32*
-  store i32 %2, i32* %3, align 1
-  ret void
-}
-
-; CHECK:      define void @StoreGlobal() {
-; CHECK-NEXT:   %1 = alloca i8, i32 4, align 8
-; CHECK-NEXT:   %2 = ptrtoint [4 x i8]* @bytes to i32
-; CHECK-NEXT:   %3 = bitcast i8* %1 to i32*
-; CHECK-NEXT:   store i32 %2, i32* %3, align 1
-; CHECK-NEXT:   ret void
-; CHECK-NEXT: }
-
-define void @StoreGlobalCastsReversed() {
-  %1 = alloca i8, i32 4, align 8
-  %2 = ptrtoint [4 x i8]* @bytes to i32
-  %3 = bitcast i8* %1 to i32*
-  store i32 %2, i32* %3, align 1
-  ret void
-}
-
-; CHECK:      define void @StoreGlobalCastsReversed() {
-; CHECK-NEXT:   %1 = alloca i8, i32 4, align 8
-; CHECK-NEXT:   %2 = ptrtoint [4 x i8]* @bytes to i32
-; CHECK-NEXT:   %3 = bitcast i8* %1 to i32*
-; CHECK-NEXT:   store i32 %2, i32* %3, align 1
-; CHECK-NEXT:   ret void
-; CHECK-NEXT: }
-
-define i32 @StoreGlobalCastPtr2Int() {
-  %1 = alloca i8, i32 4, align 8
-  %2 = ptrtoint [4 x i8]* @bytes to i32
-  %3 = bitcast i8* %1 to i32*
-  store i32 %2, i32* %3, align 1
-  ret i32 0
-}
-
-; CHECK:      define i32 @StoreGlobalCastPtr2Int() {
-; CHECK-NEXT:   %1 = alloca i8, i32 4, align 8
-; CHECK-NEXT:   %2 = ptrtoint [4 x i8]* @bytes to i32
-; CHECK-NEXT:   %3 = bitcast i8* %1 to i32*
-; CHECK-NEXT:   store i32 %2, i32* %3, align 1
-; CHECK-NEXT:   ret i32 0
-; CHECK-NEXT: }
-
-define void @CastAddAlloca() {
-  %1 = alloca i8, i32 4, align 8
-  %2 = add i32 1, 2
-  %3 = ptrtoint i8* %1 to i32
-  %4 = add i32 %3, 2
-  %5 = add i32 1, %3
-  %6 = add i32 %3, %3
-  ret void
-}
-
-; CHECK:      define void @CastAddAlloca() {
-; CHECK-NEXT:   %1 = alloca i8, i32 4, align 8
-; CHECK-NEXT:   %2 = add i32 1, 2
-; CHECK-NEXT:   %3 = ptrtoint i8* %1 to i32
-; CHECK-NEXT:   %4 = add i32 %3, 2
-; CHECK-NEXT:   %5 = add i32 1, %3
-; CHECK-NEXT:   %6 = add i32 %3, %3
-; CHECK-NEXT:   ret void
-; CHECK-NEXT: }
-
-define void @CastAddGlobal() {
-  %1 = add i32 1, 2
-  %2 = ptrtoint [4 x i8]* @bytes to i32
-  %3 = add i32 %2, 2
-  %4 = add i32 1, %2
-  %5 = add i32 %2, %2
-  ret void
-}
-
-; CHECK:      define void @CastAddGlobal() {
-; CHECK-NEXT:   %1 = add i32 1, 2
-; CHECK-NEXT:   %2 = ptrtoint [4 x i8]* @bytes to i32
-; CHECK-NEXT:   %3 = add i32 %2, 2
-; CHECK-NEXT:   %4 = add i32 1, %2
-; CHECK-NEXT:   %5 = add i32 %2, %2
-; CHECK-NEXT:   ret void
-; CHECK-NEXT: }
-
-define void @CastBinop() {
-  %1 = alloca i8, i32 4, align 8
-  %2 = ptrtoint i8* %1 to i32
-  %3 = ptrtoint [4 x i8]* @bytes to i32
-  %4 = sub i32 %2, %3
-  %5 = mul i32 %2, %3
-  %6 = udiv i32 %2, %3
-  %7 = urem i32 %2, %3
-  %8 = srem i32 %2, %3
-  %9 = shl i32 %2, %3
-  %10 = lshr i32 %2, %3
-  %11 = ashr i32 %2, %3
-  %12 = and i32 %2, %3
-  %13 = or i32 %2, %3
-  %14 = xor i32 %2, %3
-  ret void
-}
-
-; CHECK:      define void @CastBinop() {
-; CHECK-NEXT:   %1 = alloca i8, i32 4, align 8
-; CHECK-NEXT:   %2 = ptrtoint i8* %1 to i32
-; CHECK-NEXT:   %3 = ptrtoint [4 x i8]* @bytes to i32
-; CHECK-NEXT:   %4 = sub i32 %2, %3
-; CHECK-NEXT:   %5 = mul i32 %2, %3
-; CHECK-NEXT:   %6 = udiv i32 %2, %3
-; CHECK-NEXT:   %7 = urem i32 %2, %3
-; CHECK-NEXT:   %8 = srem i32 %2, %3
-; CHECK-NEXT:   %9 = shl i32 %2, %3
-; CHECK-NEXT:   %10 = lshr i32 %2, %3
-; CHECK-NEXT:   %11 = ashr i32 %2, %3
-; CHECK-NEXT:   %12 = and i32 %2, %3
-; CHECK-NEXT:   %13 = or i32 %2, %3
-; CHECK-NEXT:   %14 = xor i32 %2, %3
-; CHECK-NEXT:   ret void
-; CHECK-NEXT: }
-
-define void @TestSavedPtrToInt() {
-  %1 = alloca i8, i32 4, align 8
-  %2 = ptrtoint i8* %1 to i32
-  %3 = add i32 %2, 0
-  %4 = call i32 @bar(i32 %2)
-  ret void
-}
-
-; CHECK:      define void @TestSavedPtrToInt() {
-; CHECK-NEXT:   %1 = alloca i8, i32 4, align 8
-; CHECK-NEXT:   %2 = ptrtoint i8* %1 to i32
-; CHECK-NEXT:   %3 = add i32 %2, 0
-; CHECK-NEXT:   %4 = call i32 @bar(i32 %2)
-; CHECK-NEXT:   ret void
-; CHECK-NEXT: }
-
-define void @CastSelect() {
-  %1 = alloca i8, i32 4, align 8
-  %2 = select i1 true, i32 1, i32 2
-  %3 = ptrtoint i8* %1 to i32
-  %4 = select i1 true, i32 %3, i32 2
-  %5 = ptrtoint [4 x i8]* @bytes to i32
-  %6 = select i1 true, i32 1, i32 %5
-  %7 = select i1 true, i32 %3, i32 %5
-  %8 = select i1 true, i32 %5, i32 %3
-  ret void
-}
-
-; CHECK:      define void @CastSelect() {
-; CHECK-NEXT:   %1 = alloca i8, i32 4, align 8
-; CHECK-NEXT:   %2 = select i1 true, i32 1, i32 2
-; CHECK-NEXT:   %3 = ptrtoint i8* %1 to i32
-; CHECK-NEXT:   %4 = select i1 true, i32 %3, i32 2
-; CHECK-NEXT:   %5 = ptrtoint [4 x i8]* @bytes to i32
-; CHECK-NEXT:   %6 = select i1 true, i32 1, i32 %5
-; CHECK-NEXT:   %7 = select i1 true, i32 %3, i32 %5
-; CHECK-NEXT:   %8 = select i1 true, i32 %5, i32 %3
-; CHECK-NEXT:   ret void
-; CHECK-NEXT: }
-
-define void @PhiBackwardRefs(i1) {
-  %2 = alloca i8, i32 4, align 8
-  %3 = alloca i8, i32 4, align 8
-  br i1 %0, label %true, label %false
-
-true:                                             ; preds = %1
-  %4 = bitcast i8* %2 to i32*
-  %5 = load i32, i32* %4
-  %6 = ptrtoint i8* %3 to i32
-  br label %merge
-
-false:                                            ; preds = %1
-  %7 = bitcast i8* %2 to i32*
-  %8 = load i32, i32* %7
-  %9 = ptrtoint i8* %3 to i32
-  br label %merge
-
-merge:                                            ; preds = %false, %true
-  %10 = phi i32 [ %6, %true ], [ %9, %false ]
-  %11 = phi i32 [ %5, %true ], [ %8, %false ]
-  ret void
-}
-
-; CHECK:      define void @PhiBackwardRefs(i1) {
-; CHECK-NEXT:   %2 = alloca i8, i32 4, align 8
-; CHECK-NEXT:   %3 = alloca i8, i32 4, align 8
-; CHECK-NEXT:   br i1 %0, label %true, label %false
-; CHECK:      true:                                             ; preds = %1
-; CHECK-NEXT:   %4 = bitcast i8* %2 to i32*
-; CHECK-NEXT:   %5 = load i32, i32* %4
-; CHECK-NEXT:   %6 = ptrtoint i8* %3 to i32
-; CHECK-NEXT:   br label %merge
-; CHECK:      false:                                            ; preds = %1
-; CHECK-NEXT:   %7 = bitcast i8* %2 to i32*
-; CHECK-NEXT:   %8 = load i32, i32* %7
-; CHECK-NEXT:   %9 = ptrtoint i8* %3 to i32
-; CHECK-NEXT:   br label %merge
-; CHECK:      merge:                                            ; preds = %false, %true
-; CHECK-NEXT:   %10 = phi i32 [ %6, %true ], [ %9, %false ]
-; CHECK-NEXT:   %11 = phi i32 [ %5, %true ], [ %8, %false ]
-; CHECK-NEXT:   ret void
-; CHECK-NEXT: }
-
-define void @PhiForwardRefs(i1) {
-  br label %start
-
-merge:                                            ; preds = %false, %true
-  %2 = phi i32 [ %11, %true ], [ %11, %false ]
-  %3 = phi i32 [ %5, %true ], [ %7, %false ]
-  ret void
-
-true:                                             ; preds = %start
-  %4 = inttoptr i32 %9 to i32*
-  %5 = load i32, i32* %4
-  br label %merge
-
-false:                                            ; preds = %start
-  %6 = inttoptr i32 %9 to i32*
-  %7 = load i32, i32* %6
-  br label %merge
-
-start:                                            ; preds = %1
-  %8 = alloca i8, i32 4, align 8
-  %9 = ptrtoint i8* %8 to i32
-  %10 = alloca i8, i32 4, align 8
-  %11 = ptrtoint i8* %10 to i32
-  br i1 %0, label %true, label %false
-}
-
-; CHECK:      define void @PhiForwardRefs(i1) {
-; CHECK-NEXT:   br label %start
-; CHECK:      merge:                                            ; preds = %false, %true
-; CHECK-NEXT:   %2 = phi i32 [ %11, %true ], [ %11, %false ]
-; CHECK-NEXT:   %3 = phi i32 [ %5, %true ], [ %7, %false ]
-; CHECK-NEXT:   ret void
-; CHECK:      true:                                             ; preds = %start
-; CHECK-NEXT:   %4 = inttoptr i32 %9 to i32*
-; CHECK-NEXT:   %5 = load i32, i32* %4
-; CHECK-NEXT:   br label %merge
-; CHECK:      false:                                            ; preds = %start
-; CHECK-NEXT:   %6 = inttoptr i32 %9 to i32*
-; CHECK-NEXT:   %7 = load i32, i32* %6
-; CHECK-NEXT:   br label %merge
-; CHECK:      start:                                            ; preds = %1
-; CHECK-NEXT:   %8 = alloca i8, i32 4, align 8
-; CHECK-NEXT:   %9 = ptrtoint i8* %8 to i32
-; CHECK-NEXT:   %10 = alloca i8, i32 4, align 8
-; CHECK-NEXT:   %11 = ptrtoint i8* %10 to i32
-; CHECK-NEXT:   br i1 %0, label %true, label %false
-; CHECK-NEXT: }
-
-define void @PhiMergeCast(i1) {
-  %2 = alloca i8, i32 4, align 8
-  %3 = alloca i8, i32 4, align 8
-  br i1 %0, label %true, label %false
-
-true:                                             ; preds = %1
-  %4 = bitcast i8* %2 to i32*
-  %5 = load i32, i32* %4
-  %6 = ptrtoint i8* %3 to i32
-  %7 = add i32 %5, %6
-  br label %merge
-
-false:                                            ; preds = %1
-  %8 = bitcast i8* %2 to i32*
-  %9 = load i32, i32* %8
-  %10 = ptrtoint i8* %3 to i32
-  br label %merge
-
-merge:                                            ; preds = %false, %true
-  %11 = phi i32 [ %6, %true ], [ %10, %false ]
-  %12 = phi i32 [ %5, %true ], [ %9, %false ]
-  ret void
-}
-
-; CHECK:      define void @PhiMergeCast(i1) {
-; CHECK-NEXT:   %2 = alloca i8, i32 4, align 8
-; CHECK-NEXT:   %3 = alloca i8, i32 4, align 8
-; CHECK-NEXT:   br i1 %0, label %true, label %false
-; CHECK:      true:                                             ; preds = %1
-; CHECK-NEXT:   %4 = bitcast i8* %2 to i32*
-; CHECK-NEXT:   %5 = load i32, i32* %4
-; CHECK-NEXT:   %6 = ptrtoint i8* %3 to i32
-; CHECK-NEXT:   %7 = add i32 %5, %6
-; CHECK-NEXT:   br label %merge
-; CHECK:      false:                                            ; preds = %1
-; CHECK-NEXT:   %8 = bitcast i8* %2 to i32*
-; CHECK-NEXT:   %9 = load i32, i32* %8
-; CHECK-NEXT:   %10 = ptrtoint i8* %3 to i32
-; CHECK-NEXT:   br label %merge
-; CHECK:      merge:                                            ; preds = %false, %true
-; CHECK-NEXT:   %11 = phi i32 [ %6, %true ], [ %10, %false ]
-; CHECK-NEXT:   %12 = phi i32 [ %5, %true ], [ %9, %false ]
-; CHECK-NEXT:   ret void
-; CHECK-NEXT: }
-
-define void @LongReachingCasts(i1) {
-  %2 = alloca i8, i32 4, align 8
-  br i1 %0, label %Split1, label %Split2
-
-Split1:                                           ; preds = %1
-  br i1 %0, label %b1, label %b2
-
-Split2:                                           ; preds = %1
-  br i1 %0, label %b3, label %b4
-
-b1:                                               ; preds = %Split1
-  %3 = ptrtoint i8* %2 to i32
-  %4 = bitcast [4 x i8]* @bytes to i32*
-  store i32 %3, i32* %4, align 1
-  store i32 %3, i32* %4, align 1
-  ret void
-
-b2:                                               ; preds = %Split1
-  %5 = ptrtoint i8* %2 to i32
-  %6 = bitcast [4 x i8]* @bytes to i32*
-  store i32 %5, i32* %6, align 1
-  store i32 %5, i32* %6, align 1
-  ret void
-
-b3:                                               ; preds = %Split2
-  %7 = ptrtoint i8* %2 to i32
-  %8 = bitcast [4 x i8]* @bytes to i32*
-  store i32 %7, i32* %8, align 1
-  store i32 %7, i32* %8, align 1
-  ret void
-
-b4:                                               ; preds = %Split2
-  %9 = ptrtoint i8* %2 to i32
-  %10 = bitcast [4 x i8]* @bytes to i32*
-  store i32 %9, i32* %10, align 1
-  store i32 %9, i32* %10, align 1
-  ret void
-}
-
-; CHECK:      define void @LongReachingCasts(i1) {
-; CHECK-NEXT:   %2 = alloca i8, i32 4, align 8
-; CHECK-NEXT:   br i1 %0, label %Split1, label %Split2
-; CHECK:      Split1:                                           ; preds = %1
-; CHECK-NEXT:   br i1 %0, label %b1, label %b2
-; CHECK:      Split2:                                           ; preds = %1
-; CHECK-NEXT:   br i1 %0, label %b3, label %b4
-; CHECK:      b1:                                               ; preds = %Split1
-; CHECK-NEXT:   %3 = ptrtoint i8* %2 to i32
-; CHECK-NEXT:   %4 = bitcast [4 x i8]* @bytes to i32*
-; CHECK-NEXT:   store i32 %3, i32* %4, align 1
-; CHECK-NEXT:   store i32 %3, i32* %4, align 1
-; CHECK-NEXT:   ret void
-; CHECK:      b2:                                               ; preds = %Split1
-; CHECK-NEXT:   %5 = ptrtoint i8* %2 to i32
-; CHECK-NEXT:   %6 = bitcast [4 x i8]* @bytes to i32*
-; CHECK-NEXT:   store i32 %5, i32* %6, align 1
-; CHECK-NEXT:   store i32 %5, i32* %6, align 1
-; CHECK-NEXT:   ret void
-; CHECK:      b3:                                               ; preds = %Split2
-; CHECK-NEXT:   %7 = ptrtoint i8* %2 to i32
-; CHECK-NEXT:   %8 = bitcast [4 x i8]* @bytes to i32*
-; CHECK-NEXT:   store i32 %7, i32* %8, align 1
-; CHECK-NEXT:   store i32 %7, i32* %8, align 1
-; CHECK-NEXT:   ret void
-; CHECK:      b4:                                               ; preds = %Split2
-; CHECK-NEXT:   %9 = ptrtoint i8* %2 to i32
-; CHECK-NEXT:   %10 = bitcast [4 x i8]* @bytes to i32*
-; CHECK-NEXT:   store i32 %9, i32* %10, align 1
-; CHECK-NEXT:   store i32 %9, i32* %10, align 1
-; CHECK-NEXT:   ret void
-; CHECK-NEXT: }
-
-define void @SwitchVariable(i32) {
-  switch i32 %0, label %l1 [
-    i32 1, label %l2
-    i32 2, label %l2
-    i32 4, label %l3
-    i32 5, label %l3
-  ]
-                                                  ; No predecessors!
-  br label %end
-
-l1:                                               ; preds = %1
-  br label %end
-
-l2:                                               ; preds = %1, %1
-  br label %end
-
-l3:                                               ; preds = %1, %1
-  br label %end
-
-end:                                              ; preds = %l3, %l2, %l1, %2
-  ret void
-}
-
-; CHECK:      define void @SwitchVariable(i32) {
-; CHECK-NEXT:   switch i32 %0, label %l1 [
-; CHECK-NEXT:     i32 1, label %l2
-; CHECK-NEXT:     i32 2, label %l2
-; CHECK-NEXT:     i32 4, label %l3
-; CHECK-NEXT:     i32 5, label %l3
-; CHECK-NEXT:   ]
-; CHECK-NEXT:                                                   ; No predecessors!
-; CHECK-NEXT:   br label %end
-; CHECK:      l1:                                               ; preds = %1
-; CHECK-NEXT:   br label %end
-; CHECK:      l2:                                               ; preds = %1, %1
-; CHECK-NEXT:   br label %end
-; CHECK:      l3:                                               ; preds = %1, %1
-; CHECK-NEXT:   br label %end
-; CHECK:      end:                                              ; preds = %l3, %l2, %l1, %2
-; CHECK-NEXT:   ret void
-; CHECK-NEXT: }
-
-; DUMP:     PNaCl Version: 2
-
-; DUMP:     <MODULE_BLOCK>
-; DUMP-DAG:   <VERSION op0=1/>
-; DUMP-DAG:   <BLOCKINFO_BLOCK/>
-; DUMP:       <TYPE_BLOCK_ID>
-; DUMP-NEXT:    <NUMENTRY op0=9/>
-; DUMP-NEXT:    <INTEGER op0=32/>
-; DUMP-NEXT:    <VOID/>
-; DUMP-NEXT:    <INTEGER op0=8/>
-; DUMP-NEXT:    <INTEGER op0=1/>
-; DUMP-NEXT:    <FUNCTION op0=0 op1=1/>
-; DUMP-NEXT:    <FUNCTION op0=0 op1=1 op2=3/>
-; DUMP-NEXT:    <FUNCTION op0=0 op1=0 op2=0/>
-; DUMP-NEXT:    <FUNCTION op0=0 op1=1 op2=0/>
-; DUMP-NEXT:    <FUNCTION op0=0 op1=0/>
-; DUMP-NEXT:  </TYPE_BLOCK_ID>
-; DUMP-NEXT:  <FUNCTION op0=6 op1=0 op2=1 op3=0/>
-; DUMP-NEXT:  <FUNCTION op0=4 op1=0 op2=0 op3=0/>
-; DUMP-NEXT:  <FUNCTION op0=4 op1=0 op2=0 op3=0/>
-; DUMP-NEXT:  <FUNCTION op0=4 op1=0 op2=0 op3=0/>
-; DUMP-NEXT:  <FUNCTION op0=4 op1=0 op2=0 op3=0/>
-; DUMP-NEXT:  <FUNCTION op0=4 op1=0 op2=0 op3=0/>
-; DUMP-NEXT:  <FUNCTION op0=7 op1=0 op2=0 op3=0/>
-; DUMP-NEXT:  <FUNCTION op0=4 op1=0 op2=0 op3=0/>
-; DUMP-NEXT:  <FUNCTION op0=4 op1=0 op2=0 op3=0/>
-; DUMP-NEXT:  <FUNCTION op0=8 op1=0 op2=0 op3=0/>
-; DUMP-NEXT:  <FUNCTION op0=4 op1=0 op2=0 op3=0/>
-; DUMP-NEXT:  <FUNCTION op0=4 op1=0 op2=0 op3=0/>
-; DUMP-NEXT:  <FUNCTION op0=4 op1=0 op2=0 op3=0/>
-; DUMP-NEXT:  <FUNCTION op0=4 op1=0 op2=0 op3=0/>
-; DUMP-NEXT:  <FUNCTION op0=4 op1=0 op2=0 op3=0/>
-; DUMP-NEXT:  <FUNCTION op0=5 op1=0 op2=0 op3=0/>
-; DUMP-NEXT:  <FUNCTION op0=5 op1=0 op2=0 op3=0/>
-; DUMP-NEXT:  <FUNCTION op0=5 op1=0 op2=0 op3=0/>
-; DUMP-NEXT:  <FUNCTION op0=5 op1=0 op2=0 op3=0/>
-; DUMP-NEXT:  <FUNCTION op0=7 op1=0 op2=0 op3=0/>
-; DUMP-NEXT:  <GLOBALVAR_BLOCK>
-; DUMP-NEXT:    <COUNT op0=16/>
-; DUMP-NEXT:    <VAR op0=0 op1=0/>
-; DUMP-NEXT:    <DATA op0=97 op1=98 op2=99 op3=100 op4=101 op5=102
-; DUMP-NEXT:            op6=103/>
-; DUMP-NEXT:    <VAR op0=0 op1=0/>
-; DUMP-NEXT:    <RELOC op0=24/>
-; DUMP-NEXT:    <VAR op0=0 op1=0/>
-; DUMP-NEXT:    <RELOC op0=1/>
-; DUMP-NEXT:    <VAR op0=0 op1=0/>
-; DUMP-NEXT:    <COMPOUND op0=2/>
-; DUMP-NEXT:    <DATA op0=102 op1=111 op2=111/>
-; DUMP-NEXT:    <RELOC op0=1/>
-; DUMP-NEXT:    <VAR op0=0 op1=0/>
-; DUMP-NEXT:    <RELOC op0=20/>
-; DUMP-NEXT:    <VAR op0=0 op1=0/>
-; DUMP-NEXT:    <RELOC op0=24 op1=1/>
-; DUMP-NEXT:    <VAR op0=0 op1=0/>
-; DUMP-NEXT:    <RELOC op0=24 op1=4294967295/>
-; DUMP-NEXT:    <VAR op0=0 op1=0/>
-; DUMP-NEXT:    <RELOC op0=20 op1=1/>
-; DUMP-NEXT:    <VAR op0=0 op1=0/>
-; DUMP-NEXT:    <RELOC op0=20 op1=7/>
-; DUMP-NEXT:    <VAR op0=0 op1=0/>
-; DUMP-NEXT:    <RELOC op0=20 op1=9/>
-; DUMP-NEXT:    <VAR op0=0 op1=0/>
-; DUMP-NEXT:    <RELOC op0=23 op1=1/>
-; DUMP-NEXT:    <VAR op0=0 op1=0/>
-; DUMP-NEXT:    <RELOC op0=23 op1=4/>
-; DUMP-NEXT:    <VAR op0=4 op1=0/>
-; DUMP-NEXT:    <RELOC op0=1/>
-; DUMP-NEXT:    <VAR op0=0 op1=1/>
-; DUMP-NEXT:    <DATA op0=48/>
-; DUMP-NEXT:    <VAR op0=0 op1=1/>
-; DUMP-NEXT:    <ZEROFILL op0=2/>
-; DUMP-NEXT:    <VAR op0=0 op1=0/>
-; DUMP-NEXT:    <DATA op0=97 op1=98 op2=99 op3=100/>
-; DUMP-NEXT:  </GLOBALVAR_BLOCK>
-; DUMP-NEXT:  <VALUE_SYMTAB>
-; DUMP-NEXT:    <ENTRY op0=2 op1=65 op2=108 op3=108 op4=111 op5=99
-; DUMP-NEXT:             op6=67 op7=97 op8=115 op9=116 op10=83 op11=105
-; DUMP-NEXT:             op12=109 op13=112 op14=108 op15=101/>
-; DUMP-NEXT:    <ENTRY op0=10 op1=67 op2=97 op3=115 op4=116 op5=65
-; DUMP-NEXT:             op6=100 op7=100 op8=65 op9=108 op10=108 op11=111
-; DUMP-NEXT:             op12=99 op13=97/>
-; DUMP-NEXT:    <ENTRY op0=20 op1=98 op2=121 op3=116 op4=101 op5=115
-; DUMP-NEXT:             op6=55/>
-; DUMP-NEXT:    <ENTRY op0=23 op1=99 op2=111 op3=109 op4=112 op5=111
-; DUMP-NEXT:             op6=117 op7=110 op8=100/>
-; DUMP-NEXT:    <ENTRY op0=5 op1=65 op2=108 op3=108 op4=111 op5=99
-; DUMP-NEXT:             op6=67 op7=97 op8=115 op9=116 op10=79 op11=112
-; DUMP-NEXT:             op12=116/>
-; DUMP-NEXT:    <ENTRY op0=32 op1=112 op2=116 op3=114 op4=95 op5=116
-; DUMP-NEXT:             op6=111 op7=95 op8=102 op9=117 op10=110 op11=99
-; DUMP-NEXT:             op12=95 op13=97 op14=108 op15=105 op16=103 op17=110/>
-; DUMP-NEXT:    <ENTRY op0=18 op1=76 op2=111 op3=110 op4=103 op5=82
-; DUMP-NEXT:             op6=101 op7=97 op8=99 op9=104 op10=105 op11=110
-; DUMP-NEXT:             op12=103 op13=67 op14=97 op15=115 op16=116 op17=115/>
-; DUMP-NEXT:    <ENTRY op0=15 op1=80 op2=104 op3=105 op4=66 op5=97
-; DUMP-NEXT:             op6=99 op7=107 op8=119 op9=97 op10=114 op11=100
-; DUMP-NEXT:             op12=82 op13=101 op14=102 op15=115/>
-; DUMP-NEXT:    <ENTRY op0=26 op1=97 op2=100 op3=100 op4=101 op5=110
-; DUMP-NEXT:             op6=100 op7=95 op8=110 op9=101 op10=103 op11=97
-; DUMP-NEXT:             op12=116 op13=105 op14=118 op15=101/>
-; DUMP-NEXT:    <ENTRY op0=6 op1=65 op2=108 op3=108 op4=111 op5=99
-; DUMP-NEXT:             op6=66 op7=105 op8=116 op9=99 op10=97 op11=115
-; DUMP-NEXT:             op12=116/>
-; DUMP-NEXT:    <ENTRY op0=24 op1=112 op2=116 op3=114/>
-; DUMP-NEXT:    <ENTRY op0=25 op1=97 op2=100 op3=100 op4=101 op5=110
-; DUMP-NEXT:             op6=100 op7=95 op8=112 op9=116 op10=114/>
-; DUMP-NEXT:    <ENTRY op0=19 op1=83 op2=119 op3=105 op4=116 op5=99
-; DUMP-NEXT:             op6=104 op7=86 op8=97 op9=114 op10=105 op11=97
-; DUMP-NEXT:             op12=98 op13=108 op14=101/>
-; DUMP-NEXT:    <ENTRY op0=0 op1=98 op2=97 op3=114/>
-; DUMP-NEXT:    <ENTRY op0=33 op1=99 op2=104 op3=97 op4=114/>
-; DUMP-NEXT:    <ENTRY op0=3 op1=65 op2=108 op3=108 op4=111 op5=99
-; DUMP-NEXT:             op6=67 op7=97 op8=115 op9=116 op10=83 op11=105
-; DUMP-NEXT:             op12=109 op13=112 op14=108 op15=101 op16=82 op17=101
-; DUMP-NEXT:             op18=118 op19=101 op20=114 op21=115 op22=101 op23=100/>
-; DUMP-NEXT:    <ENTRY op0=22 op1=112 op2=116 op3=114 op4=95 op5=116
-; DUMP-NEXT:             op6=111 op7=95 op8=102 op9=117 op10=110 op11=99/>
-; DUMP-NEXT:    <ENTRY op0=12 op1=67 op2=97 op3=115 op4=116 op5=66
-; DUMP-NEXT:             op6=105 op7=110 op8=111 op9=112/>
-; DUMP-NEXT:    <ENTRY op0=11 op1=67 op2=97 op3=115 op4=116 op5=65
-; DUMP-NEXT:             op6=100 op7=100 op8=71 op9=108 op10=111 op11=98
-; DUMP-NEXT:             op12=97 op13=108/>
-; DUMP-NEXT:    <ENTRY op0=16 op1=80 op2=104 op3=105 op4=70 op5=111
-; DUMP-NEXT:             op6=114 op7=119 op8=97 op9=114 op10=100 op11=82
-; DUMP-NEXT:             op12=101 op13=102 op14=115/>
-; DUMP-NEXT:    <ENTRY op0=35 op1=98 op2=121 op3=116 op4=101 op5=115/>
-; DUMP-NEXT:    <ENTRY op0=4 op1=65 op2=108 op3=108 op4=111 op5=99
-; DUMP-NEXT:             op6=67 op7=97 op8=115 op9=116 op10=68 op11=101
-; DUMP-NEXT:             op12=108 op13=101 op14=116 op15=101/>
-; DUMP-NEXT:    <ENTRY op0=14 op1=67 op2=97 op3=115 op4=116 op5=83
-; DUMP-NEXT:             op6=101 op7=108 op8=101 op9=99 op10=116/>
-; DUMP-NEXT:    <ENTRY op0=1 op1=102 op2=117 op3=110 op4=99/>
-; DUMP-NEXT:    <ENTRY op0=21 op1=112 op2=116 op3=114 op4=95 op5=116
-; DUMP-NEXT:             op6=111 op7=95 op8=112 op9=116 op10=114/>
-; DUMP-NEXT:    <ENTRY op0=27 op1=97 op2=100 op3=100 op4=101 op5=110
-; DUMP-NEXT:             op6=100 op7=95 op8=97 op9=114 op10=114 op11=97
-; DUMP-NEXT:             op12=121 op13=49/>
-; DUMP-NEXT:    <ENTRY op0=28 op1=97 op2=100 op3=100 op4=101 op5=110
-; DUMP-NEXT:             op6=100 op7=95 op8=97 op9=114 op10=114 op11=97
-; DUMP-NEXT:             op12=121 op13=50/>
-; DUMP-NEXT:    <ENTRY op0=29 op1=97 op2=100 op3=100 op4=101 op5=110
-; DUMP-NEXT:             op6=100 op7=95 op8=97 op9=114 op10=114 op11=97
-; DUMP-NEXT:             op12=121 op13=51/>
-; DUMP-NEXT:    <ENTRY op0=34 op1=115 op2=104 op3=111 op4=114 op5=116/>
-; DUMP-NEXT:    <ENTRY op0=30 op1=97 op2=100 op3=100 op4=101 op5=110
-; DUMP-NEXT:             op6=100 op7=95 op8=115 op9=116 op10=114 op11=117
-; DUMP-NEXT:             op12=99 op13=116 op14=49/>
-; DUMP-NEXT:    <ENTRY op0=31 op1=97 op2=100 op3=100 op4=101 op5=110
-; DUMP-NEXT:             op6=100 op7=95 op8=115 op9=116 op10=114 op11=117
-; DUMP-NEXT:             op12=99 op13=116 op14=50/>
-; DUMP-NEXT:    <ENTRY op0=13 op1=84 op2=101 op3=115 op4=116 op5=83
-; DUMP-NEXT:             op6=97 op7=118 op8=101 op9=100 op10=80 op11=116
-; DUMP-NEXT:             op12=114 op13=84 op14=111 op15=73 op16=110 op17=116/>
-; DUMP-NEXT:    <ENTRY op0=17 op1=80 op2=104 op3=105 op4=77 op5=101
-; DUMP-NEXT:             op6=114 op7=103 op8=101 op9=67 op10=97 op11=115
-; DUMP-NEXT:             op12=116/>
-; DUMP-NEXT:    <ENTRY op0=8 op1=83 op2=116 op3=111 op4=114 op5=101
-; DUMP-NEXT:             op6=71 op7=108 op8=111 op9=98 op10=97 op11=108
-; DUMP-NEXT:             op12=67 op13=97 op14=115 op15=116 op16=115 op17=82
-; DUMP-NEXT:             op18=101 op19=118 op20=101 op21=114 op22=115 op23=101
-; DUMP-NEXT:             op24=100/>
-; DUMP-NEXT:    <ENTRY op0=7 op1=83 op2=116 op3=111 op4=114 op5=101
-; DUMP-NEXT:             op6=71 op7=108 op8=111 op9=98 op10=97 op11=108/>
-; DUMP-NEXT:    <ENTRY op0=9 op1=83 op2=116 op3=111 op4=114 op5=101
-; DUMP-NEXT:             op6=71 op7=108 op8=111 op9=98 op10=97 op11=108
-; DUMP-NEXT:             op12=67 op13=97 op14=115 op15=116 op16=80 op17=116
-; DUMP-NEXT:             op18=114 op19=50 op20=73 op21=110 op22=116/>
-; DUMP-NEXT:  </VALUE_SYMTAB>
-; DUMP-NEXT:  <FUNCTION_BLOCK>
-; DUMP-NEXT:    <DECLAREBLOCKS op0=1/>
-; DUMP-NEXT:    <INST_RET/>
-; DUMP-NEXT:  </FUNCTION_BLOCK>
-; DUMP-NEXT:  <FUNCTION_BLOCK>
-; DUMP-NEXT:    <DECLAREBLOCKS op0=1/>
-; DUMP-NEXT:    <CONSTANTS_BLOCK>
-; DUMP-NEXT:      <SETTYPE op0=0/>
-; DUMP-NEXT:      <INTEGER op0=8/>
-; DUMP-NEXT:    </CONSTANTS_BLOCK>
-; DUMP-NEXT:    <INST_ALLOCA op0=1 op1=4/>
-; DUMP-NEXT:    <INST_STORE op0=3 op1=1 op2=1/>
-; DUMP-NEXT:    <INST_RET/>
-; DUMP-NEXT:  </FUNCTION_BLOCK>
-; DUMP-NEXT:  <FUNCTION_BLOCK>
-; DUMP-NEXT:    <DECLAREBLOCKS op0=1/>
-; DUMP-NEXT:    <CONSTANTS_BLOCK>
-; DUMP-NEXT:      <SETTYPE op0=0/>
-; DUMP-NEXT:      <INTEGER op0=8/>
-; DUMP-NEXT:    </CONSTANTS_BLOCK>
-; DUMP-NEXT:    <INST_ALLOCA op0=1 op1=4/>
-; DUMP-NEXT:    <INST_STORE op0=3 op1=1 op2=1/>
-; DUMP-NEXT:    <INST_RET/>
-; DUMP-NEXT:  </FUNCTION_BLOCK>
-; DUMP-NEXT:  <FUNCTION_BLOCK>
-; DUMP-NEXT:    <DECLAREBLOCKS op0=1/>
-; DUMP-NEXT:    <CONSTANTS_BLOCK>
-; DUMP-NEXT:      <SETTYPE op0=0/>
-; DUMP-NEXT:      <INTEGER op0=8/>
-; DUMP-NEXT:    </CONSTANTS_BLOCK>
-; DUMP-NEXT:    <INST_ALLOCA op0=1 op1=4/>
-; DUMP-NEXT:    <INST_ALLOCA op0=2 op1=4/>
-; DUMP-NEXT:    <INST_RET/>
-; DUMP-NEXT:  </FUNCTION_BLOCK>
-; DUMP-NEXT:  <FUNCTION_BLOCK>
-; DUMP-NEXT:    <DECLAREBLOCKS op0=1/>
-; DUMP-NEXT:    <CONSTANTS_BLOCK>
-; DUMP-NEXT:      <SETTYPE op0=0/>
-; DUMP-NEXT:      <INTEGER op0=8/>
-; DUMP-NEXT:    </CONSTANTS_BLOCK>
-; DUMP-NEXT:    <INST_ALLOCA op0=1 op1=4/>
-; DUMP-NEXT:    <INST_STORE op0=3 op1=1 op2=1/>
-; DUMP-NEXT:    <INST_STORE op0=3 op1=1 op2=1/>
-; DUMP-NEXT:    <INST_RET/>
-; DUMP-NEXT:  </FUNCTION_BLOCK>
-; DUMP-NEXT:  <FUNCTION_BLOCK>
-; DUMP-NEXT:    <DECLAREBLOCKS op0=1/>
-; DUMP-NEXT:    <CONSTANTS_BLOCK>
-; DUMP-NEXT:      <SETTYPE op0=0/>
-; DUMP-NEXT:      <INTEGER op0=8/>
-; DUMP-NEXT:      <INTEGER op0=2/>
-; DUMP-NEXT:    </CONSTANTS_BLOCK>
-; DUMP-NEXT:    <INST_ALLOCA op0=2 op1=4/>
-; DUMP-NEXT:    <INST_BINOP op0=4 op1=2 op2=0/>
-; DUMP-NEXT:    <INST_STORE op0=6 op1=2 op2=1/>
-; DUMP-NEXT:    <INST_RET/>
-; DUMP-NEXT:  </FUNCTION_BLOCK>
-; DUMP-NEXT:  <FUNCTION_BLOCK>
-; DUMP-NEXT:    <DECLAREBLOCKS op0=1/>
-; DUMP-NEXT:    <CONSTANTS_BLOCK>
-; DUMP-NEXT:      <SETTYPE op0=0/>
-; DUMP-NEXT:      <INTEGER op0=8/>
-; DUMP-NEXT:    </CONSTANTS_BLOCK>
-; DUMP-NEXT:    <INST_ALLOCA op0=1 op1=4/>
-; DUMP-NEXT:    <INST_STORE op0=1 op1=3 op2=1/>
-; DUMP-NEXT:    <INST_RET/>
-; DUMP-NEXT:  </FUNCTION_BLOCK>
-; DUMP-NEXT:  <FUNCTION_BLOCK>
-; DUMP-NEXT:    <DECLAREBLOCKS op0=1/>
-; DUMP-NEXT:    <CONSTANTS_BLOCK>
-; DUMP-NEXT:      <SETTYPE op0=0/>
-; DUMP-NEXT:      <INTEGER op0=8/>
-; DUMP-NEXT:    </CONSTANTS_BLOCK>
-; DUMP-NEXT:    <INST_ALLOCA op0=1 op1=4/>
-; DUMP-NEXT:    <INST_STORE op0=1 op1=3 op2=1/>
-; DUMP-NEXT:    <INST_RET/>
-; DUMP-NEXT:  </FUNCTION_BLOCK>
-; DUMP-NEXT:  <FUNCTION_BLOCK>
-; DUMP-NEXT:    <DECLAREBLOCKS op0=1/>
-; DUMP-NEXT:    <CONSTANTS_BLOCK>
-; DUMP-NEXT:      <SETTYPE op0=0/>
-; DUMP-NEXT:      <INTEGER op0=8/>
-; DUMP-NEXT:      <INTEGER op0=0/>
-; DUMP-NEXT:    </CONSTANTS_BLOCK>
-; DUMP-NEXT:    <INST_ALLOCA op0=2 op1=4/>
-; DUMP-NEXT:    <INST_STORE op0=1 op1=4 op2=1/>
-; DUMP-NEXT:    <INST_RET op0=2/>
-; DUMP-NEXT:  </FUNCTION_BLOCK>
-; DUMP-NEXT:  <FUNCTION_BLOCK>
-; DUMP-NEXT:    <DECLAREBLOCKS op0=1/>
-; DUMP-NEXT:    <CONSTANTS_BLOCK>
-; DUMP-NEXT:      <SETTYPE op0=0/>
-; DUMP-NEXT:      <INTEGER op0=2/>
-; DUMP-NEXT:      <INTEGER op0=4/>
-; DUMP-NEXT:      <INTEGER op0=8/>
-; DUMP-NEXT:    </CONSTANTS_BLOCK>
-; DUMP-NEXT:    <INST_ALLOCA op0=1 op1=4/>
-; DUMP-NEXT:    <INST_BINOP op0=4 op1=3 op2=0/>
-; DUMP-NEXT:    <INST_BINOP op0=2 op1=4 op2=0/>
-; DUMP-NEXT:    <INST_BINOP op0=6 op1=3 op2=0/>
-; DUMP-NEXT:    <INST_BINOP op0=4 op1=4 op2=0/>
-; DUMP-NEXT:    <INST_RET/>
-; DUMP-NEXT:  </FUNCTION_BLOCK>
-; DUMP-NEXT:  <FUNCTION_BLOCK>
-; DUMP-NEXT:    <DECLAREBLOCKS op0=1/>
-; DUMP-NEXT:    <CONSTANTS_BLOCK>
-; DUMP-NEXT:      <SETTYPE op0=0/>
-; DUMP-NEXT:      <INTEGER op0=2/>
-; DUMP-NEXT:      <INTEGER op0=4/>
-; DUMP-NEXT:    </CONSTANTS_BLOCK>
-; DUMP-NEXT:    <INST_BINOP op0=2 op1=1 op2=0/>
-; DUMP-NEXT:    <INST_BINOP op0=4 op1=2 op2=0/>
-; DUMP-NEXT:    <INST_BINOP op0=4 op1=5 op2=0/>
-; DUMP-NEXT:    <INST_BINOP op0=6 op1=6 op2=0/>
-; DUMP-NEXT:    <INST_RET/>
-; DUMP-NEXT:  </FUNCTION_BLOCK>
-; DUMP-NEXT:  <FUNCTION_BLOCK>
-; DUMP-NEXT:    <DECLAREBLOCKS op0=1/>
-; DUMP-NEXT:    <CONSTANTS_BLOCK>
-; DUMP-NEXT:      <SETTYPE op0=0/>
-; DUMP-NEXT:      <INTEGER op0=8/>
-; DUMP-NEXT:    </CONSTANTS_BLOCK>
-; DUMP-NEXT:    <INST_ALLOCA op0=1 op1=4/>
-; DUMP-NEXT:    <INST_BINOP op0=1 op1=3 op2=1/>
-; DUMP-NEXT:    <INST_BINOP op0=2 op1=4 op2=2/>
-; DUMP-NEXT:    <INST_BINOP op0=3 op1=5 op2=3/>
-; DUMP-NEXT:    <INST_BINOP op0=4 op1=6 op2=5/>
-; DUMP-NEXT:    <INST_BINOP op0=5 op1=7 op2=6/>
-; DUMP-NEXT:    <INST_BINOP op0=6 op1=8 op2=7/>
-; DUMP-NEXT:    <INST_BINOP op0=7 op1=9 op2=8/>
-; DUMP-NEXT:    <INST_BINOP op0=8 op1=10 op2=9/>
-; DUMP-NEXT:    <INST_BINOP op0=9 op1=11 op2=10/>
-; DUMP-NEXT:    <INST_BINOP op0=10 op1=12 op2=11/>
-; DUMP-NEXT:    <INST_BINOP op0=11 op1=13 op2=12/>
-; DUMP-NEXT:    <INST_RET/>
-; DUMP-NEXT:  </FUNCTION_BLOCK>
-; DUMP-NEXT:  <FUNCTION_BLOCK>
-; DUMP-NEXT:    <DECLAREBLOCKS op0=1/>
-; DUMP-NEXT:    <CONSTANTS_BLOCK>
-; DUMP-NEXT:      <SETTYPE op0=0/>
-; DUMP-NEXT:      <INTEGER op0=8/>
-; DUMP-NEXT:      <INTEGER op0=0/>
-; DUMP-NEXT:    </CONSTANTS_BLOCK>
-; DUMP-NEXT:    <INST_ALLOCA op0=2 op1=4/>
-; DUMP-NEXT:    <INST_BINOP op0=1 op1=2 op2=0/>
-; DUMP-NEXT:    <INST_CALL op0=0 op1=40 op2=2/>
-; DUMP-NEXT:    <INST_RET/>
-; DUMP-NEXT:  </FUNCTION_BLOCK>
-; DUMP-NEXT:  <FUNCTION_BLOCK>
-; DUMP-NEXT:    <DECLAREBLOCKS op0=1/>
-; DUMP-NEXT:    <CONSTANTS_BLOCK>
-; DUMP-NEXT:      <SETTYPE op0=0/>
-; DUMP-NEXT:      <INTEGER op0=2/>
-; DUMP-NEXT:      <INTEGER op0=4/>
-; DUMP-NEXT:      <INTEGER op0=8/>
-; DUMP-NEXT:      <SETTYPE op0=3/>
-; DUMP-NEXT:      <INTEGER op0=3/>
-; DUMP-NEXT:    </CONSTANTS_BLOCK>
-; DUMP-NEXT:    <INST_ALLOCA op0=2 op1=4/>
-; DUMP-NEXT:    <INST_VSELECT op0=5 op1=4 op2=2/>
-; DUMP-NEXT:    <INST_VSELECT op0=2 op1=5 op2=3/>
-; DUMP-NEXT:    <INST_VSELECT op0=7 op1=8 op2=4/>
-; DUMP-NEXT:    <INST_VSELECT op0=4 op1=9 op2=5/>
-; DUMP-NEXT:    <INST_VSELECT op0=10 op1=5 op2=6/>
-; DUMP-NEXT:    <INST_RET/>
-; DUMP-NEXT:  </FUNCTION_BLOCK>
-; DUMP-NEXT:  <FUNCTION_BLOCK>
-; DUMP-NEXT:    <DECLAREBLOCKS op0=4/>
-; DUMP-NEXT:    <CONSTANTS_BLOCK>
-; DUMP-NEXT:      <SETTYPE op0=0/>
-; DUMP-NEXT:      <INTEGER op0=8/>
-; DUMP-NEXT:    </CONSTANTS_BLOCK>
-; DUMP-NEXT:    <INST_ALLOCA op0=1 op1=4/>
-; DUMP-NEXT:    <INST_ALLOCA op0=2 op1=4/>
-; DUMP-NEXT:    <INST_BR op0=1 op1=2 op2=4/>
-; DUMP-NEXT:    <INST_LOAD op0=2 op1=0 op2=0/>
-; DUMP-NEXT:    <INST_BR op0=3/>
-; DUMP-NEXT:    <INST_LOAD op0=3 op1=0 op2=0/>
-; DUMP-NEXT:    <INST_BR op0=3/>
-; DUMP-NEXT:    <INST_PHI op0=0 op1=6 op2=1 op3=6 op4=2/>
-; DUMP-NEXT:    <INST_PHI op0=0 op1=6 op2=1 op3=4 op4=2/>
-; DUMP-NEXT:    <INST_RET/>
-; DUMP-NEXT:  </FUNCTION_BLOCK>
-; DUMP-NEXT:  <FUNCTION_BLOCK>
-; DUMP-NEXT:    <DECLAREBLOCKS op0=5/>
-; DUMP-NEXT:    <CONSTANTS_BLOCK>
-; DUMP-NEXT:      <SETTYPE op0=0/>
-; DUMP-NEXT:      <INTEGER op0=8/>
-; DUMP-NEXT:    </CONSTANTS_BLOCK>
-; DUMP-NEXT:    <INST_BR op0=4/>
-; DUMP-NEXT:    <FORWARDTYPEREF op0=43 op1=0/>
-; DUMP-NEXT:    <INST_PHI op0=0 op1=11 op2=2 op3=11 op4=3/>
-; DUMP-NEXT:    <FORWARDTYPEREF op0=40 op1=0/>
-; DUMP-NEXT:    <FORWARDTYPEREF op0=41 op1=0/>
-; DUMP-NEXT:    <INST_PHI op0=0 op1=3 op2=2 op3=5 op4=3/>
-; DUMP-NEXT:    <INST_RET/>
-; DUMP-NEXT:    <FORWARDTYPEREF op0=42 op1=0/>
-; DUMP-NEXT:    <INST_LOAD op0=4294967294 op1=0 op2=0/>
-; DUMP-NEXT:    <INST_BR op0=1/>
-; DUMP-NEXT:    <INST_LOAD op0=4294967295 op1=0 op2=0/>
-; DUMP-NEXT:    <INST_BR op0=1/>
-; DUMP-NEXT:    <INST_ALLOCA op0=5 op1=4/>
-; DUMP-NEXT:    <INST_ALLOCA op0=6 op1=4/>
-; DUMP-NEXT:    <INST_BR op0=2 op1=3 op2=8/>
-; DUMP-NEXT:  </FUNCTION_BLOCK>
-; DUMP-NEXT:  <FUNCTION_BLOCK>
-; DUMP-NEXT:    <DECLAREBLOCKS op0=4/>
-; DUMP-NEXT:    <CONSTANTS_BLOCK>
-; DUMP-NEXT:      <SETTYPE op0=0/>
-; DUMP-NEXT:      <INTEGER op0=8/>
-; DUMP-NEXT:    </CONSTANTS_BLOCK>
-; DUMP-NEXT:    <INST_ALLOCA op0=1 op1=4/>
-; DUMP-NEXT:    <INST_ALLOCA op0=2 op1=4/>
-; DUMP-NEXT:    <INST_BR op0=1 op1=2 op2=4/>
-; DUMP-NEXT:    <INST_LOAD op0=2 op1=0 op2=0/>
-; DUMP-NEXT:    <INST_BINOP op0=1 op1=2 op2=0/>
-; DUMP-NEXT:    <INST_BR op0=3/>
-; DUMP-NEXT:    <INST_LOAD op0=4 op1=0 op2=0/>
-; DUMP-NEXT:    <INST_BR op0=3/>
-; DUMP-NEXT:    <INST_PHI op0=0 op1=8 op2=1 op3=8 op4=2/>
-; DUMP-NEXT:    <INST_PHI op0=0 op1=8 op2=1 op3=4 op4=2/>
-; DUMP-NEXT:    <INST_RET/>
-; DUMP-NEXT:  </FUNCTION_BLOCK>
-; DUMP-NEXT:  <FUNCTION_BLOCK>
-; DUMP-NEXT:    <DECLAREBLOCKS op0=7/>
-; DUMP-NEXT:    <CONSTANTS_BLOCK>
-; DUMP-NEXT:      <SETTYPE op0=0/>
-; DUMP-NEXT:      <INTEGER op0=8/>
-; DUMP-NEXT:    </CONSTANTS_BLOCK>
-; DUMP-NEXT:    <INST_ALLOCA op0=1 op1=4/>
-; DUMP-NEXT:    <INST_BR op0=1 op1=2 op2=3/>
-; DUMP-NEXT:    <INST_BR op0=3 op1=4 op2=3/>
-; DUMP-NEXT:    <INST_BR op0=5 op1=6 op2=3/>
-; DUMP-NEXT:    <INST_STORE op0=4 op1=1 op2=1/>
-; DUMP-NEXT:    <INST_STORE op0=4 op1=1 op2=1/>
-; DUMP-NEXT:    <INST_RET/>
-; DUMP-NEXT:    <INST_STORE op0=4 op1=1 op2=1/>
-; DUMP-NEXT:    <INST_STORE op0=4 op1=1 op2=1/>
-; DUMP-NEXT:    <INST_RET/>
-; DUMP-NEXT:    <INST_STORE op0=4 op1=1 op2=1/>
-; DUMP-NEXT:    <INST_STORE op0=4 op1=1 op2=1/>
-; DUMP-NEXT:    <INST_RET/>
-; DUMP-NEXT:    <INST_STORE op0=4 op1=1 op2=1/>
-; DUMP-NEXT:    <INST_STORE op0=4 op1=1 op2=1/>
-; DUMP-NEXT:    <INST_RET/>
-; DUMP-NEXT:  </FUNCTION_BLOCK>
-; DUMP-NEXT:  <FUNCTION_BLOCK>
-; DUMP-NEXT:    <DECLAREBLOCKS op0=6/>
-; DUMP-NEXT:    <INST_SWITCH op0=0 op1=1 op2=2 op3=4 op4=1 op5=1
-; DUMP-NEXT:                   op6=2 op7=3 op8=1 op9=1 op10=4 op11=3
-; DUMP-NEXT:                   op12=1 op13=1 op14=8 op15=4 op16=1 op17=1
-; DUMP-NEXT:                   op18=10 op19=4/>
-; DUMP-NEXT:    <INST_BR op0=5/>
-; DUMP-NEXT:    <INST_BR op0=5/>
-; DUMP-NEXT:    <INST_BR op0=5/>
-; DUMP-NEXT:    <INST_BR op0=5/>
-; DUMP-NEXT:    <INST_RET/>
-; DUMP-NEXT:  </FUNCTION_BLOCK>
-; DUMP-NEXT:</MODULE_BLOCK>
-
-
diff --git a/test/NaCl/Bitcode/bcdist.ll b/test/NaCl/Bitcode/bcdist.ll
deleted file mode 100644
index 606d4afcf59..00000000000
--- a/test/NaCl/Bitcode/bcdist.ll
+++ /dev/null
@@ -1,392 +0,0 @@
-; Simple test to show that we don't break distribution counts.
-
-; RUN: llvm-as < %s | pnacl-freeze | pnacl-bcanalyzer --order-blocks-by-id \
-; RUN:              | FileCheck %s
-
-@bytes7 = internal global [7 x i8] c"abcdefg"
-
-@ptr_to_ptr = internal global i32 ptrtoint (i32* @ptr to i32)
-
-@ptr_to_func = internal global i32 ptrtoint (void ()* @func to i32)
-
-@compound = internal global <{ [3 x i8], i32 }> <{ [3 x i8] c"foo", i32 ptrtoint (void ()* @func to i32) }>
-
-@ptr = internal global i32 ptrtoint ([7 x i8]* @bytes7 to i32)
-
-@addend_ptr = internal global i32 add (i32 ptrtoint (i32* @ptr to i32), i32 1)
-
-@addend_negative = internal global i32 add (i32 ptrtoint (i32* @ptr to i32), i32 -1)
-
-@addend_array1 = internal global i32 add (i32 ptrtoint ([7 x i8]* @bytes7 to i32), i32 1)
-
-@addend_array2 = internal global i32 add (i32 ptrtoint ([7 x i8]* @bytes7 to i32), i32 7)
-
-@addend_array3 = internal global i32 add (i32 ptrtoint ([7 x i8]* @bytes7 to i32), i32 9)
-
-@addend_struct1 = internal global i32 add (i32 ptrtoint (<{ [3 x i8], i32 }>* @compound to i32), i32 1)
-
-@addend_struct2 = internal global i32 add (i32 ptrtoint (<{ [3 x i8], i32 }>* @compound to i32), i32 4)
-
-@ptr_to_func_align = internal global i32 ptrtoint (void ()* @func to i32), align 8
-
-@char = internal constant [1 x i8] c"0"
-
-@short = internal constant [2 x i8] zeroinitializer
-
-@bytes = internal global [4 x i8] c"abcd"
-
-declare i32 @bar(i32)
-
-define void @func() {
-  ret void
-}
-
-define void @AllocCastSimple() {
-  %1 = alloca i8, i32 4, align 8
-  %2 = ptrtoint i8* %1 to i32
-  %3 = bitcast [4 x i8]* @bytes to i32*
-  store i32 %2, i32* %3, align 1
-  ret void
-}
-
-define void @AllocCastSimpleReversed() {
-  %1 = alloca i8, i32 4, align 8
-  %2 = ptrtoint i8* %1 to i32
-  %3 = bitcast [4 x i8]* @bytes to i32*
-  store i32 %2, i32* %3, align 1
-  ret void
-}
-
-define void @AllocCastDelete() {
-  %1 = alloca i8, i32 4, align 8
-  %2 = alloca i8, i32 4, align 8
-  ret void
-}
-
-define void @AllocCastOpt() {
-  %1 = alloca i8, i32 4, align 8
-  %2 = ptrtoint i8* %1 to i32
-  %3 = bitcast [4 x i8]* @bytes to i32*
-  store i32 %2, i32* %3, align 1
-  store i32 %2, i32* %3, align 1
-  ret void
-}
-
-define void @AllocBitcast(i32) {
-  %2 = alloca i8, i32 4, align 8
-  %3 = add i32 %0, 1
-  %4 = ptrtoint i8* %2 to i32
-  %5 = bitcast [4 x i8]* @bytes to i32*
-  store i32 %4, i32* %5, align 1
-  ret void
-}
-
-define void @StoreGlobal() {
-  %1 = alloca i8, i32 4, align 8
-  %2 = ptrtoint [4 x i8]* @bytes to i32
-  %3 = bitcast i8* %1 to i32*
-  store i32 %2, i32* %3, align 1
-  ret void
-}
-
-define void @StoreGlobalCastsReversed() {
-  %1 = alloca i8, i32 4, align 8
-  %2 = ptrtoint [4 x i8]* @bytes to i32
-  %3 = bitcast i8* %1 to i32*
-  store i32 %2, i32* %3, align 1
-  ret void
-}
-
-define i32 @StoreGlobalCastPtr2Int() {
-  %1 = alloca i8, i32 4, align 8
-  %2 = ptrtoint [4 x i8]* @bytes to i32
-  %3 = bitcast i8* %1 to i32*
-  store i32 %2, i32* %3, align 1
-  ret i32 0
-}
-
-define void @CastAddAlloca() {
-  %1 = alloca i8, i32 4, align 8
-  %2 = add i32 1, 2
-  %3 = ptrtoint i8* %1 to i32
-  %4 = add i32 %3, 2
-  %5 = add i32 1, %3
-  %6 = add i32 %3, %3
-  ret void
-}
-
-define void @CastAddGlobal() {
-  %1 = add i32 1, 2
-  %2 = ptrtoint [4 x i8]* @bytes to i32
-  %3 = add i32 %2, 2
-  %4 = add i32 1, %2
-  %5 = add i32 %2, %2
-  ret void
-}
-
-define void @CastBinop() {
-  %1 = alloca i8, i32 4, align 8
-  %2 = ptrtoint i8* %1 to i32
-  %3 = ptrtoint [4 x i8]* @bytes to i32
-  %4 = sub i32 %2, %3
-  %5 = mul i32 %2, %3
-  %6 = udiv i32 %2, %3
-  %7 = urem i32 %2, %3
-  %8 = srem i32 %2, %3
-  %9 = shl i32 %2, %3
-  %10 = lshr i32 %2, %3
-  %11 = ashr i32 %2, %3
-  %12 = and i32 %2, %3
-  %13 = or i32 %2, %3
-  %14 = xor i32 %2, %3
-  ret void
-}
-
-define void @TestSavedPtrToInt() {
-  %1 = alloca i8, i32 4, align 8
-  %2 = ptrtoint i8* %1 to i32
-  %3 = add i32 %2, 0
-  %4 = call i32 @bar(i32 %2)
-  ret void
-}
-
-define void @CastSelect() {
-  %1 = alloca i8, i32 4, align 8
-  %2 = select i1 true, i32 1, i32 2
-  %3 = ptrtoint i8* %1 to i32
-  %4 = select i1 true, i32 %3, i32 2
-  %5 = ptrtoint [4 x i8]* @bytes to i32
-  %6 = select i1 true, i32 1, i32 %5
-  %7 = select i1 true, i32 %3, i32 %5
-  %8 = select i1 true, i32 %5, i32 %3
-  ret void
-}
-
-define void @PhiBackwardRefs(i1) {
-  %2 = alloca i8, i32 4, align 8
-  %3 = alloca i8, i32 4, align 8
-  br i1 %0, label %true, label %false
-
-true:                                             ; preds = %1
-  %4 = bitcast i8* %2 to i32*
-  %5 = load i32, i32* %4
-  %6 = ptrtoint i8* %3 to i32
-  br label %merge
-
-false:                                            ; preds = %1
-  %7 = bitcast i8* %2 to i32*
-  %8 = load i32, i32* %7
-  %9 = ptrtoint i8* %3 to i32
-  br label %merge
-
-merge:                                            ; preds = %false, %true
-  %10 = phi i32 [ %6, %true ], [ %9, %false ]
-  %11 = phi i32 [ %5, %true ], [ %8, %false ]
-  ret void
-}
-
-define void @PhiForwardRefs(i1) {
-  br label %start
-
-merge:                                            ; preds = %false, %true
-  %2 = phi i32 [ %11, %true ], [ %11, %false ]
-  %3 = phi i32 [ %5, %true ], [ %7, %false ]
-  ret void
-
-true:                                             ; preds = %start
-  %4 = inttoptr i32 %9 to i32*
-  %5 = load i32, i32* %4
-  br label %merge
-
-false:                                            ; preds = %start
-  %6 = inttoptr i32 %9 to i32*
-  %7 = load i32, i32* %6
-  br label %merge
-
-start:                                            ; preds = %1
-  %8 = alloca i8, i32 4, align 8
-  %9 = ptrtoint i8* %8 to i32
-  %10 = alloca i8, i32 4, align 8
-  %11 = ptrtoint i8* %10 to i32
-  br i1 %0, label %true, label %false
-}
-
-define void @PhiMergeCast(i1) {
-  %2 = alloca i8, i32 4, align 8
-  %3 = alloca i8, i32 4, align 8
-  br i1 %0, label %true, label %false
-
-true:                                             ; preds = %1
-  %4 = bitcast i8* %2 to i32*
-  %5 = load i32, i32* %4
-  %6 = ptrtoint i8* %3 to i32
-  %7 = add i32 %5, %6
-  br label %merge
-
-false:                                            ; preds = %1
-  %8 = bitcast i8* %2 to i32*
-  %9 = load i32, i32* %8
-  %10 = ptrtoint i8* %3 to i32
-  br label %merge
-
-merge:                                            ; preds = %false, %true
-  %11 = phi i32 [ %6, %true ], [ %10, %false ]
-  %12 = phi i32 [ %5, %true ], [ %9, %false ]
-  ret void
-}
-
-define void @LongReachingCasts(i1) {
-  %2 = alloca i8, i32 4, align 8
-  br i1 %0, label %Split1, label %Split2
-
-Split1:                                           ; preds = %1
-  br i1 %0, label %b1, label %b2
-
-Split2:                                           ; preds = %1
-  br i1 %0, label %b3, label %b4
-
-b1:                                               ; preds = %Split1
-  %3 = ptrtoint i8* %2 to i32
-  %4 = bitcast [4 x i8]* @bytes to i32*
-  store i32 %3, i32* %4, align 1
-  store i32 %3, i32* %4, align 1
-  ret void
-
-b2:                                               ; preds = %Split1
-  %5 = ptrtoint i8* %2 to i32
-  %6 = bitcast [4 x i8]* @bytes to i32*
-  store i32 %5, i32* %6, align 1
-  store i32 %5, i32* %6, align 1
-  ret void
-
-b3:                                               ; preds = %Split2
-  %7 = ptrtoint i8* %2 to i32
-  %8 = bitcast [4 x i8]* @bytes to i32*
-  store i32 %7, i32* %8, align 1
-  store i32 %7, i32* %8, align 1
-  ret void
-
-b4:                                               ; preds = %Split2
-  %9 = ptrtoint i8* %2 to i32
-  %10 = bitcast [4 x i8]* @bytes to i32*
-  store i32 %9, i32* %10, align 1
-  store i32 %9, i32* %10, align 1
-  ret void
-}
-
-define void @SwitchVariable(i32) {
-  switch i32 %0, label %l1 [
-    i32 1, label %l2
-    i32 2, label %l2
-    i32 4, label %l3
-    i32 5, label %l3
-  ]
-                                                  ; No predecessors!
-  br label %end
-
-l1:                                               ; preds = %1
-  br label %end
-
-l2:                                               ; preds = %1, %1
-  br label %end
-
-l3:                                               ; preds = %1, %1
-  br label %end
-
-end:                                              ; preds = %l3, %l2, %l1, %2
-  ret void
-}
-
-
-; CHECK:      # Toplevel Blocks: 1
-
-; CHECK:      Block Histogram (8 elements):
-
-; CHECK:        %File   Count %Count    # Bits    Bits/Elmt Block
-; CHECK-NEXT:  {{.*}}       1  {{.*}}   {{.*}}       {{.*}} BLOCKINFO_BLOCK
-; CHECK-NEXT:  {{.*}}       1  {{.*}}   {{.*}}       {{.*}} MODULE_BLOCK
-
-; CHECK:          Subblocks (5 elements):
-
-; CHECK:             Count %Count Subblock
-; CHECK-NEXT:           19  82.61 FUNCTION_BLOCK
-; CHECK-NEXT:            1   4.35 GLOBALVAR_BLOCK
-; CHECK-NEXT:            1   4.35 TYPE_BLOCK_ID
-; CHECK-NEXT:            1   4.35 VALUE_SYMTAB
-; CHECK-NEXT:            1   4.35 BLOCKINFO_BLOCK
-
-; CHECK:          Record Histogram: (2 elements):
-
-; CHECK:             Count %Count    # Bits    Bits/Elmt   % Abv Record Kind
-; CHECK-NEXT:           20  95.24    {{.*}}       {{.*}}         FUNCTION
-; CHECK-NEXT:            1   4.76    {{.*}}       {{.*}}         VERSION
-
-; CHECK:        Block Histogram (continued)
-; CHECK-NEXT:   %File   Count %Count    # Bits    Bits/Elmt Block
-; CHECK-NEXT:  {{.*}}      17 {{.*}}    {{.*}}       {{.*}} CONSTANTS_BLOCK
-
-; CHECK:          Record Histogram: (2 elements):
-
-; CHECK:             Count %Count    # Bits    Bits/Elmt   % Abv Record Kind
-; CHECK-NEXT:           26  59.09    {{.*}}       {{.*}}  {{.*}} INTEGER
-; CHECK-NEXT:           18  40.91    {{.*}}       {{.*}}  {{.*}} SETTYPE
-
-; CHECK:        Block Histogram (continued)
-; CHECK-NEXT:   %File   Count %Count    # Bits    Bits/Elmt Block
-; CHECK-NEXT:  {{.*}}      19 {{.*}}    {{.*}}       {{.*}} FUNCTION_BLOCK
-
-; CHECK:          Subblocks (1 elements):
-
-; CHECK:             Count %Count Subblock
-; CHECK-NEXT:           17 100.00 CONSTANTS_BLOCK
-
-; CHECK:          Record Histogram: (12 elements):
-
-; CHECK:             Count %Count    # Bits    Bits/Elmt   % Abv Record Kind
-; CHECK-NEXT:           22  15.83    {{.*}}       {{.*}}  {{.*}} INST_RET
-; CHECK-NEXT:           22  15.83    {{.*}}       {{.*}}  {{.*}} INST_BINOP
-; CHECK-NEXT:           20  14.39    {{.*}}       {{.*}}         INST_ALLOCA
-; CHECK-NEXT:           19  13.67    {{.*}}       {{.*}}         DECLAREBLOCKS
-; CHECK-NEXT:           17  12.23    {{.*}}       {{.*}}         INST_BR
-; CHECK-NEXT:           16  11.51    {{.*}}       {{.*}}  {{.*}} INST_STORE
-; CHECK-NEXT:            6   4.32    {{.*}}       {{.*}}  {{.*}} INST_LOAD
-; CHECK-NEXT:            6   4.32    {{.*}}       {{.*}}         INST_PHI
-; CHECK-NEXT:            5   3.60    {{.*}}       {{.*}}         INST_VSELECT
-; CHECK-NEXT:            4   2.88    {{.*}}       {{.*}}  {{.*}} FORWARDTYPEREF
-; CHECK-NEXT:            1   0.72    {{.*}}       {{.*}}         INST_CALL
-; CHECK-NEXT:            1   0.72    {{.*}}       {{.*}}         INST_SWITCH
-
-; CHECK:        Block Histogram (continued)
-; CHECK-NEXT:   %File   Count %Count    # Bits    Bits/Elmt Block
-; CHECK-NEXT:  {{.*}}       1 {{.*}}    {{.*}}       {{.*}} VALUE_SYMTAB
-
-; CHECK:          Record Histogram: (1 elements):
-
-; CHECK:             Count %Count    # Bits    Bits/Elmt   % Abv Record Kind
-; CHECK-NEXT:           36 100.00    {{.*}}       {{.*}}  {{.*}} ENTRY
-
-; CHECK:        Block Histogram (continued)
-; CHECK-NEXT:   %File   Count %Count    # Bits    Bits/Elmt Block
-; CHECK-NEXT:  {{.*}}       1 {{.*}}    {{.*}}       {{.*}} TYPE_BLOCK_ID
-
-; CHECK:          Record Histogram: (4 elements):
-
-; CHECK:             Count %Count    # Bits    Bits/Elmt   % Abv Record Kind
-; CHECK-NEXT:            5  50.00    {{.*}}       {{.*}}  {{.*}} FUNCTION
-; CHECK-NEXT:            3  30.00    {{.*}}       {{.*}}         INTEGER
-; CHECK-NEXT:            1  10.00    {{.*}}       {{.*}}         VOID
-; CHECK-NEXT:            1  10.00    {{.*}}       {{.*}}         NUMENTRY
-
-; CHECK:        Block Histogram (continued)
-; CHECK-NEXT:   %File   Count %Count    # Bits    Bits/Elmt Block
-; CHECK-NEXT:  {{.*}}       1 {{.*}}    {{.*}}       {{.*}} GLOBALVAR_BLOCK
-
-; CHECK:          Record Histogram: (6 elements):
-
-; CHECK:             Count %Count    # Bits    Bits/Elmt   % Abv Record Kind
-; CHECK-NEXT:           16  45.71    {{.*}}       {{.*}}  {{.*}} VAR
-; CHECK-NEXT:           12  34.29    {{.*}}       {{.*}}  {{.*}} RELOC
-; CHECK-NEXT:            4  11.43    {{.*}}       {{.*}}  {{.*}} DATA
-; CHECK-NEXT:            1   2.86    {{.*}}       {{.*}}         COUNT
-; CHECK-NEXT:            1   2.86    {{.*}}       {{.*}}  {{.*}} ZEROFILL
-; CHECK-NEXT:            1   2.86    {{.*}}       {{.*}}  {{.*}} COMPOUND
diff --git a/test/NaCl/Bitcode/bitcast-elide.ll b/test/NaCl/Bitcode/bitcast-elide.ll
deleted file mode 100644
index f13da813b3f..00000000000
--- a/test/NaCl/Bitcode/bitcast-elide.ll
+++ /dev/null
@@ -1,179 +0,0 @@
-; Test how we handle eliding (pointer) bitcast instructions.
-
-; RUN: llvm-as < %s | pnacl-freeze  \
-; RUN:              | pnacl-bcanalyzer -dump-records \
-; RUN:              | FileCheck %s -check-prefix=PF2
-
-; RUN: llvm-as < %s | pnacl-freeze -allow-local-symbol-tables \
-; RUN:              | pnacl-thaw -allow-local-symbol-tables \
-; RUN:              | llvm-dis - | FileCheck %s -check-prefix=TD2
-
-; ------------------------------------------------------
-
-@bytes = internal global [4 x i8] c"abcd"
-
-; ------------------------------------------------------
-
-; Test that we elide the simple case of global.
-define void @SimpleLoad() {
-  %1 = bitcast [4 x i8]* @bytes to i32*
-  %2 = load i32, i32* %1, align 4
-  ret void
-}
-
-; TD2:      define void @SimpleLoad() {
-; TD2-NEXT:   %1 = bitcast [4 x i8]* @bytes to i32*
-; TD2-NEXT:   %2 = load i32, i32* %1, align 4
-; TD2-NEXT:   ret void
-; TD2-NEXT: }
-
-; PF2:       <FUNCTION_BLOCK>
-; PF2-NEXT:    <DECLAREBLOCKS op0=1/>
-; PF2-NEXT:    <INST_LOAD op0=1 op1=3 op2=0/>
-; PF2-NEXT:    <INST_RET/>
-; PF2-NEXT:  </FUNCTION_BLOCK>
-
-; ------------------------------------------------------
-
-; Test that we elide the simple case of an alloca.
-define void @SimpleLoadAlloca() {
-  %1 = alloca i8, i32 4, align 4
-  %2 = bitcast i8* %1 to i32*
-  %3 = load i32, i32* %2, align 4
-  ret void
-}
-
-; TD2:      define void @SimpleLoadAlloca() {
-; TD2-NEXT:   %1 = alloca i8, i32 4, align 4
-; TD2-NEXT:   %2 = bitcast i8* %1 to i32*
-; TD2-NEXT:   %3 = load i32, i32* %2, align 4
-; TD2-NEXT:   ret void
-; TD2-NEXT: }
-
-; PF2:        <FUNCTION_BLOCK>
-; PF2:          </CONSTANTS_BLOCK>
-; PF2-NEXT:     <INST_ALLOCA op0=1 op1=3/>
-; PF2-NEXT:     <INST_LOAD op0=1 op1=3 op2=0/>
-; PF2-NEXT:     <INST_RET/>
-; PF2-NEXT:   </FUNCTION_BLOCK>
-
-; ------------------------------------------------------
-
-; Test that we can handle multiple bitcasts.
-define i32 @TwoLoads(i32 %i) {
-  %1 = bitcast [4 x i8]* @bytes to i32*       
-  %2 = load i32, i32* %1, align 4
-  %3 = bitcast [4 x i8]* @bytes to i32*       
-  %4 = load i32, i32* %3, align 4
-  %5 = add i32 %2, %4
-  ret i32 %5
-}
-
-; TD2:      define i32 @TwoLoads(i32 %i) {
-; TD2-NEXT:   %1 = bitcast [4 x i8]* @bytes to i32*
-; TD2-NEXT:   %2 = load i32, i32* %1, align 4
-; TD2-NEXT:   %3 = load i32, i32* %1, align 4
-; TD2-NEXT:   %4 = add i32 %2, %3
-; TD2-NEXT:   ret i32 %4
-; TD2-NEXT: }
-
-; PF2:       <FUNCTION_BLOCK>
-; PF2-NEXT:    <DECLAREBLOCKS op0=1/>
-; PF2-NEXT:    <INST_LOAD op0=2 op1=3 op2=0/>
-; PF2-NEXT:    <INST_LOAD op0=3 op1=3 op2=0/>
-; PF2-NEXT:    <INST_BINOP op0=2 op1=1 op2=0/>
-; PF2-NEXT:    <INST_RET op0=1/>
-; PF2:       </FUNCTION_BLOCK>
-
-; ------------------------------------------------------
-
-; Test how we handle bitcasts if optimized in the input file.  This
-; case tests within a single block.
-define i32 @TwoLoadOptOneBlock(i32 %i) {
-  %1 = bitcast [4 x i8]* @bytes to i32*       
-  %2 = load i32, i32* %1, align 4
-  %3 = load i32, i32* %1, align 4
-  %4 = add i32 %2, %3
-  ret i32 %4
-}
-
-; TD2:      define i32 @TwoLoadOptOneBlock(i32 %i) {
-; TD2-NEXT:   %1 = bitcast [4 x i8]* @bytes to i32*
-; TD2-NEXT:   %2 = load i32, i32* %1, align 4
-; TD2-NEXT:   %3 = load i32, i32* %1, align 4
-; TD2-NEXT:   %4 = add i32 %2, %3
-; TD2-NEXT:   ret i32 %4
-; TD2-NEXT: }
-
-; PF2:       <FUNCTION_BLOCK>
-; PF2-NEXT:    <DECLAREBLOCKS op0=1/>
-; PF2-NEXT:    <INST_LOAD op0=2 op1=3 op2=0/>
-; PF2-NEXT:    <INST_LOAD op0=3 op1=3 op2=0/>
-; PF2-NEXT:    <INST_BINOP op0=2 op1=1 op2=0/>
-; PF2-NEXT:    <INST_RET op0=1/>
-; PF2:       </FUNCTION_BLOCK>
-
-; ------------------------------------------------------
-
-; Test how we handle bitcasts if optimized in the input file.  This
-; case tests accross blocks.
-define i32 @TwoLoadOptTwoBlocks(i32 %i) {
-  %1 = bitcast [4 x i8]* @bytes to i32*       
-  %2 = load i32, i32* %1, align 4
-  %3 = load i32, i32* %1, align 4
-  %4 = add i32 %2, %3
-  br label %BB
-
-BB:
-  %5 = load i32, i32* %1, align 4
-  %6 = load i32, i32* %1, align 4
-  %7 = add i32 %5, %6
-  ret i32 %4
-}
-
-; TD2:      define i32 @TwoLoadOptTwoBlocks(i32 %i) {
-; TD2-NEXT:   %1 = bitcast [4 x i8]* @bytes to i32*
-; TD2-NEXT:   %2 = load i32, i32* %1, align 4
-; TD2-NEXT:   %3 = load i32, i32* %1, align 4
-; TD2-NEXT:   %4 = add i32 %2, %3
-; TD2-NEXT:   br label %BB
-; TD2:      BB:
-; TD2-NEXT:   %5 = bitcast [4 x i8]* @bytes to i32*
-; TD2-NEXT:   %6 = load i32, i32* %5, align 4
-; TD2-NEXT:   %7 = load i32, i32* %5, align 4
-; TD2-NEXT:   %8 = add i32 %6, %7
-; TD2-NEXT:   ret i32 %4
-; TD2-NEXT: }
-
-; PF2:        <FUNCTION_BLOCK>
-; PF2-NEXT:     <DECLAREBLOCKS op0=2/>
-; PF2-NEXT:     <INST_LOAD op0=2 op1=3 op2=0/>
-; PF2-NEXT:     <INST_LOAD op0=3 op1=3 op2=0/>
-; PF2-NEXT:     <INST_BINOP op0=2 op1=1 op2=0/>
-; PF2-NEXT:     <INST_BR op0=1/>
-; PF2-NEXT:     <INST_LOAD op0=5 op1=3 op2=0/>
-; PF2-NEXT:     <INST_LOAD op0=6 op1=3 op2=0/>
-; PF2-NEXT:     <INST_BINOP op0=2 op1=1 op2=0/>
-; PF2-NEXT:     <INST_RET op0=4/>
-; PF2:        </FUNCTION_BLOCK>
-
-; ------------------------------------------------------
-
-; Test that we elide the simple case of bitcast for a store.
-define void @SimpleStore(i32 %i) {
-  %1 = bitcast [4 x i8]* @bytes to i32*
-  store i32 %i, i32* %1, align 4
-  ret void
-}
-
-; TD2:      define void @SimpleStore(i32 %i) {
-; TD2-NEXT:   %1 = bitcast [4 x i8]* @bytes to i32*
-; TD2-NEXT:   store i32 %i, i32* %1, align 4
-; TD2-NEXT:   ret void
-; TD2-NEXT: }
-
-; PF2:        <FUNCTION_BLOCK>
-; PF2-NEXT:     <DECLAREBLOCKS op0=1/>
-; PF2-NEXT:     <INST_STORE op0=2 op1=1 op2=3/>
-; PF2-NEXT:     <INST_RET/>
-; PF2:        </FUNCTION_BLOCK>
diff --git a/test/NaCl/Bitcode/call-elide.ll b/test/NaCl/Bitcode/call-elide.ll
deleted file mode 100644
index f53e9bee38e..00000000000
--- a/test/NaCl/Bitcode/call-elide.ll
+++ /dev/null
@@ -1,219 +0,0 @@
-; Test how we handle eliding pointers in call instructions.
-
-; RUN: llvm-as < %s | pnacl-freeze \
-; RUN:              | pnacl-bcanalyzer -dump-records \
-; RUN:              | FileCheck %s -check-prefix=PF2
-
-; RUN: llvm-as < %s | pnacl-freeze -allow-local-symbol-tables \
-; RUN:              | pnacl-thaw -allow-local-symbol-tables \
-; RUN:              | llvm-dis - | FileCheck %s -check-prefix=TD2
-
-; ------------------------------------------------------
-; Define some global functions/variables to be used in testing.
-
-
-@bytes = internal global [4 x i8] c"abcd"
-declare void @foo(i32 %i)
-declare i32 @llvm.nacl.setjmp(i8* %i)
-
-; ------------------------------------------------------
-; Test how we handle a direct call.
-
-define void @DirectCall() {
-  call void @foo(i32 0)
-  ret void
-}
-
-; TD2:      define void @DirectCall() {
-; TD2-NEXT:   call void @foo(i32 0)
-; TD2-NEXT:   ret void
-; TD2-NEXT: }
-
-; PF2:      <FUNCTION_BLOCK>
-; PF2:        </CONSTANTS_BLOCK>
-; PF2-NEXT:   <INST_CALL op0=0 op1=14 op2=1/>
-; PF2-NEXT:   <INST_RET/>
-; PF2-NEXT: </FUNCTION_BLOCK>
-
-; ------------------------------------------------------
-; Test how we handle a direct call with a normalized inttoptr argument.
-; Pointer arguments are only allowed for intrinsic calls.
-
-define void @DirectCallIntToPtrArg(i32 %i) {
-  %1 = inttoptr i32 %i to i8*
-  %2 = call i32 @llvm.nacl.setjmp(i8* %1)
-  ret void
-}
-
-; TD2:      define void @DirectCallIntToPtrArg(i32 %i) {
-; TD2-NEXT:   %1 = inttoptr i32 %i to i8*
-; TD2-NEXT:   %2 = call i32 @llvm.nacl.setjmp(i8* %1)
-; TD2-NEXT:   ret void
-; TD2-NEXT: }
-
-; PF2:      <FUNCTION_BLOCK>
-; PF2-NEXT:   <DECLAREBLOCKS op0=1/>
-; PF2-NEXT:   <INST_CALL op0=0 op1=13 op2=1/>
-; PF2-NEXT:   <INST_RET/>
-; PF2-NEXT: </FUNCTION_BLOCK>
-
-; ------------------------------------------------------
-; Test how we handle a direct call with a normalized ptroint argument.
-; Pointer arguments are only allowed for intrinsic calls.
-
-define void @DirectCallPtrToIntArg() {
-  %1 = alloca i8, i32 4, align 8
-  %2 = ptrtoint i8* %1 to i32
-  call void @foo(i32 %2)
-  ret void
-}
-
-; TD2:      define void @DirectCallPtrToIntArg() {
-; TD2-NEXT:   %1 = alloca i8, i32 4, align 8
-; TD2-NEXT:   %2 = ptrtoint i8* %1 to i32
-; TD2-NEXT:   call void @foo(i32 %2)
-; TD2-NEXT:   ret void
-; TD2-NEXT: }
-
-; PF2:      <FUNCTION_BLOCK>
-; PF2:        </CONSTANTS_BLOCK>
-; PF2-NEXT:   <INST_ALLOCA op0=1 op1=4/>
-; PF2-NEXT:   <INST_CALL op0=0 op1=15 op2=1/>
-; PF2-NEXT:   <INST_RET/>
-; PF2-NEXT: </FUNCTION_BLOCK>
-
-; ------------------------------------------------------
-; Test how we handle a direct call with a normalized bitcast argument.
-
-define void @DirectCallBitcastArg(i32 %i) {
-  %1 = bitcast [4 x i8]* @bytes to i8*
-  %2 = call i32 @llvm.nacl.setjmp(i8* %1)
-  ret void
-}
-
-; TD2:      define void @DirectCallBitcastArg(i32 %i) {
-; TD2-NEXT:   %1 = bitcast [4 x i8]* @bytes to i8*
-; TD2-NEXT:   %2 = call i32 @llvm.nacl.setjmp(i8* %1)
-; TD2-NEXT:   ret void
-; TD2-NEXT: }
-
-; PF2:      <FUNCTION_BLOCK>
-; PF2-NEXT:   <DECLAREBLOCKS op0=1/>
-; PF2-NEXT:   <INST_CALL op0=0 op1=13 op2=2/>
-; PF2-NEXT:   <INST_RET/>
-; PF2-NEXT: </FUNCTION_BLOCK>
-
-; ------------------------------------------------------
-; Test how we handle a direct call with a pointer to scalar conversion.
-
-define void @DirectCallScalarArg() {
-  %1 = ptrtoint [4 x i8]* @bytes to i32
-  call void @foo(i32 %1)
-  ret void
-}
-
-; TD2:      define void @DirectCallScalarArg() {
-; TD2-NEXT:   %1 = ptrtoint [4 x i8]* @bytes to i32
-; TD2-NEXT:   call void @foo(i32 %1)
-; TD2-NEXT:   ret void
-; TD2-NEXT: }
-
-; PF2:      <FUNCTION_BLOCK>
-; PF2-NEXT:   <DECLAREBLOCKS op0=1/>
-; PF2-NEXT:   <INST_CALL op0=0 op1=13 op2=1/>
-; PF2-NEXT:   <INST_RET/>
-; PF2-NEXT: </FUNCTION_BLOCK>
-
-; ------------------------------------------------------
-; Test how we handle an indirect call.
-
-define void @IndirectCall(i32 %i) {
-  %1 = inttoptr i32 %i to void (i32)*
-  call void %1(i32 %i)
-  ret void
-}
-
-; TD2:      define void @IndirectCall(i32 %i) {
-; TD2-NEXT:   %1 = inttoptr i32 %i to void (i32)*
-; TD2-NEXT:   call void %1(i32 %i)
-; TD2-NEXT:   ret void
-; TD2-NEXT: }
-
-; PF2:      <FUNCTION_BLOCK>
-; PF2-NEXT:   <DECLAREBLOCKS op0=1/>
-; PF2-NEXT:   <INST_CALL_INDIRECT op0=0 op1=1 op2=1 op3=1/>
-; PF2-NEXT:   <INST_RET/>
-; PF2-NEXT: </FUNCTION_BLOCK>
-
-; ------------------------------------------------------
-; Test how we handle an indirect call with a normalized ptrtoint argument.
-
-define void @IndirectCallPtrToIntArg(i32 %i) {
-  %1 = alloca i8, i32 4, align 8
-  %2 = inttoptr i32 %i to void (i32)*
-  %3 = ptrtoint i8* %1 to i32
-  call void %2(i32 %3)
-  ret void
-}
-
-; TD2:      define void @IndirectCallPtrToIntArg(i32 %i) {
-; TD2-NEXT:   %1 = alloca i8, i32 4, align 8
-; TD2-NEXT:   %2 = ptrtoint i8* %1 to i32
-; TD2-NEXT:   %3 = inttoptr i32 %i to void (i32)*
-; TD2-NEXT:   call void %3(i32 %2)
-; TD2-NEXT:   ret void
-; TD2-NEXT: }
-
-; PF2:      <FUNCTION_BLOCK>
-; PF2:        </CONSTANTS_BLOCK>
-; PF2-NEXT:   <INST_ALLOCA op0=1 op1=4/>
-; PF2-NEXT:   <INST_CALL_INDIRECT op0=0 op1=3 op2=1 op3=1/>
-; PF2-NEXT:   <INST_RET/>
-; PF2-NEXT:      </FUNCTION_BLOCK>
-
-; ------------------------------------------------------
-; Test how we handle an indirect call with a pointer to scalar conversion.
-
-define void @IndirectCallScalarArg(i32 %i) {
-  %1 = inttoptr i32 %i to void (i32)*
-  %2 = ptrtoint [4 x i8]* @bytes to i32
-  call void %1(i32 %2)
-  ret void
-}
-
-; TD2:      define void @IndirectCallScalarArg(i32 %i) {
-; TD2-NEXT:   %1 = ptrtoint [4 x i8]* @bytes to i32
-; TD2-NEXT:   %2 = inttoptr i32 %i to void (i32)*
-; TD2-NEXT:   call void %2(i32 %1)
-; TD2-NEXT:   ret void
-; TD2-NEXT: }
-
-; PF2:      <FUNCTION_BLOCK>
-; PF2-NEXT:   <DECLAREBLOCKS op0=1/>
-; PF2-NEXT:   <INST_CALL_INDIRECT op0=0 op1=1 op2=1 op3=2/>
-; PF2-NEXT:   <INST_RET/>
-; PF2-NEXT: </FUNCTION_BLOCK>
-
-; ------------------------------------------------------
-; Test how we handle intrinsics that can return (inherent) pointers, and
-; return statements that expect scalar values.
-
-declare i8* @llvm.nacl.read.tp()
-
-define i32 @ReturnPtrIntrinsic() {
-  %1 = call i8* @llvm.nacl.read.tp()
-  %2 = ptrtoint i8* %1 to i32
-  ret i32 %2
-}
-
-; TD2:      define i32 @ReturnPtrIntrinsic() {
-; TD2-NEXT:   %1 = call i8* @llvm.nacl.read.tp()
-; TD2-NEXT:   %2 = ptrtoint i8* %1 to i32
-; TD2-NEXT:   ret i32 %2
-; TD2-NEXT: }
-
-; PF2:      <FUNCTION_BLOCK>
-; PF2-NEXT:   <DECLAREBLOCKS op0=1/>
-; PF2-NEXT:   <INST_CALL op0=0 op1=3/>
-; PF2-NEXT:   <INST_RET op0=1/>
-; PF2-NEXT: </FUNCTION_BLOCK>
diff --git a/test/NaCl/Bitcode/cmp.ll b/test/NaCl/Bitcode/cmp.ll
deleted file mode 100644
index ac24c72704d..00000000000
--- a/test/NaCl/Bitcode/cmp.ll
+++ /dev/null
@@ -1,72 +0,0 @@
-; Tests that all comparison conditions survive through PNaCl bitcode files.
-
-; RUN: llvm-as < %s | pnacl-freeze | pnacl-thaw | llvm-dis - \
-; RUN:              | FileCheck %s
-
-define void @IntCompare() {
-  %1 = icmp eq i32 0, 1
-  %2 = icmp ne i32 0, 1
-  %3 = icmp ugt i32 0, 1
-  %4 = icmp uge i32 0, 1
-  %5 = icmp ult i32 0, 1
-  %6 = icmp ule i32 0, 1
-  %7 = icmp sgt i32 0, 1
-  %8 = icmp sge i32 0, 1
-  %9 = icmp slt i32 0, 1
-  %10 = icmp sle i32 0, 1
-  ret void
-}
-
-; CHECK: define void @IntCompare() {
-; CHECK:   %1 = icmp eq i32 0, 1
-; CHECK:   %2 = icmp ne i32 0, 1
-; CHECK:   %3 = icmp ugt i32 0, 1
-; CHECK:   %4 = icmp uge i32 0, 1
-; CHECK:   %5 = icmp ult i32 0, 1
-; CHECK:   %6 = icmp ule i32 0, 1
-; CHECK:   %7 = icmp sgt i32 0, 1
-; CHECK:   %8 = icmp sge i32 0, 1
-; CHECK:   %9 = icmp slt i32 0, 1
-; CHECK:   %10 = icmp sle i32 0, 1
-; CHECK:   ret void
-; CHECK: }
-
-define void @FloatCompare() {
-  %1 = fcmp false float 0.000000e+00, 1.000000e+00
-  %2 = fcmp oeq float 0.000000e+00, 1.000000e+00
-  %3 = fcmp ogt float 0.000000e+00, 1.000000e+00
-  %4 = fcmp oge float 0.000000e+00, 1.000000e+00
-  %5 = fcmp olt float 0.000000e+00, 1.000000e+00
-  %6 = fcmp ole float 0.000000e+00, 1.000000e+00
-  %7 = fcmp one float 0.000000e+00, 1.000000e+00
-  %8 = fcmp ord float 0.000000e+00, 1.000000e+00
-  %9 = fcmp ueq float 0.000000e+00, 1.000000e+00
-  %10 = fcmp ugt float 0.000000e+00, 1.000000e+00
-  %11 = fcmp uge float 0.000000e+00, 1.000000e+00
-  %12 = fcmp ult float 0.000000e+00, 1.000000e+00
-  %13 = fcmp ule float 0.000000e+00, 1.000000e+00
-  %14 = fcmp une float 0.000000e+00, 1.000000e+00
-  %15 = fcmp uno float 0.000000e+00, 1.000000e+00
-  %16 = fcmp true float 0.000000e+00, 1.000000e+00
-  ret void
-}
-
-; CHECK: define void @FloatCompare() {
-; CHECK:   %1 = fcmp false float 0.000000e+00, 1.000000e+00
-; CHECK:   %2 = fcmp oeq float 0.000000e+00, 1.000000e+00
-; CHECK:   %3 = fcmp ogt float 0.000000e+00, 1.000000e+00
-; CHECK:   %4 = fcmp oge float 0.000000e+00, 1.000000e+00
-; CHECK:   %5 = fcmp olt float 0.000000e+00, 1.000000e+00
-; CHECK:   %6 = fcmp ole float 0.000000e+00, 1.000000e+00
-; CHECK:   %7 = fcmp one float 0.000000e+00, 1.000000e+00
-; CHECK:   %8 = fcmp ord float 0.000000e+00, 1.000000e+00
-; CHECK:   %9 = fcmp ueq float 0.000000e+00, 1.000000e+00
-; CHECK:   %10 = fcmp ugt float 0.000000e+00, 1.000000e+00
-; CHECK:   %11 = fcmp uge float 0.000000e+00, 1.000000e+00
-; CHECK:   %12 = fcmp ult float 0.000000e+00, 1.000000e+00
-; CHECK:   %13 = fcmp ule float 0.000000e+00, 1.000000e+00
-; CHECK:   %14 = fcmp une float 0.000000e+00, 1.000000e+00
-; CHECK:   %15 = fcmp uno float 0.000000e+00, 1.000000e+00
-; CHECK:   %16 = fcmp true float 0.000000e+00, 1.000000e+00
-; CHECK:   ret void
-; CHECK: }
diff --git a/test/NaCl/Bitcode/dump.ll b/test/NaCl/Bitcode/dump.ll
deleted file mode 100644
index 9f87f0ae8be..00000000000
--- a/test/NaCl/Bitcode/dump.ll
+++ /dev/null
@@ -1,247 +0,0 @@
-; Simple tests on dump flags for pnacl-bcanalyzer.
-
-@bytes = internal global [4 x i8] c"abcd"
-
-@ptr_to_ptr = internal global i32 ptrtoint (i32* @ptr to i32)
-
-@ptr = internal global i32 ptrtoint ([4 x i8]* @bytes to i32)
-
-declare i32 @bar(i32)
-
-define void @AllocCastSimple() {
-  %1 = alloca i8, i32 4, align 8
-  %2 = ptrtoint i8* %1 to i32
-  %3 = bitcast [4 x i8]* @bytes to i32*
-  store i32 %2, i32* %3, align 1
-  ret void
-}
-
-define void @PhiBackwardRefs(i1) {
-  %2 = alloca i8, i32 4, align 8
-  %3 = alloca i8, i32 4, align 8
-  br i1 %0, label %true, label %false
-
-true:                                             ; preds = %1
-  %4 = bitcast i8* %2 to i32*
-  %5 = load i32, i32* %4
-  %6 = ptrtoint i8* %3 to i32
-  br label %merge
-
-false:                                            ; preds = %1
-  %7 = bitcast i8* %2 to i32*
-  %8 = load i32, i32* %7
-  %9 = ptrtoint i8* %3 to i32
-  br label %merge
-
-merge:                                            ; preds = %false, %true
-  %10 = phi i32 [ %6, %true ], [ %9, %false ]
-  %11 = phi i32 [ %5, %true ], [ %8, %false ]
-  ret void
-}
-
-; -------------------------------------------------
-
-; 1) Test dump of records.
-; RUN: llvm-as < %s | pnacl-freeze \
-; RUN:              | pnacl-bcanalyzer --operands-per-line=4 \
-; RUN:                   -dump-records \
-; RUN:              | FileCheck %s -check-prefix=DR
-
-; DR: <MODULE_BLOCK>
-; DR:   <VERSION op0=1/>
-; DR:   <BLOCKINFO_BLOCK/>
-; DR:   <TYPE_BLOCK_ID>
-; DR:     <NUMENTRY op0=7/>
-; DR:     <INTEGER op0=32/>
-; DR:     <VOID/>
-; DR:     <INTEGER op0=1/>
-; DR:     <INTEGER op0=8/>
-; DR:     <FUNCTION op0=0 op1=0 op2=0/>
-; DR:     <FUNCTION op0=0 op1=1/>
-; DR:     <FUNCTION op0=0 op1=1 op2=2/>
-; DR:   </TYPE_BLOCK_ID>
-; DR:   <FUNCTION op0=4 op1=0 op2=1 op3=0/>
-; DR:   <FUNCTION op0=5 op1=0 op2=0 op3=0/>
-; DR:   <FUNCTION op0=6 op1=0 op2=0 op3=0/>
-; DR:   <GLOBALVAR_BLOCK>
-; DR:     <COUNT op0=3/>
-; DR:     <VAR op0=0 op1=0/>
-; DR:     <DATA op0=97 op1=98 op2=99 op3=100/>
-; DR:     <VAR op0=0 op1=0/>
-; DR:     <RELOC op0=5/>
-; DR:     <VAR op0=0 op1=0/>
-; DR:     <RELOC op0=3/>
-; DR:   </GLOBALVAR_BLOCK>
-; DR:   <VALUE_SYMTAB>
-; DR:     <ENTRY op0=1 op1=65 op2=108 op3=108
-; DR:            op4=111 op5=99 op6=67 op7=97
-; DR:            op8=115 op9=116 op10=83 op11=105
-; DR:            op12=109 op13=112 op14=108 op15=101/>
-; DR:     <ENTRY op0=2 op1=80 op2=104 op3=105
-; DR:            op4=66 op5=97 op6=99 op7=107
-; DR:            op8=119 op9=97 op10=114 op11=100
-; DR:            op12=82 op13=101 op14=102 op15=115/>
-; DR:     <ENTRY op0=0 op1=98 op2=97 op3=114/>
-; DR:     <ENTRY op0=5 op1=112 op2=116 op3=114/>
-; DR:     <ENTRY op0=3 op1=98 op2=121 op3=116
-; DR:            op4=101 op5=115/>
-; DR:     <ENTRY op0=4 op1=112 op2=116 op3=114
-; DR:            op4=95 op5=116 op6=111 op7=95
-; DR:            op8=112 op9=116 op10=114/>
-; DR:   </VALUE_SYMTAB>
-; DR:   <FUNCTION_BLOCK>
-; DR:     <DECLAREBLOCKS op0=1/>
-; DR:     <CONSTANTS_BLOCK>
-; DR:       <SETTYPE op0=0/>
-; DR:       <INTEGER op0=8/>
-; DR:     </CONSTANTS_BLOCK>
-; DR:     <INST_ALLOCA op0=1 op1=4/>
-; DR:     <INST_STORE op0=5 op1=1 op2=1/>
-; DR:     <INST_RET/>
-; DR:   </FUNCTION_BLOCK>
-; DR:   <FUNCTION_BLOCK>
-; DR:     <DECLAREBLOCKS op0=4/>
-; DR:     <CONSTANTS_BLOCK>
-; DR:       <SETTYPE op0=0/>
-; DR:       <INTEGER op0=8/>
-; DR:     </CONSTANTS_BLOCK>
-; DR:     <INST_ALLOCA op0=1 op1=4/>
-; DR:     <INST_ALLOCA op0=2 op1=4/>
-; DR:     <INST_BR op0=1 op1=2 op2=4/>
-; DR:     <INST_LOAD op0=2 op1=0 op2=0/>
-; DR:     <INST_BR op0=3/>
-; DR:     <INST_LOAD op0=3 op1=0 op2=0/>
-; DR:     <INST_BR op0=3/>
-; DR:     <INST_PHI op0=0 op1=6 op2=1 op3=6
-; DR:               op4=2/>
-; DR:     <INST_PHI op0=0 op1=6 op2=1 op3=4
-; DR:               op4=2/>
-; DR:     <INST_RET/>
-; DR:   </FUNCTION_BLOCK>
-; DR: </MODULE_BLOCK>
-
-; -------------------------------------------------
-
-; 2) Test dump of records with details.
-; RUN: llvm-as < %s | pnacl-freeze \
-; RUN:              | pnacl-bcanalyzer --operands-per-line=4 \
-; RUN:                   --dump-records --dump-details \
-; RUN:              | FileCheck %s -check-prefix=DRWD
-
-; DRWD: <MODULE_BLOCK abbrev='ENTER_SUBBLOCK' NumWords={{.*}} BlockCodeSize=2>
-; DRWD:   <VERSION abbrev='UNABBREVIATED' op0=1/>
-; DRWD:   <BLOCKINFO_BLOCK abbrev='ENTER_SUBBLOCK' NumWords=24 BlockCodeSize=2>
-; DRWD:     <SETBID abbrev='UNABBREVIATED' block='VALUE_SYMTAB'/>
-; DRWD:     <DEFINE_ABBREV abbrev='DEFINE_ABBREV' op0='Fixed(3)' op1='VBR(8)' op2='Array'
-; DRWD:                    op3='Fixed(8)'/>
-; DRWD:     <DEFINE_ABBREV abbrev='DEFINE_ABBREV' op0='Literal(1)' op1='VBR(8)' op2='Array'
-; DRWD:                    op3='Fixed(7)'/>
-; DRWD:     <DEFINE_ABBREV abbrev='DEFINE_ABBREV' op0='Literal(1)' op1='VBR(8)' op2='Array'
-; DRWD:                    op3='Char6'/>
-; DRWD:     <DEFINE_ABBREV abbrev='DEFINE_ABBREV' op0='Literal(2)' op1='VBR(8)' op2='Array'
-; DRWD:                    op3='Char6'/>
-; DRWD:     <SETBID abbrev='UNABBREVIATED' block='CONSTANTS_BLOCK'/>
-; DRWD:     <DEFINE_ABBREV abbrev='DEFINE_ABBREV' op0='Literal(1)' op1='Fixed(3)'/>
-; DRWD:     <DEFINE_ABBREV abbrev='DEFINE_ABBREV' op0='Literal(4)' op1='VBR(8)'/>
-; DRWD:     <DEFINE_ABBREV abbrev='DEFINE_ABBREV' op0='Literal(4)' op1='Literal(0)'/>
-; DRWD:     <DEFINE_ABBREV abbrev='DEFINE_ABBREV' op0='Literal(6)' op1='VBR(8)'/>
-; DRWD:     <SETBID abbrev='UNABBREVIATED' block='FUNCTION_BLOCK'/>
-; DRWD:     <DEFINE_ABBREV abbrev='DEFINE_ABBREV' op0='Literal(20)' op1='VBR(6)' op2='VBR(4)'
-; DRWD:                    op3='VBR(4)'/>
-; DRWD:     <DEFINE_ABBREV abbrev='DEFINE_ABBREV' op0='Literal(2)' op1='VBR(6)' op2='VBR(6)'
-; DRWD:                    op3='Fixed(4)'/>
-; DRWD:     <DEFINE_ABBREV abbrev='DEFINE_ABBREV' op0='Literal(3)' op1='VBR(6)' op2='Fixed(3)'
-; DRWD:                    op3='Fixed(4)'/>
-; DRWD:     <DEFINE_ABBREV abbrev='DEFINE_ABBREV' op0='Literal(10)'/>
-; DRWD:     <DEFINE_ABBREV abbrev='DEFINE_ABBREV' op0='Literal(10)' op1='VBR(6)'/>
-; DRWD:     <DEFINE_ABBREV abbrev='DEFINE_ABBREV' op0='Literal(15)'/>
-; DRWD:     <DEFINE_ABBREV abbrev='DEFINE_ABBREV' op0='Literal(43)' op1='VBR(6)' op2='Fixed(3)'/>
-; DRWD:     <DEFINE_ABBREV abbrev='DEFINE_ABBREV' op0='Literal(24)' op1='VBR(6)' op2='VBR(6)'
-; DRWD:                    op3='VBR(4)'/>
-; DRWD:     <SETBID abbrev='UNABBREVIATED' block='GLOBALVAR_BLOCK'/>
-; DRWD:     <DEFINE_ABBREV abbrev='DEFINE_ABBREV' op0='Literal(0)' op1='VBR(6)' op2='Fixed(1)'/>
-; DRWD:     <DEFINE_ABBREV abbrev='DEFINE_ABBREV' op0='Literal(1)' op1='VBR(8)'/>
-; DRWD:     <DEFINE_ABBREV abbrev='DEFINE_ABBREV' op0='Literal(2)' op1='VBR(8)'/>
-; DRWD:     <DEFINE_ABBREV abbrev='DEFINE_ABBREV' op0='Literal(3)' op1='Array' op2='Fixed(8)'/>
-; DRWD:     <DEFINE_ABBREV abbrev='DEFINE_ABBREV' op0='Literal(4)' op1='VBR(6)'/>
-; DRWD:     <DEFINE_ABBREV abbrev='DEFINE_ABBREV' op0='Literal(4)' op1='VBR(6)' op2='VBR(6)'/>
-; DRWD:   </BLOCKINFO_BLOCK abbrev='END_BLOCK'>
-; DRWD:   <TYPE_BLOCK_ID abbrev='ENTER_SUBBLOCK' NumWords={{.*}} BlockCodeSize=3>
-; DRWD:     <DEFINE_ABBREV abbrev='DEFINE_ABBREV' op0='Literal(21)' op1='Fixed(1)' op2='Array'
-; DRWD:                    op3='Fixed(3)'/>
-; DRWD:     <NUMENTRY abbrev='UNABBREVIATED' op0=7/>
-; DRWD:     <INTEGER abbrev='UNABBREVIATED' op0=32/>
-; DRWD:     <VOID abbrev='UNABBREVIATED'/>
-; DRWD:     <INTEGER abbrev='UNABBREVIATED' op0=1/>
-; DRWD:     <INTEGER abbrev='UNABBREVIATED' op0=8/>
-; DRWD:     <FUNCTION abbrev=4 op0=0 op1=0 op2=0/>
-; DRWD:     <FUNCTION abbrev=4 op0=0 op1=1/>
-; DRWD:     <FUNCTION abbrev=4 op0=0 op1=1 op2=2/>
-; DRWD:   </TYPE_BLOCK_ID abbrev='END_BLOCK'>
-; DRWD:   <FUNCTION abbrev='UNABBREVIATED' op0=4 op1=0 op2=1
-; DRWD:             op3=0/>
-; DRWD:   <FUNCTION abbrev='UNABBREVIATED' op0=5 op1=0 op2=0
-; DRWD:             op3=0/>
-; DRWD:   <FUNCTION abbrev='UNABBREVIATED' op0=6 op1=0 op2=0
-; DRWD:             op3=0/>
-; DRWD:   <GLOBALVAR_BLOCK abbrev='ENTER_SUBBLOCK' NumWords={{.*}} BlockCodeSize=4>
-; DRWD:     <COUNT abbrev='UNABBREVIATED' op0=3/>
-; DRWD:     <VAR abbrev=4 op0=0 op1=0/>
-; DRWD:     <DATA abbrev=7 op0=97 op1=98 op2=99
-; DRWD:           op3=100/>
-; DRWD:     <VAR abbrev=4 op0=0 op1=0/>
-; DRWD:     <RELOC abbrev=8 op0=5/>
-; DRWD:     <VAR abbrev=4 op0=0 op1=0/>
-; DRWD:     <RELOC abbrev=8 op0=3/>
-; DRWD:   </GLOBALVAR_BLOCK abbrev='END_BLOCK'>
-; DRWD:   <VALUE_SYMTAB abbrev='ENTER_SUBBLOCK' NumWords={{.*}} BlockCodeSize=3>
-; DRWD:     <ENTRY abbrev=6 op0=1 op1=65 op2=108
-; DRWD:            op3=108 op4=111 op5=99 op6=67
-; DRWD:            op7=97 op8=115 op9=116 op10=83
-; DRWD:            op11=105 op12=109 op13=112 op14=108
-; DRWD:            op15=101/>
-; DRWD:     <ENTRY abbrev=6 op0=2 op1=80 op2=104
-; DRWD:            op3=105 op4=66 op5=97 op6=99
-; DRWD:            op7=107 op8=119 op9=97 op10=114
-; DRWD:            op11=100 op12=82 op13=101 op14=102
-; DRWD:            op15=115/>
-; DRWD:     <ENTRY abbrev=6 op0=0 op1=98 op2=97
-; DRWD:            op3=114/>
-; DRWD:     <ENTRY abbrev=6 op0=5 op1=112 op2=116
-; DRWD:            op3=114/>
-; DRWD:     <ENTRY abbrev=6 op0=3 op1=98 op2=121
-; DRWD:            op3=116 op4=101 op5=115/>
-; DRWD:     <ENTRY abbrev=6 op0=4 op1=112 op2=116
-; DRWD:            op3=114 op4=95 op5=116 op6=111
-; DRWD:            op7=95 op8=112 op9=116 op10=114/>
-; DRWD:   </VALUE_SYMTAB abbrev='END_BLOCK'>
-; DRWD:   <FUNCTION_BLOCK abbrev='ENTER_SUBBLOCK' NumWords={{.*}} BlockCodeSize=4>
-; DRWD:     <DECLAREBLOCKS abbrev='UNABBREVIATED' op0=1/>
-; DRWD:     <CONSTANTS_BLOCK abbrev='ENTER_SUBBLOCK' NumWords={{.*}} BlockCodeSize=3>
-; DRWD:       <SETTYPE abbrev=4 op0=0/>
-; DRWD:       <INTEGER abbrev=5 op0=8/>
-; DRWD:     </CONSTANTS_BLOCK abbrev='END_BLOCK'>
-; DRWD:     <INST_ALLOCA abbrev='UNABBREVIATED' op0=1 op1=4/>
-; DRWD:     <INST_STORE abbrev=11 op0=5 op1=1 op2=1/>
-; DRWD:     <INST_RET abbrev=7/>
-; DRWD:   </FUNCTION_BLOCK abbrev='END_BLOCK'>
-; DRWD:   <FUNCTION_BLOCK abbrev='ENTER_SUBBLOCK' NumWords={{.*}} BlockCodeSize=4>
-; DRWD:     <DECLAREBLOCKS abbrev='UNABBREVIATED' op0=4/>
-; DRWD:     <CONSTANTS_BLOCK abbrev='ENTER_SUBBLOCK' NumWords={{.*}} BlockCodeSize=3>
-; DRWD:       <SETTYPE abbrev=4 op0=0/>
-; DRWD:       <INTEGER abbrev=5 op0=8/>
-; DRWD:     </CONSTANTS_BLOCK abbrev='END_BLOCK'>
-; DRWD:     <INST_ALLOCA abbrev='UNABBREVIATED' op0=1 op1=4/>
-; DRWD:     <INST_ALLOCA abbrev='UNABBREVIATED' op0=2 op1=4/>
-; DRWD:     <INST_BR abbrev='UNABBREVIATED' op0=1 op1=2 op2=4/>
-; DRWD:     <INST_LOAD abbrev=4 op0=2 op1=0 op2=0/>
-; DRWD:     <INST_BR abbrev='UNABBREVIATED' op0=3/>
-; DRWD:     <INST_LOAD abbrev=4 op0=3 op1=0 op2=0/>
-; DRWD:     <INST_BR abbrev='UNABBREVIATED' op0=3/>
-; DRWD:     <INST_PHI abbrev='UNABBREVIATED' op0=0 op1=6 op2=1
-; DRWD:               op3=6 op4=2/>
-; DRWD:     <INST_PHI abbrev='UNABBREVIATED' op0=0 op1=6 op2=1
-; DRWD:               op3=4 op4=2/>
-; DRWD:     <INST_RET abbrev=7/>
-; DRWD:   </FUNCTION_BLOCK abbrev='END_BLOCK'>
-; DRWD: </MODULE_BLOCK abbrev='END_BLOCK'>
diff --git a/test/NaCl/Bitcode/fast.ll b/test/NaCl/Bitcode/fast.ll
deleted file mode 100644
index 2a89ca3b4d8..00000000000
--- a/test/NaCl/Bitcode/fast.ll
+++ /dev/null
@@ -1,110 +0,0 @@
-; Tests that we don't write the fast (floating point) attributes into
-; PNaCl bitcode files (i.e. flags fast, nnan, ninf, nsz, and arcp).
-
-; Test 1: Show that flags are removed.
-; RUN: llvm-as < %s | pnacl-freeze | pnacl-thaw | llvm-dis - \
-; RUN:              | FileCheck %s
-
-; Test 2: Show that the bitcode files do not contain flags (i.e.
-; the corresponding BINOP records only have 3 values, not 4).
-; RUN: llvm-as < %s | pnacl-freeze | pnacl-bcanalyzer -dump-records \
-; RUN:              | FileCheck %s -check-prefix=RECORD
-
-define void @foo() {
-  ; Show that we handle all flags for fadd
-  %1 = fadd fast double 1.000000e+00, 2.000000e+00
-  %2 = fadd nnan double 3.000000e+00, 4.000000e+00
-  %3 = fadd ninf double 5.000000e+00, 6.000000e+00
-  %4 = fadd nsz double 7.000000e+00, 8.000000e+00
-  %5 = fadd arcp double 9.000000e+00, 10.000000e+00
-
-; CHECK:   %1 = fadd double 1.000000e+00, 2.000000e+00
-; CHECK:   %2 = fadd double 3.000000e+00, 4.000000e+00
-; CHECK:   %3 = fadd double 5.000000e+00, 6.000000e+00
-; CHECK:   %4 = fadd double 7.000000e+00, 8.000000e+00
-; CHECK:   %5 = fadd double 9.000000e+00, 1.000000e+01
-
-; RECORD: <INST_BINOP op0=10 op1=9 op2=0/>
-; RECORD: <INST_BINOP op0=9 op1=8 op2=0/>
-; RECORD: <INST_BINOP op0=8 op1=7 op2=0/>
-; RECORD: <INST_BINOP op0=7 op1=6 op2=0/>
-; RECORD: <INST_BINOP op0=6 op1=5 op2=0/>
-
-  ; Show that we handle all flags for fsub
-  %6 = fsub fast double 1.000000e+00, 2.000000e+00
-  %7 = fsub nnan double 3.000000e+00, 4.000000e+00
-  %8 = fsub ninf double 5.000000e+00, 6.000000e+00
-  %9 = fsub nsz double 7.000000e+00, 8.000000e+00
-  %10 = fsub arcp double 9.000000e+00, 10.000000e+00
-
-; CHECK:   %6 = fsub double 1.000000e+00, 2.000000e+00
-; CHECK:   %7 = fsub double 3.000000e+00, 4.000000e+00
-; CHECK:   %8 = fsub double 5.000000e+00, 6.000000e+00
-; CHECK:   %9 = fsub double 7.000000e+00, 8.000000e+00
-; CHECK:   %10 = fsub double 9.000000e+00, 1.000000e+01
-
-; RECORD: <INST_BINOP op0=15 op1=14 op2=1/>
-; RECORD: <INST_BINOP op0=14 op1=13 op2=1/>
-; RECORD: <INST_BINOP op0=13 op1=12 op2=1/>
-; RECORD: <INST_BINOP op0=12 op1=11 op2=1/>
-; RECORD: <INST_BINOP op0=11 op1=10 op2=1/>
-
-  ; Show that we can handle all flags for fmul
-  %11 = fmul fast double 1.000000e+00, 2.000000e+00
-  %12 = fmul nnan double 3.000000e+00, 4.000000e+00
-  %13 = fmul ninf double 5.000000e+00, 6.000000e+00
-  %14 = fmul nsz double 7.000000e+00, 8.000000e+00
-  %15 = fmul arcp double 9.000000e+00, 10.000000e+00
-
-; CHECK:   %11 = fmul double 1.000000e+00, 2.000000e+00
-; CHECK:   %12 = fmul double 3.000000e+00, 4.000000e+00
-; CHECK:   %13 = fmul double 5.000000e+00, 6.000000e+00
-; CHECK:   %14 = fmul double 7.000000e+00, 8.000000e+00
-; CHECK:   %15 = fmul double 9.000000e+00, 1.000000e+01
-
-; RECORD: <INST_BINOP op0=20 op1=19 op2=2/>
-; RECORD: <INST_BINOP op0=19 op1=18 op2=2/>
-; RECORD: <INST_BINOP op0=18 op1=17 op2=2/>
-; RECORD: <INST_BINOP op0=17 op1=16 op2=2/>
-; RECORD: <INST_BINOP op0=16 op1=15 op2=2/>
-
-  ; Show that we can handle all flags for fdiv
-  %16 = fdiv fast double 1.000000e+00, 2.000000e+00
-  %17 = fdiv nnan double 3.000000e+00, 4.000000e+00
-  %18 = fdiv ninf double 5.000000e+00, 6.000000e+00
-  %19 = fdiv nsz double 7.000000e+00, 8.000000e+00
-  %20 = fdiv arcp double 9.000000e+00, 10.000000e+00
-
-; CHECK:   %16 = fdiv double 1.000000e+00, 2.000000e+00
-; CHECK:   %17 = fdiv double 3.000000e+00, 4.000000e+00
-; CHECK:   %18 = fdiv double 5.000000e+00, 6.000000e+00
-; CHECK:   %19 = fdiv double 7.000000e+00, 8.000000e+00
-; CHECK:   %20 = fdiv double 9.000000e+00, 1.000000e+01
-
-; RECORD: <INST_BINOP op0=25 op1=24 op2=4/>
-; RECORD: <INST_BINOP op0=24 op1=23 op2=4/>
-; RECORD: <INST_BINOP op0=23 op1=22 op2=4/>
-; RECORD: <INST_BINOP op0=22 op1=21 op2=4/>
-; RECORD: <INST_BINOP op0=21 op1=20 op2=4/>
-
-  ; Show that we can handle all flags for frem.
-  %21 = frem fast double 1.000000e+00, 2.000000e+00
-  %22 = frem nnan double 3.000000e+00, 4.000000e+00
-  %23 = frem ninf double 5.000000e+00, 6.000000e+00
-  %24 = frem nsz double 7.000000e+00, 8.000000e+00
-  %25 = frem arcp double 9.000000e+00, 10.000000e+00
-
-; CHECK:   %21 = frem double 1.000000e+00, 2.000000e+00
-; CHECK:   %22 = frem double 3.000000e+00, 4.000000e+00
-; CHECK:   %23 = frem double 5.000000e+00, 6.000000e+00
-; CHECK:   %24 = frem double 7.000000e+00, 8.000000e+00
-; CHECK:   %25 = frem double 9.000000e+00, 1.000000e+01
-
-; RECORD: <INST_BINOP op0=30 op1=29 op2=6/>
-; RECORD: <INST_BINOP op0=29 op1=28 op2=6/>
-; RECORD: <INST_BINOP op0=28 op1=27 op2=6/>
-; RECORD: <INST_BINOP op0=27 op1=26 op2=6/>
-; RECORD: <INST_BINOP op0=26 op1=25 op2=6/>
-
-  ret void
-}
diff --git a/test/NaCl/Bitcode/flags.ll b/test/NaCl/Bitcode/flags.ll
deleted file mode 100644
index 0b510e2a48c..00000000000
--- a/test/NaCl/Bitcode/flags.ll
+++ /dev/null
@@ -1,17 +0,0 @@
-; Test that we no longer support the "nuw", "nsw", or the "exact" attributes on
-; binary operators in PNaCl bitcode files, since the PNaClABI doesn't allow
-; these attributes.
-
-; RUN: llvm-as < %s | pnacl-freeze | pnacl-thaw | llvm-dis - \
-; RUN:              | FileCheck %s
-
-define void @WrapFlags(i32, i32) {
-  %3 = add nuw i32 %0, %1
-  %4 = add nsw i32 %0, %1
-  %5 = udiv exact i32 %0, %1
-  ret void
-}
-
-; CHECK: %3 = add i32 %0, %1
-; CHECK: %4 = add i32 %0, %1
-; CHECK: %5 = udiv i32 %0, %1
diff --git a/test/NaCl/Bitcode/forward-ref-decl.ll b/test/NaCl/Bitcode/forward-ref-decl.ll
deleted file mode 100644
index ac02c850d72..00000000000
--- a/test/NaCl/Bitcode/forward-ref-decl.ll
+++ /dev/null
@@ -1,59 +0,0 @@
-; RUN: llvm-as < %s | pnacl-freeze | pnacl-bcanalyzer -dump-records \
-; RUN:              | FileCheck %s
-
-; Test that FORWARDTYPEREF declarations are emitted in the correct
-; places.  These are emitted for forward value references inside
-; functions.
-
-define external void @_start(i32 %arg) {
-; CHECK: <FUNCTION_BLOCK
-
-  br label %bb1
-; CHECK: <INST_BR
-
-bb2:
-  ; This instruction contains two forward references, because %x and
-  ; %y are defined later in the function.
-  add i32 %forward1, %forward2
-; CHECK-NEXT: <FORWARDTYPEREF
-; CHECK-NEXT: <FORWARDTYPEREF
-; CHECK-NEXT: <INST_BINOP
-
-  ; The FORWARDTYPEREF declaration should only be emitted once per
-  ; value, so the following references will not emit more of them.
-  add i32 %forward1, %forward2
-; CHECK-NEXT: <INST_BINOP
-
-  ; Test another case of a forward reference.
-  call void @_start(i32 %forward3)
-; CHECK-NEXT: <FORWARDTYPEREF
-; CHECK-NEXT: <INST_CALL
-
-  ; Test that FORWARDTYPEREF is generated for phi nodes (since phi
-  ; node operands are a special case in the writer).
-  br label %bb3
-bb3:
-  phi i32 [ %forward4, %bb2 ]
-; CHECK-NEXT: <INST_BR
-; CHECK-NEXT: <FORWARDTYPEREF
-; CHECK-NEXT: <INST_PHI
-
-  ; Test that FORWARDTYPEREF is generated for switch instructions
-  ; (since switch condition operands are a special case in the
-  ; writer).
-  switch i32 %forward5, label %bb4 [i32 0, label %bb4]
-bb4:
-; CHECK-NEXT: <FORWARDTYPEREF
-; CHECK-NEXT: <INST_SWITCH
-
-  ret void
-; CHECK-NEXT: <INST_RET/>
-
-bb1:
-  %forward1 = add i32 %arg, 100
-  %forward2 = add i32 %arg, 200
-  %forward3 = add i32 %arg, 300
-  %forward4 = add i32 %arg, 400
-  %forward5 = add i32 %arg, 500
-  br label %bb2
-}
diff --git a/test/NaCl/Bitcode/forward-ref-pointer-intrinsic.ll b/test/NaCl/Bitcode/forward-ref-pointer-intrinsic.ll
deleted file mode 100644
index e416d53e999..00000000000
--- a/test/NaCl/Bitcode/forward-ref-pointer-intrinsic.ll
+++ /dev/null
@@ -1,32 +0,0 @@
-; Test forward reference of a pointer-typed intrinsic result.
-
-; RUN: llvm-as < %s | pnacl-freeze -allow-local-symbol-tables \
-; RUN:              | pnacl-thaw -allow-local-symbol-tables \
-; RUN:              | llvm-dis - | FileCheck %s -check-prefix=TD2
-
-declare i8* @llvm.nacl.read.tp()
-
-define i32 @forward_ref() {
-  br label %block1
-
-block2:
-  %1 = load i8, i8* %3
-  %2 = ptrtoint i8* %3 to i32
-  ret i32 %2
-
-block1:
-  %3 = call i8* @llvm.nacl.read.tp()
-  br label %block2
-}
-
-; TD2:      define i32 @forward_ref() {
-; TD2-NEXT:   br label %block1
-; TD2:      block2:
-; TD2-NEXT:   %1 = inttoptr i32 %4 to i8*
-; TD2-NEXT:   %2 = load i8, i8* %1
-; TD2-NEXT:   ret i32 %4
-; TD2:      block1:
-; TD2-NEXT:   %3 = call i8* @llvm.nacl.read.tp()
-; TD2-NEXT:   %4 = ptrtoint i8* %3 to i32
-; TD2-NEXT:   br label %block2
-; TD2-NEXT: }
diff --git a/test/NaCl/Bitcode/globalvars.ll b/test/NaCl/Bitcode/globalvars.ll
deleted file mode 100644
index 3bd9890aa82..00000000000
--- a/test/NaCl/Bitcode/globalvars.ll
+++ /dev/null
@@ -1,106 +0,0 @@
-; RUN: llvm-as < %s | pnacl-freeze | pnacl-thaw | llvm-dis - | FileCheck %s
-; RUN: llvm-as < %s | pnacl-freeze | pnacl-bcanalyzer -dump-records \
-; RUN:              | FileCheck %s -check-prefix=BC
-
-; Test that we generate appropriate bitcode values for global variables.
-
-; Make sure that no struct/array/pointer types are generated by the
-; global variables.
-; BC: <TYPE_BLOCK_ID
-; BC-NEXT: <NUMENTRY
-; BC-NEXT: <VOID/>
-; BC-NEXT: <FUNCTION
-; BC-NEXT: </TYPE_BLOCK_ID>
-
-; Make sure that the function declaration for function func (below)
-; appears before the global variables block.
-; BC: <FUNCTION op0=1 op1=0 op2=0 op3=0/>
-
-; Make sure we begin the globals block after function declarations.
-; BC-NEXT: <GLOBALVAR_BLOCK
-; BC-NEXT: <COUNT op0=15/>
-
-@bytes = internal global [7 x i8] c"abcdefg"
-; CHECK: @bytes = internal global [7 x i8] c"abcdefg"
-; BC-NEXT: <VAR op0=0 op1=0/>
-; BC-NEXT: <DATA op0=97 op1=98 op2=99 op3=100 op4=101 op5=102 op6=103/>
-
-
-@ptr_to_ptr = internal global i32 ptrtoint (i32* @ptr to i32)
-; CHECK: @ptr_to_ptr = internal global i32 ptrtoint (i32* @ptr to i32)
-; BC-NEXT: <VAR op0=0 op1=0/>
-; BC-NEXT: <RELOC op0=5/>
-
-@ptr_to_func = internal global i32 ptrtoint (void ()* @func to i32)
-; CHECK: @ptr_to_func = internal global i32 ptrtoint (void ()* @func to i32)
-; BC-NEXT: <VAR op0=0 op1=0/>
-; BC-NEXT: <RELOC op0=0/>
-
-@compound = internal global <{ [3 x i8], i32 }> <{ [3 x i8] c"foo", i32 ptrtoint (void ()* @func to i32) }>
-; CHECK: @compound = internal global <{ [3 x i8], i32 }> <{ [3 x i8] c"foo", i32 ptrtoint (void ()* @func to i32) }>
-; BC-NEXT: <VAR op0=0 op1=0/>
-; BC-NEXT: <COMPOUND op0=2/>
-; BC-NEXT: <DATA op0=102 op1=111 op2=111/>
-; BC-NEXT: <RELOC op0=0/>
-
-@ptr = internal global i32 ptrtoint ([7 x i8]* @bytes to i32)
-; CHECK: @ptr = internal global i32 ptrtoint ([7 x i8]* @bytes to i32)
-; BC-NEXT: <VAR op0=0 op1=0/>
-; BC-NEXT: <RELOC op0=1/>
-
-@addend_ptr = internal global i32 add (i32 ptrtoint (i32* @ptr to i32), i32 1)
-; CHECK: @addend_ptr = internal global i32 add (i32 ptrtoint (i32* @ptr to i32), i32 1)
-; BC-NEXT: <VAR op0=0 op1=0/>
-; BC-NEXT: <RELOC op0=5 op1=1/>
-
-@addend_negative = internal global i32 add (i32 ptrtoint (i32* @ptr to i32), i32 -1)
-; CHECK: @addend_negative = internal global i32 add (i32 ptrtoint (i32* @ptr to i32), i32 -1)
-; BC-NEXT: <VAR op0=0 op1=0/>
-; BC-NEXT: <RELOC op0=5 op1=4294967295/>
-
-@addend_array1 = internal global i32 add (i32 ptrtoint ([7 x i8]* @bytes to i32), i32 1)
-; CHECK: @addend_array1 = internal global i32 add (i32 ptrtoint ([7 x i8]* @bytes to i32), i32 1)
-; BC-NEXT: <VAR op0=0 op1=0/>
-; BC-NEXT: <RELOC op0=1 op1=1/>
-
-@addend_array2 = internal global i32 add (i32 ptrtoint ([7 x i8]* @bytes to i32), i32 7)
-; CHECK: @addend_array2 = internal global i32 add (i32 ptrtoint ([7 x i8]* @bytes to i32), i32 7)
-; BC-NEXT: <VAR op0=0 op1=0/>
-; BC-NEXT: <RELOC op0=1 op1=7/>
-
-@addend_array3 = internal global i32 add (i32 ptrtoint ([7 x i8]* @bytes to i32), i32 9)
-; CHECK: @addend_array3 = internal global i32 add (i32 ptrtoint ([7 x i8]* @bytes to i32), i32 9)
-; BC-NEXT: <VAR op0=0 op1=0/>
-; BC-NEXT: <RELOC op0=1 op1=9/>
-
-@addend_struct1 = internal global i32 add (i32 ptrtoint (<{ [3 x i8], i32 }>* @compound to i32), i32 1)
-; CHECK: @addend_struct1 = internal global i32 add (i32 ptrtoint (<{ [3 x i8], i32 }>* @compound to i32), i32 1)
-; BC-NEXT: <VAR op0=0 op1=0/>
-; BC-NEXT: <RELOC op0=4 op1=1/>
-
-@addend_struct2 = internal global i32 add (i32 ptrtoint (<{ [3 x i8], i32 }>* @compound to i32), i32 4)
-; CHECK: @addend_struct2 = internal global i32 add (i32 ptrtoint (<{ [3 x i8], i32 }>* @compound to i32), i32 4)
-; BC-NEXT: <VAR op0=0 op1=0/>
-; BC-NEXT: <RELOC op0=4 op1=4/>
-
-@ptr_to_func_align = internal global i32 ptrtoint (void ()* @func to i32), align 8
-; CHECK: @ptr_to_func_align = internal global i32 ptrtoint (void ()* @func to i32), align 8
-; BC-NEXT: <VAR op0=4 op1=0/>
-; BC-NEXT: <RELOC op0=0/>
-
-@char = internal constant [1 x i8] c"0"
-; CHECK: @char = internal constant [1 x i8] c"0"
-; BC-NEXT: <VAR op0=0 op1=1/>
-; BC-NEXT: <DATA op0=48/>
-
-@short = internal constant [2 x i8] zeroinitializer
-; CHECK: @short = internal constant [2 x i8] zeroinitializer
-; BC-NEXT:  <VAR op0=0 op1=1/>
-; BC-NEXT:  <ZEROFILL op0=2/>
-
-; BC-NEXT: </GLOBALVAR_BLOCK>
-
-define void @func() {
-  ret void
-}
-
diff --git a/test/NaCl/Bitcode/implicit-datalayout.ll b/test/NaCl/Bitcode/implicit-datalayout.ll
deleted file mode 100644
index 0079bcfd8e4..00000000000
--- a/test/NaCl/Bitcode/implicit-datalayout.ll
+++ /dev/null
@@ -1,9 +0,0 @@
-; RUN: llvm-as < %s | pnacl-freeze | pnacl-thaw - | llvm-dis - | FileCheck %s
-
-; The "datalayout" field is considered to be implicit in the pexe.  It
-; is not stored in the pexe; the reader adds it implicitly.
-;
-; The most important parts of the datalayout for PNaCl are the pointer
-; size and the endianness ("e" for little endian).
-
-; CHECK: target datalayout = "e-{{.*}}p:32:32:32-{{.*}}"
diff --git a/test/NaCl/Bitcode/intrinsic-pointer-args.ll b/test/NaCl/Bitcode/intrinsic-pointer-args.ll
deleted file mode 100644
index 237ef090385..00000000000
--- a/test/NaCl/Bitcode/intrinsic-pointer-args.ll
+++ /dev/null
@@ -1,55 +0,0 @@
-; Test that intrinsic declarations are read back correctly.
-
-; RUN: llvm-as < %s | pnacl-freeze | pnacl-thaw \
-; RUN:              | llvm-dis - | FileCheck %s -check-prefix=TD
-
-declare i8* @llvm.stacksave()
-declare void @llvm.stackrestore(i8*)
-
-declare i8* @llvm.nacl.read.tp()
-declare void @llvm.nacl.longjmp(i8*, i32)
-declare i32 @llvm.nacl.setjmp(i8*)
-
-declare void @llvm.memcpy.p0i8.p0i8.i32(i8* nocapture, i8* nocapture, i32, i32, i1)
-declare void @llvm.memmove.p0i8.p0i8.i32(i8* nocapture, i8* nocapture, i32, i32, i1)
-declare void @llvm.memset.p0i8.i32(i8* nocapture, i8, i32, i32, i1)
-
-declare i32 @llvm.nacl.atomic.load.i32(i32*, i32)
-declare i64 @llvm.nacl.atomic.load.i64(i64*, i32)
-
-declare i32 @llvm.nacl.atomic.cmpxchg.i32(i32*, i32, i32, i32, i32)
-declare i64 @llvm.nacl.atomic.cmpxchg.i64(i64*, i64, i64, i32, i32)
-
-declare void @llvm.nacl.atomic.store.i32(i32, i32*, i32)
-declare void @llvm.nacl.atomic.store.i64(i64, i64*, i32)
-
-declare i32 @llvm.nacl.atomic.rmw.i32(i32, i32*, i32, i32)
-declare i64 @llvm.nacl.atomic.rmw.i64(i32, i64*, i64, i32)
-
-declare i1 @llvm.nacl.atomic.is.lock.free(i32, i8*)
-
-
-; TD: declare i8* @llvm.stacksave()
-; TD: declare void @llvm.stackrestore(i8*)
-
-; TD: declare i8* @llvm.nacl.read.tp()
-; TD: declare void @llvm.nacl.longjmp(i8*, i32)
-; TD: declare i32 @llvm.nacl.setjmp(i8*)
-
-; TD: declare void @llvm.memcpy.p0i8.p0i8.i32(i8* {{.*}}, i8* {{.*}}, i32, i32, i1)
-; TD: declare void @llvm.memmove.p0i8.p0i8.i32(i8* {{.*}}, i8* {{.*}}, i32, i32, i1)
-; TD: declare void @llvm.memset.p0i8.i32(i8* {{.*}}, i8, i32, i32, i1)
-
-; TD: declare i32 @llvm.nacl.atomic.load.i32(i32*, i32)
-; TD: declare i64 @llvm.nacl.atomic.load.i64(i64*, i32)
-
-; TD: declare i32 @llvm.nacl.atomic.cmpxchg.i32(i32*, i32, i32, i32, i32)
-; TD: declare i64 @llvm.nacl.atomic.cmpxchg.i64(i64*, i64, i64, i32, i32)
-
-; TD: declare void @llvm.nacl.atomic.store.i32(i32, i32*, i32)
-; TD: declare void @llvm.nacl.atomic.store.i64(i64, i64*, i32)
-
-; TD: declare i32 @llvm.nacl.atomic.rmw.i32(i32, i32*, i32, i32)
-; TD: declare i64 @llvm.nacl.atomic.rmw.i64(i32, i64*, i64, i32)
-
-; TD: declare i1 @llvm.nacl.atomic.is.lock.free(i32, i8*)
diff --git a/test/NaCl/Bitcode/inttoptr-elide.ll b/test/NaCl/Bitcode/inttoptr-elide.ll
deleted file mode 100644
index 2ccc47b4ecf..00000000000
--- a/test/NaCl/Bitcode/inttoptr-elide.ll
+++ /dev/null
@@ -1,151 +0,0 @@
-; Test how we handle eliding inttoptr instructions.
-
-; RUN: llvm-as < %s | pnacl-freeze \
-; RUN:              | pnacl-bcanalyzer -dump-records \
-; RUN:              | FileCheck %s -check-prefix=PF2
-
-; RUN: llvm-as < %s | pnacl-freeze -allow-local-symbol-tables \
-; RUN:              | pnacl-thaw -allow-local-symbol-tables \
-; RUN:              | llvm-dis - | FileCheck %s -check-prefix=TD2
-
-; ------------------------------------------------------
-
-; Test that we elide the simple case of inttoptr of a load.
-define void @SimpleLoad(i32 %i) {
-  %1 = inttoptr i32 %i to i32*
-  %2 = load i32, i32* %1, align 4
-  ret void
-}
-
-; TD2:      define void @SimpleLoad(i32 %i) {
-; TD2-NEXT:   %1 = inttoptr i32 %i to i32*
-; TD2-NEXT:   %2 = load i32, i32* %1, align 4
-; TD2-NEXT:   ret void
-; TD2-NEXT: }
-
-; PF2:       <FUNCTION_BLOCK>
-; PF2-NEXT:    <DECLAREBLOCKS op0=1/>
-; PF2-NEXT:    <INST_LOAD op0=1 op1=3 op2=0/>
-; PF2-NEXT:    <INST_RET/>
-; PF2-NEXT:  </FUNCTION_BLOCK>
-
-; ------------------------------------------------------
-
-; Test that we can handle multiple inttoptr of loads.
-define i32 @TwoLoads(i32 %i) {
-  %1 = inttoptr i32 %i to i32*
-  %2 = load i32, i32* %1, align 4
-  %3 = inttoptr i32 %i to i32*
-  %4 = load i32, i32* %3, align 4
-  %5 = add i32 %2, %4
-  ret i32 %5
-}
-
-; TD2:      define i32 @TwoLoads(i32 %i) {
-; TD2-NEXT:   %1 = inttoptr i32 %i to i32*
-; TD2-NEXT:   %2 = load i32, i32* %1, align 4
-; TD2-NEXT:   %3 = load i32, i32* %1, align 4
-; TD2-NEXT:   %4 = add i32 %2, %3
-; TD2-NEXT:   ret i32 %4
-; TD2-NEXT: }
-
-; PF2:       <FUNCTION_BLOCK>
-; PF2-NEXT:    <DECLAREBLOCKS op0=1/>
-; PF2-NEXT:    <INST_LOAD op0=1 op1=3 op2=0/>
-; PF2-NEXT:    <INST_LOAD op0=2 op1=3 op2=0/>
-; PF2-NEXT:    <INST_BINOP op0=2 op1=1 op2=0/>
-; PF2-NEXT:    <INST_RET op0=1/>
-; PF2-NEXT:  </FUNCTION_BLOCK>
-
-; ------------------------------------------------------
-
-; Test how we handle inttoptrs, if optimized in the input file. This
-; case tests within a single block.
-define i32 @TwoLoadOptOneBlock(i32 %i) {
-  %1 = inttoptr i32 %i to i32*
-  %2 = load i32, i32* %1, align 4
-  %3 = load i32, i32* %1, align 4
-  %4 = add i32 %2, %3
-  ret i32 %4
-}
-
-; TD2:      define i32 @TwoLoadOptOneBlock(i32 %i) {
-; TD2-NEXT:   %1 = inttoptr i32 %i to i32*
-; TD2-NEXT:   %2 = load i32, i32* %1, align 4
-; TD2-NEXT:   %3 = load i32, i32* %1, align 4
-; TD2-NEXT:   %4 = add i32 %2, %3
-; TD2-NEXT:   ret i32 %4
-; TD2-NEXT: }
-
-; PF2:       <FUNCTION_BLOCK>
-; PF2-NEXT:    <DECLAREBLOCKS op0=1/>
-; PF2-NEXT:    <INST_LOAD op0=1 op1=3 op2=0/>
-; PF2-NEXT:    <INST_LOAD op0=2 op1=3 op2=0/>
-; PF2-NEXT:    <INST_BINOP op0=2 op1=1 op2=0/>
-; PF2-NEXT:    <INST_RET op0=1/>
-; PF2-NEXT:  </FUNCTION_BLOCK>
-
-; ------------------------------------------------------
-
-; Test how we handle inttoptrs if optimized in the input file.  This
-; case tests accross blocks.
-define i32 @TwoLoadOptTwoBlocks(i32 %i) {
-  %1 = inttoptr i32 %i to i32*
-  %2 = load i32, i32* %1, align 4
-  %3 = load i32, i32* %1, align 4
-  %4 = add i32 %2, %3
-  br label %BB
-
-BB:
-  %5 = load i32, i32* %1, align 4
-  %6 = load i32, i32* %1, align 4
-  %7 = add i32 %5, %6
-  ret i32 %7
-}
-
-; TD2:      define i32 @TwoLoadOptTwoBlocks(i32 %i) {
-; TD2-NEXT:   %1 = inttoptr i32 %i to i32*
-; TD2-NEXT:   %2 = load i32, i32* %1, align 4
-; TD2-NEXT:   %3 = load i32, i32* %1, align 4
-; TD2-NEXT:   %4 = add i32 %2, %3
-; TD2-NEXT:   br label %BB
-; TD2:      BB:
-; TD2-NEXT:   %5 = inttoptr i32 %i to i32*
-; TD2-NEXT:   %6 = load i32, i32* %5, align 4
-; TD2-NEXT:   %7 = load i32, i32* %5, align 4
-; TD2-NEXT:   %8 = add i32 %6, %7
-; TD2-NEXT:   ret i32 %8
-; TD2-NEXT: }
-
-; PF2:        <FUNCTION_BLOCK>
-; PF2-NEXT:     <DECLAREBLOCKS op0=2/>
-; PF2-NEXT:     <INST_LOAD op0=1 op1=3 op2=0/>
-; PF2-NEXT:     <INST_LOAD op0=2 op1=3 op2=0/>
-; PF2-NEXT:     <INST_BINOP op0=2 op1=1 op2=0/>
-; PF2-NEXT:     <INST_BR op0=1/>
-; PF2-NEXT:     <INST_LOAD op0=4 op1=3 op2=0/>
-; PF2-NEXT:     <INST_LOAD op0=5 op1=3 op2=0/>
-; PF2-NEXT:     <INST_BINOP op0=2 op1=1 op2=0/>
-; PF2-NEXT:     <INST_RET op0=1/>
-; PF2-NEXT:   </FUNCTION_BLOCK>
-
-; ------------------------------------------------------
-
-; Test that we elide the simple case of inttoptr for a store.
-define void @SimpleStore(i32 %i) {
-  %1 = inttoptr i32 %i to i32*
-  store i32 %i, i32* %1, align 4
-  ret void
-}
-
-; TD2:      define void @SimpleStore(i32 %i) {
-; TD2-NEXT:   %1 = inttoptr i32 %i to i32*
-; TD2-NEXT:   store i32 %i, i32* %1, align 4
-; TD2-NEXT:   ret void
-; TD2-NEXT: }
-
-; PF2:      <FUNCTION_BLOCK>
-; PF2-NEXT:   <DECLAREBLOCKS op0=1/>
-; PF2-NEXT:   <INST_STORE op0=1 op1=1 op2=3/>
-; PF2-NEXT:   <INST_RET/>
-; PF2-NEXT: </FUNCTION_BLOCK>
diff --git a/test/NaCl/Bitcode/inttoptr-of-ptrtoint-elide.ll b/test/NaCl/Bitcode/inttoptr-of-ptrtoint-elide.ll
deleted file mode 100644
index 3a440c0b236..00000000000
--- a/test/NaCl/Bitcode/inttoptr-of-ptrtoint-elide.ll
+++ /dev/null
@@ -1,32 +0,0 @@
-; Test that the writer elides an inttoptr of a ptrtoint.
-
-; RUN: llvm-as < %s | pnacl-freeze \
-; RUN:              | pnacl-bcanalyzer -dump-records \
-; RUN:              | FileCheck %s -check-prefix=PF2
-
-; RUN: llvm-as < %s | pnacl-freeze | pnacl-thaw \
-; RUN:              | llvm-dis - | FileCheck %s -check-prefix=TD2
-
-
-@bytes = internal global [4 x i8] c"abcd"
-
-define void @inttoptr_of_ptrtoint() {
-  ; These two instructions are usually replaced with an equivalent
-  ; bitcast, but either sequence is allowed by the PNaCl ABI verifier.
-  %1 = ptrtoint [4 x i8]* @bytes to i32
-  %2 = inttoptr i32 %1 to i8*
-  load i8, i8* %2
-  ret void
-}
-
-; TD2:      define void @inttoptr_of_ptrtoint() {
-; TD2-NEXT:   %1 = bitcast [4 x i8]* @bytes to i8*
-; TD2-NEXT:   %2 = load i8, i8* %1
-; TD2-NEXT:   ret void
-; TD2-NEXT: }
-
-; PF2:       <FUNCTION_BLOCK>
-; PF2-NEXT:    <DECLAREBLOCKS op0=1/>
-; PF2-NEXT:    <INST_LOAD {{.*}}/>
-; PF2-NEXT:    <INST_RET/>
-; PF2:       </FUNCTION_BLOCK>
diff --git a/test/NaCl/Bitcode/invalid.test b/test/NaCl/Bitcode/invalid.test
deleted file mode 100644
index ab54c9d0f19..00000000000
--- a/test/NaCl/Bitcode/invalid.test
+++ /dev/null
@@ -1,36 +0,0 @@
-; Check when separator(,) not followed by a value.
-
-RUN: not pnacl-bcfuzz -bitcode-as-text %p/Inputs/bad-separator.tbc \
-RUN:    -output - 2>&1 | FileCheck --check-prefix=BAD-SEP %s
-
-BAD-SEP: Value expected after separator, but not found
-
-; Check that comparison operator of branch is i1.
-
-RUN: pnacl-bcfuzz -bitcode-as-text %p/Inputs/bad-branch-compare.tbc \
-RUN:    -output - | not pnacl-bcdis | FileCheck --check-prefix=BAD-BC-BR-CMP %s
-
-BAD-BC-BR-CMP:      br i32 %p0, label %b1, label %b2;
-BAD-BC-BR-CMP-NEXT: Branch condition not i1
-
-RUN: pnacl-bcfuzz -bitcode-as-text %p/Inputs/bad-branch-compare.tbc \
-RUN:     -output - | not pnacl-thaw -verbose-parse-errors -o /dev/null 2>&1 \
-RUN:               | FileCheck --check-prefix=BAD-BR-CMP %s
-
-BAD-BR-CMP: Type of branch condition not i1. Found: i32
-
-; Tests that we recognize wrong abbreviation index bitsize in a blockinfo block,
-; and properly exit with an appropriate error message.
-; Tests issue: https://code.google.com/p/nativeclient/issues/detail?id=4215
-RUN: not pnacl-bcfuzz -bitcode-as-text %p/Inputs/bad-blockinfo-numbits.tbc \
-RUN:     -output - 2>&1 | FileCheck --check-prefix=BAD-BLKINFO_BITS %s
-
-BAD-BLKINFO_BITS: Numbits entry for abbreviations in blockinfo block not 2.
-BAD-BLKINFO_BITS: Failed to write bitcode
-
-; Check that we catch error where line doesn't begin with a record code.
-
-RUN: not pnacl-bcfuzz -bitcode-as-text %p/Inputs/bad-code.tbc \
-RUN:    -output - 2>&1 | FileCheck --check-prefix=BAD-CODE %s
-
-BAD-CODE: Bitcode record doesn't begin with a record code
diff --git a/test/NaCl/Bitcode/lit.local.cfg b/test/NaCl/Bitcode/lit.local.cfg
deleted file mode 100644
index 00bd9b83b52..00000000000
--- a/test/NaCl/Bitcode/lit.local.cfg
+++ /dev/null
@@ -1 +0,0 @@
-config.suffixes = ['.ll', '.c', '.cpp', '.test']
diff --git a/test/NaCl/Bitcode/no-structs.ll b/test/NaCl/Bitcode/no-structs.ll
deleted file mode 100644
index 87f5b721b00..00000000000
--- a/test/NaCl/Bitcode/no-structs.ll
+++ /dev/null
@@ -1,32 +0,0 @@
-; Tests that even though global variables can define structured types,
-; they types are not put into the bitcode file.
-
-; RUN: llvm-as < %s | pnacl-freeze \
-; RUN:              | pnacl-bcanalyzer -dump-records \
-; RUN:              | FileCheck %s -check-prefix=PF2
-
-declare void @func()
-
-@compound = internal global <{ [4 x i8], i32 }>
-    <{ [4 x i8] c"home", i32 ptrtoint (void ()* @func to i32) }>
-
-define void @CheckBitcastGlobal() {
-  %1 = bitcast <{ [4 x i8], i32}>* @compound to i32*
-  %2 = load i32, i32* %1, align 4
-  ret void
-}
-
-define void @CheckPtrToIntGlobal() {
-  %1 = ptrtoint <{ [4 x i8], i32 }>* @compound to i32
-  %2 = add i32 %1, 0
-  ret void
-}
-
-; Note that it doesn't define a struct type.
-
-; PF2:      <TYPE_BLOCK_ID>
-; PF2-NEXT:   <NUMENTRY op0=3/>
-; PF2-NEXT:   <INTEGER op0=32/>
-; PF2-NEXT:   <VOID/>
-; PF2-NEXT:   <FUNCTION op0=0 op1=1/>
-; PF2-NEXT: </TYPE_BLOCK_ID>
diff --git a/test/NaCl/Bitcode/null.ll b/test/NaCl/Bitcode/null.ll
deleted file mode 100644
index 866c2017bfa..00000000000
--- a/test/NaCl/Bitcode/null.ll
+++ /dev/null
@@ -1,82 +0,0 @@
-; Test that we no longer generate NULL for numeric constants.
-
-; RUN: llvm-as < %s | pnacl-freeze  \
-; RUN:              | pnacl-bcanalyzer -dump-records \
-; RUN:              | FileCheck %s -check-prefix=PF
-
-; RUN: llvm-as < %s | pnacl-freeze | pnacl-thaw \
-; RUN:              | llvm-dis - | FileCheck %s -check-prefix=TD
-
-; ------------------------------------------------------
-
-define void @TestIntegers() {
-  %1 = and i1 true, false
-  %2 = add i8 1, 0
-  %3 = add i16 1, 0
-  %4 = add i32 1, 0
-  %5 = add i64 1, 0
-  ret void
-}
-
-; TD:      define void @TestIntegers() {
-; TD-NEXT:   %1 = and i1 true, false
-; TD-NEXT:   %2 = add i8 1, 0
-; TD-NEXT:   %3 = add i16 1, 0
-; TD-NEXT:   %4 = add i32 1, 0
-; TD-NEXT:   %5 = add i64 1, 0
-; TD-NEXT:   ret void
-; TD-NEXT: }
-
-; PF:      <FUNCTION_BLOCK>
-; PF-NEXT:   <DECLAREBLOCKS op0=1/>
-; PF-NEXT:   <CONSTANTS_BLOCK>
-; PF-NEXT:     <SETTYPE op0=1/>
-; PF-NEXT:     <INTEGER op0=3/>
-; PF-NEXT:     <INTEGER op0=0/>
-; PF-NEXT:     <SETTYPE op0=2/>
-; PF-NEXT:     <INTEGER op0=2/>
-; PF-NEXT:     <INTEGER op0=0/>
-; PF-NEXT:     <SETTYPE op0=3/>
-; PF-NEXT:     <INTEGER op0=2/>
-; PF-NEXT:     <INTEGER op0=0/>
-; PF-NEXT:     <SETTYPE op0=4/>
-; PF-NEXT:     <INTEGER op0=2/>
-; PF-NEXT:     <INTEGER op0=0/>
-; PF-NEXT:     <SETTYPE op0=5/>
-; PF-NEXT:     <INTEGER op0=2/>
-; PF-NEXT:     <INTEGER op0=0/>
-; PF-NEXT:   </CONSTANTS_BLOCK>
-; PF-NEXT:   <INST_BINOP op0=10 op1=9 op2=10/>
-; PF-NEXT:   <INST_BINOP op0=9 op1=8 op2=0/>
-; PF-NEXT:   <INST_BINOP op0=8 op1=7 op2=0/>
-; PF-NEXT:   <INST_BINOP op0=7 op1=6 op2=0/>
-; PF-NEXT:   <INST_BINOP op0=6 op1=5 op2=0/>
-; PF-NEXT:   <INST_RET/>
-; PF-NEXT: </FUNCTION_BLOCK>
-
-define void @TestFloats() {
-  %1 = fadd float 1.0, 0.0
-  %2 = fadd double 1.0, 0.0
-  ret void
-}
-
-; TD:      define void @TestFloats() {
-; TD-NEXT:   %1 = fadd float 1.000000e+00, 0.000000e+00
-; TD-NEXT:   %2 = fadd double 1.000000e+00, 0.000000e+00
-; TD-NEXT:   ret void
-; TD-NEXT: }
-
-; PF:      <FUNCTION_BLOCK>
-; PF-NEXT:   <DECLAREBLOCKS op0=1/>
-; PF-NEXT:   <CONSTANTS_BLOCK>
-; PF-NEXT:     <SETTYPE op0=6/>
-; PF-NEXT:     <FLOAT op0=1065353216/>
-; PF-NEXT:     <FLOAT op0=0/>
-; PF-NEXT:     <SETTYPE op0=7/>
-; PF-NEXT:     <FLOAT op0=4607182418800017408/>
-; PF-NEXT:     <FLOAT op0=0/>
-; PF-NEXT:   </CONSTANTS_BLOCK>
-; PF-NEXT:   <INST_BINOP op0=4 op1=3 op2=0/>
-; PF-NEXT:   <INST_BINOP op0=3 op1=2 op2=0/>
-; PF-NEXT:   <INST_RET/>
-; PF-NEXT: </FUNCTION_BLOCK>
diff --git a/test/NaCl/Bitcode/pnacl-bcdis/Inputs/bad-zext-cast.tbc b/test/NaCl/Bitcode/pnacl-bcdis/Inputs/bad-zext-cast.tbc
deleted file mode 100644
index 2976d8b9a75..00000000000
--- a/test/NaCl/Bitcode/pnacl-bcdis/Inputs/bad-zext-cast.tbc
+++ /dev/null
@@ -1,31 +0,0 @@
-65535,8,2;
-1,1;
-65535,17,2;
-1,4;
-7,32;
-2;
-21,0,0,0;
-7,1;
-65534;
-8,2,0,0,0;
-65535,19,2;
-5,0;
-65534;
-65535,14,2;
-1,0,102,105,98;
-65534;
-65535,12,2;
-1,3;
-65535,11,2;
-1,0;
-4,2;
-65534;
-28,2,1,36;
-11,1,2,1;
-10,2;
-3,3,2,1;
-34,0,5,1;
-2,1,5,0;
-10,1;
-65534;
-65534;
diff --git a/test/NaCl/Bitcode/pnacl-bcdis/alloca.ll b/test/NaCl/Bitcode/pnacl-bcdis/alloca.ll
deleted file mode 100644
index 4be2914a2d5..00000000000
--- a/test/NaCl/Bitcode/pnacl-bcdis/alloca.ll
+++ /dev/null
@@ -1,21 +0,0 @@
-; Test alloca instructions.
-
-; RUN: llvm-as < %s | pnacl-freeze | pnacl-bccompress --remove-abbreviations \
-; RUN:              | pnacl-bcdis | FileCheck %s
-
-define void @AllocaTests(i32 %p0) {
-
-; CHECK:               |                             |  %b0:
-
-  %v0 = alloca i8, i32 1, align 4
-  %v1 = alloca i8, i32 4, align 4
-  %v2 = alloca i8, i32 %p0, align 128
-  ret void
-
-; CHECK-NEXT:    {{.*}}|    3: <19, 2, 3>            |    %v0 = alloca i8, i32 %c0, align 4;
-; CHECK-NEXT:    {{.*}}|    3: <19, 2, 3>            |    %v1 = alloca i8, i32 %c1, align 4;
-; CHECK-NEXT:    {{.*}}|    3: <19, 5, 8>            |    %v2 = alloca i8, i32 %p0, 
-; CHECK-NEXT:          |                             |        align 128;
-; CHECK-NEXT:    {{.*}}|    3: <10>                  |    ret void;
-
-}
diff --git a/test/NaCl/Bitcode/pnacl-bcdis/binops.ll b/test/NaCl/Bitcode/pnacl-bcdis/binops.ll
deleted file mode 100644
index 57c18493bcc..00000000000
--- a/test/NaCl/Bitcode/pnacl-bcdis/binops.ll
+++ /dev/null
@@ -1,350 +0,0 @@
-; Test dumping binary operations.
-
-; RUN: llvm-as < %s | pnacl-freeze | pnacl-bccompress --remove-abbreviations \
-; RUN:              | pnacl-bcdis | FileCheck %s
-
-; Test integer binary operators.
-define internal void @IntOps(i64 %p0) {
-  ; Define different sized integer ops.
-  %v0 = trunc i64 %p0 to i8
-  %v1 = trunc i64 %p0 to i16
-  %v2 = trunc i64 %p0 to i32
-  %v3 = zext i32 %v2 to i64
-  %v4 = trunc i32 %v2 to i1
-
-  %v5 = add i8 %v0, 1
-  %v6 = add i16 %v1, 2
-  %v7 = add i32 %v2, 3
-  %v8 = add i64 %v3, 4
-
-; CHECK:         {{.*}}|    3: <2, 5, 8, 0>          |    %v5 = add i8 %v0, %c2;
-; CHECK-NEXT:    {{.*}}|    3: <2, 5, 8, 0>          |    %v6 = add i16 %v1, %c3;
-; CHECK-NEXT:    {{.*}}|    3: <2, 5, 11, 0>         |    %v7 = add i32 %v2, %c1;
-; CHECK-NEXT:    {{.*}}|    3: <2, 5, 13, 0>         |    %v8 = add i64 %v3, %c0;
-
-  %v9 = sub i8 1, %v0
-  %v10 = sub i16 2, %v1
-  %v11 = sub i32 3, %v2
-  %v12 = sub i64 4, %v3
-
-; CHECK-NEXT:    {{.*}}|    3: <2, 12, 9, 1>         |    %v9 = sub i8 %c2, %v0;
-; CHECK-NEXT:    {{.*}}|    3: <2, 12, 9, 1>         |    %v10 = sub i16 %c3, %v1;
-; CHECK-NEXT:    {{.*}}|    3: <2, 15, 9, 1>         |    %v11 = sub i32 %c1, %v2;
-; CHECK-NEXT:    {{.*}}|    3: <2, 17, 9, 1>         |    %v12 = sub i64 %c0, %v3;
-
-  %v13 = mul i8 %v0, 1
-  %v14 = mul i16 %v1, 2
-  %v15 = mul i32 %v2, 3
-  %v16 = mul i64 %v3, 4
-
-; CHECK-NEXT:    {{.*}}|    3: <2, 13, 16, 2>        |    %v13 = mul i8 %v0, %c2;
-; CHECK-NEXT:    {{.*}}|    3: <2, 13, 16, 2>        |    %v14 = mul i16 %v1, %c3;
-; CHECK-NEXT:    {{.*}}|    3: <2, 13, 19, 2>        |    %v15 = mul i32 %v2, %c1;
-; CHECK-NEXT:    {{.*}}|    3: <2, 13, 21, 2>        |    %v16 = mul i64 %v3, %c0;
-
-  %v17 = udiv i8 %v0, 1
-  %v18 = udiv i16 %v1, 2
-  %v19 = udiv i32 %v2, 3
-  %v20 = udiv i64 %v3, 4
-
-; CHECK-NEXT:    {{.*}}|    3: <2, 17, 20, 3>        |    %v17 = udiv i8 %v0, %c2;
-; CHECK-NEXT:    {{.*}}|    3: <2, 17, 20, 3>        |    %v18 = udiv i16 %v1, %c3;
-; CHECK-NEXT:    {{.*}}|    3: <2, 17, 23, 3>        |    %v19 = udiv i32 %v2, %c1;
-; CHECK-NEXT:    {{.*}}|    3: <2, 17, 25, 3>        |    %v20 = udiv i64 %v3, %c0;
-
-  %v21 = sdiv i8 1, %v0
-  %v22 = sdiv i16 2, %v1
-  %v23 = sdiv i32 3, %v2
-  %v24 = sdiv i64 4, %v3
-
-; CHECK-NEXT:    {{.*}}|    3: <2, 24, 21, 4>        |    %v21 = sdiv i8 %c2, %v0;
-; CHECK-NEXT:    {{.*}}|    3: <2, 24, 21, 4>        |    %v22 = sdiv i16 %c3, %v1;
-; CHECK-NEXT:    {{.*}}|    3: <2, 27, 21, 4>        |    %v23 = sdiv i32 %c1, %v2;
-; CHECK-NEXT:    {{.*}}|    3: <2, 29, 21, 4>        |    %v24 = sdiv i64 %c0, %v3;
-
-  %v25 = urem i8 1, %v0
-  %v26 = urem i16 2, %v1
-  %v27 = urem i32 3, %v2
-  %v28 = urem i64 4, %v3
-
-; CHECK-NEXT:    {{.*}}|    3: <2, 28, 25, 5>        |    %v25 = urem i8 %c2, %v0;
-; CHECK-NEXT:    {{.*}}|    3: <2, 28, 25, 5>        |    %v26 = urem i16 %c3, %v1;
-; CHECK-NEXT:    {{.*}}|    3: <2, 31, 25, 5>        |    %v27 = urem i32 %c1, %v2;
-; CHECK-NEXT:    {{.*}}|    3: <2, 33, 25, 5>        |    %v28 = urem i64 %c0, %v3;
-
-  %v29 = srem i8 %v0, 1
-  %v30 = srem i16 %v1, 2
-  %v31 = srem i32 %v2, 3
-  %v32 = srem i64 %v3, 4
-
-; CHECK-NEXT:    {{.*}}|    3: <2, 29, 32, 6>        |    %v29 = srem i8 %v0, %c2;
-; CHECK-NEXT:    {{.*}}|    3: <2, 29, 32, 6>        |    %v30 = srem i16 %v1, %c3;
-; CHECK-NEXT:    {{.*}}|    3: <2, 29, 35, 6>        |    %v31 = srem i32 %v2, %c1;
-; CHECK-NEXT:    {{.*}}|    3: <2, 29, 37, 6>        |    %v32 = srem i64 %v3, %c0;
-
-  %v33 = shl i8 1, %v0
-  %v34 = shl i16 2, %v1
-  %v35 = shl i32 3, %v2
-  %v36 = shl i64 4, %v3
-
-; CHECK-NEXT:    {{.*}}|    3: <2, 36, 33, 7>        |    %v33 = shl i8 %c2, %v0;
-; CHECK-NEXT:    {{.*}}|    3: <2, 36, 33, 7>        |    %v34 = shl i16 %c3, %v1;
-; CHECK-NEXT:    {{.*}}|    3: <2, 39, 33, 7>        |    %v35 = shl i32 %c1, %v2;
-; CHECK-NEXT:    {{.*}}|    3: <2, 41, 33, 7>        |    %v36 = shl i64 %c0, %v3;
-
-  %v37 = lshr i8 %v0, 1
-  %v38 = lshr i16 %v1, 2
-  %v39 = lshr i32 %v2, 3
-  %v40 = lshr i64 %v3, 4
-
-; CHECK-NEXT:    {{.*}}|    3: <2, 37, 40, 8>        |    %v37 = lshr i8 %v0, %c2;
-; CHECK-NEXT:    {{.*}}|    3: <2, 37, 40, 8>        |    %v38 = lshr i16 %v1, %c3;
-; CHECK-NEXT:    {{.*}}|    3: <2, 37, 43, 8>        |    %v39 = lshr i32 %v2, %c1;
-; CHECK-NEXT:    {{.*}}|    3: <2, 37, 45, 8>        |    %v40 = lshr i64 %v3, %c0;
-
-  %v41 = ashr i8 %v0, 1
-  %v42 = ashr i16 %v1, 2
-  %v43 = ashr i32 %v2, 3
-  %v44 = ashr i64 %v3, 4
-
-; CHECK-NEXT:    {{.*}}|    3: <2, 41, 44, 9>        |    %v41 = ashr i8 %v0, %c2;
-; CHECK-NEXT:    {{.*}}|    3: <2, 41, 44, 9>        |    %v42 = ashr i16 %v1, %c3;
-; CHECK-NEXT:    {{.*}}|    3: <2, 41, 47, 9>        |    %v43 = ashr i32 %v2, %c1;
-; CHECK-NEXT:    {{.*}}|    3: <2, 41, 49, 9>        |    %v44 = ashr i64 %v3, %c0;
-
-  %v45 = and i1 %v4, 0
-  %v46 = and i8 %v0, 1
-  %v47 = and i16 %v1, 2
-  %v48 = and i32 %v2, 3
-  %v49 = and i64 %v3, 4
-
-; CHECK-NEXT:    {{.*}}|    3: <2, 41, 46, 10>       |    %v45 = and i1 %v4, %c4;
-; CHECK-NEXT:    {{.*}}|    3: <2, 46, 49, 10>       |    %v46 = and i8 %v0, %c2;
-; CHECK-NEXT:    {{.*}}|    3: <2, 46, 49, 10>       |    %v47 = and i16 %v1, %c3;
-; CHECK-NEXT:    {{.*}}|    3: <2, 46, 52, 10>       |    %v48 = and i32 %v2, %c1;
-; CHECK-NEXT:    {{.*}}|    3: <2, 46, 54, 10>       |    %v49 = and i64 %v3, %c0;
-
-  %v50 = or i1 0, %v4
-  %v51 = or i8 1, %v0
-  %v52 = or i16 2, %v1
-  %v53 = or i32 3, %v2
-  %v54 = or i64 4, %v3
-
-; CHECK-NEXT:    {{.*}}|    3: <2, 51, 46, 11>       |    %v50 = or i1 %c4, %v4;
-; CHECK-NEXT:    {{.*}}|    3: <2, 54, 51, 11>       |    %v51 = or i8 %c2, %v0;
-; CHECK-NEXT:    {{.*}}|    3: <2, 54, 51, 11>       |    %v52 = or i16 %c3, %v1;
-; CHECK-NEXT:    {{.*}}|    3: <2, 57, 51, 11>       |    %v53 = or i32 %c1, %v2;
-; CHECK-NEXT:    {{.*}}|    3: <2, 59, 51, 11>       |    %v54 = or i64 %c0, %v3;
-
-  %v55 = xor i1 %v4, 0
-  %v56 = xor i8 %v0, 1
-  %v57 = xor i16 %v1, 2
-  %v58 = xor i32 %v2, 3
-  %v59 = xor i64 %v3, 4
-
-; CHECK-NEXT:    {{.*}}|    3: <2, 51, 56, 12>       |    %v55 = xor i1 %v4, %c4;
-; CHECK-NEXT:    {{.*}}|    3: <2, 56, 59, 12>       |    %v56 = xor i8 %v0, %c2;
-; CHECK-NEXT:    {{.*}}|    3: <2, 56, 59, 12>       |    %v57 = xor i16 %v1, %c3;
-; CHECK-NEXT:    {{.*}}|    3: <2, 56, 62, 12>       |    %v58 = xor i32 %v2, %c1;
-; CHECK-NEXT:    {{.*}}|    3: <2, 56, 64, 12>       |    %v59 = xor i64 %v3, %c0;
-
-  ret void
-}
-
-
-
-; Tests integer vector binary operations.
-define internal void @IntVecOps(<16 x i8> %p0, <8 x i16> %p1, <4 x i32> %p2,
-                                 <4 x i1> %p3, <8 x i1> %p4, <16 x i1> %p5) {
-
-; CHECK:               |                             |  %b0:
-
-  %v0 = add <16 x i8> %p0, %p0
-  %v1 = add <8 x i16> %p1, %p1
-  %v2 = add <4 x i32> %p2, %p2
-
-; CHECK-NEXT:    {{.*}}|    3: <2, 6, 6, 0>          |    %v0 = add <16 x i8> %p0, %p0;
-; CHECK-NEXT:    {{.*}}|    3: <2, 6, 6, 0>          |    %v1 = add <8 x i16> %p1, %p1;
-; CHECK-NEXT:    {{.*}}|    3: <2, 6, 6, 0>          |    %v2 = add <4 x i32> %p2, %p2;
-
-  %v3 = sub <16 x i8> %p0, %p0
-  %v4 = sub <8 x i16> %p1, %p1
-  %v5 = sub <4 x i32> %p2, %p2
-
-; CHECK-NEXT:    {{.*}}|    3: <2, 9, 9, 1>          |    %v3 = sub <16 x i8> %p0, %p0;
-; CHECK-NEXT:    {{.*}}|    3: <2, 9, 9, 1>          |    %v4 = sub <8 x i16> %p1, %p1;
-; CHECK-NEXT:    {{.*}}|    3: <2, 9, 9, 1>          |    %v5 = sub <4 x i32> %p2, %p2;
-
-  %v6 = mul <16 x i8> %p0, %p0
-  %v7 = mul <8 x i16> %p1, %p1
-  %v8 = mul <4 x i32> %p2, %p2
-
-; CHECK-NEXT:    {{.*}}|    3: <2, 12, 12, 2>        |    %v6 = mul <16 x i8> %p0, %p0;
-; CHECK-NEXT:    {{.*}}|    3: <2, 12, 12, 2>        |    %v7 = mul <8 x i16> %p1, %p1;
-; CHECK-NEXT:    {{.*}}|    3: <2, 12, 12, 2>        |    %v8 = mul <4 x i32> %p2, %p2;
-
-  %v9 = sdiv <16 x i8> %p0, %p0
-  %v10 = sdiv <8 x i16> %p1, %p1
-  %v11 = sdiv <4 x i32> %p2, %p2
-
-; CHECK-NEXT:    {{.*}}|    3: <2, 15, 15, 4>        |    %v9 = sdiv <16 x i8> %p0, %p0;
-; CHECK-NEXT:    {{.*}}|    3: <2, 15, 15, 4>        |    %v10 = sdiv <8 x i16> %p1, %p1;
-; CHECK-NEXT:    {{.*}}|    3: <2, 15, 15, 4>        |    %v11 = sdiv <4 x i32> %p2, %p2;
-
-  %v12 = udiv <16 x i8> %p0, %p0
-  %v13 = udiv <8 x i16> %p1, %p1
-  %v14 = udiv <4 x i32> %p2, %p2
-
-; CHECK-NEXT:    {{.*}}|    3: <2, 18, 18, 3>        |    %v12 = udiv <16 x i8> %p0, %p0;
-; CHECK-NEXT:    {{.*}}|    3: <2, 18, 18, 3>        |    %v13 = udiv <8 x i16> %p1, %p1;
-; CHECK-NEXT:    {{.*}}|    3: <2, 18, 18, 3>        |    %v14 = udiv <4 x i32> %p2, %p2;
-
-  %v15 = srem <16 x i8> %p0, %p0
-  %v16 = srem <8 x i16> %p1, %p1
-  %v17 = srem <4 x i32> %p2, %p2
-
-; CHECK-NEXT:    {{.*}}|    3: <2, 21, 21, 6>        |    %v15 = srem <16 x i8> %p0, %p0;
-; CHECK-NEXT:    {{.*}}|    3: <2, 21, 21, 6>        |    %v16 = srem <8 x i16> %p1, %p1;
-; CHECK-NEXT:    {{.*}}|    3: <2, 21, 21, 6>        |    %v17 = srem <4 x i32> %p2, %p2;
-
-  %v18 = urem <16 x i8> %p0, %p0
-  %v19 = urem <8 x i16> %p1, %p1
-  %v20 = urem <4 x i32> %p2, %p2
-
-; CHECK-NEXT:    {{.*}}|    3: <2, 24, 24, 5>        |    %v18 = urem <16 x i8> %p0, %p0;
-; CHECK-NEXT:    {{.*}}|    3: <2, 24, 24, 5>        |    %v19 = urem <8 x i16> %p1, %p1;
-; CHECK-NEXT:    {{.*}}|    3: <2, 24, 24, 5>        |    %v20 = urem <4 x i32> %p2, %p2;
-
-  %v21 = shl <16 x i8> %p0, %p0
-  %v22 = shl <8 x i16> %p1, %p1
-  %v23 = shl <4 x i32> %p2, %p2
-
-; CHECK-NEXT:    {{.*}}|    3: <2, 27, 27, 7>        |    %v21 = shl <16 x i8> %p0, %p0;
-; CHECK-NEXT:    {{.*}}|    3: <2, 27, 27, 7>        |    %v22 = shl <8 x i16> %p1, %p1;
-; CHECK-NEXT:    {{.*}}|    3: <2, 27, 27, 7>        |    %v23 = shl <4 x i32> %p2, %p2;
-
-  %v24 = lshr <16 x i8> %p0, %p0
-  %v25 = lshr <8 x i16> %p1, %p1
-  %v26 = lshr <4 x i32> %p2, %p2
-
-; CHECK-NEXT:    {{.*}}|    3: <2, 30, 30, 8>        |    %v24 = lshr <16 x i8> %p0, %p0;
-; CHECK-NEXT:    {{.*}}|    3: <2, 30, 30, 8>        |    %v25 = lshr <8 x i16> %p1, %p1;
-; CHECK-NEXT:    {{.*}}|    3: <2, 30, 30, 8>        |    %v26 = lshr <4 x i32> %p2, %p2;
-
-  %v27 = ashr <16 x i8> %p0, %p0
-  %v28 = ashr <8 x i16> %p1, %p1
-  %v29 = ashr <4 x i32> %p2, %p2
-
-; CHECK-NEXT:    {{.*}}|    3: <2, 33, 33, 9>        |    %v27 = ashr <16 x i8> %p0, %p0;
-; CHECK-NEXT:    {{.*}}|    3: <2, 33, 33, 9>        |    %v28 = ashr <8 x i16> %p1, %p1;
-; CHECK-NEXT:    {{.*}}|    3: <2, 33, 33, 9>        |    %v29 = ashr <4 x i32> %p2, %p2;
-
-  %v30 = and <16 x i8> %p0, %p0
-  %v31 = and <8 x i16> %p1, %p1
-  %v32 = and <4 x i32> %p2, %p2
-  %v34 = and <4 x i1> %p3, %p3
-  %v35 = and <8 x i1> %p4, %p4
-  %v36 = and <16 x i1> %p5, %p5
-
-; CHECK-NEXT:    {{.*}}|    3: <2, 36, 36, 10>       |    %v30 = and <16 x i8> %p0, %p0;
-; CHECK-NEXT:    {{.*}}|    3: <2, 36, 36, 10>       |    %v31 = and <8 x i16> %p1, %p1;
-; CHECK-NEXT:    {{.*}}|    3: <2, 36, 36, 10>       |    %v32 = and <4 x i32> %p2, %p2;
-; CHECK-NEXT:    {{.*}}|    3: <2, 36, 36, 10>       |    %v33 = and <4 x i1> %p3, %p3;
-; CHECK-NEXT:    {{.*}}|    3: <2, 36, 36, 10>       |    %v34 = and <8 x i1> %p4, %p4;
-; CHECK-NEXT:    {{.*}}|    3: <2, 36, 36, 10>       |    %v35 = and <16 x i1> %p5, %p5;
-
-  %v37 = or <16 x i8> %p0, %p0
-  %v38 = or <8 x i16> %p1, %p1
-  %v39 = or <4 x i32> %p2, %p2
-  %v41 = or <4 x i1> %p3, %p3
-  %v42 = or <8 x i1> %p4, %p4
-  %v43 = or <16 x i1> %p5, %p5
-
-; CHECK-NEXT:    {{.*}}|    3: <2, 42, 42, 11>       |    %v36 = or <16 x i8> %p0, %p0;
-; CHECK-NEXT:    {{.*}}|    3: <2, 42, 42, 11>       |    %v37 = or <8 x i16> %p1, %p1;
-; CHECK-NEXT:    {{.*}}|    3: <2, 42, 42, 11>       |    %v38 = or <4 x i32> %p2, %p2;
-; CHECK-NEXT:    {{.*}}|    3: <2, 42, 42, 11>       |    %v39 = or <4 x i1> %p3, %p3;
-; CHECK-NEXT:    {{.*}}|    3: <2, 42, 42, 11>       |    %v40 = or <8 x i1> %p4, %p4;
-; CHECK-NEXT:    {{.*}}|    3: <2, 42, 42, 11>       |    %v41 = or <16 x i1> %p5, %p5;
-
-  %v44 = xor <16 x i8> %p0, %p0
-  %v45 = xor <8 x i16> %p1, %p1
-  %v46 = xor <4 x i32> %p2, %p2
-  %v48 = xor <4 x i1> %p3, %p3
-  %v49 = xor <8 x i1> %p4, %p4
-  %v50 = xor <16 x i1> %p5, %p5
-  ret void
-
-; CHECK-NEXT:    {{.*}}|    3: <2, 48, 48, 12>       |    %v42 = xor <16 x i8> %p0, %p0;
-; CHECK-NEXT:    {{.*}}|    3: <2, 48, 48, 12>       |    %v43 = xor <8 x i16> %p1, %p1;
-; CHECK-NEXT:    {{.*}}|    3: <2, 48, 48, 12>       |    %v44 = xor <4 x i32> %p2, %p2;
-; CHECK-NEXT:    {{.*}}|    3: <2, 48, 48, 12>       |    %v45 = xor <4 x i1> %p3, %p3;
-; CHECK-NEXT:    {{.*}}|    3: <2, 48, 48, 12>       |    %v46 = xor <8 x i1> %p4, %p4;
-; CHECK-NEXT:    {{.*}}|    3: <2, 48, 48, 12>       |    %v47 = xor <16 x i1> %p5, %p5;
-; CHECK-NEXT:    {{.*}}|    3: <10>                  |    ret void;
-
-}
-
-
-
-; Test floating point binary operations.
-define internal void @FloatOps(float %p0, double %p1) {
-
-; CHECK:               |                             |  %b0:
-
-  %v0 = fadd float %p0, 1.0
-  %v1 = fadd double %p1, 2.0
-
-; CHECK-NEXT:    {{.*}}|    3: <2, 4, 2, 0>          |    %v0 = fadd float %p0, %c0;
-; CHECK-NEXT:    {{.*}}|    3: <2, 4, 2, 0>          |    %v1 = fadd double %p1, %c1;
-
-  %v2 = fsub float %p0, 1.0
-  %v3 = fsub double %p1, 2.0
-
-; CHECK-NEXT:    {{.*}}|    3: <2, 6, 4, 1>          |    %v2 = fsub float %p0, %c0;
-; CHECK-NEXT:    {{.*}}|    3: <2, 6, 4, 1>          |    %v3 = fsub double %p1, %c1;
-
-  %v4 = fmul float %p0, 1.0
-  %v5 = fmul double %p1, 2.0
-
-; CHECK-NEXT:    {{.*}}|    3: <2, 8, 6, 2>          |    %v4 = fmul float %p0, %c0;
-; CHECK-NEXT:    {{.*}}|    3: <2, 8, 6, 2>          |    %v5 = fmul double %p1, %c1;
-
-  %v6 = fdiv float %p0, 1.0
-  %v7 = fdiv double %p1, 2.0
-
-; CHECK-NEXT:    {{.*}}|    3: <2, 10, 8, 4>         |    %v6 = fdiv float %p0, %c0;
-; CHECK-NEXT:    {{.*}}|    3: <2, 10, 8, 4>         |    %v7 = fdiv double %p1, %c1;
-
-  %v8 = frem float %p0, 1.0
-  %v9 = frem double %p1, 2.0
-  ret void
-
-; CHECK-NEXT:    {{.*}}|    3: <2, 12, 10, 6>        |    %v8 = frem float %p0, %c0;
-; CHECK-NEXT:    {{.*}}|    3: <2, 12, 10, 6>        |    %v9 = frem double %p1, %c1;
-; CHECK-NEXT:    {{.*}}|    3: <10>                  |    ret void;
-
-}
-
-
-
-; Tests floating point vector binary operations.
-define internal void @VecFloatOps(<4 x float> %p0) {
-
-; CHECK:               |                             |  %b0:
-
-  %v0 = fadd <4 x float> %p0, %p0
-  %v2 = fsub <4 x float> %p0, %p0
-  %v4 = fmul <4 x float> %p0, %p0
-  %v6 = fdiv <4 x float> %p0, %p0
-  %v8 = frem <4 x float> %p0, %p0
-  ret void
-
-; CHECK-NEXT:    {{.*}}|    3: <2, 1, 1, 0>          |    %v0 = fadd <4 x float> %p0, %p0;
-; CHECK-NEXT:    {{.*}}|    3: <2, 2, 2, 1>          |    %v1 = fsub <4 x float> %p0, %p0;
-; CHECK-NEXT:    {{.*}}|    3: <2, 3, 3, 2>          |    %v2 = fmul <4 x float> %p0, %p0;
-; CHECK-NEXT:    {{.*}}|    3: <2, 4, 4, 4>          |    %v3 = fdiv <4 x float> %p0, %p0;
-; CHECK-NEXT:    {{.*}}|    3: <2, 5, 5, 6>          |    %v4 = frem <4 x float> %p0, %p0;
-; CHECK-NEXT:    {{.*}}|    3: <10>                  |    ret void;
-
-}
diff --git a/test/NaCl/Bitcode/pnacl-bcdis/br.ll b/test/NaCl/Bitcode/pnacl-bcdis/br.ll
deleted file mode 100644
index ceffb8b68e5..00000000000
--- a/test/NaCl/Bitcode/pnacl-bcdis/br.ll
+++ /dev/null
@@ -1,40 +0,0 @@
-; Test branch instructions.
-
-; RUN: llvm-as < %s | pnacl-freeze | pnacl-bccompress --remove-abbreviations \
-; RUN:              | pnacl-bcdis | FileCheck %s
-
-
-define void @Ex1() {
-
-; CHECK:               |                             |  %b0:
-
-  br label %b3
-
-; CHECK-NEXT:    {{.*}}|    3: <11, 3>               |    br label %b3;
-
-b1:
-  br i1 1, label %b2, label %b4
-
-; CHECK-NEXT:          |                             |  %b1:
-; CHECK-NEXT:    {{.*}}|    3: <11, 2, 4, 1>         |    br i1 %c0, label %b2, label %b4;
-
-b2:
-  br label %b1
-
-; CHECK-NEXT:          |                             |  %b2:
-; CHECK-NEXT:    {{.*}}|    3: <11, 1>               |    br label %b1;
-
-b3:
-  ret void
-
-; CHECK-NEXT:          |                             |  %b3:
-; CHECK-NEXT:    {{.*}}|    3: <10>                  |    ret void;
-
-b4:
-  br label %b3
-}
-
-; CHECK-NEXT:          |                             |  %b4:
-; CHECK-NEXT:    {{.*}}|    3: <11, 3>               |    br label %b3;
-; CHECK-NEXT:    {{.*}}|  0: <65534>                 |  }
-
diff --git a/test/NaCl/Bitcode/pnacl-bcdis/call.ll b/test/NaCl/Bitcode/pnacl-bcdis/call.ll
deleted file mode 100644
index 55f76677832..00000000000
--- a/test/NaCl/Bitcode/pnacl-bcdis/call.ll
+++ /dev/null
@@ -1,90 +0,0 @@
-; Test call instructions
-
-; RUN: llvm-as < %s | pnacl-freeze | pnacl-bccompress --remove-abbreviations \
-; RUN:              | pnacl-bcdis | FileCheck %s
-
-; CHECK:         {{.*}}|    3: <7, 32>               |    @t0 = i32;
-; CHECK-NEXT:    {{.*}}|    3: <2>                   |    @t1 = void;
-; CHECK-NEXT:    {{.*}}|    3: <3>                   |    @t2 = float;
-
-
-; Test simple calls
-define void @foo(i32 %p0) {
-
-; CHECK:               |                             |  %b0:
-
-  %v0 = call i32 @bar(i32 %p0, i32 1)
-
-; CHECK-NEXT:    {{.*}}|    3: <34, 0, 6, 2, 1>      |    %v0 = call i32 
-; CHECK-NEXT:          |                             |        @f1(i32 %p0, i32 %c0);
-
-  %v1 = call float @bam(i32 %p0)
-
-; CHECK-NEXT:    {{.*}}|    3: <34, 0, 6, 3>         |    %v1 = call float @f2(i32 %p0);
-
-  call void @huh()
-  ret void
-
-; CHECK-NEXT:    {{.*}}|    3: <34, 0, 6>            |    call void @f3();
-; CHECK-NEXT:    {{.*}}|    3: <10>                  |    ret void;
-
-}
-
-define i32 @bar(i32 %p0, i32 %p1) {
-
-; CHECK:               |                             |  %b0:
-
-  %v0 = add i32 %p0, %p1
-  ret i32 %v0
-}
-
-define float @bam(i32 %p0) {
-
-; CHECK:               |                             |  %b0:
-
-  %v0 = sitofp i32 %p0 to float
-  ret float %v0
-}
-
-define void @huh() {
-
-; CHECK:               |                             |  %b0:
-
-  ret void
-}
-
-
-; Test indirect calls
-define void @IndirectTests(i32 %p0, i32 %p1) {
-
-; CHECK:               |                             |  %b0:
-
-  ; Simulates indirect call to @foo
-  %v0 = inttoptr i32 %p0 to void (i32)*
-  call void %v0(i32 %p1)
-
-; CHECK-NEXT:    {{.*}}|    3: <44, 0, 3, 1, 2>      |    call void %p0(i32 %p1);
-
-  ; Simulates indirect call to @bar
-  %v1 = inttoptr i32 %p0 to i32 (i32, i32)*
-  %v2 = call i32 %v1(i32 %p1, i32 1)
-
-; CHECK-NEXT:    {{.*}}|    3: <44, 0, 3, 0, 2, 1>   |    %v0 = call i32 
-; CHECK-NEXT:          |                             |        %p0(i32 %p1, i32 %c0);
-
-  ; Simulates indirect call to @bam
-  %v3 = inttoptr i32 %p0 to float (i32)*
-  %v4 = call float %v3(i32 %p1)
-
-; CHECK-NEXT:    {{.*}}|    3: <44, 0, 4, 2, 3>      |    %v1 = call float %p0(i32 %p1);
-
-  ; Simulates indirect call to @huh
-  %v5 = inttoptr i32 %p0 to void ()*
-  call void %v5()
-  ret void
-
-; CHECK-NEXT:    {{.*}}|    3: <44, 0, 5, 1>         |    call void %p0();
-; CHECK-NEXT:    {{.*}}|    3: <10>                  |    ret void;
-
-}
-
diff --git a/test/NaCl/Bitcode/pnacl-bcdis/casts.ll b/test/NaCl/Bitcode/pnacl-bcdis/casts.ll
deleted file mode 100644
index d62c002f22f..00000000000
--- a/test/NaCl/Bitcode/pnacl-bcdis/casts.ll
+++ /dev/null
@@ -1,422 +0,0 @@
-; Test dumping cast operations.
-
-; RUN: llvm-as < %s | pnacl-freeze | pnacl-bccompress --remove-abbreviations \
-; RUN:              | pnacl-bcdis | FileCheck %s
-
-
-; Test integer to integer casts.
-define internal void @Int2IntCasts() {
-
-; CHECK:               |                             |  %b0:
-
-  ; Test truncation to i1.
-  %v0 = trunc i8 1 to i1
-  %v1 = trunc i16 2 to i1
-  %v2 = trunc i32 3 to i1
-  %v3 = trunc i64 4 to i1
-
-; CHECK-NEXT:    {{.*}}|    3: <3, 3, 6, 0>          |    %v0 = trunc i8 %c2 to i1;
-; CHECK-NEXT:    {{.*}}|    3: <3, 3, 6, 0>          |    %v1 = trunc i16 %c3 to i1;
-; CHECK-NEXT:    {{.*}}|    3: <3, 7, 6, 0>          |    %v2 = trunc i32 %c0 to i1;
-; CHECK-NEXT:    {{.*}}|    3: <3, 7, 6, 0>          |    %v3 = trunc i64 %c1 to i1;
-
-  ; Verify i1 generated.
-  %v4 = and i1 %v0, %v1
-  %v5 = and i1 %v2, %v3
-
-; CHECK-NEXT:    {{.*}}|    3: <2, 4, 3, 10>         |    %v4 = and i1 %v0, %v1;
-; CHECK-NEXT:    {{.*}}|    3: <2, 3, 2, 10>         |    %v5 = and i1 %v2, %v3;
-
-  ; Test truncation to i8.
-  %v6 = trunc i16 2 to i8
-  %v7 = trunc i32 3 to i8
-  %v8 = trunc i64 4 to i8
-
-; CHECK-NEXT:    {{.*}}|    3: <3, 8, 4, 0>          |    %v6 = trunc i16 %c3 to i8;
-; CHECK-NEXT:    {{.*}}|    3: <3, 12, 4, 0>         |    %v7 = trunc i32 %c0 to i8;
-; CHECK-NEXT:    {{.*}}|    3: <3, 12, 4, 0>         |    %v8 = trunc i64 %c1 to i8;
-
-  ; Verify i8 generated.
-  %v9 = add i8 %v6, %v7
-  %v10 = add i8 %v8, %v8
-
-; CHECK-NEXT:    {{.*}}|    3: <2, 3, 2, 0>          |    %v9 = add i8 %v6, %v7;
-; CHECK-NEXT:    {{.*}}|    3: <2, 2, 2, 0>          |    %v10 = add i8 %v8, %v8;
-
-  ; Test trunction to i16.
-  %v11 = trunc i32 3 to i16
-  %v12 = trunc i64 4 to i16
-
-; CHECK-NEXT:    {{.*}}|    3: <3, 16, 5, 0>         |    %v11 = trunc i32 %c0 to i16;
-; CHECK-NEXT:    {{.*}}|    3: <3, 16, 5, 0>         |    %v12 = trunc i64 %c1 to i16;
-
-  ; Verify i16 generated.
-  %v13 = add i16 %v11, %v12
-
-; CHECK-NEXT:    {{.*}}|    3: <2, 2, 1, 0>          |    %v13 = add i16 %v11, %v12;
-  
-  ; Test truncation to i32.
-  %v14 = trunc i64 4 to i32
-
-; CHECK-NEXT:    {{.*}}|    3: <3, 18, 2, 0>         |    %v14 = trunc i64 %c1 to i32;
-
-  ; Verify i32 generated.
-  %v15 = add i32 %v14, %v14
-
-; CHECK-NEXT:    {{.*}}|    3: <2, 1, 1, 0>          |    %v15 = add i32 %v14, %v14;
-
-  ; Test zero extend to i8.
-  %v16 = zext i1 0 to i8
-
-; CHECK-NEXT:    {{.*}}|    3: <3, 17, 4, 1>         |    %v16 = zext i1 %c4 to i8;
-
-  ; Verify i8 generated.
-  %v17 = add i8 %v16, %v16
-
-; CHECK-NEXT:    {{.*}}|    3: <2, 1, 1, 0>          |    %v17 = add i8 %v16, %v16;
-
-  ; Test zero extend to i16.
-  %v18 = zext i1 0 to i16  
-  %v19 = zext i8 1 to i16
-
-; CHECK-NEXT:    {{.*}}|    3: <3, 19, 5, 1>         |    %v18 = zext i1 %c4 to i16;
-; CHECK-NEXT:    {{.*}}|    3: <3, 22, 5, 1>         |    %v19 = zext i8 %c2 to i16;
-
-  ; Verify i16 generated.
-  %v20 = add i16 %v18, %v19
-
-; CHECK-NEXT:    {{.*}}|    3: <2, 2, 1, 0>          |    %v20 = add i16 %v18, %v19;
-
-  ; Test zero extend to i32.
-  %v21 = zext i1 0 to i32
-  %v22 = zext i8 1 to i32
-  %v23 = zext i16 2 to i32
-
-; CHECK-NEXT:    {{.*}}|    3: <3, 22, 2, 1>         |    %v21 = zext i1 %c4 to i32;
-; CHECK-NEXT:    {{.*}}|    3: <3, 25, 2, 1>         |    %v22 = zext i8 %c2 to i32;
-; CHECK-NEXT:    {{.*}}|    3: <3, 25, 2, 1>         |    %v23 = zext i16 %c3 to i32;
-
-  ; Verify i32 generated.
-  %v24 = add i32 %v21, %v22
-  %v25 = add i32 %v23, %v24
-
-; CHECK-NEXT:    {{.*}}|    3: <2, 3, 2, 0>          |    %v24 = add i32 %v21, %v22;
-; CHECK-NEXT:    {{.*}}|    3: <2, 2, 1, 0>          |    %v25 = add i32 %v23, %v24;
-
-  ; Test zero extend to i64.
-  %v26 = zext i1 0 to i64
-  %v27 = zext i8 1 to i64
-  %v28 = zext i16 2 to i64
-  %v29 = zext i32 3 to i64
-
-; CHECK-NEXT:    {{.*}}|    3: <3, 27, 3, 1>         |    %v26 = zext i1 %c4 to i64;
-; CHECK-NEXT:    {{.*}}|    3: <3, 30, 3, 1>         |    %v27 = zext i8 %c2 to i64;
-; CHECK-NEXT:    {{.*}}|    3: <3, 30, 3, 1>         |    %v28 = zext i16 %c3 to i64;
-; CHECK-NEXT:    {{.*}}|    3: <3, 34, 3, 1>         |    %v29 = zext i32 %c0 to i64;
-
-  ; Verify i64 generated.
-  %v30 = add i64 %v26, %v27
-  %v31 = add i64 %v28, %v29
-
-; CHECK-NEXT:    {{.*}}|    3: <2, 4, 3, 0>          |    %v30 = add i64 %v26, %v27;
-; CHECK-NEXT:    {{.*}}|    3: <2, 3, 2, 0>          |    %v31 = add i64 %v28, %v29;
-
-  ; Test sign extend to i8.
-  %v32 = sext i1 0 to i8
-
-; CHECK-NEXT:    {{.*}}|    3: <3, 33, 4, 2>         |    %v32 = sext i1 %c4 to i8;
-
-  ; Verify i8 generated.
-  %v33 = add i8 %v32, %v32
-
-; CHECK-NEXT:    {{.*}}|    3: <2, 1, 1, 0>          |    %v33 = add i8 %v32, %v32;
-
-  ; Test sign extend to i16
-  %v34 = sext i1 0 to i16
-  %v35 = sext i8 1 to i16
-
-; CHECK-NEXT:    {{.*}}|    3: <3, 35, 5, 2>         |    %v34 = sext i1 %c4 to i16;
-; CHECK-NEXT:    {{.*}}|    3: <3, 38, 5, 2>         |    %v35 = sext i8 %c2 to i16;
-
-  ; Verify i16 generated.
-  %v36 = add i16 %v34, %v35
-
-; CHECK-NEXT:    {{.*}}|    3: <2, 2, 1, 0>          |    %v36 = add i16 %v34, %v35;
-
-  ; Test sign extend to i32.
-  %v37 = sext i1 0 to i32
-  %v38 = sext i8 1 to i32
-  %v39 = sext i16 2 to i32
-
-; CHECK-NEXT:    {{.*}}|    3: <3, 38, 2, 2>         |    %v37 = sext i1 %c4 to i32;
-; CHECK-NEXT:    {{.*}}|    3: <3, 41, 2, 2>         |    %v38 = sext i8 %c2 to i32;
-; CHECK-NEXT:    {{.*}}|    3: <3, 41, 2, 2>         |    %v39 = sext i16 %c3 to i32;
-
-  ; Verify i32 generated.
-  %v40 = add i32 %v37, %v38
-  %v41 = add i32 %v39, %v40
-
-; CHECK-NEXT:    {{.*}}|    3: <2, 3, 2, 0>          |    %v40 = add i32 %v37, %v38;
-; CHECK-NEXT:    {{.*}}|    3: <2, 2, 1, 0>          |    %v41 = add i32 %v39, %v40;
-
-  ; Test sign extend to i64.
-  %v42 = sext i1 0 to i64
-  %v43 = sext i8 1 to i64
-  %v44 = sext i16 2 to i64
-  %v45 = sext i32 3 to i64
-
-; CHECK-NEXT:    {{.*}}|    3: <3, 43, 3, 2>         |    %v42 = sext i1 %c4 to i64;
-; CHECK-NEXT:    {{.*}}|    3: <3, 46, 3, 2>         |    %v43 = sext i8 %c2 to i64;
-; CHECK-NEXT:    {{.*}}|    3: <3, 46, 3, 2>         |    %v44 = sext i16 %c3 to i64;
-; CHECK-NEXT:    {{.*}}|    3: <3, 50, 3, 2>         |    %v45 = sext i32 %c0 to i64;
-
-  ; Verify i64 generated.
-  %v46 = add i64 %v42, %v43
-  %v47 = add i64 %v44, %v45
-  ret void
-
-; CHECK-NEXT:    {{.*}}|    3: <2, 4, 3, 0>          |    %v46 = add i64 %v42, %v43;
-; CHECK-NEXT:    {{.*}}|    3: <2, 3, 2, 0>          |    %v47 = add i64 %v44, %v45;
-; CHECK-NEXT:    {{.*}}|    3: <10>                  |    ret void;
-
-}
-
-
-
-; Test float to float casts.
-define internal void @Float2FloatCasts(float %p0, double %p1) {
-
-; CHECK:               |                             |  %b0:
-
-  ; Test and verify truncation to float.
-  %v0 = fptrunc double %p1 to float
-  %v1 = fadd float %v0, %v0
-
-; CHECK-NEXT:    {{.*}}|    3: <3, 1, 0, 7>          |    %v0 = fptrunc double %p1 to float;
-; CHECK-NEXT:    {{.*}}|    3: <2, 1, 1, 0>          |    %v1 = fadd float %v0, %v0;
-
-  ; Test and verify extending to double.
-  %v2 = fpext float %p0 to double
-  %v3 = fadd double %v2, %v2
-  ret void
-
-; CHECK-NEXT:    {{.*}}|    3: <3, 4, 1, 8>          |    %v2 = fpext float %p0 to double;
-; CHECK-NEXT:    {{.*}}|    3: <2, 1, 1, 0>          |    %v3 = fadd double %v2, %v2;
-; CHECK-NEXT:    {{.*}}|    3: <10>                  |    ret void;
-
-}
-
-
-
-; Test float to int casts.
-define internal void @Float2IntCasts(float %p0, double %p1) {
-
-; CHECK:               |                             |  %b0:
-
-  ; Test and verify fptoui for i1.
-  %v0 = fptoui float %p0 to i1
-  %v1 = fptoui double %p1 to i1
-  %v2 = and i1 %v0, %v1
-
-; CHECK-NEXT:    {{.*}}|    3: <3, 2, 6, 3>          |    %v0 = fptoui float %p0 to i1;
-; CHECK-NEXT:    {{.*}}|    3: <3, 2, 6, 3>          |    %v1 = fptoui double %p1 to i1;
-; CHECK-NEXT:    {{.*}}|    3: <2, 2, 1, 10>         |    %v2 = and i1 %v0, %v1;
-
-  ; Test and verify fptoui for i8.
-  %v3 = fptoui float %p0 to i8
-  %v4 = fptoui double %p1 to i8
-  %v5 = add i8 %v3, %v4
-
-; CHECK-NEXT:    {{.*}}|    3: <3, 5, 4, 3>          |    %v3 = fptoui float %p0 to i8;
-; CHECK-NEXT:    {{.*}}|    3: <3, 5, 4, 3>          |    %v4 = fptoui double %p1 to i8;
-; CHECK-NEXT:    {{.*}}|    3: <2, 2, 1, 0>          |    %v5 = add i8 %v3, %v4;
-
-  ; Test and verify fptoui for i16.
-  %v6 = fptoui float %p0 to i16
-  %v7 = fptoui double %p1 to i16
-  %v8 = add i16 %v6, %v7
-
-; CHECK-NEXT:    {{.*}}|    3: <3, 8, 5, 3>          |    %v6 = fptoui float %p0 to i16;
-; CHECK-NEXT:    {{.*}}|    3: <3, 8, 5, 3>          |    %v7 = fptoui double %p1 to i16;
-; CHECK-NEXT:    {{.*}}|    3: <2, 2, 1, 0>          |    %v8 = add i16 %v6, %v7;
-
-  ; Test and verify fptoui for i32.
-  %v9 = fptoui float %p0 to i32
-  %v10 = fptoui double %p1 to i32
-  %v11 = and i32 %v9, %v10
-
-; CHECK-NEXT:    {{.*}}|    3: <3, 11, 2, 3>         |    %v9 = fptoui float %p0 to i32;
-; CHECK-NEXT:    {{.*}}|    3: <3, 11, 2, 3>         |    %v10 = fptoui double %p1 to i32;
-; CHECK-NEXT:    {{.*}}|    3: <2, 2, 1, 10>         |    %v11 = and i32 %v9, %v10;
-
-  ; Test and verify fptoui for i64
-  %v12 = fptoui float %p0 to i64
-  %v13 = fptoui double %p1 to i64
-  %v14 = and i64 %v12, %v13
-
-; CHECK-NEXT:    {{.*}}|    3: <3, 14, 3, 3>         |    %v12 = fptoui float %p0 to i64;
-; CHECK-NEXT:    {{.*}}|    3: <3, 14, 3, 3>         |    %v13 = fptoui double %p1 to i64;
-; CHECK-NEXT:    {{.*}}|    3: <2, 2, 1, 10>         |    %v14 = and i64 %v12, %v13;
-
-  ; Test and verify fptosi for i1.
-  %v15 = fptosi float %p0 to i1
-  %v16 = fptosi double %p1 to i1
-  %v17 = and i1 %v15, %v16
-
-; CHECK-NEXT:    {{.*}}|    3: <3, 17, 6, 4>         |    %v15 = fptosi float %p0 to i1;
-; CHECK-NEXT:    {{.*}}|    3: <3, 17, 6, 4>         |    %v16 = fptosi double %p1 to i1;
-; CHECK-NEXT:    {{.*}}|    3: <2, 2, 1, 10>         |    %v17 = and i1 %v15, %v16;
-
-  ; Test and verify fptosi for i8.
-  %v18 = fptosi float %p0 to i8
-  %v19 = fptosi double %p1 to i8
-  %v20 = add i8 %v18, %v19
-
-; CHECK-NEXT:    {{.*}}|    3: <3, 20, 4, 4>         |    %v18 = fptosi float %p0 to i8;
-; CHECK-NEXT:    {{.*}}|    3: <3, 20, 4, 4>         |    %v19 = fptosi double %p1 to i8;
-; CHECK-NEXT:    {{.*}}|    3: <2, 2, 1, 0>          |    %v20 = add i8 %v18, %v19;
-
-  ; Test and verify fptosi for i16.
-  %v21 = fptosi float %p0 to i16
-  %v22 = fptosi double %p1 to i16
-  %v23 = add i16 %v21, %v22
-
-; CHECK-NEXT:    {{.*}}|    3: <3, 23, 5, 4>         |    %v21 = fptosi float %p0 to i16;
-; CHECK-NEXT:    {{.*}}|    3: <3, 23, 5, 4>         |    %v22 = fptosi double %p1 to i16;
-; CHECK-NEXT:    {{.*}}|    3: <2, 2, 1, 0>          |    %v23 = add i16 %v21, %v22;
-
-  ; Test and verify fptosi for i32.
-  %v24 = fptosi float %p0 to i32
-  %v25 = fptosi double %p1 to i32
-  %v26 = and i32 %v24, %v25
-
-; CHECK-NEXT:    {{.*}}|    3: <3, 26, 2, 4>         |    %v24 = fptosi float %p0 to i32;
-; CHECK-NEXT:    {{.*}}|    3: <3, 26, 2, 4>         |    %v25 = fptosi double %p1 to i32;
-; CHECK-NEXT:    {{.*}}|    3: <2, 2, 1, 10>         |    %v26 = and i32 %v24, %v25;
-
-  ; Test and verify fptosi for i64
-  %v27 = fptosi float %p0 to i64
-  %v28 = fptosi double %p1 to i64
-  %v29 = and i64 %v27, %v28
-
-; CHECK-NEXT:    {{.*}}|    3: <3, 29, 3, 4>         |    %v27 = fptosi float %p0 to i64;
-; CHECK-NEXT:    {{.*}}|    3: <3, 29, 3, 4>         |    %v28 = fptosi double %p1 to i64;
-; CHECK-NEXT:    {{.*}}|    3: <2, 2, 1, 10>         |    %v29 = and i64 %v27, %v28;
-
-  ; Test and verify bitcast to i32.
-  %v30 = bitcast float %p0 to i32
-  %v31 = add i32 %v30, %v30
-
-; CHECK-NEXT:    {{.*}}|    3: <3, 32, 2, 11>        |    %v30 = bitcast float %p0 to i32;
-; CHECK-NEXT:    {{.*}}|    3: <2, 1, 1, 0>          |    %v31 = add i32 %v30, %v30;
-
-  ; Test and verify bitcast to i64.
-  %v32 = bitcast double %p1 to i64
-  %v33 = add i64 %v32, %v32
-  ret void
-
-; CHECK-NEXT:    {{.*}}|    3: <3, 33, 3, 11>        |    %v32 = bitcast double %p1 to i64;
-; CHECK-NEXT:    {{.*}}|    3: <2, 1, 1, 0>          |    %v33 = add i64 %v32, %v32;
-; CHECK-NEXT:    {{.*}}|    3: <10>                  |    ret void;
-
-}
-
-
-
-; Test int to float casts.
-define internal void @Int2FloatCasts() {
-
-; CHECK:               |                             |  %b0:
-
-  ; Test uitofp conversions to float.
-  %v0 = uitofp i1 0 to float
-  %v1 = uitofp i8 1 to float
-  %v2 = uitofp i16 2 to float
-  %v3 = uitofp i32 3 to float
-  %v4 = uitofp i64 4 to float
-
-; CHECK-NEXT:    {{.*}}|    3: <3, 1, 0, 5>          |    %v0 = uitofp i1 %c4 to float;
-; CHECK-NEXT:    {{.*}}|    3: <3, 4, 0, 5>          |    %v1 = uitofp i8 %c2 to float;
-; CHECK-NEXT:    {{.*}}|    3: <3, 4, 0, 5>          |    %v2 = uitofp i16 %c3 to float;
-; CHECK-NEXT:    {{.*}}|    3: <3, 8, 0, 5>          |    %v3 = uitofp i32 %c0 to float;
-; CHECK-NEXT:    {{.*}}|    3: <3, 8, 0, 5>          |    %v4 = uitofp i64 %c1 to float;
-
-  ; Verify floats generated.
-  %v5 = fadd float %v0, %v1
-  %v6 = fadd float %v2, %v3
-  %v7 = fadd float %v4, %v5
-
-; CHECK-NEXT:    {{.*}}|    3: <2, 5, 4, 0>          |    %v5 = fadd float %v0, %v1;
-; CHECK-NEXT:    {{.*}}|    3: <2, 4, 3, 0>          |    %v6 = fadd float %v2, %v3;
-; CHECK-NEXT:    {{.*}}|    3: <2, 3, 2, 0>          |    %v7 = fadd float %v4, %v5;
-
-  ; Test uitofp conversions to double
-  %v8 = uitofp i1 0 to double
-  %v9 = uitofp i8 1 to double
-  %v10 = uitofp i16 2 to double
-  %v11 = uitofp i32 3 to double
-  %v12 = uitofp i64 4 to double
-
-; CHECK-NEXT:    {{.*}}|    3: <3, 9, 1, 5>          |    %v8 = uitofp i1 %c4 to double;
-; CHECK-NEXT:    {{.*}}|    3: <3, 12, 1, 5>         |    %v9 = uitofp i8 %c2 to double;
-; CHECK-NEXT:    {{.*}}|    3: <3, 12, 1, 5>         |    %v10 = uitofp i16 %c3 to double;
-; CHECK-NEXT:    {{.*}}|    3: <3, 16, 1, 5>         |    %v11 = uitofp i32 %c0 to double;
-; CHECK-NEXT:    {{.*}}|    3: <3, 16, 1, 5>         |    %v12 = uitofp i64 %c1 to double;
-
-  ; Verify doubles generated.
-  %v13 = fadd double %v8, %v9
-  %v14 = fadd double %v10, %v11
-  %v15 = fadd double %v12, %v13
-
-; CHECK-NEXT:    {{.*}}|    3: <2, 5, 4, 0>          |    %v13 = fadd double %v8, %v9;
-; CHECK-NEXT:    {{.*}}|    3: <2, 4, 3, 0>          |    %v14 = fadd double %v10, %v11;
-; CHECK-NEXT:    {{.*}}|    3: <2, 3, 2, 0>          |    %v15 = fadd double %v12, %v13;
-
-  ; Test sitofp conversions to float.
-  %v16 = sitofp i1 0 to float
-  %v17 = sitofp i8 1 to float
-  %v18 = sitofp i16 2 to float
-  %v19 = sitofp i32 3 to float
-  %v20 = sitofp i64 4 to float
-
-; CHECK-NEXT:    {{.*}}|    3: <3, 17, 0, 6>         |    %v16 = sitofp i1 %c4 to float;
-; CHECK-NEXT:    {{.*}}|    3: <3, 20, 0, 6>         |    %v17 = sitofp i8 %c2 to float;
-; CHECK-NEXT:    {{.*}}|    3: <3, 20, 0, 6>         |    %v18 = sitofp i16 %c3 to float;
-; CHECK-NEXT:    {{.*}}|    3: <3, 24, 0, 6>         |    %v19 = sitofp i32 %c0 to float;
-; CHECK-NEXT:    {{.*}}|    3: <3, 24, 0, 6>         |    %v20 = sitofp i64 %c1 to float;
-
-  ; Verify floats generated.
-  %v21 = fadd float %v16, %v17
-  %v22 = fadd float %v18, %v19
-  %v23 = fadd float %v20, %v21
-
-; CHECK-NEXT:    {{.*}}|    3: <2, 5, 4, 0>          |    %v21 = fadd float %v16, %v17;
-; CHECK-NEXT:    {{.*}}|    3: <2, 4, 3, 0>          |    %v22 = fadd float %v18, %v19;
-; CHECK-NEXT:    {{.*}}|    3: <2, 3, 2, 0>          |    %v23 = fadd float %v20, %v21;
-
-  ; Test sitofp conversions to double
-  %v24 = sitofp i1 0 to double
-  %v25 = sitofp i8 1 to double
-  %v26 = sitofp i16 2 to double
-  %v27 = sitofp i32 3 to double
-  %v28 = sitofp i64 4 to double
-
-; CHECK-NEXT:    {{.*}}|    3: <3, 25, 1, 6>         |    %v24 = sitofp i1 %c4 to double;
-; CHECK-NEXT:    {{.*}}|    3: <3, 28, 1, 6>         |    %v25 = sitofp i8 %c2 to double;
-; CHECK-NEXT:    {{.*}}|    3: <3, 28, 1, 6>         |    %v26 = sitofp i16 %c3 to double;
-; CHECK-NEXT:    {{.*}}|    3: <3, 32, 1, 6>         |    %v27 = sitofp i32 %c0 to double;
-; CHECK-NEXT:    {{.*}}|    3: <3, 32, 1, 6>         |    %v28 = sitofp i64 %c1 to double;
-
-  ; Verify doubles generated.
-  %v29 = fadd double %v24, %v25
-  %v30 = fadd double %v26, %v27
-  %v31 = fadd double %v29, %v30
-  ret void
-
-; CHECK-NEXT:    {{.*}}|    3: <2, 5, 4, 0>          |    %v29 = fadd double %v24, %v25;
-; CHECK-NEXT:    {{.*}}|    3: <2, 4, 3, 0>          |    %v30 = fadd double %v26, %v27;
-; CHECK-NEXT:    {{.*}}|    3: <2, 2, 1, 0>          |    %v31 = fadd double %v29, %v30;
-; CHECK-NEXT:    {{.*}}|    3: <10>                  |    ret void;
-
-}
-
diff --git a/test/NaCl/Bitcode/pnacl-bcdis/constants.ll b/test/NaCl/Bitcode/pnacl-bcdis/constants.ll
deleted file mode 100644
index 6328bf46ae8..00000000000
--- a/test/NaCl/Bitcode/pnacl-bcdis/constants.ll
+++ /dev/null
@@ -1,201 +0,0 @@
-; Test handling of constants in function blocks.
-
-; RUN: llvm-as < %s | pnacl-freeze | pnacl-bccompress --remove-abbreviations \
-; RUN:              | pnacl-bcdis | FileCheck %s
-
-define void @TestIntegers() {
-
-; CHECK:         {{.*}}|    1: <65535, 11, 2>        |    constants {  // BlockID = 11
-; CHECK-NEXT:    {{.*}}|      3: <1, 2>              |      i8:
-; CHECK-NEXT:    {{.*}}|      3: <4, 0>              |        %c0 = i8 0;
-; CHECK-NEXT:    {{.*}}|      3: <4, 10>             |        %c1 = i8 5;
-; CHECK-NEXT:    {{.*}}|      3: <4, 11>             |        %c2 = i8 -5;
-; CHECK-NEXT:    {{.*}}|      3: <3>                 |        %c3 = i8 undef;
-; CHECK-NEXT:    {{.*}}|      3: <1, 4>              |      i16:
-; CHECK-NEXT:    {{.*}}|      3: <4, 0>              |        %c4 = i16 0;
-; CHECK-NEXT:    {{.*}}|      3: <4, 20>             |        %c5 = i16 10;
-; CHECK-NEXT:    {{.*}}|      3: <4, 21>             |        %c6 = i16 -10;
-; CHECK-NEXT:    {{.*}}|      3: <3>                 |        %c7 = i16 undef;
-; CHECK-NEXT:    {{.*}}|      3: <1, 5>              |      i32:
-; CHECK-NEXT:    {{.*}}|      3: <4, 0>              |        %c8 = i32 0;
-; CHECK-NEXT:    {{.*}}|      3: <4, 40>             |        %c9 = i32 20;
-; CHECK-NEXT:    {{.*}}|      3: <4, 41>             |        %c10 = i32 -20;
-; CHECK-NEXT:    {{.*}}|      3: <3>                 |        %c11 = i32 undef;
-; CHECK-NEXT:    {{.*}}|      3: <1, 6>              |      i64:
-; CHECK-NEXT:    {{.*}}|      3: <4, 0>              |        %c12 = i64 0;
-; CHECK-NEXT:    {{.*}}|      3: <4, 60>             |        %c13 = i64 30;
-; CHECK-NEXT:    {{.*}}|      3: <4, 61>             |        %c14 = i64 -30;
-; CHECK-NEXT:    {{.*}}|      3: <3>                 |        %c15 = i64 undef;
-; CHECK-NEXT:    {{.*}}|      3: <1, 7>              |      i1:
-; CHECK-NEXT:    {{.*}}|      3: <4, 0>              |        %c16 = i1 0;
-; CHECK-NEXT:    {{.*}}|      3: <4, 3>              |        %c17 = i1 1;
-; CHECK-NEXT:    {{.*}}|      3: <3>                 |        %c18 = i1 undef;
-; CHECK-NEXT:    {{.*}}|    0: <65534>               |      }
-; CHECK-NEXT:          |                             |  %b0:
-
-  ; Test various sized integers
-  %v0 = or i1 true, false
-  %v1 = add i8 0, 0
-  %v2 = add i8 5, 0
-  %v3 = add i8 -5, 0
-  %v4 = and i16 10, 0
-  %v5 = add i16 -10, 0
-  %v6 = add i32 20, 0
-  %v7 = add i32 -20, 0
-  %v8 = add i64 30, 0
-  %v9 = add i64 -30, 0
-
-; CHECK-NEXT:    {{.*}}|    3: <2, 2, 3, 11>         |    %v0 = or i1 %c17, %c16;
-; CHECK-NEXT:    {{.*}}|    3: <2, 20, 20, 0>        |    %v1 = add i8 %c0, %c0;
-; CHECK-NEXT:    {{.*}}|    3: <2, 20, 21, 0>        |    %v2 = add i8 %c1, %c0;
-; CHECK-NEXT:    {{.*}}|    3: <2, 20, 22, 0>        |    %v3 = add i8 %c2, %c0;
-; CHECK-NEXT:    {{.*}}|    3: <2, 18, 19, 10>       |    %v4 = and i16 %c5, %c4;
-; CHECK-NEXT:    {{.*}}|    3: <2, 18, 20, 0>        |    %v5 = add i16 %c6, %c4;
-; CHECK-NEXT:    {{.*}}|    3: <2, 16, 17, 0>        |    %v6 = add i32 %c9, %c8;
-; CHECK-NEXT:    {{.*}}|    3: <2, 16, 18, 0>        |    %v7 = add i32 %c10, %c8;
-; CHECK-NEXT:    {{.*}}|    3: <2, 14, 15, 0>        |    %v8 = add i64 %c13, %c12;
-; CHECK-NEXT:    {{.*}}|    3: <2, 14, 16, 0>        |    %v9 = add i64 %c14, %c12;
-
-  ; Test undefined integer values.
-  %v10 = xor i1 undef, 0
-  %v11 = add i8 undef, 0
-  %v12 = add i16 undef, 0
-  %v13 = add i32 undef, 0
-  %v14 = add i64 undef, 0
-  ret void
-
-; CHECK-NEXT:    {{.*}}|    3: <2, 11, 13, 12>       |    %v10 = xor i1 %c18, %c16;
-; CHECK-NEXT:    {{.*}}|    3: <2, 27, 30, 0>        |    %v11 = add i8 %c3, %c0;
-; CHECK-NEXT:    {{.*}}|    3: <2, 24, 27, 0>        |    %v12 = add i16 %c7, %c4;
-; CHECK-NEXT:    {{.*}}|    3: <2, 21, 24, 0>        |    %v13 = add i32 %c11, %c8;
-; CHECK-NEXT:    {{.*}}|    3: <2, 18, 21, 0>        |    %v14 = add i64 %c15, %c12;
-; CHECK-NEXT:    {{.*}}|    3: <10>                  |    ret void;
-}
-
-define void @TestFloats() {
-
-; CHECK:         {{.*}}|    1: <65535, 11, 2>        |    constants {  // BlockID = 11
-; CHECK-NEXT:    {{.*}}|      3: <1, 0>              |      float:
-; CHECK-NEXT:    {{.*}}|      3: <6, 0>              |        %c0 = float 0;
-; CHECK-NEXT:    {{.*}}|      3: <6, 1065353216>     |        %c1 = float 1;
-; CHECK-NEXT:    {{.*}}|      3: <6, 1088421888>     |        %c2 = float 7;
-; CHECK-NEXT:    {{.*}}|      3: <6, 1090519040>     |        %c3 = float 8;
-; CHECK-NEXT:    {{.*}}|      3: <3>                 |        %c4 = float undef;
-; CHECK-NEXT:    {{.*}}|      3: <1, 1>              |      double:
-; CHECK-NEXT:    {{.*}}|      3: <6,                 |        %c5 = double 6;
-; CHECK-NEXT:          |        4618441417868443648> |
-; CHECK-NEXT:    {{.*}}|      3: <6,                 |        %c6 = double 1;
-; CHECK-NEXT:          |        4607182418800017408> |
-; CHECK-NEXT:    {{.*}}|      3: <6, 0>              |        %c7 = double 0;
-; CHECK-NEXT:    {{.*}}|      3: <6,                 |        %c8 = double 5;
-; CHECK-NEXT:          |        4617315517961601024> |
-; CHECK-NEXT:    {{.*}}|      3: <3>                 |        %c9 = double undef;
-; CHECK-NEXT:    {{.*}}|    0: <65534>               |      }
-; CHECK-NEXT:          |                             |  %b0:
-
-  ; Test float and double constants
-  %1 = fadd float 1.0, 0.0
-  %2 = fadd double 1.0, 0.0
-  %3 = fsub float 7.000000e+00, 8.000000e+00
-  %4 = fsub double 5.000000e+00, 6.000000e+00
-
-; CHECK-NEXT:    {{.*}}|    3: <2, 9, 10, 0>         |    %v0 = fadd float %c1, %c0;
-; CHECK-NEXT:    {{.*}}|    3: <2, 5, 4, 0>          |    %v1 = fadd double %c6, %c7;
-; CHECK-NEXT:    {{.*}}|    3: <2, 10, 9, 1>         |    %v2 = fsub float %c2, %c3;
-; CHECK-NEXT:    {{.*}}|    3: <2, 5, 8, 1>          |    %v3 = fsub double %c8, %c5;
-
-  ; Test undefined float and double.
-  %5 = fadd float undef, 0.0
-  %6 = fsub double undef, 6.000000e+00
-  ret void
-
-; CHECK-NEXT:    {{.*}}|    3: <2, 10, 14, 0>        |    %v4 = fadd float %c4, %c0;
-; CHECK-NEXT:    {{.*}}|    3: <2, 6, 10, 1>         |    %v5 = fsub double %c9, %c5;
-; CHECK-NEXT:    {{.*}}|    3: <10>                  |    ret void;
-
-}
-
-; Test float Nan, +Inf, -Inf.
-; Note: llvm-as doesn't accept float hex values. Only accepts double
-; hex values.
-define float @GetFloatNan() {
-  ; Generated from NAN in <math.h>
-  ret float 0x7FF8000000000000
-
-; CHECK:         {{.*}}|    1: <65535, 11, 2>        |    constants {  // BlockID = 11
-; CHECK-NEXT:    {{.*}}|      3: <1, 0>              |      float:
-; CHECK-NEXT:    {{.*}}|      3: <6, 2143289344>     |        %c0 = float nan;
-; CHECK-NEXT:    {{.*}}|    0: <65534>               |      }
-; CHECK-NEXT:          |                             |  %b0:
-; CHECK-NEXT:    {{.*}}|    3: <10, 1>               |    ret float %c0;
-
-}
-
-define float @GetFloatInf() {
-  ; Generated from INFINITY in <math.h>
-  ret float 0x7FF0000000000000
-
-; CHECK:         {{.*}}|    1: <65535, 11, 2>        |    constants {  // BlockID = 11
-; CHECK-NEXT:    {{.*}}|      3: <1, 0>              |      float:
-; CHECK-NEXT:    {{.*}}|      3: <6, 2139095040>     |        %c0 = float inf;
-; CHECK-NEXT:    {{.*}}|    0: <65534>               |      }
-; CHECK-NEXT:          |                             |  %b0:
-; CHECK-NEXT:    {{.*}}|    3: <10, 1>               |    ret float %c0;
-
-}
-
-define float @GetFloatNegInf() {
-  ; Generated from -INFINITY in <math.h>
-  ret float 0xFFF0000000000000
-
-; CHECK:         {{.*}}|    1: <65535, 11, 2>        |    constants {  // BlockID = 11
-; CHECK-NEXT:    {{.*}}|      3: <1, 0>              |      float:
-; CHECK-NEXT:    {{.*}}|      3: <6, 4286578688>     |        %c0 = float -inf;
-; CHECK-NEXT:    {{.*}}|    0: <65534>               |      }
-; CHECK-NEXT:          |                             |  %b0:
-; CHECK-NEXT:    {{.*}}|    3: <10, 1>               |    ret float %c0;
-}
-
-; Test double Nan, +Inf, -Inf.
-define double @GetDoubleNan() {
-  ; Generated from NAN in <math.h>
-  ret double 0x7FF8000000000000
-
-; CHECK:         {{.*}}|    1: <65535, 11, 2>        |    constants {  // BlockID = 11
-; CHECK-NEXT:    {{.*}}|      3: <1, 1>              |      double:
-; CHECK-NEXT:    {{.*}}|      3: <6,                 |        %c0 = double nan;
-; CHECK-NEXT:          |        9221120237041090560> |
-; CHECK-NEXT:    {{.*}}|    0: <65534>               |      }
-; CHECK-NEXT:          |                             |  %b0:
-; CHECK-NEXT:    {{.*}}|    3: <10, 1>               |    ret double %c0;
-
-}
-
-define double @GetDoubleInf() {
-  ; Generated from INFINITY in <math.h>
-  ret double 0x7FF0000000000000
-
-; CHECK:         {{.*}}|    1: <65535, 11, 2>        |    constants {  // BlockID = 11
-; CHECK-NEXT:    {{.*}}|      3: <1, 1>              |      double:
-; CHECK-NEXT:    {{.*}}|      3: <6,                 |        %c0 = double inf;
-; CHECK-NEXT:          |        9218868437227405312> |
-; CHECK-NEXT:    {{.*}}|    0: <65534>               |      }
-; CHECK-NEXT:          |                             |  %b0:
-; CHECK-NEXT:    {{.*}}|    3: <10, 1>               |    ret double %c0;
-
-}
-
-define double @GetDoubleNegInf() {
-  ; Generated from -INFINITY in <math.h>
-  ret double 0xFFF0000000000000
-
-; CHECK:         {{.*}}|    1: <65535, 11, 2>        |    constants {  // BlockID = 11
-; CHECK-NEXT:    {{.*}}|      3: <1, 1>              |      double:
-; CHECK-NEXT:    {{.*}}|      3: <6,                 |        %c0 = double -inf;
-; CHECK-NEXT:          |        18442240474082181120>|
-; CHECK-NEXT:    {{.*}}|    0: <65534>               |      }
-; CHECK-NEXT:          |                             |  %b0:
-; CHECK-NEXT:    {{.*}}|    3: <10, 1>               |    ret double %c0;
-
-}
-
diff --git a/test/NaCl/Bitcode/pnacl-bcdis/fcmp.ll b/test/NaCl/Bitcode/pnacl-bcdis/fcmp.ll
deleted file mode 100644
index 5115f1e4368..00000000000
--- a/test/NaCl/Bitcode/pnacl-bcdis/fcmp.ll
+++ /dev/null
@@ -1,226 +0,0 @@
-; Tests the fcmp instruction.
-
-; RUN: llvm-as < %s | pnacl-freeze | pnacl-bccompress --remove-abbreviations \
-; RUN:              | pnacl-bcdis | FileCheck %s
-
-
-; Test fcmp on primitive types.
-define void @SimpleFcmpOps(float %p0, double %p1) {
-
-; CHECK:               |                             |  %b0:
-
-  %v0 = fcmp false float %p0, 2.0
-  %v1 = fcmp false double %p1, 3.0
-
-; CHECK-NEXT:    {{.*}}|    3: <28, 4, 2, 0>         |    %v0 = fcmp false float %p0, %c0;
-; CHECK-NEXT:    {{.*}}|    3: <28, 4, 2, 0>         |    %v1 = fcmp false double %p1, %c1;
-
-  %v2 = fcmp oeq float %p0, 2.0
-  %v3 = fcmp oeq double %p1, 3.0
-
-; CHECK-NEXT:    {{.*}}|    3: <28, 6, 4, 1>         |    %v2 = fcmp oeq float %p0, %c0;
-; CHECK-NEXT:    {{.*}}|    3: <28, 6, 4, 1>         |    %v3 = fcmp oeq double %p1, %c1;
-
-  %v4 = fcmp ogt float %p0, 2.0
-  %v5 = fcmp ogt double %p1, 3.0
-
-; CHECK-NEXT:    {{.*}}|    3: <28, 8, 6, 2>         |    %v4 = fcmp ogt float %p0, %c0;
-; CHECK-NEXT:    {{.*}}|    3: <28, 8, 6, 2>         |    %v5 = fcmp ogt double %p1, %c1;
-
-  %v6 = fcmp oge float %p0, 2.0
-  %v7 = fcmp oge double %p1, 3.0
-
-; CHECK-NEXT:    {{.*}}|    3: <28, 10, 8, 3>        |    %v6 = fcmp oge float %p0, %c0;
-; CHECK-NEXT:    {{.*}}|    3: <28, 10, 8, 3>        |    %v7 = fcmp oge double %p1, %c1;
-
-  %v8 = fcmp olt float %p0, 2.0
-  %v9 = fcmp olt double %p1, 3.0
-
-; CHECK-NEXT:    {{.*}}|    3: <28, 12, 10, 4>       |    %v8 = fcmp olt float %p0, %c0;
-; CHECK-NEXT:    {{.*}}|    3: <28, 12, 10, 4>       |    %v9 = fcmp olt double %p1, %c1;
-
-  %v10 = fcmp ole float %p0, 2.0
-  %v11 = fcmp ole double %p1, 3.0
-
-; CHECK-NEXT:    {{.*}}|    3: <28, 14, 12, 5>       |    %v10 = fcmp ole float %p0, %c0;
-; CHECK-NEXT:    {{.*}}|    3: <28, 14, 12, 5>       |    %v11 = fcmp ole double %p1, %c1;
-
-  %v12 = fcmp one float %p0, 2.0
-  %v13 = fcmp one double %p1, 3.0
-
-; CHECK-NEXT:    {{.*}}|    3: <28, 16, 14, 6>       |    %v12 = fcmp one float %p0, %c0;
-; CHECK-NEXT:    {{.*}}|    3: <28, 16, 14, 6>       |    %v13 = fcmp one double %p1, %c1;
-
-  %v14 = fcmp ord float %p0, 2.0
-  %v15 = fcmp ord double %p1, 3.0
-
-; CHECK-NEXT:    {{.*}}|    3: <28, 18, 16, 7>       |    %v14 = fcmp ord float %p0, %c0;
-; CHECK-NEXT:    {{.*}}|    3: <28, 18, 16, 7>       |    %v15 = fcmp ord double %p1, %c1;
-
-  %v16 = fcmp ueq float %p0, 2.0
-  %v17 = fcmp ueq double %p1, 3.0
-
-; CHECK-NEXT:    {{.*}}|    3: <28, 20, 18, 9>       |    %v16 = fcmp ueq float %p0, %c0;
-; CHECK-NEXT:    {{.*}}|    3: <28, 20, 18, 9>       |    %v17 = fcmp ueq double %p1, %c1;
-
-  %v18 = fcmp ugt float %p0, 2.0
-  %v19 = fcmp ugt double %p1, 3.0
-
-; CHECK-NEXT:    {{.*}}|    3: <28, 22, 20, 10>      |    %v18 = fcmp ugt float %p0, %c0;
-; CHECK-NEXT:    {{.*}}|    3: <28, 22, 20, 10>      |    %v19 = fcmp ugt double %p1, %c1;
-
-  %v20 = fcmp uge float %p0, 2.0
-  %v21 = fcmp uge double %p1, 3.0
-
-; CHECK-NEXT:    {{.*}}|    3: <28, 24, 22, 11>      |    %v20 = fcmp uge float %p0, %c0;
-; CHECK-NEXT:    {{.*}}|    3: <28, 24, 22, 11>      |    %v21 = fcmp uge double %p1, %c1;
-
-  %v22 = fcmp ult float %p0, 2.0
-  %v23 = fcmp ult double %p1, 3.0
-
-; CHECK-NEXT:    {{.*}}|    3: <28, 26, 24, 12>      |    %v22 = fcmp ult float %p0, %c0;
-; CHECK-NEXT:    {{.*}}|    3: <28, 26, 24, 12>      |    %v23 = fcmp ult double %p1, %c1;
-
-  %v24 = fcmp ule float %p0, 2.0
-  %v25 = fcmp ule double %p1, 3.0
-
-; CHECK-NEXT:    {{.*}}|    3: <28, 28, 26, 13>      |    %v24 = fcmp ule float %p0, %c0;
-; CHECK-NEXT:    {{.*}}|    3: <28, 28, 26, 13>      |    %v25 = fcmp ule double %p1, %c1;
-
-  %v26 = fcmp une float %p0, 2.0
-  %v27 = fcmp une double %p1, 3.0
-
-; CHECK-NEXT:    {{.*}}|    3: <28, 30, 28, 14>      |    %v26 = fcmp une float %p0, %c0;
-; CHECK-NEXT:    {{.*}}|    3: <28, 30, 28, 14>      |    %v27 = fcmp une double %p1, %c1;
-
-  %v28 = fcmp uno float %p0, 2.0
-  %v29 = fcmp uno double %p1, 3.0
-
-; CHECK-NEXT:    {{.*}}|    3: <28, 32, 30, 8>       |    %v28 = fcmp uno float %p0, %c0;
-; CHECK-NEXT:    {{.*}}|    3: <28, 32, 30, 8>       |    %v29 = fcmp uno double %p1, %c1;
-
-  %v30 = fcmp true float %p0, 2.0
-  %v31 = fcmp true double %p1, 3.0
-
-; CHECK-NEXT:    {{.*}}|    3: <28, 34, 32, 15>      |    %v30 = fcmp true float %p0, %c0;
-; CHECK-NEXT:    {{.*}}|    3: <28, 34, 32, 15>      |    %v31 = fcmp true double %p1, %c1;
-
-  ; Verifies result is i1.
-  %v32 = and i1 %v0, %v16
-  %v33 = and i1 %v1, %v17
-  %v34 = and i1 %v2, %v18
-  %v35 = and i1 %v3, %v19
-  %v36 = and i1 %v4, %v20
-  %v37 = and i1 %v5, %v21
-  %v38 = and i1 %v6, %v22
-  %v39 = and i1 %v7, %v23
-  %v40 = and i1 %v8, %v24
-  %v41 = and i1 %v9, %v25
-  %v42 = and i1 %v10, %v26
-  %v43 = and i1 %v11, %v27
-  %v44 = and i1 %v12, %v28
-  %v45 = and i1 %v13, %v29
-  %v46 = and i1 %v14, %v30
-  %v47 = and i1 %v15, %v31
-  ret void
-
-; CHECK-NEXT:    {{.*}}|    3: <2, 32, 16, 10>       |    %v32 = and i1 %v0, %v16;
-; CHECK-NEXT:    {{.*}}|    3: <2, 32, 16, 10>       |    %v33 = and i1 %v1, %v17;
-; CHECK-NEXT:    {{.*}}|    3: <2, 32, 16, 10>       |    %v34 = and i1 %v2, %v18;
-; CHECK-NEXT:    {{.*}}|    3: <2, 32, 16, 10>       |    %v35 = and i1 %v3, %v19;
-; CHECK-NEXT:    {{.*}}|    3: <2, 32, 16, 10>       |    %v36 = and i1 %v4, %v20;
-; CHECK-NEXT:    {{.*}}|    3: <2, 32, 16, 10>       |    %v37 = and i1 %v5, %v21;
-; CHECK-NEXT:    {{.*}}|    3: <2, 32, 16, 10>       |    %v38 = and i1 %v6, %v22;
-; CHECK-NEXT:    {{.*}}|    3: <2, 32, 16, 10>       |    %v39 = and i1 %v7, %v23;
-; CHECK-NEXT:    {{.*}}|    3: <2, 32, 16, 10>       |    %v40 = and i1 %v8, %v24;
-; CHECK-NEXT:    {{.*}}|    3: <2, 32, 16, 10>       |    %v41 = and i1 %v9, %v25;
-; CHECK-NEXT:    {{.*}}|    3: <2, 32, 16, 10>       |    %v42 = and i1 %v10, %v26;
-; CHECK-NEXT:    {{.*}}|    3: <2, 32, 16, 10>       |    %v43 = and i1 %v11, %v27;
-; CHECK-NEXT:    {{.*}}|    3: <2, 32, 16, 10>       |    %v44 = and i1 %v12, %v28;
-; CHECK-NEXT:    {{.*}}|    3: <2, 32, 16, 10>       |    %v45 = and i1 %v13, %v29;
-; CHECK-NEXT:    {{.*}}|    3: <2, 32, 16, 10>       |    %v46 = and i1 %v14, %v30;
-; CHECK-NEXT:    {{.*}}|    3: <2, 32, 16, 10>       |    %v47 = and i1 %v15, %v31;
-; CHECK-NEXT:    {{.*}}|    3: <10>                  |    ret void;
-
-}
-
-
-
-; Tests floating vector compares
-define internal void @VecFloatOps(<4 x float> %p0) {
-
-; CHECK:               |                             |  %b0:
-
-  %v0 = fcmp false <4 x float> %p0, %p0
-  %v1 = fcmp oeq <4 x float> %p0, %p0
-  %v2 = fcmp ogt <4 x float> %p0, %p0
-  %v3 = fcmp oge <4 x float> %p0, %p0
-  %v4 = fcmp olt <4 x float> %p0, %p0
-  %v5 = fcmp ole <4 x float> %p0, %p0
-  %v6 = fcmp one <4 x float> %p0, %p0
-  %v7 = fcmp ord <4 x float> %p0, %p0
-  %v8 = fcmp ueq <4 x float> %p0, %p0
-  %v9 = fcmp ugt <4 x float> %p0, %p0
-  %v10 = fcmp uge <4 x float> %p0, %p0
-  %v11 = fcmp ult <4 x float> %p0, %p0
-  %v12 = fcmp ule <4 x float> %p0, %p0
-  %v13 = fcmp une <4 x float> %p0, %p0
-  %v14 = fcmp uno <4 x float> %p0, %p0
-  %v15 = fcmp true <4 x float> %p0, %p0
-
-; CHECK-NEXT:    {{.*}}|    3: <28, 1, 1, 0>         |    %v0 = fcmp false <4 x float> %p0, 
-; CHECK-NEXT:          |                             |        %p0;
-; CHECK-NEXT:    {{.*}}|    3: <28, 2, 2, 1>         |    %v1 = fcmp oeq <4 x float> %p0, 
-; CHECK-NEXT:          |                             |        %p0;
-; CHECK-NEXT:    {{.*}}|    3: <28, 3, 3, 2>         |    %v2 = fcmp ogt <4 x float> %p0, 
-; CHECK-NEXT:          |                             |        %p0;
-; CHECK-NEXT:    {{.*}}|    3: <28, 4, 4, 3>         |    %v3 = fcmp oge <4 x float> %p0, 
-; CHECK-NEXT:          |                             |        %p0;
-; CHECK-NEXT:    {{.*}}|    3: <28, 5, 5, 4>         |    %v4 = fcmp olt <4 x float> %p0, 
-; CHECK-NEXT:          |                             |        %p0;
-; CHECK-NEXT:    {{.*}}|    3: <28, 6, 6, 5>         |    %v5 = fcmp ole <4 x float> %p0, 
-; CHECK-NEXT:          |                             |        %p0;
-; CHECK-NEXT:    {{.*}}|    3: <28, 7, 7, 6>         |    %v6 = fcmp one <4 x float> %p0, 
-; CHECK-NEXT:          |                             |        %p0;
-; CHECK-NEXT:    {{.*}}|    3: <28, 8, 8, 7>         |    %v7 = fcmp ord <4 x float> %p0, 
-; CHECK-NEXT:          |                             |        %p0;
-; CHECK-NEXT:    {{.*}}|    3: <28, 9, 9, 9>         |    %v8 = fcmp ueq <4 x float> %p0, 
-; CHECK-NEXT:          |                             |        %p0;
-; CHECK-NEXT:    {{.*}}|    3: <28, 10, 10, 10>      |    %v9 = fcmp ugt <4 x float> %p0, 
-; CHECK-NEXT:          |                             |        %p0;
-; CHECK-NEXT:    {{.*}}|    3: <28, 11, 11, 11>      |    %v10 = fcmp uge <4 x float> %p0, 
-; CHECK-NEXT:          |                             |        %p0;
-; CHECK-NEXT:    {{.*}}|    3: <28, 12, 12, 12>      |    %v11 = fcmp ult <4 x float> %p0, 
-; CHECK-NEXT:          |                             |        %p0;
-; CHECK-NEXT:    {{.*}}|    3: <28, 13, 13, 13>      |    %v12 = fcmp ule <4 x float> %p0, 
-; CHECK-NEXT:          |                             |        %p0;
-; CHECK-NEXT:    {{.*}}|    3: <28, 14, 14, 14>      |    %v13 = fcmp une <4 x float> %p0, 
-; CHECK-NEXT:          |                             |        %p0;
-; CHECK-NEXT:    {{.*}}|    3: <28, 15, 15, 8>       |    %v14 = fcmp uno <4 x float> %p0, 
-; CHECK-NEXT:          |                             |        %p0;
-; CHECK-NEXT:    {{.*}}|    3: <28, 16, 16, 15>      |    %v15 = fcmp true <4 x float> %p0, 
-; CHECK-NEXT:          |                             |        %p0;
-
-  ; Verify the result is a vector of the right size.
-  %v16 = and <4 x i1> %v0, %v1
-  %v17 = and <4 x i1> %v2, %v3
-  %v18 = and <4 x i1> %v4, %v5
-  %v19 = and <4 x i1> %v6, %v7
-  %v20 = and <4 x i1> %v8, %v9
-  %v21 = and <4 x i1> %v10, %v11
-  %v22 = and <4 x i1> %v12, %v13
-  %v23 = and <4 x i1> %v14, %v15
-  ret void
-
-; CHECK-NEXT:    {{.*}}|    3: <2, 16, 15, 10>       |    %v16 = and <4 x i1> %v0, %v1;
-; CHECK-NEXT:    {{.*}}|    3: <2, 15, 14, 10>       |    %v17 = and <4 x i1> %v2, %v3;
-; CHECK-NEXT:    {{.*}}|    3: <2, 14, 13, 10>       |    %v18 = and <4 x i1> %v4, %v5;
-; CHECK-NEXT:    {{.*}}|    3: <2, 13, 12, 10>       |    %v19 = and <4 x i1> %v6, %v7;
-; CHECK-NEXT:    {{.*}}|    3: <2, 12, 11, 10>       |    %v20 = and <4 x i1> %v8, %v9;
-; CHECK-NEXT:    {{.*}}|    3: <2, 11, 10, 10>       |    %v21 = and <4 x i1> %v10, %v11;
-; CHECK-NEXT:    {{.*}}|    3: <2, 10, 9, 10>        |    %v22 = and <4 x i1> %v12, %v13;
-; CHECK-NEXT:    {{.*}}|    3: <2, 9, 8, 10>         |    %v23 = and <4 x i1> %v14, %v15;
-; CHECK-NEXT:    {{.*}}|    3: <10>                  |    ret void;
-
-}
-
diff --git a/test/NaCl/Bitcode/pnacl-bcdis/forwarddcl.ll b/test/NaCl/Bitcode/pnacl-bcdis/forwarddcl.ll
deleted file mode 100644
index 23f425a9d1e..00000000000
--- a/test/NaCl/Bitcode/pnacl-bcdis/forwarddcl.ll
+++ /dev/null
@@ -1,94 +0,0 @@
-; Test if forward declare works.
-
-; RUN: llvm-as < %s | pnacl-freeze | pnacl-bccompress --remove-abbreviations \
-; RUN:              | pnacl-bcdis | FileCheck %s
-
-; Tests case where forward declare will be added because of loop carried
-; dependency.
-define void @LoopCarriedDep() {
-
-; CHECK:         {{.*}}|    1: <65535, 11, 2>        |    constants {  // BlockID = 11
-; CHECK-NEXT:    {{.*}}|      3: <1, 0>              |      i32:
-; CHECK-NEXT:    {{.*}}|      3: <4, 2>              |        %c0 = i32 1;
-; CHECK-NEXT:    {{.*}}|      3: <4, 4>              |        %c1 = i32 2;
-; CHECK-NEXT:    {{.*}}|    0: <65534>               |      }
-
-b0:
-  %v0 = add i32 1, 2
-  br label %b1
-
-; CHECK-NEXT:          |                             |  %b0:
-; CHECK-NEXT:    {{.*}}|    3: <2, 2, 1, 0>          |    %v0 = add i32 %c0, %c1;
-; CHECK-NEXT:    {{.*}}|    3: <11, 1>               |    br label %b1;
-
-b1:
-  %v1 = phi i32 [%v0, %b0], [%v2, %b1]
-  %v2 = add i32 %v1, 1
-  br label %b1
-
-; CHECK-NEXT:          |                             |  %b1:
-; CHECK-NEXT:    {{.*}}|    3: <43, 6, 0>            |    declare i32 %v2;
-; CHECK-NEXT:    {{.*}}|    3: <16, 0, 2, 0, 3, 1>   |    %v1 = phi i32 [%v0, %b0], 
-; CHECK-NEXT:          |                             |        [%v2, %b1];
-; CHECK-NEXT:    {{.*}}|    3: <2, 1, 4, 0>          |    %v2 = add i32 %v1, %c0;
-; CHECK-NEXT:    {{.*}}|    3: <11, 1>               |    br label %b1;
-
-}
-
-; Test case where we backward branch.
-define void @BackBranch(i32 %p0) {
-  br label %b4
-
-; CHECK:               |                             |  %b0:
-; CHECK-NEXT:    {{.*}}|    3: <11, 4>               |    br label %b4;
-
-b1:
-  %v0 = add i32 %p0, %v3
-  br label %b6
-
-; CHECK-NEXT:          |                             |  %b1:
-; CHECK-NEXT:    {{.*}}|    3: <43, 7, 0>            |    declare i32 %v3;
-; CHECK-NEXT:    {{.*}}|    3: <2, 2, 4294967293, 0> |    %v0 = add i32 %p0, %v3;
-; CHECK-NEXT:    {{.*}}|    3: <11, 6>               |    br label %b6;
-
-b2:
-  %v1 = add i32 %p0, %v4
-  br label %b6
-
-; CHECK-NEXT:          |                             |  %b2:
-; CHECK-NEXT:    {{.*}}|    3: <43, 8, 0>            |    declare i32 %v4;
-; CHECK-NEXT:    {{.*}}|    3: <2, 3, 4294967293, 0> |    %v1 = add i32 %p0, %v4;
-; CHECK-NEXT:    {{.*}}|    3: <11, 6>               |    br label %b6;
-
-b3:
-  %v2 = add i32 %p0, %v3 ; No forward declare, already done!
-  br label %b6
-
-; CHECK-NEXT:          |                             |  %b3:
-; CHECK-NEXT:    {{.*}}|    3: <2, 4, 4294967295, 0> |    %v2 = add i32 %p0, %v3;
-; CHECK-NEXT:    {{.*}}|    3: <11, 6>               |    br label %b6;
-
-b4:
-  %v3 = add i32 %p0, %p0
-  br i1 1, label %b1, label %b5
-
-; CHECK-NEXT:          |                             |  %b4:
-; CHECK-NEXT:    {{.*}}|    3: <2, 5, 5, 0>          |    %v3 = add i32 %p0, %p0;
-; CHECK-NEXT:    {{.*}}|    3: <11, 1, 5, 5>         |    br i1 %c0, label %b1, label %b5;
-
-b5:
-  %v4 = add i32 %v3, %p0
-  br i1 1, label %b2, label %b3
-
-; CHECK-NEXT:          |                             |  %b5:
-; CHECK-NEXT:    {{.*}}|    3: <2, 1, 6, 0>          |    %v4 = add i32 %v3, %p0;
-; CHECK-NEXT:    {{.*}}|    3: <11, 2, 3, 6>         |    br i1 %c0, label %b2, label %b3;
-
-b6:
-  ret void
-
-; CHECK-NEXT:          |                             |  %b6:
-; CHECK-NEXT:    {{.*}}|    3: <10>                  |    ret void;
-
-}
-
diff --git a/test/NaCl/Bitcode/pnacl-bcdis/global-fwd-ref.ll b/test/NaCl/Bitcode/pnacl-bcdis/global-fwd-ref.ll
deleted file mode 100644
index 4a85dbd9fca..00000000000
--- a/test/NaCl/Bitcode/pnacl-bcdis/global-fwd-ref.ll
+++ /dev/null
@@ -1,46 +0,0 @@
-; Tests that forward (relocation) reference to a global works.
-
-; RUN: llvm-as < %s | pnacl-freeze | pnacl-bccompress --remove-abbreviations \
-; RUN:              | pnacl-bcdis | FileCheck %s
-
-; CHECK:       0:0|<65532, 80, 69, 88, 69, 1, 0,|Magic Number: 'PEXE' (80, 69, 88, 69)
-; CHECK-NEXT:     | 8, 0, 17, 0, 4, 0, 2, 0, 0, |PNaCl Version: 2
-; CHECK-NEXT:     | 0>                          |
-; CHECK-NEXT: 16:0|1: <65535, 8, 2>             |module {  // BlockID = 8
-; CHECK-NEXT: 24:0|  3: <1, 1>                  |  version 1;
-; CHECK-NEXT: 26:4|  1: <65535, 17, 2>          |  types {  // BlockID = 17
-; CHECK-NEXT: 36:0|    3: <1, 2>                |    count 2;
-; CHECK-NEXT: 38:4|    3: <2>                   |    @t0 = void;
-; CHECK-NEXT: 40:2|    3: <21, 0, 0>            |    @t1 = void ();
-; CHECK-NEXT: 43:4|  0: <65534>                 |  }
-
-declare void @f0()
-
-; CHECK-NEXT: 44:0|  3: <8, 1, 0, 1, 0>         |  declare external void @f0();
-; CHECK-NEXT: 48:6|  1: <65535, 19, 2>          |  globals {  // BlockID = 19
-; CHECK-NEXT: 56:0|    3: <5, 2>                |    count 2;
-
-@g0 = internal global <{ i32 , i32 , i32 }>
-        <{ i32 ptrtoint (void ()* @f0 to i32),
-           i32 ptrtoint (<{ i32 , i32 , i32 }>* @g0 to i32),
-           i32 ptrtoint ([4 x i8]* @g1 to i32) ; forward reference!
-        }>, align 1
-
-; CHECK-NEXT: 58:4|    3: <0, 1, 0>             |    var @g0, align 1,
-; CHECK-NEXT: 61:6|    3: <1, 3>                |      initializers 3 {
-; CHECK-NEXT: 64:2|    3: <4, 0>                |        reloc @f0;
-; CHECK-NEXT: 66:6|    3: <4, 1>                |        reloc @g0;
-; CHECK-NEXT: 69:2|    3: <4, 2>                |        reloc @g1;
-; CHECK-NEXT:     |                             |      }
-
-@g1 = internal global [4 x i8] zeroinitializer, align 4
-
-; CHECK-NEXT: 71:6|    3: <0, 3, 0>             |    var @g1, align 4,
-; CHECK-NEXT: 75:0|    3: <2, 4>                |      zerofill 4;
-; CHECK-NEXT: 77:4|  0: <65534>                 |  }
-; CHECK-NEXT: 80:0|  1: <65535, 14, 2>          |  valuesymtab {  // BlockID = 14
-; CHECK-NEXT: 88:0|    3: <1, 0, 102, 48>       |    @f0 : "f0";
-; CHECK-NEXT: 93:4|    3: <1, 1, 103, 48>       |    @g0 : "g0";
-; CHECK-NEXT: 99:0|    3: <1, 2, 103, 49>       |    @g1 : "g1";
-; CHECK-NEXT:104:4|  0: <65534>                 |  }
-; CHECK-NEXT:108:0|0: <65534>                   |}
diff --git a/test/NaCl/Bitcode/pnacl-bcdis/global-with-neg-offset.ll b/test/NaCl/Bitcode/pnacl-bcdis/global-with-neg-offset.ll
deleted file mode 100644
index 7e33663a3b4..00000000000
--- a/test/NaCl/Bitcode/pnacl-bcdis/global-with-neg-offset.ll
+++ /dev/null
@@ -1,85 +0,0 @@
-; Show that we detect when a negative offset is used within a relocation.
-
-; RUN: llvm-as < %s | pnacl-freeze | pnacl-bcdis  | FileCheck %s
-
-@bytes = internal global [7 x i8] c"abcdefg"
-@addend_negative = internal global i32 add (i32 ptrtoint ([7 x i8]* @bytes to i32), i32 -1)
-
-; CHECK:            0:0|<65532, 80, 69, 88, 69, 1, 0,|Magic Number: 'PEXE' (80, 69, 88, 69)
-; CHECK-NEXT:          | 8, 0, 17, 0, 4, 0, 2, 0, 0, |PNaCl Version: 2
-; CHECK-NEXT:          | 0>                          |
-; CHECK-NEXT:      16:0|1: <65535, 8, 2>             |module {  // BlockID = 8
-; CHECK-NEXT:      24:0|  3: <1, 1>                  |  version 1;
-; CHECK-NEXT:      26:4|  1: <65535, 0, 2>           |  abbreviations {  // BlockID = 0
-; CHECK-NEXT:      36:0|    3: <1, 14>               |    valuesymtab:
-; CHECK-NEXT:      38:4|    2: <65533, 4, 0, 1, 3, 0,|      @a0 = abbrev <fixed(3), vbr(8), 
-; CHECK-NEXT:          |        2, 8, 0, 3, 0, 1, 8> |                   array(fixed(8))>;
-; CHECK-NEXT:      43:2|    2: <65533, 4, 1, 1, 0, 2,|      @a1 = abbrev <1, vbr(8), 
-; CHECK-NEXT:          |        8, 0, 3, 0, 1, 7>    |                   array(fixed(7))>;
-; CHECK-NEXT:      48:0|    2: <65533, 4, 1, 1, 0, 2,|      @a2 = abbrev <1, vbr(8), 
-; CHECK-NEXT:          |        8, 0, 3, 0, 4>       |                   array(char6)>;
-; CHECK-NEXT:      52:1|    2: <65533, 4, 1, 2, 0, 2,|      @a3 = abbrev <2, vbr(8), 
-; CHECK-NEXT:          |        8, 0, 3, 0, 4>       |                   array(char6)>;
-; CHECK-NEXT:      56:2|    3: <1, 11>               |    constants:
-; CHECK-NEXT:      58:6|    2: <65533, 2, 1, 1, 0, 1,|      @a0 = abbrev <1, 0>;
-; CHECK-NEXT:          |        0, 0>                |
-; CHECK-NEXT:      61:7|    2: <65533, 2, 1, 4, 0, 2,|      @a1 = abbrev <4, vbr(8)>;
-; CHECK-NEXT:          |        8>                   |
-; CHECK-NEXT:      65:0|    2: <65533, 2, 1, 4, 1, 0>|      @a2 = abbrev <4, 0>;
-; CHECK-NEXT:      68:1|    2: <65533, 2, 1, 6, 0, 2,|      @a3 = abbrev <6, vbr(8)>;
-; CHECK-NEXT:          |        8>                   |
-; CHECK-NEXT:      71:2|    3: <1, 12>               |    function:
-; CHECK-NEXT:      73:6|    2: <65533, 4, 1, 20, 0,  |      @a0 = abbrev <20, vbr(6), vbr(4),
-; CHECK-NEXT:          |        2, 6, 0, 2, 4, 0, 2, |                   vbr(4)>;
-; CHECK-NEXT:          |        4>                   |
-; CHECK-NEXT:      79:1|    2: <65533, 4, 1, 2, 0, 2,|      @a1 = abbrev <2, vbr(6), vbr(6), 
-; CHECK-NEXT:          |        6, 0, 2, 6, 0, 1, 4> |                   fixed(4)>;
-; CHECK-NEXT:      84:4|    2: <65533, 4, 1, 3, 0, 2,|      @a2 = abbrev <3, vbr(6), 0, 
-; CHECK-NEXT:          |        6, 0, 1, 0, 0, 0, 1, |                   fixed(4)>;
-; CHECK-NEXT:          |        4>                   |
-; CHECK-NEXT:      89:7|    2: <65533, 1, 1, 10>     |      @a3 = abbrev <10>;
-; CHECK-NEXT:      91:7|    2: <65533, 2, 1, 10, 0,  |      @a4 = abbrev <10, vbr(6)>;
-; CHECK-NEXT:          |        2, 6>                |
-; CHECK-NEXT:      95:0|    2: <65533, 1, 1, 15>     |      @a5 = abbrev <15>;
-; CHECK-NEXT:      97:0|    2: <65533, 3, 1, 43, 0,  |      @a6 = abbrev <43, vbr(6), 0>;
-; CHECK-NEXT:          |        2, 6, 0, 1, 0, 0>    |
-; CHECK-NEXT:     101:2|    2: <65533, 4, 1, 24, 0,  |      @a7 = abbrev <24, vbr(6), vbr(6),
-; CHECK-NEXT:          |        2, 6, 0, 2, 6, 0, 2, |                   vbr(4)>;
-; CHECK-NEXT:          |        4>                   |
-; CHECK-NEXT:     106:5|    3: <1, 19>               |    globals:
-; CHECK-NEXT:     109:1|    2: <65533, 3, 1, 0, 0, 2,|      @a0 = abbrev <0, vbr(6), 
-; CHECK-NEXT:          |        6, 0, 1, 1>          |                   fixed(1)>;
-; CHECK-NEXT:     113:3|    2: <65533, 2, 1, 1, 0, 2,|      @a1 = abbrev <1, vbr(8)>;
-; CHECK-NEXT:          |        8>                   |
-; CHECK-NEXT:     116:4|    2: <65533, 2, 1, 2, 0, 2,|      @a2 = abbrev <2, vbr(8)>;
-; CHECK-NEXT:          |        8>                   |
-; CHECK-NEXT:     119:5|    2: <65533, 3, 1, 3, 0, 3,|      @a3 = abbrev <3, array(fixed(8))>
-; CHECK-NEXT:          |        0, 1, 8>             |          ;
-; CHECK-NEXT:     123:2|    2: <65533, 2, 1, 4, 0, 2,|      @a4 = abbrev <4, vbr(6)>;
-; CHECK-NEXT:          |        6>                   |
-; CHECK-NEXT:     126:3|    2: <65533, 3, 1, 4, 0, 2,|      @a5 = abbrev <4, vbr(6), vbr(6)>;
-; CHECK-NEXT:          |        6, 0, 2, 6>          |
-; CHECK-NEXT:     130:5|  0: <65534>                 |  }
-; CHECK-NEXT:     132:0|  1: <65535, 17, 3>          |  types {  // BlockID = 17
-; CHECK-NEXT:     140:0|    2: <65533, 4, 1, 21, 0,  |    %a0 = abbrev <21, fixed(1), 
-; CHECK-NEXT:          |        1, 1, 0, 3, 0, 1, 0, |                  array(0)>;
-; CHECK-NEXT:          |        0>                   |
-; CHECK-NEXT:     144:7|    3: <1, 0>                |    count 0;
-; CHECK-NEXT:     147:4|  0: <65534>                 |  }
-; CHECK-NEXT:     148:0|  1: <65535, 19, 4>          |  globals {  // BlockID = 19
-; CHECK-NEXT:     156:0|    3: <5, 2>                |    count 2;
-; CHECK-NEXT:     158:6|    4: <0, 0, 0>             |    var @g0, align 0, <@a0>
-; CHECK-NEXT:     160:1|    7: <3, 97, 98, 99, 100,  |      { 97,  98,  99, 100, 101, 102, 
-; CHECK-NEXT:          |        101, 102, 103>       |       103} <@a3>
-; CHECK-NEXT:     168:3|    4: <0, 0, 0>             |    var @g1, align 0, <@a0>
-; CHECK-NEXT:     169:6|    9: <4, 0, 4294967295>    |      reloc @g0 - 1; <@a5>
-; CHECK-NEXT:     176:2|  0: <65534>                 |  }
-; CHECK-NEXT:     180:0|  1: <65535, 14, 3>          |  valuesymtab {  // BlockID = 14
-; CHECK-NEXT:     188:0|    6: <1, 1, 97, 100, 100,  |    @g1 : "addend_negative"; <@a2>
-; CHECK-NEXT:          |        101, 110, 100, 95,   |
-; CHECK-NEXT:          |        110, 101, 103, 97,   |
-; CHECK-NEXT:          |        116, 105, 118, 101>  |
-; CHECK-NEXT:     201:3|    6: <1, 0, 98, 121, 116,  |    @g0 : "bytes"; <@a2>
-; CHECK-NEXT:          |        101, 115>            |
-; CHECK-NEXT:     207:2|  0: <65534>                 |  }
-; CHECK-NEXT:     208:0|0: <65534>                   |}
diff --git a/test/NaCl/Bitcode/pnacl-bcdis/globals-ending-with-compound.ll b/test/NaCl/Bitcode/pnacl-bcdis/globals-ending-with-compound.ll
deleted file mode 100644
index 2404191a7f5..00000000000
--- a/test/NaCl/Bitcode/pnacl-bcdis/globals-ending-with-compound.ll
+++ /dev/null
@@ -1,92 +0,0 @@
-; Test that we properly handle the globals block correctly, when it ends with
-; a compound initializer.
-
-; RUN: llvm-as < %s | pnacl-freeze | pnacl-bcdis | FileCheck %s
-
-@bytes = internal global [7 x i8] c"abcdefg"
-@compound = internal global <{ [3 x i8], i32 }> <{ [3 x i8] c"foo", i32 ptrtoint (void ()* @func to i32) }>
-declare void @func();
-
-; CHECK:            0:0|<65532, 80, 69, 88, 69, 1, 0,|Magic Number: 'PEXE' (80, 69, 88, 69)
-; CHECK-NEXT:          | 8, 0, 17, 0, 4, 0, 2, 0, 0, |PNaCl Version: 2
-; CHECK-NEXT:          | 0>                          |
-; CHECK-NEXT:      16:0|1: <65535, 8, 2>             |module {  // BlockID = 8
-; CHECK-NEXT:      24:0|  3: <1, 1>                  |  version 1;
-; CHECK-NEXT:      26:4|  1: <65535, 0, 2>           |  abbreviations {  // BlockID = 0
-; CHECK-NEXT:      36:0|    3: <1, 14>               |    valuesymtab:
-; CHECK-NEXT:      38:4|    2: <65533, 4, 0, 1, 3, 0,|      @a0 = abbrev <fixed(3), vbr(8), 
-; CHECK-NEXT:          |        2, 8, 0, 3, 0, 1, 8> |                   array(fixed(8))>;
-; CHECK-NEXT:      43:2|    2: <65533, 4, 1, 1, 0, 2,|      @a1 = abbrev <1, vbr(8), 
-; CHECK-NEXT:          |        8, 0, 3, 0, 1, 7>    |                   array(fixed(7))>;
-; CHECK-NEXT:      48:0|    2: <65533, 4, 1, 1, 0, 2,|      @a2 = abbrev <1, vbr(8), 
-; CHECK-NEXT:          |        8, 0, 3, 0, 4>       |                   array(char6)>;
-; CHECK-NEXT:      52:1|    2: <65533, 4, 1, 2, 0, 2,|      @a3 = abbrev <2, vbr(8), 
-; CHECK-NEXT:          |        8, 0, 3, 0, 4>       |                   array(char6)>;
-; CHECK-NEXT:      56:2|    3: <1, 11>               |    constants:
-; CHECK-NEXT:      58:6|    2: <65533, 2, 1, 1, 0, 1,|      @a0 = abbrev <1, fixed(2)>;
-; CHECK-NEXT:          |        2>                   |
-; CHECK-NEXT:      61:7|    2: <65533, 2, 1, 4, 0, 2,|      @a1 = abbrev <4, vbr(8)>;
-; CHECK-NEXT:          |        8>                   |
-; CHECK-NEXT:      65:0|    2: <65533, 2, 1, 4, 1, 0>|      @a2 = abbrev <4, 0>;
-; CHECK-NEXT:      68:1|    2: <65533, 2, 1, 6, 0, 2,|      @a3 = abbrev <6, vbr(8)>;
-; CHECK-NEXT:          |        8>                   |
-; CHECK-NEXT:      71:2|    3: <1, 12>               |    function:
-; CHECK-NEXT:      73:6|    2: <65533, 4, 1, 20, 0,  |      @a0 = abbrev <20, vbr(6), vbr(4),
-; CHECK-NEXT:          |        2, 6, 0, 2, 4, 0, 2, |                   vbr(4)>;
-; CHECK-NEXT:          |        4>                   |
-; CHECK-NEXT:      79:1|    2: <65533, 4, 1, 2, 0, 2,|      @a1 = abbrev <2, vbr(6), vbr(6), 
-; CHECK-NEXT:          |        6, 0, 2, 6, 0, 1, 4> |                   fixed(4)>;
-; CHECK-NEXT:      84:4|    2: <65533, 4, 1, 3, 0, 2,|      @a2 = abbrev <3, vbr(6), 
-; CHECK-NEXT:          |        6, 0, 1, 2, 0, 1, 4> |                   fixed(2), fixed(4)>;
-; CHECK-NEXT:      89:7|    2: <65533, 1, 1, 10>     |      @a3 = abbrev <10>;
-; CHECK-NEXT:      91:7|    2: <65533, 2, 1, 10, 0,  |      @a4 = abbrev <10, vbr(6)>;
-; CHECK-NEXT:          |        2, 6>                |
-; CHECK-NEXT:      95:0|    2: <65533, 1, 1, 15>     |      @a5 = abbrev <15>;
-; CHECK-NEXT:      97:0|    2: <65533, 3, 1, 43, 0,  |      @a6 = abbrev <43, vbr(6), 
-; CHECK-NEXT:          |        2, 6, 0, 1, 2>       |                   fixed(2)>;
-; CHECK-NEXT:     101:2|    2: <65533, 4, 1, 24, 0,  |      @a7 = abbrev <24, vbr(6), vbr(6),
-; CHECK-NEXT:          |        2, 6, 0, 2, 6, 0, 2, |                   vbr(4)>;
-; CHECK-NEXT:          |        4>                   |
-; CHECK-NEXT:     106:5|    3: <1, 19>               |    globals:
-; CHECK-NEXT:     109:1|    2: <65533, 3, 1, 0, 0, 2,|      @a0 = abbrev <0, vbr(6), 
-; CHECK-NEXT:          |        6, 0, 1, 1>          |                   fixed(1)>;
-; CHECK-NEXT:     113:3|    2: <65533, 2, 1, 1, 0, 2,|      @a1 = abbrev <1, vbr(8)>;
-; CHECK-NEXT:          |        8>                   |
-; CHECK-NEXT:     116:4|    2: <65533, 2, 1, 2, 0, 2,|      @a2 = abbrev <2, vbr(8)>;
-; CHECK-NEXT:          |        8>                   |
-; CHECK-NEXT:     119:5|    2: <65533, 3, 1, 3, 0, 3,|      @a3 = abbrev <3, array(fixed(8))>
-; CHECK-NEXT:          |        0, 1, 8>             |          ;
-; CHECK-NEXT:     123:2|    2: <65533, 2, 1, 4, 0, 2,|      @a4 = abbrev <4, vbr(6)>;
-; CHECK-NEXT:          |        6>                   |
-; CHECK-NEXT:     126:3|    2: <65533, 3, 1, 4, 0, 2,|      @a5 = abbrev <4, vbr(6), vbr(6)>;
-; CHECK-NEXT:          |        6, 0, 2, 6>          |
-; CHECK-NEXT:     130:5|  0: <65534>                 |  }
-; CHECK-NEXT:     132:0|  1: <65535, 17, 3>          |  types {  // BlockID = 17
-; CHECK-NEXT:     140:0|    2: <65533, 4, 1, 21, 0,  |    %a0 = abbrev <21, fixed(1), 
-; CHECK-NEXT:          |        1, 1, 0, 3, 0, 1, 2> |                  array(fixed(2))>;
-; CHECK-NEXT:     144:7|    3: <1, 2>                |    count 2;
-; CHECK-NEXT:     147:4|    3: <2>                   |    @t0 = void;
-; CHECK-NEXT:     149:3|    4: <21, 0, 0>            |    @t1 = void (); <%a0>
-; CHECK-NEXT:     150:7|  0: <65534>                 |  }
-; CHECK-NEXT:     152:0|  3: <8, 1, 0, 1, 0>         |  declare external void @f0();
-; CHECK-NEXT:     156:6|  1: <65535, 19, 4>          |  globals {  // BlockID = 19
-; CHECK-NEXT:     164:0|    3: <5, 2>                |    count 2;
-; CHECK-NEXT:     166:6|    4: <0, 0, 0>             |    var @g0, align 0, <@a0>
-; CHECK-NEXT:     168:1|    7: <3, 97, 98, 99, 100,  |      { 97,  98,  99, 100, 101, 102, 
-; CHECK-NEXT:          |        101, 102, 103>       |       103} <@a3>
-; CHECK-NEXT:     176:3|    4: <0, 0, 0>             |    var @g1, align 0, <@a0>
-; CHECK-NEXT:     177:6|    5: <1, 2>                |      initializers 2 { <@a1>
-; CHECK-NEXT:     179:2|    7: <3, 102, 111, 111>    |        {102, 111, 111} <@a3>
-; CHECK-NEXT:     183:4|    8: <4, 0>                |        reloc @f0; <@a4>
-; CHECK-NEXT:          |                             |      }
-; CHECK-NEXT:     184:6|  0: <65534>                 |  }
-; CHECK-NEXT:     188:0|  1: <65535, 14, 3>          |  valuesymtab {  // BlockID = 14
-; CHECK-NEXT:     196:0|    6: <1, 2, 99, 111, 109,  |    @g1 : "compound"; <@a2>
-; CHECK-NEXT:          |        112, 111, 117, 110,  |
-; CHECK-NEXT:          |        100>                 |
-; CHECK-NEXT:     204:1|    6: <1, 1, 98, 121, 116,  |    @g0 : "bytes"; <@a2>
-; CHECK-NEXT:          |        101, 115>            |
-; CHECK-NEXT:     210:0|    6: <1, 0, 102, 117, 110, |    @f0 : "func"; <@a2>
-; CHECK-NEXT:          |        99>                  |
-; CHECK-NEXT:     215:1|  0: <65534>                 |  }
-; CHECK-NEXT:     216:0|0: <65534>                   |}
diff --git a/test/NaCl/Bitcode/pnacl-bcdis/globals.ll b/test/NaCl/Bitcode/pnacl-bcdis/globals.ll
deleted file mode 100644
index d081b0b524d..00000000000
--- a/test/NaCl/Bitcode/pnacl-bcdis/globals.ll
+++ /dev/null
@@ -1,155 +0,0 @@
-; Simple test to see if we handle global variables.
-
-; RUN: llvm-as < %s | pnacl-freeze | pnacl-bcdis | FileCheck %s
-
-
-@zero = internal global [4 x i8] zeroinitializer, align 4
-@bytes = internal global [7 x i8] c"abcdefg"
-@ptr_to_ptr = internal global i32 ptrtoint (i32* @ptr to i32)
-@ptr_to_func = internal global i32 ptrtoint (void ()* @func to i32)
-@compound = internal global <{ [3 x i8], i32 }> <{ [3 x i8] c"foo", i32 ptrtoint (void ()* @func to i32) }>
-@ptr = internal global i32 ptrtoint ([7 x i8]* @bytes to i32)
-@addend_ptr = internal global i32 add (i32 ptrtoint (i32* @ptr to i32), i32 1)
-@addend_array1 = internal global i32 add (i32 ptrtoint ([7 x i8]* @bytes to i32), i32 1)
-@addend_array2 = internal global i32 add (i32 ptrtoint ([7 x i8]* @bytes to i32), i32 7)
-@addend_array3 = internal global i32 add (i32 ptrtoint ([7 x i8]* @bytes to i32), i32 9)
-@addend_struct1 = internal global i32 add (i32 ptrtoint (<{ [3 x i8], i32 }>* @compound to i32), i32 1)
-@addend_struct2 = internal global i32 add (i32 ptrtoint (<{ [3 x i8], i32 }>* @compound to i32), i32 4)
-
-declare void @func();
-
-; CHECK:            0:0|<65532, 80, 69, 88, 69, 1, 0,|Magic Number: 'PEXE' (80, 69, 88, 69)
-; CHECK-NEXT:          | 8, 0, 17, 0, 4, 0, 2, 0, 0, |PNaCl Version: 2
-; CHECK-NEXT:          | 0>                          |
-; CHECK-NEXT:      16:0|1: <65535, 8, 2>             |module {  // BlockID = 8
-; CHECK-NEXT:      24:0|  3: <1, 1>                  |  version 1;
-; CHECK-NEXT:      26:4|  1: <65535, 0, 2>           |  abbreviations {  // BlockID = 0
-; CHECK-NEXT:      36:0|    3: <1, 14>               |    valuesymtab:
-; CHECK-NEXT:      38:4|    2: <65533, 4, 0, 1, 3, 0,|      @a0 = abbrev <fixed(3), vbr(8), 
-; CHECK-NEXT:          |        2, 8, 0, 3, 0, 1, 8> |                   array(fixed(8))>;
-; CHECK-NEXT:      43:2|    2: <65533, 4, 1, 1, 0, 2,|      @a1 = abbrev <1, vbr(8), 
-; CHECK-NEXT:          |        8, 0, 3, 0, 1, 7>    |                   array(fixed(7))>;
-; CHECK-NEXT:      48:0|    2: <65533, 4, 1, 1, 0, 2,|      @a2 = abbrev <1, vbr(8), 
-; CHECK-NEXT:          |        8, 0, 3, 0, 4>       |                   array(char6)>;
-; CHECK-NEXT:      52:1|    2: <65533, 4, 1, 2, 0, 2,|      @a3 = abbrev <2, vbr(8), 
-; CHECK-NEXT:          |        8, 0, 3, 0, 4>       |                   array(char6)>;
-; CHECK-NEXT:      56:2|    3: <1, 11>               |    constants:
-; CHECK-NEXT:      58:6|    2: <65533, 2, 1, 1, 0, 1,|      @a0 = abbrev <1, fixed(2)>;
-; CHECK-NEXT:          |        2>                   |
-; CHECK-NEXT:      61:7|    2: <65533, 2, 1, 4, 0, 2,|      @a1 = abbrev <4, vbr(8)>;
-; CHECK-NEXT:          |        8>                   |
-; CHECK-NEXT:      65:0|    2: <65533, 2, 1, 4, 1, 0>|      @a2 = abbrev <4, 0>;
-; CHECK-NEXT:      68:1|    2: <65533, 2, 1, 6, 0, 2,|      @a3 = abbrev <6, vbr(8)>;
-; CHECK-NEXT:          |        8>                   |
-; CHECK-NEXT:      71:2|    3: <1, 12>               |    function:
-; CHECK-NEXT:      73:6|    2: <65533, 4, 1, 20, 0,  |      @a0 = abbrev <20, vbr(6), vbr(4),
-; CHECK-NEXT:          |        2, 6, 0, 2, 4, 0, 2, |                   vbr(4)>;
-; CHECK-NEXT:          |        4>                   |
-; CHECK-NEXT:      79:1|    2: <65533, 4, 1, 2, 0, 2,|      @a1 = abbrev <2, vbr(6), vbr(6), 
-; CHECK-NEXT:          |        6, 0, 2, 6, 0, 1, 4> |                   fixed(4)>;
-; CHECK-NEXT:      84:4|    2: <65533, 4, 1, 3, 0, 2,|      @a2 = abbrev <3, vbr(6), 
-; CHECK-NEXT:          |        6, 0, 1, 2, 0, 1, 4> |                   fixed(2), fixed(4)>;
-; CHECK-NEXT:      89:7|    2: <65533, 1, 1, 10>     |      @a3 = abbrev <10>;
-; CHECK-NEXT:      91:7|    2: <65533, 2, 1, 10, 0,  |      @a4 = abbrev <10, vbr(6)>;
-; CHECK-NEXT:          |        2, 6>                |
-; CHECK-NEXT:      95:0|    2: <65533, 1, 1, 15>     |      @a5 = abbrev <15>;
-; CHECK-NEXT:      97:0|    2: <65533, 3, 1, 43, 0,  |      @a6 = abbrev <43, vbr(6), 
-; CHECK-NEXT:          |        2, 6, 0, 1, 2>       |                   fixed(2)>;
-; CHECK-NEXT:     101:2|    2: <65533, 4, 1, 24, 0,  |      @a7 = abbrev <24, vbr(6), vbr(6),
-; CHECK-NEXT:          |        2, 6, 0, 2, 6, 0, 2, |                   vbr(4)>;
-; CHECK-NEXT:          |        4>                   |
-; CHECK-NEXT:     106:5|    3: <1, 19>               |    globals:
-; CHECK-NEXT:     109:1|    2: <65533, 3, 1, 0, 0, 2,|      @a0 = abbrev <0, vbr(6), 
-; CHECK-NEXT:          |        6, 0, 1, 1>          |                   fixed(1)>;
-; CHECK-NEXT:     113:3|    2: <65533, 2, 1, 1, 0, 2,|      @a1 = abbrev <1, vbr(8)>;
-; CHECK-NEXT:          |        8>                   |
-; CHECK-NEXT:     116:4|    2: <65533, 2, 1, 2, 0, 2,|      @a2 = abbrev <2, vbr(8)>;
-; CHECK-NEXT:          |        8>                   |
-; CHECK-NEXT:     119:5|    2: <65533, 3, 1, 3, 0, 3,|      @a3 = abbrev <3, array(fixed(8))>
-; CHECK-NEXT:          |        0, 1, 8>             |          ;
-; CHECK-NEXT:     123:2|    2: <65533, 2, 1, 4, 0, 2,|      @a4 = abbrev <4, vbr(6)>;
-; CHECK-NEXT:          |        6>                   |
-; CHECK-NEXT:     126:3|    2: <65533, 3, 1, 4, 0, 2,|      @a5 = abbrev <4, vbr(6), vbr(6)>;
-; CHECK-NEXT:          |        6, 0, 2, 6>          |
-; CHECK-NEXT:     130:5|  0: <65534>                 |  }
-; CHECK-NEXT:     132:0|  1: <65535, 17, 3>          |  types {  // BlockID = 17
-; CHECK-NEXT:     140:0|    2: <65533, 4, 1, 21, 0,  |    %a0 = abbrev <21, fixed(1), 
-; CHECK-NEXT:          |        1, 1, 0, 3, 0, 1, 2> |                  array(fixed(2))>;
-; CHECK-NEXT:     144:7|    3: <1, 2>                |    count 2;
-; CHECK-NEXT:     147:4|    3: <2>                   |    @t0 = void;
-; CHECK-NEXT:     149:3|    4: <21, 0, 0>            |    @t1 = void (); <%a0>
-; CHECK-NEXT:     150:7|  0: <65534>                 |  }
-; CHECK-NEXT:     152:0|  3: <8, 1, 0, 1, 0>         |  declare external void @f0();
-; CHECK-NEXT:     156:6|  1: <65535, 19, 4>          |  globals {  // BlockID = 19
-; CHECK-NEXT:     164:0|    3: <5, 12>               |    count 12;
-; CHECK-NEXT:     166:6|    4: <0, 3, 0>             |    var @g0, align 4, <@a0>
-; CHECK-NEXT:     168:1|    6: <2, 4>                |      zerofill 4; <@a2>
-; CHECK-NEXT:     169:5|    4: <0, 0, 0>             |    var @g1, align 0, <@a0>
-; CHECK-NEXT:     171:0|    7: <3, 97, 98, 99, 100,  |      { 97,  98,  99, 100, 101, 102, 
-; CHECK-NEXT:          |        101, 102, 103>       |       103} <@a3>
-; CHECK-NEXT:     179:2|    4: <0, 0, 0>             |    var @g2, align 0, <@a0>
-; CHECK-NEXT:     180:5|    8: <4, 6>                |      reloc @g5; <@a4>
-; CHECK-NEXT:     181:7|    4: <0, 0, 0>             |    var @g3, align 0, <@a0>
-; CHECK-NEXT:     183:2|    8: <4, 0>                |      reloc @f0; <@a4>
-; CHECK-NEXT:     184:4|    4: <0, 0, 0>             |    var @g4, align 0, <@a0>
-; CHECK-NEXT:     185:7|    5: <1, 2>                |      initializers 2 { <@a1>
-; CHECK-NEXT:     187:3|    7: <3, 102, 111, 111>    |        {102, 111, 111} <@a3>
-; CHECK-NEXT:     191:5|    8: <4, 0>                |        reloc @f0; <@a4>
-; CHECK-NEXT:          |                             |      }
-; CHECK-NEXT:     192:7|    4: <0, 0, 0>             |    var @g5, align 0, <@a0>
-; CHECK-NEXT:     194:2|    8: <4, 2>                |      reloc @g1; <@a4>
-; CHECK-NEXT:     195:4|    4: <0, 0, 0>             |    var @g6, align 0, <@a0>
-; CHECK-NEXT:     196:7|    9: <4, 6, 1>             |      reloc @g5 + 1; <@a5>
-; CHECK-NEXT:     198:7|    4: <0, 0, 0>             |    var @g7, align 0, <@a0>
-; CHECK-NEXT:     200:2|    9: <4, 2, 1>             |      reloc @g1 + 1; <@a5>
-; CHECK-NEXT:     202:2|    4: <0, 0, 0>             |    var @g8, align 0, <@a0>
-; CHECK-NEXT:     203:5|    9: <4, 2, 7>             |      reloc @g1 + 7; <@a5>
-; CHECK-NEXT:     205:5|    4: <0, 0, 0>             |    var @g9, align 0, <@a0>
-; CHECK-NEXT:     207:0|    9: <4, 2, 9>             |      reloc @g1 + 9; <@a5>
-; CHECK-NEXT:     209:0|    4: <0, 0, 0>             |    var @g10, align 0, <@a0>
-; CHECK-NEXT:     210:3|    9: <4, 5, 1>             |      reloc @g4 + 1; <@a5>
-; CHECK-NEXT:     212:3|    4: <0, 0, 0>             |    var @g11, align 0, <@a0>
-; CHECK-NEXT:     213:6|    9: <4, 5, 4>             |      reloc @g4 + 4; <@a5>
-; CHECK-NEXT:     215:6|  0: <65534>                 |  }
-; CHECK-NEXT:     220:0|  1: <65535, 14, 3>          |  valuesymtab {  // BlockID = 14
-; CHECK-NEXT:     228:0|    6: <1, 1, 122, 101, 114, |    @g0 : "zero"; <@a2>
-; CHECK-NEXT:          |        111>                 |
-; CHECK-NEXT:     233:1|    6: <1, 4, 112, 116, 114, |    @g3 : "ptr_to_func"; <@a2>
-; CHECK-NEXT:          |        95, 116, 111, 95,    |
-; CHECK-NEXT:          |        102, 117, 110, 99>   |
-; CHECK-NEXT:     243:4|    6: <1, 5, 99, 111, 109,  |    @g4 : "compound"; <@a2>
-; CHECK-NEXT:          |        112, 111, 117, 110,  |
-; CHECK-NEXT:          |        100>                 |
-; CHECK-NEXT:     251:5|    6: <1, 2, 98, 121, 116,  |    @g1 : "bytes"; <@a2>
-; CHECK-NEXT:          |        101, 115>            |
-; CHECK-NEXT:     257:4|    6: <1, 0, 102, 117, 110, |    @f0 : "func"; <@a2>
-; CHECK-NEXT:          |        99>                  |
-; CHECK-NEXT:     262:5|    6: <1, 3, 112, 116, 114, |    @g2 : "ptr_to_ptr"; <@a2>
-; CHECK-NEXT:          |        95, 116, 111, 95,    |
-; CHECK-NEXT:          |        112, 116, 114>       |
-; CHECK-NEXT:     272:2|    6: <1, 8, 97, 100, 100,  |    @g7 : "addend_array1"; <@a2>
-; CHECK-NEXT:          |        101, 110, 100, 95,   |
-; CHECK-NEXT:          |        97, 114, 114, 97,    |
-; CHECK-NEXT:          |        121, 49>             |
-; CHECK-NEXT:     284:1|    6: <1, 9, 97, 100, 100,  |    @g8 : "addend_array2"; <@a2>
-; CHECK-NEXT:          |        101, 110, 100, 95,   |
-; CHECK-NEXT:          |        97, 114, 114, 97,    |
-; CHECK-NEXT:          |        121, 50>             |
-; CHECK-NEXT:     296:0|    6: <1, 10, 97, 100, 100, |    @g9 : "addend_array3"; <@a2>
-; CHECK-NEXT:          |        101, 110, 100, 95,   |
-; CHECK-NEXT:          |        97, 114, 114, 97,    |
-; CHECK-NEXT:          |        121, 51>             |
-; CHECK-NEXT:     307:7|    6: <1, 7, 97, 100, 100,  |    @g6 : "addend_ptr"; <@a2>
-; CHECK-NEXT:          |        101, 110, 100, 95,   |
-; CHECK-NEXT:          |        112, 116, 114>       |
-; CHECK-NEXT:     317:4|    6: <1, 11, 97, 100, 100, |    @g10 : "addend_struct1"; <@a2>
-; CHECK-NEXT:          |        101, 110, 100, 95,   |
-; CHECK-NEXT:          |        115, 116, 114, 117,  |
-; CHECK-NEXT:          |        99, 116, 49>         |
-; CHECK-NEXT:     330:1|    6: <1, 6, 112, 116, 114> |    @g5 : "ptr"; <@a2>
-; CHECK-NEXT:     334:4|    6: <1, 12, 97, 100, 100, |    @g11 : "addend_struct2"; <@a2>
-; CHECK-NEXT:          |        101, 110, 100, 95,   |
-; CHECK-NEXT:          |        115, 116, 114, 117,  |
-; CHECK-NEXT:          |        99, 116, 50>         |
-; CHECK-NEXT:     347:1|  0: <65534>                 |  }
-; CHECK-NEXT:     348:0|0: <65534>                   |}
diff --git a/test/NaCl/Bitcode/pnacl-bcdis/icmp.ll b/test/NaCl/Bitcode/pnacl-bcdis/icmp.ll
deleted file mode 100644
index d89fe65f395..00000000000
--- a/test/NaCl/Bitcode/pnacl-bcdis/icmp.ll
+++ /dev/null
@@ -1,318 +0,0 @@
-; Tests the icmp instruction.
-
-; RUN: llvm-as < %s | pnacl-freeze | pnacl-bccompress --remove-abbreviations \
-; RUN:              | pnacl-bcdis | FileCheck %s
-
-; Test icmp on primitive types.
-define void @SimpleIcmpOps() {
-
-; CHECK:         {{.*}}|    1: <65535, 11, 2>        |    constants {  // BlockID = 11
-; CHECK-NEXT:    {{.*}}|      3: <1, 0>              |      i1:
-; CHECK-NEXT:    {{.*}}|      3: <4, 0>              |        %c0 = i1 0;
-; CHECK-NEXT:    {{.*}}|      3: <1, 4>              |      i8:
-; CHECK-NEXT:    {{.*}}|      3: <4, 2>              |        %c1 = i8 1;
-; CHECK-NEXT:    {{.*}}|      3: <1, 6>              |      i16:
-; CHECK-NEXT:    {{.*}}|      3: <4, 4>              |        %c2 = i16 2;
-; CHECK-NEXT:    {{.*}}|      3: <1, 8>              |      i32:
-; CHECK-NEXT:    {{.*}}|      3: <4, 6>              |        %c3 = i32 3;
-; CHECK-NEXT:    {{.*}}|      3: <1, 10>              |      i64:
-; CHECK-NEXT:    {{.*}}|      3: <4, 8>              |        %c4 = i64 4;
-; CHECK-NEXT:    {{.*}}|    0: <65534>               |      }
-; CHECK-NEXT:          |                             |  %b0:
-
-  %v0 = icmp eq i1 0, 0
-  %v1 = icmp eq i8 1, 1
-  %v2 = icmp eq i16 2, 2
-  %v3 = icmp eq i32 3, 3
-  %v4 = icmp eq i64 4, 4
-
-; CHECK-NEXT:    {{.*}}|    3: <28, 5, 5, 32>       |    %v0 = icmp eq i1 %c0, %c0;
-; CHECK-NEXT:    {{.*}}|    3: <28, 5, 5, 32>       |    %v1 = icmp eq i8 %c1, %c1;
-; CHECK-NEXT:    {{.*}}|    3: <28, 5, 5, 32>       |    %v2 = icmp eq i16 %c2, %c2;
-; CHECK-NEXT:    {{.*}}|    3: <28, 5, 5, 32>       |    %v3 = icmp eq i32 %c3, %c3;
-; CHECK-NEXT:    {{.*}}|    3: <28, 5, 5, 32>       |    %v4 = icmp eq i64 %c4, %c4;
-
-  %v5 = icmp ne i1 0, 0
-  %v6 = icmp ne i8 1, 1
-  %v7 = icmp ne i16 2, 2
-  %v8 = icmp ne i32 3, 3
-  %v9 = icmp ne i64 4, 4
-
-; CHECK-NEXT:    {{.*}}|    3: <28, 10, 10, 33>      |    %v5 = icmp ne i1 %c0, %c0;
-; CHECK-NEXT:    {{.*}}|    3: <28, 10, 10, 33>      |    %v6 = icmp ne i8 %c1, %c1;
-; CHECK-NEXT:    {{.*}}|    3: <28, 10, 10, 33>      |    %v7 = icmp ne i16 %c2, %c2;
-; CHECK-NEXT:    {{.*}}|    3: <28, 10, 10, 33>      |    %v8 = icmp ne i32 %c3, %c3;
-; CHECK-NEXT:    {{.*}}|    3: <28, 10, 10, 33>      |    %v9 = icmp ne i64 %c4, %c4;
-
-  %v10 = icmp ugt i1 0, 0
-  %v11 = icmp ugt i8 1, 1
-  %v12 = icmp ugt i16 2, 2
-  %v13 = icmp ugt i32 3, 3
-  %v14 = icmp ugt i64 4, 4
-
-; CHECK-NEXT:    {{.*}}|    3: <28, 15, 15, 34>      |    %v10 = icmp ugt i1 %c0, %c0;
-; CHECK-NEXT:    {{.*}}|    3: <28, 15, 15, 34>      |    %v11 = icmp ugt i8 %c1, %c1;
-; CHECK-NEXT:    {{.*}}|    3: <28, 15, 15, 34>      |    %v12 = icmp ugt i16 %c2, %c2;
-; CHECK-NEXT:    {{.*}}|    3: <28, 15, 15, 34>      |    %v13 = icmp ugt i32 %c3, %c3;
-; CHECK-NEXT:    {{.*}}|    3: <28, 15, 15, 34>      |    %v14 = icmp ugt i64 %c4, %c4;
-
-  %v15 = icmp uge i1 0, 0
-  %v16 = icmp uge i8 1, 1
-  %v17 = icmp uge i16 2, 2
-  %v18 = icmp uge i32 3, 3
-  %v19 = icmp uge i64 4, 4
-
-; CHECK-NEXT:    {{.*}}|    3: <28, 20, 20, 35>      |    %v15 = icmp uge i1 %c0, %c0;
-; CHECK-NEXT:    {{.*}}|    3: <28, 20, 20, 35>      |    %v16 = icmp uge i8 %c1, %c1;
-; CHECK-NEXT:    {{.*}}|    3: <28, 20, 20, 35>      |    %v17 = icmp uge i16 %c2, %c2;
-; CHECK-NEXT:    {{.*}}|    3: <28, 20, 20, 35>      |    %v18 = icmp uge i32 %c3, %c3;
-; CHECK-NEXT:    {{.*}}|    3: <28, 20, 20, 35>      |    %v19 = icmp uge i64 %c4, %c4;
-
-  %v20 = icmp ult i1 0, 0
-  %v21 = icmp ult i8 1, 1
-  %v22 = icmp ult i16 2, 2
-  %v23 = icmp ult i32 3, 3
-  %v24 = icmp ult i64 4, 4
-
-; CHECK-NEXT:    {{.*}}|    3: <28, 25, 25, 36>      |    %v20 = icmp ult i1 %c0, %c0;
-; CHECK-NEXT:    {{.*}}|    3: <28, 25, 25, 36>      |    %v21 = icmp ult i8 %c1, %c1;
-; CHECK-NEXT:    {{.*}}|    3: <28, 25, 25, 36>      |    %v22 = icmp ult i16 %c2, %c2;
-; CHECK-NEXT:    {{.*}}|    3: <28, 25, 25, 36>      |    %v23 = icmp ult i32 %c3, %c3;
-; CHECK-NEXT:    {{.*}}|    3: <28, 25, 25, 36>      |    %v24 = icmp ult i64 %c4, %c4;
-
-  %v25 = icmp ule i1 0, 0
-  %v26 = icmp ule i8 1, 1
-  %v27 = icmp ule i16 2, 2
-  %v28 = icmp ule i32 3, 3
-  %v29 = icmp ule i64 4, 4
-
-; CHECK-NEXT:    {{.*}}|    3: <28, 30, 30, 37>      |    %v25 = icmp ule i1 %c0, %c0;
-; CHECK-NEXT:    {{.*}}|    3: <28, 30, 30, 37>      |    %v26 = icmp ule i8 %c1, %c1;
-; CHECK-NEXT:    {{.*}}|    3: <28, 30, 30, 37>      |    %v27 = icmp ule i16 %c2, %c2;
-; CHECK-NEXT:    {{.*}}|    3: <28, 30, 30, 37>      |    %v28 = icmp ule i32 %c3, %c3;
-; CHECK-NEXT:    {{.*}}|    3: <28, 30, 30, 37>      |    %v29 = icmp ule i64 %c4, %c4;
-
-  %v30 = icmp sgt i1 0, 0
-  %v31 = icmp sgt i8 1, 1
-  %v32 = icmp sgt i16 2, 2
-  %v33 = icmp sgt i32 3, 3
-  %v34 = icmp sgt i64 4, 4
-
-; CHECK-NEXT:    {{.*}}|    3: <28, 35, 35, 38>      |    %v30 = icmp sgt i1 %c0, %c0;
-; CHECK-NEXT:    {{.*}}|    3: <28, 35, 35, 38>      |    %v31 = icmp sgt i8 %c1, %c1;
-; CHECK-NEXT:    {{.*}}|    3: <28, 35, 35, 38>      |    %v32 = icmp sgt i16 %c2, %c2;
-; CHECK-NEXT:    {{.*}}|    3: <28, 35, 35, 38>      |    %v33 = icmp sgt i32 %c3, %c3;
-; CHECK-NEXT:    {{.*}}|    3: <28, 35, 35, 38>      |    %v34 = icmp sgt i64 %c4, %c4;
-
-  %v35 = icmp sge i1 0, 0
-  %v36 = icmp sge i8 1, 1
-  %v37 = icmp sge i16 2, 2
-  %v38 = icmp sge i32 3, 3
-  %v39 = icmp sge i64 4, 4
-
-; CHECK-NEXT:    {{.*}}|    3: <28, 40, 40, 39>      |    %v35 = icmp sge i1 %c0, %c0;
-; CHECK-NEXT:    {{.*}}|    3: <28, 40, 40, 39>      |    %v36 = icmp sge i8 %c1, %c1;
-; CHECK-NEXT:    {{.*}}|    3: <28, 40, 40, 39>      |    %v37 = icmp sge i16 %c2, %c2;
-; CHECK-NEXT:    {{.*}}|    3: <28, 40, 40, 39>      |    %v38 = icmp sge i32 %c3, %c3;
-; CHECK-NEXT:    {{.*}}|    3: <28, 40, 40, 39>      |    %v39 = icmp sge i64 %c4, %c4;
-
-  %v40 = icmp slt i1 0, 0
-  %v41 = icmp slt i8 1, 1
-  %v42 = icmp slt i16 2, 2
-  %v43 = icmp slt i32 3, 3
-  %v44 = icmp slt i64 4, 4
-
-; CHECK-NEXT:    {{.*}}|    3: <28, 45, 45, 40>      |    %v40 = icmp slt i1 %c0, %c0;
-; CHECK-NEXT:    {{.*}}|    3: <28, 45, 45, 40>      |    %v41 = icmp slt i8 %c1, %c1;
-; CHECK-NEXT:    {{.*}}|    3: <28, 45, 45, 40>      |    %v42 = icmp slt i16 %c2, %c2;
-; CHECK-NEXT:    {{.*}}|    3: <28, 45, 45, 40>      |    %v43 = icmp slt i32 %c3, %c3;
-; CHECK-NEXT:    {{.*}}|    3: <28, 45, 45, 40>      |    %v44 = icmp slt i64 %c4, %c4;
-
-  %v45 = icmp sle i1 0, 0
-  %v46 = icmp sle i8 1, 1
-  %v47 = icmp sle i16 2, 2
-  %v48 = icmp sle i32 3, 3
-  %v49 = icmp sle i64 4, 4
-
-; CHECK-NEXT:    {{.*}}|    3: <28, 50, 50, 41>      |    %v45 = icmp sle i1 %c0, %c0;
-; CHECK-NEXT:    {{.*}}|    3: <28, 50, 50, 41>      |    %v46 = icmp sle i8 %c1, %c1;
-; CHECK-NEXT:    {{.*}}|    3: <28, 50, 50, 41>      |    %v47 = icmp sle i16 %c2, %c2;
-; CHECK-NEXT:    {{.*}}|    3: <28, 50, 50, 41>      |    %v48 = icmp sle i32 %c3, %c3;
-; CHECK-NEXT:    {{.*}}|    3: <28, 50, 50, 41>      |    %v49 = icmp sle i64 %c4, %c4;
-
-  ; Verifies result is i1.
-  %v50 = and i1 %v0, %v1
-  %v51 = and i1 %v2, %v3
-  %v52 = and i1 %v4, %v5
-  %v53 = and i1 %v6, %v7
-  %v54 = and i1 %v8, %v9
-  %v55 = and i1 %v10, %v11
-  %v56 = and i1 %v12, %v13
-  %v57 = and i1 %v14, %v15
-  %v58 = and i1 %v16, %v17
-  %v59 = and i1 %v18, %v19
-  %v60 = and i1 %v20, %v21
-  %v61 = and i1 %v22, %v23
-  %v62 = and i1 %v24, %v25
-  %v63 = and i1 %v26, %v27
-  %v64 = and i1 %v28, %v29
-  %v65 = and i1 %v30, %v31
-  %v66 = and i1 %v32, %v33
-  %v67 = and i1 %v34, %v35
-  %v68 = and i1 %v36, %v37
-  %v69 = and i1 %v38, %v39
-  ret void
-
-; CHECK-NEXT:    {{.*}}|    3: <2, 50, 49, 10>       |    %v50 = and i1 %v0, %v1;
-; CHECK-NEXT:    {{.*}}|    3: <2, 49, 48, 10>       |    %v51 = and i1 %v2, %v3;
-; CHECK-NEXT:    {{.*}}|    3: <2, 48, 47, 10>       |    %v52 = and i1 %v4, %v5;
-; CHECK-NEXT:    {{.*}}|    3: <2, 47, 46, 10>       |    %v53 = and i1 %v6, %v7;
-; CHECK-NEXT:    {{.*}}|    3: <2, 46, 45, 10>       |    %v54 = and i1 %v8, %v9;
-; CHECK-NEXT:    {{.*}}|    3: <2, 45, 44, 10>       |    %v55 = and i1 %v10, %v11;
-; CHECK-NEXT:    {{.*}}|    3: <2, 44, 43, 10>       |    %v56 = and i1 %v12, %v13;
-; CHECK-NEXT:    {{.*}}|    3: <2, 43, 42, 10>       |    %v57 = and i1 %v14, %v15;
-; CHECK-NEXT:    {{.*}}|    3: <2, 42, 41, 10>       |    %v58 = and i1 %v16, %v17;
-; CHECK-NEXT:    {{.*}}|    3: <2, 41, 40, 10>       |    %v59 = and i1 %v18, %v19;
-; CHECK-NEXT:    {{.*}}|    3: <2, 40, 39, 10>       |    %v60 = and i1 %v20, %v21;
-; CHECK-NEXT:    {{.*}}|    3: <2, 39, 38, 10>       |    %v61 = and i1 %v22, %v23;
-; CHECK-NEXT:    {{.*}}|    3: <2, 38, 37, 10>       |    %v62 = and i1 %v24, %v25;
-; CHECK-NEXT:    {{.*}}|    3: <2, 37, 36, 10>       |    %v63 = and i1 %v26, %v27;
-; CHECK-NEXT:    {{.*}}|    3: <2, 36, 35, 10>       |    %v64 = and i1 %v28, %v29;
-; CHECK-NEXT:    {{.*}}|    3: <2, 35, 34, 10>       |    %v65 = and i1 %v30, %v31;
-; CHECK-NEXT:    {{.*}}|    3: <2, 34, 33, 10>       |    %v66 = and i1 %v32, %v33;
-; CHECK-NEXT:    {{.*}}|    3: <2, 33, 32, 10>       |    %v67 = and i1 %v34, %v35;
-; CHECK-NEXT:    {{.*}}|    3: <2, 32, 31, 10>       |    %v68 = and i1 %v36, %v37;
-; CHECK-NEXT:    {{.*}}|    3: <2, 31, 30, 10>       |    %v69 = and i1 %v38, %v39;
-; CHECK-NEXT:    {{.*}}|    3: <10>                  |    ret void;
-
-}
-
-
-; Tests integer vector compares.
-define void @VectorIcmpOps(<16 x i8> %p0, <8 x i16> %p1, <4 x i32> %p2,
-                           <16 x i1> %p3, <8 x i1> %p4, <4 x i1> %p5) {
-
-; CHECK:         {{.*}}|    3: <1, 1>                |    blocks 1;
-; CHECK-NEXT:          |                             |  %b0:
-
-  %v0 = icmp eq <16 x i8> %p0, %p0
-  %v1 = icmp eq <8 x i16> %p1, %p1
-  %v2 = icmp eq <4 x i32> %p2, %p2
-
-; CHECK-NEXT:    {{.*}}|    3: <28, 6, 6, 32>        |    %v0 = icmp eq <16 x i8> %p0, %p0;
-; CHECK-NEXT:    {{.*}}|    3: <28, 6, 6, 32>        |    %v1 = icmp eq <8 x i16> %p1, %p1;
-; CHECK-NEXT:    {{.*}}|    3: <28, 6, 6, 32>        |    %v2 = icmp eq <4 x i32> %p2, %p2;
-
-  %v3 = icmp ne <16 x i8> %p0, %p0
-  %v4 = icmp ne <8 x i16> %p1, %p1
-  %v5 = icmp ne <4 x i32> %p2, %p2
-
-; CHECK-NEXT:    {{.*}}|    3: <28, 9, 9, 33>        |    %v3 = icmp ne <16 x i8> %p0, %p0;
-; CHECK-NEXT:    {{.*}}|    3: <28, 9, 9, 33>        |    %v4 = icmp ne <8 x i16> %p1, %p1;
-; CHECK-NEXT:    {{.*}}|    3: <28, 9, 9, 33>        |    %v5 = icmp ne <4 x i32> %p2, %p2;
-
-  %v6 = icmp ugt <16 x i8> %p0, %p0
-  %v7 = icmp ugt <8 x i16> %p1, %p1
-  %v8 = icmp ugt <4 x i32> %p2, %p2
-
-; CHECK-NEXT:    {{.*}}|    3: <28, 12, 12, 34>      |    %v6 = icmp ugt <16 x i8> %p0, %p0;
-; CHECK-NEXT:    {{.*}}|    3: <28, 12, 12, 34>      |    %v7 = icmp ugt <8 x i16> %p1, %p1;
-; CHECK-NEXT:    {{.*}}|    3: <28, 12, 12, 34>      |    %v8 = icmp ugt <4 x i32> %p2, %p2;
-
-  %v9 = icmp uge <16 x i8> %p0, %p0
-  %v10 = icmp uge <8 x i16> %p1, %p1
-  %v11 = icmp uge <4 x i32> %p2, %p2
-
-; CHECK-NEXT:    {{.*}}|    3: <28, 15, 15, 35>      |    %v9 = icmp uge <16 x i8> %p0, %p0;
-; CHECK-NEXT:    {{.*}}|    3: <28, 15, 15, 35>      |    %v10 = icmp uge <8 x i16> %p1, %p1;
-; CHECK-NEXT:    {{.*}}|    3: <28, 15, 15, 35>      |    %v11 = icmp uge <4 x i32> %p2, %p2;
-
-  %v12 = icmp ult <16 x i8> %p0, %p0
-  %v13 = icmp ult <8 x i16> %p1, %p1
-  %v14 = icmp ult <4 x i32> %p2, %p2
-
-; CHECK-NEXT:    {{.*}}|    3: <28, 18, 18, 36>      |    %v12 = icmp ult <16 x i8> %p0, %p0;
-; CHECK-NEXT:    {{.*}}|    3: <28, 18, 18, 36>      |    %v13 = icmp ult <8 x i16> %p1, %p1;
-; CHECK-NEXT:    {{.*}}|    3: <28, 18, 18, 36>      |    %v14 = icmp ult <4 x i32> %p2, %p2;
-
-  %v15 = icmp ule <16 x i8> %p0, %p0
-  %v16 = icmp ule <8 x i16> %p1, %p1
-  %v17 = icmp ule <4 x i32> %p2, %p2
-
-; CHECK-NEXT:    {{.*}}|    3: <28, 21, 21, 37>      |    %v15 = icmp ule <16 x i8> %p0, %p0;
-; CHECK-NEXT:    {{.*}}|    3: <28, 21, 21, 37>      |    %v16 = icmp ule <8 x i16> %p1, %p1;
-; CHECK-NEXT:    {{.*}}|    3: <28, 21, 21, 37>      |    %v17 = icmp ule <4 x i32> %p2, %p2;
-
-  %v18 = icmp sgt <16 x i8> %p0, %p0
-  %v19 = icmp sgt <8 x i16> %p1, %p1
-  %v20 = icmp sgt <4 x i32> %p2, %p2
-
-; CHECK-NEXT:    {{.*}}|    3: <28, 24, 24, 38>      |    %v18 = icmp sgt <16 x i8> %p0, %p0;
-; CHECK-NEXT:    {{.*}}|    3: <28, 24, 24, 38>      |    %v19 = icmp sgt <8 x i16> %p1, %p1;
-; CHECK-NEXT:    {{.*}}|    3: <28, 24, 24, 38>      |    %v20 = icmp sgt <4 x i32> %p2, %p2;
-
-  %v21 = icmp sge <16 x i8> %p0, %p0
-  %v22 = icmp sge <8 x i16> %p1, %p1
-  %v23 = icmp sge <4 x i32> %p2, %p2
-
-; CHECK-NEXT:    {{.*}}|    3: <28, 27, 27, 39>      |    %v21 = icmp sge <16 x i8> %p0, %p0;
-; CHECK-NEXT:    {{.*}}|    3: <28, 27, 27, 39>      |    %v22 = icmp sge <8 x i16> %p1, %p1;
-; CHECK-NEXT:    {{.*}}|    3: <28, 27, 27, 39>      |    %v23 = icmp sge <4 x i32> %p2, %p2;
-
-  %v24 = icmp slt <16 x i8> %p0, %p0
-  %v25 = icmp slt <8 x i16> %p1, %p1
-  %v26 = icmp slt <4 x i32> %p2, %p2
-
-; CHECK-NEXT:    {{.*}}|    3: <28, 30, 30, 40>      |    %v24 = icmp slt <16 x i8> %p0, %p0;
-; CHECK-NEXT:    {{.*}}|    3: <28, 30, 30, 40>      |    %v25 = icmp slt <8 x i16> %p1, %p1;
-; CHECK-NEXT:    {{.*}}|    3: <28, 30, 30, 40>      |    %v26 = icmp slt <4 x i32> %p2, %p2;
-
-  %v27 = icmp sle <16 x i8> %p0, %p0
-  %v28 = icmp sle <8 x i16> %p1, %p1
-  %v29 = icmp sle <4 x i32> %p2, %p2
-
-; CHECK-NEXT:    {{.*}}|    3: <28, 33, 33, 41>      |    %v27 = icmp sle <16 x i8> %p0, %p0;
-; CHECK-NEXT:    {{.*}}|    3: <28, 33, 33, 41>      |    %v28 = icmp sle <8 x i16> %p1, %p1;
-; CHECK-NEXT:    {{.*}}|    3: <28, 33, 33, 41>      |    %v29 = icmp sle <4 x i32> %p2, %p2;
-
-  ; Verify result types are vectors of right size.
-  %v30 = and <16 x i1> %v0, %v3
-  %v31 = and <16 x i1> %v6, %v9
-  %v32 = and <16 x i1> %v12, %v15
-  %v33 = and <16 x i1> %v18, %v21
-  %v34 = and <16 x i1> %v24, %v27
-
-; CHECK-NEXT:    {{.*}}|    3: <2, 30, 27, 10>       |    %v30 = and <16 x i1> %v0, %v3;
-; CHECK-NEXT:    {{.*}}|    3: <2, 25, 22, 10>       |    %v31 = and <16 x i1> %v6, %v9;
-; CHECK-NEXT:    {{.*}}|    3: <2, 20, 17, 10>       |    %v32 = and <16 x i1> %v12, %v15;
-; CHECK-NEXT:    {{.*}}|    3: <2, 15, 12, 10>       |    %v33 = and <16 x i1> %v18, %v21;
-; CHECK-NEXT:    {{.*}}|    3: <2, 10, 7, 10>        |    %v34 = and <16 x i1> %v24, %v27;
-
-  %v35 = and <8 x i1> %v1, %v4
-  %v36 = and <8 x i1> %v7, %v10
-  %v37 = and <8 x i1> %v13, %v16
-  %v38 = and <8 x i1> %v19, %v22
-  %v39 = and <8 x i1> %v25, %v28
-
-; CHECK-NEXT:    {{.*}}|    3: <2, 34, 31, 10>       |    %v35 = and <8 x i1> %v1, %v4;
-; CHECK-NEXT:    {{.*}}|    3: <2, 29, 26, 10>       |    %v36 = and <8 x i1> %v7, %v10;
-; CHECK-NEXT:    {{.*}}|    3: <2, 24, 21, 10>       |    %v37 = and <8 x i1> %v13, %v16;
-; CHECK-NEXT:    {{.*}}|    3: <2, 19, 16, 10>       |    %v38 = and <8 x i1> %v19, %v22;
-; CHECK-NEXT:    {{.*}}|    3: <2, 14, 11, 10>       |    %v39 = and <8 x i1> %v25, %v28;
-
-
-  %v40 = and <4 x i1> %v2, %v5
-  %v41 = and <4 x i1> %v8, %v11
-  %v42 = and <4 x i1> %v14, %v17
-  %v43 = and <4 x i1> %v20, %v23
-  %v44 = and <4 x i1> %v26, %v29
-  ret void
-
-; CHECK-NEXT:    {{.*}}|    3: <2, 38, 35, 10>       |    %v40 = and <4 x i1> %v2, %v5;
-; CHECK-NEXT:    {{.*}}|    3: <2, 33, 30, 10>       |    %v41 = and <4 x i1> %v8, %v11;
-; CHECK-NEXT:    {{.*}}|    3: <2, 28, 25, 10>       |    %v42 = and <4 x i1> %v14, %v17;
-; CHECK-NEXT:    {{.*}}|    3: <2, 23, 20, 10>       |    %v43 = and <4 x i1> %v20, %v23;
-; CHECK-NEXT:    {{.*}}|    3: <2, 18, 15, 10>       |    %v44 = and <4 x i1> %v26, %v29;
-; CHECK-NEXT:    {{.*}}|    3: <10>                  |    ret void;
-
-}
diff --git a/test/NaCl/Bitcode/pnacl-bcdis/intrinsics-bad-types.ll b/test/NaCl/Bitcode/pnacl-bcdis/intrinsics-bad-types.ll
deleted file mode 100644
index c3f1e88862b..00000000000
--- a/test/NaCl/Bitcode/pnacl-bcdis/intrinsics-bad-types.ll
+++ /dev/null
@@ -1,16 +0,0 @@
-; Tests calls to intrinsics with bad type signatures.
-
-; RUN: llvm-as < %s | pnacl-freeze | pnacl-bccompress --remove-abbreviations \
-; RUN:              | not pnacl-bcdis | FileCheck %s
-
-; Error where return type should be i32.
-declare void @llvm.nacl.setjmp(i8*)
-; CHECK: Error({{.*}}): Intrinsic llvm.nacl.setjmp expects return type i32. Found: void
-
-; Error where type of 2nd parameter is wrong (should be i32).
-declare i32 @llvm.nacl.atomic.load.i32(i32*, i64)
-; CHECK: Error({{.*}}): Intrinsic llvm.nacl.atomic.load.i32 expects i32 for argument 2. Found: i64
-
-; Error where too many arguments are specified.
-declare void @llvm.nacl.atomic.store.i64(i64, i64*, i32, i32)
-; CHECK: Error({{.*}}): Intrinsic llvm.nacl.atomic.store.i64 expects 3 arguments. Found: 4
diff --git a/test/NaCl/Bitcode/pnacl-bcdis/intrinsics.ll b/test/NaCl/Bitcode/pnacl-bcdis/intrinsics.ll
deleted file mode 100644
index 46fafc64c25..00000000000
--- a/test/NaCl/Bitcode/pnacl-bcdis/intrinsics.ll
+++ /dev/null
@@ -1,25 +0,0 @@
-; Tests calls to intrinsics.
-
-; RUN: llvm-as < %s | pnacl-freeze | pnacl-bccompress --remove-abbreviations \
-; RUN:              | pnacl-bcdis | FileCheck %s
-
-; Test simple intrinsic call.
-declare i8* @llvm.nacl.read.tp()
-
-define i32 @ReturnPtrIntrinsic() {
-  %1 = call i8* @llvm.nacl.read.tp()
-  %2 = ptrtoint i8* %1 to i32
-  ret i32 %2
-}
-
-; Test intrinsic call with i1 argument.
-declare external i32 @llvm.ctlz.i32(i32, i1)
-
-define i32 @f() {
-  %v0 = call i32 @llvm.ctlz.i32(i32 0, i1 0)
-  ret i32 %v0
-}
-
-; CHECK-NOT: Error
-
-
diff --git a/test/NaCl/Bitcode/pnacl-bcdis/invalid.test b/test/NaCl/Bitcode/pnacl-bcdis/invalid.test
deleted file mode 100644
index efb49a7e39d..00000000000
--- a/test/NaCl/Bitcode/pnacl-bcdis/invalid.test
+++ /dev/null
@@ -1,5 +0,0 @@
-RUN: pnacl-bcfuzz -bitcode-as-text %p/Inputs/bad-zext-cast.tbc -output - \
-RUN:   | not pnacl-bcdis | FileCheck --check-prefix=BAD-ZEXT-CAST %s
-
-BAD-ZEXT-CAST:      %v1 = zext i32 %p0 to i32 (i32);
-BAD-ZEXT-CAST-NEXT: Invalid cast 'zext'. Not defined on i32 to i32 (i32)
diff --git a/test/NaCl/Bitcode/pnacl-bcdis/lit.local.cfg b/test/NaCl/Bitcode/pnacl-bcdis/lit.local.cfg
deleted file mode 100644
index e531cacd4d8..00000000000
--- a/test/NaCl/Bitcode/pnacl-bcdis/lit.local.cfg
+++ /dev/null
@@ -1 +0,0 @@
-config.suffixes = ['.ll', '.test']
diff --git a/test/NaCl/Bitcode/pnacl-bcdis/load.ll b/test/NaCl/Bitcode/pnacl-bcdis/load.ll
deleted file mode 100644
index ecf762b7122..00000000000
--- a/test/NaCl/Bitcode/pnacl-bcdis/load.ll
+++ /dev/null
@@ -1,191 +0,0 @@
-; Test the load instruction.
-
-; RUN: llvm-as < %s | pnacl-freeze | pnacl-bccompress --remove-abbreviations \
-; RUN:              | not pnacl-bcdis | FileCheck %s
-
-; Test valid loads.
-define void @GoodTests(i32 %p0) {
-; CHECK:               |                             |  %b0:
-
-  %ai8 = inttoptr i32 %p0 to i8*
-  %v0 = load i8, i8* %ai8, align 1
-
-; CHECK-NEXT:    {{.*}}|    3: <20, 1, 1, 3>         |    %v0 = load i8* %p0, align 1;
-
-  %ai16 = inttoptr i32 %p0 to i16*
-  %v1 = load i16, i16* %ai16, align 1
-
-; CHECK-NEXT:    {{.*}}|    3: <20, 2, 1, 4>         |    %v1 = load i16* %p0, align 1;
-
-  %ai32 = inttoptr i32 %p0 to i32*
-  %v2 = load i32, i32* %ai32, align 1
-
-; CHECK-NEXT:    {{.*}}|    3: <20, 3, 1, 0>         |    %v2 = load i32* %p0, align 1;
-
-  %ai64 = inttoptr i32 %p0 to i64*
-  %v3 = load i64, i64* %ai64, align 1
-
-; CHECK-NEXT:    {{.*}}|    3: <20, 4, 1, 5>         |    %v3 = load i64* %p0, align 1;
-
-  %af = inttoptr i32 %p0 to float*
-  %v4 = load float, float* %af, align 1
-  %v5 = load float, float* %af, align 4
-
-; CHECK-NEXT:    {{.*}}|    3: <20, 5, 1, 1>         |    %v4 = load float* %p0, align 1;
-; CHECK-NEXT:    {{.*}}|    3: <20, 6, 3, 1>         |    %v5 = load float* %p0, align 4;
-
-  %ad = inttoptr i32 %p0 to double*
-  %v6 = load double, double* %ad, align 1
-  %v7 = load double, double* %ad, align 8
-
-; CHECK-NEXT:    {{.*}}|    3: <20, 7, 1, 2>         |    %v6 = load double* %p0, align 1;
-; CHECK-NEXT:    {{.*}}|    3: <20, 8, 4, 2>         |    %v7 = load double* %p0, align 8;
-
-  %av16_i8 = inttoptr i32 %p0 to <16 x i8>*
-  %v8 = load <16 x i8>, <16 x i8>* %av16_i8, align 1
-
-; CHECK-NEXT:    {{.*}}|    3: <20, 9, 1, 6>         |    %v8 = load <16 x i8>* %p0, align 1;
-
-  %av8_i16 = inttoptr i32 %p0 to <8 x i16>*
-  %v9 = load <8 x i16>, <8 x i16>* %av8_i16, align 2
-
-; CHECK-NEXT:    {{.*}}|    3: <20, 10, 2, 7>        |    %v9 = load <8 x i16>* %p0, align 2;
-
-  %av4_i32 = inttoptr i32 %p0 to <4 x i32>*
-  %v10 = load <4 x i32>, <4 x i32>* %av4_i32, align 4
-
-; CHECK-NEXT:    {{.*}}|    3: <20, 11, 3, 8>        |    %v10 = load <4 x i32>* %p0, 
-; CHECK-NEXT:          |                             |        align 4;
-
-  %av4_f = inttoptr i32 %p0 to <4 x float>*
-  %v11 = load <4 x float>, <4 x float>* %av4_f, align 4
-
-; CHECK-NEXT:    {{.*}}|    3: <20, 12, 3, 9>        |    %v11 = load <4 x float>* %p0, 
-; CHECK-NEXT:          |                             |        align 4;
-
-  ; Verify correct type loaded.
-  %v12 = add i8 %v0, %v0
-  %v13 = add i16 %v1, %v1
-  %v14 = add i32 %v2, %v2
-  %v15 = add i64 %v3, %v3
-  %v16 = fadd float %v4, %v5
-  %v17 = fadd double %v6, %v7
-  %v18 = add <16 x i8> %v8, %v8
-  %v19 = add <8 x i16> %v9, %v9
-  %v20 = add <4 x i32> %v10, %v10
-  %v21 = fadd <4 x float> %v11, %v11
-
-; CHECK-NEXT:    {{.*}}|    3: <2, 12, 12, 0>        |    %v12 = add i8 %v0, %v0;
-; CHECK-NEXT:    {{.*}}|    3: <2, 12, 12, 0>        |    %v13 = add i16 %v1, %v1;
-; CHECK-NEXT:    {{.*}}|    3: <2, 12, 12, 0>        |    %v14 = add i32 %v2, %v2;
-; CHECK-NEXT:    {{.*}}|    3: <2, 12, 12, 0>        |    %v15 = add i64 %v3, %v3;
-; CHECK-NEXT:    {{.*}}|    3: <2, 12, 11, 0>        |    %v16 = fadd float %v4, %v5;
-; CHECK-NEXT:    {{.*}}|    3: <2, 11, 10, 0>        |    %v17 = fadd double %v6, %v7;
-; CHECK-NEXT:    {{.*}}|    3: <2, 10, 10, 0>        |    %v18 = add <16 x i8> %v8, %v8;
-; CHECK-NEXT:    {{.*}}|    3: <2, 10, 10, 0>        |    %v19 = add <8 x i16> %v9, %v9;
-; CHECK-NEXT:    {{.*}}|    3: <2, 10, 10, 0>        |    %v20 = add <4 x i32> %v10, %v10;
-; CHECK-NEXT:    {{.*}}|    3: <2, 10, 10, 0>        |    %v21 = fadd <4 x float> %v11, %v11;
-
-  ret void
-; CHECK-NEXT:    {{.*}}|    3: <10>                  |    ret void;
-}
-
-
-
-; Test invalid loads.
-define void @BadTests(i32 %p0) {
-; CHECK:               |                             |  %b0:
-
-  %ai1 = inttoptr i32 %p0 to i1*
-  %v0 = load i1, i1* %ai1
-
-; CHECK-NEXT:    {{.*}}|    3: <20, 1, 0, 10>        |    %v0 = load i1* %p0, align 0;
-; CHECK-NEXT:Error({{.*}}): load: Illegal alignment for i1. Expects: 1
-
-  %ai8 = inttoptr i32 %p0 to i8*
-  %v1 = load i8, i8* %ai8
-; CHECK-NEXT:    {{.*}}|    3: <20, 2, 0, 3>         |    %v1 = load i8* %p0, align 0;
-; CHECK-NEXT:Error({{.*}}): load: Illegal alignment for i8. Expects: 1
-
-  %ai16 = inttoptr i32 %p0 to i16*
-  %v2 = load i16, i16* %ai16
-
-; CHECK-NEXT:    {{.*}}|    3: <20, 3, 0, 4>         |    %v2 = load i16* %p0, align 0;
-; CHECK-NEXT:Error({{.*}}): load: Illegal alignment for i16. Expects: 1
-
-  %ai32 = inttoptr i32 %p0 to i32*
-  %v3 = load i32, i32* %ai32
-
-; CHECK-NEXT:    {{.*}}|    3: <20, 4, 0, 0>         |    %v3 = load i32* %p0, align 0;
-; CHECK-NEXT:Error({{.*}}): load: Illegal alignment for i32. Expects: 1
-
-  %ai64 = inttoptr i32 %p0 to i64*
-  %v4 = load i64, i64* %ai64
-
-; CHECK-NEXT:    {{.*}}|    3: <20, 5, 0, 5>         |    %v4 = load i64* %p0, align 0;
-; CHECK-NEXT:Error({{.*}}): load: Illegal alignment for i64. Expects: 1
-
-  %af = inttoptr i32 %p0 to float*
-  %v5 = load float, float* %af
-
-; CHECK-NEXT:    {{.*}}|    3: <20, 6, 0, 1>         |    %v5 = load float* %p0, align 0;
-; CHECK-NEXT:Error({{.*}}): load: Illegal alignment for float. Expects: 1 or 4
-
-  %ad = inttoptr i32 %p0 to double*
-  %v6 = load double, double* %ad, align 4
-
-; CHECK-NEXT:    {{.*}}|    3: <20, 7, 3, 2>         |    %v6 = load double* %p0, align 4;
-; CHECK-NEXT:Error({{.*}}): load: Illegal alignment for double. Expects: 1 or 8
-
-  %av16_i8 = inttoptr i32 %p0 to <16 x i8>*
-  %v7 = load <16 x i8>, <16 x i8>* %av16_i8, align 8
-
-; CHECK-NEXT:    {{.*}}|    3: <20, 8, 4, 6>         |    %v7 = load <16 x i8>* %p0, align 8;
-; CHECK-NEXT:Error({{.*}}): load: Illegal alignment for <16 x i8>. Expects: 1
-
-  %av8_i16 = inttoptr i32 %p0 to <8 x i16>*
-  %v8 = load <8 x i16>, <8 x i16>* %av8_i16, align 1
-
-; CHECK-NEXT:    {{.*}}|    3: <20, 9, 1, 7>         |    %v8 = load <8 x i16>* %p0, align 1;
-; CHECK-NEXT:Error({{.*}}): load: Illegal alignment for <8 x i16>. Expects: 2
-
-  %av4_i32 = inttoptr i32 %p0 to <4 x i32>*
-  %v9 = load <4 x i32>, <4 x i32>* %av4_i32, align 1
-
-; CHECK-NEXT:    {{.*}}|    3: <20, 10, 1, 8>        |    %v9 = load <4 x i32>* %p0, align 1;
-; CHECK-NEXT:Error({{.*}}): load: Illegal alignment for <4 x i32>. Expects: 4
-
-  %av4_f = inttoptr i32 %p0 to <4 x float>*
-  %v10 = load <4 x float>, <4 x float>* %av4_f, align 1
-
-; CHECK-NEXT:    {{.*}}|    3: <20, 11, 1, 9>        |    %v10 = load <4 x float>* %p0, 
-; CHECK-NEXT:          |                             |        align 1;
-; CHECK-NEXT:Error({{.*}}): load: Illegal alignment for <4 x float>. Expects: 4
-
-  ; Note: vectors of form <N x i1> can't be loaded because no alignment
-  ; value is valid.
-
-  %av4_i1 = inttoptr i32 %p0 to <4 x i1>*
-  %v11 = load <4 x i1>, <4 x i1>* %av4_i1, align 1
-
-; CHECK-NEXT:    {{.*}}|    3: <20, 12, 1, 13>       |    %v11 = load <4 x i1>* %p0, align 1;
-; CHECK-NEXT:Error({{.*}}): load: Not allowed for type: <4 x i1>
-
-  %av8_i1 = inttoptr i32 %p0 to <8 x i1>*
-  %v12 = load <8 x i1>, <8 x i1>* %av8_i1, align 1
-
-; CHECK-NEXT:    {{.*}}|    3: <20, 13, 1, 14>       |    %v12 = load <8 x i1>* %p0, align 1;
-; CHECK-NEXT:Error({{.*}}): load: Not allowed for type: <8 x i1>
-
-  %av16_i1 = inttoptr i32 %p0 to <16 x i1>*
-  %v13 = load <16 x i1>, <16 x i1>* %av16_i1, align 1
-
-; CHECK-NEXT:    {{.*}}|    3: <20, 14, 1, 15>       |    %v13 = load <16 x i1>* %p0, 
-; CHECK-NEXT:          |                             |        align 1;
-; CHECK-NEXT:Error({{.*}}): load: Not allowed for type: <16 x i1>
-
-  ret void
-; CHECK-NEXT:    {{.*}}|    3: <10>                  |    ret void;
-
-}
-
diff --git a/test/NaCl/Bitcode/pnacl-bcdis/no-abbreviations.ll b/test/NaCl/Bitcode/pnacl-bcdis/no-abbreviations.ll
deleted file mode 100644
index 593e9699259..00000000000
--- a/test/NaCl/Bitcode/pnacl-bcdis/no-abbreviations.ll
+++ /dev/null
@@ -1,288 +0,0 @@
-; Simple test to show the difference between no abbreviations, and
-; the ones provided by pnacl-freeze. Differences are show by the
-; addresses for records.
-
-; RUN: llvm-as < %s | pnacl-freeze --allow-local-symbol-tables \
-; RUN:              | pnacl-bcdis --allow-local-symbol-tables \
-; RUN:              | FileCheck %s --check-prefix ABV
-
-; RUN: llvm-as < %s | pnacl-freeze --allow-local-symbol-tables \
-; RUN:                --align-bitcode-records \
-; RUN:              | pnacl-bcdis --allow-local-symbol-tables \
-; RUN:              | FileCheck %s --check-prefix ALN
-
-; RUN: llvm-as < %s | pnacl-freeze --allow-local-symbol-tables \
-; RUN:              | pnacl-bccompress --remove-abbreviations \
-; RUN:              | pnacl-bcdis --allow-local-symbol-tables \
-; RUN:              | FileCheck %s --check-prefix NOABV
-
-define i32 @fact(i32 %n) {
-  %v1 = icmp eq i32 0, 1
-  br i1 %v1, label %true, label %false
-true:
-  ret i32 1
-false:
-  %v2 = sub i32 %n, 1
-  %v3 = call i32 @fact(i32 %v2)
-  %v4 = mul i32 %n, %v3
-  ret i32 %v4
-}
-
-; ABV:            0:0|<65532, 80, 69, 88, 69, 1, 0,|Magic Number: 'PEXE' (80, 69, 88, 69)
-; ABV-NEXT:          | 8, 0, 17, 0, 4, 0, 2, 0, 0, |PNaCl Version: 2
-; ABV-NEXT:          | 0>                          |
-; ABV-NEXT:      16:0|1: <65535, 8, 2>             |module {  // BlockID = 8
-; ABV-NEXT:      24:0|  3: <1, 1>                  |  version 1;
-; ABV-NEXT:      26:4|  1: <65535, 0, 2>           |  abbreviations {  // BlockID = 0
-; ABV-NEXT:      36:0|    3: <1, 14>               |    valuesymtab:
-; ABV-NEXT:      38:4|    2: <65533, 4, 0, 1, 3, 0,|      @a0 = abbrev <fixed(3), vbr(8), 
-; ABV-NEXT:          |        2, 8, 0, 3, 0, 1, 8> |                   array(fixed(8))>;
-; ABV-NEXT:      43:2|    2: <65533, 4, 1, 1, 0, 2,|      @a1 = abbrev <1, vbr(8), 
-; ABV-NEXT:          |        8, 0, 3, 0, 1, 7>    |                   array(fixed(7))>;
-; ABV-NEXT:      48:0|    2: <65533, 4, 1, 1, 0, 2,|      @a2 = abbrev <1, vbr(8), 
-; ABV-NEXT:          |        8, 0, 3, 0, 4>       |                   array(char6)>;
-; ABV-NEXT:      52:1|    2: <65533, 4, 1, 2, 0, 2,|      @a3 = abbrev <2, vbr(8), 
-; ABV-NEXT:          |        8, 0, 3, 0, 4>       |                   array(char6)>;
-; ABV-NEXT:      56:2|    3: <1, 11>               |    constants:
-; ABV-NEXT:      58:6|    2: <65533, 2, 1, 1, 0, 1,|      @a0 = abbrev <1, fixed(3)>;
-; ABV-NEXT:          |        3>                   |
-; ABV-NEXT:      61:7|    2: <65533, 2, 1, 4, 0, 2,|      @a1 = abbrev <4, vbr(8)>;
-; ABV-NEXT:          |        8>                   |
-; ABV-NEXT:      65:0|    2: <65533, 2, 1, 4, 1, 0>|      @a2 = abbrev <4, 0>;
-; ABV-NEXT:      68:1|    2: <65533, 2, 1, 6, 0, 2,|      @a3 = abbrev <6, vbr(8)>;
-; ABV-NEXT:          |        8>                   |
-; ABV-NEXT:      71:2|    3: <1, 12>               |    function:
-; ABV-NEXT:      73:6|    2: <65533, 4, 1, 20, 0,  |      @a0 = abbrev <20, vbr(6), vbr(4),
-; ABV-NEXT:          |        2, 6, 0, 2, 4, 0, 2, |                   vbr(4)>;
-; ABV-NEXT:          |        4>                   |
-; ABV-NEXT:      79:1|    2: <65533, 4, 1, 2, 0, 2,|      @a1 = abbrev <2, vbr(6), vbr(6), 
-; ABV-NEXT:          |        6, 0, 2, 6, 0, 1, 4> |                   fixed(4)>;
-; ABV-NEXT:      84:4|    2: <65533, 4, 1, 3, 0, 2,|      @a2 = abbrev <3, vbr(6), 
-; ABV-NEXT:          |        6, 0, 1, 3, 0, 1, 4> |                   fixed(3), fixed(4)>;
-; ABV-NEXT:      89:7|    2: <65533, 1, 1, 10>     |      @a3 = abbrev <10>;
-; ABV-NEXT:      91:7|    2: <65533, 2, 1, 10, 0,  |      @a4 = abbrev <10, vbr(6)>;
-; ABV-NEXT:          |        2, 6>                |
-; ABV-NEXT:      95:0|    2: <65533, 1, 1, 15>     |      @a5 = abbrev <15>;
-; ABV-NEXT:      97:0|    2: <65533, 3, 1, 43, 0,  |      @a6 = abbrev <43, vbr(6), 
-; ABV-NEXT:          |        2, 6, 0, 1, 3>       |                   fixed(3)>;
-; ABV-NEXT:     101:2|    2: <65533, 4, 1, 24, 0,  |      @a7 = abbrev <24, vbr(6), vbr(6),
-; ABV-NEXT:          |        2, 6, 0, 2, 6, 0, 2, |                   vbr(4)>;
-; ABV-NEXT:          |        4>                   |
-; ABV-NEXT:     106:5|    3: <1, 19>               |    globals:
-; ABV-NEXT:     109:1|    2: <65533, 3, 1, 0, 0, 2,|      @a0 = abbrev <0, vbr(6), 
-; ABV-NEXT:          |        6, 0, 1, 1>          |                   fixed(1)>;
-; ABV-NEXT:     113:3|    2: <65533, 2, 1, 1, 0, 2,|      @a1 = abbrev <1, vbr(8)>;
-; ABV-NEXT:          |        8>                   |
-; ABV-NEXT:     116:4|    2: <65533, 2, 1, 2, 0, 2,|      @a2 = abbrev <2, vbr(8)>;
-; ABV-NEXT:          |        8>                   |
-; ABV-NEXT:     119:5|    2: <65533, 3, 1, 3, 0, 3,|      @a3 = abbrev <3, array(fixed(8))>
-; ABV-NEXT:          |        0, 1, 8>             |          ;
-; ABV-NEXT:     123:2|    2: <65533, 2, 1, 4, 0, 2,|      @a4 = abbrev <4, vbr(6)>;
-; ABV-NEXT:          |        6>                   |
-; ABV-NEXT:     126:3|    2: <65533, 3, 1, 4, 0, 2,|      @a5 = abbrev <4, vbr(6), vbr(6)>;
-; ABV-NEXT:          |        6, 0, 2, 6>          |
-; ABV-NEXT:     130:5|  0: <65534>                 |  }
-; ABV-NEXT:     132:0|  1: <65535, 17, 3>          |  types {  // BlockID = 17
-; ABV-NEXT:     140:0|    2: <65533, 4, 1, 21, 0,  |    %a0 = abbrev <21, fixed(1), 
-; ABV-NEXT:          |        1, 1, 0, 3, 0, 1, 3> |                  array(fixed(3))>;
-; ABV-NEXT:     144:7|    3: <1, 4>                |    count 4;
-; ABV-NEXT:     147:4|    3: <7, 32>               |    @t0 = i32;
-; ABV-NEXT:     150:7|    3: <2>                   |    @t1 = void;
-; ABV-NEXT:     152:6|    4: <21, 0, 0, 0>         |    @t2 = i32 (i32); <%a0>
-; ABV-NEXT:     154:6|    3: <7, 1>                |    @t3 = i1;
-; ABV-NEXT:     157:3|  0: <65534>                 |  }
-; ABV-NEXT:     160:0|  3: <8, 2, 0, 0, 0>         |  define external i32 @f0(i32);
-; ABV-NEXT:     164:6|  1: <65535, 19, 4>          |  globals {  // BlockID = 19
-; ABV-NEXT:     172:0|    3: <5, 0>                |    count 0;
-; ABV-NEXT:     174:6|  0: <65534>                 |  }
-; ABV-NEXT:     176:0|  1: <65535, 14, 3>          |  valuesymtab {  // BlockID = 14
-; ABV-NEXT:     184:0|    6: <1, 0, 102, 97, 99,   |    @f0 : "fact"; <@a2>
-; ABV-NEXT:          |        116>                 |
-; ABV-NEXT:     189:1|  0: <65534>                 |  }
-; ABV-NEXT:     192:0|  1: <65535, 12, 4>          |  function i32 @f0(i32 %p0) {  
-; ABV-NEXT:          |                             |                   // BlockID = 12
-; ABV-NEXT:     200:0|    3: <1, 3>                |    blocks 3;
-; ABV-NEXT:     202:6|    1: <65535, 11, 3>        |    constants {  // BlockID = 11
-; ABV-NEXT:     212:0|      4: <1, 0>              |      i32: <@a0>
-; ABV-NEXT:     212:6|      5: <4, 2>              |        %c0 = i32 1; <@a1>
-; ABV-NEXT:     214:1|      6: <4, 0>              |        %c1 = i32 0; <@a2>
-; ABV-NEXT:     214:4|    0: <65534>               |      }
-; ABV-NEXT:          |                             |  %b0:
-; ABV-NEXT:     216:0|    3: <28, 1, 2, 32>        |    %v0 = icmp eq i32 %c1, %c0;
-; ABV-NEXT:     221:0|    3: <11, 1, 2, 1>         |    br i1 %v0, label %b1, label %b2;
-; ABV-NEXT:          |                             |  %b1:
-; ABV-NEXT:     225:2|    8: <10, 3>               |    ret i32 %c0; <@a4>
-; ABV-NEXT:          |                             |  %b2:
-; ABV-NEXT:     226:4|    5: <2, 4, 3, 1>          |    %v1 = sub i32 %p0, %c0; <@a1>
-; ABV-NEXT:     229:0|    3: <34, 0, 6, 1>         |    %v2 = call i32 @f0(i32 %v1);
-; ABV-NEXT:     234:0|    5: <2, 6, 1, 2>          |    %v3 = mul i32 %p0, %v2; <@a1>
-; ABV-NEXT:     236:4|    8: <10, 1>               |    ret i32 %v3; <@a4>
-; ABV-NEXT:     237:6|    1: <65535, 14, 3>        |    valuesymtab {  // BlockID = 14
-; ABV-NEXT:     244:0|      7: <2, 1, 116, 114,    |      %b1 : "true"; <@a3>
-; ABV-NEXT:          |        117, 101>            |
-; ABV-NEXT:     249:1|      6: <1, 4, 118, 49>     |      %v0 : "v1"; <@a2>
-; ABV-NEXT:     252:6|      6: <1, 5, 118, 50>     |      %v1 : "v2"; <@a2>
-; ABV-NEXT:     256:3|      6: <1, 6, 118, 51>     |      %v2 : "v3"; <@a2>
-; ABV-NEXT:     260:0|      6: <1, 7, 118, 52>     |      %v3 : "v4"; <@a2>
-; ABV-NEXT:     263:5|      7: <2, 2, 102, 97, 108,|      %b2 : "false"; <@a3>
-; ABV-NEXT:          |        115, 101>            |
-; ABV-NEXT:     269:4|      6: <1, 1, 110>         |      %p0 : "n"; <@a2>
-; ABV-NEXT:     272:3|    0: <65534>               |    }
-; ABV-NEXT:     276:0|  0: <65534>                 |  }
-; ABV-NEXT:     280:0|0: <65534>                   |}
-
-; ALN:            0:0|<65532, 80, 69, 88, 69, 2, 0,|Magic Number: 'PEXE' (80, 69, 88, 69)
-; ALN-NEXT:          | 12, 0, 17, 0, 4, 0, 2, 0, 0,|PNaCl Version: 2
-; ALN-NEXT:          | 0, 34, 0, 0, 0>             |Align bitcode records: true
-; ALN-NEXT:Warning(0:0): Unsupported: Align bitcode records: true
-; ALN-NEXT:      20:0|1: <65535, 8, 2>             |module {  // BlockID = 8
-; ALN-NEXT:      28:0|  3: <1, 1>                  |  version 1;
-; ALN-NEXT:      31:0|  1: <65535, 0, 2>           |  abbreviations {  // BlockID = 0
-; ALN-NEXT:      40:0|    3: <1, 14>               |    valuesymtab:
-; ALN-NEXT:      43:0|    2: <65533, 4, 0, 1, 3, 0,|      @a0 = abbrev <fixed(3), vbr(8), 
-; ALN-NEXT:          |        2, 8, 0, 3, 0, 1, 8> |                   array(fixed(8))>;
-; ALN-NEXT:      48:0|    2: <65533, 4, 1, 1, 0, 2,|      @a1 = abbrev <1, vbr(8), 
-; ALN-NEXT:          |        8, 0, 3, 0, 1, 7>    |                   array(fixed(7))>;
-; ALN-NEXT:      53:0|    2: <65533, 4, 1, 1, 0, 2,|      @a2 = abbrev <1, vbr(8), 
-; ALN-NEXT:          |        8, 0, 3, 0, 4>       |                   array(char6)>;
-; ALN-NEXT:      58:0|    2: <65533, 4, 1, 2, 0, 2,|      @a3 = abbrev <2, vbr(8), 
-; ALN-NEXT:          |        8, 0, 3, 0, 4>       |                   array(char6)>;
-; ALN-NEXT:      63:0|    3: <1, 11>               |    constants:
-; ALN-NEXT:      66:0|    2: <65533, 2, 1, 1, 0, 1,|      @a0 = abbrev <1, fixed(3)>;
-; ALN-NEXT:          |        3>                   |
-; ALN-NEXT:      70:0|    2: <65533, 2, 1, 4, 0, 2,|      @a1 = abbrev <4, vbr(8)>;
-; ALN-NEXT:          |        8>                   |
-; ALN-NEXT:      74:0|    2: <65533, 2, 1, 4, 1, 0>|      @a2 = abbrev <4, 0>;
-; ALN-NEXT:      78:0|    2: <65533, 2, 1, 6, 0, 2,|      @a3 = abbrev <6, vbr(8)>;
-; ALN-NEXT:          |        8>                   |
-; ALN-NEXT:      82:0|    3: <1, 12>               |    function:
-; ALN-NEXT:      85:0|    2: <65533, 4, 1, 20, 0,  |      @a0 = abbrev <20, vbr(6), vbr(4),
-; ALN-NEXT:          |        2, 6, 0, 2, 4, 0, 2, |                   vbr(4)>;
-; ALN-NEXT:          |        4>                   |
-; ALN-NEXT:      91:0|    2: <65533, 4, 1, 2, 0, 2,|      @a1 = abbrev <2, vbr(6), vbr(6), 
-; ALN-NEXT:          |        6, 0, 2, 6, 0, 1, 4> |                   fixed(4)>;
-; ALN-NEXT:      97:0|    2: <65533, 4, 1, 3, 0, 2,|      @a2 = abbrev <3, vbr(6), 
-; ALN-NEXT:          |        6, 0, 1, 3, 0, 1, 4> |                   fixed(3), fixed(4)>;
-; ALN-NEXT:     103:0|    2: <65533, 1, 1, 10>     |      @a3 = abbrev <10>;
-; ALN-NEXT:     105:0|    2: <65533, 2, 1, 10, 0,  |      @a4 = abbrev <10, vbr(6)>;
-; ALN-NEXT:          |        2, 6>                |
-; ALN-NEXT:     109:0|    2: <65533, 1, 1, 15>     |      @a5 = abbrev <15>;
-; ALN-NEXT:     111:0|    2: <65533, 3, 1, 43, 0,  |      @a6 = abbrev <43, vbr(6), 
-; ALN-NEXT:          |        2, 6, 0, 1, 3>       |                   fixed(3)>;
-; ALN-NEXT:     116:0|    2: <65533, 4, 1, 24, 0,  |      @a7 = abbrev <24, vbr(6), vbr(6),
-; ALN-NEXT:          |        2, 6, 0, 2, 6, 0, 2, |                   vbr(4)>;
-; ALN-NEXT:          |        4>                   |
-; ALN-NEXT:     122:0|    3: <1, 19>               |    globals:
-; ALN-NEXT:     125:0|    2: <65533, 3, 1, 0, 0, 2,|      @a0 = abbrev <0, vbr(6), 
-; ALN-NEXT:          |        6, 0, 1, 1>          |                   fixed(1)>;
-; ALN-NEXT:     130:0|    2: <65533, 2, 1, 1, 0, 2,|      @a1 = abbrev <1, vbr(8)>;
-; ALN-NEXT:          |        8>                   |
-; ALN-NEXT:     134:0|    2: <65533, 2, 1, 2, 0, 2,|      @a2 = abbrev <2, vbr(8)>;
-; ALN-NEXT:          |        8>                   |
-; ALN-NEXT:     138:0|    2: <65533, 3, 1, 3, 0, 3,|      @a3 = abbrev <3, array(fixed(8))>
-; ALN-NEXT:          |        0, 1, 8>             |          ;
-; ALN-NEXT:     142:0|    2: <65533, 2, 1, 4, 0, 2,|      @a4 = abbrev <4, vbr(6)>;
-; ALN-NEXT:          |        6>                   |
-; ALN-NEXT:     146:0|    2: <65533, 3, 1, 4, 0, 2,|      @a5 = abbrev <4, vbr(6), vbr(6)>;
-; ALN-NEXT:          |        6, 0, 2, 6>          |
-; ALN-NEXT:     151:0|  0: <65534>                 |  }
-; ALN-NEXT:     152:0|  1: <65535, 17, 3>          |  types {  // BlockID = 17
-; ALN-NEXT:     160:0|    2: <65533, 4, 1, 21, 0,  |    %a0 = abbrev <21, fixed(1), 
-; ALN-NEXT:          |        1, 1, 0, 3, 0, 1, 3> |                  array(fixed(3))>;
-; ALN-NEXT:     165:0|    3: <1, 4>                |    count 4;
-; ALN-NEXT:     168:0|    3: <7, 32>               |    @t0 = i32;
-; ALN-NEXT:     172:0|    3: <2>                   |    @t1 = void;
-; ALN-NEXT:     174:0|    4: <21, 0, 0, 0>         |    @t2 = i32 (i32); <%a0>
-; ALN-NEXT:     176:0|    3: <7, 1>                |    @t3 = i1;
-; ALN-NEXT:     179:0|  0: <65534>                 |  }
-; ALN-NEXT:     180:0|  3: <8, 2, 0, 0, 0>         |  define external i32 @f0(i32);
-; ALN-NEXT:     185:0|  1: <65535, 19, 4>          |  globals {  // BlockID = 19
-; ALN-NEXT:     192:0|    3: <5, 0>                |    count 0;
-; ALN-NEXT:     195:0|  0: <65534>                 |  }
-; ALN-NEXT:     196:0|  1: <65535, 14, 3>          |  valuesymtab {  // BlockID = 14
-; ALN-NEXT:     204:0|    6: <1, 0, 102, 97, 99,   |    @f0 : "fact"; <@a2>
-; ALN-NEXT:          |        116>                 |
-; ALN-NEXT:     210:0|  0: <65534>                 |  }
-; ALN-NEXT:     212:0|  1: <65535, 12, 4>          |  function i32 @f0(i32 %p0) {  
-; ALN-NEXT:          |                             |                   // BlockID = 12
-; ALN-NEXT:     220:0|    3: <1, 3>                |    blocks 3;
-; ALN-NEXT:     223:0|    1: <65535, 11, 3>        |    constants {  // BlockID = 11
-; ALN-NEXT:     232:0|      4: <1, 0>              |      i32: <@a0>
-; ALN-NEXT:     233:0|      5: <4, 2>              |        %c0 = i32 1; <@a1>
-; ALN-NEXT:     235:0|      6: <4, 0>              |        %c1 = i32 0; <@a2>
-; ALN-NEXT:     236:0|    0: <65534>               |      }
-; ALN-NEXT:          |                             |  %b0:
-; ALN-NEXT:     240:0|    3: <28, 1, 2, 32>        |    %v0 = icmp eq i32 %c1, %c0;
-; ALN-NEXT:     245:0|    3: <11, 1, 2, 1>         |    br i1 %v0, label %b1, label %b2;
-; ALN-NEXT:          |                             |  %b1:
-; ALN-NEXT:     250:0|    8: <10, 3>               |    ret i32 %c0; <@a4>
-; ALN-NEXT:          |                             |  %b2:
-; ALN-NEXT:     252:0|    5: <2, 4, 3, 1>          |    %v1 = sub i32 %p0, %c0; <@a1>
-; ALN-NEXT:     255:0|    3: <34, 0, 6, 1>         |    %v2 = call i32 @f0(i32 %v1);
-; ALN-NEXT:     260:0|    5: <2, 6, 1, 2>          |    %v3 = mul i32 %p0, %v2; <@a1>
-; ALN-NEXT:     263:0|    8: <10, 1>               |    ret i32 %v3; <@a4>
-; ALN-NEXT:     265:0|    1: <65535, 14, 3>        |    valuesymtab {  // BlockID = 14
-; ALN-NEXT:     272:0|      7: <2, 1, 116, 114,    |      %b1 : "true"; <@a3>
-; ALN-NEXT:          |        117, 101>            |
-; ALN-NEXT:     278:0|      6: <1, 4, 118, 49>     |      %v0 : "v1"; <@a2>
-; ALN-NEXT:     282:0|      6: <1, 5, 118, 50>     |      %v1 : "v2"; <@a2>
-; ALN-NEXT:     286:0|      6: <1, 6, 118, 51>     |      %v2 : "v3"; <@a2>
-; ALN-NEXT:     290:0|      6: <1, 7, 118, 52>     |      %v3 : "v4"; <@a2>
-; ALN-NEXT:     294:0|      7: <2, 2, 102, 97, 108,|      %b2 : "false"; <@a3>
-; ALN-NEXT:          |        115, 101>            |
-; ALN-NEXT:     300:0|      6: <1, 1, 110>         |      %p0 : "n"; <@a2>
-; ALN-NEXT:     303:0|    0: <65534>               |    }
-; ALN-NEXT:     304:0|  0: <65534>                 |  }
-; ALN-NEXT:     308:0|0: <65534>                   |}
-
-; NOABV:         0:0|<65532, 80, 69, 88, 69, 1, 0,|Magic Number: 'PEXE' (80, 69, 88, 69)
-; NOABV-NEXT:       | 8, 0, 17, 0, 4, 0, 2, 0, 0, |PNaCl Version: 2
-; NOABV-NEXT:       | 0>                          |
-; NOABV-NEXT:   16:0|1: <65535, 8, 2>             |module {  // BlockID = 8
-; NOABV-NEXT:   24:0|  3: <1, 1>                  |  version 1;
-; NOABV-NEXT:   26:4|  1: <65535, 17, 2>          |  types {  // BlockID = 17
-; NOABV-NEXT:   36:0|    3: <1, 4>                |    count 4;
-; NOABV-NEXT:   38:4|    3: <7, 32>               |    @t0 = i32;
-; NOABV-NEXT:   41:6|    3: <2>                   |    @t1 = void;
-; NOABV-NEXT:   43:4|    3: <21, 0, 0, 0>         |    @t2 = i32 (i32);
-; NOABV-NEXT:   47:4|    3: <7, 1>                |    @t3 = i1;
-; NOABV-NEXT:   50:0|  0: <65534>                 |  }
-; NOABV-NEXT:   52:0|  3: <8, 2, 0, 0, 0>         |  define external i32 @f0(i32);
-; NOABV-NEXT:   56:6|  1: <65535, 19, 2>          |  globals {  // BlockID = 19
-; NOABV-NEXT:   64:0|    3: <5, 0>                |    count 0;
-; NOABV-NEXT:   66:4|  0: <65534>                 |  }
-; NOABV-NEXT:   68:0|  1: <65535, 14, 2>          |  valuesymtab {  // BlockID = 14
-; NOABV-NEXT:   76:0|    3: <1, 0, 102, 97, 99,   |    @f0 : "fact";
-; NOABV-NEXT:       |        116>                 |
-; NOABV-NEXT:   84:4|  0: <65534>                 |  }
-; NOABV-NEXT:   88:0|  1: <65535, 12, 2>          |  function i32 @f0(i32 %p0) {  
-; NOABV-NEXT:       |                             |                   // BlockID = 12
-; NOABV-NEXT:   96:0|    3: <1, 3>                |    blocks 3;
-; NOABV-NEXT:   98:4|    1: <65535, 11, 2>        |    constants {  // BlockID = 11
-; NOABV-NEXT:  108:0|      3: <1, 0>              |      i32:
-; NOABV-NEXT:  110:4|      3: <4, 2>              |        %c0 = i32 1;
-; NOABV-NEXT:  113:0|      3: <4, 0>              |        %c1 = i32 0;
-; NOABV-NEXT:  115:4|    0: <65534>               |      }
-; NOABV-NEXT:       |                             |  %b0:
-; NOABV-NEXT:  116:0|    3: <28, 1, 2, 32>        |    %v0 = icmp eq i32 %c1, %c0;
-; NOABV-NEXT:  120:6|    3: <11, 1, 2, 1>         |    br i1 %v0, label %b1, label %b2;
-; NOABV-NEXT:       |                             |  %b1:
-; NOABV-NEXT:  124:6|    3: <10, 3>               |    ret i32 %c0;
-; NOABV-NEXT:       |                             |  %b2:
-; NOABV-NEXT:  127:2|    3: <2, 4, 3, 1>          |    %v1 = sub i32 %p0, %c0;
-; NOABV-NEXT:  131:2|    3: <34, 0, 6, 1>         |    %v2 = call i32 @f0(i32 %v1);
-; NOABV-NEXT:  136:0|    3: <2, 6, 1, 2>          |    %v3 = mul i32 %p0, %v2;
-; NOABV-NEXT:  140:0|    3: <10, 1>               |    ret i32 %v3;
-; NOABV-NEXT:  142:4|    1: <65535, 14, 2>        |    valuesymtab {  // BlockID = 14
-; NOABV-NEXT:  152:0|      3: <2, 1, 116, 114,    |      %b1 : "true";
-; NOABV-NEXT:       |        117, 101>            |
-; NOABV-NEXT:  160:4|      3: <1, 4, 118, 49>     |      %v0 : "v1";
-; NOABV-NEXT:  166:0|      3: <1, 5, 118, 50>     |      %v1 : "v2";
-; NOABV-NEXT:  171:4|      3: <1, 6, 118, 51>     |      %v2 : "v3";
-; NOABV-NEXT:  177:0|      3: <1, 7, 118, 52>     |      %v3 : "v4";
-; NOABV-NEXT:  182:4|      3: <2, 2, 102, 97, 108,|      %b2 : "false";
-; NOABV-NEXT:       |        115, 101>            |
-; NOABV-NEXT:  192:4|      3: <1, 1, 110>         |      %p0 : "n";
-; NOABV-NEXT:  196:4|    0: <65534>               |    }
-; NOABV-NEXT:  200:0|  0: <65534>                 |  }
-; NOABV-NEXT:  204:0|0: <65534>                   |}
diff --git a/test/NaCl/Bitcode/pnacl-bcdis/phi.ll b/test/NaCl/Bitcode/pnacl-bcdis/phi.ll
deleted file mode 100644
index 2c8758cdad6..00000000000
--- a/test/NaCl/Bitcode/pnacl-bcdis/phi.ll
+++ /dev/null
@@ -1,77 +0,0 @@
-; Simple test for phi nodes.
-
-; RUN: llvm-as < %s | pnacl-freeze | pnacl-bccompress --remove-abbreviations \
-; RUN:              | pnacl-bcdis | FileCheck %s
-
-
-
-; Test phi node introduced by branch merge.
-define void @PhiBranchMerge() {
-
-
-b0:
-  br i1 0, label %b1, label %b2
-
-; CHECK:               |                             |  %b0:
-; CHECK-NEXT:    {{.*}}|    3: <11, 1, 2, 1>         |    br i1 %c1, label %b1, label %b2;
-
-b1:
-  %v0 = add i32 1, 1
-  %v1 = sub i32 1, 1
-  br label %b3
-
-; CHECK-NEXT:          |                             |  %b1:
-; CHECK-NEXT:    {{.*}}|    3: <2, 2, 2, 0>          |    %v0 = add i32 %c0, %c0;
-; CHECK-NEXT:    {{.*}}|    3: <2, 3, 3, 1>          |    %v1 = sub i32 %c0, %c0;
-; CHECK-NEXT:    {{.*}}|    3: <11, 3>               |    br label %b3;
-
-b2:
-  %v2 = mul i32 1, 1
-  %v3 = udiv i32 1, 1
-  br label %b3
-
-; CHECK-NEXT:          |                             |  %b2:
-; CHECK-NEXT:    {{.*}}|    3: <2, 4, 4, 2>          |    %v2 = mul i32 %c0, %c0;
-; CHECK-NEXT:    {{.*}}|    3: <2, 5, 5, 3>          |    %v3 = udiv i32 %c0, %c0;
-; CHECK-NEXT:    {{.*}}|    3: <11, 3>               |    br label %b3;
-
-b3:
-  %v4 = phi i32 [ %v0, %b1 ], [ %v2, %b2 ]
-  %v5 = phi i32 [ %v1, %b1 ], [ %v3, %b2 ]
-  ret void
-
-; CHECK-NEXT:          |                             |  %b3:
-; CHECK-NEXT:    {{.*}}|    3: <16, 0, 8, 1, 4, 2>   |    %v4 = phi i32 [%v0, %b1], 
-; CHECK-NEXT:    {{.*}}|                             |        [%v2, %b2];
-; CHECK-NEXT:    {{.*}}|    3: <16, 0, 8, 1, 4, 2>   |    %v5 = phi i32 [%v1, %b1], 
-; CHECK-NEXT:          |                             |        [%v3, %b2];
-; CHECK-NEXT:    {{.*}}|    3: <10>                  |    ret void;
-
-}
-
-
-
-; Test phi node introduced by loop.
-define void @PhiLoopCarried(i32 %p0) {
-
-b0:
-  %v0 = add i32 %p0, 1
-  br label %b1
-
-; CHECK:               |                             |  %b0:
-; CHECK-NEXT:    {{.*}}|    3: <2, 2, 1, 0>          |    %v0 = add i32 %p0, %c0;
-; CHECK-NEXT:    {{.*}}|    3: <11, 1>               |    br label %b1;
-
-b1:
-  %v1 = phi i32 [%v0, %b0], [%v2, %b1]
-  %v2 = add i32 %v1, 1
-  br label %b1
-
-; CHECK-NEXT:          |                             |  %b1:
-; CHECK-NEXT:    {{.*}}|    3: <43, 6, 0>            |    declare i32 %v2;
-; CHECK-NEXT:    {{.*}}|    3: <16, 0, 2, 0, 3, 1>   |    %v1 = phi i32 [%v0, %b0], 
-; CHECK-NEXT:          |                             |        [%v2, %b1];
-; CHECK-NEXT:    {{.*}}|    3: <2, 1, 3, 0>          |    %v2 = add i32 %v1, %c0;
-; CHECK-NEXT:    {{.*}}|    3: <11, 1>               |    br label %b1;
-
-}
diff --git a/test/NaCl/Bitcode/pnacl-bcdis/ret.ll b/test/NaCl/Bitcode/pnacl-bcdis/ret.ll
deleted file mode 100644
index f7e6b5221fd..00000000000
--- a/test/NaCl/Bitcode/pnacl-bcdis/ret.ll
+++ /dev/null
@@ -1,55 +0,0 @@
-; Test return instructions for various types.
-
-; RUN: llvm-as < %s | pnacl-freeze | pnacl-bccompress --remove-abbreviations \
-; RUN:              | pnacl-bcdis | FileCheck %s
-
-define internal void @fvoid() {
-  ret void
-
-; CHECK:               |                             |  %b0:
-; CHECK-NEXT:    {{.*}}|    3: <10>                  |    ret void;
-
-}
-
-define internal i32 @fi32(i32 %p0) {
-  ret i32 %p0
-
-; CHECK:               |                             |  %b0:
-; CHECK:         {{.*}}|    3: <10, 1>               |    ret i32 %p0;
-
-}
-
-
-define internal i64 @fi64(i64 %p0) {
-  ret i64 %p0
-
-; CHECK:               |                             |  %b0:
-; CHECK:         {{.*}}|    3: <10, 1>               |    ret i64 %p0;
-
-}
-
-define internal float @ffloat(float %p0) {
-  ret float %p0
-
-; CHECK:               |                             |  %b0:
-; CHECK:         {{.*}}|    3: <10, 1>               |    ret float %p0;
-
-}
-
-
-define internal double @fdouble(double %p0) {
-  ret double %p0
-
-; CHECK:               |                             |  %b0:
-; CHECK:         {{.*}}|    3: <10, 1>               |    ret double %p0;
-
-}
-
-define internal <4 x i32> @fi32vec(<4 x i32> %p0) {
-  ret <4 x i32> %p0
-
-; CHECK:               |                             |  %b0:
-; CHECK-NEXT:    {{.*}}|    3: <10, 1>               |    ret <4 x i32> %p0;
-
-}
-
diff --git a/test/NaCl/Bitcode/pnacl-bcdis/select.ll b/test/NaCl/Bitcode/pnacl-bcdis/select.ll
deleted file mode 100644
index 8c77d3ba309..00000000000
--- a/test/NaCl/Bitcode/pnacl-bcdis/select.ll
+++ /dev/null
@@ -1,118 +0,0 @@
-; Test the select instruction.
-
-; RUN: llvm-as < %s | pnacl-freeze | pnacl-bccompress --remove-abbreviations \
-; RUN:              | pnacl-bcdis | FileCheck %s
-
-; Test selecting integer types.
-define void @TestIntTypes() {
-
-; CHECK:               |                             |  %b0:
-
-  %v0 = select i1 0, i1 0, i1 0
-  %v1 = select i1 %v0, i8 1, i8 1
-  %v2 = select i1 %v0, i16 2, i16 2
-  %v3 = select i1 %v0, i32 3, i32 3
-  %v4 = select i1 %v0, i64 4, i64 4
-
-; CHECK-NEXT:    {{.*}}|    3: <29, 5, 5, 5>         |    %v0 = select i1 %c0, i1 %c0, 
-; CHECK-NEXT:          |                             |        i1 %c0;
-; CHECK-NEXT:    {{.*}}|    3: <29, 4, 4, 1>         |    %v1 = select i1 %v0, i8 %c2, 
-; CHECK-NEXT:          |                             |        i8 %c2;
-; CHECK-NEXT:    {{.*}}|    3: <29, 4, 4, 2>         |    %v2 = select i1 %v0, i16 %c3, 
-; CHECK-NEXT:          |                             |        i16 %c3;
-; CHECK-NEXT:    {{.*}}|    3: <29, 7, 7, 3>         |    %v3 = select i1 %v0, i32 %c1, 
-; CHECK-NEXT:          |                             |        i32 %c1;
-; CHECK-NEXT:    {{.*}}|    3: <29, 5, 5, 4>         |    %v4 = select i1 %v0, i64 %c4, 
-; CHECK-NEXT:          |                             |        i64 %c4;
-
-  ; Verify computed results of right size.
-  %v5 = and i1 %v0, %v0
-  %v6 = add i8 %v1, %v1
-  %v7 = add i16 %v2, %v2
-  %v8 = add i32 %v3, %v3
-  %v9 = add i64 %v4, %v4
-  ret void
-
-; CHECK-NEXT:    {{.*}}|    3: <2, 5, 5, 10>         |    %v5 = and i1 %v0, %v0;
-; CHECK-NEXT:    {{.*}}|    3: <2, 5, 5, 0>          |    %v6 = add i8 %v1, %v1;
-; CHECK-NEXT:    {{.*}}|    3: <2, 5, 5, 0>          |    %v7 = add i16 %v2, %v2;
-; CHECK-NEXT:    {{.*}}|    3: <2, 5, 5, 0>          |    %v8 = add i32 %v3, %v3;
-; CHECK-NEXT:    {{.*}}|    3: <2, 5, 5, 0>          |    %v9 = add i64 %v4, %v4;
-; CHECK-NEXT:    {{.*}}|    3: <10>                  |    ret void;
-
-}
-
-; Test selecting floating types.
-define void @TestFloatTypes(float %p0, double %p1) {
-
-; CHECK:               |                             |  %b0:
-
-  %v0 = select i1 0, float %p0, float %p0
-  %v1 = select i1 0, double %p1, double %p1
-
-; CHECK-NEXT:    {{.*}}|    3: <29, 3, 3, 1>         |    %v0 = select i1 %c0, float %p0, 
-; CHECK-NEXT:          |                             |        float %p0;
-; CHECK-NEXT:    {{.*}}|    3: <29, 3, 3, 2>         |    %v1 = select i1 %c0, double %p1, 
-; CHECK-NEXT:          |                             |        double %p1;
-
-  ; Verify computed results are of right size.
-  %v2 = fadd float %v0, %v0
-  %v3 = fadd double %v1, %v1
-  ret void
-
-; CHECK-NEXT:    {{.*}}|    3: <2, 2, 2, 0>          |    %v2 = fadd float %v0, %v0;
-; CHECK-NEXT:    {{.*}}|    3: <2, 2, 2, 0>          |    %v3 = fadd double %v1, %v1;
-; CHECK-NEXT:    {{.*}}|    3: <10>                  |    ret void;
-
-}
-
-; Test select on integer vectors
-define void @TestVecIntTypes(i32 %p0, <16 x i8> %p1, <8 x i16> %p2, <4 x i32> %p3) {
-
-  %v0 = trunc i32 %p0 to i1;
-  %v1 = select i1 %v0, <16 x i8> %p1, <16 x i8> %p1
-  %v2 = select i1 %v0, <8 x i16> %p2, <8 x i16> %p2
-  %v3 = select i1 %v0, <4 x i32> %p3, <4 x i32> %p3
-
-; CHECK:         {{.*}}|    {{.*}}                   |    {{.*}} trunc
-
-; CHECK-NEXT:    {{.*}}|    3: <29, 4, 4, 1>         |    %v1 = select i1 %v0, <16 x i8> %p1,
-; CHECK-NEXT:          |                             |        <16 x i8> %p1;
-; CHECK-NEXT:    {{.*}}|    3: <29, 4, 4, 2>         |    %v2 = select i1 %v0, <8 x i16> %p2,
-; CHECK-NEXT:          |                             |        <8 x i16> %p2;
-; CHECK-NEXT:    {{.*}}|    3: <29, 4, 4, 3>         |    %v3 = select i1 %v0, <4 x i32> %p3,
-; CHECK-NEXT:          |                             |        <4 x i32> %p3;
-
-  ; Verify computed results are of right type.
-  %v4 = and <16 x i8> %v1, %v1
-  %v5 = add <8 x i16> %v2, %v2
-  %v6 = add <4 x i32> %v3, %v3
-  ret void
-
-; CHECK-NEXT:    {{.*}}|    3: <2, 3, 3, 10>         |    %v4 = and <16 x i8> %v1, %v1;
-; CHECK-NEXT:    {{.*}}|    3: <2, 3, 3, 0>          |    %v5 = add <8 x i16> %v2, %v2;
-; CHECK-NEXT:    {{.*}}|    3: <2, 3, 3, 0>          |    %v6 = add <4 x i32> %v3, %v3;
-; CHECK-NEXT:    {{.*}}|    3: <10>                  |    ret void;
-
-}
-
-
-; Test select on floating vectors
-define void @TestVecFloatTypes(i32 %p0, <4 x float> %p1) {
-
-  %v0 = trunc i32 %p0 to i1;  
-  %v1 = select i1 %v0, <4 x float> %p1, <4 x float> %p1
-
-; CHECK:         {{.*}}|    {{.*}}                   |    {{.*}} trunc
-; CHECK-NEXT:    {{.*}}|    3: <29, 2, 2, 1>         |    %v1 = select i1 %v0, 
-; CHECK-NEXT:          |                             |        <4 x float> %p1, 
-; CHECK-NEXT:          |                             |        <4 x float> %p1;
-
-  ; Verify computed results are of right type.
-  %v2 = fadd <4 x float> %v1, %v1
-  ret void
-
-; CHECK-NEXT:    {{.*}}|    3: <2, 1, 1, 0>          |    %v2 = fadd <4 x float> %v1, %v1;
-; CHECK-NEXT:    {{.*}}|    3: <10>                  |    ret void;
-
-}
diff --git a/test/NaCl/Bitcode/pnacl-bcdis/store.ll b/test/NaCl/Bitcode/pnacl-bcdis/store.ll
deleted file mode 100644
index 7527c2199fd..00000000000
--- a/test/NaCl/Bitcode/pnacl-bcdis/store.ll
+++ /dev/null
@@ -1,242 +0,0 @@
-; Test the store instruction.
-
-; RUN: llvm-as < %s | pnacl-freeze | pnacl-bccompress --remove-abbreviations \
-; RUN:              | not pnacl-bcdis | FileCheck %s
-
-; Test valid stores.
-define void @GoodTests(i32 %p0) {
-
-; CHECK:         {{.*}}|    1: <65535, 11, 2>        |    constants {  // BlockID = 11
-; CHECK-NEXT:    {{.*}}|      3: <1, 1>              |      i32:
-; CHECK-NEXT:    {{.*}}|      3: <4, 6>              |        %c0 = i32 3;
-; CHECK-NEXT:    {{.*}}|      3: <1, 10>             |      <4 x i32>:
-; CHECK-NEXT:    {{.*}}|      3: <3>                 |        %c1 = <4 x i32> undef;
-; CHECK-NEXT:    {{.*}}|      3: <1, 9>              |      <8 x i16>:
-; CHECK-NEXT:    {{.*}}|      3: <3>                 |        %c2 = <8 x i16> undef;
-; CHECK-NEXT:    {{.*}}|      3: <1, 8>              |      <16 x i8>:
-; CHECK-NEXT:    {{.*}}|      3: <3>                 |        %c3 = <16 x i8> undef;
-; CHECK-NEXT:    {{.*}}|      3: <1, 7>              |      i64:
-; CHECK-NEXT:    {{.*}}|      3: <4, 8>              |        %c4 = i64 4;
-; CHECK-NEXT:    {{.*}}|      3: <1, 4>              |      i8:
-; CHECK-NEXT:    {{.*}}|      3: <4, 4>              |        %c5 = i8 2;
-; CHECK-NEXT:    {{.*}}|      3: <1, 5>              |      i16:
-; CHECK-NEXT:    {{.*}}|      3: <4, 4>              |        %c6 = i16 2;
-; CHECK-NEXT:    {{.*}}|      3: <1, 3>              |      double:
-; CHECK-NEXT:    {{.*}}|      3: <6,                 |        %c7 = double 2;
-; CHECK-NEXT:          |        4611686018427387904> |
-; CHECK-NEXT:    {{.*}}|      3: <6,                 |        %c8 = double 1;
-; CHECK-NEXT:          |        4607182418800017408> |
-; CHECK-NEXT:    {{.*}}|      3: <1, 2>              |      float:
-; CHECK-NEXT:    {{.*}}|      3: <6, 1073741824>     |        %c9 = float 2;
-; CHECK-NEXT:    {{.*}}|      3: <6, 1065353216>     |        %c10 = float 1;
-; CHECK-NEXT:    {{.*}}|      3: <1, 11>             |      <4 x float>:
-; CHECK-NEXT:    {{.*}}|      3: <3>                 |        %c11 = <4 x float> undef;
-; CHECK-NEXT:    {{.*}}|    0: <65534>               |      }
-; CHECK-NEXT:          |                             |  %b0:
-
-  %ai8 = inttoptr i32 %p0 to i8*
-  store i8 2, i8* %ai8, align 1
-
-; CHECK-NEXT:    {{.*}}|    3: <24, 13, 7, 1>        |    store i8 %c5, i8* %p0, align 1;
-
-  %ai16 = inttoptr i32 %p0 to i16*
-  store i16 2, i16* %ai16, align 1
-
-; CHECK-NEXT:    {{.*}}|    3: <24, 13, 6, 1>        |    store i16 %c6, i16* %p0, align 1;
-
-  %ai32 = inttoptr i32 %p0 to i32*
-  store i32 3, i32* %ai32, align 1
-
-; CHECK-NEXT:    {{.*}}|    3: <24, 13, 12, 1>       |    store i32 %c0, i32* %p0, align 1;
-
-  %ai64 = inttoptr i32 %p0 to i64*
-  store i64 4, i64* %ai64, align 1
-
-; CHECK-NEXT:    {{.*}}|    3: <24, 13, 8, 1>        |    store i64 %c4, i64* %p0, align 1;
-
-  %af = inttoptr i32 %p0 to float*
-  store float 1.0, float* %af, align 1
-  store float 2.0, float* %af, align 4
-
-; CHECK-NEXT:    {{.*}}|    3: <24, 13, 2, 1>        |    store float %c10, float* %p0, 
-; CHECK-NEXT:          |                             |        align 1;
-; CHECK-NEXT:    {{.*}}|    3: <24, 13, 3, 3>        |    store float %c9, float* %p0, 
-; CHECK-NEXT:          |                             |        align 4;
-
-  %ad = inttoptr i32 %p0 to double*
-  store double 1.0, double* %ad, align 1
-  store double 2.0, double* %ad, align 8
-
-; CHECK-NEXT:    {{.*}}|    3: <24, 13, 4, 1>        |    store double %c8, double* %p0, 
-; CHECK-NEXT:          |                             |        align 1;
-; CHECK-NEXT:    {{.*}}|    3: <24, 13, 5, 4>        |    store double %c7, double* %p0, 
-; CHECK-NEXT:          |                             |        align 8;
-
-  %av16_i8 = inttoptr i32 %p0 to <16 x i8>*
-  store <16 x i8> undef, <16 x i8>* %av16_i8, align 1
-
-; CHECK-NEXT:    {{.*}}|    3: <24, 13, 9, 1>        |    store <16 x i8> %c3, 
-; CHECK-NEXT:          |                             |        <16 x i8>* %p0, align 1;
-
-  %av8_i16 = inttoptr i32 %p0 to <8 x i16>*
-  store <8 x i16> undef, <8 x i16>* %av8_i16, align 2
-
-; CHECK-NEXT:    {{.*}}|    3: <24, 13, 10, 2>       |    store <8 x i16> %c2, 
-; CHECK-NEXT:          |                             |        <8 x i16>* %p0, align 2;
-
-  %av4_i32 = inttoptr i32 %p0 to <4 x i32>*
-  store <4 x i32> undef,  <4 x i32>* %av4_i32, align 4
-
-; CHECK-NEXT:    {{.*}}|    3: <24, 13, 11, 3>       |    store <4 x i32> %c1, 
-; CHECK-NEXT:          |                             |        <4 x i32>* %p0, align 4;
-
-  %av4_f = inttoptr i32 %p0 to <4 x float>*
-  store <4 x float> undef,  <4 x float>* %av4_f, align 4
-; CHECK-NEXT:    {{.*}}|    3: <24, 13, 1, 3>        |    store <4 x float> %c11, 
-; CHECK-NEXT:          |                             |        <4 x float>* %p0, align 4;
-
-  ret void
-
-; CHECK-NEXT:    {{.*}}|    3: <10>                  |    ret void;
-
-}
-
-
-; Test invalid stores.
-define void @BadTests(i32 %p0) {
-
-; CHECK:         {{.*}}|    1: <65535, 11, 2>        |    constants {  // BlockID = 11
-; CHECK-NEXT:    {{.*}}|      3: <1, 1>              |      i32:
-; CHECK-NEXT:    {{.*}}|      3: <4, 6>              |        %c0 = i32 3;
-; CHECK-NEXT:    {{.*}}|      3: <1, 15>             |      <16 x i1>:
-; CHECK-NEXT:    {{.*}}|      3: <3>                 |        %c1 = <16 x i1> undef;
-; CHECK-NEXT:    {{.*}}|      3: <1, 14>             |      <8 x i1>:
-; CHECK-NEXT:    {{.*}}|      3: <3>                 |        %c2 = <8 x i1> undef;
-; CHECK-NEXT:    {{.*}}|      3: <1, 4>              |      i8:
-; CHECK-NEXT:    {{.*}}|      3: <4, 2>              |        %c3 = i8 1;
-; CHECK-NEXT:    {{.*}}|      3: <1, 5>              |      i16:
-; CHECK-NEXT:    {{.*}}|      3: <4, 4>              |        %c4 = i16 2;
-; CHECK-NEXT:    {{.*}}|      3: <1, 6>              |      i1:
-; CHECK-NEXT:    {{.*}}|      3: <4, 0>              |        %c5 = i1 0;
-; CHECK-NEXT:    {{.*}}|      3: <1, 7>              |      i64:
-; CHECK-NEXT:    {{.*}}|      3: <4, 8>              |        %c6 = i64 4;
-; CHECK-NEXT:    {{.*}}|      3: <1, 8>              |      <16 x i8>:
-; CHECK-NEXT:    {{.*}}|      3: <3>                 |        %c7 = <16 x i8> undef;
-; CHECK-NEXT:    {{.*}}|      3: <1, 9>              |      <8 x i16>:
-; CHECK-NEXT:    {{.*}}|      3: <3>                 |        %c8 = <8 x i16> undef;
-; CHECK-NEXT:    {{.*}}|      3: <1, 10>             |      <4 x i32>:
-; CHECK-NEXT:    {{.*}}|      3: <3>                 |        %c9 = <4 x i32> undef;
-; CHECK-NEXT:    {{.*}}|      3: <1, 13>             |      <4 x i1>:
-; CHECK-NEXT:    {{.*}}|      3: <3>                 |        %c10 = <4 x i1> undef;
-; CHECK-NEXT:    {{.*}}|      3: <1, 11>             |      <4 x float>:
-; CHECK-NEXT:    {{.*}}|      3: <3>                 |        %c11 = <4 x float> undef;
-; CHECK-NEXT:    {{.*}}|      3: <1, 3>              |      double:
-; CHECK-NEXT:    {{.*}}|      3: <6,                 |        %c12 = double 2;
-; CHECK-NEXT:          |        4611686018427387904> |
-; CHECK-NEXT:    {{.*}}|      3: <1, 2>              |      float:
-; CHECK-NEXT:    {{.*}}|      3: <6, 1065353216>     |        %c13 = float 1;
-; CHECK-NEXT:    {{.*}}|    0: <65534>               |      }
-; CHECK-NEXT:          |                             |  %b0:
-
-  %ai1 = inttoptr i32 %p0 to i1*
-  store i1 0, i1* %ai1
-
-; CHECK-NEXT:    {{.*}}|    3: <24, 15, 9, 0>        |    store i1 %c5, i1* %p0, align 0;
-; CHECK-NEXT:Error({{.*}}): store: Illegal alignment for i1. Expects: 1
-
-  %ai8 = inttoptr i32 %p0 to i8*
-  store i8 1, i8* %ai8
-
-; CHECK-NEXT:    {{.*}}|    3: <24, 15, 11, 0>        |    store i8 %c3, i8* %p0, align 0;
-; CHECK-NEXT:Error({{.*}}): store: Illegal alignment for i8. Expects: 1
-
-  %ai16 = inttoptr i32 %p0 to i16*
-  store i16 2, i16* %ai16
-
-; CHECK-NEXT:    {{.*}}|    3: <24, 15, 10, 0>       |    store i16 %c4, i16* %p0, align 0;
-; CHECK-NEXT:Error({{.*}}): store: Illegal alignment for i16. Expects: 1
-
-  %ai32 = inttoptr i32 %p0 to i32*
-  store i32 3, i32* %ai32
-
-; CHECK-NEXT:    {{.*}}|    3: <24, 15, 14, 0>      |    store i32 %c0, i32* %p0, align 0;
-; CHECK-NEXT:Error({{.*}}): store: Illegal alignment for i32. Expects: 1
-
-  %ai64 = inttoptr i32 %p0 to i64*
-  store i64 4, i64* %ai64
-
-; CHECK-NEXT:    {{.*}}|    3: <24, 15, 8, 0>        |    store i64 %c6, i64* %p0, align 0;
-; CHECK-NEXT:Error({{.*}}): store: Illegal alignment for i64. Expects: 1
-
-  %af = inttoptr i32 %p0 to float*
-  store float 1.0, float* %af
-
-; CHECK-NEXT:    {{.*}}|    3: <24, 15, 1, 0>        |    store float %c13, float* %p0, 
-; CHECK-NEXT:          |                             |        align 0;
-; CHECK-NEXT:Error({{.*}}): store: Illegal alignment for float. Expects: 1 or 4
-
-  %ad = inttoptr i32 %p0 to double*
-  store double 2.0, double* %ad, align 4
-
-; CHECK-NEXT:    {{.*}}|    3: <24, 15, 2, 3>        |    store double %c12, double* %p0, 
-; CHECK-NEXT:          |                             |        align 4;
-; CHECK-NEXT:Error({{.*}}): store: Illegal alignment for double. Expects: 1 or 8
-
-  %av16_i8 = inttoptr i32 %p0 to <16 x i8>*
-  store <16 x i8> undef, <16 x i8>* %av16_i8, align 4
-
-; CHECK-NEXT:    {{.*}}|    3: <24, 15, 7, 3>        |    store <16 x i8> %c7, 
-; CHECK-NEXT:          |                             |        <16 x i8>* %p0, align 4;
-; CHECK-NEXT:Error({{.*}}): store: Illegal alignment for <16 x i8>. Expects: 1
-
-  %av8_i16 = inttoptr i32 %p0 to <8 x i16>*
-  store <8 x i16> undef, <8 x i16>* %av8_i16, align 1
-
-; CHECK-NEXT:    {{.*}}|    3: <24, 15, 6, 1>        |    store <8 x i16> %c8, 
-; CHECK-NEXT:          |                             |        <8 x i16>* %p0, align 1;
-; CHECK-NEXT:Error({{.*}}): store: Illegal alignment for <8 x i16>. Expects: 2
-
-  %av4_i32 = inttoptr i32 %p0 to <4 x i32>*
-  store <4 x i32> undef,  <4 x i32>* %av4_i32, align 1
-
-; CHECK-NEXT:    {{.*}}|    3: <24, 15, 5, 1>        |    store <4 x i32> %c9, 
-; CHECK-NEXT:          |                             |        <4 x i32>* %p0, align 1;
-; CHECK-NEXT:Error({{.*}}): store: Illegal alignment for <4 x i32>. Expects: 4
-
-  %av4_f = inttoptr i32 %p0 to <4 x float>*
-  store <4 x float> undef,  <4 x float>* %av4_f, align 1
-
-; CHECK-NEXT:    {{.*}}|    3: <24, 15, 3, 1>        |    store <4 x float> %c11, 
-; CHECK-NEXT:          |                             |        <4 x float>* %p0, align 1;
-; CHECK-NEXT:Error({{.*}}): store: Illegal alignment for <4 x float>. Expects: 4
-
-  ; Note: vectors of form <N x i1> can't be stored because no alignment
-  ; value is valid.
-
-  %av4_i1 = inttoptr i32 %p0 to <4 x i1>*
-  store <4 x i1> undef,  <4 x i1>* %av4_i1, align 1
-
-; CHECK-NEXT:    {{.*}}|    3: <24, 15, 4, 1>        |    store <4 x i1> %c10, <4 x i1>* %p0,
-; CHECK-NEXT:          |                             |        align 1;
-; CHECK-NEXT:Error({{.*}}): store: Not allowed for type: <4 x i1>
-
-  %av8_i1 = inttoptr i32 %p0 to <8 x i1>*
-  store <8 x i1> undef,  <8 x i1>* %av8_i1, align 1
-
-; CHECK-NEXT:    {{.*}}|    3: <24, 15, 12, 1>       |    store <8 x i1> %c2, <8 x i1>* %p0, 
-; CHECK-NEXT:          |                             |        align 1;
-; CHECK-NEXT:Error({{.*}}): store: Not allowed for type: <8 x i1>
-
-  %av16_i1 = inttoptr i32 %p0 to <16 x i1>*
-  store <16 x i1> undef,  <16 x i1>* %av16_i1, align 1
-
-; CHECK-NEXT:    {{.*}}|    3: <24, 15, 13, 1>       |    store <16 x i1> %c1, 
-; CHECK-NEXT:          |                             |        <16 x i1>* %p0, align 1;
-; CHECK-NEXT:Error({{.*}}): store: Not allowed for type: <16 x i1>
-
-  ret void
-
-; CHECK-NEXT:    {{.*}}|    3: <10>                  |    ret void;
-
-}
-
diff --git a/test/NaCl/Bitcode/pnacl-bcdis/switch.ll b/test/NaCl/Bitcode/pnacl-bcdis/switch.ll
deleted file mode 100644
index 5bd242be772..00000000000
--- a/test/NaCl/Bitcode/pnacl-bcdis/switch.ll
+++ /dev/null
@@ -1,73 +0,0 @@
-; Test switch statements.
-
-; RUN: llvm-as < %s | pnacl-freeze | pnacl-bccompress --remove-abbreviations \
-; RUN:              | pnacl-bcdis | FileCheck %s
-
-
-; Test case where we switch on a variable.
-define void @SwitchVariable(i32 %p0) {
-
-; CHECK:               |                             |  %b0:
-
-  switch i32 %p0, label %b1 [
-    i32 1, label %b2
-    i32 2, label %b2
-    i32 4, label %b3
-    i32 5, label %b3
-  ]
-
-; CHECK-NEXT:    {{.*}}|    3: <12, 1, 1, 1, 4, 1, 1,|    switch i32 %p0 {
-; CHECK-NEXT:          |        2, 2, 1, 1, 4, 2, 1, |      default: br label %b1;
-; CHECK-NEXT:          |        1, 8, 3, 1, 1, 10, 3>|      i32 1: br label %b2;
-; CHECK-NEXT:          |                             |      i32 2: br label %b2;
-; CHECK-NEXT:          |                             |      i32 4: br label %b3;
-; CHECK-NEXT:          |                             |      i32 5: br label %b3;
-; CHECK-NEXT:          |                             |    }
-
-b1:
-  br label %b4
-
-
-b2:
-  br label %b4
-
-b3:
-  br label %b4
-
-b4:
-  ret void
-}
-
-; Test case where we switch on a constant.
-define void @SwitchConstant() {
-
-; CHECK:               |                             |  %b0:
-
-  switch i32 3, label %b1 [
-    i32 2, label %b2
-    i32 1, label %b2
-    i32 5, label %b3
-    i32 4, label %b3
-  ]
-
-; CHECK-NEXT:    {{.*}}|    3: <12, 1, 1, 1, 4, 1, 1,|    switch i32 %c0 {
-; CHECK-NEXT:          |        4, 2, 1, 1, 2, 2, 1, |      default: br label %b1;
-; CHECK-NEXT:          |        1, 10, 3, 1, 1, 8, 3>|      i32 2: br label %b2;
-; CHECK-NEXT:          |                             |      i32 1: br label %b2;
-; CHECK-NEXT:          |                             |      i32 5: br label %b3;
-; CHECK-NEXT:          |                             |      i32 4: br label %b3;
-; CHECK-NEXT:          |                             |    }
-
-b1:
-  br label %b4
-
-b2:
-  br label %b4
-
-b3:
-  br label %b4
-
-b4:
-  ret void
-}
-
diff --git a/test/NaCl/Bitcode/pnacl-bcdis/types-and-fcn-addrs.ll b/test/NaCl/Bitcode/pnacl-bcdis/types-and-fcn-addrs.ll
deleted file mode 100644
index 2f85eb93191..00000000000
--- a/test/NaCl/Bitcode/pnacl-bcdis/types-and-fcn-addrs.ll
+++ /dev/null
@@ -1,85 +0,0 @@
-; Simple test to see if we handle all types. Note: Types are generated
-; via function declarations.
-
-; RUN: llvm-as < %s | pnacl-freeze | pnacl-bccompress --remove-abbreviations \
-; RUN:              | pnacl-bcdis | FileCheck %s
-
-declare void @func();
-
-declare i32 @test(i32, float, i32, i32, i64);
-
-declare i64 @merge(i32, i32);
-
-declare double @Float2Double(float);
-
-declare float @Double2Float(double);
-
-declare void @indefargs(i32, ...);
-
-declare <4 x i32> @Mul(<4 x i32>, <4 x i32>);
-
-; Show example where function name can't be modeled as a char6.
-declare i32 @foo$bar(i32);
-
-; CHECK:         0:0|<65532, 80, 69, 88, 69, 1, 0,|Magic Number: 'PEXE' (80, 69, 88, 69)
-; CHECK-NEXT:       | 8, 0, 17, 0, 4, 0, 2, 0, 0, |PNaCl Version: 2
-; CHECK-NEXT:       | 0>                          |
-; CHECK-NEXT:   16:0|1: <65535, 8, 2>             |module {  // BlockID = 8
-; CHECK-NEXT:   24:0|  3: <1, 1>                  |  version 1;
-; CHECK-NEXT:   26:4|  1: <65535, 17, 2>          |  types {  // BlockID = 17
-; CHECK-NEXT:   36:0|    3: <1, 14>               |    count 14;
-; CHECK-NEXT:   38:4|    3: <7, 32>               |    @t0 = i32;
-; CHECK-NEXT:   41:6|    3: <3>                   |    @t1 = float;
-; CHECK-NEXT:   43:4|    3: <12, 4, 0>            |    @t2 = <4 x i32>;
-; CHECK-NEXT:   46:6|    3: <7, 64>               |    @t3 = i64;
-; CHECK-NEXT:   50:0|    3: <4>                   |    @t4 = double;
-; CHECK-NEXT:   51:6|    3: <2>                   |    @t5 = void;
-; CHECK-NEXT:   53:4|    3: <21, 0, 5>            |    @t6 = void ();
-; CHECK-NEXT:   56:6|    3: <21, 0, 0, 0, 1, 0, 0,|    @t7 = 
-; CHECK-NEXT:       |        3>                   |        i32 (i32, float, i32, i32, i64)
-; CHECK-NEXT:       |                             |        ;
-; CHECK-NEXT:   63:6|    3: <21, 0, 3, 0, 0>      |    @t8 = i64 (i32, i32);
-; CHECK-NEXT:   68:4|    3: <21, 0, 4, 1>         |    @t9 = double (float);
-; CHECK-NEXT:   72:4|    3: <21, 0, 1, 4>         |    @t10 = float (double);
-; CHECK-NEXT:   76:4|    3: <21, 0, 5, 0>         |    @t11 = void (i32);
-; CHECK-NEXT:   80:4|    3: <21, 0, 2, 2, 2>      |    @t12 = 
-; CHECK-NEXT:       |                             |        <4 x i32> 
-; CHECK-NEXT:       |                             |        (<4 x i32>, <4 x i32>);
-; CHECK-NEXT:   85:2|    3: <21, 0, 0, 0>         |    @t13 = i32 (i32);
-; CHECK-NEXT:   89:2|  0: <65534>                 |  }
-; CHECK-NEXT:   92:0|  3: <8, 6, 0, 1, 0>         |  declare external void @f0();
-; CHECK-NEXT:   96:6|  3: <8, 7, 0, 1, 0>         |  declare external 
-; CHECK-NEXT:       |                             |      i32 
-; CHECK-NEXT:       |                             |      @f1(i32, float, i32, i32, i64);
-; CHECK-NEXT:  101:4|  3: <8, 8, 0, 1, 0>         |  declare external i64 @f2(i32, i32);
-; CHECK-NEXT:  106:2|  3: <8, 9, 0, 1, 0>         |  declare external double @f3(float);
-; CHECK-NEXT:  111:0|  3: <8, 10, 0, 1, 0>        |  declare external float @f4(double);
-; CHECK-NEXT:  115:6|  3: <8, 11, 0, 1, 0>        |  declare external void @f5(i32);
-; CHECK-NEXT:  120:4|  3: <8, 12, 0, 1, 0>        |  declare external 
-; CHECK-NEXT:       |                             |      <4 x i32> 
-; CHECK-NEXT:       |                             |      @f6(<4 x i32>, <4 x i32>);
-; CHECK-NEXT:  125:2|  3: <8, 13, 0, 1, 0>        |  declare external i32 @f7(i32);
-; CHECK-NEXT:  130:0|  1: <65535, 19, 2>          |  globals {  // BlockID = 19
-; CHECK-NEXT:  136:0|    3: <5, 0>                |    count 0;
-; CHECK-NEXT:  138:4|  0: <65534>                 |  }
-; CHECK-NEXT:  140:0|  1: <65535, 14, 2>          |  valuesymtab {  // BlockID = 14
-; CHECK-NEXT:  148:0|    3: <1, 1, 116, 101, 115, |    @f1 : "test";
-; CHECK-NEXT:       |        116>                 |
-; CHECK-NEXT:  156:4|    3: <1, 2, 109, 101, 114, |    @f2 : "merge";
-; CHECK-NEXT:       |        103, 101>            |
-; CHECK-NEXT:  166:4|    3: <1, 3, 70, 108, 111,  |    @f3 : "Float2Double";
-; CHECK-NEXT:       |        97, 116, 50, 68, 111,|
-; CHECK-NEXT:       |        117, 98, 108, 101>   |
-; CHECK-NEXT:  187:0|    3: <1, 4, 68, 111, 117,  |    @f4 : "Double2Float";
-; CHECK-NEXT:       |        98, 108, 101, 50, 70,|
-; CHECK-NEXT:       |        108, 111, 97, 116>   |
-; CHECK-NEXT:  207:4|    3: <1, 5, 105, 110, 100, |    @f5 : "indefargs";
-; CHECK-NEXT:       |        101, 102, 97, 114,   |
-; CHECK-NEXT:       |        103, 115>            |
-; CHECK-NEXT:  223:4|    3: <1, 0, 102, 117, 110, |    @f0 : "func";
-; CHECK-NEXT:       |        99>                  |
-; CHECK-NEXT:  232:0|    3: <1, 7, 102, 111, 111, |    @f7 : {'f', 'o', 'o',  36, 'b', 
-; CHECK-NEXT:       |        36, 98, 97, 114>     |           'a', 'r'}
-; CHECK-NEXT:  245:0|    3: <1, 6, 77, 117, 108>  |    @f6 : "Mul";
-; CHECK-NEXT:  252:0|  0: <65534>                 |  }
-; CHECK-NEXT:  256:0|0: <65534>                   |}
diff --git a/test/NaCl/Bitcode/pnacl-bcdis/unreachable.ll b/test/NaCl/Bitcode/pnacl-bcdis/unreachable.ll
deleted file mode 100644
index 9380299d915..00000000000
--- a/test/NaCl/Bitcode/pnacl-bcdis/unreachable.ll
+++ /dev/null
@@ -1,42 +0,0 @@
-; Test unreachable statement (without worrying if legal in context).
-
-; RUN: llvm-as < %s | pnacl-freeze | pnacl-bccompress --remove-abbreviations \
-; RUN:              | pnacl-bcdis | FileCheck %s
-
-
-
-; Note: This code only tests parsing of unreachable, not correctness
-; of use.
-define void @TestUnreach(i32 %p0) {
-
-; CHECK:               |                             |  %b0:
-
-  %v0 = trunc i32 %p0 to i1
-  br i1 %v0, label %b1, label %b2
-
-b1:
-  br i1 %v0, label %b3, label %b4
-
-b2:
-  br i1 %v0, label %b5, label %b3
-
-
-b3:
-  unreachable
-
-; CHECK:               |                             |  %b3:
-; CHECK-NEXT:    {{.*}}|    3: <15>                  |    unreachable;
-
-b4:
-  unreachable
-
-; CHECK:               |                             |  %b4:
-; CHECK-NEXT:    {{.*}}|    3: <15>                  |    unreachable;
-
-b5:
-  unreachable
-
-; CHECK:               |                             |  %b5:
-; CHECK-NEXT:    {{.*}}|    3: <15>                  |    unreachable;
-
-}
diff --git a/test/NaCl/Bitcode/pnacl-bcdis/vectorops.ll b/test/NaCl/Bitcode/pnacl-bcdis/vectorops.ll
deleted file mode 100644
index d55cbd37cff..00000000000
--- a/test/NaCl/Bitcode/pnacl-bcdis/vectorops.ll
+++ /dev/null
@@ -1,169 +0,0 @@
-; Tests vector inserts/extracts.
-
-; RUN: llvm-as < %s | pnacl-freeze | pnacl-bccompress --remove-abbreviations \
-; RUN:              | not pnacl-bcdis | FileCheck %s
-
-define void @GoodVectorOps() {
-
-; CHECK:         {{.*}}|    1: <65535, 11, 2>        |    constants {  // BlockID = 11
-; CHECK-NEXT:    {{.*}}|      3: <1, 0>              |      i32:
-; CHECK-NEXT:    {{.*}}|      3: <4, 2>              |        %c0 = i32 1;
-; CHECK-NEXT:    {{.*}}|      3: <4, 0>              |        %c1 = i32 0;
-; CHECK-NEXT:    {{.*}}|      3: <4, 20>             |        %c2 = i32 10;
-; CHECK-NEXT:    {{.*}}|      3: <1, 1>              |      i1:
-; CHECK-NEXT:    {{.*}}|      3: <4, 3>              |        %c3 = i1 1;
-; CHECK-NEXT:    {{.*}}|      3: <4, 0>              |        %c4 = i1 0;
-; CHECK-NEXT:    {{.*}}|      3: <1, 11>             |      <4 x i32>:
-; CHECK-NEXT:    {{.*}}|      3: <3>                 |        %c5 = <4 x i32> undef;
-; CHECK-NEXT:    {{.*}}|      3: <1, 3>              |      i8:
-; CHECK-NEXT:    {{.*}}|      3: <4, 0>              |        %c6 = i8 0;
-; CHECK-NEXT:    {{.*}}|      3: <4, 20>             |        %c7 = i8 10;
-; CHECK-NEXT:    {{.*}}|      3: <1, 4>              |      i16:
-; CHECK-NEXT:    {{.*}}|      3: <4, 2>              |        %c8 = i16 1;
-; CHECK-NEXT:    {{.*}}|      3: <4, 0>              |        %c9 = i16 0;
-; CHECK-NEXT:    {{.*}}|      3: <1, 10>             |      <8 x i16>:
-; CHECK-NEXT:    {{.*}}|      3: <3>                 |        %c10 = <8 x i16> undef;
-; CHECK-NEXT:    {{.*}}|      3: <1, 6>              |      <4 x i1>:
-; CHECK-NEXT:    {{.*}}|      3: <3>                 |        %c11 = <4 x i1> undef;
-; CHECK-NEXT:    {{.*}}|      3: <1, 7>              |      <8 x i1>:
-; CHECK-NEXT:    {{.*}}|      3: <3>                 |        %c12 = <8 x i1> undef;
-; CHECK-NEXT:    {{.*}}|      3: <1, 8>              |      <16 x i1>:
-; CHECK-NEXT:    {{.*}}|      3: <3>                 |        %c13 = <16 x i1> undef;
-; CHECK-NEXT:    {{.*}}|      3: <1, 9>              |      <16 x i8>:
-; CHECK-NEXT:    {{.*}}|      3: <3>                 |        %c14 = <16 x i8> undef;
-; CHECK-NEXT:    {{.*}}|      3: <1, 5>              |      <4 x float>:
-; CHECK-NEXT:    {{.*}}|      3: <3>                 |        %c15 = <4 x float> undef;
-; CHECK-NEXT:    {{.*}}|      3: <1, 2>              |      float:
-; CHECK-NEXT:    {{.*}}|      3: <6, 1065353216>     |        %c16 = float 1;
-; CHECK-NEXT:    {{.*}}|    0: <65534>               |      }
-; CHECK-NEXT:          |                             |  %b0:
-
-  %v0 = insertelement <4 x i1> undef, i1 1, i32 0
-  %v1 = insertelement <4 x i1> %v0, i1 0, i32 1
-  %v2 = extractelement <4 x i1> %v1, i32 0
-  %v3 = and i1 %v2, 1
-
-; CHECK-NEXT:    {{.*}}|    3: <7, 6, 14, 16>        |    %v0  =  
-; CHECK-NEXT:          |                             |        insertelement <4 x i1> %c11, 
-; CHECK-NEXT:          |                             |        i1 %c3, i32 %c1;
-; CHECK-NEXT:    {{.*}}|    3: <7, 1, 14, 18>        |    %v1  =  insertelement <4 x i1> %v0,
-; CHECK-NEXT:          |                             |        i1 %c4, i32 %c0;
-; CHECK-NEXT:    {{.*}}|    3: <6, 1, 18>            |    %v2  =  
-; CHECK-NEXT:          |                             |        extractelement <4 x i1> %v1, 
-; CHECK-NEXT:          |                             |        i32 %c1;
-; CHECK-NEXT:    {{.*}}|    3: <2, 1, 17, 10>        |    %v3 = and i1 %v2, %c3;
-
-  %v4 = insertelement <8 x i1> undef, i1 1, i32 1
-  %v5 = insertelement <8 x i1> %v4, i1 0, i32 0
-  %v6 = extractelement <8 x i1> %v5, i32 1
-  %v7 = and i1 %v6, 1
-
-; CHECK-NEXT:    {{.*}}|    3: <7, 9, 18, 21>        |    %v4  =  
-; CHECK-NEXT:          |                             |        insertelement <8 x i1> %c12, 
-; CHECK-NEXT:          |                             |        i1 %c3, i32 %c0;
-; CHECK-NEXT:    {{.*}}|    3: <7, 1, 18, 21>        |    %v5  =  insertelement <8 x i1> %v4,
-; CHECK-NEXT:          |                             |        i1 %c4, i32 %c1;
-; CHECK-NEXT:    {{.*}}|    3: <6, 1, 23>            |    %v6  =  
-; CHECK-NEXT:          |                             |        extractelement <8 x i1> %v5, 
-; CHECK-NEXT:          |                             |        i32 %c0;
-; CHECK-NEXT:    {{.*}}|    3: <2, 1, 21, 10>        |    %v7 = and i1 %v6, %c3;
-
-  %v8 = insertelement <16 x i1> undef, i1 1, i32 0
-  %v9 = insertelement <16 x i1> %v8, i1 0, i32 1
-  %v10 = extractelement <16 x i1> %v9, i32 1
-  %v11 = and i1 %v10, 1
-
-; CHECK-NEXT:    {{.*}}|    3: <7, 12, 22, 24>       |    %v8  =  
-; CHECK-NEXT:          |                             |        insertelement <16 x i1> %c13, 
-; CHECK-NEXT:          |                             |        i1 %c3, i32 %c1;
-; CHECK-NEXT:    {{.*}}|    3: <7, 1, 22, 26>        |    %v9  =  
-; CHECK-NEXT:          |                             |        insertelement <16 x i1> %v8, 
-; CHECK-NEXT:          |                             |        i1 %c4, i32 %c0;
-; CHECK-NEXT:    {{.*}}|    3: <6, 1, 27>            |    %v10  =  
-; CHECK-NEXT:          |                             |        extractelement <16 x i1> %v9, 
-; CHECK-NEXT:          |                             |        i32 %c0;
-; CHECK-NEXT:    {{.*}}|    3: <2, 1, 25, 10>        |    %v11 = and i1 %v10, %c3;
-  
-  %v12 = insertelement <16 x i8> undef, i8 0, i32 1
-  %v13 = insertelement <16 x i8> %v12, i8 10, i32 10
-  %v14 = extractelement <16 x i8> %v13, i32 1
-  %v15 = add i8 %v14, 0
-
-; CHECK-NEXT:    {{.*}}|    3: <7, 15, 23, 29>       |    %v12  =  
-; CHECK-NEXT:          |                             |        insertelement <16 x i8> %c14, 
-; CHECK-NEXT:          |                             |        i8 %c6, i32 %c0;
-; CHECK-NEXT:    {{.*}}|    3: <7, 1, 23, 28>        |    %v13  =  
-; CHECK-NEXT:          |                             |        insertelement <16 x i8> %v12, 
-; CHECK-NEXT:          |                             |        i8 %c7, i32 %c2;
-; CHECK-NEXT:    {{.*}}|    3: <6, 1, 31>            |    %v14  =  
-; CHECK-NEXT:          |                             |        extractelement <16 x i8> %v13, 
-; CHECK-NEXT:          |                             |        i32 %c0;
-; CHECK-NEXT:    {{.*}}|    3: <2, 1, 26, 0>         |    %v15 = add i8 %v14, %c6;
-
-  %v16 = insertelement <8 x i16> undef, i16 1, i32 1
-  %v17 = insertelement <8 x i16> %v16, i16 0, i32 0
-  %v18 = extractelement <8 x i16> %v17, i32 1
-  %v19 = add i16 %v18, 1
-
-; CHECK-NEXT:    {{.*}}|    3: <7, 23, 25, 33>       |    %v16  =  
-; CHECK-NEXT:          |                             |        insertelement <8 x i16> %c10, 
-; CHECK-NEXT:          |                             |        i16 %c8, i32 %c0;
-; CHECK-NEXT:    {{.*}}|    3: <7, 1, 25, 33>        |    %v17  =  
-; CHECK-NEXT:          |                             |        insertelement <8 x i16> %v16, 
-; CHECK-NEXT:          |                             |        i16 %c9, i32 %c1;
-; CHECK-NEXT:    {{.*}}|    3: <6, 1, 35>            |    %v18  =  
-; CHECK-NEXT:          |                             |        extractelement <8 x i16> %v17, 
-; CHECK-NEXT:          |                             |        i32 %c0;
-; CHECK-NEXT:    {{.*}}|    3: <2, 1, 28, 0>         |    %v19 = add i16 %v18, %c8;
-
-  %v20 = insertelement <4 x i32> undef, i32 1, i32 0
-  %v21 = insertelement <4 x i32> %v20, i32 0, i32 1
-  %v22 = extractelement <4 x i32> %v21, i32 0
-  %v23 = add i32 %v22, 1
-
-; CHECK-NEXT:    {{.*}}|    3: <7, 32, 37, 36>       |    %v20  =  
-; CHECK-NEXT:          |                             |        insertelement <4 x i32> %c5, 
-; CHECK-NEXT:          |                             |        i32 %c0, i32 %c1;
-; CHECK-NEXT:    {{.*}}|    3: <7, 1, 37, 38>        |    %v21  =  
-; CHECK-NEXT:          |                             |        insertelement <4 x i32> %v20, 
-; CHECK-NEXT:          |                             |        i32 %c1, i32 %c0;
-; CHECK-NEXT:    {{.*}}|    3: <6, 1, 38>            |    %v22  =  
-; CHECK-NEXT:          |                             |        extractelement <4 x i32> %v21, 
-; CHECK-NEXT:          |                             |        i32 %c1;
-; CHECK-NEXT:    {{.*}}|    3: <2, 1, 40, 0>         |    %v23 = add i32 %v22, %c0;
-
-  %v24 = insertelement <4 x float> undef, float 1.0, i32 0
-  %v25 = insertelement <4 x float> %v24, float 1.0, i32 1
-  %v26 = extractelement <4 x float> %v25, i32 0
-  %v27 = fadd float %v26, 1.0
-
-; CHECK-NEXT:    {{.*}}|    3: <7, 26, 25, 40>       |    %v24  =  
-; CHECK-NEXT:          |                             |        insertelement <4 x float> %c15,
-; CHECK-NEXT:          |                             |        float %c16, i32 %c1;
-; CHECK-NEXT:    {{.*}}|    3: <7, 1, 26, 42>        |    %v25  =  
-; CHECK-NEXT:          |                             |        insertelement <4 x float> %v24,
-; CHECK-NEXT:          |                             |        float %c16, i32 %c0;
-; CHECK-NEXT:    {{.*}}|    3: <6, 1, 42>            |    %v26  =  
-; CHECK-NEXT:          |                             |        extractelement <4 x float> %v25
-; CHECK-NEXT:          |                             |        , i32 %c1;
-; CHECK-NEXT:    {{.*}}|    3: <2, 1, 28, 0>         |    %v27 = fadd float %v26, %c16;
-
-  ret void
-; CHECK-NEXT:    {{.*}}|    3: <10>                  |    ret void;
-
-}
-
-define void @BadVectorOps(i32 %p0) {
-; CHECK:               |                             |  %b0:
-
-  %v0 = insertelement <4 x float> undef, float 1.0, i32 %p0
-
-; CHECK-NEXT:    {{.*}}|    3: <7, 1, 2, 3>          |    %v0  =  
-; CHECK-NEXT:          |                             |        insertelement <4 x float> %c1, 
-; CHECK-NEXT:          |                             |        float %c0, i32 %p0;
-; CHECK-NEXT:Error({{.*}}): insertelement: Vector index not constant: %p0
-
-  ret void
-; CHECK-NEXT:    {{.*}}|    3: <10>                  |    ret void;
-}
-
diff --git a/test/NaCl/Bitcode/ptrtoint-elide.ll b/test/NaCl/Bitcode/ptrtoint-elide.ll
deleted file mode 100644
index ffc22731418..00000000000
--- a/test/NaCl/Bitcode/ptrtoint-elide.ll
+++ /dev/null
@@ -1,811 +0,0 @@
-; Test how we handle eliding ptrtoint instructions.
-
-; RUN: llvm-as < %s | pnacl-freeze \
-; RUN:              | pnacl-bcanalyzer -dump-records \
-; RUN:              | FileCheck %s -check-prefix=PF2
-
-; RUN: llvm-as < %s | pnacl-freeze -allow-local-symbol-tables \
-; RUN:              | pnacl-thaw -allow-local-symbol-tables \
-; RUN:              | llvm-dis - | FileCheck %s -check-prefix=TD2
-
-; ------------------------------------------------------
-
-declare i32 @bar(i32)
-
-@bytes = internal global [4 x i8] c"abcd"
-
-; ------------------------------------------------------
-
-; Show simple case where we use ptrtoint
-define void @AllocCastSimple() {
-  %1 = alloca i8, i32 4, align 8
-  %2 = ptrtoint i8* %1 to i32
-  %3 = bitcast [4 x i8]* @bytes to i32*
-  store i32 %2, i32* %3, align 1
-  ret void
-}
-
-; TD2:      define void @AllocCastSimple() {
-; TD2-NEXT:   %1 = alloca i8, i32 4, align 8
-; TD2-NEXT:   %2 = ptrtoint i8* %1 to i32
-; TD2-NEXT:   %3 = bitcast [4 x i8]* @bytes to i32*
-; TD2-NEXT:   store i32 %2, i32* %3, align 1
-; TD2-NEXT:   ret void
-; TD2-NEXT: }
-
-; PF2:        <FUNCTION_BLOCK>
-; PF2:          </CONSTANTS_BLOCK>
-; PF2-NEXT:     <INST_ALLOCA op0=1 op1=4/>
-; PF2-NEXT:     <INST_STORE op0=3 op1=1 op2=1/>
-; PF2-NEXT:     <INST_RET/>
-; PF2-NEXT:   </FUNCTION_BLOCK>
-
-; ------------------------------------------------------
-
-; Same as above, but with the cast order changed. Shows
-; that we always inject casts back in a fixed order. Hence,
-; the casts will be reversed.
-define void @AllocCastSimpleReversed() {
-  %1 = alloca i8, i32 4, align 8
-  %2 = bitcast [4 x i8]* @bytes to i32*
-  %3 = ptrtoint i8* %1 to i32
-  store i32 %3, i32* %2, align 1
-  ret void
-}
-
-; TD2:      define void @AllocCastSimpleReversed() {
-; TD2-NEXT:   %1 = alloca i8, i32 4, align 8
-; TD2-NEXT:   %2 = ptrtoint i8* %1 to i32
-; TD2-NEXT:   %3 = bitcast [4 x i8]* @bytes to i32*
-; TD2-NEXT:   store i32 %2, i32* %3, align 1
-; TD2-NEXT:   ret void
-; TD2-NEXT: }
-
-; PF2:        <FUNCTION_BLOCK>
-; PF2:          </CONSTANTS_BLOCK>
-; PF2-NEXT:     <INST_ALLOCA op0=1 op1=4/>
-; PF2-NEXT:     <INST_STORE op0=3 op1=1 op2=1/>
-; PF2-NEXT:     <INST_RET/>
-; PF2-NEXT:   </FUNCTION_BLOCK>
-
-; ------------------------------------------------------
-
-; Show case where we delete ptrtoint because they aren't used.
-define void @AllocCastDelete() {
-  %1 = alloca i8, i32 4, align 8
-  %2 = ptrtoint i8* %1 to i32
-  %3 = alloca i8, i32 4, align 8
-  %4 = ptrtoint i8* %3 to i32
-  ret void
-}
-
-; TD2:      define void @AllocCastDelete() {
-; TD2-NEXT:   %1 = alloca i8, i32 4, align 8
-; TD2-NEXT:   %2 = alloca i8, i32 4, align 8
-; TD2-NEXT:   ret void
-; TD2-NEXT: }
-
-; PF2:        <FUNCTION_BLOCK>
-; PF2:          </CONSTANTS_BLOCK>
-; PF2-NEXT:     <INST_ALLOCA op0=1 op1=4/>
-; PF2-NEXT:     <INST_ALLOCA op0=2 op1=4/>
-; PF2-NEXT:     <INST_RET/>
-; PF2-NEXT:   </FUNCTION_BLOCK>
-
-; ------------------------------------------------------
-
-; Show case where we have optimized the ptrtoint (and bitcast) into a
-; single instruction, and will only be inserted before the first use
-; in the block.
-define void @AllocCastOpt() {
-  %1 = alloca i8, i32 4, align 8
-  %2 = bitcast [4 x i8]* @bytes to i32*
-  %3 = ptrtoint i8* %1 to i32
-  store i32 %3, i32* %2, align 1
-  store i32 %3, i32* %2, align 1
-  ret void
-}
-
-; TD2:      define void @AllocCastOpt() {
-; TD2-NEXT:   %1 = alloca i8, i32 4, align 8
-; TD2-NEXT:   %2 = ptrtoint i8* %1 to i32
-; TD2-NEXT:   %3 = bitcast [4 x i8]* @bytes to i32*
-; TD2-NEXT:   store i32 %2, i32* %3, align 1
-; TD2-NEXT:   store i32 %2, i32* %3, align 1
-; TD2-NEXT:   ret void
-; TD2-NEXT: }
-
-; PF2:        <FUNCTION_BLOCK>
-; PF2:          </CONSTANTS_BLOCK>
-; PF2-NEXT:     <INST_ALLOCA op0=1 op1=4/>
-; PF2-NEXT:     <INST_STORE op0=3 op1=1 op2=1/>
-; PF2-NEXT:     <INST_STORE op0=3 op1=1 op2=1/>
-; PF2-NEXT:     <INST_RET/>
-; PF2-NEXT:   </FUNCTION_BLOCK>
-
-; ------------------------------------------------------
-
-; Show case where ptrtoint (and bitcast) for store are not immediately
-; before the store, the casts will be moved to the store.
-define void @AllocCastMove(i32) {
-  %2 = alloca i8, i32 4, align 8
-  %3 = bitcast [4 x i8]* @bytes to i32*
-  %4 = ptrtoint i8* %2 to i32
-  %5 = add i32 %0, 1
-  store i32 %4, i32* %3, align 1
-  ret void
-}
-
-; TD2:      define void @AllocCastMove(i32) {
-; TD2-NEXT:   %2 = alloca i8, i32 4, align 8
-; TD2-NEXT:   %3 = add i32 %0, 1
-; TD2-NEXT:   %4 = ptrtoint i8* %2 to i32
-; TD2-NEXT:   %5 = bitcast [4 x i8]* @bytes to i32*
-; TD2-NEXT:   store i32 %4, i32* %5, align 1
-; TD2-NEXT:   ret void
-; TD2-NEXT: }
-
-; PF2:        <FUNCTION_BLOCK>
-; PF2:          </CONSTANTS_BLOCK>
-; PF2-NEXT:     <INST_ALLOCA op0=2 op1=4/>
-; PF2-NEXT:     <INST_BINOP op0=4 op1=2 op2=0/>
-; PF2-NEXT:     <INST_STORE op0=6 op1=2 op2=1/>
-; PF2-NEXT:     <INST_RET/>
-; PF2-NEXT:   </FUNCTION_BLOCK>
-
-; ------------------------------------------------------
-
-; Show case where ptrtoint on global variable is merged in a store, and
-; order is kept.
-define void @StoreGlobal() {
-  %1 = alloca i8, i32 4, align 8
-  %2 = ptrtoint [4 x i8]* @bytes to i32
-  %3 = bitcast i8* %1 to i32*
-  store i32 %2, i32* %3, align 1
-  ret void
-}
-
-; TD2:      define void @StoreGlobal() {
-; TD2-NEXT:   %1 = alloca i8, i32 4, align 8
-; TD2-NEXT:   %2 = ptrtoint [4 x i8]* @bytes to i32
-; TD2-NEXT:   %3 = bitcast i8* %1 to i32*
-; TD2-NEXT:   store i32 %2, i32* %3, align 1
-; TD2-NEXT:   ret void
-; TD2-NEXT: }
-
-; PF2:        <FUNCTION_BLOCK>
-; PF2:          </CONSTANTS_BLOCK>
-; PF2-NEXT:     <INST_ALLOCA op0=1 op1=4/>
-; PF2-NEXT:     <INST_STORE op0=1 op1=3 op2=1/>
-; PF2-NEXT:     <INST_RET/>
-; PF2-NEXT:   </FUNCTION_BLOCK>
-
-; ------------------------------------------------------
-
-; Same as above, but with cast order reversed.
-define void @StoreGlobalCastsReversed() {
-  %1 = alloca i8, i32 4, align 8
-  %2 = bitcast i8* %1 to i32*
-  %3 = ptrtoint [4 x i8]* @bytes to i32
-  store i32 %3, i32* %2, align 1
-  ret void
-}
-
-; TD2:      define void @StoreGlobalCastsReversed() {
-; TD2-NEXT:   %1 = alloca i8, i32 4, align 8
-; TD2-NEXT:   %2 = ptrtoint [4 x i8]* @bytes to i32
-; TD2-NEXT:   %3 = bitcast i8* %1 to i32*
-; TD2-NEXT:   store i32 %2, i32* %3, align 1
-; TD2-NEXT:   ret void
-; TD2-NEXT: }
-
-; PF2:        <FUNCTION_BLOCK>
-; PF2:          </CONSTANTS_BLOCK>
-; PF2-NEXT:     <INST_ALLOCA op0=1 op1=4/>
-; PF2-NEXT:     <INST_STORE op0=1 op1=3 op2=1/>
-; PF2-NEXT:     <INST_RET/>
-; PF2-NEXT:   </FUNCTION_BLOCK>
-
-; ------------------------------------------------------
-
-; Show that we will move the ptrtoint of a global to the use.
-define i32 @StoreGlobalMovePtr2Int() {
-  %1 = ptrtoint [4 x i8]* @bytes to i32
-  %2 = alloca i8, i32 4, align 8
-  %3 = bitcast i8* %2 to i32*
-  store i32 %1, i32* %3, align 1
-  ret i32 0
-}
-
-; TD2:      define i32 @StoreGlobalMovePtr2Int() {
-; TD2-NEXT:   %1 = alloca i8, i32 4, align 8
-; TD2-NEXT:   %2 = ptrtoint [4 x i8]* @bytes to i32
-; TD2-NEXT:   %3 = bitcast i8* %1 to i32*
-; TD2-NEXT:   store i32 %2, i32* %3, align 1
-; TD2-NEXT:   ret i32 0
-; TD2-NEXT: }
-
-; PF2:        <FUNCTION_BLOCK>
-; PF2:          </CONSTANTS_BLOCK>
-; PF2-NEXT:     <INST_ALLOCA op0=2 op1=4/>
-; PF2-NEXT:     <INST_STORE op0=1 op1=4 op2=1/>
-; PF2-NEXT:     <INST_RET op0=2/>
-; PF2-NEXT:   </FUNCTION_BLOCK>
-
-; ------------------------------------------------------
-
-; Show that we handle add instructions with pointer casts.
-define void @CastAddAlloca() {
-  %1 = alloca i8, i32 4, align 8
-  %2 = ptrtoint i8* %1 to i32
-
-  ; Simple add.
-  %3 = add i32 1, 2
-
-  ; Cast first.
-  %4 = add i32 %2, 2
-
-  ; Cast second.
-  %5 = add i32 1, %2
-
-  ; Cast both.
-  %6 = add i32 %2, %2
-
-  ret void
-}
-
-; TD2:      define void @CastAddAlloca() {
-; TD2-NEXT:   %1 = alloca i8, i32 4, align 8
-; TD2-NEXT:   %2 = add i32 1, 2
-; TD2-NEXT:   %3 = ptrtoint i8* %1 to i32
-; TD2-NEXT:   %4 = add i32 %3, 2
-; TD2-NEXT:   %5 = add i32 1, %3
-; TD2-NEXT:   %6 = add i32 %3, %3
-; TD2-NEXT:   ret void
-; TD2-NEXT: }
-
-; PF2:        <FUNCTION_BLOCK>
-; PF2:          </CONSTANTS_BLOCK>
-; PF2-NEXT:     <INST_ALLOCA op0=1 op1=4/>
-; PF2-NEXT:     <INST_BINOP op0=4 op1=3 op2=0/>
-; PF2-NEXT:     <INST_BINOP op0=2 op1=4 op2=0/>
-; PF2-NEXT:     <INST_BINOP op0=6 op1=3 op2=0/>
-; PF2-NEXT:     <INST_BINOP op0=4 op1=4 op2=0/>
-; PF2-NEXT:     <INST_RET/>
-; PF2-NEXT:   </FUNCTION_BLOCK>
-
-; ------------------------------------------------------
-
-; Show that we handle add instructions with pointer casts.
-define void @CastAddGlobal() {
-  %1 = ptrtoint [4 x i8]* @bytes to i32
-
-  ; Simple Add.
-  %2 = add i32 1, 2
-
-  ; Cast first.
-  %3 = add i32 %1, 2
-
-  ; Cast Second.
-  %4 = add i32 1, %1
-
-  ; Cast both.
-  %5 = add i32 %1, %1
-  ret void
-}
-
-; TD2:      define void @CastAddGlobal() {
-; TD2-NEXT:   %1 = add i32 1, 2
-; TD2-NEXT:   %2 = ptrtoint [4 x i8]* @bytes to i32
-; TD2-NEXT:   %3 = add i32 %2, 2
-; TD2-NEXT:   %4 = add i32 1, %2
-; TD2-NEXT:   %5 = add i32 %2, %2
-; TD2-NEXT:   ret void
-; TD2-NEXT: }
-
-; PF2:        <FUNCTION_BLOCK>
-; PF2:          </CONSTANTS_BLOCK>
-; PF2-NEXT:     <INST_BINOP op0=2 op1=1 op2=0/>
-; PF2-NEXT:     <INST_BINOP op0=4 op1=2 op2=0/>
-; PF2-NEXT:     <INST_BINOP op0=4 op1=5 op2=0/>
-; PF2-NEXT:     <INST_BINOP op0=6 op1=6 op2=0/>
-; PF2-NEXT:     <INST_RET/>
-; PF2-NEXT:   </FUNCTION_BLOCK>
-
-; ------------------------------------------------------
-
-; Show that we can handle pointer conversions for other scalar binary operators.
-define void @CastBinop() {
-  %1 = alloca i8, i32 4, align 8
-  %2 = ptrtoint i8* %1 to i32
-  %3 = ptrtoint [4 x i8]* @bytes to i32
-  %4 = sub i32 %2, %3
-  %5 = mul i32 %2, %3
-  %6 = udiv i32 %2, %3
-  %7 = urem i32 %2, %3
-  %8 = srem i32 %2, %3
-  %9 = shl i32 %2, %3
-  %10 = lshr i32 %2, %3
-  %11 = ashr i32 %2, %3
-  %12 = and i32 %2, %3
-  %13 = or i32 %2, %3
-  %14 = xor i32 %2, %3
-  ret void
-}
-
-; TD2:      define void @CastBinop() {
-; TD2-NEXT:   %1 = alloca i8, i32 4, align 8
-; TD2-NEXT:   %2 = ptrtoint i8* %1 to i32
-; TD2-NEXT:   %3 = ptrtoint [4 x i8]* @bytes to i32
-; TD2-NEXT:   %4 = sub i32 %2, %3
-; TD2-NEXT:   %5 = mul i32 %2, %3
-; TD2-NEXT:   %6 = udiv i32 %2, %3
-; TD2-NEXT:   %7 = urem i32 %2, %3
-; TD2-NEXT:   %8 = srem i32 %2, %3
-; TD2-NEXT:   %9 = shl i32 %2, %3
-; TD2-NEXT:   %10 = lshr i32 %2, %3
-; TD2-NEXT:   %11 = ashr i32 %2, %3
-; TD2-NEXT:   %12 = and i32 %2, %3
-; TD2-NEXT:   %13 = or i32 %2, %3
-; TD2-NEXT:   %14 = xor i32 %2, %3
-; TD2-NEXT:   ret void
-; TD2-NEXT: }
-
-; PF2:        <FUNCTION_BLOCK>
-; PF2:          </CONSTANTS_BLOCK>
-; PF2-NEXT:     <INST_ALLOCA op0=1 op1=4/>
-; PF2-NEXT:     <INST_BINOP op0=1 op1=3 op2=1/>
-; PF2-NEXT:     <INST_BINOP op0=2 op1=4 op2=2/>
-; PF2-NEXT:     <INST_BINOP op0=3 op1=5 op2=3/>
-; PF2-NEXT:     <INST_BINOP op0=4 op1=6 op2=5/>
-; PF2-NEXT:     <INST_BINOP op0=5 op1=7 op2=6/>
-; PF2-NEXT:     <INST_BINOP op0=6 op1=8 op2=7/>
-; PF2-NEXT:     <INST_BINOP op0=7 op1=9 op2=8/>
-; PF2-NEXT:     <INST_BINOP op0=8 op1=10 op2=9/>
-; PF2-NEXT:     <INST_BINOP op0=9 op1=11 op2=10/>
-; PF2-NEXT:     <INST_BINOP op0=10 op1=12 op2=11/>
-; PF2-NEXT:     <INST_BINOP op0=11 op1=13 op2=12/>
-; PF2-NEXT:     <INST_RET/>
-; PF2-NEXT:   </FUNCTION_BLOCK>
-
-; ------------------------------------------------------
-
-; Show that we handle (non-special) bitcasts by converting pointer
-; casts to integer.
-define void @TestCasts() {
-  %1 = alloca i8, i32 4, align 8
-  %2 = ptrtoint i8* %1 to i32
-
-  %3 = trunc i32 257 to i8
-  %4 = trunc i32 %2 to i8
-
-  %5 = zext i32 257 to i64
-  %6 = zext i32 %2 to i64
-
-  %7 = sext i32 -1 to i64
-  %8 = sext i32 %2 to i64
-
-  %9 = uitofp i32 1 to float
-  %10 = uitofp i32 %2 to float
-
-  %11 = sitofp i32 -1 to float
-  %12 = sitofp i32 %2 to float
-  ret void
-}
-
-; TD2:      define void @TestCasts() {
-; TD2-NEXT:   %1 = alloca i8, i32 4, align 8
-; TD2-NEXT:   %2 = trunc i32 257 to i8
-; TD2-NEXT:   %3 = ptrtoint i8* %1 to i32
-; TD2-NEXT:   %4 = trunc i32 %3 to i8
-; TD2-NEXT:   %5 = zext i32 257 to i64
-; TD2-NEXT:   %6 = zext i32 %3 to i64
-; TD2-NEXT:   %7 = sext i32 -1 to i64
-; TD2-NEXT:   %8 = sext i32 %3 to i64
-; TD2-NEXT:   %9 = uitofp i32 1 to float
-; TD2-NEXT:   %10 = uitofp i32 %3 to float
-; TD2-NEXT:   %11 = sitofp i32 -1 to float
-; TD2-NEXT:   %12 = sitofp i32 %3 to float
-; TD2-NEXT:   ret void
-; TD2-NEXT: }
-
-; PF2:        <FUNCTION_BLOCK>
-; PF2:          </CONSTANTS_BLOCK>
-; PF2-NEXT:     <INST_ALLOCA op0=2 op1=4/>
-; PF2-NEXT:     <INST_CAST op0=5 op1={{.*}} op2=0/>
-; PF2-NEXT:     <INST_CAST op0=2 op1={{.*}} op2=0/>
-; PF2-NEXT:     <INST_CAST op0=7 op1={{.*}} op2=1/>
-; PF2-NEXT:     <INST_CAST op0=4 op1={{.*}} op2=1/>
-; PF2-NEXT:     <INST_CAST op0=8 op1={{.*}} op2=2/>
-; PF2-NEXT:     <INST_CAST op0=6 op1={{.*}} op2=2/>
-; PF2-NEXT:     <INST_CAST op0=8 op1={{.*}} op2=5/>
-; PF2-NEXT:     <INST_CAST op0=8 op1={{.*}} op2=5/>
-; PF2-NEXT:     <INST_CAST op0=12 op1={{.*}} op2=6/>
-; PF2-NEXT:     <INST_CAST op0=10 op1={{.*}} op2=6/>
-; PF2-NEXT:     <INST_RET/>
-; PF2-NEXT:   </FUNCTION_BLOCK>
-
-; ------------------------------------------------------
-
-; Show that we elide a ptrtoint cast for a call.
-define void @TestSavedPtrToInt() {
-  %1 = alloca i8, i32 4, align 8
-  %2 = ptrtoint i8* %1 to i32
-  %3 = add i32 %2, 0
-  %4 = call i32 @bar(i32 %2)
-  ret void
-}
-
-; TD2:      define void @TestSavedPtrToInt() {
-; TD2-NEXT:   %1 = alloca i8, i32 4, align 8
-; TD2-NEXT:   %2 = ptrtoint i8* %1 to i32
-; TD2-NEXT:   %3 = add i32 %2, 0
-; TD2-NEXT:   %4 = call i32 @bar(i32 %2)
-; TD2-NEXT:   ret void
-; TD2-NEXT: }
-
-; PF2:        <FUNCTION_BLOCK>
-; PF2:          </CONSTANTS_BLOCK>
-; PF2-NEXT:     <INST_ALLOCA op0=2 op1=4/>
-; PF2-NEXT:     <INST_BINOP op0=1 op1=2 op2=0/>
-; PF2-NEXT:     <INST_CALL op0=0 op1=25 op2=2/>
-; PF2-NEXT:     <INST_RET/>
-; PF2-NEXT:   </FUNCTION_BLOCK>
-
-; ------------------------------------------------------
-
-; Show that we can handle pointer conversions for icmp.
-define void @CastIcmp() {
-  %1 = alloca i8, i32 4, align 8
-  %2 = ptrtoint i8* %1 to i32
-  %3 = ptrtoint [4 x i8]* @bytes to i32
-  %4 = icmp eq i32 1, 2
-  %5 = icmp eq i32 %2, 2
-  %6 = icmp eq i32 1, %3
-  %7 = icmp eq i32 %2, %3
-  %8 = icmp eq i32 %3, %2
-  ret void
-}
-
-; TD2:      define void @CastIcmp() {
-; TD2-NEXT:   %1 = alloca i8, i32 4, align 8
-; TD2-NEXT:   %2 = icmp eq i32 1, 2
-; TD2-NEXT:   %3 = ptrtoint i8* %1 to i32
-; TD2-NEXT:   %4 = icmp eq i32 %3, 2
-; TD2-NEXT:   %5 = ptrtoint [4 x i8]* @bytes to i32
-; TD2-NEXT:   %6 = icmp eq i32 1, %5
-; TD2-NEXT:   %7 = icmp eq i32 %3, %5
-; TD2-NEXT:   %8 = icmp eq i32 %5, %3
-; TD2-NEXT:   ret void
-; TD2-NEXT: }
-
-; PF2:        <FUNCTION_BLOCK>
-; PF2:          </CONSTANTS_BLOCK>
-; PF2-NEXT:     <INST_ALLOCA op0=1 op1=4/>
-; PF2-NEXT:     <INST_CMP2 op0=4 op1=3 op2=32/>
-; PF2-NEXT:     <INST_CMP2 op0=2 op1=4 op2=32/>
-; PF2-NEXT:     <INST_CMP2 op0=6 op1=7 op2=32/>
-; PF2-NEXT:     <INST_CMP2 op0=4 op1=8 op2=32/>
-; PF2-NEXT:     <INST_CMP2 op0=9 op1=5 op2=32/>
-; PF2-NEXT:     <INST_RET/>
-; PF2-NEXT:   </FUNCTION_BLOCK>
-
-; ------------------------------------------------------
-
-; Show that we can handle pointer conversions for Select.
-define void @CastSelect() {
-  %1 = alloca i8, i32 4, align 8
-  %2 = ptrtoint i8* %1 to i32
-  %3 = ptrtoint [4 x i8]* @bytes to i32
-  %4 = select i1 true, i32 1, i32 2
-  %5 = select i1 true, i32 %2, i32 2
-  %6 = select i1 true, i32 1, i32 %3
-  %7 = select i1 true, i32 %2, i32 %3
-  %8 = select i1 true, i32 %3, i32 %2
-  ret void
-}
-
-; TD2:      define void @CastSelect() {
-; TD2-NEXT:   %1 = alloca i8, i32 4, align 8
-; TD2-NEXT:   %2 = select i1 true, i32 1, i32 2
-; TD2-NEXT:   %3 = ptrtoint i8* %1 to i32
-; TD2-NEXT:   %4 = select i1 true, i32 %3, i32 2
-; TD2-NEXT:   %5 = ptrtoint [4 x i8]* @bytes to i32
-; TD2-NEXT:   %6 = select i1 true, i32 1, i32 %5
-; TD2-NEXT:   %7 = select i1 true, i32 %3, i32 %5
-; TD2-NEXT:   %8 = select i1 true, i32 %5, i32 %3
-; TD2-NEXT:   ret void
-; TD2-NEXT: }
-
-; PF2:        <FUNCTION_BLOCK>
-; PF2:          </CONSTANTS_BLOCK>
-; PF2-NEXT:     <INST_ALLOCA op0=2 op1=4/>
-; PF2-NEXT:     <INST_VSELECT op0=5 op1=4 op2=2/>
-; PF2-NEXT:     <INST_VSELECT op0=2 op1=5 op2=3/>
-; PF2-NEXT:     <INST_VSELECT op0=7 op1=8 op2=4/>
-; PF2-NEXT:     <INST_VSELECT op0=4 op1=9 op2=5/>
-; PF2-NEXT:     <INST_VSELECT op0=10 op1=5 op2=6/>
-; PF2-NEXT:     <INST_RET/>
-; PF2-NEXT:   </FUNCTION_BLOCK>
-
-; ------------------------------------------------------
-
-; Show that if a phi node refers to a pointer cast, we add
-; them at the end of the incoming block.
-define void @PhiBackwardRefs(i1) {
-  %2 = alloca i8, i32 4, align 8
-  %3 = bitcast i8* %2 to i32*
-  %4 = alloca i8, i32 4, align 8
-  %5 = ptrtoint i8* %4 to i32
-  br i1 %0, label %true, label %false
-
-true:
-  %6 = load i32, i32* %3
-  br label %merge
-
-false:
-  %7 = load i32, i32* %3
-  br label %merge
-
-merge:
-  %8 = phi i32 [%5, %true], [%5, %false]
-  %9 = phi i32 [%6, %true], [%7, %false]
-  ret void
-}
-
-; TD2:      define void @PhiBackwardRefs(i1) {
-; TD2-NEXT:   %2 = alloca i8, i32 4, align 8
-; TD2-NEXT:   %3 = alloca i8, i32 4, align 8
-; TD2-NEXT:   br i1 %0, label %true, label %false
-; TD2:      true:
-; TD2-NEXT:   %4 = bitcast i8* %2 to i32*
-; TD2-NEXT:   %5 = load i32, i32* %4
-; TD2-NEXT:   %6 = ptrtoint i8* %3 to i32
-; TD2-NEXT:   br label %merge
-; TD2:      false:
-; TD2-NEXT:   %7 = bitcast i8* %2 to i32*
-; TD2-NEXT:   %8 = load i32, i32* %7
-; TD2-NEXT:   %9 = ptrtoint i8* %3 to i32
-; TD2-NEXT:   br label %merge
-; TD2:      merge:
-; TD2-NEXT:   %10 = phi i32 [ %6, %true ], [ %9, %false ]
-; TD2-NEXT:   %11 = phi i32 [ %5, %true ], [ %8, %false ]
-; TD2-NEXT:   ret void
-; TD2-NEXT: }
-
-; PF2:        <FUNCTION_BLOCK>
-; PF2:          </CONSTANTS_BLOCK>
-; PF2-NEXT:     <INST_ALLOCA op0=1 op1=4/>
-; PF2-NEXT:     <INST_ALLOCA op0=2 op1=4/>
-; PF2-NEXT:     <INST_BR op0=1 op1=2 op2=4/>
-; PF2-NEXT:     <INST_LOAD op0=2 op1=0 op2=0/>
-; PF2-NEXT:     <INST_BR op0=3/>
-; PF2-NEXT:     <INST_LOAD op0=3 op1=0 op2=0/>
-; PF2-NEXT:     <INST_BR op0=3/>
-; PF2-NEXT:     <INST_PHI op0=0 op1=6 op2=1 op3=6 op4=2/>
-; PF2-NEXT:     <INST_PHI op0=0 op1=6 op2=1 op3=4 op4=2/>
-; PF2-NEXT:     <INST_RET/>
-; PF2:        </FUNCTION_BLOCK>
-
-; ------------------------------------------------------
-
-; Like PhiBackwardRefs except the phi nodes forward reference
-; instructions instead of backwards references.
-define void @PhiForwardRefs(i1) {
-  br label %start
-
-merge:
-  %2 = phi i32 [%9, %true], [%9, %false]
-  %3 = phi i32 [%4, %true], [%5, %false]
-  ret void
-
-true:
-  %4 = load i32, i32* %7
-  br label %merge
-
-false:
-  %5 = load i32, i32* %7
-  br label %merge
-
-start:
-  %6 = alloca i8, i32 4, align 8
-  %7 = bitcast i8* %6 to i32*
-  %8 = alloca i8, i32 4, align 8
-  %9 = ptrtoint i8* %8 to i32
-  br i1 %0, label %true, label %false
-}
-
-; TD2:      define void @PhiForwardRefs(i1) {
-; TD2-NEXT:   br label %start
-; TD2:      merge
-; TD2-NEXT:   %2 = phi i32 [ %11, %true ], [ %11, %false ]
-; TD2-NEXT:   %3 = phi i32 [ %5, %true ], [ %7, %false ]
-; TD2-NEXT:   ret void
-; TD2:      true:
-; TD2-NEXT:   %4 = inttoptr i32 %9 to i32*
-; TD2-NEXT:   %5 = load i32, i32* %4
-; TD2-NEXT:   br label %merge
-; TD2:      false:
-; TD2-NEXT:   %6 = inttoptr i32 %9 to i32*
-; TD2-NEXT:   %7 = load i32, i32* %6
-; TD2-NEXT:   br label %merge
-; TD2:      start:
-; TD2-NEXT:   %8 = alloca i8, i32 4, align 8
-; TD2-NEXT:   %9 = ptrtoint i8* %8 to i32
-; TD2-NEXT:   %10 = alloca i8, i32 4, align 8
-; TD2-NEXT:   %11 = ptrtoint i8* %10 to i32
-; TD2-NEXT:   br i1 %0, label %true, label %false
-; TD2-NEXT: }
-
-; PF2:        <FUNCTION_BLOCK>
-; PF2:          </CONSTANTS_BLOCK>
-; PF2-NEXT:     <INST_BR op0=4/>
-; PF2-NEXT:     <FORWARDTYPEREF op0=28 op1=0/>
-; PF2-NEXT:     <INST_PHI op0=0 op1=11 op2=2 op3=11 op4=3/>
-; PF2-NEXT:     <FORWARDTYPEREF op0=25 op1=0/>
-; PF2-NEXT:     <FORWARDTYPEREF op0=26 op1=0/>
-; PF2-NEXT:     <INST_PHI op0=0 op1=3 op2=2 op3=5 op4=3/>
-; PF2-NEXT:     <INST_RET/>
-; PF2-NEXT:     <FORWARDTYPEREF op0=27 op1=0/>
-; PF2-NEXT:     <INST_LOAD op0=4294967294 op1=0 op2=0/>
-; PF2-NEXT:     <INST_BR op0=1/>
-; PF2-NEXT:     <INST_LOAD op0=4294967295 op1=0 op2=0/>
-; PF2-NEXT:     <INST_BR op0=1/>
-; PF2-NEXT:     <INST_ALLOCA op0=5 op1=4/>
-; PF2-NEXT:     <INST_ALLOCA op0=6 op1=4/>
-; PF2-NEXT:     <INST_BR op0=2 op1=3 op2=8/>
-; PF2:        </FUNCTION_BLOCK>
-
-; ------------------------------------------------------
-
-; Show that if a phi node incoming block already has a pointer cast,
-; we use it instead of adding one at the end of the block. In this
-; example, we reuse instruction %7 in block true for phi node %10.
-define void @PhiMergeCast(i1) {
-  %2 = alloca i8, i32 4, align 8
-  %3 = bitcast i8* %2 to i32*
-  %4 = alloca i8, i32 4, align 8
-  %5 = ptrtoint i8* %4 to i32
-  br i1 %0, label %true, label %false
-
-true:
-  %6 = load i32, i32* %3
-  %7 = ptrtoint i8* %4 to i32
-  %8 = add i32 %6, %7
-  br label %merge
-
-false:
-  %9 = load i32, i32* %3
-  br label %merge
-
-merge:
-  %10 = phi i32 [%5, %true], [%5, %false]
-  %11 = phi i32 [%6, %true], [%9, %false]
-  ret void
-}
-
-; TD2:      define void @PhiMergeCast(i1) {
-; TD2-NEXT:   %2 = alloca i8, i32 4, align 8
-; TD2-NEXT:   %3 = alloca i8, i32 4, align 8
-; TD2-NEXT:   br i1 %0, label %true, label %false
-; TD2:      true:
-; TD2-NEXT:   %4 = bitcast i8* %2 to i32*
-; TD2-NEXT:   %5 = load i32, i32* %4
-; TD2-NEXT:   %6 = ptrtoint i8* %3 to i32
-; TD2-NEXT:   %7 = add i32 %5, %6
-; TD2-NEXT:   br label %merge
-; TD2:      false:
-; TD2-NEXT:   %8 = bitcast i8* %2 to i32*
-; TD2-NEXT:   %9 = load i32, i32* %8
-; TD2-NEXT:   %10 = ptrtoint i8* %3 to i32
-; TD2-NEXT:   br label %merge
-; TD2:      merge:
-; TD2-NEXT:   %11 = phi i32 [ %6, %true ], [ %10, %false ]
-; TD2-NEXT:   %12 = phi i32 [ %5, %true ], [ %9, %false ]
-; TD2-NEXT:   ret void
-; TD2-NEXT: }
-
-; PF2:        <FUNCTION_BLOCK>
-; PF2:          </CONSTANTS_BLOCK>
-; PF2-NEXT:     <INST_ALLOCA op0=1 op1=4/>
-; PF2-NEXT:     <INST_ALLOCA op0=2 op1=4/>
-; PF2-NEXT:     <INST_BR op0=1 op1=2 op2=4/>
-; PF2-NEXT:     <INST_LOAD op0=2 op1=0 op2=0/>
-; PF2-NEXT:     <INST_BINOP op0=1 op1=2 op2=0/>
-; PF2-NEXT:     <INST_BR op0=3/>
-; PF2-NEXT:     <INST_LOAD op0=4 op1=0 op2=0/>
-; PF2-NEXT:     <INST_BR op0=3/>
-; PF2-NEXT:     <INST_PHI op0=0 op1=8 op2=1 op3=8 op4=2/>
-; PF2-NEXT:     <INST_PHI op0=0 op1=8 op2=1 op3=4 op4=2/>
-; PF2-NEXT:     <INST_RET/>
-; PF2:        </FUNCTION_BLOCK>
-
-; ------------------------------------------------------
-
-; Show that we must introduce a cast reference for each
-; reachable block, but one is sufficient.
-define void @LongReachingCasts(i1) {
-  %2 = alloca i8, i32 4, align 8
-  %3 = ptrtoint i8* %2 to i32
-  %4 = bitcast [4 x i8]* @bytes to i32*
-  br i1 %0, label %Split1, label %Split2
-
-Split1:
-  br i1 %0, label %b1, label %b2
-
-Split2:
-  br i1 %0, label %b3, label %b4
-
-b1:
-  store i32 %3, i32* %4, align 1
-  store i32 %3, i32* %4, align 1
-  ret void
-
-b2:
-  store i32 %3, i32* %4, align 1
-  store i32 %3, i32* %4, align 1
-  ret void
-
-b3:
-  store i32 %3, i32* %4, align 1
-  store i32 %3, i32* %4, align 1
-  ret void
-
-b4:
-  store i32 %3, i32* %4, align 1
-  store i32 %3, i32* %4, align 1
-  ret void
-}
-
-; TD2:      define void @LongReachingCasts(i1) {
-; TD2-NEXT:   %2 = alloca i8, i32 4, align 8
-; TD2-NEXT:   br i1 %0, label %Split1, label %Split2
-; TD2:      Split1:
-; TD2-NEXT:   br i1 %0, label %b1, label %b2
-; TD2:      Split2:
-; TD2-NEXT:   br i1 %0, label %b3, label %b4
-; TD2:      b1:
-; TD2-NEXT:   %3 = ptrtoint i8* %2 to i32
-; TD2-NEXT:   %4 = bitcast [4 x i8]* @bytes to i32*
-; TD2-NEXT:   store i32 %3, i32* %4, align 1
-; TD2-NEXT:   store i32 %3, i32* %4, align 1
-; TD2-NEXT:   ret void
-; TD2:      b2:
-; TD2-NEXT:   %5 = ptrtoint i8* %2 to i32
-; TD2-NEXT:   %6 = bitcast [4 x i8]* @bytes to i32*
-; TD2-NEXT:   store i32 %5, i32* %6, align 1
-; TD2-NEXT:   store i32 %5, i32* %6, align 1
-; TD2-NEXT:   ret void
-; TD2:      b3:
-; TD2-NEXT:   %7 = ptrtoint i8* %2 to i32
-; TD2-NEXT:   %8 = bitcast [4 x i8]* @bytes to i32*
-; TD2-NEXT:   store i32 %7, i32* %8, align 1
-; TD2-NEXT:   store i32 %7, i32* %8, align 1
-; TD2-NEXT:   ret void
-; TD2:      b4:
-; TD2-NEXT:   %9 = ptrtoint i8* %2 to i32
-; TD2-NEXT:   %10 = bitcast [4 x i8]* @bytes to i32*
-; TD2-NEXT:   store i32 %9, i32* %10, align 1
-; TD2-NEXT:   store i32 %9, i32* %10, align 1
-; TD2-NEXT:   ret void
-; TD2-NEXT: }
-
-; PF2:        <FUNCTION_BLOCK>
-; PF2:          </CONSTANTS_BLOCK>
-; PF2-NEXT:     <INST_ALLOCA op0=1 op1=4/>
-; PF2-NEXT:     <INST_BR op0=1 op1=2 op2=3/>
-; PF2-NEXT:     <INST_BR op0=3 op1=4 op2=3/>
-; PF2-NEXT:     <INST_BR op0=5 op1=6 op2=3/>
-; PF2-NEXT:     <INST_STORE op0=4 op1=1 op2=1/>
-; PF2-NEXT:     <INST_STORE op0=4 op1=1 op2=1/>
-; PF2-NEXT:     <INST_RET/>
-; PF2-NEXT:     <INST_STORE op0=4 op1=1 op2=1/>
-; PF2-NEXT:     <INST_STORE op0=4 op1=1 op2=1/>
-; PF2-NEXT:     <INST_RET/>
-; PF2-NEXT:     <INST_STORE op0=4 op1=1 op2=1/>
-; PF2-NEXT:     <INST_STORE op0=4 op1=1 op2=1/>
-; PF2-NEXT:     <INST_RET/>
-; PF2-NEXT:     <INST_STORE op0=4 op1=1 op2=1/>
-; PF2-NEXT:     <INST_STORE op0=4 op1=1 op2=1/>
-; PF2-NEXT:     <INST_RET/>
-; PF2:        </FUNCTION_BLOCK>
diff --git a/test/NaCl/Bitcode/records-as-text.ll b/test/NaCl/Bitcode/records-as-text.ll
deleted file mode 100644
index 177dacfb420..00000000000
--- a/test/NaCl/Bitcode/records-as-text.ll
+++ /dev/null
@@ -1,165 +0,0 @@
-; Show that we can convert PNaCl bitcode into simplified record text
-; (for fuzzing).
-
-; RUN: llvm-as < %s | pnacl-freeze | pnacl-bcdis \
-; RUN:              | FileCheck %s -check-prefix=BC
-
-; RUN: llvm-as < %s | pnacl-freeze \
-; RUN:              | pnacl-bcfuzz -convert-to-text -output - \
-; RUN:              | FileCheck %s --check-prefix=TB
-
-; RUN: llvm-as < %s | pnacl-freeze -allow-local-symbol-tables \
-; RUN:              | pnacl-bcfuzz -convert-to-text -output - \
-; RUN:              | pnacl-bcfuzz -bitcode-as-text -output - \
-; RUN:              | pnacl-thaw -allow-local-symbol-tables \
-; RUN:              | llvm-dis - \
-; RUN:              | FileCheck %s --check-prefix=IN
-
-define i32 @fact(i32 %p0) {
-  %v0 = icmp ult i32 %p0, 1
-  br i1 %v0, label %true, label %false
-true:
-  ret i32 1
-false:
-  %v2 = sub i32 %p0, 1
-  %v3 = call i32 @fact(i32 %v2)
-  %v4 = mul i32 %v3, %p0
-  ret i32 %v4
-}
-
-; IN: define i32 @fact(i32 %p0) {
-; IN:   %v0 = icmp ult i32 %p0, 1
-; IN:   br i1 %v0, label %true, label %false
-; IN: true:
-; IN:   ret i32 1
-; IN: false:
-; IN:   %v2 = sub i32 %p0, 1
-; IN:   %v3 = call i32 @fact(i32 %v2)
-; IN:   %v4 = mul i32 %v3, %p0
-; IN:   ret i32 %v4
-; IN: }
-
-; BC:        0:0|<65532, 80, 69, 88, 69, 1, 0,|Magic Number: 'PEXE' (80, 69, 88, 69)
-; BC-NEXT:      | 8, 0, 17, 0, 4, 0, 2, 0, 0, |PNaCl Version: 2
-; BC-NEXT:      | 0>                          |
-; BC-NEXT:  16:0|1: <65535, 8, 2>             |module {  // BlockID = 8
-; BC-NEXT:  24:0|  3: <1, 1>                  |  version 1;
-; BC-NEXT:  26:4|  1: <65535, 0, 2>           |  abbreviations {  // BlockID = 0
-; BC-NEXT:  36:0|    3: <1, 14>               |    valuesymtab:
-; BC-NEXT:  38:4|    2: <65533, 4, 0, 1, 3, 0,|      @a0 = abbrev <fixed(3), vbr(8), 
-; BC-NEXT:      |        2, 8, 0, 3, 0, 1, 8> |                   array(fixed(8))>;
-; BC-NEXT:  43:2|    2: <65533, 4, 1, 1, 0, 2,|      @a1 = abbrev <1, vbr(8), 
-; BC-NEXT:      |        8, 0, 3, 0, 1, 7>    |                   array(fixed(7))>;
-; BC-NEXT:  48:0|    2: <65533, 4, 1, 1, 0, 2,|      @a2 = abbrev <1, vbr(8), 
-; BC-NEXT:      |        8, 0, 3, 0, 4>       |                   array(char6)>;
-; BC-NEXT:  52:1|    2: <65533, 4, 1, 2, 0, 2,|      @a3 = abbrev <2, vbr(8), 
-; BC-NEXT:      |        8, 0, 3, 0, 4>       |                   array(char6)>;
-; BC-NEXT:  56:2|    3: <1, 11>               |    constants:
-; BC-NEXT:  58:6|    2: <65533, 2, 1, 1, 0, 1,|      @a0 = abbrev <1, fixed(3)>;
-; BC-NEXT:      |        3>                   |
-; BC-NEXT:  61:7|    2: <65533, 2, 1, 4, 0, 2,|      @a1 = abbrev <4, vbr(8)>;
-; BC-NEXT:      |        8>                   |
-; BC-NEXT:  65:0|    2: <65533, 2, 1, 4, 1, 0>|      @a2 = abbrev <4, 0>;
-; BC-NEXT:  68:1|    2: <65533, 2, 1, 6, 0, 2,|      @a3 = abbrev <6, vbr(8)>;
-; BC-NEXT:      |        8>                   |
-; BC-NEXT:  71:2|    3: <1, 12>               |    function:
-; BC-NEXT:  73:6|    2: <65533, 4, 1, 20, 0,  |      @a0 = abbrev <20, vbr(6), vbr(4),
-; BC-NEXT:      |        2, 6, 0, 2, 4, 0, 2, |                   vbr(4)>;
-; BC-NEXT:      |        4>                   |
-; BC-NEXT:  79:1|    2: <65533, 4, 1, 2, 0, 2,|      @a1 = abbrev <2, vbr(6), vbr(6), 
-; BC-NEXT:      |        6, 0, 2, 6, 0, 1, 4> |                   fixed(4)>;
-; BC-NEXT:  84:4|    2: <65533, 4, 1, 3, 0, 2,|      @a2 = abbrev <3, vbr(6), 
-; BC-NEXT:      |        6, 0, 1, 3, 0, 1, 4> |                   fixed(3), fixed(4)>;
-; BC-NEXT:  89:7|    2: <65533, 1, 1, 10>     |      @a3 = abbrev <10>;
-; BC-NEXT:  91:7|    2: <65533, 2, 1, 10, 0,  |      @a4 = abbrev <10, vbr(6)>;
-; BC-NEXT:      |        2, 6>                |
-; BC-NEXT:  95:0|    2: <65533, 1, 1, 15>     |      @a5 = abbrev <15>;
-; BC-NEXT:  97:0|    2: <65533, 3, 1, 43, 0,  |      @a6 = abbrev <43, vbr(6), 
-; BC-NEXT:      |        2, 6, 0, 1, 3>       |                   fixed(3)>;
-; BC-NEXT: 101:2|    2: <65533, 4, 1, 24, 0,  |      @a7 = abbrev <24, vbr(6), vbr(6),
-; BC-NEXT:      |        2, 6, 0, 2, 6, 0, 2, |                   vbr(4)>;
-; BC-NEXT:      |        4>                   |
-; BC-NEXT: 106:5|    3: <1, 19>               |    globals:
-; BC-NEXT: 109:1|    2: <65533, 3, 1, 0, 0, 2,|      @a0 = abbrev <0, vbr(6), 
-; BC-NEXT:      |        6, 0, 1, 1>          |                   fixed(1)>;
-; BC-NEXT: 113:3|    2: <65533, 2, 1, 1, 0, 2,|      @a1 = abbrev <1, vbr(8)>;
-; BC-NEXT:      |        8>                   |
-; BC-NEXT: 116:4|    2: <65533, 2, 1, 2, 0, 2,|      @a2 = abbrev <2, vbr(8)>;
-; BC-NEXT:      |        8>                   |
-; BC-NEXT: 119:5|    2: <65533, 3, 1, 3, 0, 3,|      @a3 = abbrev <3, array(fixed(8))>
-; BC-NEXT:      |        0, 1, 8>             |          ;
-; BC-NEXT: 123:2|    2: <65533, 2, 1, 4, 0, 2,|      @a4 = abbrev <4, vbr(6)>;
-; BC-NEXT:      |        6>                   |
-; BC-NEXT: 126:3|    2: <65533, 3, 1, 4, 0, 2,|      @a5 = abbrev <4, vbr(6), vbr(6)>;
-; BC-NEXT:      |        6, 0, 2, 6>          |
-; BC-NEXT: 130:5|  0: <65534>                 |  }
-; BC-NEXT: 132:0|  1: <65535, 17, 3>          |  types {  // BlockID = 17
-; BC-NEXT: 140:0|    2: <65533, 4, 1, 21, 0,  |    %a0 = abbrev <21, fixed(1), 
-; BC-NEXT:      |        1, 1, 0, 3, 0, 1, 3> |                  array(fixed(3))>;
-; BC-NEXT: 144:7|    3: <1, 4>                |    count 4;
-; BC-NEXT: 147:4|    3: <7, 32>               |    @t0 = i32;
-; BC-NEXT: 150:7|    3: <2>                   |    @t1 = void;
-; BC-NEXT: 152:6|    4: <21, 0, 0, 0>         |    @t2 = i32 (i32); <%a0>
-; BC-NEXT: 154:6|    3: <7, 1>                |    @t3 = i1;
-; BC-NEXT: 157:3|  0: <65534>                 |  }
-; BC-NEXT: 160:0|  3: <8, 2, 0, 0, 0>         |  define external i32 @f0(i32);
-; BC-NEXT: 164:6|  1: <65535, 19, 4>          |  globals {  // BlockID = 19
-; BC-NEXT: 172:0|    3: <5, 0>                |    count 0;
-; BC-NEXT: 174:6|  0: <65534>                 |  }
-; BC-NEXT: 176:0|  1: <65535, 14, 3>          |  valuesymtab {  // BlockID = 14
-; BC-NEXT: 184:0|    6: <1, 0, 102, 97, 99,   |    @f0 : "fact"; <@a2>
-; BC-NEXT:      |        116>                 |
-; BC-NEXT: 189:1|  0: <65534>                 |  }
-; BC-NEXT: 192:0|  1: <65535, 12, 4>          |  function i32 @f0(i32 %p0) {  
-; BC-NEXT:      |                             |                   // BlockID = 12
-; BC-NEXT: 200:0|    3: <1, 3>                |    blocks 3;
-; BC-NEXT: 202:6|    1: <65535, 11, 3>        |    constants {  // BlockID = 11
-; BC-NEXT: 212:0|      4: <1, 0>              |      i32: <@a0>
-; BC-NEXT: 212:6|      5: <4, 2>              |        %c0 = i32 1; <@a1>
-; BC-NEXT: 214:1|    0: <65534>               |      }
-; BC-NEXT:      |                             |  %b0:
-; BC-NEXT: 216:0|    3: <28, 2, 1, 36>        |    %v0 = icmp ult i32 %p0, %c0;
-; BC-NEXT: 221:0|    3: <11, 1, 2, 1>         |    br i1 %v0, label %b1, label %b2;
-; BC-NEXT:      |                             |  %b1:
-; BC-NEXT: 225:2|    8: <10, 2>               |    ret i32 %c0; <@a4>
-; BC-NEXT:      |                             |  %b2:
-; BC-NEXT: 226:4|    5: <2, 3, 2, 1>          |    %v1 = sub i32 %p0, %c0; <@a1>
-; BC-NEXT: 229:0|    3: <34, 0, 5, 1>         |    %v2 = call i32 @f0(i32 %v1);
-; BC-NEXT: 234:0|    5: <2, 1, 5, 2>          |    %v3 = mul i32 %v2, %p0; <@a1>
-; BC-NEXT: 236:4|    8: <10, 1>               |    ret i32 %v3; <@a4>
-; BC-NEXT: 237:6|  0: <65534>                 |  }
-; BC-NEXT: 240:0|0: <65534>                   |}
-
-; TB:      65535,8,2;
-; TB-NEXT: 1,1;
-; NOTE: The blockinfo block has been removed.
-; TB-NEXT: 65535,17,2;
-; NOTE: The local type abbreviation has been removed.
-; TB-NEXT: 1,4;
-; TB-NEXT: 7,32;
-; TB-NEXT: 2;
-; TB-NEXT: 21,0,0,0;
-; TB-NEXT: 7,1;
-; TB-NEXT: 65534;
-; TB-NEXT: 8,2,0,0,0;
-; TB-NEXT: 65535,19,2;
-; TB-NEXT: 5,0;
-; TB-NEXT: 65534;
-; TB-NEXT: 65535,14,2;
-; TB-NEXT: 1,0,102,97,99,116;
-; TB-NEXT: 65534;
-; TB-NEXT: 65535,12,2;
-; TB-NEXT: 1,3;
-; TB-NEXT: 65535,11,2;
-; TB-NEXT: 1,0;
-; TB-NEXT: 4,2;
-; TB-NEXT: 65534;
-; TB-NEXT: 28,2,1,36;
-; TB-NEXT: 11,1,2,1;
-; TB-NEXT: 10,2;
-; TB-NEXT: 2,3,2,1;
-; TB-NEXT: 34,0,5,1;
-; TB-NEXT: 2,1,5,2;
-; TB-NEXT: 10,1;
-; TB-NEXT: 65534;
-; TB-NEXT: 65534;
diff --git a/test/NaCl/Bitcode/switch-ptr.ll b/test/NaCl/Bitcode/switch-ptr.ll
deleted file mode 100644
index bab44cd1ce8..00000000000
--- a/test/NaCl/Bitcode/switch-ptr.ll
+++ /dev/null
@@ -1,27 +0,0 @@
-; Tests if we handle pointer argument for switch statement.
-; https://code.google.com/p/nativeclient/issues/detail?id=4050
-
-; RUN: llvm-as < %s | pnacl-freeze \
-; RUN:              | pnacl-thaw \
-; RUN:              | llvm-dis -o - | FileCheck %s
-
-@hi = internal global [1 x i8] zeroinitializer, align 1
-define internal void @foo() {
-  %e = ptrtoint [1 x i8]* @hi to i32
-  switch i32 %e, label %baz3 [
-    i32 -4, label %baz1
-    i32 -8, label %baz2
-  ]
-baz1:
-  ret void
-baz2:
-  ret void
-baz3:
-  ret void
-}
-
-; CHECK-LABEL: @foo
-; CHECK:       %1 = ptrtoint [1 x i8]* @hi to i32
-; CHECK-NEXT:  switch i32 %1, label
-; CHECK-NEXT:     i32
-; CHECK-NEXT:     i32
diff --git a/test/NaCl/Bitcode/switch.ll b/test/NaCl/Bitcode/switch.ll
deleted file mode 100644
index 3d5ac91281c..00000000000
--- a/test/NaCl/Bitcode/switch.ll
+++ /dev/null
@@ -1,51 +0,0 @@
-; Test that we no longer put VECTOR/ARRAY type entries, associated with
-; switch instructions, into the bitcode file.
-
-; RUN: llvm-as < %s | pnacl-freeze \
-; RUN:              | pnacl-bcanalyzer -dump-records \
-; RUN:              | FileCheck %s -check-prefix=PF2
-
-; Test case where we switch on a variable.
-define void @SwitchVariable(i32) {
-  switch i32 %0, label %l1 [
-    i32 1, label %l2
-    i32 2, label %l2
-    i32 4, label %l3
-    i32 5, label %l3
-  ]
-  br label %end
-l1:
-  br label %end
-l2:
-  br label %end
-l3:
-  br label %end
-end:
-  ret void
-}
-
-; Test case where we switch on a constant.
-define void @SwitchConstant(i32) {
-  switch i32 3, label %l1 [
-    i32 1, label %l2
-    i32 2, label %l2
-    i32 4, label %l3
-    i32 5, label %l3
-  ]
-  br label %end
-l1:
-  br label %end
-l2:
-  br label %end
-l3:
-  br label %end
-end:
-  ret void
-}
-
-; PF2:      <TYPE_BLOCK_ID>
-; PF2-NEXT:   <NUMENTRY op0=3/>
-; PF2-NEXT:   <VOID/>
-; PF2-NEXT:   <INTEGER op0=32/>
-; PF2-NEXT:   <FUNCTION op0={{.*}} op1={{.*}} op2={{.*}}/>
-; PF2-NEXT: </TYPE_BLOCK_ID>
diff --git a/test/NaCl/Bitcode/unsupported.ll b/test/NaCl/Bitcode/unsupported.ll
deleted file mode 100644
index 9e37e65d2da..00000000000
--- a/test/NaCl/Bitcode/unsupported.ll
+++ /dev/null
@@ -1,98 +0,0 @@
-; Tests that check how we handle supported and unsupported versions of
-; the same input. Includes code that tests that we loose names for
-; pointer casts in local value symbol tables.
-
-define i8 @foo(i32 %i) {
-  %v1 = add i32 %i, %i
-  %v2 = inttoptr i32 %v1 to i8*
-  %v3 = load i8, i8* %v2
-  ret i8 %v3
-}
-
-; Test source effects of running with only supported bitcode features.
-; RUN: llvm-as < %s | pnacl-freeze | pnacl-thaw | llvm-dis - \
-; RUN:              | FileCheck %s -check-prefix=CSUP
-
-
-; CSUP:      define i8 @foo(i32) {
-; CSUP-NEXT:   %2 = add i32 %0, %0
-; CSUP-NEXT:   %3 = inttoptr i32 %2 to i8*
-; CSUP-NEXT:   %4 = load i8, i8* %3
-; CSUP-NEXT:   ret i8 %4
-; CSUP-NEXT: }
-
-; Test source effects of running with unsupported bitcode features.
-; RUN: llvm-as < %s | pnacl-freeze -allow-local-symbol-tables \
-; RUN:              | pnacl-thaw -allow-local-symbol-tables \
-; RUN:              | llvm-dis - | FileCheck %s -check-prefix=CUNS
-
-; CUNS:      define i8 @foo(i32 %i) {
-; CUNS-NEXT:   %v1 = add i32 %i, %i
-; CUNS-NEXT:   %1 = inttoptr i32 %v1 to i8*
-; CUNS-NEXT:   %v3 = load i8, i8* %1
-; CUNS-NEXT:   ret i8 %v3
-; CUNS-NEXT: }
-
-; Test dump effects of running with only supported bitcode features.
-; RUN: llvm-as < %s | pnacl-freeze | pnacl-bcanalyzer -dump-records \
-; RUN:              | FileCheck %s -check-prefix=DSUP
-
-; DSUP:      <MODULE_BLOCK>
-; DSUP-NEXT:   <VERSION op0=1/>
-; DSUP-NEXT:   <BLOCKINFO_BLOCK/>
-; DSUP-NEXT:   <TYPE_BLOCK_ID>
-; DSUP-NEXT:     <NUMENTRY op0=4/>
-; DSUP-NEXT:     <INTEGER op0=32/>
-; DSUP-NEXT:     <INTEGER op0=8/>
-; DSUP-NEXT:     <FUNCTION op0=0 op1=1 op2=0/>
-; DSUP-NEXT:     <VOID/>
-; DSUP-NEXT:   </TYPE_BLOCK_ID>
-; DSUP-NEXT:   <FUNCTION op0=2 op1=0 op2=0 op3=0/>
-; DSUP-NEXT:   <GLOBALVAR_BLOCK>
-; DSUP-NEXT:     <COUNT op0=0/>
-; DSUP-NEXT:   </GLOBALVAR_BLOCK>
-; DSUP-NEXT:   <VALUE_SYMTAB>
-; DSUP-NEXT:     <ENTRY op0=0 op1=102 op2=111 op3=111/>
-; DSUP-NEXT:   </VALUE_SYMTAB>
-; DSUP-NEXT:   <FUNCTION_BLOCK>
-; DSUP-NEXT:     <DECLAREBLOCKS op0=1/>
-; DSUP-NEXT:     <INST_BINOP op0=1 op1=1 op2=0/>
-; DSUP-NEXT:     <INST_LOAD op0=1 op1=0 op2=1/>
-; DSUP-NEXT:     <INST_RET op0=1/>
-; DSUP-NEXT:   </FUNCTION_BLOCK>
-; DSUP-NEXT: </MODULE_BLOCK>
-
-; Test dump effects of running with unsupported bitcode features.
-; RUN: llvm-as < %s | pnacl-freeze -allow-local-symbol-tables \
-; RUN:              | pnacl-bcanalyzer -dump-records \
-; RUN:              | FileCheck %s -check-prefix=DUNS
-
-; DUNS:      <MODULE_BLOCK>
-; DUNS-NEXT:   <VERSION op0=1/>
-; DUNS-NEXT:   <BLOCKINFO_BLOCK/>
-; DUNS-NEXT:   <TYPE_BLOCK_ID>
-; DUNS-NEXT:     <NUMENTRY op0=4/>
-; DUNS-NEXT:     <INTEGER op0=32/>
-; DUNS-NEXT:     <INTEGER op0=8/>
-; DUNS-NEXT:     <FUNCTION op0=0 op1=1 op2=0/>
-; DUNS-NEXT:     <VOID/>
-; DUNS-NEXT:   </TYPE_BLOCK_ID>
-; DUNS-NEXT:   <FUNCTION op0=2 op1=0 op2=0 op3=0/>
-; DUNS-NEXT:   <GLOBALVAR_BLOCK>
-; DUNS-NEXT:     <COUNT op0=0/>
-; DUNS-NEXT:   </GLOBALVAR_BLOCK>
-; DUNS-NEXT:   <VALUE_SYMTAB>
-; DUNS-NEXT:     <ENTRY op0=0 op1=102 op2=111 op3=111/>
-; DUNS-NEXT:   </VALUE_SYMTAB>
-; DUNS-NEXT:   <FUNCTION_BLOCK>
-; DUNS-NEXT:     <DECLAREBLOCKS op0=1/>
-; DUNS-NEXT:     <INST_BINOP op0=1 op1=1 op2=0/>
-; DUNS-NEXT:     <INST_LOAD op0=1 op1=0 op2=1/>
-; DUNS-NEXT:     <INST_RET op0=1/>
-; DUNS-NEXT:     <VALUE_SYMTAB>
-; DUNS-NEXT:       <ENTRY op0=2 op1=118 op2=49/>
-; DUNS-NEXT:       <ENTRY op0=1 op1=105/>
-; DUNS-NEXT:       <ENTRY op0=3 op1=118 op2=51/>
-; DUNS-NEXT:     </VALUE_SYMTAB>
-; DUNS-NEXT:   </FUNCTION_BLOCK>
-; DUNS-NEXT: </MODULE_BLOCK>
diff --git a/test/NaCl/Bitcode/vector.ll b/test/NaCl/Bitcode/vector.ll
deleted file mode 100644
index 544261c9c11..00000000000
--- a/test/NaCl/Bitcode/vector.ll
+++ /dev/null
@@ -1,57 +0,0 @@
-; Tests that vector operations survive through PNaCl bitcode files.
-
-; RUN: llvm-as < %s | pnacl-freeze | pnacl-thaw | llvm-dis - \
-; RUN:              | FileCheck %s
-
-define internal void @loadstore(i32 %addr) { ; CHECK-LABEL: loadstore
-  %ptr16xi8 = inttoptr i32 %addr to <16 x i8>*        ; CHECK-NEXT: %[[ptr16xi8:[0-9]+]] = inttoptr i32 %[[addr:[0-9]+]] to <16 x i8>*
-  %l16xi8 = load <16 x i8>, <16 x i8>* %ptr16xi8, align 1        ; CHECK-NEXT: %[[l16xi8:[0-9]+]] = load <16 x i8>, <16 x i8>* %[[ptr16xi8]], align 1
-  %ptr8xi16 = inttoptr i32 %addr to <8 x i16>*        ; CHECK-NEXT: %[[ptr8xi16:[0-9]+]] = inttoptr i32 %[[addr]] to <8 x i16>*
-  %l8xi16 = load <8 x i16>, <8 x i16>* %ptr8xi16, align 2        ; CHECK-NEXT: %[[l8xi16:[0-9]+]] = load <8 x i16>, <8 x i16>* %[[ptr8xi16]], align 2
-  %ptr4xi32 = inttoptr i32 %addr to <4 x i32>*        ; CHECK-NEXT: %[[ptr4xi32:[0-9]+]] = inttoptr i32 %[[addr]] to <4 x i32>*
-  %l4xi32 = load <4 x i32>, <4 x i32>* %ptr4xi32, align 4        ; CHECK-NEXT: %[[l4xi32:[0-9]+]] = load <4 x i32>, <4 x i32>* %[[ptr4xi32]], align 4
-  %ptr4xfloat = inttoptr i32 %addr to <4 x float>*    ; CHECK-NEXT: %[[ptr4xfloat:[0-9]+]] = inttoptr i32 %[[addr]] to <4 x float>*
-  %l4xfloat = load <4 x float>, <4 x float>* %ptr4xfloat, align 4  ; CHECK-NEXT: %[[l4xfloat:[0-9]+]] = load <4 x float>, <4 x float>* %[[ptr4xfloat]], align 4
-
-  store <16 x i8> undef, <16 x i8>* %ptr16xi8, align 1       ; CHECK-NEXT: store <16 x i8> undef, <16 x i8>* %[[ptr16xi8]], align 1
-  store <8 x i16> undef, <8 x i16>* %ptr8xi16, align 2       ; CHECK-NEXT: store <8 x i16> undef, <8 x i16>* %[[ptr8xi16]], align 2
-  store <4 x i32> undef, <4 x i32>* %ptr4xi32, align 4       ; CHECK-NEXT: store <4 x i32> undef, <4 x i32>* %[[ptr4xi32]], align 4
-  store <4 x float> undef, <4 x float>* %ptr4xfloat, align 4 ; CHECK-NEXT: store <4 x float> undef, <4 x float>* %[[ptr4xfloat]], align 4
-
-  ret void ; CHECK-NEXT: ret void
-}
-
-define internal void @binops() {      ; CHECK-LABEL: binops
-  %1 = add <4 x i32> undef, undef     ; CHECK-NEXT: %1 = add <4 x i32> undef, undef
-  %2 = fadd <4 x float> undef, undef  ; CHECK-NEXT: %2 = fadd <4 x float> undef, undef
-  %3 = sub <4 x i32> undef, undef     ; CHECK-NEXT: %3 = sub <4 x i32> undef, undef
-  %4 = fsub <4 x float> undef, undef  ; CHECK-NEXT: %4 = fsub <4 x float> undef, undef
-  %5 = mul <4 x i32> undef, undef     ; CHECK-NEXT: %5 = mul <4 x i32> undef, undef
-  %6 = fmul <4 x float> undef, undef  ; CHECK-NEXT: %6 = fmul <4 x float> undef, undef
-  %7 = udiv <4 x i32> undef, undef    ; CHECK-NEXT: %7 = udiv <4 x i32> undef, undef
-  %8 = sdiv <4 x i32> undef, undef    ; CHECK-NEXT: %8 = sdiv <4 x i32> undef, undef
-  %9 = fdiv <4 x float> undef, undef  ; CHECK-NEXT: %9 = fdiv <4 x float> undef, undef
-  %10 = urem <4 x i32> undef, undef   ; CHECK-NEXT: %10 = urem <4 x i32> undef, undef
-  %11 = srem <4 x i32> undef, undef   ; CHECK-NEXT: %11 = srem <4 x i32> undef, undef
-  %12 = frem <4 x float> undef, undef ; CHECK-NEXT: %12 = frem <4 x float> undef, undef
-  %13 = shl <4 x i32> undef, undef    ; CHECK-NEXT: %13 = shl <4 x i32> undef, undef
-  %14 = lshr <4 x i32> undef, undef   ; CHECK-NEXT: %14 = lshr <4 x i32> undef, undef
-  %15 = ashr <4 x i32> undef, undef   ; CHECK-NEXT: %15 = ashr <4 x i32> undef, undef
-  %16 = and <4 x i32> undef, undef    ; CHECK-NEXT: %16 = and <4 x i32> undef, undef
-  %17 = or <4 x i32> undef, undef     ; CHECK-NEXT: %17 = or <4 x i32> undef, undef
-  %18 = xor <4 x i32> undef, undef    ; CHECK-NEXT: %18 = xor <4 x i32> undef, undef
-  %19 = select <4 x i1> undef, <4 x i32> undef, <4 x i32> undef ; CHECK-NEXT: %19 = select <4 x i1> undef, <4 x i32> undef, <4 x i32> undef
-  ret void                            ; CHECK-NEXT: ret void
-}
-
-define internal void @insert_extract() { ; CHECK-LABEL: insert_extract
-  %1 = extractelement <4 x i32> undef, i32 0       ; CHECK-NEXT: %1 = extractelement <4 x i32> undef, i32 0
-  %2 = extractelement <4 x i32> undef, i32 1       ; CHECK-NEXT: %2 = extractelement <4 x i32> undef, i32 1
-  %3 = extractelement <4 x i32> undef, i32 2       ; CHECK-NEXT: %3 = extractelement <4 x i32> undef, i32 2
-  %4 = extractelement <4 x i32> undef, i32 3       ; CHECK-NEXT: %4 = extractelement <4 x i32> undef, i32 3
-  %5 = insertelement <4 x i32> undef, i32 1, i32 0 ; CHECK-NEXT: %5 = insertelement <4 x i32> undef, i32 1, i32 0
-  %6 = insertelement <4 x i32> undef, i32 1, i32 1 ; CHECK-NEXT: %6 = insertelement <4 x i32> undef, i32 1, i32 1
-  %7 = insertelement <4 x i32> undef, i32 1, i32 2 ; CHECK-NEXT: %7 = insertelement <4 x i32> undef, i32 1, i32 2
-  %8 = insertelement <4 x i32> undef, i32 1, i32 3 ; CHECK-NEXT: %8 = insertelement <4 x i32> undef, i32 1, i32 3
-  ret void                                         ; CHECK-NEXT: ret void
-}
diff --git a/test/NaCl/Localmods/lit.local.cfg b/test/NaCl/Localmods/lit.local.cfg
deleted file mode 100644
index c6106e4746f..00000000000
--- a/test/NaCl/Localmods/lit.local.cfg
+++ /dev/null
@@ -1 +0,0 @@
-config.suffixes = ['.ll']
diff --git a/test/NaCl/PNaClABI/abi-addrspace.ll b/test/NaCl/PNaClABI/abi-addrspace.ll
deleted file mode 100644
index 9a737b513a7..00000000000
--- a/test/NaCl/PNaClABI/abi-addrspace.ll
+++ /dev/null
@@ -1,16 +0,0 @@
-; RUN: not pnacl-abicheck < %s | FileCheck %s
-
-; This test checks that the "addrspace" pointer attribute is rejected
-; by the PNaCl ABI verifier.  The only allowed address space value is
-; 0 (the default).
-
-@var = addrspace(1) global [4 x i8] c"xxxx"
-; CHECK: Variable var has addrspace attribute (disallowed)
-
-define void @func() {
-  inttoptr i32 0 to i32 addrspace(2)*
-; CHECK: disallowed: bad result type: {{.*}} inttoptr {{.*}} addrspace
-  ret void
-}
-
-; CHECK-NOT: disallowed
diff --git a/test/NaCl/PNaClABI/abi-aliases.ll b/test/NaCl/PNaClABI/abi-aliases.ll
deleted file mode 100644
index a49f3f78973..00000000000
--- a/test/NaCl/PNaClABI/abi-aliases.ll
+++ /dev/null
@@ -1,7 +0,0 @@
-; RUN: not pnacl-abicheck < %s | FileCheck %s
-
-@aliased_var = internal global [1 x i8] c"x"
-; CHECK-NOT: disallowed
-
-@alias1 = alias [1 x i8]* @aliased_var
-; CHECK: Variable alias1 is an alias (disallowed)
diff --git a/test/NaCl/PNaClABI/abi-alignment.ll b/test/NaCl/PNaClABI/abi-alignment.ll
deleted file mode 100644
index 2a601b7153c..00000000000
--- a/test/NaCl/PNaClABI/abi-alignment.ll
+++ /dev/null
@@ -1,103 +0,0 @@
-; RUN: not pnacl-abicheck < %s | FileCheck %s
-
-; Test the "align" attributes that are allowed on load and store
-; instructions.
-
-
-declare void @llvm.memcpy.p0i8.p0i8.i32(i8*, i8*, i32, i32, i1)
-declare void @llvm.memmove.p0i8.p0i8.i32(i8*, i8*, i32, i32, i1)
-declare void @llvm.memset.p0i8.i32(i8*, i8, i32, i32, i1)
-
-
-define internal void @allowed_cases(i32 %ptr, float %f, double %d) {
-  %ptr.i32 = inttoptr i32 %ptr to i32*
-  load i32, i32* %ptr.i32, align 1
-  store i32 123, i32* %ptr.i32, align 1
-
-  %ptr.float = inttoptr i32 %ptr to float*
-  load float, float* %ptr.float, align 1
-  load float, float* %ptr.float, align 4
-  store float %f, float* %ptr.float, align 1
-  store float %f, float* %ptr.float, align 4
-
-  %ptr.double = inttoptr i32 %ptr to double*
-  load double, double* %ptr.double, align 1
-  load double, double* %ptr.double, align 8
-  store double %d, double* %ptr.double, align 1
-  store double %d, double* %ptr.double, align 8
-
-  ; memcpy() et el take an alignment parameter, which is allowed to be 1.
-  %ptr.p = inttoptr i32 %ptr to i8*
-  call void @llvm.memcpy.p0i8.p0i8.i32(i8* %ptr.p, i8* %ptr.p,
-                                       i32 10, i32 1, i1 false)
-  call void @llvm.memmove.p0i8.p0i8.i32(i8* %ptr.p, i8* %ptr.p,
-                                        i32 10, i32 1, i1 false)
-  call void @llvm.memset.p0i8.i32(i8* %ptr.p, i8 99,
-                                  i32 10, i32 1, i1 false)
-
-  ret void
-}
-; CHECK-NOT: disallowed
-
-
-define internal void @rejected_cases(i32 %ptr, float %f, double %d, i32 %align) {
-  %ptr.i32 = inttoptr i32 %ptr to i32*
-  load i32, i32* %ptr.i32, align 4
-  store i32 123, i32* %ptr.i32, align 4
-; CHECK: disallowed: bad alignment: {{.*}} load i32{{.*}} align 4
-; CHECK-NEXT: disallowed: bad alignment: store i32{{.*}} align 4
-
-  ; Unusual, not-very-useful alignments are rejected.
-  %ptr.float = inttoptr i32 %ptr to float*
-  load float, float* %ptr.float, align 2
-  load float, float* %ptr.float, align 8
-  store float %f, float* %ptr.float, align 2
-  store float %f, float* %ptr.float, align 8
-; CHECK-NEXT: disallowed: bad alignment: {{.*}} load float{{.*}} align 2
-; CHECK-NEXT: disallowed: bad alignment: {{.*}} load float{{.*}} align 8
-; CHECK-NEXT: disallowed: bad alignment: store float{{.*}} align 2
-; CHECK-NEXT: disallowed: bad alignment: store float{{.*}} align 8
-
-  %ptr.double = inttoptr i32 %ptr to double*
-  load double, double* %ptr.double, align 2
-  load double, double* %ptr.double, align 4
-  store double %d, double* %ptr.double, align 2
-  store double %d, double* %ptr.double, align 4
-; CHECK-NEXT: disallowed: bad alignment: {{.*}} load double{{.*}} align 2
-; CHECK-NEXT: disallowed: bad alignment: {{.*}} load double{{.*}} align 4
-; CHECK-NEXT: disallowed: bad alignment: store double{{.*}} align 2
-; CHECK-NEXT: disallowed: bad alignment: store double{{.*}} align 4
-
-  ; Non-pessimistic alignments for memcpy() et al are rejected.
-  %ptr.p = inttoptr i32 %ptr to i8*
-  call void @llvm.memcpy.p0i8.p0i8.i32(i8* %ptr.p, i8* %ptr.p,
-                                       i32 10, i32 4, i1 false)
-  call void @llvm.memmove.p0i8.p0i8.i32(i8* %ptr.p, i8* %ptr.p,
-                                        i32 10, i32 4, i1 false)
-  call void @llvm.memset.p0i8.i32(i8* %ptr.p, i8 99,
-                                  i32 10, i32 4, i1 false)
-; CHECK-NEXT: bad alignment: call void @llvm.memcpy
-; CHECK-NEXT: bad alignment: call void @llvm.memmove
-; CHECK-NEXT: bad alignment: call void @llvm.memset
-
-  ; Check that the verifier does not crash if the alignment argument
-  ; is not a constant.
-  call void @llvm.memcpy.p0i8.p0i8.i32(i8* %ptr.p, i8* %ptr.p,
-                                       i32 10, i32 %align, i1 false)
-  call void @llvm.memmove.p0i8.p0i8.i32(i8* %ptr.p, i8* %ptr.p,
-                                        i32 10, i32 %align, i1 false)
-  call void @llvm.memset.p0i8.i32(i8* %ptr.p, i8 99,
-                                  i32 10, i32 %align, i1 false)
-; CHECK-NEXT: bad alignment: call void @llvm.memcpy
-; CHECK-NEXT: bad alignment: call void @llvm.memmove
-; CHECK-NEXT: bad alignment: call void @llvm.memset
-
-  ret void
-}
-; CHECK-NOT: disallowed
-
-
-; This stops the verifier from complaining about the lack of an entry point.
-define void @_start(i32 %arg) {
-  ret void
-}
diff --git a/test/NaCl/PNaClABI/abi-arithmetic-attributes.ll b/test/NaCl/PNaClABI/abi-arithmetic-attributes.ll
deleted file mode 100644
index 6be379b104a..00000000000
--- a/test/NaCl/PNaClABI/abi-arithmetic-attributes.ll
+++ /dev/null
@@ -1,41 +0,0 @@
-; RUN: not pnacl-abicheck < %s | FileCheck %s
-
-; This tests that the arithmetic attributes "nuw" and "nsw" ("no
-; unsigned wrap" and "no signed wrap") and "exact" are disallowed by
-; the PNaCl ABI verifier.
-
-define internal void @allowed_cases() {
-  %add = add i32 1, 2
-  %shl = shl i32 3, 4
-  %udiv = udiv i32 4, 2
-  %lshr = lshr i32 2, 1
-  %ashr = ashr i32 2, 1
-  ret void
-}
-; CHECK-NOT: disallowed
-
-
-define internal void @rejected_cases() {
-  %add = add nsw i32 1, 2
-; CHECK: disallowed: has "nsw" attribute: %add
-  %shl1 = shl nuw i32 3, 4
-; CHECK-NEXT: disallowed: has "nuw" attribute: %shl1
-  %sub = sub nsw nuw i32 5, 6
-; CHECK-NEXT: disallowed: has "nuw" attribute: %sub
-
-  %lshr = lshr exact i32 2, 1
-; CHECK-NEXT: disallowed: has "exact" attribute: %lshr
-  %ashr = ashr exact i32 2, 1
-; CHECK-NEXT: disallowed: has "exact" attribute: %ashr
-  %udiv = udiv exact i32 4, 2
-; CHECK-NEXT: disallowed: has "exact" attribute: %udiv
-
-  ret void
-}
-; CHECK-NOT: disallowed
-
-
-; This stops the verifier from complaining about the lack of an entry point.
-define void @_start(i32 %arg) {
-  ret void
-}
diff --git a/test/NaCl/PNaClABI/abi-atomics.ll b/test/NaCl/PNaClABI/abi-atomics.ll
deleted file mode 100644
index 43295a93e76..00000000000
--- a/test/NaCl/PNaClABI/abi-atomics.ll
+++ /dev/null
@@ -1,535 +0,0 @@
-; RUN: not pnacl-abicheck < %s | FileCheck %s
-
-declare i8 @llvm.nacl.atomic.load.i8(i8*, i32)
-declare i16 @llvm.nacl.atomic.load.i16(i16*, i32)
-declare i32 @llvm.nacl.atomic.load.i32(i32*, i32)
-declare i64 @llvm.nacl.atomic.load.i64(i64*, i32)
-declare void @llvm.nacl.atomic.store.i8(i8, i8*, i32)
-declare void @llvm.nacl.atomic.store.i16(i16, i16*, i32)
-declare void @llvm.nacl.atomic.store.i32(i32, i32*, i32)
-declare void @llvm.nacl.atomic.store.i64(i64, i64*, i32)
-declare i8 @llvm.nacl.atomic.rmw.i8(i32, i8*, i8, i32)
-declare i16 @llvm.nacl.atomic.rmw.i16(i32, i16*, i16, i32)
-declare i32 @llvm.nacl.atomic.rmw.i32(i32, i32*, i32, i32)
-declare i64 @llvm.nacl.atomic.rmw.i64(i32, i64*, i64, i32)
-declare i8 @llvm.nacl.atomic.cmpxchg.i8(i8*, i8, i8, i32, i32)
-declare i16 @llvm.nacl.atomic.cmpxchg.i16(i16*, i16, i16, i32, i32)
-declare i32 @llvm.nacl.atomic.cmpxchg.i32(i32*, i32, i32, i32, i32)
-declare i64 @llvm.nacl.atomic.cmpxchg.i64(i64*, i64, i64, i32, i32)
-declare void @llvm.nacl.atomic.fence(i32)
-declare void @llvm.nacl.atomic.fence.all()
-declare i1 @llvm.nacl.atomic.is.lock.free(i32, i8*)
-
-; This stops the verifier from complaining about the lack of an entry point.
-define void @_start(i32 %arg) {
-  ret void
-}
-
-
-; Load
-
-define internal i32 @test_load_invalid_7() {
-  %ptr = inttoptr i32 undef to i32*
-  %1 = call i32 @llvm.nacl.atomic.load.i32(i32* %ptr, i32 7)
-  ret i32 %1
-}
-; CHECK: test_load_invalid_7 disallowed: invalid memory order
-
-define internal i32 @test_load_invalid_0() {
-  %ptr = inttoptr i32 undef to i32*
-  %1 = call i32 @llvm.nacl.atomic.load.i32(i32* %ptr, i32 0)
-  ret i32 %1
-}
-; CHECK: test_load_invalid_0 disallowed: invalid memory order
-
-define internal i32 @test_load_seqcst() {
-  %ptr = inttoptr i32 undef to i32*
-  %1 = call i32 @llvm.nacl.atomic.load.i32(i32* %ptr, i32 6)
-  ret i32 %1
-}
-; CHECK-NOT: disallowed
-
-define internal i32 @test_load_acqrel() {
-  %ptr = inttoptr i32 undef to i32*
-  %1 = call i32 @llvm.nacl.atomic.load.i32(i32* %ptr, i32 5)
-  ret i32 %1
-}
-; CHECK: test_load_acqrel disallowed: invalid memory order
-
-define internal i32 @test_load_release() {
-  %ptr = inttoptr i32 undef to i32*
-  %1 = call i32 @llvm.nacl.atomic.load.i32(i32* %ptr, i32 4)
-  ret i32 %1
-}
-; CHECK: test_load_release disallowed: invalid memory order
-
-define internal i32 @test_load_acquire() {
-  %ptr = inttoptr i32 undef to i32*
-  %1 = call i32 @llvm.nacl.atomic.load.i32(i32* %ptr, i32 3)
-  ret i32 %1
-}
-; CHECK-NOT: disallowed
-
-define internal i32 @test_load_consume() {
-  %ptr = inttoptr i32 undef to i32*
-  %1 = call i32 @llvm.nacl.atomic.load.i32(i32* %ptr, i32 2)
-  ret i32 %1
-}
-; CHECK: test_load_consume disallowed: invalid memory order
-
-define internal i32 @test_load_relaxed() {
-  %ptr = inttoptr i32 undef to i32*
-  %1 = call i32 @llvm.nacl.atomic.load.i32(i32* %ptr, i32 1)
-  ret i32 %1
-}
-; CHECK: test_load_relaxed disallowed: invalid memory order
-
-
-; Store
-
-define internal void @test_store_invalid_7() {
-  %ptr = inttoptr i32 undef to i32*
-  call void @llvm.nacl.atomic.store.i32(i32 undef, i32* %ptr, i32 7)
-  ret void
-}
-; CHECK: test_store_invalid_7 disallowed: invalid memory order
-
-define internal void @test_store_invalid_0() {
-  %ptr = inttoptr i32 undef to i32*
-  call void @llvm.nacl.atomic.store.i32(i32 undef, i32* %ptr, i32 0)
-  ret void
-}
-; CHECK: test_store_invalid_0 disallowed: invalid memory order
-
-define internal void @test_store_seqcst() {
-  %ptr = inttoptr i32 undef to i32*
-  call void @llvm.nacl.atomic.store.i32(i32 undef, i32* %ptr, i32 6)
-  ret void
-}
-; CHECK-NOT: disallowed
-
-define internal void @test_store_acqrel() {
-  %ptr = inttoptr i32 undef to i32*
-  call void @llvm.nacl.atomic.store.i32(i32 undef, i32* %ptr, i32 5)
-  ret void
-}
-; CHECK: test_store_acqrel disallowed: invalid memory order
-
-define internal void @test_store_release() {
-  %ptr = inttoptr i32 undef to i32*
-  call void @llvm.nacl.atomic.store.i32(i32 undef, i32* %ptr, i32 4)
-  ret void
-}
-; CHECK-NOT: disallowed
-
-define internal void @test_store_acquire() {
-  %ptr = inttoptr i32 undef to i32*
-  call void @llvm.nacl.atomic.store.i32(i32 undef, i32* %ptr, i32 3)
-  ret void
-}
-; CHECK: test_store_acquire disallowed: invalid memory order
-
-define internal void @test_store_consume() {
-  %ptr = inttoptr i32 undef to i32*
-  call void @llvm.nacl.atomic.store.i32(i32 undef, i32* %ptr, i32 2)
-  ret void
-}
-; CHECK: test_store_consume disallowed: invalid memory order
-
-define internal void @test_store_relaxed() {
-  %ptr = inttoptr i32 undef to i32*
-  call void @llvm.nacl.atomic.store.i32(i32 undef, i32* %ptr, i32 1)
-  ret void
-}
-; CHECK: test_store_relaxed disallowed: invalid memory order
-
-
-; rmw
-
-define internal i32 @test_rmw_invalid_7() {
-  %ptr = inttoptr i32 undef to i32*
-  %1 = call i32 @llvm.nacl.atomic.rmw.i32(i32 1, i32* %ptr, i32 0, i32 7)
-  ret i32 %1
-}
-; CHECK: test_rmw_invalid_7 disallowed: invalid memory order
-
-define internal i32 @test_rmw_invalid_0() {
-  %ptr = inttoptr i32 undef to i32*
-  %1 = call i32 @llvm.nacl.atomic.rmw.i32(i32 1, i32* %ptr, i32 0, i32 0)
-  ret i32 %1
-}
-; CHECK: test_rmw_invalid_0 disallowed: invalid memory order
-
-define internal i32 @test_rmw_seqcst() {
-  %ptr = inttoptr i32 undef to i32*
-  %1 = call i32 @llvm.nacl.atomic.rmw.i32(i32 1, i32* %ptr, i32 0, i32 6)
-  ret i32 %1
-}
-; CHECK-NOT: disallowed
-
-define internal i32 @test_rmw_acqrel() {
-  %ptr = inttoptr i32 undef to i32*
-  %1 = call i32 @llvm.nacl.atomic.rmw.i32(i32 1, i32* %ptr, i32 0, i32 5)
-  ret i32 %1
-}
-; CHECK-NOT: disallowed
-
-define internal i32 @test_rmw_release() {
-  %ptr = inttoptr i32 undef to i32*
-  %1 = call i32 @llvm.nacl.atomic.rmw.i32(i32 1, i32* %ptr, i32 0, i32 4)
-  ret i32 %1
-}
-; CHECK-NOT: disallowed
-
-define internal i32 @test_rmw_acquire() {
-  %ptr = inttoptr i32 undef to i32*
-  %1 = call i32 @llvm.nacl.atomic.rmw.i32(i32 1, i32* %ptr, i32 0, i32 3)
-  ret i32 %1
-}
-; CHECK-NOT: disallowed
-
-define internal i32 @test_rmw_consume() {
-  %ptr = inttoptr i32 undef to i32*
-  %1 = call i32 @llvm.nacl.atomic.rmw.i32(i32 1, i32* %ptr, i32 0, i32 2)
-  ret i32 %1
-}
-; CHECK: test_rmw_consume disallowed: invalid memory order
-
-define internal i32 @test_rmw_relaxed() {
-  %ptr = inttoptr i32 undef to i32*
-  %1 = call i32 @llvm.nacl.atomic.rmw.i32(i32 1, i32* %ptr, i32 0, i32 1)
-  ret i32 %1
-}
-; CHECK: test_rmw_relaxed disallowed: invalid memory order
-
-
-; cmpxchg
-
-define internal i32 @test_cmpxchg_invalid_7(i32 %oldval, i32 %newval) {
-  %ptr = inttoptr i32 undef to i32*
-  %1 = call i32 @llvm.nacl.atomic.cmpxchg.i32(i32* %ptr, i32 %oldval, i32 %newval, i32 7, i32 7)
-  ret i32 %1
-}
-; CHECK: test_cmpxchg_invalid_7 disallowed: invalid memory order
-
-define internal i32 @test_cmpxchg_invalid_0(i32 %oldval, i32 %newval) {
-  %ptr = inttoptr i32 undef to i32*
-  %1 = call i32 @llvm.nacl.atomic.cmpxchg.i32(i32* %ptr, i32 %oldval, i32 %newval, i32 0, i32 0)
-  ret i32 %1
-}
-; CHECK: test_cmpxchg_invalid_0 disallowed: invalid memory order
-
-; seq_cst
-
-define internal i32 @test_cmpxchg_seqcst_seqcst(i32 %oldval, i32 %newval) {
-  %ptr = inttoptr i32 undef to i32*
-  %1 = call i32 @llvm.nacl.atomic.cmpxchg.i32(i32* %ptr, i32 %oldval, i32 %newval, i32 6, i32 6)
-  ret i32 %1
-}
-; CHECK-NOT: disallowed
-
-define internal i32 @test_cmpxchg_seqcst_acqrel(i32 %oldval, i32 %newval) {
-  %ptr = inttoptr i32 undef to i32*
-  %1 = call i32 @llvm.nacl.atomic.cmpxchg.i32(i32* %ptr, i32 %oldval, i32 %newval, i32 6, i32 5)
-  ret i32 %1
-}
-; CHECK: test_cmpxchg_seqcst_acqrel disallowed: invalid memory order
-
-define internal i32 @test_cmpxchg_seqcst_release(i32 %oldval, i32 %newval) {
-  %ptr = inttoptr i32 undef to i32*
-  %1 = call i32 @llvm.nacl.atomic.cmpxchg.i32(i32* %ptr, i32 %oldval, i32 %newval, i32 6, i32 4)
-  ret i32 %1
-}
-; CHECK: test_cmpxchg_seqcst_release disallowed: invalid memory order
-
-define internal i32 @test_cmpxchg_seqcst_acquire(i32 %oldval, i32 %newval) {
-  %ptr = inttoptr i32 undef to i32*
-  %1 = call i32 @llvm.nacl.atomic.cmpxchg.i32(i32* %ptr, i32 %oldval, i32 %newval, i32 6, i32 3)
-  ret i32 %1
-}
-; CHECK-NOT: disallowed
-
-define internal i32 @test_cmpxchg_seqcst_consume(i32 %oldval, i32 %newval) {
-  %ptr = inttoptr i32 undef to i32*
-  %1 = call i32 @llvm.nacl.atomic.cmpxchg.i32(i32* %ptr, i32 %oldval, i32 %newval, i32 6, i32 2)
-  ret i32 %1
-}
-; CHECK: test_cmpxchg_seqcst_consume disallowed: invalid memory order
-
-define internal i32 @test_cmpxchg_seqcst_relaxed(i32 %oldval, i32 %newval) {
-  %ptr = inttoptr i32 undef to i32*
-  %1 = call i32 @llvm.nacl.atomic.cmpxchg.i32(i32* %ptr, i32 %oldval, i32 %newval, i32 6, i32 1)
-  ret i32 %1
-}
-; CHECK: test_cmpxchg_seqcst_relaxed disallowed: invalid memory order
-
-; acq_rel
-
-define internal i32 @test_cmpxchg_acqrel_seqcst(i32 %oldval, i32 %newval) {
-  %ptr = inttoptr i32 undef to i32*
-  %1 = call i32 @llvm.nacl.atomic.cmpxchg.i32(i32* %ptr, i32 %oldval, i32 %newval, i32 5, i32 6)
-  ret i32 %1
-}
-; CHECK: test_cmpxchg_acqrel_seqcst disallowed: invalid memory order
-
-define internal i32 @test_cmpxchg_acqrel_acqrel(i32 %oldval, i32 %newval) {
-  %ptr = inttoptr i32 undef to i32*
-  %1 = call i32 @llvm.nacl.atomic.cmpxchg.i32(i32* %ptr, i32 %oldval, i32 %newval, i32 5, i32 5)
-  ret i32 %1
-}
-; CHECK: test_cmpxchg_acqrel_acqrel disallowed: invalid memory order
-
-define internal i32 @test_cmpxchg_acqrel_release(i32 %oldval, i32 %newval) {
-  %ptr = inttoptr i32 undef to i32*
-  %1 = call i32 @llvm.nacl.atomic.cmpxchg.i32(i32* %ptr, i32 %oldval, i32 %newval, i32 5, i32 4)
-  ret i32 %1
-}
-; CHECK: test_cmpxchg_acqrel_release disallowed: invalid memory order
-
-define internal i32 @test_cmpxchg_acqrel_acquire(i32 %oldval, i32 %newval) {
-  %ptr = inttoptr i32 undef to i32*
-  %1 = call i32 @llvm.nacl.atomic.cmpxchg.i32(i32* %ptr, i32 %oldval, i32 %newval, i32 5, i32 3)
-  ret i32 %1
-}
-; CHECK-NOT: disallowed
-
-define internal i32 @test_cmpxchg_acqrel_consume(i32 %oldval, i32 %newval) {
-  %ptr = inttoptr i32 undef to i32*
-  %1 = call i32 @llvm.nacl.atomic.cmpxchg.i32(i32* %ptr, i32 %oldval, i32 %newval, i32 5, i32 2)
-  ret i32 %1
-}
-; CHECK: test_cmpxchg_acqrel_consume disallowed: invalid memory order
-
-define internal i32 @test_cmpxchg_acqrel_relaxed(i32 %oldval, i32 %newval) {
-  %ptr = inttoptr i32 undef to i32*
-  %1 = call i32 @llvm.nacl.atomic.cmpxchg.i32(i32* %ptr, i32 %oldval, i32 %newval, i32 5, i32 1)
-  ret i32 %1
-}
-; CHECK: test_cmpxchg_acqrel_relaxed disallowed: invalid memory order
-
-; release
-
-define internal i32 @test_cmpxchg_release_seqcst(i32 %oldval, i32 %newval) {
-  %ptr = inttoptr i32 undef to i32*
-  %1 = call i32 @llvm.nacl.atomic.cmpxchg.i32(i32* %ptr, i32 %oldval, i32 %newval, i32 4, i32 6)
-  ret i32 %1
-}
-; CHECK: test_cmpxchg_release_seqcst disallowed: invalid memory order
-
-define internal i32 @test_cmpxchg_release_acqrel(i32 %oldval, i32 %newval) {
-  %ptr = inttoptr i32 undef to i32*
-  %1 = call i32 @llvm.nacl.atomic.cmpxchg.i32(i32* %ptr, i32 %oldval, i32 %newval, i32 4, i32 5)
-  ret i32 %1
-}
-; CHECK: test_cmpxchg_release_acqrel disallowed: invalid memory order
-
-define internal i32 @test_cmpxchg_release_release(i32 %oldval, i32 %newval) {
-  %ptr = inttoptr i32 undef to i32*
-  %1 = call i32 @llvm.nacl.atomic.cmpxchg.i32(i32* %ptr, i32 %oldval, i32 %newval, i32 4, i32 4)
-  ret i32 %1
-}
-; CHECK: test_cmpxchg_release_release disallowed: invalid memory order
-
-define internal i32 @test_cmpxchg_release_acquire(i32 %oldval, i32 %newval) {
-  %ptr = inttoptr i32 undef to i32*
-  %1 = call i32 @llvm.nacl.atomic.cmpxchg.i32(i32* %ptr, i32 %oldval, i32 %newval, i32 4, i32 3)
-  ret i32 %1
-}
-; CHECK: test_cmpxchg_release_acquire disallowed: invalid memory order
-
-define internal i32 @test_cmpxchg_release_consume(i32 %oldval, i32 %newval) {
-  %ptr = inttoptr i32 undef to i32*
-  %1 = call i32 @llvm.nacl.atomic.cmpxchg.i32(i32* %ptr, i32 %oldval, i32 %newval, i32 4, i32 2)
-  ret i32 %1
-}
-; CHECK: test_cmpxchg_release_consume disallowed: invalid memory order
-
-define internal i32 @test_cmpxchg_release_relaxed(i32 %oldval, i32 %newval) {
-  %ptr = inttoptr i32 undef to i32*
-  %1 = call i32 @llvm.nacl.atomic.cmpxchg.i32(i32* %ptr, i32 %oldval, i32 %newval, i32 4, i32 1)
-  ret i32 %1
-}
-; CHECK: test_cmpxchg_release_relaxed disallowed: invalid memory order
-
-; acquire
-
-define internal i32 @test_cmpxchg_acquire_seqcst(i32 %oldval, i32 %newval) {
-  %ptr = inttoptr i32 undef to i32*
-  %1 = call i32 @llvm.nacl.atomic.cmpxchg.i32(i32* %ptr, i32 %oldval, i32 %newval, i32 3, i32 6)
-  ret i32 %1
-}
-; CHECK: test_cmpxchg_acquire_seqcst disallowed: invalid memory order
-
-define internal i32 @test_cmpxchg_acquire_acqrel(i32 %oldval, i32 %newval) {
-  %ptr = inttoptr i32 undef to i32*
-  %1 = call i32 @llvm.nacl.atomic.cmpxchg.i32(i32* %ptr, i32 %oldval, i32 %newval, i32 3, i32 5)
-  ret i32 %1
-}
-; CHECK: test_cmpxchg_acquire_acqrel disallowed: invalid memory order
-
-define internal i32 @test_cmpxchg_acquire_release(i32 %oldval, i32 %newval) {
-  %ptr = inttoptr i32 undef to i32*
-  %1 = call i32 @llvm.nacl.atomic.cmpxchg.i32(i32* %ptr, i32 %oldval, i32 %newval, i32 3, i32 4)
-  ret i32 %1
-}
-; CHECK: test_cmpxchg_acquire_release disallowed: invalid memory order
-
-define internal i32 @test_cmpxchg_acquire_acquire(i32 %oldval, i32 %newval) {
-  %ptr = inttoptr i32 undef to i32*
-  %1 = call i32 @llvm.nacl.atomic.cmpxchg.i32(i32* %ptr, i32 %oldval, i32 %newval, i32 3, i32 3)
-  ret i32 %1
-}
-; CHECK-NOT: disallowed
-
-define internal i32 @test_cmpxchg_acquire_consume(i32 %oldval, i32 %newval) {
-  %ptr = inttoptr i32 undef to i32*
-  %1 = call i32 @llvm.nacl.atomic.cmpxchg.i32(i32* %ptr, i32 %oldval, i32 %newval, i32 3, i32 2)
-  ret i32 %1
-}
-; CHECK: test_cmpxchg_acquire_consume disallowed: invalid memory order
-
-define internal i32 @test_cmpxchg_acquire_relaxed(i32 %oldval, i32 %newval) {
-  %ptr = inttoptr i32 undef to i32*
-  %1 = call i32 @llvm.nacl.atomic.cmpxchg.i32(i32* %ptr, i32 %oldval, i32 %newval, i32 3, i32 1)
-  ret i32 %1
-}
-; CHECK: test_cmpxchg_acquire_relaxed disallowed: invalid memory order
-
-; consume
-
-define internal i32 @test_cmpxchg_consume_seqcst(i32 %oldval, i32 %newval) {
-  %ptr = inttoptr i32 undef to i32*
-  %1 = call i32 @llvm.nacl.atomic.cmpxchg.i32(i32* %ptr, i32 %oldval, i32 %newval, i32 2, i32 6)
-  ret i32 %1
-}
-; CHECK: test_cmpxchg_consume_seqcst disallowed: invalid memory order
-
-define internal i32 @test_cmpxchg_consume_acqrel(i32 %oldval, i32 %newval) {
-  %ptr = inttoptr i32 undef to i32*
-  %1 = call i32 @llvm.nacl.atomic.cmpxchg.i32(i32* %ptr, i32 %oldval, i32 %newval, i32 2, i32 5)
-  ret i32 %1
-}
-; CHECK: test_cmpxchg_consume_acqrel disallowed: invalid memory order
-
-define internal i32 @test_cmpxchg_consume_release(i32 %oldval, i32 %newval) {
-  %ptr = inttoptr i32 undef to i32*
-  %1 = call i32 @llvm.nacl.atomic.cmpxchg.i32(i32* %ptr, i32 %oldval, i32 %newval, i32 2, i32 4)
-  ret i32 %1
-}
-; CHECK: test_cmpxchg_consume_release disallowed: invalid memory order
-
-define internal i32 @test_cmpxchg_consume_acquire(i32 %oldval, i32 %newval) {
-  %ptr = inttoptr i32 undef to i32*
-  %1 = call i32 @llvm.nacl.atomic.cmpxchg.i32(i32* %ptr, i32 %oldval, i32 %newval, i32 2, i32 3)
-  ret i32 %1
-}
-; CHECK: test_cmpxchg_consume_acquire disallowed: invalid memory order
-
-define internal i32 @test_cmpxchg_consume_consume(i32 %oldval, i32 %newval) {
-  %ptr = inttoptr i32 undef to i32*
-  %1 = call i32 @llvm.nacl.atomic.cmpxchg.i32(i32* %ptr, i32 %oldval, i32 %newval, i32 2, i32 2)
-  ret i32 %1
-}
-; CHECK: test_cmpxchg_consume_consume disallowed: invalid memory order
-
-define internal i32 @test_cmpxchg_consume_relaxed(i32 %oldval, i32 %newval) {
-  %ptr = inttoptr i32 undef to i32*
-  %1 = call i32 @llvm.nacl.atomic.cmpxchg.i32(i32* %ptr, i32 %oldval, i32 %newval, i32 2, i32 1)
-  ret i32 %1
-}
-; CHECK: test_cmpxchg_consume_relaxed disallowed: invalid memory order
-
-; relaxed
-
-define internal i32 @test_cmpxchg_relaxed_seqcst(i32 %oldval, i32 %newval) {
-  %ptr = inttoptr i32 undef to i32*
-  %1 = call i32 @llvm.nacl.atomic.cmpxchg.i32(i32* %ptr, i32 %oldval, i32 %newval, i32 1, i32 6)
-  ret i32 %1
-}
-; CHECK: test_cmpxchg_relaxed_seqcst disallowed: invalid memory order
-
-define internal i32 @test_cmpxchg_relaxed_acqrel(i32 %oldval, i32 %newval) {
-  %ptr = inttoptr i32 undef to i32*
-  %1 = call i32 @llvm.nacl.atomic.cmpxchg.i32(i32* %ptr, i32 %oldval, i32 %newval, i32 1, i32 5)
-  ret i32 %1
-}
-; CHECK: test_cmpxchg_relaxed_acqrel disallowed: invalid memory order
-
-define internal i32 @test_cmpxchg_relaxed_release(i32 %oldval, i32 %newval) {
-  %ptr = inttoptr i32 undef to i32*
-  %1 = call i32 @llvm.nacl.atomic.cmpxchg.i32(i32* %ptr, i32 %oldval, i32 %newval, i32 1, i32 4)
-  ret i32 %1
-}
-; CHECK: test_cmpxchg_relaxed_release disallowed: invalid memory order
-
-define internal i32 @test_cmpxchg_relaxed_acquire(i32 %oldval, i32 %newval) {
-  %ptr = inttoptr i32 undef to i32*
-  %1 = call i32 @llvm.nacl.atomic.cmpxchg.i32(i32* %ptr, i32 %oldval, i32 %newval, i32 1, i32 3)
-  ret i32 %1
-}
-; CHECK: test_cmpxchg_relaxed_acquire disallowed: invalid memory order
-
-define internal i32 @test_cmpxchg_relaxed_consume(i32 %oldval, i32 %newval) {
-  %ptr = inttoptr i32 undef to i32*
-  %1 = call i32 @llvm.nacl.atomic.cmpxchg.i32(i32* %ptr, i32 %oldval, i32 %newval, i32 1, i32 2)
-  ret i32 %1
-}
-; CHECK: test_cmpxchg_relaxed_consume disallowed: invalid memory order
-
-define internal i32 @test_cmpxchg_relaxed_relaxed(i32 %oldval, i32 %newval) {
-  %ptr = inttoptr i32 undef to i32*
-  %1 = call i32 @llvm.nacl.atomic.cmpxchg.i32(i32* %ptr, i32 %oldval, i32 %newval, i32 1, i32 1)
-  ret i32 %1
-}
-; CHECK: test_cmpxchg_relaxed_relaxed disallowed: invalid memory order
-
-
-; fence
-
-define internal void @test_fence_invalid_7() {
-  call void @llvm.nacl.atomic.fence(i32 7)
-  ret void
-}
-; CHECK: test_fence_invalid_7 disallowed: invalid memory order
-
-define internal void @test_fence_invalid_0() {
-  call void @llvm.nacl.atomic.fence(i32 0)
-  ret void
-}
-; CHECK: test_fence_invalid_0 disallowed: invalid memory order
-
-define internal void @test_fence_seqcst() {
-  call void @llvm.nacl.atomic.fence(i32 6)
-  ret void
-}
-; CHECK-NOT: disallowed
-
-define internal void @test_fence_acqrel() {
-  call void @llvm.nacl.atomic.fence(i32 5)
-  ret void
-}
-; CHECK-NOT: disallowed
-
-define internal void @test_fence_acquire() {
-  call void @llvm.nacl.atomic.fence(i32 4)
-  ret void
-}
-; CHECK-NOT: disallowed
-
-define internal void @test_fence_release() {
-  call void @llvm.nacl.atomic.fence(i32 3)
-  ret void
-}
-; CHECK-NOT: disallowed
-
-define internal void @test_fence_consume() {
-  call void @llvm.nacl.atomic.fence(i32 2)
-  ret void
-}
-; CHECK: test_fence_consume disallowed: invalid memory order
-
-define internal void @test_fence_relaxed() {
-  call void @llvm.nacl.atomic.fence(i32 1)
-  ret void
-}
-; CHECK: test_fence_relaxed disallowed: invalid memory order
diff --git a/test/NaCl/PNaClABI/abi-bad-intrinsic.ll b/test/NaCl/PNaClABI/abi-bad-intrinsic.ll
deleted file mode 100644
index bd5eacef85d..00000000000
--- a/test/NaCl/PNaClABI/abi-bad-intrinsic.ll
+++ /dev/null
@@ -1,10 +0,0 @@
-; RUN: not pnacl-abicheck < %s | FileCheck %s
-
-; This intrinsic is declared with the wrong type, using i32* arguments
-; instead of i8*.  Check that the ABI verifier rejects this.  This
-; must be tested in a separate .ll file from the correct intrinsic
-; declarations.
-
-declare void @llvm.memcpy.p0i8.p0i8.i32(i32* %dest, i32* %src,
-                                        i32 %len, i32 %align, i1 %isvolatile)
-; CHECK: Function llvm.memcpy.p0i8.p0i8.i32 is a disallowed LLVM intrinsic
diff --git a/test/NaCl/PNaClABI/abi-blockaddress.ll b/test/NaCl/PNaClABI/abi-blockaddress.ll
deleted file mode 100644
index 31f803a3c5f..00000000000
--- a/test/NaCl/PNaClABI/abi-blockaddress.ll
+++ /dev/null
@@ -1,11 +0,0 @@
-; RUN: not pnacl-abicheck < %s | FileCheck %s
-
-define void @func_with_block() {
-  br label %some_block
-some_block:
-  ret void
-}
-; CHECK-NOT: disallowed
-
-@blockaddr = global i8* blockaddress(@func_with_block, %some_block)
-; CHECK: Global variable blockaddr has non-flattened initializer (disallowed): i8* blockaddress(@func_with_block, %some_block)
diff --git a/test/NaCl/PNaClABI/abi-call-attributes.ll b/test/NaCl/PNaClABI/abi-call-attributes.ll
deleted file mode 100644
index 284503208be..00000000000
--- a/test/NaCl/PNaClABI/abi-call-attributes.ll
+++ /dev/null
@@ -1,20 +0,0 @@
-; RUN: not pnacl-abicheck < %s | FileCheck %s
-
-define void @func(i32 %arg) {
-  ret void
-}
-
-define void @calls() {
-  call void @func(i32 1) noreturn nounwind
-; CHECK: disallowed: bad call attributes: call void @func(i32 1) #
-
-  call void @func(i32 inreg 1)
-; CHECK-NEXT: disallowed: bad call attributes: call void @func(i32 inreg 1)
-
-  call fastcc void @func(i32 1)
-; CHECK-NEXT: disallowed: bad calling convention: call fastcc void @func(i32 1)
-
-  ret void
-}
-
-; CHECK-NOT: disallowed
diff --git a/test/NaCl/PNaClABI/abi-debug-info.ll b/test/NaCl/PNaClABI/abi-debug-info.ll
deleted file mode 100644
index 7773c2f8c6e..00000000000
--- a/test/NaCl/PNaClABI/abi-debug-info.ll
+++ /dev/null
@@ -1,41 +0,0 @@
-; RUN: not pnacl-abicheck < %s | FileCheck %s
-; RUN: not pnacl-abicheck -pnaclabi-allow-debug-metadata < %s | \
-; RUN:   FileCheck %s --check-prefix=DBG
-
-; DBG-NOT: disallowed
-
-; A debuginfo version is required.
-!llvm.module.flags = !{!0}
-!0 = !{i32 1, !"Debug Info Version", i32 3}
-!1 = !{}
-!2 = !MDExpression()
-
-declare void @llvm.dbg.declare(metadata, metadata, metadata)
-declare void @llvm.dbg.value(metadata, i64, metadata)
-
-; CHECK: Function llvm.dbg.declare is a disallowed LLVM intrinsic
-; CHECK: Function llvm.dbg.value is a disallowed LLVM intrinsic
-
-
-define internal void @debug_declare(i32 %val) {
-  ; We normally expect llvm.dbg.declare to be used on an alloca.
-  %var = alloca [4 x i8]
-  tail call void @llvm.dbg.declare(metadata [4 x i8]* %var, metadata !1, metadata !2)
-  tail call void @llvm.dbg.declare(metadata i32 %val, metadata !1, metadata !2)
-  ret void
-}
-
-define internal void @debug_value(i32 %ptr_as_int, i32 %val) {
-  %ptr = inttoptr i32 %ptr_as_int to i8*
-  tail call void @llvm.dbg.value(metadata i8* %ptr, i64 2, metadata !1)
-  tail call void @llvm.dbg.value(metadata i32 %val, i64 1, metadata !1)
-  ret void
-}
-
-; FileCheck gives an error if its input file is empty, so ensure that
-; the output of pnacl-abicheck is non-empty by generating at least one
-; error.
-declare void @bad_func(ppc_fp128 %bad_arg)
-; DBG: Function bad_func has disallowed type: void (ppc_fp128)
-; Check that there are no complaints about dbg.declare or dbg.value
-; DBG-NOT: dbg
\ No newline at end of file
diff --git a/test/NaCl/PNaClABI/abi-entry-point-required.ll b/test/NaCl/PNaClABI/abi-entry-point-required.ll
deleted file mode 100644
index 9bd8a6d12ef..00000000000
--- a/test/NaCl/PNaClABI/abi-entry-point-required.ll
+++ /dev/null
@@ -1,8 +0,0 @@
-; RUN: not pnacl-abicheck < %s | FileCheck %s
-
-; This is not a valid entry point because it's declared "internal".
-define internal void @_start(i32 %arg) {
-  ret void
-}
-
-; CHECK: Module has no entry point (disallowed)
diff --git a/test/NaCl/PNaClABI/abi-externals-multiple.ll b/test/NaCl/PNaClABI/abi-externals-multiple.ll
deleted file mode 100644
index a75d2f9b2e6..00000000000
--- a/test/NaCl/PNaClABI/abi-externals-multiple.ll
+++ /dev/null
@@ -1,9 +0,0 @@
-; RUN: not pnacl-abicheck < %s | FileCheck %s
-
-define external void @_start() {
-  ret void
-}
-
-@__pnacl_pso_root = global [4 x i8] c"abcd"
-
-; CHECK: Module has multiple entry points (disallowed)
diff --git a/test/NaCl/PNaClABI/abi-externals-whitelist-pso.ll b/test/NaCl/PNaClABI/abi-externals-whitelist-pso.ll
deleted file mode 100644
index ed9af5c0b79..00000000000
--- a/test/NaCl/PNaClABI/abi-externals-whitelist-pso.ll
+++ /dev/null
@@ -1,8 +0,0 @@
-; RUN: not pnacl-abicheck < %s | FileCheck %s
-
-@global_var = global [4 x i8] c"abcd"
-; CHECK: global_var is not a valid external symbol (disallowed)
-
-; __pnacl_pso_root is whitelisted.
-@__pnacl_pso_root = global [4 x i8] c"abcd"
-; CHECK-NOT: disallowed
diff --git a/test/NaCl/PNaClABI/abi-externals-whitelist.ll b/test/NaCl/PNaClABI/abi-externals-whitelist.ll
deleted file mode 100644
index bdb95968041..00000000000
--- a/test/NaCl/PNaClABI/abi-externals-whitelist.ll
+++ /dev/null
@@ -1,28 +0,0 @@
-; RUN: not pnacl-abicheck < %s | FileCheck %s
-
-; Make sure that external symbols are properly rejected or accepted
-
-
-@global_var = global [4 x i8] c"abcd"
-; CHECK: global_var is not a valid external symbol (disallowed)
-
-
-define void @foo() {
-  ret void
-}
-; CHECK: foo is not a valid external symbol (disallowed)
-
-define external void @main() {
-  ret void
-}
-; CHECK: main is not a valid external symbol (disallowed)
-
-define external void @_start() {
-  ret void
-}
-; _start is whitelisted
-; CHECK-NOT: disallowed
-
-; Intrinsics can be external too
-declare void @llvm.trap()
-
diff --git a/test/NaCl/PNaClABI/abi-flattened-globals.ll b/test/NaCl/PNaClABI/abi-flattened-globals.ll
deleted file mode 100644
index aff301371e2..00000000000
--- a/test/NaCl/PNaClABI/abi-flattened-globals.ll
+++ /dev/null
@@ -1,69 +0,0 @@
-; RUN: not pnacl-abicheck < %s | FileCheck %s
-
-
-; Allowed cases
-
-@bytes = internal global [7 x i8] c"abcdefg"
-
-@ptr_to_ptr = internal global i32 ptrtoint (i32* @ptr to i32)
-@ptr_to_func = internal global i32 ptrtoint (void ()* @func to i32)
-
-@compound = internal global <{ [3 x i8], i32 }>
-    <{ [3 x i8] c"foo", i32 ptrtoint (void ()* @func to i32) }>
-
-@ptr = internal global i32 ptrtoint ([7 x i8]* @bytes to i32)
-
-@addend_ptr = internal global i32 add (i32 ptrtoint (i32* @ptr to i32), i32 1)
-@addend_negative = internal global i32 add (i32 ptrtoint (i32* @ptr to i32), i32 -1)
-
-@addend_array1 = internal global i32 add (i32 ptrtoint ([7 x i8]* @bytes to i32), i32 1)
-@addend_array2 = internal global i32 add (i32 ptrtoint ([7 x i8]* @bytes to i32), i32 7)
-@addend_array3 = internal global i32 add (i32 ptrtoint ([7 x i8]* @bytes to i32), i32 9)
-
-@addend_struct1 = internal global i32 add (i32 ptrtoint (<{ [3 x i8], i32 }>* @compound to i32), i32 1)
-@addend_struct2 = internal global i32 add (i32 ptrtoint (<{ [3 x i8], i32 }>* @compound to i32), i32 4)
-
-; CHECK-NOT: disallowed
-
-
-; Disallowed cases
-
-@bad_external = external global [1 x i8]
-; CHECK: Global variable bad_external has no initializer (disallowed)
-
-@bad_int = internal global i32 0
-; CHECK: Global variable bad_int has non-flattened initializer (disallowed): i32 0
-
-@bad_size = internal global i64 ptrtoint ([7 x i8]* @bytes to i64)
-; CHECK: Global variable bad_size has non-flattened initializer
-
-; "null" is not allowed.
-@bad_ptr = internal global i8* null
-; CHECK: Global variable bad_ptr has non-flattened initializer
-
-@bad_ptr2 = internal global i64 ptrtoint (i8* null to i64)
-; CHECK: Global variable bad_ptr2 has non-flattened initializer
-
-@bad_sub = internal global i32 sub (i32 ptrtoint (i32* @ptr to i32), i32 1)
-; CHECK: Global variable bad_sub has non-flattened initializer
-
-; i16 not allowed here.
-@bad_compound = internal global <{ i32, i16 }>
-    <{ i32 ptrtoint (void ()* @func to i32), i16 0 }>
-; CHECK: Global variable bad_compound has non-flattened initializer
-
-; The struct type must be packed.
-@non_packed_struct = internal global { [3 x i8], i32 }
-    { [3 x i8] c"foo", i32 ptrtoint (void ()* @func to i32) }
-; CHECK: Global variable non_packed_struct has non-flattened initializer
-
-; The struct type must be anonymous.
-%struct = type <{ [3 x i8], i32 }>
-@named_struct = internal global %struct
-    <{ [3 x i8] c"foo", i32 ptrtoint (void ()* @func to i32) }>
-; CHECK: Global variable named_struct has non-flattened initializer
-
-
-define internal void @func() {
-  ret void
-}
diff --git a/test/NaCl/PNaClABI/abi-i1-operations.ll b/test/NaCl/PNaClABI/abi-i1-operations.ll
deleted file mode 100644
index 585fce3a047..00000000000
--- a/test/NaCl/PNaClABI/abi-i1-operations.ll
+++ /dev/null
@@ -1,164 +0,0 @@
-; RUN: not pnacl-abicheck < %s | FileCheck %s
-
-; Most arithmetic operations are not very useful on i1, so use of i1
-; is restricted to a subset of operations.
-
-
-; i1 is allowed on these bitwise operations because:
-;  * These operations never overflow.
-;  * They do get generated in practice for combining conditions.
-define internal void @allowed_cases() {
-  %and = and i1 0, 0
-  %or = or i1 0, 0
-  %xor = xor i1 0, 0
-
-  %v4and = and <4 x i1> undef, undef
-  %v4or = or <4 x i1> undef, undef
-  %v4xor = xor <4 x i1> undef, undef
-
-  %v8and = and <8 x i1> undef, undef
-  %v8or = or <8 x i1> undef, undef
-  %v8xor = xor <8 x i1> undef, undef
-
-  %v16and = and <16 x i1> undef, undef
-  %v16or = or <16 x i1> undef, undef
-  %v16xor = xor <16 x i1> undef, undef
-  ret void
-}
-; CHECK-NOT: disallowed
-
-
-define internal void @rejected_cases(i32 %ptr) {
-  ; Loads and stores of i1 are disallowed.  This is done by rejecting
-  ; i1* as a pointer type.
-  %ptr.p = inttoptr i32 %ptr to i1*
-; CHECK: disallowed: bad result type: i1* %ptr.p = inttoptr
-  %vptr.p = inttoptr i32 %ptr to <4 x i1>*
-; CHECK: disallowed: bad result type: <4 x i1>* %vptr.p = inttoptr
-  load i1, i1* %ptr.p, align 1
-; CHECK-NEXT: disallowed: bad pointer: {{.*}} load i1, i1*
-  load <4 x i1>, <4 x i1>* %vptr.p, align 4
-; CHECK-NEXT: disallowed: bad pointer: {{.*}} load <4 x i1>, <4 x i1>*
-
-  ; Check that we don't allow weird sizes.
-  add i31 0, 0
-; CHECK-NEXT:  disallowed: Invalid integer arithmetic type: {{.*}} add i31
-
-  ; i1 arithmetic is of dubious usefulness, so it is rejected.
-  add i1 0, 0
-; CHECK-NEXT: disallowed: Invalid integer arithmetic type: {{.*}} add i1
-  sub i1 0, 0
-; CHECK-NEXT: disallowed: Invalid integer arithmetic type: {{.*}} sub i1
-  mul i1 0, 0
-; CHECK-NEXT: disallowed: Invalid integer arithmetic type: {{.*}} = mul i1
-  udiv i1 0, 0
-; CHECK-NEXT: disallowed: Invalid integer arithmetic type: {{.*}} = udiv i1
-  sdiv i1 0, 0
-; CHECK-NEXT: disallowed: Invalid integer arithmetic type: {{.*}} = sdiv i1
-  urem i1 0, 0
-; CHECK-NEXT: disallowed: Invalid integer arithmetic type: {{.*}} = urem i1
-  srem i1 0, 0
-; CHECK-NEXT: disallowed: Invalid integer arithmetic type: {{.*}} = srem i1
-  shl i1 0, 0
-; CHECK-NEXT: disallowed: Invalid integer arithmetic type: {{.*}} = shl i1
-  lshr i1 0, 0
-; CHECK-NEXT: disallowed: Invalid integer arithmetic type: {{.*}} lshr i1
-  ashr i1 0, 0
-; CHECK-NEXT: disallowed: Invalid integer arithmetic type: {{.*}} ashr i1
-  add <4 x i1> undef, undef
-; CHECK-NEXT: disallowed: Invalid integer arithmetic type: {{.*}} add <4 x i1>
-  sub <4 x i1> undef, undef
-; CHECK-NEXT: disallowed: Invalid integer arithmetic type: {{.*}} sub <4 x i1>
-  mul <4 x i1> undef, undef
-; CHECK-NEXT: disallowed: Invalid integer arithmetic type: {{.*}} mul <4 x i1>
-  udiv <4 x i1> undef, undef
-; CHECK-NEXT: disallowed: Invalid integer arithmetic type: {{.*}} udiv <4 x i1>
-  sdiv <4 x i1> undef, undef
-; CHECK-NEXT: disallowed: Invalid integer arithmetic type: {{.*}} sdiv <4 x i1>
-  urem <4 x i1> undef, undef
-; CHECK-NEXT: disallowed: Invalid integer arithmetic type: {{.*}} urem <4 x i1>
-  srem <4 x i1> undef, undef
-; CHECK-NEXT: disallowed: Invalid integer arithmetic type: {{.*}} srem <4 x i1>
-  shl <4 x i1> undef, undef
-; CHECK-NEXT: disallowed: Invalid integer arithmetic type: {{.*}} shl <4 x i1>
-  lshr <4 x i1> undef, undef
-; CHECK-NEXT: disallowed: Invalid integer arithmetic type: {{.*}} lshr <4 x i1>
-  ashr <4 x i1> undef, undef
-; CHECK-NEXT: disallowed: Invalid integer arithmetic type: {{.*}} ashr <4 x i1>
-  add <8 x i1> undef, undef
-; CHECK-NEXT: disallowed: Invalid integer arithmetic type: {{.*}} add <8 x i1>
-  sub <8 x i1> undef, undef
-; CHECK-NEXT: disallowed: Invalid integer arithmetic type: {{.*}} sub <8 x i1>
-  mul <8 x i1> undef, undef
-; CHECK-NEXT: disallowed: Invalid integer arithmetic type: {{.*}} mul <8 x i1>
-  udiv <8 x i1> undef, undef
-; CHECK-NEXT: disallowed: Invalid integer arithmetic type: {{.*}} udiv <8 x i1>
-  sdiv <8 x i1> undef, undef
-; CHECK-NEXT: disallowed: Invalid integer arithmetic type: {{.*}} sdiv <8 x i1>
-  urem <8 x i1> undef, undef
-; CHECK-NEXT: disallowed: Invalid integer arithmetic type: {{.*}} urem <8 x i1>
-  srem <8 x i1> undef, undef
-; CHECK-NEXT: disallowed: Invalid integer arithmetic type: {{.*}} srem <8 x i1>
-  shl <8 x i1> undef, undef
-; CHECK-NEXT: disallowed: Invalid integer arithmetic type: {{.*}} shl <8 x i1>
-  lshr <8 x i1> undef, undef
-; CHECK-NEXT: disallowed: Invalid integer arithmetic type: {{.*}} lshr <8 x i1>
-  ashr <8 x i1> undef, undef
-; CHECK-NEXT: disallowed: Invalid integer arithmetic type: {{.*}} ashr <8 x i1>
-  add <16 x i1> undef, undef
-; CHECK-NEXT: disallowed: Invalid integer arithmetic type: {{.*}} add <16 x i1>
-  sub <16 x i1> undef, undef
-; CHECK-NEXT: disallowed: Invalid integer arithmetic type: {{.*}} sub <16 x i1>
-  mul <16 x i1> undef, undef
-; CHECK-NEXT: disallowed: Invalid integer arithmetic type: {{.*}} mul <16 x i1>
-  udiv <16 x i1> undef, undef
-; CHECK-NEXT: disallowed: Invalid integer arithmetic type: {{.*}} udiv <16 x i1>
-  sdiv <16 x i1> undef, undef
-; CHECK-NEXT: disallowed: Invalid integer arithmetic type: {{.*}} sdiv <16 x i1>
-  urem <16 x i1> undef, undef
-; CHECK-NEXT: disallowed: Invalid integer arithmetic type: {{.*}} urem <16 x i1>
-  srem <16 x i1> undef, undef
-; CHECK-NEXT: disallowed: Invalid integer arithmetic type: {{.*}} srem <16 x i1>
-  shl <16 x i1> undef, undef
-; CHECK-NEXT: disallowed: Invalid integer arithmetic type: {{.*}} shl <16 x i1>
-  lshr <16 x i1> undef, undef
-; CHECK-NEXT: disallowed: Invalid integer arithmetic type: {{.*}} lshr <16 x i1>
-  ashr <16 x i1> undef, undef
-; CHECK-NEXT: disallowed: Invalid integer arithmetic type: {{.*}} ashr <16 x i1>
-
-  ; The same applies to i1 comparisons.
-  icmp eq i1 0, 0
-; CHECK-NEXT: disallowed: Invalid integer arithmetic type: {{.*}} icmp eq i1
-  icmp ult i1 0, 0
-; CHECK-NEXT: disallowed: Invalid integer arithmetic type: {{.*}} icmp ult i1
-  icmp eq <4 x i1> undef, undef
-; CHECK-NEXT: disallowed: Invalid integer arithmetic type: {{.*}} icmp eq <4 x i1>
-  icmp ult <4 x i1> undef, undef
-; CHECK-NEXT: disallowed: Invalid integer arithmetic type: {{.*}} icmp ult <4 x i1>
-  icmp eq <8 x i1> undef, undef
-; CHECK-NEXT: disallowed: Invalid integer arithmetic type: {{.*}} icmp eq <8 x i1>
-  icmp ult <8 x i1> undef, undef
-; CHECK-NEXT: disallowed: Invalid integer arithmetic type: {{.*}} icmp ult <8 x i1>
-  icmp eq <16 x i1> undef, undef
-; CHECK-NEXT: disallowed: Invalid integer arithmetic type: {{.*}} icmp eq <16 x i1>
-  icmp ult <16 x i1> undef, undef
-; CHECK-NEXT: disallowed: Invalid integer arithmetic type: {{.*}} icmp ult <16 x i1>
-
-  ; There should be no implicit zero-extension in alloca.
-  alloca i8, i1 1
-; CHECK-NEXT: disallowed: alloca array size is not i32
-
-  ; Switch on i1 is not useful.  "br" should be used instead.
-  switch i1 0, label %next [i1 0, label %next]
-; CHECK-NEXT: disallowed: switch on i1
-next:
-
-  ret void
-}
-; CHECK-NOT: disallowed
-
-
-; This stops the verifier from complaining about the lack of an entry point.
-define void @_start(i32 %arg) {
-  ret void
-}
diff --git a/test/NaCl/PNaClABI/abi-metadata.ll b/test/NaCl/PNaClABI/abi-metadata.ll
deleted file mode 100644
index f5498d05351..00000000000
--- a/test/NaCl/PNaClABI/abi-metadata.ll
+++ /dev/null
@@ -1,27 +0,0 @@
-; RUN: not pnacl-abicheck < %s | FileCheck %s
-; RUN: not pnacl-abicheck -pnaclabi-allow-debug-metadata < %s | FileCheck %s --check-prefix=DEBUG
-
-
-; Metadata is not part of the PNaCl's stable ABI, so normally the ABI
-; checker rejects metadata entirely.  However, for debugging support,
-; pre-finalized pexes may contain metadata.  When checking a
-; pre-finalized pexe, the ABI checker does not check the types in the
-; metadata.
-
-; DEBUG-NOT: Named metadata node llvm.dbg.cu is disallowed
-; CHECK: Named metadata node llvm.dbg.cu is disallowed
-!llvm.dbg.cu = !{!0}
-!0 = !{ half 0.0}
-
-; A debuginfo version must always be specified.
-; DEBUG-NOT: ignoring debug info with an invalid version
-; CHECK-NOT: ignoring debug info with an invalid version
-; DEBUG-NOT: Named metadata node llvm.module.flags is disallowed
-; CHECK: Named metadata node llvm.module.flags is disallowed
-!llvm.module.flags = !{!1}
-!1 = !{i32 1, !"Debug Info Version", i32 3}
-
-; CHECK: Named metadata node madeup is disallowed
-; DEBUG: Named metadata node madeup is disallowed
-!madeup = !{!2}
-!2 = !{ half 1.0}
diff --git a/test/NaCl/PNaClABI/abi-small-arguments.ll b/test/NaCl/PNaClABI/abi-small-arguments.ll
deleted file mode 100644
index 696fac16b8c..00000000000
--- a/test/NaCl/PNaClABI/abi-small-arguments.ll
+++ /dev/null
@@ -1,52 +0,0 @@
-; RUN: not pnacl-abicheck < %s | FileCheck %s
-
-define void @arg_i1(i1 %bad) {
-  ret void
-}
-; CHECK: Function arg_i1 has disallowed type:
-
-define void @arg_i16(i32 %allowed, i16 %bad) {
-  ret void
-}
-; CHECK: Function arg_i16 has disallowed type:
-
-define i1 @return_i1() {
-  ret i1 0
-}
-; CHECK: Function return_i1 has disallowed type:
-
-define i8 @return_i8() {
-  ret i8 0
-}
-; CHECK: Function return_i8 has disallowed type:
-
-
-define void @bad_direct_calls() {
-  call void @arg_i1(i1 0)
-; CHECK: bad function callee operand: call void @arg_i1
-
-  call void @arg_i16(i32 0, i16 0)
-; CHECK-NEXT: bad function callee operand: call void @arg_i16
-
-  %result1 = call i1 @return_i1()
-; CHECK-NEXT: bad function callee operand: {{.*}} call i1 @return_i1
-
-  %result2 = call i8 @return_i8()
-; CHECK-NEXT: bad function callee operand: {{.*}} call i8 @return_i8
-
-  ret void
-}
-
-define void @bad_indirect_calls(i32 %ptr) {
-  %func1 = inttoptr i32 %ptr to void (i8)*
-; CHECK: bad result type: void (i8)* %func1
-  call void %func1(i8 0)
-; CHECK: bad function callee operand: {{.*}} %func1
-
-  %func2 = inttoptr i32 %ptr to i16 ()*
-; CHECK: bad result type: i16 ()* %func2
-  %result3 = call i16 %func2()
-; CHECK: bad function callee operand: {{.*}} %func2
-
-  ret void
-}
diff --git a/test/NaCl/PNaClABI/abi-stripped-pointers.ll b/test/NaCl/PNaClABI/abi-stripped-pointers.ll
deleted file mode 100644
index 16b884e7224..00000000000
--- a/test/NaCl/PNaClABI/abi-stripped-pointers.ll
+++ /dev/null
@@ -1,138 +0,0 @@
-; RUN: not pnacl-abicheck < %s | FileCheck %s
-
-; This test checks that the PNaCl ABI verifier enforces the normal
-; form introduced by the ReplacePtrsWithInts pass.
-
-
-@var = global [4 x i8] c"xxxx"
-@ptr = global i32 ptrtoint ([4 x i8]* @var to i32)
-
-declare i8* @llvm.nacl.read.tp()
-
-
-define internal void @pointer_arg(i8* %arg) {
-  ret void
-}
-; CHECK: Function pointer_arg has disallowed type
-
-define internal i8* @pointer_return() {
-  unreachable
-}
-; CHECK-NEXT: Function pointer_return has disallowed type
-
-define internal void @func() {
-  ret void
-}
-
-define internal void @func_with_arg(i32 %arg) {
-  ret void
-}
-
-
-define internal void @allowed_cases(i32 %arg) {
-  inttoptr i32 123 to i8*
-
-  ptrtoint [4 x i8]* @var to i32
-
-  %alloc = alloca i8
-  ptrtoint i8* %alloc to i32
-  load i8, i8* %alloc, align 1
-
-  ; These instructions may use a NormalizedPtr, which may be a global.
-  load i32, i32* @ptr, align 1
-  store i32 123, i32* @ptr, align 1
-
-  ; A NormalizedPtr may be a bitcast.
-  %ptr_bitcast = bitcast [4 x i8]* @var to i32*
-  load i32, i32* %ptr_bitcast, align 1
-
-  ; A NormalizedPtr may be an inttoptr.
-  %ptr_from_int = inttoptr i32 123 to i32*
-  load i32, i32* %ptr_from_int, align 1
-
-  ; Check direct and indirect function calls.
-  %func_as_int = ptrtoint void ()* @func to i32
-  %func_ptr = inttoptr i32 %func_as_int to void ()*
-  call void %func_ptr()
-  call void @func()
-  call void @func_with_arg(i32 123)
-
-  ; Intrinsic calls may return pointers.
-  %thread_ptr = call i8* @llvm.nacl.read.tp()
-  ptrtoint i8* %thread_ptr to i32
-
-  ; Bitcasts between non-pointers are not restricted
-  bitcast i64 0 to double
-  bitcast i32 0 to float
-
-  ; ConstantInts and Arguments are allowed as operands.
-  add i32 %arg, 123
-
-  ret void
-}
-; CHECK-NOT: disallowed
-
-
-define internal void @bad_cases() {
-entry:
-  ptrtoint [4 x i8]* @var to i16
-; CHECK: Function bad_cases disallowed: non-i32 ptrtoint
-
-  inttoptr i16 123 to i8*
-; CHECK-NEXT: non-i32 inttoptr
-
-  %a = alloca i32
-; CHECK-NEXT: non-i8 alloca: %a
-  %a2 = alloca [4 x i8]
-; CHECK-NEXT: non-i8 alloca: %a2
-
-  store i32 0, i32* null, align 1
-; CHECK-NEXT: bad pointer
-
-  store i32 0, i32* undef, align 1
-; CHECK-NEXT: bad pointer
-
-  %bc = bitcast i32* @ptr to i31*
-; CHECK-NEXT: bad result type
-  store i31 0, i31* %bc, align 1
-; CHECK-NEXT: bad pointer
-
-  ; Only one level of bitcasts is allowed.
-  %b = bitcast i32* %a to i8*
-  %c = bitcast i8* %b to i16*
-; CHECK-NEXT: operand not InherentPtr
-
-  br label %block
-block:
-  %phi1 = phi i8* [ undef, %entry ]
-; CHECK-NEXT: bad operand: %phi1
-  %phi2 = phi i32* [ undef, %entry ]
-; CHECK-NEXT: bad operand: %phi2
-
-  icmp eq i32* @ptr, @ptr
-; CHECK-NEXT: Expects integer arithmetic type: {{.*}} icmp eq i32*
-  icmp eq void ()* @func, @func
-; CHECK-NEXT: Expects integer arithmetic type: {{.*}} icmp eq void ()* 
-  icmp eq i31 0, 0
-; CHECK-NEXT: Invalid integer arithmetic type: {{.*}} icmp eq i31
-
-  call void null()
-; CHECK-NEXT: bad function callee operand
-
-  call void @func_with_arg(i32 ptrtoint (i32* @ptr to i32))
-; CHECK-NEXT: bad operand
-
-  ; Taking the address of an intrinsic is not allowed.
-  ptrtoint i8* ()* @llvm.nacl.read.tp to i32
-; CHECK-NEXT: operand not InherentPtr
-
-  ret void
-}
-
-; CHECK-NOT: disallowed
-
-
-; This stops the verifier from complaining about the lack of an entry point.
-define void @_start(i32 %arg) {
-  ret void
-}
diff --git a/test/NaCl/PNaClABI/abi-switch.ll b/test/NaCl/PNaClABI/abi-switch.ll
deleted file mode 100644
index 8fd6b75ebb7..00000000000
--- a/test/NaCl/PNaClABI/abi-switch.ll
+++ /dev/null
@@ -1,37 +0,0 @@
-; RUN: not pnacl-abicheck < %s | FileCheck %s
-
-@var = internal global [4 x i8] c"xxxx"
-
-
-; CHECK-NOT: disallowed
-
-define internal void @bad_cases() {
-  ; ConstantExprs should be rejected here.
-  switch i32 ptrtoint ([4 x i8]* @var to i32), label %next [i32 0, label %next]
-; CHECK: disallowed: bad switch condition
-next:
-
-  ; Bad integer type.
-  switch i32 0, label %next [i99 0, label %next]
-; CHECK: bad switch case
-
-  ; Bad integer type.
-  switch i32 0, label %next [i32 0, label %next
-                             i99 1, label %next]
-; CHECK: bad switch case
-
-  ; Note that the reader only allows ConstantInts in the label list.
-  ; We don't need to check the following, because the reader rejects
-  ; it:
-  ; switch i32 0, label %next [i32 ptrtoint (i32* @ptr to i32), label %next]
-
-  ret void
-}
-
-; CHECK-NOT: disallowed
-
-
-; This stops the verifier from complaining about the lack of an entry point.
-define void @_start(i32 %arg) {
-  ret void
-}
diff --git a/test/NaCl/PNaClABI/abi-varargs.ll b/test/NaCl/PNaClABI/abi-varargs.ll
deleted file mode 100644
index f2f55d9b873..00000000000
--- a/test/NaCl/PNaClABI/abi-varargs.ll
+++ /dev/null
@@ -1,13 +0,0 @@
-; RUN: not pnacl-abicheck < %s | FileCheck %s
-
-define void @varargs_func(i32 %arg, ...) {
-  ret void
-}
-; CHECK: Function varargs_func has disallowed type: void (i32, ...)
-
-define void @call_varargs_func(i32 %ptr) {
-  %ptr2 = inttoptr i32 %ptr to void (i32, ...)*
-  call void (i32, ...) %ptr2(i32 123)
-  ret void
-}
-; CHECK: Function call_varargs_func disallowed: bad function callee operand: call void (i32, ...)
diff --git a/test/NaCl/PNaClABI/abi-visibility.ll b/test/NaCl/PNaClABI/abi-visibility.ll
deleted file mode 100644
index 40629ffc710..00000000000
--- a/test/NaCl/PNaClABI/abi-visibility.ll
+++ /dev/null
@@ -1,15 +0,0 @@
-; RUN: not pnacl-abicheck < %s | FileCheck %s
-
-; Disallow the visibility attributes set by
-; __attribute__((visibility("hidden"))) and
-; __attribute__((visibility("protected"))).
-
-define hidden void @visibility_hidden() {
-  ret void
-}
-; CHECK: Function visibility_hidden has disallowed visibility: hidden
-
-define protected void @visibility_protected() {
-  ret void
-}
-; CHECK: Function visibility_protected has disallowed visibility: protected
diff --git a/test/NaCl/PNaClABI/function-signatures.ll b/test/NaCl/PNaClABI/function-signatures.ll
deleted file mode 100644
index 5ff1054ea29..00000000000
--- a/test/NaCl/PNaClABI/function-signatures.ll
+++ /dev/null
@@ -1,181 +0,0 @@
-; RUN: not pnacl-abicheck < %s | FileCheck %s
-
-; Test type-checking of function signatures.
-
-; CHECK: Function f_arg_i1 has disallowed type: void (i1)
-; CHECK: Function f_ret_i1 has disallowed type: i1 ()
-define internal void @f_arg_i1(i1 %a) {
-  ret void
-}
-define internal i1 @f_ret_i1() {
-  ret i1 undef
-}
-
-; CHECK: Function f_arg_i8 has disallowed type: void (i8)
-; CHECK: Function f_ret_i8 has disallowed type: i8 ()
-define internal void @f_arg_i8(i8 %a) {
-  ret void
-}
-define internal i8 @f_ret_i8() {
-  ret i8 undef
-}
-
-; CHECK: Function f_arg_i16 has disallowed type: void (i16)
-; CHECK: Function f_ret_i16 has disallowed type: i16 ()
-define internal void @f_arg_i16(i16 %a) {
-  ret void
-}
-define internal i16 @f_ret_i16() {
-  ret i16 undef
-}
-
-; CHECK-NOT: f_{{[a-z]+}}_i32 {{.*}} disallowed
-define internal void @f_arg_i32(i32 %a) {
-  ret void
-}
-define internal i32 @f_ret_i32() {
-  ret i32 undef
-}
-
-; CHECK-NOT: f_{{[a-z]+}}_i64 {{.*}} disallowed
-define internal void @f_arg_i64(i64 %a) {
-  ret void
-}
-define internal i64 @f_ret_i64() {
-  ret i64 undef
-}
-
-; CHECK: Function f_arg_i128 has disallowed type: void (i128)
-; CHECK: Function f_ret_i128 has disallowed type: i128 ()
-define internal void @f_arg_i128(i128 %a) {
-  ret void
-}
-define internal i128 @f_ret_i128() {
-  ret i128 undef
-}
-
-; CHECK-NOT: f_{{[a-z]+}}_float {{.*}} disallowed
-define internal void @f_arg_float(float %a) {
-  ret void
-}
-define internal float @f_ret_float() {
-  ret float undef
-}
-
-; CHECK-NOT: f_{{[a-z]+}}_double {{.*}} disallowed
-define internal void @f_arg_double(double %a) {
-  ret void
-}
-define internal double @f_ret_double() {
-  ret double undef
-}
-
-; CHECK: Function f_arg_1xi1 has disallowed type: void (<1 x i1>)
-; CHECK: Function f_ret_1xi1 has disallowed type: <1 x i1> ()
-define internal void @f_arg_1xi1(<1 x i1> %a) {
-  ret void
-}
-define internal <1 x i1> @f_ret_1xi1() {
-  ret <1 x i1> undef
-}
-
-; CHECK: Function f_arg_2xi1 has disallowed type: void (<2 x i1>)
-; CHECK: Function f_ret_2xi1 has disallowed type: <2 x i1> ()
-define internal void @f_arg_2xi1(<2 x i1> %a) {
-  ret void
-}
-define internal <2 x i1> @f_ret_2xi1() {
-  ret <2 x i1> undef
-}
-
-; CHECK-NOT: f_{{[a-z]+}}_4xi1 {{.*}} disallowed
-define internal void @f_arg_4xi1(<4 x i1> %a) {
-  ret void
-}
-define internal <4 x i1> @f_ret_4xi1() {
-  ret <4 x i1> undef
-}
-
-; CHECK-NOT: f_{{[a-z]+}}_8xi1 {{.*}} disallowed
-define internal void @f_arg_8xi1(<8 x i1> %a) {
-  ret void
-}
-define internal <8 x i1> @f_ret_8xi1() {
-  ret <8 x i1> undef
-}
-
-; CHECK-NOT: f_{{[a-z]+}}_16xi1 {{.*}} disallowed
-define internal void @f_arg_16xi1(<16 x i1> %a) {
-  ret void
-}
-define internal <16 x i1> @f_ret_16xi1() {
-  ret <16 x i1> undef
-}
-
-; CHECK: Function f_arg_32xi1 has disallowed type: void (<32 x i1>)
-; CHECK: Function f_ret_32xi1 has disallowed type: <32 x i1> ()
-define internal void @f_arg_32xi1(<32 x i1> %a) {
-  ret void
-}
-define internal <32 x i1> @f_ret_32xi1() {
-  ret <32 x i1> undef
-}
-
-; CHECK: Function f_arg_64xi1 has disallowed type: void (<64 x i1>)
-; CHECK: Function f_ret_64xi1 has disallowed type: <64 x i1> ()
-define internal void @f_arg_64xi1(<64 x i1> %a) {
-  ret void
-}
-define internal <64 x i1> @f_ret_64xi1() {
-  ret <64 x i1> undef
-}
-
-; CHECK-NOT: f_{{[a-z]+}}_16xi8 {{.*}} disallowed
-define internal void @f_arg_16xi8(<16 x i8> %a) {
-  ret void
-}
-define internal <16 x i8> @f_ret_16xi8() {
-  ret <16 x i8> undef
-}
-
-; CHECK-NOT: f_{{[a-z]+}}_8xi16 {{.*}} disallowed
-define internal void @f_arg_8xi16(<8 x i16> %a) {
-  ret void
-}
-define internal <8 x i16> @f_ret_8xi16() {
-  ret <8 x i16> undef
-}
-
-; CHECK-NOT: f_{{[a-z]+}}_4xi32 {{.*}} disallowed
-define internal void @f_arg_4xi32(<4 x i32> %a) {
-  ret void
-}
-define internal <4 x i32> @f_ret_4xi32() {
-  ret <4 x i32> undef
-}
-
-; CHECK: Function f_arg_2xi64 has disallowed type: void (<2 x i64>)
-; CHECK: Function f_ret_2xi64 has disallowed type: <2 x i64> ()
-define internal void @f_arg_2xi64(<2 x i64> %a) {
-  ret void
-}
-define internal <2 x i64> @f_ret_2xi64() {
-  ret <2 x i64> undef
-}
-
-; CHECK-NOT: f_{{[a-z]+}}_4xfloat {{.*}} disallowed
-define internal void @f_arg_4xfloat(<4 x float> %a) {
-  ret void
-}
-define internal <4 x float> @f_ret_4xfloat() {
-  ret <4 x float> undef
-}
-
-; CHECK: Function f_arg_2xdouble has disallowed type: void (<2 x double>)
-; CHECK: Function f_ret_2xdouble has disallowed type: <2 x double> ()
-define internal void @f_arg_2xdouble(<2 x double> %a) {
-  ret void
-}
-define internal <2 x double> @f_ret_2xdouble() {
-  ret <2 x double> undef
-}
diff --git a/test/NaCl/PNaClABI/global-attributes.ll b/test/NaCl/PNaClABI/global-attributes.ll
deleted file mode 100644
index 0690bd4a5f8..00000000000
--- a/test/NaCl/PNaClABI/global-attributes.ll
+++ /dev/null
@@ -1,68 +0,0 @@
-; RUN: not pnacl-abicheck < %s | FileCheck %s
-
-; Global variable attributes
-
-; CHECK: Variable var_with_section has disallowed "section" attribute
-@var_with_section = internal global [1 x i8] zeroinitializer, section ".some_section"
-
-; PNaCl programs can depend on data alignments in general, so we allow
-; "align" on global variables.
-; CHECK-NOT: var_with_alignment
-@var_with_alignment = internal global [4 x i8] zeroinitializer, align 8
-
-; TLS variables must be expanded out by ExpandTls.
-; CHECK-NEXT: Variable tls_var has disallowed "thread_local" attribute
-@tls_var = internal thread_local global [4 x i8] zeroinitializer
-
-; CHECK-NEXT: Variable var_with_unnamed_addr has disallowed "unnamed_addr" attribute
-@var_with_unnamed_addr = internal unnamed_addr constant [1 x i8] c"x"
-
-; CHECK-NEXT: Variable var_ext_init has disallowed "externally_initialized" attribute
-@var_ext_init = internal externally_initialized global [1 x i8] c"x"
-
-
-; Function attributes
-
-; CHECK-NEXT: Function func_with_attrs has disallowed attributes: noreturn nounwind
-define internal void @func_with_attrs() noreturn nounwind {
-  ret void
-}
-
-; CHECK-NEXT: Function func_with_arg_attrs has disallowed attributes: inreg zeroext
-define internal void @func_with_arg_attrs(i32 inreg zeroext) {
-  ret void
-}
-
-; CHECK-NEXT: Function func_with_callingconv has disallowed calling convention: fastcc (8)
-define internal fastcc void @func_with_callingconv() {
-  ret void
-}
-
-; CHECK-NEXT: Function func_with_section has disallowed "section" attribute
-define internal void @func_with_section() section ".some_section" {
-  ret void
-}
-
-; CHECK-NEXT: Function func_with_alignment has disallowed "align" attribute
-define internal void @func_with_alignment() align 1 {
-  ret void
-}
-
-; CHECK-NEXT: Function func_with_gc has disallowed "gc" attribute
-define internal void @func_with_gc() gc "my_gc_func" {
-  ret void
-}
-
-; CHECK-NEXT: Function func_with_unnamed_addr has disallowed "unnamed_addr" attribute
-define internal void @func_with_unnamed_addr() unnamed_addr {
-  ret void
-}
-
-; CHECK-NOT: disallowed
-; If another check is added, there should be a check-not in between each check
-
-
-; This stops the verifier from complaining about the lack of an entry point.
-define void @_start(i32 %arg) {
-  ret void
-}
diff --git a/test/NaCl/PNaClABI/instcombine.ll b/test/NaCl/PNaClABI/instcombine.ll
deleted file mode 100644
index e21eea45c4a..00000000000
--- a/test/NaCl/PNaClABI/instcombine.ll
+++ /dev/null
@@ -1,24 +0,0 @@
-; RUN: opt < %s -instcombine -S | FileCheck %s
-; Test that instcombine does not introduce non-power-of-two integers into
-; the module
-
-target datalayout = "p:32:32:32"
-
-; This test is a counterpart to icmp_shl16 in
-; test/Transforms/InstCombine/icmp.ll, which should still pass.
-; CHECK: @icmp_shl31
-; CHECK-NOT: i31
-define i1 @icmp_shl31(i32 %x) {
-  %shl = shl i32 %x, 1
-  %cmp = icmp slt i32 %shl, 36
-  ret i1 %cmp
-}
-
-; Check that we don't introduce i4, which is a power of 2 but still not allowed.
-; CHECK: @icmp_shl4
-; CHECK-NOT: i4
-define i1 @icmp_shl4(i32 %x) {
-  %shl = shl i32 %x, 28
-  %cmp = icmp slt i32 %shl, 1073741824
-  ret i1 %cmp
-}
diff --git a/test/NaCl/PNaClABI/instructions.ll b/test/NaCl/PNaClABI/instructions.ll
deleted file mode 100644
index dacc29d697f..00000000000
--- a/test/NaCl/PNaClABI/instructions.ll
+++ /dev/null
@@ -1,431 +0,0 @@
-; RUN: not pnacl-abicheck < %s | FileCheck %s
-; Test instruction opcodes allowed by PNaCl ABI
-
-target datalayout = "e-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-p:32:32:32-v128:32:32"
-target triple = "le32-unknown-nacl"
-
-define internal void @terminators() {
-; CHECK: ERROR: Function terminators
-; Terminator instructions
-terminators:
- ret void
- br i1 0, label %next2, label %next
-next:
- switch i32 1, label %next2 [i32 0, label %next]
-next2:
-  unreachable
-; CHECK-NOT: disallowed
-; CHECK: Function terminators disallowed: bad instruction opcode: indirectbr
-  indirectbr i8* undef, [label %next, label %next2]
-}
-
-define internal void @binops() {
-; CHECK-NOT: ERROR: Function binops
-; Binary operations
-  %a1 = add i32 0, 0
-  %a2 = fadd float 0.0, 0.0
-  %a3 = sub i32 0, 0
-  %a4 = fsub float 0.0, 0.0
-  %a5 = mul i32 0, 0
-  %a6 = fmul float 0.0, 0.0
-  %a7 = udiv i32 0, 1
-  %a8 = sdiv i32 0, 1
-  %a9 = fdiv float 0.0, 1.0
-  %a10 = urem i32 0, 1
-  %a11 = srem i32 0, 1
-  %a12 = frem float 0.0, 1.0
-; Bitwise binary operations
-  %a13 = shl i32 1, 1
-  %a14 = lshr i32 1, 1
-  %a15 = ashr i32 1, 1
-  %a16 = and i32 1, 1
-  %a17 = or i32 1, 1
-  %a18 = xor i32 1, 1
-  ret void
-}
-; CHECK-NOT: disallowed
-
-define internal void @vector_binops(<4 x i32> %i, <4 x float> %f) {
-; CHECK-NOT: ERROR: Function vector_binops
-; Binary operations
-  %a1 = add <4 x i32> %i, %i
-  %a2 = fadd <4 x float> %f, %f
-  %a3 = sub <4 x i32> %i, %i
-  %a4 = fsub <4 x float> %f, %f
-  %a5 = mul <4 x i32> %i, %i
-  %a6 = fmul <4 x float> %f, %f
-  %a7 = udiv <4 x i32> %i, %i
-  %a8 = sdiv <4 x i32> %i, %i
-  %a9 = fdiv <4 x float> %f, %f
-  %a10 = urem <4 x i32> %i, %i
-  %a11 = srem <4 x i32> %i, %i
-  %a12 = frem <4 x float> %f, %f
-; Bitwise binary operations
-  %a13 = shl <4 x i32> %i, %i
-  %a14 = lshr <4 x i32> %i, %i
-  %a15 = ashr <4 x i32> %i, %i
-  %a16 = and <4 x i32> %i, %i
-  %a17 = or <4 x i32> %i, %i
-  %a18 = xor <4 x i32> %i, %i
-  ret void
-}
-; CHECK-NOT: disallowed
-
-define internal void @vectors_ok(<4 x i32> %i) {
-; CHECK-NOT: ERROR: Function vectors_ok
-  %eu4xi32.0 = extractelement <4 x i32> undef, i32 0
-  %eu4xi32.1 = extractelement <4 x i32> undef, i32 1
-  %eu4xi32.2 = extractelement <4 x i32> undef, i32 2
-  %eu4xi32.3 = extractelement <4 x i32> undef, i32 3
-
-  %ev4xi32.0 = extractelement <4 x i32> %i, i32 0
-  %ev4xi32.1 = extractelement <4 x i32> %i, i32 1
-  %ev4xi32.2 = extractelement <4 x i32> %i, i32 2
-  %ev4xi32.3 = extractelement <4 x i32> %i, i32 3
-
-  %iu4xi32.0 = insertelement <4 x i32> undef, i32 1, i32 0
-  %iu4xi32.1 = insertelement <4 x i32> undef, i32 1, i32 1
-  %iu4xi32.2 = insertelement <4 x i32> undef, i32 1, i32 2
-  %iu4xi32.3 = insertelement <4 x i32> undef, i32 1, i32 3
-
-  %iv4xi32.0 = insertelement <4 x i32> %i, i32 1, i32 0
-  %iv4xi32.1 = insertelement <4 x i32> %i, i32 1, i32 1
-  %iv4xi32.2 = insertelement <4 x i32> %i, i32 1, i32 2
-  %iv4xi32.3 = insertelement <4 x i32> %i, i32 1, i32 3
-
-  ret void
-}
-; CHECK-NOT: disallowed
-
-define internal void @vectors_bad(i32 %idx) {
-; CHECK: ERROR: Function vectors_bad
-
-  %e.var.idx = extractelement <4 x i32> <i32 0, i32 0, i32 0, i32 0>, i32 %idx ; CHECK-NEXT: disallowed: non-constant vector insert/extract index: {{.*}} extractelement
-  %e.oob.idx = extractelement <4 x i32> <i32 0, i32 0, i32 0, i32 0>, i32 4 ; CHECK-NEXT: disallowed: out of range vector insert/extract index: {{.*}} extractelement
-  %e.vec.imm = extractelement <4 x i32> <i32 0, i32 1, i32 2, i32 3>, i32 0 ; CHECK-NEXT: disallowed: bad operand: {{.*}} extractelement
-  %i.var.idx = insertelement <4 x i32> undef, i32 42, i32 %idx ; CHECK-NEXT: disallowed: non-constant vector insert/extract index: {{.*}} insertelement
-  %i.oob.idx = insertelement <4 x i32> undef, i32 42, i32 4 ; CHECK-NEXT: disallowed: out of range vector insert/extract index: {{.*}} insertelement
-  %i.vec.imm = insertelement <4 x i32> <i32 0, i32 1, i32 2, i32 3>, i32 42, i32 0 ; CHECK-NEXT: disallowed: bad operand: {{.*}} insertelement
-  %a3 = shufflevector <4 x i32> undef, <4 x i32> undef, <4 x i32> undef ; CHECK-NEXT: disallowed: bad instruction opcode: {{.*}} shufflevector
-
-  ret void
-}
-
-define internal void @vectors_bad_zeroinitializer() {
-; CHECK: ERROR: Function vectors_bad_zeroinitializer
-
-  ; zeroinitializer isn't allowed, it should be globalized instead.
-  %ez4xi32.0 = extractelement <4 x i32> <i32 0, i32 0, i32 0, i32 0>, i32 0 ; CHECK-NEXT: disallowed: bad operand: {{.*}} extractelement {{.*}} zeroinitializer
-  %ez4xi32.1 = extractelement <4 x i32> <i32 0, i32 0, i32 0, i32 0>, i32 1 ; CHECK-NEXT: disallowed: bad operand: {{.*}} extractelement {{.*}} zeroinitializer
-  %ez4xi32.2 = extractelement <4 x i32> <i32 0, i32 0, i32 0, i32 0>, i32 2 ; CHECK-NEXT: disallowed: bad operand: {{.*}} extractelement {{.*}} zeroinitializer
-  %ez4xi32.3 = extractelement <4 x i32> <i32 0, i32 0, i32 0, i32 0>, i32 3 ; CHECK-NEXT: disallowed: bad operand: {{.*}} extractelement {{.*}} zeroinitializer
-  %iz4xi32.0 = insertelement <4 x i32> <i32 0, i32 0, i32 0, i32 0>, i32 1, i32 0 ; CHECK-NEXT: disallowed: bad operand: {{.*}} insertelement {{.*}} zeroinitializer
-  %iz4xi32.1 = insertelement <4 x i32> <i32 0, i32 0, i32 0, i32 0>, i32 1, i32 1 ; CHECK-NEXT: disallowed: bad operand: {{.*}} insertelement {{.*}} zeroinitializer
-  %iz4xi32.2 = insertelement <4 x i32> <i32 0, i32 0, i32 0, i32 0>, i32 1, i32 2 ; CHECK-NEXT: disallowed: bad operand: {{.*}} insertelement {{.*}} zeroinitializer
-  %iz4xi32.3 = insertelement <4 x i32> <i32 0, i32 0, i32 0, i32 0>, i32 1, i32 3 ; CHECK-NEXT: disallowed: bad operand: {{.*}} insertelement {{.*}} zeroinitializer
-
-  ret void
-}
-
-define internal void @aggregates() {
-; CHECK: ERROR: Function aggregates
-
-; Aggregate operations
-  %a1 = extractvalue { i32, i32 } { i32 0, i32 0 }, 0
-; CHECK: disallowed: bad instruction opcode: {{.*}} extractvalue
-
-  %a2 = insertvalue {i32, float} undef, i32 1, 0
-; CHECK-NEXT: disallowed: bad instruction opcode: {{.*}} insertvalue
-
-  ret void
-}
-
-define internal void @memory() {
-; CHECK: ERROR: Function memory
-; Memory operations
-  %a1 = alloca i8, i32 4
-  %ptr = inttoptr i32 0 to i32*
-  %a2 = load i32, i32* %ptr, align 1
-  store i32 undef, i32* %ptr, align 1
- ; CHECK-NOT: disallowed
-  %a4 = getelementptr { i32, i32}, { i32, i32}* undef ; CHECK-NEXT: disallowed: bad instruction opcode: {{.*}} getelementptr
-  ret void
-}
-
-define internal void @vector_memory() {
-; CHECK: ERROR: Function vector_memory
-; Vector memory operations.
-  %ptr16xi8 = inttoptr i32 0 to <16 x i8>*
-  %ptr8xi16 = inttoptr i32 0 to <8 x i16>*
-  %ptr4xi32 = inttoptr i32 0 to <4 x i32>*
-  %ptr4xfloat = inttoptr i32 0 to <4 x float>*
-
-  %l16xi8 = load <16 x i8>, <16 x i8>* %ptr16xi8, align 1
-  %l8xi16 = load <8 x i16>, <8 x i16>* %ptr8xi16, align 2
-  %l4xi32 = load <4 x i32>, <4 x i32>* %ptr4xi32, align 4
-  %l4xfloat = load <4 x float>, <4 x float>* %ptr4xfloat, align 4
-
-  store <16 x i8> undef, <16 x i8>* %ptr16xi8, align 1
-  store <8 x i16> undef, <8 x i16>* %ptr8xi16, align 2
-  store <4 x i32> undef, <4 x i32>* %ptr4xi32, align 4
-  store <4 x float> undef, <4 x float>* %ptr4xfloat, align 4
-
- ; CHECK-NOT: disallowed
- ; The following vector types are disallowed.
-  %ptr2xi1 = inttoptr i32 0 to <2 x i1>*         ; CHECK-NEXT: disallowed: bad result type: <2 x i1>*
-  %ptr4xi1 = inttoptr i32 0 to <4 x i1>*         ; CHECK-NEXT: disallowed: bad result type: <4 x i1>*
-  %ptr8xi1 = inttoptr i32 0 to <8 x i1>*	 ; CHECK-NEXT: disallowed: bad result type: <8 x i1>*
-  %ptr16xi1 = inttoptr i32 0 to <16 x i1>*	 ; CHECK-NEXT: disallowed: bad result type: <16 x i1>*
-  %ptr32xi1 = inttoptr i32 0 to <32 x i1>*       ; CHECK-NEXT: disallowed: bad result type: <32 x i1>*
-  %ptr64xi1 = inttoptr i32 0 to <64 x i1>*       ; CHECK-NEXT: disallowed: bad result type: <64 x i1>*
-  %ptr2xi8 = inttoptr i32 0 to <2 x i8>*         ; CHECK-NEXT: disallowed: bad result type: <2 x i8>*
-  %ptr4xi8 = inttoptr i32 0 to <4 x i8>*         ; CHECK-NEXT: disallowed: bad result type: <4 x i8>*
-  %ptr32xi8 = inttoptr i32 0 to <32 x i8>*       ; CHECK-NEXT: disallowed: bad result type: <32 x i8>*
-  %ptr64xi8 = inttoptr i32 0 to <64 x i8>*       ; CHECK-NEXT: disallowed: bad result type: <64 x i8>*
-  %ptr2xi16 = inttoptr i32 0 to <2 x i16>*       ; CHECK-NEXT: disallowed: bad result type: <2 x i16>*
-  %ptr4xi16 = inttoptr i32 0 to <4 x i16>*       ; CHECK-NEXT: disallowed: bad result type: <4 x i16>*
-  %ptr16xi16 = inttoptr i32 0 to <16 x i16>*     ; CHECK-NEXT: disallowed: bad result type: <16 x i16>*
-  %ptr32xi16 = inttoptr i32 0 to <32 x i16>*     ; CHECK-NEXT: disallowed: bad result type: <32 x i16>*
-  %ptr2xi32 = inttoptr i32 0 to <2 x i32>*       ; CHECK-NEXT: disallowed: bad result type: <2 x i32>*
-  %ptr8xi32 = inttoptr i32 0 to <8 x i32>*       ; CHECK-NEXT: disallowed: bad result type: <8 x i32>*
-  %ptr16xi32 = inttoptr i32 0 to <16 x i32>*     ; CHECK-NEXT: disallowed: bad result type: <16 x i32>*
-  %ptr2xi64 = inttoptr i32 0 to <2 x i64>*       ; CHECK-NEXT: disallowed: bad result type: <2 x i64>*
-  %ptr4xi64 = inttoptr i32 0 to <4 x i64>*       ; CHECK-NEXT: disallowed: bad result type: <4 x i64>*
-  %ptr8xi64 = inttoptr i32 0 to <8 x i64>*       ; CHECK-NEXT: disallowed: bad result type: <8 x i64>*
-  %ptr2xfloat = inttoptr i32 0 to <2 x float>*   ; CHECK-NEXT: disallowed: bad result type: <2 x float>*
-  %ptr8xfloat = inttoptr i32 0 to <8 x float>*   ; CHECK-NEXT: disallowed: bad result type: <8 x float>*
-  %ptr16xfloat = inttoptr i32 0 to <16 x float>* ; CHECK-NEXT: disallowed: bad result type: <16 x float>*
-  %ptr2xdouble = inttoptr i32 0 to <2 x double>* ; CHECK-NEXT: disallowed: bad result type: <2 x double>*
-  %ptr4xdouble = inttoptr i32 0 to <4 x double>* ; CHECK-NEXT: disallowed: bad result type: <4 x double>*
-  %ptr8xdouble = inttoptr i32 0 to <8 x double>* ; CHECK-NEXT: disallowed: bad result type: <8 x double>*
-
-  ; i1 vector pointers are simply disallowed, their alignment is inconsequential.
-  %l4xi1 = load <4 x i1>, <4 x i1>* %ptr4xi1, align 1    ; CHECK-NEXT: disallowed: bad pointer: %l4xi1 = load <4 x i1>, <4 x i1>* %ptr4xi1, align 1
-  %l8xi1 = load <8 x i1>, <8 x i1>* %ptr8xi1, align 1    ; CHECK-NEXT: disallowed: bad pointer: %l8xi1 = load <8 x i1>, <8 x i1>* %ptr8xi1, align 1
-  %l16xi1 = load <16 x i1>, <16 x i1>* %ptr16xi1, align 1 ; CHECK-NEXT: disallowed: bad pointer: %l16xi1 = load <16 x i1>, <16 x i1>* %ptr16xi1, align 1
-
-  store <4 x i1> undef, <4 x i1>* %ptr4xi1, align 1    ; CHECK-NEXT: disallowed: bad pointer: store <4 x i1> undef, <4 x i1>* %ptr4xi1, align 1
-  store <8 x i1> undef, <8 x i1>* %ptr8xi1, align 1    ; CHECK-NEXT: disallowed: bad pointer: store <8 x i1> undef, <8 x i1>* %ptr8xi1, align 1
-  store <16 x i1> undef, <16 x i1>* %ptr16xi1, align 1 ; CHECK-NEXT: disallowed: bad pointer: store <16 x i1> undef, <16 x i1>* %ptr16xi1, align 1
-
-  ; Under- or over-aligned load/store are disallowed.
-  %a1_8xi16 = load <8 x i16>, <8 x i16>* %ptr8xi16, align 1       ; CHECK-NEXT: disallowed: bad alignment: %a1_8xi16 = load <8 x i16>, <8 x i16>* %ptr8xi16, align 1
-  %a1_4xi32 = load <4 x i32>, <4 x i32>* %ptr4xi32, align 1       ; CHECK-NEXT: disallowed: bad alignment:	%a1_4xi32 = load <4 x i32>, <4 x i32>* %ptr4xi32, align 1
-  %a1_4xfloat = load <4 x float>, <4 x float>* %ptr4xfloat, align 1 ; CHECK-NEXT: disallowed: bad alignment:	%a1_4xfloat = load <4 x float>, <4 x float>* %ptr4xfloat, align 1
-
-  %a16_16xi8 = load <16 x i8>, <16 x i8>* %ptr16xi8, align 16       ; CHECK-NEXT: disallowed: bad alignment: %a16_16xi8 = load <16 x i8>, <16 x i8>* %ptr16xi8, align 16
-  %a16_8xi16 = load <8 x i16>, <8 x i16>* %ptr8xi16, align 16       ; CHECK-NEXT: disallowed: bad alignment: %a16_8xi16 = load <8 x i16>, <8 x i16>* %ptr8xi16, align 16
-  %a16_4xi32 = load <4 x i32>, <4 x i32>* %ptr4xi32, align 16       ; CHECK-NEXT: disallowed: bad alignment: %a16_4xi32 = load <4 x i32>, <4 x i32>* %ptr4xi32, align 16
-  %a16_4xfloat = load <4 x float>, <4 x float>* %ptr4xfloat, align 16 ; CHECK-NEXT: disallowed: bad alignment: %a16_4xfloat = load <4 x float>, <4 x float>* %ptr4xfloat, align 16
-
-  store <8 x i16> undef, <8 x i16>* %ptr8xi16, align 1       ; CHECK-NEXT: disallowed: bad alignment: store <8 x i16> undef, <8 x i16>* %ptr8xi16, align 1
-  store <4 x i32> undef, <4 x i32>* %ptr4xi32, align 1	     ; CHECK-NEXT: disallowed: bad alignment: store <4 x i32> undef, <4 x i32>* %ptr4xi32, align 1
-  store <4 x float> undef, <4 x float>* %ptr4xfloat, align 1 ; CHECK-NEXT: disallowed: bad alignment: store <4 x float> undef, <4 x float>* %ptr4xfloat, align 1
-
-  store <16 x i8> undef, <16 x i8>* %ptr16xi8, align 16       ; CHECK-NEXT: disallowed: bad alignment: store <16 x i8> undef, <16 x i8>* %ptr16xi8, align 16
-  store <8 x i16> undef, <8 x i16>* %ptr8xi16, align 16	      ; CHECK-NEXT: disallowed: bad alignment: store <8 x i16> undef, <8 x i16>* %ptr8xi16, align 16
-  store <4 x i32> undef, <4 x i32>* %ptr4xi32, align 16	      ; CHECK-NEXT: disallowed: bad alignment: store <4 x i32> undef, <4 x i32>* %ptr4xi32, align 16
-  store <4 x float> undef, <4 x float>* %ptr4xfloat, align 16 ; CHECK-NEXT: disallowed: bad alignment: store <4 x float> undef, <4 x float>* %ptr4xfloat, align 16
-
- ret void
-}
-
-define internal void @atomic() {
-; CHECK: ERROR: Function atomic
-  %ptr = inttoptr i32 0 to i32*
-
- ; CHECK-NOT: disallowed
-  %la = load atomic i32, i32* %ptr seq_cst, align 4                      ; CHECK: disallowed: atomic load: {{.*}} load atomic
-  %lv = load volatile i32, i32* %ptr, align 4                            ; CHECK: disallowed: volatile load: {{.*}} load volatile
-  store atomic i32 undef, i32* %ptr seq_cst, align 4                ; CHECK: disallowed: atomic store: store atomic
-  store volatile i32 undef, i32* %ptr, align 4                      ; CHECK: disallowed: volatile store: store volatile
-  fence acq_rel                                                     ; CHECK: disallowed: bad instruction opcode: fence
-  %cmpx = cmpxchg i32* %ptr, i32 undef, i32 undef acq_rel monotonic ; CHECK: disallowed: bad instruction opcode: {{.*}} cmpxchg
-  %crm = atomicrmw add i32* %ptr, i32 1 acquire                     ; CHECK: disallowed: bad instruction opcode: {{.*}} atomicrmw
-  ret void
-}
-
-define internal void @atomic_vector() {
-; CHECK: ERROR: Function atomic_vector
-  %ptr = inttoptr i32 0 to <4 x i32>*
-
- ; CHECK-NOT: disallowed
-  %la = load atomic <4 x i32>, <4 x i32>* %ptr seq_cst, align 1             ; CHECK: disallowed: atomic load: {{.*}} load atomic
-  %lv = load volatile <4 x i32>, <4 x i32>* %ptr, align 1                   ; CHECK: disallowed: volatile load: {{.*}} load volatile
-  store atomic <4 x i32> undef, <4 x i32>* %ptr seq_cst, align 1 ; CHECK: disallowed: atomic store: store atomic
-  store volatile <4 x i32> undef, <4 x i32>* %ptr, align 1       ; CHECK: disallowed: volatile store: store volatile
-  ret void
-}
-
-define internal void @conversion() {
-; CHECK-NOT: Function conversion
-; Conversion operations
-  %t = trunc i32 undef to i8
-  %z = zext i8 undef to i32
-  %s = sext i8 undef to i32
-  %ft = fptrunc double undef to float
-  %fe = fpext float undef to double
-  %fu32 = fptoui float undef to i32
-  %fs32 = fptosi float undef to i32
-  %fu64 = fptoui double undef to i64
-  %fs64 = fptosi double undef to i64
-  %uf32 = uitofp i32 undef to float
-  %sf32 = sitofp i32 undef to float
-  %uf64 = uitofp i64 undef to double
-  %sf64 = sitofp i64 undef to double
-  ret void
-}
-; CHECK-NOT: disallowed
-
-define internal void @vector_conversion() {
-; CHECK: ERROR: Function vector_conversion
-  %t1 = trunc <4 x i32> undef to <4 x i16> ; CHECK-NEXT: bad result type: <4 x i16>
-  %t2 = trunc <4 x i32> undef to <4 x i8> ; CHECK-NEXT: bad result type: <4 x i8>
-  %t3 = trunc <8 x i16> undef to <8 x i8> ; CHECK-NEXT: bad result type: <8 x i8>
-  %z1 = zext <8 x i16> undef to <8 x i32> ; CHECK-NEXT: bad result type: <8 x i32>
-  %z2 = zext <8 x i1> undef to <8 x i32> ; CHECK-NEXT: bad result type: <8 x i32>
-  %z4 = zext <8 x i1> undef to <8 x i8> ; CHECK-NEXT: bad result type: <8 x i8>
-  %z5 = zext <16 x i8> undef to <16 x i16> ; CHECK-NEXT: bad result type: <16 x i16>
-  %z6 = zext <16 x i8> undef to <16 x i32> ; CHECK-NEXT: bad result type: <16 x i32>
-  %z7 = zext <16 x i1> undef to <16 x i32> ; CHECK-NEXT: bad result type: <16 x i32>
-  %z8 = zext <16 x i1> undef to <16 x i16> ; CHECK-NEXT: bad result type: <16 x i16>
-  %s1 = sext <8 x i16> undef to <8 x i32> ; CHECK-NEXT: bad result type: <8 x i32>
-  %s2 = sext <8 x i1> undef to <8 x i32> ; CHECK-NEXT: bad result type: <8 x i32>
-  %s4 = sext <8 x i1> undef to <8 x i8> ; CHECK-NEXT: bad result type: <8 x i8>
-  %s5 = sext <16 x i8> undef to <16 x i16> ; CHECK-NEXT: bad result type: <16 x i16>
-  %s6 = sext <16 x i8> undef to <16 x i32> ; CHECK-NEXT: bad result type: <16 x i32>
-  %s7 = sext <16 x i1> undef to <16 x i32> ; CHECK-NEXT: bad result type: <16 x i32>
-  %s8 = sext <16 x i1> undef to <16 x i16> ; CHECK-NEXT: bad result type: <16 x i16>
-  %ft = fptrunc <2 x double> undef to <2 x float> ; CHECK-NEXT: bad operand
-  %fe = fpext <4 x float> undef to <4 x double> ; CHECK-NEXT: bad result type: <4 x double>
-  %fu64 = fptoui <2 x double> undef to <2 x i64> ; CHECK-NEXT: bad operand
-  %fs64 = fptosi <2 x double> undef to <2 x i64> ; CHECK-NEXT: bad operand
-  %uf64 = uitofp <2 x i64> undef to <2 x double> ; CHECK-NEXT: bad operand
-  %sf64 = sitofp <2 x i64> undef to <2 x double> ; CHECK-NEXT: bad operand
-
-  ; Conversions to allowed types should work.
-  %t4 = trunc <4 x i32> undef to <4 x i1> ; CHECK-NOT: bad
-  %t5 = trunc <8 x i16> undef to <8 x i1> ; CHECK-NOT: bad
-  %t6 = trunc <16 x i8> undef to <16 x i1> ; CHECK-NOT: bad
-  %z10 = zext <4 x i1> undef to <4 x i32> ; CHECK-NOT: bad
-  %z11 = zext <8 x i1> undef to <8 x i16> ; CHECK-NOT: bad
-  %z12 = zext <16 x i1> undef to <16 x i8> ; CHECK-NOT: bad
-  %s10 = sext <4 x i1> undef to <4 x i32> ; CHECK-NOT: bad
-  %s11 = sext <8 x i1> undef to <8 x i16> ; CHECK-NOT: bad
-  %s12 = sext <16 x i1> undef to <16 x i8> ; CHECK-NOT: bad
-  %fu32 = fptoui <4 x float> undef to <4 x i32> ; CHECK-NOT: bad
-  %fs32 = fptosi <4 x float> undef to <4 x i32> ; CHECK-NOT: bad
-  %uf32 = uitofp <4 x i32> undef to <4 x float> ; CHECK-NOT: bad
-  %sf32 = sitofp <4 x i32> undef to <4 x float> ; CHECK-NOT: bad
-  ret void
-}
-; CHECK-NOT: disallowed
-
-define internal void @other() {
-; CHECK-NOT: Function other
-entry:
-  %icmp8 = icmp eq i8 undef, undef
-  %icmp16 = icmp eq i16 undef, undef
-  %icmp32 = icmp eq i32 undef, undef
-  %icmp64 = icmp eq i64 undef, undef
-  %vicmp8 = icmp eq <16 x i8> undef, undef
-  %vicmp16 = icmp eq <8 x i16> undef, undef
-  %vicmp32 = icmp eq <4 x i32> undef, undef
-  %fcmp32 = fcmp oeq float undef, undef
-  %fcmp64 = fcmp oeq double undef, undef
-  %vfcmp = fcmp oeq <4 x float> undef, undef
-  br i1 undef, label %foo, label %bar
-foo:
-; phi predecessor labels have to match to appease module verifier
-  %phi1 = phi i1 [0, %entry], [0, %foo]
-  %phi8 = phi i8 [0, %entry], [0, %foo]
-  %phi16 = phi i16 [0, %entry], [0, %foo]
-  %phi32 = phi i32 [0, %entry], [0, %foo]
-  %phi64 = phi i64 [0, %entry], [0, %foo]
-  %vphi4x1 = phi <4 x i1> [undef, %entry], [undef, %foo]
-  %vphi8x1 = phi <8 x i1> [undef, %entry], [undef, %foo]
-  %vphi16x1 = phi <16 x i1> [undef, %entry], [undef, %foo]
-  %vphi8 = phi <16 x i8> [undef, %entry], [undef, %foo]
-  %vphi16 = phi <8 x i16> [undef, %entry], [undef, %foo]
-  %vphi32 = phi <4 x i32> [undef, %entry], [undef, %foo]
-
-  %select = select i1 true, i8 undef, i8 undef
-
-  %vselect4x1 = select i1 true, <4 x i1> undef, <4 x i1> undef
-  %vselect8x1 = select i1 true, <8 x i1> undef, <8 x i1> undef
-  %vselect16x1 = select i1 true, <16 x i1> undef, <16 x i1> undef
-  %vselect8 = select i1 true, <16 x i8> undef, <16 x i8> undef
-  %vselect16 = select i1 true, <8 x i16> undef, <8 x i16> undef
-  %vselect32 = select i1 true, <4 x i32> undef, <4 x i32> undef
-
-  %vvselect4x1 = select <4 x i1> undef, <4 x i1> undef, <4 x i1> undef
-  %vvselect8x1 = select <8 x i1> undef, <8 x i1> undef, <8 x i1> undef
-  %vvselect16x1 = select <16 x i1> undef, <16 x i1> undef, <16 x i1> undef
-  %vvselect8 = select <16 x i1> undef, <16 x i8> undef, <16 x i8> undef
-  %vvselect16 = select <8 x i1> undef, <8 x i16> undef, <8 x i16> undef
-  %vvselect32 = select <4 x i1> undef, <4 x i32> undef, <4 x i32> undef
-
-  call void @conversion()
-  br i1 undef, label %foo, label %bar
-bar:
-  ret void
-}
-; CHECK-NOT: disallowed
-
-define internal void @throwing_func() {
-; CHECK-NOT: Function throwing_func
-  ret void
-}
-; CHECK-NOT: disallowed
-
-define internal void @personality_func() {
-; CHECK-NOT: Function personality_func
-  ret void
-}
-; CHECK-NOT: disallowed
-
-define internal void @invoke_func() {
-; CHECK: ERROR: Function invoke_func
-  invoke void @throwing_func() to label %ok unwind label %onerror
-; CHECK-NOT: disallowed
-; CHECK: disallowed: bad instruction opcode: invoke
-ok:
-  ret void
-onerror:
-  %lp = landingpad i32
-      personality i8* bitcast (void ()* @personality_func to i8*)
-      catch i32* null
-; CHECK: disallowed: bad instruction opcode: {{.*}} landingpad
-  resume i32 %lp
-; CHECK: disallowed: bad instruction opcode: resume
-}
-
-define internal i32 @va_arg(i32 %va_list_as_int) {
-; CHECK: ERROR: Function va_arg
-  %va_list = inttoptr i32 %va_list_as_int to i8*
-  %val = va_arg i8* %va_list, i32
-  ret i32 %val
-}
-; CHECK-NOT: disallowed
-; CHECK: disallowed: bad instruction opcode: {{.*}} va_arg
-
-@global_var = internal global [4 x i8] zeroinitializer
-
-define internal void @constantexpr() {
-; CHECK: ERROR: Function constantexpr
-  ptrtoint i8* getelementptr ([4 x i8], [4 x i8]* @global_var, i32 1, i32 0) to i32
-  ret void
-}
-; CHECK-NOT: disallowed
-; CHECK: disallowed: operand not InherentPtr: %1 = ptrtoint i8* getelementptr
-
-define internal void @inline_asm() {
-; CHECK: ERROR: Function inline_asm
-  call void asm "foo", ""()
-  ret void
-}
-; CHECK-NOT: disallowed
-; CHECK: disallowed: inline assembly: call void asm "foo", ""()
-
-; CHECK-NOT: disallowed
-; If another check is added, there should be a check-not in between each check
diff --git a/test/NaCl/PNaClABI/intrinsics.ll b/test/NaCl/PNaClABI/intrinsics.ll
deleted file mode 100644
index 4d1f29b9149..00000000000
--- a/test/NaCl/PNaClABI/intrinsics.ll
+++ /dev/null
@@ -1,168 +0,0 @@
-; RUN: not pnacl-abicheck < %s | FileCheck %s
-; RUN: not pnacl-abicheck -pnaclabi-allow-debug-metadata < %s | \
-; RUN:   FileCheck %s --check-prefix=DBG
-
-; Test that only white-listed intrinsics are allowed.
-
-; ===================================
-; Always allowed intrinsics.
-
-declare void @llvm.memcpy.p0i8.p0i8.i32(i8* %dest, i8* %src,
-                                        i32 %len, i32 %align, i1 %isvolatile)
-declare void @llvm.memmove.p0i8.p0i8.i32(i8* %dest, i8* %src,
-                                         i32 %len, i32 %align, i1 %isvolatile)
-declare void @llvm.memset.p0i8.i32(i8* %dest, i8 %val,
-                                    i32 %len, i32 %align, i1 %isvolatile)
-
-declare i8* @llvm.nacl.read.tp()
-
-declare i8 @llvm.nacl.atomic.load.i8(i8*, i32)
-declare i16 @llvm.nacl.atomic.load.i16(i16*, i32)
-declare i32 @llvm.nacl.atomic.load.i32(i32*, i32)
-declare i64 @llvm.nacl.atomic.load.i64(i64*, i32)
-declare void @llvm.nacl.atomic.store.i8(i8, i8*, i32)
-declare void @llvm.nacl.atomic.store.i16(i16, i16*, i32)
-declare void @llvm.nacl.atomic.store.i32(i32, i32*, i32)
-declare void @llvm.nacl.atomic.store.i64(i64, i64*, i32)
-declare i8 @llvm.nacl.atomic.rmw.i8(i32, i8*, i8, i32)
-declare i16 @llvm.nacl.atomic.rmw.i16(i32, i16*, i16, i32)
-declare i32 @llvm.nacl.atomic.rmw.i32(i32, i32*, i32, i32)
-declare i64 @llvm.nacl.atomic.rmw.i64(i32, i64*, i64, i32)
-declare i8 @llvm.nacl.atomic.cmpxchg.i8(i8*, i8, i8, i32, i32)
-declare i16 @llvm.nacl.atomic.cmpxchg.i16(i16*, i16, i16, i32, i32)
-declare i32 @llvm.nacl.atomic.cmpxchg.i32(i32*, i32, i32, i32, i32)
-declare i64 @llvm.nacl.atomic.cmpxchg.i64(i64*, i64, i64, i32, i32)
-declare void @llvm.nacl.atomic.fence(i32)
-declare void @llvm.nacl.atomic.fence.all()
-declare i1 @llvm.nacl.atomic.is.lock.free(i32, i8*)
-
-declare i16 @llvm.bswap.i16(i16)
-declare i32 @llvm.bswap.i32(i32)
-declare i64 @llvm.bswap.i64(i64)
-
-declare i32 @llvm.cttz.i32(i32, i1)
-declare i64 @llvm.cttz.i64(i64, i1)
-
-declare i32 @llvm.ctlz.i32(i32, i1)
-declare i64 @llvm.ctlz.i64(i64, i1)
-
-declare i32 @llvm.ctpop.i32(i32)
-declare i64 @llvm.ctpop.i64(i64)
-
-declare void @llvm.trap()
-
-declare float @llvm.fabs.f32(float)
-declare double @llvm.fabs.f64(double)
-declare <4 x float> @llvm.fabs.v4f32(<4 x float>)
-
-declare float @llvm.sqrt.f32(float)
-declare double @llvm.sqrt.f64(double)
-
-declare i8* @llvm.stacksave()
-declare void @llvm.stackrestore(i8*)
-
-declare void @llvm.nacl.longjmp(i8*, i32)
-declare i32 @llvm.nacl.setjmp(i8*)
-
-; CHECK-NOT: disallowed
-
-; ===================================
-; Always disallowed intrinsics.
-
-; CHECK: Function llvm.adjust.trampoline is a disallowed LLVM intrinsic
-; DBG: Function llvm.adjust.trampoline is a disallowed LLVM intrinsic
-declare i8* @llvm.adjust.trampoline(i8*)
-
-; CHECK: Function llvm.init.trampoline is a disallowed LLVM intrinsic
-; DBG: Function llvm.init.trampoline is a disallowed LLVM intrinsic
-declare void @llvm.init.trampoline(i8*, i8*, i8*)
-
-; CHECK: Function llvm.x86.aesni.aeskeygenassist is a disallowed LLVM intrinsic
-; DBG: Function llvm.x86.aesni.aeskeygenassist is a disallowed LLVM intrinsic
-declare <2 x i64> @llvm.x86.aesni.aeskeygenassist(<2 x i64>, i8)
-
-; CHECK: Function llvm.va_copy is a disallowed LLVM intrinsic
-; DBG: Function llvm.va_copy is a disallowed LLVM intrinsic
-declare void @llvm.va_copy(i8*, i8*)
-
-; CHECK: Function llvm.bswap.i1 is a disallowed LLVM intrinsic
-declare i1 @llvm.bswap.i1(i1)
-
-; CHECK: Function llvm.bswap.i8 is a disallowed LLVM intrinsic
-declare i8 @llvm.bswap.i8(i8)
-
-; CHECK: Function llvm.ctlz.i16 is a disallowed LLVM intrinsic
-declare i16 @llvm.ctlz.i16(i16, i1)
-
-; CHECK: Function llvm.cttz.i16 is a disallowed LLVM intrinsic
-declare i16 @llvm.cttz.i16(i16, i1)
-
-; CHECK: Function llvm.ctpop.i16 is a disallowed LLVM intrinsic
-declare i16 @llvm.ctpop.i16(i16)
-
-; CHECK: Function llvm.lifetime.start is a disallowed LLVM intrinsic
-declare void @llvm.lifetime.start(i64, i8* nocapture)
-
-; CHECK: Function llvm.lifetime.end is a disallowed LLVM intrinsic
-declare void @llvm.lifetime.end(i64, i8* nocapture)
-
-; CHECK: Function llvm.frameaddress is a disallowed LLVM intrinsic
-declare i8* @llvm.frameaddress(i32 %level)
-
-; CHECK: Function llvm.returnaddress is a disallowed LLVM intrinsic
-declare i8* @llvm.returnaddress(i32 %level)
-
-; CHECK: Function llvm.fabs.f16 is a disallowed LLVM intrinsic
-declare half @llvm.fabs.f16(half)
-
-; CHECK: Function llvm.fabs.v2f16 is a disallowed LLVM intrinsic
-declare <2 x half> @llvm.fabs.v2f16(<2 x half>)
-; CHECK: Function llvm.fabs.v4f16 is a disallowed LLVM intrinsic
-declare <4 x half> @llvm.fabs.v4f16(<4 x half>)
-; CHECK: Function llvm.fabs.v8f16 is a disallowed LLVM intrinsic
-declare <8 x half> @llvm.fabs.v8f16(<8 x half>)
-
-; CHECK: Function llvm.fabs.v2f32 is a disallowed LLVM intrinsic
-declare <2 x float> @llvm.fabs.v2f32(<2 x float>)
-; CHECK: Function llvm.fabs.v8f32 is a disallowed LLVM intrinsic
-declare <8 x float> @llvm.fabs.v8f32(<8 x float>)
-
-; CHECK: Function llvm.fabs.v2f64 is a disallowed LLVM intrinsic
-declare <2 x double> @llvm.fabs.v2f64(<2 x double>)
-; CHECK: Function llvm.fabs.v4f64 is a disallowed LLVM intrinsic
-declare <4 x double> @llvm.fabs.v4f64(<4 x double>)
-; CHECK: Function llvm.fabs.v8f64 is a disallowed LLVM intrinsic
-declare <8 x double> @llvm.fabs.v8f64(<8 x double>)
-
-; CHECK: Function llvm.sqrt.fp128 is a disallowed LLVM intrinsic
-declare fp128 @llvm.sqrt.fp128(fp128)
-
-; The variants with 64-bit %len arguments are disallowed.
-; CHECK: Function llvm.memcpy.p0i8.p0i8.i64 is a disallowed LLVM intrinsic
-declare void @llvm.memcpy.p0i8.p0i8.i64(i8* %dest, i8* %src,
-                                        i64 %len, i32 %align, i1 %isvolatile)
-; CHECK: Function llvm.memmove.p0i8.p0i8.i64 is a disallowed LLVM intrinsic
-declare void @llvm.memmove.p0i8.p0i8.i64(i8* %dest, i8* %src,
-                                         i64 %len, i32 %align, i1 %isvolatile)
-; CHECK: Function llvm.memset.p0i8.i64 is a disallowed LLVM intrinsic
-declare void @llvm.memset.p0i8.i64(i8* %dest, i8 %val,
-                                    i64 %len, i32 %align, i1 %isvolatile)
-
-; Test that the ABI checker checks the full function name.
-; CHECK: Function llvm.memset.foo is a disallowed LLVM intrinsic
-declare void @llvm.memset.foo(i8* %dest, i8 %val,
-                              i64 %len, i32 %align, i1 %isvolatile)
-
-; A debuginfo version is required.
-!llvm.module.flags = !{!0}
-!0 = !{i32 1, !"Debug Info Version", i32 3}
-
-; ===================================
-; Debug info intrinsics, which are disallowed by default.
-
-; CHECK: Function llvm.dbg.value is a disallowed LLVM intrinsic
-; DBG-NOT: Function llvm.dbg.value is a disallowed LLVM intrinsic
-declare void @llvm.dbg.value(metadata, i64, metadata, metadata)
-; CHECK: Function llvm.dbg.declare is a disallowed LLVM intrinsic
-; DBG-NOT: Function llvm.dbg.declare is a disallowed LLVM intrinsic
-declare void @llvm.dbg.declare(metadata, metadata, metadata)
diff --git a/test/NaCl/PNaClABI/linkagetypes.ll b/test/NaCl/PNaClABI/linkagetypes.ll
deleted file mode 100644
index 005497a69f6..00000000000
--- a/test/NaCl/PNaClABI/linkagetypes.ll
+++ /dev/null
@@ -1,78 +0,0 @@
-; RUN: not pnacl-abicheck < %s | FileCheck %s
-; Test linkage types allowed by PNaCl ABI
-
-target datalayout = "e-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-p:32:32:32-v128:32:32"
-target triple = "le32-unknown-nacl"
-
-
-@gv_internal = internal global [1 x i8] c"x"
-; CHECK-NOT: disallowed
-
-@gv_private = private global [1 x i8] c"x"
-; CHECK: Variable gv_private has disallowed linkage type: private
-@gv_linkonce = linkonce global [1 x i8] c"x"
-; CHECK: gv_linkonce has disallowed linkage type: linkonce
-@gv_linkonce_odr = linkonce_odr global [1 x i8] c"x"
-; CHECK: gv_linkonce_odr has disallowed linkage type: linkonce_odr
-@gv_weak = weak global [1 x i8] c"x"
-; CHECK: gv_weak has disallowed linkage type: weak
-@gv_weak_odr = weak_odr global [1 x i8] c"x"
-; CHECK: gv_weak_odr has disallowed linkage type: weak_odr
-@gv_common = common global [1 x i8] c"x"
-; CHECK: gv_common has disallowed linkage type: common
-@gv_appending = appending global [1 x i8] zeroinitializer
-; CHECK: gv_appending has disallowed linkage type: appending
-@gv_dllimport = external dllimport global [1 x i8]
-; CHECK: gv_dllimport is not a valid external symbol (disallowed)
-@gv_dllexport = dllexport global [1 x i8] c"x"
-; CHECK: gv_dllexport is not a valid external symbol (disallowed)
-@gv_extern_weak = extern_weak global [1 x i8]
-; CHECK: gv_extern_weak has disallowed linkage type: extern_weak
-@gv_available_externally = available_externally global [1 x i8] c"x"
-; CHECK: gv_available_externally has disallowed linkage type: available_externally
-
-
-; CHECK-NOT: disallowed
-; CHECK-NOT: internal_func
-; internal linkage is allowed, and should not appear in error output.
-define internal void @internal_func() {
-  ret void
-}
-
-; CHECK: Function private_func has disallowed linkage type: private
-define private void @private_func() {
-  ret void
-}
-; CHECK: Function external_func is declared but not defined (disallowed)
-declare external void @external_func()
-; CHECK: linkonce_func has disallowed linkage type: linkonce
-define linkonce void @linkonce_func() {
-  ret void
-}
-; CHECK-NEXT: linkonce_odr_func has disallowed linkage type: linkonce_odr
-define linkonce_odr void @linkonce_odr_func() {
-  ret void
-}
-; CHECK-NEXT: weak_func has disallowed linkage type: weak
-define weak void @weak_func() {
-  ret void
-}
-; CHECK-NEXT: weak_odr_func has disallowed linkage type: weak_odr
-define weak_odr void @weak_odr_func() {
-  ret void
-}
-; CHECK-NEXT: dllimport_func is declared but not defined (disallowed)
-; CHECK-NEXT: dllimport_func is not a valid external symbol (disallowed)
-declare dllimport void @dllimport_func()
-; CHECK-NEXT: dllexport_func is not a valid external symbol (disallowed)
-define dllexport void @dllexport_func() {
-  ret void
-}
-; CHECK-NEXT: Function extern_weak_func is declared but not defined (disallowed)
-; CHECK-NEXT: Function extern_weak_func has disallowed linkage type: extern_weak
-declare extern_weak void @extern_weak_func()
-
-; CHECK-NEXT: Function avail_ext_func has disallowed linkage type: available_externally
-define available_externally void @avail_ext_func() {
-  ret void
-}
diff --git a/test/NaCl/PNaClABI/lit.local.cfg b/test/NaCl/PNaClABI/lit.local.cfg
deleted file mode 100644
index c6106e4746f..00000000000
--- a/test/NaCl/PNaClABI/lit.local.cfg
+++ /dev/null
@@ -1 +0,0 @@
-config.suffixes = ['.ll']
diff --git a/test/NaCl/PNaClABI/module-asm.ll b/test/NaCl/PNaClABI/module-asm.ll
deleted file mode 100644
index cd32554f59f..00000000000
--- a/test/NaCl/PNaClABI/module-asm.ll
+++ /dev/null
@@ -1,4 +0,0 @@
-; RUN: not pnacl-abicheck < %s | FileCheck %s
-
-module asm "foo"
-; CHECK: Module contains disallowed top-level inline assembly
diff --git a/test/NaCl/PNaClABI/types-function.ll b/test/NaCl/PNaClABI/types-function.ll
deleted file mode 100644
index dfb876c1142..00000000000
--- a/test/NaCl/PNaClABI/types-function.ll
+++ /dev/null
@@ -1,40 +0,0 @@
-; RUN: not pnacl-abicheck < %s | FileCheck %s
-; Test type-checking in function bodies. This test is not intended to verify
-; all the rules about the various types, but instead to make sure that types
-; stashed in various places in function bodies are caught.
-
-@a2 = private global i17 zeroinitializer
-
-; CHECK: Function func has disallowed type: void (i15)
-declare void @func(i15 %arg)
-
-!llvm.foo = !{!0}
-!0 = !{ half 0.0}
-
-define void @types() {
-; CHECK: bad result type: {{.*}} fptrunc
-  %h1 = fptrunc double undef to half
-
-; CHECK: bad operand: {{.*}} bitcast half
-  %h2 = bitcast half 0.0 to i16
-
-; see below...
-  %h3 = fadd double 0.0, fpext (half 0.0 to double)
-
-; CHECK: bad pointer: store
-  store i32 0, i32* bitcast (i17* @a2 to i32*), align 1
-
-; CHECK: bad function callee operand: call void @func(i15 1)
-  call void @func(i15 1)
-
-; CHECK: Function types has disallowed instruction metadata: !foo
-  ret void, !foo !0
-}
-; CHECK-NOT: disallowed
-
-
-; TODO:
-; the bitcode reader seems to expand some operations inline
-; (e.g. fpext, sext, uitofp) such that doing something like
-;   %h3 = fadd double 0.0, fpext (half 0.0 to double)
-; means the verifier pass will never see the fpext or its operands
diff --git a/test/NaCl/PNaClABI/types.ll b/test/NaCl/PNaClABI/types.ll
deleted file mode 100644
index d5ca57739d9..00000000000
--- a/test/NaCl/PNaClABI/types.ll
+++ /dev/null
@@ -1,206 +0,0 @@
-; RUN: not pnacl-abicheck < %s | FileCheck %s
-; Test types allowed by PNaCl ABI
-
-
-; CHECK: Function badReturn has disallowed type: half* ()
-define internal half* @badReturn() {
-  unreachable
-}
-
-; CHECK: Function badArgType1 has disallowed type: void (half, i32)
-define internal void @badArgType1(half %a, i32 %b) {
-  ret void
-}
-; CHECK: Function badArgType2 has disallowed type: void (i32, half)
-define internal void @badArgType2(i32 %a, half %b) {
-  ret void
-}
-
-
-define internal void @func() {
-entry:
-  br label %block
-block:
-
-  ; We test for allowed/disallowed types via phi nodes.  This gives us
-  ; a uniform way to test any type.
-
-  ; Allowed types
-
-  ; Scalars.
-  phi i1 [ undef, %entry ]
-  phi i8 [ undef, %entry ]
-  phi i16 [ undef, %entry ]
-  phi i32 [ undef, %entry ]
-  phi i64 [ undef, %entry ]
-  phi float [ undef, %entry ]
-  phi double [ undef, %entry ]
-  ; Vectors.
-  phi <4 x i1> [ undef, %entry ]
-  phi <8 x i1> [ undef, %entry ]
-  phi <16 x i1> [ undef, %entry ]
-  phi <16 x i8> [ undef, %entry ]
-  phi <8 x i16> [ undef, %entry ]
-  phi <4 x i32> [ undef, %entry ]
-  phi <4 x float> [ undef, %entry ]
-; CHECK-NOT: disallowed
-
-
-  ; Disallowed integer types
-
-  phi i4 [ undef, %entry ]
-; CHECK: Function func disallowed: bad operand: {{.*}} i4
-
-  phi i33 [ undef, %entry ]
-; CHECK-NEXT: disallowed: bad operand: {{.*}} i33
-
-  phi i128 [ undef, %entry ]
-; CHECK-NEXT: disallowed: bad operand: {{.*}} i128
-
-
-  ; Disallowed floating point types
-
-  phi half [ undef, %entry ]
-; CHECK-NEXT: disallowed: bad operand: {{.*}} half
-
-  phi x86_fp80 [ undef, %entry ]
-; CHECK-NEXT: disallowed: bad operand: {{.*}} x86_fp80
-
-  phi fp128 [ undef, %entry ]
-; CHECK-NEXT: disallowed: bad operand: {{.*}} fp128
-
-  phi ppc_fp128 [ undef, %entry ]
-; CHECK-NEXT: disallowed: bad operand: {{.*}} ppc_fp128
-
-  phi x86_mmx [ undef, %entry ]
-; CHECK-NEXT: disallowed: bad operand: {{.*}} x86_mmx
-
-
-  ; Derived types are disallowed too
-
-  phi i32* [ undef, %entry ]
-; CHECK-NEXT: disallowed: bad operand: {{.*}} i32*
-
-  phi [1 x i32] [ undef, %entry ]
-; CHECK-NEXT: disallowed: bad operand: {{.*}} [1 x i32]
-
-  phi { i32, float } [ undef, %entry ]
-; CHECK-NEXT: disallowed: bad operand: {{.*}} { i32, float }
-
-  phi void (i32)* [ undef, %entry ]
-; CHECK-NEXT: disallowed: bad operand: {{.*}} void (i32)*
-
-  phi <{ i8, i32 }> [ undef, %entry ]
-; CHECK-NEXT: disallowed: bad operand: {{.*}} <{ i8, i32 }>
-
-  ; Vector types of weird bit width are disallowed
-  phi <1 x i1> [ undef, %entry ]
-  phi <3 x i1> [ undef, %entry ]
-  phi <17 x i1> [ undef, %entry ]
-  phi <1 x i32> [ undef, %entry ]
-  phi <2 x i32> [ undef, %entry ]
-  phi <3 x i32> [ undef, %entry ]
-  phi <5 x i32> [ undef, %entry ]
-; CHECK-NEXT: disallowed: bad operand: {{.*}} <1 x i1>
-; CHECK-NEXT: disallowed: bad operand: {{.*}} <3 x i1>
-; CHECK-NEXT: disallowed: bad operand: {{.*}} <17 x i1>
-; CHECK-NEXT: disallowed: bad operand: {{.*}} <1 x i32>
-; CHECK-NEXT: disallowed: bad operand: {{.*}} <2 x i32>
-; CHECK-NEXT: disallowed: bad operand: {{.*}} <3 x i32>
-; CHECK-NEXT: disallowed: bad operand: {{.*}} <5 x i32>
-
-  ; i64 and double vectors are currently disallowed, and so are the
-  ; corresponding bool vectors.
-  phi <2 x i1> [ undef, %entry ]
-  phi <2 x i64> [ undef, %entry ]
-  phi <2 x double> [ undef, %entry ]
-; CHECK-NEXT: disallowed: bad operand: {{.*}} <2 x i1>
-; CHECK-NEXT: disallowed: bad operand: {{.*}} <2 x i64>
-; CHECK-NEXT: disallowed: bad operand: {{.*}} <2 x double>
-
-  ; 256-bit width vectors are currently disallowed, and so are the
-  ; corresponding bool vectors.
-  phi <32 x i1> [ undef, %entry ]
-  phi <32 x i8> [ undef, %entry ]
-  phi <16 x i16> [ undef, %entry ]
-  phi <8 x i32> [ undef, %entry ]
-  phi <4 x i64> [ undef, %entry ]
-  phi <8 x float> [ undef, %entry ]
-  phi <4 x double> [ undef, %entry ]
-; CHECK-NEXT: disallowed: bad operand: {{.*}} <32 x i1>
-; CHECK-NEXT: disallowed: bad operand: {{.*}} <32 x i8>
-; CHECK-NEXT: disallowed: bad operand: {{.*}} <16 x i16>
-; CHECK-NEXT: disallowed: bad operand: {{.*}} <8 x i32>
-; CHECK-NEXT: disallowed: bad operand: {{.*}} <4 x i64>
-; CHECK-NEXT: disallowed: bad operand: {{.*}} <8 x float>
-; CHECK-NEXT: disallowed: bad operand: {{.*}} <4 x double>
-
-  ; 512-bit width vectors are currently disallowed, and so are the
-  ; corresponding bool vectors.
-  phi <64 x i1> [ undef, %entry ]
-  phi <64 x i8> [ undef, %entry ]
-  phi <32 x i16> [ undef, %entry ]
-  phi <16 x i32> [ undef, %entry ]
-  phi <8 x i64> [ undef, %entry ]
-  phi <16 x float> [ undef, %entry ]
-  phi <8 x double> [ undef, %entry ]
-; CHECK-NEXT: disallowed: bad operand: {{.*}} <64 x i1>
-; CHECK-NEXT: disallowed: bad operand: {{.*}} <64 x i8>
-; CHECK-NEXT: disallowed: bad operand: {{.*}} <32 x i16>
-; CHECK-NEXT: disallowed: bad operand: {{.*}} <16 x i32>
-; CHECK-NEXT: disallowed: bad operand: {{.*}} <8 x i64>
-; CHECK-NEXT: disallowed: bad operand: {{.*}} <16 x float>
-; CHECK-NEXT: disallowed: bad operand: {{.*}} <8 x double>
-
-  ret void
-}
-
-
-; Named types. With the current implementation, named types are legal
-; until they are actually attempted to be used. Might want to fix that.
-%struct.s1 = type { half, float}
-%struct.s2 = type { i32, i32}
-
-define internal void @func2() {
-entry:
-  br label %block
-block:
-
-  phi %struct.s1 [ undef, %entry ]
-; CHECK: disallowed: bad operand: {{.*}} %struct.s1
-
-  phi %struct.s2 [ undef, %entry ]
-; CHECK-NEXT: disallowed: bad operand: {{.*}} %struct.s2
-
-  ret void
-}
-
-
-; Circularities:  here to make sure the verifier doesn't crash or assert.
-
-; This oddity is perfectly legal according to the IR and ABI verifiers.
-; Might want to fix that. (good luck initializing one of these, though.)
-%struct.snake = type { i32, %struct.tail }
-%struct.tail = type { %struct.snake, i32 }
-
-%struct.linked = type { i32, %struct.linked * }
-
-define internal void @func3() {
-entry:
-  br label %block
-block:
-
-  phi %struct.snake [ undef, %entry ]
-; CHECK: disallowed: bad operand: {{.*}} %struct.snake
-
-  phi %struct.linked [ undef, %entry ]
-; CHECK-NEXT: disallowed: bad operand: {{.*}} %struct.linked
-
-  ret void
-}
-
-
-; This stops the verifier from complaining about the lack of an entry point.
-define void @_start(i32 %arg) {
-  ret void
-}
diff --git a/test/NaCl/PNaClLLC/lit.local.cfg b/test/NaCl/PNaClLLC/lit.local.cfg
deleted file mode 100644
index c6106e4746f..00000000000
--- a/test/NaCl/PNaClLLC/lit.local.cfg
+++ /dev/null
@@ -1 +0,0 @@
-config.suffixes = ['.ll']
diff --git a/test/NaCl/PNaClLLC/test-runs-verify.ll b/test/NaCl/PNaClLLC/test-runs-verify.ll
deleted file mode 100644
index 9e7c9d32a5c..00000000000
--- a/test/NaCl/PNaClLLC/test-runs-verify.ll
+++ /dev/null
@@ -1,13 +0,0 @@
-; RUN: not pnacl-llc -mtriple=i386-unknown-nacl -filetype=asm %s -o - 2>&1 | FileCheck %s
-
-; Test that the Verifier pass is running in pnacl-llc.
-
-define i32 @f1(i32 %x) {
-  %y = add i32 %z, 1
-  %z = add i32 %x, 1
-  ret i32 %y
-; CHECK: Instruction does not dominate all uses!
-; CHECK-NEXT:  %z = add i32 %x, 1
-; CHECK-NEXT:  %y = add i32 %z, 1
-}
-
diff --git a/test/NaCl/X86/asm_memory.ll b/test/NaCl/X86/asm_memory.ll
deleted file mode 100644
index 91aad00172e..00000000000
--- a/test/NaCl/X86/asm_memory.ll
+++ /dev/null
@@ -1,15 +0,0 @@
-; RUN: pnacl-llc -mtriple=x86_64-unknown-nacl -filetype=asm %s -o - \
-; RUN:   | FileCheck %s
-
-; CHECK: func:
-define void @func(i32* %i) {
-entry:
-  %0 = load i32, i32* %i, align 4
-; Check that the inline asm expression is correctly transformed to NaCl
-; pseudo-segment memory operand syntax.
-; CHECK: movl %e{{[a-z]+}}, %e[[REG:[a-z]{2}]]
-; CHECK: movl %nacl:(%r15,%r[[REG]]), %eax
-  call void asm sideeffect "mov $0, %eax", "*m,~{dirflag},~{fpsr},~{flags}"(i32* %i)
-  ret void
-}
-
diff --git a/test/NaCl/X86/asm_mplus.ll b/test/NaCl/X86/asm_mplus.ll
deleted file mode 100644
index 2b002c70257..00000000000
--- a/test/NaCl/X86/asm_mplus.ll
+++ /dev/null
@@ -1,14 +0,0 @@
-; RUN: pnacl-llc -mtriple=x86_64-unknown-nacl -filetype=asm %s -o - \
-; RUN:   | FileCheck %s
-
-target datalayout = "e-m:e-p:32:32-i64:64-f80:128-n8:16:32:64-S128"
-target triple = "x86_64--nacl"
-
-; Test that we can handle 2 memory constraints in one inline asm
-define i32 @f(i32* %a, i32 %b) {
-entry:
-; CHECK: lock
-; CHECK: xaddl %e{{[a-z]*}}, %nacl:(%r15,%r{{[a-z]*}})
-  %r = call i32 asm sideeffect "lock; xaddl $0, $1", "=r,=*m,0,*m,~{memory},~{dirflag},~{fpsr},~{flags}"(i32* %a, i32 %b, i32* %a)
-  ret i32 %r
-}
diff --git a/test/NaCl/X86/dynamic-stack-alloc.ll b/test/NaCl/X86/dynamic-stack-alloc.ll
deleted file mode 100644
index 2c61d417cd4..00000000000
--- a/test/NaCl/X86/dynamic-stack-alloc.ll
+++ /dev/null
@@ -1,47 +0,0 @@
-; RUN: llc -mtriple=x86_64-nacl %s -o - | FileCheck %s
-; RUN: llc -mtriple=x86_64-nacl -fast-isel %s -o - | FileCheck %s
-
-; Test calling with many parameters (such that some need to go on the stack),
-; along with dynamic stack allocation and x86-64 NaCl stack/frame
-; register sandboxing.
-
-declare i32 @callee(i32 %a1, i32 %a2, i32 %a3, i32 %a4, i32 %a5, i32 %a6, i32 %a7, i32 %a8, i8* %ptr)
-
-define i32 @dyn_stack_alloc (i32 %a1, i32 %a2, i32 %a3, i32 %a4, i32 %a5, i32 %a6, i32 %a7, i32 %a8, i32 %amt) {
-  %p = alloca i8, i32 %amt, align 1
-  %x = call i32 @callee(i32 %a1, i32 %a2, i32 %a3, i32 %a4, i32 %a5, i32 %a6, i32 %a7, i32 %a8, i8* %p)
-  ret i32 %x
-}
-
-; CHECK-LABEL: dyn_stack_alloc
-; Set up frame pointer and save two registers.
-; CHECK: movl %ebp, %eax
-; CHECK-NEXT: pushq %rax
-; CHECK: movq %rsp, %rbp
-; CHECK-NEXT: .L
-; CHECK-NEXT: .cfi_def_cfa_register %rbp
-; CHECK-NEXT: pushq
-; CHECK-NEXT: pushq
-; CHECK-NOT: pushq
-
-; The dynamic alloc (aligned):
-; CHECK: movl 32(%rbp), [[AMT:%.*]]
-; CHECK: movl %esp, [[TMP:%.*]]
-; CHECK-NEXT: addl $15, [[AMT]]
-; CHECK-NEXT: andl $-16, [[AMT]]
-; CHECK-NEXT: subl [[AMT]], [[TMP]]
-; Set the SP based on the final value of TMP.
-; CHECK-NEXT: naclrestsp_noflags [[TMP]], %r15
-
-; Make more room for call arguments:
-; CHECK: naclsspq $32, %r15
-; CHECK: callq callee
-
-; Restore stack pointer from frame pointer - 16. It is -16 because
-; the code still needs to pop the two pushed regs, before restoring
-; the frame pointer from the stack.
-; CHECK: naclspadj $-16, %r15
-; CHECK-NEXT: popq
-; CHECK-NEXT: popq
-; CHECK-NEXT: naclrestbp (%rsp), %r15
-; CHECK-NEXT: naclaspq $8, %r15
diff --git a/test/NaCl/X86/dynamic-stack-alloc2.ll b/test/NaCl/X86/dynamic-stack-alloc2.ll
deleted file mode 100644
index 864ee3cba15..00000000000
--- a/test/NaCl/X86/dynamic-stack-alloc2.ll
+++ /dev/null
@@ -1,23 +0,0 @@
-; RUN: llc -mtriple=x86_64-nacl %s -o - | FileCheck %s
-
-target datalayout = "e-p:32:32-i64:64-n32"
-target triple = "le32-unknown-nacl"
-
-; CHECK-LABEL: @foo
-; CHECK: .bundle_lock
-; CHECK: leal -16({{.*}}), %esp
-; CHECK: addq %r15, %rsp
-; CHECK: .bundle_unlock
-define hidden void @foo() {
-entry:
-  br label %bb1
-; The alloca must be in a non-entry block so it gets lowered in the dag as
-; a dynamic_stackalloc node
-bb1:
-  %0 = alloca i8, i32 16, align 16
-  %1 = load i8, i8* %0, align 1
-  %call5 = call i32 @bar(i8 %1)
-  unreachable
-}
-
-declare hidden i32 @bar(i8)
\ No newline at end of file
diff --git a/test/NaCl/X86/intrinsics-bitmanip.ll b/test/NaCl/X86/intrinsics-bitmanip.ll
deleted file mode 100644
index ff20ec9b38b..00000000000
--- a/test/NaCl/X86/intrinsics-bitmanip.ll
+++ /dev/null
@@ -1,74 +0,0 @@
-; RUN: pnacl-llc -mtriple=i686-unknown-nacl -O0 -filetype=asm %s -o - | \
-; RUN:   FileCheck %s --check-prefix=NACL32
-; RUN: pnacl-llc -mtriple=i686-unknown-nacl -filetype=asm %s -o - | \
-; RUN:   FileCheck %s --check-prefix=NACL32
-; RUN: pnacl-llc -mtriple=x86_64-unknown-nacl -O0 -filetype=asm %s -o - | \
-; RUN:   FileCheck %s --check-prefix=NACL64
-; RUN: pnacl-llc -mtriple=x86_64-unknown-nacl -filetype=asm %s -o - | \
-; RUN:   FileCheck %s --check-prefix=NACL64
-
-; Test that various bit manipulation intrinsics are supported by the
-; NaCl X86-32 and X86-64 backends.
-
-declare i16 @llvm.bswap.i16(i16)
-declare i32 @llvm.bswap.i32(i32)
-declare i64 @llvm.bswap.i64(i64)
-
-; NACL32: test_bswap_16
-; NACL32: rolw $8, %{{.*}}
-; NACL64: test_bswap_16
-; NACL64: rolw $8, %{{.*}}
-define i16 @test_bswap_16(i16 %a) {
-  %b = call i16 @llvm.bswap.i16(i16 %a)
-  ret i16 %b
-}
-
-; NACL32: test_bswap_const_16
-; NACL32: movw $-12885, %ax # imm = 0xFFFFFFFFFFFFCDAB
-; NACL64: test_bswap_const_16
-; NACL64: movw $-12885, %ax # imm = 0xFFFFFFFFFFFFCDAB
-define i16 @test_bswap_const_16() {
-  ; 0xabcd
-  %a = call i16 @llvm.bswap.i16(i16 43981)
-  ret i16 %a
-}
-
-; NACL32: test_bswap_32
-; NACL32: bswapl %eax
-; NACL64: test_bswap_32
-; NACL64: bswapl %edi
-define i32 @test_bswap_32(i32 %a) {
-  %b = call i32 @llvm.bswap.i32(i32 %a)
-  ret i32 %b
-}
-
-; NACL32: test_bswap_const_32
-; NACL32: movl $32492971, %eax # imm = 0x1EFCDAB
-; NACL64: test_bswap_const_32
-; NACL64: movl $32492971, %eax # imm = 0x1EFCDAB
-define i32 @test_bswap_const_32() {
-  ; 0xabcdef01
-  %a = call i32 @llvm.bswap.i32(i32 2882400001)
-  ret i32 %a
-}
-
-; NACL32: test_bswap_64
-; NACL32: bswapl %e{{.*}}
-; NACL32: bswapl %e{{.*}}
-; NACL64: test_bswap_64
-; NACL64: bswapq %rdi
-define i64 @test_bswap_64(i64 %a) {
-  %b = call i64 @llvm.bswap.i64(i64 %a)
-  ret i64 %b
-}
-
-; NACL32: test_bswap_const_64
-; NACL32: movl $32492971, %eax # imm = 0x1EFCDAB
-; NACL32: movl $-1989720797, %edx # imm = 0xFFFFFFFF89674523
-; NACL64: test_bswap_const_64
-; NACL64: movabsq	$-8545785751253561941, %rax # imm = 0x8967452301EFCDAB
-define i64 @test_bswap_const_64(i64 %a) {
-  ; 0xabcdef01 23456789
-  %b = call i64 @llvm.bswap.i64(i64 12379813738877118345)
-  ret i64 %b
-}
diff --git a/test/NaCl/X86/lit.local.cfg b/test/NaCl/X86/lit.local.cfg
deleted file mode 100644
index 56bf0085957..00000000000
--- a/test/NaCl/X86/lit.local.cfg
+++ /dev/null
@@ -1,6 +0,0 @@
-config.suffixes = ['.ll', '.s']
-
-targets = set(config.root.targets_to_build.split())
-if not 'X86' in targets:
-    config.unsupported = True
-
diff --git a/test/NaCl/X86/llc-make-symbols-hidden.ll b/test/NaCl/X86/llc-make-symbols-hidden.ll
deleted file mode 100644
index f18bca32494..00000000000
--- a/test/NaCl/X86/llc-make-symbols-hidden.ll
+++ /dev/null
@@ -1,27 +0,0 @@
-; RUN: pnacl-llc -mtriple=i686-unknown-nacl -filetype=asm %s -o - \
-; RUN:   -relocation-model=pic | FileCheck %s
-
-; This should have "hidden" automatically added to it.
-declare void @callee()
-
-define void @caller() {
-  tail call void @callee()
-  ret void
-}
-; CHECK-LABEL: caller
-; As we automatically make callee hidden, we should see neither an
-; access to GOT nor jmp instruction to PLT.
-; CHECK-NOT: _GLOBAL_OFFSET_TABLE_
-; CHECK-NOT: jmp callee@PLT
-; CHECK: jmp callee
-
-; This should have "hidden" automatically added to it.
-@tls_var = external thread_local global i32
-
-define i32* @get_tls_addr() {
-  ret i32* @tls_var
-}
-; CHECK-LABEL: get_tls_addr
-; There must be no general dynamic TLS.
-; CHECK-NOT: @TLSGD
-; CHECK: tls_var@TLSLDM
diff --git a/test/NaCl/X86/llc-malign-double.ll b/test/NaCl/X86/llc-malign-double.ll
deleted file mode 100644
index 7bf497d8622..00000000000
--- a/test/NaCl/X86/llc-malign-double.ll
+++ /dev/null
@@ -1,54 +0,0 @@
-; RUN: pnacl-llc -mtriple=i386-pc-linux -filetype=asm %s -o - | FileCheck %s
-; RUN: pnacl-llc -mtriple=i386-pc-linux -malign-double -filetype=asm %s -o -| FileCheck %s --check-prefix=MALIGN
-; RUN: pnacl-llc -mtriple=i386-unknown-nacl -filetype=asm %s -o -| FileCheck %s --check-prefix=MALIGN
-
-; Test that the -malign-double flag causes i64 and f64 types to have alignment
-; 8 instead of 4.
-
-%structi64 = type { i32, i64 }
-
-define i32 @check_i64_size() {
-; CHECK-LABEL: check_i64_size:
-  %starti = ptrtoint %structi64* null to i32
-  %endp = getelementptr %structi64, %structi64* null, i32 1
-  %endi = ptrtoint %structi64* %endp to i32
-  %diff = sub i32 %endi, %starti
-; CHECK: movl $12, %eax
-; MALIGN: movl $16, %eax
-  ret i32 %diff
-}
-
-define i32 @check_i64_offset() {
-; CHECK-LABEL: check_i64_offset:
-  %starti = ptrtoint %structi64* null to i32
-  %endp = getelementptr %structi64, %structi64* null, i32 0, i32 1
-  %endi = ptrtoint i64* %endp to i32
-  %diff = sub i32 %endi, %starti
-; CHECK: movl $4, %eax
-; MALIGN: movl $8, %eax
-  ret i32 %diff
-}
-
-%structf64 = type { i32, double }
-
-define i32 @check_f64_size() {
-; CHECK-LABEL: check_f64_size:
-  %starti = ptrtoint %structf64* null to i32
-  %endp = getelementptr %structf64, %structf64* null, i32 1
-  %endi = ptrtoint %structf64* %endp to i32
-  %diff = sub i32 %endi, %starti
-; CHECK: movl $12, %eax
-; MALIGN: movl $16, %eax
-  ret i32 %diff
-}
-
-define i32 @check_f64_offset() {
-; CHECK-LABEL: check_f64_offset:
-  %starti = ptrtoint %structf64* null to i32
-  %endp = getelementptr %structf64, %structf64* null, i32 0, i32 1
-  %endi = ptrtoint double* %endp to i32
-  %diff = sub i32 %endi, %starti
-; CHECK: movl $4, %eax
-; MALIGN: movl $8, %eax
-  ret i32 %diff
-}
diff --git a/test/NaCl/X86/nacl-calls.ll b/test/NaCl/X86/nacl-calls.ll
deleted file mode 100644
index 36ccb39c216..00000000000
--- a/test/NaCl/X86/nacl-calls.ll
+++ /dev/null
@@ -1,149 +0,0 @@
-; RUN: pnacl-llc -mtriple=i686-unknown-nacl -filetype=asm %s -o - \
-; RUN:   | FileCheck %s --check-prefix=NACL32
-; RUN: pnacl-llc -mtriple=i686-unknown-nacl -filetype=asm -O0 %s -o - \
-; RUN:   | FileCheck %s --check-prefix=NACL32O0
-; RUN: pnacl-llc -mtriple=x86_64-unknown-nacl -filetype=asm %s -o - \
-; RUN:   | FileCheck %s --check-prefix=NACL64
-; RUN: pnacl-llc -mtriple=x86_64-unknown-nacl -filetype=asm -O0 %s -o - \
-; RUN:   | FileCheck %s --check-prefix=NACL64
-
-;;;;
-; Call to a NaCl trampoline (specific addresses).
-define i32 @call_address() {
-entry:
-  call void inttoptr (i32 66496 to void ()*)()
-  ret i32 0
-}
-; NACL32-LABEL: call_address
-; NACL32: calll 66496
-; NACL32O0-LABEL: call_address
-; NACL32O0: movl $66496, [[REG:%[a-z0-9]+]]
-; NACL32O0: naclcall {{.*}}[[REG]]
-; NACL64-LABEL: call_address
-; NACL64: movl $66496, %{{.*}}
-; NACL64: naclcall %{{.*}},%r15
-
-define fastcc i32 @tail_call_address(i32 %arg) {
-entry:
-  %call1 = tail call fastcc i32 inttoptr (i32 66496 to i32 (i32)*)(i32 %arg)
-  ret i32 %call1
-}
-; NACL32-LABEL: tail_call_address
-; NACL32: movl $66496, [[REG:%[a-z0-9]+]]
-; NACL32: nacljmp {{.*}}[[REG]]
-; NACL32O0-LABEL: tail_call_address
-; NACL32O0: movl $66496, [[REG:%[a-z0-9]+]]
-; NACL32O0: nacljmp {{.*}}[[REG]]
-; NACL64-LABEL: tail_call_address
-; NACL64: movl $66496, %{{.*}}
-; NACL64: nacljmp %{{.*}}, %r15
-
-
-;;;;;
-; Call to another function (external/internal), directly.
-
-declare void @other_function()
-
-define internal void @call_other_function() {
-  call void @other_function()
-  ret void
-}
-
-define void @call_other_function2() {
-  call void @call_other_function()
-  ret void
-}
-; NACL32-LABEL: call_other_function
-; NACL32: calll other_function
-; NACL32-LABEL: call_other_function2
-; NACL32: calll call_other_function
-; NACL32O0-LABEL: call_other_function
-; NACL32O0: calll other_function
-; NACL64-LABEL: call_other_function
-; NACL64: callq other_function
-; NACL64-LABEL: call_other_function2
-; NACL64: callq call_other_function
-
-
-declare fastcc i32 @other_function_fast()
-
-define internal fastcc i32 @tail_call_other_function() {
-  %i = tail call fastcc i32 @other_function_fast()
-  ret i32 %i
-}
-
-define fastcc i32 @tail_call_other_function2() {
-  %i = tail call fastcc i32 @tail_call_other_function()
-  ret i32 %i
-}
-; NACL32-LABEL: tail_call_other_function
-; NACL32: jmp other_function_fast
-; NACL32-LABEL: tail_call_other_function2
-; NACL32: jmp tail_call_other_function
-; NACL32O0-LABEL: tail_call_other_function
-; NACL32O0: jmp other_function_fast
-; NACL32O0-LABEL: tail_call_other_function2
-; NACL32O0: jmp tail_call_other_function
-; NACL64-LABEL: tail_call_other_function
-; NACL64: jmp other_function_fast
-; NACL64-LABEL: tail_call_other_function2
-; NACL64: jmp tail_call_other_function
-
-;;;;;
-; Indirect call, but not a specific address.
-
-@fp = external global i32 (i32)*
-
-; With a load.
-define i32 @call_indirect() {
-  %1 = load i32 (i32)*, i32 (i32)** @fp, align 4
-  %call1 = call i32 %1(i32 10)
-  ret i32 %call1
-}
-; NACL32-LABEL: call_indirect
-; NACL32: movl fp, [[REG:%[a-z0-9]+]]
-; NACL32: naclcall {{.*}}[[REG]]
-; NACL32O0-LABEL: call_indirect
-; NACL32O0: movl fp, [[REG:%[a-z0-9]+]]
-; NACL32O0: naclcall {{.*}}[[REG]]
-; NACL64-LABEL: call_indirect
-; NACL64: movl fp({{.*}}), %{{.*}}
-; NACL64: naclcall %{{.*}},%r15
-
-define fastcc i32 @tail_call_indirect() {
-  %1 = load i32 (i32)*, i32 (i32)** @fp, align 4
-  %call1 = tail call fastcc i32 %1(i32 10)
-  ret i32 %call1
-}
-; NACL32-LABEL: tail_call_indirect
-; NACL32: movl fp, [[REG:%[a-z0-9]+]]
-; NACL32: nacljmp {{.*}}[[REG]]
-; NACL32O0-LABEL: tail_call_indirect
-; NACL32O0: movl fp, [[REG:%[a-z0-9]+]]
-; NACL32O0: nacljmp {{.*}}[[REG]]
-; NACL64-LABEL: tail_call_indirect
-; NACL64: movl fp({{.*}}), %{{.*}}
-; NACL64: nacljmp %{{.*}}, %r15
-
-; "Without" a load (may load from stack on x86-32 still).
-define i32 @call_indirect_arg(i32 ()* %argfp) {
-  %call1 = call i32 %argfp()
-  ret i32 %call1
-}
-; NACL32-LABEL: call_indirect_arg
-; NACL32: naclcall {{%[a-z0-9]+}}
-; NACL32O0-LABEL: call_indirect_arg
-; NACL32O0: naclcall {{%[a-z0-9]+}}
-; NACL64-LABEL: call_indirect_arg
-; NACL64: naclcall {{%[a-z0-9]+}},%r15
-
-define fastcc i32 @tail_call_indirect_arg(i32 ()* %argfp) {
-  %call1 = tail call fastcc i32 %argfp()
-  ret i32 %call1
-}
-; NACL32-LABEL: tail_call_indirect_arg
-; NACL32: nacljmp {{%[a-z0-9]+}}
-; NACL32O0-LABEL: tail_call_indirect_arg
-; NACL32O0: nacljmp {{%[a-z0-9]+}}
-; NACL64-LABEL: tail_call_indirect_arg
-; NACL64: nacljmp {{%[a-z0-9]+}}, %r15
diff --git a/test/NaCl/X86/nacl-read-tp-intrinsic.ll b/test/NaCl/X86/nacl-read-tp-intrinsic.ll
deleted file mode 100644
index 501bf8b9bcb..00000000000
--- a/test/NaCl/X86/nacl-read-tp-intrinsic.ll
+++ /dev/null
@@ -1,43 +0,0 @@
-; RUN: pnacl-llc -mtriple=i386-unknown-nacl -filetype=asm %s -o - \
-; RUN:   | FileCheck -check-prefix=X32 %s
-
-; RUN: pnacl-llc -mtriple=i386-unknown-nacl -filetype=asm -mtls-use-call %s -o - \
-; RUN:   | FileCheck -check-prefix=USE_CALL %s
-
-; RUN: pnacl-llc -mtriple=x86_64-unknown-nacl -filetype=asm %s -o - \
-; RUN:   | FileCheck -check-prefix=USE_CALL %s
-
-; "-mtls-use-call" should not make any difference on x86-64.
-; RUN: pnacl-llc -mtriple=x86_64-unknown-nacl -filetype=asm -mtls-use-call %s -o - \
-; RUN:   | FileCheck -check-prefix=USE_CALL %s
-
-
-declare i8* @llvm.nacl.read.tp()
-
-define i8* @get_thread_pointer() {
-  %tp = call i8* @llvm.nacl.read.tp()
-  ret i8* %tp
-}
-
-; X32: get_thread_pointer:
-; X32: movl %gs:0, %eax
-
-; USE_CALL: get_thread_pointer:
-; USE_CALL: call{{[lq]?}} __nacl_read_tp
-
-
-; Make sure that we do not generate:
-;   movl $1000, %eax
-;   addl %gs:0, %eax
-; The x86-32 NaCl validator only accepts %gs with "mov", not with
-; "add".  Note that we had to use a large immediate to trigger the bug
-; and generate the code above.
-define i8* @get_thread_pointer_add() {
-  %tp = call i8* @llvm.nacl.read.tp()
-  %result = getelementptr i8, i8* %tp, i32 1000
-  ret i8* %result
-}
-
-; X32: get_thread_pointer_add:
-; X32: movl %gs:0, %eax
-; X32: addl $1000, %eax
diff --git a/test/NaCl/X86/nacl-setlongjmp-intrinsics.ll b/test/NaCl/X86/nacl-setlongjmp-intrinsics.ll
deleted file mode 100644
index 3e43979a31d..00000000000
--- a/test/NaCl/X86/nacl-setlongjmp-intrinsics.ll
+++ /dev/null
@@ -1,18 +0,0 @@
-; RUN: pnacl-llc -mtriple=i386-unknown-nacl -filetype=asm %s -o - \
-; RUN:  | FileCheck %s --check-prefix=X86
-; Test that @llvm.nacl.{set|long}jmp intrinsics calls get translated to library
-; calls as expected.
-
-declare i32 @llvm.nacl.setjmp(i8*)
-declare void @llvm.nacl.longjmp(i8*, i32)
-
-define void @foo(i8* %arg) {
-  %num = call i32 @llvm.nacl.setjmp(i8* %arg)
-; X86: calll setjmp
-
-  call void @llvm.nacl.longjmp(i8* %arg, i32 %num)
-; X86: calll longjmp
-
-  ret void
-}
-
diff --git a/test/NaCl/X86/nacl64-addrmodes.ll b/test/NaCl/X86/nacl64-addrmodes.ll
deleted file mode 100644
index 51c0ce28b32..00000000000
--- a/test/NaCl/X86/nacl64-addrmodes.ll
+++ /dev/null
@@ -1,131 +0,0 @@
-; RUN: pnacl-llc -mtriple=x86_64-unknown-nacl -filetype=asm %s -O0 -o - \
-; RUN:   | FileCheck %s
-
-; RUN: pnacl-llc -mtriple=x86_64-unknown-nacl -filetype=asm %s -O2 -o - \
-; RUN:   | FileCheck %s
-
-; Check that we don't try to fold a negative displacement into a memory
-; reference
-define i16 @negativedisp(i32 %b) {
-; CHECK: negativedisp
-  %a = alloca [1 x i16], align 2
-  %add = add nsw i32 1073741824, %b
-  %arrayidx = getelementptr inbounds [1 x i16], [1 x i16]* %a, i32 0, i32 %add
-; CHECK-NOT: nacl:-2147483648(
-  %c = load i16, i16* %arrayidx, align 2
-  ret i16 %c
-}
-
-@main.m2 = internal constant [1 x [1 x i32]] [[1 x i32] [i32 -60417067]], align 4
-define i1 @largeconst() nounwind {
-; CHECK: largeconst
-entry:
-  %retval = alloca i32, align 4
-  %i = alloca i32, align 4
-  %j = alloca i32, align 4
-  %madat = alloca i32*, align 4
-  store i32 0, i32* %retval
-  store i32 -270770481, i32* %i, align 4
-  store i32 -1912319477, i32* %j, align 4
-  %0 = load i32, i32* %j, align 4
-  %mul = mul nsw i32 %0, 233468377
-  %add = add nsw i32 %mul, 689019309
-  %1 = load i32, i32* %i, align 4
-  %mul1 = mul nsw i32 %1, 947877507
-  %add2 = add nsw i32 %mul1, 1574375955
-  %arrayidx = getelementptr inbounds [1 x i32], [1 x i32]* getelementptr inbounds ([1 x [1 x i32]], [1 x [1 x i32]]* @main.m2, i32 0, i32 0), i32 %add2
-  %2 = bitcast [1 x i32]* %arrayidx to i32*
-  %arrayidx3 = getelementptr inbounds i32, i32* %2, i32 %add
-  store i32* %arrayidx3, i32** %madat, align 4
-; Ensure the large constant doesn't get folded into the load
-; CHECK: nacl:(%r15
-  %3 = load i32*, i32** %madat, align 4
-  %4 = load i32, i32* %3, align 4
-  %conv = zext i32 %4 to i64
-  %5 = load i32, i32* %j, align 4
-  %mul4 = mul nsw i32 %5, 233468377
-  %add5 = add nsw i32 %mul4, 689019309
-  %6 = load i32, i32* %i, align 4
-  %mul6 = mul nsw i32 %6, 947877507
-  %add7 = add nsw i32 %mul6, 1574375955
-  %arrayidx8 = getelementptr inbounds [1 x i32], [1 x i32]* getelementptr inbounds ([1 x [1 x i32]], [1 x [1 x i32]]* @main.m2, i32 0, i32 0), i32 %add7
-  %7 = bitcast [1 x i32]* %arrayidx8 to i32*
-  %arrayidx9 = getelementptr inbounds i32, i32* %7, i32 %add5
-; Ensure the large constant doesn't get folded into the load
-; CHECK: nacl:(%r15
-  %8 = load i32, i32* %arrayidx9, align 4
-  %conv10 = zext i32 %8 to i64
-  %mul11 = mul nsw i64 3795428823, %conv10
-  %9 = load i32, i32* %j, align 4
-  %mul12 = mul nsw i32 %9, 233468377
-  %add13 = add nsw i32 %mul12, 689019309
-  %conv14 = sext i32 %add13 to i64
-  %rem = srem i64 %conv14, 4294967295
-  %xor = xor i64 2597389499, %rem
-  %mul15 = mul nsw i64 %xor, 3795428823
-  %sub = sub nsw i64 %mul11, %mul15
-  %add16 = add nsw i64 %sub, 3829710203
-  %mul17 = mul nsw i64 %add16, 2824337475
-  %add18 = add nsw i64 %mul17, 2376483023
-  %cmp = icmp eq i64 %conv, %add18
-  ret i1 %cmp
-}
-
-
-@main.array = private unnamed_addr constant [1 x i64] [i64 1438933078946427748], align 8
-
-define i1 @largeconst_frameindex() nounwind {
-; CHECK: largeconst_frameindex
-entry:
-  %retval = alloca i32, align 4
-  %r_Ng = alloca i64, align 8
-  %i = alloca i32, align 4
-  %adat = alloca i64*, align 4
-  %array = alloca [1 x i64], align 8
-  store i32 0, i32* %retval
-  store i32 -270770481, i32* %i, align 4
-  %0 = bitcast [1 x i64]* %array to i8*
-  call void @llvm.memcpy.p0i8.p0i8.i32(i8* %0, i8* bitcast ([1 x i64]* @main.array to i8*), i32 8, i32 8, i1 false)
-  store i32 -270770481, i32* %i, align 4
-  %1 = load i32, i32* %i, align 4
-  %mul = mul i32 %1, 947877507
-  %add = add i32 %mul, 1574375955
-  %2 = bitcast [1 x i64]* %array to i64*
-  %arrayidx = getelementptr inbounds i64, i64* %2, i32 %add
-; Ensure the large constant didn't get folded into the load
-; CHECK: nacl:(%r15
-  %3 = load i64, i64* %arrayidx, align 8
-  %add1 = add i64 %3, -5707596139582126917
-  %4 = load i32, i32* %i, align 4
-  %mul2 = mul i32 %4, 947877507
-  %add3 = add i32 %mul2, 1574375955
-  %5 = bitcast [1 x i64]* %array to i64*
-  %arrayidx4 = getelementptr inbounds i64, i64* %5, i32 %add3
-  store i64 %add1, i64* %arrayidx4, align 8
-  %6 = load i32, i32* %i, align 4
-  %mul5 = mul nsw i32 %6, 947877507
-  %add6 = add nsw i32 %mul5, 1574375955
-  %arrayidx7 = getelementptr inbounds [1 x i64], [1 x i64]* %array, i32 0, i32 %add6
-; CHECK: nacl:(%r15
-  %7 = load i64, i64* %arrayidx7, align 8
-  %add8 = add i64 %7, -5707596139582126917
-  %8 = load i32, i32* %i, align 4
-  %mul9 = mul nsw i32 %8, 947877507
-  %add10 = add nsw i32 %mul9, 1574375955
-  %arrayidx11 = getelementptr inbounds [1 x i64], [1 x i64]* %array, i32 0, i32 %add10
-  store i64 %add8, i64* %arrayidx11, align 8
-  %9 = load i32, i32* %i, align 4
-  %mul12 = mul nsw i32 %9, 947877507
-  %add13 = add nsw i32 %mul12, 1574375955
-  %10 = bitcast [1 x i64]* %array to i64*
-  %arrayidx14 = getelementptr inbounds i64, i64* %10, i32 %add13
-  store i64* %arrayidx14, i64** %adat, align 4
-  %11 = load i64*, i64** %adat, align 4
-  %12 = load i64, i64* %11, align 8
-  %mul15 = mul i64 %12, -1731288434922394955
-  %add16 = add i64 %mul15, -7745351015538694962
-  store i64 %add16, i64* %r_Ng, align 8
-  ret i1 0
-}
-
-declare void @llvm.memcpy.p0i8.p0i8.i32(i8* nocapture, i8* nocapture, i32, i32, i1) nounwind
diff --git a/test/NaCl/X86/negative-addend.ll b/test/NaCl/X86/negative-addend.ll
deleted file mode 100644
index 47ecd108648..00000000000
--- a/test/NaCl/X86/negative-addend.ll
+++ /dev/null
@@ -1,27 +0,0 @@
-; RUN: pnacl-llc -mtriple=i386-unknown-nacl -filetype=obj %s -o - \
-; RUN:   | llvm-objdump -r - | FileCheck %s -check-prefix=X8632
-; RUN: pnacl-llc -mtriple=x86_64-unknown-nacl -filetype=obj %s -o - \
-; RUN:   | llvm-objdump -r - | FileCheck %s -check-prefix=X8664
-
-; Check that "add" works for negative values when used as a
-; ConstantExpr in a global variable initializer.
-; See: https://code.google.com/p/nativeclient/issues/detail?id=3548
-
-
-; @spacer and @var end up in the BSS section.
-; @spacer is at offset 0.  @var is at offset 4096 = 0x1000.
-
-@spacer = internal global [4096 x i8] zeroinitializer
-@var = internal global i32 zeroinitializer
-
-@negative_offset = internal global i32 add
-    (i32 ptrtoint (i32* @var to i32), i32 -8)
-
-; Note that the addend 4294971384 below equals 0x100000ff8, where
-; 0xff8 comes from subtracting 8 from the offset of @var.
-
-; X8632: RELOCATION RECORDS
-; X8632-NEXT: 0 R_386_32 .bss
-
-; X8664: RELOCATION RECORDS
-; X8664-NEXT: 0 R_X86_64_32 .bss+4294971384
diff --git a/test/NaCl/X86/no-global-in-disp-x86-64.ll b/test/NaCl/X86/no-global-in-disp-x86-64.ll
deleted file mode 100644
index 6578f2ecc47..00000000000
--- a/test/NaCl/X86/no-global-in-disp-x86-64.ll
+++ /dev/null
@@ -1,50 +0,0 @@
-; RUN: pnacl-llc -O2 -mtriple=x86_64-none-nacl < %s | \
-; RUN:     FileCheck %s --check-prefix=NACLON
-; RUN: pnacl-llc -O2 -mtriple=x86_64-linux     < %s | \
-; RUN:     FileCheck %s --check-prefix=NACLOFF
-
-; This test is derived from the following C code:
-;
-; int myglobal[100];
-; void test(int arg)
-; {
-;   myglobal[arg] = arg;
-;   myglobal[arg+1] = arg;
-; }
-; int main(int argc, char **argv)
-; {
-;   test(argc);
-; }
-;
-; The goal is NOT to produce an instruction with "myglobal" as the
-; displacement value in any addressing mode, e.g. this (bad) instruction:
-;
-; movl %eax, %nacl:myglobal(%r15,%rax,4)
-;
-; The NACLOFF test is a canary that tries to ensure that the NACLON test is
-; testing the right thing.  If the NACLOFF test starts failing, it's likely
-; that the LLVM -O2 optimizations are no longer generating the problematic
-; pattern that NACLON tests for.  In that case, the test should be modified.
-
-
-@myglobal = global [100 x i32] zeroinitializer, align 4
-
-define void @test(i32 %arg) #0 {
-entry:
-; NACLON: test:
-; NACLON-NOT: mov{{.*}}nacl:myglobal(
-; NACLOFF: test:
-; NACLOFF: mov{{.*}}myglobal(
-  %arg.addr = alloca i32, align 4
-  store i32 %arg, i32* %arg.addr, align 4
-  %0 = load i32, i32* %arg.addr, align 4
-  %1 = load i32, i32* %arg.addr, align 4
-  %arrayidx = getelementptr inbounds [100 x i32], [100 x i32]* @myglobal, i32 0, i32 %1
-  store i32 %0, i32* %arrayidx, align 4
-  %2 = load i32, i32* %arg.addr, align 4
-  %3 = load i32, i32* %arg.addr, align 4
-  %add = add nsw i32 %3, 1
-  %arrayidx1 = getelementptr inbounds [100 x i32], [100 x i32]* @myglobal, i32 0, i32 %add
-  store i32 %2, i32* %arrayidx1, align 4
-  ret void
-}
diff --git a/test/NaCl/X86/pic-nacl-x8632.ll b/test/NaCl/X86/pic-nacl-x8632.ll
deleted file mode 100644
index eded4026888..00000000000
--- a/test/NaCl/X86/pic-nacl-x8632.ll
+++ /dev/null
@@ -1,21 +0,0 @@
-; RUN: llc < %s -mcpu=generic -mtriple=i686-unknown-nacl -relocation-model=pic -asm-verbose=false -post-RA-scheduler=false | FileCheck %s
-
-@ptr = external global i32*
-@dst = external global i32
-@src = external global i32
-
-define void @test0() nounwind {
-entry:
-    store i32* @dst, i32** @ptr
-    %tmp.s = load i32, i32* @src
-    store i32 %tmp.s, i32* @dst
-    ret void
-; CHECK-LABEL:    test0:
-; CHECK:	calll	.L0$pb
-; CHECK-NEXT: .L0$pb:
-; CHECK-NEXT:	popl
-; CHECK:	addl	$_GLOBAL_OFFSET_TABLE_+(.-.L0$pb),
-; CHECK:	movl	dst@GOT(%eax),
-; CHECK:	movl	ptr@GOT(%eax),
-; CHECK:	movl	src@GOT(%eax),
-}
diff --git a/test/NaCl/X86/pnacl-avoids-r11-x86-64.c b/test/NaCl/X86/pnacl-avoids-r11-x86-64.c
deleted file mode 100644
index 65b921486fd..00000000000
--- a/test/NaCl/X86/pnacl-avoids-r11-x86-64.c
+++ /dev/null
@@ -1,79 +0,0 @@
-/*
-  Object file built using:
-  pnacl-clang -S -O2 -emit-llvm -o pnacl-avoids-r11-x86-64.ll \
-      pnacl-avoids-r11-x86-64.c
-  Then the comments below should be pasted into the .ll file,
-  replacing "RUNxxx" with "RUN".
-
-; The NACLON test verifies that %r11 and %r11d are not used except as
-; part of the return sequence.
-;
-; RUNxxx: pnacl-llc -O2 -mtriple=x86_64-none-nacl < %s | \
-; RUNxxx:     FileCheck %s --check-prefix=NACLON
-;
-; The NACLOFF test verifies that %r11 would normally be used if PNaCl
-; weren't reserving r11 for its own uses, to be sure NACLON is a
-; valid test.
-;
-; RUNxxx: pnacl-llc -O2 -mtriple=x86_64-linux < %s | \
-; RUNxxx:     FileCheck %s --check-prefix=NACLOFF
-;
-; NACLON: RegisterPressure:
-; NACLON-NOT: %r11
-; NACLON: popq %r11
-; NACLON: nacljmp %r11, %r15
-;
-; NACLOFF: RegisterPressure:
-; NACLOFF: %r11
-; NACLOFF: ret
-
-*/
-
-// Function RegisterPressure() tries to induce maximal integer
-// register pressure in a ~16 register machine, for both scratch and
-// preserved registers.  Repeated calls to Use() are designed to
-// use all the preserved registers.  The calculations on the local
-// variables between function calls are designed to use all the
-// scratch registers.
-
-void RegisterPressure(void)
-{
-  extern void Use(int, int, int, int, int, int, int, int,
-                  int, int, int, int, int, int, int, int);
-  extern int GetValue(void);
-  extern volatile int v1a, v1b, v2a, v2b, v3a, v3b, v4a, v4b;
-
-  int i00 = GetValue();
-  int i01 = GetValue();
-  int i02 = GetValue();
-  int i03 = GetValue();
-  int i04 = GetValue();
-  int i05 = GetValue();
-  int i06 = GetValue();
-  int i07 = GetValue();
-  int i08 = GetValue();
-  int i09 = GetValue();
-  int i10 = GetValue();
-  int i11 = GetValue();
-  int i12 = GetValue();
-  int i13 = GetValue();
-  int i14 = GetValue();
-  int i15 = GetValue();
-
-  Use(i00, i01, i02, i03, i04, i05, i06, i07,
-      i08, i09, i10, i11, i12, i13, i14, i15);
-  Use(i00, i01, i02, i03, i04, i05, i06, i07,
-      i08, i09, i10, i11, i12, i13, i14, i15);
-  v1a = i00 + i01 + i02 + i03 + i04 + i05 + i06 + i07;
-  v1b = i08 + i09 + i10 + i11 + i12 + i13 + i14 + i15;
-  v2a = i00 + i01 + i02 + i03 + i08 + i09 + i10 + i11;
-  v2b = i04 + i05 + i06 + i07 + i12 + i13 + i14 + i15;
-  v3a = i00 + i01 + i04 + i05 + i08 + i09 + i12 + i13;
-  v3b = i02 + i03 + i06 + i07 + i10 + i11 + i14 + i15;
-  v4a = i00 + i02 + i04 + i06 + i08 + i10 + i12 + i14;
-  v4b = i01 + i03 + i05 + i07 + i09 + i11 + i13 + i15;
-  Use(i00, i01, i02, i03, i04, i05, i06, i07,
-      i08, i09, i10, i11, i12, i13, i14, i15);
-  Use(i00, i01, i02, i03, i04, i05, i06, i07,
-      i08, i09, i10, i11, i12, i13, i14, i15);
-}
diff --git a/test/NaCl/X86/pnacl-avoids-r11-x86-64.ll b/test/NaCl/X86/pnacl-avoids-r11-x86-64.ll
deleted file mode 100644
index f753b7a0c9a..00000000000
--- a/test/NaCl/X86/pnacl-avoids-r11-x86-64.ll
+++ /dev/null
@@ -1,127 +0,0 @@
-; The NACLON test verifies that %r11 and %r11d are not used except as
-; part of the return sequence.
-;
-; RUN: pnacl-llc -O2 -mtriple=x86_64-none-nacl < %s | \
-; RUN:     FileCheck %s --check-prefix=NACLON
-;
-; The NACLOFF test verifies that %r11 would normally be used if PNaCl
-; weren't reserving r11 for its own uses, to be sure NACLON is a
-; valid test.
-;
-; RUN: pnacl-llc -O2 -mtriple=x86_64-linux < %s | \
-; RUN:     FileCheck %s --check-prefix=NACLOFF
-;
-; NACLON: RegisterPressure:
-; NACLON-NOT: %r11
-; NACLON: popq %r11
-; NACLON: nacljmp %r11d, %r15
-;
-; NACLOFF: RegisterPressure:
-; NACLOFF: %r11
-; NACLOFF: ret
-; ModuleID = 'pnacl-avoids-r11-x86-64.c'
-target datalayout = "e-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-p:32:32:32-v128:32:32"
-target triple = "le32-unknown-nacl"
-
-@v1a = external global i32
-@v1b = external global i32
-@v2a = external global i32
-@v2b = external global i32
-@v3a = external global i32
-@v3b = external global i32
-@v4a = external global i32
-@v4b = external global i32
-
-; Function Attrs: nounwind
-define void @RegisterPressure() #0 {
-entry:
-  %call = tail call i32 @GetValue() #2
-  %call1 = tail call i32 @GetValue() #2
-  %call2 = tail call i32 @GetValue() #2
-  %call3 = tail call i32 @GetValue() #2
-  %call4 = tail call i32 @GetValue() #2
-  %call5 = tail call i32 @GetValue() #2
-  %call6 = tail call i32 @GetValue() #2
-  %call7 = tail call i32 @GetValue() #2
-  %call8 = tail call i32 @GetValue() #2
-  %call9 = tail call i32 @GetValue() #2
-  %call10 = tail call i32 @GetValue() #2
-  %call11 = tail call i32 @GetValue() #2
-  %call12 = tail call i32 @GetValue() #2
-  %call13 = tail call i32 @GetValue() #2
-  %call14 = tail call i32 @GetValue() #2
-  %call15 = tail call i32 @GetValue() #2
-  tail call void @Use(i32 %call, i32 %call1, i32 %call2, i32 %call3, i32 %call4, i32 %call5, i32 %call6, i32 %call7, i32 %call8, i32 %call9, i32 %call10, i32 %call11, i32 %call12, i32 %call13, i32 %call14, i32 %call15) #2
-  tail call void @Use(i32 %call, i32 %call1, i32 %call2, i32 %call3, i32 %call4, i32 %call5, i32 %call6, i32 %call7, i32 %call8, i32 %call9, i32 %call10, i32 %call11, i32 %call12, i32 %call13, i32 %call14, i32 %call15) #2
-  %add = add nsw i32 %call1, %call
-  %add16 = add nsw i32 %add, %call2
-  %add17 = add nsw i32 %add16, %call3
-  %add18 = add nsw i32 %add17, %call4
-  %add19 = add nsw i32 %add18, %call5
-  %add20 = add nsw i32 %add19, %call6
-  %add21 = add nsw i32 %add20, %call7
-  store volatile i32 %add21, i32* @v1a, align 4
-  %add22 = add nsw i32 %call9, %call8
-  %add23 = add nsw i32 %add22, %call10
-  %add24 = add nsw i32 %add23, %call11
-  %add25 = add nsw i32 %add24, %call12
-  %add26 = add nsw i32 %add25, %call13
-  %add27 = add nsw i32 %add26, %call14
-  %add28 = add nsw i32 %add27, %call15
-  store volatile i32 %add28, i32* @v1b, align 4
-  %add32 = add nsw i32 %call8, %add17
-  %add33 = add nsw i32 %add32, %call9
-  %add34 = add nsw i32 %add33, %call10
-  %add35 = add nsw i32 %add34, %call11
-  store volatile i32 %add35, i32* @v2a, align 4
-  %add36 = add nsw i32 %call5, %call4
-  %add37 = add nsw i32 %add36, %call6
-  %add38 = add nsw i32 %add37, %call7
-  %add39 = add nsw i32 %add38, %call12
-  %add40 = add nsw i32 %add39, %call13
-  %add41 = add nsw i32 %add40, %call14
-  %add42 = add nsw i32 %add41, %call15
-  store volatile i32 %add42, i32* @v2b, align 4
-  %add44 = add nsw i32 %call4, %add
-  %add45 = add nsw i32 %add44, %call5
-  %add46 = add nsw i32 %add45, %call8
-  %add47 = add nsw i32 %add46, %call9
-  %add48 = add nsw i32 %add47, %call12
-  %add49 = add nsw i32 %add48, %call13
-  store volatile i32 %add49, i32* @v3a, align 4
-  %add50 = add nsw i32 %call3, %call2
-  %add51 = add nsw i32 %add50, %call6
-  %add52 = add nsw i32 %add51, %call7
-  %add53 = add nsw i32 %add52, %call10
-  %add54 = add nsw i32 %add53, %call11
-  %add55 = add nsw i32 %add54, %call14
-  %add56 = add nsw i32 %add55, %call15
-  store volatile i32 %add56, i32* @v3b, align 4
-  %add57 = add nsw i32 %call2, %call
-  %add58 = add nsw i32 %add57, %call4
-  %add59 = add nsw i32 %add58, %call6
-  %add60 = add nsw i32 %add59, %call8
-  %add61 = add nsw i32 %add60, %call10
-  %add62 = add nsw i32 %add61, %call12
-  %add63 = add nsw i32 %add62, %call14
-  store volatile i32 %add63, i32* @v4a, align 4
-  %add64 = add nsw i32 %call3, %call1
-  %add65 = add nsw i32 %add64, %call5
-  %add66 = add nsw i32 %add65, %call7
-  %add67 = add nsw i32 %add66, %call9
-  %add68 = add nsw i32 %add67, %call11
-  %add69 = add nsw i32 %add68, %call13
-  %add70 = add nsw i32 %add69, %call15
-  store volatile i32 %add70, i32* @v4b, align 4
-  tail call void @Use(i32 %call, i32 %call1, i32 %call2, i32 %call3, i32 %call4, i32 %call5, i32 %call6, i32 %call7, i32 %call8, i32 %call9, i32 %call10, i32 %call11, i32 %call12, i32 %call13, i32 %call14, i32 %call15) #2
-  tail call void @Use(i32 %call, i32 %call1, i32 %call2, i32 %call3, i32 %call4, i32 %call5, i32 %call6, i32 %call7, i32 %call8, i32 %call9, i32 %call10, i32 %call11, i32 %call12, i32 %call13, i32 %call14, i32 %call15) #2
-  ret void
-}
-
-declare i32 @GetValue() #1
-
-declare void @Use(i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32) #1
-
-attributes #0 = { nounwind "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf"="true" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "unsafe-fp-math"="false" "use-soft-float"="false" }
-attributes #1 = { "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf"="true" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "unsafe-fp-math"="false" "use-soft-float"="false" }
-attributes #2 = { nounwind }
diff --git a/test/NaCl/X86/pnacl-hides-sandbox-x86-64.c b/test/NaCl/X86/pnacl-hides-sandbox-x86-64.c
deleted file mode 100644
index e3ffba0baf6..00000000000
--- a/test/NaCl/X86/pnacl-hides-sandbox-x86-64.c
+++ /dev/null
@@ -1,54 +0,0 @@
-/*
-  Object file built using:
-  pnacl-clang -S -O0 -emit-llvm -o pnacl-hides-sandbox-x86-64.ll \
-      pnacl-hides-sandbox-x86-64.c
-*/
-
-#include <stdlib.h>
-#include <stdio.h>
-
-void TestDirectCall(void) {
-  extern void DirectCallTarget(void);
-  DirectCallTarget();
-}
-
-void TestIndirectCall(void) {
-  extern void (*IndirectCallTarget)(void);
-  IndirectCallTarget();
-}
-
-void TestMaskedFramePointer(int Arg) {
-  extern void Consume(void *);
-  // Calling alloca() is one way to force the rbp frame pointer.
-  void *Tmp = alloca(Arg);
-  Consume(Tmp);
-}
-
-void TestMaskedFramePointerVarargs(int Arg, ...) {
-  extern void Consume(void *);
-  void *Tmp = alloca(Arg);
-  Consume(Tmp);
-}
-
-void TestIndirectJump(int Arg) {
-  switch (Arg) {
-  case 2:
-    puts("Prime 1");
-    break;
-  case 3:
-    puts("Prime 2");
-    break;
-  case 5:
-    puts("Prime 3");
-    break;
-  case 7:
-    puts("Prime 4");
-    break;
-  case 11:
-    puts("Prime 5");
-    break;
-  }
-}
-
-void TestReturn(void) {
-}
diff --git a/test/NaCl/X86/pnacl-hides-sandbox-x86-64.ll b/test/NaCl/X86/pnacl-hides-sandbox-x86-64.ll
deleted file mode 100644
index 8abaa6a2074..00000000000
--- a/test/NaCl/X86/pnacl-hides-sandbox-x86-64.ll
+++ /dev/null
@@ -1,203 +0,0 @@
-; RUN: pnacl-llc -O2 -mtriple=x86_64-none-nacl -filetype=obj < %s | \
-; RUN:     llvm-objdump -d -r - | FileCheck %s
-; RUN: pnacl-llc -O2 -mtriple=x86_64-none-nacl -filetype=obj < %s | \
-; RUN:     llvm-objdump -d -r - | FileCheck %s --check-prefix=NOCALLRET
-; RUN: pnacl-llc -O2 -mtriple=x86_64-none-nacl -filetype=obj \
-; RUN:     -relocation-model=pic < %s | \
-; RUN:     llvm-objdump -d -r - | FileCheck %s --check-prefix=PIC
-
-; RUN: pnacl-llc -O2 -mtriple=x86_64-none-nacl -filetype=asm < %s | \
-; RUN: FileCheck %s --check-prefix=ASM
-
-; ModuleID = 'pnacl-hides-sandbox-x86-64.c'
-target datalayout = "e-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-p:32:32:32-v128:32:32"
-target triple = "le32-unknown-nacl"
-
-@IndirectCallTarget = external global void ()*
-@.str = private unnamed_addr constant [8 x i8] c"Prime 1\00", align 1
-@.str1 = private unnamed_addr constant [8 x i8] c"Prime 2\00", align 1
-@.str2 = private unnamed_addr constant [8 x i8] c"Prime 3\00", align 1
-@.str3 = private unnamed_addr constant [8 x i8] c"Prime 4\00", align 1
-@.str4 = private unnamed_addr constant [8 x i8] c"Prime 5\00", align 1
-
-; Function Attrs: nounwind
-define void @TestDirectCall() #0 {
-entry:
-  call void @DirectCallTarget()
-  ret void
-}
-; CHECK-LABEL: TestDirectCall:
-; Push only the bottom 32-bits of the frame pointer
-; CHECK: movl %ebp, %eax
-; CHECK-NEXT: pushq %rax
-; Push the immediate return address
-; CHECK:      pushq $0
-; CHECK-NEXT: .text
-; Immediate jump to the target
-; CHECK:      jmp 0
-; CHECK-NEXT: DirectCallTarget
-; The return sequence should use %r11
-; CHECK: popq %r11
-; CHECK: andl $-32, %r11d
-; CHECK-NEXT: addq %r15, %r11
-; CHECK-NEXT: jmpq *%r11
-
-; PIC-LABEL: TestDirectCall
-; PIC: leal 19(%rip), %r10d
-; PIC-NEXT: pushq %r10
-; PIC-NEXT: jmp 0
-; PIC-NEXT: DirectCallTarget
-
-; ASM-LABEL: TestDirectCall:
-; Push only the bottom 32-bits of the frame pointer
-; ASM: movl %ebp, %eax
-; ASM-NEXT: pushq %rax
-; In asm, direct calls are just 'call'
-; ASM: callq DirectCallTarget
-; The return sequence should use %r11
-; ASM: popq %r11
-; ASM: nacljmp %r11d, %r15
-
-declare hidden void @DirectCallTarget() #1
-
-; Function Attrs: nounwind
-define void @TestIndirectCall() #0 {
-entry:
-  %0 = load void ()*, void ()** @IndirectCallTarget, align 4
-  call void %0()
-  ret void
-}
-; CHECK-LABEL: TestIndirectCall:
-; Push the immediate return address
-; CHECK:      pushq $0
-; CHECK-NEXT: .text
-; Fixed sequence for indirect jump
-; CHECK:      andl $-32, %r11d
-; CHECK-NEXT: addq %r15, %r11
-; CHECK-NEXT: jmpq *%r11
-
-; PIC-LABEL: TestIndirectCall:
-; Ensure that the mov of the call target happens before the return address
-; calculation
-; PIC: movl {{.*}}, %r11d
-; Calculate and push the return address
-; PIC-NEXT: leal {{[0-9]+}}(%rip), %r10d
-; PIC-NEXT: pushq %r10
-; Fixed sequence for indirect jump
-; PIC: andl $-32, %r11d
-; PIC-NEXT: addq %r15, %r11
-; PIC-NEXT: jmpq *%r11
-
-; Function Attrs: nounwind
-define void @TestMaskedFramePointer(i32 %Arg) #0 {
-entry:
-  %Arg.addr = alloca i32, align 4
-  %Tmp = alloca i8*, align 4
-  store i32 %Arg, i32* %Arg.addr, align 4
-  %0 = load i32, i32* %Arg.addr, align 4
-  %1 = alloca i8, i32 %0
-  store i8* %1, i8** %Tmp, align 4
-  %2 = load i8*, i8** %Tmp, align 4
-  call void @Consume(i8* %2)
-  ret void
-}
-; Verify that the old frame pointer isn't leaked when saved
-; CHECK: TestMaskedFramePointer:
-; CHECK: movl    %ebp, %eax
-; CHECK: pushq   %rax
-; CHECK: movq    %rsp, %rbp
-
-declare void @Consume(i8*) #1
-
-; Function Attrs: nounwind
-define void @TestMaskedFramePointerVarargs(i32 %Arg, ...) #0 {
-entry:
-  %Arg.addr = alloca i32, align 4
-  %Tmp = alloca i8*, align 4
-  store i32 %Arg, i32* %Arg.addr, align 4
-  %0 = load i32, i32* %Arg.addr, align 4
-  %1 = alloca i8, i32 %0
-  store i8* %1, i8** %Tmp, align 4
-  %2 = load i8*, i8** %Tmp, align 4
-  call void @Consume(i8* %2)
-  ret void
-}
-; Verify use of r10 instead of rax in the presence of varargs,
-; when saving the old rbp.
-; CHECK: TestMaskedFramePointerVarargs:
-; CHECK: movl    %ebp, %r10d
-; CHECK: pushq   %r10
-; CHECK: movq    %rsp, %rbp
-
-; Function Attrs: nounwind
-define void @TestIndirectJump(i32 %Arg) #0 {
-entry:
-  %Arg.addr = alloca i32, align 4
-  store i32 %Arg, i32* %Arg.addr, align 4
-  %0 = load i32, i32* %Arg.addr, align 4
-  switch i32 %0, label %sw.epilog [
-    i32 2, label %sw.bb
-    i32 3, label %sw.bb1
-    i32 5, label %sw.bb3
-    i32 7, label %sw.bb5
-    i32 11, label %sw.bb7
-  ]
-
-sw.bb:                                            ; preds = %entry
-  %call = call i32 @puts(i8* getelementptr inbounds ([8 x i8], [8 x i8]* @.str, i32 0, i32 0))
-  br label %sw.epilog
-
-sw.bb1:                                           ; preds = %entry
-  %call2 = call i32 @puts(i8* getelementptr inbounds ([8 x i8], [8 x i8]* @.str1, i32 0, i32 0))
-  br label %sw.epilog
-
-sw.bb3:                                           ; preds = %entry
-  %call4 = call i32 @puts(i8* getelementptr inbounds ([8 x i8], [8 x i8]* @.str2, i32 0, i32 0))
-  br label %sw.epilog
-
-sw.bb5:                                           ; preds = %entry
-  %call6 = call i32 @puts(i8* getelementptr inbounds ([8 x i8], [8 x i8]* @.str3, i32 0, i32 0))
-  br label %sw.epilog
-
-sw.bb7:                                           ; preds = %entry
-  %call8 = call i32 @puts(i8* getelementptr inbounds ([8 x i8], [8 x i8]* @.str4, i32 0, i32 0))
-  br label %sw.epilog
-
-sw.epilog:                                        ; preds = %entry, %sw.bb7, %sw.bb5, %sw.bb3, %sw.bb1, %sw.bb
-  ret void
-}
-; Test the indirect jump sequence derived from a "switch" statement.
-; CHECK: TestIndirectJump:
-; CHECK:      andl $-32, %r11d
-; CHECK-NEXT: addq %r15, %r11
-; CHECK-NEXT: jmpq *%r11
-; At least 4 "jmp"s due to 5 switch cases
-; CHECK:      jmp
-; CHECK:      jmp
-; CHECK:      jmp
-; CHECK:      jmp
-; At least 1 direct call to puts()
-; CHECK:      pushq $0
-; CHECK-NEXT: .text
-; CHECK:      jmp 0
-; CHECK-NEXT: puts
-
-declare i32 @puts(i8*) #1
-
-; Function Attrs: nounwind
-define void @TestReturn() #0 {
-entry:
-  ret void
-}
-; Return sequence is just the indirect jump sequence
-; CHECK: TestReturn:
-; CHECK:      andl $-32, %r11d
-; CHECK-NEXT: addq %r15, %r11
-; CHECK-NEXT: jmpq *%r11
-
-attributes #0 = { nounwind "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf"="true" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "unsafe-fp-math"="false" "use-soft-float"="false" }
-attributes #1 = { "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf"="true" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "unsafe-fp-math"="false" "use-soft-float"="false" }
-
-; Special test that no "call" or "ret" instructions are generated.
-; NOCALLRET-NOT: call
-; NOCALLRET-NOT: ret
diff --git a/test/NaCl/X86/ro-jumptable.ll b/test/NaCl/X86/ro-jumptable.ll
deleted file mode 100644
index 79538c33584..00000000000
--- a/test/NaCl/X86/ro-jumptable.ll
+++ /dev/null
@@ -1,75 +0,0 @@
-; RUN: llc -mtriple=i386-unknown-nacl -filetype=asm %s -o - \
-; RUN:  | FileCheck %s
-
-declare i32 @bar(i32)
-
-; CHECK-LABEL: @foo
-; Check that the jump table for the switch goes in .rodata
-; CHECK: .section .rodata
-; CHECK: .long .LBB0
-define void @foo(i32 %a) {
-entry:
-  switch i32 %a, label %sw.epilog [
-    i32 3, label %sw.bb
-    i32 2, label %sw.bb1
-    i32 1, label %sw.bb3
-    i32 0, label %sw.bb5
-  ]
-
-sw.bb:                                            ; preds = %entry
-  %call = call i32 @bar(i32 1)
-  br label %sw.bb1
-
-sw.bb1:                                           ; preds = %entry, %sw.bb
-  %call2 = call i32 @bar(i32 2)
-  br label %sw.bb3
-
-sw.bb3:                                           ; preds = %entry, %sw.bb1
-  %call4 = call i32 @bar(i32 3)
-  br label %sw.bb5
-
-sw.bb5:                                           ; preds = %entry, %sw.bb3
-  %call6 = call i32 @bar(i32 4)
-  br label %sw.epilog
-
-sw.epilog:                                        ; preds = %sw.bb5, %entry
-  ret void
-}
-
-
-; CHECK: .section .text.foo_linkonce,"axG",@progbits,foo_linkonce,comdat
-; CHECK: @foo_linkonce
-; Check that the jump table for the comdat function goes into a comdat
-; group uniqued like the function
-; CHECK: .section .rodata.foo_linkonce,"aG",@progbits,foo_linkonce,comdat
-; CHECK: .long .LBB1
-$foo_linkonce = comdat any
-define linkonce_odr void @foo_linkonce(i32 %a) comdat {
-entry:
-  switch i32 %a, label %sw.epilog [
-    i32 3, label %sw.bb
-    i32 2, label %sw.bb1
-    i32 1, label %sw.bb3
-    i32 0, label %sw.bb5
-  ]
-
-sw.bb:                                            ; preds = %entry
-  %call = call i32 @bar(i32 1)
-  br label %sw.bb1
-
-sw.bb1:                                           ; preds = %entry, %sw.bb
-  %call2 = call i32 @bar(i32 2)
-  br label %sw.bb3
-
-sw.bb3:                                           ; preds = %entry, %sw.bb1
-  %call4 = call i32 @bar(i32 3)
-  br label %sw.bb5
-
-sw.bb5:                                           ; preds = %entry, %sw.bb3
-  %call6 = call i32 @bar(i32 4)
-  br label %sw.epilog
-
-sw.epilog:                                        ; preds = %sw.bb5, %entry
-  ret void
-}
-
diff --git a/test/NaCl/X86/stack-adjust.ll b/test/NaCl/X86/stack-adjust.ll
deleted file mode 100644
index c21a3e42bfa..00000000000
--- a/test/NaCl/X86/stack-adjust.ll
+++ /dev/null
@@ -1,12 +0,0 @@
-; RUN: llc -mtriple=x86_64-pc-nacl -force-align-stack %s -o - | FileCheck %s
-
-; Check that stack alignment (ANDri32 with the stack pointer) expands to
-; the bundle-locked adjustment for asm output, instead of a naclandsp pseudo
-define void @stackalign () {
-; CHECK-LABEL: stackalign
-  ret void
-; CHECK: .bundle_lock
-; CHECK-NEXT: andl $-8, %esp
-; CHECK-NEXT: leaq (%rsp,%r15), %rsp
-; CHECK-NEXT: .bundle_unlock
-}
\ No newline at end of file
diff --git a/test/Object/AMDGPU/elf-definitios.yaml b/test/Object/AMDGPU/elf-definitios.yaml
new file mode 100644
index 00000000000..1d9821a978d
--- /dev/null
+++ b/test/Object/AMDGPU/elf-definitios.yaml
@@ -0,0 +1,12 @@
+# RUN: yaml2obj -format=elf %s > %t.o
+# RUN: llvm-readobj -file-headers %t.o | FileCheck %s
+
+# CHECK:  Machine: EM_AMDGPU (0xE0)
+
+---
+FileHeader:
+  Class:   ELFCLASS64
+  Data:    ELFDATA2LSB
+  Type:    ET_REL
+  Machine: EM_AMDGPU
+...
diff --git a/test/Object/ARM/nm-mapping-symbol.s b/test/Object/ARM/nm-mapping-symbol.s
new file mode 100644
index 00000000000..485c1cc39d7
--- /dev/null
+++ b/test/Object/ARM/nm-mapping-symbol.s
@@ -0,0 +1,11 @@
+// RUN: llvm-mc %s -o %t.o -filetype=obj -triple=armv7-pc-linux
+// RUN: llvm-readobj -t %t.o | FileCheck %s
+// RUN: llvm-nm %t.o | FileCheck -allow-empty --check-prefix=NM %s
+
+// Test that nm doesn't print the mapping symbols
+
+// CHECK: Name: $d.0
+// NM-NOT: $d.0
+
+        .section        .foobar,"",%progbits
+        .asciz  "foo"
diff --git a/test/Object/Inputs/corrupt-invalid-dynamic-table-size.elf.x86-64 b/test/Object/Inputs/corrupt-invalid-dynamic-table-size.elf.x86-64
new file mode 100755
index 00000000000..ee29a4162e3
Binary files /dev/null and b/test/Object/Inputs/corrupt-invalid-dynamic-table-size.elf.x86-64 differ
diff --git a/test/Object/Inputs/corrupt-invalid-phentsize.elf.x86-64 b/test/Object/Inputs/corrupt-invalid-phentsize.elf.x86-64
new file mode 100755
index 00000000000..148d4c5fd64
Binary files /dev/null and b/test/Object/Inputs/corrupt-invalid-phentsize.elf.x86-64 differ
diff --git a/test/Object/Inputs/corrupt-invalid-relocation-size.elf.x86-64 b/test/Object/Inputs/corrupt-invalid-relocation-size.elf.x86-64
new file mode 100755
index 00000000000..2798ab34e17
Binary files /dev/null and b/test/Object/Inputs/corrupt-invalid-relocation-size.elf.x86-64 differ
diff --git a/test/Object/Inputs/corrupt-invalid-strtab.elf.x86-64 b/test/Object/Inputs/corrupt-invalid-strtab.elf.x86-64
new file mode 100755
index 00000000000..bdaa3cf9208
Binary files /dev/null and b/test/Object/Inputs/corrupt-invalid-strtab.elf.x86-64 differ
diff --git a/test/Object/Inputs/corrupt-invalid-virtual-addr.elf.x86-64 b/test/Object/Inputs/corrupt-invalid-virtual-addr.elf.x86-64
new file mode 100755
index 00000000000..58d995933ec
Binary files /dev/null and b/test/Object/Inputs/corrupt-invalid-virtual-addr.elf.x86-64 differ
diff --git a/test/Object/Inputs/elf-mip64-reloc.o b/test/Object/Inputs/elf-mip64-reloc.o
new file mode 100644
index 00000000000..5e689254a43
Binary files /dev/null and b/test/Object/Inputs/elf-mip64-reloc.o differ
diff --git a/test/Object/Inputs/invalid-bad-rel-type.elf b/test/Object/Inputs/invalid-bad-rel-type.elf
new file mode 100644
index 00000000000..2caebcdd1e5
Binary files /dev/null and b/test/Object/Inputs/invalid-bad-rel-type.elf differ
diff --git a/test/Object/Inputs/invalid-bad-section-address.coff b/test/Object/Inputs/invalid-bad-section-address.coff
new file mode 100644
index 00000000000..8d96e013751
Binary files /dev/null and b/test/Object/Inputs/invalid-bad-section-address.coff differ
diff --git a/test/Object/Inputs/invalid-section-index.elf b/test/Object/Inputs/invalid-section-index.elf
new file mode 100644
index 00000000000..a019d8a5479
Binary files /dev/null and b/test/Object/Inputs/invalid-section-index.elf differ
diff --git a/test/Object/Inputs/invalid-section-size.elf b/test/Object/Inputs/invalid-section-size.elf
new file mode 100644
index 00000000000..c111a4c6262
Binary files /dev/null and b/test/Object/Inputs/invalid-section-size.elf differ
diff --git a/test/Object/Inputs/invalid-sh_entsize.elf b/test/Object/Inputs/invalid-sh_entsize.elf
new file mode 100755
index 00000000000..9ea80731491
Binary files /dev/null and b/test/Object/Inputs/invalid-sh_entsize.elf differ
diff --git a/test/Object/Inputs/invalid-strtab-non-null.elf b/test/Object/Inputs/invalid-strtab-non-null.elf
new file mode 100644
index 00000000000..f52c0a14013
Binary files /dev/null and b/test/Object/Inputs/invalid-strtab-non-null.elf differ
diff --git a/test/Object/Inputs/invalid-strtab-size.elf b/test/Object/Inputs/invalid-strtab-size.elf
new file mode 100644
index 00000000000..fb1974613ab
Binary files /dev/null and b/test/Object/Inputs/invalid-strtab-size.elf differ
diff --git a/test/Object/Inputs/invalid-strtab-type.elf b/test/Object/Inputs/invalid-strtab-type.elf
new file mode 100644
index 00000000000..2a072ebe51d
Binary files /dev/null and b/test/Object/Inputs/invalid-strtab-type.elf differ
diff --git a/test/Object/Inputs/invalid-symbol-table-size.elf b/test/Object/Inputs/invalid-symbol-table-size.elf
new file mode 100755
index 00000000000..8329abbbd71
Binary files /dev/null and b/test/Object/Inputs/invalid-symbol-table-size.elf differ
diff --git a/test/Object/Inputs/macho-invalid-getsection-index b/test/Object/Inputs/macho-invalid-getsection-index
deleted file mode 100644
index b7e4b95bc9b..00000000000
Binary files a/test/Object/Inputs/macho-invalid-getsection-index and /dev/null differ
diff --git a/test/Object/Inputs/macho-invalid-header b/test/Object/Inputs/macho-invalid-header
new file mode 100644
index 00000000000..da52d43a03a
Binary files /dev/null and b/test/Object/Inputs/macho-invalid-header differ
diff --git a/test/Object/Inputs/macho-invalid-section-index-getSectionRawFinalSegmentName b/test/Object/Inputs/macho-invalid-section-index-getSectionRawFinalSegmentName
deleted file mode 100644
index e3f65865922..00000000000
Binary files a/test/Object/Inputs/macho-invalid-section-index-getSectionRawFinalSegmentName and /dev/null differ
diff --git a/test/Object/Inputs/macho64-invalid-getsection-index b/test/Object/Inputs/macho64-invalid-getsection-index
deleted file mode 100644
index a2a7bc10c4f..00000000000
Binary files a/test/Object/Inputs/macho64-invalid-getsection-index and /dev/null differ
diff --git a/test/Object/Inputs/macho64-invalid-incomplete-segment-load-command b/test/Object/Inputs/macho64-invalid-incomplete-segment-load-command
new file mode 100644
index 00000000000..82ec7246bdb
Binary files /dev/null and b/test/Object/Inputs/macho64-invalid-incomplete-segment-load-command differ
diff --git a/test/Object/Inputs/main-ret-zero-pe-i386.dll b/test/Object/Inputs/main-ret-zero-pe-i386.dll
new file mode 100755
index 00000000000..d4a343088b1
Binary files /dev/null and b/test/Object/Inputs/main-ret-zero-pe-i386.dll differ
diff --git a/test/Object/Inputs/main-ret-zero-pe-i386.exe b/test/Object/Inputs/main-ret-zero-pe-i386.exe
new file mode 100755
index 00000000000..366767b9b77
Binary files /dev/null and b/test/Object/Inputs/main-ret-zero-pe-i386.exe differ
diff --git a/test/Object/Inputs/no-section-header-string-table.elf-x86-64 b/test/Object/Inputs/no-section-header-string-table.elf-x86-64
new file mode 100644
index 00000000000..9024ad9b757
Binary files /dev/null and b/test/Object/Inputs/no-section-header-string-table.elf-x86-64 differ
diff --git a/test/Object/Inputs/no-section-table.so b/test/Object/Inputs/no-section-table.so
new file mode 100644
index 00000000000..fd176ebf7ce
Binary files /dev/null and b/test/Object/Inputs/no-section-table.so differ
diff --git a/test/Object/Inputs/no-start-symbol.elf-x86_64 b/test/Object/Inputs/no-start-symbol.elf-x86_64
new file mode 100644
index 00000000000..b2c9edc300d
Binary files /dev/null and b/test/Object/Inputs/no-start-symbol.elf-x86_64 differ
diff --git a/test/Object/Inputs/program-headers.mips b/test/Object/Inputs/program-headers.mips
index 54ebfea31f0..e3485b22374 100755
Binary files a/test/Object/Inputs/program-headers.mips and b/test/Object/Inputs/program-headers.mips differ
diff --git a/test/Object/Inputs/rel-no-sec-table.elf-x86-64 b/test/Object/Inputs/rel-no-sec-table.elf-x86-64
new file mode 100755
index 00000000000..d527305d659
Binary files /dev/null and b/test/Object/Inputs/rel-no-sec-table.elf-x86-64 differ
diff --git a/test/Object/Inputs/stackmap-test.macho-x86-64 b/test/Object/Inputs/stackmap-test.macho-x86-64
new file mode 100644
index 00000000000..588c5aa6685
Binary files /dev/null and b/test/Object/Inputs/stackmap-test.macho-x86-64 differ
diff --git a/test/Object/Inputs/symtab-only.a b/test/Object/Inputs/symtab-only.a
new file mode 100644
index 00000000000..611236bf3a1
Binary files /dev/null and b/test/Object/Inputs/symtab-only.a differ
diff --git a/test/Object/Inputs/thin-path.a b/test/Object/Inputs/thin-path.a
new file mode 100644
index 00000000000..d48fd874689
Binary files /dev/null and b/test/Object/Inputs/thin-path.a differ
diff --git a/test/Object/Inputs/trivial-object-test2.macho-x86-64 b/test/Object/Inputs/trivial-object-test2.macho-x86-64
new file mode 100644
index 00000000000..75eedb6dd31
Binary files /dev/null and b/test/Object/Inputs/trivial-object-test2.macho-x86-64 differ
diff --git a/test/Object/Mips/abi-flags.yaml b/test/Object/Mips/abi-flags.yaml
new file mode 100644
index 00000000000..8a11fc4124c
--- /dev/null
+++ b/test/Object/Mips/abi-flags.yaml
@@ -0,0 +1,63 @@
+# RUN: yaml2obj -format=elf %s > %t
+# RUN: llvm-readobj -mips-abi-flags %t | FileCheck -check-prefix=OBJ %s
+# RUN: obj2yaml %t | FileCheck -check-prefix=YAML %s
+
+# OBJ:      MIPS ABI Flags {
+# OBJ-NEXT:   Version: 0
+# OBJ-NEXT:   ISA: MIPS64r5
+# OBJ-NEXT:   ISA Extension: Cavium Networks Octeon3 (0x13)
+# OBJ-NEXT:   ASEs [ (0x103)
+# OBJ-NEXT:     DSP (0x1)
+# OBJ-NEXT:     DSPR2 (0x2)
+# OBJ-NEXT:     VZ (0x100)
+# OBJ-NEXT:   ]
+# OBJ-NEXT:   FP ABI: Hard float (double precision) (0x1)
+# OBJ-NEXT:   GPR size: 64
+# OBJ-NEXT:   CPR1 size: 64
+# OBJ-NEXT:   CPR2 size: 0
+# OBJ-NEXT:   Flags 1 [ (0x1)
+# OBJ-NEXT:     ODDSPREG (0x1)
+# OBJ-NEXT:   ]
+# OBJ-NEXT:   Flags 2: 0x0
+# OBJ-NEXT: }
+
+# YAML:      Sections:
+# YAML-NEXT: - Name:            .MIPS.abiflags
+# YAML-NEXT:   Type:            SHT_MIPS_ABIFLAGS
+# YAML-NEXT:   AddressAlign:    0x0000000000000008
+# YAML-NEXT:   ISA:             MIPS64
+# YAML-NEXT:   ISARevision:     0x05
+# YAML-NEXT:   ISAExtension:    EXT_OCTEON3
+# YAML-NEXT:   ASEs:            [ DSP, DSPR2, VIRT ]
+# YAML-NEXT:   FpABI:           FP_DOUBLE
+# YAML-NEXT:   GPRSize:         REG_64
+# YAML-NEXT:   CPR1Size:        REG_64
+# YAML-NEXT:   Flags1:          [ ODDSPREG ]
+
+FileHeader:
+  Class:   ELFCLASS64
+  Data:    ELFDATA2MSB
+  Type:    ET_REL
+  Machine: EM_MIPS
+
+Sections:
+- Name: .MIPS.abiflags
+  Type: SHT_MIPS_ABIFLAGS
+  AddressAlign: 8
+  Version:      0
+  ISA:          MIPS64
+  ISARevision:  5
+  ISAExtension: EXT_OCTEON3
+  ASEs:         [ DSP, DSPR2, VIRT ]
+  FpABI:        FP_DOUBLE
+  GPRSize:      REG_64
+  CPR1Size:     REG_64
+  CPR2Size:     REG_NONE
+  Flags1:       [ ODDSPREG ]
+  Flags2:       0x0
+
+Symbols:
+  Local:
+    - Name:    .MIPS.abiflags
+      Type:    STT_SECTION
+      Section: .MIPS.abiflags
diff --git a/test/Object/Mips/elf-abi.yaml b/test/Object/Mips/elf-abi.yaml
new file mode 100644
index 00000000000..5f11b14da59
--- /dev/null
+++ b/test/Object/Mips/elf-abi.yaml
@@ -0,0 +1,136 @@
+# RUN: yaml2obj -format=elf -docnum=1 %s > %t.o32
+# RUN: llvm-readobj -file-headers %t.o32 | FileCheck -check-prefix=O32OBJ %s
+# RUN: obj2yaml %t.o32 | FileCheck -check-prefix=O32YAML %s
+
+# O32OBJ:      Flags [ (0x50001002)
+# O32OBJ-NEXT:   EF_MIPS_ABI_O32 (0x1000)
+# O32OBJ-NEXT:   EF_MIPS_ARCH_32 (0x50000000)
+# O32OBJ-NEXT:   EF_MIPS_PIC (0x2)
+# O32OBJ-NEXT: ]
+
+# O32YAML:  Flags:  [ EF_MIPS_PIC, EF_MIPS_ABI_O32, EF_MIPS_ARCH_32 ]
+
+# RUN: yaml2obj -format=elf -docnum=2 %s > %t.o64
+# RUN: llvm-readobj -file-headers %t.o64 | FileCheck -check-prefix=O64OBJ %s
+# RUN: obj2yaml %t.o64 | FileCheck -check-prefix=O64YAML %s
+
+# O64OBJ:      Flags [ (0x60002002)
+# O64OBJ-NEXT:   EF_MIPS_ABI_O64 (0x2000)
+# O64OBJ-NEXT:   EF_MIPS_ARCH_64 (0x60000000)
+# O64OBJ-NEXT:   EF_MIPS_PIC (0x2)
+# O64OBJ-NEXT: ]
+
+# O64YAML:  Flags:  [ EF_MIPS_PIC, EF_MIPS_ABI_O64, EF_MIPS_ARCH_64 ]
+
+# RUN: yaml2obj -format=elf -docnum=3 %s > %t.eabi32
+# RUN: llvm-readobj -file-headers %t.eabi32 | FileCheck -check-prefix=E32OBJ %s
+# RUN: obj2yaml %t.eabi32 | FileCheck -check-prefix=E32YAML %s
+
+# E32OBJ:      Flags [ (0x50003002)
+# E32OBJ-NEXT:   EF_MIPS_ABI_EABI32 (0x3000)
+# E32OBJ-NEXT:   EF_MIPS_ARCH_32 (0x50000000)
+# E32OBJ-NEXT:   EF_MIPS_PIC (0x2)
+# E32OBJ-NEXT: ]
+
+# E32YAML:  Flags:  [ EF_MIPS_PIC, EF_MIPS_ABI_EABI32, EF_MIPS_ARCH_32 ]
+
+# RUN: yaml2obj -format=elf -docnum=4 %s > %t.eabi64
+# RUN: llvm-readobj -file-headers %t.eabi64 | FileCheck -check-prefix=E64OBJ %s
+# RUN: obj2yaml %t.eabi64 | FileCheck -check-prefix=E64YAML %s
+
+# E64OBJ:      Flags [ (0x60004002)
+# E64OBJ-NEXT:   EF_MIPS_ABI_EABI64 (0x4000)
+# E64OBJ-NEXT:   EF_MIPS_ARCH_64 (0x60000000)
+# E64OBJ-NEXT:   EF_MIPS_PIC (0x2)
+# E64OBJ-NEXT: ]
+
+# E64YAML:  Flags:  [ EF_MIPS_PIC, EF_MIPS_ABI_EABI64, EF_MIPS_ARCH_64 ]
+
+# o32
+---
+FileHeader:
+  Class:           ELFCLASS32
+  Data:            ELFDATA2LSB
+  Type:            ET_REL
+  Machine:         EM_MIPS
+  Flags:           [ EF_MIPS_PIC, EF_MIPS_ABI_O32, EF_MIPS_ARCH_32 ]
+Sections:
+  - Name:            .text
+    Type:            SHT_PROGBITS
+    Flags:           [ SHF_ALLOC, SHF_EXECINSTR ]
+    AddressAlign:    16
+    Size:            4
+
+Symbols:
+  Global:
+    - Name:    T1
+      Section: .text
+      Value:   0
+      Size:    4
+
+# o64
+---
+FileHeader:
+  Class:           ELFCLASS64
+  Data:            ELFDATA2LSB
+  Type:            ET_REL
+  Machine:         EM_MIPS
+  Flags:           [ EF_MIPS_PIC, EF_MIPS_ABI_O64, EF_MIPS_ARCH_64 ]
+Sections:
+  - Name:            .text
+    Type:            SHT_PROGBITS
+    Flags:           [ SHF_ALLOC, SHF_EXECINSTR ]
+    AddressAlign:    16
+    Size:            4
+
+Symbols:
+  Global:
+    - Name:    T1
+      Section: .text
+      Value:   0
+      Size:    4
+
+# eabio32
+---
+FileHeader:
+  Class:           ELFCLASS32
+  Data:            ELFDATA2LSB
+  Type:            ET_REL
+  Machine:         EM_MIPS
+  Flags:           [ EF_MIPS_PIC, EF_MIPS_ABI_EABI32, EF_MIPS_ARCH_32 ]
+Sections:
+  - Name:            .text
+    Type:            SHT_PROGBITS
+    Flags:           [ SHF_ALLOC, SHF_EXECINSTR ]
+    AddressAlign:    16
+    Size:            4
+
+Symbols:
+  Global:
+    - Name:    T1
+      Section: .text
+      Value:   0
+      Size:    4
+
+# eabi64
+---
+FileHeader:
+  Class:           ELFCLASS64
+  Data:            ELFDATA2LSB
+  Type:            ET_REL
+  Machine:         EM_MIPS
+  Flags:           [ EF_MIPS_PIC, EF_MIPS_ABI_EABI64, EF_MIPS_ARCH_64 ]
+Sections:
+  - Name:            .text
+    Type:            SHT_PROGBITS
+    Flags:           [ SHF_ALLOC, SHF_EXECINSTR ]
+    AddressAlign:    16
+    Size:            4
+
+Symbols:
+  Global:
+    - Name:    T1
+      Section: .text
+      Value:   0
+      Size:    4
+...
diff --git a/test/Object/Mips/elf-flags.yaml b/test/Object/Mips/elf-flags.yaml
new file mode 100644
index 00000000000..f79be9d4e07
--- /dev/null
+++ b/test/Object/Mips/elf-flags.yaml
@@ -0,0 +1,52 @@
+# RUN: yaml2obj -format=elf %s > %t
+# RUN: llvm-readobj -file-headers %t | FileCheck -check-prefix=OBJ %s
+# RUN: obj2yaml %t | FileCheck -check-prefix=YAML %s
+
+# OBJ:      Flags [ (0x9E8B1727)
+# OBJ-NEXT:   EF_MIPS_32BITMODE (0x100)
+# OBJ-NEXT:   EF_MIPS_ABI2 (0x20)
+# OBJ-NEXT:   EF_MIPS_ABI_O32 (0x1000)
+# OBJ-NEXT:   EF_MIPS_ARCH_32R6 (0x90000000)
+# OBJ-NEXT:   EF_MIPS_ARCH_ASE_M16 (0x4000000)
+# OBJ-NEXT:   EF_MIPS_ARCH_ASE_MDMX (0x8000000)
+# OBJ-NEXT:   EF_MIPS_CPIC (0x4)
+# OBJ-NEXT:   EF_MIPS_FP64 (0x200)
+# OBJ-NEXT:   EF_MIPS_MACH_OCTEON (0x8B0000)
+# OBJ-NEXT:   EF_MIPS_MICROMIPS (0x2000000)
+# OBJ-NEXT:   EF_MIPS_NAN2008 (0x400)
+# OBJ-NEXT:   EF_MIPS_NOREORDER (0x1)
+# OBJ-NEXT:   EF_MIPS_PIC (0x2)
+# OBJ-NEXT: ]
+
+# YAML:      FileHeader:
+# YAML-NEXT:   Class:           ELFCLASS32
+# YAML-NEXT:   Data:            ELFDATA2LSB
+# YAML-NEXT:   Type:            ET_REL
+# YAML-NEXT:   Machine:         EM_MIPS
+# YAML-NEXT:   Flags:           [ EF_MIPS_NOREORDER, EF_MIPS_PIC, EF_MIPS_CPIC, EF_MIPS_ABI2, EF_MIPS_32BITMODE, EF_MIPS_FP64, EF_MIPS_NAN2008, EF_MIPS_MICROMIPS, EF_MIPS_ARCH_ASE_M16, EF_MIPS_ARCH_ASE_MDMX, EF_MIPS_ABI_O32, EF_MIPS_MACH_OCTEON, EF_MIPS_ARCH_32R6 ]
+
+---
+FileHeader:
+  Class:    ELFCLASS32
+  Data:     ELFDATA2LSB
+  Type:     ET_REL
+  Machine:  EM_MIPS
+  Flags:    [ EF_MIPS_NOREORDER, EF_MIPS_PIC, EF_MIPS_CPIC, EF_MIPS_ABI2,
+              EF_MIPS_32BITMODE, EF_MIPS_FP64, EF_MIPS_NAN2008,
+              EF_MIPS_ABI_O32, EF_MIPS_ARCH_32R6,
+              EF_MIPS_MICROMIPS, EF_MIPS_ARCH_ASE_M16,
+              EF_MIPS_ARCH_ASE_MDMX, EF_MIPS_MACH_OCTEON ]
+Sections:
+  - Name:          .text
+    Type:          SHT_PROGBITS
+    Flags:         [ SHF_ALLOC, SHF_EXECINSTR ]
+    AddressAlign:  16
+    Size:          4
+
+Symbols:
+  Global:
+    - Name:    T1
+      Section: .text
+      Value:   0
+      Size:    4
+...
diff --git a/test/Object/Mips/elf-mips64-rel.yaml b/test/Object/Mips/elf-mips64-rel.yaml
index 8b595099b9e..7c41b78e6ad 100644
--- a/test/Object/Mips/elf-mips64-rel.yaml
+++ b/test/Object/Mips/elf-mips64-rel.yaml
@@ -58,6 +58,7 @@ Sections:
   - Name:            .rela.text
     Type:            SHT_RELA
     Flags:           [ SHF_INFO_LINK ]
+    AddressAlign:    0x8
     Info:            .text
     Relocations:
       - Offset:      0x14
diff --git a/test/Object/Mips/reloc-visit.test b/test/Object/Mips/reloc-visit.test
new file mode 100644
index 00000000000..d75551a387c
--- /dev/null
+++ b/test/Object/Mips/reloc-visit.test
@@ -0,0 +1,6 @@
+RUN: llvm-dwarfdump -debug-dump=info %p/../Inputs/elf-mip64-reloc.o 2>&1 | \
+RUN:   FileCheck %s
+
+CHECK: DW_AT_name [DW_FORM_strp]	( .debug_str[0x00000037] = "<stdin>")
+CHECK: DW_AT_name [DW_FORM_strp]	( .debug_str[0x00000054] = "foo")
+CHECK: DW_AT_name [DW_FORM_strp]	( .debug_str[0x00000058] = "int")
diff --git a/test/Object/X86/nm-coff.s b/test/Object/X86/nm-coff.s
new file mode 100644
index 00000000000..b8f28c7e52a
--- /dev/null
+++ b/test/Object/X86/nm-coff.s
@@ -0,0 +1,9 @@
+// RUN: llvm-mc %s -o %t -filetype=obj -triple=x86_64-pc-win32
+// RUN: llvm-nm --undefined-only %t | FileCheck %s
+// CHECK: w foo
+
+g:
+	movl	foo(%rip), %eax
+	retq
+
+	.weak	foo
diff --git a/test/Object/X86/nm-macho.s b/test/Object/X86/nm-macho.s
new file mode 100644
index 00000000000..7bdfa34ca36
--- /dev/null
+++ b/test/Object/X86/nm-macho.s
@@ -0,0 +1,9 @@
+// RUN: llvm-mc %s -o %t -filetype=obj -triple=x86_64-apple-darwin
+// RUN: llvm-nm -n %t | FileCheck %s
+// CHECK: 0000000000000000 t _f
+// CHECK: 0000000000000004 C _a
+
+_f:
+	retq
+
+	.comm	_a,4
diff --git a/test/Object/X86/nm-print-size.s b/test/Object/X86/nm-print-size.s
new file mode 100644
index 00000000000..a755ffa3ded
--- /dev/null
+++ b/test/Object/X86/nm-print-size.s
@@ -0,0 +1,18 @@
+// RUN: llvm-mc %s -o %t -filetype=obj -triple=x86_64-pc-linux
+// RUN: llvm-nm --print-size %t | FileCheck %s
+
+// CHECK: 0000000000000000 ffffffffffffffff n a
+// CHECK: 0000000000000000 0000000000000000 N b
+// CHECK: 0000000000000004 0000000000000004 C c
+// CHECK: ffffffffffffffff 0000000000000000 a d
+
+        .section foo
+a:
+        .size a, 0xffffffffffffffff
+
+        .global b
+b:
+
+        .comm c,4,8
+
+d = 0xffffffffffffffff
diff --git a/test/Object/X86/no-start-symbol.test b/test/Object/X86/no-start-symbol.test
new file mode 100644
index 00000000000..b468894c7b6
--- /dev/null
+++ b/test/Object/X86/no-start-symbol.test
@@ -0,0 +1,9 @@
+RUN: llvm-objdump -d %p/../Inputs/no-start-symbol.elf-x86_64 | FileCheck %s
+
+Test that we disassemble the start of the section.
+
+CHECK: Disassembly of section .text:
+CHECK-NEXT:  .text:
+CHECK-NEXT:       0:	90 	nop
+CHECK: foo:
+CHECK-NEXT:       1:	90 	nop
diff --git a/test/Object/archive-extract.test b/test/Object/archive-extract.test
new file mode 100644
index 00000000000..50372d530d8
--- /dev/null
+++ b/test/Object/archive-extract.test
@@ -0,0 +1,56 @@
+; This test just makes sure that llvm-ar can extract bytecode members
+; from various style archives.
+
+; RUN: cd %T
+
+; RUN: rm -f very_long_bytecode_file_name.bc
+; RUN: llvm-ar p %p/Inputs/GNU.a very_long_bytecode_file_name.bc | \
+; RUN:   cmp -s %p/Inputs/very_long_bytecode_file_name.bc -
+; RUN: llvm-ar x %p/Inputs/GNU.a very_long_bytecode_file_name.bc
+; RUN: cmp -s %p/Inputs/very_long_bytecode_file_name.bc \
+; RUN:        very_long_bytecode_file_name.bc
+
+; RUN: rm -f very_long_bytecode_file_name.bc
+; RUN: llvm-ar p %p/Inputs/MacOSX.a very_long_bytecode_file_name.bc | \
+; RUN:   cmp -s %p/Inputs/very_long_bytecode_file_name.bc -
+; RUN: llvm-ar x %p/Inputs/MacOSX.a very_long_bytecode_file_name.bc
+; RUN: cmp -s %p/Inputs/very_long_bytecode_file_name.bc \
+; RUN:        very_long_bytecode_file_name.bc
+
+; RUN: rm -f very_long_bytecode_file_name.bc
+; RUN: llvm-ar p %p/Inputs/SVR4.a very_long_bytecode_file_name.bc | \
+; RUN:   cmp -s %p/Inputs/very_long_bytecode_file_name.bc -
+; RUN: llvm-ar x %p/Inputs/SVR4.a very_long_bytecode_file_name.bc
+; RUN: cmp -s %p/Inputs/very_long_bytecode_file_name.bc \
+; RUN:        very_long_bytecode_file_name.bc
+
+; RUN: rm -f very_long_bytecode_file_name.bc
+; RUN: llvm-ar p %p/Inputs/xpg4.a very_long_bytecode_file_name.bc |\
+; RUN:   cmp -s %p/Inputs/very_long_bytecode_file_name.bc -
+; RUN: llvm-ar x %p/Inputs/xpg4.a very_long_bytecode_file_name.bc
+; RUN: cmp -s %p/Inputs/very_long_bytecode_file_name.bc \
+; RUN:        very_long_bytecode_file_name.bc
+
+
+; Test that the 'o' option is working by extracting a file, putting it in o
+; new archive and checking that date.
+; RUN: rm -f very_long_bytecode_file_name.bc
+; RUN: llvm-ar xo %p/Inputs/GNU.a very_long_bytecode_file_name.bc
+; RUN: rm -f %t.a
+; RUN: llvm-ar rcU %t.a very_long_bytecode_file_name.bc
+; RUN: env TZ=GMT llvm-ar tv %t.a | FileCheck %s
+
+CHECK: 1465 2004-11-19 03:01:31.000000000 very_long_bytecode_file_name.bc
+
+
+RUN: not llvm-ar x %p/Inputs/GNU.a foo.o 2>&1 | FileCheck --check-prefix=NOTFOUND %s
+NOTFOUND: foo.o was not found
+
+RUN: not llvm-ar x %p/Inputs/thin.a foo.o 2>&1 | FileCheck %s --check-prefix=THINEXTRACT
+THINEXTRACT: extracting from a thin archive is not supported
+
+RUN: llvm-ar p %p/Inputs/thin.a evenlen | FileCheck %s --check-prefix=EVENLEN
+EVENLEN: evenlen
+
+RUN: not llvm-ar p %p/Inputs/thin-path.a t/test2.o | FileCheck %s --check-prefix=MISSING
+MISSING: {{N|n}}o such file or directory.
diff --git a/test/Object/archive-format.test b/test/Object/archive-format.test
index f076123d2ea..651a9b9ef15 100644
--- a/test/Object/archive-format.test
+++ b/test/Object/archive-format.test
@@ -6,14 +6,59 @@ RUN: cd %t
 
 RUN: echo -n bar. > 0123456789abcde
 RUN: echo -n zed. > 0123456789abcdef
+RUN: mkdir -p foo
+RUN: echo -n bar2 > foo/0123456789abcde
+RUN: echo -n zed2 > foo/0123456789abcdef
 
-RUN: rm -f test.a
-RUN: llvm-ar rc test.a 0123456789abcde 0123456789abcdef
-RUN: cat test.a | FileCheck -strict-whitespace %s
+RUN: rm -f %t.a
+RUN: llvm-ar --format=gnu rc %t.a 0123456789abcde 0123456789abcdef foo/0123456789abcde foo/0123456789abcdef
+RUN: cat %t.a | FileCheck -strict-whitespace %s
 
 CHECK:      !<arch>
-CHECK-NEXT: //                                              18        `
+CHECK-NEXT: //                                              36        `
+CHECK-NEXT: 0123456789abcdef/
 CHECK-NEXT: 0123456789abcdef/
-CHECK-NEXT: 0123456789abcde/{{................................}}4         `
-CHECK-NEXT: bar./0              {{................................}}4         `
+CHECK-NEXT: 0123456789abcde/0           0     0     644     4         `
+CHECK-NEXT: bar.
+CHECK-SAME: /0              0           0     0     644     4         `
 CHECK-NEXT: zed.
+CHECK-SAME: 0123456789abcde/0           0     0     644     4         `
+CHECK-NEXT: bar2
+CHECK-SAME: /18             0           0     0     644     4         `
+CHECK-NEXT: zed2
+
+RUN: rm -f %t.a
+RUN: llvm-ar --format=bsd rc %t.a 0123456789abcde 0123456789abcdef
+RUN: cat %t.a | FileCheck -strict-whitespace --check-prefix=BSD %s
+
+BSD:      !<arch>
+BSD-NEXT: #1/20           0           0     0     644     24        `
+BSD-NEXT: 0123456789abcde{{.....}}bar.
+BSD-SAME: #1/16           0           0     0     644     20        `
+BSD-NEXT: 0123456789abcdefzed.
+
+RUN: rm -f test.a
+RUN: llvm-ar --format=gnu rcT test.a 0123456789abcde 0123456789abcdef
+RUN: cat test.a | FileCheck -strict-whitespace --check-prefix=THIN %s
+THIN: !<thin>
+THIN-NEXT: //                                              36        `
+THIN-NEXT: 0123456789abcde/
+THIN-NEXT: 0123456789abcdef/{{$}}
+THIN: {{^$}}
+THIN: /0              0           0     0     644     4         `
+THIN-NEXT: /17             0           0     0     644     4         `
+
+RUN: mkdir -p bar
+RUN: rm -f bar/test.a
+RUN: llvm-ar --format=gnu rcT bar/test.a 0123456789abcde 0123456789abcdef foo/0123456789abcde foo/0123456789abcdef
+RUN: cat bar/test.a | FileCheck -strict-whitespace --check-prefix=THIN-PATH %s
+THIN-PATH: !<thin>
+THIN-PATH-NEXT: //                                              90        `
+THIN-PATH-NEXT: ..{{/|\\}}0123456789abcde/
+THIN-PATH-NEXT: ..{{/|\\}}0123456789abcdef/
+THIN-PATH-NEXT: ..{{/|\\}}foo{{/|\\}}0123456789abcde/
+THIN-PATH-NEXT: ..{{/|\\}}foo{{/|\\}}0123456789abcdef/
+THIN-PATH-NEXT: /0              0           0     0     644     4         `
+THIN-PATH-NEXT: /20             0           0     0     644     4         `
+THIN-PATH-NEXT: /41             0           0     0     644     4         `
+THIN-PATH-NEXT: /65             0           0     0     644     4         `
diff --git a/test/Object/archive-symtab.test b/test/Object/archive-symtab.test
index 01f17bcc8b6..c50833d6a77 100644
--- a/test/Object/archive-symtab.test
+++ b/test/Object/archive-symtab.test
@@ -1,5 +1,5 @@
 RUN: rm -f %t.a
-RUN: llvm-ar rcs %t.a %p/Inputs/trivial-object-test.elf-x86-64 %p/Inputs/trivial-object-test2.elf-x86-64
+RUN: llvm-ar rcsU %t.a %p/Inputs/trivial-object-test.elf-x86-64 %p/Inputs/trivial-object-test2.elf-x86-64
 RUN: llvm-nm -M %t.a | FileCheck %s
 
 CHECK: Archive map
@@ -8,6 +8,17 @@ CHECK-NEXT: foo in trivial-object-test2.elf-x86-64
 CHECK-NEXT: main in trivial-object-test2.elf-x86-64
 CHECK-NOT: bar
 
+
+RUN: rm -f %t.a
+RUN: llvm-ar --format=gnu rcT %t.a %p/Inputs/trivial-object-test.elf-x86-64 %p/Inputs/trivial-object-test2.elf-x86-64
+RUN: llvm-nm -M %t.a | FileCheck --check-prefix=THIN %s
+
+THIN: Archive map
+THIN-NEXT: main in {{.*}}/Inputs/trivial-object-test.elf-x86-64
+THIN-NEXT: foo in {{.*}}/Inputs/trivial-object-test2.elf-x86-64
+THIN-NEXT: main in {{.*}}/Inputs/trivial-object-test2.elf-x86-64
+
+
 CHECK: trivial-object-test.elf-x86-64:
 CHECK-NEXT:                  U SomeOtherFunction
 CHECK-NEXT: 0000000000000000 T main
@@ -19,7 +30,7 @@ CHECK-NEXT: 0000000000000006 T foo
 CHECK-NEXT: 0000000000000016 T main
 
 RUN: rm -f %t.a
-RUN: llvm-ar rcS %t.a %p/Inputs/trivial-object-test.elf-x86-64 %p/Inputs/trivial-object-test2.elf-x86-64
+RUN: llvm-ar rcSU %t.a %p/Inputs/trivial-object-test.elf-x86-64 %p/Inputs/trivial-object-test2.elf-x86-64
 RUN: llvm-nm -M %t.a | FileCheck %s --check-prefix=NOMAP
 
 NOMAP-NOT: Archive map
@@ -54,7 +65,7 @@ RUN: llvm-nm -M %t.a | FileCheck %s --check-prefix=CORRUPT
 repeate the test with llvm-ranlib
 
 RUN: rm -f %t.a
-RUN: llvm-ar rcS %t.a %p/Inputs/trivial-object-test.elf-x86-64 %p/Inputs/trivial-object-test2.elf-x86-64
+RUN: llvm-ar rcSU %t.a %p/Inputs/trivial-object-test.elf-x86-64 %p/Inputs/trivial-object-test2.elf-x86-64
 RUN: llvm-nm -M %t.a | FileCheck %s --check-prefix=NOMAP
 
 RUN: llvm-ranlib %t.a
@@ -66,3 +77,32 @@ RUN: llvm-nm -M %p/Inputs/macho-archive-unsorted-x86_64.a | FileCheck %s --check
 BSD-MachO: Archive map
 BSD-MachO: _bar in bar.o
 BSD-MachO: _foo in foo.o
+
+RUN: rm -f %t.a
+RUN: llvm-ar --format=bsd rcsU %t.a %p/Inputs/trivial-object-test.macho-x86-64 %p/Inputs/trivial-object-test2.macho-x86-64
+RUN: llvm-nm -M %t.a | FileCheck --check-prefix=MACHO %s
+
+MACHO: Archive map
+MACHO-NEXT: _main in trivial-object-test.macho-x86-64
+MACHO-NEXT: _foo in trivial-object-test2.macho-x86-64
+MACHO-NEXT: _main in trivial-object-test2.macho-x86-64
+MACHO-NOT: bar
+
+MACHO: trivial-object-test.macho-x86-64
+MACHO-NEXT: 0000000000000028 s L_.str
+MACHO-NEXT:                  U _SomeOtherFunction
+MACHO-NEXT: 0000000000000000 T _main
+MACHO-NEXT:                  U _puts
+
+MACHO: trivial-object-test2.macho-x86-64
+MACHO-NEXT: 0000000000000000 t _bar
+MACHO-NEXT: 0000000000000001 T _foo
+MACHO-NEXT: 0000000000000002 T _main
+
+Test that we pad the symbol table so that it ends in a multiple of 4 bytes:
+8 + 60 + 36 == 104
+RUN: rm -f %t.a
+RUN: llvm-ar --format=bsd rcsU %t.a %p/Inputs/trivial-object-test.macho-x86-64
+RUN: FileCheck --check-prefix=MACHO-SYMTAB-ALIGN %s < %t.a
+MACHO-SYMTAB-ALIGN: !<arch>
+MACHO-SYMTAB-ALIGN-NEXT: #1/12           {{..........}}  0     0     0       36        `
diff --git a/test/Object/archive-toc.test b/test/Object/archive-toc.test
index 79a6e0e0ba8..cca4b868c46 100644
--- a/test/Object/archive-toc.test
+++ b/test/Object/archive-toc.test
@@ -34,3 +34,13 @@ THIN:      rw-r--r-- 1000/1000      8 2014-12-16 00:56:27.000000000 evenlen
 THIN-NEXT: rw-r--r-- 1000/1000      7 2014-12-16 00:56:27.000000000 oddlen
 THIN-NEXT: rwxr-xr-x 1000/1000   1465 2014-12-16 00:56:27.000000000 very_long_bytecode_file_name.bc
 THIN-NEXT: rw-r--r-- 1000/1000   2280 2014-12-16 00:56:27.000000000 IsNAN.o
+
+Test reading an archive with just a symbol table. We use to reject them.
+RUN: llvm-ar tv %p/Inputs/symtab-only.a | FileCheck --allow-empty --check-prefix=EMPTY %s
+EMPTY-NOT: {{.}}
+
+Test reading a thin archive with directory names.
+RUN: env TZ=GMT llvm-ar tv %p/Inputs/thin-path.a | FileCheck %s --check-prefix=THINPATH -strict-whitespace
+
+THINPATH: rw-r--r-- 0/0   1224 1970-01-01 00:00:00.000000000 test.o
+THINPATH-NEXT: rw-r--r-- 0/0   1224 1970-01-01 00:00:00.000000000 t/test2.o
diff --git a/test/Object/archive-update.test b/test/Object/archive-update.test
index 91ca8102c7e..ef1a71f60f4 100644
--- a/test/Object/archive-update.test
+++ b/test/Object/archive-update.test
@@ -8,28 +8,43 @@ RUN: mkdir -p %t.older
 RUN: echo older > %t.older/evenlen
 
 Either the shell supports the 'touch' command with a flag to manually set the
-mtime or we sleep for over a second so that the mtime is definitely observable.
-RUN: touch -m -t 200001010000 %t.older/evenlen || sleep 1.1
+mtime or we sleep for over two seconds so that the mtime is definitely
+observable.
+RUN: touch -m -t 200001010000 %t.older/evenlen || sleep 2.1
 
 RUN: mkdir -p %t.newer
 RUN: echo newer > %t.newer/evenlen
 RUN: touch %t.newer/evenlen
 
 Create an achive with the newest file
-RUN: llvm-ar r %t.a %t.newer/evenlen
+RUN: llvm-ar rU %t.a %t.newer/evenlen
 RUN: llvm-ar p %t.a | FileCheck --check-prefix=NEWER %s
 
 Check that without the 'u' option the member is replaced with an older file.
-RUN: llvm-ar r %t.a %t.older/evenlen
+RUN: llvm-ar rU %t.a %t.older/evenlen
 RUN: llvm-ar p %t.a | FileCheck --check-prefix=OLDER %s
 
 Check that with the 'u' option the member is replaced with a newer file.
-RUN: llvm-ar ru %t.a %t.newer/evenlen
+RUN: llvm-ar ruU %t.a %t.newer/evenlen
 RUN: llvm-ar p %t.a | FileCheck --check-prefix=NEWER %s
 
 Check that with the 'u' option the member is not replaced with an older file.
-RUN: llvm-ar ru %t.a %t.older/evenlen
+RUN: llvm-ar ruU %t.a %t.older/evenlen
 RUN: llvm-ar p %t.a | FileCheck --check-prefix=NEWER %s
 
 NEWER: newer
 OLDER: older
+
+RUN: rm -f %t.a
+RUN: echo foo > foo
+RUN: echo bar > bar
+RUN: llvm-ar --format=gnu rcT %t.a foo
+RUN: llvm-ar --format=gnu rcT %t.a bar
+RUN: llvm-ar t %t.a | FileCheck --check-prefix=BOTH-FILES %s
+BOTH-FILES: foo
+BOTH-FILES: bar
+
+RUN: rm -f %t.a
+RUN: llvm-ar --format=gnu rc %t.a foo
+RUN: not llvm-ar --format=gnu rcT %t.a bar 2>&1 | FileCheck --check-prefix=ERROR %s
+ERROR: Cannot convert a regular archive to a thin one.
diff --git a/test/Object/coff-archive.test b/test/Object/coff-archive.test
index 239a96b4c35..c8051ebe1bf 100644
--- a/test/Object/coff-archive.test
+++ b/test/Object/coff-archive.test
@@ -66,7 +66,26 @@ CHECKIDX: 00000000 N .debug$T
 CHECKIDX: 00000000 i .drectve
 CHECKIDX: 00000001 a @feat.00
 CHECKIDX: 00ab9d1b a @comp.id
+
 CHECKIDX: Debug\mymath.obj:
+CHECKIDX:          U ??2@YAPAXI@Z
+CHECKIDX:          U ??3@YAXPAX@Z
+CHECKIDX:          U ??_7type_info@@6B@
+CHECKIDX:          w ??_Einvalid_argument@std@@UAEPAXI@Z
+CHECKIDX:          w ??_Elogic_error@std@@UAEPAXI@Z
+CHECKIDX:          U ??_Ginvalid_argument@std@@UAEPAXI@Z
+CHECKIDX:          U ??_Glogic_error@std@@UAEPAXI@Z
+CHECKIDX:          U ?what@exception@std@@UBEPBDXZ
+CHECKIDX:          U @__security_check_cookie@4
+CHECKIDX:          U __CxxThrowException@8
+CHECKIDX:          U __RTC_CheckEsp
+CHECKIDX:          U __RTC_InitBase
+CHECKIDX:          U __RTC_Shutdown
+CHECKIDX:          U ___CxxFrameHandler3
+CHECKIDX:          U ___security_cookie
+CHECKIDX:          U __fltused
+CHECKIDX:          U __imp_??0exception@std@@QAE@ABQBD@Z
+CHECKIDX:          U __imp_??1exception@std@@UAE@XZ
 CHECKIDX: 00000000 d .data
 CHECKIDX: 00000000 d .data
 CHECKIDX: 00000000 d .data
@@ -205,21 +224,3 @@ CHECKIDX: 00000004 R ??_7logic_error@std@@6B@
 CHECKIDX: 00000008 r __ehfuncinfo$?Divide@MyMathFuncs@MathFuncs@@SANNN@Z
 CHECKIDX: 0000000e t __ehhandler$?Divide@MyMathFuncs@MathFuncs@@SANNN@Z
 CHECKIDX: 00ab9d1b a @comp.id
-CHECKIDX:          U ??2@YAPAXI@Z
-CHECKIDX:          U ??3@YAXPAX@Z
-CHECKIDX:          U ??_7type_info@@6B@
-CHECKIDX:          w ??_Einvalid_argument@std@@UAEPAXI@Z
-CHECKIDX:          w ??_Elogic_error@std@@UAEPAXI@Z
-CHECKIDX:          U ??_Ginvalid_argument@std@@UAEPAXI@Z
-CHECKIDX:          U ??_Glogic_error@std@@UAEPAXI@Z
-CHECKIDX:          U ?what@exception@std@@UBEPBDXZ
-CHECKIDX:          U @__security_check_cookie@4
-CHECKIDX:          U __CxxThrowException@8
-CHECKIDX:          U __RTC_CheckEsp
-CHECKIDX:          U __RTC_InitBase
-CHECKIDX:          U __RTC_Shutdown
-CHECKIDX:          U ___CxxFrameHandler3
-CHECKIDX:          U ___security_cookie
-CHECKIDX:          U __fltused
-CHECKIDX:          U __imp_??0exception@std@@QAE@ABQBD@Z
-CHECKIDX:          U __imp_??1exception@std@@UAE@XZ
diff --git a/test/Object/coff-invalid.test b/test/Object/coff-invalid.test
new file mode 100644
index 00000000000..b85543dcfe8
--- /dev/null
+++ b/test/Object/coff-invalid.test
@@ -0,0 +1,13 @@
+RUN: llvm-readobj -s %p/Inputs/invalid-bad-section-address.coff 2>&1 | \
+RUN: FileCheck --check-prefix=SECTIONS %s
+
+SECTIONS:      Section {
+SECTIONS-NEXT:   Number: 1
+SECTIONS-NEXT:   Name: .text (2E 74 65 78 74 00 00 00)
+SECTIONS-NEXT:   VirtualSize: 0x0
+SECTIONS-NEXT:   VirtualAddress: 0x1000000
+
+RUN: not llvm-readobj -r %p/Inputs/invalid-bad-section-address.coff 2>&1 | \
+RUN: FileCheck %s
+
+CHECK: Sections with relocations should have an address of 0
diff --git a/test/Object/corrupt.test b/test/Object/corrupt.test
index ef72a0979b5..05a8d0fab5c 100644
--- a/test/Object/corrupt.test
+++ b/test/Object/corrupt.test
@@ -2,23 +2,57 @@
 RUN: not llvm-readobj %p/Inputs/corrupt.elf-x86-64 -sections \
 RUN:     2>&1 | FileCheck --check-prefix=SECNAME %s
 
+SECNAME: Error reading file: Invalid data was encountered while parsing the file.
+
+
 // Section data offset past end of file.
 RUN: not llvm-readobj %p/Inputs/corrupt.elf-x86-64 -sections -section-data \
 RUN:     2>&1 | FileCheck --check-prefix=SECDATA %s
 
+SECDATA: Error reading file: Invalid data was encountered while parsing the file.
+
+
 // Symbol name offset overflows string table.
 RUN: not llvm-readobj %p/Inputs/corrupt.elf-x86-64 -symbols \
 RUN:     2>&1 | FileCheck --check-prefix=SYMNAME %s
 
+SYMNAME: Error reading file: Invalid data was encountered while parsing the file.
+
+
 // Version index in .gnu.version overflows the version map.
 RUN: not llvm-readobj %p/Inputs/corrupt-version.elf-x86_64 -dt \
 RUN:     2>&1 | FileCheck --check-prefix=VER %s
 
-SECNAME: Error reading file: Invalid data was encountered while parsing the file.
+VER: Error reading file: Invalid data was encountered while parsing the file.
 
-SECDATA: Error reading file: Invalid data was encountered while parsing the file.
-SECDATA: Error reading file: Invalid data was encountered while parsing the file.
 
-SYMNAME: Error reading file: Invalid data was encountered while parsing the file.
+// The file is missing the dynamic string table but has references to it.
+RUN: not llvm-readobj -dynamic-table %p/Inputs/corrupt-invalid-strtab.elf.x86-64 \
+RUN:     2>&1 | FileCheck --check-prefix=STRTAB %s
 
-VER: Error reading file: Invalid data was encountered while parsing the file.
+STRTAB: Invalid dynamic string table reference
+
+RUN: not llvm-readobj -program-headers \
+RUN:   %p/Inputs/corrupt-invalid-phentsize.elf.x86-64 2>&1 | \
+RUN:   FileCheck --check-prefix=PHENTSIZE %s
+
+PHENTSIZE: Invalid program header size
+
+RUN: not llvm-readobj -dynamic-table \
+RUN:   %p/Inputs/corrupt-invalid-virtual-addr.elf.x86-64 2>&1 | \
+RUN:   FileCheck --check-prefix=VIRTADDR %s
+
+VIRTADDR: Virtual address is not in any segment
+
+
+RUN: not llvm-readobj -dyn-relocations \
+RUN:   %p/Inputs/corrupt-invalid-relocation-size.elf.x86-64 2>&1 | \
+RUN:   FileCheck --check-prefix=RELOC %s
+
+RELOC:  Invalid relocation entry size
+
+RUN: not llvm-readobj -dyn-relocations \
+RUN:   %p/Inputs/corrupt-invalid-dynamic-table-size.elf.x86-64 2>&1 | \
+RUN:   FileCheck --check-prefix=DYN-TABLE-SIZE %s
+
+DYN-TABLE-SIZE:  Invalid dynamic table size
diff --git a/test/Object/dllimport-globalref.ll b/test/Object/dllimport-globalref.ll
new file mode 100644
index 00000000000..dd518bc2266
--- /dev/null
+++ b/test/Object/dllimport-globalref.ll
@@ -0,0 +1,14 @@
+; RUN: llvm-as %s -o - | llvm-nm - | FileCheck %s
+
+; We should technically emit an unmangled reference to f here,
+; but no existing linker needs this.
+
+; XFAIL: *
+
+target datalayout = "e-m:w-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-pc-windows-msvc"
+
+; CHECK: U f
+
+declare dllimport void @f()
+@fp = constant void ()* @f
diff --git a/test/Object/dllimport.ll b/test/Object/dllimport.ll
new file mode 100644
index 00000000000..afdb4562cc9
--- /dev/null
+++ b/test/Object/dllimport.ll
@@ -0,0 +1,17 @@
+; RUN: llvm-as %s -o - | llvm-nm - | FileCheck %s
+
+target datalayout = "e-m:w-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-pc-windows-msvc"
+
+; CHECK: U __imp_f
+; CHECK: U __imp_v
+; CHECK: T g
+
+declare dllimport void @f()
+@v = external dllimport global i32
+
+define void @g() {
+  call void @f()
+  store i32 42, i32* @v
+  ret void
+}
diff --git a/test/Object/extract.ll b/test/Object/extract.ll
deleted file mode 100644
index 8d77f09fe88..00000000000
--- a/test/Object/extract.ll
+++ /dev/null
@@ -1,45 +0,0 @@
-; This isn't really an assembly file, its just here to run the test.
-
-; This test just makes sure that llvm-ar can extract bytecode members
-; from various style archives.
-
-; RUN: cd %T
-
-; RUN: rm -f very_long_bytecode_file_name.bc
-; RUN: llvm-ar p %p/Inputs/GNU.a very_long_bytecode_file_name.bc | \
-; RUN:   cmp -s %p/Inputs/very_long_bytecode_file_name.bc -
-; RUN: llvm-ar x %p/Inputs/GNU.a very_long_bytecode_file_name.bc
-; RUN: cmp -s %p/Inputs/very_long_bytecode_file_name.bc \
-; RUN:        very_long_bytecode_file_name.bc
-
-; RUN: rm -f very_long_bytecode_file_name.bc
-; RUN: llvm-ar p %p/Inputs/MacOSX.a very_long_bytecode_file_name.bc | \
-; RUN:   cmp -s %p/Inputs/very_long_bytecode_file_name.bc -
-; RUN: llvm-ar x %p/Inputs/MacOSX.a very_long_bytecode_file_name.bc
-; RUN: cmp -s %p/Inputs/very_long_bytecode_file_name.bc \
-; RUN:        very_long_bytecode_file_name.bc
-
-; RUN: rm -f very_long_bytecode_file_name.bc
-; RUN: llvm-ar p %p/Inputs/SVR4.a very_long_bytecode_file_name.bc | \
-; RUN:   cmp -s %p/Inputs/very_long_bytecode_file_name.bc -
-; RUN: llvm-ar x %p/Inputs/SVR4.a very_long_bytecode_file_name.bc
-; RUN: cmp -s %p/Inputs/very_long_bytecode_file_name.bc \
-; RUN:        very_long_bytecode_file_name.bc
-
-; RUN: rm -f very_long_bytecode_file_name.bc
-; RUN: llvm-ar p %p/Inputs/xpg4.a very_long_bytecode_file_name.bc |\
-; RUN:   cmp -s %p/Inputs/very_long_bytecode_file_name.bc -
-; RUN: llvm-ar x %p/Inputs/xpg4.a very_long_bytecode_file_name.bc
-; RUN: cmp -s %p/Inputs/very_long_bytecode_file_name.bc \
-; RUN:        very_long_bytecode_file_name.bc
-
-
-; Test that the 'o' option is working by extracting a file, putting it in o
-; new archive and checking that date.
-; RUN: rm -f very_long_bytecode_file_name.bc
-; RUN: llvm-ar xo %p/Inputs/GNU.a very_long_bytecode_file_name.bc
-; RUN: rm -f %t.a
-; RUN: llvm-ar rc %t.a very_long_bytecode_file_name.bc
-; RUN: env TZ=GMT llvm-ar tv %t.a | FileCheck %s
-
-CHECK: 1465 2004-11-19 03:01:31.000000000 very_long_bytecode_file_name.bc
diff --git a/test/Object/invalid.test b/test/Object/invalid.test
new file mode 100644
index 00000000000..a4219ed592d
--- /dev/null
+++ b/test/Object/invalid.test
@@ -0,0 +1,50 @@
+RUN: not llvm-dwarfdump %p/Inputs/invalid-bad-rel-type.elf 2>&1 | FileCheck %s
+RUN: not llvm-objdump -s %p/Inputs/invalid-strtab-type.elf 2>&1 | FileCheck %s
+RUN: not llvm-objdump -s %p/Inputs/invalid-strtab-size.elf 2>&1 | FileCheck %s
+CHECK: Invalid data was encountered while parsing the file
+
+RUN: not llvm-objdump -s %p/Inputs/invalid-strtab-non-null.elf 2>&1 | FileCheck --check-prefix=NON-NULL %s
+NON-NULL: String table must end with a null terminator
+
+Test the sh_entsize are invalid
+RUN: llvm-readobj -s %p/Inputs/invalid-sh_entsize.elf | FileCheck --check-prefix=SECTION %s
+
+SECTION:         Name: .dynsym
+SECTION-NEXT:    Type: SHT_DYNSYM
+SECTION-NEXT:    Flags [
+SECTION-NEXT:      SHF_ALLOC
+SECTION-NEXT:    ]
+SECTION-NEXT:    Address:
+SECTION-NEXT:    Offset:
+SECTION-NEXT:    Size:
+SECTION-NEXT:    Link:
+SECTION-NEXT:    Info:
+SECTION-NEXT:    AddressAlignment:
+SECTION-NEXT:    EntrySize: 32
+
+SECTION:         Name: .symtab
+SECTION-NEXT:    Type: SHT_SYMTAB
+SECTION-NEXT:    Flags [
+SECTION-NEXT:    ]
+SECTION-NEXT:    Address:
+SECTION-NEXT:    Offset:
+SECTION-NEXT:    Size:
+SECTION-NEXT:    Link:
+SECTION-NEXT:    Info:
+SECTION-NEXT:    AddressAlignment:
+SECTION-NEXT:    EntrySize: 32
+
+RUN: not llvm-readobj -t %p/Inputs/invalid-sh_entsize.elf 2>&1 | FileCheck --check-prefix=INVALID-SYM-SIZE %s
+RUN: not llvm-readobj --dyn-symbols %p/Inputs/invalid-sh_entsize.elf 2>&1 | FileCheck --check-prefix=INVALID-SYM-SIZE %s
+INVALID-SYM-SIZE: Invalid symbol size
+
+RUN: not llvm-readobj -t %p/Inputs/invalid-section-index.elf 2>&1 | FileCheck --check-prefix=INVALID-SECTION-INDEX %s
+
+INVALID-SECTION-INDEX: Invalid section index
+
+RUN: not llvm-readobj -s %p/Inputs/invalid-section-size.elf 2>&1 | FileCheck --check-prefix=INVALID-SECTION-SIZE %s
+INVALID-SECTION-SIZE: Invalid section header entry size (e_shentsize) in ELF header
+
+
+RUN: not llvm-readobj -t %p/Inputs/invalid-symbol-table-size.elf 2>&1 | FileCheck --check-prefix=INVALID-SYMTAB-SIZE %s
+INVALID-SYMTAB-SIZE: Invalid symbol table size
diff --git a/test/Object/lit.local.cfg b/test/Object/lit.local.cfg
index d74d039d684..ec8ad451d2d 100644
--- a/test/Object/lit.local.cfg
+++ b/test/Object/lit.local.cfg
@@ -1 +1 @@
-config.suffixes = ['.test', '.ll', '.yaml']
+config.suffixes = ['.test', '.ll', '.s', '.yaml']
diff --git a/test/Object/macho-invalid.test b/test/Object/macho-invalid.test
index ac4bbeb0da1..f4aa1e0c298 100644
--- a/test/Object/macho-invalid.test
+++ b/test/Object/macho-invalid.test
@@ -3,49 +3,40 @@ RUN: llvm-objdump -private-headers %p/Inputs/macho-invalid-zero-ncmds
 
 RUN: not llvm-objdump -private-headers %p/Inputs/macho64-invalid-incomplete-load-command 2>&1 \
 RUN:      | FileCheck -check-prefix INCOMPLETE-LOADC %s
+INCOMPLETE-LOADC: Invalid data was encountered while parsing the file.
 
 RUN: not llvm-objdump -private-headers %p/Inputs/macho-invalid-too-small-load-command 2>&1 \
 RUN:      | FileCheck -check-prefix SMALL-LOADC-SIZE %s
 RUN: not llvm-objdump -private-headers %p/Inputs/macho64-invalid-too-small-load-command 2>&1 \
 RUN:      | FileCheck -check-prefix SMALL-LOADC-SIZE %s
+SMALL-LOADC-SIZE: Mach-O load command with size < 8 bytes
 
 RUN: not llvm-objdump -private-headers %p/Inputs/macho-invalid-too-small-segment-load-command 2>&1 \
 RUN:      | FileCheck -check-prefix SMALL-SEGLOADC-SIZE %s
 RUN: not llvm-objdump -private-headers %p/Inputs/macho64-invalid-too-small-segment-load-command 2>&1 \
 RUN:      | FileCheck -check-prefix SMALL-SEGLOADC-SIZE %s
+SMALL-SEGLOADC-SIZE: Mach-O segment load command size is too small
 
 RUN: not llvm-objdump -private-headers %p/Inputs/macho-invalid-no-size-for-sections 2>&1 \
 RUN:      | FileCheck -check-prefix TOO-MANY-SECTS %s
 RUN: not llvm-objdump -private-headers %p/Inputs/macho64-invalid-no-size-for-sections 2>&1 \
 RUN:      | FileCheck -check-prefix TOO-MANY-SECTS %s
+TOO-MANY-SECTS: Mach-O segment load command contains too many sections
 
 RUN: not llvm-objdump -t %p/Inputs/macho-invalid-bad-symbol-index 2>&1 \
 RUN:      | FileCheck -check-prefix BAD-SYMBOL %s
+BAD-SYMBOL: Requested symbol index is out of range
 
 RUN: not llvm-objdump -t %p/Inputs/macho-invalid-symbol-name-past-eof 2>&1 \
 RUN:      | FileCheck -check-prefix NAME-PAST-EOF %s
-
-RUN: not llvm-objdump -t %p/Inputs/macho-invalid-section-index-getSectionRawFinalSegmentName 2>&1 \
-RUN:      | FileCheck -check-prefix INVALID-SECTION-IDX-SEG-NAME %s
+NAME-PAST-EOF: Symbol name entry points before beginning or past end of file
 
 RUN: not llvm-nm %p/Inputs/macho-invalid-section-index-getSectionRawName 2>&1 \
-RUN:      | FileCheck -check-prefix INVALID-SECTION-IDX-SECT-NAME %s
-
-RUN: not llvm-objdump -t %p/Inputs/macho-invalid-getsection-index 2>&1 \
-RUN:      | FileCheck -check-prefix INVALID-SECTION-IDX-GETSECT %s
+RUN:      | FileCheck -check-prefix INVALID-SECTION-IDX-SYMBOL-SEC %s
+INVALID-SECTION-IDX-SYMBOL-SEC: getSymbolSection: Invalid section index
 
-RUN: not llvm-objdump -t %p/Inputs/macho64-invalid-getsection-index 2>&1 \
-RUN:      | FileCheck -check-prefix INVALID-SECTION-IDX-GETSECT64 %s
-
-
-SMALL-LOADC-SIZE: Load command with size < 8 bytes
-SMALL-SEGLOADC-SIZE: Segment load command size is too small
-INCOMPLETE-LOADC: Malformed MachO file
-TOO-MANY-SECTS: Number of sections too large for size of load command
-BAD-SYMBOL: Requested symbol index is out of range
-NAME-PAST-EOF: Symbol name entry points before beginning or past end of file
+RUN: not llvm-objdump -private-headers %p/Inputs/macho-invalid-header 2>&1 | FileCheck -check-prefix INVALID-HEADER %s
+INVALID-HEADER: Invalid data was encountered while parsing the file
 
-INVALID-SECTION-IDX-SEG-NAME: getSectionRawFinalSegmentName: Invalid section index
-INVALID-SECTION-IDX-SECT-NAME: getSectionRawName: Invalid section index
-INVALID-SECTION-IDX-GETSECT: getSection: Invalid section index
-INVALID-SECTION-IDX-GETSECT64: getSection64: Invalid section index
+RUN: not llvm-objdump -private-headers %p/Inputs/macho64-invalid-incomplete-segment-load-command 2>&1 | FileCheck -check-prefix INCOMPLETE-SEGMENT-LOADC %s
+INCOMPLETE-SEGMENT-LOADC: Invalid data was encountered while parsing the file
diff --git a/test/Object/nm-archive.test b/test/Object/nm-archive.test
index a9ae9cbbfbd..8f8cdbe9f08 100644
--- a/test/Object/nm-archive.test
+++ b/test/Object/nm-archive.test
@@ -24,6 +24,15 @@ RUN: rm -f %t2
 RUN: llvm-ar rcs %t2 %t1
 RUN: llvm-nm %t2 | FileCheck %s -check-prefix BITCODE
 
+RUN: rm -f %t2
+RUN: llvm-lib /out:%t2 %t1
+RUN: llvm-nm %t2 | FileCheck %s -check-prefix BITCODE
+
+RUN: rm -f %t2
+RUN: echo /out:%t2 %t1 > %t.rsp
+RUN: llvm-lib @%t.rsp
+RUN: llvm-nm %t2 | FileCheck %s -check-prefix BITCODE
+
 BITCODE:          U SomeOtherFunction
 BITCODE-NEXT:          T main
 BITCODE-NEXT:          U puts
@@ -49,3 +58,12 @@ GNU AR is able to parse the unaligned member and warns about the member with
 the unknown format. We should probably simply warn on both. For now just check
 that we don't produce an error.
 RUN: llvm-nm %p/Inputs/corrupt-archive.a
+
+
+RUN: llvm-nm %p/Inputs/thin.a | FileCheck %s -check-prefix THIN
+
+THIN: IsNAN.o:
+THIN: 00000014 T _ZN4llvm5IsNANEd
+THIN: 00000000 T _ZN4llvm5IsNANEf
+THIN:          U __isnan
+THIN:          U __isnanf
diff --git a/test/Object/nm-pe-image.test b/test/Object/nm-pe-image.test
new file mode 100644
index 00000000000..f95aeebf727
--- /dev/null
+++ b/test/Object/nm-pe-image.test
@@ -0,0 +1,31 @@
+The executable was generated like so:
+$ cat t.c
+int main() { return 0; }
+$ clang --target=i686-windows -c t.c -o t.o
+$ lld-link2 t.o -out:t.exe -entry:main -debug
+
+It has a mingw-style symbol table in the executable, which MSVC-produced images
+don't have.
+
+RUN: llvm-nm %p/Inputs/main-ret-zero-pe-i386.exe \
+RUN:         | FileCheck %s -check-prefix PE-EXE
+
+PE-EXE: 00401000 t .bss
+PE-EXE: 00401000 t .data
+PE-EXE: 00401000 t .text
+PE-EXE: 00401000 T _main
+
+The DLL was generated like so:
+$ cat t.c
+int DllMain(void *mod, long reason, void *reserved) { return 1; }
+$ clang --target=i686-windows -c t.c -o t.o
+$ lld-link2 t.o -out:t.exe -entry:DllMain -debug -dll
+
+
+RUN: llvm-nm %p/Inputs/main-ret-zero-pe-i386.dll \
+RUN:         | FileCheck %s -check-prefix PE-DLL
+
+PE-DLL: 10001000 t .bss
+PE-DLL: 10001000 t .data
+PE-DLL: 10001000 t .text
+PE-DLL: 10001000 T _DllMain
diff --git a/test/Object/nm-trivial-object.test b/test/Object/nm-trivial-object.test
index c576c236fc0..def6549d656 100644
--- a/test/Object/nm-trivial-object.test
+++ b/test/Object/nm-trivial-object.test
@@ -1,9 +1,11 @@
 RUN: yaml2obj %p/Inputs/COFF/i386.yaml | llvm-nm -a -S - \
-RUN:         | FileCheck %s -check-prefix COFF
+RUN:         | FileCheck %s -check-prefix COFF32
 RUN: yaml2obj %p/Inputs/COFF/x86-64.yaml | llvm-nm -a -S - \
-RUN:         | FileCheck %s -check-prefix COFF
+RUN:         | FileCheck %s -check-prefix COFF64
 RUN: llvm-nm %p/Inputs/trivial-object-test.elf-i386 \
 RUN:         | FileCheck %s -check-prefix ELF
+RUN: llvm-nm %p/Inputs/trivial-object-test.elf-i386 -S \
+RUN:         | FileCheck %s -check-prefix ELF-SIZE
 RUN: llvm-nm -o %p/Inputs/trivial-object-test.elf-i386 \
 RUN:         | FileCheck %s -check-prefix ELF-o
 RUN: llvm-nm -u %p/Inputs/trivial-object-test.elf-i386 \
@@ -48,20 +50,28 @@ RUN: cp %p/Inputs/trivial-object-test.macho-i386 a.out
 RUN: llvm-nm | FileCheck %s -check-prefix A-OUT
 
 
-COFF: 00000000 {{.*}} d .data
-COFF: 00000000 {{.*}} t .text
-COFF: 00000000 0000000d d L{{_?}}.str
-COFF:          U {{_?}}SomeOtherFunction
-COFF: 00000000 {{.*}} T {{_?}}main
-COFF:          U {{_?}}puts
+COFF32: 00000000 d .data
+COFF32-NEXT: 00000000 t .text
+COFF32-NEXT: 00000000 d L_.str
+COFF32-NEXT:          U _SomeOtherFunction
+COFF32-NEXT: 00000000 T _main
+COFF32-NEXT:          U _puts
 
-COFF-COMMON: 00000000 00000000 b .bss
-COFF-COMMON-NEXT: 00000000 00000000 d .data
-COFF-COMMON-NEXT: 00000000 00000014 d .drectve
-COFF-COMMON-NEXT: 00000000 00000000 n .file
-COFF-COMMON-NEXT: 00000000 00000014 r .rdata$zzz
-COFF-COMMON-NEXT: 00000000 00000000 t .text
-COFF-COMMON-NEXT:          00000004 C _a
+COFF64: 00000000 d .data
+COFF64-NEXT: 00000000 t .text
+COFF64-NEXT: 00000000 r ??__Ex@@YAXXZ
+COFF64-NEXT: 00000000 d L.str
+COFF64-NEXT:          U SomeOtherFunction
+COFF64-NEXT: 00000000 T main
+COFF64-NEXT:          U puts
+
+COFF-COMMON: 00000000 b .bss
+COFF-COMMON-NEXT: 00000000 d .data
+COFF-COMMON-NEXT: 00000000 d .drectve
+COFF-COMMON-NEXT: 00000000 n .file
+COFF-COMMON-NEXT: 00000000 r .rdata$zzz
+COFF-COMMON-NEXT: 00000000 t .text
+COFF-COMMON-NEXT:          C _a
 
 
 ELF-NOT:      U
@@ -69,6 +79,11 @@ ELF:          U SomeOtherFunction
 ELF: 00000000 T main
 ELF:          U puts
 
+FIXME: we should not print the size of undefined symbols.
+ELF-SIZE:               00000000 U SomeOtherFunction
+ELF-SIZE-NEXT: 00000000 00000024 T main
+ELF-SIZE-NEXT:          00000000 U puts
+
 ELF-o: {{.*}}/trivial-object-test.elf-i386:          U SomeOtherFunction
 ELF-o: {{.*}}/trivial-object-test.elf-i386: 00000000 T main
 ELF-o: {{.*}}/trivial-object-test.elf-i386:          U puts
diff --git a/test/Object/no-section-header-string-table.test b/test/Object/no-section-header-string-table.test
new file mode 100644
index 00000000000..1f449328cab
--- /dev/null
+++ b/test/Object/no-section-header-string-table.test
@@ -0,0 +1,10 @@
+RUN: llvm-readobj %p/Inputs/no-section-header-string-table.elf-x86-64 --sections \
+RUN:   | FileCheck %s
+
+CHECK:    Type: SHT_PROGBITS (0x1)
+CHECK:    Type: SHT_PROGBITS (0x1)
+CHECK:    Type: SHT_PROGBITS (0x1)
+CHECK:    Type: SHT_RELA (0x4)
+CHECK:    Type: SHT_SYMTAB (0x2)
+CHECK:    Type: SHT_STRTAB (0x3)
+CHECK:    Type: SHT_STRTAB (0x3)
diff --git a/test/Object/no-section-table.test b/test/Object/no-section-table.test
new file mode 100644
index 00000000000..77fb98a4d40
--- /dev/null
+++ b/test/Object/no-section-table.test
@@ -0,0 +1,36 @@
+RUN: llvm-readobj %p/Inputs/no-section-table.so -hash-table -dynamic-table \
+RUN:   | FileCheck %s
+
+CHECK: DynamicSection [ (24 entries)
+CHECK:   Tag                Type                 Name/Value
+CHECK:   0x0000000000000001 NEEDED               SharedLibrary (libc.so.6)
+CHECK:   0x000000000000000C INIT                 0x4B8
+CHECK:   0x000000000000000D FINI                 0x618
+CHECK:   0x0000000000000019 INIT_ARRAY           0x2006C0
+CHECK:   0x000000000000001B INIT_ARRAYSZ         8 (bytes)
+CHECK:   0x000000000000001A FINI_ARRAY           0x2006C8
+CHECK:   0x000000000000001C FINI_ARRAYSZ         8 (bytes)
+CHECK:   0x0000000000000004 HASH                 0x158
+CHECK:   0x0000000000000005 STRTAB               0x2D8
+CHECK:   0x0000000000000006 SYMTAB               0x1A0
+CHECK:   0x000000000000000A STRSZ                179 (bytes)
+CHECK:   0x000000000000000B SYMENT               24 (bytes)
+CHECK:   0x0000000000000003 PLTGOT               0x2008C0
+CHECK:   0x0000000000000002 PLTRELSZ             48 (bytes)
+CHECK:   0x0000000000000014 PLTREL               RELA
+CHECK:   0x0000000000000017 JMPREL               0x488
+CHECK:   0x0000000000000007 RELA                 0x3C8
+CHECK:   0x0000000000000008 RELASZ               192 (bytes)
+CHECK:   0x0000000000000009 RELAENT              24 (bytes)
+CHECK:   0x000000006FFFFFFE VERNEED              0x3A8
+CHECK:   0x000000006FFFFFFF VERNEEDNUM           1
+CHECK:   0x000000006FFFFFF0 VERSYM               0x38C
+CHECK:   0x000000006FFFFFF9 unknown              0x3
+CHECK:   0x0000000000000000 NULL                 0x0
+CHECK: ]
+CHECK: HashTable {
+CHECK:   Num Buckets: 3
+CHECK:   Num Chains: 13
+CHECK:   Buckets: [12, 10, 11]
+CHECK:   Chains: [0, 0, 0, 0, 2, 3, 4, 0, 7, 5, 6, 8, 9]
+CHECK: }
diff --git a/test/Object/obj2yaml.test b/test/Object/obj2yaml.test
index e654dcdccf5..8054b23eb56 100644
--- a/test/Object/obj2yaml.test
+++ b/test/Object/obj2yaml.test
@@ -234,7 +234,7 @@ ELF-MIPSEL-NEXT:   - Name:            .bss
 ELF-MIPSEL-NEXT:     Type:            SHT_NOBITS
 ELF-MIPSEL-NEXT:     Flags:           [ SHF_WRITE, SHF_ALLOC ]
 ELF-MIPSEL-NEXT:     AddressAlign:    0x0000000000000004
-ELF-MIPSEL-NEXT:     Content:         48656C6C
+ELF-MIPSEL-NEXT:     Size:            0x0000000000000004
 ELF-MIPSEL-NEXT:   - Name:            .mdebug.abi32
 ELF-MIPSEL-NEXT:     Type:            SHT_PROGBITS
 ELF-MIPSEL-NEXT:     AddressAlign:    0x0000000000000001
@@ -253,7 +253,12 @@ ELF-MIPSEL-NEXT:   - Name:            .MIPS.abiflags
 ELF-MIPSEL-NEXT:     Type:            SHT_MIPS_ABIFLAGS
 ELF-MIPSEL-NEXT:     Flags:           [ SHF_ALLOC ]
 ELF-MIPSEL-NEXT:     AddressAlign:    0x0000000000000008
-ELF-MIPSEL-NEXT:     Content:         '000020010101000100000000000000000100000000000000'
+ELF-MIPSEL-NEXT:     ISA:             MIPS32
+ELF-MIPSEL-NEXT:     ISARevision:     0x01
+ELF-MIPSEL-NEXT:     FpABI:           FP_DOUBLE
+ELF-MIPSEL-NEXT:     GPRSize:         REG_32
+ELF-MIPSEL-NEXT:     CPR1Size:        REG_32
+ELF-MIPSEL-NEXT:     Flags1:          [ ODDSPREG ]
 ELF-MIPSEL-NEXT: Symbols:
 ELF-MIPSEL-NEXT:   Local:
 ELF-MIPSEL-NEXT:     - Name:            trivial.ll
@@ -262,26 +267,19 @@ ELF-MIPSEL-NEXT:     - Name:            '$.str'
 ELF-MIPSEL-NEXT:       Type:            STT_OBJECT
 ELF-MIPSEL-NEXT:       Section:         .rodata.str1.1
 ELF-MIPSEL-NEXT:       Size:            0x000000000000000D
-ELF-MIPSEL-NEXT:     - Name:            .text
-ELF-MIPSEL-NEXT:       Type:            STT_SECTION
+ELF-MIPSEL-NEXT:     - Type:            STT_SECTION
 ELF-MIPSEL-NEXT:       Section:         .text
-ELF-MIPSEL-NEXT:     - Name:            .data
-ELF-MIPSEL-NEXT:       Type:            STT_SECTION
+ELF-MIPSEL-NEXT:     - Type:            STT_SECTION
 ELF-MIPSEL-NEXT:       Section:         .data
-ELF-MIPSEL-NEXT:     - Name:            .bss
-ELF-MIPSEL-NEXT:       Type:            STT_SECTION
+ELF-MIPSEL-NEXT:     - Type:            STT_SECTION
 ELF-MIPSEL-NEXT:       Section:         .bss
-ELF-MIPSEL-NEXT:     - Name:            .mdebug.abi32
-ELF-MIPSEL-NEXT:       Type:            STT_SECTION
+ELF-MIPSEL-NEXT:     - Type:            STT_SECTION
 ELF-MIPSEL-NEXT:       Section:         .mdebug.abi32
-ELF-MIPSEL-NEXT:     - Name:            .rodata.str1.1
-ELF-MIPSEL-NEXT:       Type:            STT_SECTION
+ELF-MIPSEL-NEXT:     - Type:            STT_SECTION
 ELF-MIPSEL-NEXT:       Section:         .rodata.str1.1
-ELF-MIPSEL-NEXT:     - Name:            .reginfo
-ELF-MIPSEL-NEXT:       Type:            STT_SECTION
+ELF-MIPSEL-NEXT:     - Type:            STT_SECTION
 ELF-MIPSEL-NEXT:       Section:         .reginfo
-ELF-MIPSEL-NEXT:     - Name:            .MIPS.abiflags
-ELF-MIPSEL-NEXT:       Type:            STT_SECTION
+ELF-MIPSEL-NEXT:     - Type:            STT_SECTION
 ELF-MIPSEL-NEXT:       Section:         .MIPS.abiflags
 ELF-MIPSEL-NEXT:   Global:
 ELF-MIPSEL-NEXT:     - Name:            main
@@ -326,7 +324,6 @@ ELF-MIPS64EL-NEXT:   - Name:            .bss
 ELF-MIPS64EL-NEXT:     Type:            SHT_NOBITS
 ELF-MIPS64EL-NEXT:     Flags:           [ SHF_WRITE, SHF_ALLOC ]
 ELF-MIPS64EL-NEXT:     AddressAlign:    0x0000000000000010
-ELF-MIPS64EL-NEXT:     Content:         ''
 ELF-MIPS64EL-NEXT:   - Name:            .MIPS.options
 ELF-MIPS64EL-NEXT:     Type:            SHT_MIPS_OPTIONS
 ELF-MIPS64EL-NEXT:     Flags:           [ SHF_ALLOC ]
@@ -338,22 +335,17 @@ ELF-MIPS64EL-NEXT:     AddressAlign:    0x0000000000000004
 ELF-MIPS64EL-NEXT:     Content:         ''
 ELF-MIPS64EL-NEXT: Symbols:
 ELF-MIPS64EL-NEXT:   Local:
-ELF-MIPS64EL-NEXT:     - Name:            .text
-ELF-MIPS64EL-NEXT:       Type:            STT_SECTION
+ELF-MIPS64EL-NEXT:     - Type:            STT_SECTION
 ELF-MIPS64EL-NEXT:       Section:         .text
-ELF-MIPS64EL-NEXT:     - Name:            .data
-ELF-MIPS64EL-NEXT:       Type:            STT_SECTION
+ELF-MIPS64EL-NEXT:     - Type:            STT_SECTION
 ELF-MIPS64EL-NEXT:       Section:         .data
-ELF-MIPS64EL-NEXT:     - Name:            .bss
-ELF-MIPS64EL-NEXT:       Type:            STT_SECTION
+ELF-MIPS64EL-NEXT:     - Type:            STT_SECTION
 ELF-MIPS64EL-NEXT:       Section:         .bss
 ELF-MIPS64EL-NEXT:     - Name:            bar
 ELF-MIPS64EL-NEXT:       Section:         .data
-ELF-MIPS64EL-NEXT:     - Name:            .MIPS.options
-ELF-MIPS64EL-NEXT:       Type:            STT_SECTION
+ELF-MIPS64EL-NEXT:     - Type:            STT_SECTION
 ELF-MIPS64EL-NEXT:       Section:         .MIPS.options
-ELF-MIPS64EL-NEXT:     - Name:            .pdr
-ELF-MIPS64EL-NEXT:       Type:            STT_SECTION
+ELF-MIPS64EL-NEXT:     - Type:            STT_SECTION
 ELF-MIPS64EL-NEXT:       Section:         .pdr
 ELF-MIPS64EL-NEXT:   Global:
 ELF-MIPS64EL-NEXT:     - Name:            zed
@@ -389,7 +381,7 @@ ELF-X86-64-NEXT:     AddressAlign:    0x0000000000000008
 ELF-X86-64-NEXT:     Info:            .text
 ELF-X86-64-NEXT:     Relocations:
 ELF-X86-64-NEXT:       - Offset:          0x000000000000000D
-ELF-X86-64-NEXT:         Symbol:          .rodata.str1.1
+ELF-X86-64-NEXT:         Symbol:          ''
 ELF-X86-64-NEXT:         Type:            R_X86_64_32S
 ELF-X86-64-NEXT:       - Offset:          0x0000000000000012
 ELF-X86-64-NEXT:         Symbol:          puts
@@ -403,14 +395,11 @@ ELF-X86-64-NEXT: Symbols:
 ELF-X86-64-NEXT:   Local:
 ELF-X86-64-NEXT:     - Name:            trivial-object-test.s
 ELF-X86-64-NEXT:       Type:            STT_FILE
-ELF-X86-64-NEXT:     - Name:            .text
-ELF-X86-64-NEXT:       Type:            STT_SECTION
+ELF-X86-64-NEXT:     - Type:            STT_SECTION
 ELF-X86-64-NEXT:       Section:         .text
-ELF-X86-64-NEXT:     - Name:            .rodata.str1.1
-ELF-X86-64-NEXT:       Type:            STT_SECTION
+ELF-X86-64-NEXT:     - Type:            STT_SECTION
 ELF-X86-64-NEXT:       Section:         .rodata.str1.1
-ELF-X86-64-NEXT:     - Name:            .note.GNU-stack
-ELF-X86-64-NEXT:       Type:            STT_SECTION
+ELF-X86-64-NEXT:     - Type:            STT_SECTION
 ELF-X86-64-NEXT:       Section:         .note.GNU-stack
 ELF-X86-64-NEXT:   Global:
 ELF-X86-64-NEXT:     - Name:            main
diff --git a/test/Object/objdump-symbol-table.test b/test/Object/objdump-symbol-table.test
index c94b0777355..e66faecf4cc 100644
--- a/test/Object/objdump-symbol-table.test
+++ b/test/Object/objdump-symbol-table.test
@@ -30,12 +30,14 @@ ELF-i386: 00000000         *UND*  00000000 puts
 
 macho-i386: trivial-object-test.macho-i386:        file format Mach-O 32-bit i386
 macho-i386: SYMBOL TABLE:
-macho-i386: 00000000 g     F __TEXT,__text  00000024 _main
-macho-i386: 00000000         *UND*  00000000 _SomeOtherFunction
-macho-i386: 00000000         *UND*  00000000 _puts
+macho-i386: 00000000 g     F __TEXT,__text  _main
+macho-i386: 00000000         *UND*          _SomeOtherFunction
+macho-i386: 00000000         *UND*          _puts
 
 ELF-shared: shared-object-test.elf-i386:     file format
 ELF-shared: SYMBOL TABLE:
 ELF-shared: 00000200 l     F .text 00000003 local_func
+ELF-shared: 000012b8 l       *ABS* 00000000 .hidden _GLOBAL_OFFSET_TABLE_
+ELF-shared: 00001248 l       *ABS* 00000000 .hidden _DYNAMIC
 ELF-shared: 000012c4 g       .data 00000004 defined_sym
 ELF-shared: 000001f0 g     F .text 00000003 global_func
diff --git a/test/Object/readobj-shared-object.test b/test/Object/readobj-shared-object.test
index 516d4c699e4..508caca9717 100644
--- a/test/Object/readobj-shared-object.test
+++ b/test/Object/readobj-shared-object.test
@@ -128,61 +128,61 @@ ELF: ]
 
 ELF: Symbols [
 ELF:   Symbol {
-ELF:     Name: .hash
+ELF:     Name: (0)
 ELF:     Binding: Local
 ELF:     Type: Section
 ELF:     Section: .hash
 ELF:   }
 ELF:   Symbol {
-ELF:     Name: .dynsym
+ELF:     Name: (0)
 ELF:     Binding: Local
 ELF:     Type: Section
 ELF:     Section: .dynsym
 ELF:   }
 ELF:   Symbol {
-ELF:     Name: .dynstr
+ELF:     Name: (0)
 ELF:     Binding: Local
 ELF:     Type: Section
 ELF:     Section: .dynstr
 ELF:   }
 ELF:   Symbol {
-ELF:     Name: .text
+ELF:     Name: (0)
 ELF:     Binding: Local
 ELF:     Type: Section
 ELF:     Section: .text
 ELF:   }
 ELF:   Symbol {
-ELF:     Name: .eh_frame
+ELF:     Name: (0)
 ELF:     Binding: Local
 ELF:     Type: Section
 ELF:     Section: .eh_frame
 ELF:   }
 ELF:   Symbol {
-ELF:     Name: .tdata
+ELF:     Name: (0)
 ELF:     Binding: Local
 ELF:     Type: Section
 ELF:     Section: .tdata
 ELF:   }
 ELF:   Symbol {
-ELF:     Name: .dynamic
+ELF:     Name: (0)
 ELF:     Binding: Local
 ELF:     Type: Section
 ELF:     Section: .dynamic
 ELF:   }
 ELF:   Symbol {
-ELF:     Name: .got.plt
+ELF:     Name: (0)
 ELF:     Binding: Local
 ELF:     Type: Section
 ELF:     Section: .got.plt
 ELF:   }
 ELF:   Symbol {
-ELF:     Name: .data
+ELF:     Name: (0)
 ELF:     Binding: Local
 ELF:     Type: Section
 ELF:     Section: .data
 ELF:   }
 ELF:   Symbol {
-ELF:     Name: .bss
+ELF:     Name: (0)
 ELF:     Binding: Local
 ELF:     Type: Section
 ELF:     Section: .bss
diff --git a/test/Object/relocation-executable.test b/test/Object/relocation-executable.test
index 1236035d9f6..93d4dee3089 100644
--- a/test/Object/relocation-executable.test
+++ b/test/Object/relocation-executable.test
@@ -1,5 +1,7 @@
 RUN: llvm-readobj -r -expand-relocs %p/Inputs/hello-world.elf-x86-64 \
 RUN:   | FileCheck %s
+RUN: llvm-readobj -dyn-relocations -expand-relocs \
+RUN:   %p/Inputs/hello-world.elf-x86-64 | FileCheck %s --check-prefix=DYN
 
 // CHECK:     Relocations [
 // CHECK-NEXT:  Section (8) .rela.dyn {
@@ -24,3 +26,24 @@ RUN:   | FileCheck %s
 // CHECK-NEXT:      Addend: 0x0
 // CHECK-NEXT:    }
 // CHECK-NEXT:  }
+
+// DYN:      Dynamic Relocations {
+// DYN-NEXT:   Relocation {
+// DYN-NEXT:     Offset: 0x4018D8
+// DYN-NEXT:     Type: R_X86_64_GLOB_DAT (6)
+// DYN-NEXT:     Symbol: __gmon_start__
+// DYN-NEXT:     Addend: 0x0
+// DYN-NEXT:   }
+// DYN-NEXT: }
+
+RUN: llvm-readobj -dyn-relocations -expand-relocs \
+RUN:   %p/Inputs/rel-no-sec-table.elf-x86-64 | FileCheck %s --check-prefix=DYN2
+
+// DYN2:      Dynamic Relocations {
+// DYN2-NEXT:   Relocation {
+// DYN2-NEXT:     Offset: 0x12F0
+// DYN2-NEXT:     Type: R_X86_64_GLOB_DAT (6)
+// DYN2-NEXT:     Symbol: g
+// DYN2-NEXT:     Addend: 0x0
+// DYN2-NEXT:   }
+// DYN2-NEXT: }
diff --git a/test/Object/stackmap-dump.test b/test/Object/stackmap-dump.test
new file mode 100644
index 00000000000..71710fb6194
--- /dev/null
+++ b/test/Object/stackmap-dump.test
@@ -0,0 +1,16 @@
+RUN: llvm-readobj -stackmap %p/Inputs/stackmap-test.macho-x86-64 | FileCheck %s
+
+CHECK:      LLVM StackMap Version: 1
+CHECK-NEXT: Num Functions: 1
+CHECK-NEXT:   Function address: 0, stack size: 16
+CHECK-NEXT: Num Constants: 1
+CHECK-NEXT:   #1: 10000000000
+CHECK-NEXT: Num Records: 1
+CHECK-NEXT:   Record ID: 2, instruction offset: 1
+CHECK-NEXT:     5 locations:
+CHECK-NEXT:       #1: Register R#5
+CHECK-NEXT:       #2: Constant 10
+CHECK-NEXT:       #3: ConstantIndex #0 (10000000000)
+CHECK-NEXT:       #4: Direct R#4 + -8
+CHECK-NEXT:       #5: Indirect [R#6 + -16]
+CHECK-NEXT:     1 live-outs: [ R#7 (8-bytes) ]
diff --git a/test/Object/yaml2obj-elf-alignment.yaml b/test/Object/yaml2obj-elf-alignment.yaml
new file mode 100644
index 00000000000..8f2f985177f
--- /dev/null
+++ b/test/Object/yaml2obj-elf-alignment.yaml
@@ -0,0 +1,53 @@
+# Check that yaml2obj takes in account section AddressAlign field.
+
+# RUN: yaml2obj -format=elf %s > %t
+# RUN: llvm-readobj -s %t | FileCheck %s
+
+# CHECK:      Section {
+# CHECK:        Index: 2
+# CHECK-NEXT:   Name: .data
+# CHECK-NEXT:   Type: SHT_PROGBITS
+# CHECK-NEXT:   Flags [
+# CHECK-NEXT:     SHF_ALLOC
+# CHECK-NEXT:     SHF_WRITE
+# CHECK-NEXT:   ]
+# CHECK-NEXT:   Address: 0x0
+# CHECK-NEXT:   Offset: 0x{{[0-9A-F]*}}00
+# CHECK-NEXT:   Size: 4
+# CHECK-NEXT:   Link: 0
+# CHECK-NEXT:   Info: 0
+# CHECK-NEXT:   AddressAlignment: 256
+# CHECK-NEXT:   EntrySize: 0
+# CHECK-NEXT: }
+
+---
+FileHeader:
+  Class:    ELFCLASS32
+  Data:     ELFDATA2LSB
+  Type:     ET_REL
+  Machine:  EM_MIPS
+  Flags:    [ EF_MIPS_CPIC, EF_MIPS_ABI_O32, EF_MIPS_ARCH_32 ]
+
+Sections:
+  - Name:          .text
+    Type:          SHT_PROGBITS
+    Flags:         [ SHF_ALLOC, SHF_EXECINSTR ]
+    AddressAlign:  8
+    Size:          4
+  - Name:          .data
+    Type:          SHT_PROGBITS
+    Flags:         [ SHF_WRITE, SHF_ALLOC ]
+    AddressAlign:  256
+    Size:          4
+
+Symbols:
+  Global:
+    - Name:     T0
+      Type:     STT_FUNC
+      Section:  .text
+      Size:     4
+    - Name:     D0
+      Type:     STT_OBJECT
+      Section:  .data
+      Size:     4
+...
diff --git a/test/Object/yaml2obj-elf-rel-noref.yaml b/test/Object/yaml2obj-elf-rel-noref.yaml
new file mode 100644
index 00000000000..4a13acd1fd3
--- /dev/null
+++ b/test/Object/yaml2obj-elf-rel-noref.yaml
@@ -0,0 +1,77 @@
+# RUN: yaml2obj -format=elf %s | llvm-readobj -sections -relocations - | FileCheck %s
+
+!ELF
+FileHeader:        !FielHeader
+  Class:           ELFCLASS32
+  Data:            ELFDATA2LSB
+  Type:            ET_REL
+  Machine:         EM_ARM
+  Flags:           [ EF_ARM_EABI_VER5 ]
+Sections:        
+  - Name:            .text
+    Type:            SHT_PROGBITS
+    Flags:           [ SHF_ALLOC, SHF_EXECINSTR ]
+    AddressAlign:    0x0000000000000004
+    Content:         03308FE0022093E7000052E31EFF2F01
+  - Name:            .rel.text
+    Type:            SHT_REL
+    Flags:           [ SHF_INFO_LINK ]
+    Link:            .symtab
+    AddressAlign:    0x0000000000000004
+    Info:            .text
+    Relocations:     
+      - Offset:          0x000000000000000C
+        Symbol:          ''
+        Type:            R_ARM_V4BX
+  - Name:            .data
+    Type:            SHT_PROGBITS
+    Flags:           [ SHF_WRITE, SHF_ALLOC ]
+    AddressAlign:    0x0000000000000001
+    Content:         ''
+  - Name:            .bss
+    Type:            SHT_NOBITS
+    Flags:           [ SHF_WRITE, SHF_ALLOC ]
+    AddressAlign:    0x0000000000000001
+    Size:            0
+  - Name:            .ARM.attributes
+    Type:            SHT_ARM_ATTRIBUTES
+    AddressAlign:    0x0000000000000001
+    Content:         4115000000616561626900010B000000060208010901
+Symbols:         
+  Local:           
+    - Name:            .text
+      Type:            STT_SECTION
+      Section:         .text
+    - Name:            .data
+      Type:            STT_SECTION
+      Section:         .data
+    - Name:            .bss
+      Type:            STT_SECTION
+      Section:         .bss
+    - Name:            '$a'
+      Section:         .text
+    - Name:            call_weak_fn
+      Type:            STT_FUNC
+      Section:         .text
+    - Name:            .ARM.attributes
+      Type:            STT_SECTION
+      Section:         .ARM.attributes
+
+# CHECK:        Section {
+# CHECK-NEXT:     Index: 0
+# CHECK:        }
+# CHECK:        Section {
+# CHECK-NEXT:     Index: 1
+# CHECK-NEXT:     Name: .text (5)
+# CHECK:        }
+# CHECK-NEXT:   Section {
+# CHECK-NEXT:     Index: 2
+# CHECK-NEXT:     Name: .rel.text (1)
+# CHECK-NEXT:     Type: SHT_REL (0x9)
+# CHECK-NEXT:     Flags [ (0x40)
+# CHECK:	}
+# CHECK:        Relocations [
+# CHECK-NEXT:     Section (2) .rel.text {
+# CHECK-NEXT:       0xC R_ARM_V4BX - 0x0
+# CHECK-NEXT:     }
+# CHECK-NEXT:   ]
diff --git a/test/Object/yaml2obj-elf-rel.yaml b/test/Object/yaml2obj-elf-rel.yaml
index 6a7ed459eff..ba3640cfefa 100644
--- a/test/Object/yaml2obj-elf-rel.yaml
+++ b/test/Object/yaml2obj-elf-rel.yaml
@@ -75,7 +75,7 @@ Symbols:
 # CHECK-NEXT:     Flags [ (0x0)
 # CHECK-NEXT:     ]
 # CHECK-NEXT:     Address: 0x0
-# CHECK-NEXT:     Offset: 0x160
+# CHECK-NEXT:     Offset: 0x{{[0-9A-F]+}}
 # CHECK-NEXT:     Size: 24
 # CHECK-NEXT:     Link: 4
 # CHECK-NEXT:     Info: 1
@@ -89,7 +89,7 @@ Symbols:
 # CHECK-NEXT:     Flags [ (0x0)
 # CHECK-NEXT:     ]
 # CHECK-NEXT:     Address: 0x0
-# CHECK-NEXT:     Offset: 0x180
+# CHECK-NEXT:     Offset: 0x{{[0-9A-F]+}}
 # CHECK-NEXT:     Size: 36
 # CHECK-NEXT:     Link: 4
 # CHECK-NEXT:     Info: 1
diff --git a/test/Object/yaml2obj-elf-section-basic.yaml b/test/Object/yaml2obj-elf-section-basic.yaml
index 56a3fd6e5f1..69d3ae91b71 100644
--- a/test/Object/yaml2obj-elf-section-basic.yaml
+++ b/test/Object/yaml2obj-elf-section-basic.yaml
@@ -52,7 +52,7 @@ Sections:
 # CHECK-NEXT:       SHF_ALLOC (0x2)
 # CHECK-NEXT:     ]
 # CHECK-NEXT:     Address: 0xCAFECAFE
-# CHECK-NEXT:     Offset: 0x1D0
+# CHECK-NEXT:     Offset: 0x{{[0-9A-F]+}}
 # CHECK-NEXT:     Size: 8
 # CHECK-NEXT:     Link: 0
 # CHECK-NEXT:     Info: 0
@@ -65,12 +65,15 @@ Sections:
 # CHECK:        Section {
 # CHECK:          Name: .symtab (25)
 # CHECK:          Type: SHT_SYMTAB (0x2)
+# CHECK:          AddressAlignment: 8
 # CHECK:        }
 # CHECK:        Section {
 # CHECK:          Name: .strtab (17)
 # CHECK:          Type: SHT_STRTAB (0x3)
+# CHECK:          AddressAlignment: 1
 # CHECK:        }
 # CHECK:        Section {
 # CHECK:          Name: .shstrtab (7)
 # CHECK:          Type: SHT_STRTAB (0x3)
+# CHECK:          AddressAlignment: 1
 # CHECK:        }
diff --git a/test/Object/yaml2obj-elf-symbol-basic.yaml b/test/Object/yaml2obj-elf-symbol-basic.yaml
index 6d49ddd1c42..b17c0429bd4 100644
--- a/test/Object/yaml2obj-elf-symbol-basic.yaml
+++ b/test/Object/yaml2obj-elf-symbol-basic.yaml
@@ -9,6 +9,7 @@ Sections:
   - Name: .text
     Type: SHT_PROGBITS
     Flags: [ SHF_ALLOC, SHF_EXECINSTR ]
+    AddressAlign: 0x4
     Content: "90EBFE" # x86 machine code
                       #   NOP ; To make main's `Value` non-zero (for testing).
                       # main:
diff --git a/test/Other/2008-10-15-MissingSpace.ll b/test/Other/2008-10-15-MissingSpace.ll
index bc78e84a0af..37b3f0cb94d 100644
--- a/test/Other/2008-10-15-MissingSpace.ll
+++ b/test/Other/2008-10-15-MissingSpace.ll
@@ -1,14 +1,14 @@
 ; RUN: llvm-as < %s | llvm-dis | FileCheck %s
 ; PR2894
 declare void @g()
-define void @f() {
+define void @f() personality i32 (...)* @__gxx_personality_v0 {
 ; CHECK:  invoke void @g()
 ; CHECK:           to label %d unwind label %c
   invoke void @g() to label %d unwind label %c
 d:
   ret void
 c:
-  %exn = landingpad {i8*, i32} personality i32 (...)* @__gxx_personality_v0
+  %exn = landingpad {i8*, i32}
             cleanup
   ret void
 }
diff --git a/test/Other/2009-03-31-CallGraph.ll b/test/Other/2009-03-31-CallGraph.ll
index 1e178308464..189c408d74c 100644
--- a/test/Other/2009-03-31-CallGraph.ll
+++ b/test/Other/2009-03-31-CallGraph.ll
@@ -1,5 +1,5 @@
 ; RUN: opt < %s -inline -prune-eh -disable-output
-define void @f2() {
+define void @f2() personality i32 (...)* @__gxx_personality_v0 {
     invoke void @f6()
         to label %ok1 unwind label %lpad1
 
@@ -7,7 +7,7 @@ ok1:
     ret void
 
 lpad1:
-    landingpad {i8*, i32} personality i32 (...)* @__gxx_personality_v0
+    landingpad {i8*, i32}
             cleanup
     invoke void @f4()
         to label %ok2 unwind label %lpad2
@@ -17,7 +17,7 @@ ok2:
     unreachable
 
 lpad2:
-    %exn = landingpad {i8*, i32} personality i32 (...)* @__gxx_personality_v0
+    %exn = landingpad {i8*, i32}
             cleanup
     unreachable
 }
diff --git a/test/Other/extract.ll b/test/Other/extract.ll
index 8b0c835d574..08675d8bff3 100644
--- a/test/Other/extract.ll
+++ b/test/Other/extract.ll
@@ -7,19 +7,22 @@
 ; llvm-extract uses lazy bitcode loading, so make sure it correctly reads
 ; from bitcode files in addition to assembly files.
 
-; CHECK: define hidden void @foo() {
+; CHECK: define hidden void @foo() comdat($x) {
 ; CHECK:   ret void
 ; CHECK: }
 
 ; The private linkage for foo() should be changed to external linkage and
 ; hidden visibility added.
 ; DELETE: declare hidden void @foo()
+; DELETE-NOT: comdat
 ; DELETE: define void @bar() {
 ; DELETE:   call void @foo()
 ; DELETE:   ret void
 ; DELETE: }
 
-define private void @foo() {
+$x = comdat any
+
+define private void @foo() comdat($x) {
   ret void
 }
 define void @bar() {
diff --git a/test/Other/opt-override-mcpu-mattr.ll b/test/Other/opt-override-mcpu-mattr.ll
new file mode 100644
index 00000000000..84d4209a996
--- /dev/null
+++ b/test/Other/opt-override-mcpu-mattr.ll
@@ -0,0 +1,13 @@
+; RUN: opt < %s -mtriple=x86_64-apple-darwin -mcpu=broadwell -mattr=+avx2 -S | FileCheck %s
+
+; Check that opt can rewrite function attributes target-cpu and target-features
+; using command line options -mcpu and -mattr.
+
+; CHECK: attributes #0 = { nounwind readnone ssp uwtable "target-cpu"="broadwell" "target-features"="+avx2" "use-soft-float"="false" }
+
+define i32 @foo1() #0 {
+entry:
+  ret i32 0
+}
+
+attributes #0 = { nounwind readnone ssp uwtable "target-cpu"="core2" "target-features"="+ssse3,+cx16,+sse,+sse2,+sse3" "use-soft-float"="false" }
diff --git a/test/TableGen/AsmPredicateCondsEmission.td b/test/TableGen/AsmPredicateCondsEmission.td
index ba5898fbebd..c1fe0d2aba2 100644
--- a/test/TableGen/AsmPredicateCondsEmission.td
+++ b/test/TableGen/AsmPredicateCondsEmission.td
@@ -28,4 +28,4 @@ def foo : Instruction {
   let Predicates = [Pred1, Pred2];
 }
 
-// CHECK: return ((Bits & arch::AssemblerCondition2));
+// CHECK: return (Bits[arch::AssemblerCondition2]);
diff --git a/test/TableGen/MultiClassDefName.td b/test/TableGen/MultiClassDefName.td
index 811d92667eb..fafa1ad3081 100644
--- a/test/TableGen/MultiClassDefName.td
+++ b/test/TableGen/MultiClassDefName.td
@@ -39,3 +39,45 @@ multiclass MC2<string name> {
 
 defm : MC2<"bar">;
 
+multiclass MC3<string s> {
+  def ZFizz#s : C<s>;
+}
+
+defm : MC3<"Buzz">;
+
+// CHECK: def ZFizzBuzz
+// CHECK: string name = "Buzz";
+// CHECK-NOT: MC3::s
+
+multiclass MC4<string s> {
+  def NAME#s : C<s>;
+}
+
+defm ZTagazok : MC4<"AToi">;
+
+// CHECK: def ZTagazokAToi
+// CHECK: string name = "AToi";
+// CHECK-NOT: MC4::s
+
+multiclass MC5<C c> {
+  def NAME#c.name : C<c.name>;
+}
+
+def CTiger : C<"Tiger">;
+defm Zebra : MC5<CTiger>;
+
+// CHECK: def ZebraTiger
+// CHECK: string name = "Tiger";
+// CHECK-NOT: MC5::c
+
+multiclass MC6<C c> {
+  def NAME#Tiger#c.name : C<c.name>;
+}
+
+def CAligator : C<"Aligator">;
+defm Zebra : MC6<CAligator>;
+
+// CHECK: def ZebraTigerAligator
+// CHECK: string name = "Aligator";
+// CHECK-NOT: MC6::c
+
diff --git a/test/TableGen/cast-list-initializer.td b/test/TableGen/cast-list-initializer.td
new file mode 100644
index 00000000000..4c83773a5a7
--- /dev/null
+++ b/test/TableGen/cast-list-initializer.td
@@ -0,0 +1,10 @@
+// RUN: llvm-tblgen %s | FileCheck %s
+
+class Foo<bits<8> b> {
+// CHECK: list<int> ListOfInts = [170];
+// CHECK: list<int> AnotherList = [170, 7];
+  list<int> ListOfInts = [!cast<int>(b)];
+  list<int> AnotherList = [!cast<int>(b), !cast<int>({1, 1, 1})];
+}
+
+def : Foo<{1, 0, 1, 0, 1, 0, 1, 0}>;
diff --git a/test/TableGen/intrinsic-varargs.td b/test/TableGen/intrinsic-varargs.td
index 935a6250380..380d79d4a22 100644
--- a/test/TableGen/intrinsic-varargs.td
+++ b/test/TableGen/intrinsic-varargs.td
@@ -23,7 +23,7 @@ class Intrinsic<string name, list<LLVMType> param_types = []> {
 }
 
 // isVoid needs to match the definition in ValueTypes.td
-def isVoid : ValueType<0, 56>;   // Produces no value
+def isVoid : ValueType<0, 57>;   // Produces no value
 def llvm_vararg_ty : LLVMType<isVoid>;   // this means vararg here
 
 // CHECK: /* 0 */ 0, 28, 0,
diff --git a/test/TableGen/trydecode-emission.td b/test/TableGen/trydecode-emission.td
new file mode 100644
index 00000000000..91c0e123857
--- /dev/null
+++ b/test/TableGen/trydecode-emission.td
@@ -0,0 +1,43 @@
+// RUN: llvm-tblgen -gen-disassembler -I %p/../../include %s | FileCheck %s
+
+// Check that if decoding of an instruction fails and the instruction does not
+// have a complete decoder method that can determine if the bitpattern is valid
+// or not then the decoder tries to find a more general instruction that
+// matches the bitpattern too.
+
+include "llvm/Target/Target.td"
+
+def archInstrInfo : InstrInfo { }
+
+def arch : Target {
+  let InstructionSet = archInstrInfo;
+}
+
+class TestInstruction : Instruction {
+  let Size = 1;
+  let OutOperandList = (outs);
+  let InOperandList = (ins);
+  field bits<8> Inst;
+  field bits<8> SoftFail = 0;
+}
+
+def InstA : TestInstruction {
+  let Inst = {0,0,0,0,?,?,?,?};
+  let AsmString = "InstA";
+}
+
+def InstB : TestInstruction {
+  let Inst = {0,0,0,0,0,0,?,?};
+  let AsmString = "InstB";
+  let DecoderMethod = "DecodeInstB";
+  let hasCompleteDecoder = 0;
+}
+
+// CHECK:      /* 0 */       MCD::OPC_ExtractField, 4, 4,  // Inst{7-4} ...
+// CHECK-NEXT: /* 3 */       MCD::OPC_FilterValue, 0, 14, 0, // Skip to: 21
+// CHECK-NEXT: /* 7 */       MCD::OPC_CheckField, 2, 2, 0, 5, 0, // Skip to: 18
+// CHECK-NEXT: /* 13 */      MCD::OPC_TryDecode, 24, 0, 0, 0, // Opcode: InstB, skip to: 18
+// CHECK-NEXT: /* 18 */      MCD::OPC_Decode, 23, 1, // Opcode: InstA
+// CHECK-NEXT: /* 21 */      MCD::OPC_Fail,
+
+// CHECK: if (DecodeInstB(MI, insn, Address, Decoder) == MCDisassembler::Fail) { DecodeComplete = false; return MCDisassembler::Fail; }
diff --git a/test/TableGen/trydecode-emission2.td b/test/TableGen/trydecode-emission2.td
new file mode 100644
index 00000000000..56ca6d33c24
--- /dev/null
+++ b/test/TableGen/trydecode-emission2.td
@@ -0,0 +1,44 @@
+// RUN: llvm-tblgen -gen-disassembler -I %p/../../include %s | FileCheck %s
+
+include "llvm/Target/Target.td"
+
+def archInstrInfo : InstrInfo { }
+
+def arch : Target {
+  let InstructionSet = archInstrInfo;
+}
+
+class TestInstruction : Instruction {
+  let Size = 1;
+  let OutOperandList = (outs);
+  let InOperandList = (ins);
+  field bits<8> Inst;
+  field bits<8> SoftFail = 0;
+}
+
+def InstA : TestInstruction {
+  let Inst = {0,0,0,0,0,0,?,?};
+  let AsmString = "InstA";
+  let DecoderMethod = "DecodeInstA";
+  let hasCompleteDecoder = 0;
+}
+
+def InstB : TestInstruction {
+  let Inst = {0,0,0,?,?,0,1,1};
+  let AsmString = "InstB";
+  let DecoderMethod = "DecodeInstB";
+  let hasCompleteDecoder = 0;
+}
+
+// CHECK:      /* 0 */       MCD::OPC_ExtractField, 2, 1,  // Inst{2} ...
+// CHECK-NEXT: /* 3 */       MCD::OPC_FilterValue, 0, 29, 0, // Skip to: 36
+// CHECK-NEXT: /* 7 */       MCD::OPC_ExtractField, 5, 3,  // Inst{7-5} ...
+// CHECK-NEXT: /* 10 */      MCD::OPC_FilterValue, 0, 22, 0, // Skip to: 36
+// CHECK-NEXT: /* 14 */      MCD::OPC_CheckField, 0, 2, 3, 5, 0, // Skip to: 25
+// CHECK-NEXT: /* 20 */      MCD::OPC_TryDecode, 24, 0, 0, 0, // Opcode: InstB, skip to: 25
+// CHECK-NEXT: /* 25 */      MCD::OPC_CheckField, 3, 2, 0, 5, 0, // Skip to: 36
+// CHECK-NEXT: /* 31 */      MCD::OPC_TryDecode, 23, 1, 0, 0, // Opcode: InstA, skip to: 36
+// CHECK-NEXT: /* 36 */      MCD::OPC_Fail,
+
+// CHECK: if (DecodeInstB(MI, insn, Address, Decoder) == MCDisassembler::Fail) { DecodeComplete = false; return MCDisassembler::Fail; }
+// CHECK: if (DecodeInstA(MI, insn, Address, Decoder) == MCDisassembler::Fail) { DecodeComplete = false; return MCDisassembler::Fail; }
diff --git a/test/TableGen/trydecode-emission3.td b/test/TableGen/trydecode-emission3.td
new file mode 100644
index 00000000000..ad21eefa897
--- /dev/null
+++ b/test/TableGen/trydecode-emission3.td
@@ -0,0 +1,44 @@
+// RUN: llvm-tblgen -gen-disassembler -I %p/../../include %s | FileCheck %s
+
+include "llvm/Target/Target.td"
+
+def archInstrInfo : InstrInfo { }
+
+def arch : Target {
+  let InstructionSet = archInstrInfo;
+}
+
+class TestInstruction : Instruction {
+  let Size = 1;
+  let OutOperandList = (outs);
+  let InOperandList = (ins);
+  field bits<8> Inst;
+  field bits<8> SoftFail = 0;
+}
+
+def InstA : TestInstruction {
+  let Inst = {0,0,0,0,?,?,?,?};
+  let AsmString = "InstA";
+}
+
+def InstBOp : Operand<i32> {
+  let DecoderMethod = "DecodeInstBOp";
+  let hasCompleteDecoder = 0;
+}
+
+def InstB : TestInstruction {
+  bits<2> op;
+  let Inst{7-2} = {0,0,0,0,0,0};
+  let Inst{1-0} = op;
+  let OutOperandList = (outs InstBOp:$op);
+  let AsmString = "InstB";
+}
+
+// CHECK:      /* 0 */       MCD::OPC_ExtractField, 4, 4,  // Inst{7-4} ...
+// CHECK-NEXT: /* 3 */       MCD::OPC_FilterValue, 0, 14, 0, // Skip to: 21
+// CHECK-NEXT: /* 7 */       MCD::OPC_CheckField, 2, 2, 0, 5, 0, // Skip to: 18
+// CHECK-NEXT: /* 13 */      MCD::OPC_TryDecode, 24, 0, 0, 0, // Opcode: InstB, skip to: 18
+// CHECK-NEXT: /* 18 */      MCD::OPC_Decode, 23, 1, // Opcode: InstA
+// CHECK-NEXT: /* 21 */      MCD::OPC_Fail,
+
+// CHECK: if (DecodeInstBOp(MI, tmp, Address, Decoder) == MCDisassembler::Fail) { DecodeComplete = false; return MCDisassembler::Fail; }
diff --git a/test/Transforms/ADCE/2003-09-10-UnwindInstFail.ll b/test/Transforms/ADCE/2003-09-10-UnwindInstFail.ll
index 6bbcfdb67ec..607bf2e5809 100644
--- a/test/Transforms/ADCE/2003-09-10-UnwindInstFail.ll
+++ b/test/Transforms/ADCE/2003-09-10-UnwindInstFail.ll
@@ -1,6 +1,6 @@
 ; RUN: opt < %s -adce -disable-output
 
-define void @test() {
+define void @test() personality i32 (...)* @__gxx_personality_v0 {
         br i1 false, label %then, label %endif
 
 then:           ; preds = %0
@@ -8,7 +8,7 @@ then:           ; preds = %0
                         to label %invoke_cont unwind label %invoke_catch
 
 invoke_catch:           ; preds = %then
-        %exn = landingpad {i8*, i32} personality i32 (...)* @__gxx_personality_v0
+        %exn = landingpad {i8*, i32}
                  cleanup
         resume { i8*, i32 } %exn
 
diff --git a/test/Transforms/ADCE/2005-02-17-PHI-Invoke-Crash.ll b/test/Transforms/ADCE/2005-02-17-PHI-Invoke-Crash.ll
index 4ddc2f180a2..068ad2bc1d8 100644
--- a/test/Transforms/ADCE/2005-02-17-PHI-Invoke-Crash.ll
+++ b/test/Transforms/ADCE/2005-02-17-PHI-Invoke-Crash.ll
@@ -6,7 +6,7 @@ declare void @_ZN10QByteArray6resizeEi()
 
 declare void @q_atomic_decrement()
 
-define void @_ZNK10QByteArray13leftJustifiedEicb() {
+define void @_ZNK10QByteArray13leftJustifiedEicb() personality i32 (...)* @__gxx_personality_v0 {
 entry:
         invoke void @strlen( )
                         to label %tmp.3.i.noexc unwind label %invoke_catch.0
@@ -15,7 +15,7 @@ tmp.3.i.noexc:          ; preds = %entry
         br i1 false, label %then.0, label %else.0
 
 invoke_catch.0:         ; preds = %entry
-        %exn.0 = landingpad {i8*, i32} personality i32 (...)* @__gxx_personality_v0
+        %exn.0 = landingpad {i8*, i32}
                  cleanup
         invoke void @q_atomic_decrement( )
                         to label %tmp.1.i.i183.noexc unwind label %terminate
@@ -28,7 +28,7 @@ then.0:         ; preds = %tmp.3.i.noexc
                         to label %invoke_cont.1 unwind label %invoke_catch.1
 
 invoke_catch.1:         ; preds = %then.0
-        %exn.1 = landingpad {i8*, i32} personality i32 (...)* @__gxx_personality_v0
+        %exn.1 = landingpad {i8*, i32}
                  cleanup
         invoke void @q_atomic_decrement( )
                         to label %tmp.1.i.i162.noexc unwind label %terminate
@@ -44,7 +44,7 @@ else.0:         ; preds = %tmp.3.i.noexc
 
 terminate:              ; preds = %invoke_catch.1, %invoke_catch.0
         %dbg.0.1 = phi {  }* [ null, %invoke_catch.1 ], [ null, %invoke_catch.0 ]               ; <{  }*> [#uses=0]
-        %exn = landingpad {i8*, i32} personality i32 (...)* @__gxx_personality_v0
+        %exn = landingpad {i8*, i32}
                  cleanup
         unreachable
 }
diff --git a/test/Transforms/ADCE/dce_pure_invoke.ll b/test/Transforms/ADCE/dce_pure_invoke.ll
index 8e785180457..e01c9feaeb0 100644
--- a/test/Transforms/ADCE/dce_pure_invoke.ll
+++ b/test/Transforms/ADCE/dce_pure_invoke.ll
@@ -2,7 +2,7 @@
 
 declare i32 @strlen(i8*) readnone
 
-define i32 @test() {
+define i32 @test() personality i32 (...)* @__gxx_personality_v0 {
 	; invoke of pure function should not be deleted!
 	invoke i32 @strlen( i8* null ) readnone
 			to label %Cont unwind label %Other		; <i32>:1 [#uses=0]
@@ -11,7 +11,7 @@ Cont:		; preds = %0
 	ret i32 0
 
 Other:		; preds = %0
-         %exn = landingpad {i8*, i32} personality i32 (...)* @__gxx_personality_v0
+         %exn = landingpad {i8*, i32}
                   cleanup
 	ret i32 1
 }
diff --git a/test/Transforms/AddDiscriminators/basic.ll b/test/Transforms/AddDiscriminators/basic.ll
index 9ab0edf4f5e..cabf707fe5d 100644
--- a/test/Transforms/AddDiscriminators/basic.ll
+++ b/test/Transforms/AddDiscriminators/basic.ll
@@ -41,22 +41,22 @@ attributes #0 = { nounwind uwtable "less-precise-fpmad"="false" "no-frame-pointe
 !llvm.module.flags = !{!7, !8}
 !llvm.ident = !{!9}
 
-!0 = !MDCompileUnit(language: DW_LANG_C99, producer: "clang version 3.5 ", isOptimized: false, emissionKind: 0, file: !1, enums: !2, retainedTypes: !2, subprograms: !3, globals: !2, imports: !2)
-!1 = !MDFile(filename: "basic.c", directory: ".")
+!0 = !DICompileUnit(language: DW_LANG_C99, producer: "clang version 3.5 ", isOptimized: false, emissionKind: 0, file: !1, enums: !2, retainedTypes: !2, subprograms: !3, globals: !2, imports: !2)
+!1 = !DIFile(filename: "basic.c", directory: ".")
 !2 = !{}
 !3 = !{!4}
-!4 = !MDSubprogram(name: "foo", line: 1, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 1, file: !1, scope: !5, type: !6, function: void (i32)* @foo, variables: !2)
-!5 = !MDFile(filename: "basic.c", directory: ".")
-!6 = !MDSubroutineType(types: !2)
+!4 = !DISubprogram(name: "foo", line: 1, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 1, file: !1, scope: !5, type: !6, function: void (i32)* @foo, variables: !2)
+!5 = !DIFile(filename: "basic.c", directory: ".")
+!6 = !DISubroutineType(types: !2)
 !7 = !{i32 2, !"Dwarf Version", i32 4}
 !8 = !{i32 1, !"Debug Info Version", i32 3}
 !9 = !{!"clang version 3.5 "}
-!10 = !MDLocation(line: 3, scope: !11)
-!11 = distinct !MDLexicalBlock(line: 3, column: 0, file: !1, scope: !4)
-!12 = !MDLocation(line: 4, scope: !4)
-
-; CHECK: ![[FOO:[0-9]+]] = !MDSubprogram(name: "foo"
-; CHECK: ![[BLOCK:[0-9]+]] = distinct !MDLexicalBlock(scope: ![[FOO]],{{.*}} line: 3)
-; CHECK: ![[THEN]] = !MDLocation(line: 3, scope: ![[BLOCKFILE:[0-9]+]])
-; CHECK: ![[BLOCKFILE]] = !MDLexicalBlockFile(scope: ![[BLOCK]],{{.*}} discriminator: 1)
-; CHECK: ![[END]] = !MDLocation(line: 4, scope: ![[FOO]])
+!10 = !DILocation(line: 3, scope: !11)
+!11 = distinct !DILexicalBlock(line: 3, column: 0, file: !1, scope: !4)
+!12 = !DILocation(line: 4, scope: !4)
+
+; CHECK: ![[FOO:[0-9]+]] = !DISubprogram(name: "foo"
+; CHECK: ![[BLOCK:[0-9]+]] = distinct !DILexicalBlock(scope: ![[FOO]],{{.*}} line: 3)
+; CHECK: ![[THEN]] = !DILocation(line: 3, scope: ![[BLOCKFILE:[0-9]+]])
+; CHECK: ![[BLOCKFILE]] = !DILexicalBlockFile(scope: ![[BLOCK]],{{.*}} discriminator: 1)
+; CHECK: ![[END]] = !DILocation(line: 4, scope: ![[FOO]])
diff --git a/test/Transforms/AddDiscriminators/first-only.ll b/test/Transforms/AddDiscriminators/first-only.ll
index 59bcb0543ff..7f1ea2b15cf 100644
--- a/test/Transforms/AddDiscriminators/first-only.ll
+++ b/test/Transforms/AddDiscriminators/first-only.ll
@@ -50,34 +50,34 @@ attributes #0 = { nounwind uwtable "less-precise-fpmad"="false" "no-frame-pointe
 !llvm.module.flags = !{!7, !8}
 !llvm.ident = !{!9}
 
-!0 = !MDCompileUnit(language: DW_LANG_C99, producer: "clang version 3.5 (trunk 199750) (llvm/trunk 199751)", isOptimized: false, emissionKind: 0, file: !1, enums: !2, retainedTypes: !2, subprograms: !3, globals: !2, imports: !2)
-!1 = !MDFile(filename: "first-only.c", directory: ".")
+!0 = !DICompileUnit(language: DW_LANG_C99, producer: "clang version 3.5 (trunk 199750) (llvm/trunk 199751)", isOptimized: false, emissionKind: 0, file: !1, enums: !2, retainedTypes: !2, subprograms: !3, globals: !2, imports: !2)
+!1 = !DIFile(filename: "first-only.c", directory: ".")
 !2 = !{}
 !3 = !{!4}
-!4 = !MDSubprogram(name: "foo", line: 1, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 1, file: !1, scope: !5, type: !6, function: void (i32)* @foo, variables: !2)
-!5 = !MDFile(filename: "first-only.c", directory: ".")
-!6 = !MDSubroutineType(types: !{null})
+!4 = !DISubprogram(name: "foo", line: 1, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 1, file: !1, scope: !5, type: !6, function: void (i32)* @foo, variables: !2)
+!5 = !DIFile(filename: "first-only.c", directory: ".")
+!6 = !DISubroutineType(types: !{null})
 !7 = !{i32 2, !"Dwarf Version", i32 4}
 !8 = !{i32 1, !"Debug Info Version", i32 3}
 !9 = !{!"clang version 3.5 (trunk 199750) (llvm/trunk 199751)"}
-!10 = !MDLocation(line: 3, scope: !11)
+!10 = !DILocation(line: 3, scope: !11)
 
-!11 = distinct !MDLexicalBlock(line: 3, column: 0, file: !1, scope: !4)
-; CHECK: ![[FOO:[0-9]+]] = !MDSubprogram(name: "foo"
-; CHECK: ![[BLOCK1:[0-9]+]] = distinct !MDLexicalBlock(scope: ![[FOO]],{{.*}} line: 3)
+!11 = distinct !DILexicalBlock(line: 3, column: 0, file: !1, scope: !4)
+; CHECK: ![[FOO:[0-9]+]] = !DISubprogram(name: "foo"
+; CHECK: ![[BLOCK1:[0-9]+]] = distinct !DILexicalBlock(scope: ![[FOO]],{{.*}} line: 3)
 
-!12 = !MDLocation(line: 3, scope: !13)
+!12 = !DILocation(line: 3, scope: !13)
 
-!13 = distinct !MDLexicalBlock(line: 3, column: 0, file: !1, scope: !11)
-; CHECK: !MDLexicalBlockFile(scope: ![[BLOCK2:[0-9]+]],{{.*}} discriminator: 1)
+!13 = distinct !DILexicalBlock(line: 3, column: 0, file: !1, scope: !11)
+; CHECK: !DILexicalBlockFile(scope: ![[BLOCK2:[0-9]+]],{{.*}} discriminator: 1)
 
-!14 = !MDLocation(line: 4, scope: !13)
-; CHECK: ![[BLOCK2]] = distinct !MDLexicalBlock(scope: ![[BLOCK1]],{{.*}} line: 3)
+!14 = !DILocation(line: 4, scope: !13)
+; CHECK: ![[BLOCK2]] = distinct !DILexicalBlock(scope: ![[BLOCK1]],{{.*}} line: 3)
 
-!15 = !MDLocation(line: 5, scope: !13)
-; CHECK: ![[THEN]] = !MDLocation(line: 4, scope: ![[BLOCK2]])
+!15 = !DILocation(line: 5, scope: !13)
+; CHECK: ![[THEN]] = !DILocation(line: 4, scope: ![[BLOCK2]])
 
-!16 = !MDLocation(line: 6, scope: !4)
-; CHECK: ![[BR]] = !MDLocation(line: 5, scope: ![[BLOCK2]])
-; CHECK: ![[END]] = !MDLocation(line: 6, scope: ![[FOO]])
+!16 = !DILocation(line: 6, scope: !4)
+; CHECK: ![[BR]] = !DILocation(line: 5, scope: ![[BLOCK2]])
+; CHECK: ![[END]] = !DILocation(line: 6, scope: ![[FOO]])
 
diff --git a/test/Transforms/AddDiscriminators/multiple.ll b/test/Transforms/AddDiscriminators/multiple.ll
index 587baad43ee..621a7117571 100644
--- a/test/Transforms/AddDiscriminators/multiple.ll
+++ b/test/Transforms/AddDiscriminators/multiple.ll
@@ -51,22 +51,22 @@ attributes #0 = { nounwind uwtable "less-precise-fpmad"="false" "no-frame-pointe
 !llvm.module.flags = !{!7, !8}
 !llvm.ident = !{!9}
 
-!0 = !MDCompileUnit(language: DW_LANG_C99, producer: "clang version 3.5 (trunk 199750) (llvm/trunk 199751)", isOptimized: false, emissionKind: 0, file: !1, enums: !2, retainedTypes: !2, subprograms: !3, globals: !2, imports: !2)
-!1 = !MDFile(filename: "multiple.c", directory: ".")
+!0 = !DICompileUnit(language: DW_LANG_C99, producer: "clang version 3.5 (trunk 199750) (llvm/trunk 199751)", isOptimized: false, emissionKind: 0, file: !1, enums: !2, retainedTypes: !2, subprograms: !3, globals: !2, imports: !2)
+!1 = !DIFile(filename: "multiple.c", directory: ".")
 !2 = !{}
 !3 = !{!4}
-!4 = !MDSubprogram(name: "foo", line: 1, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 1, file: !1, scope: !5, type: !6, function: void (i32)* @foo, variables: !2)
-!5 = !MDFile(filename: "multiple.c", directory: ".")
-!6 = !MDSubroutineType(types: !{null, !13})
-!13 = !MDBasicType(encoding: DW_ATE_signed, name: "int", size: 32, align: 32)
+!4 = !DISubprogram(name: "foo", line: 1, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 1, file: !1, scope: !5, type: !6, function: void (i32)* @foo, variables: !2)
+!5 = !DIFile(filename: "multiple.c", directory: ".")
+!6 = !DISubroutineType(types: !{null, !13})
+!13 = !DIBasicType(encoding: DW_ATE_signed, name: "int", size: 32, align: 32)
 !7 = !{i32 2, !"Dwarf Version", i32 4}
 !8 = !{i32 1, !"Debug Info Version", i32 3}
 !9 = !{!"clang version 3.5 (trunk 199750) (llvm/trunk 199751)"}
-!10 = !MDLocation(line: 3, scope: !11)
-!11 = distinct !MDLexicalBlock(line: 3, column: 0, file: !1, scope: !4)
-!12 = !MDLocation(line: 4, scope: !4)
+!10 = !DILocation(line: 3, scope: !11)
+!11 = distinct !DILexicalBlock(line: 3, column: 0, file: !1, scope: !4)
+!12 = !DILocation(line: 4, scope: !4)
 
-; CHECK: ![[THEN]] = !MDLocation(line: 3, scope: ![[THENBLOCK:[0-9]+]])
-; CHECK: ![[THENBLOCK]] = !MDLexicalBlockFile(scope: ![[SCOPE:[0-9]+]],{{.*}} discriminator: 1)
-; CHECK: ![[ELSE]] = !MDLocation(line: 3, scope: ![[ELSEBLOCK:[0-9]+]])
-; CHECK: ![[ELSEBLOCK]] = !MDLexicalBlockFile(scope: ![[SCOPE]],{{.*}} discriminator: 2)
+; CHECK: ![[THEN]] = !DILocation(line: 3, scope: ![[THENBLOCK:[0-9]+]])
+; CHECK: ![[THENBLOCK]] = !DILexicalBlockFile(scope: ![[SCOPE:[0-9]+]],{{.*}} discriminator: 1)
+; CHECK: ![[ELSE]] = !DILocation(line: 3, scope: ![[ELSEBLOCK:[0-9]+]])
+; CHECK: ![[ELSEBLOCK]] = !DILexicalBlockFile(scope: ![[SCOPE]],{{.*}} discriminator: 2)
diff --git a/test/Transforms/AddDiscriminators/no-discriminators.ll b/test/Transforms/AddDiscriminators/no-discriminators.ll
index 5ef0d00119c..895967e73b4 100644
--- a/test/Transforms/AddDiscriminators/no-discriminators.ll
+++ b/test/Transforms/AddDiscriminators/no-discriminators.ll
@@ -17,7 +17,7 @@ entry:
   %retval = alloca i32, align 4
   %i.addr = alloca i64, align 8
   store i64 %i, i64* %i.addr, align 8
-  call void @llvm.dbg.declare(metadata i64* %i.addr, metadata !13, metadata !MDExpression()), !dbg !14
+  call void @llvm.dbg.declare(metadata i64* %i.addr, metadata !13, metadata !DIExpression()), !dbg !14
   %0 = load i64, i64* %i.addr, align 8, !dbg !15
 ; CHECK:  %0 = load i64, i64* %i.addr, align 8, !dbg ![[ENTRY:[0-9]+]]
   %cmp = icmp slt i64 %0, 5, !dbg !15
@@ -48,25 +48,25 @@ attributes #1 = { nounwind readnone }
 !llvm.module.flags = !{!10, !11}
 !llvm.ident = !{!12}
 
-!0 = !MDCompileUnit(language: DW_LANG_C99, producer: "clang version 3.5.0 ", isOptimized: false, emissionKind: 1, file: !1, enums: !2, retainedTypes: !2, subprograms: !3, globals: !2, imports: !2)
-!1 = !MDFile(filename: "no-discriminators", directory: ".")
+!0 = !DICompileUnit(language: DW_LANG_C99, producer: "clang version 3.5.0 ", isOptimized: false, emissionKind: 1, file: !1, enums: !2, retainedTypes: !2, subprograms: !3, globals: !2, imports: !2)
+!1 = !DIFile(filename: "no-discriminators", directory: ".")
 !2 = !{}
 !3 = !{!4}
-!4 = !MDSubprogram(name: "foo", line: 1, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 1, file: !1, scope: !5, type: !6, function: i32 (i64)* @foo, variables: !2)
-; CHECK: ![[FOO:[0-9]+]] = !MDSubprogram(name: "foo"
-!5 = !MDFile(filename: "no-discriminators", directory: ".")
-!6 = !MDSubroutineType(types: !7)
+!4 = !DISubprogram(name: "foo", line: 1, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 1, file: !1, scope: !5, type: !6, function: i32 (i64)* @foo, variables: !2)
+; CHECK: ![[FOO:[0-9]+]] = !DISubprogram(name: "foo"
+!5 = !DIFile(filename: "no-discriminators", directory: ".")
+!6 = !DISubroutineType(types: !7)
 !7 = !{!8, !9}
-!8 = !MDBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
-!9 = !MDBasicType(tag: DW_TAG_base_type, name: "long int", size: 64, align: 64, encoding: DW_ATE_signed)
+!8 = !DIBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
+!9 = !DIBasicType(tag: DW_TAG_base_type, name: "long int", size: 64, align: 64, encoding: DW_ATE_signed)
 !10 = !{i32 2, !"Dwarf Version", i32 2}
 ; CHECK: !{i32 2, !"Dwarf Version", i32 2}
 !11 = !{i32 1, !"Debug Info Version", i32 3}
 !12 = !{!"clang version 3.5.0 "}
-!13 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "i", line: 1, arg: 1, scope: !4, file: !5, type: !9)
-!14 = !MDLocation(line: 1, scope: !4)
-!15 = !MDLocation(line: 2, scope: !16)
-; CHECK: ![[ENTRY]] = !MDLocation(line: 2, scope: ![[BLOCK:[0-9]+]])
-!16 = distinct !MDLexicalBlock(line: 2, column: 0, file: !1, scope: !4)
-; CHECK: ![[BLOCK]] = distinct !MDLexicalBlock(scope: ![[FOO]],{{.*}} line: 2)
-!17 = !MDLocation(line: 3, scope: !4)
+!13 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "i", line: 1, arg: 1, scope: !4, file: !5, type: !9)
+!14 = !DILocation(line: 1, scope: !4)
+!15 = !DILocation(line: 2, scope: !16)
+; CHECK: ![[ENTRY]] = !DILocation(line: 2, scope: ![[BLOCK:[0-9]+]])
+!16 = distinct !DILexicalBlock(line: 2, column: 0, file: !1, scope: !4)
+; CHECK: ![[BLOCK]] = distinct !DILexicalBlock(scope: ![[FOO]],{{.*}} line: 2)
+!17 = !DILocation(line: 3, scope: !4)
diff --git a/test/Transforms/ArgumentPromotion/crash.ll b/test/Transforms/ArgumentPromotion/crash.ll
index dbd343ae920..d3f412da14d 100644
--- a/test/Transforms/ArgumentPromotion/crash.ll
+++ b/test/Transforms/ArgumentPromotion/crash.ll
@@ -1,7 +1,7 @@
 ; RUN: opt -inline -argpromotion < %s
 ; rdar://7879828
 
-define void @foo() {
+define void @foo() personality i32 (...)* @__gxx_personality_v0 {
   invoke void @foo2()
           to label %if.end432 unwind label %for.end520 
 
@@ -9,7 +9,7 @@ if.end432:
   unreachable
 
 for.end520: 
-  %exn = landingpad {i8*, i32} personality i32 (...)* @__gxx_personality_v0
+  %exn = landingpad {i8*, i32}
            cleanup
   unreachable
 }
diff --git a/test/Transforms/ArgumentPromotion/dbg.ll b/test/Transforms/ArgumentPromotion/dbg.ll
index 69bd4d22a61..17a34cb6223 100644
--- a/test/Transforms/ArgumentPromotion/dbg.ll
+++ b/test/Transforms/ArgumentPromotion/dbg.ll
@@ -1,6 +1,6 @@
 ; RUN: opt < %s -argpromotion -S | FileCheck %s
 ; CHECK: call void @test(i32 %
-; CHECK: !MDSubprogram(name: "test",{{.*}} function: void (i32)* @test
+; CHECK: !DISubprogram(name: "test",{{.*}} function: void (i32)* @test
 
 declare void @sink(i32)
 
@@ -20,8 +20,8 @@ define void @caller(i32** %Y) {
 !llvm.dbg.cu = !{!3}
 
 !0 = !{i32 2, !"Debug Info Version", i32 3}
-!1 = !MDLocation(line: 8, scope: !2)
-!2 = !MDSubprogram(name: "test", line: 3, isLocal: true, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 3, scope: null, function: void (i32**)* @test)
-!3 = !MDCompileUnit(language: DW_LANG_C_plus_plus, producer: "clang version 3.5.0 ", isOptimized: false, emissionKind: 2, file: !5, subprograms: !4)
+!1 = !DILocation(line: 8, scope: !2)
+!2 = !DISubprogram(name: "test", line: 3, isLocal: true, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 3, scope: null, function: void (i32**)* @test)
+!3 = !DICompileUnit(language: DW_LANG_C_plus_plus, producer: "clang version 3.5.0 ", isOptimized: false, emissionKind: 2, file: !5, subprograms: !4)
 !4 = !{!2}
-!5 = !MDFile(filename: "test.c", directory: "")
+!5 = !DIFile(filename: "test.c", directory: "")
diff --git a/test/Transforms/ArgumentPromotion/sret.ll b/test/Transforms/ArgumentPromotion/sret.ll
new file mode 100644
index 00000000000..8e5521f48d1
--- /dev/null
+++ b/test/Transforms/ArgumentPromotion/sret.ll
@@ -0,0 +1,28 @@
+; RUN: opt < %s -argpromotion -S | FileCheck %s
+
+target datalayout = "e-m:w-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-pc-windows-msvc"
+
+; CHECK: define internal void @add(i32 %[[THIS1:.*]], i32 %[[THIS2:.*]], i32* noalias %[[SR:.*]])
+define internal void @add({i32, i32}* %this, i32* sret %r) {
+  %ap = getelementptr {i32, i32}, {i32, i32}* %this, i32 0, i32 0
+  %bp = getelementptr {i32, i32}, {i32, i32}* %this, i32 0, i32 1
+  %a = load i32, i32* %ap
+  %b = load i32, i32* %bp
+  ; CHECK: %[[AB:.*]] = add i32 %[[THIS1]], %[[THIS2]]
+  %ab = add i32 %a, %b
+  ; CHECK: store i32 %[[AB]], i32* %[[SR]]
+  store i32 %ab, i32* %r
+  ret void
+}
+
+; CHECK: define void @f()
+define void @f() {
+  ; CHECK: %[[R:.*]] = alloca i32
+  %r = alloca i32
+  %pair = alloca {i32, i32}
+
+  ; CHECK: call void @add(i32 %{{.*}}, i32 %{{.*}}, i32* noalias %[[R]])
+  call void @add({i32, i32}* %pair, i32* sret %r)
+  ret void
+}
diff --git a/test/Transforms/BBVectorize/simple3.ll b/test/Transforms/BBVectorize/simple3.ll
index 6edf7f07ac1..da7f9414941 100644
--- a/test/Transforms/BBVectorize/simple3.ll
+++ b/test/Transforms/BBVectorize/simple3.ll
@@ -4,12 +4,12 @@ target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f3
 ; Basic depth-3 chain
 define double @test1(double %A1, double %A2, double %A3, double %B1, double %B2, double %B3) {
 ; CHECK-LABEL: @test1(
-; CHECK: %X1.v.i1.11 = insertelement <3 x double> undef, double %B1, i32 0
-; CHECK: %X1.v.i1.22 = insertelement <3 x double> %X1.v.i1.11, double %B2, i32 1
-; CHECK: %X1.v.i1 = insertelement <3 x double> %X1.v.i1.22, double %B3, i32 2
-; CHECK: %X1.v.i0.13 = insertelement <3 x double> undef, double %A1, i32 0
-; CHECK: %X1.v.i0.24 = insertelement <3 x double> %X1.v.i0.13, double %A2, i32 1
-; CHECK: %X1.v.i0 = insertelement <3 x double> %X1.v.i0.24, double %A3, i32 2
+; CHECK: %X1.v.i1.1.1 = insertelement <3 x double> undef, double %B1, i32 0
+; CHECK: %X1.v.i1.2.2 = insertelement <3 x double> %X1.v.i1.1.1, double %B2, i32 1
+; CHECK: %X1.v.i1 = insertelement <3 x double> %X1.v.i1.2.2, double %B3, i32 2
+; CHECK: %X1.v.i0.1.3 = insertelement <3 x double> undef, double %A1, i32 0
+; CHECK: %X1.v.i0.2.4 = insertelement <3 x double> %X1.v.i0.1.3, double %A2, i32 1
+; CHECK: %X1.v.i0 = insertelement <3 x double> %X1.v.i0.2.4, double %A3, i32 2
 	%X1 = fsub double %A1, %B1
 	%X2 = fsub double %A2, %B2
 	%X3 = fsub double %A3, %B3
@@ -24,11 +24,11 @@ define double @test1(double %A1, double %A2, double %A3, double %B1, double %B2,
 ; CHECK: %Z1 = fadd <3 x double> %Y1, %X1.v.i1
         %R1 = fmul double %Z1, %Z2
 	%R  = fmul double %R1, %Z3
-; CHECK: %Z1.v.r210 = extractelement <3 x double> %Z1, i32 2
+; CHECK: %Z1.v.r2.10 = extractelement <3 x double> %Z1, i32 2
 ; CHECK: %Z1.v.r1 = extractelement <3 x double> %Z1, i32 0
 ; CHECK: %Z1.v.r2 = extractelement <3 x double> %Z1, i32 1
 ; CHECK: %R1 = fmul double %Z1.v.r1, %Z1.v.r2
-; CHECK: %R = fmul double %R1, %Z1.v.r210
+; CHECK: %R = fmul double %R1, %Z1.v.r2.10
 	ret double %R
 ; CHECK: ret double %R
 }
diff --git a/test/Transforms/BDCE/dce-pure.ll b/test/Transforms/BDCE/dce-pure.ll
index 6a432fcc42d..a379fa4a003 100644
--- a/test/Transforms/BDCE/dce-pure.ll
+++ b/test/Transforms/BDCE/dce-pure.ll
@@ -11,7 +11,7 @@ define void @test1() {
 ; CHECK: ret void
 }
 
-define i32 @test2() {
+define i32 @test2() personality i32 (...)* @__gxx_personality_v0 {
   ; invoke of pure function should not be deleted!
   invoke i32 @strlen( i8* null ) readnone
                   to label %Cont unwind label %Other
@@ -20,7 +20,7 @@ Cont:           ; preds = %0
   ret i32 0
 
 Other:          ; preds = %0
-   %exn = landingpad {i8*, i32} personality i32 (...)* @__gxx_personality_v0
+   %exn = landingpad {i8*, i32}
             cleanup
   ret i32 1
 
diff --git a/test/Transforms/BDCE/order.ll b/test/Transforms/BDCE/order.ll
index 301f4476ab5..728624a5f23 100644
--- a/test/Transforms/BDCE/order.ll
+++ b/test/Transforms/BDCE/order.ll
@@ -4,7 +4,7 @@ target triple = "x86_64-unknown-linux-gnu"
 
 declare i32 @__gxx_personality_v0(...)
 
-define fastcc void @_ZN11__sanitizerL12TestRegistryEPNS_14ThreadRegistryEb() #0 {
+define fastcc void @_ZN11__sanitizerL12TestRegistryEPNS_14ThreadRegistryEb() #0 personality i8* bitcast (i32 (...)* @__gxx_personality_v0 to i8*) {
 entry:
   br i1 undef, label %if.else, label %entry.if.end_crit_edge
 
@@ -22,7 +22,7 @@ lpad65.loopexit.split-lp.loopexit.split-lp:
   br label %if.else
 
 lpad65.loopexit.split-lp.loopexit.split-lp.loopexit:
-  %lpad.loopexit1121 = landingpad { i8*, i32 } personality i8* bitcast (i32 (...)* @__gxx_personality_v0 to i8*)
+  %lpad.loopexit1121 = landingpad { i8*, i32 }
           cleanup
   br label %lpad65.loopexit.split-lp.loopexit.split-lp
 
diff --git a/test/Transforms/BranchFolding/2007-10-19-InlineAsmDirectives.ll b/test/Transforms/BranchFolding/2007-10-19-InlineAsmDirectives.ll
index d4b94fe62c7..56a99d191b3 100644
--- a/test/Transforms/BranchFolding/2007-10-19-InlineAsmDirectives.ll
+++ b/test/Transforms/BranchFolding/2007-10-19-InlineAsmDirectives.ll
@@ -1,7 +1,10 @@
-; RUN: opt < %s -O3 -o - | llc -no-integrated-as -o - | grep bork_directive | wc -l | grep 2
+; RUN: opt < %s -O3 | llc -no-integrated-as | FileCheck %s
 
 ;; We don't want branch folding to fold asm directives.
 
+; CHECK: bork_directive
+; CHECK: bork_directive
+
 define void @bork(i32 %param) {
 entry:
 	%tmp = icmp eq i32 %param, 0
diff --git a/test/Transforms/CodeExtractor/2004-03-18-InvokeHandling.ll b/test/Transforms/CodeExtractor/2004-03-18-InvokeHandling.ll
index ff5de6b73e2..fd9814c545f 100644
--- a/test/Transforms/CodeExtractor/2004-03-18-InvokeHandling.ll
+++ b/test/Transforms/CodeExtractor/2004-03-18-InvokeHandling.ll
@@ -4,7 +4,7 @@ declare i32 @_IO_getc()
 
 declare void @__errno_location()
 
-define void @yylex() {
+define void @yylex() personality i32 (...)* @__gcc_personality_v0 {
 entry:
 	switch i32 0, label %label.126 [
 		 i32 0, label %return
@@ -190,7 +190,7 @@ return:		; preds = %entry
 	ret void
 
 LongJmpBlkPre:		; preds = %endif.52, %then.40
-        %exn = landingpad { i8*, i32 } personality i32 (...)* @__gcc_personality_v0
+        %exn = landingpad { i8*, i32 }
                  catch i8* null
 	ret void
 }
diff --git a/test/Transforms/CodeExtractor/2004-11-12-InvokeExtract.ll b/test/Transforms/CodeExtractor/2004-11-12-InvokeExtract.ll
index 92603d9e634..0a83681a071 100644
--- a/test/Transforms/CodeExtractor/2004-11-12-InvokeExtract.ll
+++ b/test/Transforms/CodeExtractor/2004-11-12-InvokeExtract.ll
@@ -1,5 +1,5 @@
 ; RUN: opt < %s -extract-blocks -disable-output
-define i32 @foo() {
+define i32 @foo() personality i32 (...)* @__gcc_personality_v0 {
         br label %EB
 
 EB:             ; preds = %0
@@ -10,7 +10,7 @@ Cont:           ; preds = %EB
         ret i32 %V
 
 Unw:            ; preds = %EB
-        %exn = landingpad { i8*, i32 } personality i32 (...)* @__gcc_personality_v0
+        %exn = landingpad { i8*, i32 }
                  catch i8* null
         resume { i8*, i32 } %exn
 }
diff --git a/test/Transforms/CodeGenPrepare/AMDGPU/lit.local.cfg b/test/Transforms/CodeGenPrepare/AMDGPU/lit.local.cfg
new file mode 100644
index 00000000000..6baccf05fff
--- /dev/null
+++ b/test/Transforms/CodeGenPrepare/AMDGPU/lit.local.cfg
@@ -0,0 +1,3 @@
+if not 'AMDGPU' in config.root.targets:
+    config.unsupported = True
+
diff --git a/test/Transforms/CodeGenPrepare/AMDGPU/no-sink-addrspacecast.ll b/test/Transforms/CodeGenPrepare/AMDGPU/no-sink-addrspacecast.ll
new file mode 100644
index 00000000000..f6f898fae21
--- /dev/null
+++ b/test/Transforms/CodeGenPrepare/AMDGPU/no-sink-addrspacecast.ll
@@ -0,0 +1,49 @@
+; RUN: opt -S -codegenprepare -mtriple=amdgcn-unknown-unknown < %s | FileCheck -check-prefix=ASC -check-prefix=COMMON %s
+
+; COMMON-LABEL: @test_sink_ptrtoint_asc(
+; ASC: addrspacecast
+; ASC-NOT: ptrtoint
+; ASC-NOT: inttoptr
+
+define void @test_sink_ptrtoint_asc(float addrspace(1)* nocapture %arg, float addrspace(1)* nocapture readonly %arg1, float addrspace(3)* %arg2) #0 {
+bb:
+  %tmp = getelementptr inbounds float, float addrspace(3)* %arg2, i32 16
+  %tmp2 = tail call i32 @llvm.r600.read.tidig.x() #1
+  %tmp3 = sext i32 %tmp2 to i64
+  %tmp4 = getelementptr inbounds float, float addrspace(1)* %arg1, i64 %tmp3
+  %tmp5 = load float, float addrspace(1)* %tmp4, align 4
+  %tmp6 = addrspacecast float addrspace(3)* %tmp to float addrspace(4)*
+  %tmp7 = fcmp olt float %tmp5, 8.388608e+06
+  br i1 %tmp7, label %bb8, label %bb14
+
+bb8:                                              ; preds = %bb
+  %tmp9 = tail call float @llvm.fma.f32(float %tmp5, float 0x3FE45F3060000000, float 5.000000e-01) #1
+  %tmp10 = fmul float %tmp9, 0x3E74442D00000000
+  %tmp11 = fsub float -0.000000e+00, %tmp10
+  %tmp12 = tail call float @llvm.fma.f32(float %tmp9, float 0x3E74442D00000000, float %tmp11) #1
+  store float %tmp12, float addrspace(4)* %tmp6, align 4
+  %tmp13 = fsub float -0.000000e+00, %tmp12
+  br label %bb15
+
+bb14:                                             ; preds = %bb
+  store float 2.000000e+00, float addrspace(4)* %tmp6, align 4
+  br label %bb15
+
+bb15:                                             ; preds = %bb14, %bb8
+  %tmp16 = phi float [ 0.000000e+00, %bb14 ], [ %tmp13, %bb8 ]
+  %tmp17 = fsub float -0.000000e+00, %tmp16
+  %tmp18 = tail call float @llvm.fma.f32(float 1.000000e+00, float 0x3FF0AAAAA0000000, float %tmp17) #1
+  %tmp19 = fsub float 2.187500e-01, %tmp18
+  %tmp20 = fsub float 7.187500e-01, %tmp19
+  %tmp21 = fcmp ogt float %tmp5, 1.600000e+01
+  %tmp22 = select i1 %tmp21, float 0x7FF8000000000000, float %tmp20
+  %tmp23 = getelementptr inbounds float, float addrspace(1)* %arg, i64 %tmp3
+  store float %tmp22, float addrspace(1)* %tmp23, align 4
+  ret void
+}
+
+declare float @llvm.fma.f32(float, float, float) #1
+declare i32 @llvm.r600.read.tidig.x() #1
+
+attributes #0 = { nounwind }
+attributes #1 = { nounwind readnone }
diff --git a/test/Transforms/CodeGenPrepare/statepoint-relocate.ll b/test/Transforms/CodeGenPrepare/statepoint-relocate.ll
index 1baf4abbc94..b6898b37311 100644
--- a/test/Transforms/CodeGenPrepare/statepoint-relocate.ll
+++ b/test/Transforms/CodeGenPrepare/statepoint-relocate.ll
@@ -10,9 +10,9 @@ define i32 @test_sor_basic(i32* %base) gc "statepoint-example" {
 ; CHECK: getelementptr i32, i32* %base-new, i32 15
 entry:
        %ptr = getelementptr i32, i32* %base, i32 15
-       %tok = call i32 (i1 ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_i1f(i1 ()* @return_i1, i32 0, i32 0, i32 0, i32* %base, i32* %ptr)
-       %base-new = call i32* @llvm.experimental.gc.relocate.p0i32(i32 %tok, i32 4, i32 4)
-       %ptr-new = call i32* @llvm.experimental.gc.relocate.p0i32(i32 %tok, i32 4, i32 5)
+       %tok = call i32 (i64, i32, i1 ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_i1f(i64 0, i32 0, i1 ()* @return_i1, i32 0, i32 0, i32 0, i32 0, i32* %base, i32* %ptr)
+       %base-new = call i32* @llvm.experimental.gc.relocate.p0i32(i32 %tok,  i32 7, i32 7)
+       %ptr-new = call i32* @llvm.experimental.gc.relocate.p0i32(i32 %tok, i32 7, i32 8)
        %ret = load i32, i32* %ptr-new
        ret i32 %ret
 }
@@ -20,15 +20,15 @@ entry:
 define i32 @test_sor_two_derived(i32* %base) gc "statepoint-example" {
 ; CHECK: getelementptr i32, i32* %base, i32 15
 ; CHECK: getelementptr i32, i32* %base, i32 12
-; CHECK: getelementptr i32, i32* %base-new, i32 15
 ; CHECK: getelementptr i32, i32* %base-new, i32 12
+; CHECK: getelementptr i32, i32* %base-new, i32 15
 entry:
        %ptr = getelementptr i32, i32* %base, i32 15
        %ptr2 = getelementptr i32, i32* %base, i32 12
-       %tok = call i32 (i1 ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_i1f(i1 ()* @return_i1, i32 0, i32 0, i32 0, i32* %base, i32* %ptr, i32* %ptr2)
-       %base-new = call i32* @llvm.experimental.gc.relocate.p0i32(i32 %tok, i32 4, i32 4)
-       %ptr-new = call i32* @llvm.experimental.gc.relocate.p0i32(i32 %tok, i32 4, i32 5)
-       %ptr2-new = call i32* @llvm.experimental.gc.relocate.p0i32(i32 %tok, i32 4, i32 6)
+       %tok = call i32 (i64, i32, i1 ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_i1f(i64 0, i32 0, i1 ()* @return_i1, i32 0, i32 0, i32 0, i32 0, i32* %base, i32* %ptr, i32* %ptr2)
+       %base-new = call i32* @llvm.experimental.gc.relocate.p0i32(i32 %tok,  i32 7, i32 7)
+       %ptr-new = call i32* @llvm.experimental.gc.relocate.p0i32(i32 %tok, i32 7, i32 8)
+       %ptr2-new = call i32* @llvm.experimental.gc.relocate.p0i32(i32 %tok, i32 7, i32 9)
        %ret = load i32, i32* %ptr-new
        ret i32 %ret
 }
@@ -38,9 +38,9 @@ define i32 @test_sor_ooo(i32* %base) gc "statepoint-example" {
 ; CHECK: getelementptr i32, i32* %base-new, i32 15
 entry:
        %ptr = getelementptr i32, i32* %base, i32 15
-       %tok = call i32 (i1 ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_i1f(i1 ()* @return_i1, i32 0, i32 0, i32 0, i32* %base, i32* %ptr)
-       %ptr-new = call i32* @llvm.experimental.gc.relocate.p0i32(i32 %tok, i32 4, i32 5)
-       %base-new = call i32* @llvm.experimental.gc.relocate.p0i32(i32 %tok, i32 4, i32 4)
+       %tok = call i32 (i64, i32, i1 ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_i1f(i64 0, i32 0, i1 ()* @return_i1, i32 0, i32 0, i32 0, i32 0, i32* %base, i32* %ptr)
+       %ptr-new = call i32* @llvm.experimental.gc.relocate.p0i32(i32 %tok, i32 7, i32 8)
+       %base-new = call i32* @llvm.experimental.gc.relocate.p0i32(i32 %tok,  i32 7, i32 7)
        %ret = load i32, i32* %ptr-new
        ret i32 %ret
 }
@@ -50,9 +50,9 @@ define i32 @test_sor_gep_smallint([3 x i32]* %base) gc "statepoint-example" {
 ; CHECK: getelementptr [3 x i32], [3 x i32]* %base-new, i32 0, i32 2
 entry:
        %ptr = getelementptr [3 x i32], [3 x i32]* %base, i32 0, i32 2
-       %tok = call i32 (i1 ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_i1f(i1 ()* @return_i1, i32 0, i32 0, i32 0, [3 x i32]* %base, i32* %ptr)
-       %base-new = call [3 x i32]* @llvm.experimental.gc.relocate.p0a3i32(i32 %tok, i32 4, i32 4)
-       %ptr-new = call i32* @llvm.experimental.gc.relocate.p0i32(i32 %tok, i32 4, i32 5)
+       %tok = call i32 (i64, i32, i1 ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_i1f(i64 0, i32 0, i1 ()* @return_i1, i32 0, i32 0, i32 0, i32 0, [3 x i32]* %base, i32* %ptr)
+       %base-new = call [3 x i32]* @llvm.experimental.gc.relocate.p0a3i32(i32 %tok,  i32 7, i32 7)
+       %ptr-new = call i32* @llvm.experimental.gc.relocate.p0i32(i32 %tok, i32 7, i32 8)
        %ret = load i32, i32* %ptr-new
        ret i32 %ret
 }
@@ -62,27 +62,27 @@ define i32 @test_sor_gep_largeint([3 x i32]* %base) gc "statepoint-example" {
 ; CHECK-NOT: getelementptr [3 x i32], [3 x i32]* %base-new, i32 0, i32 21
 entry:
        %ptr = getelementptr [3 x i32], [3 x i32]* %base, i32 0, i32 21
-       %tok = call i32 (i1 ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_i1f(i1 ()* @return_i1, i32 0, i32 0, i32 0, [3 x i32]* %base, i32* %ptr)
-       %base-new = call [3 x i32]* @llvm.experimental.gc.relocate.p0a3i32(i32 %tok, i32 4, i32 4)
-       %ptr-new = call i32* @llvm.experimental.gc.relocate.p0i32(i32 %tok, i32 4, i32 5)
+       %tok = call i32 (i64, i32, i1 ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_i1f(i64 0, i32 0, i1 ()* @return_i1, i32 0, i32 0, i32 0, i32 0, [3 x i32]* %base, i32* %ptr)
+       %base-new = call [3 x i32]* @llvm.experimental.gc.relocate.p0a3i32(i32 %tok,  i32 7, i32 7)
+       %ptr-new = call i32* @llvm.experimental.gc.relocate.p0i32(i32 %tok, i32 7, i32 8)
        %ret = load i32, i32* %ptr-new
        ret i32 %ret
 }
 
 define i32 @test_sor_noop(i32* %base) gc "statepoint-example" {
 ; CHECK: getelementptr i32, i32* %base, i32 15
-; CHECK: call i32* @llvm.experimental.gc.relocate.p0i32(i32 %tok, i32 4, i32 5)
-; CHECK: call i32* @llvm.experimental.gc.relocate.p0i32(i32 %tok, i32 4, i32 6)
+; CHECK: call i32* @llvm.experimental.gc.relocate.p0i32(i32 %tok, i32 7, i32 8)
+; CHECK: call i32* @llvm.experimental.gc.relocate.p0i32(i32 %tok, i32 7, i32 9)
 entry:
        %ptr = getelementptr i32, i32* %base, i32 15
        %ptr2 = getelementptr i32, i32* %base, i32 12
-       %tok = call i32 (i1 ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_i1f(i1 ()* @return_i1, i32 0, i32 0, i32 0, i32* %base, i32* %ptr, i32* %ptr2)
-       %ptr-new = call i32* @llvm.experimental.gc.relocate.p0i32(i32 %tok, i32 4, i32 5)
-       %ptr2-new = call i32* @llvm.experimental.gc.relocate.p0i32(i32 %tok, i32 4, i32 6)
+       %tok = call i32 (i64, i32, i1 ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_i1f(i64 0, i32 0, i1 ()* @return_i1, i32 0, i32 0, i32 0, i32 0, i32* %base, i32* %ptr, i32* %ptr2)
+       %ptr-new = call i32* @llvm.experimental.gc.relocate.p0i32(i32 %tok, i32 7, i32 8)
+       %ptr2-new = call i32* @llvm.experimental.gc.relocate.p0i32(i32 %tok, i32 7, i32 9)
        %ret = load i32, i32* %ptr-new
        ret i32 %ret
 }
 
-declare i32 @llvm.experimental.gc.statepoint.p0f_i1f(i1 ()*, i32, i32, ...)
+declare i32 @llvm.experimental.gc.statepoint.p0f_i1f(i64, i32, i1 ()*, i32, i32, ...)
 declare i32* @llvm.experimental.gc.relocate.p0i32(i32, i32, i32)
 declare [3 x i32]* @llvm.experimental.gc.relocate.p0a3i32(i32, i32, i32)
diff --git a/test/Transforms/ConstProp/InsertElement.ll b/test/Transforms/ConstProp/InsertElement.ll
index d249c2eb1c5..011ad3f57f4 100644
--- a/test/Transforms/ConstProp/InsertElement.ll
+++ b/test/Transforms/ConstProp/InsertElement.ll
@@ -1,12 +1,32 @@
 ; RUN: opt < %s -constprop -S | FileCheck %s
 
+; CHECK-LABEL: @test1
 define i32 @test1() {
   %A = bitcast i32 2139171423 to float
   %B = insertelement <1 x float> undef, float %A, i32 0
   %C = extractelement <1 x float> %B, i32 0
   %D = bitcast float %C to i32
   ret i32 %D
-; CHECK: @test1
 ; CHECK: ret i32 2139171423
 }
 
+; CHECK-LABEL: @insertelement
+define <4 x i64> @insertelement() {
+  %vec1 = insertelement <4 x i64> undef, i64 -1, i32 0
+  %vec2 = insertelement <4 x i64> %vec1, i64 -2, i32 1
+  %vec3 = insertelement <4 x i64> %vec2, i64 -3, i32 2
+  %vec4 = insertelement <4 x i64> %vec3, i64 -4, i32 3
+  ; CHECK: ret <4 x i64> <i64 -1, i64 -2, i64 -3, i64 -4>
+  ret <4 x i64> %vec4
+}
+
+; CHECK-LABEL: @insertelement_undef
+define <4 x i64> @insertelement_undef() {
+  %vec1 = insertelement <4 x i64> undef, i64 -1, i32 0
+  %vec2 = insertelement <4 x i64> %vec1, i64 -2, i32 1
+  %vec3 = insertelement <4 x i64> %vec2, i64 -3, i32 2
+  %vec4 = insertelement <4 x i64> %vec3, i64 -4, i32 3
+  %vec5 = insertelement <4 x i64> %vec3, i64 -5, i32 4
+  ; CHECK: ret <4 x i64> undef
+  ret <4 x i64> %vec5
+}
diff --git a/test/Transforms/ConstProp/convert-from-fp16.ll b/test/Transforms/ConstProp/convert-from-fp16.ll
new file mode 100644
index 00000000000..bb903231317
--- /dev/null
+++ b/test/Transforms/ConstProp/convert-from-fp16.ll
@@ -0,0 +1,97 @@
+; RUN: opt -constprop -S < %s | FileCheck %s
+
+; Verify that we don't crash with an assertion failure when constant folding
+; a call to intrinsic 'convert.from.fp16' if the return type is not 'float'.
+
+define float @fold_from_fp16_to_fp32() {
+; CHECK-LABEL: @fold_from_fp16_to_fp32
+; CHECK: ret float 0.000000e+00
+entry:
+  %0 = call float @llvm.convert.from.fp16.f32(i16 0)
+  ret float %0
+}
+
+define double @fold_from_fp16_to_fp64() {
+; CHECK-LABEL: @fold_from_fp16_to_fp64
+; CHECK: ret double 0.000000e+00
+entry:
+  %0 = call double @llvm.convert.from.fp16.f64(i16 0)
+  ret double %0
+}
+
+define x86_fp80 @fold_from_fp16_to_fp80() {
+; CHECK-LABEL: @fold_from_fp16_to_fp80
+; CHECK: ret x86_fp80 0xK00000000000000000000
+entry:
+  %0 = call x86_fp80 @llvm.convert.from.fp16.f80(i16 0)
+  ret x86_fp80 %0
+}
+
+define fp128 @fold_from_fp16_to_fp128() {
+; CHECK-LABEL: @fold_from_fp16_to_fp128
+; CHECK: ret fp128 0xL00000000000000000000000000000000
+entry:
+  %0 = call fp128 @llvm.convert.from.fp16.f128(i16 0)
+  ret fp128 %0
+}
+
+define ppc_fp128 @fold_from_fp16_to_ppcfp128() {
+; CHECK-LABEL: @fold_from_fp16_to_ppcfp128
+; CHECK: ret ppc_fp128 0xM00000000000000000000000000000000
+entry:
+  %0 = call ppc_fp128 @llvm.convert.from.fp16.ppcf128(i16 0)
+  ret ppc_fp128 %0
+}
+
+define float @fold_from_fp16_to_fp32_b() {
+; CHECK-LABEL: @fold_from_fp16_to_fp32_b
+; CHECK: ret float 4.000000e+00
+entry:
+  %0 = call i16 @llvm.convert.to.fp16.f64(double 4.0)
+  %1 = call float @llvm.convert.from.fp16.f32(i16 %0)
+  ret float %1
+}
+
+define double @fold_from_fp16_to_fp64_b() {
+; CHECK-LABEL: @fold_from_fp16_to_fp64_b
+; CHECK: ret double 4.000000e+00
+entry:
+  %0 = call i16 @llvm.convert.to.fp16.f64(double 4.0)
+  %1 = call double @llvm.convert.from.fp16.f64(i16 %0)
+  ret double %1
+}
+
+define x86_fp80 @fold_from_fp16_to_fp80_b() {
+; CHECK-LABEL: @fold_from_fp16_to_fp80_b
+; CHECK: ret x86_fp80 0xK40018000000000000000
+entry:
+  %0 = call i16 @llvm.convert.to.fp16.f64(double 4.0)
+  %1 = call x86_fp80 @llvm.convert.from.fp16.f80(i16 %0)
+  ret x86_fp80 %1
+}
+
+define fp128 @fold_from_fp16_to_fp128_b() {
+; CHECK-LABEL: @fold_from_fp16_to_fp128_b
+; CHECK: ret fp128 0xL00000000000000004001000000000000
+entry:
+  %0 = call i16 @llvm.convert.to.fp16.f64(double 4.0)
+  %1 = call fp128 @llvm.convert.from.fp16.f128(i16 %0)
+  ret fp128 %1
+}
+
+define ppc_fp128 @fold_from_fp16_to_ppcfp128_b() {
+; CHECK-LABEL: @fold_from_fp16_to_ppcfp128_b
+; CHECK: ret ppc_fp128 0xM40100000000000000000000000000000
+entry:
+  %0 = call i16 @llvm.convert.to.fp16.f64(double 4.0)
+  %1 = call ppc_fp128 @llvm.convert.from.fp16.ppcf128(i16 %0)
+  ret ppc_fp128 %1
+}
+
+
+declare i16 @llvm.convert.to.fp16.f64(double)
+declare float @llvm.convert.from.fp16.f32(i16)
+declare double @llvm.convert.from.fp16.f64(i16)
+declare x86_fp80 @llvm.convert.from.fp16.f80(i16)
+declare fp128 @llvm.convert.from.fp16.f128(i16)
+declare ppc_fp128 @llvm.convert.from.fp16.ppcf128(i16)
diff --git a/test/Transforms/ConstProp/loads.ll b/test/Transforms/ConstProp/loads.ll
index 5426ad0f8ad..89387ad06ba 100644
--- a/test/Transforms/ConstProp/loads.ll
+++ b/test/Transforms/ConstProp/loads.ll
@@ -40,13 +40,16 @@ define i16 @test2_addrspacecast() {
   %r = load i16, i16 addrspace(1)* addrspacecast(i32* getelementptr ({{i32,i8},i32}, {{i32,i8},i32}* @g1, i32 0, i32 0, i32 0) to i16 addrspace(1)*)
   ret i16 %r
 
+; FIXME: Should be able to load through a constant addrspacecast.
 ; 0xBEEF
 ; LE-LABEL: @test2_addrspacecast(
-; LE: ret i16 -16657
+; XLE: ret i16 -16657
+; LE: load i16, i16 addrspace(1)* addrspacecast
 
 ; 0xDEAD
 ; BE-LABEL: @test2_addrspacecast(
-; BE: ret i16 -8531
+; XBE: ret i16 -8531
+; BE: load i16, i16 addrspace(1)* addrspacecast
 }
 
 ; Load of second 16 bits of 32-bit value.
diff --git a/test/Transforms/CorrelatedValuePropagation/select.ll b/test/Transforms/CorrelatedValuePropagation/select.ll
new file mode 100644
index 00000000000..d88e3e462a2
--- /dev/null
+++ b/test/Transforms/CorrelatedValuePropagation/select.ll
@@ -0,0 +1,75 @@
+; RUN: opt < %s -correlated-propagation -S | FileCheck %s
+
+; CHECK-LABEL: @simple(
+define i8 @simple(i1) {
+entry:
+  %s = select i1 %0, i8 0, i8 1
+  br i1 %0, label %then, label %else
+
+then:
+; CHECK: ret i8 0
+  %a = phi i8 [ %s, %entry ]
+  ret i8 %a
+
+else:
+; CHECK: ret i8 1
+  %b = phi i8 [ %s, %entry ]
+  ret i8 %b
+}
+
+; CHECK-LABEL: @loop(
+define void @loop(i32) {
+entry:
+  br label %loop
+
+loop:
+  %idx = phi i32 [ %0, %entry ], [ %sel, %loop ]
+; CHECK: %idx = phi i32 [ %0, %entry ], [ %2, %loop ]
+  %1 = icmp eq i32 %idx, 0
+  %2 = add i32 %idx, -1
+  %sel = select i1 %1, i32 0, i32 %2
+  br i1 %1, label %out, label %loop
+
+out:
+  ret void
+}
+
+; CHECK-LABEL: @not_correlated(
+define i8 @not_correlated(i1, i1) {
+entry:
+  %s = select i1 %0, i8 0, i8 1
+  br i1 %1, label %then, label %else
+
+then:
+; CHECK: ret i8 %s
+  %a = phi i8 [ %s, %entry ]
+  ret i8 %a
+
+else:
+; CHECK: ret i8 %s
+  %b = phi i8 [ %s, %entry ]
+  ret i8 %b
+}
+
+@c = global i32 0, align 4
+@b = global i32 0, align 4
+
+; CHECK-LABEL: @PR23752(
+define i32 @PR23752() {
+entry:
+  br label %for.body
+
+for.body:
+  %phi = phi i32 [ 0, %entry ], [ %sel, %for.body ]
+  %sel = select i1 icmp sgt (i32* @b, i32* @c), i32 %phi, i32 1
+  %cmp = icmp ne i32 %sel, 1
+  br i1 %cmp, label %for.body, label %if.end
+
+; CHECK:      %[[sel:.*]] = select i1 icmp sgt (i32* @b, i32* @c), i32 0, i32 1
+; CHECK-NEXT: %[[cmp:.*]] = icmp ne i32 %[[sel]], 1
+; CHECK-NEXT: br i1 %[[cmp]]
+
+if.end:
+  ret i32 %sel
+; CHECK: ret i32 %[[sel]]
+}
diff --git a/test/Transforms/DeadArgElim/2009-03-17-MRE-Invoke.ll b/test/Transforms/DeadArgElim/2009-03-17-MRE-Invoke.ll
index fc25daca1c2..4adae850e78 100644
--- a/test/Transforms/DeadArgElim/2009-03-17-MRE-Invoke.ll
+++ b/test/Transforms/DeadArgElim/2009-03-17-MRE-Invoke.ll
@@ -5,18 +5,18 @@ define internal { i32, i32 } @foo() {
   ret {i32,i32} {i32 42, i32 4}
 }
 
-define i32 @bar() {
+define i32 @bar() personality i32 (...)* @__gxx_personality_v0 {
   %x = invoke {i32,i32} @foo() to label %T unwind label %T2
 T:
   %y = extractvalue {i32,i32} %x, 1
   ret i32 %y
 T2:
-  %exn = landingpad {i8*, i32} personality i32 (...)* @__gxx_personality_v0
+  %exn = landingpad {i8*, i32}
             cleanup
   unreachable
 }
 
-define i32 @bar2() {
+define i32 @bar2() personality i32 (...)* @__gxx_personality_v0 {
 entry:
   %x = invoke {i32,i32} @foo() to label %T unwind label %T2
 T:
@@ -24,7 +24,7 @@ T:
   %y = extractvalue {i32,i32} %x, 1
   ret i32 %y
 T2:
-  %exn = landingpad {i8*, i32} personality i32 (...)* @__gxx_personality_v0
+  %exn = landingpad {i8*, i32}
             cleanup
   unreachable
 }
diff --git a/test/Transforms/DeadArgElim/2010-04-30-DbgInfo.ll b/test/Transforms/DeadArgElim/2010-04-30-DbgInfo.ll
index b834a7509de..bd9704eeef4 100644
--- a/test/Transforms/DeadArgElim/2010-04-30-DbgInfo.ll
+++ b/test/Transforms/DeadArgElim/2010-04-30-DbgInfo.ll
@@ -4,10 +4,10 @@
 
 define i8* @vfs_addname(i8* %name, i32 %len, i32 %hash, i32 %flags) nounwind ssp {
 entry:
-  call void @llvm.dbg.value(metadata i8* %name, i64 0, metadata !0, metadata !MDExpression()), !dbg !MDLocation(scope: !1)
-  call void @llvm.dbg.value(metadata i32 %len, i64 0, metadata !10, metadata !MDExpression()), !dbg !MDLocation(scope: !1)
-  call void @llvm.dbg.value(metadata i32 %hash, i64 0, metadata !11, metadata !MDExpression()), !dbg !MDLocation(scope: !1)
-  call void @llvm.dbg.value(metadata i32 %flags, i64 0, metadata !12, metadata !MDExpression()), !dbg !MDLocation(scope: !1)
+  call void @llvm.dbg.value(metadata i8* %name, i64 0, metadata !0, metadata !DIExpression()), !dbg !DILocation(scope: !1)
+  call void @llvm.dbg.value(metadata i32 %len, i64 0, metadata !10, metadata !DIExpression()), !dbg !DILocation(scope: !1)
+  call void @llvm.dbg.value(metadata i32 %hash, i64 0, metadata !11, metadata !DIExpression()), !dbg !DILocation(scope: !1)
+  call void @llvm.dbg.value(metadata i32 %flags, i64 0, metadata !12, metadata !DIExpression()), !dbg !DILocation(scope: !1)
 ; CHECK:  call fastcc i8* @add_name_internal(i8* %name, i32 %hash) [[NUW:#[0-9]+]], !dbg !{{[0-9]+}}
   %0 = call fastcc i8* @add_name_internal(i8* %name, i32 %len, i32 %hash, i8 zeroext 0, i32 %flags) nounwind, !dbg !13 ; <i8*> [#uses=1]
   ret i8* %0, !dbg !13
@@ -17,11 +17,11 @@ declare void @llvm.dbg.declare(metadata, metadata, metadata) nounwind readnone
 
 define internal fastcc i8* @add_name_internal(i8* %name, i32 %len, i32 %hash, i8 zeroext %extra, i32 %flags) noinline nounwind ssp {
 entry:
-  call void @llvm.dbg.value(metadata i8* %name, i64 0, metadata !15, metadata !MDExpression()), !dbg !MDLocation(scope: !16)
-  call void @llvm.dbg.value(metadata i32 %len, i64 0, metadata !20, metadata !MDExpression()), !dbg !MDLocation(scope: !16)
-  call void @llvm.dbg.value(metadata i32 %hash, i64 0, metadata !21, metadata !MDExpression()), !dbg !MDLocation(scope: !16)
-  call void @llvm.dbg.value(metadata i8 %extra, i64 0, metadata !22, metadata !MDExpression()), !dbg !MDLocation(scope: !16)
-  call void @llvm.dbg.value(metadata i32 %flags, i64 0, metadata !23, metadata !MDExpression()), !dbg !MDLocation(scope: !16)
+  call void @llvm.dbg.value(metadata i8* %name, i64 0, metadata !15, metadata !DIExpression()), !dbg !DILocation(scope: !16)
+  call void @llvm.dbg.value(metadata i32 %len, i64 0, metadata !20, metadata !DIExpression()), !dbg !DILocation(scope: !16)
+  call void @llvm.dbg.value(metadata i32 %hash, i64 0, metadata !21, metadata !DIExpression()), !dbg !DILocation(scope: !16)
+  call void @llvm.dbg.value(metadata i8 %extra, i64 0, metadata !22, metadata !DIExpression()), !dbg !DILocation(scope: !16)
+  call void @llvm.dbg.value(metadata i32 %flags, i64 0, metadata !23, metadata !DIExpression()), !dbg !DILocation(scope: !16)
   %0 = icmp eq i32 %hash, 0, !dbg !24             ; <i1> [#uses=1]
   br i1 %0, label %bb, label %bb1, !dbg !24
 
@@ -45,34 +45,34 @@ declare void @llvm.dbg.value(metadata, i64, metadata, metadata) nounwind readnon
 
 !llvm.dbg.cu = !{!3}
 !llvm.module.flags = !{!30}
-!0 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "name", line: 8, arg: 0, scope: !1, file: !2, type: !6)
-!1 = !MDSubprogram(name: "vfs_addname", linkageName: "vfs_addname", line: 12, isLocal: false, isDefinition: true, virtualIndex: 6, isOptimized: false, file: !28, scope: !2, type: !4)
-!2 = !MDFile(filename: "tail.c", directory: "/Users/echeng/LLVM/radars/r7927803/")
-!3 = !MDCompileUnit(language: DW_LANG_C89, producer: "4.2.1 (Based on Apple Inc. build 5658) (LLVM build 9999)", isOptimized: true, emissionKind: 0, file: !28, enums: !29, retainedTypes: !29)
-!4 = !MDSubroutineType(types: !5)
+!0 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "name", line: 8, arg: 1, scope: !1, file: !2, type: !6)
+!1 = !DISubprogram(name: "vfs_addname", linkageName: "vfs_addname", line: 12, isLocal: false, isDefinition: true, virtualIndex: 6, isOptimized: false, file: !28, scope: !2, type: !4)
+!2 = !DIFile(filename: "tail.c", directory: "/Users/echeng/LLVM/radars/r7927803/")
+!3 = !DICompileUnit(language: DW_LANG_C89, producer: "4.2.1 (Based on Apple Inc. build 5658) (LLVM build 9999)", isOptimized: true, emissionKind: 0, file: !28, enums: !29, retainedTypes: !29)
+!4 = !DISubroutineType(types: !5)
 !5 = !{!6, !6, !9, !9, !9}
-!6 = !MDDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, file: !28, scope: !2, baseType: !7)
-!7 = !MDDerivedType(tag: DW_TAG_const_type, size: 8, align: 8, file: !28, scope: !2, baseType: !8)
-!8 = !MDBasicType(tag: DW_TAG_base_type, name: "char", size: 8, align: 8, encoding: DW_ATE_signed_char)
-!9 = !MDBasicType(tag: DW_TAG_base_type, name: "unsigned int", size: 32, align: 32, encoding: DW_ATE_unsigned)
-!10 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "len", line: 9, arg: 0, scope: !1, file: !2, type: !9)
-!11 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "hash", line: 10, arg: 0, scope: !1, file: !2, type: !9)
-!12 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "flags", line: 11, arg: 0, scope: !1, file: !2, type: !9)
-!13 = !MDLocation(line: 13, scope: !14)
-!14 = distinct !MDLexicalBlock(line: 12, column: 0, file: !28, scope: !1)
-!15 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "name", line: 17, arg: 0, scope: !16, file: !2, type: !6)
-!16 = !MDSubprogram(name: "add_name_internal", linkageName: "add_name_internal", line: 22, isLocal: true, isDefinition: true, virtualIndex: 6, isOptimized: false, file: !28, scope: !2, type: !17)
-!17 = !MDSubroutineType(types: !18)
+!6 = !DIDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, file: !28, scope: !2, baseType: !7)
+!7 = !DIDerivedType(tag: DW_TAG_const_type, size: 8, align: 8, file: !28, scope: !2, baseType: !8)
+!8 = !DIBasicType(tag: DW_TAG_base_type, name: "char", size: 8, align: 8, encoding: DW_ATE_signed_char)
+!9 = !DIBasicType(tag: DW_TAG_base_type, name: "unsigned int", size: 32, align: 32, encoding: DW_ATE_unsigned)
+!10 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "len", line: 9, arg: 2, scope: !1, file: !2, type: !9)
+!11 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "hash", line: 10, arg: 3, scope: !1, file: !2, type: !9)
+!12 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "flags", line: 11, arg: 4, scope: !1, file: !2, type: !9)
+!13 = !DILocation(line: 13, scope: !14)
+!14 = distinct !DILexicalBlock(line: 12, column: 0, file: !28, scope: !1)
+!15 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "name", line: 17, arg: 1, scope: !16, file: !2, type: !6)
+!16 = !DISubprogram(name: "add_name_internal", linkageName: "add_name_internal", line: 22, isLocal: true, isDefinition: true, virtualIndex: 6, isOptimized: false, file: !28, scope: !2, type: !17)
+!17 = !DISubroutineType(types: !18)
 !18 = !{!6, !6, !9, !9, !19, !9}
-!19 = !MDBasicType(tag: DW_TAG_base_type, name: "unsigned char", size: 8, align: 8, encoding: DW_ATE_unsigned_char)
-!20 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "len", line: 18, arg: 0, scope: !16, file: !2, type: !9)
-!21 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "hash", line: 19, arg: 0, scope: !16, file: !2, type: !9)
-!22 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "extra", line: 20, arg: 0, scope: !16, file: !2, type: !19)
-!23 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "flags", line: 21, arg: 0, scope: !16, file: !2, type: !9)
-!24 = !MDLocation(line: 23, scope: !25)
-!25 = distinct !MDLexicalBlock(line: 22, column: 0, file: !28, scope: !16)
-!26 = !MDLocation(line: 24, scope: !25)
-!27 = !MDLocation(line: 26, scope: !25)
-!28 = !MDFile(filename: "tail.c", directory: "/Users/echeng/LLVM/radars/r7927803/")
+!19 = !DIBasicType(tag: DW_TAG_base_type, name: "unsigned char", size: 8, align: 8, encoding: DW_ATE_unsigned_char)
+!20 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "len", line: 18, arg: 2, scope: !16, file: !2, type: !9)
+!21 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "hash", line: 19, arg: 3, scope: !16, file: !2, type: !9)
+!22 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "extra", line: 20, arg: 4, scope: !16, file: !2, type: !19)
+!23 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "flags", line: 21, arg: 5, scope: !16, file: !2, type: !9)
+!24 = !DILocation(line: 23, scope: !25)
+!25 = distinct !DILexicalBlock(line: 22, column: 0, file: !28, scope: !16)
+!26 = !DILocation(line: 24, scope: !25)
+!27 = !DILocation(line: 26, scope: !25)
+!28 = !DIFile(filename: "tail.c", directory: "/Users/echeng/LLVM/radars/r7927803/")
 !29 = !{}
 !30 = !{i32 1, !"Debug Info Version", i32 3}
diff --git a/test/Transforms/DeadArgElim/dbginfo.ll b/test/Transforms/DeadArgElim/dbginfo.ll
index 95e18e1c613..efafd9bbc9e 100644
--- a/test/Transforms/DeadArgElim/dbginfo.ll
+++ b/test/Transforms/DeadArgElim/dbginfo.ll
@@ -14,7 +14,7 @@
 ; the function->debug info mapping on update to ensure it's accurate when used
 ; again for the next removal.
 
-; CHECK: !MDSubprogram(name: "f1",{{.*}} function: void ()* @_ZL2f1iz
+; CHECK: !DISubprogram(name: "f1",{{.*}} function: void ()* @_ZL2f1iz
 
 ; Check that debug info metadata for subprograms stores pointers to
 ; updated LLVM functions.
@@ -47,24 +47,24 @@ attributes #2 = { nounwind readnone }
 !llvm.module.flags = !{!12, !13}
 !llvm.ident = !{!14}
 
-!0 = !MDCompileUnit(language: DW_LANG_C_plus_plus, producer: "clang version 3.6.0 ", isOptimized: false, emissionKind: 1, file: !1, enums: !2, retainedTypes: !2, subprograms: !3, globals: !2, imports: !2)
-!1 = !MDFile(filename: "dbg.cpp", directory: "/tmp/dbginfo")
+!0 = !DICompileUnit(language: DW_LANG_C_plus_plus, producer: "clang version 3.6.0 ", isOptimized: false, emissionKind: 1, file: !1, enums: !2, retainedTypes: !2, subprograms: !3, globals: !2, imports: !2)
+!1 = !DIFile(filename: "dbg.cpp", directory: "/tmp/dbginfo")
 !2 = !{}
 !3 = !{!4, !8}
-!4 = !MDSubprogram(name: "f2", linkageName: "_Z2f2v", line: 4, isLocal: false, isDefinition: true, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 4, file: !1, scope: !5, type: !6, function: void ()* @_Z2f2v, variables: !2)
-!5 = !MDFile(filename: "dbg.cpp", directory: "/tmp/dbginfo")
-!6 = !MDSubroutineType(types: !7)
+!4 = !DISubprogram(name: "f2", linkageName: "_Z2f2v", line: 4, isLocal: false, isDefinition: true, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 4, file: !1, scope: !5, type: !6, function: void ()* @_Z2f2v, variables: !2)
+!5 = !DIFile(filename: "dbg.cpp", directory: "/tmp/dbginfo")
+!6 = !DISubroutineType(types: !7)
 !7 = !{null}
-!8 = !MDSubprogram(name: "f1", linkageName: "_ZL2f1iz", line: 1, isLocal: true, isDefinition: true, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 1, file: !1, scope: !5, type: !9, function: void (i32, ...)* @_ZL2f1iz, variables: !2)
-!9 = !MDSubroutineType(types: !10)
+!8 = !DISubprogram(name: "f1", linkageName: "_ZL2f1iz", line: 1, isLocal: true, isDefinition: true, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 1, file: !1, scope: !5, type: !9, function: void (i32, ...)* @_ZL2f1iz, variables: !2)
+!9 = !DISubroutineType(types: !10)
 !10 = !{null, !11, null}
-!11 = !MDBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
+!11 = !DIBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
 !12 = !{i32 2, !"Dwarf Version", i32 4}
 !13 = !{i32 2, !"Debug Info Version", i32 3}
 !14 = !{!"clang version 3.6.0 "}
-!15 = !MDLocation(line: 5, column: 3, scope: !4)
-!16 = !MDLocation(line: 6, column: 1, scope: !4)
-!17 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "", line: 1, arg: 1, scope: !8, file: !5, type: !11)
-!18 = !MDExpression()
-!19 = !MDLocation(line: 1, column: 19, scope: !8)
-!20 = !MDLocation(line: 2, column: 1, scope: !8)
+!15 = !DILocation(line: 5, column: 3, scope: !4)
+!16 = !DILocation(line: 6, column: 1, scope: !4)
+!17 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "", line: 1, arg: 1, scope: !8, file: !5, type: !11)
+!18 = !DIExpression()
+!19 = !DILocation(line: 1, column: 19, scope: !8)
+!20 = !DILocation(line: 2, column: 1, scope: !8)
diff --git a/test/Transforms/DeadStoreElimination/inst-limits.ll b/test/Transforms/DeadStoreElimination/inst-limits.ll
index 87de2ae3ebd..54e41c8b413 100644
--- a/test/Transforms/DeadStoreElimination/inst-limits.ll
+++ b/test/Transforms/DeadStoreElimination/inst-limits.ll
@@ -118,7 +118,7 @@ entry:
 
   ; Insert a meaningless dbg.value intrinsic; it should have no
   ; effect on the working of DSE in any way.
-  call void @llvm.dbg.value(metadata i32* undef, i64 0, metadata !10, metadata !MDExpression()), !dbg !MDLocation(scope: !4)
+  call void @llvm.dbg.value(metadata i32* undef, i64 0, metadata !10, metadata !DIExpression()), !dbg !DILocation(scope: !4)
 
   ; CHECK:  store i32 -1, i32* @x, align 4
   store i32 -1, i32* @x, align 4
@@ -245,17 +245,17 @@ declare void @llvm.dbg.value(metadata, i64, metadata, metadata)
 !llvm.dbg.cu = !{!0}
 !llvm.module.flags = !{!11, !13}
 
-!0 = !MDCompileUnit(language: DW_LANG_C_plus_plus, producer: "clang version 3.4", isOptimized: true, emissionKind: 0, file: !1, enums: !2, retainedTypes: !2, subprograms: !3, globals: !2, imports: !2)
-!1 = !MDFile(filename: "test.c", directory: "/home/tmp")
+!0 = !DICompileUnit(language: DW_LANG_C_plus_plus, producer: "clang version 3.4", isOptimized: true, emissionKind: 0, file: !1, enums: !2, retainedTypes: !2, subprograms: !3, globals: !2, imports: !2)
+!1 = !DIFile(filename: "test.c", directory: "/home/tmp")
 !2 = !{}
 !3 = !{!4}
-!4 = !MDSubprogram(name: "test_within_limit", line: 3, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 4, file: !1, scope: !5, type: !6, function: i32 ()* @test_within_limit, variables: !2)
-!5 = !MDFile(filename: "test.c", directory: "/home/tmp")
-!6 = !MDSubroutineType(types: !7)
+!4 = !DISubprogram(name: "test_within_limit", line: 3, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 4, file: !1, scope: !5, type: !6, function: i32 ()* @test_within_limit, variables: !2)
+!5 = !DIFile(filename: "test.c", directory: "/home/tmp")
+!6 = !DISubroutineType(types: !7)
 !7 = !{!8}
-!8 = !MDBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
+!8 = !DIBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
 !9 = !{!10}
-!10 = !MDLocalVariable(tag: DW_TAG_auto_variable, name: "x", scope: !4, type: !8)
+!10 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "x", scope: !4, type: !8)
 !11 = !{i32 2, !"Dwarf Version", i32 4}
 !12 = !{i32* undef}
 
diff --git a/test/Transforms/EarlyCSE/conditional.ll b/test/Transforms/EarlyCSE/conditional.ll
new file mode 100644
index 00000000000..5b5a74f112b
--- /dev/null
+++ b/test/Transforms/EarlyCSE/conditional.ll
@@ -0,0 +1,107 @@
+; RUN: opt -early-cse -S < %s | FileCheck %s
+
+; Can we CSE a known condition to a constant?
+define i1 @test(i8* %p) {
+; CHECK-LABEL: @test
+entry:
+  %cnd1 = icmp eq i8* %p, null
+  br i1 %cnd1, label %taken, label %untaken
+
+taken:
+; CHECK-LABEL: taken:
+; CHECK-NEXT: ret i1 true
+  %cnd2 = icmp eq i8* %p, null
+  ret i1 %cnd2
+
+untaken:
+; CHECK-LABEL: untaken:
+; CHECK-NEXT: ret i1 false
+  %cnd3 = icmp eq i8* %p, null
+  ret i1 %cnd3
+}
+
+; We can CSE the condition, but we *don't* know it's value after the merge
+define i1 @test_neg1(i8* %p) {
+; CHECK-LABEL: @test_neg1
+entry:
+  %cnd1 = icmp eq i8* %p, null
+  br i1 %cnd1, label %taken, label %untaken
+
+taken:
+  br label %merge
+
+untaken:
+  br label %merge
+
+merge:
+; CHECK-LABEL: merge:
+; CHECK-NEXT: ret i1 %cnd1
+  %cnd3 = icmp eq i8* %p, null
+  ret i1 %cnd3
+}
+
+; Check specifically for a case where we have a unique predecessor, but
+; not a single predecessor.  We can not know the value of the condition here.
+define i1 @test_neg2(i8* %p) {
+; CHECK-LABEL: @test_neg2
+entry:
+  %cnd1 = icmp eq i8* %p, null
+  br i1 %cnd1, label %merge, label %merge
+
+merge:
+; CHECK-LABEL: merge:
+; CHECK-NEXT: ret i1 %cnd1
+  %cnd3 = icmp eq i8* %p, null
+  ret i1 %cnd3
+}
+
+; Replace a use rather than CSE
+define i1 @test2(i8* %p) {
+; CHECK-LABEL: @test2
+entry:
+  %cnd = icmp eq i8* %p, null
+  br i1 %cnd, label %taken, label %untaken
+
+taken:
+; CHECK-LABEL: taken:
+; CHECK-NEXT: ret i1 true
+  ret i1 %cnd
+
+untaken:
+; CHECK-LABEL: untaken:
+; CHECK-NEXT: ret i1 false
+  ret i1 %cnd
+}
+
+; Not legal to replace use given it's not dominated by edge
+define i1 @test2_neg1(i8* %p) {
+; CHECK-LABEL: @test2_neg1
+entry:
+  %cnd1 = icmp eq i8* %p, null
+  br i1 %cnd1, label %taken, label %untaken
+
+taken:
+  br label %merge
+
+untaken:
+  br label %merge
+
+merge:
+; CHECK-LABEL: merge:
+; CHECK-NEXT: ret i1 %cnd1
+  ret i1 %cnd1
+}
+
+; Another single predecessor test, but for dominated use
+define i1 @test2_neg2(i8* %p) {
+; CHECK-LABEL: @test2_neg2
+entry:
+  %cnd1 = icmp eq i8* %p, null
+  br i1 %cnd1, label %merge, label %merge
+
+merge:
+; CHECK-LABEL: merge:
+; CHECK-NEXT: ret i1 %cnd1
+  ret i1 %cnd1
+}
+
diff --git a/test/Transforms/EarlyCSE/edge.ll b/test/Transforms/EarlyCSE/edge.ll
new file mode 100644
index 00000000000..267db679967
--- /dev/null
+++ b/test/Transforms/EarlyCSE/edge.ll
@@ -0,0 +1,173 @@
+; RUN: opt -early-cse -S < %s | FileCheck %s
+; Same as GVN/edge.ll, but updated to reflect EarlyCSE's less powerful
+; implementation.  EarlyCSE currently doesn't exploit equality comparisons
+; against constants.
+
+define i32 @f1(i32 %x) {
+  ; CHECK-LABEL: define i32 @f1(
+bb0:
+  %cmp = icmp eq i32 %x, 0
+  br i1 %cmp, label %bb2, label %bb1
+bb1:
+  br label %bb2
+bb2:
+  %cond = phi i32 [ %x, %bb0 ], [ 0, %bb1 ]
+  %foo = add i32 %cond, %x
+  ret i32 %foo
+  ; CHECK: bb2:
+  ; CHECK: %cond = phi i32 [ %x, %bb0 ], [ 0, %bb1 ]
+}
+
+define i32 @f2(i32 %x) {
+  ; CHECK-LABEL: define i32 @f2(
+bb0:
+  %cmp = icmp ne i32 %x, 0
+  br i1 %cmp, label %bb1, label %bb2
+bb1:
+  br label %bb2
+bb2:
+  %cond = phi i32 [ %x, %bb0 ], [ 0, %bb1 ]
+  %foo = add i32 %cond, %x
+  ret i32 %foo
+  ; CHECK: bb2:
+  ; CHECK: %cond = phi i32 [ %x, %bb0 ], [ 0, %bb1 ]
+}
+
+define i32 @f3(i32 %x) {
+  ; CHECK-LABEL: define i32 @f3(
+bb0:
+  switch i32 %x, label %bb1 [ i32 0, label %bb2]
+bb1:
+  br label %bb2
+bb2:
+  %cond = phi i32 [ %x, %bb0 ], [ 0, %bb1 ]
+  %foo = add i32 %cond, %x
+  ret i32 %foo
+  ; CHECK: bb2:
+  ; CHECK: %cond = phi i32 [ %x, %bb0 ], [ 0, %bb1 ]
+}
+
+declare void @g(i1)
+define void @f4(i8 * %x)  {
+; CHECK-LABEL: define void @f4(
+bb0:
+  %y = icmp eq i8* null, %x
+  br i1 %y, label %bb2, label %bb1
+bb1:
+  br label %bb2
+bb2:
+  %zed = icmp eq i8* null, %x
+  call void @g(i1 %zed)
+; CHECK: call void @g(i1 %y)
+  ret void
+}
+
+define double @fcmp_oeq_not_zero(double %x, double %y) {
+entry:
+  %cmp = fcmp oeq double %y, 2.0
+  br i1 %cmp, label %if, label %return
+
+if:
+  %div = fdiv double %x, %y
+  br label %return
+
+return:
+  %retval = phi double [ %div, %if ], [ %x, %entry ]
+  ret double %retval
+
+; CHECK-LABEL: define double @fcmp_oeq_not_zero(
+; CHECK: %div = fdiv double %x, %y
+}
+
+define double @fcmp_une_not_zero(double %x, double %y) {
+entry:
+  %cmp = fcmp une double %y, 2.0
+  br i1 %cmp, label %return, label %else
+
+else:
+  %div = fdiv double %x, %y
+  br label %return
+
+return:
+  %retval = phi double [ %div, %else ], [ %x, %entry ]
+  ret double %retval
+
+; CHECK-LABEL: define double @fcmp_une_not_zero(
+; CHECK: %div = fdiv double %x, %y
+}
+
+; PR22376 - We can't propagate zero constants because -0.0 
+; compares equal to 0.0. If %y is -0.0 in this test case,
+; we would produce the wrong sign on the infinity return value.
+define double @fcmp_oeq_zero(double %x, double %y) {
+entry:
+  %cmp = fcmp oeq double %y, 0.0
+  br i1 %cmp, label %if, label %return
+
+if:
+  %div = fdiv double %x, %y
+  br label %return
+
+return:
+  %retval = phi double [ %div, %if ], [ %x, %entry ]
+  ret double %retval
+
+; CHECK-LABEL: define double @fcmp_oeq_zero(
+; CHECK: %div = fdiv double %x, %y
+}
+
+define double @fcmp_une_zero(double %x, double %y) {
+entry:
+  %cmp = fcmp une double %y, -0.0
+  br i1 %cmp, label %return, label %else
+
+else:
+  %div = fdiv double %x, %y
+  br label %return
+
+return:
+  %retval = phi double [ %div, %else ], [ %x, %entry ]
+  ret double %retval
+
+; CHECK-LABEL: define double @fcmp_une_zero(
+; CHECK: %div = fdiv double %x, %y
+}
+
+; We also cannot propagate a value if it's not a constant.
+; This is because the value could be 0.0 or -0.0.
+
+define double @fcmp_oeq_maybe_zero(double %x, double %y, double %z1, double %z2) {
+entry:
+ %z = fadd double %z1, %z2
+ %cmp = fcmp oeq double %y, %z
+ br i1 %cmp, label %if, label %return
+
+if:
+ %div = fdiv double %x, %z
+ br label %return
+
+return:
+ %retval = phi double [ %div, %if ], [ %x, %entry ]
+ ret double %retval
+
+; CHECK-LABEL: define double @fcmp_oeq_maybe_zero(
+; CHECK: %div = fdiv double %x, %z
+}
+
+define double @fcmp_une_maybe_zero(double %x, double %y, double %z1, double %z2) {
+entry:
+ %z = fadd double %z1, %z2
+ %cmp = fcmp une double %y, %z
+ br i1 %cmp, label %return, label %else
+
+else:
+ %div = fdiv double %x, %z
+ br label %return
+
+return:
+ %retval = phi double [ %div, %else ], [ %x, %entry ]
+ ret double %retval
+
+; CHECK-LABEL: define double @fcmp_une_maybe_zero(
+; CHECK: %div = fdiv double %x, %z
+}
diff --git a/test/Transforms/EarlyCSE/read-reg.ll b/test/Transforms/EarlyCSE/read-reg.ll
new file mode 100644
index 00000000000..83a299912c7
--- /dev/null
+++ b/test/Transforms/EarlyCSE/read-reg.ll
@@ -0,0 +1,34 @@
+; RUN: opt -S -early-cse < %s | FileCheck %s
+target datalayout = "E-m:e-i64:64-n32:64"
+target triple = "powerpc64-unknown-linux-gnu"
+
+; Function Attrs: nounwind
+define i64 @f(i64 %x) #0 {
+entry:
+  %0 = call i64 @llvm.read_register.i64(metadata !0)
+  call void bitcast (void (...)* @foo to void ()*)()
+  %1 = call i64 @llvm.read_register.i64(metadata !0)
+  %add = add nsw i64 %0, %1
+  ret i64 %add
+}
+
+; CHECK-LABEL: @f
+; CHECK: call i64 @llvm.read_register.i64
+; CHECK: call i64 @llvm.read_register.i64
+
+; Function Attrs: nounwind readnone
+declare i64 @llvm.read_register.i64(metadata) #1
+
+; Function Attrs: nounwind
+declare void @llvm.write_register.i64(metadata, i64) #2
+
+declare void @foo(...)
+
+attributes #0 = { nounwind }
+attributes #1 = { nounwind readnone }
+attributes #2 = { nounwind }
+
+!llvm.named.register.r1 = !{!0}
+
+!0 = !{!"r1"}
+
diff --git a/test/Transforms/EliminateAvailableExternally/visibility.ll b/test/Transforms/EliminateAvailableExternally/visibility.ll
new file mode 100644
index 00000000000..9966fcf30e8
--- /dev/null
+++ b/test/Transforms/EliminateAvailableExternally/visibility.ll
@@ -0,0 +1,11 @@
+; RUN: opt -elim-avail-extern -S < %s | FileCheck %s
+
+; CHECK: declare hidden void @f()
+define available_externally hidden void @f() {
+  ret void
+}
+
+define void @g() {
+  call void @f()
+  ret void
+}
diff --git a/test/Transforms/FunctionAttrs/nocapture.ll b/test/Transforms/FunctionAttrs/nocapture.ll
index 55a3dc4aa35..4057b2a86e3 100644
--- a/test/Transforms/FunctionAttrs/nocapture.ll
+++ b/test/Transforms/FunctionAttrs/nocapture.ll
@@ -47,13 +47,13 @@ define i1 @c5(i32* %q, i32 %bitno) {
 declare void @throw_if_bit_set(i8*, i8) readonly
 
 ; CHECK: define i1 @c6(i8* readonly %q, i8 %bit)
-define i1 @c6(i8* %q, i8 %bit) {
+define i1 @c6(i8* %q, i8 %bit) personality i32 (...)* @__gxx_personality_v0 {
 	invoke void @throw_if_bit_set(i8* %q, i8 %bit)
 		to label %ret0 unwind label %ret1
 ret0:
 	ret i1 0
 ret1:
-        %exn = landingpad {i8*, i32} personality i32 (...)* @__gxx_personality_v0
+        %exn = landingpad {i8*, i32}
                  cleanup
 	ret i1 1
 }
diff --git a/test/Transforms/FunctionAttrs/readnone.ll b/test/Transforms/FunctionAttrs/readnone.ll
new file mode 100644
index 00000000000..eddcdd2641d
--- /dev/null
+++ b/test/Transforms/FunctionAttrs/readnone.ll
@@ -0,0 +1,13 @@
+; RUN: opt < %s -functionattrs -S | FileCheck %s
+
+; CHECK: define void @bar(i8* nocapture readnone)
+define void @bar(i8* readonly) {
+  call void @foo(i8* %0)
+    ret void
+}
+
+; CHECK: define void @foo(i8* nocapture readnone)
+define void @foo(i8* readonly) {
+  call void @bar(i8* %0)
+  ret void
+}
diff --git a/test/Transforms/GCOVProfiling/function-numbering.ll b/test/Transforms/GCOVProfiling/function-numbering.ll
index d39a7e17966..5a704e4d047 100644
--- a/test/Transforms/GCOVProfiling/function-numbering.ll
+++ b/test/Transforms/GCOVProfiling/function-numbering.ll
@@ -40,17 +40,17 @@ define void @baz() {
 !llvm.module.flags = !{!9, !10}
 !llvm.ident = !{!11}
 
-!0 = !MDCompileUnit(language: DW_LANG_C99, producer: "clang version 3.6.0 ", isOptimized: false, emissionKind: 2, file: !1, enums: !2, retainedTypes: !2, subprograms: !3, globals: !2, imports: !2)
-!1 = !MDFile(filename: ".../llvm/test/Transforms/GCOVProfiling/function-numbering.ll", directory: "")
+!0 = !DICompileUnit(language: DW_LANG_C99, producer: "clang version 3.6.0 ", isOptimized: false, emissionKind: 2, file: !1, enums: !2, retainedTypes: !2, subprograms: !3, globals: !2, imports: !2)
+!1 = !DIFile(filename: ".../llvm/test/Transforms/GCOVProfiling/function-numbering.ll", directory: "")
 !2 = !{}
 !3 = !{!4, !7, !8}
-!4 = !MDSubprogram(name: "foo", line: 1, isLocal: false, isDefinition: true, isOptimized: false, scopeLine: 1, file: !1, scope: !5, type: !6, function: void ()* @foo, variables: !2)
-!5 = !MDFile(filename: ".../llvm/test/Transforms/GCOVProfiling/function-numbering.ll", directory: "")
-!6 = !MDSubroutineType(types: !2)
-!7 = !MDSubprogram(name: "bar", line: 2, isLocal: false, isDefinition: true, isOptimized: false, scopeLine: 2, file: !1, scope: !5, type: !6, function: void ()* @bar, variables: !2)
-!8 = !MDSubprogram(name: "baz", line: 3, isLocal: false, isDefinition: true, isOptimized: false, scopeLine: 3, file: !1, scope: !5, type: !6, function: void ()* @baz, variables: !2)
+!4 = !DISubprogram(name: "foo", line: 1, isLocal: false, isDefinition: true, isOptimized: false, scopeLine: 1, file: !1, scope: !5, type: !6, function: void ()* @foo, variables: !2)
+!5 = !DIFile(filename: ".../llvm/test/Transforms/GCOVProfiling/function-numbering.ll", directory: "")
+!6 = !DISubroutineType(types: !2)
+!7 = !DISubprogram(name: "bar", line: 2, isLocal: false, isDefinition: true, isOptimized: false, scopeLine: 2, file: !1, scope: !5, type: !6, function: void ()* @bar, variables: !2)
+!8 = !DISubprogram(name: "baz", line: 3, isLocal: false, isDefinition: true, isOptimized: false, scopeLine: 3, file: !1, scope: !5, type: !6, function: void ()* @baz, variables: !2)
 !9 = !{i32 2, !"Dwarf Version", i32 2}
 !10 = !{i32 2, !"Debug Info Version", i32 3}
 !11 = !{!"clang version 3.6.0 "}
-!12 = !MDLocation(line: 1, column: 13, scope: !4)
-!13 = !MDLocation(line: 3, column: 13, scope: !8)
+!12 = !DILocation(line: 1, column: 13, scope: !4)
+!13 = !DILocation(line: 3, column: 13, scope: !8)
diff --git a/test/Transforms/GCOVProfiling/global-ctor.ll b/test/Transforms/GCOVProfiling/global-ctor.ll
index c064d9a46f6..29c46d6c210 100644
--- a/test/Transforms/GCOVProfiling/global-ctor.ll
+++ b/test/Transforms/GCOVProfiling/global-ctor.ll
@@ -38,19 +38,19 @@ attributes #1 = { "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "
 !llvm.gcov = !{!16}
 !llvm.ident = !{!12}
 
-!0 = !MDCompileUnit(language: DW_LANG_C_plus_plus, producer: "clang version 3.5.0 (trunk 210217)", isOptimized: false, emissionKind: 2, file: !1, enums: !2, retainedTypes: !2, subprograms: !3, globals: !2, imports: !2)
-!1 = !MDFile(filename: "<stdin>", directory: "/home/nlewycky")
+!0 = !DICompileUnit(language: DW_LANG_C_plus_plus, producer: "clang version 3.5.0 (trunk 210217)", isOptimized: false, emissionKind: 2, file: !1, enums: !2, retainedTypes: !2, subprograms: !3, globals: !2, imports: !2)
+!1 = !DIFile(filename: "<stdin>", directory: "/home/nlewycky")
 !2 = !{}
 !3 = !{!4, !8}
-!4 = !MDSubprogram(name: "__cxx_global_var_init", line: 2, isLocal: true, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 2, file: !5, scope: !6, type: !7, function: void ()* @__cxx_global_var_init, variables: !2)
-!5 = !MDFile(filename: "global-ctor.ll", directory: "/home/nlewycky")
-!6 = !MDFile(filename: "global-ctor.ll", directory: "/home/nlewycky")
-!7 = !MDSubroutineType(types: !2)
-!8 = !MDSubprogram(name: "", linkageName: "_GLOBAL__sub_I_global-ctor.ll", isLocal: true, isDefinition: true, virtualIndex: 6, flags: DIFlagArtificial, isOptimized: false, file: !1, scope: !9, type: !7, function: void ()* @_GLOBAL__sub_I_global-ctor.ll, variables: !2)
-!9 = !MDFile(filename: "<stdin>", directory: "/home/nlewycky")
+!4 = !DISubprogram(name: "__cxx_global_var_init", line: 2, isLocal: true, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 2, file: !5, scope: !6, type: !7, function: void ()* @__cxx_global_var_init, variables: !2)
+!5 = !DIFile(filename: "global-ctor.ll", directory: "/home/nlewycky")
+!6 = !DIFile(filename: "global-ctor.ll", directory: "/home/nlewycky")
+!7 = !DISubroutineType(types: !2)
+!8 = !DISubprogram(name: "", linkageName: "_GLOBAL__sub_I_global-ctor.ll", isLocal: true, isDefinition: true, virtualIndex: 6, flags: DIFlagArtificial, isOptimized: false, file: !1, scope: !9, type: !7, function: void ()* @_GLOBAL__sub_I_global-ctor.ll, variables: !2)
+!9 = !DIFile(filename: "<stdin>", directory: "/home/nlewycky")
 !10 = !{i32 2, !"Dwarf Version", i32 4}
 !11 = !{i32 2, !"Debug Info Version", i32 3}
 !12 = !{!"clang version 3.5.0 (trunk 210217)"}
-!13 = !MDLocation(line: 2, scope: !4)
-!14 = !MDLocation(line: 0, scope: !15)
-!15 = !MDLexicalBlockFile(discriminator: 0, file: !5, scope: !8)
+!13 = !DILocation(line: 2, scope: !4)
+!14 = !DILocation(line: 0, scope: !15)
+!15 = !DILexicalBlockFile(discriminator: 0, file: !5, scope: !8)
diff --git a/test/Transforms/GCOVProfiling/linezero.ll b/test/Transforms/GCOVProfiling/linezero.ll
index d56228255fb..9e172b752d7 100644
--- a/test/Transforms/GCOVProfiling/linezero.ll
+++ b/test/Transforms/GCOVProfiling/linezero.ll
@@ -18,17 +18,17 @@ entry:
   %__begin = alloca i8*, align 8
   %__end = alloca i8*, align 8
   %spec = alloca i8, align 1
-  call void @llvm.dbg.declare(metadata %struct.vector** %__range, metadata !27, metadata !MDExpression()), !dbg !30
+  call void @llvm.dbg.declare(metadata %struct.vector** %__range, metadata !27, metadata !DIExpression()), !dbg !30
   br label %0
 
 ; <label>:0                                       ; preds = %entry
   call void @_Z13TagFieldSpecsv(), !dbg !31
   store %struct.vector* %ref.tmp, %struct.vector** %__range, align 8, !dbg !31
-  call void @llvm.dbg.declare(metadata i8** %__begin, metadata !32, metadata !MDExpression()), !dbg !30
+  call void @llvm.dbg.declare(metadata i8** %__begin, metadata !32, metadata !DIExpression()), !dbg !30
   %1 = load %struct.vector*, %struct.vector** %__range, align 8, !dbg !31
   %call = call i8* @_ZN6vector5beginEv(%struct.vector* %1), !dbg !31
   store i8* %call, i8** %__begin, align 8, !dbg !31
-  call void @llvm.dbg.declare(metadata i8** %__end, metadata !33, metadata !MDExpression()), !dbg !30
+  call void @llvm.dbg.declare(metadata i8** %__end, metadata !33, metadata !DIExpression()), !dbg !30
   %2 = load %struct.vector*, %struct.vector** %__range, align 8, !dbg !31
   %call1 = call i8* @_ZN6vector3endEv(%struct.vector* %2), !dbg !31
   store i8* %call1, i8** %__end, align 8, !dbg !31
@@ -41,7 +41,7 @@ for.cond:                                         ; preds = %for.inc, %0
   br i1 %cmp, label %for.body, label %for.end, !dbg !34
 
 for.body:                                         ; preds = %for.cond
-  call void @llvm.dbg.declare(metadata i8* %spec, metadata !37, metadata !MDExpression()), !dbg !31
+  call void @llvm.dbg.declare(metadata i8* %spec, metadata !37, metadata !DIExpression()), !dbg !31
   %5 = load i8*, i8** %__begin, align 8, !dbg !38
   %6 = load i8, i8* %5, align 1, !dbg !38
   store i8 %6, i8* %spec, align 1, !dbg !38
@@ -93,49 +93,49 @@ attributes #3 = { noreturn nounwind }
 !llvm.gcov = !{!25}
 !llvm.ident = !{!26}
 
-!0 = !MDCompileUnit(language: DW_LANG_C_plus_plus, producer: "clang version 3.5.0 (trunk 209871)", isOptimized: false, emissionKind: 1, file: !1, enums: !2, retainedTypes: !3, subprograms: !14, globals: !2, imports: !2)
-!1 = !MDFile(filename: "<stdin>", directory: "PATTERN")
+!0 = !DICompileUnit(language: DW_LANG_C_plus_plus, producer: "clang version 3.5.0 (trunk 209871)", isOptimized: false, emissionKind: 1, file: !1, enums: !2, retainedTypes: !3, subprograms: !14, globals: !2, imports: !2)
+!1 = !DIFile(filename: "<stdin>", directory: "PATTERN")
 !2 = !{}
 !3 = !{!4}
-!4 = !MDCompositeType(tag: DW_TAG_structure_type, name: "vector", line: 21, size: 8, align: 8, file: !5, elements: !6, identifier: "_ZTS6vector")
-!5 = !MDFile(filename: "linezero.cc", directory: "PATTERN")
+!4 = !DICompositeType(tag: DW_TAG_structure_type, name: "vector", line: 21, size: 8, align: 8, file: !5, elements: !6, identifier: "_ZTS6vector")
+!5 = !DIFile(filename: "linezero.cc", directory: "PATTERN")
 !6 = !{!7, !13}
-!7 = !MDSubprogram(name: "begin", linkageName: "_ZN6vector5beginEv", line: 25, isLocal: false, isDefinition: false, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 25, file: !5, scope: !"_ZTS6vector", type: !8)
-!8 = !MDSubroutineType(types: !9)
+!7 = !DISubprogram(name: "begin", linkageName: "_ZN6vector5beginEv", line: 25, isLocal: false, isDefinition: false, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 25, file: !5, scope: !"_ZTS6vector", type: !8)
+!8 = !DISubroutineType(types: !9)
 !9 = !{!10, !12}
-!10 = !MDDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, baseType: !11)
-!11 = !MDBasicType(tag: DW_TAG_base_type, name: "char", size: 8, align: 8, encoding: DW_ATE_signed_char)
-!12 = !MDDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, flags: DIFlagArtificial | DIFlagObjectPointer, baseType: !"_ZTS6vector")
-!13 = !MDSubprogram(name: "end", linkageName: "_ZN6vector3endEv", line: 26, isLocal: false, isDefinition: false, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 26, file: !5, scope: !"_ZTS6vector", type: !8)
+!10 = !DIDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, baseType: !11)
+!11 = !DIBasicType(tag: DW_TAG_base_type, name: "char", size: 8, align: 8, encoding: DW_ATE_signed_char)
+!12 = !DIDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, flags: DIFlagArtificial | DIFlagObjectPointer, baseType: !"_ZTS6vector")
+!13 = !DISubprogram(name: "end", linkageName: "_ZN6vector3endEv", line: 26, isLocal: false, isDefinition: false, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 26, file: !5, scope: !"_ZTS6vector", type: !8)
 !14 = !{!15, !20}
-!15 = !MDSubprogram(name: "test", linkageName: "_Z4testv", line: 50, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 50, file: !5, scope: !16, type: !17, function: i32 ()* @_Z4testv, variables: !2)
-!16 = !MDFile(filename: "linezero.cc", directory: "PATTERN")
-!17 = !MDSubroutineType(types: !18)
+!15 = !DISubprogram(name: "test", linkageName: "_Z4testv", line: 50, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 50, file: !5, scope: !16, type: !17, function: i32 ()* @_Z4testv, variables: !2)
+!16 = !DIFile(filename: "linezero.cc", directory: "PATTERN")
+!17 = !DISubroutineType(types: !18)
 !18 = !{!19}
-!19 = !MDBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
-!20 = !MDSubprogram(name: "f1", linkageName: "_Z2f1v", line: 54, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 54, file: !5, scope: !16, type: !21, function: void ()* @_Z2f1v, variables: !2)
-!21 = !MDSubroutineType(types: !22)
+!19 = !DIBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
+!20 = !DISubprogram(name: "f1", linkageName: "_Z2f1v", line: 54, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 54, file: !5, scope: !16, type: !21, function: void ()* @_Z2f1v, variables: !2)
+!21 = !DISubroutineType(types: !22)
 !22 = !{null}
 !23 = !{i32 2, !"Dwarf Version", i32 4}
 !24 = !{i32 2, !"Debug Info Version", i32 3}
 !25 = !{!"PATTERN/linezero.o", !0}
 !26 = !{!"clang version 3.5.0 (trunk 209871)"}
-!27 = !MDLocalVariable(tag: DW_TAG_auto_variable, name: "__range", flags: DIFlagArtificial, scope: !28, type: !29)
-!28 = distinct !MDLexicalBlock(line: 51, column: 0, file: !5, scope: !15)
-!29 = !MDDerivedType(tag: DW_TAG_rvalue_reference_type, baseType: !"_ZTS6vector")
-!30 = !MDLocation(line: 0, scope: !28)
-!31 = !MDLocation(line: 51, scope: !28)
-!32 = !MDLocalVariable(tag: DW_TAG_auto_variable, name: "__begin", flags: DIFlagArtificial, scope: !28, type: !10)
-!33 = !MDLocalVariable(tag: DW_TAG_auto_variable, name: "__end", flags: DIFlagArtificial, scope: !28, type: !10)
-!34 = !MDLocation(line: 51, scope: !35)
-!35 = distinct !MDLexicalBlock(line: 51, column: 0, file: !5, scope: !36)
-!36 = distinct !MDLexicalBlock(line: 51, column: 0, file: !5, scope: !28)
-!37 = !MDLocalVariable(tag: DW_TAG_auto_variable, name: "spec", line: 51, scope: !28, file: !16, type: !11)
-!38 = !MDLocation(line: 51, scope: !39)
-!39 = distinct !MDLexicalBlock(line: 51, column: 0, file: !5, scope: !28)
-!40 = !MDLocation(line: 51, scope: !41)
-!41 = distinct !MDLexicalBlock(line: 51, column: 0, file: !5, scope: !28)
-!42 = !MDLocation(line: 51, scope: !43)
-!43 = distinct !MDLexicalBlock(line: 51, column: 0, file: !5, scope: !28)
-!44 = !MDLocation(line: 52, scope: !15)
-!45 = !MDLocation(line: 54, scope: !20)
+!27 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "__range", flags: DIFlagArtificial, scope: !28, type: !29)
+!28 = distinct !DILexicalBlock(line: 51, column: 0, file: !5, scope: !15)
+!29 = !DIDerivedType(tag: DW_TAG_rvalue_reference_type, baseType: !"_ZTS6vector")
+!30 = !DILocation(line: 0, scope: !28)
+!31 = !DILocation(line: 51, scope: !28)
+!32 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "__begin", flags: DIFlagArtificial, scope: !28, type: !10)
+!33 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "__end", flags: DIFlagArtificial, scope: !28, type: !10)
+!34 = !DILocation(line: 51, scope: !35)
+!35 = distinct !DILexicalBlock(line: 51, column: 0, file: !5, scope: !36)
+!36 = distinct !DILexicalBlock(line: 51, column: 0, file: !5, scope: !28)
+!37 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "spec", line: 51, scope: !28, file: !16, type: !11)
+!38 = !DILocation(line: 51, scope: !39)
+!39 = distinct !DILexicalBlock(line: 51, column: 0, file: !5, scope: !28)
+!40 = !DILocation(line: 51, scope: !41)
+!41 = distinct !DILexicalBlock(line: 51, column: 0, file: !5, scope: !28)
+!42 = !DILocation(line: 51, scope: !43)
+!43 = distinct !DILexicalBlock(line: 51, column: 0, file: !5, scope: !28)
+!44 = !DILocation(line: 52, scope: !15)
+!45 = !DILocation(line: 54, scope: !20)
diff --git a/test/Transforms/GCOVProfiling/linkagename.ll b/test/Transforms/GCOVProfiling/linkagename.ll
index 22442611bec..7a4119802c9 100644
--- a/test/Transforms/GCOVProfiling/linkagename.ll
+++ b/test/Transforms/GCOVProfiling/linkagename.ll
@@ -13,15 +13,15 @@ entry:
 !llvm.module.flags = !{!10}
 !llvm.gcov = !{!9}
 
-!0 = !MDCompileUnit(language: DW_LANG_C_plus_plus, producer: "clang version 3.3 (trunk 177323)", isOptimized: false, emissionKind: 0, file: !2, enums: !3, retainedTypes: !3, subprograms: !4, globals: !3, imports:  !3)
-!1 = !MDFile(filename: "hello.cc", directory: "/home/nlewycky")
-!2 = !MDFile(filename: "hello.cc", directory: "/home/nlewycky")
+!0 = !DICompileUnit(language: DW_LANG_C_plus_plus, producer: "clang version 3.3 (trunk 177323)", isOptimized: false, emissionKind: 0, file: !2, enums: !3, retainedTypes: !3, subprograms: !4, globals: !3, imports:  !3)
+!1 = !DIFile(filename: "hello.cc", directory: "/home/nlewycky")
+!2 = !DIFile(filename: "hello.cc", directory: "/home/nlewycky")
 !3 = !{}
 !4 = !{!5}
-!5 = !MDSubprogram(name: "foo", linkageName: "_Z3foov", line: 1, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 1, file: !1, scope: !1, type: !6, function: void ()* @_Z3foov, variables: !3)
-!6 = !MDSubroutineType(types: !7)
+!5 = !DISubprogram(name: "foo", linkageName: "_Z3foov", line: 1, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 1, file: !1, scope: !1, type: !6, function: void ()* @_Z3foov, variables: !3)
+!6 = !DISubroutineType(types: !7)
 !7 = !{null}
-!8 = !MDLocation(line: 1, scope: !5)
+!8 = !DILocation(line: 1, scope: !5)
 
 
 !10 = !{i32 1, !"Debug Info Version", i32 3}
diff --git a/test/Transforms/GCOVProfiling/return-block.ll b/test/Transforms/GCOVProfiling/return-block.ll
index 1f1933ca72e..38b5b75e3c2 100644
--- a/test/Transforms/GCOVProfiling/return-block.ll
+++ b/test/Transforms/GCOVProfiling/return-block.ll
@@ -44,25 +44,25 @@ attributes #2 = { nounwind }
 !llvm.module.flags = !{!11, !12}
 !llvm.ident = !{!13}
 
-!0 = !MDCompileUnit(language: DW_LANG_C99, producer: "clang version 3.6.0 (trunk 223182)", isOptimized: true, emissionKind: 1, file: !1, enums: !2, retainedTypes: !2, subprograms: !3, globals: !8, imports: !2)
-!1 = !MDFile(filename: ".../llvm/test/Transforms/GCOVProfiling/return-block.ll", directory: "")
+!0 = !DICompileUnit(language: DW_LANG_C99, producer: "clang version 3.6.0 (trunk 223182)", isOptimized: true, emissionKind: 1, file: !1, enums: !2, retainedTypes: !2, subprograms: !3, globals: !8, imports: !2)
+!1 = !DIFile(filename: ".../llvm/test/Transforms/GCOVProfiling/return-block.ll", directory: "")
 !2 = !{}
 !3 = !{!4}
-!4 = !MDSubprogram(name: "test", line: 5, isLocal: false, isDefinition: true, isOptimized: true, scopeLine: 5, file: !1, scope: !5, type: !6, function: void ()* @test, variables: !2)
-!5 = !MDFile(filename: ".../llvm/test/Transforms/GCOVProfiling/return-block.ll", directory: "")
-!6 = !MDSubroutineType(types: !7)
+!4 = !DISubprogram(name: "test", line: 5, isLocal: false, isDefinition: true, isOptimized: true, scopeLine: 5, file: !1, scope: !5, type: !6, function: void ()* @test, variables: !2)
+!5 = !DIFile(filename: ".../llvm/test/Transforms/GCOVProfiling/return-block.ll", directory: "")
+!6 = !DISubroutineType(types: !7)
 !7 = !{null}
 !8 = !{!9}
-!9 = !MDGlobalVariable(name: "A", line: 3, isLocal: false, isDefinition: true, scope: null, file: !5, type: !10, variable: i32* @A)
-!10 = !MDBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
+!9 = !DIGlobalVariable(name: "A", line: 3, isLocal: false, isDefinition: true, scope: null, file: !5, type: !10, variable: i32* @A)
+!10 = !DIBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
 !11 = !{i32 2, !"Dwarf Version", i32 4}
 !12 = !{i32 2, !"Debug Info Version", i32 3}
 !13 = !{!"clang version 3.6.0 (trunk 223182)"}
-!14 = !MDLocation(line: 6, column: 3, scope: !4)
-!15 = !MDLocation(line: 7, column: 7, scope: !4)
-!16 = !MDLocation(line: 8, column: 5, scope: !17)
-!17 = distinct !MDLexicalBlock(line: 7, column: 7, file: !1, scope: !4)
-!18 = !MDLocation(line: 9, column: 1, scope: !4)
+!14 = !DILocation(line: 6, column: 3, scope: !4)
+!15 = !DILocation(line: 7, column: 7, scope: !4)
+!16 = !DILocation(line: 8, column: 5, scope: !17)
+!17 = distinct !DILexicalBlock(line: 7, column: 7, file: !1, scope: !4)
+!18 = !DILocation(line: 9, column: 1, scope: !4)
 
 ; There should be no destination edges for the exit block.
 ; CHECK: Block : 1 Counter : 0
diff --git a/test/Transforms/GCOVProfiling/version.ll b/test/Transforms/GCOVProfiling/version.ll
index 60bc6a02bc0..487e72123cb 100644
--- a/test/Transforms/GCOVProfiling/version.ll
+++ b/test/Transforms/GCOVProfiling/version.ll
@@ -16,15 +16,15 @@ define void @test() {
 !llvm.dbg.cu = !{!0}
 !llvm.module.flags = !{!12}
 
-!0 = !MDCompileUnit(language: DW_LANG_C_plus_plus, producer: "clang version 3.3 (trunk 176994)", isOptimized: false, emissionKind: 0, file: !11, enums: !3, retainedTypes: !3, subprograms: !4, globals: !3)
-!2 = !MDFile(filename: "version", directory: "/usr/local/google/home/nlewycky")
+!0 = !DICompileUnit(language: DW_LANG_C_plus_plus, producer: "clang version 3.3 (trunk 176994)", isOptimized: false, emissionKind: 0, file: !11, enums: !3, retainedTypes: !3, subprograms: !4, globals: !3)
+!2 = !DIFile(filename: "version", directory: "/usr/local/google/home/nlewycky")
 !3 = !{}
 !4 = !{!5}
-!5 = !MDSubprogram(name: "test", line: 1, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 1, file: !10, scope: !6, type: !7, function: void ()* @test, variables: !3)
-!6 = !MDFile(filename: "<stdin>", directory: ".")
-!7 = !MDSubroutineType(types: !{null})
-!8 = !MDLocation(line: 1, scope: !5)
+!5 = !DISubprogram(name: "test", line: 1, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 1, file: !10, scope: !6, type: !7, function: void ()* @test, variables: !3)
+!6 = !DIFile(filename: "<stdin>", directory: ".")
+!7 = !DISubroutineType(types: !{null})
+!8 = !DILocation(line: 1, scope: !5)
 ;; !9 is added through the echo line at the top.
-!10 = !MDFile(filename: "<stdin>", directory: ".")
-!11 = !MDFile(filename: "version", directory: "/usr/local/google/home/nlewycky")
+!10 = !DIFile(filename: "<stdin>", directory: ".")
+!11 = !DIFile(filename: "version", directory: "/usr/local/google/home/nlewycky")
 !12 = !{i32 1, !"Debug Info Version", i32 3}
diff --git a/test/Transforms/GVN/2010-05-08-OneBit.ll b/test/Transforms/GVN/2010-05-08-OneBit.ll
index 0e3fa4b8a49..562b3d820d0 100644
--- a/test/Transforms/GVN/2010-05-08-OneBit.ll
+++ b/test/Transforms/GVN/2010-05-08-OneBit.ll
@@ -4,7 +4,7 @@
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64"
 target triple = "x86_64-unknown-linux-gnu"
 
-define i32 @main(i32 %argc, i8** nocapture %argv) {
+define i32 @main(i32 %argc, i8** nocapture %argv) personality i32 (...)* @__gxx_personality_v0 {
 entry:
   %0 = getelementptr inbounds i8, i8* undef, i64 5    ; <i8*> [#uses=1]
   %1 = bitcast i8* %0 to i32*                     ; <i32*> [#uses=1]
@@ -45,7 +45,7 @@ k151.i.i:                                         ; preds = %k133.i.i
   ret i32 0
 
 landing_pad:                                      ; preds = %l147.i.i, %l129.i.i, %l117.i.i
-  %exn = landingpad {i8*, i32} personality i32 (...)* @__gxx_personality_v0
+  %exn = landingpad {i8*, i32}
             cleanup
   switch i32 undef, label %fin [
     i32 1, label %catch1
diff --git a/test/Transforms/GVN/2011-09-07-TypeIdFor.ll b/test/Transforms/GVN/2011-09-07-TypeIdFor.ll
index 314b5bb113a..d6b69d3eb4e 100644
--- a/test/Transforms/GVN/2011-09-07-TypeIdFor.ll
+++ b/test/Transforms/GVN/2011-09-07-TypeIdFor.ll
@@ -17,13 +17,13 @@ declare void @__cxa_end_catch()
 
 declare i32 @__gxx_personality_v0(i32, i64, i8*, i8*)
 
-define void @_Z3foov() uwtable {
+define void @_Z3foov() uwtable personality i32 (i32, i64, i8*, i8*)* @__gxx_personality_v0 {
 entry:
   invoke void @_Z4barv()
           to label %return unwind label %lpad
 
 lpad:                                             ; preds = %entry
-  %0 = landingpad { i8*, i32 } personality i32 (i32, i64, i8*, i8*)* @__gxx_personality_v0
+  %0 = landingpad { i8*, i32 }
           catch %struct.__fundamental_type_info_pseudo* @_ZTIi
           catch %struct.__fundamental_type_info_pseudo* @_ZTIb
           catch %struct.__fundamental_type_info_pseudo* @_ZTIi
diff --git a/test/Transforms/GVN/basic-undef-test.ll b/test/Transforms/GVN/basic-undef-test.ll
new file mode 100644
index 00000000000..4f940956780
--- /dev/null
+++ b/test/Transforms/GVN/basic-undef-test.ll
@@ -0,0 +1,15 @@
+; RUN: opt -basicaa -gvn -S < %s | FileCheck %s
+; ModuleID = 'test3.ll'
+target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
+
+define i32 @main(i32 *%foo)  {
+entry:
+; CHECK: load i32, i32* %foo, align 4
+  %0 = load i32, i32* %foo, align 4
+  store i32 5, i32* undef, align 4
+; CHECK-NOT: load i32, i32* %foo, align 4
+  %1 = load i32, i32* %foo, align 4
+; CHECK: add i32 %0, %0
+  %2 = add i32 %0, %1
+  ret i32 %2
+}
diff --git a/test/Transforms/GVN/br-identical.ll b/test/Transforms/GVN/br-identical.ll
new file mode 100644
index 00000000000..dfb7abe9a5d
--- /dev/null
+++ b/test/Transforms/GVN/br-identical.ll
@@ -0,0 +1,38 @@
+; RUN: opt -gvn -S -o - %s | FileCheck %s
+
+; If a branch has two identical successors, we cannot declare either dead.
+
+define void @widget(i1 %p) {
+entry:
+  br label %bb2
+
+bb2:
+  %t1 = phi i64 [ 0, %entry ], [ %t5, %bb7 ]
+  %t2 = add i64 %t1, 1
+  %t3 = icmp ult i64 0, %t2
+  br i1 %t3, label %bb3, label %bb4
+
+bb3:
+  %t4 = call i64 @f()
+  br label %bb4
+
+bb4:
+  ; CHECK-NOT: phi {{.*}} undef
+  %foo = phi i64 [ %t4, %bb3 ], [ 0, %bb2 ]
+  br i1 %p, label %bb5, label %bb6
+
+bb5:
+  br i1 true, label %bb7, label %bb7
+
+bb6:
+  br i1 true, label %bb7, label %bb7
+
+bb7:
+  %t5 = add i64 %t1, 1
+  br i1 %p, label %bb2, label %bb8
+
+bb8:
+  ret void
+}
+
+declare i64 @f()
diff --git a/test/Transforms/GVN/cond_br2.ll b/test/Transforms/GVN/cond_br2.ll
index 02154a78391..baa282ec200 100644
--- a/test/Transforms/GVN/cond_br2.ll
+++ b/test/Transforms/GVN/cond_br2.ll
@@ -9,7 +9,7 @@ target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f3
 %"union.llvm::SmallVectorBase::U" = type { x86_fp80 }
 
 ; Function Attrs: ssp uwtable
-define void @_Z4testv() #0 {
+define void @_Z4testv() #0 personality i8* bitcast (i32 (...)* @__gxx_personality_v0 to i8*) {
 ; CHECK: @_Z4testv()
 ; CHECK: invoke.cont:
 ; CHECK: br i1 true, label %new.notnull.i11, label %if.end.i14
@@ -98,7 +98,7 @@ _ZN4llvm11SmallVectorIiLj8EED1Ev.exit21:          ; preds = %invoke.cont3, %if.t
   ret void
 
 lpad:                                             ; preds = %if.end.i14, %if.end.i, %invoke.cont2
-  %12 = landingpad { i8*, i32 } personality i8* bitcast (i32 (...)* @__gxx_personality_v0 to i8*)
+  %12 = landingpad { i8*, i32 }
           cleanup
   %13 = load i8*, i8** %BeginX.i.i.i.i.i.i, align 16, !tbaa !4
   %cmp.i.i.i.i = icmp eq i8* %13, %1
diff --git a/test/Transforms/GVN/load-pre-nonlocal.ll b/test/Transforms/GVN/load-pre-nonlocal.ll
index c75e54db628..e9827a158ad 100644
--- a/test/Transforms/GVN/load-pre-nonlocal.ll
+++ b/test/Transforms/GVN/load-pre-nonlocal.ll
@@ -53,30 +53,35 @@ for.end:
 ; %1 is partially redundant if %0 can be widened to a 64-bit load.
 
 ; CHECK-LABEL: define i32 @overaligned_load
+; CHECK: if.then:
+; CHECK:   %0 = load i64
+; CHECK:   [[LSHR:%[0-9]+]] = lshr i64 %0, 32, !dbg [[LSHR_LOC:![0-9]+]]
+; CHECK:   trunc i64 [[LSHR]] to i32
 ; CHECK: if.end:
 ; CHECK-NOT: %1 = load i32, i32*
+; CHECK: [[LSHR_LOC]] = !DILocation(line: 101, column: 1, scope: !{{.*}})
 
 define i32 @overaligned_load(i32 %a, i32* nocapture %b) {
 entry:
-  %cmp = icmp sgt i32 %a, 0
-  br i1 %cmp, label %if.then, label %if.else
+  %cmp = icmp sgt i32 %a, 0, !dbg !14
+  br i1 %cmp, label %if.then, label %if.else, !dbg !14
 
 if.then:
-  %0 = load i32, i32* getelementptr inbounds (%struct.S1, %struct.S1* @s1, i64 0, i32 0), align 8, !tbaa !5
-  br label %if.end
+  %0 = load i32, i32* getelementptr inbounds (%struct.S1, %struct.S1* @s1, i64 0, i32 0), align 8, !tbaa !5, !dbg !15
+  br label %if.end, !dbg !15
 
 if.else:
-  %arrayidx = getelementptr inbounds i32, i32* %b, i64 2
-  store i32 10, i32* %arrayidx, align 4, !tbaa !5
-  br label %if.end
+  %arrayidx = getelementptr inbounds i32, i32* %b, i64 2, !dbg !16
+  store i32 10, i32* %arrayidx, align 4, !tbaa !5, !dbg !16
+  br label %if.end, !dbg !16
 
 if.end:
   %i.0 = phi i32 [ %0, %if.then ], [ 0, %if.else ]
   %p.0 = phi i32* [ getelementptr inbounds (%struct.S1, %struct.S1* @s1, i64 0, i32 0), %if.then ], [ %b, %if.else ]
-  %add.ptr = getelementptr inbounds i32, i32* %p.0, i64 1
-  %1 = load i32, i32* %add.ptr, align 4, !tbaa !5
-  %add1 = add nsw i32 %1, %i.0
-  ret i32 %add1
+  %add.ptr = getelementptr inbounds i32, i32* %p.0, i64 1, !dbg !17
+  %1 = load i32, i32* %add.ptr, align 4, !tbaa !5, !dbg !17
+  %add1 = add nsw i32 %1, %i.0, !dbg !17
+  ret i32 %add1, !dbg !17
 }
 
 !1 = !{!2, !2, i64 0}
@@ -85,3 +90,18 @@ if.end:
 !4 = !{!"Simple C/C++ TBAA"}
 !5 = !{!6, !6, i64 0}
 !6 = !{!"int", !3, i64 0}
+
+!llvm.module.flags = !{!7, !8, !9}
+!7 = !{i32 2, !"Dwarf Version", i32 4}
+!8 = !{i32 2, !"Debug Info Version", i32 3}
+!9 = !{i32 1, !"PIC Level", i32 2}
+
+!10 = !{}
+!11 = !DISubroutineType(types: !10)
+!12 = !DIFile(filename: "test.cpp", directory: "/tmp")
+!13 = !DISubprogram(name: "test", scope: !12, file: !12, line: 99, type: !11, isLocal: false, isDefinition: true, scopeLine: 100, flags: DIFlagPrototyped, isOptimized: false, function: i32 (i32, i32*)* @overaligned_load, variables: !10)
+!14 = !DILocation(line: 100, column: 1, scope: !13)
+!15 = !DILocation(line: 101, column: 1, scope: !13)
+!16 = !DILocation(line: 102, column: 1, scope: !13)
+!17 = !DILocation(line: 103, column: 1, scope: !13)
+
diff --git a/test/Transforms/GVN/phi-translate.ll b/test/Transforms/GVN/phi-translate.ll
index 6068b05aadf..9e37b882f22 100644
--- a/test/Transforms/GVN/phi-translate.ll
+++ b/test/Transforms/GVN/phi-translate.ll
@@ -4,28 +4,49 @@ target datalayout = "e-p:64:64:64"
 
 ; CHECK-LABEL: @foo(
 ; CHECK: entry.end_crit_edge:
-; CHECK:   %n.pre = load i32, i32* %q.phi.trans.insert
+; CHECK:   %j.phi.trans.insert = sext i32 %x to i64, !dbg [[J_LOC:![0-9]+]]
+; CHECK:   %q.phi.trans.insert = getelementptr {{.*}}, !dbg [[Q_LOC:![0-9]+]]
+; CHECK:   %n.pre = load i32, i32* %q.phi.trans.insert, !dbg [[N_LOC:![0-9]+]]
 ; CHECK: then:
 ; CHECK:   store i32 %z
 ; CHECK: end:
-; CHECK:   %n = phi i32 [ %n.pre, %entry.end_crit_edge ], [ %z, %then ]
+; CHECK:   %n = phi i32 [ %n.pre, %entry.end_crit_edge ], [ %z, %then ], !dbg [[N_LOC]]
 ; CHECK:   ret i32 %n
 
+; CHECK-DAG: [[J_LOC]] = !DILocation(line: 45, column: 1, scope: !{{.*}})
+; CHECK-DAG: [[Q_LOC]] = !DILocation(line: 46, column: 1, scope: !{{.*}})
+; CHECK-DAG: [[N_LOC]] = !DILocation(line: 47, column: 1, scope: !{{.*}})
+
 @G = external global [100 x i32]
 define i32 @foo(i32 %x, i32 %z) {
 entry:
-  %tobool = icmp eq i32 %z, 0
-  br i1 %tobool, label %end, label %then
+  %tobool = icmp eq i32 %z, 0, !dbg !7
+  br i1 %tobool, label %end, label %then, !dbg !7
 
 then:
-  %i = sext i32 %x to i64
-  %p = getelementptr [100 x i32], [100 x i32]* @G, i64 0, i64 %i
-  store i32 %z, i32* %p
-  br label %end
+  %i = sext i32 %x to i64, !dbg !8
+  %p = getelementptr [100 x i32], [100 x i32]* @G, i64 0, i64 %i, !dbg !8
+  store i32 %z, i32* %p, !dbg !8
+  br label %end, !dbg !8
 
 end:
-  %j = sext i32 %x to i64
-  %q = getelementptr [100 x i32], [100 x i32]* @G, i64 0, i64 %j
-  %n = load i32, i32* %q
-  ret i32 %n
+  %j = sext i32 %x to i64, !dbg !9
+  %q = getelementptr [100 x i32], [100 x i32]* @G, i64 0, i64 %j, !dbg !10
+  %n = load i32, i32* %q, !dbg !11
+  ret i32 %n, !dbg !11
 }
+
+!llvm.module.flags = !{!0, !1, !2}
+!0 = !{i32 2, !"Dwarf Version", i32 4}
+!1 = !{i32 2, !"Debug Info Version", i32 3}
+!2 = !{i32 1, !"PIC Level", i32 2}
+
+!3 = !{}
+!4 = !DISubroutineType(types: !3)
+!5 = !DIFile(filename: "a.cc", directory: "/tmp")
+!6 = !DISubprogram(name: "foo", scope: !5, file: !5, line: 42, type: !4, isLocal: false, isDefinition: true, scopeLine: 43, flags: DIFlagPrototyped, isOptimized: false, function: i32 (i32, i32)* @foo, variables: !3)
+!7 = !DILocation(line: 43, column: 1, scope: !6)
+!8 = !DILocation(line: 44, column: 1, scope: !6)
+!9 = !DILocation(line: 45, column: 1, scope: !6)
+!10 = !DILocation(line: 46, column: 1, scope: !6)
+!11 = !DILocation(line: 47, column: 1, scope: !6)
diff --git a/test/Transforms/GVN/pr12979.ll b/test/Transforms/GVN/pr12979.ll
index 0198a56513e..919c22de02a 100644
--- a/test/Transforms/GVN/pr12979.ll
+++ b/test/Transforms/GVN/pr12979.ll
@@ -77,3 +77,17 @@ define i32 @test7(i32 %x, i32 %y) {
   %foo = add i32 %add1, %add2
   ret i32 %foo
 }
+
+declare void @mumble(i2, i2)
+
+define void @test8(i2 %x) {
+; CHECK-LABEL: @test8(
+; CHECK:      %[[ashr:.*]] = ashr i2 %x, 1
+; CHECK-NEXT: call void @mumble(i2 %[[ashr]], i2 %[[ashr]])
+; CHECK-NEXT: ret void
+
+  %ashr0 = ashr exact i2 %x, 1
+  %ashr1 = ashr i2 %x, 1
+  call void @mumble(i2 %ashr0, i2 %ashr1)
+  ret void
+}
diff --git a/test/Transforms/GVN/pre-new-inst.ll b/test/Transforms/GVN/pre-new-inst.ll
new file mode 100644
index 00000000000..238b8a687cc
--- /dev/null
+++ b/test/Transforms/GVN/pre-new-inst.ll
@@ -0,0 +1,29 @@
+; RUN: opt -basicaa -gvn -S %s | FileCheck %s
+
+%MyStruct = type { i32, i32 }
+define i8 @foo(i64 %in, i8* %arr) {
+  %addr = alloca %MyStruct
+  %dead = trunc i64 %in to i32
+  br i1 undef, label %next, label %tmp
+
+tmp:
+  call void @bar()
+  br label %next
+
+next:
+  %addr64 = bitcast %MyStruct* %addr to i64*
+  store i64 %in, i64* %addr64
+  br label %final
+
+final:
+  %addr32 = getelementptr %MyStruct, %MyStruct* %addr, i32 0, i32 0
+  %idx32 = load i32, i32* %addr32
+
+; CHECK: %resptr = getelementptr i8, i8* %arr, i32 %dead
+  %resptr = getelementptr i8, i8* %arr, i32 %idx32
+  %res = load i8, i8* %resptr
+
+  ret i8 %res
+}
+
+declare void @bar()
diff --git a/test/Transforms/GVN/unreachable_block_infinite_loop.ll b/test/Transforms/GVN/unreachable_block_infinite_loop.ll
index fca5a28b38d..a47e9e4c3a0 100644
--- a/test/Transforms/GVN/unreachable_block_infinite_loop.ll
+++ b/test/Transforms/GVN/unreachable_block_infinite_loop.ll
@@ -12,3 +12,32 @@ unreachable_block:
     ret i32 %a
 }
 
+define i32 @pr23096_test0() {
+entry:
+  br label %bb0
+
+bb1:
+  %ptr1 = ptrtoint i32* %ptr2 to i64
+  %ptr2 = inttoptr i64 %ptr1 to i32*
+  br i1 undef, label %bb0, label %bb1
+
+bb0:
+  %phi = phi i32* [ undef, %entry ], [ %ptr2, %bb1 ]
+  %load = load i32, i32* %phi
+  ret i32 %load
+}
+
+define i32 @pr23096_test1() {
+entry:
+  br label %bb0
+
+bb1:
+  %ptr1 = getelementptr i32, i32* %ptr2, i32 0
+  %ptr2 = getelementptr i32, i32* %ptr1, i32 0
+  br i1 undef, label %bb0, label %bb1
+
+bb0:
+  %phi = phi i32* [ undef, %entry ], [ %ptr2, %bb1 ]
+  %load = load i32, i32* %phi
+  ret i32 %load
+}
diff --git a/test/Transforms/GlobalOpt/2009-03-05-dbg.ll b/test/Transforms/GlobalOpt/2009-03-05-dbg.ll
index f52a31398b2..07d3979acaf 100644
--- a/test/Transforms/GlobalOpt/2009-03-05-dbg.ll
+++ b/test/Transforms/GlobalOpt/2009-03-05-dbg.ll
@@ -55,25 +55,25 @@ declare void @llvm.dbg.value(metadata, i64, metadata, metadata) nounwind readnon
 
 !llvm.dbg.gv = !{!0}
 
-!0 = !MDGlobalVariable(name: "Stop", line: 2, isLocal: true, isDefinition: true, scope: !1, file: !1, type: !2, variable: i32* @Stop)
-!1 = !MDCompileUnit(language: DW_LANG_C89, producer: "4.2.1 (Based on Apple Inc. build 5658) (LLVM build)", isOptimized: true, emissionKind: 0, file: !20, enums: !21, retainedTypes: !21)
-!2 = !MDBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
-!3 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "i", line: 4, arg: 0, scope: !4, file: !1, type: !2)
-!4 = !MDSubprogram(name: "foo", linkageName: "foo", line: 4, isLocal: false, isDefinition: true, virtualIndex: 6, isOptimized: false, scope: !1, type: !5)
-!5 = !MDSubroutineType(types: !6)
+!0 = !DIGlobalVariable(name: "Stop", line: 2, isLocal: true, isDefinition: true, scope: !1, file: !1, type: !2, variable: i32* @Stop)
+!1 = !DICompileUnit(language: DW_LANG_C89, producer: "4.2.1 (Based on Apple Inc. build 5658) (LLVM build)", isOptimized: true, emissionKind: 0, file: !20, enums: !21, retainedTypes: !21)
+!2 = !DIBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
+!3 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "i", line: 4, arg: 1, scope: !4, file: !1, type: !2)
+!4 = !DISubprogram(name: "foo", linkageName: "foo", line: 4, isLocal: false, isDefinition: true, virtualIndex: 6, isOptimized: false, scope: !1, type: !5)
+!5 = !DISubroutineType(types: !6)
 !6 = !{!2, !2}
-!7 = !MDLocation(line: 5, scope: !8)
-!8 = distinct !MDLexicalBlock(line: 0, column: 0, file: !20, scope: !4)
-!9 = !MDLocation(line: 6, scope: !8)
-!10 = !MDLocation(line: 7, scope: !8)
-!11 = !MDLocation(line: 9, scope: !8)
-!12 = !MDLocation(line: 11, scope: !8)
-!13 = !MDLocation(line: 14, scope: !14)
-!14 = distinct !MDLexicalBlock(line: 0, column: 0, file: !20, scope: !15)
-!15 = !MDSubprogram(name: "bar", linkageName: "bar", line: 13, isLocal: false, isDefinition: true, virtualIndex: 6, isOptimized: false, scope: !1, type: !16)
-!16 = !MDSubroutineType(types: !17)
+!7 = !DILocation(line: 5, scope: !8)
+!8 = distinct !DILexicalBlock(line: 0, column: 0, file: !20, scope: !4)
+!9 = !DILocation(line: 6, scope: !8)
+!10 = !DILocation(line: 7, scope: !8)
+!11 = !DILocation(line: 9, scope: !8)
+!12 = !DILocation(line: 11, scope: !8)
+!13 = !DILocation(line: 14, scope: !14)
+!14 = distinct !DILexicalBlock(line: 0, column: 0, file: !20, scope: !15)
+!15 = !DISubprogram(name: "bar", linkageName: "bar", line: 13, isLocal: false, isDefinition: true, virtualIndex: 6, isOptimized: false, scope: !1, type: !16)
+!16 = !DISubroutineType(types: !17)
 !17 = !{!2}
-!18 = !MDLocation(line: 15, scope: !14)
-!19 = !MDLocation(line: 16, scope: !14)
-!20 = !MDFile(filename: "g.c", directory: "/tmp")
+!18 = !DILocation(line: 15, scope: !14)
+!19 = !DILocation(line: 16, scope: !14)
+!20 = !DIFile(filename: "g.c", directory: "/tmp")
 !21 = !{i32 0}
diff --git a/test/Transforms/GlobalOpt/cleanup-pointer-root-users.ll b/test/Transforms/GlobalOpt/cleanup-pointer-root-users.ll
index b6dfdea0610..16da5315db0 100644
--- a/test/Transforms/GlobalOpt/cleanup-pointer-root-users.ll
+++ b/test/Transforms/GlobalOpt/cleanup-pointer-root-users.ll
@@ -30,7 +30,7 @@ define void @test2() {
 declare i8* @strdup(i8*)
 declare void @foo2(i8*)
 
-define void @test3() uwtable {
+define void @test3() uwtable personality i32 (i32, i64, i8*, i8*)* @__gxx_personality_v0 {
 ; CHECK-LABEL: @test3(
 ; CHECK-NOT: bb1:
 ; CHECK-NOT: bb2:
@@ -41,7 +41,7 @@ bb1:
   store i8* %ptr, i8** @glbl
   unreachable
 bb2:
-  %tmp1 = landingpad { i8*, i32 } personality i32 (i32, i64, i8*, i8*)* @__gxx_personality_v0
+  %tmp1 = landingpad { i8*, i32 }
           cleanup
   resume { i8*, i32 } %tmp1
 }
diff --git a/test/Transforms/GlobalOpt/invoke.ll b/test/Transforms/GlobalOpt/invoke.ll
index c1f499c38a3..a0f7890a985 100644
--- a/test/Transforms/GlobalOpt/invoke.ll
+++ b/test/Transforms/GlobalOpt/invoke.ll
@@ -11,7 +11,7 @@ define i32 @one() {
   ret i32 1
 }
 
-define void @_GLOBAL__I_a() {
+define void @_GLOBAL__I_a() personality i8* undef {
 bb:
   %tmp1 = invoke i32 @one()
           to label %bb2 unwind label %bb4
@@ -21,7 +21,7 @@ bb2:                                              ; preds = %bb
   ret void
 
 bb4:                                              ; preds = %bb
-  %tmp5 = landingpad { i8*, i32 } personality i8* undef
+  %tmp5 = landingpad { i8*, i32 }
           filter [0 x i8*] zeroinitializer
   unreachable
 }
diff --git a/test/Transforms/GlobalOpt/tls.ll b/test/Transforms/GlobalOpt/tls.ll
index f3cb4a65704..d010b96188f 100644
--- a/test/Transforms/GlobalOpt/tls.ll
+++ b/test/Transforms/GlobalOpt/tls.ll
@@ -1,4 +1,5 @@
 ; RUN: opt < %s -globalopt -S | FileCheck %s
+; RUN: opt -emulated-tls < %s -globalopt -S | FileCheck %s
 
 declare void @wait()
 declare void @signal()
diff --git a/test/Transforms/IPConstantProp/return-argument.ll b/test/Transforms/IPConstantProp/return-argument.ll
index 927531b996e..0290adc4b78 100644
--- a/test/Transforms/IPConstantProp/return-argument.ll
+++ b/test/Transforms/IPConstantProp/return-argument.ll
@@ -27,7 +27,7 @@ define internal { i32, i32 } @foo(i32 %A, i32 %B) {
         ret { i32, i32 } %Z
 }
 
-define void @caller(i1 %C) {
+define void @caller(i1 %C) personality i32 (...)* @__gxx_personality_v0 {
         %Q = alloca i32
         ;; Call incdec to see if %W is properly replaced by %Q
         %W = call i32* @incdec(i1 %C, i32* %Q )             ; <i32> [#uses=1]
@@ -46,7 +46,7 @@ OK:
         br label %RET
 
 LPAD:
-        %exn = landingpad {i8*, i32} personality i32 (...)* @__gxx_personality_v0
+        %exn = landingpad {i8*, i32}
                  cleanup
         br label %RET
 
diff --git a/test/Transforms/IPConstantProp/return-constant.ll b/test/Transforms/IPConstantProp/return-constant.ll
index 499d3832952..195420d6aad 100644
--- a/test/Transforms/IPConstantProp/return-constant.ll
+++ b/test/Transforms/IPConstantProp/return-constant.ll
@@ -1,5 +1,4 @@
-; RUN: opt < %s -ipconstprop -instcombine | \
-; RUN:    llvm-dis | grep "ret i1 true" | count 2
+; RUN: opt < %s -ipconstprop -instcombine -S | grep "ret i1 true" | count 2
 define internal i32 @foo(i1 %C) {
         br i1 %C, label %T, label %F
 
@@ -16,13 +15,13 @@ define i1 @caller(i1 %C) {
         ret i1 %Y
 }
 
-define i1 @invokecaller(i1 %C) {
+define i1 @invokecaller(i1 %C) personality i32 (...)* @__gxx_personality_v0 {
         %X = invoke i32 @foo( i1 %C ) to label %OK unwind label %FAIL             ; <i32> [#uses=1]
 OK:
         %Y = icmp ne i32 %X, 0          ; <i1> [#uses=1]
         ret i1 %Y 
 FAIL:
-        %exn = landingpad {i8*, i32} personality i32 (...)* @__gxx_personality_v0
+        %exn = landingpad {i8*, i32}
                  cleanup
         ret i1 false
 }
diff --git a/test/Transforms/IndVarSimplify/2004-04-05-InvokeCastCrash.ll b/test/Transforms/IndVarSimplify/2004-04-05-InvokeCastCrash.ll
index 0c88e83975c..06eec7dfe28 100644
--- a/test/Transforms/IndVarSimplify/2004-04-05-InvokeCastCrash.ll
+++ b/test/Transforms/IndVarSimplify/2004-04-05-InvokeCastCrash.ll
@@ -112,13 +112,13 @@ declare void @_ZNK4llvm19MachineInstrBuilder7addSImmEi()
 
 declare i32 @__gxx_personality_v0(...)
 
-define void @_ZN4llvm11_GLOBAL__N_22InsertPrologEpilogCode20runOnMachineFunctionERNS_15MachineFunctionE(%"struct.llvm::MachineFunction"* %F) {
+define void @_ZN4llvm11_GLOBAL__N_22InsertPrologEpilogCode20runOnMachineFunctionERNS_15MachineFunctionE(%"struct.llvm::MachineFunction"* %F) personality i32 (...)* @__gxx_personality_v0 {
 entry:
 	%tmp.8.i = invoke %"struct.llvm::TargetFrameInfo"* null( %"struct.llvm::TargetMachine"* null )
 			to label %invoke_cont.0.i unwind label %invoke_catch.0.i		; <%"struct.llvm::TargetFrameInfo"*> [#uses=0]
 
 invoke_catch.0.i:		; preds = %invoke_cont.49.i, %invoke_cont.48.i, %invoke_cont.47.i, %invoke_cont.i53.i, %no_exit.i, %invoke_cont.44.i, %invoke_cont.43.i, %invoke_cont.42.i, %invoke_cont.41.i, %invoke_cont.40.i, %invoke_cont.39.i, %invoke_cont.38.i, %invoke_cont.37.i, %then.2.i, %invoke_cont.35.i, %invoke_cont.34.i, %then.1.i, %endif.0.i, %invoke_cont.9.i, %invoke_cont.8.i, %invoke_cont.7.i, %invoke_cont.i.i, %then.0.i, %invoke_cont.4.i, %invoke_cont.3.i, %invoke_cont.2.i, %invoke_cont.1.i, %endif.0.i.i, %tmp.7.i.noexc.i, %invoke_cont.0.i, %entry
-        %exn0.i = landingpad {i8*, i32} personality i32 (...)* @__gxx_personality_v0
+        %exn0.i = landingpad {i8*, i32}
                  cleanup
 	ret void
 
@@ -168,7 +168,7 @@ tmp.0.i.noexc.i:		; preds = %then.0.i
 			to label %invoke_cont.i.i unwind label %cond_true.i.i
 
 cond_true.i.i:		; preds = %tmp.0.i.noexc.i
-        %exn.i.i = landingpad {i8*, i32} personality i32 (...)* @__gxx_personality_v0
+        %exn.i.i = landingpad {i8*, i32}
                  cleanup
 	ret void
 
@@ -262,7 +262,7 @@ tmp.0.i.noexc55.i:		; preds = %no_exit.i
 			to label %invoke_cont.i53.i unwind label %cond_true.i52.i
 
 cond_true.i52.i:		; preds = %tmp.0.i.noexc55.i
-        %exn.i52.i = landingpad {i8*, i32} personality i32 (...)* @__gxx_personality_v0
+        %exn.i52.i = landingpad {i8*, i32}
                  cleanup
 	ret void
 
diff --git a/test/Transforms/IndVarSimplify/2005-02-11-InvokeCrash.ll b/test/Transforms/IndVarSimplify/2005-02-11-InvokeCrash.ll
index a5706ca6198..926b82fdf14 100644
--- a/test/Transforms/IndVarSimplify/2005-02-11-InvokeCrash.ll
+++ b/test/Transforms/IndVarSimplify/2005-02-11-InvokeCrash.ll
@@ -1,6 +1,6 @@
 ; RUN: opt < %s -indvars -disable-output
 
-define void @_ZN5ArrayISt7complexIdEEC2ERK10dim_vector() {
+define void @_ZN5ArrayISt7complexIdEEC2ERK10dim_vector() personality i32 (...)* @__gxx_personality_v0 {
 entry:
         %tmp.7 = invoke i32 @_ZN5ArrayISt7complexIdEE8get_sizeERK10dim_vector( )
                         to label %invoke_cont.0 unwind label %cond_true.1               ; <i32> [#uses=2]
@@ -16,7 +16,7 @@ no_exit.i:              ; preds = %no_exit.i, %invoke_cont.0
         br label %no_exit.i
 
 cond_true.1:            ; preds = %entry
-        %exn = landingpad {i8*, i32} personality i32 (...)* @__gxx_personality_v0
+        %exn = landingpad {i8*, i32}
                  cleanup
         resume { i8*, i32 } %exn
 }
diff --git a/test/Transforms/IndVarSimplify/2005-02-17-TruncateExprCrash.ll b/test/Transforms/IndVarSimplify/2005-02-17-TruncateExprCrash.ll
index 16ad635a920..a0dac7a29cc 100644
--- a/test/Transforms/IndVarSimplify/2005-02-17-TruncateExprCrash.ll
+++ b/test/Transforms/IndVarSimplify/2005-02-17-TruncateExprCrash.ll
@@ -4,13 +4,13 @@ declare void @q_atomic_increment()
 
 declare void @_Z9qt_assertPKcS0_i()
 
-define void @_ZN13QMetaResourceC1EPKh() {
+define void @_ZN13QMetaResourceC1EPKh() personality i32 (...)* @__gxx_personality_v0 {
 entry:
 	invoke void @_Z9qt_assertPKcS0_i( )
 			to label %endif.1 unwind label %then.i.i551
 
 then.i.i551:		; preds = %entry
-        %exn551 = landingpad {i8*, i32} personality i32 (...)* @__gxx_personality_v0
+        %exn551 = landingpad {i8*, i32}
                  cleanup
 	ret void
 
@@ -22,7 +22,7 @@ then.2:		; preds = %endif.1
 			to label %loopentry.0 unwind label %invoke_catch.6
 
 invoke_catch.6:		; preds = %then.2
-        %exn6 = landingpad {i8*, i32} personality i32 (...)* @__gxx_personality_v0
+        %exn6 = landingpad {i8*, i32}
                  cleanup
 	ret void
 
diff --git a/test/Transforms/IndVarSimplify/crash.ll b/test/Transforms/IndVarSimplify/crash.ll
index aa6a2ee1652..63683ff5611 100644
--- a/test/Transforms/IndVarSimplify/crash.ll
+++ b/test/Transforms/IndVarSimplify/crash.ll
@@ -62,7 +62,7 @@ declare void @__go_undefer()
 
 declare i32 @__gccgo_personality_v0(i32, i64, i8*, i8*)
 
-define void @main.main() uwtable {
+define void @main.main() uwtable personality i32 (i32, i64, i8*, i8*)* @__gccgo_personality_v0 {
 entry:
   invoke void @__go_panic() noreturn
           to label %0 unwind label %"5.i"
@@ -75,12 +75,12 @@ entry:
           to label %main.f.exit unwind label %"7.i"
 
 "5.i":                                            ; preds = %entry
-  %1 = landingpad { i8*, i32 } personality i32 (i32, i64, i8*, i8*)* @__gccgo_personality_v0
+  %1 = landingpad { i8*, i32 }
           catch i8* null
   br label %"3.i"
 
 "7.i":                                            ; preds = %"3.i"
-  %2 = landingpad { i8*, i32 } personality i32 (i32, i64, i8*, i8*)* @__gccgo_personality_v0
+  %2 = landingpad { i8*, i32 }
           catch i8* null
   br label %"3.i"
 
diff --git a/test/Transforms/IndVarSimplify/exit_value_test2.ll b/test/Transforms/IndVarSimplify/exit_value_test2.ll
new file mode 100644
index 00000000000..24e3e95a891
--- /dev/null
+++ b/test/Transforms/IndVarSimplify/exit_value_test2.ll
@@ -0,0 +1,52 @@
+; PR23538
+; RUN: opt < %s -indvars -loop-deletion -S | FileCheck %s
+
+; Check IndVarSimplify should not replace exit value because or else
+; udiv will be introduced by expand and the cost will be high.
+;
+; CHECK-LABEL: @_Z3fooPKcjj(
+; CHECK-NOT: udiv
+
+declare void @_Z3mixRjj(i32* dereferenceable(4), i32)
+declare void @llvm.lifetime.start(i64, i8* nocapture)
+declare void @llvm.lifetime.end(i64, i8* nocapture)
+
+define i32 @_Z3fooPKcjj(i8* nocapture readonly %s, i32 %len, i32 %c) {
+entry:
+  %a = alloca i32, align 4
+  %tmp = bitcast i32* %a to i8*
+  call void @llvm.lifetime.start(i64 4, i8* %tmp)
+  store i32 -1640531527, i32* %a, align 4
+  %cmp8 = icmp ugt i32 %len, 11
+  br i1 %cmp8, label %while.body.lr.ph, label %while.end
+
+while.body.lr.ph:                                 ; preds = %entry
+  br label %while.body
+
+while.body:                                       ; preds = %while.body, %while.body.lr.ph
+  %keylen.010 = phi i32 [ %len, %while.body.lr.ph ], [ %sub, %while.body ]
+  %s.addr.09 = phi i8* [ %s, %while.body.lr.ph ], [ %add.ptr, %while.body ]
+  %tmp1 = bitcast i8* %s.addr.09 to i32*
+  %tmp2 = load i32, i32* %tmp1, align 4
+  %shl.i = shl i32 %tmp2, 1
+  %and.i = and i32 %shl.i, 16843008
+  %tmp3 = load i32, i32* %a, align 4
+  %sub.i = add i32 %tmp3, %tmp2
+  %add = sub i32 %sub.i, %and.i
+  store i32 %add, i32* %a, align 4
+  %add.ptr = getelementptr inbounds i8, i8* %s.addr.09, i64 12
+  %sub = add i32 %keylen.010, -12
+  %cmp = icmp ugt i32 %sub, 11
+  br i1 %cmp, label %while.body, label %while.cond.while.end_crit_edge
+
+while.cond.while.end_crit_edge:                   ; preds = %while.body
+  %sub.lcssa = phi i32 [ %sub, %while.body ]
+  br label %while.end
+
+while.end:                                        ; preds = %while.cond.while.end_crit_edge, %entry
+  %keylen.0.lcssa = phi i32 [ %sub.lcssa, %while.cond.while.end_crit_edge ], [ %len, %entry ]
+  call void @_Z3mixRjj(i32* dereferenceable(4) %a, i32 %keylen.0.lcssa)
+  %tmp4 = load i32, i32* %a, align 4
+  call void @llvm.lifetime.end(i64 4, i8* %tmp)
+  ret i32 %tmp4
+}
diff --git a/test/Transforms/IndVarSimplify/exit_value_test3.ll b/test/Transforms/IndVarSimplify/exit_value_test3.ll
new file mode 100644
index 00000000000..2051d2a46b2
--- /dev/null
+++ b/test/Transforms/IndVarSimplify/exit_value_test3.ll
@@ -0,0 +1,24 @@
+; RUN: opt < %s -indvars -loop-deletion -S |FileCheck %s
+
+; Check IndVarSimplify should replace exit value even if the expansion cost
+; is high because the loop can be deleted after the exit value rewrite.
+;
+; CHECK-LABEL: @_Z3fooPKcjj(
+; CHECK: udiv
+; CHECK: [[LABEL:^[a-zA-Z0-9_.]+]]:
+; CHECK-NOT: br {{.*}} [[LABEL]]
+
+define i32 @_Z3fooPKcjj(i8* nocapture readnone %s, i32 %len, i32 %c) #0 {
+entry:
+  br label %while.cond
+
+while.cond:                                       ; preds = %while.cond, %entry
+  %klen.0 = phi i32 [ %len, %entry ], [ %sub, %while.cond ]
+  %cmp = icmp ugt i32 %klen.0, 11
+  %sub = add i32 %klen.0, -12
+  br i1 %cmp, label %while.cond, label %while.end
+
+while.end:                                        ; preds = %while.cond
+  %klen.0.lcssa = phi i32 [ %klen.0, %while.cond ]
+  ret i32 %klen.0.lcssa
+}
diff --git a/test/Transforms/IndVarSimplify/interesting-invoke-use.ll b/test/Transforms/IndVarSimplify/interesting-invoke-use.ll
index 69bea6eaaed..131b02c3aae 100644
--- a/test/Transforms/IndVarSimplify/interesting-invoke-use.ll
+++ b/test/Transforms/IndVarSimplify/interesting-invoke-use.ll
@@ -11,7 +11,7 @@ target triple = "i386-pc-linux-gnu"
 @.str7 = external constant [24 x i8]            ; <[24 x i8]*> [#uses=1]
 @C.17.316 = external constant %struct.string___XUB              ; <%struct.string___XUB*> [#uses=1]
 
-define void @_ada_c35503g() {
+define void @_ada_c35503g() personality i32 (...)* @__gxx_personality_v0 {
 entry:
   br label %bb
 
@@ -47,7 +47,7 @@ bb178:          ; preds = %invcont127
   br label %bb123
 
 lpad266:                ; preds = %invcont129, %bb128, %bb123
-  %exn = landingpad {i8*, i32} personality i32 (...)* @__gxx_personality_v0
+  %exn = landingpad {i8*, i32}
             cleanup
   unreachable
 }
diff --git a/test/Transforms/IndVarSimplify/lcssa-preservation.ll b/test/Transforms/IndVarSimplify/lcssa-preservation.ll
index f69c96ce021..5d502f3b6ea 100644
--- a/test/Transforms/IndVarSimplify/lcssa-preservation.ll
+++ b/test/Transforms/IndVarSimplify/lcssa-preservation.ll
@@ -1,5 +1,4 @@
-; RUN: opt < %s -indvars -S | FileCheck %s
-;
+; RUN: opt < %s -indvars -replexitval=always -S | FileCheck %s
 ; Make sure IndVars preserves LCSSA form, especially across loop nests. 
 
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64"
diff --git a/test/Transforms/IndVarSimplify/loop-invariant-conditions.ll b/test/Transforms/IndVarSimplify/loop-invariant-conditions.ll
new file mode 100644
index 00000000000..eee321da239
--- /dev/null
+++ b/test/Transforms/IndVarSimplify/loop-invariant-conditions.ll
@@ -0,0 +1,279 @@
+; RUN: opt -S -indvars %s | FileCheck %s
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+define void @test1(i64 %start) {
+; CHECK-LABEL: @test1
+entry:
+  br label %loop
+
+loop:
+  %indvars.iv = phi i64 [ %start, %entry ], [ %indvars.iv.next, %loop ]
+  %indvars.iv.next = add nsw i64 %indvars.iv, 1
+; CHECK: %cmp1 = icmp slt i64 %start, -1
+  %cmp1 = icmp slt i64 %indvars.iv, -1
+  br i1 %cmp1, label %for.end, label %loop
+
+for.end:                                          ; preds = %if.end, %entry
+  ret void
+}
+
+define void @test2(i64 %start) {
+; CHECK-LABEL: @test2
+entry:
+  br label %loop
+
+loop:
+  %indvars.iv = phi i64 [ %start, %entry ], [ %indvars.iv.next, %loop ]
+  %indvars.iv.next = add nsw i64 %indvars.iv, 1
+; CHECK: %cmp1 = icmp sle i64 %start, -1
+  %cmp1 = icmp sle i64 %indvars.iv, -1
+  br i1 %cmp1, label %for.end, label %loop
+
+for.end:                                          ; preds = %if.end, %entry
+  ret void
+}
+
+; As long as the test dominates the backedge, we're good
+define void @test3(i64 %start) {
+; CHECK-LABEL: @test3
+entry:
+  br label %loop
+
+loop:
+  %indvars.iv = phi i64 [ %start, %entry ], [ %indvars.iv.next, %backedge ]
+  %indvars.iv.next = add nsw i64 %indvars.iv, 1
+  %cmp = icmp eq i64 %indvars.iv.next, 25
+  br i1 %cmp, label %backedge, label %for.end
+
+backedge:
+  ; prevent flattening, needed to make sure we're testing what we intend
+  call void @foo() 
+; CHECK: %cmp1 = icmp slt i64 %start, -1
+  %cmp1 = icmp slt i64 %indvars.iv, -1
+  br i1 %cmp1, label %for.end, label %loop
+
+for.end:                                          ; preds = %if.end, %entry
+  ret void
+}
+
+define void @test4(i64 %start) {
+; CHECK-LABEL: @test4
+entry:
+  br label %loop
+
+loop:
+  %indvars.iv = phi i64 [ %start, %entry ], [ %indvars.iv.next, %backedge ]
+  %indvars.iv.next = add nsw i64 %indvars.iv, 1
+  %cmp = icmp eq i64 %indvars.iv.next, 25
+  br i1 %cmp, label %backedge, label %for.end
+
+backedge:
+  ; prevent flattening, needed to make sure we're testing what we intend
+  call void @foo() 
+; CHECK: %cmp1 = icmp sgt i64 %start, -1
+  %cmp1 = icmp sgt i64 %indvars.iv, -1
+  br i1 %cmp1, label %loop, label %for.end
+
+for.end:                                          ; preds = %if.end, %entry
+  ret void
+}
+
+define void @test5(i64 %start) {
+; CHECK-LABEL: @test5
+entry:
+  br label %loop
+
+loop:
+  %indvars.iv = phi i64 [ %start, %entry ], [ %indvars.iv.next, %backedge ]
+  %indvars.iv.next = add nuw i64 %indvars.iv, 1
+  %cmp = icmp eq i64 %indvars.iv.next, 25
+  br i1 %cmp, label %backedge, label %for.end
+
+backedge:
+  ; prevent flattening, needed to make sure we're testing what we intend
+  call void @foo() 
+; CHECK: %cmp1 = icmp ugt i64 %start, 100
+  %cmp1 = icmp ugt i64 %indvars.iv, 100
+  br i1 %cmp1, label %loop, label %for.end
+
+for.end:                                          ; preds = %if.end, %entry
+  ret void
+}
+
+define void @test6(i64 %start) {
+; CHECK-LABEL: @test6
+entry:
+  br label %loop
+
+loop:
+  %indvars.iv = phi i64 [ %start, %entry ], [ %indvars.iv.next, %backedge ]
+  %indvars.iv.next = add nuw i64 %indvars.iv, 1
+  %cmp = icmp eq i64 %indvars.iv.next, 25
+  br i1 %cmp, label %backedge, label %for.end
+
+backedge:
+  ; prevent flattening, needed to make sure we're testing what we intend
+  call void @foo() 
+; CHECK: %cmp1 = icmp ult i64 %start, 100
+  %cmp1 = icmp ult i64 %indvars.iv, 100
+  br i1 %cmp1, label %for.end, label %loop
+
+for.end:                                          ; preds = %if.end, %entry
+  ret void
+}
+
+define void @test7(i64 %start, i64* %inc_ptr) {
+; CHECK-LABEL: @test7
+entry:
+  %inc = load i64, i64* %inc_ptr, !range !0
+  %ok = icmp sge i64 %inc, 0
+  br i1 %ok, label %loop, label %for.end
+
+loop:
+  %indvars.iv = phi i64 [ %start, %entry ], [ %indvars.iv.next, %loop ]
+  %indvars.iv.next = add nsw i64 %indvars.iv, %inc
+; CHECK: %cmp1 = icmp slt i64 %start, -1
+  %cmp1 = icmp slt i64 %indvars.iv, -1
+  br i1 %cmp1, label %for.end, label %loop
+
+for.end:                                          ; preds = %if.end, %entry
+  ret void
+}
+
+!0 = !{i64 0, i64 100}
+
+; Negative test - we can't show that the internal branch executes, so we can't
+; fold the test to a loop invariant one.
+define void @test1_neg(i64 %start) {
+; CHECK-LABEL: @test1_neg
+entry:
+  br label %loop
+
+loop:
+  %indvars.iv = phi i64 [ %start, %entry ], [ %indvars.iv.next, %backedge ]
+  %indvars.iv.next = add nsw i64 %indvars.iv, 1
+  %cmp = icmp eq i64 %indvars.iv.next, 25
+  br i1 %cmp, label %backedge, label %skip
+skip:
+  ; prevent flattening, needed to make sure we're testing what we intend
+  call void @foo() 
+; CHECK: %cmp1 = icmp slt i64 %indvars.iv, -1
+  %cmp1 = icmp slt i64 %indvars.iv, -1
+  br i1 %cmp1, label %for.end, label %backedge
+backedge:
+  ; prevent flattening, needed to make sure we're testing what we intend
+  call void @foo() 
+  br label %loop
+
+for.end:                                          ; preds = %if.end, %entry
+  ret void
+}
+
+; Slightly subtle version of @test4 where the icmp dominates the backedge,
+; but the exit branch doesn't.  
+define void @test2_neg(i64 %start) {
+; CHECK-LABEL: @test2_neg
+entry:
+  br label %loop
+
+loop:
+  %indvars.iv = phi i64 [ %start, %entry ], [ %indvars.iv.next, %backedge ]
+  %indvars.iv.next = add nsw i64 %indvars.iv, 1
+  %cmp = icmp eq i64 %indvars.iv.next, 25
+; CHECK: %cmp1 = icmp slt i64 %indvars.iv, -1
+  %cmp1 = icmp slt i64 %indvars.iv, -1
+  br i1 %cmp, label %backedge, label %skip
+skip:
+  ; prevent flattening, needed to make sure we're testing what we intend
+  call void @foo() 
+  br i1 %cmp1, label %for.end, label %backedge
+backedge:
+  ; prevent flattening, needed to make sure we're testing what we intend
+  call void @foo() 
+  br label %loop
+
+for.end:                                          ; preds = %if.end, %entry
+  ret void
+}
+
+; The branch has to exit the loop if the condition is true
+define void @test3_neg(i64 %start) {
+; CHECK-LABEL: @test3_neg
+entry:
+  br label %loop
+
+loop:
+  %indvars.iv = phi i64 [ %start, %entry ], [ %indvars.iv.next, %loop ]
+  %indvars.iv.next = add nsw i64 %indvars.iv, 1
+; CHECK: %cmp1 = icmp slt i64 %indvars.iv, -1
+  %cmp1 = icmp slt i64 %indvars.iv, -1
+  br i1 %cmp1, label %loop, label %for.end
+
+for.end:                                          ; preds = %if.end, %entry
+  ret void
+}
+
+define void @test4_neg(i64 %start) {
+; CHECK-LABEL: @test4_neg
+entry:
+  br label %loop
+
+loop:
+  %indvars.iv = phi i64 [ %start, %entry ], [ %indvars.iv.next, %backedge ]
+  %indvars.iv.next = add nsw i64 %indvars.iv, 1
+  %cmp = icmp eq i64 %indvars.iv.next, 25
+  br i1 %cmp, label %backedge, label %for.end
+
+backedge:
+  ; prevent flattening, needed to make sure we're testing what we intend
+  call void @foo() 
+; CHECK: %cmp1 = icmp sgt i64 %indvars.iv, -1
+  %cmp1 = icmp sgt i64 %indvars.iv, -1
+
+; %cmp1 can be made loop invariant only if the branch below goes to
+; %the header when %cmp1 is true.
+  br i1 %cmp1, label %for.end, label %loop
+
+for.end:                                          ; preds = %if.end, %entry
+  ret void
+}
+
+define void @test5_neg(i64 %start, i64 %inc) {
+; CHECK-LABEL: @test5_neg
+entry:
+  br label %loop
+
+loop:
+  %indvars.iv = phi i64 [ %start, %entry ], [ %indvars.iv.next, %loop ]
+  %indvars.iv.next = add nsw i64 %indvars.iv, %inc
+; CHECK: %cmp1 = icmp slt i64 %indvars.iv, -1
+  %cmp1 = icmp slt i64 %indvars.iv, -1
+  br i1 %cmp1, label %for.end, label %loop
+
+for.end:                                          ; preds = %if.end, %entry
+  ret void
+}
+
+define void @test8(i64 %start, i64* %inc_ptr) {
+; CHECK-LABEL: @test8
+entry:
+  %inc = load i64, i64* %inc_ptr, !range !1
+  %ok = icmp sge i64 %inc, 0
+  br i1 %ok, label %loop, label %for.end
+
+loop:
+  %indvars.iv = phi i64 [ %start, %entry ], [ %indvars.iv.next, %loop ]
+  %indvars.iv.next = add nsw i64 %indvars.iv, %inc
+; CHECK: %cmp1 = icmp slt i64 %indvars.iv, -1
+  %cmp1 = icmp slt i64 %indvars.iv, -1
+  br i1 %cmp1, label %for.end, label %loop
+
+for.end:                                          ; preds = %if.end, %entry
+  ret void
+}
+
+!1 = !{i64 -1, i64 100}
+
+
+declare void @foo()
diff --git a/test/Transforms/IndVarSimplify/lrev-existing-umin.ll b/test/Transforms/IndVarSimplify/lrev-existing-umin.ll
new file mode 100644
index 00000000000..961c9fd944d
--- /dev/null
+++ b/test/Transforms/IndVarSimplify/lrev-existing-umin.ll
@@ -0,0 +1,36 @@
+; RUN: opt -S -indvars < %s | FileCheck %s
+
+define void @f(i32 %length.i.88, i32 %length.i, i8* %tmp12, i32 %tmp10, i8* %tmp8) {
+; CHECK-LABEL: @f(
+not_zero11.preheader:
+  %tmp13 = icmp ugt i32 %length.i, %length.i.88
+  %tmp14 = select i1 %tmp13, i32 %length.i.88, i32 %length.i
+  %tmp15 = icmp sgt i32 %tmp14, 0
+  br i1 %tmp15, label %not_zero11, label %not_zero11.postloop
+
+not_zero11:
+  %v_1 = phi i32 [ %tmp22, %not_zero11 ], [ 0, %not_zero11.preheader ]
+  %tmp16 = zext i32 %v_1 to i64
+  %tmp17 = getelementptr inbounds i8, i8* %tmp8, i64 %tmp16
+  %tmp18 = load i8, i8* %tmp17, align 1
+  %tmp19 = zext i8 %tmp18 to i32
+  %tmp20 = or i32 %tmp19, %tmp10
+  %tmp21 = trunc i32 %tmp20 to i8
+  %addr22 = getelementptr inbounds i8, i8* %tmp12, i64 %tmp16
+  store i8 %tmp21, i8* %addr22, align 1
+  %tmp22 = add nuw nsw i32 %v_1, 1
+  %tmp23 = icmp slt i32 %tmp22, %tmp14
+  br i1 %tmp23, label %not_zero11, label %main.exit.selector
+
+main.exit.selector:
+; CHECK-LABEL: main.exit.selector:
+; CHECK: %tmp24 = icmp slt i32 %tmp14, %length.i
+  %tmp24 = icmp slt i32 %tmp22, %length.i
+  br i1 %tmp24, label %not_zero11.postloop, label %leave
+
+leave:
+  ret void
+
+not_zero11.postloop:
+  ret void
+}
diff --git a/test/Transforms/IndVarSimplify/tripcount_infinite.ll b/test/Transforms/IndVarSimplify/tripcount_infinite.ll
index 0495b50c3e4..658598d3b7e 100644
--- a/test/Transforms/IndVarSimplify/tripcount_infinite.ll
+++ b/test/Transforms/IndVarSimplify/tripcount_infinite.ll
@@ -1,38 +1,45 @@
 ; These tests have an infinite trip count.  We obviously shouldn't remove the 
 ; loops!  :)
 ;
-; RUN: opt < %s -indvars -adce -simplifycfg -S | grep icmp | wc -l > %t2
-; RUN: llvm-as < %s | llvm-dis | grep icmp | wc -l > %t1
-; RUN: diff %t1 %t2
+; RUN: opt < %s -indvars -adce -simplifycfg -S | FileCheck %s
 
 ;; test for (i = 1; i != 100; i += 2)
 define i32 @infinite_linear() {
+; CHECK-LABEL: @infinite_linear(
 entry:
         br label %loop
 
 loop:           ; preds = %loop, %entry
+; CHECK-LABEL: loop:
         %i = phi i32 [ 1, %entry ], [ %i.next, %loop ]          ; <i32> [#uses=3]
         %i.next = add i32 %i, 2         ; <i32> [#uses=1]
         %c = icmp ne i32 %i, 100                ; <i1> [#uses=1]
+; CHECK: icmp
+; CHECK: br
         br i1 %c, label %loop, label %loopexit
 
 loopexit:               ; preds = %loop
+; CHECK-LABEL: loopexit:
         ret i32 %i
 }
 
 ;; test for (i = 1; i*i != 63; ++i)
 define i32 @infinite_quadratic() {
+; CHECK-LABEL: @infinite_quadratic(
 entry:
         br label %loop
 
 loop:           ; preds = %loop, %entry
+; CHECK-LABEL: loop:
         %i = phi i32 [ 1, %entry ], [ %i.next, %loop ]          ; <i32> [#uses=4]
         %isquare = mul i32 %i, %i               ; <i32> [#uses=1]
         %i.next = add i32 %i, 1         ; <i32> [#uses=1]
         %c = icmp ne i32 %isquare, 63           ; <i1> [#uses=1]
+; CHECK: icmp
+; CHECK: br
         br i1 %c, label %loop, label %loopexit
 
 loopexit:               ; preds = %loop
+; CHECK-LABEL: loopexit:
         ret i32 %i
 }
-
diff --git a/test/Transforms/Inline/2003-09-14-InlineValue.ll b/test/Transforms/Inline/2003-09-14-InlineValue.ll
index 98bc08b378e..4f1f61b4c07 100644
--- a/test/Transforms/Inline/2003-09-14-InlineValue.ll
+++ b/test/Transforms/Inline/2003-09-14-InlineValue.ll
@@ -8,7 +8,7 @@ define internal i32 @Callee() {
         ret i32 %J
 }
 
-define i32 @Caller() {
+define i32 @Caller() personality i32 (...)* @__gxx_personality_v0 {
         %V = invoke i32 @Callee( )
                         to label %Ok unwind label %Bad          ; <i32> [#uses=1]
 
@@ -16,7 +16,7 @@ Ok:             ; preds = %0
         ret i32 %V
 
 Bad:            ; preds = %0
-        %exn = landingpad {i8*, i32} personality i32 (...)* @__gxx_personality_v0
+        %exn = landingpad {i8*, i32}
                  cleanup
         ret i32 0
 }
diff --git a/test/Transforms/Inline/2003-09-22-PHINodeInlineFail.ll b/test/Transforms/Inline/2003-09-22-PHINodeInlineFail.ll
index df0b472bb2a..9a5fcaeea7d 100644
--- a/test/Transforms/Inline/2003-09-22-PHINodeInlineFail.ll
+++ b/test/Transforms/Inline/2003-09-22-PHINodeInlineFail.ll
@@ -1,6 +1,6 @@
 ; RUN: opt < %s -inline -disable-output
 
-define i32 @main() {
+define i32 @main() personality i32 (...)* @__gxx_personality_v0 {
 entry:
         invoke void @__main( )
                         to label %LongJmpBlkPost unwind label %LongJmpBlkPre
@@ -10,7 +10,7 @@ LongJmpBlkPost:
 
 LongJmpBlkPre:
         %i.3 = phi i32 [ 0, %entry ]
-        %exn = landingpad {i8*, i32} personality i32 (...)* @__gxx_personality_v0
+        %exn = landingpad {i8*, i32}
                  cleanup
         ret i32 0
 }
diff --git a/test/Transforms/Inline/2003-09-22-PHINodesInExceptionDest.ll b/test/Transforms/Inline/2003-09-22-PHINodesInExceptionDest.ll
index d5416a20597..2311cdab518 100644
--- a/test/Transforms/Inline/2003-09-22-PHINodesInExceptionDest.ll
+++ b/test/Transforms/Inline/2003-09-22-PHINodesInExceptionDest.ll
@@ -1,6 +1,6 @@
 ; RUN: opt < %s -inline -disable-output
 
-define i32 @main() {
+define i32 @main() personality i32 (...)* @__gxx_personality_v0 {
 entry:
         invoke void @__main( )
                         to label %Call2Invoke unwind label %LongJmpBlkPre
@@ -10,7 +10,7 @@ Call2Invoke:            ; preds = %entry
 
 LongJmpBlkPre:          ; preds = %Call2Invoke, %entry
         %i.3 = phi i32 [ 0, %entry ]
-        %exn = landingpad {i8*, i32} personality i32 (...)* @__gxx_personality_v0
+        %exn = landingpad {i8*, i32}
                  cleanup
         br label %exit
 
diff --git a/test/Transforms/Inline/2003-09-22-PHINodesInNormalInvokeDest.ll b/test/Transforms/Inline/2003-09-22-PHINodesInNormalInvokeDest.ll
index 43bdd309c98..ce7d1fb3276 100644
--- a/test/Transforms/Inline/2003-09-22-PHINodesInNormalInvokeDest.ll
+++ b/test/Transforms/Inline/2003-09-22-PHINodesInNormalInvokeDest.ll
@@ -1,6 +1,6 @@
 ; RUN: opt < %s -inline -disable-output
 
-define i32 @main() {
+define i32 @main() personality i32 (...)* @__gxx_personality_v0 {
 entry:
         invoke void @__main( )
                         to label %else unwind label %RethrowExcept
@@ -13,7 +13,7 @@ LJDecisionBB:           ; preds = %else
         br label %else
 
 RethrowExcept:          ; preds = %entry
-        %exn = landingpad {i8*, i32} personality i32 (...)* @__gxx_personality_v0
+        %exn = landingpad {i8*, i32}
                  cleanup
         ret i32 0
 }
diff --git a/test/Transforms/Inline/2006-11-09-InlineCGUpdate-2.ll b/test/Transforms/Inline/2006-11-09-InlineCGUpdate-2.ll
index ee5a378b187..b4d630d8e38 100644
--- a/test/Transforms/Inline/2006-11-09-InlineCGUpdate-2.ll
+++ b/test/Transforms/Inline/2006-11-09-InlineCGUpdate-2.ll
@@ -126,13 +126,13 @@ entry:
 	unreachable
 }
 
-define fastcc void @_ZSt19__throw_logic_errorPKc() {
+define fastcc void @_ZSt19__throw_logic_errorPKc() personality i32 (...)* @__gxx_personality_v0 {
 entry:
 	invoke fastcc void @_ZNSt11logic_errorC1ERKSs( )
 			to label %try_exit.0 unwind label %try_catch.0
 
 try_catch.0:		; preds = %entry
-        %exn = landingpad {i8*, i32} personality i32 (...)* @__gxx_personality_v0
+        %exn = landingpad {i8*, i32}
                  catch i8* null
 	resume { i8*, i32 } %exn
 
@@ -157,13 +157,13 @@ entry:
 	unreachable
 }
 
-define fastcc void @_ZNSt12length_errorC1ERKSs() {
+define fastcc void @_ZNSt12length_errorC1ERKSs() personality i32 (...)* @__gxx_personality_v0 {
 entry:
 	invoke fastcc void @_ZNSsC1ERKSs( )
 			to label %_ZNSt11logic_errorC2ERKSs.exit unwind label %invoke_catch.i
 
 invoke_catch.i:		; preds = %entry
-        %exn = landingpad {i8*, i32} personality i32 (...)* @__gxx_personality_v0
+        %exn = landingpad {i8*, i32}
                  catch i8* null
 	resume { i8*, i32 } %exn
 
@@ -195,14 +195,14 @@ endif.1.i:		; preds = %then.1.i, %entry
 	unreachable
 }
 
-define fastcc void @_ZNSsC1ERKSs() {
+define fastcc void @_ZNSsC1ERKSs() personality i32 (...)* @__gxx_personality_v0 {
 entry:
 	call fastcc void @_ZNSs4_Rep7_M_grabERKSaIcES2_( )
 	invoke fastcc void @_ZNSaIcEC1ERKS_( )
 			to label %invoke_cont.1 unwind label %invoke_catch.1
 
 invoke_catch.1:		; preds = %entry
-        %exn = landingpad {i8*, i32} personality i32 (...)* @__gxx_personality_v0
+        %exn = landingpad {i8*, i32}
                  catch i8* null
 	call fastcc void @_ZNSaIcED1Ev( )
 	resume { i8*, i32 } %exn
diff --git a/test/Transforms/Inline/2006-11-09-InlineCGUpdate.ll b/test/Transforms/Inline/2006-11-09-InlineCGUpdate.ll
index fb5a4b512b9..8a613e534c7 100644
--- a/test/Transforms/Inline/2006-11-09-InlineCGUpdate.ll
+++ b/test/Transforms/Inline/2006-11-09-InlineCGUpdate.ll
@@ -170,14 +170,14 @@ endif.1.i:		; preds = %entry
 	unreachable
 }
 
-define fastcc void @_ZNSsC1ERKSs() {
+define fastcc void @_ZNSsC1ERKSs() personality i32 (...)* @__gxx_personality_v0 {
 entry:
 	call fastcc void @_ZNSs4_Rep7_M_grabERKSaIcES2_( )
 	invoke fastcc void @_ZNSaIcEC1ERKS_( )
 			to label %invoke_cont.1 unwind label %invoke_catch.1
 
 invoke_catch.1:		; preds = %entry
-        %exn = landingpad {i8*, i32} personality i32 (...)* @__gxx_personality_v0
+        %exn = landingpad {i8*, i32}
                  catch i8* null
 	call fastcc void @_ZNSaIcED1Ev( )
 	resume { i8*, i32 } %exn
@@ -301,13 +301,13 @@ entry:
 	unreachable
 }
 
-define fastcc void @_ZNSt12length_errorC1ERKSs() {
+define fastcc void @_ZNSt12length_errorC1ERKSs() personality i32 (...)* @__gxx_personality_v0 {
 entry:
 	invoke fastcc void @_ZNSsC1ERKSs( )
 			to label %_ZNSt11logic_errorC2ERKSs.exit unwind label %invoke_catch.i
 
 invoke_catch.i:		; preds = %entry
-        %exn = landingpad {i8*, i32} personality i32 (...)* @__gxx_personality_v0
+        %exn = landingpad {i8*, i32}
                  catch i8* null
 	resume { i8*, i32 } %exn
 
diff --git a/test/Transforms/Inline/2007-04-15-InlineEH.ll b/test/Transforms/Inline/2007-04-15-InlineEH.ll
index b114537490c..d8f94c8f005 100644
--- a/test/Transforms/Inline/2007-04-15-InlineEH.ll
+++ b/test/Transforms/Inline/2007-04-15-InlineEH.ll
@@ -12,7 +12,7 @@ entry:
 	unreachable
 }
 
-define fastcc void @bc__support__high_resolution_time__initialize_clock_rate() {
+define fastcc void @bc__support__high_resolution_time__initialize_clock_rate() personality i32 (...)* @__gxx_personality_v0 {
 entry:
 	invoke void @gnat__os_lib__getenv( %struct.gnat__strings__string_access* null )
 			to label %invcont unwind label %cleanup144
@@ -33,7 +33,7 @@ invcont67:		; preds = %invcont65
 	ret void
 
 cleanup144:		; preds = %invcont65, %invcont64, %invcont, %entry
-        %exn = landingpad {i8*, i32} personality i32 (...)* @__gxx_personality_v0
+        %exn = landingpad {i8*, i32}
                  cleanup
 	resume { i8*, i32 } %exn
 }
diff --git a/test/Transforms/Inline/X86/inline-target-attr.ll b/test/Transforms/Inline/X86/inline-target-attr.ll
new file mode 100644
index 00000000000..c59f4b4594b
--- /dev/null
+++ b/test/Transforms/Inline/X86/inline-target-attr.ll
@@ -0,0 +1,35 @@
+; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -S -inline | FileCheck %s
+; Check that we only inline when we have compatible target attributes.
+; X86 has implemented a target attribute that will verify that the attribute
+; sets are compatible.
+
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+define i32 @foo() #0 {
+entry:
+  %call = call i32 (...) @baz()
+  ret i32 %call
+; CHECK-LABEL: foo
+; CHECK: call i32 (...) @baz()
+}
+declare i32 @baz(...) #0
+
+define i32 @bar() #1 {
+entry:
+  %call = call i32 @foo()
+  ret i32 %call
+; CHECK-LABEL: bar
+; CHECK: call i32 (...) @baz()
+}
+
+define i32 @qux() #0 {
+entry:
+  %call = call i32 @bar()
+  ret i32 %call
+; CHECK-LABEL: qux
+; CHECK: call i32 @bar()
+}
+
+attributes #0 = { "target-cpu"="x86-64" "target-features"="+sse,+sse2" }
+attributes #1 = { "target-cpu"="x86-64" "target-features"="+sse,+sse2,+sse3,+sse4.1,+sse4.2,+ssse3" }
diff --git a/test/Transforms/Inline/X86/lit.local.cfg b/test/Transforms/Inline/X86/lit.local.cfg
new file mode 100644
index 00000000000..e71f3cc4c41
--- /dev/null
+++ b/test/Transforms/Inline/X86/lit.local.cfg
@@ -0,0 +1,3 @@
+if not 'X86' in config.root.targets:
+    config.unsupported = True
+
diff --git a/test/Transforms/Inline/alloca-dbgdeclare.ll b/test/Transforms/Inline/alloca-dbgdeclare.ll
index 7e649a22863..286f2931ff2 100644
--- a/test/Transforms/Inline/alloca-dbgdeclare.ll
+++ b/test/Transforms/Inline/alloca-dbgdeclare.ll
@@ -82,60 +82,60 @@ attributes #3 = { noreturn nounwind }
 !llvm.module.flags = !{!28, !29}
 !llvm.ident = !{!30}
 
-!0 = !MDCompileUnit(language: DW_LANG_C_plus_plus, producer: "clang version 3.7.0 (trunk 227480) (llvm/trunk 227517)", isOptimized: true, emissionKind: 1, file: !1, enums: !2, retainedTypes: !3, subprograms: !14, globals: !25, imports: !2)
-!1 = !MDFile(filename: "<stdin>", directory: "")
+!0 = !DICompileUnit(language: DW_LANG_C_plus_plus, producer: "clang version 3.7.0 (trunk 227480) (llvm/trunk 227517)", isOptimized: true, emissionKind: 1, file: !1, enums: !2, retainedTypes: !3, subprograms: !14, globals: !25, imports: !2)
+!1 = !DIFile(filename: "<stdin>", directory: "")
 !2 = !{}
 !3 = !{!4}
-!4 = !MDCompositeType(tag: DW_TAG_structure_type, name: "A", line: 1, size: 192, align: 64, file: !5, elements: !6, identifier: "_ZTS1A")
-!5 = !MDFile(filename: "test.cpp", directory: "")
+!4 = !DICompositeType(tag: DW_TAG_structure_type, name: "A", line: 1, size: 192, align: 64, file: !5, elements: !6, identifier: "_ZTS1A")
+!5 = !DIFile(filename: "test.cpp", directory: "")
 !6 = !{!7, !9}
-!7 = !MDDerivedType(tag: DW_TAG_member, name: "arg0", line: 2, size: 32, align: 32, file: !5, scope: !"_ZTS1A", baseType: !8)
-!8 = !MDBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
-!9 = !MDDerivedType(tag: DW_TAG_member, name: "arg1", line: 3, size: 128, align: 64, offset: 64, file: !5, scope: !"_ZTS1A", baseType: !10)
-!10 = !MDCompositeType(tag: DW_TAG_array_type, size: 128, align: 64, baseType: !11, elements: !12)
-!11 = !MDBasicType(tag: DW_TAG_base_type, name: "double", size: 64, align: 64, encoding: DW_ATE_float)
+!7 = !DIDerivedType(tag: DW_TAG_member, name: "arg0", line: 2, size: 32, align: 32, file: !5, scope: !"_ZTS1A", baseType: !8)
+!8 = !DIBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
+!9 = !DIDerivedType(tag: DW_TAG_member, name: "arg1", line: 3, size: 128, align: 64, offset: 64, file: !5, scope: !"_ZTS1A", baseType: !10)
+!10 = !DICompositeType(tag: DW_TAG_array_type, size: 128, align: 64, baseType: !11, elements: !12)
+!11 = !DIBasicType(tag: DW_TAG_base_type, name: "double", size: 64, align: 64, encoding: DW_ATE_float)
 !12 = !{!13}
-!13 = !MDSubrange(count: 2)
+!13 = !DISubrange(count: 2)
 !14 = !{!15, !21, !24}
-!15 = !MDSubprogram(name: "fn3", linkageName: "_Z3fn31A", line: 6, isLocal: false, isDefinition: true, flags: DIFlagPrototyped, isOptimized: true, scopeLine: 6, file: !5, scope: !16, type: !17, function: void (%struct.A*)* @_Z3fn31A, variables: !19)
-!16 = !MDFile(filename: "test.cpp", directory: "")
-!17 = !MDSubroutineType(types: !18)
+!15 = !DISubprogram(name: "fn3", linkageName: "_Z3fn31A", line: 6, isLocal: false, isDefinition: true, flags: DIFlagPrototyped, isOptimized: true, scopeLine: 6, file: !5, scope: !16, type: !17, function: void (%struct.A*)* @_Z3fn31A, variables: !19)
+!16 = !DIFile(filename: "test.cpp", directory: "")
+!17 = !DISubroutineType(types: !18)
 !18 = !{null, !"_ZTS1A"}
 !19 = !{!20}
-!20 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "p1", line: 6, arg: 1, scope: !15, file: !16, type: !"_ZTS1A")
-!21 = !MDSubprogram(name: "fn4", linkageName: "_Z3fn4v", line: 11, isLocal: false, isDefinition: true, flags: DIFlagPrototyped, isOptimized: true, scopeLine: 11, file: !5, scope: !16, type: !22, function: void ()* @_Z3fn4v, variables: !2)
-!22 = !MDSubroutineType(types: !23)
+!20 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "p1", line: 6, arg: 1, scope: !15, file: !16, type: !"_ZTS1A")
+!21 = !DISubprogram(name: "fn4", linkageName: "_Z3fn4v", line: 11, isLocal: false, isDefinition: true, flags: DIFlagPrototyped, isOptimized: true, scopeLine: 11, file: !5, scope: !16, type: !22, function: void ()* @_Z3fn4v, variables: !2)
+!22 = !DISubroutineType(types: !23)
 !23 = !{null}
-!24 = !MDSubprogram(name: "fn5", linkageName: "_Z3fn5v", line: 13, isLocal: false, isDefinition: true, flags: DIFlagPrototyped, isOptimized: true, scopeLine: 13, file: !5, scope: !16, type: !22, function: void ()* @_Z3fn5v, variables: !2)
+!24 = !DISubprogram(name: "fn5", linkageName: "_Z3fn5v", line: 13, isLocal: false, isDefinition: true, flags: DIFlagPrototyped, isOptimized: true, scopeLine: 13, file: !5, scope: !16, type: !22, function: void ()* @_Z3fn5v, variables: !2)
 !25 = !{!26, !27}
-!26 = !MDGlobalVariable(name: "a", line: 4, isLocal: false, isDefinition: true, scope: null, file: !16, type: !"_ZTS1A", variable: %struct.A* @a)
-!27 = !MDGlobalVariable(name: "b", line: 4, isLocal: false, isDefinition: true, scope: null, file: !16, type: !"_ZTS1A", variable: %struct.A* @b)
+!26 = !DIGlobalVariable(name: "a", line: 4, isLocal: false, isDefinition: true, scope: null, file: !16, type: !"_ZTS1A", variable: %struct.A* @a)
+!27 = !DIGlobalVariable(name: "b", line: 4, isLocal: false, isDefinition: true, scope: null, file: !16, type: !"_ZTS1A", variable: %struct.A* @b)
 !28 = !{i32 2, !"Dwarf Version", i32 4}
 !29 = !{i32 2, !"Debug Info Version", i32 3}
 !30 = !{!"clang version 3.7.0 (trunk 227480) (llvm/trunk 227517)"}
-!31 = !MDExpression(DW_OP_deref)
-!32 = !MDLocation(line: 6, scope: !15)
-!33 = !MDLocation(line: 7, scope: !34)
-!34 = distinct !MDLexicalBlock(line: 7, column: 0, file: !5, scope: !15)
+!31 = !DIExpression(DW_OP_deref)
+!32 = !DILocation(line: 6, scope: !15)
+!33 = !DILocation(line: 7, scope: !34)
+!34 = distinct !DILexicalBlock(line: 7, column: 0, file: !5, scope: !15)
 !35 = !{!36, !37, i64 0}
 !36 = !{!"_ZTS1A", !37, i64 0, !38, i64 8}
 !37 = !{!"int", !38, i64 0}
 !38 = !{!"omnipotent char", !39, i64 0}
 !39 = !{!"Simple C/C++ TBAA"}
-!40 = !MDLocation(line: 7, scope: !15)
-!41 = !MDLocation(line: 8, scope: !34)
+!40 = !DILocation(line: 7, scope: !15)
+!41 = !DILocation(line: 8, scope: !34)
 !42 = !{i64 0, i64 4, !43, i64 8, i64 16, !44}
 !43 = !{!37, !37, i64 0}
 !44 = !{!38, !38, i64 0}
-!45 = !MDLocation(line: 9, scope: !15)
-!46 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "p1", line: 6, arg: 1, scope: !15, file: !16, type: !"_ZTS1A")
-!47 = distinct !MDLocation(line: 11, scope: !21)
-!48 = !MDExpression(DW_OP_bit_piece, 32, 160)
-!49 = !MDLocation(line: 6, scope: !15, inlinedAt: !47)
-!50 = !MDLocation(line: 11, scope: !21)
-!51 = !MDExpression(DW_OP_bit_piece, 0, 32)
-!52 = !MDLocation(line: 7, scope: !34, inlinedAt: !47)
-!53 = !MDLocation(line: 7, scope: !15, inlinedAt: !47)
-!54 = !MDLocation(line: 8, scope: !34, inlinedAt: !47)
-!55 = !MDLocation(line: 14, scope: !24)
-!56 = !MDLocation(line: 15, scope: !24)
+!45 = !DILocation(line: 9, scope: !15)
+!46 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "p1", line: 6, arg: 1, scope: !15, file: !16, type: !"_ZTS1A")
+!47 = distinct !DILocation(line: 11, scope: !21)
+!48 = !DIExpression(DW_OP_bit_piece, 32, 160)
+!49 = !DILocation(line: 6, scope: !15, inlinedAt: !47)
+!50 = !DILocation(line: 11, scope: !21)
+!51 = !DIExpression(DW_OP_bit_piece, 0, 32)
+!52 = !DILocation(line: 7, scope: !34, inlinedAt: !47)
+!53 = !DILocation(line: 7, scope: !15, inlinedAt: !47)
+!54 = !DILocation(line: 8, scope: !34, inlinedAt: !47)
+!55 = !DILocation(line: 14, scope: !24)
+!56 = !DILocation(line: 15, scope: !24)
diff --git a/test/Transforms/Inline/byval-tail-call.ll b/test/Transforms/Inline/byval-tail-call.ll
index 1e50463ed80..7553b76cf15 100644
--- a/test/Transforms/Inline/byval-tail-call.ll
+++ b/test/Transforms/Inline/byval-tail-call.ll
@@ -33,7 +33,7 @@ define void @frob(i32* %x) {
 ; CHECK: %[[POS:.*]] = alloca i32
 ; CHECK: %[[VAL:.*]] = load i32, i32* %x
 ; CHECK: store i32 %[[VAL]], i32* %[[POS]]
-; CHECK: {{^ *}}call void @ext(i32* %[[POS]]
+; CHECK: {{^ *}}call void @ext(i32* nonnull %[[POS]]
 ; CHECK: tail call void @ext(i32* null)
 ; CHECK: ret void
   tail call void @qux(i32* byval %x)
diff --git a/test/Transforms/Inline/callgraph-update.ll b/test/Transforms/Inline/callgraph-update.ll
index b96fbc39c2f..1a1799e5cfb 100644
--- a/test/Transforms/Inline/callgraph-update.ll
+++ b/test/Transforms/Inline/callgraph-update.ll
@@ -21,7 +21,7 @@ define internal fastcc void @parse() {
 	ret void
 }
 
-define void @main() {
+define void @main() personality i32 (...)* @__gxx_personality_v0 {
 	invoke fastcc void @parse()
 			to label %invcont unwind label %lpad
 
@@ -29,7 +29,7 @@ invcont:
 	unreachable
 
 lpad:
-        %exn = landingpad {i8*, i32} personality i32 (...)* @__gxx_personality_v0
+        %exn = landingpad {i8*, i32}
                  cleanup
 	unreachable
 }
diff --git a/test/Transforms/Inline/crash-lifetime-marker.ll b/test/Transforms/Inline/crash-lifetime-marker.ll
new file mode 100644
index 00000000000..42e926b1f63
--- /dev/null
+++ b/test/Transforms/Inline/crash-lifetime-marker.ll
@@ -0,0 +1,24 @@
+; RUN: opt < %s -inline -S | FileCheck %s
+
+; InlineFunction would assert inside the loop that leaves lifetime markers if
+; there was an zero-sized AllocaInst. Check that it doesn't assert and doesn't
+; leave lifetime markers in that case.
+
+declare i32 @callee2(i8*)
+
+define i32 @callee1(i32 %count) {
+  %a0 = alloca i8, i32 %count, align 4
+  %call0 = call i32 @callee2(i8* %a0)
+  ret i32 %call0
+}
+
+; CHECK-LABEL: define i32 @caller1(
+; CHECK: [[ALLOCA:%[a-z0-9\.]+]] = alloca i8
+; CHECK-NOT: call void @llvm.lifetime.start(
+; CHECK: call i32 @callee2(i8* [[ALLOCA]])
+; CHECK-NOT: call void @llvm.lifetime.end(
+
+define i32 @caller1(i32 %count) {
+  %call0 = call i32 @callee1(i32 0)
+  ret i32 %call0
+}
diff --git a/test/Transforms/Inline/crash.ll b/test/Transforms/Inline/crash.ll
index e2cd49c2516..ec1c867bd05 100644
--- a/test/Transforms/Inline/crash.ll
+++ b/test/Transforms/Inline/crash.ll
@@ -59,7 +59,7 @@ declare fastcc void @list_Rplacd1284() nounwind ssp
 ;============================
 ; PR5208
 
-define void @AAA() {
+define void @AAA() personality i32 (...)* @__gxx_personality_v0 {
 entry:
   %A = alloca i8, i32 undef, align 1
   invoke fastcc void @XXX()
@@ -69,7 +69,7 @@ invcont98:
   unreachable
 
 lpad156:                            
-  %exn = landingpad {i8*, i32} personality i32 (...)* @__gxx_personality_v0
+  %exn = landingpad {i8*, i32}
             cleanup
   unreachable
 }
@@ -78,7 +78,7 @@ declare i32 @__gxx_personality_v0(...)
 
 declare fastcc void @YYY()
 
-define internal fastcc void @XXX() {
+define internal fastcc void @XXX() personality i32 (...)* @__gxx_personality_v0 {
 entry:
   %B = alloca i8, i32 undef, align 1
   invoke fastcc void @YYY()
@@ -88,7 +88,7 @@ bb260:
   ret void
 
 lpad:                               
-  %exn = landingpad {i8*, i32} personality i32 (...)* @__gxx_personality_v0
+  %exn = landingpad {i8*, i32}
             cleanup
   resume { i8*, i32 } %exn
 }
@@ -102,7 +102,7 @@ entry:
   ret void
 }
 
-define void @f4(i32 %size) ssp {
+define void @f4(i32 %size) ssp personality i32 (...)* @__gxx_personality_v0 {
 entry:
   invoke void @f1(void ()* @f3)
           to label %invcont3 unwind label %lpad18
@@ -111,7 +111,7 @@ invcont3:                                         ; preds = %bb1
   ret void
 
 lpad18:                                           ; preds = %invcont3, %bb1
-  %exn = landingpad {i8*, i32} personality i32 (...)* @__gxx_personality_v0
+  %exn = landingpad {i8*, i32}
             cleanup
   unreachable
 }
diff --git a/test/Transforms/Inline/debug-info-duplicate-calls.ll b/test/Transforms/Inline/debug-info-duplicate-calls.ll
index a67fac5850e..4e3d9d92fdb 100644
--- a/test/Transforms/Inline/debug-info-duplicate-calls.ll
+++ b/test/Transforms/Inline/debug-info-duplicate-calls.ll
@@ -33,10 +33,10 @@
 ; CHECK: call void @_Z2f1v(), !dbg [[fcs2_f4_f3cs1_f2:![0-9]+]]
 ; CHECK: call void @_Z2f1v(), !dbg [[fcs2_f4_f3cs2_f2:![0-9]+]]
 
-; CHECK-DAG: [[F:![0-9]+]]  = !MDSubprogram(name: "f"
-; CHECK-DAG: [[F2:![0-9]+]] = !MDSubprogram(name: "f2"
-; CHECK-DAG: [[F3:![0-9]+]] = !MDSubprogram(name: "f3"
-; CHECK-DAG: [[F4:![0-9]+]] = !MDSubprogram(name: "f4"
+; CHECK-DAG: [[F:![0-9]+]]  = !DISubprogram(name: "f"
+; CHECK-DAG: [[F2:![0-9]+]] = !DISubprogram(name: "f2"
+; CHECK-DAG: [[F3:![0-9]+]] = !DISubprogram(name: "f3"
+; CHECK-DAG: [[F4:![0-9]+]] = !DISubprogram(name: "f4"
 
 ; CHECK: [[fcs1_f4_f3cs1_f2]] = {{.*}}, scope: [[F2]], inlinedAt: [[fcs1_f4_f3cs1:![0-9]+]])
 ; CHECK: [[fcs1_f4_f3cs1]] = {{.*}}, scope: [[F3]], inlinedAt: [[fcs1_f4:![0-9]+]])
@@ -98,24 +98,24 @@ attributes #2 = { "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "n
 !llvm.module.flags = !{!10, !11}
 !llvm.ident = !{!12}
 
-!0 = !MDCompileUnit(language: DW_LANG_C_plus_plus, producer: "clang version 3.7.0 (trunk 226474) (llvm/trunk 226478)", isOptimized: false, emissionKind: 2, file: !1, enums: !2, retainedTypes: !2, subprograms: !3, globals: !2, imports: !2)
-!1 = !MDFile(filename: "debug-info-duplicate-calls.cpp", directory: "/tmp/dbginfo")
+!0 = !DICompileUnit(language: DW_LANG_C_plus_plus, producer: "clang version 3.7.0 (trunk 226474) (llvm/trunk 226478)", isOptimized: false, emissionKind: 2, file: !1, enums: !2, retainedTypes: !2, subprograms: !3, globals: !2, imports: !2)
+!1 = !DIFile(filename: "debug-info-duplicate-calls.cpp", directory: "/tmp/dbginfo")
 !2 = !{}
 !3 = !{!4, !7, !8, !9}
-!4 = !MDSubprogram(name: "f", line: 13, isLocal: false, isDefinition: true, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 13, file: !1, scope: !5, type: !6, function: void ()* @_Z1fv, variables: !2)
-!5 = !MDFile(filename: "debug-info-duplicate-calls.cpp", directory: "/tmp/dbginfo")
-!6 = !MDSubroutineType(types: !2)
-!7 = !MDSubprogram(name: "f4", line: 10, isLocal: false, isDefinition: true, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 10, file: !1, scope: !5, type: !6, function: void ()* @_Z2f4v, variables: !2)
-!8 = !MDSubprogram(name: "f3", line: 7, isLocal: false, isDefinition: true, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 7, file: !1, scope: !5, type: !6, function: void ()* @_Z2f3v, variables: !2)
-!9 = !MDSubprogram(name: "f2", line: 4, isLocal: false, isDefinition: true, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 4, file: !1, scope: !5, type: !6, function: void ()* @_Z2f2v, variables: !2)
+!4 = !DISubprogram(name: "f", line: 13, isLocal: false, isDefinition: true, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 13, file: !1, scope: !5, type: !6, function: void ()* @_Z1fv, variables: !2)
+!5 = !DIFile(filename: "debug-info-duplicate-calls.cpp", directory: "/tmp/dbginfo")
+!6 = !DISubroutineType(types: !2)
+!7 = !DISubprogram(name: "f4", line: 10, isLocal: false, isDefinition: true, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 10, file: !1, scope: !5, type: !6, function: void ()* @_Z2f4v, variables: !2)
+!8 = !DISubprogram(name: "f3", line: 7, isLocal: false, isDefinition: true, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 7, file: !1, scope: !5, type: !6, function: void ()* @_Z2f3v, variables: !2)
+!9 = !DISubprogram(name: "f2", line: 4, isLocal: false, isDefinition: true, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 4, file: !1, scope: !5, type: !6, function: void ()* @_Z2f2v, variables: !2)
 !10 = !{i32 2, !"Dwarf Version", i32 4}
 !11 = !{i32 2, !"Debug Info Version", i32 3}
 !12 = !{!"clang version 3.7.0 (trunk 226474) (llvm/trunk 226478)"}
-!13 = !MDLocation(line: 14, column: 3, scope: !4)
-!14 = !MDLocation(line: 15, column: 1, scope: !4)
-!15 = !MDLocation(line: 11, column: 3, scope: !7)
-!16 = !MDLocation(line: 12, column: 1, scope: !7)
-!17 = !MDLocation(line: 8, column: 3, scope: !8)
-!18 = !MDLocation(line: 9, column: 1, scope: !8)
-!19 = !MDLocation(line: 5, column: 3, scope: !9)
-!20 = !MDLocation(line: 6, column: 1, scope: !9)
+!13 = !DILocation(line: 14, column: 3, scope: !4)
+!14 = !DILocation(line: 15, column: 1, scope: !4)
+!15 = !DILocation(line: 11, column: 3, scope: !7)
+!16 = !DILocation(line: 12, column: 1, scope: !7)
+!17 = !DILocation(line: 8, column: 3, scope: !8)
+!18 = !DILocation(line: 9, column: 1, scope: !8)
+!19 = !DILocation(line: 5, column: 3, scope: !9)
+!20 = !DILocation(line: 6, column: 1, scope: !9)
diff --git a/test/Transforms/Inline/debug-invoke.ll b/test/Transforms/Inline/debug-invoke.ll
index c35300c7671..ca407acdd65 100644
--- a/test/Transforms/Inline/debug-invoke.ll
+++ b/test/Transforms/Inline/debug-invoke.ll
@@ -4,9 +4,9 @@
 
 ; CHECK: invoke void @test()
 ; CHECK-NEXT: to label {{.*}} unwind label {{.*}}, !dbg [[INL_LOC:!.*]]
-; CHECK: [[SP:.*]] = !MDSubprogram(
-; CHECK: [[INL_LOC]] = !MDLocation(line: 1, scope: [[SP]], inlinedAt: [[INL_AT:.*]])
-; CHECK: [[INL_AT]] = distinct !MDLocation(line: 2, scope: [[SP]])
+; CHECK: [[SP:.*]] = !DISubprogram(
+; CHECK: [[INL_LOC]] = !DILocation(line: 1, scope: [[SP]], inlinedAt: [[INL_AT:.*]])
+; CHECK: [[INL_AT]] = distinct !DILocation(line: 2, scope: [[SP]])
 
 declare void @test()
 declare i32 @__gxx_personality_v0(...)
@@ -17,7 +17,7 @@ define void @inl() #0 {
   ret void
 }
 
-define void @caller() {
+define void @caller() personality i8* bitcast (i32 (...)* @__gxx_personality_v0 to i8*) {
   invoke void @inl()
     to label %cont unwind label %lpad, !dbg !4
 
@@ -25,13 +25,13 @@ cont:
   ret void
 
 lpad:
-  landingpad { i8*, i32 } personality i8* bitcast (i32 (...)* @__gxx_personality_v0 to i8*)
+  landingpad { i8*, i32 }
     cleanup
   ret void
 }
 
 !llvm.module.flags = !{!1}
 !1 = !{i32 2, !"Debug Info Version", i32 3}
-!2 = !MDSubprogram()
-!3 = !MDLocation(line: 1, scope: !2)
-!4 = !MDLocation(line: 2, scope: !2)
+!2 = !DISubprogram()
+!3 = !DILocation(line: 1, scope: !2)
+!4 = !DILocation(line: 2, scope: !2)
diff --git a/test/Transforms/Inline/frameescape.ll b/test/Transforms/Inline/frameescape.ll
index fb336024f93..6615fe9a76e 100644
--- a/test/Transforms/Inline/frameescape.ll
+++ b/test/Transforms/Inline/frameescape.ll
@@ -1,13 +1,13 @@
 ; RUN: opt -inline -S < %s | FileCheck %s
 
-; PR23216: We can't inline functions using llvm.frameescape.
+; PR23216: We can't inline functions using llvm.localescape.
 
-declare void @llvm.frameescape(...)
+declare void @llvm.localescape(...)
 declare i8* @llvm.frameaddress(i32)
-declare i8* @llvm.framerecover(i8*, i8*, i32)
+declare i8* @llvm.localrecover(i8*, i8*, i32)
 
 define internal void @foo(i8* %fp) {
-  %a.i8 = call i8* @llvm.framerecover(i8* bitcast (i32 ()* @bar to i8*), i8* %fp, i32 0)
+  %a.i8 = call i8* @llvm.localrecover(i8* bitcast (i32 ()* @bar to i8*), i8* %fp, i32 0)
   %a = bitcast i8* %a.i8 to i32*
   store i32 42, i32* %a
   ret void
@@ -16,7 +16,7 @@ define internal void @foo(i8* %fp) {
 define internal i32 @bar() {
 entry:
   %a = alloca i32
-  call void (...) @llvm.frameescape(i32* %a)
+  call void (...) @llvm.localescape(i32* %a)
   %fp = call i8* @llvm.frameaddress(i32 0)
   tail call void @foo(i8* %fp)
   %r = load i32, i32* %a
@@ -27,7 +27,7 @@ entry:
 define internal i32 @bar_alwaysinline() alwaysinline {
 entry:
   %a = alloca i32
-  call void (...) @llvm.frameescape(i32* %a)
+  call void (...) @llvm.localescape(i32* %a)
   tail call void @foo(i8* null)
   ret i32 0
 }
diff --git a/test/Transforms/Inline/ignore-debug-info.ll b/test/Transforms/Inline/ignore-debug-info.ll
index 4cbd2fa481a..eb92bc52cc9 100644
--- a/test/Transforms/Inline/ignore-debug-info.ll
+++ b/test/Transforms/Inline/ignore-debug-info.ll
@@ -12,11 +12,11 @@ declare void @llvm.dbg.value(metadata, i64, metadata, metadata) #1
 
 define <4 x float> @inner_vectors(<4 x float> %a, <4 x float> %b) {
 entry:
-  call void @llvm.dbg.value(metadata i32 undef, i64 0, metadata !MDLocalVariable(tag: DW_TAG_auto_variable, scope: !6), metadata !MDExpression()), !dbg !MDLocation(scope: !6)
+  call void @llvm.dbg.value(metadata i32 undef, i64 0, metadata !DILocalVariable(tag: DW_TAG_auto_variable, scope: !6), metadata !DIExpression()), !dbg !DILocation(scope: !6)
   %mul = fmul <4 x float> %a, <float 3.000000e+00, float 3.000000e+00, float 3.000000e+00, float 3.000000e+00>
-  call void @llvm.dbg.value(metadata i32 undef, i64 0, metadata !MDLocalVariable(tag: DW_TAG_auto_variable, scope: !6), metadata !MDExpression()), !dbg !MDLocation(scope: !6)
+  call void @llvm.dbg.value(metadata i32 undef, i64 0, metadata !DILocalVariable(tag: DW_TAG_auto_variable, scope: !6), metadata !DIExpression()), !dbg !DILocation(scope: !6)
   %mul1 = fmul <4 x float> %b, <float 5.000000e+00, float 5.000000e+00, float 5.000000e+00, float 5.000000e+00>
-  call void @llvm.dbg.value(metadata i32 undef, i64 0, metadata !MDLocalVariable(tag: DW_TAG_auto_variable, scope: !6), metadata !MDExpression()), !dbg !MDLocation(scope: !6)
+  call void @llvm.dbg.value(metadata i32 undef, i64 0, metadata !DILocalVariable(tag: DW_TAG_auto_variable, scope: !6), metadata !DIExpression()), !dbg !DILocation(scope: !6)
   %add = fadd <4 x float> %mul, %mul1
   ret <4 x float> %add
 }
@@ -27,10 +27,10 @@ define float @outer_vectors(<4 x float> %a, <4 x float> %b) {
 ; CHECK: ret float
 
 entry:
-  call void @llvm.dbg.value(metadata i32 undef, i64 0, metadata !MDLocalVariable(tag: DW_TAG_auto_variable, scope: !6), metadata !MDExpression()), !dbg !MDLocation(scope: !6)
-  call void @llvm.dbg.value(metadata i32 undef, i64 0, metadata !MDLocalVariable(tag: DW_TAG_auto_variable, scope: !6), metadata !MDExpression()), !dbg !MDLocation(scope: !6)
+  call void @llvm.dbg.value(metadata i32 undef, i64 0, metadata !DILocalVariable(tag: DW_TAG_auto_variable, scope: !6), metadata !DIExpression()), !dbg !DILocation(scope: !6)
+  call void @llvm.dbg.value(metadata i32 undef, i64 0, metadata !DILocalVariable(tag: DW_TAG_auto_variable, scope: !6), metadata !DIExpression()), !dbg !DILocation(scope: !6)
   %call = call <4 x float> @inner_vectors(<4 x float> %a, <4 x float> %b)
-  call void @llvm.dbg.value(metadata i32 undef, i64 0, metadata !MDLocalVariable(tag: DW_TAG_auto_variable, scope: !6), metadata !MDExpression()), !dbg !MDLocation(scope: !6)
+  call void @llvm.dbg.value(metadata i32 undef, i64 0, metadata !DILocalVariable(tag: DW_TAG_auto_variable, scope: !6), metadata !DIExpression()), !dbg !DILocation(scope: !6)
   %vecext = extractelement <4 x float> %call, i32 0
   %vecext1 = extractelement <4 x float> %call, i32 1
   %add = fadd float %vecext, %vecext1
@@ -47,10 +47,10 @@ attributes #0 = { nounwind readnone }
 !llvm.module.flags = !{!3, !4}
 !llvm.ident = !{!5}
 
-!0 = !MDCompileUnit(language: DW_LANG_C_plus_plus, isOptimized: false, emissionKind: 0, file: !1, enums: !2, retainedTypes: !2, subprograms: !{!6}, globals: !2, imports: !2)
-!1 = !MDFile(filename: "test.c", directory: "")
+!0 = !DICompileUnit(language: DW_LANG_C_plus_plus, isOptimized: false, emissionKind: 0, file: !1, enums: !2, retainedTypes: !2, subprograms: !{!6}, globals: !2, imports: !2)
+!1 = !DIFile(filename: "test.c", directory: "")
 !2 = !{}
 !3 = !{i32 2, !"Dwarf Version", i32 4}
 !4 = !{i32 1, !"Debug Info Version", i32 3}
 !5 = !{!""}
-!6 = !MDSubprogram()
+!6 = !DISubprogram()
diff --git a/test/Transforms/Inline/inline-constexpr-addrspacecast-argument.ll b/test/Transforms/Inline/inline-constexpr-addrspacecast-argument.ll
new file mode 100644
index 00000000000..498a995ecd4
--- /dev/null
+++ b/test/Transforms/Inline/inline-constexpr-addrspacecast-argument.ll
@@ -0,0 +1,30 @@
+; RUN: opt -S -inline < %s | FileCheck %s
+
+target datalayout = "e-p3:32:32-p4:64:64-n32"
+
+@lds = internal addrspace(3) global [64 x i64] zeroinitializer
+
+; CHECK-LABEL: @constexpr_addrspacecast_ptr_size_change(
+; CHECK: load i64, i64 addrspace(4)* getelementptr (i64, i64 addrspace(4)* addrspacecast (i64 addrspace(3)* getelementptr inbounds ([64 x i64], [64 x i64] addrspace(3)* @lds, i32 0, i32 0) to i64 addrspace(4)*), i64 undef)
+; CHECK-NEXT: br
+define void @constexpr_addrspacecast_ptr_size_change() #0 {
+  %tmp0 = call i32 @foo(i64 addrspace(4)* addrspacecast (i64 addrspace(3)* getelementptr inbounds ([64 x i64], [64 x i64] addrspace(3)* @lds, i32 0, i32 0) to i64 addrspace(4)*)) #1
+  ret void
+}
+
+define i32 @foo(i64 addrspace(4)* %arg) #1 {
+bb:
+  %tmp = getelementptr i64, i64 addrspace(4)* %arg, i64 undef
+  %tmp1 = load i64, i64 addrspace(4)* %tmp
+  br i1 undef, label %bb2, label %bb3
+
+bb2:
+  store i64 0, i64 addrspace(4)* %tmp
+  br label %bb3
+
+bb3:
+  unreachable
+}
+
+attributes #0 = { nounwind }
+attributes #1 = { alwaysinline nounwind }
diff --git a/test/Transforms/Inline/inline-invoke-tail.ll b/test/Transforms/Inline/inline-invoke-tail.ll
index 5fef4baee9e..f4b80653d01 100644
--- a/test/Transforms/Inline/inline-invoke-tail.ll
+++ b/test/Transforms/Inline/inline-invoke-tail.ll
@@ -10,7 +10,7 @@ define internal void @foo(i32* %p, i32* %q) {
 
 declare void @llvm.memcpy.i32(i8* nocapture, i8* nocapture, i32, i32) nounwind
 
-define i32 @main() {
+define i32 @main() personality i32 (...)* @__gxx_personality_v0 {
 	%a = alloca i32		; <i32*> [#uses=3]
 	%b = alloca i32		; <i32*> [#uses=2]
 	store i32 1, i32* %a, align 4
@@ -23,7 +23,7 @@ invcont:
 	ret i32 %retval
 
 lpad:
-        %exn = landingpad {i8*, i32} personality i32 (...)* @__gxx_personality_v0
+        %exn = landingpad {i8*, i32}
             catch i8* null
 	unreachable
 }
diff --git a/test/Transforms/Inline/inline-invoke-with-asm-call.ll b/test/Transforms/Inline/inline-invoke-with-asm-call.ll
index 876f8d7455b..93bb0358055 100644
--- a/test/Transforms/Inline/inline-invoke-with-asm-call.ll
+++ b/test/Transforms/Inline/inline-invoke-with-asm-call.ll
@@ -8,7 +8,7 @@ target triple = "x86_64-apple-darwin"
 ; Make sure we are generating "call asm" instead of "invoke asm".
 ; CHECK: call void asm
 ; CHECK-LABEL: @callee_with_asm
-define void @caller() {
+define void @caller() personality i8* bitcast (i32 (...)* @__objc_personality_v0 to i8*) {
   br i1 undef, label %1, label %4
 
 ; <label>:1
@@ -16,7 +16,7 @@ define void @caller() {
           to label %4 unwind label %2
 
 ; <label>:2
-  %3 = landingpad { i8*, i32 } personality i8* bitcast (i32 (...)* @__objc_personality_v0 to i8*)
+  %3 = landingpad { i8*, i32 }
           cleanup
   resume { i8*, i32 } undef
 
diff --git a/test/Transforms/Inline/inline_dbg_declare.ll b/test/Transforms/Inline/inline_dbg_declare.ll
index e34a43b3492..f3ad7ef8b8f 100644
--- a/test/Transforms/Inline/inline_dbg_declare.ll
+++ b/test/Transforms/Inline/inline_dbg_declare.ll
@@ -67,33 +67,33 @@ attributes #1 = { nounwind readnone }
 !llvm.module.flags = !{!13, !14}
 !llvm.ident = !{!15}
 
-!0 = !MDCompileUnit(language: DW_LANG_C99, producer: "clang version 3.6.0 (trunk)", isOptimized: false, emissionKind: 1, file: !1, enums: !2, retainedTypes: !2, subprograms: !3, globals: !2, imports: !2)
-!1 = !MDFile(filename: "foo.c", directory: "")
+!0 = !DICompileUnit(language: DW_LANG_C99, producer: "clang version 3.6.0 (trunk)", isOptimized: false, emissionKind: 1, file: !1, enums: !2, retainedTypes: !2, subprograms: !3, globals: !2, imports: !2)
+!1 = !DIFile(filename: "foo.c", directory: "")
 !2 = !{}
 !3 = !{!4, !9}
-!4 = !MDSubprogram(name: "foo", line: 1, isLocal: false, isDefinition: true, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 2, file: !1, scope: !5, type: !6, function: float (float)* @foo, variables: !2)
-!5 = !MDFile(filename: "foo.c", directory: "")
-!6 = !MDSubroutineType(types: !7)
+!4 = !DISubprogram(name: "foo", line: 1, isLocal: false, isDefinition: true, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 2, file: !1, scope: !5, type: !6, function: float (float)* @foo, variables: !2)
+!5 = !DIFile(filename: "foo.c", directory: "")
+!6 = !DISubroutineType(types: !7)
 !7 = !{!8, !8}
-!8 = !MDBasicType(tag: DW_TAG_base_type, name: "float", size: 32, align: 32, encoding: DW_ATE_float)
-!9 = !MDSubprogram(name: "bar", line: 6, isLocal: false, isDefinition: true, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 7, file: !1, scope: !5, type: !10, function: void (float*)* @bar, variables: !2)
-!10 = !MDSubroutineType(types: !11)
+!8 = !DIBasicType(tag: DW_TAG_base_type, name: "float", size: 32, align: 32, encoding: DW_ATE_float)
+!9 = !DISubprogram(name: "bar", line: 6, isLocal: false, isDefinition: true, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 7, file: !1, scope: !5, type: !10, function: void (float*)* @bar, variables: !2)
+!10 = !DISubroutineType(types: !11)
 !11 = !{null, !12}
-!12 = !MDDerivedType(tag: DW_TAG_pointer_type, size: 32, align: 32, baseType: !8)
+!12 = !DIDerivedType(tag: DW_TAG_pointer_type, size: 32, align: 32, baseType: !8)
 !13 = !{i32 2, !"Dwarf Version", i32 4}
 !14 = !{i32 2, !"Debug Info Version", i32 3}
 !15 = !{!"clang version 3.6.0 (trunk)"}
-!16 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "x", line: 1, arg: 1, scope: !4, file: !5, type: !8)
-!17 = !MDExpression()
-!18 = !MDLocation(line: 1, column: 17, scope: !4)
-!19 = !MDLocation(line: 3, column: 5, scope: !4)
-!20 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "dst", line: 6, arg: 1, scope: !9, file: !5, type: !12)
-!21 = !MDLocation(line: 6, column: 17, scope: !9)
-!22 = !MDLocation(line: 8, column: 14, scope: !9)
-!23 = !MDLocation(line: 9, column: 1, scope: !9)
+!16 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "x", line: 1, arg: 1, scope: !4, file: !5, type: !8)
+!17 = !DIExpression()
+!18 = !DILocation(line: 1, column: 17, scope: !4)
+!19 = !DILocation(line: 3, column: 5, scope: !4)
+!20 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "dst", line: 6, arg: 1, scope: !9, file: !5, type: !12)
+!21 = !DILocation(line: 6, column: 17, scope: !9)
+!22 = !DILocation(line: 8, column: 14, scope: !9)
+!23 = !DILocation(line: 9, column: 1, scope: !9)
 
-; CHECK: [[FOO:![0-9]+]] = !MDSubprogram(name: "foo",
-; CHECK: [[BAR:![0-9]+]] = !MDSubprogram(name: "bar",
-; CHECK: [[m23]] = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "x", arg: 1, scope: [[FOO]]
-; CHECK: [[CALL_SITE:![0-9]+]] = distinct !MDLocation(line: 8, column: 14, scope: [[BAR]])
-; CHECK: [[m24]] = !MDLocation(line: 1, column: 17, scope: [[FOO]], inlinedAt: [[CALL_SITE]])
+; CHECK: [[FOO:![0-9]+]] = !DISubprogram(name: "foo",
+; CHECK: [[BAR:![0-9]+]] = !DISubprogram(name: "bar",
+; CHECK: [[m23]] = !DILocalVariable(tag: DW_TAG_arg_variable, name: "x", arg: 1, scope: [[FOO]]
+; CHECK: [[CALL_SITE:![0-9]+]] = distinct !DILocation(line: 8, column: 14, scope: [[BAR]])
+; CHECK: [[m24]] = !DILocation(line: 1, column: 17, scope: [[FOO]], inlinedAt: [[CALL_SITE]])
diff --git a/test/Transforms/Inline/inline_invoke.ll b/test/Transforms/Inline/inline_invoke.ll
index c53bb5aa17b..2ef216e2d38 100644
--- a/test/Transforms/Inline/inline_invoke.ll
+++ b/test/Transforms/Inline/inline_invoke.ll
@@ -28,7 +28,7 @@ declare void @__cxa_end_catch()
 
 declare void @_ZSt9terminatev()
 
-define internal void @test0_in() alwaysinline uwtable ssp {
+define internal void @test0_in() alwaysinline uwtable ssp personality i32 (...)* @__gxx_personality_v0 {
 entry:
   %a = alloca %struct.A, align 1
   %b = alloca %struct.A, align 1
@@ -45,7 +45,7 @@ invoke.cont1:
   ret void
 
 lpad:
-  %exn = landingpad {i8*, i32} personality i32 (...)* @__gxx_personality_v0
+  %exn = landingpad {i8*, i32}
             cleanup
   invoke void @_ZN1AD1Ev(%struct.A* %a)
           to label %invoke.cont2 unwind label %terminate.lpad
@@ -54,13 +54,13 @@ invoke.cont2:
   resume { i8*, i32 } %exn
 
 terminate.lpad:
-  %exn1 = landingpad {i8*, i32} personality i32 (...)* @__gxx_personality_v0
+  %exn1 = landingpad {i8*, i32}
             catch i8* null
   call void @_ZSt9terminatev() noreturn nounwind
   unreachable
 }
 
-define void @test0_out() uwtable ssp {
+define void @test0_out() uwtable ssp personality i32 (...)* @__gxx_personality_v0 {
 entry:
   invoke void @test0_in()
           to label %ret unwind label %lpad
@@ -69,7 +69,7 @@ ret:
   ret void
 
 lpad:                                             ; preds = %entry
-  %exn = landingpad {i8*, i32} personality i32 (...)* @__gxx_personality_v0
+  %exn = landingpad {i8*, i32}
             catch i8* bitcast (i8** @_ZTIi to i8*)
   %eh.exc = extractvalue { i8*, i32 } %exn, 0
   %eh.selector = extractvalue { i8*, i32 } %exn, 1
@@ -93,7 +93,7 @@ eh.resume:
 ; CHECK:      invoke void @_ZN1AC1Ev(%struct.A* [[B]])
 ; CHECK:      invoke void @_ZN1AD1Ev(%struct.A* [[B]])
 ; CHECK:      invoke void @_ZN1AD1Ev(%struct.A* [[A]])
-; CHECK:      landingpad { i8*, i32 } personality i32 (...)* @__gxx_personality_v0
+; CHECK:      landingpad { i8*, i32 }
 ; CHECK-NEXT:    cleanup
 ; CHECK-NEXT:    catch i8* bitcast (i8** @_ZTIi to i8*)
 ; CHECK-NEXT: invoke void @_ZN1AD1Ev(%struct.A* [[A]])
@@ -101,7 +101,7 @@ eh.resume:
 ; CHECK: [[LBL]]:
 ; CHECK-NEXT: br label %[[LPAD:[^\s]+]]
 ; CHECK:      ret void
-; CHECK:      landingpad { i8*, i32 } personality i32 (...)* @__gxx_personality_v0
+; CHECK:      landingpad { i8*, i32 }
 ; CHECK-NEXT:    catch i8* bitcast (i8** @_ZTIi to i8*)
 ; CHECK-NEXT: br label %[[LPAD]]
 ; CHECK: [[LPAD]]:
@@ -113,7 +113,7 @@ eh.resume:
 
 ;; Test 1 - Correctly handle phis in outer landing pads.
 
-define void @test1_out() uwtable ssp {
+define void @test1_out() uwtable ssp personality i32 (...)* @__gxx_personality_v0 {
 entry:
   invoke void @test0_in()
           to label %cont unwind label %lpad
@@ -128,7 +128,7 @@ ret:
 lpad:
   %x = phi i32 [ 0, %entry ], [ 1, %cont ]
   %y = phi i32 [ 1, %entry ], [ 4, %cont ]
-  %exn = landingpad {i8*, i32} personality i32 (...)* @__gxx_personality_v0
+  %exn = landingpad {i8*, i32}
             catch i8* bitcast (i8** @_ZTIi to i8*)
   %eh.exc = extractvalue { i8*, i32 } %exn, 0
   %eh.selector = extractvalue { i8*, i32 } %exn, 1
@@ -163,7 +163,7 @@ eh.resume:
 
 ; Inner landing pad from first inlining.
 ; CHECK:    [[LPAD1]]:
-; CHECK-NEXT: [[LPADVAL1:%.*]] = landingpad { i8*, i32 } personality i32 (...)* @__gxx_personality_v0
+; CHECK-NEXT: [[LPADVAL1:%.*]] = landingpad { i8*, i32 }
 ; CHECK-NEXT:    cleanup
 ; CHECK-NEXT:    catch i8* bitcast (i8** @_ZTIi to i8*)
 ; CHECK-NEXT: invoke void @_ZN1AD1Ev(%struct.A* [[A1]])
@@ -182,7 +182,7 @@ eh.resume:
 
 ; Inner landing pad from second inlining.
 ; CHECK:    [[LPAD2]]:
-; CHECK-NEXT: [[LPADVAL2:%.*]] = landingpad { i8*, i32 } personality i32 (...)* @__gxx_personality_v0
+; CHECK-NEXT: [[LPADVAL2:%.*]] = landingpad { i8*, i32 }
 ; CHECK-NEXT:   cleanup
 ; CHECK-NEXT:   catch i8* bitcast (i8** @_ZTIi to i8*)
 ; CHECK-NEXT: invoke void @_ZN1AD1Ev(%struct.A* [[A2]])
@@ -195,7 +195,7 @@ eh.resume:
 ; CHECK:    [[LPAD]]:
 ; CHECK-NEXT: [[X:%.*]] = phi i32 [ 0, %entry ], [ 0, {{%.*}} ], [ 1, %cont ], [ 1, {{%.*}} ]
 ; CHECK-NEXT: [[Y:%.*]] = phi i32 [ 1, %entry ], [ 1, {{%.*}} ], [ 4, %cont ], [ 4, {{%.*}} ]
-; CHECK-NEXT: [[LPADVAL:%.*]] = landingpad { i8*, i32 } personality i32 (...)* @__gxx_personality_v0
+; CHECK-NEXT: [[LPADVAL:%.*]] = landingpad { i8*, i32 }
 ; CHECK-NEXT:   catch i8* bitcast (i8** @_ZTIi to i8*)
 ; CHECK-NEXT: br label %[[LPAD_JOIN2]]
 
@@ -221,7 +221,7 @@ eh.resume:
 
 
 ;; Test 2 - Don't make invalid IR for inlines into landing pads without eh.exception calls
-define void @test2_out() uwtable ssp {
+define void @test2_out() uwtable ssp personality i32 (...)* @__gxx_personality_v0 {
 entry:
   invoke void @test0_in()
           to label %ret unwind label %lpad
@@ -230,7 +230,7 @@ ret:
   ret void
 
 lpad:
-  %exn = landingpad {i8*, i32} personality i32 (...)* @__gxx_personality_v0
+  %exn = landingpad {i8*, i32}
             cleanup
   call void @_ZSt9terminatev()
   unreachable
@@ -250,7 +250,7 @@ lpad:
 
 
 ;; Test 3 - Deal correctly with split unwind edges.
-define void @test3_out() uwtable ssp {
+define void @test3_out() uwtable ssp personality i32 (...)* @__gxx_personality_v0 {
 entry:
   invoke void @test0_in()
           to label %ret unwind label %lpad
@@ -259,7 +259,7 @@ ret:
   ret void
 
 lpad:
-  %exn = landingpad {i8*, i32} personality i32 (...)* @__gxx_personality_v0
+  %exn = landingpad {i8*, i32}
             catch i8* bitcast (i8** @_ZTIi to i8*)
   br label %lpad.cont
 
@@ -269,7 +269,7 @@ lpad.cont:
 }
 
 ; CHECK: define void @test3_out()
-; CHECK:      landingpad { i8*, i32 } personality i32 (...)* @__gxx_personality_v0
+; CHECK:      landingpad { i8*, i32 }
 ; CHECK-NEXT:    cleanup
 ; CHECK-NEXT:    catch i8* bitcast (i8** @_ZTIi to i8*)
 ; CHECK-NEXT: invoke void @_ZN1AD1Ev(
@@ -284,7 +284,7 @@ lpad.cont:
 
 
 ;; Test 4 - Split unwind edges with a dominance problem
-define void @test4_out() uwtable ssp {
+define void @test4_out() uwtable ssp personality i32 (...)* @__gxx_personality_v0 {
 entry:
   invoke void @test0_in()
           to label %cont unwind label %lpad.crit
@@ -297,13 +297,13 @@ ret:
   ret void
 
 lpad.crit:
-  %exn = landingpad {i8*, i32} personality i32 (...)* @__gxx_personality_v0
+  %exn = landingpad {i8*, i32}
             catch i8* bitcast (i8** @_ZTIi to i8*)
   call void @opaque() nounwind
   br label %terminate
 
 lpad:
-  %exn2 = landingpad {i8*, i32} personality i32 (...)* @__gxx_personality_v0
+  %exn2 = landingpad {i8*, i32}
             catch i8* bitcast (i8** @_ZTIi to i8*)
   br label %terminate
 
@@ -315,7 +315,7 @@ terminate:
 }
 
 ; CHECK: define void @test4_out()
-; CHECK:      landingpad { i8*, i32 } personality i32 (...)* @__gxx_personality_v0
+; CHECK:      landingpad { i8*, i32 }
 ; CHECK-NEXT:    cleanup
 ; CHECK-NEXT:    catch i8* bitcast (i8** @_ZTIi to i8*)
 ; CHECK-NEXT: invoke void @_ZN1AD1Ev(
@@ -325,7 +325,7 @@ terminate:
 ; CHECK:      invoke void @opaque()
 ; CHECK-NEXT:                  unwind label %lpad
 ; CHECK:    lpad.crit:
-; CHECK-NEXT: landingpad { i8*, i32 } personality i32 (...)* @__gxx_personality_v0
+; CHECK-NEXT: landingpad { i8*, i32 }
 ; CHECK-NEXT:   catch i8* bitcast (i8** @_ZTIi to i8*)
 ; CHECK-NEXT: br label %[[JOIN]]
 ; CHECK:    [[JOIN]]:
@@ -333,7 +333,7 @@ terminate:
 ; CHECK-NEXT: call void @opaque() [[NUW:#[0-9]+]]
 ; CHECK-NEXT: br label %[[FIX:[^\s]+]]
 ; CHECK:    lpad:
-; CHECK-NEXT: landingpad { i8*, i32 } personality i32 (...)* @__gxx_personality_v0
+; CHECK-NEXT: landingpad { i8*, i32 }
 ; CHECK-NEXT:   catch i8* bitcast (i8** @_ZTIi to i8*)
 ; CHECK-NEXT: br label %[[FIX]]
 ; CHECK:    [[FIX]]:
diff --git a/test/Transforms/Inline/inline_returns_twice.ll b/test/Transforms/Inline/inline_returns_twice.ll
index 36042640cc0..8a131bca749 100644
--- a/test/Transforms/Inline/inline_returns_twice.ll
+++ b/test/Transforms/Inline/inline_returns_twice.ll
@@ -37,7 +37,7 @@ entry:
   ret i32 %add
 }
 
-define i32 @inner3() {
+define i32 @inner3() personality i8* null {
 entry:
   %invoke = invoke i32 @a() returns_twice
       to label %cont unwind label %lpad
@@ -47,7 +47,7 @@ cont:
   ret i32 %add
 
 lpad:
-  %lp = landingpad i32 personality i8* null cleanup
+  %lp = landingpad i32 cleanup
   resume i32 %lp
 }
 
@@ -60,7 +60,7 @@ entry:
   ret i32 %add
 }
 
-define i32 @inner4() returns_twice {
+define i32 @inner4() returns_twice personality i8* null {
 entry:
   %invoke = invoke i32 @a() returns_twice
       to label %cont unwind label %lpad
@@ -70,7 +70,7 @@ cont:
   ret i32 %add
 
 lpad:
-  %lp = landingpad i32 personality i8* null cleanup
+  %lp = landingpad i32 cleanup
   resume i32 %lp
 }
 
diff --git a/test/Transforms/Inline/invoke-cleanup.ll b/test/Transforms/Inline/invoke-cleanup.ll
index 457ae2addeb..2750b79fbe1 100644
--- a/test/Transforms/Inline/invoke-cleanup.ll
+++ b/test/Transforms/Inline/invoke-cleanup.ll
@@ -6,13 +6,13 @@ declare void @external_func()
 @exception_type2 = external global i8
 
 
-define internal void @inner() {
+define internal void @inner() personality i8* null {
   invoke void @external_func()
       to label %cont unwind label %lpad
 cont:
   ret void
 lpad:
-  %lp = landingpad i32 personality i8* null
+  %lp = landingpad i32
       catch i8* @exception_type1
   resume i32 %lp
 }
@@ -21,13 +21,13 @@ lpad:
 ; this call site (PR17872), otherwise C++ destructors will not be
 ; called when they should be.
 
-define void @outer() {
+define void @outer() personality i8* null {
   invoke void @inner()
       to label %cont unwind label %lpad
 cont:
   ret void
 lpad:
-  %lp = landingpad i32 personality i8* null
+  %lp = landingpad i32
       cleanup
       catch i8* @exception_type2
   resume i32 %lp
diff --git a/test/Transforms/Inline/invoke-combine-clauses.ll b/test/Transforms/Inline/invoke-combine-clauses.ll
index 89a4cc951c7..e3788d6d643 100644
--- a/test/Transforms/Inline/invoke-combine-clauses.ll
+++ b/test/Transforms/Inline/invoke-combine-clauses.ll
@@ -12,13 +12,13 @@ declare void @abort()
 ; inlined function caused "catch i8* @exception_outer" to appear
 ; multiple times in the resulting landingpad.
 
-define internal void @inner_multiple_resume() {
+define internal void @inner_multiple_resume() personality i8* null {
   invoke void @external_func()
       to label %cont unwind label %lpad
 cont:
   ret void
 lpad:
-  %lp = landingpad i32 personality i8* null
+  %lp = landingpad i32
       catch i8* @exception_inner
   %cond = load i1, i1* @condition
   br i1 %cond, label %resume1, label %resume2
@@ -28,13 +28,13 @@ resume2:
   resume i32 2
 }
 
-define void @outer_multiple_resume() {
+define void @outer_multiple_resume() personality i8* null {
   invoke void @inner_multiple_resume()
       to label %cont unwind label %lpad
 cont:
   ret void
 lpad:
-  %lp = landingpad i32 personality i8* null
+  %lp = landingpad i32
       catch i8* @exception_outer
   resume i32 %lp
 }
@@ -50,25 +50,25 @@ lpad:
 ; inlined function caused "catch i8* @exception_outer" to appear
 ; multiple times in the resulting landingpad.
 
-define internal void @inner_resume_and_call() {
+define internal void @inner_resume_and_call() personality i8* null {
   call void @external_func()
   invoke void @external_func()
       to label %cont unwind label %lpad
 cont:
   ret void
 lpad:
-  %lp = landingpad i32 personality i8* null
+  %lp = landingpad i32
       catch i8* @exception_inner
   resume i32 %lp
 }
 
-define void @outer_resume_and_call() {
+define void @outer_resume_and_call() personality i8* null {
   invoke void @inner_resume_and_call()
       to label %cont unwind label %lpad
 cont:
   ret void
 lpad:
-  %lp = landingpad i32 personality i8* null
+  %lp = landingpad i32
       catch i8* @exception_outer
   resume i32 %lp
 }
@@ -86,26 +86,26 @@ lpad:
 ; function (since the outer function's landingpad will not be
 ; reachable), but it's OK to include this clause.
 
-define internal void @inner_no_resume_or_call() {
+define internal void @inner_no_resume_or_call() personality i8* null {
   invoke void @external_func()
       to label %cont unwind label %lpad
 cont:
   ret void
 lpad:
-  %lp = landingpad i32 personality i8* null
+  %lp = landingpad i32
       catch i8* @exception_inner
   ; A landingpad might have no "resume" if a C++ destructor aborts.
   call void @abort() noreturn nounwind
   unreachable
 }
 
-define void @outer_no_resume_or_call() {
+define void @outer_no_resume_or_call() personality i8* null {
   invoke void @inner_no_resume_or_call()
       to label %cont unwind label %lpad
 cont:
   ret void
 lpad:
-  %lp = landingpad i32 personality i8* null
+  %lp = landingpad i32
       catch i8* @exception_outer
   resume i32 %lp
 }
diff --git a/test/Transforms/Inline/invoke-cost.ll b/test/Transforms/Inline/invoke-cost.ll
index 84d33ad5512..24f2893e90f 100644
--- a/test/Transforms/Inline/invoke-cost.ll
+++ b/test/Transforms/Inline/invoke-cost.ll
@@ -10,7 +10,7 @@ declare i8* @__cxa_begin_catch(i8*)
 declare void @__cxa_end_catch()
 declare void @_ZSt9terminatev()
 
-define void @inner1() {
+define void @inner1() personality i32 (...)* @__gxx_personality_v0 {
 entry:
   invoke void @f() to label %cont1 unwind label %terminate.lpad
 
@@ -27,7 +27,7 @@ cont4:
   ret void
 
 terminate.lpad:
-  landingpad {i8*, i32} personality i32 (...)* @__gxx_personality_v0
+  landingpad {i8*, i32}
             catch i8* null
   call void @_ZSt9terminatev() noreturn nounwind
   unreachable
diff --git a/test/Transforms/Inline/invoke_test-1.ll b/test/Transforms/Inline/invoke_test-1.ll
index 922351fd461..8cb6362f349 100644
--- a/test/Transforms/Inline/invoke_test-1.ll
+++ b/test/Transforms/Inline/invoke_test-1.ll
@@ -12,7 +12,7 @@ define internal void @callee() {
 }
 
 ; caller returns true if might_throw throws an exception...
-define i32 @caller() {
+define i32 @caller() personality i32 (...)* @__gxx_personality_v0 {
         invoke void @callee( )
                         to label %cont unwind label %exc
 
@@ -20,7 +20,7 @@ cont:           ; preds = %0
         ret i32 0
 
 exc:            ; preds = %0
-        %exn = landingpad {i8*, i32} personality i32 (...)* @__gxx_personality_v0
+        %exn = landingpad {i8*, i32}
                  cleanup
         ret i32 1
 }
diff --git a/test/Transforms/Inline/invoke_test-2.ll b/test/Transforms/Inline/invoke_test-2.ll
index 680a5ca2542..b3119b99337 100644
--- a/test/Transforms/Inline/invoke_test-2.ll
+++ b/test/Transforms/Inline/invoke_test-2.ll
@@ -6,7 +6,7 @@
 
 declare void @might_throw()
 
-define internal i32 @callee() {
+define internal i32 @callee() personality i32 (...)* @__gxx_personality_v0 {
         invoke void @might_throw( )
                         to label %cont unwind label %exc
 
@@ -14,13 +14,13 @@ cont:           ; preds = %0
         ret i32 0
 
 exc:            ; preds = %0
-        %exn = landingpad {i8*, i32} personality i32 (...)* @__gxx_personality_v0
+        %exn = landingpad {i8*, i32}
                  cleanup
         ret i32 1
 }
 
 ; caller returns true if might_throw throws an exception... callee cannot throw.
-define i32 @caller() {
+define i32 @caller() personality i32 (...)* @__gxx_personality_v0 {
         %X = invoke i32 @callee( )
                         to label %cont unwind label %UnreachableExceptionHandler                ; <i32> [#uses=1]
 
@@ -28,7 +28,7 @@ cont:           ; preds = %0
         ret i32 %X
 
 UnreachableExceptionHandler:            ; preds = %0
-        %exn = landingpad {i8*, i32} personality i32 (...)* @__gxx_personality_v0
+        %exn = landingpad {i8*, i32}
                  cleanup
         ret i32 -1
 }
diff --git a/test/Transforms/Inline/invoke_test-3.ll b/test/Transforms/Inline/invoke_test-3.ll
index f5ce95aa516..5eabfdd417d 100644
--- a/test/Transforms/Inline/invoke_test-3.ll
+++ b/test/Transforms/Inline/invoke_test-3.ll
@@ -5,7 +5,7 @@
 
 declare void @might_throw()
 
-define internal i32 @callee() {
+define internal i32 @callee() personality i32 (...)* @__gxx_personality_v0 {
         invoke void @might_throw( )
                         to label %cont unwind label %exc
 
@@ -14,14 +14,14 @@ cont:           ; preds = %0
 
 exc:            ; preds = %0a
        ; This just rethrows the exception!
-        %exn = landingpad {i8*, i32} personality i32 (...)* @__gxx_personality_v0
+        %exn = landingpad {i8*, i32}
                  cleanup
         resume { i8*, i32 } %exn
 }
 
 ; caller returns true if might_throw throws an exception... which gets
 ; propagated by callee.
-define i32 @caller() {
+define i32 @caller() personality i32 (...)* @__gxx_personality_v0 {
         %X = invoke i32 @callee( )
                         to label %cont unwind label %Handler            ; <i32> [#uses=1]
 
@@ -30,7 +30,7 @@ cont:           ; preds = %0
 
 Handler:                ; preds = %0
 ; This consumes an exception thrown by might_throw
-        %exn = landingpad {i8*, i32} personality i32 (...)* @__gxx_personality_v0
+        %exn = landingpad {i8*, i32}
                  cleanup
         ret i32 1
 }
diff --git a/test/Transforms/Inline/noalias-cs.ll b/test/Transforms/Inline/noalias-cs.ll
index 8528a391cf9..0bff1882e83 100644
--- a/test/Transforms/Inline/noalias-cs.ll
+++ b/test/Transforms/Inline/noalias-cs.ll
@@ -34,13 +34,13 @@ entry:
 ; CHECK:   %arrayidx.i = getelementptr inbounds float, float* %a, i64 7
 ; CHECK:   store float %1, float* %arrayidx.i, align 4, !noalias !16
 ; CHECK:   %2 = load float, float* %a, align 4, !alias.scope !16, !noalias !17
-; CHECK:   %arrayidx.i.i1 = getelementptr inbounds float, float* %b, i64 5
-; CHECK:   store float %2, float* %arrayidx.i.i1, align 4, !alias.scope !21, !noalias !22
-; CHECK:   %arrayidx1.i.i2 = getelementptr inbounds float, float* %b, i64 8
-; CHECK:   store float %2, float* %arrayidx1.i.i2, align 4, !alias.scope !23, !noalias !24
+; CHECK:   %arrayidx.i.i.1 = getelementptr inbounds float, float* %b, i64 5
+; CHECK:   store float %2, float* %arrayidx.i.i.1, align 4, !alias.scope !21, !noalias !22
+; CHECK:   %arrayidx1.i.i.2 = getelementptr inbounds float, float* %b, i64 8
+; CHECK:   store float %2, float* %arrayidx1.i.i.2, align 4, !alias.scope !23, !noalias !24
 ; CHECK:   %3 = load float, float* %a, align 4, !alias.scope !16
-; CHECK:   %arrayidx.i3 = getelementptr inbounds float, float* %b, i64 7
-; CHECK:   store float %3, float* %arrayidx.i3, align 4, !alias.scope !16
+; CHECK:   %arrayidx.i.3 = getelementptr inbounds float, float* %b, i64 7
+; CHECK:   store float %3, float* %arrayidx.i.3, align 4, !alias.scope !16
 ; CHECK:   ret void
 ; CHECK: }
 
diff --git a/test/Transforms/Inline/noalias2.ll b/test/Transforms/Inline/noalias2.ll
index 432fccf431c..df135b0a318 100644
--- a/test/Transforms/Inline/noalias2.ll
+++ b/test/Transforms/Inline/noalias2.ll
@@ -61,8 +61,8 @@ entry:
 ; CHECK:   %arrayidx.i = getelementptr inbounds float, float* %a, i64 7
 ; CHECK:   store float %1, float* %arrayidx.i, align 4, !alias.scope !14, !noalias !13
 ; CHECK:   %2 = load float, float* %c, align 4, !noalias !15
-; CHECK:   %arrayidx.i1 = getelementptr inbounds float, float* %a, i64 6
-; CHECK:   store float %2, float* %arrayidx.i1, align 4, !alias.scope !19, !noalias !20
+; CHECK:   %arrayidx.i.1 = getelementptr inbounds float, float* %a, i64 6
+; CHECK:   store float %2, float* %arrayidx.i.1, align 4, !alias.scope !19, !noalias !20
 ; CHECK:   %arrayidx1.i = getelementptr inbounds float, float* %b, i64 8
 ; CHECK:   store float %2, float* %arrayidx1.i, align 4, !alias.scope !20, !noalias !19
 ; CHECK:   %3 = load float, float* %c, align 4
diff --git a/test/Transforms/Inline/nonnull.ll b/test/Transforms/Inline/nonnull.ll
new file mode 100644
index 00000000000..4aa0c28bfc7
--- /dev/null
+++ b/test/Transforms/Inline/nonnull.ll
@@ -0,0 +1,45 @@
+; RUN: opt -S -inline %s | FileCheck %s
+
+declare void @foo()
+declare void @bar()
+
+define void @callee(i8* %arg) {
+  %cmp = icmp eq i8* %arg, null
+  br i1 %cmp, label %expensive, label %done
+
+; This block is designed to be too expensive to inline.  We can only inline
+; callee if this block is known to be dead.
+expensive:
+  call void @foo()
+  call void @foo()
+  call void @foo()
+  call void @foo()
+  call void @foo()
+  call void @foo()
+  call void @foo()
+  call void @foo()
+  call void @foo()
+  call void @foo()
+  ret void
+
+done:
+  call void @bar()
+  ret void
+}
+
+; Positive test - arg is known non null
+define void @caller(i8* nonnull %arg) {
+; CHECK-LABEL: @caller
+; CHECK: call void @bar()
+  call void @callee(i8* nonnull %arg)
+  ret void
+}
+
+; Negative test - arg is not known to be non null
+define void @caller2(i8* %arg) {
+; CHECK-LABEL: @caller2
+; CHECK: call void @callee(
+  call void @callee(i8* %arg)
+  ret void
+}
+
diff --git a/test/Transforms/Inline/pr22285.ll b/test/Transforms/Inline/pr22285.ll
new file mode 100644
index 00000000000..1deae1860f0
--- /dev/null
+++ b/test/Transforms/Inline/pr22285.ll
@@ -0,0 +1,15 @@
+; RUN: opt < %s -inline -S | FileCheck %s
+
+$f1 = comdat any
+; CHECK-NOT: $f1 = comdat any
+
+define void @f2() {
+  call void @f1()
+  ret void
+}
+; CHECK-LABEL: define void @f2
+
+define linkonce_odr void @f1() comdat {
+  ret void
+}
+; CHECK-NOT: define linkonce_odr void @f1() comdat
diff --git a/test/Transforms/Inline/vector-bonus.ll b/test/Transforms/Inline/vector-bonus.ll
new file mode 100644
index 00000000000..c6745d59e5e
--- /dev/null
+++ b/test/Transforms/Inline/vector-bonus.ll
@@ -0,0 +1,37 @@
+; RUN: opt < %s -inline -inline-threshold=35  -S | FileCheck %s
+
+define i32 @bar(<4 x i32> %v, i32 %i) #0 {
+entry:
+  %cmp = icmp sgt i32 %i, 4
+  br i1 %cmp, label %if.then, label %if.else
+
+if.then:                                          ; preds = %entry
+  %mul1 = mul nsw i32 %i, %i
+  br label %return
+
+if.else:                                          ; preds = %entry
+  %add1 = add nsw i32 %i, %i
+  %add2 = add nsw i32 %i, %i
+  %add3 = add nsw i32 %i, %i
+  %add4 = add nsw i32 %i, %i
+  %add5 = add nsw i32 %i, %i
+  %add6 = add nsw i32 %i, %i
+  %vecext = extractelement <4 x i32> %v, i32 0
+  %vecext7 = extractelement <4 x i32> %v, i32 1
+  %add7 = add nsw i32 %vecext, %vecext7
+  br label %return
+
+return:                                           ; preds = %if.else, %if.then
+  %retval.0 = phi i32 [ %mul1, %if.then ], [ %add7, %if.else ]
+  ret i32 %retval.0
+}
+
+define i32 @foo(<4 x i32> %v, i32 %a) #1 {
+; CHECK-LABEL: @foo(
+; CHECK-NOT: call i32 @bar
+; CHECK: ret
+entry:
+  %call = call i32 @bar(<4 x i32> %v, i32 %a)
+  ret i32 %call
+}
+
diff --git a/test/Transforms/InstCombine/2003-10-29-CallSiteResolve.ll b/test/Transforms/InstCombine/2003-10-29-CallSiteResolve.ll
index 56493e2e5e3..1fc8aa7c278 100644
--- a/test/Transforms/InstCombine/2003-10-29-CallSiteResolve.ll
+++ b/test/Transforms/InstCombine/2003-10-29-CallSiteResolve.ll
@@ -2,7 +2,7 @@
 
 declare i32* @bar()
 
-define float* @foo() {
+define float* @foo() personality i32 (...)* @__gxx_personality_v0 {
         %tmp.11 = invoke float* bitcast (i32* ()* @bar to float* ()*)( )
                         to label %invoke_cont unwind label %X           ; <float*> [#uses=1]
 
@@ -10,7 +10,7 @@ invoke_cont:            ; preds = %0
         ret float* %tmp.11
 
 X:              ; preds = %0
-        %exn = landingpad {i8*, i32} personality i32 (...)* @__gxx_personality_v0
+        %exn = landingpad {i8*, i32}
                  cleanup
         ret float* null
 }
diff --git a/test/Transforms/InstCombine/2004-01-13-InstCombineInvokePHI.ll b/test/Transforms/InstCombine/2004-01-13-InstCombineInvokePHI.ll
index a086c015650..7471d8b3462 100644
--- a/test/Transforms/InstCombine/2004-01-13-InstCombineInvokePHI.ll
+++ b/test/Transforms/InstCombine/2004-01-13-InstCombineInvokePHI.ll
@@ -9,7 +9,7 @@
 
 declare i8* @test()
 
-define i32 @foo() {
+define i32 @foo() personality i32 (...)* @__gxx_personality_v0 {
 entry:
         br i1 true, label %cont, label %call
 
@@ -23,7 +23,7 @@ cont:           ; preds = %call, %entry
         ret i32 %V
 
 N:              ; preds = %call
-        %exn = landingpad {i8*, i32} personality i32 (...)* @__gxx_personality_v0
+        %exn = landingpad {i8*, i32}
                  cleanup
         ret i32 0
 }
diff --git a/test/Transforms/InstCombine/2008-05-09-SinkOfInvoke.ll b/test/Transforms/InstCombine/2008-05-09-SinkOfInvoke.ll
index 722f8f0fabb..4d9c19ff583 100644
--- a/test/Transforms/InstCombine/2008-05-09-SinkOfInvoke.ll
+++ b/test/Transforms/InstCombine/2008-05-09-SinkOfInvoke.ll
@@ -9,7 +9,7 @@ declare i32** @__ctype_toupper_loc() readnone
 
 declare i32** @__ctype_tolower_loc() readnone 
 
-define void @_ZNSt5ctypeIcEC2EPiPKtbm(%"struct.std::ctype<char>"* %this, i32* %unnamed_arg, i16* %__table, i8 zeroext  %__del, i64 %__refs) {
+define void @_ZNSt5ctypeIcEC2EPiPKtbm(%"struct.std::ctype<char>"* %this, i32* %unnamed_arg, i16* %__table, i8 zeroext  %__del, i64 %__refs) personality i32 (...)* @__gxx_personality_v0 {
 entry:
 	%tmp8 = invoke i32* @_ZNSt6locale5facet15_S_get_c_localeEv( )
 			to label %invcont unwind label %lpad		; <i32*> [#uses=0]
@@ -29,7 +29,7 @@ invcont37:		; preds = %invcont31
 	ret void
 
 lpad:		; preds = %invcont31, %invcont, %entry
-        %exn = landingpad {i8*, i32} personality i32 (...)* @__gxx_personality_v0
+        %exn = landingpad {i8*, i32}
                  cleanup
 	unreachable
 }
diff --git a/test/Transforms/InstCombine/AddOverFlow.ll b/test/Transforms/InstCombine/AddOverFlow.ll
index bebfd6293f9..a341cb042cc 100644
--- a/test/Transforms/InstCombine/AddOverFlow.ll
+++ b/test/Transforms/InstCombine/AddOverFlow.ll
@@ -39,7 +39,7 @@ declare i32 @__gxx_personality_v0(...);
 !0 = !{i16 0, i16 32768} ; [0, 32767]
 !1 = !{i16 0, i16 32769} ; [0, 32768]
 
-define i16 @add_bounded_values(i16 %a, i16 %b) {
+define i16 @add_bounded_values(i16 %a, i16 %b) personality i8* bitcast (i32 (...)* @__gxx_personality_v0 to i8*) {
 ; CHECK-LABEL: @add_bounded_values(
 entry:
   %c = call i16 @bounded(i16 %a), !range !0
@@ -50,12 +50,12 @@ cont:
 ; CHECK: add nuw i16 %c, %d
   ret i16 %e
 lpad:
-  %0 = landingpad { i8*, i32 } personality i8* bitcast (i32 (...)* @__gxx_personality_v0 to i8*)
+  %0 = landingpad { i8*, i32 }
           filter [0 x i8*] zeroinitializer
   ret i16 42
 }
 
-define i16 @add_bounded_values_2(i16 %a, i16 %b) {
+define i16 @add_bounded_values_2(i16 %a, i16 %b) personality i8* bitcast (i32 (...)* @__gxx_personality_v0 to i8*) {
 ; CHECK-LABEL: @add_bounded_values_2(
 entry:
   %c = call i16 @bounded(i16 %a), !range !1
@@ -67,7 +67,7 @@ cont:
 ; CHECK: add i16 %c, %d
   ret i16 %e
 lpad:
-  %0 = landingpad { i8*, i32 } personality i8* bitcast (i32 (...)* @__gxx_personality_v0 to i8*)
+  %0 = landingpad { i8*, i32 }
           filter [0 x i8*] zeroinitializer
   ret i16 42
 }
diff --git a/test/Transforms/InstCombine/LandingPadClauses.ll b/test/Transforms/InstCombine/LandingPadClauses.ll
index 0d42f7c737f..a4d77cbe8ef 100644
--- a/test/Transforms/InstCombine/LandingPadClauses.ll
+++ b/test/Transforms/InstCombine/LandingPadClauses.ll
@@ -11,7 +11,7 @@ declare i32 @__C_specific_handler(...)
 
 declare void @bar()
 
-define void @foo_generic() {
+define void @foo_generic() personality i32 (i32, i64, i8*, i8*)* @generic_personality {
 ; CHECK-LABEL: @foo_generic(
   invoke void @bar()
     to label %cont.a unwind label %lpad.a
@@ -43,7 +43,7 @@ cont.i:
   ret void
 
 lpad.a:
-  %a = landingpad { i8*, i32 } personality i32 (i32, i64, i8*, i8*)* @generic_personality
+  %a = landingpad { i8*, i32 }
           catch i32* @T1
           catch i32* @T2
           catch i32* @T1
@@ -55,7 +55,7 @@ lpad.a:
 ; CHECK-NEXT: unreachable
 
 lpad.b:
-  %b = landingpad { i8*, i32 } personality i32 (i32, i64, i8*, i8*)* @generic_personality
+  %b = landingpad { i8*, i32 }
           filter [0 x i32*] zeroinitializer
           catch i32* @T1
   unreachable
@@ -64,7 +64,7 @@ lpad.b:
 ; CHECK-NEXT: unreachable
 
 lpad.c:
-  %c = landingpad { i8*, i32 } personality i32 (i32, i64, i8*, i8*)* @generic_personality
+  %c = landingpad { i8*, i32 }
           catch i32* @T1
           filter [1 x i32*] [i32* @T1]
           catch i32* @T2
@@ -75,7 +75,7 @@ lpad.c:
 ; CHECK-NEXT: unreachable
 
 lpad.d:
-  %d = landingpad { i8*, i32 } personality i32 (i32, i64, i8*, i8*)* @generic_personality
+  %d = landingpad { i8*, i32 }
           filter [3 x i32*] zeroinitializer
   unreachable
 ; CHECK: %d = landingpad
@@ -83,7 +83,7 @@ lpad.d:
 ; CHECK-NEXT: unreachable
 
 lpad.e:
-  %e = landingpad { i8*, i32 } personality i32 (i32, i64, i8*, i8*)* @generic_personality
+  %e = landingpad { i8*, i32 }
           catch i32* @T1
           filter [3 x i32*] [i32* @T1, i32* @T2, i32* @T2]
   unreachable
@@ -93,7 +93,7 @@ lpad.e:
 ; CHECK-NEXT: unreachable
 
 lpad.f:
-  %f = landingpad { i8*, i32 } personality i32 (i32, i64, i8*, i8*)* @generic_personality
+  %f = landingpad { i8*, i32 }
           filter [2 x i32*] [i32* @T2, i32* @T1]
           filter [1 x i32*] [i32* @T1]
   unreachable
@@ -102,7 +102,7 @@ lpad.f:
 ; CHECK-NEXT: unreachable
 
 lpad.g:
-  %g = landingpad { i8*, i32 } personality i32 (i32, i64, i8*, i8*)* @generic_personality
+  %g = landingpad { i8*, i32 }
           filter [1 x i32*] [i32* @T1]
           catch i32* @T3
           filter [2 x i32*] [i32* @T2, i32* @T1]
@@ -113,7 +113,7 @@ lpad.g:
 ; CHECK-NEXT: unreachable
 
 lpad.h:
-  %h = landingpad { i8*, i32 } personality i32 (i32, i64, i8*, i8*)* @generic_personality
+  %h = landingpad { i8*, i32 }
           filter [2 x i32*] [i32* @T1, i32* null]
           filter [1 x i32*] zeroinitializer
   unreachable
@@ -122,7 +122,7 @@ lpad.h:
 ; CHECK-NEXT: unreachable
 
 lpad.i:
-  %i = landingpad { i8*, i32 } personality i32 (i32, i64, i8*, i8*)* @generic_personality
+  %i = landingpad { i8*, i32 }
           cleanup
           filter [0 x i32*] zeroinitializer
   unreachable
@@ -131,7 +131,7 @@ lpad.i:
 ; CHECK-NEXT: unreachable
 }
 
-define void @foo_cxx() {
+define void @foo_cxx() personality i32 (i32, i64, i8*, i8*)* @__gxx_personality_v0 {
 ; CHECK-LABEL: @foo_cxx(
   invoke void @bar()
     to label %cont.a unwind label %lpad.a
@@ -148,7 +148,7 @@ cont.d:
   ret void
 
 lpad.a:
-  %a = landingpad { i8*, i32 } personality i32 (i32, i64, i8*, i8*)* @__gxx_personality_v0
+  %a = landingpad { i8*, i32 }
           catch i32* null
           catch i32* @T1
   unreachable
@@ -157,7 +157,7 @@ lpad.a:
 ; CHECK-NEXT: unreachable
 
 lpad.b:
-  %b = landingpad { i8*, i32 } personality i32 (i32, i64, i8*, i8*)* @__gxx_personality_v0
+  %b = landingpad { i8*, i32 }
           filter [1 x i32*] zeroinitializer
   unreachable
 ; CHECK: %b = landingpad
@@ -165,7 +165,7 @@ lpad.b:
 ; CHECK-NEXT: unreachable
 
 lpad.c:
-  %c = landingpad { i8*, i32 } personality i32 (i32, i64, i8*, i8*)* @__gxx_personality_v0
+  %c = landingpad { i8*, i32 }
           filter [2 x i32*] [i32* @T1, i32* null]
   unreachable
 ; CHECK: %c = landingpad
@@ -173,7 +173,7 @@ lpad.c:
 ; CHECK-NEXT: unreachable
 
 lpad.d:
-  %d = landingpad { i8*, i32 } personality i32 (i32, i64, i8*, i8*)* @__gxx_personality_v0
+  %d = landingpad { i8*, i32 }
           cleanup
           catch i32* null
   unreachable
@@ -182,7 +182,7 @@ lpad.d:
 ; CHECK-NEXT: unreachable
 }
 
-define void @foo_objc() {
+define void @foo_objc() personality i32 (i32, i64, i8*, i8*)* @__objc_personality_v0 {
 ; CHECK-LABEL: @foo_objc(
   invoke void @bar()
     to label %cont.a unwind label %lpad.a
@@ -199,7 +199,7 @@ cont.d:
   ret void
 
 lpad.a:
-  %a = landingpad { i8*, i32 } personality i32 (i32, i64, i8*, i8*)* @__objc_personality_v0
+  %a = landingpad { i8*, i32 }
           catch i32* null
           catch i32* @T1
   unreachable
@@ -208,7 +208,7 @@ lpad.a:
 ; CHECK-NEXT: unreachable
 
 lpad.b:
-  %b = landingpad { i8*, i32 } personality i32 (i32, i64, i8*, i8*)* @__objc_personality_v0
+  %b = landingpad { i8*, i32 }
           filter [1 x i32*] zeroinitializer
   unreachable
 ; CHECK: %b = landingpad
@@ -216,7 +216,7 @@ lpad.b:
 ; CHECK-NEXT: unreachable
 
 lpad.c:
-  %c = landingpad { i8*, i32 } personality i32 (i32, i64, i8*, i8*)* @__objc_personality_v0
+  %c = landingpad { i8*, i32 }
           filter [2 x i32*] [i32* @T1, i32* null]
   unreachable
 ; CHECK: %c = landingpad
@@ -224,7 +224,7 @@ lpad.c:
 ; CHECK-NEXT: unreachable
 
 lpad.d:
-  %d = landingpad { i8*, i32 } personality i32 (i32, i64, i8*, i8*)* @__objc_personality_v0
+  %d = landingpad { i8*, i32 }
           cleanup
           catch i32* null
   unreachable
@@ -233,7 +233,7 @@ lpad.d:
 ; CHECK-NEXT: unreachable
 }
 
-define void @foo_seh() {
+define void @foo_seh() personality i32 (...)* @__C_specific_handler {
 ; CHECK-LABEL: @foo_seh(
   invoke void @bar()
     to label %cont.a unwind label %lpad.a
@@ -250,7 +250,7 @@ cont.d:
   ret void
 
 lpad.a:
-  %a = landingpad { i8*, i32 } personality i32 (...)* @__C_specific_handler
+  %a = landingpad { i8*, i32 }
           catch i32* null
           catch i32* @T1
   unreachable
@@ -259,7 +259,7 @@ lpad.a:
 ; CHECK-NEXT: unreachable
 
 lpad.b:
-  %b = landingpad { i8*, i32 } personality i32 (...)* @__C_specific_handler
+  %b = landingpad { i8*, i32 }
           filter [1 x i32*] zeroinitializer
   unreachable
 ; CHECK: %b = landingpad
@@ -267,7 +267,7 @@ lpad.b:
 ; CHECK-NEXT: unreachable
 
 lpad.c:
-  %c = landingpad { i8*, i32 } personality i32 (...)* @__C_specific_handler
+  %c = landingpad { i8*, i32 }
           filter [2 x i32*] [i32* @T1, i32* null]
   unreachable
 ; CHECK: %c = landingpad
@@ -275,7 +275,7 @@ lpad.c:
 ; CHECK-NEXT: unreachable
 
 lpad.d:
-  %d = landingpad { i8*, i32 } personality i32 (...)* @__C_specific_handler
+  %d = landingpad { i8*, i32 }
           cleanup
           catch i32* null
   unreachable
diff --git a/test/Transforms/InstCombine/add2.ll b/test/Transforms/InstCombine/add2.ll
index fbbba5954b3..2fe9e8cadeb 100644
--- a/test/Transforms/InstCombine/add2.ll
+++ b/test/Transforms/InstCombine/add2.ll
@@ -273,6 +273,35 @@ define i32 @mul_add_to_mul_6(i32 %x, i32 %y) {
 ; CHECK-NEXT: ret i32 %add
 }
 
+define i16 @mul_add_to_mul_7(i16 %x) {
+  %mul1 = mul nsw i16 %x, 32767
+  %add2 = add nsw i16 %x, %mul1
+  ret i16 %add2
+; CHECK-LABEL: @mul_add_to_mul_7(
+; CHECK-NEXT: %add2 = shl i16 %x, 15
+; CHECK-NEXT: ret i16 %add2
+}
+
+define i16 @mul_add_to_mul_8(i16 %a) {
+  %mul1 = mul nsw i16 %a, 16383
+  %mul2 = mul nsw i16 %a, 16384
+  %add = add nsw i16 %mul1, %mul2
+  ret i16 %add
+; CHECK-LABEL: @mul_add_to_mul_8(
+; CHECK-NEXT: %add = mul nsw i16 %a, 32767
+; CHECK-NEXT: ret i16 %add
+}
+
+define i16 @mul_add_to_mul_9(i16 %a) {
+  %mul1 = mul nsw i16 %a, 16384
+  %mul2 = mul nsw i16 %a, 16384
+  %add = add nsw i16 %mul1, %mul2
+  ret i16 %add
+; CHECK-LABEL: @mul_add_to_mul_9(
+; CHECK-NEXT: %add = shl i16 %a, 15
+; CHECK-NEXT: ret i16 %add
+}
+
 ; This test and the next test verify that when a range metadata is attached to
 ; llvm.cttz, ValueTracking correctly intersects the range specified by the
 ; metadata and the range implied by the intrinsic.
@@ -353,3 +382,26 @@ define i32 @add_nuw_nsw_or_and(i32 %x, i32 %y) {
 ; CHECK-NEXT: add nuw nsw i32 %x, %y
 ; CHECK-NEXT: ret i32
 }
+
+; A *nsw B + A *nsw C != A *nsw (B + C)
+; e.g. A = -1, B = 1, C = INT_SMAX
+
+define i8 @add_of_mul(i8 %x, i8 %y, i8 %z) {
+; CHECK-LABEL: @add_of_mul(
+ entry:
+  %mA = mul nsw i8 %x, %y
+  %mB = mul nsw i8 %x, %z
+; CHECK: %sum = mul i8
+  %sum = add nsw i8 %mA, %mB
+  ret i8 %sum
+}
+
+define i32 @add_of_selects(i1 %A, i32 %B) {
+  %sel0 = select i1 %A, i32 0, i32 -2
+  %sel1 = select i1 %A, i32 %B, i32 2
+  %add = add i32 %sel0, %sel1
+  ret i32 %add
+; CHECK-LABEL: @add_of_selects(
+; CHECK-NEXT: %[[sel:.*]] = select i1 %A, i32 %B, i32 0
+; CHECK-NEXT: ret i32 %[[sel]]
+}
diff --git a/test/Transforms/InstCombine/align-external.ll b/test/Transforms/InstCombine/align-external.ll
index ee98a012017..15f3096105b 100644
--- a/test/Transforms/InstCombine/align-external.ll
+++ b/test/Transforms/InstCombine/align-external.ll
@@ -3,16 +3,14 @@
 ; Don't assume that external global variables or those with weak linkage have
 ; their preferred alignment. They may only have the ABI minimum alignment.
 
-; CHECK: %s = shl i64 %a, 3
-; CHECK: %r = or i64 %s, ptrtoint (i32* @A to i64)
-; CHECK: %q = add i64 %r, 1
-; CHECK: ret i64 %q
-
 target datalayout = "i32:8:32"
 
 @A = external global i32
 @B = weak_odr global i32 0
 
+@C = available_externally global <4 x i32> zeroinitializer, align 4
+; CHECK: @C = available_externally global <4 x i32> zeroinitializer, align 4
+
 define i64 @foo(i64 %a) {
   %t = ptrtoint i32* @A to i64
   %s = shl i64 %a, 3
@@ -21,9 +19,23 @@ define i64 @foo(i64 %a) {
   ret i64 %q
 }
 
+; CHECK-LABEL: define i64 @foo(i64 %a)
+; CHECK: %s = shl i64 %a, 3
+; CHECK: %r = or i64 %s, ptrtoint (i32* @A to i64)
+; CHECK: %q = add i64 %r, 1
+; CHECK: ret i64 %q
+
 define i32 @bar() {
-; CHECK-LABEL: @bar(
   %r = load i32, i32* @B, align 1
-; CHECK: align 1
   ret i32 %r
 }
+
+; CHECK-LABEL: @bar()
+; CHECK: align 1
+
+define void @vec_store() {
+  store <4 x i32> <i32 0, i32 1, i32 2, i32 3>, <4 x i32>* @C, align 4
+  ret void
+}
+; CHECK: define void @vec_store()
+; CHECK: store <4 x i32> <i32 0, i32 1, i32 2, i32 3>, <4 x i32>* @C, align 4
diff --git a/test/Transforms/InstCombine/aligned-qpx.ll b/test/Transforms/InstCombine/aligned-qpx.ll
index e303ddd17d5..e9710df5670 100644
--- a/test/Transforms/InstCombine/aligned-qpx.ll
+++ b/test/Transforms/InstCombine/aligned-qpx.ll
@@ -28,6 +28,7 @@ entry:
 
 ; CHECK-LABEL: @test1a
 ; CHECK-NOT: @llvm.ppc.qpx.qvlfs
+; CHECK-NOT: load <4 x double>
 ; CHECK: ret <4 x double>
 
   %v0 = load <4 x float>, <4 x float>* %h, align 8
@@ -62,7 +63,9 @@ entry:
   ret <4 x float> %v0
 
 ; CHECK-LABEL: @test2
+; CHECK: fptrunc <4 x double> %d to <4 x float>
 ; CHECK-NOT: @llvm.ppc.qpx.qvstfs
+; CHECK-NOT: store <4 x double>
 ; CHECK: ret <4 x float>
 }
 
diff --git a/test/Transforms/InstCombine/call.ll b/test/Transforms/InstCombine/call.ll
index 47ae71f37fb..ea338f0bf58 100644
--- a/test/Transforms/InstCombine/call.ll
+++ b/test/Transforms/InstCombine/call.ll
@@ -123,7 +123,7 @@ define void @test7() {
 ; rdar://7590304
 declare void @test8a()
 
-define i8* @test8() {
+define i8* @test8() personality i32 (...)* @__gxx_personality_v0 {
 ; CHECK-LABEL: @test8(
 ; CHECK-NEXT: invoke void @test8a()
 ; Don't turn this into "unreachable": the callee and caller don't agree in
@@ -136,7 +136,7 @@ invoke.cont:                                      ; preds = %entry
   unreachable
 
 try.handler:                                      ; preds = %entry
-  %exn = landingpad {i8*, i32} personality i32 (...)* @__gxx_personality_v0
+  %exn = landingpad {i8*, i32}
             cleanup
   ret i8* null
 }
diff --git a/test/Transforms/InstCombine/cast.ll b/test/Transforms/InstCombine/cast.ll
index c96140e2197..7fe54ef8469 100644
--- a/test/Transforms/InstCombine/cast.ll
+++ b/test/Transforms/InstCombine/cast.ll
@@ -100,23 +100,23 @@ define void @test11(i32* %P) {
 }
 
 declare i32 @__gxx_personality_v0(...)
-define void @test_invoke_vararg_cast(i32* %a, i32* %b) {
+define void @test_invoke_vararg_cast(i32* %a, i32* %b) personality i8* bitcast (i32 (...)* @__gxx_personality_v0 to i8*) {
 entry:
   %0 = bitcast i32* %b to i8*
   %1 = bitcast i32* %a to i64*
-  invoke void (i32, ...)* @varargs(i32 1, i8* %0, i64* %1)
+  invoke void (i32, ...) @varargs(i32 1, i8* %0, i64* %1)
           to label %invoke.cont unwind label %lpad
 
 invoke.cont:                                      ; preds = %entry
   ret void
 
 lpad:                                             ; preds = %entry
-  %2 = landingpad { i8*, i32 } personality i8* bitcast (i32 (...)* @__gxx_personality_v0 to i8*)
+  %2 = landingpad { i8*, i32 }
           cleanup
   ret void
 ; CHECK-LABEL: test_invoke_vararg_cast
 ; CHECK-LABEL: entry:
-; CHECK: invoke void (i32, ...)* @varargs(i32 1, i32* %b, i32* %a)
+; CHECK: invoke void (i32, ...) @varargs(i32 1, i32* %b, i32* %a)
 }
 
 define i8* @test13(i64 %A) {
@@ -187,8 +187,8 @@ define i32 @test21(i32 %X) {
         %c2 = sext i8 %c1 to i32                ; <i32> [#uses=1]
         %RV = and i32 %c2, 255          ; <i32> [#uses=1]
         ret i32 %RV
-; CHECK: %c21 = and i32 %X, 255
-; CHECK: ret i32 %c21
+; CHECK: %c2.1 = and i32 %X, 255
+; CHECK: ret i32 %c2.1
 }
 
 define i32 @test22(i32 %X) {
@@ -1104,3 +1104,36 @@ define i32 @PR21388(i32* %v) {
 ; CHECK-NEXT: %[[sext:.*]] = sext i1 %[[icmp]] to i32
 ; CHECK-NEXT: ret i32 %[[sext]]
 }
+
+define float @sitofp_zext(i16 %a) {
+; CHECK-LABEL: @sitofp_zext(
+; CHECK-NEXT: %[[sitofp:.*]] = uitofp i16 %a to float
+; CHECK-NEXT: ret float %[[sitofp]]
+  %zext = zext i16 %a to i32
+  %sitofp = sitofp i32 %zext to float
+  ret float %sitofp
+}
+
+define i1 @PR23309(i32 %A, i32 %B) {
+; CHECK-LABEL: @PR23309(
+; CHECK-NEXT: %[[sub:.*]] = sub i32 %A, %B
+; CHECK-NEXT: %[[and:.*]] = and i32 %[[sub]], 1
+; CHECK-NEXT: %[[cmp:.*]] = icmp ne i32 %[[and]], 0
+; CHECK-NEXT: ret i1 %[[cmp]]
+  %add = add i32 %A, -4
+  %sub = sub nsw i32 %add, %B
+  %trunc = trunc i32 %sub to i1
+  ret i1 %trunc
+}
+
+define i1 @PR23309v2(i32 %A, i32 %B) {
+; CHECK-LABEL: @PR23309v2(
+; CHECK-NEXT: %[[sub:.*]] = add i32 %A, %B
+; CHECK-NEXT: %[[and:.*]] = and i32 %[[sub]], 1
+; CHECK-NEXT: %[[cmp:.*]] = icmp ne i32 %[[and]], 0
+; CHECK-NEXT: ret i1 %[[cmp]]
+  %add = add i32 %A, -4
+  %sub = add nuw i32 %add, %B
+  %trunc = trunc i32 %sub to i1
+  ret i1 %trunc
+}
diff --git a/test/Transforms/InstCombine/crash.ll b/test/Transforms/InstCombine/crash.ll
index 2d93ecd59bf..fbb9675c0f1 100644
--- a/test/Transforms/InstCombine/crash.ll
+++ b/test/Transforms/InstCombine/crash.ll
@@ -131,11 +131,11 @@ define i32 @test5a() {
        ret i32 0
 }
 
-define void @test5() {
+define void @test5() personality i32 (...)* @__gxx_personality_v0 {
   store i1 true, i1* undef
   %r = invoke i32 @test5a() to label %exit unwind label %unwind
 unwind:
-  %exn = landingpad {i8*, i32} personality i32 (...)* @__gxx_personality_v0
+  %exn = landingpad {i8*, i32}
           cleanup
   br label %exit
 exit:
@@ -159,7 +159,7 @@ entry:
 %class.RuleBasedBreakIterator = type { i64 ()* }
 %class.UStack = type { i8** }
 
-define i32 @_ZN22RuleBasedBreakIterator15checkDictionaryEi(%class.RuleBasedBreakIterator* %this, i32 %x) align 2 {
+define i32 @_ZN22RuleBasedBreakIterator15checkDictionaryEi(%class.RuleBasedBreakIterator* %this, i32 %x) align 2 personality i8* bitcast (i32 (...)* @__gxx_personality_v0 to i8*) {
 entry:
   %breaks = alloca %class.UStack, align 4         ; <%class.UStack*> [#uses=3]
   call void @_ZN6UStackC1Ei(%class.UStack* %breaks, i32 0)
@@ -167,13 +167,13 @@ entry:
   br i1 %tobool, label %cond.end, label %cond.false
 
 terminate.handler:                                ; preds = %ehcleanup
-  %exc = landingpad { i8*, i32 } personality i8* bitcast (i32 (...)* @__gxx_personality_v0 to i8*)
+  %exc = landingpad { i8*, i32 }
            cleanup
   call void @_ZSt9terminatev() noreturn nounwind
   unreachable
 
 ehcleanup:                                        ; preds = %cond.false
-  %exc1 = landingpad { i8*, i32 } personality i8* bitcast (i32 (...)* @__gxx_personality_v0 to i8*)
+  %exc1 = landingpad { i8*, i32 }
            catch i8* null
   invoke void @_ZN6UStackD1Ev(%class.UStack* %breaks)
           to label %cont unwind label %terminate.handler
@@ -207,7 +207,7 @@ declare void @_Unwind_Resume_or_Rethrow(i8*)
 
 
 ; rdar://7590304
-define i8* @test10(i8* %self, i8* %tmp3) {
+define i8* @test10(i8* %self, i8* %tmp3) personality i32 (...)* @__gxx_personality_v0 {
 entry:
   store i1 true, i1* undef
   store i1 true, i1* undef
@@ -218,7 +218,7 @@ invoke.cont:                                      ; preds = %entry
   unreachable
 
 try.handler:                                      ; preds = %entry
-  %exn = landingpad {i8*, i32} personality i32 (...)* @__gxx_personality_v0
+  %exn = landingpad {i8*, i32}
            catch i8* null
   ret i8* %self
 }
@@ -376,7 +376,7 @@ return:                                           ; No predecessors!
 declare void @test18b() noreturn
 declare void @test18foo(double**)
 declare void @test18a() noreturn
-define fastcc void @test18x(i8* %t0, i1 %b) uwtable align 2 {
+define fastcc void @test18x(i8* %t0, i1 %b) uwtable align 2 personality i32 (...)* @__gxx_personality_v0 {
 entry:
   br i1 %b, label %e1, label %e2
 e1:
@@ -389,7 +389,7 @@ e2:
           to label %u unwind label %lpad
 lpad:
   %t5 = phi double** [ %t2, %e1 ], [ %t4, %e2 ]
-  %lpad.nonloopexit262 = landingpad { i8*, i32 } personality i32 (...)* @__gxx_personality_v0
+  %lpad.nonloopexit262 = landingpad { i8*, i32 }
           cleanup
   call void @test18foo(double** %t5)
   unreachable
diff --git a/test/Transforms/InstCombine/debug-line.ll b/test/Transforms/InstCombine/debug-line.ll
index 8e335b82639..823ec98ebe2 100644
--- a/test/Transforms/InstCombine/debug-line.ll
+++ b/test/Transforms/InstCombine/debug-line.ll
@@ -15,14 +15,14 @@ declare i32 @printf(i8*, ...)
 !llvm.module.flags = !{!10}
 !llvm.dbg.sp = !{!0}
 
-!0 = !MDSubprogram(name: "foo", line: 4, isLocal: false, isDefinition: true, virtualIndex: 6, isOptimized: false, file: !8, scope: !1, type: !3, function: void ()* @foo)
-!1 = !MDFile(filename: "m.c", directory: "/private/tmp")
-!2 = !MDCompileUnit(language: DW_LANG_C99, producer: "clang", isOptimized: true, emissionKind: 0, file: !8, enums: !{}, retainedTypes: !{}, subprograms: !9)
-!3 = !MDSubroutineType(types: !4)
+!0 = !DISubprogram(name: "foo", line: 4, isLocal: false, isDefinition: true, virtualIndex: 6, isOptimized: false, file: !8, scope: !1, type: !3, function: void ()* @foo)
+!1 = !DIFile(filename: "m.c", directory: "/private/tmp")
+!2 = !DICompileUnit(language: DW_LANG_C99, producer: "clang", isOptimized: true, emissionKind: 0, file: !8, enums: !{}, retainedTypes: !{}, subprograms: !9)
+!3 = !DISubroutineType(types: !4)
 !4 = !{null}
-!5 = !MDLocation(line: 5, column: 2, scope: !6)
-!6 = distinct !MDLexicalBlock(line: 4, column: 12, file: !8, scope: !0)
-!7 = !MDLocation(line: 6, column: 1, scope: !6)
-!8 = !MDFile(filename: "m.c", directory: "/private/tmp")
+!5 = !DILocation(line: 5, column: 2, scope: !6)
+!6 = distinct !DILexicalBlock(line: 4, column: 12, file: !8, scope: !0)
+!7 = !DILocation(line: 6, column: 1, scope: !6)
+!8 = !DIFile(filename: "m.c", directory: "/private/tmp")
 !9 = !{!0}
 !10 = !{i32 1, !"Debug Info Version", i32 3}
diff --git a/test/Transforms/InstCombine/debuginfo.ll b/test/Transforms/InstCombine/debuginfo.ll
index 0a2ea0e6d58..3875bcc9b8c 100644
--- a/test/Transforms/InstCombine/debuginfo.ll
+++ b/test/Transforms/InstCombine/debuginfo.ll
@@ -14,11 +14,11 @@ entry:
   store i8* %__dest, i8** %__dest.addr, align 8
 ; CHECK-NOT: call void @llvm.dbg.declare
 ; CHECK: call void @llvm.dbg.value
-  call void @llvm.dbg.declare(metadata i8** %__dest.addr, metadata !0, metadata !MDExpression()), !dbg !16
+  call void @llvm.dbg.declare(metadata i8** %__dest.addr, metadata !0, metadata !DIExpression()), !dbg !16
   store i32 %__val, i32* %__val.addr, align 4
-  call void @llvm.dbg.declare(metadata i32* %__val.addr, metadata !7, metadata !MDExpression()), !dbg !18
+  call void @llvm.dbg.declare(metadata i32* %__val.addr, metadata !7, metadata !DIExpression()), !dbg !18
   store i64 %__len, i64* %__len.addr, align 8
-  call void @llvm.dbg.declare(metadata i64* %__len.addr, metadata !9, metadata !MDExpression()), !dbg !20
+  call void @llvm.dbg.declare(metadata i64* %__len.addr, metadata !9, metadata !DIExpression()), !dbg !20
   %tmp = load i8*, i8** %__dest.addr, align 8, !dbg !21
   %tmp1 = load i32, i32* %__val.addr, align 4, !dbg !21
   %tmp2 = load i64, i64* %__len.addr, align 8, !dbg !21
@@ -31,29 +31,29 @@ entry:
 !llvm.dbg.cu = !{!3}
 !llvm.module.flags = !{!30}
 
-!0 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "__dest", line: 78, arg: 1, scope: !1, file: !2, type: !6)
-!1 = !MDSubprogram(name: "foobar", line: 79, isLocal: true, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: true, scopeLine: 79, file: !27, scope: !2, type: !4, function: i8* (i8*, i32, i64)* @foobar, variables: !25)
-!2 = !MDFile(filename: "string.h", directory: "Game")
-!3 = !MDCompileUnit(language: DW_LANG_C99, producer: "clang version 3.0 (trunk 127710)", isOptimized: true, emissionKind: 0, file: !28, enums: !29, retainedTypes: !29, subprograms: !24)
-!4 = !MDSubroutineType(types: !5)
+!0 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "__dest", line: 78, arg: 1, scope: !1, file: !2, type: !6)
+!1 = !DISubprogram(name: "foobar", line: 79, isLocal: true, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: true, scopeLine: 79, file: !27, scope: !2, type: !4, function: i8* (i8*, i32, i64)* @foobar, variables: !25)
+!2 = !DIFile(filename: "string.h", directory: "Game")
+!3 = !DICompileUnit(language: DW_LANG_C99, producer: "clang version 3.0 (trunk 127710)", isOptimized: true, emissionKind: 0, file: !28, enums: !29, retainedTypes: !29, subprograms: !24)
+!4 = !DISubroutineType(types: !5)
 !5 = !{!6}
-!6 = !MDDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, scope: !3, baseType: null)
-!7 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "__val", line: 78, arg: 2, scope: !1, file: !2, type: !8)
-!8 = !MDBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
-!9 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "__len", line: 78, arg: 3, scope: !1, file: !2, type: !10)
-!10 = !MDDerivedType(tag: DW_TAG_typedef, name: "size_t", line: 80, file: !27, scope: !3, baseType: !11)
-!11 = !MDDerivedType(tag: DW_TAG_typedef, name: "__darwin_size_t", line: 90, file: !27, scope: !3, baseType: !12)
-!12 = !MDBasicType(tag: DW_TAG_base_type, name: "long unsigned int", size: 64, align: 64, encoding: DW_ATE_unsigned)
-!16 = !MDLocation(line: 78, column: 28, scope: !1)
-!18 = !MDLocation(line: 78, column: 40, scope: !1)
-!20 = !MDLocation(line: 78, column: 54, scope: !1)
-!21 = !MDLocation(line: 80, column: 3, scope: !22)
-!22 = distinct !MDLexicalBlock(line: 80, column: 3, file: !27, scope: !23)
-!23 = distinct !MDLexicalBlock(line: 79, column: 1, file: !27, scope: !1)
+!6 = !DIDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, scope: !3, baseType: null)
+!7 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "__val", line: 78, arg: 2, scope: !1, file: !2, type: !8)
+!8 = !DIBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
+!9 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "__len", line: 78, arg: 3, scope: !1, file: !2, type: !10)
+!10 = !DIDerivedType(tag: DW_TAG_typedef, name: "size_t", line: 80, file: !27, scope: !3, baseType: !11)
+!11 = !DIDerivedType(tag: DW_TAG_typedef, name: "__darwin_size_t", line: 90, file: !27, scope: !3, baseType: !12)
+!12 = !DIBasicType(tag: DW_TAG_base_type, name: "long unsigned int", size: 64, align: 64, encoding: DW_ATE_unsigned)
+!16 = !DILocation(line: 78, column: 28, scope: !1)
+!18 = !DILocation(line: 78, column: 40, scope: !1)
+!20 = !DILocation(line: 78, column: 54, scope: !1)
+!21 = !DILocation(line: 80, column: 3, scope: !22)
+!22 = distinct !DILexicalBlock(line: 80, column: 3, file: !27, scope: !23)
+!23 = distinct !DILexicalBlock(line: 79, column: 1, file: !27, scope: !1)
 !24 = !{!1}
 !25 = !{!0, !7, !9}
-!26 = !MDFile(filename: "bits.c", directory: "Game")
-!27 = !MDFile(filename: "string.h", directory: "Game")
-!28 = !MDFile(filename: "bits.c", directory: "Game")
+!26 = !DIFile(filename: "bits.c", directory: "Game")
+!27 = !DIFile(filename: "string.h", directory: "Game")
+!28 = !DIFile(filename: "bits.c", directory: "Game")
 !29 = !{}
 !30 = !{i32 1, !"Debug Info Version", i32 3}
diff --git a/test/Transforms/InstCombine/demand_shrink_nsw.ll b/test/Transforms/InstCombine/demand_shrink_nsw.ll
new file mode 100644
index 00000000000..f4917429516
--- /dev/null
+++ b/test/Transforms/InstCombine/demand_shrink_nsw.ll
@@ -0,0 +1,26 @@
+; RUN: opt -instcombine -o - -S %s | FileCheck %s
+
+; The constant at %v35 should be shrunk, but this must lead to the nsw flag of
+; %v43 getting removed so that %v44 is not illegally optimized away.
+; CHECK-LABEL: @foo
+; CHECK: %v35 = add nuw i32 %v34, 1362915575
+; ...
+; CHECK: add nuw i32 %v42, 1533579450
+; CHECK-NEXT: %v44 = or i32 %v43, -2147483648
+; CHECK-NEXT: %v45 = xor i32 %v44, 749011377
+; CHECK-NEXT: ret i32 %v45
+define i32 @foo(i32 %arg) {
+  %v33 = and i32 %arg, 223
+  %v34 = xor i32 %v33, 29
+  %v35 = add nuw i32 %v34, 3510399223
+  %v37 = or i32 %v34, 1874836915
+  %v38 = and i32 %v34, 221
+  %v39 = xor i32 %v38, 1874836915
+  %v40 = xor i32 %v37, %v39
+  %v41 = shl nsw nuw i32 %v40, 1
+  %v42 = sub i32 %v35, %v41
+  %v43 = add nsw i32 %v42, 1533579450
+  %v44 = or i32 %v43, -2147483648
+  %v45 = xor i32 %v44, 749011377
+  ret i32 %v45
+}
diff --git a/test/Transforms/InstCombine/descale-zero.ll b/test/Transforms/InstCombine/descale-zero.ll
deleted file mode 100644
index 4347be48e61..00000000000
--- a/test/Transforms/InstCombine/descale-zero.ll
+++ /dev/null
@@ -1,20 +0,0 @@
-; RUN: opt < %s -instcombine -S | FileCheck %s
-
-target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
-target triple = "x86_64-apple-macosx10.10.0"
-
-define internal i8* @descale_zero() {
-entry:
-; CHECK: load i8*, i8** inttoptr (i64 48 to i8**), align 16
-; CHECK-NEXT: ret i8*
-  %i16_ptr = load i16*, i16** inttoptr (i64 48 to i16**), align 16
-  %num = load i64, i64* inttoptr (i64 64 to i64*), align 64
-  %num_times_2 = shl i64 %num, 1
-  %num_times_2_plus_4 = add i64 %num_times_2, 4
-  %i8_ptr = bitcast i16* %i16_ptr to i8*
-  %i8_ptr_num_times_2_plus_4 = getelementptr i8, i8* %i8_ptr, i64 %num_times_2_plus_4
-  %num_times_neg2 = mul i64 %num, -2
-  %num_times_neg2_minus_4 = add i64 %num_times_neg2, -4
-  %addr = getelementptr i8, i8* %i8_ptr_num_times_2_plus_4, i64 %num_times_neg2_minus_4
-  ret i8* %addr
-}
diff --git a/test/Transforms/InstCombine/fpcast.ll b/test/Transforms/InstCombine/fpcast.ll
index 8319624b87c..93a64e6b49b 100644
--- a/test/Transforms/InstCombine/fpcast.ll
+++ b/test/Transforms/InstCombine/fpcast.ll
@@ -85,3 +85,11 @@ define float @test8(float %V) {
 ; CHECK-NEXT: %[[trunc:.*]] = fptrunc double %frem to float
 ; CHECK-NEXT: ret float %trunc
 }
+
+; CHECK-LABEL: @test_fptrunc_fptrunc
+; CHECK-NOT: fptrunc double {{.*}} to half
+define half @test_fptrunc_fptrunc(double %V) {
+  %t1 = fptrunc double %V to float
+  %t2 = fptrunc float %t1 to half
+  ret half %t2
+}
diff --git a/test/Transforms/InstCombine/gc.relocate.ll b/test/Transforms/InstCombine/gc.relocate.ll
index 5ecde19b7a4..a51aac10eb5 100644
--- a/test/Transforms/InstCombine/gc.relocate.ll
+++ b/test/Transforms/InstCombine/gc.relocate.ll
@@ -6,7 +6,7 @@ target datalayout = "e-p:32:32:32-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-f3
 ; then the return attribute of gc.relocate is dereferenceable(N).
 
 declare zeroext i1 @return_i1()
-declare i32 @llvm.experimental.gc.statepoint.p0f_i1f(i1 ()*, i32, i32, ...)
+declare i32 @llvm.experimental.gc.statepoint.p0f_i1f(i64, i32, i1 ()*, i32, i32, ...)
 declare i32 addrspace(1)* @llvm.experimental.gc.relocate.p1i32(i32, i32, i32)
 
 define i32 addrspace(1)* @deref(i32 addrspace(1)* dereferenceable(8) %dparam) gc "statepoint-example" {
@@ -15,7 +15,7 @@ define i32 addrspace(1)* @deref(i32 addrspace(1)* dereferenceable(8) %dparam) gc
 ; CHECK: call dereferenceable(8)
 entry:
     %load = load i32, i32 addrspace(1)* %dparam
-    %tok = tail call i32 (i1 ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_i1f(i1 ()* @return_i1, i32 0, i32 0, i32 0, i32 addrspace(1)* %dparam)
-    %relocate = call i32 addrspace(1)* @llvm.experimental.gc.relocate.p1i32(i32 %tok, i32 4, i32 4)
+    %tok = tail call i32 (i64, i32, i1 ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_i1f(i64 0, i32 0, i1 ()* @return_i1, i32 0, i32 0, i32 0, i32 0, i32 addrspace(1)* %dparam)
+    %relocate = call i32 addrspace(1)* @llvm.experimental.gc.relocate.p1i32(i32 %tok,  i32 7, i32 7)
     ret i32 addrspace(1)* %relocate
 }
diff --git a/test/Transforms/InstCombine/gepphigep.ll b/test/Transforms/InstCombine/gepphigep.ll
index 5ae3171b39f..b98ea4cd115 100644
--- a/test/Transforms/InstCombine/gepphigep.ll
+++ b/test/Transforms/InstCombine/gepphigep.ll
@@ -59,7 +59,7 @@ bb:
 
 ; Check that instcombine doesn't insert GEPs before landingpad.
 
-define i32 @test3(%struct3* %dm, i1 %tmp4, i64 %tmp9, i64 %tmp19, i64 %tmp20, i64 %tmp21) {
+define i32 @test3(%struct3* %dm, i1 %tmp4, i64 %tmp9, i64 %tmp19, i64 %tmp20, i64 %tmp21) personality i8* bitcast (i32 (...)* @__gxx_personality_v0 to i8*) {
 bb:
   %tmp = getelementptr inbounds %struct3, %struct3* %dm, i64 0
   br i1 %tmp4, label %bb1, label %bb2
@@ -84,7 +84,7 @@ bb4:
   ret i32 0
 
 bb5:
-  %tmp27 = landingpad { i8*, i32 } personality i8* bitcast (i32 (...)* @__gxx_personality_v0 to i8*) catch i8* bitcast (i8** @_ZTIi to i8*)
+  %tmp27 = landingpad { i8*, i32 } catch i8* bitcast (i8** @_ZTIi to i8*)
   %tmp34 = getelementptr inbounds %struct4, %struct4* %phi, i64 %tmp21, i32 1
   %tmp35 = getelementptr inbounds %struct2, %struct2* %tmp34, i64 0, i32 1
   %tmp25 = load i32, i32* %tmp35, align 4
@@ -92,7 +92,7 @@ bb5:
 
 ; CHECK-LABEL: @test3(
 ; CHECK: bb5:
-; CHECK-NEXT: {{.*}}landingpad { i8*, i32 } personality i8* bitcast (i32 (...)* @__gxx_personality_v0 to i8*)
+; CHECK-NEXT: {{.*}}landingpad { i8*, i32 }
 }
 
 @_ZTIi = external constant i8*
diff --git a/test/Transforms/InstCombine/getelementptr.ll b/test/Transforms/InstCombine/getelementptr.ll
index 9bb1b12a697..276ada91f3c 100644
--- a/test/Transforms/InstCombine/getelementptr.ll
+++ b/test/Transforms/InstCombine/getelementptr.ll
@@ -104,8 +104,8 @@ define i32* @test7(i32* %I, i64 %C, i64 %D) {
         %B = getelementptr i32, i32* %A, i64 %D
         ret i32* %B
 ; CHECK-LABEL: @test7(
-; CHECK: %A.sum = add i64 %C, %D
-; CHECK: getelementptr i32, i32* %I, i64 %A.sum
+; CHECK: %A = getelementptr i32, i32* %I, i64 %C
+; CHECK: %B = getelementptr i32, i32* %A, i64 %D
 }
 
 define i8* @test8([10 x i32]* %X) {
diff --git a/test/Transforms/InstCombine/intrinsics.ll b/test/Transforms/InstCombine/intrinsics.ll
index f9ccf51a621..88f03249827 100644
--- a/test/Transforms/InstCombine/intrinsics.ll
+++ b/test/Transforms/InstCombine/intrinsics.ll
@@ -17,6 +17,13 @@ declare i32 @llvm.cttz.i32(i32, i1) nounwind readnone
 declare i32 @llvm.ctlz.i32(i32, i1) nounwind readnone
 declare i32 @llvm.ctpop.i32(i32) nounwind readnone
 declare i8 @llvm.ctlz.i8(i8, i1) nounwind readnone
+declare double @llvm.cos.f64(double %Val) nounwind readonly
+declare double @llvm.sin.f64(double %Val) nounwind readonly
+declare double @llvm.floor.f64(double %Val) nounwind readonly
+declare double @llvm.ceil.f64(double %Val) nounwind readonly
+declare double @llvm.trunc.f64(double %Val) nounwind readonly
+declare double @llvm.rint.f64(double %Val) nounwind readonly
+declare double @llvm.nearbyint.f64(double %Val) nounwind readonly
 
 define i8 @uaddtest1(i8 %A, i8 %B) {
   %x = call %overflow.result @llvm.uadd.with.overflow.i8(i8 %A, i8 %B)
@@ -417,3 +424,91 @@ define %ov.result.32 @ssubtest_reorder(i8 %a) {
 ; CHECK-NEXT: %1 = insertvalue %ov.result.32 { i32 undef, i1 false }, i32 %x, 0
 ; CHECK-NEXT:  ret %ov.result.32 %1
 }
+
+define %ov.result.32 @never_overflows_ssub_test0(i32 %a) {
+  %x = call %ov.result.32 @llvm.ssub.with.overflow.i32(i32 %a, i32 0)
+  ret %ov.result.32 %x
+; CHECK-LABEL: @never_overflows_ssub_test0
+; CHECK-NEXT: %[[x:.*]] = insertvalue %ov.result.32 { i32 undef, i1 false }, i32 %a, 0
+; CHECK-NEXT:  ret %ov.result.32 %[[x]]
+}
+
+define void @cos(double *%P) {
+entry:
+  %B = tail call double @llvm.cos.f64(double 0.0) nounwind
+  store volatile double %B, double* %P
+
+  ret void
+; CHECK-LABEL: @cos(
+; CHECK: store volatile double 1.000000e+00, double* %P
+}
+
+define void @sin(double *%P) {
+entry:
+  %B = tail call double @llvm.sin.f64(double 0.0) nounwind
+  store volatile double %B, double* %P
+
+  ret void
+; CHECK-LABEL: @sin(
+; CHECK: store volatile double 0.000000e+00, double* %P
+}
+
+define void @floor(double *%P) {
+entry:
+  %B = tail call double @llvm.floor.f64(double 1.5) nounwind
+  store volatile double %B, double* %P
+  %C = tail call double @llvm.floor.f64(double -1.5) nounwind
+  store volatile double %C, double* %P
+  ret void
+; CHECK-LABEL: @floor(
+; CHECK: store volatile double 1.000000e+00, double* %P, align 8
+; CHECK: store volatile double -2.000000e+00, double* %P, align 8
+}
+
+define void @ceil(double *%P) {
+entry:
+  %B = tail call double @llvm.ceil.f64(double 1.5) nounwind
+  store volatile double %B, double* %P
+  %C = tail call double @llvm.ceil.f64(double -1.5) nounwind
+  store volatile double %C, double* %P
+  ret void
+; CHECK-LABEL: @ceil(
+; CHECK: store volatile double 2.000000e+00, double* %P, align 8
+; CHECK: store volatile double -1.000000e+00, double* %P, align 8
+}
+
+define void @trunc(double *%P) {
+entry:
+  %B = tail call double @llvm.trunc.f64(double 1.5) nounwind
+  store volatile double %B, double* %P
+  %C = tail call double @llvm.trunc.f64(double -1.5) nounwind
+  store volatile double %C, double* %P
+  ret void
+; CHECK-LABEL: @trunc(
+; CHECK: store volatile double 1.000000e+00, double* %P, align 8
+; CHECK: store volatile double -1.000000e+00, double* %P, align 8
+}
+
+define void @rint(double *%P) {
+entry:
+  %B = tail call double @llvm.rint.f64(double 1.5) nounwind
+  store volatile double %B, double* %P
+  %C = tail call double @llvm.rint.f64(double -1.5) nounwind
+  store volatile double %C, double* %P
+  ret void
+; CHECK-LABEL: @rint(
+; CHECK: store volatile double 2.000000e+00, double* %P, align 8
+; CHECK: store volatile double -2.000000e+00, double* %P, align 8
+}
+
+define void @nearbyint(double *%P) {
+entry:
+  %B = tail call double @llvm.nearbyint.f64(double 1.5) nounwind
+  store volatile double %B, double* %P
+  %C = tail call double @llvm.nearbyint.f64(double -1.5) nounwind
+  store volatile double %C, double* %P
+  ret void
+; CHECK-LABEL: @nearbyint(
+; CHECK: store volatile double 2.000000e+00, double* %P, align 8
+; CHECK: store volatile double -2.000000e+00, double* %P, align 8
+}
diff --git a/test/Transforms/InstCombine/invoke.ll b/test/Transforms/InstCombine/invoke.ll
index c4b58de6194..ee08ae14887 100644
--- a/test/Transforms/InstCombine/invoke.ll
+++ b/test/Transforms/InstCombine/invoke.ll
@@ -8,7 +8,7 @@ declare i8* @_Znwm(i64)
 
 
 ; CHECK-LABEL: @f1(
-define i64 @f1() nounwind uwtable ssp {
+define i64 @f1() nounwind uwtable ssp personality i8* bitcast (i32 (...)* @__gxx_personality_v0 to i8*) {
 entry:
 ; CHECK: nvoke noalias i8* undef()
   %call = invoke noalias i8* undef()
@@ -20,7 +20,7 @@ invoke.cont:
   ret i64 %0
 
 lpad:
-  %1 = landingpad { i8*, i32 } personality i8* bitcast (i32 (...)* @__gxx_personality_v0 to i8*)
+  %1 = landingpad { i8*, i32 }
           filter [0 x i8*] zeroinitializer
   %2 = extractvalue { i8*, i32 } %1, 0
   tail call void @__cxa_call_unexpected(i8* %2) noreturn nounwind
@@ -28,7 +28,7 @@ lpad:
 }
 
 ; CHECK-LABEL: @f2(
-define i64 @f2() nounwind uwtable ssp {
+define i64 @f2() nounwind uwtable ssp personality i8* bitcast (i32 (...)* @__gxx_personality_v0 to i8*) {
 entry:
 ; CHECK: nvoke noalias i8* null()
   %call = invoke noalias i8* null()
@@ -40,7 +40,7 @@ invoke.cont:
   ret i64 %0
 
 lpad:
-  %1 = landingpad { i8*, i32 } personality i8* bitcast (i32 (...)* @__gxx_personality_v0 to i8*)
+  %1 = landingpad { i8*, i32 }
           filter [0 x i8*] zeroinitializer
   %2 = extractvalue { i8*, i32 } %1, 0
   tail call void @__cxa_call_unexpected(i8* %2) noreturn nounwind
@@ -48,7 +48,7 @@ lpad:
 }
 
 ; CHECK-LABEL: @f3(
-define void @f3() nounwind uwtable ssp {
+define void @f3() nounwind uwtable ssp personality i8* bitcast (i32 (...)* @__gxx_personality_v0 to i8*) {
 ; CHECK: invoke void @llvm.donothing()
   %call = invoke noalias i8* @_Znwm(i64 13)
           to label %invoke.cont unwind label %lpad
@@ -57,7 +57,7 @@ invoke.cont:
   ret void
 
 lpad:
-  %1 = landingpad { i8*, i32 } personality i8* bitcast (i32 (...)* @__gxx_personality_v0 to i8*)
+  %1 = landingpad { i8*, i32 }
           filter [0 x i8*] zeroinitializer
   %2 = extractvalue { i8*, i32 } %1, 0
   tail call void @__cxa_call_unexpected(i8* %2) noreturn nounwind
diff --git a/test/Transforms/InstCombine/load-bitcast32.ll b/test/Transforms/InstCombine/load-bitcast32.ll
new file mode 100644
index 00000000000..b1c78a8a314
--- /dev/null
+++ b/test/Transforms/InstCombine/load-bitcast32.ll
@@ -0,0 +1,79 @@
+; RUN: opt -instcombine -S < %s | FileCheck %s
+
+target datalayout = "p:32:32:32"
+
+
+define i64* @test1(i8* %x) {
+entry:
+; CHECK-LABEL: @test1(
+; CHECK: load i64, i64*
+; CHECK: ret
+  %a = bitcast i8* %x to i64*
+  %b = load i64, i64* %a
+  %c = inttoptr i64 %b to i64*
+
+  ret i64* %c
+}
+
+define i32* @test2(i8* %x) {
+entry:
+; CHECK-LABEL: @test2(
+; CHECK: load i32*, i32**
+; CHECK: ret
+  %a = bitcast i8* %x to i32*
+  %b = load i32, i32* %a
+  %c = inttoptr i32 %b to i32*
+
+  ret i32* %c
+}
+
+define i64* @test3(i8* %x) {
+entry:
+; CHECK-LABEL: @test3(
+; CHECK: load i64*, i64**
+; CHECK: ret
+  %a = bitcast i8* %x to i32*
+  %b = load i32, i32* %a
+  %c = inttoptr i32 %b to i64*
+
+  ret i64* %c
+}
+
+define i64 @test4(i8* %x) {
+entry:
+; CHECK-LABEL: @test4(
+; CHECK: load i32, i32*
+; CHECK: zext
+; CHECK: ret
+  %a = bitcast i8* %x to i64**
+  %b = load i64*, i64** %a
+  %c = ptrtoint i64* %b to i64
+
+  ret i64 %c
+}
+
+define i32 @test5(i8* %x) {
+entry:
+; CHECK-LABEL: @test5(
+; CHECK: load i32, i32*
+; CHECK: ret
+  %a = bitcast i8* %x to i32**
+  %b = load i32*, i32** %a
+  %c = ptrtoint i32* %b to i32
+
+  ret i32 %c
+}
+
+define i64 @test6(i8* %x) {
+entry:
+; CHECK-LABEL: @test6(
+; CHECK: load i32, i32*
+; CHECK: zext
+; CHECK: ret
+  %a = bitcast i8* %x to i32**
+  %b = load i32*, i32** %a
+  %c = ptrtoint i32* %b to i64
+
+  ret i64 %c
+}
+
diff --git a/test/Transforms/InstCombine/load-bitcast64.ll b/test/Transforms/InstCombine/load-bitcast64.ll
new file mode 100644
index 00000000000..d14c686d83e
--- /dev/null
+++ b/test/Transforms/InstCombine/load-bitcast64.ll
@@ -0,0 +1,78 @@
+; RUN: opt -instcombine -S < %s | FileCheck %s
+
+target datalayout = "p:64:64:64"
+
+
+define i64* @test1(i8* %x) {
+entry:
+; CHECK-LABEL: @test1(
+; CHECK: load i64*, i64**
+; CHECK: ret
+  %a = bitcast i8* %x to i64*
+  %b = load i64, i64* %a
+  %c = inttoptr i64 %b to i64*
+
+  ret i64* %c
+}
+
+define i32* @test2(i8* %x) {
+entry:
+; CHECK-LABEL: @test2(
+; CHECK: load i32, i32*
+; CHECK: ret
+  %a = bitcast i8* %x to i32*
+  %b = load i32, i32* %a
+  %c = inttoptr i32 %b to i32*
+
+  ret i32* %c
+}
+
+define i64* @test3(i8* %x) {
+entry:
+; CHECK-LABEL: @test3(
+; CHECK: load i32, i32*
+; CHECK: ret
+  %a = bitcast i8* %x to i32*
+  %b = load i32, i32* %a
+  %c = inttoptr i32 %b to i64*
+
+  ret i64* %c
+}
+
+define i64 @test4(i8* %x) {
+entry:
+; CHECK-LABEL: @test4(
+; CHECK: load i64, i64*
+; CHECK: ret
+  %a = bitcast i8* %x to i64**
+  %b = load i64*, i64** %a
+  %c = ptrtoint i64* %b to i64
+
+  ret i64 %c
+}
+
+define i32 @test5(i8* %x) {
+entry:
+; CHECK-LABEL: @test5(
+; CHECK: load i64, i64*
+; CHECK: trunc
+; CHECK: ret
+  %a = bitcast i8* %x to i32**
+  %b = load i32*, i32** %a
+  %c = ptrtoint i32* %b to i32
+
+  ret i32 %c
+}
+
+define i64 @test6(i8* %x) {
+entry:
+; CHECK-LABEL: @test6(
+; CHECK: load i64, i64*
+; CHECK: ret
+  %a = bitcast i8* %x to i32**
+  %b = load i32*, i32** %a
+  %c = ptrtoint i32* %b to i64
+
+  ret i64 %c
+}
+
diff --git a/test/Transforms/InstCombine/load-combine-metadata.ll b/test/Transforms/InstCombine/load-combine-metadata.ll
new file mode 100644
index 00000000000..9b9c1fe607b
--- /dev/null
+++ b/test/Transforms/InstCombine/load-combine-metadata.ll
@@ -0,0 +1,29 @@
+; RUN: opt -instcombine -S < %s | FileCheck %s
+
+target datalayout = "e-m:e-p:64:64:64-i64:64-f80:128-n8:16:32:64-S128"
+
+; CHECK-LABEL: @test_load_load_combine_metadata(
+; Check that range and AA metadata is combined
+; CHECK: %[[V:.*]] = load i32, i32* %0
+; CHECK-SAME: !tbaa !{{[0-9]+}}
+; CHECK-SAME: !range ![[RANGE:[0-9]+]]
+; CHECK: store i32 %[[V]], i32* %1
+; CHECK: store i32 %[[V]], i32* %2
+define void @test_load_load_combine_metadata(i32*, i32*, i32*) {
+  %a = load i32, i32* %0, !tbaa !8, !range !0, !alias.scope !5, !noalias !6
+  %b = load i32, i32* %0, !tbaa !8, !range !1
+  store i32 %a, i32* %1
+  store i32 %b, i32* %2
+  ret void
+}
+
+; CHECK: ![[RANGE]] = !{i32 0, i32 1, i32 8, i32 9}
+!0 = !{ i32 0, i32 1 }
+!1 = !{ i32 8, i32 9 }
+!2 = !{!2}
+!3 = !{!3, !2}
+!4 = !{!4, !2}
+!5 = !{!3}
+!6 = !{!4}
+!7 = !{ !"tbaa root" }
+!8 = !{ !7, !7, i64 0 }
diff --git a/test/Transforms/InstCombine/load_combine_aa.ll b/test/Transforms/InstCombine/load_combine_aa.ll
new file mode 100644
index 00000000000..b84b81ddd5d
--- /dev/null
+++ b/test/Transforms/InstCombine/load_combine_aa.ll
@@ -0,0 +1,15 @@
+; RUN: opt -basicaa -instcombine -S < %s | FileCheck %s
+
+; CHECK-LABEL: @test_load_combine_aa(
+; CHECK: %[[V:.*]] = load i32, i32* %0
+; CHECK: store i32 0, i32* %3
+; CHECK: store i32 %[[V]], i32* %1
+; CHECK: store i32 %[[V]], i32* %2
+define void @test_load_combine_aa(i32*, i32*, i32*, i32* noalias) {
+  %a = load i32, i32* %0
+  store i32 0, i32* %3
+  %b = load i32, i32* %0
+  store i32 %a, i32* %1
+  store i32 %b, i32* %2
+  ret void
+}
diff --git a/test/Transforms/InstCombine/malloc-free-delete.ll b/test/Transforms/InstCombine/malloc-free-delete.ll
index dc04adb0696..138001ace95 100644
--- a/test/Transforms/InstCombine/malloc-free-delete.ll
+++ b/test/Transforms/InstCombine/malloc-free-delete.ll
@@ -127,7 +127,7 @@ declare i32 @__gxx_personality_v0(...)
 declare void @_ZN1AC2Ev(i8* %this)
 
 ; CHECK-LABEL: @test7(
-define void @test7() {
+define void @test7() personality i8* bitcast (i32 (...)* @__gxx_personality_v0 to i8*) {
 entry:
   %nt = alloca i8
   ; CHECK-NOT: call {{.*}}@_ZnwmRKSt9nothrow_t(
@@ -139,7 +139,7 @@ entry:
   unreachable
 
 lpad.i:                                           ; preds = %entry
-  %0 = landingpad { i8*, i32 } personality i8* bitcast (i32 (...)* @__gxx_personality_v0 to i8*) cleanup
+  %0 = landingpad { i8*, i32 } cleanup
   ; CHECK-NOT: call {{.*}}@_ZdlPvRKSt9nothrow_t(
   call void @_ZdlPvRKSt9nothrow_t(i8* %call.i, i8* %nt) builtin nounwind
   resume { i8*, i32 } %0
diff --git a/test/Transforms/InstCombine/max-of-nots.ll b/test/Transforms/InstCombine/max-of-nots.ll
index 41e3038cbe2..dd9c54b0673 100644
--- a/test/Transforms/InstCombine/max-of-nots.ll
+++ b/test/Transforms/InstCombine/max-of-nots.ll
@@ -66,3 +66,20 @@ define i32 @compute_min_pessimization(i32 %x, i32 %y) {
   %min = sub i32 -1, %not_min
   ret i32 %min
 }
+
+define i32 @max_of_nots(i32 %x, i32 %y) {
+; CHECK-LABEL: @max_of_nots(
+; CHECK-NEXT: icmp
+; CHECK-NEXT: select
+; CHECK-NEXT: icmp
+; CHECK-NEXT: select
+; CHECK-NEXT: xor
+; CHECK-NEXT: ret
+  %c0 = icmp sgt i32 %y, 0
+  %xor_y = xor i32 %y, -1
+  %s0 = select i1 %c0, i32 %xor_y, i32 -1
+  %xor_x = xor i32 %x, -1
+  %c1 = icmp slt i32 %s0, %xor_x
+  %smax96 = select i1 %c1, i32 %xor_x, i32 %s0
+  ret i32 %smax96
+}
diff --git a/test/Transforms/InstCombine/minmax-fold.ll b/test/Transforms/InstCombine/minmax-fold.ll
new file mode 100644
index 00000000000..4ed0c78216d
--- /dev/null
+++ b/test/Transforms/InstCombine/minmax-fold.ll
@@ -0,0 +1,112 @@
+; RUN: opt -S -instcombine < %s | FileCheck %s
+
+; CHECK-LABEL: @t1
+; CHECK-NEXT: icmp
+; CHECK-NEXT: select
+; CHECK-NEXT: sext
+define i64 @t1(i32 %a) {
+  ; This is the canonical form for a type-changing min/max.
+  %1 = icmp slt i32 %a, 5
+  %2 = select i1 %1, i32 %a, i32 5
+  %3 = sext i32 %2 to i64
+  ret i64 %3
+}
+
+; CHECK-LABEL: @t2
+; CHECK-NEXT: icmp
+; CHECK-NEXT: select
+; CHECK-NEXT: sext
+define i64 @t2(i32 %a) {
+  ; Check this is converted into canonical form, as above.
+  %1 = icmp slt i32 %a, 5
+  %2 = sext i32 %a to i64
+  %3 = select i1 %1, i64 %2, i64 5
+  ret i64 %3
+}
+
+; CHECK-LABEL: @t3
+; CHECK-NEXT: icmp
+; CHECK-NEXT: select
+; CHECK-NEXT: zext
+define i64 @t3(i32 %a) {
+  ; Same as @t2, with flipped operands and zext instead of sext.
+  %1 = icmp ult i32 %a, 5
+  %2 = zext i32 %a to i64
+  %3 = select i1 %1, i64 5, i64 %2
+  ret i64 %3
+}
+
+; CHECK-LABEL: @t4
+; CHECK-NEXT: icmp
+; CHECK-NEXT: select
+; CHECK-NEXT: trunc
+define i32 @t4(i64 %a) {
+  ; Same again, with trunc.
+  %1 = icmp slt i64 %a, 5
+  %2 = trunc i64 %a to i32
+  %3 = select i1 %1, i32 %2, i32 5
+  ret i32 %3
+}
+
+; CHECK-LABEL: @t5
+; CHECK-NEXT: icmp
+; CHECK-NEXT: zext
+; CHECK-NEXT: select
+define i64 @t5(i32 %a) {
+  ; Same as @t3, but with mismatched signedness between icmp and zext.
+  ; InstCombine should leave this alone.
+  %1 = icmp slt i32 %a, 5
+  %2 = zext i32 %a to i64
+  %3 = select i1 %1, i64 5, i64 %2
+  ret i64 %3
+}
+
+; CHECK-LABEL: @t6
+; CHECK-NEXT: icmp
+; CHECK-NEXT: select
+; CHECK-NEXT: sitofp
+define float @t6(i32 %a) {
+  %1 = icmp slt i32 %a, 0
+  %2 = select i1 %1, i32 %a, i32 0
+  %3 = sitofp i32 %2 to float
+  ret float %3
+}
+
+; CHECK-LABEL: @t7
+; CHECK-NEXT: icmp
+; CHECK-NEXT: select
+; CHECK-NEXT: trunc
+define i16 @t7(i32 %a) {
+  %1 = icmp slt i32 %a, -32768
+  %2 = trunc i32 %a to i16
+  %3 = select i1 %1, i16 %2, i16 -32768
+  ret i16 %3
+}
+
+; Just check for no infinite loop. InstSimplify liked to
+; "simplify" -32767 by removing all the sign bits,
+; which led to a canonicalization fight between different
+; parts of instcombine.
+define i32 @t8(i64 %a, i32 %b) {
+  %1 = icmp slt i64 %a, -32767
+  %2 = select i1 %1, i64 %a, i64 -32767
+  %3 = trunc i64 %2 to i32
+  %4 = icmp slt i32 %b, 42
+  %5 = select i1 %4, i32 42, i32 %3
+  %6 = icmp ne i32 %5, %b
+  %7 = zext i1 %6 to i32
+  ret i32 %7
+}
+
+; Ensure this doesn't get converted to a min/max.
+; CHECK-LABEL: @t9
+; CHECK-NEXT: icmp
+; CHECK-NEXT: sext
+; CHECK-NEXT: 4294967295
+; CHECK-NEXT: ret
+define i64 @t9(i32 %a) {
+  %1 = icmp sgt i32 %a, -1
+  %2 = sext i32 %a to i64
+  %3 = select i1 %1, i64 %2, i64 4294967295
+  ret i64 %3
+}
diff --git a/test/Transforms/InstCombine/mul.ll b/test/Transforms/InstCombine/mul.ll
index de09f129cdf..6e67c7fa08a 100644
--- a/test/Transforms/InstCombine/mul.ll
+++ b/test/Transforms/InstCombine/mul.ll
@@ -288,3 +288,19 @@ define i32 @test31(i32 %V) {
 ; CHECK:      %[[mul:.*]] = shl i32 %V, zext (i1 icmp ne (i32* inttoptr (i64 1 to i32*), i32* @PR22087) to i32)
 ; CHECK-NEXT: ret i32 %[[mul]]
 }
+
+define i32 @test32(i32 %X) {
+; CHECK-LABEL: @test32
+  %mul = mul nsw i32 %X, -2147483648
+  ret i32 %mul
+; CHECK:      %[[shl:.*]] = shl i32 %X, 31
+; CHECK-NEXT: ret i32 %[[shl]]
+}
+
+define i32 @test33(i32 %X) {
+; CHECK-LABEL: @test33
+  %mul = mul nsw i32 %X, 1073741824
+; CHECK:      %[[shl:.*]] = shl nsw i32 %X, 30
+; CHECK-NEXT: ret i32 %[[shl]]
+  ret i32 %mul
+}
diff --git a/test/Transforms/InstCombine/objsize-64.ll b/test/Transforms/InstCombine/objsize-64.ll
index 5046724038a..866bc4f937b 100644
--- a/test/Transforms/InstCombine/objsize-64.ll
+++ b/test/Transforms/InstCombine/objsize-64.ll
@@ -18,7 +18,7 @@ define i64 @f1(i8 **%esc) {
 
 
 ; CHECK-LABEL: @f2(
-define i64 @f2(i8** %esc) nounwind uwtable ssp {
+define i64 @f2(i8** %esc) nounwind uwtable ssp personality i8* bitcast (i32 (...)* @__gxx_personality_v0 to i8*) {
 entry:
 ; CHECK: invoke noalias i8* @_Znwm(i64 13)
   %call = invoke noalias i8* @_Znwm(i64 13)
@@ -31,7 +31,7 @@ invoke.cont:
   ret i64 %0
 
 lpad:
-  %1 = landingpad { i8*, i32 } personality i8* bitcast (i32 (...)* @__gxx_personality_v0 to i8*)
+  %1 = landingpad { i8*, i32 }
           filter [0 x i8*] zeroinitializer
   %2 = extractvalue { i8*, i32 } %1, 0
   tail call void @__cxa_call_unexpected(i8* %2) noreturn nounwind
diff --git a/test/Transforms/InstCombine/pr23751.ll b/test/Transforms/InstCombine/pr23751.ll
new file mode 100644
index 00000000000..d7840be2f83
--- /dev/null
+++ b/test/Transforms/InstCombine/pr23751.ll
@@ -0,0 +1,13 @@
+; RUN: opt -instcombine -S < %s | FileCheck %s
+
+@d = common global i32 0, align 4
+
+define i1 @f(i8 zeroext %p) #1 {
+; CHECK-NOT: ret i1 false
+  %1 = zext i8 %p to i32
+  %2 = load i32, i32* @d, align 4
+  %3 = or i32 %2, -2
+  %4 = add nsw i32 %3, %1
+  %5 = icmp ugt i32 %1, %4
+  ret i1 %5
+}
diff --git a/test/Transforms/InstCombine/pr23809.ll b/test/Transforms/InstCombine/pr23809.ll
new file mode 100644
index 00000000000..06c7ce20ba8
--- /dev/null
+++ b/test/Transforms/InstCombine/pr23809.ll
@@ -0,0 +1,22 @@
+; RUN: opt < %s -instcombine -S | FileCheck %s
+
+; InstCombine should preserve the call to @llvm.assume.
+define i32 @icmp(i32 %a, i32 %b) {
+; CHECK-LABEL: @icmp(
+  %sum = add i32 %a, %b
+  %1 = icmp sge i32 %sum, 0
+  call void @llvm.assume(i1 %1)
+; CHECK: call void @llvm.assume
+  ret i32 %sum
+}
+
+define float @fcmp(float %a, float %b) {
+; CHECK-LABEL: @fcmp(
+  %sum = fadd float %a, %b
+  %1 = fcmp oge float %sum, 0.0
+  call void @llvm.assume(i1 %1)
+; CHECK: call void @llvm.assume
+  ret float %sum
+}
+
+declare void @llvm.assume(i1)
diff --git a/test/Transforms/InstCombine/select.ll b/test/Transforms/InstCombine/select.ll
index ef122c92092..fdf1199a66f 100644
--- a/test/Transforms/InstCombine/select.ll
+++ b/test/Transforms/InstCombine/select.ll
@@ -1265,7 +1265,7 @@ define i32 @test77(i1 %flag, i32* %x) {
 ; load does.
 ; CHECK-LABEL: @test77(
 ; CHECK: %[[A:.*]] = alloca i32, align 1
-; CHECK: call void @scribble_on_i32(i32* %[[A]])
+; CHECK: call void @scribble_on_i32(i32* nonnull %[[A]])
 ; CHECK: store i32 0, i32* %x
 ; CHECK: %[[P:.*]] = select i1 %flag, i32* %[[A]], i32* %x
 ; CHECK: load i32, i32* %[[P]]
@@ -1296,6 +1296,23 @@ entry:
   ret i32 %v
 }
 
+define i32 @test78_neg(i1 %flag, i32* %x, i32* %y, i32* %z) {
+; The same as @test78 but we can't speculate the load because it can trap
+; if under-aligned.
+; CHECK-LABEL: @test78_neg(
+; CHECK: %p = select i1 %flag, i32* %x, i32* %y
+; CHECK-NEXT: %v = load i32, i32* %p, align 16
+; CHECK-NEXT: ret i32 %v
+entry:
+  store i32 0, i32* %x
+  store i32 0, i32* %y
+  ; Block forwarding by storing to %z which could alias either %x or %y.
+  store i32 42, i32* %z
+  %p = select i1 %flag, i32* %x, i32* %y
+  %v = load i32, i32* %p, align 16
+  ret i32 %v
+}
+
 define float @test79(i1 %flag, float* %x, i32* %y, i32* %z) {
 ; Test that we can speculate the loads around the select even when we can't
 ; fold the load completely away.
@@ -1520,3 +1537,28 @@ define i32 @test_select_select1(i32 %a, i32 %r0, i32 %r1, i32 %v1, i32 %v2) {
   %s1 = select i1 %c1, i32 %r0, i32 %s0
   ret i32 %s1
 }
+
+define i32 @test_max_of_min(i32 %a) {
+; MAX(MIN(%a, -1), -1) == -1
+; CHECK-LABEL: @test_max_of_min(
+; CHECK: ret i32 -1
+  %not_a = xor i32 %a, -1
+  %c0 = icmp sgt i32 %a, 0
+  %s0 = select i1 %c0, i32 %not_a, i32 -1
+  %c1 = icmp sgt i32 %s0, -1
+  %s1 = select i1 %c1, i32 %s0, i32 -1
+  ret i32 %s1
+}
+
+
+define i32 @PR23757(i32 %x) {
+; CHECK-LABEL: @PR23757
+; CHECK:      %[[cmp:.*]] = icmp eq i32 %x, 2147483647
+; CHECK-NEXT: %[[add:.*]] = add nsw i32 %x, 1
+; CHECK-NEXT: %[[sel:.*]] = select i1 %[[cmp]], i32 -2147483648, i32 %[[add]]
+; CHECK-NEXT: ret i32 %[[sel]]
+  %cmp = icmp eq i32 %x, 2147483647
+  %add = add nsw i32 %x, 1
+  %sel = select i1 %cmp, i32 -2147483648, i32 %add
+  ret i32 %sel
+}
diff --git a/test/Transforms/InstCombine/shift.ll b/test/Transforms/InstCombine/shift.ll
index 0b5b5deb68c..79c2ae28105 100644
--- a/test/Transforms/InstCombine/shift.ll
+++ b/test/Transforms/InstCombine/shift.ll
@@ -575,7 +575,7 @@ entry:
 ; CHECK: %0 = shl i8 %tmp4, 2
 ; CHECK: %tmp54 = and i8 %0, 16
   %tmp55 = xor i8 %tmp54, %tmp51
-; CHECK: ret i8 %tmp551
+; CHECK: ret i8 %tmp55.1
   ret i8 %tmp55
 }
 
@@ -743,7 +743,7 @@ define i32 @test57(i32 %x) {
   %or = or i32 %shl, 7
   ret i32 %or
 ; CHECK-LABEL: @test57(
-; CHECK: %shl = shl i32 %shr1, 4
+; CHECK: %shl = shl i32 %shr.1, 4
 }
 
 
diff --git a/test/Transforms/InstCombine/statepoint.ll b/test/Transforms/InstCombine/statepoint.ll
index ec991176da3..f904f207bfd 100644
--- a/test/Transforms/InstCombine/statepoint.ll
+++ b/test/Transforms/InstCombine/statepoint.ll
@@ -7,8 +7,8 @@ declare void @func()
 
 define i1 @test_negative(i32 addrspace(1)* %p) gc "statepoint-example" {
 entry:
-  %safepoint_token = tail call i32 (void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(void ()* @func, i32 0, i32 0, i32 0, i32 addrspace(1)* %p)
-  %pnew = call i32 addrspace(1)* @llvm.experimental.gc.relocate.p1i32(i32 %safepoint_token, i32 4, i32 4)
+  %safepoint_token = tail call i32 (i64, i32, void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 0, i32 0, void ()* @func, i32 0, i32 0, i32 0, i32 0, i32 addrspace(1)* %p)
+  %pnew = call i32 addrspace(1)* @llvm.experimental.gc.relocate.p1i32(i32 %safepoint_token,  i32 7, i32 7)
   %cmp = icmp eq i32 addrspace(1)* %pnew, null
   ret i1 %cmp
 ; CHECK-LABEL: test_negative
@@ -18,8 +18,8 @@ entry:
 
 define i1 @test_nonnull(i32 addrspace(1)* nonnull %p) gc "statepoint-example" {
 entry:
-  %safepoint_token = tail call i32 (void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(void ()* @func, i32 0, i32 0, i32 0, i32 addrspace(1)* %p)
-  %pnew = call i32 addrspace(1)* @llvm.experimental.gc.relocate.p1i32(i32 %safepoint_token, i32 4, i32 4)
+  %safepoint_token = tail call i32 (i64, i32, void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 0, i32 0, void ()* @func, i32 0, i32 0, i32 0, i32 0, i32 addrspace(1)* %p)
+  %pnew = call i32 addrspace(1)* @llvm.experimental.gc.relocate.p1i32(i32 %safepoint_token,  i32 7, i32 7)
   %cmp = icmp eq i32 addrspace(1)* %pnew, null
   ret i1 %cmp
 ; CHECK-LABEL: test_nonnull
@@ -28,8 +28,8 @@ entry:
 
 define i1 @test_null() gc "statepoint-example" {
 entry:
-  %safepoint_token = tail call i32 (void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(void ()* @func, i32 0, i32 0, i32 0, i32 addrspace(1)* null)
-  %pnew = call i32 addrspace(1)* @llvm.experimental.gc.relocate.p1i32(i32 %safepoint_token, i32 4, i32 4)
+  %safepoint_token = tail call i32 (i64, i32, void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 0, i32 0, void ()* @func, i32 0, i32 0, i32 0, i32 0, i32 addrspace(1)* null)
+  %pnew = call i32 addrspace(1)* @llvm.experimental.gc.relocate.p1i32(i32 %safepoint_token,  i32 7, i32 7)
   %cmp = icmp eq i32 addrspace(1)* %pnew, null
   ret i1 %cmp
 ; CHECK-LABEL: test_null
@@ -39,8 +39,8 @@ entry:
 
 define i1 @test_undef() gc "statepoint-example" {
 entry:
-  %safepoint_token = tail call i32 (void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(void ()* @func, i32 0, i32 0, i32 0, i32 addrspace(1)* undef)
-  %pnew = call i32 addrspace(1)* @llvm.experimental.gc.relocate.p1i32(i32 %safepoint_token, i32 4, i32 4)
+  %safepoint_token = tail call i32 (i64, i32, void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 0, i32 0, void ()* @func, i32 0, i32 0, i32 0, i32 0, i32 addrspace(1)* undef)
+  %pnew = call i32 addrspace(1)* @llvm.experimental.gc.relocate.p1i32(i32 %safepoint_token,  i32 7, i32 7)
   %cmp = icmp eq i32 addrspace(1)* %pnew, null
   ret i1 %cmp
 ; CHECK-LABEL: test_undef
@@ -48,5 +48,5 @@ entry:
 ; CHECK: ret i1 undef
 }
 
-declare i32 @llvm.experimental.gc.statepoint.p0f_isVoidf(void ()*, i32, i32, ...)
+declare i32 @llvm.experimental.gc.statepoint.p0f_isVoidf(i64, i32, void ()*, i32, i32, ...)
 declare i32 addrspace(1)* @llvm.experimental.gc.relocate.p1i32(i32, i32, i32) #3
diff --git a/test/Transforms/InstCombine/sub.ll b/test/Transforms/InstCombine/sub.ll
index c76d8d0e220..b1c7b7245cf 100644
--- a/test/Transforms/InstCombine/sub.ll
+++ b/test/Transforms/InstCombine/sub.ll
@@ -550,3 +550,25 @@ define i32 @test46(i32 %x, i32 %y) {
 ; CHECK-NEXT: %sub = and i32 %y, %x.not
 ; CHECK: ret i32 %sub
 }
+
+define i32 @test47(i1 %A, i32 %B, i32 %C, i32 %D) {
+  %sel0 = select i1 %A, i32 %D, i32 %B
+  %sel1 = select i1 %A, i32 %C, i32 %B
+  %sub = sub i32 %sel0, %sel1
+  ret i32 %sub
+; CHECK-LABEL: @test47(
+; CHECK-NEXT: %[[sub:.*]] = sub i32 %D, %C
+; CHECK-NEXT: %[[sel:.*]] = select i1 %A, i32 %[[sub]], i32 0
+; CHECK-NEXT: ret i32 %[[sel]]
+}
+
+define i32 @test48(i1 %A, i32 %B, i32 %C, i32 %D) {
+  %sel0 = select i1 %A, i32 %B, i32 %D
+  %sel1 = select i1 %A, i32 %B, i32 %C
+  %sub = sub i32 %sel0, %sel1
+  ret i32 %sub
+; CHECK-LABEL: @test48(
+; CHECK-NEXT: %[[sub:.*]] = sub i32 %D, %C
+; CHECK-NEXT: %[[sel:.*]] = select i1 %A, i32 0, i32 %[[sub]]
+; CHECK-NEXT: ret i32 %[[sel]]
+}
diff --git a/test/Transforms/InstCombine/unpack-fca.ll b/test/Transforms/InstCombine/unpack-fca.ll
index 5f63330b931..48bb157956a 100644
--- a/test/Transforms/InstCombine/unpack-fca.ll
+++ b/test/Transforms/InstCombine/unpack-fca.ll
@@ -12,20 +12,103 @@ declare i32 @A.foo(%A* nocapture %this)
 
 declare i8* @allocmemory(i64)
 
-define void @structA() {
+define void @storeA() {
 body:
   %0 = tail call i8* @allocmemory(i64 32)
   %1 = bitcast i8* %0 to %A*
+; CHECK-LABEL: storeA
 ; CHECK: store %A__vtbl* @A__vtblZ
   store %A { %A__vtbl* @A__vtblZ }, %A* %1, align 8
   ret void
 }
 
-define void @structOfA() {
+define void @storeStructOfA() {
 body:
   %0 = tail call i8* @allocmemory(i64 32)
   %1 = bitcast i8* %0 to { %A }*
+; CHECK-LABEL: storeStructOfA
 ; CHECK: store %A__vtbl* @A__vtblZ
   store { %A } { %A { %A__vtbl* @A__vtblZ } }, { %A }* %1, align 8
   ret void
 }
+
+define void @storeArrayOfA() {
+body:
+  %0 = tail call i8* @allocmemory(i64 32)
+  %1 = bitcast i8* %0 to [1 x %A]*
+; CHECK-LABEL: storeArrayOfA
+; CHECK: store %A__vtbl* @A__vtblZ
+  store [1 x %A] [%A { %A__vtbl* @A__vtblZ }], [1 x %A]* %1, align 8
+  ret void
+}
+
+define void @storeStructOfArrayOfA() {
+body:
+  %0 = tail call i8* @allocmemory(i64 32)
+  %1 = bitcast i8* %0 to { [1 x %A] }*
+; CHECK-LABEL: storeStructOfArrayOfA
+; CHECK: store %A__vtbl* @A__vtblZ
+  store { [1 x %A] } { [1 x %A] [%A { %A__vtbl* @A__vtblZ }] }, { [1 x %A] }* %1, align 8
+  ret void
+}
+
+define %A @loadA() {
+body:
+  %0 = tail call i8* @allocmemory(i64 32)
+  %1 = bitcast i8* %0 to %A*
+; CHECK-LABEL: loadA
+; CHECK: load %A__vtbl*,
+; CHECK: insertvalue %A undef, %A__vtbl* {{.*}}, 0
+  %2 = load %A, %A* %1, align 8
+  ret %A %2
+}
+
+define { %A } @loadStructOfA() {
+body:
+  %0 = tail call i8* @allocmemory(i64 32)
+  %1 = bitcast i8* %0 to { %A }*
+; CHECK-LABEL: loadStructOfA
+; CHECK: load %A__vtbl*,
+; CHECK: insertvalue %A undef, %A__vtbl* {{.*}}, 0
+; CHECK: insertvalue { %A } undef, %A {{.*}}, 0
+  %2 = load { %A }, { %A }* %1, align 8
+  ret { %A } %2
+}
+
+define [1 x %A] @loadArrayOfA() {
+body:
+  %0 = tail call i8* @allocmemory(i64 32)
+  %1 = bitcast i8* %0 to [1 x %A]*
+; CHECK-LABEL: loadArrayOfA
+; CHECK: load %A__vtbl*,
+; CHECK: insertvalue %A undef, %A__vtbl* {{.*}}, 0
+; CHECK: insertvalue [1 x %A] undef, %A {{.*}}, 0
+  %2 = load [1 x %A], [1 x %A]* %1, align 8
+  ret [1 x %A] %2
+}
+
+define { [1 x %A] } @loadStructOfArrayOfA() {
+body:
+  %0 = tail call i8* @allocmemory(i64 32)
+  %1 = bitcast i8* %0 to { [1 x %A] }*
+; CHECK-LABEL: loadStructOfArrayOfA
+; CHECK: load %A__vtbl*,
+; CHECK: insertvalue %A undef, %A__vtbl* {{.*}}, 0
+; CHECK: insertvalue [1 x %A] undef, %A {{.*}}, 0
+; CHECK: insertvalue { [1 x %A] } undef, [1 x %A] {{.*}}, 0
+  %2 = load { [1 x %A] }, { [1 x %A] }* %1, align 8
+  ret { [1 x %A] } %2
+}
+
+define { %A } @structOfA() {
+body:
+  %0 = tail call i8* @allocmemory(i64 32)
+  %1 = bitcast i8* %0 to { %A }*
+; CHECK-LABEL: structOfA
+; CHECK: store %A__vtbl* @A__vtblZ
+  store { %A } { %A { %A__vtbl* @A__vtblZ } }, { %A }* %1, align 8
+  %2 = load { %A }, { %A }* %1, align 8
+; CHECK-NOT: load
+; CHECK: ret { %A } { %A { %A__vtbl* @A__vtblZ } }
+  ret { %A } %2
+}
diff --git a/test/Transforms/InstCombine/vec_demanded_elts.ll b/test/Transforms/InstCombine/vec_demanded_elts.ll
index 4245c7a3c13..dabbeb05a34 100644
--- a/test/Transforms/InstCombine/vec_demanded_elts.ll
+++ b/test/Transforms/InstCombine/vec_demanded_elts.ll
@@ -138,22 +138,6 @@ declare i64 @llvm.x86.sse2.cvtsd2si64(<2 x double>)
 declare i32 @llvm.x86.sse2.cvttsd2si(<2 x double>)
 declare i64 @llvm.x86.sse2.cvttsd2si64(<2 x double>)
 
-; <rdar://problem/6945110>
-define <4 x i32> @kernel3_vertical(<4 x i16> * %src, <8 x i16> * %foo) nounwind {
-entry:
-	%tmp = load <4 x i16>, <4 x i16>* %src
-	%tmp1 = load <8 x i16>, <8 x i16>* %foo
-; CHECK: %tmp2 = shufflevector
-	%tmp2 = shufflevector <4 x i16> %tmp, <4 x i16> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef>
-; pmovzxwd ignores the upper 64-bits of its input; -instcombine should remove this shuffle:
-; CHECK-NOT: shufflevector
-	%tmp3 = shufflevector <8 x i16> %tmp1, <8 x i16> %tmp2, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 4, i32 5, i32 6, i32 7>
-; CHECK-NEXT: pmovzxwd
-	%0 = call <4 x i32> @llvm.x86.sse41.pmovzxwd(<8 x i16> %tmp3)
-	ret <4 x i32> %0
-}
-declare <4 x i32> @llvm.x86.sse41.pmovzxwd(<8 x i16>) nounwind readnone
-
 define <4 x float> @dead_shuffle_elt(<4 x float> %x, <2 x float> %y) nounwind {
 entry:
 ; CHECK-LABEL: define <4 x float> @dead_shuffle_elt(
@@ -210,130 +194,6 @@ define <4 x float> @test_select(float %f, float %g) {
   ret <4 x float> %ret
 }
 
-; We should optimize these two redundant insertqi into one
-; CHECK: define <2 x i64> @testInsertTwice(<2 x i64> %v, <2 x i64> %i)
-define <2 x i64> @testInsertTwice(<2 x i64> %v, <2 x i64> %i) {
-; CHECK: call <2 x i64> @llvm.x86.sse4a.insertqi(<2 x i64> %v, <2 x i64> %i, i8 32, i8 32)
-; CHECK-NOT: insertqi
-  %1 = tail call <2 x i64> @llvm.x86.sse4a.insertqi(<2 x i64> %v, <2 x i64> %i, i8 32, i8 32)
-  %2 = tail call <2 x i64> @llvm.x86.sse4a.insertqi(<2 x i64> %1, <2 x i64> %i, i8 32, i8 32)
-  ret <2 x i64> %2
-}
-
-; The result of this insert is the second arg, since the top 64 bits of
-; the result are undefined, and we copy the bottom 64 bits from the
-; second arg
-; CHECK: define <2 x i64> @testInsert64Bits(<2 x i64> %v, <2 x i64> %i)
-define <2 x i64> @testInsert64Bits(<2 x i64> %v, <2 x i64> %i) {
-; CHECK: ret <2 x i64> %i
-  %1 = tail call <2 x i64> @llvm.x86.sse4a.insertqi(<2 x i64> %v, <2 x i64> %i, i8 64, i8 0)
-  ret <2 x i64> %1
-}
-
-; Test the several types of ranges and ordering that exist for two insertqi
-; CHECK: define <2 x i64> @testInsertContainedRange(<2 x i64> %v, <2 x i64> %i)
-define <2 x i64> @testInsertContainedRange(<2 x i64> %v, <2 x i64> %i) {
-; CHECK: %[[RES:.*]] = call <2 x i64> @llvm.x86.sse4a.insertqi(<2 x i64> %v, <2 x i64> %i, i8 32, i8 0)
-; CHECK: ret <2 x i64> %[[RES]]
-  %1 = tail call <2 x i64> @llvm.x86.sse4a.insertqi(<2 x i64> %v, <2 x i64> %i, i8 32, i8 0)
-  %2 = tail call <2 x i64> @llvm.x86.sse4a.insertqi(<2 x i64> %1, <2 x i64> %i, i8 16, i8 16)
-  ret <2 x i64> %2
-}
-
-; CHECK: define <2 x i64> @testInsertContainedRange_2(<2 x i64> %v, <2 x i64> %i)
-define <2 x i64> @testInsertContainedRange_2(<2 x i64> %v, <2 x i64> %i) {
-; CHECK: %[[RES:.*]] = call <2 x i64> @llvm.x86.sse4a.insertqi(<2 x i64> %v, <2 x i64> %i, i8 32, i8 0)
-; CHECK: ret <2 x i64> %[[RES]]
-  %1 = tail call <2 x i64> @llvm.x86.sse4a.insertqi(<2 x i64> %v, <2 x i64> %i, i8 16, i8 16)
-  %2 = tail call <2 x i64> @llvm.x86.sse4a.insertqi(<2 x i64> %1, <2 x i64> %i, i8 32, i8 0)
-  ret <2 x i64> %2
-}
-
-; CHECK: define <2 x i64> @testInsertOverlappingRange(<2 x i64> %v, <2 x i64> %i)
-define <2 x i64> @testInsertOverlappingRange(<2 x i64> %v, <2 x i64> %i) {
-; CHECK: %[[RES:.*]] = call <2 x i64> @llvm.x86.sse4a.insertqi(<2 x i64> %v, <2 x i64> %i, i8 48, i8 0)
-; CHECK: ret <2 x i64> %[[RES]]
-  %1 = tail call <2 x i64> @llvm.x86.sse4a.insertqi(<2 x i64> %v, <2 x i64> %i, i8 32, i8 0)
-  %2 = tail call <2 x i64> @llvm.x86.sse4a.insertqi(<2 x i64> %1, <2 x i64> %i, i8 32, i8 16)
-  ret <2 x i64> %2
-}
-
-; CHECK: define <2 x i64> @testInsertOverlappingRange_2(<2 x i64> %v, <2 x i64> %i)
-define <2 x i64> @testInsertOverlappingRange_2(<2 x i64> %v, <2 x i64> %i) {
-; CHECK: %[[RES:.*]] = call <2 x i64> @llvm.x86.sse4a.insertqi(<2 x i64> %v, <2 x i64> %i, i8 48, i8 0)
-; CHECK: ret <2 x i64> %[[RES]]
-  %1 = tail call <2 x i64> @llvm.x86.sse4a.insertqi(<2 x i64> %v, <2 x i64> %i, i8 32, i8 16)
-  %2 = tail call <2 x i64> @llvm.x86.sse4a.insertqi(<2 x i64> %1, <2 x i64> %i, i8 32, i8 0)
-  ret <2 x i64> %2
-}
-
-; CHECK: define <2 x i64> @testInsertAdjacentRange(<2 x i64> %v, <2 x i64> %i)
-define <2 x i64> @testInsertAdjacentRange(<2 x i64> %v, <2 x i64> %i) {
-; CHECK: %[[RES:.*]] = call <2 x i64> @llvm.x86.sse4a.insertqi(<2 x i64> %v, <2 x i64> %i, i8 48, i8 0)
-; CHECK: ret <2 x i64> %[[RES]]
-  %1 = tail call <2 x i64> @llvm.x86.sse4a.insertqi(<2 x i64> %v, <2 x i64> %i, i8 32, i8 0)
-  %2 = tail call <2 x i64> @llvm.x86.sse4a.insertqi(<2 x i64> %1, <2 x i64> %i, i8 16, i8 32)
-  ret <2 x i64> %2
-}
-
-; CHECK: define <2 x i64> @testInsertAdjacentRange_2(<2 x i64> %v, <2 x i64> %i)
-define <2 x i64> @testInsertAdjacentRange_2(<2 x i64> %v, <2 x i64> %i) {
-; CHECK: %[[RES:.*]] = call <2 x i64> @llvm.x86.sse4a.insertqi(<2 x i64> %v, <2 x i64> %i, i8 48, i8 0)
-; CHECK: ret <2 x i64> %[[RES]]
-  %1 = tail call <2 x i64> @llvm.x86.sse4a.insertqi(<2 x i64> %v, <2 x i64> %i, i8 16, i8 32)
-  %2 = tail call <2 x i64> @llvm.x86.sse4a.insertqi(<2 x i64> %1, <2 x i64> %i, i8 32, i8 0)
-  ret <2 x i64> %2
-}
-
-; CHECK: define <2 x i64> @testInsertDisjointRange(<2 x i64> %v, <2 x i64> %i)
-define <2 x i64> @testInsertDisjointRange(<2 x i64> %v, <2 x i64> %i) {
-; CHECK: tail call <2 x i64> @llvm.x86.sse4a.insertqi(<2 x i64> %v, <2 x i64> %i, i8 16, i8 0)
-; CHECK: tail call <2 x i64> @llvm.x86.sse4a.insertqi(<2 x i64> %1, <2 x i64> %i, i8 16, i8 32)
-  %1 = tail call <2 x i64> @llvm.x86.sse4a.insertqi(<2 x i64> %v, <2 x i64> %i, i8 16, i8 0)
-  %2 = tail call <2 x i64> @llvm.x86.sse4a.insertqi(<2 x i64> %1, <2 x i64> %i, i8 16, i8 32)
-  ret <2 x i64> %2
-}
-
-; CHECK: define <2 x i64> @testInsertDisjointRange_2(<2 x i64> %v, <2 x i64> %i)
-define <2 x i64> @testInsertDisjointRange_2(<2 x i64> %v, <2 x i64> %i) {
-; CHECK:  tail call <2 x i64> @llvm.x86.sse4a.insertqi(<2 x i64> %v, <2 x i64> %i, i8 16, i8 0)
-; CHECK:  tail call <2 x i64> @llvm.x86.sse4a.insertqi(<2 x i64> %1, <2 x i64> %i, i8 16, i8 32)
-  %1 = tail call <2 x i64> @llvm.x86.sse4a.insertqi(<2 x i64> %v, <2 x i64> %i, i8 16, i8 0)
-  %2 = tail call <2 x i64> @llvm.x86.sse4a.insertqi(<2 x i64> %1, <2 x i64> %i, i8 16, i8 32)
-  ret <2 x i64> %2
-}
-
-; CHECK: define <2 x i64> @testZeroLength(<2 x i64> %v, <2 x i64> %i)
-define <2 x i64> @testZeroLength(<2 x i64> %v, <2 x i64> %i) {
-; CHECK: ret <2 x i64> %i
-  %1 = tail call <2 x i64> @llvm.x86.sse4a.insertqi(<2 x i64> %v, <2 x i64> %i, i8 0, i8 0)
-  ret <2 x i64> %1
-}
-
-; CHECK: define <2 x i64> @testUndefinedInsertq_1(<2 x i64> %v, <2 x i64> %i)
-define <2 x i64> @testUndefinedInsertq_1(<2 x i64> %v, <2 x i64> %i) {
-; CHECK: ret <2 x i64> undef
-  %1 = tail call <2 x i64> @llvm.x86.sse4a.insertqi(<2 x i64> %v, <2 x i64> %i, i8 0, i8 16)
-  ret <2 x i64> %1
-}
-
-; CHECK: define <2 x i64> @testUndefinedInsertq_2(<2 x i64> %v, <2 x i64> %i)
-define <2 x i64> @testUndefinedInsertq_2(<2 x i64> %v, <2 x i64> %i) {
-; CHECK: ret <2 x i64> undef
-  %1 = tail call <2 x i64> @llvm.x86.sse4a.insertqi(<2 x i64> %v, <2 x i64> %i, i8 48, i8 32)
-  ret <2 x i64> %1
-}
-
-; CHECK: define <2 x i64> @testUndefinedInsertq_3(<2 x i64> %v, <2 x i64> %i)
-define <2 x i64> @testUndefinedInsertq_3(<2 x i64> %v, <2 x i64> %i) {
-; CHECK: ret <2 x i64> undef
-  %1 = tail call <2 x i64> @llvm.x86.sse4a.insertqi(<2 x i64> %v, <2 x i64> %i, i8 64, i8 16)
-  ret <2 x i64> %1
-}
-
-; CHECK: declare <2 x i64> @llvm.x86.sse4a.insertqi
-declare <2 x i64> @llvm.x86.sse4a.insertqi(<2 x i64>, <2 x i64>, i8, i8) nounwind
-
 declare <4 x float> @llvm.x86.avx.vpermilvar.ps(<4 x float>, <4 x i32>)
 define <4 x float> @test_vpermilvar_ps(<4 x float> %v) {
 ; CHECK-LABEL: @test_vpermilvar_ps(
diff --git a/test/Transforms/InstCombine/x86-insertps.ll b/test/Transforms/InstCombine/x86-insertps.ll
index 487c7274ca1..092bf7c6c62 100644
--- a/test/Transforms/InstCombine/x86-insertps.ll
+++ b/test/Transforms/InstCombine/x86-insertps.ll
@@ -30,14 +30,47 @@ define <4 x float> @insertps_0xff(<4 x float> %v1, <4 x float> %v2) {
 ; CHECK-NEXT:  ret <4 x float> zeroinitializer
 }
 
-; If some zero mask bits are set, we do not change anything.
+; If some zero mask bits are set that do not override the insertion, we do not change anything.
 
-define <4 x float> @insertps_0x03(<4 x float> %v1, <4 x float> %v2) {
-  %res = call <4 x float> @llvm.x86.sse41.insertps(<4 x float> %v1, <4 x float> %v2, i8 3)
+define <4 x float> @insertps_0x0c(<4 x float> %v1, <4 x float> %v2) {
+  %res = call <4 x float> @llvm.x86.sse41.insertps(<4 x float> %v1, <4 x float> %v2, i8 12)
   ret <4 x float> %res
 
-; CHECK-LABEL: @insertps_0x03
-; CHECK-NEXT:  call <4 x float> @llvm.x86.sse41.insertps(<4 x float> %v1, <4 x float> %v2, i8 3)
+; CHECK-LABEL: @insertps_0x0c
+; CHECK-NEXT:  call <4 x float> @llvm.x86.sse41.insertps(<4 x float> %v1, <4 x float> %v2, i8 12)
+; CHECK-NEXT:  ret <4 x float>
+}
+
+; ...unless both input vectors are the same operand.
+
+define <4 x float> @insertps_0x15_single_input(<4 x float> %v1) {
+  %res = call <4 x float> @llvm.x86.sse41.insertps(<4 x float> %v1, <4 x float> %v1, i8 21)
+  ret <4 x float> %res
+
+; CHECK-LABEL: @insertps_0x15_single_input
+; CHECK-NEXT:  shufflevector <4 x float> %v1, <4 x float> <float 0.000000e+00, float undef, float 0.000000e+00, float undef>, <4 x i32> <i32 4, i32 0, i32 6, i32 3>
+; CHECK-NEXT:  ret <4 x float>
+}
+
+; The zero mask overrides the insertion lane.
+
+define <4 x float> @insertps_0x1a_single_input(<4 x float> %v1) {
+  %res = call <4 x float> @llvm.x86.sse41.insertps(<4 x float> %v1, <4 x float> %v1, i8 26)
+  ret <4 x float> %res
+
+; CHECK-LABEL: @insertps_0x1a_single_input
+; CHECK-NEXT:  shufflevector <4 x float> %v1, <4 x float> <float undef, float 0.000000e+00, float undef, float 0.000000e+00>, <4 x i32> <i32 0, i32 5, i32 2, i32 7>
+; CHECK-NEXT:  ret <4 x float>
+}
+
+; The zero mask overrides the insertion lane, so the second input vector is not used.
+
+define <4 x float> @insertps_0xc1(<4 x float> %v1, <4 x float> %v2) {
+  %res = call <4 x float> @llvm.x86.sse41.insertps(<4 x float> %v1, <4 x float> %v2, i8 193)
+  ret <4 x float> %res
+
+; CHECK-LABEL: @insertps_0xc1
+; CHECK-NEXT:  shufflevector <4 x float> %v1, <4 x float> <float 0.000000e+00, float undef, float undef, float undef>, <4 x i32> <i32 4, i32 1, i32 2, i32 3>
 ; CHECK-NEXT:  ret <4 x float>
 }
 
diff --git a/test/Transforms/InstCombine/x86-pmovsx.ll b/test/Transforms/InstCombine/x86-pmovsx.ll
new file mode 100644
index 00000000000..31bdc59b16a
--- /dev/null
+++ b/test/Transforms/InstCombine/x86-pmovsx.ll
@@ -0,0 +1,136 @@
+; RUN: opt < %s -instcombine -S | FileCheck %s
+
+declare <4 x i32>  @llvm.x86.sse41.pmovsxbd(<16 x i8>) nounwind readnone
+declare <2 x i64>  @llvm.x86.sse41.pmovsxbq(<16 x i8>) nounwind readnone
+declare <8 x i16>  @llvm.x86.sse41.pmovsxbw(<16 x i8>) nounwind readnone
+declare <2 x i64>  @llvm.x86.sse41.pmovsxdq(<4 x i32>) nounwind readnone
+declare <4 x i32>  @llvm.x86.sse41.pmovsxwd(<8 x i16>) nounwind readnone
+declare <2 x i64>  @llvm.x86.sse41.pmovsxwq(<8 x i16>) nounwind readnone
+
+declare <8 x i32>  @llvm.x86.avx2.pmovsxbd(<16 x i8>) nounwind readnone
+declare <4 x i64>  @llvm.x86.avx2.pmovsxbq(<16 x i8>) nounwind readnone
+declare <16 x i16> @llvm.x86.avx2.pmovsxbw(<16 x i8>) nounwind readnone
+declare <4 x i64>  @llvm.x86.avx2.pmovsxdq(<4 x i32>) nounwind readnone
+declare <8 x i32>  @llvm.x86.avx2.pmovsxwd(<8 x i16>) nounwind readnone
+declare <4 x i64>  @llvm.x86.avx2.pmovsxwq(<8 x i16>) nounwind readnone
+
+;
+; Basic sign extension tests
+;
+
+define <4 x i32> @sse41_pmovsxbd(<16 x i8> %v) nounwind readnone {
+; CHECK-LABEL: @sse41_pmovsxbd
+; CHECK-NEXT:  shufflevector <16 x i8> %v, <16 x i8> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:  sext <4 x i8> %1 to <4 x i32>
+; CHECK-NEXT:  ret <4 x i32> %2
+
+  %res = call <4 x i32> @llvm.x86.sse41.pmovsxbd(<16 x i8> %v)
+  ret <4 x i32> %res
+}
+
+define <2 x i64> @sse41_pmovsxbq(<16 x i8> %v) nounwind readnone {
+; CHECK-LABEL: @sse41_pmovsxbq
+; CHECK-NEXT:  shufflevector <16 x i8> %v, <16 x i8> undef, <2 x i32> <i32 0, i32 1>
+; CHECK-NEXT:  sext <2 x i8> %1 to <2 x i64>
+; CHECK-NEXT:  ret <2 x i64> %2
+
+  %res = call <2 x i64> @llvm.x86.sse41.pmovsxbq(<16 x i8> %v)
+  ret <2 x i64> %res
+}
+
+define <8 x i16> @sse41_pmovsxbw(<16 x i8> %v) nounwind readnone {
+; CHECK-LABEL: @sse41_pmovsxbw
+; CHECK-NEXT:  shufflevector <16 x i8> %v, <16 x i8> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; CHECK-NEXT:  sext <8 x i8> %1 to <8 x i16>
+; CHECK-NEXT:  ret <8 x i16> %2
+
+  %res = call <8 x i16> @llvm.x86.sse41.pmovsxbw(<16 x i8> %v)
+  ret <8 x i16> %res
+}
+
+define <2 x i64> @sse41_pmovsxdq(<4 x i32> %v) nounwind readnone {
+; CHECK-LABEL: @sse41_pmovsxdq
+; CHECK-NEXT:  shufflevector <4 x i32> %v, <4 x i32> undef, <2 x i32> <i32 0, i32 1>
+; CHECK-NEXT:  sext <2 x i32> %1 to <2 x i64>
+; CHECK-NEXT:  ret <2 x i64> %2
+
+  %res = call <2 x i64> @llvm.x86.sse41.pmovsxdq(<4 x i32> %v)
+  ret <2 x i64> %res
+}
+
+define <4 x i32> @sse41_pmovsxwd(<8 x i16> %v) nounwind readnone {
+; CHECK-LABEL: @sse41_pmovsxwd
+; CHECK-NEXT:  shufflevector <8 x i16> %v, <8 x i16> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:  sext <4 x i16> %1 to <4 x i32>
+; CHECK-NEXT:  ret <4 x i32> %2
+
+  %res = call <4 x i32> @llvm.x86.sse41.pmovsxwd(<8 x i16> %v)
+  ret <4 x i32> %res
+}
+
+define <2 x i64> @sse41_pmovsxwq(<8 x i16> %v) nounwind readnone {
+; CHECK-LABEL: @sse41_pmovsxwq
+; CHECK-NEXT:  shufflevector <8 x i16> %v, <8 x i16> undef, <2 x i32> <i32 0, i32 1>
+; CHECK-NEXT:  sext <2 x i16> %1 to <2 x i64>
+; CHECK-NEXT:  ret <2 x i64> %2
+
+  %res = call <2 x i64> @llvm.x86.sse41.pmovsxwq(<8 x i16> %v)
+  ret <2 x i64> %res
+}
+
+define <8 x i32> @avx2_pmovsxbd(<16 x i8> %v) nounwind readnone {
+; CHECK-LABEL: @avx2_pmovsxbd
+; CHECK-NEXT:  shufflevector <16 x i8> %v, <16 x i8> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; CHECK-NEXT:  sext <8 x i8> %1 to <8 x i32>
+; CHECK-NEXT:  ret <8 x i32> %2
+
+  %res = call <8 x i32> @llvm.x86.avx2.pmovsxbd(<16 x i8> %v)
+  ret <8 x i32> %res
+}
+
+define <4 x i64> @avx2_pmovsxbq(<16 x i8> %v) nounwind readnone {
+; CHECK-LABEL: @avx2_pmovsxbq
+; CHECK-NEXT:  shufflevector <16 x i8> %v, <16 x i8> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:  sext <4 x i8> %1 to <4 x i64>
+; CHECK-NEXT:  ret <4 x i64> %2
+
+  %res = call <4 x i64> @llvm.x86.avx2.pmovsxbq(<16 x i8> %v)
+  ret <4 x i64> %res
+}
+
+define <16 x i16> @avx2_pmovsxbw(<16 x i8> %v) nounwind readnone {
+; CHECK-LABEL: @avx2_pmovsxbw
+; CHECK-NEXT:  sext <16 x i8> %v to <16 x i16>
+; CHECK-NEXT:  ret <16 x i16> %1
+
+  %res = call <16 x i16> @llvm.x86.avx2.pmovsxbw(<16 x i8> %v)
+  ret <16 x i16> %res
+}
+
+define <4 x i64> @avx2_pmovsxdq(<4 x i32> %v) nounwind readnone {
+; CHECK-LABEL: @avx2_pmovsxdq
+; CHECK-NEXT:  sext <4 x i32> %v to <4 x i64>
+; CHECK-NEXT:  ret <4 x i64> %1
+
+  %res = call <4 x i64> @llvm.x86.avx2.pmovsxdq(<4 x i32> %v)
+  ret <4 x i64> %res
+}
+
+define <8 x i32> @avx2_pmovsxwd(<8 x i16> %v) nounwind readnone {
+; CHECK-LABEL: @avx2_pmovsxwd
+; CHECK-NEXT:  sext <8 x i16> %v to <8 x i32>
+; CHECK-NEXT:  ret <8 x i32> %1
+
+  %res = call <8 x i32> @llvm.x86.avx2.pmovsxwd(<8 x i16> %v)
+  ret <8 x i32> %res
+}
+
+define <4 x i64> @avx2_pmovsxwq(<8 x i16> %v) nounwind readnone {
+; CHECK-LABEL: @avx2_pmovsxwq
+; CHECK-NEXT:  shufflevector <8 x i16> %v, <8 x i16> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:  sext <4 x i16> %1 to <4 x i64>
+; CHECK-NEXT:  ret <4 x i64> %2
+
+  %res = call <4 x i64> @llvm.x86.avx2.pmovsxwq(<8 x i16> %v)
+  ret <4 x i64> %res
+}
diff --git a/test/Transforms/InstCombine/x86-pmovzx.ll b/test/Transforms/InstCombine/x86-pmovzx.ll
new file mode 100644
index 00000000000..31028cba26e
--- /dev/null
+++ b/test/Transforms/InstCombine/x86-pmovzx.ll
@@ -0,0 +1,136 @@
+; RUN: opt < %s -instcombine -S | FileCheck %s
+
+declare <4 x i32>  @llvm.x86.sse41.pmovzxbd(<16 x i8>) nounwind readnone
+declare <2 x i64>  @llvm.x86.sse41.pmovzxbq(<16 x i8>) nounwind readnone
+declare <8 x i16>  @llvm.x86.sse41.pmovzxbw(<16 x i8>) nounwind readnone
+declare <2 x i64>  @llvm.x86.sse41.pmovzxdq(<4 x i32>) nounwind readnone
+declare <4 x i32>  @llvm.x86.sse41.pmovzxwd(<8 x i16>) nounwind readnone
+declare <2 x i64>  @llvm.x86.sse41.pmovzxwq(<8 x i16>) nounwind readnone
+
+declare <8 x i32>  @llvm.x86.avx2.pmovzxbd(<16 x i8>) nounwind readnone
+declare <4 x i64>  @llvm.x86.avx2.pmovzxbq(<16 x i8>) nounwind readnone
+declare <16 x i16> @llvm.x86.avx2.pmovzxbw(<16 x i8>) nounwind readnone
+declare <4 x i64>  @llvm.x86.avx2.pmovzxdq(<4 x i32>) nounwind readnone
+declare <8 x i32>  @llvm.x86.avx2.pmovzxwd(<8 x i16>) nounwind readnone
+declare <4 x i64>  @llvm.x86.avx2.pmovzxwq(<8 x i16>) nounwind readnone
+
+;
+; Basic zero extension tests
+;
+
+define <4 x i32> @sse41_pmovzxbd(<16 x i8> %v) nounwind readnone {
+; CHECK-LABEL: @sse41_pmovzxbd
+; CHECK-NEXT:  shufflevector <16 x i8> %v, <16 x i8> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:  zext <4 x i8> %1 to <4 x i32>
+; CHECK-NEXT:  ret <4 x i32> %2
+
+  %res = call <4 x i32> @llvm.x86.sse41.pmovzxbd(<16 x i8> %v)
+  ret <4 x i32> %res
+}
+
+define <2 x i64> @sse41_pmovzxbq(<16 x i8> %v) nounwind readnone {
+; CHECK-LABEL: @sse41_pmovzxbq
+; CHECK-NEXT:  shufflevector <16 x i8> %v, <16 x i8> undef, <2 x i32> <i32 0, i32 1>
+; CHECK-NEXT:  zext <2 x i8> %1 to <2 x i64>
+; CHECK-NEXT:  ret <2 x i64> %2
+
+  %res = call <2 x i64> @llvm.x86.sse41.pmovzxbq(<16 x i8> %v)
+  ret <2 x i64> %res
+}
+
+define <8 x i16> @sse41_pmovzxbw(<16 x i8> %v) nounwind readnone {
+; CHECK-LABEL: @sse41_pmovzxbw
+; CHECK-NEXT:  shufflevector <16 x i8> %v, <16 x i8> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; CHECK-NEXT:  zext <8 x i8> %1 to <8 x i16>
+; CHECK-NEXT:  ret <8 x i16> %2
+
+  %res = call <8 x i16> @llvm.x86.sse41.pmovzxbw(<16 x i8> %v)
+  ret <8 x i16> %res
+}
+
+define <2 x i64> @sse41_pmovzxdq(<4 x i32> %v) nounwind readnone {
+; CHECK-LABEL: @sse41_pmovzxdq
+; CHECK-NEXT:  shufflevector <4 x i32> %v, <4 x i32> undef, <2 x i32> <i32 0, i32 1>
+; CHECK-NEXT:  zext <2 x i32> %1 to <2 x i64>
+; CHECK-NEXT:  ret <2 x i64> %2
+
+  %res = call <2 x i64> @llvm.x86.sse41.pmovzxdq(<4 x i32> %v)
+  ret <2 x i64> %res
+}
+
+define <4 x i32> @sse41_pmovzxwd(<8 x i16> %v) nounwind readnone {
+; CHECK-LABEL: @sse41_pmovzxwd
+; CHECK-NEXT:  shufflevector <8 x i16> %v, <8 x i16> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:  zext <4 x i16> %1 to <4 x i32>
+; CHECK-NEXT:  ret <4 x i32> %2
+
+  %res = call <4 x i32> @llvm.x86.sse41.pmovzxwd(<8 x i16> %v)
+  ret <4 x i32> %res
+}
+
+define <2 x i64> @sse41_pmovzxwq(<8 x i16> %v) nounwind readnone {
+; CHECK-LABEL: @sse41_pmovzxwq
+; CHECK-NEXT:  shufflevector <8 x i16> %v, <8 x i16> undef, <2 x i32> <i32 0, i32 1>
+; CHECK-NEXT:  zext <2 x i16> %1 to <2 x i64>
+; CHECK-NEXT:  ret <2 x i64> %2
+
+  %res = call <2 x i64> @llvm.x86.sse41.pmovzxwq(<8 x i16> %v)
+  ret <2 x i64> %res
+}
+
+define <8 x i32> @avx2_pmovzxbd(<16 x i8> %v) nounwind readnone {
+; CHECK-LABEL: @avx2_pmovzxbd
+; CHECK-NEXT:  shufflevector <16 x i8> %v, <16 x i8> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; CHECK-NEXT:  zext <8 x i8> %1 to <8 x i32>
+; CHECK-NEXT:  ret <8 x i32> %2
+
+  %res = call <8 x i32> @llvm.x86.avx2.pmovzxbd(<16 x i8> %v)
+  ret <8 x i32> %res
+}
+
+define <4 x i64> @avx2_pmovzxbq(<16 x i8> %v) nounwind readnone {
+; CHECK-LABEL: @avx2_pmovzxbq
+; CHECK-NEXT:  shufflevector <16 x i8> %v, <16 x i8> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:  zext <4 x i8> %1 to <4 x i64>
+; CHECK-NEXT:  ret <4 x i64> %2
+
+  %res = call <4 x i64> @llvm.x86.avx2.pmovzxbq(<16 x i8> %v)
+  ret <4 x i64> %res
+}
+
+define <16 x i16> @avx2_pmovzxbw(<16 x i8> %v) nounwind readnone {
+; CHECK-LABEL: @avx2_pmovzxbw
+; CHECK-NEXT:  zext <16 x i8> %v to <16 x i16>
+; CHECK-NEXT:  ret <16 x i16> %1
+
+  %res = call <16 x i16> @llvm.x86.avx2.pmovzxbw(<16 x i8> %v)
+  ret <16 x i16> %res
+}
+
+define <4 x i64> @avx2_pmovzxdq(<4 x i32> %v) nounwind readnone {
+; CHECK-LABEL: @avx2_pmovzxdq
+; CHECK-NEXT:  zext <4 x i32> %v to <4 x i64>
+; CHECK-NEXT:  ret <4 x i64> %1
+
+  %res = call <4 x i64> @llvm.x86.avx2.pmovzxdq(<4 x i32> %v)
+  ret <4 x i64> %res
+}
+
+define <8 x i32> @avx2_pmovzxwd(<8 x i16> %v) nounwind readnone {
+; CHECK-LABEL: @avx2_pmovzxwd
+; CHECK-NEXT:  zext <8 x i16> %v to <8 x i32>
+; CHECK-NEXT:  ret <8 x i32> %1
+
+  %res = call <8 x i32> @llvm.x86.avx2.pmovzxwd(<8 x i16> %v)
+  ret <8 x i32> %res
+}
+
+define <4 x i64> @avx2_pmovzxwq(<8 x i16> %v) nounwind readnone {
+; CHECK-LABEL: @avx2_pmovzxwq
+; CHECK-NEXT:  shufflevector <8 x i16> %v, <8 x i16> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:  zext <4 x i16> %1 to <4 x i64>
+; CHECK-NEXT:  ret <4 x i64> %2
+
+  %res = call <4 x i64> @llvm.x86.avx2.pmovzxwq(<8 x i16> %v)
+  ret <4 x i64> %res
+}
diff --git a/test/Transforms/InstCombine/x86-sse4a.ll b/test/Transforms/InstCombine/x86-sse4a.ll
new file mode 100644
index 00000000000..9766fe9329d
--- /dev/null
+++ b/test/Transforms/InstCombine/x86-sse4a.ll
@@ -0,0 +1,125 @@
+; RUN: opt < %s -instcombine -S | FileCheck %s
+
+; We should optimize these two redundant insertqi into one
+; CHECK: define <2 x i64> @testInsertTwice(<2 x i64> %v, <2 x i64> %i)
+define <2 x i64> @testInsertTwice(<2 x i64> %v, <2 x i64> %i) {
+; CHECK: call <2 x i64> @llvm.x86.sse4a.insertqi(<2 x i64> %v, <2 x i64> %i, i8 32, i8 32)
+; CHECK-NOT: insertqi
+  %1 = tail call <2 x i64> @llvm.x86.sse4a.insertqi(<2 x i64> %v, <2 x i64> %i, i8 32, i8 32)
+  %2 = tail call <2 x i64> @llvm.x86.sse4a.insertqi(<2 x i64> %1, <2 x i64> %i, i8 32, i8 32)
+  ret <2 x i64> %2
+}
+
+; The result of this insert is the second arg, since the top 64 bits of
+; the result are undefined, and we copy the bottom 64 bits from the
+; second arg
+; CHECK: define <2 x i64> @testInsert64Bits(<2 x i64> %v, <2 x i64> %i)
+define <2 x i64> @testInsert64Bits(<2 x i64> %v, <2 x i64> %i) {
+; CHECK: ret <2 x i64> %i
+  %1 = tail call <2 x i64> @llvm.x86.sse4a.insertqi(<2 x i64> %v, <2 x i64> %i, i8 64, i8 0)
+  ret <2 x i64> %1
+}
+
+; Test the several types of ranges and ordering that exist for two insertqi
+; CHECK: define <2 x i64> @testInsertContainedRange(<2 x i64> %v, <2 x i64> %i)
+define <2 x i64> @testInsertContainedRange(<2 x i64> %v, <2 x i64> %i) {
+; CHECK: %[[RES:.*]] = call <2 x i64> @llvm.x86.sse4a.insertqi(<2 x i64> %v, <2 x i64> %i, i8 32, i8 0)
+; CHECK: ret <2 x i64> %[[RES]]
+  %1 = tail call <2 x i64> @llvm.x86.sse4a.insertqi(<2 x i64> %v, <2 x i64> %i, i8 32, i8 0)
+  %2 = tail call <2 x i64> @llvm.x86.sse4a.insertqi(<2 x i64> %1, <2 x i64> %i, i8 16, i8 16)
+  ret <2 x i64> %2
+}
+
+; CHECK: define <2 x i64> @testInsertContainedRange_2(<2 x i64> %v, <2 x i64> %i)
+define <2 x i64> @testInsertContainedRange_2(<2 x i64> %v, <2 x i64> %i) {
+; CHECK: %[[RES:.*]] = call <2 x i64> @llvm.x86.sse4a.insertqi(<2 x i64> %v, <2 x i64> %i, i8 32, i8 0)
+; CHECK: ret <2 x i64> %[[RES]]
+  %1 = tail call <2 x i64> @llvm.x86.sse4a.insertqi(<2 x i64> %v, <2 x i64> %i, i8 16, i8 16)
+  %2 = tail call <2 x i64> @llvm.x86.sse4a.insertqi(<2 x i64> %1, <2 x i64> %i, i8 32, i8 0)
+  ret <2 x i64> %2
+}
+
+; CHECK: define <2 x i64> @testInsertOverlappingRange(<2 x i64> %v, <2 x i64> %i)
+define <2 x i64> @testInsertOverlappingRange(<2 x i64> %v, <2 x i64> %i) {
+; CHECK: %[[RES:.*]] = call <2 x i64> @llvm.x86.sse4a.insertqi(<2 x i64> %v, <2 x i64> %i, i8 48, i8 0)
+; CHECK: ret <2 x i64> %[[RES]]
+  %1 = tail call <2 x i64> @llvm.x86.sse4a.insertqi(<2 x i64> %v, <2 x i64> %i, i8 32, i8 0)
+  %2 = tail call <2 x i64> @llvm.x86.sse4a.insertqi(<2 x i64> %1, <2 x i64> %i, i8 32, i8 16)
+  ret <2 x i64> %2
+}
+
+; CHECK: define <2 x i64> @testInsertOverlappingRange_2(<2 x i64> %v, <2 x i64> %i)
+define <2 x i64> @testInsertOverlappingRange_2(<2 x i64> %v, <2 x i64> %i) {
+; CHECK: %[[RES:.*]] = call <2 x i64> @llvm.x86.sse4a.insertqi(<2 x i64> %v, <2 x i64> %i, i8 48, i8 0)
+; CHECK: ret <2 x i64> %[[RES]]
+  %1 = tail call <2 x i64> @llvm.x86.sse4a.insertqi(<2 x i64> %v, <2 x i64> %i, i8 32, i8 16)
+  %2 = tail call <2 x i64> @llvm.x86.sse4a.insertqi(<2 x i64> %1, <2 x i64> %i, i8 32, i8 0)
+  ret <2 x i64> %2
+}
+
+; CHECK: define <2 x i64> @testInsertAdjacentRange(<2 x i64> %v, <2 x i64> %i)
+define <2 x i64> @testInsertAdjacentRange(<2 x i64> %v, <2 x i64> %i) {
+; CHECK: %[[RES:.*]] = call <2 x i64> @llvm.x86.sse4a.insertqi(<2 x i64> %v, <2 x i64> %i, i8 48, i8 0)
+; CHECK: ret <2 x i64> %[[RES]]
+  %1 = tail call <2 x i64> @llvm.x86.sse4a.insertqi(<2 x i64> %v, <2 x i64> %i, i8 32, i8 0)
+  %2 = tail call <2 x i64> @llvm.x86.sse4a.insertqi(<2 x i64> %1, <2 x i64> %i, i8 16, i8 32)
+  ret <2 x i64> %2
+}
+
+; CHECK: define <2 x i64> @testInsertAdjacentRange_2(<2 x i64> %v, <2 x i64> %i)
+define <2 x i64> @testInsertAdjacentRange_2(<2 x i64> %v, <2 x i64> %i) {
+; CHECK: %[[RES:.*]] = call <2 x i64> @llvm.x86.sse4a.insertqi(<2 x i64> %v, <2 x i64> %i, i8 48, i8 0)
+; CHECK: ret <2 x i64> %[[RES]]
+  %1 = tail call <2 x i64> @llvm.x86.sse4a.insertqi(<2 x i64> %v, <2 x i64> %i, i8 16, i8 32)
+  %2 = tail call <2 x i64> @llvm.x86.sse4a.insertqi(<2 x i64> %1, <2 x i64> %i, i8 32, i8 0)
+  ret <2 x i64> %2
+}
+
+; CHECK: define <2 x i64> @testInsertDisjointRange(<2 x i64> %v, <2 x i64> %i)
+define <2 x i64> @testInsertDisjointRange(<2 x i64> %v, <2 x i64> %i) {
+; CHECK: tail call <2 x i64> @llvm.x86.sse4a.insertqi(<2 x i64> %v, <2 x i64> %i, i8 16, i8 0)
+; CHECK: tail call <2 x i64> @llvm.x86.sse4a.insertqi(<2 x i64> %1, <2 x i64> %i, i8 16, i8 32)
+  %1 = tail call <2 x i64> @llvm.x86.sse4a.insertqi(<2 x i64> %v, <2 x i64> %i, i8 16, i8 0)
+  %2 = tail call <2 x i64> @llvm.x86.sse4a.insertqi(<2 x i64> %1, <2 x i64> %i, i8 16, i8 32)
+  ret <2 x i64> %2
+}
+
+; CHECK: define <2 x i64> @testInsertDisjointRange_2(<2 x i64> %v, <2 x i64> %i)
+define <2 x i64> @testInsertDisjointRange_2(<2 x i64> %v, <2 x i64> %i) {
+; CHECK:  tail call <2 x i64> @llvm.x86.sse4a.insertqi(<2 x i64> %v, <2 x i64> %i, i8 16, i8 0)
+; CHECK:  tail call <2 x i64> @llvm.x86.sse4a.insertqi(<2 x i64> %1, <2 x i64> %i, i8 16, i8 32)
+  %1 = tail call <2 x i64> @llvm.x86.sse4a.insertqi(<2 x i64> %v, <2 x i64> %i, i8 16, i8 0)
+  %2 = tail call <2 x i64> @llvm.x86.sse4a.insertqi(<2 x i64> %1, <2 x i64> %i, i8 16, i8 32)
+  ret <2 x i64> %2
+}
+
+; CHECK: define <2 x i64> @testZeroLength(<2 x i64> %v, <2 x i64> %i)
+define <2 x i64> @testZeroLength(<2 x i64> %v, <2 x i64> %i) {
+; CHECK: ret <2 x i64> %i
+  %1 = tail call <2 x i64> @llvm.x86.sse4a.insertqi(<2 x i64> %v, <2 x i64> %i, i8 0, i8 0)
+  ret <2 x i64> %1
+}
+
+; CHECK: define <2 x i64> @testUndefinedInsertq_1(<2 x i64> %v, <2 x i64> %i)
+define <2 x i64> @testUndefinedInsertq_1(<2 x i64> %v, <2 x i64> %i) {
+; CHECK: ret <2 x i64> undef
+  %1 = tail call <2 x i64> @llvm.x86.sse4a.insertqi(<2 x i64> %v, <2 x i64> %i, i8 0, i8 16)
+  ret <2 x i64> %1
+}
+
+; CHECK: define <2 x i64> @testUndefinedInsertq_2(<2 x i64> %v, <2 x i64> %i)
+define <2 x i64> @testUndefinedInsertq_2(<2 x i64> %v, <2 x i64> %i) {
+; CHECK: ret <2 x i64> undef
+  %1 = tail call <2 x i64> @llvm.x86.sse4a.insertqi(<2 x i64> %v, <2 x i64> %i, i8 48, i8 32)
+  ret <2 x i64> %1
+}
+
+; CHECK: define <2 x i64> @testUndefinedInsertq_3(<2 x i64> %v, <2 x i64> %i)
+define <2 x i64> @testUndefinedInsertq_3(<2 x i64> %v, <2 x i64> %i) {
+; CHECK: ret <2 x i64> undef
+  %1 = tail call <2 x i64> @llvm.x86.sse4a.insertqi(<2 x i64> %v, <2 x i64> %i, i8 64, i8 16)
+  ret <2 x i64> %1
+}
+
+; CHECK: declare <2 x i64> @llvm.x86.sse4a.insertqi
+declare <2 x i64> @llvm.x86.sse4a.insertqi(<2 x i64>, <2 x i64>, i8, i8) nounwind
diff --git a/test/Transforms/InstCombine/xor.ll b/test/Transforms/InstCombine/xor.ll
index c8debcbac22..33d5a2a9fda 100644
--- a/test/Transforms/InstCombine/xor.ll
+++ b/test/Transforms/InstCombine/xor.ll
@@ -63,8 +63,8 @@ define i32 @test7(i32 %A, i32 %B) {
 ; CHECK-LABEL: @test7(
 ; CHECK-NEXT: %A1 = and i32 %A, 7
 ; CHECK-NEXT: %B1 = and i32 %B, 128
-; CHECK-NEXT: %C11 = or i32 %A1, %B1
-; CHECK-NEXT: ret i32 %C11
+; CHECK-NEXT: %C1.1 = or i32 %A1, %B1
+; CHECK-NEXT: ret i32 %C1.1
 	%A1 = and i32 %A, 7		; <i32> [#uses=1]
 	%B1 = and i32 %B, 128		; <i32> [#uses=1]
 	%C1 = xor i32 %A1, %B1		; <i32> [#uses=1]
@@ -96,8 +96,8 @@ define i1 @test9(i8 %A) {
 define i8 @test10(i8 %A) {
 ; CHECK-LABEL: @test10(
 ; CHECK-NEXT: %B = and i8 %A, 3
-; CHECK-NEXT: %C1 = or i8 %B, 4
-; CHECK-NEXT: ret i8 %C1
+; CHECK-NEXT: %C.1 = or i8 %B, 4
+; CHECK-NEXT: ret i8 %C.1
 	%B = and i8 %A, 3		; <i8> [#uses=1]
 	%C = xor i8 %B, 4		; <i8> [#uses=1]
 	ret i8 %C
diff --git a/test/Transforms/InstSimplify/2011-09-05-InsertExtractValue.ll b/test/Transforms/InstSimplify/2011-09-05-InsertExtractValue.ll
index 3514b347937..7e391aba304 100644
--- a/test/Transforms/InstSimplify/2011-09-05-InsertExtractValue.ll
+++ b/test/Transforms/InstSimplify/2011-09-05-InsertExtractValue.ll
@@ -2,13 +2,13 @@
 
 declare void @bar()
 
-define void @test1() {
+define void @test1() personality i32 (i32, i64, i8*, i8*)* @__gxx_personality_v0 {
 entry:
   invoke void @bar() to label %cont unwind label %lpad
 cont:
   ret void
 lpad:
-  %ex = landingpad { i8*, i32 } personality i32 (i32, i64, i8*, i8*)* @__gxx_personality_v0 cleanup
+  %ex = landingpad { i8*, i32 } cleanup
   %exc_ptr = extractvalue { i8*, i32 } %ex, 0
   %filter = extractvalue { i8*, i32 } %ex, 1
   %exc_ptr2 = insertvalue { i8*, i32 } undef, i8* %exc_ptr, 0
@@ -27,3 +27,12 @@ define { i8, i32 } @test2({ i8*, i32 } %x) {
   ret { i8, i32 } %ins
 ; CHECK-LABEL: @test2(
 }
+
+define i32 @test3(i32 %a, float %b) {
+  %agg1 = insertvalue {i32, float} undef, i32 %a, 0
+  %agg2 = insertvalue {i32, float} %agg1, float %b, 1
+  %ev = extractvalue {i32, float} %agg2, 0
+  ret i32 %ev
+; CHECK-LABEL: @test3(
+; CHECK: ret i32 %a
+}
diff --git a/test/Transforms/InstSimplify/call.ll b/test/Transforms/InstSimplify/call.ll
index ac94e75f489..465529aef66 100644
--- a/test/Transforms/InstSimplify/call.ll
+++ b/test/Transforms/InstSimplify/call.ll
@@ -1,6 +1,9 @@
 ; RUN: opt < %s -instsimplify -S | FileCheck %s
 
 declare {i8, i1} @llvm.uadd.with.overflow.i8(i8 %a, i8 %b)
+declare {i8, i1} @llvm.usub.with.overflow.i8(i8 %a, i8 %b)
+declare {i8, i1} @llvm.ssub.with.overflow.i8(i8 %a, i8 %b)
+declare {i8, i1} @llvm.umul.with.overflow.i8(i8 %a, i8 %b)
 
 define i1 @test_uadd1() {
 ; CHECK-LABEL: @test_uadd1(
@@ -18,6 +21,27 @@ define i8 @test_uadd2() {
 ; CHECK-NEXT: ret i8 42
 }
 
+define {i8, i1} @test_usub1(i8 %V) {
+; CHECK-LABEL: @test_usub1(
+  %x = call {i8, i1} @llvm.usub.with.overflow.i8(i8 %V, i8 %V)
+  ret {i8, i1} %x
+; CHECK-NEXT: ret { i8, i1 } zeroinitializer
+}
+
+define {i8, i1} @test_ssub1(i8 %V) {
+; CHECK-LABEL: @test_ssub1(
+  %x = call {i8, i1} @llvm.ssub.with.overflow.i8(i8 %V, i8 %V)
+  ret {i8, i1} %x
+; CHECK-NEXT: ret { i8, i1 } zeroinitializer
+}
+
+define {i8, i1} @test_umul1(i8 %V) {
+; CHECK-LABEL: @test_umul1(
+  %x = call {i8, i1} @llvm.umul.with.overflow.i8(i8 %V, i8 0)
+  ret {i8, i1} %x
+; CHECK-NEXT: ret { i8, i1 } zeroinitializer
+}
+
 declare i256 @llvm.cttz.i256(i256 %src, i1 %is_zero_undef)
 
 define i256 @test_cttz() {
diff --git a/test/Transforms/InstSimplify/fast-math.ll b/test/Transforms/InstSimplify/fast-math.ll
index e7fb14d7e78..90532fa5db8 100644
--- a/test/Transforms/InstSimplify/fast-math.ll
+++ b/test/Transforms/InstSimplify/fast-math.ll
@@ -70,17 +70,17 @@ define float @fadd_fsub_0(float %a) {
   ret float %ret
 }
 
-; fsub nnan ninf x, x ==> 0.0
+; fsub nnan x, x ==> 0.0
 ; CHECK-LABEL: @fsub_x_x(
 define float @fsub_x_x(float %a) {
 ; X - X ==> 0
-  %zero1 = fsub nnan ninf float %a, %a
+  %zero1 = fsub nnan float %a, %a
 
 ; Dont fold
 ; CHECK: %no_zero1 = fsub
   %no_zero1 = fsub ninf float %a, %a
 ; CHECK: %no_zero2 = fsub
-  %no_zero2 = fsub nnan float %a, %a
+  %no_zero2 = fsub float %a, %a
 ; CHECK: %no_zero = fadd
   %no_zero = fadd float %no_zero1, %no_zero2
 
@@ -114,3 +114,60 @@ define double @fdiv_zero_by_x(double %X) {
   ret double %r
 ; CHECK: ret double 0
 }
+
+define float @fdiv_self(float %f) {
+  %div = fdiv nnan float %f, %f
+  ret float %div
+; CHECK-LABEL: fdiv_self
+; CHECK: ret float 1.000000e+00
+}
+
+define float @fdiv_self_invalid(float %f) {
+  %div = fdiv float %f, %f
+  ret float %div
+; CHECK-LABEL: fdiv_self_invalid
+; CHECK: %div = fdiv float %f, %f
+; CHECK-NEXT: ret float %div
+}
+
+define float @fdiv_neg1(float %f) {
+  %neg = fsub fast float -0.000000e+00, %f
+  %div = fdiv nnan float %neg, %f
+  ret float %div
+; CHECK-LABEL: fdiv_neg1
+; CHECK: ret float -1.000000e+00
+}
+
+define float @fdiv_neg2(float %f) {
+  %neg = fsub fast float 0.000000e+00, %f
+  %div = fdiv nnan float %neg, %f
+  ret float %div
+; CHECK-LABEL: fdiv_neg2
+; CHECK: ret float -1.000000e+00
+}
+
+define float @fdiv_neg_invalid(float %f) {
+  %neg = fsub fast float -0.000000e+00, %f
+  %div = fdiv float %neg, %f
+  ret float %div
+; CHECK-LABEL: fdiv_neg_invalid
+; CHECK: %neg = fsub fast float -0.000000e+00, %f
+; CHECK-NEXT: %div = fdiv float %neg, %f
+; CHECK-NEXT: ret float %div
+}
+
+define float @fdiv_neg_swapped1(float %f) {
+  %neg = fsub float -0.000000e+00, %f
+  %div = fdiv nnan float %f, %neg
+  ret float %div
+; CHECK-LABEL: fdiv_neg_swapped1
+; CHECK: ret float -1.000000e+00
+}
+
+define float @fdiv_neg_swapped2(float %f) {
+  %neg = fsub float 0.000000e+00, %f
+  %div = fdiv nnan float %f, %neg
+  ret float %div
+; CHECK-LABEL: fdiv_neg_swapped2
+; CHECK: ret float -1.000000e+00
+}
diff --git a/test/Transforms/InstSimplify/floating-point-compare.ll b/test/Transforms/InstSimplify/floating-point-compare.ll
index af48d062b4f..8174f583453 100644
--- a/test/Transforms/InstSimplify/floating-point-compare.ll
+++ b/test/Transforms/InstSimplify/floating-point-compare.ll
@@ -58,3 +58,18 @@ define i1 @orderedLessZeroPowi(double,double) {
   ret i1 %olt
 }
 
+define i1 @nonans1(double %in1, double %in2) {
+  %cmp = fcmp nnan uno double %in1, %in2
+  ret i1 %cmp
+
+; CHECK-LABEL: @nonans1
+; CHECK-NEXT: ret i1 false
+}
+
+define i1 @nonans2(double %in1, double %in2) {
+  %cmp = fcmp nnan ord double %in1, %in2
+  ret i1 %cmp
+
+; CHECK-LABEL: @nonans2
+; CHECK-NEXT: ret i1 true
+}
diff --git a/test/Transforms/InstSimplify/undef.ll b/test/Transforms/InstSimplify/undef.ll
index f1f0b037fdb..d75dc364243 100644
--- a/test/Transforms/InstSimplify/undef.ll
+++ b/test/Transforms/InstSimplify/undef.ll
@@ -265,3 +265,17 @@ define i32 @test34(i32 %a) {
   %b = lshr i32 undef, 0
   ret i32 %b
 }
+
+; CHECK-LABEL: @test35
+; CHECK: ret i32 undef
+define i32 @test35(<4 x i32> %V) {
+  %b = extractelement <4 x i32> %V, i32 4
+  ret i32 %b
+}
+
+; CHECK-LABEL: @test36
+; CHECK: ret i32 undef
+define i32 @test36(i32 %V) {
+  %b = extractelement <4 x i32> undef, i32 %V
+  ret i32 %b
+}
diff --git a/test/Transforms/Internalize/comdat.ll b/test/Transforms/Internalize/comdat.ll
new file mode 100644
index 00000000000..8217dd603a1
--- /dev/null
+++ b/test/Transforms/Internalize/comdat.ll
@@ -0,0 +1,52 @@
+; RUN: opt < %s -internalize -internalize-public-api-list c1 -internalize-public-api-list c2 -internalize-public-api-list c3 -internalize-public-api-list c4 -S | FileCheck %s
+
+$c1 = comdat any
+$c2 = comdat any
+$c3 = comdat any
+$c4 = comdat any
+
+; CHECK: @c1_c = global i32 0, comdat($c1)
+@c1_c = global i32 0, comdat($c1)
+
+; CHECK: @c2_b = internal global i32 0{{$}}
+@c2_b = global i32 0, comdat($c2)
+
+; CHECK: @c3 = global i32 0, comdat{{$}}
+@c3 = global i32 0, comdat
+
+; CHECK: @c4_a = internal global i32 0, comdat($c4)
+@c4_a = internal global i32 0, comdat($c4)
+
+; CHECK: @c1_d = alias i32* @c1_c
+@c1_d = alias i32* @c1_c
+
+; CHECK: @c2_c = internal alias i32* @c2_b
+@c2_c = alias i32* @c2_b
+
+; CHECK: @c4 = alias i32* @c4_a
+@c4 = alias i32* @c4_a
+
+; CHECK: define void @c1() comdat {
+define void @c1() comdat {
+  ret void
+}
+
+; CHECK: define void @c1_a() comdat($c1) {
+define void @c1_a() comdat($c1) {
+  ret void
+}
+
+; CHECK: define internal void @c2() {
+define internal void @c2() comdat {
+  ret void
+}
+
+; CHECK: define internal void @c2_a() {
+define void @c2_a() comdat($c2) {
+  ret void
+}
+
+; CHECK: define void @c3_a() comdat($c3) {
+define void @c3_a() comdat($c3) {
+  ret void
+}
diff --git a/test/Transforms/JumpThreading/landing-pad.ll b/test/Transforms/JumpThreading/landing-pad.ll
index 4d49db04f6a..5dcc5aa17e6 100644
--- a/test/Transforms/JumpThreading/landing-pad.ll
+++ b/test/Transforms/JumpThreading/landing-pad.ll
@@ -42,7 +42,7 @@ entry:
   ret void
 }
 
-define void @_Z3fn1v() uwtable {
+define void @_Z3fn1v() uwtable personality i8* bitcast (i32 (...)* @__gxx_personality_v0 to i8*) {
 entry:
   %call = call noalias i8* @_Znwm() #8
   invoke void @_ZN24CompositeEditCommandImplC2Ev()
@@ -68,13 +68,13 @@ invoke.cont7:                                     ; preds = %_ZN15EditCommandImp
   ret void
 
 lpad:                                             ; preds = %entry
-  %4 = landingpad { i8*, i32 } personality i8* bitcast (i32 (...)* @__gxx_personality_v0 to i8*)
+  %4 = landingpad { i8*, i32 }
           cleanup
   call void @_ZdlPv() #9
   unreachable
 
 lpad1:                                            ; preds = %_ZN1DC1Ev.exit, %_ZN15EditCommandImpl5applyEv.exit
-  %5 = landingpad { i8*, i32 } personality i8* bitcast (i32 (...)* @__gxx_personality_v0 to i8*)
+  %5 = landingpad { i8*, i32 }
           cleanup
   %6 = load i32, i32* %1, align 4
   %tobool.i.i.i = icmp eq i32 %6, 0
@@ -91,7 +91,7 @@ _ZN1BI1DED1Ev.exit:                               ; preds = %lpad1, %if.then.i.i
   resume { i8*, i32 } undef
 
 terminate.lpad:                                   ; No predecessors!
-  %7 = landingpad { i8*, i32 } personality i8* bitcast (i32 (...)* @__gxx_personality_v0 to i8*)
+  %7 = landingpad { i8*, i32 }
           catch i8* null
   unreachable
 }
diff --git a/test/Transforms/JumpThreading/thread-cmp.ll b/test/Transforms/JumpThreading/thread-cmp.ll
new file mode 100644
index 00000000000..95089674476
--- /dev/null
+++ b/test/Transforms/JumpThreading/thread-cmp.ll
@@ -0,0 +1,69 @@
+; RUN: opt -S -jump-threading %s | FileCheck %s
+; When simplify a branch based on LVI predicates, we should replace the 
+; comparison itself with a constant (when possible) in case it's otherwise used.
+
+define i32 @test(i32* %p) {
+; CHECK-LABEL: @test
+; CHECK: icmp eq
+; CHECK-NEXT: br i1 %cmp, label %exit2, label %exit1
+; CHECK-NOT: icmp ne
+entry:
+  %cmp = icmp eq i32* %p, null
+  br i1 %cmp, label %is_null, label %not_null
+is_null:
+  %cmp2 = icmp ne i32* %p, null
+  br i1 %cmp2, label %exit1, label %exit2
+not_null:
+  %cmp3 = icmp ne i32* %p, null
+  br i1 %cmp3, label %exit1, label %exit2
+exit1:
+  ret i32 0
+exit2:
+  ret i32 1
+}
+
+declare void @use(i1)
+
+; It would not be legal to replace %cmp2 (well, in this case it actually is, 
+; but that's a CSE problem, not a LVI/jump threading problem)
+define i32 @test_negative(i32* %p) {
+; CHECK-LABEL: @test
+; CHECK: icmp ne
+; CHECK: icmp eq
+; CHECK-NEXT: br i1 %cmp, label %exit2, label %exit1
+; CHECK-NOT: icmp ne
+entry:
+  %cmp2 = icmp ne i32* %p, null
+  call void @use(i1 %cmp2)
+  %cmp = icmp eq i32* %p, null
+  br i1 %cmp, label %is_null, label %not_null
+is_null:
+  br i1 %cmp2, label %exit1, label %exit2
+not_null:
+  br i1 %cmp2, label %exit1, label %exit2
+exit1:
+  ret i32 0
+exit2:
+  ret i32 1
+}
+
+; In this case, we can remove cmp2 because it's otherwise unused
+define i32 @test2(i32* %p) {
+; CHECK-LABEL: @test
+; CHECK-LABEL: entry:
+; CHECK-NEXT: icmp eq
+; CHECK-NEXT: br i1 %cmp, label %exit2, label %exit1
+; CHECK-NOT: icmp ne
+entry:
+  %cmp2 = icmp ne i32* %p, null
+  %cmp = icmp eq i32* %p, null
+  br i1 %cmp, label %is_null, label %not_null
+is_null:
+  br i1 %cmp2, label %exit1, label %exit2
+not_null:
+  br i1 %cmp2, label %exit1, label %exit2
+exit1:
+  ret i32 0
+exit2:
+  ret i32 1
+}
diff --git a/test/Transforms/LCSSA/invoke-dest.ll b/test/Transforms/LCSSA/invoke-dest.ll
index a36dce77a2d..1523d4ff1f6 100644
--- a/test/Transforms/LCSSA/invoke-dest.ll
+++ b/test/Transforms/LCSSA/invoke-dest.ll
@@ -9,7 +9,7 @@ target datalayout = "e-p:32:32:32-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-f3
 @.str32190 = external constant [92 x i8], align 1		; <[92 x i8]*> [#uses=1]
 @.str41 = external constant [25 x i8], align 1		; <[25 x i8]*> [#uses=1]
 
-define void @_ZN8EtherBus10initializeEv() {
+define void @_ZN8EtherBus10initializeEv() personality i32 (...)* @__gxx_personality_v0 {
 entry:
 	br i1 undef, label %_ZN7cObjectnwEj.exit, label %bb.i
 
@@ -54,7 +54,7 @@ invcont11:		; preds = %invcont10
 	br i1 undef, label %bb12, label %bb18
 
 bb12:		; preds = %invcont11
-	invoke void (i8*, i8*, ...)* @_ZN6cEnvir6printfEPKcz(i8* null, i8* getelementptr ([3 x i8], [3 x i8]* @.str12, i32 0, i32 0), i32 undef)
+	invoke void (i8*, i8*, ...) @_ZN6cEnvir6printfEPKcz(i8* null, i8* getelementptr ([3 x i8], [3 x i8]* @.str12, i32 0, i32 0), i32 undef)
 			to label %bb.i.i159 unwind label %lpad119
 
 bb.i.i159:		; preds = %bb12
@@ -77,7 +77,7 @@ invcont35:		; preds = %bb34
 	br i1 undef, label %bb49, label %bb61
 
 bb49:		; preds = %invcont35
-	invoke void (i8*, i8*, ...)* @_ZNK13cSimpleModule5errorEPKcz(i8* undef, i8* getelementptr ([92 x i8], [92 x i8]* @.str32190, i32 0, i32 0))
+	invoke void (i8*, i8*, ...) @_ZNK13cSimpleModule5errorEPKcz(i8* undef, i8* getelementptr ([92 x i8], [92 x i8]* @.str32190, i32 0, i32 0))
 			to label %bb51 unwind label %lpad119
 
 bb51:		; preds = %bb49
@@ -110,17 +110,17 @@ bb106:		; preds = %invcont105, %bb61
 			to label %.noexc unwind label %lpad119		; <i8*> [#uses=1]
 
 lpad:		; preds = %_ZN7cObjectnwEj.exit
-        %exn = landingpad {i8*, i32} personality i32 (...)* @__gxx_personality_v0
+        %exn = landingpad {i8*, i32}
                  cleanup
 	br label %Unwind
 
 lpad119:		; preds = %bb106, %invcont104, %invcont103, %bb102, %bb49, %bb34, %bb12, %invcont10, %invcont9, %bb8
-        %exn119 = landingpad {i8*, i32} personality i32 (...)* @__gxx_personality_v0
+        %exn119 = landingpad {i8*, i32}
                  cleanup
 	unreachable
 
 lpad123:		; preds = %.noexc
-        %exn123 = landingpad {i8*, i32} personality i32 (...)* @__gxx_personality_v0
+        %exn123 = landingpad {i8*, i32}
                  cleanup
 	%tmp5 = icmp eq i8* %tmp4, null		; <i1> [#uses=1]
 	br i1 %tmp5, label %Unwind, label %bb.i2
diff --git a/test/Transforms/LICM/PR24013.ll b/test/Transforms/LICM/PR24013.ll
new file mode 100644
index 00000000000..4557bfcfd12
--- /dev/null
+++ b/test/Transforms/LICM/PR24013.ll
@@ -0,0 +1,19 @@
+; RUN: opt -licm -S < %s | FileCheck %s
+
+define void @f(i1 zeroext %p1) {
+; CHECK-LABEL: @f(
+entry:
+  br label %lbl
+
+lbl.loopexit:                                     ; No predecessors!
+  br label %lbl
+
+lbl:                                              ; preds = %lbl.loopexit, %entry
+  %phi = phi i32 [ %conv, %lbl.loopexit ], [ undef, %entry ]
+; CHECK: phi i32 [ undef, {{.*}} ], [ undef
+  br label %if.then.5
+
+if.then.5:                                        ; preds = %if.then.5, %lbl
+  %conv = zext i1 undef to i32
+  br label %if.then.5
+}
diff --git a/test/Transforms/LICM/debug-value.ll b/test/Transforms/LICM/debug-value.ll
index 222069813c3..62429fdc321 100644
--- a/test/Transforms/LICM/debug-value.ll
+++ b/test/Transforms/LICM/debug-value.ll
@@ -15,7 +15,7 @@ if.then:                                          ; preds = %for.body
 
 if.then27:                                        ; preds = %if.then
 ; CHECK: tail call void @llvm.dbg.value
-  tail call void @llvm.dbg.value(metadata double undef, i64 0, metadata !19, metadata !MDExpression()), !dbg !21
+  tail call void @llvm.dbg.value(metadata double undef, i64 0, metadata !19, metadata !DIExpression()), !dbg !21
   br label %for.body61.us
 
 if.end.if.end.split_crit_edge.critedge:           ; preds = %if.then
@@ -36,30 +36,30 @@ declare void @llvm.dbg.value(metadata, i64, metadata, metadata) nounwind readnon
 !llvm.module.flags = !{!26}
 !llvm.dbg.sp = !{!0, !6, !9, !10}
 
-!0 = !MDSubprogram(name: "idamax", line: 112, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, file: !25, scope: !1, type: !3)
-!1 = !MDFile(filename: "/Volumes/Lalgate/work/llvm/projects/llvm-test/SingleSource/Benchmarks/CoyoteBench/lpbench.c", directory: "/private/tmp")
-!2 = !MDCompileUnit(language: DW_LANG_C99, producer: "clang version 2.9 (trunk 127169)", isOptimized: true, emissionKind: 0, file: !25, enums: !8, retainedTypes: !8, subprograms: !8)
-!3 = !MDSubroutineType(types: !4)
+!0 = !DISubprogram(name: "idamax", line: 112, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, file: !25, scope: !1, type: !3)
+!1 = !DIFile(filename: "/Volumes/Lalgate/work/llvm/projects/llvm-test/SingleSource/Benchmarks/CoyoteBench/lpbench.c", directory: "/private/tmp")
+!2 = !DICompileUnit(language: DW_LANG_C99, producer: "clang version 2.9 (trunk 127169)", isOptimized: true, emissionKind: 0, file: !25, enums: !8, retainedTypes: !8, subprograms: !8)
+!3 = !DISubroutineType(types: !4)
 !4 = !{!5}
-!5 = !MDBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
-!6 = !MDSubprogram(name: "dscal", line: 206, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, file: !25, scope: !1, type: !7)
-!7 = !MDSubroutineType(types: !8)
+!5 = !DIBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
+!6 = !DISubprogram(name: "dscal", line: 206, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, file: !25, scope: !1, type: !7)
+!7 = !DISubroutineType(types: !8)
 !8 = !{null}
-!9 = !MDSubprogram(name: "daxpy", line: 230, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, file: !25, scope: !1, type: !7)
-!10 = !MDSubprogram(name: "dgefa", line: 267, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, file: !25, scope: !1, type: !7)
-!11 = !MDLocation(line: 281, column: 9, scope: !12)
-!12 = distinct !MDLexicalBlock(line: 272, column: 5, file: !25, scope: !13)
-!13 = distinct !MDLexicalBlock(line: 271, column: 5, file: !25, scope: !14)
-!14 = distinct !MDLexicalBlock(line: 267, column: 1, file: !25, scope: !10)
-!15 = !MDLocation(line: 271, column: 5, scope: !14)
-!16 = !MDLocation(line: 284, column: 10, scope: !17)
-!17 = distinct !MDLexicalBlock(line: 282, column: 9, file: !25, scope: !12)
+!9 = !DISubprogram(name: "daxpy", line: 230, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, file: !25, scope: !1, type: !7)
+!10 = !DISubprogram(name: "dgefa", line: 267, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, file: !25, scope: !1, type: !7)
+!11 = !DILocation(line: 281, column: 9, scope: !12)
+!12 = distinct !DILexicalBlock(line: 272, column: 5, file: !25, scope: !13)
+!13 = distinct !DILexicalBlock(line: 271, column: 5, file: !25, scope: !14)
+!14 = distinct !DILexicalBlock(line: 267, column: 1, file: !25, scope: !10)
+!15 = !DILocation(line: 271, column: 5, scope: !14)
+!16 = !DILocation(line: 284, column: 10, scope: !17)
+!17 = distinct !DILexicalBlock(line: 282, column: 9, file: !25, scope: !12)
 !18 = !{double undef}
-!19 = !MDLocalVariable(tag: DW_TAG_auto_variable, name: "temp", line: 268, scope: !14, file: !1, type: !20)
-!20 = !MDBasicType(tag: DW_TAG_base_type, name: "double", size: 64, align: 64, encoding: DW_ATE_float)
-!21 = !MDLocation(line: 286, column: 14, scope: !22)
-!22 = distinct !MDLexicalBlock(line: 285, column: 13, file: !25, scope: !17)
-!23 = !MDLocation(line: 296, column: 13, scope: !17)
-!24 = !MDLocation(line: 313, column: 1, scope: !14)
-!25 = !MDFile(filename: "/Volumes/Lalgate/work/llvm/projects/llvm-test/SingleSource/Benchmarks/CoyoteBench/lpbench.c", directory: "/private/tmp")
+!19 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "temp", line: 268, scope: !14, file: !1, type: !20)
+!20 = !DIBasicType(tag: DW_TAG_base_type, name: "double", size: 64, align: 64, encoding: DW_ATE_float)
+!21 = !DILocation(line: 286, column: 14, scope: !22)
+!22 = distinct !DILexicalBlock(line: 285, column: 13, file: !25, scope: !17)
+!23 = !DILocation(line: 296, column: 13, scope: !17)
+!24 = !DILocation(line: 313, column: 1, scope: !14)
+!25 = !DIFile(filename: "/Volumes/Lalgate/work/llvm/projects/llvm-test/SingleSource/Benchmarks/CoyoteBench/lpbench.c", directory: "/private/tmp")
 !26 = !{i32 1, !"Debug Info Version", i32 3}
diff --git a/test/Transforms/LICM/hoist-deref-load.ll b/test/Transforms/LICM/hoist-deref-load.ll
index 4d0ae452921..c384a858d1e 100644
--- a/test/Transforms/LICM/hoist-deref-load.ll
+++ b/test/Transforms/LICM/hoist-deref-load.ll
@@ -164,5 +164,229 @@ for.end:                                          ; preds = %for.inc, %entry
   ret void
 }
 
-attributes #0 = { nounwind uwtable }
+; This test represents the following function:
+; void test1(int * __restrict__ a, int *b, int &c, int n) {
+;   if (c != null)
+;     for (int i = 0; i < n; ++i)
+;       if (a[i] > 0)
+;         a[i] = c*b[i];
+; }
+; and we want to hoist the load of %c out of the loop. This can be done only
+; because the dereferenceable_or_null attribute is on %c and there is a null
+; check on %c.
+
+; CHECK-LABEL: @test5
+; CHECK: load i32, i32* %c, align 4
+; CHECK: for.body:
+
+define void @test5(i32* noalias %a, i32* %b, i32* dereferenceable_or_null(4) %c, i32 %n) #0 {
+entry:
+  %not_null = icmp ne i32* %c, null
+  br i1 %not_null, label %not.null, label %for.end
+
+not.null:
+  %cmp11 = icmp sgt i32 %n, 0
+  br i1 %cmp11, label %for.body, label %for.end
+
+for.body:                                         ; preds = %not.null, %for.inc
+  %indvars.iv = phi i64 [ %indvars.iv.next, %for.inc ], [ 0, %not.null ]
+  %arrayidx = getelementptr inbounds i32, i32* %a, i64 %indvars.iv
+  %0 = load i32, i32* %arrayidx, align 4
+  %cmp1 = icmp sgt i32 %0, 0
+  br i1 %cmp1, label %if.then, label %for.inc
+
+if.then:                                          ; preds = %for.body
+  %1 = load i32, i32* %c, align 4
+  %arrayidx3 = getelementptr inbounds i32, i32* %b, i64 %indvars.iv
+  %2 = load i32, i32* %arrayidx3, align 4
+  %mul = mul nsw i32 %2, %1
+  store i32 %mul, i32* %arrayidx, align 4
+  br label %for.inc
+
+for.inc:                                          ; preds = %for.body, %if.then
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+  %lftr.wideiv = trunc i64 %indvars.iv.next to i32
+  %exitcond = icmp eq i32 %lftr.wideiv, %n
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:                                          ; preds = %for.inc, %entry, %not.null
+  ret void
+}
+
+; This is the same as @test5, but without the null check on %c.
+; Without this check, we should not hoist the load of %c.
+
+; This test case has an icmp on c but the use of this comparison is
+; not a branch. 
+
+; CHECK-LABEL: @test6
+; CHECK: if.then:
+; CHECK: load i32, i32* %c, align 4
+
+define i1 @test6(i32* noalias %a, i32* %b, i32* dereferenceable_or_null(4) %c, i32 %n) #0 {
+entry:
+  %not_null = icmp ne i32* %c, null
+  %cmp11 = icmp sgt i32 %n, 0
+  br i1 %cmp11, label %for.body, label %for.end
+
+for.body:                                         ; preds = %entry, %for.inc
+  %indvars.iv = phi i64 [ %indvars.iv.next, %for.inc ], [ 0, %entry ]
+  %arrayidx = getelementptr inbounds i32, i32* %a, i64 %indvars.iv
+  %0 = load i32, i32* %arrayidx, align 4
+  %cmp1 = icmp sgt i32 %0, 0
+  br i1 %cmp1, label %if.then, label %for.inc
+
+if.then:                                          ; preds = %for.body
+  %1 = load i32, i32* %c, align 4
+  %arrayidx3 = getelementptr inbounds i32, i32* %b, i64 %indvars.iv
+  %2 = load i32, i32* %arrayidx3, align 4
+  %mul = mul nsw i32 %2, %1
+  store i32 %mul, i32* %arrayidx, align 4
+  br label %for.inc
+
+for.inc:                                          ; preds = %for.body, %if.then
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+  %lftr.wideiv = trunc i64 %indvars.iv.next to i32
+  %exitcond = icmp eq i32 %lftr.wideiv, %n
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:                                          ; preds = %for.inc, %entry
+  ret i1 %not_null
+}
+
+; This test represents the following function:
+; void test1(int * __restrict__ a, int *b, int **cptr, int n) {
+;   c = *cptr;
+;   for (int i = 0; i < n; ++i)
+;     if (a[i] > 0)
+;       a[i] = (*c)*b[i];
+; }
+; and we want to hoist the load of %c out of the loop. This can be done only
+; because the dereferenceable meatdata on the c = *cptr load.
+
+; CHECK-LABEL: @test7
+; CHECK: load i32, i32* %c, align 4
+; CHECK: for.body:
+
+define void @test7(i32* noalias %a, i32* %b, i32** %cptr, i32 %n) #0 {
+entry:
+  %c = load i32*, i32** %cptr, !dereferenceable !0
+  %cmp11 = icmp sgt i32 %n, 0
+  br i1 %cmp11, label %for.body, label %for.end
+
+for.body:                                         ; preds = %entry, %for.inc
+  %indvars.iv = phi i64 [ %indvars.iv.next, %for.inc ], [ 0, %entry ]
+  %arrayidx = getelementptr inbounds i32, i32* %a, i64 %indvars.iv
+  %0 = load i32, i32* %arrayidx, align 4
+  %cmp1 = icmp sgt i32 %0, 0
+  br i1 %cmp1, label %if.then, label %for.inc
 
+if.then:                                          ; preds = %for.body
+  %1 = load i32, i32* %c, align 4
+  %arrayidx3 = getelementptr inbounds i32, i32* %b, i64 %indvars.iv
+  %2 = load i32, i32* %arrayidx3, align 4
+  %mul = mul nsw i32 %2, %1
+  store i32 %mul, i32* %arrayidx, align 4
+  br label %for.inc
+
+for.inc:                                          ; preds = %for.body, %if.then
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+  %lftr.wideiv = trunc i64 %indvars.iv.next to i32
+  %exitcond = icmp eq i32 %lftr.wideiv, %n
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:                                          ; preds = %for.inc, %entry
+  ret void
+}
+
+; This test represents the following function:
+; void test1(int * __restrict__ a, int *b, int **cptr, int n) {
+;   c = *cptr;
+;   if (c != null)
+;     for (int i = 0; i < n; ++i)
+;       if (a[i] > 0)
+;         a[i] = (*c)*b[i];
+; }
+; and we want to hoist the load of %c out of the loop. This can be done only
+; because the dereferenceable_or_null meatdata on the c = *cptr load and there 
+; is a null check on %c.
+
+; CHECK-LABEL: @test8
+; CHECK: load i32, i32* %c, align 4
+; CHECK: for.body:
+
+define void @test8(i32* noalias %a, i32* %b, i32** %cptr, i32 %n) #0 {
+entry:
+  %c = load i32*, i32** %cptr, !dereferenceable_or_null !0
+  %not_null = icmp ne i32* %c, null
+  br i1 %not_null, label %not.null, label %for.end
+
+not.null:
+  %cmp11 = icmp sgt i32 %n, 0
+  br i1 %cmp11, label %for.body, label %for.end
+
+for.body:                                         ; preds = %not.null, %for.inc
+  %indvars.iv = phi i64 [ %indvars.iv.next, %for.inc ], [ 0, %not.null ]
+  %arrayidx = getelementptr inbounds i32, i32* %a, i64 %indvars.iv
+  %0 = load i32, i32* %arrayidx, align 4
+  %cmp1 = icmp sgt i32 %0, 0
+  br i1 %cmp1, label %if.then, label %for.inc
+
+if.then:                                          ; preds = %for.body
+  %1 = load i32, i32* %c, align 4
+  %arrayidx3 = getelementptr inbounds i32, i32* %b, i64 %indvars.iv
+  %2 = load i32, i32* %arrayidx3, align 4
+  %mul = mul nsw i32 %2, %1
+  store i32 %mul, i32* %arrayidx, align 4
+  br label %for.inc
+
+for.inc:                                          ; preds = %for.body, %if.then
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+  %lftr.wideiv = trunc i64 %indvars.iv.next to i32
+  %exitcond = icmp eq i32 %lftr.wideiv, %n
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:                                          ; preds = %for.inc, %entry, %not.null
+  ret void
+}
+
+; This is the same as @test8, but without the null check on %c.
+; Without this check, we should not hoist the load of %c.
+
+; CHECK-LABEL: @test9
+; CHECK: if.then:
+; CHECK: load i32, i32* %c, align 4
+
+define void @test9(i32* noalias %a, i32* %b, i32** %cptr, i32 %n) #0 {
+entry:
+  %c = load i32*, i32** %cptr, !dereferenceable_or_null !0
+  %cmp11 = icmp sgt i32 %n, 0
+  br i1 %cmp11, label %for.body, label %for.end
+
+for.body:                                         ; preds = %entry, %for.inc
+  %indvars.iv = phi i64 [ %indvars.iv.next, %for.inc ], [ 0, %entry ]
+  %arrayidx = getelementptr inbounds i32, i32* %a, i64 %indvars.iv
+  %0 = load i32, i32* %arrayidx, align 4
+  %cmp1 = icmp sgt i32 %0, 0
+  br i1 %cmp1, label %if.then, label %for.inc
+
+if.then:                                          ; preds = %for.body
+  %1 = load i32, i32* %c, align 4
+  %arrayidx3 = getelementptr inbounds i32, i32* %b, i64 %indvars.iv
+  %2 = load i32, i32* %arrayidx3, align 4
+  %mul = mul nsw i32 %2, %1
+  store i32 %mul, i32* %arrayidx, align 4
+  br label %for.inc
+
+for.inc:                                          ; preds = %for.body, %if.then
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+  %lftr.wideiv = trunc i64 %indvars.iv.next to i32
+  %exitcond = icmp eq i32 %lftr.wideiv, %n
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:                                          ; preds = %for.inc, %entry
+  ret void
+}
+
+attributes #0 = { nounwind uwtable }
+!0 = !{i64 4}
diff --git a/test/Transforms/LICM/pr23608.ll b/test/Transforms/LICM/pr23608.ll
new file mode 100644
index 00000000000..249bc6bf5f6
--- /dev/null
+++ b/test/Transforms/LICM/pr23608.ll
@@ -0,0 +1,50 @@
+; RUN: opt -S -licm %s | FileCheck %s
+; ModuleID = '../pr23608.ll'
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+%struct.PyFrameObject = type { i32 }
+
+@a = common global %struct.PyFrameObject* null, align 8
+@__msan_origin_tls = external thread_local(initialexec) global i32
+
+define void @fn1() {
+entry:
+  br label %indirectgoto
+
+while.cond:                                       ; preds = %indirectgoto, %bb15
+  %tmp = load %struct.PyFrameObject*, %struct.PyFrameObject** @a, align 8
+  %_msld = load i64, i64* inttoptr (i64 and (i64 ptrtoint (%struct.PyFrameObject** @a to i64), i64 -70368744177665) to i64*), align 8
+  %tmp1 = load i32, i32* inttoptr (i64 add (i64 and (i64 ptrtoint (%struct.PyFrameObject** @a to i64), i64 -70368744177665), i64 35184372088832) to i32*), align 8
+  %f_iblock = getelementptr inbounds %struct.PyFrameObject, %struct.PyFrameObject* %tmp, i64 0, i32 0
+  br label %bb2
+
+bb:                                               ; preds = %while.cond
+  call void @__msan_warning_noreturn()
+  unreachable
+
+bb2:                                              ; preds = %while.cond
+  %tmp3 = load i32, i32* %f_iblock, align 4
+  %tmp4 = ptrtoint i32* %f_iblock to i64
+  %tmp8 = inttoptr i64 %tmp4 to i32*
+  %tobool = icmp eq i64 %tmp4, 0
+  br i1 %tobool, label %bb13, label %bb15
+
+bb13:                                             ; preds = %bb2
+; CHECK-LABEL bb13:
+; CHECK: %tmp8.le = inttoptr
+  %.lcssa7 = phi i32* [ %tmp8, %bb2 ]
+  call void @__msan_warning_noreturn()
+  unreachable
+
+bb15:                                             ; preds = %bb2
+  br i1 %tobool, label %while.end, label %while.cond
+
+while.end:                                        ; preds = %bb15
+  ret void
+
+indirectgoto:                                     ; preds = %indirectgoto, %entry
+  indirectbr i8* null, [label %indirectgoto, label %while.cond]
+}
+
+declare void @__msan_warning_noreturn()
diff --git a/test/Transforms/LoopDistribute/basic-with-memchecks.ll b/test/Transforms/LoopDistribute/basic-with-memchecks.ll
new file mode 100644
index 00000000000..3aced485041
--- /dev/null
+++ b/test/Transforms/LoopDistribute/basic-with-memchecks.ll
@@ -0,0 +1,110 @@
+; RUN: opt -basicaa -loop-distribute -verify-loop-info -verify-dom-info -S \
+; RUN:   < %s | FileCheck %s
+
+; RUN: opt -basicaa -loop-distribute -loop-vectorize -force-vector-width=4 \
+; RUN:   -verify-loop-info -verify-dom-info -S < %s | \
+; RUN:   FileCheck --check-prefix=VECTORIZE %s
+
+; The memcheck version of basic.ll.  We should distribute and vectorize the
+; second part of this loop with 5 memchecks (A+1 x {C, D, E} + C x {A, B})
+;
+;   for (i = 0; i < n; i++) {
+;     A[i + 1] = A[i] * B[i];
+; -------------------------------
+;     C[i] = D[i] * E[i];
+;   }
+
+target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-apple-macosx10.10.0"
+
+@B = common global i32* null, align 8
+@A = common global i32* null, align 8
+@C = common global i32* null, align 8
+@D = common global i32* null, align 8
+@E = common global i32* null, align 8
+
+define void @f() {
+entry:
+  %a = load i32*, i32** @A, align 8
+  %b = load i32*, i32** @B, align 8
+  %c = load i32*, i32** @C, align 8
+  %d = load i32*, i32** @D, align 8
+  %e = load i32*, i32** @E, align 8
+  br label %for.body
+
+; We have two compares for each array overlap check.
+; Since the checks to A and A + 4 get merged, this will give us a
+; total of 8 compares.
+;
+; CHECK: for.body.lver.memcheck:
+; CHECK:     = icmp
+; CHECK:     = icmp
+
+; CHECK:     = icmp
+; CHECK:     = icmp
+
+; CHECK:     = icmp
+; CHECK:     = icmp
+
+; CHECK:     = icmp
+; CHECK:     = icmp
+
+; CHECK-NOT: = icmp
+; CHECK:     br i1 %memcheck.conflict, label %for.body.ph.lver.orig, label %for.body.ph.ldist1
+
+; The non-distributed loop that the memchecks fall back on.
+
+; CHECK: for.body.ph.lver.orig:
+; CHECK:     br label %for.body.lver.orig
+; CHECK: for.body.lver.orig:
+; CHECK:    br i1 %exitcond.lver.orig, label %for.end, label %for.body.lver.orig
+
+; Verify the two distributed loops.
+
+; CHECK: for.body.ph.ldist1:
+; CHECK:     br label %for.body.ldist1
+; CHECK: for.body.ldist1:
+; CHECK:    %mulA.ldist1 = mul i32 %loadB.ldist1, %loadA.ldist1
+; CHECK:    br i1 %exitcond.ldist1, label %for.body.ph, label %for.body.ldist1
+
+; CHECK: for.body.ph:
+; CHECK:    br label %for.body
+; CHECK: for.body:
+; CHECK:    %mulC = mul i32 %loadD, %loadE
+; CHECK: for.end:
+
+
+; VECTORIZE: mul <4 x i32>
+
+for.body:                                         ; preds = %for.body, %entry
+  %ind = phi i64 [ 0, %entry ], [ %add, %for.body ]
+
+  %arrayidxA = getelementptr inbounds i32, i32* %a, i64 %ind
+  %loadA = load i32, i32* %arrayidxA, align 4
+
+  %arrayidxB = getelementptr inbounds i32, i32* %b, i64 %ind
+  %loadB = load i32, i32* %arrayidxB, align 4
+
+  %mulA = mul i32 %loadB, %loadA
+
+  %add = add nuw nsw i64 %ind, 1
+  %arrayidxA_plus_4 = getelementptr inbounds i32, i32* %a, i64 %add
+  store i32 %mulA, i32* %arrayidxA_plus_4, align 4
+
+  %arrayidxD = getelementptr inbounds i32, i32* %d, i64 %ind
+  %loadD = load i32, i32* %arrayidxD, align 4
+
+  %arrayidxE = getelementptr inbounds i32, i32* %e, i64 %ind
+  %loadE = load i32, i32* %arrayidxE, align 4
+
+  %mulC = mul i32 %loadD, %loadE
+
+  %arrayidxC = getelementptr inbounds i32, i32* %c, i64 %ind
+  store i32 %mulC, i32* %arrayidxC, align 4
+
+  %exitcond = icmp eq i64 %add, 20
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:                                          ; preds = %for.body
+  ret void
+}
diff --git a/test/Transforms/LoopDistribute/basic.ll b/test/Transforms/LoopDistribute/basic.ll
new file mode 100644
index 00000000000..f19fa8bb1b1
--- /dev/null
+++ b/test/Transforms/LoopDistribute/basic.ll
@@ -0,0 +1,83 @@
+; RUN: opt -basicaa -loop-distribute -verify-loop-info -verify-dom-info -S \
+; RUN:   < %s | FileCheck %s
+
+; RUN: opt -basicaa -loop-distribute -verify-loop-info -verify-dom-info \
+; RUN:   -loop-accesses -analyze < %s | FileCheck %s --check-prefix=ANALYSIS
+
+; RUN: opt -basicaa -loop-distribute -loop-vectorize -force-vector-width=4 -S \
+; RUN:   < %s | FileCheck %s --check-prefix=VECTORIZE
+
+; We should distribute this loop into a safe (2nd statement) and unsafe loop
+; (1st statement):
+;   for (i = 0; i < n; i++) {
+;     A[i + 1] = A[i] * B[i];
+;     =======================
+;     C[i] = D[i] * E[i];
+;   }
+
+target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-apple-macosx10.10.0"
+
+define void @f(i32* noalias %a,
+               i32* noalias %b,
+               i32* noalias %c,
+               i32* noalias %d,
+               i32* noalias %e) {
+entry:
+  br label %for.body
+
+; Verify the two distributed loops.
+
+; CHECK: entry.split.ldist1:
+; CHECK:    br label %for.body.ldist1
+; CHECK: for.body.ldist1:
+; CHECK:    %mulA.ldist1 = mul i32 %loadB.ldist1, %loadA.ldist1
+; CHECK:    br i1 %exitcond.ldist1, label %entry.split, label %for.body.ldist1
+
+; CHECK: entry.split:
+; CHECK:    br label %for.body
+; CHECK: for.body:
+; CHECK:    %mulC = mul i32 %loadD, %loadE
+; CHECK: for.end:
+
+
+; ANALYSIS: for.body:
+; ANALYSIS-NEXT: Memory dependences are safe{{$}}
+; ANALYSIS: for.body.ldist1:
+; ANALYSIS-NEXT: Report: unsafe dependent memory operations in loop
+
+
+; VECTORIZE: mul <4 x i32>
+
+for.body:                                         ; preds = %for.body, %entry
+  %ind = phi i64 [ 0, %entry ], [ %add, %for.body ]
+
+  %arrayidxA = getelementptr inbounds i32, i32* %a, i64 %ind
+  %loadA = load i32, i32* %arrayidxA, align 4
+
+  %arrayidxB = getelementptr inbounds i32, i32* %b, i64 %ind
+  %loadB = load i32, i32* %arrayidxB, align 4
+
+  %mulA = mul i32 %loadB, %loadA
+
+  %add = add nuw nsw i64 %ind, 1
+  %arrayidxA_plus_4 = getelementptr inbounds i32, i32* %a, i64 %add
+  store i32 %mulA, i32* %arrayidxA_plus_4, align 4
+
+  %arrayidxD = getelementptr inbounds i32, i32* %d, i64 %ind
+  %loadD = load i32, i32* %arrayidxD, align 4
+
+  %arrayidxE = getelementptr inbounds i32, i32* %e, i64 %ind
+  %loadE = load i32, i32* %arrayidxE, align 4
+
+  %mulC = mul i32 %loadD, %loadE
+
+  %arrayidxC = getelementptr inbounds i32, i32* %c, i64 %ind
+  store i32 %mulC, i32* %arrayidxC, align 4
+
+  %exitcond = icmp eq i64 %add, 20
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:                                          ; preds = %for.body
+  ret void
+}
diff --git a/test/Transforms/LoopDistribute/crash-in-memcheck-generation.ll b/test/Transforms/LoopDistribute/crash-in-memcheck-generation.ll
new file mode 100644
index 00000000000..c81ddf59fa7
--- /dev/null
+++ b/test/Transforms/LoopDistribute/crash-in-memcheck-generation.ll
@@ -0,0 +1,59 @@
+; RUN: opt -basicaa -loop-distribute -loop-vectorize -force-vector-width=4 \
+; RUN:   -verify-loop-info -verify-dom-info -S < %s | FileCheck %s
+
+; If only A and B can alias here, we don't need memchecks to distribute since
+; A and B are in the same partition.  This used to cause a crash in the
+; memcheck generation.
+;
+;   for (i = 0; i < n; i++) {
+;     A[i + 1] = A[i] * B[i];
+; ------------------------------
+;     C[i] = D[i] * E[i];
+;   }
+
+target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-apple-macosx10.10.0"
+
+define void @f(i32*  %a,
+               i32*  %b,
+               i32* noalias %c,
+               i32* noalias %d,
+               i32* noalias %e) {
+entry:
+  br label %for.body
+
+; CHECK-NOT: memcheck:
+; CHECK: mul <4 x i32>
+
+for.body:                                         ; preds = %for.body, %entry
+  %ind = phi i64 [ 0, %entry ], [ %add, %for.body ]
+
+  %arrayidxA = getelementptr inbounds i32, i32* %a, i64 %ind
+  %loadA = load i32, i32* %arrayidxA, align 4
+
+  %arrayidxB = getelementptr inbounds i32, i32* %b, i64 %ind
+  %loadB = load i32, i32* %arrayidxB, align 4
+
+  %mulA = mul i32 %loadB, %loadA
+
+  %add = add nuw nsw i64 %ind, 1
+  %arrayidxA_plus_4 = getelementptr inbounds i32, i32* %a, i64 %add
+  store i32 %mulA, i32* %arrayidxA_plus_4, align 4
+
+  %arrayidxD = getelementptr inbounds i32, i32* %d, i64 %ind
+  %loadD = load i32, i32* %arrayidxD, align 4
+
+  %arrayidxE = getelementptr inbounds i32, i32* %e, i64 %ind
+  %loadE = load i32, i32* %arrayidxE, align 4
+
+  %mulC = mul i32 %loadD, %loadE
+
+  %arrayidxC = getelementptr inbounds i32, i32* %c, i64 %ind
+  store i32 %mulC, i32* %arrayidxC, align 4
+
+  %exitcond = icmp eq i64 %add, 20
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:                                          ; preds = %for.body
+  ret void
+}
diff --git a/test/Transforms/LoopDistribute/no-if-convert.ll b/test/Transforms/LoopDistribute/no-if-convert.ll
new file mode 100644
index 00000000000..fcd8b65731c
--- /dev/null
+++ b/test/Transforms/LoopDistribute/no-if-convert.ll
@@ -0,0 +1,95 @@
+; RUN: opt -basicaa -loop-distribute -verify-loop-info -verify-dom-info -S < %s \
+; RUN:   | FileCheck %s
+
+; We should distribute this loop along === but not along ---.  The last
+; partition won't be vectorized due to conditional stores so it's better to
+; keep it with the second partition which has a dependence cycle.
+
+; (1st statement):
+;   for (i = 0; i < n; i++) {
+;     C[i] = D[i] * E[i];
+;=============================
+;     A[i + 1] = A[i] * B[i];
+;-----------------------------
+;     if (F[i])
+;        G[i] = H[i] * J[i];
+;   }
+
+target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-apple-macosx10.10.0"
+
+define void @f(i32* noalias %a,
+               i32* noalias %b,
+               i32* noalias %c,
+               i32* noalias %d,
+               i32* noalias %e,
+               i32* noalias %g,
+               i32* noalias %h,
+               i32* noalias %j,
+               i64 %x) {
+entry:
+  br label %for.body
+
+; Ensure that we have only two partitions, the first with one multiplication
+; and the second with two.
+
+; CHECK: for.body.ldist1:
+; CHECK:    %mulC.ldist1 = mul i32 %loadD.ldist1, %loadE.ldist1
+; CHECK:    br i1 %exitcond.ldist1, label %entry.split, label %for.body.ldist1
+; CHECK: entry.split:
+; CHECK:    br label %for.body
+; CHECK: for.body:
+; CHECK:    %mulA = mul i32 %loadB, %loadA
+; CHECK:    %mulG = mul i32 %loadH, %loadJ
+; CHECK: for.end:
+
+for.body:                                         ; preds = %for.body, %entry
+  %ind = phi i64 [ 0, %entry ], [ %add, %if.end ]
+
+  %arrayidxD = getelementptr inbounds i32, i32* %d, i64 %ind
+  %loadD = load i32, i32* %arrayidxD, align 4
+
+  %arrayidxE = getelementptr inbounds i32, i32* %e, i64 %ind
+  %loadE = load i32, i32* %arrayidxE, align 4
+
+  %mulC = mul i32 %loadD, %loadE
+
+  %arrayidxC = getelementptr inbounds i32, i32* %c, i64 %ind
+  store i32 %mulC, i32* %arrayidxC, align 4
+
+
+  %arrayidxA = getelementptr inbounds i32, i32* %a, i64 %ind
+  %loadA = load i32, i32* %arrayidxA, align 4
+
+  %arrayidxB = getelementptr inbounds i32, i32* %b, i64 %ind
+  %loadB = load i32, i32* %arrayidxB, align 4
+
+  %mulA = mul i32 %loadB, %loadA
+
+  %add = add nuw nsw i64 %ind, 1
+  %arrayidxA_plus_4 = getelementptr inbounds i32, i32* %a, i64 %add
+  store i32 %mulA, i32* %arrayidxA_plus_4, align 4
+
+  %if.cond = icmp eq i64 %ind, %x
+  br i1 %if.cond, label %if.then, label %if.end
+
+if.then:
+  %arrayidxH = getelementptr inbounds i32, i32* %h, i64 %ind
+  %loadH = load i32, i32* %arrayidxH, align 4
+
+  %arrayidxJ = getelementptr inbounds i32, i32* %j, i64 %ind
+  %loadJ = load i32, i32* %arrayidxJ, align 4
+
+  %mulG = mul i32 %loadH, %loadJ
+
+  %arrayidxG = getelementptr inbounds i32, i32* %g, i64 %ind
+  store i32 %mulG, i32* %arrayidxG, align 4
+  br label %if.end
+
+if.end:
+  %exitcond = icmp eq i64 %add, 20
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:                                          ; preds = %for.body
+  ret void
+}
diff --git a/test/Transforms/LoopDistribute/outside-use.ll b/test/Transforms/LoopDistribute/outside-use.ll
new file mode 100644
index 00000000000..7a3fe1be067
--- /dev/null
+++ b/test/Transforms/LoopDistribute/outside-use.ll
@@ -0,0 +1,69 @@
+; RUN: opt -loop-distribute -verify-loop-info -verify-dom-info -S < %s \
+; RUN:   | FileCheck %s
+
+; Check that definitions used outside the loop are handled correctly: (1) they
+; are not dropped (2) when version the loop, a phi is added to merge the value
+; from the non-distributed loop and the distributed loop.
+;
+;   for (i = 0; i < n; i++) {
+;     A[i + 1] = A[i] * B[i];
+;   ==========================
+;     sum += C[i];
+;   }
+
+target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-apple-macosx10.10.0"
+
+@B = common global i32* null, align 8
+@A = common global i32* null, align 8
+@C = common global i32* null, align 8
+@D = common global i32* null, align 8
+@E = common global i32* null, align 8
+@SUM = common global i32 0, align 8
+
+define void @f() {
+entry:
+  %a = load i32*, i32** @A, align 8
+  %b = load i32*, i32** @B, align 8
+  %c = load i32*, i32** @C, align 8
+  %d = load i32*, i32** @D, align 8
+  %e = load i32*, i32** @E, align 8
+
+  br label %for.body
+
+; CHECK: for.body.ldist1:
+; CHECK:   %mulA.ldist1 = mul i32 %loadB.ldist1, %loadA.ldist1
+; CHECK: for.body.ph:
+; CHECK: for.body:
+; CHECK:   %sum_add = add nuw nsw i32 %sum, %loadC
+; CHECK: for.end:
+; CHECK:   %sum_add.lver = phi i32 [ %sum_add, %for.body ], [ %sum_add.lver.orig, %for.body.lver.orig ]
+
+for.body:                                         ; preds = %for.body, %entry
+  %ind = phi i64 [ 0, %entry ], [ %add, %for.body ]
+  %sum = phi i32 [ 0, %entry ], [ %sum_add, %for.body ]
+
+  %arrayidxA = getelementptr inbounds i32, i32* %a, i64 %ind
+  %loadA = load i32, i32* %arrayidxA, align 4
+
+  %arrayidxB = getelementptr inbounds i32, i32* %b, i64 %ind
+  %loadB = load i32, i32* %arrayidxB, align 4
+
+  %mulA = mul i32 %loadB, %loadA
+
+  %add = add nuw nsw i64 %ind, 1
+  %arrayidxA_plus_4 = getelementptr inbounds i32, i32* %a, i64 %add
+  store i32 %mulA, i32* %arrayidxA_plus_4, align 4
+
+  %arrayidxC = getelementptr inbounds i32, i32* %c, i64 %ind
+  %loadC = load i32, i32* %arrayidxC, align 4
+
+  %sum_add = add nuw nsw i32 %sum, %loadC
+
+  %exitcond = icmp eq i64 %add, 20
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:                                          ; preds = %for.body
+  store i32 %sum_add, i32* @SUM, align 4
+  ret void
+}
diff --git a/test/Transforms/LoopDistribute/program-order.ll b/test/Transforms/LoopDistribute/program-order.ll
new file mode 100644
index 00000000000..b534b79af71
--- /dev/null
+++ b/test/Transforms/LoopDistribute/program-order.ll
@@ -0,0 +1,65 @@
+; RUN: opt -loop-distribute -S -verify-loop-info -verify-dom-info < %s \
+; RUN:   | FileCheck %s
+
+; Distributing this loop to avoid the dependence cycle would require to
+; reorder S1 and S2 to form the two partitions: {S2} | {S1, S3}.  The analysis
+; provided by LoopAccessAnalysis does not allow us to reorder memory
+; operations so make sure we bail on this loop.
+;
+;   for (i = 0; i < n; i++) {
+;     S1: d = D[i];
+;     S2: A[i + 1] = A[i] * B[i];
+;     S3: C[i] = d * E[i];
+;   }
+
+target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-apple-macosx10.10.0"
+
+define void @f(i32* noalias %a,
+               i32* noalias %b,
+               i32* noalias %c,
+               i32* noalias %d,
+               i32* noalias %e) {
+entry:
+  br label %for.body
+
+; CHECK: entry:
+; CHECK:    br label %for.body
+; CHECK: for.body:
+; CHECK:    br i1 %exitcond, label %for.end, label %for.body
+; CHECK: for.end:
+; CHECK:    ret void
+
+for.body:                                         ; preds = %for.body, %entry
+  %ind = phi i64 [ 0, %entry ], [ %add, %for.body ]
+
+  %arrayidxA = getelementptr inbounds i32, i32* %a, i64 %ind
+  %loadA = load i32, i32* %arrayidxA, align 4
+
+  %arrayidxB = getelementptr inbounds i32, i32* %b, i64 %ind
+  %loadB = load i32, i32* %arrayidxB, align 4
+
+  %mulA = mul i32 %loadB, %loadA
+
+  %arrayidxD = getelementptr inbounds i32, i32* %d, i64 %ind
+  %loadD = load i32, i32* %arrayidxD, align 4
+
+  %add = add nuw nsw i64 %ind, 1
+  %arrayidxA_plus_4 = getelementptr inbounds i32, i32* %a, i64 %add
+  store i32 %mulA, i32* %arrayidxA_plus_4, align 4
+
+  %arrayidxC = getelementptr inbounds i32, i32* %c, i64 %ind
+
+  %arrayidxE = getelementptr inbounds i32, i32* %e, i64 %ind
+  %loadE = load i32, i32* %arrayidxE, align 4
+
+  %mulC = mul i32 %loadD, %loadE
+
+  store i32 %mulC, i32* %arrayidxC, align 4
+
+  %exitcond = icmp eq i64 %add, 20
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:                                          ; preds = %for.body
+  ret void
+}
diff --git a/test/Transforms/LoopIdiom/AMDGPU/lit.local.cfg b/test/Transforms/LoopIdiom/AMDGPU/lit.local.cfg
new file mode 100644
index 00000000000..6baccf05fff
--- /dev/null
+++ b/test/Transforms/LoopIdiom/AMDGPU/lit.local.cfg
@@ -0,0 +1,3 @@
+if not 'AMDGPU' in config.root.targets:
+    config.unsupported = True
+
diff --git a/test/Transforms/LoopIdiom/R600/popcnt.ll b/test/Transforms/LoopIdiom/AMDGPU/popcnt.ll
similarity index 100%
rename from test/Transforms/LoopIdiom/R600/popcnt.ll
rename to test/Transforms/LoopIdiom/AMDGPU/popcnt.ll
diff --git a/test/Transforms/LoopIdiom/R600/lit.local.cfg b/test/Transforms/LoopIdiom/R600/lit.local.cfg
deleted file mode 100644
index 4086e8d681c..00000000000
--- a/test/Transforms/LoopIdiom/R600/lit.local.cfg
+++ /dev/null
@@ -1,3 +0,0 @@
-if not 'R600' in config.root.targets:
-    config.unsupported = True
-
diff --git a/test/Transforms/LoopIdiom/basic.ll b/test/Transforms/LoopIdiom/basic.ll
index 488cffc67dc..c633ae95d16 100644
--- a/test/Transforms/LoopIdiom/basic.ll
+++ b/test/Transforms/LoopIdiom/basic.ll
@@ -5,7 +5,7 @@ target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f3
 ; CHECK: @.memset_pattern = private unnamed_addr constant [4 x i32] [i32 1, i32 1, i32 1, i32 1]
 
 ; For @test13_pattern
-; CHECK: @.memset_pattern1 = private unnamed_addr constant [2 x i32*] [i32* @G, i32* @G]
+; CHECK: @.memset_pattern.1 = private unnamed_addr constant [2 x i32*] [i32* @G, i32* @G]
 
 target triple = "x86_64-apple-darwin10.0.0"
 
@@ -69,7 +69,7 @@ for.end:                                          ; preds = %for.body, %entry
   ret void
 ; CHECK-LABEL: @test2(
 ; CHECK: br i1 %cmp10,
-; CHECK: %0 = mul i64 %Size, 4
+; CHECK: %0 = shl i64 %Size, 2
 ; CHECK: call void @llvm.memset.p0i8.i64(i8* %Base1, i8 1, i64 %0, i32 4, i1 false)
 ; CHECK-NOT: store
 }
diff --git a/test/Transforms/LoopIdiom/ctpop-multiple-users-crash.ll b/test/Transforms/LoopIdiom/ctpop-multiple-users-crash.ll
new file mode 100644
index 00000000000..ddb7bdbe7d1
--- /dev/null
+++ b/test/Transforms/LoopIdiom/ctpop-multiple-users-crash.ll
@@ -0,0 +1,34 @@
+; RUN: opt -loop-idiom -S < %s | FileCheck %s
+
+target datalayout = "e-m:o-i64:64-i128:128-n32:64-S128"
+target triple = "arm64-apple-ios8.0.0"
+
+; When we replace the precondition with a ctpop, we need to ensure
+; that only the first branch reads the ctpop.  The store prior
+; to that should continue to read from the original compare.
+
+; CHECK: %tobool.5 = icmp ne i32 %num, 0
+; CHECK: store i1 %tobool.5, i1* %ptr
+
+define internal fastcc i32 @num_bits_set(i32 %num, i1* %ptr) #1 {
+entry:
+  %tobool.5 = icmp ne i32 %num, 0
+  store i1 %tobool.5, i1* %ptr
+  br i1 %tobool.5, label %for.body.lr.ph, label %for.end
+
+for.body.lr.ph:                                   ; preds = %entry
+  br label %for.body
+
+for.body:                                         ; preds = %for.body.lr.ph, %for.body
+  %count.07 = phi i32 [ 0, %for.body.lr.ph ], [ %inc, %for.body ]
+  %num.addr.06 = phi i32 [ %num, %for.body.lr.ph ], [ %and, %for.body ]
+  %sub = add i32 %num.addr.06, -1
+  %and = and i32 %sub, %num.addr.06
+  %inc = add nsw i32 %count.07, 1
+  %tobool = icmp ne i32 %and, 0
+  br i1 %tobool, label %for.body, label %for.end
+
+for.end:                                          ; preds = %for.cond.for.end_crit_edge, %entry
+  %count.0.lcssa = phi i32 [ %inc, %for.body ], [ 0, %entry ]
+  ret i32 %count.0.lcssa
+}
\ No newline at end of file
diff --git a/test/Transforms/LoopIdiom/debug-line.ll b/test/Transforms/LoopIdiom/debug-line.ll
index 1bedf7b1ef9..bcd862d7a72 100644
--- a/test/Transforms/LoopIdiom/debug-line.ll
+++ b/test/Transforms/LoopIdiom/debug-line.ll
@@ -5,8 +5,8 @@ target triple = "x86_64-apple-darwin10.0.0"
 
 define void @foo(double* nocapture %a) nounwind ssp {
 entry:
-  tail call void @llvm.dbg.value(metadata double* %a, i64 0, metadata !5, metadata !MDExpression()), !dbg !8
-  tail call void @llvm.dbg.value(metadata i32 0, i64 0, metadata !10, metadata !MDExpression()), !dbg !14
+  tail call void @llvm.dbg.value(metadata double* %a, i64 0, metadata !5, metadata !DIExpression()), !dbg !8
+  tail call void @llvm.dbg.value(metadata i32 0, i64 0, metadata !10, metadata !DIExpression()), !dbg !14
   br label %for.body
 
 for.body:                                         ; preds = %entry, %for.body
@@ -19,7 +19,7 @@ for.body:                                         ; preds = %entry, %for.body
   br i1 %exitcond, label %for.body, label %for.end, !dbg !14
 
 for.end:                                          ; preds = %for.body
-  tail call void @llvm.dbg.value(metadata !{null}, i64 0, metadata !10, metadata !MDExpression()), !dbg !16
+  tail call void @llvm.dbg.value(metadata !{null}, i64 0, metadata !10, metadata !DIExpression()), !dbg !16
   ret void, !dbg !17
 }
 
@@ -30,23 +30,23 @@ declare void @llvm.dbg.value(metadata, i64, metadata, metadata) nounwind readnon
 !llvm.module.flags = !{!19}
 !llvm.dbg.sp = !{!0}
 
-!0 = !MDSubprogram(name: "foo", line: 2, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, file: !18, scope: !1, type: !3, function: void (double*)* @foo)
-!1 = !MDFile(filename: "li.c", directory: "/private/tmp")
-!2 = !MDCompileUnit(language: DW_LANG_C99, producer: "clang version 2.9 (trunk 127165:127174)", isOptimized: true, emissionKind: 0, file: !18, enums: !9, retainedTypes: !9)
-!3 = !MDSubroutineType(types: !4)
+!0 = !DISubprogram(name: "foo", line: 2, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, file: !18, scope: !1, type: !3, function: void (double*)* @foo)
+!1 = !DIFile(filename: "li.c", directory: "/private/tmp")
+!2 = !DICompileUnit(language: DW_LANG_C99, producer: "clang version 2.9 (trunk 127165:127174)", isOptimized: true, emissionKind: 0, file: !18, enums: !9, retainedTypes: !9)
+!3 = !DISubroutineType(types: !4)
 !4 = !{null}
-!5 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "a", line: 2, arg: 1, scope: !0, file: !1, type: !6)
-!6 = !MDDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, scope: !2, baseType: !7)
-!7 = !MDBasicType(tag: DW_TAG_base_type, name: "double", size: 64, align: 64, encoding: DW_ATE_float)
-!8 = !MDLocation(line: 2, column: 18, scope: !0)
+!5 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "a", line: 2, arg: 1, scope: !0, file: !1, type: !6)
+!6 = !DIDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, scope: !2, baseType: !7)
+!7 = !DIBasicType(tag: DW_TAG_base_type, name: "double", size: 64, align: 64, encoding: DW_ATE_float)
+!8 = !DILocation(line: 2, column: 18, scope: !0)
 !9 = !{}
-!10 = !MDLocalVariable(tag: DW_TAG_auto_variable, name: "i", line: 3, scope: !11, file: !1, type: !13)
-!11 = distinct !MDLexicalBlock(line: 3, column: 3, file: !18, scope: !12)
-!12 = distinct !MDLexicalBlock(line: 2, column: 21, file: !18, scope: !0)
-!13 = !MDBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
-!14 = !MDLocation(line: 3, column: 3, scope: !12)
-!15 = !MDLocation(line: 4, column: 5, scope: !11)
-!16 = !MDLocation(line: 3, column: 29, scope: !11)
-!17 = !MDLocation(line: 5, column: 1, scope: !12)
-!18 = !MDFile(filename: "li.c", directory: "/private/tmp")
+!10 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "i", line: 3, scope: !11, file: !1, type: !13)
+!11 = distinct !DILexicalBlock(line: 3, column: 3, file: !18, scope: !12)
+!12 = distinct !DILexicalBlock(line: 2, column: 21, file: !18, scope: !0)
+!13 = !DIBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
+!14 = !DILocation(line: 3, column: 3, scope: !12)
+!15 = !DILocation(line: 4, column: 5, scope: !11)
+!16 = !DILocation(line: 3, column: 29, scope: !11)
+!17 = !DILocation(line: 5, column: 1, scope: !12)
+!18 = !DIFile(filename: "li.c", directory: "/private/tmp")
 !19 = !{i32 1, !"Debug Info Version", i32 3}
diff --git a/test/Transforms/LoopInterchange/reductions.ll b/test/Transforms/LoopInterchange/reductions.ll
new file mode 100644
index 00000000000..86a44da7cf2
--- /dev/null
+++ b/test/Transforms/LoopInterchange/reductions.ll
@@ -0,0 +1,235 @@
+; RUN: opt < %s -basicaa -loop-interchange -S | FileCheck %s
+
+@A = common global [500 x [500 x i32]] zeroinitializer
+@X = common global i32 0
+@B = common global [500 x [500 x i32]] zeroinitializer
+@Y = common global i32 0
+
+;;  for( int i=1;i<N;i++)
+;;    for( int j=1;j<N;j++)
+;;      X+=A[j][i];
+
+define void @reduction_01(i32 %N) {
+entry:
+  %cmp16 = icmp sgt i32 %N, 1
+  br i1 %cmp16, label %for.body3.lr.ph, label %for.end8
+
+for.body3.lr.ph:                                  ; preds = %entry, %for.cond1.for.inc6_crit_edge
+  %indvars.iv18 = phi i64 [ %indvars.iv.next19, %for.cond1.for.inc6_crit_edge ], [ 1, %entry ]
+  %X.promoted = load i32, i32* @X
+  br label %for.body3
+
+for.body3:                                        ; preds = %for.body3, %for.body3.lr.ph
+  %indvars.iv = phi i64 [ 1, %for.body3.lr.ph ], [ %indvars.iv.next, %for.body3 ]
+  %add15 = phi i32 [ %X.promoted, %for.body3.lr.ph ], [ %add, %for.body3 ]
+  %arrayidx5 = getelementptr inbounds [500 x [500 x i32]], [500 x [500 x i32]]* @A, i64 0, i64 %indvars.iv, i64 %indvars.iv18
+  %0 = load i32, i32* %arrayidx5
+  %add = add nsw i32 %add15, %0
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+  %lftr.wideiv = trunc i64 %indvars.iv.next to i32
+  %exitcond = icmp eq i32 %lftr.wideiv, %N
+  br i1 %exitcond, label %for.cond1.for.inc6_crit_edge, label %for.body3
+
+for.cond1.for.inc6_crit_edge:                     ; preds = %for.body3
+  store i32 %add, i32* @X
+  %indvars.iv.next19 = add nuw nsw i64 %indvars.iv18, 1
+  %lftr.wideiv20 = trunc i64 %indvars.iv.next19 to i32
+  %exitcond21 = icmp eq i32 %lftr.wideiv20, %N
+  br i1 %exitcond21, label %for.end8, label %for.body3.lr.ph
+
+for.end8:                                         ; preds = %for.cond1.for.inc6_crit_edge, %entry
+  ret void
+}
+
+;; Loop is interchanged check that the phi nodes are split and the promoted value is used instead of the reduction phi.
+; CHECK-LABEL: @reduction_01
+; CHECK:  for.body3:                                        ; preds = %for.body3.preheader, %for.body3.split
+; CHECK:    %indvars.iv = phi i64 [ %indvars.iv.next, %for.body3.split ], [ 1, %for.body3.preheader ]
+; CHECK:    br label %for.body3.lr.ph.preheader
+; CHECK:    %add = add nsw i32 %X.promoted
+
+
+;; Test for more than 1 reductions inside a loop.
+;;  for( int i=1;i<N;i++)
+;;    for( int j=1;j<N;j++)
+;;      for( int k=1;k<N;k++) {
+;;        X+=A[k][j];
+;;        Y+=B[k][i];
+;;      }
+
+define void @reduction_02(i32 %N)  {
+entry:
+  %cmp34 = icmp sgt i32 %N, 1
+  br i1 %cmp34, label %for.cond4.preheader.preheader, label %for.end19
+
+for.cond4.preheader.preheader:                    ; preds = %entry, %for.inc17
+  %indvars.iv40 = phi i64 [ %indvars.iv.next41, %for.inc17 ], [ 1, %entry ]
+  br label %for.body6.lr.ph
+
+for.body6.lr.ph:                                  ; preds = %for.cond4.for.inc14_crit_edge, %for.cond4.preheader.preheader
+  %indvars.iv36 = phi i64 [ %indvars.iv.next37, %for.cond4.for.inc14_crit_edge ], [ 1, %for.cond4.preheader.preheader ]
+  %X.promoted = load i32, i32* @X
+  %Y.promoted = load i32, i32* @Y
+  br label %for.body6
+
+for.body6:                                        ; preds = %for.body6, %for.body6.lr.ph
+  %indvars.iv = phi i64 [ 1, %for.body6.lr.ph ], [ %indvars.iv.next, %for.body6 ]
+  %add1331 = phi i32 [ %Y.promoted, %for.body6.lr.ph ], [ %add13, %for.body6 ]
+  %add30 = phi i32 [ %X.promoted, %for.body6.lr.ph ], [ %add, %for.body6 ]
+  %arrayidx8 = getelementptr inbounds [500 x [500 x i32]], [500 x [500 x i32]]* @A, i64 0, i64 %indvars.iv, i64 %indvars.iv36
+  %0 = load i32, i32* %arrayidx8
+  %add = add nsw i32 %add30, %0
+  %arrayidx12 = getelementptr inbounds [500 x [500 x i32]], [500 x [500 x i32]]* @B, i64 0, i64 %indvars.iv, i64 %indvars.iv40
+  %1 = load i32, i32* %arrayidx12
+  %add13 = add nsw i32 %add1331, %1
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+  %lftr.wideiv = trunc i64 %indvars.iv.next to i32
+  %exitcond = icmp eq i32 %lftr.wideiv, %N
+  br i1 %exitcond, label %for.cond4.for.inc14_crit_edge, label %for.body6
+
+for.cond4.for.inc14_crit_edge:                    ; preds = %for.body6
+  store i32 %add, i32* @X
+  store i32 %add13, i32* @Y
+  %indvars.iv.next37 = add nuw nsw i64 %indvars.iv36, 1
+  %lftr.wideiv38 = trunc i64 %indvars.iv.next37 to i32
+  %exitcond39 = icmp eq i32 %lftr.wideiv38, %N
+  br i1 %exitcond39, label %for.inc17, label %for.body6.lr.ph
+
+for.inc17:                                        ; preds = %for.cond4.for.inc14_crit_edge
+  %indvars.iv.next41 = add nuw nsw i64 %indvars.iv40, 1
+  %lftr.wideiv42 = trunc i64 %indvars.iv.next41 to i32
+  %exitcond43 = icmp eq i32 %lftr.wideiv42, %N
+  br i1 %exitcond43, label %for.end19, label %for.cond4.preheader.preheader
+
+for.end19:                                        ; preds = %for.inc17, %entry
+  ret void
+}
+
+;; Loop is interchanged check that the phi nodes are split and the promoted value is used instead of the reduction phi.
+; CHECK-LABEL: @reduction_02
+; CHECK:  for.body6:                                        ; preds = %for.body6.preheader, %for.body6.split
+; CHECK:    %indvars.iv = phi i64 [ %indvars.iv.next, %for.body6.split ], [ 1, %for.body6.preheader ]
+; CHECK:    br label %for.cond4.preheader.preheader.preheader
+; CHECK:    %add13 = add nsw i32 %Y.promoted
+
+
+;; Not tightly nested. Do not interchange.
+;;  for( int i=1;i<N;i++)
+;;    for( int j=1;j<N;j++) {
+;;      for( int k=1;k<N;k++) {
+;;        X+=A[k][j];
+;;      }
+;;      Y+=B[j][i];
+;;    }
+define void @reduction_03(i32 %N)  {
+entry:
+  %cmp35 = icmp sgt i32 %N, 1
+  br i1 %cmp35, label %for.cond4.preheader.lr.ph, label %for.end19
+
+for.cond4.preheader.lr.ph:                        ; preds = %entry, %for.cond1.for.inc17_crit_edge
+  %indvars.iv41 = phi i64 [ %indvars.iv.next42, %for.cond1.for.inc17_crit_edge ], [ 1, %entry ]
+  %Y.promoted = load i32, i32* @Y
+  br label %for.body6.lr.ph
+
+for.body6.lr.ph:                                  ; preds = %for.cond4.preheader.lr.ph, %for.cond4.for.end_crit_edge
+  %indvars.iv37 = phi i64 [ 1, %for.cond4.preheader.lr.ph ], [ %indvars.iv.next38, %for.cond4.for.end_crit_edge ]
+  %add1334 = phi i32 [ %Y.promoted, %for.cond4.preheader.lr.ph ], [ %add13, %for.cond4.for.end_crit_edge ]
+  %X.promoted = load i32, i32* @X
+  br label %for.body6
+
+for.body6:                                        ; preds = %for.body6, %for.body6.lr.ph
+  %indvars.iv = phi i64 [ 1, %for.body6.lr.ph ], [ %indvars.iv.next, %for.body6 ]
+  %add31 = phi i32 [ %X.promoted, %for.body6.lr.ph ], [ %add, %for.body6 ]
+  %arrayidx8 = getelementptr inbounds [500 x [500 x i32]], [500 x [500 x i32]]* @A, i64 0, i64 %indvars.iv, i64 %indvars.iv37
+  %0 = load i32, i32* %arrayidx8
+  %add = add nsw i32 %add31, %0
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+  %lftr.wideiv = trunc i64 %indvars.iv.next to i32
+  %exitcond = icmp eq i32 %lftr.wideiv, %N
+  br i1 %exitcond, label %for.cond4.for.end_crit_edge, label %for.body6
+
+for.cond4.for.end_crit_edge:                      ; preds = %for.body6
+  store i32 %add, i32* @X
+  %arrayidx12 = getelementptr inbounds [500 x [500 x i32]], [500 x [500 x i32]]* @B, i64 0, i64 %indvars.iv37, i64 %indvars.iv41
+  %1 = load i32, i32* %arrayidx12
+  %add13 = add nsw i32 %add1334, %1
+  %indvars.iv.next38 = add nuw nsw i64 %indvars.iv37, 1
+  %lftr.wideiv39 = trunc i64 %indvars.iv.next38 to i32
+  %exitcond40 = icmp eq i32 %lftr.wideiv39, %N
+  br i1 %exitcond40, label %for.cond1.for.inc17_crit_edge, label %for.body6.lr.ph
+
+for.cond1.for.inc17_crit_edge:                    ; preds = %for.cond4.for.end_crit_edge
+  store i32 %add13, i32* @Y
+  %indvars.iv.next42 = add nuw nsw i64 %indvars.iv41, 1
+  %lftr.wideiv43 = trunc i64 %indvars.iv.next42 to i32
+  %exitcond44 = icmp eq i32 %lftr.wideiv43, %N
+  br i1 %exitcond44, label %for.end19, label %for.cond4.preheader.lr.ph
+
+for.end19:                                        ; preds = %for.cond1.for.inc17_crit_edge, %entry
+  ret void
+}
+;; Not tightly nested. Do not interchange.
+;; Not interchanged hence the phi's in the inner loop will not be split. Check for the same.
+; CHECK-LABEL: @reduction_03
+; CHECK:  for.body6:                                        ; preds = %for.body6.preheader, %for.body6
+; CHECK:    %indvars.iv = phi i64 [ %indvars.iv.next, %for.body6 ], [ 1, %for.body6.preheader ]
+; CHECK:    %add31 = phi i32 [ %add, %for.body6 ], [ %X.promoted, %for.body6.preheader ]
+
+
+
+;; Multiple use of reduction not safe. Do not interchange.
+;;  for( int i=1;i<N;i++)
+;;    for( int j=1;j<N;j++)
+;;      for( int k=1;k<N;k++) {
+;;        X+=A[k][j];
+;;        Y+=X;
+;;      }
+define void @reduction_04(i32 %N) {
+entry:
+  %cmp28 = icmp sgt i32 %N, 1
+  br i1 %cmp28, label %for.cond4.preheader.preheader, label %for.end15
+
+for.cond4.preheader.preheader:                    ; preds = %entry, %for.inc13
+  %i.029 = phi i32 [ %inc14, %for.inc13 ], [ 1, %entry ]
+  br label %for.body6.lr.ph
+
+for.body6.lr.ph:                                  ; preds = %for.cond4.for.inc10_crit_edge, %for.cond4.preheader.preheader
+  %indvars.iv30 = phi i64 [ %indvars.iv.next31, %for.cond4.for.inc10_crit_edge ], [ 1, %for.cond4.preheader.preheader ]
+  %X.promoted = load i32, i32* @X
+  %Y.promoted = load i32, i32* @Y
+  br label %for.body6
+
+for.body6:                                        ; preds = %for.body6, %for.body6.lr.ph
+  %indvars.iv = phi i64 [ 1, %for.body6.lr.ph ], [ %indvars.iv.next, %for.body6 ]
+  %add925 = phi i32 [ %Y.promoted, %for.body6.lr.ph ], [ %add9, %for.body6 ]
+  %add24 = phi i32 [ %X.promoted, %for.body6.lr.ph ], [ %add, %for.body6 ]
+  %arrayidx8 = getelementptr inbounds [500 x [500 x i32]], [500 x [500 x i32]]* @A, i64 0, i64 %indvars.iv, i64 %indvars.iv30
+  %0 = load i32, i32* %arrayidx8
+  %add = add nsw i32 %add24, %0
+  %add9 = add nsw i32 %add925, %add
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+  %lftr.wideiv = trunc i64 %indvars.iv.next to i32
+  %exitcond = icmp eq i32 %lftr.wideiv, %N
+  br i1 %exitcond, label %for.cond4.for.inc10_crit_edge, label %for.body6
+
+for.cond4.for.inc10_crit_edge:                    ; preds = %for.body6
+  store i32 %add, i32* @X
+  store i32 %add9, i32* @Y
+  %indvars.iv.next31 = add nuw nsw i64 %indvars.iv30, 1
+  %lftr.wideiv32 = trunc i64 %indvars.iv.next31 to i32
+  %exitcond33 = icmp eq i32 %lftr.wideiv32, %N
+  br i1 %exitcond33, label %for.inc13, label %for.body6.lr.ph
+
+for.inc13:                                        ; preds = %for.cond4.for.inc10_crit_edge
+  %inc14 = add nuw nsw i32 %i.029, 1
+  %exitcond34 = icmp eq i32 %inc14, %N
+  br i1 %exitcond34, label %for.end15, label %for.cond4.preheader.preheader
+
+for.end15:                                        ; preds = %for.inc13, %entry
+  ret void
+}
+;; Not interchanged hence the phi's in the inner loop will not be split. Check for the same.
+; CHECK-LABEL: @reduction_04
+; CHECK:  for.body6:                                        ; preds = %for.body6.preheader, %for.body6
+; CHECK:    %indvars.iv = phi i64 [ %indvars.iv.next, %for.body6 ], [ 1, %for.body6.preheader ]
+; CHECK:    %add925 = phi i32 [ %add9, %for.body6 ], [ %Y.promoted, %for.body6.preheader ]
diff --git a/test/Transforms/LoopReroll/negative.ll b/test/Transforms/LoopReroll/negative.ll
new file mode 100644
index 00000000000..36f6806e1c3
--- /dev/null
+++ b/test/Transforms/LoopReroll/negative.ll
@@ -0,0 +1,48 @@
+; RUN: opt -S  -loop-reroll   %s | FileCheck %s
+target triple = "aarch64--linux-gnu"
+@buf = global [16 x i8] c"\0A\0A\0A\0A\0A\0A\0A\0A\0A\0A\0A\0A\0A\0A\0A\0A", align 1
+
+define i32 @test1(i32 %len, i8* nocapture readonly %buf) #0 {
+entry:
+  %cmp.13 = icmp sgt i32 %len, 1
+  br i1 %cmp.13, label %while.body.lr.ph, label %while.end
+
+while.body.lr.ph:                                 ; preds = %entry
+  br label %while.body
+
+while.body:
+;CHECK-LABEL: while.body:
+;CHECK-NEXT:    %indvar = phi i32 [ %indvar.next, %while.body ], [ 0, %while.body.lr.ph ]
+;CHECK-NEXT:    %sum4.015 = phi i64 [ 0, %while.body.lr.ph ], [ %add, %while.body ]
+;CHECK-NOT:     %sub5 = add nsw i32 %len.addr.014, -1
+;CHECK-NOT:     %sub5 = add nsw i32 %len.addr.014, -2
+;CHECK:    br i1 %exitcond, label %while.cond.while.end_crit_edge, label %while.body
+
+  %sum4.015 = phi i64 [ 0, %while.body.lr.ph ], [ %add4, %while.body ]
+  %len.addr.014 = phi i32 [ %len, %while.body.lr.ph ], [ %sub5, %while.body ]
+  %idxprom = sext i32 %len.addr.014 to i64
+  %arrayidx = getelementptr inbounds i8, i8* %buf, i64 %idxprom
+  %0 = load i8, i8* %arrayidx, align 1
+  %conv = zext i8 %0 to i64
+  %add = add i64 %conv, %sum4.015
+  %sub = add nsw i32 %len.addr.014, -1
+  %idxprom1 = sext i32 %sub to i64
+  %arrayidx2 = getelementptr inbounds i8, i8* %buf, i64 %idxprom1
+  %1 = load i8, i8* %arrayidx2, align 1
+  %conv3 = zext i8 %1 to i64
+  %add4 = add i64 %add, %conv3
+  %sub5 = add nsw i32 %len.addr.014, -2
+  %cmp = icmp sgt i32 %sub5, 1
+  br i1 %cmp, label %while.body, label %while.cond.while.end_crit_edge
+
+while.cond.while.end_crit_edge:                   ; preds = %while.body
+  %add4.lcssa = phi i64 [ %add4, %while.body ]
+  %phitmp = trunc i64 %add4.lcssa to i32
+  br label %while.end
+
+while.end:                                        ; preds = %while.cond.while.end_crit_edge, %entry
+  %sum4.0.lcssa = phi i32 [ %phitmp, %while.cond.while.end_crit_edge ], [ 0, %entry ]
+  ret i32 %sum4.0.lcssa
+  unreachable
+}
+
diff --git a/test/Transforms/LoopReroll/nonconst_lb.ll b/test/Transforms/LoopReroll/nonconst_lb.ll
index 5effa42d07c..96090e8dc0b 100644
--- a/test/Transforms/LoopReroll/nonconst_lb.ll
+++ b/test/Transforms/LoopReroll/nonconst_lb.ll
@@ -52,7 +52,7 @@ for.end:                                          ; preds = %for.body, %entry
 ; CHECK:   %0 = add i32 %n, -1
 ; CHECK:   %1 = sub i32 %0, %m
 ; CHECK:   %2 = lshr i32 %1, 2
-; CHECK:   %3 = mul i32 %2, 4
+; CHECK:   %3 = shl i32 %2, 2
 ; CHECK:   %4 = add i32 %m, %3
 ; CHECK:   %5 = add i32 %4, 3
 ; CHECK:   br label %for.body
@@ -132,7 +132,7 @@ for.end:                                          ; preds = %for.body, %entry
 ; CHECK:   %0 = add i32 %n, -1
 ; CHECK:   %1 = sub i32 %0, %rem
 ; CHECK:   %2 = lshr i32 %1, 2
-; CHECK:   %3 = mul i32 %2, 4
+; CHECK:   %3 = shl i32 %2, 2
 ; CHECK:   %4 = add i32 %rem, %3
 ; CHECK:   %5 = add i32 %4, 3
 ; CHECK:   br label %for.body
diff --git a/test/Transforms/LoopRotate/dbgvalue.ll b/test/Transforms/LoopRotate/dbgvalue.ll
index 88348b0288c..d65884698d4 100644
--- a/test/Transforms/LoopRotate/dbgvalue.ll
+++ b/test/Transforms/LoopRotate/dbgvalue.ll
@@ -15,9 +15,9 @@ tailrecurse:                                      ; preds = %if.then, %entry
   %x.tr = phi i32 [ %x, %entry ], [ %call, %if.then ]
   %y.tr = phi i32 [ %y, %entry ], [ %call9, %if.then ]
   %z.tr = phi i32 [ %z, %entry ], [ %call14, %if.then ]
-  tail call void @llvm.dbg.value(metadata i32 %x.tr, i64 0, metadata !6, metadata !MDExpression()), !dbg !7
-  tail call void @llvm.dbg.value(metadata i32 %y.tr, i64 0, metadata !8, metadata !MDExpression()), !dbg !9
-  tail call void @llvm.dbg.value(metadata i32 %z.tr, i64 0, metadata !10, metadata !MDExpression()), !dbg !11
+  tail call void @llvm.dbg.value(metadata i32 %x.tr, i64 0, metadata !6, metadata !DIExpression()), !dbg !7
+  tail call void @llvm.dbg.value(metadata i32 %y.tr, i64 0, metadata !8, metadata !DIExpression()), !dbg !9
+  tail call void @llvm.dbg.value(metadata i32 %z.tr, i64 0, metadata !10, metadata !DIExpression()), !dbg !11
   %cmp = icmp slt i32 %y.tr, %x.tr, !dbg !12
   br i1 %cmp, label %if.then, label %if.end, !dbg !12
 
@@ -72,7 +72,7 @@ for.body:
 
 for.inc:
   %dec = add i64 %i.0, -1
-  tail call void @llvm.dbg.value(metadata i64 %dec, i64 0, metadata !MDLocalVariable(tag: DW_TAG_auto_variable, scope: !0), metadata !MDExpression()), !dbg !MDLocation(scope: !0)
+  tail call void @llvm.dbg.value(metadata i64 %dec, i64 0, metadata !DILocalVariable(tag: DW_TAG_auto_variable, scope: !0), metadata !DIExpression()), !dbg !DILocation(scope: !0)
   br label %for.cond
 
 for.end:
@@ -84,24 +84,24 @@ for.end:
 !llvm.module.flags = !{!20}
 !llvm.dbg.sp = !{!0}
 
-!0 = !MDSubprogram(name: "tak", line: 32, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, file: !18, scope: !1, type: !3, function: i32 (i32, i32, i32)* @tak)
-!1 = !MDFile(filename: "/Volumes/Lalgate/cj/llvm/projects/llvm-test/SingleSource/Benchmarks/BenchmarkGame/recursive.c", directory: "/Volumes/Lalgate/cj/D/projects/llvm-test/SingleSource/Benchmarks/BenchmarkGame")
-!2 = !MDCompileUnit(language: DW_LANG_C99, producer: "clang version 2.9 (trunk 125492)", isOptimized: true, emissionKind: 0, file: !18, enums: !19, retainedTypes: !19)
-!3 = !MDSubroutineType(types: !4)
+!0 = !DISubprogram(name: "tak", line: 32, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, file: !18, scope: !1, type: !3, function: i32 (i32, i32, i32)* @tak)
+!1 = !DIFile(filename: "/Volumes/Lalgate/cj/llvm/projects/llvm-test/SingleSource/Benchmarks/BenchmarkGame/recursive.c", directory: "/Volumes/Lalgate/cj/D/projects/llvm-test/SingleSource/Benchmarks/BenchmarkGame")
+!2 = !DICompileUnit(language: DW_LANG_C99, producer: "clang version 2.9 (trunk 125492)", isOptimized: true, emissionKind: 0, file: !18, enums: !19, retainedTypes: !19)
+!3 = !DISubroutineType(types: !4)
 !4 = !{!5}
-!5 = !MDBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
-!6 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "x", line: 32, arg: 0, scope: !0, file: !1, type: !5)
-!7 = !MDLocation(line: 32, column: 13, scope: !0)
-!8 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "y", line: 32, arg: 0, scope: !0, file: !1, type: !5)
-!9 = !MDLocation(line: 32, column: 20, scope: !0)
-!10 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "z", line: 32, arg: 0, scope: !0, file: !1, type: !5)
-!11 = !MDLocation(line: 32, column: 27, scope: !0)
-!12 = !MDLocation(line: 33, column: 3, scope: !13)
-!13 = distinct !MDLexicalBlock(line: 32, column: 30, file: !18, scope: !0)
-!14 = !MDLocation(line: 34, column: 5, scope: !15)
-!15 = distinct !MDLexicalBlock(line: 33, column: 14, file: !18, scope: !13)
-!16 = !MDLocation(line: 36, column: 3, scope: !13)
-!17 = !MDLocation(line: 37, column: 1, scope: !13)
-!18 = !MDFile(filename: "/Volumes/Lalgate/cj/llvm/projects/llvm-test/SingleSource/Benchmarks/BenchmarkGame/recursive.c", directory: "/Volumes/Lalgate/cj/D/projects/llvm-test/SingleSource/Benchmarks/BenchmarkGame")
+!5 = !DIBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
+!6 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "x", line: 32, arg: 1, scope: !0, file: !1, type: !5)
+!7 = !DILocation(line: 32, column: 13, scope: !0)
+!8 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "y", line: 32, arg: 2, scope: !0, file: !1, type: !5)
+!9 = !DILocation(line: 32, column: 20, scope: !0)
+!10 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "z", line: 32, arg: 3, scope: !0, file: !1, type: !5)
+!11 = !DILocation(line: 32, column: 27, scope: !0)
+!12 = !DILocation(line: 33, column: 3, scope: !13)
+!13 = distinct !DILexicalBlock(line: 32, column: 30, file: !18, scope: !0)
+!14 = !DILocation(line: 34, column: 5, scope: !15)
+!15 = distinct !DILexicalBlock(line: 33, column: 14, file: !18, scope: !13)
+!16 = !DILocation(line: 36, column: 3, scope: !13)
+!17 = !DILocation(line: 37, column: 1, scope: !13)
+!18 = !DIFile(filename: "/Volumes/Lalgate/cj/llvm/projects/llvm-test/SingleSource/Benchmarks/BenchmarkGame/recursive.c", directory: "/Volumes/Lalgate/cj/D/projects/llvm-test/SingleSource/Benchmarks/BenchmarkGame")
 !19 = !{i32 0}
 !20 = !{i32 1, !"Debug Info Version", i32 3}
diff --git a/test/Transforms/LoopRotate/multiple-exits.ll b/test/Transforms/LoopRotate/multiple-exits.ll
index f31ed7f1c5b..f38c855b9c8 100644
--- a/test/Transforms/LoopRotate/multiple-exits.ll
+++ b/test/Transforms/LoopRotate/multiple-exits.ll
@@ -87,7 +87,7 @@ declare i32 @bar(i32)
 @_ZTIi = external constant i8*
 
 ; Verify dominators.
-define void @test3(i32 %x) {
+define void @test3(i32 %x) personality i8* bitcast (i32 (...)* @__gxx_personality_v0 to i8*) {
 entry:
   %cmp2 = icmp eq i32 0, %x
   br i1 %cmp2, label %try.cont.loopexit, label %for.body.lr.ph
@@ -106,7 +106,7 @@ for.inc:                                          ; preds = %for.body
   br i1 %cmp, label %for.cond.try.cont.loopexit_crit_edge, label %for.body
 
 lpad:                                             ; preds = %for.body
-  %0 = landingpad { i8*, i32 } personality i8* bitcast (i32 (...)* @__gxx_personality_v0 to i8*)
+  %0 = landingpad { i8*, i32 }
           catch i8* bitcast (i8** @_ZTIi to i8*)
   %1 = extractvalue { i8*, i32 } %0, 0
   %2 = extractvalue { i8*, i32 } %0, 1
@@ -132,7 +132,7 @@ for.inc.i:                                        ; preds = %for.body.i
   br i1 %cmp.i, label %for.cond.i.invoke.cont2.loopexit_crit_edge, label %for.body.i
 
 lpad.i:                                           ; preds = %for.body.i
-  %5 = landingpad { i8*, i32 } personality i8* bitcast (i32 (...)* @__gxx_personality_v0 to i8*)
+  %5 = landingpad { i8*, i32 }
           catch i8* bitcast (i8** @_ZTIi to i8*)
   %6 = extractvalue { i8*, i32 } %5, 0
   %7 = extractvalue { i8*, i32 } %5, 1
@@ -149,7 +149,7 @@ invoke.cont2.i:                                   ; preds = %catch.i
   br label %invoke.cont2
 
 lpad1.i:                                          ; preds = %catch.i
-  %9 = landingpad { i8*, i32 } personality i8* bitcast (i32 (...)* @__gxx_personality_v0 to i8*)
+  %9 = landingpad { i8*, i32 }
           cleanup
   %10 = extractvalue { i8*, i32 } %9, 0
   %11 = extractvalue { i8*, i32 } %9, 1
diff --git a/test/Transforms/LoopRotate/oz-disable.ll b/test/Transforms/LoopRotate/oz-disable.ll
new file mode 100644
index 00000000000..7a6a9bf33a1
--- /dev/null
+++ b/test/Transforms/LoopRotate/oz-disable.ll
@@ -0,0 +1,30 @@
+; REQUIRES: asserts
+; RUN: opt < %s -S -Os -debug -debug-only=loop-rotate 2>&1 | FileCheck %s -check-prefix=OS
+; RUN: opt < %s -S -Oz -debug -debug-only=loop-rotate 2>&1 | FileCheck %s -check-prefix=OZ
+
+; Loop should be rotated for -Os but not for -Oz.
+; OS: rotating Loop at depth 1
+; OZ-NOT: rotating Loop at depth 1
+
+@e = global i32 10
+
+declare void @use(i32)
+
+define void @test() {
+entry:
+  %end = load i32, i32* @e
+  br label %loop
+
+loop:
+  %n.phi = phi i32 [ %n, %loop.fin ], [ 0, %entry ]
+  %cond = icmp eq i32 %n.phi, %end
+  br i1 %cond, label %exit, label %loop.fin
+
+loop.fin:
+  %n = add i32 %n.phi, 1
+  call void @use(i32 %n)
+  br label %loop
+
+exit:
+  ret void
+}
diff --git a/test/Transforms/LoopSimplify/2007-10-28-InvokeCrash.ll b/test/Transforms/LoopSimplify/2007-10-28-InvokeCrash.ll
index 0534a0bf7d0..9f65d68202f 100644
--- a/test/Transforms/LoopSimplify/2007-10-28-InvokeCrash.ll
+++ b/test/Transforms/LoopSimplify/2007-10-28-InvokeCrash.ll
@@ -3,7 +3,7 @@
 target datalayout = "e-p:32:32:32-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-f32:32:32-f64:32:64-v64:64:64-v128:128:128-a0:0:64-s0:0:64-f80:32:32"
 target triple = "i686-pc-mingw32"
 
-define void @func() {
+define void @func() personality i32 (...)* @__gxx_personality_v0 {
 bb_init:
 	br label %bb_main
 
@@ -18,7 +18,7 @@ invcont17.normaldest:		; preds = %invcont17.normaldest917, %bb_main
 	br label %bb_main
 
 invcont17.normaldest.normaldest:		; No predecessors!
-        %exn = landingpad {i8*, i32} personality i32 (...)* @__gxx_personality_v0
+        %exn = landingpad {i8*, i32}
                  catch i8* null
         store i32 %tmp23, i32* undef
 	br label %bb_main
diff --git a/test/Transforms/LoopSimplify/2011-12-14-LandingpadHeader.ll b/test/Transforms/LoopSimplify/2011-12-14-LandingpadHeader.ll
index 39471eb0d11..cb9dd4124e2 100644
--- a/test/Transforms/LoopSimplify/2011-12-14-LandingpadHeader.ll
+++ b/test/Transforms/LoopSimplify/2011-12-14-LandingpadHeader.ll
@@ -3,7 +3,7 @@
 
 @catchtypeinfo = external unnamed_addr constant { i8*, i8*, i8* }
 
-define void @main() uwtable ssp {
+define void @main() uwtable ssp personality i8* bitcast (i32 (...)* @__gxx_personality_v0 to i8*) {
 entry:
   invoke void @f1()
           to label %try.cont19 unwind label %catch
@@ -17,7 +17,7 @@ entry:
 ; CHECK: br label %catch
 
 catch:                                            ; preds = %if.else, %entry
-  %0 = landingpad { i8*, i32 } personality i8* bitcast (i32 (...)* @__gxx_personality_v0 to i8*)
+  %0 = landingpad { i8*, i32 }
           catch i8* bitcast ({ i8*, i8*, i8* }* @catchtypeinfo to i8*)
   invoke void @f3()
           to label %if.else unwind label %eh.resume
@@ -30,7 +30,7 @@ try.cont19:                                       ; preds = %if.else, %entry
   ret void
 
 eh.resume:                                        ; preds = %catch
-  %1 = landingpad { i8*, i32 } personality i8* bitcast (i32 (...)* @__gxx_personality_v0 to i8*)
+  %1 = landingpad { i8*, i32 }
           cleanup
           catch i8* bitcast ({ i8*, i8*, i8* }* @catchtypeinfo to i8*)
   resume { i8*, i32 } undef
diff --git a/test/Transforms/LoopSimplify/dbg-loc.ll b/test/Transforms/LoopSimplify/dbg-loc.ll
new file mode 100644
index 00000000000..073319bdac3
--- /dev/null
+++ b/test/Transforms/LoopSimplify/dbg-loc.ll
@@ -0,0 +1,90 @@
+; Check that LoopSimplify creates debug locations in synthesized basic blocks.
+; RUN: opt -loop-simplify %s -S -o - | FileCheck %s
+
+%union.anon = type { i32 }
+%"Length" = type <{ %union.anon, i8, i8, i8, i8 }>
+declare void @bar(%"Length"*) #3
+@catchtypeinfo = external unnamed_addr constant { i8*, i8*, i8* }
+declare i32 @__gxx_personality_v0(...)
+declare void @f1()
+declare void @f2()
+declare void @f3()
+
+; CHECK-LABEL: @foo
+; CHECK:       for.body.preheader:
+; CHECK-NEXT:    br label %for.body, !dbg [[PREHEADER_LOC:![0-9]+]]
+; CHECK:       for.end.loopexit:
+; CHECK-NEXT:    br label %for.end, !dbg [[LOOPEXIT_LOC:![0-9]+]]
+
+define linkonce_odr hidden void @foo(%"Length"* %begin, %"Length"* %end) nounwind ssp uwtable align 2 {
+entry:
+  %cmp.4 = icmp eq %"Length"* %begin, %end, !dbg !7
+  br i1 %cmp.4, label %for.end, label %for.body, !dbg !8
+
+for.body:                                         ; preds = %entry, %length.exit
+  %begin.sink5 = phi %"Length"* [ %incdec.ptr, %length.exit ], [ %begin, %entry ]
+  %m_type.i.i.i = getelementptr inbounds %"Length", %"Length"* %begin.sink5, i64 0, i32 2, !dbg !9
+  %0 = load i8, i8* %m_type.i.i.i, align 1, !dbg !9
+  %cmp.i.i = icmp eq i8 %0, 9, !dbg !7
+  br i1 %cmp.i.i, label %if.then.i, label %length.exit, !dbg !8
+
+if.then.i:                                        ; preds = %for.body
+  tail call void @bar(%"Length"* %begin.sink5) #7, !dbg !10
+  br label %length.exit, !dbg !10
+
+length.exit:                        ; preds = %for.body, %if.then.i
+  %incdec.ptr = getelementptr inbounds %"Length", %"Length"* %begin.sink5, i64 1, !dbg !11
+  %cmp = icmp eq %"Length"* %incdec.ptr, %end, !dbg !7
+  br i1 %cmp, label %for.end, label %for.body, !dbg !8
+
+for.end:                                          ; preds = %length.exit, %entry
+  ret void, !dbg !12
+}
+
+; CHECK-LABEL: @with_landingpad
+; CHECK: catch.preheader:
+; CHECK:   br label %catch, !dbg [[LPAD_PREHEADER_LOC:![0-9]+]]
+; CHECK: catch.preheader.split-lp:
+; CHECK:   br label %catch, !dbg [[LPAD_PREHEADER_LOC]]
+
+define void @with_landingpad() uwtable ssp personality i8* bitcast (i32 (...)* @__gxx_personality_v0 to i8*) {
+entry:
+  invoke void @f1() to label %try.cont19 unwind label %catch, !dbg !13
+
+catch:                                            ; preds = %if.else, %entry
+  %0 = landingpad { i8*, i32 }
+          catch i8* bitcast ({ i8*, i8*, i8* }* @catchtypeinfo to i8*), !dbg !13
+  invoke void @f3() to label %if.else unwind label %eh.resume, !dbg !13
+
+if.else:                                          ; preds = %catch
+  invoke void @f2() to label %try.cont19 unwind label %catch, !dbg !13
+
+try.cont19:                                       ; preds = %if.else, %entry
+  ret void, !dbg !13
+
+eh.resume:                                        ; preds = %catch
+  %1 = landingpad { i8*, i32 }
+          cleanup catch i8* bitcast ({ i8*, i8*, i8* }* @catchtypeinfo to i8*), !dbg !13
+  resume { i8*, i32 } undef, !dbg !13
+}
+
+; CHECK-DAG: [[PREHEADER_LOC]] = !DILocation(line: 73, column: 27, scope: !{{[0-9]+}})
+; CHECK-DAG: [[LOOPEXIT_LOC]] = !DILocation(line: 75, column: 9, scope: !{{[0-9]+}})
+; CHECK-DAG: [[LPAD_PREHEADER_LOC]] = !DILocation(line: 85, column: 1, scope: !{{[0-9]+}})
+
+!llvm.module.flags = !{!0, !1, !2}
+!0 = !{i32 2, !"Dwarf Version", i32 4}
+!1 = !{i32 2, !"Debug Info Version", i32 3}
+!2 = !{i32 1, !"PIC Level", i32 2}
+
+!3 = !{}
+!4 = !DISubroutineType(types: !3)
+!5 = !DIFile(filename: "Vector.h", directory: "/tmp")
+!6 = !DISubprogram(name: "destruct", scope: !5, file: !5, line: 71, type: !4, isLocal: false, isDefinition: true, scopeLine: 72, flags: DIFlagPrototyped, isOptimized: false, function: void (%"Length"*, %"Length"*)* @foo, variables: !3)
+!7 = !DILocation(line: 73, column: 38, scope: !6)
+!8 = !DILocation(line: 73, column: 13, scope: !6)
+!9 = !DILocation(line: 73, column: 27, scope: !6)
+!10 = !DILocation(line: 74, column: 17, scope: !6)
+!11 = !DILocation(line: 73, column: 46, scope: !6)
+!12 = !DILocation(line: 75, column: 9, scope: !6)
+!13 = !DILocation(line: 85, column: 1, scope: !6)
diff --git a/test/Transforms/LoopSimplify/single-backedge.ll b/test/Transforms/LoopSimplify/single-backedge.ll
index aedd6f23091..92fbdca8a6d 100644
--- a/test/Transforms/LoopSimplify/single-backedge.ll
+++ b/test/Transforms/LoopSimplify/single-backedge.ll
@@ -5,19 +5,35 @@
 ; RUN: opt < %s -indvars -S | FileCheck %s
 ; CHECK: Loop.backedge:
 ; CHECK-NOT: br
-; CHECK: br label %Loop
+; CHECK: br label %Loop, !dbg [[BACKEDGE_LOC:![0-9]+]]
+
+; CHECK: [[BACKEDGE_LOC]] = !DILocation(line: 101, column: 1, scope: !{{.*}})
 
 define i32 @test(i1 %C) {
 ; <label>:0
-	br label %Loop
-Loop:		; preds = %BE2, %BE1, %0
-	%IV = phi i32 [ 1, %0 ], [ %IV2, %BE1 ], [ %IV2, %BE2 ]		; <i32> [#uses=2]
-	store i32 %IV, i32* null
-	%IV2 = add i32 %IV, 2		; <i32> [#uses=2]
-	br i1 %C, label %BE1, label %BE2
-BE1:		; preds = %Loop
-	br label %Loop
-BE2:		; preds = %Loop
-	br label %Loop
+  br label %Loop, !dbg !6
+Loop: ; preds = %BE2, %BE1, %0
+  %IV = phi i32 [ 1, %0 ], [ %IV2, %BE1 ], [ %IV2, %BE2 ] ; <i32> [#uses=2]
+  store i32 %IV, i32* null, !dbg !7
+  %IV2 = add i32 %IV, 2, !dbg !8 ; <i32> [#uses=2]
+  br i1 %C, label %BE1, label %BE2, !dbg !9
+BE1:  ; preds = %Loop
+  br label %Loop, !dbg !10
+BE2:    ; preds = %n br label %Loop
+  br label %Loop, !dbg !11
 }
 
+!llvm.module.flags = !{!0, !1}
+!0 = !{i32 2, !"Dwarf Version", i32 4}
+!1 = !{i32 2, !"Debug Info Version", i32 3}
+
+!2 = !{}
+!3 = !DISubroutineType(types: !2)
+!4 = !DIFile(filename: "atomic.cpp", directory: "/tmp")
+!5 = !DISubprogram(name: "test", scope: !4, file: !4, line: 99, type: !3, isLocal: false, isDefinition: true, scopeLine: 100, flags: DIFlagPrototyped, isOptimized: false, variables: !2)
+!6 = !DILocation(line: 100, column: 1, scope: !5)
+!7 = !DILocation(line: 101, column: 1, scope: !5)
+!8 = !DILocation(line: 102, column: 1, scope: !5)
+!9 = !DILocation(line: 103, column: 1, scope: !5)
+!10 = !DILocation(line: 104, column: 1, scope: !5)
+!11 = !DILocation(line: 105, column: 1, scope: !5)
diff --git a/test/Transforms/LoopStrengthReduce/2011-10-06-ReusePhi.ll b/test/Transforms/LoopStrengthReduce/2011-10-06-ReusePhi.ll
index cc8226e1258..5923a42fa65 100644
--- a/test/Transforms/LoopStrengthReduce/2011-10-06-ReusePhi.ll
+++ b/test/Transforms/LoopStrengthReduce/2011-10-06-ReusePhi.ll
@@ -12,8 +12,8 @@ target datalayout = "n8:16:32:64"
 ; CHECK-LABEL: @test(
 ; multiplies are hoisted out of the loop
 ; CHECK: while.body.lr.ph:
-; CHECK: mul i64
-; CHECK: mul i64
+; CHECK: shl i64
+; CHECK: shl i64
 ; GEPs are ugly
 ; CHECK: while.body:
 ; CHECK: phi
diff --git a/test/Transforms/LoopStrengthReduce/X86/ivchain-stress-X86.ll b/test/Transforms/LoopStrengthReduce/X86/ivchain-stress-X86.ll
index 7925bf01020..24be0dc42d6 100644
--- a/test/Transforms/LoopStrengthReduce/X86/ivchain-stress-X86.ll
+++ b/test/Transforms/LoopStrengthReduce/X86/ivchain-stress-X86.ll
@@ -23,7 +23,7 @@
 ; X32: add
 ; X32: add
 ; X32: add
-; X32: add
+; X32: leal
 ; X32: %for.body.3
 define void @sharedidx(i8* nocapture %a, i8* nocapture %b, i8* nocapture %c, i32 %s, i32 %len) nounwind ssp {
 entry:
diff --git a/test/Transforms/LoopStrengthReduce/X86/zext-of-scale.ll b/test/Transforms/LoopStrengthReduce/X86/zext-of-scale.ll
new file mode 100644
index 00000000000..d0972fe9765
--- /dev/null
+++ b/test/Transforms/LoopStrengthReduce/X86/zext-of-scale.ll
@@ -0,0 +1,70 @@
+; RUN: opt  < %s -S -loop-reduce | FileCheck %s
+
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+%struct = type { [8 x i8] }
+
+declare void @use_32(i32)
+declare void @use_64(i64)
+
+define void @f(i32 %tmp156, i32* %length_buf_1, i32* %length_buf_0, %struct* %b,
+                %struct* %c, %struct* %d, %struct* %e, i32* %length_buf_2,
+                i32 %tmp160) {
+; CHECK-LABEL: @f(
+entry:
+  %begin151 = getelementptr inbounds %struct, %struct* %b, i64 0, i32 0, i64 12
+  %tmp21 = bitcast i8* %begin151 to i32*
+  %begin157 = getelementptr inbounds %struct, %struct* %c, i64 0, i32 0, i64 16
+  %tmp23 = bitcast i8* %begin157 to double*
+  %begin163 = getelementptr inbounds %struct, %struct* %d, i64 0, i32 0, i64 16
+  %tmp25 = bitcast i8* %begin163 to double*
+  %length.i820 = load i32, i32* %length_buf_1, align 4, !range !0
+  %enter = icmp ne i32 %tmp156, -1
+  br i1 %enter, label %ok_146, label %block_81_2
+
+ok_146:
+  %var_13 = phi double [ %tmp186, %ok_161 ], [ 0.000000e+00, %entry ]
+  %var_17 = phi i32 [ %tmp187, %ok_161 ], [ %tmp156, %entry ]
+  %tmp174 = zext i32 %var_17 to i64
+  %tmp175 = icmp ult i32 %var_17, %length.i820
+  br i1 %tmp175, label %ok_152, label %block_81_2
+
+ok_152:
+  %tmp176 = getelementptr inbounds i32, i32* %tmp21, i64 %tmp174
+  %tmp177 = load i32, i32* %tmp176, align 4
+  %tmp178 = zext i32 %tmp177 to i64
+  %length.i836 = load i32, i32* %length_buf_2, align 4, !range !0
+  %tmp179 = icmp ult i32 %tmp177, %length.i836
+  br i1 %tmp179, label %ok_158, label %block_81_2
+
+ok_158:
+  %tmp180 = getelementptr inbounds double, double* %tmp23, i64 %tmp178
+  %tmp181 = load double, double* %tmp180, align 8
+  %length.i = load i32, i32* %length_buf_0, align 4, !range !0
+  %tmp182 = icmp slt i32 %var_17, %length.i
+  br i1 %tmp182, label %ok_161, label %block_81_2
+
+ok_161:
+; CHECK-LABEL: ok_161:
+; CHECK: add
+; CHECK-NOT: add
+  %tmp183 = getelementptr inbounds double, double* %tmp25, i64 %tmp174
+  %tmp184 = load double, double* %tmp183, align 8
+  %tmp185 = fmul double %tmp181, %tmp184
+  %tmp186 = fadd double %var_13, %tmp185
+  %tmp187 = add nsw i32 %var_17, 1
+  %tmp188 = icmp slt i32 %tmp187, %tmp160
+; CHECK: br
+  br i1 %tmp188, label %ok_146, label %block_81
+
+block_81:
+  call void @use_64(i64 %tmp174)  ;; pre-inc use
+  call void @use_32(i32 %tmp187)  ;; post-inc use
+  ret void
+
+block_81_2:
+  ret void
+}
+
+!0 = !{i32 0, i32 2147483647}
diff --git a/test/Transforms/LoopStrengthReduce/dominate-assert.ll b/test/Transforms/LoopStrengthReduce/dominate-assert.ll
index bca234c6f3c..ff26c76b3c1 100644
--- a/test/Transforms/LoopStrengthReduce/dominate-assert.ll
+++ b/test/Transforms/LoopStrengthReduce/dominate-assert.ll
@@ -4,7 +4,7 @@
 declare i8* @_Znwm()
 declare i32 @__gxx_personality_v0(...)
 declare void @g()
-define void @f() {
+define void @f() personality i8* bitcast (i32 (...)* @__gxx_personality_v0 to i8*) {
 bb0:
   br label %bb1
 bb1:
@@ -18,7 +18,7 @@ bb3:
   %v3 = invoke noalias i8* @_Znwm()
           to label %bb5 unwind label %bb4
 bb4:
-  %v4 = landingpad { i8*, i32 } personality i8* bitcast (i32 (...)* @__gxx_personality_v0 to i8*)
+  %v4 = landingpad { i8*, i32 }
           cleanup
   br label %bb9
 bb5:
@@ -32,7 +32,7 @@ bb6:
 bb7:
   unreachable
 bb8:
-  %v7 = landingpad { i8*, i32 } personality i8* bitcast (i32 (...)* @__gxx_personality_v0 to i8*)
+  %v7 = landingpad { i8*, i32 }
           cleanup
   br label %bb9
 bb9:
@@ -40,7 +40,7 @@ bb9:
 }
 
 
-define void @h() {
+define void @h() personality i8* bitcast (i32 (...)* @__gxx_personality_v0 to i8*) {
 bb1:
   invoke void @g() optsize
           to label %bb2 unwind label %bb5
@@ -54,17 +54,17 @@ bb3:
 bb4:
   ret void
 bb5:
-  %tmp = landingpad { i8*, i32 } personality i8* bitcast (i32 (...)* @__gxx_personality_v0 to i8*)
+  %tmp = landingpad { i8*, i32 }
           cleanup
   invoke void @g() optsize
           to label %bb4 unwind label %bb7
 bb6:
-  %tmp1 = landingpad { i8*, i32 } personality i8* bitcast (i32 (...)* @__gxx_personality_v0 to i8*)
+  %tmp1 = landingpad { i8*, i32 }
           cleanup
   %arraydestroy.isempty = icmp eq i8* undef, %arrayctor.cur
   ret void
 bb7:
-  %lpad.nonloopexit = landingpad { i8*, i32 } personality i8* bitcast (i32 (...)* @__gxx_personality_v0 to i8*)
+  %lpad.nonloopexit = landingpad { i8*, i32 }
           catch i8* null
   ret void
 }
diff --git a/test/Transforms/LoopStrengthReduce/ephemeral.ll b/test/Transforms/LoopStrengthReduce/ephemeral.ll
new file mode 100644
index 00000000000..a0d1d44b1ba
--- /dev/null
+++ b/test/Transforms/LoopStrengthReduce/ephemeral.ll
@@ -0,0 +1,41 @@
+; RUN: opt < %s -loop-reduce -S | FileCheck %s
+
+target datalayout = "e-i64:64-v16:16-v32:32-n16:32:64"
+
+; for (int i = 0; i < n; ++i) {
+;   use(i * 5 + 3);
+;   // i * a + b is ephemeral and shouldn't be promoted by LSR
+;   __builtin_assume(i * a + b >= 0);
+; }
+define void @ephemeral(i32 %a, i32 %b, i32 %n) {
+; CHECK-LABEL: @ephemeral(
+entry:
+  br label %loop
+
+loop:
+  %i = phi i32 [ 0, %entry ], [ %inc, %loop ]
+  ; Only i and i * 5 + 3 should be indvars, not i * a + b.
+; CHECK: phi i32
+; CHECK: phi i32
+; CHECK-NOT: phi i32
+  %inc = add nsw i32 %i, 1
+  %exitcond = icmp eq i32 %inc, %n
+
+  %0 = mul nsw i32 %i, 5
+  %1 = add nsw i32 %0, 3
+  call void @use(i32 %1)
+
+  %2 = mul nsw i32 %i, %a
+  %3 = add nsw i32 %2, %b
+  %4 = icmp sgt i32 %3, -1
+  call void @llvm.assume(i1 %4)
+
+  br i1 %exitcond, label %exit, label %loop
+
+exit:
+  ret void
+}
+
+declare void @use(i32)
+
+declare void @llvm.assume(i1)
diff --git a/test/Transforms/LoopStrengthReduce/negative-scale.ll b/test/Transforms/LoopStrengthReduce/negative-scale.ll
new file mode 100644
index 00000000000..fdde1b6bf9d
--- /dev/null
+++ b/test/Transforms/LoopStrengthReduce/negative-scale.ll
@@ -0,0 +1,28 @@
+; RUN: opt < %s -loop-reduce -S | FileCheck %s
+
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64"
+
+declare void @foo(i8)
+
+define void @not_addressing_mode(i8* %input, i64 %n) {
+; CHECK-LABEL: @not_addressing_mode(
+entry:
+  br label %loop
+
+loop:
+; CHECK: loop:
+; CHECK: %lsr.iv = phi i8* [ {{%[^,]+}}, %loop ], [ %input, %entry ]
+  %i = phi i64 [ 0, %entry ], [ %i.next, %loop ]
+  %i.next = add i64 %i, 1
+  %j = mul i64 %i, -2
+  ; (%input - 2 * %j) is not foldable. Worth another indvar.
+  %p = getelementptr i8, i8* %input, i64 %j
+  %v = load i8, i8* %p
+; CHECK: %v = load i8, i8* %lsr.iv
+  call void @foo(i8 %v)
+  %exitcond = icmp slt i64 %i.next, %n
+  br i1 %exitcond, label %exit, label %loop
+
+exit:
+  ret void
+}
diff --git a/test/Transforms/LoopStrengthReduce/post-inc-icmpzero.ll b/test/Transforms/LoopStrengthReduce/post-inc-icmpzero.ll
index 092b274bfc0..466566ed8a0 100644
--- a/test/Transforms/LoopStrengthReduce/post-inc-icmpzero.ll
+++ b/test/Transforms/LoopStrengthReduce/post-inc-icmpzero.ll
@@ -6,7 +6,7 @@
 
 ; CHECK:   [[r1:%[a-z0-9]+]] = sub i64 %sub.ptr.lhs.cast, %sub.ptr.rhs.cast
 ; CHECK:   [[r2:%[a-z0-9]+]] = lshr i64 [[r1]], 1
-; CHECK:   [[r3:%[a-z0-9]+]] = mul i64 [[r2]], 2
+; CHECK:   [[r3:%[a-z0-9]+]] = shl i64 [[r2]], 1
 ; CHECK:   br label %for.body
 ; CHECK: for.body:
 ; CHECK:   %lsr.iv2 = phi i64 [ %lsr.iv.next, %for.body ], [ [[r3]], %for.body.lr.ph ]
diff --git a/test/Transforms/LoopStrengthReduce/pr12018.ll b/test/Transforms/LoopStrengthReduce/pr12018.ll
index 9bdbf4e2b1d..b15961a7790 100644
--- a/test/Transforms/LoopStrengthReduce/pr12018.ll
+++ b/test/Transforms/LoopStrengthReduce/pr12018.ll
@@ -16,7 +16,7 @@ for.body:                                         ; preds = %_ZN8nsTArray9Elemen
   %tmp = bitcast %struct.nsTArrayHeader* %add.ptr.i to %struct.nsTArray*
   %arrayidx = getelementptr inbounds %struct.nsTArray, %struct.nsTArray* %tmp, i32 %i.06
   %add = add nsw i32 %i.06, 1
-  call void @llvm.dbg.value(metadata %struct.nsTArray* %aValues, i64 0, metadata !0, metadata !MDExpression()) nounwind, !dbg !MDLocation(scope: !MDSubprogram())
+  call void @llvm.dbg.value(metadata %struct.nsTArray* %aValues, i64 0, metadata !0, metadata !DIExpression()) nounwind, !dbg !DILocation(scope: !DISubprogram())
   br label %_ZN8nsTArray9ElementAtEi.exit
 
 _ZN8nsTArray9ElementAtEi.exit:                    ; preds = %for.body
@@ -35,4 +35,4 @@ declare %struct.nsTArrayHeader* @_ZN8nsTArray4Hdr2Ev()
 
 declare void @llvm.dbg.value(metadata, i64, metadata, metadata) nounwind readnone
 
-!0 = !MDLocalVariable(tag: DW_TAG_arg_variable, scope: !MDSubprogram())
+!0 = !DILocalVariable(tag: DW_TAG_arg_variable, scope: !DISubprogram())
diff --git a/test/Transforms/LoopStrengthReduce/sext-ind-var.ll b/test/Transforms/LoopStrengthReduce/sext-ind-var.ll
new file mode 100644
index 00000000000..9fca3f97094
--- /dev/null
+++ b/test/Transforms/LoopStrengthReduce/sext-ind-var.ll
@@ -0,0 +1,36 @@
+; RUN: opt -loop-reduce -S < %s | FileCheck %s
+
+target datalayout = "e-i64:64-v16:16-v32:32-n16:32:64"
+target triple = "nvptx64-unknown-unknown"
+
+; LSR used not to be able to generate a float* induction variable in
+; these cases due to scalar evolution not propagating nsw from an
+; instruction to the SCEV, preventing distributing sext into the
+; corresponding addrec.
+
+define float @testadd(float* %input, i32 %offset, i32 %numIterations) {
+; CHECK-LABEL: @testadd
+; CHECK: sext i32 %offset to i64
+; CHECK: loop:
+; CHECK-DAG: phi float*
+; CHECK-DAG: phi i32
+; CHECK-NOT: sext
+
+entry:
+  br label %loop
+
+loop:
+  %i = phi i32 [ %nexti, %loop ], [ 0, %entry ]
+  %sum = phi float [ %nextsum, %loop ], [ 0.000000e+00, %entry ]
+  %index32 = add nuw nsw i32 %i, %offset
+  %index64 = sext i32 %index32 to i64
+  %ptr = getelementptr inbounds float, float* %input, i64 %index64
+  %addend = load float, float* %ptr, align 4
+  %nextsum = fadd float %sum, %addend
+  %nexti = add nuw nsw i32 %i, 1
+  %exitcond = icmp eq i32 %nexti, %numIterations
+  br i1 %exitcond, label %exit, label %loop
+
+exit:
+  ret float %nextsum
+}
diff --git a/test/Transforms/LoopStrengthReduce/shl.ll b/test/Transforms/LoopStrengthReduce/shl.ll
new file mode 100644
index 00000000000..bb9cb39f437
--- /dev/null
+++ b/test/Transforms/LoopStrengthReduce/shl.ll
@@ -0,0 +1,38 @@
+; RUN: opt < %s -loop-reduce -gvn -S | FileCheck %s
+
+target datalayout = "e-i64:64-v16:16-v32:32-n16:32:64"
+
+define void @_Z3fooPfll(float* nocapture readonly %input, i64 %n, i64 %s) {
+; CHECK-LABEL: @_Z3fooPfll(
+entry:
+  %mul = shl nsw i64 %s, 2
+; CHECK: %mul = shl i64 %s, 2
+  tail call void @_Z3bazl(i64 %mul) #2
+; CHECK-NEXT: call void @_Z3bazl(i64 %mul)
+  %cmp.5 = icmp sgt i64 %n, 0
+  br i1 %cmp.5, label %for.body.preheader, label %for.cond.cleanup
+
+for.body.preheader:                               ; preds = %entry
+  br label %for.body
+
+for.cond.cleanup.loopexit:                        ; preds = %for.body
+  br label %for.cond.cleanup
+
+for.cond.cleanup:                                 ; preds = %for.cond.cleanup.loopexit, %entry
+  ret void
+
+for.body:                                         ; preds = %for.body.preheader, %for.body
+  %i.06 = phi i64 [ %add, %for.body ], [ 0, %for.body.preheader ]
+  %arrayidx = getelementptr inbounds float, float* %input, i64 %i.06
+; LoopStrengthReduce should reuse %mul as the stride.
+; CHECK: getelementptr i1, i1* {{[^,]+}}, i64 %mul
+  %0 = load float, float* %arrayidx, align 4
+  tail call void @_Z3barf(float %0) #2
+  %add = add nsw i64 %i.06, %s
+  %cmp = icmp slt i64 %add, %n
+  br i1 %cmp, label %for.body, label %for.cond.cleanup.loopexit
+}
+
+declare void @_Z3bazl(i64)
+
+declare void @_Z3barf(float)
diff --git a/test/Transforms/LoopUnroll/PowerPC/a2-high-cost-trip-count-computation.ll b/test/Transforms/LoopUnroll/PowerPC/a2-high-cost-trip-count-computation.ll
new file mode 100644
index 00000000000..bd5f68f9f20
--- /dev/null
+++ b/test/Transforms/LoopUnroll/PowerPC/a2-high-cost-trip-count-computation.ll
@@ -0,0 +1,27 @@
+; RUN: opt < %s -S -mtriple=powerpc64-unknown-linux-gnu -mcpu=a2 -loop-unroll | FileCheck %s
+
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
+
+;; Check that we do emit expensive instructions to compute trip
+;; counts when unrolling loops on the a2 (because we unroll a lot).
+
+define i32 @test(i64 %v12, i8* %array, i64* %loc) {
+; CHECK-LABEL: @test(
+; CHECK: udiv
+entry:
+  %step = load i64, i64* %loc, !range !0
+  br label %loop
+
+loop:                                           ; preds = %entry, %loop
+  %k.015 = phi i64 [ %v15, %loop ], [ %v12, %entry ]
+  %v14 = getelementptr inbounds i8, i8* %array, i64 %k.015
+  store i8 0, i8* %v14
+  %v15 = add nuw nsw i64 %k.015, %step
+  %v16 = icmp slt i64 %v15, 8193
+  br i1 %v16, label %loop, label %loopexit
+
+loopexit:                             ; preds = %loop
+  ret i32 0
+}
+
+!0 = !{i64 1, i64 100}
diff --git a/test/Transforms/LoopUnroll/X86/mmx.ll b/test/Transforms/LoopUnroll/X86/mmx.ll
new file mode 100644
index 00000000000..2c4aa086e83
--- /dev/null
+++ b/test/Transforms/LoopUnroll/X86/mmx.ll
@@ -0,0 +1,24 @@
+; RUN: opt < %s -S -loop-unroll | FileCheck %s
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+define x86_mmx @f() #0 {
+entry:
+  br label %for.body
+
+for.body:                                         ; preds = %for.body, %entry
+  %phi = phi i32 [ 1, %entry ], [ %add, %for.body ]
+  %add = add i32 %phi, 1
+  %cmp = icmp eq i32 %phi, 0
+  br i1 %cmp, label %exit, label %for.body
+
+exit:                                             ; preds = %for.body
+  %ret = phi x86_mmx [ undef, %for.body ]
+  ; CHECK: %[[ret_unr:.*]] = phi x86_mmx [ undef,
+  ; CHECK: %[[ret_ph:.*]]  = phi x86_mmx [ undef,
+  ; CHECK: %[[ret:.*]] = phi x86_mmx [ %[[ret_unr]], {{.*}} ], [ %[[ret_ph]]
+  ; CHECK: ret x86_mmx %[[ret]]
+  ret x86_mmx %ret
+}
+
+attributes #0 = { "target-cpu"="x86-64" }
diff --git a/test/Transforms/LoopUnroll/X86/partial.ll b/test/Transforms/LoopUnroll/X86/partial.ll
index 4566f792deb..104a38779e5 100644
--- a/test/Transforms/LoopUnroll/X86/partial.ll
+++ b/test/Transforms/LoopUnroll/X86/partial.ll
@@ -86,17 +86,20 @@ for.body:                                         ; preds = %entry, %for.body
   %reduction.026 = phi i16 [ %add14, %for.body ], [ 0, %entry ]
   %arrayidx = getelementptr inbounds i16, i16* %arr, i64 %indvars.iv
   %0 = load i16, i16* %arrayidx, align 2
-  %add = add i16 %0, %reduction.026
+  %mul = shl i16 %0, 1
+  %add = add i16 %mul, %reduction.026
   %sext = mul i64 %indvars.iv, 12884901888
   %idxprom3 = ashr exact i64 %sext, 32
   %arrayidx4 = getelementptr inbounds i16, i16* %arr, i64 %idxprom3
   %1 = load i16, i16* %arrayidx4, align 2
-  %add7 = add i16 %add, %1
+  %mul2 = shl i16 %1, 1
+  %add7 = add i16 %add, %mul2
   %sext28 = mul i64 %indvars.iv, 21474836480
   %idxprom10 = ashr exact i64 %sext28, 32
   %arrayidx11 = getelementptr inbounds i16, i16* %arr, i64 %idxprom10
   %2 = load i16, i16* %arrayidx11, align 2
-  %add14 = add i16 %add7, %2
+  %mul3 = shl i16 %2, 1
+  %add14 = add i16 %add7, %mul3
   %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
   %lftr.wideiv = trunc i64 %indvars.iv.next to i32
   %exitcond = icmp eq i32 %lftr.wideiv, %n
diff --git a/test/Transforms/LoopUnroll/full-unroll-bad-cost.ll b/test/Transforms/LoopUnroll/full-unroll-bad-cost.ll
new file mode 100644
index 00000000000..e5694fbeb0c
--- /dev/null
+++ b/test/Transforms/LoopUnroll/full-unroll-bad-cost.ll
@@ -0,0 +1,58 @@
+; RUN: opt -S -loop-unroll < %s | FileCheck %s
+
+; LLVM should not try to fully unroll this loop.
+
+declare void @f()
+declare void @g()
+declare void @h()
+
+define void @trivial_loop() {
+; CHECK-LABEL: @trivial_loop(
+ entry:
+  br label %loop
+
+ loop:
+  %idx = phi i32 [ 0, %entry ], [ %idx.inc, %loop ]
+  %idx.inc = add i32 %idx, 1
+  call void @f()
+  call void @g()
+  call void @h()
+  call void @f()
+  call void @g()
+  call void @h()
+  call void @f()
+  call void @g()
+  call void @h()
+  call void @f()
+  call void @g()
+  call void @h()
+  call void @f()
+  call void @g()
+  call void @h()
+  %be = icmp slt i32 %idx, 268435456
+  br i1 %be, label %loop, label %exit
+
+; CHECK: loop:
+; CHECK-NEXT:  %idx = phi i32 [ 0, %entry ], [ %idx.inc, %loop ]
+; CHECK-NEXT:  %idx.inc = add i32 %idx, 1
+; CHECK-NEXT:  call void @f()
+; CHECK-NEXT:  call void @g()
+; CHECK-NEXT:  call void @h()
+; CHECK-NEXT:  call void @f()
+; CHECK-NEXT:  call void @g()
+; CHECK-NEXT:  call void @h()
+; CHECK-NEXT:  call void @f()
+; CHECK-NEXT:  call void @g()
+; CHECK-NEXT:  call void @h()
+; CHECK-NEXT:  call void @f()
+; CHECK-NEXT:  call void @g()
+; CHECK-NEXT:  call void @h()
+; CHECK-NEXT:  call void @f()
+; CHECK-NEXT:  call void @g()
+; CHECK-NEXT:  call void @h()
+; CHECK-NEXT:  %be = icmp slt i32 %idx, 268435456
+; CHECK-NEXT:  br i1 %be, label %loop, label %exit
+
+ exit:
+  ret void
+}
diff --git a/test/Transforms/LoopUnroll/full-unroll-crashers.ll b/test/Transforms/LoopUnroll/full-unroll-crashers.ll
new file mode 100644
index 00000000000..eebf5a0a2f2
--- /dev/null
+++ b/test/Transforms/LoopUnroll/full-unroll-crashers.ll
@@ -0,0 +1,83 @@
+; Check that we don't crash on corner cases.
+; RUN: opt < %s -S -loop-unroll -unroll-max-iteration-count-to-analyze=1000 -unroll-threshold=10 -unroll-percent-dynamic-cost-saved-threshold=20 -o /dev/null
+target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
+
+@known_constant = internal unnamed_addr constant [10 x i32] [i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1], align 16
+
+define void @foo1() {
+entry:
+  br label %for.body
+
+for.body:
+  %phi = phi i64 [ 0, %entry ], [ %inc, %for.body ]
+  %idx = zext i32 undef to i64
+  %add.ptr = getelementptr inbounds i64, i64* null, i64 %idx
+  %inc = add nuw nsw i64 %phi, 1
+  %cmp = icmp ult i64 %inc, 999
+  br i1 %cmp, label %for.body, label %for.exit
+
+for.exit:
+  ret void
+}
+
+define void @foo2() {
+entry:
+  br label %for.body
+
+for.body:
+  %phi = phi i64 [ 0, %entry ], [ %inc, %for.body ]
+  %x = getelementptr i32, <4 x i32*> undef, <4 x i32> <i32 1, i32 1, i32 1, i32 1>
+  %inc = add nuw nsw i64 %phi, 1
+  %cmp = icmp ult i64 %inc, 999
+  br i1 %cmp, label %for.body, label %for.exit
+
+for.exit:
+  ret void
+}
+
+define void @cmp_undef() {
+entry:
+  br label %for.body
+
+for.body:                                         ; preds = %for.inc, %entry
+  %iv.0 = phi i64 [ 0, %entry ], [ %iv.1, %for.inc ]
+  %arrayidx1 = getelementptr inbounds [10 x i32], [10 x i32]* @known_constant, i64 0, i64 %iv.0
+  %x1 = load i32, i32* %arrayidx1, align 4
+  %cmp = icmp eq i32 %x1, undef
+  br i1 %cmp, label %if.then, label %for.inc
+
+if.then:                                          ; preds = %for.body
+  br label %for.inc
+
+for.inc:                                          ; preds = %for.body, %if.then
+  %iv.1 = add nuw nsw i64 %iv.0, 1
+  %exitcond = icmp eq i64 %iv.1, 10
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:                                          ; preds = %for.inc
+  ret void
+}
+
+define void @switch() {
+entry:
+  br label %for.body
+
+for.body:
+  %iv.0 = phi i64 [ 0, %entry ], [ %iv.1, %for.inc ]
+  %arrayidx1 = getelementptr inbounds [10 x i32], [10 x i32]* @known_constant, i64 0, i64 %iv.0
+  %x1 = load i32, i32* %arrayidx1, align 4
+  switch i32 %x1, label %l1 [
+  ]
+
+l1:
+  %x2 = add i32 %x1, 2
+  br label %for.inc
+
+for.inc:
+  %iv.1 = add nuw nsw i64 %iv.0, 1
+  %exitcond = icmp eq i64 %iv.1, 10
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:
+  ret void
+}
diff --git a/test/Transforms/LoopUnroll/full-unroll-heuristics-cast.ll b/test/Transforms/LoopUnroll/full-unroll-heuristics-cast.ll
new file mode 100644
index 00000000000..cd8cfd75424
--- /dev/null
+++ b/test/Transforms/LoopUnroll/full-unroll-heuristics-cast.ll
@@ -0,0 +1,97 @@
+; RUN: opt < %s -S -loop-unroll -unroll-max-iteration-count-to-analyze=100 -unroll-dynamic-cost-savings-discount=1000 -unroll-threshold=10 -unroll-percent-dynamic-cost-saved-threshold=50 | FileCheck %s
+target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
+
+@known_constant = internal unnamed_addr constant [10 x i32] [i32 0, i32 1, i32 0, i32 1, i32 0, i32 1, i32 0, i32 1, i32 0, i32 1], align 16
+
+; We should be able to propagate constant data through different types of
+; casts. For example, in this test we have a load, which becomes constant after
+; unrolling, which then is truncated to i8. Obviously, truncated value is also a
+; constant, which can be used in the further simplifications.
+;
+; We expect this loop to be unrolled, because in this case load would become
+; constant, which is 0 in many cases, and which, in its turn, helps to simplify
+; following multiplication and addition. In total, unrolling should help to
+; optimize  ~60% of all instructions in this case.
+;
+; CHECK-LABEL: @const_load_trunc
+; CHECK-NOT: br i1
+; CHECK: ret i8 %
+define i8 @const_load_trunc(i32* noalias nocapture readonly %src) {
+entry:
+  br label %loop
+
+loop:                                                ; preds = %loop, %entry
+  %iv = phi i64 [ 0, %entry ], [ %inc, %loop ]
+  %r  = phi i8 [ 0, %entry ], [ %add, %loop ]
+  %arrayidx = getelementptr inbounds i32, i32* %src, i64 %iv
+  %src_element = load i32, i32* %arrayidx, align 4
+  %array_const_idx = getelementptr inbounds [10 x i32], [10 x i32]* @known_constant, i64 0, i64 %iv
+  %const_array_element = load i32, i32* %array_const_idx, align 4
+  %x = trunc i32 %src_element to i8
+  %y = trunc i32 %const_array_element to i8
+  %mul = mul nsw i8 %x, %y
+  %add = add nsw i8 %mul, %r
+  %inc = add nuw nsw i64 %iv, 1
+  %exitcond86.i = icmp eq i64 %inc, 10
+  br i1 %exitcond86.i, label %loop.end, label %loop
+
+loop.end:                                            ; preds = %loop
+  %r.lcssa = phi i8 [ %r, %loop ]
+  ret i8 %r.lcssa
+}
+
+; The same test as before, but with ZEXT instead of TRUNC.
+; CHECK-LABEL: @const_load_zext
+; CHECK-NOT: br i1
+; CHECK: ret i64 %
+define i64 @const_load_zext(i32* noalias nocapture readonly %src) {
+entry:
+  br label %loop
+
+loop:                                                ; preds = %loop, %entry
+  %iv = phi i64 [ 0, %entry ], [ %inc, %loop ]
+  %r  = phi i64 [ 0, %entry ], [ %add, %loop ]
+  %arrayidx = getelementptr inbounds i32, i32* %src, i64 %iv
+  %src_element = load i32, i32* %arrayidx, align 4
+  %array_const_idx = getelementptr inbounds [10 x i32], [10 x i32]* @known_constant, i64 0, i64 %iv
+  %const_array_element = load i32, i32* %array_const_idx, align 4
+  %x = zext i32 %src_element to i64
+  %y = zext i32 %const_array_element to i64
+  %mul = mul nsw i64 %x, %y
+  %add = add nsw i64 %mul, %r
+  %inc = add nuw nsw i64 %iv, 1
+  %exitcond86.i = icmp eq i64 %inc, 10
+  br i1 %exitcond86.i, label %loop.end, label %loop
+
+loop.end:                                            ; preds = %loop
+  %r.lcssa = phi i64 [ %r, %loop ]
+  ret i64 %r.lcssa
+}
+
+; The same test as the first one, but with SEXT instead of TRUNC.
+; CHECK-LABEL: @const_load_sext
+; CHECK-NOT: br i1
+; CHECK: ret i64 %
+define i64 @const_load_sext(i32* noalias nocapture readonly %src) {
+entry:
+  br label %loop
+
+loop:                                                ; preds = %loop, %entry
+  %iv = phi i64 [ 0, %entry ], [ %inc, %loop ]
+  %r  = phi i64 [ 0, %entry ], [ %add, %loop ]
+  %arrayidx = getelementptr inbounds i32, i32* %src, i64 %iv
+  %src_element = load i32, i32* %arrayidx, align 4
+  %array_const_idx = getelementptr inbounds [10 x i32], [10 x i32]* @known_constant, i64 0, i64 %iv
+  %const_array_element = load i32, i32* %array_const_idx, align 4
+  %x = sext i32 %src_element to i64
+  %y = sext i32 %const_array_element to i64
+  %mul = mul nsw i64 %x, %y
+  %add = add nsw i64 %mul, %r
+  %inc = add nuw nsw i64 %iv, 1
+  %exitcond86.i = icmp eq i64 %inc, 10
+  br i1 %exitcond86.i, label %loop.end, label %loop
+
+loop.end:                                            ; preds = %loop
+  %r.lcssa = phi i64 [ %r, %loop ]
+  ret i64 %r.lcssa
+}
diff --git a/test/Transforms/LoopUnroll/full-unroll-heuristics-cmp.ll b/test/Transforms/LoopUnroll/full-unroll-heuristics-cmp.ll
new file mode 100644
index 00000000000..f7758fa2200
--- /dev/null
+++ b/test/Transforms/LoopUnroll/full-unroll-heuristics-cmp.ll
@@ -0,0 +1,207 @@
+; RUN: opt < %s -S -loop-unroll -unroll-max-iteration-count-to-analyze=100 -unroll-dynamic-cost-savings-discount=1000 -unroll-threshold=10 -unroll-percent-dynamic-cost-saved-threshold=50 | FileCheck %s
+target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
+
+@known_constant = internal unnamed_addr constant [10 x i32] [i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1], align 16
+
+; We should be able to propagate constant data through comparisons.
+; For example, in this test we have a load, which becomes constant after
+; unrolling, making comparison with 0 also known to be 0 (false) - and that
+; will trigger further simplifications.
+;
+; We expect this loop to be unrolled, because in this case load would become
+; constant, which is always 1, and which, in its turn, helps to simplify
+; following comparison, zero-extension, and addition. In total, unrolling should help to
+; optimize more than 50% of all instructions in this case.
+;
+; CHECK-LABEL: @const_compare
+; CHECK-NOT: br i1 %
+; CHECK: ret i32
+define i32 @const_compare(i32* noalias nocapture readonly %b) {
+entry:
+  br label %for.body
+
+for.body:                                         ; preds = %for.inc, %entry
+  %iv.0 = phi i64 [ 0, %entry ], [ %iv.1, %for.body ]
+  %r.0 = phi i32 [ 0, %entry ], [ %r.1, %for.body ]
+  %arrayidx1 = getelementptr inbounds [10 x i32], [10 x i32]* @known_constant, i64 0, i64 %iv.0
+  %x1 = load i32, i32* %arrayidx1, align 4
+  %cmp = icmp eq i32 %x1, 0
+  %cast = zext i1 %cmp to i32
+  %iv.1 = add nuw nsw i64 %iv.0, 1
+  %r.1 = add i32 %r.0, %cast
+  %exitcond = icmp eq i64 %iv.1, 10
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:                                          ; preds = %for.inc
+  ret i32 %r.1
+}
+
+; If we can figure out result of comparison on each iteration, we can resolve
+; the depending branch. That means, that the unrolled version of the loop would
+; have less code, because we don't need not-taken basic blocks there.
+; This test checks that this is taken into consideration.
+; We expect this loop to be unrolled, because the most complicated part of its
+; body (if.then block) is never actually executed.
+; CHECK-LABEL: @branch_folded
+; CHECK-NOT: br i1 %
+; CHECK: ret i32
+define i32 @branch_folded(i32* noalias nocapture readonly %b) {
+entry:
+  br label %for.body
+
+for.body:                                         ; preds = %for.inc, %entry
+  %iv.0 = phi i64 [ 0, %entry ], [ %iv.1, %for.inc ]
+  %r.0 = phi i32 [ 0, %entry ], [ %r.1, %for.inc ]
+  %arrayidx1 = getelementptr inbounds [10 x i32], [10 x i32]* @known_constant, i64 0, i64 %iv.0
+  %x1 = load i32, i32* %arrayidx1, align 4
+  %cmp = icmp eq i32 %x1, 0
+  %iv.1 = add nuw nsw i64 %iv.0, 1
+  br i1 %cmp, label %if.then, label %for.inc
+
+if.then:                                          ; preds = %for.body
+  %arrayidx2 = getelementptr inbounds i32, i32* %b, i64 %iv.0
+  %x2 = load i32, i32* %arrayidx2, align 4
+  %add = add nsw i32 %x2, %r.0
+  br label %for.inc
+
+for.inc:                                          ; preds = %for.body, %if.then
+  %r.1 = phi i32 [ %add, %if.then ], [ %x1, %for.body ]
+  %exitcond = icmp eq i64 %iv.1, 10
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:                                          ; preds = %for.inc
+  ret i32 %r.1
+}
+
+; This test is similar to the previous one, but in this we use IV in comparison
+; (not a loaded value as we did there).
+; CHECK-LABEL: @branch_iv
+; CHECK-NOT: br i1 %
+; CHECK: ret i64
+define i64 @branch_iv(i64* noalias nocapture readonly %b) {
+entry:
+  br label %for.body
+
+for.body:                                         ; preds = %for.inc, %entry
+  %indvars.iv = phi i64 [ 0, %entry ], [ %tmp3, %for.inc ]
+  %r.030 = phi i64 [ 0, %entry ], [ %r.1, %for.inc ]
+  %cmp3 = icmp eq i64 %indvars.iv, 5
+  %tmp3 = add nuw nsw i64 %indvars.iv, 1
+  br i1 %cmp3, label %if.then, label %for.inc
+
+if.then:                                          ; preds = %for.body
+  %arrayidx2 = getelementptr inbounds i64, i64* %b, i64 %tmp3
+  %tmp1 = load i64, i64* %arrayidx2, align 4
+  %add = add nsw i64 %tmp1, %r.030
+  br label %for.inc
+
+for.inc:                                          ; preds = %if.then, %for.body
+  %r.1 = phi i64 [ %add, %if.then ], [ %r.030, %for.body ]
+  %exitcond = icmp eq i64 %tmp3, 20
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:                                          ; preds = %for.inc
+  ret i64 %r.1
+}
+
+; Induction variables are often casted to another type, and that shouldn't
+; prevent us from folding branches. Tthis test specifically checks if we can
+; handle this. Other than thatm it's similar to the previous test.
+; CHECK-LABEL: @branch_iv_trunc
+; CHECK-NOT:   br i1 %
+; CHECK:   ret i32
+define i32 @branch_iv_trunc(i32* noalias nocapture readonly %b) {
+entry:
+  br label %for.body
+
+for.body:                                         ; preds = %for.inc, %entry
+  %indvars.iv = phi i64 [ 0, %entry ], [ %tmp3, %for.inc ]
+  %r.030 = phi i32 [ 0, %entry ], [ %r.1, %for.inc ]
+  %tmp2 = trunc i64 %indvars.iv to i32
+  %cmp3 = icmp eq i32 %tmp2, 5
+  %tmp3 = add nuw nsw i64 %indvars.iv, 1
+  br i1 %cmp3, label %if.then, label %for.inc
+
+if.then:                                          ; preds = %for.body
+  %arrayidx2 = getelementptr inbounds i32, i32* %b, i64 %tmp3
+  %tmp1 = load i32, i32* %arrayidx2, align 4
+  %add = add nsw i32 %tmp1, %r.030
+  br label %for.inc
+
+for.inc:                                          ; preds = %if.then, %for.body
+  %r.1 = phi i32 [ %add, %if.then ], [ %r.030, %for.body ]
+  %exitcond = icmp eq i64 %tmp3, 10
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:                                          ; preds = %for.inc
+  ret i32 %r.1
+}
+
+; Check that we don't crash when we analyze icmp with pointer-typed IV and a
+; pointer.
+; CHECK-LABEL: @ptr_cmp_crash
+; CHECK:   ret void
+define void @ptr_cmp_crash() {
+entry:
+  br label %while.body
+
+while.body:
+  %iv.0 = phi i32* [ getelementptr inbounds ([10 x i32], [10 x i32]* @known_constant, i64 0, i64 0), %entry ], [ %iv.1, %while.body ]
+  %iv.1 = getelementptr inbounds i32, i32* %iv.0, i64 1
+  %exitcond = icmp eq i32* %iv.1, getelementptr inbounds ([10 x i32], [10 x i32]* @known_constant, i64 0, i64 9)
+  br i1 %exitcond, label %loop.exit, label %while.body
+
+loop.exit:
+  ret void
+}
+
+; Check that we don't crash when we analyze ptrtoint cast.
+; CHECK-LABEL: @ptrtoint_cast_crash
+; CHECK:   ret void
+define void @ptrtoint_cast_crash(i8 * %a) {
+entry:
+  %limit = getelementptr i8, i8* %a, i64 512
+  br label %loop.body
+
+loop.body:
+  %iv.0 = phi i8* [ %a, %entry ], [ %iv.1, %loop.body ]
+  %cast = ptrtoint i8* %iv.0 to i64
+  %iv.1 = getelementptr inbounds i8, i8* %iv.0, i64 1
+  %exitcond = icmp ne i8* %iv.1, %limit
+  br i1 %exitcond, label %loop.body, label %loop.exit
+
+loop.exit:
+  ret void
+}
+
+; Loop unroller should be able to predict that a comparison would become
+; constant if the operands are pointers with the same base and constant
+; offsets.
+; We expect this loop to be unrolled, since most of its instructions would
+; become constant after it.
+; CHECK-LABEL: @ptr_cmp
+; CHECK-NOT:   br i1 %
+; CHECK:   ret i64
+define i64 @ptr_cmp(i8 * %a) {
+entry:
+  %limit = getelementptr i8, i8* %a, i64 40
+  %start.iv2 = getelementptr i8, i8* %a, i64 7
+  br label %loop.body
+
+loop.body:
+  %iv.0 = phi i8* [ %a, %entry ], [ %iv.1, %loop.body ]
+  %iv2.0 = phi i8* [ %start.iv2, %entry ], [ %iv2.1, %loop.body ]
+  %r.0 = phi i64 [ 0, %entry ], [ %r.1, %loop.body ]
+  %cast = ptrtoint i8* %iv.0 to i64
+  %cmp = icmp eq i8* %iv2.0, %iv.0
+  %sub = sext i1 %cmp to i64
+  %mul = mul i64 %sub, %cast
+  %r.1 = add i64 %r.0, %mul
+  %iv.1 = getelementptr inbounds i8, i8* %iv.0, i64 1
+  %iv2.1 = getelementptr inbounds i8, i8* %iv2.0, i64 1
+  %exitcond = icmp ne i8* %iv.1, %limit
+  br i1 %exitcond, label %loop.body, label %loop.exit
+
+loop.exit:
+  ret i64 %r.1
+}
diff --git a/test/Transforms/LoopUnroll/full-unroll-heuristics.ll b/test/Transforms/LoopUnroll/full-unroll-heuristics.ll
index a9104adeb97..904a65a1bc0 100644
--- a/test/Transforms/LoopUnroll/full-unroll-heuristics.ll
+++ b/test/Transforms/LoopUnroll/full-unroll-heuristics.ll
@@ -1,8 +1,8 @@
 ; In this test we check how heuristics for complete unrolling work. We have
 ; three knobs:
 ;  1) -unroll-threshold
-;  2) -unroll-absolute-threshold and
-;  3) -unroll-percent-of-optimized-for-complete-unroll
+;  3) -unroll-percent-dynamic-cost-saved-threshold and
+;  2) -unroll-dynamic-cost-savings-discount
 ;
 ; They control loop-unrolling according to the following rules:
 ;  * If size of unrolled loop exceeds the absoulte threshold, we don't unroll
@@ -17,10 +17,10 @@
 ; optimizations to remove ~55% of the instructions, the loop body size is 9,
 ; and unrolled size is 65.
 
-; RUN: opt < %s -S -loop-unroll -unroll-max-iteration-count-to-analyze=1000 -unroll-absolute-threshold=10  -unroll-threshold=10  -unroll-percent-of-optimized-for-complete-unroll=30 | FileCheck %s -check-prefix=TEST1
-; RUN: opt < %s -S -loop-unroll -unroll-max-iteration-count-to-analyze=1000 -unroll-absolute-threshold=100 -unroll-threshold=10  -unroll-percent-of-optimized-for-complete-unroll=30 | FileCheck %s -check-prefix=TEST2
-; RUN: opt < %s -S -loop-unroll -unroll-max-iteration-count-to-analyze=1000 -unroll-absolute-threshold=100 -unroll-threshold=10  -unroll-percent-of-optimized-for-complete-unroll=80 | FileCheck %s -check-prefix=TEST3
-; RUN: opt < %s -S -loop-unroll -unroll-max-iteration-count-to-analyze=1000 -unroll-absolute-threshold=100 -unroll-threshold=100 -unroll-percent-of-optimized-for-complete-unroll=80 | FileCheck %s -check-prefix=TEST4
+; RUN: opt < %s -S -loop-unroll -unroll-max-iteration-count-to-analyze=1000 -unroll-threshold=10  -unroll-percent-dynamic-cost-saved-threshold=20 -unroll-dynamic-cost-savings-discount=0 | FileCheck %s -check-prefix=TEST1
+; RUN: opt < %s -S -loop-unroll -unroll-max-iteration-count-to-analyze=1000 -unroll-threshold=10  -unroll-percent-dynamic-cost-saved-threshold=20 -unroll-dynamic-cost-savings-discount=90 | FileCheck %s -check-prefix=TEST2
+; RUN: opt < %s -S -loop-unroll -unroll-max-iteration-count-to-analyze=1000 -unroll-threshold=10  -unroll-percent-dynamic-cost-saved-threshold=80 -unroll-dynamic-cost-savings-discount=90 | FileCheck %s -check-prefix=TEST3
+; RUN: opt < %s -S -loop-unroll -unroll-max-iteration-count-to-analyze=1000 -unroll-threshold=100 -unroll-percent-dynamic-cost-saved-threshold=80 -unroll-dynamic-cost-savings-discount=0 | FileCheck %s -check-prefix=TEST4
 
 ; If the absolute threshold is too low, or if we can't optimize away requested
 ; percent of instructions, we shouldn't unroll:
diff --git a/test/Transforms/LoopUnroll/runtime-loop1.ll b/test/Transforms/LoopUnroll/runtime-loop1.ll
index 7684e394290..de61e847a5a 100644
--- a/test/Transforms/LoopUnroll/runtime-loop1.ll
+++ b/test/Transforms/LoopUnroll/runtime-loop1.ll
@@ -2,29 +2,50 @@
 
 ; This tests that setting the unroll count works
 
+; CHECK: for.body.preheader:
+; CHECK:   br {{.*}} label %for.body.prol, label %for.body.preheader.split, !dbg [[PH_LOC:![0-9]+]]
 ; CHECK: for.body.prol:
-; CHECK: br label %for.body.preheader.split
+; CHECK:   br label %for.body.preheader.split, !dbg [[BODY_LOC:![0-9]+]]
+; CHECK: for.body.preheader.split:
+; CHECK:   br {{.*}} label %for.end.loopexit, label %for.body.preheader.split.split, !dbg [[PH_LOC]]
 ; CHECK: for.body:
-; CHECK: br i1 %exitcond.1, label %for.end.loopexit.unr-lcssa, label %for.body
+; CHECK:   br i1 %exitcond.1, label %for.end.loopexit.unr-lcssa, label %for.body, !dbg [[BODY_LOC]]
 ; CHECK-NOT: br i1 %exitcond.4, label %for.end.loopexit{{.*}}, label %for.body
 
+; CHECK-DAG: [[PH_LOC]] = !DILocation(line: 101, column: 1, scope: !{{.*}})
+; CHECK-DAG: [[BODY_LOC]] = !DILocation(line: 102, column: 1, scope: !{{.*}})
+
 define i32 @test(i32* nocapture %a, i32 %n) nounwind uwtable readonly {
 entry:
-  %cmp1 = icmp eq i32 %n, 0
-  br i1 %cmp1, label %for.end, label %for.body
+  %cmp1 = icmp eq i32 %n, 0, !dbg !7
+  br i1 %cmp1, label %for.end, label %for.body, !dbg !7
 
 for.body:                                         ; preds = %for.body, %entry
   %indvars.iv = phi i64 [ %indvars.iv.next, %for.body ], [ 0, %entry ]
   %sum.02 = phi i32 [ %add, %for.body ], [ 0, %entry ]
-  %arrayidx = getelementptr inbounds i32, i32* %a, i64 %indvars.iv
-  %0 = load i32, i32* %arrayidx, align 4
-  %add = add nsw i32 %0, %sum.02
-  %indvars.iv.next = add i64 %indvars.iv, 1
-  %lftr.wideiv = trunc i64 %indvars.iv.next to i32
-  %exitcond = icmp eq i32 %lftr.wideiv, %n
-  br i1 %exitcond, label %for.end, label %for.body
+  %arrayidx = getelementptr inbounds i32, i32* %a, i64 %indvars.iv, !dbg !8
+  %0 = load i32, i32* %arrayidx, align 4, !dbg !8
+  %add = add nsw i32 %0, %sum.02, !dbg !8
+  %indvars.iv.next = add i64 %indvars.iv, 1, !dbg !9
+  %lftr.wideiv = trunc i64 %indvars.iv.next to i32, !dbg !9
+  %exitcond = icmp eq i32 %lftr.wideiv, %n, !dbg !9
+  br i1 %exitcond, label %for.end, label %for.body, !dbg !9
 
 for.end:                                          ; preds = %for.body, %entry
   %sum.0.lcssa = phi i32 [ 0, %entry ], [ %add, %for.body ]
-  ret i32 %sum.0.lcssa
+  ret i32 %sum.0.lcssa, !dbg !10
 }
+
+!llvm.module.flags = !{!0, !1, !2}
+!0 = !{i32 2, !"Dwarf Version", i32 4}
+!1 = !{i32 2, !"Debug Info Version", i32 3}
+!2 = !{i32 1, !"PIC Level", i32 2}
+
+!3 = !{}
+!4 = !DISubroutineType(types: !3)
+!5 = !DIFile(filename: "test.cpp", directory: "/tmp")
+!6 = !DISubprogram(name: "test", scope: !5, file: !5, line: 99, type: !4, isLocal: false, isDefinition: true, scopeLine: 100, flags: DIFlagPrototyped, isOptimized: false, function: i32 (i32*, i32)* @test, variables: !3)
+!7 = !DILocation(line: 100, column: 1, scope: !6)
+!8 = !DILocation(line: 101, column: 1, scope: !6)
+!9 = !DILocation(line: 102, column: 1, scope: !6)
+!10 = !DILocation(line: 103, column: 1, scope: !6)
diff --git a/test/Transforms/LoopUnroll/unroll-cleanup.ll b/test/Transforms/LoopUnroll/unroll-cleanup.ll
new file mode 100644
index 00000000000..1e42203876e
--- /dev/null
+++ b/test/Transforms/LoopUnroll/unroll-cleanup.ll
@@ -0,0 +1,85 @@
+; PR23524
+; The test is to check redundency produced by loop unroll pass
+; should be cleaned up by later pass.
+; RUN: opt < %s -O2 -S | FileCheck %s
+
+; After loop unroll:
+;       %dec18 = add nsw i32 %dec18.in, -1
+;       ...
+;       %dec18.1 = add nsw i32 %dec18, -1
+; should be merged to:
+;       %dec18.1 = add nsw i32 %dec18.in, -2
+;
+; CHECK-LABEL: @_Z3fn1v(
+; CHECK: %dec18.1 = add nsw i32 %dec18.in, -2
+
+; ModuleID = '<stdin>'
+target triple = "x86_64-unknown-linux-gnu"
+
+@b = global i32 0, align 4
+@c = global i32 0, align 4
+
+; Function Attrs: nounwind uwtable
+define void @_Z3fn1v() #0 {
+entry:
+  %tmp = load i32, i32* @b, align 4
+  %tobool20 = icmp eq i32 %tmp, 0
+  br i1 %tobool20, label %for.end6, label %for.body.lr.ph
+
+for.body.lr.ph:                                   ; preds = %entry
+  br label %for.body
+
+for.cond1.for.cond.loopexit_crit_edge:            ; preds = %for.inc
+  %add.ptr.lcssa = phi i16* [ %add.ptr, %for.inc ]
+  %incdec.ptr.lcssa = phi i8* [ %incdec.ptr, %for.inc ]
+  br label %for.cond.loopexit
+
+for.cond.loopexit:                                ; preds = %for.body, %for.cond1.for.cond.loopexit_crit_edge
+  %r.1.lcssa = phi i16* [ %add.ptr.lcssa, %for.cond1.for.cond.loopexit_crit_edge ], [ %r.022, %for.body ]
+  %a.1.lcssa = phi i8* [ %incdec.ptr.lcssa, %for.cond1.for.cond.loopexit_crit_edge ], [ %a.021, %for.body ]
+  %tmp1 = load i32, i32* @b, align 4
+  %tobool = icmp eq i32 %tmp1, 0
+  br i1 %tobool, label %for.cond.for.end6_crit_edge, label %for.body
+
+for.body:                                         ; preds = %for.cond.loopexit, %for.body.lr.ph
+  %r.022 = phi i16* [ undef, %for.body.lr.ph ], [ %r.1.lcssa, %for.cond.loopexit ]
+  %a.021 = phi i8* [ undef, %for.body.lr.ph ], [ %a.1.lcssa, %for.cond.loopexit ]
+  %tmp2 = load i32, i32* @c, align 4
+  %tobool215 = icmp eq i32 %tmp2, 0
+  br i1 %tobool215, label %for.cond.loopexit, label %for.body3.lr.ph
+
+for.body3.lr.ph:                                  ; preds = %for.body
+  br label %for.body3
+
+for.body3:                                        ; preds = %for.inc, %for.body3.lr.ph
+  %dec18.in = phi i32 [ %tmp2, %for.body3.lr.ph ], [ %dec18, %for.inc ]
+  %r.117 = phi i16* [ %r.022, %for.body3.lr.ph ], [ %add.ptr, %for.inc ]
+  %a.116 = phi i8* [ %a.021, %for.body3.lr.ph ], [ %incdec.ptr, %for.inc ]
+  %dec18 = add nsw i32 %dec18.in, -1
+  %tmp3 = load i8, i8* %a.116, align 1
+  %cmp = icmp eq i8 %tmp3, 0
+  br i1 %cmp, label %if.then, label %for.inc
+
+if.then:                                          ; preds = %for.body3
+  %arrayidx = getelementptr inbounds i16, i16* %r.117, i64 1
+  store i16 0, i16* %arrayidx, align 2
+  store i16 0, i16* %r.117, align 2
+  %arrayidx5 = getelementptr inbounds i16, i16* %r.117, i64 2
+  store i16 0, i16* %arrayidx5, align 2
+  br label %for.inc
+
+for.inc:                                          ; preds = %if.then, %for.body3
+  %incdec.ptr = getelementptr inbounds i8, i8* %a.116, i64 1
+  %add.ptr = getelementptr inbounds i16, i16* %r.117, i64 3
+  %tobool2 = icmp eq i32 %dec18, 0
+  br i1 %tobool2, label %for.cond1.for.cond.loopexit_crit_edge, label %for.body3, !llvm.loop !0
+
+for.cond.for.end6_crit_edge:                      ; preds = %for.cond.loopexit
+  br label %for.end6
+
+for.end6:                                         ; preds = %for.cond.for.end6_crit_edge, %entry
+  ret void
+}
+
+!0 = !{!0, !1}
+!1 = !{!"llvm.loop.unroll.count", i32 2}
diff --git a/test/Transforms/LoopUnroll/unroll-pragmas.ll b/test/Transforms/LoopUnroll/unroll-pragmas.ll
index 1354181becd..8e0d77513cc 100644
--- a/test/Transforms/LoopUnroll/unroll-pragmas.ll
+++ b/test/Transforms/LoopUnroll/unroll-pragmas.ll
@@ -86,9 +86,9 @@ for.end:                                          ; preds = %for.body
 ; #pragma clang loop unroll(full)
 ; Loop should be fully unrolled.
 ;
-; CHECK-LABEL: @loop64_with_enable(
+; CHECK-LABEL: @loop64_with_full(
 ; CHECK-NOT: br i1
-define void @loop64_with_enable(i32* nocapture %a) {
+define void @loop64_with_full(i32* nocapture %a) {
 entry:
   br label %for.body
 
@@ -139,14 +139,13 @@ for.end:                                          ; preds = %for.body
 !6 = !{!"llvm.loop.unroll.count", i32 4}
 
 ; #pragma clang loop unroll(full)
-; Full unrolling is requested, but loop has a dynamic trip count so
+; Full unrolling is requested, but loop has a runtime trip count so
 ; no unrolling should occur.
 ;
-; CHECK-LABEL: @dynamic_loop_with_enable(
+; CHECK-LABEL: @runtime_loop_with_full(
 ; CHECK: store i32
 ; CHECK-NOT: store i32
-; CHECK: br i1
-define void @dynamic_loop_with_enable(i32* nocapture %a, i32 %b) {
+define void @runtime_loop_with_full(i32* nocapture %a, i32 %b) {
 entry:
   %cmp3 = icmp sgt i32 %b, 0
   br i1 %cmp3, label %for.body, label %for.end, !llvm.loop !8
@@ -168,22 +167,22 @@ for.end:                                          ; preds = %for.body, %entry
 !8 = !{!8, !4}
 
 ; #pragma clang loop unroll_count(4)
-; Loop has a dynamic trip count.  Unrolling should occur, but no
-; conditional branches can be removed.
+; Loop has a runtime trip count.  Runtime unrolling should occur and loop
+; should be duplicated (original and 4x unrolled).
 ;
-; CHECK-LABEL: @dynamic_loop_with_count4(
+; CHECK-LABEL: @runtime_loop_with_count4(
+; CHECK: for.body.prol:
+; CHECK: store
 ; CHECK-NOT: store
 ; CHECK: br i1
+; CHECK: for.body
 ; CHECK: store
-; CHECK: br i1
 ; CHECK: store
-; CHECK: br i1
 ; CHECK: store
-; CHECK: br i1
 ; CHECK: store
+; CHECK-NOT: store
 ; CHECK: br i1
-; CHECK-NOT: br i1
-define void @dynamic_loop_with_count4(i32* nocapture %a, i32 %b) {
+define void @runtime_loop_with_count4(i32* nocapture %a, i32 %b) {
 entry:
   %cmp3 = icmp sgt i32 %b, 0
   br i1 %cmp3, label %for.body, label %for.end, !llvm.loop !9
diff --git a/test/Transforms/LoopUnswitch/2011-09-26-EHCrash.ll b/test/Transforms/LoopUnswitch/2011-09-26-EHCrash.ll
index 0b7f91fee15..1a929d68573 100644
--- a/test/Transforms/LoopUnswitch/2011-09-26-EHCrash.ll
+++ b/test/Transforms/LoopUnswitch/2011-09-26-EHCrash.ll
@@ -6,7 +6,7 @@ target triple = "x86_64-apple-macosx10.7.2"
 %class.MyContainer.1.3.19.29 = type { [6 x %class.MyMemVarClass.0.2.18.28*] }
 %class.MyMemVarClass.0.2.18.28 = type { i32 }
 
-define void @_ZN11MyContainer1fEi(%class.MyContainer.1.3.19.29* %this, i32 %doit) uwtable ssp align 2 {
+define void @_ZN11MyContainer1fEi(%class.MyContainer.1.3.19.29* %this, i32 %doit) uwtable ssp align 2 personality i8* bitcast (i32 (...)* @__gxx_personality_v0 to i8*) {
 entry:
   br label %for.cond
 
@@ -38,7 +38,7 @@ invoke.cont:                                      ; preds = %delete.notnull
   br label %for.inc
 
 lpad:                                             ; preds = %delete.notnull
-  %1 = landingpad { i8*, i32 } personality i8* bitcast (i32 (...)* @__gxx_personality_v0 to i8*)
+  %1 = landingpad { i8*, i32 }
           cleanup
   %2 = extractvalue { i8*, i32 } %1, 0
   %3 = extractvalue { i8*, i32 } %1, 1
diff --git a/test/Transforms/LoopUnswitch/2011-11-18-SimpleSwitch.ll b/test/Transforms/LoopUnswitch/2011-11-18-SimpleSwitch.ll
index a35596aff11..abeea209f63 100644
--- a/test/Transforms/LoopUnswitch/2011-11-18-SimpleSwitch.ll
+++ b/test/Transforms/LoopUnswitch/2011-11-18-SimpleSwitch.ll
@@ -34,7 +34,7 @@
 ; CHECK-NEXT:   br label %loop_begin.us1
 
 ; CHECK:      loop_begin.us1:                                   ; preds = %loop_begin.backedge.us5, %.split.split.us
-; CHECK-NEXT:   %var_val.us2 = load i32, i32* %var
+; CHECK-NEXT:   %var_val.us.2 = load i32, i32* %var
 ; CHECK-NEXT:   switch i32 2, label %default.us-lcssa.us-lcssa.us [
 ; CHECK-NEXT:     i32 1, label %inc.us4
 ; CHECK-NEXT:     i32 2, label %dec.us3
diff --git a/test/Transforms/LoopUnswitch/2011-11-18-TwoSwitches.ll b/test/Transforms/LoopUnswitch/2011-11-18-TwoSwitches.ll
index 20f03c987eb..02552ea5cc4 100644
--- a/test/Transforms/LoopUnswitch/2011-11-18-TwoSwitches.ll
+++ b/test/Transforms/LoopUnswitch/2011-11-18-TwoSwitches.ll
@@ -65,7 +65,7 @@
 ; CHECK-NEXT:   br label %loop_begin.us1
 
 ; CHECK:      loop_begin.us1:                                   ; preds = %loop_begin.backedge.us6, %.split.split.us
-; CHECK-NEXT:   %var_val.us2 = load i32, i32* %var
+; CHECK-NEXT:   %var_val.us.2 = load i32, i32* %var
 ; CHECK-NEXT:   switch i32 %c, label %second_switch.us3 [
 ; CHECK-NEXT:     i32 1, label %loop_begin.inc_crit_edge.us
 ; CHECK-NEXT:   ]
diff --git a/test/Transforms/LoopUnswitch/2012-04-30-LoopUnswitch-LPad-Crash.ll b/test/Transforms/LoopUnswitch/2012-04-30-LoopUnswitch-LPad-Crash.ll
index 223fbf18bf5..2c1847a545b 100644
--- a/test/Transforms/LoopUnswitch/2012-04-30-LoopUnswitch-LPad-Crash.ll
+++ b/test/Transforms/LoopUnswitch/2012-04-30-LoopUnswitch-LPad-Crash.ll
@@ -8,7 +8,7 @@ target triple = "x86_64-apple-macosx10.7.0"
 %class.B.21.41.65.101.137.157.177.197.237.241.245.249.261.293.301.337.345.378 = type { %class.A.20.40.64.100.136.156.176.196.236.240.244.248.260.292.300.336.344.377* }
 %class.A.20.40.64.100.136.156.176.196.236.240.244.248.260.292.300.336.344.377 = type { i8 }
 
-define void @_Z23get_reconstruction_pathv() uwtable ssp {
+define void @_Z23get_reconstruction_pathv() uwtable ssp personality i8* bitcast (i32 (...)* @__gxx_personality_v0 to i8*) {
 entry:
   %c = alloca %class.D.22.42.66.102.138.158.178.198.238.242.246.250.262.294.302.338.346.379, align 8
   br label %for.cond
@@ -33,7 +33,7 @@ invoke.cont6:                                     ; preds = %invoke.cont4
   br i1 undef, label %for.cond3, label %for.end
 
 lpad:                                             ; preds = %for.end, %invoke.cont4, %for.cond3, %invoke.cont, %for.cond
-  %0 = landingpad { i8*, i32 } personality i8* bitcast (i32 (...)* @__gxx_personality_v0 to i8*)
+  %0 = landingpad { i8*, i32 }
           cleanup
   resume { i8*, i32 } undef
 
diff --git a/test/Transforms/LoopUnswitch/2015-06-17-Metadata.ll b/test/Transforms/LoopUnswitch/2015-06-17-Metadata.ll
new file mode 100644
index 00000000000..d536da1e8b6
--- /dev/null
+++ b/test/Transforms/LoopUnswitch/2015-06-17-Metadata.ll
@@ -0,0 +1,77 @@
+;RUN: opt  -loop-unswitch -simplifycfg -S < %s | FileCheck %s
+
+define i32 @foo(i32 %a, i32 %b) {
+;CHECK-LABEL: foo
+entry:
+  br label %for.body.lr.ph
+
+for.body.lr.ph:                                   ; preds = %entry
+  %cmp0 = icmp sgt i32 %b, 0
+  br i1 %cmp0, label %for.body, label %for.cond.cleanup
+
+for.body:                                         ; preds = %for.inc, %for.body.lr.ph
+  %inc.i = phi i32 [ 0, %for.body.lr.ph ], [ %inc, %for.inc ]
+  %mul.i = phi i32 [ 3, %for.body.lr.ph ], [ %mul.p, %for.inc ]
+  %add.i = phi i32 [ %a, %for.body.lr.ph ], [ %add.p, %for.inc ]
+  %cmp1 = icmp eq i32 %a, 12345
+  br i1 %cmp1, label %if.then, label %if.else, !prof !0
+; CHECK: %cmp1 = icmp eq i32 %a, 12345
+; CHECK-NEXT: br i1 %cmp1, label %if.then.us, label %if.else, !prof !0
+if.then:                                          ; preds = %for.body
+; CHECK: if.then.us:
+; CHECK: add nsw i32 %{{.*}}, 123
+; CHECK: %exitcond.us = icmp eq i32 %inc.us, %b
+; CHECK: br i1 %exitcond.us, label %for.cond.cleanup, label %if.then.us
+  %add = add nsw i32 %add.i, 123
+  br label %for.inc
+
+if.else:                                          ; preds = %for.body
+  %mul = mul nsw i32 %mul.i, %b
+  br label %for.inc
+; CHECK: if.else:
+; CHECK: %mul = mul nsw i32 %mul.i, %b
+; CHECK: %inc = add nuw nsw i32 %inc.i, 1
+; CHECK: %exitcond = icmp eq i32 %inc, %b
+; CHECK: br i1 %exitcond, label %for.cond.cleanup, label %if.else
+for.inc:                                          ; preds = %if.then, %if.else
+  %mul.p = phi i32 [ %b, %if.then ], [ %mul, %if.else ]
+  %add.p = phi i32 [ %add, %if.then ], [ %a, %if.else ]
+  %inc = add nuw nsw i32 %inc.i, 1
+  %exitcond = icmp eq i32 %inc, %b
+  br i1 %exitcond, label %for.cond.cleanup, label %for.body
+
+for.cond.cleanup:                                 ; preds = %for.inc, %for.body.lr.ph
+  %t2 = phi i32 [ %b, %for.body.lr.ph ], [ %mul.p, %for.inc ]
+  %t1 = phi i32 [ %a, %for.body.lr.ph ], [ %add.p, %for.inc ]
+  %add3 = add nsw i32 %t2, %t1
+  ret i32 %add3
+}
+
+define void @foo_swapped(i32 %a, i32 %b) {
+;CHECK-LABEL: foo_swapped
+entry:
+  br label %for.body
+;CHECK: entry:
+;CHECK-NEXT: %cmp1 = icmp eq i32 1, 2
+;CHECK-NEXT: br i1 %cmp1, label %for.body, label %for.cond.cleanup.split, !prof !1
+;CHECK: for.body:
+for.body:                                         ; preds = %for.inc, %entry
+  %inc.i = phi i32 [ 0, %entry ], [ %inc, %if.then ]
+  %add.i = phi i32 [ 100, %entry ], [ %add, %if.then ]
+  %inc = add nuw nsw i32 %inc.i, 1
+  %cmp1 = icmp eq i32 1, 2
+  br i1 %cmp1, label %if.then, label  %for.cond.cleanup, !prof !0
+
+if.then:                                          ; preds = %for.body
+  %add = add nsw i32 %a, %add.i
+
+  %exitcond = icmp eq i32 %inc, %b
+  br i1 %exitcond, label %for.cond.cleanup, label %for.body
+
+for.cond.cleanup:                                 ; preds = %for.inc, %for.body.lr.ph, %for.body
+  ret void
+}
+!0 = !{!"branch_weights", i32 64, i32 4}
+
+;CHECK: !0 = !{!"branch_weights", i32 64, i32 4}
+;CHECK: !1 = !{!"branch_weights", i32 4, i32 64}
diff --git a/test/Transforms/LoopUnswitch/LIV-loop-condtion.ll b/test/Transforms/LoopUnswitch/LIV-loop-condtion.ll
new file mode 100644
index 00000000000..dd436f474ea
--- /dev/null
+++ b/test/Transforms/LoopUnswitch/LIV-loop-condtion.ll
@@ -0,0 +1,28 @@
+; RUN: opt < %s -loop-unswitch -loop-unswitch-threshold=0 -S 2>&1 | FileCheck %s
+
+; This is to test trivial loop unswitch only happens when trivial condition
+; itself is an LIV loop condition (not partial LIV which could occur in and/or).
+
+define i32 @test(i1 %cond1, i32 %var1) {
+entry:
+  br label %loop_begin
+
+loop_begin:
+  %var3 = phi i32 [%var1, %entry], [%var2, %do_something]
+  %cond2 = icmp eq i32 %var3, 10
+  %cond.and = and i1 %cond1, %cond2
+  
+; %cond.and only has %cond1 as LIV so no unswitch should happen.
+; CHECK: br i1 %cond.and, label %do_something, label %loop_exit
+  br i1 %cond.and, label %do_something, label %loop_exit 
+
+do_something:
+  %var2 = add i32 %var3, 1
+  call void @some_func() noreturn nounwind
+  br label %loop_begin
+
+loop_exit:
+  ret i32 0
+}
+
+declare void @some_func() noreturn 
\ No newline at end of file
diff --git a/test/Transforms/LoopUnswitch/infinite-loop.ll b/test/Transforms/LoopUnswitch/infinite-loop.ll
index e79d874d9ca..3d1c895edec 100644
--- a/test/Transforms/LoopUnswitch/infinite-loop.ll
+++ b/test/Transforms/LoopUnswitch/infinite-loop.ll
@@ -9,23 +9,23 @@
 ; It can trivially unswitch on the false cas of condition %a though.
 
 ; STATS: 2 loop-unswitch - Number of branches unswitched
-; STATS: 1 loop-unswitch - Number of unswitches that are trivial
+; STATS: 2 loop-unswitch - Number of unswitches that are trivial
 
 ; CHECK-LABEL: @func_16(
 ; CHECK-NEXT: entry:
 ; CHECK-NEXT: br i1 %a, label %entry.split, label %abort0.split
 
 ; CHECK: entry.split:
-; CHECK-NEXT: br i1 %b, label %cond.end.us, label %abort1
+; CHECK-NEXT: br i1 %b, label %cond.end, label %abort1.split
 
-; CHECK: cond.end.us:
-; CHECK-NEXT: br label %cond.end.us
+; CHECK: cond.end:
+; CHECK-NEXT: br label %cond.end
 
 ; CHECK: abort0.split:
 ; CHECK-NEXT: call void @end0() [[NOR_NUW:#[0-9]+]]
 ; CHECK-NEXT: unreachable
 
-; CHECK: abort1:
+; CHECK: abort1.split:
 ; CHECK-NEXT: call void @end1() [[NOR_NUW]]
 ; CHECK-NEXT: unreachable
 
diff --git a/test/Transforms/LoopUnswitch/trivial-unswitch.ll b/test/Transforms/LoopUnswitch/trivial-unswitch.ll
new file mode 100644
index 00000000000..db3328278da
--- /dev/null
+++ b/test/Transforms/LoopUnswitch/trivial-unswitch.ll
@@ -0,0 +1,47 @@
+; RUN: opt < %s -loop-unswitch -loop-unswitch-threshold=0 -verify-loop-info -S < %s 2>&1 | FileCheck %s
+
+; This test contains two trivial unswitch condition in one loop. 
+; LoopUnswitch pass should be able to unswitch the second one 
+; after unswitching the first one.
+
+
+; CHECK:  br i1 %cond1, label %..split_crit_edge, label %.loop_exit.split_crit_edge
+
+; CHECK:  ..split_crit_edge:                                ; preds = %0
+; CHECK:    br label %.split
+
+; CHECK:  .split:                                           ; preds = %..split_crit_edge
+; CHECK:    br i1 %cond2, label %.split..split.split_crit_edge, label %.split.loop_exit.split1_crit_edge
+
+; CHECK:  .split..split.split_crit_edge:                    ; preds = %.split
+; CHECK:    br label %.split.split
+
+; CHECK:  .split.split:                                     ; preds = %.split..split.split_crit_edge
+; CHECK:    br label %loop_begin
+
+; CHECK:  loop_begin:                                       ; preds = %do_something, %.split.split
+; CHECK:    br i1 true, label %continue, label %loop_exit
+
+; CHECK:  continue:                                         ; preds = %loop_begin
+; CHECK:    %var_val = load i32, i32* %var
+; CHECK:    br i1 true, label %do_something, label %loop_exit
+
+define i32 @test(i32* %var, i1 %cond1, i1 %cond2) {
+  br label %loop_begin
+
+loop_begin:  
+  br i1 %cond1, label %continue, label %loop_exit	; first trivial condition
+
+continue:
+  %var_val = load i32, i32* %var
+  br i1 %cond2, label %do_something, label %loop_exit	; second trivial condition  
+
+do_something:
+  call void @some_func() noreturn nounwind
+  br label %loop_begin
+
+loop_exit:
+  ret i32 0
+}
+
+declare void @some_func() noreturn
\ No newline at end of file
diff --git a/test/Transforms/LoopVectorize/AArch64/arbitrary-induction-step.ll b/test/Transforms/LoopVectorize/AArch64/arbitrary-induction-step.ll
index 4cd703f6458..f16ee4171da 100644
--- a/test/Transforms/LoopVectorize/AArch64/arbitrary-induction-step.ll
+++ b/test/Transforms/LoopVectorize/AArch64/arbitrary-induction-step.ll
@@ -1,5 +1,5 @@
-; RUN: opt -S < %s -loop-vectorize 2>&1 | FileCheck %s
-; RUN: opt -S < %s -loop-vectorize -force-vector-interleave=1 -force-vector-width=2 | FileCheck %s --check-prefix=FORCE-VEC
+; RUN: opt -S < %s -loop-vectorize -force-vector-interleave=2 -force-vector-width=4 -enable-interleaved-mem-accesses=true | FileCheck %s
+; RUN: opt -S < %s -loop-vectorize -force-vector-interleave=1 -force-vector-width=2 -enable-interleaved-mem-accesses=true | FileCheck %s --check-prefix=FORCE-VEC
 
 target datalayout = "e-m:e-i64:64-i128:128-n32:64-S128"
 target triple = "aarch64--linux-gnueabi"
@@ -102,26 +102,23 @@ for.end:                                          ; preds = %for.body
 ;   }
 
 ; CHECK-LABEL: @ptr_ind_plus2(
-; CHECK: load i32, i32*
-; CHECK: load i32, i32*
-; CHECK: load i32, i32*
-; CHECK: load i32, i32*
-; CHECK: mul nsw i32
-; CHECK: mul nsw i32
-; CHECK: add nsw i32
-; CHECK: add nsw i32
-; CHECK: %index.next = add i64 %index, 2
-; CHECK: %21 = icmp eq i64 %index.next, 1024
+; CHECK: %[[V0:.*]] = load <8 x i32>
+; CHECK: shufflevector <8 x i32> %[[V0]], <8 x i32> undef, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
+; CHECK: shufflevector <8 x i32> %[[V0]], <8 x i32> undef, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
+; CHECK: %[[V1:.*]] = load <8 x i32>
+; CHECK: shufflevector <8 x i32> %[[V1]], <8 x i32> undef, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
+; CHECK: shufflevector <8 x i32> %[[V1]], <8 x i32> undef, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
+; CHECK: mul nsw <4 x i32>
+; CHECK: mul nsw <4 x i32>
+; CHECK: add nsw <4 x i32>
+; CHECK: add nsw <4 x i32>
+; CHECK: %index.next = add i64 %index, 8
+; CHECK: icmp eq i64 %index.next, 1024
 
 ; FORCE-VEC-LABEL: @ptr_ind_plus2(
-; FORCE-VEC: load i32, i32*
-; FORCE-VEC: insertelement <2 x i32>
-; FORCE-VEC: load i32, i32*
-; FORCE-VEC: insertelement <2 x i32>
-; FORCE-VEC: load i32, i32*
-; FORCE-VEC: insertelement <2 x i32>
-; FORCE-VEC: load i32, i32*
-; FORCE-VEC: insertelement <2 x i32>
+; FORCE-VEC: %[[V:.*]] = load <4 x i32>
+; FORCE-VEC: shufflevector <4 x i32> %[[V]], <4 x i32> undef, <2 x i32> <i32 0, i32 2>
+; FORCE-VEC: shufflevector <4 x i32> %[[V]], <4 x i32> undef, <2 x i32> <i32 1, i32 3>
 ; FORCE-VEC: mul nsw <2 x i32>
 ; FORCE-VEC: add nsw <2 x i32>
 ; FORCE-VEC: %index.next = add i64 %index, 2
diff --git a/test/Transforms/LoopVectorize/AArch64/interleaved_cost.ll b/test/Transforms/LoopVectorize/AArch64/interleaved_cost.ll
new file mode 100644
index 00000000000..fe61ef794a4
--- /dev/null
+++ b/test/Transforms/LoopVectorize/AArch64/interleaved_cost.ll
@@ -0,0 +1,39 @@
+; RUN: opt -S -debug-only=loop-vectorize -loop-vectorize -instcombine -enable-interleaved-mem-accesses=true  < %s 2>&1 | FileCheck %s
+; REQUIRES: asserts
+
+target datalayout = "e-m:e-i64:64-i128:128-n32:64-S128"
+target triple = "aarch64--linux-gnueabi"
+
+@AB = common global [1024 x i8] zeroinitializer, align 4
+@CD = common global [1024 x i8] zeroinitializer, align 4
+
+define void @test_byte_interleaved_cost(i8 %C, i8 %D) {
+entry:
+  br label %for.body
+
+; 8xi8 and 16xi8 are valid i8 vector types, so the cost of the interleaved
+; access group is 2.
+
+; CHECK: LV: Found an estimated cost of 2 for VF 8 For instruction:   %tmp = load i8, i8* %arrayidx0, align 4
+; CHECK: LV: Found an estimated cost of 2 for VF 16 For instruction:   %tmp = load i8, i8* %arrayidx0, align 4
+
+for.body:                                         ; preds = %for.body, %entry
+  %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+  %arrayidx0 = getelementptr inbounds [1024 x i8], [1024 x i8]* @AB, i64 0, i64 %indvars.iv
+  %tmp = load i8, i8* %arrayidx0, align 4
+  %tmp1 = or i64 %indvars.iv, 1
+  %arrayidx1 = getelementptr inbounds [1024 x i8], [1024 x i8]* @AB, i64 0, i64 %tmp1
+  %tmp2 = load i8, i8* %arrayidx1, align 4
+  %add = add nsw i8 %tmp, %C
+  %mul = mul nsw i8 %tmp2, %D
+  %arrayidx2 = getelementptr inbounds [1024 x i8], [1024 x i8]* @CD, i64 0, i64 %indvars.iv
+  store i8 %add, i8* %arrayidx2, align 4
+  %arrayidx3 = getelementptr inbounds [1024 x i8], [1024 x i8]* @CD, i64 0, i64 %tmp1
+  store i8 %mul, i8* %arrayidx3, align 4
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 2
+  %cmp = icmp slt i64 %indvars.iv.next, 1024
+  br i1 %cmp, label %for.body, label %for.end
+
+for.end:                                          ; preds = %for.body
+  ret void
+}
diff --git a/test/Transforms/LoopVectorize/ARM/interleaved_cost.ll b/test/Transforms/LoopVectorize/ARM/interleaved_cost.ll
new file mode 100644
index 00000000000..0dcd8e24500
--- /dev/null
+++ b/test/Transforms/LoopVectorize/ARM/interleaved_cost.ll
@@ -0,0 +1,39 @@
+; RUN: opt -S -debug-only=loop-vectorize -loop-vectorize -instcombine -enable-interleaved-mem-accesses=true  < %s 2>&1 | FileCheck %s
+; REQUIRES: asserts
+
+target datalayout = "e-m:e-i64:64-i128:128-n32:64-S128"
+target triple = "armv8--linux-gnueabihf"
+
+@AB = common global [1024 x i8] zeroinitializer, align 4
+@CD = common global [1024 x i8] zeroinitializer, align 4
+
+define void @test_byte_interleaved_cost(i8 %C, i8 %D) {
+entry:
+  br label %for.body
+
+; 8xi8 and 16xi8 are valid i8 vector types, so the cost of the interleaved
+; access group is 2.
+
+; CHECK: LV: Found an estimated cost of 2 for VF 8 For instruction:   %tmp = load i8, i8* %arrayidx0, align 4
+; CHECK: LV: Found an estimated cost of 2 for VF 16 For instruction:   %tmp = load i8, i8* %arrayidx0, align 4
+
+for.body:                                         ; preds = %for.body, %entry
+  %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+  %arrayidx0 = getelementptr inbounds [1024 x i8], [1024 x i8]* @AB, i64 0, i64 %indvars.iv
+  %tmp = load i8, i8* %arrayidx0, align 4
+  %tmp1 = or i64 %indvars.iv, 1
+  %arrayidx1 = getelementptr inbounds [1024 x i8], [1024 x i8]* @AB, i64 0, i64 %tmp1
+  %tmp2 = load i8, i8* %arrayidx1, align 4
+  %add = add nsw i8 %tmp, %C
+  %mul = mul nsw i8 %tmp2, %D
+  %arrayidx2 = getelementptr inbounds [1024 x i8], [1024 x i8]* @CD, i64 0, i64 %indvars.iv
+  store i8 %add, i8* %arrayidx2, align 4
+  %arrayidx3 = getelementptr inbounds [1024 x i8], [1024 x i8]* @CD, i64 0, i64 %tmp1
+  store i8 %mul, i8* %arrayidx3, align 4
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 2
+  %cmp = icmp slt i64 %indvars.iv.next, 1024
+  br i1 %cmp, label %for.body, label %for.end
+
+for.end:                                          ; preds = %for.body
+  ret void
+}
diff --git a/test/Transforms/LoopVectorize/X86/metadata-enable.ll b/test/Transforms/LoopVectorize/X86/metadata-enable.ll
index ba8e11e5874..74c0c16086f 100644
--- a/test/Transforms/LoopVectorize/X86/metadata-enable.ll
+++ b/test/Transforms/LoopVectorize/X86/metadata-enable.ll
@@ -60,7 +60,7 @@ for.body:                                         ; preds = %for.body, %entry
   %arrayidx2 = getelementptr inbounds i32, i32* %a, i64 %indvars.iv
   store i32 %add, i32* %arrayidx2, align 4
   %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
-  %exitcond = icmp eq i64 %indvars.iv.next, 32
+  %exitcond = icmp eq i64 %indvars.iv.next, 64
   br i1 %exitcond, label %for.end, label %for.body, !llvm.loop !0
 
 for.end:                                          ; preds = %for.body
@@ -111,7 +111,7 @@ for.body:                                         ; preds = %for.body, %entry
   %arrayidx2 = getelementptr inbounds i32, i32* %a, i64 %indvars.iv
   store i32 %add, i32* %arrayidx2, align 4
   %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
-  %exitcond = icmp eq i64 %indvars.iv.next, 32
+  %exitcond = icmp eq i64 %indvars.iv.next, 64
   br i1 %exitcond, label %for.end, label %for.body
 
 for.end:                                          ; preds = %for.body
@@ -162,7 +162,7 @@ for.body:                                         ; preds = %for.body, %entry
   %arrayidx2 = getelementptr inbounds i32, i32* %a, i64 %indvars.iv
   store i32 %add, i32* %arrayidx2, align 4
   %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
-  %exitcond = icmp eq i64 %indvars.iv.next, 32
+  %exitcond = icmp eq i64 %indvars.iv.next, 64
   br i1 %exitcond, label %for.end, label %for.body, !llvm.loop !2
 
 for.end:                                          ; preds = %for.body
diff --git a/test/Transforms/LoopVectorize/X86/ptr-indvar-crash.ll b/test/Transforms/LoopVectorize/X86/ptr-indvar-crash.ll
index 56ea2f4c31d..13ceaef4dbb 100644
--- a/test/Transforms/LoopVectorize/X86/ptr-indvar-crash.ll
+++ b/test/Transforms/LoopVectorize/X86/ptr-indvar-crash.ll
@@ -1,4 +1,4 @@
-; RUN: opt -slp-vectorizer -S %s
+; RUN: opt -loop-vectorize -S %s
 
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
diff --git a/test/Transforms/LoopVectorize/X86/unroll-small-loops.ll b/test/Transforms/LoopVectorize/X86/unroll-small-loops.ll
index 4411da3f0a9..69d2a319a8c 100644
--- a/test/Transforms/LoopVectorize/X86/unroll-small-loops.ll
+++ b/test/Transforms/LoopVectorize/X86/unroll-small-loops.ll
@@ -47,9 +47,11 @@ define i32 @foo(i32* nocapture %A) nounwind uwtable ssp {
 ; CHECK-VECTOR: store <4 x i32>
 ; CHECK-VECTOR: ret
 ;
+; For x86, loop unroll in loop vectorizer is disabled when VF==1.
+;
 ; CHECK-SCALAR-LABEL: @bar(
 ; CHECK-SCALAR: store i32
-; CHECK-SCALAR: store i32
+; CHECK-SCALAR-NOT: store i32
 ; CHECK-SCALAR: ret
 define i32 @bar(i32* nocapture %A, i32 %n) nounwind uwtable ssp {
   %1 = icmp sgt i32 %n, 0
diff --git a/test/Transforms/LoopVectorize/X86/veclib-calls.ll b/test/Transforms/LoopVectorize/X86/veclib-calls.ll
index 62e0a44b508..6f8f5223ce4 100644
--- a/test/Transforms/LoopVectorize/X86/veclib-calls.ll
+++ b/test/Transforms/LoopVectorize/X86/veclib-calls.ll
@@ -180,3 +180,453 @@ for.body:                                         ; preds = %entry, %for.body
 for.end:                                          ; preds = %for.body, %entry
   ret void
 }
+
+;CHECK-LABEL: @ceil_f32(
+;CHECK: vceilf{{.*}}<4 x float>
+;CHECK: ret void
+declare float @ceilf(float) nounwind readnone
+define void @ceil_f32(i32 %n, float* noalias %y, float* noalias %x) nounwind uwtable {
+entry:
+  %cmp6 = icmp sgt i32 %n, 0
+  br i1 %cmp6, label %for.body, label %for.end
+
+for.body:                                         ; preds = %entry, %for.body
+  %indvars.iv = phi i64 [ %indvars.iv.next, %for.body ], [ 0, %entry ]
+  %arrayidx = getelementptr inbounds float, float* %y, i64 %indvars.iv
+  %0 = load float, float* %arrayidx, align 4
+  %call = tail call float @ceilf(float %0) nounwind readnone
+  %arrayidx2 = getelementptr inbounds float, float* %x, i64 %indvars.iv
+  store float %call, float* %arrayidx2, align 4
+  %indvars.iv.next = add i64 %indvars.iv, 1
+  %lftr.wideiv = trunc i64 %indvars.iv.next to i32
+  %exitcond = icmp eq i32 %lftr.wideiv, %n
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:                                          ; preds = %for.body, %entry
+  ret void
+}
+
+;CHECK-LABEL: @floor_f32(
+;CHECK: vfloorf{{.*}}<4 x float>
+;CHECK: ret void
+declare float @floorf(float) nounwind readnone
+define void @floor_f32(i32 %n, float* noalias %y, float* noalias %x) nounwind uwtable {
+entry:
+  %cmp6 = icmp sgt i32 %n, 0
+  br i1 %cmp6, label %for.body, label %for.end
+
+for.body:                                         ; preds = %entry, %for.body
+  %indvars.iv = phi i64 [ %indvars.iv.next, %for.body ], [ 0, %entry ]
+  %arrayidx = getelementptr inbounds float, float* %y, i64 %indvars.iv
+  %0 = load float, float* %arrayidx, align 4
+  %call = tail call float @floorf(float %0) nounwind readnone
+  %arrayidx2 = getelementptr inbounds float, float* %x, i64 %indvars.iv
+  store float %call, float* %arrayidx2, align 4
+  %indvars.iv.next = add i64 %indvars.iv, 1
+  %lftr.wideiv = trunc i64 %indvars.iv.next to i32
+  %exitcond = icmp eq i32 %lftr.wideiv, %n
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:                                          ; preds = %for.body, %entry
+  ret void
+}
+
+;CHECK-LABEL: @expm1_f32(
+;CHECK: vexpm1f{{.*}}<4 x float>
+;CHECK: ret void
+declare float @expm1f(float) nounwind readnone
+define void @expm1_f32(i32 %n, float* noalias %y, float* noalias %x) nounwind uwtable {
+entry:
+  %cmp6 = icmp sgt i32 %n, 0
+  br i1 %cmp6, label %for.body, label %for.end
+
+for.body:                                         ; preds = %entry, %for.body
+  %indvars.iv = phi i64 [ %indvars.iv.next, %for.body ], [ 0, %entry ]
+  %arrayidx = getelementptr inbounds float, float* %y, i64 %indvars.iv
+  %0 = load float, float* %arrayidx, align 4
+  %call = tail call float @expm1f(float %0) nounwind readnone
+  %arrayidx2 = getelementptr inbounds float, float* %x, i64 %indvars.iv
+  store float %call, float* %arrayidx2, align 4
+  %indvars.iv.next = add i64 %indvars.iv, 1
+  %lftr.wideiv = trunc i64 %indvars.iv.next to i32
+  %exitcond = icmp eq i32 %lftr.wideiv, %n
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:                                          ; preds = %for.body, %entry
+  ret void
+}
+
+;CHECK-LABEL: @log1p_f32(
+;CHECK: vlog1pf{{.*}}<4 x float>
+;CHECK: ret void
+declare float @log1pf(float) nounwind readnone
+define void @log1p_f32(i32 %n, float* noalias %y, float* noalias %x) nounwind uwtable {
+entry:
+  %cmp6 = icmp sgt i32 %n, 0
+  br i1 %cmp6, label %for.body, label %for.end
+
+for.body:                                         ; preds = %entry, %for.body
+  %indvars.iv = phi i64 [ %indvars.iv.next, %for.body ], [ 0, %entry ]
+  %arrayidx = getelementptr inbounds float, float* %y, i64 %indvars.iv
+  %0 = load float, float* %arrayidx, align 4
+  %call = tail call float @log1pf(float %0) nounwind readnone
+  %arrayidx2 = getelementptr inbounds float, float* %x, i64 %indvars.iv
+  store float %call, float* %arrayidx2, align 4
+  %indvars.iv.next = add i64 %indvars.iv, 1
+  %lftr.wideiv = trunc i64 %indvars.iv.next to i32
+  %exitcond = icmp eq i32 %lftr.wideiv, %n
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:                                          ; preds = %for.body, %entry
+  ret void
+}
+
+;CHECK-LABEL: @log10_f32(
+;CHECK: vlog10f{{.*}}<4 x float>
+;CHECK: ret void
+declare float @log10f(float) nounwind readnone
+define void @log10_f32(i32 %n, float* noalias %y, float* noalias %x) nounwind uwtable {
+entry:
+  %cmp6 = icmp sgt i32 %n, 0
+  br i1 %cmp6, label %for.body, label %for.end
+
+for.body:                                         ; preds = %entry, %for.body
+  %indvars.iv = phi i64 [ %indvars.iv.next, %for.body ], [ 0, %entry ]
+  %arrayidx = getelementptr inbounds float, float* %y, i64 %indvars.iv
+  %0 = load float, float* %arrayidx, align 4
+  %call = tail call float @log10f(float %0) nounwind readnone
+  %arrayidx2 = getelementptr inbounds float, float* %x, i64 %indvars.iv
+  store float %call, float* %arrayidx2, align 4
+  %indvars.iv.next = add i64 %indvars.iv, 1
+  %lftr.wideiv = trunc i64 %indvars.iv.next to i32
+  %exitcond = icmp eq i32 %lftr.wideiv, %n
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:                                          ; preds = %for.body, %entry
+  ret void
+}
+
+;CHECK-LABEL: @logb_f32(
+;CHECK: vlogbf{{.*}}<4 x float>
+;CHECK: ret void
+declare float @logbf(float) nounwind readnone
+define void @logb_f32(i32 %n, float* noalias %y, float* noalias %x) nounwind uwtable {
+entry:
+  %cmp6 = icmp sgt i32 %n, 0
+  br i1 %cmp6, label %for.body, label %for.end
+
+for.body:                                         ; preds = %entry, %for.body
+  %indvars.iv = phi i64 [ %indvars.iv.next, %for.body ], [ 0, %entry ]
+  %arrayidx = getelementptr inbounds float, float* %y, i64 %indvars.iv
+  %0 = load float, float* %arrayidx, align 4
+  %call = tail call float @logbf(float %0) nounwind readnone
+  %arrayidx2 = getelementptr inbounds float, float* %x, i64 %indvars.iv
+  store float %call, float* %arrayidx2, align 4
+  %indvars.iv.next = add i64 %indvars.iv, 1
+  %lftr.wideiv = trunc i64 %indvars.iv.next to i32
+  %exitcond = icmp eq i32 %lftr.wideiv, %n
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:                                          ; preds = %for.body, %entry
+  ret void
+}
+
+;CHECK-LABEL: @sin_f32(
+;CHECK: vsinf{{.*}}<4 x float>
+;CHECK: ret void
+declare float @sinf(float) nounwind readnone
+define void @sin_f32(i32 %n, float* noalias %y, float* noalias %x) nounwind uwtable {
+entry:
+  %cmp6 = icmp sgt i32 %n, 0
+  br i1 %cmp6, label %for.body, label %for.end
+
+for.body:                                         ; preds = %entry, %for.body
+  %indvars.iv = phi i64 [ %indvars.iv.next, %for.body ], [ 0, %entry ]
+  %arrayidx = getelementptr inbounds float, float* %y, i64 %indvars.iv
+  %0 = load float, float* %arrayidx, align 4
+  %call = tail call float @sinf(float %0) nounwind readnone
+  %arrayidx2 = getelementptr inbounds float, float* %x, i64 %indvars.iv
+  store float %call, float* %arrayidx2, align 4
+  %indvars.iv.next = add i64 %indvars.iv, 1
+  %lftr.wideiv = trunc i64 %indvars.iv.next to i32
+  %exitcond = icmp eq i32 %lftr.wideiv, %n
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:                                          ; preds = %for.body, %entry
+  ret void
+}
+
+;CHECK-LABEL: @cos_f32(
+;CHECK: vcosf{{.*}}<4 x float>
+;CHECK: ret void
+declare float @cosf(float) nounwind readnone
+define void @cos_f32(i32 %n, float* noalias %y, float* noalias %x) nounwind uwtable {
+entry:
+  %cmp6 = icmp sgt i32 %n, 0
+  br i1 %cmp6, label %for.body, label %for.end
+
+for.body:                                         ; preds = %entry, %for.body
+  %indvars.iv = phi i64 [ %indvars.iv.next, %for.body ], [ 0, %entry ]
+  %arrayidx = getelementptr inbounds float, float* %y, i64 %indvars.iv
+  %0 = load float, float* %arrayidx, align 4
+  %call = tail call float @cosf(float %0) nounwind readnone
+  %arrayidx2 = getelementptr inbounds float, float* %x, i64 %indvars.iv
+  store float %call, float* %arrayidx2, align 4
+  %indvars.iv.next = add i64 %indvars.iv, 1
+  %lftr.wideiv = trunc i64 %indvars.iv.next to i32
+  %exitcond = icmp eq i32 %lftr.wideiv, %n
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:                                          ; preds = %for.body, %entry
+  ret void
+}
+
+;CHECK-LABEL: @tan_f32(
+;CHECK: vtanf{{.*}}<4 x float>
+;CHECK: ret void
+declare float @tanf(float) nounwind readnone
+define void @tan_f32(i32 %n, float* noalias %y, float* noalias %x) nounwind uwtable {
+entry:
+  %cmp6 = icmp sgt i32 %n, 0
+  br i1 %cmp6, label %for.body, label %for.end
+
+for.body:                                         ; preds = %entry, %for.body
+  %indvars.iv = phi i64 [ %indvars.iv.next, %for.body ], [ 0, %entry ]
+  %arrayidx = getelementptr inbounds float, float* %y, i64 %indvars.iv
+  %0 = load float, float* %arrayidx, align 4
+  %call = tail call float @tanf(float %0) nounwind readnone
+  %arrayidx2 = getelementptr inbounds float, float* %x, i64 %indvars.iv
+  store float %call, float* %arrayidx2, align 4
+  %indvars.iv.next = add i64 %indvars.iv, 1
+  %lftr.wideiv = trunc i64 %indvars.iv.next to i32
+  %exitcond = icmp eq i32 %lftr.wideiv, %n
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:                                          ; preds = %for.body, %entry
+  ret void
+}
+
+;CHECK-LABEL: @asin_f32(
+;CHECK: vasinf{{.*}}<4 x float>
+;CHECK: ret void
+declare float @asinf(float) nounwind readnone
+define void @asin_f32(i32 %n, float* noalias %y, float* noalias %x) nounwind uwtable {
+entry:
+  %cmp6 = icmp sgt i32 %n, 0
+  br i1 %cmp6, label %for.body, label %for.end
+
+for.body:                                         ; preds = %entry, %for.body
+  %indvars.iv = phi i64 [ %indvars.iv.next, %for.body ], [ 0, %entry ]
+  %arrayidx = getelementptr inbounds float, float* %y, i64 %indvars.iv
+  %0 = load float, float* %arrayidx, align 4
+  %call = tail call float @asinf(float %0) nounwind readnone
+  %arrayidx2 = getelementptr inbounds float, float* %x, i64 %indvars.iv
+  store float %call, float* %arrayidx2, align 4
+  %indvars.iv.next = add i64 %indvars.iv, 1
+  %lftr.wideiv = trunc i64 %indvars.iv.next to i32
+  %exitcond = icmp eq i32 %lftr.wideiv, %n
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:                                          ; preds = %for.body, %entry
+  ret void
+}
+
+;CHECK-LABEL: @acos_f32(
+;CHECK: vacosf{{.*}}<4 x float>
+;CHECK: ret void
+declare float @acosf(float) nounwind readnone
+define void @acos_f32(i32 %n, float* noalias %y, float* noalias %x) nounwind uwtable {
+entry:
+  %cmp6 = icmp sgt i32 %n, 0
+  br i1 %cmp6, label %for.body, label %for.end
+
+for.body:                                         ; preds = %entry, %for.body
+  %indvars.iv = phi i64 [ %indvars.iv.next, %for.body ], [ 0, %entry ]
+  %arrayidx = getelementptr inbounds float, float* %y, i64 %indvars.iv
+  %0 = load float, float* %arrayidx, align 4
+  %call = tail call float @acosf(float %0) nounwind readnone
+  %arrayidx2 = getelementptr inbounds float, float* %x, i64 %indvars.iv
+  store float %call, float* %arrayidx2, align 4
+  %indvars.iv.next = add i64 %indvars.iv, 1
+  %lftr.wideiv = trunc i64 %indvars.iv.next to i32
+  %exitcond = icmp eq i32 %lftr.wideiv, %n
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:                                          ; preds = %for.body, %entry
+  ret void
+}
+
+;CHECK-LABEL: @atan_f32(
+;CHECK: vatanf{{.*}}<4 x float>
+;CHECK: ret void
+declare float @atanf(float) nounwind readnone
+define void @atan_f32(i32 %n, float* noalias %y, float* noalias %x) nounwind uwtable {
+entry:
+  %cmp6 = icmp sgt i32 %n, 0
+  br i1 %cmp6, label %for.body, label %for.end
+
+for.body:                                         ; preds = %entry, %for.body
+  %indvars.iv = phi i64 [ %indvars.iv.next, %for.body ], [ 0, %entry ]
+  %arrayidx = getelementptr inbounds float, float* %y, i64 %indvars.iv
+  %0 = load float, float* %arrayidx, align 4
+  %call = tail call float @atanf(float %0) nounwind readnone
+  %arrayidx2 = getelementptr inbounds float, float* %x, i64 %indvars.iv
+  store float %call, float* %arrayidx2, align 4
+  %indvars.iv.next = add i64 %indvars.iv, 1
+  %lftr.wideiv = trunc i64 %indvars.iv.next to i32
+  %exitcond = icmp eq i32 %lftr.wideiv, %n
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:                                          ; preds = %for.body, %entry
+  ret void
+}
+
+;CHECK-LABEL: @sinh_f32(
+;CHECK: vsinhf{{.*}}<4 x float>
+;CHECK: ret void
+declare float @sinhf(float) nounwind readnone
+define void @sinh_f32(i32 %n, float* noalias %y, float* noalias %x) nounwind uwtable {
+entry:
+  %cmp6 = icmp sgt i32 %n, 0
+  br i1 %cmp6, label %for.body, label %for.end
+
+for.body:                                         ; preds = %entry, %for.body
+  %indvars.iv = phi i64 [ %indvars.iv.next, %for.body ], [ 0, %entry ]
+  %arrayidx = getelementptr inbounds float, float* %y, i64 %indvars.iv
+  %0 = load float, float* %arrayidx, align 4
+  %call = tail call float @sinhf(float %0) nounwind readnone
+  %arrayidx2 = getelementptr inbounds float, float* %x, i64 %indvars.iv
+  store float %call, float* %arrayidx2, align 4
+  %indvars.iv.next = add i64 %indvars.iv, 1
+  %lftr.wideiv = trunc i64 %indvars.iv.next to i32
+  %exitcond = icmp eq i32 %lftr.wideiv, %n
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:                                          ; preds = %for.body, %entry
+  ret void
+}
+
+;CHECK-LABEL: @cosh_f32(
+;CHECK: vcoshf{{.*}}<4 x float>
+;CHECK: ret void
+declare float @coshf(float) nounwind readnone
+define void @cosh_f32(i32 %n, float* noalias %y, float* noalias %x) nounwind uwtable {
+entry:
+  %cmp6 = icmp sgt i32 %n, 0
+  br i1 %cmp6, label %for.body, label %for.end
+
+for.body:                                         ; preds = %entry, %for.body
+  %indvars.iv = phi i64 [ %indvars.iv.next, %for.body ], [ 0, %entry ]
+  %arrayidx = getelementptr inbounds float, float* %y, i64 %indvars.iv
+  %0 = load float, float* %arrayidx, align 4
+  %call = tail call float @coshf(float %0) nounwind readnone
+  %arrayidx2 = getelementptr inbounds float, float* %x, i64 %indvars.iv
+  store float %call, float* %arrayidx2, align 4
+  %indvars.iv.next = add i64 %indvars.iv, 1
+  %lftr.wideiv = trunc i64 %indvars.iv.next to i32
+  %exitcond = icmp eq i32 %lftr.wideiv, %n
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:                                          ; preds = %for.body, %entry
+  ret void
+}
+
+;CHECK-LABEL: @tanh_f32(
+;CHECK: vtanhf{{.*}}<4 x float>
+;CHECK: ret void
+declare float @tanhf(float) nounwind readnone
+define void @tanh_f32(i32 %n, float* noalias %y, float* noalias %x) nounwind uwtable {
+entry:
+  %cmp6 = icmp sgt i32 %n, 0
+  br i1 %cmp6, label %for.body, label %for.end
+
+for.body:                                         ; preds = %entry, %for.body
+  %indvars.iv = phi i64 [ %indvars.iv.next, %for.body ], [ 0, %entry ]
+  %arrayidx = getelementptr inbounds float, float* %y, i64 %indvars.iv
+  %0 = load float, float* %arrayidx, align 4
+  %call = tail call float @tanhf(float %0) nounwind readnone
+  %arrayidx2 = getelementptr inbounds float, float* %x, i64 %indvars.iv
+  store float %call, float* %arrayidx2, align 4
+  %indvars.iv.next = add i64 %indvars.iv, 1
+  %lftr.wideiv = trunc i64 %indvars.iv.next to i32
+  %exitcond = icmp eq i32 %lftr.wideiv, %n
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:                                          ; preds = %for.body, %entry
+  ret void
+}
+
+;CHECK-LABEL: @asinh_f32(
+;CHECK: vasinhf{{.*}}<4 x float>
+;CHECK: ret void
+declare float @asinhf(float) nounwind readnone
+define void @asinh_f32(i32 %n, float* noalias %y, float* noalias %x) nounwind uwtable {
+entry:
+  %cmp6 = icmp sgt i32 %n, 0
+  br i1 %cmp6, label %for.body, label %for.end
+
+for.body:                                         ; preds = %entry, %for.body
+  %indvars.iv = phi i64 [ %indvars.iv.next, %for.body ], [ 0, %entry ]
+  %arrayidx = getelementptr inbounds float, float* %y, i64 %indvars.iv
+  %0 = load float, float* %arrayidx, align 4
+  %call = tail call float @asinhf(float %0) nounwind readnone
+  %arrayidx2 = getelementptr inbounds float, float* %x, i64 %indvars.iv
+  store float %call, float* %arrayidx2, align 4
+  %indvars.iv.next = add i64 %indvars.iv, 1
+  %lftr.wideiv = trunc i64 %indvars.iv.next to i32
+  %exitcond = icmp eq i32 %lftr.wideiv, %n
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:                                          ; preds = %for.body, %entry
+  ret void
+}
+
+;CHECK-LABEL: @acosh_f32(
+;CHECK: vacoshf{{.*}}<4 x float>
+;CHECK: ret void
+declare float @acoshf(float) nounwind readnone
+define void @acosh_f32(i32 %n, float* noalias %y, float* noalias %x) nounwind uwtable {
+entry:
+  %cmp6 = icmp sgt i32 %n, 0
+  br i1 %cmp6, label %for.body, label %for.end
+
+for.body:                                         ; preds = %entry, %for.body
+  %indvars.iv = phi i64 [ %indvars.iv.next, %for.body ], [ 0, %entry ]
+  %arrayidx = getelementptr inbounds float, float* %y, i64 %indvars.iv
+  %0 = load float, float* %arrayidx, align 4
+  %call = tail call float @acoshf(float %0) nounwind readnone
+  %arrayidx2 = getelementptr inbounds float, float* %x, i64 %indvars.iv
+  store float %call, float* %arrayidx2, align 4
+  %indvars.iv.next = add i64 %indvars.iv, 1
+  %lftr.wideiv = trunc i64 %indvars.iv.next to i32
+  %exitcond = icmp eq i32 %lftr.wideiv, %n
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:                                          ; preds = %for.body, %entry
+  ret void
+}
+
+;CHECK-LABEL: @atanh_f32(
+;CHECK: vatanhf{{.*}}<4 x float>
+;CHECK: ret void
+declare float @atanhf(float) nounwind readnone
+define void @atanh_f32(i32 %n, float* noalias %y, float* noalias %x) nounwind uwtable {
+entry:
+  %cmp6 = icmp sgt i32 %n, 0
+  br i1 %cmp6, label %for.body, label %for.end
+
+for.body:                                         ; preds = %entry, %for.body
+  %indvars.iv = phi i64 [ %indvars.iv.next, %for.body ], [ 0, %entry ]
+  %arrayidx = getelementptr inbounds float, float* %y, i64 %indvars.iv
+  %0 = load float, float* %arrayidx, align 4
+  %call = tail call float @atanhf(float %0) nounwind readnone
+  %arrayidx2 = getelementptr inbounds float, float* %x, i64 %indvars.iv
+  store float %call, float* %arrayidx2, align 4
+  %indvars.iv.next = add i64 %indvars.iv, 1
+  %lftr.wideiv = trunc i64 %indvars.iv.next to i32
+  %exitcond = icmp eq i32 %lftr.wideiv, %n
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:                                          ; preds = %for.body, %entry
+  ret void
+}
diff --git a/test/Transforms/LoopVectorize/X86/vectorization-remarks-missed.ll b/test/Transforms/LoopVectorize/X86/vectorization-remarks-missed.ll
index 60ad3c6dd7a..65cabb05f2f 100644
--- a/test/Transforms/LoopVectorize/X86/vectorization-remarks-missed.ll
+++ b/test/Transforms/LoopVectorize/X86/vectorization-remarks-missed.ll
@@ -122,40 +122,40 @@ attributes #0 = { nounwind }
 !llvm.module.flags = !{!9, !10}
 !llvm.ident = !{!11}
 
-!0 = !MDCompileUnit(language: DW_LANG_C_plus_plus, producer: "clang version 3.5.0", isOptimized: true, runtimeVersion: 6, emissionKind: 2, file: !1, enums: !2, retainedTypes: !2, subprograms: !3, globals: !2, imports: !2)
-!1 = !MDFile(filename: "source.cpp", directory: ".")
+!0 = !DICompileUnit(language: DW_LANG_C_plus_plus, producer: "clang version 3.5.0", isOptimized: true, runtimeVersion: 6, emissionKind: 2, file: !1, enums: !2, retainedTypes: !2, subprograms: !3, globals: !2, imports: !2)
+!1 = !DIFile(filename: "source.cpp", directory: ".")
 !2 = !{}
 !3 = !{!4, !7, !8}
-!4 = !MDSubprogram(name: "test", line: 1, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: true, scopeLine: 1, file: !1, scope: !5, type: !6, function: void (i32*, i32)* @_Z4testPii, variables: !2)
-!5 = !MDFile(filename: "source.cpp", directory: ".")
-!6 = !MDSubroutineType(types: !2)
-!7 = !MDSubprogram(name: "test_disabled", line: 10, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: true, scopeLine: 10, file: !1, scope: !5, type: !6, function: void (i32*, i32)* @_Z13test_disabledPii, variables: !2)
-!8 = !MDSubprogram(name: "test_array_bounds", line: 16, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: true, scopeLine: 16, file: !1, scope: !5, type: !6, function: void (i32*, i32*, i32)* @_Z17test_array_boundsPiS_i, variables: !2)
+!4 = !DISubprogram(name: "test", line: 1, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: true, scopeLine: 1, file: !1, scope: !5, type: !6, function: void (i32*, i32)* @_Z4testPii, variables: !2)
+!5 = !DIFile(filename: "source.cpp", directory: ".")
+!6 = !DISubroutineType(types: !2)
+!7 = !DISubprogram(name: "test_disabled", line: 10, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: true, scopeLine: 10, file: !1, scope: !5, type: !6, function: void (i32*, i32)* @_Z13test_disabledPii, variables: !2)
+!8 = !DISubprogram(name: "test_array_bounds", line: 16, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: true, scopeLine: 16, file: !1, scope: !5, type: !6, function: void (i32*, i32*, i32)* @_Z17test_array_boundsPiS_i, variables: !2)
 !9 = !{i32 2, !"Dwarf Version", i32 2}
 !10 = !{i32 2, !"Debug Info Version", i32 3}
 !11 = !{!"clang version 3.5.0"}
-!12 = !MDLocation(line: 3, column: 8, scope: !13)
-!13 = distinct !MDLexicalBlock(line: 3, column: 3, file: !1, scope: !4)
+!12 = !DILocation(line: 3, column: 8, scope: !13)
+!13 = distinct !DILexicalBlock(line: 3, column: 3, file: !1, scope: !4)
 !14 = !{!14, !15, !15}
 !15 = !{!"llvm.loop.vectorize.enable", i1 true}
-!16 = !MDLocation(line: 4, column: 5, scope: !17)
-!17 = distinct !MDLexicalBlock(line: 3, column: 36, file: !1, scope: !13)
+!16 = !DILocation(line: 4, column: 5, scope: !17)
+!17 = distinct !DILexicalBlock(line: 3, column: 36, file: !1, scope: !13)
 !18 = !{!19, !19, i64 0}
 !19 = !{!"int", !20, i64 0}
 !20 = !{!"omnipotent char", !21, i64 0}
 !21 = !{!"Simple C/C++ TBAA"}
-!22 = !MDLocation(line: 5, column: 9, scope: !23)
-!23 = distinct !MDLexicalBlock(line: 5, column: 9, file: !1, scope: !17)
-!24 = !MDLocation(line: 8, column: 1, scope: !4)
-!25 = !MDLocation(line: 12, column: 8, scope: !26)
-!26 = distinct !MDLexicalBlock(line: 12, column: 3, file: !1, scope: !7)
+!22 = !DILocation(line: 5, column: 9, scope: !23)
+!23 = distinct !DILexicalBlock(line: 5, column: 9, file: !1, scope: !17)
+!24 = !DILocation(line: 8, column: 1, scope: !4)
+!25 = !DILocation(line: 12, column: 8, scope: !26)
+!26 = distinct !DILexicalBlock(line: 12, column: 3, file: !1, scope: !7)
 !27 = !{!27, !28, !29}
 !28 = !{!"llvm.loop.interleave.count", i32 1}
 !29 = !{!"llvm.loop.vectorize.width", i32 1}
-!30 = !MDLocation(line: 13, column: 5, scope: !26)
-!31 = !MDLocation(line: 14, column: 1, scope: !7)
-!32 = !MDLocation(line: 18, column: 8, scope: !33)
-!33 = distinct !MDLexicalBlock(line: 18, column: 3, file: !1, scope: !8)
+!30 = !DILocation(line: 13, column: 5, scope: !26)
+!31 = !DILocation(line: 14, column: 1, scope: !7)
+!32 = !DILocation(line: 18, column: 8, scope: !33)
+!33 = distinct !DILexicalBlock(line: 18, column: 3, file: !1, scope: !8)
 !34 = !{!34, !15}
-!35 = !MDLocation(line: 19, column: 5, scope: !33)
-!36 = !MDLocation(line: 20, column: 1, scope: !8)
+!35 = !DILocation(line: 19, column: 5, scope: !33)
+!36 = !DILocation(line: 20, column: 1, scope: !8)
diff --git a/test/Transforms/LoopVectorize/X86/vectorization-remarks.ll b/test/Transforms/LoopVectorize/X86/vectorization-remarks.ll
index a4e895afcd2..8640950be32 100644
--- a/test/Transforms/LoopVectorize/X86/vectorization-remarks.ll
+++ b/test/Transforms/LoopVectorize/X86/vectorization-remarks.ll
@@ -5,13 +5,13 @@
 ; This code has all the !dbg annotations needed to track source line information,
 ; but is missing the llvm.dbg.cu annotation. This prevents code generation from
 ; emitting debug info in the final output.
-; RUN: llc -mtriple x86_64-pc-linux-gnu %s -o - | FileCheck -check-prefix=DEBUG-OUTPUT %s
+; RUN: llc < %s -mtriple x86_64-pc-linux-gnu -o - | FileCheck -check-prefix=DEBUG-OUTPUT %s
 ; DEBUG-OUTPUT-NOT: .loc
 ; DEBUG-OUTPUT-NOT: {{.*}}.debug_info
 
-; VECTORIZED: remark: vectorization-remarks.c:17:8: vectorized loop (vectorization factor: 4, unrolling interleave factor: 1)
-; UNROLLED: remark: vectorization-remarks.c:17:8: unrolled with interleaving factor 4 (vectorization not beneficial)
 ; NONE: remark: vectorization-remarks.c:17:8: loop not vectorized: vector width and interleave count are explicitly set to 1
+; VECTORIZED: remark: vectorization-remarks.c:17:8: vectorized loop (vectorization width: 4, interleaved count: 1)
+; UNROLLED: remark: vectorization-remarks.c:17:8: interleaved by 4 (vectorization not beneficial)
 
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
 
@@ -49,26 +49,26 @@ declare void @ibar(i32*) #1
 !llvm.module.flags = !{!7, !8}
 !llvm.ident = !{!9}
 
-!1 = !MDFile(filename: "vectorization-remarks.c", directory: ".")
+!1 = !DIFile(filename: "vectorization-remarks.c", directory: ".")
 !2 = !{}
 !3 = !{!4}
-!4 = !MDSubprogram(name: "foo", line: 5, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: true, scopeLine: 6, file: !1, scope: !5, type: !6, function: i32 (i32)* @foo, variables: !2)
-!5 = !MDFile(filename: "vectorization-remarks.c", directory: ".")
-!6 = !MDSubroutineType(types: !2)
+!4 = !DISubprogram(name: "foo", line: 5, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: true, scopeLine: 6, file: !1, scope: !5, type: !6, function: i32 (i32)* @foo, variables: !2)
+!5 = !DIFile(filename: "vectorization-remarks.c", directory: ".")
+!6 = !DISubroutineType(types: !2)
 !7 = !{i32 2, !"Dwarf Version", i32 4}
 !8 = !{i32 1, !"Debug Info Version", i32 3}
 !9 = !{!"clang version 3.5.0 "}
-!10 = !MDLocation(line: 8, column: 3, scope: !4)
+!10 = !DILocation(line: 8, column: 3, scope: !4)
 !11 = !{!12, !12, i64 0}
 !12 = !{!"int", !13, i64 0}
 !13 = !{!"omnipotent char", !14, i64 0}
 !14 = !{!"Simple C/C++ TBAA"}
-!15 = !MDLocation(line: 17, column: 8, scope: !16)
-!16 = distinct !MDLexicalBlock(line: 17, column: 8, file: !1, scope: !17)
-!17 = distinct !MDLexicalBlock(line: 17, column: 8, file: !1, scope: !18)
-!18 = distinct !MDLexicalBlock(line: 17, column: 3, file: !1, scope: !4)
-!19 = !MDLocation(line: 18, column: 5, scope: !20)
-!20 = distinct !MDLexicalBlock(line: 17, column: 27, file: !1, scope: !18)
+!15 = !DILocation(line: 17, column: 8, scope: !16)
+!16 = distinct !DILexicalBlock(line: 17, column: 8, file: !1, scope: !17)
+!17 = distinct !DILexicalBlock(line: 17, column: 8, file: !1, scope: !18)
+!18 = distinct !DILexicalBlock(line: 17, column: 3, file: !1, scope: !4)
+!19 = !DILocation(line: 18, column: 5, scope: !20)
+!20 = distinct !DILexicalBlock(line: 17, column: 27, file: !1, scope: !18)
 !21 = !{!13, !13, i64 0}
-!22 = !MDLocation(line: 20, column: 3, scope: !4)
-!23 = !MDLocation(line: 21, column: 3, scope: !4)
+!22 = !DILocation(line: 20, column: 3, scope: !4)
+!23 = !DILocation(line: 21, column: 3, scope: !4)
diff --git a/test/Transforms/LoopVectorize/conditional-assignment.ll b/test/Transforms/LoopVectorize/conditional-assignment.ll
index 178a0e56bfc..f41f08df07a 100644
--- a/test/Transforms/LoopVectorize/conditional-assignment.ll
+++ b/test/Transforms/LoopVectorize/conditional-assignment.ll
@@ -36,23 +36,23 @@ attributes #0 = { nounwind }
 !llvm.module.flags = !{!7, !8}
 !llvm.ident = !{!9}
 
-!0 = !MDCompileUnit(language: DW_LANG_C99, producer: "clang version 3.6.0", isOptimized: true, emissionKind: 2, file: !1, enums: !2, retainedTypes: !2, subprograms: !3, globals: !2, imports: !2)
-!1 = !MDFile(filename: "source.c", directory: ".")
+!0 = !DICompileUnit(language: DW_LANG_C99, producer: "clang version 3.6.0", isOptimized: true, emissionKind: 2, file: !1, enums: !2, retainedTypes: !2, subprograms: !3, globals: !2, imports: !2)
+!1 = !DIFile(filename: "source.c", directory: ".")
 !2 = !{}
 !3 = !{!4}
-!4 = !MDSubprogram(name: "conditional_store", line: 1, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: true, scopeLine: 1, file: !1, scope: !5, type: !6, function: void (i32*)* @conditional_store, variables: !2)
-!5 = !MDFile(filename: "source.c", directory: ".")
-!6 = !MDSubroutineType(types: !2)
+!4 = !DISubprogram(name: "conditional_store", line: 1, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: true, scopeLine: 1, file: !1, scope: !5, type: !6, function: void (i32*)* @conditional_store, variables: !2)
+!5 = !DIFile(filename: "source.c", directory: ".")
+!6 = !DISubroutineType(types: !2)
 !7 = !{i32 2, !"Dwarf Version", i32 2}
 !8 = !{i32 2, !"Debug Info Version", i32 3}
 !9 = !{!"clang version 3.6.0"}
-!10 = !MDLocation(line: 2, column: 8, scope: !11)
-!11 = distinct !MDLexicalBlock(line: 2, column: 3, file: !1, scope: !4)
-!12 = !MDLocation(line: 3, column: 9, scope: !13)
-!13 = distinct !MDLexicalBlock(line: 3, column: 9, file: !1, scope: !11)
+!10 = !DILocation(line: 2, column: 8, scope: !11)
+!11 = distinct !DILexicalBlock(line: 2, column: 3, file: !1, scope: !4)
+!12 = !DILocation(line: 3, column: 9, scope: !13)
+!13 = distinct !DILexicalBlock(line: 3, column: 9, file: !1, scope: !11)
 !14 = !{!15, !15, i64 0}
 !15 = !{!"int", !16, i64 0}
 !16 = !{!"omnipotent char", !17, i64 0}
 !17 = !{!"Simple C/C++ TBAA"}
-!18 = !MDLocation(line: 3, column: 29, scope: !13)
-!19 = !MDLocation(line: 4, column: 1, scope: !4)
+!18 = !DILocation(line: 3, column: 29, scope: !13)
+!19 = !DILocation(line: 4, column: 1, scope: !4)
diff --git a/test/Transforms/LoopVectorize/control-flow.ll b/test/Transforms/LoopVectorize/control-flow.ll
index ae0e6c67ccb..c56f9122e46 100644
--- a/test/Transforms/LoopVectorize/control-flow.ll
+++ b/test/Transforms/LoopVectorize/control-flow.ll
@@ -55,24 +55,24 @@ attributes #0 = { nounwind }
 !llvm.module.flags = !{!7, !8}
 !llvm.ident = !{!9}
 
-!0 = !MDCompileUnit(language: DW_LANG_C_plus_plus, producer: "clang version 3.5.0", isOptimized: true, runtimeVersion: 6, emissionKind: 2, file: !1, enums: !2, retainedTypes: !2, subprograms: !3, globals: !2, imports: !2)
-!1 = !MDFile(filename: "source.cpp", directory: ".")
+!0 = !DICompileUnit(language: DW_LANG_C_plus_plus, producer: "clang version 3.5.0", isOptimized: true, runtimeVersion: 6, emissionKind: 2, file: !1, enums: !2, retainedTypes: !2, subprograms: !3, globals: !2, imports: !2)
+!1 = !DIFile(filename: "source.cpp", directory: ".")
 !2 = !{}
 !3 = !{!4}
-!4 = !MDSubprogram(name: "test", line: 1, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: true, scopeLine: 2, file: !1, scope: !5, type: !6, function: i32 (i32*, i32)* @_Z4testPii, variables: !2)
-!5 = !MDFile(filename: "source.cpp", directory: ".")
-!6 = !MDSubroutineType(types: !2)
+!4 = !DISubprogram(name: "test", line: 1, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: true, scopeLine: 2, file: !1, scope: !5, type: !6, function: i32 (i32*, i32)* @_Z4testPii, variables: !2)
+!5 = !DIFile(filename: "source.cpp", directory: ".")
+!6 = !DISubroutineType(types: !2)
 !7 = !{i32 2, !"Dwarf Version", i32 2}
 !8 = !{i32 2, !"Debug Info Version", i32 3}
 !9 = !{!"clang version 3.5.0"}
-!10 = !MDLocation(line: 3, column: 8, scope: !11)
-!11 = distinct !MDLexicalBlock(line: 3, column: 3, file: !1, scope: !4)
-!12 = !MDLocation(line: 5, column: 9, scope: !13)
-!13 = distinct !MDLexicalBlock(line: 5, column: 9, file: !1, scope: !14)
-!14 = distinct !MDLexicalBlock(line: 4, column: 3, file: !1, scope: !11)
+!10 = !DILocation(line: 3, column: 8, scope: !11)
+!11 = distinct !DILexicalBlock(line: 3, column: 3, file: !1, scope: !4)
+!12 = !DILocation(line: 5, column: 9, scope: !13)
+!13 = distinct !DILexicalBlock(line: 5, column: 9, file: !1, scope: !14)
+!14 = distinct !DILexicalBlock(line: 4, column: 3, file: !1, scope: !11)
 !15 = !{!16, !16, i64 0}
 !16 = !{!"int", !17, i64 0}
 !17 = !{!"omnipotent char", !18, i64 0}
 !18 = !{!"Simple C/C++ TBAA"}
-!19 = !MDLocation(line: 8, column: 7, scope: !13)
-!20 = !MDLocation(line: 12, column: 3, scope: !4)
+!19 = !DILocation(line: 8, column: 7, scope: !13)
+!20 = !DILocation(line: 12, column: 3, scope: !4)
diff --git a/test/Transforms/LoopVectorize/dbg.value.ll b/test/Transforms/LoopVectorize/dbg.value.ll
index c794ca00af9..c7440f84b2c 100644
--- a/test/Transforms/LoopVectorize/dbg.value.ll
+++ b/test/Transforms/LoopVectorize/dbg.value.ll
@@ -11,7 +11,7 @@ target triple = "x86_64-apple-macosx10.8.0"
 ; CHECK-LABEL: @test(
 define i32 @test() #0 {
 entry:
-  tail call void @llvm.dbg.value(metadata i32 0, i64 0, metadata !9, metadata !MDExpression()), !dbg !18
+  tail call void @llvm.dbg.value(metadata i32 0, i64 0, metadata !9, metadata !DIExpression()), !dbg !18
   br label %for.body, !dbg !18
 
 for.body:
@@ -25,7 +25,7 @@ for.body:
   %arrayidx4 = getelementptr inbounds [1024 x i32], [1024 x i32]* @A, i64 0, i64 %indvars.iv, !dbg !19
   store i32 %add, i32* %arrayidx4, align 4, !dbg !19
   %indvars.iv.next = add i64 %indvars.iv, 1, !dbg !18
-  tail call void @llvm.dbg.value(metadata !{null}, i64 0, metadata !9, metadata !MDExpression()), !dbg !18
+  tail call void @llvm.dbg.value(metadata !{null}, i64 0, metadata !9, metadata !DIExpression()), !dbg !18
   %lftr.wideiv = trunc i64 %indvars.iv.next to i32, !dbg !18
   %exitcond = icmp ne i32 %lftr.wideiv, 1024, !dbg !18
   br i1 %exitcond, label %for.body, label %for.end, !dbg !18
@@ -44,27 +44,27 @@ attributes #1 = { nounwind readnone }
 !llvm.dbg.cu = !{!0}
 !llvm.module.flags = !{!26}
 
-!0 = !MDCompileUnit(language: DW_LANG_C_plus_plus, producer: "clang", isOptimized: true, emissionKind: 0, file: !25, enums: !1, retainedTypes: !1, subprograms: !2, globals: !11)
+!0 = !DICompileUnit(language: DW_LANG_C_plus_plus, producer: "clang", isOptimized: true, emissionKind: 0, file: !25, enums: !1, retainedTypes: !1, subprograms: !2, globals: !11)
 !1 = !{}
 !2 = !{!3}
-!3 = !MDSubprogram(name: "test", linkageName: "test", line: 5, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: true, scopeLine: 5, file: !25, scope: !4, type: !5, function: i32 ()* @test, variables: !8)
-!4 = !MDFile(filename: "test", directory: "/path/to/somewhere")
-!5 = !MDSubroutineType(types: !6)
+!3 = !DISubprogram(name: "test", linkageName: "test", line: 5, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: true, scopeLine: 5, file: !25, scope: !4, type: !5, function: i32 ()* @test, variables: !8)
+!4 = !DIFile(filename: "test", directory: "/path/to/somewhere")
+!5 = !DISubroutineType(types: !6)
 !6 = !{!7}
-!7 = !MDBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
+!7 = !DIBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
 !8 = !{!9}
-!9 = !MDLocalVariable(tag: DW_TAG_auto_variable, name: "i", line: 6, scope: !10, file: !4, type: !7)
-!10 = distinct !MDLexicalBlock(line: 6, column: 0, file: !25, scope: !3)
+!9 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "i", line: 6, scope: !10, file: !4, type: !7)
+!10 = distinct !DILexicalBlock(line: 6, column: 0, file: !25, scope: !3)
 !11 = !{!12, !16, !17}
-!12 = !MDGlobalVariable(name: "A", line: 1, isLocal: false, isDefinition: true, scope: null, file: !4, type: !13, variable: [1024 x i32]* @A)
-!13 = !MDCompositeType(tag: DW_TAG_array_type, size: 32768, align: 32, baseType: !7, elements: !14)
+!12 = !DIGlobalVariable(name: "A", line: 1, isLocal: false, isDefinition: true, scope: null, file: !4, type: !13, variable: [1024 x i32]* @A)
+!13 = !DICompositeType(tag: DW_TAG_array_type, size: 32768, align: 32, baseType: !7, elements: !14)
 !14 = !{!15}
 !15 = !{i32 786465, i64 0, i64 1024}
-!16 = !MDGlobalVariable(name: "B", line: 2, isLocal: false, isDefinition: true, scope: null, file: !4, type: !13, variable: [1024 x i32]* @B)
-!17 = !MDGlobalVariable(name: "C", line: 3, isLocal: false, isDefinition: true, scope: null, file: !4, type: !13, variable: [1024 x i32]* @C)
-!18 = !MDLocation(line: 6, scope: !10)
-!19 = !MDLocation(line: 7, scope: !20)
-!20 = distinct !MDLexicalBlock(line: 6, column: 0, file: !25, scope: !10)
-!24 = !MDLocation(line: 9, scope: !3)
-!25 = !MDFile(filename: "test", directory: "/path/to/somewhere")
+!16 = !DIGlobalVariable(name: "B", line: 2, isLocal: false, isDefinition: true, scope: null, file: !4, type: !13, variable: [1024 x i32]* @B)
+!17 = !DIGlobalVariable(name: "C", line: 3, isLocal: false, isDefinition: true, scope: null, file: !4, type: !13, variable: [1024 x i32]* @C)
+!18 = !DILocation(line: 6, scope: !10)
+!19 = !DILocation(line: 7, scope: !20)
+!20 = distinct !DILexicalBlock(line: 6, column: 0, file: !25, scope: !10)
+!24 = !DILocation(line: 9, scope: !3)
+!25 = !DIFile(filename: "test", directory: "/path/to/somewhere")
 !26 = !{i32 1, !"Debug Info Version", i32 3}
diff --git a/test/Transforms/LoopVectorize/debugloc.ll b/test/Transforms/LoopVectorize/debugloc.ll
index b9f98cfef04..e691afdd693 100644
--- a/test/Transforms/LoopVectorize/debugloc.ll
+++ b/test/Transforms/LoopVectorize/debugloc.ll
@@ -19,10 +19,10 @@ target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f3
 
 define i32 @f(i32* nocapture %a, i32 %size) #0 {
 entry:
-  tail call void @llvm.dbg.value(metadata i32* %a, i64 0, metadata !13, metadata !MDExpression()), !dbg !19
-  tail call void @llvm.dbg.value(metadata i32 %size, i64 0, metadata !14, metadata !MDExpression()), !dbg !19
-  tail call void @llvm.dbg.value(metadata i32 0, i64 0, metadata !15, metadata !MDExpression()), !dbg !20
-  tail call void @llvm.dbg.value(metadata i32 0, i64 0, metadata !16, metadata !MDExpression()), !dbg !21
+  tail call void @llvm.dbg.value(metadata i32* %a, i64 0, metadata !13, metadata !DIExpression()), !dbg !19
+  tail call void @llvm.dbg.value(metadata i32 %size, i64 0, metadata !14, metadata !DIExpression()), !dbg !19
+  tail call void @llvm.dbg.value(metadata i32 0, i64 0, metadata !15, metadata !DIExpression()), !dbg !20
+  tail call void @llvm.dbg.value(metadata i32 0, i64 0, metadata !16, metadata !DIExpression()), !dbg !21
   %cmp4 = icmp eq i32 %size, 0, !dbg !21
   br i1 %cmp4, label %for.end, label %for.body.lr.ph, !dbg !21
 
@@ -35,9 +35,9 @@ for.body:                                         ; preds = %for.body.lr.ph, %fo
   %arrayidx = getelementptr inbounds i32, i32* %a, i64 %indvars.iv, !dbg !22
   %0 = load i32, i32* %arrayidx, align 4, !dbg !22
   %add = add i32 %0, %sum.05, !dbg !22
-  tail call void @llvm.dbg.value(metadata i32 %add.lcssa, i64 0, metadata !15, metadata !MDExpression()), !dbg !22
+  tail call void @llvm.dbg.value(metadata i32 %add.lcssa, i64 0, metadata !15, metadata !DIExpression()), !dbg !22
   %indvars.iv.next = add i64 %indvars.iv, 1, !dbg !21
-  tail call void @llvm.dbg.value(metadata !{null}, i64 0, metadata !16, metadata !MDExpression()), !dbg !21
+  tail call void @llvm.dbg.value(metadata !{null}, i64 0, metadata !16, metadata !DIExpression()), !dbg !21
   %lftr.wideiv = trunc i64 %indvars.iv.next to i32, !dbg !21
   %exitcond = icmp ne i32 %lftr.wideiv, %size, !dbg !21
   br i1 %exitcond, label %for.body, label %for.cond.for.end_crit_edge, !dbg !21
@@ -63,28 +63,28 @@ attributes #1 = { nounwind readnone }
 !llvm.dbg.cu = !{!0}
 !llvm.module.flags = !{!18, !27}
 
-!0 = !MDCompileUnit(language: DW_LANG_C99, producer: "clang version 3.4 (trunk 185038) (llvm/trunk 185097)", isOptimized: true, emissionKind: 0, file: !1, enums: !2, retainedTypes: !2, subprograms: !3, globals: !2, imports: !2)
-!1 = !MDFile(filename: "-", directory: "/Volumes/Data/backedup/dev/os/llvm/debug")
+!0 = !DICompileUnit(language: DW_LANG_C99, producer: "clang version 3.4 (trunk 185038) (llvm/trunk 185097)", isOptimized: true, emissionKind: 0, file: !1, enums: !2, retainedTypes: !2, subprograms: !3, globals: !2, imports: !2)
+!1 = !DIFile(filename: "-", directory: "/Volumes/Data/backedup/dev/os/llvm/debug")
 !2 = !{}
 !3 = !{!4}
-!4 = !MDSubprogram(name: "f", line: 3, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: true, scopeLine: 3, file: !5, scope: !6, type: !7, function: i32 (i32*, i32)* @f, variables: !12)
-!5 = !MDFile(filename: "<stdin>", directory: "/Volumes/Data/backedup/dev/os/llvm/debug")
-!6 = !MDFile(filename: "<stdin>", directory: "/Volumes/Data/backedup/dev/os/llvm/debug")
-!7 = !MDSubroutineType(types: !8)
+!4 = !DISubprogram(name: "f", line: 3, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: true, scopeLine: 3, file: !5, scope: !6, type: !7, function: i32 (i32*, i32)* @f, variables: !12)
+!5 = !DIFile(filename: "<stdin>", directory: "/Volumes/Data/backedup/dev/os/llvm/debug")
+!6 = !DIFile(filename: "<stdin>", directory: "/Volumes/Data/backedup/dev/os/llvm/debug")
+!7 = !DISubroutineType(types: !8)
 !8 = !{!9, !10, !11}
-!9 = !MDBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
-!10 = !MDDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, baseType: !9)
-!11 = !MDBasicType(tag: DW_TAG_base_type, name: "unsigned int", size: 32, align: 32, encoding: DW_ATE_unsigned)
+!9 = !DIBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
+!10 = !DIDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, baseType: !9)
+!11 = !DIBasicType(tag: DW_TAG_base_type, name: "unsigned int", size: 32, align: 32, encoding: DW_ATE_unsigned)
 !12 = !{!13, !14, !15, !16}
-!13 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "a", line: 3, arg: 1, scope: !4, file: !6, type: !10)
-!14 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "size", line: 3, arg: 2, scope: !4, file: !6, type: !11)
-!15 = !MDLocalVariable(tag: DW_TAG_auto_variable, name: "sum", line: 4, scope: !4, file: !6, type: !11)
-!16 = !MDLocalVariable(tag: DW_TAG_auto_variable, name: "i", line: 5, scope: !17, file: !6, type: !11)
-!17 = distinct !MDLexicalBlock(line: 5, column: 0, file: !5, scope: !4)
+!13 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "a", line: 3, arg: 1, scope: !4, file: !6, type: !10)
+!14 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "size", line: 3, arg: 2, scope: !4, file: !6, type: !11)
+!15 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "sum", line: 4, scope: !4, file: !6, type: !11)
+!16 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "i", line: 5, scope: !17, file: !6, type: !11)
+!17 = distinct !DILexicalBlock(line: 5, column: 0, file: !5, scope: !4)
 !18 = !{i32 2, !"Dwarf Version", i32 3}
-!19 = !MDLocation(line: 3, scope: !4)
-!20 = !MDLocation(line: 4, scope: !4)
-!21 = !MDLocation(line: 5, scope: !17)
-!22 = !MDLocation(line: 6, scope: !17)
-!26 = !MDLocation(line: 7, scope: !4)
+!19 = !DILocation(line: 3, scope: !4)
+!20 = !DILocation(line: 4, scope: !4)
+!21 = !DILocation(line: 5, scope: !17)
+!22 = !DILocation(line: 6, scope: !17)
+!26 = !DILocation(line: 7, scope: !4)
 !27 = !{i32 1, !"Debug Info Version", i32 3}
diff --git a/test/Transforms/LoopVectorize/interleaved-accesses.ll b/test/Transforms/LoopVectorize/interleaved-accesses.ll
new file mode 100644
index 00000000000..d7237a5c27d
--- /dev/null
+++ b/test/Transforms/LoopVectorize/interleaved-accesses.ll
@@ -0,0 +1,467 @@
+; RUN: opt -S -loop-vectorize -instcombine -force-vector-width=4 -force-vector-interleave=1 -enable-interleaved-mem-accesses=true -runtime-memory-check-threshold=24 < %s | FileCheck %s
+
+target datalayout = "e-m:e-i64:64-i128:128-n32:64-S128"
+
+; Check vectorization on an interleaved load group of factor 2 and an interleaved
+; store group of factor 2.
+
+; int AB[1024];
+; int CD[1024];
+;  void test_array_load2_store2(int C, int D) {
+;   for (int i = 0; i < 1024; i+=2) {
+;     int A = AB[i];
+;     int B = AB[i+1];
+;     CD[i] = A + C;
+;     CD[i+1] = B * D;
+;   }
+; }
+
+; CHECK-LABEL: @test_array_load2_store2(
+; CHECK: %wide.vec = load <8 x i32>, <8 x i32>* %{{.*}}, align 4
+; CHECK: shufflevector <8 x i32> %wide.vec, <8 x i32> undef, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
+; CHECK: shufflevector <8 x i32> %wide.vec, <8 x i32> undef, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
+; CHECK: add nsw <4 x i32>
+; CHECK: mul nsw <4 x i32>
+; CHECK: %interleaved.vec = shufflevector <4 x i32> {{.*}}, <8 x i32> <i32 0, i32 4, i32 1, i32 5, i32 2, i32 6, i32 3, i32 7>
+; CHECK: store <8 x i32> %interleaved.vec, <8 x i32>* %{{.*}}, align 4
+
+@AB = common global [1024 x i32] zeroinitializer, align 4
+@CD = common global [1024 x i32] zeroinitializer, align 4
+
+define void @test_array_load2_store2(i32 %C, i32 %D) {
+entry:
+  br label %for.body
+
+for.body:                                         ; preds = %for.body, %entry
+  %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+  %arrayidx0 = getelementptr inbounds [1024 x i32], [1024 x i32]* @AB, i64 0, i64 %indvars.iv
+  %tmp = load i32, i32* %arrayidx0, align 4
+  %tmp1 = or i64 %indvars.iv, 1
+  %arrayidx1 = getelementptr inbounds [1024 x i32], [1024 x i32]* @AB, i64 0, i64 %tmp1
+  %tmp2 = load i32, i32* %arrayidx1, align 4
+  %add = add nsw i32 %tmp, %C
+  %mul = mul nsw i32 %tmp2, %D
+  %arrayidx2 = getelementptr inbounds [1024 x i32], [1024 x i32]* @CD, i64 0, i64 %indvars.iv
+  store i32 %add, i32* %arrayidx2, align 4
+  %arrayidx3 = getelementptr inbounds [1024 x i32], [1024 x i32]* @CD, i64 0, i64 %tmp1
+  store i32 %mul, i32* %arrayidx3, align 4
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 2
+  %cmp = icmp slt i64 %indvars.iv.next, 1024
+  br i1 %cmp, label %for.body, label %for.end
+
+for.end:                                          ; preds = %for.body
+  ret void
+}
+
+; int A[3072];
+; struct ST S[1024];
+; void test_struct_st3() {
+;   int *ptr = A;
+;   for (int i = 0; i < 1024; i++) {
+;     int X1 = *ptr++;
+;     int X2 = *ptr++;
+;     int X3 = *ptr++;
+;     T[i].x = X1 + 1;
+;     T[i].y = X2 + 2;
+;     T[i].z = X3 + 3;
+;   }
+; }
+
+; CHECK-LABEL: @test_struct_array_load3_store3(
+; CHECK: %wide.vec = load <12 x i32>, <12 x i32>* {{.*}}, align 4
+; CHECK: shufflevector <12 x i32> %wide.vec, <12 x i32> undef, <4 x i32> <i32 0, i32 3, i32 6, i32 9>
+; CHECK: shufflevector <12 x i32> %wide.vec, <12 x i32> undef, <4 x i32> <i32 1, i32 4, i32 7, i32 10>
+; CHECK: shufflevector <12 x i32> %wide.vec, <12 x i32> undef, <4 x i32> <i32 2, i32 5, i32 8, i32 11>
+; CHECK: add nsw <4 x i32> {{.*}}, <i32 1, i32 1, i32 1, i32 1>
+; CHECK: add nsw <4 x i32> {{.*}}, <i32 2, i32 2, i32 2, i32 2>
+; CHECK: add nsw <4 x i32> {{.*}}, <i32 3, i32 3, i32 3, i32 3>
+; CHECK: shufflevector <4 x i32> {{.*}}, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; CHECK: shufflevector <4 x i32> {{.*}}, <4 x i32> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef>
+; CHECK: %interleaved.vec = shufflevector <8 x i32> {{.*}}, <12 x i32> <i32 0, i32 4, i32 8, i32 1, i32 5, i32 9, i32 2, i32 6, i32 10, i32 3, i32 7, i32 11>
+; CHECK: store <12 x i32> %interleaved.vec, <12 x i32>* {{.*}}, align 4
+
+%struct.ST3 = type { i32, i32, i32 }
+@A = common global [3072 x i32] zeroinitializer, align 4
+@S = common global [1024 x %struct.ST3] zeroinitializer, align 4
+
+define void @test_struct_array_load3_store3() {
+entry:
+  br label %for.body
+
+for.body:                                         ; preds = %for.body, %entry
+  %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+  %ptr.016 = phi i32* [ getelementptr inbounds ([3072 x i32], [3072 x i32]* @A, i64 0, i64 0), %entry ], [ %incdec.ptr2, %for.body ]
+  %incdec.ptr = getelementptr inbounds i32, i32* %ptr.016, i64 1
+  %tmp = load i32, i32* %ptr.016, align 4
+  %incdec.ptr1 = getelementptr inbounds i32, i32* %ptr.016, i64 2
+  %tmp1 = load i32, i32* %incdec.ptr, align 4
+  %incdec.ptr2 = getelementptr inbounds i32, i32* %ptr.016, i64 3
+  %tmp2 = load i32, i32* %incdec.ptr1, align 4
+  %add = add nsw i32 %tmp, 1
+  %x = getelementptr inbounds [1024 x %struct.ST3], [1024 x %struct.ST3]* @S, i64 0, i64 %indvars.iv, i32 0
+  store i32 %add, i32* %x, align 4
+  %add3 = add nsw i32 %tmp1, 2
+  %y = getelementptr inbounds [1024 x %struct.ST3], [1024 x %struct.ST3]* @S, i64 0, i64 %indvars.iv, i32 1
+  store i32 %add3, i32* %y, align 4
+  %add6 = add nsw i32 %tmp2, 3
+  %z = getelementptr inbounds [1024 x %struct.ST3], [1024 x %struct.ST3]* @S, i64 0, i64 %indvars.iv, i32 2
+  store i32 %add6, i32* %z, align 4
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+  %exitcond = icmp eq i64 %indvars.iv.next, 1024
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:                                          ; preds = %for.body
+  ret void
+}
+
+; Check vectorization on an interleaved load group of factor 4.
+
+; struct ST4{
+;   int x;
+;   int y;
+;   int z;
+;   int w;
+; };
+; int test_struct_load4(struct ST4 *S) {
+;   int r = 0;
+;   for (int i = 0; i < 1024; i++) {
+;      r += S[i].x;
+;      r -= S[i].y;
+;      r += S[i].z;
+;      r -= S[i].w;
+;   }
+;   return r;
+; }
+
+; CHECK-LABEL: @test_struct_load4(
+; CHECK: %wide.vec = load <16 x i32>, <16 x i32>* {{.*}}, align 4
+; CHECK: shufflevector <16 x i32> %wide.vec, <16 x i32> undef, <4 x i32> <i32 0, i32 4, i32 8, i32 12>
+; CHECK: shufflevector <16 x i32> %wide.vec, <16 x i32> undef, <4 x i32> <i32 1, i32 5, i32 9, i32 13>
+; CHECK: shufflevector <16 x i32> %wide.vec, <16 x i32> undef, <4 x i32> <i32 2, i32 6, i32 10, i32 14>
+; CHECK: shufflevector <16 x i32> %wide.vec, <16 x i32> undef, <4 x i32> <i32 3, i32 7, i32 11, i32 15>
+; CHECK: add nsw <4 x i32>
+; CHECK: sub <4 x i32>
+; CHECK: add nsw <4 x i32>
+; CHECK: sub <4 x i32>
+
+%struct.ST4 = type { i32, i32, i32, i32 }
+
+define i32 @test_struct_load4(%struct.ST4* nocapture readonly %S) {
+entry:
+  br label %for.body
+
+for.body:                                         ; preds = %for.body, %entry
+  %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+  %r.022 = phi i32 [ 0, %entry ], [ %sub8, %for.body ]
+  %x = getelementptr inbounds %struct.ST4, %struct.ST4* %S, i64 %indvars.iv, i32 0
+  %tmp = load i32, i32* %x, align 4
+  %add = add nsw i32 %tmp, %r.022
+  %y = getelementptr inbounds %struct.ST4, %struct.ST4* %S, i64 %indvars.iv, i32 1
+  %tmp1 = load i32, i32* %y, align 4
+  %sub = sub i32 %add, %tmp1
+  %z = getelementptr inbounds %struct.ST4, %struct.ST4* %S, i64 %indvars.iv, i32 2
+  %tmp2 = load i32, i32* %z, align 4
+  %add5 = add nsw i32 %sub, %tmp2
+  %w = getelementptr inbounds %struct.ST4, %struct.ST4* %S, i64 %indvars.iv, i32 3
+  %tmp3 = load i32, i32* %w, align 4
+  %sub8 = sub i32 %add5, %tmp3
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+  %exitcond = icmp eq i64 %indvars.iv.next, 1024
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:                                          ; preds = %for.body
+  ret i32 %sub8
+}
+
+; Check vectorization on an interleaved store group of factor 4.
+
+; void test_struct_store4(int *A, struct ST4 *B) {
+;   int *ptr = A;
+;   for (int i = 0; i < 1024; i++) {
+;     int X = *ptr++;
+;     B[i].x = X + 1;
+;     B[i].y = X * 2;
+;     B[i].z = X + 3;
+;     B[i].w = X + 4;
+;   }
+; }
+
+; CHECK-LABEL: @test_struct_store4(
+; CHECK: %[[LD:.*]] = load <4 x i32>, <4 x i32>* 
+; CHECK: add nsw <4 x i32> %[[LD]], <i32 1, i32 1, i32 1, i32 1>
+; CHECK: shl nsw <4 x i32> %[[LD]], <i32 1, i32 1, i32 1, i32 1>
+; CHECK: add nsw <4 x i32> %[[LD]], <i32 3, i32 3, i32 3, i32 3>
+; CHECK: add nsw <4 x i32> %[[LD]], <i32 4, i32 4, i32 4, i32 4>
+; CHECK: shufflevector <4 x i32> {{.*}}, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; CHECK: shufflevector <4 x i32> {{.*}}, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; CHECK: %interleaved.vec = shufflevector <8 x i32> {{.*}}, <16 x i32> <i32 0, i32 4, i32 8, i32 12, i32 1, i32 5, i32 9, i32 13, i32 2, i32 6, i32 10, i32 14, i32 3, i32 7, i32 11, i32 15>
+; CHECK: store <16 x i32> %interleaved.vec, <16 x i32>* {{.*}}, align 4
+
+define void @test_struct_store4(i32* noalias nocapture readonly %A, %struct.ST4* noalias nocapture %B) {
+entry:
+  br label %for.body
+
+for.cond.cleanup:                                 ; preds = %for.body
+  ret void
+
+for.body:                                         ; preds = %for.body, %entry
+  %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+  %ptr.024 = phi i32* [ %A, %entry ], [ %incdec.ptr, %for.body ]
+  %incdec.ptr = getelementptr inbounds i32, i32* %ptr.024, i64 1
+  %tmp = load i32, i32* %ptr.024, align 4
+  %add = add nsw i32 %tmp, 1
+  %x = getelementptr inbounds %struct.ST4, %struct.ST4* %B, i64 %indvars.iv, i32 0
+  store i32 %add, i32* %x, align 4
+  %mul = shl nsw i32 %tmp, 1
+  %y = getelementptr inbounds %struct.ST4, %struct.ST4* %B, i64 %indvars.iv, i32 1
+  store i32 %mul, i32* %y, align 4
+  %add3 = add nsw i32 %tmp, 3
+  %z = getelementptr inbounds %struct.ST4, %struct.ST4* %B, i64 %indvars.iv, i32 2
+  store i32 %add3, i32* %z, align 4
+  %add6 = add nsw i32 %tmp, 4
+  %w = getelementptr inbounds %struct.ST4, %struct.ST4* %B, i64 %indvars.iv, i32 3
+  store i32 %add6, i32* %w, align 4
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+  %exitcond = icmp eq i64 %indvars.iv.next, 1024
+  br i1 %exitcond, label %for.cond.cleanup, label %for.body
+}
+
+; Check vectorization on a reverse interleaved load group of factor 2 and
+; a reverse interleaved store group of factor 2.
+
+; struct ST2 {
+;  int x;
+;  int y;
+; };
+;
+; void test_reversed_load2_store2(struct ST2 *A, struct ST2 *B) {
+;   for (int i = 1023; i >= 0; i--) {
+;     int a = A[i].x + i;  // interleaved load of index 0
+;     int b = A[i].y - i;  // interleaved load of index 1
+;     B[i].x = a;          // interleaved store of index 0
+;     B[i].y = b;          // interleaved store of index 1
+;   }
+; }
+
+; CHECK-LABEL: @test_reversed_load2_store2(
+; CHECK: %wide.vec = load <8 x i32>, <8 x i32>* {{.*}}, align 4
+; CHECK: shufflevector <8 x i32> %wide.vec, <8 x i32> undef, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
+; CHECK: shufflevector <4 x i32> {{.*}}, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
+; CHECK: shufflevector <8 x i32> %wide.vec, <8 x i32> undef, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
+; CHECK: shufflevector <4 x i32> {{.*}}, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
+; CHECK: add nsw <4 x i32>
+; CHECK: sub nsw <4 x i32>
+; CHECK: shufflevector <4 x i32> {{.*}}, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
+; CHECK: shufflevector <4 x i32> {{.*}}, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
+; CHECK: %interleaved.vec = shufflevector <4 x i32> {{.*}}, <8 x i32> <i32 0, i32 4, i32 1, i32 5, i32 2, i32 6, i32 3, i32 7>
+; CHECK: store <8 x i32> %interleaved.vec, <8 x i32>* %{{.*}}, align 4
+
+%struct.ST2 = type { i32, i32 }
+
+define void @test_reversed_load2_store2(%struct.ST2* noalias nocapture readonly %A, %struct.ST2* noalias nocapture %B) {
+entry:
+  br label %for.body
+
+for.cond.cleanup:                                 ; preds = %for.body
+  ret void
+
+for.body:                                         ; preds = %for.body, %entry
+  %indvars.iv = phi i64 [ 1023, %entry ], [ %indvars.iv.next, %for.body ]
+  %x = getelementptr inbounds %struct.ST2, %struct.ST2* %A, i64 %indvars.iv, i32 0
+  %tmp = load i32, i32* %x, align 4
+  %tmp1 = trunc i64 %indvars.iv to i32
+  %add = add nsw i32 %tmp, %tmp1
+  %y = getelementptr inbounds %struct.ST2, %struct.ST2* %A, i64 %indvars.iv, i32 1
+  %tmp2 = load i32, i32* %y, align 4
+  %sub = sub nsw i32 %tmp2, %tmp1
+  %x5 = getelementptr inbounds %struct.ST2, %struct.ST2* %B, i64 %indvars.iv, i32 0
+  store i32 %add, i32* %x5, align 4
+  %y8 = getelementptr inbounds %struct.ST2, %struct.ST2* %B, i64 %indvars.iv, i32 1
+  store i32 %sub, i32* %y8, align 4
+  %indvars.iv.next = add nsw i64 %indvars.iv, -1
+  %cmp = icmp sgt i64 %indvars.iv, 0
+  br i1 %cmp, label %for.body, label %for.cond.cleanup
+}
+
+; Check vectorization on an interleaved load group of factor 2 with 1 gap
+; (missing the load of odd elements).
+
+; void even_load(int *A, int *B) {
+;  for (unsigned i = 0; i < 1024; i+=2)
+;     B[i/2] = A[i] * 2;
+; }
+
+; CHECK-LABEL: @even_load(
+; CHECK: %wide.vec = load <8 x i32>, <8 x i32>* %{{.*}}, align 4
+; CHECK: %strided.vec = shufflevector <8 x i32> %wide.vec, <8 x i32> undef, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
+; CHECK-NOT: shufflevector <8 x i32> %wide.vec, <8 x i32> undef, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
+; CHECK: shl nsw <4 x i32> %strided.vec, <i32 1, i32 1, i32 1, i32 1>
+
+define void @even_load(i32* noalias nocapture readonly %A, i32* noalias nocapture %B) {
+entry:
+  br label %for.body
+
+for.cond.cleanup:                                 ; preds = %for.body
+  ret void
+
+for.body:                                         ; preds = %for.body, %entry
+  %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+  %arrayidx = getelementptr inbounds i32, i32* %A, i64 %indvars.iv
+  %tmp = load i32, i32* %arrayidx, align 4
+  %mul = shl nsw i32 %tmp, 1
+  %tmp1 = lshr exact i64 %indvars.iv, 1
+  %arrayidx2 = getelementptr inbounds i32, i32* %B, i64 %tmp1
+  store i32 %mul, i32* %arrayidx2, align 4
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 2
+  %cmp = icmp ult i64 %indvars.iv.next, 1024
+  br i1 %cmp, label %for.body, label %for.cond.cleanup
+}
+
+; Check vectorization on interleaved access groups identified from mixed
+; loads/stores.
+; void mixed_load2_store2(int *A, int *B) {
+;   for (unsigned i = 0; i < 1024; i+=2)  {
+;     B[i] = A[i] * A[i+1];
+;     B[i+1] = A[i] + A[i+1];
+;   }
+; }
+
+; CHECK-LABEL: @mixed_load2_store2(
+; CHECK: %wide.vec = load <8 x i32>, <8 x i32>* {{.*}}, align 4
+; CHECK: shufflevector <8 x i32> %wide.vec, <8 x i32> undef, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
+; CHECK: shufflevector <8 x i32> %wide.vec, <8 x i32> undef, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
+; CHECK: %interleaved.vec = shufflevector <4 x i32> %{{.*}}, <8 x i32> <i32 0, i32 4, i32 1, i32 5, i32 2, i32 6, i32 3, i32 7>
+; CHECK: store <8 x i32> %interleaved.vec
+
+define void @mixed_load2_store2(i32* noalias nocapture readonly %A, i32* noalias nocapture %B) {
+entry:
+  br label %for.body
+
+for.cond.cleanup:                                 ; preds = %for.body
+  ret void
+
+for.body:                                         ; preds = %for.body, %entry
+  %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+  %arrayidx = getelementptr inbounds i32, i32* %A, i64 %indvars.iv
+  %tmp = load i32, i32* %arrayidx, align 4
+  %tmp1 = or i64 %indvars.iv, 1
+  %arrayidx2 = getelementptr inbounds i32, i32* %A, i64 %tmp1
+  %tmp2 = load i32, i32* %arrayidx2, align 4
+  %mul = mul nsw i32 %tmp2, %tmp
+  %arrayidx4 = getelementptr inbounds i32, i32* %B, i64 %indvars.iv
+  store i32 %mul, i32* %arrayidx4, align 4
+  %tmp3 = load i32, i32* %arrayidx, align 4
+  %tmp4 = load i32, i32* %arrayidx2, align 4
+  %add10 = add nsw i32 %tmp4, %tmp3
+  %arrayidx13 = getelementptr inbounds i32, i32* %B, i64 %tmp1
+  store i32 %add10, i32* %arrayidx13, align 4
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 2
+  %cmp = icmp ult i64 %indvars.iv.next, 1024
+  br i1 %cmp, label %for.body, label %for.cond.cleanup
+}
+
+; Check vectorization on interleaved access groups identified from mixed
+; loads/stores.
+; void mixed_load3_store3(int *A) {
+;   for (unsigned i = 0; i < 1024; i++)  {
+;     *A++ += i;
+;     *A++ += i;
+;     *A++ += i;
+;   }
+; }
+
+; CHECK-LABEL: @mixed_load3_store3(
+; CHECK: %wide.vec = load <12 x i32>, <12 x i32>* {{.*}}, align 4
+; CHECK: shufflevector <12 x i32> %wide.vec, <12 x i32> undef, <4 x i32> <i32 0, i32 3, i32 6, i32 9>
+; CHECK: shufflevector <12 x i32> %wide.vec, <12 x i32> undef, <4 x i32> <i32 1, i32 4, i32 7, i32 10>
+; CHECK: shufflevector <12 x i32> %wide.vec, <12 x i32> undef, <4 x i32> <i32 2, i32 5, i32 8, i32 11>
+; CHECK: %interleaved.vec = shufflevector <8 x i32> %{{.*}}, <12 x i32> <i32 0, i32 4, i32 8, i32 1, i32 5, i32 9, i32 2, i32 6, i32 10, i32 3, i32 7, i32 11>
+; CHECK: store <12 x i32> %interleaved.vec, <12 x i32>* %{{.*}}, align 4
+
+define void @mixed_load3_store3(i32* nocapture %A) {
+entry:
+  br label %for.body
+
+for.cond.cleanup:                                 ; preds = %for.body
+  ret void
+
+for.body:                                         ; preds = %for.body, %entry
+  %i.013 = phi i32 [ 0, %entry ], [ %inc, %for.body ]
+  %A.addr.012 = phi i32* [ %A, %entry ], [ %incdec.ptr3, %for.body ]
+  %incdec.ptr = getelementptr inbounds i32, i32* %A.addr.012, i64 1
+  %tmp = load i32, i32* %A.addr.012, align 4
+  %add = add i32 %tmp, %i.013
+  store i32 %add, i32* %A.addr.012, align 4
+  %incdec.ptr1 = getelementptr inbounds i32, i32* %A.addr.012, i64 2
+  %tmp1 = load i32, i32* %incdec.ptr, align 4
+  %add2 = add i32 %tmp1, %i.013
+  store i32 %add2, i32* %incdec.ptr, align 4
+  %incdec.ptr3 = getelementptr inbounds i32, i32* %A.addr.012, i64 3
+  %tmp2 = load i32, i32* %incdec.ptr1, align 4
+  %add4 = add i32 %tmp2, %i.013
+  store i32 %add4, i32* %incdec.ptr1, align 4
+  %inc = add nuw nsw i32 %i.013, 1
+  %exitcond = icmp eq i32 %inc, 1024
+  br i1 %exitcond, label %for.cond.cleanup, label %for.body
+}
+
+; Check vectorization on interleaved access groups with members having different
+; kinds of type.
+
+; struct IntFloat {
+;   int a;
+;   float b;
+; };
+; 
+; int SA;
+; float SB;
+;
+; void int_float_struct(struct IntFloat *A) {
+;   int SumA;
+;   float SumB;
+;   for (unsigned i = 0; i < 1024; i++)  {
+;     SumA += A[i].a;
+;     SumB += A[i].b;
+;   }
+;   SA = SumA;
+;   SB = SumB;
+; }
+
+; CHECK-LABEL: @int_float_struct(
+; CHECK: %wide.vec = load <8 x i32>, <8 x i32>* %{{.*}}, align 4
+; CHECK: %[[V0:.*]] = shufflevector <8 x i32> %wide.vec, <8 x i32> undef, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
+; CHECK: %[[V1:.*]] = shufflevector <8 x i32> %wide.vec, <8 x i32> undef, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
+; CHECK: bitcast <4 x i32> %[[V1]] to <4 x float>
+; CHECK: add nsw <4 x i32>
+; CHECK: fadd fast <4 x float>
+
+%struct.IntFloat = type { i32, float }
+
+@SA = common global i32 0, align 4
+@SB = common global float 0.000000e+00, align 4
+
+define void @int_float_struct(%struct.IntFloat* nocapture readonly %A) #0 {
+entry:
+  br label %for.body
+
+for.cond.cleanup:                                 ; preds = %for.body
+  store i32 %add, i32* @SA, align 4
+  store float %add3, float* @SB, align 4
+  ret void
+
+for.body:                                         ; preds = %for.body, %entry
+  %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+  %SumB.014 = phi float [ undef, %entry ], [ %add3, %for.body ]
+  %SumA.013 = phi i32 [ undef, %entry ], [ %add, %for.body ]
+  %a = getelementptr inbounds %struct.IntFloat, %struct.IntFloat* %A, i64 %indvars.iv, i32 0
+  %tmp = load i32, i32* %a, align 4
+  %add = add nsw i32 %tmp, %SumA.013
+  %b = getelementptr inbounds %struct.IntFloat, %struct.IntFloat* %A, i64 %indvars.iv, i32 1
+  %tmp1 = load float, float* %b, align 4
+  %add3 = fadd fast float %SumB.014, %tmp1
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+  %exitcond = icmp eq i64 %indvars.iv.next, 1024
+  br i1 %exitcond, label %for.cond.cleanup, label %for.body
+}
+
+attributes #0 = { "unsafe-fp-math"="true" }
diff --git a/test/Transforms/LoopVectorize/no_array_bounds.ll b/test/Transforms/LoopVectorize/no_array_bounds.ll
index 1a2fda1a9a6..f7c7ff7732b 100644
--- a/test/Transforms/LoopVectorize/no_array_bounds.ll
+++ b/test/Transforms/LoopVectorize/no_array_bounds.ll
@@ -72,30 +72,30 @@ attributes #0 = { nounwind }
 !llvm.module.flags = !{!7, !8}
 !llvm.ident = !{!9}
 
-!0 = !MDCompileUnit(language: DW_LANG_C_plus_plus, producer: "clang version 3.5.0", isOptimized: true, emissionKind: 2, file: !1, enums: !2, retainedTypes: !2, subprograms: !3, globals: !2, imports: !2)
-!1 = !MDFile(filename: "no_array_bounds.cpp", directory: ".")
+!0 = !DICompileUnit(language: DW_LANG_C_plus_plus, producer: "clang version 3.5.0", isOptimized: true, emissionKind: 2, file: !1, enums: !2, retainedTypes: !2, subprograms: !3, globals: !2, imports: !2)
+!1 = !DIFile(filename: "no_array_bounds.cpp", directory: ".")
 !2 = !{}
 !3 = !{!4}
-!4 = !MDSubprogram(name: "test", line: 1, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: true, scopeLine: 2, file: !1, scope: !5, type: !6, function: void (i32*, i32*, i32)* @_Z4testPiS_i, variables: !2)
-!5 = !MDFile(filename: "no_array_bounds.cpp", directory: ".")
-!6 = !MDSubroutineType(types: !2)
+!4 = !DISubprogram(name: "test", line: 1, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: true, scopeLine: 2, file: !1, scope: !5, type: !6, function: void (i32*, i32*, i32)* @_Z4testPiS_i, variables: !2)
+!5 = !DIFile(filename: "no_array_bounds.cpp", directory: ".")
+!6 = !DISubroutineType(types: !2)
 !7 = !{i32 2, !"Dwarf Version", i32 2}
 !8 = !{i32 2, !"Debug Info Version", i32 3}
 !9 = !{!"clang version 3.5.0"}
-!10 = !MDLocation(line: 4, column: 8, scope: !11)
-!11 = distinct !MDLexicalBlock(line: 4, column: 3, file: !1, scope: !4)
+!10 = !DILocation(line: 4, column: 8, scope: !11)
+!11 = distinct !DILexicalBlock(line: 4, column: 3, file: !1, scope: !4)
 !12 = !{!12, !13}
 !13 = !{!"llvm.loop.vectorize.enable", i1 true}
-!14 = !MDLocation(line: 5, column: 5, scope: !15)
-!15 = distinct !MDLexicalBlock(line: 4, column: 36, file: !1, scope: !11)
-!16 = !MDLocation(line: 9, column: 8, scope: !17)
-!17 = distinct !MDLexicalBlock(line: 9, column: 3, file: !1, scope: !4)
+!14 = !DILocation(line: 5, column: 5, scope: !15)
+!15 = distinct !DILexicalBlock(line: 4, column: 36, file: !1, scope: !11)
+!16 = !DILocation(line: 9, column: 8, scope: !17)
+!17 = distinct !DILexicalBlock(line: 9, column: 3, file: !1, scope: !4)
 !18 = !{!18, !13, !19}
 !19 = !{!"llvm.loop.vectorize.width", i32 1}
-!20 = !MDLocation(line: 10, column: 5, scope: !21)
-!21 = distinct !MDLexicalBlock(line: 9, column: 36, file: !1, scope: !17)
+!20 = !DILocation(line: 10, column: 5, scope: !21)
+!21 = distinct !DILexicalBlock(line: 9, column: 36, file: !1, scope: !17)
 !22 = !{!23, !23, i64 0}
 !23 = !{!"int", !24, i64 0}
 !24 = !{!"omnipotent char", !25, i64 0}
 !25 = !{!"Simple C/C++ TBAA"}
-!26 = !MDLocation(line: 12, column: 1, scope: !4)
+!26 = !DILocation(line: 12, column: 1, scope: !4)
diff --git a/test/Transforms/LoopVectorize/no_switch.ll b/test/Transforms/LoopVectorize/no_switch.ll
index 14d2fab8936..1f139c26d79 100644
--- a/test/Transforms/LoopVectorize/no_switch.ll
+++ b/test/Transforms/LoopVectorize/no_switch.ll
@@ -59,28 +59,28 @@ attributes #0 = { nounwind }
 !llvm.module.flags = !{!7, !8}
 !llvm.ident = !{!9}
 
-!0 = !MDCompileUnit(language: DW_LANG_C_plus_plus, producer: "clang version 3.5.0", isOptimized: true, runtimeVersion: 6, emissionKind: 2, file: !1, enums: !2, retainedTypes: !2, subprograms: !3, globals: !2, imports: !2)
-!1 = !MDFile(filename: "source.cpp", directory: ".")
+!0 = !DICompileUnit(language: DW_LANG_C_plus_plus, producer: "clang version 3.5.0", isOptimized: true, runtimeVersion: 6, emissionKind: 2, file: !1, enums: !2, retainedTypes: !2, subprograms: !3, globals: !2, imports: !2)
+!1 = !DIFile(filename: "source.cpp", directory: ".")
 !2 = !{}
 !3 = !{!4}
-!4 = !MDSubprogram(name: "test_switch", line: 1, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: true, scopeLine: 1, file: !1, scope: !5, type: !6, function: void (i32*, i32)* @_Z11test_switchPii, variables: !2)
-!5 = !MDFile(filename: "source.cpp", directory: ".")
-!6 = !MDSubroutineType(types: !2)
+!4 = !DISubprogram(name: "test_switch", line: 1, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: true, scopeLine: 1, file: !1, scope: !5, type: !6, function: void (i32*, i32)* @_Z11test_switchPii, variables: !2)
+!5 = !DIFile(filename: "source.cpp", directory: ".")
+!6 = !DISubroutineType(types: !2)
 !7 = !{i32 2, !"Dwarf Version", i32 2}
 !8 = !{i32 2, !"Debug Info Version", i32 3}
 !9 = !{!"clang version 3.5.0"}
-!10 = !MDLocation(line: 3, column: 8, scope: !11)
-!11 = distinct !MDLexicalBlock(line: 3, column: 3, file: !1, scope: !4)
+!10 = !DILocation(line: 3, column: 8, scope: !11)
+!11 = distinct !DILexicalBlock(line: 3, column: 3, file: !1, scope: !4)
 !12 = !{!12, !13, !13}
 !13 = !{!"llvm.loop.vectorize.enable", i1 true}
-!14 = !MDLocation(line: 4, column: 5, scope: !15)
-!15 = distinct !MDLexicalBlock(line: 3, column: 36, file: !1, scope: !11)
+!14 = !DILocation(line: 4, column: 5, scope: !15)
+!15 = distinct !DILexicalBlock(line: 3, column: 36, file: !1, scope: !11)
 !16 = !{!17, !17, i64 0}
 !17 = !{!"int", !18, i64 0}
 !18 = !{!"omnipotent char", !19, i64 0}
 !19 = !{!"Simple C/C++ TBAA"}
-!20 = !MDLocation(line: 6, column: 7, scope: !21)
-!21 = distinct !MDLexicalBlock(line: 4, column: 18, file: !1, scope: !15)
-!22 = !MDLocation(line: 7, column: 5, scope: !21)
-!23 = !MDLocation(line: 9, column: 7, scope: !21)
-!24 = !MDLocation(line: 14, column: 1, scope: !4)
+!20 = !DILocation(line: 6, column: 7, scope: !21)
+!21 = distinct !DILexicalBlock(line: 4, column: 18, file: !1, scope: !15)
+!22 = !DILocation(line: 7, column: 5, scope: !21)
+!23 = !DILocation(line: 9, column: 7, scope: !21)
+!24 = !DILocation(line: 14, column: 1, scope: !4)
diff --git a/test/Transforms/LoopVectorize/optsize.ll b/test/Transforms/LoopVectorize/optsize.ll
new file mode 100644
index 00000000000..e183fda099a
--- /dev/null
+++ b/test/Transforms/LoopVectorize/optsize.ll
@@ -0,0 +1,34 @@
+; This test verifies that the loop vectorizer will NOT produce a tail
+; loop with Optimize for size attibute.
+; REQUIRES: asserts
+; RUN: opt < %s -loop-vectorize -Os -debug -debug-only=loop-vectorize -S 2>&1 | FileCheck %s
+
+;CHECK-NOT: <2 x i8>
+;CHECK-NOT: <4 x i8>
+;CHECK: Aborting. A tail loop is required in Os.
+
+target datalayout = "E-m:e-p:32:32-i64:32-f64:32:64-a:0:32-n32-S128"
+
+@tab = common global [32 x i8] zeroinitializer, align 1
+
+; Function Attrs: nounwind optsize
+define i32 @foo() #0 {
+entry:
+  br label %for.body
+
+for.body:                                         ; preds = %for.body, %entry
+  %i.08 = phi i32 [ 0, %entry ], [ %inc, %for.body ]
+  %arrayidx = getelementptr inbounds [32 x i8], [32 x i8]* @tab, i32 0, i32 %i.08
+  %0 = load i8, i8* %arrayidx, align 1
+  %cmp1 = icmp eq i8 %0, 0
+  %. = select i1 %cmp1, i8 2, i8 1
+  store i8 %., i8* %arrayidx, align 1
+  %inc = add nsw i32 %i.08, 1
+  %exitcond = icmp eq i32 %i.08, 202
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:                                          ; preds = %for.body
+  ret i32 0
+}
+
+attributes #0 = { nounwind optsize "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
diff --git a/test/Transforms/LoopVectorize/runtime-check.ll b/test/Transforms/LoopVectorize/runtime-check.ll
index b1c5d402c45..1f07d3f6959 100644
--- a/test/Transforms/LoopVectorize/runtime-check.ll
+++ b/test/Transforms/LoopVectorize/runtime-check.ll
@@ -9,30 +9,31 @@ target triple = "x86_64-apple-macosx10.9.0"
 ;     a[i] = b[i] * 3;
 ; }
 
+;CHECK-LABEL: define i32 @foo
 ;CHECK: for.body.preheader:
-;CHECK: br i1 %cmp.zero, label %middle.block, label %vector.memcheck
+;CHECK: br i1 %cmp.zero, label %middle.block, label %vector.memcheck, !dbg [[BODY_LOC:![0-9]+]]
 ;CHECK: vector.memcheck:
-;CHECK: br i1 %memcheck.conflict, label %middle.block, label %vector.ph
+;CHECK: br i1 %memcheck.conflict, label %middle.block, label %vector.ph, !dbg [[BODY_LOC]]
 ;CHECK: load <4 x float>
 define i32 @foo(float* nocapture %a, float* nocapture %b, i32 %n) nounwind uwtable ssp {
 entry:
-  %cmp6 = icmp sgt i32 %n, 0
-  br i1 %cmp6, label %for.body, label %for.end
+  %cmp6 = icmp sgt i32 %n, 0, !dbg !6
+  br i1 %cmp6, label %for.body, label %for.end, !dbg !6
 
 for.body:                                         ; preds = %entry, %for.body
-  %indvars.iv = phi i64 [ %indvars.iv.next, %for.body ], [ 0, %entry ]
-  %arrayidx = getelementptr inbounds float, float* %b, i64 %indvars.iv
-  %0 = load float, float* %arrayidx, align 4
-  %mul = fmul float %0, 3.000000e+00
-  %arrayidx2 = getelementptr inbounds float, float* %a, i64 %indvars.iv
-  store float %mul, float* %arrayidx2, align 4
-  %indvars.iv.next = add i64 %indvars.iv, 1
-  %lftr.wideiv = trunc i64 %indvars.iv.next to i32
-  %exitcond = icmp eq i32 %lftr.wideiv, %n
-  br i1 %exitcond, label %for.end, label %for.body
+  %indvars.iv = phi i64 [ %indvars.iv.next, %for.body ], [ 0, %entry ], !dbg !7
+  %arrayidx = getelementptr inbounds float, float* %b, i64 %indvars.iv, !dbg !7
+  %0 = load float, float* %arrayidx, align 4, !dbg !7
+  %mul = fmul float %0, 3.000000e+00, !dbg !7
+  %arrayidx2 = getelementptr inbounds float, float* %a, i64 %indvars.iv, !dbg !7
+  store float %mul, float* %arrayidx2, align 4, !dbg !7
+  %indvars.iv.next = add i64 %indvars.iv, 1, !dbg !7
+  %lftr.wideiv = trunc i64 %indvars.iv.next to i32, !dbg !7
+  %exitcond = icmp eq i32 %lftr.wideiv, %n, !dbg !7
+  br i1 %exitcond, label %for.end, label %for.body, !dbg !7
 
 for.end:                                          ; preds = %for.body, %entry
-  ret i32 undef
+  ret i32 undef, !dbg !8
 }
 
 ; Make sure that we try to vectorize loops with a runtime check if the
@@ -62,3 +63,17 @@ for.body:
 loopexit:
   ret void
 }
+
+; CHECK: [[BODY_LOC]] = !DILocation(line: 101, column: 1, scope: !{{.*}})
+
+!llvm.module.flags = !{!0, !1}
+!0 = !{i32 2, !"Dwarf Version", i32 4}
+!1 = !{i32 2, !"Debug Info Version", i32 3}
+
+!2 = !{}
+!3 = !DISubroutineType(types: !2)
+!4 = !DIFile(filename: "test.cpp", directory: "/tmp")
+!5 = !DISubprogram(name: "foo", scope: !4, file: !4, line: 99, type: !3, isLocal: false, isDefinition: true, scopeLine: 100, flags: DIFlagPrototyped, isOptimized: false, variables: !2)
+!6 = !DILocation(line: 100, column: 1, scope: !5)
+!7 = !DILocation(line: 101, column: 1, scope: !5)
+!8 = !DILocation(line: 102, column: 1, scope: !5)
diff --git a/test/Transforms/LoopVectorize/unroll.ll b/test/Transforms/LoopVectorize/unroll.ll
new file mode 100644
index 00000000000..74076f69510
--- /dev/null
+++ b/test/Transforms/LoopVectorize/unroll.ll
@@ -0,0 +1,37 @@
+; This test makes sure that loop will not be unrolled in vectorization if VF computed
+; equals to 1.
+; RUN: opt < %s -loop-vectorize -S | FileCheck %s
+
+; Make sure there are no geps being merged.
+; CHECK-LABEL: @foo(
+; CHECK: getelementptr
+; CHECK-NOT: getelementptr
+
+@N = common global i32 0, align 4
+@a = common global [1000 x i32] zeroinitializer, align 16
+
+define void @foo() #0 {
+entry:
+  %0 = load i32, i32* @N, align 4
+  %cmp5 = icmp sgt i32 %0, 0
+  br i1 %cmp5, label %for.body.lr.ph, label %for.end
+
+for.body.lr.ph:                                   ; preds = %entry
+  %conv = sext i32 %0 to i64
+  br label %for.body
+
+for.body:                                         ; preds = %for.body.lr.ph, %for.body
+  %i.06 = phi i64 [ 0, %for.body.lr.ph ], [ %inc, %for.body ]
+  %mul = mul nuw nsw i64 %i.06, 7
+  %arrayidx = getelementptr inbounds [1000 x i32], [1000 x i32]* @a, i64 0, i64 %mul
+  store i32 3, i32* %arrayidx, align 4
+  %inc = add nuw nsw i64 %i.06, 1
+  %cmp = icmp slt i64 %inc, %conv
+  br i1 %cmp, label %for.body, label %for.end.loopexit
+
+for.end.loopexit:                                 ; preds = %for.body
+  br label %for.end
+
+for.end:                                          ; preds = %for.end.loopexit, %entry
+  ret void
+}
diff --git a/test/Transforms/LowerBitSets/nonglobal.ll b/test/Transforms/LowerBitSets/nonglobal.ll
new file mode 100644
index 00000000000..7591e31e352
--- /dev/null
+++ b/test/Transforms/LowerBitSets/nonglobal.ll
@@ -0,0 +1,19 @@
+; RUN: opt -S -lowerbitsets < %s | FileCheck %s
+
+target datalayout = "e-p:32:32"
+
+; CHECK-NOT: @b = alias
+@a = constant i32 1
+@b = constant [2 x i32] [i32 2, i32 3]
+
+!0 = !{!"bitset1", i32* @a, i32 0}
+!1 = !{!"bitset1", i32* bitcast ([2 x i32]* @b to i32*), i32 0}
+
+!llvm.bitsets = !{ !0, !1 }
+
+declare i1 @llvm.bitset.test(i8* %ptr, metadata %bitset) nounwind readnone
+
+define i1 @foo(i8* %p) {
+  %x = call i1 @llvm.bitset.test(i8* %p, metadata !"bitset1")
+  ret i1 %x
+}
diff --git a/test/Transforms/LowerBitSets/simple.ll b/test/Transforms/LowerBitSets/simple.ll
index 8694d67f460..0fcdf0b36d6 100644
--- a/test/Transforms/LowerBitSets/simple.ll
+++ b/test/Transforms/LowerBitSets/simple.ll
@@ -39,8 +39,8 @@ target datalayout = "e-p:32:32"
 !llvm.bitsets = !{ !0, !1, !2, !3, !4, !5, !6, !7 }
 
 ; CHECK: @bits_use{{[0-9]*}} = private alias i8* @bits{{[0-9]*}}
-; CHECK: @bits_use{{[0-9]*}} = private alias i8* @bits{{[0-9]*}}
-; CHECK: @bits_use{{[0-9]*}} = private alias i8* @bits{{[0-9]*}}
+; CHECK: @bits_use.{{[0-9]*}} = private alias i8* @bits{{[0-9]*}}
+; CHECK: @bits_use.{{[0-9]*}} = private alias i8* @bits{{[0-9]*}}
 
 ; CHECK: @a = alias getelementptr inbounds ({ i32, [0 x i8], [63 x i32], [4 x i8], i32, [0 x i8], [2 x i32] }, { i32, [0 x i8], [63 x i32], [4 x i8], i32, [0 x i8], [2 x i32] }* [[G]], i32 0, i32 0)
 ; CHECK: @b = alias getelementptr inbounds ({ i32, [0 x i8], [63 x i32], [4 x i8], i32, [0 x i8], [2 x i32] }, { i32, [0 x i8], [63 x i32], [4 x i8], i32, [0 x i8], [2 x i32] }* [[G]], i32 0, i32 2)
@@ -62,7 +62,7 @@ target datalayout = "e-p:32:32"
 ; CHECK-DARWIN: [[G]] = private constant
 
 ; CHECK: @bits{{[0-9]*}} = private alias getelementptr inbounds ([68 x i8], [68 x i8]* [[BA]], i32 0, i32 0)
-; CHECK: @bits{{[0-9]*}} = private alias getelementptr inbounds ([68 x i8], [68 x i8]* [[BA]], i32 0, i32 0)
+; CHECK: @bits.{{[0-9]*}} = private alias getelementptr inbounds ([68 x i8], [68 x i8]* [[BA]], i32 0, i32 0)
 
 declare i1 @llvm.bitset.test(i8* %ptr, metadata %bitset) nounwind readnone
 
@@ -80,7 +80,7 @@ define i1 @foo(i32* %p) {
   ; CHECK: [[R6:%[^ ]*]] = icmp ult i32 [[R5]], 68
   ; CHECK: br i1 [[R6]]
 
-  ; CHECK: [[R8:%[^ ]*]] = getelementptr i8, i8* @bits_use{{[0-9]*}}, i32 [[R5]]
+  ; CHECK: [[R8:%[^ ]*]] = getelementptr i8, i8* @bits_use.{{[0-9]*}}, i32 [[R5]]
   ; CHECK: [[R9:%[^ ]*]] = load i8, i8* [[R8]]
   ; CHECK: [[R10:%[^ ]*]] = and i8 [[R9]], 1
   ; CHECK: [[R11:%[^ ]*]] = icmp ne i8 [[R10]], 0
@@ -123,7 +123,7 @@ define i1 @baz(i32* %p) {
   ; CHECK: [[T6:%[^ ]*]] = icmp ult i32 [[T5]], 66
   ; CHECK: br i1 [[T6]]
 
-  ; CHECK: [[T8:%[^ ]*]] = getelementptr i8, i8* @bits_use{{[0-9]*}}, i32 [[T5]]
+  ; CHECK: [[T8:%[^ ]*]] = getelementptr i8, i8* @bits_use.{{[0-9]*}}, i32 [[T5]]
   ; CHECK: [[T9:%[^ ]*]] = load i8, i8* [[T8]]
   ; CHECK: [[T10:%[^ ]*]] = and i8 [[T9]], 2
   ; CHECK: [[T11:%[^ ]*]] = icmp ne i8 [[T10]], 0
diff --git a/test/Transforms/LowerBitSets/unnamed.ll b/test/Transforms/LowerBitSets/unnamed.ll
new file mode 100644
index 00000000000..6f108e22d02
--- /dev/null
+++ b/test/Transforms/LowerBitSets/unnamed.ll
@@ -0,0 +1,20 @@
+; RUN: opt -S -lowerbitsets < %s | FileCheck %s
+
+target datalayout = "e-p:32:32"
+
+; CHECK: @{{[0-9]+}} = alias
+; CHECK: @{{[0-9]+}} = alias
+@0 = constant i32 1
+@1 = constant [2 x i32] [i32 2, i32 3]
+
+!0 = !{!"bitset1", i32* @0, i32 0}
+!1 = !{!"bitset1", [2 x i32]* @1, i32 4}
+
+!llvm.bitsets = !{ !0, !1 }
+
+declare i1 @llvm.bitset.test(i8* %ptr, metadata %bitset) nounwind readnone
+
+define i1 @foo(i8* %p) {
+  %x = call i1 @llvm.bitset.test(i8* %p, metadata !"bitset1")
+  ret i1 %x
+}
diff --git a/test/Transforms/LowerInvoke/2003-12-10-Crash.ll b/test/Transforms/LowerInvoke/2003-12-10-Crash.ll
index fca8e868018..559f629ff9e 100644
--- a/test/Transforms/LowerInvoke/2003-12-10-Crash.ll
+++ b/test/Transforms/LowerInvoke/2003-12-10-Crash.ll
@@ -6,7 +6,7 @@ declare void @baz()
 
 declare void @bar()
 
-define void @foo() {
+define void @foo() personality i32 (...)* @__gxx_personality_v0 {
 then:
 	invoke void @baz( )
 			to label %invoke_cont.0 unwind label %try_catch
@@ -15,7 +15,7 @@ invoke_cont.0:		; preds = %then
 			to label %try_exit unwind label %try_catch
 try_catch:		; preds = %invoke_cont.0, %then
 	%__tmp.0 = phi i32* [ null, %invoke_cont.0 ], [ null, %then ]		; <i32*> [#uses=0]
-  %res = landingpad { i8* } personality i32 (...)* @__gxx_personality_v0
+  %res = landingpad { i8* }
           cleanup
 	ret void
 try_exit:		; preds = %invoke_cont.0
diff --git a/test/Transforms/LowerInvoke/lowerinvoke.ll b/test/Transforms/LowerInvoke/lowerinvoke.ll
index 05c19be7a9a..ad78df3e221 100644
--- a/test/Transforms/LowerInvoke/lowerinvoke.ll
+++ b/test/Transforms/LowerInvoke/lowerinvoke.ll
@@ -2,7 +2,7 @@
 
 declare i32 @external_func(i64 %arg)
 
-define i32 @invoke_test(i64 %arg) {
+define i32 @invoke_test(i64 %arg) personality i8* null {
 entry:
   %result = invoke fastcc i32 @external_func(i64 inreg %arg)
       to label %cont unwind label %lpad
@@ -10,7 +10,7 @@ cont:
   ret i32 %result
 lpad:
   %phi = phi i32 [ 99, %entry ]
-  %lp = landingpad { i8*, i32 } personality i8* null cleanup
+  %lp = landingpad { i8*, i32 } cleanup
   ret i32 %phi
 }
 
diff --git a/test/Transforms/LowerSwitch/feature.ll b/test/Transforms/LowerSwitch/feature.ll
index 09d25f0b06d..b82d9345543 100644
--- a/test/Transforms/LowerSwitch/feature.ll
+++ b/test/Transforms/LowerSwitch/feature.ll
@@ -4,49 +4,49 @@
 ; On output we should got binary comparison tree. Check that all is fine.
 
 ;CHECK:     entry:
-;CHECK-NEXT:  br label %NodeBlock19
+;CHECK-NEXT:  br label %NodeBlock.19
 
-;CHECK:     NodeBlock19:                                      ; preds = %entry
-;CHECK-NEXT:  %Pivot20 = icmp slt i32 %tmp158, 10
-;CHECK-NEXT:  br i1 %Pivot20, label %NodeBlock5, label %NodeBlock17
+;CHECK:     NodeBlock.19:                                      ; preds = %entry
+;CHECK-NEXT:  %Pivot.20 = icmp slt i32 %tmp158, 10
+;CHECK-NEXT:  br i1 %Pivot.20, label %NodeBlock.5, label %NodeBlock.17
 
-;CHECK:     NodeBlock17:                                      ; preds = %NodeBlock19
-;CHECK-NEXT:  %Pivot18 = icmp slt i32 %tmp158, 13
-;CHECK-NEXT:  br i1 %Pivot18, label %NodeBlock9, label %NodeBlock15
+;CHECK:     NodeBlock.17:                                      ; preds = %NodeBlock.19
+;CHECK-NEXT:  %Pivot.18 = icmp slt i32 %tmp158, 13
+;CHECK-NEXT:  br i1 %Pivot.18, label %NodeBlock.9, label %NodeBlock.15
 
-;CHECK:     NodeBlock15:                                      ; preds = %NodeBlock17
-;CHECK-NEXT:  %Pivot16 = icmp slt i32 %tmp158, 14
-;CHECK-NEXT:  br i1 %Pivot16, label %bb330, label %NodeBlock13
+;CHECK:     NodeBlock.15:                                      ; preds = %NodeBlock.17
+;CHECK-NEXT:  %Pivot.16 = icmp slt i32 %tmp158, 14
+;CHECK-NEXT:  br i1 %Pivot.16, label %bb330, label %NodeBlock.13
 
-;CHECK:     NodeBlock13:                                      ; preds = %NodeBlock15
-;CHECK-NEXT:  %Pivot14 = icmp slt i32 %tmp158, 15
-;CHECK-NEXT:  br i1 %Pivot14, label %bb332, label %LeafBlock11
+;CHECK:     NodeBlock.13:                                      ; preds = %NodeBlock.15
+;CHECK-NEXT:  %Pivot.14 = icmp slt i32 %tmp158, 15
+;CHECK-NEXT:  br i1 %Pivot.14, label %bb332, label %LeafBlock.11
 
-;CHECK:     LeafBlock11:                                      ; preds = %NodeBlock13
+;CHECK:     LeafBlock.11:                                      ; preds = %NodeBlock.13
 ;CHECK-NEXT:  %SwitchLeaf12 = icmp eq i32 %tmp158, 15
 ;CHECK-NEXT:  br i1 %SwitchLeaf12, label %bb334, label %NewDefault
 
-;CHECK:     NodeBlock9:                                       ; preds = %NodeBlock17
-;CHECK-NEXT:  %Pivot10 = icmp slt i32 %tmp158, 11
-;CHECK-NEXT:  br i1 %Pivot10, label %bb324, label %NodeBlock7
+;CHECK:     NodeBlock.9:                                       ; preds = %NodeBlock.17
+;CHECK-NEXT:  %Pivot.10 = icmp slt i32 %tmp158, 11
+;CHECK-NEXT:  br i1 %Pivot.10, label %bb324, label %NodeBlock.7
 
-;CHECK:     NodeBlock7:                                       ; preds = %NodeBlock9
-;CHECK-NEXT:  %Pivot8 = icmp slt i32 %tmp158, 12
-;CHECK-NEXT:  br i1 %Pivot8, label %bb326, label %bb328
+;CHECK:     NodeBlock.7:                                       ; preds = %NodeBlock.9
+;CHECK-NEXT:  %Pivot.8 = icmp slt i32 %tmp158, 12
+;CHECK-NEXT:  br i1 %Pivot.8, label %bb326, label %bb328
 
-;CHECK:     NodeBlock5:                                       ; preds = %NodeBlock19
-;CHECK-NEXT:  %Pivot6 = icmp slt i32 %tmp158, 7
-;CHECK-NEXT:  br i1 %Pivot6, label %NodeBlock, label %NodeBlock3
+;CHECK:     NodeBlock.5:                                       ; preds = %NodeBlock.19
+;CHECK-NEXT:  %Pivot.6 = icmp slt i32 %tmp158, 7
+;CHECK-NEXT:  br i1 %Pivot.6, label %NodeBlock, label %NodeBlock.3
 
-;CHECK:     NodeBlock3:                                       ; preds = %NodeBlock5
-;CHECK-NEXT:  %Pivot4 = icmp slt i32 %tmp158, 8
-;CHECK-NEXT:  br i1 %Pivot4, label %bb, label %NodeBlock1
+;CHECK:     NodeBlock.3:                                       ; preds = %NodeBlock.5
+;CHECK-NEXT:  %Pivot.4 = icmp slt i32 %tmp158, 8
+;CHECK-NEXT:  br i1 %Pivot.4, label %bb, label %NodeBlock.1
 
-;CHECK:     NodeBlock1:                                       ; preds = %NodeBlock3
-;CHECK-NEXT:  %Pivot2 = icmp slt i32 %tmp158, 9
-;CHECK-NEXT:  br i1 %Pivot2, label %bb338, label %bb322
+;CHECK:     NodeBlock.1:                                       ; preds = %NodeBlock.3
+;CHECK-NEXT:  %Pivot.2 = icmp slt i32 %tmp158, 9
+;CHECK-NEXT:  br i1 %Pivot.2, label %bb338, label %bb322
 
-;CHECK:     NodeBlock:                                        ; preds = %NodeBlock5
+;CHECK:     NodeBlock:                                        ; preds = %NodeBlock.5
 ;CHECK-NEXT:  %Pivot = icmp slt i32 %tmp158, 0
 ;CHECK-NEXT:  br i1 %Pivot, label %LeafBlock, label %bb338
 
diff --git a/test/Transforms/Mem2Reg/ConvertDebugInfo.ll b/test/Transforms/Mem2Reg/ConvertDebugInfo.ll
index 7a55b07c549..2a0fee97f24 100644
--- a/test/Transforms/Mem2Reg/ConvertDebugInfo.ll
+++ b/test/Transforms/Mem2Reg/ConvertDebugInfo.ll
@@ -7,13 +7,13 @@ entry:
   %retval = alloca double                         ; <double*> [#uses=2]
   %0 = alloca double                              ; <double*> [#uses=2]
   %"alloca point" = bitcast i32 0 to i32          ; <i32> [#uses=0]
-  call void @llvm.dbg.declare(metadata i32* %i_addr, metadata !0, metadata !MDExpression()), !dbg !8
+  call void @llvm.dbg.declare(metadata i32* %i_addr, metadata !0, metadata !DIExpression()), !dbg !8
 ; CHECK: call void @llvm.dbg.value(metadata i32 %i, i64 0, metadata ![[IVAR:[0-9]*]], metadata {{.*}})
 ; CHECK: call void @llvm.dbg.value(metadata double %j, i64 0, metadata ![[JVAR:[0-9]*]], metadata {{.*}})
-; CHECK: ![[IVAR]] = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "i"
-; CHECK: ![[JVAR]] = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "j"
+; CHECK: ![[IVAR]] = !DILocalVariable(tag: DW_TAG_arg_variable, name: "i"
+; CHECK: ![[JVAR]] = !DILocalVariable(tag: DW_TAG_arg_variable, name: "j"
   store i32 %i, i32* %i_addr
-  call void @llvm.dbg.declare(metadata double* %j_addr, metadata !9, metadata !MDExpression()), !dbg !8
+  call void @llvm.dbg.declare(metadata double* %j_addr, metadata !9, metadata !DIExpression()), !dbg !8
   store double %j, double* %j_addr
   %1 = load i32, i32* %i_addr, align 4, !dbg !10       ; <i32> [#uses=1]
   %2 = add nsw i32 %1, 1, !dbg !10                ; <i32> [#uses=1]
@@ -35,18 +35,18 @@ declare void @llvm.dbg.declare(metadata, metadata, metadata) nounwind readnone
 !llvm.dbg.cu = !{!3}
 !llvm.module.flags = !{!14}
 
-!0 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "i", line: 2, arg: 0, scope: !1, file: !2, type: !7)
-!1 = !MDSubprogram(name: "testfunc", linkageName: "testfunc", line: 2, isLocal: false, isDefinition: true, virtualIndex: 6, isOptimized: false, scopeLine: 2, file: !12, scope: !2, type: !4, function: double (i32, double)* @testfunc)
-!2 = !MDFile(filename: "testfunc.c", directory: "/tmp")
-!3 = !MDCompileUnit(language: DW_LANG_C89, producer: "4.2.1 (Based on Apple Inc. build 5658) (LLVM build)", isOptimized: true, emissionKind: 0, file: !12, enums: !13, retainedTypes: !13)
-!4 = !MDSubroutineType(types: !5)
+!0 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "i", line: 2, arg: 1, scope: !1, file: !2, type: !7)
+!1 = !DISubprogram(name: "testfunc", linkageName: "testfunc", line: 2, isLocal: false, isDefinition: true, virtualIndex: 6, isOptimized: false, scopeLine: 2, file: !12, scope: !2, type: !4, function: double (i32, double)* @testfunc)
+!2 = !DIFile(filename: "testfunc.c", directory: "/tmp")
+!3 = !DICompileUnit(language: DW_LANG_C89, producer: "4.2.1 (Based on Apple Inc. build 5658) (LLVM build)", isOptimized: true, emissionKind: 0, file: !12, enums: !13, retainedTypes: !13)
+!4 = !DISubroutineType(types: !5)
 !5 = !{!6, !7, !6}
-!6 = !MDBasicType(tag: DW_TAG_base_type, name: "double", size: 64, align: 64, encoding: DW_ATE_float)
-!7 = !MDBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
-!8 = !MDLocation(line: 2, scope: !1)
-!9 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "j", line: 2, arg: 0, scope: !1, file: !2, type: !6)
-!10 = !MDLocation(line: 3, scope: !11)
-!11 = distinct !MDLexicalBlock(line: 2, column: 0, file: !12, scope: !1)
-!12 = !MDFile(filename: "testfunc.c", directory: "/tmp")
+!6 = !DIBasicType(tag: DW_TAG_base_type, name: "double", size: 64, align: 64, encoding: DW_ATE_float)
+!7 = !DIBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
+!8 = !DILocation(line: 2, scope: !1)
+!9 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "j", line: 2, arg: 2, scope: !1, file: !2, type: !6)
+!10 = !DILocation(line: 3, scope: !11)
+!11 = distinct !DILexicalBlock(line: 2, column: 0, file: !12, scope: !1)
+!12 = !DIFile(filename: "testfunc.c", directory: "/tmp")
 !13 = !{}
 !14 = !{i32 1, !"Debug Info Version", i32 3}
diff --git a/test/Transforms/Mem2Reg/ConvertDebugInfo2.ll b/test/Transforms/Mem2Reg/ConvertDebugInfo2.ll
index e60acc4ca9b..aff2e608d90 100644
--- a/test/Transforms/Mem2Reg/ConvertDebugInfo2.ll
+++ b/test/Transforms/Mem2Reg/ConvertDebugInfo2.ll
@@ -11,14 +11,14 @@ entry:
   %z_addr.i = alloca i8*                          ; <i8**> [#uses=2]
   %a_addr = alloca i32                            ; <i32*> [#uses=2]
   %"alloca point" = bitcast i32 0 to i32          ; <i32> [#uses=0]
-  call void @llvm.dbg.declare(metadata i32* %a_addr, metadata !0, metadata !MDExpression()), !dbg !7
+  call void @llvm.dbg.declare(metadata i32* %a_addr, metadata !0, metadata !DIExpression()), !dbg !7
   store i32 %a, i32* %a_addr
   %0 = load i32, i32* %a_addr, align 4, !dbg !8        ; <i32> [#uses=1]
-  call void @llvm.dbg.declare(metadata i32* %x_addr.i, metadata !9, metadata !MDExpression()) nounwind, !dbg !15
+  call void @llvm.dbg.declare(metadata i32* %x_addr.i, metadata !9, metadata !DIExpression()) nounwind, !dbg !15
   store i32 %0, i32* %x_addr.i
-  call void @llvm.dbg.declare(metadata i64* %y_addr.i, metadata !16, metadata !MDExpression()) nounwind, !dbg !15
+  call void @llvm.dbg.declare(metadata i64* %y_addr.i, metadata !16, metadata !DIExpression()) nounwind, !dbg !15
   store i64 55, i64* %y_addr.i
-  call void @llvm.dbg.declare(metadata i8** %z_addr.i, metadata !17, metadata !MDExpression()) nounwind, !dbg !15
+  call void @llvm.dbg.declare(metadata i8** %z_addr.i, metadata !17, metadata !DIExpression()) nounwind, !dbg !15
   store i8* bitcast (void (i32)* @baz to i8*), i8** %z_addr.i
   %1 = load i32, i32* %x_addr.i, align 4, !dbg !18     ; <i32> [#uses=1]
   %2 = load i64, i64* %y_addr.i, align 8, !dbg !18     ; <i64> [#uses=1]
@@ -32,26 +32,26 @@ return:                                           ; preds = %entry
 
 !llvm.dbg.cu = !{!3}
 !llvm.module.flags = !{!22}
-!0 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "a", line: 8, arg: 0, scope: !1, file: !2, type: !6)
-!1 = !MDSubprogram(name: "baz", linkageName: "baz", line: 8, isLocal: false, isDefinition: true, virtualIndex: 6, isOptimized: false, scopeLine: 8, file: !20, scope: !2, type: !4, function: void (i32)* @baz)
-!2 = !MDFile(filename: "bar.c", directory: "/tmp/")
-!3 = !MDCompileUnit(language: DW_LANG_C89, producer: "4.2.1 (Based on Apple Inc. build 5658) (LLVM build)", isOptimized: true, emissionKind: 0, file: !20, enums: !21, retainedTypes: !21)
-!4 = !MDSubroutineType(types: !5)
+!0 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "a", line: 8, arg: 1, scope: !1, file: !2, type: !6)
+!1 = !DISubprogram(name: "baz", linkageName: "baz", line: 8, isLocal: false, isDefinition: true, virtualIndex: 6, isOptimized: false, scopeLine: 8, file: !20, scope: !2, type: !4, function: void (i32)* @baz)
+!2 = !DIFile(filename: "bar.c", directory: "/tmp/")
+!3 = !DICompileUnit(language: DW_LANG_C89, producer: "4.2.1 (Based on Apple Inc. build 5658) (LLVM build)", isOptimized: true, emissionKind: 0, file: !20, enums: !21, retainedTypes: !21)
+!4 = !DISubroutineType(types: !5)
 !5 = !{null, !6}
-!6 = !MDBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
-!7 = !MDLocation(line: 8, scope: !1)
-!8 = !MDLocation(line: 9, scope: !1)
-!9 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "x", line: 4, arg: 0, scope: !10, file: !2, type: !6)
-!10 = !MDSubprogram(name: "bar", linkageName: "bar", line: 4, isLocal: true, isDefinition: true, virtualIndex: 6, isOptimized: false, scopeLine: 4, file: !20, scope: !2, type: !11)
-!11 = !MDSubroutineType(types: !12)
+!6 = !DIBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
+!7 = !DILocation(line: 8, scope: !1)
+!8 = !DILocation(line: 9, scope: !1)
+!9 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "x", line: 4, arg: 1, scope: !10, file: !2, type: !6)
+!10 = !DISubprogram(name: "bar", linkageName: "bar", line: 4, isLocal: true, isDefinition: true, virtualIndex: 6, isOptimized: false, scopeLine: 4, file: !20, scope: !2, type: !11)
+!11 = !DISubroutineType(types: !12)
 !12 = !{null, !6, !13, !14}
-!13 = !MDBasicType(tag: DW_TAG_base_type, name: "long int", size: 64, align: 64, encoding: DW_ATE_signed)
-!14 = !MDDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, file: !20, scope: !2, baseType: null)
-!15 = !MDLocation(line: 4, scope: !10, inlinedAt: !8)
-!16 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "y", line: 4, arg: 0, scope: !10, file: !2, type: !13)
-!17 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "z", line: 4, arg: 0, scope: !10, file: !2, type: !14)
-!18 = !MDLocation(line: 5, scope: !10, inlinedAt: !8)
-!19 = !MDLocation(line: 10, scope: !1)
-!20 = !MDFile(filename: "bar.c", directory: "/tmp/")
+!13 = !DIBasicType(tag: DW_TAG_base_type, name: "long int", size: 64, align: 64, encoding: DW_ATE_signed)
+!14 = !DIDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, file: !20, scope: !2, baseType: null)
+!15 = !DILocation(line: 4, scope: !10, inlinedAt: !8)
+!16 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "y", line: 4, arg: 2, scope: !10, file: !2, type: !13)
+!17 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "z", line: 4, arg: 3, scope: !10, file: !2, type: !14)
+!18 = !DILocation(line: 5, scope: !10, inlinedAt: !8)
+!19 = !DILocation(line: 10, scope: !1)
+!20 = !DIFile(filename: "bar.c", directory: "/tmp/")
 !21 = !{}
 !22 = !{i32 1, !"Debug Info Version", i32 3}
diff --git a/test/Transforms/Mem2Reg/crash.ll b/test/Transforms/Mem2Reg/crash.ll
index a4a31b112ee..d7ed1dd13e8 100644
--- a/test/Transforms/Mem2Reg/crash.ll
+++ b/test/Transforms/Mem2Reg/crash.ll
@@ -3,7 +3,7 @@
 
 declare i32 @test1f()
 
-define i32 @test1() {
+define i32 @test1() personality i32 (...)* @__gxx_personality_v0 {
 entry:
   %whichFlag = alloca i32
   %A = invoke i32 @test1f()
@@ -18,7 +18,7 @@ bb15:
   ret i32 %B
 
 lpad86:
-  %exn = landingpad {i8*, i32} personality i32 (...)* @__gxx_personality_v0
+  %exn = landingpad {i8*, i32}
            cleanup
   br label %bb15
   
diff --git a/test/Transforms/Mem2Reg/pr24179.ll b/test/Transforms/Mem2Reg/pr24179.ll
new file mode 100644
index 00000000000..e4216ce4daa
--- /dev/null
+++ b/test/Transforms/Mem2Reg/pr24179.ll
@@ -0,0 +1,44 @@
+; RUN: opt -mem2reg < %s -S | FileCheck %s
+
+declare i32 @def(i32)
+declare i1 @use(i32)
+
+; Special case of a single-BB alloca does not apply here since the load
+; is affected by the following store. Expect this case to be identified
+; and a PHI node to be created.
+define void @test1() {
+; CHECK-LABEL: @test1(
+ entry:
+  %t = alloca i32
+  br label %loop
+
+ loop:
+  %v = load i32, i32* %t
+  %c = call i1 @use(i32 %v)
+; CHECK: [[PHI:%.*]] = phi i32 [ undef, %entry ], [ %n, %loop ]
+; CHECK: call i1 @use(i32 [[PHI]])
+  %n = call i32 @def(i32 7)
+  store i32 %n, i32* %t
+  br i1 %c, label %loop, label %exit
+
+ exit:
+  ret void
+}
+
+; Same as above, except there is no following store. The alloca should just be
+; replaced with an undef
+define void @test2() {
+; CHECK-LABEL: @test2(
+ entry:
+  %t = alloca i32
+  br label %loop
+
+ loop:
+  %v = load i32, i32* %t
+  %c = call i1 @use(i32 %v)
+; CHECK: %c = call i1 @use(i32 undef)
+  br i1 %c, label %loop, label %exit
+
+ exit:
+  ret void
+}
diff --git a/test/Transforms/MemCpyOpt/callslot_deref.ll b/test/Transforms/MemCpyOpt/callslot_deref.ll
index a1f5ae19d24..4c3e3e86bf3 100644
--- a/test/Transforms/MemCpyOpt/callslot_deref.ll
+++ b/test/Transforms/MemCpyOpt/callslot_deref.ll
@@ -17,10 +17,11 @@ define void @must_remove_memcpy(i8* noalias nocapture dereferenceable(4096) %dst
 }
 
 ; memset touch more bytes than those guaranteed to be dereferenceable
+; We can't remove the memcpy, but we can turn it into an independent memset.
 define void @must_not_remove_memcpy(i8* noalias nocapture dereferenceable(1024) %dst) {
 ; CHECK-LABEL: @must_not_remove_memcpy(
 ; CHECK: call void @llvm.memset.p0i8.i64
-; CHECK: call void @llvm.memcpy.p0i8.p0i8.i64
+; CHECK: call void @llvm.memset.p0i8.i64
   %src = alloca [4096 x i8], align 1
   %p = getelementptr inbounds [4096 x i8], [4096 x i8]* %src, i64 0, i64 0
   call void @llvm.memset.p0i8.i64(i8* %p, i8 0, i64 4096, i32 1, i1 false)
diff --git a/test/Transforms/MemCpyOpt/memcpy-to-memset-with-lifetimes.ll b/test/Transforms/MemCpyOpt/memcpy-to-memset-with-lifetimes.ll
index 626317614c9..e3e57f09d88 100644
--- a/test/Transforms/MemCpyOpt/memcpy-to-memset-with-lifetimes.ll
+++ b/test/Transforms/MemCpyOpt/memcpy-to-memset-with-lifetimes.ll
@@ -40,7 +40,7 @@ entry-block:
 ; CHECK:         %[[a_cast:[^=]+]] = bitcast [8 x i64]* %[[a]] to i8*
 ; CHECK:         call void @llvm.memset.p0i8.i64(i8* %[[a_cast]], i8 0, i64 64
 ; CHECK:         %[[sret_cast:[^=]+]] = bitcast [8 x i64]* %sret to i8*
-; CHECK:         call void @llvm.memcpy.p0i8.p0i8.i64(i8* %[[sret_cast]], i8* %[[a_cast]], i64 64
+; CHECK:         call void @llvm.memset.p0i8.i64(i8* %[[sret_cast]], i8 0, i64 64
 ; CHECK:         call void @llvm.memset.p0i8.i64(i8* %[[a_cast]], i8 42, i64 32
 ; CHECK:         %[[out_cast:[^=]+]] = bitcast [8 x i64]* %out to i8*
 ; CHECK:         call void @llvm.memcpy.p0i8.p0i8.i64(i8* %[[out_cast]], i8* %[[a_cast]], i64 64
diff --git a/test/Transforms/MemCpyOpt/memset-memcpy-redundant-memset.ll b/test/Transforms/MemCpyOpt/memset-memcpy-redundant-memset.ll
new file mode 100644
index 00000000000..993c8a1bb2d
--- /dev/null
+++ b/test/Transforms/MemCpyOpt/memset-memcpy-redundant-memset.ll
@@ -0,0 +1,168 @@
+; RUN: opt -basicaa -memcpyopt -S %s | FileCheck %s
+
+target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
+
+; CHECK-LABEL: define void @test
+; CHECK-DAG: [[DST:%[0-9]+]] = getelementptr i8, i8* %dst, i64 %src_size
+; CHECK-DAG: [[ULE:%[0-9]+]] = icmp ule i64 %dst_size, %src_size
+; CHECK-DAG: [[SIZEDIFF:%[0-9]+]] = sub i64 %dst_size, %src_size
+; CHECK-DAG: [[SIZE:%[0-9]+]] = select i1 [[ULE]], i64 0, i64 [[SIZEDIFF]]
+; CHECK-NEXT: call void @llvm.memset.p0i8.i64(i8* [[DST]], i8 %c, i64 [[SIZE]], i32 1, i1 false)
+; CHECK-NEXT: call void @llvm.memcpy.p0i8.p0i8.i64(i8* %dst, i8* %src, i64 %src_size, i32 1, i1 false)
+; CHECK-NEXT: ret void
+define void @test(i8* %src, i64 %src_size, i8* %dst, i64 %dst_size, i8 %c) {
+  call void @llvm.memset.p0i8.i64(i8* %dst, i8 %c, i64 %dst_size, i32 1, i1 false)
+  call void @llvm.memcpy.p0i8.p0i8.i64(i8* %dst, i8* %src, i64 %src_size, i32 1, i1 false)
+  ret void
+}
+
+; CHECK-LABEL: define void @test_different_types_i32_i64
+; CHECK-DAG: [[DSTSIZE:%[0-9]+]] = zext i32 %dst_size to i64
+; CHECK-DAG: [[DST:%[0-9]+]] = getelementptr i8, i8* %dst, i64 %src_size
+; CHECK-DAG: [[ULE:%[0-9]+]] = icmp ule i64 [[DSTSIZE]], %src_size
+; CHECK-DAG: [[SIZEDIFF:%[0-9]+]] = sub i64 [[DSTSIZE]], %src_size
+; CHECK-DAG: [[SIZE:%[0-9]+]] = select i1 [[ULE]], i64 0, i64 [[SIZEDIFF]]
+; CHECK-NEXT: call void @llvm.memset.p0i8.i64(i8* [[DST]], i8 %c, i64 [[SIZE]], i32 1, i1 false)
+; CHECK-NEXT: call void @llvm.memcpy.p0i8.p0i8.i64(i8* %dst, i8* %src, i64 %src_size, i32 1, i1 false)
+; CHECK-NEXT: ret void
+define void @test_different_types_i32_i64(i8* %dst, i8* %src, i32 %dst_size, i64 %src_size, i8 %c) {
+  call void @llvm.memset.p0i8.i32(i8* %dst, i8 %c, i32 %dst_size, i32 1, i1 false)
+  call void @llvm.memcpy.p0i8.p0i8.i64(i8* %dst, i8* %src, i64 %src_size, i32 1, i1 false)
+  ret void
+}
+
+; CHECK-LABEL: define void @test_different_types_i128_i32
+; CHECK-DAG: [[SRCSIZE:%[0-9]+]] = zext i32 %src_size to i128
+; CHECK-DAG: [[DST:%[0-9]+]] = getelementptr i8, i8* %dst, i128 [[SRCSIZE]]
+; CHECK-DAG: [[ULE:%[0-9]+]] = icmp ule i128 %dst_size, [[SRCSIZE]]
+; CHECK-DAG: [[SIZEDIFF:%[0-9]+]] = sub i128 %dst_size, [[SRCSIZE]]
+; CHECK-DAG: [[SIZE:%[0-9]+]] = select i1 [[ULE]], i128 0, i128 [[SIZEDIFF]]
+; CHECK-NEXT: call void @llvm.memset.p0i8.i128(i8* [[DST]], i8 %c, i128 [[SIZE]], i32 1, i1 false)
+; CHECK-NEXT: call void @llvm.memcpy.p0i8.p0i8.i32(i8* %dst, i8* %src, i32 %src_size, i32 1, i1 false)
+; CHECK-NEXT: ret void
+define void @test_different_types_i128_i32(i8* %dst, i8* %src, i128 %dst_size, i32 %src_size, i8 %c) {
+  call void @llvm.memset.p0i8.i128(i8* %dst, i8 %c, i128 %dst_size, i32 1, i1 false)
+  call void @llvm.memcpy.p0i8.p0i8.i32(i8* %dst, i8* %src, i32 %src_size, i32 1, i1 false)
+  ret void
+}
+
+; CHECK-LABEL: define void @test_different_types_i32_i128
+; CHECK-DAG: [[DSTSIZE:%[0-9]+]] = zext i32 %dst_size to i128
+; CHECK-DAG: [[DST:%[0-9]+]] = getelementptr i8, i8* %dst, i128 %src_size
+; CHECK-DAG: [[ULE:%[0-9]+]] = icmp ule i128 [[DSTSIZE]], %src_size
+; CHECK-DAG: [[SIZEDIFF:%[0-9]+]] = sub i128 [[DSTSIZE]], %src_size
+; CHECK-DAG: [[SIZE:%[0-9]+]] = select i1 [[ULE]], i128 0, i128 [[SIZEDIFF]]
+; CHECK-NEXT: call void @llvm.memset.p0i8.i128(i8* [[DST]], i8 %c, i128 [[SIZE]], i32 1, i1 false)
+; CHECK-NEXT: call void @llvm.memcpy.p0i8.p0i8.i128(i8* %dst, i8* %src, i128 %src_size, i32 1, i1 false)
+; CHECK-NEXT: ret void
+define void @test_different_types_i32_i128(i8* %dst, i8* %src, i32 %dst_size, i128 %src_size, i8 %c) {
+  call void @llvm.memset.p0i8.i32(i8* %dst, i8 %c, i32 %dst_size, i32 1, i1 false)
+  call void @llvm.memcpy.p0i8.p0i8.i128(i8* %dst, i8* %src, i128 %src_size, i32 1, i1 false)
+  ret void
+}
+
+; CHECK-LABEL: define void @test_different_types_i64_i32
+; CHECK-DAG: [[SRCSIZE:%[0-9]+]] = zext i32 %src_size to i64
+; CHECK-DAG: [[DST:%[0-9]+]] = getelementptr i8, i8* %dst, i64 [[SRCSIZE]]
+; CHECK-DAG: [[ULE:%[0-9]+]] = icmp ule i64 %dst_size, [[SRCSIZE]]
+; CHECK-DAG: [[SIZEDIFF:%[0-9]+]] = sub i64 %dst_size, [[SRCSIZE]]
+; CHECK-DAG: [[SIZE:%[0-9]+]] = select i1 [[ULE]], i64 0, i64 [[SIZEDIFF]]
+; CHECK-NEXT: call void @llvm.memset.p0i8.i64(i8* [[DST]], i8 %c, i64 [[SIZE]], i32 1, i1 false)
+; CHECK-NEXT: call void @llvm.memcpy.p0i8.p0i8.i32(i8* %dst, i8* %src, i32 %src_size, i32 1, i1 false)
+; CHECK-NEXT: ret void
+define void @test_different_types_i64_i32(i8* %dst, i8* %src, i64 %dst_size, i32 %src_size, i8 %c) {
+  call void @llvm.memset.p0i8.i64(i8* %dst, i8 %c, i64 %dst_size, i32 1, i1 false)
+  call void @llvm.memcpy.p0i8.p0i8.i32(i8* %dst, i8* %src, i32 %src_size, i32 1, i1 false)
+  ret void
+}
+
+; CHECK-LABEL: define void @test_align_same
+; CHECK: call void @llvm.memset.p0i8.i64(i8* {{.*}}, i8 0, i64 {{.*}}, i32 8, i1 false)
+define void @test_align_same(i8* %src, i8* %dst, i64 %dst_size) {
+  call void @llvm.memset.p0i8.i64(i8* %dst, i8 0, i64 %dst_size, i32 8, i1 false)
+  call void @llvm.memcpy.p0i8.p0i8.i64(i8* %dst, i8* %src, i64 80, i32 1, i1 false)
+  ret void
+}
+
+; CHECK-LABEL: define void @test_align_min
+; CHECK: call void @llvm.memset.p0i8.i64(i8* {{.*}}, i8 0, i64 {{.*}}, i32 4, i1 false)
+define void @test_align_min(i8* %src, i8* %dst, i64 %dst_size) {
+  call void @llvm.memset.p0i8.i64(i8* %dst, i8 0, i64 %dst_size, i32 8, i1 false)
+  call void @llvm.memcpy.p0i8.p0i8.i64(i8* %dst, i8* %src, i64 36, i32 1, i1 false)
+  ret void
+}
+
+; CHECK-LABEL: define void @test_align_memcpy
+; CHECK: call void @llvm.memset.p0i8.i64(i8* {{.*}}, i8 0, i64 {{.*}}, i32 8, i1 false)
+define void @test_align_memcpy(i8* %src, i8* %dst, i64 %dst_size) {
+  call void @llvm.memset.p0i8.i64(i8* %dst, i8 0, i64 %dst_size, i32 1, i1 false)
+  call void @llvm.memcpy.p0i8.p0i8.i64(i8* %dst, i8* %src, i64 80, i32 8, i1 false)
+  ret void
+}
+
+; CHECK-LABEL: define void @test_non_i8_dst_type
+; CHECK-NEXT: %dst = bitcast i64* %dst_pi64 to i8*
+; CHECK-DAG: [[DST:%[0-9]+]] = getelementptr i8, i8* %dst, i64 %src_size
+; CHECK-DAG: [[ULE:%[0-9]+]] = icmp ule i64 %dst_size, %src_size
+; CHECK-DAG: [[SIZEDIFF:%[0-9]+]] = sub i64 %dst_size, %src_size
+; CHECK-DAG: [[SIZE:%[0-9]+]] = select i1 [[ULE]], i64 0, i64 [[SIZEDIFF]]
+; CHECK-NEXT: call void @llvm.memset.p0i8.i64(i8* [[DST]], i8 %c, i64 [[SIZE]], i32 1, i1 false)
+; CHECK-NEXT: call void @llvm.memcpy.p0i8.p0i8.i64(i8* %dst, i8* %src, i64 %src_size, i32 1, i1 false)
+; CHECK-NEXT: ret void
+define void @test_non_i8_dst_type(i8* %src, i64 %src_size, i64* %dst_pi64, i64 %dst_size, i8 %c) {
+  %dst = bitcast i64* %dst_pi64 to i8*
+  call void @llvm.memset.p0i8.i64(i8* %dst, i8 %c, i64 %dst_size, i32 1, i1 false)
+  call void @llvm.memcpy.p0i8.p0i8.i64(i8* %dst, i8* %src, i64 %src_size, i32 1, i1 false)
+  ret void
+}
+
+; CHECK-LABEL: define void @test_different_dst
+; CHECK-NEXT: call void @llvm.memset.p0i8.i64(i8* %dst, i8 0, i64 %dst_size, i32 1, i1 false)
+; CHECK-NEXT: call void @llvm.memcpy.p0i8.p0i8.i64(i8* %dst2, i8* %src, i64 %src_size, i32 1, i1 false)
+; CHECK-NEXT: ret void
+define void @test_different_dst(i8* %dst2, i8* %src, i64 %src_size, i8* %dst, i64 %dst_size) {
+  call void @llvm.memset.p0i8.i64(i8* %dst, i8 0, i64 %dst_size, i32 1, i1 false)
+  call void @llvm.memcpy.p0i8.p0i8.i64(i8* %dst2, i8* %src, i64 %src_size, i32 1, i1 false)
+  ret void
+}
+
+; Make sure we also take into account dependencies on the destination.
+
+; CHECK-LABEL: define i8 @test_intermediate_read
+; CHECK-NEXT: call void @llvm.memset.p0i8.i64(i8* %a, i8 0, i64 64, i32 1, i1 false)
+; CHECK-NEXT: %r = load i8, i8* %a
+; CHECK-NEXT: call void @llvm.memcpy.p0i8.p0i8.i64(i8* %a, i8* %b, i64 24, i32 1, i1 false)
+; CHECK-NEXT: ret i8 %r
+define i8 @test_intermediate_read(i8* %a, i8* %b) #0 {
+  call void @llvm.memset.p0i8.i64(i8* %a, i8 0, i64 64, i32 1, i1 false)
+  %r = load i8, i8* %a
+  call void @llvm.memcpy.p0i8.p0i8.i64(i8* %a, i8* %b, i64 24, i32 1, i1 false)
+  ret i8 %r
+}
+
+%struct = type { [8 x i8], [8 x i8] }
+
+; CHECK-LABEL: define void @test_intermediate_write
+; CHECK-NEXT: %a = alloca %struct
+; CHECK-NEXT: %a0 = getelementptr %struct, %struct* %a, i32 0, i32 0, i32 0
+; CHECK-NEXT: %a1 = getelementptr %struct, %struct* %a, i32 0, i32 1, i32 0
+; CHECK-NEXT: call void @llvm.memset.p0i8.i64(i8* %a0, i8 0, i64 16, i32 1, i1 false)
+; CHECK-NEXT: store i8 1, i8* %a1
+; CHECK-NEXT: call void @llvm.memcpy.p0i8.p0i8.i64(i8* %a0, i8* %b, i64 8, i32 1, i1 false)
+; CHECK-NEXT: ret void
+define void @test_intermediate_write(i8* %b) #0 {
+  %a = alloca %struct
+  %a0 = getelementptr %struct, %struct* %a, i32 0, i32 0, i32 0
+  %a1 = getelementptr %struct, %struct* %a, i32 0, i32 1, i32 0
+  call void @llvm.memset.p0i8.i64(i8* %a0, i8 0, i64 16, i32 1, i1 false)
+  store i8 1, i8* %a1
+  call void @llvm.memcpy.p0i8.p0i8.i64(i8* %a0, i8* %b, i64 8, i32 1, i1 false)
+  ret void
+}
+
+declare void @llvm.memset.p0i8.i64(i8* nocapture, i8, i64, i32, i1)
+declare void @llvm.memcpy.p0i8.p0i8.i64(i8* nocapture, i8* nocapture readonly, i64, i32, i1)
+declare void @llvm.memset.p0i8.i32(i8* nocapture, i8, i32, i32, i1)
+declare void @llvm.memcpy.p0i8.p0i8.i32(i8* nocapture, i8* nocapture readonly, i32, i32, i1)
+declare void @llvm.memset.p0i8.i128(i8* nocapture, i8, i128, i32, i1)
+declare void @llvm.memcpy.p0i8.p0i8.i128(i8* nocapture, i8* nocapture readonly, i128, i32, i1)
diff --git a/test/Transforms/MemCpyOpt/memset-memcpy-to-2x-memset.ll b/test/Transforms/MemCpyOpt/memset-memcpy-to-2x-memset.ll
new file mode 100644
index 00000000000..1c567042ce7
--- /dev/null
+++ b/test/Transforms/MemCpyOpt/memset-memcpy-to-2x-memset.ll
@@ -0,0 +1,101 @@
+; RUN: opt -memcpyopt -S %s | FileCheck %s
+
+target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
+
+; CHECK-LABEL: define void @test(
+; CHECK-NEXT: call void @llvm.memset.p0i8.i64(i8* %dst1, i8 %c, i64 128, i32 1, i1 false)
+; CHECK-NEXT: call void @llvm.memset.p0i8.i64(i8* %dst2, i8 %c, i64 128, i32 8, i1 false)
+; CHECK-NEXT: ret void
+define void @test(i8* %dst1, i8* %dst2, i8 %c) {
+  call void @llvm.memset.p0i8.i64(i8* %dst1, i8 %c, i64 128, i32 1, i1 false)
+  call void @llvm.memcpy.p0i8.p0i8.i64(i8* %dst2, i8* %dst1, i64 128, i32 8, i1 false)
+  ret void
+}
+
+; CHECK-LABEL: define void @test_smaller_memcpy(
+; CHECK-NEXT: call void @llvm.memset.p0i8.i64(i8* %dst1, i8 %c, i64 128, i32 1, i1 false)
+; CHECK-NEXT: call void @llvm.memset.p0i8.i64(i8* %dst2, i8 %c, i64 100, i32 1, i1 false)
+; CHECK-NEXT: ret void
+define void @test_smaller_memcpy(i8* %dst1, i8* %dst2, i8 %c) {
+  call void @llvm.memset.p0i8.i64(i8* %dst1, i8 %c, i64 128, i32 1, i1 false)
+  call void @llvm.memcpy.p0i8.p0i8.i64(i8* %dst2, i8* %dst1, i64 100, i32 1, i1 false)
+  ret void
+}
+
+; CHECK-LABEL: define void @test_smaller_memset(
+; CHECK-NEXT: call void @llvm.memset.p0i8.i64(i8* %dst1, i8 %c, i64 100, i32 1, i1 false)
+; CHECK-NEXT: call void @llvm.memcpy.p0i8.p0i8.i64(i8* %dst2, i8* %dst1, i64 128, i32 1, i1 false)
+; CHECK-NEXT: ret void
+define void @test_smaller_memset(i8* %dst1, i8* %dst2, i8 %c) {
+  call void @llvm.memset.p0i8.i64(i8* %dst1, i8 %c, i64 100, i32 1, i1 false)
+  call void @llvm.memcpy.p0i8.p0i8.i64(i8* %dst2, i8* %dst1, i64 128, i32 1, i1 false)
+  ret void
+}
+
+; CHECK-LABEL: define void @test_align_memset(
+; CHECK-NEXT: call void @llvm.memset.p0i8.i64(i8* %dst1, i8 %c, i64 128, i32 8, i1 false)
+; CHECK-NEXT: call void @llvm.memset.p0i8.i64(i8* %dst2, i8 %c, i64 128, i32 1, i1 false)
+; CHECK-NEXT: ret void
+define void @test_align_memset(i8* %dst1, i8* %dst2, i8 %c) {
+  call void @llvm.memset.p0i8.i64(i8* %dst1, i8 %c, i64 128, i32 8, i1 false)
+  call void @llvm.memcpy.p0i8.p0i8.i64(i8* %dst2, i8* %dst1, i64 128, i32 1, i1 false)
+  ret void
+}
+
+; CHECK-LABEL: define void @test_different_types(
+; CHECK-NEXT: call void @llvm.memset.p0i8.i64(i8* %dst1, i8 %c, i64 128, i32 8, i1 false)
+; CHECK-NEXT: call void @llvm.memset.p0i8.i32(i8* %dst2, i8 %c, i32 100, i32 1, i1 false)
+; CHECK-NEXT: ret void
+define void @test_different_types(i8* %dst1, i8* %dst2, i8 %c) {
+  call void @llvm.memset.p0i8.i64(i8* %dst1, i8 %c, i64 128, i32 8, i1 false)
+  call void @llvm.memcpy.p0i8.p0i8.i32(i8* %dst2, i8* %dst1, i32 100, i32 1, i1 false)
+  ret void
+}
+
+; CHECK-LABEL: define void @test_different_types_2(
+; CHECK-NEXT: call void @llvm.memset.p0i8.i32(i8* %dst1, i8 %c, i32 128, i32 8, i1 false)
+; CHECK-NEXT: call void @llvm.memset.p0i8.i64(i8* %dst2, i8 %c, i64 100, i32 1, i1 false)
+; CHECK-NEXT: ret void
+define void @test_different_types_2(i8* %dst1, i8* %dst2, i8 %c) {
+  call void @llvm.memset.p0i8.i32(i8* %dst1, i8 %c, i32 128, i32 8, i1 false)
+  call void @llvm.memcpy.p0i8.p0i8.i64(i8* %dst2, i8* %dst1, i64 100, i32 1, i1 false)
+  ret void
+}
+
+; CHECK-LABEL: define void @test_different_source_gep(
+; CHECK-NEXT: call void @llvm.memset.p0i8.i64(i8* %dst1, i8 %c, i64 128, i32 1, i1 false)
+; CHECK-NEXT: %p = getelementptr i8, i8* %dst1, i64 64
+; CHECK-NEXT: call void @llvm.memcpy.p0i8.p0i8.i64(i8* %dst2, i8* %p, i64 64, i32 1, i1 false)
+; CHECK-NEXT: ret void
+define void @test_different_source_gep(i8* %dst1, i8* %dst2, i8 %c) {
+  call void @llvm.memset.p0i8.i64(i8* %dst1, i8 %c, i64 128, i32 1, i1 false)
+  ; FIXME: We could optimize this as well.
+  %p = getelementptr i8, i8* %dst1, i64 64
+  call void @llvm.memcpy.p0i8.p0i8.i64(i8* %dst2, i8* %p, i64 64, i32 1, i1 false)
+  ret void
+}
+
+; CHECK-LABEL: define void @test_variable_size_1(
+; CHECK-NEXT: call void @llvm.memset.p0i8.i64(i8* %dst1, i8 %c, i64 %dst1_size, i32 1, i1 false)
+; CHECK-NEXT: call void @llvm.memcpy.p0i8.p0i8.i64(i8* %dst2, i8* %dst1, i64 128, i32 1, i1 false)
+; CHECK-NEXT: ret void
+define void @test_variable_size_1(i8* %dst1, i64 %dst1_size, i8* %dst2, i8 %c) {
+  call void @llvm.memset.p0i8.i64(i8* %dst1, i8 %c, i64 %dst1_size, i32 1, i1 false)
+  call void @llvm.memcpy.p0i8.p0i8.i64(i8* %dst2, i8* %dst1, i64 128, i32 1, i1 false)
+  ret void
+}
+
+; CHECK-LABEL: define void @test_variable_size_2(
+; CHECK-NEXT: call void @llvm.memset.p0i8.i64(i8* %dst1, i8 %c, i64 128, i32 1, i1 false)
+; CHECK-NEXT: call void @llvm.memcpy.p0i8.p0i8.i64(i8* %dst2, i8* %dst1, i64 %dst2_size, i32 1, i1 false)
+; CHECK-NEXT: ret void
+define void @test_variable_size_2(i8* %dst1, i8* %dst2, i64 %dst2_size, i8 %c) {
+  call void @llvm.memset.p0i8.i64(i8* %dst1, i8 %c, i64 128, i32 1, i1 false)
+  call void @llvm.memcpy.p0i8.p0i8.i64(i8* %dst2, i8* %dst1, i64 %dst2_size, i32 1, i1 false)
+  ret void
+}
+
+declare void @llvm.memset.p0i8.i64(i8* nocapture, i8, i64, i32, i1)
+declare void @llvm.memcpy.p0i8.p0i8.i64(i8* nocapture, i8* nocapture readonly, i64, i32, i1)
+declare void @llvm.memset.p0i8.i32(i8* nocapture, i8, i32, i32, i1)
+declare void @llvm.memcpy.p0i8.p0i8.i32(i8* nocapture, i8* nocapture readonly, i32, i32, i1)
diff --git a/test/Transforms/MergeFunc/2011-02-08-RemoveEqual.ll b/test/Transforms/MergeFunc/2011-02-08-RemoveEqual.ll
index 4735ea5e61d..97e8ed5a6ed 100644
--- a/test/Transforms/MergeFunc/2011-02-08-RemoveEqual.ll
+++ b/test/Transforms/MergeFunc/2011-02-08-RemoveEqual.ll
@@ -22,7 +22,7 @@ target triple = "i386-pc-linux-gnu"
 @.str = external constant [1 x i8], align 1
 @_ZTVN2kc22impl_fileline_FileLineE = external constant [13 x i32 (...)*], align 32
 
-define void @_ZN2kc22impl_fileline_FileLineC2EPNS_20impl_casestring__StrEi(%"struct.kc::impl_fileline_FileLine"* %this, %"struct.kc::impl_casestring__Str"* %_file, i32 %_line) align 2 {
+define void @_ZN2kc22impl_fileline_FileLineC2EPNS_20impl_casestring__StrEi(%"struct.kc::impl_fileline_FileLine"* %this, %"struct.kc::impl_casestring__Str"* %_file, i32 %_line) align 2 personality i8* bitcast (i32 (...)* @__gxx_personality_v0 to i8*) {
 entry:
   %this_addr = alloca %"struct.kc::impl_fileline_FileLine"*, align 4
   %_file_addr = alloca %"struct.kc::impl_casestring__Str"*, align 4
@@ -75,7 +75,7 @@ bb2:                                              ; preds = %bb1, %invcont
   ret void
 
 lpad:                                             ; preds = %bb
-  %eh_ptr = landingpad { i8*, i32 } personality i8* bitcast (i32 (...)* @__gxx_personality_v0 to i8*)
+  %eh_ptr = landingpad { i8*, i32 }
               cleanup
   %exn = extractvalue { i8*, i32 } %eh_ptr, 0
   store i8* %exn, i8** %eh_exception
@@ -148,7 +148,7 @@ return:                                           ; preds = %bb1, %entry
   ret void
 }
 
-define void @_ZN2kc22impl_fileline_FileLineC1EPNS_20impl_casestring__StrEi(%"struct.kc::impl_fileline_FileLine"* %this, %"struct.kc::impl_casestring__Str"* %_file, i32 %_line) align 2 {
+define void @_ZN2kc22impl_fileline_FileLineC1EPNS_20impl_casestring__StrEi(%"struct.kc::impl_fileline_FileLine"* %this, %"struct.kc::impl_casestring__Str"* %_file, i32 %_line) align 2 personality i8* bitcast (i32 (...)* @__gxx_personality_v0 to i8*) {
 entry:
   %this_addr = alloca %"struct.kc::impl_fileline_FileLine"*, align 4
   %_file_addr = alloca %"struct.kc::impl_casestring__Str"*, align 4
@@ -201,7 +201,7 @@ bb2:                                              ; preds = %bb1, %invcont
   ret void
 
 lpad:                                             ; preds = %bb
-  %eh_ptr = landingpad { i8*, i32 } personality i8* bitcast (i32 (...)* @__gxx_personality_v0 to i8*)
+  %eh_ptr = landingpad { i8*, i32 }
               cleanup
   %exn = extractvalue { i8*, i32 } %eh_ptr, 0
   store i8* %exn, i8** %eh_exception
diff --git a/test/Transforms/MergeFunc/alloca.ll b/test/Transforms/MergeFunc/alloca.ll
new file mode 100644
index 00000000000..d9f66d0911d
--- /dev/null
+++ b/test/Transforms/MergeFunc/alloca.ll
@@ -0,0 +1,33 @@
+; RUN: opt -mergefunc -S < %s | FileCheck %s
+
+;; Make sure that two different sized allocas are not treated as equal.
+
+target datalayout = "e-m:w-p:32:32-i64:64-f80:32-n8:16:32-S32"
+
+%kv1 = type { i32, i32 }
+%kv2 = type { i8 }
+
+
+define void @a(i8 *%f) {
+  %v = alloca %kv1, align 8
+  %f_2 = bitcast i8* %f to void (%kv1 *)*
+  call void %f_2(%kv1 * %v)
+  call void %f_2(%kv1 * %v)
+  call void %f_2(%kv1 * %v)
+  call void %f_2(%kv1 * %v)
+  ret void
+}
+
+; CHECK-LABEL: define void @b
+; CHECK-NOT: call @a
+; CHECK: ret
+
+define void @b(i8 *%f) {
+  %v = alloca %kv2, align 8
+  %f_2 = bitcast i8* %f to void (%kv2 *)*
+  call void %f_2(%kv2 * %v)
+  call void %f_2(%kv2 * %v)
+  call void %f_2(%kv2 * %v)
+  call void %f_2(%kv2 * %v)
+  ret void
+}
diff --git a/test/Transforms/MergeFunc/apply_function_attributes.ll b/test/Transforms/MergeFunc/apply_function_attributes.ll
new file mode 100644
index 00000000000..fa7f052f7ff
--- /dev/null
+++ b/test/Transforms/MergeFunc/apply_function_attributes.ll
@@ -0,0 +1,40 @@
+; RUN: opt -S -mergefunc < %s | FileCheck %s
+
+%Opaque_type = type opaque
+%S2i = type <{ i64, i64 }>
+%D2i = type <{ i64, i64 }>
+%Di = type <{ i32 }>
+%Si = type <{ i32 }>
+
+define void @B(%Opaque_type* sret %a, %S2i* %b, i32* %xp, i32* %yp) {
+  %x = load i32, i32* %xp
+  %y = load i32, i32* %yp
+  %sum = add i32 %x, %y
+  %sum2 = add i32 %sum, %y
+  %sum3 = add i32 %sum2, %y
+  ret void
+}
+
+define void @C(%Opaque_type* sret %a, %S2i* %b, i32* %xp, i32* %yp) {
+  %x = load i32, i32* %xp
+  %y = load i32, i32* %yp
+  %sum = add i32 %x, %y
+  %sum2 = add i32 %sum, %y
+  %sum3 = add i32 %sum2, %y
+  ret void
+}
+
+define void @A(%Opaque_type* sret %a, %D2i* %b, i32* %xp, i32* %yp) {
+  %x = load i32, i32* %xp
+  %y = load i32, i32* %yp
+  %sum = add i32 %x, %y
+  %sum2 = add i32 %sum, %y
+  %sum3 = add i32 %sum2, %y
+  ret void
+}
+
+; Make sure we transfer the parameter attributes to the call site.
+; CHECK-LABEL: define void @C(%Opaque_type* sret
+; CHECK:  tail call void bitcast (void (%Opaque_type*, %D2i*, i32*, i32*)* @A to void (%Opaque_type*, %S2i*, i32*, i32*)*)(%Opaque_type* sret %0, %S2i* %1, i32* %2, i32* %3)
+; CHECK:  ret void
+
diff --git a/test/Transforms/MergeFunc/call-and-invoke-with-ranges.ll b/test/Transforms/MergeFunc/call-and-invoke-with-ranges.ll
index b2083cb5c70..b955e3c9582 100644
--- a/test/Transforms/MergeFunc/call-and-invoke-with-ranges.ll
+++ b/test/Transforms/MergeFunc/call-and-invoke-with-ranges.ll
@@ -26,18 +26,18 @@ define i8 @call_different_range() {
   ret i8 %out
 }
 
-define i8 @invoke_with_range() {
+define i8 @invoke_with_range() personality i8* undef {
   %out = invoke i8 @dummy() to label %next unwind label %lpad, !range !0
 
 next:
   ret i8 %out
 
 lpad:
-  %pad = landingpad { i8*, i32 } personality i8* undef cleanup
+  %pad = landingpad { i8*, i32 } cleanup
   resume { i8*, i32 } zeroinitializer
 }
 
-define i8 @invoke_no_range() {
+define i8 @invoke_no_range() personality i8* undef {
 ; CHECK-LABEL: @invoke_no_range()
 ; CHECK-NEXT: invoke i8 @dummy
   %out = invoke i8 @dummy() to label %next unwind label %lpad
@@ -46,11 +46,11 @@ next:
   ret i8 %out
 
 lpad:
-  %pad = landingpad { i8*, i32 } personality i8* undef cleanup
+  %pad = landingpad { i8*, i32 } cleanup
   resume { i8*, i32 } zeroinitializer
 }
 
-define i8 @invoke_different_range() {
+define i8 @invoke_different_range() personality i8* undef {
 ; CHECK-LABEL: @invoke_different_range()
 ; CHECK-NEXT: invoke i8 @dummy
   %out = invoke i8 @dummy() to label %next unwind label %lpad, !range !1
@@ -59,20 +59,20 @@ next:
   ret i8 %out
 
 lpad:
-  %pad = landingpad { i8*, i32 } personality i8* undef cleanup
+  %pad = landingpad { i8*, i32 } cleanup
   resume { i8*, i32 } zeroinitializer
 }
 
-define i8 @call_same_range() {
-; CHECK-LABEL: @call_same_range
+define i8 @call_with_same_range() {
+; CHECK-LABEL: @call_with_same_range
 ; CHECK: tail call i8 @call_with_range
   bitcast i8 0 to i8
   %out = call i8 @dummy(), !range !0
   ret i8 %out
 }
 
-define i8 @invoke_same_range() {
-; CHECK-LABEL: @invoke_same_range()
+define i8 @invoke_with_same_range() personality i8* undef {
+; CHECK-LABEL: @invoke_with_same_range()
 ; CHECK: tail call i8 @invoke_with_range()
   %out = invoke i8 @dummy() to label %next unwind label %lpad, !range !0
 
@@ -80,7 +80,7 @@ next:
   ret i8 %out
 
 lpad:
-  %pad = landingpad { i8*, i32 } personality i8* undef cleanup
+  %pad = landingpad { i8*, i32 } cleanup
   resume { i8*, i32 } zeroinitializer
 }
 
diff --git a/test/Transforms/MergeFunc/fold-weak.ll b/test/Transforms/MergeFunc/fold-weak.ll
index 4df6e39c125..f8a18887890 100644
--- a/test/Transforms/MergeFunc/fold-weak.ll
+++ b/test/Transforms/MergeFunc/fold-weak.ll
@@ -1,17 +1,47 @@
-; RUN: opt < %s -mergefunc -S > %t
-; RUN: grep "define weak" %t | count 2
-; RUN: grep "call" %t | count 2
-; XFAIL: *
-
-; This test is off for a bit as we change this particular sort of folding to
-; only apply on ELF systems and not Mach-O systems.
+; RUN: opt -S -mergefunc < %s | FileCheck %s
 
 define weak i32 @sum(i32 %x, i32 %y) {
   %sum = add i32 %x, %y
-  ret i32 %sum
+  %sum2 = add i32 %sum, %y
+  %sum3 = add i32 %sum2, %y
+  ret i32 %sum3
 }
 
 define weak i32 @add(i32 %x, i32 %y) {
   %sum = add i32 %x, %y
-  ret i32 %sum
+  %sum2 = add i32 %sum, %y
+  %sum3 = add i32 %sum2, %y
+  ret i32 %sum3
+}
+
+; Don't replace a weak function use by another equivalent function. We don't
+; know whether the symbol that will ulitmately be linked is equivalent - we
+; don't know that the weak definition is the definitive definition or whether it
+; will be overriden by a stronger definition).
+
+; CHECK-LABEL: define private i32 @0
+; CHECK: add i32
+; CHECK: add i32
+; CHECK: add i32
+; CHECK: ret
+
+; CHECK-LABEL: define i32 @use_weak
+; CHECK: call i32 @add
+; CHECK: call i32 @sum
+; CHECK: ret
+
+; CHECK-LABEL: define weak i32 @sum
+; CHECK:  tail call i32 @0
+; CHECK:  ret
+
+; CHECK-LABEL: define weak i32 @add
+; CHECK:  tail call i32 @0
+; CHECK:  ret
+
+
+define i32 @use_weak(i32 %a, i32 %b) {
+  %res = call i32 @add(i32 %a, i32 %b)
+  %res2 = call i32 @sum(i32 %a, i32 %b)
+  %res3 = add i32 %res, %res2
+  ret i32 %res3
 }
diff --git a/test/Transforms/MergeFunc/linkonce_odr.ll b/test/Transforms/MergeFunc/linkonce_odr.ll
new file mode 100644
index 00000000000..1ad0d727f83
--- /dev/null
+++ b/test/Transforms/MergeFunc/linkonce_odr.ll
@@ -0,0 +1,30 @@
+; RUN: opt -S -mergefunc < %s | FileCheck %s
+
+; Replacments should be totally ordered on the function name.
+; If we don't do this we  can end up with one module defining a thunk for @funA
+; and another module defining a thunk for @funB.
+;
+; The problem with this is that the linker could then choose these two stubs
+; each of the two modules and we end up with two stubs calling each other.
+
+; CHECK-LABEL: define linkonce_odr i32 @funA
+; CHECK-NEXT:    add
+; CHECK:         ret
+
+; CHECK-LABEL: define linkonce_odr i32 @funB
+; CHECK-NEXT:    tail call i32 @funA(i32 %0, i32 %1)
+; CHECK-NEXT:    ret
+
+define linkonce_odr i32 @funB(i32 %x, i32 %y) {
+  %sum = add i32 %x, %y
+  %sum2 = add i32 %x, %sum
+  %sum3 = add i32 %x, %sum2
+  ret i32 %sum3
+}
+
+define linkonce_odr i32 @funA(i32 %x, i32 %y) {
+  %sum = add i32 %x, %y
+  %sum2 = add i32 %x, %sum
+  %sum3 = add i32 %x, %sum2
+  ret i32 %sum3
+}
diff --git a/test/Transforms/MergeFunc/self-referential-global.ll b/test/Transforms/MergeFunc/self-referential-global.ll
new file mode 100644
index 00000000000..d3d1c62aa7f
--- /dev/null
+++ b/test/Transforms/MergeFunc/self-referential-global.ll
@@ -0,0 +1,40 @@
+; RUN: opt -mergefunc -disable-output < %s
+
+; A linked list type and simple payload
+%LL = type { %S, %LL* }
+%S = type { void (%S*, i32)* }
+
+; Table refers to itself via GEP
+@Table = internal global [3 x %LL] [%LL { %S { void (%S*, i32)* @B }, %LL* getelementptr inbounds ([3 x %LL], [3 x %LL]* @Table, i32 0, i32 0) }, %LL { %S { void (%S*, i32)* @A }, %LL* getelementptr inbounds ([3 x %LL], [3 x %LL]* @Table, i32 0, i32 0) }, %LL { %S { void (%S*, i32)* @A }, %LL* getelementptr inbounds ([3 x %LL], [3 x %LL]* @Table, i32 0, i32 0) }], align 16
+
+; The body of this is irrelevant; it is long so that mergefunc doesn't skip it as a small function.
+define internal void @A(%S* %self, i32 %a) {
+  %1 = add i32 %a, 32
+  %2 = add i32 %1, 32
+  %3 = add i32 %2, 32
+  %4 = add i32 %3, 32
+  %5 = add i32 %4, 32
+  %6 = add i32 %5, 32
+  %7 = add i32 %6, 32
+  %8 = add i32 %7, 32
+  %9 = add i32 %8, 32
+  %10 = add i32 %9, 32
+  %11 = add i32 %10, 32
+  ret void
+}
+
+define internal void @B(%S* %self, i32 %a) {
+  %1 = add i32 %a, 32
+  %2 = add i32 %1, 32
+  %3 = add i32 %2, 32
+  %4 = add i32 %3, 32
+  %5 = add i32 %4, 32
+  %6 = add i32 %5, 32
+  %7 = add i32 %6, 32
+  %8 = add i32 %7, 32
+  %9 = add i32 %8, 32
+  %10 = add i32 %9, 32
+  %11 = add i32 %10, 32
+  ret void
+}
+
diff --git a/test/Transforms/MinSFI/abi-verifier-whitelist-syscalls-global.ll b/test/Transforms/MinSFI/abi-verifier-whitelist-syscalls-global.ll
deleted file mode 100644
index 0ad5b1c90ed..00000000000
--- a/test/Transforms/MinSFI/abi-verifier-whitelist-syscalls-global.ll
+++ /dev/null
@@ -1,13 +0,0 @@
-; RUN: not opt %s -verify-pnaclabi-module -pnaclabi-allow-minsfi-syscalls -S \
-; RUN:   2>&1 | FileCheck %s
-
-target datalayout = "p:32:32:32"
-target triple = "le32-unknown-nacl"
-
-define i32 @_start(i32 %args) {
-  ret i32 0
-}
-
-@__minsfi_syscall_dummy = global [4 x i8] c"DATA"
-
-; CHECK: __minsfi_syscall_dummy is not a valid external symbol
diff --git a/test/Transforms/MinSFI/abi-verifier-whitelist-syscalls-wrong-type.ll b/test/Transforms/MinSFI/abi-verifier-whitelist-syscalls-wrong-type.ll
deleted file mode 100644
index 409b3ef8ef0..00000000000
--- a/test/Transforms/MinSFI/abi-verifier-whitelist-syscalls-wrong-type.ll
+++ /dev/null
@@ -1,21 +0,0 @@
-; RUN: not opt %s -verify-pnaclabi-module -pnaclabi-allow-minsfi-syscalls -S \
-; RUN:   2>&1 | FileCheck %s
-
-target datalayout = "p:32:32:32"
-target triple = "le32-unknown-nacl"
-
-define void @_start() {
-  ret void
-}
-
-declare i32 @__minsfi_syscall_good()
-; CHECK-NOT: __minsfi_syscall_good is not a valid external symbol
-
-declare void @__minsfi_syscall_bad_void()
-; CHECK: __minsfi_syscall_bad_void is not a valid external symbol
-
-declare i64 @__minsfi_syscall_bad_i64()
-; CHECK: __minsfi_syscall_bad_i64 is not a valid external symbol
-
-declare float @__minsfi_syscall_bad_float()
-; CHECK: __minsfi_syscall_bad_float is not a valid external symbol
diff --git a/test/Transforms/MinSFI/abi-verifier-whitelist-syscalls.ll b/test/Transforms/MinSFI/abi-verifier-whitelist-syscalls.ll
deleted file mode 100644
index 68f458590e6..00000000000
--- a/test/Transforms/MinSFI/abi-verifier-whitelist-syscalls.ll
+++ /dev/null
@@ -1,18 +0,0 @@
-; RUN: opt %s -verify-pnaclabi-module -pnaclabi-allow-minsfi-syscalls -S \
-; RUN:   | FileCheck -check-prefix=CHECK-WITHFLAG %s
-; RUN: not opt %s -verify-pnaclabi-module -S 2>&1 \
-; RUN:   | FileCheck -check-prefix=CHECK-NOFLAG %s
-
-target datalayout = "p:32:32:32"
-target triple = "le32-unknown-nacl"
-
-define i32 @_start(i32 %args) {
-  ret i32 0
-}
-
-declare i32 @__minsfi_syscall_dummy(i32)
-
-; CHECK-NOFLAG: __minsfi_syscall_dummy is declared but not defined
-; CHECK-NOFLAG: __minsfi_syscall_dummy is not a valid external symbol
-
-; CHECK-WITHFLAG: declare i32 @__minsfi_syscall_dummy(i32)
diff --git a/test/Transforms/MinSFI/allocate-data-segment-empty.ll b/test/Transforms/MinSFI/allocate-data-segment-empty.ll
deleted file mode 100644
index 7b366976a27..00000000000
--- a/test/Transforms/MinSFI/allocate-data-segment-empty.ll
+++ /dev/null
@@ -1,12 +0,0 @@
-; RUN: opt %s -minsfi-allocate-data-segment -S | FileCheck %s
-
-target datalayout = "p:32:32:32"
-target triple = "le32-unknown-nacl"
-
-define void @foo() {
-  ret void
-}
-
-; CHECK: %__sfi_data_segment = type <{}>
-; CHECK: @__sfi_data_segment = constant %__sfi_data_segment zeroinitializer
-; CHECK: @__sfi_data_segment_size = constant i32 0 
diff --git a/test/Transforms/MinSFI/allocate-data-segment.ll b/test/Transforms/MinSFI/allocate-data-segment.ll
deleted file mode 100644
index 6c73e1f73c3..00000000000
--- a/test/Transforms/MinSFI/allocate-data-segment.ll
+++ /dev/null
@@ -1,46 +0,0 @@
-; RUN: opt %s -minsfi-allocate-data-segment -S | FileCheck %s
-
-target datalayout = "p:32:32:32"
-target triple = "le32-unknown-nacl"
-
-; Every global variable in PNaCl IR must have an initializer. This is an
-; exhaustive list of all the possible formats.
-
-@a = global [1 x i8] zeroinitializer                       ; no alignment
-@b = global [3 x i8] c"ABC"                                ; no alignment
-@c = global [5 x i8] c"ABCDE", align 16                    ; explicit 16B align
-@d = global i32 ptrtoint ([3 x i8]* @b to i32)             ; implicit 4B align
-@e = global i32 add (i32 ptrtoint ([5 x i8]* @c to i32), i32 3), align 2
-                                                           ; implicit 4B align
-@f = global <{ [1 x i8], i32 }> 
-       <{ [1 x i8] zeroinitializer, 
-          i32 ptrtoint ([5 x i8]* @c to i32) }>            ; no alignment
-@g = global [1 x i8] zeroinitializer                       ; no alignment
-
-; Use each variable to verify its location
- 
-define void @use_variables() {
-  %a = load [1 x i8], [1 x i8]* @a
-  %b = load [3 x i8], [3 x i8]* @b
-  %c = load [5 x i8], [5 x i8]* @c
-  %d = load i32, i32* @d
-  %e = load i32, i32* @e
-  %f = load <{ [1 x i8], i32 }>, <{ [1 x i8], i32 }>* @f
-  %g = load [1 x i8], [1 x i8]* @g
-  ret void
-}
-
-; CHECK: %__sfi_data_segment = type <{ [1 x i8], [3 x i8], [12 x i8], [5 x i8], [3 x i8], i32, i32, <{ [1 x i8], i32 }>, [1 x i8] }>
-; CHECK: @__sfi_data_segment = constant %__sfi_data_segment <{ [1 x i8] zeroinitializer, [3 x i8] c"ABC", [12 x i8] zeroinitializer, [5 x i8] c"ABCDE", [3 x i8] zeroinitializer, i32 65537, i32 65555, <{ [1 x i8], i32 }> <{ [1 x i8] zeroinitializer, i32 65552 }>, [1 x i8] zeroinitializer }>
-; CHECK: @__sfi_data_segment_size = constant i32 38 
-
-; CHECK-LABEL: define void @use_variables() {
-; CHECK-NEXT:    %a = load [1 x i8], [1 x i8]* inttoptr (i32 65536 to [1 x i8]*)
-; CHECK-NEXT:    %b = load [3 x i8], [3 x i8]* inttoptr (i32 65537 to [3 x i8]*)
-; CHECK-NEXT:    %c = load [5 x i8], [5 x i8]* inttoptr (i32 65552 to [5 x i8]*)
-; CHECK-NEXT:    %d = load i32, i32* inttoptr (i32 65560 to i32*)
-; CHECK-NEXT:    %e = load i32, i32* inttoptr (i32 65564 to i32*)
-; CHECK-NEXT:    %f = load <{ [1 x i8], i32 }>, <{ [1 x i8], i32 }>* inttoptr (i32 65568 to <{ [1 x i8], i32 }>*)
-; CHECK-NEXT:    %g = load [1 x i8], [1 x i8]* inttoptr (i32 65573 to [1 x i8]*)
-; CHECK-NEXT:    ret void
-; CHECK-NEXT:  }
diff --git a/test/Transforms/MinSFI/expand-allocas-globalvar-exists.ll b/test/Transforms/MinSFI/expand-allocas-globalvar-exists.ll
deleted file mode 100644
index 04f6f3c6739..00000000000
--- a/test/Transforms/MinSFI/expand-allocas-globalvar-exists.ll
+++ /dev/null
@@ -1,24 +0,0 @@
-; RUN: opt %s -minsfi-expand-allocas -S | FileCheck %s
-
-target datalayout = "p:32:32:32"
-target triple = "le32-unknown-nacl"
-
-@__sfi_stack_ptr = external global i32
-
-; CHECK: @__sfi_stack_ptr1 = internal global i32
-
-define i8* @test_correct_global_var_used() {
-  %ptr = alloca i8
-  ret i8* %ptr
-}
-
-; CHECK-LABEL: define i8* @test_correct_global_var_used() {
-; CHECK-NEXT:    %frame_top = load i32, i32* @__sfi_stack_ptr1
-; CHECK-NEXT:    %1 = sub i32 %frame_top, 1
-; CHECK-NEXT:    %ptr = inttoptr i32 %1 to i8*
-; CHECK-NEXT:    ret i8* %ptr
-; CHECK-NEXT:  }
-
-define i32 @_start_minsfi(i32 %args) {
-  ret i32 0
-}
diff --git a/test/Transforms/MinSFI/expand-allocas-no-start.ll b/test/Transforms/MinSFI/expand-allocas-no-start.ll
deleted file mode 100644
index 219ff9f9ae9..00000000000
--- a/test/Transforms/MinSFI/expand-allocas-no-start.ll
+++ /dev/null
@@ -1,10 +0,0 @@
-; RUN: not opt %s -minsfi-expand-allocas -S 2>&1 | FileCheck %s
-
-target datalayout = "p:32:32:32"
-target triple = "le32-unknown-nacl"
-
-define i32 @foo(i32 %arg1) {
-  ret i32 0
-}
-
-; CHECK: ExpandAllocas: Module does not have an entry function
diff --git a/test/Transforms/MinSFI/expand-allocas-start-argscount.ll b/test/Transforms/MinSFI/expand-allocas-start-argscount.ll
deleted file mode 100644
index 9434cd15283..00000000000
--- a/test/Transforms/MinSFI/expand-allocas-start-argscount.ll
+++ /dev/null
@@ -1,10 +0,0 @@
-; RUN: not opt %s -minsfi-expand-allocas -S 2>&1 | FileCheck %s
-
-target datalayout = "p:32:32:32"
-target triple = "le32-unknown-nacl"
-
-define i32 @_start_minsfi(float %arg1) {
-  ret i32 0
-}
-
-; CHECK: ExpandAllocas: Invalid signature of _start_minsfi
diff --git a/test/Transforms/MinSFI/expand-allocas-start-argstype.ll b/test/Transforms/MinSFI/expand-allocas-start-argstype.ll
deleted file mode 100644
index 0ae6e8b691d..00000000000
--- a/test/Transforms/MinSFI/expand-allocas-start-argstype.ll
+++ /dev/null
@@ -1,10 +0,0 @@
-; RUN: not opt %s -minsfi-expand-allocas -S 2>&1 | FileCheck %s
-
-target datalayout = "p:32:32:32"
-target triple = "le32-unknown-nacl"
-
-define i32 @_start_minsfi(i32 %arg1, i32 %arg2) {
-  ret i32 0
-}
-
-; CHECK: ExpandAllocas: Invalid signature of _start_minsfi
diff --git a/test/Transforms/MinSFI/expand-allocas.ll b/test/Transforms/MinSFI/expand-allocas.ll
deleted file mode 100644
index be6356218c1..00000000000
--- a/test/Transforms/MinSFI/expand-allocas.ll
+++ /dev/null
@@ -1,260 +0,0 @@
-; RUN: opt %s -minsfi-expand-allocas -S | FileCheck %s
-
-!llvm.module.flags = !{!0}
-!0 = !{i32 1, !"Debug Info Version", i32 3}
-
-target datalayout = "p:32:32:32"
-target triple = "le32-unknown-nacl"
-
-; Check that the stack pointer global variable is created. This does not check 
-; the initial value of the stack ptr.
-; CHECK: @__sfi_stack_ptr = internal global i32
-
-declare i8* @llvm.stacksave()
-declare void @llvm.stackrestore(i8* %ptr)
-
-declare void @foo()
-
-define i32 @test_no_alloca(i1 %cond) {
-  br i1 %cond, label %IfOne, label %IfNotOne
-IfOne:
-  call void @foo()
-  ret i32 2
-IfNotOne:
-  ret i32 3
-}
-
-; CHECK-LABEL: define i32 @test_no_alloca(i1 %cond) {
-; CHECK-NEXT:    br i1 %cond, label %IfOne, label %IfNotOne
-; CHECK:       IfOne:
-; CHECK-NEXT:    call void @foo()
-; CHECK-NEXT:    ret i32 2
-; CHECK:       IfNotOne:
-; CHECK-NEXT:    ret i32 3
-; CHECK-NEXT:  }
-
-define i8* @test_const_alloca() {
-  %ptr = alloca i8, i32 9, !dbg !1
-  ret i8* %ptr
-}
-
-; CHECK-LABEL: define i8* @test_const_alloca() {
-; CHECK-NEXT:    %frame_top = load i32, i32* @__sfi_stack_ptr
-; CHECK-NEXT:    %1 = sub i32 %frame_top, 9
-; CHECK-NEXT:    %ptr = inttoptr i32 %1 to i8*, !dbg !1
-; CHECK-NEXT:    ret i8* %ptr
-; CHECK-NEXT:  }
-
-define i8* @test_const_alloca_align() {
-  %ptr = alloca i8, i32 9, align 536870912  ; biggest possible alignment
-  ret i8* %ptr
-}
-
-; CHECK-LABEL: define i8* @test_const_alloca_align() {
-; CHECK-NEXT:    %frame_top = load i32, i32* @__sfi_stack_ptr
-; CHECK-NEXT:    %1 = sub i32 %frame_top, 9
-; CHECK-NEXT:    %2 = and i32 %1, -536870912
-; CHECK-NEXT:    %ptr = inttoptr i32 %2 to i8*
-; CHECK-NEXT:    ret i8* %ptr
-; CHECK-NEXT:  }
-
-define i8* @test_variable_length_alloca(i32 %size) {
-  %ptr = alloca i8, i32 %size, !dbg !1
-  ret i8* %ptr
-}
-
-; CHECK-LABEL: define i8* @test_variable_length_alloca(i32 %size) {
-; CHECK-NEXT:    %frame_top = load i32, i32* @__sfi_stack_ptr
-; CHECK-NEXT:    %1 = sub i32 %frame_top, %size
-; CHECK-NEXT:    %ptr = inttoptr i32 %1 to i8*, !dbg !1
-; CHECK-NEXT:    ret i8* %ptr
-; CHECK-NEXT:  }
-
-define i8* @test_variable_length_alloca_align(i32 %size) {
-  %ptr = alloca i8, i32 %size, align 32
-  ret i8* %ptr
-}
-
-; CHECK-LABEL: define i8* @test_variable_length_alloca_align(i32 %size) {
-; CHECK-NEXT:    %frame_top = load i32, i32* @__sfi_stack_ptr
-; CHECK-NEXT:    %1 = sub i32 %frame_top, %size
-; CHECK-NEXT:    %2 = and i32 %1, -32
-; CHECK-NEXT:    %ptr = inttoptr i32 %2 to i8*
-; CHECK-NEXT:    ret i8* %ptr
-; CHECK-NEXT:  }
-
-define i8* @test_const_after_const_alloca() {
-  %ptr1 = alloca i8, i32 4
-  %ptr2 = alloca i8, i32 8
-  ret i8* %ptr2
-}
-
-; CHECK-LABEL: define i8* @test_const_after_const_alloca() {
-; CHECK-NEXT:    %frame_top = load i32, i32* @__sfi_stack_ptr
-; CHECK-NEXT:    %1 = sub i32 %frame_top, 4
-; CHECK-NEXT:    %ptr1 = inttoptr i32 %1 to i8*
-; CHECK-NEXT:    %2 = sub i32 %1, 8
-; CHECK-NEXT:    %ptr2 = inttoptr i32 %2 to i8*
-; CHECK-NEXT:    ret i8* %ptr2
-; CHECK-NEXT:  }
-
-define i8* @test_const_after_variable_alloca(i32 %size) {
-  %ptr1 = alloca i8, i32 %size
-  %ptr2 = alloca i8, i32 19
-  ret i8* %ptr2
-}
-
-; CHECK-LABEL: define i8* @test_const_after_variable_alloca(i32 %size) {
-; CHECK-NEXT:    %frame_top = load i32, i32* @__sfi_stack_ptr
-; CHECK-NEXT:    %1 = sub i32 %frame_top, %size
-; CHECK-NEXT:    %ptr1 = inttoptr i32 %1 to i8*
-; CHECK-NEXT:    %2 = sub i32 %1, 19
-; CHECK-NEXT:    %ptr2 = inttoptr i32 %2 to i8*
-; CHECK-NEXT:    ret i8* %ptr2
-; CHECK-NEXT:  }
-
-define i8* @test_stacksave() {
-  %ptr = call i8* @llvm.stacksave(), !dbg !1
-  ret i8* %ptr
-}
-
-; CHECK-LABEL: define i8* @test_stacksave() {
-; CHECK-NEXT:    %frame_top = load i32, i32* @__sfi_stack_ptr
-; CHECK-NEXT:    %ptr = inttoptr i32 %frame_top to i8*, !dbg !1
-; CHECK-NEXT:    ret i8* %ptr
-; CHECK-NEXT:  }
-
-define i8* @test_stacksave_after_alloca() {
-  %ptr1 = alloca i8, i32 11
-  %ptr2 = call i8* @llvm.stacksave(), !dbg !1
-  ret i8* %ptr2
-}
-
-; CHECK-LABEL: define i8* @test_stacksave_after_alloca() {
-; CHECK-NEXT:    %frame_top = load i32, i32* @__sfi_stack_ptr
-; CHECK-NEXT:    %1 = sub i32 %frame_top, 11
-; CHECK-NEXT:    %ptr1 = inttoptr i32 %1 to i8*
-; CHECK-NEXT:    %ptr2 = inttoptr i32 %1 to i8*, !dbg !1
-; CHECK-NEXT:    ret i8* %ptr2
-; CHECK-NEXT:  }
-
-define i8* @test_stackrestore(i8* %new_stack) {
-  call void @llvm.stackrestore(i8* %new_stack), !dbg !1
-  %ptr = alloca i8, i32 5, !dbg !3
-  ret i8* %ptr
-}
-
-; CHECK-LABEL: define i8* @test_stackrestore(i8* %new_stack) {
-; CHECK-NEXT:    frame_top = load i32, i32* @__sfi_stack_ptr
-; CHECK-NEXT:    %1 = ptrtoint i8* %new_stack to i32, !dbg !1
-; CHECK-NEXT:    %2 = sub i32 %1, 5
-; CHECK-NEXT:    %ptr = inttoptr i32 %2 to i8*, !dbg !3
-; CHECK-NEXT:    ret i8* %ptr
-; CHECK-NEXT:  }
-
-define i8* @test_stackrestore_after_push(i8* %new_stack) {
-  %ptr1 = alloca i8, i32 5, !dbg !1
-  call void @llvm.stackrestore(i8* %new_stack), !dbg !3
-  %ptr2 = alloca i8, i32 6, !dbg !4
-  ret i8* %ptr2
-}
-
-; CHECK-LABEL: define i8* @test_stackrestore_after_push(i8* %new_stack) {
-; CHECK-NEXT:    %frame_top = load i32, i32* @__sfi_stack_ptr
-; CHECK-NEXT:    %1 = sub i32 %frame_top, 5
-; CHECK-NEXT:    %ptr1 = inttoptr i32 %1 to i8*, !dbg !1
-; CHECK-NEXT:    %2 = ptrtoint i8* %new_stack to i32, !dbg !3
-; CHECK-NEXT:    %3 = sub i32 %2, 6
-; CHECK-NEXT:    %ptr2 = inttoptr i32 %3 to i8*, !dbg !4
-; CHECK-NEXT:    ret i8* %ptr2
-; CHECK-NEXT:  }
-
-define i8* @test_stackptr_bb_propagation(i1 %cond) {
-  %ptr1 = alloca i8, i32 8
-  br i1 %cond, label %IfOne, label %IfNotOne
-IfOne:
-  ret i8* %ptr1
-IfNotOne:
-  %ptr2 = alloca i8, i32 4
-  ret i8* %ptr2
-}
-
-; CHECK-LABEL: define i8* @test_stackptr_bb_propagation(i1 %cond) {
-; CHECK-NEXT:    %frame_top = load i32, i32* @__sfi_stack_ptr
-; CHECK-NEXT:    %1 = sub i32 %frame_top, 8
-; CHECK-NEXT:    %ptr1 = inttoptr i32 %1 to i8*
-; CHECK-NEXT:    br i1 %cond, label %IfOne, label %IfNotOne
-; CHECK:       IfOne:
-; CHECK-NEXT:    %2 = phi i32 [ %1, %0 ]
-; CHECK-NEXT:    ret i8* %ptr1
-; CHECK:       IfNotOne:
-; CHECK-NEXT:    %3 = phi i32 [ %1, %0 ]
-; CHECK-NEXT:    %4 = sub i32 %3, 4
-; CHECK-NEXT:    %ptr2 = inttoptr i32 %4 to i8*
-; CHECK-NEXT:    ret i8* %ptr2
-; CHECK-NEXT:  }
-
-define i8* @test_scoped_alloca(i1 %cmp) {
-  br label %loop
-loop:
-  %stacksave = call i8* @llvm.stacksave()
-  %ptr = alloca i8, i32 17
-  call void @llvm.stackrestore(i8* %stacksave)
-  br i1 %cmp, label %done, label %loop
-done:
-  ret i8* %ptr
-}
-
-; CHECK-LABEL: define i8* @test_scoped_alloca(i1 %cmp) {
-; CHECK-NEXT:    %frame_top = load i32, i32* @__sfi_stack_ptr
-; CHECK-NEXT:    br label %loop
-; CHECK:       loop:
-; CHECK-NEXT:    %1 = phi i32 [ %frame_top, %0 ], [ %3, %loop ]
-; CHECK-NEXT:    %stacksave = inttoptr i32 %1 to i8*
-; CHECK-NEXT:    %2 = sub i32 %1, 17
-; CHECK-NEXT:    %ptr = inttoptr i32 %2 to i8*
-; CHECK-NEXT:    %3 = ptrtoint i8* %stacksave to i32
-; CHECK-NEXT:    br i1 %cmp, label %done, label %loop
-; CHECK:       done:
-; CHECK-NEXT:    %4 = phi i32 [ %3, %loop ]
-; CHECK-NEXT:    ret i8* %ptr
-; CHECK-NEXT:  }
-
-define i8* @test_global_ptr_updates(i32 %size) {
-  %ptr0 = call i8* @llvm.stacksave()
-  %ptr1 = alloca i8, i32 32
-  %ptr2 = alloca i8, i32 %size
-  call void @llvm.stackrestore(i8* %ptr0)
-  call void @foo()
-  ret i8* %ptr2
-}
-
-; CHECK-LABEL: define i8* @test_global_ptr_updates(i32 %size) {
-; CHECK-NEXT:    %frame_top = load i32, i32* @__sfi_stack_ptr
-; CHECK-NEXT:    %ptr0 = inttoptr i32 %frame_top to i8*
-; CHECK-NEXT:    %1 = sub i32 %frame_top, 32
-; CHECK-NEXT:    store i32 %1, i32* @__sfi_stack_ptr
-; CHECK-NEXT:    %ptr1 = inttoptr i32 %1 to i8*
-; CHECK-NEXT:    %2 = sub i32 %1, %size
-; CHECK-NEXT:    store i32 %2, i32* @__sfi_stack_ptr
-; CHECK-NEXT:    %ptr2 = inttoptr i32 %2 to i8*
-; CHECK-NEXT:    %3 = ptrtoint i8* %ptr0 to i32
-; CHECK-NEXT:    store i32 %3, i32* @__sfi_stack_ptr
-; CHECK-NEXT:    call void @foo()
-; CHECK-NEXT:    store i32 %frame_top, i32* @__sfi_stack_ptr
-; CHECK-NEXT:    ret i8* %ptr2
-; CHECK-NEXT:  }
-
-define i32 @_start_minsfi(i32 %args) {
-  ret i32 0
-}
-
-; CHECK-LABEL: define i32 @_start_minsfi(i32 %args) {
-; CHECK-NEXT:    store i32 %args, i32* @__sfi_stack_ptr
-; CHECK-NEXT:    ret i32 0
-; CHECK-NEXT:  }
-
-!1 = !MDLocation(line: 1, column: 13, scope: !2)
-!2 = !MDSubprogram(name: "foo")
-!3 = !MDLocation(line: 2, column: 10, scope: !2)
-!4 = !MDLocation(line: 2, column: 3, scope: !2)
diff --git a/test/Transforms/MinSFI/lit.local.cfg b/test/Transforms/MinSFI/lit.local.cfg
deleted file mode 100644
index a43fd3ebdd5..00000000000
--- a/test/Transforms/MinSFI/lit.local.cfg
+++ /dev/null
@@ -1,3 +0,0 @@
-# -*- Python -*-
-
-config.suffixes = ['.ll']
diff --git a/test/Transforms/MinSFI/rename-entry-point-no-start.ll b/test/Transforms/MinSFI/rename-entry-point-no-start.ll
deleted file mode 100644
index 785a233b5f0..00000000000
--- a/test/Transforms/MinSFI/rename-entry-point-no-start.ll
+++ /dev/null
@@ -1,10 +0,0 @@
-; RUN: not opt %s -minsfi-rename-entry-point -S 2>&1 | FileCheck %s
-
-target datalayout = "p:32:32:32"
-target triple = "le32-unknown-nacl"
-
-define i32 @foo() {
-  ret i32 0
-}
-
-; CHECK: RenameEntryPoint: The module does not contain a function named '_start'
diff --git a/test/Transforms/MinSFI/rename-entry-point-startminsfi-exists.ll b/test/Transforms/MinSFI/rename-entry-point-startminsfi-exists.ll
deleted file mode 100644
index 388aaea0e86..00000000000
--- a/test/Transforms/MinSFI/rename-entry-point-startminsfi-exists.ll
+++ /dev/null
@@ -1,12 +0,0 @@
-; RUN: not opt %s -minsfi-rename-entry-point -S 2>&1 | FileCheck %s
-
-target datalayout = "p:32:32:32"
-target triple = "le32-unknown-nacl"
-
-@_start_minsfi = constant i32 1234
-
-define i32 @_start() {
-  ret i32 0
-}
-
-; CHECK: RenameEntryPoint: The module already contains a value named '_start_minsfi'
diff --git a/test/Transforms/MinSFI/rename-entry-point.ll b/test/Transforms/MinSFI/rename-entry-point.ll
deleted file mode 100644
index 9c966f054c5..00000000000
--- a/test/Transforms/MinSFI/rename-entry-point.ll
+++ /dev/null
@@ -1,12 +0,0 @@
-; RUN: opt %s -minsfi-rename-entry-point -S | FileCheck %s
-
-target datalayout = "p:32:32:32"
-target triple = "le32-unknown-nacl"
-
-define i32 @_start() {
-  ret i32 0
-}
-
-; CHECK-LABEL: define i32 @_start_minsfi() {
-; CHECK-NEXT:    ret i32 0
-; CHECK-NEXT:  }
diff --git a/test/Transforms/MinSFI/sandbox-indirect-calls-invalid-call.ll b/test/Transforms/MinSFI/sandbox-indirect-calls-invalid-call.ll
deleted file mode 100644
index 9af3e5ae428..00000000000
--- a/test/Transforms/MinSFI/sandbox-indirect-calls-invalid-call.ll
+++ /dev/null
@@ -1,15 +0,0 @@
-; RUN: not opt %s -minsfi-sandbox-indirect-calls -S 2>&1 | FileCheck %s
-
-target datalayout = "p:32:32:32"
-target triple = "le32-unknown-nacl"
-
-declare void @fn()
-
-declare i32 @foo(void ()*)
-
-define i32 @bar() {
-  %ret = call i32 @foo(void ()* @fn)
-  ret i32 %ret
-}
-
-; CHECK: Invalid reference to function @fn
diff --git a/test/Transforms/MinSFI/sandbox-indirect-calls-wrong-use.ll b/test/Transforms/MinSFI/sandbox-indirect-calls-wrong-use.ll
deleted file mode 100644
index ea44340d825..00000000000
--- a/test/Transforms/MinSFI/sandbox-indirect-calls-wrong-use.ll
+++ /dev/null
@@ -1,14 +0,0 @@
-; RUN: not opt %s -minsfi-sandbox-indirect-calls -S 2>&1 | FileCheck %s
-
-target datalayout = "p:32:32:32"
-target triple = "le32-unknown-nacl"
-
-declare i32 @foo(i32)
-
-define i32 @bar() {
-  %fn = bitcast i32 (i32)* @foo to i32 ()*
-  %ret = call i32 %fn()
-  ret i32 %ret
-}
-
-; CHECK: Invalid reference to function @foo
diff --git a/test/Transforms/MinSFI/sandbox-indirect-calls.ll b/test/Transforms/MinSFI/sandbox-indirect-calls.ll
deleted file mode 100644
index 38bafad6728..00000000000
--- a/test/Transforms/MinSFI/sandbox-indirect-calls.ll
+++ /dev/null
@@ -1,105 +0,0 @@
-; RUN: opt %s -minsfi-sandbox-indirect-calls -S | FileCheck %s
-
-!llvm.module.flags = !{!0}
-!0 = !{i32 1, !"Debug Info Version", i32 3}
-
-target datalayout = "p:32:32:32"
-target triple = "le32-unknown-nacl"
-
-declare void @fn_v_v()
-declare void @fn_v_i_not_addr_taken(i32)
-declare void @fn_v_i_1(i32)
-declare i32  @fn_i_i_not_addr_taken(i32)
-declare i32  @fn_i_ii(i32, i32)
-declare void @fn_v_i_2(i32)
-
-declare void @foo_2ptr(i32, i32)
-
-; CHECK-DAG: [[TAB_V_V:@__sfi_function_table[0-9]*]]  = internal constant [8 x void ()*]        [void ()* null,        void ()* @fn_v_v,     void ()* null,         void ()* null,            void ()* null,         void ()* null,        void ()* null,        void ()* null]
-; CHECK-DAG: [[TAB_V_I:@__sfi_function_table[0-9]*]]  = internal constant [8 x void (i32)*]     [void (i32)* null,     void (i32)* null,     void (i32)* @fn_v_i_1, void (i32)* null,         void (i32)* @fn_v_i_2, void (i32)* null,     void (i32)* null,     void (i32)* null]
-; CHECK-DAG: [[TAB_I_II:@__sfi_function_table[0-9]*]] = internal constant [8 x i32 (i32, i32)*] [i32 (i32, i32)* null, i32 (i32, i32)* null, i32 (i32, i32)* null,  i32 (i32, i32)* @fn_i_ii, i32 (i32, i32)* null,  i32 (i32, i32)* null, i32 (i32, i32)* null, i32 (i32, i32)* null]
-
-@test_global_inits = 
-  internal constant i32 ptrtoint (void (i32)* @fn_v_i_2 to i32)
-
-; CHECK-DAG: @test_global_inits = internal constant i32 4
-
-define void @test_direct_calls_not_replaced() {
-  call void @fn_v_i_not_addr_taken(i32 5)
-  call i32 @fn_i_i_not_addr_taken(i32 7)
-  ret void
-}
-
-; CHECK-LABEL: define void @test_direct_calls_not_replaced() {
-; CHECK-NEXT:    call void @fn_v_i_not_addr_taken(i32 5)
-; CHECK-NEXT:    call i32 @fn_i_i_not_addr_taken(i32 7)
-; CHECK-NEXT:    ret void
-; CHECK-NEXT:  }
-
-define void @test_ptr_consts_replaced() {
-  call void @foo_2ptr(i32 ptrtoint (void (i32)* @fn_v_i_1 to i32),
-                      i32 ptrtoint (void ()* @fn_v_v to i32))
-  ret void
-}
-
-; CHECK-LABEL: define void @test_ptr_consts_replaced() {
-; CHECK-NEXT:    call void @foo_2ptr(i32 2, i32 1)
-; CHECK-NEXT:    ret void
-; CHECK-NEXT:  }
-
-define void @test_ptr_insts_replaced() {
-  %ptr1 = ptrtoint void (i32)* @fn_v_i_2 to i32
-  %ptr2 = ptrtoint i32 (i32, i32)* @fn_i_ii to i32
-  call void @foo_2ptr(i32 %ptr1, i32 %ptr2)
-  ret void
-}
-
-; CHECK-LABEL: define void @test_ptr_insts_replaced() {
-; CHECK-NEXT:    call void @foo_2ptr(i32 4, i32 3)
-; CHECK-NEXT:    ret void
-; CHECK-NEXT:  }
-
-define void @test_indirect_calls(i32 %index_v_i, i32 %index_i_ii) {
-  %fn_v_i = inttoptr i32 %index_v_i to void (i32)*, !dbg !1
-  call void %fn_v_i(i32 7), !dbg !3
-  call void %fn_v_i(i32 9), !dbg !4
-  %fn_i_ii = inttoptr i32 %index_i_ii to i32 (i32, i32)*, !dbg !5
-  call i32 %fn_i_ii(i32 11, i32 13), !dbg !6
-  ret void
-}
-
-; CHECK-LABEL: define void @test_indirect_calls(i32 %index_v_i, i32 %index_i_ii) {
-; CHECK-NEXT:    %1 = and i32 %index_v_i, 7
-; CHECK-NEXT:    %2 = getelementptr [8 x void (i32)*], [8 x void (i32)*]* [[TAB_V_I]], i32 0, i32 %1
-; CHECK-NEXT:    %3 = load void (i32)*, void (i32)** %2, !dbg !1
-; CHECK-NEXT:    call void %3(i32 7), !dbg !3
-; CHECK-NEXT:    %4 = and i32 %index_v_i, 7
-; CHECK-NEXT:    %5 = getelementptr [8 x void (i32)*], [8 x void (i32)*]* [[TAB_V_I]], i32 0, i32 %4
-; CHECK-NEXT:    %6 = load void (i32)*, void (i32)** %5, !dbg !1
-; CHECK-NEXT:    call void %6(i32 9), !dbg !4
-; CHECK-NEXT:    %7 = and i32 %index_i_ii, 7
-; CHECK-NEXT:    %8 = getelementptr [8 x i32 (i32, i32)*], [8 x i32 (i32, i32)*]* [[TAB_I_II]], i32 0, i32 %7
-; CHECK-NEXT:    %9 = load i32 (i32, i32)*, i32 (i32, i32)** %8, !dbg !5
-; CHECK-NEXT:    call i32 %9(i32 11, i32 13), !dbg !6
-; CHECK-NEXT:    ret void
-; CHECK-NEXT:  }
-
-define float @test_call_without_a_table(i32 %index) {
-  %fn = inttoptr i32 %index to float (float)*
-  %ret = call float %fn(float 0.000000e+00)
-  ret float %ret
-}
-
-; CHECK-LABEL: define float @test_call_without_a_table(i32 %index) {
-; CHECK-NEXT:    call void @llvm.trap()
-; CHECK-NEXT:    %ret = call float null(float 0.000000e+00)
-; CHECK-NEXT:    ret float %ret
-; CHECK-NEXT:  }
-
-
-!1 = !MDLocation(line: 1, column: 13, scope: !2)
-!2 = !MDSubprogram(name: "foo")
-!3 = !MDLocation(line: 2, column: 10, scope: !2)
-!4 = !MDLocation(line: 2, column: 3, scope: !2)
-!5 = !MDLocation(line: 23, column: 3, scope: !2)
-!6 = !MDLocation(line: 234, column: 3, scope: !2)
diff --git a/test/Transforms/MinSFI/sandbox-memory-accesses-gep.ll b/test/Transforms/MinSFI/sandbox-memory-accesses-gep.ll
deleted file mode 100644
index 2532ac3d592..00000000000
--- a/test/Transforms/MinSFI/sandbox-memory-accesses-gep.ll
+++ /dev/null
@@ -1,141 +0,0 @@
-; RUN: opt %s -expand-getelementptr -replace-ptrs-with-ints \
-; RUN:        -minsfi-sandbox-memory-accesses -S \
-; RUN:   | FileCheck %s -check-prefix=CHECK-GEP
-; RUN: opt %s -expand-getelementptr -replace-ptrs-with-ints \
-; RUN:        -minsfi-sandbox-memory-accesses -minsfi-ptrsize=20 -S \
-; RUN:   | FileCheck %s -check-prefix=CHECK-GEP-MASK
-; RUN: opt %s -expand-getelementptr -minsfi-sandbox-memory-accesses \
-; RUN:        -minsfi-ptrsize=20 -S \ 
-; RUN:   | FileCheck %s
-
-target datalayout = "p:32:32:32"
-target triple = "le32-unknown-nacl"
-
-declare void @llvm.memcpy.p0i8.p0i8.i32(i8* nocapture, i8* nocapture readonly, i32, i32, i1)
-declare void @llvm.memmove.p0i8.p0i8.i32(i8* nocapture, i8* nocapture readonly, i32, i32, i1)
-declare void @llvm.memset.p0i8.i32(i8* nocapture, i8, i32, i32, i1)
-
-; This test verifies that the pass recognizes the pointer arithmetic pattern
-; produced by the ExpandGetElementPtr pass and that it emits a more efficient
-; address sandboxing than in the general case.
-
-define i32 @test_load_elementptr([100 x i32]* %foo) {
-  %elem = getelementptr inbounds [100 x i32], [100 x i32]* %foo, i32 0, i32 97
-  %val = load i32, i32* %elem
-  ret i32 %val
-}
-
-; CHECK-GEP-LABEL: define i32 @test_load_elementptr(i32 %foo) {
-; CHECK-GEP-NEXT:    %mem_base = load i64, i64* @__sfi_memory_base
-; CHECK-GEP-NEXT:    %1 = zext i32 %foo to i64
-; CHECK-GEP-NEXT:    %2 = add i64 %mem_base, %1
-; CHECK-GEP-NEXT:    %3 = add i64 %2, 388
-; CHECK-GEP-NEXT:    %4 = inttoptr i64 %3 to i32*
-; CHECK-GEP-NEXT:    %val = load i32, i32* %4
-; CHECK-GEP-NEXT:    ret i32 %val
-; CHECK-GEP-NEXT:  }
-
-; CHECK-GEP-MASK-LABEL: define i32 @test_load_elementptr(i32 %foo) {
-; CHECK-GEP-MASK-NEXT:    %mem_base = load i64, i64* @__sfi_memory_base
-; CHECK-GEP-MASK-NEXT:    %1 = and i32 %foo, 1048575
-; CHECK-GEP-MASK-NEXT:    %2 = zext i32 %1 to i64
-; CHECK-GEP-MASK-NEXT:    %3 = add i64 %mem_base, %2
-; CHECK-GEP-MASK-NEXT:    %4 = add i64 %3, 388
-; CHECK-GEP-MASK-NEXT:    %5 = inttoptr i64 %4 to i32*
-; CHECK-GEP-MASK-NEXT:    %val = load i32, i32* %5
-; CHECK-GEP-MASK-NEXT:    ret i32 %val
-; CHECK-GEP-MASK-NEXT:  }
-
-define <4 x float> @test_max_offset(i32 %x) {
-  %1 = add i32 %x, 1048560  ; 1MB - 16B
-  %ptr = inttoptr i32 %1 to <4 x float>*
-  %val = load <4 x float>, <4 x float>* %ptr
-  ret <4 x float> %val
-}
-
-; CHECK-LABEL: define <4 x float> @test_max_offset(i32 %x) {
-; CHECK-NEXT:    %mem_base = load i64, i64* @__sfi_memory_base
-; CHECK-NEXT:    %1 = and i32 %x, 1048575
-; CHECK-NEXT:    %2 = zext i32 %1 to i64
-; CHECK-NEXT:    %3 = add i64 %mem_base, %2
-; CHECK-NEXT:    %4 = add i64 %3, 1048560
-; CHECK-NEXT:    %5 = inttoptr i64 %4 to <4 x float>*
-; CHECK-NEXT:    %val = load <4 x float>, <4 x float>* %5
-; CHECK-NEXT:    ret <4 x float> %val
-; CHECK-NEXT:  }
-
-; This will not get optimized as it could access memory past the guard region.
-define <4 x float> @test_offset_overflow(i32 %x) {
-  %1 = add i32 %x, 1048561
-  %ptr = inttoptr i32 %1 to <4 x float>*
-  %val = load <4 x float>, <4 x float>* %ptr
-  ret <4 x float> %val
-}
-
-; CHECK-LABEL: define <4 x float> @test_offset_overflow(i32 %x) {
-; CHECK-NEXT:    %mem_base = load i64, i64* @__sfi_memory_base
-; CHECK-NEXT:    %1 = add i32 %x, 1048561
-; CHECK-NEXT:    %ptr = inttoptr i32 %1 to <4 x float>*
-; CHECK-NEXT:    %2 = ptrtoint <4 x float>* %ptr to i32
-; CHECK-NEXT:    %3 = and i32 %2, 1048575
-; CHECK-NEXT:    %4 = zext i32 %3 to i64
-; CHECK-NEXT:    %5 = add i64 %mem_base, %4
-; CHECK-NEXT:    %6 = inttoptr i64 %5 to <4 x float>*
-; CHECK-NEXT:    %val = load <4 x float>, <4 x float>* %6
-; CHECK-NEXT:    ret <4 x float> %val
-; CHECK-NEXT:  }
-
-define void @test_not_applied_on_memcpy(i32 %x) {
-  %1 = add i32 %x, 1024
-  %ptr = inttoptr i32 %1 to i8*
-  call void @llvm.memcpy.p0i8.p0i8.i32(i8* %ptr, i8* %ptr, i32 2048, i32 4, i1 false);
-  ret void
-}
-
-; CHECK-LABEL: define void @test_not_applied_on_memcpy(i32 %x) {
-; CHECK:         [[IPTR1:%[0-9]+]] = ptrtoint i8* %ptr to i32 
-; CHECK-NEXT:    [[AND1:%[0-9]+]] = and i32 [[IPTR1]], 1048575
-; CHECK-NEXT:    [[ZEXT1:%[0-9]+]] = zext i32 [[AND1]] to i64
-; CHECK-NEXT:    [[BASE1:%[0-9]+]] = add i64 %mem_base, [[ZEXT1]]
-; CHECK-NEXT:    inttoptr i64 [[BASE1]] to i8*
-; CHECK:         [[IPTR2:%[0-9]+]] = ptrtoint i8* %ptr to i32 
-; CHECK-NEXT:    [[AND2:%[0-9]+]] = and i32 [[IPTR2]], 1048575
-; CHECK-NEXT:    [[ZEXT2:%[0-9]+]] = zext i32 [[AND2]] to i64
-; CHECK-NEXT:    [[BASE2:%[0-9]+]] = add i64 %mem_base, [[ZEXT2]]
-; CHECK-NEXT:    inttoptr i64 [[BASE2]] to i8*
-; CHECK:         call void @llvm.memcpy.p0i8.p0i8.i32
-
-define void @test_not_applied_on_memmove(i32 %x) {
-  %1 = add i32 %x, 1024
-  %ptr = inttoptr i32 %1 to i8*
-  call void @llvm.memmove.p0i8.p0i8.i32(i8* %ptr, i8* %ptr, i32 2048, i32 4, i1 false);
-  ret void
-}
-
-; CHECK-LABEL: define void @test_not_applied_on_memmove(i32 %x) {
-; CHECK:         [[IPTR1:%[0-9]+]] = ptrtoint i8* %ptr to i32 
-; CHECK-NEXT:    [[AND1:%[0-9]+]] = and i32 [[IPTR1]], 1048575
-; CHECK-NEXT:    [[ZEXT1:%[0-9]+]] = zext i32 [[AND1]] to i64
-; CHECK-NEXT:    [[BASE1:%[0-9]+]] = add i64 %mem_base, [[ZEXT1]]
-; CHECK-NEXT:    inttoptr i64 [[BASE1]] to i8*
-; CHECK:         [[IPTR2:%[0-9]+]] = ptrtoint i8* %ptr to i32 
-; CHECK-NEXT:    [[AND2:%[0-9]+]] = and i32 [[IPTR2]], 1048575
-; CHECK-NEXT:    [[ZEXT2:%[0-9]+]] = zext i32 [[AND2]] to i64
-; CHECK-NEXT:    [[BASE2:%[0-9]+]] = add i64 %mem_base, [[ZEXT2]]
-; CHECK-NEXT:    inttoptr i64 [[BASE2]] to i8*
-; CHECK:         call void @llvm.memmove.p0i8.p0i8.i32
-
-define void @test_not_applied_on_memset(i32 %x) {
-  %1 = add i32 %x, 1024
-  %ptr = inttoptr i32 %1 to i8*
-  call void @llvm.memset.p0i8.i32(i8* %ptr, i8 3, i32 2048, i32 4, i1 false);
-  ret void
-}
-
-; CHECK-LABEL: define void @test_not_applied_on_memset(i32 %x) {
-; CHECK:         [[IPTR:%[0-9]+]] = ptrtoint i8* %ptr to i32 
-; CHECK-NEXT:    [[AND:%[0-9]+]] = and i32 [[IPTR]], 1048575
-; CHECK-NEXT:    [[ZEXT:%[0-9]+]] = zext i32 [[AND]] to i64
-; CHECK-NEXT:    [[BASE:%[0-9]+]] = add i64 %mem_base, [[ZEXT]]
-; CHECK-NEXT:    inttoptr i64 [[BASE]] to i8*
-; CHECK:         call void @llvm.memset.p0i8.i32
diff --git a/test/Transforms/MinSFI/sandbox-memory-accesses-unhandled-instruction.ll b/test/Transforms/MinSFI/sandbox-memory-accesses-unhandled-instruction.ll
deleted file mode 100644
index e08baa6aa76..00000000000
--- a/test/Transforms/MinSFI/sandbox-memory-accesses-unhandled-instruction.ll
+++ /dev/null
@@ -1,14 +0,0 @@
-; RUN: not opt %s -minsfi-sandbox-memory-accesses -S 2>&1 | FileCheck %s
-
-; The SandboxMemoryAccess pass should fail if it encounters an unexpected 
-; instruction such as this 'atomicrmw'. This mechanism protects MinSFI 
-; from unsafe operations which it does not handle appearing in the bitcode. 
-; This could be a result of a bug in the compiler or a newly introduced
-; LLVM instruction.
-
-define i32 @test_unhandled_instr(i32* %ptr) {
-  %old = atomicrmw add i32* %ptr, i32 1 acquire
-  ret i32 %old
-}
-
-; CHECK: LLVM ERROR: SandboxMemoryAccesses: unexpected instruction with pointer-type operands
diff --git a/test/Transforms/MinSFI/sandbox-memory-accesses-unhandled-intrinsic.ll b/test/Transforms/MinSFI/sandbox-memory-accesses-unhandled-intrinsic.ll
deleted file mode 100644
index b8a23046cdf..00000000000
--- a/test/Transforms/MinSFI/sandbox-memory-accesses-unhandled-intrinsic.ll
+++ /dev/null
@@ -1,16 +0,0 @@
-; RUN: not opt %s -minsfi-sandbox-memory-accesses -S 2>&1 | FileCheck %s
-
-; The SandboxMemoryAccess pass should fail if it encounters an unexpected 
-; intrinsic such as this '@llvm.objectsize'. This mechanism protects MinSFI 
-; from unsafe operations it does not handle appearing in the bitcode. 
-; This could be a result of a bug in the compiler or a newly introduced
-; LLVM instruction.
-
-declare i32 @llvm.objectsize.i32(i8*, i1)
-
-define i32 @test_unhandled_intrinsic(i8* %ptr) {
-  %val = call i32 @llvm.objectsize.i32(i8* %ptr, i1 true)
-  ret i32 %val
-}
-
-; CHECK: LLVM ERROR: SandboxMemoryAccesses: unexpected instruction with pointer-type operands
diff --git a/test/Transforms/MinSFI/sandbox-memory-accesses.ll b/test/Transforms/MinSFI/sandbox-memory-accesses.ll
deleted file mode 100644
index f9082b0be04..00000000000
--- a/test/Transforms/MinSFI/sandbox-memory-accesses.ll
+++ /dev/null
@@ -1,520 +0,0 @@
-; RUN: opt %s -minsfi-sandbox-memory-accesses -S | FileCheck %s
-; RUN: opt %s -minsfi-ptrsize=20 -minsfi-sandbox-memory-accesses -S \
-; RUN: | FileCheck %s -check-prefix=CHECK-MASK
-
-!llvm.module.flags = !{!0}
-!0 = !{i32 1, !"Debug Info Version", i32 3}
-
-target datalayout = "p:32:32:32"
-target triple = "le32-unknown-nacl"
-
-; CHECK:  @__sfi_memory_base = external global i64
-; CHECK:  @__sfi_pointer_size = constant i32 32
-; CHECK-MASK:  @__sfi_pointer_size = constant i32 20
-
-declare void @llvm.memcpy.p0i8.p0i8.i32(i8* nocapture, i8* nocapture readonly, i32, i32, i1)
-declare void @llvm.memmove.p0i8.p0i8.i32(i8* nocapture, i8* nocapture readonly, i32, i32, i1)
-declare void @llvm.memset.p0i8.i32(i8* nocapture, i8, i32, i32, i1)
-
-declare i32 @llvm.nacl.atomic.load.i32(i32*, i32)
-declare void @llvm.nacl.atomic.store.i32(i32, i32*, i32)
-declare i32 @llvm.nacl.atomic.rmw.i32(i32, i32*, i32, i32)
-declare i32 @llvm.nacl.atomic.cmpxchg.i32(i32*, i32, i32, i32, i32)
-
-declare i64 @llvm.nacl.atomic.load.i64(i64*, i32)
-declare void @llvm.nacl.atomic.store.i64(i64, i64*, i32)
-declare i64 @llvm.nacl.atomic.rmw.i64(i32, i64*, i64, i32)
-declare i64 @llvm.nacl.atomic.cmpxchg.i64(i64*, i64, i64, i32, i32)
-
-declare void @llvm.nacl.atomic.fence(i32)
-declare void @llvm.nacl.atomic.fence.all()
-declare i1 @llvm.nacl.atomic.is.lock.free(i32, i8*)
-
-define i32 @test_no_sandbox(i32 %x, i32 %y) {
-  %sum = add i32 %x, %y
-  ret i32 %sum
-}
-
-; CHECK-LABEL: define i32 @test_no_sandbox(i32 %x, i32 %y) {
-; CHECK-NOT:     @__sfi_memory_base
-; CHECK-NEXT:    %sum = add i32 %x, %y
-; CHECK-NEXT:    ret i32 %sum
-; CHECK-NEXT:  }
-
-define i32 @test_load(i32* %ptr) {
-  %val = load i32, i32* %ptr
-  ret i32 %val
-}
-
-; CHECK-LABEL: define i32 @test_load(i32* %ptr) {
-; CHECK-NEXT:    %mem_base = load i64, i64* @__sfi_memory_base
-; CHECK-NEXT:    %1 = ptrtoint i32* %ptr to i32
-; CHECK-NEXT:    %2 = zext i32 %1 to i64
-; CHECK-NEXT:    %3 = add i64 %mem_base, %2
-; CHECK-NEXT:    %4 = inttoptr i64 %3 to i32* 
-; CHECK-NEXT:    %val = load i32, i32* %4
-; CHECK-NEXT:    ret i32 %val
-; CHECK-NEXT:  }
-
-; CHECK-MASK-LABEL: define i32 @test_load(i32* %ptr) {
-; CHECK-MASK-NEXT:    %mem_base = load i64, i64* @__sfi_memory_base
-; CHECK-MASK-NEXT:    %1 = ptrtoint i32* %ptr to i32
-; CHECK-MASK-NEXT:    %2 = and i32 %1, 1048575
-; CHECK-MASK-NEXT:    %3 = zext i32 %2 to i64
-; CHECK-MASK-NEXT:    %4 = add i64 %mem_base, %3
-; CHECK-MASK-NEXT:    %5 = inttoptr i64 %4 to i32* 
-; CHECK-MASK-NEXT:    %val = load i32, i32* %5
-; CHECK-MASK-NEXT:    ret i32 %val
-; CHECK-MASK-NEXT:  }
-
-define void @test_store(i32* %ptr) {
-  store i32 1234, i32* %ptr
-  ret void
-}
-
-; CHECK-LABEL: define void @test_store(i32* %ptr) {
-; CHECK-NEXT:    %mem_base = load i64, i64* @__sfi_memory_base
-; CHECK-NEXT:    %1 = ptrtoint i32* %ptr to i32
-; CHECK-NEXT:    %2 = zext i32 %1 to i64
-; CHECK-NEXT:    %3 = add i64 %mem_base, %2
-; CHECK-NEXT:    %4 = inttoptr i64 %3 to i32* 
-; CHECK-NEXT:    store i32 1234, i32* %4
-; CHECK-NEXT:    ret void
-; CHECK-NEXT:  }
-
-; CHECK-MASK-LABEL: define void @test_store(i32* %ptr) {
-; CHECK-MASK-NEXT:    %mem_base = load i64, i64* @__sfi_memory_base
-; CHECK-MASK-NEXT:    %1 = ptrtoint i32* %ptr to i32
-; CHECK-MASK-NEXT:    %2 = and i32 %1, 1048575
-; CHECK-MASK-NEXT:    %3 = zext i32 %2 to i64
-; CHECK-MASK-NEXT:    %4 = add i64 %mem_base, %3
-; CHECK-MASK-NEXT:    %5 = inttoptr i64 %4 to i32* 
-; CHECK-MASK-NEXT:    store i32 1234, i32* %5
-; CHECK-MASK-NEXT:    ret void
-; CHECK-MASK-NEXT:  }
-
-define void @test_memcpy_32(i8* %dest, i8* %src, i32 %len) {
-  call void @llvm.memcpy.p0i8.p0i8.i32(i8* %dest, i8* %src, i32 %len, i32 4, i1 false)
-  ret void
-}
-
-; CHECK-LABEL: define void @test_memcpy_32(i8* %dest, i8* %src, i32 %len) {
-; CHECK-NEXT:    %mem_base = load i64, i64* @__sfi_memory_base
-; CHECK-NEXT:    %1 = ptrtoint i8* %dest to i32
-; CHECK-NEXT:    %2 = zext i32 %1 to i64
-; CHECK-NEXT:    %3 = add i64 %mem_base, %2
-; CHECK-NEXT:    %4 = inttoptr i64 %3 to i8* 
-; CHECK-NEXT:    %5 = ptrtoint i8* %src to i32
-; CHECK-NEXT:    %6 = zext i32 %5 to i64
-; CHECK-NEXT:    %7 = add i64 %mem_base, %6
-; CHECK-NEXT:    %8 = inttoptr i64 %7 to i8* 
-; CHECK-NEXT:    call void @llvm.memcpy.p0i8.p0i8.i32(i8* %4, i8* %8, i32 %len, i32 4, i1 false)
-; CHECK-NEXT:    ret void
-; CHECK-NEXT:  }
-
-; CHECK-MASK-LABEL: define void @test_memcpy_32(i8* %dest, i8* %src, i32 %len) {
-; CHECK-MASK:         %11 = and i32 %len, 1048575
-; CHECK-MASK-NEXT:    call void @llvm.memcpy.p0i8.p0i8.i32(i8* %5, i8* %10, i32 %11, i32 4, i1 false)
-
-define void @test_memmove_32(i8* %dest, i8* %src, i32 %len) {
-  call void @llvm.memmove.p0i8.p0i8.i32(i8* %dest, i8* %src, i32 %len, i32 4, i1 false)
-  ret void
-}
-
-; CHECK-LABEL: define void @test_memmove_32(i8* %dest, i8* %src, i32 %len) {
-; CHECK-NEXT:    %mem_base = load i64, i64* @__sfi_memory_base
-; CHECK-NEXT:    %1 = ptrtoint i8* %dest to i32
-; CHECK-NEXT:    %2 = zext i32 %1 to i64
-; CHECK-NEXT:    %3 = add i64 %mem_base, %2
-; CHECK-NEXT:    %4 = inttoptr i64 %3 to i8* 
-; CHECK-NEXT:    %5 = ptrtoint i8* %src to i32
-; CHECK-NEXT:    %6 = zext i32 %5 to i64
-; CHECK-NEXT:    %7 = add i64 %mem_base, %6
-; CHECK-NEXT:    %8 = inttoptr i64 %7 to i8* 
-; CHECK-NEXT:    call void @llvm.memmove.p0i8.p0i8.i32(i8* %4, i8* %8, i32 %len, i32 4, i1 false)
-; CHECK-NEXT:    ret void
-; CHECK-NEXT:  }
-
-; CHECK-MASK-LABEL: define void @test_memmove_32(i8* %dest, i8* %src, i32 %len) {
-; CHECK-MASK:         %11 = and i32 %len, 1048575
-; CHECK-MASK-NEXT:    call void @llvm.memmove.p0i8.p0i8.i32(i8* %5, i8* %10, i32 %11, i32 4, i1 false)
-
-define void @test_memset_32(i8* %dest, i32 %len) {
-  call void @llvm.memset.p0i8.i32(i8* %dest, i8 5, i32 %len, i32 4, i1 false)
-  ret void
-}
-
-; CHECK-LABEL: define void @test_memset_32(i8* %dest, i32 %len) {
-; CHECK-NEXT:    %mem_base = load i64, i64* @__sfi_memory_base
-; CHECK-NEXT:    %1 = ptrtoint i8* %dest to i32
-; CHECK-NEXT:    %2 = zext i32 %1 to i64
-; CHECK-NEXT:    %3 = add i64 %mem_base, %2
-; CHECK-NEXT:    %4 = inttoptr i64 %3 to i8* 
-; CHECK-NEXT:    call void @llvm.memset.p0i8.i32(i8* %4, i8 5, i32 %len, i32 4, i1 false)
-; CHECK-NEXT:    ret void
-; CHECK-NEXT:  }
-
-; CHECK-MASK-LABEL: define void @test_memset_32(i8* %dest, i32 %len) {
-; CHECK-MASK:         %6 = and i32 %len, 1048575
-; CHECK-MASK-NEXT:    call void @llvm.memset.p0i8.i32(i8* %5, i8 5, i32 %6, i32 4, i1 false)
-
-define i32 @test_atomic_load_32(i32* %ptr) {
-  %val = call i32 @llvm.nacl.atomic.load.i32(i32* %ptr, i32 1)
-  ret i32 %val
-}
-
-; CHECK-LABEL: define i32 @test_atomic_load_32(i32* %ptr) {
-; CHECK-NEXT:    %mem_base = load i64, i64* @__sfi_memory_base
-; CHECK-NEXT:    %1 = ptrtoint i32* %ptr to i32
-; CHECK-NEXT:    %2 = zext i32 %1 to i64
-; CHECK-NEXT:    %3 = add i64 %mem_base, %2
-; CHECK-NEXT:    %4 = inttoptr i64 %3 to i32* 
-; CHECK-NEXT:    %val = call i32 @llvm.nacl.atomic.load.i32(i32* %4, i32 1)
-; CHECK-NEXT:    ret i32 %val
-; CHECK-NEXT:  }
-
-; CHECK-MASK-LABEL: define i32 @test_atomic_load_32(i32* %ptr) {
-; CHECK-MASK:         %2 = and i32 %1, 1048575
-
-define i64 @test_atomic_load_64(i64* %ptr) {
-  %val = call i64 @llvm.nacl.atomic.load.i64(i64* %ptr, i32 1)
-  ret i64 %val
-}
-
-; CHECK-LABEL: define i64 @test_atomic_load_64(i64* %ptr) {
-; CHECK-NEXT:    %mem_base = load i64, i64* @__sfi_memory_base
-; CHECK-NEXT:    %1 = ptrtoint i64* %ptr to i32
-; CHECK-NEXT:    %2 = zext i32 %1 to i64
-; CHECK-NEXT:    %3 = add i64 %mem_base, %2
-; CHECK-NEXT:    %4 = inttoptr i64 %3 to i64* 
-; CHECK-NEXT:    %val = call i64 @llvm.nacl.atomic.load.i64(i64* %4, i32 1)
-; CHECK-NEXT:    ret i64 %val
-; CHECK-NEXT:  }
-
-; CHECK-MASK-LABEL: define i64 @test_atomic_load_64(i64* %ptr) {
-; CHECK-MASK:         %2 = and i32 %1, 1048575
-
-define void @test_atomic_store_32(i32* %ptr) {
-  call void @llvm.nacl.atomic.store.i32(i32 1234, i32* %ptr, i32 1)
-  ret void
-}
-
-; CHECK-LABEL: define void @test_atomic_store_32(i32* %ptr) {
-; CHECK-NEXT:    %mem_base = load i64, i64* @__sfi_memory_base
-; CHECK-NEXT:    %1 = ptrtoint i32* %ptr to i32
-; CHECK-NEXT:    %2 = zext i32 %1 to i64
-; CHECK-NEXT:    %3 = add i64 %mem_base, %2
-; CHECK-NEXT:    %4 = inttoptr i64 %3 to i32* 
-; CHECK-NEXT:    call void @llvm.nacl.atomic.store.i32(i32 1234, i32* %4, i32 1)
-; CHECK-NEXT:    ret void
-; CHECK-NEXT:  }
-
-; CHECK-MASK-LABEL: define void @test_atomic_store_32(i32* %ptr) {
-; CHECK-MASK:         %2 = and i32 %1, 1048575
-
-define void @test_atomic_store_64(i64* %ptr) {
-  call void @llvm.nacl.atomic.store.i64(i64 1234, i64* %ptr, i32 1)
-  ret void
-}
-
-; CHECK-LABEL: define void @test_atomic_store_64(i64* %ptr) {
-; CHECK-NEXT:    %mem_base = load i64, i64* @__sfi_memory_base
-; CHECK-NEXT:    %1 = ptrtoint i64* %ptr to i32
-; CHECK-NEXT:    %2 = zext i32 %1 to i64
-; CHECK-NEXT:    %3 = add i64 %mem_base, %2
-; CHECK-NEXT:    %4 = inttoptr i64 %3 to i64* 
-; CHECK-NEXT:    call void @llvm.nacl.atomic.store.i64(i64 1234, i64* %4, i32 1)
-; CHECK-NEXT:    ret void
-; CHECK-NEXT:  }
-
-; CHECK-MASK-LABEL: define void @test_atomic_store_64(i64* %ptr) {
-; CHECK-MASK:         %2 = and i32 %1, 1048575
-
-define i32 @test_atomic_rmw_32(i32* %ptr) {
-  %val = call i32 @llvm.nacl.atomic.rmw.i32(i32 1, i32* %ptr, i32 1234, i32 1)
-  ret i32 %val
-}
-
-; CHECK-LABEL: define i32 @test_atomic_rmw_32(i32* %ptr) {
-; CHECK-NEXT:    %mem_base = load i64, i64* @__sfi_memory_base
-; CHECK-NEXT:    %1 = ptrtoint i32* %ptr to i32
-; CHECK-NEXT:    %2 = zext i32 %1 to i64
-; CHECK-NEXT:    %3 = add i64 %mem_base, %2
-; CHECK-NEXT:    %4 = inttoptr i64 %3 to i32* 
-; CHECK-NEXT:    %val = call i32 @llvm.nacl.atomic.rmw.i32(i32 1, i32* %4, i32 1234, i32 1)
-; CHECK-NEXT:    ret i32 %val
-; CHECK-NEXT:  }
-
-; CHECK-MASK-LABEL: define i32 @test_atomic_rmw_32(i32* %ptr) {
-; CHECK-MASK:         %2 = and i32 %1, 1048575
-
-define i64 @test_atomic_rmw_64(i64* %ptr) {
-  %val = call i64 @llvm.nacl.atomic.rmw.i64(i32 1, i64* %ptr, i64 1234, i32 1)
-  ret i64 %val
-}
-
-; CHECK-LABEL: define i64 @test_atomic_rmw_64(i64* %ptr) {
-; CHECK-NEXT:    %mem_base = load i64, i64* @__sfi_memory_base
-; CHECK-NEXT:    %1 = ptrtoint i64* %ptr to i32
-; CHECK-NEXT:    %2 = zext i32 %1 to i64
-; CHECK-NEXT:    %3 = add i64 %mem_base, %2
-; CHECK-NEXT:    %4 = inttoptr i64 %3 to i64* 
-; CHECK-NEXT:    %val = call i64 @llvm.nacl.atomic.rmw.i64(i32 1, i64* %4, i64 1234, i32 1)
-; CHECK-NEXT:    ret i64 %val
-; CHECK-NEXT:  }
-
-; CHECK-MASK-LABEL: define i64 @test_atomic_rmw_64(i64* %ptr) {
-; CHECK-MASK:         %2 = and i32 %1, 1048575
-
-define i32 @test_atomic_cmpxchg_32(i32* %ptr) {
-  %val = call i32 @llvm.nacl.atomic.cmpxchg.i32(i32* %ptr, i32 0, i32 1, i32 1, i32 1)
-  ret i32 %val
-}
-
-; CHECK-LABEL: define i32 @test_atomic_cmpxchg_32(i32* %ptr) {
-; CHECK-NEXT:    %mem_base = load i64, i64* @__sfi_memory_base
-; CHECK-NEXT:    %1 = ptrtoint i32* %ptr to i32
-; CHECK-NEXT:    %2 = zext i32 %1 to i64
-; CHECK-NEXT:    %3 = add i64 %mem_base, %2
-; CHECK-NEXT:    %4 = inttoptr i64 %3 to i32*
-; CHECK-NEXT:    %val = call i32 @llvm.nacl.atomic.cmpxchg.i32(i32* %4, i32 0, i32 1, i32 1, i32 1)
-; CHECK-NEXT:    ret i32 %val
-; CHECK-NEXT:  }
-
-; CHECK-MASK-LABEL: define i32 @test_atomic_cmpxchg_32(i32* %ptr) {
-; CHECK-MASK:         %2 = and i32 %1, 1048575
-
-define i64 @test_atomic_cmpxchg_64(i64* %ptr) {
-  %val = call i64 @llvm.nacl.atomic.cmpxchg.i64(i64* %ptr, i64 0, i64 1, i32 1, i32 1)
-  ret i64 %val
-}
-
-; CHECK-LABEL: define i64 @test_atomic_cmpxchg_64(i64* %ptr) {
-; CHECK-NEXT:    %mem_base = load i64, i64* @__sfi_memory_base
-; CHECK-NEXT:    %1 = ptrtoint i64* %ptr to i32
-; CHECK-NEXT:    %2 = zext i32 %1 to i64
-; CHECK-NEXT:    %3 = add i64 %mem_base, %2
-; CHECK-NEXT:    %4 = inttoptr i64 %3 to i64*
-; CHECK-NEXT:    %val = call i64 @llvm.nacl.atomic.cmpxchg.i64(i64* %4, i64 0, i64 1, i32 1, i32 1)
-; CHECK-NEXT:    ret i64 %val
-; CHECK-NEXT:  }
-
-; CHECK-MASK-LABEL: define i64 @test_atomic_cmpxchg_64(i64* %ptr) {
-; CHECK-MASK:         %2 = and i32 %1, 1048575
-
-define void @test_atomic_fence() {
-  call void @llvm.nacl.atomic.fence(i32 1)
-  ret void
-}
-
-; CHECK-LABEL: define void @test_atomic_fence() {
-; CHECK-NEXT:    call void @llvm.nacl.atomic.fence(i32 1)
-; CHECK-NEXT:    ret void
-; CHECK-NEXT:  }
-
-define void @test_atomic_fence_all() {
-  call void @llvm.nacl.atomic.fence.all()
-  ret void
-}
-
-; CHECK-LABEL: define void @test_atomic_fence_all() {
-; CHECK-NEXT:    call void @llvm.nacl.atomic.fence.all()
-; CHECK-NEXT:    ret void
-; CHECK-NEXT:  }
-
-define i1 @test_atomic_is_lock_free(i8* %ptr) {
-  %val = call i1 @llvm.nacl.atomic.is.lock.free(i32 4, i8* %ptr)
-  ret i1 %val
-}
-
-; CHECK-LABEL: define i1 @test_atomic_is_lock_free(i8* %ptr) {
-; CHECK-NEXT:    %mem_base = load i64, i64* @__sfi_memory_base
-; CHECK-NEXT:    %1 = ptrtoint i8* %ptr to i32
-; CHECK-NEXT:    %2 = zext i32 %1 to i64
-; CHECK-NEXT:    %3 = add i64 %mem_base, %2
-; CHECK-NEXT:    %4 = inttoptr i64 %3 to i8*
-; CHECK-NEXT:    %val = call i1 @llvm.nacl.atomic.is.lock.free(i32 4, i8* %4)
-; CHECK-NEXT:    ret i1 %val
-; CHECK-NEXT:  }
-
-define void @test_bitcast_whitelisted(i32 %val) {
-  %ptr = inttoptr i32 %val to i8*
-  %ptr.bc = bitcast i8* %ptr to i32*
-  ret void
-}
-
-; CHECK-LABEL: define void @test_bitcast_whitelisted(i32 %val) {
-; CHECK-NEXT:    %ptr = inttoptr i32 %val to i8*
-; CHECK-NEXT:    %ptr.bc = bitcast i8* %ptr to i32*
-; CHECK-NEXT:    ret void
-; CHECK-NEXT:  }
-
-; -----------------------------------------------------------------------------
-; Test the special case which optimizes sandboxing of the output of
-; the ExpandGetElementPtr pass. 
-
-; this won't get optimized because IntToPtr is not casting a result of an Add  
-define i32 @test_no_opt__cast_not_add(i32 %ptr_int) {
-  %ptr = inttoptr i32 %ptr_int to i32*
-  %val = load i32, i32* %ptr
-  ret i32 %val
-}
-
-; CHECK-LABEL: define i32 @test_no_opt__cast_not_add(i32 %ptr_int) {
-; CHECK-NEXT:    %mem_base = load i64, i64* @__sfi_memory_base
-; CHECK-NEXT:    %ptr = inttoptr i32 %ptr_int to i32*
-; CHECK-NEXT:    %1 = ptrtoint i32* %ptr to i32
-; CHECK-NEXT:    %2 = zext i32 %1 to i64
-; CHECK-NEXT:    %3 = add i64 %mem_base, %2
-; CHECK-NEXT:    %4 = inttoptr i64 %3 to i32*
-; CHECK-NEXT:    %val = load i32, i32* %4
-; CHECK-NEXT:    ret i32 %val
-; CHECK-NEXT:  }
-
-; this won't get optimized because the cast is not from i32 
-define i32 @test_no_opt__cast_not_32(i64 %ptr_int1) {
-  %ptr_sum = add i64 %ptr_int1, 5
-  %ptr = inttoptr i64 %ptr_sum to i32*
-  %val = load i32, i32* %ptr
-  ret i32 %val
-}
-
-; CHECK-LABEL: define i32 @test_no_opt__cast_not_32(i64 %ptr_int1) {
-; CHECK-NEXT:    %mem_base = load i64, i64* @__sfi_memory_base
-; CHECK-NEXT:    %ptr_sum = add i64 %ptr_int1, 5
-; CHECK-NEXT:    %ptr = inttoptr i64 %ptr_sum to i32*
-; CHECK-NEXT:    %1 = ptrtoint i32* %ptr to i32
-; CHECK-NEXT:    %2 = zext i32 %1 to i64
-; CHECK-NEXT:    %3 = add i64 %mem_base, %2
-; CHECK-NEXT:    %4 = inttoptr i64 %3 to i32*
-; CHECK-NEXT:    %val = load i32, i32* %4
-; CHECK-NEXT:    ret i32 %val
-; CHECK-NEXT:  }
-
-; this won't get optimized because the Add's 2nd operand is not a constant  
-define i32 @test_no_opt__add_not_constant(i32 %ptr_int1, i32 %ptr_int2) {
-  %ptr_sum = add i32 %ptr_int1, %ptr_int2  
-  %ptr = inttoptr i32 %ptr_sum to i32*
-  %val = load i32, i32* %ptr
-  ret i32 %val
-}
-
-; CHECK-LABEL: define i32 @test_no_opt__add_not_constant(i32 %ptr_int1, i32 %ptr_int2) {
-; CHECK-NEXT:    %mem_base = load i64, i64* @__sfi_memory_base
-; CHECK-NEXT:    %ptr_sum = add i32 %ptr_int1, %ptr_int2  
-; CHECK-NEXT:    %ptr = inttoptr i32 %ptr_sum to i32*
-; CHECK-NEXT:    %1 = ptrtoint i32* %ptr to i32
-; CHECK-NEXT:    %2 = zext i32 %1 to i64
-; CHECK-NEXT:    %3 = add i64 %mem_base, %2
-; CHECK-NEXT:    %4 = inttoptr i64 %3 to i32*
-; CHECK-NEXT:    %val = load i32, i32* %4
-; CHECK-NEXT:    ret i32 %val
-; CHECK-NEXT:  }
-
-; this won't get optimized because the Add's 2nd operand is not positive
-define i32 @test_no_opt__add_not_positive(i32 %ptr_int) {
-  %ptr_sum = add i32 %ptr_int, -5  
-  %ptr = inttoptr i32 %ptr_sum to i32*
-  %val = load i32, i32* %ptr
-  ret i32 %val
-}
-
-; CHECK-LABEL: define i32 @test_no_opt__add_not_positive(i32 %ptr_int) {
-; CHECK-NEXT:    %mem_base = load i64, i64* @__sfi_memory_base
-; CHECK-NEXT:    %ptr_sum = add i32 %ptr_int, -5  
-; CHECK-NEXT:    %ptr = inttoptr i32 %ptr_sum to i32*
-; CHECK-NEXT:    %1 = ptrtoint i32* %ptr to i32
-; CHECK-NEXT:    %2 = zext i32 %1 to i64
-; CHECK-NEXT:    %3 = add i64 %mem_base, %2
-; CHECK-NEXT:    %4 = inttoptr i64 %3 to i32*
-; CHECK-NEXT:    %val = load i32, i32* %4
-; CHECK-NEXT:    ret i32 %val
-; CHECK-NEXT:  }
-
-define i32 @test_opt_dont_remove_cast_if_used(i32 %ptr_int, i32 %replace) {
-  %ptr_sum = add i32 %ptr_int, 5  
-  %ptr = inttoptr i32 %ptr_sum to i32*
-  %val = load i32, i32* %ptr             ; %ptr is used later => keep cast
-  store i32 %replace, i32* %ptr     ; %ptr not used any more => remove cast
-  ret i32 %val
-}
-
-; CHECK-LABEL: define i32 @test_opt_dont_remove_cast_if_used(i32 %ptr_int, i32 %replace) {
-; CHECK-NEXT:    %mem_base = load i64, i64* @__sfi_memory_base
-; CHECK-NEXT:    %1 = zext i32 %ptr_int to i64
-; CHECK-NEXT:    %2 = add i64 %mem_base, %1
-; CHECK-NEXT:    %3 = add i64 %2, 5
-; CHECK-NEXT:    %4 = inttoptr i64 %3 to i32*
-; CHECK-NEXT:    %val = load i32, i32* %4
-; CHECK-NEXT:    %5 = zext i32 %ptr_int to i64
-; CHECK-NEXT:    %6 = add i64 %mem_base, %5
-; CHECK-NEXT:    %7 = add i64 %6, 5
-; CHECK-NEXT:    %8 = inttoptr i64 %7 to i32*
-; CHECK-NEXT:    store i32 %replace, i32* %8
-; CHECK-NEXT:    ret i32 %val
-; CHECK-NEXT:  }
-
-define i32 @test_opt_dont_remove_add_if_used(i32 %ptr_int, i32 %replace) {
-  %ptr_sum = add i32 %ptr_int, 5  
-  %ptr = inttoptr i32 %ptr_sum to i32*
-  store i32 %replace, i32* %ptr
-  ret i32 %ptr_sum
-}
-
-; CHECK-LABEL: define i32 @test_opt_dont_remove_add_if_used(i32 %ptr_int, i32 %replace) {
-; CHECK-NEXT:    %mem_base = load i64, i64* @__sfi_memory_base
-; CHECK-NEXT:    %ptr_sum = add i32 %ptr_int, 5  
-; CHECK-NEXT:    %1 = zext i32 %ptr_int to i64
-; CHECK-NEXT:    %2 = add i64 %mem_base, %1
-; CHECK-NEXT:    %3 = add i64 %2, 5
-; CHECK-NEXT:    %4 = inttoptr i64 %3 to i32*
-; CHECK-NEXT:    store i32 %replace, i32* %4
-; CHECK-NEXT:    ret i32 %ptr_sum
-; CHECK-NEXT:  }
-
-
-; ------------------------------------------------------------------------------
-; Check that dbg symbols are preserved
-
-define void @test_len_dbg(i8* %dest, i8* %src, i32 %len) {
-  call void @llvm.memcpy.p0i8.p0i8.i32(i8* %dest, i8* %src, i32 %len, i32 4, i1 false), !dbg !1
-  ret void
-}
-
-; CHECK-LABEL: define void @test_len_dbg(i8* %dest, i8* %src, i32 %len) {
-; CHECK-NEXT:    %mem_base = load i64, i64* @__sfi_memory_base
-; CHECK-NEXT:    %1 = ptrtoint i8* %dest to i32
-; CHECK-NEXT:    %2 = zext i32 %1 to i64
-; CHECK-NEXT:    %3 = add i64 %mem_base, %2
-; CHECK-NEXT:    %4 = inttoptr i64 %3 to i8*
-; CHECK-NEXT:    %5 = ptrtoint i8* %src to i32
-; CHECK-NEXT:    %6 = zext i32 %5 to i64
-; CHECK-NEXT:    %7 = add i64 %mem_base, %6
-; CHECK-NEXT:    %8 = inttoptr i64 %7 to i8*
-; CHECK-NEXT:    call void @llvm.memcpy.p0i8.p0i8.i32(i8* %4, i8* %8, i32 %len, i32 4, i1 false), !dbg !1
-; CHECK-NEXT:    ret void
-; CHECK-NEXT:  }
-
-define void @test_opt_dbg(i32 %ptr_int, i32 %replace) {
-  %ptr_sum = add i32 %ptr_int, 5, !dbg !1
-  %ptr = inttoptr i32 %ptr_sum to i32*, !dbg !3
-  store i32 %replace, i32* %ptr, !dbg !4
-  ret void, !dbg !5
-}
-
-; CHECK-LABEL: define void @test_opt_dbg(i32 %ptr_int, i32 %replace) {
-; CHECK-NEXT:    %mem_base = load i64, i64* @__sfi_memory_base
-; CHECK-NEXT:    %1 = zext i32 %ptr_int to i64
-; CHECK-NEXT:    %2 = add i64 %mem_base, %1
-; CHECK-NEXT:    %3 = add i64 %2, 5, !dbg !1
-; CHECK-NEXT:    %4 = inttoptr i64 %3 to i32*, !dbg !3
-; CHECK-NEXT:    store i32 %replace, i32* %4, !dbg !4
-; CHECK-NEXT:    ret void, !dbg !5
-; CHECK-NEXT:  }
-
-
-!1 = !MDLocation(line: 1, column: 13, scope: !2)
-!2 = !MDSubprogram(name: "foo")
-!3 = !MDLocation(line: 2, column: 10, scope: !2)
-!4 = !MDLocation(line: 2, column: 3, scope: !2)
-!5 = !MDLocation(line: 23, column: 3, scope: !2)
diff --git a/test/Transforms/MinSFI/strip-tls.ll b/test/Transforms/MinSFI/strip-tls.ll
deleted file mode 100644
index 41c33bd0985..00000000000
--- a/test/Transforms/MinSFI/strip-tls.ll
+++ /dev/null
@@ -1,23 +0,0 @@
-; RUN: opt %s -minsfi-strip-tls -S | FileCheck %s
-
-target datalayout = "p:32:32:32"
-target triple = "le32-unknown-nacl"
-
-@internal_shared = internal global i32 111
-@internal_thread_local = internal thread_local global i32 222 
-
-@external_shared = external global i64
-@external_thread_local = external thread_local global i64 
-
-@other_attr = thread_local addrspace(5) constant float 1.000000e+00, 
-              section "foo", align 4
-@tls_model = thread_local(initialexec) global i32 555
-
-; CHECK: @internal_shared = internal global i32 111
-; CHECK: @internal_thread_local = internal global i32 222 
-
-; CHECK: @external_shared = external global i64
-; CHECK: @external_thread_local = external global i64
-
-; CHECK: @other_attr = addrspace(5) constant float 1.000000e+00, section "foo", align 4
-; CHECK: @tls_model = global i32 555
diff --git a/test/Transforms/MinSFI/substitute-undefs.ll b/test/Transforms/MinSFI/substitute-undefs.ll
deleted file mode 100644
index dd1bc487ccf..00000000000
--- a/test/Transforms/MinSFI/substitute-undefs.ll
+++ /dev/null
@@ -1,98 +0,0 @@
-; RUN: opt %s -minsfi-substitute-undefs -S | FileCheck %s
-
-target datalayout = "p:32:32:32"
-target triple = "le32-unknown-nacl"
-
-define i1 @test_undef_i1() {
-  ret i1 undef                             ; replaced with 0x0
-}
-
-; CHECK-LABEL: define i1 @test_undef_i1() {
-; CHECK-NEXT:    ret i1 false
-; CHECK-NEXT:  }
-
-define i8 @test_undef_i8() {
-  ret i8 undef                             ; replaced with 0xBE
-}
-
-; CHECK-LABEL: define i8 @test_undef_i8() {
-; CHECK-NEXT:    ret i8 -66
-; CHECK-NEXT:  }
-
-define i64 @test_undef_i64() {
-  ret i64 undef                            ; replaced with 0xBAADF00DCAFEBABE
-}
-
-; CHECK-LABEL: define i64 @test_undef_i64() {
-; CHECK-NEXT:    ret i64 -4995072469653079362
-; CHECK-NEXT:  }
-
-define <8 x i16> @test_undef_i16vec() {
-  ret <8 x i16> undef                      ; replace with a vector of 0xBABE
-}
-
-; CHECK-LABEL: define <8 x i16> @test_undef_i16vec() {
-; CHECK-NEXT:    ret <8 x i16> <i16 -17730, i16 -17730, i16 -17730, i16 -17730, i16 -17730, i16 -17730, i16 -17730, i16 -17730>
-; CHECK-NEXT:  }
-
-define float @test_undef_float() {
-  ret float undef                          ; replaced with pi
-}
-
-; CHECK-LABEL: define float @test_undef_float() {
-; CHECK-NEXT:    ret float 0x400921FB60000000
-; CHECK-NEXT:  }
-
-define double @test_undef_double() {
-  ret double undef                         ; replaced with pi
-}
-
-; CHECK-LABEL: define double @test_undef_double() {
-; CHECK-NEXT:    ret double 0x400921FB54442EEA
-; CHECK-NEXT:  }
-
-define <4 x float> @test_undef_floatvec() {
-  ret <4 x float> undef                    ; replaced with a vector of pi
-}
-
-; CHECK-LABEL: define <4 x float> @test_undef_floatvec() {
-; CHECK-NEXT:    ret <4 x float> <float 0x400921FB60000000, float 0x400921FB60000000, float 0x400921FB60000000, float 0x400921FB60000000>
-; CHECK-NEXT:  }
-
-
-
-declare void @foo(i32, float)
-
-define void @test_more_operands() {
-  call void @foo(i32 undef, float undef)
-  ret void
-}
-
-; CHECK-LABEL: define void @test_more_operands() {
-; CHECK-NEXT:    call void @foo(i32 -889275714, float 0x400921FB60000000)
-; CHECK-NEXT:    ret void
-; CHECK-NEXT:  }
-
-define void @test_more_basicblocks() {
-entry:
-  br label %loop
-
-loop:
-  %i = phi i16 [ undef, %entry ], [ %next, %loop ]
-  %next = add i16 %i, 1
-  br i1 undef, label %exit, label %loop
-
-exit:
-  ret void
-}
-
-; CHECK-LABEL: define void @test_more_basicblocks() {
-; CHECK-NEXT:  entry:
-; CHECK-NEXT:    br label %loop
-; CHECK:       loop:
-; CHECK-NEXT:    %i = phi i16 [ -17730, %entry ], [ %next, %loop ]
-; CHECK-NEXT:    %next = add i16 %i, 1
-; CHECK-NEXT:    br i1 false, label %exit
-; CHECK:       exit:
-; CHECK-NEXT:    ret void
-; CHECK-NEXT:  }
diff --git a/test/Transforms/NaryReassociate/NVPTX/lit.local.cfg b/test/Transforms/NaryReassociate/NVPTX/lit.local.cfg
new file mode 100644
index 00000000000..2cb98eb371b
--- /dev/null
+++ b/test/Transforms/NaryReassociate/NVPTX/lit.local.cfg
@@ -0,0 +1,2 @@
+if not 'NVPTX' in config.root.targets:
+    config.unsupported = True
diff --git a/test/Transforms/NaryReassociate/NVPTX/nary-gep.ll b/test/Transforms/NaryReassociate/NVPTX/nary-gep.ll
new file mode 100644
index 00000000000..92fbd20d298
--- /dev/null
+++ b/test/Transforms/NaryReassociate/NVPTX/nary-gep.ll
@@ -0,0 +1,126 @@
+; RUN: opt < %s -nary-reassociate -early-cse -S | FileCheck %s
+
+target datalayout = "e-i64:64-v16:16-v32:32-n16:32:64"
+target triple = "nvptx64-unknown-unknown"
+
+declare void @foo(float*)
+
+; foo(&a[i]);
+; foo(&a[i + j]);
+;   =>
+; t = &a[i];
+; foo(t);
+; foo(t + j);
+define void @reassociate_gep(float* %a, i64 %i, i64 %j) {
+; CHECK-LABEL: @reassociate_gep(
+  %1 = add i64 %i, %j
+  %2 = getelementptr float, float* %a, i64 %i
+; CHECK: [[t1:[^ ]+]] = getelementptr float, float* %a, i64 %i
+  call void @foo(float* %2)
+; CHECK: call void @foo(float* [[t1]])
+  %3 = getelementptr float, float* %a, i64 %1
+; CHECK: [[t2:[^ ]+]] = getelementptr float, float* [[t1]], i64 %j
+  call void @foo(float* %3)
+; CHECK: call void @foo(float* [[t2]])
+  ret void
+}
+
+; foo(&a[sext(j)]);
+; foo(&a[sext(i +nsw j)]);
+; foo(&a[sext((i +nsw j) +nsw i)]);
+;   =>
+; t1 = &a[sext(j)];
+; foo(t1);
+; t2 = t1 + sext(i);
+; foo(t2);
+; t3 = t2 + sext(i); // sext(i) should be GVN'ed.
+; foo(t3);
+define void @reassociate_gep_nsw(float* %a, i32 %i, i32 %j) {
+; CHECK-LABEL: @reassociate_gep_nsw(
+  %idxprom.j = sext i32 %j to i64
+  %1 = getelementptr float, float* %a, i64 %idxprom.j
+; CHECK: [[t1:[^ ]+]] = getelementptr float, float* %a, i64 %idxprom.j
+  call void @foo(float* %1)
+; CHECK: call void @foo(float* [[t1]])
+
+  %2 = add nsw i32 %i, %j
+  %idxprom.2 = sext i32 %2 to i64
+  %3 = getelementptr float, float* %a, i64 %idxprom.2
+; CHECK: [[sexti:[^ ]+]] = sext i32 %i to i64
+; CHECK: [[t2:[^ ]+]] = getelementptr float, float* [[t1]], i64 [[sexti]]
+  call void @foo(float* %3)
+; CHECK: call void @foo(float* [[t2]])
+
+  %4 = add nsw i32 %2, %i
+  %idxprom.4 = sext i32 %4 to i64
+  %5 = getelementptr float, float* %a, i64 %idxprom.4
+; CHECK: [[t3:[^ ]+]] = getelementptr float, float* [[t2]], i64 [[sexti]]
+  call void @foo(float* %5)
+; CHECK: call void @foo(float* [[t3]])
+
+  ret void
+}
+
+; assume(j >= 0);
+; foo(&a[zext(j)]);
+; assume(i + j >= 0);
+; foo(&a[zext(i + j)]);
+;   =>
+; t1 = &a[zext(j)];
+; foo(t1);
+; t2 = t1 + sext(i);
+; foo(t2);
+define void @reassociate_gep_assume(float* %a, i32 %i, i32 %j) {
+; CHECK-LABEL: @reassociate_gep_assume(
+  ; assume(j >= 0)
+  %cmp = icmp sgt i32 %j, -1
+  call void @llvm.assume(i1 %cmp)
+  %1 = add i32 %i, %j
+  %cmp2 = icmp sgt i32 %1, -1
+  call void @llvm.assume(i1 %cmp2)
+
+  %idxprom.j = zext i32 %j to i64
+  %2 = getelementptr float, float* %a, i64 %idxprom.j
+; CHECK: [[t1:[^ ]+]] = getelementptr float, float* %a, i64 %idxprom.j
+  call void @foo(float* %2)
+; CHECK: call void @foo(float* [[t1]])
+
+  %idxprom.1 = zext i32 %1 to i64
+  %3 = getelementptr float, float* %a, i64 %idxprom.1
+; CHECK: [[sexti:[^ ]+]] = sext i32 %i to i64
+; CHECK: [[t2:[^ ]+]] = getelementptr float, float* [[t1]], i64 [[sexti]]
+  call void @foo(float* %3)
+; CHECK: call void @foo(float* [[t2]])
+
+  ret void
+}
+
+; Do not split the second GEP because sext(i + j) != sext(i) + sext(j).
+define void @reassociate_gep_no_nsw(float* %a, i32 %i, i32 %j) {
+; CHECK-LABEL: @reassociate_gep_no_nsw(
+  %1 = add i32 %i, %j
+  %2 = getelementptr float, float* %a, i32 %j
+; CHECK: getelementptr float, float* %a, i32 %j
+  call void @foo(float* %2)
+  %3 = getelementptr float, float* %a, i32 %1
+; CHECK: getelementptr float, float* %a, i32 %1
+  call void @foo(float* %3)
+  ret void
+}
+
+define void @reassociate_gep_128(float* %a, i128 %i, i128 %j) {
+; CHECK-LABEL: @reassociate_gep_128(
+  %1 = add i128 %i, %j
+  %2 = getelementptr float, float* %a, i128 %i
+; CHECK: [[t1:[^ ]+]] = getelementptr float, float* %a, i128 %i
+  call void @foo(float* %2)
+; CHECK: call void @foo(float* [[t1]])
+  %3 = getelementptr float, float* %a, i128 %1
+; CHECK: [[truncj:[^ ]+]] = trunc i128 %j to i64
+; CHECK: [[t2:[^ ]+]] = getelementptr float, float* [[t1]], i64 [[truncj]]
+  call void @foo(float* %3)
+; CHECK: call void @foo(float* [[t2]])
+  ret void
+}
+
+declare void @llvm.assume(i1)
diff --git a/test/Transforms/NaryReassociate/NVPTX/nary-slsr.ll b/test/Transforms/NaryReassociate/NVPTX/nary-slsr.ll
new file mode 100644
index 00000000000..2360fe7f1b7
--- /dev/null
+++ b/test/Transforms/NaryReassociate/NVPTX/nary-slsr.ll
@@ -0,0 +1,47 @@
+; RUN: opt < %s -slsr -nary-reassociate -S | FileCheck %s
+; RUN: llc < %s -march=nvptx64 -mcpu=sm_20 | FileCheck %s --check-prefix=PTX
+
+target datalayout = "e-i64:64-v16:16-v32:32-n16:32:64"
+
+; foo((a + b) + c);
+; foo((a + b * 2) + c);
+; foo((a + b * 3) + c);
+;   =>
+; abc = (a + b) + c;
+; foo(abc);
+; ab2c = abc + b;
+; foo(ab2c);
+; ab3c = ab2c + b;
+; foo(ab3c);
+define void @nary_reassociate_after_slsr(i32 %a, i32 %b, i32 %c) {
+; CHECK-LABEL: @nary_reassociate_after_slsr(
+; PTX-LABEL: .visible .func nary_reassociate_after_slsr(
+; PTX: ld.param.u32 [[b:%r[0-9]+]], [nary_reassociate_after_slsr_param_1];
+  %ab = add i32 %a, %b
+  %abc = add i32 %ab, %c
+  call void @foo(i32 %abc)
+; CHECK: call void @foo(i32 %abc)
+; PTX: st.param.b32 [param0+0], [[abc:%r[0-9]+]];
+
+  %b2 = shl i32 %b, 1
+  %ab2 = add i32 %a, %b2
+  %ab2c = add i32 %ab2, %c
+; CHECK-NEXT: %ab2c = add i32 %abc, %b
+; PTX: add.s32 [[ab2c:%r[0-9]+]], [[abc]], [[b]]
+  call void @foo(i32 %ab2c)
+; CHECK-NEXT: call void @foo(i32 %ab2c)
+; PTX: st.param.b32 [param0+0], [[ab2c]];
+
+  %b3 = mul i32 %b, 3
+  %ab3 = add i32 %a, %b3
+  %ab3c = add i32 %ab3, %c
+; CHECK-NEXT: %ab3c = add i32 %ab2c, %b
+; PTX: add.s32 [[ab3c:%r[0-9]+]], [[ab2c]], [[b]]
+  call void @foo(i32 %ab3c)
+; CHECK-NEXT: call void @foo(i32 %ab3c)
+; PTX: st.param.b32 [param0+0], [[ab3c]];
+
+  ret void
+}
+
+declare void @foo(i32)
diff --git a/test/Transforms/NaryReassociate/nary-add.ll b/test/Transforms/NaryReassociate/nary-add.ll
index 39d7c59ef9d..b3093ff6ecd 100644
--- a/test/Transforms/NaryReassociate/nary-add.ll
+++ b/test/Transforms/NaryReassociate/nary-add.ll
@@ -196,3 +196,14 @@ define void @iterative(i32 %a, i32 %b, i32 %c) {
 
   ret void
 }
+
+define void @avoid_infinite_loop(i32 %a, i32 %b) {
+; CHECK-LABEL: @avoid_infinite_loop
+  %ab = add i32 %a, %b
+; CHECK-NEXT: %ab
+  %ab2 = add i32 %ab, %b
+; CHECK-NEXT: %ab2
+  call void @foo(i32 %ab2)
+; CHECK-NEXT: @foo(i32 %ab2)
+  ret void
+}
diff --git a/test/Transforms/ObjCARC/basic.ll b/test/Transforms/ObjCARC/basic.ll
index c4f73e7dc9b..9fc5ad1f100 100644
--- a/test/Transforms/ObjCARC/basic.ll
+++ b/test/Transforms/ObjCARC/basic.ll
@@ -1289,7 +1289,7 @@ entry:
 ; CHECK: %tmp1 = tail call i8* @objc_retain(i8* %tmp) [[NUW]]
 ; CHECK-NEXT: invoke
 ; CHECK: }
-define void @test20(double* %self) {
+define void @test20(double* %self) personality i32 (...)* @__gxx_personality_v0 {
 if.then12:
   %tmp = bitcast double* %self to i8*
   %tmp1 = call i8* @objc_retain(i8* %tmp) nounwind
@@ -1302,7 +1302,7 @@ invoke.cont23:                                    ; preds = %if.then12
 
 lpad20:                                           ; preds = %invoke.cont23, %if.then12
   %tmp502 = phi double* [ undef, %invoke.cont23 ], [ %self, %if.then12 ]
-  %exn = landingpad {i8*, i32} personality i32 (...)* @__gxx_personality_v0
+  %exn = landingpad {i8*, i32}
            cleanup
   unreachable
 
@@ -2684,8 +2684,8 @@ define {<2 x float>, <2 x float>} @"\01-[A z]"({}* %self, i8* nocapture %_cmd) n
 invoke.cont:
   %0 = bitcast {}* %self to i8*
   %1 = tail call i8* @objc_retain(i8* %0) nounwind
-  tail call void @llvm.dbg.value(metadata {}* %self, i64 0, metadata !MDLocalVariable(tag: DW_TAG_auto_variable, scope: !2), metadata !MDExpression()), !dbg !MDLocation(scope: !2)
-  tail call void @llvm.dbg.value(metadata {}* %self, i64 0, metadata !MDLocalVariable(tag: DW_TAG_auto_variable, scope: !2), metadata !MDExpression()), !dbg !MDLocation(scope: !2)
+  tail call void @llvm.dbg.value(metadata {}* %self, i64 0, metadata !DILocalVariable(tag: DW_TAG_auto_variable, scope: !2), metadata !DIExpression()), !dbg !DILocation(scope: !2)
+  tail call void @llvm.dbg.value(metadata {}* %self, i64 0, metadata !DILocalVariable(tag: DW_TAG_auto_variable, scope: !2), metadata !DIExpression()), !dbg !DILocation(scope: !2)
   %ivar = load i64, i64* @"OBJC_IVAR_$_A.myZ", align 8
   %add.ptr = getelementptr i8, i8* %0, i64 %ivar
   %tmp1 = bitcast i8* %add.ptr to float*
@@ -3018,7 +3018,7 @@ define void @test67(i8* %x) {
 
 !0 = !{}
 !1 = !{i32 1, !"Debug Info Version", i32 3}
-!2 = !MDSubprogram()
+!2 = !DISubprogram()
 
 ; CHECK: attributes #0 = { nounwind readnone }
 ; CHECK: attributes [[NUW]] = { nounwind }
diff --git a/test/Transforms/ObjCARC/contract-testcases.ll b/test/Transforms/ObjCARC/contract-testcases.ll
index 74a4a7f989c..e6d34a9426f 100644
--- a/test/Transforms/ObjCARC/contract-testcases.ll
+++ b/test/Transforms/ObjCARC/contract-testcases.ll
@@ -67,12 +67,12 @@ bb7:                                              ; preds = %bb6, %bb6, %bb5
 ; call, handle the case where it's an invoke in a different basic block.
 ; rdar://11714057
 
-; CHECK: define void @_Z6doTestP8NSString() {
+; CHECK: define void @_Z6doTestP8NSString() personality i8* bitcast (i32 (...)* @__gxx_personality_sj0 to i8*) {
 ; CHECK: invoke.cont:                                      ; preds = %entry
 ; CHECK-NEXT: call void asm sideeffect "mov\09r7, r7\09\09@ marker for objc_retainAutoreleaseReturnValue", ""()
 ; CHECK-NEXT: %tmp = tail call i8* @objc_retainAutoreleasedReturnValue(i8* %call) [[NUW:#[0-9]+]]
 ; CHECK: }
-define void @_Z6doTestP8NSString() {
+define void @_Z6doTestP8NSString() personality i8* bitcast (i32 (...)* @__gxx_personality_sj0 to i8*) {
 entry:
   %call = invoke i8* bitcast (i8* (i8*, i8*, ...)* @objc_msgSend to i8* ()*)()
           to label %invoke.cont unwind label %lpad
@@ -82,7 +82,7 @@ invoke.cont:                                      ; preds = %entry
   unreachable
 
 lpad:                                             ; preds = %entry
-  %tmp1 = landingpad { i8*, i32 } personality i8* bitcast (i32 (...)* @__gxx_personality_sj0 to i8*)
+  %tmp1 = landingpad { i8*, i32 }
           cleanup
   resume { i8*, i32 } undef
 }
diff --git a/test/Transforms/ObjCARC/ensure-that-exception-unwind-path-is-visited.ll b/test/Transforms/ObjCARC/ensure-that-exception-unwind-path-is-visited.ll
index 25135a317f9..db3a780f91b 100644
--- a/test/Transforms/ObjCARC/ensure-that-exception-unwind-path-is-visited.ll
+++ b/test/Transforms/ObjCARC/ensure-that-exception-unwind-path-is-visited.ll
@@ -34,17 +34,17 @@ target triple = "x86_64-apple-macosx10.9.0"
 @"\01L_OBJC_SELECTOR_REFERENCES_5" = internal global i8* getelementptr inbounds ([14 x i8], [14 x i8]* @"\01L_OBJC_METH_VAR_NAME_4", i64 0, i64 0), section "__DATA, __objc_selrefs, literal_pointers, no_dead_strip"
 @llvm.used = appending global [6 x i8*] [i8* bitcast (%struct._class_t** @"\01L_OBJC_CLASSLIST_REFERENCES_$_" to i8*), i8* getelementptr inbounds ([4 x i8], [4 x i8]* @"\01L_OBJC_METH_VAR_NAME_", i32 0, i32 0), i8* bitcast (i8** @"\01L_OBJC_SELECTOR_REFERENCES_" to i8*), i8* bitcast (%struct._class_t** @"\01L_OBJC_CLASSLIST_REFERENCES_$_1" to i8*), i8* getelementptr inbounds ([14 x i8], [14 x i8]* @"\01L_OBJC_METH_VAR_NAME_4", i32 0, i32 0), i8* bitcast (i8** @"\01L_OBJC_SELECTOR_REFERENCES_5" to i8*)], section "llvm.metadata"
 
-define i32 @main() uwtable ssp {
+define i32 @main() uwtable ssp personality i8* bitcast (i32 (...)* @__objc_personality_v0 to i8*) {
 entry:
   %tmp = load %struct._class_t*, %struct._class_t** @"\01L_OBJC_CLASSLIST_REFERENCES_$_", align 8, !dbg !37
   %tmp1 = load i8*, i8** @"\01L_OBJC_SELECTOR_REFERENCES_", align 8, !dbg !37, !invariant.load !38
   %tmp2 = bitcast %struct._class_t* %tmp to i8*, !dbg !37
 ; CHECK: call i8* bitcast (i8* (i8*, i8*, ...)* @objc_msgSend to i8* (i8*, i8*)*)(i8* %tmp2, i8* %tmp1)
   %call = call i8* bitcast (i8* (i8*, i8*, ...)* @objc_msgSend to i8* (i8*, i8*)*)(i8* %tmp2, i8* %tmp1), !dbg !37, !clang.arc.no_objc_arc_exceptions !38
-  call void @llvm.dbg.value(metadata i8* %call, i64 0, metadata !25, metadata !MDExpression()), !dbg !37
+  call void @llvm.dbg.value(metadata i8* %call, i64 0, metadata !25, metadata !DIExpression()), !dbg !37
 ; CHECK: call i8* @objc_retain(i8* %call) [[NUW:#[0-9]+]]
   %tmp3 = call i8* @objc_retain(i8* %call) nounwind, !dbg !39
-  call void @llvm.dbg.value(metadata i8* %call, i64 0, metadata !25, metadata !MDExpression()), !dbg !39
+  call void @llvm.dbg.value(metadata i8* %call, i64 0, metadata !25, metadata !DIExpression()), !dbg !39
   invoke fastcc void @ThrowFunc(i8* %call)
           to label %eh.cont unwind label %lpad, !dbg !40, !clang.arc.no_objc_arc_exceptions !38
 
@@ -54,11 +54,11 @@ eh.cont:                                          ; preds = %entry
   br label %if.end, !dbg !43
 
 lpad:                                             ; preds = %entry
-  %tmp4 = landingpad { i8*, i32 } personality i8* bitcast (i32 (...)* @__objc_personality_v0 to i8*)
+  %tmp4 = landingpad { i8*, i32 }
           catch i8* null, !dbg !40
   %tmp5 = extractvalue { i8*, i32 } %tmp4, 0, !dbg !40
   %exn.adjusted = call i8* @objc_begin_catch(i8* %tmp5) nounwind, !dbg !44
-  call void @llvm.dbg.value(metadata i8 0, i64 0, metadata !21, metadata !MDExpression()), !dbg !46
+  call void @llvm.dbg.value(metadata i8 0, i64 0, metadata !21, metadata !DIExpression()), !dbg !46
   call void @objc_end_catch(), !dbg !49, !clang.arc.no_objc_arc_exceptions !38
 ; CHECK: call void @objc_release(i8* %call)
   call void @objc_release(i8* %call) nounwind, !dbg !42, !clang.imprecise_release !38
@@ -87,7 +87,7 @@ declare void @objc_exception_rethrow()
 define internal fastcc void @ThrowFunc(i8* %obj) uwtable noinline ssp {
 entry:
   %tmp = call i8* @objc_retain(i8* %obj) nounwind
-  call void @llvm.dbg.value(metadata i8* %obj, i64 0, metadata !32, metadata !MDExpression()), !dbg !55
+  call void @llvm.dbg.value(metadata i8* %obj, i64 0, metadata !32, metadata !DIExpression()), !dbg !55
   %tmp1 = load %struct._class_t*, %struct._class_t** @"\01L_OBJC_CLASSLIST_REFERENCES_$_1", align 8, !dbg !56
   %tmp2 = load i8*, i8** @"\01L_OBJC_SELECTOR_REFERENCES_5", align 8, !dbg !56, !invariant.load !38
   %tmp3 = bitcast %struct._class_t* %tmp1 to i8*, !dbg !56
@@ -113,60 +113,60 @@ declare void @llvm.dbg.value(metadata, i64, metadata, metadata) nounwind readnon
 !llvm.dbg.cu = !{!0}
 !llvm.module.flags = !{!33, !34, !35, !36, !61}
 
-!0 = !MDCompileUnit(language: DW_LANG_ObjC, producer: "clang version 3.3 ", isOptimized: true, runtimeVersion: 2, emissionKind: 0, file: !60, enums: !1, retainedTypes: !1, subprograms: !3, globals: !1)
+!0 = !DICompileUnit(language: DW_LANG_ObjC, producer: "clang version 3.3 ", isOptimized: true, runtimeVersion: 2, emissionKind: 0, file: !60, enums: !1, retainedTypes: !1, subprograms: !3, globals: !1)
 !1 = !{}
 !3 = !{!5, !27}
-!5 = !MDSubprogram(name: "main", line: 9, isLocal: false, isDefinition: true, virtualIndex: 6, isOptimized: true, scopeLine: 10, file: !60, scope: !6, type: !7, function: i32 ()* @main, variables: !11)
-!6 = !MDFile(filename: "test.m", directory: "/Volumes/Files/gottesmmcab/Radar/12906997")
-!7 = !MDSubroutineType(types: !8)
+!5 = !DISubprogram(name: "main", line: 9, isLocal: false, isDefinition: true, virtualIndex: 6, isOptimized: true, scopeLine: 10, file: !60, scope: !6, type: !7, function: i32 ()* @main, variables: !11)
+!6 = !DIFile(filename: "test.m", directory: "/Volumes/Files/gottesmmcab/Radar/12906997")
+!7 = !DISubroutineType(types: !8)
 !8 = !{!9}
-!9 = !MDBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
+!9 = !DIBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
 !11 = !{!12, !21, !25}
-!12 = !MDLocalVariable(tag: DW_TAG_auto_variable, name: "obj", line: 11, scope: !13, file: !6, type: !14)
-!13 = distinct !MDLexicalBlock(line: 10, column: 0, file: !60, scope: !5)
-!14 = !MDDerivedType(tag: DW_TAG_typedef, name: "id", line: 11, file: !60, baseType: !15)
-!15 = !MDDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, file: !60, baseType: !16)
-!16 = !MDCompositeType(tag: DW_TAG_structure_type, name: "objc_object", file: !60, elements: !17)
+!12 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "obj", line: 11, scope: !13, file: !6, type: !14)
+!13 = distinct !DILexicalBlock(line: 10, column: 0, file: !60, scope: !5)
+!14 = !DIDerivedType(tag: DW_TAG_typedef, name: "id", line: 11, file: !60, baseType: !15)
+!15 = !DIDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, file: !60, baseType: !16)
+!16 = !DICompositeType(tag: DW_TAG_structure_type, name: "objc_object", file: !60, elements: !17)
 !17 = !{!18}
-!18 = !MDDerivedType(tag: DW_TAG_member, name: "isa", size: 64, file: !60, scope: !16, baseType: !19)
-!19 = !MDDerivedType(tag: DW_TAG_pointer_type, size: 64, baseType: !20)
-!20 = !MDCompositeType(tag: DW_TAG_structure_type, name: "objc_class", flags: DIFlagFwdDecl, file: !60)
-!21 = !MDLocalVariable(tag: DW_TAG_auto_variable, name: "ok", line: 13, scope: !22, file: !6, type: !23)
-!22 = distinct !MDLexicalBlock(line: 12, column: 0, file: !60, scope: !13)
-!23 = !MDDerivedType(tag: DW_TAG_typedef, name: "BOOL", line: 62, file: !60, baseType: !24)
-!24 = !MDBasicType(tag: DW_TAG_base_type, name: "signed char", size: 8, align: 8, encoding: DW_ATE_signed_char)
-!25 = !MDLocalVariable(tag: DW_TAG_auto_variable, name: "obj2", line: 15, scope: !26, file: !6, type: !14)
-!26 = distinct !MDLexicalBlock(line: 14, column: 0, file: !60, scope: !22)
-!27 = !MDSubprogram(name: "ThrowFunc", line: 4, isLocal: true, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: true, scopeLine: 5, file: !60, scope: !6, type: !28, function: void (i8*)* @ThrowFunc, variables: !31)
-!28 = !MDSubroutineType(types: !29)
+!18 = !DIDerivedType(tag: DW_TAG_member, name: "isa", size: 64, file: !60, scope: !16, baseType: !19)
+!19 = !DIDerivedType(tag: DW_TAG_pointer_type, size: 64, baseType: !20)
+!20 = !DICompositeType(tag: DW_TAG_structure_type, name: "objc_class", flags: DIFlagFwdDecl, file: !60)
+!21 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "ok", line: 13, scope: !22, file: !6, type: !23)
+!22 = distinct !DILexicalBlock(line: 12, column: 0, file: !60, scope: !13)
+!23 = !DIDerivedType(tag: DW_TAG_typedef, name: "BOOL", line: 62, file: !60, baseType: !24)
+!24 = !DIBasicType(tag: DW_TAG_base_type, name: "signed char", size: 8, align: 8, encoding: DW_ATE_signed_char)
+!25 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "obj2", line: 15, scope: !26, file: !6, type: !14)
+!26 = distinct !DILexicalBlock(line: 14, column: 0, file: !60, scope: !22)
+!27 = !DISubprogram(name: "ThrowFunc", line: 4, isLocal: true, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: true, scopeLine: 5, file: !60, scope: !6, type: !28, function: void (i8*)* @ThrowFunc, variables: !31)
+!28 = !DISubroutineType(types: !29)
 !29 = !{null, !14}
 !31 = !{!32}
-!32 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "obj", line: 4, arg: 1, scope: !27, file: !6, type: !14)
+!32 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "obj", line: 4, arg: 1, scope: !27, file: !6, type: !14)
 !33 = !{i32 1, !"Objective-C Version", i32 2}
 !34 = !{i32 1, !"Objective-C Image Info Version", i32 0}
 !35 = !{i32 1, !"Objective-C Image Info Section", !"__DATA, __objc_imageinfo, regular, no_dead_strip"}
 !36 = !{i32 4, !"Objective-C Garbage Collection", i32 0}
-!37 = !MDLocation(line: 11, scope: !13)
+!37 = !DILocation(line: 11, scope: !13)
 !38 = !{}
-!39 = !MDLocation(line: 15, scope: !26)
-!40 = !MDLocation(line: 17, scope: !41)
-!41 = distinct !MDLexicalBlock(line: 16, column: 0, file: !60, scope: !26)
-!42 = !MDLocation(line: 22, scope: !26)
-!43 = !MDLocation(line: 23, scope: !22)
-!44 = !MDLocation(line: 19, scope: !41)
+!39 = !DILocation(line: 15, scope: !26)
+!40 = !DILocation(line: 17, scope: !41)
+!41 = distinct !DILexicalBlock(line: 16, column: 0, file: !60, scope: !26)
+!42 = !DILocation(line: 22, scope: !26)
+!43 = !DILocation(line: 23, scope: !22)
+!44 = !DILocation(line: 19, scope: !41)
 !45 = !{i8 0}
-!46 = !MDLocation(line: 20, scope: !47)
-!47 = distinct !MDLexicalBlock(line: 19, column: 0, file: !60, scope: !48)
-!48 = distinct !MDLexicalBlock(line: 19, column: 0, file: !60, scope: !26)
-!49 = !MDLocation(line: 21, scope: !47)
-!50 = !MDLocation(line: 24, scope: !51)
-!51 = distinct !MDLexicalBlock(line: 23, column: 0, file: !60, scope: !22)
-!52 = !MDLocation(line: 25, scope: !51)
-!53 = !MDLocation(line: 27, scope: !13)
-!54 = !MDLocation(line: 28, scope: !13)
-!55 = !MDLocation(line: 4, scope: !27)
-!56 = !MDLocation(line: 6, scope: !57)
-!57 = distinct !MDLexicalBlock(line: 5, column: 0, file: !60, scope: !27)
-!58 = !MDLocation(line: 7, scope: !57)
-!60 = !MDFile(filename: "test.m", directory: "/Volumes/Files/gottesmmcab/Radar/12906997")
+!46 = !DILocation(line: 20, scope: !47)
+!47 = distinct !DILexicalBlock(line: 19, column: 0, file: !60, scope: !48)
+!48 = distinct !DILexicalBlock(line: 19, column: 0, file: !60, scope: !26)
+!49 = !DILocation(line: 21, scope: !47)
+!50 = !DILocation(line: 24, scope: !51)
+!51 = distinct !DILexicalBlock(line: 23, column: 0, file: !60, scope: !22)
+!52 = !DILocation(line: 25, scope: !51)
+!53 = !DILocation(line: 27, scope: !13)
+!54 = !DILocation(line: 28, scope: !13)
+!55 = !DILocation(line: 4, scope: !27)
+!56 = !DILocation(line: 6, scope: !57)
+!57 = distinct !DILexicalBlock(line: 5, column: 0, file: !60, scope: !27)
+!58 = !DILocation(line: 7, scope: !57)
+!60 = !DIFile(filename: "test.m", directory: "/Volumes/Files/gottesmmcab/Radar/12906997")
 !61 = !{i32 1, !"Debug Info Version", i32 3}
diff --git a/test/Transforms/ObjCARC/invoke.ll b/test/Transforms/ObjCARC/invoke.ll
index 5ef5184154b..06105c17397 100644
--- a/test/Transforms/ObjCARC/invoke.ll
+++ b/test/Transforms/ObjCARC/invoke.ll
@@ -18,7 +18,7 @@ declare i8* @returner()
 ; CHECK:   call void @objc_release(i8* %zipFile) [[NUW]], !clang.imprecise_release !0
 ; CHECK:   ret void
 ; CHECK-NEXT: }
-define void @test0(i8* %zipFile) {
+define void @test0(i8* %zipFile) personality i32 (...)* @__gxx_personality_v0 {
 entry:
   call i8* @objc_retain(i8* %zipFile) nounwind
   call void @use_pointer(i8* %zipFile)
@@ -30,7 +30,7 @@ invoke.cont:                                      ; preds = %entry
   ret void
 
 lpad:                                             ; preds = %entry
-  %exn = landingpad {i8*, i32} personality i32 (...)* @__gxx_personality_v0
+  %exn = landingpad {i8*, i32}
            cleanup
   call void @objc_release(i8* %zipFile) nounwind, !clang.imprecise_release !0
   ret void
@@ -50,7 +50,7 @@ lpad:                                             ; preds = %entry
 ; CHECK: done:
 ; CHECK-NEXT: ret void
 ; CHECK-NEXT: }
-define void @test1(i8* %zipFile) {
+define void @test1(i8* %zipFile) personality i32 (...)* @__gxx_personality_v0 {
 entry:
   call i8* @objc_retain(i8* %zipFile) nounwind
   call void @use_pointer(i8* %zipFile)
@@ -62,7 +62,7 @@ invoke.cont:                                      ; preds = %entry
   br label %done
 
 lpad:                                             ; preds = %entry
-  %exn = landingpad {i8*, i32} personality i32 (...)* @__gxx_personality_v0
+  %exn = landingpad {i8*, i32}
            cleanup
   call void @callee()
   br label %done
@@ -75,7 +75,7 @@ done:
 ; The optimizer should ignore invoke unwind paths consistently.
 ; PR12265
 
-; CHECK: define void @test2() {
+; CHECK: define void @test2() personality i8* bitcast (i32 (...)* @__objc_personality_v0 to i8*) {
 ; CHECK: invoke.cont:
 ; CHECK-NEXT: call i8* @objc_retain
 ; CHECK-NOT: @objc_r
@@ -85,7 +85,7 @@ done:
 ; CHECK: finally.rethrow:
 ; CHECK-NOT: @objc
 ; CHECK: }
-define void @test2() {
+define void @test2() personality i8* bitcast (i32 (...)* @__objc_personality_v0 to i8*) {
 entry:
   %call = invoke i8* bitcast (i8* (i8*, i8*, ...)* @objc_msgSend to i8* ()*)()
           to label %invoke.cont unwind label %finally.rethrow, !clang.arc.no_objc_arc_exceptions !0
@@ -101,7 +101,7 @@ finally.cont:                                     ; preds = %invoke.cont
   ret void
 
 finally.rethrow:                                  ; preds = %invoke.cont, %entry
-  %tmp2 = landingpad { i8*, i32 } personality i8* bitcast (i32 (...)* @__objc_personality_v0 to i8*)
+  %tmp2 = landingpad { i8*, i32 }
           catch i8* null
   unreachable
 }
@@ -113,7 +113,7 @@ finally.rethrow:                                  ; preds = %invoke.cont, %entry
 ; CHECK-NEXT: call void @objc_release(i8* %p) [[NUW]]
 ; CHECK-NEXT: ret void
 ; CHECK-NEXT: }
-define void @test3(i8* %p, i1 %b) {
+define void @test3(i8* %p, i1 %b) personality i8* bitcast (i32 (...)* @__objc_personality_v0 to i8*) {
 entry:
   %0 = call i8* @objc_retain(i8* %p)
   call void @callee()
@@ -128,7 +128,7 @@ if.else:
           to label %if.end unwind label %lpad, !clang.arc.no_objc_arc_exceptions !0
 
 lpad:
-  %r = landingpad { i8*, i32 } personality i8* bitcast (i32 (...)* @__objc_personality_v0 to i8*)
+  %r = landingpad { i8*, i32 }
        cleanup
   ret void
 
@@ -141,7 +141,7 @@ if.end:
 
 ; CHECK-LABEL: define void @test4(
 ; CHECK: lpad:
-; CHECK-NEXT: %r = landingpad { i8*, i32 } personality i8* bitcast (i32 (...)* @__objc_personality_v0 to i8*)
+; CHECK-NEXT: %r = landingpad { i8*, i32 }
 ; CHECK-NEXT: cleanup
 ; CHECK-NEXT: call void @objc_release(i8* %p) [[NUW]]
 ; CHECK-NEXT: ret void
@@ -149,7 +149,7 @@ if.end:
 ; CHECK-NEXT: call void @objc_release(i8* %p) [[NUW]]
 ; CHECK-NEXT: ret void
 ; CHECK-NEXT: }
-define void @test4(i8* %p, i1 %b) {
+define void @test4(i8* %p, i1 %b) personality i8* bitcast (i32 (...)* @__objc_personality_v0 to i8*) {
 entry:
   %0 = call i8* @objc_retain(i8* %p)
   call void @callee()
@@ -164,7 +164,7 @@ if.else:
           to label %if.end unwind label %lpad
 
 lpad:
-  %r = landingpad { i8*, i32 } personality i8* bitcast (i32 (...)* @__objc_personality_v0 to i8*)
+  %r = landingpad { i8*, i32 }
        cleanup
   call void @objc_release(i8* %p)
   ret void
@@ -180,13 +180,13 @@ if.end:
 ; CHECK-LABEL: define void @test5(
 ; CHECK: call i8* @objc_retainAutoreleasedReturnValue(i8* %z)
 ; CHECK: }
-define void @test5() {
+define void @test5() personality i8* bitcast (i32 (...)* @__objc_personality_v0 to i8*) {
 entry:
   %z = invoke i8* @returner()
           to label %if.end unwind label %lpad, !clang.arc.no_objc_arc_exceptions !0
 
 lpad:
-  %r13 = landingpad { i8*, i32 } personality i8* bitcast (i32 (...)* @__objc_personality_v0 to i8*)
+  %r13 = landingpad { i8*, i32 }
           cleanup
   ret void
 
@@ -200,13 +200,13 @@ if.end:
 ; CHECK-LABEL: define void @test6(
 ; CHECK: call i8* @objc_retain(i8* %z)
 ; CHECK: }
-define void @test6() {
+define void @test6() personality i8* bitcast (i32 (...)* @__objc_personality_v0 to i8*) {
 entry:
   %z = invoke i8* @returner()
           to label %if.end unwind label %lpad, !clang.arc.no_objc_arc_exceptions !0
 
 lpad:
-  %r13 = landingpad { i8*, i32 } personality i8* bitcast (i32 (...)* @__objc_personality_v0 to i8*)
+  %r13 = landingpad { i8*, i32 }
           cleanup
   ret void
 
diff --git a/test/Transforms/ObjCARC/path-overflow.ll b/test/Transforms/ObjCARC/path-overflow.ll
index d2396531012..21fcc86f946 100644
--- a/test/Transforms/ObjCARC/path-overflow.ll
+++ b/test/Transforms/ObjCARC/path-overflow.ll
@@ -29,7 +29,7 @@ declare i32 @__gxx_personality_sj0(...)
 declare i32 @__objc_personality_v0(...)
 
 
-define hidden void @test1() {
+define hidden void @test1() personality i8* bitcast (i32 (...)* @__gxx_personality_sj0 to i8*) {
 entry:
   br i1 undef, label %msgSend.nullinit, label %msgSend.call
 
@@ -864,7 +864,7 @@ bb222:                                            ; preds = %bb20, %bb19
 }
 
 ; Function Attrs: ssp
-define void @test3() #1 {
+define void @test3() #1 personality i8* bitcast (i32 (...)* @__gxx_personality_sj0 to i8*) {
 entry:
   %call2 = invoke i8* bitcast (i8* (i8*, i8*, ...)* @objc_msgSend to i8* (i8*, i8*, i8*)*)(i8* undef, i8* undef, i8* bitcast (%struct.NSConstantString* @_unnamed_cfstring to i8*))
           to label %invoke.cont unwind label %lpad
@@ -881,7 +881,7 @@ land.rhs:                                         ; preds = %invoke.cont4
           to label %land.end unwind label %lpad3
 
 land.end:                                         ; preds = %land.rhs, %invoke.cont4
-  invoke void (i8*, ...)* @NSLog(i8* bitcast (%struct.NSConstantString* @_unnamed_cfstring to i8*), i8* undef, i8* bitcast (%struct.NSConstantString* @_unnamed_cfstring to i8*))
+  invoke void (i8*, ...) @NSLog(i8* bitcast (%struct.NSConstantString* @_unnamed_cfstring to i8*), i8* undef, i8* bitcast (%struct.NSConstantString* @_unnamed_cfstring to i8*))
           to label %invoke.cont.i unwind label %lpad.i
 
 invoke.cont.i:                                    ; preds = %land.end
@@ -891,12 +891,12 @@ if.then.i:                                        ; preds = %invoke.cont.i
   br label %invoke.cont8
 
 lpad.i:                                           ; preds = %land.end
-  %tmp13 = landingpad { i8*, i32 } personality i8* bitcast (i32 (...)* @__gxx_personality_sj0 to i8*)
+  %tmp13 = landingpad { i8*, i32 }
           cleanup
   unreachable
 
 invoke.cont8:                                     ; preds = %if.then.i, %invoke.cont.i
-  %call18 = invoke i8* (i8*, i8*, i8*, ...)* bitcast (i8* (i8*, i8*, ...)* @objc_msgSend to i8* (i8*, i8*, i8*, ...)*)(i8* undef, i8* undef, i8* bitcast (%struct.NSConstantString* @_unnamed_cfstring to i8*), i8* undef)
+  %call18 = invoke i8* (i8*, i8*, i8*, ...) bitcast (i8* (i8*, i8*, ...)* @objc_msgSend to i8* (i8*, i8*, i8*, ...)*)(i8* undef, i8* undef, i8* bitcast (%struct.NSConstantString* @_unnamed_cfstring to i8*), i8* undef)
           to label %invoke.cont17 unwind label %lpad16
 
 invoke.cont17:                                    ; preds = %invoke.cont8
@@ -904,7 +904,7 @@ invoke.cont17:                                    ; preds = %invoke.cont8
           to label %invoke.cont21 unwind label %lpad20
 
 invoke.cont21:                                    ; preds = %invoke.cont17
-  invoke void (i8*, ...)* @NSLog(i8* bitcast (%struct.NSConstantString* @_unnamed_cfstring to i8*), i8* undef, i8* bitcast (%struct.NSConstantString* @_unnamed_cfstring to i8*))
+  invoke void (i8*, ...) @NSLog(i8* bitcast (%struct.NSConstantString* @_unnamed_cfstring to i8*), i8* undef, i8* bitcast (%struct.NSConstantString* @_unnamed_cfstring to i8*))
           to label %invoke.cont.i1980 unwind label %lpad.i1982
 
 invoke.cont.i1980:                                ; preds = %invoke.cont21
@@ -914,7 +914,7 @@ if.then.i1981:                                    ; preds = %invoke.cont.i1980
   br label %invoke.cont24
 
 lpad.i1982:                                       ; preds = %invoke.cont21
-  %tmp28 = landingpad { i8*, i32 } personality i8* bitcast (i32 (...)* @__gxx_personality_sj0 to i8*)
+  %tmp28 = landingpad { i8*, i32 }
           cleanup
   unreachable
 
@@ -930,7 +930,7 @@ land.rhs39:                                       ; preds = %invoke.cont36
           to label %land.end43 unwind label %lpad35
 
 land.end43:                                       ; preds = %land.rhs39, %invoke.cont36
-  invoke void (i8*, ...)* @NSLog(i8* bitcast (%struct.NSConstantString* @_unnamed_cfstring to i8*), i8* undef, i8* bitcast (%struct.NSConstantString* @_unnamed_cfstring to i8*))
+  invoke void (i8*, ...) @NSLog(i8* bitcast (%struct.NSConstantString* @_unnamed_cfstring to i8*), i8* undef, i8* bitcast (%struct.NSConstantString* @_unnamed_cfstring to i8*))
           to label %invoke.cont.i1986 unwind label %lpad.i1988
 
 invoke.cont.i1986:                                ; preds = %land.end43
@@ -940,7 +940,7 @@ if.then.i1987:                                    ; preds = %invoke.cont.i1986
   br label %invoke.cont44
 
 lpad.i1988:                                       ; preds = %land.end43
-  %tmp42 = landingpad { i8*, i32 } personality i8* bitcast (i32 (...)* @__gxx_personality_sj0 to i8*)
+  %tmp42 = landingpad { i8*, i32 }
           cleanup
   unreachable
 
@@ -960,7 +960,7 @@ invoke.cont62:                                    ; preds = %land.rhs58
           to label %land.end70 unwind label %lpad66.body.thread
 
 land.end70:                                       ; preds = %invoke.cont62, %invoke.cont52
-  invoke void (i8*, ...)* @NSLog(i8* bitcast (%struct.NSConstantString* @_unnamed_cfstring to i8*), i8* undef, i8* bitcast (%struct.NSConstantString* @_unnamed_cfstring to i8*))
+  invoke void (i8*, ...) @NSLog(i8* bitcast (%struct.NSConstantString* @_unnamed_cfstring to i8*), i8* undef, i8* bitcast (%struct.NSConstantString* @_unnamed_cfstring to i8*))
           to label %invoke.cont.i1992 unwind label %lpad66.body
 
 invoke.cont.i1992:                                ; preds = %land.end70
@@ -970,7 +970,7 @@ if.then.i1993:                                    ; preds = %invoke.cont.i1992
   br label %invoke.cont71
 
 invoke.cont71:                                    ; preds = %if.then.i1993, %invoke.cont.i1992
-  invoke void (i8*, ...)* @NSLog(i8* bitcast (%struct.NSConstantString* @_unnamed_cfstring to i8*), i8* undef, i8* bitcast (%struct.NSConstantString* @_unnamed_cfstring to i8*))
+  invoke void (i8*, ...) @NSLog(i8* bitcast (%struct.NSConstantString* @_unnamed_cfstring to i8*), i8* undef, i8* bitcast (%struct.NSConstantString* @_unnamed_cfstring to i8*))
           to label %invoke.cont.i1998 unwind label %lpad.i2000
 
 invoke.cont.i1998:                                ; preds = %invoke.cont71
@@ -980,7 +980,7 @@ if.then.i1999:                                    ; preds = %invoke.cont.i1998
   br label %invoke.cont91
 
 lpad.i2000:                                       ; preds = %invoke.cont71
-  %tmp74 = landingpad { i8*, i32 } personality i8* bitcast (i32 (...)* @__gxx_personality_sj0 to i8*)
+  %tmp74 = landingpad { i8*, i32 }
           cleanup
   br label %ehcleanup102
 
@@ -993,7 +993,7 @@ invoke.cont95:                                    ; preds = %invoke.cont91
           to label %invoke.cont97 unwind label %lpad94
 
 invoke.cont97:                                    ; preds = %invoke.cont95
-  invoke void (i8*, ...)* @NSLog(i8* bitcast (%struct.NSConstantString* @_unnamed_cfstring to i8*), i8* undef, i8* bitcast (%struct.NSConstantString* @_unnamed_cfstring to i8*))
+  invoke void (i8*, ...) @NSLog(i8* bitcast (%struct.NSConstantString* @_unnamed_cfstring to i8*), i8* undef, i8* bitcast (%struct.NSConstantString* @_unnamed_cfstring to i8*))
           to label %invoke.cont.i2004 unwind label %lpad.i2006
 
 invoke.cont.i2004:                                ; preds = %invoke.cont97
@@ -1003,7 +1003,7 @@ if.then.i2005:                                    ; preds = %invoke.cont.i2004
   br label %invoke.cont100
 
 lpad.i2006:                                       ; preds = %invoke.cont97
-  %tmp82 = landingpad { i8*, i32 } personality i8* bitcast (i32 (...)* @__gxx_personality_sj0 to i8*)
+  %tmp82 = landingpad { i8*, i32 }
           cleanup
   unreachable
 
@@ -1012,7 +1012,7 @@ invoke.cont100:                                   ; preds = %if.then.i2005, %inv
           to label %invoke.cont110 unwind label %lpad109
 
 invoke.cont110:                                   ; preds = %invoke.cont100
-  invoke void (i8*, ...)* @NSLog(i8* bitcast (%struct.NSConstantString* @_unnamed_cfstring to i8*), i8* undef, i8* bitcast (%struct.NSConstantString* @_unnamed_cfstring to i8*))
+  invoke void (i8*, ...) @NSLog(i8* bitcast (%struct.NSConstantString* @_unnamed_cfstring to i8*), i8* undef, i8* bitcast (%struct.NSConstantString* @_unnamed_cfstring to i8*))
           to label %invoke.cont.i2010 unwind label %lpad.i2012
 
 invoke.cont.i2010:                                ; preds = %invoke.cont110
@@ -1022,21 +1022,21 @@ if.then.i2011:                                    ; preds = %invoke.cont.i2010
   br label %invoke.cont117
 
 lpad.i2012:                                       ; preds = %invoke.cont110
-  %tmp98 = landingpad { i8*, i32 } personality i8* bitcast (i32 (...)* @__gxx_personality_sj0 to i8*)
+  %tmp98 = landingpad { i8*, i32 }
           cleanup
   unreachable
 
 invoke.cont117:                                   ; preds = %if.then.i2011, %invoke.cont.i2010
-  invoke void (i8*, ...)* @NSLog(i8* bitcast (%struct.NSConstantString* @_unnamed_cfstring to i8*), i8* undef, i8* bitcast (%struct.NSConstantString* @_unnamed_cfstring to i8*))
+  invoke void (i8*, ...) @NSLog(i8* bitcast (%struct.NSConstantString* @_unnamed_cfstring to i8*), i8* undef, i8* bitcast (%struct.NSConstantString* @_unnamed_cfstring to i8*))
           to label %invoke.cont.i2022 unwind label %lpad156.body
 
 lpad:                                             ; preds = %entry
-  %tmp118 = landingpad { i8*, i32 } personality i8* bitcast (i32 (...)* @__gxx_personality_sj0 to i8*)
+  %tmp118 = landingpad { i8*, i32 }
           cleanup
   br label %ehcleanup
 
 lpad3:                                            ; preds = %land.rhs, %invoke.cont
-  %tmp119 = landingpad { i8*, i32 } personality i8* bitcast (i32 (...)* @__gxx_personality_sj0 to i8*)
+  %tmp119 = landingpad { i8*, i32 }
           cleanup
   br label %ehcleanup
 
@@ -1044,12 +1044,12 @@ ehcleanup:                                        ; preds = %lpad3, %lpad
   unreachable
 
 lpad16:                                           ; preds = %invoke.cont8
-  %tmp121 = landingpad { i8*, i32 } personality i8* bitcast (i32 (...)* @__gxx_personality_sj0 to i8*)
+  %tmp121 = landingpad { i8*, i32 }
           cleanup
   br label %ehcleanup26
 
 lpad20:                                           ; preds = %invoke.cont17
-  %tmp122 = landingpad { i8*, i32 } personality i8* bitcast (i32 (...)* @__gxx_personality_sj0 to i8*)
+  %tmp122 = landingpad { i8*, i32 }
           cleanup
   br label %ehcleanup26
 
@@ -1057,32 +1057,32 @@ ehcleanup26:                                      ; preds = %lpad20, %lpad16
   unreachable
 
 lpad35:                                           ; preds = %land.rhs39, %invoke.cont24
-  %tmp124 = landingpad { i8*, i32 } personality i8* bitcast (i32 (...)* @__gxx_personality_sj0 to i8*)
+  %tmp124 = landingpad { i8*, i32 }
           cleanup
   unreachable
 
 lpad51:                                           ; preds = %invoke.cont44
-  %tmp125 = landingpad { i8*, i32 } personality i8* bitcast (i32 (...)* @__gxx_personality_sj0 to i8*)
+  %tmp125 = landingpad { i8*, i32 }
           cleanup
   unreachable
 
 lpad61:                                           ; preds = %land.rhs58
-  %tmp127 = landingpad { i8*, i32 } personality i8* bitcast (i32 (...)* @__gxx_personality_sj0 to i8*)
+  %tmp127 = landingpad { i8*, i32 }
           cleanup
   unreachable
 
 lpad66.body.thread:                               ; preds = %invoke.cont62
-  %tmp128 = landingpad { i8*, i32 } personality i8* bitcast (i32 (...)* @__gxx_personality_sj0 to i8*)
+  %tmp128 = landingpad { i8*, i32 }
           cleanup
   unreachable
 
 lpad66.body:                                      ; preds = %land.end70
-  %tmp129 = landingpad { i8*, i32 } personality i8* bitcast (i32 (...)* @__gxx_personality_sj0 to i8*)
+  %tmp129 = landingpad { i8*, i32 }
           cleanup
   unreachable
 
 lpad94:                                           ; preds = %invoke.cont95, %invoke.cont91
-  %tmp133 = landingpad { i8*, i32 } personality i8* bitcast (i32 (...)* @__gxx_personality_sj0 to i8*)
+  %tmp133 = landingpad { i8*, i32 }
           cleanup
   br label %ehcleanup102
 
@@ -1090,7 +1090,7 @@ ehcleanup102:                                     ; preds = %lpad94, %lpad.i2000
   unreachable
 
 lpad109:                                          ; preds = %invoke.cont100
-  %tmp134 = landingpad { i8*, i32 } personality i8* bitcast (i32 (...)* @__gxx_personality_sj0 to i8*)
+  %tmp134 = landingpad { i8*, i32 }
           cleanup
   unreachable
 
@@ -1101,7 +1101,7 @@ if.then.i2023:                                    ; preds = %invoke.cont.i2022
   br label %invoke.cont157
 
 invoke.cont157:                                   ; preds = %if.then.i2023, %invoke.cont.i2022
-  invoke void (i8*, ...)* @NSLog(i8* bitcast (%struct.NSConstantString* @_unnamed_cfstring to i8*), i8* undef, i8* bitcast (%struct.NSConstantString* @_unnamed_cfstring to i8*))
+  invoke void (i8*, ...) @NSLog(i8* bitcast (%struct.NSConstantString* @_unnamed_cfstring to i8*), i8* undef, i8* bitcast (%struct.NSConstantString* @_unnamed_cfstring to i8*))
           to label %invoke.cont.i2028 unwind label %lpad164.body
 
 invoke.cont.i2028:                                ; preds = %invoke.cont157
@@ -1119,7 +1119,7 @@ invoke.cont184:                                   ; preds = %invoke.cont165
           to label %invoke.cont185 unwind label %lpad183
 
 invoke.cont185:                                   ; preds = %invoke.cont184
-  invoke void (i8*, ...)* @NSLog(i8* bitcast (%struct.NSConstantString* @_unnamed_cfstring to i8*), i8* undef, i8* bitcast (%struct.NSConstantString* @_unnamed_cfstring to i8*))
+  invoke void (i8*, ...) @NSLog(i8* bitcast (%struct.NSConstantString* @_unnamed_cfstring to i8*), i8* undef, i8* bitcast (%struct.NSConstantString* @_unnamed_cfstring to i8*))
           to label %invoke.cont.i2034 unwind label %lpad.i2036
 
 invoke.cont.i2034:                                ; preds = %invoke.cont185
@@ -1129,7 +1129,7 @@ if.then.i2035:                                    ; preds = %invoke.cont.i2034
   br label %invoke.cont190
 
 lpad.i2036:                                       ; preds = %invoke.cont185
-  %tmp168 = landingpad { i8*, i32 } personality i8* bitcast (i32 (...)* @__gxx_personality_sj0 to i8*)
+  %tmp168 = landingpad { i8*, i32 }
           cleanup
   br label %lpad183.body
 
@@ -1146,7 +1146,7 @@ invoke.cont201:                                   ; preds = %invoke.cont197
           to label %invoke.cont204 unwind label %lpad203
 
 invoke.cont204:                                   ; preds = %invoke.cont201
-  invoke void (i8*, ...)* @NSLog(i8* bitcast (%struct.NSConstantString* @_unnamed_cfstring to i8*), i8* undef, i8* bitcast (%struct.NSConstantString* @_unnamed_cfstring to i8*))
+  invoke void (i8*, ...) @NSLog(i8* bitcast (%struct.NSConstantString* @_unnamed_cfstring to i8*), i8* undef, i8* bitcast (%struct.NSConstantString* @_unnamed_cfstring to i8*))
           to label %invoke.cont.i2040 unwind label %lpad.i2042
 
 invoke.cont.i2040:                                ; preds = %invoke.cont204
@@ -1156,7 +1156,7 @@ if.then.i2041:                                    ; preds = %invoke.cont.i2040
   br label %invoke.cont207
 
 lpad.i2042:                                       ; preds = %invoke.cont204
-  %tmp181 = landingpad { i8*, i32 } personality i8* bitcast (i32 (...)* @__gxx_personality_sj0 to i8*)
+  %tmp181 = landingpad { i8*, i32 }
           cleanup
   unreachable
 
@@ -1165,7 +1165,7 @@ invoke.cont207:                                   ; preds = %if.then.i2041, %inv
           to label %invoke.cont208 unwind label %lpad203
 
 invoke.cont208:                                   ; preds = %invoke.cont207
-  invoke void (i8*, ...)* @NSLog(i8* bitcast (%struct.NSConstantString* @_unnamed_cfstring to i8*), i8* undef, i8* bitcast (%struct.NSConstantString* @_unnamed_cfstring to i8*))
+  invoke void (i8*, ...) @NSLog(i8* bitcast (%struct.NSConstantString* @_unnamed_cfstring to i8*), i8* undef, i8* bitcast (%struct.NSConstantString* @_unnamed_cfstring to i8*))
           to label %invoke.cont.i2046 unwind label %lpad212.body
 
 invoke.cont.i2046:                                ; preds = %invoke.cont208
@@ -1183,7 +1183,7 @@ invoke.cont221:                                   ; preds = %invoke.cont213
           to label %invoke.cont228 unwind label %lpad227
 
 invoke.cont228:                                   ; preds = %invoke.cont221
-  invoke void (i8*, ...)* @NSLog(i8* bitcast (%struct.NSConstantString* @_unnamed_cfstring to i8*), i8* undef, i8* bitcast (%struct.NSConstantString* @_unnamed_cfstring to i8*))
+  invoke void (i8*, ...) @NSLog(i8* bitcast (%struct.NSConstantString* @_unnamed_cfstring to i8*), i8* undef, i8* bitcast (%struct.NSConstantString* @_unnamed_cfstring to i8*))
           to label %invoke.cont.i2052 unwind label %lpad.i2054
 
 invoke.cont.i2052:                                ; preds = %invoke.cont228
@@ -1193,7 +1193,7 @@ if.then.i2053:                                    ; preds = %invoke.cont.i2052
   br label %invoke.cont231
 
 lpad.i2054:                                       ; preds = %invoke.cont228
-  %tmp198 = landingpad { i8*, i32 } personality i8* bitcast (i32 (...)* @__gxx_personality_sj0 to i8*)
+  %tmp198 = landingpad { i8*, i32 }
           cleanup
   unreachable
 
@@ -1202,7 +1202,7 @@ invoke.cont231:                                   ; preds = %if.then.i2053, %inv
           to label %invoke.cont232 unwind label %lpad227
 
 invoke.cont232:                                   ; preds = %invoke.cont231
-  invoke void (i8*, ...)* @NSLog(i8* bitcast (%struct.NSConstantString* @_unnamed_cfstring to i8*), i8* undef, i8* bitcast (%struct.NSConstantString* @_unnamed_cfstring to i8*))
+  invoke void (i8*, ...) @NSLog(i8* bitcast (%struct.NSConstantString* @_unnamed_cfstring to i8*), i8* undef, i8* bitcast (%struct.NSConstantString* @_unnamed_cfstring to i8*))
           to label %invoke.cont.i2058 unwind label %lpad236.body
 
 invoke.cont.i2058:                                ; preds = %invoke.cont232
@@ -1248,7 +1248,7 @@ invoke.cont274:                                   ; preds = %invoke.cont266
           to label %invoke.cont278 unwind label %lpad277
 
 invoke.cont278:                                   ; preds = %invoke.cont274
-  invoke void (i8*, ...)* @NSLog(i8* bitcast (%struct.NSConstantString* @_unnamed_cfstring to i8*), i8* undef, i8* bitcast (%struct.NSConstantString* @_unnamed_cfstring to i8*))
+  invoke void (i8*, ...) @NSLog(i8* bitcast (%struct.NSConstantString* @_unnamed_cfstring to i8*), i8* undef, i8* bitcast (%struct.NSConstantString* @_unnamed_cfstring to i8*))
           to label %invoke.cont.i2064 unwind label %lpad.i2066
 
 invoke.cont.i2064:                                ; preds = %invoke.cont278
@@ -1258,7 +1258,7 @@ if.then.i2065:                                    ; preds = %invoke.cont.i2064
   br label %invoke.cont281
 
 lpad.i2066:                                       ; preds = %invoke.cont278
-  %tmp253 = landingpad { i8*, i32 } personality i8* bitcast (i32 (...)* @__gxx_personality_sj0 to i8*)
+  %tmp253 = landingpad { i8*, i32 }
           cleanup
   unreachable
 
@@ -1294,7 +1294,7 @@ land.rhs335:                                      ; preds = %invoke.cont321
           to label %land.end344 unwind label %lpad340.body.thread
 
 land.end344:                                      ; preds = %land.rhs335, %invoke.cont321
-  invoke void (i8*, ...)* @NSLog(i8* bitcast (%struct.NSConstantString* @_unnamed_cfstring to i8*), i8* undef, i8* bitcast (%struct.NSConstantString* @_unnamed_cfstring to i8*))
+  invoke void (i8*, ...) @NSLog(i8* bitcast (%struct.NSConstantString* @_unnamed_cfstring to i8*), i8* undef, i8* bitcast (%struct.NSConstantString* @_unnamed_cfstring to i8*))
           to label %invoke.cont.i2070 unwind label %lpad340.body
 
 invoke.cont.i2070:                                ; preds = %land.end344
@@ -1316,7 +1316,7 @@ invoke.cont364:                                   ; preds = %invoke.cont361
           to label %invoke.cont370 unwind label %lpad369
 
 invoke.cont370:                                   ; preds = %invoke.cont364
-  invoke void (i8*, ...)* @NSLog(i8* bitcast (%struct.NSConstantString* @_unnamed_cfstring to i8*), i8* undef, i8* bitcast (%struct.NSConstantString* @_unnamed_cfstring to i8*))
+  invoke void (i8*, ...) @NSLog(i8* bitcast (%struct.NSConstantString* @_unnamed_cfstring to i8*), i8* undef, i8* bitcast (%struct.NSConstantString* @_unnamed_cfstring to i8*))
           to label %invoke.cont.i2076 unwind label %lpad.i2078
 
 invoke.cont.i2076:                                ; preds = %invoke.cont370
@@ -1326,7 +1326,7 @@ if.then.i2077:                                    ; preds = %invoke.cont.i2076
   br label %invoke.cont373
 
 lpad.i2078:                                       ; preds = %invoke.cont370
-  %tmp340 = landingpad { i8*, i32 } personality i8* bitcast (i32 (...)* @__gxx_personality_sj0 to i8*)
+  %tmp340 = landingpad { i8*, i32 }
           cleanup
   unreachable
 
@@ -1343,7 +1343,7 @@ invoke.cont382:                                   ; preds = %invoke.cont376
           to label %invoke.cont383 unwind label %lpad381
 
 invoke.cont383:                                   ; preds = %invoke.cont382
-  invoke void (i8*, ...)* @NSLog(i8* bitcast (%struct.NSConstantString* @_unnamed_cfstring to i8*), i8* undef, i8* bitcast (%struct.NSConstantString* @_unnamed_cfstring to i8*))
+  invoke void (i8*, ...) @NSLog(i8* bitcast (%struct.NSConstantString* @_unnamed_cfstring to i8*), i8* undef, i8* bitcast (%struct.NSConstantString* @_unnamed_cfstring to i8*))
           to label %invoke.cont.i2082 unwind label %lpad.i2084
 
 invoke.cont.i2082:                                ; preds = %invoke.cont383
@@ -1353,7 +1353,7 @@ if.then.i2083:                                    ; preds = %invoke.cont.i2082
   br label %invoke.cont392
 
 lpad.i2084:                                       ; preds = %invoke.cont383
-  %tmp360 = landingpad { i8*, i32 } personality i8* bitcast (i32 (...)* @__gxx_personality_sj0 to i8*)
+  %tmp360 = landingpad { i8*, i32 }
           cleanup
   unreachable
 
@@ -1374,7 +1374,7 @@ invoke.cont399:                                   ; preds = %invoke.cont396
           to label %invoke.cont402 unwind label %lpad401
 
 invoke.cont402:                                   ; preds = %invoke.cont399
-  invoke void (i8*, ...)* @NSLog(i8* bitcast (%struct.NSConstantString* @_unnamed_cfstring to i8*), i8* undef, i8* bitcast (%struct.NSConstantString* @_unnamed_cfstring to i8*))
+  invoke void (i8*, ...) @NSLog(i8* bitcast (%struct.NSConstantString* @_unnamed_cfstring to i8*), i8* undef, i8* bitcast (%struct.NSConstantString* @_unnamed_cfstring to i8*))
           to label %invoke.cont.i2088 unwind label %lpad.i2090
 
 invoke.cont.i2088:                                ; preds = %invoke.cont402
@@ -1384,7 +1384,7 @@ if.then.i2089:                                    ; preds = %invoke.cont.i2088
   br label %invoke.cont405
 
 lpad.i2090:                                       ; preds = %invoke.cont402
-  %tmp370 = landingpad { i8*, i32 } personality i8* bitcast (i32 (...)* @__gxx_personality_sj0 to i8*)
+  %tmp370 = landingpad { i8*, i32 }
           cleanup
   unreachable
 
@@ -1401,7 +1401,7 @@ invoke.cont409:                                   ; preds = %invoke.cont408
           to label %invoke.cont412 unwind label %lpad411
 
 invoke.cont412:                                   ; preds = %invoke.cont409
-  invoke void (i8*, ...)* @NSLog(i8* bitcast (%struct.NSConstantString* @_unnamed_cfstring to i8*), i8* undef, i8* bitcast (%struct.NSConstantString* @_unnamed_cfstring to i8*))
+  invoke void (i8*, ...) @NSLog(i8* bitcast (%struct.NSConstantString* @_unnamed_cfstring to i8*), i8* undef, i8* bitcast (%struct.NSConstantString* @_unnamed_cfstring to i8*))
           to label %invoke.cont.i2094 unwind label %lpad.i2096
 
 invoke.cont.i2094:                                ; preds = %invoke.cont412
@@ -1411,7 +1411,7 @@ if.then.i2095:                                    ; preds = %invoke.cont.i2094
   br label %invoke.cont418
 
 lpad.i2096:                                       ; preds = %invoke.cont412
-  %tmp380 = landingpad { i8*, i32 } personality i8* bitcast (i32 (...)* @__gxx_personality_sj0 to i8*)
+  %tmp380 = landingpad { i8*, i32 }
           cleanup
   unreachable
 
@@ -1432,7 +1432,7 @@ invoke.cont426:                                   ; preds = %invoke.cont423
           to label %invoke.cont429 unwind label %lpad428
 
 invoke.cont429:                                   ; preds = %invoke.cont426
-  invoke void (i8*, ...)* @NSLog(i8* bitcast (%struct.NSConstantString* @_unnamed_cfstring to i8*), i8* undef, i8* bitcast (%struct.NSConstantString* @_unnamed_cfstring to i8*))
+  invoke void (i8*, ...) @NSLog(i8* bitcast (%struct.NSConstantString* @_unnamed_cfstring to i8*), i8* undef, i8* bitcast (%struct.NSConstantString* @_unnamed_cfstring to i8*))
           to label %invoke.cont.i2100 unwind label %lpad.i2102
 
 invoke.cont.i2100:                                ; preds = %invoke.cont429
@@ -1442,7 +1442,7 @@ if.then.i2101:                                    ; preds = %invoke.cont.i2100
   br label %invoke.cont432
 
 lpad.i2102:                                       ; preds = %invoke.cont429
-  %tmp390 = landingpad { i8*, i32 } personality i8* bitcast (i32 (...)* @__gxx_personality_sj0 to i8*)
+  %tmp390 = landingpad { i8*, i32 }
           cleanup
   unreachable
 
@@ -1451,7 +1451,7 @@ invoke.cont432:                                   ; preds = %if.then.i2101, %inv
           to label %invoke.cont435 unwind label %lpad381
 
 invoke.cont435:                                   ; preds = %invoke.cont432
-  invoke void (i8*, ...)* @NSLog(i8* bitcast (%struct.NSConstantString* @_unnamed_cfstring to i8*), i8* undef, i8* bitcast (%struct.NSConstantString* @_unnamed_cfstring to i8*))
+  invoke void (i8*, ...) @NSLog(i8* bitcast (%struct.NSConstantString* @_unnamed_cfstring to i8*), i8* undef, i8* bitcast (%struct.NSConstantString* @_unnamed_cfstring to i8*))
           to label %invoke.cont.i2106 unwind label %lpad.i2108
 
 invoke.cont.i2106:                                ; preds = %invoke.cont435
@@ -1459,12 +1459,12 @@ invoke.cont.i2106:                                ; preds = %invoke.cont435
           to label %invoke.cont443 unwind label %lpad381
 
 lpad.i2108:                                       ; preds = %invoke.cont435
-  %tmp396 = landingpad { i8*, i32 } personality i8* bitcast (i32 (...)* @__gxx_personality_sj0 to i8*)
+  %tmp396 = landingpad { i8*, i32 }
           cleanup
   unreachable
 
 invoke.cont443:                                   ; preds = %invoke.cont.i2106
-  invoke void (i8*, ...)* @NSLog(i8* bitcast (%struct.NSConstantString* @_unnamed_cfstring to i8*), i8* undef, i8* bitcast (%struct.NSConstantString* @_unnamed_cfstring to i8*))
+  invoke void (i8*, ...) @NSLog(i8* bitcast (%struct.NSConstantString* @_unnamed_cfstring to i8*), i8* undef, i8* bitcast (%struct.NSConstantString* @_unnamed_cfstring to i8*))
           to label %invoke.cont.i2112 unwind label %lpad.i2114
 
 invoke.cont.i2112:                                ; preds = %invoke.cont443
@@ -1474,7 +1474,7 @@ if.then.i2113:                                    ; preds = %invoke.cont.i2112
   br label %invoke.cont449
 
 lpad.i2114:                                       ; preds = %invoke.cont443
-  %tmp402 = landingpad { i8*, i32 } personality i8* bitcast (i32 (...)* @__gxx_personality_sj0 to i8*)
+  %tmp402 = landingpad { i8*, i32 }
           cleanup
   unreachable
 
@@ -1487,7 +1487,7 @@ invoke.cont452:                                   ; preds = %invoke.cont449
           to label %invoke.cont455 unwind label %lpad454
 
 invoke.cont455:                                   ; preds = %invoke.cont452
-  invoke void (i8*, ...)* @NSLog(i8* bitcast (%struct.NSConstantString* @_unnamed_cfstring to i8*), i8* undef, i8* bitcast (%struct.NSConstantString* @_unnamed_cfstring to i8*))
+  invoke void (i8*, ...) @NSLog(i8* bitcast (%struct.NSConstantString* @_unnamed_cfstring to i8*), i8* undef, i8* bitcast (%struct.NSConstantString* @_unnamed_cfstring to i8*))
           to label %invoke.cont.i2118 unwind label %lpad.i2120
 
 invoke.cont.i2118:                                ; preds = %invoke.cont455
@@ -1497,7 +1497,7 @@ if.then.i2119:                                    ; preds = %invoke.cont.i2118
   br label %invoke.cont458
 
 lpad.i2120:                                       ; preds = %invoke.cont455
-  %tmp408 = landingpad { i8*, i32 } personality i8* bitcast (i32 (...)* @__gxx_personality_sj0 to i8*)
+  %tmp408 = landingpad { i8*, i32 }
           cleanup
   unreachable
 
@@ -1506,7 +1506,7 @@ invoke.cont458:                                   ; preds = %if.then.i2119, %inv
           to label %invoke.cont460 unwind label %lpad381
 
 invoke.cont460:                                   ; preds = %invoke.cont458
-  invoke void (i8*, ...)* @NSLog(i8* bitcast (%struct.NSConstantString* @_unnamed_cfstring to i8*), i8* undef, i8* bitcast (%struct.NSConstantString* @_unnamed_cfstring to i8*))
+  invoke void (i8*, ...) @NSLog(i8* bitcast (%struct.NSConstantString* @_unnamed_cfstring to i8*), i8* undef, i8* bitcast (%struct.NSConstantString* @_unnamed_cfstring to i8*))
           to label %invoke.cont.i2124 unwind label %lpad.i2126
 
 invoke.cont.i2124:                                ; preds = %invoke.cont460
@@ -1516,7 +1516,7 @@ if.then.i2125:                                    ; preds = %invoke.cont.i2124
   br label %invoke.cont466
 
 lpad.i2126:                                       ; preds = %invoke.cont460
-  %tmp414 = landingpad { i8*, i32 } personality i8* bitcast (i32 (...)* @__gxx_personality_sj0 to i8*)
+  %tmp414 = landingpad { i8*, i32 }
           cleanup
   br label %ehcleanup477
 
@@ -1525,7 +1525,7 @@ invoke.cont466:                                   ; preds = %if.then.i2125, %inv
           to label %invoke.cont469 unwind label %lpad381
 
 invoke.cont469:                                   ; preds = %invoke.cont466
-  invoke void (i8*, ...)* @NSLog(i8* bitcast (%struct.NSConstantString* @_unnamed_cfstring to i8*), i8* undef, i8* bitcast (%struct.NSConstantString* @_unnamed_cfstring to i8*))
+  invoke void (i8*, ...) @NSLog(i8* bitcast (%struct.NSConstantString* @_unnamed_cfstring to i8*), i8* undef, i8* bitcast (%struct.NSConstantString* @_unnamed_cfstring to i8*))
           to label %invoke.cont.i2130 unwind label %lpad.i2132
 
 invoke.cont.i2130:                                ; preds = %invoke.cont469
@@ -1535,7 +1535,7 @@ if.then.i2131:                                    ; preds = %invoke.cont.i2130
   br label %invoke.cont475
 
 lpad.i2132:                                       ; preds = %invoke.cont469
-  %tmp420 = landingpad { i8*, i32 } personality i8* bitcast (i32 (...)* @__gxx_personality_sj0 to i8*)
+  %tmp420 = landingpad { i8*, i32 }
           cleanup
   br label %ehcleanup477
 
@@ -1574,7 +1574,7 @@ msgSend.null-receiver:                            ; preds = %invoke.cont512
   br label %msgSend.cont
 
 msgSend.cont:                                     ; preds = %msgSend.null-receiver, %msgSend.call
-  invoke void (i8*, ...)* @NSLog(i8* bitcast (%struct.NSConstantString* @_unnamed_cfstring to i8*), i8* undef, i8* bitcast (%struct.NSConstantString* @_unnamed_cfstring to i8*))
+  invoke void (i8*, ...) @NSLog(i8* bitcast (%struct.NSConstantString* @_unnamed_cfstring to i8*), i8* undef, i8* bitcast (%struct.NSConstantString* @_unnamed_cfstring to i8*))
           to label %invoke.cont.i2136 unwind label %lpad.i2138
 
 invoke.cont.i2136:                                ; preds = %msgSend.cont
@@ -1584,7 +1584,7 @@ if.then.i2137:                                    ; preds = %invoke.cont.i2136
   br label %invoke.cont521
 
 lpad.i2138:                                       ; preds = %msgSend.cont
-  %tmp468 = landingpad { i8*, i32 } personality i8* bitcast (i32 (...)* @__gxx_personality_sj0 to i8*)
+  %tmp468 = landingpad { i8*, i32 }
           cleanup
   unreachable
 
@@ -1601,7 +1601,7 @@ invoke.cont531:                                   ; preds = %invoke.cont528
           to label %invoke.cont534 unwind label %lpad533
 
 invoke.cont534:                                   ; preds = %invoke.cont531
-  invoke void (i8*, ...)* @NSLog(i8* bitcast (%struct.NSConstantString* @_unnamed_cfstring to i8*), i8* undef, i8* bitcast (%struct.NSConstantString* @_unnamed_cfstring to i8*))
+  invoke void (i8*, ...) @NSLog(i8* bitcast (%struct.NSConstantString* @_unnamed_cfstring to i8*), i8* undef, i8* bitcast (%struct.NSConstantString* @_unnamed_cfstring to i8*))
           to label %invoke.cont.i2142 unwind label %lpad.i2144
 
 invoke.cont.i2142:                                ; preds = %invoke.cont534
@@ -1611,7 +1611,7 @@ if.then.i2143:                                    ; preds = %invoke.cont.i2142
   br label %invoke.cont540
 
 lpad.i2144:                                       ; preds = %invoke.cont534
-  %tmp486 = landingpad { i8*, i32 } personality i8* bitcast (i32 (...)* @__gxx_personality_sj0 to i8*)
+  %tmp486 = landingpad { i8*, i32 }
           cleanup
   unreachable
 
@@ -1633,7 +1633,7 @@ invoke.cont548:                                   ; preds = %invoke.cont546
 
 invoke.cont554:                                   ; preds = %invoke.cont548
   %tmp499 = call i8* @objc_retain(i8* bitcast (%struct.NSConstantString* @_unnamed_cfstring to i8*)) #3
-  invoke void (i8*, ...)* @NSLog(i8* bitcast (%struct.NSConstantString* @_unnamed_cfstring to i8*), i8* %tmp499, i8* bitcast (%struct.NSConstantString* @_unnamed_cfstring to i8*))
+  invoke void (i8*, ...) @NSLog(i8* bitcast (%struct.NSConstantString* @_unnamed_cfstring to i8*), i8* %tmp499, i8* bitcast (%struct.NSConstantString* @_unnamed_cfstring to i8*))
           to label %invoke.cont.i2148 unwind label %lpad.i2150
 
 invoke.cont.i2148:                                ; preds = %invoke.cont554
@@ -1642,7 +1642,7 @@ invoke.cont.i2148:                                ; preds = %invoke.cont554
           to label %invoke.cont566 unwind label %lpad565
 
 lpad.i2150:                                       ; preds = %invoke.cont554
-  %tmp500 = landingpad { i8*, i32 } personality i8* bitcast (i32 (...)* @__gxx_personality_sj0 to i8*)
+  %tmp500 = landingpad { i8*, i32 }
           cleanup
   call void @objc_release(i8* %tmp499) #3, !clang.imprecise_release !0
   unreachable
@@ -1659,17 +1659,17 @@ invoke.cont581:                                   ; preds = %invoke.cont572
   unreachable
 
 lpad156.body:                                     ; preds = %invoke.cont117
-  %tmp1157 = landingpad { i8*, i32 } personality i8* bitcast (i32 (...)* @__gxx_personality_sj0 to i8*)
+  %tmp1157 = landingpad { i8*, i32 }
           cleanup
   unreachable
 
 lpad164.body:                                     ; preds = %invoke.cont157
-  %tmp1158 = landingpad { i8*, i32 } personality i8* bitcast (i32 (...)* @__gxx_personality_sj0 to i8*)
+  %tmp1158 = landingpad { i8*, i32 }
           cleanup
   unreachable
 
 lpad183:                                          ; preds = %invoke.cont184, %invoke.cont165
-  %tmp1159 = landingpad { i8*, i32 } personality i8* bitcast (i32 (...)* @__gxx_personality_sj0 to i8*)
+  %tmp1159 = landingpad { i8*, i32 }
           cleanup
   br label %lpad183.body
 
@@ -1677,37 +1677,37 @@ lpad183.body:                                     ; preds = %lpad183, %lpad.i203
   unreachable
 
 lpad196:                                          ; preds = %invoke.cont190
-  %tmp1160 = landingpad { i8*, i32 } personality i8* bitcast (i32 (...)* @__gxx_personality_sj0 to i8*)
+  %tmp1160 = landingpad { i8*, i32 }
           cleanup
   unreachable
 
 lpad200:                                          ; preds = %invoke.cont197
-  %tmp1161 = landingpad { i8*, i32 } personality i8* bitcast (i32 (...)* @__gxx_personality_sj0 to i8*)
+  %tmp1161 = landingpad { i8*, i32 }
           cleanup
   unreachable
 
 lpad203:                                          ; preds = %invoke.cont207, %invoke.cont201
-  %tmp1162 = landingpad { i8*, i32 } personality i8* bitcast (i32 (...)* @__gxx_personality_sj0 to i8*)
+  %tmp1162 = landingpad { i8*, i32 }
           cleanup
   unreachable
 
 lpad212.body:                                     ; preds = %invoke.cont208
-  %tmp1163 = landingpad { i8*, i32 } personality i8* bitcast (i32 (...)* @__gxx_personality_sj0 to i8*)
+  %tmp1163 = landingpad { i8*, i32 }
           cleanup
   unreachable
 
 lpad220:                                          ; preds = %invoke.cont213
-  %tmp1164 = landingpad { i8*, i32 } personality i8* bitcast (i32 (...)* @__gxx_personality_sj0 to i8*)
+  %tmp1164 = landingpad { i8*, i32 }
           cleanup
   br label %eh.resume
 
 lpad227:                                          ; preds = %invoke.cont231, %invoke.cont221
-  %tmp1166 = landingpad { i8*, i32 } personality i8* bitcast (i32 (...)* @__gxx_personality_sj0 to i8*)
+  %tmp1166 = landingpad { i8*, i32 }
           cleanup
   br label %ehcleanup239
 
 lpad236.body:                                     ; preds = %invoke.cont232
-  %tmp1167 = landingpad { i8*, i32 } personality i8* bitcast (i32 (...)* @__gxx_personality_sj0 to i8*)
+  %tmp1167 = landingpad { i8*, i32 }
           cleanup
   br label %ehcleanup239
 
@@ -1715,27 +1715,27 @@ ehcleanup239:                                     ; preds = %lpad236.body, %lpad
   unreachable
 
 lpad244:                                          ; preds = %invoke.cont245, %invoke.cont237
-  %tmp1168 = landingpad { i8*, i32 } personality i8* bitcast (i32 (...)* @__gxx_personality_sj0 to i8*)
+  %tmp1168 = landingpad { i8*, i32 }
           cleanup
   unreachable
 
 lpad249:                                          ; preds = %invoke.cont247
-  %tmp1169 = landingpad { i8*, i32 } personality i8* bitcast (i32 (...)* @__gxx_personality_sj0 to i8*)
+  %tmp1169 = landingpad { i8*, i32 }
           cleanup
   unreachable
 
 lpad252:                                          ; preds = %invoke.cont250
-  %tmp1170 = landingpad { i8*, i32 } personality i8* bitcast (i32 (...)* @__gxx_personality_sj0 to i8*)
+  %tmp1170 = landingpad { i8*, i32 }
           cleanup
   br label %ehcleanup263
 
 lpad255:                                          ; preds = %invoke.cont253
-  %tmp1171 = landingpad { i8*, i32 } personality i8* bitcast (i32 (...)* @__gxx_personality_sj0 to i8*)
+  %tmp1171 = landingpad { i8*, i32 }
           cleanup
   br label %ehcleanup263
 
 lpad258:                                          ; preds = %invoke.cont256
-  %tmp1172 = landingpad { i8*, i32 } personality i8* bitcast (i32 (...)* @__gxx_personality_sj0 to i8*)
+  %tmp1172 = landingpad { i8*, i32 }
           cleanup
   unreachable
 
@@ -1743,107 +1743,107 @@ ehcleanup263:                                     ; preds = %lpad255, %lpad252
   unreachable
 
 lpad265:                                          ; preds = %invoke.cont259
-  %tmp1173 = landingpad { i8*, i32 } personality i8* bitcast (i32 (...)* @__gxx_personality_sj0 to i8*)
+  %tmp1173 = landingpad { i8*, i32 }
           cleanup
   unreachable
 
 lpad273:                                          ; preds = %invoke.cont266
-  %tmp1175 = landingpad { i8*, i32 } personality i8* bitcast (i32 (...)* @__gxx_personality_sj0 to i8*)
+  %tmp1175 = landingpad { i8*, i32 }
           cleanup
   unreachable
 
 lpad277:                                          ; preds = %invoke.cont274
-  %tmp1176 = landingpad { i8*, i32 } personality i8* bitcast (i32 (...)* @__gxx_personality_sj0 to i8*)
+  %tmp1176 = landingpad { i8*, i32 }
           cleanup
   unreachable
 
 lpad289:                                          ; preds = %invoke.cont281
-  %tmp1177 = landingpad { i8*, i32 } personality i8* bitcast (i32 (...)* @__gxx_personality_sj0 to i8*)
+  %tmp1177 = landingpad { i8*, i32 }
           cleanup
   unreachable
 
 lpad301:                                          ; preds = %invoke.cont290
-  %tmp1180 = landingpad { i8*, i32 } personality i8* bitcast (i32 (...)* @__gxx_personality_sj0 to i8*)
+  %tmp1180 = landingpad { i8*, i32 }
           cleanup
   unreachable
 
 lpad308:                                          ; preds = %invoke.cont302
-  %tmp1182 = landingpad { i8*, i32 } personality i8* bitcast (i32 (...)* @__gxx_personality_sj0 to i8*)
+  %tmp1182 = landingpad { i8*, i32 }
           cleanup
   unreachable
 
 lpad311:                                          ; preds = %invoke.cont309
-  %tmp1183 = landingpad { i8*, i32 } personality i8* bitcast (i32 (...)* @__gxx_personality_sj0 to i8*)
+  %tmp1183 = landingpad { i8*, i32 }
           cleanup
   unreachable
 
 lpad314:                                          ; preds = %invoke.cont312
-  %tmp1184 = landingpad { i8*, i32 } personality i8* bitcast (i32 (...)* @__gxx_personality_sj0 to i8*)
+  %tmp1184 = landingpad { i8*, i32 }
           cleanup
   unreachable
 
 lpad320:                                          ; preds = %invoke.cont315
-  %tmp1186 = landingpad { i8*, i32 } personality i8* bitcast (i32 (...)* @__gxx_personality_sj0 to i8*)
+  %tmp1186 = landingpad { i8*, i32 }
           cleanup
   unreachable
 
 lpad340.body.thread:                              ; preds = %land.rhs335
-  %tmp1188 = landingpad { i8*, i32 } personality i8* bitcast (i32 (...)* @__gxx_personality_sj0 to i8*)
+  %tmp1188 = landingpad { i8*, i32 }
           cleanup
   unreachable
 
 lpad340.body:                                     ; preds = %land.end344
-  %tmp1189 = landingpad { i8*, i32 } personality i8* bitcast (i32 (...)* @__gxx_personality_sj0 to i8*)
+  %tmp1189 = landingpad { i8*, i32 }
           cleanup
   unreachable
 
 lpad360:                                          ; preds = %invoke.cont345
-  %tmp1191 = landingpad { i8*, i32 } personality i8* bitcast (i32 (...)* @__gxx_personality_sj0 to i8*)
+  %tmp1191 = landingpad { i8*, i32 }
           cleanup
   br label %eh.resume
 
 lpad363:                                          ; preds = %invoke.cont373, %invoke.cont361
-  %tmp1192 = landingpad { i8*, i32 } personality i8* bitcast (i32 (...)* @__gxx_personality_sj0 to i8*)
+  %tmp1192 = landingpad { i8*, i32 }
           cleanup
   unreachable
 
 lpad369:                                          ; preds = %invoke.cont364
-  %tmp1194 = landingpad { i8*, i32 } personality i8* bitcast (i32 (...)* @__gxx_personality_sj0 to i8*)
+  %tmp1194 = landingpad { i8*, i32 }
           cleanup
   unreachable
 
 lpad381:                                          ; preds = %invoke.cont466, %invoke.cont458, %invoke.cont449, %invoke.cont.i2106, %invoke.cont432, %invoke.cont422, %invoke.cont418, %invoke.cont408, %invoke.cont405, %invoke.cont395, %invoke.cont392, %invoke.cont382, %invoke.cont376
-  %tmp1196 = landingpad { i8*, i32 } personality i8* bitcast (i32 (...)* @__gxx_personality_sj0 to i8*)
+  %tmp1196 = landingpad { i8*, i32 }
           cleanup
   br label %ehcleanup477
 
 lpad398:                                          ; preds = %invoke.cont396
-  %tmp1199 = landingpad { i8*, i32 } personality i8* bitcast (i32 (...)* @__gxx_personality_sj0 to i8*)
+  %tmp1199 = landingpad { i8*, i32 }
           cleanup
   unreachable
 
 lpad401:                                          ; preds = %invoke.cont399
-  %tmp1200 = landingpad { i8*, i32 } personality i8* bitcast (i32 (...)* @__gxx_personality_sj0 to i8*)
+  %tmp1200 = landingpad { i8*, i32 }
           cleanup
   unreachable
 
 lpad411:                                          ; preds = %invoke.cont409
-  %tmp1201 = landingpad { i8*, i32 } personality i8* bitcast (i32 (...)* @__gxx_personality_sj0 to i8*)
+  %tmp1201 = landingpad { i8*, i32 }
           cleanup
   unreachable
 
 lpad425:                                          ; preds = %invoke.cont423
-  %tmp1203 = landingpad { i8*, i32 } personality i8* bitcast (i32 (...)* @__gxx_personality_sj0 to i8*)
+  %tmp1203 = landingpad { i8*, i32 }
           cleanup
   br label %ehcleanup477
 
 lpad428:                                          ; preds = %invoke.cont426
-  %tmp1204 = landingpad { i8*, i32 } personality i8* bitcast (i32 (...)* @__gxx_personality_sj0 to i8*)
+  %tmp1204 = landingpad { i8*, i32 }
           cleanup
   unreachable
 
 lpad454:                                          ; preds = %invoke.cont452
-  %tmp1207 = landingpad { i8*, i32 } personality i8* bitcast (i32 (...)* @__gxx_personality_sj0 to i8*)
+  %tmp1207 = landingpad { i8*, i32 }
           cleanup
   unreachable
 
@@ -1851,47 +1851,47 @@ ehcleanup477:                                     ; preds = %lpad425, %lpad381,
   unreachable
 
 lpad489:                                          ; preds = %invoke.cont546, %invoke.cont540, %invoke.cont528, %invoke.cont509, %invoke.cont499, %invoke.cont475
-  %tmp1211 = landingpad { i8*, i32 } personality i8* bitcast (i32 (...)* @__gxx_personality_sj0 to i8*)
+  %tmp1211 = landingpad { i8*, i32 }
           cleanup
   br label %ehcleanup560
 
 lpad498:                                          ; preds = %invoke.cont490
-  %tmp1214 = landingpad { i8*, i32 } personality i8* bitcast (i32 (...)* @__gxx_personality_sj0 to i8*)
+  %tmp1214 = landingpad { i8*, i32 }
           cleanup
   unreachable
 
 lpad505:                                          ; preds = %invoke.cont503
-  %tmp1215 = landingpad { i8*, i32 } personality i8* bitcast (i32 (...)* @__gxx_personality_sj0 to i8*)
+  %tmp1215 = landingpad { i8*, i32 }
           cleanup
   unreachable
 
 lpad508:                                          ; preds = %invoke.cont506
-  %tmp1216 = landingpad { i8*, i32 } personality i8* bitcast (i32 (...)* @__gxx_personality_sj0 to i8*)
+  %tmp1216 = landingpad { i8*, i32 }
           cleanup
   unreachable
 
 lpad514:                                          ; preds = %msgSend.call
-  %tmp1217 = landingpad { i8*, i32 } personality i8* bitcast (i32 (...)* @__gxx_personality_sj0 to i8*)
+  %tmp1217 = landingpad { i8*, i32 }
           cleanup
   unreachable
 
 lpad527:                                          ; preds = %invoke.cont521
-  %tmp1219 = landingpad { i8*, i32 } personality i8* bitcast (i32 (...)* @__gxx_personality_sj0 to i8*)
+  %tmp1219 = landingpad { i8*, i32 }
           cleanup
   br label %ehcleanup560
 
 lpad533:                                          ; preds = %invoke.cont531
-  %tmp1220 = landingpad { i8*, i32 } personality i8* bitcast (i32 (...)* @__gxx_personality_sj0 to i8*)
+  %tmp1220 = landingpad { i8*, i32 }
           cleanup
   unreachable
 
 lpad545:                                          ; preds = %invoke.cont543
-  %tmp1222 = landingpad { i8*, i32 } personality i8* bitcast (i32 (...)* @__gxx_personality_sj0 to i8*)
+  %tmp1222 = landingpad { i8*, i32 }
           cleanup
   unreachable
 
 lpad553:                                          ; preds = %invoke.cont548
-  %tmp1224 = landingpad { i8*, i32 } personality i8* bitcast (i32 (...)* @__gxx_personality_sj0 to i8*)
+  %tmp1224 = landingpad { i8*, i32 }
           cleanup
   unreachable
 
@@ -1899,17 +1899,17 @@ ehcleanup560:                                     ; preds = %lpad527, %lpad489
   br label %eh.resume
 
 lpad565:                                          ; preds = %invoke.cont.i2148
-  %tmp1225 = landingpad { i8*, i32 } personality i8* bitcast (i32 (...)* @__gxx_personality_sj0 to i8*)
+  %tmp1225 = landingpad { i8*, i32 }
           cleanup
   unreachable
 
 lpad571:                                          ; preds = %invoke.cont566
-  %tmp1227 = landingpad { i8*, i32 } personality i8* bitcast (i32 (...)* @__gxx_personality_sj0 to i8*)
+  %tmp1227 = landingpad { i8*, i32 }
           cleanup
   unreachable
 
 lpad580:                                          ; preds = %invoke.cont572
-  %tmp1228 = landingpad { i8*, i32 } personality i8* bitcast (i32 (...)* @__gxx_personality_sj0 to i8*)
+  %tmp1228 = landingpad { i8*, i32 }
           cleanup
   br label %eh.resume
 
@@ -1919,7 +1919,7 @@ eh.resume:                                        ; preds = %lpad580, %ehcleanup
 
 @"OBJC_EHTYPE_$_NSException" = external global i8
 
-define void @test4() {
+define void @test4() personality i8* bitcast (i32 (...)* @__objc_personality_v0 to i8*) {
 entry:
   br i1 undef, label %if.end13, label %if.then10
 
@@ -2173,7 +2173,7 @@ if.then430:                                       ; preds = %eh.cont
   br label %if.end439
 
 lpad:                                             ; preds = %if.end399
-  %2 = landingpad { i8*, i32 } personality i8* bitcast (i32 (...)* @__objc_personality_v0 to i8*)
+  %2 = landingpad { i8*, i32 }
           catch i8* @"OBJC_EHTYPE_$_NSException"
   unreachable
 
diff --git a/test/Transforms/ObjCARC/retain-not-declared.ll b/test/Transforms/ObjCARC/retain-not-declared.ll
index 416202222aa..f7ac908a76e 100644
--- a/test/Transforms/ObjCARC/retain-not-declared.ll
+++ b/test/Transforms/ObjCARC/retain-not-declared.ll
@@ -34,7 +34,7 @@ entry:
 ; CHECK: @objc_release(
 ; CHECK: @objc_release(
 ; CHECK: }
-define void @test1(i8* %call88) nounwind {
+define void @test1(i8* %call88) nounwind personality i32 (...)* @__gxx_personality_v0 {
 entry:
   %tmp1 = call i8* @objc_retainAutoreleasedReturnValue(i8* %call88) nounwind
   %call94 = invoke i8* bitcast (i8* (i8*, i8*, ...)* @objc_msgSend to i8* (i8*)*)(i8* %tmp1)
@@ -51,12 +51,12 @@ invoke.cont102:                                   ; preds = %invoke.cont93
   unreachable
 
 lpad91:                                           ; preds = %entry
-  %exn91 = landingpad {i8*, i32} personality i32 (...)* @__gxx_personality_v0
+  %exn91 = landingpad {i8*, i32}
               cleanup
   unreachable
 
 lpad100:                                          ; preds = %invoke.cont93
-  %exn100 = landingpad {i8*, i32} personality i32 (...)* @__gxx_personality_v0
+  %exn100 = landingpad {i8*, i32}
               cleanup
   call void @objc_release(i8* %tmp2) nounwind, !clang.imprecise_release !0
   unreachable
diff --git a/test/Transforms/ObjCARC/split-backedge.ll b/test/Transforms/ObjCARC/split-backedge.ll
index 2507173f4b7..6851487ed50 100644
--- a/test/Transforms/ObjCARC/split-backedge.ll
+++ b/test/Transforms/ObjCARC/split-backedge.ll
@@ -10,7 +10,7 @@
 ; CHECK: call void @objc_release(i8* %call) [[NUW]]
 ; CHECK: call void @objc_release(i8* %call) [[NUW]]
 ; CHECK: call void @objc_release(i8* %cond) [[NUW]]
-define void @test0() {
+define void @test0() personality i8* bitcast (i32 (...)* @__objc_personality_v0 to i8*) {
 entry:
   br label %while.body
 
@@ -34,7 +34,7 @@ invoke.cont1:                                     ; preds = %invoke.cont
   br label %while.body
 
 lpad:                                             ; preds = %invoke.cont, %while.body
-  %t4 = landingpad { i8*, i32 } personality i8* bitcast (i32 (...)* @__objc_personality_v0 to i8*)
+  %t4 = landingpad { i8*, i32 }
           catch i8* null
   ret void
 }
diff --git a/test/Transforms/PhaseOrdering/gdce.ll b/test/Transforms/PhaseOrdering/gdce.ll
index 6f79eb81583..fa62f92500b 100644
--- a/test/Transforms/PhaseOrdering/gdce.ll
+++ b/test/Transforms/PhaseOrdering/gdce.ll
@@ -67,7 +67,7 @@ entry:
   ret void
 }
 
-define linkonce_odr void @_ZN4BaseD0Ev(%class.Base* %this) unnamed_addr uwtable ssp align 2 {
+define linkonce_odr void @_ZN4BaseD0Ev(%class.Base* %this) unnamed_addr uwtable ssp align 2 personality i8* bitcast (i32 (...)* @__gxx_personality_v0 to i8*) {
 entry:
   %this.addr = alloca %class.Base*, align 8
   %exn.slot = alloca i8*
@@ -83,7 +83,7 @@ invoke.cont:                                      ; preds = %entry
   ret void
 
 lpad:                                             ; preds = %entry
-  %1 = landingpad { i8*, i32 } personality i8* bitcast (i32 (...)* @__gxx_personality_v0 to i8*)
+  %1 = landingpad { i8*, i32 }
           cleanup
   %2 = extractvalue { i8*, i32 } %1, 0
   store i8* %2, i8** %exn.slot
diff --git a/test/Transforms/PlaceSafepoints/basic.ll b/test/Transforms/PlaceSafepoints/basic.ll
index 4dbff586582..32aa4da68f2 100644
--- a/test/Transforms/PlaceSafepoints/basic.ll
+++ b/test/Transforms/PlaceSafepoints/basic.ll
@@ -73,8 +73,8 @@ define i1 @test_call_with_result() gc "statepoint-example" {
 ; inserted for a function that takes 1 argument.
 ; CHECK: gc.statepoint.p0f_isVoidf
 ; CHECK: gc.statepoint.p0f_i1i1f
-; CHECK: (i1 (i1)* @i1_return_i1, i32 1, i32 0, i1 false, i32 0)
-; CHECK: %call12 = call i1 @llvm.experimental.gc.result.i1
+; CHECK: (i64 2882400000, i32 0, i1 (i1)* @i1_return_i1, i32 1, i32 0, i1 false, i32 0, i32 0)
+; CHECK: %call1.2 = call i1 @llvm.experimental.gc.result.i1
 entry:
   %call1 = tail call i1 (i1) @i1_return_i1(i1 false)
   ret i1 %call1
diff --git a/test/Transforms/PlaceSafepoints/call-in-loop.ll b/test/Transforms/PlaceSafepoints/call-in-loop.ll
index a220fc96c10..9edfeb75e0c 100644
--- a/test/Transforms/PlaceSafepoints/call-in-loop.ll
+++ b/test/Transforms/PlaceSafepoints/call-in-loop.ll
@@ -15,7 +15,7 @@ entry:
 
 loop:
 ; CHECK-LABEL: loop
-; CHECK: @llvm.experimental.gc.statepoint.p0f_isVoidf(void ()* @foo
+; CHECK: @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 2882400000, i32 0, void ()* @foo
 ; CHECK-NOT: statepoint
   call void @foo()
   br label %loop
diff --git a/test/Transforms/PlaceSafepoints/call_gc_result.ll b/test/Transforms/PlaceSafepoints/call_gc_result.ll
new file mode 100644
index 00000000000..d78a0989c3b
--- /dev/null
+++ b/test/Transforms/PlaceSafepoints/call_gc_result.ll
@@ -0,0 +1,38 @@
+;; RUN: opt %s -place-safepoints -S | FileCheck %s
+
+;; This test is to verify that gc_result from a call statepoint
+;; can have preceding phis in its parent basic block. Unlike
+;; invoke statepoint, call statepoint does not terminate the
+;; block, and thus its gc_result is in the same block with the
+;; call statepoint.
+
+declare i32 @foo()
+
+define i32 @test1(i1 %cond, i32 %a) gc "statepoint-example" {
+entry:
+  br i1 %cond, label %branch1, label %branch2
+  
+branch1:
+  %b = add i32 %a, 1
+  br label %merge
+ 
+branch2:
+  br label %merge
+
+merge:
+;; CHECK: 		%phi = phi i32 [ %a, %branch2 ], [ %b, %branch1 ]
+;; CHECK-NEXT:  %safepoint_token.1 = call i32 (i64, i32, i32 ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_i32f(i64 2882400000, i32 0, i32 ()* @foo, i32 0, i32 0, i32 0, i32 0)
+;; CHECK-NEXT:  %ret.2 = call i32 @llvm.experimental.gc.result.i32(i32 %safepoint_token.1)
+  %phi = phi i32 [ %a, %branch2 ], [ %b, %branch1 ]
+  %ret = call i32 @foo()
+  ret i32 %ret
+}
+
+; This function is inlined when inserting a poll.
+declare void @do_safepoint()
+define void @gc.safepoint_poll() {
+; CHECK-LABEL: gc.safepoint_poll
+entry:
+  call void @do_safepoint()
+  ret void
+}
diff --git a/test/Transforms/PlaceSafepoints/finite-loops.ll b/test/Transforms/PlaceSafepoints/finite-loops.ll
index 8b64d240603..3cc7158afcf 100644
--- a/test/Transforms/PlaceSafepoints/finite-loops.ll
+++ b/test/Transforms/PlaceSafepoints/finite-loops.ll
@@ -77,4 +77,4 @@ define void @gc.safepoint_poll() {
 entry:
   call void @do_safepoint()
   ret void
-}
+}
\ No newline at end of file
diff --git a/test/Transforms/PlaceSafepoints/invokes.ll b/test/Transforms/PlaceSafepoints/invokes.ll
index 5fd5bea7e5f..a93e4545bc9 100644
--- a/test/Transforms/PlaceSafepoints/invokes.ll
+++ b/test/Transforms/PlaceSafepoints/invokes.ll
@@ -3,7 +3,7 @@
 declare i64 addrspace(1)* @"some_call"(i64 addrspace(1)*)
 declare i32 @"personality_function"()
 
-define i64 addrspace(1)* @test_basic(i64 addrspace(1)* %obj, i64 addrspace(1)* %obj1) gc "statepoint-example" {
+define i64 addrspace(1)* @test_basic(i64 addrspace(1)* %obj, i64 addrspace(1)* %obj1) gc "statepoint-example" personality i32 ()* @"personality_function" {
 ; CHECK-LABEL: entry:
 entry:
   ; CHECK: invoke
@@ -24,12 +24,12 @@ normal_return:
 ; CHECK: ret i64
 
 exceptional_return:
-  %landing_pad4 = landingpad {i8*, i32} personality i32 ()* @"personality_function"
+  %landing_pad4 = landingpad {i8*, i32}
           cleanup
   ret i64 addrspace(1)* %obj1
 }
 
-define i64 addrspace(1)* @test_two_invokes(i64 addrspace(1)* %obj, i64 addrspace(1)* %obj1) gc "statepoint-example" {
+define i64 addrspace(1)* @test_two_invokes(i64 addrspace(1)* %obj, i64 addrspace(1)* %obj1) gc "statepoint-example" personality i32 ()* @"personality_function" {
 ; CHECK-LABEL: entry:
 entry:
   ; CHECK: invoke 
@@ -56,12 +56,13 @@ normal_return:
 ; CHECK: ret i64
 
 exceptional_return:
-  %landing_pad4 = landingpad {i8*, i32} personality i32 ()* @"personality_function"
+  %landing_pad4 = landingpad {i8*, i32}
           cleanup
   ret i64 addrspace(1)* %obj1
 }
 
-define i64 addrspace(1)* @test_phi_node(i1 %cond, i64 addrspace(1)* %obj) gc "statepoint-example" {
+define i64 addrspace(1)* @test_phi_node(i1 %cond, i64 addrspace(1)* %obj) gc "statepoint-example" personality i32 ()* @"personality_function" {
+; CHECK-LABEL: @test_phi_node
 ; CHECK-LABEL: entry:
 entry:
   br i1 %cond, label %left, label %right
@@ -74,15 +75,15 @@ right:
   %ret_val_right = invoke i64 addrspace(1)* @"some_call"(i64 addrspace(1)* %obj)
                      to label %merge unwind label %exceptional_return
 
-; CHECK-LABEL: merge1:
+; CHECK: merge[[A:[0-9]]]:
 ; CHECK: gc.result
-; CHECK: br label %merge
+; CHECK: br label %[[with_phi:merge[0-9]*]]
 
-; CHECK-LABEL: merge3:
+; CHECK: merge[[B:[0-9]]]:
 ; CHECK: gc.result
-; CHECK: br label %merge
+; CHECK: br label %[[with_phi]]
 
-; CHECK-LABEL: merge:
+; CHECK: [[with_phi]]:
 ; CHECK: phi
 ; CHECK: ret i64 addrspace(1)* %ret_val
 merge:
@@ -93,7 +94,7 @@ merge:
 ; CHECK: ret i64 addrspace(1)*
 
 exceptional_return:
-  %landing_pad4 = landingpad {i8*, i32} personality i32 ()* @"personality_function"
+  %landing_pad4 = landingpad {i8*, i32}
           cleanup
   ret i64 addrspace(1)* %obj
 }
diff --git a/test/Transforms/PlaceSafepoints/memset.ll b/test/Transforms/PlaceSafepoints/memset.ll
new file mode 100644
index 00000000000..534b2f12058
--- /dev/null
+++ b/test/Transforms/PlaceSafepoints/memset.ll
@@ -0,0 +1,20 @@
+; RUN: opt -S -place-safepoints %s | FileCheck %s
+
+define void @test(i32, i8 addrspace(1)* %ptr) gc "statepoint-example" {
+; CHECK-LABEL: @test
+; CHECK-NEXT: llvm.memset
+; CHECK: do_safepoint
+; CHECK: @foo
+  call void @llvm.memset.p1i8.i64(i8 addrspace(1)* %ptr, i8 0, i64 24, i32 8, i1 false)
+  call void @foo()
+  ret void
+}
+
+declare void @foo()
+declare void @llvm.memset.p1i8.i64(i8 addrspace(1)*, i8, i64, i32, i1)
+
+declare void @do_safepoint()
+define void @gc.safepoint_poll() {
+  call void @do_safepoint()
+  ret void
+}
diff --git a/test/Transforms/PlaceSafepoints/patchable-statepoints.ll b/test/Transforms/PlaceSafepoints/patchable-statepoints.ll
new file mode 100644
index 00000000000..fd653174a6a
--- /dev/null
+++ b/test/Transforms/PlaceSafepoints/patchable-statepoints.ll
@@ -0,0 +1,44 @@
+; RUN: opt -place-safepoints -S < %s | FileCheck %s
+
+declare void @f()
+declare i32 @personality_function()
+
+define void @test_id() gc "statepoint-example" personality i32 ()* @personality_function {
+; CHECK-LABEL: @test_id(
+entry:
+; CHECK-LABEL: entry:
+; CHECK: invoke i32 (i64, i32, void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 100, i32 0, void ()* @f
+  invoke void @f()  "statepoint-id"="100" to label %normal_return unwind label %exceptional_return
+
+normal_return:
+  ret void
+
+exceptional_return:
+  %landing_pad4 = landingpad {i8*, i32} cleanup
+  ret void
+}
+
+define void @test_num_patch_bytes() gc "statepoint-example" personality i32 ()* @personality_function {
+; CHECK-LABEL: @test_num_patch_bytes(
+entry:
+; CHECK-LABEL: entry:
+; CHECK: invoke i32 (i64, i32, void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 2882400000, i32 99, void ()* @f,
+  invoke void @f()  "statepoint-num-patch-bytes"="99" to label %normal_return unwind label %exceptional_return
+
+normal_return:
+  ret void
+
+exceptional_return:
+  %landing_pad4 = landingpad {i8*, i32} cleanup
+  ret void
+}
+
+declare void @do_safepoint()
+define void @gc.safepoint_poll() {
+entry:
+  call void @do_safepoint()
+  ret void
+}
+
+; CHECK-NOT: statepoint-id
+; CHECK-NOT: statepoint-num-patch_bytes
diff --git a/test/Transforms/PlaceSafepoints/split-backedge.ll b/test/Transforms/PlaceSafepoints/split-backedge.ll
index 176b54f558e..b9fad45709d 100644
--- a/test/Transforms/PlaceSafepoints/split-backedge.ll
+++ b/test/Transforms/PlaceSafepoints/split-backedge.ll
@@ -22,12 +22,12 @@ exit:
 ; to be sure this keeps working.
 define void @test2(i32, i1 %cond) gc "statepoint-example" {
 ; CHECK-LABEL: @test2
-; CHECK-LABE: loop.loopexit.split
-; CHECK: gc.statepoint
-; CHECK-NEXT: br label %loop
-; CHECK-LABEL: loop2.loop2_crit_edge
+; CHECK-LABEL: loop2.loop2_crit_edge:
 ; CHECK: gc.statepoint
 ; CHECK-NEXT: br label %loop2
+; CHECK-LABEL: loop2.loop_crit_edge:
+; CHECK: gc.statepoint
+; CHECK-NEXT: br label %loop
 entry:
   br label %loop
 
@@ -43,4 +43,4 @@ define void @gc.safepoint_poll() {
 entry:
   call void @do_safepoint()
   ret void
-}
+}
\ No newline at end of file
diff --git a/test/Transforms/PlaceSafepoints/statepoint-calling-conventions.ll b/test/Transforms/PlaceSafepoints/statepoint-calling-conventions.ll
new file mode 100644
index 00000000000..6048f63c7f7
--- /dev/null
+++ b/test/Transforms/PlaceSafepoints/statepoint-calling-conventions.ll
@@ -0,0 +1,42 @@
+; RUN: opt -place-safepoints -S < %s | FileCheck %s
+
+; Ensure that the gc.statepoint calls / invokes we generate carry over
+; the right calling conventions.
+
+define i64 addrspace(1)* @test_invoke_format(i64 addrspace(1)* %obj, i64 addrspace(1)* %obj1) gc "statepoint-example" personality i32 ()* @personality {
+; CHECK-LABEL: @test_invoke_format(
+; CHECK-LABEL: entry:
+; CHECK: invoke coldcc i32 (i64, i32, i64 addrspace(1)* (i64 addrspace(1)*)*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_p1i64p1i64f(i64 2882400000, i32 0, i64 addrspace(1)* (i64 addrspace(1)*)* @callee, i32 1, i32 0, i64 addrspace(1)* %obj, i32 0, i32 0)
+entry:
+  %ret_val = invoke coldcc i64 addrspace(1)* @callee(i64 addrspace(1)* %obj)
+               to label %normal_return unwind label %exceptional_return
+
+normal_return:
+  ret i64 addrspace(1)* %ret_val
+
+exceptional_return:
+  %landing_pad4 = landingpad {i8*, i32}
+          cleanup
+  ret i64 addrspace(1)* %obj1
+}
+
+define i64 addrspace(1)* @test_call_format(i64 addrspace(1)* %obj, i64 addrspace(1)* %obj1) gc "statepoint-example" {
+; CHECK-LABEL: @test_call_format(
+; CHECK-LABEL: entry:
+; CHECK: call coldcc i32 (i64, i32, i64 addrspace(1)* (i64 addrspace(1)*)*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_p1i64p1i64f(i64 2882400000, i32 0, i64 addrspace(1)* (i64 addrspace(1)*)* @callee, i32 1, i32 0, i64 addrspace(1)* %obj, i32 0, i32 0)
+entry:
+  %ret_val = call coldcc i64 addrspace(1)* @callee(i64 addrspace(1)* %obj)
+  ret i64 addrspace(1)* %ret_val
+}
+
+; This function is inlined when inserting a poll.
+declare void @do_safepoint()
+define void @gc.safepoint_poll() {
+; CHECK-LABEL: gc.safepoint_poll
+entry:
+  call void @do_safepoint()
+  ret void
+}
+
+declare coldcc i64 addrspace(1)* @callee(i64 addrspace(1)*)
+declare i32 @personality()
diff --git a/test/Transforms/PlaceSafepoints/statepoint-coreclr.ll b/test/Transforms/PlaceSafepoints/statepoint-coreclr.ll
new file mode 100644
index 00000000000..dc468966877
--- /dev/null
+++ b/test/Transforms/PlaceSafepoints/statepoint-coreclr.ll
@@ -0,0 +1,31 @@
+; RUN: opt %s -S -place-safepoints | FileCheck %s
+
+; Basic test to make sure that safepoints are placed
+; for CoreCLR GC
+
+declare void @foo()
+
+define void @test_simple_call() gc "coreclr" {
+; CHECK-LABEL: test_simple_call
+entry:
+  br label %other
+other:
+; CHECK-LABEL: other
+; CHECK: statepoint
+; CHECK-NOT: gc.result
+  call void @foo()
+  ret void
+}
+
+; This function is inlined when inserting a poll.  To avoid recursive
+; issues, make sure we don't place safepoints in it.
+declare void @do_safepoint()
+define void @gc.safepoint_poll() {
+; CHECK-LABEL: gc.safepoint_poll
+; CHECK-LABEL: entry
+; CHECK-NEXT: do_safepoint
+; CHECK-NEXT: ret void
+entry:
+  call void @do_safepoint()
+  ret void
+}
diff --git a/test/Transforms/PlaceSafepoints/statepoint-format.ll b/test/Transforms/PlaceSafepoints/statepoint-format.ll
new file mode 100644
index 00000000000..496091f552d
--- /dev/null
+++ b/test/Transforms/PlaceSafepoints/statepoint-format.ll
@@ -0,0 +1,42 @@
+; RUN: opt -place-safepoints -S < %s | FileCheck %s
+
+; Ensure that the gc.statepoint calls / invokes we generate have the
+; set of arguments we expect it to have.
+
+define i64 addrspace(1)* @test_invoke_format(i64 addrspace(1)* %obj, i64 addrspace(1)* %obj1) gc "statepoint-example" personality i32 ()* @personality {
+; CHECK-LABEL: @test_invoke_format(
+; CHECK-LABEL: entry:
+; CHECK: invoke i32 (i64, i32, i64 addrspace(1)* (i64 addrspace(1)*)*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_p1i64p1i64f(i64 2882400000, i32 0, i64 addrspace(1)* (i64 addrspace(1)*)* @callee, i32 1, i32 0, i64 addrspace(1)* %obj, i32 0, i32 0)
+entry:
+  %ret_val = invoke i64 addrspace(1)* @callee(i64 addrspace(1)* %obj)
+               to label %normal_return unwind label %exceptional_return
+
+normal_return:
+  ret i64 addrspace(1)* %ret_val
+
+exceptional_return:
+  %landing_pad4 = landingpad {i8*, i32}
+          cleanup
+  ret i64 addrspace(1)* %obj1
+}
+
+define i64 addrspace(1)* @test_call_format(i64 addrspace(1)* %obj, i64 addrspace(1)* %obj1) gc "statepoint-example" {
+; CHECK-LABEL: @test_call_format(
+; CHECK-LABEL: entry:
+; CHECK: call i32 (i64, i32, i64 addrspace(1)* (i64 addrspace(1)*)*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_p1i64p1i64f(i64 2882400000, i32 0, i64 addrspace(1)* (i64 addrspace(1)*)* @callee, i32 1, i32 0, i64 addrspace(1)* %obj, i32 0, i32 0)
+entry:
+  %ret_val = call i64 addrspace(1)* @callee(i64 addrspace(1)* %obj)
+  ret i64 addrspace(1)* %ret_val
+}
+
+; This function is inlined when inserting a poll.
+declare void @do_safepoint()
+define void @gc.safepoint_poll() {
+; CHECK-LABEL: gc.safepoint_poll
+entry:
+  call void @do_safepoint()
+  ret void
+}
+
+declare i64 addrspace(1)* @callee(i64 addrspace(1)*)
+declare i32 @personality()
diff --git a/test/Transforms/PlaceSafepoints/statepoint-frameescape.ll b/test/Transforms/PlaceSafepoints/statepoint-frameescape.ll
new file mode 100644
index 00000000000..c4e250957a8
--- /dev/null
+++ b/test/Transforms/PlaceSafepoints/statepoint-frameescape.ll
@@ -0,0 +1,29 @@
+; RUN: opt %s -S -place-safepoints | FileCheck %s
+
+declare void @llvm.localescape(...)
+
+; Do we insert the entry safepoint after the localescape intrinsic?
+define void @parent() gc "statepoint-example" {
+; CHECK-LABEL: @parent
+entry:
+; CHECK-LABEL: entry
+; CHECK-NEXT: alloca
+; CHECK-NEXT: localescape
+; CHECK-NEXT: statepoint
+  %ptr = alloca i32
+  call void (...) @llvm.localescape(i32* %ptr)
+  ret void
+}
+
+; This function is inlined when inserting a poll.  To avoid recursive 
+; issues, make sure we don't place safepoints in it.
+declare void @do_safepoint()
+define void @gc.safepoint_poll() {
+; CHECK-LABEL: gc.safepoint_poll
+; CHECK-LABEL: entry
+; CHECK-NEXT: do_safepoint
+; CHECK-NEXT: ret void 
+entry:
+  call void @do_safepoint()
+  ret void
+}
\ No newline at end of file
diff --git a/test/Transforms/PruneEH/pr23971.ll b/test/Transforms/PruneEH/pr23971.ll
new file mode 100644
index 00000000000..8a8a591fff0
--- /dev/null
+++ b/test/Transforms/PruneEH/pr23971.ll
@@ -0,0 +1,21 @@
+; RUN: opt -S -prune-eh < %s | FileCheck %s
+
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+define void @f() #0 {
+entry:
+  call void asm sideeffect "ret\0A\09", "~{dirflag},~{fpsr},~{flags}"()
+  unreachable
+}
+
+define i32 @g() {
+entry:
+  call void @f()
+  ret i32 42
+}
+
+; CHECK-LABEL: define i32 @g()
+; CHECK: ret i32 42
+
+attributes #0 = { naked noinline }
diff --git a/test/Transforms/PruneEH/recursivetest.ll b/test/Transforms/PruneEH/recursivetest.ll
index bc002ae70d3..0b2399a9a29 100644
--- a/test/Transforms/PruneEH/recursivetest.ll
+++ b/test/Transforms/PruneEH/recursivetest.ll
@@ -1,23 +1,23 @@
 ; RUN: opt < %s -prune-eh -S | not grep invoke
 
-define internal i32 @foo() {
+define internal i32 @foo() personality i32 (...)* @__gxx_personality_v0 {
 	invoke i32 @foo( )
 			to label %Normal unwind label %Except		; <i32>:1 [#uses=0]
 Normal:		; preds = %0
 	ret i32 12
 Except:		; preds = %0
-        landingpad { i8*, i32 } personality i32 (...)* @__gxx_personality_v0
+        landingpad { i8*, i32 }
                 catch i8* null
 	ret i32 123
 }
 
-define i32 @caller() {
+define i32 @caller() personality i32 (...)* @__gxx_personality_v0 {
 	invoke i32 @foo( )
 			to label %Normal unwind label %Except		; <i32>:1 [#uses=0]
 Normal:		; preds = %0
 	ret i32 0
 Except:		; preds = %0
-        landingpad { i8*, i32 } personality i32 (...)* @__gxx_personality_v0
+        landingpad { i8*, i32 }
                 catch i8* null
 	ret i32 1
 }
diff --git a/test/Transforms/PruneEH/seh-nounwind.ll b/test/Transforms/PruneEH/seh-nounwind.ll
index 4b69ae4b28d..043a792eb6d 100644
--- a/test/Transforms/PruneEH/seh-nounwind.ll
+++ b/test/Transforms/PruneEH/seh-nounwind.ll
@@ -10,13 +10,13 @@ entry:
   ret i32 %div
 }
 
-define i32 @main() nounwind {
+define i32 @main() nounwind personality i8* bitcast (i32 (...)* @__C_specific_handler to i8*) {
 entry:
   %call = invoke i32 @div(i32 10, i32 0)
           to label %__try.cont unwind label %lpad
 
 lpad:
-  %0 = landingpad { i8*, i32 } personality i8* bitcast (i32 (...)* @__C_specific_handler to i8*)
+  %0 = landingpad { i8*, i32 }
           catch i8* null
   br label %__try.cont
 
diff --git a/test/Transforms/PruneEH/simpletest.ll b/test/Transforms/PruneEH/simpletest.ll
index 6154a80b68a..cbc5592cfb2 100644
--- a/test/Transforms/PruneEH/simpletest.ll
+++ b/test/Transforms/PruneEH/simpletest.ll
@@ -7,7 +7,7 @@ define internal void @foo() {
 	ret void
 }
 
-define i32 @caller() {
+define i32 @caller() personality i32 (...)* @__gxx_personality_v0 {
 	invoke void @foo( )
 			to label %Normal unwind label %Except
 
@@ -15,7 +15,7 @@ Normal:		; preds = %0
 	ret i32 0
 
 Except:		; preds = %0
-        landingpad { i8*, i32 } personality i32 (...)* @__gxx_personality_v0
+        landingpad { i8*, i32 }
                 catch i8* null
 	ret i32 1
 }
diff --git a/test/Transforms/Reassociate/basictest.ll b/test/Transforms/Reassociate/basictest.ll
index 015d3b0bee9..c557017b4c6 100644
--- a/test/Transforms/Reassociate/basictest.ll
+++ b/test/Transforms/Reassociate/basictest.ll
@@ -169,7 +169,11 @@ define i32 @test11(i32 %W) {
 ; CHECK-NEXT: ret i32
 }
 
+declare void @mumble(i32)
+
 define i32 @test12(i32 %X) {
+  %X.neg = sub nsw nuw i32 0, %X
+  call void @mumble(i32 %X.neg)
   %A = sub i32 1, %X
   %B = sub i32 2, %X
   %C = sub i32 3, %X
@@ -177,8 +181,8 @@ define i32 @test12(i32 %X) {
   %Z = add i32 %Y, %C
   ret i32 %Z
 ; CHECK-LABEL: @test12
-; CHECK-NEXT: mul i32 %X, -3
-; CHECK-NEXT: add i32{{.*}}, 6
+; CHECK: %[[mul:.*]] = mul i32 %X, -3
+; CHECK-NEXT: add i32 %[[mul]], 6
 ; CHECK-NEXT: ret i32
 }
 
@@ -202,8 +206,8 @@ define i32 @test14(i32 %X1, i32 %X2) {
   ret i32 %D
 
 ; CHECK-LABEL: @test14
-; CHECK-NEXT: sub i32 %X1, %X2
-; CHECK-NEXT: mul i32 %B2, 47
+; CHECK-NEXT: %[[SUB:.*]] = sub i32 %X1, %X2
+; CHECK-NEXT: mul i32 %[[SUB]], 47
 ; CHECK-NEXT: ret i32
 }
 
diff --git a/test/Transforms/Reassociate/canonicalize-neg-const.ll b/test/Transforms/Reassociate/canonicalize-neg-const.ll
index e85a963f6dd..465460cb53b 100644
--- a/test/Transforms/Reassociate/canonicalize-neg-const.ll
+++ b/test/Transforms/Reassociate/canonicalize-neg-const.ll
@@ -49,18 +49,6 @@ define double @test3(double %x, double %y) {
   ret double %mul3
 }
 
-; Canonicalize (x - -1234 * y)
-define i64 @test4(i64 %x, i64 %y) {
-; CHECK-LABEL: @test4
-; CHECK-NEXT: mul i64 %y, 1234
-; CHECK-NEXT: add i64 %mul, %x
-; CHECK-NEXT: ret i64 %sub
-
-  %mul = mul i64 %y, -1234
-  %sub = sub i64 %x, %mul
-  ret i64 %sub
-}
-
 ; Canonicalize (x - -0.1234 * y)
 define double @test5(double %x, double %y) {
 ; CHECK-LABEL: @test5
@@ -156,3 +144,13 @@ define double @test12(double %x, double %y) {
   %add = fadd double %div, %x
   ret double %add
 }
+
+; Don't create an NSW violation
+define i4 @test13(i4 %x) {
+; CHECK-LABEL: @test13
+; CHECK-NEXT: %[[mul:.*]] = mul nsw i4 %x, -2
+; CHECK-NEXT: %[[add:.*]] = add i4 %[[mul]], 3
+  %mul = mul nsw i4 %x, -2
+  %add = add i4 %mul, 3
+  ret i4 %add
+}
diff --git a/test/Transforms/Reassociate/wrap-flags.ll b/test/Transforms/Reassociate/wrap-flags.ll
index e3304b6a7bb..f56719d32c2 100644
--- a/test/Transforms/Reassociate/wrap-flags.ll
+++ b/test/Transforms/Reassociate/wrap-flags.ll
@@ -32,3 +32,14 @@ entry:
   %mul2 = add i32 %mul, 1
   ret i32 %mul2
 }
+
+; CHECK-LABEL: @pr23926(
+; CHECK:       %[[X1_neg:.*]] = sub i2 0, %X1
+; CHECK-NEXT:  %[[sub_one:.*]] = add i2 %[[X1_neg]], -1
+; CHECK-NEXT:  %[[add:.*]] = add i2 %[[sub_one]], %X2
+; CHECK-NEXT:  ret i2 %[[add]]
+define i2 @pr23926(i2 %X1, i2 %X2) {
+  %add = add nuw i2 %X1, 1
+  %sub = sub nuw nsw i2 %X2, %add
+  ret i2 %sub
+}
diff --git a/test/Transforms/Reg2Mem/crash.ll b/test/Transforms/Reg2Mem/crash.ll
index 02fed94b852..52dfeaf0265 100644
--- a/test/Transforms/Reg2Mem/crash.ll
+++ b/test/Transforms/Reg2Mem/crash.ll
@@ -13,13 +13,13 @@ declare void @f4_()
 
 declare void @_Z12xxxdtsP10xxxpq()
 
-define hidden void @_ZN12xxxyzIi9xxxwLi29ELi0EE4f3NewES0_i() ssp align 2 {
+define hidden void @_ZN12xxxyzIi9xxxwLi29ELi0EE4f3NewES0_i() ssp align 2 personality i8* bitcast (i32 (...)* @__gxx_personality_sj0 to i8*) {
 bb:
   invoke void @f4_()
           to label %bb1 unwind label %.thread
 
 .thread:                                          ; preds = %bb
-  %tmp = landingpad { i8*, i32 } personality i8* bitcast (i32 (...)* @__gxx_personality_sj0 to i8*)
+  %tmp = landingpad { i8*, i32 }
           cleanup
   br label %bb13
 
@@ -32,13 +32,13 @@ bb1:                                              ; preds = %bb
           to label %bb6 unwind label %bb2
 
 bb2:                                              ; preds = %.noexc
-  %tmp3 = landingpad { i8*, i32 } personality i8* bitcast (i32 (...)* @__gxx_personality_sj0 to i8*)
+  %tmp3 = landingpad { i8*, i32 }
           cleanup
   invoke void @f3()
           to label %.body unwind label %bb4
 
 bb4:                                              ; preds = %bb2
-  %tmp5 = landingpad { i8*, i32 } personality i8* bitcast (i32 (...)* @__gxx_personality_sj0 to i8*)
+  %tmp5 = landingpad { i8*, i32 }
           catch i8* null
   unreachable
 
@@ -54,13 +54,13 @@ bb7:                                              ; preds = %_ZN6xxxdIN12xxxyzIi
   ret void
 
 bb8:                                              ; preds = %_ZN6xxxdIN12xxxyzIi9xxxwLi29ELi0EE4fr1jS3_.exit
-  %tmp9 = landingpad { i8*, i32 } personality i8* bitcast (i32 (...)* @__gxx_personality_sj0 to i8*)
+  %tmp9 = landingpad { i8*, i32 }
           cleanup
   br label %_ZN10xxxpqdlev.exit
 
 bb10:                                             ; preds = %bb6, %bb1
   %.1 = phi i1 [ true, %bb1 ], [ false, %bb6 ]
-  %tmp11 = landingpad { i8*, i32 } personality i8* bitcast (i32 (...)* @__gxx_personality_sj0 to i8*)
+  %tmp11 = landingpad { i8*, i32 }
           cleanup
   br label %.body
 
@@ -80,7 +80,7 @@ _ZN10xxxpqdlev.exit:                              ; preds = %bb13, %bb12, %bb8
   resume { i8*, i32 } undef
 
 bb14:                                             ; preds = %bb13, %.body
-  %tmp15 = landingpad { i8*, i32 } personality i8* bitcast (i32 (...)* @__gxx_personality_sj0 to i8*)
+  %tmp15 = landingpad { i8*, i32 }
           catch i8* null
   unreachable
 }
diff --git a/test/Transforms/RewriteStatepointsForGC/base-pointers-1.ll b/test/Transforms/RewriteStatepointsForGC/base-pointers-1.ll
index e6e5efcc26d..8c360cd7242 100644
--- a/test/Transforms/RewriteStatepointsForGC/base-pointers-1.ll
+++ b/test/Transforms/RewriteStatepointsForGC/base-pointers-1.ll
@@ -1,6 +1,6 @@
 ; RUN: opt %s -rewrite-statepoints-for-gc -spp-print-base-pointers -S 2>&1 | FileCheck %s
 
-; CHECK: derived %merged_value base %base_phi
+; CHECK: derived %merged_value base %merged_value.base
 
 declare void @site_for_call_safpeoint()
 
@@ -18,11 +18,11 @@ there:
 
 merge:
 ; CHECK-LABEL: merge:
-; CHECK:   %base_phi = phi i64 addrspace(1)* [ %base_obj_x, %here ], [ %base_obj_y, %there ]
+; CHECK:   %merged_value.base = phi i64 addrspace(1)* [ %base_obj_x, %here ], [ %base_obj_y, %there ]
   %merged_value = phi i64 addrspace(1)* [ %x, %here ], [ %y, %there ]
-  %safepoint_token = call i32 (void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(void ()* @site_for_call_safpeoint, i32 0, i32 0, i32 5, i32 0, i32 -1, i32 0, i32 0, i32 0)
+  %safepoint_token = call i32 (i64, i32, void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 0, i32 0, void ()* @site_for_call_safpeoint, i32 0, i32 0, i32 0, i32 5, i32 0, i32 -1, i32 0, i32 0, i32 0)
   ret i64 addrspace(1)* %merged_value
 }
 
 declare void @foo()
-declare i32 @llvm.experimental.gc.statepoint.p0f_isVoidf(void ()*, i32, i32, ...)
+declare i32 @llvm.experimental.gc.statepoint.p0f_isVoidf(i64, i32, void ()*, i32, i32, ...)
diff --git a/test/Transforms/RewriteStatepointsForGC/base-pointers-10.ll b/test/Transforms/RewriteStatepointsForGC/base-pointers-10.ll
index fa800eadc55..61574e53ef8 100644
--- a/test/Transforms/RewriteStatepointsForGC/base-pointers-10.ll
+++ b/test/Transforms/RewriteStatepointsForGC/base-pointers-10.ll
@@ -1,8 +1,9 @@
 ; RUN: opt %s -rewrite-statepoints-for-gc -spp-print-base-pointers -S 2>&1 | FileCheck %s
 
-; CHECK: derived %next_x base %base_obj_x
-; CHECK: derived %next_y base %base_obj_y
-; CHECK: derived %next base %base_phi
+; CHECK: Base Pairs (w/o Relocation):
+; CHECK-DAG: derived %next base %next.base
+; CHECK-DAG: derived %next_x base %base_obj_x
+; CHECK-DAG: derived %next_y base %base_obj_y
 
 declare i1 @runtime_value()
 declare void @do_safepoint()
@@ -30,8 +31,8 @@ false:
 
 merge:
   %next = phi i64 addrspace(1)* [ %next_x, %true ], [ %next_y, %false ]
-  %safepoint_token = call i32 (void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(void ()* @do_safepoint, i32 0, i32 0, i32 5, i32 0, i32 -1, i32 0, i32 0, i32 0)
+  %safepoint_token = call i32 (i64, i32, void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 0, i32 0, void ()* @do_safepoint, i32 0, i32 0, i32 0, i32 5, i32 0, i32 -1, i32 0, i32 0, i32 0)
   br label %loop
 }
 
-declare i32 @llvm.experimental.gc.statepoint.p0f_isVoidf(void ()*, i32, i32, ...)
+declare i32 @llvm.experimental.gc.statepoint.p0f_isVoidf(i64, i32, void ()*, i32, i32, ...)
diff --git a/test/Transforms/RewriteStatepointsForGC/base-pointers-11.ll b/test/Transforms/RewriteStatepointsForGC/base-pointers-11.ll
index c7fd32e6a2e..2a7a7444ade 100644
--- a/test/Transforms/RewriteStatepointsForGC/base-pointers-11.ll
+++ b/test/Transforms/RewriteStatepointsForGC/base-pointers-11.ll
@@ -12,15 +12,15 @@ entry:
 loop:                                             ; preds = %loop, %entry
 ; CHECK-LABEL: loop:
 ; CHECK: phi i64 addrspace(1)*
-; CHECK-DAG:  [ %base_obj.relocated, %loop ] 
+; CHECK-DAG:  [ %base_obj.relocated.casted, %loop ] 
 ; CHECK-DAG:  [ %base_obj, %entry ]
 ; CHECK:  %current = phi i64 addrspace(1)* 
 ; CHECK-DAG:  [ %obj, %entry ]
-; CHECK-DAG:  [ %next.relocated, %loop ]
+; CHECK-DAG:  [ %next.relocated.casted, %loop ]
   %current = phi i64 addrspace(1)* [ %obj, %entry ], [ %next, %loop ]
   %next = getelementptr i64, i64 addrspace(1)* %current, i32 1
-  %safepoint_token = call i32 (void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(void ()* @do_safepoint, i32 0, i32 0, i32 5, i32 0, i32 -1, i32 0, i32 0, i32 0)
+  %safepoint_token = call i32 (i64, i32, void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 0, i32 0, void ()* @do_safepoint, i32 0, i32 0, i32 0, i32 5, i32 0, i32 -1, i32 0, i32 0, i32 0)
   br label %loop
 }
 
-declare i32 @llvm.experimental.gc.statepoint.p0f_isVoidf(void ()*, i32, i32, ...)
+declare i32 @llvm.experimental.gc.statepoint.p0f_isVoidf(i64, i32, void ()*, i32, i32, ...)
\ No newline at end of file
diff --git a/test/Transforms/RewriteStatepointsForGC/base-pointers-2.ll b/test/Transforms/RewriteStatepointsForGC/base-pointers-2.ll
index 6509d23ca03..c2877d8b539 100644
--- a/test/Transforms/RewriteStatepointsForGC/base-pointers-2.ll
+++ b/test/Transforms/RewriteStatepointsForGC/base-pointers-2.ll
@@ -13,9 +13,9 @@ there:
 
 merge:
   %merged_value = phi i64 addrspace(1)* [ %base_obj, %entry ], [ %derived_obj, %there ]
-  %safepoint_token = call i32 (void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(void ()* @foo, i32 0, i32 0, i32 5, i32 0, i32 -1, i32 0, i32 0, i32 0)
+  %safepoint_token = call i32 (i64, i32, void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 0, i32 0, void ()* @foo, i32 0, i32 0, i32 0, i32 5, i32 0, i32 -1, i32 0, i32 0, i32 0)
   ret i64 addrspace(1)* %merged_value
 }
 
 declare void @foo()
-declare i32 @llvm.experimental.gc.statepoint.p0f_isVoidf(void ()*, i32, i32, ...)
+declare i32 @llvm.experimental.gc.statepoint.p0f_isVoidf(i64, i32, void ()*, i32, i32, ...)
\ No newline at end of file
diff --git a/test/Transforms/RewriteStatepointsForGC/base-pointers-3.ll b/test/Transforms/RewriteStatepointsForGC/base-pointers-3.ll
index 7a80f3163b4..f72201b5138 100644
--- a/test/Transforms/RewriteStatepointsForGC/base-pointers-3.ll
+++ b/test/Transforms/RewriteStatepointsForGC/base-pointers-3.ll
@@ -12,9 +12,9 @@ loop:
   %current.i32 = bitcast i64 addrspace(1)* %current to i32 addrspace(1)*
   %next.i32 = getelementptr i32, i32 addrspace(1)* %current.i32, i32 1
   %next.i64 = bitcast i32 addrspace(1)* %next.i32 to i64 addrspace(1)*
-  %safepoint_token = call i32 (void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(void ()* @do_safepoint, i32 0, i32 0, i32 5, i32 0, i32 -1, i32 0, i32 0, i32 0)
+  %safepoint_token = call i32 (i64, i32, void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 0, i32 0, void ()* @do_safepoint, i32 0, i32 0, i32 0, i32 5, i32 0, i32 -1, i32 0, i32 0, i32 0)
   br label %loop
 }
 
 declare void @do_safepoint()
-declare i32 @llvm.experimental.gc.statepoint.p0f_isVoidf(void ()*, i32, i32, ...)
+declare i32 @llvm.experimental.gc.statepoint.p0f_isVoidf(i64, i32, void ()*, i32, i32, ...)
\ No newline at end of file
diff --git a/test/Transforms/RewriteStatepointsForGC/base-pointers-4.ll b/test/Transforms/RewriteStatepointsForGC/base-pointers-4.ll
index 33619ad4e48..9ffe08628ef 100644
--- a/test/Transforms/RewriteStatepointsForGC/base-pointers-4.ll
+++ b/test/Transforms/RewriteStatepointsForGC/base-pointers-4.ll
@@ -1,7 +1,7 @@
 ; RUN: opt %s -rewrite-statepoints-for-gc -spp-print-base-pointers -S 2>&1 | FileCheck %s
 
 
-; CHECK: derived %obj_to_consume base %base_phi
+; CHECK: derived %obj_to_consume base %obj_to_consume.base
 
 declare void @foo()
 declare i64 addrspace(1)* @generate_obj()
@@ -13,9 +13,9 @@ entry:
 
 loop:
 ; CHECK: loop:
-; CHECK:  %safepoint_token1 = call i32 (i64 addrspace(1)* ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_p1i64f(i64 addrspace(1)* ()* @generate_obj, i32 0, i32 0, i32 5, i32 0, i32 -1, i32 0, i32 0, i3
+; CHECK:  %safepoint_token1 = call i32 (i64, i32, i64 addrspace(1)* ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_p1i64f(i64 0, i32 0, i64 addrspace(1)* ()* @generate_obj, i32 0, i32 0, i32 0, i32 5, i32 0, i32 -1, i32 0, i32 0, i3
 ; CHECK-NEXT:  %obj2 = call i64 addrspace(1)* @llvm.experimental.gc.result
-  %safepoint_token1 = call i32 (i64 addrspace(1)* ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_p1i64f(i64 addrspace(1)* ()* @generate_obj, i32 0, i32 0, i32 5, i32 0, i32 -1, i32 0, i32 0, i32 0)
+  %safepoint_token1 = call i32 (i64, i32, i64 addrspace(1)* ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_p1i64f(i64 0, i32 0, i64 addrspace(1)* ()* @generate_obj, i32 0, i32 0, i32 0, i32 5, i32 0, i32 -1, i32 0, i32 0, i32 0)
   %obj2 = call i64 addrspace(1)* @llvm.experimental.gc.result.p1i64(i32 %safepoint_token1)
   switch i32 %condition, label %dest_a [
     i32 0, label %dest_b
@@ -33,21 +33,21 @@ dest_c:
 
 merge:
 ; CHECK: merge:
-; CHECK:  %base_phi = phi i64 addrspace(1)* [ %obj2, %dest_a ], [ null, %dest_b ], [ null, %dest_c ]
+; CHECK:  %obj_to_consume.base = phi i64 addrspace(1)* [ %obj2, %dest_a ], [ null, %dest_b ], [ null, %dest_c ]
 ; CHECK:  %obj_to_consume = phi i64 addrspace(1)* [ %obj2, %dest_a ], [ null, %dest_b ], [ null, %dest_c ]
 
   %obj_to_consume = phi i64 addrspace(1)* [ %obj2, %dest_a ], [ null, %dest_b ], [ null, %dest_c ]
-  %safepoint_token3 = call i32 (void (i64 addrspace(1)*)*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidp1i64f(void (i64 addrspace(1)*)* @consume_obj, i32 1, i32 0, i64 addrspace(1)* %obj_to_consume, i32 5, i32 0, i32 -1, i32 0, i32 0, i32 0)
+  %safepoint_token3 = call i32 (i64, i32, void (i64 addrspace(1)*)*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidp1i64f(i64 0, i32 0, void (i64 addrspace(1)*)* @consume_obj, i32 1, i32 0, i64 addrspace(1)* %obj_to_consume, i32 0, i32 5, i32 0, i32 -1, i32 0, i32 0, i32 0)
   br label %merge.split
 
 merge.split:                                      ; preds = %merge
-  %safepoint_token = call i32 (void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(void ()* @foo, i32 0, i32 0, i32 5, i32 0, i32 -1, i32 0, i32 0, i32 0)
+  %safepoint_token = call i32 (i64, i32, void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 0, i32 0, void ()* @foo, i32 0, i32 0, i32 0, i32 5, i32 0, i32 -1, i32 0, i32 0, i32 0)
   br label %loop
 }
 
 
 ; Function Attrs: nounwind
 declare i64 addrspace(1)* @llvm.experimental.gc.result.p1i64(i32) #0
-declare i32 @llvm.experimental.gc.statepoint.p0f_isVoidf(void ()*, i32, i32, ...)
-declare i32 @llvm.experimental.gc.statepoint.p0f_p1i64f(i64 addrspace(1)* ()*, i32, i32, ...)
-declare i32 @llvm.experimental.gc.statepoint.p0f_isVoidp1i64f(void (i64 addrspace(1)*)*, i32, i32, ...)
+declare i32 @llvm.experimental.gc.statepoint.p0f_isVoidf(i64, i32, void ()*, i32, i32, ...)
+declare i32 @llvm.experimental.gc.statepoint.p0f_p1i64f(i64, i32, i64 addrspace(1)* ()*, i32, i32, ...)
+declare i32 @llvm.experimental.gc.statepoint.p0f_isVoidp1i64f(i64, i32, void (i64 addrspace(1)*)*, i32, i32, ...)
diff --git a/test/Transforms/RewriteStatepointsForGC/base-pointers-5.ll b/test/Transforms/RewriteStatepointsForGC/base-pointers-5.ll
index 0800504e041..a34e4cb8ae0 100644
--- a/test/Transforms/RewriteStatepointsForGC/base-pointers-5.ll
+++ b/test/Transforms/RewriteStatepointsForGC/base-pointers-5.ll
@@ -1,6 +1,6 @@
 ; RUN: opt %s -rewrite-statepoints-for-gc -spp-print-base-pointers -S 2>&1 | FileCheck %s
 
-; CHECK: derived %merged_value base %base_phi
+; CHECK: derived %merged_value base %merged_value.base
 
 declare void @foo()
 
@@ -20,12 +20,11 @@ there:
 
 merge:
 ; CHECK: merge:
-; CHECK:  %base_phi = phi i64 addrspace(1)* [ %base_obj_x, %bump ], [ %base_obj_y, %there ]
+; CHECK:  %merged_value.base = phi i64 addrspace(1)* [ %base_obj_x, %bump ], [ %base_obj_y, %there ]
 ; CHECK-NEXT:  %merged_value = phi i64 addrspace(1)* [ %base_obj_x, %bump ], [ %y, %there ]  
   %merged_value = phi i64 addrspace(1)* [ %base_obj_x, %bump ], [ %y, %there ]
-  %safepoint_token = call i32 (void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(void ()* @foo, i32 0, i32 0, i32 5, i32 0, i32 -1, i32 0, i32 0, i32 0)
+  %safepoint_token = call i32 (i64, i32, void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 0, i32 0, void ()* @foo, i32 0, i32 0, i32 0, i32 5, i32 0, i32 -1, i32 0, i32 0, i32 0)
   ret i64 addrspace(1)* %merged_value
 }
 
-declare i32 @llvm.experimental.gc.statepoint.p0f_isVoidf(void ()*, i32, i32, ...)
-
+declare i32 @llvm.experimental.gc.statepoint.p0f_isVoidf(i64, i32, void ()*, i32, i32, ...)
diff --git a/test/Transforms/RewriteStatepointsForGC/base-pointers-6.ll b/test/Transforms/RewriteStatepointsForGC/base-pointers-6.ll
index f286ed9d60b..ace2df2cf61 100644
--- a/test/Transforms/RewriteStatepointsForGC/base-pointers-6.ll
+++ b/test/Transforms/RewriteStatepointsForGC/base-pointers-6.ll
@@ -1,6 +1,6 @@
 ; RUN: opt %s -rewrite-statepoints-for-gc -spp-print-base-pointers -S 2>&1 | FileCheck %s
 
-; CHECK: derived %merged_value base %base_phi
+; CHECK: derived %merged_value base %merged_value.base
 
 declare void @site_for_call_safpeoint()
 
@@ -30,12 +30,12 @@ there:
 
 merge:
 ; CHECK: merge:
-; CHECK:  %base_phi = phi i64 addrspace(1)* [ %base_obj_x, %merge_here ], [ %base_obj_y, %there ]
+; CHECK:  %merged_value.base = phi i64 addrspace(1)* [ %base_obj_x, %merge_here ], [ %base_obj_y, %there ]
 ; CHECK-NEXT:  %merged_value = phi i64 addrspace(1)* [ %x, %merge_here ], [ %y, %there ]  
   %merged_value = phi i64 addrspace(1)* [ %x, %merge_here ], [ %y, %there ]
-  %safepoint_token = call i32 (void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(void ()* @site_for_call_safpeoint, i32 0, i32 0, i32 5, i32 0, i32 -1, i32 0, i32 0, i32 0)
+  %safepoint_token = call i32 (i64, i32, void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 0, i32 0, void ()* @site_for_call_safpeoint, i32 0, i32 0, i32 0, i32 5, i32 0, i32 -1, i32 0, i32 0, i32 0)
   ret i64 addrspace(1)* %merged_value
 }
 
 declare void @do_safepoint()
-declare i32 @llvm.experimental.gc.statepoint.p0f_isVoidf(void ()*, i32, i32, ...)
+declare i32 @llvm.experimental.gc.statepoint.p0f_isVoidf(i64, i32, void ()*, i32, i32, ...)
diff --git a/test/Transforms/RewriteStatepointsForGC/base-pointers-7.ll b/test/Transforms/RewriteStatepointsForGC/base-pointers-7.ll
index f51e1179717..c944f3474d3 100644
--- a/test/Transforms/RewriteStatepointsForGC/base-pointers-7.ll
+++ b/test/Transforms/RewriteStatepointsForGC/base-pointers-7.ll
@@ -1,6 +1,6 @@
 ; RUN: opt %s -rewrite-statepoints-for-gc -spp-print-base-pointers -S 2>&1 | FileCheck %s
 
-; CHECK: derived %merged_value base %base_phi
+; CHECK: derived %merged_value base %merged_value.base
 
 declare void @site_for_call_safpeoint()
 
@@ -24,7 +24,7 @@ bump_here_b:
 
 merge_here:
 ; CHECK: merge_here:
-; CHECK-DAG: %base_phi
+; CHECK-DAG: %x.base
 ; CHECK-DAG: phi i64 addrspace(1)*
 ; CHECK-DAG: [ %base_obj_x, %bump_here_a ]
 ; CHECK-DAG: [ %base_obj_y, %bump_here_b ]
@@ -37,17 +37,16 @@ there:
 
 merge:
 ; CHECK: merge:
-; CHECK-DAG:  %base_phi1
+; CHECK-DAG:  %merged_value.base
 ; CHECK-DAG: phi i64 addrspace(1)*
 ; CHECK-DAG: %merge_here
 ; CHECK-DAG: [ %base_obj_y, %there ]
 ; CHECK:  %merged_value = phi i64 addrspace(1)* [ %x, %merge_here ], [ %y, %there ]  
   %merged_value = phi i64 addrspace(1)* [ %x, %merge_here ], [ %y, %there ]
 
-  %safepoint_token = call i32 (void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(void ()* @site_for_call_safpeoint, i32 0, i32 0, i32 5, i32 0, i32 -1, i32 0, i32 0, i32 0)
+  %safepoint_token = call i32 (i64, i32, void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 0, i32 0, void ()* @site_for_call_safpeoint, i32 0, i32 0, i32 0, i32 5, i32 0, i32 -1, i32 0, i32 0, i32 0)
   ret i64 addrspace(1)* %merged_value
 }
 
 declare void @do_safepoint()
-declare i32 @llvm.experimental.gc.statepoint.p0f_isVoidf(void ()*, i32, i32, ...)
-
+declare i32 @llvm.experimental.gc.statepoint.p0f_isVoidf(i64, i32, void ()*, i32, i32, ...)
diff --git a/test/Transforms/RewriteStatepointsForGC/base-pointers-8.ll b/test/Transforms/RewriteStatepointsForGC/base-pointers-8.ll
index 07bc42b2ea4..b89be7daa67 100644
--- a/test/Transforms/RewriteStatepointsForGC/base-pointers-8.ll
+++ b/test/Transforms/RewriteStatepointsForGC/base-pointers-8.ll
@@ -24,7 +24,7 @@ check_for_null:
 loop_back:
   %next_element_ptr = getelementptr i64 addrspace(1)*, i64 addrspace(1)* addrspace(1)* %current_element_ptr, i32 1
   %next_index = add i32 %index, 1
-  %safepoint_token = call i32 (void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(void ()* @do_safepoint, i32 0, i32 0, i32 5, i32 0, i32 -1, i32 0, i32 0, i32 0)
+  %safepoint_token = call i32 (i64, i32, void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 0, i32 0, void ()* @do_safepoint, i32 0, i32 0, i32 0, i32 5, i32 0, i32 -1, i32 0, i32 0, i32 0)
   br label %loop_check
 
 not_found:
@@ -35,4 +35,4 @@ found:
 }
 
 declare void @do_safepoint()
-declare i32 @llvm.experimental.gc.statepoint.p0f_isVoidf(void ()*, i32, i32, ...)
+declare i32 @llvm.experimental.gc.statepoint.p0f_isVoidf(i64, i32, void ()*, i32, i32, ...)
\ No newline at end of file
diff --git a/test/Transforms/RewriteStatepointsForGC/base-pointers-9.ll b/test/Transforms/RewriteStatepointsForGC/base-pointers-9.ll
index 973ac7eaf36..848633b2a27 100644
--- a/test/Transforms/RewriteStatepointsForGC/base-pointers-9.ll
+++ b/test/Transforms/RewriteStatepointsForGC/base-pointers-9.ll
@@ -13,9 +13,9 @@ loop:
   %condition = call i1 @runtime_value()
   %maybe_next = getelementptr i64, i64 addrspace(1)* %current, i32 1
   %next = select i1 %condition, i64 addrspace(1)* %maybe_next, i64 addrspace(1)* %current
-  %safepoint_token = call i32 (void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(void ()* @do_safepoint, i32 0, i32 0, i32 5, i32 0, i32 -1, i32 0, i32 0, i32 0)
+  %safepoint_token = call i32 (i64, i32, void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 0, i32 0, void ()* @do_safepoint, i32 0, i32 0, i32 0, i32 5, i32 0, i32 -1, i32 0, i32 0, i32 0)
   br label %loop
 }
 
 declare void @do_safepoint()
-declare i32 @llvm.experimental.gc.statepoint.p0f_isVoidf(void ()*, i32, i32, ...)
+declare i32 @llvm.experimental.gc.statepoint.p0f_isVoidf(i64, i32, void ()*, i32, i32, ...)
\ No newline at end of file
diff --git a/test/Transforms/RewriteStatepointsForGC/base-pointers.ll b/test/Transforms/RewriteStatepointsForGC/base-pointers.ll
index eab963f79f6..2b5b44aa695 100644
--- a/test/Transforms/RewriteStatepointsForGC/base-pointers.ll
+++ b/test/Transforms/RewriteStatepointsForGC/base-pointers.ll
@@ -13,10 +13,10 @@ entry:
 
 loop:
 ; CHECK: phi i64 addrspace(1)* 
-; CHECK-DAG: [ %obj.relocated, %loop ]
+; CHECK-DAG: [ %obj.relocated.casted, %loop ]
 ; CHECK-DAG: [ %obj, %entry ]
   call void @use_obj(i64 addrspace(1)* %obj)
-  %safepoint_token = call i32 (void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(void ()* @do_safepoint, i32 0, i32 0, i32 5, i32 0, i32 -1, i32 0, i32 0, i32 0)
+  %safepoint_token = call i32 (i64, i32, void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 0, i32 0, void ()* @do_safepoint, i32 0, i32 0, i32 0, i32 5, i32 0, i32 -1, i32 0, i32 0, i32 0)
   br label %loop
 }
 
@@ -55,9 +55,9 @@ define i64 addrspace(1)* @test1(i32 %caller, i8 addrspace(1)* %a, i8 addrspace(1
 
  merge:
 ; CHECK: merge:
-; CHECK-NEXT: %base_phi = phi i64 addrspace(1)* [ [[CAST_L]], %left ], [ [[CAST_L]], %left ], [ [[CAST_L]], %left ], [ [[CAST_R]], %right ], !is_base_value !0
+; CHECK-NEXT: %value.base = phi i64 addrspace(1)* [ [[CAST_L]], %left ], [ [[CAST_L]], %left ], [ [[CAST_L]], %left ], [ [[CAST_R]], %right ], !is_base_value !0
   %value = phi i64 addrspace(1)* [ %a.cast, %left], [ %a.cast, %left], [ %a.cast, %left], [ %b.cast, %right]
-  %safepoint_token = call i32 (void (i64 addrspace(1)*)*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidp1i64f(void (i64 addrspace(1)*)* @parse_point, i32 1, i32 0, i64 addrspace(1)* %value, i32 5, i32 0, i32 0, i32 0, i32 0, i32 0)
+  %safepoint_token = call i32 (i64, i32, void (i64 addrspace(1)*)*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidp1i64f(i64 0, i32 0, void (i64 addrspace(1)*)* @parse_point, i32 1, i32 0, i64 addrspace(1)* %value, i32 0, i32 5, i32 0, i32 0, i32 0, i32 0, i32 0)
 
   ret i64 addrspace(1)* %value
 }
@@ -74,16 +74,16 @@ entry:
 
 loop:                                             ; preds = %loop, %entry
 ; CHECK-LABEL: loop
-; CHECK:   %base_phi = phi i64 addrspace(1)*
+; CHECK:   %current.base = phi i64 addrspace(1)*
 ; CHECK-DAG: [ %base_obj, %entry ]
 ; Given the two selects are equivelent, so are their base phis - ideally,
 ; we'd have commoned these, but that's a missed optimization, not correctness.
-; CHECK-DAG: [ [[DISCARD:%base_select.*.relocated]], %loop ]
-; CHECK-NOT: base_phi2
+; CHECK-DAG: [ [[DISCARD:%.*.base.relocated.casted]], %loop ]
+; CHECK-NOT: extra.base
+; CHECK: next.base = select
 ; CHECK: next = select
-; CHECK: base_select
+; CHECK: extra2.base = select
 ; CHECK: extra2 = select
-; CHECK: base_select
 ; CHECK: statepoint
 ;; Both 'next' and 'extra2' are live across the backedge safepoint...
   %current = phi i64 addrspace(1)* [ %obj, %entry ], [ %next, %loop ]
@@ -91,10 +91,10 @@ loop:                                             ; preds = %loop, %entry
   %nexta = getelementptr i64, i64 addrspace(1)* %current, i32 1
   %next = select i1 %cnd, i64 addrspace(1)* %nexta, i64 addrspace(1)* %base_arg2
   %extra2 = select i1 %cnd, i64 addrspace(1)* %nexta, i64 addrspace(1)* %base_arg2
-  %safepoint_token = call i32 (void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(void ()* @foo, i32 0, i32 0, i32 5, i32 0, i32 -1, i32 0, i32 0, i32 0)
+  %safepoint_token = call i32 (i64, i32, void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 0, i32 0, void ()* @foo, i32 0, i32 0, i32 0, i32 5, i32 0, i32 -1, i32 0, i32 0, i32 0)
   br label %loop
 }
 
 declare void @foo()
-declare i32 @llvm.experimental.gc.statepoint.p0f_isVoidf(void ()*, i32, i32, ...)
-declare i32 @llvm.experimental.gc.statepoint.p0f_isVoidp1i64f(void (i64 addrspace(1)*)*, i32, i32, ...)
+declare i32 @llvm.experimental.gc.statepoint.p0f_isVoidf(i64, i32, void ()*, i32, i32, ...)
+declare i32 @llvm.experimental.gc.statepoint.p0f_isVoidp1i64f(i64, i32, void (i64 addrspace(1)*)*, i32, i32, ...)
diff --git a/test/Transforms/RewriteStatepointsForGC/basics.ll b/test/Transforms/RewriteStatepointsForGC/basics.ll
index 1720cc1c7a3..2a61924a592 100644
--- a/test/Transforms/RewriteStatepointsForGC/basics.ll
+++ b/test/Transforms/RewriteStatepointsForGC/basics.ll
@@ -1,5 +1,5 @@
 ; This is a collection of really basic tests for gc.statepoint rewriting.
-; RUN:  opt %s -rewrite-statepoints-for-gc -S | FileCheck %s
+; RUN:  opt %s -rewrite-statepoints-for-gc -spp-rematerialization-threshold=0 -S | FileCheck %s
 
 declare void @foo()
 
@@ -10,7 +10,7 @@ define i8 addrspace(1)* @test1(i8 addrspace(1)* %obj) gc "statepoint-example" {
 ; CHECK-NEXT: gc.statepoint
 ; CHECK-NEXT: %obj.relocated = call coldcc i8 addrspace(1)*
 entry:
-  call i32 (void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(void ()* @foo, i32 0, i32 0, i32 5, i32 0, i32 -1, i32 0, i32 0, i32 0)
+  call i32 (i64, i32, void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 0, i32 0, void ()* @foo, i32 0, i32 0, i32 0, i32 5, i32 0, i32 -1, i32 0, i32 0, i32 0)
   ret i8 addrspace(1)* %obj
 }
 
@@ -23,8 +23,8 @@ define i8 addrspace(1)* @test2(i8 addrspace(1)* %obj) gc "statepoint-example" {
 ; CHECK-NEXT: gc.statepoint
 ; CHECK-NEXT: %obj.relocated1 = call coldcc i8 addrspace(1)*
 entry:
-  call i32 (void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(void ()* @foo, i32 0, i32 0, i32 5, i32 0, i32 -1, i32 0, i32 0, i32 0)
-  call i32 (void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(void ()* @foo, i32 0, i32 0, i32 5, i32 0, i32 -1, i32 0, i32 0, i32 0)
+  call i32 (i64, i32, void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 0, i32 0, void ()* @foo, i32 0, i32 0, i32 0, i32 5, i32 0, i32 -1, i32 0, i32 0, i32 0)
+  call i32 (i64, i32, void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 0, i32 0, void ()* @foo, i32 0, i32 0, i32 0, i32 5, i32 0, i32 -1, i32 0, i32 0, i32 0)
   ret i8 addrspace(1)* %obj
 }
 
@@ -39,7 +39,7 @@ define i8 @test3(i8 addrspace(1)* %obj) gc "statepoint-example" {
 ; CHECK-NEXT: load i8, i8 addrspace(1)* %obj.relocated
 entry:
   %derived = getelementptr i8, i8 addrspace(1)* %obj, i64 10
-  call i32 (void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(void ()* @foo, i32 0, i32 0, i32 5, i32 0, i32 -1, i32 0, i32 0, i32 0)
+  call i32 (i64, i32, void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 0, i32 0, void ()* @foo, i32 0, i32 0, i32 0, i32 5, i32 0, i32 -1, i32 0, i32 0, i32 0)
 
   %a = load i8, i8 addrspace(1)* %derived
   %b = load i8, i8 addrspace(1)* %obj
@@ -57,14 +57,14 @@ taken:
 ; CHECK-LABEL: taken:
 ; CHECK-NEXT: gc.statepoint
 ; CHECK-NEXT: %obj.relocated = call coldcc i8 addrspace(1)*
-  call i32 (void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(void ()* @foo, i32 0, i32 0, i32 5, i32 0, i32 -1, i32 0, i32 0, i32 0)
+  call i32 (i64, i32, void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 0, i32 0, void ()* @foo, i32 0, i32 0, i32 0, i32 5, i32 0, i32 -1, i32 0, i32 0, i32 0)
   br label %merge
 
 untaken:
 ; CHECK-LABEL: untaken:
 ; CHECK-NEXT: gc.statepoint
 ; CHECK-NEXT: %obj.relocated1 = call coldcc i8 addrspace(1)*
-  call i32 (void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(void ()* @foo, i32 0, i32 0, i32 5, i32 0, i32 -1, i32 0, i32 0, i32 0)
+  call i32 (i64, i32, void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 0, i32 0, void ()* @foo, i32 0, i32 0, i32 0, i32 5, i32 0, i32 -1, i32 0, i32 0, i32 0)
   br label %merge
 
 merge:
@@ -81,8 +81,8 @@ define i8 addrspace(1)* @test5(i8 addrspace(1)* %obj) gc "ocaml" {
 ; CHECK-NEXT: gc.statepoint
 ; CHECK-NOT: %obj.relocated = call coldcc i8 addrspace(1)*
 entry:
-  call i32 (void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(void ()* @foo, i32 0, i32 0, i32 5, i32 0, i32 -1, i32 0, i32 0, i32 0)
+  call i32 (i64, i32, void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 0, i32 0, void ()* @foo, i32 0, i32 0, i32 0, i32 5, i32 0, i32 -1, i32 0, i32 0, i32 0)
   ret i8 addrspace(1)* %obj
 }
 
-declare i32 @llvm.experimental.gc.statepoint.p0f_isVoidf(void ()*, i32, i32, ...)
+declare i32 @llvm.experimental.gc.statepoint.p0f_isVoidf(i64, i32, void ()*, i32, i32, ...)
\ No newline at end of file
diff --git a/test/Transforms/RewriteStatepointsForGC/constants.ll b/test/Transforms/RewriteStatepointsForGC/constants.ll
new file mode 100644
index 00000000000..a30fdd7034a
--- /dev/null
+++ b/test/Transforms/RewriteStatepointsForGC/constants.ll
@@ -0,0 +1,60 @@
+; RUN: opt -S -rewrite-statepoints-for-gc %s | FileCheck %s
+
+declare void @foo()
+declare i32 @llvm.experimental.gc.statepoint.p0f_isVoidf(i64, i32, void ()*, i32, i32, ...)
+
+; constants don't get relocated.
+define i8 @test() gc "statepoint-example" {
+; CHECK-LABEL: @test
+; CHECK: gc.statepoint
+; CHECK-NEXT: load i8, i8 addrspace(1)* inttoptr (i64 15 to i8 addrspace(1)*)
+entry:
+  call i32 (i64, i32, void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 0, i32 0, void ()* @foo, i32 0, i32 0, i32 0, i32 0)
+  %res = load i8, i8 addrspace(1)* inttoptr (i64 15 to i8 addrspace(1)*)
+  ret i8 %res
+}
+
+
+; Mostly just here to show reasonable code test can come from.  
+define i8 @test2(i8 addrspace(1)* %p) gc "statepoint-example" {
+; CHECK-LABEL: @test2
+; CHECK: gc.statepoint
+; CHECK-NEXT: gc.relocate
+; CHECK-NEXT: icmp
+entry:
+  call i32 (i64, i32, void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 0, i32 0, void ()* @foo, i32 0, i32 0, i32 0, i32 0)
+  %cmp = icmp eq i8 addrspace(1)* %p, null
+  br i1 %cmp, label %taken, label %not_taken
+
+taken:
+  ret i8 0
+
+not_taken:
+  %cmp2 = icmp ne i8 addrspace(1)* %p, null
+  br i1 %cmp2, label %taken, label %dead
+
+dead:
+  ; We see that dead can't be reached, but the optimizer might not.  It's 
+  ; completely legal for it to exploit the fact that if dead executed, %p 
+  ; would have to equal null.  This can produce intermediate states which 
+  ; look like that of test above, even if arbitrary constant addresses aren't
+  ; legal in the source language
+  %addr = getelementptr i8, i8 addrspace(1)* %p, i32 15
+  %res = load i8, i8addrspace(1)* %addr
+  ret i8 %res
+}
+
+@G = addrspace(1) global i8 5
+
+; Globals don't move and thus don't get relocated
+define i8 @test3(i1 %always_true) gc "statepoint-example" {
+; CHECK-LABEL: @test3
+; CHECK: gc.statepoint
+; CHECK-NEXT: load i8, i8 addrspace(1)* @G
+entry:
+  call i32 (i64, i32, void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 0, i32 0, void ()* @foo, i32 0, i32 0, i32 0, i32 0)
+  %res = load i8, i8 addrspace(1)* @G, align 1
+  ret i8 %res
+}
+
+
diff --git a/test/Transforms/RewriteStatepointsForGC/deref-pointers.ll b/test/Transforms/RewriteStatepointsForGC/deref-pointers.ll
new file mode 100644
index 00000000000..5913db21fcf
--- /dev/null
+++ b/test/Transforms/RewriteStatepointsForGC/deref-pointers.ll
@@ -0,0 +1,77 @@
+; RUN: opt -S -rewrite-statepoints-for-gc < %s | FileCheck %s
+
+declare void @foo()
+declare i8 addrspace(1)* @some_function()
+declare void @some_function_consumer(i8 addrspace(1)*)
+declare dereferenceable(4) i8 addrspace(1)* @some_function_ret_deref()
+; CHECK: declare i8 addrspace(1)* @some_function_ret_deref()
+
+define i8 addrspace(1)* @test_deref_arg(i8 addrspace(1)* dereferenceable(4) %a) gc "statepoint-example" {
+; CHECK: define i8 addrspace(1)* @test_deref_arg(i8 addrspace(1)* %a)
+entry:
+  call i32 (i64, i32, void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 0, i32 0, void ()* @foo, i32 0, i32 0, i32 0, i32 5, i32 0, i32 -1, i32 0, i32 0, i32 0)
+  ret i8 addrspace(1)* %a
+}
+
+define i8 addrspace(1)* @test_deref_or_null_arg(i8 addrspace(1)* dereferenceable_or_null(4) %a) gc "statepoint-example" {
+; CHECK: define i8 addrspace(1)* @test_deref_or_null_arg(i8 addrspace(1)* %a)
+entry:
+  call i32 (i64, i32, void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 0, i32 0, void ()* @foo, i32 0, i32 0, i32 0, i32 5, i32 0, i32 -1, i32 0, i32 0, i32 0)
+  ret i8 addrspace(1)* %a
+}
+
+define i8 addrspace(1)* @test_deref_retval() gc "statepoint-example" {
+; CHECK-LABEL: @test_deref_retval(
+entry:
+  %a = call dereferenceable(4) i8 addrspace(1)* @some_function()
+; CHECK: %a = call i8 addrspace(1)* @some_function()
+  call i32 (i64, i32, void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 0, i32 0, void ()* @foo, i32 0, i32 0, i32 0, i32 5, i32 0, i32 -1, i32 0, i32 0, i32 0)
+  ret i8 addrspace(1)* %a
+}
+
+define i8 addrspace(1)* @test_deref_or_null_retval() gc "statepoint-example" {
+; CHECK-LABEL: @test_deref_or_null_retval(
+entry:
+  %a = call dereferenceable_or_null(4) i8 addrspace(1)* @some_function()
+; CHECK: %a = call i8 addrspace(1)* @some_function()
+  call i32 (i64, i32, void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 0, i32 0, void ()* @foo, i32 0, i32 0, i32 0, i32 5, i32 0, i32 -1, i32 0, i32 0, i32 0)
+  ret i8 addrspace(1)* %a
+}
+
+define i8 @test_md(i8 addrspace(1)* %ptr) gc "statepoint-example" {
+; CHECK-LABEL: @test_md(
+ entry:
+; CHECK: %tmp = load i8, i8 addrspace(1)* %ptr, !tbaa !0
+  %tmp = load i8, i8 addrspace(1)* %ptr, !tbaa !0
+  call i32 (i64, i32, void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 0, i32 0, void ()* @foo, i32 0, i32 0, i32 0, i32 5, i32 0, i32 -1, i32 0, i32 0, i32 0)
+  ret i8 %tmp
+}
+
+define i8 addrspace(1)* @test_decl_only_attribute(i8 addrspace(1)* %ptr) gc "statepoint-example" {
+; CHECK-LABEL: @test_decl_only_attribute(
+entry:
+; No change here, but the prototype of some_function_ret_deref should have changed.
+; CHECK: call i8 addrspace(1)* @some_function_ret_deref()
+  %a = call i8 addrspace(1)* @some_function_ret_deref()
+  call i32 (i64, i32, void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 0, i32 0, void ()* @foo, i32 0, i32 0, i32 0, i32 5, i32 0, i32 -1, i32 0, i32 0, i32 0)
+  ret i8 addrspace(1)* %a
+}
+
+define i8 addrspace(1)* @test_callsite_arg_attribute(i8 addrspace(1)* %ptr) gc "statepoint-example" {
+; CHECK-LABEL: @test_callsite_arg_attribute(
+entry:
+; CHECK: call void @some_function_consumer(i8 addrspace(1)* %ptr)
+  call void @some_function_consumer(i8 addrspace(1)* dereferenceable(4) %ptr)
+  call i32 (i64, i32, void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 0, i32 0, void ()* @foo, i32 0, i32 0, i32 0, i32 5, i32 0, i32 -1, i32 0, i32 0, i32 0)
+  ret i8 addrspace(1)* %ptr
+}
+
+declare i32 @llvm.experimental.gc.statepoint.p0f_isVoidf(i64, i32, void ()*, i32, i32, ...)
+
+!0 = !{!1, !1, i64 0, i64 1}
+!1 = !{!"red", !2}
+!2 = !{!"blue"}
+
+; CHECK: !0 = !{!1, !1, i64 0}
+; CHECK: !1 = !{!"red", !2}
+; CHECK: !2 = !{!"blue"}
diff --git a/test/Transforms/RewriteStatepointsForGC/gc_relocate_creation.ll b/test/Transforms/RewriteStatepointsForGC/gc_relocate_creation.ll
new file mode 100644
index 00000000000..355ffa634f3
--- /dev/null
+++ b/test/Transforms/RewriteStatepointsForGC/gc_relocate_creation.ll
@@ -0,0 +1,20 @@
+; RUN: opt %s -rewrite-statepoints-for-gc -S 2>&1 | FileCheck %s
+; This test is to verify gc.relocate can handle pointer to vector of
+; pointers (<2 x i32 addrspace(1)*> addrspace(1)* in this case).
+; The old scheme to create a gc.relocate of <2 x i32 addrspace(1)*> addrspace(1)*
+; type will fail because llvm does not support mangling vector of pointers.
+; The new scheme will create all gc.relocate to i8 addrspace(1)* type and
+; then bitcast to the correct type.
+
+declare void @foo()
+declare void @use(...)
+declare i32 @llvm.experimental.gc.statepoint.p0f_isVoidf(i64, i32, void ()*, i32, i32, ...)
+
+define void @test1(<2 x i32 addrspace(1)*> addrspace(1)* %obj) gc "statepoint-example" {
+entry:
+  %safepoint_token = call i32 (i64, i32, void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 0, i32 0, void ()* @foo, i32 0, i32 0, i32 0, i32 0)
+; CHECK: %obj.relocated = call coldcc i8 addrspace(1)* @llvm.experimental.gc.relocate.p1i8(i32 %safepoint_token, i32 7, i32 7)
+; CHECK-NEXT:  %obj.relocated.casted = bitcast i8 addrspace(1)* %obj.relocated to <2 x i32 addrspace(1)*> addrspace(1)*
+  call void (...) @use(<2 x i32 addrspace(1)*> addrspace(1)* %obj)
+  ret void
+}
diff --git a/test/Transforms/RewriteStatepointsForGC/live-vector.ll b/test/Transforms/RewriteStatepointsForGC/live-vector.ll
index c1a693ed716..26ad73737ad 100644
--- a/test/Transforms/RewriteStatepointsForGC/live-vector.ll
+++ b/test/Transforms/RewriteStatepointsForGC/live-vector.ll
@@ -7,9 +7,10 @@ define i64 addrspace(1)* @test(i64 addrspace(1)* %obj) gc "statepoint-example" {
 ; CHECK-LABEL: test
 ; CHECK: gc.statepoint
 ; CHECK-NEXT: gc.relocate
-; CHECK-NEXT: ret i64 addrspace(1)* %obj.relocated
+; CHECK-NEXT: bitcast
+; CHECK-NEXT: ret i64 addrspace(1)* %obj.relocated.casted
 entry:
-  %safepoint_token = call i32 (void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(void ()* @do_safepoint, i32 0, i32 0, i32 0)
+  %safepoint_token = call i32 (i64, i32, void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 0, i32 0, void ()* @do_safepoint, i32 0, i32 0, i32 0, i32 0)
   ret i64 addrspace(1)* %obj
 }
 
@@ -20,12 +21,14 @@ define <2 x i64 addrspace(1)*> @test2(<2 x i64 addrspace(1)*> %obj) gc "statepoi
 ; CHECK-NEXT: extractelement
 ; CHECK-NEXT: gc.statepoint
 ; CHECK-NEXT: gc.relocate
+; CHECK-NEXT: bitcast
 ; CHECK-NEXT: gc.relocate
+; CHECK-NEXT: bitcast
 ; CHECK-NEXT: insertelement
 ; CHECK-NEXT: insertelement
-; CHECK-NEXT: ret <2 x i64 addrspace(1)*> %5
+; CHECK-NEXT: ret <2 x i64 addrspace(1)*> %7
 entry:
-  %safepoint_token = call i32 (void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(void ()* @do_safepoint, i32 0, i32 0, i32 0)
+  %safepoint_token = call i32 (i64, i32, void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 0, i32 0, void ()* @do_safepoint, i32 0, i32 0, i32 0, i32 0)
   ret <2 x i64 addrspace(1)*> %obj
 }
 
@@ -37,20 +40,22 @@ define <2 x i64 addrspace(1)*> @test3(<2 x i64 addrspace(1)*>* %ptr) gc "statepo
 ; CHECK-NEXT: extractelement
 ; CHECK-NEXT: gc.statepoint
 ; CHECK-NEXT: gc.relocate
+; CHECK-NEXT: bitcast
 ; CHECK-NEXT: gc.relocate
+; CHECK-NEXT: bitcast
 ; CHECK-NEXT: insertelement
 ; CHECK-NEXT: insertelement
-; CHECK-NEXT: ret <2 x i64 addrspace(1)*> %5
+; CHECK-NEXT: ret <2 x i64 addrspace(1)*> %7
 entry:
   %obj = load <2 x i64 addrspace(1)*>, <2 x i64 addrspace(1)*>* %ptr
-  %safepoint_token = call i32 (void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(void ()* @do_safepoint, i32 0, i32 0, i32 0)
+  %safepoint_token = call i32 (i64, i32, void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 0, i32 0, void ()* @do_safepoint, i32 0, i32 0, i32 0, i32 0)
   ret <2 x i64 addrspace(1)*> %obj
 }
 
 declare i32 @fake_personality_function()
 
 ; When a statepoint is an invoke rather than a call
-define <2 x i64 addrspace(1)*> @test4(<2 x i64 addrspace(1)*>* %ptr) gc "statepoint-example" {
+define <2 x i64 addrspace(1)*> @test4(<2 x i64 addrspace(1)*>* %ptr) gc "statepoint-example" personality i32 ()* @fake_personality_function {
 ; CHECK-LABEL: test4
 ; CHECK: load
 ; CHECK-NEXT: extractelement
@@ -58,30 +63,99 @@ define <2 x i64 addrspace(1)*> @test4(<2 x i64 addrspace(1)*>* %ptr) gc "statepo
 ; CHECK-NEXT: gc.statepoint
 entry:
   %obj = load <2 x i64 addrspace(1)*>, <2 x i64 addrspace(1)*>* %ptr
-  invoke i32 (void ()*, i32, i32, ...)* @llvm.experimental.gc.statepoint.p0f_isVoidf(void ()* @do_safepoint, i32 0, i32 0, i32 0)
+  invoke i32 (i64, i32, void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 0, i32 0, void ()* @do_safepoint, i32 0, i32 0, i32 0, i32 0)
           to label %normal_return unwind label %exceptional_return
 
 ; CHECK-LABEL: normal_return:
 ; CHECK: gc.relocate
+; CHECK-NEXT: bitcast
 ; CHECK-NEXT: gc.relocate
+; CHECK-NEXT: bitcast
 ; CHECK-NEXT: insertelement
 ; CHECK-NEXT: insertelement
-; CHECK-NEXT: ret <2 x i64 addrspace(1)*> %6
+; CHECK-NEXT: ret <2 x i64 addrspace(1)*> %8
 normal_return:                                    ; preds = %entry
   ret <2 x i64 addrspace(1)*> %obj
 
 ; CHECK-LABEL: exceptional_return:
 ; CHECK: gc.relocate
+; CHECK-NEXT: bitcast
 ; CHECK-NEXT: gc.relocate
+; CHECK-NEXT: bitcast
 ; CHECK-NEXT: insertelement
 ; CHECK-NEXT: insertelement
-; CHECK-NEXT: ret <2 x i64 addrspace(1)*> %10
+; CHECK-NEXT: ret <2 x i64 addrspace(1)*> %14
 exceptional_return:                               ; preds = %entry
-  %landing_pad4 = landingpad { i8*, i32 } personality i32 ()* @fake_personality_function
+  %landing_pad4 = landingpad { i8*, i32 }
           cleanup
   ret <2 x i64 addrspace(1)*> %obj
 }
 
+; Can we handle an insert element with a constant offset?  This effectively
+; tests both the equal and inequal case since we have to relocate both indices
+; in the vector.
+define <2 x i64 addrspace(1)*> @test5(i64 addrspace(1)* %p) 
+     gc "statepoint-example" {
+; CHECK-LABEL: test5
+; CHECK: insertelement
+; CHECK-NEXT: extractelement
+; CHECK-NEXT: extractelement
+; CHECK-NEXT: gc.statepoint
+; CHECK-NEXT: gc.relocate
+; CHECK-NEXT: bitcast
+; CHECK-NEXT: gc.relocate
+; CHECK-NEXT: bitcast
+; CHECK-NEXT: insertelement
+; CHECK-NEXT: insertelement
+; CHECK-NEXT: ret <2 x i64 addrspace(1)*> %7
+entry:
+  %vec = insertelement <2 x i64 addrspace(1)*> undef, i64 addrspace(1)* %p, i32 0
+  %safepoint_token = call i32 (i64, i32, void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 0, i32 0, void ()* @do_safepoint, i32 0, i32 0, i32 0, i32 0)
+  ret <2 x i64 addrspace(1)*> %vec
+}
+
+
+; A base vector from a load
+define <2 x i64 addrspace(1)*> @test6(i1 %cnd, <2 x i64 addrspace(1)*>* %ptr) 
+    gc "statepoint-example" {
+; CHECK-LABEL: test6
+; CHECK-LABEL: merge:
+; CHECK-NEXT: = phi
+; CHECK-NEXT: = phi
+; CHECK-NEXT: extractelement
+; CHECK-NEXT: extractelement
+; CHECK-NEXT: extractelement
+; CHECK-NEXT: extractelement
+; CHECK-NEXT: gc.statepoint
+; CHECK-NEXT: gc.relocate
+; CHECK-NEXT: bitcast
+; CHECK-NEXT: gc.relocate
+; CHECK-NEXT: bitcast
+; CHECK-NEXT: gc.relocate
+; CHECK-NEXT: bitcast
+; CHECK-NEXT: gc.relocate
+; CHECK-NEXT: bitcast
+; CHECK-NEXT: insertelement
+; CHECK-NEXT: insertelement
+; CHECK-NEXT: insertelement
+; CHECK-NEXT: insertelement
+; CHECK-NEXT: ret <2 x i64 addrspace(1)*>
+entry:
+  br i1 %cnd, label %taken, label %untaken
+taken:
+  %obja = load <2 x i64 addrspace(1)*>, <2 x i64 addrspace(1)*>* %ptr
+  br label %merge
+untaken:
+  %objb = load <2 x i64 addrspace(1)*>, <2 x i64 addrspace(1)*>* %ptr
+  br label %merge
+
+merge:
+  %obj = phi <2 x i64 addrspace(1)*> [%obja, %taken], [%objb, %untaken]
+  %safepoint_token = call i32 (i64, i32, void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 0, i32 0, void ()* @do_safepoint, i32 0, i32 0, i32 0, i32 0)
+  ret <2 x i64 addrspace(1)*> %obj
+}
+
+
 declare void @do_safepoint()
 
-declare i32 @llvm.experimental.gc.statepoint.p0f_isVoidf(void ()*, i32, i32, ...)
+declare i32 @llvm.experimental.gc.statepoint.p0f_isVoidf(i64, i32, void ()*, i32, i32, ...)
diff --git a/test/Transforms/RewriteStatepointsForGC/liveness-basics.ll b/test/Transforms/RewriteStatepointsForGC/liveness-basics.ll
index 0990c68daf9..6bc4d532449 100644
--- a/test/Transforms/RewriteStatepointsForGC/liveness-basics.ll
+++ b/test/Transforms/RewriteStatepointsForGC/liveness-basics.ll
@@ -1,6 +1,6 @@
 ; A collection of liveness test cases to ensure we're reporting the
 ; correct live values at statepoints
-; RUN: opt -rewrite-statepoints-for-gc -S < %s | FileCheck %s
+; RUN: opt -rewrite-statepoints-for-gc -spp-rematerialization-threshold=0 -S < %s | FileCheck %s
 
 
 ; Tests to make sure we consider %obj live in both the taken and untaken 
@@ -13,22 +13,24 @@ entry:
 taken:
 ; CHECK-LABEL: taken:
 ; CHECK-NEXT: gc.statepoint
-; CHECK-NEXT: %obj.relocated = call coldcc i64 addrspace(1)*
+; CHECK-NEXT: %obj.relocated = call coldcc i8 addrspace(1)*
+; CHECK-NEXT: bitcast
 ; CHECK-NEXT: br label %merge
-  call i32 (void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(void ()* @foo, i32 0, i32 0, i32 0)
+  call i32 (i64, i32, void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 0, i32 0, void ()* @foo, i32 0, i32 0, i32 0, i32 0)
   br label %merge
 
 untaken:
 ; CHECK-LABEL: untaken:
 ; CHECK-NEXT: gc.statepoint
-; CHECK-NEXT: %obj.relocated1 = call coldcc i64 addrspace(1)*
+; CHECK-NEXT: %obj.relocated1 = call coldcc i8 addrspace(1)*
+; CHECK-NEXT: bitcast
 ; CHECK-NEXT: br label %merge
-  call i32 (void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(void ()* @foo, i32 0, i32 0, i32 0)
+  call i32 (i64, i32, void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 0, i32 0, void ()* @foo, i32 0, i32 0, i32 0, i32 0)
   br label %merge
 
 merge:
 ; CHECK-LABEL: merge:
-; CHECK-NEXT: %.0 = phi i64 addrspace(1)* [ %obj.relocated, %taken ], [ %obj.relocated1, %untaken ]
+; CHECK-NEXT: %.0 = phi i64 addrspace(1)* [ %obj.relocated.casted, %taken ], [ %obj.relocated1.casted, %untaken ]
 ; CHECK-NEXT: ret i64 addrspace(1)* %.0
   ret i64 addrspace(1)* %obj
 }
@@ -40,7 +42,7 @@ entry:
 ; CHECK-LABEL: entry:
 ; CHECK-NEXT:  gc.statepoint
 ; CHECK-NEXT:  br
-  call i32 (void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(void ()* @foo, i32 0, i32 0, i32 0)
+  call i32 (i64, i32, void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 0, i32 0, void ()* @foo, i32 0, i32 0, i32 0, i32 0)
   br i1 %cmp, label %taken, label %untaken
 
 taken:
@@ -48,10 +50,11 @@ taken:
 ; CHECK-NEXT:  %obj = load
 ; CHECK-NEXT:  gc.statepoint
 ; CHECK-NEXT:  gc.relocate
-; CHECK-NEXT:  ret i64 addrspace(1)* %obj.relocated
+; CHECK-NEXT: bitcast
+; CHECK-NEXT:  ret i64 addrspace(1)* %obj.relocated.casted
 
   %obj = load i64 addrspace(1)*, i64 addrspace(1)** %loc
-  call i32 (void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(void ()* @foo, i32 0, i32 0, i32 0)
+  call i32 (i64, i32, void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 0, i32 0, void ()* @foo, i32 0, i32 0, i32 0, i32 0)
   ret i64 addrspace(1)* %obj
 
 untaken:
@@ -70,18 +73,19 @@ taken:
 ; CHECK-NEXT: gc.statepoint
 ; CHECK-NEXT: %obj = load
 ; CHECK-NEXT: gc.statepoint
-; CHECK-NEXT: %obj.relocated = call coldcc i64 addrspace(1)*
+; CHECK-NEXT: %obj.relocated = call coldcc i8 addrspace(1)*
+; CHECK-NEXT: bitcast
 ; CHECK-NEXT: br label %merge
-  call i32 (void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(void ()* @foo, i32 0, i32 0, i32 0)
+  call i32 (i64, i32, void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 0, i32 0, void ()* @foo, i32 0, i32 0, i32 0, i32 0)
   %obj = load i64 addrspace(1)*, i64 addrspace(1)** %loc
-  call i32 (void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(void ()* @foo, i32 0, i32 0, i32 0)
+  call i32 (i64, i32, void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 0, i32 0, void ()* @foo, i32 0, i32 0, i32 0, i32 0)
   br label %merge
 
 untaken:
 ; CHECK-LABEL: taken:
 ; CHECK-NEXT: gc.statepoint
 ; CHECK-NEXT: br label %merge
-  call i32 (void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(void ()* @foo, i32 0, i32 0, i32 0)
+  call i32 (i64, i32, void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 0, i32 0, void ()* @foo, i32 0, i32 0, i32 0, i32 0)
   br label %merge
 
 merge:
@@ -97,18 +101,22 @@ entry:
 ; CHECK-LABEL: entry:
 ; CHECK-NEXT:  %derived = getelementptr
 ; CHECK-NEXT:  gc.statepoint
-; CHECK-NEXT:  %derived.relocated = 
+; CHECK-NEXT:  %derived.relocated =
+; CHECK-NEXT:  bitcast 
 ; CHECK-NEXT:  %obj.relocated =
+; CHECK-NEXT:  bitcast
 ; CHECK-NEXT:  gc.statepoint
-; CHECK-NEXT:  %derived.relocated1 = 
+; CHECK-NEXT:  %derived.relocated1 =
+; CHECK-NEXT:  bitcast 
 ; Note: It's legal to relocate obj again, but not strictly needed
 ; CHECK-NEXT:  %obj.relocated2 =
-; CHECK-NEXT:  ret i64 addrspace(1)* %derived.relocated1
+; CHECK-NEXT:  bitcast
+; CHECK-NEXT:  ret i64 addrspace(1)* %derived.relocated1.casted
 ; 
   %derived = getelementptr i64, i64 addrspace(1)* %obj, i64 8
-  call i32 (void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(void ()* @foo, i32 0, i32 0, i32 0)
+  call i32 (i64, i32, void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 0, i32 0, void ()* @foo, i32 0, i32 0, i32 0, i32 0)
 
-  call i32 (void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(void ()* @foo, i32 0, i32 0, i32 0)
+  call i32 (i64, i32, void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 0, i32 0, void ()* @foo, i32 0, i32 0, i32 0, i32 0)
   ret i64 addrspace(1)* %derived
 }
 
@@ -125,9 +133,10 @@ entry:
 taken:
 ; CHECK-LABEL: taken:
 ; CHECK-NEXT: gc.statepoint
-; CHECK-NEXT: %obj.relocated = call coldcc i64 addrspace(1)*
+; CHECK-NEXT: %obj.relocated = call coldcc i8 addrspace(1)*
+; CHECK-NEXT: bitcast
 ; CHECK-NEXT: br label %merge
-  call i32 (void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(void ()* @foo, i32 0, i32 0, i32 0)
+  call i32 (i64, i32, void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 0, i32 0, void ()* @foo, i32 0, i32 0, i32 0, i32 0)
   br label %merge
 
 untaken:
@@ -154,5 +163,4 @@ final:
 
 declare void @foo()
 
-declare i32 @llvm.experimental.gc.statepoint.p0f_isVoidf(void ()*, i32, i32, ...)
-
+declare i32 @llvm.experimental.gc.statepoint.p0f_isVoidf(i64, i32, void ()*, i32, i32, ...)
diff --git a/test/Transforms/RewriteStatepointsForGC/preprocess.ll b/test/Transforms/RewriteStatepointsForGC/preprocess.ll
index 29a7dcc4256..012fff5c9e1 100644
--- a/test/Transforms/RewriteStatepointsForGC/preprocess.ll
+++ b/test/Transforms/RewriteStatepointsForGC/preprocess.ll
@@ -13,10 +13,11 @@ next:
 ; CHECK-LABEL: next:
 ; CHECK-NEXT: gc.statepoint
 ; CHECK-NEXT: gc.relocate
-; CHECK-NEXT: @consume(i64 addrspace(1)* %obj.relocated)
-; CHECK-NEXT: @consume(i64 addrspace(1)* %obj.relocated)
+; CHECK-NEXT: bitcast
+; CHECK-NEXT: @consume(i64 addrspace(1)* %obj.relocated.casted)
+; CHECK-NEXT: @consume(i64 addrspace(1)* %obj.relocated.casted)
   %obj2 = phi i64 addrspace(1)* [ %obj, %entry ]
-  call i32 (void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(void ()* @foo, i32 0, i32 0, i32 0)
+  call i32 (i64, i32, void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 0, i32 0, void ()* @foo, i32 0, i32 0, i32 0, i32 0)
   call void (...) @consume(i64 addrspace(1)* %obj2)
   call void (...) @consume(i64 addrspace(1)* %obj)
   ret void
@@ -32,27 +33,27 @@ define void @test7() gc "statepoint-example" {
 
 unreached:
   %obj = phi i64 addrspace(1)* [null, %unreached]
-  call i32 (void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(void ()* @foo, i32 0, i32 0, i32 0)
+  call i32 (i64, i32, void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 0, i32 0, void ()* @foo, i32 0, i32 0, i32 0, i32 0)
   call void (...) @consume(i64 addrspace(1)* %obj)
   br label %unreached
 }
 
 ; Need to delete unreachable gc.statepoint invoke - tested seperately given
 ; a correct implementation could only remove the instructions, not the block
-define void @test8() gc "statepoint-example" {
+define void @test8() gc "statepoint-example" personality i32 ()* undef {
 ; CHECK-LABEL: test8
 ; CHECK-NOT: gc.statepoint
   ret void
 
 unreached:
-  invoke i32 (void ()*, i32, i32, ...)* @llvm.experimental.gc.statepoint.p0f_isVoidf(void ()* @foo, i32 0, i32 0, i32 0)
+  invoke i32 (i64, i32, void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 0, i32 0, void ()* @foo, i32 0, i32 0, i32 0, i32 0)
           to label %normal_return unwind label %exceptional_return
 
 normal_return:                                    ; preds = %entry
   ret void
 
 exceptional_return:                               ; preds = %entry
-  %landing_pad4 = landingpad { i8*, i32 } personality i32 ()* undef
+  %landing_pad4 = landingpad { i8*, i32 }
           cleanup
   ret void
 }
@@ -61,5 +62,4 @@ declare void @foo()
 ; Bound the last check-not
 ; CHECK-LABEL: @foo
 
-declare i32 @llvm.experimental.gc.statepoint.p0f_isVoidf(void ()*, i32, i32, ...)
-
+declare i32 @llvm.experimental.gc.statepoint.p0f_isVoidf(i64, i32, void ()*, i32, i32, ...)
diff --git a/test/Transforms/RewriteStatepointsForGC/relocate_invoke_result.ll b/test/Transforms/RewriteStatepointsForGC/relocate_invoke_result.ll
index 73ebf2f86d6..1a5289b2665 100644
--- a/test/Transforms/RewriteStatepointsForGC/relocate_invoke_result.ll
+++ b/test/Transforms/RewriteStatepointsForGC/relocate_invoke_result.ll
@@ -10,24 +10,24 @@ declare void @gc_call()
 declare i32* @fake_personality_function()
 
 ; Function Attrs: nounwind
-define i64* addrspace(1)* @test() gc "statepoint-example" {
+define i64* addrspace(1)* @test() gc "statepoint-example" personality i32* ()* @fake_personality_function {
 entry:
   %obj = invoke i64* addrspace(1)* @non_gc_call()
           to label %normal_dest unwind label %unwind_dest
 
 unwind_dest: 
-  %lpad = landingpad { i8*, i32 } personality i32* ()* @fake_personality_function
+  %lpad = landingpad { i8*, i32 }
           cleanup
   resume { i8*, i32 } undef
 
 normal_dest:
 ;; CHECK-LABEL: normal_dest:
 ;; CHECK-NEXT: gc.statepoint
-;; CHECK-NEXT: %obj.relocated = call coldcc i64* addrspace(1)*                                      
-  %safepoint_token = call i32 (void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(void ()* @gc_call, i32 0, i32 0, i32 5, i32 0, i32 -1, i32 0, i32 0, i32 0)
+;; CHECK-NEXT: %obj.relocated = call coldcc i8 addrspace(1)*
+;; CHECK-NEXT: bitcast
+  %safepoint_token = call i32 (i64, i32, void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 0, i32 0, void ()* @gc_call, i32 0, i32 0, i32 0, i32 5, i32 0, i32 -1, i32 0, i32 0, i32 0)
   ret i64* addrspace(1)* %obj
 }
 
-declare i32 @llvm.experimental.gc.statepoint.p0f_isVoidf(void ()*, i32, i32, ...)
-
+declare i32 @llvm.experimental.gc.statepoint.p0f_isVoidf(i64, i32, void ()*, i32, i32, ...)
 
diff --git a/test/Transforms/RewriteStatepointsForGC/relocation.ll b/test/Transforms/RewriteStatepointsForGC/relocation.ll
index da092ee3d24..d7a84e5820c 100644
--- a/test/Transforms/RewriteStatepointsForGC/relocation.ll
+++ b/test/Transforms/RewriteStatepointsForGC/relocation.ll
@@ -1,4 +1,4 @@
-; RUN: opt %s -rewrite-statepoints-for-gc -S 2>&1 | FileCheck %s
+; RUN: opt %s -rewrite-statepoints-for-gc -spp-rematerialization-threshold=0 -S 2>&1 | FileCheck %s
 
 
 declare void @foo()
@@ -9,12 +9,12 @@ entry:
 ; CHECK-LABEL: @test1
 ; CHECK-DAG: %obj.relocated
 ; CHECK-DAG: %obj2.relocated
-  %safepoint_token = call i32 (void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(void ()* @foo, i32 0, i32 0, i32 0)
+  %safepoint_token = call i32 (i64, i32, void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 0, i32 0, void ()* @foo, i32 0, i32 0, i32 0, i32 0)
   br label %joint
 
 joint:
 ; CHECK-LABEL: joint:
-; CHECK: %phi1 = phi i64 addrspace(1)* [ %obj.relocated, %entry ], [ %obj3, %joint2 ]
+; CHECK: %phi1 = phi i64 addrspace(1)* [ %obj.relocated.casted, %entry ], [ %obj3, %joint2 ]
   %phi1 = phi i64 addrspace(1)* [ %obj, %entry ], [ %obj3, %joint2 ]
   br i1 %condition, label %use, label %joint2
 
@@ -23,8 +23,8 @@ use:
 
 joint2:
 ; CHECK-LABEL: joint2:
-; CHECK: %phi2 = phi i64 addrspace(1)* [ %obj.relocated, %use ], [ %obj2.relocated, %joint ]
-; CHECK: %obj3 = getelementptr i64, i64 addrspace(1)* %obj2.relocated, i32 1
+; CHECK: %phi2 = phi i64 addrspace(1)* [ %obj.relocated.casted, %use ], [ %obj2.relocated.casted, %joint ]
+; CHECK: %obj3 = getelementptr i64, i64 addrspace(1)* %obj2.relocated.casted, i32 1
   %phi2 = phi i64 addrspace(1)* [ %obj, %use ], [ %obj2, %joint ]
   %obj3 = getelementptr i64, i64 addrspace(1)* %obj2, i32 1
   br label %joint
@@ -45,9 +45,9 @@ entry:
 
 loop:
 ; CHECK: loop:
-; CHECK-DAG: [ %obj_init.relocated, %loop.backedge ]
+; CHECK-DAG: [ %obj_init.relocated.casted, %loop.backedge ]
 ; CHECK-DAG: [ %obj_init, %entry ]
-; CHECK-DAG: [ %obj.relocated, %loop.backedge ]
+; CHECK-DAG: [ %obj.relocated.casted, %loop.backedge ]
 ; CHECK-DAG: [ %obj, %entry ]
   %index = phi i32 [ 0, %entry ], [ %index.inc, %loop.backedge ]
 ; CHECK-NOT: %location = getelementptr i64, i64 addrspace(1)* %obj, i32 %index
@@ -61,7 +61,7 @@ loop_x:
   br label %loop.backedge
 
 loop.backedge:
-  %safepoint_token = call i32 (void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(void ()* @do_safepoint, i32 0, i32 0, i32 0)
+  %safepoint_token = call i32 (i64, i32, void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 0, i32 0, void ()* @do_safepoint, i32 0, i32 0, i32 0, i32 0)
   br label %loop
 
 loop_y:
@@ -79,14 +79,14 @@ if_branch:
 ; CHECK-LABEL: if_branch:
 ; CHECK: gc.statepoint
 ; CHECK: gc.relocate
-  %safepoint_token = call i32 (void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(void ()* @foo, i32 0, i32 0, i32 0)
+  %safepoint_token = call i32 (i64, i32, void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 0, i32 0, void ()* @foo, i32 0, i32 0, i32 0, i32 0)
   br label %join
 
 else_branch:
 ; CHECK-LABEL: else_branch:
 ; CHECK: gc.statepoint
 ; CHECK: gc.relocate
-  %safepoint_token1 = call i32 (void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(void ()* @foo, i32 0, i32 0, i32 0)
+  %safepoint_token1 = call i32 (i64, i32, void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 0, i32 0, void ()* @foo, i32 0, i32 0, i32 0, i32 0)
   br label %join
 
 join:
@@ -108,9 +108,10 @@ entry:
 ; CHECK-LABEL: @test3
 ; CHECK: gc.statepoint
 ; CHECK-NEXT: gc.relocate
+; CHECK-NEXT: bitcast
 ; CHECK-NEXT: gc.statepoint
-  %safepoint_token = call i32 (void (i64)*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidi64f(void (i64)* undef, i32 1, i32 0, i64 undef, i32 5, i32 0, i32 -1, i32 0, i32 0, i32 0)
-  %safepoint_token1 = call i32 (i32 (i64 addrspace(1)*)*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_i32p1i64f(i32 (i64 addrspace(1)*)* undef, i32 1, i32 0, i64 addrspace(1)* %obj, i32 5, i32 0, i32 -1, i32 0, i32 0, i32 0)
+  %safepoint_token = call i32 (i64, i32, void (i64)*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidi64f(i64 0, i32 0, void (i64)* undef, i32 1, i32 0, i64 undef, i32 0, i32 5, i32 0, i32 -1, i32 0, i32 0, i32 0)
+  %safepoint_token1 = call i32 (i64, i32, i32 (i64 addrspace(1)*)*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_i32p1i64f(i64 0, i32 0, i32 (i64 addrspace(1)*)* undef, i32 1, i32 0, i64 addrspace(1)* %obj, i32 0, i32 5, i32 0, i32 -1, i32 0, i32 0, i32 0)
   ret void
 }
 
@@ -123,9 +124,9 @@ define void @test4() gc "statepoint-example" {
 ; CHECK: gc.statepoint
 ; CHECK: gc.relocate
 ; CHECK: @use(i8 addrspace(1)* %res.relocated)
-  %safepoint_token2 = tail call i32 (i8 addrspace(1)* ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_p1i8f(i8 addrspace(1)* ()* undef, i32 0, i32 0, i32 0)
+  %safepoint_token2 = tail call i32 (i64, i32, i8 addrspace(1)* ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_p1i8f(i64 0, i32 0, i8 addrspace(1)* ()* undef, i32 0, i32 0, i32 0, i32 0)
   %res = call i8 addrspace(1)* @llvm.experimental.gc.result.ptr.p1i8(i32 %safepoint_token2)
-  call i32 (i8 addrspace(1)* ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_p1i8f(i8 addrspace(1)* ()* undef, i32 0, i32 0, i32 0)
+  call i32 (i64, i32, i8 addrspace(1)* ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_p1i8f(i64 0, i32 0, i8 addrspace(1)* ()* undef, i32 0, i32 0, i32 0, i32 0)
   call void (...) @use(i8 addrspace(1)* %res)
   unreachable
 }
@@ -135,7 +136,7 @@ define void @test4() gc "statepoint-example" {
 define void @test5(i8 addrspace(1)* %arg) gc "statepoint-example" {
 ; CHECK-LABEL: test5
 entry:
-  call i32 (i8 addrspace(1)* ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_p1i8f(i8 addrspace(1)* ()* undef, i32 0, i32 0, i32 0)
+  call i32 (i64, i32, i8 addrspace(1)* ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_p1i8f(i64 0, i32 0, i8 addrspace(1)* ()* undef, i32 0, i32 0, i32 0, i32 0)
   switch i32 undef, label %kill [
     i32 10, label %merge
     i32 13, label %merge
@@ -169,7 +170,7 @@ do_safepoint:
 ; CHECK: arg1.relocated = 
 ; CHECK: arg2.relocated = 
 ; CHECK: arg3.relocated = 
-  call i32 (void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(void ()* @foo, i32 0, i32 0, i32 3, i8 addrspace(1)* %arg1, i8 addrspace(1)* %arg2, i8 addrspace(1)* %arg3)
+  call i32 (i64, i32, void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 0, i32 0, void ()* @foo, i32 0, i32 0, i32 0, i32 3, i8 addrspace(1)* %arg1, i8 addrspace(1)* %arg2, i8 addrspace(1)* %arg3)
   br label %gc.safepoint_poll.exit2
 
 gc.safepoint_poll.exit2:
@@ -208,7 +209,7 @@ outer-inc:
 ; CHECK-LABEL: outer-inc:
 ; CHECK: %arg1.relocated
 ; CHECK: %arg2.relocated
-  %safepoint_token = call i32 (void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(void ()* @foo, i32 0, i32 0, i32 2, i8 addrspace(1)* %arg1, i8 addrspace(1)* %arg2)
+  %safepoint_token = call i32 (i64, i32, void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 0, i32 0, void ()* @foo, i32 0, i32 0, i32 0, i32 2, i8 addrspace(1)* %arg1, i8 addrspace(1)* %arg2)
   br label %outer-loop
 }
 
@@ -237,7 +238,7 @@ inner-loop:
 ; CHECK: gc.statepoint
 ; CHECK: %arg1.relocated
 ; CHECK: %arg2.relocated
-  %safepoint_token = call i32 (void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(void ()* @foo, i32 0, i32 0, i32 2, i8 addrspace(1)* %arg1, i8 addrspace(1)* %arg2)
+  %safepoint_token = call i32 (i64, i32, void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 0, i32 0, void ()* @foo, i32 0, i32 0, i32 0, i32 2, i8 addrspace(1)* %arg1, i8 addrspace(1)* %arg2)
   br i1 %cmp, label %inner-loop, label %outer-inc
 
 outer-inc:
@@ -257,15 +258,15 @@ branch2:
   br i1 %condition, label %callbb, label %join2
 
 callbb:
-  %safepoint_token = call i32 (void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(void ()* @foo, i32 0, i32 0, i32 5, i32 0, i32 -1, i32 0, i32 0, i32 0)
+  %safepoint_token = call i32 (i64, i32, void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 0, i32 0, void ()* @foo, i32 0, i32 0, i32 0, i32 5, i32 0, i32 -1, i32 0, i32 0, i32 0)
   br label %join
 
 join:
 ; CHECK-LABEL: join:
-; CHECK: phi i64 addrspace(1)* [ %obj.relocated, %callbb ], [ %obj, %entry ]
+; CHECK: phi i64 addrspace(1)* [ %obj.relocated.casted, %callbb ], [ %obj, %entry ]
 ; CHECK: phi i64 addrspace(1)* 
 ; CHECK-DAG: [ %obj, %entry ]
-; CHECK-DAG: [ %obj2.relocated, %callbb ]
+; CHECK-DAG: [ %obj2.relocated.casted, %callbb ]
   ; This is a phi outside the dominator region of the new defs inserted by
   ; the safepoint, BUT we can't stop the search here or we miss the second
   ; phi below.
@@ -284,12 +285,11 @@ join2:
 
 declare void @do_safepoint()
 
-declare i32 @llvm.experimental.gc.statepoint.p0f_isVoidf(void ()*, i32, i32, ...)
-declare i32 @llvm.experimental.gc.statepoint.p0f_p1i8f(i8 addrspace(1)* ()*, i32, i32, ...)
-declare i32 @llvm.experimental.gc.statepoint.p0f_isVoidi64f(void (i64)*, i32, i32, ...)
-declare i32 @llvm.experimental.gc.statepoint.p0f_i32p1i64f(i32 (i64 addrspace(1)*)*, i32, i32, ...)
+declare i32 @llvm.experimental.gc.statepoint.p0f_isVoidf(i64, i32, void ()*, i32, i32, ...)
+declare i32 @llvm.experimental.gc.statepoint.p0f_p1i8f(i64, i32, i8 addrspace(1)* ()*, i32, i32, ...)
+declare i32 @llvm.experimental.gc.statepoint.p0f_isVoidi64f(i64, i32, void (i64)*, i32, i32, ...)
+declare i32 @llvm.experimental.gc.statepoint.p0f_i32p1i64f(i64, i32, i32 (i64 addrspace(1)*)*, i32, i32, ...)
 declare i8 addrspace(1)* @llvm.experimental.gc.result.ptr.p1i8(i32) #3
 
 
 
-
diff --git a/test/Transforms/RewriteStatepointsForGC/rematerialize-derived-pointers.ll b/test/Transforms/RewriteStatepointsForGC/rematerialize-derived-pointers.ll
new file mode 100644
index 00000000000..f04e7c797ca
--- /dev/null
+++ b/test/Transforms/RewriteStatepointsForGC/rematerialize-derived-pointers.ll
@@ -0,0 +1,222 @@
+; RUN: opt %s -rewrite-statepoints-for-gc -S 2>&1 | FileCheck %s
+
+declare void @use_obj16(i16 addrspace(1)*)
+declare void @use_obj32(i32 addrspace(1)*)
+declare void @use_obj64(i64 addrspace(1)*)
+declare void @do_safepoint()
+
+define void @"test_gep_const"(i32 addrspace(1)* %base) gc "statepoint-example" {
+; CHECK-LABEL: test_gep_const
+entry:
+  %ptr = getelementptr i32, i32 addrspace(1)* %base, i32 15
+  ; CHECK: getelementptr i32, i32 addrspace(1)* %base, i32 15
+  %sp = call i32 (i64, i32, void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 0, i32 0, void ()* @do_safepoint, i32 0, i32 0, i32 0, i32 0)
+  ; CHECK: %base.relocated = call coldcc i8 addrspace(1)* @llvm.experimental.gc.relocate.p1i8(i32 %sp, i32 7, i32 7)
+  ; CHECK: bitcast i8 addrspace(1)* %base.relocated to i32 addrspace(1)*
+  ; CHECK: getelementptr i32, i32 addrspace(1)* %base.relocated.casted, i32 15
+  call void @use_obj32(i32 addrspace(1)* %base)
+  call void @use_obj32(i32 addrspace(1)* %ptr)
+  ret void
+}
+
+define void @"test_gep_idx"(i32 addrspace(1)* %base, i32 %idx) gc "statepoint-example" {
+; CHECK-LABEL: test_gep_idx
+entry:
+  %ptr = getelementptr i32, i32 addrspace(1)* %base, i32 %idx
+  ; CHECK: getelementptr
+  %sp = call i32 (i64, i32, void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 0, i32 0, void ()* @do_safepoint, i32 0, i32 0, i32 0, i32 0)
+  ; CHECK: %base.relocated = call coldcc i8 addrspace(1)* @llvm.experimental.gc.relocate.p1i8(i32 %sp, i32 7, i32 7)
+  ; CHECK: %base.relocated.casted = bitcast i8 addrspace(1)* %base.relocated to i32 addrspace(1)*
+  ; CHECK: getelementptr i32, i32 addrspace(1)* %base.relocated.casted, i32 %idx
+  call void @use_obj32(i32 addrspace(1)* %base)
+  call void @use_obj32(i32 addrspace(1)* %ptr)
+  ret void
+}
+
+define void @"test_bitcast"(i32 addrspace(1)* %base) gc "statepoint-example" {
+; CHECK-LABEL: test_bitcast
+entry:
+  %ptr = bitcast i32 addrspace(1)* %base to i64 addrspace(1)*
+  ; CHECK: bitcast i32 addrspace(1)* %base to i64 addrspace(1)*
+  %sp = call i32 (i64, i32, void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 0, i32 0, void ()* @do_safepoint, i32 0, i32 0, i32 0, i32 0)
+  ; CHECK: %base.relocated = call coldcc i8 addrspace(1)* @llvm.experimental.gc.relocate.p1i8(i32 %sp, i32 7, i32 7)
+  ; CHECK: %base.relocated.casted = bitcast i8 addrspace(1)* %base.relocated to i32 addrspace(1)*
+  ; CHECK: bitcast i32 addrspace(1)* %base.relocated.casted to i64 addrspace(1)*
+  call void @use_obj32(i32 addrspace(1)* %base)
+  call void @use_obj64(i64 addrspace(1)* %ptr)
+  ret void
+}
+
+define void @"test_bitcast_gep"(i32 addrspace(1)* %base) gc "statepoint-example" {
+; CHECK-LABEL: test_bitcast_gep
+entry:
+  %ptr.gep = getelementptr i32, i32 addrspace(1)* %base, i32 15
+  ; CHECK: getelementptr
+  %ptr.cast = bitcast i32 addrspace(1)* %ptr.gep to i64 addrspace(1)*
+  ; CHECK: bitcast
+  %sp = call i32 (i64, i32, void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 0, i32 0, void ()* @do_safepoint, i32 0, i32 0, i32 0, i32 0)
+  ; CHECK: gc.relocate
+  ; CHECK: bitcast
+  ; CHECK: getelementptr
+  ; CHECK: bitcast
+  call void @use_obj32(i32 addrspace(1)* %base)
+  call void @use_obj64(i64 addrspace(1)* %ptr.cast)
+  ret void
+}
+
+define void @"test_intersecting_chains"(i32 addrspace(1)* %base, i32 %idx) gc "statepoint-example" {
+; CHECK-LABEL: test_intersecting_chains
+entry:
+  %ptr.gep = getelementptr i32, i32 addrspace(1)* %base, i32 15
+  ; CHECK: getelementptr
+  %ptr.cast = bitcast i32 addrspace(1)* %ptr.gep to i64 addrspace(1)*
+  ; CHECK: bitcast
+  %ptr.cast2 = bitcast i32 addrspace(1)* %ptr.gep to i16 addrspace(1)*
+  ; CHECK: bitcast
+  %sp = call i32 (i64, i32, void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 0, i32 0, void ()* @do_safepoint, i32 0, i32 0, i32 0, i32 0)
+  ; CHECK: getelementptr
+  ; CHECK: bitcast
+  ; CHECK: getelementptr
+  ; CHECK: bitcast
+  call void @use_obj64(i64 addrspace(1)* %ptr.cast)
+  call void @use_obj16(i16 addrspace(1)* %ptr.cast2)
+  ret void
+}
+
+define void @"test_cost_threshold"(i32 addrspace(1)* %base, i32 %idx1, i32 %idx2, i32 %idx3) gc "statepoint-example" {
+; CHECK-LABEL: test_cost_threshold
+entry:
+  %ptr.gep = getelementptr i32, i32 addrspace(1)* %base, i32 15
+  ; CHECK: getelementptr
+  %ptr.gep2 = getelementptr i32, i32 addrspace(1)* %ptr.gep, i32 %idx1
+  ; CHECK: getelementptr
+  %ptr.gep3 = getelementptr i32, i32 addrspace(1)* %ptr.gep2, i32 %idx2
+  ; CHECK: getelementptr
+  %ptr.gep4 = getelementptr i32, i32 addrspace(1)* %ptr.gep3, i32 %idx3
+  ; CHECK: getelementptr
+  %ptr.cast = bitcast i32 addrspace(1)* %ptr.gep4 to i64 addrspace(1)*
+  ; CHECK: bitcast
+  %sp = call i32 (i64, i32, void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 0, i32 0, void ()* @do_safepoint, i32 0, i32 0, i32 0, i32 0)
+  ; CHECK: gc.relocate
+  ; CHECK: bitcast
+  ; CHECK: gc.relocate
+  ; CHECK: bitcast
+  call void @use_obj64(i64 addrspace(1)* %ptr.cast)
+  ret void
+}
+
+define void @"test_two_derived"(i32 addrspace(1)* %base) gc "statepoint-example" {
+; CHECK-LABEL: test_two_derived
+entry:
+  %ptr = getelementptr i32, i32 addrspace(1)* %base, i32 15
+  %ptr2 = getelementptr i32, i32 addrspace(1)* %base, i32 12
+  ; CHECK: getelementptr
+  ; CHECK: getelementptr
+  %sp = call i32 (i64, i32, void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 0, i32 0, void ()* @do_safepoint, i32 0, i32 0, i32 0, i32 0)
+  ; CHECK: gc.relocate
+  ; CHECK: bitcast
+  ; CHECK: getelementptr
+  ; CHECK: getelementptr
+  call void @use_obj32(i32 addrspace(1)* %ptr)
+  call void @use_obj32(i32 addrspace(1)* %ptr2)
+  ret void
+}
+
+define void @"test_gep_smallint_array"([3 x i32] addrspace(1)* %base) gc "statepoint-example" {
+; CHECK-LABEL: test_gep_smallint_array
+entry:
+  %ptr = getelementptr [3 x i32], [3 x i32] addrspace(1)* %base, i32 0, i32 2
+  ; CHECK: getelementptr
+  %sp = call i32 (i64, i32, void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 0, i32 0, void ()* @do_safepoint, i32 0, i32 0, i32 0, i32 0)
+  ; CHECK: gc.relocate
+  ; CHECK: bitcast
+  ; CHECK: getelementptr
+  call void @use_obj32(i32 addrspace(1)* %ptr)
+  ret void
+}
+
+declare i32 @fake_personality_function()
+
+define void @"test_invoke"(i32 addrspace(1)* %base) gc "statepoint-example" personality i32 ()* @fake_personality_function {
+; CHECK-LABEL: test_invoke
+entry:
+  %ptr.gep = getelementptr i32, i32 addrspace(1)* %base, i32 15
+  ; CHECK: getelementptr
+  %ptr.cast = bitcast i32 addrspace(1)* %ptr.gep to i64 addrspace(1)*
+  ; CHECK: bitcast
+  %ptr.cast2 = bitcast i32 addrspace(1)* %ptr.gep to i16 addrspace(1)*
+  ; CHECK: bitcast
+  %sp = invoke i32 (i64, i32, void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 0, i32 0, void ()* @do_safepoint, i32 0, i32 0, i32 0, i32 0)
+                to label %normal unwind label %exception
+
+normal:
+  ; CHECK-LABEL: normal:
+  ; CHECK: gc.relocate
+  ; CHECK: bitcast
+  ; CHECK: getelementptr
+  ; CHECK: bitcast
+  ; CHECK: getelementptr
+  ; CHECK: bitcast
+  call void @use_obj64(i64 addrspace(1)* %ptr.cast)
+  call void @use_obj16(i16 addrspace(1)* %ptr.cast2)
+  ret void
+
+exception:
+  ; CHECK-LABEL: exception:
+  %landing_pad4 = landingpad { i8*, i32 }
+          cleanup
+  ; CHECK: gc.relocate
+  ; CHECK: bitcast
+  ; CHECK: getelementptr
+  ; CHECK: bitcast
+  ; CHECK: getelementptr
+  ; CHECK: bitcast
+  call void @use_obj64(i64 addrspace(1)* %ptr.cast)
+  call void @use_obj16(i16 addrspace(1)* %ptr.cast2)
+  ret void
+}
+
+define void @"test_loop"(i32 addrspace(1)* %base) gc "statepoint-example" {
+; CHECK-LABEL: test_loop
+entry:
+  %ptr.gep = getelementptr i32, i32 addrspace(1)* %base, i32 15
+  ; CHECK: getelementptr
+  br label %loop
+
+loop:
+  ; CHECK: phi i32 addrspace(1)* [ %ptr.gep, %entry ], [ %ptr.gep.remat, %loop ]
+  ; CHECK: phi i32 addrspace(1)* [ %base, %entry ], [ %base.relocated.casted, %loop ]
+  call void @use_obj32(i32 addrspace(1)* %ptr.gep)
+  %sp = call i32 (i64, i32, void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 0, i32 0, void ()* @do_safepoint, i32 0, i32 0, i32 0, i32 0)
+  ; CHECK: gc.relocate
+  ; CHECK: bitcast
+  ; CHECK: getelementptr
+  br label %loop
+}
+
+define void @"test_too_long"(i32 addrspace(1)* %base) gc "statepoint-example" {
+; CHECK-LABEL: test_too_long
+entry:
+  %ptr.gep   = getelementptr i32, i32 addrspace(1)* %base, i32 15
+  %ptr.gep1  = getelementptr i32, i32 addrspace(1)* %ptr.gep, i32 15
+  %ptr.gep2  = getelementptr i32, i32 addrspace(1)* %ptr.gep1, i32 15
+  %ptr.gep3  = getelementptr i32, i32 addrspace(1)* %ptr.gep2, i32 15
+  %ptr.gep4  = getelementptr i32, i32 addrspace(1)* %ptr.gep3, i32 15
+  %ptr.gep5  = getelementptr i32, i32 addrspace(1)* %ptr.gep4, i32 15
+  %ptr.gep6  = getelementptr i32, i32 addrspace(1)* %ptr.gep5, i32 15
+  %ptr.gep7  = getelementptr i32, i32 addrspace(1)* %ptr.gep6, i32 15
+  %ptr.gep8  = getelementptr i32, i32 addrspace(1)* %ptr.gep7, i32 15
+  %ptr.gep9  = getelementptr i32, i32 addrspace(1)* %ptr.gep8, i32 15
+  %ptr.gep10 = getelementptr i32, i32 addrspace(1)* %ptr.gep9, i32 15
+  %ptr.gep11 = getelementptr i32, i32 addrspace(1)* %ptr.gep10, i32 15
+  %sp = call i32 (i64, i32, void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 0, i32 0, void ()* @do_safepoint, i32 0, i32 0, i32 0, i32 0)
+  ; CHECK: gc.relocate
+  ; CHECK: bitcast
+  ; CHECK: gc.relocate
+  ; CHECK: bitcast
+  call void @use_obj32(i32 addrspace(1)* %ptr.gep11)
+  ret void
+}
+
+
+declare i32 @llvm.experimental.gc.statepoint.p0f_isVoidf(i64, i32, void ()*, i32, i32, ...)
diff --git a/test/Transforms/SCCP/2003-08-26-InvokeHandling.ll b/test/Transforms/SCCP/2003-08-26-InvokeHandling.ll
index fb1926eca25..eb308afc0e6 100644
--- a/test/Transforms/SCCP/2003-08-26-InvokeHandling.ll
+++ b/test/Transforms/SCCP/2003-08-26-InvokeHandling.ll
@@ -3,7 +3,7 @@
 
 declare void @foo()
 
-define i32 @test(i1 %cond) {
+define i32 @test(i1 %cond) personality i8* bitcast (i32 (...)* @__gxx_personality_v0 to i8*) {
 Entry:
 	br i1 %cond, label %Inv, label %Cont
 Inv:		; preds = %Entry
@@ -12,7 +12,7 @@ Inv:		; preds = %Entry
 Ok:		; preds = %Inv
 	br label %Cont
 LPad:
-        %val = landingpad { i8*, i32 } personality i8* bitcast (i32 (...)* @__gxx_personality_v0 to i8*)
+        %val = landingpad { i8*, i32 }
                  catch i8* null
         br label %Cont
 Cont:		; preds = %Ok, %Inv, %Entry
diff --git a/test/Transforms/SCCP/2004-11-16-DeadInvoke.ll b/test/Transforms/SCCP/2004-11-16-DeadInvoke.ll
index e7eb101c147..47d9d835656 100644
--- a/test/Transforms/SCCP/2004-11-16-DeadInvoke.ll
+++ b/test/Transforms/SCCP/2004-11-16-DeadInvoke.ll
@@ -2,13 +2,13 @@
 
 declare i32 @foo()
 
-define void @caller() {
+define void @caller() personality i8* bitcast (i32 (...)* @__gxx_personality_v0 to i8*) {
 	br i1 true, label %T, label %F
 F:		; preds = %0
 	%X = invoke i32 @foo( )
 			to label %T unwind label %LP		; <i32> [#uses=0]
 LP:
-        %val = landingpad { i8*, i32 } personality i8* bitcast (i32 (...)* @__gxx_personality_v0 to i8*)
+        %val = landingpad { i8*, i32 }
                  catch i8* null
         br label %T
 T:
diff --git a/test/Transforms/SCCP/2007-05-16-InvokeCrash.ll b/test/Transforms/SCCP/2007-05-16-InvokeCrash.ll
index a5a42f1b407..7d29f6cabcb 100644
--- a/test/Transforms/SCCP/2007-05-16-InvokeCrash.ll
+++ b/test/Transforms/SCCP/2007-05-16-InvokeCrash.ll
@@ -1,7 +1,7 @@
 ; RUN: opt < %s -sccp -disable-output
 ; PR1431
 
-define void @_ada_bench() {
+define void @_ada_bench() personality i8* bitcast (i32 (...)* @__gxx_personality_v0 to i8*) {
 entry:
 	br label %cond_next
 cond_next:		; preds = %cond_next, %entry
@@ -31,7 +31,7 @@ bb149:		; preds = %bb114
 bb177:		; preds = %bb149
 	unreachable
 cleanup:		; preds = %bb149, %bb114, %bb67
-        %val = landingpad { i8*, i32 } personality i8* bitcast (i32 (...)* @__gxx_personality_v0 to i8*)
+        %val = landingpad { i8*, i32 }
                  cleanup
 	resume { i8*, i32 } %val
 }
diff --git a/test/Transforms/SCCP/2009-01-14-IPSCCP-Invoke.ll b/test/Transforms/SCCP/2009-01-14-IPSCCP-Invoke.ll
index c05f897f1a3..f3e54804f30 100644
--- a/test/Transforms/SCCP/2009-01-14-IPSCCP-Invoke.ll
+++ b/test/Transforms/SCCP/2009-01-14-IPSCCP-Invoke.ll
@@ -2,12 +2,12 @@
 ; RUN: opt < %s -ipsccp -S | grep "ret i32 undef"
 ; PR3325
 
-define i32 @main() {
+define i32 @main() personality i8* bitcast (i32 (...)* @__gxx_personality_v0 to i8*) {
 	%tmp1 = invoke i32 @f()
 			to label %UnifiedReturnBlock unwind label %lpad
 
 lpad:
-        %val = landingpad { i8*, i32 } personality i8* bitcast (i32 (...)* @__gxx_personality_v0 to i8*)
+        %val = landingpad { i8*, i32 }
                  cleanup
 	unreachable
 
diff --git a/test/Transforms/SCCP/crash.ll b/test/Transforms/SCCP/crash.ll
index 88528902d72..3ec1fd21745 100644
--- a/test/Transforms/SCCP/crash.ll
+++ b/test/Transforms/SCCP/crash.ll
@@ -27,3 +27,8 @@ define i32 @test2([4 x i32] %A) {
   %B = extractvalue [4 x i32] %A, 1
   ret i32 %B
 }
+
+define x86_mmx @test3() {
+  %load = load x86_mmx, x86_mmx* null
+  ret x86_mmx %load
+}
diff --git a/test/Transforms/SCCP/global-alias-constprop.ll b/test/Transforms/SCCP/global-alias-constprop.ll
new file mode 100644
index 00000000000..453755b066b
--- /dev/null
+++ b/test/Transforms/SCCP/global-alias-constprop.ll
@@ -0,0 +1,11 @@
+; RUN: opt < %s -sccp -S | FileCheck %s
+
+@0 = private unnamed_addr constant [2 x i32] [i32 -1, i32 1]
+@"\01??_7A@@6B@" = unnamed_addr alias getelementptr inbounds ([2 x i32], [2 x i32]* @0, i32 0, i32 1)
+
+; CHECK: ret i32 1
+
+define i32 @main() {
+  %a = load i32, i32* @"\01??_7A@@6B@"
+  ret i32 %a
+}
diff --git a/test/Transforms/SCCP/ipsccp-basic.ll b/test/Transforms/SCCP/ipsccp-basic.ll
index c74063f3300..bf37134545e 100644
--- a/test/Transforms/SCCP/ipsccp-basic.ll
+++ b/test/Transforms/SCCP/ipsccp-basic.ll
@@ -82,7 +82,7 @@ define internal {i64,i64} @test4a() {
   ret {i64,i64} %b
 }
 
-define i64 @test4b() {
+define i64 @test4b() personality i8* bitcast (i32 (...)* @__gxx_personality_v0 to i8*) {
   %a = invoke {i64,i64} @test4a()
           to label %A unwind label %B
 A:
@@ -90,7 +90,7 @@ A:
   %c = call i64 @test4c(i64 %b)
   ret i64 %c
 B:
-  %val = landingpad { i8*, i32 } personality i8* bitcast (i32 (...)* @__gxx_personality_v0 to i8*)
+  %val = landingpad { i8*, i32 }
            catch i8* null
   ret i64 0
 }
@@ -116,14 +116,14 @@ define internal {i64,i64} @test5a() {
   ret {i64,i64} %b
 }
 
-define i64 @test5b() {
+define i64 @test5b() personality i8* bitcast (i32 (...)* @__gxx_personality_v0 to i8*) {
   %a = invoke {i64,i64} @test5a()
           to label %A unwind label %B
 A:
   %c = call i64 @test5c({i64,i64} %a)
   ret i64 %c
 B:
-  %val = landingpad { i8*, i32 } personality i8* bitcast (i32 (...)* @__gxx_personality_v0 to i8*)
+  %val = landingpad { i8*, i32 }
            catch i8* null
   ret i64 0
 }
diff --git a/test/Transforms/SLPVectorizer/AMDGPU/lit.local.cfg b/test/Transforms/SLPVectorizer/AMDGPU/lit.local.cfg
new file mode 100644
index 00000000000..6baccf05fff
--- /dev/null
+++ b/test/Transforms/SLPVectorizer/AMDGPU/lit.local.cfg
@@ -0,0 +1,3 @@
+if not 'AMDGPU' in config.root.targets:
+    config.unsupported = True
+
diff --git a/test/Transforms/SLPVectorizer/R600/simplebb.ll b/test/Transforms/SLPVectorizer/AMDGPU/simplebb.ll
similarity index 92%
rename from test/Transforms/SLPVectorizer/R600/simplebb.ll
rename to test/Transforms/SLPVectorizer/AMDGPU/simplebb.ll
index 9ed86f88147..35763953911 100644
--- a/test/Transforms/SLPVectorizer/R600/simplebb.ll
+++ b/test/Transforms/SLPVectorizer/AMDGPU/simplebb.ll
@@ -1,4 +1,9 @@
 ; RUN: opt -S -march=r600 -mcpu=cayman -basicaa -slp-vectorizer -dce < %s | FileCheck %s
+; XFAIL: *
+; 
+; FIXME: If this test expects to be vectorized, the TTI must indicate that the target
+;        has vector registers of the expected width.
+;        Currently, it says there are 8 vector registers that are 32-bits wide.
 
 target datalayout = "e-p:32:32:32-p3:16:16:16-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-v16:16:16-v24:32:32-v32:32:32-v48:64:64-v64:64:64-v96:128:128-v128:128:128-v192:256:256-v256:256:256-v512:512:512-v1024:1024:1024-v2048:2048:2048-n32:64"
 
diff --git a/test/Transforms/SLPVectorizer/R600/lit.local.cfg b/test/Transforms/SLPVectorizer/R600/lit.local.cfg
deleted file mode 100644
index 4086e8d681c..00000000000
--- a/test/Transforms/SLPVectorizer/R600/lit.local.cfg
+++ /dev/null
@@ -1,3 +0,0 @@
-if not 'R600' in config.root.targets:
-    config.unsupported = True
-
diff --git a/test/Transforms/SLPVectorizer/X86/call.ll b/test/Transforms/SLPVectorizer/X86/call.ll
index b76ac2c15c6..a55cc1bf793 100644
--- a/test/Transforms/SLPVectorizer/X86/call.ll
+++ b/test/Transforms/SLPVectorizer/X86/call.ll
@@ -119,10 +119,10 @@ entry:
 }
 
 
-; CHECK: declare <2 x double> @llvm.sin.v2f64(<2 x double>) #0
-; CHECK: declare <2 x double> @llvm.cos.v2f64(<2 x double>) #0
-; CHECK: declare <2 x double> @llvm.pow.v2f64(<2 x double>, <2 x double>) #0
-; CHECK: declare <2 x double> @llvm.exp2.v2f64(<2 x double>) #0
+; CHECK: declare <2 x double> @llvm.sin.v2f64(<2 x double>) [[ATTR0:#[0-9]+]]
+; CHECK: declare <2 x double> @llvm.cos.v2f64(<2 x double>) [[ATTR0]]
+; CHECK: declare <2 x double> @llvm.pow.v2f64(<2 x double>, <2 x double>) [[ATTR0]]
+; CHECK: declare <2 x double> @llvm.exp2.v2f64(<2 x double>) [[ATTR0]]
 
-; CHECK: attributes #0 = { nounwind readnone }
+; CHECK: attributes [[ATTR0]] = { nounwind readnone }
 
diff --git a/test/Transforms/SLPVectorizer/X86/cse.ll b/test/Transforms/SLPVectorizer/X86/cse.ll
index 9f56e219599..8d25b3661dc 100644
--- a/test/Transforms/SLPVectorizer/X86/cse.ll
+++ b/test/Transforms/SLPVectorizer/X86/cse.ll
@@ -12,11 +12,8 @@ target triple = "i386-apple-macosx10.8.0"
 
 ;CHECK-LABEL: @test(
 ;CHECK: load <2 x double>
-;CHECK: fadd <2 x double>
-;CHECK: store <2 x double>
-;CHECK: insertelement <2 x double>
-;CHECK: fadd <2 x double>
-;CHECK: store <2 x double>
+;CHECK: fadd <4 x double>
+;CHECK: store <4 x double>
 ;CHECK: ret i32
 
 define i32 @test(double* nocapture %G) {
@@ -48,11 +45,12 @@ entry:
 ;  A[2] = A[2] * 7.6 * n + 3.0;
 ;  A[3] = A[3] * 7.4 * n + 4.0;
 ;}
-;CHECK-LABEL: @foo(
-;CHECK: insertelement <2 x double>
-;CHECK: insertelement <2 x double>
-;CHECK-NOT: insertelement <2 x double>
-;CHECK: ret
+; CHECK-LABEL: @foo(
+; CHECK: load <4 x double>
+; CHECK: fmul <4 x double>
+; CHECK: fmul <4 x double>
+; CHECK: fadd <4 x double>
+; CHECK: store <4 x double>
 define i32 @foo(double* nocapture %A, i32 %n) {
 entry:
   %0 = load double, double* %A, align 8
@@ -93,7 +91,7 @@ entry:
 ; }
 
 ; We can't merge the gather sequences because one does not dominate the other.
-; CHECK: test2
+; CHECK-LABEL: @test2(
 ; CHECK: insertelement
 ; CHECK: insertelement
 ; CHECK: insertelement
@@ -140,11 +138,12 @@ define i32 @test2(double* nocapture %G, i32 %k) {
 ;  A[2] = A[2] * 7.9 * n + 6.0;
 ;  A[3] = A[3] * 7.9 * n + 6.0;
 ;}
-;CHECK-LABEL: @foo4(
-;CHECK: insertelement <2 x double>
-;CHECK: insertelement <2 x double>
-;CHECK-NOT: insertelement <2 x double>
-;CHECK: ret
+; CHECK-LABEL: @foo4(
+; CHECK: load <4 x double>
+; CHECK: fmul <4 x double>
+; CHECK: fmul <4 x double>
+; CHECK: fadd <4 x double>
+; CHECK: store <4 x double>
 define i32 @foo4(double* nocapture %A, i32 %n) {
 entry:
   %0 = load double, double* %A, align 8
diff --git a/test/Transforms/SLPVectorizer/X86/debug_info.ll b/test/Transforms/SLPVectorizer/X86/debug_info.ll
index 60ff35851dd..1d349fbc98b 100644
--- a/test/Transforms/SLPVectorizer/X86/debug_info.ll
+++ b/test/Transforms/SLPVectorizer/X86/debug_info.ll
@@ -18,16 +18,16 @@ target triple = "x86_64-apple-macosx10.7.0"
 ;CHECK: load <2 x double>, <2 x double>* {{.*}}, !dbg ![[LOC]]
 ;CHECK: store <2 x double> {{.*}}, !dbg ![[LOC2:[0-9]+]]
 ;CHECK: ret
-;CHECK: ![[LOC]] = !MDLocation(line: 4, scope:
-;CHECK: ![[LOC2]] = !MDLocation(line: 7, scope:
+;CHECK: ![[LOC]] = !DILocation(line: 4, scope:
+;CHECK: ![[LOC2]] = !DILocation(line: 7, scope:
 
 define i32 @depth(double* nocapture %A, i32 %m) #0 {
 entry:
-  tail call void @llvm.dbg.value(metadata double* %A, i64 0, metadata !12, metadata !MDExpression()), !dbg !19
-  tail call void @llvm.dbg.value(metadata i32 %m, i64 0, metadata !13, metadata !MDExpression()), !dbg !19
-  tail call void @llvm.dbg.value(metadata i32 00, i64 0, metadata !14, metadata !MDExpression()), !dbg !21
-  tail call void @llvm.dbg.value(metadata i32 02, i64 0, metadata !15, metadata !MDExpression()), !dbg !21
-  tail call void @llvm.dbg.value(metadata i32 0, i64 0, metadata !16, metadata !MDExpression()), !dbg !23
+  tail call void @llvm.dbg.value(metadata double* %A, i64 0, metadata !12, metadata !DIExpression()), !dbg !19
+  tail call void @llvm.dbg.value(metadata i32 %m, i64 0, metadata !13, metadata !DIExpression()), !dbg !19
+  tail call void @llvm.dbg.value(metadata i32 00, i64 0, metadata !14, metadata !DIExpression()), !dbg !21
+  tail call void @llvm.dbg.value(metadata i32 02, i64 0, metadata !15, metadata !DIExpression()), !dbg !21
+  tail call void @llvm.dbg.value(metadata i32 0, i64 0, metadata !16, metadata !DIExpression()), !dbg !23
   %cmp8 = icmp sgt i32 %m, 0, !dbg !23
   br i1 %cmp8, label %for.body.lr.ph, label %for.end, !dbg !23
 
@@ -57,33 +57,33 @@ attributes #1 = { nounwind readnone }
 !llvm.dbg.cu = !{!0}
 !llvm.module.flags = !{!18, !32}
 
-!0 = !MDCompileUnit(language: DW_LANG_C99, producer: "clang version 3.4 (trunk 187335) (llvm/trunk 187335:187340M)", isOptimized: true, emissionKind: 0, file: !1, enums: !2, retainedTypes: !2, subprograms: !3, globals: !2, imports: !2)
-!1 = !MDFile(filename: "file.c", directory: "/Users/nadav")
+!0 = !DICompileUnit(language: DW_LANG_C99, producer: "clang version 3.4 (trunk 187335) (llvm/trunk 187335:187340M)", isOptimized: true, emissionKind: 0, file: !1, enums: !2, retainedTypes: !2, subprograms: !3, globals: !2, imports: !2)
+!1 = !DIFile(filename: "file.c", directory: "/Users/nadav")
 !2 = !{}
 !3 = !{!4}
-!4 = !MDSubprogram(name: "depth", line: 1, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: true, scopeLine: 1, file: !1, scope: !5, type: !6, function: i32 (double*, i32)* @depth, variables: !11)
-!5 = !MDFile(filename: "file.c", directory: "/Users/nadav")
-!6 = !MDSubroutineType(types: !7)
+!4 = !DISubprogram(name: "depth", line: 1, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: true, scopeLine: 1, file: !1, scope: !5, type: !6, function: i32 (double*, i32)* @depth, variables: !11)
+!5 = !DIFile(filename: "file.c", directory: "/Users/nadav")
+!6 = !DISubroutineType(types: !7)
 !7 = !{!8, !9, !8}
-!8 = !MDBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
-!9 = !MDDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, baseType: !10)
-!10 = !MDBasicType(tag: DW_TAG_base_type, name: "double", size: 64, align: 64, encoding: DW_ATE_float)
+!8 = !DIBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
+!9 = !DIDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, baseType: !10)
+!10 = !DIBasicType(tag: DW_TAG_base_type, name: "double", size: 64, align: 64, encoding: DW_ATE_float)
 !11 = !{!12, !13, !14, !15, !16}
-!12 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "A", line: 1, arg: 1, scope: !4, file: !5, type: !9)
-!13 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "m", line: 1, arg: 2, scope: !4, file: !5, type: !8)
-!14 = !MDLocalVariable(tag: DW_TAG_auto_variable, name: "y0", line: 2, scope: !4, file: !5, type: !10)
-!15 = !MDLocalVariable(tag: DW_TAG_auto_variable, name: "y1", line: 2, scope: !4, file: !5, type: !10)
-!16 = !MDLocalVariable(tag: DW_TAG_auto_variable, name: "i", line: 3, scope: !17, file: !5, type: !8)
-!17 = distinct !MDLexicalBlock(line: 3, column: 0, file: !1, scope: !4)
+!12 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "A", line: 1, arg: 1, scope: !4, file: !5, type: !9)
+!13 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "m", line: 1, arg: 2, scope: !4, file: !5, type: !8)
+!14 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "y0", line: 2, scope: !4, file: !5, type: !10)
+!15 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "y1", line: 2, scope: !4, file: !5, type: !10)
+!16 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "i", line: 3, scope: !17, file: !5, type: !8)
+!17 = distinct !DILexicalBlock(line: 3, column: 0, file: !1, scope: !4)
 !18 = !{i32 2, !"Dwarf Version", i32 2}
-!19 = !MDLocation(line: 1, scope: !4)
+!19 = !DILocation(line: 1, scope: !4)
 !20 = !{double 0.000000e+00}
-!21 = !MDLocation(line: 2, scope: !4)
+!21 = !DILocation(line: 2, scope: !4)
 !22 = !{double 1.000000e+00}
-!23 = !MDLocation(line: 3, scope: !17)
-!24 = !MDLocation(line: 4, scope: !25)
-!25 = distinct !MDLexicalBlock(line: 3, column: 0, file: !1, scope: !17)
-!29 = !MDLocation(line: 5, scope: !25)
-!30 = !MDLocation(line: 7, scope: !4)
-!31 = !MDLocation(line: 8, scope: !4)
+!23 = !DILocation(line: 3, scope: !17)
+!24 = !DILocation(line: 4, scope: !25)
+!25 = distinct !DILexicalBlock(line: 3, column: 0, file: !1, scope: !17)
+!29 = !DILocation(line: 5, scope: !25)
+!30 = !DILocation(line: 7, scope: !4)
+!31 = !DILocation(line: 8, scope: !4)
 !32 = !{i32 1, !"Debug Info Version", i32 3}
diff --git a/test/Transforms/SLPVectorizer/X86/gep.ll b/test/Transforms/SLPVectorizer/X86/gep.ll
index 3f952d7b242..d10f2b6015d 100644
--- a/test/Transforms/SLPVectorizer/X86/gep.ll
+++ b/test/Transforms/SLPVectorizer/X86/gep.ll
@@ -1,5 +1,6 @@
 ; RUN: opt < %s -basicaa -slp-vectorizer -S |FileCheck %s
 target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-unknown"
 
 ; Test if SLP can handle GEP expressions.
 ; The test perform the following action:
diff --git a/test/Transforms/SLPVectorizer/X86/loopinvariant.ll b/test/Transforms/SLPVectorizer/X86/loopinvariant.ll
index 0c16c34a188..dace4b35b87 100644
--- a/test/Transforms/SLPVectorizer/X86/loopinvariant.ll
+++ b/test/Transforms/SLPVectorizer/X86/loopinvariant.ll
@@ -1,22 +1,19 @@
-; RUN: opt < %s -basicaa -slp-vectorizer -dce -S -mtriple=x86_64-apple-macosx10.8.0 -mcpu=corei7-avx | FileCheck %s
+; RUN: opt < %s -basicaa -slp-vectorizer -S -mcpu=corei7-avx | FileCheck %s
 
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
 target triple = "x86_64-apple-macosx10.8.0"
 
 ;CHECK-LABEL: @foo(
-;CHECK: load <4 x i32>
-;CHECK: add nsw <4 x i32>
-;CHECK: store <4 x i32>
-;CHECK: load <4 x i32>
-;CHECK: add nsw <4 x i32>
-;CHECK: store <4 x i32>
+;CHECK: load <8 x i32>
+;CHECK: add nsw <8 x i32>
+;CHECK: store <8 x i32>
 ;CHECK: ret
-define i32 @foo(i32* nocapture %A, i32 %n) #0 {
+define i32 @foo(i32* nocapture %A, i32 %n) {
 entry:
   %cmp62 = icmp sgt i32 %n, 0
   br i1 %cmp62, label %for.body, label %for.end
 
-for.body:                                         ; preds = %entry, %for.body
+for.body:
   %indvars.iv = phi i64 [ %indvars.iv.next, %for.body ], [ 0, %entry ]
   %arrayidx = getelementptr inbounds i32, i32* %A, i64 %indvars.iv
   %0 = load i32, i32* %arrayidx, align 4
@@ -62,8 +59,7 @@ for.body:                                         ; preds = %entry, %for.body
   %cmp = icmp slt i32 %15, %n
   br i1 %cmp, label %for.body, label %for.end
 
-for.end:                                          ; preds = %for.body, %entry
+for.end:
   ret i32 undef
 }
 
-attributes #0 = { nounwind ssp uwtable "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf"="true" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "unsafe-fp-math"="false" "use-soft-float"="false" }
diff --git a/test/Transforms/SLPVectorizer/X86/ordering.ll b/test/Transforms/SLPVectorizer/X86/ordering.ll
index 0fa72c94c27..dfe95ac824e 100644
--- a/test/Transforms/SLPVectorizer/X86/ordering.ll
+++ b/test/Transforms/SLPVectorizer/X86/ordering.ll
@@ -21,7 +21,7 @@ entry:
 declare i8* @objc_msgSend(i8*, i8*, ...)
 declare i32 @personality_v0(...)
 
-define void @invoketest() {
+define void @invoketest() personality i8* bitcast (i32 (...)* @personality_v0 to i8*) {
 entry:
   br i1 undef, label %cond.true, label %cond.false
 
@@ -67,7 +67,7 @@ if.then63:
   br label %if.end98
 
 lpad:
-  %l = landingpad { i8*, i32 } personality i8* bitcast (i32 (...)* @personality_v0 to i8*)
+  %l = landingpad { i8*, i32 }
           cleanup
   resume { i8*, i32 } %l
 
diff --git a/test/Transforms/SLPVectorizer/X86/phi_landingpad.ll b/test/Transforms/SLPVectorizer/X86/phi_landingpad.ll
index 6d2d5e3540c..b47a6ce2a26 100644
--- a/test/Transforms/SLPVectorizer/X86/phi_landingpad.ll
+++ b/test/Transforms/SLPVectorizer/X86/phi_landingpad.ll
@@ -2,7 +2,8 @@
 
 target datalayout = "f64:64:64-v64:64:64"
 
-define void @test_phi_in_landingpad() {
+define void @test_phi_in_landingpad() personality i8*
+          bitcast (i32 (...)* @__gxx_personality_v0 to i8*) {
 entry:
   invoke void @foo()
           to label %inner unwind label %lpad
@@ -16,8 +17,7 @@ inner:
 lpad:
   %x1 = phi double [ undef, %entry ], [ undef, %inner ]
   %y1 = phi double [ undef, %entry ], [ undef, %inner ]
-  landingpad { i8*, i32 } personality i8*
-          bitcast (i32 (...)* @__gxx_personality_v0 to i8*) catch i8* null
+  landingpad { i8*, i32 } catch i8* null
   br label %done
 
 done:
diff --git a/test/Transforms/SLPVectorizer/X86/pr19657.ll b/test/Transforms/SLPVectorizer/X86/pr19657.ll
index a687aec7610..32f8da4c7ee 100644
--- a/test/Transforms/SLPVectorizer/X86/pr19657.ll
+++ b/test/Transforms/SLPVectorizer/X86/pr19657.ll
@@ -1,73 +1,45 @@
-; RUN: opt < %s -O1 -basicaa -slp-vectorizer -dce -S -mtriple=x86_64-unknown-linux-gnu -mcpu=corei7-avx | FileCheck %s
+; RUN: opt < %s -basicaa -slp-vectorizer -S -mcpu=corei7-avx | FileCheck %s
+; RUN: opt < %s -basicaa -slp-vectorizer -slp-max-reg-size=128 -S -mcpu=corei7-avx | FileCheck %s --check-prefix=V128
 
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
 
-;CHECK: load <2 x double>, <2 x double>*
-;CHECK: fadd <2 x double>
-;CHECK: store <2 x double>
+; CHECK-LABEL: @foo(
+; CHECK: load <4 x double>
+; CHECK: fadd <4 x double>
+; CHECK: fadd <4 x double>
+; CHECK: store <4 x double>
 
-; Function Attrs: nounwind uwtable
-define void @foo(double* %x) #0 {
-  %1 = alloca double*, align 8
-  store double* %x, double** %1, align 8
-  %2 = load double*, double** %1, align 8
-  %3 = getelementptr inbounds double, double* %2, i64 0
-  %4 = load double, double* %3, align 8
-  %5 = load double*, double** %1, align 8
-  %6 = getelementptr inbounds double, double* %5, i64 0
-  %7 = load double, double* %6, align 8
-  %8 = fadd double %4, %7
-  %9 = load double*, double** %1, align 8
-  %10 = getelementptr inbounds double, double* %9, i64 0
-  %11 = load double, double* %10, align 8
-  %12 = fadd double %8, %11
-  %13 = load double*, double** %1, align 8
-  %14 = getelementptr inbounds double, double* %13, i64 0
-  store double %12, double* %14, align 8
-  %15 = load double*, double** %1, align 8
-  %16 = getelementptr inbounds double, double* %15, i64 1
-  %17 = load double, double* %16, align 8
-  %18 = load double*, double** %1, align 8
-  %19 = getelementptr inbounds double, double* %18, i64 1
-  %20 = load double, double* %19, align 8
-  %21 = fadd double %17, %20
-  %22 = load double*, double** %1, align 8
-  %23 = getelementptr inbounds double, double* %22, i64 1
-  %24 = load double, double* %23, align 8
-  %25 = fadd double %21, %24
-  %26 = load double*, double** %1, align 8
-  %27 = getelementptr inbounds double, double* %26, i64 1
-  store double %25, double* %27, align 8
-  %28 = load double*, double** %1, align 8
-  %29 = getelementptr inbounds double, double* %28, i64 2
-  %30 = load double, double* %29, align 8
-  %31 = load double*, double** %1, align 8
-  %32 = getelementptr inbounds double, double* %31, i64 2
-  %33 = load double, double* %32, align 8
-  %34 = fadd double %30, %33
-  %35 = load double*, double** %1, align 8
-  %36 = getelementptr inbounds double, double* %35, i64 2
-  %37 = load double, double* %36, align 8
-  %38 = fadd double %34, %37
-  %39 = load double*, double** %1, align 8
-  %40 = getelementptr inbounds double, double* %39, i64 2
-  store double %38, double* %40, align 8
-  %41 = load double*, double** %1, align 8
-  %42 = getelementptr inbounds double, double* %41, i64 3
-  %43 = load double, double* %42, align 8
-  %44 = load double*, double** %1, align 8
-  %45 = getelementptr inbounds double, double* %44, i64 3
-  %46 = load double, double* %45, align 8
-  %47 = fadd double %43, %46
-  %48 = load double*, double** %1, align 8
-  %49 = getelementptr inbounds double, double* %48, i64 3
-  %50 = load double, double* %49, align 8
-  %51 = fadd double %47, %50
-  %52 = load double*, double** %1, align 8
-  %53 = getelementptr inbounds double, double* %52, i64 3
-  store double %51, double* %53, align 8
+; V128-LABEL: @foo(
+; V128: load <2 x double>
+; V128: fadd <2 x double>
+; V128: fadd <2 x double>
+; V128: store <2 x double>
+; V128: load <2 x double>
+; V128: fadd <2 x double>
+; V128: fadd <2 x double>
+; V128: store <2 x double>
+
+define void @foo(double* %x) {
+  %1 = load double, double* %x, align 8
+  %2 = fadd double %1, %1
+  %3 = fadd double %2, %1
+  store double %3, double* %x, align 8
+  %4 = getelementptr inbounds double, double* %x, i64 1
+  %5 = load double, double* %4, align 8
+  %6 = fadd double %5, %5
+  %7 = fadd double %6, %5
+  store double %7, double* %4, align 8
+  %8 = getelementptr inbounds double, double* %x, i64 2
+  %9 = load double, double* %8, align 8
+  %10 = fadd double %9, %9
+  %11 = fadd double %10, %9
+  store double %11, double* %8, align 8
+  %12 = getelementptr inbounds double, double* %x, i64 3
+  %13 = load double, double* %12, align 8
+  %14 = fadd double %13, %13
+  %15 = fadd double %14, %13
+  store double %15, double* %12, align 8
   ret void
 }
 
-attributes #0 = { nounwind }
diff --git a/test/Transforms/SLPVectorizer/X86/pr23510.ll b/test/Transforms/SLPVectorizer/X86/pr23510.ll
new file mode 100644
index 00000000000..efdb0ecd999
--- /dev/null
+++ b/test/Transforms/SLPVectorizer/X86/pr23510.ll
@@ -0,0 +1,38 @@
+; PR23510
+; RUN: opt < %s -basicaa -slp-vectorizer -S | FileCheck %s
+
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+; CHECK-LABEL: @_Z3fooPml(
+; CHECK: lshr <2 x i64>
+; CHECK: lshr <2 x i64>
+
+@total = global i64 0, align 8
+
+define void @_Z3fooPml(i64* nocapture %a, i64 %i) {
+entry:
+  %tmp = load i64, i64* %a, align 8
+  %shr = lshr i64 %tmp, 4
+  store i64 %shr, i64* %a, align 8
+  %arrayidx1 = getelementptr inbounds i64, i64* %a, i64 1
+  %tmp1 = load i64, i64* %arrayidx1, align 8
+  %shr2 = lshr i64 %tmp1, 4
+  store i64 %shr2, i64* %arrayidx1, align 8
+  %arrayidx3 = getelementptr inbounds i64, i64* %a, i64 %i
+  %tmp2 = load i64, i64* %arrayidx3, align 8
+  %tmp3 = load i64, i64* @total, align 8
+  %add = add i64 %tmp3, %tmp2
+  store i64 %add, i64* @total, align 8
+  %tmp4 = load i64, i64* %a, align 8
+  %shr5 = lshr i64 %tmp4, 4
+  store i64 %shr5, i64* %a, align 8
+  %tmp5 = load i64, i64* %arrayidx1, align 8
+  %shr7 = lshr i64 %tmp5, 4
+  store i64 %shr7, i64* %arrayidx1, align 8
+  %tmp6 = load i64, i64* %arrayidx3, align 8
+  %tmp7 = load i64, i64* @total, align 8
+  %add9 = add i64 %tmp7, %tmp6
+  store i64 %add9, i64* @total, align 8
+  ret void
+}
diff --git a/test/Transforms/SLPVectorizer/X86/tiny-tree.ll b/test/Transforms/SLPVectorizer/X86/tiny-tree.ll
index 6c93222ef93..26af8165bc3 100644
--- a/test/Transforms/SLPVectorizer/X86/tiny-tree.ll
+++ b/test/Transforms/SLPVectorizer/X86/tiny-tree.ll
@@ -153,3 +153,19 @@ define void @store_splat(float*, float) {
   store float %1, float* %6, align 4
   ret void
 }
+
+
+; CHECK-LABEL: store_const
+; CHECK: store <4 x i32>
+define void @store_const(i32* %a) {
+entry:
+  %ptr0 = getelementptr inbounds i32, i32* %a, i64 0
+  store i32 10, i32* %ptr0, align 4
+  %ptr1 = getelementptr inbounds i32, i32* %a, i64 1
+  store i32 30, i32* %ptr1, align 4
+  %ptr2 = getelementptr inbounds i32, i32* %a, i64 2
+  store i32 20, i32* %ptr2, align 4
+  %ptr3 = getelementptr inbounds i32, i32* %a, i64 3
+  store i32 40, i32* %ptr3, align 4
+  ret void
+}
diff --git a/test/Transforms/SROA/basictest.ll b/test/Transforms/SROA/basictest.ll
index a59192d718c..ad2794167a5 100644
--- a/test/Transforms/SROA/basictest.ll
+++ b/test/Transforms/SROA/basictest.ll
@@ -1195,20 +1195,24 @@ entry:
   %a = alloca <{ i1 }>, align 8
   %b = alloca <{ i1 }>, align 8
 ; CHECK:      %[[a:.*]] = alloca i8, align 8
+; CHECK-NEXT: %[[b:.*]] = alloca i8, align 8
 
   %b.i1 = bitcast <{ i1 }>* %b to i1*
   store i1 %x, i1* %b.i1, align 8
   %b.i8 = bitcast <{ i1 }>* %b to i8*
   %foo = load i8, i8* %b.i8, align 1
-; CHECK-NEXT: %[[ext:.*]] = zext i1 %x to i8
-; CHECK-NEXT: store i8 %[[ext]], i8* %[[a]], align 8
-; CHECK-NEXT: {{.*}} = load i8, i8* %[[a]], align 8
+; CHECK-NEXT: %[[b_cast:.*]] = bitcast i8* %[[b]] to i1*
+; CHECK-NEXT: store i1 %x, i1* %[[b_cast]], align 8
+; CHECK-NEXT: {{.*}} = load i8, i8* %[[b]], align 8
 
   %a.i8 = bitcast <{ i1 }>* %a to i8*
   call void @llvm.memcpy.p0i8.p0i8.i32(i8* %a.i8, i8* %b.i8, i32 1, i32 1, i1 false) nounwind
   %bar = load i8, i8* %a.i8, align 1
   %a.i1 = getelementptr inbounds <{ i1 }>, <{ i1 }>* %a, i32 0, i32 0
   %baz = load i1, i1* %a.i1, align 1
+; CHECK-NEXT: %[[copy:.*]] = load i8, i8* %[[b]], align 8
+; CHECK-NEXT: store i8 %[[copy]], i8* %[[a]], align 8
+; CHECK-NEXT: {{.*}} = load i8, i8* %[[a]], align 8
 ; CHECK-NEXT: %[[a_cast:.*]] = bitcast i8* %[[a]] to i1*
 ; CHECK-NEXT: {{.*}} = load i1, i1* %[[a_cast]], align 8
 
@@ -1595,3 +1599,14 @@ entry:
   store i32 %load, i32* %a.gep1
   ret void
 }
+
+define void @PR23737() {
+; CHECK-LABEL: @PR23737(
+; CHECK: store atomic volatile {{.*}} seq_cst
+; CHECK: load atomic volatile {{.*}} seq_cst
+entry:
+  %ptr = alloca i64, align 8
+  store atomic volatile i64 0, i64* %ptr seq_cst, align 8
+  %load = load atomic volatile i64, i64* %ptr seq_cst, align 8
+  ret void
+}
diff --git a/test/Transforms/SROA/big-endian.ll b/test/Transforms/SROA/big-endian.ll
index b5a04ca8e64..4de7bfcb898 100644
--- a/test/Transforms/SROA/big-endian.ll
+++ b/test/Transforms/SROA/big-endian.ll
@@ -112,3 +112,126 @@ entry:
 ; CHECK-NEXT: %[[ret:.*]] = zext i56 %[[insert4]] to i64
 ; CHECK-NEXT: ret i64 %[[ret]]
 }
+
+define i64 @PR14132(i1 %flag) {
+; CHECK-LABEL: @PR14132(
+; Here we form a PHI-node by promoting the pointer alloca first, and then in
+; order to promote the other two allocas, we speculate the load of the
+; now-phi-node-pointer. In doing so we end up loading a 64-bit value from an i8
+; alloca. While this is a bit dubious, we were asserting on trying to
+; rewrite it. The trick is that the code using the value may carefully take
+; steps to only use the not-undef bits, and so we need to at least loosely
+; support this. This test is particularly interesting because how we handle
+; a load of an i64 from an i8 alloca is dependent on endianness.
+entry:
+  %a = alloca i64, align 8
+  %b = alloca i8, align 8
+  %ptr = alloca i64*, align 8
+; CHECK-NOT: alloca
+
+  %ptr.cast = bitcast i64** %ptr to i8**
+  store i64 0, i64* %a
+  store i8 1, i8* %b
+  store i64* %a, i64** %ptr
+  br i1 %flag, label %if.then, label %if.end
+
+if.then:
+  store i8* %b, i8** %ptr.cast
+  br label %if.end
+; CHECK-NOT: store
+; CHECK: %[[ext:.*]] = zext i8 1 to i64
+; CHECK: %[[shift:.*]] = shl i64 %[[ext]], 56
+
+if.end:
+  %tmp = load i64*, i64** %ptr
+  %result = load i64, i64* %tmp
+; CHECK-NOT: load
+; CHECK: %[[result:.*]] = phi i64 [ %[[shift]], %if.then ], [ 0, %entry ]
+
+  ret i64 %result
+; CHECK-NEXT: ret i64 %[[result]]
+}
+
+declare void @f(i64 %x, i32 %y)
+
+define void @test3() {
+; CHECK-LABEL: @test3(
+;
+; This is a test that specifically exercises the big-endian lowering because it
+; ends up splitting a 64-bit integer into two smaller integers and has a number
+; of tricky aspects (the i24 type) that make that hard. Historically, SROA
+; would miscompile this by either dropping a most significant byte or least
+; significant byte due to shrinking the [4,8) slice to an i24, or by failing to
+; move the bytes around correctly.
+;
+; The magical number 34494054408 is used because it has bits set in various
+; bytes so that it is clear if those bytes fail to be propagated.
+;
+; If you're debugging this, rather than using the direct magical numbers, run
+; the IR through '-sroa -instcombine'. With '-instcombine' these will be
+; constant folded, and if the i64 doesn't round-trip correctly, you've found
+; a bug!
+;
+entry:
+  %a = alloca { i32, i24 }, align 4
+; CHECK-NOT: alloca
+
+  %tmp0 = bitcast { i32, i24 }* %a to i64*
+  store i64 34494054408, i64* %tmp0
+  %tmp1 = load i64, i64* %tmp0, align 4
+  %tmp2 = bitcast { i32, i24 }* %a to i32*
+  %tmp3 = load i32, i32* %tmp2, align 4
+; CHECK: %[[HI_EXT:.*]] = zext i32 134316040 to i64
+; CHECK: %[[HI_INPUT:.*]] = and i64 undef, -4294967296
+; CHECK: %[[HI_MERGE:.*]] = or i64 %[[HI_INPUT]], %[[HI_EXT]]
+; CHECK: %[[LO_EXT:.*]] = zext i32 8 to i64
+; CHECK: %[[LO_SHL:.*]] = shl i64 %[[LO_EXT]], 32
+; CHECK: %[[LO_INPUT:.*]] = and i64 %[[HI_MERGE]], 4294967295
+; CHECK: %[[LO_MERGE:.*]] = or i64 %[[LO_INPUT]], %[[LO_SHL]]
+
+  call void @f(i64 %tmp1, i32 %tmp3)
+; CHECK: call void @f(i64 %[[LO_MERGE]], i32 8)
+  ret void
+; CHECK: ret void
+}
+
+define void @test4() {
+; CHECK-LABEL: @test4
+;
+; Much like @test3, this is specifically testing big-endian management of data.
+; Also similarly, it uses constants with particular bits set to help track
+; whether values are corrupted, and can be easily evaluated by running through
+; -instcombine to see that the i64 round-trips.
+;
+entry:
+  %a = alloca { i32, i24 }, align 4
+  %a2 = alloca i64, align 4
+; CHECK-NOT: alloca
+
+  store i64 34494054408, i64* %a2
+  %tmp0 = bitcast { i32, i24 }* %a to i8*
+  %tmp1 = bitcast i64* %a2 to i8*
+  call void @llvm.memcpy.p0i8.p0i8.i64(i8* %tmp0, i8* %tmp1, i64 8, i32 4, i1 false)
+; CHECK: %[[LO_SHR:.*]] = lshr i64 34494054408, 32
+; CHECK: %[[LO_START:.*]] = trunc i64 %[[LO_SHR]] to i32
+; CHECK: %[[HI_START:.*]] = trunc i64 34494054408 to i32
+
+  %tmp2 = bitcast { i32, i24 }* %a to i64*
+  %tmp3 = load i64, i64* %tmp2, align 4
+  %tmp4 = bitcast { i32, i24 }* %a to i32*
+  %tmp5 = load i32, i32* %tmp4, align 4
+; CHECK: %[[HI_EXT:.*]] = zext i32 %[[HI_START]] to i64
+; CHECK: %[[HI_INPUT:.*]] = and i64 undef, -4294967296
+; CHECK: %[[HI_MERGE:.*]] = or i64 %[[HI_INPUT]], %[[HI_EXT]]
+; CHECK: %[[LO_EXT:.*]] = zext i32 %[[LO_START]] to i64
+; CHECK: %[[LO_SHL:.*]] = shl i64 %[[LO_EXT]], 32
+; CHECK: %[[LO_INPUT:.*]] = and i64 %[[HI_MERGE]], 4294967295
+; CHECK: %[[LO_MERGE:.*]] = or i64 %[[LO_INPUT]], %[[LO_SHL]]
+
+  call void @f(i64 %tmp3, i32 %tmp5)
+; CHECK: call void @f(i64 %[[LO_MERGE]], i32 %[[LO_START]])
+  ret void
+; CHECK: ret void
+}
+
+declare void @llvm.memcpy.p0i8.p0i8.i64(i8*, i8*, i64, i32, i1)
diff --git a/test/Transforms/SROA/phi-and-select.ll b/test/Transforms/SROA/phi-and-select.ll
index e97bd66d052..fb76548b1d1 100644
--- a/test/Transforms/SROA/phi-and-select.ll
+++ b/test/Transforms/SROA/phi-and-select.ll
@@ -438,26 +438,26 @@ define i64 @PR14132(i1 %flag) {
 ; steps to only use the not-undef bits, and so we need to at least loosely
 ; support this..
 entry:
-  %a = alloca i64
-  %b = alloca i8
-  %ptr = alloca i64*
+  %a = alloca i64, align 8
+  %b = alloca i8, align 8
+  %ptr = alloca i64*, align 8
 ; CHECK-NOT: alloca
 
   %ptr.cast = bitcast i64** %ptr to i8**
-  store i64 0, i64* %a
-  store i8 1, i8* %b
-  store i64* %a, i64** %ptr
+  store i64 0, i64* %a, align 8
+  store i8 1, i8* %b, align 8
+  store i64* %a, i64** %ptr, align 8
   br i1 %flag, label %if.then, label %if.end
 
 if.then:
-  store i8* %b, i8** %ptr.cast
+  store i8* %b, i8** %ptr.cast, align 8
   br label %if.end
 ; CHECK-NOT: store
 ; CHECK: %[[ext:.*]] = zext i8 1 to i64
 
 if.end:
-  %tmp = load i64*, i64** %ptr
-  %result = load i64, i64* %tmp
+  %tmp = load i64*, i64** %ptr, align 8
+  %result = load i64, i64* %tmp, align 8
 ; CHECK-NOT: load
 ; CHECK: %[[result:.*]] = phi i64 [ %[[ext]], %if.then ], [ 0, %entry ]
 
diff --git a/test/Transforms/SafeStack/addr-taken.ll b/test/Transforms/SafeStack/addr-taken.ll
new file mode 100644
index 00000000000..0780a01fa89
--- /dev/null
+++ b/test/Transforms/SafeStack/addr-taken.ll
@@ -0,0 +1,22 @@
+; RUN: opt -safe-stack -S -mtriple=i386-pc-linux-gnu < %s -o - | FileCheck %s
+; RUN: opt -safe-stack -S -mtriple=x86_64-pc-linux-gnu < %s -o - | FileCheck %s
+
+@.str = private unnamed_addr constant [4 x i8] c"%s\0A\00", align 1
+
+; Address-of local taken (j = &a)
+; Requires protector.
+
+define void @foo() nounwind uwtable safestack {
+entry:
+  ; CHECK: __safestack_unsafe_stack_ptr
+  %retval = alloca i32, align 4
+  %a = alloca i32, align 4
+  %j = alloca i32*, align 8
+  store i32 0, i32* %retval
+  %0 = load i32, i32* %a, align 4
+  %add = add nsw i32 %0, 1
+  store i32 %add, i32* %a, align 4
+  store i32* %a, i32** %j, align 8
+  ret void
+}
+
diff --git a/test/Transforms/SafeStack/array-aligned.ll b/test/Transforms/SafeStack/array-aligned.ll
new file mode 100644
index 00000000000..4676903ec77
--- /dev/null
+++ b/test/Transforms/SafeStack/array-aligned.ll
@@ -0,0 +1,39 @@
+; RUN: opt -safe-stack -S -mtriple=i386-pc-linux-gnu < %s -o - | FileCheck %s
+; RUN: opt -safe-stack -S -mtriple=x86_64-pc-linux-gnu < %s -o - | FileCheck %s
+
+@.str = private unnamed_addr constant [4 x i8] c"%s\0A\00", align 1
+
+; array of [16 x i8]
+
+define void @foo(i8* %a) nounwind uwtable safestack {
+entry:
+  ; CHECK: %[[USP:.*]] = load i8*, i8** @__safestack_unsafe_stack_ptr
+
+  ; CHECK: %[[USST:.*]] = getelementptr i8, i8* %[[USP]], i32 -16
+
+  ; CHECK: store i8* %[[USST]], i8** @__safestack_unsafe_stack_ptr
+
+  ; CHECK: %[[AADDR:.*]] = alloca i8*, align 8
+  %a.addr = alloca i8*, align 8
+
+  ; CHECK: %[[BUFPTR:.*]] = getelementptr i8, i8* %[[USP]], i32 -16
+  ; CHECK: %[[BUFPTR2:.*]] = bitcast i8* %[[BUFPTR]] to [16 x i8]*
+  %buf = alloca [16 x i8], align 16
+
+  ; CHECK: store i8* {{.*}}, i8** %[[AADDR]], align 8
+  store i8* %a, i8** %a.addr, align 8
+
+  ; CHECK: %[[GEP:.*]] = getelementptr inbounds [16 x i8], [16 x i8]* %[[BUFPTR2]], i32 0, i32 0
+  %gep = getelementptr inbounds [16 x i8], [16 x i8]* %buf, i32 0, i32 0
+
+  ; CHECK: %[[A2:.*]] = load i8*, i8** %[[AADDR]], align 8
+  %a2 = load i8*, i8** %a.addr, align 8
+
+  ; CHECK: call i8* @strcpy(i8* %[[GEP]], i8* %[[A2]])
+  %call = call i8* @strcpy(i8* %gep, i8* %a2)
+
+  ; CHECK: store i8* %[[USP]], i8** @__safestack_unsafe_stack_ptr
+  ret void
+}
+
+declare i8* @strcpy(i8*, i8*)
diff --git a/test/Transforms/SafeStack/array.ll b/test/Transforms/SafeStack/array.ll
new file mode 100644
index 00000000000..6036bfc2c9c
--- /dev/null
+++ b/test/Transforms/SafeStack/array.ll
@@ -0,0 +1,38 @@
+; RUN: opt -safe-stack -S -mtriple=i386-pc-linux-gnu < %s -o - | FileCheck %s
+; RUN: opt -safe-stack -S -mtriple=x86_64-pc-linux-gnu < %s -o - | FileCheck %s
+
+; array [4 x i8]
+; Requires protector.
+
+define void @foo(i8* %a) nounwind uwtable safestack {
+entry:
+  ; CHECK: %[[USP:.*]] = load i8*, i8** @__safestack_unsafe_stack_ptr
+
+  ; CHECK: %[[USST:.*]] = getelementptr i8, i8* %[[USP]], i32 -16
+
+  ; CHECK: store i8* %[[USST]], i8** @__safestack_unsafe_stack_ptr
+
+  ; CHECK: %[[AADDR:.*]] = alloca i8*, align 8
+  %a.addr = alloca i8*, align 8
+
+  ; CHECK: %[[BUFPTR:.*]] = getelementptr i8, i8* %[[USP]], i32 -4
+  ; CHECK: %[[BUFPTR2:.*]] = bitcast i8* %[[BUFPTR]] to [4 x i8]*
+  %buf = alloca [4 x i8], align 1
+
+  ; CHECK: store i8* {{.*}}, i8** %[[AADDR]], align 8
+  store i8* %a, i8** %a.addr, align 8
+
+  ; CHECK: %[[GEP:.*]] = getelementptr inbounds [4 x i8], [4 x i8]* %[[BUFPTR2]], i32 0, i32 0
+  %gep = getelementptr inbounds [4 x i8], [4 x i8]* %buf, i32 0, i32 0
+
+  ; CHECK: %[[A2:.*]] = load i8*, i8** %[[AADDR]], align 8
+  %a2 = load i8*, i8** %a.addr, align 8
+
+  ; CHECK: call i8* @strcpy(i8* %[[GEP]], i8* %[[A2]])
+  %call = call i8* @strcpy(i8* %gep, i8* %a2)
+
+  ; CHECK: store i8* %[[USP]], i8** @__safestack_unsafe_stack_ptr
+  ret void
+}
+
+declare i8* @strcpy(i8*, i8*)
diff --git a/test/Transforms/SafeStack/call.ll b/test/Transforms/SafeStack/call.ll
new file mode 100644
index 00000000000..ac12ec02b0b
--- /dev/null
+++ b/test/Transforms/SafeStack/call.ll
@@ -0,0 +1,20 @@
+; RUN: opt -safe-stack -S -mtriple=i386-pc-linux-gnu < %s -o - | FileCheck %s
+; RUN: opt -safe-stack -S -mtriple=x86_64-pc-linux-gnu < %s -o - | FileCheck %s
+
+@.str = private unnamed_addr constant [4 x i8] c"%s\0A\00", align 1
+
+; no arrays / no nested arrays
+; Requires no protector.
+
+; CHECK-LABEL: @foo(
+define void @foo(i8* %a) nounwind uwtable safestack {
+entry:
+  ; CHECK-NOT: __safestack_unsafe_stack_ptr
+  %a.addr = alloca i8*, align 8
+  store i8* %a, i8** %a.addr, align 8
+  %0 = load i8*, i8** %a.addr, align 8
+  %call = call i32 (i8*, ...) @printf(i8* getelementptr inbounds ([4 x i8], [4 x i8]* @.str, i32 0, i32 0), i8* %0)
+  ret void
+}
+
+declare i32 @printf(i8*, ...)
diff --git a/test/Transforms/SafeStack/cast.ll b/test/Transforms/SafeStack/cast.ll
new file mode 100644
index 00000000000..df6273a117c
--- /dev/null
+++ b/test/Transforms/SafeStack/cast.ll
@@ -0,0 +1,17 @@
+; RUN: opt -safe-stack -S -mtriple=i386-pc-linux-gnu < %s -o - | FileCheck %s
+; RUN: opt -safe-stack -S -mtriple=x86_64-pc-linux-gnu < %s -o - | FileCheck %s
+
+@.str = private unnamed_addr constant [4 x i8] c"%s\0A\00", align 1
+
+; PtrToInt/IntToPtr Cast
+; Requires no protector.
+
+; CHECK-LABEL: @foo(
+define void @foo() nounwind uwtable safestack {
+entry:
+  ; CHECK-NOT: __safestack_unsafe_stack_ptr
+  %a = alloca i32, align 4
+  %0 = ptrtoint i32* %a to i64
+  %1 = inttoptr i64 %0 to i32*
+  ret void
+}
diff --git a/test/Transforms/SafeStack/constant-gep-call.ll b/test/Transforms/SafeStack/constant-gep-call.ll
new file mode 100644
index 00000000000..456c1cb1596
--- /dev/null
+++ b/test/Transforms/SafeStack/constant-gep-call.ll
@@ -0,0 +1,26 @@
+; RUN: opt -safe-stack -S -mtriple=i386-pc-linux-gnu < %s -o - | FileCheck %s
+; RUN: opt -safe-stack -S -mtriple=x86_64-pc-linux-gnu < %s -o - | FileCheck %s
+
+%struct.nest = type { %struct.pair, %struct.pair }
+%struct.pair = type { i32, i32 }
+
+@.str = private unnamed_addr constant [4 x i8] c"%s\0A\00", align 1
+
+; Nested structure, no arrays, no address-of expressions.
+; Verify that the resulting gep-of-gep does not incorrectly trigger
+; a safe stack protector.
+; safestack attribute
+; Requires no protector.
+; CHECK-LABEL: @foo(
+define void @foo() nounwind uwtable safestack {
+entry:
+  ; CHECK-NOT: __safestack_unsafe_stack_ptr
+  %c = alloca %struct.nest, align 4
+  %b = getelementptr inbounds %struct.nest, %struct.nest* %c, i32 0, i32 1
+  %_a = getelementptr inbounds %struct.pair, %struct.pair* %b, i32 0, i32 0
+  %0 = load i32, i32* %_a, align 4
+  %call = call i32 (i8*, ...) @printf(i8* getelementptr inbounds ([4 x i8], [4 x i8]* @.str, i32 0, i32 0), i32 %0)
+  ret void
+}
+
+declare i32 @printf(i8*, ...)
diff --git a/test/Transforms/SafeStack/constant-gep.ll b/test/Transforms/SafeStack/constant-gep.ll
new file mode 100644
index 00000000000..6468a761dd5
--- /dev/null
+++ b/test/Transforms/SafeStack/constant-gep.ll
@@ -0,0 +1,20 @@
+; RUN: opt -safe-stack -S -mtriple=i386-pc-linux-gnu < %s -o - | FileCheck %s
+; RUN: opt -safe-stack -S -mtriple=x86_64-pc-linux-gnu < %s -o - | FileCheck %s
+
+%class.A = type { [2 x i8] }
+
+@.str = private unnamed_addr constant [4 x i8] c"%s\0A\00", align 1
+
+; [2 x i8] in a class
+;  safestack attribute
+; Requires no protector.
+; CHECK-LABEL: @foo(
+define signext i8 @foo() nounwind uwtable safestack {
+entry:
+  ; CHECK-NOT: __safestack_unsafe_stack_ptr
+  %a = alloca %class.A, align 1
+  %array = getelementptr inbounds %class.A, %class.A* %a, i32 0, i32 0
+  %arrayidx = getelementptr inbounds [2 x i8], [2 x i8]* %array, i32 0, i64 0
+  %0 = load i8, i8* %arrayidx, align 1
+  ret i8 %0
+}
diff --git a/test/Transforms/SafeStack/constant-geps.ll b/test/Transforms/SafeStack/constant-geps.ll
new file mode 100644
index 00000000000..8a6f7549bb4
--- /dev/null
+++ b/test/Transforms/SafeStack/constant-geps.ll
@@ -0,0 +1,28 @@
+; RUN: opt -safe-stack -S -mtriple=i386-pc-linux-gnu < %s -o - | FileCheck %s
+; RUN: opt -safe-stack -S -mtriple=x86_64-pc-linux-gnu < %s -o - | FileCheck %s
+
+%struct.deep = type { %union.anon }
+%union.anon = type { %struct.anon }
+%struct.anon = type { %struct.anon.0 }
+%struct.anon.0 = type { %union.anon.1 }
+%union.anon.1 = type { [2 x i8] }
+
+@.str = private unnamed_addr constant [4 x i8] c"%s\0A\00", align 1
+
+; [2 x i8] nested in several layers of structs and unions
+;  safestack attribute
+; Requires no protector.
+; CHECK-LABEL: @foo(
+define signext i8 @foo() nounwind uwtable safestack {
+entry:
+  ; CHECK-NOT: __safestack_unsafe_stack_ptr
+  %x = alloca %struct.deep, align 1
+  %b = getelementptr inbounds %struct.deep, %struct.deep* %x, i32 0, i32 0
+  %c = bitcast %union.anon* %b to %struct.anon*
+  %d = getelementptr inbounds %struct.anon, %struct.anon* %c, i32 0, i32 0
+  %e = getelementptr inbounds %struct.anon.0, %struct.anon.0* %d, i32 0, i32 0
+  %array = bitcast %union.anon.1* %e to [2 x i8]*
+  %arrayidx = getelementptr inbounds [2 x i8], [2 x i8]* %array, i32 0, i64 0
+  %0 = load i8, i8* %arrayidx, align 1
+  ret i8 %0
+}
diff --git a/test/Transforms/SafeStack/dynamic-alloca.ll b/test/Transforms/SafeStack/dynamic-alloca.ll
new file mode 100644
index 00000000000..bfec66f82a2
--- /dev/null
+++ b/test/Transforms/SafeStack/dynamic-alloca.ll
@@ -0,0 +1,21 @@
+; RUN: opt -safe-stack -S -mtriple=i386-pc-linux-gnu < %s -o - | FileCheck %s
+; RUN: opt -safe-stack -S -mtriple=x86_64-pc-linux-gnu < %s -o - | FileCheck %s
+
+@.str = private unnamed_addr constant [4 x i8] c"%s\0A\00", align 1
+
+; Variable sized alloca
+;  safestack attribute
+; Requires protector.
+define void @foo(i32 %n) nounwind uwtable safestack {
+entry:
+  ; CHECK: __safestack_unsafe_stack_ptr
+  %n.addr = alloca i32, align 4
+  %a = alloca i32*, align 8
+  store i32 %n, i32* %n.addr, align 4
+  %0 = load i32, i32* %n.addr, align 4
+  %conv = sext i32 %0 to i64
+  %1 = alloca i8, i64 %conv
+  %2 = bitcast i8* %1 to i32*
+  store i32* %2, i32** %a, align 8
+  ret void
+}
diff --git a/test/Transforms/SafeStack/escape-addr-pointer.ll b/test/Transforms/SafeStack/escape-addr-pointer.ll
new file mode 100644
index 00000000000..615d711f62c
--- /dev/null
+++ b/test/Transforms/SafeStack/escape-addr-pointer.ll
@@ -0,0 +1,23 @@
+; RUN: opt -safe-stack -S -mtriple=i386-pc-linux-gnu < %s -o - | FileCheck %s
+; RUN: opt -safe-stack -S -mtriple=x86_64-pc-linux-gnu < %s -o - | FileCheck %s
+
+@.str = private unnamed_addr constant [4 x i8] c"%s\0A\00", align 1
+
+; Addr-of a pointer
+;  safestack attribute
+; Requires protector.
+define void @foo() nounwind uwtable safestack {
+entry:
+  ; CHECK: __safestack_unsafe_stack_ptr
+  %a = alloca i32*, align 8
+  %b = alloca i32**, align 8
+  %call = call i32* @getp()
+  store i32* %call, i32** %a, align 8
+  store i32** %a, i32*** %b, align 8
+  %0 = load i32**, i32*** %b, align 8
+  call void @funcall2(i32** %0)
+  ret void
+}
+
+declare void @funcall2(i32**)
+declare i32* @getp()
diff --git a/test/Transforms/SafeStack/escape-bitcast-store.ll b/test/Transforms/SafeStack/escape-bitcast-store.ll
new file mode 100644
index 00000000000..9d556a6782a
--- /dev/null
+++ b/test/Transforms/SafeStack/escape-bitcast-store.ll
@@ -0,0 +1,23 @@
+; RUN: opt -safe-stack -S -mtriple=i386-pc-linux-gnu < %s -o - | FileCheck %s
+; RUN: opt -safe-stack -S -mtriple=x86_64-pc-linux-gnu < %s -o - | FileCheck %s
+
+@.str = private unnamed_addr constant [4 x i8] c"%s\0A\00", align 1
+
+; Addr-of a local cast to a ptr of a different type
+;   (e.g., int a; ... ; float *b = &a;)
+;  safestack attribute
+; Requires protector.
+define void @foo() nounwind uwtable safestack {
+entry:
+  ; CHECK: __safestack_unsafe_stack_ptr
+  %a = alloca i32, align 4
+  %b = alloca float*, align 8
+  store i32 0, i32* %a, align 4
+  %0 = bitcast i32* %a to float*
+  store float* %0, float** %b, align 8
+  %1 = load float*, float** %b, align 8
+  %call = call i32 (i8*, ...) @printf(i8* getelementptr inbounds ([4 x i8], [4 x i8]* @.str, i32 0, i32 0), float* %1)
+  ret void
+}
+
+declare i32 @printf(i8*, ...)
diff --git a/test/Transforms/SafeStack/escape-bitcast-store2.ll b/test/Transforms/SafeStack/escape-bitcast-store2.ll
new file mode 100644
index 00000000000..5f1f873f422
--- /dev/null
+++ b/test/Transforms/SafeStack/escape-bitcast-store2.ll
@@ -0,0 +1,20 @@
+; RUN: opt -safe-stack -S -mtriple=i386-pc-linux-gnu < %s -o - | FileCheck %s
+; RUN: opt -safe-stack -S -mtriple=x86_64-pc-linux-gnu < %s -o - | FileCheck %s
+
+@.str = private unnamed_addr constant [4 x i8] c"%s\0A\00", align 1
+
+; Addr-of a local cast to a ptr of a different type (optimized)
+;   (e.g., int a; ... ; float *b = &a;)
+;  safestack attribute
+; Requires protector.
+define void @foo() nounwind uwtable safestack {
+entry:
+  ; CHECK: __safestack_unsafe_stack_ptr
+  %a = alloca i32, align 4
+  store i32 0, i32* %a, align 4
+  %0 = bitcast i32* %a to float*
+  call void @funfloat(float* %0) nounwind
+  ret void
+}
+
+declare void @funfloat(float*)
diff --git a/test/Transforms/SafeStack/escape-call.ll b/test/Transforms/SafeStack/escape-call.ll
new file mode 100644
index 00000000000..ce09780d2ea
--- /dev/null
+++ b/test/Transforms/SafeStack/escape-call.ll
@@ -0,0 +1,16 @@
+; RUN: opt -safe-stack -S -mtriple=i386-pc-linux-gnu < %s -o - | FileCheck %s
+; RUN: opt -safe-stack -S -mtriple=x86_64-pc-linux-gnu < %s -o - | FileCheck %s
+
+@.str = private unnamed_addr constant [4 x i8] c"%s\0A\00", align 1
+
+; Passing addr-of to function call
+; Requires protector.
+define void @foo() nounwind uwtable safestack {
+entry:
+  ; CHECK: __safestack_unsafe_stack_ptr
+  %b = alloca i32, align 4
+  call void @funcall(i32* %b) nounwind
+  ret void
+}
+
+declare void @funcall(i32*)
diff --git a/test/Transforms/SafeStack/escape-casted-pointer.ll b/test/Transforms/SafeStack/escape-casted-pointer.ll
new file mode 100644
index 00000000000..bf6ce1d6b2a
--- /dev/null
+++ b/test/Transforms/SafeStack/escape-casted-pointer.ll
@@ -0,0 +1,24 @@
+; RUN: opt -safe-stack -S -mtriple=i386-pc-linux-gnu < %s -o - | FileCheck %s
+; RUN: opt -safe-stack -S -mtriple=x86_64-pc-linux-gnu < %s -o - | FileCheck %s
+
+@.str = private unnamed_addr constant [4 x i8] c"%s\0A\00", align 1
+
+; Addr-of a casted pointer
+;  safestack attribute
+; Requires protector.
+define void @foo() nounwind uwtable safestack {
+entry:
+  ; CHECK: __safestack_unsafe_stack_ptr
+  %a = alloca i32*, align 8
+  %b = alloca float**, align 8
+  %call = call i32* @getp()
+  store i32* %call, i32** %a, align 8
+  %0 = bitcast i32** %a to float**
+  store float** %0, float*** %b, align 8
+  %1 = load float**, float*** %b, align 8
+  call void @funfloat2(float** %1)
+  ret void
+}
+
+declare void @funfloat2(float**)
+declare i32* @getp()
diff --git a/test/Transforms/SafeStack/escape-gep-call.ll b/test/Transforms/SafeStack/escape-gep-call.ll
new file mode 100644
index 00000000000..42b5dd5c1e7
--- /dev/null
+++ b/test/Transforms/SafeStack/escape-gep-call.ll
@@ -0,0 +1,20 @@
+; RUN: opt -safe-stack -S -mtriple=i386-pc-linux-gnu < %s -o - | FileCheck %s
+; RUN: opt -safe-stack -S -mtriple=x86_64-pc-linux-gnu < %s -o - | FileCheck %s
+
+%struct.pair = type { i32, i32 }
+
+@.str = private unnamed_addr constant [4 x i8] c"%s\0A\00", align 1
+
+; Addr-of struct element, GEP followed by callinst.
+;  safestack attribute
+; Requires protector.
+define void @foo() nounwind uwtable safestack {
+entry:
+  ; CHECK: __safestack_unsafe_stack_ptr
+  %c = alloca %struct.pair, align 4
+  %y = getelementptr inbounds %struct.pair, %struct.pair* %c, i64 0, i32 1
+  %call = call i32 (i8*, ...) @printf(i8* getelementptr inbounds ([4 x i8], [4 x i8]* @.str, i64 0, i64 0), i32* %y) nounwind
+  ret void
+}
+
+declare i32 @printf(i8*, ...)
diff --git a/test/Transforms/SafeStack/escape-gep-invoke.ll b/test/Transforms/SafeStack/escape-gep-invoke.ll
new file mode 100644
index 00000000000..8495ff985f6
--- /dev/null
+++ b/test/Transforms/SafeStack/escape-gep-invoke.ll
@@ -0,0 +1,34 @@
+; RUN: opt -safe-stack -S -mtriple=i386-pc-linux-gnu < %s -o - | FileCheck %s
+; RUN: opt -safe-stack -S -mtriple=x86_64-pc-linux-gnu < %s -o - | FileCheck %s
+
+%struct.pair = type { i32, i32 }
+
+@.str = private unnamed_addr constant [4 x i8] c"%s\0A\00", align 1
+
+; Addr-of a struct element passed into an invoke instruction.
+;   (GEP followed by an invoke)
+;  safestack attribute
+; Requires protector.
+define i32 @foo() uwtable safestack personality i8* bitcast (i32 (...)* @__gxx_personality_v0 to i8*) {
+entry:
+  ; CHECK: __safestack_unsafe_stack_ptr
+  %c = alloca %struct.pair, align 4
+  %exn.slot = alloca i8*
+  %ehselector.slot = alloca i32
+  %a = getelementptr inbounds %struct.pair, %struct.pair* %c, i32 0, i32 0
+  store i32 0, i32* %a, align 4
+  %a1 = getelementptr inbounds %struct.pair, %struct.pair* %c, i32 0, i32 0
+  invoke void @_Z3exceptPi(i32* %a1)
+          to label %invoke.cont unwind label %lpad
+
+invoke.cont:
+  ret i32 0
+
+lpad:
+  %0 = landingpad { i8*, i32 }
+          catch i8* null
+  ret i32 0
+}
+
+declare void @_Z3exceptPi(i32*)
+declare i32 @__gxx_personality_v0(...)
diff --git a/test/Transforms/SafeStack/escape-gep-negative.ll b/test/Transforms/SafeStack/escape-gep-negative.ll
new file mode 100644
index 00000000000..80d405de36d
--- /dev/null
+++ b/test/Transforms/SafeStack/escape-gep-negative.ll
@@ -0,0 +1,18 @@
+; RUN: opt -safe-stack -S -mtriple=i386-pc-linux-gnu < %s -o - | FileCheck %s
+; RUN: opt -safe-stack -S -mtriple=x86_64-pc-linux-gnu < %s -o - | FileCheck %s
+
+@.str = private unnamed_addr constant [4 x i8] c"%s\0A\00", align 1
+
+; Addr-of a local, optimized into a GEP (e.g., &a - 12)
+;  safestack attribute
+; Requires protector.
+define void @foo() nounwind uwtable safestack {
+entry:
+  ; CHECK: __safestack_unsafe_stack_ptr
+  %a = alloca i32, align 4
+  %add.ptr5 = getelementptr inbounds i32, i32* %a, i64 -12
+  %call = call i32 (i8*, ...) @printf(i8* getelementptr inbounds ([4 x i8], [4 x i8]* @.str, i64 0, i64 0), i32* %add.ptr5) nounwind
+  ret void
+}
+
+declare i32 @printf(i8*, ...)
diff --git a/test/Transforms/SafeStack/escape-gep-ptrtoint.ll b/test/Transforms/SafeStack/escape-gep-ptrtoint.ll
new file mode 100644
index 00000000000..73a8e58fb08
--- /dev/null
+++ b/test/Transforms/SafeStack/escape-gep-ptrtoint.ll
@@ -0,0 +1,22 @@
+; RUN: opt -safe-stack -S -mtriple=i386-pc-linux-gnu < %s -o - | FileCheck %s
+; RUN: opt -safe-stack -S -mtriple=x86_64-pc-linux-gnu < %s -o - | FileCheck %s
+
+%struct.pair = type { i32, i32 }
+
+@.str = private unnamed_addr constant [4 x i8] c"%s\0A\00", align 1
+
+; Addr-of struct element, GEP followed by ptrtoint.
+;  safestack attribute
+; Requires protector.
+define void @foo() nounwind uwtable safestack {
+entry:
+  ; CHECK: __safestack_unsafe_stack_ptr
+  %c = alloca %struct.pair, align 4
+  %b = alloca i32*, align 8
+  %y = getelementptr inbounds %struct.pair, %struct.pair* %c, i32 0, i32 1
+  %0 = ptrtoint i32* %y to i64
+  %call = call i32 (i8*, ...) @printf(i8* getelementptr inbounds ([4 x i8], [4 x i8]* @.str, i32 0, i32 0), i64 %0)
+  ret void
+}
+
+declare i32 @printf(i8*, ...)
diff --git a/test/Transforms/SafeStack/escape-gep-store.ll b/test/Transforms/SafeStack/escape-gep-store.ll
new file mode 100644
index 00000000000..7c6c0a318b1
--- /dev/null
+++ b/test/Transforms/SafeStack/escape-gep-store.ll
@@ -0,0 +1,23 @@
+; RUN: opt -safe-stack -S -mtriple=i386-pc-linux-gnu < %s -o - | FileCheck %s
+; RUN: opt -safe-stack -S -mtriple=x86_64-pc-linux-gnu < %s -o - | FileCheck %s
+
+%struct.pair = type { i32, i32 }
+
+@.str = private unnamed_addr constant [4 x i8] c"%s\0A\00", align 1
+
+; Addr-of struct element. (GEP followed by store).
+;  safestack attribute
+; Requires protector.
+define void @foo() nounwind uwtable safestack {
+entry:
+  ; CHECK: __safestack_unsafe_stack_ptr
+  %c = alloca %struct.pair, align 4
+  %b = alloca i32*, align 8
+  %y = getelementptr inbounds %struct.pair, %struct.pair* %c, i32 0, i32 1
+  store i32* %y, i32** %b, align 8
+  %0 = load i32*, i32** %b, align 8
+  %call = call i32 (i8*, ...) @printf(i8* getelementptr inbounds ([4 x i8], [4 x i8]* @.str, i32 0, i32 0), i32* %0)
+  ret void
+}
+
+declare i32 @printf(i8*, ...)
diff --git a/test/Transforms/SafeStack/escape-phi-call.ll b/test/Transforms/SafeStack/escape-phi-call.ll
new file mode 100644
index 00000000000..10b6c1fdce4
--- /dev/null
+++ b/test/Transforms/SafeStack/escape-phi-call.ll
@@ -0,0 +1,36 @@
+; RUN: opt -safe-stack -S -mtriple=i386-pc-linux-gnu < %s -o - | FileCheck %s
+; RUN: opt -safe-stack -S -mtriple=x86_64-pc-linux-gnu < %s -o - | FileCheck %s
+
+@.str = private unnamed_addr constant [4 x i8] c"%s\0A\00", align 1
+
+; Addr-of in phi instruction
+; Requires protector.
+define void @foo() nounwind uwtable safestack {
+entry:
+  ; CHECK: __safestack_unsafe_stack_ptr
+  %x = alloca double, align 8
+  %call = call double @testi_aux() nounwind
+  store double %call, double* %x, align 8
+  %cmp = fcmp ogt double %call, 3.140000e+00
+  br i1 %cmp, label %if.then, label %if.else
+
+if.then:                                          ; preds = %entry
+  %call1 = call double @testi_aux() nounwind
+  store double %call1, double* %x, align 8
+  br label %if.end4
+
+if.else:                                          ; preds = %entry
+  %cmp2 = fcmp ogt double %call, 1.000000e+00
+  br i1 %cmp2, label %if.then3, label %if.end4
+
+if.then3:                                         ; preds = %if.else
+  br label %if.end4
+
+if.end4:                                          ; preds = %if.else, %if.then3, %if.then
+  %y.0 = phi double* [ null, %if.then ], [ %x, %if.then3 ], [ null, %if.else ]
+  %call5 = call i32 (i8*, ...) @printf(i8* getelementptr inbounds ([4 x i8], [4 x i8]* @.str, i64 0, i64 0), double* %y.0) nounwind
+  ret void
+}
+
+declare double @testi_aux()
+declare i32 @printf(i8*, ...)
diff --git a/test/Transforms/SafeStack/escape-select-call.ll b/test/Transforms/SafeStack/escape-select-call.ll
new file mode 100644
index 00000000000..9e54dd8e140
--- /dev/null
+++ b/test/Transforms/SafeStack/escape-select-call.ll
@@ -0,0 +1,22 @@
+; RUN: opt -safe-stack -S -mtriple=i386-pc-linux-gnu < %s -o - | FileCheck %s
+; RUN: opt -safe-stack -S -mtriple=x86_64-pc-linux-gnu < %s -o - | FileCheck %s
+
+@.str = private unnamed_addr constant [4 x i8] c"%s\0A\00", align 1
+
+; Addr-of in select instruction
+; safestack attribute
+; Requires protector.
+define void @foo() nounwind uwtable safestack {
+entry:
+  ; CHECK: __safestack_unsafe_stack_ptr
+  %x = alloca double, align 8
+  %call = call double @testi_aux() nounwind
+  store double %call, double* %x, align 8
+  %cmp2 = fcmp ogt double %call, 0.000000e+00
+  %y.1 = select i1 %cmp2, double* %x, double* null
+  %call2 = call i32 (i8*, ...) @printf(i8* getelementptr inbounds ([4 x i8], [4 x i8]* @.str, i32 0, i32 0), double* %y.1)
+  ret void
+}
+
+declare double @testi_aux()
+declare i32 @printf(i8*, ...)
diff --git a/test/Transforms/SafeStack/escape-vector.ll b/test/Transforms/SafeStack/escape-vector.ll
new file mode 100644
index 00000000000..76b01c7c430
--- /dev/null
+++ b/test/Transforms/SafeStack/escape-vector.ll
@@ -0,0 +1,21 @@
+; RUN: opt -safe-stack -S -mtriple=i386-pc-linux-gnu < %s -o - | FileCheck %s
+; RUN: opt -safe-stack -S -mtriple=x86_64-pc-linux-gnu < %s -o - | FileCheck %s
+
+%struct.vec = type { <4 x i32> }
+
+@.str = private unnamed_addr constant [4 x i8] c"%s\0A\00", align 1
+
+; Addr-of a vector nested in a struct
+;  safestack attribute
+; Requires protector.
+define void @foo() nounwind uwtable safestack {
+entry:
+  ; CHECK: __safestack_unsafe_stack_ptr
+  %c = alloca %struct.vec, align 16
+  %y = getelementptr inbounds %struct.vec, %struct.vec* %c, i64 0, i32 0
+  %add.ptr = getelementptr inbounds <4 x i32>, <4 x i32>* %y, i64 -12
+  %call = call i32 (i8*, ...) @printf(i8* getelementptr inbounds ([4 x i8], [4 x i8]* @.str, i64 0, i64 0), <4 x i32>* %add.ptr) nounwind
+  ret void
+}
+
+declare i32 @printf(i8*, ...)
diff --git a/test/Transforms/SafeStack/invoke.ll b/test/Transforms/SafeStack/invoke.ll
new file mode 100644
index 00000000000..bfebc336cfd
--- /dev/null
+++ b/test/Transforms/SafeStack/invoke.ll
@@ -0,0 +1,33 @@
+; RUN: opt -safe-stack -S -mtriple=i386-pc-linux-gnu < %s -o - | FileCheck %s
+; RUN: opt -safe-stack -S -mtriple=x86_64-pc-linux-gnu < %s -o - | FileCheck %s
+
+@.str = private unnamed_addr constant [4 x i8] c"%s\0A\00", align 1
+
+; Addr-of a variable passed into an invoke instruction.
+;  safestack attribute
+; Requires protector and stack restore after landing pad.
+define i32 @foo() uwtable safestack personality i8* bitcast (i32 (...)* @__gxx_personality_v0 to i8*) {
+entry:
+  ; CHECK: %[[SP:.*]] = load i8*, i8** @__safestack_unsafe_stack_ptr
+  ; CHECK: %[[STATICTOP:.*]] = getelementptr i8, i8* %[[SP]], i32 -16
+  %a = alloca i32, align 4
+  %exn.slot = alloca i8*
+  %ehselector.slot = alloca i32
+  store i32 0, i32* %a, align 4
+  invoke void @_Z3exceptPi(i32* %a)
+          to label %invoke.cont unwind label %lpad
+
+invoke.cont:
+  ret i32 0
+
+lpad:
+  ; CHECK: landingpad
+  ; CHECK-NEXT: catch
+  %0 = landingpad { i8*, i32 }
+          catch i8* null
+  ; CHECK-NEXT: store i8* %[[STATICTOP]], i8** @__safestack_unsafe_stack_ptr
+  ret i32 0
+}
+
+declare void @_Z3exceptPi(i32*)
+declare i32 @__gxx_personality_v0(...)
diff --git a/test/Transforms/SafeStack/no-attr.ll b/test/Transforms/SafeStack/no-attr.ll
new file mode 100644
index 00000000000..d9bcefd3c84
--- /dev/null
+++ b/test/Transforms/SafeStack/no-attr.ll
@@ -0,0 +1,27 @@
+; RUN: opt -safe-stack -S -mtriple=i386-pc-linux-gnu < %s -o - | FileCheck %s
+; RUN: opt -safe-stack -S -mtriple=x86_64-pc-linux-gnu < %s -o - | FileCheck %s
+
+@.str = private unnamed_addr constant [4 x i8] c"%s\0A\00", align 1
+
+; no safestack attribute
+; Requires no protector.
+
+; CHECK-NOT: __safestack_unsafe_stack_ptr
+
+; CHECK: @foo
+define void @foo(i8* %a) nounwind uwtable {
+entry:
+  ; CHECK-NOT: __safestack_unsafe_stack_ptr
+  %a.addr = alloca i8*, align 8
+  %buf = alloca [16 x i8], align 16
+  store i8* %a, i8** %a.addr, align 8
+  %arraydecay = getelementptr inbounds [16 x i8], [16 x i8]* %buf, i32 0, i32 0
+  %0 = load i8*, i8** %a.addr, align 8
+  %call = call i8* @strcpy(i8* %arraydecay, i8* %0)
+  %arraydecay1 = getelementptr inbounds [16 x i8], [16 x i8]* %buf, i32 0, i32 0
+  %call2 = call i32 (i8*, ...) @printf(i8* getelementptr inbounds ([4 x i8], [4 x i8]* @.str, i32 0, i32 0), i8* %arraydecay1)
+  ret void
+}
+
+declare i8* @strcpy(i8*, i8*)
+declare i32 @printf(i8*, ...)
diff --git a/test/Transforms/SafeStack/phi-cycle.ll b/test/Transforms/SafeStack/phi-cycle.ll
new file mode 100644
index 00000000000..026e88785cb
--- /dev/null
+++ b/test/Transforms/SafeStack/phi-cycle.ll
@@ -0,0 +1,50 @@
+; RUN: opt -safe-stack -S -mtriple=i386-pc-linux-gnu < %s -o - | FileCheck %s
+; RUN: opt -safe-stack -S -mtriple=x86_64-pc-linux-gnu < %s -o - | FileCheck %s
+
+%struct.small = type { i8 }
+
+@.str = private unnamed_addr constant [4 x i8] c"%s\0A\00", align 1
+
+; Address-of a structure taken in a function with a loop where
+; the alloca is an incoming value to a PHI node and a use of that PHI
+; node is also an incoming value.
+; Verify that the address-of analysis does not get stuck in infinite
+; recursion when chasing the alloca through the PHI nodes.
+; Requires protector.
+define i32 @foo(i32 %arg) nounwind uwtable safestack {
+bb:
+  ; CHECK: __safestack_unsafe_stack_ptr
+  %tmp = alloca %struct.small*, align 8
+  %tmp1 = call i32 (...) @dummy(%struct.small** %tmp) nounwind
+  %tmp2 = load %struct.small*, %struct.small** %tmp, align 8
+  %tmp3 = ptrtoint %struct.small* %tmp2 to i64
+  %tmp4 = trunc i64 %tmp3 to i32
+  %tmp5 = icmp sgt i32 %tmp4, 0
+  br i1 %tmp5, label %bb6, label %bb21
+
+bb6:                                              ; preds = %bb17, %bb
+  %tmp7 = phi %struct.small* [ %tmp19, %bb17 ], [ %tmp2, %bb ]
+  %tmp8 = phi i64 [ %tmp20, %bb17 ], [ 1, %bb ]
+  %tmp9 = phi i32 [ %tmp14, %bb17 ], [ %tmp1, %bb ]
+  %tmp10 = getelementptr inbounds %struct.small, %struct.small* %tmp7, i64 0, i32 0
+  %tmp11 = load i8, i8* %tmp10, align 1
+  %tmp12 = icmp eq i8 %tmp11, 1
+  %tmp13 = add nsw i32 %tmp9, 8
+  %tmp14 = select i1 %tmp12, i32 %tmp13, i32 %tmp9
+  %tmp15 = trunc i64 %tmp8 to i32
+  %tmp16 = icmp eq i32 %tmp15, %tmp4
+  br i1 %tmp16, label %bb21, label %bb17
+
+bb17:                                             ; preds = %bb6
+  %tmp18 = getelementptr inbounds %struct.small*, %struct.small** %tmp, i64 %tmp8
+  %tmp19 = load %struct.small*, %struct.small** %tmp18, align 8
+  %tmp20 = add i64 %tmp8, 1
+  br label %bb6
+
+bb21:                                             ; preds = %bb6, %bb
+  %tmp22 = phi i32 [ %tmp1, %bb ], [ %tmp14, %bb6 ]
+  %tmp23 = call i32 (...) @dummy(i32 %tmp22) nounwind
+  ret i32 undef
+}
+
+declare i32 @dummy(...)
diff --git a/test/Transforms/SafeStack/setjmp.ll b/test/Transforms/SafeStack/setjmp.ll
new file mode 100644
index 00000000000..e38bff68e94
--- /dev/null
+++ b/test/Transforms/SafeStack/setjmp.ll
@@ -0,0 +1,37 @@
+; RUN: opt -safe-stack -S -mtriple=i386-pc-linux-gnu < %s -o - | FileCheck %s
+; RUN: opt -safe-stack -S -mtriple=x86_64-pc-linux-gnu < %s -o - | FileCheck %s
+
+%struct.__jmp_buf_tag = type { [8 x i64], i32, %struct.__sigset_t }
+%struct.__sigset_t = type { [16 x i64] }
+
+@.str = private unnamed_addr constant [4 x i8] c"%s\0A\00", align 1
+@buf = internal global [1 x %struct.__jmp_buf_tag] zeroinitializer, align 16
+
+; setjmp/longjmp test.
+; Requires protector.
+define i32 @foo() nounwind uwtable safestack {
+entry:
+  ; CHECK: %[[SP:.*]] = load i8*, i8** @__safestack_unsafe_stack_ptr
+  ; CHECK: %[[STATICTOP:.*]] = getelementptr i8, i8* %[[SP]], i32 -16
+  %retval = alloca i32, align 4
+  %x = alloca i32, align 4
+  store i32 0, i32* %retval
+  store i32 42, i32* %x, align 4
+  %call = call i32 @_setjmp(%struct.__jmp_buf_tag* getelementptr inbounds ([1 x %struct.__jmp_buf_tag], [1 x %struct.__jmp_buf_tag]* @buf, i32 0, i32 0)) returns_twice
+  ; CHECK: setjmp
+  ; CHECK-NEXT: store i8* %[[STATICTOP]], i8** @__safestack_unsafe_stack_ptr
+  %tobool = icmp ne i32 %call, 0
+  br i1 %tobool, label %if.else, label %if.then
+if.then:                                          ; preds = %entry
+  call void @funcall(i32* %x)
+  br label %if.end
+if.else:                                          ; preds = %entry
+  call i32 (...) @dummy()
+  br label %if.end
+if.end:                                           ; preds = %if.else, %if.then
+  ret i32 0
+}
+
+declare i32 @_setjmp(%struct.__jmp_buf_tag*)
+declare void @funcall(i32*)
+declare i32 @dummy(...)
diff --git a/test/Transforms/SafeStack/setjmp2.ll b/test/Transforms/SafeStack/setjmp2.ll
new file mode 100644
index 00000000000..65fd920d63d
--- /dev/null
+++ b/test/Transforms/SafeStack/setjmp2.ll
@@ -0,0 +1,42 @@
+; RUN: opt -safe-stack -S -mtriple=i386-pc-linux-gnu < %s -o - | FileCheck %s
+; RUN: opt -safe-stack -S -mtriple=x86_64-pc-linux-gnu < %s -o - | FileCheck %s
+
+%struct.__jmp_buf_tag = type { [8 x i64], i32, %struct.__sigset_t }
+%struct.__sigset_t = type { [16 x i64] }
+
+@.str = private unnamed_addr constant [4 x i8] c"%s\0A\00", align 1
+@buf = internal global [1 x %struct.__jmp_buf_tag] zeroinitializer, align 16
+
+; setjmp/longjmp test with dynamically sized array.
+; Requires protector.
+; CHECK: @foo(i32 %[[ARG:.*]])
+define i32 @foo(i32 %size) nounwind uwtable safestack {
+entry:
+  ; CHECK: %[[DYNPTR:.*]] = alloca i8*
+  ; CHECK-NEXT: %[[SP:.*]] = load i8*, i8** @__safestack_unsafe_stack_ptr
+  ; CHECK-NEXT: store i8* %[[SP]], i8** %[[DYNPTR]]
+
+  ; CHECK-NEXT: %[[ZEXT:.*]] = zext i32 %[[ARG]] to i64
+  ; CHECK-NEXT: %[[MUL:.*]] = mul i64 %[[ZEXT]], 4
+  ; CHECK-NEXT: %[[SP2:.*]] = load i8*, i8** @__safestack_unsafe_stack_ptr
+  ; CHECK-NEXT: %[[PTRTOINT:.*]] = ptrtoint i8* %[[SP2]] to i64
+  ; CHECK-NEXT: %[[SUB:.*]] = sub i64 %[[PTRTOINT]], %[[MUL]]
+  ; CHECK-NEXT: %[[AND:.*]] = and i64 %[[SUB]], -16
+  ; CHECK-NEXT: %[[INTTOPTR:.*]] = inttoptr i64 %[[AND]] to i8*
+  ; CHECK-NEXT: store i8* %[[INTTOPTR]], i8** @__safestack_unsafe_stack_ptr
+  ; CHECK-NEXT: store i8* %[[INTTOPTR]], i8** %unsafe_stack_dynamic_ptr
+  ; CHECK-NEXT: %[[ALLOCA:.*]] = inttoptr i64 %[[SUB]] to i32*
+  %a = alloca i32, i32 %size
+
+  ; CHECK: setjmp
+  ; CHECK-NEXT: %[[LOAD:.*]] = load i8*, i8** %[[DYNPTR]]
+  ; CHECK-NEXT: store i8* %[[LOAD]], i8** @__safestack_unsafe_stack_ptr
+  %call = call i32 @_setjmp(%struct.__jmp_buf_tag* getelementptr inbounds ([1 x %struct.__jmp_buf_tag], [1 x %struct.__jmp_buf_tag]* @buf, i32 0, i32 0)) returns_twice
+
+  ; CHECK: call void @funcall(i32* %[[ALLOCA]])
+  call void @funcall(i32* %a)
+  ret i32 0
+}
+
+declare i32 @_setjmp(%struct.__jmp_buf_tag*)
+declare void @funcall(i32*)
diff --git a/test/Transforms/SafeStack/struct.ll b/test/Transforms/SafeStack/struct.ll
new file mode 100644
index 00000000000..12a0085a2cc
--- /dev/null
+++ b/test/Transforms/SafeStack/struct.ll
@@ -0,0 +1,41 @@
+; RUN: opt -safe-stack -S -mtriple=i386-pc-linux-gnu < %s -o - | FileCheck %s
+; RUN: opt -safe-stack -S -mtriple=x86_64-pc-linux-gnu < %s -o - | FileCheck %s
+
+%struct.foo = type { [16 x i8] }
+
+@.str = private unnamed_addr constant [4 x i8] c"%s\0A\00", align 1
+
+; struct { [16 x i8] }
+
+define void @foo(i8* %a) nounwind uwtable safestack {
+entry:
+  ; CHECK: %[[USP:.*]] = load i8*, i8** @__safestack_unsafe_stack_ptr
+
+  ; CHECK: %[[USST:.*]] = getelementptr i8, i8* %[[USP]], i32 -16
+
+  ; CHECK: store i8* %[[USST]], i8** @__safestack_unsafe_stack_ptr
+
+  ; CHECK: %[[AADDR:.*]] = alloca i8*, align 8
+  %a.addr = alloca i8*, align 8
+
+  ; CHECK: %[[BUFPTR:.*]] = getelementptr i8, i8* %[[USP]], i32 -16
+  ; CHECK: %[[BUFPTR2:.*]] = bitcast i8* %[[BUFPTR]] to %struct.foo*
+  %buf = alloca %struct.foo, align 1
+
+  ; CHECK: store i8* {{.*}}, i8** %[[AADDR]], align 8
+  store i8* %a, i8** %a.addr, align 8
+
+  ; CHECK: %[[GEP:.*]] = getelementptr inbounds %struct.foo, %struct.foo* %[[BUFPTR2]], i32 0, i32 0, i32 0
+  %gep = getelementptr inbounds %struct.foo, %struct.foo* %buf, i32 0, i32 0, i32 0
+
+  ; CHECK: %[[A:.*]] = load i8*, i8** %[[AADDR]], align 8
+  %a2 = load i8*, i8** %a.addr, align 8
+
+  ; CHECK: call i8* @strcpy(i8* %[[GEP]], i8* %[[A]])
+  %call = call i8* @strcpy(i8* %gep, i8* %a2)
+
+  ; CHECK: store i8* %[[USP]], i8** @__safestack_unsafe_stack_ptr
+  ret void
+}
+
+declare i8* @strcpy(i8*, i8*)
diff --git a/test/Transforms/SampleProfile/Inputs/entry_counts.prof b/test/Transforms/SampleProfile/Inputs/entry_counts.prof
new file mode 100644
index 00000000000..5c2172b5a4d
--- /dev/null
+++ b/test/Transforms/SampleProfile/Inputs/entry_counts.prof
@@ -0,0 +1,3 @@
+empty:100:13293
+0: 0
+1: 100
diff --git a/test/Transforms/SampleProfile/branch.ll b/test/Transforms/SampleProfile/branch.ll
index ea41ee161f0..25bd455a044 100644
--- a/test/Transforms/SampleProfile/branch.ll
+++ b/test/Transforms/SampleProfile/branch.ll
@@ -32,22 +32,22 @@ define i32 @main(i32 %argc, i8** nocapture readonly %argv) #0 {
 ; CHECK: Printing analysis 'Branch Probability Analysis' for function 'main':
 
 entry:
-  tail call void @llvm.dbg.value(metadata i32 %argc, i64 0, metadata !13, metadata !MDExpression()), !dbg !27
-  tail call void @llvm.dbg.value(metadata i8** %argv, i64 0, metadata !14, metadata !MDExpression()), !dbg !27
+  tail call void @llvm.dbg.value(metadata i32 %argc, i64 0, metadata !13, metadata !DIExpression()), !dbg !27
+  tail call void @llvm.dbg.value(metadata i8** %argv, i64 0, metadata !14, metadata !DIExpression()), !dbg !27
   %cmp = icmp slt i32 %argc, 2, !dbg !28
   br i1 %cmp, label %return, label %if.end, !dbg !28
-; CHECK: edge entry -> return probability is 1 / 2 = 50%
-; CHECK: edge entry -> if.end probability is 1 / 2 = 50%
+; CHECK: edge entry -> return probability is 0 / 1 = 0%
+; CHECK: edge entry -> if.end probability is 1 / 1 = 100%
 
 if.end:                                           ; preds = %entry
   %arrayidx = getelementptr inbounds i8*, i8** %argv, i64 1, !dbg !30
   %0 = load i8*, i8** %arrayidx, align 8, !dbg !30, !tbaa !31
   %call = tail call i32 @atoi(i8* %0) #4, !dbg !30
-  tail call void @llvm.dbg.value(metadata i32 %call, i64 0, metadata !17, metadata !MDExpression()), !dbg !30
+  tail call void @llvm.dbg.value(metadata i32 %call, i64 0, metadata !17, metadata !DIExpression()), !dbg !30
   %cmp1 = icmp sgt i32 %call, 100, !dbg !35
   br i1 %cmp1, label %for.body, label %if.end6, !dbg !35
-; CHECK: edge if.end -> for.body probability is 1 / 2 = 50%
-; CHECK: edge if.end -> if.end6 probability is 1 / 2 = 50%
+; CHECK: edge if.end -> for.body probability is 0 / 1 = 0%
+; CHECK: edge if.end -> if.end6 probability is 1 / 1 = 100%
 
 for.body:                                         ; preds = %if.end, %for.body
   %u.016 = phi i32 [ %inc, %for.body ], [ 0, %if.end ]
@@ -55,18 +55,18 @@ for.body:                                         ; preds = %if.end, %for.body
   %add = fadd double %s.015, 3.049000e+00, !dbg !36
   %conv = sitofp i32 %u.016 to double, !dbg !36
   %add4 = fadd double %add, %conv, !dbg !36
-  tail call void @llvm.dbg.value(metadata double %add4, i64 0, metadata !18, metadata !MDExpression()), !dbg !36
+  tail call void @llvm.dbg.value(metadata double %add4, i64 0, metadata !18, metadata !DIExpression()), !dbg !36
   %div = fdiv double 3.940000e+00, %s.015, !dbg !37
   %mul = fmul double %div, 3.200000e-01, !dbg !37
   %add5 = fadd double %add4, %mul, !dbg !37
   %sub = fsub double %add4, %add5, !dbg !37
-  tail call void @llvm.dbg.value(metadata double %sub, i64 0, metadata !18, metadata !MDExpression()), !dbg !37
+  tail call void @llvm.dbg.value(metadata double %sub, i64 0, metadata !18, metadata !DIExpression()), !dbg !37
   %inc = add nsw i32 %u.016, 1, !dbg !38
-  tail call void @llvm.dbg.value(metadata i32 %inc, i64 0, metadata !21, metadata !MDExpression()), !dbg !38
+  tail call void @llvm.dbg.value(metadata i32 %inc, i64 0, metadata !21, metadata !DIExpression()), !dbg !38
   %exitcond = icmp eq i32 %inc, %call, !dbg !38
   br i1 %exitcond, label %if.end6, label %for.body, !dbg !38
-; CHECK: edge for.body -> if.end6 probability is 1 / 10227 = 0.00977804
-; CHECK: edge for.body -> for.body probability is 10226 / 10227 = 99.9902% [HOT edge]
+; CHECK: edge for.body -> if.end6 probability is 0 / 10226 = 0%
+; CHECK: edge for.body -> for.body probability is 10226 / 10226 = 100% [HOT edge]
 
 if.end6:                                          ; preds = %for.body, %if.end
   %result.0 = phi double [ 0.000000e+00, %if.end ], [ %sub, %for.body ]
@@ -98,46 +98,46 @@ attributes #4 = { nounwind readonly }
 !llvm.module.flags = !{!25, !42}
 !llvm.ident = !{!26}
 
-!0 = !MDCompileUnit(language: DW_LANG_C_plus_plus, producer: "clang version 3.4 (trunk 192896) (llvm/trunk 192895)", isOptimized: true, emissionKind: 0, file: !1, enums: !2, retainedTypes: !2, subprograms: !3, globals: !2, imports: !2)
-!1 = !MDFile(filename: "branch.cc", directory: ".")
+!0 = !DICompileUnit(language: DW_LANG_C_plus_plus, producer: "clang version 3.4 (trunk 192896) (llvm/trunk 192895)", isOptimized: true, emissionKind: 0, file: !1, enums: !2, retainedTypes: !2, subprograms: !3, globals: !2, imports: !2)
+!1 = !DIFile(filename: "branch.cc", directory: ".")
 !2 = !{}
 !3 = !{!4}
-!4 = !MDSubprogram(name: "main", line: 4, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: true, scopeLine: 4, file: !1, scope: !5, type: !6, function: i32 (i32, i8**)* @main, variables: !12)
-!5 = !MDFile(filename: "branch.cc", directory: ".")
-!6 = !MDSubroutineType(types: !7)
+!4 = !DISubprogram(name: "main", line: 4, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: true, scopeLine: 4, file: !1, scope: !5, type: !6, function: i32 (i32, i8**)* @main, variables: !12)
+!5 = !DIFile(filename: "branch.cc", directory: ".")
+!6 = !DISubroutineType(types: !7)
 !7 = !{!8, !8, !9}
-!8 = !MDBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
-!9 = !MDDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, baseType: !10)
-!10 = !MDDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, baseType: !11)
-!11 = !MDBasicType(tag: DW_TAG_base_type, name: "char", size: 8, align: 8, encoding: DW_ATE_signed_char)
+!8 = !DIBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
+!9 = !DIDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, baseType: !10)
+!10 = !DIDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, baseType: !11)
+!11 = !DIBasicType(tag: DW_TAG_base_type, name: "char", size: 8, align: 8, encoding: DW_ATE_signed_char)
 !12 = !{!13, !14, !15, !17, !18, !21, !23}
-!13 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "argc", line: 4, arg: 1, scope: !4, file: !5, type: !8)
-!14 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "argv", line: 4, arg: 2, scope: !4, file: !5, type: !9)
-!15 = !MDLocalVariable(tag: DW_TAG_auto_variable, name: "result", line: 7, scope: !4, file: !5, type: !16)
-!16 = !MDBasicType(tag: DW_TAG_base_type, name: "double", size: 64, align: 64, encoding: DW_ATE_float)
-!17 = !MDLocalVariable(tag: DW_TAG_auto_variable, name: "limit", line: 8, scope: !4, file: !5, type: !8)
-!18 = !MDLocalVariable(tag: DW_TAG_auto_variable, name: "s", line: 10, scope: !19, file: !5, type: !16)
-!19 = distinct !MDLexicalBlock(line: 9, column: 0, file: !1, scope: !20)
-!20 = distinct !MDLexicalBlock(line: 9, column: 0, file: !1, scope: !4)
-!21 = !MDLocalVariable(tag: DW_TAG_auto_variable, name: "u", line: 11, scope: !22, file: !5, type: !8)
-!22 = distinct !MDLexicalBlock(line: 11, column: 0, file: !1, scope: !19)
-!23 = !MDLocalVariable(tag: DW_TAG_auto_variable, name: "x", line: 12, scope: !24, file: !5, type: !16)
-!24 = distinct !MDLexicalBlock(line: 11, column: 0, file: !1, scope: !22)
+!13 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "argc", line: 4, arg: 1, scope: !4, file: !5, type: !8)
+!14 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "argv", line: 4, arg: 2, scope: !4, file: !5, type: !9)
+!15 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "result", line: 7, scope: !4, file: !5, type: !16)
+!16 = !DIBasicType(tag: DW_TAG_base_type, name: "double", size: 64, align: 64, encoding: DW_ATE_float)
+!17 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "limit", line: 8, scope: !4, file: !5, type: !8)
+!18 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "s", line: 10, scope: !19, file: !5, type: !16)
+!19 = distinct !DILexicalBlock(line: 9, column: 0, file: !1, scope: !20)
+!20 = distinct !DILexicalBlock(line: 9, column: 0, file: !1, scope: !4)
+!21 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "u", line: 11, scope: !22, file: !5, type: !8)
+!22 = distinct !DILexicalBlock(line: 11, column: 0, file: !1, scope: !19)
+!23 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "x", line: 12, scope: !24, file: !5, type: !16)
+!24 = distinct !DILexicalBlock(line: 11, column: 0, file: !1, scope: !22)
 !25 = !{i32 2, !"Dwarf Version", i32 4}
 !26 = !{!"clang version 3.4 (trunk 192896) (llvm/trunk 192895)"}
-!27 = !MDLocation(line: 4, scope: !4)
-!28 = !MDLocation(line: 5, scope: !29)
-!29 = distinct !MDLexicalBlock(line: 5, column: 0, file: !1, scope: !4)
-!30 = !MDLocation(line: 8, scope: !4)
+!27 = !DILocation(line: 4, scope: !4)
+!28 = !DILocation(line: 5, scope: !29)
+!29 = distinct !DILexicalBlock(line: 5, column: 0, file: !1, scope: !4)
+!30 = !DILocation(line: 8, scope: !4)
 !31 = !{!32, !32, i64 0}
 !32 = !{!"any pointer", !33, i64 0}
 !33 = !{!"omnipotent char", !34, i64 0}
 !34 = !{!"Simple C/C++ TBAA"}
-!35 = !MDLocation(line: 9, scope: !20)
-!36 = !MDLocation(line: 13, scope: !24)
-!37 = !MDLocation(line: 14, scope: !24)
-!38 = !MDLocation(line: 11, scope: !22)
-!39 = !MDLocation(line: 20, scope: !4)
-!40 = !MDLocation(line: 21, scope: !4)
-!41 = !MDLocation(line: 22, scope: !4)
+!35 = !DILocation(line: 9, scope: !20)
+!36 = !DILocation(line: 13, scope: !24)
+!37 = !DILocation(line: 14, scope: !24)
+!38 = !DILocation(line: 11, scope: !22)
+!39 = !DILocation(line: 20, scope: !4)
+!40 = !DILocation(line: 21, scope: !4)
+!41 = !DILocation(line: 22, scope: !4)
 !42 = !{i32 1, !"Debug Info Version", i32 3}
diff --git a/test/Transforms/SampleProfile/calls.ll b/test/Transforms/SampleProfile/calls.ll
index a484995e39c..10f43a1a612 100644
--- a/test/Transforms/SampleProfile/calls.ll
+++ b/test/Transforms/SampleProfile/calls.ll
@@ -52,8 +52,8 @@ while.cond:                                       ; preds = %if.end, %entry
   store i32 %inc, i32* %i, align 4, !dbg !14
   %cmp = icmp slt i32 %0, 400000000, !dbg !14
   br i1 %cmp, label %while.body, label %while.end, !dbg !14
-; CHECK: edge while.cond -> while.body probability is 5391 / 5392 = 99.9815% [HOT edge]
-; CHECK: edge while.cond -> while.end probability is 1 / 5392 = 0.018546%
+; CHECK: edge while.cond -> while.body probability is 5391 / 5391 = 100% [HOT edge]
+; CHECK: edge while.cond -> while.end probability is 0 / 5391 = 0%
 
 while.body:                                       ; preds = %while.cond
   %1 = load i32, i32* %i, align 4, !dbg !16
@@ -63,8 +63,8 @@ while.body:                                       ; preds = %while.cond
 ; both branches out of while.body had the same weight. In reality,
 ; the edge while.body->if.then is taken most of the time.
 ;
-; CHECK: edge while.body -> if.then probability is 5752 / 5753 = 99.9826% [HOT edge]
-; CHECK: edge while.body -> if.else probability is 1 / 5753 = 0.0173822%
+; CHECK: edge while.body -> if.then probability is 5752 / 5752 = 100% [HOT edge]
+; CHECK: edge while.body -> if.else probability is 0 / 5752 = 0%
 
 
 if.then:                                          ; preds = %while.body
@@ -92,29 +92,29 @@ declare i32 @printf(i8*, ...) #2
 !llvm.module.flags = !{!8, !9}
 !llvm.ident = !{!10}
 
-!0 = !MDCompileUnit(language: DW_LANG_C_plus_plus, producer: "clang version 3.5 ", isOptimized: false, emissionKind: 0, file: !1, enums: !2, retainedTypes: !2, subprograms: !3, globals: !2, imports: !2)
-!1 = !MDFile(filename: "calls.cc", directory: ".")
+!0 = !DICompileUnit(language: DW_LANG_C_plus_plus, producer: "clang version 3.5 ", isOptimized: false, emissionKind: 0, file: !1, enums: !2, retainedTypes: !2, subprograms: !3, globals: !2, imports: !2)
+!1 = !DIFile(filename: "calls.cc", directory: ".")
 !2 = !{}
 !3 = !{!4, !7}
-!4 = !MDSubprogram(name: "sum", line: 3, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 3, file: !1, scope: !5, type: !6, function: i32 (i32, i32)* @_Z3sumii, variables: !2)
-!5 = !MDFile(filename: "calls.cc", directory: ".")
-!6 = !MDSubroutineType(types: !2)
-!7 = !MDSubprogram(name: "main", line: 7, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 7, file: !1, scope: !5, type: !6, function: i32 ()* @main, variables: !2)
+!4 = !DISubprogram(name: "sum", line: 3, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 3, file: !1, scope: !5, type: !6, function: i32 (i32, i32)* @_Z3sumii, variables: !2)
+!5 = !DIFile(filename: "calls.cc", directory: ".")
+!6 = !DISubroutineType(types: !2)
+!7 = !DISubprogram(name: "main", line: 7, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 7, file: !1, scope: !5, type: !6, function: i32 ()* @main, variables: !2)
 !8 = !{i32 2, !"Dwarf Version", i32 4}
 !9 = !{i32 1, !"Debug Info Version", i32 3}
 !10 = !{!"clang version 3.5 "}
-!11 = !MDLocation(line: 4, scope: !4)
-!12 = !MDLocation(line: 8, scope: !7)
-!13 = !MDLocation(line: 9, scope: !7)
-!14 = !MDLocation(line: 9, scope: !15)
-!15 = !MDLexicalBlockFile(discriminator: 1, file: !1, scope: !7)
-!16 = !MDLocation(line: 10, scope: !17)
-!17 = distinct !MDLexicalBlock(line: 10, column: 0, file: !1, scope: !7)
-!18 = !MDLocation(line: 10, scope: !19)
-!19 = !MDLexicalBlockFile(discriminator: 1, file: !1, scope: !17)
-!20 = !MDLocation(line: 10, scope: !21)
-!21 = !MDLexicalBlockFile(discriminator: 2, file: !1, scope: !17)
-!22 = !MDLocation(line: 10, scope: !23)
-!23 = !MDLexicalBlockFile(discriminator: 3, file: !1, scope: !17)
-!24 = !MDLocation(line: 11, scope: !7)
-!25 = !MDLocation(line: 12, scope: !7)
+!11 = !DILocation(line: 4, scope: !4)
+!12 = !DILocation(line: 8, scope: !7)
+!13 = !DILocation(line: 9, scope: !7)
+!14 = !DILocation(line: 9, scope: !15)
+!15 = !DILexicalBlockFile(discriminator: 1, file: !1, scope: !7)
+!16 = !DILocation(line: 10, scope: !17)
+!17 = distinct !DILexicalBlock(line: 10, column: 0, file: !1, scope: !7)
+!18 = !DILocation(line: 10, scope: !19)
+!19 = !DILexicalBlockFile(discriminator: 1, file: !1, scope: !17)
+!20 = !DILocation(line: 10, scope: !21)
+!21 = !DILexicalBlockFile(discriminator: 2, file: !1, scope: !17)
+!22 = !DILocation(line: 10, scope: !23)
+!23 = !DILexicalBlockFile(discriminator: 3, file: !1, scope: !17)
+!24 = !DILocation(line: 11, scope: !7)
+!25 = !DILocation(line: 12, scope: !7)
diff --git a/test/Transforms/SampleProfile/discriminator.ll b/test/Transforms/SampleProfile/discriminator.ll
index 1f3a12fc2ba..2445c5c5d60 100644
--- a/test/Transforms/SampleProfile/discriminator.ll
+++ b/test/Transforms/SampleProfile/discriminator.ll
@@ -66,25 +66,25 @@ while.end:                                        ; preds = %while.cond
 !llvm.module.flags = !{!7, !8}
 !llvm.ident = !{!9}
 
-!0 = !MDCompileUnit(language: DW_LANG_C99, producer: "clang version 3.5 ", isOptimized: false, emissionKind: 0, file: !1, enums: !2, retainedTypes: !2, subprograms: !3, globals: !2, imports: !2)
-!1 = !MDFile(filename: "discriminator.c", directory: ".")
+!0 = !DICompileUnit(language: DW_LANG_C99, producer: "clang version 3.5 ", isOptimized: false, emissionKind: 0, file: !1, enums: !2, retainedTypes: !2, subprograms: !3, globals: !2, imports: !2)
+!1 = !DIFile(filename: "discriminator.c", directory: ".")
 !2 = !{}
 !3 = !{!4}
-!4 = !MDSubprogram(name: "foo", line: 1, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 1, file: !1, scope: !5, type: !6, function: i32 (i32)* @foo, variables: !2)
-!5 = !MDFile(filename: "discriminator.c", directory: ".")
-!6 = !MDSubroutineType(types: !2)
+!4 = !DISubprogram(name: "foo", line: 1, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 1, file: !1, scope: !5, type: !6, function: i32 (i32)* @foo, variables: !2)
+!5 = !DIFile(filename: "discriminator.c", directory: ".")
+!6 = !DISubroutineType(types: !2)
 !7 = !{i32 2, !"Dwarf Version", i32 4}
 !8 = !{i32 1, !"Debug Info Version", i32 3}
 !9 = !{!"clang version 3.5 "}
-!10 = !MDLocation(line: 2, scope: !4)
-!11 = !MDLocation(line: 3, scope: !4)
-!12 = !MDLocation(line: 3, scope: !13)
-!13 = !MDLexicalBlockFile(discriminator: 1, file: !1, scope: !4)
-!14 = !MDLocation(line: 4, scope: !15)
-!15 = distinct !MDLexicalBlock(line: 4, column: 0, file: !1, scope: !16)
-!16 = distinct !MDLexicalBlock(line: 3, column: 0, file: !1, scope: !4)
-!17 = !MDLocation(line: 4, scope: !18)
-!18 = !MDLexicalBlockFile(discriminator: 1, file: !1, scope: !15)
-!19 = !MDLocation(line: 5, scope: !16)
-!20 = !MDLocation(line: 6, scope: !16)
-!21 = !MDLocation(line: 7, scope: !4)
+!10 = !DILocation(line: 2, scope: !4)
+!11 = !DILocation(line: 3, scope: !4)
+!12 = !DILocation(line: 3, scope: !13)
+!13 = !DILexicalBlockFile(discriminator: 1, file: !1, scope: !4)
+!14 = !DILocation(line: 4, scope: !15)
+!15 = distinct !DILexicalBlock(line: 4, column: 0, file: !1, scope: !16)
+!16 = distinct !DILexicalBlock(line: 3, column: 0, file: !1, scope: !4)
+!17 = !DILocation(line: 4, scope: !18)
+!18 = !DILexicalBlockFile(discriminator: 1, file: !1, scope: !15)
+!19 = !DILocation(line: 5, scope: !16)
+!20 = !DILocation(line: 6, scope: !16)
+!21 = !DILocation(line: 7, scope: !4)
diff --git a/test/Transforms/SampleProfile/entry_counts.ll b/test/Transforms/SampleProfile/entry_counts.ll
new file mode 100644
index 00000000000..bf66e693a9d
--- /dev/null
+++ b/test/Transforms/SampleProfile/entry_counts.ll
@@ -0,0 +1,24 @@
+; RUN: opt < %s -sample-profile -sample-profile-file=%S/Inputs/entry_counts.prof -S | FileCheck %s
+
+; According to the profile, function empty() was called 13,293 times.
+; CHECK: {{.*}} = !{!"function_entry_count", i64 13293}
+
+define void @empty() {
+entry:
+  ret void, !dbg !9
+}
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!6, !7}
+!llvm.ident = !{!8}
+
+!0 = !DICompileUnit(language: DW_LANG_C99, file: !1, producer: "clang version 3.7.0 (trunk 237249) (llvm/trunk 237261)", isOptimized: false, runtimeVersion: 0, emissionKind: 2, enums: !2, retainedTypes: !2, subprograms: !3, globals: !2, imports: !2)
+!1 = !DIFile(filename: "entry_counts.c", directory: "/usr/local/google/home/dnovillo/llvm/test/pgo")
+!2 = !{}
+!3 = !{!4}
+!4 = !DISubprogram(name: "empty", scope: !1, file: !1, line: 1, type: !5, isLocal: false, isDefinition: true, scopeLine: 1, isOptimized: false, function: void ()* @empty, variables: !2)
+!5 = !DISubroutineType(types: !2)
+!6 = !{i32 2, !"Dwarf Version", i32 4}
+!7 = !{i32 2, !"Debug Info Version", i32 3}
+!8 = !{!"clang version 3.7.0 (trunk 237249) (llvm/trunk 237261)"}
+!9 = !DILocation(line: 1, column: 15, scope: !4)
diff --git a/test/Transforms/SampleProfile/fnptr.ll b/test/Transforms/SampleProfile/fnptr.ll
index 9b6653e60dd..368da42fc8a 100644
--- a/test/Transforms/SampleProfile/fnptr.ll
+++ b/test/Transforms/SampleProfile/fnptr.ll
@@ -129,27 +129,27 @@ declare i32 @printf(i8* nocapture readonly, ...) #1
 
 !0 = !{i32 2, !"Debug Info Version", i32 3}
 !1 = !{!"clang version 3.6.0 "}
-!2 = !MDLocation(line: 9, column: 3, scope: !3)
-!3 = !MDSubprogram(name: "foo", line: 8, isLocal: false, isDefinition: true, flags: DIFlagPrototyped, isOptimized: true, scopeLine: 8, file: !4, scope: !5, type: !6, function: double (i32)* @_Z3fooi, variables: !7)
-!4 = !MDFile(filename: "fnptr.cc", directory: ".")
-!5 = !MDFile(filename: "fnptr.cc", directory: ".")
-!6 = !MDSubroutineType(types: !7)
+!2 = !DILocation(line: 9, column: 3, scope: !3)
+!3 = !DISubprogram(name: "foo", line: 8, isLocal: false, isDefinition: true, flags: DIFlagPrototyped, isOptimized: true, scopeLine: 8, file: !4, scope: !5, type: !6, function: double (i32)* @_Z3fooi, variables: !7)
+!4 = !DIFile(filename: "fnptr.cc", directory: ".")
+!5 = !DIFile(filename: "fnptr.cc", directory: ".")
+!6 = !DISubroutineType(types: !7)
 !7 = !{}
-!8 = !MDLocation(line: 9, column: 14, scope: !3)
-!9 = !MDLocation(line: 13, column: 3, scope: !10)
-!10 = !MDSubprogram(name: "bar", line: 12, isLocal: false, isDefinition: true, flags: DIFlagPrototyped, isOptimized: true, scopeLine: 12, file: !4, scope: !5, type: !6, function: double (i32)* @_Z3bari, variables: !7)
-!11 = !MDLocation(line: 13, column: 14, scope: !10)
-!12 = !MDLocation(line: 19, column: 3, scope: !13)
-!13 = !MDSubprogram(name: "main", line: 16, isLocal: false, isDefinition: true, flags: DIFlagPrototyped, isOptimized: true, scopeLine: 16, file: !4, scope: !5, type: !6, function: i32 ()* @main, variables: !7)
-!14 = !MDLocation(line: 20, column: 5, scope: !13)
-!15 = !MDLocation(line: 21, column: 15, scope: !13)
-!16 = !MDLocation(line: 22, column: 11, scope: !13)
+!8 = !DILocation(line: 9, column: 14, scope: !3)
+!9 = !DILocation(line: 13, column: 3, scope: !10)
+!10 = !DISubprogram(name: "bar", line: 12, isLocal: false, isDefinition: true, flags: DIFlagPrototyped, isOptimized: true, scopeLine: 12, file: !4, scope: !5, type: !6, function: double (i32)* @_Z3bari, variables: !7)
+!11 = !DILocation(line: 13, column: 14, scope: !10)
+!12 = !DILocation(line: 19, column: 3, scope: !13)
+!13 = !DISubprogram(name: "main", line: 16, isLocal: false, isDefinition: true, flags: DIFlagPrototyped, isOptimized: true, scopeLine: 16, file: !4, scope: !5, type: !6, function: i32 ()* @main, variables: !7)
+!14 = !DILocation(line: 20, column: 5, scope: !13)
+!15 = !DILocation(line: 21, column: 15, scope: !13)
+!16 = !DILocation(line: 22, column: 11, scope: !13)
 !17 = !{!"branch_weights", i32 534, i32 2064}
-!18 = !MDLocation(line: 23, column: 14, scope: !13)
-!19 = !MDLocation(line: 25, column: 14, scope: !13)
-!20 = !MDLocation(line: 20, column: 28, scope: !13)
+!18 = !DILocation(line: 23, column: 14, scope: !13)
+!19 = !DILocation(line: 25, column: 14, scope: !13)
+!20 = !DILocation(line: 20, column: 28, scope: !13)
 !21 = !{!"branch_weights", i32 0, i32 1075}
-!22 = !MDLocation(line: 19, column: 26, scope: !13)
+!22 = !DILocation(line: 19, column: 26, scope: !13)
 !23 = !{!"branch_weights", i32 0, i32 534}
-!24 = !MDLocation(line: 27, column: 3, scope: !13)
-!25 = !MDLocation(line: 28, column: 3, scope: !13)
+!24 = !DILocation(line: 27, column: 3, scope: !13)
+!25 = !DILocation(line: 28, column: 3, scope: !13)
diff --git a/test/Transforms/SampleProfile/propagate.ll b/test/Transforms/SampleProfile/propagate.ll
index 523122a11e5..620d125875f 100644
--- a/test/Transforms/SampleProfile/propagate.ll
+++ b/test/Transforms/SampleProfile/propagate.ll
@@ -73,8 +73,8 @@ for.cond:                                         ; preds = %for.inc16, %if.else
   %5 = load i64, i64* %N.addr, align 8, !dbg !15
   %cmp1 = icmp slt i64 %4, %5, !dbg !15
   br i1 %cmp1, label %for.body, label %for.end18, !dbg !15
-; CHECK: edge for.cond -> for.body probability is 10 / 11 = 90.9091% [HOT edge]
-; CHECK: edge for.cond -> for.end18 probability is 1 / 11 = 9.09091%
+; CHECK: edge for.cond -> for.body probability is 10 / 10 = 100% [HOT edge]
+; CHECK: edge for.cond -> for.end18 probability is 0 / 10 = 0%
 
 for.body:                                         ; preds = %for.cond
   %6 = load i64, i64* %i, align 8, !dbg !18
@@ -119,8 +119,8 @@ for.cond8:                                        ; preds = %for.inc, %if.else7
   %14 = load i64, i64* %i, align 8, !dbg !28
   %cmp10 = icmp slt i64 %conv9, %14, !dbg !28
   br i1 %cmp10, label %for.body11, label %for.end, !dbg !28
-; CHECK: edge for.cond8 -> for.body11 probability is 16191 / 16192 = 99.9938% [HOT edge]
-; CHECK: edge for.cond8 -> for.end probability is 1 / 16192 = 0.00617589%
+; CHECK: edge for.cond8 -> for.body11 probability is 16191 / 16191 = 100% [HOT edge]
+; CHECK: edge for.cond8 -> for.end probability is 0 / 16191 = 0%
 
 for.body11:                                       ; preds = %for.cond8
   %15 = load i32, i32* %j, align 4, !dbg !31
@@ -198,46 +198,46 @@ attributes #2 = { "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "n
 !llvm.module.flags = !{!8, !9}
 !llvm.ident = !{!10}
 
-!0 = !MDCompileUnit(language: DW_LANG_C_plus_plus, producer: "clang version 3.5 ", isOptimized: false, emissionKind: 0, file: !1, enums: !2, retainedTypes: !2, subprograms: !3, globals: !2, imports: !2)
-!1 = !MDFile(filename: "propagate.cc", directory: ".")
+!0 = !DICompileUnit(language: DW_LANG_C_plus_plus, producer: "clang version 3.5 ", isOptimized: false, emissionKind: 0, file: !1, enums: !2, retainedTypes: !2, subprograms: !3, globals: !2, imports: !2)
+!1 = !DIFile(filename: "propagate.cc", directory: ".")
 !2 = !{}
 !3 = !{!4, !7}
-!4 = !MDSubprogram(name: "foo", line: 3, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 3, file: !1, scope: !5, type: !6, function: i64 (i32, i32, i64)* @_Z3fooiil, variables: !2)
-!5 = !MDFile(filename: "propagate.cc", directory: ".")
-!6 = !MDSubroutineType(types: !{null})
-!7 = !MDSubprogram(name: "main", line: 24, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 24, file: !1, scope: !5, type: !6, function: i32 ()* @main, variables: !2)
+!4 = !DISubprogram(name: "foo", line: 3, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 3, file: !1, scope: !5, type: !6, function: i64 (i32, i32, i64)* @_Z3fooiil, variables: !2)
+!5 = !DIFile(filename: "propagate.cc", directory: ".")
+!6 = !DISubroutineType(types: !{null})
+!7 = !DISubprogram(name: "main", line: 24, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 24, file: !1, scope: !5, type: !6, function: i32 ()* @main, variables: !2)
 !8 = !{i32 2, !"Dwarf Version", i32 4}
 !9 = !{i32 1, !"Debug Info Version", i32 3}
 !10 = !{!"clang version 3.5 "}
-!11 = !MDLocation(line: 4, scope: !12)
-!12 = distinct !MDLexicalBlock(line: 4, column: 0, file: !1, scope: !4)
-!13 = !MDLocation(line: 5, scope: !14)
-!14 = distinct !MDLexicalBlock(line: 4, column: 0, file: !1, scope: !12)
-!15 = !MDLocation(line: 7, scope: !16)
-!16 = distinct !MDLexicalBlock(line: 7, column: 0, file: !1, scope: !17)
-!17 = distinct !MDLexicalBlock(line: 6, column: 0, file: !1, scope: !12)
-!18 = !MDLocation(line: 8, scope: !19)
-!19 = distinct !MDLexicalBlock(line: 8, column: 0, file: !1, scope: !20)
-!20 = distinct !MDLexicalBlock(line: 7, column: 0, file: !1, scope: !16)
-!21 = !MDLocation(line: 9, scope: !19)
-!22 = !MDLocation(line: 10, scope: !23)
-!23 = distinct !MDLexicalBlock(line: 10, column: 0, file: !1, scope: !20)
-!24 = !MDLocation(line: 11, scope: !25)
-!25 = distinct !MDLexicalBlock(line: 10, column: 0, file: !1, scope: !23)
-!26 = !MDLocation(line: 12, scope: !25)
-!27 = !MDLocation(line: 13, scope: !25)
-!28 = !MDLocation(line: 14, scope: !29)
-!29 = distinct !MDLexicalBlock(line: 14, column: 0, file: !1, scope: !30)
-!30 = distinct !MDLexicalBlock(line: 13, column: 0, file: !1, scope: !23)
-!31 = !MDLocation(line: 15, scope: !32)
-!32 = distinct !MDLexicalBlock(line: 14, column: 0, file: !1, scope: !29)
-!33 = !MDLocation(line: 16, scope: !32)
-!34 = !MDLocation(line: 17, scope: !32)
-!35 = !MDLocation(line: 19, scope: !20)
-!36 = !MDLocation(line: 21, scope: !4)
-!37 = !MDLocation(line: 22, scope: !4)
-!38 = !MDLocation(line: 25, scope: !7)
-!39 = !MDLocation(line: 26, scope: !7)
-!40 = !MDLocation(line: 27, scope: !7)
-!41 = !MDLocation(line: 28, scope: !7)
-!42 = !MDLocation(line: 29, scope: !7)
+!11 = !DILocation(line: 4, scope: !12)
+!12 = distinct !DILexicalBlock(line: 4, column: 0, file: !1, scope: !4)
+!13 = !DILocation(line: 5, scope: !14)
+!14 = distinct !DILexicalBlock(line: 4, column: 0, file: !1, scope: !12)
+!15 = !DILocation(line: 7, scope: !16)
+!16 = distinct !DILexicalBlock(line: 7, column: 0, file: !1, scope: !17)
+!17 = distinct !DILexicalBlock(line: 6, column: 0, file: !1, scope: !12)
+!18 = !DILocation(line: 8, scope: !19)
+!19 = distinct !DILexicalBlock(line: 8, column: 0, file: !1, scope: !20)
+!20 = distinct !DILexicalBlock(line: 7, column: 0, file: !1, scope: !16)
+!21 = !DILocation(line: 9, scope: !19)
+!22 = !DILocation(line: 10, scope: !23)
+!23 = distinct !DILexicalBlock(line: 10, column: 0, file: !1, scope: !20)
+!24 = !DILocation(line: 11, scope: !25)
+!25 = distinct !DILexicalBlock(line: 10, column: 0, file: !1, scope: !23)
+!26 = !DILocation(line: 12, scope: !25)
+!27 = !DILocation(line: 13, scope: !25)
+!28 = !DILocation(line: 14, scope: !29)
+!29 = distinct !DILexicalBlock(line: 14, column: 0, file: !1, scope: !30)
+!30 = distinct !DILexicalBlock(line: 13, column: 0, file: !1, scope: !23)
+!31 = !DILocation(line: 15, scope: !32)
+!32 = distinct !DILexicalBlock(line: 14, column: 0, file: !1, scope: !29)
+!33 = !DILocation(line: 16, scope: !32)
+!34 = !DILocation(line: 17, scope: !32)
+!35 = !DILocation(line: 19, scope: !20)
+!36 = !DILocation(line: 21, scope: !4)
+!37 = !DILocation(line: 22, scope: !4)
+!38 = !DILocation(line: 25, scope: !7)
+!39 = !DILocation(line: 26, scope: !7)
+!40 = !DILocation(line: 27, scope: !7)
+!41 = !DILocation(line: 28, scope: !7)
+!42 = !DILocation(line: 29, scope: !7)
diff --git a/test/Transforms/ScalarRepl/2011-09-22-PHISpeculateInvoke.ll b/test/Transforms/ScalarRepl/2011-09-22-PHISpeculateInvoke.ll
index 9a24662f411..bff6566d178 100644
--- a/test/Transforms/ScalarRepl/2011-09-22-PHISpeculateInvoke.ll
+++ b/test/Transforms/ScalarRepl/2011-09-22-PHISpeculateInvoke.ll
@@ -10,7 +10,7 @@ declare void @extern_fn(i32*)
 declare i32 @extern_fn2(i32)
 declare i32 @__gcc_personality_v0(i32, i64, i8*, i8*)
 
-define void @odd_fn(i1) noinline {
+define void @odd_fn(i1) noinline personality i32 (i32, i64, i8*, i8*)* @__gcc_personality_v0 {
   %retptr1 = alloca i32
   %retptr2 = alloca i32
   br i1 %0, label %then, label %else
@@ -30,7 +30,7 @@ join:                                             ; preds = %then, %else
   ret void
 
 unwind:                                           ; preds = %then
-  %info = landingpad { i8*, i32 } personality i32 (i32, i64, i8*, i8*)* @__gcc_personality_v0
+  %info = landingpad { i8*, i32 }
           cleanup
   call void @extern_fn(i32* null)
   unreachable
diff --git a/test/Transforms/ScalarRepl/debuginfo-preserved.ll b/test/Transforms/ScalarRepl/debuginfo-preserved.ll
index 10bad7d664e..8e5d4ff773d 100644
--- a/test/Transforms/ScalarRepl/debuginfo-preserved.ll
+++ b/test/Transforms/ScalarRepl/debuginfo-preserved.ll
@@ -17,10 +17,10 @@ entry:
   %b.addr = alloca i32, align 4
   %c = alloca i32, align 4
   store i32 %a, i32* %a.addr, align 4
-  call void @llvm.dbg.declare(metadata i32* %a.addr, metadata !6, metadata !MDExpression()), !dbg !7
+  call void @llvm.dbg.declare(metadata i32* %a.addr, metadata !6, metadata !DIExpression()), !dbg !7
   store i32 %b, i32* %b.addr, align 4
-  call void @llvm.dbg.declare(metadata i32* %b.addr, metadata !8, metadata !MDExpression()), !dbg !9
-  call void @llvm.dbg.declare(metadata i32* %c, metadata !10, metadata !MDExpression()), !dbg !12
+  call void @llvm.dbg.declare(metadata i32* %b.addr, metadata !8, metadata !DIExpression()), !dbg !9
+  call void @llvm.dbg.declare(metadata i32* %c, metadata !10, metadata !DIExpression()), !dbg !12
   %tmp = load i32, i32* %a.addr, align 4, !dbg !13
   store i32 %tmp, i32* %c, align 4, !dbg !13
   %tmp1 = load i32, i32* %a.addr, align 4, !dbg !14
@@ -42,24 +42,24 @@ declare void @llvm.dbg.declare(metadata, metadata, metadata) nounwind readnone
 !llvm.dbg.cu = !{!0}
 !llvm.module.flags = !{!20}
 
-!0 = !MDCompileUnit(language: DW_LANG_C99, producer: "clang version 3.0 (trunk 131941)", isOptimized: false, emissionKind: 0, file: !18, enums: !19, retainedTypes: !19, subprograms: !17)
-!1 = !MDSubprogram(name: "f", line: 1, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 1, file: !18, scope: !2, type: !3, function: i32 (i32, i32)* @f)
-!2 = !MDFile(filename: "/d/j/debug-test.c", directory: "/Volumes/Data/b")
-!3 = !MDSubroutineType(types: !4)
+!0 = !DICompileUnit(language: DW_LANG_C99, producer: "clang version 3.0 (trunk 131941)", isOptimized: false, emissionKind: 0, file: !18, enums: !19, retainedTypes: !19, subprograms: !17)
+!1 = !DISubprogram(name: "f", line: 1, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 1, file: !18, scope: !2, type: !3, function: i32 (i32, i32)* @f)
+!2 = !DIFile(filename: "/d/j/debug-test.c", directory: "/Volumes/Data/b")
+!3 = !DISubroutineType(types: !4)
 !4 = !{!5}
-!5 = !MDBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
-!6 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "a", line: 1, arg: 1, scope: !1, file: !2, type: !5)
-!7 = !MDLocation(line: 1, column: 11, scope: !1)
-!8 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "b", line: 1, arg: 2, scope: !1, file: !2, type: !5)
-!9 = !MDLocation(line: 1, column: 18, scope: !1)
-!10 = !MDLocalVariable(tag: DW_TAG_auto_variable, name: "c", line: 2, scope: !11, file: !2, type: !5)
-!11 = distinct !MDLexicalBlock(line: 1, column: 21, file: !18, scope: !1)
-!12 = !MDLocation(line: 2, column: 9, scope: !11)
-!13 = !MDLocation(line: 2, column: 14, scope: !11)
-!14 = !MDLocation(line: 3, column: 5, scope: !11)
-!15 = !MDLocation(line: 4, column: 5, scope: !11)
-!16 = !MDLocation(line: 5, column: 5, scope: !11)
+!5 = !DIBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
+!6 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "a", line: 1, arg: 1, scope: !1, file: !2, type: !5)
+!7 = !DILocation(line: 1, column: 11, scope: !1)
+!8 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "b", line: 1, arg: 2, scope: !1, file: !2, type: !5)
+!9 = !DILocation(line: 1, column: 18, scope: !1)
+!10 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "c", line: 2, scope: !11, file: !2, type: !5)
+!11 = distinct !DILexicalBlock(line: 1, column: 21, file: !18, scope: !1)
+!12 = !DILocation(line: 2, column: 9, scope: !11)
+!13 = !DILocation(line: 2, column: 14, scope: !11)
+!14 = !DILocation(line: 3, column: 5, scope: !11)
+!15 = !DILocation(line: 4, column: 5, scope: !11)
+!16 = !DILocation(line: 5, column: 5, scope: !11)
 !17 = !{!1}
-!18 = !MDFile(filename: "/d/j/debug-test.c", directory: "/Volumes/Data/b")
+!18 = !DIFile(filename: "/d/j/debug-test.c", directory: "/Volumes/Data/b")
 !19 = !{}
 !20 = !{i32 1, !"Debug Info Version", i32 3}
diff --git a/test/Transforms/Scalarizer/dbginfo.ll b/test/Transforms/Scalarizer/dbginfo.ll
index 37a810f9114..3770a3e8c64 100644
--- a/test/Transforms/Scalarizer/dbginfo.ll
+++ b/test/Transforms/Scalarizer/dbginfo.ll
@@ -37,9 +37,9 @@ define void @f1(<4 x i32>* nocapture %a, <4 x i32>* nocapture readonly %b, <4 x
 ; CHECK: store i32 %add.i3, i32* %a.i3, align 4, !dbg ![[TAG1]], !tbaa ![[TAG2]]
 ; CHECK: ret void
 entry:
-  tail call void @llvm.dbg.value(metadata <4 x i32>* %a, i64 0, metadata !15, metadata !MDExpression()), !dbg !20
-  tail call void @llvm.dbg.value(metadata <4 x i32>* %b, i64 0, metadata !16, metadata !MDExpression()), !dbg !20
-  tail call void @llvm.dbg.value(metadata <4 x i32>* %c, i64 0, metadata !17, metadata !MDExpression()), !dbg !20
+  tail call void @llvm.dbg.value(metadata <4 x i32>* %a, i64 0, metadata !15, metadata !DIExpression()), !dbg !20
+  tail call void @llvm.dbg.value(metadata <4 x i32>* %b, i64 0, metadata !16, metadata !DIExpression()), !dbg !20
+  tail call void @llvm.dbg.value(metadata <4 x i32>* %c, i64 0, metadata !17, metadata !DIExpression()), !dbg !20
   %bval = load <4 x i32>, <4 x i32>* %b, align 16, !dbg !21, !tbaa !22
   %cval = load <4 x i32>, <4 x i32>* %c, align 16, !dbg !21, !tbaa !22
   %add = add <4 x i32> %bval, %cval, !dbg !21
@@ -57,30 +57,30 @@ attributes #1 = { nounwind readnone }
 !llvm.module.flags = !{!18, !26}
 !llvm.ident = !{!19}
 
-!0 = !MDCompileUnit(language: DW_LANG_C99, producer: "clang version 3.4 (trunk 194134) (llvm/trunk 194126)", isOptimized: true, emissionKind: 0, file: !1, enums: !2, retainedTypes: !2, subprograms: !3, globals: !2, imports: !2)
-!1 = !MDFile(filename: "/tmp/add.c", directory: "/home/richards/llvm/build")
+!0 = !DICompileUnit(language: DW_LANG_C99, producer: "clang version 3.4 (trunk 194134) (llvm/trunk 194126)", isOptimized: true, emissionKind: 0, file: !1, enums: !2, retainedTypes: !2, subprograms: !3, globals: !2, imports: !2)
+!1 = !DIFile(filename: "/tmp/add.c", directory: "/home/richards/llvm/build")
 !2 = !{}
 !3 = !{!4}
-!4 = !MDSubprogram(name: "f1", line: 3, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: true, scopeLine: 4, file: !1, scope: !5, type: !6, function: void (<4 x i32>*, <4 x i32>*, <4 x i32>*)* @f1, variables: !14)
-!5 = !MDFile(filename: "/tmp/add.c", directory: "/home/richards/llvm/build")
-!6 = !MDSubroutineType(types: !7)
+!4 = !DISubprogram(name: "f1", line: 3, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: true, scopeLine: 4, file: !1, scope: !5, type: !6, function: void (<4 x i32>*, <4 x i32>*, <4 x i32>*)* @f1, variables: !14)
+!5 = !DIFile(filename: "/tmp/add.c", directory: "/home/richards/llvm/build")
+!6 = !DISubroutineType(types: !7)
 !7 = !{null, !8, !8, !8}
-!8 = !MDDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, baseType: !9)
-!9 = !MDDerivedType(tag: DW_TAG_typedef, name: "V4SI", line: 1, file: !1, baseType: !10)
-!10 = !MDCompositeType(tag: DW_TAG_array_type, size: 128, align: 128, flags: DIFlagVector, baseType: !11, elements: !12)
-!11 = !MDBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
+!8 = !DIDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, baseType: !9)
+!9 = !DIDerivedType(tag: DW_TAG_typedef, name: "V4SI", line: 1, file: !1, baseType: !10)
+!10 = !DICompositeType(tag: DW_TAG_array_type, size: 128, align: 128, flags: DIFlagVector, baseType: !11, elements: !12)
+!11 = !DIBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
 !12 = !{!13}
-!13 = !MDSubrange(count: 4)
+!13 = !DISubrange(count: 4)
 !14 = !{!15, !16, !17}
-!15 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "a", line: 3, arg: 1, scope: !4, file: !5, type: !8)
-!16 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "b", line: 3, arg: 2, scope: !4, file: !5, type: !8)
-!17 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "c", line: 3, arg: 3, scope: !4, file: !5, type: !8)
+!15 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "a", line: 3, arg: 1, scope: !4, file: !5, type: !8)
+!16 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "b", line: 3, arg: 2, scope: !4, file: !5, type: !8)
+!17 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "c", line: 3, arg: 3, scope: !4, file: !5, type: !8)
 !18 = !{i32 2, !"Dwarf Version", i32 4}
 !19 = !{!"clang version 3.4 (trunk 194134) (llvm/trunk 194126)"}
-!20 = !MDLocation(line: 3, scope: !4)
-!21 = !MDLocation(line: 5, scope: !4)
+!20 = !DILocation(line: 3, scope: !4)
+!21 = !DILocation(line: 5, scope: !4)
 !22 = !{!23, !23, i64 0}
 !23 = !{!"omnipotent char", !24, i64 0}
 !24 = !{!"Simple C/C++ TBAA"}
-!25 = !MDLocation(line: 6, scope: !4)
+!25 = !DILocation(line: 6, scope: !4)
 !26 = !{i32 1, !"Debug Info Version", i32 3}
diff --git a/test/Transforms/Scalarizer/store-bug.ll b/test/Transforms/Scalarizer/store-bug.ll
new file mode 100644
index 00000000000..84c2b3f840a
--- /dev/null
+++ b/test/Transforms/Scalarizer/store-bug.ll
@@ -0,0 +1,25 @@
+; RUN: opt -scalarizer -scalarize-load-store -S < %s | FileCheck %s
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
+
+; This input caused the scalarizer not to clear cached results
+; properly.
+;
+; Any regressions should trigger an assert in the scalarizer.
+
+define void @func(<4 x float> %val, <4 x float> *%ptr) {
+  store <4 x float> %val, <4 x float> *%ptr
+  ret void
+; CHECK: store float %val.i0, float* %ptr.i0, align 16
+; CHECK: store float %val.i1, float* %ptr.i1, align 4
+; CHECK: store float %val.i2, float* %ptr.i2, align 8
+; CHECK: store float %val.i3, float* %ptr.i3, align 4
+}
+
+define void @func.copy(<4 x float> %val, <4 x float> *%ptr) {
+  store <4 x float> %val, <4 x float> *%ptr
+  ret void
+; CHECK: store float %val.i0, float* %ptr.i0, align 16
+; CHECK: store float %val.i1, float* %ptr.i1, align 4
+; CHECK: store float %val.i2, float* %ptr.i2, align 8
+; CHECK: store float %val.i3, float* %ptr.i3, align 4
+}
diff --git a/test/Transforms/SeparateConstOffsetFromGEP/AMDGPU/lit.local.cfg b/test/Transforms/SeparateConstOffsetFromGEP/AMDGPU/lit.local.cfg
new file mode 100644
index 00000000000..6baccf05fff
--- /dev/null
+++ b/test/Transforms/SeparateConstOffsetFromGEP/AMDGPU/lit.local.cfg
@@ -0,0 +1,3 @@
+if not 'AMDGPU' in config.root.targets:
+    config.unsupported = True
+
diff --git a/test/Transforms/SeparateConstOffsetFromGEP/AMDGPU/split-gep-and-gvn-addrspace-addressing-modes.ll b/test/Transforms/SeparateConstOffsetFromGEP/AMDGPU/split-gep-and-gvn-addrspace-addressing-modes.ll
new file mode 100644
index 00000000000..527634db0f5
--- /dev/null
+++ b/test/Transforms/SeparateConstOffsetFromGEP/AMDGPU/split-gep-and-gvn-addrspace-addressing-modes.ll
@@ -0,0 +1,94 @@
+; RUN: opt -mtriple=amdgcn-- -S -separate-const-offset-from-gep -reassociate-geps-verify-no-dead-code -gvn < %s | FileCheck -check-prefix=IR %s
+
+target datalayout = "e-p:32:32-p1:64:64-p2:64:64-p3:32:32-p4:64:64-p5:32:32-p24:64:64-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-v2048:2048-n32:64"
+
+@array = internal addrspace(2) constant [4096 x [32 x float]] zeroinitializer, align 4
+
+; IR-LABEL: @sum_of_array(
+; IR: [[BASE_PTR:%[a-zA-Z0-9]+]] = getelementptr inbounds [4096 x [32 x float]], [4096 x [32 x float]] addrspace(2)* @array, i64 0, i64 %{{[a-zA-Z0-9]+}}, i64 %{{[a-zA-Z0-9]+}}
+; IR: getelementptr float, float addrspace(2)* [[BASE_PTR]], i64 1
+; IR: getelementptr float, float addrspace(2)* [[BASE_PTR]], i64 32
+; IR: getelementptr float, float addrspace(2)* [[BASE_PTR]], i64 33
+define void @sum_of_array(i32 %x, i32 %y, float addrspace(1)* nocapture %output) {
+  %tmp = sext i32 %y to i64
+  %tmp1 = sext i32 %x to i64
+  %tmp2 = getelementptr inbounds [4096 x [32 x float]], [4096 x [32 x float]] addrspace(2)* @array, i64 0, i64 %tmp1, i64 %tmp
+  %tmp4 = load float, float addrspace(2)* %tmp2, align 4
+  %tmp5 = fadd float %tmp4, 0.000000e+00
+  %tmp6 = add i32 %y, 1
+  %tmp7 = sext i32 %tmp6 to i64
+  %tmp8 = getelementptr inbounds [4096 x [32 x float]], [4096 x [32 x float]] addrspace(2)* @array, i64 0, i64 %tmp1, i64 %tmp7
+  %tmp10 = load float, float addrspace(2)* %tmp8, align 4
+  %tmp11 = fadd float %tmp5, %tmp10
+  %tmp12 = add i32 %x, 1
+  %tmp13 = sext i32 %tmp12 to i64
+  %tmp14 = getelementptr inbounds [4096 x [32 x float]], [4096 x [32 x float]] addrspace(2)* @array, i64 0, i64 %tmp13, i64 %tmp
+  %tmp16 = load float, float addrspace(2)* %tmp14, align 4
+  %tmp17 = fadd float %tmp11, %tmp16
+  %tmp18 = getelementptr inbounds [4096 x [32 x float]], [4096 x [32 x float]] addrspace(2)* @array, i64 0, i64 %tmp13, i64 %tmp7
+  %tmp20 = load float, float addrspace(2)* %tmp18, align 4
+  %tmp21 = fadd float %tmp17, %tmp20
+  store float %tmp21, float addrspace(1)* %output, align 4
+  ret void
+}
+
+@array2 = internal addrspace(2) constant [4096 x [4 x float]] zeroinitializer, align 4
+
+; Some of the indices go over the maximum mubuf offset, so don't split them.
+
+; IR-LABEL: @sum_of_array_over_max_mubuf_offset(
+; IR: [[BASE_PTR:%[a-zA-Z0-9]+]] = getelementptr inbounds [4096 x [4 x float]], [4096 x [4 x float]] addrspace(2)* @array2, i64 0, i64 %{{[a-zA-Z0-9]+}}, i64 %{{[a-zA-Z0-9]+}}
+; IR: getelementptr float, float addrspace(2)* [[BASE_PTR]], i64 255
+; IR: add i32 %x, 256
+; IR: getelementptr inbounds [4096 x [4 x float]], [4096 x [4 x float]] addrspace(2)* @array2, i64 0, i64 %{{[a-zA-Z0-9]+}}, i64 %{{[a-zA-Z0-9]+}}
+; IR: getelementptr inbounds [4096 x [4 x float]], [4096 x [4 x float]] addrspace(2)* @array2, i64 0, i64 %{{[a-zA-Z0-9]+}}, i64 %{{[a-zA-Z0-9]+}}
+define void @sum_of_array_over_max_mubuf_offset(i32 %x, i32 %y, float addrspace(1)* nocapture %output) {
+  %tmp = sext i32 %y to i64
+  %tmp1 = sext i32 %x to i64
+  %tmp2 = getelementptr inbounds [4096 x [4 x float]], [4096 x [4 x float]] addrspace(2)* @array2, i64 0, i64 %tmp1, i64 %tmp
+  %tmp4 = load float, float addrspace(2)* %tmp2, align 4
+  %tmp5 = fadd float %tmp4, 0.000000e+00
+  %tmp6 = add i32 %y, 255
+  %tmp7 = sext i32 %tmp6 to i64
+  %tmp8 = getelementptr inbounds [4096 x [4 x float]], [4096 x [4 x float]] addrspace(2)* @array2, i64 0, i64 %tmp1, i64 %tmp7
+  %tmp10 = load float, float addrspace(2)* %tmp8, align 4
+  %tmp11 = fadd float %tmp5, %tmp10
+  %tmp12 = add i32 %x, 256
+  %tmp13 = sext i32 %tmp12 to i64
+  %tmp14 = getelementptr inbounds [4096 x [4 x float]], [4096 x [4 x float]] addrspace(2)* @array2, i64 0, i64 %tmp13, i64 %tmp
+  %tmp16 = load float, float addrspace(2)* %tmp14, align 4
+  %tmp17 = fadd float %tmp11, %tmp16
+  %tmp18 = getelementptr inbounds [4096 x [4 x float]], [4096 x [4 x float]] addrspace(2)* @array2, i64 0, i64 %tmp13, i64 %tmp7
+  %tmp20 = load float, float addrspace(2)* %tmp18, align 4
+  %tmp21 = fadd float %tmp17, %tmp20
+  store float %tmp21, float addrspace(1)* %output, align 4
+  ret void
+}
+
+
+@lds_array = internal addrspace(3) global [4096 x [4 x float]] undef, align 4
+
+; DS instructions have a larger immediate offset, so make sure these are OK.
+; IR-LABEL: @sum_of_lds_array_over_max_mubuf_offset(
+; IR: [[BASE_PTR:%[a-zA-Z0-9]+]] = getelementptr inbounds [4096 x [4 x float]], [4096 x [4 x float]] addrspace(3)* @lds_array, i32 0, i32 %{{[a-zA-Z0-9]+}}, i32 %{{[a-zA-Z0-9]+}}
+; IR: getelementptr float, float addrspace(3)* [[BASE_PTR]], i32 255
+; IR: getelementptr float, float addrspace(3)* [[BASE_PTR]], i32 16128
+; IR: getelementptr float, float addrspace(3)* [[BASE_PTR]], i32 16383
+define void @sum_of_lds_array_over_max_mubuf_offset(i32 %x, i32 %y, float addrspace(1)* nocapture %output) {
+  %tmp2 = getelementptr inbounds [4096 x [4 x float]], [4096 x [4 x float]] addrspace(3)* @lds_array, i32 0, i32 %x, i32 %y
+  %tmp4 = load float, float addrspace(3)* %tmp2, align 4
+  %tmp5 = fadd float %tmp4, 0.000000e+00
+  %tmp6 = add i32 %y, 255
+  %tmp8 = getelementptr inbounds [4096 x [4 x float]], [4096 x [4 x float]] addrspace(3)* @lds_array, i32 0, i32 %x, i32 %tmp6
+  %tmp10 = load float, float addrspace(3)* %tmp8, align 4
+  %tmp11 = fadd float %tmp5, %tmp10
+  %tmp12 = add i32 %x, 4032
+  %tmp14 = getelementptr inbounds [4096 x [4 x float]], [4096 x [4 x float]] addrspace(3)* @lds_array, i32 0, i32 %tmp12, i32 %y
+  %tmp16 = load float, float addrspace(3)* %tmp14, align 4
+  %tmp17 = fadd float %tmp11, %tmp16
+  %tmp18 = getelementptr inbounds [4096 x [4 x float]], [4096 x [4 x float]] addrspace(3)* @lds_array, i32 0, i32 %tmp12, i32 %tmp6
+  %tmp20 = load float, float addrspace(3)* %tmp18, align 4
+  %tmp21 = fadd float %tmp17, %tmp20
+  store float %tmp21, float addrspace(1)* %output, align 4
+  ret void
+}
diff --git a/test/Transforms/SeparateConstOffsetFromGEP/NVPTX/split-gep-and-gvn.ll b/test/Transforms/SeparateConstOffsetFromGEP/NVPTX/split-gep-and-gvn.ll
index 9ee492df94e..073313d40e7 100644
--- a/test/Transforms/SeparateConstOffsetFromGEP/NVPTX/split-gep-and-gvn.ll
+++ b/test/Transforms/SeparateConstOffsetFromGEP/NVPTX/split-gep-and-gvn.ll
@@ -1,5 +1,5 @@
 ; RUN: llc < %s -march=nvptx64 -mcpu=sm_20 | FileCheck %s --check-prefix=PTX
-; RUN: opt < %s -S -separate-const-offset-from-gep -gvn -dce | FileCheck %s --check-prefix=IR
+; RUN: opt < %s -S -separate-const-offset-from-gep -reassociate-geps-verify-no-dead-code -gvn | FileCheck %s --check-prefix=IR
 
 ; Verifies the SeparateConstOffsetFromGEP pass.
 ; The following code computes
diff --git a/test/Transforms/SeparateConstOffsetFromGEP/NVPTX/split-gep.ll b/test/Transforms/SeparateConstOffsetFromGEP/NVPTX/split-gep.ll
index 574f6e963ee..2fdd158a35e 100644
--- a/test/Transforms/SeparateConstOffsetFromGEP/NVPTX/split-gep.ll
+++ b/test/Transforms/SeparateConstOffsetFromGEP/NVPTX/split-gep.ll
@@ -1,4 +1,4 @@
-; RUN: opt < %s -separate-const-offset-from-gep -dce -S | FileCheck %s
+; RUN: opt < %s -separate-const-offset-from-gep -reassociate-geps-verify-no-dead-code -S | FileCheck %s
 
 ; Several unit tests for -separate-const-offset-from-gep. The transformation
 ; heavily relies on TargetTransformInfo, so we put these tests under
diff --git a/test/Transforms/SeparateConstOffsetFromGEP/NVPTX/value-tracking-domtree.ll b/test/Transforms/SeparateConstOffsetFromGEP/NVPTX/value-tracking-domtree.ll
new file mode 100644
index 00000000000..d1a0f33d5a2
--- /dev/null
+++ b/test/Transforms/SeparateConstOffsetFromGEP/NVPTX/value-tracking-domtree.ll
@@ -0,0 +1,33 @@
+; RUN: opt < %s -separate-const-offset-from-gep -value-tracking-dom-conditions -reassociate-geps-verify-no-dead-code -S | FileCheck %s
+
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "nvptx64-unknown-unknown"
+
+; if (i == 4)
+;   p = &input[i | 3];
+;
+; =>
+;
+; if (i == 4) {
+;   base = &input[i];
+;   p = &base[3];
+; }
+;
+; We should treat (i | 3) as (i + 3) because i is guaranteed to be 4, which
+; does not share any set bits with 3.
+define float* @guarded_or(float* %input, i64 %i) {
+; CHECK-LABEL: @guarded_or(
+entry:
+  %is4 = icmp eq i64 %i, 4
+  br i1 %is4, label %then, label %exit
+
+then:
+  %or = or i64 %i, 3
+  %p = getelementptr inbounds float, float* %input, i64 %or
+; CHECK: [[base:[^ ]+]] = getelementptr float, float* %input, i64 %i
+; CHECK: getelementptr float, float* [[base]], i64 3
+  ret float* %p
+
+exit:
+  ret float* null
+}
diff --git a/test/Transforms/SimplifyCFG/2003-08-05-InvokeCrash.ll b/test/Transforms/SimplifyCFG/2003-08-05-InvokeCrash.ll
index 7551e8fb747..fe3a603a042 100644
--- a/test/Transforms/SimplifyCFG/2003-08-05-InvokeCrash.ll
+++ b/test/Transforms/SimplifyCFG/2003-08-05-InvokeCrash.ll
@@ -2,13 +2,13 @@
 ;
 ; RUN: opt < %s -simplifycfg -disable-output
 
-define i32 @test() {
+define i32 @test() personality i8* bitcast (i32 (...)* @__gxx_personality_v0 to i8*) {
 	%A = invoke i32 @test( )
 			to label %Ret unwind label %Ret2		; <i32> [#uses=1]
 Ret:		; preds = %0
 	ret i32 %A
 Ret2:		; preds = %0
-        %val = landingpad { i8*, i32 } personality i8* bitcast (i32 (...)* @__gxx_personality_v0 to i8*)
+        %val = landingpad { i8*, i32 }
                  catch i8* null
 	ret i32 undef
 }
diff --git a/test/Transforms/SimplifyCFG/2005-10-02-InvokeSimplify.ll b/test/Transforms/SimplifyCFG/2005-10-02-InvokeSimplify.ll
index 2ecdc95259c..c71f05bf0a3 100644
--- a/test/Transforms/SimplifyCFG/2005-10-02-InvokeSimplify.ll
+++ b/test/Transforms/SimplifyCFG/2005-10-02-InvokeSimplify.ll
@@ -1,10 +1,10 @@
 ; RUN: opt < %s -simplifycfg -disable-output
 
-define i1 @foo() {
+define i1 @foo() personality i8* bitcast (i32 (...)* @__gxx_personality_v0 to i8*) {
 	%X = invoke i1 @foo( )
 			to label %N unwind label %F		; <i1> [#uses=1]
 F:		; preds = %0
-        %val = landingpad { i8*, i32 } personality i8* bitcast (i32 (...)* @__gxx_personality_v0 to i8*)
+        %val = landingpad { i8*, i32 }
                  catch i8* null
 	ret i1 false
 N:		; preds = %0
diff --git a/test/Transforms/SimplifyCFG/2007-11-22-InvokeNoUnwind.ll b/test/Transforms/SimplifyCFG/2007-11-22-InvokeNoUnwind.ll
index 76f41e8fc21..c38d71ccd28 100644
--- a/test/Transforms/SimplifyCFG/2007-11-22-InvokeNoUnwind.ll
+++ b/test/Transforms/SimplifyCFG/2007-11-22-InvokeNoUnwind.ll
@@ -4,7 +4,7 @@
 
 declare i32 @func(i8*) nounwind
 
-define i32 @test() {
+define i32 @test() personality i32 (...)* @__gxx_personality_v0 {
 	invoke i32 @func( i8* null )
 			to label %Cont unwind label %Other		; <i32>:1 [#uses=0]
 
@@ -12,7 +12,7 @@ Cont:		; preds = %0
 	ret i32 0
 
 Other:		; preds = %0
-	landingpad { i8*, i32 } personality i32 (...)* @__gxx_personality_v0
+	landingpad { i8*, i32 }
 		catch i8* null
 	ret i32 1
 }
diff --git a/test/Transforms/SimplifyCFG/2010-03-30-InvokeCrash.ll b/test/Transforms/SimplifyCFG/2010-03-30-InvokeCrash.ll
index 333336de767..d545739bc74 100644
--- a/test/Transforms/SimplifyCFG/2010-03-30-InvokeCrash.ll
+++ b/test/Transforms/SimplifyCFG/2010-03-30-InvokeCrash.ll
@@ -5,7 +5,7 @@ target triple = "x86_64-unknown-linux-gnu"
 
 declare void @bar(i32)
 
-define void @foo() {
+define void @foo() personality i32 (...)* @__gxx_personality_v0 {
 entry:
  invoke void @bar(i32 undef)
          to label %r unwind label %u
@@ -14,7 +14,7 @@ r:                                                ; preds = %entry
  ret void
 
 u:                                                ; preds = %entry
- %val = landingpad { i8*, i32 } personality i32 (...)* @__gxx_personality_v0
+ %val = landingpad { i8*, i32 }
           cleanup
  resume { i8*, i32 } %val
 }
diff --git a/test/Transforms/SimplifyCFG/2011-09-05-TrivialLPad.ll b/test/Transforms/SimplifyCFG/2011-09-05-TrivialLPad.ll
index 7558419a2eb..111434b7fcd 100644
--- a/test/Transforms/SimplifyCFG/2011-09-05-TrivialLPad.ll
+++ b/test/Transforms/SimplifyCFG/2011-09-05-TrivialLPad.ll
@@ -5,7 +5,7 @@
 
 declare void @bar()
 
-define i32 @foo() {
+define i32 @foo() personality i32 (i32, i64, i8*, i8*)* @__gxx_personality_v0 {
 entry:
   invoke void @bar()
           to label %return unwind label %lpad
@@ -14,7 +14,7 @@ return:
   ret i32 0
 
 lpad:
-  %lp = landingpad { i8*, i32 } personality i32 (i32, i64, i8*, i8*)* @__gxx_personality_v0
+  %lp = landingpad { i8*, i32 }
           cleanup
   resume { i8*, i32 } %lp
 }
diff --git a/test/Transforms/SimplifyCFG/R600/cttz-ctlz.ll b/test/Transforms/SimplifyCFG/AMDGPU/cttz-ctlz.ll
similarity index 100%
rename from test/Transforms/SimplifyCFG/R600/cttz-ctlz.ll
rename to test/Transforms/SimplifyCFG/AMDGPU/cttz-ctlz.ll
diff --git a/test/Transforms/SimplifyCFG/AMDGPU/lit.local.cfg b/test/Transforms/SimplifyCFG/AMDGPU/lit.local.cfg
new file mode 100644
index 00000000000..2a665f06be7
--- /dev/null
+++ b/test/Transforms/SimplifyCFG/AMDGPU/lit.local.cfg
@@ -0,0 +1,2 @@
+if not 'AMDGPU' in config.root.targets:
+    config.unsupported = True
diff --git a/test/Transforms/SimplifyCFG/R600/lit.local.cfg b/test/Transforms/SimplifyCFG/R600/lit.local.cfg
deleted file mode 100644
index ad9ce2541ef..00000000000
--- a/test/Transforms/SimplifyCFG/R600/lit.local.cfg
+++ /dev/null
@@ -1,2 +0,0 @@
-if not 'R600' in config.root.targets:
-    config.unsupported = True
diff --git a/test/Transforms/SimplifyCFG/UnreachableEliminate.ll b/test/Transforms/SimplifyCFG/UnreachableEliminate.ll
index 8718c552c69..87872a6a8a1 100644
--- a/test/Transforms/SimplifyCFG/UnreachableEliminate.ll
+++ b/test/Transforms/SimplifyCFG/UnreachableEliminate.ll
@@ -13,7 +13,7 @@ F:
         ret void
 }
 
-define void @test2() {
+define void @test2() personality i32 (...)* @__gxx_personality_v0 {
 ; CHECK-LABEL: @test2(
 ; CHECK: entry:
 ; CHECK-NEXT: call void @test2()
@@ -22,7 +22,7 @@ entry:
         invoke void @test2( )
                         to label %N unwind label %U
 U:
-  %res = landingpad { i8* } personality i32 (...)* @__gxx_personality_v0
+  %res = landingpad { i8* }
           cleanup
         unreachable
 N:
diff --git a/test/Transforms/SimplifyCFG/X86/switch_to_lookup_table.ll b/test/Transforms/SimplifyCFG/X86/switch_to_lookup_table.ll
index cdc0488991a..cae1a91bd43 100644
--- a/test/Transforms/SimplifyCFG/X86/switch_to_lookup_table.ll
+++ b/test/Transforms/SimplifyCFG/X86/switch_to_lookup_table.ll
@@ -7,22 +7,22 @@ target triple = "x86_64-unknown-linux-gnu"
 ; CHECK: @switch.table = private unnamed_addr constant [7 x i32] [i32 55, i32 123, i32 0, i32 -1, i32 27, i32 62, i32 1]
 
 ; The float table for @h
-; CHECK: @switch.table1 = private unnamed_addr constant [4 x float] [float 0x40091EB860000000, float 0x3FF3BE76C0000000, float 0x4012449BA0000000, float 0x4001AE1480000000]
+; CHECK: @switch.table.1 = private unnamed_addr constant [4 x float] [float 0x40091EB860000000, float 0x3FF3BE76C0000000, float 0x4012449BA0000000, float 0x4001AE1480000000]
 
 ; The table for @foostring
-; CHECK: @switch.table2 = private unnamed_addr constant [4 x i8*] [i8* getelementptr inbounds ([4 x i8], [4 x i8]* @.str, i64 0, i64 0), i8* getelementptr inbounds ([4 x i8], [4 x i8]* @.str1, i64 0, i64 0), i8* getelementptr inbounds ([4 x i8], [4 x i8]* @.str2, i64 0, i64 0), i8* getelementptr inbounds ([4 x i8], [4 x i8]* @.str3, i64 0, i64 0)]
+; CHECK: @switch.table.2 = private unnamed_addr constant [4 x i8*] [i8* getelementptr inbounds ([4 x i8], [4 x i8]* @.str, i64 0, i64 0), i8* getelementptr inbounds ([4 x i8], [4 x i8]* @.str1, i64 0, i64 0), i8* getelementptr inbounds ([4 x i8], [4 x i8]* @.str2, i64 0, i64 0), i8* getelementptr inbounds ([4 x i8], [4 x i8]* @.str3, i64 0, i64 0)]
 
 ; The table for @earlyreturncrash
-; CHECK: @switch.table3 = private unnamed_addr constant [4 x i32] [i32 42, i32 9, i32 88, i32 5]
+; CHECK: @switch.table.3 = private unnamed_addr constant [4 x i32] [i32 42, i32 9, i32 88, i32 5]
 
 ; The table for @large.
-; CHECK: @switch.table4 = private unnamed_addr constant [199 x i32] [i32 1, i32 4, i32 9,
+; CHECK: @switch.table.4 = private unnamed_addr constant [199 x i32] [i32 1, i32 4, i32 9,
 
 ; The table for @cprop
-; CHECK: @switch.table5 = private unnamed_addr constant [7 x i32] [i32 5, i32 42, i32 126, i32 -452, i32 128, i32 6, i32 7]
+; CHECK: @switch.table.5 = private unnamed_addr constant [7 x i32] [i32 5, i32 42, i32 126, i32 -452, i32 128, i32 6, i32 7]
 
 ; The table for @unreachable_case
-; CHECK: @switch.table6 = private unnamed_addr constant [9 x i32] [i32 0, i32 0, i32 0, i32 2, i32 -1, i32 1, i32 1, i32 1, i32 1]
+; CHECK: @switch.table.6 = private unnamed_addr constant [9 x i32] [i32 0, i32 0, i32 0, i32 2, i32 -1, i32 1, i32 1, i32 1, i32 1]
 
 ; A simple int-to-int selection switch.
 ; It is dense enough to be replaced by table lookup.
@@ -97,7 +97,7 @@ sw.epilog:
 ; CHECK-NEXT: %switch.shiftamt = mul i32 %switch.tableidx, 8
 ; CHECK-NEXT: %switch.downshift = lshr i32 89655594, %switch.shiftamt
 ; CHECK-NEXT: %switch.masked = trunc i32 %switch.downshift to i8
-; CHECK-NEXT: %switch.gep = getelementptr inbounds [4 x float], [4 x float]* @switch.table1, i32 0, i32 %switch.tableidx
+; CHECK-NEXT: %switch.gep = getelementptr inbounds [4 x float], [4 x float]* @switch.table.1, i32 0, i32 %switch.tableidx
 ; CHECK-NEXT: %switch.load = load float, float* %switch.gep
 ; CHECK-NEXT: br label %sw.epilog
 ; CHECK: sw.epilog:
@@ -144,7 +144,7 @@ return:
 ; CHECK-NEXT: %0 = icmp ult i32 %switch.tableidx, 4
 ; CHECK-NEXT: br i1 %0, label %switch.lookup, label %return
 ; CHECK: switch.lookup:
-; CHECK-NEXT: %switch.gep = getelementptr inbounds [4 x i8*], [4 x i8*]* @switch.table2, i32 0, i32 %switch.tableidx
+; CHECK-NEXT: %switch.gep = getelementptr inbounds [4 x i8*], [4 x i8*]* @switch.table.2, i32 0, i32 %switch.tableidx
 ; CHECK-NEXT: %switch.load = load i8*, i8** %switch.gep
 ; CHECK-NEXT: ret i8* %switch.load
 }
@@ -173,7 +173,7 @@ sw.epilog:
 
 ; CHECK-LABEL: @earlyreturncrash(
 ; CHECK: switch.lookup:
-; CHECK-NEXT: %switch.gep = getelementptr inbounds [4 x i32], [4 x i32]* @switch.table3, i32 0, i32 %switch.tableidx
+; CHECK-NEXT: %switch.gep = getelementptr inbounds [4 x i32], [4 x i32]* @switch.table.3, i32 0, i32 %switch.tableidx
 ; CHECK-NEXT: %switch.load = load i32, i32* %switch.gep
 ; CHECK-NEXT: ret i32 %switch.load
 ; CHECK: sw.epilog:
@@ -749,7 +749,7 @@ return:
 
 ; CHECK-LABEL: @cprop(
 ; CHECK: switch.lookup:
-; CHECK: %switch.gep = getelementptr inbounds [7 x i32], [7 x i32]* @switch.table5, i32 0, i32 %switch.tableidx
+; CHECK: %switch.gep = getelementptr inbounds [7 x i32], [7 x i32]* @switch.table.5, i32 0, i32 %switch.tableidx
 }
 
 define i32 @unreachable_case(i32 %x)  {
@@ -778,7 +778,7 @@ return:
 
 ; CHECK-LABEL: @unreachable_case(
 ; CHECK: switch.lookup:
-; CHECK: getelementptr inbounds [9 x i32], [9 x i32]* @switch.table6, i32 0, i32 %switch.tableidx
+; CHECK: getelementptr inbounds [9 x i32], [9 x i32]* @switch.table.6, i32 0, i32 %switch.tableidx
 }
 
 define i32 @unreachable_default(i32 %x)  {
@@ -805,7 +805,7 @@ return:
 ; CHECK-NEXT: %switch.tableidx = sub i32 %x, 0
 ; CHECK-NOT: icmp
 ; CHECK-NOT: br 1i
-; CHECK-NEXT: %switch.gep = getelementptr inbounds [4 x i32], [4 x i32]* @switch.table7, i32 0, i32 %switch.tableidx
+; CHECK-NEXT: %switch.gep = getelementptr inbounds [4 x i32], [4 x i32]* @switch.table.7, i32 0, i32 %switch.tableidx
 ; CHECK-NEXT: %switch.load = load i32, i32* %switch.gep
 ; CHECK-NEXT: ret i32 %switch.load
 }
@@ -1273,3 +1273,32 @@ end:
 ; CHECK-LABEL: @pr20210
 ; CHECK: switch i8 %x
 }
+
+; Make sure we do not crash due to trying to generate an unguarded
+; lookup (since i3 can only hold values in the range of explicit
+; values) and simultaneously trying to generate a branch to deal with
+; the fact that we have holes in the range.
+define i32 @covered_switch_with_bit_tests(i3) {
+entry:
+  switch i3 %0, label %l6 [
+    i3 -3, label %l5
+    i3 -4, label %l5
+    i3 3, label %l1
+    i3 2, label %l1
+  ]
+
+l1: br label %l2
+
+l2:
+  %x = phi i32 [ 1, %l1 ], [ 2, %l5 ]
+  br label %l6
+
+l5: br label %l2
+
+l6:
+  %r = phi i32 [ %x, %l2 ], [ 0, %entry ]
+  ret i32 %r
+; CHECK-LABEL: @covered_switch_with_bit_tests
+; CHECK: entry
+; CHECK-NEXT: switch
+}
diff --git a/test/Transforms/SimplifyCFG/branch-fold-dbg.ll b/test/Transforms/SimplifyCFG/branch-fold-dbg.ll
index 3ddd351f92b..fac5b186e89 100644
--- a/test/Transforms/SimplifyCFG/branch-fold-dbg.ll
+++ b/test/Transforms/SimplifyCFG/branch-fold-dbg.ll
@@ -41,19 +41,19 @@ declare void @llvm.dbg.value(metadata, i64, metadata, metadata) nounwind readnon
 
 !llvm.dbg.sp = !{!0}
 
-!0 = !MDSubprogram(name: "foo", line: 231, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, file: !15, scope: !1, type: !3, function: void (i32)* @foo)
-!1 = !MDFile(filename: "a.c", directory: "/private/tmp")
-!2 = !MDCompileUnit(language: DW_LANG_C99, producer: "clang (trunk 129006)", isOptimized: true, emissionKind: 0, file: !15, enums: !4, retainedTypes: !4)
-!3 = !MDSubroutineType(types: !4)
+!0 = !DISubprogram(name: "foo", line: 231, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, file: !15, scope: !1, type: !3, function: void (i32)* @foo)
+!1 = !DIFile(filename: "a.c", directory: "/private/tmp")
+!2 = !DICompileUnit(language: DW_LANG_C99, producer: "clang (trunk 129006)", isOptimized: true, emissionKind: 0, file: !15, enums: !4, retainedTypes: !4)
+!3 = !DISubroutineType(types: !4)
 !4 = !{null}
-!5 = !MDLocation(line: 131, column: 2, scope: !0)
-!6 = !MDLocation(line: 134, column: 2, scope: !0)
-!7 = !MDLocalVariable(tag: DW_TAG_auto_variable, name: "bar", line: 232, scope: !8, file: !1, type: !9)
-!8 = distinct !MDLexicalBlock(line: 231, column: 1, file: !15, scope: !0)
-!9 = !MDDerivedType(tag: DW_TAG_pointer_type, size: 32, align: 32, scope: !2, baseType: !10)
-!10 = !MDDerivedType(tag: DW_TAG_const_type, scope: !2, baseType: !11)
-!11 = !MDBasicType(tag: DW_TAG_base_type, name: "unsigned int", size: 32, align: 32, encoding: DW_ATE_unsigned)
-!12 = !MDLocation(line: 232, column: 40, scope: !8)
-!13 = !MDLocation(line: 234, column: 2, scope: !8)
-!14 = !MDLocation(line: 274, column: 1, scope: !8)
-!15 = !MDFile(filename: "a.c", directory: "/private/tmp")
+!5 = !DILocation(line: 131, column: 2, scope: !0)
+!6 = !DILocation(line: 134, column: 2, scope: !0)
+!7 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "bar", line: 232, scope: !8, file: !1, type: !9)
+!8 = distinct !DILexicalBlock(line: 231, column: 1, file: !15, scope: !0)
+!9 = !DIDerivedType(tag: DW_TAG_pointer_type, size: 32, align: 32, scope: !2, baseType: !10)
+!10 = !DIDerivedType(tag: DW_TAG_const_type, scope: !2, baseType: !11)
+!11 = !DIBasicType(tag: DW_TAG_base_type, name: "unsigned int", size: 32, align: 32, encoding: DW_ATE_unsigned)
+!12 = !DILocation(line: 232, column: 40, scope: !8)
+!13 = !DILocation(line: 234, column: 2, scope: !8)
+!14 = !DILocation(line: 274, column: 1, scope: !8)
+!15 = !DIFile(filename: "a.c", directory: "/private/tmp")
diff --git a/test/Transforms/SimplifyCFG/duplicate-landingpad.ll b/test/Transforms/SimplifyCFG/duplicate-landingpad.ll
index 54028774d20..93c55f0064c 100644
--- a/test/Transforms/SimplifyCFG/duplicate-landingpad.ll
+++ b/test/Transforms/SimplifyCFG/duplicate-landingpad.ll
@@ -6,7 +6,7 @@ declare void @fn()
 
 
 ; CHECK-LABEL: @test1
-define void @test1() {
+define void @test1() personality i32 (...)* @__gxx_personality_v0 {
 entry:
 ; CHECK-LABEL: entry:
 ; CHECK: to label %invoke2 unwind label %lpad2
@@ -23,17 +23,17 @@ invoke.cont:
   ret void
 
 lpad1:
-  %exn = landingpad {i8*, i32} personality i32 (...)* @__gxx_personality_v0
+  %exn = landingpad {i8*, i32}
          cleanup
   br label %shared_resume
 
 lpad2:
 ; CHECK-LABEL: lpad2:
-; CHECK: landingpad { i8*, i32 } personality i32 (...)* @__gxx_personality_v0
+; CHECK: landingpad { i8*, i32 }
 ; CHECK-NEXT: cleanup
 ; CHECK-NEXT: call void @fn()
 ; CHECK-NEXT: ret void
-  %exn2 = landingpad {i8*, i32} personality i32 (...)* @__gxx_personality_v0
+  %exn2 = landingpad {i8*, i32}
           cleanup
   br label %shared_resume
 
@@ -43,7 +43,7 @@ shared_resume:
 }
 
 ; Don't trigger if blocks aren't the same/empty
-define void @neg1() {
+define void @neg1() personality i32 (...)* @__gxx_personality_v0 {
 ; CHECK-LABEL: @neg1
 entry:
 ; CHECK-LABEL: entry:
@@ -61,13 +61,13 @@ invoke.cont:
   ret void
 
 lpad1:
-  %exn = landingpad {i8*, i32} personality i32 (...)* @__gxx_personality_v0
+  %exn = landingpad {i8*, i32}
          filter [0 x i8*] zeroinitializer
   call void @fn()
   br label %shared_resume
 
 lpad2:
-  %exn2 = landingpad {i8*, i32} personality i32 (...)* @__gxx_personality_v0
+  %exn2 = landingpad {i8*, i32}
           cleanup
   br label %shared_resume
 
@@ -77,7 +77,7 @@ shared_resume:
 }
 
 ; Should not trigger when the landing pads are not the exact same
-define void @neg2() {
+define void @neg2() personality i32 (...)* @__gxx_personality_v0 {
 ; CHECK-LABEL: @neg2
 entry:
 ; CHECK-LABEL: entry:
@@ -95,12 +95,12 @@ invoke.cont:
   ret void
 
 lpad1:
-  %exn = landingpad {i8*, i32} personality i32 (...)* @__gxx_personality_v0
+  %exn = landingpad {i8*, i32}
          filter [0 x i8*] zeroinitializer
   br label %shared_resume
 
 lpad2:
-  %exn2 = landingpad {i8*, i32} personality i32 (...)* @__gxx_personality_v0
+  %exn2 = landingpad {i8*, i32}
           cleanup
   br label %shared_resume
 
diff --git a/test/Transforms/SimplifyCFG/hoist-dbgvalue.ll b/test/Transforms/SimplifyCFG/hoist-dbgvalue.ll
index ee7df267116..345cf6282e3 100644
--- a/test/Transforms/SimplifyCFG/hoist-dbgvalue.ll
+++ b/test/Transforms/SimplifyCFG/hoist-dbgvalue.ll
@@ -1,8 +1,8 @@
 ; RUN: opt -simplifycfg -S < %s | FileCheck %s
 
 define i32 @foo(i32 %i) nounwind ssp {
-  call void @llvm.dbg.value(metadata i32 %i, i64 0, metadata !6, metadata !MDExpression()), !dbg !7
-  call void @llvm.dbg.value(metadata i32 0, i64 0, metadata !9, metadata !MDExpression()), !dbg !11
+  call void @llvm.dbg.value(metadata i32 %i, i64 0, metadata !6, metadata !DIExpression()), !dbg !7
+  call void @llvm.dbg.value(metadata i32 0, i64 0, metadata !9, metadata !DIExpression()), !dbg !11
   %1 = icmp ne i32 %i, 0, !dbg !12
 ;CHECK: call i32 (...) @bar()
 ;CHECK-NEXT: llvm.dbg.value
@@ -10,12 +10,12 @@ define i32 @foo(i32 %i) nounwind ssp {
 
 ; <label>:2                                       ; preds = %0
   %3 = call i32 (...) @bar(), !dbg !13
-  call void @llvm.dbg.value(metadata i32 %3, i64 0, metadata !9, metadata !MDExpression()), !dbg !13
+  call void @llvm.dbg.value(metadata i32 %3, i64 0, metadata !9, metadata !DIExpression()), !dbg !13
   br label %6, !dbg !15
 
 ; <label>:4                                       ; preds = %0
   %5 = call i32 (...) @bar(), !dbg !16
-  call void @llvm.dbg.value(metadata i32 %5, i64 0, metadata !9, metadata !MDExpression()), !dbg !16
+  call void @llvm.dbg.value(metadata i32 %5, i64 0, metadata !9, metadata !DIExpression()), !dbg !16
   br label %6, !dbg !18
 
 ; <label>:6                                       ; preds = %4, %2
@@ -32,25 +32,25 @@ declare void @llvm.dbg.value(metadata, i64, metadata, metadata) nounwind readnon
 !llvm.module.flags = !{!21}
 !llvm.dbg.sp = !{!0}
 
-!0 = !MDSubprogram(name: "foo", line: 2, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, file: !20, scope: !1, type: !3, function: i32 (i32)* @foo)
-!1 = !MDFile(filename: "b.c", directory: "/private/tmp")
-!2 = !MDCompileUnit(language: DW_LANG_C99, producer: "clang", isOptimized: true, emissionKind: 0, file: !20, enums: !8, retainedTypes: !8)
-!3 = !MDSubroutineType(types: !4)
+!0 = !DISubprogram(name: "foo", line: 2, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, file: !20, scope: !1, type: !3, function: i32 (i32)* @foo)
+!1 = !DIFile(filename: "b.c", directory: "/private/tmp")
+!2 = !DICompileUnit(language: DW_LANG_C99, producer: "clang", isOptimized: true, emissionKind: 0, file: !20, enums: !8, retainedTypes: !8)
+!3 = !DISubroutineType(types: !4)
 !4 = !{!5}
-!5 = !MDBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
-!6 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "i", line: 2, arg: 1, scope: !0, file: !1, type: !5)
-!7 = !MDLocation(line: 2, column: 13, scope: !0)
+!5 = !DIBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
+!6 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "i", line: 2, arg: 1, scope: !0, file: !1, type: !5)
+!7 = !DILocation(line: 2, column: 13, scope: !0)
 !8 = !{i32 0}
-!9 = !MDLocalVariable(tag: DW_TAG_auto_variable, name: "k", line: 3, scope: !10, file: !1, type: !5)
-!10 = distinct !MDLexicalBlock(line: 2, column: 16, file: !20, scope: !0)
-!11 = !MDLocation(line: 3, column: 12, scope: !10)
-!12 = !MDLocation(line: 4, column: 3, scope: !10)
-!13 = !MDLocation(line: 5, column: 5, scope: !14)
-!14 = distinct !MDLexicalBlock(line: 4, column: 10, file: !20, scope: !10)
-!15 = !MDLocation(line: 6, column: 3, scope: !14)
-!16 = !MDLocation(line: 7, column: 5, scope: !17)
-!17 = distinct !MDLexicalBlock(line: 6, column: 10, file: !20, scope: !10)
-!18 = !MDLocation(line: 8, column: 3, scope: !17)
-!19 = !MDLocation(line: 9, column: 3, scope: !10)
-!20 = !MDFile(filename: "b.c", directory: "/private/tmp")
+!9 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "k", line: 3, scope: !10, file: !1, type: !5)
+!10 = distinct !DILexicalBlock(line: 2, column: 16, file: !20, scope: !0)
+!11 = !DILocation(line: 3, column: 12, scope: !10)
+!12 = !DILocation(line: 4, column: 3, scope: !10)
+!13 = !DILocation(line: 5, column: 5, scope: !14)
+!14 = distinct !DILexicalBlock(line: 4, column: 10, file: !20, scope: !10)
+!15 = !DILocation(line: 6, column: 3, scope: !14)
+!16 = !DILocation(line: 7, column: 5, scope: !17)
+!17 = distinct !DILexicalBlock(line: 6, column: 10, file: !20, scope: !10)
+!18 = !DILocation(line: 8, column: 3, scope: !17)
+!19 = !DILocation(line: 9, column: 3, scope: !10)
+!20 = !DIFile(filename: "b.c", directory: "/private/tmp")
 !21 = !{i32 1, !"Debug Info Version", i32 3}
diff --git a/test/Transforms/SimplifyCFG/invoke.ll b/test/Transforms/SimplifyCFG/invoke.ll
index 5f513ac3e6d..b7fd7d62ccf 100644
--- a/test/Transforms/SimplifyCFG/invoke.ll
+++ b/test/Transforms/SimplifyCFG/invoke.ll
@@ -10,7 +10,7 @@ declare i32 @fn()
 
 
 ; CHECK-LABEL: @f1(
-define i8* @f1() nounwind uwtable ssp {
+define i8* @f1() nounwind uwtable ssp personality i8* bitcast (i32 (...)* @__gxx_personality_v0 to i8*) {
 entry:
 ; CHECK: call void @llvm.trap()
 ; CHECK: unreachable
@@ -21,7 +21,7 @@ invoke.cont:
   ret i8* %call
 
 lpad:
-  %0 = landingpad { i8*, i32 } personality i8* bitcast (i32 (...)* @__gxx_personality_v0 to i8*)
+  %0 = landingpad { i8*, i32 }
           filter [0 x i8*] zeroinitializer
   %1 = extractvalue { i8*, i32 } %0, 0
   tail call void @__cxa_call_unexpected(i8* %1) noreturn nounwind
@@ -29,7 +29,7 @@ lpad:
 }
 
 ; CHECK-LABEL: @f2(
-define i8* @f2() nounwind uwtable ssp {
+define i8* @f2() nounwind uwtable ssp personality i8* bitcast (i32 (...)* @__gxx_personality_v0 to i8*) {
 entry:
 ; CHECK: call void @llvm.trap()
 ; CHECK: unreachable
@@ -40,7 +40,7 @@ invoke.cont:
   ret i8* %call
 
 lpad:
-  %0 = landingpad { i8*, i32 } personality i8* bitcast (i32 (...)* @__gxx_personality_v0 to i8*)
+  %0 = landingpad { i8*, i32 }
           filter [0 x i8*] zeroinitializer
   %1 = extractvalue { i8*, i32 } %0, 0
   tail call void @__cxa_call_unexpected(i8* %1) noreturn nounwind
@@ -48,7 +48,7 @@ lpad:
 }
 
 ; CHECK-LABEL: @f3(
-define i32 @f3() nounwind uwtable ssp {
+define i32 @f3() nounwind uwtable ssp personality i8* bitcast (i32 (...)* @__gxx_personality_v0 to i8*) {
 ; CHECK-NEXT: entry
 entry:
 ; CHECK-NEXT: ret i32 3
@@ -59,7 +59,7 @@ invoke.cont:
   ret i32 3
 
 lpad:
-  %0 = landingpad { i8*, i32 } personality i8* bitcast (i32 (...)* @__gxx_personality_v0 to i8*)
+  %0 = landingpad { i8*, i32 }
           filter [0 x i8*] zeroinitializer
   %1 = extractvalue { i8*, i32 } %0, 0
   tail call void @__cxa_call_unexpected(i8* %1) noreturn nounwind
@@ -67,7 +67,7 @@ lpad:
 }
 
 ; CHECK-LABEL: @f4(
-define i32 @f4() nounwind uwtable ssp {
+define i32 @f4() nounwind uwtable ssp personality i8* bitcast (i32 (...)* @__gxx_personality_v0 to i8*) {
 ; CHECK-NEXT: entry
 entry:
 ; CHECK-NEXT: call i32 @read_only()
@@ -79,7 +79,7 @@ invoke.cont:
   ret i32 %call
 
 lpad:
-  %0 = landingpad { i8*, i32 } personality i8* bitcast (i32 (...)* @__gxx_personality_v0 to i8*)
+  %0 = landingpad { i8*, i32 }
           filter [0 x i8*] zeroinitializer
   %1 = extractvalue { i8*, i32 } %0, 0
   tail call void @__cxa_call_unexpected(i8* %1) noreturn nounwind
@@ -87,7 +87,7 @@ lpad:
 }
 
 ; CHECK-LABEL: @f5(
-define i32 @f5(i1 %cond, i8* %a, i8* %b) {
+define i32 @f5(i1 %cond, i8* %a, i8* %b) personality i8* bitcast (i32 (...)* @__gxx_personality_v0 to i8*) {
 entry:
   br i1 %cond, label %x, label %y
 
@@ -110,7 +110,7 @@ cont:
 lpad:
 ; CHECK-NOT: phi
   %phi2 = phi i8* [%a, %x], [%b, %y]
-  %0 = landingpad { i8*, i32 } personality i8* bitcast (i32 (...)* @__gxx_personality_v0 to i8*)
+  %0 = landingpad { i8*, i32 }
           filter [0 x i8*] zeroinitializer
 ; CHECK: __cxa_call_unexpected(i8* %a)
   tail call void @__cxa_call_unexpected(i8* %phi2) noreturn nounwind
@@ -118,7 +118,7 @@ lpad:
 }
 
 ; CHECK-LABEL: @f6(
-define void @f6() {
+define void @f6() personality i8* bitcast (i32 (...)* @__gxx_personality_v0 to i8*) {
 entry:
   invoke void @purefn()
           to label %invoke.cont1 unwind label %lpad
@@ -133,7 +133,7 @@ invoke.cont2:
 lpad:
 ; CHECK-NOT: phi
   %tmp = phi i8* [ null, %invoke.cont1 ], [ null, %entry ]
-  landingpad { i8*, i32 } personality i8* bitcast (i32 (...)* @__gxx_personality_v0 to i8*)
+  landingpad { i8*, i32 }
           cleanup
   ret void
 }
diff --git a/test/Transforms/SimplifyCFG/invoke_unwind.ll b/test/Transforms/SimplifyCFG/invoke_unwind.ll
index 435bed0c295..3b4c09d96f7 100644
--- a/test/Transforms/SimplifyCFG/invoke_unwind.ll
+++ b/test/Transforms/SimplifyCFG/invoke_unwind.ll
@@ -4,7 +4,7 @@ declare void @bar()
 
 ; This testcase checks to see if the simplifycfg pass is converting invoke
 ; instructions to call instructions if the handler just rethrows the exception.
-define i32 @test1() {
+define i32 @test1() personality i32 (...)* @__gxx_personality_v0 {
 ; CHECK-LABEL: @test1(
 ; CHECK-NEXT: call void @bar()
 ; CHECK-NEXT: ret i32 0
@@ -12,7 +12,7 @@ define i32 @test1() {
                         to label %1 unwind label %Rethrow
         ret i32 0
 Rethrow:
-        %exn = landingpad {i8*, i32} personality i32 (...)* @__gxx_personality_v0
+        %exn = landingpad {i8*, i32}
                  catch i8* null
         resume { i8*, i32 } %exn
 }
diff --git a/test/Transforms/SimplifyCFG/seh-nounwind.ll b/test/Transforms/SimplifyCFG/seh-nounwind.ll
index 3845e319890..c380c6ce2ce 100644
--- a/test/Transforms/SimplifyCFG/seh-nounwind.ll
+++ b/test/Transforms/SimplifyCFG/seh-nounwind.ll
@@ -10,13 +10,13 @@ entry:
   ret i32 %div
 }
 
-define i32 @main() nounwind {
+define i32 @main() nounwind personality i8* bitcast (i32 (...)* @__C_specific_handler to i8*) {
 entry:
   %call = invoke i32 @div(i32 10, i32 0)
           to label %__try.cont unwind label %lpad
 
 lpad:
-  %0 = landingpad { i8*, i32 } personality i8* bitcast (i32 (...)* @__C_specific_handler to i8*)
+  %0 = landingpad { i8*, i32 }
           catch i8* null
   br label %__try.cont
 
diff --git a/test/Transforms/SimplifyCFG/statepoint-invoke-unwind.ll b/test/Transforms/SimplifyCFG/statepoint-invoke-unwind.ll
new file mode 100644
index 00000000000..994e47eb0d6
--- /dev/null
+++ b/test/Transforms/SimplifyCFG/statepoint-invoke-unwind.ll
@@ -0,0 +1,24 @@
+; RUN: opt < %s -simplifycfg -S | FileCheck %s
+; Test that statepoint intrinsic is marked with Throwable attribute and it is
+; not optimized into call
+
+declare i64 addrspace(1)* @gc_call()
+declare i32 @llvm.experimental.gc.statepoint.p0f_p1i64f(i64, i32, i64 addrspace(1)* ()*, i32, i32, ...)
+declare i32* @fake_personality_function()
+
+define i32 @test() gc "statepoint-example" personality i32* ()* @fake_personality_function {
+; CHECK-LABEL: test
+entry:
+  ; CHECK-LABEL: entry:
+  ; CHECK-NEXT: %sp = invoke i32 (i64, i32, i64 addrspace(1)* ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_p1i64f
+  %sp = invoke i32 (i64, i32, i64 addrspace(1)* ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_p1i64f(i64 0, i32 0, i64 addrspace(1)* ()* @gc_call, i32 0, i32 0, i32 0, i32 0)
+                to label %normal unwind label %exception
+
+exception:
+  %lpad = landingpad { i8*, i32 }
+          cleanup
+  ret i32 0
+
+normal:
+  ret i32 1
+}
diff --git a/test/Transforms/SimplifyCFG/trap-debugloc.ll b/test/Transforms/SimplifyCFG/trap-debugloc.ll
index 65c7e4153fe..fedf6b172d7 100644
--- a/test/Transforms/SimplifyCFG/trap-debugloc.ll
+++ b/test/Transforms/SimplifyCFG/trap-debugloc.ll
@@ -11,14 +11,14 @@ define void @foo() nounwind ssp {
 !llvm.module.flags = !{!10}
 !llvm.dbg.sp = !{!0}
 
-!0 = !MDSubprogram(name: "foo", line: 3, isLocal: false, isDefinition: true, virtualIndex: 6, isOptimized: false, file: !8, scope: !1, type: !3, function: void ()* @foo)
-!1 = !MDFile(filename: "foo.c", directory: "/private/tmp")
-!2 = !MDCompileUnit(language: DW_LANG_C99, producer: "Apple clang version 3.0 (tags/Apple/clang-206.1) (based on LLVM 3.0svn)", isOptimized: true, emissionKind: 0, file: !8, enums: !{}, retainedTypes: !{}, subprograms: !9)
-!3 = !MDSubroutineType(types: !4)
+!0 = !DISubprogram(name: "foo", line: 3, isLocal: false, isDefinition: true, virtualIndex: 6, isOptimized: false, file: !8, scope: !1, type: !3, function: void ()* @foo)
+!1 = !DIFile(filename: "foo.c", directory: "/private/tmp")
+!2 = !DICompileUnit(language: DW_LANG_C99, producer: "Apple clang version 3.0 (tags/Apple/clang-206.1) (based on LLVM 3.0svn)", isOptimized: true, emissionKind: 0, file: !8, enums: !{}, retainedTypes: !{}, subprograms: !9)
+!3 = !DISubroutineType(types: !4)
 !4 = !{null}
-!5 = !MDLocation(line: 4, column: 2, scope: !6)
-!6 = distinct !MDLexicalBlock(line: 3, column: 12, file: !8, scope: !0)
-!7 = !MDLocation(line: 5, column: 1, scope: !6)
-!8 = !MDFile(filename: "foo.c", directory: "/private/tmp")
+!5 = !DILocation(line: 4, column: 2, scope: !6)
+!6 = distinct !DILexicalBlock(line: 3, column: 12, file: !8, scope: !0)
+!7 = !DILocation(line: 5, column: 1, scope: !6)
+!8 = !DIFile(filename: "foo.c", directory: "/private/tmp")
 !9 = !{!0}
 !10 = !{i32 1, !"Debug Info Version", i32 3}
diff --git a/test/Transforms/Sink/convergent.ll b/test/Transforms/Sink/convergent.ll
new file mode 100644
index 00000000000..49207dbc992
--- /dev/null
+++ b/test/Transforms/Sink/convergent.ll
@@ -0,0 +1,24 @@
+; RUN: opt -sink -S < %s | FileCheck %s
+
+; Verify that IR sinking does not move convergent operations to
+; blocks that are not control equivalent.
+
+; CHECK: define i32 @foo
+; CHECK: entry
+; CHECK-NEXT: call i32 @bar
+; CHECK-NEXT: br i1 %arg
+
+define i32 @foo(i1 %arg) {
+entry:
+  %c = call i32 @bar() readonly convergent
+  br i1 %arg, label %then, label %end
+
+then:
+  ret i32 %c
+
+end:
+  ret i32 0
+}
+
+declare i32 @bar() readonly convergent
+
diff --git a/test/Transforms/SpeculativeExecution/spec.ll b/test/Transforms/SpeculativeExecution/spec.ll
new file mode 100644
index 00000000000..b0953c85dd4
--- /dev/null
+++ b/test/Transforms/SpeculativeExecution/spec.ll
@@ -0,0 +1,195 @@
+; RUN: opt < %s -S -speculative-execution \
+; RUN:   -spec-exec-max-speculation-cost 4 -spec-exec-max-not-hoisted 3 \
+; RUN:   | FileCheck %s
+
+target datalayout = "e-i64:64-v16:16-v32:32-n16:32:64"
+
+; Hoist in if-then pattern.
+define void @ifThen() {
+; CHECK-LABEL: @ifThen(
+; CHECK: %x = add i32 2, 3
+; CHECK: br i1 true
+  br i1 true, label %a, label %b
+; CHECK: a:
+a:
+  %x = add i32 2, 3
+; CHECK: br label
+  br label %b
+; CHECK: b:
+b:
+; CHECK: ret void
+  ret void
+}
+
+; Hoist in if-else pattern.
+define void @ifElse() {
+; CHECK-LABEL: @ifElse(
+; CHECK: %x = add i32 2, 3
+; CHECK: br i1 true
+  br i1 true, label %b, label %a
+; CHECK: a:
+a:
+  %x = add i32 2, 3
+; CHECK: br label
+  br label %b
+; CHECK: b:
+b:
+; CHECK: ret void
+  ret void
+}
+
+; Hoist in if-then-else pattern if it is equivalent to if-then.
+define void @ifElseThenAsIfThen() {
+; CHECK-LABEL: @ifElseThenAsIfThen(
+; CHECK: %x = add i32 2, 3
+; CHECK: br
+  br i1 true, label %a, label %b
+; CHECK: a:
+a:
+  %x = add i32 2, 3
+; CHECK: br label
+  br label %c
+; CHECK: b:
+b:
+  br label %c
+; CHECK: c
+c:
+  ret void
+}
+
+; Hoist in if-then-else pattern if it is equivalent to if-else.
+define void @ifElseThenAsIfElse() {
+; CHECK-LABEL: @ifElseThenAsIfElse(
+; CHECK: %x = add i32 2, 3
+; CHECK: br
+  br i1 true, label %b, label %a
+; CHECK: a:
+a:
+  %x = add i32 2, 3
+; CHECK: br label
+  br label %c
+; CHECK: b:
+b:
+  br label %c
+; CHECK: c
+c:
+  ret void
+}
+
+; Do not hoist if-then-else pattern if it is not equivalent to if-then
+; or if-else.
+define void @ifElseThen() {
+; CHECK-LABEL: @ifElseThen(
+; CHECK: br
+  br i1 true, label %a, label %b
+; CHECK: a:
+a:
+; CHECK: %x = add
+  %x = add i32 2, 3
+; CHECK: br label
+  br label %c
+; CHECK: b:
+b:
+; CHECK: %y = add
+  %y = add i32 2, 3
+  br label %c
+; CHECK: c
+c:
+  ret void
+}
+
+; Do not hoist loads and do not hoist an instruction past a definition of
+; an operand.
+define void @doNotHoistPastDef() {
+; CHECK-LABEL: @doNotHoistPastDef(
+  br i1 true, label %b, label %a
+; CHECK-NOT: load
+; CHECK-NOT: add
+; CHECK: a:
+a:
+; CHECK: %def = load
+  %def = load i32, i32* null
+; CHECK: %use = add
+  %use = add i32 %def, 0
+  br label %b
+; CHECK: b:
+b:
+  ret void
+}
+
+; Case with nothing to speculate.
+define void @nothingToSpeculate() {
+; CHECK-LABEL: @nothingToSpeculate(
+  br i1 true, label %b, label %a
+; CHECK: a:
+a:
+; CHECK: %def = load
+  %def = load i32, i32* null
+  br label %b
+; CHECK: b:
+b:
+  ret void
+}
+
+; Still hoist if an operand is defined before the block or is itself hoisted.
+define void @hoistIfNotPastDef() {
+; CHECK-LABEL: @hoistIfNotPastDef(
+; CHECK: %x = load
+  %x = load i32, i32* null
+; CHECK: %y = add i32 %x, 1
+; CHECK: %z = add i32 %y, 1
+; CHECK: br
+  br i1 true, label %b, label %a
+; CHECK: a:
+a:
+  %y = add i32 %x, 1
+  %z = add i32 %y, 1
+  br label %b
+; CHECK: b:
+b:
+  ret void
+}
+
+; Do not hoist if the speculation cost is too high.
+define void @costTooHigh() {
+; CHECK-LABEL: @costTooHigh(
+; CHECK: br
+  br i1 true, label %b, label %a
+; CHECK: a:
+a:
+; CHECK: %r1 = add
+  %r1 = add i32 1, 1
+; CHECK: %r2 = add
+  %r2 = add i32 1, 1
+; CHECK: %r3 = add
+  %r3 = add i32 1, 1
+; CHECK: %r4 = add
+  %r4 = add i32 1, 1
+; CHECK: %r5 = add
+  %r5 = add i32 1, 1
+  br label %b
+; CHECK: b:
+b:
+  ret void
+}
+
+; Do not hoist if too many instructions are left behind.
+define void @tooMuchLeftBehind() {
+; CHECK-LABEL: @tooMuchLeftBehind(
+; CHECK: br
+  br i1 true, label %b, label %a
+; CHECK: a:
+a:
+; CHECK: %x = load
+  %x = load i32, i32* null
+; CHECK: %r1 = add
+  %r1 = add i32 %x, 1
+; CHECK: %r2 = add
+  %r2 = add i32 %x, 1
+; CHECK: %r3 = add
+  %r3 = add i32 %x, 1
+  br label %b
+; CHECK: b:
+b:
+  ret void
+}
diff --git a/test/Transforms/StraightLineStrengthReduce/AMDGPU/lit.local.cfg b/test/Transforms/StraightLineStrengthReduce/AMDGPU/lit.local.cfg
new file mode 100644
index 00000000000..2a665f06be7
--- /dev/null
+++ b/test/Transforms/StraightLineStrengthReduce/AMDGPU/lit.local.cfg
@@ -0,0 +1,2 @@
+if not 'AMDGPU' in config.root.targets:
+    config.unsupported = True
diff --git a/test/Transforms/StraightLineStrengthReduce/AMDGPU/pr23975.ll b/test/Transforms/StraightLineStrengthReduce/AMDGPU/pr23975.ll
new file mode 100644
index 00000000000..f587a93bf1e
--- /dev/null
+++ b/test/Transforms/StraightLineStrengthReduce/AMDGPU/pr23975.ll
@@ -0,0 +1,20 @@
+; RUN: opt < %s -slsr -S | FileCheck %s
+
+target datalayout = "e-p:32:32-p1:64:64-p2:64:64-p3:32:32-p4:64:64-p5:32:32-p24:64:64-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-v2048:2048-n32:64"
+target triple = "amdgcn--"
+
+%struct.Matrix4x4 = type { [4 x [4 x float]] }
+
+; Function Attrs: nounwind
+define fastcc void @Accelerator_Intersect(%struct.Matrix4x4 addrspace(1)* nocapture readonly %leafTransformations) #0 {
+; CHECK-LABEL:  @Accelerator_Intersect(
+entry:
+  %tmp = sext i32 undef to i64
+  %arrayidx114 = getelementptr inbounds %struct.Matrix4x4, %struct.Matrix4x4 addrspace(1)* %leafTransformations, i64 %tmp
+  %tmp1 = getelementptr %struct.Matrix4x4, %struct.Matrix4x4 addrspace(1)* %leafTransformations, i64 %tmp, i32 0, i64 0, i64 0
+; CHECK: %tmp1 = getelementptr %struct.Matrix4x4, %struct.Matrix4x4 addrspace(1)* %leafTransformations, i64 %tmp, i32 0, i64 0, i64 0
+  %tmp2 = load <4 x float>, <4 x float> addrspace(1)* undef, align 4
+  ret void
+}
+
+attributes #0 = { nounwind "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "no-realign-stack" "stack-protector-buffer-size"="8" "target-cpu"="tahiti" "unsafe-fp-math"="false" "use-soft-float"="false" }
diff --git a/test/Transforms/StraightLineStrengthReduce/AMDGPU/reassociate-geps-and-slsr-addrspace.ll b/test/Transforms/StraightLineStrengthReduce/AMDGPU/reassociate-geps-and-slsr-addrspace.ll
new file mode 100644
index 00000000000..278250a9c80
--- /dev/null
+++ b/test/Transforms/StraightLineStrengthReduce/AMDGPU/reassociate-geps-and-slsr-addrspace.ll
@@ -0,0 +1,107 @@
+; RUN: opt -S -mtriple=amdgcn-- -separate-const-offset-from-gep -slsr -gvn < %s | FileCheck %s
+
+target datalayout = "e-p:32:32-p1:64:64-p2:64:64-p3:32:32-p4:64:64-p5:32:32-p24:64:64-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-v2048:2048-n32:64"
+
+
+; CHECK-LABEL: @slsr_after_reassociate_global_geps_mubuf_max_offset(
+; CHECK: [[b1:%[0-9]+]] = getelementptr float, float addrspace(1)* %arr, i64 [[bump:%[0-9]+]]
+; CHECK: [[b2:%[0-9]+]] = getelementptr float, float addrspace(1)* [[b1]], i64 [[bump]]
+define void @slsr_after_reassociate_global_geps_mubuf_max_offset(float addrspace(1)* %out, float addrspace(1)* noalias %arr, i32 %i) {
+bb:
+  %i2 = shl nsw i32 %i, 1
+  %j1 = add nsw i32 %i, 1023
+  %tmp = sext i32 %j1 to i64
+  %p1 = getelementptr inbounds float, float addrspace(1)* %arr, i64 %tmp
+  %tmp3 = bitcast float addrspace(1)* %p1 to i32 addrspace(1)*
+  %v11 = load i32, i32 addrspace(1)* %tmp3, align 4
+  %tmp4 = bitcast float addrspace(1)* %out to i32 addrspace(1)*
+  store i32 %v11, i32 addrspace(1)* %tmp4, align 4
+
+  %j2 = add nsw i32 %i2, 1023
+  %tmp5 = sext i32 %j2 to i64
+  %p2 = getelementptr inbounds float, float addrspace(1)* %arr, i64 %tmp5
+  %tmp6 = bitcast float addrspace(1)* %p2 to i32 addrspace(1)*
+  %v22 = load i32, i32 addrspace(1)* %tmp6, align 4
+  %tmp7 = bitcast float addrspace(1)* %out to i32 addrspace(1)*
+  store i32 %v22, i32 addrspace(1)* %tmp7, align 4
+
+  ret void
+}
+
+; CHECK-LABEL: @slsr_after_reassociate_global_geps_over_mubuf_max_offset(
+; CHECK: %j1 = add nsw i32 %i, 1024
+; CHECK: %tmp = sext i32 %j1 to i64
+; CHECK: getelementptr inbounds float, float addrspace(1)* %arr, i64 %tmp
+; CHECK: getelementptr inbounds float, float addrspace(1)* %arr, i64 %tmp5
+define void @slsr_after_reassociate_global_geps_over_mubuf_max_offset(float addrspace(1)* %out, float addrspace(1)* noalias %arr, i32 %i) {
+bb:
+  %i2 = shl nsw i32 %i, 1
+  %j1 = add nsw i32 %i, 1024
+  %tmp = sext i32 %j1 to i64
+  %p1 = getelementptr inbounds float, float addrspace(1)* %arr, i64 %tmp
+  %tmp3 = bitcast float addrspace(1)* %p1 to i32 addrspace(1)*
+  %v11 = load i32, i32 addrspace(1)* %tmp3, align 4
+  %tmp4 = bitcast float addrspace(1)* %out to i32 addrspace(1)*
+  store i32 %v11, i32 addrspace(1)* %tmp4, align 4
+
+  %j2 = add nsw i32 %i2, 1024
+  %tmp5 = sext i32 %j2 to i64
+  %p2 = getelementptr inbounds float, float addrspace(1)* %arr, i64 %tmp5
+  %tmp6 = bitcast float addrspace(1)* %p2 to i32 addrspace(1)*
+  %v22 = load i32, i32 addrspace(1)* %tmp6, align 4
+  %tmp7 = bitcast float addrspace(1)* %out to i32 addrspace(1)*
+  store i32 %v22, i32 addrspace(1)* %tmp7, align 4
+
+  ret void
+}
+
+; CHECK-LABEL: @slsr_after_reassociate_lds_geps_ds_max_offset(
+; CHECK: [[B1:%[0-9]+]] = getelementptr float, float addrspace(3)* %arr, i32 %i
+; CHECK: getelementptr float, float addrspace(3)* [[B1]], i32 16383
+
+; CHECK: [[B2:%[0-9]+]] = getelementptr float, float addrspace(3)* [[B1]], i32 %i
+; CHECK: getelementptr float, float addrspace(3)* [[B2]], i32 16383
+define void @slsr_after_reassociate_lds_geps_ds_max_offset(float addrspace(1)* %out, float addrspace(3)* noalias %arr, i32 %i) {
+bb:
+  %i2 = shl nsw i32 %i, 1
+  %j1 = add nsw i32 %i, 16383
+  %p1 = getelementptr inbounds float, float addrspace(3)* %arr, i32 %j1
+  %tmp3 = bitcast float addrspace(3)* %p1 to i32 addrspace(3)*
+  %v11 = load i32, i32 addrspace(3)* %tmp3, align 4
+  %tmp4 = bitcast float addrspace(1)* %out to i32 addrspace(1)*
+  store i32 %v11, i32 addrspace(1)* %tmp4, align 4
+
+  %j2 = add nsw i32 %i2, 16383
+  %p2 = getelementptr inbounds float, float addrspace(3)* %arr, i32 %j2
+  %tmp6 = bitcast float addrspace(3)* %p2 to i32 addrspace(3)*
+  %v22 = load i32, i32 addrspace(3)* %tmp6, align 4
+  %tmp7 = bitcast float addrspace(1)* %out to i32 addrspace(1)*
+  store i32 %v22, i32 addrspace(1)* %tmp7, align 4
+
+  ret void
+}
+
+; CHECK-LABEL: @slsr_after_reassociate_lds_geps_over_ds_max_offset(
+; CHECK: %j1 = add nsw i32 %i, 16384
+; CHECK: getelementptr inbounds float, float addrspace(3)* %arr, i32 %j1
+; CHECK: %j2 = add i32 %j1, %i
+; CHECK: getelementptr inbounds float, float addrspace(3)* %arr, i32 %j2
+define void @slsr_after_reassociate_lds_geps_over_ds_max_offset(float addrspace(1)* %out, float addrspace(3)* noalias %arr, i32 %i) {
+bb:
+  %i2 = shl nsw i32 %i, 1
+  %j1 = add nsw i32 %i, 16384
+  %p1 = getelementptr inbounds float, float addrspace(3)* %arr, i32 %j1
+  %tmp3 = bitcast float addrspace(3)* %p1 to i32 addrspace(3)*
+  %v11 = load i32, i32 addrspace(3)* %tmp3, align 4
+  %tmp4 = bitcast float addrspace(1)* %out to i32 addrspace(1)*
+  store i32 %v11, i32 addrspace(1)* %tmp4, align 4
+
+  %j2 = add nsw i32 %i2, 16384
+  %p2 = getelementptr inbounds float, float addrspace(3)* %arr, i32 %j2
+  %tmp6 = bitcast float addrspace(3)* %p2 to i32 addrspace(3)*
+  %v22 = load i32, i32 addrspace(3)* %tmp6, align 4
+  %tmp7 = bitcast float addrspace(1)* %out to i32 addrspace(1)*
+  store i32 %v22, i32 addrspace(1)* %tmp7, align 4
+
+  ret void
+}
diff --git a/test/Transforms/StraightLineStrengthReduce/NVPTX/lit.local.cfg b/test/Transforms/StraightLineStrengthReduce/NVPTX/lit.local.cfg
new file mode 100644
index 00000000000..2cb98eb371b
--- /dev/null
+++ b/test/Transforms/StraightLineStrengthReduce/NVPTX/lit.local.cfg
@@ -0,0 +1,2 @@
+if not 'NVPTX' in config.root.targets:
+    config.unsupported = True
diff --git a/test/Transforms/StraightLineStrengthReduce/NVPTX/reassociate-geps-and-slsr.ll b/test/Transforms/StraightLineStrengthReduce/NVPTX/reassociate-geps-and-slsr.ll
new file mode 100644
index 00000000000..03c0356eb5b
--- /dev/null
+++ b/test/Transforms/StraightLineStrengthReduce/NVPTX/reassociate-geps-and-slsr.ll
@@ -0,0 +1,74 @@
+; RUN: opt < %s -separate-const-offset-from-gep -slsr -gvn -S | FileCheck %s
+; RUN: llc < %s -march=nvptx64 -mcpu=sm_35 | FileCheck %s --check-prefix=PTX
+
+target datalayout = "e-i64:64-v16:16-v32:32-n16:32:64"
+target triple = "nvptx64-unknown-unknown"
+
+; arr[i + 5]
+; arr[i * 2 + 5]
+; arr[i * 3 + 5]
+; arr[i * 4 + 5]
+;
+;   => reassociate-geps
+;
+; *(&arr[i] + 5)
+; *(&arr[i * 2] + 5)
+; *(&arr[i * 3] + 5)
+; *(&arr[i * 4] + 5)
+;
+;   => slsr
+;
+; p1 = &arr[i]
+; *(p1 + 5)
+; p2 = p1 + i
+; *(p2 + 5)
+; p3 = p2 + i
+; *(p3 + 5)
+; p4 = p3 + i
+; *(p4 + 5)
+define void @slsr_after_reassociate_geps(float* %arr, i32 %i) {
+; CHECK-LABEL: @slsr_after_reassociate_geps(
+; PTX-LABEL: .visible .func slsr_after_reassociate_geps(
+; PTX: ld.param.u64 [[arr:%rd[0-9]+]], [slsr_after_reassociate_geps_param_0];
+; PTX: ld.param.u32 [[i:%r[0-9]+]], [slsr_after_reassociate_geps_param_1];
+  %i2 = shl nsw i32 %i, 1
+  %i3 = mul nsw i32 %i, 3
+  %i4 = shl nsw i32 %i, 2
+
+  %j1 = add nsw i32 %i, 5
+  %p1 = getelementptr inbounds float, float* %arr, i32 %j1
+; CHECK: [[b1:%[0-9]+]] = getelementptr float, float* %arr, i64 [[bump:%[0-9]+]]
+; PTX: mul.wide.s32 [[i4:%rd[0-9]+]], [[i]], 4;
+; PTX: add.s64 [[base1:%rd[0-9]+]], [[arr]], [[i4]];
+  %v1 = load float, float* %p1, align 4
+; PTX: ld.f32 {{%f[0-9]+}}, {{\[}}[[base1]]+20];
+  call void @foo(float %v1)
+
+  %j2 = add nsw i32 %i2, 5
+  %p2 = getelementptr inbounds float, float* %arr, i32 %j2
+; CHECK: [[b2:%[0-9]+]] = getelementptr float, float* [[b1]], i64 [[bump]]
+; PTX: add.s64 [[base2:%rd[0-9]+]], [[base1]], [[i4]];
+  %v2 = load float, float* %p2, align 4
+; PTX: ld.f32 {{%f[0-9]+}}, {{\[}}[[base2]]+20];
+  call void @foo(float %v2)
+
+  %j3 = add nsw i32 %i3, 5
+  %p3 = getelementptr inbounds float, float* %arr, i32 %j3
+; CHECK: [[b3:%[0-9]+]] = getelementptr float, float* [[b2]], i64 [[bump]]
+; PTX: add.s64 [[base3:%rd[0-9]+]], [[base2]], [[i4]];
+  %v3 = load float, float* %p3, align 4
+; PTX: ld.f32 {{%f[0-9]+}}, {{\[}}[[base3]]+20];
+  call void @foo(float %v3)
+
+  %j4 = add nsw i32 %i4, 5
+  %p4 = getelementptr inbounds float, float* %arr, i32 %j4
+; CHECK: [[b4:%[0-9]+]] = getelementptr float, float* [[b3]], i64 [[bump]]
+; PTX: add.s64 [[base4:%rd[0-9]+]], [[base3]], [[i4]];
+  %v4 = load float, float* %p4, align 4
+; PTX: ld.f32 {{%f[0-9]+}}, {{\[}}[[base4]]+20];
+  call void @foo(float %v4)
+
+  ret void
+}
+
+declare void @foo(float)
diff --git a/test/Transforms/StraightLineStrengthReduce/NVPTX/speculative-slsr.ll b/test/Transforms/StraightLineStrengthReduce/NVPTX/speculative-slsr.ll
new file mode 100644
index 00000000000..cb73565b152
--- /dev/null
+++ b/test/Transforms/StraightLineStrengthReduce/NVPTX/speculative-slsr.ll
@@ -0,0 +1,71 @@
+; RUN: llc < %s -march=nvptx64 -mcpu=sm_35 | FileCheck %s
+
+target datalayout = "e-i64:64-v16:16-v32:32-n16:32:64"
+target triple = "nvptx64-nvidia-cuda"
+
+; CUDA code
+; __global__ void foo(int b, int s) {
+;   #pragma unroll
+;   for (int i = 0; i < 4; ++i) {
+;     if (cond(i))
+;       use((b + i) * s);
+;   }
+; }
+define void @foo(i32 %b, i32 %s) {
+; CHECK-LABEL: .visible .entry foo(
+entry:
+; CHECK: ld.param.u32 [[s:%r[0-9]+]], [foo_param_1];
+; CHECK: ld.param.u32 [[b:%r[0-9]+]], [foo_param_0];
+  %call = tail call zeroext i1 @cond(i32 0)
+  br i1 %call, label %if.then, label %for.inc
+
+if.then:                                          ; preds = %entry
+  %mul = mul nsw i32 %b, %s
+; CHECK: mul.lo.s32 [[a0:%r[0-9]+]], [[b]], [[s]]
+  tail call void @use(i32 %mul)
+  br label %for.inc
+
+for.inc:                                          ; preds = %entry, %if.then
+  %call.1 = tail call zeroext i1 @cond(i32 1)
+  br i1 %call.1, label %if.then.1, label %for.inc.1
+
+if.then.1:                                        ; preds = %for.inc
+  %add.1 = add nsw i32 %b, 1
+  %mul.1 = mul nsw i32 %add.1, %s
+; CHECK: add.s32 [[a1:%r[0-9]+]], [[a0]], [[s]]
+  tail call void @use(i32 %mul.1)
+  br label %for.inc.1
+
+for.inc.1:                                        ; preds = %if.then.1, %for.inc
+  %call.2 = tail call zeroext i1 @cond(i32 2)
+  br i1 %call.2, label %if.then.2, label %for.inc.2
+
+if.then.2:                                        ; preds = %for.inc.1
+  %add.2 = add nsw i32 %b, 2
+  %mul.2 = mul nsw i32 %add.2, %s
+; CHECK: add.s32 [[a2:%r[0-9]+]], [[a1]], [[s]]
+  tail call void @use(i32 %mul.2)
+  br label %for.inc.2
+
+for.inc.2:                                        ; preds = %if.then.2, %for.inc.1
+  %call.3 = tail call zeroext i1 @cond(i32 3)
+  br i1 %call.3, label %if.then.3, label %for.inc.3
+
+if.then.3:                                        ; preds = %for.inc.2
+  %add.3 = add nsw i32 %b, 3
+  %mul.3 = mul nsw i32 %add.3, %s
+; CHECK: add.s32 [[a3:%r[0-9]+]], [[a2]], [[s]]
+  tail call void @use(i32 %mul.3)
+  br label %for.inc.3
+
+for.inc.3:                                        ; preds = %if.then.3, %for.inc.2
+  ret void
+}
+
+declare zeroext i1 @cond(i32)
+
+declare void @use(i32)
+
+!nvvm.annotations = !{!0}
+
+!0 = !{void (i32, i32)* @foo, !"kernel", i32 1}
diff --git a/test/Transforms/StraightLineStrengthReduce/X86/no-slsr.ll b/test/Transforms/StraightLineStrengthReduce/X86/no-slsr.ll
index e2201ce23f6..f11cbc5897a 100644
--- a/test/Transforms/StraightLineStrengthReduce/X86/no-slsr.ll
+++ b/test/Transforms/StraightLineStrengthReduce/X86/no-slsr.ll
@@ -1,4 +1,4 @@
-; RUN: opt < %s -slsr -gvn -dce -S | FileCheck %s
+; RUN: opt < %s -slsr -gvn -S | FileCheck %s
 
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
diff --git a/test/Transforms/StraightLineStrengthReduce/slsr-add.ll b/test/Transforms/StraightLineStrengthReduce/slsr-add.ll
index 4c79be0577e..e25ddc2888a 100644
--- a/test/Transforms/StraightLineStrengthReduce/slsr-add.ll
+++ b/test/Transforms/StraightLineStrengthReduce/slsr-add.ll
@@ -1,4 +1,4 @@
-; RUN: opt < %s -slsr -gvn -dce -S | FileCheck %s
+; RUN: opt < %s -slsr -gvn -S | FileCheck %s
 
 target datalayout = "e-i64:64-v16:16-v32:32-n16:32:64"
 
diff --git a/test/Transforms/StraightLineStrengthReduce/slsr-gep.ll b/test/Transforms/StraightLineStrengthReduce/slsr-gep.ll
index 3944739aeaa..bd92780a036 100644
--- a/test/Transforms/StraightLineStrengthReduce/slsr-gep.ll
+++ b/test/Transforms/StraightLineStrengthReduce/slsr-gep.ll
@@ -1,4 +1,4 @@
-; RUN: opt < %s -slsr -gvn -dce -S | FileCheck %s
+; RUN: opt < %s -slsr -gvn -S | FileCheck %s
 
 target datalayout = "e-i64:64-v16:16-v32:32-n16:32:64"
 
diff --git a/test/Transforms/StraightLineStrengthReduce/slsr-mul.ll b/test/Transforms/StraightLineStrengthReduce/slsr-mul.ll
index 1c7333dab9f..56b1d1fc5cd 100644
--- a/test/Transforms/StraightLineStrengthReduce/slsr-mul.ll
+++ b/test/Transforms/StraightLineStrengthReduce/slsr-mul.ll
@@ -1,4 +1,4 @@
-; RUN: opt < %s -slsr -gvn -dce -S | FileCheck %s
+; RUN: opt < %s -slsr -gvn -S | FileCheck %s
 
 target datalayout = "e-i64:64-v16:16-v32:32-n16:32:64"
 
@@ -23,6 +23,50 @@ define void @slsr1(i32 %b, i32 %s) {
   ret void
 }
 
+define void @non_canonicalized(i32 %b, i32 %s) {
+; CHECK-LABEL: @non_canonicalized(
+  ; foo(b * s);
+  %mul0 = mul i32 %b, %s
+; CHECK: mul i32
+; CHECK-NOT: mul i32
+  call void @foo(i32 %mul0)
+
+  ; foo((1 + b) * s);
+  %b1 = add i32 1, %b
+  %mul1 = mul i32 %b1, %s
+  call void @foo(i32 %mul1)
+
+  ; foo((2 + b) * s);
+  %b2 = add i32 2, %b
+  %mul2 = mul i32 %b2, %s
+  call void @foo(i32 %mul2)
+
+  ret void
+}
+
+define void @or(i32 %a, i32 %s) {
+  %b = shl i32 %a, 1
+; CHECK-LABEL: @or(
+  ; foo(b * s);
+  %mul0 = mul i32 %b, %s
+; CHECK: [[base:[^ ]+]] = mul i32
+  call void @foo(i32 %mul0)
+
+  ; foo((b | 1) * s);
+  %b1 = or i32 %b, 1
+  %mul1 = mul i32 %b1, %s
+; CHECK: add i32 [[base]], %s
+  call void @foo(i32 %mul1)
+
+  ; foo((b | 2) * s);
+  %b2 = or i32 %b, 2
+  %mul2 = mul i32 %b2, %s
+; CHECK: mul i32 %b2, %s
+  call void @foo(i32 %mul2)
+
+  ret void
+}
+
 ; foo(a * b)
 ; foo((a + 1) * b)
 ; foo(a * (b + 1))
diff --git a/test/Transforms/StripSymbols/2010-06-30-StripDebug.ll b/test/Transforms/StripSymbols/2010-06-30-StripDebug.ll
index 23c8a5810a6..3b1fd74b981 100644
--- a/test/Transforms/StripSymbols/2010-06-30-StripDebug.ll
+++ b/test/Transforms/StripSymbols/2010-06-30-StripDebug.ll
@@ -18,17 +18,17 @@ declare void @llvm.dbg.value(metadata, i64, metadata, metadata) nounwind readnon
 !llvm.dbg.lv.foo = !{!5}
 !llvm.dbg.gv = !{!8}
 
-!0 = !MDSubprogram(name: "foo", linkageName: "foo", line: 2, isLocal: false, isDefinition: true, virtualIndex: 6, isOptimized: true, file: !12, scope: !1, type: !3, function: void ()* @foo)
-!1 = !MDFile(filename: "b.c", directory: "/tmp")
-!2 = !MDCompileUnit(language: DW_LANG_C89, producer: "4.2.1 (Based on Apple Inc. build 5658) (LLVM build)", isOptimized: true, emissionKind: 0, file: !12, enums: !4, retainedTypes: !4)
-!3 = !MDSubroutineType(types: !4)
+!0 = !DISubprogram(name: "foo", linkageName: "foo", line: 2, isLocal: false, isDefinition: true, virtualIndex: 6, isOptimized: true, file: !12, scope: !1, type: !3, function: void ()* @foo)
+!1 = !DIFile(filename: "b.c", directory: "/tmp")
+!2 = !DICompileUnit(language: DW_LANG_C89, producer: "4.2.1 (Based on Apple Inc. build 5658) (LLVM build)", isOptimized: true, emissionKind: 0, file: !12, enums: !4, retainedTypes: !4)
+!3 = !DISubroutineType(types: !4)
 !4 = !{null}
-!5 = !MDLocalVariable(tag: DW_TAG_auto_variable, name: "y", line: 3, scope: !6, file: !1, type: !7)
-!6 = distinct !MDLexicalBlock(line: 2, column: 0, file: !12, scope: !0)
-!7 = !MDBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
-!8 = !MDGlobalVariable(name: "x", line: 1, isLocal: false, isDefinition: true, scope: !1, file: !1, type: !7, variable: i32* @x)
+!5 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "y", line: 3, scope: !6, file: !1, type: !7)
+!6 = distinct !DILexicalBlock(line: 2, column: 0, file: !12, scope: !0)
+!7 = !DIBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
+!8 = !DIGlobalVariable(name: "x", line: 1, isLocal: false, isDefinition: true, scope: !1, file: !1, type: !7, variable: i32* @x)
 !9 = !{i32 0}
-!10 = !MDLocation(line: 3, scope: !6)
-!11 = !MDLocation(line: 4, scope: !6)
-!12 = !MDFile(filename: "b.c", directory: "/tmp")
+!10 = !DILocation(line: 3, scope: !6)
+!11 = !DILocation(line: 4, scope: !6)
+!12 = !DIFile(filename: "b.c", directory: "/tmp")
 !13 = !{i32 1, !"Debug Info Version", i32 3}
diff --git a/test/Transforms/StripSymbols/2010-08-25-crash.ll b/test/Transforms/StripSymbols/2010-08-25-crash.ll
index 7fd3bcc4f51..f77ed11d912 100644
--- a/test/Transforms/StripSymbols/2010-08-25-crash.ll
+++ b/test/Transforms/StripSymbols/2010-08-25-crash.ll
@@ -7,17 +7,17 @@ entry:
 !llvm.dbg.cu = !{!2}
 !llvm.module.flags = !{!14}
 
-!0 = !MDSubprogram(name: "foo", linkageName: "foo", line: 3, isLocal: false, isDefinition: true, virtualIndex: 6, isOptimized: false, file: !10, scope: !1, type: !3, function: i32 ()* @foo)
-!1 = !MDFile(filename: "/tmp/a.c", directory: "/Volumes/Lalgate/clean/D.CW")
-!2 = !MDCompileUnit(language: DW_LANG_C99, producer: "clang version 2.8 (trunk 112062)", isOptimized: true, emissionKind: 1, file: !10, enums: !11, retainedTypes: !11, subprograms: !12, globals: !13)
-!3 = !MDSubroutineType(types: !4)
+!0 = !DISubprogram(name: "foo", linkageName: "foo", line: 3, isLocal: false, isDefinition: true, virtualIndex: 6, isOptimized: false, file: !10, scope: !1, type: !3, function: i32 ()* @foo)
+!1 = !DIFile(filename: "/tmp/a.c", directory: "/Volumes/Lalgate/clean/D.CW")
+!2 = !DICompileUnit(language: DW_LANG_C99, producer: "clang version 2.8 (trunk 112062)", isOptimized: true, emissionKind: 1, file: !10, enums: !11, retainedTypes: !11, subprograms: !12, globals: !13)
+!3 = !DISubroutineType(types: !4)
 !4 = !{!5}
-!5 = !MDBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
-!6 = !MDGlobalVariable(name: "i", linkageName: "i", line: 2, isLocal: true, isDefinition: true, scope: !1, file: !1, type: !7)
-!7 = !MDDerivedType(tag: DW_TAG_const_type, file: !10, scope: !1, baseType: !5)
-!8 = !MDLocation(line: 3, column: 13, scope: !9)
-!9 = distinct !MDLexicalBlock(line: 3, column: 11, file: !10, scope: !0)
-!10 = !MDFile(filename: "/tmp/a.c", directory: "/Volumes/Lalgate/clean/D.CW")
+!5 = !DIBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
+!6 = !DIGlobalVariable(name: "i", linkageName: "i", line: 2, isLocal: true, isDefinition: true, scope: !1, file: !1, type: !7)
+!7 = !DIDerivedType(tag: DW_TAG_const_type, file: !10, scope: !1, baseType: !5)
+!8 = !DILocation(line: 3, column: 13, scope: !9)
+!9 = distinct !DILexicalBlock(line: 3, column: 11, file: !10, scope: !0)
+!10 = !DIFile(filename: "/tmp/a.c", directory: "/Volumes/Lalgate/clean/D.CW")
 !11 = !{}
 !12 = !{!0}
 !13 = !{!6}
diff --git a/test/Transforms/StripSymbols/strip-dead-debug-info.ll b/test/Transforms/StripSymbols/strip-dead-debug-info.ll
index d9511b8cb4a..08a1d507949 100644
--- a/test/Transforms/StripSymbols/strip-dead-debug-info.ll
+++ b/test/Transforms/StripSymbols/strip-dead-debug-info.ll
@@ -18,7 +18,7 @@ entry:
 ; Function Attrs: nounwind readonly ssp
 define i32 @foo(i32 %i) #2 {
 entry:
-  tail call void @llvm.dbg.value(metadata i32 %i, i64 0, metadata !15, metadata !MDExpression()), !dbg !20
+  tail call void @llvm.dbg.value(metadata i32 %i, i64 0, metadata !15, metadata !DIExpression()), !dbg !20
   %.0 = load i32, i32* @xyz, align 4
   ret i32 %.0, !dbg !21
 }
@@ -30,29 +30,29 @@ attributes #2 = { nounwind readonly ssp }
 !llvm.dbg.cu = !{!0}
 !llvm.module.flags = !{!25}
 
-!0 = !MDCompileUnit(language: DW_LANG_C89, producer: "4.2.1 (Based on Apple Inc. build 5658) (LLVM build)", isOptimized: true, emissionKind: 1, file: !1, enums: !{}, retainedTypes: !{}, subprograms: !23, globals: !24)
-!1 = !MDFile(filename: "g.c", directory: "/tmp/")
+!0 = !DICompileUnit(language: DW_LANG_C89, producer: "4.2.1 (Based on Apple Inc. build 5658) (LLVM build)", isOptimized: true, emissionKind: 1, file: !1, enums: !{}, retainedTypes: !{}, subprograms: !23, globals: !24)
+!1 = !DIFile(filename: "g.c", directory: "/tmp/")
 !2 = !{null}
-!3 = !MDSubprogram(name: "bar", line: 5, isLocal: true, isDefinition: true, virtualIndex: 6, isOptimized: true, file: !1, scope: null, type: !4)
-!4 = !MDSubroutineType(types: !2)
-!5 = !MDFile(filename: "g.c", directory: "/tmp/")
-!6 = !MDSubprogram(name: "fn", linkageName: "fn", line: 6, isLocal: false, isDefinition: true, virtualIndex: 6, isOptimized: true, file: !1, scope: null, type: !7, function: i32 ()* @fn)
-!7 = !MDSubroutineType(types: !8)
+!3 = !DISubprogram(name: "bar", line: 5, isLocal: true, isDefinition: true, virtualIndex: 6, isOptimized: true, file: !1, scope: null, type: !4)
+!4 = !DISubroutineType(types: !2)
+!5 = !DIFile(filename: "g.c", directory: "/tmp/")
+!6 = !DISubprogram(name: "fn", linkageName: "fn", line: 6, isLocal: false, isDefinition: true, virtualIndex: 6, isOptimized: true, file: !1, scope: null, type: !7, function: i32 ()* @fn)
+!7 = !DISubroutineType(types: !8)
 !8 = !{!9}
-!9 = !MDBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
-!10 = !MDSubprogram(name: "foo", linkageName: "foo", line: 7, isLocal: false, isDefinition: true, virtualIndex: 6, isOptimized: true, file: !1, scope: null, type: !11, function: i32 (i32)* @foo)
-!11 = !MDSubroutineType(types: !12)
+!9 = !DIBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
+!10 = !DISubprogram(name: "foo", linkageName: "foo", line: 7, isLocal: false, isDefinition: true, virtualIndex: 6, isOptimized: true, file: !1, scope: null, type: !11, function: i32 (i32)* @foo)
+!11 = !DISubroutineType(types: !12)
 !12 = !{!9, !9}
-!13 = !MDLocalVariable(tag: DW_TAG_auto_variable, name: "bb", line: 5, scope: !14, file: !5, type: !9)
-!14 = distinct !MDLexicalBlock(line: 5, column: 0, file: !1, scope: !3)
-!15 = !MDLocalVariable(tag: DW_TAG_arg_variable, name: "i", line: 7, arg: 0, scope: !10, file: !5, type: !9)
-!16 = !MDGlobalVariable(name: "abcd", line: 2, isLocal: true, isDefinition: true, scope: !5, file: !5, type: !9)
-!17 = !MDGlobalVariable(name: "xyz", line: 3, isLocal: false, isDefinition: true, scope: !5, file: !5, type: !9, variable: i32* @xyz)
-!18 = !MDLocation(line: 6, scope: !19)
-!19 = distinct !MDLexicalBlock(line: 6, column: 0, file: !1, scope: !6)
-!20 = !MDLocation(line: 7, scope: !10)
-!21 = !MDLocation(line: 10, scope: !22)
-!22 = distinct !MDLexicalBlock(line: 7, column: 0, file: !1, scope: !10)
+!13 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "bb", line: 5, scope: !14, file: !5, type: !9)
+!14 = distinct !DILexicalBlock(line: 5, column: 0, file: !1, scope: !3)
+!15 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "i", line: 7, arg: 1, scope: !10, file: !5, type: !9)
+!16 = !DIGlobalVariable(name: "abcd", line: 2, isLocal: true, isDefinition: true, scope: !5, file: !5, type: !9)
+!17 = !DIGlobalVariable(name: "xyz", line: 3, isLocal: false, isDefinition: true, scope: !5, file: !5, type: !9, variable: i32* @xyz)
+!18 = !DILocation(line: 6, scope: !19)
+!19 = distinct !DILexicalBlock(line: 6, column: 0, file: !1, scope: !6)
+!20 = !DILocation(line: 7, scope: !10)
+!21 = !DILocation(line: 10, scope: !22)
+!22 = distinct !DILexicalBlock(line: 7, column: 0, file: !1, scope: !10)
 !23 = !{!3, !6, !10}
 !24 = !{!16, !17}
 !25 = !{i32 1, !"Debug Info Version", i32 3}
diff --git a/test/Transforms/Util/lowerswitch.ll b/test/Transforms/Util/lowerswitch.ll
index 1eddb43c1a0..6e21f916c03 100644
--- a/test/Transforms/Util/lowerswitch.ll
+++ b/test/Transforms/Util/lowerswitch.ll
@@ -3,7 +3,7 @@
 ; Test that we don't crash and have a different basic block for each incoming edge.
 define void @test0() {
 ; CHECK-LABEL: @test0
-; CHECK: %merge = phi i64 [ 1, %BB3 ], [ 0, %NewDefault ], [ 0, %NodeBlock5 ], [ 0, %LeafBlock1 ]
+; CHECK: %merge = phi i64 [ 1, %BB3 ], [ 0, %NewDefault ], [ 0, %NodeBlock.5 ], [ 0, %LeafBlock.1 ]
 BB1:
   switch i32 undef, label %BB2 [
     i32 3, label %BB2
@@ -43,9 +43,9 @@ bb2:
 
 bb3:
 ; CHECK-LABEL: bb3
-; CHECK: %tmp = phi i32 [ 1, %NodeBlock ], [ 0, %bb2 ], [ 1, %LeafBlock3 ]
+; CHECK: %tmp = phi i32 [ 1, %NodeBlock ], [ 0, %bb2 ], [ 1, %LeafBlock.3 ]
   %tmp = phi i32 [ 1, %bb1 ], [ 0, %bb2 ], [ 1, %bb1 ], [ 1, %bb1 ]
-; CHECK-NEXT: %tmp2 = phi i32 [ 2, %NodeBlock ], [ 5, %bb2 ], [ 2, %LeafBlock3 ]
+; CHECK-NEXT: %tmp2 = phi i32 [ 2, %NodeBlock ], [ 5, %bb2 ], [ 2, %LeafBlock.3 ]
   %tmp2 = phi i32 [ 2, %bb1 ], [ 2, %bb1 ], [ 5, %bb2 ], [ 2, %bb1 ]
   br label %exit
 
diff --git a/test/Verifier/comdat-decl1.ll b/test/Verifier/comdat-decl1.ll
new file mode 100644
index 00000000000..aee56b63627
--- /dev/null
+++ b/test/Verifier/comdat-decl1.ll
@@ -0,0 +1,5 @@
+; RUN: not llvm-as %s -o /dev/null 2>&1 | FileCheck %s
+
+$v = comdat any
+@v = available_externally global i32 0, comdat
+; CHECK: Declaration may not be in a Comdat!
diff --git a/test/Verifier/comdat-decl2.ll b/test/Verifier/comdat-decl2.ll
new file mode 100644
index 00000000000..fcd3d5d0eee
--- /dev/null
+++ b/test/Verifier/comdat-decl2.ll
@@ -0,0 +1,5 @@
+; RUN: not llvm-as %s -o /dev/null 2>&1 | FileCheck %s
+
+$v = comdat any
+@v = external global i32, comdat
+; CHECK: Declaration may not be in a Comdat!
diff --git a/test/Verifier/dbg-typerefs.ll b/test/Verifier/dbg-typerefs.ll
index 55ee03e206b..dc50fce0715 100644
--- a/test/Verifier/dbg-typerefs.ll
+++ b/test/Verifier/dbg-typerefs.ll
@@ -7,26 +7,26 @@
 ; Make a bunch of type references.  Note that !4 references !"0.bad" (instead
 ; of !"4.bad") to test error ordering.
 !typerefs = !{!1, !2, !3, !4}
-!1 = !MDDerivedType(tag: DW_TAG_pointer_type, size: 32, align: 32, baseType: !"1.good")
-!2 = !MDDerivedType(tag: DW_TAG_pointer_type, size: 32, align: 32, baseType: !"2.bad")
-!3 = !MDDerivedType(tag: DW_TAG_pointer_type, size: 32, align: 32, baseType: !"3.good")
-!4 = !MDDerivedType(tag: DW_TAG_pointer_type, size: 32, align: 32, baseType: !"0.bad")
+!1 = !DIDerivedType(tag: DW_TAG_pointer_type, size: 32, align: 32, baseType: !"1.good")
+!2 = !DIDerivedType(tag: DW_TAG_pointer_type, size: 32, align: 32, baseType: !"2.bad")
+!3 = !DIDerivedType(tag: DW_TAG_pointer_type, size: 32, align: 32, baseType: !"3.good")
+!4 = !DIDerivedType(tag: DW_TAG_pointer_type, size: 32, align: 32, baseType: !"0.bad")
 
 ; Add a minimal compile unit to resolve some of the type references.
 !llvm.dbg.cu = !{!5}
-!5 = !MDCompileUnit(file: !6, language: DW_LANG_C99, retainedTypes: !7)
-!6 = !MDFile(filename: "file.c", directory: "/path/to/dir")
+!5 = !DICompileUnit(file: !6, language: DW_LANG_C99, retainedTypes: !7)
+!6 = !DIFile(filename: "file.c", directory: "/path/to/dir")
 !7 = !{!8, !9}
-!8 = !MDCompositeType(tag: DW_TAG_structure_type, identifier: "1.good")
-!9 = !MDCompositeType(tag: DW_TAG_structure_type, identifier: "3.good")
+!8 = !DICompositeType(tag: DW_TAG_structure_type, identifier: "1.good")
+!9 = !DICompositeType(tag: DW_TAG_structure_type, identifier: "3.good")
 
 ; CHECK:      assembly parsed, but does not verify
 ; CHECK-NEXT: unresolved type ref
 ; CHECK-NEXT: !"0.bad"
-; CHECK-NEXT: !MDDerivedType(tag: DW_TAG_pointer_type
+; CHECK-NEXT: !DIDerivedType(tag: DW_TAG_pointer_type
 ; CHECK-SAME:                baseType: !"0.bad"
 ; CHECK-NEXT: unresolved type ref
 ; CHECK-NEXT: !"2.bad"
-; CHECK-NEXT: !MDDerivedType(tag: DW_TAG_pointer_type
+; CHECK-NEXT: !DIDerivedType(tag: DW_TAG_pointer_type
 ; CHECK-SAME:                baseType: !"2.bad"
 ; CHECK-NOT:  unresolved
diff --git a/test/Verifier/dbg.ll b/test/Verifier/dbg.ll
index 3cff1e05157..395806b1299 100644
--- a/test/Verifier/dbg.ll
+++ b/test/Verifier/dbg.ll
@@ -2,9 +2,9 @@
 
 define void @foo() {
 entry:
-  br label %exit, !dbg !MDLocation(scope: !MDSubprogram(), inlinedAt: !{})
+  br label %exit, !dbg !DILocation(scope: !DISubprogram(), inlinedAt: !{})
 ; CHECK: inlined-at should be a location
-; CHECK-NEXT: !{{[0-9]+}} = !MDLocation(line: 0, scope: !{{[0-9]+}}, inlinedAt: ![[IA:[0-9]+]])
+; CHECK-NEXT: !{{[0-9]+}} = !DILocation(line: 0, scope: !{{[0-9]+}}, inlinedAt: ![[IA:[0-9]+]])
 ; CHECK-NEXT: ![[IA]] = !{}
 
 exit:
diff --git a/test/Verifier/dominates.ll b/test/Verifier/dominates.ll
index 17e2c339996..c5424bbede3 100644
--- a/test/Verifier/dominates.ll
+++ b/test/Verifier/dominates.ll
@@ -10,14 +10,14 @@ define i32 @f1(i32 %x) {
 }
 
 declare i32 @g()
-define void @f2(i32 %x) {
+define void @f2(i32 %x) personality i32 ()* @g {
 bb0:
   %y1 = invoke i32 @g() to label %bb1 unwind label %bb2
 bb1:
   ret void
 bb2:
   %y2 = phi i32 [%y1, %bb0]
-  %y3 = landingpad i32 personality i32 ()* @g
+  %y3 = landingpad i32
           cleanup
   ret void
 ; CHECK: Instruction does not dominate all uses!
@@ -26,13 +26,13 @@ bb2:
 ; CHECK-NEXT:  %y2 = phi i32 [ %y1, %bb0 ]
 }
 
-define void @f3(i32 %x) {
+define void @f3(i32 %x) personality i32 ()* @g {
 bb0:
   %y1 = invoke i32 @g() to label %bb1 unwind label %bb2
 bb1:
   ret void
 bb2:
-  %y2 = landingpad i32 personality i32 ()* @g
+  %y2 = landingpad i32
           cleanup
   br label %bb3
 bb3:
diff --git a/test/Verifier/frameescape.ll b/test/Verifier/frameescape.ll
index 1fb9387eb2f..074098b990d 100644
--- a/test/Verifier/frameescape.ll
+++ b/test/Verifier/frameescape.ll
@@ -1,69 +1,69 @@
 ; RUN: not llvm-as %s -o /dev/null 2>&1 | FileCheck %s
 
-declare void @llvm.frameescape(...)
-declare i8* @llvm.framerecover(i8*, i8*, i32)
+declare void @llvm.localescape(...)
+declare i8* @llvm.localrecover(i8*, i8*, i32)
 
 define internal void @f() {
   %a = alloca i8
-  call void (...) @llvm.frameescape(i8* %a)
-  call void (...) @llvm.frameescape(i8* %a)
+  call void (...) @llvm.localescape(i8* %a)
+  call void (...) @llvm.localescape(i8* %a)
   ret void
 }
-; CHECK: multiple calls to llvm.frameescape in one function
+; CHECK: multiple calls to llvm.localescape in one function
 
 define internal void @g() {
 entry:
   %a = alloca i8
   br label %not_entry
 not_entry:
-  call void (...) @llvm.frameescape(i8* %a)
+  call void (...) @llvm.localescape(i8* %a)
   ret void
 }
-; CHECK: llvm.frameescape used outside of entry block
+; CHECK: llvm.localescape used outside of entry block
 
 define internal void @h() {
-  call i8* @llvm.framerecover(i8* null, i8* null, i32 0)
+  call i8* @llvm.localrecover(i8* null, i8* null, i32 0)
   ret void
 }
-; CHECK: llvm.framerecover first argument must be function defined in this module
+; CHECK: llvm.localrecover first argument must be function defined in this module
 
 @global = constant i8 0
 
 declare void @declaration()
 
 define internal void @i() {
-  call i8* @llvm.framerecover(i8* @global, i8* null, i32 0)
+  call i8* @llvm.localrecover(i8* @global, i8* null, i32 0)
   ret void
 }
-; CHECK: llvm.framerecover first argument must be function defined in this module
+; CHECK: llvm.localrecover first argument must be function defined in this module
 
 define internal void @j() {
-  call i8* @llvm.framerecover(i8* bitcast(void()* @declaration to i8*), i8* null, i32 0)
+  call i8* @llvm.localrecover(i8* bitcast(void()* @declaration to i8*), i8* null, i32 0)
   ret void
 }
-; CHECK: llvm.framerecover first argument must be function defined in this module
+; CHECK: llvm.localrecover first argument must be function defined in this module
 
 define internal void @k(i32 %n) {
-  call i8* @llvm.framerecover(i8* bitcast(void()* @f to i8*), i8* null, i32 %n)
+  call i8* @llvm.localrecover(i8* bitcast(void()* @f to i8*), i8* null, i32 %n)
   ret void
 }
-; CHECK: idx argument of llvm.framerecover must be a constant int
+; CHECK: idx argument of llvm.localrecover must be a constant int
 
 define internal void @l(i8* %b) {
   %a = alloca i8
-  call void (...) @llvm.frameescape(i8* %a, i8* %b)
+  call void (...) @llvm.localescape(i8* %a, i8* %b)
   ret void
 }
-; CHECK: llvm.frameescape only accepts static allocas
+; CHECK: llvm.localescape only accepts static allocas
 
 define internal void @m() {
   %a = alloca i8
-  call void (...) @llvm.frameescape(i8* %a)
+  call void (...) @llvm.localescape(i8* %a)
   ret void
 }
 
 define internal void @n(i8* %fp) {
-  call i8* @llvm.framerecover(i8* bitcast(void ()* @m to i8*), i8* %fp, i32 1)
+  call i8* @llvm.localrecover(i8* bitcast(void ()* @m to i8*), i8* %fp, i32 1)
   ret void
 }
-; CHECK: all indices passed to llvm.framerecover must be less than the number of arguments passed ot llvm.frameescape in the parent function
+; CHECK: all indices passed to llvm.localrecover must be less than the number of arguments passed ot llvm.localescape in the parent function
diff --git a/test/Verifier/function-metadata-bad.ll b/test/Verifier/function-metadata-bad.ll
new file mode 100644
index 00000000000..9e7ba225408
--- /dev/null
+++ b/test/Verifier/function-metadata-bad.ll
@@ -0,0 +1,35 @@
+; RUN: not llvm-as < %s -o /dev/null 2>&1 | FileCheck %s
+
+define i32 @bad1() !prof !0 {
+  ret i32 0
+}
+
+!0 = !{i32 123, i32 3}
+; CHECK: assembly parsed, but does not verify as correct!
+; CHECK-NEXT: expected string with name of the !prof annotation
+; CHECK-NEXT: !0 = !{i32 123, i32 3}
+
+define i32 @bad2() !prof !1 {
+  ret i32 0
+}
+
+!1 = !{!"function_entry_count"}
+; CHECK-NEXT: !prof annotations should have exactly 2 operands
+; CHECK-NEXT: !1 = !{!"function_entry_count"}
+
+
+define i32 @bad3() !prof !2 {
+  ret i32 0
+}
+
+!2 = !{!"some_other_count", i64 200}
+; CHECK-NEXT: first operand should be 'function_entry_count'
+; CHECK-NEXT: !2 = !{!"some_other_count", i64 200}
+
+define i32 @bad4() !prof !3 {
+  ret i32 0
+}
+
+!3 = !{!"function_entry_count", !"string"}
+; CHECK-NEXT: expected integer argument to function_entry_count
+; CHECK-NEXT: !3 = !{!"function_entry_count", !"string"}
diff --git a/test/Verifier/function-metadata-good.ll b/test/Verifier/function-metadata-good.ll
new file mode 100644
index 00000000000..3d73a1c0694
--- /dev/null
+++ b/test/Verifier/function-metadata-good.ll
@@ -0,0 +1,8 @@
+; RUN: llvm-as < %s -o /dev/null 2>&1
+
+; Function foo() is called 2,304 times at runtime.
+define i32 @foo() !prof !0 {
+  ret i32 0
+}
+
+!0 = !{!"function_entry_count", i32 2304}
diff --git a/test/Verifier/gc_relocate_addrspace.ll b/test/Verifier/gc_relocate_addrspace.ll
new file mode 100644
index 00000000000..ddc1b230a8c
--- /dev/null
+++ b/test/Verifier/gc_relocate_addrspace.ll
@@ -0,0 +1,23 @@
+; RUN: not llvm-as -disable-output <%s 2>&1 | FileCheck %s
+; This is to verify that gc_relocate must return a pointer with the same
+; address space with the relocated value.
+
+; CHECK:       gc.relocate: relocating a pointer shouldn't change its address space
+; CHECK-NEXT:  %obj.relocated = call coldcc i8* @llvm.experimental.gc.relocate.p0i8(i32 %safepoint_token, i32 7, i32 7) ;
+
+declare void @foo()
+
+; Function Attrs: nounwind
+declare i32 @llvm.experimental.gc.statepoint.p0f_isVoidf(i64, i32, void ()*, i32, i32, ...) #0
+
+define void @test1(i64 addrspace(1)* %obj) gc "statepoint-example" {
+entry:
+  %safepoint_token = call i32 (i64, i32, void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 0, i32 0, void ()* @foo, i32 0, i32 0, i32 0, i32 0, i64 addrspace(1)* %obj)
+  %obj.relocated = call coldcc i8* @llvm.experimental.gc.relocate.p0i8(i32 %safepoint_token, i32 7, i32 7) ; (%obj, %obj)
+  ret void
+}
+
+; Function Attrs: nounwind
+declare i8* @llvm.experimental.gc.relocate.p0i8(i32, i32, i32) #0
+
+attributes #0 = { nounwind }
diff --git a/test/Verifier/gc_relocate_operand.ll b/test/Verifier/gc_relocate_operand.ll
new file mode 100644
index 00000000000..c28b8d87036
--- /dev/null
+++ b/test/Verifier/gc_relocate_operand.ll
@@ -0,0 +1,21 @@
+; RUN: not llvm-as -disable-output <%s 2>&1 | FileCheck %s
+; This is to verify that the relocated value by gc_relocate must be a pointer type.
+
+; CHECK: gc.relocate: relocated value must be a gc pointer
+
+declare void @foo()
+
+declare i32 @llvm.experimental.gc.statepoint.p0f_isVoidf(i64, i32, void ()*, i32, i32, ...)
+
+define void @test1(i64 %obj) gc "statepoint-example" {
+entry:
+  %safepoint_token = call i32 (i64, i32, void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 0, i32 0, void ()* @foo, i32 0, i32 0, i32 0, i32 0, i64 %obj)
+  %obj.relocated = call coldcc i8 addrspace(1)* @llvm.experimental.gc.relocate.p1i8(i32 %safepoint_token, i32 7, i32 7) ; (%obj, %obj)
+  ret void
+}
+
+; Function Attrs: nounwind
+declare i8 addrspace(1)* @llvm.experimental.gc.relocate.p1i8(i32, i32, i32) #0
+
+attributes #0 = { nounwind }
+
diff --git a/test/Verifier/gc_relocate_return.ll b/test/Verifier/gc_relocate_return.ll
new file mode 100644
index 00000000000..3957d4c0cec
--- /dev/null
+++ b/test/Verifier/gc_relocate_return.ll
@@ -0,0 +1,22 @@
+; RUN: not llvm-as -disable-output <%s 2>&1 | FileCheck %s
+; This is to verify that gc_relocate must return a pointer type, which is defined
+; in intrinsics.td.
+
+; CHECK: Intrinsic has incorrect return type!
+
+declare void @foo()
+
+declare i32 @llvm.experimental.gc.statepoint.p0f_isVoidf(i64, i32, void ()*, i32, i32, ...)
+
+define void @test1(<2 x i32 addrspace(1)*> addrspace(1)* %obj) gc "statepoint-example" {
+entry:
+  %safepoint_token = call i32 (i64, i32, void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 0, i32 0, void ()* @foo, i32 0, i32 0, i32 0, i32 0, <2 x i32 addrspace(1)*> addrspace(1)* %obj)
+  %obj.relocated = call coldcc i8 @llvm.experimental.gc.relocate.i8(i32 %safepoint_token, i32 7, i32 7) ; (%obj, %obj)
+  ret void
+}
+
+; Function Attrs: nounwind
+declare i8 @llvm.experimental.gc.relocate.i8(i32, i32, i32) #0
+
+attributes #0 = { nounwind }
+
diff --git a/test/Verifier/invalid-statepoint.ll b/test/Verifier/invalid-statepoint.ll
index d3a5bb80fbc..6c3525a16df 100644
--- a/test/Verifier/invalid-statepoint.ll
+++ b/test/Verifier/invalid-statepoint.ll
@@ -5,16 +5,16 @@
 declare zeroext i1 @return0i1()
 
 ; Function Attrs: nounwind
-declare i32 @llvm.experimental.gc.statepoint.p0f0i1f(i1 ()*, i32, i32, ...) #0
+declare i32 @llvm.experimental.gc.statepoint.p0f0i1f(i64, i32, i1 ()*, i32, i32, ...) #0
 
 ; Function Attrs: nounwind
 declare i32 addrspace(1)* @llvm.experimental.gc.relocate.p1i32(i32, i32, i32) #0
 
 define i32 addrspace(1)* @0(i32 addrspace(1)* %dparam) {
   %a00 = load i32, i32 addrspace(1)* %dparam
-  %to0 = call i32 (i1 ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f0i1f(i1 ()* @return0i1, i32 9, i32 0, i2 0, i32 addrspace(1)* %dparam)
-  %relocate = call i32 addrspace(1)* @llvm.experimental.gc.relocate.p1i32(i32 %to0, i32 0, i32 4)
+  %to0 = call i32 (i64, i32, i1 ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f0i1f(i64 0, i32 0, i1 ()* @return0i1, i32 9, i32 0, i2 0, i32 addrspace(1)* %dparam)
+  %relocate = call i32 addrspace(1)* @llvm.experimental.gc.relocate.p1i32(i32 %to0, i32 2, i32 6)
   ret i32 addrspace(1)* %relocate
 }
 
-attributes #0 = { nounwind }
+attributes #0 = { nounwind }
\ No newline at end of file
diff --git a/test/Verifier/invalid-statepoint2.ll b/test/Verifier/invalid-statepoint2.ll
index e3dd9225e91..c8b453c31e6 100644
--- a/test/Verifier/invalid-statepoint2.ll
+++ b/test/Verifier/invalid-statepoint2.ll
@@ -5,15 +5,14 @@
 declare void @use(...)
 declare i8 addrspace(1)* @llvm.experimental.gc.relocate.p1i8(i32, i32, i32)
 declare i64 addrspace(1)* @llvm.experimental.gc.relocate.p1i64(i32, i32, i32)
-declare i32 @llvm.experimental.gc.statepoint.p0f_isVoidf(void ()*, i32, i32, ...)
+declare i32 @llvm.experimental.gc.statepoint.p0f_isVoidf(i64, i32, void ()*, i32, i32, ...)
 declare i32 @"personality_function"()
 
 ;; Basic usage
 define i64 addrspace(1)* @test1(i8 addrspace(1)* %arg, i32 %val) gc "statepoint-example" {
 entry:
   %cast = bitcast i8 addrspace(1)* %arg to i64 addrspace(1)*
-  %safepoint_token = call i32 (void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(void ()* undef, i32 0, i32 0, i32 %val, i32 0, i32 0, i32 0, i32 10, i32 0, i8 addrspace(1)* %arg, i64 addrspace(1)* %cast, i8 addrspace(1)* %arg, i8 addrspace(1)* %arg)
-  %reloc = call i64 addrspace(1)* @llvm.experimental.gc.relocate.p1i64(i32 %safepoint_token, i32 9, i32 10)
+  %safepoint_token = call i32 (i64, i32, void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 0, i32 0, void ()* undef, i32 0, i32 0, i32 0, i32 %val, i32 0, i32 0, i32 0, i32 10, i32 0, i8 addrspace(1)* %arg, i64 addrspace(1)* %cast, i8 addrspace(1)* %arg, i8 addrspace(1)* %arg)
+  %reloc = call i64 addrspace(1)* @llvm.experimental.gc.relocate.p1i64(i32 %safepoint_token, i32 12, i32 13)
   ret i64 addrspace(1)* %reloc
 }
-
diff --git a/test/Verifier/invoke.ll b/test/Verifier/invoke.ll
index e80cfcf830b..8fa9923c0cb 100644
--- a/test/Verifier/invoke.ll
+++ b/test/Verifier/invoke.ll
@@ -2,7 +2,7 @@
 
 ; PR1042
 define i32 @foo() {
-; CHECK: The unwind destination does not have a landingpad instruction
+; CHECK: The unwind destination does not have an exception handling instruction
 	%A = invoke i32 @foo( )
 			to label %L unwind label %L		; <i32> [#uses=1]
 L:		; preds = %0, %0
@@ -18,7 +18,7 @@ L1:		; preds = %0
 L2:		; preds = %0
 	br label %L
 L:		; preds = %L2, %L1, %L1
-; CHECK: The unwind destination does not have a landingpad instruction
+; CHECK: The unwind destination does not have an exception handling instruction
 	ret i32 %A
 }
 
@@ -29,7 +29,7 @@ declare void @llvm.trap()
 declare i8 @llvm.expect.i8(i8,i8)
 declare i32 @fn(i8 (i8, i8)*)
 
-define void @f1() {
+define void @f1() personality i8* bitcast (i32 (...)* @__gxx_personality_v0 to i8*) {
 entry:
 ; OK
   invoke void @llvm.donothing()
@@ -39,12 +39,12 @@ conta:
   ret void
 
 contb:
-  %0 = landingpad { i8*, i32 } personality i8* bitcast (i32 (...)* @__gxx_personality_v0 to i8*)
+  %0 = landingpad { i8*, i32 }
           filter [0 x i8*] zeroinitializer
   ret void
 }
 
-define i8 @f2() {
+define i8 @f2() personality i8* bitcast (i32 (...)* @__gxx_personality_v0 to i8*) {
 entry:
 ; CHECK: Cannot invoke an intrinsinc other than donothing or patchpoint
   invoke void @llvm.trap()
@@ -54,7 +54,7 @@ cont:
   ret i8 3
 
 lpad:
-  %0 = landingpad { i8*, i32 } personality i8* bitcast (i32 (...)* @__gxx_personality_v0 to i8*)
+  %0 = landingpad { i8*, i32 }
           filter [0 x i8*] zeroinitializer
   ret i8 2
 }
@@ -66,14 +66,14 @@ entry:
   ret i32 %call
 }
 
-define void @f4() {
+define void @f4() personality i8* bitcast (i32 (...)* @__gxx_personality_v0 to i8*) {
 entry:
   invoke void @llvm.donothing()
   to label %cont unwind label %cont
 
 cont:
 ; CHECK: Block containing LandingPadInst must be jumped to only by the unwind edge of an invoke.
-  %0 = landingpad { i8*, i32 } personality i8* bitcast (i32 (...)* @__gxx_personality_v0 to i8*)
+  %0 = landingpad { i8*, i32 }
           filter [0 x i8*] zeroinitializer
   ret void
 }
diff --git a/test/Verifier/llvm.dbg.declare-address.ll b/test/Verifier/llvm.dbg.declare-address.ll
index 0c23d6ea397..ba132ad5346 100644
--- a/test/Verifier/llvm.dbg.declare-address.ll
+++ b/test/Verifier/llvm.dbg.declare-address.ll
@@ -6,7 +6,7 @@
 define void @foo(i32 %a) {
 entry:
   %s = alloca i32
-  call void @llvm.dbg.declare(metadata !"", metadata !MDLocalVariable(tag: DW_TAG_arg_variable, scope: !1), metadata !MDExpression()), !dbg !MDLocation(scope: !1)
+  call void @llvm.dbg.declare(metadata !"", metadata !DILocalVariable(tag: DW_TAG_arg_variable, scope: !1), metadata !DIExpression()), !dbg !DILocation(scope: !1)
   ret void
 }
 
@@ -14,4 +14,4 @@ declare void @llvm.dbg.declare(metadata, metadata, metadata)
 
 !llvm.module.flags = !{!0}
 !0 = !{i32 2, !"Debug Info Version", i32 3}
-!1 = !MDSubprogram()
+!1 = !DISubprogram()
diff --git a/test/Verifier/llvm.dbg.declare-expression.ll b/test/Verifier/llvm.dbg.declare-expression.ll
index 0491fdc95aa..4c1bd65230a 100644
--- a/test/Verifier/llvm.dbg.declare-expression.ll
+++ b/test/Verifier/llvm.dbg.declare-expression.ll
@@ -6,7 +6,7 @@
 define void @foo(i32 %a) {
 entry:
   %s = alloca i32
-  call void @llvm.dbg.declare(metadata i32* %s, metadata !MDLocalVariable(tag: DW_TAG_arg_variable, scope: !1), metadata !"")
+  call void @llvm.dbg.declare(metadata i32* %s, metadata !DILocalVariable(tag: DW_TAG_arg_variable, scope: !1), metadata !"")
   ret void
 }
 
@@ -14,4 +14,4 @@ declare void @llvm.dbg.declare(metadata, metadata, metadata)
 
 !llvm.module.flags = !{!0}
 !0 = !{i32 2, !"Debug Info Version", i32 3}
-!1 = !MDSubprogram()
+!1 = !DISubprogram()
diff --git a/test/Verifier/llvm.dbg.declare-variable.ll b/test/Verifier/llvm.dbg.declare-variable.ll
index 763f4637366..deef50fd717 100644
--- a/test/Verifier/llvm.dbg.declare-variable.ll
+++ b/test/Verifier/llvm.dbg.declare-variable.ll
@@ -6,7 +6,7 @@
 define void @foo(i32 %a) {
 entry:
   %s = alloca i32
-  call void @llvm.dbg.declare(metadata i32* %s, metadata !"", metadata !MDExpression()), !dbg !MDLocation(scope: !1)
+  call void @llvm.dbg.declare(metadata i32* %s, metadata !"", metadata !DIExpression()), !dbg !DILocation(scope: !1)
   ret void
 }
 
@@ -14,4 +14,4 @@ declare void @llvm.dbg.declare(metadata, metadata, metadata)
 
 !llvm.module.flags = !{!0}
 !0 = !{i32 2, !"Debug Info Version", i32 3}
-!1 = !MDSubprogram()
+!1 = !DISubprogram()
diff --git a/test/Verifier/llvm.dbg.intrinsic-dbg-attachment.ll b/test/Verifier/llvm.dbg.intrinsic-dbg-attachment.ll
index 3615960df63..717794b4274 100644
--- a/test/Verifier/llvm.dbg.intrinsic-dbg-attachment.ll
+++ b/test/Verifier/llvm.dbg.intrinsic-dbg-attachment.ll
@@ -4,8 +4,8 @@ entry:
   call void @llvm.dbg.value(
       metadata i8* undef,
       i64 0,
-      metadata !MDLocalVariable(tag: DW_TAG_arg_variable, scope: !1),
-      metadata !MDExpression())
+      metadata !DILocalVariable(tag: DW_TAG_arg_variable, scope: !1),
+      metadata !DIExpression())
 ; CHECK-LABEL: llvm.dbg.value intrinsic requires a !dbg attachment
 ; CHECK-NEXT: call void @llvm.dbg.value({{.*}})
 ; CHECK-NEXT: label %entry
@@ -13,8 +13,8 @@ entry:
 
   call void @llvm.dbg.declare(
       metadata i8* undef,
-      metadata !MDLocalVariable(tag: DW_TAG_arg_variable, scope: !1),
-      metadata !MDExpression())
+      metadata !DILocalVariable(tag: DW_TAG_arg_variable, scope: !1),
+      metadata !DIExpression())
 ; CHECK-LABEL: llvm.dbg.declare intrinsic requires a !dbg attachment
 ; CHECK-NEXT: call void @llvm.dbg.declare({{.*}})
 ; CHECK-NEXT: label %entry
@@ -23,31 +23,31 @@ entry:
   call void @llvm.dbg.value(
       metadata i8* undef,
       i64 0,
-      metadata !MDLocalVariable(tag: DW_TAG_arg_variable, scope: !1),
-      metadata !MDExpression()),
-    !dbg !MDLocation(scope: !2)
+      metadata !DILocalVariable(tag: DW_TAG_arg_variable, scope: !1),
+      metadata !DIExpression()),
+    !dbg !DILocation(scope: !2)
 ; CHECK-LABEL: mismatched subprogram between llvm.dbg.value variable and !dbg attachment
 ; CHECK-NEXT: call void @llvm.dbg.value({{[^,]+, [^,]+}}, metadata ![[VAR:[0-9]+]], {{[^,]+}}), !dbg ![[LOC:[0-9]+]]
 ; CHECK-NEXT: label %entry
 ; CHECK-NEXT: void ()* @foo
-; CHECK-NEXT: ![[VAR]] = !MDLocalVariable({{.*}}scope: ![[VARSP:[0-9]+]]
-; CHECK-NEXT: ![[VARSP]] = !MDSubprogram(
-; CHECK-NEXT: ![[LOC]] = !MDLocation({{.*}}scope: ![[LOCSP:[0-9]+]]
-; CHECK-NEXT: ![[LOCSP]] = !MDSubprogram(
+; CHECK-NEXT: ![[VAR]] = !DILocalVariable({{.*}}scope: ![[VARSP:[0-9]+]]
+; CHECK-NEXT: ![[VARSP]] = !DISubprogram(
+; CHECK-NEXT: ![[LOC]] = !DILocation({{.*}}scope: ![[LOCSP:[0-9]+]]
+; CHECK-NEXT: ![[LOCSP]] = !DISubprogram(
 
   call void @llvm.dbg.declare(
       metadata i8* undef,
-      metadata !MDLocalVariable(tag: DW_TAG_arg_variable, scope: !1),
-      metadata !MDExpression()),
-    !dbg !MDLocation(scope: !2)
+      metadata !DILocalVariable(tag: DW_TAG_arg_variable, scope: !1),
+      metadata !DIExpression()),
+    !dbg !DILocation(scope: !2)
 ; CHECK-LABEL: mismatched subprogram between llvm.dbg.declare variable and !dbg attachment
 ; CHECK-NEXT: call void @llvm.dbg.declare({{[^,]+}}, metadata ![[VAR:[0-9]+]], {{.*[^,]+}}), !dbg ![[LOC:[0-9]+]]
 ; CHECK-NEXT: label %entry
 ; CHECK-NEXT: void ()* @foo
-; CHECK-NEXT: ![[VAR]] = !MDLocalVariable({{.*}}scope: ![[VARSP:[0-9]+]]
-; CHECK-NEXT: ![[VARSP]] = !MDSubprogram(
-; CHECK-NEXT: ![[LOC]] = !MDLocation({{.*}}scope: ![[LOCSP:[0-9]+]]
-; CHECK-NEXT: ![[LOCSP]] = !MDSubprogram(
+; CHECK-NEXT: ![[VAR]] = !DILocalVariable({{.*}}scope: ![[VARSP:[0-9]+]]
+; CHECK-NEXT: ![[VARSP]] = !DISubprogram(
+; CHECK-NEXT: ![[LOC]] = !DILocation({{.*}}scope: ![[LOCSP:[0-9]+]]
+; CHECK-NEXT: ![[LOCSP]] = !DISubprogram(
 
   ret void
 }
@@ -57,5 +57,5 @@ declare void @llvm.dbg.declare(metadata, metadata, metadata)
 
 !llvm.module.flags = !{!0}
 !0 = !{i32 2, !"Debug Info Version", i32 3}
-!1 = !MDSubprogram(name: "foo")
-!2 = !MDSubprogram(name: "bar")
+!1 = !DISubprogram(name: "foo")
+!2 = !DISubprogram(name: "bar")
diff --git a/test/Verifier/llvm.dbg.value-expression.ll b/test/Verifier/llvm.dbg.value-expression.ll
index 18350a144e6..78a7c50f233 100644
--- a/test/Verifier/llvm.dbg.value-expression.ll
+++ b/test/Verifier/llvm.dbg.value-expression.ll
@@ -6,7 +6,7 @@
 define void @foo(i32 %a) {
 entry:
   %s = alloca i32
-  call void @llvm.dbg.value(metadata i32* %s, i64 0, metadata !MDLocalVariable(tag: DW_TAG_arg_variable, scope: !1), metadata !""), !dbg !MDLocation(scope: !1)
+  call void @llvm.dbg.value(metadata i32* %s, i64 0, metadata !DILocalVariable(tag: DW_TAG_arg_variable, scope: !1), metadata !""), !dbg !DILocation(scope: !1)
   ret void
 }
 
@@ -14,4 +14,4 @@ declare void @llvm.dbg.value(metadata, i64, metadata, metadata)
 
 !llvm.module.flags = !{!0}
 !0 = !{i32 2, !"Debug Info Version", i32 3}
-!1 = !MDSubprogram()
+!1 = !DISubprogram()
diff --git a/test/Verifier/llvm.dbg.value-value.ll b/test/Verifier/llvm.dbg.value-value.ll
index 3d82be767b9..1acb5f8132f 100644
--- a/test/Verifier/llvm.dbg.value-value.ll
+++ b/test/Verifier/llvm.dbg.value-value.ll
@@ -6,7 +6,7 @@
 define void @foo(i32 %a) {
 entry:
   %s = alloca i32
-  call void @llvm.dbg.value(metadata !"", i64 0, metadata !MDLocalVariable(tag: DW_TAG_arg_variable, scope: !1), metadata !MDExpression()), !dbg !MDLocation(scope: !1)
+  call void @llvm.dbg.value(metadata !"", i64 0, metadata !DILocalVariable(tag: DW_TAG_arg_variable, scope: !1), metadata !DIExpression()), !dbg !DILocation(scope: !1)
   ret void
 }
 
@@ -14,4 +14,4 @@ declare void @llvm.dbg.value(metadata, i64, metadata, metadata)
 
 !llvm.module.flags = !{!0}
 !0 = !{i32 2, !"Debug Info Version", i32 3}
-!1 = !MDSubprogram()
+!1 = !DISubprogram()
diff --git a/test/Verifier/llvm.dbg.value-variable.ll b/test/Verifier/llvm.dbg.value-variable.ll
index ba6a687e0ab..66329f9fdaa 100644
--- a/test/Verifier/llvm.dbg.value-variable.ll
+++ b/test/Verifier/llvm.dbg.value-variable.ll
@@ -6,7 +6,7 @@
 define void @foo(i32 %a) {
 entry:
   %s = alloca i32
-  call void @llvm.dbg.value(metadata i32* %s, i64 0, metadata !"", metadata !MDExpression()), !dbg !MDLocation(scope: !1)
+  call void @llvm.dbg.value(metadata i32* %s, i64 0, metadata !"", metadata !DIExpression()), !dbg !DILocation(scope: !1)
   ret void
 }
 
@@ -14,4 +14,4 @@ declare void @llvm.dbg.value(metadata, i64, metadata, metadata)
 
 !llvm.module.flags = !{!0}
 !0 = !{i32 2, !"Debug Info Version", i32 3}
-!1 = !MDSubprogram()
+!1 = !DISubprogram()
diff --git a/test/Verifier/mdcompositetype-templateparams-tuple.ll b/test/Verifier/mdcompositetype-templateparams-tuple.ll
index a2e2c780d81..8de6fac50ea 100644
--- a/test/Verifier/mdcompositetype-templateparams-tuple.ll
+++ b/test/Verifier/mdcompositetype-templateparams-tuple.ll
@@ -1,11 +1,11 @@
 ; RUN: not llvm-as < %s -disable-output 2>&1 | FileCheck %s
 
 ; CHECK:      invalid template params
-; CHECK-NEXT: !2 = !MDCompositeType(
+; CHECK-NEXT: !2 = !DICompositeType(
 ; CHECK-SAME:                       templateParams: !1
-; CHECK-NEXT: !1 = !MDTemplateTypeParameter(
+; CHECK-NEXT: !1 = !DITemplateTypeParameter(
 
 !named = !{!0, !1, !2}
-!0 = !MDBasicType(name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
-!1 = !MDTemplateTypeParameter(name: "T", type: !0)
-!2 = !MDCompositeType(tag: DW_TAG_structure_type, name: "IntTy", size: 32, align: 32, templateParams: !1)
+!0 = !DIBasicType(name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
+!1 = !DITemplateTypeParameter(name: "T", type: !0)
+!2 = !DICompositeType(tag: DW_TAG_structure_type, name: "IntTy", size: 32, align: 32, templateParams: !1)
diff --git a/test/Verifier/mdcompositetype-templateparams.ll b/test/Verifier/mdcompositetype-templateparams.ll
index 72909e3e418..5d4bef2232f 100644
--- a/test/Verifier/mdcompositetype-templateparams.ll
+++ b/test/Verifier/mdcompositetype-templateparams.ll
@@ -1,12 +1,12 @@
 ; RUN: not llvm-as < %s -disable-output 2>&1 | FileCheck %s
 
 ; CHECK:      invalid template parameter
-; CHECK-NEXT: !2 = !MDCompositeType(
+; CHECK-NEXT: !2 = !DICompositeType(
 ; CHECK-SAME:                       templateParams: !1
 ; CHECK-NEXT: !1 = !{!0}
-; CHECK-NEXT: !0 = !MDBasicType(
+; CHECK-NEXT: !0 = !DIBasicType(
 
 !named = !{!0, !1, !2}
-!0 = !MDBasicType(name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
+!0 = !DIBasicType(name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
 !1 = !{!0}
-!2 = !MDCompositeType(tag: DW_TAG_structure_type, name: "IntTy", size: 32, align: 32, templateParams: !1)
+!2 = !DICompositeType(tag: DW_TAG_structure_type, name: "IntTy", size: 32, align: 32, templateParams: !1)
diff --git a/test/Verifier/range-2.ll b/test/Verifier/range-2.ll
index b7c9a6e65e6..6362cb757ed 100644
--- a/test/Verifier/range-2.ll
+++ b/test/Verifier/range-2.ll
@@ -47,7 +47,7 @@ entry:
 }
 
 ; We can annotate the range of the return value of an INVOKE.
-define void @invoke_all(i8* %x) {
+define void @invoke_all(i8* %x) personality i8* bitcast (i32 (...)* @__gxx_personality_v0 to i8*) {
 entry:
   %v1 = invoke i8 @f1(i8* %x) to label %cont unwind label %lpad, !range !0
   %v2 = invoke i8 @f2(i8* %x) to label %cont unwind label %lpad, !range !1
@@ -59,7 +59,7 @@ cont:
   ret void
 
 lpad:
-  %4 = landingpad { i8*, i32 } personality i8* bitcast (i32 (...)* @__gxx_personality_v0 to i8*)
+  %4 = landingpad { i8*, i32 }
           filter [0 x i8*] zeroinitializer
   ret void
 }
diff --git a/test/Verifier/statepoint.ll b/test/Verifier/statepoint.ll
index 61d8b7756cb..2807620f79e 100644
--- a/test/Verifier/statepoint.ll
+++ b/test/Verifier/statepoint.ll
@@ -3,18 +3,18 @@
 declare void @use(...)
 declare i8 addrspace(1)* @llvm.experimental.gc.relocate.p1i8(i32, i32, i32)
 declare i64 addrspace(1)* @llvm.experimental.gc.relocate.p1i64(i32, i32, i32)
-declare i32 @llvm.experimental.gc.statepoint.p0f_isVoidf(void ()*, i32, i32, ...)
+declare i32 @llvm.experimental.gc.statepoint.p0f_isVoidf(i64, i32, void ()*, i32, i32, ...)
 declare i32 @"personality_function"()
 
 ;; Basic usage
 define i64 addrspace(1)* @test1(i8 addrspace(1)* %arg) gc "statepoint-example" {
 entry:
   %cast = bitcast i8 addrspace(1)* %arg to i64 addrspace(1)*
-  %safepoint_token = call i32 (void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(void ()* undef, i32 0, i32 0, i32 5, i32 0, i32 0, i32 0, i32 10, i32 0, i8 addrspace(1)* %arg, i64 addrspace(1)* %cast, i8 addrspace(1)* %arg, i8 addrspace(1)* %arg)
-  %reloc = call i64 addrspace(1)* @llvm.experimental.gc.relocate.p1i64(i32 %safepoint_token, i32 9, i32 10)
+  %safepoint_token = call i32 (i64, i32, void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 0, i32 0, void ()* undef, i32 0, i32 0, i32 0, i32 5, i32 0, i32 0, i32 0, i32 10, i32 0, i8 addrspace(1)* %arg, i64 addrspace(1)* %cast, i8 addrspace(1)* %arg, i8 addrspace(1)* %arg)
+  %reloc = call i64 addrspace(1)* @llvm.experimental.gc.relocate.p1i64(i32 %safepoint_token, i32 12, i32 13)
   ;; It is perfectly legal to relocate the same value multiple times...
-  %reloc2 = call i64 addrspace(1)* @llvm.experimental.gc.relocate.p1i64(i32 %safepoint_token, i32 9, i32 10)
-  %reloc3 = call i8 addrspace(1)* @llvm.experimental.gc.relocate.p1i8(i32 %safepoint_token, i32 10, i32 9)
+  %reloc2 = call i64 addrspace(1)* @llvm.experimental.gc.relocate.p1i64(i32 %safepoint_token, i32 12, i32 13)
+  %reloc3 = call i8 addrspace(1)* @llvm.experimental.gc.relocate.p1i8(i32 %safepoint_token, i32 13, i32 12)
   ret i64 addrspace(1)* %reloc
 ; CHECK-LABEL: test1
 ; CHECK: statepoint
@@ -39,8 +39,8 @@ notequal:
   ret void
 
 equal:
-  %safepoint_token = call i32 (void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(void ()* undef, i32 0, i32 0, i32 5, i32 0, i32 0, i32 0, i32 10, i32 0, i8 addrspace(1)* %arg, i64 addrspace(1)* %cast, i8 addrspace(1)* %arg, i8 addrspace(1)* %arg)
-  %reloc = call i64 addrspace(1)* @llvm.experimental.gc.relocate.p1i64(i32 %safepoint_token, i32 9, i32 10)
+  %safepoint_token = call i32 (i64, i32, void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 0, i32 0, void ()* undef, i32 0, i32 0, i32 0, i32 5, i32 0, i32 0, i32 0, i32 10, i32 0, i8 addrspace(1)* %arg, i64 addrspace(1)* %cast, i8 addrspace(1)* %arg, i8 addrspace(1)* %arg)
+  %reloc = call i64 addrspace(1)* @llvm.experimental.gc.relocate.p1i64(i32 %safepoint_token, i32 12, i32 13)
   call void undef(i64 addrspace(1)* %reloc)
   ret void
 ; CHECK-LABEL: test2
@@ -52,12 +52,12 @@ equal:
 }
 
 ; Basic test for invoke statepoints
-define i8 addrspace(1)* @test3(i8 addrspace(1)* %obj, i8 addrspace(1)* %obj1) gc "statepoint-example" {
+define i8 addrspace(1)* @test3(i8 addrspace(1)* %obj, i8 addrspace(1)* %obj1) gc "statepoint-example" personality i32 ()* @"personality_function" {
 ; CHECK-LABEL: test3
 entry:
   ; CHECK-LABEL: entry
   ; CHECK: statepoint
-  %0 = invoke i32 (void ()*, i32, i32, ...)* @llvm.experimental.gc.statepoint.p0f_isVoidf(void ()* undef, i32 0, i32 0, i32 5, i32 0, i32 -1, i32 0, i32 0, i32 0, i8 addrspace(1)* %obj, i8 addrspace(1)* %obj1)
+  %0 = invoke i32 (i64, i32, void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 0, i32 0, void ()* undef, i32 0, i32 0, i32 0, i32 5, i32 0, i32 -1, i32 0, i32 0, i32 0, i8 addrspace(1)* %obj, i8 addrspace(1)* %obj1)
           to label %normal_dest unwind label %exceptional_return
 
 normal_dest:
@@ -65,19 +65,18 @@ normal_dest:
   ; CHECK: gc.relocate
   ; CHECK: gc.relocate
   ; CHECK: ret
-  %obj.relocated = call coldcc i8 addrspace(1)* @llvm.experimental.gc.relocate.p1i8(i32 %0, i32 9, i32 9)
-  %obj1.relocated = call coldcc i8 addrspace(1)* @llvm.experimental.gc.relocate.p1i8(i32 %0, i32 10, i32 10)
+  %obj.relocated = call coldcc i8 addrspace(1)* @llvm.experimental.gc.relocate.p1i8(i32 %0, i32 12, i32 12)
+  %obj1.relocated = call coldcc i8 addrspace(1)* @llvm.experimental.gc.relocate.p1i8(i32 %0, i32 12, i32 12)
   ret i8 addrspace(1)* %obj.relocated
 
 exceptional_return:
   ; CHECK-LABEL: exceptional_return
   ; CHECK: gc.relocate
   ; CHECK: gc.relocate
-  %landing_pad = landingpad { i8*, i32 } personality i32 ()* @"personality_function"
+  %landing_pad = landingpad { i8*, i32 }
           cleanup
   %relocate_token = extractvalue { i8*, i32 } %landing_pad, 1
-  %obj.relocated1 = call coldcc i8 addrspace(1)* @llvm.experimental.gc.relocate.p1i8(i32 %relocate_token, i32 9, i32 9)
-  %obj1.relocated1 = call coldcc i8 addrspace(1)* @llvm.experimental.gc.relocate.p1i8(i32 %relocate_token, i32 10, i32 10)
+  %obj.relocated1 = call coldcc i8 addrspace(1)* @llvm.experimental.gc.relocate.p1i8(i32 %relocate_token, i32 12, i32 12)
+  %obj1.relocated1 = call coldcc i8 addrspace(1)* @llvm.experimental.gc.relocate.p1i8(i32 %relocate_token, i32 12, i32 12)
   ret i8 addrspace(1)* %obj1.relocated1
 }
-
diff --git a/test/YAMLParser/bool.data b/test/YAMLParser/bool.test
similarity index 100%
rename from test/YAMLParser/bool.data
rename to test/YAMLParser/bool.test
diff --git a/test/YAMLParser/construct-bool.data b/test/YAMLParser/construct-bool.test
similarity index 100%
rename from test/YAMLParser/construct-bool.data
rename to test/YAMLParser/construct-bool.test
diff --git a/test/YAMLParser/construct-custom.data b/test/YAMLParser/construct-custom.test
similarity index 100%
rename from test/YAMLParser/construct-custom.data
rename to test/YAMLParser/construct-custom.test
diff --git a/test/YAMLParser/construct-float.data b/test/YAMLParser/construct-float.test
similarity index 100%
rename from test/YAMLParser/construct-float.data
rename to test/YAMLParser/construct-float.test
diff --git a/test/YAMLParser/construct-int.data b/test/YAMLParser/construct-int.test
similarity index 100%
rename from test/YAMLParser/construct-int.data
rename to test/YAMLParser/construct-int.test
diff --git a/test/YAMLParser/construct-map.data b/test/YAMLParser/construct-map.test
similarity index 100%
rename from test/YAMLParser/construct-map.data
rename to test/YAMLParser/construct-map.test
diff --git a/test/YAMLParser/construct-merge.data b/test/YAMLParser/construct-merge.test
similarity index 100%
rename from test/YAMLParser/construct-merge.data
rename to test/YAMLParser/construct-merge.test
diff --git a/test/YAMLParser/construct-null.data b/test/YAMLParser/construct-null.test
similarity index 100%
rename from test/YAMLParser/construct-null.data
rename to test/YAMLParser/construct-null.test
diff --git a/test/YAMLParser/construct-omap.data b/test/YAMLParser/construct-omap.test
similarity index 100%
rename from test/YAMLParser/construct-omap.data
rename to test/YAMLParser/construct-omap.test
diff --git a/test/YAMLParser/construct-pairs.data b/test/YAMLParser/construct-pairs.test
similarity index 100%
rename from test/YAMLParser/construct-pairs.data
rename to test/YAMLParser/construct-pairs.test
diff --git a/test/YAMLParser/construct-seq.data b/test/YAMLParser/construct-seq.test
similarity index 100%
rename from test/YAMLParser/construct-seq.data
rename to test/YAMLParser/construct-seq.test
diff --git a/test/YAMLParser/construct-set.data b/test/YAMLParser/construct-set.test
similarity index 100%
rename from test/YAMLParser/construct-set.data
rename to test/YAMLParser/construct-set.test
diff --git a/test/YAMLParser/construct-str-ascii.data b/test/YAMLParser/construct-str-ascii.test
similarity index 100%
rename from test/YAMLParser/construct-str-ascii.data
rename to test/YAMLParser/construct-str-ascii.test
diff --git a/test/YAMLParser/construct-str.data b/test/YAMLParser/construct-str.test
similarity index 100%
rename from test/YAMLParser/construct-str.data
rename to test/YAMLParser/construct-str.test
diff --git a/test/YAMLParser/construct-timestamp.data b/test/YAMLParser/construct-timestamp.test
similarity index 100%
rename from test/YAMLParser/construct-timestamp.data
rename to test/YAMLParser/construct-timestamp.test
diff --git a/test/YAMLParser/construct-value.data b/test/YAMLParser/construct-value.test
similarity index 100%
rename from test/YAMLParser/construct-value.data
rename to test/YAMLParser/construct-value.test
diff --git a/test/YAMLParser/duplicate-key.former-loader-error.data b/test/YAMLParser/duplicate-key.former-loader-error.test
similarity index 100%
rename from test/YAMLParser/duplicate-key.former-loader-error.data
rename to test/YAMLParser/duplicate-key.former-loader-error.test
diff --git a/test/YAMLParser/duplicate-mapping-key.former-loader-error.data b/test/YAMLParser/duplicate-mapping-key.former-loader-error.test
similarity index 100%
rename from test/YAMLParser/duplicate-mapping-key.former-loader-error.data
rename to test/YAMLParser/duplicate-mapping-key.former-loader-error.test
diff --git a/test/YAMLParser/duplicate-merge-key.former-loader-error.data b/test/YAMLParser/duplicate-merge-key.former-loader-error.test
similarity index 100%
rename from test/YAMLParser/duplicate-merge-key.former-loader-error.data
rename to test/YAMLParser/duplicate-merge-key.former-loader-error.test
diff --git a/test/YAMLParser/duplicate-value-key.former-loader-error.data b/test/YAMLParser/duplicate-value-key.former-loader-error.test
similarity index 100%
rename from test/YAMLParser/duplicate-value-key.former-loader-error.data
rename to test/YAMLParser/duplicate-value-key.former-loader-error.test
diff --git a/test/YAMLParser/emit-block-scalar-in-simple-key-context-bug.data b/test/YAMLParser/emit-block-scalar-in-simple-key-context-bug.test
similarity index 100%
rename from test/YAMLParser/emit-block-scalar-in-simple-key-context-bug.data
rename to test/YAMLParser/emit-block-scalar-in-simple-key-context-bug.test
diff --git a/test/YAMLParser/empty-document-bug.data b/test/YAMLParser/empty-document-bug.test
similarity index 100%
rename from test/YAMLParser/empty-document-bug.data
rename to test/YAMLParser/empty-document-bug.test
diff --git a/test/YAMLParser/float.data b/test/YAMLParser/float.test
similarity index 100%
rename from test/YAMLParser/float.data
rename to test/YAMLParser/float.test
diff --git a/test/YAMLParser/int.data b/test/YAMLParser/int.test
similarity index 100%
rename from test/YAMLParser/int.data
rename to test/YAMLParser/int.test
diff --git a/test/YAMLParser/invalid-single-quote-bug.data b/test/YAMLParser/invalid-single-quote-bug.test
similarity index 100%
rename from test/YAMLParser/invalid-single-quote-bug.data
rename to test/YAMLParser/invalid-single-quote-bug.test
diff --git a/test/YAMLParser/merge.data b/test/YAMLParser/merge.test
similarity index 100%
rename from test/YAMLParser/merge.data
rename to test/YAMLParser/merge.test
diff --git a/test/YAMLParser/more-floats.data b/test/YAMLParser/more-floats.test
similarity index 100%
rename from test/YAMLParser/more-floats.data
rename to test/YAMLParser/more-floats.test
diff --git a/test/YAMLParser/negative-float-bug.data b/test/YAMLParser/negative-float-bug.test
similarity index 100%
rename from test/YAMLParser/negative-float-bug.data
rename to test/YAMLParser/negative-float-bug.test
diff --git a/test/YAMLParser/null.data b/test/YAMLParser/null.test
similarity index 100%
rename from test/YAMLParser/null.data
rename to test/YAMLParser/null.test
diff --git a/test/YAMLParser/resolver.data b/test/YAMLParser/resolver.test
similarity index 100%
rename from test/YAMLParser/resolver.data
rename to test/YAMLParser/resolver.test
diff --git a/test/YAMLParser/run-parser-crash-bug.data b/test/YAMLParser/run-parser-crash-bug.test
similarity index 100%
rename from test/YAMLParser/run-parser-crash-bug.data
rename to test/YAMLParser/run-parser-crash-bug.test
diff --git a/test/YAMLParser/scan-document-end-bug.data b/test/YAMLParser/scan-document-end-bug.test
similarity index 100%
rename from test/YAMLParser/scan-document-end-bug.data
rename to test/YAMLParser/scan-document-end-bug.test
diff --git a/test/YAMLParser/scan-line-break-bug.data b/test/YAMLParser/scan-line-break-bug.test
similarity index 100%
rename from test/YAMLParser/scan-line-break-bug.data
rename to test/YAMLParser/scan-line-break-bug.test
diff --git a/test/YAMLParser/single-dot-is-not-float-bug.data b/test/YAMLParser/single-dot-is-not-float-bug.test
similarity index 100%
rename from test/YAMLParser/single-dot-is-not-float-bug.data
rename to test/YAMLParser/single-dot-is-not-float-bug.test
diff --git a/test/YAMLParser/sloppy-indentation.data b/test/YAMLParser/sloppy-indentation.test
similarity index 100%
rename from test/YAMLParser/sloppy-indentation.data
rename to test/YAMLParser/sloppy-indentation.test
diff --git a/test/YAMLParser/spec-02-01.data b/test/YAMLParser/spec-02-01.test
similarity index 100%
rename from test/YAMLParser/spec-02-01.data
rename to test/YAMLParser/spec-02-01.test
diff --git a/test/YAMLParser/spec-02-02.data b/test/YAMLParser/spec-02-02.test
similarity index 100%
rename from test/YAMLParser/spec-02-02.data
rename to test/YAMLParser/spec-02-02.test
diff --git a/test/YAMLParser/spec-02-03.data b/test/YAMLParser/spec-02-03.test
similarity index 100%
rename from test/YAMLParser/spec-02-03.data
rename to test/YAMLParser/spec-02-03.test
diff --git a/test/YAMLParser/spec-02-04.data b/test/YAMLParser/spec-02-04.test
similarity index 100%
rename from test/YAMLParser/spec-02-04.data
rename to test/YAMLParser/spec-02-04.test
diff --git a/test/YAMLParser/spec-02-05.data b/test/YAMLParser/spec-02-05.test
similarity index 100%
rename from test/YAMLParser/spec-02-05.data
rename to test/YAMLParser/spec-02-05.test
diff --git a/test/YAMLParser/spec-02-06.data b/test/YAMLParser/spec-02-06.test
similarity index 100%
rename from test/YAMLParser/spec-02-06.data
rename to test/YAMLParser/spec-02-06.test
diff --git a/test/YAMLParser/spec-02-07.data b/test/YAMLParser/spec-02-07.test
similarity index 100%
rename from test/YAMLParser/spec-02-07.data
rename to test/YAMLParser/spec-02-07.test
diff --git a/test/YAMLParser/spec-02-08.data b/test/YAMLParser/spec-02-08.test
similarity index 100%
rename from test/YAMLParser/spec-02-08.data
rename to test/YAMLParser/spec-02-08.test
diff --git a/test/YAMLParser/spec-02-09.data b/test/YAMLParser/spec-02-09.test
similarity index 100%
rename from test/YAMLParser/spec-02-09.data
rename to test/YAMLParser/spec-02-09.test
diff --git a/test/YAMLParser/spec-02-10.data b/test/YAMLParser/spec-02-10.test
similarity index 100%
rename from test/YAMLParser/spec-02-10.data
rename to test/YAMLParser/spec-02-10.test
diff --git a/test/YAMLParser/spec-02-11.data b/test/YAMLParser/spec-02-11.test
similarity index 100%
rename from test/YAMLParser/spec-02-11.data
rename to test/YAMLParser/spec-02-11.test
diff --git a/test/YAMLParser/spec-02-12.data b/test/YAMLParser/spec-02-12.test
similarity index 100%
rename from test/YAMLParser/spec-02-12.data
rename to test/YAMLParser/spec-02-12.test
diff --git a/test/YAMLParser/spec-02-13.data b/test/YAMLParser/spec-02-13.test
similarity index 100%
rename from test/YAMLParser/spec-02-13.data
rename to test/YAMLParser/spec-02-13.test
diff --git a/test/YAMLParser/spec-02-14.data b/test/YAMLParser/spec-02-14.test
similarity index 100%
rename from test/YAMLParser/spec-02-14.data
rename to test/YAMLParser/spec-02-14.test
diff --git a/test/YAMLParser/spec-02-15.data b/test/YAMLParser/spec-02-15.test
similarity index 100%
rename from test/YAMLParser/spec-02-15.data
rename to test/YAMLParser/spec-02-15.test
diff --git a/test/YAMLParser/spec-02-16.data b/test/YAMLParser/spec-02-16.test
similarity index 100%
rename from test/YAMLParser/spec-02-16.data
rename to test/YAMLParser/spec-02-16.test
diff --git a/test/YAMLParser/spec-02-17.data b/test/YAMLParser/spec-02-17.test
similarity index 100%
rename from test/YAMLParser/spec-02-17.data
rename to test/YAMLParser/spec-02-17.test
diff --git a/test/YAMLParser/spec-02-18.data b/test/YAMLParser/spec-02-18.test
similarity index 100%
rename from test/YAMLParser/spec-02-18.data
rename to test/YAMLParser/spec-02-18.test
diff --git a/test/YAMLParser/spec-02-19.data b/test/YAMLParser/spec-02-19.test
similarity index 100%
rename from test/YAMLParser/spec-02-19.data
rename to test/YAMLParser/spec-02-19.test
diff --git a/test/YAMLParser/spec-02-20.data b/test/YAMLParser/spec-02-20.test
similarity index 100%
rename from test/YAMLParser/spec-02-20.data
rename to test/YAMLParser/spec-02-20.test
diff --git a/test/YAMLParser/spec-02-21.data b/test/YAMLParser/spec-02-21.test
similarity index 100%
rename from test/YAMLParser/spec-02-21.data
rename to test/YAMLParser/spec-02-21.test
diff --git a/test/YAMLParser/spec-02-22.data b/test/YAMLParser/spec-02-22.test
similarity index 100%
rename from test/YAMLParser/spec-02-22.data
rename to test/YAMLParser/spec-02-22.test
diff --git a/test/YAMLParser/spec-02-23.data b/test/YAMLParser/spec-02-23.test
similarity index 100%
rename from test/YAMLParser/spec-02-23.data
rename to test/YAMLParser/spec-02-23.test
diff --git a/test/YAMLParser/spec-02-24.data b/test/YAMLParser/spec-02-24.test
similarity index 100%
rename from test/YAMLParser/spec-02-24.data
rename to test/YAMLParser/spec-02-24.test
diff --git a/test/YAMLParser/spec-02-25.data b/test/YAMLParser/spec-02-25.test
similarity index 100%
rename from test/YAMLParser/spec-02-25.data
rename to test/YAMLParser/spec-02-25.test
diff --git a/test/YAMLParser/spec-02-26.data b/test/YAMLParser/spec-02-26.test
similarity index 100%
rename from test/YAMLParser/spec-02-26.data
rename to test/YAMLParser/spec-02-26.test
diff --git a/test/YAMLParser/spec-02-27.data b/test/YAMLParser/spec-02-27.test
similarity index 100%
rename from test/YAMLParser/spec-02-27.data
rename to test/YAMLParser/spec-02-27.test
diff --git a/test/YAMLParser/spec-02-28.data b/test/YAMLParser/spec-02-28.test
similarity index 100%
rename from test/YAMLParser/spec-02-28.data
rename to test/YAMLParser/spec-02-28.test
diff --git a/test/YAMLParser/spec-05-01-utf8.data b/test/YAMLParser/spec-05-01-utf8.test
similarity index 100%
rename from test/YAMLParser/spec-05-01-utf8.data
rename to test/YAMLParser/spec-05-01-utf8.test
diff --git a/test/YAMLParser/spec-05-02-utf8.data b/test/YAMLParser/spec-05-02-utf8.data
deleted file mode 100644
index 028f41bb557..00000000000
--- a/test/YAMLParser/spec-05-02-utf8.data
+++ /dev/null
@@ -1,7 +0,0 @@
-# RUN: yaml-bench -canonical %s 2>&1 | FileCheck %s
-
-# Invalid use of BOM
-# inside a
-# document.
-
-# CHECK: error
diff --git a/test/YAMLParser/spec-05-02-utf8.test b/test/YAMLParser/spec-05-02-utf8.test
new file mode 100644
index 00000000000..3666f990e47
--- /dev/null
+++ b/test/YAMLParser/spec-05-02-utf8.test
@@ -0,0 +1,7 @@
+# RUN: not yaml-bench -canonical %s 2>&1 | FileCheck %s
+
+# Invalid use of BOM
+# inside a
+# document.
+
+# CHECK: error
diff --git a/test/YAMLParser/spec-05-03.data b/test/YAMLParser/spec-05-03.test
similarity index 100%
rename from test/YAMLParser/spec-05-03.data
rename to test/YAMLParser/spec-05-03.test
diff --git a/test/YAMLParser/spec-05-04.data b/test/YAMLParser/spec-05-04.test
similarity index 100%
rename from test/YAMLParser/spec-05-04.data
rename to test/YAMLParser/spec-05-04.test
diff --git a/test/YAMLParser/spec-05-05.data b/test/YAMLParser/spec-05-05.test
similarity index 100%
rename from test/YAMLParser/spec-05-05.data
rename to test/YAMLParser/spec-05-05.test
diff --git a/test/YAMLParser/spec-05-06.data b/test/YAMLParser/spec-05-06.test
similarity index 100%
rename from test/YAMLParser/spec-05-06.data
rename to test/YAMLParser/spec-05-06.test
diff --git a/test/YAMLParser/spec-05-07.data b/test/YAMLParser/spec-05-07.test
similarity index 100%
rename from test/YAMLParser/spec-05-07.data
rename to test/YAMLParser/spec-05-07.test
diff --git a/test/YAMLParser/spec-05-08.data b/test/YAMLParser/spec-05-08.test
similarity index 100%
rename from test/YAMLParser/spec-05-08.data
rename to test/YAMLParser/spec-05-08.test
diff --git a/test/YAMLParser/spec-05-09.data b/test/YAMLParser/spec-05-09.test
similarity index 100%
rename from test/YAMLParser/spec-05-09.data
rename to test/YAMLParser/spec-05-09.test
diff --git a/test/YAMLParser/spec-05-10.data b/test/YAMLParser/spec-05-10.data
deleted file mode 100644
index bab2c1b89ca..00000000000
--- a/test/YAMLParser/spec-05-10.data
+++ /dev/null
@@ -1,6 +0,0 @@
-# RUN: yaml-bench -canonical %s 2>&1 | FileCheck %s
-
-commercial-at: @text
-grave-accent: `text
-
-# CHECK: error
diff --git a/test/YAMLParser/spec-05-10.test b/test/YAMLParser/spec-05-10.test
new file mode 100644
index 00000000000..14112c1eed5
--- /dev/null
+++ b/test/YAMLParser/spec-05-10.test
@@ -0,0 +1,6 @@
+# RUN: not yaml-bench -canonical %s 2>&1 | FileCheck %s
+
+commercial-at: @text
+grave-accent: `text
+
+# CHECK: error
diff --git a/test/YAMLParser/spec-05-11.data b/test/YAMLParser/spec-05-11.test
similarity index 100%
rename from test/YAMLParser/spec-05-11.data
rename to test/YAMLParser/spec-05-11.test
diff --git a/test/YAMLParser/spec-05-12.data b/test/YAMLParser/spec-05-12.data
deleted file mode 100644
index eedfc08cf38..00000000000
--- a/test/YAMLParser/spec-05-12.data
+++ /dev/null
@@ -1,16 +0,0 @@
-# RUN: yaml-bench -canonical %s 2>&1 | FileCheck %s
-#
-# We don't currently reject tabs as indentation.
-# XFAIL: *
-
-# Tabs do's and don'ts:
-# comment:
-quoted: "Quoted		"
-block: |
-  void main() {
-  	printf("Hello, world!\n");
-  }
-elsewhere:	# separation
-	indentation, in	plain scalar
-
-# CHECK: error
diff --git a/test/YAMLParser/spec-05-12.test b/test/YAMLParser/spec-05-12.test
new file mode 100644
index 00000000000..7ecc36dee8b
--- /dev/null
+++ b/test/YAMLParser/spec-05-12.test
@@ -0,0 +1,18 @@
+# RUN: yaml-bench -canonical %s 2>&1 | FileCheck %s
+#
+# FIXME: This test should actually fail. Yaml-bench should report an error
+# that a tab is being used to indent a plain scalar at line 15.
+# We don't currently reject tabs as indentation.
+
+# Tabs do's and don'ts:
+# comment:
+quoted: "Quoted		"
+block: |
+  void main() {
+  	printf("Hello, world!\n");
+  }
+elsewhere:	# separation
+	indentation, in	plain scalar
+
+
+# CHECK: !!str "Quoted\t\t"
diff --git a/test/YAMLParser/spec-05-13.data b/test/YAMLParser/spec-05-13.test
similarity index 100%
rename from test/YAMLParser/spec-05-13.data
rename to test/YAMLParser/spec-05-13.test
diff --git a/test/YAMLParser/spec-05-14.data b/test/YAMLParser/spec-05-14.test
similarity index 100%
rename from test/YAMLParser/spec-05-14.data
rename to test/YAMLParser/spec-05-14.test
diff --git a/test/YAMLParser/spec-05-15.data b/test/YAMLParser/spec-05-15.data
deleted file mode 100644
index 27dbd8396ea..00000000000
--- a/test/YAMLParser/spec-05-15.data
+++ /dev/null
@@ -1,7 +0,0 @@
-# RUN: yaml-bench -canonical %s 2>&1 | FileCheck %s
-
-Bad escapes:
-  "\c
-  \xq-"
-
-# CHECK: error
diff --git a/test/YAMLParser/spec-05-15.test b/test/YAMLParser/spec-05-15.test
new file mode 100644
index 00000000000..46228bae215
--- /dev/null
+++ b/test/YAMLParser/spec-05-15.test
@@ -0,0 +1,7 @@
+# RUN: not yaml-bench -canonical %s 2>&1 | FileCheck %s
+
+Bad escapes:
+  "\c
+  \xq-"
+
+# CHECK: error
diff --git a/test/YAMLParser/spec-06-01.data b/test/YAMLParser/spec-06-01.test
similarity index 100%
rename from test/YAMLParser/spec-06-01.data
rename to test/YAMLParser/spec-06-01.test
diff --git a/test/YAMLParser/spec-06-02.data b/test/YAMLParser/spec-06-02.test
similarity index 100%
rename from test/YAMLParser/spec-06-02.data
rename to test/YAMLParser/spec-06-02.test
diff --git a/test/YAMLParser/spec-06-03.data b/test/YAMLParser/spec-06-03.test
similarity index 100%
rename from test/YAMLParser/spec-06-03.data
rename to test/YAMLParser/spec-06-03.test
diff --git a/test/YAMLParser/spec-06-04.data b/test/YAMLParser/spec-06-04.test
similarity index 100%
rename from test/YAMLParser/spec-06-04.data
rename to test/YAMLParser/spec-06-04.test
diff --git a/test/YAMLParser/spec-06-05.data b/test/YAMLParser/spec-06-05.test
similarity index 100%
rename from test/YAMLParser/spec-06-05.data
rename to test/YAMLParser/spec-06-05.test
diff --git a/test/YAMLParser/spec-06-06.data b/test/YAMLParser/spec-06-06.test
similarity index 100%
rename from test/YAMLParser/spec-06-06.data
rename to test/YAMLParser/spec-06-06.test
diff --git a/test/YAMLParser/spec-06-07.data b/test/YAMLParser/spec-06-07.test
similarity index 100%
rename from test/YAMLParser/spec-06-07.data
rename to test/YAMLParser/spec-06-07.test
diff --git a/test/YAMLParser/spec-06-08.data b/test/YAMLParser/spec-06-08.test
similarity index 100%
rename from test/YAMLParser/spec-06-08.data
rename to test/YAMLParser/spec-06-08.test
diff --git a/test/YAMLParser/spec-07-01.data b/test/YAMLParser/spec-07-01.test
similarity index 100%
rename from test/YAMLParser/spec-07-01.data
rename to test/YAMLParser/spec-07-01.test
diff --git a/test/YAMLParser/spec-07-02.data b/test/YAMLParser/spec-07-02.test
similarity index 100%
rename from test/YAMLParser/spec-07-02.data
rename to test/YAMLParser/spec-07-02.test
diff --git a/test/YAMLParser/spec-07-03.data b/test/YAMLParser/spec-07-03.data
deleted file mode 100644
index c4a5299562a..00000000000
--- a/test/YAMLParser/spec-07-03.data
+++ /dev/null
@@ -1,7 +0,0 @@
-# RUN: yaml-bench -canonical %s 2>&1 | FileCheck %s
-
-%YAML 1.1
-%YAML 1.1
-foo
-
-# CHECK: error
diff --git a/test/YAMLParser/spec-07-03.test b/test/YAMLParser/spec-07-03.test
new file mode 100644
index 00000000000..76266b896d7
--- /dev/null
+++ b/test/YAMLParser/spec-07-03.test
@@ -0,0 +1,7 @@
+# RUN: not yaml-bench -canonical %s 2>&1 | FileCheck %s
+
+%YAML 1.1
+%YAML 1.1
+foo
+
+# CHECK: error
diff --git a/test/YAMLParser/spec-07-04.data b/test/YAMLParser/spec-07-04.test
similarity index 100%
rename from test/YAMLParser/spec-07-04.data
rename to test/YAMLParser/spec-07-04.test
diff --git a/test/YAMLParser/spec-07-05.data b/test/YAMLParser/spec-07-05.data
deleted file mode 100644
index f7cff3a8390..00000000000
--- a/test/YAMLParser/spec-07-05.data
+++ /dev/null
@@ -1,10 +0,0 @@
-# RUN: yaml-bench -canonical %s 2>&1 | FileCheck %s
-#
-# We don't currently parse TAG directives.
-# XFAIL: *
-
-%TAG ! !foo
-%TAG ! !foo
-bar
-
-# CHECK: error
diff --git a/test/YAMLParser/spec-07-05.test b/test/YAMLParser/spec-07-05.test
new file mode 100644
index 00000000000..447cba81902
--- /dev/null
+++ b/test/YAMLParser/spec-07-05.test
@@ -0,0 +1,8 @@
+# RUN: not yaml-bench -canonical %s 2>&1 | FileCheck %s
+#
+# We don't currently parse TAG directives.
+# CHECK: error: Unexpected token
+
+%TAG ! !foo
+%TAG ! !foo
+bar
diff --git a/test/YAMLParser/spec-07-06.data b/test/YAMLParser/spec-07-06.test
similarity index 100%
rename from test/YAMLParser/spec-07-06.data
rename to test/YAMLParser/spec-07-06.test
diff --git a/test/YAMLParser/spec-07-07a.data b/test/YAMLParser/spec-07-07a.test
similarity index 100%
rename from test/YAMLParser/spec-07-07a.data
rename to test/YAMLParser/spec-07-07a.test
diff --git a/test/YAMLParser/spec-07-07b.data b/test/YAMLParser/spec-07-07b.test
similarity index 100%
rename from test/YAMLParser/spec-07-07b.data
rename to test/YAMLParser/spec-07-07b.test
diff --git a/test/YAMLParser/spec-07-08.data b/test/YAMLParser/spec-07-08.test
similarity index 100%
rename from test/YAMLParser/spec-07-08.data
rename to test/YAMLParser/spec-07-08.test
diff --git a/test/YAMLParser/spec-07-09.data b/test/YAMLParser/spec-07-09.test
similarity index 100%
rename from test/YAMLParser/spec-07-09.data
rename to test/YAMLParser/spec-07-09.test
diff --git a/test/YAMLParser/spec-07-10.data b/test/YAMLParser/spec-07-10.test
similarity index 100%
rename from test/YAMLParser/spec-07-10.data
rename to test/YAMLParser/spec-07-10.test
diff --git a/test/YAMLParser/spec-07-11.data b/test/YAMLParser/spec-07-11.test
similarity index 100%
rename from test/YAMLParser/spec-07-11.data
rename to test/YAMLParser/spec-07-11.test
diff --git a/test/YAMLParser/spec-07-12a.data b/test/YAMLParser/spec-07-12a.test
similarity index 100%
rename from test/YAMLParser/spec-07-12a.data
rename to test/YAMLParser/spec-07-12a.test
diff --git a/test/YAMLParser/spec-07-12b.data b/test/YAMLParser/spec-07-12b.test
similarity index 100%
rename from test/YAMLParser/spec-07-12b.data
rename to test/YAMLParser/spec-07-12b.test
diff --git a/test/YAMLParser/spec-07-13.data b/test/YAMLParser/spec-07-13.test
similarity index 100%
rename from test/YAMLParser/spec-07-13.data
rename to test/YAMLParser/spec-07-13.test
diff --git a/test/YAMLParser/spec-08-01.data b/test/YAMLParser/spec-08-01.test
similarity index 100%
rename from test/YAMLParser/spec-08-01.data
rename to test/YAMLParser/spec-08-01.test
diff --git a/test/YAMLParser/spec-08-02.data b/test/YAMLParser/spec-08-02.test
similarity index 100%
rename from test/YAMLParser/spec-08-02.data
rename to test/YAMLParser/spec-08-02.test
diff --git a/test/YAMLParser/spec-08-03.data b/test/YAMLParser/spec-08-03.test
similarity index 100%
rename from test/YAMLParser/spec-08-03.data
rename to test/YAMLParser/spec-08-03.test
diff --git a/test/YAMLParser/spec-08-04.data b/test/YAMLParser/spec-08-04.data
deleted file mode 100644
index 73c493d7637..00000000000
--- a/test/YAMLParser/spec-08-04.data
+++ /dev/null
@@ -1,9 +0,0 @@
-# RUN: yaml-bench -canonical %s 2>&1 | FileCheck %s
-#
-# We don't currently look at the content of literal tags.
-# XFAIL: *
-
-- !<!> foo
-- !<$:?> bar
-
-# CHECK: error
diff --git a/test/YAMLParser/spec-08-04.test b/test/YAMLParser/spec-08-04.test
new file mode 100644
index 00000000000..5b502ae3a17
--- /dev/null
+++ b/test/YAMLParser/spec-08-04.test
@@ -0,0 +1,7 @@
+# RUN: not yaml-bench -canonical %s 2>&1 | FileCheck %s
+#
+# We don't currently look at the content of literal tags.
+# CHECK: error: Unknown tag handle
+
+- !<!> foo
+- !<$:?> bar
diff --git a/test/YAMLParser/spec-08-05.data b/test/YAMLParser/spec-08-05.test
similarity index 100%
rename from test/YAMLParser/spec-08-05.data
rename to test/YAMLParser/spec-08-05.test
diff --git a/test/YAMLParser/spec-08-06.data b/test/YAMLParser/spec-08-06.data
deleted file mode 100644
index 9844f53a4ef..00000000000
--- a/test/YAMLParser/spec-08-06.data
+++ /dev/null
@@ -1,12 +0,0 @@
-# RUN: yaml-bench -canonical %s 2>&1 | FileCheck %s
-#
-# We don't currently validate tags.
-# XFAIL: *
-
-%TAG !o! tag:ben-kiki.org,2000:
----
-- !$a!b foo
-- !o! bar
-- !h!type baz
-
-# CHECK: error
diff --git a/test/YAMLParser/spec-08-06.test b/test/YAMLParser/spec-08-06.test
new file mode 100644
index 00000000000..b457d59b569
--- /dev/null
+++ b/test/YAMLParser/spec-08-06.test
@@ -0,0 +1,11 @@
+# RUN: not yaml-bench -canonical %s 2>&1 | FileCheck %s
+#
+# We don't currently validate tags.
+# CHECK: error: Unknown tag handle
+
+%TAG !o! tag:ben-kiki.org,2000:
+---
+- !$a!b foo
+- !o! bar
+- !h!type baz
+
diff --git a/test/YAMLParser/spec-08-07.data b/test/YAMLParser/spec-08-07.test
similarity index 100%
rename from test/YAMLParser/spec-08-07.data
rename to test/YAMLParser/spec-08-07.test
diff --git a/test/YAMLParser/spec-08-08.data b/test/YAMLParser/spec-08-08.test
similarity index 100%
rename from test/YAMLParser/spec-08-08.data
rename to test/YAMLParser/spec-08-08.test
diff --git a/test/YAMLParser/spec-08-09.data b/test/YAMLParser/spec-08-09.test
similarity index 100%
rename from test/YAMLParser/spec-08-09.data
rename to test/YAMLParser/spec-08-09.test
diff --git a/test/YAMLParser/spec-08-10.data b/test/YAMLParser/spec-08-10.test
similarity index 100%
rename from test/YAMLParser/spec-08-10.data
rename to test/YAMLParser/spec-08-10.test
diff --git a/test/YAMLParser/spec-08-11.data b/test/YAMLParser/spec-08-11.test
similarity index 100%
rename from test/YAMLParser/spec-08-11.data
rename to test/YAMLParser/spec-08-11.test
diff --git a/test/YAMLParser/spec-08-12.data b/test/YAMLParser/spec-08-12.test
similarity index 100%
rename from test/YAMLParser/spec-08-12.data
rename to test/YAMLParser/spec-08-12.test
diff --git a/test/YAMLParser/spec-08-13.data b/test/YAMLParser/spec-08-13.test
similarity index 100%
rename from test/YAMLParser/spec-08-13.data
rename to test/YAMLParser/spec-08-13.test
diff --git a/test/YAMLParser/spec-08-14.data b/test/YAMLParser/spec-08-14.test
similarity index 100%
rename from test/YAMLParser/spec-08-14.data
rename to test/YAMLParser/spec-08-14.test
diff --git a/test/YAMLParser/spec-08-15.data b/test/YAMLParser/spec-08-15.test
similarity index 100%
rename from test/YAMLParser/spec-08-15.data
rename to test/YAMLParser/spec-08-15.test
diff --git a/test/YAMLParser/spec-09-01.data b/test/YAMLParser/spec-09-01.test
similarity index 100%
rename from test/YAMLParser/spec-09-01.data
rename to test/YAMLParser/spec-09-01.test
diff --git a/test/YAMLParser/spec-09-02.data b/test/YAMLParser/spec-09-02.data
deleted file mode 100644
index 9d8a58ca33a..00000000000
--- a/test/YAMLParser/spec-09-02.data
+++ /dev/null
@@ -1,14 +0,0 @@
-# RUN: yaml-bench -canonical %s 2>&1 | FileCheck %s
-#
-# Indent trimming is not yet implemented.
-# XFAIL: *
-
- "as space
- trimmed
-
- specific
-
- escaped	\
- none"
-
-# CHECK: !!str "as space trimmed\nspecific\nescaped\tnone"
diff --git a/test/YAMLParser/spec-09-02.test b/test/YAMLParser/spec-09-02.test
new file mode 100644
index 00000000000..3f8e49a8bd3
--- /dev/null
+++ b/test/YAMLParser/spec-09-02.test
@@ -0,0 +1,14 @@
+# RUN: yaml-bench -canonical %s 2>&1 | FileCheck %s
+
+ "as space
+ trimmed
+
+ specific
+
+ escaped	\
+ none"
+
+# FIXME: The string below should actually be
+#   "as space trimmed\nspecific\nescaped\tnone", but the parser currently has
+#   a bug when parsing multiline quoted strings.
+# CHECK: !!str "as space\n trimmed\n specific\n escaped\t none"
diff --git a/test/YAMLParser/spec-09-03.data b/test/YAMLParser/spec-09-03.test
similarity index 100%
rename from test/YAMLParser/spec-09-03.data
rename to test/YAMLParser/spec-09-03.test
diff --git a/test/YAMLParser/spec-09-04.data b/test/YAMLParser/spec-09-04.test
similarity index 100%
rename from test/YAMLParser/spec-09-04.data
rename to test/YAMLParser/spec-09-04.test
diff --git a/test/YAMLParser/spec-09-05.data b/test/YAMLParser/spec-09-05.test
similarity index 100%
rename from test/YAMLParser/spec-09-05.data
rename to test/YAMLParser/spec-09-05.test
diff --git a/test/YAMLParser/spec-09-06.data b/test/YAMLParser/spec-09-06.test
similarity index 100%
rename from test/YAMLParser/spec-09-06.data
rename to test/YAMLParser/spec-09-06.test
diff --git a/test/YAMLParser/spec-09-07.data b/test/YAMLParser/spec-09-07.test
similarity index 100%
rename from test/YAMLParser/spec-09-07.data
rename to test/YAMLParser/spec-09-07.test
diff --git a/test/YAMLParser/spec-09-08.data b/test/YAMLParser/spec-09-08.test
similarity index 100%
rename from test/YAMLParser/spec-09-08.data
rename to test/YAMLParser/spec-09-08.test
diff --git a/test/YAMLParser/spec-09-09.data b/test/YAMLParser/spec-09-09.test
similarity index 100%
rename from test/YAMLParser/spec-09-09.data
rename to test/YAMLParser/spec-09-09.test
diff --git a/test/YAMLParser/spec-09-10.data b/test/YAMLParser/spec-09-10.test
similarity index 100%
rename from test/YAMLParser/spec-09-10.data
rename to test/YAMLParser/spec-09-10.test
diff --git a/test/YAMLParser/spec-09-11.data b/test/YAMLParser/spec-09-11.test
similarity index 100%
rename from test/YAMLParser/spec-09-11.data
rename to test/YAMLParser/spec-09-11.test
diff --git a/test/YAMLParser/spec-09-12.data b/test/YAMLParser/spec-09-12.test
similarity index 100%
rename from test/YAMLParser/spec-09-12.data
rename to test/YAMLParser/spec-09-12.test
diff --git a/test/YAMLParser/spec-09-13.data b/test/YAMLParser/spec-09-13.test
similarity index 100%
rename from test/YAMLParser/spec-09-13.data
rename to test/YAMLParser/spec-09-13.test
diff --git a/test/YAMLParser/spec-09-14.data b/test/YAMLParser/spec-09-14.data
deleted file mode 100644
index a83fcd45e2c..00000000000
--- a/test/YAMLParser/spec-09-14.data
+++ /dev/null
@@ -1,21 +0,0 @@
-# RUN: yaml-bench -canonical %s 2>&1 | FileCheck %s
-#
-# Not quite sure why this doesn't fail.
-# XFAIL: *
-
----
---- ||| : foo
-... >>>: bar
----
-[
----
-,
-... ,
-{
---- :
-... # Nested
-}
-]
-...
-
-# CHECK: error
diff --git a/test/YAMLParser/spec-09-14.test b/test/YAMLParser/spec-09-14.test
new file mode 100644
index 00000000000..5f028f920fc
--- /dev/null
+++ b/test/YAMLParser/spec-09-14.test
@@ -0,0 +1,18 @@
+# RUN: not yaml-bench -canonical %s 2>&1 | FileCheck %s
+#
+# CHECK: error: Expected a line break after block scalar header
+
+---
+--- ||| : foo
+... >>>: bar
+---
+[
+---
+,
+... ,
+{
+--- :
+... # Nested
+}
+]
+...
diff --git a/test/YAMLParser/spec-09-15.data b/test/YAMLParser/spec-09-15.test
similarity index 100%
rename from test/YAMLParser/spec-09-15.data
rename to test/YAMLParser/spec-09-15.test
diff --git a/test/YAMLParser/spec-09-16.data b/test/YAMLParser/spec-09-16.test
similarity index 100%
rename from test/YAMLParser/spec-09-16.data
rename to test/YAMLParser/spec-09-16.test
diff --git a/test/YAMLParser/spec-09-17.data b/test/YAMLParser/spec-09-17.test
similarity index 100%
rename from test/YAMLParser/spec-09-17.data
rename to test/YAMLParser/spec-09-17.test
diff --git a/test/YAMLParser/spec-09-18.data b/test/YAMLParser/spec-09-18.data
deleted file mode 100644
index ac623f9973f..00000000000
--- a/test/YAMLParser/spec-09-18.data
+++ /dev/null
@@ -1,11 +0,0 @@
-# RUN: yaml-bench -canonical %s
-
-- | # Just the style
- literal
-- >1 # Indentation indicator
-  folded
-- |+ # Chomping indicator
- keep
-
-- >-1 # Both indicators
-  strip
diff --git a/test/YAMLParser/spec-09-18.test b/test/YAMLParser/spec-09-18.test
new file mode 100644
index 00000000000..cb05bb3774c
--- /dev/null
+++ b/test/YAMLParser/spec-09-18.test
@@ -0,0 +1,15 @@
+# RUN: yaml-bench -canonical %s | FileCheck %s
+# CHECK: !!str "literal\n"
+# CHECK: !!str " folded\n"
+# CHECK: !!str "keep\n\n"
+# CHECK: !!str " strip"
+
+- | # Just the style
+ literal
+- >1 # Indentation indicator
+  folded
+- |+ # Chomping indicator
+ keep
+
+- >-1 # Both indicators
+  strip
diff --git a/test/YAMLParser/spec-09-19.data b/test/YAMLParser/spec-09-19.data
deleted file mode 100644
index 52aa157137b..00000000000
--- a/test/YAMLParser/spec-09-19.data
+++ /dev/null
@@ -1,6 +0,0 @@
-# RUN: yaml-bench -canonical %s
-
-- |
- literal
-- >
- folded
diff --git a/test/YAMLParser/spec-09-19.test b/test/YAMLParser/spec-09-19.test
new file mode 100644
index 00000000000..f385717b035
--- /dev/null
+++ b/test/YAMLParser/spec-09-19.test
@@ -0,0 +1,8 @@
+# RUN: yaml-bench -canonical %s | FileCheck %s
+# CHECK: !!str "literal\n"
+# CHECK: !!str "folded\n"
+
+- |
+ literal
+- >
+ folded
diff --git a/test/YAMLParser/spec-09-20.data b/test/YAMLParser/spec-09-20.data
deleted file mode 100644
index 86fc7ab9a2e..00000000000
--- a/test/YAMLParser/spec-09-20.data
+++ /dev/null
@@ -1,13 +0,0 @@
-# RUN: yaml-bench -canonical %s
-
-- |
- detected
-- >
- 
-  
-  # detected
-- |1
-  explicit
-- >
- 	
- detected
diff --git a/test/YAMLParser/spec-09-20.test b/test/YAMLParser/spec-09-20.test
new file mode 100644
index 00000000000..47c255b8573
--- /dev/null
+++ b/test/YAMLParser/spec-09-20.test
@@ -0,0 +1,17 @@
+# RUN: yaml-bench -canonical %s | FileCheck %s
+# CHECK: !!str "detected\n"
+# CHECK: !!str "\n\n# detected\n"
+# CHECK: !!str " explicit\n"
+# CHECK: !!str "\t\ndetected\n"
+
+- |
+ detected
+- >
+ 
+  
+  # detected
+- |1
+  explicit
+- >
+ 	
+ detected
diff --git a/test/YAMLParser/spec-09-21.data b/test/YAMLParser/spec-09-21.data
deleted file mode 100644
index 6eb7917a97e..00000000000
--- a/test/YAMLParser/spec-09-21.data
+++ /dev/null
@@ -1,12 +0,0 @@
-# RUN: yaml-bench -canonical %s 2>&1 | FileCheck %s
-
-- |
-
- text
-- >
-  text
- text
-- |1
- text
-
-# CHECK: error
diff --git a/test/YAMLParser/spec-09-21.test b/test/YAMLParser/spec-09-21.test
new file mode 100644
index 00000000000..661f986917a
--- /dev/null
+++ b/test/YAMLParser/spec-09-21.test
@@ -0,0 +1,12 @@
+# RUN: not yaml-bench -canonical %s 2>&1 | FileCheck %s
+
+- |
+
+ text
+- >
+  text
+ text
+- |1
+ text
+
+# CHECK: 8:2: error: A text line is less indented than the block scalar
diff --git a/test/YAMLParser/spec-09-22.data b/test/YAMLParser/spec-09-22.data
deleted file mode 100644
index b95faa50b5d..00000000000
--- a/test/YAMLParser/spec-09-22.data
+++ /dev/null
@@ -1,6 +0,0 @@
-# RUN: yaml-bench -canonical %s
-
-strip: |-
-  text
clip: |
-  textÂ…keep: |+
-  text

diff --git a/test/YAMLParser/spec-09-22.test b/test/YAMLParser/spec-09-22.test
new file mode 100644
index 00000000000..726358dd5ba
--- /dev/null
+++ b/test/YAMLParser/spec-09-22.test
@@ -0,0 +1,12 @@
+# RUN: yaml-bench -canonical %s | FileCheck %s
+# CHECK: !!str "text"
+# CHECK: !!str "text\n"
+# CHECK: !!str "text\n\n"
+
+strip: |-
+  text
+clip: |
+  text
+keep: |+
+  text
+
diff --git a/test/YAMLParser/spec-09-23.data b/test/YAMLParser/spec-09-23.test
similarity index 100%
rename from test/YAMLParser/spec-09-23.data
rename to test/YAMLParser/spec-09-23.test
diff --git a/test/YAMLParser/spec-09-24.data b/test/YAMLParser/spec-09-24.data
deleted file mode 100644
index f08eae6a80e..00000000000
--- a/test/YAMLParser/spec-09-24.data
+++ /dev/null
@@ -1,8 +0,0 @@
-# RUN: yaml-bench -canonical %s
-
-strip: >-
-
-clip: >
-
-keep: |+
-
diff --git a/test/YAMLParser/spec-09-24.test b/test/YAMLParser/spec-09-24.test
new file mode 100644
index 00000000000..9cf51413855
--- /dev/null
+++ b/test/YAMLParser/spec-09-24.test
@@ -0,0 +1,13 @@
+# RUN: yaml-bench -canonical %s | FileCheck %s
+# CHECK: ? !!str "strip"
+# CHECK: : !!str ""
+# CHECK: ? !!str "clip"
+# CHECK: : !!str ""
+# CHECK: ? !!str "keep"
+# CHECK: : !!str "\n"
+
+strip: >-
+
+clip: >
+
+keep: |+
diff --git a/test/YAMLParser/spec-09-25.data b/test/YAMLParser/spec-09-25.data
deleted file mode 100644
index b15edb523d2..00000000000
--- a/test/YAMLParser/spec-09-25.data
+++ /dev/null
@@ -1,5 +0,0 @@
-# RUN: yaml-bench -canonical %s
-
-| # Simple block scalar
- literal
- 	text
diff --git a/test/YAMLParser/spec-09-25.test b/test/YAMLParser/spec-09-25.test
new file mode 100644
index 00000000000..697b47e9ac7
--- /dev/null
+++ b/test/YAMLParser/spec-09-25.test
@@ -0,0 +1,6 @@
+# RUN: yaml-bench -canonical %s | FileCheck %s
+# CHECK: !!str "literal\n\ttext\n"
+
+| # Simple block scalar
+ literal
+ 	text
diff --git a/test/YAMLParser/spec-09-26.data b/test/YAMLParser/spec-09-26.data
deleted file mode 100644
index 286740ed39c..00000000000
--- a/test/YAMLParser/spec-09-26.data
+++ /dev/null
@@ -1,10 +0,0 @@
-# RUN: yaml-bench -canonical %s
-
-|
- 
-  
-  literal
- 
-  text
-
- # Comment
diff --git a/test/YAMLParser/spec-09-26.test b/test/YAMLParser/spec-09-26.test
new file mode 100644
index 00000000000..c8f31aea2b3
--- /dev/null
+++ b/test/YAMLParser/spec-09-26.test
@@ -0,0 +1,11 @@
+# RUN: yaml-bench -canonical %s | FileCheck %s
+# CHECK: !!str "\n\nliteral\n\ntext\n"
+
+|
+ 
+  
+  literal
+ 
+  text
+
+ # Comment
diff --git a/test/YAMLParser/spec-09-27.data b/test/YAMLParser/spec-09-27.data
deleted file mode 100644
index 286740ed39c..00000000000
--- a/test/YAMLParser/spec-09-27.data
+++ /dev/null
@@ -1,10 +0,0 @@
-# RUN: yaml-bench -canonical %s
-
-|
- 
-  
-  literal
- 
-  text
-
- # Comment
diff --git a/test/YAMLParser/spec-09-28.data b/test/YAMLParser/spec-09-28.data
deleted file mode 100644
index 286740ed39c..00000000000
--- a/test/YAMLParser/spec-09-28.data
+++ /dev/null
@@ -1,10 +0,0 @@
-# RUN: yaml-bench -canonical %s
-
-|
- 
-  
-  literal
- 
-  text
-
- # Comment
diff --git a/test/YAMLParser/spec-09-29.data b/test/YAMLParser/spec-09-29.test
similarity index 100%
rename from test/YAMLParser/spec-09-29.data
rename to test/YAMLParser/spec-09-29.test
diff --git a/test/YAMLParser/spec-09-30.data b/test/YAMLParser/spec-09-30.test
similarity index 100%
rename from test/YAMLParser/spec-09-30.data
rename to test/YAMLParser/spec-09-30.test
diff --git a/test/YAMLParser/spec-09-31.data b/test/YAMLParser/spec-09-31.test
similarity index 100%
rename from test/YAMLParser/spec-09-31.data
rename to test/YAMLParser/spec-09-31.test
diff --git a/test/YAMLParser/spec-09-32.data b/test/YAMLParser/spec-09-32.test
similarity index 100%
rename from test/YAMLParser/spec-09-32.data
rename to test/YAMLParser/spec-09-32.test
diff --git a/test/YAMLParser/spec-09-33.data b/test/YAMLParser/spec-09-33.test
similarity index 100%
rename from test/YAMLParser/spec-09-33.data
rename to test/YAMLParser/spec-09-33.test
diff --git a/test/YAMLParser/spec-10-01.data b/test/YAMLParser/spec-10-01.test
similarity index 100%
rename from test/YAMLParser/spec-10-01.data
rename to test/YAMLParser/spec-10-01.test
diff --git a/test/YAMLParser/spec-10-02.data b/test/YAMLParser/spec-10-02.test
similarity index 100%
rename from test/YAMLParser/spec-10-02.data
rename to test/YAMLParser/spec-10-02.test
diff --git a/test/YAMLParser/spec-10-03.data b/test/YAMLParser/spec-10-03.test
similarity index 100%
rename from test/YAMLParser/spec-10-03.data
rename to test/YAMLParser/spec-10-03.test
diff --git a/test/YAMLParser/spec-10-04.data b/test/YAMLParser/spec-10-04.test
similarity index 100%
rename from test/YAMLParser/spec-10-04.data
rename to test/YAMLParser/spec-10-04.test
diff --git a/test/YAMLParser/spec-10-05.data b/test/YAMLParser/spec-10-05.test
similarity index 100%
rename from test/YAMLParser/spec-10-05.data
rename to test/YAMLParser/spec-10-05.test
diff --git a/test/YAMLParser/spec-10-06.data b/test/YAMLParser/spec-10-06.test
similarity index 100%
rename from test/YAMLParser/spec-10-06.data
rename to test/YAMLParser/spec-10-06.test
diff --git a/test/YAMLParser/spec-10-07.data b/test/YAMLParser/spec-10-07.test
similarity index 100%
rename from test/YAMLParser/spec-10-07.data
rename to test/YAMLParser/spec-10-07.test
diff --git a/test/YAMLParser/spec-10-08.data b/test/YAMLParser/spec-10-08.data
deleted file mode 100644
index 53faeb9e001..00000000000
--- a/test/YAMLParser/spec-10-08.data
+++ /dev/null
@@ -1,13 +0,0 @@
-# RUN: yaml-bench -canonical %s 2>&1 | FileCheck %s
-#
-# This fails because even without a key token, some contexts (in this case flow
-# maps) allow implicit null keys, which mix with this in weird ways.
-# XFAIL: *
-
-{
-multi-line
- simple key : value,
-very long ...................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................(>1KB)................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................... key: value
-}
-
-# CHECK: error
diff --git a/test/YAMLParser/spec-10-08.test b/test/YAMLParser/spec-10-08.test
new file mode 100644
index 00000000000..c4cb8e65ef2
--- /dev/null
+++ b/test/YAMLParser/spec-10-08.test
@@ -0,0 +1,13 @@
+# RUN: yaml-bench -canonical %s 2>&1 | FileCheck %s
+#
+# FIXME: This test should fail. Yaml bench should report an error that a simple
+# key spans across multiple lines and that another simple key is longer than
+# 1024 characters.
+
+{
+multi-line
+ simple key : value,
+very long ...................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................(>1KB)................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................... key: value
+}
+
+# CHECK: ? !!str "very long
diff --git a/test/YAMLParser/spec-10-09.data b/test/YAMLParser/spec-10-09.test
similarity index 100%
rename from test/YAMLParser/spec-10-09.data
rename to test/YAMLParser/spec-10-09.test
diff --git a/test/YAMLParser/spec-10-10.data b/test/YAMLParser/spec-10-10.test
similarity index 100%
rename from test/YAMLParser/spec-10-10.data
rename to test/YAMLParser/spec-10-10.test
diff --git a/test/YAMLParser/spec-10-11.data b/test/YAMLParser/spec-10-11.test
similarity index 100%
rename from test/YAMLParser/spec-10-11.data
rename to test/YAMLParser/spec-10-11.test
diff --git a/test/YAMLParser/spec-10-12.data b/test/YAMLParser/spec-10-12.test
similarity index 100%
rename from test/YAMLParser/spec-10-12.data
rename to test/YAMLParser/spec-10-12.test
diff --git a/test/YAMLParser/spec-10-13.data b/test/YAMLParser/spec-10-13.test
similarity index 100%
rename from test/YAMLParser/spec-10-13.data
rename to test/YAMLParser/spec-10-13.test
diff --git a/test/YAMLParser/spec-10-14.data b/test/YAMLParser/spec-10-14.test
similarity index 100%
rename from test/YAMLParser/spec-10-14.data
rename to test/YAMLParser/spec-10-14.test
diff --git a/test/YAMLParser/spec-10-15.data b/test/YAMLParser/spec-10-15.test
similarity index 100%
rename from test/YAMLParser/spec-10-15.data
rename to test/YAMLParser/spec-10-15.test
diff --git a/test/YAMLParser/str.data b/test/YAMLParser/str.test
similarity index 100%
rename from test/YAMLParser/str.data
rename to test/YAMLParser/str.test
diff --git a/test/YAMLParser/timestamp-bugs.data b/test/YAMLParser/timestamp-bugs.test
similarity index 100%
rename from test/YAMLParser/timestamp-bugs.data
rename to test/YAMLParser/timestamp-bugs.test
diff --git a/test/YAMLParser/timestamp.data b/test/YAMLParser/timestamp.test
similarity index 100%
rename from test/YAMLParser/timestamp.data
rename to test/YAMLParser/timestamp.test
diff --git a/test/YAMLParser/utf8-implicit.data b/test/YAMLParser/utf8-implicit.test
similarity index 100%
rename from test/YAMLParser/utf8-implicit.data
rename to test/YAMLParser/utf8-implicit.test
diff --git a/test/YAMLParser/utf8.data b/test/YAMLParser/utf8.test
similarity index 100%
rename from test/YAMLParser/utf8.data
rename to test/YAMLParser/utf8.test
diff --git a/test/YAMLParser/value.data b/test/YAMLParser/value.test
similarity index 100%
rename from test/YAMLParser/value.data
rename to test/YAMLParser/value.test
diff --git a/test/YAMLParser/yaml.data b/test/YAMLParser/yaml.test
similarity index 100%
rename from test/YAMLParser/yaml.data
rename to test/YAMLParser/yaml.test
diff --git a/test/lit.cfg b/test/lit.cfg
index 75520dc53b6..6d3c4125642 100644
--- a/test/lit.cfg
+++ b/test/lit.cfg
@@ -236,6 +236,7 @@ for pattern in [r"\bbugpoint\b(?!-)",
                 r"\bllvm-dwarfdump\b",
                 r"\bllvm-extract\b",
                 r"\bllvm-go\b",
+                r"\bllvm-lib\b",
                 r"\bllvm-link\b",
                 r"\bllvm-lto\b",
                 r"\bllvm-mc\b",
@@ -251,17 +252,10 @@ for pattern in [r"\bbugpoint\b(?!-)",
                 r"\bllvm-c-test\b",
                 r"\bmacho-dump\b",
                 NOJUNK + r"\bopt\b",
-                # @LOCALMOD-START - match pnacl-* tools
-                r"\bpnacl-llc\b",
-                r"\bpnacl-bcanalyzer\b",
-                r"\bpnacl-bcdis\b",
-                r"\bpnacl-bcfuzz\b",
-                r"\bpnacl-freeze\b",
-                r"\bpnacl-thaw\b",
-                # @LOCALMOD-END
                 r"\bFileCheck\b",
                 r"\bobj2yaml\b",
                 r"\byaml2obj\b",
+                r"\byaml-bench\b",
                 r"\bverify-uselistorder\b",
                 # Handle these specially as they are strings searched
                 # for during testing.
@@ -275,8 +269,12 @@ for pattern in [r"\bbugpoint\b(?!-)",
                           pattern)
     tool_pipe = tool_match.group(2)
     tool_name = tool_match.group(4)
-    tool_path = lit.util.which(tool_name, llvm_tools_dir)
-    if not tool_path:
+    # Did the user specify the tool path + arguments? This allows things like
+    # llvm-lit "-Dllc=llc -enable-misched -verify-machineinstrs"
+    tool_path = lit_config.params.get(tool_name)
+    if tool_path is None:
+        tool_path = lit.util.which(tool_name, llvm_tools_dir)
+    if tool_path is None:
         # Warn, but still provide a substitution.
         lit_config.note('Did not find ' + tool_name + ' in ' + llvm_tools_dir)
         tool_path = llvm_tools_dir + '/' + tool_name
diff --git a/test/tools/dsymutil/ARM/fat-arch-not-found.test b/test/tools/dsymutil/ARM/fat-arch-not-found.test
new file mode 100644
index 00000000000..89b518cd16f
--- /dev/null
+++ b/test/tools/dsymutil/ARM/fat-arch-not-found.test
@@ -0,0 +1,13 @@
+# REQUIRES: object-emission
+# RUN: llvm-dsymutil -oso-prepend-path=%p/../Inputs -y %s -o - 2>&1 | FileCheck %s
+
+---
+triple:          'armv7-apple-darwin'
+objects:
+  - filename: libfat-test.a(fat-test.o)
+    symbols:
+      - { sym: _armv7_var, objAddr: 0x0, binAddr: 0x1000, size: 0x4 }
+...
+
+# CHECK: libfat-test.a(fat-test.o): No object file for requested architecture
+
diff --git a/test/tools/dsymutil/ARM/lit.local.cfg b/test/tools/dsymutil/ARM/lit.local.cfg
new file mode 100644
index 00000000000..236e1d34416
--- /dev/null
+++ b/test/tools/dsymutil/ARM/lit.local.cfg
@@ -0,0 +1,2 @@
+if not 'ARM' in config.root.targets:
+    config.unsupported = True
diff --git a/test/tools/dsymutil/Inputs/fat-test.c b/test/tools/dsymutil/Inputs/fat-test.c
new file mode 100644
index 00000000000..42c1fce1ec3
--- /dev/null
+++ b/test/tools/dsymutil/Inputs/fat-test.c
@@ -0,0 +1,17 @@
+/* Compile with:
+   clang -c -g -arch x86_64h -arch x86_64 -arch i386 fat-test.c
+   libtool -static -o libfat-test.a fat-test.o
+
+   To reduce the size of the fat .o:
+   lipo -thin i386 -o fat-test.i386.o fat-test.o 
+   lipo -thin x86_64 -o fat-test.x86_64.o fat-test.o 
+   lipo -thin x86_64h -o fat-test.x86_64h.o fat-test.o 
+   lipo -create -arch x86_64h fat-test.x86_64h.o -arch x86_64 fat-test.x86_64.o -arch i386 fat-test.i386.o -o fat-test.o -segalign i386 8 -segalign x86_64 8 -segalign x86_64h 8
+ */
+#ifdef __x86_64h__
+int x86_64h_var;
+#elif defined(__x86_64__)
+int x86_64_var;
+#else
+int i386_var;
+#endif
diff --git a/test/tools/dsymutil/Inputs/fat-test.o b/test/tools/dsymutil/Inputs/fat-test.o
new file mode 100644
index 00000000000..8159cc74977
Binary files /dev/null and b/test/tools/dsymutil/Inputs/fat-test.o differ
diff --git a/test/tools/dsymutil/Inputs/frame-dw2.ll b/test/tools/dsymutil/Inputs/frame-dw2.ll
new file mode 100644
index 00000000000..7ffc9339731
--- /dev/null
+++ b/test/tools/dsymutil/Inputs/frame-dw2.ll
@@ -0,0 +1,71 @@
+; Generated from frame.c on Darwin with '-arch i386 -g -emit-llvm'
+; ModuleID = 'frame.c'
+target datalayout = "e-m:o-p:32:32-f64:32:64-f80:128-n8:16:32-S128"
+target triple = "i386-apple-macosx10.11.0"
+
+; Function Attrs: nounwind ssp
+define i32 @bar(i32 %b) #0 {
+entry:
+  %b.addr = alloca i32, align 4
+  %var = alloca i32, align 4
+  store i32 %b, i32* %b.addr, align 4
+  call void @llvm.dbg.declare(metadata i32* %b.addr, metadata !13, metadata !14), !dbg !15
+  call void @llvm.dbg.declare(metadata i32* %var, metadata !16, metadata !14), !dbg !17
+  %0 = load i32, i32* %b.addr, align 4, !dbg !18
+  %add = add nsw i32 %0, 1, !dbg !19
+  store i32 %add, i32* %var, align 4, !dbg !17
+  %call = call i32 @foo(i32* %var), !dbg !20
+  ret i32 %call, !dbg !21
+}
+
+; Function Attrs: nounwind readnone
+declare void @llvm.dbg.declare(metadata, metadata, metadata) #1
+
+declare i32 @foo(i32*) #2
+
+; Function Attrs: nounwind ssp
+define i32 @baz(i32 %b) #0 {
+entry:
+  %b.addr = alloca i32, align 4
+  store i32 %b, i32* %b.addr, align 4
+  call void @llvm.dbg.declare(metadata i32* %b.addr, metadata !22, metadata !14), !dbg !23
+  %0 = load i32, i32* %b.addr, align 4, !dbg !24
+  %call = call i32 @bar(i32 %0), !dbg !25
+  ret i32 %call, !dbg !26
+}
+
+attributes #0 = { nounwind ssp "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="yonah" "target-features"="+cx16,+sse,+sse2,+sse3" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #1 = { nounwind readnone }
+attributes #2 = { "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="yonah" "target-features"="+cx16,+sse,+sse2,+sse3" "unsafe-fp-math"="false" "use-soft-float"="false" }
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!9, !10, !11}
+!llvm.ident = !{!12}
+
+!0 = !DICompileUnit(language: DW_LANG_C99, file: !1, producer: "clang version 3.7.0 (trunk 239176) (llvm/trunk 239190)", isOptimized: false, runtimeVersion: 0, emissionKind: 1, enums: !2, retainedTypes: !2, subprograms: !3, globals: !2, imports: !2)
+!1 = !DIFile(filename: "frame.c", directory: "/tmp")
+!2 = !{}
+!3 = !{!4, !8}
+!4 = !DISubprogram(name: "bar", scope: !1, file: !1, line: 3, type: !5, isLocal: false, isDefinition: true, scopeLine: 3, flags: DIFlagPrototyped, isOptimized: false, function: i32 (i32)* @bar, variables: !2)
+!5 = !DISubroutineType(types: !6)
+!6 = !{!7, !7}
+!7 = !DIBasicType(name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
+!8 = !DISubprogram(name: "baz", scope: !1, file: !1, line: 8, type: !5, isLocal: false, isDefinition: true, scopeLine: 8, flags: DIFlagPrototyped, isOptimized: false, function: i32 (i32)* @baz, variables: !2)
+!9 = !{i32 2, !"Dwarf Version", i32 2}
+!10 = !{i32 2, !"Debug Info Version", i32 3}
+!11 = !{i32 1, !"PIC Level", i32 2}
+!12 = !{!"clang version 3.7.0 (trunk 239176) (llvm/trunk 239190)"}
+!13 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "b", arg: 1, scope: !4, file: !1, line: 3, type: !7)
+!14 = !DIExpression()
+!15 = !DILocation(line: 3, column: 13, scope: !4)
+!16 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "var", scope: !4, file: !1, line: 4, type: !7)
+!17 = !DILocation(line: 4, column: 6, scope: !4)
+!18 = !DILocation(line: 4, column: 12, scope: !4)
+!19 = !DILocation(line: 4, column: 14, scope: !4)
+!20 = !DILocation(line: 5, column: 9, scope: !4)
+!21 = !DILocation(line: 5, column: 2, scope: !4)
+!22 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "b", arg: 1, scope: !8, file: !1, line: 8, type: !7)
+!23 = !DILocation(line: 8, column: 13, scope: !8)
+!24 = !DILocation(line: 9, column: 13, scope: !8)
+!25 = !DILocation(line: 9, column: 9, scope: !8)
+!26 = !DILocation(line: 9, column: 2, scope: !8)
diff --git a/test/tools/dsymutil/Inputs/frame-dw4.ll b/test/tools/dsymutil/Inputs/frame-dw4.ll
new file mode 100644
index 00000000000..c8674b13e58
--- /dev/null
+++ b/test/tools/dsymutil/Inputs/frame-dw4.ll
@@ -0,0 +1,71 @@
+; Generated from frame.c on Darwin with '-arch i386 -gdwarf-4 -emit-llvm'
+; ModuleID = 'frame.c'
+target datalayout = "e-m:o-p:32:32-f64:32:64-f80:128-n8:16:32-S128"
+target triple = "i386-apple-macosx10.11.0"
+
+; Function Attrs: nounwind ssp
+define i32 @bar(i32 %b) #0 {
+entry:
+  %b.addr = alloca i32, align 4
+  %var = alloca i32, align 4
+  store i32 %b, i32* %b.addr, align 4
+  call void @llvm.dbg.declare(metadata i32* %b.addr, metadata !13, metadata !14), !dbg !15
+  call void @llvm.dbg.declare(metadata i32* %var, metadata !16, metadata !14), !dbg !17
+  %0 = load i32, i32* %b.addr, align 4, !dbg !18
+  %add = add nsw i32 %0, 1, !dbg !19
+  store i32 %add, i32* %var, align 4, !dbg !17
+  %call = call i32 @foo(i32* %var), !dbg !20
+  ret i32 %call, !dbg !21
+}
+
+; Function Attrs: nounwind readnone
+declare void @llvm.dbg.declare(metadata, metadata, metadata) #1
+
+declare i32 @foo(i32*) #2
+
+; Function Attrs: nounwind ssp
+define i32 @baz(i32 %b) #0 {
+entry:
+  %b.addr = alloca i32, align 4
+  store i32 %b, i32* %b.addr, align 4
+  call void @llvm.dbg.declare(metadata i32* %b.addr, metadata !22, metadata !14), !dbg !23
+  %0 = load i32, i32* %b.addr, align 4, !dbg !24
+  %call = call i32 @bar(i32 %0), !dbg !25
+  ret i32 %call, !dbg !26
+}
+
+attributes #0 = { nounwind ssp "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="yonah" "target-features"="+cx16,+sse,+sse2,+sse3" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #1 = { nounwind readnone }
+attributes #2 = { "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="yonah" "target-features"="+cx16,+sse,+sse2,+sse3" "unsafe-fp-math"="false" "use-soft-float"="false" }
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!9, !10, !11}
+!llvm.ident = !{!12}
+
+!0 = !DICompileUnit(language: DW_LANG_C99, file: !1, producer: "clang version 3.7.0 (trunk 239176) (llvm/trunk 239190)", isOptimized: false, runtimeVersion: 0, emissionKind: 1, enums: !2, retainedTypes: !2, subprograms: !3, globals: !2, imports: !2)
+!1 = !DIFile(filename: "frame.c", directory: "/tmp")
+!2 = !{}
+!3 = !{!4, !8}
+!4 = !DISubprogram(name: "bar", scope: !1, file: !1, line: 3, type: !5, isLocal: false, isDefinition: true, scopeLine: 3, flags: DIFlagPrototyped, isOptimized: false, function: i32 (i32)* @bar, variables: !2)
+!5 = !DISubroutineType(types: !6)
+!6 = !{!7, !7}
+!7 = !DIBasicType(name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
+!8 = !DISubprogram(name: "baz", scope: !1, file: !1, line: 8, type: !5, isLocal: false, isDefinition: true, scopeLine: 8, flags: DIFlagPrototyped, isOptimized: false, function: i32 (i32)* @baz, variables: !2)
+!9 = !{i32 2, !"Dwarf Version", i32 4}
+!10 = !{i32 2, !"Debug Info Version", i32 3}
+!11 = !{i32 1, !"PIC Level", i32 2}
+!12 = !{!"clang version 3.7.0 (trunk 239176) (llvm/trunk 239190)"}
+!13 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "b", arg: 1, scope: !4, file: !1, line: 3, type: !7)
+!14 = !DIExpression()
+!15 = !DILocation(line: 3, column: 13, scope: !4)
+!16 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "var", scope: !4, file: !1, line: 4, type: !7)
+!17 = !DILocation(line: 4, column: 6, scope: !4)
+!18 = !DILocation(line: 4, column: 12, scope: !4)
+!19 = !DILocation(line: 4, column: 14, scope: !4)
+!20 = !DILocation(line: 5, column: 9, scope: !4)
+!21 = !DILocation(line: 5, column: 2, scope: !4)
+!22 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "b", arg: 1, scope: !8, file: !1, line: 8, type: !7)
+!23 = !DILocation(line: 8, column: 13, scope: !8)
+!24 = !DILocation(line: 9, column: 13, scope: !8)
+!25 = !DILocation(line: 9, column: 9, scope: !8)
+!26 = !DILocation(line: 9, column: 2, scope: !8)
diff --git a/test/tools/dsymutil/Inputs/frame.c b/test/tools/dsymutil/Inputs/frame.c
new file mode 100644
index 00000000000..9ca082d4ae5
--- /dev/null
+++ b/test/tools/dsymutil/Inputs/frame.c
@@ -0,0 +1,10 @@
+int foo(int *f);
+
+int bar(int b) {
+	int var = b + 1;
+	return foo(&var);
+}
+
+int baz(int b) {
+	return bar(b);
+}
diff --git a/test/tools/dsymutil/Inputs/libfat-test.a b/test/tools/dsymutil/Inputs/libfat-test.a
new file mode 100644
index 00000000000..6b34b0fd6f9
Binary files /dev/null and b/test/tools/dsymutil/Inputs/libfat-test.a differ
diff --git a/test/tools/dsymutil/Inputs/odr-types.h b/test/tools/dsymutil/Inputs/odr-types.h
new file mode 100644
index 00000000000..09057dc9683
--- /dev/null
+++ b/test/tools/dsymutil/Inputs/odr-types.h
@@ -0,0 +1,51 @@
+struct S {
+	int I;
+
+	void incr() __attribute__((always_inline)) { I++; }
+	void incr(int Add) __attribute__((always_inline)) { I += Add; }
+
+	typedef int SInt;
+
+	struct Nested {
+		double D;
+
+		template<typename T> void init(T Val) { D = double(Val); }
+	};
+
+	Nested D;
+
+public:
+	int foo() { return I; }
+};
+
+typedef S AliasForS;
+
+namespace N {
+class C {
+	AliasForS S;
+};
+}
+
+namespace N {
+namespace N {
+class C {
+        int S;
+};
+}
+}
+
+namespace {
+	class AnonC {
+	};
+}
+
+union U {
+	class C {} C;
+	struct S {} S;
+};
+
+inline int func() {
+	struct CInsideFunc { int i; };
+	auto functor = []() { CInsideFunc dummy; return dummy.i; };
+	return functor();
+}
diff --git a/test/tools/dsymutil/Inputs/odr1.cpp b/test/tools/dsymutil/Inputs/odr1.cpp
new file mode 100644
index 00000000000..722247b2504
--- /dev/null
+++ b/test/tools/dsymutil/Inputs/odr1.cpp
@@ -0,0 +1,11 @@
+#include "odr-types.h"
+
+int foo() {
+	AliasForS s;
+	N::C nc;
+	N::N::C nnc;
+	AnonC ac;
+	U u;
+
+	return func();
+}
diff --git a/test/tools/dsymutil/Inputs/odr1.ll b/test/tools/dsymutil/Inputs/odr1.ll
new file mode 100644
index 00000000000..818e0effa3e
--- /dev/null
+++ b/test/tools/dsymutil/Inputs/odr1.ll
@@ -0,0 +1,153 @@
+; Generated from odr1.cpp and odr-types.h by running:
+; clang -emit-llvm -g -S -std=c++11 odr1.cpp
+; ModuleID = 'odr1.cpp'
+target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-apple-macosx10.11.0"
+
+%struct.S = type { i32, %"struct.S::Nested" }
+%"struct.S::Nested" = type { double }
+%"class.N::C" = type { %struct.S }
+%"class.N::N::C" = type { i32 }
+%"class.(anonymous namespace)::AnonC" = type { i8 }
+%union.U = type { %"class.U::C" }
+%"class.U::C" = type { i8 }
+%class.anon = type { i8 }
+%struct.CInsideFunc = type { i32 }
+
+; Function Attrs: ssp uwtable
+define i32 @_Z3foov() #0 {
+entry:
+  %s = alloca %struct.S, align 8
+  %nc = alloca %"class.N::C", align 8
+  %nnc = alloca %"class.N::N::C", align 4
+  %ac = alloca %"class.(anonymous namespace)::AnonC", align 1
+  %u = alloca %union.U, align 1
+  call void @llvm.dbg.declare(metadata %struct.S* %s, metadata !59, metadata !60), !dbg !61
+  call void @llvm.dbg.declare(metadata %"class.N::C"* %nc, metadata !62, metadata !60), !dbg !63
+  call void @llvm.dbg.declare(metadata %"class.N::N::C"* %nnc, metadata !64, metadata !60), !dbg !65
+  call void @llvm.dbg.declare(metadata %"class.(anonymous namespace)::AnonC"* %ac, metadata !66, metadata !60), !dbg !69
+  call void @llvm.dbg.declare(metadata %union.U* %u, metadata !70, metadata !60), !dbg !71
+  %call = call i32 @_Z4funcv(), !dbg !72
+  ret i32 %call, !dbg !73
+}
+
+; Function Attrs: nounwind readnone
+declare void @llvm.dbg.declare(metadata, metadata, metadata) #1
+
+; Function Attrs: inlinehint ssp uwtable
+define linkonce_odr i32 @_Z4funcv() #2 {
+entry:
+  %functor = alloca %class.anon, align 1
+  call void @llvm.dbg.declare(metadata %class.anon* %functor, metadata !74, metadata !60), !dbg !75
+  %call = call i32 @_ZZ4funcvENKUlvE_clEv(%class.anon* %functor), !dbg !76
+  ret i32 %call, !dbg !77
+}
+
+; Function Attrs: inlinehint nounwind ssp uwtable
+define linkonce_odr i32 @_ZZ4funcvENKUlvE_clEv(%class.anon* %this) #3 align 2 {
+entry:
+  %this.addr = alloca %class.anon*, align 8
+  %dummy = alloca %struct.CInsideFunc, align 4
+  store %class.anon* %this, %class.anon** %this.addr, align 8
+  call void @llvm.dbg.declare(metadata %class.anon** %this.addr, metadata !78, metadata !60), !dbg !80
+  %this1 = load %class.anon*, %class.anon** %this.addr
+  call void @llvm.dbg.declare(metadata %struct.CInsideFunc* %dummy, metadata !81, metadata !60), !dbg !82
+  %i = getelementptr inbounds %struct.CInsideFunc, %struct.CInsideFunc* %dummy, i32 0, i32 0, !dbg !83
+  %0 = load i32, i32* %i, align 4, !dbg !83
+  ret i32 %0, !dbg !84
+}
+
+attributes #0 = { ssp uwtable "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="core2" "target-features"="+cx16,+sse,+sse2,+sse3,+ssse3" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #1 = { nounwind readnone }
+attributes #2 = { inlinehint ssp uwtable "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="core2" "target-features"="+cx16,+sse,+sse2,+sse3,+ssse3" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #3 = { inlinehint nounwind ssp uwtable "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="core2" "target-features"="+cx16,+sse,+sse2,+sse3,+ssse3" "unsafe-fp-math"="false" "use-soft-float"="false" }
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!55, !56, !57}
+!llvm.ident = !{!58}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C_plus_plus, file: !1, producer: "clang version 3.8.0 (trunk 242534)", isOptimized: false, runtimeVersion: 0, emissionKind: 1, enums: !2, retainedTypes: !3, subprograms: !52)
+!1 = !DIFile(filename: "odr1.cpp", directory: "/Inputs")
+!2 = !{}
+!3 = !{!4, !20, !24, !29, !33, !37, !38, !39, !49}
+!4 = !DICompositeType(tag: DW_TAG_structure_type, name: "S", file: !5, line: 1, size: 128, align: 64, elements: !6, identifier: "_ZTS1S")
+!5 = !DIFile(filename: "./odr-types.h", directory: "/Inputs")
+!6 = !{!7, !9, !10, !14, !17}
+!7 = !DIDerivedType(tag: DW_TAG_member, name: "I", scope: !"_ZTS1S", file: !5, line: 2, baseType: !8, size: 32, align: 32)
+!8 = !DIBasicType(name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
+!9 = !DIDerivedType(tag: DW_TAG_member, name: "D", scope: !"_ZTS1S", file: !5, line: 15, baseType: !"_ZTSN1S6NestedE", size: 64, align: 64, offset: 64)
+!10 = !DISubprogram(name: "incr", linkageName: "_ZN1S4incrEv", scope: !"_ZTS1S", file: !5, line: 4, type: !11, isLocal: false, isDefinition: false, scopeLine: 4, flags: DIFlagPrototyped, isOptimized: false)
+!11 = !DISubroutineType(types: !12)
+!12 = !{null, !13}
+!13 = !DIDerivedType(tag: DW_TAG_pointer_type, baseType: !"_ZTS1S", size: 64, align: 64, flags: DIFlagArtificial | DIFlagObjectPointer)
+!14 = !DISubprogram(name: "incr", linkageName: "_ZN1S4incrEi", scope: !"_ZTS1S", file: !5, line: 5, type: !15, isLocal: false, isDefinition: false, scopeLine: 5, flags: DIFlagPrototyped, isOptimized: false)
+!15 = !DISubroutineType(types: !16)
+!16 = !{null, !13, !8}
+!17 = !DISubprogram(name: "foo", linkageName: "_ZN1S3fooEv", scope: !"_ZTS1S", file: !5, line: 18, type: !18, isLocal: false, isDefinition: false, scopeLine: 18, flags: DIFlagPrototyped, isOptimized: false)
+!18 = !DISubroutineType(types: !19)
+!19 = !{!8, !13}
+!20 = !DICompositeType(tag: DW_TAG_structure_type, name: "Nested", scope: !"_ZTS1S", file: !5, line: 9, size: 64, align: 64, elements: !21, identifier: "_ZTSN1S6NestedE")
+!21 = !{!22}
+!22 = !DIDerivedType(tag: DW_TAG_member, name: "D", scope: !"_ZTSN1S6NestedE", file: !5, line: 10, baseType: !23, size: 64, align: 64)
+!23 = !DIBasicType(name: "double", size: 64, align: 64, encoding: DW_ATE_float)
+!24 = !DICompositeType(tag: DW_TAG_class_type, name: "C", scope: !25, file: !5, line: 24, size: 128, align: 64, elements: !26, identifier: "_ZTSN1N1CE")
+!25 = !DINamespace(name: "N", scope: null, file: !5, line: 23)
+!26 = !{!27}
+!27 = !DIDerivedType(tag: DW_TAG_member, name: "S", scope: !"_ZTSN1N1CE", file: !5, line: 25, baseType: !28, size: 128, align: 64)
+!28 = !DIDerivedType(tag: DW_TAG_typedef, name: "AliasForS", file: !5, line: 21, baseType: !"_ZTS1S")
+!29 = !DICompositeType(tag: DW_TAG_class_type, name: "C", scope: !30, file: !5, line: 31, size: 32, align: 32, elements: !31, identifier: "_ZTSN1N1N1CE")
+!30 = !DINamespace(name: "N", scope: !25, file: !5, line: 30)
+!31 = !{!32}
+!32 = !DIDerivedType(tag: DW_TAG_member, name: "S", scope: !"_ZTSN1N1N1CE", file: !5, line: 32, baseType: !8, size: 32, align: 32)
+!33 = !DICompositeType(tag: DW_TAG_union_type, name: "U", file: !5, line: 42, size: 8, align: 8, elements: !34, identifier: "_ZTS1U")
+!34 = !{!35, !36}
+!35 = !DIDerivedType(tag: DW_TAG_member, name: "C", scope: !"_ZTS1U", file: !5, line: 43, baseType: !"_ZTSN1U1CE", size: 8, align: 8)
+!36 = !DIDerivedType(tag: DW_TAG_member, name: "S", scope: !"_ZTS1U", file: !5, line: 44, baseType: !"_ZTSN1U1SE", size: 8, align: 8)
+!37 = !DICompositeType(tag: DW_TAG_class_type, name: "C", scope: !"_ZTS1U", file: !5, line: 43, size: 8, align: 8, elements: !2, identifier: "_ZTSN1U1CE")
+!38 = !DICompositeType(tag: DW_TAG_structure_type, name: "S", scope: !"_ZTS1U", file: !5, line: 44, size: 8, align: 8, elements: !2, identifier: "_ZTSN1U1SE")
+!39 = !DICompositeType(tag: DW_TAG_class_type, scope: !40, file: !5, line: 49, size: 8, align: 8, elements: !43, identifier: "_ZTSZ4funcvEUlvE_")
+!40 = !DISubprogram(name: "func", linkageName: "_Z4funcv", scope: !5, file: !5, line: 47, type: !41, isLocal: false, isDefinition: true, scopeLine: 47, flags: DIFlagPrototyped, isOptimized: false, function: i32 ()* @_Z4funcv, variables: !2)
+!41 = !DISubroutineType(types: !42)
+!42 = !{!8}
+!43 = !{!44}
+!44 = !DISubprogram(name: "operator()", scope: !"_ZTSZ4funcvEUlvE_", file: !5, line: 49, type: !45, isLocal: false, isDefinition: false, scopeLine: 49, flags: DIFlagPublic | DIFlagPrototyped, isOptimized: false)
+!45 = !DISubroutineType(types: !46)
+!46 = !{!8, !47}
+!47 = !DIDerivedType(tag: DW_TAG_pointer_type, baseType: !48, size: 64, align: 64, flags: DIFlagArtificial | DIFlagObjectPointer)
+!48 = !DIDerivedType(tag: DW_TAG_const_type, baseType: !"_ZTSZ4funcvEUlvE_")
+!49 = !DICompositeType(tag: DW_TAG_structure_type, name: "CInsideFunc", scope: !40, file: !5, line: 48, size: 32, align: 32, elements: !50, identifier: "_ZTSZ4funcvE11CInsideFunc")
+!50 = !{!51}
+!51 = !DIDerivedType(tag: DW_TAG_member, name: "i", scope: !"_ZTSZ4funcvE11CInsideFunc", file: !5, line: 48, baseType: !8, size: 32, align: 32)
+!52 = !{!53, !40, !54}
+!53 = !DISubprogram(name: "foo", linkageName: "_Z3foov", scope: !1, file: !1, line: 3, type: !41, isLocal: false, isDefinition: true, scopeLine: 3, flags: DIFlagPrototyped, isOptimized: false, function: i32 ()* @_Z3foov, variables: !2)
+!54 = !DISubprogram(name: "operator()", linkageName: "_ZZ4funcvENKUlvE_clEv", scope: !"_ZTSZ4funcvEUlvE_", file: !5, line: 49, type: !45, isLocal: false, isDefinition: true, scopeLine: 49, flags: DIFlagPrototyped, isOptimized: false, function: i32 (%class.anon*)* @_ZZ4funcvENKUlvE_clEv, declaration: !44, variables: !2)
+!55 = !{i32 2, !"Dwarf Version", i32 2}
+!56 = !{i32 2, !"Debug Info Version", i32 3}
+!57 = !{i32 1, !"PIC Level", i32 2}
+!58 = !{!"clang version 3.8.0 (trunk 242534)"}
+!59 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "s", scope: !53, file: !1, line: 4, type: !28)
+!60 = !DIExpression()
+!61 = !DILocation(line: 4, column: 12, scope: !53)
+!62 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "nc", scope: !53, file: !1, line: 5, type: !"_ZTSN1N1CE")
+!63 = !DILocation(line: 5, column: 7, scope: !53)
+!64 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "nnc", scope: !53, file: !1, line: 6, type: !"_ZTSN1N1N1CE")
+!65 = !DILocation(line: 6, column: 10, scope: !53)
+!66 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "ac", scope: !53, file: !1, line: 7, type: !67)
+!67 = !DICompositeType(tag: DW_TAG_class_type, name: "AnonC", scope: !68, file: !5, line: 38, size: 8, align: 8, elements: !2)
+!68 = !DINamespace(scope: null, file: !5, line: 37)
+!69 = !DILocation(line: 7, column: 8, scope: !53)
+!70 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "u", scope: !53, file: !1, line: 8, type: !"_ZTS1U")
+!71 = !DILocation(line: 8, column: 4, scope: !53)
+!72 = !DILocation(line: 10, column: 9, scope: !53)
+!73 = !DILocation(line: 10, column: 2, scope: !53)
+!74 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "functor", scope: !40, file: !5, line: 49, type: !"_ZTSZ4funcvEUlvE_")
+!75 = !DILocation(line: 49, column: 7, scope: !40)
+!76 = !DILocation(line: 50, column: 9, scope: !40)
+!77 = !DILocation(line: 50, column: 2, scope: !40)
+!78 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "this", arg: 1, scope: !54, type: !79, flags: DIFlagArtificial | DIFlagObjectPointer)
+!79 = !DIDerivedType(tag: DW_TAG_pointer_type, baseType: !48, size: 64, align: 64)
+!80 = !DILocation(line: 0, scope: !54)
+!81 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "dummy", scope: !54, file: !5, line: 49, type: !"_ZTSZ4funcvE11CInsideFunc")
+!82 = !DILocation(line: 49, column: 36, scope: !54)
+!83 = !DILocation(line: 49, column: 56, scope: !54)
+!84 = !DILocation(line: 49, column: 43, scope: !54)
diff --git a/test/tools/dsymutil/Inputs/odr2.cpp b/test/tools/dsymutil/Inputs/odr2.cpp
new file mode 100644
index 00000000000..4c26fb2d677
--- /dev/null
+++ b/test/tools/dsymutil/Inputs/odr2.cpp
@@ -0,0 +1,7 @@
+#include "odr-types.h"
+
+int bar() {
+	S s;
+	s.incr();
+	return s.foo();
+}
diff --git a/test/tools/dsymutil/Inputs/odr2.ll b/test/tools/dsymutil/Inputs/odr2.ll
new file mode 100644
index 00000000000..da66a2d7153
--- /dev/null
+++ b/test/tools/dsymutil/Inputs/odr2.ll
@@ -0,0 +1,99 @@
+; Generated from odr2.cpp and odr-types.h by running:
+; clang -emit-llvm -g -S -std=c++11 odr2.cpp
+; ModuleID = 'odr2.cpp'
+target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-apple-macosx10.11.0"
+
+%struct.S = type { i32, %"struct.S::Nested" }
+%"struct.S::Nested" = type { double }
+
+; Function Attrs: ssp uwtable
+define i32 @_Z3barv() #0 {
+entry:
+  %this.addr.i = alloca %struct.S*, align 8
+  %s = alloca %struct.S, align 8
+  call void @llvm.dbg.declare(metadata %struct.S* %s, metadata !34, metadata !35), !dbg !36
+  store %struct.S* %s, %struct.S** %this.addr.i, align 8, !dbg !37
+  %this1.i = load %struct.S*, %struct.S** %this.addr.i, !dbg !37
+  %I.i = getelementptr inbounds %struct.S, %struct.S* %this1.i, i32 0, i32 0, !dbg !38
+  %0 = load i32, i32* %I.i, align 4, !dbg !40
+  %inc.i = add nsw i32 %0, 1, !dbg !40
+  store i32 %inc.i, i32* %I.i, align 4, !dbg !40
+  %call = call i32 @_ZN1S3fooEv(%struct.S* %s), !dbg !41
+  call void @llvm.dbg.declare(metadata %struct.S** %this.addr.i, metadata !42, metadata !35), !dbg !44
+  ret i32 %call, !dbg !45
+}
+
+; Function Attrs: nounwind readnone
+declare void @llvm.dbg.declare(metadata, metadata, metadata) #1
+
+; Function Attrs: nounwind ssp uwtable
+define linkonce_odr i32 @_ZN1S3fooEv(%struct.S* %this) #2 align 2 {
+entry:
+  %this.addr = alloca %struct.S*, align 8
+  store %struct.S* %this, %struct.S** %this.addr, align 8
+  call void @llvm.dbg.declare(metadata %struct.S** %this.addr, metadata !46, metadata !35), !dbg !47
+  %this1 = load %struct.S*, %struct.S** %this.addr
+  %I = getelementptr inbounds %struct.S, %struct.S* %this1, i32 0, i32 0, !dbg !48
+  %0 = load i32, i32* %I, align 4, !dbg !48
+  ret i32 %0, !dbg !49
+}
+
+attributes #0 = { ssp uwtable "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="core2" "target-features"="+cx16,+sse,+sse2,+sse3,+ssse3" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #1 = { nounwind readnone }
+attributes #2 = { nounwind ssp uwtable "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="core2" "target-features"="+cx16,+sse,+sse2,+sse3,+ssse3" "unsafe-fp-math"="false" "use-soft-float"="false" }
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!30, !31, !32}
+!llvm.ident = !{!33}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C_plus_plus, file: !1, producer: "clang version 3.8.0 (trunk 242534)", isOptimized: false, runtimeVersion: 0, emissionKind: 1, enums: !2, retainedTypes: !3, subprograms: !24)
+!1 = !DIFile(filename: "odr2.cpp", directory: "/Inputs")
+!2 = !{}
+!3 = !{!4, !20}
+!4 = !DICompositeType(tag: DW_TAG_structure_type, name: "S", file: !5, line: 1, size: 128, align: 64, elements: !6, identifier: "_ZTS1S")
+!5 = !DIFile(filename: "./odr-types.h", directory: "/Inputs")
+!6 = !{!7, !9, !10, !14, !17}
+!7 = !DIDerivedType(tag: DW_TAG_member, name: "I", scope: !"_ZTS1S", file: !5, line: 2, baseType: !8, size: 32, align: 32)
+!8 = !DIBasicType(name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
+!9 = !DIDerivedType(tag: DW_TAG_member, name: "D", scope: !"_ZTS1S", file: !5, line: 15, baseType: !"_ZTSN1S6NestedE", size: 64, align: 64, offset: 64)
+!10 = !DISubprogram(name: "incr", linkageName: "_ZN1S4incrEv", scope: !"_ZTS1S", file: !5, line: 4, type: !11, isLocal: false, isDefinition: false, scopeLine: 4, flags: DIFlagPrototyped, isOptimized: false)
+!11 = !DISubroutineType(types: !12)
+!12 = !{null, !13}
+!13 = !DIDerivedType(tag: DW_TAG_pointer_type, baseType: !"_ZTS1S", size: 64, align: 64, flags: DIFlagArtificial | DIFlagObjectPointer)
+!14 = !DISubprogram(name: "incr", linkageName: "_ZN1S4incrEi", scope: !"_ZTS1S", file: !5, line: 5, type: !15, isLocal: false, isDefinition: false, scopeLine: 5, flags: DIFlagPrototyped, isOptimized: false)
+!15 = !DISubroutineType(types: !16)
+!16 = !{null, !13, !8}
+!17 = !DISubprogram(name: "foo", linkageName: "_ZN1S3fooEv", scope: !"_ZTS1S", file: !5, line: 18, type: !18, isLocal: false, isDefinition: false, scopeLine: 18, flags: DIFlagPrototyped, isOptimized: false)
+!18 = !DISubroutineType(types: !19)
+!19 = !{!8, !13}
+!20 = !DICompositeType(tag: DW_TAG_structure_type, name: "Nested", scope: !"_ZTS1S", file: !5, line: 9, size: 64, align: 64, elements: !21, identifier: "_ZTSN1S6NestedE")
+!21 = !{!22}
+!22 = !DIDerivedType(tag: DW_TAG_member, name: "D", scope: !"_ZTSN1S6NestedE", file: !5, line: 10, baseType: !23, size: 64, align: 64)
+!23 = !DIBasicType(name: "double", size: 64, align: 64, encoding: DW_ATE_float)
+!24 = !{!25, !28, !29}
+!25 = !DISubprogram(name: "bar", linkageName: "_Z3barv", scope: !1, file: !1, line: 3, type: !26, isLocal: false, isDefinition: true, scopeLine: 3, flags: DIFlagPrototyped, isOptimized: false, function: i32 ()* @_Z3barv, variables: !2)
+!26 = !DISubroutineType(types: !27)
+!27 = !{!8}
+!28 = !DISubprogram(name: "incr", linkageName: "_ZN1S4incrEv", scope: !"_ZTS1S", file: !5, line: 4, type: !11, isLocal: false, isDefinition: true, scopeLine: 4, flags: DIFlagPrototyped, isOptimized: false, declaration: !10, variables: !2)
+!29 = !DISubprogram(name: "foo", linkageName: "_ZN1S3fooEv", scope: !"_ZTS1S", file: !5, line: 18, type: !18, isLocal: false, isDefinition: true, scopeLine: 18, flags: DIFlagPrototyped, isOptimized: false, function: i32 (%struct.S*)* @_ZN1S3fooEv, declaration: !17, variables: !2)
+!30 = !{i32 2, !"Dwarf Version", i32 2}
+!31 = !{i32 2, !"Debug Info Version", i32 3}
+!32 = !{i32 1, !"PIC Level", i32 2}
+!33 = !{!"clang version 3.8.0 (trunk 242534)"}
+!34 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "s", scope: !25, file: !1, line: 4, type: !"_ZTS1S")
+!35 = !DIExpression()
+!36 = !DILocation(line: 4, column: 4, scope: !25)
+!37 = !DILocation(line: 5, column: 2, scope: !25)
+!38 = !DILocation(line: 4, column: 47, scope: !28, inlinedAt: !39)
+!39 = distinct !DILocation(line: 5, column: 2, scope: !25)
+!40 = !DILocation(line: 4, column: 48, scope: !28, inlinedAt: !39)
+!41 = !DILocation(line: 6, column: 9, scope: !25)
+!42 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "this", arg: 1, scope: !28, type: !43, flags: DIFlagArtificial | DIFlagObjectPointer)
+!43 = !DIDerivedType(tag: DW_TAG_pointer_type, baseType: !"_ZTS1S", size: 64, align: 64)
+!44 = !DILocation(line: 0, scope: !28, inlinedAt: !39)
+!45 = !DILocation(line: 6, column: 2, scope: !25)
+!46 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "this", arg: 1, scope: !29, type: !43, flags: DIFlagArtificial | DIFlagObjectPointer)
+!47 = !DILocation(line: 0, scope: !29)
+!48 = !DILocation(line: 18, column: 21, scope: !29)
+!49 = !DILocation(line: 18, column: 14, scope: !29)
diff --git a/test/tools/dsymutil/Inputs/odr3.cpp b/test/tools/dsymutil/Inputs/odr3.cpp
new file mode 100644
index 00000000000..1200c670a48
--- /dev/null
+++ b/test/tools/dsymutil/Inputs/odr3.cpp
@@ -0,0 +1,7 @@
+#include "odr-types.h"
+
+int bar() {
+	S s;
+	s.incr(42);
+	return s.foo();
+}
diff --git a/test/tools/dsymutil/Inputs/odr3.ll b/test/tools/dsymutil/Inputs/odr3.ll
new file mode 100644
index 00000000000..d0fd23963c2
--- /dev/null
+++ b/test/tools/dsymutil/Inputs/odr3.ll
@@ -0,0 +1,106 @@
+; Generated from odr3.cpp and odr-types.h by running:
+; clang -emit-llvm -g -S -std=c++11 odr3.cpp
+; ModuleID = 'odr3.cpp'
+target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-apple-macosx10.11.0"
+
+%struct.S = type { i32, %"struct.S::Nested" }
+%"struct.S::Nested" = type { double }
+
+; Function Attrs: ssp uwtable
+define i32 @_Z3barv() #0 {
+entry:
+  %this.addr.i = alloca %struct.S*, align 8
+  %Add.addr.i = alloca i32, align 4
+  %s = alloca %struct.S, align 8
+  call void @llvm.dbg.declare(metadata %struct.S* %s, metadata !34, metadata !35), !dbg !36
+  store %struct.S* %s, %struct.S** %this.addr.i, align 8, !dbg !37
+  store i32 42, i32* %Add.addr.i, align 4, !dbg !37
+  %this1.i = load %struct.S*, %struct.S** %this.addr.i, !dbg !37
+  %0 = load i32, i32* %Add.addr.i, align 4, !dbg !38
+  %I.i = getelementptr inbounds %struct.S, %struct.S* %this1.i, i32 0, i32 0, !dbg !40
+  %1 = load i32, i32* %I.i, align 4, !dbg !41
+  %add.i = add nsw i32 %1, %0, !dbg !41
+  store i32 %add.i, i32* %I.i, align 4, !dbg !41
+  %call = call i32 @_ZN1S3fooEv(%struct.S* %s), !dbg !42
+  call void @llvm.dbg.declare(metadata %struct.S** %this.addr.i, metadata !43, metadata !35), !dbg !45
+  call void @llvm.dbg.declare(metadata i32* %Add.addr.i, metadata !46, metadata !35), !dbg !47
+  ret i32 %call, !dbg !48
+}
+
+; Function Attrs: nounwind readnone
+declare void @llvm.dbg.declare(metadata, metadata, metadata) #1
+
+; Function Attrs: nounwind ssp uwtable
+define linkonce_odr i32 @_ZN1S3fooEv(%struct.S* %this) #2 align 2 {
+entry:
+  %this.addr = alloca %struct.S*, align 8
+  store %struct.S* %this, %struct.S** %this.addr, align 8
+  call void @llvm.dbg.declare(metadata %struct.S** %this.addr, metadata !49, metadata !35), !dbg !50
+  %this1 = load %struct.S*, %struct.S** %this.addr
+  %I = getelementptr inbounds %struct.S, %struct.S* %this1, i32 0, i32 0, !dbg !51
+  %0 = load i32, i32* %I, align 4, !dbg !51
+  ret i32 %0, !dbg !52
+}
+
+attributes #0 = { ssp uwtable "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="core2" "target-features"="+cx16,+sse,+sse2,+sse3,+ssse3" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #1 = { nounwind readnone }
+attributes #2 = { nounwind ssp uwtable "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="core2" "target-features"="+cx16,+sse,+sse2,+sse3,+ssse3" "unsafe-fp-math"="false" "use-soft-float"="false" }
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!30, !31, !32}
+!llvm.ident = !{!33}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C_plus_plus, file: !1, producer: "clang version 3.8.0 (trunk 242534)", isOptimized: false, runtimeVersion: 0, emissionKind: 1, enums: !2, retainedTypes: !3, subprograms: !24)
+!1 = !DIFile(filename: "odr3.cpp", directory: "/Inputs")
+!2 = !{}
+!3 = !{!4, !20}
+!4 = !DICompositeType(tag: DW_TAG_structure_type, name: "S", file: !5, line: 1, size: 128, align: 64, elements: !6, identifier: "_ZTS1S")
+!5 = !DIFile(filename: "./odr-types.h", directory: "/Inputs")
+!6 = !{!7, !9, !10, !14, !17}
+!7 = !DIDerivedType(tag: DW_TAG_member, name: "I", scope: !"_ZTS1S", file: !5, line: 2, baseType: !8, size: 32, align: 32)
+!8 = !DIBasicType(name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
+!9 = !DIDerivedType(tag: DW_TAG_member, name: "D", scope: !"_ZTS1S", file: !5, line: 15, baseType: !"_ZTSN1S6NestedE", size: 64, align: 64, offset: 64)
+!10 = !DISubprogram(name: "incr", linkageName: "_ZN1S4incrEv", scope: !"_ZTS1S", file: !5, line: 4, type: !11, isLocal: false, isDefinition: false, scopeLine: 4, flags: DIFlagPrototyped, isOptimized: false)
+!11 = !DISubroutineType(types: !12)
+!12 = !{null, !13}
+!13 = !DIDerivedType(tag: DW_TAG_pointer_type, baseType: !"_ZTS1S", size: 64, align: 64, flags: DIFlagArtificial | DIFlagObjectPointer)
+!14 = !DISubprogram(name: "incr", linkageName: "_ZN1S4incrEi", scope: !"_ZTS1S", file: !5, line: 5, type: !15, isLocal: false, isDefinition: false, scopeLine: 5, flags: DIFlagPrototyped, isOptimized: false)
+!15 = !DISubroutineType(types: !16)
+!16 = !{null, !13, !8}
+!17 = !DISubprogram(name: "foo", linkageName: "_ZN1S3fooEv", scope: !"_ZTS1S", file: !5, line: 18, type: !18, isLocal: false, isDefinition: false, scopeLine: 18, flags: DIFlagPrototyped, isOptimized: false)
+!18 = !DISubroutineType(types: !19)
+!19 = !{!8, !13}
+!20 = !DICompositeType(tag: DW_TAG_structure_type, name: "Nested", scope: !"_ZTS1S", file: !5, line: 9, size: 64, align: 64, elements: !21, identifier: "_ZTSN1S6NestedE")
+!21 = !{!22}
+!22 = !DIDerivedType(tag: DW_TAG_member, name: "D", scope: !"_ZTSN1S6NestedE", file: !5, line: 10, baseType: !23, size: 64, align: 64)
+!23 = !DIBasicType(name: "double", size: 64, align: 64, encoding: DW_ATE_float)
+!24 = !{!25, !28, !29}
+!25 = !DISubprogram(name: "bar", linkageName: "_Z3barv", scope: !1, file: !1, line: 3, type: !26, isLocal: false, isDefinition: true, scopeLine: 3, flags: DIFlagPrototyped, isOptimized: false, function: i32 ()* @_Z3barv, variables: !2)
+!26 = !DISubroutineType(types: !27)
+!27 = !{!8}
+!28 = !DISubprogram(name: "incr", linkageName: "_ZN1S4incrEi", scope: !"_ZTS1S", file: !5, line: 5, type: !15, isLocal: false, isDefinition: true, scopeLine: 5, flags: DIFlagPrototyped, isOptimized: false, declaration: !14, variables: !2)
+!29 = !DISubprogram(name: "foo", linkageName: "_ZN1S3fooEv", scope: !"_ZTS1S", file: !5, line: 18, type: !18, isLocal: false, isDefinition: true, scopeLine: 18, flags: DIFlagPrototyped, isOptimized: false, function: i32 (%struct.S*)* @_ZN1S3fooEv, declaration: !17, variables: !2)
+!30 = !{i32 2, !"Dwarf Version", i32 2}
+!31 = !{i32 2, !"Debug Info Version", i32 3}
+!32 = !{i32 1, !"PIC Level", i32 2}
+!33 = !{!"clang version 3.8.0 (trunk 242534)"}
+!34 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "s", scope: !25, file: !1, line: 4, type: !"_ZTS1S")
+!35 = !DIExpression()
+!36 = !DILocation(line: 4, column: 4, scope: !25)
+!37 = !DILocation(line: 5, column: 2, scope: !25)
+!38 = !DILocation(line: 5, column: 59, scope: !28, inlinedAt: !39)
+!39 = distinct !DILocation(line: 5, column: 2, scope: !25)
+!40 = !DILocation(line: 5, column: 54, scope: !28, inlinedAt: !39)
+!41 = !DILocation(line: 5, column: 56, scope: !28, inlinedAt: !39)
+!42 = !DILocation(line: 6, column: 9, scope: !25)
+!43 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "this", arg: 1, scope: !28, type: !44, flags: DIFlagArtificial | DIFlagObjectPointer)
+!44 = !DIDerivedType(tag: DW_TAG_pointer_type, baseType: !"_ZTS1S", size: 64, align: 64)
+!45 = !DILocation(line: 0, scope: !28, inlinedAt: !39)
+!46 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "Add", arg: 2, scope: !28, file: !5, line: 5, type: !8)
+!47 = !DILocation(line: 5, column: 16, scope: !28, inlinedAt: !39)
+!48 = !DILocation(line: 6, column: 2, scope: !25)
+!49 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "this", arg: 1, scope: !29, type: !44, flags: DIFlagArtificial | DIFlagObjectPointer)
+!50 = !DILocation(line: 0, scope: !29)
+!51 = !DILocation(line: 18, column: 21, scope: !29)
+!52 = !DILocation(line: 18, column: 14, scope: !29)
diff --git a/test/tools/dsymutil/Inputs/odr4.cpp b/test/tools/dsymutil/Inputs/odr4.cpp
new file mode 100644
index 00000000000..948dbdbd8e5
--- /dev/null
+++ b/test/tools/dsymutil/Inputs/odr4.cpp
@@ -0,0 +1,8 @@
+namespace {
+	class AnonC {
+	};
+}
+
+void baz() {
+	AnonC ac;
+}
diff --git a/test/tools/dsymutil/Inputs/odr4.ll b/test/tools/dsymutil/Inputs/odr4.ll
new file mode 100644
index 00000000000..ecaf85bd34f
--- /dev/null
+++ b/test/tools/dsymutil/Inputs/odr4.ll
@@ -0,0 +1,43 @@
+; Generated from odr4.cpp and odr-types.h by running:
+; clang -emit-llvm -g -S -std=c++11 odr4.cpp
+; ModuleID = 'odr4.cpp'
+target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-apple-macosx10.11.0"
+
+%"class.(anonymous namespace)::AnonC" = type { i8 }
+
+; Function Attrs: nounwind ssp uwtable
+define void @_Z3bazv() #0 {
+entry:
+  %ac = alloca %"class.(anonymous namespace)::AnonC", align 1
+  call void @llvm.dbg.declare(metadata %"class.(anonymous namespace)::AnonC"* %ac, metadata !11, metadata !14), !dbg !15
+  ret void, !dbg !16
+}
+
+; Function Attrs: nounwind readnone
+declare void @llvm.dbg.declare(metadata, metadata, metadata) #1
+
+attributes #0 = { nounwind ssp uwtable "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="core2" "target-features"="+cx16,+sse,+sse2,+sse3,+ssse3" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #1 = { nounwind readnone }
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!7, !8, !9}
+!llvm.ident = !{!10}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C_plus_plus, file: !1, producer: "clang version 3.8.0 (trunk 242534)", isOptimized: false, runtimeVersion: 0, emissionKind: 1, enums: !2, subprograms: !3)
+!1 = !DIFile(filename: "odr4.cpp", directory: "/Inputs")
+!2 = !{}
+!3 = !{!4}
+!4 = !DISubprogram(name: "baz", linkageName: "_Z3bazv", scope: !1, file: !1, line: 6, type: !5, isLocal: false, isDefinition: true, scopeLine: 6, flags: DIFlagPrototyped, isOptimized: false, function: void ()* @_Z3bazv, variables: !2)
+!5 = !DISubroutineType(types: !6)
+!6 = !{null}
+!7 = !{i32 2, !"Dwarf Version", i32 2}
+!8 = !{i32 2, !"Debug Info Version", i32 3}
+!9 = !{i32 1, !"PIC Level", i32 2}
+!10 = !{!"clang version 3.8.0 (trunk 242534)"}
+!11 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "ac", scope: !4, file: !1, line: 7, type: !12)
+!12 = !DICompositeType(tag: DW_TAG_class_type, name: "AnonC", scope: !13, file: !1, line: 2, size: 8, align: 8, elements: !2)
+!13 = !DINamespace(scope: null, file: !1, line: 1)
+!14 = !DIExpression()
+!15 = !DILocation(line: 7, column: 8, scope: !4)
+!16 = !DILocation(line: 8, column: 1, scope: !4)
diff --git a/test/tools/dsymutil/Inputs/odr5.cpp b/test/tools/dsymutil/Inputs/odr5.cpp
new file mode 100644
index 00000000000..fd694239e37
--- /dev/null
+++ b/test/tools/dsymutil/Inputs/odr5.cpp
@@ -0,0 +1,7 @@
+#include "odr-types.h"
+
+double baz() {
+	S::Nested d;
+	d.init(0);
+	return d.D;
+}
diff --git a/test/tools/dsymutil/Inputs/odr5.ll b/test/tools/dsymutil/Inputs/odr5.ll
new file mode 100644
index 00000000000..7b738be5427
--- /dev/null
+++ b/test/tools/dsymutil/Inputs/odr5.ll
@@ -0,0 +1,101 @@
+; Generated from odr5.cpp and odr-types.h by running:
+; clang -emit-llvm -g -S -std=c++11 odr5.cpp
+; ModuleID = 'odr5.cpp'
+target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-apple-macosx10.11.0"
+
+%"struct.S::Nested" = type { double }
+
+; Function Attrs: ssp uwtable
+define double @_Z3bazv() #0 {
+entry:
+  %d = alloca %"struct.S::Nested", align 8
+  call void @llvm.dbg.declare(metadata %"struct.S::Nested"* %d, metadata !39, metadata !40), !dbg !41
+  call void @_ZN1S6Nested4initIiEEvT_(%"struct.S::Nested"* %d, i32 0), !dbg !42
+  %D = getelementptr inbounds %"struct.S::Nested", %"struct.S::Nested"* %d, i32 0, i32 0, !dbg !43
+  %0 = load double, double* %D, align 8, !dbg !43
+  ret double %0, !dbg !44
+}
+
+; Function Attrs: nounwind readnone
+declare void @llvm.dbg.declare(metadata, metadata, metadata) #1
+
+; Function Attrs: nounwind ssp uwtable
+define linkonce_odr void @_ZN1S6Nested4initIiEEvT_(%"struct.S::Nested"* %this, i32 %Val) #2 align 2 {
+entry:
+  %this.addr = alloca %"struct.S::Nested"*, align 8
+  %Val.addr = alloca i32, align 4
+  store %"struct.S::Nested"* %this, %"struct.S::Nested"** %this.addr, align 8
+  call void @llvm.dbg.declare(metadata %"struct.S::Nested"** %this.addr, metadata !45, metadata !40), !dbg !47
+  store i32 %Val, i32* %Val.addr, align 4
+  call void @llvm.dbg.declare(metadata i32* %Val.addr, metadata !48, metadata !40), !dbg !49
+  %this1 = load %"struct.S::Nested"*, %"struct.S::Nested"** %this.addr
+  %0 = load i32, i32* %Val.addr, align 4, !dbg !50
+  %conv = sitofp i32 %0 to double, !dbg !50
+  %D = getelementptr inbounds %"struct.S::Nested", %"struct.S::Nested"* %this1, i32 0, i32 0, !dbg !51
+  store double %conv, double* %D, align 8, !dbg !52
+  ret void, !dbg !53
+}
+
+attributes #0 = { ssp uwtable "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="core2" "target-features"="+cx16,+sse,+sse2,+sse3,+ssse3" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #1 = { nounwind readnone }
+attributes #2 = { nounwind ssp uwtable "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="core2" "target-features"="+cx16,+sse,+sse2,+sse3,+ssse3" "unsafe-fp-math"="false" "use-soft-float"="false" }
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!35, !36, !37}
+!llvm.ident = !{!38}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C_plus_plus, file: !1, producer: "clang version 3.8.0 (trunk 242534)", isOptimized: false, runtimeVersion: 0, emissionKind: 1, enums: !2, retainedTypes: !3, subprograms: !24)
+!1 = !DIFile(filename: "odr5.cpp", directory: "/Inputs")
+!2 = !{}
+!3 = !{!4, !20, !23}
+!4 = !DICompositeType(tag: DW_TAG_structure_type, name: "S", file: !5, line: 1, size: 128, align: 64, elements: !6, identifier: "_ZTS1S")
+!5 = !DIFile(filename: "./odr-types.h", directory: "/Inputs")
+!6 = !{!7, !9, !10, !14, !17}
+!7 = !DIDerivedType(tag: DW_TAG_member, name: "I", scope: !"_ZTS1S", file: !5, line: 2, baseType: !8, size: 32, align: 32)
+!8 = !DIBasicType(name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
+!9 = !DIDerivedType(tag: DW_TAG_member, name: "D", scope: !"_ZTS1S", file: !5, line: 15, baseType: !"_ZTSN1S6NestedE", size: 64, align: 64, offset: 64)
+!10 = !DISubprogram(name: "incr", linkageName: "_ZN1S4incrEv", scope: !"_ZTS1S", file: !5, line: 4, type: !11, isLocal: false, isDefinition: false, scopeLine: 4, flags: DIFlagPrototyped, isOptimized: false)
+!11 = !DISubroutineType(types: !12)
+!12 = !{null, !13}
+!13 = !DIDerivedType(tag: DW_TAG_pointer_type, baseType: !"_ZTS1S", size: 64, align: 64, flags: DIFlagArtificial | DIFlagObjectPointer)
+!14 = !DISubprogram(name: "incr", linkageName: "_ZN1S4incrEi", scope: !"_ZTS1S", file: !5, line: 5, type: !15, isLocal: false, isDefinition: false, scopeLine: 5, flags: DIFlagPrototyped, isOptimized: false)
+!15 = !DISubroutineType(types: !16)
+!16 = !{null, !13, !8}
+!17 = !DISubprogram(name: "foo", linkageName: "_ZN1S3fooEv", scope: !"_ZTS1S", file: !5, line: 18, type: !18, isLocal: false, isDefinition: false, scopeLine: 18, flags: DIFlagPrototyped, isOptimized: false)
+!18 = !DISubroutineType(types: !19)
+!19 = !{!8, !13}
+!20 = !DICompositeType(tag: DW_TAG_structure_type, name: "Nested", scope: !"_ZTS1S", file: !5, line: 9, size: 64, align: 64, elements: !21, identifier: "_ZTSN1S6NestedE")
+!21 = !{!22}
+!22 = !DIDerivedType(tag: DW_TAG_member, name: "D", scope: !"_ZTSN1S6NestedE", file: !5, line: 10, baseType: !23, size: 64, align: 64)
+!23 = !DIBasicType(name: "double", size: 64, align: 64, encoding: DW_ATE_float)
+!24 = !{!25, !28}
+!25 = !DISubprogram(name: "baz", linkageName: "_Z3bazv", scope: !1, file: !1, line: 3, type: !26, isLocal: false, isDefinition: true, scopeLine: 3, flags: DIFlagPrototyped, isOptimized: false, function: double ()* @_Z3bazv, variables: !2)
+!26 = !DISubroutineType(types: !27)
+!27 = !{!23}
+!28 = !DISubprogram(name: "init<int>", linkageName: "_ZN1S6Nested4initIiEEvT_", scope: !"_ZTSN1S6NestedE", file: !5, line: 12, type: !29, isLocal: false, isDefinition: true, scopeLine: 12, flags: DIFlagPrototyped, isOptimized: false, function: void (%"struct.S::Nested"*, i32)* @_ZN1S6Nested4initIiEEvT_, templateParams: !32, declaration: !34, variables: !2)
+!29 = !DISubroutineType(types: !30)
+!30 = !{null, !31, !8}
+!31 = !DIDerivedType(tag: DW_TAG_pointer_type, baseType: !"_ZTSN1S6NestedE", size: 64, align: 64, flags: DIFlagArtificial | DIFlagObjectPointer)
+!32 = !{!33}
+!33 = !DITemplateTypeParameter(name: "T", type: !8)
+!34 = !DISubprogram(name: "init<int>", linkageName: "_ZN1S6Nested4initIiEEvT_", scope: !"_ZTSN1S6NestedE", file: !5, line: 12, type: !29, isLocal: false, isDefinition: false, scopeLine: 12, flags: DIFlagPrototyped, isOptimized: false, templateParams: !32)
+!35 = !{i32 2, !"Dwarf Version", i32 2}
+!36 = !{i32 2, !"Debug Info Version", i32 3}
+!37 = !{i32 1, !"PIC Level", i32 2}
+!38 = !{!"clang version 3.8.0 (trunk 242534)"}
+!39 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "d", scope: !25, file: !1, line: 4, type: !"_ZTSN1S6NestedE")
+!40 = !DIExpression()
+!41 = !DILocation(line: 4, column: 12, scope: !25)
+!42 = !DILocation(line: 5, column: 2, scope: !25)
+!43 = !DILocation(line: 6, column: 11, scope: !25)
+!44 = !DILocation(line: 6, column: 2, scope: !25)
+!45 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "this", arg: 1, scope: !28, type: !46, flags: DIFlagArtificial | DIFlagObjectPointer)
+!46 = !DIDerivedType(tag: DW_TAG_pointer_type, baseType: !"_ZTSN1S6NestedE", size: 64, align: 64)
+!47 = !DILocation(line: 0, scope: !28)
+!48 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "Val", arg: 2, scope: !28, file: !5, line: 12, type: !8)
+!49 = !DILocation(line: 12, column: 36, scope: !28)
+!50 = !DILocation(line: 12, column: 54, scope: !28)
+!51 = !DILocation(line: 12, column: 43, scope: !28)
+!52 = !DILocation(line: 12, column: 45, scope: !28)
+!53 = !DILocation(line: 12, column: 60, scope: !28)
diff --git a/test/tools/dsymutil/Inputs/odr6.cpp b/test/tools/dsymutil/Inputs/odr6.cpp
new file mode 100644
index 00000000000..2893856ce2a
--- /dev/null
+++ b/test/tools/dsymutil/Inputs/odr6.cpp
@@ -0,0 +1,7 @@
+#include "odr-types.h"
+
+double baz() {
+	S::Nested d;
+	d.init(0.0);
+	return d.D;
+}
diff --git a/test/tools/dsymutil/Inputs/odr6.ll b/test/tools/dsymutil/Inputs/odr6.ll
new file mode 100644
index 00000000000..c17099f1f97
--- /dev/null
+++ b/test/tools/dsymutil/Inputs/odr6.ll
@@ -0,0 +1,100 @@
+; Generated from odr6.cpp and odr-types.h by running:
+; clang -emit-llvm -g -S -std=c++11 odr6.cpp
+; ModuleID = 'odr6.cpp'
+target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-apple-macosx10.11.0"
+
+%"struct.S::Nested" = type { double }
+
+; Function Attrs: ssp uwtable
+define double @_Z3bazv() #0 {
+entry:
+  %d = alloca %"struct.S::Nested", align 8
+  call void @llvm.dbg.declare(metadata %"struct.S::Nested"* %d, metadata !39, metadata !40), !dbg !41
+  call void @_ZN1S6Nested4initIdEEvT_(%"struct.S::Nested"* %d, double 0.000000e+00), !dbg !42
+  %D = getelementptr inbounds %"struct.S::Nested", %"struct.S::Nested"* %d, i32 0, i32 0, !dbg !43
+  %0 = load double, double* %D, align 8, !dbg !43
+  ret double %0, !dbg !44
+}
+
+; Function Attrs: nounwind readnone
+declare void @llvm.dbg.declare(metadata, metadata, metadata) #1
+
+; Function Attrs: nounwind ssp uwtable
+define linkonce_odr void @_ZN1S6Nested4initIdEEvT_(%"struct.S::Nested"* %this, double %Val) #2 align 2 {
+entry:
+  %this.addr = alloca %"struct.S::Nested"*, align 8
+  %Val.addr = alloca double, align 8
+  store %"struct.S::Nested"* %this, %"struct.S::Nested"** %this.addr, align 8
+  call void @llvm.dbg.declare(metadata %"struct.S::Nested"** %this.addr, metadata !45, metadata !40), !dbg !47
+  store double %Val, double* %Val.addr, align 8
+  call void @llvm.dbg.declare(metadata double* %Val.addr, metadata !48, metadata !40), !dbg !49
+  %this1 = load %"struct.S::Nested"*, %"struct.S::Nested"** %this.addr
+  %0 = load double, double* %Val.addr, align 8, !dbg !50
+  %D = getelementptr inbounds %"struct.S::Nested", %"struct.S::Nested"* %this1, i32 0, i32 0, !dbg !51
+  store double %0, double* %D, align 8, !dbg !52
+  ret void, !dbg !53
+}
+
+attributes #0 = { ssp uwtable "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="core2" "target-features"="+cx16,+sse,+sse2,+sse3,+ssse3" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #1 = { nounwind readnone }
+attributes #2 = { nounwind ssp uwtable "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="core2" "target-features"="+cx16,+sse,+sse2,+sse3,+ssse3" "unsafe-fp-math"="false" "use-soft-float"="false" }
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!35, !36, !37}
+!llvm.ident = !{!38}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C_plus_plus, file: !1, producer: "clang version 3.8.0 (trunk 242534)", isOptimized: false, runtimeVersion: 0, emissionKind: 1, enums: !2, retainedTypes: !3, subprograms: !24)
+!1 = !DIFile(filename: "odr6.cpp", directory: "/Inputs")
+!2 = !{}
+!3 = !{!4, !20, !23}
+!4 = !DICompositeType(tag: DW_TAG_structure_type, name: "S", file: !5, line: 1, size: 128, align: 64, elements: !6, identifier: "_ZTS1S")
+!5 = !DIFile(filename: "./odr-types.h", directory: "/Inputs")
+!6 = !{!7, !9, !10, !14, !17}
+!7 = !DIDerivedType(tag: DW_TAG_member, name: "I", scope: !"_ZTS1S", file: !5, line: 2, baseType: !8, size: 32, align: 32)
+!8 = !DIBasicType(name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
+!9 = !DIDerivedType(tag: DW_TAG_member, name: "D", scope: !"_ZTS1S", file: !5, line: 15, baseType: !"_ZTSN1S6NestedE", size: 64, align: 64, offset: 64)
+!10 = !DISubprogram(name: "incr", linkageName: "_ZN1S4incrEv", scope: !"_ZTS1S", file: !5, line: 4, type: !11, isLocal: false, isDefinition: false, scopeLine: 4, flags: DIFlagPrototyped, isOptimized: false)
+!11 = !DISubroutineType(types: !12)
+!12 = !{null, !13}
+!13 = !DIDerivedType(tag: DW_TAG_pointer_type, baseType: !"_ZTS1S", size: 64, align: 64, flags: DIFlagArtificial | DIFlagObjectPointer)
+!14 = !DISubprogram(name: "incr", linkageName: "_ZN1S4incrEi", scope: !"_ZTS1S", file: !5, line: 5, type: !15, isLocal: false, isDefinition: false, scopeLine: 5, flags: DIFlagPrototyped, isOptimized: false)
+!15 = !DISubroutineType(types: !16)
+!16 = !{null, !13, !8}
+!17 = !DISubprogram(name: "foo", linkageName: "_ZN1S3fooEv", scope: !"_ZTS1S", file: !5, line: 18, type: !18, isLocal: false, isDefinition: false, scopeLine: 18, flags: DIFlagPrototyped, isOptimized: false)
+!18 = !DISubroutineType(types: !19)
+!19 = !{!8, !13}
+!20 = !DICompositeType(tag: DW_TAG_structure_type, name: "Nested", scope: !"_ZTS1S", file: !5, line: 9, size: 64, align: 64, elements: !21, identifier: "_ZTSN1S6NestedE")
+!21 = !{!22}
+!22 = !DIDerivedType(tag: DW_TAG_member, name: "D", scope: !"_ZTSN1S6NestedE", file: !5, line: 10, baseType: !23, size: 64, align: 64)
+!23 = !DIBasicType(name: "double", size: 64, align: 64, encoding: DW_ATE_float)
+!24 = !{!25, !28}
+!25 = !DISubprogram(name: "baz", linkageName: "_Z3bazv", scope: !1, file: !1, line: 3, type: !26, isLocal: false, isDefinition: true, scopeLine: 3, flags: DIFlagPrototyped, isOptimized: false, function: double ()* @_Z3bazv, variables: !2)
+!26 = !DISubroutineType(types: !27)
+!27 = !{!23}
+!28 = !DISubprogram(name: "init<double>", linkageName: "_ZN1S6Nested4initIdEEvT_", scope: !"_ZTSN1S6NestedE", file: !5, line: 12, type: !29, isLocal: false, isDefinition: true, scopeLine: 12, flags: DIFlagPrototyped, isOptimized: false, function: void (%"struct.S::Nested"*, double)* @_ZN1S6Nested4initIdEEvT_, templateParams: !32, declaration: !34, variables: !2)
+!29 = !DISubroutineType(types: !30)
+!30 = !{null, !31, !23}
+!31 = !DIDerivedType(tag: DW_TAG_pointer_type, baseType: !"_ZTSN1S6NestedE", size: 64, align: 64, flags: DIFlagArtificial | DIFlagObjectPointer)
+!32 = !{!33}
+!33 = !DITemplateTypeParameter(name: "T", type: !23)
+!34 = !DISubprogram(name: "init<double>", linkageName: "_ZN1S6Nested4initIdEEvT_", scope: !"_ZTSN1S6NestedE", file: !5, line: 12, type: !29, isLocal: false, isDefinition: false, scopeLine: 12, flags: DIFlagPrototyped, isOptimized: false, templateParams: !32)
+!35 = !{i32 2, !"Dwarf Version", i32 2}
+!36 = !{i32 2, !"Debug Info Version", i32 3}
+!37 = !{i32 1, !"PIC Level", i32 2}
+!38 = !{!"clang version 3.8.0 (trunk 242534)"}
+!39 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "d", scope: !25, file: !1, line: 4, type: !"_ZTSN1S6NestedE")
+!40 = !DIExpression()
+!41 = !DILocation(line: 4, column: 12, scope: !25)
+!42 = !DILocation(line: 5, column: 2, scope: !25)
+!43 = !DILocation(line: 6, column: 11, scope: !25)
+!44 = !DILocation(line: 6, column: 2, scope: !25)
+!45 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "this", arg: 1, scope: !28, type: !46, flags: DIFlagArtificial | DIFlagObjectPointer)
+!46 = !DIDerivedType(tag: DW_TAG_pointer_type, baseType: !"_ZTSN1S6NestedE", size: 64, align: 64)
+!47 = !DILocation(line: 0, scope: !28)
+!48 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "Val", arg: 2, scope: !28, file: !5, line: 12, type: !23)
+!49 = !DILocation(line: 12, column: 36, scope: !28)
+!50 = !DILocation(line: 12, column: 54, scope: !28)
+!51 = !DILocation(line: 12, column: 43, scope: !28)
+!52 = !DILocation(line: 12, column: 45, scope: !28)
+!53 = !DILocation(line: 12, column: 60, scope: !28)
diff --git a/test/tools/dsymutil/Inputs/odr7.cpp b/test/tools/dsymutil/Inputs/odr7.cpp
new file mode 100644
index 00000000000..6056b486972
--- /dev/null
+++ b/test/tools/dsymutil/Inputs/odr7.cpp
@@ -0,0 +1,5 @@
+#include "odr-types.h"
+
+void foo() {
+	S::Nested N;
+}
diff --git a/test/tools/dsymutil/Inputs/odr7.ll b/test/tools/dsymutil/Inputs/odr7.ll
new file mode 100644
index 00000000000..b4df117f1cf
--- /dev/null
+++ b/test/tools/dsymutil/Inputs/odr7.ll
@@ -0,0 +1,62 @@
+; Generated from odr7.cpp and odr-types.h by running:
+; clang -emit-llvm -g -S -std=c++11 odr7.cpp
+; ModuleID = 'odr7.cpp'
+target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-apple-macosx10.11.0"
+
+%"struct.S::Nested" = type { double }
+
+; Function Attrs: nounwind ssp uwtable
+define void @_Z3foov() #0 {
+entry:
+  %N = alloca %"struct.S::Nested", align 8
+  call void @llvm.dbg.declare(metadata %"struct.S::Nested"* %N, metadata !32, metadata !33), !dbg !34
+  ret void, !dbg !35
+}
+
+; Function Attrs: nounwind readnone
+declare void @llvm.dbg.declare(metadata, metadata, metadata) #1
+
+attributes #0 = { nounwind ssp uwtable "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="core2" "target-features"="+cx16,+sse,+sse2,+sse3,+ssse3" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #1 = { nounwind readnone }
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!28, !29, !30}
+!llvm.ident = !{!31}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C_plus_plus, file: !1, producer: "clang version 3.8.0 (trunk 242534)", isOptimized: false, runtimeVersion: 0, emissionKind: 1, enums: !2, retainedTypes: !3, subprograms: !24)
+!1 = !DIFile(filename: "odr7.cpp", directory: "/Inputs")
+!2 = !{}
+!3 = !{!4, !20}
+!4 = !DICompositeType(tag: DW_TAG_structure_type, name: "S", file: !5, line: 1, size: 128, align: 64, elements: !6, identifier: "_ZTS1S")
+!5 = !DIFile(filename: "./odr-types.h", directory: "/Inputs")
+!6 = !{!7, !9, !10, !14, !17}
+!7 = !DIDerivedType(tag: DW_TAG_member, name: "I", scope: !"_ZTS1S", file: !5, line: 2, baseType: !8, size: 32, align: 32)
+!8 = !DIBasicType(name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
+!9 = !DIDerivedType(tag: DW_TAG_member, name: "D", scope: !"_ZTS1S", file: !5, line: 15, baseType: !"_ZTSN1S6NestedE", size: 64, align: 64, offset: 64)
+!10 = !DISubprogram(name: "incr", linkageName: "_ZN1S4incrEv", scope: !"_ZTS1S", file: !5, line: 4, type: !11, isLocal: false, isDefinition: false, scopeLine: 4, flags: DIFlagPrototyped, isOptimized: false)
+!11 = !DISubroutineType(types: !12)
+!12 = !{null, !13}
+!13 = !DIDerivedType(tag: DW_TAG_pointer_type, baseType: !"_ZTS1S", size: 64, align: 64, flags: DIFlagArtificial | DIFlagObjectPointer)
+!14 = !DISubprogram(name: "incr", linkageName: "_ZN1S4incrEi", scope: !"_ZTS1S", file: !5, line: 5, type: !15, isLocal: false, isDefinition: false, scopeLine: 5, flags: DIFlagPrototyped, isOptimized: false)
+!15 = !DISubroutineType(types: !16)
+!16 = !{null, !13, !8}
+!17 = !DISubprogram(name: "foo", linkageName: "_ZN1S3fooEv", scope: !"_ZTS1S", file: !5, line: 18, type: !18, isLocal: false, isDefinition: false, scopeLine: 18, flags: DIFlagPrototyped, isOptimized: false)
+!18 = !DISubroutineType(types: !19)
+!19 = !{!8, !13}
+!20 = !DICompositeType(tag: DW_TAG_structure_type, name: "Nested", scope: !"_ZTS1S", file: !5, line: 9, size: 64, align: 64, elements: !21, identifier: "_ZTSN1S6NestedE")
+!21 = !{!22}
+!22 = !DIDerivedType(tag: DW_TAG_member, name: "D", scope: !"_ZTSN1S6NestedE", file: !5, line: 10, baseType: !23, size: 64, align: 64)
+!23 = !DIBasicType(name: "double", size: 64, align: 64, encoding: DW_ATE_float)
+!24 = !{!25}
+!25 = !DISubprogram(name: "foo", linkageName: "_Z3foov", scope: !1, file: !1, line: 3, type: !26, isLocal: false, isDefinition: true, scopeLine: 3, flags: DIFlagPrototyped, isOptimized: false, function: void ()* @_Z3foov, variables: !2)
+!26 = !DISubroutineType(types: !27)
+!27 = !{null}
+!28 = !{i32 2, !"Dwarf Version", i32 2}
+!29 = !{i32 2, !"Debug Info Version", i32 3}
+!30 = !{i32 1, !"PIC Level", i32 2}
+!31 = !{!"clang version 3.8.0 (trunk 242534)"}
+!32 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "N", scope: !25, file: !1, line: 4, type: !"_ZTSN1S6NestedE")
+!33 = !DIExpression()
+!34 = !DILocation(line: 4, column: 12, scope: !25)
+!35 = !DILocation(line: 5, column: 1, scope: !25)
diff --git a/test/tools/dsymutil/X86/basic-linking-x86.test b/test/tools/dsymutil/X86/basic-linking-x86.test
index 1059e233f94..19b4e3bef66 100644
--- a/test/tools/dsymutil/X86/basic-linking-x86.test
+++ b/test/tools/dsymutil/X86/basic-linking-x86.test
@@ -6,6 +6,8 @@ RUN: llvm-dsymutil -o %t2 -oso-prepend-path=%p/.. %p/../Inputs/basic.macho.x86_6
 RUN: llvm-dwarfdump %t2 | FileCheck %s
 RUN: llvm-dsymutil -o - -oso-prepend-path=%p/.. %p/../Inputs/basic.macho.x86_64 | llvm-dwarfdump - | FileCheck %s --check-prefix=CHECK --check-prefix=BASIC
 RUN: llvm-dsymutil -o - -oso-prepend-path=%p/.. %p/../Inputs/basic-archive.macho.x86_64 | llvm-dwarfdump - | FileCheck %s --check-prefix=CHECK --check-prefix=ARCHIVE
+RUN: llvm-dsymutil -dump-debug-map -oso-prepend-path=%p/.. %p/../Inputs/basic.macho.x86_64 | llvm-dsymutil -y -o - - | llvm-dwarfdump - | FileCheck %s --check-prefix=CHECK --check-prefix=BASIC
+RUN: llvm-dsymutil -dump-debug-map -oso-prepend-path=%p/.. %p/../Inputs/basic-archive.macho.x86_64 | llvm-dsymutil -o - -y - | llvm-dwarfdump - | FileCheck %s --check-prefix=CHECK --check-prefix=ARCHIVE
 
 CHECK: file format Mach-O 64-bit x86-64
 
@@ -118,7 +120,7 @@ ARCHIVE:    DW_AT_location [DW_FORM_block1] (<0x09> 03 08 10 00 00 01 00 00 00 )
 CHECK:    DW_TAG_volatile_type [10]
 CHECK:      DW_AT_type [DW_FORM_ref4]       (cu + 0x0041 => {0x00000167})
 CHECK:    DW_TAG_base_type [4]
-CHACK:      DW_AT_name [DW_FORM_strp]       ( .debug_str[0x00000060] = "int")
+CHECK:      DW_AT_name [DW_FORM_strp]       ( .debug_str[0x00000060] = "int")
 CHECK:    DW_TAG_subprogram [2] *
 CHECK:      DW_AT_name [DW_FORM_strp]       ( .debug_str[0x0000009b] = "bar")
 CHECK:      DW_AT_type [DW_FORM_ref4]       (cu + 0x0041 => {0x00000167})
diff --git a/test/tools/dsymutil/X86/basic-lto-linking-x86.test b/test/tools/dsymutil/X86/basic-lto-linking-x86.test
index 22b6e0808d8..395234e9616 100644
--- a/test/tools/dsymutil/X86/basic-lto-linking-x86.test
+++ b/test/tools/dsymutil/X86/basic-lto-linking-x86.test
@@ -1,5 +1,6 @@
 REQUIRES: shell
 RUN: llvm-dsymutil -o - -oso-prepend-path=%p/.. %p/../Inputs/basic-lto.macho.x86_64 | llvm-dwarfdump - | FileCheck %s
+RUN: llvm-dsymutil -oso-prepend-path=%p/.. -dump-debug-map %p/../Inputs/basic-lto.macho.x86_64 | llvm-dsymutil -o - -y - | llvm-dwarfdump - | FileCheck %s
 
 CHECK: file format Mach-O 64-bit x86-64
 
diff --git a/test/tools/dsymutil/X86/fat-archive-input-i386.test b/test/tools/dsymutil/X86/fat-archive-input-i386.test
new file mode 100644
index 00000000000..7f17ff48372
--- /dev/null
+++ b/test/tools/dsymutil/X86/fat-archive-input-i386.test
@@ -0,0 +1,16 @@
+# REQUIRES: object-emission
+# RUN: llvm-dsymutil -oso-prepend-path=%p/../Inputs -y %s -o - | llvm-dwarfdump -debug-dump=info - | FileCheck %s
+
+---
+triple:          'i386-apple-darwin'
+objects:
+  - filename: libfat-test.a(fat-test.o)
+    symbols:
+      - { sym: _i386_var, objAddr: 0x0, binAddr: 0x1000, size: 0x4 }
+...
+
+# CHECK: .debug_info contents:
+# CHECK: DW_TAG_variable
+# CHECK-NOT: {{DW_TAG|NULL}}
+# CHECK:  DW_AT_name{{.*}}"i386_var"
+
diff --git a/test/tools/dsymutil/X86/fat-object-input-x86_64.test b/test/tools/dsymutil/X86/fat-object-input-x86_64.test
new file mode 100644
index 00000000000..489afece120
--- /dev/null
+++ b/test/tools/dsymutil/X86/fat-object-input-x86_64.test
@@ -0,0 +1,16 @@
+# REQUIRES: object-emission
+# RUN: llvm-dsymutil -oso-prepend-path=%p/../Inputs -y %s -o - | llvm-dwarfdump -debug-dump=info - | FileCheck %s
+
+---
+triple:          'x86_64-apple-darwin'
+objects:
+  - filename: fat-test.o
+    symbols:
+      - { sym: _x86_64_var, objAddr: 0x0, binAddr: 0x1000, size: 0x4 }
+...
+
+# CHECK: .debug_info contents:
+# CHECK: DW_TAG_variable
+# CHECK-NOT: {{DW_TAG|NULL}}
+# CHECK:  DW_AT_name{{.*}}"x86_64_var"
+
diff --git a/test/tools/dsymutil/X86/fat-object-input-x86_64h.test b/test/tools/dsymutil/X86/fat-object-input-x86_64h.test
new file mode 100644
index 00000000000..f611f5516f5
--- /dev/null
+++ b/test/tools/dsymutil/X86/fat-object-input-x86_64h.test
@@ -0,0 +1,16 @@
+# REQUIRES: object-emission
+# RUN: llvm-dsymutil -oso-prepend-path=%p/../Inputs -y %s -o - | llvm-dwarfdump -debug-dump=info - | FileCheck %s
+
+---
+triple:          'x86_64h-apple-darwin'
+objects:
+  - filename: fat-test.o
+    symbols:
+      - { sym: _x86_64h_var, objAddr: 0x0, binAddr: 0x1000, size: 0x4 }
+...
+
+# CHECK: .debug_info contents:
+# CHECK: DW_TAG_variable
+# CHECK-NOT: {{DW_TAG|NULL}}
+# CHECK:  DW_AT_name{{.*}}"x86_64h_var"
+
diff --git a/test/tools/dsymutil/X86/frame-1.test b/test/tools/dsymutil/X86/frame-1.test
new file mode 100644
index 00000000000..fed18122d00
--- /dev/null
+++ b/test/tools/dsymutil/X86/frame-1.test
@@ -0,0 +1,32 @@
+# REQUIRES: object-emission
+# RUN: rm -rf %t
+# RUN: mkdir -p %t
+# RUN: llc -filetype=obj %p/../Inputs/frame-dw2.ll -o %t/frame-dw2.o
+# RUN: llvm-dsymutil -oso-prepend-path=%t -y %s -o - | llvm-dwarfdump -debug-dump=frames - | FileCheck %s
+
+# This test is meant to verify that identical CIEs will get reused
+# in the same file but also inbetween files. For this to happen, we
+# link twice the same file using this made-up debug map:
+
+---
+triple:          'i386-apple-darwin'
+objects:
+  - filename: frame-dw2.o
+    symbols:
+      - { sym: _bar, objAddr: 0x0, binAddr: 0x1000, size: 0x12 }
+      - { sym: _baz, objAddr: 0x0, binAddr: 0x2000, size: 0x12 }
+  - filename: frame-dw2.o
+    symbols:
+      - { sym: _baz, objAddr: 0x0, binAddr: 0x3000, size: 0x12 }
+...
+
+# CHECK: .debug_frame contents:
+# CHECK: 00000000 {{[0-9a-f]*}} ffffffff CIE
+# CHECK-NOT: FDE
+# CHECK:  FDE cie=00000000 pc=00001000...00001
+# CHECK-NOT: FDE
+# CHECK:  FDE cie=00000000 pc=00002000...00002
+# CHECK-NOT: FDE
+# CHECK:  FDE cie=00000000 pc=00003000...00003
+# CHECK-NOT: FDE
+
diff --git a/test/tools/dsymutil/X86/frame-2.test b/test/tools/dsymutil/X86/frame-2.test
new file mode 100644
index 00000000000..92520290dad
--- /dev/null
+++ b/test/tools/dsymutil/X86/frame-2.test
@@ -0,0 +1,47 @@
+# REQUIRES: object-emission
+# RUN: rm -rf %t
+# RUN: mkdir -p %t
+# RUN: llc -filetype=obj %p/../Inputs/frame-dw2.ll -o %t/frame-dw2.o
+# RUN: llc -filetype=obj %p/../Inputs/frame-dw4.ll -o %t/frame-dw4.o
+# RUN: llvm-dsymutil -oso-prepend-path=%t -y %s -o - | llvm-dwarfdump -debug-dump=frames - | FileCheck %s
+
+# Check the handling of multiple different CIEs. To have CIEs that
+# appear to be different, use a dwarf2 version of the file along with
+# a dwarf 4 version. The CIE header version (and layout) will be different.
+# FIXME: this test also checks that we didn't reuse the first CIE when it
+# appears again. This is a behavior we inherited from dsymutil-classic
+# but this should be fixed (see comment in patchFrameInfoForObject())
+---
+triple:          'i386-apple-darwin'
+objects:
+  - filename: frame-dw2.o
+    symbols:
+      - { sym: _bar, objAddr: 0x0, binAddr: 0x1000, size: 0x12 }
+      - { sym: _baz, objAddr: 0x0, binAddr: 0x2000, size: 0x12 }
+  - filename: frame-dw4.o
+    symbols:
+      - { sym: _baz, objAddr: 0x0, binAddr: 0x3000, size: 0x12 }
+  - filename: frame-dw2.o
+    symbols:
+      - { sym: _bar, objAddr: 0x0, binAddr: 0x4000, size: 0x12 }
+...
+
+# CHECK: .debug_frame contents:
+# CHECK: 00000000 {{[0-9a-f]*}} ffffffff CIE
+# CHECK-NEXT: Version:{{.*}}1
+# CHECK-NOT: FDE
+# CHECK:  FDE cie=00000000 pc=00001000...00001
+# CHECK-NOT: FDE
+# CHECK:  FDE cie=00000000 pc=00002000...00002
+# CHECK-NOT: FDE
+# CHECK: [[CIEDW4:[0-9a-f]*]] 00000010 ffffffff CIE
+# CHECK-NEXT: Version:{{.*}}4
+# CHECK-NOT: FDE
+# CHECK:  FDE cie=[[CIEDW4]] pc=00003000...00003
+# CHECK-NOT: FDE
+# CHECK: [[CIEDW2:[0-9a-f]*]] {{[0-9a-f]*}} ffffffff CIE
+# CHECK-NEXT: Version:{{.*}}1
+# CHECK-NOT: FDE
+# CHECK:  FDE cie=[[CIEDW2]] pc=00004000...00004
+# CHECK-NOT: FDE
+
diff --git a/test/tools/dsymutil/X86/odr-1.test b/test/tools/dsymutil/X86/odr-1.test
new file mode 100644
index 00000000000..607cd054b19
--- /dev/null
+++ b/test/tools/dsymutil/X86/odr-1.test
@@ -0,0 +1,423 @@
+# REQUIRES: object-emission
+# RUN: rm -rf %t
+# RUN: mkdir -p %t
+# RUN: llc -filetype=obj %p/../Inputs/odr1.ll -o %t/odr1.o
+# RUN: llc -filetype=obj %p/../Inputs/odr2.ll -o %t/odr2.o
+# RUN: llc -filetype=obj %p/../Inputs/odr3.ll -o %t/odr3.o
+# RUN: llc -filetype=obj %p/../Inputs/odr4.ll -o %t/odr4.o
+# RUN: llc -filetype=obj %p/../Inputs/odr5.ll -o %t/odr5.o
+# RUN: llc -filetype=obj %p/../Inputs/odr6.ll -o %t/odr6.o
+# RUN: llc -filetype=obj %p/../Inputs/odr7.ll -o %t/odr7.o
+# RUN: llvm-dsymutil -oso-prepend-path=%t -y %s -o - | llvm-dwarfdump -debug-dump=info - | FileCheck -check-prefix=ODR -check-prefix=CHECK %s
+# RUN: llvm-dsymutil -oso-prepend-path=%t -y %s -no-odr -o - | llvm-dwarfdump -debug-dump=info - | FileCheck -check-prefix=NOODR -check-prefix=CHECK %s
+
+# Totally made up debug map to test ODR uniquing
+
+---
+triple:          'x86_64-apple-darwin'
+objects:
+  - filename: odr1.o
+    symbols:
+      - { sym: __Z3foov, objAddr: 0x0, binAddr: 0x1000, size: 0x12 }
+      - { sym: __Z4funcv, objAddr: 0x0, binAddr: 0x2000, size: 0x12 }
+      - { sym: __ZZ4funcvENKUlvE_clEv,  objAddr: 0x0, binAddr: 0x3000, size: 0x12 }
+
+# Check that all our types are in the first CU.
+
+# CHECK: TAG_compile_unit
+# CHECK-NOT: {{DW_TAG|NULL}}
+# CHECK: AT_name{{.*}}"odr1.cpp"
+
+# This is "struct S"
+
+# CHECK: 0x[[S:[0-9a-f]*]]:{{.*}}DW_TAG_structure_type
+# CHECK-NEXT: DW_AT_name{{.*}}"S"
+# CHECK-NOT: NULL
+# CHECK: 0x[[NESTED:[0-9a-f]*]]:{{.*}}DW_TAG_structure_type
+# CHECK-NOT: {{DW_TAG|NULL}}
+# CHECK: DW_AT_name{{.*}}"Nested"
+# CHECK: NULL
+# CHECK: DW_TAG_subprogram
+# CHECK: DW_AT_MIPS_linkage_name{{.*}}"_ZN1S4incrEv"
+# CHECK: NULL
+# CHECK: DW_TAG_subprogram
+# CHECK: DW_AT_MIPS_linkage_name{{.*}}"_ZN1S4incrEi"
+# CHECK: NULL
+# CHECK: DW_TAG_subprogram
+# CHECK: DW_AT_MIPS_linkage_name{{.*}}"_ZN1S3fooEv"
+# CHECK: NULL
+# CHECK: NULL
+
+# This is "class N::C"
+
+# CHECK: DW_TAG_namespace
+# CHECK-NEXT: DW_AT_name{{.*}}"N"
+# CHECK-NOT: NULL
+# CHECK: 0x[[NC:[0-9a-f]*]]:{{.*}}DW_TAG_class_type
+# CHECK-NEXT: DW_AT_name{{.*}}"C"
+# CHECK: NULL
+
+# This is "class N::N::C"
+
+# CHECK: DW_TAG_namespace
+# CHECK-NEXT: DW_AT_name{{.*}}"N"
+# CHECK-NOT: NULL
+# CHECK: 0x[[NNC:[0-9a-f]*]]:{{.*}}DW_TAG_class_type
+# CHECK-NEXT: DW_AT_name{{.*}}"C"
+# CHECK: NULL
+# CHECK: NULL
+# CHECK: NULL
+
+# This is "AliasForS"
+# CHECK: 0x[[ALIASFORS:[0-9a-f]*]]:{{.*}}DW_TAG_typedef
+# CHECK-NEXT: DW_AT_type{{.*}}[[S]]
+# CHECK-NEXT: DW_AT_name{{.*}}"AliasForS"
+
+# This is "union U"
+
+# CHECK:  0x[[U:[0-9a-f]*]]:{{.*}}DW_TAG_union_type
+# CHECK-NEXT: DW_AT_name{{.*}}"U"
+# CHECK-NOT: NULL
+# CHECK:  0x[[UC:[0-9a-f]*]]:{{.*}}DW_TAG_class_type
+# CHECK-NOT: NULL
+# CHECK:  0x[[US:[0-9a-f]*]]:{{.*}}DW_TAG_structure_type
+# CHECK: NULL
+
+# This is "func" free function
+
+# CHECK: DW_TAG_subprogram
+# CHECK-NOT: {{DW_TAG|NULL}}
+# CHECK: DW_AT_name{{.*}}"func"
+# CHECK: 0x[[CINSIDEFUNC:[0-9a-f]*]]:{{.*}}DW_TAG_structure_type
+# CHECK-NEXT: DW_AT_name{{.*}}"CInsideFunc"
+
+# This is "(anonymous namespace)::AnonC"
+
+# CHECK: DW_TAG_namespace
+# CHECK-NOT: {{DW_AT_name|NULL|DW_TAG}}
+# CHECK: 0x[[ANONC:[0-9a-f]*]]:{{.*}}DW_TAG_class_type
+# CHECK-NEXT: DW_AT_name{{.*}}"AnonC"
+
+  - filename: odr1.o
+    symbols:
+      - { sym: __Z3foov, objAddr: 0x0, binAddr: 0x4000, size: 0x12 }
+      - { sym: __Z4funcv, objAddr: 0x0, binAddr: 0x5000, size: 0x12 }
+      - { sym: __ZZ4funcvENKUlvE_clEv,  objAddr: 0x0, binAddr: 0x6000, size: 0x12 }
+
+# We relink the same file a second time. In the ODR case, everything (except for the
+# union for now) should be uniqued. In the non-ODR case, we should get every type
+# duplicated.
+
+# CHECK: TAG_compile_unit
+# CHECK-NOT: {{DW_TAG|NULL}}
+# CHECK: AT_name{{.*}}"odr1.cpp"
+
+# ODR: DW_TAG_union_type
+# ODR-NEXT: DW_AT_name{{.*}}"U"
+# Types defined inside the union should be uniqued:
+# ODR: DW_TAG_member
+# ODR-NEXT: DW_AT_name{{.*}}"C"
+# ODR-NOT: {{NULL|DW_TAG}}
+# ODR: DW_AT_type{{.*}}[[UC]]
+# ODR: DW_TAG_member
+# ODR-NEXT: DW_AT_name{{.*}}"S"
+# ODR-NOT: {{NULL|DW_TAG}}
+# ODR: DW_AT_type{{.*}}[[US]]
+
+# Skip func
+# ODR: DW_TAG_subprogram
+# ODR-NOT: {{NULL|DW_TAG}}
+# ODR: DW_AT_name{{.*}}"func"
+# ODR: NULL
+
+# ODR: DW_TAG_subprogram
+# ODR-NOT: {{NULL|DW_TAG}}
+# ODR: DW_AT_name{{.*}}"foo"
+# ODR-NOT: NULL
+# ODR: DW_TAG_variable
+# ODR-NOT: {{DW_TAG|NULL}}
+# ODR: DW_AT_name{{.*}}"s"
+# ODR-NOT: {{DW_TAG|NULL}}
+# ODR: DW_AT_type{{.*}}[[ALIASFORS]]
+# ODR: DW_TAG_variable
+# ODR-NOT: {{DW_TAG|NULL}}
+# ODR: DW_AT_name{{.*}}"nc"
+# ODR-NOT: {{DW_TAG|NULL}}
+# ODR: DW_AT_type{{.*}}[[NC]]
+# ODR: DW_TAG_variable
+# ODR-NOT: {{DW_TAG|NULL}}
+# ODR: DW_AT_name{{.*}}"nnc"
+# ODR-NOT: {{DW_TAG|NULL}}
+# ODR: DW_AT_type{{.*}}[[NNC]]
+# ODR: DW_TAG_variable
+# ODR-NOT: {{DW_TAG|NULL}}
+# ODR: DW_AT_name{{.*}}"ac"
+# ODR-NOT: {{DW_TAG|NULL}}
+# ODR: DW_AT_type{{.*}}[[ANONC]]
+
+# ODR: DW_TAG_subprogram
+# ODR-NOT: {{NULL|DW_TAG}}
+# ODR: linkage_name{{.*}}"_ZZ4funcvENKUlvE_clEv"
+# ODR-NOT: NULL
+# ODR: DW_TAG_variable
+# ODR-NOT: DW_TAG
+# ODR: DW_AT_name{{.*}}"dummy"
+# ODR-NOT: NULL
+# ODR: DW_AT_type{{.*}}[[CINSIDEFUNC]]
+
+# With no ODR uniquing, we should get copies of all the types:
+
+# This is "struct S"
+# NOODR: 0x[[DUP_S:[0-9a-f]*]]:{{.*}}DW_TAG_structure_type
+# NOODR-NEXT: DW_AT_name{{.*}}"S"
+
+# This is "class N::C"
+# NOODR: DW_TAG_namespace
+# NOODR-NEXT: DW_AT_name{{.*}}"N"
+# NOODR: 0x[[DUP_NC:[0-9a-f]*]]:{{.*}}DW_TAG_class_type
+# NOODR-NEXT: DW_AT_name{{.*}}"C"
+
+# This is "class N::N::C"
+# NOODR: DW_TAG_namespace
+# NOODR-NEXT: DW_AT_name{{.*}}"N"
+# NOODR: 0x[[DUP_NNC:[0-9a-f]*]]:{{.*}}DW_TAG_class_type
+# NOODR-NEXT: DW_AT_name{{.*}}"C"
+
+# This is "AliasForS"
+# NOODR: 0x[[DUP_ALIASFORS:[0-9a-f]*]]:{{.*}}DW_TAG_typedef
+# NOODR-NOT: {{NULL|DW_TAG}}
+# NOODR: DW_AT_name{{.*}}"AliasForS"
+
+# This is "union U"
+
+# NOODR:  0x[[U:[0-9a-f]*]]:{{.*}}DW_TAG_union_type
+# NOODR-NEXT: DW_AT_name{{.*}}"U"
+# NOODR-NOT: NULL
+# NOODR:  0x[[DUP_UC:[0-9a-f]*]]:{{.*}}DW_TAG_class_type
+# NOODR-NOT: NULL
+# NOODR:  0x[[DUP_US:[0-9a-f]*]]:{{.*}}DW_TAG_structure_type
+# NOODR: NULL
+
+# This is "func" free function
+
+# NOODR: DW_TAG_subprogram
+# NOODR-NOT: {{DW_TAG|NULL}}
+# NOODR: DW_AT_name{{.*}}"func"
+# NOODR: 0x[[DUP_CINSIDEFUNC:[0-9a-f]*]]:{{.*}}DW_TAG_structure_type
+# NOODR-NEXT: DW_AT_name{{.*}}"CInsideFunc"
+
+# NOODR: DW_TAG_subprogram
+# NOODR-NOT: {{NULL|DW_TAG}}
+# NOODR: DW_AT_name{{.*}}"foo"
+# NOODR-NOT: NULL
+# NOODR: DW_TAG_variable
+# NOODR-NOT: {{DW_TAG|NULL}}
+# NOODR: DW_AT_name{{.*}}"s"
+# NOODR-NOT: {{DW_TAG|NULL}}
+# NOODR: DW_AT_type{{.*}}[[DUP_ALIASFORS]]
+# NOODR: DW_TAG_variable
+# NOODR-NOT: {{DW_TAG|NULL}}
+# NOODR: DW_AT_name{{.*}}"nc"
+# NOODR-NOT: {{DW_TAG|NULL}}
+# NOODR: DW_AT_type{{.*}}[[DUP_NC]]
+# NOODR: DW_TAG_variable
+# NOODR-NOT: {{DW_TAG|NULL}}
+# NOODR: DW_AT_name{{.*}}"nnc"
+# NOODR-NOT: {{DW_TAG|NULL}}
+# NOODR: DW_AT_type{{.*}}[[DUP_NNC]]
+# NOODR: DW_TAG_variable
+# NOODR-NOT: {{DW_TAG|NULL}}
+# NOODR: DW_AT_name{{.*}}"ac"
+# NOODR-NOT: {{DW_TAG|NULL}}
+# NOODR: DW_AT_type{{.*}}0x[[DUP_ANONC:[0-9a-f]*]]
+
+# This is the lanbda inside func
+
+# NOODR: DW_TAG_subprogram
+# NOODR-NOT: {{NULL|DW_TAG}}
+# NOODR: linkage_name{{.*}}"_ZZ4funcvENKUlvE_clEv"
+# NOODR-NOT: NULL
+# NOODR: DW_TAG_variable
+# NOODR-NOT: DW_TAG
+# NOODR: DW_AT_name{{.*}}"dummy"
+# NOODR-NOT: NULL
+# NOODR: DW_AT_type{{.*}}[[DUP_CINSIDEFUNC]]
+
+# This is "(anonymous namespace)::AnonC"
+
+# NOODR: DW_TAG_namespace
+# NOODR-NOT: {{DW_AT_name|NULL|DW_TAG}}
+# NOODR: 0x[[DUP_ANONC]]:{{.*}}DW_TAG_class_type
+# NOODR-NEXT: DW_AT_name{{.*}}"AnonC"
+
+  - filename: odr2.o
+    symbols:
+      - { sym: __ZN1S3fooEv, objAddr: 0x0, binAddr: 0x7000, size: 0x12 }
+      - { sym: __Z3barv, objAddr: 0x0, binAddr: 0x8000, size: 0x12 }
+  - filename: odr3.o
+    symbols:
+      - { sym: __ZN1S3fooEv, objAddr: 0x0, binAddr: 0x8000, size: 0x12 }
+      - { sym: __Z3barv, objAddr: 0x0, binAddr: 0x9000, size: 0x12 }
+
+# odr2.cpp and odr3.cpp test that a simple overloaded function doesn't break the
+# uniquing (contrary to what we'll see with template/artificial) functions. 
+
+# CHECK: TAG_compile_unit
+# CHECK-NOT: {{DW_TAG|NULL}}
+# CHECK: AT_name{{.*}}"odr2.cpp"
+
+# NO-ODR: DW_TAG_structure_type
+# ODR-NOT: DW_TAG_structure_type
+
+# ODR: DW_TAG_subprogram
+# ODR: DW_AT_specification{{.*}}4incr
+# ODR: DW_TAG_formal_parameter
+# ODR-NEXT: DW_AT_name{{.*}}"this"
+# ODR-NEXT: DW_AT_type{{.*}}0x00000000[[S_PTR:[0-9a-f]*]]
+# ODR: 0x[[S_PTR]]:{{.*}}DW_TAG_pointer_type
+# ODR-NEXT: DW_AT_type{{.*}}[[S]]
+# ODR: DW_TAG_subprogram
+# ODR-NOT: {{DW_TAG|NULL}}
+# ODR: DW_AT_name{{.*}}"bar"
+# ODR-NOT: NULL
+# ODR: DW_TAG_variable
+# ODR-NOT: {{DW_TAG|NULL}}
+# ODR: DW_AT_type{{.*}}[[S]]
+# ODR-NOT NULL
+# DOR: DW_TAG_inlined_subroutine
+# ODR-NOT NULL
+# ODR: DW_TAG_formal_parameter
+# ODR-NOT {{NULL|DW_TAG}}
+# ODR: DW_AT_type{{.*}}[[S_PTR]]
+
+# CHECK: TAG_compile_unit
+# CHECK-NOT: {{DW_TAG|NULL}}
+# CHECK: AT_name{{.*}}"odr3.cpp"
+
+# NO-ODR: DW_TAG_structure_type
+# ODR-NOT: DW_TAG_structure_type
+
+# ODR: DW_TAG_subprogram
+# ODR: DW_AT_specification{{.*}}4incr
+# ODR: DW_TAG_formal_parameter
+# ODR-NEXT: DW_AT_name{{.*}}"this"
+# ODR-NEXT: DW_AT_type{{.*}}0x00000000[[S_PTR2:[0-9a-f]*]]
+# ODR: 0x[[S_PTR2]]:{{.*}}DW_TAG_pointer_type
+# ODR-NEXT: DW_AT_type{{.*}}[[S]]
+# ODR: DW_TAG_subprogram
+# ODR-NOT: {{DW_TAG|NULL}}
+# ODR: DW_AT_name{{.*}}"bar"
+# ODR-NOT: NULL
+# ODR: DW_TAG_variable
+# ODR-NOT: {{DW_TAG|NULL}}
+# ODR: DW_AT_type{{.*}}[[S]]
+# ODR-NOT NULL
+# DOR: DW_TAG_inlined_subroutine
+# ODR-NOT NULL
+# ODR: DW_TAG_formal_parameter
+# ODR-NOT {{NULL|DW_TAG}}
+# ODR: DW_AT_type{{.*}}[[S_PTR2]]
+
+  - filename: odr4.o
+    symbols:
+      - { sym: __Z3bazv, objAddr: 0x0, binAddr: 0xa000, size: 0x12 }
+
+# odr4.cpp helps check that anonymous namespaces with similarly named contents do
+# not get uniqued.
+
+# CHECK: TAG_compile_unit
+# CHECK-NOT: {{DW_TAG|NULL}}
+# CHECK: AT_name{{.*}}"odr4.cpp"
+
+# CHECK: DW_TAG_subprogram
+# CHECK-NOT: NULL
+# CHECK: DW_TAG_variable
+# CHECK-NOT: DW_TAG
+# ODR: DW_AT_type{{.*}}[[LOCALANONC:........]])
+# NOODR: DW_AT_type{{.*}}[[LOCALANONC:........]]})
+
+# CHECK: DW_TAG_namespace
+# CHECK-NOT: DW_AT_name
+# CHECK: [[LOCALANONC]]{{.*}}DW_TAG_class_type
+# CHECK-NOT: {{NULL|DW_TAG}}
+# CHECK: DW_AT_name{{.*}}"AnonC"
+
+  - filename: odr5.o
+    symbols:
+      - { sym: __Z3bazv, objAddr: 0x0, binAddr: 0xb000, size: 0x12 }
+      - { sym: __ZN1S6Nested4initIiEEvT_, objAddr: 0x0, binAddr: 0xc000, size: 0x12 }
+  - filename: odr6.o
+    symbols:
+      - { sym: __Z3bazv, objAddr: 0x0, binAddr: 0xd000, size: 0x12 }
+      - { sym: __ZN1S6Nested4initIdEEvT_, objAddr: 0x0, binAddr: 0xe000, size: 0x12 }
+
+# odr5.cpp and odr6.cpp instanciate a template member function of the S class.
+# They instanciate it with different types. As DWARF only describes the actual
+# intances, these members aren't described in the uniqued class definition of
+# odr1.cpp. Both these files should contain a new copy of S' definition with
+# the template instance included.
+
+# CHECK: TAG_compile_unit
+# CHECK-NOT: {{DW_TAG|NULL}}
+# CHECK: AT_name{{.*}}"odr5.cpp"
+
+# CHECK: 0x{{[0-9a-f]*}}:{{.*}}DW_TAG_structure_type
+# CHECK-NEXT: DW_AT_name{{.*}}"S"
+# CHECK-NOT: NULL
+# CHECK: 0x[[NESTED2:[0-9a-f]*]]:{{.*}}DW_TAG_structure_type
+# CHECK-NOT: {{DW_TAG|NULL}}
+# CHECK: DW_AT_name{{.*}}"Nested"
+# CHECK-NOT: NULL
+# CHECK: 0x[[INITTEMPLATE:[0-9a-f]*]]:{{.*}}DW_TAG_subprogram
+# CHECK-NEXT:{{.*}}"_ZN1S6Nested4init
+
+# CHECK: DW_AT_specification{{.*}}[[INITTEMPLATE]]
+# CHECK: DW_TAG_formal_parameter
+# CHECK-NOT: DW_TAG
+# CHECK: DW_AT_type{{.*}}[[NESTED_PTR:[0-9a-f]{8}]]{{[}]?}})
+
+# CHECK: 0x[[NESTED_PTR]]{{.*}}DW_TAG_pointer_type
+# ODR-NEXT: DW_AT_type{{.*}}[[NESTED]]
+# NOODR-NEXT: DW_AT_type{{.*}}[[NESTED2]]
+
+# CHECK: TAG_compile_unit
+# CHECK-NOT: {{DW_TAG|NULL}}
+# CHECK: AT_name{{.*}}"odr6.cpp"
+
+# CHECK: 0x{{[0-9a-f]*}}:{{.*}}DW_TAG_structure_type
+# CHECK-NEXT: DW_AT_name{{.*}}"S"
+# CHECK-NOT: NULL
+# CHECK: 0x[[NESTED3:[0-9a-f]*]]:{{.*}}DW_TAG_structure_type
+# CHECK-NOT: {{DW_TAG|NULL}}
+# CHECK: DW_AT_name{{.*}}"Nested"
+# CHECK-NOT: NULL
+# CHECK: 0x[[INITTEMPLATE2:[0-9a-f]*]]:{{.*}}DW_TAG_subprogram
+# CHECK-NEXT:{{.*}}"_ZN1S6Nested4init
+
+# CHECK: DW_AT_specification{{.*}}[[INITTEMPLATE2]]
+# CHECK: DW_TAG_formal_parameter
+# CHECK-NOT: DW_TAG
+# CHECK: DW_AT_type{{.*}}[[NESTED_PTR2:[0-9a-f]{8}]]{{[}]?}})
+
+# CHECK: 0x[[NESTED_PTR2]]{{.*}}DW_TAG_pointer_type
+# ODR-NEXT: DW_AT_type{{.*}}[[NESTED]]
+# NOODR-NEXT: DW_AT_type{{.*}}[[NESTED3]]
+
+  - filename: odr7.o
+    symbols:
+      - { sym: __Z3foov, objAddr: 0x0, binAddr: 0xf000, size: 0x12 }
+
+# Check that a reference to a nested class correctly refers to the original
+# definition
+
+# CHECK: TAG_compile_unit
+# CHECK-NOT: {{DW_TAG|NULL}}
+# CHECK: AT_name{{.*}}"odr7.cpp"
+
+# ODR: DW_TAG_subprogram
+# ODR-NOT: NULL
+# ODR: DW_TAG_variable
+# ODR-NOT: DW_TAG
+# ODR: DW_AT_type{{.*}}[[NESTED]]
+...
+
diff --git a/test/tools/dsymutil/archive-timestamp.test b/test/tools/dsymutil/archive-timestamp.test
new file mode 100644
index 00000000000..f3f2162fa59
--- /dev/null
+++ b/test/tools/dsymutil/archive-timestamp.test
@@ -0,0 +1,24 @@
+# RUN: llvm-dsymutil -no-output -oso-prepend-path=%p -y %s 2>&1 | FileCheck %s
+
+# This is the archive member part of basic-archive.macho.x86_64 debug map with corrupted timestamps.
+
+# CHECK: warning: {{.*}}libbasic.a(basic2.macho.x86_64.o): {{[Nn]o}} such file
+# CHECK: warning: {{.*}}libbasic.a(basic3.macho.x86_64.o): {{[Nn]o}} such file
+
+---
+triple:          'x86_64-apple-darwin'
+objects:
+  - filename:        '/Inputs/libbasic.a(basic2.macho.x86_64.o)'
+    timestamp:       141869239
+    symbols:
+      - { sym: _foo, objAddr: 0x0000000000000020, binAddr: 0x0000000100000ED0, size: 0x00000050 }
+      - { sym: _private_int, objAddr: 0x0000000000000560, binAddr: 0x0000000100001004, size: 0x00000000 }
+      - { sym: _inc, objAddr: 0x0000000000000070, binAddr: 0x0000000100000F20, size: 0x00000017 }
+      - { sym: _baz, objAddr: 0x0000000000000310, binAddr: 0x0000000100001000, size: 0x00000000 }
+  - filename:        '/Inputs/libbasic.a(basic3.macho.x86_64.o)'
+    timestamp:       418692393
+    symbols:
+      - { sym: _val, objAddr: 0x0000000000000004, binAddr: 0x0000000100001008, size: 0x00000000 }
+      - { sym: _bar, objAddr: 0x0000000000000020, binAddr: 0x0000000100000F40, size: 0x00000050 }
+      - { sym: _inc, objAddr: 0x0000000000000070, binAddr: 0x0000000100000F90, size: 0x00000019 }
+...
diff --git a/test/tools/dsymutil/basic-linking.test b/test/tools/dsymutil/basic-linking.test
index ec6a5b77146..362df19a024 100644
--- a/test/tools/dsymutil/basic-linking.test
+++ b/test/tools/dsymutil/basic-linking.test
@@ -1,6 +1,6 @@
-RUN: llvm-dsymutil -no-output -v -oso-prepend-path=%p %p/Inputs/basic.macho.x86_64 | FileCheck %s
-RUN: llvm-dsymutil -no-output -v -oso-prepend-path=%p %p/Inputs/basic-lto.macho.x86_64 | FileCheck %s --check-prefix=CHECK-LTO
-RUN: llvm-dsymutil -no-output -v -oso-prepend-path=%p %p/Inputs/basic-archive.macho.x86_64 | FileCheck %s --check-prefix=CHECK-ARCHIVE
+RUN: llvm-dsymutil -no-output -verbose -oso-prepend-path=%p %p/Inputs/basic.macho.x86_64 | FileCheck %s
+RUN: llvm-dsymutil -no-output -verbose -oso-prepend-path=%p %p/Inputs/basic-lto.macho.x86_64 | FileCheck %s --check-prefix=CHECK-LTO
+RUN: llvm-dsymutil -no-output -verbose -oso-prepend-path=%p %p/Inputs/basic-archive.macho.x86_64 | FileCheck %s --check-prefix=CHECK-ARCHIVE
 
 This test check the basic Dwarf linking process through the debug dumps.
 
diff --git a/test/tools/dsymutil/debug-map-parsing.test b/test/tools/dsymutil/debug-map-parsing.test
index 3422316191f..9415c94873b 100644
--- a/test/tools/dsymutil/debug-map-parsing.test
+++ b/test/tools/dsymutil/debug-map-parsing.test
@@ -1,40 +1,42 @@
-RUN: llvm-dsymutil -v -parse-only -oso-prepend-path=%p %p/Inputs/basic.macho.x86_64 | FileCheck %s
-RUN: llvm-dsymutil -v -parse-only -oso-prepend-path=%p %p/Inputs/basic-lto.macho.x86_64 | FileCheck %s --check-prefix=CHECK-LTO
-RUN: llvm-dsymutil -v -parse-only -oso-prepend-path=%p %p/Inputs/basic-archive.macho.x86_64 | FileCheck %s --check-prefix=CHECK-ARCHIVE
-RUN: llvm-dsymutil -v -parse-only %p/Inputs/basic.macho.x86_64 2>&1 | FileCheck %s --check-prefix=NOT-FOUND
-RUN: not llvm-dsymutil -v -parse-only %p/Inputs/inexistant 2>&1 | FileCheck %s --check-prefix=NO-EXECUTABLE
+RUN: llvm-dsymutil -dump-debug-map -oso-prepend-path=%p %p/Inputs/basic.macho.x86_64 | FileCheck %s
+RUN: llvm-dsymutil -dump-debug-map -oso-prepend-path=%p %p/Inputs/basic-lto.macho.x86_64 | FileCheck %s --check-prefix=CHECK-LTO
+RUN: llvm-dsymutil -verbose -dump-debug-map -oso-prepend-path=%p %p/Inputs/basic-archive.macho.x86_64 | FileCheck %s --check-prefix=CHECK-ARCHIVE
+RUN: llvm-dsymutil -dump-debug-map %p/Inputs/basic.macho.x86_64 2>&1 | FileCheck %s --check-prefix=NOT-FOUND
+RUN: not llvm-dsymutil -dump-debug-map %p/Inputs/inexistant 2>&1 | FileCheck %s --check-prefix=NO-EXECUTABLE
 
 
 Check that We can parse the debug map of the basic executable.
 
 CHECK-NOT: error
-CHECK: DEBUG MAP: x86_64-unknown-unknown-macho
-CHECK: /Inputs/basic1.macho.x86_64.o:
-CHECK: 	0000000000000000 => 0000000100000ea0+0x24	_main
-CHECK: /Inputs/basic2.macho.x86_64.o:
-CHECK: 	0000000000000310 => 0000000100001000+0x0	_baz
-CHECK: 	0000000000000020 => 0000000100000ed0+0x50	_foo
-CHECK: 	0000000000000070 => 0000000100000f20+0x17	_inc
-CHECK: 	0000000000000560 => 0000000100001008+0x0	_private_int
-CHECK: /Inputs/basic3.macho.x86_64.o:
-CHECK: 	0000000000000020 => 0000000100000f40+0x50	_bar
-CHECK: 	0000000000000070 => 0000000100000f90+0x19	_inc
-CHECK: 	0000000000000004 => 0000000100001004+0x0	_val
-CHECK: END DEBUG MAP
+CHECK: ---
+CHECK: triple: 'x86_64-apple-darwin'
+CHECK: filename:{{.*}}/Inputs/basic1.macho.x86_64.o
+CHECK-DAG: sym: _main, objAddr: 0x0000000000000000, binAddr: 0x0000000100000EA0, size: 0x00000024
+CHECK: filename{{.*}}/Inputs/basic2.macho.x86_64.o
+CHECK-DAG: 	sym: _baz, objAddr: 0x0000000000000310, binAddr: 0x0000000100001000, size: 0x00000000
+CHECK-DAG: 	sym: _foo, objAddr: 0x0000000000000020, binAddr: 0x0000000100000ED0, size: 0x00000050
+CHECK-DAG: 	sym: _inc, objAddr: 0x0000000000000070, binAddr: 0x0000000100000F20, size: 0x00000017
+CHECK-DAG: 	sym: _private_int, objAddr: 0x0000000000000560, binAddr: 0x0000000100001008, size: 0x00000000
+CHECK: filename{{.*}}/Inputs/basic3.macho.x86_64.o
+CHECK-DAG: 	sym: _bar, objAddr: 0x0000000000000020, binAddr: 0x0000000100000F40, size: 0x00000050
+CHECK-DAG: 	sym: _inc, objAddr: 0x0000000000000070, binAddr: 0x0000000100000F90, size: 0x00000019
+CHECK-DAG: 	sym: _val, objAddr: 0x0000000000000004, binAddr: 0x0000000100001004, size: 0x00000000
+CHECK: ...
 
 
 Check that we can parse the debug-map of the basic-lto executable
 
 CHECK-LTO-NOT: error
-CHECK-LTO: DEBUG MAP: x86_64-unknown-unknown-macho
-CHECK-LTO: /Inputs/basic-lto.macho.x86_64.o:
-CHECK-LTO: 	0000000000000050 => 0000000100000f90+0x24	_bar
-CHECK-LTO: 	0000000000000658 => 0000000100001000+0x0	_baz
-CHECK-LTO: 	0000000000000010 => 0000000100000f50+0x40	_foo
-CHECK-LTO: 	0000000000000000 => 0000000100000f40+0x10	_main
-CHECK-LTO: 	00000000000008e8 => 0000000100001008+0x0	_private_int
-CHECK-LTO: 	00000000000008ec => 0000000100001004+0x0	_val
-CHECK-LTO: END DEBUG MAP
+CHECK-LTO: ---
+CHECK-LTO: triple: 'x86_64-apple-darwin'
+CHECK-LTO: /Inputs/basic-lto.macho.x86_64.o
+CHECK-LTO-DAG: 	sym: _bar, objAddr: 0x0000000000000050, binAddr: 0x0000000100000F90, size: 0x00000024
+CHECK-LTO-DAG: 	sym: _baz, objAddr: 0x0000000000000658, binAddr: 0x0000000100001000, size: 0x00000000
+CHECK-LTO-DAG: 	sym: _foo, objAddr: 0x0000000000000010, binAddr: 0x0000000100000F50, size: 0x00000040
+CHECK-LTO-DAG: 	sym: _main, objAddr: 0x0000000000000000, binAddr: 0x0000000100000F40, size: 0x00000010
+CHECK-LTO-DAG: 	sym: _private_int, objAddr: 0x00000000000008E8, binAddr: 0x0000000100001008, size: 0x00000000
+CHECK-LTO-DAG: 	sym: _val, objAddr: 0x00000000000008EC, binAddr: 0x0000000100001004, size: 0x00000000
+CHECK-LTO: ...
 
 Check thet we correctly handle debug maps with archive members (including only
 opening the archive once if mulitple of its members are used).
@@ -48,20 +50,20 @@ CHECK-ARCHIVE-NEXT: 	opened new archive {{.*}}/libbasic.a'
 CHECK-ARCHIVE-NEXT: 	found member in current archive.
 CHECK-ARCHIVE-NEXT: trying to open {{.*}}/libbasic.a(basic3.macho.x86_64.o)'
 CHECK-ARCHIVE-NEXT: 	found member in current archive.
-CHECK-ARCHIVE: DEBUG MAP: x86_64-unknown-unknown-macho
-CHECK-ARCHIVE:       object addr =>  executable addr	symbol name
-CHECK-ARCHIVE: /Inputs/basic1.macho.x86_64.o:
-CHECK-ARCHIVE: 	0000000000000000 => 0000000100000ea0+0x24	_main
-CHECK-ARCHIVE: /Inputs/./libbasic.a(basic2.macho.x86_64.o):
-CHECK-ARCHIVE: 	0000000000000310 => 0000000100001000+0x0	_baz
-CHECK-ARCHIVE: 	0000000000000020 => 0000000100000ed0+0x50	_foo
-CHECK-ARCHIVE: 	0000000000000070 => 0000000100000f20+0x17	_inc
-CHECK-ARCHIVE: 	0000000000000560 => 0000000100001004+0x0	_private_int
-CHECK-ARCHIVE: /Inputs/./libbasic.a(basic3.macho.x86_64.o):
-CHECK-ARCHIVE: 	0000000000000020 => 0000000100000f40+0x50	_bar
-CHECK-ARCHIVE: 	0000000000000070 => 0000000100000f90+0x19	_inc
-CHECK-ARCHIVE: 	0000000000000004 => 0000000100001008+0x0	_val
-CHECK-ARCHIVE: END DEBUG MAP
+CHECK-ARCHIVE: ---
+CHECK-ARCHIVE: triple: 'x86_64-apple-darwin'
+CHECK-ARCHIVE: /Inputs/basic1.macho.x86_64.o
+CHECK-ARCHIVE-DAG: 	sym: _main, objAddr: 0x0000000000000000, binAddr: 0x0000000100000EA0, size: 0x00000024
+CHECK-ARCHIVE: /Inputs/./libbasic.a(basic2.macho.x86_64.o)
+CHECK-ARCHIVE-DAG: 	sym: _baz, objAddr: 0x0000000000000310, binAddr: 0x0000000100001000, size: 0x00000000
+CHECK-ARCHIVE-DAG: 	sym: _foo, objAddr: 0x0000000000000020, binAddr: 0x0000000100000ED0, size: 0x00000050
+CHECK-ARCHIVE-DAG: 	sym: _inc, objAddr: 0x0000000000000070, binAddr: 0x0000000100000F20, size: 0x00000017
+CHECK-ARCHIVE-DAG: 	sym: _private_int, objAddr: 0x0000000000000560, binAddr: 0x0000000100001004, size: 0x00000000
+CHECK-ARCHIVE: /Inputs/./libbasic.a(basic3.macho.x86_64.o)
+CHECK-ARCHIVE-DAG: 	sym: _bar, objAddr: 0x0000000000000020, binAddr: 0x0000000100000F40, size: 0x00000050
+CHECK-ARCHIVE-DAG: 	sym: _inc, objAddr: 0x0000000000000070, binAddr: 0x0000000100000F90, size: 0x00000019
+CHECK-ARCHIVE-DAG: 	sym: _val, objAddr: 0x0000000000000004, binAddr: 0x0000000100001008, size: 0x00000000
+CHECK-ARCHIVE: ...
 
 Check that we warn about missing object files (this presumes that the files aren't
 present in the machine's /Inputs/ folder, which should be a pretty safe bet).
@@ -69,11 +71,11 @@ present in the machine's /Inputs/ folder, which should be a pretty safe bet).
 NOT-FOUND: cannot open{{.*}}"/Inputs/basic1.macho.x86_64.o": {{[Nn]o}} such file
 NOT-FOUND: cannot open{{.*}}"/Inputs/basic2.macho.x86_64.o": {{[Nn]o}} such file
 NOT-FOUND: cannot open{{.*}}"/Inputs/basic3.macho.x86_64.o": {{[Nn]o}} such file
-NOT-FOUND: DEBUG MAP:
-NOT-FOUND-NEXT:  object addr =>  executable addr	symbol name
-NOT-FOUND-NEXT: END DEBUG MAP
+NOT-FOUND: ---
+NOT-FOUND-NEXT: triple: 'x86_64-apple-darwin'
+NOT-FOUND-NEXT: ...
 
 Check that we correctly error out on invalid executatble.
 
 NO-EXECUTABLE: cannot parse{{.*}}/inexistant": {{[Nn]o}} such file
-NO-EXECUTABLE-NOT: DEBUG MAP
+NO-EXECUTABLE-NOT: ---
diff --git a/test/tools/dsymutil/yaml-object-address-rewrite.test b/test/tools/dsymutil/yaml-object-address-rewrite.test
new file mode 100644
index 00000000000..cebda0c62c0
--- /dev/null
+++ b/test/tools/dsymutil/yaml-object-address-rewrite.test
@@ -0,0 +1,47 @@
+# RUN: llvm-dsymutil -dump-debug-map -oso-prepend-path=%p -y %s | FileCheck %s
+#
+# The YAML debug map bellow is the one from basic-archive.macho.x86_64 with
+# the object addresses set to zero. Check that the YAML import is able to
+# rewrite these addresses to the right values.
+#
+# CHECK: ---
+# CHECK-NEXT: triple:{{.*}}'x86_64-apple-darwin'
+# CHECK-NEXT: objects:
+# CHECK-NEXT: filename:{{.*}}/Inputs/basic1.macho.x86_64.o
+# CHECK-NEXT: timestamp: 0
+# CHECK-NEXT: symbols:
+# CHECK-NEXT: sym: _main, objAddr: 0x0000000000000000, binAddr: 0x0000000100000EA0, size: 0x00000024
+# CHECK-NEXT: filename:{{.*}}/Inputs/./libbasic.a(basic2.macho.x86_64.o)'
+# CHECK-NEXT: timestamp: 0
+# CHECK-NEXT: symbols:
+# CHECK-DAG:   sym: _foo, objAddr: 0x0000000000000020, binAddr: 0x0000000100000ED0, size: 0x00000050
+# CHECK-DAG:   sym: _private_int, objAddr: 0x0000000000000560, binAddr: 0x0000000100001004, size: 0x00000000
+# CHECK-DAG:   sym: _inc, objAddr: 0x0000000000000070, binAddr: 0x0000000100000F20, size: 0x00000017
+# CHECK-DAG:   sym: _baz, objAddr: 0x0000000000000310, binAddr: 0x0000000100001000, size: 0x00000000
+# CHECK-NOT: { sym:
+# CHECK-NEXT: filename:{{.*}}/Inputs/./libbasic.a(basic3.macho.x86_64.o)'
+# CHECK-NEXT: timestamp: 0
+# CHECK-NEXT: symbols:
+# CHECK-DAG:   sym: _val, objAddr: 0x0000000000000004, binAddr: 0x0000000100001008, size: 0x00000000
+# CHECK-DAG:   sym: _bar, objAddr: 0x0000000000000020, binAddr: 0x0000000100000F40, size: 0x00000050
+# CHECK-DAG:   sym: _inc, objAddr: 0x0000000000000070, binAddr: 0x0000000100000F90, size: 0x00000019
+# CHECK-NOT: { sym:
+# CHECK-NEXT: ...
+---
+triple:          'x86_64-apple-darwin'
+objects:
+  - filename: /Inputs/basic1.macho.x86_64.o
+    symbols:
+      - { sym: _main, objAddr: 0x0, binAddr: 0x0000000100000EA0, size: 0x00000024 }
+  - filename: /Inputs/./libbasic.a(basic2.macho.x86_64.o)
+    symbols:
+      - { sym: _foo, objAddr: 0x0, binAddr: 0x0000000100000ED0, size: 0x00000050 }
+      - { sym: _private_int, objAddr: 0x0, binAddr: 0x0000000100001004, size: 0x00000000 }
+      - { sym: _inc, objAddr: 0x0, binAddr: 0x0000000100000F20, size: 0x00000017 }
+      - { sym: _baz, objAddr: 0x0, binAddr: 0x0000000100001000, size: 0x00000000 }
+  - filename: /Inputs/./libbasic.a(basic3.macho.x86_64.o)
+    symbols:
+      - { sym: _val, objAddr: 0x0, binAddr: 0x0000000100001008, size: 0x00000000 }
+      - { sym: _bar, objAddr: 0x0, binAddr: 0x0000000100000F40, size: 0x00000050 }
+      - { sym: _inc, objAddr: 0x0, binAddr: 0x0000000100000F90, size: 0x00000019 }
+...
diff --git a/test/tools/gold/emit-llvm.ll b/test/tools/gold/emit-llvm.ll
index f851fbfb5e0..bfb90c4bc28 100644
--- a/test/tools/gold/emit-llvm.ll
+++ b/test/tools/gold/emit-llvm.ll
@@ -12,6 +12,7 @@
 ; RUN:    -shared %t.o -o %t3.o
 ; RUN: llvm-dis %t3.o.bc -o - | FileCheck %s
 ; RUN: llvm-dis %t3.o.opt.bc -o - | FileCheck --check-prefix=OPT %s
+; RUN: llvm-nm %t3.o.o | FileCheck --check-prefix=NM %s
 
 ; RUN: rm -f %t4.o
 ; RUN: %gold -plugin %llvmshlibdir/LLVMgold.so \
@@ -19,6 +20,8 @@
 ; RUN:    -shared %t.o -o %t4.o
 ; RUN: not test -a %t4.o
 
+; NM: T f3
+
 target triple = "x86_64-unknown-linux-gnu"
 
 @g7 = extern_weak global i32
diff --git a/test/tools/gold/slp-vectorize.ll b/test/tools/gold/slp-vectorize.ll
index a75f0b38cc1..30950b2d2de 100644
--- a/test/tools/gold/slp-vectorize.ll
+++ b/test/tools/gold/slp-vectorize.ll
@@ -1,6 +1,6 @@
 ; RUN: llvm-as %s -o %t.o
 
-; RUN: %gold -plugin %llvmshlibdir/LLVMgold.so \
+; RUN: %gold -m elf_x86_64 -plugin %llvmshlibdir/LLVMgold.so \
 ; RUN:    --plugin-opt=save-temps \
 ; RUN:    -shared %t.o -o %t2.o
 ; RUN: llvm-dis %t2.o.opt.bc -o - | FileCheck %s
@@ -8,7 +8,7 @@
 ; test that the vectorizer is run.
 ; CHECK: fadd <4 x float>
 
-target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
 
 define void @f(float* nocapture %x) {
   %tmp = load float, float* %x, align 4
diff --git a/test/tools/llvm-cov/Inputs/lineExecutionCounts.proftext b/test/tools/llvm-cov/Inputs/lineExecutionCounts.proftext
index 1966d42ce30..38387a6237a 100644
--- a/test/tools/llvm-cov/Inputs/lineExecutionCounts.proftext
+++ b/test/tools/llvm-cov/Inputs/lineExecutionCounts.proftext
@@ -1,8 +1,8 @@
 main
 0x000000000028434d
 5
-1
+161
 0
-100
-1
+16100
+161
 0
diff --git a/test/tools/llvm-cov/Inputs/regionMarkers.proftext b/test/tools/llvm-cov/Inputs/regionMarkers.proftext
index 1966d42ce30..5199edc5394 100644
--- a/test/tools/llvm-cov/Inputs/regionMarkers.proftext
+++ b/test/tools/llvm-cov/Inputs/regionMarkers.proftext
@@ -1,8 +1,8 @@
 main
 0x000000000028434d
 5
-1
+1111000
 0
-100
-1
+111100000
+1111000
 0
diff --git a/test/tools/llvm-cov/showLineExecutionCounts.cpp b/test/tools/llvm-cov/showLineExecutionCounts.cpp
index d5254da788b..83fee977e6d 100644
--- a/test/tools/llvm-cov/showLineExecutionCounts.cpp
+++ b/test/tools/llvm-cov/showLineExecutionCounts.cpp
@@ -1,30 +1,30 @@
 // Basic handling of line counts.
 // RUN: llvm-profdata merge %S/Inputs/lineExecutionCounts.proftext -o %t.profdata
 
-// before any coverage              // WHOLE-FILE:    | [[@LINE]]|// before
-                                    // FILTER-NOT:    | [[@LINE-1]]|// before
-int main() {                             // CHECK:   1| [[@LINE]]|int main(
-  int x = 0;                             // CHECK:   1| [[@LINE]]|  int x
-                                         // CHECK:   1| [[@LINE]]|
-  if (x) {                               // CHECK:   0| [[@LINE]]|  if (x)
-    x = 0;                               // CHECK:   0| [[@LINE]]|    x = 0
-  } else {                               // CHECK:   1| [[@LINE]]|  } else
-    x = 1;                               // CHECK:   1| [[@LINE]]|    x = 1
-  }                                      // CHECK:   1| [[@LINE]]|  }
-                                         // CHECK:   1| [[@LINE]]|
-  for (int i = 0; i < 100; ++i) {        // CHECK: 101| [[@LINE]]|  for (
-    x = 1;                               // CHECK: 100| [[@LINE]]|    x = 1
-  }                                      // CHECK: 100| [[@LINE]]|  }
-                                         // CHECK:   1| [[@LINE]]|
-  x = x < 10 ? x + 1 : x - 1;            // CHECK:   1| [[@LINE]]|  x =
-  x = x > 10 ?                           // CHECK:   1| [[@LINE]]|  x =
-        x - 1:                           // CHECK:   0| [[@LINE]]|        x
-        x + 1;                           // CHECK:   1| [[@LINE]]|        x
-                                         // CHECK:   1| [[@LINE]]|
-  return 0;                              // CHECK:   1| [[@LINE]]|  return
-}                                        // CHECK:   1| [[@LINE]]|}
-// after coverage                   // WHOLE-FILE:    | [[@LINE]]|// after
-                                    // FILTER-NOT:    | [[@LINE-1]]|// after
+// before any coverage              // WHOLE-FILE:      | [[@LINE]]|// before
+                                    // FILTER-NOT:      | [[@LINE-1]]|// before
+int main() {                             // CHECK:   161| [[@LINE]]|int main(
+  int x = 0;                             // CHECK:   161| [[@LINE]]|  int x
+                                         // CHECK:   161| [[@LINE]]|
+  if (x) {                               // CHECK:     0| [[@LINE]]|  if (x)
+    x = 0;                               // CHECK:     0| [[@LINE]]|    x = 0
+  } else {                               // CHECK:   161| [[@LINE]]|  } else
+    x = 1;                               // CHECK:   161| [[@LINE]]|    x = 1
+  }                                      // CHECK:   161| [[@LINE]]|  }
+                                         // CHECK:   161| [[@LINE]]|
+  for (int i = 0; i < 100; ++i) {        // CHECK: 16.2k| [[@LINE]]|  for (
+    x = 1;                               // CHECK: 16.1k| [[@LINE]]|    x = 1
+  }                                      // CHECK: 16.1k| [[@LINE]]|  }
+                                         // CHECK:   161| [[@LINE]]|
+  x = x < 10 ? x + 1 : x - 1;            // CHECK:   161| [[@LINE]]|  x =
+  x = x > 10 ?                           // CHECK:   161| [[@LINE]]|  x =
+        x - 1:                           // CHECK:     0| [[@LINE]]|        x
+        x + 1;                           // CHECK:   161| [[@LINE]]|        x
+                                         // CHECK:   161| [[@LINE]]|
+  return 0;                              // CHECK:   161| [[@LINE]]|  return
+}                                        // CHECK:   161| [[@LINE]]|}
+// after coverage                   // WHOLE-FILE:      | [[@LINE]]|// after
+                                    // FILTER-NOT:      | [[@LINE-1]]|// after
 
 // RUN: llvm-cov show %S/Inputs/lineExecutionCounts.covmapping -instr-profile %t.profdata -filename-equivalence %s | FileCheck -check-prefix=CHECK -check-prefix=WHOLE-FILE %s
 // RUN: llvm-cov show %S/Inputs/lineExecutionCounts.covmapping -instr-profile %t.profdata -filename-equivalence -name=main %s | FileCheck -check-prefix=CHECK -check-prefix=FILTER %s
diff --git a/test/tools/llvm-cov/showRegionMarkers.cpp b/test/tools/llvm-cov/showRegionMarkers.cpp
index d1b83d25985..5507367c0ed 100644
--- a/test/tools/llvm-cov/showRegionMarkers.cpp
+++ b/test/tools/llvm-cov/showRegionMarkers.cpp
@@ -1,23 +1,23 @@
 // RUN: llvm-profdata merge %S/Inputs/regionMarkers.proftext -o %t.profdata
 
-int main() {                      // CHECK: Marker at [[@LINE]]:12 = 1
+int main() {                      // CHECK: Marker at [[@LINE]]:12 = 1.11M
   int x = 0;
 
   if (x) {                        // CHECK: Marker at [[@LINE]]:10 = 0
     x = 0;
-  } else {                        // CHECK: Marker at [[@LINE]]:10 = 1
+  } else {                        // CHECK: Marker at [[@LINE]]:10 = 1.11M
     x = 1;
   }
-                                  // CHECK: Marker at [[@LINE+2]]:19 = 101
-                                  // CHECK: Marker at [[@LINE+1]]:28 = 100
-  for (int i = 0; i < 100; ++i) { // CHECK: Marker at [[@LINE]]:33 = 100
+                                  // CHECK: Marker at [[@LINE+2]]:19 = 112M
+                                  // CHECK: Marker at [[@LINE+1]]:28 = 111M
+  for (int i = 0; i < 100; ++i) { // CHECK: Marker at [[@LINE]]:33 = 111M
     x = 1;
   }
-                                  // CHECK: Marker at [[@LINE+1]]:16 = 1
+                                  // CHECK: Marker at [[@LINE+1]]:16 = 1.11M
   x = x < 10 ? x + 1 : x - 1;     // CHECK: Marker at [[@LINE]]:24 = 0
   x = x > 10 ?
         x - 1:                    // CHECK: Marker at [[@LINE]]:9 = 0
-        x + 1;                    // CHECK: Marker at [[@LINE]]:9 = 1
+        x + 1;                    // CHECK: Marker at [[@LINE]]:9 = 1.11M
 
   return 0;
 }
diff --git a/test/tools/llvm-cxxdump/X86/lit.local.cfg b/test/tools/llvm-cxxdump/X86/lit.local.cfg
new file mode 100644
index 00000000000..c8625f4d9d2
--- /dev/null
+++ b/test/tools/llvm-cxxdump/X86/lit.local.cfg
@@ -0,0 +1,2 @@
+if not 'X86' in config.root.targets:
+    config.unsupported = True
diff --git a/test/tools/llvm-cxxdump/X86/sym-size.s b/test/tools/llvm-cxxdump/X86/sym-size.s
new file mode 100644
index 00000000000..c2527523385
--- /dev/null
+++ b/test/tools/llvm-cxxdump/X86/sym-size.s
@@ -0,0 +1,47 @@
+// RUN: llvm-mc %s -o %t -filetype=obj -triple=x86_64-pc-win32
+// RUN: llvm-cxxdump %t | FileCheck %s
+
+// CHECK:      ??_8B@@7B@[0]: 8
+// CHECK-NEXT: ??_8B@@7B@[4]: 9
+// CHECK-NEXT: ??_8C@@7B@[0]: 10
+// CHECK-NEXT: ??_8C@@7B@[4]: 11
+// CHECK-NEXT: ??_8D@@7B0@@[0]: 0
+// CHECK-NEXT: ??_8D@@7B0@@[4]: 1
+// CHECK-NEXT: ??_8D@@7B0@@[8]: 2
+// CHECK-NEXT: ??_8D@@7B0@@[12]: 3
+// CHECK-NEXT: ??_8D@@7BB@@@[0]: 4
+// CHECK-NEXT: ??_8D@@7BB@@@[4]: 5
+// CHECK-NEXT: ??_8D@@7BC@@@[0]: 6
+// CHECK-NEXT: ??_8D@@7BC@@@[4]: 7
+// CHECK-NEXT: ??_8XYZ[0]: 10
+// CHECK-NEXT: ??_8XYZ[4]: 11
+
+	.section	.rdata,"dr"
+	.globl	"??_8D@@7B0@@"
+"??_8D@@7B0@@":
+	.long	0
+	.long	1
+	.long	2
+	.long	3
+
+	.globl	"??_8D@@7BB@@@"
+"??_8D@@7BB@@@":
+	.long	4
+	.long	5
+
+	.globl	"??_8D@@7BC@@@"
+"??_8D@@7BC@@@":
+	.long	6
+	.long	7
+
+	.globl	"??_8B@@7B@"
+"??_8B@@7B@":
+	.long	8
+	.long	9
+
+	.globl	"??_8C@@7B@"
+"??_8C@@7B@":
+	.long	10
+	.long	11
+
+"??_8XYZ" = "??_8C@@7B@"
diff --git a/test/tools/llvm-mc/fatal_warnings.test b/test/tools/llvm-mc/fatal_warnings.test
new file mode 100644
index 00000000000..df73ad8b7cf
--- /dev/null
+++ b/test/tools/llvm-mc/fatal_warnings.test
@@ -0,0 +1,5 @@
+# RUN: not llvm-mc --fatal-warnings %s 2>&1 | FileCheck %s
+# XFAIL: hexagon
+
+# CHECK: error: .warning directive invoked in source file
+.warning
diff --git a/test/tools/llvm-mc/no_warnings.test b/test/tools/llvm-mc/no_warnings.test
new file mode 100644
index 00000000000..f542f1773c0
--- /dev/null
+++ b/test/tools/llvm-mc/no_warnings.test
@@ -0,0 +1,5 @@
+# RUN: llvm-mc --no-warn %s 2>&1 | FileCheck %s
+# XFAIL: hexagon
+
+# CHECK-NOT: warning:
+.warning
diff --git a/test/tools/llvm-objdump/ARM/macho-arm-and-thumb.test b/test/tools/llvm-objdump/ARM/macho-arm-and-thumb.test
index 720b78fa89e..f741f6cb27a 100644
--- a/test/tools/llvm-objdump/ARM/macho-arm-and-thumb.test
+++ b/test/tools/llvm-objdump/ARM/macho-arm-and-thumb.test
@@ -1,4 +1,5 @@
 @ RUN: llvm-mc < %s -triple armv7-apple-darwin -filetype=obj | llvm-objdump -m -d - | FileCheck %s
+@ RUN: llvm-mc < %s -triple armv7-apple-darwin -filetype=obj | llvm-objdump -arch armv7 -m -d - | FileCheck %s
 
 .thumb
 .thumb_func _t
diff --git a/test/tools/llvm-objdump/Inputs/section-filter.obj b/test/tools/llvm-objdump/Inputs/section-filter.obj
new file mode 100644
index 00000000000..7dc5dae26b7
Binary files /dev/null and b/test/tools/llvm-objdump/Inputs/section-filter.obj differ
diff --git a/test/tools/llvm-objdump/X86/Inputs/disassemble-data.obj b/test/tools/llvm-objdump/X86/Inputs/disassemble-data.obj
new file mode 100644
index 00000000000..6381120a1c8
Binary files /dev/null and b/test/tools/llvm-objdump/X86/Inputs/disassemble-data.obj differ
diff --git a/test/tools/llvm-objdump/X86/disassemble-data.test b/test/tools/llvm-objdump/X86/disassemble-data.test
new file mode 100644
index 00000000000..e9c4e7e1e5f
--- /dev/null
+++ b/test/tools/llvm-objdump/X86/disassemble-data.test
@@ -0,0 +1,4 @@
+// This test checks that -D disassembles from a data section
+// RUN: llvm-objdump -D %p/Inputs/disassemble-data.obj | FileCheck %s
+
+// CHECK: Disassembly of section .data:
\ No newline at end of file
diff --git a/test/tools/llvm-objdump/X86/macho-archive-headers.test b/test/tools/llvm-objdump/X86/macho-archive-headers.test
index 3d9043e671a..1253b46529b 100644
--- a/test/tools/llvm-objdump/X86/macho-archive-headers.test
+++ b/test/tools/llvm-objdump/X86/macho-archive-headers.test
@@ -1,5 +1,9 @@
 RUN: llvm-objdump %p/Inputs/macho-universal-archive.x86_64.i386 -macho -archive-headers -arch all \
 RUN: | FileCheck %s
+RUN: llvm-objdump %p/Inputs/macho-universal-archive.x86_64.i386 -macho -archive-headers -arch all -archive-member-offsets \
+RUN: | FileCheck %s -check-prefix=OFFSETS
+RUN: llvm-objdump %p/Inputs/macho-universal-archive.x86_64.i386 -macho -archive-headers -arch all -non-verbose \
+RUN: | FileCheck %s -check-prefix=NON-VERBOSE
 
 # Note the date as printed by ctime(3) is time zone dependent and not checked.
 CHECK: Archive : {{.*}}/macho-universal-archive.x86_64.i386 (architecture x86_64)
@@ -8,3 +12,17 @@ CHECK: -rw-r--r--124/0     860 {{.*}} hello.o
 CHECK: Archive : {{.*}}/macho-universal-archive.x86_64.i386 (architecture i386)
 CHECK: -rw-r--r--124/11     60 {{.*}} __.SYMDEF SORTED
 CHECK: -rw-r--r--124/0     388 {{.*}} foo.o
+
+OFFSETS: Archive : {{.*}}/macho-universal-archive.x86_64.i386 (architecture x86_64)
+OFFSETS: 8	-rw-r--r--124/11     44 {{.*}} __.SYMDEF SORTED
+OFFSETS: 112	-rw-r--r--124/0     860 {{.*}} hello.o
+OFFSETS: Archive : {{.*}}/macho-universal-archive.x86_64.i386 (architecture i386)
+OFFSETS: 8	-rw-r--r--124/11     60 {{.*}} __.SYMDEF SORTED
+OFFSETS: 128	-rw-r--r--124/0     388 {{.*}} foo.o
+
+NON-VERBOSE: Archive : {{.*}}/macho-universal-archive.x86_64.i386 (architecture x86_64)
+NON-VERBOSE: 0100644 124/11     44 1399572709 #1/20
+NON-VERBOSE: 0100644 124/0     860 1399501499 #1/12
+NON-VERBOSE: Archive : {{.*}}/macho-universal-archive.x86_64.i386 (architecture i386)
+NON-VERBOSE: 0100644 124/11     60 1399572709 #1/20
+NON-VERBOSE: 0100644 124/0     388 1399572697 #1/12
diff --git a/test/tools/llvm-objdump/X86/macho-symbol-table.test b/test/tools/llvm-objdump/X86/macho-symbol-table.test
index 5b9a4f84976..826d78af68b 100644
--- a/test/tools/llvm-objdump/X86/macho-symbol-table.test
+++ b/test/tools/llvm-objdump/X86/macho-symbol-table.test
@@ -1,8 +1,8 @@
 RUN: llvm-objdump -macho -t %p/Inputs/hello.obj.macho-x86_64 | FileCheck %s
 
 CHECK: SYMBOL TABLE:
-CHECK: 000000000000003b l     F __TEXT,__cstring	0000000d L_.str
-CHECK: 0000000000000068 l     F __TEXT,__eh_frame	00000018 EH_frame0
-CHECK: 0000000000000000 g     F __TEXT,__text	0000003b _main
-CHECK: 0000000000000080 g     F __TEXT,__eh_frame	00000028 _main.eh
-CHECK: 0000000000000000         *UND*	00000000 _printf
+CHECK: 000000000000003b l     F __TEXT,__cstring	L_.str
+CHECK: 0000000000000068 l     F __TEXT,__eh_frame	EH_frame0
+CHECK: 0000000000000000 g     F __TEXT,__text	        _main
+CHECK: 0000000000000080 g     F __TEXT,__eh_frame	_main.eh
+CHECK: 0000000000000000         *UND*	                _printf
diff --git a/test/tools/llvm-objdump/invalid-input.test b/test/tools/llvm-objdump/invalid-input.test
new file mode 100644
index 00000000000..20a901d0b2d
--- /dev/null
+++ b/test/tools/llvm-objdump/invalid-input.test
@@ -0,0 +1,6 @@
+RUN: not llvm-objdump -t %p/missing-file 2>&1 | FileCheck %s -check-prefix=NO_SUCH_FILE
+# Don't check the OS-dependent message "No such file or directory".
+NO_SUCH_FILE: '{{.*}}missing-file':
+
+RUN: not llvm-objdump -t %s 2>&1 | FileCheck %s -check-prefix=UNKNOWN_FILE_TYPE
+UNKNOWN_FILE_TYPE: '{{.*}}invalid-input.test': The file was not recognized as a valid object file
diff --git a/test/tools/llvm-objdump/macho-sections.test b/test/tools/llvm-objdump/macho-sections.test
index 31efd110a5c..07c2b52ebfc 100644
--- a/test/tools/llvm-objdump/macho-sections.test
+++ b/test/tools/llvm-objdump/macho-sections.test
@@ -1,5 +1,3 @@
 # RUN: llvm-objdump -macho -section=__data %p/Inputs/bind2.macho-x86_64 | FileCheck %s
-# RUN: llvm-objdump -macho -section=__data -raw %p/Inputs/bind2.macho-x86_64 | FileCheck --check-prefix=RAW %s
 
 # CHECK: bind2.macho-x86_64:
-# RAW-NOT: bind2.macho-x86_64:
diff --git a/test/tools/llvm-objdump/section-filter.test b/test/tools/llvm-objdump/section-filter.test
new file mode 100644
index 00000000000..9c7ab31b0d7
--- /dev/null
+++ b/test/tools/llvm-objdump/section-filter.test
@@ -0,0 +1,7 @@
+// This test checks that --section works correctly
+// RUN: llvm-objdump -h %p/Inputs/section-filter.obj -j=.text \
+// RUN: --section=.bss | FileCheck %s
+
+# CHECK: .text
+# CHECK-NOT: .data
+# CHECK: .bss
\ No newline at end of file
diff --git a/test/tools/llvm-pdbdump/Inputs/LoadAddressTest.cpp b/test/tools/llvm-pdbdump/Inputs/LoadAddressTest.cpp
new file mode 100644
index 00000000000..bb6ed4ac553
--- /dev/null
+++ b/test/tools/llvm-pdbdump/Inputs/LoadAddressTest.cpp
@@ -0,0 +1,6 @@
+// Compile with "cl /c /Zi /GR- LoadAddressTest.cpp"
+// Link with "link LoadAddressTest.obj /debug /nodefaultlib /entry:main"
+
+int main(int argc, char **argv) {
+  return 0;
+}
diff --git a/test/tools/llvm-pdbdump/Inputs/LoadAddressTest.pdb b/test/tools/llvm-pdbdump/Inputs/LoadAddressTest.pdb
new file mode 100644
index 00000000000..e302e9bf3b6
Binary files /dev/null and b/test/tools/llvm-pdbdump/Inputs/LoadAddressTest.pdb differ
diff --git a/test/tools/llvm-pdbdump/load-address.test b/test/tools/llvm-pdbdump/load-address.test
new file mode 100644
index 00000000000..7a5a4dbff67
--- /dev/null
+++ b/test/tools/llvm-pdbdump/load-address.test
@@ -0,0 +1,10 @@
+; RUN: llvm-pdbdump -externals %p/Inputs/LoadAddressTest.pdb \
+; RUN:    | FileCheck --check-prefix=RVA %s
+; RUN: llvm-pdbdump -externals -load-address=0x40000000 \
+; RUN: %p/Inputs/LoadAddressTest.pdb | FileCheck --check-prefix=VA %s
+
+; RVA: ---EXTERNALS---
+; RVA: [0x00001010] _main
+
+; VA: ---EXTERNALS---
+; VA: [0x40001010] _main
diff --git a/test/tools/llvm-profdata/raw-magic-but-no-header.test b/test/tools/llvm-profdata/raw-magic-but-no-header.test
index 6db723c3e25..b2a697042b0 100644
--- a/test/tools/llvm-profdata/raw-magic-but-no-header.test
+++ b/test/tools/llvm-profdata/raw-magic-but-no-header.test
@@ -3,4 +3,4 @@ RUN: not llvm-profdata show %t 2>&1 | FileCheck %s
 RUN: printf '\377lprofr\201' > %t
 RUN: not llvm-profdata show %t 2>&1 | FileCheck %s
 
-CHECK: error: {{.+}}: Invalid header
+CHECK: error: {{.+}}: Invalid profile data (file header is corrupt)
diff --git a/test/tools/llvm-readobj/Inputs/abiflags.obj.elf-mips b/test/tools/llvm-readobj/Inputs/abiflags.obj.elf-mips
new file mode 100644
index 00000000000..edcd50becec
Binary files /dev/null and b/test/tools/llvm-readobj/Inputs/abiflags.obj.elf-mips differ
diff --git a/test/tools/llvm-readobj/Inputs/abiflags.obj.elf-mipsel b/test/tools/llvm-readobj/Inputs/abiflags.obj.elf-mipsel
new file mode 100644
index 00000000000..bb5f43d2862
Binary files /dev/null and b/test/tools/llvm-readobj/Inputs/abiflags.obj.elf-mipsel differ
diff --git a/test/tools/llvm-readobj/Inputs/dtflags.elf-x86-64 b/test/tools/llvm-readobj/Inputs/dtflags.elf-x86-64
new file mode 100755
index 00000000000..c6e8501d9e7
Binary files /dev/null and b/test/tools/llvm-readobj/Inputs/dtflags.elf-x86-64 differ
diff --git a/test/tools/llvm-readobj/Inputs/got-plt.exe.elf-mipsel b/test/tools/llvm-readobj/Inputs/got-plt.exe.elf-mipsel
new file mode 100755
index 00000000000..8cdc69ffa56
Binary files /dev/null and b/test/tools/llvm-readobj/Inputs/got-plt.exe.elf-mipsel differ
diff --git a/test/tools/llvm-readobj/Inputs/mips-options.elf-mips64el b/test/tools/llvm-readobj/Inputs/mips-options.elf-mips64el
new file mode 100755
index 00000000000..4b6525731dd
Binary files /dev/null and b/test/tools/llvm-readobj/Inputs/mips-options.elf-mips64el differ
diff --git a/test/tools/llvm-readobj/Inputs/reginfo.obj.elf-mipsel b/test/tools/llvm-readobj/Inputs/reginfo.obj.elf-mipsel
new file mode 100644
index 00000000000..5cd09809f8b
Binary files /dev/null and b/test/tools/llvm-readobj/Inputs/reginfo.obj.elf-mipsel differ
diff --git a/test/tools/llvm-readobj/codeview-linetables.test b/test/tools/llvm-readobj/codeview-linetables.test
index b2acee1200b..d124e6e2d45 100644
--- a/test/tools/llvm-readobj/codeview-linetables.test
+++ b/test/tools/llvm-readobj/codeview-linetables.test
@@ -104,6 +104,7 @@ MFUN32-NEXT:     PayloadSize: 0x8
 MFUN32:        ]
 MFUN32-NEXT:   FunctionLineTable [
 MFUN32-NEXT:     FunctionName: _x
+MFUN32-NEXT:     Flags: 0x0
 MFUN32-NEXT:     CodeSize: 0xA
 MFUN32-NEXT:     FilenameSegment [
 MFUN32-NEXT:       Filename: d:\source.c
@@ -114,6 +115,7 @@ MFUN32-NEXT:     ]
 MFUN32-NEXT:   ]
 MFUN32-NEXT:   FunctionLineTable [
 MFUN32-NEXT:     FunctionName: _y
+MFUN32-NEXT:     Flags: 0x0
 MFUN32-NEXT:     CodeSize: 0xA
 MFUN32-NEXT:     FilenameSegment [
 MFUN32-NEXT:       Filename: d:\source.c
@@ -124,6 +126,7 @@ MFUN32-NEXT:     ]
 MFUN32-NEXT:   ]
 MFUN32-NEXT:   FunctionLineTable [
 MFUN32-NEXT:     FunctionName: _f
+MFUN32-NEXT:     Flags: 0x0
 MFUN32-NEXT:     CodeSize: 0x14
 MFUN32-NEXT:     FilenameSegment [
 MFUN32-NEXT:       Filename: d:\source.c
@@ -201,6 +204,7 @@ MFUN64-NEXT:     PayloadSize: 0x8
 MFUN64:        ]
 MFUN64-NEXT:   FunctionLineTable [
 MFUN64-NEXT:     FunctionName: x
+MFUN64-NEXT:     Flags: 0x0
 MFUN64-NEXT:     CodeSize: 0xE
 MFUN64-NEXT:     FilenameSegment [
 MFUN64-NEXT:       Filename: d:\source.c
@@ -211,6 +215,7 @@ MFUN64-NEXT:     ]
 MFUN64-NEXT:   ]
 MFUN64-NEXT:   FunctionLineTable [
 MFUN64-NEXT:     FunctionName: y
+MFUN64-NEXT:     Flags: 0x0
 MFUN64-NEXT:     CodeSize: 0xE
 MFUN64-NEXT:     FilenameSegment [
 MFUN64-NEXT:       Filename: d:\source.c
@@ -221,6 +226,7 @@ MFUN64-NEXT:     ]
 MFUN64-NEXT:   ]
 MFUN64-NEXT:   FunctionLineTable [
 MFUN64-NEXT:     FunctionName: f
+MFUN64-NEXT:     Flags: 0x0
 MFUN64-NEXT:     CodeSize: 0x18
 MFUN64-NEXT:     FilenameSegment [
 MFUN64-NEXT:       Filename: d:\source.c
@@ -296,6 +302,7 @@ MFILE32-NEXT:     PayloadSize: 0x8
 MFILE32:        ]
 MFILE32-NEXT:   FunctionLineTable [
 MFILE32-NEXT:     FunctionName: _f
+MFILE32-NEXT:     Flags: 0x0
 MFILE32-NEXT:     CodeSize: 0x14
 MFILE32-NEXT:     FilenameSegment [
 MFILE32-NEXT:       Filename: d:\input.c
@@ -352,6 +359,7 @@ MFILE64-NEXT:     PayloadSize: 0x8
 MFILE64:        ]
 MFILE64-NEXT:   FunctionLineTable [
 MFILE64-NEXT:     FunctionName: f
+MFILE64-NEXT:     Flags: 0x0
 MFILE64-NEXT:     CodeSize: 0x18
 MFILE64-NEXT:     FilenameSegment [
 MFILE64-NEXT:       Filename: d:\input.c
@@ -399,6 +407,7 @@ MCOMDAT-NEXT:   CodeSize: 0x7
 MCOMDAT-NEXT: }
 MCOMDAT:      FunctionLineTable [
 MCOMDAT-NEXT:   FunctionName: ?f@@YAHXZ
+MCOMDAT-NEXT:   Flags: 0x0
 MCOMDAT-NEXT:   CodeSize: 0x7
 MCOMDAT-NEXT:   FilenameSegment [
 MCOMDAT-NEXT:     Filename: c:\src\test.cc
@@ -414,6 +423,7 @@ MCOMDAT-NEXT:   CodeSize: 0x7
 MCOMDAT-NEXT: }
 MCOMDAT:      FunctionLineTable [
 MCOMDAT-NEXT:   FunctionName: ?g@@YAHXZ
+MCOMDAT-NEXT:   Flags: 0x0
 MCOMDAT-NEXT:   CodeSize: 0x7
 MCOMDAT-NEXT:   FilenameSegment [
 MCOMDAT-NEXT:     Filename: c:\src\test.cc
diff --git a/test/tools/llvm-readobj/elf-dtflags.test b/test/tools/llvm-readobj/elf-dtflags.test
new file mode 100644
index 00000000000..4e6c90d9f06
--- /dev/null
+++ b/test/tools/llvm-readobj/elf-dtflags.test
@@ -0,0 +1,10 @@
+// Test that llvm-readobj dumps DF_XXX and DF_1_XXX flags correctly.
+// The input was generated using the following:
+// $ clang -Wl,-z,origin -Wl,-z,now example.c
+// $ cat example.c
+// int main(void) { return (0); }
+
+RUN: llvm-readobj -dynamic-table %p/Inputs/dtflags.elf-x86-64 | FileCheck %s
+
+CHECK: 0x000000000000001E FLAGS                ORIGIN BIND_NOW
+CHECK: 0x000000006FFFFFFB FLAGS_1              NOW ORIGIN
diff --git a/test/tools/llvm-readobj/mips-abiflags.test b/test/tools/llvm-readobj/mips-abiflags.test
new file mode 100644
index 00000000000..b4c769be207
--- /dev/null
+++ b/test/tools/llvm-readobj/mips-abiflags.test
@@ -0,0 +1,42 @@
+RUN: llvm-readobj -mips-abi-flags %p/Inputs/abiflags.obj.elf-mipsel | \
+RUN:   FileCheck -check-prefix=EL64 %s
+RUN: llvm-readobj -mips-abi-flags %p/Inputs/abiflags.obj.elf-mips | \
+RUN:   FileCheck -check-prefix=BE32 %s
+
+EL64:      MIPS ABI Flags {
+EL64-NEXT:   Version: 0
+EL64-NEXT:   ISA: MIPS64r5
+EL64-NEXT:   ISA Extension: Cavium Networks Octeon3 (0x13)
+EL64-NEXT:   ASEs [ (0x103)
+EL64-NEXT:     DSP (0x1)
+EL64-NEXT:     DSPR2 (0x2)
+EL64-NEXT:     VZ (0x100)
+EL64-NEXT:   ]
+EL64-NEXT:   FP ABI: Hard float (double precision) (0x1)
+EL64-NEXT:   GPR size: 64
+EL64-NEXT:   CPR1 size: 64
+EL64-NEXT:   CPR2 size: 0
+EL64-NEXT:   Flags 1 [ (0x1)
+EL64-NEXT:     ODDSPREG (0x1)
+EL64-NEXT:   ]
+EL64-NEXT:   Flags 2: 0x0
+EL64-NEXT: }
+
+BE32:      MIPS ABI Flags {
+BE32-NEXT:   Version: 0
+BE32-NEXT:   ISA: MIPS32r2
+BE32-NEXT:   ISA Extension: None (0x0)
+BE32-NEXT:   ASEs [ (0x803)
+BE32-NEXT:     DSP (0x1)
+BE32-NEXT:     DSPR2 (0x2)
+BE32-NEXT:     microMIPS (0x800)
+BE32-NEXT:   ]
+BE32-NEXT:   FP ABI: Soft float (0x3)
+BE32-NEXT:   GPR size: 32
+BE32-NEXT:   CPR1 size: 0
+BE32-NEXT:   CPR2 size: 0
+BE32-NEXT:   Flags 1 [ (0x1)
+BE32-NEXT:     ODDSPREG (0x1)
+BE32-NEXT:   ]
+BE32-NEXT:   Flags 2: 0x0
+BE32-NEXT: }
diff --git a/test/tools/llvm-readobj/mips-options.test b/test/tools/llvm-readobj/mips-options.test
new file mode 100644
index 00000000000..7a2959dd032
--- /dev/null
+++ b/test/tools/llvm-readobj/mips-options.test
@@ -0,0 +1,22 @@
+# Check DT_MIPS_OPTIONS .dynamic section tag reading
+
+RUN: llvm-readobj -dynamic-table %p/Inputs/mips-options.elf-mips64el | \
+RUN:   FileCheck %s
+
+CHECK:      DynamicSection [ (14 entries)
+CHECK-NEXT:   Tag                Type                 Name/Value
+CHECK-NEXT:   0x0000000000000004 HASH                 0x158
+CHECK-NEXT:   0x0000000000000005 STRTAB               0x1C0
+CHECK-NEXT:   0x0000000000000006 SYMTAB               0x178
+CHECK-NEXT:   0x000000000000000A STRSZ                7 (bytes)
+CHECK-NEXT:   0x000000000000000B SYMENT               24 (bytes)
+CHECK-NEXT:   0x0000000070000001 MIPS_RLD_VERSION     1
+CHECK-NEXT:   0x0000000070000005 MIPS_FLAGS           NOTPOT 
+CHECK-NEXT:   0x0000000070000006 MIPS_BASE_ADDRESS    0x0
+CHECK-NEXT:   0x000000007000000A MIPS_LOCAL_GOTNO     2
+CHECK-NEXT:   0x0000000070000011 MIPS_SYMTABNO        3
+CHECK-NEXT:   0x0000000070000013 MIPS_GOTSYM          0x3
+CHECK-NEXT:   0x0000000000000003 PLTGOT               0x2000
+CHECK-NEXT:   0x0000000070000029 MIPS_OPTIONS         0x1000
+CHECK-NEXT:   0x0000000000000000 NULL                 0x0
+CHECK-NEXT: ]
diff --git a/test/tools/llvm-readobj/mips-plt.test b/test/tools/llvm-readobj/mips-plt.test
new file mode 100644
index 00000000000..ab0824b0be6
--- /dev/null
+++ b/test/tools/llvm-readobj/mips-plt.test
@@ -0,0 +1,34 @@
+RUN: llvm-readobj -mips-plt-got %p/Inputs/got-plt.exe.elf-mipsel | FileCheck %s
+
+CHECK:      PLT GOT {
+CHECK-NEXT:   Reserved entries [
+CHECK-NEXT:     Entry {
+CHECK-NEXT:       Address: 0x410814
+CHECK-NEXT:       Initial: 0x0
+CHECK-NEXT:       Purpose: PLT lazy resolver
+CHECK-NEXT:     }
+CHECK-NEXT:     Entry {
+CHECK-NEXT:       Address: 0x410818
+CHECK-NEXT:       Initial: 0x0
+CHECK-NEXT:       Purpose: Module pointer
+CHECK-NEXT:     }
+CHECK-NEXT:   ]
+CHECK-NEXT:   Entries [
+CHECK-NEXT:     Entry {
+CHECK-NEXT:       Address: 0x41081C
+CHECK-NEXT:       Initial: 0x4007C0
+CHECK-NEXT:       Value: 0x0
+CHECK-NEXT:       Type: Function (0x2)
+CHECK-NEXT:       Section: Undefined (0x0)
+CHECK-NEXT:       Name: puts@GLIBC_2.0 (71)
+CHECK-NEXT:     }
+CHECK-NEXT:     Entry {
+CHECK-NEXT:       Address: 0x410820
+CHECK-NEXT:       Initial: 0x4007C0
+CHECK-NEXT:       Value: 0x0
+CHECK-NEXT:       Type: Function (0x2)
+CHECK-NEXT:       Section: Undefined (0x0)
+CHECK-NEXT:       Name: __libc_start_main@GLIBC_2.0 (53)
+CHECK-NEXT:     }
+CHECK-NEXT:   ]
+CHECK-NEXT: }
diff --git a/test/tools/llvm-readobj/mips-reginfo.test b/test/tools/llvm-readobj/mips-reginfo.test
new file mode 100644
index 00000000000..d983d0a6344
--- /dev/null
+++ b/test/tools/llvm-readobj/mips-reginfo.test
@@ -0,0 +1,10 @@
+RUN: llvm-readobj -mips-reginfo %p/Inputs/reginfo.obj.elf-mipsel | FileCheck %s
+
+CHECK:      MIPS RegInfo {
+CHECK-NEXT:   GP: 0x7FEF
+CHECK-NEXT:   General Mask: 0xB00001F6
+CHECK-NEXT:   Co-Proc Mask0: 0x0
+CHECK-NEXT:   Co-Proc Mask1: 0x0
+CHECK-NEXT:   Co-Proc Mask2: 0x0
+CHECK-NEXT:   Co-Proc Mask3: 0x0
+CHECK-NEXT: }
diff --git a/test/tools/llvm-readobj/program-headers.test b/test/tools/llvm-readobj/program-headers.test
index f014c039042..8e8855d8b8e 100644
--- a/test/tools/llvm-readobj/program-headers.test
+++ b/test/tools/llvm-readobj/program-headers.test
@@ -83,10 +83,22 @@ ELF-MIPS-NEXT: AddressSize: 32bit
 ELF-MIPS-NEXT: LoadName:
 ELF-MIPS-NEXT: ProgramHeaders [
 ELF-MIPS-NEXT:   ProgramHeader {
+ELF-MIPS-NEXT:     Type: PT_MIPS_ABIFLAGS (0x70000003)
+ELF-MIPS-NEXT:     Offset: 0x98
+ELF-MIPS-NEXT:     VirtualAddress: 0x400098
+ELF-MIPS-NEXT:     PhysicalAddress: 0x400098
+ELF-MIPS-NEXT:     FileSize: 24
+ELF-MIPS-NEXT:     MemSize: 24
+ELF-MIPS-NEXT:     Flags [ (0x4)
+ELF-MIPS-NEXT:       PF_R (0x4)
+ELF-MIPS-NEXT:     ]
+ELF-MIPS-NEXT:     Alignment: 8
+ELF-MIPS-NEXT:   }
+ELF-MIPS-NEXT:   ProgramHeader {
 ELF-MIPS-NEXT:     Type: PT_MIPS_REGINFO (0x70000000)
-ELF-MIPS-NEXT:     Offset: 0x74
-ELF-MIPS-NEXT:     VirtualAddress: 0x400074
-ELF-MIPS-NEXT:     PhysicalAddress: 0x400074
+ELF-MIPS-NEXT:     Offset: 0xB0
+ELF-MIPS-NEXT:     VirtualAddress: 0x4000B0
+ELF-MIPS-NEXT:     PhysicalAddress: 0x4000B0
 ELF-MIPS-NEXT:     FileSize: 24
 ELF-MIPS-NEXT:     MemSize: 24
 ELF-MIPS-NEXT:     Flags [ (0x4)
@@ -99,8 +111,8 @@ ELF-MIPS-NEXT:     Type: PT_LOAD (0x1)
 ELF-MIPS-NEXT:     Offset: 0x0
 ELF-MIPS-NEXT:     VirtualAddress: 0x400000
 ELF-MIPS-NEXT:     PhysicalAddress: 0x400000
-ELF-MIPS-NEXT:     FileSize: 160
-ELF-MIPS-NEXT:     MemSize: 160
+ELF-MIPS-NEXT:     FileSize: 224
+ELF-MIPS-NEXT:     MemSize: 224
 ELF-MIPS-NEXT:     Flags [ (0x5)
 ELF-MIPS-NEXT:       PF_R (0x4)
 ELF-MIPS-NEXT:       PF_X (0x1)
diff --git a/test/tools/llvm-readobj/relocations.test b/test/tools/llvm-readobj/relocations.test
index 2e11aa27c37..229fef54fb8 100644
--- a/test/tools/llvm-readobj/relocations.test
+++ b/test/tools/llvm-readobj/relocations.test
@@ -10,9 +10,9 @@ RUN: llvm-readobj -r %p/Inputs/trivial.obj.macho-i386 \
 RUN:   | FileCheck %s -check-prefix MACHO-I386
 RUN: llvm-readobj -r %p/Inputs/trivial.obj.macho-x86-64 \
 RUN:   | FileCheck %s -check-prefix MACHO-X86-64
-RUN: llvm-readobj -r %p/Inputs/trivial.obj.macho-ppc \
+RUN: llvm-readobj -r --expand-relocs %p/Inputs/trivial.obj.macho-ppc \
 RUN:   | FileCheck %s -check-prefix MACHO-PPC
-RUN: llvm-readobj -r %p/Inputs/trivial.obj.macho-ppc64 \
+RUN: llvm-readobj -r --expand-relocs %p/Inputs/trivial.obj.macho-ppc64 \
 RUN:   | FileCheck %s -check-prefix MACHO-PPC64
 RUN: llvm-readobj -r -expand-relocs %p/Inputs/trivial.obj.macho-arm \
 RUN:   | FileCheck %s -check-prefix MACHO-ARM
@@ -59,39 +59,159 @@ MACHO-X86-64-NEXT:]
 
 MACHO-PPC: Relocations [
 MACHO-PPC-NEXT:   Section __text {
-MACHO-PPC-NEXT:     0x24 0 2 n/a PPC_RELOC_LO16_SECTDIFF 1 0x64
-MACHO-PPC-NEXT:     0x0 0 2 n/a PPC_RELOC_PAIR 1 0xC
-MACHO-PPC-NEXT:     0x1C 0 2 n/a PPC_RELOC_HA16_SECTDIFF 1 0x64
-MACHO-PPC-NEXT:     0x58 0 2 n/a PPC_RELOC_PAIR 1 0xC
-MACHO-PPC-NEXT:     0x18 1 2 0 PPC_RELOC_BR24 0 0x2
+MACHO-PPC-NEXT:     Relocation {
+MACHO-PPC-NEXT:       Offset: 0x24
+MACHO-PPC-NEXT:       PCRel: 0
+MACHO-PPC-NEXT:       Length: 2
+MACHO-PPC-NEXT:       Type: PPC_RELOC_LO16_SECTDIFF (11)
+MACHO-PPC-NEXT:       Value: 0x64
+MACHO-PPC-NEXT:     }
+MACHO-PPC-NEXT:     Relocation {
+MACHO-PPC-NEXT:       Offset: 0x0
+MACHO-PPC-NEXT:       PCRel: 0
+MACHO-PPC-NEXT:       Length: 2
+MACHO-PPC-NEXT:       Type: PPC_RELOC_PAIR (1)
+MACHO-PPC-NEXT:       Value: 0xC
+MACHO-PPC-NEXT:     }
+MACHO-PPC-NEXT:     Relocation {
+MACHO-PPC-NEXT:       Offset: 0x1C
+MACHO-PPC-NEXT:       PCRel: 0
+MACHO-PPC-NEXT:       Length: 2
+MACHO-PPC-NEXT:       Type: PPC_RELOC_HA16_SECTDIFF (12)
+MACHO-PPC-NEXT:       Value: 0x64
+MACHO-PPC-NEXT:     }
+MACHO-PPC-NEXT:     Relocation {
+MACHO-PPC-NEXT:       Offset: 0x58
+MACHO-PPC-NEXT:       PCRel: 0
+MACHO-PPC-NEXT:       Length: 2
+MACHO-PPC-NEXT:       Type: PPC_RELOC_PAIR (1)
+MACHO-PPC-NEXT:       Value: 0xC
+MACHO-PPC-NEXT:     }
+MACHO-PPC-NEXT:     Relocation {
+MACHO-PPC-NEXT:       Offset: 0x18
+MACHO-PPC-NEXT:       PCRel: 1
+MACHO-PPC-NEXT:       Length: 2
+MACHO-PPC-NEXT:       Type: PPC_RELOC_BR24 (3)
+MACHO-PPC-NEXT:       Section: __picsymbolstub1 (2)
+MACHO-PPC-NEXT:     }
 MACHO-PPC-NEXT:   }
 MACHO-PPC-NEXT:   Section __picsymbolstub1 {
-MACHO-PPC-NEXT:     0x14 0 2 n/a PPC_RELOC_LO16_SECTDIFF 1 0x68
-MACHO-PPC-NEXT:     0x0 0 2 n/a PPC_RELOC_PAIR 1 0x48
-MACHO-PPC-NEXT:     0xC 0 2 n/a PPC_RELOC_HA16_SECTDIFF 1 0x68
-MACHO-PPC-NEXT:     0x20 0 2 n/a PPC_RELOC_PAIR 1 0x48
+MACHO-PPC-NEXT:     Relocation {
+MACHO-PPC-NEXT:       Offset: 0x14
+MACHO-PPC-NEXT:       PCRel: 0
+MACHO-PPC-NEXT:       Length: 2
+MACHO-PPC-NEXT:       Type: PPC_RELOC_LO16_SECTDIFF (11)
+MACHO-PPC-NEXT:       Value: 0x68
+MACHO-PPC-NEXT:     }
+MACHO-PPC-NEXT:     Relocation {
+MACHO-PPC-NEXT:       Offset: 0x0
+MACHO-PPC-NEXT:       PCRel: 0
+MACHO-PPC-NEXT:       Length: 2
+MACHO-PPC-NEXT:       Type: PPC_RELOC_PAIR (1)
+MACHO-PPC-NEXT:       Value: 0x48
+MACHO-PPC-NEXT:     }
+MACHO-PPC-NEXT:     Relocation {
+MACHO-PPC-NEXT:       Offset: 0xC
+MACHO-PPC-NEXT:       PCRel: 0
+MACHO-PPC-NEXT:       Length: 2
+MACHO-PPC-NEXT:       Type: PPC_RELOC_HA16_SECTDIFF (12)
+MACHO-PPC-NEXT:       Value: 0x68
+MACHO-PPC-NEXT:     }
+MACHO-PPC-NEXT:     Relocation {
+MACHO-PPC-NEXT:       Offset: 0x20
+MACHO-PPC-NEXT:       PCRel: 0
+MACHO-PPC-NEXT:       Length: 2
+MACHO-PPC-NEXT:       Type: PPC_RELOC_PAIR (1)
+MACHO-PPC-NEXT:       Value: 0x48
+MACHO-PPC-NEXT:     }
 MACHO-PPC-NEXT:   }
 MACHO-PPC-NEXT:   Section __la_symbol_ptr {
-MACHO-PPC-NEXT:     0x0 0 2 1 PPC_RELOC_VANILLA 0 dyld_stub_binding_helper
+MACHO-PPC-NEXT:     Relocation {
+MACHO-PPC-NEXT:       Offset: 0x0
+MACHO-PPC-NEXT:       PCRel: 0
+MACHO-PPC-NEXT:       Length: 2
+MACHO-PPC-NEXT:       Type: PPC_RELOC_VANILLA (0)
+MACHO-PPC-NEXT:       Symbol: dyld_stub_binding_helper (5)
+MACHO-PPC-NEXT:     }
 MACHO-PPC-NEXT:   }
 MACHO-PPC-NEXT: ]
 
 MACHO-PPC64: Relocations [
 MACHO-PPC64-NEXT:   Section __text {
-MACHO-PPC64-NEXT:     0x24 0 2 n/a 1 0x64
-MACHO-PPC64-NEXT:     0x0 0 2 n/a 1 0xC
-MACHO-PPC64-NEXT:     0x1C 0 2 n/a 1 0x64
-MACHO-PPC64-NEXT:     0x58 0 2 n/a 1 0xC
-MACHO-PPC64-NEXT:     0x18 1 2 0 0 0x2
+MACHO-PPC64-NEXT:     Relocation {
+MACHO-PPC64-NEXT:       Offset: 0x24
+MACHO-PPC64-NEXT:       PCRel: 0
+MACHO-PPC64-NEXT:       Length: 2
+MACHO-PPC64-NEXT:       Type:  (14)
+MACHO-PPC64-NEXT:       Value: 0x64
+MACHO-PPC64-NEXT:     }
+MACHO-PPC64-NEXT:     Relocation {
+MACHO-PPC64-NEXT:       Offset: 0x0
+MACHO-PPC64-NEXT:       PCRel: 0
+MACHO-PPC64-NEXT:       Length: 2
+MACHO-PPC64-NEXT:       Type:  (1)
+MACHO-PPC64-NEXT:       Value: 0xC
+MACHO-PPC64-NEXT:     }
+MACHO-PPC64-NEXT:     Relocation {
+MACHO-PPC64-NEXT:       Offset: 0x1C
+MACHO-PPC64-NEXT:       PCRel: 0
+MACHO-PPC64-NEXT:       Length: 2
+MACHO-PPC64-NEXT:       Type:  (12)
+MACHO-PPC64-NEXT:       Value: 0x64
+MACHO-PPC64-NEXT:     }
+MACHO-PPC64-NEXT:     Relocation {
+MACHO-PPC64-NEXT:       Offset: 0x58
+MACHO-PPC64-NEXT:       PCRel: 0
+MACHO-PPC64-NEXT:       Length: 2
+MACHO-PPC64-NEXT:       Type:  (1)
+MACHO-PPC64-NEXT:       Value: 0xC
+MACHO-PPC64-NEXT:     }
+MACHO-PPC64-NEXT:     Relocation {
+MACHO-PPC64-NEXT:       Offset: 0x18
+MACHO-PPC64-NEXT:       PCRel: 1
+MACHO-PPC64-NEXT:       Length: 2
+MACHO-PPC64-NEXT:       Type:  (3)
+MACHO-PPC64-NEXT:       Section: __picsymbolstub1 (2)
+MACHO-PPC64-NEXT:     }
 MACHO-PPC64-NEXT:   }
 MACHO-PPC64-NEXT:   Section __picsymbolstub1 {
-MACHO-PPC64-NEXT:     0x14 0 2 n/a 1 0x6C
-MACHO-PPC64-NEXT:     0x0 0 2 n/a 1 0x48
-MACHO-PPC64-NEXT:     0xC 0 2 n/a 1 0x6C
-MACHO-PPC64-NEXT:     0x24 0 2 n/a 1 0x48
+MACHO-PPC64-NEXT:     Relocation {
+MACHO-PPC64-NEXT:       Offset: 0x14
+MACHO-PPC64-NEXT:       PCRel: 0
+MACHO-PPC64-NEXT:       Length: 2
+MACHO-PPC64-NEXT:       Type:  (14)
+MACHO-PPC64-NEXT:       Value: 0x6C
+MACHO-PPC64-NEXT:     }
+MACHO-PPC64-NEXT:     Relocation {
+MACHO-PPC64-NEXT:       Offset: 0x0
+MACHO-PPC64-NEXT:       PCRel: 0
+MACHO-PPC64-NEXT:       Length: 2
+MACHO-PPC64-NEXT:       Type:  (1)
+MACHO-PPC64-NEXT:       Value: 0x48
+MACHO-PPC64-NEXT:     }
+MACHO-PPC64-NEXT:     Relocation {
+MACHO-PPC64-NEXT:       Offset: 0xC
+MACHO-PPC64-NEXT:       PCRel: 0
+MACHO-PPC64-NEXT:       Length: 2
+MACHO-PPC64-NEXT:       Type:  (12)
+MACHO-PPC64-NEXT:       Value: 0x6C
+MACHO-PPC64-NEXT:     }
+MACHO-PPC64-NEXT:     Relocation {
+MACHO-PPC64-NEXT:       Offset: 0x24
+MACHO-PPC64-NEXT:       PCRel: 0
+MACHO-PPC64-NEXT:       Length: 2
+MACHO-PPC64-NEXT:       Type:  (1)
+MACHO-PPC64-NEXT:       Value: 0x48
+MACHO-PPC64-NEXT:     }
 MACHO-PPC64-NEXT:   }
 MACHO-PPC64-NEXT:   Section __la_symbol_ptr {
-MACHO-PPC64-NEXT:     0x0 0 3 1 0 dyld_stub_binding_helper
+MACHO-PPC64-NEXT:     Relocation {
+MACHO-PPC64-NEXT:       Offset: 0x0
+MACHO-PPC64-NEXT:       PCRel: 0
+MACHO-PPC64-NEXT:       Length: 3
+MACHO-PPC64-NEXT:       Type:  (0)
+MACHO-PPC64-NEXT:       Symbol: dyld_stub_binding_helper (5)
+MACHO-PPC64-NEXT:     }
 MACHO-PPC64-NEXT:   }
 MACHO-PPC64-NEXT: ]
 
@@ -102,82 +222,64 @@ MACHO-ARM-NEXT:      Relocation {
 MACHO-ARM-NEXT:        Offset: 0x38
 MACHO-ARM-NEXT:        PCRel: 0
 MACHO-ARM-NEXT:        Length: 2
-MACHO-ARM-NEXT:        Extern: N/A
 MACHO-ARM-NEXT:        Type: ARM_RELOC_SECTDIFF (2)
-MACHO-ARM-NEXT:        Symbol: 0x40
-MACHO-ARM-NEXT:        Scattered: 1
+MACHO-ARM-NEXT:        Value: 0x40
 MACHO-ARM-NEXT:      }
 MACHO-ARM-NEXT:      Relocation {
 MACHO-ARM-NEXT:        Offset: 0x0
 MACHO-ARM-NEXT:        PCRel: 0
 MACHO-ARM-NEXT:        Length: 2
-MACHO-ARM-NEXT:        Extern: N/A
 MACHO-ARM-NEXT:        Type: ARM_RELOC_PAIR (1)
-MACHO-ARM-NEXT:        Symbol: 0x28
-MACHO-ARM-NEXT:        Scattered: 1
+MACHO-ARM-NEXT:        Value: 0x28
 MACHO-ARM-NEXT:      }
 MACHO-ARM-NEXT:      Relocation {
 MACHO-ARM-NEXT:        Offset: 0x20
 MACHO-ARM-NEXT:        PCRel: 1
 MACHO-ARM-NEXT:        Length: 2
-MACHO-ARM-NEXT:        Extern: 1
 MACHO-ARM-NEXT:        Type: ARM_RELOC_BR24 (5)
 MACHO-ARM-NEXT:        Symbol: _g
-MACHO-ARM-NEXT:        Scattered: 0
 MACHO-ARM-NEXT:      }
 MACHO-ARM-NEXT:      Relocation {
 MACHO-ARM-NEXT:        Offset: 0x1C
 MACHO-ARM-NEXT:        PCRel: 0
 MACHO-ARM-NEXT:        Length: 1
-MACHO-ARM-NEXT:        Extern: 1
 MACHO-ARM-NEXT:        Type: ARM_RELOC_HALF (8)
 MACHO-ARM-NEXT:        Symbol: _g
-MACHO-ARM-NEXT:        Scattered: 0
 MACHO-ARM-NEXT:      }
 MACHO-ARM-NEXT:      Relocation {
 MACHO-ARM-NEXT:        Offset: 0x0
 MACHO-ARM-NEXT:        PCRel: 0
 MACHO-ARM-NEXT:        Length: 1
-MACHO-ARM-NEXT:        Extern: 0
 MACHO-ARM-NEXT:        Type: ARM_RELOC_PAIR (1)
-MACHO-ARM-NEXT:        Symbol: 0xFFFFFF
-MACHO-ARM-NEXT:        Scattered: 0
+MACHO-ARM-NEXT:        Section: -
 MACHO-ARM-NEXT:      }
 MACHO-ARM-NEXT:      Relocation {
 MACHO-ARM-NEXT:        Offset: 0x18
 MACHO-ARM-NEXT:        PCRel: 0
 MACHO-ARM-NEXT:        Length: 0
-MACHO-ARM-NEXT:        Extern: 1
 MACHO-ARM-NEXT:        Type: ARM_RELOC_HALF (8)
 MACHO-ARM-NEXT:        Symbol: _g
-MACHO-ARM-NEXT:        Scattered: 0
 MACHO-ARM-NEXT:      }
 MACHO-ARM-NEXT:      Relocation {
 MACHO-ARM-NEXT:        Offset: 0x0
 MACHO-ARM-NEXT:        PCRel: 0
 MACHO-ARM-NEXT:        Length: 0
-MACHO-ARM-NEXT:        Extern: 0
 MACHO-ARM-NEXT:        Type: ARM_RELOC_PAIR (1)
-MACHO-ARM-NEXT:        Symbol: 0xFFFFFF
-MACHO-ARM-NEXT:        Scattered: 0
+MACHO-ARM-NEXT:        Section: -
 MACHO-ARM-NEXT:      }
 MACHO-ARM-NEXT:      Relocation {
 MACHO-ARM-NEXT:        Offset: 0xC
 MACHO-ARM-NEXT:        PCRel: 0
 MACHO-ARM-NEXT:        Length: 2
-MACHO-ARM-NEXT:        Extern: N/A
 MACHO-ARM-NEXT:        Type: ARM_RELOC_SECTDIFF (2)
-MACHO-ARM-NEXT:        Symbol: 0x44
-MACHO-ARM-NEXT:        Scattered: 1
+MACHO-ARM-NEXT:        Value: 0x44
 MACHO-ARM-NEXT:      }
 MACHO-ARM-NEXT:      Relocation {
 MACHO-ARM-NEXT:        Offset: 0x0
 MACHO-ARM-NEXT:        PCRel: 0
 MACHO-ARM-NEXT:        Length: 2
-MACHO-ARM-NEXT:        Extern: N/A
 MACHO-ARM-NEXT:        Type: ARM_RELOC_PAIR (1)
-MACHO-ARM-NEXT:        Symbol: 0x4
-MACHO-ARM-NEXT:        Scattered: 1
+MACHO-ARM-NEXT:        Value: 0x4
 MACHO-ARM-NEXT:      }
 MACHO-ARM-NEXT:    }
 MACHO-ARM-NEXT:  ]
diff --git a/test/tools/llvm-readobj/sections-ext.test b/test/tools/llvm-readobj/sections-ext.test
index 4024878d2bd..19b7aa0516d 100644
--- a/test/tools/llvm-readobj/sections-ext.test
+++ b/test/tools/llvm-readobj/sections-ext.test
@@ -6,9 +6,9 @@ RUN: llvm-readobj -s -st -sr -sd %p/Inputs/trivial.obj.macho-i386 \
 RUN:   | FileCheck %s -check-prefix MACHO-I386
 RUN: llvm-readobj -s -st -sr -sd %p/Inputs/trivial.obj.macho-x86-64 \
 RUN:   | FileCheck %s -check-prefix MACHO-X86-64
-RUN: llvm-readobj -s -st -sr -sd %p/Inputs/trivial.obj.macho-ppc \
+RUN: llvm-readobj --expand-relocs -s -st -sr -sd %p/Inputs/trivial.obj.macho-ppc \
 RUN:   | FileCheck %s -check-prefix MACHO-PPC
-RUN: llvm-readobj -s -st -sr -sd %p/Inputs/trivial.obj.macho-ppc64 \
+RUN: llvm-readobj --expand-relocs -s -st -sr -sd %p/Inputs/trivial.obj.macho-ppc64 \
 RUN:   | FileCheck %s -check-prefix MACHO-PPC64
 RUN: llvm-readobj -expand-relocs -s -st -sr -sd %p/Inputs/trivial.obj.macho-arm \
 RUN:   | FileCheck %s -check-prefix MACHO-ARM
@@ -110,7 +110,7 @@ ELF-NEXT:     Relocations [
 ELF-NEXT:     ]
 ELF-NEXT:     Symbols [
 ELF-NEXT:       Symbol {
-ELF-NEXT:         Name: .text (0)
+ELF-NEXT:         Name: (0)
 ELF-NEXT:         Value: 0x0
 ELF-NEXT:         Size: 0
 ELF-NEXT:         Binding: Local (0x0)
@@ -298,11 +298,41 @@ MACHO-PPC-NEXT:     ]
 MACHO-PPC-NEXT:     Reserved1: 0x0
 MACHO-PPC-NEXT:     Reserved2: 0x0
 MACHO-PPC-NEXT:     Relocations [
-MACHO-PPC-NEXT:       0x24 0 2 n/a PPC_RELOC_LO16_SECTDIFF 1 0x64
-MACHO-PPC-NEXT:       0x0 0 2 n/a PPC_RELOC_PAIR 1 0xC
-MACHO-PPC-NEXT:       0x1C 0 2 n/a PPC_RELOC_HA16_SECTDIFF 1 0x64
-MACHO-PPC-NEXT:       0x58 0 2 n/a PPC_RELOC_PAIR 1 0xC
-MACHO-PPC-NEXT:       0x18 1 2 0 PPC_RELOC_BR24 0 0x2
+MACHO-PPC-NEXT:       Relocation {
+MACHO-PPC-NEXT:         Offset: 0x24
+MACHO-PPC-NEXT:         PCRel: 0
+MACHO-PPC-NEXT:         Length: 2
+MACHO-PPC-NEXT:         Type: PPC_RELOC_LO16_SECTDIFF (11)
+MACHO-PPC-NEXT:         Value: 0x64
+MACHO-PPC-NEXT:       }
+MACHO-PPC-NEXT:       Relocation {
+MACHO-PPC-NEXT:         Offset: 0x0
+MACHO-PPC-NEXT:         PCRel: 0
+MACHO-PPC-NEXT:         Length: 2
+MACHO-PPC-NEXT:         Type: PPC_RELOC_PAIR (1)
+MACHO-PPC-NEXT:         Value: 0xC
+MACHO-PPC-NEXT:       }
+MACHO-PPC-NEXT:       Relocation {
+MACHO-PPC-NEXT:         Offset: 0x1C
+MACHO-PPC-NEXT:         PCRel: 0
+MACHO-PPC-NEXT:         Length: 2
+MACHO-PPC-NEXT:         Type: PPC_RELOC_HA16_SECTDIFF (12)
+MACHO-PPC-NEXT:         Value: 0x64
+MACHO-PPC-NEXT:       }
+MACHO-PPC-NEXT:       Relocation {
+MACHO-PPC-NEXT:         Offset: 0x58
+MACHO-PPC-NEXT:         PCRel: 0
+MACHO-PPC-NEXT:         Length: 2
+MACHO-PPC-NEXT:         Type: PPC_RELOC_PAIR (1)
+MACHO-PPC-NEXT:         Value: 0xC
+MACHO-PPC-NEXT:       }
+MACHO-PPC-NEXT:       Relocation {
+MACHO-PPC-NEXT:         Offset: 0x18
+MACHO-PPC-NEXT:         PCRel: 1
+MACHO-PPC-NEXT:         Length: 2
+MACHO-PPC-NEXT:         Type: PPC_RELOC_BR24 (3)
+MACHO-PPC-NEXT:         Section: __picsymbolstub1 (2)
+MACHO-PPC-NEXT:       }
 MACHO-PPC-NEXT:     ]
 MACHO-PPC-NEXT:     Symbols [
 MACHO-PPC-NEXT:       Symbol {
@@ -341,10 +371,34 @@ MACHO-PPC-NEXT:     ]
 MACHO-PPC-NEXT:     Reserved1: 0x0
 MACHO-PPC-NEXT:     Reserved2: 0x20
 MACHO-PPC-NEXT:     Relocations [
-MACHO-PPC-NEXT:       0x14 0 2 n/a PPC_RELOC_LO16_SECTDIFF 1 0x68
-MACHO-PPC-NEXT:       0x0 0 2 n/a PPC_RELOC_PAIR 1 0x48
-MACHO-PPC-NEXT:       0xC 0 2 n/a PPC_RELOC_HA16_SECTDIFF 1 0x68
-MACHO-PPC-NEXT:       0x20 0 2 n/a PPC_RELOC_PAIR 1 0x48
+MACHO-PPC-NEXT:       Relocation {
+MACHO-PPC-NEXT:         Offset: 0x14
+MACHO-PPC-NEXT:         PCRel: 0
+MACHO-PPC-NEXT:         Length: 2
+MACHO-PPC-NEXT:         Type: PPC_RELOC_LO16_SECTDIFF (11)
+MACHO-PPC-NEXT:         Value: 0x68
+MACHO-PPC-NEXT:       }
+MACHO-PPC-NEXT:       Relocation {
+MACHO-PPC-NEXT:         Offset: 0x0
+MACHO-PPC-NEXT:         PCRel: 0
+MACHO-PPC-NEXT:         Length: 2
+MACHO-PPC-NEXT:         Type: PPC_RELOC_PAIR (1)
+MACHO-PPC-NEXT:         Value: 0x48
+MACHO-PPC-NEXT:       }
+MACHO-PPC-NEXT:       Relocation {
+MACHO-PPC-NEXT:         Offset: 0xC
+MACHO-PPC-NEXT:         PCRel: 0
+MACHO-PPC-NEXT:         Length: 2
+MACHO-PPC-NEXT:         Type: PPC_RELOC_HA16_SECTDIFF (12)
+MACHO-PPC-NEXT:         Value: 0x68
+MACHO-PPC-NEXT:       }
+MACHO-PPC-NEXT:       Relocation {
+MACHO-PPC-NEXT:         Offset: 0x20
+MACHO-PPC-NEXT:         PCRel: 0
+MACHO-PPC-NEXT:         Length: 2
+MACHO-PPC-NEXT:         Type: PPC_RELOC_PAIR (1)
+MACHO-PPC-NEXT:         Value: 0x48
+MACHO-PPC-NEXT:       }
 MACHO-PPC-NEXT:     ]
 MACHO-PPC-NEXT:     Symbols [
 MACHO-PPC-NEXT:     ]
@@ -425,7 +479,13 @@ MACHO-PPC-NEXT:     ]
 MACHO-PPC-NEXT:     Reserved1: 0x2
 MACHO-PPC-NEXT:     Reserved2: 0x0
 MACHO-PPC-NEXT:     Relocations [
-MACHO-PPC-NEXT:       0x0 0 2 1 PPC_RELOC_VANILLA 0 dyld_stub_binding_helper
+MACHO-PPC-NEXT:       Relocation {
+MACHO-PPC-NEXT:         Offset: 0x0
+MACHO-PPC-NEXT:         PCRel: 0
+MACHO-PPC-NEXT:         Length: 2
+MACHO-PPC-NEXT:         Type: PPC_RELOC_VANILLA (0)
+MACHO-PPC-NEXT:         Symbol: dyld_stub_binding_helper (5)
+MACHO-PPC-NEXT:       }
 MACHO-PPC-NEXT:     ]
 MACHO-PPC-NEXT:     Symbols [
 MACHO-PPC-NEXT:     ]
@@ -455,11 +515,41 @@ MACHO-PPC64-NEXT:     ]
 MACHO-PPC64-NEXT:     Reserved1: 0x0
 MACHO-PPC64-NEXT:     Reserved2: 0x0
 MACHO-PPC64-NEXT:     Relocations [
-MACHO-PPC64-NEXT:       0x24 0 2 n/a 1 0x64
-MACHO-PPC64-NEXT:       0x0 0 2 n/a 1 0xC
-MACHO-PPC64-NEXT:       0x1C 0 2 n/a 1 0x64
-MACHO-PPC64-NEXT:       0x58 0 2 n/a 1 0xC
-MACHO-PPC64-NEXT:       0x18 1 2 0 0 0x2
+MACHO-PPC64-NEXT:       Relocation {
+MACHO-PPC64-NEXT:         Offset: 0x24
+MACHO-PPC64-NEXT:         PCRel: 0
+MACHO-PPC64-NEXT:         Length: 2
+MACHO-PPC64-NEXT:         Type:  (14)
+MACHO-PPC64-NEXT:         Value: 0x64
+MACHO-PPC64-NEXT:       }
+MACHO-PPC64-NEXT:       Relocation {
+MACHO-PPC64-NEXT:         Offset: 0x0
+MACHO-PPC64-NEXT:         PCRel: 0
+MACHO-PPC64-NEXT:         Length: 2
+MACHO-PPC64-NEXT:         Type:  (1)
+MACHO-PPC64-NEXT:         Value: 0xC
+MACHO-PPC64-NEXT:       }
+MACHO-PPC64-NEXT:       Relocation {
+MACHO-PPC64-NEXT:         Offset: 0x1C
+MACHO-PPC64-NEXT:         PCRel: 0
+MACHO-PPC64-NEXT:         Length: 2
+MACHO-PPC64-NEXT:         Type:  (12)
+MACHO-PPC64-NEXT:         Value: 0x64
+MACHO-PPC64-NEXT:       }
+MACHO-PPC64-NEXT:       Relocation {
+MACHO-PPC64-NEXT:         Offset: 0x58
+MACHO-PPC64-NEXT:         PCRel: 0
+MACHO-PPC64-NEXT:         Length: 2
+MACHO-PPC64-NEXT:         Type:  (1)
+MACHO-PPC64-NEXT:         Value: 0xC
+MACHO-PPC64-NEXT:       }
+MACHO-PPC64-NEXT:       Relocation {
+MACHO-PPC64-NEXT:         Offset: 0x18
+MACHO-PPC64-NEXT:         PCRel: 1
+MACHO-PPC64-NEXT:         Length: 2
+MACHO-PPC64-NEXT:         Type:  (3)
+MACHO-PPC64-NEXT:         Section: __picsymbolstub1 (2)
+MACHO-PPC64-NEXT:       }
 MACHO-PPC64-NEXT:     ]
 MACHO-PPC64-NEXT:     Symbols [
 MACHO-PPC64-NEXT:       Symbol {
@@ -498,10 +588,34 @@ MACHO-PPC64-NEXT:     ]
 MACHO-PPC64-NEXT:     Reserved1: 0x0
 MACHO-PPC64-NEXT:     Reserved2: 0x20
 MACHO-PPC64-NEXT:     Relocations [
-MACHO-PPC64-NEXT:       0x14 0 2 n/a 1 0x6C
-MACHO-PPC64-NEXT:       0x0 0 2 n/a 1 0x48
-MACHO-PPC64-NEXT:       0xC 0 2 n/a 1 0x6C
-MACHO-PPC64-NEXT:       0x24 0 2 n/a 1 0x48
+MACHO-PPC64-NEXT:       Relocation {
+MACHO-PPC64-NEXT:         Offset: 0x14
+MACHO-PPC64-NEXT:         PCRel: 0
+MACHO-PPC64-NEXT:         Length: 2
+MACHO-PPC64-NEXT:         Type:  (14)
+MACHO-PPC64-NEXT:         Value: 0x6C
+MACHO-PPC64-NEXT:       }
+MACHO-PPC64-NEXT:       Relocation {
+MACHO-PPC64-NEXT:         Offset: 0x0
+MACHO-PPC64-NEXT:         PCRel: 0
+MACHO-PPC64-NEXT:         Length: 2
+MACHO-PPC64-NEXT:         Type:  (1)
+MACHO-PPC64-NEXT:         Value: 0x48
+MACHO-PPC64-NEXT:       }
+MACHO-PPC64-NEXT:       Relocation {
+MACHO-PPC64-NEXT:         Offset: 0xC
+MACHO-PPC64-NEXT:         PCRel: 0
+MACHO-PPC64-NEXT:         Length: 2
+MACHO-PPC64-NEXT:         Type:  (12)
+MACHO-PPC64-NEXT:         Value: 0x6C
+MACHO-PPC64-NEXT:       }
+MACHO-PPC64-NEXT:       Relocation {
+MACHO-PPC64-NEXT:         Offset: 0x24
+MACHO-PPC64-NEXT:         PCRel: 0
+MACHO-PPC64-NEXT:         Length: 2
+MACHO-PPC64-NEXT:         Type:  (1)
+MACHO-PPC64-NEXT:         Value: 0x48
+MACHO-PPC64-NEXT:       }
 MACHO-PPC64-NEXT:     ]
 MACHO-PPC64-NEXT:     Symbols [
 MACHO-PPC64-NEXT:     ]
@@ -582,7 +696,13 @@ MACHO-PPC64-NEXT:     ]
 MACHO-PPC64-NEXT:     Reserved1: 0x2
 MACHO-PPC64-NEXT:     Reserved2: 0x0
 MACHO-PPC64-NEXT:     Relocations [
-MACHO-PPC64-NEXT:       0x0 0 3 1 0 dyld_stub_binding_helper
+MACHO-PPC64-NEXT:       Relocation {
+MACHO-PPC64-NEXT:         Offset: 0x0
+MACHO-PPC64-NEXT:         PCRel: 0
+MACHO-PPC64-NEXT:         Length: 3
+MACHO-PPC64-NEXT:         Type:  (0)
+MACHO-PPC64-NEXT:         Symbol: dyld_stub_binding_helper (5)
+MACHO-PPC64-NEXT:       }
 MACHO-PPC64-NEXT:     ]
 MACHO-PPC64-NEXT:     Symbols [
 MACHO-PPC64-NEXT:     ]
@@ -615,82 +735,64 @@ MACHO-ARM-NEXT:       Relocation {
 MACHO-ARM-NEXT:         Offset: 0x38
 MACHO-ARM-NEXT:         PCRel: 0
 MACHO-ARM-NEXT:         Length: 2
-MACHO-ARM-NEXT:         Extern: N/A
 MACHO-ARM-NEXT:         Type: ARM_RELOC_SECTDIFF (2)
-MACHO-ARM-NEXT:         Symbol: 0x40
-MACHO-ARM-NEXT:         Scattered: 1
+MACHO-ARM-NEXT:         Value: 0x40
 MACHO-ARM-NEXT:       }
 MACHO-ARM-NEXT:       Relocation {
 MACHO-ARM-NEXT:         Offset: 0x0
 MACHO-ARM-NEXT:         PCRel: 0
 MACHO-ARM-NEXT:         Length: 2
-MACHO-ARM-NEXT:         Extern: N/A
 MACHO-ARM-NEXT:         Type: ARM_RELOC_PAIR (1)
-MACHO-ARM-NEXT:         Symbol: 0x28
-MACHO-ARM-NEXT:         Scattered: 1
+MACHO-ARM-NEXT:         Value: 0x28
 MACHO-ARM-NEXT:       }
 MACHO-ARM-NEXT:       Relocation {
 MACHO-ARM-NEXT:         Offset: 0x20
 MACHO-ARM-NEXT:         PCRel: 1
 MACHO-ARM-NEXT:         Length: 2
-MACHO-ARM-NEXT:         Extern: 1
 MACHO-ARM-NEXT:         Type: ARM_RELOC_BR24 (5)
 MACHO-ARM-NEXT:         Symbol: _g
-MACHO-ARM-NEXT:         Scattered: 0
 MACHO-ARM-NEXT:       }
 MACHO-ARM-NEXT:       Relocation {
 MACHO-ARM-NEXT:         Offset: 0x1C
 MACHO-ARM-NEXT:         PCRel: 0
 MACHO-ARM-NEXT:         Length: 1
-MACHO-ARM-NEXT:         Extern: 1
 MACHO-ARM-NEXT:         Type: ARM_RELOC_HALF (8)
 MACHO-ARM-NEXT:         Symbol: _g
-MACHO-ARM-NEXT:         Scattered: 0
 MACHO-ARM-NEXT:       }
 MACHO-ARM-NEXT:       Relocation {
 MACHO-ARM-NEXT:         Offset: 0x0
 MACHO-ARM-NEXT:         PCRel: 0
 MACHO-ARM-NEXT:         Length: 1
-MACHO-ARM-NEXT:         Extern: 0
 MACHO-ARM-NEXT:         Type: ARM_RELOC_PAIR (1)
-MACHO-ARM-NEXT:         Symbol: 0xFFFFFF
-MACHO-ARM-NEXT:         Scattered: 0
+MACHO-ARM-NEXT:         Section: -
 MACHO-ARM-NEXT:       }
 MACHO-ARM-NEXT:       Relocation {
 MACHO-ARM-NEXT:         Offset: 0x18
 MACHO-ARM-NEXT:         PCRel: 0
 MACHO-ARM-NEXT:         Length: 0
-MACHO-ARM-NEXT:         Extern: 1
 MACHO-ARM-NEXT:         Type: ARM_RELOC_HALF (8)
 MACHO-ARM-NEXT:         Symbol: _g
-MACHO-ARM-NEXT:         Scattered: 0
 MACHO-ARM-NEXT:       }
 MACHO-ARM-NEXT:       Relocation {
 MACHO-ARM-NEXT:         Offset: 0x0
 MACHO-ARM-NEXT:         PCRel: 0
 MACHO-ARM-NEXT:         Length: 0
-MACHO-ARM-NEXT:         Extern: 0
 MACHO-ARM-NEXT:         Type: ARM_RELOC_PAIR (1)
-MACHO-ARM-NEXT:         Symbol: 0xFFFFFF
-MACHO-ARM-NEXT:         Scattered: 0
+MACHO-ARM-NEXT:         Section: -
 MACHO-ARM-NEXT:       }
 MACHO-ARM-NEXT:       Relocation {
 MACHO-ARM-NEXT:         Offset: 0xC
 MACHO-ARM-NEXT:         PCRel: 0
 MACHO-ARM-NEXT:         Length: 2
-MACHO-ARM-NEXT:         Extern: N/A
 MACHO-ARM-NEXT:         Type: ARM_RELOC_SECTDIFF (2)
-MACHO-ARM-NEXT:         Symbol: 0x44
-MACHO-ARM-NEXT:         Scattered: 1
+MACHO-ARM-NEXT:         Value: 0x44
 MACHO-ARM-NEXT:       }
 MACHO-ARM-NEXT:       Relocation {
 MACHO-ARM-NEXT:         Offset: 0x0
 MACHO-ARM-NEXT:         PCRel: 0
 MACHO-ARM-NEXT:         Length: 2
-MACHO-ARM-NEXT:         Extern: N/A
 MACHO-ARM-NEXT:         Type: ARM_RELOC_PAIR (1)
-MACHO-ARM-NEXT:         Symbol: 0x4
-MACHO-ARM-NEXT:         Scattered: 1
+MACHO-ARM-NEXT:         Value: 0x4
 MACHO-ARM-NEXT:       }
 MACHO-ARM-NEXT:     ]
 MACHO-ARM-NEXT:     Symbols [
diff --git a/test/tools/llvm-symbolizer/Inputs/fat.c b/test/tools/llvm-symbolizer/Inputs/fat.c
new file mode 100644
index 00000000000..0331c09fe5f
--- /dev/null
+++ b/test/tools/llvm-symbolizer/Inputs/fat.c
@@ -0,0 +1,15 @@
+/* Compile with:
+   clang -arch armv7 -arch armv7m -arch armv7em -arch x86_64 -arch x86_64h -c
+*/
+
+#ifdef __x86_64h__
+void x86_64h_function() {}
+#elif defined(__x86_64__)
+void x86_64_function() {}
+#elif defined(__ARM_ARCH_7EM__)
+void armv7em_function() {}
+#elif defined(__ARM_ARCH_7M__)
+void armv7m_function() {}
+#elif defined(__ARM_ARCH_7A__)
+void armv7_function() {}
+#endif
diff --git a/test/tools/llvm-symbolizer/Inputs/fat.o b/test/tools/llvm-symbolizer/Inputs/fat.o
new file mode 100644
index 00000000000..947cfc2d8cd
Binary files /dev/null and b/test/tools/llvm-symbolizer/Inputs/fat.o differ
diff --git a/test/tools/llvm-symbolizer/fat.test b/test/tools/llvm-symbolizer/fat.test
new file mode 100644
index 00000000000..1ecd1abb356
--- /dev/null
+++ b/test/tools/llvm-symbolizer/fat.test
@@ -0,0 +1,11 @@
+RUN: echo 0 | llvm-symbolizer -obj=%p/Inputs/fat.o -default-arch=x86_64 | FileCheck --check-prefix=X86_64 %s
+RUN: echo 0 | llvm-symbolizer -obj=%p/Inputs/fat.o -default-arch=x86_64h | FileCheck --check-prefix=X86_64H %s
+RUN: echo 0 | llvm-symbolizer -obj=%p/Inputs/fat.o -default-arch=armv7 | FileCheck --check-prefix=ARMV7 %s
+RUN: echo 0 | llvm-symbolizer -obj=%p/Inputs/fat.o -default-arch=armv7em | FileCheck --check-prefix=ARMV7EM %s
+RUN: echo 0 | llvm-symbolizer -obj=%p/Inputs/fat.o -default-arch=armv7m | FileCheck --check-prefix=ARMV7M %s
+
+X86_64: x86_64_function
+X86_64H: x86_64h_function
+ARMV7: armv7_function
+ARMV7EM: armv7em_function
+ARMV7M: armv7m_function
diff --git a/test/tools/llvm-symbolizer/pdb/Inputs/test.cpp b/test/tools/llvm-symbolizer/pdb/Inputs/test.cpp
new file mode 100644
index 00000000000..f1f98af4190
--- /dev/null
+++ b/test/tools/llvm-symbolizer/pdb/Inputs/test.cpp
@@ -0,0 +1,18 @@
+// To generate the corresponding EXE/PDB, run:
+// cl /Zi test.cpp
+
+namespace NS {
+struct Foo {
+  void bar() {}
+};
+}
+
+void foo() {
+}
+
+int main() {
+  foo();
+  
+  NS::Foo f;
+  f.bar();
+}
diff --git a/test/tools/llvm-symbolizer/pdb/Inputs/test.exe b/test/tools/llvm-symbolizer/pdb/Inputs/test.exe
new file mode 100644
index 00000000000..80fb34bb6dc
Binary files /dev/null and b/test/tools/llvm-symbolizer/pdb/Inputs/test.exe differ
diff --git a/test/tools/llvm-symbolizer/pdb/Inputs/test.exe.input b/test/tools/llvm-symbolizer/pdb/Inputs/test.exe.input
new file mode 100644
index 00000000000..affda60449b
--- /dev/null
+++ b/test/tools/llvm-symbolizer/pdb/Inputs/test.exe.input
@@ -0,0 +1,4 @@
+0x401030
+0x401040
+0x401060
+0x500000
diff --git a/test/tools/llvm-symbolizer/pdb/Inputs/test.pdb b/test/tools/llvm-symbolizer/pdb/Inputs/test.pdb
new file mode 100644
index 00000000000..974e565e87f
Binary files /dev/null and b/test/tools/llvm-symbolizer/pdb/Inputs/test.pdb differ
diff --git a/test/tools/llvm-symbolizer/pdb/lit.local.cfg b/test/tools/llvm-symbolizer/pdb/lit.local.cfg
new file mode 100644
index 00000000000..8c58f166640
--- /dev/null
+++ b/test/tools/llvm-symbolizer/pdb/lit.local.cfg
@@ -0,0 +1 @@
+config.unsupported = not config.have_dia_sdk
diff --git a/test/tools/llvm-symbolizer/pdb/pdb.test b/test/tools/llvm-symbolizer/pdb/pdb.test
new file mode 100644
index 00000000000..b5d0f15fbcb
--- /dev/null
+++ b/test/tools/llvm-symbolizer/pdb/pdb.test
@@ -0,0 +1,18 @@
+RUN: llvm-symbolizer -obj="%p/Inputs/test.exe" < "%p/Inputs/test.exe.input" | \
+RUN:    FileCheck %s --check-prefix=CHECK
+RUN: llvm-symbolizer -obj="%p/Inputs/test.exe" -demangle=false < \
+RUN:    "%p/Inputs/test.exe.input" | FileCheck %s --check-prefix=CHECK-NO-DEMANGLE
+
+CHECK: foo(void)
+CHECK-NEXT: test.cpp:10
+CHECK: _main
+CHECK-NEXT: test.cpp:13:0
+CHECK: NS::Foo::bar(void)
+CHECK-NEXT: test.cpp:6:0
+
+CHECK-NO-DEMANGLE: foo
+CHECK-NO-DEMANGLE-NEXT: test.cpp:10
+CHECK-NO-DEMANGLE: _main
+CHECK-LINKAGE-NAME-NEXT: test.cpp:13:0
+CHECK-NO-DEMANGLE: bar
+CHECK-LINKAGE-NAME-NEXT: test.cpp:6:0
diff --git a/tools/CMakeLists.txt b/tools/CMakeLists.txt
index 7f3fbcdf2fc..203169350c5 100644
--- a/tools/CMakeLists.txt
+++ b/tools/CMakeLists.txt
@@ -1,113 +1,44 @@
-add_llvm_tool_subdirectory(llvm-config)
+# This file will recurse into all subdirectories that contain CMakeLists.txt
+# Setting variables that match the pattern LLVM_TOOL_{NAME}_BUILD to Off will
+# prevent traversing into a directory.
+#
+# The only tools that need to be explicitly added are ones that have explicit
+# ordering requirements.
+
+# Iterates all the subdirectories to create CMake options to enable/disable
+# traversing each directory.
+create_llvm_tool_options()
 
 # Build polly before the tools: the tools link against polly when
 # LINK_POLLY_INTO_TOOLS is set.
 if(WITH_POLLY)
   add_llvm_external_project(polly)
-else(WITH_POLLY)
-  list(APPEND LLVM_IMPLICIT_PROJECT_IGNORE "${LLVM_MAIN_SRC_DIR}/tools/polly")
-endif(WITH_POLLY)
-
-if( LLVM_BUILD_LLVM_DYLIB )
-  add_llvm_tool_subdirectory(llvm-shlib)
 else()
-  ignore_llvm_tool_subdirectory(llvm-shlib)
+  set(LLVM_TOOL_POLLY_BUILD Off)
 endif()
 
-add_llvm_tool_subdirectory(opt)
-add_llvm_tool_subdirectory(llvm-as)
-add_llvm_tool_subdirectory(llvm-dis)
-add_llvm_tool_subdirectory(llvm-mc)
-
-add_llvm_tool_subdirectory(llc)
-
-if (ENABLE_PNACL) # XXX Emscripten: Disable PNaCl build (unless -DENABLE_PNACL=1 is specified), PNaCl is not needed for Emscripten.
-  add_llvm_tool_subdirectory(pnacl-llc)
-else()
-  ignore_llvm_tool_subdirectory(pnacl-llc)
+if(NOT LLVM_BUILD_LLVM_DYLIB )
+  set(LLVM_TOOL_LLVM_SHLIB_BUILD Off)
 endif()
 
-add_llvm_tool_subdirectory(llvm-ar)
-add_llvm_tool_subdirectory(llvm-nm)
-add_llvm_tool_subdirectory(llvm-size)
-
-add_llvm_tool_subdirectory(llvm-cov)
-add_llvm_tool_subdirectory(llvm-profdata)
-add_llvm_tool_subdirectory(llvm-link)
-add_llvm_tool_subdirectory(lli)
-
-add_llvm_tool_subdirectory(llvm-extract)
-add_llvm_tool_subdirectory(llvm-diff)
-add_llvm_tool_subdirectory(macho-dump)
-add_llvm_tool_subdirectory(llvm-objdump)
-add_llvm_tool_subdirectory(llvm-readobj)
-add_llvm_tool_subdirectory(llvm-rtdyld)
-add_llvm_tool_subdirectory(llvm-dwarfdump)
-add_llvm_tool_subdirectory(dsymutil)
-add_llvm_tool_subdirectory(llvm-cxxdump)
-if( LLVM_USE_INTEL_JITEVENTS )
-  add_llvm_tool_subdirectory(llvm-jitlistener)
-else()
-  ignore_llvm_tool_subdirectory(llvm-jitlistener)
-endif( LLVM_USE_INTEL_JITEVENTS )
-
-add_llvm_tool_subdirectory(bugpoint)
-add_llvm_tool_subdirectory(bugpoint-passes)
-add_llvm_tool_subdirectory(llvm-bcanalyzer)
-add_llvm_tool_subdirectory(llvm-stress)
-add_llvm_tool_subdirectory(llvm-mcmarkup)
-
-add_llvm_tool_subdirectory(verify-uselistorder)
-
-add_llvm_tool_subdirectory(llvm-symbolizer)
-if (ENABLE_PNACL) # XXX Emscripten: Disable PNaCl build (unless -DENABLE_PNACL=1 is specified), PNaCl is not needed for Emscripten.
-  add_llvm_tool_subdirectory(pnacl-abicheck)
-  add_llvm_tool_subdirectory(pnacl-bcanalyzer)
-  add_llvm_tool_subdirectory(pnacl-bccompress)
-  add_llvm_tool_subdirectory(pnacl-bcdis)
-  add_llvm_tool_subdirectory(pnacl-bcfuzz)
-  add_llvm_tool_subdirectory(pnacl-freeze)
-  add_llvm_tool_subdirectory(pnacl-thaw)
-else()
-  ignore_llvm_tool_subdirectory(pnacl-abicheck)
-  ignore_llvm_tool_subdirectory(pnacl-bcanalyzer)
-  ignore_llvm_tool_subdirectory(pnacl-bccompress)
-  ignore_llvm_tool_subdirectory(pnacl-bcdis)
-  ignore_llvm_tool_subdirectory(pnacl-bcfuzz)
-  ignore_llvm_tool_subdirectory(pnacl-freeze)
-  ignore_llvm_tool_subdirectory(pnacl-thaw)
+if(NOT LLVM_USE_INTEL_JITEVENTS )
+  set(LLVM_TOOL_LLVM_JITLISTENER_BUILD Off)
 endif()
 
-add_llvm_tool_subdirectory(llvm-c-test)
-
-add_llvm_tool_subdirectory(obj2yaml)
-add_llvm_tool_subdirectory(yaml2obj)
-
-add_llvm_tool_subdirectory(llvm-go)
-
-add_llvm_tool_subdirectory(llvm-pdbdump)
-
-if(NOT CYGWIN AND LLVM_ENABLE_PIC)
-  add_llvm_tool_subdirectory(lto)
-  add_llvm_tool_subdirectory(llvm-lto)
-else()
-  ignore_llvm_tool_subdirectory(lto)
-  ignore_llvm_tool_subdirectory(llvm-lto)
+if(CYGWIN OR NOT LLVM_ENABLE_PIC)
+  set(LLVM_TOOL_LTO_BUILD Off)
+  set(LLVM_TOOL_LLVM_LTO_BUILD Off)
 endif()
 
-add_llvm_tool_subdirectory(gold)
-
+# Projects supported via LLVM_EXTERNAL_*_SOURCE_DIR need to be explicitly
+# specified.
 add_llvm_external_project(clang)
 add_llvm_external_project(llgo)
-add_llvm_external_project(subzero) # LOCALMOD
+add_llvm_external_project(lld)
+add_llvm_external_project(lldb)
 
-if( NOT LLVM_INCLUDE_TOOLS STREQUAL "bootstrap-only" )
-  add_llvm_external_project(lld)
-  add_llvm_external_project(lldb)
-
-  # Automatically add remaining sub-directories containing a 'CMakeLists.txt'
-  # file as external projects.
-  add_llvm_implicit_external_projects()
-endif()
+# Automatically add remaining sub-directories containing a 'CMakeLists.txt'
+# file as external projects.
+add_llvm_implicit_projects()
 
 set(LLVM_COMMON_DEPENDS ${LLVM_COMMON_DEPENDS} PARENT_SCOPE)
diff --git a/tools/LLVMBuild.txt b/tools/LLVMBuild.txt
index 3354c74566b..acf61b0268c 100644
--- a/tools/LLVMBuild.txt
+++ b/tools/LLVMBuild.txt
@@ -16,7 +16,33 @@
 ;===------------------------------------------------------------------------===;
 
 [common]
-subdirectories = pnacl-llc pnacl-abicheck pnacl-bcanalyzer pnacl-bccompress pnacl-bcdis pnacl-bcfuzz pnacl-freeze pnacl-thaw bugpoint llc lli llvm-ar llvm-as llvm-bcanalyzer llvm-cov llvm-diff llvm-dis llvm-dwarfdump llvm-extract llvm-jitlistener llvm-link llvm-lto llvm-mc llvm-nm llvm-objdump llvm-pdbdump llvm-profdata llvm-rtdyld llvm-size macho-dump opt llvm-mcmarkup verify-uselistorder dsymutil
+subdirectories =
+ bugpoint
+ dsymutil
+ llc
+ lli
+ llvm-ar
+ llvm-as
+ llvm-bcanalyzer
+ llvm-cov
+ llvm-diff
+ llvm-dis
+ llvm-dwarfdump
+ llvm-extract
+ llvm-jitlistener
+ llvm-link
+ llvm-lto
+ llvm-mc
+ llvm-mcmarkup
+ llvm-nm
+ llvm-objdump
+ llvm-pdbdump
+ llvm-profdata
+ llvm-rtdyld
+ llvm-size
+ macho-dump
+ opt
+ verify-uselistorder
 
 [component_0]
 type = Group
diff --git a/tools/Makefile b/tools/Makefile
index 68584e8a060..a47710f111a 100644
--- a/tools/Makefile
+++ b/tools/Makefile
@@ -19,20 +19,15 @@ else
   OPTIONAL_PARALLEL_DIRS := clang
 endif
 
-OPTIONAL_PARALLEL_DIRS += $(SUBZERO_SRC_ROOT) # @LOCALMOD
-
-# Build LLD and LLDB if present. Note LLDB must be built last as it depends on
+# Build LLDB if present. Note LLDB must be built last as it depends on
 # the wider LLVM infrastructure (including Clang).
-OPTIONAL_PARALLEL_DIRS += lld
 OPTIONAL_DIRS := lldb
 
 # NOTE: The tools are organized into five groups of four consisting of one
 # large and three small executables. This is done to minimize memory load
 # in parallel builds.  Please retain this ordering.
 DIRS := llvm-config
-PARALLEL_DIRS := pnacl-llc pnacl-abicheck pnacl-bcanalyzer pnacl-bcfuzz \
-                 pnacl-freeze pnacl-thaw pnacl-bccompress pnacl-bcdis \
-                 opt llvm-as llvm-dis llc llvm-ar llvm-nm llvm-link \
+PARALLEL_DIRS := opt llvm-as llvm-dis llc llvm-ar llvm-nm llvm-link \
                  lli llvm-extract llvm-mc bugpoint llvm-bcanalyzer llvm-diff \
                  macho-dump llvm-objdump llvm-readobj llvm-rtdyld \
                  llvm-dwarfdump llvm-cov llvm-size llvm-stress llvm-mcmarkup \
@@ -47,13 +42,8 @@ endif
 # Let users override the set of tools to build from the command line.
 ifdef ONLY_TOOLS
   OPTIONAL_PARALLEL_DIRS :=
-  # @LOCALMOD-BEGIN -- Allow subzero to be built as a tool if requested.
-  ifneq ($(findstring subzero,$(ONLY_TOOLS)),)
-    OPTIONAL_PARALLEL_DIRS += $(SUBZERO_SRC_ROOT)
-  endif
-  # @LOCALMOD-END
   OPTIONAL_DIRS := $(findstring lldb,$(ONLY_TOOLS))
-  PARALLEL_DIRS := $(filter-out lldb subzero,$(ONLY_TOOLS)) # @LOCALMOD
+  PARALLEL_DIRS := $(filter-out lldb,$(ONLY_TOOLS))
 endif
 
 # These libraries build as dynamic libraries (.dylib /.so), they can only be
diff --git a/tools/bugpoint/CMakeLists.txt b/tools/bugpoint/CMakeLists.txt
index e8eb4fe56d3..6e5138a0449 100644
--- a/tools/bugpoint/CMakeLists.txt
+++ b/tools/bugpoint/CMakeLists.txt
@@ -10,9 +10,6 @@ set(LLVM_LINK_COMPONENTS
   Instrumentation
   Linker
   # @LOCALMOD-BEGIN
-  MinSFITransforms
-  NaClAnalysis
-  NaClBitReader
   NaClTransforms
   JSBackend
   # @LOCALMOD-END
diff --git a/tools/bugpoint/CrashDebugger.cpp b/tools/bugpoint/CrashDebugger.cpp
index 6f41d3030d0..e2aaf6b01fc 100644
--- a/tools/bugpoint/CrashDebugger.cpp
+++ b/tools/bugpoint/CrashDebugger.cpp
@@ -385,8 +385,7 @@ bool ReduceCrashingBlocks::TestBlocks(std::vector<const BasicBlock*> &BBs) {
   std::vector<std::pair<std::string, std::string> > BlockInfo;
 
   for (BasicBlock *BB : Blocks)
-    BlockInfo.push_back(std::make_pair(BB->getParent()->getName(),
-                                       BB->getName()));
+    BlockInfo.emplace_back(BB->getParent()->getName(), BB->getName());
 
   // Now run the CFG simplify pass on the function...
   std::vector<std::string> Passes;
diff --git a/tools/bugpoint/LLVMBuild.txt b/tools/bugpoint/LLVMBuild.txt
index 6c121b02b62..fab5c1c1e96 100644
--- a/tools/bugpoint/LLVMBuild.txt
+++ b/tools/bugpoint/LLVMBuild.txt
@@ -19,4 +19,16 @@
 type = Tool
 name = bugpoint
 parent = Tools
-required_libraries = AsmParser BitReader BitWriter IRReader IPO Instrumentation Linker Scalar ObjCARC CodeGen NaClTransforms NaClAnalysis MinSFITransforms NaClBitReader JSBackend
+required_libraries =
+ AsmParser
+ BitReader
+ BitWriter
+ CodeGen
+ IRReader
+ IPO
+ Instrumentation
+ JSBackend
+ Linker
+ NaClTransforms
+ ObjCARC
+ Scalar
diff --git a/tools/bugpoint/ListReducer.h b/tools/bugpoint/ListReducer.h
index a0bb570a5cb..f08bc97a998 100644
--- a/tools/bugpoint/ListReducer.h
+++ b/tools/bugpoint/ListReducer.h
@@ -75,6 +75,11 @@ struct ListReducer {
     // Maximal number of allowed splitting iterations,
     // before the elements are randomly shuffled.
     const unsigned MaxIterationsWithoutProgress = 3;
+
+    // Maximal number of allowed single-element trim iterations. We add a
+    // threshhold here as single-element reductions may otherwise take a
+    // very long time to complete.
+    const unsigned MaxTrimIterationsWithoutBackJump = 3;
     bool ShufflingEnabled = true;
 
 Backjump:
@@ -157,6 +162,7 @@ struct ListReducer {
     if (TheList.size() > 2) {
       bool Changed = true;
       std::vector<ElTy> EmptyList;
+      unsigned TrimIterations = 0;
       while (Changed) {  // Trimming loop.
         Changed = false;
         
@@ -186,9 +192,9 @@ struct ListReducer {
           if (!Error.empty())
             return true;
         }
-        // This can take a long time if left uncontrolled.  For now, don't
-        // iterate.
-        break;
+        if (TrimIterations >= MaxTrimIterationsWithoutBackJump)
+          break;
+        TrimIterations++;
       }
     }
 
diff --git a/tools/bugpoint/Makefile b/tools/bugpoint/Makefile
index c993b6ff0f1..974a201af68 100644
--- a/tools/bugpoint/Makefile
+++ b/tools/bugpoint/Makefile
@@ -8,11 +8,10 @@
 ##===----------------------------------------------------------------------===##
 
 LEVEL := ../..
-
 TOOLNAME := bugpoint
 LINK_COMPONENTS := asmparser instrumentation scalaropts ipo linker bitreader \
                    bitwriter irreader vectorize objcarcopts codegen          \
-                   nacltransforms naclanalysis minsfitransforms naclbitreader
+                   nacltransforms
 
 # Support plugins.
 NO_DEAD_STRIP := 1
diff --git a/tools/bugpoint/Miscompilation.cpp b/tools/bugpoint/Miscompilation.cpp
index 53631d25c39..fad16368698 100644
--- a/tools/bugpoint/Miscompilation.cpp
+++ b/tools/bugpoint/Miscompilation.cpp
@@ -386,10 +386,8 @@ static bool ExtractLoops(BugDriver &BD,
       // that masked the error.  Stop loop extraction now.
 
       std::vector<std::pair<std::string, FunctionType*> > MisCompFunctions;
-      for (unsigned i = 0, e = MiscompiledFunctions.size(); i != e; ++i) {
-        Function *F = MiscompiledFunctions[i];
-        MisCompFunctions.push_back(std::make_pair(F->getName(),
-                                                  F->getFunctionType()));
+      for (Function *F : MiscompiledFunctions) {
+        MisCompFunctions.emplace_back(F->getName(), F->getFunctionType());
       }
 
       if (Linker::LinkModules(ToNotOptimize, ToOptimizeLoopExtracted))
@@ -414,8 +412,7 @@ static bool ExtractLoops(BugDriver &BD,
     for (Module::iterator I = ToOptimizeLoopExtracted->begin(),
            E = ToOptimizeLoopExtracted->end(); I != E; ++I)
       if (!I->isDeclaration())
-        MisCompFunctions.push_back(std::make_pair(I->getName(),
-                                                  I->getFunctionType()));
+        MisCompFunctions.emplace_back(I->getName(), I->getFunctionType());
 
     // Okay, great!  Now we know that we extracted a loop and that loop
     // extraction both didn't break the program, and didn't mask the problem.
@@ -596,8 +593,7 @@ static bool ExtractBlocks(BugDriver &BD,
   for (Module::iterator I = Extracted->begin(), E = Extracted->end();
        I != E; ++I)
     if (!I->isDeclaration())
-      MisCompFunctions.push_back(std::make_pair(I->getName(),
-                                                I->getFunctionType()));
+      MisCompFunctions.emplace_back(I->getName(), I->getFunctionType());
 
   if (Linker::LinkModules(ProgClone, Extracted.get()))
     exit(1);
diff --git a/tools/bugpoint/bugpoint.cpp b/tools/bugpoint/bugpoint.cpp
index fd175b3da3a..0a6a4bdf957 100644
--- a/tools/bugpoint/bugpoint.cpp
+++ b/tools/bugpoint/bugpoint.cpp
@@ -50,7 +50,7 @@ TimeoutValue("timeout", cl::init(300), cl::value_desc("seconds"),
 static cl::opt<int>
 MemoryLimit("mlimit", cl::init(-1), cl::value_desc("MBytes"),
             cl::desc("Maximum amount of memory to use. 0 disables check."
-                     " Defaults to 300MB (800MB under valgrind)."));
+                     " Defaults to 400MB (800MB under valgrind)."));
 
 static cl::opt<bool>
 UseValgrind("enable-valgrind",
@@ -138,12 +138,10 @@ int main(int argc, char **argv) {
 
   // @LOCALMOD-BEGIN
   initializeAddPNaClExternalDeclsPass(Registry);
-  initializeAllocateDataSegmentPass(Registry);
   initializeBackendCanonicalizePass(Registry);
   initializeCanonicalizeMemIntrinsicsPass(Registry);
   initializeCleanupUsedGlobalsMetadataPass(Registry);
   initializeConstantInsertExtractElementIndexPass(Registry);
-  initializeExpandAllocasPass(Registry);
   initializeExpandArithWithOverflowPass(Registry);
   initializeExpandByValPass(Registry);
   initializeExpandConstantExprPass(Registry);
@@ -164,28 +162,21 @@ int main(int argc, char **argv) {
   initializeInsertDivideCheckPass(Registry);
   initializeInternalizeUsedGlobalsPass(Registry);
   initializeNormalizeAlignmentPass(Registry);
-  initializePNaClABIVerifyFunctionsPass(Registry);
-  initializePNaClABIVerifyModulePass(Registry);
   initializePNaClSjLjEHPass(Registry);
   initializePromoteI1OpsPass(Registry);
   initializePromoteIntegersPass(Registry);
   initializeRemoveAsmMemoryPass(Registry);
-  initializeRenameEntryPointPass(Registry);
   initializeReplacePtrsWithIntsPass(Registry);
   initializeResolveAliasesPass(Registry);
   initializeResolvePNaClIntrinsicsPass(Registry);
   initializeRewriteAtomicsPass(Registry);
   initializeRewriteLLVMIntrinsicsPass(Registry);
   initializeRewritePNaClLibraryCallsPass(Registry);
-  initializeSandboxIndirectCallsPass(Registry);
-  initializeSandboxMemoryAccessesPass(Registry);
   initializeSimplifyAllocasPass(Registry);
   initializeSimplifyStructRegSignaturesPass(Registry);
   initializeStripAttributesPass(Registry);
   initializeStripMetadataPass(Registry);
   initializeStripModuleFlagsPass(Registry);
-  initializeStripTlsPass(Registry);
-  initializeSubstituteUndefsPass(Registry);
   // Emscripten passes:
   initializeExpandI64Pass(Registry);
   initializeExpandInsertExtractElementPass(Registry);
@@ -218,7 +209,7 @@ int main(int argc, char **argv) {
     if (sys::RunningOnValgrind() || UseValgrind)
       MemoryLimit = 800;
     else
-      MemoryLimit = 300;
+      MemoryLimit = 400;
   }
 
   BugDriver D(argv[0], FindBugs, TimeoutValue, MemoryLimit,
diff --git a/tools/dsymutil/BinaryHolder.cpp b/tools/dsymutil/BinaryHolder.cpp
index ad66105bc24..7ff4fd69957 100644
--- a/tools/dsymutil/BinaryHolder.cpp
+++ b/tools/dsymutil/BinaryHolder.cpp
@@ -13,25 +13,59 @@
 //===----------------------------------------------------------------------===//
 
 #include "BinaryHolder.h"
+#include "llvm/Object/MachO.h"
 #include "llvm/Support/raw_ostream.h"
 
 namespace llvm {
 namespace dsymutil {
 
-ErrorOr<MemoryBufferRef>
-BinaryHolder::GetMemoryBufferForFile(StringRef Filename) {
+Triple BinaryHolder::getTriple(const object::MachOObjectFile &Obj) {
+  // If a ThumbTriple is returned, use it instead of the standard
+  // one. This is because the thumb triple always allows to create a
+  // target, whereas the non-thumb one might not.
+  Triple ThumbTriple;
+  Triple T = Obj.getArch(nullptr, &ThumbTriple);
+  return ThumbTriple.getArch() ? ThumbTriple : T;
+}
+
+static std::vector<MemoryBufferRef>
+getMachOFatMemoryBuffers(StringRef Filename, MemoryBuffer &Mem,
+                         object::MachOUniversalBinary &Fat) {
+  std::vector<MemoryBufferRef> Buffers;
+  StringRef FatData = Fat.getData();
+  for (auto It = Fat.begin_objects(), End = Fat.end_objects(); It != End;
+       ++It) {
+    StringRef ObjData = FatData.substr(It->getOffset(), It->getSize());
+    Buffers.emplace_back(ObjData, Filename);
+  }
+  return Buffers;
+}
+
+void BinaryHolder::changeBackingMemoryBuffer(
+    std::unique_ptr<MemoryBuffer> &&Buf) {
+  CurrentArchives.clear();
+  CurrentObjectFiles.clear();
+  CurrentFatBinary.reset();
+
+  CurrentMemoryBuffer = std::move(Buf);
+}
+
+ErrorOr<std::vector<MemoryBufferRef>>
+BinaryHolder::GetMemoryBuffersForFile(StringRef Filename,
+                                      sys::TimeValue Timestamp) {
   if (Verbose)
     outs() << "trying to open '" << Filename << "'\n";
 
   // Try that first as it doesn't involve any filesystem access.
-  if (auto ErrOrArchiveMember = GetArchiveMemberBuffer(Filename))
-    return *ErrOrArchiveMember;
+  if (auto ErrOrArchiveMembers = GetArchiveMemberBuffers(Filename, Timestamp))
+    return *ErrOrArchiveMembers;
 
   // If the name ends with a closing paren, there is a huge chance
   // it is an archive member specification.
   if (Filename.endswith(")"))
-    if (auto ErrOrArchiveMember = MapArchiveAndGetMemberBuffer(Filename))
-      return *ErrOrArchiveMember;
+    if (auto ErrOrArchiveMembers =
+            MapArchiveAndGetMemberBuffers(Filename, Timestamp))
+      return *ErrOrArchiveMembers;
 
   // Otherwise, just try opening a standard file. If this is an
   // archive member specifiaction and any of the above didn't handle it
@@ -42,39 +76,67 @@ BinaryHolder::GetMemoryBufferForFile(StringRef Filename) {
   if (auto Err = ErrOrFile.getError())
     return Err;
 
+  changeBackingMemoryBuffer(std::move(*ErrOrFile));
   if (Verbose)
     outs() << "\tloaded file.\n";
-  CurrentArchive.reset();
-  CurrentMemoryBuffer = std::move(ErrOrFile.get());
-  return CurrentMemoryBuffer->getMemBufferRef();
+
+  auto ErrOrFat = object::MachOUniversalBinary::create(
+      CurrentMemoryBuffer->getMemBufferRef());
+  if (ErrOrFat.getError()) {
+    // Not a fat binary must be a standard one. Return a one element vector.
+    return std::vector<MemoryBufferRef>{CurrentMemoryBuffer->getMemBufferRef()};
+  }
+
+  CurrentFatBinary = std::move(*ErrOrFat);
+  return getMachOFatMemoryBuffers(Filename, *CurrentMemoryBuffer,
+                                  *CurrentFatBinary);
 }
 
-ErrorOr<MemoryBufferRef>
-BinaryHolder::GetArchiveMemberBuffer(StringRef Filename) {
-  if (!CurrentArchive)
+ErrorOr<std::vector<MemoryBufferRef>>
+BinaryHolder::GetArchiveMemberBuffers(StringRef Filename,
+                                      sys::TimeValue Timestamp) {
+  if (CurrentArchives.empty())
     return make_error_code(errc::no_such_file_or_directory);
 
-  StringRef CurArchiveName = CurrentArchive->getFileName();
+  StringRef CurArchiveName = CurrentArchives.front()->getFileName();
   if (!Filename.startswith(Twine(CurArchiveName, "(").str()))
     return make_error_code(errc::no_such_file_or_directory);
 
   // Remove the archive name and the parens around the archive member name.
   Filename = Filename.substr(CurArchiveName.size() + 1).drop_back();
 
-  for (const auto &Child : CurrentArchive->children()) {
-    if (auto NameOrErr = Child.getName())
-      if (*NameOrErr == Filename) {
-        if (Verbose)
-          outs() << "\tfound member in current archive.\n";
-        return Child.getMemoryBufferRef();
+  std::vector<MemoryBufferRef> Buffers;
+  Buffers.reserve(CurrentArchives.size());
+
+  for (const auto &CurrentArchive : CurrentArchives) {
+    for (const auto &Child : CurrentArchive->children()) {
+      if (auto NameOrErr = Child.getName()) {
+        if (*NameOrErr == Filename) {
+          if (Timestamp != sys::TimeValue::PosixZeroTime() &&
+              Timestamp != Child.getLastModified()) {
+            if (Verbose)
+              outs() << "\tmember had timestamp mismatch.\n";
+            continue;
+          }
+          if (Verbose)
+            outs() << "\tfound member in current archive.\n";
+          auto ErrOrMem = Child.getMemoryBufferRef();
+          if (auto Err = ErrOrMem.getError())
+            return Err;
+          Buffers.push_back(*ErrOrMem);
+        }
       }
+    }
   }
 
-  return make_error_code(errc::no_such_file_or_directory);
+  if (Buffers.empty())
+    return make_error_code(errc::no_such_file_or_directory);
+  return Buffers;
 }
 
-ErrorOr<MemoryBufferRef>
-BinaryHolder::MapArchiveAndGetMemberBuffer(StringRef Filename) {
+ErrorOr<std::vector<MemoryBufferRef>>
+BinaryHolder::MapArchiveAndGetMemberBuffers(StringRef Filename,
+                                            sys::TimeValue Timestamp) {
   StringRef ArchiveFilename = Filename.substr(0, Filename.find('('));
 
   auto ErrOrBuff = MemoryBuffer::getFileOrSTDIN(ArchiveFilename);
@@ -83,29 +145,62 @@ BinaryHolder::MapArchiveAndGetMemberBuffer(StringRef Filename) {
 
   if (Verbose)
     outs() << "\topened new archive '" << ArchiveFilename << "'\n";
-  auto ErrOrArchive = object::Archive::create((*ErrOrBuff)->getMemBufferRef());
-  if (auto Err = ErrOrArchive.getError())
-    return Err;
 
-  CurrentArchive = std::move(*ErrOrArchive);
-  CurrentMemoryBuffer = std::move(*ErrOrBuff);
+  changeBackingMemoryBuffer(std::move(*ErrOrBuff));
+  std::vector<MemoryBufferRef> ArchiveBuffers;
+  auto ErrOrFat = object::MachOUniversalBinary::create(
+      CurrentMemoryBuffer->getMemBufferRef());
+  if (ErrOrFat.getError()) {
+    // Not a fat binary must be a standard one.
+    ArchiveBuffers.push_back(CurrentMemoryBuffer->getMemBufferRef());
+  } else {
+    CurrentFatBinary = std::move(*ErrOrFat);
+    ArchiveBuffers = getMachOFatMemoryBuffers(
+        ArchiveFilename, *CurrentMemoryBuffer, *CurrentFatBinary);
+  }
 
-  return GetArchiveMemberBuffer(Filename);
+  for (auto MemRef : ArchiveBuffers) {
+    auto ErrOrArchive = object::Archive::create(MemRef);
+    if (auto Err = ErrOrArchive.getError())
+      return Err;
+    CurrentArchives.push_back(std::move(*ErrOrArchive));
+  }
+  return GetArchiveMemberBuffers(Filename, Timestamp);
 }
 
 ErrorOr<const object::ObjectFile &>
-BinaryHolder::GetObjectFile(StringRef Filename) {
-  auto ErrOrMemBufferRef = GetMemoryBufferForFile(Filename);
-  if (auto Err = ErrOrMemBufferRef.getError())
-    return Err;
+BinaryHolder::getObjfileForArch(const Triple &T) {
+  for (const auto &Obj : CurrentObjectFiles) {
+    if (const auto *MachO = dyn_cast<object::MachOObjectFile>(Obj.get())) {
+      if (getTriple(*MachO).str() == T.str())
+        return *MachO;
+    } else if (Obj->getArch() == T.getArch())
+      return *Obj;
+  }
 
-  auto ErrOrObjectFile =
-      object::ObjectFile::createObjectFile(*ErrOrMemBufferRef);
-  if (auto Err = ErrOrObjectFile.getError())
+  return make_error_code(object::object_error::arch_not_found);
+}
+
+ErrorOr<std::vector<const object::ObjectFile *>>
+BinaryHolder::GetObjectFiles(StringRef Filename, sys::TimeValue Timestamp) {
+  auto ErrOrMemBufferRefs = GetMemoryBuffersForFile(Filename, Timestamp);
+  if (auto Err = ErrOrMemBufferRefs.getError())
     return Err;
 
-  CurrentObjectFile = std::move(*ErrOrObjectFile);
-  return *CurrentObjectFile;
+  std::vector<const object::ObjectFile *> Objects;
+  Objects.reserve(ErrOrMemBufferRefs->size());
+
+  CurrentObjectFiles.clear();
+  for (auto MemBuf : *ErrOrMemBufferRefs) {
+    auto ErrOrObjectFile = object::ObjectFile::createObjectFile(MemBuf);
+    if (auto Err = ErrOrObjectFile.getError())
+      return Err;
+
+    Objects.push_back(ErrOrObjectFile->get());
+    CurrentObjectFiles.push_back(std::move(*ErrOrObjectFile));
+  }
+
+  return std::move(Objects);
 }
 }
 }
diff --git a/tools/dsymutil/BinaryHolder.h b/tools/dsymutil/BinaryHolder.h
index 04871b5d585..9d7b4bd8787 100644
--- a/tools/dsymutil/BinaryHolder.h
+++ b/tools/dsymutil/BinaryHolder.h
@@ -14,11 +14,14 @@
 #ifndef LLVM_TOOLS_DSYMUTIL_BINARYHOLDER_H
 #define LLVM_TOOLS_DSYMUTIL_BINARYHOLDER_H
 
+#include "llvm/ADT/Triple.h"
 #include "llvm/Object/Archive.h"
 #include "llvm/Object/Error.h"
+#include "llvm/Object/MachOUniversal.h"
 #include "llvm/Object/ObjectFile.h"
 #include "llvm/Support/Errc.h"
 #include "llvm/Support/ErrorOr.h"
+#include "llvm/Support/TimeValue.h"
 
 namespace llvm {
 namespace dsymutil {
@@ -35,69 +38,98 @@ namespace dsymutil {
 /// meaning that a mapping request will invalidate the previous memory
 /// mapping.
 class BinaryHolder {
-  std::unique_ptr<object::Archive> CurrentArchive;
+  std::vector<std::unique_ptr<object::Archive>> CurrentArchives;
   std::unique_ptr<MemoryBuffer> CurrentMemoryBuffer;
-  std::unique_ptr<object::ObjectFile> CurrentObjectFile;
+  std::vector<std::unique_ptr<object::ObjectFile>> CurrentObjectFiles;
+  std::unique_ptr<object::MachOUniversalBinary> CurrentFatBinary;
   bool Verbose;
 
-  /// \brief Get the MemoryBufferRef for the file specification in \p
-  /// Filename from the current archive.
+  /// Get the MemoryBufferRefs for the file specification in \p
+  /// Filename from the current archive. Multiple buffers are returned
+  /// when there are multiple architectures available for the
+  /// requested file.
   ///
   /// This function performs no system calls, it just looks up a
   /// potential match for the given \p Filename in the currently
   /// mapped archive if there is one.
-  ErrorOr<MemoryBufferRef> GetArchiveMemberBuffer(StringRef Filename);
+  ErrorOr<std::vector<MemoryBufferRef>>
+  GetArchiveMemberBuffers(StringRef Filename, sys::TimeValue Timestamp);
 
-  /// \brief Interpret Filename as an archive member specification,
-  /// map the corresponding archive to memory and return the
-  /// MemoryBufferRef corresponding to the described member.
-  ErrorOr<MemoryBufferRef> MapArchiveAndGetMemberBuffer(StringRef Filename);
+  /// Interpret Filename as an archive member specification map the
+  /// corresponding archive to memory and return the MemoryBufferRefs
+  /// corresponding to the described member. Multiple buffers are
+  /// returned when there are multiple architectures available for the
+  /// requested file.
+  ErrorOr<std::vector<MemoryBufferRef>>
+  MapArchiveAndGetMemberBuffers(StringRef Filename, sys::TimeValue Timestamp);
 
-  /// \brief Return the MemoryBufferRef that holds the memory
-  /// mapping for the given \p Filename. This function will try to
-  /// parse archive member specifications of the form
-  /// /path/to/archive.a(member.o).
+  /// Return the MemoryBufferRef that holds the memory mapping for the
+  /// given \p Filename. This function will try to parse archive
+  /// member specifications of the form /path/to/archive.a(member.o).
   ///
-  /// The returned MemoryBufferRef points to a buffer owned by this
+  /// The returned MemoryBufferRefs points to a buffer owned by this
   /// object. The buffer is valid until the next call to
   /// GetMemoryBufferForFile() on this object.
-  ErrorOr<MemoryBufferRef> GetMemoryBufferForFile(StringRef Filename);
+  /// Multiple buffers are returned when there are multiple
+  /// architectures available for the requested file.
+  ErrorOr<std::vector<MemoryBufferRef>>
+  GetMemoryBuffersForFile(StringRef Filename, sys::TimeValue Timestamp);
+
+  void changeBackingMemoryBuffer(std::unique_ptr<MemoryBuffer> &&MemBuf);
+  ErrorOr<const object::ObjectFile &> getObjfileForArch(const Triple &T);
 
 public:
   BinaryHolder(bool Verbose) : Verbose(Verbose) {}
 
-  /// \brief Get the ObjectFile designated by the \p Filename. This
+  /// Get the ObjectFiles designated by the \p Filename. This
   /// might be an archive member specification of the form
   /// /path/to/archive.a(member.o).
   ///
   /// Calling this function invalidates the previous mapping owned by
-  /// the BinaryHolder.
-  ErrorOr<const object::ObjectFile &> GetObjectFile(StringRef Filename);
+  /// the BinaryHolder. Multiple buffers are returned when there are
+  /// multiple architectures available for the requested file.
+  ErrorOr<std::vector<const object::ObjectFile *>>
+  GetObjectFiles(StringRef Filename,
+                 sys::TimeValue Timestamp = sys::TimeValue::PosixZeroTime());
 
-  /// \brief Wraps GetObjectFile() to return a derived ObjectFile type.
+  /// Wraps GetObjectFiles() to return a derived ObjectFile type.
   template <typename ObjectFileType>
-  ErrorOr<const ObjectFileType &> GetFileAs(StringRef Filename) {
-    auto ErrOrObjFile = GetObjectFile(Filename);
+  ErrorOr<std::vector<const ObjectFileType *>>
+  GetFilesAs(StringRef Filename,
+             sys::TimeValue Timestamp = sys::TimeValue::PosixZeroTime()) {
+    auto ErrOrObjFile = GetObjectFiles(Filename, Timestamp);
     if (auto Err = ErrOrObjFile.getError())
       return Err;
-    if (const auto *Derived = dyn_cast<ObjectFileType>(CurrentObjectFile.get()))
-      return *Derived;
-    return make_error_code(object::object_error::invalid_file_type);
+
+    std::vector<const ObjectFileType *> Objects;
+    Objects.reserve((*ErrOrObjFile).size());
+    for (const auto &Obj : *ErrOrObjFile) {
+      const auto *Derived = dyn_cast<ObjectFileType>(Obj);
+      if (!Derived)
+        return make_error_code(object::object_error::invalid_file_type);
+      Objects.push_back(Derived);
+    }
+    return std::move(Objects);
   }
 
-  /// \brief Access the currently owned ObjectFile. As successfull
-  /// call to GetObjectFile() or GetFileAs() must have been performed
-  /// before calling this.
-  const object::ObjectFile &Get() {
-    assert(CurrentObjectFile);
-    return *CurrentObjectFile;
+  /// Access the currently owned ObjectFile with architecture \p T. As
+  /// successfull call to GetObjectFiles() or GetFilesAs() must have
+  /// been performed before calling this.
+  ErrorOr<const object::ObjectFile &> Get(const Triple &T) {
+    return getObjfileForArch(T);
   }
 
-  /// \brief Access to a derived version of the currently owned
+  /// Access to a derived version of the currently owned
   /// ObjectFile. The conversion must be known to be valid.
-  template <typename ObjectFileType> const ObjectFileType &GetAs() {
-    return cast<ObjectFileType>(*CurrentObjectFile);
+  template <typename ObjectFileType>
+  ErrorOr<const ObjectFileType &> GetAs(const Triple &T) {
+    auto ErrOrObj = Get(T);
+    if (auto Err = ErrOrObj.getError())
+      return Err;
+    return cast<ObjectFileType>(*ErrOrObj);
   }
+
+  static Triple getTriple(const object::MachOObjectFile &Obj);
 };
 }
 }
diff --git a/tools/dsymutil/CMakeLists.txt b/tools/dsymutil/CMakeLists.txt
index 59b37a9b290..88f9f1f083d 100644
--- a/tools/dsymutil/CMakeLists.txt
+++ b/tools/dsymutil/CMakeLists.txt
@@ -5,6 +5,7 @@ set(LLVM_LINK_COMPONENTS
   MC
   Object
   Support
+  Target
   )
 
 add_llvm_tool(llvm-dsymutil
diff --git a/tools/dsymutil/DebugMap.cpp b/tools/dsymutil/DebugMap.cpp
index 9fa3f788a89..e29c885b5ac 100644
--- a/tools/dsymutil/DebugMap.cpp
+++ b/tools/dsymutil/DebugMap.cpp
@@ -7,6 +7,7 @@
 //
 //===----------------------------------------------------------------------===//
 #include "DebugMap.h"
+#include "BinaryHolder.h"
 #include "llvm/ADT/STLExtras.h"
 #include "llvm/ADT/iterator_range.h"
 #include "llvm/Support/DataTypes.h"
@@ -19,8 +20,9 @@ namespace dsymutil {
 
 using namespace llvm::object;
 
-DebugMapObject::DebugMapObject(StringRef ObjectFilename)
-    : Filename(ObjectFilename) {}
+DebugMapObject::DebugMapObject(StringRef ObjectFilename,
+                               sys::TimeValue Timestamp)
+    : Filename(ObjectFilename), Timestamp(Timestamp) {}
 
 bool DebugMapObject::addSymbol(StringRef Name, uint64_t ObjectAddress,
                                uint64_t LinkedAddress, uint32_t Size) {
@@ -46,8 +48,9 @@ void DebugMapObject::print(raw_ostream &OS) const {
       [](const Entry &LHS, const Entry &RHS) { return LHS.first < RHS.first; });
   for (const auto &Sym : Entries) {
     OS << format("\t%016" PRIx64 " => %016" PRIx64 "+0x%x\t%s\n",
-                 Sym.second.ObjectAddress, Sym.second.BinaryAddress,
-                 Sym.second.Size, Sym.first.data());
+                 uint64_t(Sym.second.ObjectAddress),
+                 uint64_t(Sym.second.BinaryAddress), uint32_t(Sym.second.Size),
+                 Sym.first.data());
   }
   OS << '\n';
 }
@@ -56,8 +59,9 @@ void DebugMapObject::print(raw_ostream &OS) const {
 void DebugMapObject::dump() const { print(errs()); }
 #endif
 
-DebugMapObject &DebugMap::addDebugMapObject(StringRef ObjectFilePath) {
-  Objects.emplace_back(new DebugMapObject(ObjectFilePath));
+DebugMapObject &DebugMap::addDebugMapObject(StringRef ObjectFilePath,
+                                            sys::TimeValue Timestamp) {
+  Objects.emplace_back(new DebugMapObject(ObjectFilePath, Timestamp));
   return *Objects.back();
 }
 
@@ -78,15 +82,165 @@ DebugMapObject::lookupObjectAddress(uint64_t Address) const {
 }
 
 void DebugMap::print(raw_ostream &OS) const {
-  OS << "DEBUG MAP: " << BinaryTriple.getTriple()
-     << "\n\tobject addr =>  executable addr\tsymbol name\n";
-  for (const auto &Obj : objects())
-    Obj->print(OS);
-  OS << "END DEBUG MAP\n";
+  yaml::Output yout(OS, /* Ctxt = */ nullptr, /* WrapColumn = */ 0);
+  yout << const_cast<DebugMap &>(*this);
 }
 
 #ifndef NDEBUG
 void DebugMap::dump() const { print(errs()); }
 #endif
+
+namespace {
+struct YAMLContext {
+  StringRef PrependPath;
+  Triple BinaryTriple;
+};
+}
+
+ErrorOr<std::unique_ptr<DebugMap>>
+DebugMap::parseYAMLDebugMap(StringRef InputFile, StringRef PrependPath,
+                            bool Verbose) {
+  auto ErrOrFile = MemoryBuffer::getFileOrSTDIN(InputFile);
+  if (auto Err = ErrOrFile.getError())
+    return Err;
+
+  YAMLContext Ctxt;
+
+  Ctxt.PrependPath = PrependPath;
+
+  std::unique_ptr<DebugMap> Res;
+  yaml::Input yin((*ErrOrFile)->getBuffer(), &Ctxt);
+  yin >> Res;
+
+  if (auto EC = yin.error())
+    return EC;
+
+  return std::move(Res);
+}
+}
+
+namespace yaml {
+
+// Normalize/Denormalize between YAML and a DebugMapObject.
+struct MappingTraits<dsymutil::DebugMapObject>::YamlDMO {
+  YamlDMO(IO &io) { Timestamp = 0; }
+  YamlDMO(IO &io, dsymutil::DebugMapObject &Obj);
+  dsymutil::DebugMapObject denormalize(IO &IO);
+
+  std::string Filename;
+  sys::TimeValue::SecondsType Timestamp;
+  std::vector<dsymutil::DebugMapObject::YAMLSymbolMapping> Entries;
+};
+
+void MappingTraits<std::pair<std::string, DebugMapObject::SymbolMapping>>::
+    mapping(IO &io, std::pair<std::string, DebugMapObject::SymbolMapping> &s) {
+  io.mapRequired("sym", s.first);
+  io.mapRequired("objAddr", s.second.ObjectAddress);
+  io.mapRequired("binAddr", s.second.BinaryAddress);
+  io.mapOptional("size", s.second.Size);
+}
+
+void MappingTraits<dsymutil::DebugMapObject>::mapping(
+    IO &io, dsymutil::DebugMapObject &DMO) {
+  MappingNormalization<YamlDMO, dsymutil::DebugMapObject> Norm(io, DMO);
+  io.mapRequired("filename", Norm->Filename);
+  io.mapOptional("timestamp", Norm->Timestamp);
+  io.mapRequired("symbols", Norm->Entries);
+}
+
+void ScalarTraits<Triple>::output(const Triple &val, void *,
+                                  llvm::raw_ostream &out) {
+  out << val.str();
+}
+
+StringRef ScalarTraits<Triple>::input(StringRef scalar, void *, Triple &value) {
+  value = Triple(scalar);
+  return StringRef();
+}
+
+size_t
+SequenceTraits<std::vector<std::unique_ptr<dsymutil::DebugMapObject>>>::size(
+    IO &io, std::vector<std::unique_ptr<dsymutil::DebugMapObject>> &seq) {
+  return seq.size();
+}
+
+dsymutil::DebugMapObject &
+SequenceTraits<std::vector<std::unique_ptr<dsymutil::DebugMapObject>>>::element(
+    IO &, std::vector<std::unique_ptr<dsymutil::DebugMapObject>> &seq,
+    size_t index) {
+  if (index >= seq.size()) {
+    seq.resize(index + 1);
+    seq[index].reset(new dsymutil::DebugMapObject);
+  }
+  return *seq[index];
+}
+
+void MappingTraits<dsymutil::DebugMap>::mapping(IO &io,
+                                                dsymutil::DebugMap &DM) {
+  io.mapRequired("triple", DM.BinaryTriple);
+  if (void *Ctxt = io.getContext())
+    reinterpret_cast<YAMLContext *>(Ctxt)->BinaryTriple = DM.BinaryTriple;
+  io.mapOptional("objects", DM.Objects);
+}
+
+void MappingTraits<std::unique_ptr<dsymutil::DebugMap>>::mapping(
+    IO &io, std::unique_ptr<dsymutil::DebugMap> &DM) {
+  if (!DM)
+    DM.reset(new DebugMap());
+  io.mapRequired("triple", DM->BinaryTriple);
+  if (void *Ctxt = io.getContext())
+    reinterpret_cast<YAMLContext *>(Ctxt)->BinaryTriple = DM->BinaryTriple;
+  io.mapOptional("objects", DM->Objects);
+}
+
+MappingTraits<dsymutil::DebugMapObject>::YamlDMO::YamlDMO(
+    IO &io, dsymutil::DebugMapObject &Obj) {
+  Filename = Obj.Filename;
+  Timestamp = Obj.getTimestamp().toEpochTime();
+  Entries.reserve(Obj.Symbols.size());
+  for (auto &Entry : Obj.Symbols)
+    Entries.push_back(std::make_pair(Entry.getKey(), Entry.getValue()));
+}
+
+dsymutil::DebugMapObject
+MappingTraits<dsymutil::DebugMapObject>::YamlDMO::denormalize(IO &IO) {
+  BinaryHolder BinHolder(/* Verbose =*/false);
+  const auto &Ctxt = *reinterpret_cast<YAMLContext *>(IO.getContext());
+  SmallString<80> Path(Ctxt.PrependPath);
+  StringMap<uint64_t> SymbolAddresses;
+
+  sys::path::append(Path, Filename);
+  auto ErrOrObjectFiles = BinHolder.GetObjectFiles(Path);
+  if (auto EC = ErrOrObjectFiles.getError()) {
+    llvm::errs() << "warning: Unable to open " << Path << " " << EC.message()
+                 << '\n';
+  } else if (auto ErrOrObjectFile = BinHolder.Get(Ctxt.BinaryTriple)) {
+    // Rewrite the object file symbol addresses in the debug map. The
+    // YAML input is mainly used to test llvm-dsymutil without
+    // requiring binaries checked-in. If we generate the object files
+    // during the test, we can't hardcode the symbols addresses, so
+    // look them up here and rewrite them.
+    for (const auto &Sym : ErrOrObjectFile->symbols()) {
+      uint64_t Address = Sym.getValue();
+      ErrorOr<StringRef> Name = Sym.getName();
+      if (!Name)
+        continue;
+      SymbolAddresses[*Name] = Address;
+    }
+  }
+
+  sys::TimeValue TV;
+  TV.fromEpochTime(Timestamp);
+  dsymutil::DebugMapObject Res(Path, TV);
+  for (auto &Entry : Entries) {
+    auto &Mapping = Entry.second;
+    uint64_t ObjAddress = Mapping.ObjectAddress;
+    auto AddressIt = SymbolAddresses.find(Entry.first);
+    if (AddressIt != SymbolAddresses.end())
+      ObjAddress = AddressIt->getValue();
+    Res.addSymbol(Entry.first, ObjAddress, Mapping.BinaryAddress, Mapping.Size);
+  }
+  return Res;
+}
 }
 }
diff --git a/tools/dsymutil/DebugMap.h b/tools/dsymutil/DebugMap.h
index ee48b093d4f..29347ed0f50 100644
--- a/tools/dsymutil/DebugMap.h
+++ b/tools/dsymutil/DebugMap.h
@@ -28,6 +28,9 @@
 #include "llvm/Object/ObjectFile.h"
 #include "llvm/Support/ErrorOr.h"
 #include "llvm/Support/Format.h"
+#include "llvm/Support/Path.h"
+#include "llvm/Support/TimeValue.h"
+#include "llvm/Support/YAMLTraits.h"
 #include <vector>
 
 namespace llvm {
@@ -66,6 +69,12 @@ class DebugMap {
   typedef std::vector<std::unique_ptr<DebugMapObject>> ObjectContainer;
   ObjectContainer Objects;
 
+  /// For YAML IO support.
+  ///@{
+  friend yaml::MappingTraits<std::unique_ptr<DebugMap>>;
+  friend yaml::MappingTraits<DebugMap>;
+  DebugMap() = default;
+  ///@}
 public:
   DebugMap(const Triple &BinaryTriple) : BinaryTriple(BinaryTriple) {}
 
@@ -81,7 +90,8 @@ class DebugMap {
 
   /// This function adds an DebugMapObject to the list owned by this
   /// debug map.
-  DebugMapObject &addDebugMapObject(StringRef ObjectFilePath);
+  DebugMapObject &addDebugMapObject(StringRef ObjectFilePath,
+                                    sys::TimeValue Timestamp);
 
   const Triple &getTriple() const { return BinaryTriple; }
 
@@ -90,6 +100,10 @@ class DebugMap {
 #ifndef NDEBUG
   void dump() const;
 #endif
+
+  /// Read a debug map for \a InputFile.
+  static ErrorOr<std::unique_ptr<DebugMap>>
+  parseYAMLDebugMap(StringRef InputFile, StringRef PrependPath, bool Verbose);
 };
 
 /// \brief The DebugMapObject represents one object file described by
@@ -99,12 +113,14 @@ class DebugMap {
 class DebugMapObject {
 public:
   struct SymbolMapping {
-    uint64_t ObjectAddress;
-    uint64_t BinaryAddress;
-    uint32_t Size;
+    yaml::Hex64 ObjectAddress;
+    yaml::Hex64 BinaryAddress;
+    yaml::Hex32 Size;
     SymbolMapping(uint64_t ObjectAddress, uint64_t BinaryAddress, uint32_t Size)
         : ObjectAddress(ObjectAddress), BinaryAddress(BinaryAddress),
           Size(Size) {}
+    /// For YAML IO support
+    SymbolMapping() = default;
   };
 
   typedef StringMapEntry<SymbolMapping> DebugMapEntry;
@@ -125,6 +141,8 @@ class DebugMapObject {
 
   llvm::StringRef getObjectFilename() const { return Filename; }
 
+  sys::TimeValue getTimestamp() const { return Timestamp; }
+
   iterator_range<StringMap<SymbolMapping>::const_iterator> symbols() const {
     return make_range(Symbols.begin(), Symbols.end());
   }
@@ -136,11 +154,80 @@ class DebugMapObject {
 private:
   friend class DebugMap;
   /// DebugMapObjects can only be constructed by the owning DebugMap.
-  DebugMapObject(StringRef ObjectFilename);
+  DebugMapObject(StringRef ObjectFilename, sys::TimeValue Timestamp);
 
   std::string Filename;
+  sys::TimeValue Timestamp;
   StringMap<SymbolMapping> Symbols;
   DenseMap<uint64_t, DebugMapEntry *> AddressToMapping;
+
+  /// For YAMLIO support.
+  ///@{
+  typedef std::pair<std::string, SymbolMapping> YAMLSymbolMapping;
+  friend yaml::MappingTraits<dsymutil::DebugMapObject>;
+  friend yaml::SequenceTraits<std::vector<std::unique_ptr<DebugMapObject>>>;
+  friend yaml::SequenceTraits<std::vector<YAMLSymbolMapping>>;
+  DebugMapObject() = default;
+
+public:
+  DebugMapObject &operator=(DebugMapObject RHS) {
+    std::swap(Filename, RHS.Filename);
+    std::swap(Timestamp, RHS.Timestamp);
+    std::swap(Symbols, RHS.Symbols);
+    std::swap(AddressToMapping, RHS.AddressToMapping);
+    return *this;
+  }
+  DebugMapObject(DebugMapObject &&RHS) {
+    Filename = std::move(RHS.Filename);
+    Timestamp = std::move(RHS.Timestamp);
+    Symbols = std::move(RHS.Symbols);
+    AddressToMapping = std::move(RHS.AddressToMapping);
+  }
+  ///@}
+};
+}
+}
+
+LLVM_YAML_IS_SEQUENCE_VECTOR(llvm::dsymutil::DebugMapObject::YAMLSymbolMapping)
+
+namespace llvm {
+namespace yaml {
+
+using namespace llvm::dsymutil;
+
+template <>
+struct MappingTraits<std::pair<std::string, DebugMapObject::SymbolMapping>> {
+  static void mapping(IO &io,
+                      std::pair<std::string, DebugMapObject::SymbolMapping> &s);
+  static const bool flow = true;
+};
+
+template <> struct MappingTraits<dsymutil::DebugMapObject> {
+  struct YamlDMO;
+  static void mapping(IO &io, dsymutil::DebugMapObject &DMO);
+};
+
+template <> struct ScalarTraits<Triple> {
+  static void output(const Triple &val, void *, llvm::raw_ostream &out);
+  static StringRef input(StringRef scalar, void *, Triple &value);
+  static bool mustQuote(StringRef) { return true; }
+};
+
+template <>
+struct SequenceTraits<std::vector<std::unique_ptr<dsymutil::DebugMapObject>>> {
+  static size_t
+  size(IO &io, std::vector<std::unique_ptr<dsymutil::DebugMapObject>> &seq);
+  static dsymutil::DebugMapObject &
+  element(IO &, std::vector<std::unique_ptr<dsymutil::DebugMapObject>> &seq,
+          size_t index);
+};
+
+template <> struct MappingTraits<dsymutil::DebugMap> {
+  static void mapping(IO &io, dsymutil::DebugMap &DM);
+};
+
+template <> struct MappingTraits<std::unique_ptr<dsymutil::DebugMap>> {
+  static void mapping(IO &io, std::unique_ptr<dsymutil::DebugMap> &DM);
 };
 }
 }
diff --git a/tools/dsymutil/DwarfLinker.cpp b/tools/dsymutil/DwarfLinker.cpp
index c9c05cd54ac..484c86aa5df 100644
--- a/tools/dsymutil/DwarfLinker.cpp
+++ b/tools/dsymutil/DwarfLinker.cpp
@@ -15,6 +15,7 @@
 #include "llvm/ADT/STLExtras.h"
 #include "llvm/CodeGen/AsmPrinter.h"
 #include "llvm/CodeGen/DIE.h"
+#include "llvm/Config/config.h"
 #include "llvm/DebugInfo/DWARF/DWARFContext.h"
 #include "llvm/DebugInfo/DWARF/DWARFDebugInfoEntry.h"
 #include "llvm/DebugInfo/DWARF/DWARFFormValue.h"
@@ -27,6 +28,7 @@
 #include "llvm/MC/MCObjectFileInfo.h"
 #include "llvm/MC/MCRegisterInfo.h"
 #include "llvm/MC/MCStreamer.h"
+#include "llvm/MC/MCSubtargetInfo.h"
 #include "llvm/Object/MachO.h"
 #include "llvm/Support/Dwarf.h"
 #include "llvm/Support/LEB128.h"
@@ -59,6 +61,132 @@ using HalfOpenIntervalMap =
 
 typedef HalfOpenIntervalMap<uint64_t, int64_t> FunctionIntervals;
 
+// FIXME: Delete this structure.
+struct PatchLocation {
+  DIE::value_iterator I;
+
+  PatchLocation() = default;
+  PatchLocation(DIE::value_iterator I) : I(I) {}
+
+  void set(uint64_t New) const {
+    assert(I);
+    const auto &Old = *I;
+    assert(Old.getType() == DIEValue::isInteger);
+    *I = DIEValue(Old.getAttribute(), Old.getForm(), DIEInteger(New));
+  }
+
+  uint64_t get() const {
+    assert(I);
+    return I->getDIEInteger().getValue();
+  }
+};
+
+class CompileUnit;
+struct DeclMapInfo;
+class NonRelocatableStringpool;
+
+/// A DeclContext is a named program scope that is used for ODR
+/// uniquing of types.
+/// The set of DeclContext for the ODR-subject parts of a Dwarf link
+/// is expanded (and uniqued) with each new object file processed. We
+/// need to determine the context of each DIE in an linked object file
+/// to see if the corresponding type has already been emitted.
+///
+/// The contexts are conceptually organised as a tree (eg. a function
+/// scope is contained in a namespace scope that contains other
+/// scopes), but storing/accessing them in an actual tree is too
+/// inefficient: we need to be able to very quickly query a context
+/// for a given child context by name. Storing a StringMap in each
+/// DeclContext would be too space inefficient.
+/// The solution here is to give each DeclContext a link to its parent
+/// (this allows to walk up the tree), but to query the existance of a
+/// specific DeclContext using a separate DenseMap keyed on the hash
+/// of the fully qualified name of the context.
+class DeclContext {
+  unsigned QualifiedNameHash;
+  uint32_t Line;
+  uint32_t ByteSize;
+  uint16_t Tag;
+  StringRef Name;
+  StringRef File;
+  const DeclContext &Parent;
+  const DWARFDebugInfoEntryMinimal *LastSeenDIE;
+  uint32_t LastSeenCompileUnitID;
+  uint32_t CanonicalDIEOffset;
+
+  friend DeclMapInfo;
+
+public:
+  typedef DenseSet<DeclContext *, DeclMapInfo> Map;
+
+  DeclContext()
+      : QualifiedNameHash(0), Line(0), ByteSize(0),
+        Tag(dwarf::DW_TAG_compile_unit), Name(), File(), Parent(*this),
+        LastSeenDIE(nullptr), LastSeenCompileUnitID(0), CanonicalDIEOffset(0) {}
+
+  DeclContext(unsigned Hash, uint32_t Line, uint32_t ByteSize, uint16_t Tag,
+              StringRef Name, StringRef File, const DeclContext &Parent,
+              const DWARFDebugInfoEntryMinimal *LastSeenDIE = nullptr,
+              unsigned CUId = 0)
+      : QualifiedNameHash(Hash), Line(Line), ByteSize(ByteSize), Tag(Tag),
+        Name(Name), File(File), Parent(Parent), LastSeenDIE(LastSeenDIE),
+        LastSeenCompileUnitID(CUId), CanonicalDIEOffset(0) {}
+
+  uint32_t getQualifiedNameHash() const { return QualifiedNameHash; }
+
+  bool setLastSeenDIE(CompileUnit &U, const DWARFDebugInfoEntryMinimal *Die);
+
+  uint32_t getCanonicalDIEOffset() const { return CanonicalDIEOffset; }
+  void setCanonicalDIEOffset(uint32_t Offset) { CanonicalDIEOffset = Offset; }
+
+  uint16_t getTag() const { return Tag; }
+  StringRef getName() const { return Name; }
+};
+
+/// Info type for the DenseMap storing the DeclContext pointers.
+struct DeclMapInfo : private DenseMapInfo<DeclContext *> {
+  using DenseMapInfo<DeclContext *>::getEmptyKey;
+  using DenseMapInfo<DeclContext *>::getTombstoneKey;
+
+  static unsigned getHashValue(const DeclContext *Ctxt) {
+    return Ctxt->QualifiedNameHash;
+  }
+
+  static bool isEqual(const DeclContext *LHS, const DeclContext *RHS) {
+    if (RHS == getEmptyKey() || RHS == getTombstoneKey())
+      return RHS == LHS;
+    return LHS->QualifiedNameHash == RHS->QualifiedNameHash &&
+           LHS->Line == RHS->Line && LHS->ByteSize == RHS->ByteSize &&
+           LHS->Name.data() == RHS->Name.data() &&
+           LHS->File.data() == RHS->File.data() &&
+           LHS->Parent.QualifiedNameHash == RHS->Parent.QualifiedNameHash;
+  }
+};
+
+/// This class gives a tree-like API to the DenseMap that stores the
+/// DeclContext objects. It also holds the BumpPtrAllocator where
+/// these objects will be allocated.
+class DeclContextTree {
+  BumpPtrAllocator Allocator;
+  DeclContext Root;
+  DeclContext::Map Contexts;
+
+public:
+  /// Get the child of \a Context described by \a DIE in \a Unit. The
+  /// required strings will be interned in \a StringPool.
+  /// \returns The child DeclContext along with one bit that is set if
+  /// this context is invalid.
+  /// FIXME: the invalid bit along the return value is to emulate some
+  /// dsymutil-classic functionality. See the fucntion definition for
+  /// a more thorough discussion of its use.
+  PointerIntPair<DeclContext *, 1>
+  getChildDeclContext(DeclContext &Context,
+                      const DWARFDebugInfoEntryMinimal *DIE, CompileUnit &Unit,
+                      NonRelocatableStringpool &StringPool);
+
+  DeclContext &getRoot() { return Root; }
+};
+
 /// \brief Stores all information relating to a compile unit, be it in
 /// its original instance in the object file to its brand new cloned
 /// and linked DIE tree.
@@ -67,16 +195,26 @@ class CompileUnit {
   /// \brief Information gathered about a DIE in the object file.
   struct DIEInfo {
     int64_t AddrAdjust; ///< Address offset to apply to the described entity.
+    DeclContext *Ctxt;  ///< ODR Declaration context.
     DIE *Clone;         ///< Cloned version of that DIE.
     uint32_t ParentIdx; ///< The index of this DIE's parent.
     bool Keep;          ///< Is the DIE part of the linked output?
     bool InDebugMap;    ///< Was this DIE's entity found in the map?
   };
 
-  CompileUnit(DWARFUnit &OrigUnit, unsigned ID)
+  CompileUnit(DWARFUnit &OrigUnit, unsigned ID, bool CanUseODR)
       : OrigUnit(OrigUnit), ID(ID), LowPc(UINT64_MAX), HighPc(0), RangeAlloc(),
-        Ranges(RangeAlloc), UnitRangeAttribute(nullptr) {
+        Ranges(RangeAlloc) {
     Info.resize(OrigUnit.getNumDIEs());
+
+    const auto *CUDie = OrigUnit.getUnitDIE(false);
+    unsigned Lang = CUDie->getAttributeValueAsUnsignedConstant(
+        &OrigUnit, dwarf::DW_AT_language, 0);
+    HasODR = CanUseODR && (Lang == dwarf::DW_LANG_C_plus_plus ||
+                           Lang == dwarf::DW_LANG_C_plus_plus_03 ||
+                           Lang == dwarf::DW_LANG_C_plus_plus_11 ||
+                           Lang == dwarf::DW_LANG_C_plus_plus_14 ||
+                           Lang == dwarf::DW_LANG_ObjC_plus_plus);
   }
 
   CompileUnit(CompileUnit &&RHS)
@@ -92,8 +230,10 @@ class CompileUnit {
 
   unsigned getUniqueID() const { return ID; }
 
-  DIE *getOutputUnitDIE() const { return CUDie.get(); }
-  void setOutputUnitDIE(DIE *Die) { CUDie.reset(Die); }
+  DIE *getOutputUnitDIE() const { return CUDie; }
+  void setOutputUnitDIE(DIE *Die) { CUDie = Die; }
+
+  bool hasODR() const { return HasODR; }
 
   DIEInfo &getInfo(unsigned Idx) { return Info[Idx]; }
   const DIEInfo &getInfo(unsigned Idx) const { return Info[Idx]; }
@@ -105,13 +245,15 @@ class CompileUnit {
   uint64_t getLowPc() const { return LowPc; }
   uint64_t getHighPc() const { return HighPc; }
 
-  DIEInteger *getUnitRangesAttribute() const { return UnitRangeAttribute; }
+  Optional<PatchLocation> getUnitRangesAttribute() const {
+    return UnitRangeAttribute;
+  }
   const FunctionIntervals &getFunctionRanges() const { return Ranges; }
-  const std::vector<DIEInteger *> &getRangesAttributes() const {
+  const std::vector<PatchLocation> &getRangesAttributes() const {
     return RangeAttributes;
   }
 
-  const std::vector<std::pair<DIEInteger *, int64_t>> &
+  const std::vector<std::pair<PatchLocation, int64_t>> &
   getLocationAttributes() const {
     return LocationAttributes;
   }
@@ -124,9 +266,10 @@ class CompileUnit {
 
   /// \brief Keep track of a forward reference to DIE \p Die in \p
   /// RefUnit by \p Attr. The attribute should be fixed up later to
-  /// point to the absolute offset of \p Die in the debug_info section.
+  /// point to the absolute offset of \p Die in the debug_info section
+  /// or to the canonical offset of \p Ctxt if it is non-null.
   void noteForwardReference(DIE *Die, const CompileUnit *RefUnit,
-                            DIEInteger *Attr);
+                            DeclContext *Ctxt, PatchLocation Attr);
 
   /// \brief Apply all fixups recored by noteForwardReference().
   void fixupForwardReferences();
@@ -137,11 +280,11 @@ class CompileUnit {
 
   /// \brief Keep track of a DW_AT_range attribute that we will need to
   /// patch up later.
-  void noteRangeAttribute(const DIE &Die, DIEInteger *Attr);
+  void noteRangeAttribute(const DIE &Die, PatchLocation Attr);
 
   /// \brief Keep track of a location attribute pointing to a location
   /// list in the debug_loc section.
-  void noteLocationAttribute(DIEInteger *Attr, int64_t PcOffset);
+  void noteLocationAttribute(PatchLocation Attr, int64_t PcOffset);
 
   /// \brief Add a name accelerator entry for \p Die with \p Name
   /// which is stored in the string table at \p Offset.
@@ -153,8 +296,8 @@ class CompileUnit {
   void addTypeAccelerator(const DIE *Die, const char *Name, uint32_t Offset);
 
   struct AccelInfo {
-    StringRef Name; ///< Name of the entry.
-    const DIE *Die; ///< DIE this entry describes.
+    StringRef Name;      ///< Name of the entry.
+    const DIE *Die;      ///< DIE this entry describes.
     uint32_t NameOffset; ///< Offset of Name in the string pool.
     bool SkipPubSection; ///< Emit this entry only in the apple_* sections.
 
@@ -167,11 +310,27 @@ class CompileUnit {
   const std::vector<AccelInfo> &getPubnames() const { return Pubnames; }
   const std::vector<AccelInfo> &getPubtypes() const { return Pubtypes; }
 
+  /// Get the full path for file \a FileNum in the line table
+  const char *getResolvedPath(unsigned FileNum) {
+    if (FileNum >= ResolvedPaths.size())
+      return nullptr;
+    return ResolvedPaths[FileNum].size() ? ResolvedPaths[FileNum].c_str()
+                                         : nullptr;
+  }
+
+  /// Set the fully resolved path for the line-table's file \a FileNum
+  /// to \a Path.
+  void setResolvedPath(unsigned FileNum, const std::string &Path) {
+    if (ResolvedPaths.size() <= FileNum)
+      ResolvedPaths.resize(FileNum + 1);
+    ResolvedPaths[FileNum] = Path;
+  }
+
 private:
   DWARFUnit &OrigUnit;
   unsigned ID;
-  std::vector<DIEInfo> Info;  ///< DIE info indexed by DIE index.
-  std::unique_ptr<DIE> CUDie; ///< Root of the linked DIE tree.
+  std::vector<DIEInfo> Info; ///< DIE info indexed by DIE index.
+  DIE *CUDie;                ///< Root of the linked DIE tree.
 
   uint64_t StartOffset;
   uint64_t NextUnitOffset;
@@ -185,8 +344,8 @@ class CompileUnit {
   /// The offsets for the attributes in this array couldn't be set while
   /// cloning because for cross-cu forward refences the target DIE's
   /// offset isn't known you emit the reference attribute.
-  std::vector<std::tuple<DIE *, const CompileUnit *, DIEInteger *>>
-      ForwardDIEReferences;
+  std::vector<std::tuple<DIE *, const CompileUnit *, DeclContext *,
+                         PatchLocation>> ForwardDIEReferences;
 
   FunctionIntervals::Allocator RangeAlloc;
   /// \brief The ranges in that interval map are the PC ranges for
@@ -197,15 +356,15 @@ class CompileUnit {
   /// \brief DW_AT_ranges attributes to patch after we have gathered
   /// all the unit's function addresses.
   /// @{
-  std::vector<DIEInteger *> RangeAttributes;
-  DIEInteger *UnitRangeAttribute;
+  std::vector<PatchLocation> RangeAttributes;
+  Optional<PatchLocation> UnitRangeAttribute;
   /// @}
 
   /// \brief Location attributes that need to be transfered from th
   /// original debug_loc section to the liked one. They are stored
   /// along with the PC offset that is to be applied to their
   /// function's address.
-  std::vector<std::pair<DIEInteger *, int64_t>> LocationAttributes;
+  std::vector<std::pair<PatchLocation, int64_t>> LocationAttributes;
 
   /// \brief Accelerator entries for the unit, both for the pub*
   /// sections and the apple* ones.
@@ -213,6 +372,12 @@ class CompileUnit {
   std::vector<AccelInfo> Pubnames;
   std::vector<AccelInfo> Pubtypes;
   /// @}
+
+  /// Cached resolved paths from the line table.
+  std::vector<std::string> ResolvedPaths;
+
+  /// Is this unit subject to the ODR rule?
+  bool HasODR;
 };
 
 uint64_t CompileUnit::computeNextUnitOffset() {
@@ -228,8 +393,8 @@ uint64_t CompileUnit::computeNextUnitOffset() {
 /// \brief Keep track of a forward cross-cu reference from this unit
 /// to \p Die that lives in \p RefUnit.
 void CompileUnit::noteForwardReference(DIE *Die, const CompileUnit *RefUnit,
-                                       DIEInteger *Attr) {
-  ForwardDIEReferences.emplace_back(Die, RefUnit, Attr);
+                                       DeclContext *Ctxt, PatchLocation Attr) {
+  ForwardDIEReferences.emplace_back(Die, RefUnit, Ctxt, Attr);
 }
 
 /// \brief Apply all fixups recorded by noteForwardReference().
@@ -237,9 +402,13 @@ void CompileUnit::fixupForwardReferences() {
   for (const auto &Ref : ForwardDIEReferences) {
     DIE *RefDie;
     const CompileUnit *RefUnit;
-    DIEInteger *Attr;
-    std::tie(RefDie, RefUnit, Attr) = Ref;
-    Attr->setValue(RefDie->getOffset() + RefUnit->getStartOffset());
+    PatchLocation Attr;
+    DeclContext *Ctxt;
+    std::tie(RefDie, RefUnit, Ctxt, Attr) = Ref;
+    if (Ctxt && Ctxt->getCanonicalDIEOffset())
+      Attr.set(Ctxt->getCanonicalDIEOffset());
+    else
+      Attr.set(RefDie->getOffset() + RefUnit->getStartOffset());
   }
 }
 
@@ -250,14 +419,14 @@ void CompileUnit::addFunctionRange(uint64_t FuncLowPc, uint64_t FuncHighPc,
   this->HighPc = std::max(HighPc, FuncHighPc + PcOffset);
 }
 
-void CompileUnit::noteRangeAttribute(const DIE &Die, DIEInteger *Attr) {
+void CompileUnit::noteRangeAttribute(const DIE &Die, PatchLocation Attr) {
   if (Die.getTag() != dwarf::DW_TAG_compile_unit)
     RangeAttributes.push_back(Attr);
   else
     UnitRangeAttribute = Attr;
 }
 
-void CompileUnit::noteLocationAttribute(DIEInteger *Attr, int64_t PcOffset) {
+void CompileUnit::noteLocationAttribute(PatchLocation Attr, int64_t PcOffset) {
   LocationAttributes.emplace_back(Attr, PcOffset);
 }
 
@@ -386,10 +555,11 @@ class DwarfStreamer {
   uint32_t RangesSectionSize;
   uint32_t LocSectionSize;
   uint32_t LineSectionSize;
+  uint32_t FrameSectionSize;
 
   /// \brief Emit the pubnames or pubtypes section contribution for \p
   /// Unit into \p Sec. The data is provided in \p Names.
-  void emitPubSectionForUnit(const MCSection *Sec, StringRef Name,
+  void emitPubSectionForUnit(MCSection *Sec, StringRef Name,
                              const CompileUnit &Unit,
                              const std::vector<CompileUnit::AccelInfo> &Names);
 
@@ -459,6 +629,15 @@ class DwarfStreamer {
 
   /// \brief Emit the .debug_pubtypes contribution for \p Unit.
   void emitPubTypesForUnit(const CompileUnit &Unit);
+
+  /// \brief Emit a CIE.
+  void emitCIE(StringRef CIEBytes);
+
+  /// \brief Emit an FDE with data \p Bytes.
+  void emitFDE(uint32_t CIEOffset, uint32_t AddreSize, uint32_t Address,
+               StringRef Bytes);
+
+  uint32_t getFrameSectionSize() const { return FrameSectionSize; }
 };
 
 bool DwarfStreamer::init(Triple TheTriple, StringRef OutputFilename) {
@@ -484,7 +663,7 @@ bool DwarfStreamer::init(Triple TheTriple, StringRef OutputFilename) {
 
   MOFI.reset(new MCObjectFileInfo);
   MC.reset(new MCContext(MAI.get(), MRI.get(), MOFI.get()));
-  MOFI->InitMCObjectFileInfo(TripleName, Reloc::Default, CodeModel::Default,
+  MOFI->InitMCObjectFileInfo(TheTriple, Reloc::Default, CodeModel::Default,
                              *MC);
 
   MAB = TheTarget->createMCAsmBackend(*MRI, TripleName, "");
@@ -528,6 +707,7 @@ bool DwarfStreamer::init(Triple TheTriple, StringRef OutputFilename) {
   RangesSectionSize = 0;
   LocSectionSize = 0;
   LineSectionSize = 0;
+  FrameSectionSize = 0;
 
   return true;
 }
@@ -584,10 +764,10 @@ void DwarfStreamer::emitDIE(DIE &Die) {
 
 /// \brief Emit the debug_str section stored in \p Pool.
 void DwarfStreamer::emitStrings(const NonRelocatableStringpool &Pool) {
-  Asm->OutStreamer.SwitchSection(MOFI->getDwarfStrSection());
+  Asm->OutStreamer->SwitchSection(MOFI->getDwarfStrSection());
   for (auto *Entry = Pool.getFirstEntry(); Entry;
        Entry = Pool.getNextEntry(Entry))
-    Asm->OutStreamer.EmitBytes(
+    Asm->OutStreamer->EmitBytes(
         StringRef(Entry->getKey().data(), Entry->getKey().size() + 1));
 }
 
@@ -668,13 +848,13 @@ void DwarfStreamer::emitUnitRangesEntries(CompileUnit &Unit,
     unsigned Padding = OffsetToAlignment(HeaderSize, TupleSize);
 
     Asm->EmitLabelDifference(EndLabel, BeginLabel, 4); // Arange length
-    Asm->OutStreamer.EmitLabel(BeginLabel);
+    Asm->OutStreamer->EmitLabel(BeginLabel);
     Asm->EmitInt16(dwarf::DW_ARANGES_VERSION); // Version number
     Asm->EmitInt32(Unit.getStartOffset());     // Corresponding unit's offset
     Asm->EmitInt8(AddressSize);                // Address size
     Asm->EmitInt8(0);                          // Segment size
 
-    Asm->OutStreamer.EmitFill(Padding, 0x0);
+    Asm->OutStreamer->EmitFill(Padding, 0x0);
 
     for (auto Range = Ranges.begin(), End = Ranges.end(); Range != End;
          ++Range) {
@@ -686,9 +866,9 @@ void DwarfStreamer::emitUnitRangesEntries(CompileUnit &Unit,
     }
 
     // Emit terminator
-    Asm->OutStreamer.EmitIntValue(0, AddressSize);
-    Asm->OutStreamer.EmitIntValue(0, AddressSize);
-    Asm->OutStreamer.EmitLabel(EndLabel);
+    Asm->OutStreamer->EmitIntValue(0, AddressSize);
+    Asm->OutStreamer->EmitIntValue(0, AddressSize);
+    Asm->OutStreamer->EmitLabel(EndLabel);
   }
 
   if (!DoDebugRanges)
@@ -716,8 +896,7 @@ void DwarfStreamer::emitUnitRangesEntries(CompileUnit &Unit,
 /// point to the new entries.
 void DwarfStreamer::emitLocationsForUnit(const CompileUnit &Unit,
                                          DWARFContext &Dwarf) {
-  const std::vector<std::pair<DIEInteger *, int64_t>> &Attributes =
-      Unit.getLocationAttributes();
+  const auto &Attributes = Unit.getLocationAttributes();
 
   if (Attributes.empty())
     return;
@@ -728,7 +907,7 @@ void DwarfStreamer::emitLocationsForUnit(const CompileUnit &Unit,
   const DWARFSection &InputSec = Dwarf.getLocSection();
   DataExtractor Data(InputSec.Data, Dwarf.isLittleEndian(), AddressSize);
   DWARFUnit &OrigUnit = Unit.getOrigUnit();
-  const auto *OrigUnitDie = OrigUnit.getCompileUnitDIE(false);
+  const auto *OrigUnitDie = OrigUnit.getUnitDIE(false);
   int64_t UnitPcOffset = 0;
   uint64_t OrigLowPc = OrigUnitDie->getAttributeValueAsAddress(
       &OrigUnit, dwarf::DW_AT_low_pc, -1ULL);
@@ -736,8 +915,8 @@ void DwarfStreamer::emitLocationsForUnit(const CompileUnit &Unit,
     UnitPcOffset = int64_t(OrigLowPc) - Unit.getLowPc();
 
   for (const auto &Attr : Attributes) {
-    uint32_t Offset = Attr.first->getValue();
-    Attr.first->setValue(LocSectionSize);
+    uint32_t Offset = Attr.first.get();
+    Attr.first.set(LocSectionSize);
     // This is the quantity to add to the old location address to get
     // the correct address for the new one.
     int64_t LocPcOffset = Attr.second + UnitPcOffset;
@@ -746,16 +925,16 @@ void DwarfStreamer::emitLocationsForUnit(const CompileUnit &Unit,
       uint64_t High = Data.getUnsigned(&Offset, AddressSize);
       LocSectionSize += 2 * AddressSize;
       if (Low == 0 && High == 0) {
-        Asm->OutStreamer.EmitIntValue(0, AddressSize);
-        Asm->OutStreamer.EmitIntValue(0, AddressSize);
+        Asm->OutStreamer->EmitIntValue(0, AddressSize);
+        Asm->OutStreamer->EmitIntValue(0, AddressSize);
         break;
       }
-      Asm->OutStreamer.EmitIntValue(Low + LocPcOffset, AddressSize);
-      Asm->OutStreamer.EmitIntValue(High + LocPcOffset, AddressSize);
+      Asm->OutStreamer->EmitIntValue(Low + LocPcOffset, AddressSize);
+      Asm->OutStreamer->EmitIntValue(High + LocPcOffset, AddressSize);
       uint64_t Length = Data.getU16(&Offset);
-      Asm->OutStreamer.EmitIntValue(Length, 2);
+      Asm->OutStreamer->EmitIntValue(Length, 2);
       // Just copy the bytes over.
-      Asm->OutStreamer.EmitBytes(
+      Asm->OutStreamer->EmitBytes(
           StringRef(InputSec.Data.substr(Offset, Length)));
       Offset += Length;
       LocSectionSize += Length + 2;
@@ -769,13 +948,13 @@ void DwarfStreamer::emitLineTableForUnit(StringRef PrologueBytes,
                                          unsigned PointerSize) {
   // Switch to the section where the table will be emitted into.
   MS->SwitchSection(MC->getObjectFileInfo()->getDwarfLineSection());
-  MCSymbol *LineStartSym = MC->CreateTempSymbol();
-  MCSymbol *LineEndSym = MC->CreateTempSymbol();
+  MCSymbol *LineStartSym = MC->createTempSymbol();
+  MCSymbol *LineEndSym = MC->createTempSymbol();
 
   // The first 4 bytes is the total length of the information for this
   // compilation unit (not including these 4 bytes for the length).
   Asm->EmitLabelDifference(LineEndSym, LineStartSym, 4);
-  Asm->OutStreamer.EmitLabel(LineStartSym);
+  Asm->OutStreamer->EmitLabel(LineStartSym);
   // Copy Prologue.
   MS->EmitBytes(PrologueBytes);
   LineSectionSize += PrologueBytes.size() + 4;
@@ -912,13 +1091,13 @@ void DwarfStreamer::emitLineTableForUnit(StringRef PrologueBytes,
 /// \brief Emit the pubnames or pubtypes section contribution for \p
 /// Unit into \p Sec. The data is provided in \p Names.
 void DwarfStreamer::emitPubSectionForUnit(
-    const MCSection *Sec, StringRef SecName, const CompileUnit &Unit,
+    MCSection *Sec, StringRef SecName, const CompileUnit &Unit,
     const std::vector<CompileUnit::AccelInfo> &Names) {
   if (Names.empty())
     return;
 
   // Start the dwarf pubnames section.
-  Asm->OutStreamer.SwitchSection(Sec);
+  Asm->OutStreamer->SwitchSection(Sec);
   MCSymbol *BeginLabel = Asm->createTempSymbol("pub" + SecName + "_begin");
   MCSymbol *EndLabel = Asm->createTempSymbol("pub" + SecName + "_end");
 
@@ -931,21 +1110,21 @@ void DwarfStreamer::emitPubSectionForUnit(
     if (!HeaderEmitted) {
       // Emit the header.
       Asm->EmitLabelDifference(EndLabel, BeginLabel, 4); // Length
-      Asm->OutStreamer.EmitLabel(BeginLabel);
+      Asm->OutStreamer->EmitLabel(BeginLabel);
       Asm->EmitInt16(dwarf::DW_PUBNAMES_VERSION); // Version
-      Asm->EmitInt32(Unit.getStartOffset()); // Unit offset
+      Asm->EmitInt32(Unit.getStartOffset());      // Unit offset
       Asm->EmitInt32(Unit.getNextUnitOffset() - Unit.getStartOffset()); // Size
       HeaderEmitted = true;
     }
     Asm->EmitInt32(Name.Die->getOffset());
-    Asm->OutStreamer.EmitBytes(
+    Asm->OutStreamer->EmitBytes(
         StringRef(Name.Name.data(), Name.Name.size() + 1));
   }
 
   if (!HeaderEmitted)
     return;
   Asm->EmitInt32(0); // End marker.
-  Asm->OutStreamer.EmitLabel(EndLabel);
+  Asm->OutStreamer->EmitLabel(EndLabel);
 }
 
 /// \brief Emit .debug_pubnames for \p Unit.
@@ -960,6 +1139,28 @@ void DwarfStreamer::emitPubTypesForUnit(const CompileUnit &Unit) {
                         "types", Unit, Unit.getPubtypes());
 }
 
+/// \brief Emit a CIE into the debug_frame section.
+void DwarfStreamer::emitCIE(StringRef CIEBytes) {
+  MS->SwitchSection(MC->getObjectFileInfo()->getDwarfFrameSection());
+
+  MS->EmitBytes(CIEBytes);
+  FrameSectionSize += CIEBytes.size();
+}
+
+/// \brief Emit a FDE into the debug_frame section. \p FDEBytes
+/// contains the FDE data without the length, CIE offset and address
+/// which will be replaced with the paramter values.
+void DwarfStreamer::emitFDE(uint32_t CIEOffset, uint32_t AddrSize,
+                            uint32_t Address, StringRef FDEBytes) {
+  MS->SwitchSection(MC->getObjectFileInfo()->getDwarfFrameSection());
+
+  MS->EmitIntValue(FDEBytes.size() + 4 + AddrSize, 4);
+  MS->EmitIntValue(CIEOffset, 4);
+  MS->EmitIntValue(Address, AddrSize);
+  MS->EmitBytes(FDEBytes);
+  FrameSectionSize += FDEBytes.size() + 8 + AddrSize;
+}
+
 /// \brief The core of the Dwarf linking logic.
 ///
 /// The link of the dwarf information from the object files will be
@@ -978,7 +1179,7 @@ class DwarfLinker {
 public:
   DwarfLinker(StringRef OutputFilename, const LinkOptions &Options)
       : OutputFilename(OutputFilename), Options(Options),
-        BinHolder(Options.Verbose) {}
+        BinHolder(Options.Verbose), LastCIEOffset(0) {}
 
   ~DwarfLinker() {
     for (auto *Abbrev : Abbreviations)
@@ -1050,6 +1251,7 @@ class DwarfLinker {
     TF_InFunctionScope = 1 << 1, ///< Current scope is a fucntion scope.
     TF_DependencyWalk = 1 << 2,  ///< Walking the dependencies of a kept DIE.
     TF_ParentWalk = 1 << 3,      ///< Walking up the parents of a kept DIE.
+    TF_ODR = 1 << 4,             ///< Use the ODR whhile keeping dependants.
   };
 
   /// \brief Mark the passed DIE as well as all the ones it depends on
@@ -1057,7 +1259,7 @@ class DwarfLinker {
   void keepDIEAndDenpendencies(const DWARFDebugInfoEntryMinimal &DIE,
                                CompileUnit::DIEInfo &MyInfo,
                                const DebugMapObject &DMO, CompileUnit &CU,
-                               unsigned Flags);
+                               bool UseODR);
 
   unsigned shouldKeepDIE(const DWARFDebugInfoEntryMinimal &DIE,
                          CompileUnit &Unit, CompileUnit::DIEInfo &MyInfo,
@@ -1176,6 +1378,10 @@ class DwarfLinker {
   /// \brief Emit the accelerator entries for \p Unit.
   void emitAcceleratorEntriesForUnit(CompileUnit &Unit);
 
+  /// \brief Patch the frame info for an object file and emit it.
+  void patchFrameInfoForObject(const DebugMapObject &, DWARFContext &,
+                               unsigned AddressSize);
+
   /// \brief DIELoc objects that need to be destructed (but not freed!).
   std::vector<DIELoc *> DIELocs;
   /// \brief DIEBlock objects that need to be destructed (but not freed!).
@@ -1184,11 +1390,14 @@ class DwarfLinker {
   BumpPtrAllocator DIEAlloc;
   /// @}
 
+  /// ODR Contexts for that link.
+  DeclContextTree ODRContexts;
+
   /// \defgroup Helpers Various helper methods.
   ///
   /// @{
   const DWARFDebugInfoEntryMinimal *
-  resolveDIEReference(DWARFFormValue &RefValue, const DWARFUnit &Unit,
+  resolveDIEReference(const DWARFFormValue &RefValue, const DWARFUnit &Unit,
                       const DWARFDebugInfoEntryMinimal &DIE,
                       CompileUnit *&ReferencedCU);
 
@@ -1201,6 +1410,11 @@ class DwarfLinker {
                      const DWARFDebugInfoEntryMinimal *DIE = nullptr) const;
 
   bool createStreamer(Triple TheTriple, StringRef OutputFilename);
+
+  /// \brief Attempt to load a debug object from disk.
+  ErrorOr<const object::ObjectFile &> loadObject(BinaryHolder &BinaryHolder,
+                                                 DebugMapObject &Obj,
+                                                 const DebugMap &Map);
   /// @}
 
 private:
@@ -1225,6 +1439,16 @@ class DwarfLinker {
   ///
   /// See startDebugObject() for a more complete description of its use.
   std::map<uint64_t, std::pair<uint64_t, int64_t>> Ranges;
+
+  /// \brief The CIEs that have been emitted in the output
+  /// section. The actual CIE data serves a the key to this StringMap,
+  /// this takes care of comparing the semantics of CIEs defined in
+  /// different object files.
+  StringMap<uint32_t> EmittedCIEs;
+
+  /// Offset of the last CIE that has been emitted in the output
+  /// debug_frame section.
+  uint32_t LastCIEOffset;
 };
 
 /// \brief Similar to DWARFUnitSection::getUnitForOffset(), but
@@ -1243,7 +1467,7 @@ CompileUnit *DwarfLinker::getUnitForOffset(unsigned Offset) {
 /// CompileUnit which is stored into \p ReferencedCU.
 /// \returns null if resolving fails for any reason.
 const DWARFDebugInfoEntryMinimal *DwarfLinker::resolveDIEReference(
-    DWARFFormValue &RefValue, const DWARFUnit &Unit,
+    const DWARFFormValue &RefValue, const DWARFUnit &Unit,
     const DWARFDebugInfoEntryMinimal &DIE, CompileUnit *&RefCU) {
   assert(RefValue.isFormClass(DWARFFormValue::FC_Reference));
   uint64_t RefOffset = *RefValue.getAsReference(&Unit);
@@ -1256,6 +1480,220 @@ const DWARFDebugInfoEntryMinimal *DwarfLinker::resolveDIEReference(
   return nullptr;
 }
 
+/// \returns whether the passed \a Attr type might contain a DIE
+/// reference suitable for ODR uniquing.
+static bool isODRAttribute(uint16_t Attr) {
+  switch (Attr) {
+  default:
+    return false;
+  case dwarf::DW_AT_type:
+  case dwarf::DW_AT_containing_type:
+  case dwarf::DW_AT_specification:
+  case dwarf::DW_AT_abstract_origin:
+  case dwarf::DW_AT_import:
+    return true;
+  }
+  llvm_unreachable("Improper attribute.");
+}
+
+/// Set the last DIE/CU a context was seen in and, possibly invalidate
+/// the context if it is ambiguous.
+///
+/// In the current implementation, we don't handle overloaded
+/// functions well, because the argument types are not taken into
+/// account when computing the DeclContext tree.
+///
+/// Some of this is mitigated byt using mangled names that do contain
+/// the arguments types, but sometimes (eg. with function templates)
+/// we don't have that. In that case, just do not unique anything that
+/// refers to the contexts we are not able to distinguish.
+///
+/// If a context that is not a namespace appears twice in the same CU,
+/// we know it is ambiguous. Make it invalid.
+bool DeclContext::setLastSeenDIE(CompileUnit &U,
+                                 const DWARFDebugInfoEntryMinimal *Die) {
+  if (LastSeenCompileUnitID == U.getUniqueID()) {
+    DWARFUnit &OrigUnit = U.getOrigUnit();
+    uint32_t FirstIdx = OrigUnit.getDIEIndex(LastSeenDIE);
+    U.getInfo(FirstIdx).Ctxt = nullptr;
+    return false;
+  }
+
+  LastSeenCompileUnitID = U.getUniqueID();
+  LastSeenDIE = Die;
+  return true;
+}
+
+/// Get the child context of \a Context corresponding to \a DIE.
+///
+/// \returns the child context or null if we shouldn't track children
+/// contexts. It also returns an additional bit meaning 'invalid'. An
+/// invalid context means it shouldn't be considered for uniquing, but
+/// its not returning null, because some children of that context
+/// might be uniquing candidates.
+/// FIXME: this is for dsymutil-classic compatibility, I don't think
+/// it buys us much.
+PointerIntPair<DeclContext *, 1> DeclContextTree::getChildDeclContext(
+    DeclContext &Context, const DWARFDebugInfoEntryMinimal *DIE, CompileUnit &U,
+    NonRelocatableStringpool &StringPool) {
+  unsigned Tag = DIE->getTag();
+
+  // FIXME: dsymutil-classic compat: We should bail out here if we
+  // have a specification or an abstract_origin. We will get the
+  // parent context wrong here.
+
+  switch (Tag) {
+  default:
+    // By default stop gathering child contexts.
+    return PointerIntPair<DeclContext *, 1>(nullptr);
+  case dwarf::DW_TAG_compile_unit:
+    // FIXME: Add support for DW_TAG_module.
+    return PointerIntPair<DeclContext *, 1>(&Context);
+  case dwarf::DW_TAG_subprogram:
+    // Do not unique anything inside CU local functions.
+    if ((Context.getTag() == dwarf::DW_TAG_namespace ||
+         Context.getTag() == dwarf::DW_TAG_compile_unit) &&
+        !DIE->getAttributeValueAsUnsignedConstant(&U.getOrigUnit(),
+                                                  dwarf::DW_AT_external, 0))
+      return PointerIntPair<DeclContext *, 1>(nullptr);
+  // Fallthrough
+  case dwarf::DW_TAG_member:
+  case dwarf::DW_TAG_namespace:
+  case dwarf::DW_TAG_structure_type:
+  case dwarf::DW_TAG_class_type:
+  case dwarf::DW_TAG_union_type:
+  case dwarf::DW_TAG_enumeration_type:
+  case dwarf::DW_TAG_typedef:
+    // Artificial things might be ambiguous, because they might be
+    // created on demand. For example implicitely defined constructors
+    // are ambiguous because of the way we identify contexts, and they
+    // won't be generated everytime everywhere.
+    if (DIE->getAttributeValueAsUnsignedConstant(&U.getOrigUnit(),
+                                                 dwarf::DW_AT_artificial, 0))
+      return PointerIntPair<DeclContext *, 1>(nullptr);
+    break;
+  }
+
+  const char *Name = DIE->getName(&U.getOrigUnit(), DINameKind::LinkageName);
+  const char *ShortName = DIE->getName(&U.getOrigUnit(), DINameKind::ShortName);
+  StringRef NameRef;
+  StringRef ShortNameRef;
+  StringRef FileRef;
+
+  if (Name)
+    NameRef = StringPool.internString(Name);
+  else if (Tag == dwarf::DW_TAG_namespace)
+    // FIXME: For dsymutil-classic compatibility. I think uniquing
+    // within anonymous namespaces is wrong. There is no ODR guarantee
+    // there.
+    NameRef = StringPool.internString("(anonymous namespace)");
+
+  if (ShortName && ShortName != Name)
+    ShortNameRef = StringPool.internString(ShortName);
+  else
+    ShortNameRef = NameRef;
+
+  if (Tag != dwarf::DW_TAG_class_type && Tag != dwarf::DW_TAG_structure_type &&
+      Tag != dwarf::DW_TAG_union_type &&
+      Tag != dwarf::DW_TAG_enumeration_type && NameRef.empty())
+    return PointerIntPair<DeclContext *, 1>(nullptr);
+
+  std::string File;
+  unsigned Line = 0;
+  unsigned ByteSize = 0;
+
+  // Gather some discriminating data about the DeclContext we will be
+  // creating: File, line number and byte size. This shouldn't be
+  // necessary, because the ODR is just about names, but given that we
+  // do some approximations with overloaded functions and anonymous
+  // namespaces, use these additional data points to make the process safer.
+  ByteSize = DIE->getAttributeValueAsUnsignedConstant(
+      &U.getOrigUnit(), dwarf::DW_AT_byte_size, UINT64_MAX);
+  if (Tag != dwarf::DW_TAG_namespace || !Name) {
+    if (unsigned FileNum = DIE->getAttributeValueAsUnsignedConstant(
+            &U.getOrigUnit(), dwarf::DW_AT_decl_file, 0)) {
+      if (const auto *LT = U.getOrigUnit().getContext().getLineTableForUnit(
+              &U.getOrigUnit())) {
+        // FIXME: dsymutil-classic compatibility. I'd rather not
+        // unique anything in anonymous namespaces, but if we do, then
+        // verify that the file and line correspond.
+        if (!Name && Tag == dwarf::DW_TAG_namespace)
+          FileNum = 1;
+
+        // FIXME: Passing U.getOrigUnit().getCompilationDir()
+        // instead of "" would allow more uniquing, but for now, do
+        // it this way to match dsymutil-classic.
+        if (LT->getFileNameByIndex(
+                FileNum, "",
+                DILineInfoSpecifier::FileLineInfoKind::AbsoluteFilePath,
+                File)) {
+          Line = DIE->getAttributeValueAsUnsignedConstant(
+              &U.getOrigUnit(), dwarf::DW_AT_decl_line, 0);
+#ifdef HAVE_REALPATH
+          // Cache the resolved paths, because calling realpath is expansive.
+          if (const char *ResolvedPath = U.getResolvedPath(FileNum)) {
+            File = ResolvedPath;
+          } else {
+            char RealPath[PATH_MAX + 1];
+            RealPath[PATH_MAX] = 0;
+            if (::realpath(File.c_str(), RealPath))
+              File = RealPath;
+            U.setResolvedPath(FileNum, File);
+          }
+#endif
+          FileRef = StringPool.internString(File);
+        }
+      }
+    }
+  }
+
+  if (!Line && NameRef.empty())
+    return PointerIntPair<DeclContext *, 1>(nullptr);
+
+  // FIXME: dsymutil-classic compat won't unique the same type
+  // presented once as a struct and once as a class. Use the Tag in
+  // the fully qualified name hash to get the same effect.
+  // We hash NameRef, which is the mangled name, in order to get most
+  // overloaded functions resolvec correctly.
+  unsigned Hash = hash_combine(Context.getQualifiedNameHash(), Tag, NameRef);
+
+  // FIXME: dsymutil-classic compatibility: when we don't have a name,
+  // use the filename.
+  if (Tag == dwarf::DW_TAG_namespace && NameRef == "(anonymous namespace)")
+    Hash = hash_combine(Hash, FileRef);
+
+  // Now look if this context already exists.
+  DeclContext Key(Hash, Line, ByteSize, Tag, NameRef, FileRef, Context);
+  auto ContextIter = Contexts.find(&Key);
+
+  if (ContextIter == Contexts.end()) {
+    // The context wasn't found.
+    bool Inserted;
+    DeclContext *NewContext =
+        new (Allocator) DeclContext(Hash, Line, ByteSize, Tag, NameRef, FileRef,
+                                    Context, DIE, U.getUniqueID());
+    std::tie(ContextIter, Inserted) = Contexts.insert(NewContext);
+    assert(Inserted && "Failed to insert DeclContext");
+    (void)Inserted;
+  } else if (Tag != dwarf::DW_TAG_namespace &&
+             !(*ContextIter)->setLastSeenDIE(U, DIE)) {
+    // The context was found, but it is ambiguous with another context
+    // in the same file. Mark it invalid.
+    return PointerIntPair<DeclContext *, 1>(*ContextIter, /* Invalid= */ 1);
+  }
+
+  assert(ContextIter != Contexts.end());
+  // FIXME: dsymutil-classic compatibility. Union types aren't
+  // uniques, but their children might be.
+  if ((Tag == dwarf::DW_TAG_subprogram &&
+       Context.getTag() != dwarf::DW_TAG_structure_type &&
+       Context.getTag() != dwarf::DW_TAG_class_type) ||
+      (Tag == dwarf::DW_TAG_union_type))
+    return PointerIntPair<DeclContext *, 1>(*ContextIter, /* Invalid= */ 1);
+
+  return PointerIntPair<DeclContext *, 1>(*ContextIter);
+}
+
 /// \brief Get the potential name and mangled name for the entity
 /// described by \p Die and store them in \Info if they are not
 /// already there.
@@ -1302,14 +1740,30 @@ bool DwarfLinker::createStreamer(Triple TheTriple, StringRef OutputFilename) {
 /// \brief Recursive helper to gather the child->parent relationships in the
 /// original compile unit.
 static void gatherDIEParents(const DWARFDebugInfoEntryMinimal *DIE,
-                             unsigned ParentIdx, CompileUnit &CU) {
+                             unsigned ParentIdx, CompileUnit &CU,
+                             DeclContext *CurrentDeclContext,
+                             NonRelocatableStringpool &StringPool,
+                             DeclContextTree &Contexts) {
   unsigned MyIdx = CU.getOrigUnit().getDIEIndex(DIE);
-  CU.getInfo(MyIdx).ParentIdx = ParentIdx;
+  CompileUnit::DIEInfo &Info = CU.getInfo(MyIdx);
+
+  Info.ParentIdx = ParentIdx;
+  if (CU.hasODR()) {
+    if (CurrentDeclContext) {
+      auto PtrInvalidPair = Contexts.getChildDeclContext(*CurrentDeclContext,
+                                                         DIE, CU, StringPool);
+      CurrentDeclContext = PtrInvalidPair.getPointer();
+      Info.Ctxt =
+          PtrInvalidPair.getInt() ? nullptr : PtrInvalidPair.getPointer();
+    } else
+      Info.Ctxt = CurrentDeclContext = nullptr;
+  }
 
   if (DIE->hasChildren())
     for (auto *Child = DIE->getFirstChild(); Child && !Child->isNULL();
          Child = Child->getSibling())
-      gatherDIEParents(Child, MyIdx, CU);
+      gatherDIEParents(Child, MyIdx, CU, CurrentDeclContext, StringPool,
+                       Contexts);
 }
 
 static bool dieNeedsChildrenToBeMeaningful(uint32_t Tag) {
@@ -1327,6 +1781,12 @@ static bool dieNeedsChildrenToBeMeaningful(uint32_t Tag) {
   llvm_unreachable("Invalid Tag");
 }
 
+static unsigned getRefAddrSize(const DWARFUnit &U) {
+  if (U.getVersion() == 2)
+    return U.getAddressByteSize();
+  return 4;
+}
+
 void DwarfLinker::startDebugObject(DWARFContext &Dwarf, DebugMapObject &Obj) {
   Units.reserve(Dwarf.getNumCompileUnits());
   NextValidReloc = 0;
@@ -1358,10 +1818,10 @@ void DwarfLinker::endDebugObject() {
   ValidRelocs.clear();
   Ranges.clear();
 
-  for (auto *Block : DIEBlocks)
-    Block->~DIEBlock();
-  for (auto *Loc : DIELocs)
-    Loc->~DIELoc();
+  for (auto I = DIEBlocks.begin(), E = DIEBlocks.end(); I != E; ++I)
+    (*I)->~DIEBlock();
+  for (auto I = DIELocs.begin(), E = DIELocs.end(); I != E; ++I)
+    (*I)->~DIELoc();
 
   DIEBlocks.clear();
   DIELocs.clear();
@@ -1382,8 +1842,8 @@ void DwarfLinker::findValidRelocsMachO(const object::SectionRef &Section,
     object::DataRefImpl RelocDataRef = Reloc.getRawDataRefImpl();
     MachO::any_relocation_info MachOReloc = Obj.getRelocation(RelocDataRef);
     unsigned RelocSize = 1 << Obj.getAnyRelocationLength(MachOReloc);
-    uint64_t Offset64;
-    if ((RelocSize != 4 && RelocSize != 8) || Reloc.getOffset(Offset64)) {
+    uint64_t Offset64 = Reloc.getOffset();
+    if ((RelocSize != 4 && RelocSize != 8)) {
       reportWarning(" unsupported relocation in debug_info section.");
       continue;
     }
@@ -1393,12 +1853,12 @@ void DwarfLinker::findValidRelocsMachO(const object::SectionRef &Section,
 
     auto Sym = Reloc.getSymbol();
     if (Sym != Obj.symbol_end()) {
-      StringRef SymbolName;
-      if (Sym->getName(SymbolName)) {
+      ErrorOr<StringRef> SymbolName = Sym->getName();
+      if (!SymbolName) {
         reportWarning("error getting relocation symbol name.");
         continue;
       }
-      if (const auto *Mapping = DMO.lookupSymbol(SymbolName))
+      if (const auto *Mapping = DMO.lookupSymbol(*SymbolName))
         ValidRelocs.emplace_back(Offset64, RelocSize, Addend, Mapping);
     } else if (const auto *Mapping = DMO.lookupObjectAddress(Addend)) {
       // Do not store the addend. The addend was the address of the
@@ -1476,15 +1936,15 @@ bool DwarfLinker::hasValidRelocation(uint32_t StartOffset, uint32_t EndOffset,
     return false;
 
   const auto &ValidReloc = ValidRelocs[NextValidReloc++];
+  const auto &Mapping = ValidReloc.Mapping->getValue();
   if (Options.Verbose)
     outs() << "Found valid debug map entry: " << ValidReloc.Mapping->getKey()
            << " " << format("\t%016" PRIx64 " => %016" PRIx64,
-                            ValidReloc.Mapping->getValue().ObjectAddress,
-                            ValidReloc.Mapping->getValue().BinaryAddress);
+                            uint64_t(Mapping.ObjectAddress),
+                            uint64_t(Mapping.BinaryAddress));
 
-  Info.AddrAdjust = int64_t(ValidReloc.Mapping->getValue().BinaryAddress) +
-                    ValidReloc.Addend -
-                    ValidReloc.Mapping->getValue().ObjectAddress;
+  Info.AddrAdjust = int64_t(Mapping.BinaryAddress) + ValidReloc.Addend -
+                    Mapping.ObjectAddress;
   Info.InDebugMap = true;
   return true;
 }
@@ -1633,15 +2093,16 @@ unsigned DwarfLinker::shouldKeepDIE(const DWARFDebugInfoEntryMinimal &DIE,
 void DwarfLinker::keepDIEAndDenpendencies(const DWARFDebugInfoEntryMinimal &DIE,
                                           CompileUnit::DIEInfo &MyInfo,
                                           const DebugMapObject &DMO,
-                                          CompileUnit &CU, unsigned Flags) {
+                                          CompileUnit &CU, bool UseODR) {
   const DWARFUnit &Unit = CU.getOrigUnit();
   MyInfo.Keep = true;
 
   // First mark all the parent chain as kept.
   unsigned AncestorIdx = MyInfo.ParentIdx;
   while (!CU.getInfo(AncestorIdx).Keep) {
+    unsigned ODRFlag = UseODR ? TF_ODR : 0;
     lookForDIEsToKeep(*Unit.getDIEAtIndex(AncestorIdx), DMO, CU,
-                      TF_ParentWalk | TF_Keep | TF_DependencyWalk);
+                      TF_ParentWalk | TF_Keep | TF_DependencyWalk | ODRFlag);
     AncestorIdx = CU.getInfo(AncestorIdx).ParentIdx;
   }
 
@@ -1662,9 +2123,27 @@ void DwarfLinker::keepDIEAndDenpendencies(const DWARFDebugInfoEntryMinimal &DIE,
 
     Val.extractValue(Data, &Offset, &Unit);
     CompileUnit *ReferencedCU;
-    if (const auto *RefDIE = resolveDIEReference(Val, Unit, DIE, ReferencedCU))
+    if (const auto *RefDIE =
+            resolveDIEReference(Val, Unit, DIE, ReferencedCU)) {
+      uint32_t RefIdx = ReferencedCU->getOrigUnit().getDIEIndex(RefDIE);
+      CompileUnit::DIEInfo &Info = ReferencedCU->getInfo(RefIdx);
+      // If the referenced DIE has a DeclContext that has already been
+      // emitted, then do not keep the one in this CU. We'll link to
+      // the canonical DIE in cloneDieReferenceAttribute.
+      // FIXME: compatibility with dsymutil-classic. UseODR shouldn't
+      // be necessary and could be advantageously replaced by
+      // ReferencedCU->hasODR() && CU.hasODR().
+      // FIXME: compatibility with dsymutil-classic. There is no
+      // reason not to unique ref_addr references.
+      if (AttrSpec.Form != dwarf::DW_FORM_ref_addr && UseODR && Info.Ctxt &&
+          Info.Ctxt != ReferencedCU->getInfo(Info.ParentIdx).Ctxt &&
+          Info.Ctxt->getCanonicalDIEOffset() && isODRAttribute(AttrSpec.Attr))
+        continue;
+
+      unsigned ODRFlag = UseODR ? TF_ODR : 0;
       lookForDIEsToKeep(*RefDIE, DMO, *ReferencedCU,
-                        TF_Keep | TF_DependencyWalk);
+                        TF_Keep | TF_DependencyWalk | ODRFlag);
+    }
   }
 }
 
@@ -1699,9 +2178,10 @@ void DwarfLinker::lookForDIEsToKeep(const DWARFDebugInfoEntryMinimal &DIE,
     Flags = shouldKeepDIE(DIE, CU, MyInfo, Flags);
 
   // If it is a newly kept DIE mark it as well as all its dependencies as kept.
-  if (!AlreadyKept && (Flags & TF_Keep))
-    keepDIEAndDenpendencies(DIE, MyInfo, DMO, CU, Flags);
-
+  if (!AlreadyKept && (Flags & TF_Keep)) {
+    bool UseOdr = (Flags & TF_DependencyWalk) ? (Flags & TF_ODR) : CU.hasODR();
+    keepDIEAndDenpendencies(DIE, MyInfo, DMO, CU, UseOdr);
+  }
   // The TF_ParentWalk flag tells us that we are currently walking up
   // the parent chain of a required DIE, and we don't want to mark all
   // the children of the parents as kept (consider for example a
@@ -1758,8 +2238,8 @@ unsigned DwarfLinker::cloneStringAttribute(DIE &Die, AttributeSpec AttrSpec,
   // Switch everything to out of line strings.
   const char *String = *Val.getAsCString(&U);
   unsigned Offset = StringPool.getStringOffset(String);
-  Die.addValue(dwarf::Attribute(AttrSpec.Attr), dwarf::DW_FORM_strp,
-               new (DIEAlloc) DIEInteger(Offset));
+  Die.addValue(DIEAlloc, dwarf::Attribute(AttrSpec.Attr), dwarf::DW_FORM_strp,
+               DIEInteger(Offset));
   return 4;
 }
 
@@ -1770,56 +2250,70 @@ unsigned DwarfLinker::cloneDieReferenceAttribute(
     DIE &Die, const DWARFDebugInfoEntryMinimal &InputDIE,
     AttributeSpec AttrSpec, unsigned AttrSize, const DWARFFormValue &Val,
     CompileUnit &Unit) {
-  uint32_t Ref = *Val.getAsReference(&Unit.getOrigUnit());
+  const DWARFUnit &U = Unit.getOrigUnit();
+  uint32_t Ref = *Val.getAsReference(&U);
   DIE *NewRefDie = nullptr;
   CompileUnit *RefUnit = nullptr;
-  const DWARFDebugInfoEntryMinimal *RefDie = nullptr;
-
-  if (!(RefUnit = getUnitForOffset(Ref)) ||
-      !(RefDie = RefUnit->getOrigUnit().getDIEForOffset(Ref))) {
-    const char *AttributeString = dwarf::AttributeString(AttrSpec.Attr);
-    if (!AttributeString)
-      AttributeString = "DW_AT_???";
-    reportWarning(Twine("Missing DIE for ref in attribute ") + AttributeString +
-                      ". Dropping.",
-                  &Unit.getOrigUnit(), &InputDIE);
+  DeclContext *Ctxt = nullptr;
+
+  const DWARFDebugInfoEntryMinimal *RefDie =
+      resolveDIEReference(Val, U, InputDIE, RefUnit);
+
+  // If the referenced DIE is not found,  drop the attribute.
+  if (!RefDie)
     return 0;
-  }
 
   unsigned Idx = RefUnit->getOrigUnit().getDIEIndex(RefDie);
   CompileUnit::DIEInfo &RefInfo = RefUnit->getInfo(Idx);
+
+  // If we already have emitted an equivalent DeclContext, just point
+  // at it.
+  if (isODRAttribute(AttrSpec.Attr)) {
+    Ctxt = RefInfo.Ctxt;
+    if (Ctxt && Ctxt->getCanonicalDIEOffset()) {
+      DIEInteger Attr(Ctxt->getCanonicalDIEOffset());
+      Die.addValue(DIEAlloc, dwarf::Attribute(AttrSpec.Attr),
+                   dwarf::DW_FORM_ref_addr, Attr);
+      return getRefAddrSize(U);
+    }
+  }
+
   if (!RefInfo.Clone) {
     assert(Ref > InputDIE.getOffset());
     // We haven't cloned this DIE yet. Just create an empty one and
     // store it. It'll get really cloned when we process it.
-    RefInfo.Clone = new DIE(dwarf::Tag(RefDie->getTag()));
+    RefInfo.Clone = DIE::get(DIEAlloc, dwarf::Tag(RefDie->getTag()));
   }
   NewRefDie = RefInfo.Clone;
 
-  if (AttrSpec.Form == dwarf::DW_FORM_ref_addr) {
+  if (AttrSpec.Form == dwarf::DW_FORM_ref_addr ||
+      (Unit.hasODR() && isODRAttribute(AttrSpec.Attr))) {
     // We cannot currently rely on a DIEEntry to emit ref_addr
     // references, because the implementation calls back to DwarfDebug
     // to find the unit offset. (We don't have a DwarfDebug)
     // FIXME: we should be able to design DIEEntry reliance on
     // DwarfDebug away.
-    DIEInteger *Attr;
+    uint64_t Attr;
     if (Ref < InputDIE.getOffset()) {
       // We must have already cloned that DIE.
       uint32_t NewRefOffset =
           RefUnit->getStartOffset() + NewRefDie->getOffset();
-      Attr = new (DIEAlloc) DIEInteger(NewRefOffset);
+      Attr = NewRefOffset;
+      Die.addValue(DIEAlloc, dwarf::Attribute(AttrSpec.Attr),
+                   dwarf::DW_FORM_ref_addr, DIEInteger(Attr));
     } else {
       // A forward reference. Note and fixup later.
-      Attr = new (DIEAlloc) DIEInteger(0xBADDEF);
-      Unit.noteForwardReference(NewRefDie, RefUnit, Attr);
+      Attr = 0xBADDEF;
+      Unit.noteForwardReference(
+          NewRefDie, RefUnit, Ctxt,
+          Die.addValue(DIEAlloc, dwarf::Attribute(AttrSpec.Attr),
+                       dwarf::DW_FORM_ref_addr, DIEInteger(Attr)));
     }
-    Die.addValue(dwarf::Attribute(AttrSpec.Attr), dwarf::DW_FORM_ref_addr,
-                 Attr);
-    return AttrSize;
+    return getRefAddrSize(U);
   }
 
-  Die.addValue(dwarf::Attribute(AttrSpec.Attr), dwarf::Form(AttrSpec.Form),
-               new (DIEAlloc) DIEEntry(*NewRefDie));
+  Die.addValue(DIEAlloc, dwarf::Attribute(AttrSpec.Attr),
+               dwarf::Form(AttrSpec.Form), DIEEntry(*NewRefDie));
   return AttrSize;
 }
 
@@ -1830,23 +2324,29 @@ unsigned DwarfLinker::cloneBlockAttribute(DIE &Die, AttributeSpec AttrSpec,
                                           const DWARFFormValue &Val,
                                           unsigned AttrSize) {
   DIE *Attr;
-  DIEValue *Value;
+  DIEValue Value;
   DIELoc *Loc = nullptr;
   DIEBlock *Block = nullptr;
   // Just copy the block data over.
   if (AttrSpec.Form == dwarf::DW_FORM_exprloc) {
-    Loc = new (DIEAlloc) DIELoc();
+    Loc = new (DIEAlloc) DIELoc;
     DIELocs.push_back(Loc);
   } else {
-    Block = new (DIEAlloc) DIEBlock();
+    Block = new (DIEAlloc) DIEBlock;
     DIEBlocks.push_back(Block);
   }
   Attr = Loc ? static_cast<DIE *>(Loc) : static_cast<DIE *>(Block);
-  Value = Loc ? static_cast<DIEValue *>(Loc) : static_cast<DIEValue *>(Block);
+
+  if (Loc)
+    Value = DIEValue(dwarf::Attribute(AttrSpec.Attr),
+                     dwarf::Form(AttrSpec.Form), Loc);
+  else
+    Value = DIEValue(dwarf::Attribute(AttrSpec.Attr),
+                     dwarf::Form(AttrSpec.Form), Block);
   ArrayRef<uint8_t> Bytes = *Val.getAsBlock();
   for (auto Byte : Bytes)
-    Attr->addValue(static_cast<dwarf::Attribute>(0), dwarf::DW_FORM_data1,
-                   new (DIEAlloc) DIEInteger(Byte));
+    Attr->addValue(DIEAlloc, static_cast<dwarf::Attribute>(0),
+                   dwarf::DW_FORM_data1, DIEInteger(Byte));
   // FIXME: If DIEBlock and DIELoc just reuses the Size field of
   // the DIE class, this if could be replaced by
   // Attr->setSize(Bytes.size()).
@@ -1856,8 +2356,7 @@ unsigned DwarfLinker::cloneBlockAttribute(DIE &Die, AttributeSpec AttrSpec,
     else
       Block->ComputeSize(&Streamer->getAsmPrinter());
   }
-  Die.addValue(dwarf::Attribute(AttrSpec.Attr), dwarf::Form(AttrSpec.Form),
-               Value);
+  Die.addValue(DIEAlloc, Value);
   return AttrSize;
 }
 
@@ -1891,9 +2390,8 @@ unsigned DwarfLinker::cloneAddressAttribute(DIE &Die, AttributeSpec AttrSpec,
       Addr = (Info.OrigHighPc ? Info.OrigHighPc : Addr) + Info.PCOffset;
   }
 
-  Die.addValue(static_cast<dwarf::Attribute>(AttrSpec.Attr),
-               static_cast<dwarf::Form>(AttrSpec.Form),
-               new (DIEAlloc) DIEInteger(Addr));
+  Die.addValue(DIEAlloc, static_cast<dwarf::Attribute>(AttrSpec.Attr),
+               static_cast<dwarf::Form>(AttrSpec.Form), DIEInteger(Addr));
   return Unit.getOrigUnit().getAddressByteSize();
 }
 
@@ -1921,20 +2419,20 @@ unsigned DwarfLinker::cloneScalarAttribute(
                   &Unit.getOrigUnit(), &InputDIE);
     return 0;
   }
-  DIEInteger *Attr = new (DIEAlloc) DIEInteger(Value);
+  PatchLocation Patch =
+      Die.addValue(DIEAlloc, dwarf::Attribute(AttrSpec.Attr),
+                   dwarf::Form(AttrSpec.Form), DIEInteger(Value));
   if (AttrSpec.Attr == dwarf::DW_AT_ranges)
-    Unit.noteRangeAttribute(Die, Attr);
+    Unit.noteRangeAttribute(Die, Patch);
   // A more generic way to check for location attributes would be
   // nice, but it's very unlikely that any other attribute needs a
   // location list.
   else if (AttrSpec.Attr == dwarf::DW_AT_location ||
            AttrSpec.Attr == dwarf::DW_AT_frame_base)
-    Unit.noteLocationAttribute(Attr, Info.PCOffset);
+    Unit.noteLocationAttribute(Patch, Info.PCOffset);
   else if (AttrSpec.Attr == dwarf::DW_AT_declaration && Value)
     Info.IsDeclaration = true;
 
-  Die.addValue(dwarf::Attribute(AttrSpec.Attr), dwarf::Form(AttrSpec.Form),
-               Attr);
   return AttrSize;
 }
 
@@ -2087,9 +2585,17 @@ DIE *DwarfLinker::cloneDIE(const DWARFDebugInfoEntryMinimal &InputDIE,
   // (see cloneDieReferenceAttribute()).
   DIE *Die = Info.Clone;
   if (!Die)
-    Die = Info.Clone = new DIE(dwarf::Tag(InputDIE.getTag()));
+    Die = Info.Clone = DIE::get(DIEAlloc, dwarf::Tag(InputDIE.getTag()));
   assert(Die->getTag() == InputDIE.getTag());
   Die->setOffset(OutOffset);
+  if (Unit.hasODR() && Die->getTag() != dwarf::DW_TAG_namespace && Info.Ctxt &&
+      Info.Ctxt != Unit.getInfo(Info.ParentIdx).Ctxt &&
+      !Info.Ctxt->getCanonicalDIEOffset()) {
+    // We are about to emit a DIE that is the root of its own valid
+    // DeclContext tree. Make the current offset the canonical offset
+    // for this context.
+    Info.Ctxt->setCanonicalDIEOffset(OutOffset + Unit.getStartOffset());
+  }
 
   // Extract and clone every attribute.
   DataExtractor Data = U.getDebugInfoExtractor();
@@ -2156,14 +2662,15 @@ DIE *DwarfLinker::cloneDIE(const DWARFDebugInfoEntryMinimal &InputDIE,
     Unit.addTypeAccelerator(Die, AttrInfo.Name, AttrInfo.NameOffset);
   }
 
-  DIEAbbrev &NewAbbrev = Die->getAbbrev();
+  DIEAbbrev NewAbbrev = Die->generateAbbrev();
   // If a scope DIE is kept, we must have kept at least one child. If
   // it's not the case, we'll just be emitting one wasteful end of
   // children marker, but things won't break.
   if (InputDIE.hasChildren())
     NewAbbrev.setChildrenFlag(dwarf::DW_CHILDREN_yes);
   // Assign a permanent abbrev number
-  AssignAbbrev(Die->getAbbrev());
+  AssignAbbrev(NewAbbrev);
+  Die->setAbbrevNumber(NewAbbrev.getNumber());
 
   // Add the size of the abbreviation number to the output offset.
   OutOffset += getULEB128Size(Die->getAbbrevNumber());
@@ -2178,7 +2685,7 @@ DIE *DwarfLinker::cloneDIE(const DWARFDebugInfoEntryMinimal &InputDIE,
   for (auto *Child = InputDIE.getFirstChild(); Child && !Child->isNULL();
        Child = Child->getSibling()) {
     if (DIE *Clone = cloneDIE(*Child, Unit, PCOffset, OutOffset)) {
-      Die->addChild(std::unique_ptr<DIE>(Clone));
+      Die->addChild(Clone);
       OutOffset = Clone->getOffset() + Clone->getSize();
     }
   }
@@ -2202,7 +2709,7 @@ void DwarfLinker::patchRangesForUnit(const CompileUnit &Unit,
                                OrigDwarf.isLittleEndian(), AddressSize);
   auto InvalidRange = FunctionRanges.end(), CurrRange = InvalidRange;
   DWARFUnit &OrigUnit = Unit.getOrigUnit();
-  const auto *OrigUnitDie = OrigUnit.getCompileUnitDIE(false);
+  const auto *OrigUnitDie = OrigUnit.getUnitDIE(false);
   uint64_t OrigLowPc = OrigUnitDie->getAttributeValueAsAddress(
       &OrigUnit, dwarf::DW_AT_low_pc, -1ULL);
   // Ranges addresses are based on the unit's low_pc. Compute the
@@ -2212,8 +2719,8 @@ void DwarfLinker::patchRangesForUnit(const CompileUnit &Unit,
     UnitPcOffset = int64_t(OrigLowPc) - Unit.getLowPc();
 
   for (const auto &RangeAttribute : Unit.getRangesAttributes()) {
-    uint32_t Offset = RangeAttribute->getValue();
-    RangeAttribute->setValue(Streamer->getRangesSectionSize());
+    uint32_t Offset = RangeAttribute.get();
+    RangeAttribute.set(Streamer->getRangesSectionSize());
     RangeList.extract(RangeExtractor, &Offset);
     const auto &Entries = RangeList.getEntries();
     const DWARFDebugRangeList::RangeListEntry &First = Entries.front();
@@ -2240,10 +2747,10 @@ void DwarfLinker::patchRangesForUnit(const CompileUnit &Unit,
 /// but for the sake of initial bit-for-bit compatibility with legacy
 /// dsymutil, we have to do it in a delayed pass.
 void DwarfLinker::generateUnitRanges(CompileUnit &Unit) const {
-  DIEInteger *Attr = Unit.getUnitRangesAttribute();
+  auto Attr = Unit.getUnitRangesAttribute();
   if (Attr)
-    Attr->setValue(Streamer->getRangesSectionSize());
-  Streamer->emitUnitRangesEntries(Unit, Attr != nullptr);
+    Attr->set(Streamer->getRangesSectionSize());
+  Streamer->emitUnitRangesEntries(Unit, static_cast<bool>(Attr));
 }
 
 /// \brief Insert the new line info sequence \p Seq into the current
@@ -2280,30 +2787,30 @@ static void insertLineSequence(std::vector<DWARFDebugLine::Row> &Seq,
   Seq.clear();
 }
 
+static void patchStmtList(DIE &Die, DIEInteger Offset) {
+  for (auto &V : Die.values())
+    if (V.getAttribute() == dwarf::DW_AT_stmt_list) {
+      V = DIEValue(V.getAttribute(), V.getForm(), Offset);
+      return;
+    }
+
+  llvm_unreachable("Didn't find DW_AT_stmt_list in cloned DIE!");
+}
+
 /// \brief Extract the line table for \p Unit from \p OrigDwarf, and
 /// recreate a relocated version of these for the address ranges that
 /// are present in the binary.
 void DwarfLinker::patchLineTableForUnit(CompileUnit &Unit,
                                         DWARFContext &OrigDwarf) {
-  const DWARFDebugInfoEntryMinimal *CUDie =
-      Unit.getOrigUnit().getCompileUnitDIE();
+  const DWARFDebugInfoEntryMinimal *CUDie = Unit.getOrigUnit().getUnitDIE();
   uint64_t StmtList = CUDie->getAttributeValueAsSectionOffset(
       &Unit.getOrigUnit(), dwarf::DW_AT_stmt_list, -1ULL);
   if (StmtList == -1ULL)
     return;
 
   // Update the cloned DW_AT_stmt_list with the correct debug_line offset.
-  if (auto *OutputDIE = Unit.getOutputUnitDIE()) {
-    const auto &Abbrev = OutputDIE->getAbbrev().getData();
-    auto Stmt = std::find_if(
-        Abbrev.begin(), Abbrev.end(), [](const DIEAbbrevData &AbbrevData) {
-          return AbbrevData.getAttribute() == dwarf::DW_AT_stmt_list;
-        });
-    assert(Stmt < Abbrev.end() && "Didn't find DW_AT_stmt_list in cloned DIE!");
-    DIEInteger *StmtAttr =
-        cast<DIEInteger>(OutputDIE->getValues()[Stmt - Abbrev.begin()]);
-    StmtAttr->setValue(Streamer->getLineSectionSize());
-  }
+  if (auto *OutputDIE = Unit.getOutputUnitDIE())
+    patchStmtList(*OutputDIE, DIEInteger(Streamer->getLineSectionSize()));
 
   // Parse the original line info for the unit.
   DWARFDebugLine::LineTable LineTable;
@@ -2421,6 +2928,104 @@ void DwarfLinker::emitAcceleratorEntriesForUnit(CompileUnit &Unit) {
   Streamer->emitPubTypesForUnit(Unit);
 }
 
+/// \brief Read the frame info stored in the object, and emit the
+/// patched frame descriptions for the linked binary.
+///
+/// This is actually pretty easy as the data of the CIEs and FDEs can
+/// be considered as black boxes and moved as is. The only thing to do
+/// is to patch the addresses in the headers.
+void DwarfLinker::patchFrameInfoForObject(const DebugMapObject &DMO,
+                                          DWARFContext &OrigDwarf,
+                                          unsigned AddrSize) {
+  StringRef FrameData = OrigDwarf.getDebugFrameSection();
+  if (FrameData.empty())
+    return;
+
+  DataExtractor Data(FrameData, OrigDwarf.isLittleEndian(), 0);
+  uint32_t InputOffset = 0;
+
+  // Store the data of the CIEs defined in this object, keyed by their
+  // offsets.
+  DenseMap<uint32_t, StringRef> LocalCIES;
+
+  while (Data.isValidOffset(InputOffset)) {
+    uint32_t EntryOffset = InputOffset;
+    uint32_t InitialLength = Data.getU32(&InputOffset);
+    if (InitialLength == 0xFFFFFFFF)
+      return reportWarning("Dwarf64 bits no supported");
+
+    uint32_t CIEId = Data.getU32(&InputOffset);
+    if (CIEId == 0xFFFFFFFF) {
+      // This is a CIE, store it.
+      StringRef CIEData = FrameData.substr(EntryOffset, InitialLength + 4);
+      LocalCIES[EntryOffset] = CIEData;
+      // The -4 is to account for the CIEId we just read.
+      InputOffset += InitialLength - 4;
+      continue;
+    }
+
+    uint32_t Loc = Data.getUnsigned(&InputOffset, AddrSize);
+
+    // Some compilers seem to emit frame info that doesn't start at
+    // the function entry point, thus we can't just lookup the address
+    // in the debug map. Use the linker's range map to see if the FDE
+    // describes something that we can relocate.
+    auto Range = Ranges.upper_bound(Loc);
+    if (Range != Ranges.begin())
+      --Range;
+    if (Range == Ranges.end() || Range->first > Loc ||
+        Range->second.first <= Loc) {
+      // The +4 is to account for the size of the InitialLength field itself.
+      InputOffset = EntryOffset + InitialLength + 4;
+      continue;
+    }
+
+    // This is an FDE, and we have a mapping.
+    // Have we already emitted a corresponding CIE?
+    StringRef CIEData = LocalCIES[CIEId];
+    if (CIEData.empty())
+      return reportWarning("Inconsistent debug_frame content. Dropping.");
+
+    // Look if we already emitted a CIE that corresponds to the
+    // referenced one (the CIE data is the key of that lookup).
+    auto IteratorInserted = EmittedCIEs.insert(
+        std::make_pair(CIEData, Streamer->getFrameSectionSize()));
+    // If there is no CIE yet for this ID, emit it.
+    if (IteratorInserted.second ||
+        // FIXME: dsymutil-classic only caches the last used CIE for
+        // reuse. Mimic that behavior for now. Just removing that
+        // second half of the condition and the LastCIEOffset variable
+        // makes the code DTRT.
+        LastCIEOffset != IteratorInserted.first->getValue()) {
+      LastCIEOffset = Streamer->getFrameSectionSize();
+      IteratorInserted.first->getValue() = LastCIEOffset;
+      Streamer->emitCIE(CIEData);
+    }
+
+    // Emit the FDE with updated address and CIE pointer.
+    // (4 + AddrSize) is the size of the CIEId + initial_location
+    // fields that will get reconstructed by emitFDE().
+    unsigned FDERemainingBytes = InitialLength - (4 + AddrSize);
+    Streamer->emitFDE(IteratorInserted.first->getValue(), AddrSize,
+                      Loc + Range->second.second,
+                      FrameData.substr(InputOffset, FDERemainingBytes));
+    InputOffset += FDERemainingBytes;
+  }
+}
+
+ErrorOr<const object::ObjectFile &>
+DwarfLinker::loadObject(BinaryHolder &BinaryHolder, DebugMapObject &Obj,
+                        const DebugMap &Map) {
+  auto ErrOrObjs =
+      BinaryHolder.GetObjectFiles(Obj.getObjectFilename(), Obj.getTimestamp());
+  if (std::error_code EC = ErrOrObjs.getError())
+    reportWarning(Twine(Obj.getObjectFilename()) + ": " + EC.message());
+  auto ErrOrObj = BinaryHolder.Get(Map.getTriple());
+  if (std::error_code EC = ErrOrObj.getError())
+    reportWarning(Twine(Obj.getObjectFilename()) + ": " + EC.message());
+  return ErrOrObj;
+}
+
 bool DwarfLinker::link(const DebugMap &Map) {
 
   if (Map.begin() == Map.end()) {
@@ -2440,11 +3045,9 @@ bool DwarfLinker::link(const DebugMap &Map) {
 
     if (Options.Verbose)
       outs() << "DEBUG MAP OBJECT: " << Obj->getObjectFilename() << "\n";
-    auto ErrOrObj = BinHolder.GetObjectFile(Obj->getObjectFilename());
-    if (std::error_code EC = ErrOrObj.getError()) {
-      reportWarning(Twine(Obj->getObjectFilename()) + ": " + EC.message());
+    auto ErrOrObj = loadObject(BinHolder, *Obj, Map);
+    if (!ErrOrObj)
       continue;
-    }
 
     // Look for relocations that correspond to debug map entries.
     if (!findValidRelocsInDebugInfo(*ErrOrObj, *Obj)) {
@@ -2460,13 +3063,14 @@ bool DwarfLinker::link(const DebugMap &Map) {
     // In a first phase, just read in the debug info and store the DIE
     // parent links that we will use during the next phase.
     for (const auto &CU : DwarfContext.compile_units()) {
-      auto *CUDie = CU->getCompileUnitDIE(false);
+      auto *CUDie = CU->getUnitDIE(false);
       if (Options.Verbose) {
         outs() << "Input compilation unit:";
         CUDie->dump(outs(), CU.get(), 0);
       }
-      Units.emplace_back(*CU, UnitID++);
-      gatherDIEParents(CUDie, 0, Units.back());
+      Units.emplace_back(*CU, UnitID++, !Options.NoODR);
+      gatherDIEParents(CUDie, 0, Units.back(), &ODRContexts.getRoot(),
+                       StringPool, ODRContexts);
     }
 
     // Then mark all the DIEs that need to be present in the linked
@@ -2475,7 +3079,7 @@ bool DwarfLinker::link(const DebugMap &Map) {
     // references require the ParentIdx to be setup for every CU in
     // the object file before calling this.
     for (auto &CurrentUnit : Units)
-      lookForDIEsToKeep(*CurrentUnit.getOrigUnit().getCompileUnitDIE(), *Obj,
+      lookForDIEsToKeep(*CurrentUnit.getOrigUnit().getUnitDIE(), *Obj,
                         CurrentUnit, 0);
 
     // The calls to applyValidRelocs inside cloneDIE will walk the
@@ -2488,7 +3092,7 @@ bool DwarfLinker::link(const DebugMap &Map) {
     // to clone/emit.
     if (!ValidRelocs.empty())
       for (auto &CurrentUnit : Units) {
-        const auto *InputDIE = CurrentUnit.getOrigUnit().getCompileUnitDIE();
+        const auto *InputDIE = CurrentUnit.getOrigUnit().getUnitDIE();
         CurrentUnit.setStartOffset(OutputDebugInfoSize);
         DIE *OutputDIE = cloneDIE(*InputDIE, CurrentUnit, 0 /* PCOffset */,
                                   11 /* Unit Header size */);
@@ -2518,6 +3122,10 @@ bool DwarfLinker::link(const DebugMap &Map) {
         Streamer->emitDIE(*CurrentUnit.getOutputUnitDIE());
       }
 
+    if (!ValidRelocs.empty() && !Options.NoOutput && !Units.empty())
+      patchFrameInfoForObject(*Obj, DwarfContext,
+                              Units[0].getOrigUnit().getAddressByteSize());
+
     // Clean-up before starting working on the next object.
     endDebugObject();
   }
diff --git a/tools/dsymutil/MachODebugMapParser.cpp b/tools/dsymutil/MachODebugMapParser.cpp
index bf64303b9ea..33554f12122 100644
--- a/tools/dsymutil/MachODebugMapParser.cpp
+++ b/tools/dsymutil/MachODebugMapParser.cpp
@@ -55,11 +55,11 @@ class MachODebugMapParser {
   const char *CurrentFunctionName;
   uint64_t CurrentFunctionAddress;
 
-  void switchToNewDebugMapObject(StringRef Filename);
+  void switchToNewDebugMapObject(StringRef Filename, sys::TimeValue Timestamp);
   void resetParserState();
   uint64_t getMainBinarySymbolAddress(StringRef Name);
-  void loadMainBinarySymbols();
-  void loadCurrentObjectFileSymbols();
+  void loadMainBinarySymbols(const MachOObjectFile &MainBinary);
+  void loadCurrentObjectFileSymbols(const object::MachOObjectFile &Obj);
   void handleStabSymbolTableEntry(uint32_t StringIndex, uint8_t Type,
                                   uint8_t SectionIndex, uint16_t Flags,
                                   uint64_t Value);
@@ -84,41 +84,47 @@ void MachODebugMapParser::resetParserState() {
 /// Create a new DebugMapObject. This function resets the state of the
 /// parser that was referring to the last object file and sets
 /// everything up to add symbols to the new one.
-void MachODebugMapParser::switchToNewDebugMapObject(StringRef Filename) {
+void MachODebugMapParser::switchToNewDebugMapObject(StringRef Filename,
+                                                    sys::TimeValue Timestamp) {
   resetParserState();
 
   SmallString<80> Path(PathPrefix);
   sys::path::append(Path, Filename);
 
-  auto MachOOrError = CurrentObjectHolder.GetFileAs<MachOObjectFile>(Path);
+  auto MachOOrError =
+      CurrentObjectHolder.GetFilesAs<MachOObjectFile>(Path, Timestamp);
   if (auto Error = MachOOrError.getError()) {
     Warning(Twine("cannot open debug object \"") + Path.str() + "\": " +
             Error.message() + "\n");
     return;
   }
 
-  loadCurrentObjectFileSymbols();
-  CurrentDebugMapObject = &Result->addDebugMapObject(Path);
-}
+  auto ErrOrAchObj =
+      CurrentObjectHolder.GetAs<MachOObjectFile>(Result->getTriple());
+  if (auto Err = ErrOrAchObj.getError()) {
+    return Warning(Twine("cannot open debug object \"") + Path.str() + "\": " +
+                   Err.message() + "\n");
+  }
 
-static Triple getTriple(const object::MachOObjectFile &Obj) {
-  Triple TheTriple("unknown-unknown-unknown");
-  TheTriple.setArch(Triple::ArchType(Obj.getArch()));
-  TheTriple.setObjectFormat(Triple::MachO);
-  return TheTriple;
+  CurrentDebugMapObject = &Result->addDebugMapObject(Path, Timestamp);
+  loadCurrentObjectFileSymbols(*ErrOrAchObj);
 }
 
 /// This main parsing routine tries to open the main binary and if
 /// successful iterates over the STAB entries. The real parsing is
 /// done in handleStabSymbolTableEntry.
 ErrorOr<std::unique_ptr<DebugMap>> MachODebugMapParser::parse() {
-  auto MainBinOrError = MainBinaryHolder.GetFileAs<MachOObjectFile>(BinaryPath);
+  auto MainBinOrError =
+      MainBinaryHolder.GetFilesAs<MachOObjectFile>(BinaryPath);
   if (auto Error = MainBinOrError.getError())
     return Error;
 
-  const MachOObjectFile &MainBinary = *MainBinOrError;
-  loadMainBinarySymbols();
-  Result = make_unique<DebugMap>(getTriple(MainBinary));
+  if (MainBinOrError->size() != 1)
+    return make_error_code(object::object_error::invalid_file_type);
+
+  const MachOObjectFile &MainBinary = *MainBinOrError->front();
+  loadMainBinarySymbols(MainBinary);
+  Result = make_unique<DebugMap>(BinaryHolder::getTriple(MainBinary));
   MainBinaryStrings = MainBinary.getStringTableData();
   for (const SymbolRef &Symbol : MainBinary.symbols()) {
     const DataRefImpl &DRI = Symbol.getRawDataRefImpl();
@@ -144,8 +150,11 @@ void MachODebugMapParser::handleStabSymbolTableEntry(uint32_t StringIndex,
   const char *Name = &MainBinaryStrings.data()[StringIndex];
 
   // An N_OSO entry represents the start of a new object file description.
-  if (Type == MachO::N_OSO)
-    return switchToNewDebugMapObject(Name);
+  if (Type == MachO::N_OSO) {
+    sys::TimeValue Timestamp;
+    Timestamp.fromEpochTime(Value);
+    return switchToNewDebugMapObject(Name, Timestamp);
+  }
 
   // If the last N_OSO object file wasn't found,
   // CurrentDebugMapObject will be null. Do not update anything
@@ -160,8 +169,6 @@ void MachODebugMapParser::handleStabSymbolTableEntry(uint32_t StringIndex,
     // symbol table to find its address as it might not be in the
     // debug map (for common symbols).
     Value = getMainBinarySymbolAddress(Name);
-    if (Value == UnknownAddressOrSize)
-      return;
     break;
   case MachO::N_FUN:
     // Functions are scopes in STABS. They have an end marker that
@@ -193,16 +200,16 @@ void MachODebugMapParser::handleStabSymbolTableEntry(uint32_t StringIndex,
 }
 
 /// Load the current object file symbols into CurrentObjectAddresses.
-void MachODebugMapParser::loadCurrentObjectFileSymbols() {
+void MachODebugMapParser::loadCurrentObjectFileSymbols(
+    const object::MachOObjectFile &Obj) {
   CurrentObjectAddresses.clear();
 
-  for (auto Sym : CurrentObjectHolder.Get().symbols()) {
-    StringRef Name;
-    uint64_t Addr;
-    if (Sym.getAddress(Addr) || Addr == UnknownAddressOrSize ||
-        Sym.getName(Name))
+  for (auto Sym : Obj.symbols()) {
+    uint64_t Addr = Sym.getValue();
+    ErrorOr<StringRef> Name = Sym.getName();
+    if (!Name)
       continue;
-    CurrentObjectAddresses[Name] = Addr;
+    CurrentObjectAddresses[*Name] = Addr;
   }
 }
 
@@ -212,31 +219,34 @@ void MachODebugMapParser::loadCurrentObjectFileSymbols() {
 uint64_t MachODebugMapParser::getMainBinarySymbolAddress(StringRef Name) {
   auto Sym = MainBinarySymbolAddresses.find(Name);
   if (Sym == MainBinarySymbolAddresses.end())
-    return UnknownAddressOrSize;
+    return 0;
   return Sym->second;
 }
 
 /// Load the interesting main binary symbols' addresses into
 /// MainBinarySymbolAddresses.
-void MachODebugMapParser::loadMainBinarySymbols() {
-  const MachOObjectFile &MainBinary = MainBinaryHolder.GetAs<MachOObjectFile>();
+void MachODebugMapParser::loadMainBinarySymbols(
+    const MachOObjectFile &MainBinary) {
   section_iterator Section = MainBinary.section_end();
+  MainBinarySymbolAddresses.clear();
   for (const auto &Sym : MainBinary.symbols()) {
-    SymbolRef::Type Type;
+    SymbolRef::Type Type = Sym.getType();
     // Skip undefined and STAB entries.
-    if (Sym.getType(Type) || (Type & SymbolRef::ST_Debug) ||
-        (Type & SymbolRef::ST_Unknown))
+    if ((Type & SymbolRef::ST_Debug) || (Type & SymbolRef::ST_Unknown))
       continue;
-    StringRef Name;
-    uint64_t Addr;
     // The only symbols of interest are the global variables. These
     // are the only ones that need to be queried because the address
     // of common data won't be described in the debug map. All other
     // addresses should be fetched for the debug map.
-    if (Sym.getAddress(Addr) || Addr == UnknownAddressOrSize ||
-        !(Sym.getFlags() & SymbolRef::SF_Global) || Sym.getSection(Section) ||
-        Section->isText() || Sym.getName(Name) || Name.size() == 0 ||
-        Name[0] == '\0')
+    if (!(Sym.getFlags() & SymbolRef::SF_Global) || Sym.getSection(Section) ||
+        Section == MainBinary.section_end() || Section->isText())
+      continue;
+    uint64_t Addr = Sym.getValue();
+    ErrorOr<StringRef> NameOrErr = Sym.getName();
+    if (!NameOrErr)
+      continue;
+    StringRef Name = *NameOrErr;
+    if (Name.size() == 0 || Name[0] == '\0')
       continue;
     MainBinarySymbolAddresses[Name] = Addr;
   }
@@ -244,10 +254,16 @@ void MachODebugMapParser::loadMainBinarySymbols() {
 
 namespace llvm {
 namespace dsymutil {
-llvm::ErrorOr<std::unique_ptr<DebugMap>>
-parseDebugMap(StringRef InputFile, StringRef PrependPath, bool Verbose) {
-  MachODebugMapParser Parser(InputFile, PrependPath, Verbose);
-  return Parser.parse();
+llvm::ErrorOr<std::unique_ptr<DebugMap>> parseDebugMap(StringRef InputFile,
+                                                       StringRef PrependPath,
+                                                       bool Verbose,
+                                                       bool InputIsYAML) {
+  if (!InputIsYAML) {
+    MachODebugMapParser Parser(InputFile, PrependPath, Verbose);
+    return Parser.parse();
+  } else {
+    return DebugMap::parseYAMLDebugMap(InputFile, PrependPath, Verbose);
+  }
 }
 }
 }
diff --git a/tools/dsymutil/dsymutil.cpp b/tools/dsymutil/dsymutil.cpp
index 4fc91b029f6..1d5fb0ff8cf 100644
--- a/tools/dsymutil/dsymutil.cpp
+++ b/tools/dsymutil/dsymutil.cpp
@@ -27,29 +27,45 @@ using namespace llvm::dsymutil;
 namespace {
 using namespace llvm::cl;
 
+OptionCategory DsymCategory("Specific Options");
+static opt<bool> Help("h", desc("Alias for -help"), Hidden);
+static opt<bool> Version("v", desc("Alias for -version"), Hidden);
+
 static opt<std::string> InputFile(Positional, desc("<input file>"),
-                                  init("a.out"));
+                                  init("a.out"), cat(DsymCategory));
 
-static opt<std::string> OutputFileOpt("o", desc("Specify the output file."
-                                                " default: <input file>.dwarf"),
-                                      value_desc("filename"));
+static opt<std::string>
+    OutputFileOpt("o",
+                  desc("Specify the output file. default: <input file>.dwarf"),
+                  value_desc("filename"), cat(DsymCategory));
 
-static opt<std::string> OsoPrependPath("oso-prepend-path",
-                                       desc("Specify a directory to prepend "
-                                            "to the paths of object files."),
-                                       value_desc("path"));
+static opt<std::string> OsoPrependPath(
+    "oso-prepend-path",
+    desc("Specify a directory to prepend to the paths of object files."),
+    value_desc("path"), cat(DsymCategory));
 
-static opt<bool> Verbose("v", desc("Verbosity level"), init(false));
+static opt<bool> Verbose("verbose", desc("Verbosity level"), init(false),
+                         cat(DsymCategory));
 
-static opt<bool> NoOutput("no-output", desc("Do the link in memory, but do "
-                                            "not emit the result file."),
-                          init(false));
+static opt<bool>
+    NoOutput("no-output",
+             desc("Do the link in memory, but do not emit the result file."),
+             init(false), cat(DsymCategory));
 
 static opt<bool>
-    ParseOnly("parse-only",
-              desc("Only parse the debug map, do not actaully link "
-                   "the DWARF."),
-              init(false));
+    NoODR("no-odr",
+          desc("Do not use ODR (One Definition Rule) for type uniquing."),
+          init(false), cat(DsymCategory));
+
+static opt<bool> DumpDebugMap(
+    "dump-debug-map",
+    desc("Parse and dump the debug map to standard output. Not DWARF link "
+         "will take place."),
+    init(false), cat(DsymCategory));
+
+static opt<bool> InputIsYAMLDebugMap(
+    "y", desc("Treat the input file is a YAML debug map rather than a binary."),
+    init(false), cat(DsymCategory));
 }
 
 int main(int argc, char **argv) {
@@ -58,11 +74,28 @@ int main(int argc, char **argv) {
   llvm::llvm_shutdown_obj Shutdown;
   LinkOptions Options;
 
-  llvm::cl::ParseCommandLineOptions(argc, argv, "llvm dsymutil\n");
-  auto DebugMapPtrOrErr = parseDebugMap(InputFile, OsoPrependPath, Verbose);
+  HideUnrelatedOptions(DsymCategory);
+  llvm::cl::ParseCommandLineOptions(
+      argc, argv,
+      "manipulate archived DWARF debug symbol files.\n\n"
+      "dsymutil links the DWARF debug information found in the object files\n"
+      "for the executable <input file> by using debug symbols information\n"
+      "contained in its symbol table.\n");
+
+  if (Help)
+    PrintHelpMessage();
+
+  if (Version) {
+    llvm::cl::PrintVersionMessage();
+    return 0;
+  }
+
+  auto DebugMapPtrOrErr =
+      parseDebugMap(InputFile, OsoPrependPath, Verbose, InputIsYAMLDebugMap);
 
   Options.Verbose = Verbose;
   Options.NoOutput = NoOutput;
+  Options.NoODR = NoODR;
 
   llvm::InitializeAllTargetInfos();
   llvm::InitializeAllTargetMCs();
@@ -75,10 +108,10 @@ int main(int argc, char **argv) {
     return 1;
   }
 
-  if (Verbose)
+  if (Verbose || DumpDebugMap)
     (*DebugMapPtrOrErr)->print(llvm::outs());
 
-  if (ParseOnly)
+  if (DumpDebugMap)
     return 0;
 
   std::string OutputFile;
diff --git a/tools/dsymutil/dsymutil.h b/tools/dsymutil/dsymutil.h
index e9f7cd95187..cd530a1177a 100644
--- a/tools/dsymutil/dsymutil.h
+++ b/tools/dsymutil/dsymutil.h
@@ -27,15 +27,17 @@ namespace dsymutil {
 struct LinkOptions {
   bool Verbose;  ///< Verbosity
   bool NoOutput; ///< Skip emitting output
+  bool NoODR;    ///< Do not unique types according to ODR
 
   LinkOptions() : Verbose(false), NoOutput(false) {}
 };
 
 /// \brief Extract the DebugMap from the given file.
 /// The file has to be a MachO object file.
-llvm::ErrorOr<std::unique_ptr<DebugMap>>
-parseDebugMap(StringRef InputFile, StringRef PrependPath = "",
-              bool Verbose = false);
+llvm::ErrorOr<std::unique_ptr<DebugMap>> parseDebugMap(StringRef InputFile,
+                                                       StringRef PrependPath,
+                                                       bool Verbose,
+                                                       bool InputIsYAML);
 
 /// \brief Link the Dwarf debuginfo as directed by the passed DebugMap
 /// \p DM into a DwarfFile named \p OutputFilename.
diff --git a/tools/gold/gold-plugin.cpp b/tools/gold/gold-plugin.cpp
index f608d2a885f..18cd621317e 100644
--- a/tools/gold/gold-plugin.cpp
+++ b/tools/gold/gold-plugin.cpp
@@ -82,14 +82,6 @@ static std::list<claimed_file> Modules;
 static std::vector<std::string> Cleanup;
 static llvm::TargetOptions TargetOpts;
 
-// @LOCALMOD-BEGIN
-// Callback for getting the number of --wrap'd symbols.
-static ld_plugin_get_num_wrapped get_num_wrapped = NULL;
-
-// Callback for getting the name of a wrapped symbol.
-static ld_plugin_get_wrapped get_wrapped = NULL;
-// @LOCALMOD-END
-
 namespace options {
   enum OutputType {
     OT_NORMAL,
@@ -240,14 +232,6 @@ ld_plugin_status onload(ld_plugin_tv *tv) {
       case LDPT_GET_VIEW:
         get_view = tv->tv_u.tv_get_view;
         break;
-      // @LOCALMOD-BEGIN
-      case LDPT_GET_WRAPPED:
-        get_wrapped = tv->tv_u.tv_get_wrapped;
-        break;
-      case LDPT_GET_NUM_WRAPPED:
-        get_num_wrapped = tv->tv_u.tv_get_num_wrapped;
-        break;
-      // @LOCALMOD-END
       case LDPT_MESSAGE:
         message = tv->tv_u.tv_message;
         break;
@@ -732,8 +716,7 @@ getModuleForFile(LLVMContext &Context, claimed_file &F,
 }
 
 static void runLTOPasses(Module &M, TargetMachine &TM) {
-  if (const DataLayout *DL = TM.getDataLayout())
-    M.setDataLayout(*DL);
+  M.setDataLayout(TM.createDataLayout());
 
   legacy::PassManager passes;
   passes.add(createTargetTransformInfoWrapperPass(TM.getTargetIRAnalysis()));
@@ -803,15 +786,20 @@ static void codegen(Module &M) {
   legacy::PassManager CodeGenPasses;
 
   SmallString<128> Filename;
+  if (!options::obj_path.empty())
+    Filename = options::obj_path;
+  else if (options::TheOutputType == options::OT_SAVE_TEMPS)
+    Filename = output_name + ".o";
+
   int FD;
-  if (options::obj_path.empty()) {
+  bool TempOutFile = Filename.empty();
+  if (TempOutFile) {
     std::error_code EC =
         sys::fs::createTemporaryFile("lto-llvm", "o", FD, Filename);
     if (EC)
       message(LDPL_FATAL, "Could not create temporary file: %s",
               EC.message().c_str());
   } else {
-    Filename = options::obj_path;
     std::error_code EC =
         sys::fs::openFileForWrite(Filename.c_str(), FD, sys::fs::F_None);
     if (EC)
@@ -832,38 +820,10 @@ static void codegen(Module &M) {
             "Unable to add .o file to the link. File left behind in: %s",
             Filename.c_str());
 
-  if (options::obj_path.empty())
+  if (TempOutFile)
     Cleanup.push_back(Filename.c_str());
 }
 
-// @LOCALMOD-BEGIN
-static void wrapSymbol(Module *M, const char *sym) {
-  std::string wrapSymName("__wrap_");
-  wrapSymName += sym;
-  std::string realSymName("__real_");
-  realSymName += sym;
-
-  Constant *SymGV = M->getNamedValue(sym);
-
-  // Replace uses of "sym" with "__wrap_sym".
-  if (SymGV) {
-    Constant *WrapGV = M->getNamedValue(wrapSymName);
-    if (!WrapGV)
-      WrapGV = M->getOrInsertGlobal(wrapSymName, SymGV->getType());
-    SymGV->replaceAllUsesWith(
-        ConstantExpr::getBitCast(WrapGV, SymGV->getType()));
-  }
-
-  // Replace uses of "__real_sym" with "sym".
-  if (Constant *RealGV = M->getNamedValue(realSymName)) {
-    if (!SymGV)
-      SymGV = M->getOrInsertGlobal(sym, RealGV->getType());
-    RealGV->replaceAllUsesWith(
-        ConstantExpr::getBitCast(SymGV, RealGV->getType()));
-  }
-}
-// @LOCALMOD-END
-
 /// gold informs us that all symbols have been read. At this point, we use
 /// get_symbols to see if any of our definitions have been overridden by a
 /// native object file. Then, perform optimization and codegen.
@@ -914,23 +874,6 @@ static ld_plugin_status allSymbolsReadHook(raw_fd_ostream *ApiFile) {
       internalize(*GV);
   }
 
-  // @LOCALMOD-BEGIN
-  // Perform symbol wrapping.
-  unsigned num_wrapped;
-  if ((*get_num_wrapped)(&num_wrapped) != LDPS_OK) {
-    (*message)(LDPL_ERROR, "Unable to get the number of wrapped symbols.");
-    return LDPS_ERR;
-  }
-  for (unsigned i = 0; i < num_wrapped; ++i) {
-    const char *sym;
-    if ((*get_wrapped)(i, &sym) != LDPS_OK) {
-      (*message)(LDPL_ERROR, "Unable to wrap symbol %u/%u.", i, num_wrapped);
-      return LDPS_ERR;
-    }
-    wrapSymbol(Combined.get(), sym);
-  }
-  // @LOCALMOD-END
-
   if (options::TheOutputType == options::OT_DISABLE)
     return LDPS_OK;
 
diff --git a/tools/llc/CMakeLists.txt b/tools/llc/CMakeLists.txt
index f81717de589..462df0f31ce 100644
--- a/tools/llc/CMakeLists.txt
+++ b/tools/llc/CMakeLists.txt
@@ -7,9 +7,11 @@ set(LLVM_LINK_COMPONENTS
   IRReader
   MC
   NaClTransforms
+  MIRParser
   ScalarOpts
   SelectionDAG
   Support
+  Target
   )
 
 # Support plugins.
diff --git a/tools/llc/LLVMBuild.txt b/tools/llc/LLVMBuild.txt
index c05575d11b0..2960e9a83d6 100644
--- a/tools/llc/LLVMBuild.txt
+++ b/tools/llc/LLVMBuild.txt
@@ -19,4 +19,5 @@
 type = Tool
 name = llc
 parent = Tools
-required_libraries = AsmParser BitReader IRReader all-targets NaClTransforms
+required_libraries = AsmParser BitReader IRReader MIRParser all-targets NaClTransforms
+
diff --git a/tools/llc/Makefile b/tools/llc/Makefile
index 4b9c235692e..e4420bab96b 100644
--- a/tools/llc/Makefile
+++ b/tools/llc/Makefile
@@ -9,7 +9,7 @@
 
 LEVEL := ../..
 TOOLNAME := llc
-LINK_COMPONENTS := nacltransforms all-targets bitreader asmparser irreader nacltransforms
+LINK_COMPONENTS := nacltransforms all-targets bitreader asmparser irreader mirparser nacltransforms
 
 # Support plugins.
 NO_DEAD_STRIP := 1
diff --git a/tools/llc/llc.cpp b/tools/llc/llc.cpp
index ab50f2a7802..c51c012391b 100644
--- a/tools/llc/llc.cpp
+++ b/tools/llc/llc.cpp
@@ -20,6 +20,7 @@
 #include "llvm/CodeGen/CommandFlags.h"
 #include "llvm/CodeGen/LinkAllAsmWriterComponents.h"
 #include "llvm/CodeGen/LinkAllCodegenComponents.h"
+#include "llvm/CodeGen/MIRParser/MIRParser.h"
 #include "llvm/IR/DataLayout.h"
 #include "llvm/IR/IRPrintingPasses.h"
 #include "llvm/IR/LLVMContext.h"
@@ -109,6 +110,8 @@ GetOutputStream(const char *TargetName, Triple::OSType OS,
       StringRef IFN = InputFilename;
       if (IFN.endswith(".bc") || IFN.endswith(".ll"))
         OutputFilename = IFN.drop_back(3);
+      else if (IFN.endswith(".mir"))
+        OutputFilename = IFN.drop_back(4);
       else
         OutputFilename = IFN;
 
@@ -207,6 +210,7 @@ static int compileModule(char **argv, LLVMContext &Context) {
   // Load the module to be compiled...
   SMDiagnostic Err;
   std::unique_ptr<Module> M;
+  std::unique_ptr<MIRParser> MIR;
   Triple TheTriple;
 
   bool SkipModule = MCPU == "help" ||
@@ -214,7 +218,14 @@ static int compileModule(char **argv, LLVMContext &Context) {
 
   // If user just wants to list available options, skip module loading
   if (!SkipModule) {
-    M = parseIRFile(InputFilename, Err, Context);
+    if (StringRef(InputFilename).endswith_lower(".mir")) {
+      MIR = createMIRParserFromFile(InputFilename, Err, Context);
+      if (MIR) {
+        M = MIR->parseLLVMModule();
+        assert(M && "parseLLVMModule should exit on failure");
+      }
+    } else
+      M = parseIRFile(InputFilename, Err, Context);
     if (!M) {
       Err.print(argv[0], errs());
       return 1;
@@ -248,32 +259,7 @@ static int compileModule(char **argv, LLVMContext &Context) {
     return 1;
   }
 
-  // Package up features to be passed to target/subtarget
-  std::string FeaturesStr;
-  if (!MAttrs.empty() || MCPU == "native") {
-    SubtargetFeatures Features;
-
-    // If user asked for the 'native' CPU, we need to autodetect features.
-    // This is necessary for x86 where the CPU might not support all the
-    // features the autodetected CPU name lists in the target. For example,
-    // not all Sandybridge processors support AVX.
-    if (MCPU == "native") {
-      StringMap<bool> HostFeatures;
-      if (sys::getHostCPUFeatures(HostFeatures))
-        for (auto &F : HostFeatures)
-          Features.AddFeature(F.first(), F.second);
-    }
-
-    for (unsigned i = 0; i != MAttrs.size(); ++i)
-      Features.AddFeature(MAttrs[i]);
-    FeaturesStr = Features.getString();
-  }
-
-  // If user asked for the 'native' CPU, autodetect here. If autodection fails,
-  // this will set the CPU to an empty string which tells the target to
-  // pick a basic default.
-  if (MCPU == "native")
-    MCPU = sys::getHostCPUName();
+  std::string CPUStr = getCPUStr(), FeaturesStr = getFeaturesStr();
 
   CodeGenOpt::Level OLvl = CodeGenOpt::Default;
   switch (OptLevel) {
@@ -294,8 +280,9 @@ static int compileModule(char **argv, LLVMContext &Context) {
   Options.MCOptions.AsmVerbose = AsmVerbose;
 
   std::unique_ptr<TargetMachine> Target(
-      TheTarget->createTargetMachine(TheTriple.getTriple(), MCPU, FeaturesStr,
+      TheTarget->createTargetMachine(TheTriple.getTriple(), CPUStr, FeaturesStr,
                                      Options, RelocModel, CMModel, OLvl));
+
   assert(Target && "Could not allocate target machine!");
 
   // If we don't have a module then just exit now. We do this down
@@ -305,9 +292,8 @@ static int compileModule(char **argv, LLVMContext &Context) {
     return 0;
 
   assert(M && "Should have exited if we didn't have a module!");
-
-  if (GenerateSoftFloatCalls)
-    FloatABIForCalls = FloatABI::Soft;
+  if (FloatABIForCalls != FloatABI::Default)
+    Options.FloatABIType = FloatABIForCalls;
 
   // Figure out where we are going to send the output.
   std::unique_ptr<tool_output_file> Out =
@@ -326,8 +312,11 @@ static int compileModule(char **argv, LLVMContext &Context) {
   PM.add(new TargetLibraryInfoWrapperPass(TLII));
 
   // Add the target data from the target machine, if it exists, or the module.
-  if (const DataLayout *DL = Target->getDataLayout())
-    M->setDataLayout(*DL);
+  M->setDataLayout(Target->createDataLayout());
+
+  // Override function attributes based on CPUStr, FeaturesStr, and command line
+  // flags.
+  setFunctionAttributes(CPUStr, FeaturesStr, *M);
 
   if (RelaxAll.getNumOccurrences() > 0 &&
       FileType != TargetMachine::CGFT_ObjectFile)
@@ -343,29 +332,44 @@ static int compileModule(char **argv, LLVMContext &Context) {
       OS = BOS.get();
     }
 
+    AnalysisID StartBeforeID = nullptr;
     AnalysisID StartAfterID = nullptr;
     AnalysisID StopAfterID = nullptr;
     const PassRegistry *PR = PassRegistry::getPassRegistry();
-    if (!StartAfter.empty()) {
-      const PassInfo *PI = PR->getPassInfo(StartAfter);
-      if (!PI) {
-        errs() << argv[0] << ": start-after pass is not registered.\n";
+    if (!RunPass.empty()) {
+      if (!StartAfter.empty() || !StopAfter.empty()) {
+        errs() << argv[0] << ": start-after and/or stop-after passes are "
+                             "redundant when run-pass is specified.\n";
         return 1;
       }
-      StartAfterID = PI->getTypeInfo();
-    }
-    if (!StopAfter.empty()) {
-      const PassInfo *PI = PR->getPassInfo(StopAfter);
+      const PassInfo *PI = PR->getPassInfo(RunPass);
       if (!PI) {
-        errs() << argv[0] << ": stop-after pass is not registered.\n";
+        errs() << argv[0] << ": run-pass pass is not registered.\n";
         return 1;
       }
-      StopAfterID = PI->getTypeInfo();
+      StopAfterID = StartBeforeID = PI->getTypeInfo();
+    } else {
+      if (!StartAfter.empty()) {
+        const PassInfo *PI = PR->getPassInfo(StartAfter);
+        if (!PI) {
+          errs() << argv[0] << ": start-after pass is not registered.\n";
+          return 1;
+        }
+        StartAfterID = PI->getTypeInfo();
+      }
+      if (!StopAfter.empty()) {
+        const PassInfo *PI = PR->getPassInfo(StopAfter);
+        if (!PI) {
+          errs() << argv[0] << ": stop-after pass is not registered.\n";
+          return 1;
+        }
+        StopAfterID = PI->getTypeInfo();
+      }
     }
 
     // Ask the target to add backend passes as necessary.
-    if (Target->addPassesToEmitFile(PM, *OS, FileType, NoVerify, StartAfterID,
-                                    StopAfterID)) {
+    if (Target->addPassesToEmitFile(PM, *OS, FileType, NoVerify, StartBeforeID,
+                                    StartAfterID, StopAfterID, MIR.get())) {
       errs() << argv[0] << ": target does not support generation of this"
              << " file type!\n";
       return 1;
diff --git a/tools/lli/CMakeLists.txt b/tools/lli/CMakeLists.txt
index f98275b5fe6..4af05969af1 100644
--- a/tools/lli/CMakeLists.txt
+++ b/tools/lli/CMakeLists.txt
@@ -14,6 +14,8 @@ set(LLVM_LINK_COMPONENTS
   RuntimeDyld
   SelectionDAG
   Support
+  Target
+  TransformUtils
   native
   )
 
diff --git a/tools/lli/LLVMBuild.txt b/tools/lli/LLVMBuild.txt
index 4c14c47bd0d..9d889bf4c2e 100644
--- a/tools/lli/LLVMBuild.txt
+++ b/tools/lli/LLVMBuild.txt
@@ -22,4 +22,14 @@ subdirectories = ChildTarget
 type = Tool
 name = lli
 parent = Tools
-required_libraries = AsmParser BitReader IRReader Instrumentation Interpreter MCJIT NativeCodeGen SelectionDAG Native
+required_libraries =
+ AsmParser
+ BitReader
+ IRReader
+ Instrumentation
+ Interpreter
+ MCJIT
+ Native
+ NativeCodeGen
+ SelectionDAG
+ TransformUtils
diff --git a/tools/lli/OrcLazyJIT.cpp b/tools/lli/OrcLazyJIT.cpp
index ff8baf0b079..4ac2ccffcd5 100644
--- a/tools/lli/OrcLazyJIT.cpp
+++ b/tools/lli/OrcLazyJIT.cpp
@@ -11,6 +11,7 @@
 #include "llvm/ExecutionEngine/Orc/OrcTargetSupport.h"
 #include "llvm/Support/Debug.h"
 #include "llvm/Support/DynamicLibrary.h"
+#include <cstdio>
 #include <system_error>
 
 using namespace llvm;
@@ -61,7 +62,7 @@ OrcLazyJIT::TransformFtor OrcLazyJIT::createDebugDumper() {
 
   switch (OrcDumpKind) {
   case DumpKind::NoDump:
-    return [](std::unique_ptr<Module> M) { return std::move(M); };
+    return [](std::unique_ptr<Module> M) { return M; };
 
   case DumpKind::DumpFuncsToStdOut:
     return [](std::unique_ptr<Module> M) {
@@ -79,7 +80,7 @@ OrcLazyJIT::TransformFtor OrcLazyJIT::createDebugDumper() {
       }
 
       printf("]\n");
-      return std::move(M);
+      return M;
     };
 
   case DumpKind::DumpModsToStdErr:
@@ -87,7 +88,7 @@ OrcLazyJIT::TransformFtor OrcLazyJIT::createDebugDumper() {
              dbgs() << "----- Module Start -----\n" << *M
                     << "----- Module End -----\n";
 
-             return std::move(M);
+             return M;
            };
 
   case DumpKind::DumpModsToDisk:
@@ -101,12 +102,15 @@ OrcLazyJIT::TransformFtor OrcLazyJIT::createDebugDumper() {
                exit(1);
              }
              Out << *M;
-             return std::move(M);
+             return M;
            };
   }
   llvm_unreachable("Unknown DumpKind");
 }
 
+// Defined in lli.cpp.
+CodeGenOpt::Level getOptLevel();
+
 int llvm::runOrcLazyJIT(std::unique_ptr<Module> M, int ArgC, char* ArgV[]) {
   // Add the program's symbols into the JIT's search space.
   if (sys::DynamicLibrary::LoadLibraryPermanently(nullptr)) {
@@ -116,7 +120,10 @@ int llvm::runOrcLazyJIT(std::unique_ptr<Module> M, int ArgC, char* ArgV[]) {
 
   // Grab a target machine and try to build a factory function for the
   // target-specific Orc callback manager.
-  auto TM = std::unique_ptr<TargetMachine>(EngineBuilder().selectTarget());
+  EngineBuilder EB;
+  EB.setOptLevel(getOptLevel());
+  auto TM = std::unique_ptr<TargetMachine>(EB.selectTarget());
+  M->setDataLayout(TM->createDataLayout());
   auto &Context = getGlobalContext();
   auto CallbackMgrBuilder =
     OrcLazyJIT::createCallbackManagerBuilder(Triple(TM->getTargetTriple()));
@@ -125,12 +132,13 @@ int llvm::runOrcLazyJIT(std::unique_ptr<Module> M, int ArgC, char* ArgV[]) {
   // manager for this target. Bail out.
   if (!CallbackMgrBuilder) {
     errs() << "No callback manager available for target '"
-           << TM->getTargetTriple() << "'.\n";
+           << TM->getTargetTriple().str() << "'.\n";
     return 1;
   }
 
   // Everything looks good. Build the JIT.
-  OrcLazyJIT J(std::move(TM), Context, CallbackMgrBuilder);
+  auto &DL = M->getDataLayout();
+  OrcLazyJIT J(std::move(TM), DL, Context, CallbackMgrBuilder);
 
   // Add the module, look up main and run it.
   auto MainHandle = J.addModule(std::move(M));
diff --git a/tools/lli/OrcLazyJIT.h b/tools/lli/OrcLazyJIT.h
index 2b2db6e668c..5019523c7f0 100644
--- a/tools/lli/OrcLazyJIT.h
+++ b/tools/lli/OrcLazyJIT.h
@@ -21,7 +21,6 @@
 #include "llvm/ExecutionEngine/Orc/ExecutionUtils.h"
 #include "llvm/ExecutionEngine/Orc/IRCompileLayer.h"
 #include "llvm/ExecutionEngine/Orc/IRTransformLayer.h"
-#include "llvm/ExecutionEngine/Orc/LazyEmittingLayer.h"
 #include "llvm/ExecutionEngine/Orc/ObjectLinkingLayer.h"
 #include "llvm/ExecutionEngine/RTDyldMemoryManager.h"
 #include "llvm/IR/LLVMContext.h"
@@ -37,9 +36,7 @@ class OrcLazyJIT {
   typedef std::function<std::unique_ptr<Module>(std::unique_ptr<Module>)>
     TransformFtor;
   typedef orc::IRTransformLayer<CompileLayerT, TransformFtor> IRDumpLayerT;
-  typedef orc::LazyEmittingLayer<IRDumpLayerT> LazyEmitLayerT;
-  typedef orc::CompileOnDemandLayer<LazyEmitLayerT,
-                                    CompileCallbackMgr> CODLayerT;
+  typedef orc::CompileOnDemandLayer<IRDumpLayerT, CompileCallbackMgr> CODLayerT;
   typedef CODLayerT::ModuleSetHandleT ModuleHandleT;
 
   typedef std::function<
@@ -49,18 +46,17 @@ class OrcLazyJIT {
     CallbackManagerBuilder;
 
   static CallbackManagerBuilder createCallbackManagerBuilder(Triple T);
-
-  OrcLazyJIT(std::unique_ptr<TargetMachine> TM, LLVMContext &Context,
-             CallbackManagerBuilder &BuildCallbackMgr)
-    : TM(std::move(TM)),
-      Mang(this->TM->getDataLayout()),
-      ObjectLayer(),
-      CompileLayer(ObjectLayer, orc::SimpleCompiler(*this->TM)),
-      IRDumpLayer(CompileLayer, createDebugDumper()),
-      LazyEmitLayer(IRDumpLayer),
-      CCMgr(BuildCallbackMgr(IRDumpLayer, CCMgrMemMgr, Context)),
-      CODLayer(LazyEmitLayer, *CCMgr),
-      CXXRuntimeOverrides([this](const std::string &S) { return mangle(S); }) {}
+  const DataLayout &DL;
+
+  OrcLazyJIT(std::unique_ptr<TargetMachine> TM, const DataLayout &DL,
+             LLVMContext &Context, CallbackManagerBuilder &BuildCallbackMgr)
+      : DL(DL), TM(std::move(TM)), ObjectLayer(),
+        CompileLayer(ObjectLayer, orc::SimpleCompiler(*this->TM)),
+        IRDumpLayer(CompileLayer, createDebugDumper()),
+        CCMgr(BuildCallbackMgr(IRDumpLayer, CCMgrMemMgr, Context)),
+        CODLayer(IRDumpLayer, *CCMgr, false),
+        CXXRuntimeOverrides(
+            [this](const std::string &S) { return mangle(S); }) {}
 
   ~OrcLazyJIT() {
     // Run any destructors registered with __cxa_atexit.
@@ -78,7 +74,7 @@ class OrcLazyJIT {
   ModuleHandleT addModule(std::unique_ptr<Module> M) {
     // Attach a data-layout if one isn't already present.
     if (M->getDataLayout().isDefault())
-      M->setDataLayout(*TM->getDataLayout());
+      M->setDataLayout(DL);
 
     // Record the static constructors and destructors. We have to do this before
     // we hand over ownership of the module to the JIT.
@@ -92,22 +88,24 @@ class OrcLazyJIT {
     //   1) Search the JIT symbols.
     //   2) Check for C++ runtime overrides.
     //   3) Search the host process (LLI)'s symbol table.
-    auto Resolver =
+    std::shared_ptr<RuntimeDyld::SymbolResolver> Resolver =
       orc::createLambdaResolver(
         [this](const std::string &Name) {
-
           if (auto Sym = CODLayer.findSymbol(Name, true))
-            return RuntimeDyld::SymbolInfo(Sym.getAddress(), Sym.getFlags());
-
+            return RuntimeDyld::SymbolInfo(Sym.getAddress(),
+                                           Sym.getFlags());
           if (auto Sym = CXXRuntimeOverrides.searchOverrides(Name))
             return Sym;
 
-          if (auto Addr = RTDyldMemoryManager::getSymbolAddressInProcess(Name))
+          if (auto Addr =
+              RTDyldMemoryManager::getSymbolAddressInProcess(Name))
             return RuntimeDyld::SymbolInfo(Addr, JITSymbolFlags::Exported);
 
           return RuntimeDyld::SymbolInfo(nullptr);
         },
-        [](const std::string &Name) { return RuntimeDyld::SymbolInfo(nullptr); }
+        [](const std::string &Name) {
+          return RuntimeDyld::SymbolInfo(nullptr);
+        }
       );
 
     // Add the module to the JIT.
@@ -120,8 +118,7 @@ class OrcLazyJIT {
     orc::CtorDtorRunner<CODLayerT> CtorRunner(std::move(CtorNames), H);
     CtorRunner.runViaLayer(CODLayer);
 
-    IRStaticDestructorRunners.push_back(
-        orc::CtorDtorRunner<CODLayerT>(std::move(DtorNames), H));
+    IRStaticDestructorRunners.emplace_back(std::move(DtorNames), H);
 
     return H;
   }
@@ -135,12 +132,11 @@ class OrcLazyJIT {
   }
 
 private:
-
   std::string mangle(const std::string &Name) {
     std::string MangledName;
     {
       raw_string_ostream MangledNameStream(MangledName);
-      Mang.getNameWithPrefix(MangledNameStream, Name);
+      Mangler::getNameWithPrefix(MangledNameStream, Name, DL);
     }
     return MangledName;
   }
@@ -148,13 +144,11 @@ class OrcLazyJIT {
   static TransformFtor createDebugDumper();
 
   std::unique_ptr<TargetMachine> TM;
-  Mangler Mang;
   SectionMemoryManager CCMgrMemMgr;
 
   ObjLayerT ObjectLayer;
   CompileLayerT CompileLayer;
   IRDumpLayerT IRDumpLayer;
-  LazyEmitLayerT LazyEmitLayer;
   std::unique_ptr<CompileCallbackMgr> CCMgr;
   CODLayerT CODLayer;
 
diff --git a/tools/lli/lli.cpp b/tools/lli/lli.cpp
index c1e3522743a..949b444cc28 100644
--- a/tools/lli/lli.cpp
+++ b/tools/lli/lli.cpp
@@ -235,23 +235,6 @@ namespace {
                      clEnumValN(FloatABI::Hard, "hard",
                                 "Hard float ABI (uses FP registers)"),
                      clEnumValEnd));
-  cl::opt<bool>
-// In debug builds, make this default to true.
-#ifdef NDEBUG
-#define EMIT_DEBUG false
-#else
-#define EMIT_DEBUG true
-#endif
-  EmitJitDebugInfo("jit-emit-debug",
-    cl::desc("Emit debug information to debugger"),
-    cl::init(EMIT_DEBUG));
-#undef EMIT_DEBUG
-
-  static cl::opt<bool>
-  EmitJitDebugInfoToDisk("jit-emit-debug-to-disk",
-    cl::Hidden,
-    cl::desc("Emit debug info objfiles to disk"),
-    cl::init(false));
 }
 
 //===----------------------------------------------------------------------===//
@@ -279,8 +262,7 @@ class LLIObjectCache : public ObjectCache {
     if (!getCacheFilename(ModuleID, CacheName))
       return;
     if (!CacheDir.empty()) { // Create user-defined cache dir.
-      SmallString<128> dir(CacheName);
-      sys::path::remove_filename(dir);
+      SmallString<128> dir(sys::path::parent_path(CacheName));
       sys::fs::create_directories(Twine(dir));
     }
     std::error_code EC;
@@ -382,6 +364,19 @@ static void addCygMingExtraModule(ExecutionEngine *EE,
   EE->addModule(std::move(M));
 }
 
+CodeGenOpt::Level getOptLevel() {
+  switch (OptLevel) {
+  default:
+    errs() << "lli: Invalid optimization level.\n";
+    exit(1);
+  case '0': return CodeGenOpt::None;
+  case '1': return CodeGenOpt::Less;
+  case ' ':
+  case '2': return CodeGenOpt::Default;
+  case '3': return CodeGenOpt::Aggressive;
+  }
+  llvm_unreachable("Unrecognized opt level.");
+}
 
 //===----------------------------------------------------------------------===//
 // main Driver function
@@ -468,31 +463,11 @@ int main(int argc, char **argv, char * const *envp) {
     exit(1);
   }
 
-  CodeGenOpt::Level OLvl = CodeGenOpt::Default;
-  switch (OptLevel) {
-  default:
-    errs() << argv[0] << ": invalid optimization level.\n";
-    return 1;
-  case ' ': break;
-  case '0': OLvl = CodeGenOpt::None; break;
-  case '1': OLvl = CodeGenOpt::Less; break;
-  case '2': OLvl = CodeGenOpt::Default; break;
-  case '3': OLvl = CodeGenOpt::Aggressive; break;
-  }
-  builder.setOptLevel(OLvl);
+  builder.setOptLevel(getOptLevel());
 
   TargetOptions Options;
-  Options.UseSoftFloat = GenerateSoftFloatCalls;
   if (FloatABIForCalls != FloatABI::Default)
     Options.FloatABIType = FloatABIForCalls;
-  if (GenerateSoftFloatCalls)
-    FloatABIForCalls = FloatABI::Soft;
-
-  // Remote target execution doesn't handle EH or debug registration.
-  if (!RemoteMCJIT) {
-    Options.JITEmitDebugInfo = EmitJitDebugInfo;
-    Options.JITEmitDebugInfoToDisk = EmitJitDebugInfoToDisk;
-  }
 
   builder.setTargetOptions(Options);
 
diff --git a/tools/llvm-ar/CMakeLists.txt b/tools/llvm-ar/CMakeLists.txt
index 3782c87e4d3..05ffe36fa7f 100644
--- a/tools/llvm-ar/CMakeLists.txt
+++ b/tools/llvm-ar/CMakeLists.txt
@@ -1,6 +1,7 @@
 set(LLVM_LINK_COMPONENTS
   ${LLVM_TARGETS_TO_BUILD}
   Core
+  LibDriver
   Object
   Support
   )
@@ -26,4 +27,13 @@ add_custom_command(OUTPUT ${llvm_ranlib}
 add_custom_target(llvm-ranlib ALL DEPENDS ${llvm_ranlib})
 set_target_properties(llvm-ranlib PROPERTIES FOLDER Tools)
 
+set(llvm_lib "${LLVM_RUNTIME_OUTPUT_INTDIR}/llvm-lib${CMAKE_EXECUTABLE_SUFFIX}")
+
+add_custom_command(OUTPUT ${llvm_lib}
+                   COMMAND ${CMAKE_COMMAND} -E ${LLVM_LINK_OR_COPY} "${llvm_ar_binary}" "${llvm_lib}"
+                   DEPENDS llvm-ar)
+
+add_custom_target(llvm-lib ALL DEPENDS ${llvm_lib})
+set_target_properties(llvm-lib PROPERTIES FOLDER Tools)
+
 install(SCRIPT install_symlink.cmake -DCMAKE_INSTALL_PREFIX=\"${CMAKE_INSTALL_PREFIX}\")
diff --git a/tools/llvm-ar/Makefile b/tools/llvm-ar/Makefile
index e10d6ac6ab6..824bb947300 100644
--- a/tools/llvm-ar/Makefile
+++ b/tools/llvm-ar/Makefile
@@ -10,9 +10,12 @@
 LEVEL := ../..
 TOOLNAME := llvm-ar
 TOOLALIAS = llvm-ranlib
-LINK_COMPONENTS := all-targets bitreader support object
+LINK_COMPONENTS := all-targets bitreader libdriver support object
 
 # This tool has no plugins, optimize startup time.
 TOOL_NO_EXPORTS := 1
 
 include $(LEVEL)/Makefile.common
+
+all-local::
+	$(Verb) $(AliasTool) $(notdir $(ToolBuildPath)) $(ToolDir)/llvm-lib$(EXEEXT)
diff --git a/tools/llvm-ar/install_symlink.cmake b/tools/llvm-ar/install_symlink.cmake
index e313897b8b3..d48431b128d 100644
--- a/tools/llvm-ar/install_symlink.cmake
+++ b/tools/llvm-ar/install_symlink.cmake
@@ -23,3 +23,9 @@ message("Creating llvm-ranlib")
 execute_process(
   COMMAND "${CMAKE_COMMAND}" -E ${LINK_OR_COPY} "llvm-ar${EXECUTABLE_SUFFIX}" "llvm-ranlib${EXECUTABLE_SUFFIX}"
   WORKING_DIRECTORY "${bindir}")
+
+message("Creating llvm-lib")
+
+execute_process(
+  COMMAND "${CMAKE_COMMAND}" -E ${LINK_OR_COPY} "llvm-ar${EXECUTABLE_SUFFIX}" "llvm-lib${EXECUTABLE_SUFFIX}"
+  WORKING_DIRECTORY "${bindir}")
diff --git a/tools/llvm-ar/llvm-ar.cpp b/tools/llvm-ar/llvm-ar.cpp
index f813465daaf..ec3cfcb5cad 100644
--- a/tools/llvm-ar/llvm-ar.cpp
+++ b/tools/llvm-ar/llvm-ar.cpp
@@ -13,9 +13,12 @@
 //===----------------------------------------------------------------------===//
 
 #include "llvm/ADT/StringSwitch.h"
+#include "llvm/ADT/Triple.h"
 #include "llvm/IR/LLVMContext.h"
 #include "llvm/IR/Module.h"
+#include "llvm/LibDriver/LibDriver.h"
 #include "llvm/Object/Archive.h"
+#include "llvm/Object/ArchiveWriter.h"
 #include "llvm/Object/ObjectFile.h"
 #include "llvm/Support/CommandLine.h"
 #include "llvm/Support/Errc.h"
@@ -45,16 +48,9 @@ using namespace llvm;
 // The name this program was invoked as.
 static StringRef ToolName;
 
-static const char *TemporaryOutput;
-static int TmpArchiveFD = -1;
-
 // Show the error message and exit.
 LLVM_ATTRIBUTE_NORETURN static void fail(Twine Error) {
   outs() << ToolName << ": " << Error << ".\n";
-  if (TmpArchiveFD != -1)
-    close(TmpArchiveFD);
-  if (TemporaryOutput)
-    sys::fs::remove(TemporaryOutput);
   exit(1);
 }
 
@@ -75,6 +71,16 @@ static cl::list<std::string>
 
 static cl::opt<bool> MRI("M", cl::desc(""));
 
+namespace {
+enum Format { Default, GNU, BSD };
+}
+
+static cl::opt<Format>
+    FormatOpt("format", cl::desc("Archive format to create"),
+              cl::values(clEnumValN(Default, "defalut", "default"),
+                         clEnumValN(GNU, "gnu", "gnu"),
+                         clEnumValN(BSD, "bsd", "bsd"), clEnumValEnd));
+
 std::string Options;
 
 // Provide additional help output explaining the operations and modifiers of
@@ -123,6 +129,8 @@ static bool OriginalDates = false; ///< 'o' modifier
 static bool OnlyUpdate = false;    ///< 'u' modifier
 static bool Verbose = false;       ///< 'v' modifier
 static bool Symtab = true;         ///< 's' modifier
+static bool Deterministic = true;  ///< 'D' and 'U' modifiers
+static bool Thin = false;          ///< 'T' modifier
 
 // Relative Positional Argument (for insert/move). This variable holds
 // the name of the archive member to which the 'a', 'b' or 'i' modifier
@@ -239,6 +247,15 @@ static ArchiveOperation parseCommandLine() {
       AddBefore = true;
       NumPositional++;
       break;
+    case 'D':
+      Deterministic = true;
+      break;
+    case 'U':
+      Deterministic = false;
+      break;
+    case 'T':
+      Thin = true;
+      break;
     default:
       cl::PrintHelpMessage();
     }
@@ -282,11 +299,13 @@ static ArchiveOperation parseCommandLine() {
 
 // Implements the 'p' operation. This function traverses the archive
 // looking for members that match the path list.
-static void doPrint(StringRef Name, object::Archive::child_iterator I) {
+static void doPrint(StringRef Name, const object::Archive::Child &C) {
   if (Verbose)
     outs() << "Printing " << Name << "\n";
 
-  StringRef Data = I->getBuffer();
+  ErrorOr<StringRef> DataOrErr = C.getBuffer();
+  failIfError(DataOrErr.getError());
+  StringRef Data = *DataOrErr;
   outs().write(Data.data(), Data.size());
 }
 
@@ -311,16 +330,16 @@ static void printMode(unsigned mode) {
 // the file names of each of the members. However, if verbose mode is requested
 // ('v' modifier) then the file type, permission mode, user, group, size, and
 // modification time are also printed.
-static void doDisplayTable(StringRef Name, object::Archive::child_iterator I) {
+static void doDisplayTable(StringRef Name, const object::Archive::Child &C) {
   if (Verbose) {
-    sys::fs::perms Mode = I->getAccessMode();
+    sys::fs::perms Mode = C.getAccessMode();
     printMode((Mode >> 6) & 007);
     printMode((Mode >> 3) & 007);
     printMode(Mode & 007);
-    outs() << ' ' << I->getUID();
-    outs() << '/' << I->getGID();
-    outs() << ' ' << format("%6llu", I->getSize());
-    outs() << ' ' << I->getLastModified().str();
+    outs() << ' ' << C.getUID();
+    outs() << '/' << C.getGID();
+    outs() << ' ' << format("%6llu", C.getSize());
+    outs() << ' ' << C.getLastModified().str();
     outs() << ' ';
   }
   outs() << Name << "\n";
@@ -328,9 +347,9 @@ static void doDisplayTable(StringRef Name, object::Archive::child_iterator I) {
 
 // Implement the 'x' operation. This function extracts files back to the file
 // system.
-static void doExtract(StringRef Name, object::Archive::child_iterator I) {
+static void doExtract(StringRef Name, const object::Archive::Child &C) {
   // Retain the original mode.
-  sys::fs::perms Mode = I->getAccessMode();
+  sys::fs::perms Mode = C.getAccessMode();
   SmallString<128> Storage = Name;
 
   int FD;
@@ -342,7 +361,7 @@ static void doExtract(StringRef Name, object::Archive::child_iterator I) {
     raw_fd_ostream file(FD, false);
 
     // Get the data and its length
-    StringRef Data = I->getBuffer();
+    StringRef Data = *C.getBuffer();
 
     // Write the data.
     file.write(Data.data(), Data.size());
@@ -352,7 +371,7 @@ static void doExtract(StringRef Name, object::Archive::child_iterator I) {
   // now.
   if (OriginalDates)
     failIfError(
-        sys::fs::setLastModificationAndAccessTime(FD, I->getLastModified()));
+        sys::fs::setLastModificationAndAccessTime(FD, C.getLastModified()));
 
   if (close(FD))
     fail("Could not close the file");
@@ -378,100 +397,59 @@ static bool shouldCreateArchive(ArchiveOperation Op) {
 
 static void performReadOperation(ArchiveOperation Operation,
                                  object::Archive *OldArchive) {
-  for (object::Archive::child_iterator I = OldArchive->child_begin(),
-                                       E = OldArchive->child_end();
-       I != E; ++I) {
-    ErrorOr<StringRef> NameOrErr = I->getName();
+  if (Operation == Extract && OldArchive->isThin()) {
+    errs() << "extracting from a thin archive is not supported\n";
+    std::exit(1);
+  }
+
+  bool Filter = !Members.empty();
+  for (const object::Archive::Child &C : OldArchive->children()) {
+    ErrorOr<StringRef> NameOrErr = C.getName();
     failIfError(NameOrErr.getError());
     StringRef Name = NameOrErr.get();
 
-    if (!Members.empty() &&
-        std::find(Members.begin(), Members.end(), Name) == Members.end())
-      continue;
+    if (Filter) {
+      auto I = std::find(Members.begin(), Members.end(), Name);
+      if (I == Members.end())
+        continue;
+      Members.erase(I);
+    }
 
     switch (Operation) {
     default:
       llvm_unreachable("Not a read operation");
     case Print:
-      doPrint(Name, I);
+      doPrint(Name, C);
       break;
     case DisplayTable:
-      doDisplayTable(Name, I);
+      doDisplayTable(Name, C);
       break;
     case Extract:
-      doExtract(Name, I);
+      doExtract(Name, C);
       break;
     }
   }
+  if (Members.empty())
+    return;
+  for (StringRef Name : Members)
+    errs() << Name << " was not found\n";
+  std::exit(1);
 }
 
-namespace {
-class NewArchiveIterator {
-  bool IsNewMember;
-  StringRef Name;
-
-  object::Archive::child_iterator OldI;
-
-  StringRef NewFilename;
-
-public:
-  NewArchiveIterator(object::Archive::child_iterator I, StringRef Name);
-  NewArchiveIterator(StringRef I, StringRef Name);
-  NewArchiveIterator();
-  bool isNewMember() const;
-  StringRef getName() const;
-
-  object::Archive::child_iterator getOld() const;
-
-  StringRef getNew() const;
-  int getFD(sys::fs::file_status &NewStatus) const;
-  const sys::fs::file_status &getStatus() const;
-};
-}
-
-NewArchiveIterator::NewArchiveIterator() {}
-
-NewArchiveIterator::NewArchiveIterator(object::Archive::child_iterator I,
-                                       StringRef Name)
-    : IsNewMember(false), Name(Name), OldI(I) {}
-
-NewArchiveIterator::NewArchiveIterator(StringRef NewFilename, StringRef Name)
-    : IsNewMember(true), Name(Name), NewFilename(NewFilename) {}
-
-StringRef NewArchiveIterator::getName() const { return Name; }
-
-bool NewArchiveIterator::isNewMember() const { return IsNewMember; }
-
-object::Archive::child_iterator NewArchiveIterator::getOld() const {
-  assert(!IsNewMember);
-  return OldI;
-}
-
-StringRef NewArchiveIterator::getNew() const {
-  assert(IsNewMember);
-  return NewFilename;
-}
-
-int NewArchiveIterator::getFD(sys::fs::file_status &NewStatus) const {
-  assert(IsNewMember);
-  int NewFD;
-  failIfError(sys::fs::openFileForRead(NewFilename, NewFD), NewFilename);
-  assert(NewFD != -1);
-
-  failIfError(sys::fs::status(NewFD, NewStatus), NewFilename);
-
-  // Opening a directory doesn't make sense. Let it fail.
-  // Linux cannot open directories with open(2), although
-  // cygwin and *bsd can.
-  if (NewStatus.type() == sys::fs::file_type::directory_file)
-    failIfError(make_error_code(errc::is_a_directory), NewFilename);
-
-  return NewFD;
+void addMember(std::vector<NewArchiveIterator> &Members, StringRef FileName,
+               int Pos = -1) {
+  NewArchiveIterator NI(FileName);
+  if (Pos == -1)
+    Members.push_back(NI);
+  else
+    Members[Pos] = NI;
 }
 
-template <typename T>
-void addMember(std::vector<NewArchiveIterator> &Members, T I, StringRef Name,
+void addMember(std::vector<NewArchiveIterator> &Members,
+               object::Archive::child_iterator I, StringRef Name,
                int Pos = -1) {
+  if (Thin && !I->getParent()->isThin())
+    fail("Cannot convert a regular archive to a thin one");
   NewArchiveIterator NI(I, Name);
   if (Pos == -1)
     Members.push_back(NI);
@@ -566,7 +544,7 @@ computeNewArchiveMembers(ArchiveOperation Operation,
         addMember(Ret, Child, Name);
         break;
       case IA_AddNewMeber:
-        addMember(Ret, *MemberI, Name);
+        addMember(Ret, *MemberI);
         break;
       case IA_Delete:
         break;
@@ -574,7 +552,7 @@ computeNewArchiveMembers(ArchiveOperation Operation,
         addMember(Moved, Child, Name);
         break;
       case IA_MoveNewMember:
-        addMember(Moved, *MemberI, Name);
+        addMember(Moved, *MemberI);
         break;
       }
       if (MemberI != Members.end())
@@ -594,280 +572,47 @@ computeNewArchiveMembers(ArchiveOperation Operation,
   assert(unsigned(InsertPos) <= Ret.size());
   Ret.insert(Ret.begin() + InsertPos, Moved.begin(), Moved.end());
 
-  Ret.insert(Ret.begin() + InsertPos, Members.size(), NewArchiveIterator());
+  Ret.insert(Ret.begin() + InsertPos, Members.size(), NewArchiveIterator(""));
   int Pos = InsertPos;
   for (auto &Member : Members) {
-    StringRef Name = sys::path::filename(Member);
-    addMember(Ret, Member, Name, Pos);
+    addMember(Ret, Member, Pos);
     ++Pos;
   }
 
   return Ret;
 }
 
-template <typename T>
-static void printWithSpacePadding(raw_fd_ostream &OS, T Data, unsigned Size,
-				  bool MayTruncate = false) {
-  uint64_t OldPos = OS.tell();
-  OS << Data;
-  unsigned SizeSoFar = OS.tell() - OldPos;
-  if (Size > SizeSoFar) {
-    unsigned Remaining = Size - SizeSoFar;
-    for (unsigned I = 0; I < Remaining; ++I)
-      OS << ' ';
-  } else if (Size < SizeSoFar) {
-    assert(MayTruncate && "Data doesn't fit in Size");
-    // Some of the data this is used for (like UID) can be larger than the
-    // space available in the archive format. Truncate in that case.
-    OS.seek(OldPos + Size);
-  }
-}
-
-static void print32BE(raw_fd_ostream &Out, unsigned Val) {
-  for (int I = 3; I >= 0; --I) {
-    char V = (Val >> (8 * I)) & 0xff;
-    Out << V;
-  }
-}
-
-static void printRestOfMemberHeader(raw_fd_ostream &Out,
-                                    const sys::TimeValue &ModTime, unsigned UID,
-                                    unsigned GID, unsigned Perms,
-                                    unsigned Size) {
-  printWithSpacePadding(Out, ModTime.toEpochTime(), 12);
-  printWithSpacePadding(Out, UID, 6, true);
-  printWithSpacePadding(Out, GID, 6, true);
-  printWithSpacePadding(Out, format("%o", Perms), 8);
-  printWithSpacePadding(Out, Size, 10);
-  Out << "`\n";
-}
-
-static void printMemberHeader(raw_fd_ostream &Out, StringRef Name,
-                              const sys::TimeValue &ModTime, unsigned UID,
-                              unsigned GID, unsigned Perms, unsigned Size) {
-  printWithSpacePadding(Out, Twine(Name) + "/", 16);
-  printRestOfMemberHeader(Out, ModTime, UID, GID, Perms, Size);
-}
-
-static void printMemberHeader(raw_fd_ostream &Out, unsigned NameOffset,
-                              const sys::TimeValue &ModTime, unsigned UID,
-                              unsigned GID, unsigned Perms, unsigned Size) {
-  Out << '/';
-  printWithSpacePadding(Out, NameOffset, 15);
-  printRestOfMemberHeader(Out, ModTime, UID, GID, Perms, Size);
-}
-
-static void writeStringTable(raw_fd_ostream &Out,
-                             ArrayRef<NewArchiveIterator> Members,
-                             std::vector<unsigned> &StringMapIndexes) {
-  unsigned StartOffset = 0;
-  for (ArrayRef<NewArchiveIterator>::iterator I = Members.begin(),
-                                              E = Members.end();
-       I != E; ++I) {
-    StringRef Name = I->getName();
-    if (Name.size() < 16)
-      continue;
-    if (StartOffset == 0) {
-      printWithSpacePadding(Out, "//", 58);
-      Out << "`\n";
-      StartOffset = Out.tell();
-    }
-    StringMapIndexes.push_back(Out.tell() - StartOffset);
-    Out << Name << "/\n";
-  }
-  if (StartOffset == 0)
-    return;
-  if (Out.tell() % 2)
-    Out << '\n';
-  int Pos = Out.tell();
-  Out.seek(StartOffset - 12);
-  printWithSpacePadding(Out, Pos - StartOffset, 10);
-  Out.seek(Pos);
-}
-
-// Returns the offset of the first reference to a member offset.
-static unsigned writeSymbolTable(raw_fd_ostream &Out,
-                                 ArrayRef<NewArchiveIterator> Members,
-                                 ArrayRef<MemoryBufferRef> Buffers,
-                                 std::vector<unsigned> &MemberOffsetRefs) {
-  unsigned StartOffset = 0;
-  unsigned MemberNum = 0;
-  std::string NameBuf;
-  raw_string_ostream NameOS(NameBuf);
-  unsigned NumSyms = 0;
-  LLVMContext &Context = getGlobalContext();
-  for (ArrayRef<NewArchiveIterator>::iterator I = Members.begin(),
-                                              E = Members.end();
-       I != E; ++I, ++MemberNum) {
-    MemoryBufferRef MemberBuffer = Buffers[MemberNum];
-    ErrorOr<std::unique_ptr<object::SymbolicFile>> ObjOrErr =
-        object::SymbolicFile::createSymbolicFile(
-            MemberBuffer, sys::fs::file_magic::unknown, &Context);
-    if (!ObjOrErr)
-      continue;  // FIXME: check only for "not an object file" errors.
-    object::SymbolicFile &Obj = *ObjOrErr.get();
-
-    if (!StartOffset) {
-      printMemberHeader(Out, "", sys::TimeValue::now(), 0, 0, 0, 0);
-      StartOffset = Out.tell();
-      print32BE(Out, 0);
-    }
-
-    for (const object::BasicSymbolRef &S : Obj.symbols()) {
-      uint32_t Symflags = S.getFlags();
-      if (Symflags & object::SymbolRef::SF_FormatSpecific)
-        continue;
-      if (!(Symflags & object::SymbolRef::SF_Global))
-        continue;
-      if (Symflags & object::SymbolRef::SF_Undefined)
-        continue;
-      failIfError(S.printName(NameOS));
-      NameOS << '\0';
-      ++NumSyms;
-      MemberOffsetRefs.push_back(MemberNum);
-      print32BE(Out, 0);
-    }
-  }
-  Out << NameOS.str();
-
-  if (StartOffset == 0)
-    return 0;
-
-  if (Out.tell() % 2)
-    Out << '\0';
-
-  unsigned Pos = Out.tell();
-  Out.seek(StartOffset - 12);
-  printWithSpacePadding(Out, Pos - StartOffset, 10);
-  Out.seek(StartOffset);
-  print32BE(Out, NumSyms);
-  Out.seek(Pos);
-  return StartOffset + 4;
-}
-
 static void
 performWriteOperation(ArchiveOperation Operation, object::Archive *OldArchive,
-                      std::vector<NewArchiveIterator> &NewMembers) {
-  SmallString<128> TmpArchive;
-  failIfError(sys::fs::createUniqueFile(ArchiveName + ".temp-archive-%%%%%%%.a",
-                                        TmpArchiveFD, TmpArchive));
-
-  TemporaryOutput = TmpArchive.c_str();
-  tool_output_file Output(TemporaryOutput, TmpArchiveFD);
-  raw_fd_ostream &Out = Output.os();
-  Out << "!<arch>\n";
-
-  std::vector<unsigned> MemberOffsetRefs;
-
-  std::vector<std::unique_ptr<MemoryBuffer>> Buffers;
-  std::vector<MemoryBufferRef> Members;
-  std::vector<sys::fs::file_status> NewMemberStatus;
-
-  for (unsigned I = 0, N = NewMembers.size(); I < N; ++I) {
-    NewArchiveIterator &Member = NewMembers[I];
-    MemoryBufferRef MemberRef;
-
-    if (Member.isNewMember()) {
-      StringRef Filename = Member.getNew();
-      NewMemberStatus.resize(NewMemberStatus.size() + 1);
-      sys::fs::file_status &Status = NewMemberStatus.back();
-      int FD = Member.getFD(Status);
-      ErrorOr<std::unique_ptr<MemoryBuffer>> MemberBufferOrErr =
-          MemoryBuffer::getOpenFile(FD, Filename, Status.getSize(), false);
-      failIfError(MemberBufferOrErr.getError(), Filename);
-      if (close(FD) != 0)
-        fail("Could not close file");
-      Buffers.push_back(std::move(MemberBufferOrErr.get()));
-      MemberRef = Buffers.back()->getMemBufferRef();
-    } else {
-      object::Archive::child_iterator OldMember = Member.getOld();
-      ErrorOr<MemoryBufferRef> MemberBufferOrErr =
-          OldMember->getMemoryBufferRef();
-      failIfError(MemberBufferOrErr.getError());
-      MemberRef = MemberBufferOrErr.get();
-    }
-    Members.push_back(MemberRef);
-  }
-
-  unsigned MemberReferenceOffset = 0;
-  if (Symtab) {
-    MemberReferenceOffset =
-        writeSymbolTable(Out, NewMembers, Members, MemberOffsetRefs);
-  }
-
-  std::vector<unsigned> StringMapIndexes;
-  writeStringTable(Out, NewMembers, StringMapIndexes);
-
-  unsigned MemberNum = 0;
-  unsigned LongNameMemberNum = 0;
-  unsigned NewMemberNum = 0;
-  std::vector<unsigned> MemberOffset;
-  for (std::vector<NewArchiveIterator>::iterator I = NewMembers.begin(),
-                                                 E = NewMembers.end();
-       I != E; ++I, ++MemberNum) {
-
-    unsigned Pos = Out.tell();
-    MemberOffset.push_back(Pos);
-
-    MemoryBufferRef File = Members[MemberNum];
-    if (I->isNewMember()) {
-      StringRef FileName = I->getNew();
-      const sys::fs::file_status &Status = NewMemberStatus[NewMemberNum];
-      NewMemberNum++;
-
-      StringRef Name = sys::path::filename(FileName);
-      if (Name.size() < 16)
-        printMemberHeader(Out, Name, Status.getLastModificationTime(),
-                          Status.getUser(), Status.getGroup(),
-                          Status.permissions(), Status.getSize());
-      else
-        printMemberHeader(Out, StringMapIndexes[LongNameMemberNum++],
-                          Status.getLastModificationTime(), Status.getUser(),
-                          Status.getGroup(), Status.permissions(),
-                          Status.getSize());
-    } else {
-      object::Archive::child_iterator OldMember = I->getOld();
-      StringRef Name = I->getName();
-
-      if (Name.size() < 16)
-        printMemberHeader(Out, Name, OldMember->getLastModified(),
-                          OldMember->getUID(), OldMember->getGID(),
-                          OldMember->getAccessMode(), OldMember->getSize());
-      else
-        printMemberHeader(Out, StringMapIndexes[LongNameMemberNum++],
-                          OldMember->getLastModified(), OldMember->getUID(),
-                          OldMember->getGID(), OldMember->getAccessMode(),
-                          OldMember->getSize());
-    }
-
-    Out << File.getBuffer();
-
-    if (Out.tell() % 2)
-      Out << '\n';
+                      std::vector<NewArchiveIterator> *NewMembersP) {
+  object::Archive::Kind Kind;
+  switch (FormatOpt) {
+  case Default: {
+    Triple T(sys::getProcessTriple());
+    if (T.isOSDarwin())
+      Kind = object::Archive::K_BSD;
+    else
+      Kind = object::Archive::K_GNU;
+    break;
   }
-
-  if (MemberReferenceOffset) {
-    Out.seek(MemberReferenceOffset);
-    for (unsigned MemberNum : MemberOffsetRefs)
-      print32BE(Out, MemberOffset[MemberNum]);
+  case GNU:
+    Kind = object::Archive::K_GNU;
+    break;
+  case BSD:
+    Kind = object::Archive::K_BSD;
+    break;
   }
-
-  Output.keep();
-  Out.close();
-  sys::fs::rename(TemporaryOutput, ArchiveName);
-  TemporaryOutput = nullptr;
-}
-
-static void
-performWriteOperation(ArchiveOperation Operation, object::Archive *OldArchive,
-                      std::vector<NewArchiveIterator> *NewMembersP) {
   if (NewMembersP) {
-    performWriteOperation(Operation, OldArchive, *NewMembersP);
+    std::pair<StringRef, std::error_code> Result = writeArchive(
+        ArchiveName, *NewMembersP, Symtab, Kind, Deterministic, Thin);
+    failIfError(Result.second, Result.first);
     return;
   }
   std::vector<NewArchiveIterator> NewMembers =
       computeNewArchiveMembers(Operation, OldArchive);
-  performWriteOperation(Operation, OldArchive, NewMembers);
+  auto Result =
+      writeArchive(ArchiveName, NewMembers, Symtab, Kind, Deterministic, Thin);
+  failIfError(Result.second, Result.first);
 }
 
 static void createSymbolTable(object::Archive *OldArchive) {
@@ -989,7 +734,7 @@ static void runMRIScript() {
       break;
     }
     case MRICommand::AddMod:
-      addMember(NewMembers, Rest, sys::path::filename(Rest));
+      addMember(NewMembers, Rest);
       break;
     case MRICommand::Create:
       Create = true;
@@ -1036,6 +781,15 @@ int main(int argc, char **argv) {
   PrettyStackTraceProgram X(argc, argv);
   llvm_shutdown_obj Y;  // Call llvm_shutdown() on exit.
 
+  llvm::InitializeAllTargetInfos();
+  llvm::InitializeAllTargetMCs();
+  llvm::InitializeAllAsmParsers();
+
+  StringRef Stem = sys::path::stem(ToolName);
+  if (Stem.find("ranlib") == StringRef::npos &&
+      Stem.find("lib") != StringRef::npos)
+    return libDriverMain(makeArrayRef(argv, argc));
+
   // Have the command line options parsed and handle things
   // like --help and --version.
   cl::ParseCommandLineOptions(argc, argv,
@@ -1043,14 +797,9 @@ int main(int argc, char **argv) {
     "  This program archives bitcode files into single libraries\n"
   );
 
-  llvm::InitializeAllTargetInfos();
-  llvm::InitializeAllTargetMCs();
-  llvm::InitializeAllAsmParsers();
-
-  StringRef Stem = sys::path::stem(ToolName);
   if (Stem.find("ar") != StringRef::npos)
     return ar_main();
   if (Stem.find("ranlib") != StringRef::npos)
     return ranlib_main();
-  fail("Not ranlib or ar!");
+  fail("Not ranlib, ar or lib!");
 }
diff --git a/tools/llvm-as/llvm-as.cpp b/tools/llvm-as/llvm-as.cpp
index c6b3f934202..4455d24fb60 100644
--- a/tools/llvm-as/llvm-as.cpp
+++ b/tools/llvm-as/llvm-as.cpp
@@ -62,14 +62,8 @@ static void WriteOutputFile(const Module *M) {
     if (InputFilename == "-") {
       OutputFilename = "-";
     } else {
-      std::string IFN = InputFilename;
-      int Len = IFN.length();
-      if (IFN[Len-3] == '.' && IFN[Len-2] == 'l' && IFN[Len-1] == 'l') {
-        // Source ends in .ll
-        OutputFilename = std::string(IFN.begin(), IFN.end()-3);
-      } else {
-        OutputFilename = IFN;   // Append a .bc to it
-      }
+      StringRef IFN = InputFilename;
+      OutputFilename = (IFN.endswith(".ll") ? IFN.drop_back(3) : IFN).str();
       OutputFilename += ".bc";
     }
   }
diff --git a/tools/llvm-bcanalyzer/llvm-bcanalyzer.cpp b/tools/llvm-bcanalyzer/llvm-bcanalyzer.cpp
index 8aa5697f6ad..33b876e266c 100644
--- a/tools/llvm-bcanalyzer/llvm-bcanalyzer.cpp
+++ b/tools/llvm-bcanalyzer/llvm-bcanalyzer.cpp
@@ -66,6 +66,10 @@ static cl::opt<std::string>
   BlockInfoFilename("block-info",
                     cl::desc("Use the BLOCK_INFO from the given file"));
 
+static cl::opt<bool>
+  ShowBinaryBlobs("show-binary-blobs",
+                  cl::desc("Print binary blobs using hex escapes"));
+
 namespace {
 
 /// CurStreamTypeType - A type for CurStreamType
@@ -142,26 +146,30 @@ static const char *GetCodeName(unsigned CodeID, unsigned BlockID,
 
   if (CurStreamType != LLVMIRBitstream) return nullptr;
 
+#define STRINGIFY_CODE(PREFIX, CODE)                                           \
+  case bitc::PREFIX##_##CODE:                                                  \
+    return #CODE;
   switch (BlockID) {
   default: return nullptr;
   case bitc::MODULE_BLOCK_ID:
     switch (CodeID) {
     default: return nullptr;
-    case bitc::MODULE_CODE_VERSION:     return "VERSION";
-    case bitc::MODULE_CODE_TRIPLE:      return "TRIPLE";
-    case bitc::MODULE_CODE_DATALAYOUT:  return "DATALAYOUT";
-    case bitc::MODULE_CODE_ASM:         return "ASM";
-    case bitc::MODULE_CODE_SECTIONNAME: return "SECTIONNAME";
-    case bitc::MODULE_CODE_DEPLIB:      return "DEPLIB"; // FIXME: Remove in 4.0
-    case bitc::MODULE_CODE_GLOBALVAR:   return "GLOBALVAR";
-    case bitc::MODULE_CODE_FUNCTION:    return "FUNCTION";
-    case bitc::MODULE_CODE_ALIAS:       return "ALIAS";
-    case bitc::MODULE_CODE_PURGEVALS:   return "PURGEVALS";
-    case bitc::MODULE_CODE_GCNAME:      return "GCNAME";
+      STRINGIFY_CODE(MODULE_CODE, VERSION)
+      STRINGIFY_CODE(MODULE_CODE, TRIPLE)
+      STRINGIFY_CODE(MODULE_CODE, DATALAYOUT)
+      STRINGIFY_CODE(MODULE_CODE, ASM)
+      STRINGIFY_CODE(MODULE_CODE, SECTIONNAME)
+      STRINGIFY_CODE(MODULE_CODE, DEPLIB) // FIXME: Remove in 4.0
+      STRINGIFY_CODE(MODULE_CODE, GLOBALVAR)
+      STRINGIFY_CODE(MODULE_CODE, FUNCTION)
+      STRINGIFY_CODE(MODULE_CODE, ALIAS)
+      STRINGIFY_CODE(MODULE_CODE, PURGEVALS)
+      STRINGIFY_CODE(MODULE_CODE, GCNAME)
     }
   case bitc::PARAMATTR_BLOCK_ID:
     switch (CodeID) {
     default: return nullptr;
+    // FIXME: Should these be different?
     case bitc::PARAMATTR_CODE_ENTRY_OLD: return "ENTRY";
     case bitc::PARAMATTR_CODE_ENTRY:     return "ENTRY";
     case bitc::PARAMATTR_GRP_CODE_ENTRY: return "ENTRY";
@@ -169,112 +177,134 @@ static const char *GetCodeName(unsigned CodeID, unsigned BlockID,
   case bitc::TYPE_BLOCK_ID_NEW:
     switch (CodeID) {
     default: return nullptr;
-    case bitc::TYPE_CODE_NUMENTRY:     return "NUMENTRY";
-    case bitc::TYPE_CODE_VOID:         return "VOID";
-    case bitc::TYPE_CODE_FLOAT:        return "FLOAT";
-    case bitc::TYPE_CODE_DOUBLE:       return "DOUBLE";
-    case bitc::TYPE_CODE_LABEL:        return "LABEL";
-    case bitc::TYPE_CODE_OPAQUE:       return "OPAQUE";
-    case bitc::TYPE_CODE_INTEGER:      return "INTEGER";
-    case bitc::TYPE_CODE_POINTER:      return "POINTER";
-    case bitc::TYPE_CODE_ARRAY:        return "ARRAY";
-    case bitc::TYPE_CODE_VECTOR:       return "VECTOR";
-    case bitc::TYPE_CODE_X86_FP80:     return "X86_FP80";
-    case bitc::TYPE_CODE_FP128:        return "FP128";
-    case bitc::TYPE_CODE_PPC_FP128:    return "PPC_FP128";
-    case bitc::TYPE_CODE_METADATA:     return "METADATA";
-    case bitc::TYPE_CODE_STRUCT_ANON:  return "STRUCT_ANON";
-    case bitc::TYPE_CODE_STRUCT_NAME:  return "STRUCT_NAME";
-    case bitc::TYPE_CODE_STRUCT_NAMED: return "STRUCT_NAMED";
-    case bitc::TYPE_CODE_FUNCTION:     return "FUNCTION";
+      STRINGIFY_CODE(TYPE_CODE, NUMENTRY)
+      STRINGIFY_CODE(TYPE_CODE, VOID)
+      STRINGIFY_CODE(TYPE_CODE, FLOAT)
+      STRINGIFY_CODE(TYPE_CODE, DOUBLE)
+      STRINGIFY_CODE(TYPE_CODE, LABEL)
+      STRINGIFY_CODE(TYPE_CODE, OPAQUE)
+      STRINGIFY_CODE(TYPE_CODE, INTEGER)
+      STRINGIFY_CODE(TYPE_CODE, POINTER)
+      STRINGIFY_CODE(TYPE_CODE, ARRAY)
+      STRINGIFY_CODE(TYPE_CODE, VECTOR)
+      STRINGIFY_CODE(TYPE_CODE, X86_FP80)
+      STRINGIFY_CODE(TYPE_CODE, FP128)
+      STRINGIFY_CODE(TYPE_CODE, PPC_FP128)
+      STRINGIFY_CODE(TYPE_CODE, METADATA)
+      STRINGIFY_CODE(TYPE_CODE, STRUCT_ANON)
+      STRINGIFY_CODE(TYPE_CODE, STRUCT_NAME)
+      STRINGIFY_CODE(TYPE_CODE, STRUCT_NAMED)
+      STRINGIFY_CODE(TYPE_CODE, FUNCTION)
     }
 
   case bitc::CONSTANTS_BLOCK_ID:
     switch (CodeID) {
     default: return nullptr;
-    case bitc::CST_CODE_SETTYPE:         return "SETTYPE";
-    case bitc::CST_CODE_NULL:            return "NULL";
-    case bitc::CST_CODE_UNDEF:           return "UNDEF";
-    case bitc::CST_CODE_INTEGER:         return "INTEGER";
-    case bitc::CST_CODE_WIDE_INTEGER:    return "WIDE_INTEGER";
-    case bitc::CST_CODE_FLOAT:           return "FLOAT";
-    case bitc::CST_CODE_AGGREGATE:       return "AGGREGATE";
-    case bitc::CST_CODE_STRING:          return "STRING";
-    case bitc::CST_CODE_CSTRING:         return "CSTRING";
-    case bitc::CST_CODE_CE_BINOP:        return "CE_BINOP";
-    case bitc::CST_CODE_CE_CAST:         return "CE_CAST";
-    case bitc::CST_CODE_CE_GEP:          return "CE_GEP";
-    case bitc::CST_CODE_CE_INBOUNDS_GEP: return "CE_INBOUNDS_GEP";
-    case bitc::CST_CODE_CE_SELECT:       return "CE_SELECT";
-    case bitc::CST_CODE_CE_EXTRACTELT:   return "CE_EXTRACTELT";
-    case bitc::CST_CODE_CE_INSERTELT:    return "CE_INSERTELT";
-    case bitc::CST_CODE_CE_SHUFFLEVEC:   return "CE_SHUFFLEVEC";
-    case bitc::CST_CODE_CE_CMP:          return "CE_CMP";
-    case bitc::CST_CODE_INLINEASM:       return "INLINEASM";
-    case bitc::CST_CODE_CE_SHUFVEC_EX:   return "CE_SHUFVEC_EX";
+      STRINGIFY_CODE(CST_CODE, SETTYPE)
+      STRINGIFY_CODE(CST_CODE, NULL)
+      STRINGIFY_CODE(CST_CODE, UNDEF)
+      STRINGIFY_CODE(CST_CODE, INTEGER)
+      STRINGIFY_CODE(CST_CODE, WIDE_INTEGER)
+      STRINGIFY_CODE(CST_CODE, FLOAT)
+      STRINGIFY_CODE(CST_CODE, AGGREGATE)
+      STRINGIFY_CODE(CST_CODE, STRING)
+      STRINGIFY_CODE(CST_CODE, CSTRING)
+      STRINGIFY_CODE(CST_CODE, CE_BINOP)
+      STRINGIFY_CODE(CST_CODE, CE_CAST)
+      STRINGIFY_CODE(CST_CODE, CE_GEP)
+      STRINGIFY_CODE(CST_CODE, CE_INBOUNDS_GEP)
+      STRINGIFY_CODE(CST_CODE, CE_SELECT)
+      STRINGIFY_CODE(CST_CODE, CE_EXTRACTELT)
+      STRINGIFY_CODE(CST_CODE, CE_INSERTELT)
+      STRINGIFY_CODE(CST_CODE, CE_SHUFFLEVEC)
+      STRINGIFY_CODE(CST_CODE, CE_CMP)
+      STRINGIFY_CODE(CST_CODE, INLINEASM)
+      STRINGIFY_CODE(CST_CODE, CE_SHUFVEC_EX)
     case bitc::CST_CODE_BLOCKADDRESS:    return "CST_CODE_BLOCKADDRESS";
-    case bitc::CST_CODE_DATA:            return "DATA";
+      STRINGIFY_CODE(CST_CODE, DATA)
     }
   case bitc::FUNCTION_BLOCK_ID:
     switch (CodeID) {
     default: return nullptr;
-    case bitc::FUNC_CODE_DECLAREBLOCKS: return "DECLAREBLOCKS";
-
-    case bitc::FUNC_CODE_INST_BINOP:        return "INST_BINOP";
-    case bitc::FUNC_CODE_INST_CAST:         return "INST_CAST";
-    case bitc::FUNC_CODE_INST_GEP_OLD:
-      return "INST_GEP_OLD";
-    case bitc::FUNC_CODE_INST_INBOUNDS_GEP_OLD:
-      return "INST_INBOUNDS_GEP_OLD";
-    case bitc::FUNC_CODE_INST_SELECT:       return "INST_SELECT";
-    case bitc::FUNC_CODE_INST_EXTRACTELT:   return "INST_EXTRACTELT";
-    case bitc::FUNC_CODE_INST_INSERTELT:    return "INST_INSERTELT";
-    case bitc::FUNC_CODE_INST_SHUFFLEVEC:   return "INST_SHUFFLEVEC";
-    case bitc::FUNC_CODE_INST_CMP:          return "INST_CMP";
-
-    case bitc::FUNC_CODE_INST_RET:          return "INST_RET";
-    case bitc::FUNC_CODE_INST_BR:           return "INST_BR";
-    case bitc::FUNC_CODE_INST_SWITCH:       return "INST_SWITCH";
-    case bitc::FUNC_CODE_INST_INVOKE:       return "INST_INVOKE";
-    case bitc::FUNC_CODE_INST_UNREACHABLE:  return "INST_UNREACHABLE";
-
-    case bitc::FUNC_CODE_INST_PHI:          return "INST_PHI";
-    case bitc::FUNC_CODE_INST_ALLOCA:       return "INST_ALLOCA";
-    case bitc::FUNC_CODE_INST_LOAD:         return "INST_LOAD";
-    case bitc::FUNC_CODE_INST_VAARG:        return "INST_VAARG";
-    case bitc::FUNC_CODE_INST_STORE:        return "INST_STORE";
-    case bitc::FUNC_CODE_INST_EXTRACTVAL:   return "INST_EXTRACTVAL";
-    case bitc::FUNC_CODE_INST_INSERTVAL:    return "INST_INSERTVAL";
-    case bitc::FUNC_CODE_INST_CMP2:         return "INST_CMP2";
-    case bitc::FUNC_CODE_INST_VSELECT:      return "INST_VSELECT";
-    case bitc::FUNC_CODE_DEBUG_LOC_AGAIN:   return "DEBUG_LOC_AGAIN";
-    case bitc::FUNC_CODE_INST_CALL:         return "INST_CALL";
-    case bitc::FUNC_CODE_DEBUG_LOC:         return "DEBUG_LOC";
-    case bitc::FUNC_CODE_INST_GEP:
-      return "INST_GEP";
+      STRINGIFY_CODE(FUNC_CODE, DECLAREBLOCKS)
+      STRINGIFY_CODE(FUNC_CODE, INST_BINOP)
+      STRINGIFY_CODE(FUNC_CODE, INST_CAST)
+      STRINGIFY_CODE(FUNC_CODE, INST_GEP_OLD)
+      STRINGIFY_CODE(FUNC_CODE, INST_INBOUNDS_GEP_OLD)
+      STRINGIFY_CODE(FUNC_CODE, INST_SELECT)
+      STRINGIFY_CODE(FUNC_CODE, INST_EXTRACTELT)
+      STRINGIFY_CODE(FUNC_CODE, INST_INSERTELT)
+      STRINGIFY_CODE(FUNC_CODE, INST_SHUFFLEVEC)
+      STRINGIFY_CODE(FUNC_CODE, INST_CMP)
+      STRINGIFY_CODE(FUNC_CODE, INST_RET)
+      STRINGIFY_CODE(FUNC_CODE, INST_BR)
+      STRINGIFY_CODE(FUNC_CODE, INST_SWITCH)
+      STRINGIFY_CODE(FUNC_CODE, INST_INVOKE)
+      STRINGIFY_CODE(FUNC_CODE, INST_UNREACHABLE)
+      STRINGIFY_CODE(FUNC_CODE, INST_CLEANUPRET)
+      STRINGIFY_CODE(FUNC_CODE, INST_CATCHRET)
+      STRINGIFY_CODE(FUNC_CODE, INST_CATCHPAD)
+      STRINGIFY_CODE(FUNC_CODE, INST_CATCHENDPAD)
+      STRINGIFY_CODE(FUNC_CODE, INST_TERMINATEPAD)
+      STRINGIFY_CODE(FUNC_CODE, INST_PHI)
+      STRINGIFY_CODE(FUNC_CODE, INST_ALLOCA)
+      STRINGIFY_CODE(FUNC_CODE, INST_LOAD)
+      STRINGIFY_CODE(FUNC_CODE, INST_VAARG)
+      STRINGIFY_CODE(FUNC_CODE, INST_STORE)
+      STRINGIFY_CODE(FUNC_CODE, INST_EXTRACTVAL)
+      STRINGIFY_CODE(FUNC_CODE, INST_INSERTVAL)
+      STRINGIFY_CODE(FUNC_CODE, INST_CMP2)
+      STRINGIFY_CODE(FUNC_CODE, INST_VSELECT)
+      STRINGIFY_CODE(FUNC_CODE, DEBUG_LOC_AGAIN)
+      STRINGIFY_CODE(FUNC_CODE, INST_CALL)
+      STRINGIFY_CODE(FUNC_CODE, DEBUG_LOC)
+      STRINGIFY_CODE(FUNC_CODE, INST_GEP)
     }
   case bitc::VALUE_SYMTAB_BLOCK_ID:
     switch (CodeID) {
     default: return nullptr;
-    case bitc::VST_CODE_ENTRY: return "ENTRY";
-    case bitc::VST_CODE_BBENTRY: return "BBENTRY";
+    STRINGIFY_CODE(VST_CODE, ENTRY)
+    STRINGIFY_CODE(VST_CODE, BBENTRY)
     }
   case bitc::METADATA_ATTACHMENT_ID:
     switch(CodeID) {
     default:return nullptr;
-    case bitc::METADATA_ATTACHMENT: return "METADATA_ATTACHMENT";
+      STRINGIFY_CODE(METADATA, ATTACHMENT)
     }
   case bitc::METADATA_BLOCK_ID:
     switch(CodeID) {
     default:return nullptr;
-    case bitc::METADATA_STRING:      return "METADATA_STRING";
-    case bitc::METADATA_NAME:        return "METADATA_NAME";
-    case bitc::METADATA_KIND:        return "METADATA_KIND";
-    case bitc::METADATA_NODE:        return "METADATA_NODE";
-    case bitc::METADATA_VALUE:       return "METADATA_VALUE";
-    case bitc::METADATA_OLD_NODE:    return "METADATA_OLD_NODE";
-    case bitc::METADATA_OLD_FN_NODE: return "METADATA_OLD_FN_NODE";
-    case bitc::METADATA_NAMED_NODE:  return "METADATA_NAMED_NODE";
+      STRINGIFY_CODE(METADATA, STRING)
+      STRINGIFY_CODE(METADATA, NAME)
+      STRINGIFY_CODE(METADATA, KIND)
+      STRINGIFY_CODE(METADATA, NODE)
+      STRINGIFY_CODE(METADATA, VALUE)
+      STRINGIFY_CODE(METADATA, OLD_NODE)
+      STRINGIFY_CODE(METADATA, OLD_FN_NODE)
+      STRINGIFY_CODE(METADATA, NAMED_NODE)
+      STRINGIFY_CODE(METADATA, DISTINCT_NODE)
+      STRINGIFY_CODE(METADATA, LOCATION)
+      STRINGIFY_CODE(METADATA, GENERIC_DEBUG)
+      STRINGIFY_CODE(METADATA, SUBRANGE)
+      STRINGIFY_CODE(METADATA, ENUMERATOR)
+      STRINGIFY_CODE(METADATA, BASIC_TYPE)
+      STRINGIFY_CODE(METADATA, FILE)
+      STRINGIFY_CODE(METADATA, DERIVED_TYPE)
+      STRINGIFY_CODE(METADATA, COMPOSITE_TYPE)
+      STRINGIFY_CODE(METADATA, SUBROUTINE_TYPE)
+      STRINGIFY_CODE(METADATA, COMPILE_UNIT)
+      STRINGIFY_CODE(METADATA, SUBPROGRAM)
+      STRINGIFY_CODE(METADATA, LEXICAL_BLOCK)
+      STRINGIFY_CODE(METADATA, LEXICAL_BLOCK_FILE)
+      STRINGIFY_CODE(METADATA, NAMESPACE)
+      STRINGIFY_CODE(METADATA, TEMPLATE_TYPE)
+      STRINGIFY_CODE(METADATA, TEMPLATE_VALUE)
+      STRINGIFY_CODE(METADATA, GLOBAL_VAR)
+      STRINGIFY_CODE(METADATA, LOCAL_VAR)
+      STRINGIFY_CODE(METADATA, EXPRESSION)
+      STRINGIFY_CODE(METADATA, OBJC_PROPERTY)
+      STRINGIFY_CODE(METADATA, IMPORTED_ENTITY)
+      STRINGIFY_CODE(METADATA, MODULE)
     }
   case bitc::USELIST_BLOCK_ID:
     switch(CodeID) {
@@ -283,6 +313,7 @@ static const char *GetCodeName(unsigned CodeID, unsigned BlockID,
     case bitc::USELIST_CODE_BB:      return "USELIST_CODE_BB";
     }
   }
+#undef STRINGIFY_CODE
 }
 
 struct PerRecordStats {
@@ -460,17 +491,22 @@ static bool ParseBlock(BitstreamCursor &Stream, unsigned BlockID,
 
       if (Blob.data()) {
         outs() << " blob data = ";
-        bool BlobIsPrintable = true;
-        for (unsigned i = 0, e = Blob.size(); i != e; ++i)
-          if (!isprint(static_cast<unsigned char>(Blob[i]))) {
-            BlobIsPrintable = false;
-            break;
-          }
-
-        if (BlobIsPrintable)
-          outs() << "'" << Blob << "'";
-        else
-          outs() << "unprintable, " << Blob.size() << " bytes.";
+        if (ShowBinaryBlobs) {
+          outs() << "'";
+          outs().write_escaped(Blob, /*hex=*/true) << "'";
+        } else {
+          bool BlobIsPrintable = true;
+          for (unsigned i = 0, e = Blob.size(); i != e; ++i)
+            if (!isprint(static_cast<unsigned char>(Blob[i]))) {
+              BlobIsPrintable = false;
+              break;
+            }
+
+          if (BlobIsPrintable)
+            outs() << "'" << Blob << "'";
+          else
+            outs() << "unprintable, " << Blob.size() << " bytes.";          
+        }
       }
 
       outs() << "\n";
diff --git a/tools/llvm-config/BuildVariables.inc.in b/tools/llvm-config/BuildVariables.inc.in
index 3f51f491a7a..66f7a16a4ce 100644
--- a/tools/llvm-config/BuildVariables.inc.in
+++ b/tools/llvm-config/BuildVariables.inc.in
@@ -26,3 +26,5 @@
 #define LLVM_LIBDIR_SUFFIX "@LLVM_LIBDIR_SUFFIX@"
 #define LLVM_TARGETS_BUILT "@LLVM_TARGETS_BUILT@"
 #define LLVM_SYSTEM_LIBS "@LLVM_SYSTEM_LIBS@"
+#define BUILD_SHARED_LIBS "@BUILD_SHARED_LIBS@"
+#define WAS_BUILT_WITH_CMAKE "@WAS_BUILT_WITH_CMAKE@"
diff --git a/tools/llvm-config/CMakeLists.txt b/tools/llvm-config/CMakeLists.txt
index d2ef45a1681..3d5c9e68e11 100644
--- a/tools/llvm-config/CMakeLists.txt
+++ b/tools/llvm-config/CMakeLists.txt
@@ -20,6 +20,7 @@ set(LLVM_CXXFLAGS "${CMAKE_CXX_FLAGS} ${CMAKE_CXX_FLAGS_${uppercase_CMAKE_BUILD_
 set(LLVM_LDFLAGS "${CMAKE_CXX_LINK_FLAGS}")
 set(LLVM_BUILDMODE ${CMAKE_BUILD_TYPE})
 set(LLVM_SYSTEM_LIBS ${SYSTEM_LIBS})
+set(WAS_BUILT_WITH_CMAKE "ON")
 string(REPLACE ";" " " LLVM_TARGETS_BUILT "${LLVM_TARGETS_TO_BUILD}")
 configure_file(${BUILDVARIABLES_SRCPATH} ${BUILDVARIABLES_OBJPATH} @ONLY)
 
diff --git a/tools/llvm-config/Makefile b/tools/llvm-config/Makefile
index 1ff8b6f0406..11e5d338771 100644
--- a/tools/llvm-config/Makefile
+++ b/tools/llvm-config/Makefile
@@ -65,6 +65,12 @@ $(ObjDir)/BuildVariables.inc: $(BUILDVARIABLES_SRCPATH) Makefile $(ObjDir)/.dir
 	  >> temp.sed
 	$(Verb) $(ECHO) 's/@LLVM_TARGETS_BUILT@/$(subst /,\/,$(TARGETS_TO_BUILD))/' \
 	  >> temp.sed
+	$(if $(filter-out $(ENABLE_SHARED),0),\
+	  $(Verb) $(ECHO) 's/@BUILD_SHARED_LIBS@/ON/',\
+	  $(Verb) $(ECHO) 's/@BUILD_SHARED_LIBS@/OFF/') \
+	  >> temp.sed
+	$(Verb) $(ECHO) 's/@WAS_BUILT_WITH_CMAKE@/OFF/' \
+	  >> temp.sed
 	$(Verb) $(SED) -f temp.sed < $< > $@
 	$(Verb) $(RM) temp.sed
 
diff --git a/tools/llvm-config/llvm-config.cpp b/tools/llvm-config/llvm-config.cpp
index 879b9ab0945..c007d71baf1 100644
--- a/tools/llvm-config/llvm-config.cpp
+++ b/tools/llvm-config/llvm-config.cpp
@@ -349,6 +349,30 @@ int main(int argc, char **argv) {
                              /*IncludeNonInstalled=*/IsInDevelopmentTree);
 
     if (PrintLibs || PrintLibNames || PrintLibFiles) {
+      // If LLVM was built as a shared library, there will be only one thing
+      // that users should link against.
+      const bool IsSharedLib = (std::strcmp(BUILD_SHARED_LIBS, "ON") == 0);
+      const bool WasBuiltWithCMake = (std::strcmp(WAS_BUILT_WITH_CMAKE, "ON") == 0);
+      // CMake correctly builds components as separate shared libraries, however
+      // autoconfig/make builds components a static libraries and then links
+      // them all together to form a single shared library. Thus, only when
+      // `WAS_BUILT_WITH_CMAKE` is `OFF` and `BUILD_SHARED_LIBS` is `ON` do we
+      // override `RequiredLibs` with the single library name.
+      if (IsSharedLib && !WasBuiltWithCMake) {
+        RequiredLibs.clear();
+        std::string Name = "libLLVM-" PACKAGE_VERSION;
+        const Triple HostTriple(LLVM_DEFAULT_TARGET_TRIPLE);
+        if (HostTriple.isOSWindows()) {
+          Name += ".dll";
+        } else if (HostTriple.isOSDarwin()) {
+          Name += ".dylib";
+        } else {
+          // default to linux' ext:
+          Name += ".so";
+        }
+        RequiredLibs.push_back(Name);
+      }
+
       for (unsigned i = 0, e = RequiredLibs.size(); i != e; ++i) {
         StringRef Lib = RequiredLibs[i];
         if (i)
@@ -360,8 +384,23 @@ int main(int argc, char **argv) {
           OS << ActiveLibDir << '/' << Lib;
         } else if (PrintLibs) {
           // If this is a typical library name, include it using -l.
-          if (Lib.startswith("lib") && Lib.endswith(".a")) {
-            OS << "-l" << Lib.slice(3, Lib.size()-2);
+          if (Lib.startswith("lib")) {
+            size_t FromEnd = 0;
+            if (Lib.endswith(".a")) {
+              FromEnd = 2;
+            } else if (Lib.endswith(".so")) {
+              FromEnd = 3;
+            } else if (Lib.endswith(".dylib")) {
+              FromEnd = 6;
+            } else {
+              FromEnd = 0;
+            }
+
+            if (FromEnd != 0) {
+              OS << "-l" << Lib.slice(3, Lib.size() - FromEnd);
+            } else {
+              OS << "-l:" << Lib;
+            }
             continue;
           }
 
diff --git a/tools/llvm-cov/CodeCoverage.cpp b/tools/llvm-cov/CodeCoverage.cpp
index 0331a02dfcc..8dc4d665f23 100644
--- a/tools/llvm-cov/CodeCoverage.cpp
+++ b/tools/llvm-cov/CodeCoverage.cpp
@@ -89,7 +89,7 @@ class CodeCoverageTool {
       LoadedSourceFiles;
   bool CompareFilenamesOnly;
   StringMap<std::string> RemappedFilenames;
-  llvm::Triple::ArchType CoverageArch;
+  std::string CoverageArch;
 };
 }
 
@@ -116,8 +116,7 @@ CodeCoverageTool::getSourceFile(StringRef SourceFile) {
     error(EC.message(), SourceFile);
     return EC;
   }
-  LoadedSourceFiles.push_back(
-      std::make_pair(SourceFile, std::move(Buffer.get())));
+  LoadedSourceFiles.emplace_back(SourceFile, std::move(Buffer.get()));
   return *LoadedSourceFiles.back().second;
 }
 
@@ -195,7 +194,20 @@ CodeCoverageTool::createSourceFileView(StringRef SourceFile,
   return View;
 }
 
+static bool modifiedTimeGT(StringRef LHS, StringRef RHS) {
+  sys::fs::file_status Status;
+  if (sys::fs::status(LHS, Status))
+    return false;
+  auto LHSTime = Status.getLastModificationTime();
+  if (sys::fs::status(RHS, Status))
+    return false;
+  auto RHSTime = Status.getLastModificationTime();
+  return LHSTime > RHSTime;
+}
+
 std::unique_ptr<CoverageMapping> CodeCoverageTool::load() {
+  if (modifiedTimeGT(ObjectFilename, PGOFilename))
+    errs() << "warning: profile data may be out of date - object is newer\n";
   auto CoverageOrErr = CoverageMapping::load(ObjectFilename, PGOFilename,
                                              CoverageArch);
   if (std::error_code EC = CoverageOrErr.getError()) {
@@ -337,15 +349,12 @@ int CodeCoverageTool::run(Command Cmd, int argc, const char **argv) {
       Filters.push_back(std::unique_ptr<CoverageFilter>(StatFilterer));
     }
 
-    if (Arch.empty())
-      CoverageArch = llvm::Triple::ArchType::UnknownArch;
-    else {
-      CoverageArch = Triple(Arch).getArch();
-      if (CoverageArch == llvm::Triple::ArchType::UnknownArch) {
-        errs() << "error: Unknown architecture: " << Arch << "\n";
-        return 1;
-      }
+    if (!Arch.empty() &&
+        Triple(Arch).getArch() == llvm::Triple::ArchType::UnknownArch) {
+      errs() << "error: Unknown architecture: " << Arch << "\n";
+      return 1;
     }
+    CoverageArch = Arch;
 
     for (const auto &File : InputSourceFiles) {
       SmallString<128> Path(File);
diff --git a/tools/llvm-cov/SourceCoverageView.cpp b/tools/llvm-cov/SourceCoverageView.cpp
index 015099c7d02..58c8a679529 100644
--- a/tools/llvm-cov/SourceCoverageView.cpp
+++ b/tools/llvm-cov/SourceCoverageView.cpp
@@ -14,6 +14,7 @@
 #include "SourceCoverageView.h"
 #include "llvm/ADT/Optional.h"
 #include "llvm/ADT/SmallString.h"
+#include "llvm/ADT/StringExtras.h"
 #include "llvm/Support/LineIterator.h"
 
 using namespace llvm;
@@ -77,6 +78,22 @@ void SourceCoverageView::renderViewDivider(unsigned Level, unsigned Length,
     OS << "-";
 }
 
+/// Format a count using engineering notation with 3 significant digits.
+static std::string formatCount(uint64_t N) {
+  std::string Number = utostr(N);
+  int Len = Number.size();
+  if (Len <= 3)
+    return Number;
+  int IntLen = Len % 3 == 0 ? 3 : Len % 3;
+  std::string Result(Number.data(), IntLen);
+  if (IntLen != 3) {
+    Result.push_back('.');
+    Result += Number.substr(IntLen, 3 - IntLen);
+  }
+  Result.push_back(" kMGTPEZY"[(Len - 1) / 3]);
+  return Result;
+}
+
 void
 SourceCoverageView::renderLineCoverageColumn(raw_ostream &OS,
                                              const LineCoverageInfo &Line) {
@@ -84,17 +101,11 @@ SourceCoverageView::renderLineCoverageColumn(raw_ostream &OS,
     OS.indent(LineCoverageColumnWidth) << '|';
     return;
   }
-  SmallString<32> Buffer;
-  raw_svector_ostream BufferOS(Buffer);
-  BufferOS << Line.ExecutionCount;
-  auto Str = BufferOS.str();
-  // Trim
-  Str = Str.substr(0, std::min(Str.size(), (size_t)LineCoverageColumnWidth));
-  // Align to the right
-  OS.indent(LineCoverageColumnWidth - Str.size());
+  std::string C = formatCount(Line.ExecutionCount);
+  OS.indent(LineCoverageColumnWidth - C.size());
   colored_ostream(OS, raw_ostream::MAGENTA,
                   Line.hasMultipleRegions() && Options.Colors)
-      << Str;
+      << C;
   OS << '|';
 }
 
@@ -111,9 +122,6 @@ void SourceCoverageView::renderLineNumberColumn(raw_ostream &OS,
 
 void SourceCoverageView::renderRegionMarkers(
     raw_ostream &OS, ArrayRef<const coverage::CoverageSegment *> Segments) {
-  SmallString<32> Buffer;
-  raw_svector_ostream BufferOS(Buffer);
-
   unsigned PrevColumn = 1;
   for (const auto *S : Segments) {
     if (!S->IsRegionEntry)
@@ -122,20 +130,16 @@ void SourceCoverageView::renderRegionMarkers(
     if (S->Col > PrevColumn)
       OS.indent(S->Col - PrevColumn);
     PrevColumn = S->Col + 1;
-    BufferOS << S->Count;
-    StringRef Str = BufferOS.str();
-    // Trim the execution count
-    Str = Str.substr(0, std::min(Str.size(), (size_t)7));
-    PrevColumn += Str.size();
-    OS << '^' << Str;
-    Buffer.clear();
+    std::string C = formatCount(S->Count);
+    PrevColumn += C.size();
+    OS << '^' << C;
   }
   OS << "\n";
 
   if (Options.Debug)
     for (const auto *S : Segments)
-      errs() << "Marker at " << S->Line << ":" << S->Col << " = " << S->Count
-             << (S->IsRegionEntry ? "\n" : " (pop)\n");
+      errs() << "Marker at " << S->Line << ":" << S->Col << " = "
+             << formatCount(S->Count) << (S->IsRegionEntry ? "\n" : " (pop)\n");
 }
 
 void SourceCoverageView::render(raw_ostream &OS, bool WholeFile,
diff --git a/tools/llvm-cov/llvm-cov.cpp b/tools/llvm-cov/llvm-cov.cpp
index bf66f583a52..8c5acaef63b 100644
--- a/tools/llvm-cov/llvm-cov.cpp
+++ b/tools/llvm-cov/llvm-cov.cpp
@@ -13,6 +13,7 @@
 
 #include "llvm/ADT/StringRef.h"
 #include "llvm/ADT/StringSwitch.h"
+#include "llvm/Support/CommandLine.h"
 #include "llvm/Support/Path.h"
 #include "llvm/Support/Process.h"
 #include "llvm/Support/raw_ostream.h"
@@ -43,6 +44,12 @@ static int helpMain(int argc, const char *argv[]) {
   return 0;
 }
 
+/// \brief Top level version information.
+static int versionMain(int argc, const char *argv[]) {
+  cl::PrintVersionMessage();
+  return 0;
+}
+
 int main(int argc, const char **argv) {
   // If argv[0] is or ends with 'gcov', always be gcov compatible
   if (sys::path::stem(argv[0]).endswith_lower("gcov"))
@@ -57,6 +64,7 @@ int main(int argc, const char **argv) {
                             .Case("report", reportMain)
                             .Case("show", showMain)
                             .Cases("-h", "-help", "--help", helpMain)
+                            .Cases("-version", "--version", versionMain)
                             .Default(nullptr);
 
     if (Func) {
diff --git a/tools/llvm-cxxdump/llvm-cxxdump.cpp b/tools/llvm-cxxdump/llvm-cxxdump.cpp
index 447d55afa8b..ee905c6f696 100644
--- a/tools/llvm-cxxdump/llvm-cxxdump.cpp
+++ b/tools/llvm-cxxdump/llvm-cxxdump.cpp
@@ -16,6 +16,7 @@
 #include "llvm/ADT/ArrayRef.h"
 #include "llvm/Object/Archive.h"
 #include "llvm/Object/ObjectFile.h"
+#include "llvm/Object/SymbolSize.h"
 #include "llvm/Support/Debug.h"
 #include "llvm/Support/Endian.h"
 #include "llvm/Support/FileSystem.h"
@@ -39,18 +40,14 @@ cl::list<std::string> InputFilenames(cl::Positional,
                                      cl::ZeroOrMore);
 } // namespace opts
 
-static int ReturnValue = EXIT_SUCCESS;
-
 namespace llvm {
 
-static bool error(std::error_code EC) {
+static void error(std::error_code EC) {
   if (!EC)
-    return false;
-
-  ReturnValue = EXIT_FAILURE;
+    return;
   outs() << "\nError reading file: " << EC.message() << ".\n";
   outs().flush();
-  return true;
+  exit(1);
 }
 
 } // namespace llvm
@@ -58,10 +55,9 @@ static bool error(std::error_code EC) {
 static void reportError(StringRef Input, StringRef Message) {
   if (Input == "-")
     Input = "<stdin>";
-
   errs() << Input << ": " << Message << "\n";
   errs().flush();
-  ReturnValue = EXIT_FAILURE;
+  exit(1);
 }
 
 static void reportError(StringRef Input, std::error_code EC) {
@@ -83,7 +79,7 @@ static SmallVectorImpl<SectionRef> &getRelocSections(const ObjectFile *Obj,
   return SectionRelocMap[Sec];
 }
 
-static bool collectRelocatedSymbols(const ObjectFile *Obj,
+static void collectRelocatedSymbols(const ObjectFile *Obj,
                                     const SectionRef &Sec, uint64_t SecAddress,
                                     uint64_t SymAddress, uint64_t SymSize,
                                     StringRef *I, StringRef *E) {
@@ -96,22 +92,18 @@ static bool collectRelocatedSymbols(const ObjectFile *Obj,
       const object::symbol_iterator RelocSymI = Reloc.getSymbol();
       if (RelocSymI == Obj->symbol_end())
         continue;
-      StringRef RelocSymName;
-      if (error(RelocSymI->getName(RelocSymName)))
-        return true;
-      uint64_t Offset;
-      if (error(Reloc.getOffset(Offset)))
-        return true;
+      ErrorOr<StringRef> RelocSymName = RelocSymI->getName();
+      error(RelocSymName.getError());
+      uint64_t Offset = Reloc.getOffset();
       if (Offset >= SymOffset && Offset < SymEnd) {
-        *I = RelocSymName;
+        *I = *RelocSymName;
         ++I;
       }
     }
   }
-  return false;
 }
 
-static bool collectRelocationOffsets(
+static void collectRelocationOffsets(
     const ObjectFile *Obj, const SectionRef &Sec, uint64_t SecAddress,
     uint64_t SymAddress, uint64_t SymSize, StringRef SymName,
     std::map<std::pair<StringRef, uint64_t>, StringRef> &Collection) {
@@ -122,17 +114,13 @@ static bool collectRelocationOffsets(
       const object::symbol_iterator RelocSymI = Reloc.getSymbol();
       if (RelocSymI == Obj->symbol_end())
         continue;
-      StringRef RelocSymName;
-      if (error(RelocSymI->getName(RelocSymName)))
-        return true;
-      uint64_t Offset;
-      if (error(Reloc.getOffset(Offset)))
-        return true;
+      ErrorOr<StringRef> RelocSymName = RelocSymI->getName();
+      error(RelocSymName.getError());
+      uint64_t Offset = Reloc.getOffset();
       if (Offset >= SymOffset && Offset < SymEnd)
-        Collection[std::make_pair(SymName, Offset - SymOffset)] = RelocSymName;
+        Collection[std::make_pair(SymName, Offset - SymOffset)] = *RelocSymName;
     }
   }
-  return false;
 }
 
 static void dumpCXXData(const ObjectFile *Obj) {
@@ -187,13 +175,17 @@ static void dumpCXXData(const ObjectFile *Obj) {
 
   uint8_t BytesInAddress = Obj->getBytesInAddress();
 
-  for (const object::SymbolRef &Sym : Obj->symbols()) {
-    StringRef SymName;
-    if (error(Sym.getName(SymName)))
-      return;
+  std::vector<std::pair<SymbolRef, uint64_t>> SymAddr =
+      object::computeSymbolSizes(*Obj);
+
+  for (auto &P : SymAddr) {
+    object::SymbolRef Sym = P.first;
+    uint64_t SymSize = P.second;
+    ErrorOr<StringRef> SymNameOrErr = Sym.getName();
+    error(SymNameOrErr.getError());
+    StringRef SymName = *SymNameOrErr;
     object::section_iterator SecI(Obj->section_begin());
-    if (error(Sym.getSection(SecI)))
-      return;
+    error(Sym.getSection(SecI));
     // Skip external symbols.
     if (SecI == Obj->section_end())
       continue;
@@ -202,11 +194,10 @@ static void dumpCXXData(const ObjectFile *Obj) {
     if (Sec.isBSS() || Sec.isVirtual())
       continue;
     StringRef SecContents;
-    if (error(Sec.getContents(SecContents)))
-      return;
-    uint64_t SymAddress, SymSize;
-    if (error(Sym.getAddress(SymAddress)) || error(Sym.getSize(SymSize)))
-      return;
+    error(Sec.getContents(SecContents));
+    ErrorOr<uint64_t> SymAddressOrErr = Sym.getAddress();
+    error(SymAddressOrErr.getError());
+    uint64_t SymAddress = *SymAddressOrErr;
     uint64_t SecAddress = Sec.getAddress();
     uint64_t SecSize = Sec.getSize();
     uint64_t SymOffset = SymAddress - SecAddress;
@@ -235,9 +226,7 @@ static void dumpCXXData(const ObjectFile *Obj) {
       COL.Data = ArrayRef<little32_t>(
           reinterpret_cast<const little32_t *>(SymContents.data()), 3);
       StringRef *I = std::begin(COL.Symbols), *E = std::end(COL.Symbols);
-      if (collectRelocatedSymbols(Obj, Sec, SecAddress, SymAddress, SymSize, I,
-                                  E))
-        return;
+      collectRelocatedSymbols(Obj, Sec, SecAddress, SymAddress, SymSize, I, E);
       COLs[SymName] = COL;
     }
     // Class hierarchy descriptors in the MS-ABI start with '??_R3'
@@ -246,9 +235,7 @@ static void dumpCXXData(const ObjectFile *Obj) {
       CHD.Data = ArrayRef<little32_t>(
           reinterpret_cast<const little32_t *>(SymContents.data()), 3);
       StringRef *I = std::begin(CHD.Symbols), *E = std::end(CHD.Symbols);
-      if (collectRelocatedSymbols(Obj, Sec, SecAddress, SymAddress, SymSize, I,
-                                  E))
-        return;
+      collectRelocatedSymbols(Obj, Sec, SecAddress, SymAddress, SymSize, I, E);
       CHDs[SymName] = CHD;
     }
     // Class hierarchy descriptors in the MS-ABI start with '??_R2'
@@ -264,9 +251,7 @@ static void dumpCXXData(const ObjectFile *Obj) {
       BCD.Data = ArrayRef<little32_t>(
           reinterpret_cast<const little32_t *>(SymContents.data()) + 1, 5);
       StringRef *I = std::begin(BCD.Symbols), *E = std::end(BCD.Symbols);
-      if (collectRelocatedSymbols(Obj, Sec, SecAddress, SymAddress, SymSize, I,
-                                  E))
-        return;
+      collectRelocatedSymbols(Obj, Sec, SecAddress, SymAddress, SymSize, I, E);
       BCDs[SymName] = BCD;
     }
     // Type descriptors in the MS-ABI start with '??_R0'
@@ -279,9 +264,7 @@ static void dumpCXXData(const ObjectFile *Obj) {
         TD.AlwaysZero = *reinterpret_cast<const little32_t *>(DataPtr);
       TD.MangledName = SymContents.drop_front(BytesInAddress * 2);
       StringRef *I = std::begin(TD.Symbols), *E = std::end(TD.Symbols);
-      if (collectRelocatedSymbols(Obj, Sec, SecAddress, SymAddress, SymSize, I,
-                                  E))
-        return;
+      collectRelocatedSymbols(Obj, Sec, SecAddress, SymAddress, SymSize, I, E);
       TDs[SymName] = TD;
     }
     // Throw descriptors in the MS-ABI start with '_TI'
@@ -312,9 +295,7 @@ static void dumpCXXData(const ObjectFile *Obj) {
       CT.VirtualBaseAdjustmentOffset = DataPtr[4];
       CT.Size = DataPtr[5];
       StringRef *I = std::begin(CT.Symbols), *E = std::end(CT.Symbols);
-      if (collectRelocatedSymbols(Obj, Sec, SecAddress, SymAddress, SymSize, I,
-                                  E))
-        return;
+      collectRelocatedSymbols(Obj, Sec, SecAddress, SymAddress, SymSize, I, E);
       CTs[SymName] = CT;
     }
     // Construction vtables in the Itanium ABI start with '_ZTT' or '__ZTT'.
@@ -565,5 +546,5 @@ int main(int argc, const char *argv[]) {
   std::for_each(opts::InputFilenames.begin(), opts::InputFilenames.end(),
                 dumpInput);
 
-  return ReturnValue;
+  return EXIT_SUCCESS;
 }
diff --git a/tools/llvm-dis/CMakeLists.txt b/tools/llvm-dis/CMakeLists.txt
index be3b2255c90..06ac051602e 100644
--- a/tools/llvm-dis/CMakeLists.txt
+++ b/tools/llvm-dis/CMakeLists.txt
@@ -1,7 +1,6 @@
 set(LLVM_LINK_COMPONENTS
   BitReader
   Core
-  NaClBitReader # @LOCALMOD
   Support
   )
 
diff --git a/tools/llvm-dis/LLVMBuild.txt b/tools/llvm-dis/LLVMBuild.txt
index cf1cbf7a40e..4525010c1fc 100644
--- a/tools/llvm-dis/LLVMBuild.txt
+++ b/tools/llvm-dis/LLVMBuild.txt
@@ -19,4 +19,4 @@
 type = Tool
 name = llvm-dis
 parent = Tools
-required_libraries = Analysis BitReader NaClBitReader
+required_libraries = Analysis BitReader
diff --git a/tools/llvm-dis/Makefile b/tools/llvm-dis/Makefile
index 0719006a15a..aeeeed0d68c 100644
--- a/tools/llvm-dis/Makefile
+++ b/tools/llvm-dis/Makefile
@@ -9,7 +9,7 @@
 
 LEVEL := ../..
 TOOLNAME := llvm-dis
-LINK_COMPONENTS := bitreader naclbitreader analysis
+LINK_COMPONENTS := bitreader analysis
 
 # This tool has no plugins, optimize startup time.
 TOOL_NO_EXPORTS := 1
diff --git a/tools/llvm-dis/llvm-dis.cpp b/tools/llvm-dis/llvm-dis.cpp
index 6123c85be18..4b7d94d5b26 100644
--- a/tools/llvm-dis/llvm-dis.cpp
+++ b/tools/llvm-dis/llvm-dis.cpp
@@ -18,7 +18,6 @@
 
 #include "llvm/IR/LLVMContext.h"
 #include "llvm/Bitcode/ReaderWriter.h"
-#include "llvm/Bitcode/NaCl/NaClReaderWriter.h"  // @LOCALMOD
 #include "llvm/IR/AssemblyAnnotationWriter.h"
 #include "llvm/IR/DebugInfo.h"
 #include "llvm/IR/DiagnosticInfo.h"
@@ -26,7 +25,6 @@
 #include "llvm/IR/IntrinsicInst.h"
 #include "llvm/IR/Module.h"
 #include "llvm/IR/Type.h"
-#include "llvm/IRReader/IRReader.h"  // @LOCALMOD
 #include "llvm/Support/CommandLine.h"
 #include "llvm/Support/DataStream.h"
 #include "llvm/Support/FileSystem.h"
@@ -35,7 +33,6 @@
 #include "llvm/Support/MemoryBuffer.h"
 #include "llvm/Support/PrettyStackTrace.h"
 #include "llvm/Support/Signals.h"
-#include "llvm/Support/StreamingMemoryObject.h" // @LOCALMOD
 #include "llvm/Support/ToolOutputFile.h"
 #include <system_error>
 using namespace llvm;
@@ -62,23 +59,11 @@ static cl::opt<bool> PreserveAssemblyUseListOrder(
     cl::desc("Preserve use-list order when writing LLVM assembly."),
     cl::init(false), cl::Hidden);
 
-// @LOCALMOD-BEGIN
-static cl::opt<NaClFileFormat>
-InputFileFormat(
-    "bitcode-format",
-    cl::desc("Define format of input bitcode file:"),
-    cl::values(
-        clEnumValN(LLVMFormat, "llvm", "LLVM bitcode file (default)"),
-        clEnumValN(PNaClFormat, "pnacl", "PNaCl bitcode file"),
-        clEnumValEnd),
-    cl::init(LLVMFormat));
-// @LOCALMOD-END
-
 namespace {
 
 static void printDebugLoc(const DebugLoc &DL, formatted_raw_ostream &OS) {
   OS << DL.getLine() << ":" << DL.getCol();
-  if (MDLocation *IDL = DL.getInlinedAt()) {
+  if (DILocation *IDL = DL.getInlinedAt()) {
     OS << "@";
     printDebugLoc(IDL, OS);
   }
@@ -95,7 +80,8 @@ class CommentWriter : public AssemblyAnnotationWriter {
     if (!V.getType()->isVoidTy()) {
       OS.PadToColumn(50);
       Padded = true;
-      OS << "; [#uses=" << V.getNumUses() << " type=" << *V.getType() << "]";  // Output # uses and type
+      // Output # uses and type
+      OS << "; [#uses=" << V.getNumUses() << " type=" << *V.getType() << "]";
     }
     if (const Instruction *I = dyn_cast<Instruction>(&V)) {
       if (const DebugLoc &DL = I->getDebugLoc()) {
@@ -109,20 +95,18 @@ class CommentWriter : public AssemblyAnnotationWriter {
         OS << "]";
       }
       if (const DbgDeclareInst *DDI = dyn_cast<DbgDeclareInst>(I)) {
-        DIVariable Var(DDI->getVariable());
         if (!Padded) {
           OS.PadToColumn(50);
           OS << ";";
         }
-        OS << " [debug variable = " << Var->getName() << "]";
+        OS << " [debug variable = " << DDI->getVariable()->getName() << "]";
       }
       else if (const DbgValueInst *DVI = dyn_cast<DbgValueInst>(I)) {
-        DIVariable Var(DVI->getVariable());
         if (!Padded) {
           OS.PadToColumn(50);
           OS << ";";
         }
-        OS << " [debug variable = " << Var->getName() << "]";
+        OS << " [debug variable = " << DVI->getVariable()->getName() << "]";
       }
     }
   }
@@ -164,51 +148,21 @@ int main(int argc, char **argv) {
   std::unique_ptr<Module> M;
 
   // Use the bitcode streaming interface
-  DataStreamer *Streamer = getDataFileStreamer(InputFilename, &ErrorMessage);
+  std::unique_ptr<DataStreamer> Streamer =
+      getDataFileStreamer(InputFilename, &ErrorMessage);
   if (Streamer) {
-    std::unique_ptr<StreamingMemoryObject> Buffer(
-        new StreamingMemoryObjectImpl(Streamer));  // @LOCALMOD
     std::string DisplayFilename;
     if (InputFilename == "-")
       DisplayFilename = "<stdin>";
     else
       DisplayFilename = InputFilename;
-
-    // @LOCALMOD-BEGIN
-    switch (InputFileFormat) {
-      case LLVMFormat: {
-        // The Module's BitcodeReader's BitstreamReader takes ownership
-        // of the StreamingMemoryObject.
-        ErrorOr<std::unique_ptr<Module>> MOrErr =
-            getStreamedBitcodeModule(DisplayFilename, Buffer.release(), Context);
-        M = std::move(*MOrErr);
-        M->materializeAllPermanently();
-        break;
-      }
-      case PNaClFormat: {
-        M.reset(getNaClStreamedBitcodeModule(
-            DisplayFilename, Buffer.release(), Context, nullptr,
-            &ErrorMessage));
-        if(M.get()) {
-          if (std::error_code EC = M->materializeAllPermanently()) {
-            ErrorMessage = EC.message();
-            M.reset();
-          }
-        } else {
-          errs() << argv[0] << ": ";
-          if (ErrorMessage.size())
-            errs() << ErrorMessage << "\n";
-          else
-            errs() << "bitcode didn't read correctly.\n";
-          return 1;
-        }
-        break;
-      }
-      case AutodetectFileFormat:
-        report_fatal_error("Command can't autodetect file format!");
-    }
-    // @LOCALMOD-END
-
+    ErrorOr<std::unique_ptr<Module>> MOrErr =
+        getStreamedBitcodeModule(DisplayFilename, std::move(Streamer), Context);
+    M = std::move(*MOrErr);
+    M->materializeAllPermanently();
+  } else {
+    errs() << argv[0] << ": " << ErrorMessage << '\n';
+    return 1;
   }
 
   // Just use stdout.  We won't actually print anything on it.
@@ -219,13 +173,9 @@ int main(int argc, char **argv) {
     if (InputFilename == "-") {
       OutputFilename = "-";
     } else {
-      const std::string &IFN = InputFilename;
-      int Len = IFN.length();
-      // If the source ends in .bc, strip it off.
-      if (IFN[Len-3] == '.' && IFN[Len-2] == 'b' && IFN[Len-1] == 'c')
-        OutputFilename = std::string(IFN.begin(), IFN.end()-3)+".ll";
-      else
-        OutputFilename = IFN+".ll";
+      StringRef IFN = InputFilename;
+      OutputFilename = (IFN.endswith(".bc") ? IFN.drop_back(3) : IFN).str();
+      OutputFilename += ".ll";
     }
   }
 
diff --git a/tools/llvm-dwarfdump/CMakeLists.txt b/tools/llvm-dwarfdump/CMakeLists.txt
index 086b1397461..9a2e53f5a4b 100644
--- a/tools/llvm-dwarfdump/CMakeLists.txt
+++ b/tools/llvm-dwarfdump/CMakeLists.txt
@@ -7,3 +7,7 @@ set(LLVM_LINK_COMPONENTS
 add_llvm_tool(llvm-dwarfdump
   llvm-dwarfdump.cpp
   )
+
+if(LLVM_USE_SANITIZE_COVERAGE)
+  add_subdirectory(fuzzer)
+endif()
diff --git a/tools/llvm-dwarfdump/fuzzer/CMakeLists.txt b/tools/llvm-dwarfdump/fuzzer/CMakeLists.txt
new file mode 100644
index 00000000000..1de35a3de47
--- /dev/null
+++ b/tools/llvm-dwarfdump/fuzzer/CMakeLists.txt
@@ -0,0 +1,14 @@
+set(LLVM_LINK_COMPONENTS
+  DebugInfoDWARF
+  Object
+  Support
+  )
+
+add_llvm_executable(llvm-dwarfdump-fuzzer
+  EXCLUDE_FROM_ALL
+  llvm-dwarfdump-fuzzer.cpp
+  )
+
+target_link_libraries(llvm-dwarfdump-fuzzer
+  LLVMFuzzer
+  )
diff --git a/tools/llvm-dwarfdump/fuzzer/llvm-dwarfdump-fuzzer.cpp b/tools/llvm-dwarfdump/fuzzer/llvm-dwarfdump-fuzzer.cpp
new file mode 100644
index 00000000000..af0ac365228
--- /dev/null
+++ b/tools/llvm-dwarfdump/fuzzer/llvm-dwarfdump-fuzzer.cpp
@@ -0,0 +1,34 @@
+//===-- llvm-dwarfdump-fuzzer.cpp - Fuzz the llvm-dwarfdump tool ----------===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+///
+/// \file
+/// \brief This file implements a function that runs llvm-dwarfdump
+///  on a single input. This function is then linked into the Fuzzer library.
+///
+//===----------------------------------------------------------------------===//
+#include "llvm/DebugInfo/DIContext.h"
+#include "llvm/DebugInfo/DWARF/DWARFContext.h"
+#include "llvm/Object/ObjectFile.h"
+#include "llvm/Support/MemoryBuffer.h"
+
+using namespace llvm;
+using namespace object;
+
+extern "C" void LLVMFuzzerTestOneInput(uint8_t *data, size_t size) {
+  std::unique_ptr<MemoryBuffer> Buff = MemoryBuffer::getMemBuffer(
+      StringRef((const char *)data, size), "", false);
+
+  ErrorOr<std::unique_ptr<ObjectFile>> ObjOrErr =
+      ObjectFile::createObjectFile(Buff->getMemBufferRef());
+  if (!ObjOrErr)
+    return;
+  ObjectFile &Obj = *ObjOrErr.get();
+  std::unique_ptr<DIContext> DICtx(new DWARFContextInMemory(Obj));
+  DICtx->dump(nulls(), DIDT_All);
+}
diff --git a/tools/llvm-dwarfdump/llvm-dwarfdump.cpp b/tools/llvm-dwarfdump/llvm-dwarfdump.cpp
index a99f57f5037..cd48d81c8de 100644
--- a/tools/llvm-dwarfdump/llvm-dwarfdump.cpp
+++ b/tools/llvm-dwarfdump/llvm-dwarfdump.cpp
@@ -13,7 +13,8 @@
 
 #include "llvm/ADT/STLExtras.h"
 #include "llvm/ADT/Triple.h"
-#include "llvm/DebugInfo/DWARF/DIContext.h"
+#include "llvm/DebugInfo/DIContext.h"
+#include "llvm/DebugInfo/DWARF/DWARFContext.h"
 #include "llvm/Object/ObjectFile.h"
 #include "llvm/Object/RelocVisitor.h"
 #include "llvm/Support/CommandLine.h"
@@ -68,25 +69,25 @@ DumpType("debug-dump", cl::init(DIDT_All),
         clEnumValN(DIDT_StrOffsetsDwo, "str_offsets.dwo", ".debug_str_offsets.dwo"),
         clEnumValEnd));
 
+static void error(StringRef Filename, std::error_code EC) {
+  if (!EC)
+    return;
+  errs() << Filename << ": " << EC.message() << "\n";
+  exit(1);
+}
+
 static void DumpInput(StringRef Filename) {
   ErrorOr<std::unique_ptr<MemoryBuffer>> BuffOrErr =
       MemoryBuffer::getFileOrSTDIN(Filename);
-
-  if (std::error_code EC = BuffOrErr.getError()) {
-    errs() << Filename << ": " << EC.message() << "\n";
-    return;
-  }
+  error(Filename, BuffOrErr.getError());
   std::unique_ptr<MemoryBuffer> Buff = std::move(BuffOrErr.get());
 
   ErrorOr<std::unique_ptr<ObjectFile>> ObjOrErr =
       ObjectFile::createObjectFile(Buff->getMemBufferRef());
-  if (std::error_code EC = ObjOrErr.getError()) {
-    errs() << Filename << ": " << EC.message() << '\n';
-    return;
-  }
+  error(Filename, ObjOrErr.getError());
   ObjectFile &Obj = *ObjOrErr.get();
 
-  std::unique_ptr<DIContext> DICtx(DIContext::getDWARFContext(Obj));
+  std::unique_ptr<DIContext> DICtx(new DWARFContextInMemory(Obj));
 
   outs() << Filename
          << ":\tfile format " << Obj.getFileFormatName() << "\n\n";
@@ -108,5 +109,5 @@ int main(int argc, char **argv) {
 
   std::for_each(InputFilenames.begin(), InputFilenames.end(), DumpInput);
 
-  return 0;
+  return EXIT_SUCCESS;
 }
diff --git a/tools/llvm-jitlistener/CMakeLists.txt b/tools/llvm-jitlistener/CMakeLists.txt
index 68a4303acef..61f8420a7ae 100644
--- a/tools/llvm-jitlistener/CMakeLists.txt
+++ b/tools/llvm-jitlistener/CMakeLists.txt
@@ -6,7 +6,7 @@ include_directories( ${LLVM_INTEL_JITEVENTS_INCDIR} )
 set(LLVM_LINK_COMPONENTS
   asmparser
   bitreader
-  debuginfo
+  DebugInfoDWARF
   inteljitevents
   interpreter
   irreader
@@ -14,6 +14,10 @@ set(LLVM_LINK_COMPONENTS
   nativecodegen
   object
   selectiondag
+  Support
+  ExecutionEngine
+  RuntimeDyld
+  Core
   )
 
 add_llvm_tool(llvm-jitlistener
diff --git a/tools/llvm-jitlistener/LLVMBuild.txt b/tools/llvm-jitlistener/LLVMBuild.txt
index e6ed20b2403..8861cc742f4 100644
--- a/tools/llvm-jitlistener/LLVMBuild.txt
+++ b/tools/llvm-jitlistener/LLVMBuild.txt
@@ -19,4 +19,13 @@
 type = Tool
 name = llvm-jitlistener
 parent = Tools
-required_libraries = AsmParser BitReader IRReader Interpreter MCJIT NativeCodeGen Object SelectionDAG Native
+required_libraries =
+ AsmParser
+ BitReader
+ IRReader
+ Interpreter
+ MCJIT
+ NativeCodeGen
+ Object
+ SelectionDAG
+ Native
diff --git a/tools/llvm-jitlistener/Makefile b/tools/llvm-jitlistener/Makefile
index 06132f9900f..4acb6a56171 100644
--- a/tools/llvm-jitlistener/Makefile
+++ b/tools/llvm-jitlistener/Makefile
@@ -1,27 +1,27 @@
-##===- tools/llvm-jitlistener/Makefile ---------------------*- Makefile -*-===##
-#
-#                     The LLVM Compiler Infrastructure
-#
-# This file is distributed under the University of Illinois Open Source
-# License. See LICENSE.TXT for details.
-#
-##===----------------------------------------------------------------------===##
-
-LEVEL := ../..
-TOOLNAME := llvm-jitlistener
-
-include $(LEVEL)/Makefile.config
-
-LINK_COMPONENTS := mcjit interpreter nativecodegen bitreader asmparser irreader selectiondag Object
-
-# If Intel JIT Events support is configured, link against the LLVM Intel JIT
-# Events interface library.  If not, this tool will do nothing useful, but it
-# will build correctly.
-ifeq ($(USE_INTEL_JITEVENTS), 1)
-  LINK_COMPONENTS += debuginfodwarf inteljitevents
-endif
-
-# This tool has no plugins, optimize startup time.
-TOOL_NO_EXPORTS := 1
-
-include $(LLVM_SRC_ROOT)/Makefile.rules
+##===- tools/llvm-jitlistener/Makefile ---------------------*- Makefile -*-===##
+#
+#                     The LLVM Compiler Infrastructure
+#
+# This file is distributed under the University of Illinois Open Source
+# License. See LICENSE.TXT for details.
+#
+##===----------------------------------------------------------------------===##
+
+LEVEL := ../..
+TOOLNAME := llvm-jitlistener
+
+include $(LEVEL)/Makefile.config
+
+LINK_COMPONENTS := mcjit interpreter nativecodegen bitreader asmparser irreader selectiondag Object
+
+# If Intel JIT Events support is configured, link against the LLVM Intel JIT
+# Events interface library.  If not, this tool will do nothing useful, but it
+# will build correctly.
+ifeq ($(USE_INTEL_JITEVENTS), 1)
+  LINK_COMPONENTS += debuginfodwarf inteljitevents
+endif
+
+# This tool has no plugins, optimize startup time.
+TOOL_NO_EXPORTS := 1
+
+include $(LLVM_SRC_ROOT)/Makefile.rules
diff --git a/tools/llvm-link/CMakeLists.txt b/tools/llvm-link/CMakeLists.txt
index 3f4ff8f69ea..d4f5be74904 100644
--- a/tools/llvm-link/CMakeLists.txt
+++ b/tools/llvm-link/CMakeLists.txt
@@ -3,7 +3,6 @@ set(LLVM_LINK_COMPONENTS
   Core
   IRReader
   Linker
-  NaClBitWriter # @LOCALMOD
   Support
   )
 
diff --git a/tools/llvm-link/llvm-link.cpp b/tools/llvm-link/llvm-link.cpp
index bcefc0b507a..369f3477fe5 100644
--- a/tools/llvm-link/llvm-link.cpp
+++ b/tools/llvm-link/llvm-link.cpp
@@ -38,6 +38,11 @@ static cl::list<std::string>
 InputFilenames(cl::Positional, cl::OneOrMore,
                cl::desc("<input bitcode files>"));
 
+static cl::list<std::string> OverridingInputs(
+    "override", cl::ZeroOrMore, cl::value_desc("filename"),
+    cl::desc(
+        "input bitcode file which can override previously defined symbol(s)"));
+
 static cl::opt<std::string>
 OutputFilename("o", cl::desc("Override output filename"), cl::init("-"),
                cl::value_desc("filename"));
@@ -107,6 +112,31 @@ static void diagnosticHandler(const DiagnosticInfo &DI) {
   errs() << '\n';
 }
 
+static bool linkFiles(const char *argv0, LLVMContext &Context, Linker &L,
+                      const cl::list<std::string> &Files,
+                      bool OverrideDuplicateSymbols) {
+  for (const auto &File : Files) {
+    std::unique_ptr<Module> M = loadFile(argv0, File, Context);
+    if (!M.get()) {
+      errs() << argv0 << ": error loading file '" << File << "'\n";
+      return false;
+    }
+
+    if (verifyModule(*M, &errs())) {
+      errs() << argv0 << ": " << File << ": error: input module is broken!\n";
+      return false;
+    }
+
+    if (Verbose)
+      errs() << "Linking in '" << File << "'\n";
+
+    if (L.linkInModule(M.get(), OverrideDuplicateSymbols))
+      return false;
+  }
+
+  return true;
+}
+
 int main(int argc, char **argv) {
   // Print a stack trace if we signal out.
   sys::PrintStackTraceOnErrorSignal();
@@ -119,24 +149,13 @@ int main(int argc, char **argv) {
   auto Composite = make_unique<Module>("llvm-link", Context);
   Linker L(Composite.get(), diagnosticHandler);
 
-  for (unsigned i = 0; i < InputFilenames.size(); ++i) {
-    std::unique_ptr<Module> M = loadFile(argv[0], InputFilenames[i], Context);
-    if (!M.get()) {
-      errs() << argv[0] << ": error loading file '" <<InputFilenames[i]<< "'\n";
-      return 1;
-    }
-
-    if (verifyModule(*M, &errs())) {
-      errs() << argv[0] << ": " << InputFilenames[i]
-             << ": error: input module is broken!\n";
-      return 1;
-    }
-
-    if (Verbose) errs() << "Linking in '" << InputFilenames[i] << "'\n";
+  // First add all the regular input files
+  if (!linkFiles(argv[0], Context, L, InputFilenames, false))
+    return 1;
 
-    if (L.linkInModule(M.get()))
-      return 1;
-  }
+  // Next the -override ones.
+  if (!linkFiles(argv[0], Context, L, OverridingInputs, true))
+    return 1;
 
   if (DumpAsm) errs() << "Here's the assembly:\n" << *Composite;
 
diff --git a/tools/llvm-lto/CMakeLists.txt b/tools/llvm-lto/CMakeLists.txt
index 0e3b5c95c07..43ff08da775 100644
--- a/tools/llvm-lto/CMakeLists.txt
+++ b/tools/llvm-lto/CMakeLists.txt
@@ -4,6 +4,7 @@ set(LLVM_LINK_COMPONENTS
   MC
   NaClTransforms
   Support
+  Target
   )
 
 add_llvm_tool(llvm-lto
diff --git a/tools/llvm-lto/llvm-lto.cpp b/tools/llvm-lto/llvm-lto.cpp
index 9cd6587c262..9678c8397e0 100644
--- a/tools/llvm-lto/llvm-lto.cpp
+++ b/tools/llvm-lto/llvm-lto.cpp
@@ -253,11 +253,9 @@ int main(int argc, char **argv) {
     CodeGen.setAttr(attrs.c_str());
 
   if (!OutputFilename.empty()) {
-    size_t len = 0;
     std::string ErrorInfo;
-    const void *Code =
-        CodeGen.compile(&len, DisableInline, DisableGVNLoadPRE,
-                        DisableLTOVectorization, ErrorInfo);
+    std::unique_ptr<MemoryBuffer> Code = CodeGen.compile(
+        DisableInline, DisableGVNLoadPRE, DisableLTOVectorization, ErrorInfo);
     if (!Code) {
       errs() << argv[0]
              << ": error compiling the code: " << ErrorInfo << "\n";
@@ -272,7 +270,7 @@ int main(int argc, char **argv) {
       return 1;
     }
 
-    FileStream.write(reinterpret_cast<const char *>(Code), len);
+    FileStream.write(Code->getBufferStart(), Code->getBufferSize());
   } else {
     std::string ErrorInfo;
     const char *OutputName = nullptr;
diff --git a/tools/llvm-mc/CMakeLists.txt b/tools/llvm-mc/CMakeLists.txt
index 6f8e9e5405f..15c6dda1b25 100644
--- a/tools/llvm-mc/CMakeLists.txt
+++ b/tools/llvm-mc/CMakeLists.txt
@@ -1,5 +1,9 @@
 set(LLVM_LINK_COMPONENTS
-  ${LLVM_TARGETS_TO_BUILD}
+  AllTargetsAsmPrinters
+  AllTargetsAsmParsers
+  AllTargetsDescs
+  AllTargetsDisassemblers
+  AllTargetsInfos
   MC
   MCParser
   Support
diff --git a/tools/llvm-mc/llvm-mc.cpp b/tools/llvm-mc/llvm-mc.cpp
index 6b5032381b1..6ecdb2eaa6d 100644
--- a/tools/llvm-mc/llvm-mc.cpp
+++ b/tools/llvm-mc/llvm-mc.cpp
@@ -18,7 +18,6 @@
 #include "llvm/MC/MCContext.h"
 #include "llvm/MC/MCInstPrinter.h"
 #include "llvm/MC/MCInstrInfo.h"
-#include "llvm/MC/MCNaCl.h"
 #include "llvm/MC/MCObjectFileInfo.h"
 #include "llvm/MC/MCParser/AsmLexer.h"
 #include "llvm/MC/MCRegisterInfo.h"
@@ -71,6 +70,9 @@ static cl::opt<bool>
 PrintImmHex("print-imm-hex", cl::init(false),
             cl::desc("Prefer hex format for immediate values"));
 
+static cl::list<std::string>
+DefineSymbol("defsym", cl::desc("Defines a symbol to be an integer constant"));
+
 enum OutputFileType {
   OFT_Null,
   OFT_AssemblyFile,
@@ -317,6 +319,26 @@ static int AsLexInput(SourceMgr &SrcMgr, MCAsmInfo &MAI,
   return Error;
 }
 
+static int fillCommandLineSymbols(MCAsmParser &Parser){
+  for(auto &I: DefineSymbol){
+    auto Pair = StringRef(I).split('=');
+    if(Pair.second.empty()){
+      errs() << "error: defsym must be of the form: sym=value: " << I;
+      return 1;
+    }
+    int64_t Value;
+    if(Pair.second.getAsInteger(0, Value)){
+      errs() << "error: Value is not an integer: " << Pair.second;
+      return 1;
+    }
+    auto &Context = Parser.getContext();
+    auto Symbol = Context.getOrCreateSymbol(Pair.first);
+    Parser.getStreamer().EmitAssignment(Symbol,
+                                        MCConstantExpr::create(Value, Context));
+  }
+  return 0;
+}
+
 static int AssembleInput(const char *ProgName, const Target *TheTarget,
                          SourceMgr &SrcMgr, MCContext &Ctx, MCStreamer &Str,
                          MCAsmInfo &MAI, MCSubtargetInfo &STI,
@@ -332,6 +354,9 @@ static int AssembleInput(const char *ProgName, const Target *TheTarget,
     return 1;
   }
 
+  int SymbolResult = fillCommandLineSymbols(*Parser);
+  if(SymbolResult)
+    return SymbolResult;
   Parser->setShowParsedOperands(ShowInstOperands);
   Parser->setTargetParser(*TAP);
 
@@ -358,7 +383,6 @@ int main(int argc, char **argv) {
   cl::ParseCommandLineOptions(argc, argv, "llvm machine code playground\n");
   MCTargetOptions MCOptions = InitMCTargetOptionsFromFlags();
   TripleName = Triple::normalize(TripleName);
-  Triple TheTriple(TripleName);
   setDwarfDebugFlags(argc, argv);
 
   setDwarfDebugProducer();
@@ -367,6 +391,9 @@ int main(int argc, char **argv) {
   const Target *TheTarget = GetTarget(ProgName);
   if (!TheTarget)
     return 1;
+  // Now that GetTarget() has (potentially) replaced TripleName, it's safe to
+  // construct the Triple object.
+  Triple TheTriple(TripleName);
 
   ErrorOr<std::unique_ptr<MemoryBuffer>> BufferPtr =
       MemoryBuffer::getFileOrSTDIN(InputFilename);
@@ -404,7 +431,7 @@ int main(int argc, char **argv) {
   // MCObjectFileInfo needs a MCContext reference in order to initialize itself.
   MCObjectFileInfo MOFI;
   MCContext Ctx(MAI.get(), MRI.get(), &MOFI, &SrcMgr);
-  MOFI.InitMCObjectFileInfo(TripleName, RelocModel, CMModel, Ctx);
+  MOFI.InitMCObjectFileInfo(TheTriple, RelocModel, CMModel, Ctx);
 
   if (SaveTempLabels)
     Ctx.setAllowTemporaryLabels(false);
@@ -448,9 +475,7 @@ int main(int argc, char **argv) {
   std::unique_ptr<MCSubtargetInfo> STI(
       TheTarget->createMCSubtargetInfo(TripleName, MCPU, FeaturesStr));
 
-  Triple T(TripleName); // @LOCALMOD
-
-  MCInstPrinter *IP = NULL;
+  MCInstPrinter *IP = nullptr;
   if (FileType == OFT_AssemblyFile) {
     IP = TheTarget->createMCInstPrinter(Triple(TripleName), OutputAsmVariant,
                                         *MAI, *MCII, *MRI);
@@ -461,7 +486,7 @@ int main(int argc, char **argv) {
     // Set up the AsmStreamer.
     MCCodeEmitter *CE = nullptr;
     MCAsmBackend *MAB = nullptr;
-    if (ShowEncoding || T.isOSNaCl()) { // @LOCALMOD
+    if (ShowEncoding) {
       CE = TheTarget->createMCCodeEmitter(*MCII, *MRI, Ctx);
       MAB = TheTarget->createMCAsmBackend(*MRI, TripleName, MCPU);
     }
@@ -470,18 +495,14 @@ int main(int argc, char **argv) {
         Ctx, std::move(FOut), /*asmverbose*/ true,
         /*useDwarfDirectory*/ true, IP, CE, MAB, ShowInst));
 
-    // @LOCALMOD-START
-    if (T.isOSNaCl()) {
-      Str->InitSections(NoExecStack);
-      initializeNaClMCStreamer(*Str.get(), Ctx, T);
-    }
-    // @LOCALMOD-END
-
   } else if (FileType == OFT_Null) {
     Str.reset(TheTarget->createNullStreamer(Ctx));
   } else {
     assert(FileType == OFT_ObjectFile && "Invalid file type!");
 
+    // Don't waste memory on names of temp labels.
+    Ctx.setUseNamesOnTempLabels(false);
+
     if (!Out->os().supportsSeeking()) {
       BOS = make_unique<buffer_ostream>(Out->os());
       OS = BOS.get();
@@ -494,14 +515,6 @@ int main(int argc, char **argv) {
                                                 /*DWARFMustBeAtTheEnd*/ false));
     if (NoExecStack)
       Str->InitSections(true);
-    // @LOCALMOD-BEGIN
-    if (T.isOSNaCl()) {
-      // If the above non-localmod does InitSections unconditionally,
-      // we can use that.
-      Str->InitSections(NoExecStack);
-      initializeNaClMCStreamer(*Str.get(), Ctx, T);
-    }
-    // @LOCALMOD-END
   }
 
   int Res = 1;
diff --git a/tools/llvm-nm/CMakeLists.txt b/tools/llvm-nm/CMakeLists.txt
index 07bec8f042d..3d40245b7e0 100644
--- a/tools/llvm-nm/CMakeLists.txt
+++ b/tools/llvm-nm/CMakeLists.txt
@@ -1,7 +1,6 @@
 set(LLVM_LINK_COMPONENTS
   ${LLVM_TARGETS_TO_BUILD}
   Core
-  NaClBitReader # @LOCALMOD
   NaClTransforms # XXX EMSCRIPTEN
   Object
   Support
diff --git a/tools/llvm-nm/LLVMBuild.txt b/tools/llvm-nm/LLVMBuild.txt
index dd987614b36..3699f962b56 100644
--- a/tools/llvm-nm/LLVMBuild.txt
+++ b/tools/llvm-nm/LLVMBuild.txt
@@ -19,4 +19,4 @@
 type = Tool
 name = llvm-nm
 parent = Tools
-required_libraries = BitReader NaClBitReader NaClTransforms Object
+required_libraries = BitReader NaClTransforms Object
diff --git a/tools/llvm-nm/Makefile b/tools/llvm-nm/Makefile
index d82bde4b0a5..ec20cef4258 100644
--- a/tools/llvm-nm/Makefile
+++ b/tools/llvm-nm/Makefile
@@ -9,7 +9,7 @@
 
 LEVEL := ../..
 TOOLNAME := llvm-nm
-LINK_COMPONENTS := all-targets bitreader naclbitreader object
+LINK_COMPONENTS := all-targets bitreader object
 
 # This tool has no plugins, optimize startup time.
 TOOL_NO_EXPORTS := 1
diff --git a/tools/llvm-nm/llvm-nm.cpp b/tools/llvm-nm/llvm-nm.cpp
index 712f8e2a610..e7ee3124ed7 100644
--- a/tools/llvm-nm/llvm-nm.cpp
+++ b/tools/llvm-nm/llvm-nm.cpp
@@ -20,8 +20,6 @@
 #include "llvm/IR/GlobalAlias.h"
 #include "llvm/IR/GlobalVariable.h"
 #include "llvm/IR/LLVMContext.h"
-#include "llvm/Bitcode/NaCl/NaClReaderWriter.h" // @LOCALMOD
-#include "llvm/IRReader/IRReader.h"             // @LOCALMOD
 #include "llvm/Object/Archive.h"
 #include "llvm/Object/COFF.h"
 #include "llvm/Object/ELFObjectFile.h"
@@ -161,21 +159,6 @@ bool MultipleFiles = false;
 bool HadError = false;
 
 std::string ToolName;
-
-// @LOCALMOD-BEGIN
-cl::opt<NaClFileFormat> InputFileFormat(
-    "bitcode-format", cl::desc("Define format of input file:"),
-    cl::values(clEnumValN(LLVMFormat, "llvm", "LLVM file (default)"),
-               clEnumValN(PNaClFormat, "pnacl", "PNaCl bitcode file"),
-               clEnumValEnd),
-    cl::init(LLVMFormat));
-
-static cl::opt<bool>
-VerboseErrors(
-    "verbose-parse-errors",
-    cl::desc("Print out more descriptive PNaCl bitcode parse errors"),
-    cl::init(false));
-// @LOCALMOD-END
 }
 
 static void error(Twine Message, Twine Path = Twine()) {
@@ -197,67 +180,25 @@ struct NMSymbol {
   uint64_t Size;
   char TypeChar;
   StringRef Name;
-  DataRefImpl Symb;
+  BasicSymbolRef Sym;
 };
 }
 
 static bool compareSymbolAddress(const NMSymbol &A, const NMSymbol &B) {
-  if (!ReverseSort) {
-    if (A.Address < B.Address)
-      return true;
-    if (A.Address == B.Address && A.Name < B.Name)
-      return true;
-    if (A.Address == B.Address && A.Name == B.Name && A.Size < B.Size)
-      return true;
-    return false;
-  }
-
-  if (A.Address > B.Address)
-    return true;
-  if (A.Address == B.Address && A.Name > B.Name)
-    return true;
-  if (A.Address == B.Address && A.Name == B.Name && A.Size > B.Size)
-    return true;
-  return false;
+  bool ADefined = !(A.Sym.getFlags() & SymbolRef::SF_Undefined);
+  bool BDefined = !(B.Sym.getFlags() & SymbolRef::SF_Undefined);
+  return std::make_tuple(ADefined, A.Address, A.Name, A.Size) <
+         std::make_tuple(BDefined, B.Address, B.Name, B.Size);
 }
 
 static bool compareSymbolSize(const NMSymbol &A, const NMSymbol &B) {
-  if (!ReverseSort) {
-    if (A.Size < B.Size)
-      return true;
-    if (A.Size == B.Size && A.Name < B.Name)
-      return true;
-    if (A.Size == B.Size && A.Name == B.Name && A.Address < B.Address)
-      return true;
-    return false;
-  }
-
-  if (A.Size > B.Size)
-    return true;
-  if (A.Size == B.Size && A.Name > B.Name)
-    return true;
-  if (A.Size == B.Size && A.Name == B.Name && A.Address > B.Address)
-    return true;
-  return false;
+  return std::make_tuple(A.Size, A.Name, A.Address) <
+         std::make_tuple(B.Size, B.Name, B.Address);
 }
 
 static bool compareSymbolName(const NMSymbol &A, const NMSymbol &B) {
-  if (!ReverseSort) {
-    if (A.Name < B.Name)
-      return true;
-    if (A.Name == B.Name && A.Size < B.Size)
-      return true;
-    if (A.Name == B.Name && A.Size == B.Size && A.Address < B.Address)
-      return true;
-    return false;
-  }
-  if (A.Name > B.Name)
-    return true;
-  if (A.Name == B.Name && A.Size > B.Size)
-    return true;
-  if (A.Name == B.Name && A.Size == B.Size && A.Address > B.Address)
-    return true;
-  return false;
+  return std::make_tuple(A.Name, A.Size, A.Address) <
+         std::make_tuple(B.Name, B.Size, B.Address);
 }
 
 static char isSymbolList64Bit(SymbolicFile &Obj) {
@@ -267,15 +208,7 @@ static char isSymbolList64Bit(SymbolicFile &Obj) {
     return false;
   if (MachOObjectFile *MachO = dyn_cast<MachOObjectFile>(&Obj))
     return MachO->is64Bit();
-  if (isa<ELF32LEObjectFile>(Obj))
-    return false;
-  if (isa<ELF64LEObjectFile>(Obj))
-    return true;
-  if (isa<ELF32BEObjectFile>(Obj))
-    return false;
-  if (isa<ELF64BEObjectFile>(Obj))
-    return true;
-  return false;
+  return cast<ELFObjectFileBase>(Obj).getBytesInAddress() == 8;
 }
 
 static StringRef CurrentFilename;
@@ -299,11 +232,12 @@ static void darwinPrintSymbol(MachOObjectFile *MachO, SymbolListT::iterator I,
   uint16_t NDesc;
   uint32_t NStrx;
   uint64_t NValue;
+  DataRefImpl SymDRI = I->Sym.getRawDataRefImpl();
   if (MachO->is64Bit()) {
     H_64 = MachO->MachOObjectFile::getHeader64();
     Filetype = H_64.filetype;
     Flags = H_64.flags;
-    STE_64 = MachO->getSymbol64TableEntry(I->Symb);
+    STE_64 = MachO->getSymbol64TableEntry(SymDRI);
     NType = STE_64.n_type;
     NSect = STE_64.n_sect;
     NDesc = STE_64.n_desc;
@@ -313,7 +247,7 @@ static void darwinPrintSymbol(MachOObjectFile *MachO, SymbolListT::iterator I,
     H = MachO->MachOObjectFile::getHeader();
     Filetype = H.filetype;
     Flags = H.flags;
-    STE = MachO->getSymbolTableEntry(I->Symb);
+    STE = MachO->getSymbolTableEntry(SymDRI);
     NType = STE.n_type;
     NSect = STE.n_sect;
     NDesc = STE.n_desc;
@@ -381,7 +315,7 @@ static void darwinPrintSymbol(MachOObjectFile *MachO, SymbolListT::iterator I,
     break;
   case MachO::N_SECT: {
     section_iterator Sec = MachO->section_end();
-    MachO->getSymbolSection(I->Symb, Sec);
+    MachO->getSymbolSection(I->Sym.getRawDataRefImpl(), Sec);
     DataRefImpl Ref = Sec->getRawDataRefImpl();
     StringRef SectionName;
     MachO->getSectionName(Ref, SectionName);
@@ -440,7 +374,7 @@ static void darwinPrintSymbol(MachOObjectFile *MachO, SymbolListT::iterator I,
   if ((NType & MachO::N_TYPE) == MachO::N_INDR) {
     outs() << I->Name << " (for ";
     StringRef IndirectName;
-    if (MachO->getIndirectName(I->Symb, IndirectName))
+    if (MachO->getIndirectName(I->Sym.getRawDataRefImpl(), IndirectName))
       outs() << "?)";
     else
       outs() << IndirectName << ")";
@@ -523,13 +457,14 @@ static void darwinPrintStab(MachOObjectFile *MachO, SymbolListT::iterator I) {
   uint8_t NType;
   uint8_t NSect;
   uint16_t NDesc;
+  DataRefImpl SymDRI = I->Sym.getRawDataRefImpl();
   if (MachO->is64Bit()) {
-    STE_64 = MachO->getSymbol64TableEntry(I->Symb);
+    STE_64 = MachO->getSymbol64TableEntry(SymDRI);
     NType = STE_64.n_type;
     NSect = STE_64.n_sect;
     NDesc = STE_64.n_desc;
   } else {
-    STE = MachO->getSymbolTableEntry(I->Symb);
+    STE = MachO->getSymbolTableEntry(SymDRI);
     NType = STE.n_type;
     NSect = STE.n_sect;
     NDesc = STE.n_desc;
@@ -551,12 +486,17 @@ static void sortAndPrintSymbolList(SymbolicFile &Obj, bool printName,
                                    std::string ArchiveName,
                                    std::string ArchitectureName) {
   if (!NoSort) {
+    std::function<bool(const NMSymbol &, const NMSymbol &)> Cmp;
     if (NumericSort)
-      std::sort(SymbolList.begin(), SymbolList.end(), compareSymbolAddress);
+      Cmp = compareSymbolAddress;
     else if (SizeSort)
-      std::sort(SymbolList.begin(), SymbolList.end(), compareSymbolSize);
+      Cmp = compareSymbolSize;
     else
-      std::sort(SymbolList.begin(), SymbolList.end(), compareSymbolName);
+      Cmp = compareSymbolName;
+
+    if (ReverseSort)
+      Cmp = [=](const NMSymbol &A, const NMSymbol &B) { return Cmp(B, A); };
+    std::sort(SymbolList.begin(), SymbolList.end(), Cmp);
   }
 
   if (!PrintFileName) {
@@ -582,11 +522,13 @@ static void sortAndPrintSymbolList(SymbolicFile &Obj, bool printName,
 
   for (SymbolListT::iterator I = SymbolList.begin(), E = SymbolList.end();
        I != E; ++I) {
-    if ((I->TypeChar != 'U') && UndefinedOnly)
+    uint32_t SymFlags = I->Sym.getFlags();
+    bool Undefined = SymFlags & SymbolRef::SF_Undefined;
+    if (!Undefined && UndefinedOnly)
       continue;
-    if ((I->TypeChar == 'U') && DefinedOnly)
+    if (Undefined && DefinedOnly)
       continue;
-    if (SizeSort && !PrintAddress && I->Size == UnknownAddressOrSize)
+    if (SizeSort && !PrintAddress)
       continue;
     if (PrintFileName) {
       if (!ArchitectureName.empty())
@@ -603,16 +545,15 @@ static void sortAndPrintSymbolList(SymbolicFile &Obj, bool printName,
     char SymbolAddrStr[18] = "";
     char SymbolSizeStr[18] = "";
 
-    if (OutputFormat == sysv || I->Address == UnknownAddressOrSize)
+    if (OutputFormat == sysv || I->TypeChar == 'U')
       strcpy(SymbolAddrStr, printBlanks);
     if (OutputFormat == sysv)
       strcpy(SymbolSizeStr, printBlanks);
 
-    if (I->Address != UnknownAddressOrSize)
+    if (I->TypeChar != 'U')
       format(printFormat, I->Address)
           .print(SymbolAddrStr, sizeof(SymbolAddrStr));
-    if (I->Size != UnknownAddressOrSize)
-      format(printFormat, I->Size).print(SymbolSizeStr, sizeof(SymbolSizeStr));
+    format(printFormat, I->Size).print(SymbolSizeStr, sizeof(SymbolSizeStr));
 
     // If OutputFormat is darwin or we are printing Mach-O symbols in hex and
     // we have a MachOObjectFile, call darwinPrintSymbol to print as darwin's
@@ -630,8 +571,7 @@ static void sortAndPrintSymbolList(SymbolicFile &Obj, bool printName,
         outs() << SymbolAddrStr << ' ';
       if (PrintSize) {
         outs() << SymbolSizeStr;
-        if (I->Size != UnknownAddressOrSize)
-          outs() << ' ';
+        outs() << ' ';
       }
       outs() << I->TypeChar;
       if (I->TypeChar == '-' && MachO)
@@ -649,25 +589,20 @@ static void sortAndPrintSymbolList(SymbolicFile &Obj, bool printName,
   SymbolList.clear();
 }
 
-template <class ELFT>
-static char getSymbolNMTypeChar(ELFObjectFile<ELFT> &Obj,
+static char getSymbolNMTypeChar(ELFObjectFileBase &Obj,
                                 basic_symbol_iterator I) {
-  typedef typename ELFObjectFile<ELFT>::Elf_Sym Elf_Sym;
-  typedef typename ELFObjectFile<ELFT>::Elf_Shdr Elf_Shdr;
-
   // OK, this is ELF
-  symbol_iterator SymI(I);
+  elf_symbol_iterator SymI(I);
 
-  DataRefImpl Symb = I->getRawDataRefImpl();
-  const Elf_Sym *ESym = Obj.getSymbol(Symb);
-  const ELFFile<ELFT> &EF = *Obj.getELFFile();
-  const Elf_Shdr *ESec = EF.getSection(ESym);
+  elf_section_iterator SecI = Obj.section_end();
+  if (error(SymI->getSection(SecI)))
+    return '?';
 
-  if (ESec) {
-    switch (ESec->sh_type) {
+  if (SecI != Obj.section_end()) {
+    switch (SecI->getType()) {
     case ELF::SHT_PROGBITS:
     case ELF::SHT_DYNAMIC:
-      switch (ESec->sh_flags) {
+      switch (SecI->getFlags()) {
       case (ELF::SHF_ALLOC | ELF::SHF_EXECINSTR):
         return 't';
       case (ELF::SHF_TLS | ELF::SHF_ALLOC | ELF::SHF_WRITE):
@@ -684,17 +619,17 @@ static char getSymbolNMTypeChar(ELFObjectFile<ELFT> &Obj,
     }
   }
 
-  if (ESym->getType() == ELF::STT_SECTION) {
-    StringRef Name;
-    if (error(SymI->getName(Name)))
+  if (SymI->getELFType() == ELF::STT_SECTION) {
+    ErrorOr<StringRef> Name = SymI->getName();
+    if (error(Name.getError()))
       return '?';
-    return StringSwitch<char>(Name)
+    return StringSwitch<char>(*Name)
         .StartsWith(".debug", 'N')
         .StartsWith(".note", 'n')
         .Default('?');
   }
 
-  return '?';
+  return 'n';
 }
 
 static char getSymbolNMTypeChar(COFFObjectFile &Obj, symbol_iterator I) {
@@ -702,11 +637,11 @@ static char getSymbolNMTypeChar(COFFObjectFile &Obj, symbol_iterator I) {
   // OK, this is COFF.
   symbol_iterator SymI(I);
 
-  StringRef Name;
-  if (error(SymI->getName(Name)))
+  ErrorOr<StringRef> Name = SymI->getName();
+  if (error(Name.getError()))
     return '?';
 
-  char Ret = StringSwitch<char>(Name)
+  char Ret = StringSwitch<char>(*Name)
                  .StartsWith(".debug", 'N')
                  .StartsWith(".sxdata", 'N')
                  .Default('?');
@@ -801,26 +736,12 @@ static char getSymbolNMTypeChar(IRObjectFile &Obj, basic_symbol_iterator I) {
   return getSymbolNMTypeChar(*GV);
 }
 
-template <class ELFT>
-static bool isELFObject(ELFObjectFile<ELFT> &Obj, symbol_iterator I) {
-  typedef typename ELFObjectFile<ELFT>::Elf_Sym Elf_Sym;
-
-  DataRefImpl Symb = I->getRawDataRefImpl();
-  const Elf_Sym *ESym = Obj.getSymbol(Symb);
-
-  return ESym->getType() == ELF::STT_OBJECT;
-}
-
 static bool isObject(SymbolicFile &Obj, basic_symbol_iterator I) {
-  if (ELF32LEObjectFile *ELF = dyn_cast<ELF32LEObjectFile>(&Obj))
-    return isELFObject(*ELF, I);
-  if (ELF64LEObjectFile *ELF = dyn_cast<ELF64LEObjectFile>(&Obj))
-    return isELFObject(*ELF, I);
-  if (ELF32BEObjectFile *ELF = dyn_cast<ELF32BEObjectFile>(&Obj))
-    return isELFObject(*ELF, I);
-  if (ELF64BEObjectFile *ELF = dyn_cast<ELF64BEObjectFile>(&Obj))
-    return isELFObject(*ELF, I);
-  return false;
+  auto *ELF = dyn_cast<ELFObjectFileBase>(&Obj);
+  if (!ELF)
+    return false;
+
+  return elf_symbol_iterator(I)->getELFType() == ELF::STT_OBJECT;
 }
 
 static char getNMTypeChar(SymbolicFile &Obj, basic_symbol_iterator I) {
@@ -847,14 +768,8 @@ static char getNMTypeChar(SymbolicFile &Obj, basic_symbol_iterator I) {
     Ret = getSymbolNMTypeChar(*COFF, I);
   else if (MachOObjectFile *MachO = dyn_cast<MachOObjectFile>(&Obj))
     Ret = getSymbolNMTypeChar(*MachO, I);
-  else if (ELF32LEObjectFile *ELF = dyn_cast<ELF32LEObjectFile>(&Obj))
-    Ret = getSymbolNMTypeChar(*ELF, I);
-  else if (ELF64LEObjectFile *ELF = dyn_cast<ELF64LEObjectFile>(&Obj))
-    Ret = getSymbolNMTypeChar(*ELF, I);
-  else if (ELF32BEObjectFile *ELF = dyn_cast<ELF32BEObjectFile>(&Obj))
-    Ret = getSymbolNMTypeChar(*ELF, I);
   else
-    Ret = getSymbolNMTypeChar(cast<ELF64BEObjectFile>(Obj), I);
+    Ret = getSymbolNMTypeChar(cast<ELFObjectFileBase>(Obj), I);
 
   if (Symflags & object::SymbolRef::SF_Global)
     Ret = toupper(Ret);
@@ -888,8 +803,8 @@ static unsigned getNsectForSegSect(MachOObjectFile *Obj) {
 // It is called once for each symbol in a Mach-O file from
 // dumpSymbolNamesFromObject() and returns the section number for that symbol
 // if it is in a section, else it returns 0.
-static unsigned getNsectInMachO(MachOObjectFile &Obj, basic_symbol_iterator I) {
-  DataRefImpl Symb = I->getRawDataRefImpl();
+static unsigned getNsectInMachO(MachOObjectFile &Obj, BasicSymbolRef Sym) {
+  DataRefImpl Symb = Sym.getRawDataRefImpl();
   if (Obj.is64Bit()) {
     MachO::nlist_64 STE = Obj.getSymbol64TableEntry(Symb);
     if ((STE.n_type & MachO::N_TYPE) == MachO::N_SECT)
@@ -906,17 +821,16 @@ static void dumpSymbolNamesFromObject(SymbolicFile &Obj, bool printName,
                                       std::string ArchiveName = std::string(),
                                       std::string ArchitectureName =
                                         std::string()) {
-  basic_symbol_iterator IBegin = Obj.symbol_begin();
-  basic_symbol_iterator IEnd = Obj.symbol_end();
+  auto Symbols = Obj.symbols();
   if (DynamicSyms) {
-    if (!Obj.isELF()) {
+    const auto *E = dyn_cast<ELFObjectFileBase>(&Obj);
+    if (!E) {
       error("File format has no dynamic symbol table", Obj.getFileName());
       return;
     }
-    std::pair<symbol_iterator, symbol_iterator> IDyn =
-        getELFDynamicSymbolIterators(&Obj);
-    IBegin = IDyn.first;
-    IEnd = IDyn.second;
+    auto DynSymbols = E->getDynamicSymbolIterators();
+    Symbols =
+        make_range<basic_symbol_iterator>(DynSymbols.begin(), DynSymbols.end());
   }
   std::string NameBuffer;
   raw_string_ostream OS(NameBuffer);
@@ -930,13 +844,13 @@ static void dumpSymbolNamesFromObject(SymbolicFile &Obj, bool printName,
     if (Nsect == 0)
       return;
   }
-  for (basic_symbol_iterator I = IBegin; I != IEnd; ++I) {
-    uint32_t SymFlags = I->getFlags();
+  for (BasicSymbolRef Sym : Symbols) {
+    uint32_t SymFlags = Sym.getFlags();
     if (!DebugSyms && (SymFlags & SymbolRef::SF_FormatSpecific))
       continue;
     if (WithoutAliases) {
       if (IRObjectFile *IR = dyn_cast<IRObjectFile>(&Obj)) {
-        const GlobalValue *GV = IR->getSymbolGV(I->getRawDataRefImpl());
+        const GlobalValue *GV = IR->getSymbolGV(Sym.getRawDataRefImpl());
         if (GV && isa<GlobalAlias>(GV))
           continue;
       }
@@ -944,24 +858,27 @@ static void dumpSymbolNamesFromObject(SymbolicFile &Obj, bool printName,
     // If a "-s segname sectname" option was specified and this is a Mach-O
     // file and this section appears in this file, Nsect will be non-zero then
     // see if this symbol is a symbol from that section and if not skip it.
-    if (Nsect && Nsect != getNsectInMachO(*MachO, I))
+    if (Nsect && Nsect != getNsectInMachO(*MachO, Sym))
       continue;
     NMSymbol S;
-    S.Size = UnknownAddressOrSize;
-    S.Address = UnknownAddressOrSize;
-    if ((PrintSize || SizeSort) && isa<ObjectFile>(Obj)) {
-      symbol_iterator SymI = I;
-      if (error(SymI->getSize(S.Size)))
-        break;
+    S.Size = 0;
+    S.Address = 0;
+    if (PrintSize) {
+      if (isa<ELFObjectFileBase>(&Obj))
+        S.Size = ELFSymbolRef(Sym).getSize();
     }
-    if (PrintAddress && isa<ObjectFile>(Obj))
-      if (error(symbol_iterator(I)->getAddress(S.Address)))
+    if (PrintAddress && isa<ObjectFile>(Obj)) {
+      SymbolRef SymRef(Sym);
+      ErrorOr<uint64_t> AddressOrErr = SymRef.getAddress();
+      if (error(AddressOrErr.getError()))
         break;
-    S.TypeChar = getNMTypeChar(Obj, I);
-    if (error(I->printName(OS)))
+      S.Address = *AddressOrErr;
+    }
+    S.TypeChar = getNMTypeChar(Obj, Sym);
+    if (error(Sym.printName(OS)))
       break;
     OS << '\0';
-    S.Symb = I->getRawDataRefImpl();
+    S.Sym = Sym;
     SymbolList.push_back(S);
   }
 
@@ -1013,35 +930,6 @@ static void dumpSymbolNamesFromFile(std::string &Filename) {
     return;
 
   LLVMContext &Context = getGlobalContext();
-
-  // @LOCALMOD-BEGIN
-  // Support parsing PNaCl bitcode files
-  switch (InputFileFormat) {
-  case LLVMFormat:
-    break;
-  case PNaClFormat:
-  case AutodetectFileFormat:
-    report_fatal_error("command only supports LLVM file format!");
-  }
-  /* TODO(jfb) This is currently broken: the code base now requires an Object.
-  if (InputFileFormat == PNaClFormat) {
-    std::string VerboseBuffer;
-    raw_string_ostream VerboseStrm(VerboseBuffer);
-    raw_ostream *Verbose = VerboseErrors ? &VerboseStrm : nullptr;
-    ErrorOr<Module *> Result =
-        NaClParseBitcodeFile(BufferOrErr.get().release(), Verbose,
-        Context);
-    if (Result) {
-      DumpSymbolNamesFromModule(Result.get());
-      delete Result;
-    } else {
-      error(VerboseStrm.str() + Result.message()), Filename);
-      return;
-    }
-  }
-  */
-  // @LOCALMOD-END
-
   ErrorOr<std::unique_ptr<Binary>> BinaryOrErr = createBinary(
       BufferOrErr.get()->getMemBufferRef(), NoLLVMBitcode ? nullptr : &Context);
   if (error(BinaryOrErr.getError(), Filename))
diff --git a/tools/llvm-objdump/CMakeLists.txt b/tools/llvm-objdump/CMakeLists.txt
index d717653685c..1f2721ab545 100644
--- a/tools/llvm-objdump/CMakeLists.txt
+++ b/tools/llvm-objdump/CMakeLists.txt
@@ -1,5 +1,6 @@
 set(LLVM_LINK_COMPONENTS
   ${LLVM_TARGETS_TO_BUILD}
+  CodeGen
   DebugInfoDWARF
   MC
   MCDisassembler
diff --git a/tools/llvm-objdump/COFFDump.cpp b/tools/llvm-objdump/COFFDump.cpp
index 4a20b91b71f..8b94a50ea28 100644
--- a/tools/llvm-objdump/COFFDump.cpp
+++ b/tools/llvm-objdump/COFFDump.cpp
@@ -161,13 +161,15 @@ static std::error_code
 resolveSectionAndAddress(const COFFObjectFile *Obj, const SymbolRef &Sym,
                          const coff_section *&ResolvedSection,
                          uint64_t &ResolvedAddr) {
-  if (std::error_code EC = Sym.getAddress(ResolvedAddr))
+  ErrorOr<uint64_t> ResolvedAddrOrErr = Sym.getAddress();
+  if (std::error_code EC = ResolvedAddrOrErr.getError())
     return EC;
+  ResolvedAddr = *ResolvedAddrOrErr;
   section_iterator iter(Obj->section_begin());
   if (std::error_code EC = Sym.getSection(iter))
     return EC;
   ResolvedSection = Obj->getCOFFSection(*iter);
-  return object_error::success;
+  return std::error_code();
 }
 
 // Given a vector of relocations for a section and an offset into this section
@@ -177,12 +179,10 @@ static std::error_code resolveSymbol(const std::vector<RelocationRef> &Rels,
   for (std::vector<RelocationRef>::const_iterator I = Rels.begin(),
                                                   E = Rels.end();
                                                   I != E; ++I) {
-    uint64_t Ofs;
-    if (std::error_code EC = I->getOffset(Ofs))
-      return EC;
+    uint64_t Ofs = I->getOffset();
     if (Ofs == Offset) {
       Sym = *I->getSymbol();
-      return object_error::success;
+      return std::error_code();
     }
   }
   return object_error::parse_failed;
@@ -204,7 +204,7 @@ getSectionContents(const COFFObjectFile *Obj,
     return EC;
   if (std::error_code EC = Obj->getSectionContents(Section, Contents))
     return EC;
-  return object_error::success;
+  return std::error_code();
 }
 
 // Given a vector of relocations for a section and an offset into this section
@@ -215,9 +215,11 @@ static std::error_code resolveSymbolName(const std::vector<RelocationRef> &Rels,
   SymbolRef Sym;
   if (std::error_code EC = resolveSymbol(Rels, Offset, Sym))
     return EC;
-  if (std::error_code EC = Sym.getName(Name))
+  ErrorOr<StringRef> NameOrErr = Sym.getName();
+  if (std::error_code EC = NameOrErr.getError())
     return EC;
-  return object_error::success;
+  Name = *NameOrErr;
+  return std::error_code();
 }
 
 static void printCOFFSymbolAddress(llvm::raw_ostream &Out,
diff --git a/tools/llvm-objdump/ELFDump.cpp b/tools/llvm-objdump/ELFDump.cpp
index 9c091a410d8..7b44e3916c4 100644
--- a/tools/llvm-objdump/ELFDump.cpp
+++ b/tools/llvm-objdump/ELFDump.cpp
@@ -24,10 +24,8 @@ using namespace llvm::object;
 template <class ELFT> void printProgramHeaders(const ELFFile<ELFT> *o) {
   typedef ELFFile<ELFT> ELFO;
   outs() << "Program Header:\n";
-  for (typename ELFO::Elf_Phdr_Iter pi = o->begin_program_headers(),
-                                    pe = o->end_program_headers();
-                                    pi != pe; ++pi) {
-    switch (pi->p_type) {
+  for (const typename ELFO::Elf_Phdr &Phdr : o->program_headers()) {
+    switch (Phdr.p_type) {
     case ELF::PT_LOAD:
       outs() << "    LOAD ";
       break;
@@ -55,22 +53,16 @@ template <class ELFT> void printProgramHeaders(const ELFFile<ELFT> *o) {
 
     const char *Fmt = ELFT::Is64Bits ? "0x%016" PRIx64 " " : "0x%08" PRIx64 " ";
 
-    outs() << "off    "
-           << format(Fmt, (uint64_t)pi->p_offset)
-           << "vaddr "
-           << format(Fmt, (uint64_t)pi->p_vaddr)
-           << "paddr "
-           << format(Fmt, (uint64_t)pi->p_paddr)
-           << format("align 2**%u\n", countTrailingZeros<uint64_t>(pi->p_align))
-           << "         filesz "
-           << format(Fmt, (uint64_t)pi->p_filesz)
-           << "memsz "
-           << format(Fmt, (uint64_t)pi->p_memsz)
-           << "flags "
-           << ((pi->p_flags & ELF::PF_R) ? "r" : "-")
-           << ((pi->p_flags & ELF::PF_W) ? "w" : "-")
-           << ((pi->p_flags & ELF::PF_X) ? "x" : "-")
-           << "\n";
+    outs() << "off    " << format(Fmt, (uint64_t)Phdr.p_offset) << "vaddr "
+           << format(Fmt, (uint64_t)Phdr.p_vaddr) << "paddr "
+           << format(Fmt, (uint64_t)Phdr.p_paddr)
+           << format("align 2**%u\n",
+                     countTrailingZeros<uint64_t>(Phdr.p_align))
+           << "         filesz " << format(Fmt, (uint64_t)Phdr.p_filesz)
+           << "memsz " << format(Fmt, (uint64_t)Phdr.p_memsz) << "flags "
+           << ((Phdr.p_flags & ELF::PF_R) ? "r" : "-")
+           << ((Phdr.p_flags & ELF::PF_W) ? "w" : "-")
+           << ((Phdr.p_flags & ELF::PF_X) ? "x" : "-") << "\n";
   }
   outs() << "\n";
 }
diff --git a/tools/llvm-objdump/MachODump.cpp b/tools/llvm-objdump/MachODump.cpp
index a491a376861..7b868fa9d0d 100644
--- a/tools/llvm-objdump/MachODump.cpp
+++ b/tools/llvm-objdump/MachODump.cpp
@@ -17,7 +17,8 @@
 #include "llvm/ADT/StringExtras.h"
 #include "llvm/ADT/Triple.h"
 #include "llvm/Config/config.h"
-#include "llvm/DebugInfo/DWARF/DIContext.h"
+#include "llvm/DebugInfo/DIContext.h"
+#include "llvm/DebugInfo/DWARF/DWARFContext.h"
 #include "llvm/MC/MCAsmInfo.h"
 #include "llvm/MC/MCContext.h"
 #include "llvm/MC/MCDisassembler.h"
@@ -66,10 +67,6 @@ static cl::opt<bool> FullLeadingAddr("full-leading-addr",
 static cl::opt<bool> NoLeadingAddr("no-leading-addr",
                                    cl::desc("Print no leading address"));
 
-static cl::opt<bool>
-    PrintImmHex("print-imm-hex",
-                cl::desc("Use hex format for immediate values"));
-
 cl::opt<bool> llvm::UniversalHeaders("universal-headers",
                                      cl::desc("Print Mach-O universal headers "
                                               "(requires -macho)"));
@@ -79,6 +76,12 @@ cl::opt<bool>
                          cl::desc("Print archive headers for Mach-O archives "
                                   "(requires -macho)"));
 
+cl::opt<bool>
+    ArchiveMemberOffsets("archive-member-offsets",
+                         cl::desc("Print the offset to each archive member for "
+                                  "Mach-O archives (requires -macho and "
+                                  "-archive-headers)"));
+
 cl::opt<bool>
     llvm::IndirectSymbols("indirect-symbols",
                           cl::desc("Print indirect symbol table for Mach-O "
@@ -94,14 +97,6 @@ cl::opt<bool>
                        cl::desc("Print the linker optimization hints for "
                                 "Mach-O objects (requires -macho)"));
 
-cl::list<std::string>
-    llvm::DumpSections("section",
-                       cl::desc("Prints the specified segment,section for "
-                                "Mach-O objects (requires -macho)"));
-
-cl::opt<bool> llvm::Raw("raw",
-                        cl::desc("Have -section dump the raw binary contents"));
-
 cl::opt<bool>
     llvm::InfoPlist("info-plist",
                     cl::desc("Print the info plist section as strings for "
@@ -175,19 +170,8 @@ static const Target *GetTarget(const MachOObjectFile *MachOObj,
 
 struct SymbolSorter {
   bool operator()(const SymbolRef &A, const SymbolRef &B) {
-    SymbolRef::Type AType, BType;
-    A.getType(AType);
-    B.getType(BType);
-
-    uint64_t AAddr, BAddr;
-    if (AType != SymbolRef::ST_Function)
-      AAddr = 0;
-    else
-      A.getAddress(AAddr);
-    if (BType != SymbolRef::ST_Function)
-      BAddr = 0;
-    else
-      B.getAddress(BAddr);
+    uint64_t AAddr = (A.getType() != SymbolRef::ST_Function) ? 0 : A.getValue();
+    uint64_t BAddr = (B.getType() != SymbolRef::ST_Function) ? 0 : B.getValue();
     return AAddr < BAddr;
   }
 };
@@ -221,19 +205,19 @@ static uint64_t DumpDataInCode(const uint8_t *bytes, uint64_t Length,
   case MachO::DICE_KIND_DATA:
     if (Length >= 4) {
       if (!NoShowRawInsn)
-        DumpBytes(ArrayRef<uint8_t>(bytes, 4));
+        dumpBytes(ArrayRef<uint8_t>(bytes, 4), outs());
       Value = bytes[3] << 24 | bytes[2] << 16 | bytes[1] << 8 | bytes[0];
       outs() << "\t.long " << Value;
       Size = 4;
     } else if (Length >= 2) {
       if (!NoShowRawInsn)
-        DumpBytes(ArrayRef<uint8_t>(bytes, 2));
+        dumpBytes(ArrayRef<uint8_t>(bytes, 2), outs());
       Value = bytes[1] << 8 | bytes[0];
       outs() << "\t.short " << Value;
       Size = 2;
     } else {
       if (!NoShowRawInsn)
-        DumpBytes(ArrayRef<uint8_t>(bytes, 2));
+        dumpBytes(ArrayRef<uint8_t>(bytes, 2), outs());
       Value = bytes[0];
       outs() << "\t.byte " << Value;
       Size = 1;
@@ -245,14 +229,14 @@ static uint64_t DumpDataInCode(const uint8_t *bytes, uint64_t Length,
     break;
   case MachO::DICE_KIND_JUMP_TABLE8:
     if (!NoShowRawInsn)
-      DumpBytes(ArrayRef<uint8_t>(bytes, 1));
+      dumpBytes(ArrayRef<uint8_t>(bytes, 1), outs());
     Value = bytes[0];
     outs() << "\t.byte " << format("%3u", Value) << "\t@ KIND_JUMP_TABLE8\n";
     Size = 1;
     break;
   case MachO::DICE_KIND_JUMP_TABLE16:
     if (!NoShowRawInsn)
-      DumpBytes(ArrayRef<uint8_t>(bytes, 2));
+      dumpBytes(ArrayRef<uint8_t>(bytes, 2), outs());
     Value = bytes[1] << 8 | bytes[0];
     outs() << "\t.short " << format("%5u", Value & 0xffff)
            << "\t@ KIND_JUMP_TABLE16\n";
@@ -261,7 +245,7 @@ static uint64_t DumpDataInCode(const uint8_t *bytes, uint64_t Length,
   case MachO::DICE_KIND_JUMP_TABLE32:
   case MachO::DICE_KIND_ABS_JUMP_TABLE32:
     if (!NoShowRawInsn)
-      DumpBytes(ArrayRef<uint8_t>(bytes, 4));
+      dumpBytes(ArrayRef<uint8_t>(bytes, 4), outs());
     Value = bytes[3] << 24 | bytes[2] << 16 | bytes[1] << 8 | bytes[0];
     outs() << "\t.long " << Value;
     if (Kind == MachO::DICE_KIND_JUMP_TABLE32)
@@ -274,16 +258,16 @@ static uint64_t DumpDataInCode(const uint8_t *bytes, uint64_t Length,
   return Size;
 }
 
-static void getSectionsAndSymbols(const MachO::mach_header Header,
-                                  MachOObjectFile *MachOObj,
+static void getSectionsAndSymbols(MachOObjectFile *MachOObj,
                                   std::vector<SectionRef> &Sections,
                                   std::vector<SymbolRef> &Symbols,
                                   SmallVectorImpl<uint64_t> &FoundFns,
                                   uint64_t &BaseSegmentAddress) {
   for (const SymbolRef &Symbol : MachOObj->symbols()) {
-    StringRef SymName;
-    Symbol.getName(SymName);
-    if (!SymName.startswith("ltmp"))
+    ErrorOr<StringRef> SymName = Symbol.getName();
+    if (std::error_code EC = SymName.getError())
+      report_fatal_error(EC.message());
+    if (!SymName->startswith("ltmp"))
       Symbols.push_back(Symbol);
   }
 
@@ -293,10 +277,8 @@ static void getSectionsAndSymbols(const MachO::mach_header Header,
     Sections.push_back(Section);
   }
 
-  MachOObjectFile::LoadCommandInfo Command =
-      MachOObj->getFirstLoadCommandInfo();
   bool BaseSegmentAddressSet = false;
-  for (unsigned i = 0;; ++i) {
+  for (const auto &Command : MachOObj->load_commands()) {
     if (Command.C.cmd == MachO::LC_FUNCTION_STARTS) {
       // We found a function starts segment, parse the addresses for later
       // consumption.
@@ -312,11 +294,6 @@ static void getSectionsAndSymbols(const MachO::mach_header Header,
         BaseSegmentAddress = SLC.vmaddr;
       }
     }
-
-    if (i == Header.ncmds - 1)
-      break;
-    else
-      Command = MachOObj->getNextLoadCommandInfo(Command);
   }
 }
 
@@ -367,9 +344,10 @@ static void PrintIndirectSymbolTable(MachOObjectFile *O, bool verbose,
       if (indirect_symbol < Symtab.nsyms) {
         symbol_iterator Sym = O->getSymbolByIndex(indirect_symbol);
         SymbolRef Symbol = *Sym;
-        StringRef SymName;
-        Symbol.getName(SymName);
-        outs() << SymName;
+        ErrorOr<StringRef> SymName = Symbol.getName();
+        if (std::error_code EC = SymName.getError())
+          report_fatal_error(EC.message());
+        outs() << *SymName;
       } else {
         outs() << "?";
       }
@@ -379,9 +357,7 @@ static void PrintIndirectSymbolTable(MachOObjectFile *O, bool verbose,
 }
 
 static void PrintIndirectSymbols(MachOObjectFile *O, bool verbose) {
-  uint32_t LoadCommandCount = O->getHeader().ncmds;
-  MachOObjectFile::LoadCommandInfo Load = O->getFirstLoadCommandInfo();
-  for (unsigned I = 0;; ++I) {
+  for (const auto &Load : O->load_commands()) {
     if (Load.C.cmd == MachO::LC_SEGMENT_64) {
       MachO::segment_command_64 Seg = O->getSegment64LoadCommand(Load);
       for (unsigned J = 0; J < Seg.nsects; ++J) {
@@ -439,10 +415,6 @@ static void PrintIndirectSymbols(MachOObjectFile *O, bool verbose) {
         }
       }
     }
-    if (I == LoadCommandCount - 1)
-      break;
-    else
-      Load = O->getNextLoadCommandInfo(Load);
   }
 }
 
@@ -546,9 +518,8 @@ static void PrintLinkOptHints(MachOObjectFile *O) {
 }
 
 static void PrintDylibs(MachOObjectFile *O, bool JustId) {
-  uint32_t LoadCommandCount = O->getHeader().ncmds;
-  MachOObjectFile::LoadCommandInfo Load = O->getFirstLoadCommandInfo();
-  for (unsigned I = 0;; ++I) {
+  unsigned Index = 0;
+  for (const auto &Load : O->load_commands()) {
     if ((JustId && Load.C.cmd == MachO::LC_ID_DYLIB) ||
         (!JustId && (Load.C.cmd == MachO::LC_ID_DYLIB ||
                      Load.C.cmd == MachO::LC_LOAD_DYLIB ||
@@ -588,13 +559,9 @@ static void PrintDylibs(MachOObjectFile *O, bool JustId) {
           outs() << "LC_LOAD_UPWARD_DYLIB ";
         else
           outs() << "LC_??? ";
-        outs() << "command " << I << "\n";
+        outs() << "command " << Index++ << "\n";
       }
     }
-    if (I == LoadCommandCount - 1)
-      break;
-    else
-      Load = O->getNextLoadCommandInfo(Load);
   }
 }
 
@@ -604,14 +571,14 @@ static void CreateSymbolAddressMap(MachOObjectFile *O,
                                    SymbolAddressMap *AddrMap) {
   // Create a map of symbol addresses to symbol names.
   for (const SymbolRef &Symbol : O->symbols()) {
-    SymbolRef::Type ST;
-    Symbol.getType(ST);
+    SymbolRef::Type ST = Symbol.getType();
     if (ST == SymbolRef::ST_Function || ST == SymbolRef::ST_Data ||
         ST == SymbolRef::ST_Other) {
-      uint64_t Address;
-      Symbol.getAddress(Address);
-      StringRef SymName;
-      Symbol.getName(SymName);
+      uint64_t Address = Symbol.getValue();
+      ErrorOr<StringRef> SymNameOrErr = Symbol.getName();
+      if (std::error_code EC = SymNameOrErr.getError())
+        report_fatal_error(EC.message());
+      StringRef SymName = *SymNameOrErr;
       if (!SymName.startswith(".objc"))
         (*AddrMap)[Address] = SymName;
     }
@@ -804,7 +771,7 @@ static void DumpLiteralPointerSection(MachOObjectFile *O,
   // Set the size of the literal pointer.
   uint32_t lp_size = O->is64Bit() ? 8 : 4;
 
-  // Collect the external relocation symbols for the the literal pointers.
+  // Collect the external relocation symbols for the literal pointers.
   std::vector<std::pair<uint64_t, SymbolRef>> Relocs;
   for (const RelocationRef &Reloc : Section.relocations()) {
     DataRefImpl Rel;
@@ -814,8 +781,7 @@ static void DumpLiteralPointerSection(MachOObjectFile *O,
     RE = O->getRelocation(Rel);
     isExtern = O->getPlainRelocationExternal(RE);
     if (isExtern) {
-      uint64_t RelocOffset;
-      Reloc.getOffset(RelocOffset);
+      uint64_t RelocOffset = Reloc.getOffset();
       symbol_iterator RelocSym = Reloc.getSymbol();
       Relocs.push_back(std::make_pair(RelocOffset, *RelocSym));
     }
@@ -849,9 +815,10 @@ static void DumpLiteralPointerSection(MachOObjectFile *O,
         [&](const std::pair<uint64_t, SymbolRef> &P) { return P.first == i; });
     if (Reloc != Relocs.end()) {
       symbol_iterator RelocSym = Reloc->second;
-      StringRef SymName;
-      RelocSym->getName(SymName);
-      outs() << "external relocation entry for symbol:" << SymName << "\n";
+      ErrorOr<StringRef> SymName = RelocSym->getName();
+      if (std::error_code EC = SymName.getError())
+        report_fatal_error(EC.message());
+      outs() << "external relocation entry for symbol:" << *SymName << "\n";
       continue;
     }
 
@@ -1034,8 +1001,8 @@ static void DumpSectionContents(StringRef Filename, MachOObjectFile *O,
   if (verbose)
     CreateSymbolAddressMap(O, &AddrMap);
 
-  for (unsigned i = 0; i < DumpSections.size(); ++i) {
-    StringRef DumpSection = DumpSections[i];
+  for (unsigned i = 0; i < FilterSections.size(); ++i) {
+    StringRef DumpSection = FilterSections[i];
     std::pair<StringRef, StringRef> DumpSegSectName;
     DumpSegSectName = DumpSection.split(',');
     StringRef DumpSegName, DumpSectName;
@@ -1071,11 +1038,6 @@ static void DumpSectionContents(StringRef Filename, MachOObjectFile *O,
         uint32_t sect_size = BytesStr.size();
         uint64_t sect_addr = Section.getAddress();
 
-        if (Raw) {
-          outs().write(BytesStr.data(), BytesStr.size());
-          continue;
-        }
-
         outs() << "Contents of (" << SegName << "," << SectName
                << ") section\n";
 
@@ -1204,8 +1166,7 @@ static void ProcessMachO(StringRef Filename, MachOObjectFile *MachOOF,
   // UniversalHeaders or ArchiveHeaders.
   if (Disassemble || PrivateHeaders || ExportsTrie || Rebase || Bind ||
       LazyBind || WeakBind || IndirectSymbols || DataInCode || LinkOptHints ||
-      DylibsUsed || DylibId || ObjcMetaData ||
-      (DumpSections.size() != 0 && !Raw)) {
+      DylibsUsed || DylibId || ObjcMetaData || (FilterSections.size() != 0)) {
     outs() << Filename;
     if (!ArchiveMemberName.empty())
       outs() << '(' << ArchiveMemberName << ')';
@@ -1228,7 +1189,7 @@ static void ProcessMachO(StringRef Filename, MachOObjectFile *MachOOF,
     PrintSectionHeaders(MachOOF);
   if (SectionContents)
     PrintSectionContents(MachOOF);
-  if (DumpSections.size() != 0)
+  if (FilterSections.size() != 0)
     DumpSectionContents(Filename, MachOOF, !NonVerbose);
   if (InfoPlist)
     DumpInfoPlistSectionContents(Filename, MachOOF);
@@ -1558,7 +1519,7 @@ void llvm::ParseInputMachO(StringRef Filename) {
   if (Archive *A = dyn_cast<Archive>(&Bin)) {
     outs() << "Archive : " << Filename << "\n";
     if (ArchiveHeaders)
-      printArchiveHeaders(A, true, false);
+      printArchiveHeaders(A, !NonVerbose, ArchiveMemberOffsets);
     for (Archive::child_iterator I = A->child_begin(), E = A->child_end();
          I != E; ++I) {
       ErrorOr<std::unique_ptr<Binary>> ChildOrErr = I->getAsBinary();
@@ -1605,7 +1566,7 @@ void llvm::ParseInputMachO(StringRef Filename) {
                 outs() << " (architecture " << ArchitectureName << ")";
               outs() << "\n";
               if (ArchiveHeaders)
-                printArchiveHeaders(A.get(), true, false);
+                printArchiveHeaders(A.get(), !NonVerbose, ArchiveMemberOffsets);
               for (Archive::child_iterator AI = A->child_begin(),
                                            AE = A->child_end();
                    AI != AE; ++AI) {
@@ -1647,7 +1608,7 @@ void llvm::ParseInputMachO(StringRef Filename) {
             std::unique_ptr<Archive> &A = *AOrErr;
             outs() << "Archive : " << Filename << "\n";
             if (ArchiveHeaders)
-              printArchiveHeaders(A.get(), true, false);
+              printArchiveHeaders(A.get(), !NonVerbose, ArchiveMemberOffsets);
             for (Archive::child_iterator AI = A->child_begin(),
                                          AE = A->child_end();
                  AI != AE; ++AI) {
@@ -1684,7 +1645,7 @@ void llvm::ParseInputMachO(StringRef Filename) {
           outs() << " (architecture " << ArchitectureName << ")";
         outs() << "\n";
         if (ArchiveHeaders)
-          printArchiveHeaders(A.get(), true, false);
+          printArchiveHeaders(A.get(), !NonVerbose, ArchiveMemberOffsets);
         for (Archive::child_iterator AI = A->child_begin(), AE = A->child_end();
              AI != AE; ++AI) {
           ErrorOr<std::unique_ptr<Binary>> ChildOrErr = AI->getAsBinary();
@@ -1781,8 +1742,7 @@ static int SymbolizerGetOpInfo(void *DisInfo, uint64_t Pc, uint64_t Offset,
     bool r_scattered = false;
     uint32_t r_value, pair_r_value, r_type;
     for (const RelocationRef &Reloc : info->S.relocations()) {
-      uint64_t RelocOffset;
-      Reloc.getOffset(RelocOffset);
+      uint64_t RelocOffset = Reloc.getOffset();
       if (RelocOffset == sect_offset) {
         Rel = Reloc.getRawDataRefImpl();
         RE = info->O->getRelocation(Rel);
@@ -1813,9 +1773,10 @@ static int SymbolizerGetOpInfo(void *DisInfo, uint64_t Pc, uint64_t Offset,
       }
     }
     if (reloc_found && isExtern) {
-      StringRef SymName;
-      Symbol.getName(SymName);
-      const char *name = SymName.data();
+      ErrorOr<StringRef> SymName = Symbol.getName();
+      if (std::error_code EC = SymName.getError())
+        report_fatal_error(EC.message());
+      const char *name = SymName->data();
       op_info->AddSymbol.Present = 1;
       op_info->AddSymbol.Name = name;
       // For i386 extern relocation entries the value in the instruction is
@@ -1859,8 +1820,7 @@ static int SymbolizerGetOpInfo(void *DisInfo, uint64_t Pc, uint64_t Offset,
     bool isExtern = false;
     SymbolRef Symbol;
     for (const RelocationRef &Reloc : info->S.relocations()) {
-      uint64_t RelocOffset;
-      Reloc.getOffset(RelocOffset);
+      uint64_t RelocOffset = Reloc.getOffset();
       if (RelocOffset == sect_offset) {
         Rel = Reloc.getRawDataRefImpl();
         RE = info->O->getRelocation(Rel);
@@ -1880,9 +1840,10 @@ static int SymbolizerGetOpInfo(void *DisInfo, uint64_t Pc, uint64_t Offset,
       // is the offset from the external symbol.
       if (info->O->getAnyRelocationPCRel(RE))
         op_info->Value -= Pc + Offset + Size;
-      StringRef SymName;
-      Symbol.getName(SymName);
-      const char *name = SymName.data();
+      ErrorOr<StringRef> SymName = Symbol.getName();
+      if (std::error_code EC = SymName.getError())
+        report_fatal_error(EC.message());
+      const char *name = SymName->data();
       unsigned Type = info->O->getAnyRelocationType(RE);
       if (Type == MachO::X86_64_RELOC_SUBTRACTOR) {
         DataRefImpl RelNext = Rel;
@@ -1896,9 +1857,10 @@ static int SymbolizerGetOpInfo(void *DisInfo, uint64_t Pc, uint64_t Offset,
           op_info->SubtractSymbol.Name = name;
           symbol_iterator RelocSymNext = info->O->getSymbolByIndex(SymbolNum);
           Symbol = *RelocSymNext;
-          StringRef SymNameNext;
-          Symbol.getName(SymNameNext);
-          name = SymNameNext.data();
+          ErrorOr<StringRef> SymNameNext = Symbol.getName();
+          if (std::error_code EC = SymNameNext.getError())
+            report_fatal_error(EC.message());
+          name = SymNameNext->data();
         }
       }
       // TODO: add the VariantKinds to op_info->VariantKind for relocation types
@@ -1929,8 +1891,7 @@ static int SymbolizerGetOpInfo(void *DisInfo, uint64_t Pc, uint64_t Offset,
     auto Reloc =
         std::find_if(info->S.relocations().begin(), info->S.relocations().end(),
                      [&](const RelocationRef &Reloc) {
-                       uint64_t RelocOffset;
-                       Reloc.getOffset(RelocOffset);
+                       uint64_t RelocOffset = Reloc.getOffset();
                        return RelocOffset == sect_offset;
                      });
 
@@ -1966,9 +1927,10 @@ static int SymbolizerGetOpInfo(void *DisInfo, uint64_t Pc, uint64_t Offset,
     }
 
     if (isExtern) {
-      StringRef SymName;
-      Symbol.getName(SymName);
-      const char *name = SymName.data();
+      ErrorOr<StringRef> SymName = Symbol.getName();
+      if (std::error_code EC = SymName.getError())
+        report_fatal_error(EC.message());
+      const char *name = SymName->data();
       op_info->AddSymbol.Present = 1;
       op_info->AddSymbol.Name = name;
       switch (r_type) {
@@ -2056,8 +2018,7 @@ static int SymbolizerGetOpInfo(void *DisInfo, uint64_t Pc, uint64_t Offset,
     auto Reloc =
         std::find_if(info->S.relocations().begin(), info->S.relocations().end(),
                      [&](const RelocationRef &Reloc) {
-                       uint64_t RelocOffset;
-                       Reloc.getOffset(RelocOffset);
+                       uint64_t RelocOffset = Reloc.getOffset();
                        return RelocOffset == sect_offset;
                      });
 
@@ -2079,9 +2040,10 @@ static int SymbolizerGetOpInfo(void *DisInfo, uint64_t Pc, uint64_t Offset,
     // NOTE: Scattered relocations don't exist on arm64.
     if (!info->O->getPlainRelocationExternal(RE))
       return 0;
-    StringRef SymName;
-    Reloc->getSymbol()->getName(SymName);
-    const char *name = SymName.data();
+    ErrorOr<StringRef> SymName = Reloc->getSymbol()->getName();
+    if (std::error_code EC = SymName.getError())
+      report_fatal_error(EC.message());
+    const char *name = SymName->data();
     op_info->AddSymbol.Present = 1;
     op_info->AddSymbol.Name = name;
 
@@ -2125,9 +2087,7 @@ static int SymbolizerGetOpInfo(void *DisInfo, uint64_t Pc, uint64_t Offset,
 // it returns a pointer to that string.  Else it returns nullptr.
 static const char *GuessCstringPointer(uint64_t ReferenceValue,
                                        struct DisassembleInfo *info) {
-  uint32_t LoadCommandCount = info->O->getHeader().ncmds;
-  MachOObjectFile::LoadCommandInfo Load = info->O->getFirstLoadCommandInfo();
-  for (unsigned I = 0;; ++I) {
+  for (const auto &Load : info->O->load_commands()) {
     if (Load.C.cmd == MachO::LC_SEGMENT_64) {
       MachO::segment_command_64 Seg = info->O->getSegment64LoadCommand(Load);
       for (unsigned J = 0; J < Seg.nsects; ++J) {
@@ -2171,10 +2131,6 @@ static const char *GuessCstringPointer(uint64_t ReferenceValue,
         }
       }
     }
-    if (I == LoadCommandCount - 1)
-      break;
-    else
-      Load = info->O->getNextLoadCommandInfo(Load);
   }
   return nullptr;
 }
@@ -2185,11 +2141,9 @@ static const char *GuessCstringPointer(uint64_t ReferenceValue,
 // symbol name being referenced by the stub or pointer.
 static const char *GuessIndirectSymbol(uint64_t ReferenceValue,
                                        struct DisassembleInfo *info) {
-  uint32_t LoadCommandCount = info->O->getHeader().ncmds;
-  MachOObjectFile::LoadCommandInfo Load = info->O->getFirstLoadCommandInfo();
   MachO::dysymtab_command Dysymtab = info->O->getDysymtabLoadCommand();
   MachO::symtab_command Symtab = info->O->getSymtabLoadCommand();
-  for (unsigned I = 0;; ++I) {
+  for (const auto &Load : info->O->load_commands()) {
     if (Load.C.cmd == MachO::LC_SEGMENT_64) {
       MachO::segment_command_64 Seg = info->O->getSegment64LoadCommand(Load);
       for (unsigned J = 0; J < Seg.nsects; ++J) {
@@ -2216,9 +2170,10 @@ static const char *GuessIndirectSymbol(uint64_t ReferenceValue,
             if (indirect_symbol < Symtab.nsyms) {
               symbol_iterator Sym = info->O->getSymbolByIndex(indirect_symbol);
               SymbolRef Symbol = *Sym;
-              StringRef SymName;
-              Symbol.getName(SymName);
-              const char *name = SymName.data();
+              ErrorOr<StringRef> SymName = Symbol.getName();
+              if (std::error_code EC = SymName.getError())
+                report_fatal_error(EC.message());
+              const char *name = SymName->data();
               return name;
             }
           }
@@ -2250,19 +2205,16 @@ static const char *GuessIndirectSymbol(uint64_t ReferenceValue,
             if (indirect_symbol < Symtab.nsyms) {
               symbol_iterator Sym = info->O->getSymbolByIndex(indirect_symbol);
               SymbolRef Symbol = *Sym;
-              StringRef SymName;
-              Symbol.getName(SymName);
-              const char *name = SymName.data();
+              ErrorOr<StringRef> SymName = Symbol.getName();
+              if (std::error_code EC = SymName.getError())
+                report_fatal_error(EC.message());
+              const char *name = SymName->data();
               return name;
             }
           }
         }
       }
     }
-    if (I == LoadCommandCount - 1)
-      break;
-    else
-      Load = info->O->getNextLoadCommandInfo(Load);
   }
   return nullptr;
 }
@@ -2349,9 +2301,7 @@ static uint64_t GuessPointerPointer(uint64_t ReferenceValue,
   selref = false;
   msgref = false;
   cfstring = false;
-  uint32_t LoadCommandCount = info->O->getHeader().ncmds;
-  MachOObjectFile::LoadCommandInfo Load = info->O->getFirstLoadCommandInfo();
-  for (unsigned I = 0;; ++I) {
+  for (const auto &Load : info->O->load_commands()) {
     if (Load.C.cmd == MachO::LC_SEGMENT_64) {
       MachO::segment_command_64 Seg = info->O->getSegment64LoadCommand(Load);
       for (unsigned J = 0; J < Seg.nsects; ++J) {
@@ -2396,10 +2346,6 @@ static uint64_t GuessPointerPointer(uint64_t ReferenceValue,
       }
     }
     // TODO: Look for LC_SEGMENT for 32-bit Mach-O files.
-    if (I == LoadCommandCount - 1)
-      break;
-    else
-      Load = info->O->getNextLoadCommandInfo(Load);
   }
   return 0;
 }
@@ -2451,10 +2397,9 @@ static const char *get_pointer_32(uint32_t Address, uint32_t &offset,
 // for the specified section offset in the specified section reference.
 // If no relocation information is found and a non-zero ReferenceValue for the
 // symbol is passed, look up that address in the info's AddrMap.
-static const char *
-get_symbol_64(uint32_t sect_offset, SectionRef S, DisassembleInfo *info,
-              uint64_t &n_value,
-              uint64_t ReferenceValue = UnknownAddressOrSize) {
+static const char *get_symbol_64(uint32_t sect_offset, SectionRef S,
+                                 DisassembleInfo *info, uint64_t &n_value,
+                                 uint64_t ReferenceValue = 0) {
   n_value = 0;
   if (!info->verbose)
     return nullptr;
@@ -2466,8 +2411,7 @@ get_symbol_64(uint32_t sect_offset, SectionRef S, DisassembleInfo *info,
   bool isExtern = false;
   SymbolRef Symbol;
   for (const RelocationRef &Reloc : S.relocations()) {
-    uint64_t RelocOffset;
-    Reloc.getOffset(RelocOffset);
+    uint64_t RelocOffset = Reloc.getOffset();
     if (RelocOffset == sect_offset) {
       Rel = Reloc.getRawDataRefImpl();
       RE = info->O->getRelocation(Rel);
@@ -2487,13 +2431,13 @@ get_symbol_64(uint32_t sect_offset, SectionRef S, DisassembleInfo *info,
   // and return its name.
   const char *SymbolName = nullptr;
   if (reloc_found && isExtern) {
-    Symbol.getAddress(n_value);
-    if (n_value == UnknownAddressOrSize)
-      n_value = 0;
-    StringRef name;
-    Symbol.getName(name);
-    if (!name.empty()) {
-      SymbolName = name.data();
+    n_value = Symbol.getValue();
+    ErrorOr<StringRef> NameOrError = Symbol.getName();
+    if (std::error_code EC = NameOrError.getError())
+      report_fatal_error(EC.message());
+    StringRef Name = *NameOrError;
+    if (!Name.empty()) {
+      SymbolName = Name.data();
       return SymbolName;
     }
   }
@@ -2509,8 +2453,7 @@ get_symbol_64(uint32_t sect_offset, SectionRef S, DisassembleInfo *info,
 
   // We did not find an external relocation entry so look up the ReferenceValue
   // as an address of a symbol and if found return that symbol's name.
-  if (ReferenceValue != UnknownAddressOrSize)
-    SymbolName = GuessSymbolName(ReferenceValue, info->AddrMap);
+  SymbolName = GuessSymbolName(ReferenceValue, info->AddrMap);
 
   return SymbolName;
 }
@@ -5648,8 +5591,7 @@ static const char *GuessLiteralPointer(uint64_t ReferenceValue,
   bool isExtern = false;
   SymbolRef Symbol;
   for (const RelocationRef &Reloc : info->S.relocations()) {
-    uint64_t RelocOffset;
-    Reloc.getOffset(RelocOffset);
+    uint64_t RelocOffset = Reloc.getOffset();
     if (RelocOffset == sect_offset) {
       Rel = Reloc.getRawDataRefImpl();
       RE = info->O->getRelocation(Rel);
@@ -5670,7 +5612,7 @@ static const char *GuessLiteralPointer(uint64_t ReferenceValue,
     if (info->O->getAnyRelocationPCRel(RE)) {
       unsigned Type = info->O->getAnyRelocationType(RE);
       if (Type == MachO::X86_64_RELOC_SIGNED) {
-        Symbol.getAddress(ReferenceValue);
+        ReferenceValue = Symbol.getValue();
       }
     }
   }
@@ -6068,7 +6010,7 @@ static void DisassembleMachO(StringRef Filename, MachOObjectFile *MachOOF,
   SmallVector<uint64_t, 8> FoundFns;
   uint64_t BaseSegmentAddress;
 
-  getSectionsAndSymbols(Header, MachOOF, Sections, Symbols, FoundFns,
+  getSectionsAndSymbols(MachOOF, Sections, Symbols, FoundFns,
                         BaseSegmentAddress);
 
   // Sort the symbols by address, just in case they didn't come in that way.
@@ -6115,10 +6057,10 @@ static void DisassembleMachO(StringRef Filename, MachOObjectFile *MachOOF,
     }
 
     // Setup the DIContext
-    diContext.reset(DIContext::getDWARFContext(*DbgObj));
+    diContext.reset(new DWARFContextInMemory(*DbgObj));
   }
 
-  if (DumpSections.size() == 0)
+  if (FilterSections.size() == 0)
     outs() << "(" << DisSegName << "," << DisSectName << ") section\n";
 
   for (unsigned SectIdx = 0; SectIdx != Sections.size(); SectIdx++) {
@@ -6143,8 +6085,7 @@ static void DisassembleMachO(StringRef Filename, MachOObjectFile *MachOOF,
     // Parse relocations.
     std::vector<std::pair<uint64_t, SymbolRef>> Relocs;
     for (const RelocationRef &Reloc : Sections[SectIdx].relocations()) {
-      uint64_t RelocOffset;
-      Reloc.getOffset(RelocOffset);
+      uint64_t RelocOffset = Reloc.getOffset();
       uint64_t SectionAddress = Sections[SectIdx].getAddress();
       RelocOffset -= SectionAddress;
 
@@ -6159,14 +6100,14 @@ static void DisassembleMachO(StringRef Filename, MachOObjectFile *MachOOF,
     SymbolAddressMap AddrMap;
     bool DisSymNameFound = false;
     for (const SymbolRef &Symbol : MachOOF->symbols()) {
-      SymbolRef::Type ST;
-      Symbol.getType(ST);
+      SymbolRef::Type ST = Symbol.getType();
       if (ST == SymbolRef::ST_Function || ST == SymbolRef::ST_Data ||
           ST == SymbolRef::ST_Other) {
-        uint64_t Address;
-        Symbol.getAddress(Address);
-        StringRef SymName;
-        Symbol.getName(SymName);
+        uint64_t Address = Symbol.getValue();
+        ErrorOr<StringRef> SymNameOrErr = Symbol.getName();
+        if (std::error_code EC = SymNameOrErr.getError())
+          report_fatal_error(EC.message());
+        StringRef SymName = *SymNameOrErr;
         AddrMap[Address] = SymName;
         if (!DisSymName.empty() && DisSymName == SymName)
           DisSymNameFound = true;
@@ -6205,11 +6146,12 @@ static void DisassembleMachO(StringRef Filename, MachOObjectFile *MachOOF,
 
     // Disassemble symbol by symbol.
     for (unsigned SymIdx = 0; SymIdx != Symbols.size(); SymIdx++) {
-      StringRef SymName;
-      Symbols[SymIdx].getName(SymName);
+      ErrorOr<StringRef> SymNameOrErr = Symbols[SymIdx].getName();
+      if (std::error_code EC = SymNameOrErr.getError())
+        report_fatal_error(EC.message());
+      StringRef SymName = *SymNameOrErr;
 
-      SymbolRef::Type ST;
-      Symbols[SymIdx].getType(ST);
+      SymbolRef::Type ST = Symbols[SymIdx].getType();
       if (ST != SymbolRef::ST_Function)
         continue;
 
@@ -6223,9 +6165,8 @@ static void DisassembleMachO(StringRef Filename, MachOObjectFile *MachOOF,
         continue;
 
       // Start at the address of the symbol relative to the section's address.
-      uint64_t Start = 0;
+      uint64_t Start = Symbols[SymIdx].getValue();
       uint64_t SectionAddress = Sections[SectIdx].getAddress();
-      Symbols[SymIdx].getAddress(Start);
       Start -= SectionAddress;
 
       // Stop disassembling either at the beginning of the next symbol or at
@@ -6234,12 +6175,11 @@ static void DisassembleMachO(StringRef Filename, MachOObjectFile *MachOOF,
       uint64_t NextSym = 0;
       uint64_t NextSymIdx = SymIdx + 1;
       while (Symbols.size() > NextSymIdx) {
-        SymbolRef::Type NextSymType;
-        Symbols[NextSymIdx].getType(NextSymType);
+        SymbolRef::Type NextSymType = Symbols[NextSymIdx].getType();
         if (NextSymType == SymbolRef::ST_Function) {
           containsNextSym =
               Sections[SectIdx].containsSymbol(Symbols[NextSymIdx]);
-          Symbols[NextSymIdx].getAddress(NextSym);
+          NextSym = Symbols[NextSymIdx].getValue();
           NextSym -= SectionAddress;
           break;
         }
@@ -6306,7 +6246,7 @@ static void DisassembleMachO(StringRef Filename, MachOObjectFile *MachOOF,
                                            DebugOut, Annotations);
         if (gotInst) {
           if (!NoShowRawInsn) {
-            DumpBytes(ArrayRef<uint8_t>(Bytes.data() + Index, Size));
+            dumpBytes(ArrayRef<uint8_t>(Bytes.data() + Index, Size), outs());
           }
           formatted_raw_ostream FormattedOS(outs());
           Annotations.flush();
@@ -6371,7 +6311,7 @@ static void DisassembleMachO(StringRef Filename, MachOObjectFile *MachOOF,
           }
           if (!NoShowRawInsn) {
             outs() << "\t";
-            DumpBytes(ArrayRef<uint8_t>(Bytes.data() + Index, InstSize));
+            dumpBytes(ArrayRef<uint8_t>(Bytes.data() + Index, InstSize), outs());
           }
           IP->printInst(&Inst, outs(), "", *STI);
           outs() << "\n";
@@ -6471,13 +6411,16 @@ static void findUnwindRelocNameAddend(const MachOObjectFile *Obj,
                                       const RelocationRef &Reloc, uint64_t Addr,
                                       StringRef &Name, uint64_t &Addend) {
   if (Reloc.getSymbol() != Obj->symbol_end()) {
-    Reloc.getSymbol()->getName(Name);
+    ErrorOr<StringRef> NameOrErr = Reloc.getSymbol()->getName();
+    if (std::error_code EC = NameOrErr.getError())
+      report_fatal_error(EC.message());
+    Name = *NameOrErr;
     Addend = Addr;
     return;
   }
 
   auto RE = Obj->getRelocation(Reloc.getRawDataRefImpl());
-  SectionRef RelocSection = Obj->getRelocationSection(RE);
+  SectionRef RelocSection = Obj->getAnyRelocationSection(RE);
 
   uint64_t SectionAddr = RelocSection.getAddress();
 
@@ -6497,7 +6440,10 @@ static void findUnwindRelocNameAddend(const MachOObjectFile *Obj,
   Sym->second.getSection(SymSection);
   if (RelocSection == *SymSection) {
     // There's a valid symbol in the same section before this reference.
-    Sym->second.getName(Name);
+    ErrorOr<StringRef> NameOrErr = Sym->second.getName();
+    if (std::error_code EC = NameOrErr.getError())
+      report_fatal_error(EC.message());
+    Name = *NameOrErr;
     Addend = Addr - Sym->first;
     return;
   }
@@ -6514,7 +6460,7 @@ static void printUnwindRelocDest(const MachOObjectFile *Obj,
   StringRef Name;
   uint64_t Addend;
 
-  if (!Reloc.getObjectFile())
+  if (!Reloc.getObject())
     return;
 
   findUnwindRelocNameAddend(Obj, Symbols, Reloc, Addr, Name, Addend);
@@ -6550,8 +6496,7 @@ printMachOCompactUnwindSection(const MachOObjectFile *Obj,
   // Next we need to look at the relocations to find out what objects are
   // actually being referred to.
   for (const RelocationRef &Reloc : CompactUnwind.relocations()) {
-    uint64_t RelocAddress;
-    Reloc.getOffset(RelocAddress);
+    uint64_t RelocAddress = Reloc.getOffset();
 
     uint32_t EntryIdx = RelocAddress / EntrySize;
     uint32_t OffsetInEntry = RelocAddress - EntryIdx * EntrySize;
@@ -6587,7 +6532,7 @@ printMachOCompactUnwindSection(const MachOObjectFile *Obj,
            << format("0x%08" PRIx32, Entry.CompactEncoding) << '\n';
 
     // 4. The personality function, if present.
-    if (Entry.PersonalityReloc.getObjectFile()) {
+    if (Entry.PersonalityReloc.getObject()) {
       outs() << "    personality function: "
              << format("0x%" PRIx64, Entry.PersonalityAddr) << ' ';
       printUnwindRelocDest(Obj, Symbols, Entry.PersonalityReloc,
@@ -6596,7 +6541,7 @@ printMachOCompactUnwindSection(const MachOObjectFile *Obj,
     }
 
     // 5. This entry's language-specific data area.
-    if (Entry.LSDAReloc.getObjectFile()) {
+    if (Entry.LSDAReloc.getObject()) {
       outs() << "    LSDA:                 " << format("0x%" PRIx64,
                                                        Entry.LSDAAddr) << ' ';
       printUnwindRelocDest(Obj, Symbols, Entry.LSDAReloc, Entry.LSDAAddr);
@@ -6840,8 +6785,7 @@ void llvm::printMachOUnwindInfo(const MachOObjectFile *Obj) {
     if (Section == Obj->section_end())
       continue;
 
-    uint64_t Addr;
-    SymRef.getAddress(Addr);
+    uint64_t Addr = SymRef.getValue();
     Symbols.insert(std::make_pair(Addr, SymRef));
   }
 
@@ -8360,15 +8304,12 @@ static void PrintLinkEditDataCommand(MachO::linkedit_data_command ld,
     outs() << "\n";
 }
 
-static void PrintLoadCommands(const MachOObjectFile *Obj, uint32_t ncmds,
-                              uint32_t filetype, uint32_t cputype,
-                              bool verbose) {
-  if (ncmds == 0)
-    return;
+static void PrintLoadCommands(const MachOObjectFile *Obj, uint32_t filetype,
+                              uint32_t cputype, bool verbose) {
   StringRef Buf = Obj->getData();
-  MachOObjectFile::LoadCommandInfo Command = Obj->getFirstLoadCommandInfo();
-  for (unsigned i = 0;; ++i) {
-    outs() << "Load command " << i << "\n";
+  unsigned Index = 0;
+  for (const auto &Command : Obj->load_commands()) {
+    outs() << "Load command " << Index++ << "\n";
     if (Command.C.cmd == MachO::LC_SEGMENT) {
       MachO::segment_command SLC = Obj->getSegmentLoadCommand(Command);
       const char *sg_segname = SLC.segname;
@@ -8487,14 +8428,10 @@ static void PrintLoadCommands(const MachOObjectFile *Obj, uint32_t ncmds,
       // TODO: get and print the raw bytes of the load command.
     }
     // TODO: print all the other kinds of load commands.
-    if (i == ncmds - 1)
-      break;
-    else
-      Command = Obj->getNextLoadCommandInfo(Command);
   }
 }
 
-static void getAndPrintMachHeader(const MachOObjectFile *Obj, uint32_t &ncmds,
+static void getAndPrintMachHeader(const MachOObjectFile *Obj,
                                   uint32_t &filetype, uint32_t &cputype,
                                   bool verbose) {
   if (Obj->is64Bit()) {
@@ -8502,7 +8439,6 @@ static void getAndPrintMachHeader(const MachOObjectFile *Obj, uint32_t &ncmds,
     H_64 = Obj->getHeader64();
     PrintMachHeader(H_64.magic, H_64.cputype, H_64.cpusubtype, H_64.filetype,
                     H_64.ncmds, H_64.sizeofcmds, H_64.flags, verbose);
-    ncmds = H_64.ncmds;
     filetype = H_64.filetype;
     cputype = H_64.cputype;
   } else {
@@ -8510,7 +8446,6 @@ static void getAndPrintMachHeader(const MachOObjectFile *Obj, uint32_t &ncmds,
     H = Obj->getHeader();
     PrintMachHeader(H.magic, H.cputype, H.cpusubtype, H.filetype, H.ncmds,
                     H.sizeofcmds, H.flags, verbose);
-    ncmds = H.ncmds;
     filetype = H.filetype;
     cputype = H.cputype;
   }
@@ -8518,11 +8453,10 @@ static void getAndPrintMachHeader(const MachOObjectFile *Obj, uint32_t &ncmds,
 
 void llvm::printMachOFileHeader(const object::ObjectFile *Obj) {
   const MachOObjectFile *file = dyn_cast<const MachOObjectFile>(Obj);
-  uint32_t ncmds = 0;
   uint32_t filetype = 0;
   uint32_t cputype = 0;
-  getAndPrintMachHeader(file, ncmds, filetype, cputype, !NonVerbose);
-  PrintLoadCommands(file, ncmds, filetype, cputype, !NonVerbose);
+  getAndPrintMachHeader(file, filetype, cputype, !NonVerbose);
+  PrintLoadCommands(file, filetype, cputype, !NonVerbose);
 }
 
 //===----------------------------------------------------------------------===//
diff --git a/tools/llvm-objdump/llvm-objdump.cpp b/tools/llvm-objdump/llvm-objdump.cpp
index 71de8adca70..da2e29fc808 100644
--- a/tools/llvm-objdump/llvm-objdump.cpp
+++ b/tools/llvm-objdump/llvm-objdump.cpp
@@ -17,9 +17,11 @@
 //===----------------------------------------------------------------------===//
 
 #include "llvm-objdump.h"
+#include "llvm/ADT/Optional.h"
 #include "llvm/ADT/STLExtras.h"
 #include "llvm/ADT/StringExtras.h"
 #include "llvm/ADT/Triple.h"
+#include "llvm/CodeGen/FaultMaps.h"
 #include "llvm/MC/MCAsmInfo.h"
 #include "llvm/MC/MCContext.h"
 #include "llvm/MC/MCDisassembler.h"
@@ -32,12 +34,14 @@
 #include "llvm/MC/MCRelocationInfo.h"
 #include "llvm/MC/MCSubtargetInfo.h"
 #include "llvm/Object/Archive.h"
+#include "llvm/Object/ELFObjectFile.h"
 #include "llvm/Object/COFF.h"
 #include "llvm/Object/MachO.h"
 #include "llvm/Object/ObjectFile.h"
 #include "llvm/Support/Casting.h"
 #include "llvm/Support/CommandLine.h"
 #include "llvm/Support/Debug.h"
+#include "llvm/Support/Errc.h"
 #include "llvm/Support/FileSystem.h"
 #include "llvm/Support/Format.h"
 #include "llvm/Support/GraphWriter.h"
@@ -68,6 +72,13 @@ static cl::alias
 Disassembled("d", cl::desc("Alias for --disassemble"),
              cl::aliasopt(Disassemble));
 
+cl::opt<bool>
+llvm::DisassembleAll("disassemble-all",
+  cl::desc("Display assembler mnemonics for the machine instructions"));
+static cl::alias
+DisassembleAlld("D", cl::desc("Alias for --disassemble-all"),
+             cl::aliasopt(DisassembleAll));
+
 cl::opt<bool>
 llvm::Relocations("r", cl::desc("Display the relocation entries in the file"));
 
@@ -92,6 +103,10 @@ llvm::LazyBind("lazy-bind", cl::desc("Display mach-o lazy binding info"));
 cl::opt<bool>
 llvm::WeakBind("weak-bind", cl::desc("Display mach-o weak binding info"));
 
+cl::opt<bool>
+llvm::RawClangAST("raw-clang-ast",
+    cl::desc("Dump the raw binary contents of the clang AST section"));
+
 static cl::opt<bool>
 MachOOpt("macho", cl::desc("Use MachO specific object file parser"));
 static cl::alias
@@ -121,6 +136,13 @@ static cl::alias
 SectionHeadersShorter("h", cl::desc("Alias for --section-headers"),
                       cl::aliasopt(SectionHeaders));
 
+cl::list<std::string>
+llvm::FilterSections("section", cl::desc("Operate on the specified sections only. "
+                                         "With -macho dump segment,section"));
+cl::alias
+static FilterSectionsj("j", cl::desc("Alias for --section"),
+                 cl::aliasopt(llvm::FilterSections));
+
 cl::list<std::string>
 llvm::MAttrs("mattr",
   cl::CommaSeparated,
@@ -147,9 +169,81 @@ static cl::alias
 PrivateHeadersShort("p", cl::desc("Alias for --private-headers"),
                     cl::aliasopt(PrivateHeaders));
 
+cl::opt<bool>
+    llvm::PrintImmHex("print-imm-hex",
+                      cl::desc("Use hex format for immediate values"));
+
+cl::opt<bool> PrintFaultMaps("fault-map-section",
+                             cl::desc("Display contents of faultmap section"));
+
 static StringRef ToolName;
 static int ReturnValue = EXIT_SUCCESS;
 
+namespace {
+typedef std::function<bool(llvm::object::SectionRef const &)> FilterPredicate;
+
+class SectionFilterIterator {
+public:
+  SectionFilterIterator(FilterPredicate P,
+                        llvm::object::section_iterator const &I,
+                        llvm::object::section_iterator const &E)
+      : Predicate(P), Iterator(I), End(E) {
+    ScanPredicate();
+  }
+  llvm::object::SectionRef operator*() const { return *Iterator; }
+  SectionFilterIterator &operator++() {
+    ++Iterator;
+    ScanPredicate();
+    return *this;
+  }
+  bool operator!=(SectionFilterIterator const &Other) const {
+    return Iterator != Other.Iterator;
+  }
+
+private:
+  void ScanPredicate() {
+    while (Iterator != End && !Predicate(*Iterator)) {
+      ++Iterator;
+    }
+  }
+  FilterPredicate Predicate;
+  llvm::object::section_iterator Iterator;
+  llvm::object::section_iterator End;
+};
+
+class SectionFilter {
+public:
+  SectionFilter(FilterPredicate P, llvm::object::ObjectFile const &O)
+      : Predicate(P), Object(O) {}
+  SectionFilterIterator begin() {
+    return SectionFilterIterator(Predicate, Object.section_begin(),
+                                 Object.section_end());
+  }
+  SectionFilterIterator end() {
+    return SectionFilterIterator(Predicate, Object.section_end(),
+                                 Object.section_end());
+  }
+
+private:
+  FilterPredicate Predicate;
+  llvm::object::ObjectFile const &Object;
+};
+SectionFilter ToolSectionFilter(llvm::object::ObjectFile const &O) {
+  return SectionFilter([](llvm::object::SectionRef const &S) {
+                         if(FilterSections.empty())
+                           return true;
+                         llvm::StringRef String;
+                         std::error_code error = S.getName(String);
+                         if (error)
+                           return false;
+                         return std::find(FilterSections.begin(),
+                                          FilterSections.end(),
+                                          String) != FilterSections.end();
+                       },
+                       O);
+}
+}
+
 bool llvm::error(std::error_code EC) {
   if (!EC)
     return false;
@@ -160,6 +254,12 @@ bool llvm::error(std::error_code EC) {
   return true;
 }
 
+static void report_error(StringRef File, std::error_code EC) {
+  assert(EC);
+  errs() << ToolName << ": '" << File << "': " << EC.message() << ".\n";
+  ReturnValue = EXIT_FAILURE;
+}
+
 static const Target *getTarget(const ObjectFile *Obj = nullptr) {
   // Figure out the target triple.
   llvm::Triple TheTriple("unknown-unknown-unknown");
@@ -194,24 +294,519 @@ static const Target *getTarget(const ObjectFile *Obj = nullptr) {
   return TheTarget;
 }
 
-void llvm::DumpBytes(ArrayRef<uint8_t> bytes) {
-  static const char hex_rep[] = "0123456789abcdef";
-  SmallString<64> output;
+bool llvm::RelocAddressLess(RelocationRef a, RelocationRef b) {
+  return a.getOffset() < b.getOffset();
+}
+
+namespace {
+class PrettyPrinter {
+public:
+  virtual ~PrettyPrinter(){}
+  virtual void printInst(MCInstPrinter &IP, const MCInst *MI,
+                         ArrayRef<uint8_t> Bytes, uint64_t Address,
+                         raw_ostream &OS, StringRef Annot,
+                         MCSubtargetInfo const &STI) {
+    outs() << format("%8" PRIx64 ":", Address);
+    if (!NoShowRawInsn) {
+      outs() << "\t";
+      dumpBytes(Bytes, outs());
+    }
+    IP.printInst(MI, outs(), "", STI);
+  }
+};
+PrettyPrinter PrettyPrinterInst;
+class HexagonPrettyPrinter : public PrettyPrinter {
+public:
+  void printLead(ArrayRef<uint8_t> Bytes, uint64_t Address,
+                 raw_ostream &OS) {
+    uint32_t opcode =
+      (Bytes[3] << 24) | (Bytes[2] << 16) | (Bytes[1] << 8) | Bytes[0];
+    OS << format("%8" PRIx64 ":", Address);
+    if (!NoShowRawInsn) {
+      OS << "\t";
+      dumpBytes(Bytes.slice(0, 4), OS);
+      OS << format("%08" PRIx32, opcode);
+    }
+  }
+  void printInst(MCInstPrinter &IP, const MCInst *MI,
+                 ArrayRef<uint8_t> Bytes, uint64_t Address,
+                 raw_ostream &OS, StringRef Annot,
+                 MCSubtargetInfo const &STI) override {
+    std::string Buffer;
+    {
+      raw_string_ostream TempStream(Buffer);
+      IP.printInst(MI, TempStream, "", STI);
+    }
+    StringRef Contents(Buffer);
+    // Split off bundle attributes
+    auto PacketBundle = Contents.rsplit('\n');
+    // Split off first instruction from the rest
+    auto HeadTail = PacketBundle.first.split('\n');
+    auto Preamble = " { ";
+    auto Separator = "";
+    while(!HeadTail.first.empty()) {
+      OS << Separator;
+      Separator = "\n";
+      printLead(Bytes, Address, OS);
+      OS << Preamble;
+      Preamble = "   ";
+      StringRef Inst;
+      auto Duplex = HeadTail.first.split('\v');
+      if(!Duplex.second.empty()){
+        OS << Duplex.first;
+        OS << "; ";
+        Inst = Duplex.second;
+      }
+      else
+        Inst = HeadTail.first;
+      OS << Inst;
+      Bytes = Bytes.slice(4);
+      Address += 4;
+      HeadTail = HeadTail.second.split('\n');
+    }
+    OS << " } " << PacketBundle.second;
+  }
+};
+HexagonPrettyPrinter HexagonPrettyPrinterInst;
+PrettyPrinter &selectPrettyPrinter(Triple const &Triple) {
+  switch(Triple.getArch()) {
+  default:
+    return PrettyPrinterInst;
+  case Triple::hexagon:
+    return HexagonPrettyPrinterInst;
+  }
+}
+}
+
+template <class ELFT>
+static std::error_code getRelocationValueString(const ELFObjectFile<ELFT> *Obj,
+                                                DataRefImpl Rel,
+                                                SmallVectorImpl<char> &Result) {
+  typedef typename ELFObjectFile<ELFT>::Elf_Sym Elf_Sym;
+  typedef typename ELFObjectFile<ELFT>::Elf_Shdr Elf_Shdr;
+  typedef typename ELFObjectFile<ELFT>::Elf_Rel Elf_Rel;
+  typedef typename ELFObjectFile<ELFT>::Elf_Rela Elf_Rela;
+
+  const ELFFile<ELFT> &EF = *Obj->getELFFile();
+
+  ErrorOr<const Elf_Shdr *> SecOrErr = EF.getSection(Rel.d.a);
+  if (std::error_code EC = SecOrErr.getError())
+    return EC;
+  const Elf_Shdr *Sec = *SecOrErr;
+  ErrorOr<const Elf_Shdr *> SymTabOrErr = EF.getSection(Sec->sh_link);
+  if (std::error_code EC = SymTabOrErr.getError())
+    return EC;
+  const Elf_Shdr *SymTab = *SymTabOrErr;
+  assert(SymTab->sh_type == ELF::SHT_SYMTAB ||
+         SymTab->sh_type == ELF::SHT_DYNSYM);
+  ErrorOr<const Elf_Shdr *> StrTabSec = EF.getSection(SymTab->sh_link);
+  if (std::error_code EC = StrTabSec.getError())
+    return EC;
+  ErrorOr<StringRef> StrTabOrErr = EF.getStringTable(*StrTabSec);
+  if (std::error_code EC = StrTabOrErr.getError())
+    return EC;
+  StringRef StrTab = *StrTabOrErr;
+  uint8_t type;
+  StringRef res;
+  int64_t addend = 0;
+  uint16_t symbol_index = 0;
+  switch (Sec->sh_type) {
+  default:
+    return object_error::parse_failed;
+  case ELF::SHT_REL: {
+    const Elf_Rel *ERel = Obj->getRel(Rel);
+    type = ERel->getType(EF.isMips64EL());
+    symbol_index = ERel->getSymbol(EF.isMips64EL());
+    // TODO: Read implicit addend from section data.
+    break;
+  }
+  case ELF::SHT_RELA: {
+    const Elf_Rela *ERela = Obj->getRela(Rel);
+    type = ERela->getType(EF.isMips64EL());
+    symbol_index = ERela->getSymbol(EF.isMips64EL());
+    addend = ERela->r_addend;
+    break;
+  }
+  }
+  const Elf_Sym *symb =
+      EF.template getEntry<Elf_Sym>(Sec->sh_link, symbol_index);
+  StringRef Target;
+  ErrorOr<const Elf_Shdr *> SymSec = EF.getSection(symb);
+  if (std::error_code EC = SymSec.getError())
+    return EC;
+  if (symb->getType() == ELF::STT_SECTION) {
+    ErrorOr<StringRef> SecName = EF.getSectionName(*SymSec);
+    if (std::error_code EC = SecName.getError())
+      return EC;
+    Target = *SecName;
+  } else {
+    ErrorOr<StringRef> SymName = symb->getName(StrTab);
+    if (!SymName)
+      return SymName.getError();
+    Target = *SymName;
+  }
+  switch (EF.getHeader()->e_machine) {
+  case ELF::EM_X86_64:
+    switch (type) {
+    case ELF::R_X86_64_PC8:
+    case ELF::R_X86_64_PC16:
+    case ELF::R_X86_64_PC32: {
+      std::string fmtbuf;
+      raw_string_ostream fmt(fmtbuf);
+      fmt << Target << (addend < 0 ? "" : "+") << addend << "-P";
+      fmt.flush();
+      Result.append(fmtbuf.begin(), fmtbuf.end());
+    } break;
+    case ELF::R_X86_64_8:
+    case ELF::R_X86_64_16:
+    case ELF::R_X86_64_32:
+    case ELF::R_X86_64_32S:
+    case ELF::R_X86_64_64: {
+      std::string fmtbuf;
+      raw_string_ostream fmt(fmtbuf);
+      fmt << Target << (addend < 0 ? "" : "+") << addend;
+      fmt.flush();
+      Result.append(fmtbuf.begin(), fmtbuf.end());
+    } break;
+    default:
+      res = "Unknown";
+    }
+    break;
+  case ELF::EM_AARCH64: {
+    std::string fmtbuf;
+    raw_string_ostream fmt(fmtbuf);
+    fmt << Target;
+    if (addend != 0)
+      fmt << (addend < 0 ? "" : "+") << addend;
+    fmt.flush();
+    Result.append(fmtbuf.begin(), fmtbuf.end());
+    break;
+  }
+  case ELF::EM_386:
+  case ELF::EM_ARM:
+  case ELF::EM_HEXAGON:
+  case ELF::EM_MIPS:
+    res = Target;
+    break;
+  default:
+    res = "Unknown";
+  }
+  if (Result.empty())
+    Result.append(res.begin(), res.end());
+  return std::error_code();
+}
+
+static std::error_code getRelocationValueString(const ELFObjectFileBase *Obj,
+                                                const RelocationRef &RelRef,
+                                                SmallVectorImpl<char> &Result) {
+  DataRefImpl Rel = RelRef.getRawDataRefImpl();
+  if (auto *ELF32LE = dyn_cast<ELF32LEObjectFile>(Obj))
+    return getRelocationValueString(ELF32LE, Rel, Result);
+  if (auto *ELF64LE = dyn_cast<ELF64LEObjectFile>(Obj))
+    return getRelocationValueString(ELF64LE, Rel, Result);
+  if (auto *ELF32BE = dyn_cast<ELF32BEObjectFile>(Obj))
+    return getRelocationValueString(ELF32BE, Rel, Result);
+  auto *ELF64BE = cast<ELF64BEObjectFile>(Obj);
+  return getRelocationValueString(ELF64BE, Rel, Result);
+}
+
+static std::error_code getRelocationValueString(const COFFObjectFile *Obj,
+                                                const RelocationRef &Rel,
+                                                SmallVectorImpl<char> &Result) {
+  symbol_iterator SymI = Rel.getSymbol();
+  ErrorOr<StringRef> SymNameOrErr = SymI->getName();
+  if (std::error_code EC = SymNameOrErr.getError())
+    return EC;
+  StringRef SymName = *SymNameOrErr;
+  Result.append(SymName.begin(), SymName.end());
+  return std::error_code();
+}
+
+static void printRelocationTargetName(const MachOObjectFile *O,
+                                      const MachO::any_relocation_info &RE,
+                                      raw_string_ostream &fmt) {
+  bool IsScattered = O->isRelocationScattered(RE);
+
+  // Target of a scattered relocation is an address.  In the interest of
+  // generating pretty output, scan through the symbol table looking for a
+  // symbol that aligns with that address.  If we find one, print it.
+  // Otherwise, we just print the hex address of the target.
+  if (IsScattered) {
+    uint32_t Val = O->getPlainRelocationSymbolNum(RE);
+
+    for (const SymbolRef &Symbol : O->symbols()) {
+      std::error_code ec;
+      ErrorOr<uint64_t> Addr = Symbol.getAddress();
+      if ((ec = Addr.getError()))
+        report_fatal_error(ec.message());
+      if (*Addr != Val)
+        continue;
+      ErrorOr<StringRef> Name = Symbol.getName();
+      if (std::error_code EC = Name.getError())
+        report_fatal_error(EC.message());
+      fmt << *Name;
+      return;
+    }
+
+    // If we couldn't find a symbol that this relocation refers to, try
+    // to find a section beginning instead.
+    for (const SectionRef &Section : ToolSectionFilter(*O)) {
+      std::error_code ec;
+
+      StringRef Name;
+      uint64_t Addr = Section.getAddress();
+      if (Addr != Val)
+        continue;
+      if ((ec = Section.getName(Name)))
+        report_fatal_error(ec.message());
+      fmt << Name;
+      return;
+    }
+
+    fmt << format("0x%x", Val);
+    return;
+  }
 
-  for (char i: bytes) {
-    output.push_back(hex_rep[(i & 0xF0) >> 4]);
-    output.push_back(hex_rep[i & 0xF]);
-    output.push_back(' ');
+  StringRef S;
+  bool isExtern = O->getPlainRelocationExternal(RE);
+  uint64_t Val = O->getPlainRelocationSymbolNum(RE);
+
+  if (isExtern) {
+    symbol_iterator SI = O->symbol_begin();
+    advance(SI, Val);
+    ErrorOr<StringRef> SOrErr = SI->getName();
+    if (!error(SOrErr.getError()))
+      S = *SOrErr;
+  } else {
+    section_iterator SI = O->section_begin();
+    // Adjust for the fact that sections are 1-indexed.
+    advance(SI, Val - 1);
+    SI->getName(S);
   }
 
-  outs() << output.c_str();
+  fmt << S;
 }
 
-bool llvm::RelocAddressLess(RelocationRef a, RelocationRef b) {
-  uint64_t a_addr, b_addr;
-  if (error(a.getOffset(a_addr))) return false;
-  if (error(b.getOffset(b_addr))) return false;
-  return a_addr < b_addr;
+static std::error_code getRelocationValueString(const MachOObjectFile *Obj,
+                                                const RelocationRef &RelRef,
+                                                SmallVectorImpl<char> &Result) {
+  DataRefImpl Rel = RelRef.getRawDataRefImpl();
+  MachO::any_relocation_info RE = Obj->getRelocation(Rel);
+
+  unsigned Arch = Obj->getArch();
+
+  std::string fmtbuf;
+  raw_string_ostream fmt(fmtbuf);
+  unsigned Type = Obj->getAnyRelocationType(RE);
+  bool IsPCRel = Obj->getAnyRelocationPCRel(RE);
+
+  // Determine any addends that should be displayed with the relocation.
+  // These require decoding the relocation type, which is triple-specific.
+
+  // X86_64 has entirely custom relocation types.
+  if (Arch == Triple::x86_64) {
+    bool isPCRel = Obj->getAnyRelocationPCRel(RE);
+
+    switch (Type) {
+    case MachO::X86_64_RELOC_GOT_LOAD:
+    case MachO::X86_64_RELOC_GOT: {
+      printRelocationTargetName(Obj, RE, fmt);
+      fmt << "@GOT";
+      if (isPCRel)
+        fmt << "PCREL";
+      break;
+    }
+    case MachO::X86_64_RELOC_SUBTRACTOR: {
+      DataRefImpl RelNext = Rel;
+      Obj->moveRelocationNext(RelNext);
+      MachO::any_relocation_info RENext = Obj->getRelocation(RelNext);
+
+      // X86_64_RELOC_SUBTRACTOR must be followed by a relocation of type
+      // X86_64_RELOC_UNSIGNED.
+      // NOTE: Scattered relocations don't exist on x86_64.
+      unsigned RType = Obj->getAnyRelocationType(RENext);
+      if (RType != MachO::X86_64_RELOC_UNSIGNED)
+        report_fatal_error("Expected X86_64_RELOC_UNSIGNED after "
+                           "X86_64_RELOC_SUBTRACTOR.");
+
+      // The X86_64_RELOC_UNSIGNED contains the minuend symbol;
+      // X86_64_RELOC_SUBTRACTOR contains the subtrahend.
+      printRelocationTargetName(Obj, RENext, fmt);
+      fmt << "-";
+      printRelocationTargetName(Obj, RE, fmt);
+      break;
+    }
+    case MachO::X86_64_RELOC_TLV:
+      printRelocationTargetName(Obj, RE, fmt);
+      fmt << "@TLV";
+      if (isPCRel)
+        fmt << "P";
+      break;
+    case MachO::X86_64_RELOC_SIGNED_1:
+      printRelocationTargetName(Obj, RE, fmt);
+      fmt << "-1";
+      break;
+    case MachO::X86_64_RELOC_SIGNED_2:
+      printRelocationTargetName(Obj, RE, fmt);
+      fmt << "-2";
+      break;
+    case MachO::X86_64_RELOC_SIGNED_4:
+      printRelocationTargetName(Obj, RE, fmt);
+      fmt << "-4";
+      break;
+    default:
+      printRelocationTargetName(Obj, RE, fmt);
+      break;
+    }
+    // X86 and ARM share some relocation types in common.
+  } else if (Arch == Triple::x86 || Arch == Triple::arm ||
+             Arch == Triple::ppc) {
+    // Generic relocation types...
+    switch (Type) {
+    case MachO::GENERIC_RELOC_PAIR: // prints no info
+      return std::error_code();
+    case MachO::GENERIC_RELOC_SECTDIFF: {
+      DataRefImpl RelNext = Rel;
+      Obj->moveRelocationNext(RelNext);
+      MachO::any_relocation_info RENext = Obj->getRelocation(RelNext);
+
+      // X86 sect diff's must be followed by a relocation of type
+      // GENERIC_RELOC_PAIR.
+      unsigned RType = Obj->getAnyRelocationType(RENext);
+
+      if (RType != MachO::GENERIC_RELOC_PAIR)
+        report_fatal_error("Expected GENERIC_RELOC_PAIR after "
+                           "GENERIC_RELOC_SECTDIFF.");
+
+      printRelocationTargetName(Obj, RE, fmt);
+      fmt << "-";
+      printRelocationTargetName(Obj, RENext, fmt);
+      break;
+    }
+    }
+
+    if (Arch == Triple::x86 || Arch == Triple::ppc) {
+      switch (Type) {
+      case MachO::GENERIC_RELOC_LOCAL_SECTDIFF: {
+        DataRefImpl RelNext = Rel;
+        Obj->moveRelocationNext(RelNext);
+        MachO::any_relocation_info RENext = Obj->getRelocation(RelNext);
+
+        // X86 sect diff's must be followed by a relocation of type
+        // GENERIC_RELOC_PAIR.
+        unsigned RType = Obj->getAnyRelocationType(RENext);
+        if (RType != MachO::GENERIC_RELOC_PAIR)
+          report_fatal_error("Expected GENERIC_RELOC_PAIR after "
+                             "GENERIC_RELOC_LOCAL_SECTDIFF.");
+
+        printRelocationTargetName(Obj, RE, fmt);
+        fmt << "-";
+        printRelocationTargetName(Obj, RENext, fmt);
+        break;
+      }
+      case MachO::GENERIC_RELOC_TLV: {
+        printRelocationTargetName(Obj, RE, fmt);
+        fmt << "@TLV";
+        if (IsPCRel)
+          fmt << "P";
+        break;
+      }
+      default:
+        printRelocationTargetName(Obj, RE, fmt);
+      }
+    } else { // ARM-specific relocations
+      switch (Type) {
+      case MachO::ARM_RELOC_HALF:
+      case MachO::ARM_RELOC_HALF_SECTDIFF: {
+        // Half relocations steal a bit from the length field to encode
+        // whether this is an upper16 or a lower16 relocation.
+        bool isUpper = Obj->getAnyRelocationLength(RE) >> 1;
+
+        if (isUpper)
+          fmt << ":upper16:(";
+        else
+          fmt << ":lower16:(";
+        printRelocationTargetName(Obj, RE, fmt);
+
+        DataRefImpl RelNext = Rel;
+        Obj->moveRelocationNext(RelNext);
+        MachO::any_relocation_info RENext = Obj->getRelocation(RelNext);
+
+        // ARM half relocs must be followed by a relocation of type
+        // ARM_RELOC_PAIR.
+        unsigned RType = Obj->getAnyRelocationType(RENext);
+        if (RType != MachO::ARM_RELOC_PAIR)
+          report_fatal_error("Expected ARM_RELOC_PAIR after "
+                             "ARM_RELOC_HALF");
+
+        // NOTE: The half of the target virtual address is stashed in the
+        // address field of the secondary relocation, but we can't reverse
+        // engineer the constant offset from it without decoding the movw/movt
+        // instruction to find the other half in its immediate field.
+
+        // ARM_RELOC_HALF_SECTDIFF encodes the second section in the
+        // symbol/section pointer of the follow-on relocation.
+        if (Type == MachO::ARM_RELOC_HALF_SECTDIFF) {
+          fmt << "-";
+          printRelocationTargetName(Obj, RENext, fmt);
+        }
+
+        fmt << ")";
+        break;
+      }
+      default: { printRelocationTargetName(Obj, RE, fmt); }
+      }
+    }
+  } else
+    printRelocationTargetName(Obj, RE, fmt);
+
+  fmt.flush();
+  Result.append(fmtbuf.begin(), fmtbuf.end());
+  return std::error_code();
+}
+
+static std::error_code getRelocationValueString(const RelocationRef &Rel,
+                                                SmallVectorImpl<char> &Result) {
+  const ObjectFile *Obj = Rel.getObject();
+  if (auto *ELF = dyn_cast<ELFObjectFileBase>(Obj))
+    return getRelocationValueString(ELF, Rel, Result);
+  if (auto *COFF = dyn_cast<COFFObjectFile>(Obj))
+    return getRelocationValueString(COFF, Rel, Result);
+  auto *MachO = cast<MachOObjectFile>(Obj);
+  return getRelocationValueString(MachO, Rel, Result);
+}
+
+/// @brief Indicates whether this relocation should hidden when listing
+/// relocations, usually because it is the trailing part of a multipart
+/// relocation that will be printed as part of the leading relocation.
+static bool getHidden(RelocationRef RelRef) {
+  const ObjectFile *Obj = RelRef.getObject();
+  auto *MachO = dyn_cast<MachOObjectFile>(Obj);
+  if (!MachO)
+    return false;
+
+  unsigned Arch = MachO->getArch();
+  DataRefImpl Rel = RelRef.getRawDataRefImpl();
+  uint64_t Type = MachO->getRelocationType(Rel);
+
+  // On arches that use the generic relocations, GENERIC_RELOC_PAIR
+  // is always hidden.
+  if (Arch == Triple::x86 || Arch == Triple::arm || Arch == Triple::ppc) {
+    if (Type == MachO::GENERIC_RELOC_PAIR)
+      return true;
+  } else if (Arch == Triple::x86_64) {
+    // On x86_64, X86_64_RELOC_UNSIGNED is hidden only when it follows
+    // an X86_64_RELOC_SUBTRACTOR.
+    if (Type == MachO::X86_64_RELOC_UNSIGNED && Rel.d.a > 0) {
+      DataRefImpl RelPrev = Rel;
+      RelPrev.d.a--;
+      uint64_t PrevType = MachO->getRelocationType(RelPrev);
+      if (PrevType == MachO::X86_64_RELOC_SUBTRACTOR)
+        return true;
+    }
+  }
+
+  return false;
 }
 
 static void DisassembleObject(const ObjectFile *Obj, bool InlineRelocs) {
@@ -280,6 +875,8 @@ static void DisassembleObject(const ObjectFile *Obj, bool InlineRelocs) {
       << '\n';
     return;
   }
+  IP->setPrintImmHex(PrintImmHex);
+  PrettyPrinter &PIP = selectPrettyPrinter(Triple(TripleName));
 
   StringRef Fmt = Obj->getBytesInAddress() > 4 ? "\t\t%016" PRIx64 ":  " :
                                                  "\t\t\t%08" PRIx64 ":  ";
@@ -288,14 +885,38 @@ static void DisassembleObject(const ObjectFile *Obj, bool InlineRelocs) {
   // in RelocSecs contain the relocations for section S.
   std::error_code EC;
   std::map<SectionRef, SmallVector<SectionRef, 1>> SectionRelocMap;
-  for (const SectionRef &Section : Obj->sections()) {
+  for (const SectionRef &Section : ToolSectionFilter(*Obj)) {
     section_iterator Sec2 = Section.getRelocatedSection();
     if (Sec2 != Obj->section_end())
       SectionRelocMap[*Sec2].push_back(Section);
   }
 
-  for (const SectionRef &Section : Obj->sections()) {
-    if (!Section.isText() || Section.isVirtual())
+  // Create a mapping from virtual address to symbol name.  This is used to
+  // pretty print the target of a call.
+  std::vector<std::pair<uint64_t, StringRef>> AllSymbols;
+  if (MIA) {
+    for (const SymbolRef &Symbol : Obj->symbols()) {
+      if (Symbol.getType() != SymbolRef::ST_Function)
+        continue;
+
+      ErrorOr<uint64_t> AddressOrErr = Symbol.getAddress();
+      if (error(AddressOrErr.getError()))
+        break;
+      uint64_t Address = *AddressOrErr;
+
+      ErrorOr<StringRef> Name = Symbol.getName();
+      if (error(Name.getError()))
+        break;
+      if (Name->empty())
+        continue;
+      AllSymbols.push_back(std::make_pair(Address, *Name));
+    }
+
+    array_pod_sort(AllSymbols.begin(), AllSymbols.end());
+  }
+
+  for (const SectionRef &Section : ToolSectionFilter(*Obj)) {
+    if (!DisassembleAll && (!Section.isText() || Section.isVirtual()))
       continue;
 
     uint64_t SectionAddr = Section.getAddress();
@@ -307,19 +928,18 @@ static void DisassembleObject(const ObjectFile *Obj, bool InlineRelocs) {
     std::vector<std::pair<uint64_t, StringRef>> Symbols;
     for (const SymbolRef &Symbol : Obj->symbols()) {
       if (Section.containsSymbol(Symbol)) {
-        uint64_t Address;
-        if (error(Symbol.getAddress(Address)))
+        ErrorOr<uint64_t> AddressOrErr = Symbol.getAddress();
+        if (error(AddressOrErr.getError()))
           break;
-        if (Address == UnknownAddressOrSize)
-          continue;
+        uint64_t Address = *AddressOrErr;
         Address -= SectionAddr;
         if (Address >= SectSize)
           continue;
 
-        StringRef Name;
-        if (error(Symbol.getName(Name)))
+        ErrorOr<StringRef> Name = Symbol.getName();
+        if (error(Name.getError()))
           break;
-        Symbols.push_back(std::make_pair(Address, Name));
+        Symbols.push_back(std::make_pair(Address, *Name));
       }
     }
 
@@ -352,11 +972,9 @@ static void DisassembleObject(const ObjectFile *Obj, bool InlineRelocs) {
       outs() << SegmentName << ",";
     outs() << name << ':';
 
-    // If the section has no symbols just insert a dummy one and disassemble
-    // the whole section.
-    if (Symbols.empty())
-      Symbols.push_back(std::make_pair(0, name));
-
+    // If the section has no symbol at the start, just insert a dummy one.
+    if (Symbols.empty() || Symbols[0].first != 0)
+      Symbols.insert(Symbols.begin(), std::make_pair(0, name));
 
     SmallString<40> Comments;
     raw_svector_ostream CommentStream(Comments);
@@ -396,14 +1014,34 @@ static void DisassembleObject(const ObjectFile *Obj, bool InlineRelocs) {
         if (DisAsm->getInstruction(Inst, Size, Bytes.slice(Index),
                                    SectionAddr + Index, DebugOut,
                                    CommentStream)) {
-          outs() << format("%8" PRIx64 ":", SectionAddr + Index);
-          if (!NoShowRawInsn) {
-            outs() << "\t";
-            DumpBytes(ArrayRef<uint8_t>(Bytes.data() + Index, Size));
-          }
-          IP->printInst(&Inst, outs(), "", *STI);
+          PIP.printInst(*IP, &Inst,
+                        Bytes.slice(Index, Size),
+                        SectionAddr + Index, outs(), "", *STI);
           outs() << CommentStream.str();
           Comments.clear();
+          if (MIA && (MIA->isCall(Inst) || MIA->isUnconditionalBranch(Inst) ||
+                      MIA->isConditionalBranch(Inst))) {
+            uint64_t Target;
+            if (MIA->evaluateBranch(Inst, SectionAddr + Index, Size, Target)) {
+              auto TargetSym = std::upper_bound(
+                  AllSymbols.begin(), AllSymbols.end(), Target,
+                  [](uint64_t LHS, const std::pair<uint64_t, StringRef> &RHS) {
+                    return LHS < RHS.first;
+                  });
+              if (TargetSym != AllSymbols.begin())
+                --TargetSym;
+              else
+                TargetSym = AllSymbols.end();
+
+              if (TargetSym != AllSymbols.end()) {
+                outs() << " <" << TargetSym->second;
+                uint64_t Disp = Target - TargetSym->first;
+                if (Disp)
+                  outs() << '+' << utohexstr(Disp);
+                outs() << '>';
+              }
+            }
+          }
           outs() << "\n";
         } else {
           errs() << ToolName << ": warning: invalid instruction encoding\n";
@@ -413,21 +1051,19 @@ static void DisassembleObject(const ObjectFile *Obj, bool InlineRelocs) {
 
         // Print relocation for instruction.
         while (rel_cur != rel_end) {
-          bool hidden = false;
-          uint64_t addr;
+          bool hidden = getHidden(*rel_cur);
+          uint64_t addr = rel_cur->getOffset();
           SmallString<16> name;
           SmallString<32> val;
 
           // If this relocation is hidden, skip it.
-          if (error(rel_cur->getHidden(hidden))) goto skip_print_rel;
           if (hidden) goto skip_print_rel;
 
-          if (error(rel_cur->getOffset(addr))) goto skip_print_rel;
           // Stop when rel_cur's address is past the current instruction.
           if (addr >= Index + Size) break;
-          if (error(rel_cur->getTypeName(name))) goto skip_print_rel;
-          if (error(rel_cur->getValueString(val))) goto skip_print_rel;
-
+          rel_cur->getTypeName(name);
+          if (error(getRelocationValueString(*rel_cur, val)))
+            goto skip_print_rel;
           outs() << format(Fmt.data(), SectionAddr + addr) << name
                  << "\t" << val << "\n";
 
@@ -447,7 +1083,7 @@ void llvm::PrintRelocations(const ObjectFile *Obj) {
   if (!Obj->isRelocatableObject())
     return;
 
-  for (const SectionRef &Section : Obj->sections()) {
+  for (const SectionRef &Section : ToolSectionFilter(*Obj)) {
     if (Section.relocation_begin() == Section.relocation_end())
       continue;
     StringRef secname;
@@ -455,19 +1091,14 @@ void llvm::PrintRelocations(const ObjectFile *Obj) {
       continue;
     outs() << "RELOCATION RECORDS FOR [" << secname << "]:\n";
     for (const RelocationRef &Reloc : Section.relocations()) {
-      bool hidden;
-      uint64_t address;
+      bool hidden = getHidden(Reloc);
+      uint64_t address = Reloc.getOffset();
       SmallString<32> relocname;
       SmallString<32> valuestr;
-      if (error(Reloc.getHidden(hidden)))
-        continue;
       if (hidden)
         continue;
-      if (error(Reloc.getTypeName(relocname)))
-        continue;
-      if (error(Reloc.getOffset(address)))
-        continue;
-      if (error(Reloc.getValueString(valuestr)))
+      Reloc.getTypeName(relocname);
+      if (error(getRelocationValueString(Reloc, valuestr)))
         continue;
       outs() << format(Fmt.data(), address) << " " << relocname << " "
              << valuestr << "\n";
@@ -480,7 +1111,7 @@ void llvm::PrintSectionHeaders(const ObjectFile *Obj) {
   outs() << "Sections:\n"
             "Idx Name          Size      Address          Type\n";
   unsigned i = 0;
-  for (const SectionRef &Section : Obj->sections()) {
+  for (const SectionRef &Section : ToolSectionFilter(*Obj)) {
     StringRef Name;
     if (error(Section.getName(Name)))
       return;
@@ -499,7 +1130,7 @@ void llvm::PrintSectionHeaders(const ObjectFile *Obj) {
 
 void llvm::PrintSectionContents(const ObjectFile *Obj) {
   std::error_code EC;
-  for (const SectionRef &Section : Obj->sections()) {
+  for (const SectionRef &Section : ToolSectionFilter(*Obj)) {
     StringRef Name;
     StringRef Contents;
     if (error(Section.getName(Name)))
@@ -608,39 +1239,31 @@ void llvm::PrintSymbolTable(const ObjectFile *o) {
     return;
   }
   for (const SymbolRef &Symbol : o->symbols()) {
-    StringRef Name;
-    uint64_t Address;
-    SymbolRef::Type Type;
-    uint64_t Size;
+    ErrorOr<uint64_t> AddressOrError = Symbol.getAddress();
+    if (error(AddressOrError.getError()))
+      continue;
+    uint64_t Address = *AddressOrError;
+    SymbolRef::Type Type = Symbol.getType();
     uint32_t Flags = Symbol.getFlags();
     section_iterator Section = o->section_end();
-    if (error(Symbol.getName(Name)))
-      continue;
-    if (error(Symbol.getAddress(Address)))
-      continue;
-    if (error(Symbol.getType(Type)))
-      continue;
-    if (error(Symbol.getSize(Size)))
-      continue;
     if (error(Symbol.getSection(Section)))
       continue;
+    StringRef Name;
+    if (Type == SymbolRef::ST_Debug && Section != o->section_end()) {
+      Section->getName(Name);
+    } else {
+      ErrorOr<StringRef> NameOrErr = Symbol.getName();
+      if (error(NameOrErr.getError()))
+        continue;
+      Name = *NameOrErr;
+    }
 
     bool Global = Flags & SymbolRef::SF_Global;
     bool Weak = Flags & SymbolRef::SF_Weak;
     bool Absolute = Flags & SymbolRef::SF_Absolute;
     bool Common = Flags & SymbolRef::SF_Common;
+    bool Hidden = Flags & SymbolRef::SF_Hidden;
 
-    if (Common) {
-      uint32_t Alignment;
-      if (error(Symbol.getAlignment(Alignment)))
-        Alignment = 0;
-      Address = Size;
-      Size = Alignment;
-    }
-    if (Address == UnknownAddressOrSize)
-      Address = 0;
-    if (Size == UnknownAddressOrSize)
-      Size = 0;
     char GlobLoc = ' ';
     if (Type != SymbolRef::ST_Unknown)
       GlobLoc = Global ? 'g' : 'l';
@@ -682,9 +1305,18 @@ void llvm::PrintSymbolTable(const ObjectFile *o) {
         SectionName = "";
       outs() << SectionName;
     }
-    outs() << '\t'
-           << format("%08" PRIx64 " ", Size)
-           << Name
+
+    outs() << '\t';
+    if (Common || isa<ELFObjectFileBase>(o)) {
+      uint64_t Val =
+          Common ? Symbol.getAlignment() : ELFSymbolRef(Symbol).getSize();
+      outs() << format("\t %08" PRIx64 " ", Val);
+    }
+
+    if (Hidden) {
+      outs() << ".hidden ";
+    }
+    outs() << Name
            << '\n';
   }
 }
@@ -759,6 +1391,86 @@ void llvm::printWeakBindTable(const ObjectFile *o) {
   }
 }
 
+/// Dump the raw contents of the __clangast section so the output can be piped
+/// into llvm-bcanalyzer.
+void llvm::printRawClangAST(const ObjectFile *Obj) {
+  if (outs().is_displayed()) {
+    errs() << "The -raw-clang-ast option will dump the raw binary contents of "
+              "the clang ast section.\n"
+              "Please redirect the output to a file or another program such as "
+              "llvm-bcanalyzer.\n";
+    return;
+  }
+
+  StringRef ClangASTSectionName("__clangast");
+  if (isa<COFFObjectFile>(Obj)) {
+    ClangASTSectionName = "clangast";
+  }
+
+  Optional<object::SectionRef> ClangASTSection;
+  for (auto Sec : ToolSectionFilter(*Obj)) {
+    StringRef Name;
+    Sec.getName(Name);
+    if (Name == ClangASTSectionName) {
+      ClangASTSection = Sec;
+      break;
+    }
+  }
+  if (!ClangASTSection)
+    return;
+
+  StringRef ClangASTContents;
+  if (error(ClangASTSection.getValue().getContents(ClangASTContents))) {
+    errs() << "Could not read the " << ClangASTSectionName << " section!\n";
+    return;
+  }
+
+  outs().write(ClangASTContents.data(), ClangASTContents.size());
+}
+
+static void printFaultMaps(const ObjectFile *Obj) {
+  const char *FaultMapSectionName = nullptr;
+
+  if (isa<ELFObjectFileBase>(Obj)) {
+    FaultMapSectionName = ".llvm_faultmaps";
+  } else if (isa<MachOObjectFile>(Obj)) {
+    FaultMapSectionName = "__llvm_faultmaps";
+  } else {
+    errs() << "This operation is only currently supported "
+              "for ELF and Mach-O executable files.\n";
+    return;
+  }
+
+  Optional<object::SectionRef> FaultMapSection;
+
+  for (auto Sec : ToolSectionFilter(*Obj)) {
+    StringRef Name;
+    Sec.getName(Name);
+    if (Name == FaultMapSectionName) {
+      FaultMapSection = Sec;
+      break;
+    }
+  }
+
+  outs() << "FaultMap table:\n";
+
+  if (!FaultMapSection.hasValue()) {
+    outs() << "<not found>\n";
+    return;
+  }
+
+  StringRef FaultMapContents;
+  if (error(FaultMapSection.getValue().getContents(FaultMapContents))) {
+    errs() << "Could not read the " << FaultMapContents << " section!\n";
+    return;
+  }
+
+  FaultMapParser FMP(FaultMapContents.bytes_begin(),
+                     FaultMapContents.bytes_end());
+
+  outs() << FMP;
+}
+
 static void printPrivateFileHeader(const ObjectFile *o) {
   if (o->isELF()) {
     printELFFileHeader(o);
@@ -770,9 +1482,12 @@ static void printPrivateFileHeader(const ObjectFile *o) {
 }
 
 static void DumpObject(const ObjectFile *o) {
-  outs() << '\n';
-  outs() << o->getFileName()
-         << ":\tfile format " << o->getFileFormatName() << "\n\n";
+  // Avoid other output when using a raw option.
+  if (!RawClangAST) {
+    outs() << '\n';
+    outs() << o->getFileName()
+           << ":\tfile format " << o->getFileFormatName() << "\n\n";
+  }
 
   if (Disassemble)
     DisassembleObject(o, Relocations);
@@ -798,6 +1513,10 @@ static void DumpObject(const ObjectFile *o) {
     printLazyBindTable(o);
   if (WeakBind)
     printWeakBindTable(o);
+  if (RawClangAST)
+    printRawClangAST(o);
+  if (PrintFaultMaps)
+    printFaultMaps(o);
 }
 
 /// @brief Dump each object file in \a a;
@@ -808,15 +1527,13 @@ static void DumpArchive(const Archive *a) {
     if (std::error_code EC = ChildOrErr.getError()) {
       // Ignore non-object files.
       if (EC != object_error::invalid_file_type)
-        errs() << ToolName << ": '" << a->getFileName() << "': " << EC.message()
-               << ".\n";
+        report_error(a->getFileName(), EC);
       continue;
     }
     if (ObjectFile *o = dyn_cast<ObjectFile>(&*ChildOrErr.get()))
       DumpObject(o);
     else
-      errs() << ToolName << ": '" << a->getFileName() << "': "
-              << "Unrecognized file type.\n";
+      report_error(a->getFileName(), object_error::invalid_file_type);
   }
 }
 
@@ -824,7 +1541,7 @@ static void DumpArchive(const Archive *a) {
 static void DumpInput(StringRef file) {
   // If file isn't stdin, check that it exists.
   if (file != "-" && !sys::fs::exists(file)) {
-    errs() << ToolName << ": '" << file << "': " << "No such file\n";
+    report_error(file, errc::no_such_file_or_directory);
     return;
   }
 
@@ -839,7 +1556,7 @@ static void DumpInput(StringRef file) {
   // Attempt to open the binary.
   ErrorOr<OwningBinary<Binary>> BinaryOrErr = createBinary(file);
   if (std::error_code EC = BinaryOrErr.getError()) {
-    errs() << ToolName << ": '" << file << "': " << EC.message() << ".\n";
+    report_error(file, EC);
     return;
   }
   Binary &Binary = *BinaryOrErr.get().getBinary();
@@ -849,7 +1566,7 @@ static void DumpInput(StringRef file) {
   else if (ObjectFile *o = dyn_cast<ObjectFile>(&Binary))
     DumpObject(o);
   else
-    errs() << ToolName << ": '" << file << "': " << "Unrecognized file type.\n";
+    report_error(file, object_error::invalid_file_type);
 }
 
 int main(int argc, char **argv) {
@@ -876,6 +1593,8 @@ int main(int argc, char **argv) {
   if (InputFilenames.size() == 0)
     InputFilenames.push_back("a.out");
 
+  if (DisassembleAll)
+    Disassemble = true;
   if (!Disassemble
       && !Relocations
       && !SectionHeaders
@@ -888,6 +1607,7 @@ int main(int argc, char **argv) {
       && !Bind
       && !LazyBind
       && !WeakBind
+      && !RawClangAST
       && !(UniversalHeaders && MachOOpt)
       && !(ArchiveHeaders && MachOOpt)
       && !(IndirectSymbols && MachOOpt)
@@ -897,7 +1617,8 @@ int main(int argc, char **argv) {
       && !(DylibsUsed && MachOOpt)
       && !(DylibId && MachOOpt)
       && !(ObjcMetaData && MachOOpt)
-      && !(DumpSections.size() != 0 && MachOOpt)) {
+      && !(FilterSections.size() != 0 && MachOOpt)
+      && !PrintFaultMaps) {
     cl::PrintHelpMessage();
     return 2;
   }
diff --git a/tools/llvm-objdump/llvm-objdump.h b/tools/llvm-objdump/llvm-objdump.h
index bde72e004a1..b19c515d3c5 100644
--- a/tools/llvm-objdump/llvm-objdump.h
+++ b/tools/llvm-objdump/llvm-objdump.h
@@ -25,9 +25,9 @@ extern cl::opt<std::string> TripleName;
 extern cl::opt<std::string> ArchName;
 extern cl::opt<std::string> MCPU;
 extern cl::list<std::string> MAttrs;
-extern cl::list<std::string> DumpSections;
-extern cl::opt<bool> Raw;
+extern cl::list<std::string> FilterSections;
 extern cl::opt<bool> Disassemble;
+extern cl::opt<bool> DisassembleAll;
 extern cl::opt<bool> NoShowRawInsn;
 extern cl::opt<bool> PrivateHeaders;
 extern cl::opt<bool> ExportsTrie;
@@ -35,6 +35,7 @@ extern cl::opt<bool> Rebase;
 extern cl::opt<bool> Bind;
 extern cl::opt<bool> LazyBind;
 extern cl::opt<bool> WeakBind;
+extern cl::opt<bool> RawClangAST;
 extern cl::opt<bool> UniversalHeaders;
 extern cl::opt<bool> ArchiveHeaders;
 extern cl::opt<bool> IndirectSymbols;
@@ -51,11 +52,11 @@ extern cl::opt<bool> SectionHeaders;
 extern cl::opt<bool> SectionContents;
 extern cl::opt<bool> SymbolTable;
 extern cl::opt<bool> UnwindInfo;
+extern cl::opt<bool> PrintImmHex;
 
 // Various helper functions.
 bool error(std::error_code ec);
 bool RelocAddressLess(object::RelocationRef a, object::RelocationRef b);
-void DumpBytes(ArrayRef<uint8_t> bytes);
 void ParseInputMachO(StringRef Filename);
 void printCOFFUnwindInfo(const object::COFFObjectFile* o);
 void printMachOUnwindInfo(const object::MachOObjectFile* o);
@@ -72,6 +73,7 @@ void printRebaseTable(const object::ObjectFile *o);
 void printBindTable(const object::ObjectFile *o);
 void printLazyBindTable(const object::ObjectFile *o);
 void printWeakBindTable(const object::ObjectFile *o);
+void printRawClangAST(const object::ObjectFile *o);
 void PrintRelocations(const object::ObjectFile *o);
 void PrintSectionHeaders(const object::ObjectFile *o);
 void PrintSectionContents(const object::ObjectFile *o);
diff --git a/tools/llvm-pdbdump/CMakeLists.txt b/tools/llvm-pdbdump/CMakeLists.txt
index 4dd339cee57..1907f917079 100644
--- a/tools/llvm-pdbdump/CMakeLists.txt
+++ b/tools/llvm-pdbdump/CMakeLists.txt
@@ -9,6 +9,7 @@ add_llvm_tool(llvm-pdbdump
   ClassDefinitionDumper.cpp
   CompilandDumper.cpp
   EnumDumper.cpp
+  ExternalSymbolDumper.cpp
   FunctionDumper.cpp
   LinePrinter.cpp
   TypeDumper.cpp
diff --git a/tools/llvm-pdbdump/CompilandDumper.cpp b/tools/llvm-pdbdump/CompilandDumper.cpp
index 86bf32d77d8..68ceb620627 100644
--- a/tools/llvm-pdbdump/CompilandDumper.cpp
+++ b/tools/llvm-pdbdump/CompilandDumper.cpp
@@ -69,7 +69,7 @@ void CompilandDumper::dump(const PDBSymbolData &Symbol) {
   case PDB_LocType::Static:
     Printer << "data: ";
     WithColor(Printer, PDB_ColorItem::Address).get()
-        << "[" << format_hex(Symbol.getRelativeVirtualAddress(), 10) << "]";
+        << "[" << format_hex(Symbol.getVirtualAddress(), 10) << "]";
     break;
   case PDB_LocType::Constant:
     Printer << "constant: ";
@@ -102,7 +102,7 @@ void CompilandDumper::dump(const PDBSymbolLabel &Symbol) {
   Printer.NewLine();
   Printer << "label ";
   WithColor(Printer, PDB_ColorItem::Address).get()
-      << "[" << format_hex(Symbol.getRelativeVirtualAddress(), 10) << "] ";
+      << "[" << format_hex(Symbol.getVirtualAddress(), 10) << "] ";
   WithColor(Printer, PDB_ColorItem::Identifier).get() << Symbol.getName();
 }
 
@@ -113,16 +113,16 @@ void CompilandDumper::dump(const PDBSymbolThunk &Symbol) {
   Printer.NewLine();
   Printer << "thunk ";
   PDB_ThunkOrdinal Ordinal = Symbol.getThunkOrdinal();
-  uint32_t RVA = Symbol.getRelativeVirtualAddress();
+  uint64_t VA = Symbol.getVirtualAddress();
   if (Ordinal == PDB_ThunkOrdinal::TrampIncremental) {
-    uint32_t Target = Symbol.getTargetRelativeVirtualAddress();
-    WithColor(Printer, PDB_ColorItem::Address).get() << format_hex(RVA, 10);
+    uint64_t Target = Symbol.getTargetVirtualAddress();
+    WithColor(Printer, PDB_ColorItem::Address).get() << format_hex(VA, 10);
     Printer << " -> ";
     WithColor(Printer, PDB_ColorItem::Address).get() << format_hex(Target, 10);
   } else {
     WithColor(Printer, PDB_ColorItem::Address).get()
-        << "[" << format_hex(RVA, 10) << " - "
-        << format_hex(RVA + Symbol.getLength(), 10) << "]";
+        << "[" << format_hex(VA, 10) << " - "
+        << format_hex(VA + Symbol.getLength(), 10) << "]";
   }
   Printer << " (";
   WithColor(Printer, PDB_ColorItem::Register).get() << Ordinal;
diff --git a/tools/llvm-pdbdump/ExternalSymbolDumper.cpp b/tools/llvm-pdbdump/ExternalSymbolDumper.cpp
new file mode 100644
index 00000000000..c4e9f474880
--- /dev/null
+++ b/tools/llvm-pdbdump/ExternalSymbolDumper.cpp
@@ -0,0 +1,40 @@
+//===- ExternalSymbolDumper.cpp -------------------------------- *- C++ *-===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+
+#include "ExternalSymbolDumper.h"
+#include "LinePrinter.h"
+
+#include "llvm/DebugInfo/PDB/PDBSymbolExe.h"
+#include "llvm/DebugInfo/PDB/PDBSymbolPublicSymbol.h"
+#include "llvm/Support/Format.h"
+
+using namespace llvm;
+
+ExternalSymbolDumper::ExternalSymbolDumper(LinePrinter &P)
+    : PDBSymDumper(true), Printer(P) {}
+
+void ExternalSymbolDumper::start(const PDBSymbolExe &Symbol) {
+  auto Vars = Symbol.findAllChildren<PDBSymbolPublicSymbol>();
+  while (auto Var = Vars->getNext())
+    Var->dump(*this);
+}
+
+void ExternalSymbolDumper::dump(const PDBSymbolPublicSymbol &Symbol) {
+  std::string LinkageName = Symbol.getName();
+  if (Printer.IsSymbolExcluded(LinkageName))
+    return;
+
+  Printer.NewLine();
+  uint64_t Addr = Symbol.getVirtualAddress();
+
+  Printer << "[";
+  WithColor(Printer, PDB_ColorItem::Address).get() << format_hex(Addr, 10);
+  Printer << "] ";
+  WithColor(Printer, PDB_ColorItem::Identifier).get() << LinkageName;
+}
diff --git a/tools/llvm-pdbdump/ExternalSymbolDumper.h b/tools/llvm-pdbdump/ExternalSymbolDumper.h
new file mode 100644
index 00000000000..d77b09cdebf
--- /dev/null
+++ b/tools/llvm-pdbdump/ExternalSymbolDumper.h
@@ -0,0 +1,32 @@
+//===- ExternalSymbolDumper.h --------------------------------- *- C++ --*-===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_TOOLS_LLVMPDBDUMP_EXTERNALSYMBOLDUMPER_H
+#define LLVM_TOOLS_LLVMPDBDUMP_EXTERNALSYMBOLDUMPER_H
+
+#include "llvm/DebugInfo/PDB/PDBSymDumper.h"
+
+namespace llvm {
+
+class LinePrinter;
+
+class ExternalSymbolDumper : public PDBSymDumper {
+public:
+  ExternalSymbolDumper(LinePrinter &P);
+
+  void start(const PDBSymbolExe &Symbol);
+
+  void dump(const PDBSymbolPublicSymbol &Symbol) override;
+
+private:
+  LinePrinter &Printer;
+};
+}
+
+#endif
diff --git a/tools/llvm-pdbdump/FunctionDumper.cpp b/tools/llvm-pdbdump/FunctionDumper.cpp
index 419f888ef2f..9584812e81a 100644
--- a/tools/llvm-pdbdump/FunctionDumper.cpp
+++ b/tools/llvm-pdbdump/FunctionDumper.cpp
@@ -109,19 +109,19 @@ void FunctionDumper::start(const PDBSymbolTypeFunctionSig &Symbol,
 }
 
 void FunctionDumper::start(const PDBSymbolFunc &Symbol, PointerType Pointer) {
-  uint32_t FuncStart = Symbol.getRelativeVirtualAddress();
-  uint32_t FuncEnd = FuncStart + Symbol.getLength();
+  uint64_t FuncStart = Symbol.getVirtualAddress();
+  uint64_t FuncEnd = FuncStart + Symbol.getLength();
 
   Printer << "func [";
   WithColor(Printer, PDB_ColorItem::Address).get() << format_hex(FuncStart, 10);
   if (auto DebugStart = Symbol.findOneChild<PDBSymbolFuncDebugStart>()) {
-    uint32_t Prologue = DebugStart->getRelativeVirtualAddress() - FuncStart;
+    uint64_t Prologue = DebugStart->getVirtualAddress() - FuncStart;
     WithColor(Printer, PDB_ColorItem::Offset).get() << "+" << Prologue;
   }
   Printer << " - ";
   WithColor(Printer, PDB_ColorItem::Address).get() << format_hex(FuncEnd, 10);
   if (auto DebugEnd = Symbol.findOneChild<PDBSymbolFuncDebugEnd>()) {
-    uint32_t Epilogue = FuncEnd - DebugEnd->getRelativeVirtualAddress();
+    uint64_t Epilogue = FuncEnd - DebugEnd->getVirtualAddress();
     WithColor(Printer, PDB_ColorItem::Offset).get() << "-" << Epilogue;
   }
   Printer << "] (";
diff --git a/tools/llvm-pdbdump/LinePrinter.h b/tools/llvm-pdbdump/LinePrinter.h
index c2a3ab60d90..b985e934190 100644
--- a/tools/llvm-pdbdump/LinePrinter.h
+++ b/tools/llvm-pdbdump/LinePrinter.h
@@ -41,7 +41,7 @@ class LinePrinter {
   void SetFilters(std::list<Regex> &List, Iter Begin, Iter End) {
     List.clear();
     for (; Begin != End; ++Begin)
-      List.push_back(StringRef(*Begin));
+      List.emplace_back(StringRef(*Begin));
   }
 
   raw_ostream &OS;
diff --git a/tools/llvm-pdbdump/VariableDumper.cpp b/tools/llvm-pdbdump/VariableDumper.cpp
index 030610c8c58..e5665b5fcaf 100644
--- a/tools/llvm-pdbdump/VariableDumper.cpp
+++ b/tools/llvm-pdbdump/VariableDumper.cpp
@@ -44,7 +44,7 @@ void VariableDumper::start(const PDBSymbolData &Var) {
     Printer.NewLine();
     Printer << "data [";
     WithColor(Printer, PDB_ColorItem::Address).get()
-        << format_hex(Var.getRelativeVirtualAddress(), 10);
+        << format_hex(Var.getVirtualAddress(), 10);
     Printer << "] ";
     WithColor(Printer, PDB_ColorItem::Keyword).get() << "static ";
     dumpSymbolTypeAndName(*VarType, Var.getName());
diff --git a/tools/llvm-pdbdump/llvm-pdbdump.cpp b/tools/llvm-pdbdump/llvm-pdbdump.cpp
index 78535ec0be9..4a4c64b80cc 100644
--- a/tools/llvm-pdbdump/llvm-pdbdump.cpp
+++ b/tools/llvm-pdbdump/llvm-pdbdump.cpp
@@ -15,6 +15,7 @@
 
 #include "llvm-pdbdump.h"
 #include "CompilandDumper.h"
+#include "ExternalSymbolDumper.h"
 #include "FunctionDumper.h"
 #include "LinePrinter.h"
 #include "TypeDumper.h"
@@ -58,6 +59,7 @@ cl::list<std::string> InputFilenames(cl::Positional,
 
 cl::OptionCategory TypeCategory("Symbol Type Options");
 cl::OptionCategory FilterCategory("Filtering Options");
+cl::OptionCategory OtherOptions("Other Options");
 
 cl::opt<bool> Compilands("compilands", cl::desc("Display compilands"),
                          cl::cat(TypeCategory));
@@ -65,11 +67,18 @@ cl::opt<bool> Symbols("symbols", cl::desc("Display symbols for each compiland"),
                       cl::cat(TypeCategory));
 cl::opt<bool> Globals("globals", cl::desc("Dump global symbols"),
                       cl::cat(TypeCategory));
+cl::opt<bool> Externals("externals", cl::desc("Dump external symbols"),
+                        cl::cat(TypeCategory));
 cl::opt<bool> Types("types", cl::desc("Display types"), cl::cat(TypeCategory));
 cl::opt<bool>
     All("all", cl::desc("Implies all other options in 'Symbol Types' category"),
         cl::cat(TypeCategory));
 
+cl::opt<uint64_t> LoadAddress(
+    "load-address",
+    cl::desc("Assume the module is loaded at the specified address"),
+    cl::cat(OtherOptions));
+
 cl::list<std::string>
     ExcludeTypes("exclude-types",
                  cl::desc("Exclude types by regular expression"),
@@ -101,7 +110,7 @@ cl::opt<bool> NoEnumDefs("no-enum-definitions",
 static void dumpInput(StringRef Path) {
   std::unique_ptr<IPDBSession> Session;
   PDB_ErrorCode Error =
-      llvm::createPDBReader(PDB_ReaderType::DIA, Path, Session);
+      llvm::loadDataForPDB(PDB_ReaderType::DIA, Path, Session);
   switch (Error) {
   case PDB_ErrorCode::Success:
     break;
@@ -121,6 +130,8 @@ static void dumpInput(StringRef Path) {
            << "'.  An unknown error occured.\n";
     return;
   }
+  if (opts::LoadAddress)
+    Session->setLoadAddress(opts::LoadAddress);
 
   LinePrinter Printer(2, outs());
 
@@ -215,6 +226,13 @@ static void dumpInput(StringRef Path) {
     }
     Printer.Unindent();
   }
+  if (opts::Externals) {
+    Printer.NewLine();
+    WithColor(Printer, PDB_ColorItem::SectionHeader).get() << "---EXTERNALS---";
+    Printer.Indent();
+    ExternalSymbolDumper Dumper(Printer);
+    Dumper.start(*GlobalScope);
+  }
   outs().flush();
 }
 
@@ -240,6 +258,7 @@ int main(int argc_, const char *argv_[]) {
     opts::Symbols = true;
     opts::Globals = true;
     opts::Types = true;
+    opts::Externals = true;
   }
   if (opts::ExcludeCompilerGenerated) {
     opts::ExcludeTypes.push_back("__vc_attributes");
diff --git a/tools/llvm-profdata/llvm-profdata.cpp b/tools/llvm-profdata/llvm-profdata.cpp
index 1bfdb181d7a..6fb48d8fad5 100644
--- a/tools/llvm-profdata/llvm-profdata.cpp
+++ b/tools/llvm-profdata/llvm-profdata.cpp
@@ -37,7 +37,9 @@ static void exitWithError(const Twine &Message, StringRef Whence = "") {
   ::exit(1);
 }
 
+namespace {
 enum ProfileKinds { instr, sample };
+}
 
 static void mergeInstrProfile(const cl::list<std::string> &Inputs,
                               StringRef OutputFilename) {
diff --git a/tools/llvm-readobj/ARMEHABIPrinter.h b/tools/llvm-readobj/ARMEHABIPrinter.h
index b15421d7571..4b8744e2015 100644
--- a/tools/llvm-readobj/ARMEHABIPrinter.h
+++ b/tools/llvm-readobj/ARMEHABIPrinter.h
@@ -310,10 +310,7 @@ class PrinterContext {
 
   typedef typename object::ELFFile<ET>::Elf_Sym Elf_Sym;
   typedef typename object::ELFFile<ET>::Elf_Shdr Elf_Shdr;
-
-  typedef typename object::ELFFile<ET>::Elf_Rel_Iter Elf_Rel_iterator;
-  typedef typename object::ELFFile<ET>::Elf_Sym_Iter Elf_Sym_iterator;
-  typedef typename object::ELFFile<ET>::Elf_Shdr_Iter Elf_Shdr_iterator;
+  typedef typename object::ELFFile<ET>::Elf_Rel Elf_Rel;
 
   static const size_t IndexTableEntrySize;
 
@@ -344,13 +341,18 @@ template <typename ET>
 const size_t PrinterContext<ET>::IndexTableEntrySize = 8;
 
 template <typename ET>
-ErrorOr<StringRef> PrinterContext<ET>::FunctionAtAddress(unsigned Section,
-                                                         uint64_t Address) const {
-  for (Elf_Sym_iterator SI = ELF->begin_symbols(), SE = ELF->end_symbols();
-       SI != SE; ++SI)
-    if (SI->st_shndx == Section && SI->st_value == Address &&
-        SI->getType() == ELF::STT_FUNC)
-      return ELF->getSymbolName(SI);
+ErrorOr<StringRef>
+PrinterContext<ET>::FunctionAtAddress(unsigned Section,
+                                      uint64_t Address) const {
+  const Elf_Shdr *Symtab = ELF->getDotSymtabSec();
+  ErrorOr<StringRef> StrTableOrErr = ELF->getStringTableForSymtab(*Symtab);
+  error(StrTableOrErr.getError());
+  StringRef StrTable = *StrTableOrErr;
+
+  for (const Elf_Sym &Sym : ELF->symbols())
+    if (Sym.st_shndx == Section && Sym.st_value == Address &&
+        Sym.getType() == ELF::STT_FUNC)
+      return Sym.getName(StrTable);
   return readobj_error::unknown_symbol;
 }
 
@@ -366,21 +368,22 @@ PrinterContext<ET>::FindExceptionTable(unsigned IndexSectionIndex,
   /// handling table.  Use this symbol to recover the actual exception handling
   /// table.
 
-  for (Elf_Shdr_iterator SI = ELF->begin_sections(), SE = ELF->end_sections();
-       SI != SE; ++SI) {
-    if (SI->sh_type == ELF::SHT_REL && SI->sh_info == IndexSectionIndex) {
-      for (Elf_Rel_iterator RI = ELF->begin_rel(&*SI), RE = ELF->end_rel(&*SI);
-           RI != RE; ++RI) {
-        if (RI->r_offset == static_cast<unsigned>(IndexTableOffset)) {
+  for (const Elf_Shdr &Sec : ELF->sections()) {
+    if (Sec.sh_type == ELF::SHT_REL && Sec.sh_info == IndexSectionIndex) {
+      for (const Elf_Rel &R : ELF->rels(&Sec)) {
+        if (R.r_offset == static_cast<unsigned>(IndexTableOffset)) {
           typename object::ELFFile<ET>::Elf_Rela RelA;
-          RelA.r_offset = RI->r_offset;
-          RelA.r_info = RI->r_info;
+          RelA.r_offset = R.r_offset;
+          RelA.r_info = R.r_info;
           RelA.r_addend = 0;
 
           std::pair<const Elf_Shdr *, const Elf_Sym *> Symbol =
-            ELF->getRelocationSymbol(&(*SI), &RelA);
+              ELF->getRelocationSymbol(&Sec, &RelA);
 
-          return ELF->getSection(Symbol.second);
+          ErrorOr<const Elf_Shdr *> Ret = ELF->getSection(Symbol.second);
+          if (std::error_code EC = Ret.getError())
+            report_fatal_error(EC.message());
+          return *Ret;
         }
       }
     }
@@ -528,20 +531,18 @@ void PrinterContext<ET>::PrintUnwindInformation() const {
   DictScope UI(SW, "UnwindInformation");
 
   int SectionIndex = 0;
-  for (Elf_Shdr_iterator SI = ELF->begin_sections(), SE = ELF->end_sections();
-       SI != SE; ++SI, ++SectionIndex) {
-    if (SI->sh_type == ELF::SHT_ARM_EXIDX) {
-      const Elf_Shdr *IT = &(*SI);
-
+  for (const Elf_Shdr &Sec : ELF->sections()) {
+    if (Sec.sh_type == ELF::SHT_ARM_EXIDX) {
       DictScope UIT(SW, "UnwindIndexTable");
 
       SW.printNumber("SectionIndex", SectionIndex);
-      if (ErrorOr<StringRef> SectionName = ELF->getSectionName(IT))
+      if (ErrorOr<StringRef> SectionName = ELF->getSectionName(&Sec))
         SW.printString("SectionName", *SectionName);
-      SW.printHex("SectionOffset", IT->sh_offset);
+      SW.printHex("SectionOffset", Sec.sh_offset);
 
-      PrintIndexTable(SectionIndex, IT);
+      PrintIndexTable(SectionIndex, &Sec);
     }
+    ++SectionIndex;
   }
 }
 }
diff --git a/tools/llvm-readobj/ARMWinEHPrinter.cpp b/tools/llvm-readobj/ARMWinEHPrinter.cpp
index 62252fcda5f..bf5ff8e1d03 100644
--- a/tools/llvm-readobj/ARMWinEHPrinter.cpp
+++ b/tools/llvm-readobj/ARMWinEHPrinter.cpp
@@ -198,18 +198,13 @@ Decoder::getSectionContaining(const COFFObjectFile &COFF, uint64_t VA) {
 ErrorOr<object::SymbolRef> Decoder::getSymbol(const COFFObjectFile &COFF,
                                               uint64_t VA, bool FunctionOnly) {
   for (const auto &Symbol : COFF.symbols()) {
-    if (FunctionOnly) {
-      SymbolRef::Type Type;
-      if (std::error_code EC = Symbol.getType(Type))
-        return EC;
-      if (Type != SymbolRef::ST_Function)
-        continue;
-    }
+    if (FunctionOnly && Symbol.getType() != SymbolRef::ST_Function)
+      continue;
 
-    uint64_t Address;
-    if (std::error_code EC = Symbol.getAddress(Address))
+    ErrorOr<uint64_t> Address = Symbol.getAddress();
+    if (std::error_code EC = Address.getError())
       return EC;
-    if (Address == VA)
+    if (*Address == VA)
       return Symbol;
   }
   return readobj_error::unknown_symbol;
@@ -219,9 +214,7 @@ ErrorOr<SymbolRef> Decoder::getRelocatedSymbol(const COFFObjectFile &,
                                                const SectionRef &Section,
                                                uint64_t Offset) {
   for (const auto &Relocation : Section.relocations()) {
-    uint64_t RelocationOffset;
-    if (auto Error = Relocation.getOffset(RelocationOffset))
-      return Error;
+    uint64_t RelocationOffset = Relocation.getOffset();
     if (RelocationOffset == Offset)
       return *Relocation.getSymbol();
   }
@@ -574,12 +567,12 @@ bool Decoder::dumpXDataRecord(const COFFObjectFile &COFF,
     if (!Symbol)
       Symbol = getSymbol(COFF, Address, /*FunctionOnly=*/true);
 
-    StringRef Name;
-    if (Symbol)
-      Symbol->getName(Name);
+    ErrorOr<StringRef> Name = Symbol->getName();
+    if (std::error_code EC = Name.getError())
+      report_fatal_error(EC.message());
 
     ListScope EHS(SW, "ExceptionHandler");
-    SW.printString("Routine", formatSymbol(Name, Address));
+    SW.printString("Routine", formatSymbol(*Name, Address));
     SW.printHex("Parameter", Parameter);
   }
 
@@ -608,8 +601,14 @@ bool Decoder::dumpUnpackedEntry(const COFFObjectFile &COFF,
   StringRef FunctionName;
   uint64_t FunctionAddress;
   if (Function) {
-    Function->getName(FunctionName);
-    Function->getAddress(FunctionAddress);
+    ErrorOr<StringRef> FunctionNameOrErr = Function->getName();
+    if (std::error_code EC = FunctionNameOrErr.getError())
+      report_fatal_error(EC.message());
+    FunctionName = *FunctionNameOrErr;
+    ErrorOr<uint64_t> FunctionAddressOrErr = Function->getAddress();
+    if (std::error_code EC = FunctionAddressOrErr.getError())
+      report_fatal_error(EC.message());
+    FunctionAddress = *FunctionAddressOrErr;
   } else {
     const pe32_header *PEHeader;
     if (COFF.getPE32Header(PEHeader))
@@ -620,13 +619,16 @@ bool Decoder::dumpUnpackedEntry(const COFFObjectFile &COFF,
   SW.printString("Function", formatSymbol(FunctionName, FunctionAddress));
 
   if (XDataRecord) {
-    StringRef Name;
-    uint64_t Address;
+    ErrorOr<StringRef> Name = XDataRecord->getName();
+    if (std::error_code EC = Name.getError())
+      report_fatal_error(EC.message());
 
-    XDataRecord->getName(Name);
-    XDataRecord->getAddress(Address);
+    ErrorOr<uint64_t> AddressOrErr = XDataRecord->getAddress();
+    if (std::error_code EC = AddressOrErr.getError())
+      report_fatal_error(EC.message());
+    uint64_t Address = *AddressOrErr;
 
-    SW.printString("ExceptionRecord", formatSymbol(Name, Address));
+    SW.printString("ExceptionRecord", formatSymbol(*Name, Address));
 
     section_iterator SI = COFF.section_end();
     if (XDataRecord->getSection(SI))
@@ -665,8 +667,12 @@ bool Decoder::dumpPackedEntry(const object::COFFObjectFile &COFF,
   StringRef FunctionName;
   uint64_t FunctionAddress;
   if (Function) {
-    Function->getName(FunctionName);
-    Function->getAddress(FunctionAddress);
+    ErrorOr<StringRef> FunctionNameOrErr = Function->getName();
+    if (std::error_code EC = FunctionNameOrErr.getError())
+      report_fatal_error(EC.message());
+    FunctionName = *FunctionNameOrErr;
+    ErrorOr<uint64_t> FunctionAddressOrErr = Function->getAddress();
+    FunctionAddress = *FunctionAddressOrErr;
   } else {
     const pe32_header *PEHeader;
     if (COFF.getPE32Header(PEHeader))
diff --git a/tools/llvm-readobj/CMakeLists.txt b/tools/llvm-readobj/CMakeLists.txt
index 30f336f76ca..87407a25802 100644
--- a/tools/llvm-readobj/CMakeLists.txt
+++ b/tools/llvm-readobj/CMakeLists.txt
@@ -1,5 +1,4 @@
 set(LLVM_LINK_COMPONENTS
-  ${LLVM_TARGETS_TO_BUILD}
   Object
   Support
   )
diff --git a/tools/llvm-readobj/COFFDumper.cpp b/tools/llvm-readobj/COFFDumper.cpp
index 141211134ff..43c4486fafc 100644
--- a/tools/llvm-readobj/COFFDumper.cpp
+++ b/tools/llvm-readobj/COFFDumper.cpp
@@ -16,6 +16,7 @@
 #include "ARMWinEHPrinter.h"
 #include "Error.h"
 #include "ObjDumper.h"
+#include "StackMapPrinter.h"
 #include "StreamWriter.h"
 #include "Win64EHDumper.h"
 #include "llvm/ADT/DenseMap.h"
@@ -47,7 +48,6 @@ class COFFDumper : public ObjDumper {
   COFFDumper(const llvm::object::COFFObjectFile *Obj, StreamWriter& Writer)
     : ObjDumper(Writer)
     , Obj(Obj) {
-    cacheRelocations();
   }
 
   void printFileHeaders() override;
@@ -60,7 +60,7 @@ class COFFDumper : public ObjDumper {
   void printCOFFExports() override;
   void printCOFFDirectives() override;
   void printCOFFBaseReloc() override;
-
+  void printStackMap() const override;
 private:
   void printSymbol(const SymbolRef &Sym);
   void printRelocation(const SectionRef &Section, const RelocationRef &Reloc);
@@ -91,6 +91,7 @@ class COFFDumper : public ObjDumper {
   typedef DenseMap<const coff_section*, std::vector<RelocationRef> > RelocMapTy;
 
   const llvm::object::COFFObjectFile *Obj;
+  bool RelocCached = false;
   RelocMapTy RelocMap;
   StringRef CVFileIndexToStringOffsetTable;
   StringRef CVStringTable;
@@ -118,11 +119,10 @@ std::error_code createCOFFDumper(const object::ObjectFile *Obj,
 // symbol used for the relocation at the offset.
 std::error_code COFFDumper::resolveSymbol(const coff_section *Section,
                                           uint64_t Offset, SymbolRef &Sym) {
+  cacheRelocations();
   const auto &Relocations = RelocMap[Section];
   for (const auto &Relocation : Relocations) {
-    uint64_t RelocationOffset;
-    if (std::error_code EC = Relocation.getOffset(RelocationOffset))
-      return EC;
+    uint64_t RelocationOffset = Relocation.getOffset();
 
     if (RelocationOffset == Offset) {
       Sym = *Relocation.getSymbol();
@@ -140,9 +140,11 @@ std::error_code COFFDumper::resolveSymbolName(const coff_section *Section,
   SymbolRef Symbol;
   if (std::error_code EC = resolveSymbol(Section, Offset, Symbol))
     return EC;
-  if (std::error_code EC = Symbol.getName(Name))
+  ErrorOr<StringRef> NameOrErr = Symbol.getName();
+  if (std::error_code EC = NameOrErr.getError())
     return EC;
-  return object_error::success;
+  Name = *NameOrErr;
+  return std::error_code();
 }
 
 static const EnumEntry<COFF::MachineTypes> ImageFileMachineType[] = {
@@ -217,6 +219,7 @@ static const EnumEntry<COFF::DLLCharacteristics> PEDLLCharacteristics[] = {
 
 static const EnumEntry<COFF::SectionCharacteristics>
 ImageSectionCharacteristics[] = {
+  LLVM_READOBJ_ENUM_ENT(COFF, IMAGE_SCN_TYPE_NOLOAD           ),
   LLVM_READOBJ_ENUM_ENT(COFF, IMAGE_SCN_TYPE_NO_PAD           ),
   LLVM_READOBJ_ENUM_ENT(COFF, IMAGE_SCN_CNT_CODE              ),
   LLVM_READOBJ_ENUM_ENT(COFF, IMAGE_SCN_CNT_INITIALIZED_DATA  ),
@@ -338,6 +341,10 @@ static std::error_code getSymbolAuxData(const COFFObjectFile *Obj,
 }
 
 void COFFDumper::cacheRelocations() {
+  if (RelocCached)
+    return;
+  RelocCached = true;
+
   for (const SectionRef &S : Obj->sections()) {
     const coff_section *Section = Obj->getCOFFSection(S);
 
@@ -379,14 +386,12 @@ void COFFDumper::printFileHeaders() {
   // Print PE header. This header does not exist if this is an object file and
   // not an executable.
   const pe32_header *PEHeader = nullptr;
-  if (error(Obj->getPE32Header(PEHeader)))
-    return;
+  error(Obj->getPE32Header(PEHeader));
   if (PEHeader)
     printPEHeader<pe32_header>(PEHeader);
 
   const pe32plus_header *PEPlusHeader = nullptr;
-  if (error(Obj->getPE32PlusHeader(PEPlusHeader)))
-    return;
+  error(Obj->getPE32PlusHeader(PEPlusHeader));
   if (PEPlusHeader)
     printPEHeader<pe32plus_header>(PEPlusHeader);
 
@@ -471,8 +476,7 @@ void COFFDumper::printBaseOfDataField(const pe32plus_header *) {}
 
 void COFFDumper::printCodeViewDebugInfo(const SectionRef &Section) {
   StringRef Data;
-  if (error(Section.getContents(Data)))
-    return;
+  error(Section.getContents(Data));
 
   SmallVector<StringRef, 10> FunctionNames;
   StringMap<StringRef> FunctionLineTables;
@@ -528,9 +532,8 @@ void COFFDumper::printCodeViewDebugInfo(const SectionRef &Section) {
         }
 
         StringRef FunctionName;
-        if (error(resolveSymbolName(Obj->getCOFFSection(Section), Offset,
-                                    FunctionName)))
-          return;
+        error(resolveSymbolName(Obj->getCOFFSection(Section), Offset,
+                                FunctionName));
         W.printString("FunctionName", FunctionName);
         if (FunctionLineTables.count(FunctionName) != 0) {
           // Saw debug info for this function already?
@@ -579,7 +582,11 @@ void COFFDumper::printCodeViewDebugInfo(const SectionRef &Section) {
     W.printString("FunctionName", Name);
 
     DataExtractor DE(FunctionLineTables[Name], true, 4);
-    uint32_t Offset = 8;  // Skip relocations.
+    uint32_t Offset = 6;  // Skip relocations.
+    uint16_t Flags = DE.getU16(&Offset);
+    W.printHex("Flags", Flags);
+    bool HasColumnInformation =
+        Flags & COFF::DEBUG_LINE_TABLES_HAVE_COLUMN_RECORDS;
     uint32_t FunctionSize = DE.getU32(&Offset);
     W.printHex("CodeSize", FunctionSize);
     while (DE.isValidOffset(Offset)) {
@@ -587,9 +594,12 @@ void COFFDumper::printCodeViewDebugInfo(const SectionRef &Section) {
       // in the line table.  The filename string is accessed using double
       // indirection to the string table subsection using the index subsection.
       uint32_t OffsetInIndex = DE.getU32(&Offset),
-               SegmentLength   = DE.getU32(&Offset),
+               SegmentLength = DE.getU32(&Offset),
                FullSegmentSize = DE.getU32(&Offset);
-      if (FullSegmentSize != 12 + 8 * SegmentLength) {
+
+      if (FullSegmentSize !=
+          12 + 8 * SegmentLength +
+              (HasColumnInformation ? 4 * SegmentLength : 0)) {
         error(object_error::parse_failed);
         return;
       }
@@ -630,6 +640,15 @@ void COFFDumper::printCodeViewDebugInfo(const SectionRef &Section) {
         format("+0x%X", PC).snprint(Buffer, 32);
         W.printNumber(Buffer, LineNumber);
       }
+      if (HasColumnInformation) {
+        for (unsigned J = 0; J != SegmentLength && DE.isValidOffset(Offset);
+             ++J) {
+          uint16_t ColStart = DE.getU16(&Offset);
+          W.printNumber("ColStart", ColStart);
+          uint16_t ColEnd = DE.getU16(&Offset);
+          W.printNumber("ColEnd", ColEnd);
+        }
+      }
     }
   }
 }
@@ -673,9 +692,8 @@ void COFFDumper::printCodeViewSymbolsSubsection(StringRef Subsection,
       uint32_t CodeSize = DE.getU32(&Offset);
       DE.getU8(&Offset, Unused, 12);
       StringRef SectionName;
-      if (error(resolveSymbolName(Obj->getCOFFSection(Section),
-                                  OffsetInSection + Offset, SectionName)))
-        return;
+      error(resolveSymbolName(Obj->getCOFFSection(Section),
+                              OffsetInSection + Offset, SectionName));
       Offset += 4;
       DE.getU8(&Offset, Unused, 3);
       StringRef DisplayName = DE.getCStr(&Offset);
@@ -726,8 +744,7 @@ void COFFDumper::printSections() {
     const coff_section *Section = Obj->getCOFFSection(Sec);
 
     StringRef Name;
-    if (error(Sec.getName(Name)))
-      Name = "";
+    error(Sec.getName(Name));
 
     DictScope D(W, "Section");
     W.printNumber("Number", SectionNumber);
@@ -766,8 +783,7 @@ void COFFDumper::printSections() {
     if (opts::SectionData &&
         !(Section->Characteristics & COFF::IMAGE_SCN_CNT_UNINITIALIZED_DATA)) {
       StringRef Data;
-      if (error(Sec.getContents(Data)))
-        break;
+      error(Sec.getContents(Data));
 
       W.printBinaryBlock("SectionData", Data);
     }
@@ -781,8 +797,7 @@ void COFFDumper::printRelocations() {
   for (const SectionRef &Section : Obj->sections()) {
     ++SectionNumber;
     StringRef Name;
-    if (error(Section.getName(Name)))
-      continue;
+    error(Section.getName(Name));
 
     bool PrintedGroup = false;
     for (const RelocationRef &Reloc : Section.relocations()) {
@@ -804,19 +819,17 @@ void COFFDumper::printRelocations() {
 
 void COFFDumper::printRelocation(const SectionRef &Section,
                                  const RelocationRef &Reloc) {
-  uint64_t Offset;
-  uint64_t RelocType;
+  uint64_t Offset = Reloc.getOffset();
+  uint64_t RelocType = Reloc.getType();
   SmallString<32> RelocName;
   StringRef SymbolName;
-  if (error(Reloc.getOffset(Offset)))
-    return;
-  if (error(Reloc.getType(RelocType)))
-    return;
-  if (error(Reloc.getTypeName(RelocName)))
-    return;
+  Reloc.getTypeName(RelocName);
   symbol_iterator Symbol = Reloc.getSymbol();
-  if (Symbol != Obj->symbol_end() && error(Symbol->getName(SymbolName)))
-    return;
+  if (Symbol != Obj->symbol_end()) {
+    ErrorOr<StringRef> SymbolNameOrErr = Symbol->getName();
+    error(SymbolNameOrErr.getError());
+    SymbolName = *SymbolNameOrErr;
+  }
 
   if (opts::ExpandRelocs) {
     DictScope Group(W, "Relocation");
@@ -893,8 +906,7 @@ void COFFDumper::printSymbol(const SymbolRef &Sym) {
   for (uint8_t I = 0; I < Symbol.getNumberOfAuxSymbols(); ++I) {
     if (Symbol.isFunctionDefinition()) {
       const coff_aux_function_definition *Aux;
-      if (error(getSymbolAuxData(Obj, Symbol, I, Aux)))
-        break;
+      error(getSymbolAuxData(Obj, Symbol, I, Aux));
 
       DictScope AS(W, "AuxFunctionDef");
       W.printNumber("TagIndex", Aux->TagIndex);
@@ -904,8 +916,7 @@ void COFFDumper::printSymbol(const SymbolRef &Sym) {
 
     } else if (Symbol.isAnyUndefined()) {
       const coff_aux_weak_external *Aux;
-      if (error(getSymbolAuxData(Obj, Symbol, I, Aux)))
-        break;
+      error(getSymbolAuxData(Obj, Symbol, I, Aux));
 
       ErrorOr<COFFSymbolRef> Linked = Obj->getSymbol(Aux->TagIndex);
       StringRef LinkedName;
@@ -922,8 +933,7 @@ void COFFDumper::printSymbol(const SymbolRef &Sym) {
 
     } else if (Symbol.isFileRecord()) {
       const char *FileName;
-      if (error(getSymbolAuxData(Obj, Symbol, I, FileName)))
-        break;
+      error(getSymbolAuxData(Obj, Symbol, I, FileName));
 
       DictScope AS(W, "AuxFileRecord");
 
@@ -933,8 +943,7 @@ void COFFDumper::printSymbol(const SymbolRef &Sym) {
       break;
     } else if (Symbol.isSectionDefinition()) {
       const coff_aux_section_definition *Aux;
-      if (error(getSymbolAuxData(Obj, Symbol, I, Aux)))
-        break;
+      error(getSymbolAuxData(Obj, Symbol, I, Aux));
 
       int32_t AuxNumber = Aux->getNumber(Symbol.isBigObj());
 
@@ -965,8 +974,7 @@ void COFFDumper::printSymbol(const SymbolRef &Sym) {
       }
     } else if (Symbol.isCLRToken()) {
       const coff_aux_clr_token *Aux;
-      if (error(getSymbolAuxData(Obj, Symbol, I, Aux)))
-        break;
+      error(getSymbolAuxData(Obj, Symbol, I, Aux));
 
       ErrorOr<COFFSymbolRef> ReferredSym =
           Obj->getSymbol(Aux->SymbolTableIndex);
@@ -1019,9 +1027,9 @@ void COFFDumper::printImportedSymbols(
     iterator_range<imported_symbol_iterator> Range) {
   for (const ImportedSymbolRef &I : Range) {
     StringRef Sym;
-    if (error(I.getSymbolName(Sym))) return;
+    error(I.getSymbolName(Sym));
     uint16_t Ordinal;
-    if (error(I.getOrdinal(Ordinal))) return;
+    error(I.getOrdinal(Ordinal));
     W.printNumber("Symbol", Sym, Ordinal);
   }
 }
@@ -1033,12 +1041,12 @@ void COFFDumper::printDelayImportedSymbols(
   for (const ImportedSymbolRef &S : Range) {
     DictScope Import(W, "Import");
     StringRef Sym;
-    if (error(S.getSymbolName(Sym))) return;
+    error(S.getSymbolName(Sym));
     uint16_t Ordinal;
-    if (error(S.getOrdinal(Ordinal))) return;
+    error(S.getOrdinal(Ordinal));
     W.printNumber("Symbol", Sym, Ordinal);
     uint64_t Addr;
-    if (error(I.getImportAddress(Index++, Addr))) return;
+    error(I.getImportAddress(Index++, Addr));
     W.printHex("Address", Addr);
   }
 }
@@ -1048,12 +1056,12 @@ void COFFDumper::printCOFFImports() {
   for (const ImportDirectoryEntryRef &I : Obj->import_directories()) {
     DictScope Import(W, "Import");
     StringRef Name;
-    if (error(I.getName(Name))) return;
+    error(I.getName(Name));
     W.printString("Name", Name);
     uint32_t Addr;
-    if (error(I.getImportLookupTableRVA(Addr))) return;
+    error(I.getImportLookupTableRVA(Addr));
     W.printHex("ImportLookupTableRVA", Addr);
-    if (error(I.getImportAddressTableRVA(Addr))) return;
+    error(I.getImportAddressTableRVA(Addr));
     W.printHex("ImportAddressTableRVA", Addr);
     printImportedSymbols(I.imported_symbols());
   }
@@ -1062,10 +1070,10 @@ void COFFDumper::printCOFFImports() {
   for (const DelayImportDirectoryEntryRef &I : Obj->delay_import_directories()) {
     DictScope Import(W, "DelayImport");
     StringRef Name;
-    if (error(I.getName(Name))) return;
+    error(I.getName(Name));
     W.printString("Name", Name);
     const delay_import_directory_table_entry *Table;
-    if (error(I.getDelayImportTable(Table))) return;
+    error(I.getDelayImportTable(Table));
     W.printHex("Attributes", Table->Attributes);
     W.printHex("ModuleHandle", Table->ModuleHandle);
     W.printHex("ImportAddressTable", Table->DelayImportAddressTable);
@@ -1083,12 +1091,9 @@ void COFFDumper::printCOFFExports() {
     StringRef Name;
     uint32_t Ordinal, RVA;
 
-    if (error(E.getSymbolName(Name)))
-      continue;
-    if (error(E.getOrdinal(Ordinal)))
-      continue;
-    if (error(E.getExportRVA(RVA)))
-      continue;
+    error(E.getSymbolName(Name));
+    error(E.getOrdinal(Ordinal));
+    error(E.getExportRVA(RVA));
 
     W.printNumber("Ordinal", Ordinal);
     W.printString("Name", Name);
@@ -1101,13 +1106,11 @@ void COFFDumper::printCOFFDirectives() {
     StringRef Contents;
     StringRef Name;
 
-    if (error(Section.getName(Name)))
-      continue;
+    error(Section.getName(Name));
     if (Name != ".drectve")
       continue;
 
-    if (error(Section.getContents(Contents)))
-      return;
+    error(Section.getContents(Contents));
 
     W.printString("Directive(s)", Contents);
   }
@@ -1131,12 +1134,39 @@ void COFFDumper::printCOFFBaseReloc() {
   for (const BaseRelocRef &I : Obj->base_relocs()) {
     uint8_t Type;
     uint32_t RVA;
-    if (error(I.getRVA(RVA)))
-      continue;
-    if (error(I.getType(Type)))
-      continue;
+    error(I.getRVA(RVA));
+    error(I.getType(Type));
     DictScope Import(W, "Entry");
     W.printString("Type", getBaseRelocTypeName(Type));
     W.printHex("Address", RVA);
   }
 }
+
+void COFFDumper::printStackMap() const {
+  object::SectionRef StackMapSection;
+  for (auto Sec : Obj->sections()) {
+    StringRef Name;
+    Sec.getName(Name);
+    if (Name == ".llvm_stackmaps") {
+      StackMapSection = Sec;
+      break;
+    }
+  }
+
+  if (StackMapSection == object::SectionRef())
+    return;
+
+  StringRef StackMapContents;
+  StackMapSection.getContents(StackMapContents);
+  ArrayRef<uint8_t> StackMapContentsArray(
+      reinterpret_cast<const uint8_t*>(StackMapContents.data()),
+      StackMapContents.size());
+
+  if (Obj->isLittleEndian())
+    prettyPrintStackMap(
+                      llvm::outs(),
+                      StackMapV1Parser<support::little>(StackMapContentsArray));
+  else
+    prettyPrintStackMap(llvm::outs(),
+                        StackMapV1Parser<support::big>(StackMapContentsArray));
+}
diff --git a/tools/llvm-readobj/ELFDumper.cpp b/tools/llvm-readobj/ELFDumper.cpp
index 39fb065810f..b4118f06665 100644
--- a/tools/llvm-readobj/ELFDumper.cpp
+++ b/tools/llvm-readobj/ELFDumper.cpp
@@ -17,6 +17,7 @@
 #include "ARMEHABIPrinter.h"
 #include "Error.h"
 #include "ObjDumper.h"
+#include "StackMapPrinter.h"
 #include "StreamWriter.h"
 #include "llvm/ADT/Optional.h"
 #include "llvm/ADT/SmallString.h"
@@ -26,6 +27,7 @@
 #include "llvm/Support/Compiler.h"
 #include "llvm/Support/Format.h"
 #include "llvm/Support/MathExtras.h"
+#include "llvm/Support/MipsABIFlags.h"
 #include "llvm/Support/raw_ostream.h"
 
 using namespace llvm;
@@ -40,12 +42,12 @@ namespace {
 template<typename ELFT>
 class ELFDumper : public ObjDumper {
 public:
-  ELFDumper(const ELFFile<ELFT> *Obj, StreamWriter &Writer)
-      : ObjDumper(Writer), Obj(Obj) {}
+  ELFDumper(const ELFFile<ELFT> *Obj, StreamWriter &Writer);
 
   void printFileHeaders() override;
   void printSections() override;
   void printRelocations() override;
+  void printDynamicRelocations() override;
   void printSymbols() override;
   void printDynamicSymbols() override;
   void printUnwindInfo() override;
@@ -53,21 +55,64 @@ class ELFDumper : public ObjDumper {
   void printDynamicTable() override;
   void printNeededLibraries() override;
   void printProgramHeaders() override;
+  void printHashTable() override;
+  void printLoadName() override;
 
   void printAttributes() override;
   void printMipsPLTGOT() override;
+  void printMipsABIFlags() override;
+  void printMipsReginfo() override;
+
+  void printStackMap() const override;
 
 private:
   typedef ELFFile<ELFT> ELFO;
   typedef typename ELFO::Elf_Shdr Elf_Shdr;
   typedef typename ELFO::Elf_Sym Elf_Sym;
-
-  void printSymbol(typename ELFO::Elf_Sym_Iter Symbol);
+  typedef typename ELFO::Elf_Dyn Elf_Dyn;
+  typedef typename ELFO::Elf_Dyn_Range Elf_Dyn_Range;
+  typedef typename ELFO::Elf_Rel Elf_Rel;
+  typedef typename ELFO::Elf_Rela Elf_Rela;
+  typedef typename ELFO::Elf_Rela_Range Elf_Rela_Range;
+  typedef typename ELFO::Elf_Phdr Elf_Phdr;
+  typedef typename ELFO::Elf_Hash Elf_Hash;
+  typedef typename ELFO::Elf_Ehdr Elf_Ehdr;
+  typedef typename ELFO::uintX_t uintX_t;
+
+  /// \brief Represents a region described by entries in the .dynamic table.
+  struct DynRegionInfo {
+    DynRegionInfo() : Addr(nullptr), Size(0), EntSize(0) {}
+    /// \brief Address in current address space.
+    const void *Addr;
+    /// \brief Size in bytes of the region.
+    uintX_t Size;
+    /// \brief Size of each entity in the region.
+    uintX_t EntSize;
+  };
+
+  void printSymbol(const Elf_Sym *Symbol, StringRef StrTable, bool IsDynamic);
 
   void printRelocations(const Elf_Shdr *Sec);
-  void printRelocation(const Elf_Shdr *Sec, typename ELFO::Elf_Rela Rel);
+  void printRelocation(const Elf_Shdr *Sec, Elf_Rela Rel);
+  void printValue(uint64_t Type, uint64_t Value);
+
+  const Elf_Rela *dyn_rela_begin() const;
+  const Elf_Rela *dyn_rela_end() const;
+  Elf_Rela_Range dyn_relas() const;
+  StringRef getDynamicString(uint64_t Offset) const;
+  const Elf_Dyn *dynamic_table_begin() const;
+  const Elf_Dyn *dynamic_table_end() const;
+  Elf_Dyn_Range dynamic_table() const {
+    return make_range(dynamic_table_begin(), dynamic_table_end());
+  }
 
   const ELFO *Obj;
+  DynRegionInfo DynRelaRegion;
+  DynRegionInfo DynamicRegion;
+  StringRef DynamicStringTable;
+  const Elf_Sym *DynSymStart = nullptr;
+  StringRef SOName;
+  const Elf_Hash *HashTable = nullptr;
 };
 
 template <class T> T errorOrDefault(ErrorOr<T> Val, T Default = T()) {
@@ -116,16 +161,17 @@ std::error_code createELFDumper(const object::ObjectFile *Obj,
 
 template <typename ELFO>
 static std::string getFullSymbolName(const ELFO &Obj,
-                                     typename ELFO::Elf_Sym_Iter Symbol) {
-  StringRef SymbolName = errorOrDefault(Obj.getSymbolName(Symbol));
-  if (!Symbol.isDynamic())
+                                     const typename ELFO::Elf_Sym *Symbol,
+                                     StringRef StrTable, bool IsDynamic) {
+  StringRef SymbolName = errorOrDefault(Symbol->getName(StrTable));
+  if (!IsDynamic)
     return SymbolName;
 
   std::string FullSymbolName(SymbolName);
 
   bool IsDefault;
   ErrorOr<StringRef> Version =
-      Obj.getSymbolVersion(nullptr, &*Symbol, IsDefault);
+      Obj.getSymbolVersion(StrTable, &*Symbol, IsDefault);
   if (Version) {
     FullSymbolName += (IsDefault ? "@@" : "@");
     FullSymbolName += *Version;
@@ -136,40 +182,49 @@ static std::string getFullSymbolName(const ELFO &Obj,
 
 template <typename ELFO>
 static void
-getSectionNameIndex(const ELFO &Obj, typename ELFO::Elf_Sym_Iter Symbol,
+getSectionNameIndex(const ELFO &Obj, const typename ELFO::Elf_Sym *Symbol,
                     StringRef &SectionName, unsigned &SectionIndex) {
   SectionIndex = Symbol->st_shndx;
-  if (SectionIndex == SHN_UNDEF) {
+  if (Symbol->isUndefined())
     SectionName = "Undefined";
-  } else if (SectionIndex >= SHN_LOPROC && SectionIndex <= SHN_HIPROC) {
+  else if (Symbol->isProcessorSpecific())
     SectionName = "Processor Specific";
-  } else if (SectionIndex >= SHN_LOOS && SectionIndex <= SHN_HIOS) {
+  else if (Symbol->isOSSpecific())
     SectionName = "Operating System Specific";
-  } else if (SectionIndex > SHN_HIOS && SectionIndex < SHN_ABS) {
-    SectionName = "Reserved";
-  } else if (SectionIndex == SHN_ABS) {
+  else if (Symbol->isAbsolute())
     SectionName = "Absolute";
-  } else if (SectionIndex == SHN_COMMON) {
+  else if (Symbol->isCommon())
     SectionName = "Common";
-  } else {
+  else if (Symbol->isReserved() && SectionIndex != SHN_XINDEX)
+    SectionName = "Reserved";
+  else {
     if (SectionIndex == SHN_XINDEX)
-      SectionIndex = Obj.getSymbolTableIndex(&*Symbol);
-    assert(SectionIndex != SHN_XINDEX &&
-           "getSymbolTableIndex should handle this");
-    const typename ELFO::Elf_Shdr *Sec = Obj.getSection(SectionIndex);
-    SectionName = errorOrDefault(Obj.getSectionName(Sec));
+      SectionIndex = Obj.getExtendedSymbolTableIndex(&*Symbol);
+    ErrorOr<const typename ELFO::Elf_Shdr *> Sec = Obj.getSection(SectionIndex);
+    error(Sec.getError());
+    SectionName = errorOrDefault(Obj.getSectionName(*Sec));
   }
 }
 
-template <class ELFT>
-static const typename ELFFile<ELFT>::Elf_Shdr *
-findSectionByAddress(const ELFFile<ELFT> *Obj, uint64_t Addr) {
+template <class ELFO>
+static const typename ELFO::Elf_Shdr *findSectionByAddress(const ELFO *Obj,
+                                                           uint64_t Addr) {
   for (const auto &Shdr : Obj->sections())
     if (Shdr.sh_addr == Addr)
       return &Shdr;
   return nullptr;
 }
 
+template <class ELFO>
+static const typename ELFO::Elf_Shdr *findSectionByName(const ELFO &Obj,
+                                                        StringRef Name) {
+  for (const auto &Shdr : Obj.sections()) {
+    if (Name == errorOrDefault(Obj.getSectionName(&Shdr)))
+      return &Shdr;
+  }
+  return nullptr;
+}
+
 static const EnumEntry<unsigned> ElfClass[] = {
   { "None",   ELF::ELFCLASSNONE },
   { "32-bit", ELF::ELFCLASS32   },
@@ -221,7 +276,7 @@ static const EnumEntry<unsigned> ElfMachineType[] = {
   LLVM_READOBJ_ENUM_ENT(ELF, EM_386          ),
   LLVM_READOBJ_ENUM_ENT(ELF, EM_68K          ),
   LLVM_READOBJ_ENUM_ENT(ELF, EM_88K          ),
-  LLVM_READOBJ_ENUM_ENT(ELF, EM_486          ),
+  LLVM_READOBJ_ENUM_ENT(ELF, EM_IAMCU        ),
   LLVM_READOBJ_ENUM_ENT(ELF, EM_860          ),
   LLVM_READOBJ_ENUM_ENT(ELF, EM_MIPS         ),
   LLVM_READOBJ_ENUM_ENT(ELF, EM_S370         ),
@@ -369,7 +424,8 @@ static const EnumEntry<unsigned> ElfMachineType[] = {
   LLVM_READOBJ_ENUM_ENT(ELF, EM_RL78         ),
   LLVM_READOBJ_ENUM_ENT(ELF, EM_VIDEOCORE5   ),
   LLVM_READOBJ_ENUM_ENT(ELF, EM_78KOR        ),
-  LLVM_READOBJ_ENUM_ENT(ELF, EM_56800EX      )
+  LLVM_READOBJ_ENUM_ENT(ELF, EM_56800EX      ),
+  LLVM_READOBJ_ENUM_ENT(ELF, EM_AMDGPU       )
 };
 
 static const EnumEntry<unsigned> ElfSymbolBindings[] = {
@@ -471,6 +527,7 @@ static const char *getElfSegmentType(unsigned Arch, unsigned Type) {
     LLVM_READOBJ_ENUM_CASE(ELF, PT_MIPS_REGINFO);
     LLVM_READOBJ_ENUM_CASE(ELF, PT_MIPS_RTPROC);
     LLVM_READOBJ_ENUM_CASE(ELF, PT_MIPS_OPTIONS);
+    LLVM_READOBJ_ENUM_CASE(ELF, PT_MIPS_ABIFLAGS);
     }
   }
 
@@ -505,10 +562,33 @@ static const EnumEntry<unsigned> ElfHeaderMipsFlags[] = {
   LLVM_READOBJ_ENUM_ENT(ELF, EF_MIPS_CPIC),
   LLVM_READOBJ_ENUM_ENT(ELF, EF_MIPS_ABI2),
   LLVM_READOBJ_ENUM_ENT(ELF, EF_MIPS_32BITMODE),
+  LLVM_READOBJ_ENUM_ENT(ELF, EF_MIPS_FP64),
   LLVM_READOBJ_ENUM_ENT(ELF, EF_MIPS_NAN2008),
   LLVM_READOBJ_ENUM_ENT(ELF, EF_MIPS_ABI_O32),
+  LLVM_READOBJ_ENUM_ENT(ELF, EF_MIPS_ABI_O64),
+  LLVM_READOBJ_ENUM_ENT(ELF, EF_MIPS_ABI_EABI32),
+  LLVM_READOBJ_ENUM_ENT(ELF, EF_MIPS_ABI_EABI64),
+  LLVM_READOBJ_ENUM_ENT(ELF, EF_MIPS_MACH_3900),
+  LLVM_READOBJ_ENUM_ENT(ELF, EF_MIPS_MACH_4010),
+  LLVM_READOBJ_ENUM_ENT(ELF, EF_MIPS_MACH_4100),
+  LLVM_READOBJ_ENUM_ENT(ELF, EF_MIPS_MACH_4650),
+  LLVM_READOBJ_ENUM_ENT(ELF, EF_MIPS_MACH_4120),
+  LLVM_READOBJ_ENUM_ENT(ELF, EF_MIPS_MACH_4111),
+  LLVM_READOBJ_ENUM_ENT(ELF, EF_MIPS_MACH_SB1),
+  LLVM_READOBJ_ENUM_ENT(ELF, EF_MIPS_MACH_OCTEON),
+  LLVM_READOBJ_ENUM_ENT(ELF, EF_MIPS_MACH_XLR),
+  LLVM_READOBJ_ENUM_ENT(ELF, EF_MIPS_MACH_OCTEON2),
+  LLVM_READOBJ_ENUM_ENT(ELF, EF_MIPS_MACH_OCTEON3),
+  LLVM_READOBJ_ENUM_ENT(ELF, EF_MIPS_MACH_5400),
+  LLVM_READOBJ_ENUM_ENT(ELF, EF_MIPS_MACH_5900),
+  LLVM_READOBJ_ENUM_ENT(ELF, EF_MIPS_MACH_5500),
+  LLVM_READOBJ_ENUM_ENT(ELF, EF_MIPS_MACH_9000),
+  LLVM_READOBJ_ENUM_ENT(ELF, EF_MIPS_MACH_LS2E),
+  LLVM_READOBJ_ENUM_ENT(ELF, EF_MIPS_MACH_LS2F),
+  LLVM_READOBJ_ENUM_ENT(ELF, EF_MIPS_MACH_LS3A),
   LLVM_READOBJ_ENUM_ENT(ELF, EF_MIPS_MICROMIPS),
   LLVM_READOBJ_ENUM_ENT(ELF, EF_MIPS_ARCH_ASE_M16),
+  LLVM_READOBJ_ENUM_ENT(ELF, EF_MIPS_ARCH_ASE_MDMX),
   LLVM_READOBJ_ENUM_ENT(ELF, EF_MIPS_ARCH_1),
   LLVM_READOBJ_ENUM_ENT(ELF, EF_MIPS_ARCH_2),
   LLVM_READOBJ_ENUM_ENT(ELF, EF_MIPS_ARCH_3),
@@ -522,9 +602,115 @@ static const EnumEntry<unsigned> ElfHeaderMipsFlags[] = {
   LLVM_READOBJ_ENUM_ENT(ELF, EF_MIPS_ARCH_64R6)
 };
 
+template <typename ELFT>
+ELFDumper<ELFT>::ELFDumper(const ELFFile<ELFT> *Obj, StreamWriter &Writer)
+    : ObjDumper(Writer), Obj(Obj) {
+
+  SmallVector<const Elf_Phdr *, 4> LoadSegments;
+  for (const Elf_Phdr &Phdr : Obj->program_headers()) {
+    if (Phdr.p_type == ELF::PT_DYNAMIC) {
+      DynamicRegion.Addr = Obj->base() + Phdr.p_offset;
+      uint64_t Size = Phdr.p_filesz;
+      if (Size % sizeof(Elf_Dyn))
+        report_fatal_error("Invalid dynamic table size");
+      DynamicRegion.Size = Phdr.p_filesz;
+      continue;
+    }
+    if (Phdr.p_type != ELF::PT_LOAD || Phdr.p_filesz == 0)
+      continue;
+    LoadSegments.push_back(&Phdr);
+  }
+
+  auto toMappedAddr = [&](uint64_t VAddr) -> const uint8_t * {
+    const Elf_Phdr **I = std::upper_bound(
+        LoadSegments.begin(), LoadSegments.end(), VAddr, compareAddr<ELFT>);
+    if (I == LoadSegments.begin())
+      report_fatal_error("Virtual address is not in any segment");
+    --I;
+    const Elf_Phdr &Phdr = **I;
+    uint64_t Delta = VAddr - Phdr.p_vaddr;
+    if (Delta >= Phdr.p_filesz)
+      report_fatal_error("Virtual address is not in any segment");
+    return Obj->base() + Phdr.p_offset + Delta;
+  };
+
+  uint64_t SONameOffset = 0;
+  const char *StringTableBegin = nullptr;
+  uint64_t StringTableSize = 0;
+  for (const Elf_Dyn &Dyn : dynamic_table()) {
+    switch (Dyn.d_tag) {
+    case ELF::DT_HASH:
+      HashTable =
+          reinterpret_cast<const Elf_Hash *>(toMappedAddr(Dyn.getPtr()));
+      break;
+    case ELF::DT_RELA:
+      DynRelaRegion.Addr = toMappedAddr(Dyn.getPtr());
+      break;
+    case ELF::DT_RELASZ:
+      DynRelaRegion.Size = Dyn.getVal();
+      break;
+    case ELF::DT_RELAENT:
+      DynRelaRegion.EntSize = Dyn.getVal();
+      break;
+    case ELF::DT_SONAME:
+      SONameOffset = Dyn.getVal();
+      break;
+    case ELF::DT_STRTAB:
+      StringTableBegin = (const char *)toMappedAddr(Dyn.getPtr());
+      break;
+    case ELF::DT_STRSZ:
+      StringTableSize = Dyn.getVal();
+      break;
+    case ELF::DT_SYMTAB:
+      DynSymStart =
+          reinterpret_cast<const Elf_Sym *>(toMappedAddr(Dyn.getPtr()));
+      break;
+    }
+  }
+  if (StringTableBegin)
+    DynamicStringTable = StringRef(StringTableBegin, StringTableSize);
+  if (SONameOffset)
+    SOName = getDynamicString(SONameOffset);
+}
+
+template <typename ELFT>
+const typename ELFDumper<ELFT>::Elf_Rela *
+ELFDumper<ELFT>::dyn_rela_begin() const {
+  if (DynRelaRegion.Size && DynRelaRegion.EntSize != sizeof(Elf_Rela))
+    report_fatal_error("Invalid relocation entry size");
+  return reinterpret_cast<const Elf_Rela *>(DynRelaRegion.Addr);
+}
+
+template <typename ELFT>
+const typename ELFDumper<ELFT>::Elf_Rela *
+ELFDumper<ELFT>::dyn_rela_end() const {
+  uint64_t Size = DynRelaRegion.Size;
+  if (Size % sizeof(Elf_Rela))
+    report_fatal_error("Invalid relocation table size");
+  return dyn_rela_begin() + Size / sizeof(Elf_Rela);
+}
+
+template <typename ELFT>
+typename ELFDumper<ELFT>::Elf_Rela_Range ELFDumper<ELFT>::dyn_relas() const {
+  return make_range(dyn_rela_begin(), dyn_rela_end());
+}
+
+template <typename ELFT>
+const typename ELFDumper<ELFT>::Elf_Dyn *
+ELFDumper<ELFT>::dynamic_table_begin() const {
+  return reinterpret_cast<const Elf_Dyn *>(DynamicRegion.Addr);
+}
+
+template <typename ELFT>
+const typename ELFDumper<ELFT>::Elf_Dyn *
+ELFDumper<ELFT>::dynamic_table_end() const {
+  uint64_t Size = DynamicRegion.Size;
+  return dynamic_table_begin() + Size / sizeof(Elf_Dyn);
+}
+
 template<class ELFT>
 void ELFDumper<ELFT>::printFileHeaders() {
-  const typename ELFO::Elf_Ehdr *Header = Obj->getHeader();
+  const Elf_Ehdr *Header = Obj->getHeader();
 
   {
     DictScope D(W, "ElfHeader");
@@ -537,7 +723,13 @@ void ELFDumper<ELFT>::printFileHeaders() {
       W.printEnum  ("DataEncoding", Header->e_ident[ELF::EI_DATA],
                       makeArrayRef(ElfDataEncoding));
       W.printNumber("FileVersion", Header->e_ident[ELF::EI_VERSION]);
-      W.printEnum  ("OS/ABI", Header->e_ident[ELF::EI_OSABI],
+
+      // Handle architecture specific OS/ABI values.
+      if (Header->e_machine == ELF::EM_AMDGPU &&
+          Header->e_ident[ELF::EI_OSABI] == ELF::ELFOSABI_AMDGPU_HSA)
+        W.printHex("OS/ABI", "AMDGPU_HSA", ELF::ELFOSABI_AMDGPU_HSA);
+      else
+        W.printEnum  ("OS/ABI", Header->e_ident[ELF::EI_OSABI],
                       makeArrayRef(ElfOSABI));
       W.printNumber("ABIVersion", Header->e_ident[ELF::EI_ABIVERSION]);
       W.printBinary("Unused", makeArrayRef(Header->e_ident).slice(ELF::EI_PAD));
@@ -551,7 +743,8 @@ void ELFDumper<ELFT>::printFileHeaders() {
     W.printHex   ("SectionHeaderOffset", Header->e_shoff);
     if (Header->e_machine == EM_MIPS)
       W.printFlags("Flags", Header->e_flags, makeArrayRef(ElfHeaderMipsFlags),
-                   unsigned(ELF::EF_MIPS_ARCH));
+                   unsigned(ELF::EF_MIPS_ARCH), unsigned(ELF::EF_MIPS_ABI),
+                   unsigned(ELF::EF_MIPS_MACH));
     else
       W.printFlags("Flags", Header->e_flags);
     W.printNumber("HeaderSize", Header->e_ehsize);
@@ -568,46 +761,49 @@ void ELFDumper<ELFT>::printSections() {
   ListScope SectionsD(W, "Sections");
 
   int SectionIndex = -1;
-  for (typename ELFO::Elf_Shdr_Iter SecI = Obj->begin_sections(),
-                                    SecE = Obj->end_sections();
-       SecI != SecE; ++SecI) {
+  for (const Elf_Shdr &Sec : Obj->sections()) {
     ++SectionIndex;
 
-    const Elf_Shdr *Section = &*SecI;
-    StringRef Name = errorOrDefault(Obj->getSectionName(Section));
+    StringRef Name = errorOrDefault(Obj->getSectionName(&Sec));
 
     DictScope SectionD(W, "Section");
     W.printNumber("Index", SectionIndex);
-    W.printNumber("Name", Name, Section->sh_name);
+    W.printNumber("Name", Name, Sec.sh_name);
     W.printHex("Type",
-               getElfSectionType(Obj->getHeader()->e_machine, Section->sh_type),
-               Section->sh_type);
-    W.printFlags ("Flags", Section->sh_flags, makeArrayRef(ElfSectionFlags));
-    W.printHex   ("Address", Section->sh_addr);
-    W.printHex   ("Offset", Section->sh_offset);
-    W.printNumber("Size", Section->sh_size);
-    W.printNumber("Link", Section->sh_link);
-    W.printNumber("Info", Section->sh_info);
-    W.printNumber("AddressAlignment", Section->sh_addralign);
-    W.printNumber("EntrySize", Section->sh_entsize);
+               getElfSectionType(Obj->getHeader()->e_machine, Sec.sh_type),
+               Sec.sh_type);
+    W.printFlags("Flags", Sec.sh_flags, makeArrayRef(ElfSectionFlags));
+    W.printHex("Address", Sec.sh_addr);
+    W.printHex("Offset", Sec.sh_offset);
+    W.printNumber("Size", Sec.sh_size);
+    W.printNumber("Link", Sec.sh_link);
+    W.printNumber("Info", Sec.sh_info);
+    W.printNumber("AddressAlignment", Sec.sh_addralign);
+    W.printNumber("EntrySize", Sec.sh_entsize);
 
     if (opts::SectionRelocations) {
       ListScope D(W, "Relocations");
-      printRelocations(Section);
+      printRelocations(&Sec);
     }
 
     if (opts::SectionSymbols) {
       ListScope D(W, "Symbols");
-      for (typename ELFO::Elf_Sym_Iter SymI = Obj->begin_symbols(),
-                                       SymE = Obj->end_symbols();
-           SymI != SymE; ++SymI) {
-        if (Obj->getSection(&*SymI) == Section)
-          printSymbol(SymI);
+      const Elf_Shdr *Symtab = Obj->getDotSymtabSec();
+      ErrorOr<StringRef> StrTableOrErr = Obj->getStringTableForSymtab(*Symtab);
+      error(StrTableOrErr.getError());
+      StringRef StrTable = *StrTableOrErr;
+
+      for (const Elf_Sym &Sym : Obj->symbols()) {
+        ErrorOr<const Elf_Shdr *> SymSec = Obj->getSection(&Sym);
+        if (!SymSec)
+          continue;
+        if (*SymSec == &Sec)
+          printSymbol(&Sym, StrTable, false);
       }
     }
 
-    if (opts::SectionData && Section->sh_type != ELF::SHT_NOBITS) {
-      ArrayRef<uint8_t> Data = errorOrDefault(Obj->getSectionContents(Section));
+    if (opts::SectionData && Sec.sh_type != ELF::SHT_NOBITS) {
+      ArrayRef<uint8_t> Data = errorOrDefault(Obj->getSectionContents(&Sec));
       W.printBinaryBlock("SectionData",
                          StringRef((const char *)Data.data(), Data.size()));
     }
@@ -619,105 +815,139 @@ void ELFDumper<ELFT>::printRelocations() {
   ListScope D(W, "Relocations");
 
   int SectionNumber = -1;
-  for (typename ELFO::Elf_Shdr_Iter SecI = Obj->begin_sections(),
-                                    SecE = Obj->end_sections();
-       SecI != SecE; ++SecI) {
+  for (const Elf_Shdr &Sec : Obj->sections()) {
     ++SectionNumber;
 
-    if (SecI->sh_type != ELF::SHT_REL && SecI->sh_type != ELF::SHT_RELA)
+    if (Sec.sh_type != ELF::SHT_REL && Sec.sh_type != ELF::SHT_RELA)
       continue;
 
-    StringRef Name = errorOrDefault(Obj->getSectionName(&*SecI));
+    StringRef Name = errorOrDefault(Obj->getSectionName(&Sec));
 
     W.startLine() << "Section (" << SectionNumber << ") " << Name << " {\n";
     W.indent();
 
-    printRelocations(&*SecI);
+    printRelocations(&Sec);
 
     W.unindent();
     W.startLine() << "}\n";
   }
 }
 
+template<class ELFT>
+void ELFDumper<ELFT>::printDynamicRelocations() {
+  W.startLine() << "Dynamic Relocations {\n";
+  W.indent();
+  for (const Elf_Rela &Rel : dyn_relas()) {
+    SmallString<32> RelocName;
+    Obj->getRelocationTypeName(Rel.getType(Obj->isMips64EL()), RelocName);
+    StringRef SymbolName;
+    uint32_t SymIndex = Rel.getSymbol(Obj->isMips64EL());
+    const Elf_Sym *Sym = DynSymStart + SymIndex;
+    SymbolName = errorOrDefault(Sym->getName(DynamicStringTable));
+    if (opts::ExpandRelocs) {
+      DictScope Group(W, "Relocation");
+      W.printHex("Offset", Rel.r_offset);
+      W.printNumber("Type", RelocName, (int)Rel.getType(Obj->isMips64EL()));
+      W.printString("Symbol", SymbolName.size() > 0 ? SymbolName : "-");
+      W.printHex("Addend", Rel.r_addend);
+    }
+    else {
+      raw_ostream& OS = W.startLine();
+      OS << W.hex(Rel.r_offset) << " " << RelocName << " "
+         << (SymbolName.size() > 0 ? SymbolName : "-") << " "
+         << W.hex(Rel.r_addend) << "\n";
+    }
+  }
+  W.unindent();
+  W.startLine() << "}\n";
+}
+
 template <class ELFT>
 void ELFDumper<ELFT>::printRelocations(const Elf_Shdr *Sec) {
   switch (Sec->sh_type) {
   case ELF::SHT_REL:
-    for (typename ELFO::Elf_Rel_Iter RI = Obj->begin_rel(Sec),
-                                     RE = Obj->end_rel(Sec);
-         RI != RE; ++RI) {
-      typename ELFO::Elf_Rela Rela;
-      Rela.r_offset = RI->r_offset;
-      Rela.r_info = RI->r_info;
+    for (const Elf_Rel &R : Obj->rels(Sec)) {
+      Elf_Rela Rela;
+      Rela.r_offset = R.r_offset;
+      Rela.r_info = R.r_info;
       Rela.r_addend = 0;
       printRelocation(Sec, Rela);
     }
     break;
   case ELF::SHT_RELA:
-    for (typename ELFO::Elf_Rela_Iter RI = Obj->begin_rela(Sec),
-                                      RE = Obj->end_rela(Sec);
-         RI != RE; ++RI) {
-      printRelocation(Sec, *RI);
-    }
+    for (const Elf_Rela &R : Obj->relas(Sec))
+      printRelocation(Sec, R);
     break;
   }
 }
 
 template <class ELFT>
-void ELFDumper<ELFT>::printRelocation(const Elf_Shdr *Sec,
-                                      typename ELFO::Elf_Rela Rel) {
+void ELFDumper<ELFT>::printRelocation(const Elf_Shdr *Sec, Elf_Rela Rel) {
   SmallString<32> RelocName;
   Obj->getRelocationTypeName(Rel.getType(Obj->isMips64EL()), RelocName);
-  StringRef SymbolName;
+  StringRef TargetName;
   std::pair<const Elf_Shdr *, const Elf_Sym *> Sym =
       Obj->getRelocationSymbol(Sec, &Rel);
-  if (Sym.first)
-    SymbolName = errorOrDefault(Obj->getSymbolName(Sym.first, Sym.second));
+  if (Sym.second && Sym.second->getType() == ELF::STT_SECTION) {
+    ErrorOr<const Elf_Shdr *> Sec = Obj->getSection(Sym.second);
+    error(Sec.getError());
+    ErrorOr<StringRef> SecName = Obj->getSectionName(*Sec);
+    if (SecName)
+      TargetName = SecName.get();
+  } else if (Sym.first) {
+    const Elf_Shdr *SymTable = Sym.first;
+    ErrorOr<StringRef> StrTableOrErr = Obj->getStringTableForSymtab(*SymTable);
+    error(StrTableOrErr.getError());
+    TargetName = errorOrDefault(Sym.second->getName(*StrTableOrErr));
+  }
 
   if (opts::ExpandRelocs) {
     DictScope Group(W, "Relocation");
     W.printHex("Offset", Rel.r_offset);
     W.printNumber("Type", RelocName, (int)Rel.getType(Obj->isMips64EL()));
-    W.printNumber("Symbol", SymbolName.size() > 0 ? SymbolName : "-",
+    W.printNumber("Symbol", TargetName.size() > 0 ? TargetName : "-",
                   Rel.getSymbol(Obj->isMips64EL()));
     W.printHex("Addend", Rel.r_addend);
   } else {
     raw_ostream& OS = W.startLine();
-    OS << W.hex(Rel.r_offset)
-       << " " << RelocName
-       << " " << (SymbolName.size() > 0 ? SymbolName : "-")
-       << " " << W.hex(Rel.r_addend)
-       << "\n";
+    OS << W.hex(Rel.r_offset) << " " << RelocName << " "
+       << (TargetName.size() > 0 ? TargetName : "-") << " "
+       << W.hex(Rel.r_addend) << "\n";
   }
 }
 
 template<class ELFT>
 void ELFDumper<ELFT>::printSymbols() {
   ListScope Group(W, "Symbols");
-  for (typename ELFO::Elf_Sym_Iter SymI = Obj->begin_symbols(),
-                                   SymE = Obj->end_symbols();
-       SymI != SymE; ++SymI) {
-    printSymbol(SymI);
-  }
+
+  const Elf_Shdr *Symtab = Obj->getDotSymtabSec();
+  ErrorOr<StringRef> StrTableOrErr = Obj->getStringTableForSymtab(*Symtab);
+  error(StrTableOrErr.getError());
+  StringRef StrTable = *StrTableOrErr;
+  for (const Elf_Sym &Sym : Obj->symbols())
+    printSymbol(&Sym, StrTable, false);
 }
 
 template<class ELFT>
 void ELFDumper<ELFT>::printDynamicSymbols() {
   ListScope Group(W, "DynamicSymbols");
 
-  for (typename ELFO::Elf_Sym_Iter SymI = Obj->begin_dynamic_symbols(),
-                                   SymE = Obj->end_dynamic_symbols();
-       SymI != SymE; ++SymI) {
-    printSymbol(SymI);
-  }
+  const Elf_Shdr *Symtab = Obj->getDotDynSymSec();
+  ErrorOr<StringRef> StrTableOrErr = Obj->getStringTableForSymtab(*Symtab);
+  error(StrTableOrErr.getError());
+  StringRef StrTable = *StrTableOrErr;
+  for (const Elf_Sym &Sym : Obj->symbols(Symtab))
+    printSymbol(&Sym, StrTable, true);
 }
 
 template <class ELFT>
-void ELFDumper<ELFT>::printSymbol(typename ELFO::Elf_Sym_Iter Symbol) {
+void ELFDumper<ELFT>::printSymbol(const Elf_Sym *Symbol, StringRef StrTable,
+                                  bool IsDynamic) {
   unsigned SectionIndex = 0;
   StringRef SectionName;
   getSectionNameIndex(*Obj, Symbol, SectionName, SectionIndex);
-  std::string FullSymbolName = getFullSymbolName(*Obj, Symbol);
+  std::string FullSymbolName =
+      getFullSymbolName(*Obj, Symbol, StrTable, IsDynamic);
 
   DictScope D(W, "Symbol");
   W.printNumber("Name", FullSymbolName, Symbol->st_name);
@@ -741,6 +971,7 @@ static const char *getTypeString(uint64_t Type) {
   LLVM_READOBJ_TYPE_CASE(FINI_ARRAY);
   LLVM_READOBJ_TYPE_CASE(FINI_ARRAYSZ);
   LLVM_READOBJ_TYPE_CASE(FLAGS);
+  LLVM_READOBJ_TYPE_CASE(FLAGS_1);
   LLVM_READOBJ_TYPE_CASE(HASH);
   LLVM_READOBJ_TYPE_CASE(INIT);
   LLVM_READOBJ_TYPE_CASE(INIT_ARRAY);
@@ -782,6 +1013,7 @@ static const char *getTypeString(uint64_t Type) {
   LLVM_READOBJ_TYPE_CASE(MIPS_GOTSYM);
   LLVM_READOBJ_TYPE_CASE(MIPS_RLD_MAP);
   LLVM_READOBJ_TYPE_CASE(MIPS_PLTGOT);
+  LLVM_READOBJ_TYPE_CASE(MIPS_OPTIONS);
   default: return "unknown";
   }
 }
@@ -799,6 +1031,34 @@ static const EnumEntry<unsigned> ElfDynamicDTFlags[] = {
   LLVM_READOBJ_DT_FLAG_ENT(DF, STATIC_TLS)
 };
 
+static const EnumEntry<unsigned> ElfDynamicDTFlags1[] = {
+  LLVM_READOBJ_DT_FLAG_ENT(DF_1, NOW),
+  LLVM_READOBJ_DT_FLAG_ENT(DF_1, GLOBAL),
+  LLVM_READOBJ_DT_FLAG_ENT(DF_1, GROUP),
+  LLVM_READOBJ_DT_FLAG_ENT(DF_1, NODELETE),
+  LLVM_READOBJ_DT_FLAG_ENT(DF_1, LOADFLTR),
+  LLVM_READOBJ_DT_FLAG_ENT(DF_1, INITFIRST),
+  LLVM_READOBJ_DT_FLAG_ENT(DF_1, NOOPEN),
+  LLVM_READOBJ_DT_FLAG_ENT(DF_1, ORIGIN),
+  LLVM_READOBJ_DT_FLAG_ENT(DF_1, DIRECT),
+  LLVM_READOBJ_DT_FLAG_ENT(DF_1, TRANS),
+  LLVM_READOBJ_DT_FLAG_ENT(DF_1, INTERPOSE),
+  LLVM_READOBJ_DT_FLAG_ENT(DF_1, NODEFLIB),
+  LLVM_READOBJ_DT_FLAG_ENT(DF_1, NODUMP),
+  LLVM_READOBJ_DT_FLAG_ENT(DF_1, CONFALT),
+  LLVM_READOBJ_DT_FLAG_ENT(DF_1, ENDFILTEE),
+  LLVM_READOBJ_DT_FLAG_ENT(DF_1, DISPRELDNE),
+  LLVM_READOBJ_DT_FLAG_ENT(DF_1, NODIRECT),
+  LLVM_READOBJ_DT_FLAG_ENT(DF_1, IGNMULDEF),
+  LLVM_READOBJ_DT_FLAG_ENT(DF_1, NOKSYMS),
+  LLVM_READOBJ_DT_FLAG_ENT(DF_1, NOHDR),
+  LLVM_READOBJ_DT_FLAG_ENT(DF_1, EDITED),
+  LLVM_READOBJ_DT_FLAG_ENT(DF_1, NORELOC),
+  LLVM_READOBJ_DT_FLAG_ENT(DF_1, SYMINTPOSE),
+  LLVM_READOBJ_DT_FLAG_ENT(DF_1, GLOBAUDIT),
+  LLVM_READOBJ_DT_FLAG_ENT(DF_1, SINGLETON)
+};
+
 static const EnumEntry<unsigned> ElfDynamicDTMipsFlags[] = {
   LLVM_READOBJ_DT_FLAG_ENT(RHF, NONE),
   LLVM_READOBJ_DT_FLAG_ENT(RHF, QUICKSTART),
@@ -840,8 +1100,15 @@ void printFlags(T Value, ArrayRef<EnumEntry<TFlag>> Flags, raw_ostream &OS) {
 }
 
 template <class ELFT>
-static void printValue(const ELFFile<ELFT> *O, uint64_t Type, uint64_t Value,
-                       bool Is64, raw_ostream &OS) {
+StringRef ELFDumper<ELFT>::getDynamicString(uint64_t Value) const {
+  if (Value >= DynamicStringTable.size())
+    reportError("Invalid dynamic string table reference");
+  return StringRef(DynamicStringTable.data() + Value);
+}
+
+template <class ELFT>
+void ELFDumper<ELFT>::printValue(uint64_t Type, uint64_t Value) {
+  raw_ostream &OS = W.getOStream();
   switch (Type) {
   case DT_PLTREL:
     if (Value == DT_REL) {
@@ -873,6 +1140,7 @@ static void printValue(const ELFFile<ELFT> *O, uint64_t Type, uint64_t Value,
   case DT_MIPS_GOTSYM:
   case DT_MIPS_RLD_MAP:
   case DT_MIPS_PLTGOT:
+  case DT_MIPS_OPTIONS:
     OS << format("0x%" PRIX64, Value);
     break;
   case DT_RELCOUNT:
@@ -896,14 +1164,14 @@ static void printValue(const ELFFile<ELFT> *O, uint64_t Type, uint64_t Value,
     OS << Value << " (bytes)";
     break;
   case DT_NEEDED:
-    OS << "SharedLibrary (" << O->getDynamicString(Value) << ")";
+    OS << "SharedLibrary (" << getDynamicString(Value) << ")";
     break;
   case DT_SONAME:
-    OS << "LibrarySoname (" << O->getDynamicString(Value) << ")";
+    OS << "LibrarySoname (" << getDynamicString(Value) << ")";
     break;
   case DT_RPATH:
   case DT_RUNPATH:
-    OS << O->getDynamicString(Value);
+    OS << getDynamicString(Value);
     break;
   case DT_MIPS_FLAGS:
     printFlags(Value, makeArrayRef(ElfDynamicDTMipsFlags), OS);
@@ -911,6 +1179,12 @@ static void printValue(const ELFFile<ELFT> *O, uint64_t Type, uint64_t Value,
   case DT_FLAGS:
     printFlags(Value, makeArrayRef(ElfDynamicDTFlags), OS);
     break;
+  case DT_FLAGS_1:
+    printFlags(Value, makeArrayRef(ElfDynamicDTFlags1), OS);
+    break;
+  default:
+    OS << format("0x%" PRIX64, Value);
+    break;
   }
 }
 
@@ -920,11 +1194,10 @@ void ELFDumper<ELFT>::printUnwindInfo() {
 }
 
 namespace {
-template <>
-void ELFDumper<ELFType<support::little, 2, false> >::printUnwindInfo() {
+template <> void ELFDumper<ELFType<support::little, false>>::printUnwindInfo() {
   const unsigned Machine = Obj->getHeader()->e_machine;
   if (Machine == EM_ARM) {
-    ARM::EHABI::PrinterContext<ELFType<support::little, 2, false> > Ctx(W, Obj);
+    ARM::EHABI::PrinterContext<ELFType<support::little, false>> Ctx(W, Obj);
     return Ctx.PrintUnwindInformation();
   }
   W.startLine() << "UnwindInfo not implemented.\n";
@@ -933,9 +1206,20 @@ void ELFDumper<ELFType<support::little, 2, false> >::printUnwindInfo() {
 
 template<class ELFT>
 void ELFDumper<ELFT>::printDynamicTable() {
-  auto DynTable = Obj->dynamic_table(true);
+  auto I = dynamic_table_begin();
+  auto E = dynamic_table_end();
+
+  if (I == E)
+    return;
+
+  --E;
+  while (I != E && E->getTag() == ELF::DT_NULL)
+    --E;
+  if (E->getTag() != ELF::DT_NULL)
+    ++E;
+  ++E;
 
-  ptrdiff_t Total = std::distance(DynTable.begin(), DynTable.end());
+  ptrdiff_t Total = std::distance(I, E);
   if (Total == 0)
     return;
 
@@ -947,12 +1231,14 @@ void ELFDumper<ELFT>::printDynamicTable() {
   W.startLine()
      << "  Tag" << (Is64 ? "                " : "        ") << "Type"
      << "                 " << "Name/Value\n";
-  for (const auto &Entry : DynTable) {
+  while (I != E) {
+    const Elf_Dyn &Entry = *I;
+    ++I;
     W.startLine()
        << "  "
        << format(Is64 ? "0x%016" PRIX64 : "0x%08" PRIX64, Entry.getTag())
        << " " << format("%-21s", getTypeString(Entry.getTag()));
-    printValue(Obj, Entry.getTag(), Entry.getVal(), Is64, OS);
+    printValue(Entry.getTag(), Entry.getVal());
     OS << "\n";
   }
 
@@ -966,9 +1252,9 @@ void ELFDumper<ELFT>::printNeededLibraries() {
   typedef std::vector<StringRef> LibsTy;
   LibsTy Libs;
 
-  for (const auto &Entry : Obj->dynamic_table())
+  for (const auto &Entry : dynamic_table())
     if (Entry.d_tag == ELF::DT_NEEDED)
-      Libs.push_back(Obj->getDynamicString(Entry.d_un.d_val));
+      Libs.push_back(getDynamicString(Entry.d_un.d_val));
 
   std::stable_sort(Libs.begin(), Libs.end());
 
@@ -981,43 +1267,54 @@ template<class ELFT>
 void ELFDumper<ELFT>::printProgramHeaders() {
   ListScope L(W, "ProgramHeaders");
 
-  for (typename ELFO::Elf_Phdr_Iter PI = Obj->begin_program_headers(),
-                                    PE = Obj->end_program_headers();
-                                    PI != PE; ++PI) {
+  for (const Elf_Phdr &Phdr : Obj->program_headers()) {
     DictScope P(W, "ProgramHeader");
-    W.printHex   ("Type",
-                  getElfSegmentType(Obj->getHeader()->e_machine, PI->p_type),
-                  PI->p_type);
-    W.printHex   ("Offset", PI->p_offset);
-    W.printHex   ("VirtualAddress", PI->p_vaddr);
-    W.printHex   ("PhysicalAddress", PI->p_paddr);
-    W.printNumber("FileSize", PI->p_filesz);
-    W.printNumber("MemSize", PI->p_memsz);
-    W.printFlags ("Flags", PI->p_flags, makeArrayRef(ElfSegmentFlags));
-    W.printNumber("Alignment", PI->p_align);
+    W.printHex("Type",
+               getElfSegmentType(Obj->getHeader()->e_machine, Phdr.p_type),
+               Phdr.p_type);
+    W.printHex("Offset", Phdr.p_offset);
+    W.printHex("VirtualAddress", Phdr.p_vaddr);
+    W.printHex("PhysicalAddress", Phdr.p_paddr);
+    W.printNumber("FileSize", Phdr.p_filesz);
+    W.printNumber("MemSize", Phdr.p_memsz);
+    W.printFlags("Flags", Phdr.p_flags, makeArrayRef(ElfSegmentFlags));
+    W.printNumber("Alignment", Phdr.p_align);
   }
 }
 
+template <typename ELFT>
+void ELFDumper<ELFT>::printHashTable() {
+  DictScope D(W, "HashTable");
+  if (!HashTable)
+    return;
+  W.printNumber("Num Buckets", HashTable->nbucket);
+  W.printNumber("Num Chains", HashTable->nchain);
+  W.printList("Buckets", HashTable->buckets());
+  W.printList("Chains", HashTable->chains());
+}
+
+template <typename ELFT> void ELFDumper<ELFT>::printLoadName() {
+  outs() << "LoadName: " << SOName << '\n';
+}
+
 template <class ELFT>
 void ELFDumper<ELFT>::printAttributes() {
   W.startLine() << "Attributes not implemented.\n";
 }
 
 namespace {
-template <>
-void ELFDumper<ELFType<support::little, 2, false> >::printAttributes() {
+template <> void ELFDumper<ELFType<support::little, false>>::printAttributes() {
   if (Obj->getHeader()->e_machine != EM_ARM) {
     W.startLine() << "Attributes not implemented.\n";
     return;
   }
 
   DictScope BA(W, "BuildAttributes");
-  for (ELFO::Elf_Shdr_Iter SI = Obj->begin_sections(), SE = Obj->end_sections();
-       SI != SE; ++SI) {
-    if (SI->sh_type != ELF::SHT_ARM_ATTRIBUTES)
+  for (const ELFO::Elf_Shdr &Sec : Obj->sections()) {
+    if (Sec.sh_type != ELF::SHT_ARM_ATTRIBUTES)
       continue;
 
-    ErrorOr<ArrayRef<uint8_t> > Contents = Obj->getSectionContents(&(*SI));
+    ErrorOr<ArrayRef<uint8_t>> Contents = Obj->getSectionContents(&Sec);
     if (!Contents)
       continue;
 
@@ -1039,89 +1336,141 @@ void ELFDumper<ELFType<support::little, 2, false> >::printAttributes() {
 namespace {
 template <class ELFT> class MipsGOTParser {
 public:
-  typedef object::ELFFile<ELFT> ObjectFile;
-  typedef typename ObjectFile::Elf_Shdr Elf_Shdr;
+  typedef object::ELFFile<ELFT> ELFO;
+  typedef typename ELFO::Elf_Shdr Elf_Shdr;
+  typedef typename ELFO::Elf_Sym Elf_Sym;
+  typedef typename ELFO::Elf_Dyn_Range Elf_Dyn_Range;
+  typedef typename ELFO::Elf_Addr GOTEntry;
+  typedef typename ELFO::Elf_Rel Elf_Rel;
+  typedef typename ELFO::Elf_Rela Elf_Rela;
 
-  MipsGOTParser(const ObjectFile *Obj, StreamWriter &W) : Obj(Obj), W(W) {}
+  MipsGOTParser(const ELFO *Obj, Elf_Dyn_Range DynTable, StreamWriter &W);
 
-  void parseGOT(const Elf_Shdr &GOTShdr);
+  void parseGOT();
+  void parsePLT();
 
 private:
-  typedef typename ObjectFile::Elf_Sym_Iter Elf_Sym_Iter;
-  typedef typename ObjectFile::Elf_Addr GOTEntry;
-  typedef typename ObjectFile::template ELFEntityIterator<const GOTEntry>
-  GOTIter;
-
-  const ObjectFile *Obj;
+  const ELFO *Obj;
   StreamWriter &W;
+  llvm::Optional<uint64_t> DtPltGot;
+  llvm::Optional<uint64_t> DtLocalGotNum;
+  llvm::Optional<uint64_t> DtGotSym;
+  llvm::Optional<uint64_t> DtMipsPltGot;
+  llvm::Optional<uint64_t> DtJmpRel;
 
   std::size_t getGOTTotal(ArrayRef<uint8_t> GOT) const;
-  GOTIter makeGOTIter(ArrayRef<uint8_t> GOT, std::size_t EntryNum);
-
-  bool getGOTTags(uint64_t &LocalGotNum, uint64_t &GotSym);
-  void printGotEntry(uint64_t GotAddr, GOTIter BeginIt, GOTIter It);
-  void printGlobalGotEntry(uint64_t GotAddr, GOTIter BeginIt, GOTIter It,
-                           Elf_Sym_Iter Sym);
+  const GOTEntry *makeGOTIter(ArrayRef<uint8_t> GOT, std::size_t EntryNum);
+
+  void printGotEntry(uint64_t GotAddr, const GOTEntry *BeginIt,
+                     const GOTEntry *It);
+  void printGlobalGotEntry(uint64_t GotAddr, const GOTEntry *BeginIt,
+                           const GOTEntry *It, const Elf_Sym *Sym,
+                           StringRef StrTable, bool IsDynamic);
+  void printPLTEntry(uint64_t PLTAddr, const GOTEntry *BeginIt,
+                     const GOTEntry *It, StringRef Purpose);
+  void printPLTEntry(uint64_t PLTAddr, const GOTEntry *BeginIt,
+                     const GOTEntry *It, StringRef StrTable,
+                     const Elf_Sym *Sym);
 };
 }
 
 template <class ELFT>
-void MipsGOTParser<ELFT>::parseGOT(const Elf_Shdr &GOTShdr) {
+MipsGOTParser<ELFT>::MipsGOTParser(const ELFO *Obj, Elf_Dyn_Range DynTable,
+                                   StreamWriter &W)
+    : Obj(Obj), W(W) {
+  for (const auto &Entry : DynTable) {
+    switch (Entry.getTag()) {
+    case ELF::DT_PLTGOT:
+      DtPltGot = Entry.getVal();
+      break;
+    case ELF::DT_MIPS_LOCAL_GOTNO:
+      DtLocalGotNum = Entry.getVal();
+      break;
+    case ELF::DT_MIPS_GOTSYM:
+      DtGotSym = Entry.getVal();
+      break;
+    case ELF::DT_MIPS_PLTGOT:
+      DtMipsPltGot = Entry.getVal();
+      break;
+    case ELF::DT_JMPREL:
+      DtJmpRel = Entry.getVal();
+      break;
+    }
+  }
+}
+
+template <class ELFT> void MipsGOTParser<ELFT>::parseGOT() {
   // See "Global Offset Table" in Chapter 5 in the following document
   // for detailed GOT description.
   // ftp://www.linux-mips.org/pub/linux/mips/doc/ABI/mipsabi.pdf
+  if (!DtPltGot) {
+    W.startLine() << "Cannot find PLTGOT dynamic table tag.\n";
+    return;
+  }
+  if (!DtLocalGotNum) {
+    W.startLine() << "Cannot find MIPS_LOCAL_GOTNO dynamic table tag.\n";
+    return;
+  }
+  if (!DtGotSym) {
+    W.startLine() << "Cannot find MIPS_GOTSYM dynamic table tag.\n";
+    return;
+  }
 
-  ErrorOr<ArrayRef<uint8_t>> GOT = Obj->getSectionContents(&GOTShdr);
-  if (!GOT) {
-    W.startLine() << "The .got section is empty.\n";
+  const Elf_Shdr *GOTShdr = findSectionByAddress(Obj, *DtPltGot);
+  if (!GOTShdr) {
+    W.startLine() << "There is no .got section in the file.\n";
     return;
   }
 
-  uint64_t DtLocalGotNum;
-  uint64_t DtGotSym;
-  if (!getGOTTags(DtLocalGotNum, DtGotSym))
+  ErrorOr<ArrayRef<uint8_t>> GOT = Obj->getSectionContents(GOTShdr);
+  if (!GOT) {
+    W.startLine() << "The .got section is empty.\n";
     return;
+  }
 
-  if (DtLocalGotNum > getGOTTotal(*GOT)) {
+  if (*DtLocalGotNum > getGOTTotal(*GOT)) {
     W.startLine() << "MIPS_LOCAL_GOTNO exceeds a number of GOT entries.\n";
     return;
   }
 
-  Elf_Sym_Iter DynSymBegin = Obj->begin_dynamic_symbols();
-  Elf_Sym_Iter DynSymEnd = Obj->end_dynamic_symbols();
+  const Elf_Shdr *DynSymSec = Obj->getDotDynSymSec();
+  ErrorOr<StringRef> StrTable = Obj->getStringTableForSymtab(*DynSymSec);
+  error(StrTable.getError());
+  const Elf_Sym *DynSymBegin = Obj->symbol_begin(DynSymSec);
+  const Elf_Sym *DynSymEnd = Obj->symbol_end(DynSymSec);
   std::size_t DynSymTotal = std::size_t(std::distance(DynSymBegin, DynSymEnd));
 
-  if (DtGotSym > DynSymTotal) {
+  if (*DtGotSym > DynSymTotal) {
     W.startLine() << "MIPS_GOTSYM exceeds a number of dynamic symbols.\n";
     return;
   }
 
-  std::size_t GlobalGotNum = DynSymTotal - DtGotSym;
+  std::size_t GlobalGotNum = DynSymTotal - *DtGotSym;
 
-  if (DtLocalGotNum + GlobalGotNum > getGOTTotal(*GOT)) {
+  if (*DtLocalGotNum + GlobalGotNum > getGOTTotal(*GOT)) {
     W.startLine() << "Number of global GOT entries exceeds the size of GOT.\n";
     return;
   }
 
-  GOTIter GotBegin = makeGOTIter(*GOT, 0);
-  GOTIter GotLocalEnd = makeGOTIter(*GOT, DtLocalGotNum);
-  GOTIter It = GotBegin;
+  const GOTEntry *GotBegin = makeGOTIter(*GOT, 0);
+  const GOTEntry *GotLocalEnd = makeGOTIter(*GOT, *DtLocalGotNum);
+  const GOTEntry *It = GotBegin;
 
   DictScope GS(W, "Primary GOT");
 
-  W.printHex("Canonical gp value", GOTShdr.sh_addr + 0x7ff0);
+  W.printHex("Canonical gp value", GOTShdr->sh_addr + 0x7ff0);
   {
     ListScope RS(W, "Reserved entries");
 
     {
       DictScope D(W, "Entry");
-      printGotEntry(GOTShdr.sh_addr, GotBegin, It++);
+      printGotEntry(GOTShdr->sh_addr, GotBegin, It++);
       W.printString("Purpose", StringRef("Lazy resolver"));
     }
 
     if (It != GotLocalEnd && (*It >> (sizeof(GOTEntry) * 8 - 1)) != 0) {
       DictScope D(W, "Entry");
-      printGotEntry(GOTShdr.sh_addr, GotBegin, It++);
+      printGotEntry(GOTShdr->sh_addr, GotBegin, It++);
       W.printString("Purpose", StringRef("Module pointer (GNU extension)"));
     }
   }
@@ -1129,80 +1478,156 @@ void MipsGOTParser<ELFT>::parseGOT(const Elf_Shdr &GOTShdr) {
     ListScope LS(W, "Local entries");
     for (; It != GotLocalEnd; ++It) {
       DictScope D(W, "Entry");
-      printGotEntry(GOTShdr.sh_addr, GotBegin, It);
+      printGotEntry(GOTShdr->sh_addr, GotBegin, It);
     }
   }
   {
     ListScope GS(W, "Global entries");
 
-    GOTIter GotGlobalEnd = makeGOTIter(*GOT, DtLocalGotNum + GlobalGotNum);
-    Elf_Sym_Iter GotDynSym = DynSymBegin + DtGotSym;
+    const GOTEntry *GotGlobalEnd =
+        makeGOTIter(*GOT, *DtLocalGotNum + GlobalGotNum);
+    const Elf_Sym *GotDynSym = DynSymBegin + *DtGotSym;
     for (; It != GotGlobalEnd; ++It) {
       DictScope D(W, "Entry");
-      printGlobalGotEntry(GOTShdr.sh_addr, GotBegin, It, GotDynSym++);
+      printGlobalGotEntry(GOTShdr->sh_addr, GotBegin, It, GotDynSym++,
+                          *StrTable, true);
     }
   }
 
-  std::size_t SpecGotNum = getGOTTotal(*GOT) - DtLocalGotNum - GlobalGotNum;
+  std::size_t SpecGotNum = getGOTTotal(*GOT) - *DtLocalGotNum - GlobalGotNum;
   W.printNumber("Number of TLS and multi-GOT entries", uint64_t(SpecGotNum));
 }
 
+template <class ELFT> void MipsGOTParser<ELFT>::parsePLT() {
+  if (!DtMipsPltGot) {
+    W.startLine() << "Cannot find MIPS_PLTGOT dynamic table tag.\n";
+    return;
+  }
+  if (!DtJmpRel) {
+    W.startLine() << "Cannot find JMPREL dynamic table tag.\n";
+    return;
+  }
+
+  const Elf_Shdr *PLTShdr = findSectionByAddress(Obj, *DtMipsPltGot);
+  if (!PLTShdr) {
+    W.startLine() << "There is no .got.plt section in the file.\n";
+    return;
+  }
+  ErrorOr<ArrayRef<uint8_t>> PLT = Obj->getSectionContents(PLTShdr);
+  if (!PLT) {
+    W.startLine() << "The .got.plt section is empty.\n";
+    return;
+  }
+
+  const Elf_Shdr *PLTRelShdr = findSectionByAddress(Obj, *DtJmpRel);
+  if (!PLTShdr) {
+    W.startLine() << "There is no .rel.plt section in the file.\n";
+    return;
+  }
+  ErrorOr<const Elf_Shdr *> SymTableOrErr =
+      Obj->getSection(PLTRelShdr->sh_link);
+  error(SymTableOrErr.getError());
+  ErrorOr<StringRef> StrTable = Obj->getStringTableForSymtab(**SymTableOrErr);
+  error(StrTable.getError());
+
+  const GOTEntry *PLTBegin = makeGOTIter(*PLT, 0);
+  const GOTEntry *PLTEnd = makeGOTIter(*PLT, getGOTTotal(*PLT));
+  const GOTEntry *It = PLTBegin;
+
+  DictScope GS(W, "PLT GOT");
+  {
+    ListScope RS(W, "Reserved entries");
+    printPLTEntry(PLTShdr->sh_addr, PLTBegin, It++, "PLT lazy resolver");
+    if (It != PLTEnd)
+      printPLTEntry(PLTShdr->sh_addr, PLTBegin, It++, "Module pointer");
+  }
+  {
+    ListScope GS(W, "Entries");
+
+    switch (PLTRelShdr->sh_type) {
+    case ELF::SHT_REL:
+      for (const Elf_Rel *RI = Obj->rel_begin(PLTRelShdr),
+                         *RE = Obj->rel_end(PLTRelShdr);
+           RI != RE && It != PLTEnd; ++RI, ++It) {
+        const Elf_Sym *Sym =
+            Obj->getRelocationSymbol(&*PLTRelShdr, &*RI).second;
+        printPLTEntry(PLTShdr->sh_addr, PLTBegin, It, *StrTable, Sym);
+      }
+      break;
+    case ELF::SHT_RELA:
+      for (const Elf_Rela *RI = Obj->rela_begin(PLTRelShdr),
+                          *RE = Obj->rela_end(PLTRelShdr);
+           RI != RE && It != PLTEnd; ++RI, ++It) {
+        const Elf_Sym *Sym =
+            Obj->getRelocationSymbol(&*PLTRelShdr, &*RI).second;
+        printPLTEntry(PLTShdr->sh_addr, PLTBegin, It, *StrTable, Sym);
+      }
+      break;
+    }
+  }
+}
+
 template <class ELFT>
 std::size_t MipsGOTParser<ELFT>::getGOTTotal(ArrayRef<uint8_t> GOT) const {
   return GOT.size() / sizeof(GOTEntry);
 }
 
 template <class ELFT>
-typename MipsGOTParser<ELFT>::GOTIter
+const typename MipsGOTParser<ELFT>::GOTEntry *
 MipsGOTParser<ELFT>::makeGOTIter(ArrayRef<uint8_t> GOT, std::size_t EntryNum) {
   const char *Data = reinterpret_cast<const char *>(GOT.data());
-  return GOTIter(sizeof(GOTEntry), Data + EntryNum * sizeof(GOTEntry));
+  return reinterpret_cast<const GOTEntry *>(Data + EntryNum * sizeof(GOTEntry));
 }
 
 template <class ELFT>
-bool MipsGOTParser<ELFT>::getGOTTags(uint64_t &LocalGotNum, uint64_t &GotSym) {
-  bool FoundLocalGotNum = false;
-  bool FoundGotSym = false;
-  for (const auto &Entry : Obj->dynamic_table()) {
-    switch (Entry.getTag()) {
-    case ELF::DT_MIPS_LOCAL_GOTNO:
-      LocalGotNum = Entry.getVal();
-      FoundLocalGotNum = true;
-      break;
-    case ELF::DT_MIPS_GOTSYM:
-      GotSym = Entry.getVal();
-      FoundGotSym = true;
-      break;
-    }
-  }
+void MipsGOTParser<ELFT>::printGotEntry(uint64_t GotAddr,
+                                        const GOTEntry *BeginIt,
+                                        const GOTEntry *It) {
+  int64_t Offset = std::distance(BeginIt, It) * sizeof(GOTEntry);
+  W.printHex("Address", GotAddr + Offset);
+  W.printNumber("Access", Offset - 0x7ff0);
+  W.printHex("Initial", *It);
+}
 
-  if (!FoundLocalGotNum) {
-    W.startLine() << "Cannot find MIPS_LOCAL_GOTNO dynamic table tag.\n";
-    return false;
-  }
+template <class ELFT>
+void MipsGOTParser<ELFT>::printGlobalGotEntry(
+    uint64_t GotAddr, const GOTEntry *BeginIt, const GOTEntry *It,
+    const Elf_Sym *Sym, StringRef StrTable, bool IsDynamic) {
+  printGotEntry(GotAddr, BeginIt, It);
 
-  if (!FoundGotSym) {
-    W.startLine() << "Cannot find MIPS_GOTSYM dynamic table tag.\n";
-    return false;
-  }
+  W.printHex("Value", Sym->st_value);
+  W.printEnum("Type", Sym->getType(), makeArrayRef(ElfSymbolTypes));
+
+  unsigned SectionIndex = 0;
+  StringRef SectionName;
+  getSectionNameIndex(*Obj, Sym, SectionName, SectionIndex);
+  W.printHex("Section", SectionName, SectionIndex);
 
-  return true;
+  std::string FullSymbolName =
+      getFullSymbolName(*Obj, Sym, StrTable, IsDynamic);
+  W.printNumber("Name", FullSymbolName, Sym->st_name);
 }
 
 template <class ELFT>
-void MipsGOTParser<ELFT>::printGotEntry(uint64_t GotAddr, GOTIter BeginIt,
-                                        GOTIter It) {
+void MipsGOTParser<ELFT>::printPLTEntry(uint64_t PLTAddr,
+                                        const GOTEntry *BeginIt,
+                                        const GOTEntry *It, StringRef Purpose) {
+  DictScope D(W, "Entry");
   int64_t Offset = std::distance(BeginIt, It) * sizeof(GOTEntry);
-  W.printHex("Address", GotAddr + Offset);
-  W.printNumber("Access", Offset - 0x7ff0);
+  W.printHex("Address", PLTAddr + Offset);
   W.printHex("Initial", *It);
+  W.printString("Purpose", Purpose);
 }
 
 template <class ELFT>
-void MipsGOTParser<ELFT>::printGlobalGotEntry(uint64_t GotAddr, GOTIter BeginIt,
-                                              GOTIter It, Elf_Sym_Iter Sym) {
-  printGotEntry(GotAddr, BeginIt, It);
-
+void MipsGOTParser<ELFT>::printPLTEntry(uint64_t PLTAddr,
+                                        const GOTEntry *BeginIt,
+                                        const GOTEntry *It, StringRef StrTable,
+                                        const Elf_Sym *Sym) {
+  DictScope D(W, "Entry");
+  int64_t Offset = std::distance(BeginIt, It) * sizeof(GOTEntry);
+  W.printHex("Address", PLTAddr + Offset);
+  W.printHex("Initial", *It);
   W.printHex("Value", Sym->st_value);
   W.printEnum("Type", Sym->getType(), makeArrayRef(ElfSymbolTypes));
 
@@ -1211,7 +1636,7 @@ void MipsGOTParser<ELFT>::printGlobalGotEntry(uint64_t GotAddr, GOTIter BeginIt,
   getSectionNameIndex(*Obj, Sym, SectionName, SectionIndex);
   W.printHex("Section", SectionName, SectionIndex);
 
-  std::string FullSymbolName = getFullSymbolName(*Obj, Sym);
+  std::string FullSymbolName = getFullSymbolName(*Obj, Sym, StrTable, true);
   W.printNumber("Name", FullSymbolName, Sym->st_name);
 }
 
@@ -1221,24 +1646,165 @@ template <class ELFT> void ELFDumper<ELFT>::printMipsPLTGOT() {
     return;
   }
 
-  llvm::Optional<uint64_t> DtPltGot;
-  for (const auto &Entry : Obj->dynamic_table()) {
-    if (Entry.getTag() == ELF::DT_PLTGOT) {
-      DtPltGot = Entry.getVal();
-      break;
-    }
+  MipsGOTParser<ELFT> GOTParser(Obj, dynamic_table(), W);
+  GOTParser.parseGOT();
+  GOTParser.parsePLT();
+}
+
+static const EnumEntry<unsigned> ElfMipsISAExtType[] = {
+  {"None",                    Mips::AFL_EXT_NONE},
+  {"Broadcom SB-1",           Mips::AFL_EXT_SB1},
+  {"Cavium Networks Octeon",  Mips::AFL_EXT_OCTEON},
+  {"Cavium Networks Octeon2", Mips::AFL_EXT_OCTEON2},
+  {"Cavium Networks OcteonP", Mips::AFL_EXT_OCTEONP},
+  {"Cavium Networks Octeon3", Mips::AFL_EXT_OCTEON3},
+  {"LSI R4010",               Mips::AFL_EXT_4010},
+  {"Loongson 2E",             Mips::AFL_EXT_LOONGSON_2E},
+  {"Loongson 2F",             Mips::AFL_EXT_LOONGSON_2F},
+  {"Loongson 3A",             Mips::AFL_EXT_LOONGSON_3A},
+  {"MIPS R4650",              Mips::AFL_EXT_4650},
+  {"MIPS R5900",              Mips::AFL_EXT_5900},
+  {"MIPS R10000",             Mips::AFL_EXT_10000},
+  {"NEC VR4100",              Mips::AFL_EXT_4100},
+  {"NEC VR4111/VR4181",       Mips::AFL_EXT_4111},
+  {"NEC VR4120",              Mips::AFL_EXT_4120},
+  {"NEC VR5400",              Mips::AFL_EXT_5400},
+  {"NEC VR5500",              Mips::AFL_EXT_5500},
+  {"RMI Xlr",                 Mips::AFL_EXT_XLR},
+  {"Toshiba R3900",           Mips::AFL_EXT_3900}
+};
+
+static const EnumEntry<unsigned> ElfMipsASEFlags[] = {
+  {"DSP",                Mips::AFL_ASE_DSP},
+  {"DSPR2",              Mips::AFL_ASE_DSPR2},
+  {"Enhanced VA Scheme", Mips::AFL_ASE_EVA},
+  {"MCU",                Mips::AFL_ASE_MCU},
+  {"MDMX",               Mips::AFL_ASE_MDMX},
+  {"MIPS-3D",            Mips::AFL_ASE_MIPS3D},
+  {"MT",                 Mips::AFL_ASE_MT},
+  {"SmartMIPS",          Mips::AFL_ASE_SMARTMIPS},
+  {"VZ",                 Mips::AFL_ASE_VIRT},
+  {"MSA",                Mips::AFL_ASE_MSA},
+  {"MIPS16",             Mips::AFL_ASE_MIPS16},
+  {"microMIPS",          Mips::AFL_ASE_MICROMIPS},
+  {"XPA",                Mips::AFL_ASE_XPA}
+};
+
+static const EnumEntry<unsigned> ElfMipsFpABIType[] = {
+  {"Hard or soft float",                  Mips::Val_GNU_MIPS_ABI_FP_ANY},
+  {"Hard float (double precision)",       Mips::Val_GNU_MIPS_ABI_FP_DOUBLE},
+  {"Hard float (single precision)",       Mips::Val_GNU_MIPS_ABI_FP_SINGLE},
+  {"Soft float",                          Mips::Val_GNU_MIPS_ABI_FP_SOFT},
+  {"Hard float (MIPS32r2 64-bit FPU 12 callee-saved)",
+   Mips::Val_GNU_MIPS_ABI_FP_OLD_64},
+  {"Hard float (32-bit CPU, Any FPU)",    Mips::Val_GNU_MIPS_ABI_FP_XX},
+  {"Hard float (32-bit CPU, 64-bit FPU)", Mips::Val_GNU_MIPS_ABI_FP_64},
+  {"Hard float compat (32-bit CPU, 64-bit FPU)",
+   Mips::Val_GNU_MIPS_ABI_FP_64A}
+};
+
+static const EnumEntry<unsigned> ElfMipsFlags1[] {
+  {"ODDSPREG", Mips::AFL_FLAGS1_ODDSPREG},
+};
+
+static int getMipsRegisterSize(uint8_t Flag) {
+  switch (Flag) {
+  case Mips::AFL_REG_NONE:
+    return 0;
+  case Mips::AFL_REG_32:
+    return 32;
+  case Mips::AFL_REG_64:
+    return 64;
+  case Mips::AFL_REG_128:
+    return 128;
+  default:
+    return -1;
   }
+}
 
-  if (!DtPltGot) {
-    W.startLine() << "Cannot find PLTGOT dynamic table tag.\n";
+template <class ELFT> void ELFDumper<ELFT>::printMipsABIFlags() {
+  const Elf_Shdr *Shdr = findSectionByName(*Obj, ".MIPS.abiflags");
+  if (!Shdr) {
+    W.startLine() << "There is no .MIPS.abiflags section in the file.\n";
+    return;
+  }
+  ErrorOr<ArrayRef<uint8_t>> Sec = Obj->getSectionContents(Shdr);
+  if (!Sec) {
+    W.startLine() << "The .MIPS.abiflags section is empty.\n";
+    return;
+  }
+  if (Sec->size() != sizeof(Elf_Mips_ABIFlags<ELFT>)) {
+    W.startLine() << "The .MIPS.abiflags section has a wrong size.\n";
     return;
   }
 
-  const Elf_Shdr *GotShdr = findSectionByAddress(Obj, *DtPltGot);
-  if (!GotShdr) {
-    W.startLine() << "There is no .got section in the file.\n";
+  auto *Flags = reinterpret_cast<const Elf_Mips_ABIFlags<ELFT> *>(Sec->data());
+
+  raw_ostream &OS = W.getOStream();
+  DictScope GS(W, "MIPS ABI Flags");
+
+  W.printNumber("Version", Flags->version);
+  W.startLine() << "ISA: ";
+  if (Flags->isa_rev <= 1)
+    OS << format("MIPS%u", Flags->isa_level);
+  else
+    OS << format("MIPS%ur%u", Flags->isa_level, Flags->isa_rev);
+  OS << "\n";
+  W.printEnum("ISA Extension", Flags->isa_ext, makeArrayRef(ElfMipsISAExtType));
+  W.printFlags("ASEs", Flags->ases, makeArrayRef(ElfMipsASEFlags));
+  W.printEnum("FP ABI", Flags->fp_abi, makeArrayRef(ElfMipsFpABIType));
+  W.printNumber("GPR size", getMipsRegisterSize(Flags->gpr_size));
+  W.printNumber("CPR1 size", getMipsRegisterSize(Flags->cpr1_size));
+  W.printNumber("CPR2 size", getMipsRegisterSize(Flags->cpr2_size));
+  W.printFlags("Flags 1", Flags->flags1, makeArrayRef(ElfMipsFlags1));
+  W.printHex("Flags 2", Flags->flags2);
+}
+
+template <class ELFT> void ELFDumper<ELFT>::printMipsReginfo() {
+  const Elf_Shdr *Shdr = findSectionByName(*Obj, ".reginfo");
+  if (!Shdr) {
+    W.startLine() << "There is no .reginfo section in the file.\n";
+    return;
+  }
+  ErrorOr<ArrayRef<uint8_t>> Sec = Obj->getSectionContents(Shdr);
+  if (!Sec) {
+    W.startLine() << "The .reginfo section is empty.\n";
     return;
   }
+  if (Sec->size() != sizeof(Elf_Mips_RegInfo<ELFT>)) {
+    W.startLine() << "The .reginfo section has a wrong size.\n";
+    return;
+  }
+
+  auto *Reginfo = reinterpret_cast<const Elf_Mips_RegInfo<ELFT> *>(Sec->data());
+
+  DictScope GS(W, "MIPS RegInfo");
+  W.printHex("GP", Reginfo->ri_gp_value);
+  W.printHex("General Mask", Reginfo->ri_gprmask);
+  W.printHex("Co-Proc Mask0", Reginfo->ri_cprmask[0]);
+  W.printHex("Co-Proc Mask1", Reginfo->ri_cprmask[1]);
+  W.printHex("Co-Proc Mask2", Reginfo->ri_cprmask[2]);
+  W.printHex("Co-Proc Mask3", Reginfo->ri_cprmask[3]);
+}
+
+template <class ELFT> void ELFDumper<ELFT>::printStackMap() const {
+  const Elf_Shdr *StackMapSection = nullptr;
+  for (const auto &Sec : Obj->sections()) {
+    ErrorOr<StringRef> Name = Obj->getSectionName(&Sec);
+    if (*Name == ".llvm_stackmaps") {
+      StackMapSection = &Sec;
+      break;
+    }
+  }
+
+  if (!StackMapSection)
+    return;
+
+  StringRef StackMapContents;
+  ErrorOr<ArrayRef<uint8_t>> StackMapContentsArray =
+    Obj->getSectionContents(StackMapSection);
 
-  MipsGOTParser<ELFT>(Obj, W).parseGOT(*GotShdr);
+  prettyPrintStackMap(
+              llvm::outs(),
+              StackMapV1Parser<ELFT::TargetEndianness>(*StackMapContentsArray));
 }
diff --git a/tools/llvm-readobj/MachODumper.cpp b/tools/llvm-readobj/MachODumper.cpp
index 7e8fdadd855..b226e5f35b5 100644
--- a/tools/llvm-readobj/MachODumper.cpp
+++ b/tools/llvm-readobj/MachODumper.cpp
@@ -14,6 +14,7 @@
 #include "llvm-readobj.h"
 #include "Error.h"
 #include "ObjDumper.h"
+#include "StackMapPrinter.h"
 #include "StreamWriter.h"
 #include "llvm/ADT/SmallString.h"
 #include "llvm/ADT/StringExtras.h"
@@ -37,6 +38,7 @@ class MachODumper : public ObjDumper {
   void printSymbols() override;
   void printDynamicSymbols() override;
   void printUnwindInfo() override;
+  void printStackMap() const override;
 
 private:
   template<class MachHeader>
@@ -203,31 +205,6 @@ static const EnumEntry<uint32_t> MachOHeaderFlags[] = {
   LLVM_READOBJ_ENUM_ENT(MachO, MH_APP_EXTENSION_SAFE),
 };
 
-static const EnumEntry<unsigned> MachOSectionTypes[] = {
-  { "Regular"                        , 0x00 },
-  { "ZeroFill"                       , 0x01 },
-  { "CStringLiterals"                , 0x02 },
-  { "4ByteLiterals"                  , 0x03 },
-  { "8ByteLiterals"                  , 0x04 },
-  { "LiteralPointers"                , 0x05 },
-  { "NonLazySymbolPointers"          , 0x06 },
-  { "LazySymbolPointers"             , 0x07 },
-  { "SymbolStubs"                    , 0x08 },
-  { "ModInitFuncs"                   , 0x09 },
-  { "ModTermFuncs"                   , 0x0A },
-  { "Coalesced"                      , 0x0B },
-  { "GBZeroFill"                     , 0x0C },
-  { "Interposing"                    , 0x0D },
-  { "16ByteLiterals"                 , 0x0E },
-  { "DTraceDOF"                      , 0x0F },
-  { "LazyDylibSymbolPoints"          , 0x10 },
-  { "ThreadLocalRegular"             , 0x11 },
-  { "ThreadLocalZerofill"            , 0x12 },
-  { "ThreadLocalVariables"           , 0x13 },
-  { "ThreadLocalVariablePointers"    , 0x14 },
-  { "ThreadLocalInitFunctionPointers", 0x15 }
-};
-
 static const EnumEntry<unsigned> MachOSectionAttributes[] = {
   { "LocReloc"         , 1 <<  0 /*S_ATTR_LOC_RELOC          */ },
   { "ExtReloc"         , 1 <<  1 /*S_ATTR_EXT_RELOC          */ },
@@ -398,8 +375,7 @@ void MachODumper::printSections(const MachOObjectFile *Obj) {
     DataRefImpl DR = Section.getRawDataRefImpl();
 
     StringRef Name;
-    if (error(Section.getName(Name)))
-      Name = "";
+    error(Section.getName(Name));
 
     ArrayRef<char> RawName = Obj->getSectionRawName(DR);
     StringRef SegmentName = Obj->getSectionFinalSegmentName(DR);
@@ -442,8 +418,7 @@ void MachODumper::printSections(const MachOObjectFile *Obj) {
       bool IsBSS = Section.isBSS();
       if (!IsBSS) {
         StringRef Data;
-        if (error(Section.getContents(Data)))
-          break;
+        error(Section.getContents(Data));
 
         W.printBinaryBlock("SectionData", Data);
       }
@@ -457,8 +432,7 @@ void MachODumper::printRelocations() {
   std::error_code EC;
   for (const SectionRef &Section : Obj->sections()) {
     StringRef Name;
-    if (error(Section.getName(Name)))
-      continue;
+    error(Section.getName(Name));
 
     bool PrintedGroup = false;
     for (const RelocationRef &Reloc : Section.relocations()) {
@@ -484,45 +458,54 @@ void MachODumper::printRelocation(const RelocationRef &Reloc) {
 
 void MachODumper::printRelocation(const MachOObjectFile *Obj,
                                   const RelocationRef &Reloc) {
-  uint64_t Offset;
+  uint64_t Offset = Reloc.getOffset();
   SmallString<32> RelocName;
-  if (error(Reloc.getOffset(Offset)))
-    return;
-  if (error(Reloc.getTypeName(RelocName)))
-    return;
+  Reloc.getTypeName(RelocName);
 
   DataRefImpl DR = Reloc.getRawDataRefImpl();
   MachO::any_relocation_info RE = Obj->getRelocation(DR);
   bool IsScattered = Obj->isRelocationScattered(RE);
-  SmallString<32> SymbolNameOrOffset("0x");
-  if (IsScattered) {
-    // Scattered relocations don't really have an associated symbol
-    // for some reason, even if one exists in the symtab at the correct address.
-    SymbolNameOrOffset += utohexstr(Obj->getScatteredRelocationValue(RE));
-  } else {
+  bool IsExtern = !IsScattered && Obj->getPlainRelocationExternal(RE);
+
+  StringRef TargetName;
+  if (IsExtern) {
     symbol_iterator Symbol = Reloc.getSymbol();
     if (Symbol != Obj->symbol_end()) {
-      StringRef SymbolName;
-      if (error(Symbol->getName(SymbolName)))
-        return;
-      SymbolNameOrOffset = SymbolName;
-    } else
-      SymbolNameOrOffset += utohexstr(Obj->getPlainRelocationSymbolNum(RE));
+      ErrorOr<StringRef> TargetNameOrErr = Symbol->getName();
+      error(TargetNameOrErr.getError());
+      TargetName = *TargetNameOrErr;
+    }
+  } else if (!IsScattered) {
+    section_iterator SecI = Obj->getRelocationSection(DR);
+    if (SecI != Obj->section_end()) {
+      error(SecI->getName(TargetName));
+    }
   }
+  if (TargetName.empty())
+    TargetName = "-";
 
   if (opts::ExpandRelocs) {
     DictScope Group(W, "Relocation");
     W.printHex("Offset", Offset);
     W.printNumber("PCRel", Obj->getAnyRelocationPCRel(RE));
     W.printNumber("Length", Obj->getAnyRelocationLength(RE));
-    if (IsScattered)
-      W.printString("Extern", StringRef("N/A"));
-    else
-      W.printNumber("Extern", Obj->getPlainRelocationExternal(RE));
     W.printNumber("Type", RelocName, Obj->getAnyRelocationType(RE));
-    W.printString("Symbol", SymbolNameOrOffset);
-    W.printNumber("Scattered", IsScattered);
+    if (IsScattered) {
+      W.printHex("Value", Obj->getScatteredRelocationValue(RE));
+    } else {
+      const char *Kind = IsExtern ? "Symbol" : "Section";
+      W.printNumber(Kind, TargetName, Obj->getPlainRelocationSymbolNum(RE));
+    }
   } else {
+    SmallString<32> SymbolNameOrOffset("0x");
+    if (IsScattered) {
+      // Scattered relocations don't really have an associated symbol for some
+      // reason, even if one exists in the symtab at the correct address.
+      SymbolNameOrOffset += utohexstr(Obj->getScatteredRelocationValue(RE));
+    } else {
+      SymbolNameOrOffset = TargetName;
+    }
+
     raw_ostream& OS = W.startLine();
     OS << W.hex(Offset)
        << " " << Obj->getAnyRelocationPCRel(RE)
@@ -552,15 +535,16 @@ void MachODumper::printDynamicSymbols() {
 
 void MachODumper::printSymbol(const SymbolRef &Symbol) {
   StringRef SymbolName;
-  if (Symbol.getName(SymbolName))
-    SymbolName = "";
+  if (ErrorOr<StringRef> SymbolNameOrErr = Symbol.getName())
+    SymbolName = *SymbolNameOrErr;
 
   MachOSymbol MOSymbol;
   getSymbol(Obj, Symbol.getRawDataRefImpl(), MOSymbol);
 
   StringRef SectionName = "";
   section_iterator SecI(Obj->section_begin());
-  if (!error(Symbol.getSection(SecI)) && SecI != Obj->section_end())
+  error(Symbol.getSection(SecI));
+  if (SecI != Obj->section_end())
     error(SecI->getName(SectionName));
 
   DictScope D(W, "Symbol");
@@ -586,3 +570,32 @@ void MachODumper::printSymbol(const SymbolRef &Symbol) {
 void MachODumper::printUnwindInfo() {
   W.startLine() << "UnwindInfo not implemented.\n";
 }
+
+void MachODumper::printStackMap() const {
+  object::SectionRef StackMapSection;
+  for (auto Sec : Obj->sections()) {
+    StringRef Name;
+    Sec.getName(Name);
+    if (Name == "__llvm_stackmaps") {
+      StackMapSection = Sec;
+      break;
+    }
+  }
+
+  if (StackMapSection == object::SectionRef())
+    return;
+
+  StringRef StackMapContents;
+  StackMapSection.getContents(StackMapContents);
+  ArrayRef<uint8_t> StackMapContentsArray(
+      reinterpret_cast<const uint8_t*>(StackMapContents.data()),
+      StackMapContents.size());
+
+  if (Obj->isLittleEndian())
+     prettyPrintStackMap(
+                      llvm::outs(),
+                      StackMapV1Parser<support::little>(StackMapContentsArray));
+  else
+     prettyPrintStackMap(llvm::outs(),
+                         StackMapV1Parser<support::big>(StackMapContentsArray));
+}
diff --git a/tools/llvm-readobj/ObjDumper.h b/tools/llvm-readobj/ObjDumper.h
index 27e658fc731..206082fa5e2 100644
--- a/tools/llvm-readobj/ObjDumper.h
+++ b/tools/llvm-readobj/ObjDumper.h
@@ -1,4 +1,4 @@
-//===-- ObjDumper.h -------------------------------------------------------===//
+//===-- ObjDumper.h ---------------------------------------------*- C++ -*-===//
 //
 //                     The LLVM Compiler Infrastructure
 //
@@ -33,15 +33,20 @@ class ObjDumper {
   virtual void printUnwindInfo() = 0;
 
   // Only implemented for ELF at this time.
+  virtual void printDynamicRelocations() { }
   virtual void printDynamicTable() { }
   virtual void printNeededLibraries() { }
   virtual void printProgramHeaders() { }
+  virtual void printHashTable() { }
+  virtual void printLoadName() {}
 
   // Only implemented for ARM ELF at this time.
   virtual void printAttributes() { }
 
   // Only implemented for MIPS ELF at this time.
   virtual void printMipsPLTGOT() { }
+  virtual void printMipsABIFlags() { }
+  virtual void printMipsReginfo() { }
 
   // Only implemented for PE/COFF.
   virtual void printCOFFImports() { }
@@ -49,6 +54,8 @@ class ObjDumper {
   virtual void printCOFFDirectives() { }
   virtual void printCOFFBaseReloc() { }
 
+  virtual void printStackMap() const = 0;
+
 protected:
   StreamWriter& W;
 };
diff --git a/tools/llvm-readobj/StackMapPrinter.h b/tools/llvm-readobj/StackMapPrinter.h
new file mode 100644
index 00000000000..92645bcf917
--- /dev/null
+++ b/tools/llvm-readobj/StackMapPrinter.h
@@ -0,0 +1,80 @@
+//===-------- StackMapPrinter.h - Pretty-print stackmaps --------*- C++ -*-===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_TOOLS_LLVM_READOBJ_STACKMAPPRINTER_H
+#define LLVM_TOOLS_LLVM_READOBJ_STACKMAPPRINTER_H
+
+#include "llvm/Object/StackMapParser.h"
+
+namespace llvm {
+
+// Pretty print a stackmap to the given ostream.
+template <typename OStreamT, typename StackMapParserT>
+void prettyPrintStackMap(OStreamT &OS, const StackMapParserT &SMP) {
+
+  OS << "LLVM StackMap Version: " << SMP.getVersion()
+     << "\nNum Functions: " << SMP.getNumFunctions();
+
+  // Functions:
+  for (const auto &F : SMP.functions())
+    OS << "\n  Function address: " << F.getFunctionAddress()
+       << ", stack size: " << F.getStackSize();
+
+  // Constants:
+  OS << "\nNum Constants: " << SMP.getNumConstants();
+  unsigned ConstantIndex = 0;
+  for (const auto &C : SMP.constants())
+    OS << "\n  #" << ++ConstantIndex << ": " << C.getValue();
+
+  // Records:
+  OS << "\nNum Records: " << SMP.getNumRecords();
+  for (const auto &R : SMP.records()) {
+    OS << "\n  Record ID: " << R.getID()
+       << ", instruction offset: " << R.getInstructionOffset()
+       << "\n    " << R.getNumLocations() << " locations:";
+
+    unsigned LocationIndex = 0;
+    for (const auto &Loc : R.locations()) {
+      OS << "\n      #" << ++LocationIndex << ": ";
+      switch (Loc.getKind()) {
+      case StackMapParserT::LocationKind::Register:
+        OS << "Register R#" << Loc.getDwarfRegNum();
+        break;
+      case StackMapParserT::LocationKind::Direct:
+        OS << "Direct R#" << Loc.getDwarfRegNum() << " + "
+           << Loc.getOffset();
+        break;
+      case StackMapParserT::LocationKind::Indirect:
+        OS << "Indirect [R#" << Loc.getDwarfRegNum() << " + "
+           << Loc.getOffset() << "]";
+        break;
+      case StackMapParserT::LocationKind::Constant:
+        OS << "Constant " << Loc.getSmallConstant();
+        break;
+      case StackMapParserT::LocationKind::ConstantIndex:
+        OS << "ConstantIndex #" << Loc.getConstantIndex() << " ("
+           << SMP.getConstant(Loc.getConstantIndex()).getValue() << ")";
+        break;
+      }
+    }
+
+    OS << "\n    " << R.getNumLiveOuts() << " live-outs: [ ";
+    for (const auto &LO : R.liveouts())
+      OS << "R#" << LO.getDwarfRegNum() << " ("
+         << LO.getSizeInBytes() << "-bytes) ";
+    OS << "]\n";
+  }
+
+ OS << "\n";
+
+}
+
+}
+
+#endif
diff --git a/tools/llvm-readobj/StreamWriter.h b/tools/llvm-readobj/StreamWriter.h
index 2fc53eeeec4..f3cc57ef940 100644
--- a/tools/llvm-readobj/StreamWriter.h
+++ b/tools/llvm-readobj/StreamWriter.h
@@ -96,9 +96,10 @@ class StreamWriter {
     }
   }
 
-  template<typename T, typename TFlag>
-  void printFlags(StringRef Label, T Value, ArrayRef<EnumEntry<TFlag> > Flags,
-                  TFlag EnumMask = TFlag(0)) {
+  template <typename T, typename TFlag>
+  void printFlags(StringRef Label, T Value, ArrayRef<EnumEntry<TFlag>> Flags,
+                  TFlag EnumMask1 = {}, TFlag EnumMask2 = {},
+                  TFlag EnumMask3 = {}) {
     typedef EnumEntry<TFlag> FlagEntry;
     typedef SmallVector<FlagEntry, 10> FlagVector;
     FlagVector SetFlags;
@@ -107,6 +108,13 @@ class StreamWriter {
       if (Flag.Value == 0)
         continue;
 
+      TFlag EnumMask{};
+      if (Flag.Value & EnumMask1)
+        EnumMask = EnumMask1;
+      else if (Flag.Value & EnumMask2)
+        EnumMask = EnumMask2;
+      else if (Flag.Value & EnumMask3)
+        EnumMask = EnumMask3;
       bool IsEnum = (Flag.Value & EnumMask) != 0;
       if ((!IsEnum && (Value & Flag.Value) == Flag.Value) ||
           (IsEnum  && (Value & EnumMask) == Flag.Value)) {
@@ -173,8 +181,8 @@ class StreamWriter {
     startLine() << Label << ": " << (Value ? "Yes" : "No") << '\n';
   }
 
-  template <typename T_>
-  void printList(StringRef Label, const SmallVectorImpl<T_> &List) {
+  template <typename T>
+  void printList(StringRef Label, const T &List) {
     startLine() << Label << ": [";
     bool Comma = false;
     for (const auto &Item : List) {
diff --git a/tools/llvm-readobj/Win64EHDumper.cpp b/tools/llvm-readobj/Win64EHDumper.cpp
index f058632a8ce..076f137d6ee 100644
--- a/tools/llvm-readobj/Win64EHDumper.cpp
+++ b/tools/llvm-readobj/Win64EHDumper.cpp
@@ -118,19 +118,19 @@ static std::string formatSymbol(const Dumper::Context &Ctx,
   std::string Buffer;
   raw_string_ostream OS(Buffer);
 
-  StringRef Name;
   SymbolRef Symbol;
-  if (Ctx.ResolveSymbol(Section, Offset, Symbol, Ctx.UserData) ||
-      Symbol.getName(Name)) {
-    OS << format(" (0x%" PRIX64 ")", Offset);
-    return OS.str();
+  if (!Ctx.ResolveSymbol(Section, Offset, Symbol, Ctx.UserData)) {
+    if (ErrorOr<StringRef> Name = Symbol.getName()) {
+      OS << *Name;
+      if (Displacement > 0)
+        OS << format(" +0x%X (0x%" PRIX64 ")", Displacement, Offset);
+      else
+        OS << format(" (0x%" PRIX64 ")", Offset);
+      return OS.str();
+    }
   }
 
-  OS << Name;
-  if (Displacement > 0)
-    OS << format(" +0x%X (0x%" PRIX64 ")", Displacement, Offset);
-  else
-    OS << format(" (0x%" PRIX64 ")", Offset);
+  OS << format(" (0x%" PRIX64 ")", Offset);
   return OS.str();
 }
 
@@ -144,15 +144,17 @@ static std::error_code resolveRelocation(const Dumper::Context &Ctx,
           Ctx.ResolveSymbol(Section, Offset, Symbol, Ctx.UserData))
     return EC;
 
-  if (std::error_code EC = Symbol.getAddress(ResolvedAddress))
+  ErrorOr<uint64_t> ResolvedAddressOrErr = Symbol.getAddress();
+  if (std::error_code EC = ResolvedAddressOrErr.getError())
     return EC;
+  ResolvedAddress = *ResolvedAddressOrErr;
 
   section_iterator SI = Ctx.COFF.section_begin();
   if (std::error_code EC = Symbol.getSection(SI))
     return EC;
 
   ResolvedSection = Ctx.COFF.getCOFFSection(*SI);
-  return object_error::success;
+  return std::error_code();
 }
 
 namespace llvm {
@@ -282,11 +284,11 @@ void Dumper::printRuntimeFunction(const Context &Ctx,
 
   const coff_section *XData;
   uint64_t Offset;
-  if (error(resolveRelocation(Ctx, Section, SectionOffset + 8, XData, Offset)))
-    return;
+  resolveRelocation(Ctx, Section, SectionOffset + 8, XData, Offset);
 
   ArrayRef<uint8_t> Contents;
-  if (error(Ctx.COFF.getSectionContents(XData, Contents)) || Contents.empty())
+  error(Ctx.COFF.getSectionContents(XData, Contents));
+  if (Contents.empty())
     return;
 
   Offset = Offset + RF.UnwindInfoOffset;
@@ -300,15 +302,15 @@ void Dumper::printRuntimeFunction(const Context &Ctx,
 void Dumper::printData(const Context &Ctx) {
   for (const auto &Section : Ctx.COFF.sections()) {
     StringRef Name;
-    if (error(Section.getName(Name)))
-      continue;
+    Section.getName(Name);
 
     if (Name != ".pdata" && !Name.startswith(".pdata$"))
       continue;
 
     const coff_section *PData = Ctx.COFF.getCOFFSection(Section);
     ArrayRef<uint8_t> Contents;
-    if (error(Ctx.COFF.getSectionContents(PData, Contents)) || Contents.empty())
+    error(Ctx.COFF.getSectionContents(PData, Contents));
+    if (Contents.empty())
       continue;
 
     const RuntimeFunction *Entries =
diff --git a/tools/llvm-readobj/llvm-readobj.cpp b/tools/llvm-readobj/llvm-readobj.cpp
index 7e6ce496723..db0ffd0141a 100644
--- a/tools/llvm-readobj/llvm-readobj.cpp
+++ b/tools/llvm-readobj/llvm-readobj.cpp
@@ -40,7 +40,6 @@
 #include <string>
 #include <system_error>
 
-
 using namespace llvm;
 using namespace llvm::object;
 
@@ -91,6 +90,10 @@ namespace opts {
     cl::desc("Alias for --relocations"),
     cl::aliasopt(Relocations));
 
+  // -dyn-relocations
+  cl::opt<bool> DynRelocs("dyn-relocations",
+    cl::desc("Display the dynamic relocation entries in the file"));
+
   // -symbols, -t
   cl::opt<bool> Symbols("symbols",
     cl::desc("Display the symbol table"));
@@ -124,6 +127,10 @@ namespace opts {
   cl::opt<bool> ProgramHeaders("program-headers",
     cl::desc("Display ELF program headers"));
 
+  // -hash-table
+  cl::opt<bool> HashTable("hash-table",
+    cl::desc("Display ELF hash table"));
+
   // -expand-relocs
   cl::opt<bool> ExpandRelocs("expand-relocs",
     cl::desc("Expand each shown relocation to multiple lines"));
@@ -148,6 +155,14 @@ namespace opts {
   MipsPLTGOT("mips-plt-got",
              cl::desc("Display the MIPS GOT and PLT GOT sections"));
 
+  // -mips-abi-flags
+  cl::opt<bool> MipsABIFlags("mips-abi-flags",
+                             cl::desc("Display the MIPS.abiflags section"));
+
+  // -mips-reginfo
+  cl::opt<bool> MipsReginfo("mips-reginfo",
+                            cl::desc("Display the MIPS .reginfo section"));
+
   // -coff-imports
   cl::opt<bool>
   COFFImports("coff-imports", cl::desc("Display the PE/COFF import table"));
@@ -165,27 +180,31 @@ namespace opts {
   cl::opt<bool>
   COFFBaseRelocs("coff-basereloc",
                  cl::desc("Display the PE/COFF .reloc section"));
-} // namespace opts
 
-static int ReturnValue = EXIT_SUCCESS;
+  // -stackmap
+  cl::opt<bool>
+  PrintStackMap("stackmap",
+                cl::desc("Display contents of stackmap section"));
+
+} // namespace opts
 
 namespace llvm {
 
-bool error(std::error_code EC) {
+void reportError(Twine Msg) {
+  outs() << Msg << "\n";
+  outs().flush();
+  exit(1);
+}
+
+void error(std::error_code EC) {
   if (!EC)
-    return false;
+    return;
 
-  ReturnValue = EXIT_FAILURE;
-  outs() << "\nError reading file: " << EC.message() << ".\n";
-  outs().flush();
-  return true;
+  reportError(Twine("\nError reading file: ") + EC.message() + ".");
 }
 
 bool relocAddressLess(RelocationRef a, RelocationRef b) {
-  uint64_t a_addr, b_addr;
-  if (error(a.getOffset(a_addr))) exit(ReturnValue);
-  if (error(b.getOffset(b_addr))) exit(ReturnValue);
-  return a_addr < b_addr;
+  return a.getOffset() < b.getOffset();
 }
 
 } // namespace llvm
@@ -194,17 +213,14 @@ static void reportError(StringRef Input, std::error_code EC) {
   if (Input == "-")
     Input = "<stdin>";
 
-  errs() << Input << ": " << EC.message() << "\n";
-  errs().flush();
-  ReturnValue = EXIT_FAILURE;
+  reportError(Twine(Input) + ": " + EC.message());
 }
 
 static void reportError(StringRef Input, StringRef Message) {
   if (Input == "-")
     Input = "<stdin>";
 
-  errs() << Input << ": " << Message << "\n";
-  ReturnValue = EXIT_FAILURE;
+  reportError(Twine(Input) + ": " + Message);
 }
 
 static bool isMipsArch(unsigned Arch) {
@@ -235,18 +251,6 @@ static std::error_code createDumper(const ObjectFile *Obj, StreamWriter &Writer,
   return readobj_error::unsupported_obj_file_format;
 }
 
-static StringRef getLoadName(const ObjectFile *Obj) {
-  if (auto *ELF = dyn_cast<ELF32LEObjectFile>(Obj))
-    return ELF->getLoadName();
-  if (auto *ELF = dyn_cast<ELF64LEObjectFile>(Obj))
-    return ELF->getLoadName();
-  if (auto *ELF = dyn_cast<ELF32BEObjectFile>(Obj))
-    return ELF->getLoadName();
-  if (auto *ELF = dyn_cast<ELF64BEObjectFile>(Obj))
-    return ELF->getLoadName();
-  llvm_unreachable("Not ELF");
-}
-
 /// @brief Dumps the specified object file.
 static void dumpObject(const ObjectFile *Obj) {
   StreamWriter Writer(outs());
@@ -263,8 +267,7 @@ static void dumpObject(const ObjectFile *Obj) {
          << Triple::getArchTypeName((llvm::Triple::ArchType)Obj->getArch())
          << "\n";
   outs() << "AddressSize: " << (8*Obj->getBytesInAddress()) << "bit\n";
-  if (Obj->isELF())
-    outs() << "LoadName: " << getLoadName(Obj) << "\n";
+  Dumper->printLoadName();
 
   if (opts::FileHeaders)
     Dumper->printFileHeaders();
@@ -272,6 +275,8 @@ static void dumpObject(const ObjectFile *Obj) {
     Dumper->printSections();
   if (opts::Relocations)
     Dumper->printRelocations();
+  if (opts::DynRelocs)
+    Dumper->printDynamicRelocations();
   if (opts::Symbols)
     Dumper->printSymbols();
   if (opts::DynamicSymbols)
@@ -284,23 +289,33 @@ static void dumpObject(const ObjectFile *Obj) {
     Dumper->printNeededLibraries();
   if (opts::ProgramHeaders)
     Dumper->printProgramHeaders();
+  if (opts::HashTable)
+    Dumper->printHashTable();
   if (Obj->getArch() == llvm::Triple::arm && Obj->isELF())
     if (opts::ARMAttributes)
       Dumper->printAttributes();
-  if (isMipsArch(Obj->getArch()) && Obj->isELF())
+  if (isMipsArch(Obj->getArch()) && Obj->isELF()) {
     if (opts::MipsPLTGOT)
       Dumper->printMipsPLTGOT();
-  if (opts::COFFImports)
-    Dumper->printCOFFImports();
-  if (opts::COFFExports)
-    Dumper->printCOFFExports();
-  if (opts::COFFDirectives)
-    Dumper->printCOFFDirectives();
-  if (opts::COFFBaseRelocs)
-    Dumper->printCOFFBaseReloc();
+    if (opts::MipsABIFlags)
+      Dumper->printMipsABIFlags();
+    if (opts::MipsReginfo)
+      Dumper->printMipsReginfo();
+  }
+  if (Obj->isCOFF()) {
+    if (opts::COFFImports)
+      Dumper->printCOFFImports();
+    if (opts::COFFExports)
+      Dumper->printCOFFExports();
+    if (opts::COFFDirectives)
+      Dumper->printCOFFDirectives();
+    if (opts::COFFBaseRelocs)
+      Dumper->printCOFFBaseReloc();
+  }
+  if (opts::PrintStackMap)
+    Dumper->printStackMap();
 }
 
-
 /// @brief Dumps each object file in \a Arc;
 static void dumpArchive(const Archive *Arc) {
   for (Archive::child_iterator ArcI = Arc->child_begin(),
@@ -361,15 +376,11 @@ static void dumpInput(StringRef File) {
     reportError(File, readobj_error::unrecognized_file_format);
 }
 
-
 int main(int argc, const char *argv[]) {
   sys::PrintStackTraceOnErrorSignal();
   PrettyStackTraceProgram X(argc, argv);
   llvm_shutdown_obj Y;
 
-  // Initialize targets.
-  llvm::InitializeAllTargetInfos();
-
   // Register the target printer for --version.
   cl::AddExtraVersionPrinter(TargetRegistry::printRegisteredTargetsForVersion);
 
@@ -382,5 +393,5 @@ int main(int argc, const char *argv[]) {
   std::for_each(opts::InputFilenames.begin(), opts::InputFilenames.end(),
                 dumpInput);
 
-  return ReturnValue;
+  return 0;
 }
diff --git a/tools/llvm-readobj/llvm-readobj.h b/tools/llvm-readobj/llvm-readobj.h
index 74b9a60d34b..58c50f58d75 100644
--- a/tools/llvm-readobj/llvm-readobj.h
+++ b/tools/llvm-readobj/llvm-readobj.h
@@ -19,7 +19,8 @@ namespace llvm {
   }
 
   // Various helper functions.
-  bool error(std::error_code ec);
+  void reportError(Twine Msg);
+  void error(std::error_code ec);
   bool relocAddressLess(object::RelocationRef A,
                         object::RelocationRef B);
 } // namespace llvm
diff --git a/tools/llvm-rtdyld/llvm-rtdyld.cpp b/tools/llvm-rtdyld/llvm-rtdyld.cpp
index 480032d3b6f..d6423a3e693 100644
--- a/tools/llvm-rtdyld/llvm-rtdyld.cpp
+++ b/tools/llvm-rtdyld/llvm-rtdyld.cpp
@@ -12,7 +12,8 @@
 //===----------------------------------------------------------------------===//
 
 #include "llvm/ADT/StringMap.h"
-#include "llvm/DebugInfo/DWARF/DIContext.h"
+#include "llvm/DebugInfo/DIContext.h"
+#include "llvm/DebugInfo/DWARF/DWARFContext.h"
 #include "llvm/ExecutionEngine/RTDyldMemoryManager.h"
 #include "llvm/ExecutionEngine/RuntimeDyld.h"
 #include "llvm/ExecutionEngine/RuntimeDyldChecker.h"
@@ -22,7 +23,9 @@
 #include "llvm/MC/MCInstPrinter.h"
 #include "llvm/MC/MCInstrInfo.h"
 #include "llvm/MC/MCRegisterInfo.h"
+#include "llvm/MC/MCSubtargetInfo.h"
 #include "llvm/Object/MachO.h"
+#include "llvm/Object/SymbolSize.h"
 #include "llvm/Support/CommandLine.h"
 #include "llvm/Support/DynamicLibrary.h"
 #include "llvm/Support/ManagedStatic.h"
@@ -45,7 +48,9 @@ InputFileList(cl::Positional, cl::ZeroOrMore,
 
 enum ActionType {
   AC_Execute,
+  AC_PrintObjectLineInfo,
   AC_PrintLineInfo,
+  AC_PrintDebugLineInfo,
   AC_Verify
 };
 
@@ -56,6 +61,10 @@ Action(cl::desc("Action to perform:"),
                              "Load, link, and execute the inputs."),
                   clEnumValN(AC_PrintLineInfo, "printline",
                              "Load, link, and print line information for each function."),
+                  clEnumValN(AC_PrintDebugLineInfo, "printdebugline",
+                             "Load, link, and print line information for each function using the debug object"),
+                  clEnumValN(AC_PrintObjectLineInfo, "printobjline",
+                             "Like -printlineinfo but does not load the object first"),
                   clEnumValN(AC_Verify, "verify",
                              "Load, link and verify the resulting memory image."),
                   clEnumValEnd));
@@ -73,6 +82,12 @@ Dylibs("dylib",
 static cl::opt<std::string>
 TripleName("triple", cl::desc("Target triple for disassembler"));
 
+static cl::opt<std::string>
+MCPU("mcpu",
+     cl::desc("Target a specific cpu type (-mcpu=help for details)"),
+     cl::value_desc("cpu-name"),
+     cl::init(""));
+
 static cl::list<std::string>
 CheckFiles("check",
            cl::desc("File containing RuntimeDyld verifier checks."),
@@ -100,8 +115,17 @@ TargetSectionSep("target-section-sep",
 
 static cl::list<std::string>
 SpecificSectionMappings("map-section",
-                        cl::desc("Map a section to a specific address."),
-                        cl::ZeroOrMore);
+                        cl::desc("For -verify only: Map a section to a "
+                                 "specific address."),
+                        cl::ZeroOrMore,
+                        cl::Hidden);
+
+static cl::list<std::string>
+DummySymbolMappings("dummy-extern",
+                    cl::desc("For -verify only: Inject a symbol into the extern "
+                             "symbol table."),
+                    cl::ZeroOrMore,
+                    cl::Hidden);
 
 /* *** */
 
@@ -131,6 +155,26 @@ class TrivialMemoryManager : public RTDyldMemoryManager {
   // explicit cache flush, otherwise JIT code manipulations (like resolved
   // relocations) will get to the data cache but not to the instruction cache.
   virtual void invalidateInstructionCache();
+
+  void addDummySymbol(const std::string &Name, uint64_t Addr) {
+    DummyExterns[Name] = Addr;
+  }
+
+  RuntimeDyld::SymbolInfo findSymbol(const std::string &Name) override {
+    auto I = DummyExterns.find(Name);
+
+    if (I != DummyExterns.end())
+      return RuntimeDyld::SymbolInfo(I->second, JITSymbolFlags::Exported);
+
+    return RTDyldMemoryManager::findSymbol(Name);
+  }
+
+  void registerEHFrames(uint8_t *Addr, uint64_t LoadAddr,
+                        size_t Size) override {}
+  void deregisterEHFrames(uint8_t *Addr, uint64_t LoadAddr,
+                          size_t Size) override {}
+private:
+  std::map<std::string, uint64_t> DummyExterns;
 };
 
 uint8_t *TrivialMemoryManager::allocateCodeSection(uintptr_t Size,
@@ -187,7 +231,9 @@ static void loadDylibs() {
 
 /* *** */
 
-static int printLineInfoForInput() {
+static int printLineInfoForInput(bool LoadObjects, bool UseDebugObj) {
+  assert(LoadObjects || !UseDebugObj);
+
   // Load any dylibs requested on the command line.
   loadDylibs();
 
@@ -214,36 +260,62 @@ static int printLineInfoForInput() {
 
     ObjectFile &Obj = **MaybeObj;
 
-    // Load the object file
-    std::unique_ptr<RuntimeDyld::LoadedObjectInfo> LoadedObjInfo =
-      Dyld.loadObject(Obj);
+    OwningBinary<ObjectFile> DebugObj;
+    std::unique_ptr<RuntimeDyld::LoadedObjectInfo> LoadedObjInfo = nullptr;
+    ObjectFile *SymbolObj = &Obj;
+    if (LoadObjects) {
+      // Load the object file
+      LoadedObjInfo =
+        Dyld.loadObject(Obj);
 
-    if (Dyld.hasError())
-      return Error(Dyld.getErrorString());
+      if (Dyld.hasError())
+        return Error(Dyld.getErrorString());
 
-    // Resolve all the relocations we can.
-    Dyld.resolveRelocations();
+      // Resolve all the relocations we can.
+      Dyld.resolveRelocations();
 
-    OwningBinary<ObjectFile> DebugObj = LoadedObjInfo->getObjectForDebug(Obj);
+      if (UseDebugObj) {
+        DebugObj = LoadedObjInfo->getObjectForDebug(Obj);
+        SymbolObj = DebugObj.getBinary();
+        LoadedObjInfo.reset();
+      }
+    }
 
     std::unique_ptr<DIContext> Context(
-      DIContext::getDWARFContext(*DebugObj.getBinary()));
+      new DWARFContextInMemory(*SymbolObj,LoadedObjInfo.get()));
+
+    std::vector<std::pair<SymbolRef, uint64_t>> SymAddr =
+        object::computeSymbolSizes(*SymbolObj);
 
     // Use symbol info to iterate functions in the object.
-    for (object::symbol_iterator I = DebugObj.getBinary()->symbol_begin(),
-                                 E = DebugObj.getBinary()->symbol_end();
-         I != E; ++I) {
-      object::SymbolRef::Type SymType;
-      if (I->getType(SymType)) continue;
-      if (SymType == object::SymbolRef::ST_Function) {
-        StringRef  Name;
-        uint64_t   Addr;
-        uint64_t   Size;
-        if (I->getName(Name)) continue;
-        if (I->getAddress(Addr)) continue;
-        if (I->getSize(Size)) continue;
-
-        outs() << "Function: " << Name << ", Size = " << Size << "\n";
+    for (const auto &P : SymAddr) {
+      object::SymbolRef Sym = P.first;
+      if (Sym.getType() == object::SymbolRef::ST_Function) {
+        ErrorOr<StringRef> Name = Sym.getName();
+        if (!Name)
+          continue;
+        ErrorOr<uint64_t> AddrOrErr = Sym.getAddress();
+        if (!AddrOrErr)
+          continue;
+        uint64_t Addr = *AddrOrErr;
+
+        uint64_t Size = P.second;
+        // If we're not using the debug object, compute the address of the
+        // symbol in memory (rather than that in the unrelocated object file)
+        // and use that to query the DWARFContext.
+        if (!UseDebugObj && LoadObjects) {
+          object::section_iterator Sec(SymbolObj->section_end());
+          Sym.getSection(Sec);
+          StringRef SecName;
+          Sec->getName(SecName);
+          uint64_t SectionLoadAddress =
+            LoadedObjInfo->getSectionLoadAddress(*Sec);
+          if (SectionLoadAddress != 0)
+            Addr += SectionLoadAddress - Sec->getAddress();
+        }
+
+        outs() << "Function: " << *Name << ", Size = " << Size
+               << ", Addr = " << Addr << "\n";
 
         DILineInfoTable Lines = Context->getLineInfoForAddressRange(Addr, Size);
         DILineInfoTable::iterator  Begin = Lines.begin();
@@ -354,7 +426,7 @@ applySpecificSectionMappings(RuntimeDyldChecker &Checker) {
   for (StringRef Mapping : SpecificSectionMappings) {
 
     size_t EqualsIdx = Mapping.find_first_of("=");
-    StringRef SectionIDStr = Mapping.substr(0, EqualsIdx);
+    std::string SectionIDStr = Mapping.substr(0, EqualsIdx);
     size_t ComaIdx = Mapping.find_first_of(",");
 
     if (ComaIdx == StringRef::npos) {
@@ -363,8 +435,8 @@ applySpecificSectionMappings(RuntimeDyldChecker &Checker) {
       exit(1);
     }
 
-    StringRef FileName = SectionIDStr.substr(0, ComaIdx);
-    StringRef SectionName = SectionIDStr.substr(ComaIdx + 1);
+    std::string FileName = SectionIDStr.substr(0, ComaIdx);
+    std::string SectionName = SectionIDStr.substr(ComaIdx + 1);
 
     uint64_t OldAddrInt;
     std::string ErrorMsg;
@@ -378,11 +450,11 @@ applySpecificSectionMappings(RuntimeDyldChecker &Checker) {
 
     void* OldAddr = reinterpret_cast<void*>(static_cast<uintptr_t>(OldAddrInt));
 
-    StringRef NewAddrStr = Mapping.substr(EqualsIdx + 1);
+    std::string NewAddrStr = Mapping.substr(EqualsIdx + 1);
     uint64_t NewAddr;
 
-    if (NewAddrStr.getAsInteger(0, NewAddr)) {
-      errs() << "Invalid section address in mapping: " << Mapping << "\n";
+    if (StringRef(NewAddrStr).getAsInteger(0, NewAddr)) {
+      errs() << "Invalid section address in mapping '" << Mapping << "'.\n";
       exit(1);
     }
 
@@ -404,9 +476,9 @@ applySpecificSectionMappings(RuntimeDyldChecker &Checker) {
 //                            Defaults to zero. Set to something big
 //                            (e.g. 1 << 32) to stress-test stubs, GOTs, etc.
 //
-static void remapSections(const llvm::Triple &TargetTriple,
-                          const TrivialMemoryManager &MemMgr,
-                          RuntimeDyldChecker &Checker) {
+static void remapSectionsAndSymbols(const llvm::Triple &TargetTriple,
+                                    TrivialMemoryManager &MemMgr,
+                                    RuntimeDyldChecker &Checker) {
 
   // Set up a work list (section addr/size pairs).
   typedef std::list<std::pair<void*, uint64_t>> WorklistT;
@@ -469,6 +541,27 @@ static void remapSections(const llvm::Triple &TargetTriple,
     Checker.getRTDyld().mapSectionAddress(CurEntry.first, NextSectionAddr);
   }
 
+  // Add dummy symbols to the memory manager.
+  for (const auto &Mapping : DummySymbolMappings) {
+    size_t EqualsIdx = Mapping.find_first_of("=");
+
+    if (EqualsIdx == StringRef::npos) {
+      errs() << "Invalid dummy symbol specification '" << Mapping
+             << "'. Should be '<symbol name>=<addr>'\n";
+      exit(1);
+    }
+
+    std::string Symbol = Mapping.substr(0, EqualsIdx);
+    std::string AddrStr = Mapping.substr(EqualsIdx + 1);
+
+    uint64_t Addr;
+    if (StringRef(AddrStr).getAsInteger(0, Addr)) {
+      errs() << "Invalid symbol mapping '" << Mapping << "'.\n";
+      exit(1);
+    }
+
+    MemMgr.addDummySymbol(Symbol, Addr);
+  }
 }
 
 // Load and link the objects specified on the command line, but do not execute
@@ -495,7 +588,7 @@ static int linkAndVerify() {
   TripleName = TheTriple.getTriple();
 
   std::unique_ptr<MCSubtargetInfo> STI(
-    TheTarget->createMCSubtargetInfo(TripleName, "", ""));
+    TheTarget->createMCSubtargetInfo(TripleName, MCPU, ""));
   assert(STI && "Unable to create subtarget info!");
 
   std::unique_ptr<MCRegisterInfo> MRI(TheTarget->createMCRegInfo(TripleName));
@@ -557,8 +650,9 @@ static int linkAndVerify() {
     }
   }
 
-  // Re-map the section addresses into the phony target address space.
-  remapSections(TheTriple, MemMgr, Checker);
+  // Re-map the section addresses into the phony target address space and add
+  // dummy symbols.
+  remapSectionsAndSymbols(TheTriple, MemMgr, Checker);
 
   // Resolve all the relocations we can.
   Dyld.resolveRelocations();
@@ -592,8 +686,12 @@ int main(int argc, char **argv) {
   switch (Action) {
   case AC_Execute:
     return executeInput();
+  case AC_PrintDebugLineInfo:
+    return printLineInfoForInput(/* LoadObjects */ true,/* UseDebugObj */ true);
   case AC_PrintLineInfo:
-    return printLineInfoForInput();
+    return printLineInfoForInput(/* LoadObjects */ true,/* UseDebugObj */false);
+  case AC_PrintObjectLineInfo:
+    return printLineInfoForInput(/* LoadObjects */false,/* UseDebugObj */false);
   case AC_Verify:
     return linkAndVerify();
   }
diff --git a/tools/llvm-shlib/CMakeLists.txt b/tools/llvm-shlib/CMakeLists.txt
index bc1b658ba72..54d71d3f632 100644
--- a/tools/llvm-shlib/CMakeLists.txt
+++ b/tools/llvm-shlib/CMakeLists.txt
@@ -15,6 +15,8 @@ if(NOT DEFINED LLVM_DYLIB_COMPONENTS)
     BitWriter
     CodeGen
     Core
+    DebugInfoDWARF
+    DebugInfoPDB
     ExecutionEngine
     IPA
     IPO
diff --git a/tools/llvm-size/llvm-size.cpp b/tools/llvm-size/llvm-size.cpp
index 0e0dd59ce92..9a6e2c1ae4b 100644
--- a/tools/llvm-size/llvm-size.cpp
+++ b/tools/llvm-size/llvm-size.cpp
@@ -97,7 +97,7 @@ static size_t getNumLengthAsString(uint64_t num) {
   return result.size();
 }
 
-/// @brief Return the the printing format for the Radix.
+/// @brief Return the printing format for the Radix.
 static const char *getRadixFmt(void) {
   switch (Radix) {
   case octal:
@@ -122,12 +122,10 @@ static void PrintDarwinSectionSizes(MachOObjectFile *MachO) {
     fmt << "0x";
   fmt << "%" << radix_fmt;
 
-  uint32_t LoadCommandCount = MachO->getHeader().ncmds;
   uint32_t Filetype = MachO->getHeader().filetype;
-  MachOObjectFile::LoadCommandInfo Load = MachO->getFirstLoadCommandInfo();
 
   uint64_t total = 0;
-  for (unsigned I = 0;; ++I) {
+  for (const auto &Load : MachO->load_commands()) {
     if (Load.C.cmd == MachO::LC_SEGMENT_64) {
       MachO::segment_command_64 Seg = MachO->getSegment64LoadCommand(Load);
       outs() << "Segment " << Seg.segname << ": "
@@ -181,10 +179,6 @@ static void PrintDarwinSectionSizes(MachOObjectFile *MachO) {
       if (Seg.nsects != 0)
         outs() << "\ttotal " << format(fmt.str().c_str(), sec_total) << "\n";
     }
-    if (I == LoadCommandCount - 1)
-      break;
-    else
-      Load = MachO->getNextLoadCommandInfo(Load);
   }
   outs() << "total " << format(fmt.str().c_str(), total) << "\n";
 }
@@ -194,14 +188,11 @@ static void PrintDarwinSectionSizes(MachOObjectFile *MachO) {
 /// This is when used when @c OutputFormat is berkeley with a Mach-O file and
 /// produces the same output as darwin's size(1) default output.
 static void PrintDarwinSegmentSizes(MachOObjectFile *MachO) {
-  uint32_t LoadCommandCount = MachO->getHeader().ncmds;
-  MachOObjectFile::LoadCommandInfo Load = MachO->getFirstLoadCommandInfo();
-
   uint64_t total_text = 0;
   uint64_t total_data = 0;
   uint64_t total_objc = 0;
   uint64_t total_others = 0;
-  for (unsigned I = 0;; ++I) {
+  for (const auto &Load : MachO->load_commands()) {
     if (Load.C.cmd == MachO::LC_SEGMENT_64) {
       MachO::segment_command_64 Seg = MachO->getSegment64LoadCommand(Load);
       if (MachO->getHeader().filetype == MachO::MH_OBJECT) {
@@ -255,10 +246,6 @@ static void PrintDarwinSegmentSizes(MachOObjectFile *MachO) {
           total_others += Seg.vmsize;
       }
     }
-    if (I == LoadCommandCount - 1)
-      break;
-    else
-      Load = MachO->getNextLoadCommandInfo(Load);
   }
   uint64_t total = total_text + total_data + total_objc + total_others;
 
diff --git a/tools/llvm-stress/llvm-stress.cpp b/tools/llvm-stress/llvm-stress.cpp
index f5e718ba65b..6a1a248a057 100644
--- a/tools/llvm-stress/llvm-stress.cpp
+++ b/tools/llvm-stress/llvm-stress.cpp
@@ -31,7 +31,8 @@
 #include <set>
 #include <sstream>
 #include <vector>
-using namespace llvm;
+
+namespace llvm {
 
 static cl::opt<unsigned> SeedCL("seed",
   cl::desc("Seed used for randomness"), cl::init(0));
@@ -42,16 +43,39 @@ static cl::opt<std::string>
 OutputFilename("o", cl::desc("Override output filename"),
                cl::value_desc("filename"));
 
-static cl::opt<bool> GenHalfFloat("generate-half-float",
-  cl::desc("Generate half-length floating-point values"), cl::init(false));
-static cl::opt<bool> GenX86FP80("generate-x86-fp80",
-  cl::desc("Generate 80-bit X86 floating-point values"), cl::init(false));
-static cl::opt<bool> GenFP128("generate-fp128",
-  cl::desc("Generate 128-bit floating-point values"), cl::init(false));
-static cl::opt<bool> GenPPCFP128("generate-ppc-fp128",
-  cl::desc("Generate 128-bit PPC floating-point values"), cl::init(false));
-static cl::opt<bool> GenX86MMX("generate-x86-mmx",
-  cl::desc("Generate X86 MMX floating-point values"), cl::init(false));
+namespace cl {
+template <> class parser<Type*> final : public basic_parser<Type*> {
+public:
+  parser(Option &O) : basic_parser(O) {}
+
+  // Parse options as IR types. Return true on error.
+  bool parse(Option &O, StringRef, StringRef Arg, Type *&Value) {
+    auto &Context = getGlobalContext();
+    if      (Arg == "half")      Value = Type::getHalfTy(Context);
+    else if (Arg == "fp128")     Value = Type::getFP128Ty(Context);
+    else if (Arg == "x86_fp80")  Value = Type::getX86_FP80Ty(Context);
+    else if (Arg == "ppc_fp128") Value = Type::getPPC_FP128Ty(Context);
+    else if (Arg == "x86_mmx")   Value = Type::getX86_MMXTy(Context);
+    else if (Arg.startswith("i")) {
+      unsigned N = 0;
+      Arg.drop_front().getAsInteger(10, N);
+      if (N > 0)
+        Value = Type::getIntNTy(Context, N);
+    }
+
+    if (!Value)
+      return O.error("Invalid IR scalar type: '" + Arg + "'!");
+    return false;
+  }
+
+  const char *getValueName() const override { return "IR scalar type"; }
+};
+}
+
+
+static cl::list<Type*> AdditionalScalarTypes("types", cl::CommaSeparated,
+  cl::desc("Additional IR scalar types "
+           "(always includes i1, i8, i16, i32, i64, float and double)"));
 
 namespace {
 /// A utility class to provide a pseudo-random number generator which is
@@ -96,24 +120,21 @@ class Random {
 
 /// Generate an empty function with a default argument list.
 Function *GenEmptyFunction(Module *M) {
-  // Type Definitions
-  std::vector<Type*> ArgsTy;
   // Define a few arguments
   LLVMContext &Context = M->getContext();
-  ArgsTy.push_back(PointerType::get(IntegerType::getInt8Ty(Context), 0));
-  ArgsTy.push_back(PointerType::get(IntegerType::getInt32Ty(Context), 0));
-  ArgsTy.push_back(PointerType::get(IntegerType::getInt64Ty(Context), 0));
-  ArgsTy.push_back(IntegerType::getInt32Ty(Context));
-  ArgsTy.push_back(IntegerType::getInt64Ty(Context));
-  ArgsTy.push_back(IntegerType::getInt8Ty(Context));
-
-  FunctionType *FuncTy = FunctionType::get(Type::getVoidTy(Context), ArgsTy, 0);
+  Type* ArgsTy[] = {
+    Type::getInt8PtrTy(Context),
+    Type::getInt32PtrTy(Context),
+    Type::getInt64PtrTy(Context),
+    Type::getInt32Ty(Context),
+    Type::getInt64Ty(Context),
+    Type::getInt8Ty(Context)
+  };
+
+  auto *FuncTy = FunctionType::get(Type::getVoidTy(Context), ArgsTy, false);
   // Pick a unique name to describe the input parameters
-  std::stringstream ss;
-  ss<<"autogen_SD"<<SeedCL;
-  Function *Func = Function::Create(FuncTy, GlobalValue::ExternalLinkage,
-                                    ss.str(), M);
-
+  Twine Name = "autogen_SD" + Twine{SeedCL};
+  auto *Func = Function::Create(FuncTy, GlobalValue::ExternalLinkage, Name, M);
   Func->setCallingConv(CallingConv::C);
   return Func;
 }
@@ -246,35 +267,22 @@ struct Modifier {
 
   /// Pick a random scalar type.
   Type *pickScalarType() {
-    Type *t = nullptr;
-    do {
-      switch (Ran->Rand() % 30) {
-      case 0: t = Type::getInt1Ty(Context); break;
-      case 1: t = Type::getInt8Ty(Context); break;
-      case 2: t = Type::getInt16Ty(Context); break;
-      case 3: case 4:
-      case 5: t = Type::getFloatTy(Context); break;
-      case 6: case 7:
-      case 8: t = Type::getDoubleTy(Context); break;
-      case 9: case 10:
-      case 11: t = Type::getInt32Ty(Context); break;
-      case 12: case 13:
-      case 14: t = Type::getInt64Ty(Context); break;
-      case 15: case 16:
-      case 17: if (GenHalfFloat) t = Type::getHalfTy(Context); break;
-      case 18: case 19:
-      case 20: if (GenX86FP80) t = Type::getX86_FP80Ty(Context); break;
-      case 21: case 22:
-      case 23: if (GenFP128) t = Type::getFP128Ty(Context); break;
-      case 24: case 25:
-      case 26: if (GenPPCFP128) t = Type::getPPC_FP128Ty(Context); break;
-      case 27: case 28:
-      case 29: if (GenX86MMX) t = Type::getX86_MMXTy(Context); break;
-      default: llvm_unreachable("Invalid scalar value");
-      }
-    } while (t == nullptr);
+    static std::vector<Type*> ScalarTypes;
+    if (ScalarTypes.empty()) {
+      ScalarTypes.assign({
+        Type::getInt1Ty(Context),
+        Type::getInt8Ty(Context),
+        Type::getInt16Ty(Context),
+        Type::getInt32Ty(Context),
+        Type::getInt64Ty(Context),
+        Type::getFloatTy(Context),
+        Type::getDoubleTy(Context)
+      });
+      ScalarTypes.insert(ScalarTypes.end(),
+        AdditionalScalarTypes.begin(), AdditionalScalarTypes.end());
+    }
 
-    return t;
+    return ScalarTypes[Ran->Rand() % ScalarTypes.size()];
   }
 
   /// Basic block to populate
@@ -620,59 +628,45 @@ static void FillFunction(Function *F, Random &R) {
   Modifier::PieceTable PT;
 
   // Consider arguments as legal values.
-  for (Function::arg_iterator it = F->arg_begin(), e = F->arg_end();
-       it != e; ++it)
-    PT.push_back(it);
+  for (auto &arg : F->args())
+    PT.push_back(&arg);
 
   // List of modifiers which add new random instructions.
-  std::vector<Modifier*> Modifiers;
-  std::unique_ptr<Modifier> LM(new LoadModifier(BB, &PT, &R));
-  std::unique_ptr<Modifier> SM(new StoreModifier(BB, &PT, &R));
-  std::unique_ptr<Modifier> EE(new ExtractElementModifier(BB, &PT, &R));
-  std::unique_ptr<Modifier> SHM(new ShuffModifier(BB, &PT, &R));
-  std::unique_ptr<Modifier> IE(new InsertElementModifier(BB, &PT, &R));
-  std::unique_ptr<Modifier> BM(new BinModifier(BB, &PT, &R));
-  std::unique_ptr<Modifier> CM(new CastModifier(BB, &PT, &R));
-  std::unique_ptr<Modifier> SLM(new SelectModifier(BB, &PT, &R));
-  std::unique_ptr<Modifier> PM(new CmpModifier(BB, &PT, &R));
-  Modifiers.push_back(LM.get());
-  Modifiers.push_back(SM.get());
-  Modifiers.push_back(EE.get());
-  Modifiers.push_back(SHM.get());
-  Modifiers.push_back(IE.get());
-  Modifiers.push_back(BM.get());
-  Modifiers.push_back(CM.get());
-  Modifiers.push_back(SLM.get());
-  Modifiers.push_back(PM.get());
+  std::vector<std::unique_ptr<Modifier>> Modifiers;
+  Modifiers.emplace_back(new LoadModifier(BB, &PT, &R));
+  Modifiers.emplace_back(new StoreModifier(BB, &PT, &R));
+  auto SM = Modifiers.back().get();
+  Modifiers.emplace_back(new ExtractElementModifier(BB, &PT, &R));
+  Modifiers.emplace_back(new ShuffModifier(BB, &PT, &R));
+  Modifiers.emplace_back(new InsertElementModifier(BB, &PT, &R));
+  Modifiers.emplace_back(new BinModifier(BB, &PT, &R));
+  Modifiers.emplace_back(new CastModifier(BB, &PT, &R));
+  Modifiers.emplace_back(new SelectModifier(BB, &PT, &R));
+  Modifiers.emplace_back(new CmpModifier(BB, &PT, &R));
 
   // Generate the random instructions
-  AllocaModifier AM(BB, &PT, &R); AM.ActN(5); // Throw in a few allocas
-  ConstModifier COM(BB, &PT, &R);  COM.ActN(40); // Throw in a few constants
+  AllocaModifier{BB, &PT, &R}.ActN(5); // Throw in a few allocas
+  ConstModifier{BB, &PT, &R}.ActN(40); // Throw in a few constants
 
-  for (unsigned i=0; i< SizeCL / Modifiers.size(); ++i)
-    for (std::vector<Modifier*>::iterator it = Modifiers.begin(),
-         e = Modifiers.end(); it != e; ++it) {
-      (*it)->Act();
-    }
+  for (unsigned i = 0; i < SizeCL / Modifiers.size(); ++i)
+    for (auto &Mod : Modifiers)
+      Mod->Act();
 
   SM->ActN(5); // Throw in a few stores.
 }
 
 static void IntroduceControlFlow(Function *F, Random &R) {
   std::vector<Instruction*> BoolInst;
-  for (BasicBlock::iterator it = F->begin()->begin(),
-       e = F->begin()->end(); it != e; ++it) {
-    if (it->getType() == IntegerType::getInt1Ty(F->getContext()))
-      BoolInst.push_back(it);
+  for (auto &Instr : F->front()) {
+    if (Instr.getType() == IntegerType::getInt1Ty(F->getContext()))
+      BoolInst.push_back(&Instr);
   }
 
   std::random_shuffle(BoolInst.begin(), BoolInst.end(), R);
 
-  for (std::vector<Instruction*>::iterator it = BoolInst.begin(),
-       e = BoolInst.end(); it != e; ++it) {
-    Instruction *Instr = *it;
+  for (auto *Instr : BoolInst) {
     BasicBlock *Curr = Instr->getParent();
-    BasicBlock::iterator Loc= Instr;
+    BasicBlock::iterator Loc = Instr;
     BasicBlock *Next = Curr->splitBasicBlock(Loc, "CF");
     Instr->moveBefore(Curr->getTerminator());
     if (Curr != &F->getEntryBlock()) {
@@ -682,13 +676,17 @@ static void IntroduceControlFlow(Function *F, Random &R) {
   }
 }
 
+}
+
 int main(int argc, char **argv) {
+  using namespace llvm;
+
   // Init LLVM, call llvm_shutdown() on exit, parse args, etc.
-  llvm::PrettyStackTraceProgram X(argc, argv);
+  PrettyStackTraceProgram X(argc, argv);
   cl::ParseCommandLineOptions(argc, argv, "llvm codegen stress-tester\n");
   llvm_shutdown_obj Y;
 
-  std::unique_ptr<Module> M(new Module("/tmp/autogen.bc", getGlobalContext()));
+  auto M = make_unique<Module>("/tmp/autogen.bc", getGlobalContext());
   Function *F = GenEmptyFunction(M.get());
 
   // Pick an initial seed value
diff --git a/tools/llvm-symbolizer/CMakeLists.txt b/tools/llvm-symbolizer/CMakeLists.txt
index 6968f1c6216..5df3b17a065 100644
--- a/tools/llvm-symbolizer/CMakeLists.txt
+++ b/tools/llvm-symbolizer/CMakeLists.txt
@@ -5,6 +5,7 @@
 
 set(LLVM_LINK_COMPONENTS
   DebugInfoDWARF
+  DebugInfoPDB
   Object
   Support
   )
diff --git a/tools/llvm-symbolizer/LLVMSymbolize.cpp b/tools/llvm-symbolizer/LLVMSymbolize.cpp
index 36061d7e684..c57c219b11d 100644
--- a/tools/llvm-symbolizer/LLVMSymbolize.cpp
+++ b/tools/llvm-symbolizer/LLVMSymbolize.cpp
@@ -14,8 +14,12 @@
 #include "LLVMSymbolize.h"
 #include "llvm/ADT/STLExtras.h"
 #include "llvm/Config/config.h"
+#include "llvm/DebugInfo/DWARF/DWARFContext.h"
+#include "llvm/DebugInfo/PDB/PDB.h"
+#include "llvm/DebugInfo/PDB/PDBContext.h"
 #include "llvm/Object/ELFObjectFile.h"
 #include "llvm/Object/MachO.h"
+#include "llvm/Object/SymbolSize.h"
 #include "llvm/Support/Casting.h"
 #include "llvm/Support/Compression.h"
 #include "llvm/Support/DataExtractor.h"
@@ -26,6 +30,12 @@
 #include <sstream>
 #include <stdlib.h>
 
+#if defined(_MSC_VER)
+#include <Windows.h>
+#include <DbgHelp.h>
+#pragma comment(lib, "dbghelp.lib")
+#endif
+
 namespace llvm {
 namespace symbolize {
 
@@ -63,31 +73,21 @@ ModuleInfo::ModuleInfo(ObjectFile *Obj, DIContext *DICtx)
       }
     }
   }
-  for (const SymbolRef &Symbol : Module->symbols()) {
-    addSymbol(Symbol, OpdExtractor.get(), OpdAddress);
-  }
-  bool NoSymbolTable = (Module->symbol_begin() == Module->symbol_end());
-  if (NoSymbolTable && Module->isELF()) {
-    // Fallback to dynamic symbol table, if regular symbol table is stripped.
-    std::pair<symbol_iterator, symbol_iterator> IDyn =
-        getELFDynamicSymbolIterators(Module);
-    for (symbol_iterator si = IDyn.first, se = IDyn.second; si != se; ++si) {
-      addSymbol(*si, OpdExtractor.get(), OpdAddress);
-    }
-  }
+  std::vector<std::pair<SymbolRef, uint64_t>> Symbols =
+      computeSymbolSizes(*Module);
+  for (auto &P : Symbols)
+    addSymbol(P.first, P.second, OpdExtractor.get(), OpdAddress);
 }
 
-void ModuleInfo::addSymbol(const SymbolRef &Symbol, DataExtractor *OpdExtractor,
-                           uint64_t OpdAddress) {
-  SymbolRef::Type SymbolType;
-  if (error(Symbol.getType(SymbolType)))
-    return;
+void ModuleInfo::addSymbol(const SymbolRef &Symbol, uint64_t SymbolSize,
+                           DataExtractor *OpdExtractor, uint64_t OpdAddress) {
+  SymbolRef::Type SymbolType = Symbol.getType();
   if (SymbolType != SymbolRef::ST_Function && SymbolType != SymbolRef::ST_Data)
     return;
-  uint64_t SymbolAddress;
-  if (error(Symbol.getAddress(SymbolAddress)) ||
-      SymbolAddress == UnknownAddressOrSize)
+  ErrorOr<uint64_t> SymbolAddressOrErr = Symbol.getAddress();
+  if (error(SymbolAddressOrErr.getError()))
     return;
+  uint64_t SymbolAddress = *SymbolAddressOrErr;
   if (OpdExtractor) {
     // For big-endian PowerPC64 ELF, symbols in the .opd section refer to
     // function descriptors. The first word of the descriptor is a pointer to
@@ -100,17 +100,10 @@ void ModuleInfo::addSymbol(const SymbolRef &Symbol, DataExtractor *OpdExtractor,
         OpdExtractor->isValidOffsetForAddress(OpdOffset32))
       SymbolAddress = OpdExtractor->getAddress(&OpdOffset32);
   }
-  uint64_t SymbolSize;
-  // Getting symbol size is linear for Mach-O files, so assume that symbol
-  // occupies the memory range up to the following symbol.
-  if (isa<MachOObjectFile>(Module))
-    SymbolSize = 0;
-  else if (error(Symbol.getSize(SymbolSize)) ||
-           SymbolSize == UnknownAddressOrSize)
-    return;
-  StringRef SymbolName;
-  if (error(Symbol.getName(SymbolName)))
+  ErrorOr<StringRef> SymbolNameOrErr = Symbol.getName();
+  if (error(SymbolNameOrErr.getError()))
     return;
+  StringRef SymbolName = *SymbolNameOrErr;
   // Mach-O symbol table names have leading underscore, skip it.
   if (Module->isMachO() && SymbolName.size() > 0 && SymbolName[0] == '_')
     SymbolName = SymbolName.drop_front();
@@ -163,6 +156,7 @@ DILineInfo ModuleInfo::symbolizeCode(
 DIInliningInfo ModuleInfo::symbolizeInlinedCode(
     uint64_t ModuleOffset, const LLVMSymbolizer::Options &Opts) const {
   DIInliningInfo InlinedContext;
+
   if (DebugInfoContext) {
     InlinedContext = DebugInfoContext->getInliningInfoForAddress(
         ModuleOffset, getDILineInfoSpecifier(Opts));
@@ -425,7 +419,7 @@ LLVMSymbolizer::getObjectFileFromBinary(Binary *Bin,
     if (I != ObjectFileForArch.end())
       return I->second;
     ErrorOr<std::unique_ptr<ObjectFile>> ParsedObj =
-        UB->getObjectForArch(Triple(ArchName).getArch());
+        UB->getObjectForArch(ArchName);
     if (ParsedObj) {
       Res = ParsedObj.get().get();
       ParsedBinariesAndObjects.push_back(std::move(ParsedObj.get()));
@@ -460,7 +454,20 @@ LLVMSymbolizer::getOrCreateModuleInfo(const std::string &ModuleName) {
     Modules.insert(make_pair(ModuleName, (ModuleInfo *)nullptr));
     return nullptr;
   }
-  DIContext *Context = DIContext::getDWARFContext(*Objects.second);
+  DIContext *Context = nullptr;
+  if (auto CoffObject = dyn_cast<COFFObjectFile>(Objects.first)) {
+    // If this is a COFF object, assume it contains PDB debug information.  If
+    // we don't find any we will fall back to the DWARF case.
+    std::unique_ptr<IPDBSession> Session;
+    PDB_ErrorCode Error = loadDataForEXE(PDB_ReaderType::DIA,
+                                         Objects.first->getFileName(), Session);
+    if (Error == PDB_ErrorCode::Success) {
+      Context = new PDBContext(*CoffObject, std::move(Session),
+                               Opts.RelativeAddresses);
+    }
+  }
+  if (!Context)
+    Context = new DWARFContextInMemory(*Objects.second);
   assert(Context);
   ModuleInfo *Info = new ModuleInfo(Objects.first, Context);
   Modules.insert(make_pair(ModuleName, Info));
@@ -507,7 +514,17 @@ std::string LLVMSymbolizer::DemangleName(const std::string &Name) {
   free(DemangledName);
   return Result;
 #else
-  return Name;
+  char DemangledName[1024] = {0};
+  DWORD result = ::UnDecorateSymbolName(
+      Name.c_str(), DemangledName, 1023,
+      UNDNAME_NO_ACCESS_SPECIFIERS |       // Strip public, private, protected
+          UNDNAME_NO_ALLOCATION_LANGUAGE | // Strip __thiscall, __stdcall, etc
+          UNDNAME_NO_THROW_SIGNATURES |    // Strip throw() specifications
+          UNDNAME_NO_MEMBER_TYPE |      // Strip virtual, static, etc specifiers
+          UNDNAME_NO_MS_KEYWORDS |      // Strip all MS extension keywords
+          UNDNAME_NO_FUNCTION_RETURNS); // Strip function return types
+
+  return (result == 0) ? Name : std::string(DemangledName);
 #endif
 }
 
diff --git a/tools/llvm-symbolizer/LLVMSymbolize.h b/tools/llvm-symbolizer/LLVMSymbolize.h
index c10922133f6..be246c3f871 100644
--- a/tools/llvm-symbolizer/LLVMSymbolize.h
+++ b/tools/llvm-symbolizer/LLVMSymbolize.h
@@ -14,7 +14,7 @@
 #define LLVM_TOOLS_LLVM_SYMBOLIZER_LLVMSYMBOLIZE_H
 
 #include "llvm/ADT/SmallVector.h"
-#include "llvm/DebugInfo/DWARF/DIContext.h"
+#include "llvm/DebugInfo/DIContext.h"
 #include "llvm/Object/MachOUniversal.h"
 #include "llvm/Object/ObjectFile.h"
 #include "llvm/Support/DataExtractor.h"
@@ -35,19 +35,20 @@ class ModuleInfo;
 class LLVMSymbolizer {
 public:
   struct Options {
-    bool UseSymbolTable : 1;
     FunctionNameKind PrintFunctions;
+    bool UseSymbolTable : 1;
     bool PrintInlining : 1;
     bool Demangle : 1;
+    bool RelativeAddresses : 1;
     std::string DefaultArch;
     std::vector<std::string> DsymHints;
-    Options(bool UseSymbolTable = true,
-            FunctionNameKind PrintFunctions = FunctionNameKind::LinkageName,
-            bool PrintInlining = true, bool Demangle = true,
+    Options(FunctionNameKind PrintFunctions = FunctionNameKind::LinkageName,
+            bool UseSymbolTable = true, bool PrintInlining = true,
+            bool Demangle = true, bool RelativeAddresses = false,
             std::string DefaultArch = "")
-        : UseSymbolTable(UseSymbolTable),
-          PrintFunctions(PrintFunctions), PrintInlining(PrintInlining),
-          Demangle(Demangle), DefaultArch(DefaultArch) {}
+        : PrintFunctions(PrintFunctions), UseSymbolTable(UseSymbolTable),
+          PrintInlining(PrintInlining), Demangle(Demangle),
+          RelativeAddresses(RelativeAddresses), DefaultArch(DefaultArch) {}
   };
 
   LLVMSymbolizer(const Options &Opts = Options()) : Opts(Opts) {}
@@ -118,7 +119,7 @@ class ModuleInfo {
                               uint64_t &Size) const;
   // For big-endian PowerPC64 ELF, OpdAddress is the address of the .opd
   // (function descriptor) section and OpdExtractor refers to its contents.
-  void addSymbol(const SymbolRef &Symbol,
+  void addSymbol(const SymbolRef &Symbol, uint64_t SymbolSize,
                  DataExtractor *OpdExtractor = nullptr,
                  uint64_t OpdAddress = 0);
   ObjectFile *Module;
diff --git a/tools/llvm-symbolizer/Makefile b/tools/llvm-symbolizer/Makefile
index c5d2cf7659c..de75befb1c9 100644
--- a/tools/llvm-symbolizer/Makefile
+++ b/tools/llvm-symbolizer/Makefile
@@ -9,7 +9,7 @@
 
 LEVEL := ../..
 TOOLNAME := llvm-symbolizer
-LINK_COMPONENTS := DebugInfoDWARF Object
+LINK_COMPONENTS := DebugInfoDWARF DebugInfoPDB Object
 
 # This tool has no plugins, optimize startup time.
 TOOL_NO_EXPORTS := 1
diff --git a/tools/llvm-symbolizer/llvm-symbolizer.cpp b/tools/llvm-symbolizer/llvm-symbolizer.cpp
index cb40472f9d2..9c9f3adbf60 100644
--- a/tools/llvm-symbolizer/llvm-symbolizer.cpp
+++ b/tools/llvm-symbolizer/llvm-symbolizer.cpp
@@ -17,6 +17,7 @@
 
 #include "LLVMSymbolize.h"
 #include "llvm/ADT/StringRef.h"
+#include "llvm/Support/COM.h"
 #include "llvm/Support/CommandLine.h"
 #include "llvm/Support/Debug.h"
 #include "llvm/Support/FileSystem.h"
@@ -48,8 +49,13 @@ static cl::opt<FunctionNameKind> ClPrintFunctions(
                clEnumValEnd));
 
 static cl::opt<bool>
-ClPrintInlining("inlining", cl::init(true),
-                cl::desc("Print all inlined frames for a given address"));
+    ClUseRelativeAddress("relative-address", cl::init(false),
+                         cl::desc("Interpret addresses as relative addresses"),
+                         cl::ReallyHidden);
+
+static cl::opt<bool>
+    ClPrintInlining("inlining", cl::init(true),
+                    cl::desc("Print all inlined frames for a given address"));
 
 static cl::opt<bool>
 ClDemangle("demangle", cl::init(true), cl::desc("Demangle function names"));
@@ -123,9 +129,12 @@ int main(int argc, char **argv) {
   PrettyStackTraceProgram X(argc, argv);
   llvm_shutdown_obj Y; // Call llvm_shutdown() on exit.
 
+  llvm::sys::InitializeCOMRAII COM(llvm::sys::COMThreadingMode::MultiThreaded);
+
   cl::ParseCommandLineOptions(argc, argv, "llvm-symbolizer\n");
-  LLVMSymbolizer::Options Opts(ClUseSymbolTable, ClPrintFunctions,
-                               ClPrintInlining, ClDemangle, ClDefaultArch);
+  LLVMSymbolizer::Options Opts(ClPrintFunctions, ClUseSymbolTable,
+                               ClPrintInlining, ClDemangle,
+                               ClUseRelativeAddress, ClDefaultArch);
   for (const auto &hint : ClDsymHint) {
     if (sys::path::extension(hint) == ".dSYM") {
       Opts.DsymHints.push_back(hint);
@@ -146,5 +155,6 @@ int main(int argc, char **argv) {
     outs() << Result << "\n";
     outs().flush();
   }
+
   return 0;
 }
diff --git a/tools/lto/CMakeLists.txt b/tools/lto/CMakeLists.txt
index 8922af6829f..0bf9b254d04 100644
--- a/tools/lto/CMakeLists.txt
+++ b/tools/lto/CMakeLists.txt
@@ -6,6 +6,7 @@ set(LLVM_LINK_COMPONENTS
   MCDisassembler
   NaClTransforms
   Support
+  Target
   )
 
 set(SOURCES
diff --git a/tools/lto/Makefile b/tools/lto/Makefile
index 2fb36aebddd..0fb57588b26 100644
--- a/tools/lto/Makefile
+++ b/tools/lto/Makefile
@@ -40,11 +40,3 @@ ifeq ($(HOST_OS),Darwin)
                           -Wl,-object_path_lto -Wl,$(TempFile)
     endif
 endif
-
-#@ LOCALMOD-BEGIN
-# This is to fix an upstream bug. It is in the process of being upstreamed.
-# This line can be removed after it has been fixed upstream and we've merged.
-ifneq ($(HOST_OS),Darwin)
-  LLVMLibsOptions := $(LLVMLibsOptions) -Wl,-soname=$(SharedPrefix)LTO$(SHLIBEXT)
-endif
-#@ LOCALMOD-END
diff --git a/tools/lto/lto.cpp b/tools/lto/lto.cpp
index cc4169fda99..5c712f18c9e 100644
--- a/tools/lto/lto.cpp
+++ b/tools/lto/lto.cpp
@@ -73,7 +73,22 @@ static void lto_initialize() {
   }
 }
 
-DEFINE_SIMPLE_CONVERSION_FUNCTIONS(LTOCodeGenerator, lto_code_gen_t)
+namespace {
+
+// This derived class owns the native object file. This helps implement the
+// libLTO API semantics, which require that the code generator owns the object
+// file.
+struct LibLTOCodeGenerator : LTOCodeGenerator {
+  LibLTOCodeGenerator() {}
+  LibLTOCodeGenerator(std::unique_ptr<LLVMContext> Context)
+      : LTOCodeGenerator(std::move(Context)) {}
+
+  std::unique_ptr<MemoryBuffer> NativeObjectFile;
+};
+
+}
+
+DEFINE_SIMPLE_CONVERSION_FUNCTIONS(LibLTOCodeGenerator, lto_code_gen_t)
 DEFINE_SIMPLE_CONVERSION_FUNCTIONS(LTOModule, lto_module_t)
 
 // Convert the subtarget features into a string to pass to LTOCodeGenerator.
@@ -208,20 +223,8 @@ lto_symbol_attributes lto_module_get_symbol_attribute(lto_module_t mod,
   return unwrap(mod)->getSymbolAttributes(index);
 }
 
-unsigned int lto_module_get_num_deplibs(lto_module_t mod) {
-  return unwrap(mod)->getDependentLibraryCount();
-}
-
-const char* lto_module_get_deplib(lto_module_t mod, unsigned int index) {
-  return unwrap(mod)->getDependentLibrary(index);
-}
-
-unsigned int lto_module_get_num_linkeropts(lto_module_t mod) {
-  return unwrap(mod)->getLinkerOptCount();
-}
-
-const char* lto_module_get_linkeropt(lto_module_t mod, unsigned int index) {
-  return unwrap(mod)->getLinkerOpt(index);
+const char* lto_module_get_linkeropts(lto_module_t mod) {
+  return unwrap(mod)->getLinkerOpts();
 }
 
 void lto_codegen_set_diagnostic_handler(lto_code_gen_t cg,
@@ -235,11 +238,10 @@ static lto_code_gen_t createCodeGen(bool InLocalContext) {
 
   TargetOptions Options = InitTargetOptionsFromCodeGenFlags();
 
-  LTOCodeGenerator *CodeGen =
-      InLocalContext ? new LTOCodeGenerator(make_unique<LLVMContext>())
-                     : new LTOCodeGenerator();
-  if (CodeGen)
-    CodeGen->setTargetOptions(Options);
+  LibLTOCodeGenerator *CodeGen =
+      InLocalContext ? new LibLTOCodeGenerator(make_unique<LLVMContext>())
+                     : new LibLTOCodeGenerator();
+  CodeGen->setTargetOptions(Options);
   return wrap(CodeGen);
 }
 
@@ -304,9 +306,13 @@ bool lto_codegen_write_merged_modules(lto_code_gen_t cg, const char *path) {
 
 const void *lto_codegen_compile(lto_code_gen_t cg, size_t *length) {
   maybeParseOptions(cg);
-  return unwrap(cg)->compile(length, DisableInline,
-                             DisableGVNLoadPRE, DisableLTOVectorization,
-                             sLastErrorString);
+  LibLTOCodeGenerator *CG = unwrap(cg);
+  CG->NativeObjectFile = CG->compile(DisableInline, DisableGVNLoadPRE,
+                                     DisableLTOVectorization, sLastErrorString);
+  if (!CG->NativeObjectFile)
+    return nullptr;
+  *length = CG->NativeObjectFile->getBufferSize();
+  return CG->NativeObjectFile->getBufferStart();
 }
 
 bool lto_codegen_optimize(lto_code_gen_t cg) {
@@ -318,7 +324,12 @@ bool lto_codegen_optimize(lto_code_gen_t cg) {
 
 const void *lto_codegen_compile_optimized(lto_code_gen_t cg, size_t *length) {
   maybeParseOptions(cg);
-  return unwrap(cg)->compileOptimized(length, sLastErrorString);
+  LibLTOCodeGenerator *CG = unwrap(cg);
+  CG->NativeObjectFile = CG->compileOptimized(sLastErrorString);
+  if (!CG->NativeObjectFile)
+    return nullptr;
+  *length = CG->NativeObjectFile->getBufferSize();
+  return CG->NativeObjectFile->getBufferStart();
 }
 
 bool lto_codegen_compile_to_file(lto_code_gen_t cg, const char **name) {
@@ -333,3 +344,13 @@ void lto_codegen_debug_options(lto_code_gen_t cg, const char *opt) {
 }
 
 unsigned int lto_api_version() { return LTO_API_VERSION; }
+
+void lto_codegen_set_should_internalize(lto_code_gen_t cg,
+                                        bool ShouldInternalize) {
+  unwrap(cg)->setShouldInternalize(ShouldInternalize);
+}
+
+void lto_codegen_set_should_embed_uselists(lto_code_gen_t cg,
+                                           lto_bool_t ShouldEmbedUselists) {
+  unwrap(cg)->setShouldEmbedUselists(ShouldEmbedUselists);
+}
diff --git a/tools/lto/lto.exports b/tools/lto/lto.exports
index 8729050123e..8bc2b0f9d31 100644
--- a/tools/lto/lto.exports
+++ b/tools/lto/lto.exports
@@ -8,10 +8,7 @@ lto_module_create_from_memory
 lto_module_create_from_memory_with_path
 lto_module_create_in_local_context
 lto_module_create_in_codegen_context
-lto_module_get_deplib
-lto_module_get_linkeropt
-lto_module_get_num_deplibs
-lto_module_get_num_linkeropts
+lto_module_get_linkeropts
 lto_module_get_num_symbols
 lto_module_get_symbol_attribute
 lto_module_get_symbol_name
@@ -41,6 +38,8 @@ lto_codegen_set_cpu
 lto_codegen_compile_to_file
 lto_codegen_optimize
 lto_codegen_compile_optimized
+lto_codegen_set_should_internalize
+lto_codegen_set_should_embed_uselists
 LLVMCreateDisasm
 LLVMCreateDisasmCPU
 LLVMDisasmDispose
diff --git a/tools/macho-dump/macho-dump.cpp b/tools/macho-dump/macho-dump.cpp
index 604f93adfbb..39c2860df35 100644
--- a/tools/macho-dump/macho-dump.cpp
+++ b/tools/macho-dump/macho-dump.cpp
@@ -340,7 +340,7 @@ DumpDylibID(const MachOObjectFile &Obj,
 }
 
 static int DumpLoadCommand(const MachOObjectFile &Obj,
-                           MachOObjectFile::LoadCommandInfo &LCI) {
+                           const MachOObjectFile::LoadCommandInfo &LCI) {
   switch (LCI.C.cmd) {
   case MachO::LC_SEGMENT:
     return DumpSegmentCommand(Obj, LCI);
@@ -369,9 +369,8 @@ static int DumpLoadCommand(const MachOObjectFile &Obj,
   }
 }
 
-
 static int DumpLoadCommand(const MachOObjectFile &Obj, unsigned Index,
-                           MachOObjectFile::LoadCommandInfo &LCI) {
+                           const MachOObjectFile::LoadCommandInfo &LCI) {
   outs() << "  # Load Command " << Index << "\n"
          << " (('command', " << LCI.C.cmd << ")\n"
          << "  ('size', " << LCI.C.cmdsize << ")\n";
@@ -423,16 +422,11 @@ int main(int argc, char **argv) {
 
   // Print the load commands.
   int Res = 0;
-  MachOObjectFile::LoadCommandInfo Command =
-    InputObject->getFirstLoadCommandInfo();
+  unsigned Index = 0;
   outs() << "('load_commands', [\n";
-  for (unsigned i = 0; ; ++i) {
-    if (DumpLoadCommand(*InputObject, i, Command))
-      break;
-
-    if (i == Header->ncmds - 1)
+  for (const auto &Load : InputObject->load_commands()) {
+    if (DumpLoadCommand(*InputObject, Index++, Load))
       break;
-    Command = InputObject->getNextLoadCommandInfo(Command);
   }
   outs() << "])\n";
 
diff --git a/tools/obj2yaml/coff2yaml.cpp b/tools/obj2yaml/coff2yaml.cpp
index 5baa644557c..f675bfe4e61 100644
--- a/tools/obj2yaml/coff2yaml.cpp
+++ b/tools/obj2yaml/coff2yaml.cpp
@@ -120,7 +120,10 @@ void COFFDumper::dumpSections(unsigned NumSections) {
       const object::coff_relocation *reloc = Obj.getCOFFRelocation(Reloc);
       COFFYAML::Relocation Rel;
       object::symbol_iterator Sym = Reloc.getSymbol();
-      Sym->getName(Rel.SymbolName);
+      ErrorOr<StringRef> SymbolNameOrErr = Sym->getName();
+      if (std::error_code EC = SymbolNameOrErr.getError())
+        report_fatal_error(EC.message());
+      Rel.SymbolName = *SymbolNameOrErr;
       Rel.VirtualAddress = reloc->VirtualAddress;
       Rel.Type = reloc->Type;
       Relocations.push_back(Rel);
@@ -271,5 +274,5 @@ std::error_code coff2yaml(raw_ostream &Out, const object::COFFObjectFile &Obj) {
   yaml::Output Yout(Out);
   Yout << Dumper.getYAMLObj();
 
-  return object::object_error::success;
+  return std::error_code();
 }
diff --git a/tools/obj2yaml/elf2yaml.cpp b/tools/obj2yaml/elf2yaml.cpp
index e606df2a084..2c6fc1bb09c 100644
--- a/tools/obj2yaml/elf2yaml.cpp
+++ b/tools/obj2yaml/elf2yaml.cpp
@@ -23,12 +23,12 @@ template <class ELFT>
 class ELFDumper {
   typedef object::Elf_Sym_Impl<ELFT> Elf_Sym;
   typedef typename object::ELFFile<ELFT>::Elf_Shdr Elf_Shdr;
-  typedef typename object::ELFFile<ELFT>::Elf_Sym_Iter Elf_Sym_Iter;
   typedef typename object::ELFFile<ELFT>::Elf_Word Elf_Word;
 
   const object::ELFFile<ELFT> &Obj;
 
-  std::error_code dumpSymbol(Elf_Sym_Iter Sym, ELFYAML::Symbol &S);
+  std::error_code dumpSymbol(const Elf_Sym *Sym, StringRef StrTable,
+                             ELFYAML::Symbol &S);
   std::error_code dumpCommonSection(const Elf_Shdr *Shdr, ELFYAML::Section &S);
   std::error_code dumpCommonRelocationSection(const Elf_Shdr *Shdr,
                                               ELFYAML::RelocationSection &S);
@@ -40,7 +40,9 @@ class ELFDumper {
   ErrorOr<ELFYAML::RelocationSection *> dumpRelaSection(const Elf_Shdr *Shdr);
   ErrorOr<ELFYAML::RawContentSection *>
   dumpContentSection(const Elf_Shdr *Shdr);
+  ErrorOr<ELFYAML::NoBitsSection *> dumpNoBitsSection(const Elf_Shdr *Shdr);
   ErrorOr<ELFYAML::Group *> dumpGroup(const Elf_Shdr *Shdr);
+  ErrorOr<ELFYAML::MipsABIFlags *> dumpMipsABIFlags(const Elf_Shdr *Shdr);
 
 public:
   ELFDumper(const object::ELFFile<ELFT> &O);
@@ -96,6 +98,20 @@ ErrorOr<ELFYAML::Object *> ELFDumper<ELFT>::dump() {
       Y->Sections.push_back(std::unique_ptr<ELFYAML::Section>(G.get()));
       break;
     }
+    case ELF::SHT_MIPS_ABIFLAGS: {
+      ErrorOr<ELFYAML::MipsABIFlags *> G = dumpMipsABIFlags(&Sec);
+      if (std::error_code EC = G.getError())
+        return EC;
+      Y->Sections.push_back(std::unique_ptr<ELFYAML::Section>(G.get()));
+      break;
+    }
+    case ELF::SHT_NOBITS: {
+      ErrorOr<ELFYAML::NoBitsSection *> S = dumpNoBitsSection(&Sec);
+      if (std::error_code EC = S.getError())
+        return EC;
+      Y->Sections.push_back(std::unique_ptr<ELFYAML::Section>(S.get()));
+      break;
+    }
     default: {
       ErrorOr<ELFYAML::RawContentSection *> S = dumpContentSection(&Sec);
       if (std::error_code EC = S.getError())
@@ -106,18 +122,24 @@ ErrorOr<ELFYAML::Object *> ELFDumper<ELFT>::dump() {
   }
 
   // Dump symbols
+  const Elf_Shdr *Symtab = Obj.getDotSymtabSec();
+  ErrorOr<StringRef> StrTableOrErr = Obj.getStringTableForSymtab(*Symtab);
+  if (std::error_code EC = StrTableOrErr.getError())
+    return EC;
+  StringRef StrTable = *StrTableOrErr;
+
   bool IsFirstSym = true;
-  for (auto SI = Obj.begin_symbols(), SE = Obj.end_symbols(); SI != SE; ++SI) {
+  for (const Elf_Sym &Sym : Obj.symbols()) {
     if (IsFirstSym) {
       IsFirstSym = false;
       continue;
     }
 
     ELFYAML::Symbol S;
-    if (std::error_code EC = ELFDumper<ELFT>::dumpSymbol(SI, S))
+    if (std::error_code EC = ELFDumper<ELFT>::dumpSymbol(&Sym, StrTable, S))
       return EC;
 
-    switch (SI->getBinding())
+    switch (Sym.getBinding())
     {
     case ELF::STB_LOCAL:
       Y->Symbols.Local.push_back(S);
@@ -137,19 +159,23 @@ ErrorOr<ELFYAML::Object *> ELFDumper<ELFT>::dump() {
 }
 
 template <class ELFT>
-std::error_code ELFDumper<ELFT>::dumpSymbol(Elf_Sym_Iter Sym,
+std::error_code ELFDumper<ELFT>::dumpSymbol(const Elf_Sym *Sym,
+                                            StringRef StrTable,
                                             ELFYAML::Symbol &S) {
   S.Type = Sym->getType();
   S.Value = Sym->st_value;
   S.Size = Sym->st_size;
   S.Other = Sym->st_other;
 
-  ErrorOr<StringRef> NameOrErr = Obj.getSymbolName(Sym);
+  ErrorOr<StringRef> NameOrErr = Sym->getName(StrTable);
   if (std::error_code EC = NameOrErr.getError())
     return EC;
   S.Name = NameOrErr.get();
 
-  const Elf_Shdr *Shdr = Obj.getSection(&*Sym);
+  ErrorOr<const Elf_Shdr *> ShdrOrErr = Obj.getSection(&*Sym);
+  if (std::error_code EC = ShdrOrErr.getError())
+    return EC;
+  const Elf_Shdr *Shdr = *ShdrOrErr;
   if (!Shdr)
     return obj2yaml_error::success;
 
@@ -174,8 +200,16 @@ std::error_code ELFDumper<ELFT>::dumpRelocation(const Elf_Shdr *Shdr,
   if (!NamePair.first)
     return obj2yaml_error::success;
 
-  ErrorOr<StringRef> NameOrErr =
-      Obj.getSymbolName(NamePair.first, NamePair.second);
+  const Elf_Shdr *SymTab = NamePair.first;
+  ErrorOr<const Elf_Shdr *> StrTabSec = Obj.getSection(SymTab->sh_link);
+  if (std::error_code EC = StrTabSec.getError())
+    return EC;
+  ErrorOr<StringRef> StrTabOrErr = Obj.getStringTable(*StrTabSec);
+  if (std::error_code EC = StrTabOrErr.getError())
+    return EC;
+  StringRef StrTab = *StrTabOrErr;
+
+  ErrorOr<StringRef> NameOrErr = NamePair.second->getName(StrTab);
   if (std::error_code EC = NameOrErr.getError())
     return EC;
   R.Symbol = NameOrErr.get();
@@ -197,12 +231,13 @@ std::error_code ELFDumper<ELFT>::dumpCommonSection(const Elf_Shdr *Shdr,
   S.Name = NameOrErr.get();
 
   if (Shdr->sh_link != ELF::SHN_UNDEF) {
-    if (const Elf_Shdr *LinkSection = Obj.getSection(Shdr->sh_link)) {
-      NameOrErr = Obj.getSectionName(LinkSection);
-      if (std::error_code EC = NameOrErr.getError())
-        return EC;
-      S.Link = NameOrErr.get();
-    }
+    ErrorOr<const Elf_Shdr *> LinkSection = Obj.getSection(Shdr->sh_link);
+    if (std::error_code EC = LinkSection.getError())
+      return EC;
+    NameOrErr = Obj.getSectionName(*LinkSection);
+    if (std::error_code EC = NameOrErr.getError())
+      return EC;
+    S.Link = NameOrErr.get();
   }
 
   return obj2yaml_error::success;
@@ -215,12 +250,14 @@ ELFDumper<ELFT>::dumpCommonRelocationSection(const Elf_Shdr *Shdr,
   if (std::error_code EC = dumpCommonSection(Shdr, S))
     return EC;
 
-  if (const Elf_Shdr *InfoSection = Obj.getSection(Shdr->sh_info)) {
-    ErrorOr<StringRef> NameOrErr = Obj.getSectionName(InfoSection);
-    if (std::error_code EC = NameOrErr.getError())
-      return EC;
-    S.Info = NameOrErr.get();
-  }
+  ErrorOr<const Elf_Shdr *> InfoSection = Obj.getSection(Shdr->sh_info);
+  if (std::error_code EC = InfoSection.getError())
+    return EC;
+
+  ErrorOr<StringRef> NameOrErr = Obj.getSectionName(*InfoSection);
+  if (std::error_code EC = NameOrErr.getError())
+    return EC;
+  S.Info = NameOrErr.get();
 
   return obj2yaml_error::success;
 }
@@ -234,8 +271,7 @@ ELFDumper<ELFT>::dumpRelSection(const Elf_Shdr *Shdr) {
   if (std::error_code EC = dumpCommonRelocationSection(Shdr, *S))
     return EC;
 
-  for (auto RI = Obj.begin_rel(Shdr), RE = Obj.end_rel(Shdr); RI != RE;
-       ++RI) {
+  for (auto RI = Obj.rel_begin(Shdr), RE = Obj.rel_end(Shdr); RI != RE; ++RI) {
     ELFYAML::Relocation R;
     if (std::error_code EC = dumpRelocation(Shdr, &*RI, R))
       return EC;
@@ -254,7 +290,7 @@ ELFDumper<ELFT>::dumpRelaSection(const Elf_Shdr *Shdr) {
   if (std::error_code EC = dumpCommonRelocationSection(Shdr, *S))
     return EC;
 
-  for (auto RI = Obj.begin_rela(Shdr), RE = Obj.end_rela(Shdr); RI != RE;
+  for (auto RI = Obj.rela_begin(Shdr), RE = Obj.rela_end(Shdr); RI != RE;
        ++RI) {
     ELFYAML::Relocation R;
     if (std::error_code EC = dumpRelocation(Shdr, &*RI, R))
@@ -283,6 +319,18 @@ ELFDumper<ELFT>::dumpContentSection(const Elf_Shdr *Shdr) {
   return S.release();
 }
 
+template <class ELFT>
+ErrorOr<ELFYAML::NoBitsSection *>
+ELFDumper<ELFT>::dumpNoBitsSection(const Elf_Shdr *Shdr) {
+  auto S = make_unique<ELFYAML::NoBitsSection>();
+
+  if (std::error_code EC = dumpCommonSection(Shdr, *S))
+    return EC;
+  S->Size = Shdr->sh_size;
+
+  return S.release();
+}
+
 template <class ELFT>
 ErrorOr<ELFYAML::Group *> ELFDumper<ELFT>::dumpGroup(const Elf_Shdr *Shdr) {
   auto S = make_unique<ELFYAML::Group>();
@@ -290,12 +338,19 @@ ErrorOr<ELFYAML::Group *> ELFDumper<ELFT>::dumpGroup(const Elf_Shdr *Shdr) {
   if (std::error_code EC = dumpCommonSection(Shdr, *S))
     return EC;
   // Get sh_info which is the signature.
-  const Elf_Sym *symbol = Obj.getSymbol(Shdr->sh_info);
-  const Elf_Shdr *symtab = Obj.getSection(Shdr->sh_link);
+  ErrorOr<const Elf_Shdr *> SymtabOrErr = Obj.getSection(Shdr->sh_link);
+  if (std::error_code EC = SymtabOrErr.getError())
+    return EC;
+  const Elf_Shdr *Symtab = *SymtabOrErr;
+  const Elf_Sym *symbol = Obj.getSymbol(Symtab, Shdr->sh_info);
+  ErrorOr<StringRef> StrTabOrErr = Obj.getStringTableForSymtab(*Symtab);
+  if (std::error_code EC = StrTabOrErr.getError())
+    return EC;
+  StringRef StrTab = *StrTabOrErr;
   auto sectionContents = Obj.getSectionContents(Shdr);
   if (std::error_code ec = sectionContents.getError())
     return ec;
-  ErrorOr<StringRef> symbolName = Obj.getSymbolName(symtab, symbol);
+  ErrorOr<StringRef> symbolName = symbol->getName(StrTab);
   if (std::error_code EC = symbolName.getError())
     return EC;
   S->Info = *symbolName;
@@ -307,8 +362,10 @@ ErrorOr<ELFYAML::Group *> ELFDumper<ELFT>::dumpGroup(const Elf_Shdr *Shdr) {
     if (groupMembers[i] == llvm::ELF::GRP_COMDAT) {
       s.sectionNameOrType = "GRP_COMDAT";
     } else {
-      const Elf_Shdr *sHdr = Obj.getSection(groupMembers[i]);
-      ErrorOr<StringRef> sectionName = Obj.getSectionName(sHdr);
+      ErrorOr<const Elf_Shdr *> sHdr = Obj.getSection(groupMembers[i]);
+      if (std::error_code EC = sHdr.getError())
+        return EC;
+      ErrorOr<StringRef> sectionName = Obj.getSectionName(*sHdr);
       if (std::error_code ec = sectionName.getError())
         return ec;
       s.sectionNameOrType = *sectionName;
@@ -318,6 +375,35 @@ ErrorOr<ELFYAML::Group *> ELFDumper<ELFT>::dumpGroup(const Elf_Shdr *Shdr) {
   return S.release();
 }
 
+template <class ELFT>
+ErrorOr<ELFYAML::MipsABIFlags *>
+ELFDumper<ELFT>::dumpMipsABIFlags(const Elf_Shdr *Shdr) {
+  assert(Shdr->sh_type == ELF::SHT_MIPS_ABIFLAGS &&
+         "Section type is not SHT_MIPS_ABIFLAGS");
+  auto S = make_unique<ELFYAML::MipsABIFlags>();
+  if (std::error_code EC = dumpCommonSection(Shdr, *S))
+    return EC;
+
+  ErrorOr<ArrayRef<uint8_t>> ContentOrErr = Obj.getSectionContents(Shdr);
+  if (std::error_code EC = ContentOrErr.getError())
+    return EC;
+
+  auto *Flags = reinterpret_cast<const object::Elf_Mips_ABIFlags<ELFT> *>(
+      ContentOrErr.get().data());
+  S->Version = Flags->version;
+  S->ISALevel = Flags->isa_level;
+  S->ISARevision = Flags->isa_rev;
+  S->GPRSize = Flags->gpr_size;
+  S->CPR1Size = Flags->cpr1_size;
+  S->CPR2Size = Flags->cpr2_size;
+  S->FpABI = Flags->fp_abi;
+  S->ISAExtension = Flags->isa_ext;
+  S->ASEs = Flags->ases;
+  S->Flags1 = Flags->flags1;
+  S->Flags2 = Flags->flags2;
+  return S.release();
+}
+
 template <class ELFT>
 static std::error_code elf2yaml(raw_ostream &Out,
                                 const object::ELFFile<ELFT> &Obj) {
@@ -330,7 +416,7 @@ static std::error_code elf2yaml(raw_ostream &Out,
   yaml::Output Yout(Out);
   Yout << *YAML;
 
-  return object::object_error::success;
+  return std::error_code();
 }
 
 std::error_code elf2yaml(raw_ostream &Out, const object::ObjectFile &Obj) {
diff --git a/tools/opt/BreakpointPrinter.cpp b/tools/opt/BreakpointPrinter.cpp
index 8f390a10e3b..363a7cd8007 100644
--- a/tools/opt/BreakpointPrinter.cpp
+++ b/tools/opt/BreakpointPrinter.cpp
@@ -29,13 +29,13 @@ struct BreakpointPrinter : public ModulePass {
 
   BreakpointPrinter(raw_ostream &out) : ModulePass(ID), Out(out) {}
 
-  void getContextName(DIDescriptor Context, std::string &N) {
-    if (auto *NS = dyn_cast<MDNamespace>(Context)) {
+  void getContextName(const DIScope *Context, std::string &N) {
+    if (auto *NS = dyn_cast<DINamespace>(Context)) {
       if (!NS->getName().empty()) {
         getContextName(NS->getScope(), N);
         N = N + NS->getName().str() + "::";
       }
-    } else if (DIType TY = dyn_cast<MDType>(Context)) {
+    } else if (auto *TY = dyn_cast<DIType>(Context)) {
       if (!TY->getName().empty()) {
         getContextName(TY->getScope().resolve(TypeIdentifierMap), N);
         N = N + TY->getName().str() + "::";
@@ -53,7 +53,7 @@ struct BreakpointPrinter : public ModulePass {
     if (NamedMDNode *NMD = M.getNamedMetadata("llvm.dbg.sp"))
       for (unsigned i = 0, e = NMD->getNumOperands(); i != e; ++i) {
         std::string Name;
-        auto *SP = cast_or_null<MDSubprogram>(NMD->getOperand(i));
+        auto *SP = cast_or_null<DISubprogram>(NMD->getOperand(i));
         if (!SP)
           continue;
         getContextName(SP->getScope().resolve(TypeIdentifierMap), Name);
diff --git a/tools/opt/CMakeLists.txt b/tools/opt/CMakeLists.txt
index cddfe9e67b7..d67b60ae32b 100644
--- a/tools/opt/CMakeLists.txt
+++ b/tools/opt/CMakeLists.txt
@@ -9,11 +9,7 @@ set(LLVM_LINK_COMPONENTS
   IRReader
   Linker
   # @LOCALMOD-BEGIN
-  NaClAnalysis
-  NaClBitWriter
-  NaClBitReader
   NaClTransforms
-  MinSFITransforms
   JSBackend
   # @LOCALMOD-END
   InstCombine
diff --git a/tools/opt/LLVMBuild.txt b/tools/opt/LLVMBuild.txt
index 03f20b75abe..3d406ffdf92 100644
--- a/tools/opt/LLVMBuild.txt
+++ b/tools/opt/LLVMBuild.txt
@@ -19,4 +19,16 @@
 type = Tool
 name = opt
 parent = Tools
-required_libraries = NaClBitWriter NaClTransforms NaClAnalysis MinSFITransforms AsmParser BitReader BitWriter CodeGen IRReader IPO Instrumentation Linker Scalar ObjCARC Passes all-targets
+required_libraries =
+ NaClTransforms
+ AsmParser
+ BitReader
+ BitWriter
+ CodeGen
+ IRReader
+ IPO
+ Instrumentation
+ Scalar
+ ObjCARC
+ Passes
+ all-targets
diff --git a/tools/opt/Makefile b/tools/opt/Makefile
index 20737abb51f..688fc8690dd 100644
--- a/tools/opt/Makefile
+++ b/tools/opt/Makefile
@@ -9,7 +9,7 @@
 
 LEVEL := ../..
 TOOLNAME := opt
-LINK_COMPONENTS := naclbitwriter nacltransforms naclanalysis minsfitransforms jsbackend bitreader bitwriter asmparser irreader instrumentation scalaropts objcarcopts ipo vectorize all-targets codegen passes linker
+LINK_COMPONENTS := nacltransforms jsbackend bitreader bitwriter asmparser irreader instrumentation scalaropts objcarcopts ipo vectorize all-targets codegen passes linker
 
 # Support plugins.
 NO_DEAD_STRIP := 1
diff --git a/tools/opt/opt.cpp b/tools/opt/opt.cpp
index 18221607e8d..4dd2e796068 100644
--- a/tools/opt/opt.cpp
+++ b/tools/opt/opt.cpp
@@ -23,7 +23,6 @@
 #include "llvm/Analysis/TargetLibraryInfo.h"
 #include "llvm/Analysis/TargetTransformInfo.h"
 #include "llvm/Bitcode/BitcodeWriterPass.h"
-#include "llvm/Bitcode/NaCl/NaClBitcodeWriterPass.h" // @LOCALMOD
 #include "llvm/CodeGen/CommandFlags.h"
 #include "llvm/IR/DataLayout.h"
 #include "llvm/IR/DebugInfo.h"
@@ -53,7 +52,6 @@
 #include "llvm/Support/ToolOutputFile.h"
 #include "llvm/Target/TargetMachine.h"
 #include "llvm/Transforms/IPO/PassManagerBuilder.h"
-#include "llvm/Transforms/MinSFI.h"  // @LOCALMOD
 #include "llvm/Transforms/NaCl.h"  // @LOCALMOD
 #include <algorithm>
 #include <memory>
@@ -149,10 +147,6 @@ static cl::opt<bool>
 PNaClABISimplifyPostOpt(
     "pnacl-abi-simplify-postopt",
     cl::desc("PNaCl ABI simplifications for after optimizations"));
-
-static cl::opt<bool>
-MinSFI("minsfi",
-       cl::desc("MinSFI sandboxing"));
 // @LOCALMOD-END
 
 static cl::opt<std::string>
@@ -204,18 +198,6 @@ static cl::opt<bool> PreserveBitcodeUseListOrder(
     "preserve-bc-uselistorder",
     cl::desc("Preserve use-list order when writing LLVM bitcode."),
     cl::init(true), cl::Hidden);
-// @LOCALMOD-BEGIN
-static cl::opt<NaClFileFormat>
-OutputFileFormat(
-    "bitcode-format",
-    cl::desc("Define format of generated bitcode file:"),
-    cl::values(
-        clEnumValN(LLVMFormat, "llvm", "LLVM bitcode file (default)"),
-        clEnumValN(PNaClFormat, "pnacl", "PNaCl bitcode file"),
-        clEnumValEnd),
-    cl::init(LLVMFormat));
-extern bool OutputFileFormatIsPNaCl;
-// @LOCALMOD-END
 
 static cl::opt<bool> PreserveAssemblyUseListOrder(
     "preserve-ll-uselistorder",
@@ -296,7 +278,9 @@ static CodeGenOpt::Level GetCodeGenOptLevel() {
 }
 
 // Returns the TargetMachine instance or zero if no triple is provided.
-static TargetMachine* GetTargetMachine(Triple TheTriple) {
+static TargetMachine* GetTargetMachine(Triple TheTriple, StringRef CPUStr,
+                                       StringRef FeaturesStr,
+                                       const TargetOptions &Options) {
   std::string Error;
   // @LOCALMOD-BEGIN: Some optimization passes like SimplifyCFG do nice
   // things for code size, but only do it if the TTI says it is okay.
@@ -313,33 +297,8 @@ static TargetMachine* GetTargetMachine(Triple TheTriple) {
     return nullptr;
   }
 
-  // Package up features to be passed to target/subtarget
-  std::string FeaturesStr;
-  if (MAttrs.size() || MCPU == "native") {
-    SubtargetFeatures Features;
-
-    // If user asked for the 'native' CPU, we need to autodetect features.
-    // This is necessary for x86 where the CPU might not support all the
-    // features the autodetected CPU name lists in the target. For example,
-    // not all Sandybridge processors support AVX.
-    if (MCPU == "native") {
-      StringMap<bool> HostFeatures;
-      if (sys::getHostCPUFeatures(HostFeatures))
-        for (auto &F : HostFeatures)
-          Features.AddFeature(F.first(), F.second);
-    }
-
-    for (unsigned i = 0; i != MAttrs.size(); ++i)
-      Features.AddFeature(MAttrs[i]);
-    FeaturesStr = Features.getString();
-  }
-
-  if (MCPU == "native")
-    MCPU = sys::getHostCPUName();
-
   return TheTarget->createTargetMachine(TheTriple.getTriple(),
-                                        MCPU, FeaturesStr,
-                                        InitTargetOptionsFromCodeGenFlags(),
+                                        CPUStr, FeaturesStr, Options,
                                         RelocModel, CMModel,
                                         GetCodeGenOptLevel());
 }
@@ -387,6 +346,7 @@ int main(int argc, char **argv) {
   initializeRewriteSymbolsPass(Registry);
   initializeWinEHPreparePass(Registry);
   initializeDwarfEHPreparePass(Registry);
+  initializeSjLjEHPreparePass(Registry);
 
 #ifdef LINK_POLLY_INTO_TOOLS
   polly::initializePollyPasses(Registry);
@@ -394,12 +354,10 @@ int main(int argc, char **argv) {
 
   // @LOCALMOD-BEGIN
   initializeAddPNaClExternalDeclsPass(Registry);
-  initializeAllocateDataSegmentPass(Registry);
   initializeBackendCanonicalizePass(Registry);
   initializeCanonicalizeMemIntrinsicsPass(Registry);
   initializeCleanupUsedGlobalsMetadataPass(Registry);
   initializeConstantInsertExtractElementIndexPass(Registry);
-  initializeExpandAllocasPass(Registry);
   initializeExpandArithWithOverflowPass(Registry);
   initializeExpandByValPass(Registry);
   initializeExpandConstantExprPass(Registry);
@@ -420,28 +378,21 @@ int main(int argc, char **argv) {
   initializeInsertDivideCheckPass(Registry);
   initializeInternalizeUsedGlobalsPass(Registry);
   initializeNormalizeAlignmentPass(Registry);
-  initializePNaClABIVerifyFunctionsPass(Registry);
-  initializePNaClABIVerifyModulePass(Registry);
   initializePNaClSjLjEHPass(Registry);
   initializePromoteI1OpsPass(Registry);
   initializePromoteIntegersPass(Registry);
   initializeRemoveAsmMemoryPass(Registry);
-  initializeRenameEntryPointPass(Registry);
   initializeReplacePtrsWithIntsPass(Registry);
   initializeResolveAliasesPass(Registry);
   initializeResolvePNaClIntrinsicsPass(Registry);
   initializeRewriteAtomicsPass(Registry);
   initializeRewriteLLVMIntrinsicsPass(Registry);
   initializeRewritePNaClLibraryCallsPass(Registry);
-  initializeSandboxIndirectCallsPass(Registry);
-  initializeSandboxMemoryAccessesPass(Registry);
   initializeSimplifyAllocasPass(Registry);
   initializeSimplifyStructRegSignaturesPass(Registry);
   initializeStripAttributesPass(Registry);
   initializeStripMetadataPass(Registry);
   initializeStripModuleFlagsPass(Registry);
-  initializeStripTlsPass(Registry);
-  initializeSubstituteUndefsPass(Registry);
   // Emscripten passes:
   initializeExpandI64Pass(Registry);
   initializeExpandInsertExtractElementPass(Registry);
@@ -540,11 +491,22 @@ int main(int argc, char **argv) {
   }
 
   Triple ModuleTriple(M->getTargetTriple());
+  std::string CPUStr, FeaturesStr;
   TargetMachine *Machine = nullptr;
-  if (ModuleTriple.getArch())
-    Machine = GetTargetMachine(ModuleTriple);
+  const TargetOptions Options = InitTargetOptionsFromCodeGenFlags();
+
+  if (ModuleTriple.getArch()) {
+    CPUStr = getCPUStr();
+    FeaturesStr = getFeaturesStr();
+    Machine = GetTargetMachine(ModuleTriple, CPUStr, FeaturesStr, Options);
+  }
+
   std::unique_ptr<TargetMachine> TM(Machine);
 
+  // Override function attributes based on CPUStr, FeaturesStr, and command line
+  // flags.
+  setFunctionAttributes(CPUStr, FeaturesStr, *M);
+
   // If the output is set to be emitted to standard out, and standard out is a
   // console, print out a warning message and refuse to do it.  We don't
   // impress anyone by spewing tons of binary goo to a terminal.
@@ -668,11 +630,6 @@ int main(int argc, char **argv) {
       PNaClABISimplifyAddPostOptPasses(&ModuleTriple, Passes);
       PNaClABISimplifyPostOpt = false;
     }
-
-    if (MinSFI && MinSFI.getPosition() < PassList.getPosition(i)) {
-      MinSFIPasses(Passes);
-      MinSFI = false;
-    }
     // @LOCALMOD-END
 
     const PassInfo *PassInf = PassList[i];
@@ -752,9 +709,6 @@ int main(int argc, char **argv) {
   // @LOCALMOD-BEGIN
   if (PNaClABISimplifyPostOpt)
     PNaClABISimplifyAddPostOptPasses(&ModuleTriple, Passes);
-
-  if (MinSFI)
-     MinSFIPasses(Passes);
   // @LOCALMOD-END
 
   // Check that the module is well formed on completion of optimization
@@ -767,19 +721,8 @@ int main(int argc, char **argv) {
       Passes.add(
           createPrintModulePass(Out->os(), "", PreserveAssemblyUseListOrder));
     else
-      // @LOCALMOD-START
-      switch (OutputFileFormat) {
-      case LLVMFormat:
-        Passes.add(createBitcodeWriterPass(Out->os(), PreserveBitcodeUseListOrder));
-        break;
-      case PNaClFormat:
-        Passes.add(createNaClBitcodeWriterPass(Out->os()));
-        break;
-      case AutodetectFileFormat:
-        report_fatal_error("Command can't autodetect file format!");
-        break;
-      }
-      // @LOCALMOD-END
+      Passes.add(
+          createBitcodeWriterPass(Out->os(), PreserveBitcodeUseListOrder));
   }
 
   // Before executing passes, print the final values of the LLVM options.
diff --git a/tools/pnacl-abicheck/CMakeLists.txt b/tools/pnacl-abicheck/CMakeLists.txt
deleted file mode 100644
index 63a86945b39..00000000000
--- a/tools/pnacl-abicheck/CMakeLists.txt
+++ /dev/null
@@ -1,11 +0,0 @@
-set(LLVM_LINK_COMPONENTS
-  BitReader
-  Core
-  IRReader
-  NaClAnalysis
-  NaClBitReader
-  Support)
-
-add_llvm_tool(pnacl-abicheck
-  pnacl-abicheck.cpp
-  )
diff --git a/tools/pnacl-abicheck/LLVMBuild.txt b/tools/pnacl-abicheck/LLVMBuild.txt
deleted file mode 100644
index 9e45f87f0ad..00000000000
--- a/tools/pnacl-abicheck/LLVMBuild.txt
+++ /dev/null
@@ -1,22 +0,0 @@
-;===- ./tools/pnacl-abicheck/LLVMBuild.txt ---------------------*- Conf -*--===;
-;
-;                     The LLVM Compiler Infrastructure
-;
-; This file is distributed under the University of Illinois Open Source
-; License. See LICENSE.TXT for details.
-;
-;===------------------------------------------------------------------------===;
-;
-; This is an LLVMBuild description file for the components in this subdirectory.
-;
-; For more information on the LLVMBuild system, please see:
-;
-;   http://llvm.org/docs/LLVMBuild.html
-;
-;===------------------------------------------------------------------------===;
-
-[component_0]
-type = Tool
-name = pnacl-abicheck
-parent = Tools
-required_libraries = AsmParser BitReader NaClBitReader IRReader NaClAnalysis
diff --git a/tools/pnacl-abicheck/Makefile b/tools/pnacl-abicheck/Makefile
deleted file mode 100644
index 97e2d223990..00000000000
--- a/tools/pnacl-abicheck/Makefile
+++ /dev/null
@@ -1,16 +0,0 @@
-#===- tools/pnacl-abicheck/Makefile ------------------------*- Makefile -*-===##
-# 
-#                     The LLVM Compiler Infrastructure
-#
-# This file is distributed under the University of Illinois Open Source
-# License. See LICENSE.TXT for details.
-# 
-##===----------------------------------------------------------------------===##
-
-LEVEL := ../..
-TOOLNAME := pnacl-abicheck
-LINK_COMPONENTS := bitreader asmparser naclbitreader irreader naclanalysis
-
-include $(LEVEL)/Makefile.common
-
-
diff --git a/tools/pnacl-abicheck/pnacl-abicheck.cpp b/tools/pnacl-abicheck/pnacl-abicheck.cpp
deleted file mode 100644
index ed326b53324..00000000000
--- a/tools/pnacl-abicheck/pnacl-abicheck.cpp
+++ /dev/null
@@ -1,103 +0,0 @@
-//===-- pnacl-abicheck.cpp - Check PNaCl bitcode ABI ----------------===//
-//
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
-//
-//===----------------------------------------------------------------------===//
-//
-// This tool checks files for compliance with the PNaCl bitcode ABI
-//
-//===----------------------------------------------------------------------===//
-
-#include "llvm/Analysis/NaCl.h"
-#include "llvm/IR/DataLayout.h"
-#include "llvm/IR/LLVMContext.h"
-#include "llvm/IR/Module.h"
-#include "llvm/IRReader/IRReader.h"
-#include "llvm/Pass.h"
-#include "llvm/IR/LegacyPassManager.h"
-#include "llvm/Support/CommandLine.h"
-#include "llvm/Support/FormattedStream.h"
-#include "llvm/Support/SourceMgr.h"
-#include <string>
-
-using namespace llvm;
-
-static cl::opt<std::string>
-InputFilename(cl::Positional, cl::desc("<input bitcode>"), cl::init("-"));
-
-static cl::opt<bool>
-VerboseErrors(
-    "verbose-parse-errors",
-    cl::desc("Print out more descriptive PNaCl bitcode parse errors"),
-    cl::init(false));
-
-static cl::opt<bool>
-Quiet("q", cl::desc("Do not print error messages"));
-
-static cl::opt<NaClFileFormat>
-InputFileFormat(
-    "bitcode-format",
-    cl::desc("Define format of input file:"),
-    cl::values(
-        clEnumValN(LLVMFormat, "llvm", "LLVM file (default)"),
-        clEnumValN(PNaClFormat, "pnacl", "PNaCl bitcode file"),
-        clEnumValEnd),
-    cl::init(AutodetectFileFormat));
-
-// Print any errors collected by the error reporter. Return true if
-// there were any.
-static bool CheckABIVerifyErrors(PNaClABIErrorReporter &Reporter,
-                                 const Twine &Name) {
-  bool HasErrors = Reporter.getErrorCount() > 0;
-  if (HasErrors) {
-    if (!Quiet) {
-      outs() << "ERROR: " << Name << " is not valid PNaCl bitcode:\n";
-      Reporter.printErrors(outs());
-    }
-  }
-  Reporter.reset();
-  return HasErrors;
-}
-
-int main(int argc, char **argv) {
-  LLVMContext &Context = getGlobalContext();
-  SMDiagnostic Err;
-  cl::ParseCommandLineOptions(argc, argv, "PNaCl Bitcode ABI checker\n");
-
-  if (Quiet)
-    VerboseErrors = false;
-
-  raw_ostream *Verbose = VerboseErrors ? &errs() : nullptr;
-  std::unique_ptr<Module> Mod(
-      NaClParseIRFile(InputFilename, InputFileFormat, Err, Verbose, Context));
-  if (Mod.get() == 0) {
-    Err.print(argv[0], errs());
-    return 1;
-  }
-  PNaClABIErrorReporter ABIErrorReporter;
-  ABIErrorReporter.setNonFatal();
-  bool ErrorsFound = false;
-
-  std::unique_ptr<ModulePass> ModuleChecker(
-      createPNaClABIVerifyModulePass(&ABIErrorReporter));
-  ModuleChecker->doInitialization(*Mod);
-  ModuleChecker->runOnModule(*Mod);
-  ErrorsFound |= CheckABIVerifyErrors(ABIErrorReporter, "Module");
-
-  std::unique_ptr<legacy::FunctionPassManager> PM(
-      new legacy::FunctionPassManager(&*Mod));
-  PM->add(createPNaClABIVerifyFunctionsPass(&ABIErrorReporter));
-
-  PM->doInitialization();
-  for (Module::iterator I = Mod->begin(), E = Mod->end(); I != E; ++I) {
-    PM->run(*I);
-    ErrorsFound |=
-        CheckABIVerifyErrors(ABIErrorReporter, "Function " + I->getName());
-  }
-  PM->doFinalization();
-
-  return ErrorsFound ? 1 : 0;
-}
diff --git a/tools/pnacl-bcanalyzer/CMakeLists.txt b/tools/pnacl-bcanalyzer/CMakeLists.txt
deleted file mode 100644
index fecbcd68c17..00000000000
--- a/tools/pnacl-bcanalyzer/CMakeLists.txt
+++ /dev/null
@@ -1,8 +0,0 @@
-set(LLVM_LINK_COMPONENTS
-  NaClBitAnalysis
-  NaClBitReader
-  Support)
-
-add_llvm_tool(pnacl-bcanalyzer
-  pnacl-bcanalyzer.cpp
-  )
diff --git a/tools/pnacl-bcanalyzer/LLVMBuild.txt b/tools/pnacl-bcanalyzer/LLVMBuild.txt
deleted file mode 100644
index 2aad716a46f..00000000000
--- a/tools/pnacl-bcanalyzer/LLVMBuild.txt
+++ /dev/null
@@ -1,22 +0,0 @@
-;===- ./tools/pnacl-bcanalyzer/LLVMBuild.txt -------------------*- Conf -*--===;
-;
-;                     The LLVM Compiler Infrastructure
-;
-; This file is distributed under the University of Illinois Open Source
-; License. See LICENSE.TXT for details.
-;
-;===------------------------------------------------------------------------===;
-;
-; This is an LLVMBuild description file for the components in this subdirectory.
-;
-; For more information on the LLVMBuild system, please see:
-;
-;   http://llvm.org/docs/LLVMBuild.html
-;
-;===------------------------------------------------------------------------===;
-
-[component_0]
-type = Tool
-name = pnacl-bcanalyzer
-parent = Tools
-required_libraries = NaClBitReader NaClBitAnalysis
diff --git a/tools/pnacl-bcanalyzer/Makefile b/tools/pnacl-bcanalyzer/Makefile
deleted file mode 100644
index f3234f38c16..00000000000
--- a/tools/pnacl-bcanalyzer/Makefile
+++ /dev/null
@@ -1,17 +0,0 @@
-##===- tools/pnacl-bcanalyzer/Makefile ---------------------*- Makefile -*-===##
-# 
-#                     The LLVM Compiler Infrastructure
-#
-# This file is distributed under the University of Illinois Open Source
-# License. See LICENSE.TXT for details.
-# 
-##===----------------------------------------------------------------------===##
-
-LEVEL := ../..
-TOOLNAME := pnacl-bcanalyzer
-LINK_COMPONENTS := naclbitreader naclbitanalysis
-
-# This tool has no plugins, optimize startup time.
-TOOL_NO_EXPORTS := 1
-
-include $(LEVEL)/Makefile.common
diff --git a/tools/pnacl-bcanalyzer/pnacl-bcanalyzer.cpp b/tools/pnacl-bcanalyzer/pnacl-bcanalyzer.cpp
deleted file mode 100644
index 6d6f8ab8639..00000000000
--- a/tools/pnacl-bcanalyzer/pnacl-bcanalyzer.cpp
+++ /dev/null
@@ -1,80 +0,0 @@
-//===-- pnacl-bcanalyzer.cpp - Bitcode Analyzer -------------------------===//
-//
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
-//
-//===----------------------------------------------------------------------===//
-//
-// This tool is a thin wrapper over NaClBitcodeAnalyzer; see
-// NaClBitcodeAnalyzer.h for more details.
-// 
-// Invoke in the following manner:
-//
-//  pnacl-bcanalyzer [options]      - Read frozen PNaCl bitcode from stdin
-//  pnacl-bcanalyzer [options] x.bc - Read frozen PNaCl bitcode from the x.bc
-//                                    file
-// Run with -help to see supported options.
-//
-//===----------------------------------------------------------------------===//
-
-#define DEBUG_TYPE "pnacl-bcanalyzer"
-
-#include "llvm/Bitcode/NaCl/NaClBitcodeAnalyzer.h"
-#include "llvm/Support/CommandLine.h"
-#include "llvm/Support/Debug.h"
-#include "llvm/Support/ManagedStatic.h"
-#include "llvm/Support/PrettyStackTrace.h"
-#include "llvm/Support/Signals.h"
-#include "llvm/Support/raw_ostream.h"
-using namespace llvm;
-
-
-static cl::opt<std::string>
-  InputFilename(cl::Positional, cl::desc("<input bitcode>"), cl::init("-"));
-
-static cl::opt<bool>
- OptDumpRecords(
-     "dump-records",
-     cl::desc("Dump contents of records in bitcode, leaving out details, "
-              "instead of displaying record distributions."),
-     cl::init(false));
-
-static cl::opt<bool>
-OptDumpDetails(
-    "dump-details",
-    cl::desc("Include details when dumping contents of records in bitcode."),
-    cl::init(false));
-
-static cl::opt<unsigned> OpsPerLine(
-    "operands-per-line",
-    cl::desc("Number of operands to print per dump line. 0 implies "
-             "all operands will be printed on the same line (default)"),
-    cl::init(0));
-
-static cl::opt<bool> OrderBlocksByID(
-    "order-blocks-by-id",
-    cl::desc("Print blocks statistics based on block id rather than size"),
-    cl::init(false));
-
-int main(int argc, char **argv) {
-  // Print a stack trace if we signal out.
-  sys::PrintStackTraceOnErrorSignal();
-  PrettyStackTraceProgram X(argc, argv);
-  llvm_shutdown_obj Y;  // Call llvm_shutdown() on exit.
-  cl::ParseCommandLineOptions(argc, argv, "pnacl-bcanalyzer file analyzer\n");
-
-  if (OptDumpDetails && !OptDumpRecords) {
-    errs() << "Can't dump details unless records are dumped!\n";
-    return 1;
-  }
-
-  AnalysisDumpOptions DumpOptions;
-  DumpOptions.DumpRecords = OptDumpRecords;
-  DumpOptions.DumpDetails = OptDumpDetails;
-  DumpOptions.OpsPerLine = OpsPerLine;
-  DumpOptions.OrderBlocksByID = OrderBlocksByID;
-
-  return AnalyzeBitcodeInFile(InputFilename, outs(), DumpOptions);
-}
diff --git a/tools/pnacl-bccompress/CMakeLists.txt b/tools/pnacl-bccompress/CMakeLists.txt
deleted file mode 100644
index f2fed9dd7a6..00000000000
--- a/tools/pnacl-bccompress/CMakeLists.txt
+++ /dev/null
@@ -1,9 +0,0 @@
-set(LLVM_LINK_COMPONENTS
-  NaClBitAnalysis
-  NaClBitReader
-  NaClBitWriter
-  Support)
-
-add_llvm_tool(pnacl-bccompress
-  pnacl-bccompress.cpp
-  )
diff --git a/tools/pnacl-bccompress/LLVMBuild.txt b/tools/pnacl-bccompress/LLVMBuild.txt
deleted file mode 100644
index 5557fd05330..00000000000
--- a/tools/pnacl-bccompress/LLVMBuild.txt
+++ /dev/null
@@ -1,22 +0,0 @@
-;===- ./tools/pnacl-bccompress/LLVMBuild.txt -------------------*- Conf -*--===;
-;
-;                     The LLVM Compiler Infrastructure
-;
-; This file is distributed under the University of Illinois Open Source
-; License. See LICENSE.TXT for details.
-;
-;===------------------------------------------------------------------------===;
-;
-; This is an LLVMBuild description file for the components in this subdirectory.
-;
-; For more information on the LLVMBuild system, please see:
-;
-;   http://llvm.org/docs/LLVMBuild.html
-;
-;===------------------------------------------------------------------------===;
-
-[component_0]
-type = Tool
-name = pnacl-bccompress
-parent = Tools
-required_libraries = NaClBitReader NaClBitWriter NaClBitAnalysis
diff --git a/tools/pnacl-bccompress/Makefile b/tools/pnacl-bccompress/Makefile
deleted file mode 100644
index daec54b6d2e..00000000000
--- a/tools/pnacl-bccompress/Makefile
+++ /dev/null
@@ -1,17 +0,0 @@
-##===- tools/pnacl-bccompress/Makefile ---------------------*- Makefile -*-===##
-# 
-#                     The LLVM Compiler Infrastructure
-#
-# This file is distributed under the University of Illinois Open Source
-# License. See LICENSE.TXT for details.
-# 
-##===----------------------------------------------------------------------===##
-
-LEVEL := ../..
-TOOLNAME := pnacl-bccompress
-LINK_COMPONENTS := naclbitreader naclbitwriter naclbitanalysis
-
-# This tool has no plugins, optimize startup time.
-TOOL_NO_EXPORTS := 1
-
-include $(LEVEL)/Makefile.common
diff --git a/tools/pnacl-bccompress/pnacl-bccompress.cpp b/tools/pnacl-bccompress/pnacl-bccompress.cpp
deleted file mode 100644
index e439662cff6..00000000000
--- a/tools/pnacl-bccompress/pnacl-bccompress.cpp
+++ /dev/null
@@ -1,135 +0,0 @@
-//===-- pnacl-bccompress.cpp - Bitcode (abbrev) compression ---------------===//
-//
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
-//
-//===----------------------------------------------------------------------===//
-//
-// This tool may be invoked in the following manner:
-//  pnacl-bccompress [options] bcin.pexe -o bcout.pexe
-//      - Read frozen PNaCl bitcode from the bcin.pexe and introduce
-//        abbreviations to compress it into bcout.pexe.
-//
-//  Options:
-//      --help      - Output information about command line switches
-//
-// This tool analyzes the data in bcin.pexe, and determines what
-// abbreviations can be added to compress the bitcode file. The result
-// is written to bcout.pexe.
-//
-//===----------------------------------------------------------------------===//
-
-#include "llvm/Bitcode/NaCl/NaClCompress.h"
-#include "llvm/Support/CommandLine.h"
-#include "llvm/Support/FileSystem.h"
-#include "llvm/Support/ManagedStatic.h"
-#include "llvm/Support/PrettyStackTrace.h"
-#include "llvm/Support/Signals.h"
-#include "llvm/Support/ToolOutputFile.h"
-
-namespace {
-
-using namespace llvm;
-
-static cl::opt<bool>
-TraceGeneratedAbbreviations(
-    "abbreviations",
-    cl::desc("Trace abbreviations added to compressed file"),
-    cl::init(false));
-
-static cl::opt<std::string>
-  InputFilename(cl::Positional, cl::desc("<input bitcode>"), cl::init("-"));
-
-static cl::opt<std::string>
-OutputFilename("o", cl::desc("Specify output filename"),
-               cl::value_desc("filename"), cl::init("-"));
-
-static cl::opt<bool>
-ShowValueDistributions(
-    "show-distributions",
-    cl::desc("Show collected value distributions in bitcode records. "
-             "Turns off compression."),
-    cl::init(false));
-
-static cl::opt<bool>
-ShowAbbrevLookupTries(
-    "show-lookup-tries",
-    cl::desc("Show lookup tries used to minimize search for \n"
-             "matching abbreviations. Turns off compression."),
-    cl::init(false));
-
-static cl::opt<bool>
-ShowAbbreviationFrequencies(
-    "show-abbreviation-frequencies",
-    cl::desc("Show how often each abbreviation is used. "
-             "Turns off compression."),
-    cl::init(false));
-
-// Note: When this flag is true, we still generate new abbreviations,
-// because we don't want to add the complexity of turning it off.
-// Rather, we simply make sure abbreviations are ignored when writing
-// out the final copy.
-static cl::opt<bool>
-RemoveAbbreviations(
-    "remove-abbreviations",
-    cl::desc("Remove abbreviations from input bitcode file."),
-    cl::init(false));
-
-static bool Fatal(const std::string &Err) {
-  errs() << Err << "\n";
-  exit(1);
-}
-
-// Reads the input file into the given buffer.
-static void ReadAndBuffer(std::unique_ptr<MemoryBuffer> &MemBuf) {
-  ErrorOr<std::unique_ptr<MemoryBuffer>> ErrOrFile =
-      MemoryBuffer::getFileOrSTDIN(InputFilename);
-  if (std::error_code EC = ErrOrFile.getError())
-    Fatal("Error reading '" + InputFilename + "': " + EC.message());
-
-  MemBuf.reset(ErrOrFile.get().release());
-  if (MemBuf->getBufferSize() % 4 != 0)
-    Fatal("Bitcode stream should be a multiple of 4 bytes in length");
-}
-
-}  // namespace
-
-int main(int argc, char **argv) {
-  // Print a stack trace if we signal out.
-  sys::PrintStackTraceOnErrorSignal();
-  PrettyStackTraceProgram X(argc, argv);
-  llvm_shutdown_obj Y;  // Call llvm_shutdown() on exit.
-  cl::ParseCommandLineOptions(argc, argv, "pnacl-bccompress file analyzer\n");
-
-  std::unique_ptr<MemoryBuffer> MemBuf;
-  ReadAndBuffer(MemBuf);
-
-  std::error_code EC;
-  std::unique_ptr<tool_output_file> OutFile(
-      new tool_output_file(OutputFilename.c_str(), EC, sys::fs::F_None));
-  if (EC)
-    Fatal(EC.message());
-
-  NaClBitcodeCompressor Compressor;
-  Compressor.Flags.TraceGeneratedAbbreviations = TraceGeneratedAbbreviations;
-  Compressor.Flags.ShowValueDistributions = ShowValueDistributions;
-  Compressor.Flags.ShowAbbrevLookupTries = ShowAbbrevLookupTries;
-  Compressor.Flags.ShowAbbreviationFrequencies = ShowAbbreviationFrequencies;
-  Compressor.Flags.RemoveAbbreviations = RemoveAbbreviations;
-
-  if (ShowValueDistributions
-      || ShowAbbreviationFrequencies
-      || ShowAbbrevLookupTries) {
-    // Assume we are only interested in analysis.
-    int ReturnStatus = !Compressor.analyze(MemBuf.get(), OutFile->os());
-    OutFile->keep();
-    return ReturnStatus;
-  }
-
-  if (!Compressor.compress(MemBuf.get(), OutFile->os()))
-    return 1;
-  OutFile->keep();
-  return 0;
-}
diff --git a/tools/pnacl-bcdis/CMakeLists.txt b/tools/pnacl-bcdis/CMakeLists.txt
deleted file mode 100644
index 94afafb4196..00000000000
--- a/tools/pnacl-bcdis/CMakeLists.txt
+++ /dev/null
@@ -1,8 +0,0 @@
-set(LLVM_LINK_COMPONENTS
-  NaClBitAnalysis
-  NaClBitReader
-  Support)
-
-add_llvm_tool(pnacl-bcdis
-  pnacl-bcdis.cpp
-  )
diff --git a/tools/pnacl-bcdis/LLVMBuild.txt b/tools/pnacl-bcdis/LLVMBuild.txt
deleted file mode 100644
index d9b4db6b38b..00000000000
--- a/tools/pnacl-bcdis/LLVMBuild.txt
+++ /dev/null
@@ -1,22 +0,0 @@
-;===- ./tools/pnacl-bcdis/LLVMBuild.txt ------------------------*- Conf -*--===;
-;
-;                     The LLVM Compiler Infrastructure
-;
-; This file is distributed under the University of Illinois Open Source
-; License. See LICENSE.TXT for details.
-;
-;===------------------------------------------------------------------------===;
-;
-; This is an LLVMBuild description file for the components in this subdirectory.
-;
-; For more information on the LLVMBuild system, please see:
-;
-;   http://llvm.org/docs/LLVMBuild.html
-;
-;===------------------------------------------------------------------------===;
-
-[component_0]
-type = Tool
-name = pnacl-bcdis
-parent = Tools
-required_libraries = NaClBitAnalysis NaClBitReader
diff --git a/tools/pnacl-bcdis/Makefile b/tools/pnacl-bcdis/Makefile
deleted file mode 100644
index ddbd965118e..00000000000
--- a/tools/pnacl-bcdis/Makefile
+++ /dev/null
@@ -1,17 +0,0 @@
-##===- tools/pnacl-bcdis/Makefile --------------------------*- Makefile -*-===##
-# 
-#                     The LLVM Compiler Infrastructure
-#
-# This file is distributed under the University of Illinois Open Source
-# License. See LICENSE.TXT for details.
-# 
-##===----------------------------------------------------------------------===##
-
-LEVEL := ../..
-TOOLNAME := pnacl-bcdis
-LINK_COMPONENTS := naclbitanalysis naclbitreader
-
-# This tool has no plugins, optimize startup time.
-TOOL_NO_EXPORTS := 1
-
-include $(LEVEL)/Makefile.common
diff --git a/tools/pnacl-bcdis/pnacl-bcdis.cpp b/tools/pnacl-bcdis/pnacl-bcdis.cpp
deleted file mode 100644
index 38de6733f1b..00000000000
--- a/tools/pnacl-bcdis/pnacl-bcdis.cpp
+++ /dev/null
@@ -1,82 +0,0 @@
-//===-- pnacl-bcdis.cpp - Disassemble pnacl bitcode -----------------------===//
-//
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
-//
-//===----------------------------------------------------------------------===//
-
-/// TODO(kschimpf): Add disassembling abbreviations.
-
-#include "llvm/Bitcode/NaCl/NaClReaderWriter.h"
-#include "llvm/Support/CommandLine.h"
-#include "llvm/Support/FileSystem.h"
-#include "llvm/Support/ManagedStatic.h"
-#include "llvm/Support/MemoryBuffer.h"
-#include "llvm/Support/PrettyStackTrace.h"
-#include "llvm/Support/Signals.h"
-#include "llvm/Support/ToolOutputFile.h"
-#include "llvm/Support/raw_ostream.h"
-#include <system_error>
-
-namespace {
-
-using namespace llvm;
-
-// The input file to read.
-static cl::opt<std::string>
-  InputFilename(cl::Positional, cl::desc("<input bitcode>"), cl::init("-"));
-
-// The output file to generate.
-static cl::opt<std::string>
-OutputFilename("o", cl::desc("Specify output filename"),
-               cl::value_desc("filename"), cl::init("-"));
-
-static cl::opt<bool>
-NoRecords("no-records",
-          cl::desc("Don't include records"),
-          cl::init(false));
-
-static cl::opt<bool>
-NoAssembly("no-assembly",
-           cl::desc("Don't include assembly"),
-           cl::init(false));
-
-// Reads and disassembles the bitcode file. Returns false
-// if successful, true otherwise.
-static bool DisassembleBitcode() {
-  // Open the bitcode file and put into a buffer.
-  ErrorOr<std::unique_ptr<MemoryBuffer>> ErrOrFile =
-      MemoryBuffer::getFileOrSTDIN(InputFilename);
-  if (std::error_code EC = ErrOrFile.getError()) {
-    errs() << "Error reading '" << InputFilename << "': " << EC.message()
-           << "\n";
-    return true;
-  }
-
-  // Create a stream to output the bitcode text to.
-  std::error_code EC;
-  raw_fd_ostream Output(OutputFilename, EC, sys::fs::F_None);
-  if (EC) {
-    errs() << EC.message() << '\n';
-    return true;
-  }
-
-  // Parse the the bitcode file.
-  return NaClObjDump(ErrOrFile.get()->getMemBufferRef(), Output, NoRecords,
-                     NoAssembly);
-}
-
-}
-
-int main(int argc, char **argv) {
-  // Print a stack trace if we signal out.
-  sys::PrintStackTraceOnErrorSignal();
-  PrettyStackTraceProgram X(argc, argv);
-  llvm_shutdown_obj Y;  // Call llvm_shutdown() on exit.
-  cl::ParseCommandLineOptions(argc, argv, "pnacl-bccompress file analyzer\n");
-
-  if (DisassembleBitcode()) return 1;
-  return 0;
-}
diff --git a/tools/pnacl-bcfuzz/CMakeLists.txt b/tools/pnacl-bcfuzz/CMakeLists.txt
deleted file mode 100644
index 783bde22f34..00000000000
--- a/tools/pnacl-bcfuzz/CMakeLists.txt
+++ /dev/null
@@ -1,10 +0,0 @@
-set(LLVM_LINK_COMPONENTS
-  Core
-  NaClBitReader
-  NaClBitTestUtils
-  NaClBitWriter
-  Support)
-
-add_llvm_tool(pnacl-bcfuzz
-  pnacl-bcfuzz.cpp
-  )
diff --git a/tools/pnacl-bcfuzz/LLVMBuild.txt b/tools/pnacl-bcfuzz/LLVMBuild.txt
deleted file mode 100644
index dd4be065c21..00000000000
--- a/tools/pnacl-bcfuzz/LLVMBuild.txt
+++ /dev/null
@@ -1,15 +0,0 @@
-;===- ./tools/pnacl-bcfuzz/LLVMBuild.txt -----------------------*- Conf -*--===;
-;
-; This is an LLVMBuild description file for the components in this subdirectory.
-;
-; For more information on the LLVMBuild system, please see:
-;
-;   http://llvm.org/docs/LLVMBuild.html
-;
-;===------------------------------------------------------------------------===;
-
-[component_0]
-type = Tool
-name = pnacl-bcfuzz
-parent = Tools
-required_libraries = NaClBitWriter NaClBitReader NaClBitTestUtils
diff --git a/tools/pnacl-bcfuzz/Makefile b/tools/pnacl-bcfuzz/Makefile
deleted file mode 100644
index dee0dd1f491..00000000000
--- a/tools/pnacl-bcfuzz/Makefile
+++ /dev/null
@@ -1,17 +0,0 @@
-##===- tools/pnacl-bcfuzz/Makefile -------------------------*- Makefile -*-===##
-# 
-#                     The LLVM Compiler Infrastructure
-#
-# This file is distributed under the University of Illinois Open Source
-# License. See LICENSE.TXT for details.
-# 
-##===----------------------------------------------------------------------===##
-
-LEVEL := ../..
-TOOLNAME := pnacl-bcfuzz
-LINK_COMPONENTS := naclbitwriter naclbitreader naclbittestutils
-
-# This tool has no plugins, optimize startup time.
-TOOL_NO_EXPORTS := 1
-
-include $(LEVEL)/Makefile.common
diff --git a/tools/pnacl-bcfuzz/pnacl-bcfuzz.cpp b/tools/pnacl-bcfuzz/pnacl-bcfuzz.cpp
deleted file mode 100644
index e7888b3ae1f..00000000000
--- a/tools/pnacl-bcfuzz/pnacl-bcfuzz.cpp
+++ /dev/null
@@ -1,224 +0,0 @@
-//===-- pnacl-bcfuzz.cpp - Record fuzzer for PNaCl bitcode ----------------===//
-//
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
-//
-//===----------------------------------------------------------------------===//
-//
-// Generates (record-level) fuzzed PNaCl bitcode files from an input
-// PNaCl bitcode file.
-//
-//===----------------------------------------------------------------------===//
-
-#include "llvm/ADT/STLExtras.h"
-#include "llvm/Bitcode/NaCl/NaClFuzz.h"
-#include "llvm/Support/CommandLine.h"
-#include "llvm/Support/FileSystem.h"
-#include "llvm/Support/ManagedStatic.h"
-#include "llvm/Support/MemoryBuffer.h"
-#include "llvm/Support/PrettyStackTrace.h"
-#include "llvm/Support/Signals.h"
-#include "llvm/Support/ToolOutputFile.h"
-
-using namespace llvm;
-using namespace naclfuzz;
-
-static cl::opt<std::string>
-InputFilename(cl::Positional, cl::desc("<frozen file>"), cl::init("-"));
-
-static cl::opt<std::string>
-OutputPrefix("output", cl::desc("<output prefix>"), cl::init(""));
-
-static cl::opt<unsigned>
-FuzzCount("count", cl::desc("Number of fuzz results to generate"),
-          cl::init(1));
-
-static cl::opt<bool>
-ConvertToTextRecords(
-    "convert-to-text",
-    cl::desc("Convert binary file to record text file (specified by -output)"),
-    cl::init(false));
-
-static cl::opt<bool>
-AcceptBitcodeRecordsAsText(
-    "bitcode-as-text",
-    cl::desc(
-        "Convert record text file to binary file (specified by -output)"),
-    cl::init(false));
-
-static cl::opt<std::string>
-RandomSeed("random-seed",
-     cl::desc("Use this value for seed of random number generator "
-              "(rather than input)"),
-     cl::init(""));
-
-static cl::opt<bool>
-ShowFuzzRecordDistribution(
-    "record-distribution",
-    cl::desc("Show distribution of record edits while fuzzing"),
-    cl::init(false));
-
-static cl::opt<bool>
-ShowFuzzEditDistribution(
-    "edit-distribution",
-    cl::desc("Show distribution of editing actions while fuzzing"),
-    cl::init(false));
-
-static cl::opt<unsigned>
-PercentageToEdit(
-    "edit-percentage",
-    cl::desc("Percentage of records to edit during fuzz (between 1 and"
-             " '-percentage-base')"),
-    cl::init(1));
-
-static cl::opt<unsigned>
-PercentageBase(
-    "percentage-base",
-    cl::desc("Base that '-edit-precentage' is defined on (defaults to 100)"),
-    cl::init(100));
-
-static cl::opt<bool>
-Verbose("verbose",
-        cl::desc("Show details of fuzzing/writing of bitcode files"),
-        cl::init(false));
-
-static void writeOutputFile(SmallVectorImpl<char> &Buffer,
-                            StringRef OutputFilename) {
-  std::error_code EC;
-  std::unique_ptr<tool_output_file> Out(
-      new tool_output_file(OutputFilename, EC, sys::fs::F_None));
-  if (EC) {
-    errs() << EC.message() << '\n';
-    exit(1);
-  }
-
-  for (SmallVectorImpl<char>::const_iterator
-           Iter = Buffer.begin(), IterEnd = Buffer.end();
-       Iter != IterEnd; ++Iter) {
-    Out->os() << *Iter;
-  }
-
-  // Declare success.
-  Out->keep();
-}
-
-static bool writeBitcode(NaClMungedBitcode &Bitcode,
-                         NaClMungedBitcode::WriteFlags &WriteFlags,
-                         StringRef OutputFile) {
-  if (Verbose) {
-    errs() << "Records:\n";
-    for (const auto &Record : Bitcode) {
-      errs() << "  " << Record << "\n";
-    }
-  }
-
-  SmallVector<char, 100> Buffer;
-  if (!Bitcode.write(Buffer, true, WriteFlags)) {
-    errs() << "Error: Failed to write bitcode: " << OutputFile << "\n";
-    return false;
-  }
-  writeOutputFile(Buffer, OutputFile);
-  return true;
-}
-
-static void writeFuzzedBitcodeFiles(NaClMungedBitcode &Bitcode,
-                                    NaClMungedBitcode::WriteFlags &WriteFlags) {
-  std::string RandSeed(RandomSeed);
-  if (RandomSeed.empty())
-    RandSeed = InputFilename;
-
-  DefaultRandomNumberGenerator Generator(RandSeed);
-  std::unique_ptr<RecordFuzzer> Fuzzer(
-      RecordFuzzer::createSimpleRecordFuzzer(Bitcode, Generator));
-
-  for (size_t i = 1; i <= FuzzCount; ++i) {
-    Generator.saltSeed(i);
-    std::string OutputFile;
-    {
-      raw_string_ostream StrBuf(OutputFile);
-      StrBuf << OutputPrefix << "-" << i;
-      StrBuf.flush();
-    }
-
-    if (Verbose)
-      errs() << "Generating " << OutputFile << "\n";
-    if (!Fuzzer->fuzz(PercentageToEdit, PercentageBase)) {
-      errs() << "Error: Fuzzing failed: " << OutputFile << "\n";
-      continue;
-    }
-    writeBitcode(Bitcode, WriteFlags, OutputFile);
-  }
-
-  if (ShowFuzzRecordDistribution)
-    Fuzzer->showRecordDistribution(outs());
-  if (ShowFuzzEditDistribution)
-    Fuzzer->showEditDistribution(outs());
-}
-
-bool writeTextualBitcodeRecords(std::unique_ptr<MemoryBuffer> InputBuffer) {
-  NaClBitcodeRecordList Records;
-  readNaClBitcodeRecordList(Records, std::move(InputBuffer));
-  SmallVector<char, 1024> OutputBuffer;
-  if (!writeNaClBitcodeRecordList(Records, OutputBuffer, errs()))
-    return false;
-  writeOutputFile(OutputBuffer, OutputPrefix);
-  return true;
-}
-
-bool writeBinaryBitcodeRecords(std::unique_ptr<MemoryBuffer> InputBuffer) {
-  std::unique_ptr<NaClBitcodeRecordList> Records
-      = make_unique<NaClBitcodeRecordList>();
-  if (std::error_code EC = readNaClTextBcRecordList(*Records,
-                                                    std::move(InputBuffer))) {
-    errs() << "Error: " << EC.message() << "\n";
-    return false;
-  }
-  NaClMungedBitcode Bitcode(std::move(Records));
-  NaClMungedBitcode::WriteFlags WriteFlags;
-  return writeBitcode(Bitcode, WriteFlags, OutputPrefix);
-}
-
-int main(int argc, char **argv) {
-  // Print a stack trace if we signal out.
-  sys::PrintStackTraceOnErrorSignal();
-  PrettyStackTraceProgram X(argc, argv);
-
-  cl::ParseCommandLineOptions(argc, argv, "Fuzz a PNaCl bitcode file\n");
-
-  if (OutputPrefix.empty()) {
-    errs() << "Output prefix not specified!\n";
-    return 1;
-  }
-
-  ErrorOr<std::unique_ptr<MemoryBuffer>>
-      MemBuf(MemoryBuffer::getFileOrSTDIN(InputFilename));
-  if (!MemBuf) {
-    errs() << MemBuf.getError().message() << "\n";
-    return 1;
-  }
-
-  if (ConvertToTextRecords)
-    return !writeTextualBitcodeRecords(std::move(MemBuf.get()));
-
-  if (AcceptBitcodeRecordsAsText)
-    return !writeBinaryBitcodeRecords(std::move(MemBuf.get()));
-
-  if (PercentageToEdit > PercentageBase) {
-    errs() << "Edit percentage " << PercentageToEdit
-           << " must not exceed: " << PercentageBase << "\n";
-    return 1;
-  }
-
-  raw_null_ostream NullStrm;
-  NaClMungedBitcode::WriteFlags WriteFlags;
-  WriteFlags.setTryToRecover(true);
-  if (!Verbose) {
-    WriteFlags.setErrStream(NullStrm);
-  }
-
-  NaClMungedBitcode Bitcode(std::move(MemBuf.get()));
-  writeFuzzedBitcodeFiles(Bitcode, WriteFlags);
-  return 0;
-}
diff --git a/tools/pnacl-freeze/CMakeLists.txt b/tools/pnacl-freeze/CMakeLists.txt
deleted file mode 100644
index 3d8c3c363e1..00000000000
--- a/tools/pnacl-freeze/CMakeLists.txt
+++ /dev/null
@@ -1,10 +0,0 @@
-set(LLVM_LINK_COMPONENTS
-  BitReader
-  Core
-  NaClBitWriter
-  NaClBitReader
-  Support)
-
-add_llvm_tool(pnacl-freeze
-  pnacl-freeze.cpp
-  )
diff --git a/tools/pnacl-freeze/LLVMBuild.txt b/tools/pnacl-freeze/LLVMBuild.txt
deleted file mode 100644
index e17f5c0f878..00000000000
--- a/tools/pnacl-freeze/LLVMBuild.txt
+++ /dev/null
@@ -1,16 +0,0 @@
-;===- ./tools/pnacl-freeze/LLVMBuild.txt -----------------------*- Conf -*--===;
-;===------------------------------------------------------------------------===;
-;
-; This is an LLVMBuild description file for the components in this subdirectory.
-;
-; For more information on the LLVMBuild system, please see:
-;
-;   http://llvm.org/docs/LLVMBuild.html
-;
-;===------------------------------------------------------------------------===;
-
-[component_0]
-type = Tool
-name = pnacl-freeze
-parent = Tools
-required_libraries = NaClBitWriter NaClBitReader BitReader
diff --git a/tools/pnacl-freeze/Makefile b/tools/pnacl-freeze/Makefile
deleted file mode 100644
index ceeb409181a..00000000000
--- a/tools/pnacl-freeze/Makefile
+++ /dev/null
@@ -1,17 +0,0 @@
-##===- tools/pnacl-freeze/Makefile -------------------------*- Makefile -*-===##
-# 
-#                     The LLVM Compiler Infrastructure
-#
-# This file is distributed under the University of Illinois Open Source
-# License. See LICENSE.TXT for details.
-# 
-##===----------------------------------------------------------------------===##
-
-LEVEL := ../..
-TOOLNAME := pnacl-freeze
-LINK_COMPONENTS := naclbitwriter naclbitreader bitreader
-
-# This tool has no plugins, optimize startup time.
-TOOL_NO_EXPORTS := 1
-
-include $(LEVEL)/Makefile.common
diff --git a/tools/pnacl-freeze/pnacl-freeze.cpp b/tools/pnacl-freeze/pnacl-freeze.cpp
deleted file mode 100644
index ea56ec3dbe0..00000000000
--- a/tools/pnacl-freeze/pnacl-freeze.cpp
+++ /dev/null
@@ -1,93 +0,0 @@
-/* Copyright 2013 The Native Client Authors. All rights reserved.
- * Use of this source code is governed by a BSD-style license that can
- * be found in the LICENSE file.
- */
-
-//===-- pnacl-freeze.cpp - The low-level NaCl bitcode freezer     --------===//
-//
-//===----------------------------------------------------------------------===//
-//
-// Generates NaCl pexe wire format.
-//
-//===----------------------------------------------------------------------===//
-
-#include "llvm/Bitcode/NaCl/NaClReaderWriter.h"
-#include "llvm/Bitcode/ReaderWriter.h"
-#include "llvm/IR/LLVMContext.h"
-#include "llvm/IR/Module.h"
-#include "llvm/Support/CommandLine.h"
-#include "llvm/Support/DataStream.h"
-#include "llvm/Support/FileSystem.h"
-#include "llvm/Support/ManagedStatic.h"
-#include "llvm/Support/PrettyStackTrace.h"
-#include "llvm/Support/Signals.h"
-#include "llvm/Support/StreamingMemoryObject.h"
-#include "llvm/Support/ToolOutputFile.h"
-
-using namespace llvm;
-
-
-static cl::opt<std::string>
-OutputFilename("o", cl::desc("Specify output filename"),
-	       cl::value_desc("filename"), cl::init("-"));
-
-static cl::opt<std::string>
-InputFilename(cl::Positional, cl::desc("<pexe file>"), cl::init("-"));
-
-static void WriteOutputFile(const Module *M) {
-
-  std::error_code EC;
-  std::unique_ptr<tool_output_file> Out(
-      new tool_output_file(OutputFilename, EC, sys::fs::F_None));
-  if (EC) {
-    errs() << EC.message() << '\n';
-    exit(1);
-  }
-
-  NaClWriteBitcodeToFile(M, Out->os(), /* AcceptSupportedOnly = */ false);
-
-  // Declare success.
-  Out->keep();
-}
-
-int main(int argc, char **argv) {
-  // Print a stack trace if we signal out.
-  sys::PrintStackTraceOnErrorSignal();
-  PrettyStackTraceProgram X(argc, argv);
-
-  LLVMContext &Context = getGlobalContext();
-  llvm_shutdown_obj Y;  // Call llvm_shutdown() on exit.
-
-  cl::ParseCommandLineOptions(argc, argv, "Generates NaCl pexe wire format\n");
-
-  std::string ErrorMessage;
-  std::unique_ptr<Module> M;
-
-  // Use the bitcode streaming interface
-  DataStreamer *streamer = getDataFileStreamer(InputFilename, &ErrorMessage);
-  std::unique_ptr<StreamingMemoryObject> Buffer(
-      new StreamingMemoryObjectImpl(streamer));
-  if (streamer) {
-    std::string DisplayFilename;
-    if (InputFilename == "-")
-      DisplayFilename = "<stdin>";
-    else
-      DisplayFilename = InputFilename;
-    ErrorOr<std::unique_ptr<Module>> MOrErr =
-        getStreamedBitcodeModule(DisplayFilename, Buffer.release(), Context);
-    M = std::move(*MOrErr);
-    M->materializeAllPermanently();
-  }
-
-  if (!M.get()) {
-    errs() << argv[0] << ": ";
-    if (ErrorMessage.size())
-      errs() << ErrorMessage << "\n";
-    else
-      errs() << "bitcode didn't read correctly.\n";
-    return 1;
-  }
-
-  WriteOutputFile(M.get());
-  return 0;
-}
diff --git a/tools/pnacl-llc/CMakeLists.txt b/tools/pnacl-llc/CMakeLists.txt
deleted file mode 100644
index edfab1c7f74..00000000000
--- a/tools/pnacl-llc/CMakeLists.txt
+++ /dev/null
@@ -1,27 +0,0 @@
-set(LLVM_LINK_COMPONENTS
-  ${LLVM_TARGETS_TO_BUILD}
-  Analysis
-  BitReader
-  CodeGen
-  Core
-  IRReader
-  MC
-  NaClAnalysis
-  NaClBitReader
-  NaClBitTestUtils
-  NaClTransforms
-  ScalarOpts
-  SelectionDAG
-  Support
-  Target)
-
-add_llvm_tool(pnacl-llc
-  srpc_main.cpp
-  SRPCStreamer.cpp
-  pnacl-llc.cpp
-  ThreadedStreamingCache.cpp
-  )
-
-if(LLVM_ENABLE_THREADS AND HAVE_LIBPTHREAD)
-  target_link_libraries(pnacl-llc pthread)
-endif()
diff --git a/tools/pnacl-llc/LLVMBuild.txt b/tools/pnacl-llc/LLVMBuild.txt
deleted file mode 100644
index d9b474d61d9..00000000000
--- a/tools/pnacl-llc/LLVMBuild.txt
+++ /dev/null
@@ -1,22 +0,0 @@
-;===- ./tools/pnacl-llc/LLVMBuild.txt --------------------------*- Conf -*--===;
-;
-;                     The LLVM Compiler Infrastructure
-;
-; This file is distributed under the University of Illinois Open Source
-; License. See LICENSE.TXT for details.
-;
-;===------------------------------------------------------------------------===;
-;
-; This is an LLVMBuild description file for the components in this subdirectory.
-;
-; For more information on the LLVMBuild system, please see:
-;
-;   http://llvm.org/docs/LLVMBuild.html
-;
-;===------------------------------------------------------------------------===;
-
-[component_0]
-type = Tool
-name = pnacl-llc
-parent = Tools
-required_libraries = Analysis AsmParser BitReader NaClBitReader IRReader all-targets NaClAnalysis NaClBitTestUtils NaClTransforms
diff --git a/tools/pnacl-llc/Makefile b/tools/pnacl-llc/Makefile
deleted file mode 100644
index 17dbd02d678..00000000000
--- a/tools/pnacl-llc/Makefile
+++ /dev/null
@@ -1,15 +0,0 @@
-#===- tools/pnacl-llc/Makefile -----------------------------*- Makefile -*-===##
-# 
-#                     The LLVM Compiler Infrastructure
-#
-# This file is distributed under the University of Illinois Open Source
-# License. See LICENSE.TXT for details.
-# 
-##===----------------------------------------------------------------------===##
-
-LEVEL := ../..
-TOOLNAME := pnacl-llc
-LINK_COMPONENTS := nacltransforms all-targets bitreader naclbitreader irreader \
-                   asmparser naclanalysis naclbittestutils nacltransforms
-
-include $(LEVEL)/Makefile.common
diff --git a/tools/pnacl-llc/SRPCStreamer.cpp b/tools/pnacl-llc/SRPCStreamer.cpp
deleted file mode 100644
index 9fd68799273..00000000000
--- a/tools/pnacl-llc/SRPCStreamer.cpp
+++ /dev/null
@@ -1,51 +0,0 @@
-//===-- SRPCStreamer.cpp - Stream bitcode over SRPC  ----------------------===//
-//
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
-//
-//===----------------------------------------------------------------------===//
-#if defined(PNACL_BROWSER_TRANSLATOR)
-#include "SRPCStreamer.h"
-
-#include <errno.h>
-
-llvm::DataStreamer *SRPCStreamer::init(void *(*Callback)(void *), void *arg,
-                                       std::string *ErrMsg) {
-  int err = pthread_create(&CompileThread, NULL, Callback, arg);
-  if (err) {
-    if (ErrMsg) *ErrMsg = std::string(strerror(errno));
-    return NULL;
-  }
-  return &Q;
-}
-
-size_t SRPCStreamer::gotChunk(unsigned char *bytes, size_t len) {
-  if (__sync_fetch_and_add(&Error, 0)) return 0; // Atomic read.
-  return Q.PutBytes(bytes, len);
-}
-
-int SRPCStreamer::streamEnd(std::string *ErrMsg) {
-  Q.SetDone();
-  int err = pthread_join(CompileThread, NULL);
-  __sync_synchronize();
-  if (Error) {
-    if (ErrMsg)
-      *ErrMsg = std::string("PNaCl Translator Error: " + ErrorMessage);
-    return 1;
-  } else if (err) {
-    if (ErrMsg) *ErrMsg = std::string(strerror(errno));
-    return err;
-  }
-  return 0;
-}
-
-void SRPCStreamer::setFatalError(const std::string& message) {
-  __sync_fetch_and_add(&Error, 1);
-  ErrorMessage = message;
-  __sync_synchronize();
-  pthread_exit(NULL);
-}
-
-#endif // __native_client__
diff --git a/tools/pnacl-llc/SRPCStreamer.h b/tools/pnacl-llc/SRPCStreamer.h
deleted file mode 100644
index c04a24a2296..00000000000
--- a/tools/pnacl-llc/SRPCStreamer.h
+++ /dev/null
@@ -1,48 +0,0 @@
-//===-- SRPCStreamer.h - Stream bitcode over SRPC  ------------------------===//
-//
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
-//
-//===----------------------------------------------------------------------===//
-//
-// Manages a Data stream where the producer pushes bytes via SRPC.
-//
-//===----------------------------------------------------------------------===//
-
-#ifndef SRPCSTREAMER_H
-#define SRPCSTREAMER_H
-
-#include <pthread.h>
-#include <string>
-#include "llvm/Support/QueueStreamer.h"
-
-// Class to manage the compliation thread and serve as the interface from
-// the SRPC thread
-class SRPCStreamer  {
-  SRPCStreamer(const SRPCStreamer &) = delete;
-  SRPCStreamer &operator=(const SRPCStreamer &) = delete;
-
-public:
-  SRPCStreamer() : Error(false) {}
-  // Initialize streamer, create a new thread running Callback, and
-  // return a pointer to the DataStreamer the threads will use to
-  // synchronize. On error, return NULL and fill in the ErrorMsg string
-  llvm::DataStreamer *init(void *(*Callback)(void *),
-                           void *arg, std::string *ErrMsg);
-  // Called by the RPC thread. Copy len bytes from buf. Return bytes copied.
-  size_t gotChunk(unsigned char *bytes, size_t len);
-  // Called by the RPC thread. Wait for the compilation thread to finish.
-  int streamEnd(std::string *ErrMsg);
-  // Called by the compilation thread. Set the error condition and also
-  // terminate the thread.
-  void setFatalError(const std::string& message);
-private:
-  int Error;
-  std::string ErrorMessage;
-  llvm::QueueStreamer Q;
-  pthread_t CompileThread;
-};
-
-#endif  // SRPCSTREAMER_H
diff --git a/tools/pnacl-llc/ThreadedFunctionQueue.h b/tools/pnacl-llc/ThreadedFunctionQueue.h
deleted file mode 100644
index 562efc4c5f7..00000000000
--- a/tools/pnacl-llc/ThreadedFunctionQueue.h
+++ /dev/null
@@ -1,117 +0,0 @@
-//===- ThreadedFunctionQueue.h - Function work units for threads -*- C++ -*-==//
-//
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
-//
-//===----------------------------------------------------------------------===//
-
-#ifndef THREADEDFUNCTIONQUEUE_H
-#define THREADEDFUNCTIONQUEUE_H
-
-#include <limits>
-
-#include "llvm/IR/Module.h"
-#include "llvm/Support/ErrorHandling.h"
-
-namespace llvm {
-
-// A "queue" that keeps track of which functions have been assigned to
-// threads and which functions have not yet been assigned. It does not
-// actually use a queue data structure and instead uses a number which
-// tracks the minimum unassigned function ID, expecting each thread
-// to have the same view of function IDs.
-class ThreadedFunctionQueue {
- public:
-  ThreadedFunctionQueue(Module *mod, unsigned NumThreads)
-      : NumThreads(NumThreads),
-        NumFunctions(0),
-        CurrentFunction(0) {
-    assert(NumThreads > 0);
-    size_t Size = 0;
-    for (Module::iterator I = mod->begin(), E = mod->end(); I != E; ++I) {
-      // Only count functions with bodies. At this point nothing should
-      // be "already materialized", so if functions with bodies are
-      // materializable.
-      if (I->isMaterializable() || !I->isDeclaration())
-        Size++;
-    }
-    if (Size > static_cast<size_t>(std::numeric_limits<int>::max()))
-      report_fatal_error("Too many functions");
-    NumFunctions = Size;
-  }
-
-  ~ThreadedFunctionQueue() {}
-
-  // Assign functions in a static manner between threads.
-  bool GrabFunctionStatic(unsigned FuncIndex, unsigned ThreadIndex) const {
-    // Note: This assumes NumThreads == SplitModuleCount, so that
-    // (a) every function of every module is covered by the NumThreads and
-    // (b) no function is covered twice by the threads.
-    assert(ThreadIndex < NumThreads);
-    return FuncIndex % NumThreads == ThreadIndex;
-  }
-
-  // Assign functions between threads dynamically.
-  // Returns true if FuncIndex is unassigned and the calling thread
-  // is assigned functions [FuncIndex, FuncIndex + ChunkSize).
-  // Returns false if the calling thread is not assigned functions
-  // [FuncIndex, FuncIndex + ChunkSize).
-  //
-  // NextIndex will be set to the next unassigned function ID, so that
-  // the calling thread will know which function ID to attempt to grab
-  // next. Each thread may have a different value for the ideal ChunkSize
-  // so it is hard to predict the next available function solely based
-  // on incrementing by ChunkSize.
-  bool GrabFunctionDynamic(unsigned FuncIndex, unsigned ChunkSize,
-                           unsigned &NextIndex) {
-    unsigned Cur = CurrentFunction;
-    if (FuncIndex < Cur) {
-      NextIndex = Cur;
-      return false;
-    }
-    NextIndex = Cur + ChunkSize;
-    unsigned Index;
-    if (Cur == (Index = __sync_val_compare_and_swap(&CurrentFunction,
-                                                    Cur, NextIndex))) {
-      return true;
-    }
-    // If this thread did not grab the function, its idea of NextIndex
-    // may be incorrect since ChunkSize can vary between threads.
-    // Reset NextIndex in that case.
-    NextIndex = Index;
-    return false;
-  }
-
-  // Returns a recommended ChunkSize for use in calling GrabFunctionDynamic().
-  // ChunkSize starts out "large" to reduce synchronization cost. However,
-  // it cannot be too large, otherwise it will encompass too many bytes
-  // and defeats streaming translation. Assigning too many functions to
-  // a single thread also throws off load balancing, so the ChunkSize is
-  // reduced when the remaining number of functions is low so that
-  // load balancing can be achieved near the end.
-  unsigned RecommendedChunkSize() const {
-    int RemainingFuncs = NumFunctions - CurrentFunction;
-    int DynamicChunkSize = RemainingFuncs / (NumThreads * 4);
-    return std::max(1, std::min(8, DynamicChunkSize));
-  }
-
-  // Total number of functions with bodies that should be processed.
-  unsigned Size() const {
-    return NumFunctions;
-  }
-
- private:
-  const unsigned NumThreads;
-  unsigned NumFunctions;
-  volatile unsigned CurrentFunction;
-
-  ThreadedFunctionQueue(
-      const ThreadedFunctionQueue&) = delete;
-  void operator=(const ThreadedFunctionQueue&) = delete;
-};
-
-} // end namespace llvm
-
-#endif
diff --git a/tools/pnacl-llc/ThreadedStreamingCache.cpp b/tools/pnacl-llc/ThreadedStreamingCache.cpp
deleted file mode 100644
index b62ee789084..00000000000
--- a/tools/pnacl-llc/ThreadedStreamingCache.cpp
+++ /dev/null
@@ -1,106 +0,0 @@
-//=- ThreadedStreamingCache.cpp - Cache for StreamingMemoryObject -*- C++ -*-=//
-//
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
-//
-//===----------------------------------------------------------------------===//
-
-#include "ThreadedStreamingCache.h"
-#include "llvm/Support/Compiler.h"
-#include "llvm/Support/Mutex.h"
-#include <cstring>
-
-using namespace llvm;
-using llvm::sys::ScopedLock;
-
-ThreadedStreamingCache::ThreadedStreamingCache(
-    llvm::StreamingMemoryObject *S) : Streamer(S),
-                                      Cache(kCacheSize),
-                                      MinObjectSize(0),
-                                      CacheBase(-1) {
-  static_assert((kCacheSize & (kCacheSize - 1)) == 0,
-                "kCacheSize must be a power of 2");
-}
-
-void ThreadedStreamingCache::fetchCacheLine(uint64_t Address) const {
-  uint64_t Base = Address & kCacheSizeMask;
-  uint64_t BytesFetched;
-  ScopedLock L(StreamerLock);
-  if (Streamer->isValidAddress(Base + kCacheSize - 1)) {
-    BytesFetched = Streamer->readBytes(&Cache[0], kCacheSize, Base);
-    if (BytesFetched != kCacheSize) {
-      llvm::report_fatal_error(
-          "fetchCacheLine failed to fetch a full cache line");
-    }
-    MinObjectSize = Base + kCacheSize;
-  } else {
-    uint64_t End = Streamer->getExtent();
-    assert(End > Address && End <= Base + kCacheSize);
-    BytesFetched = Streamer->readBytes(&Cache[0], End - Base, Base);
-    if (BytesFetched != (End - Base)) {
-      llvm::report_fatal_error(
-          "fetchCacheLine failed to fetch rest of stream");
-    }
-    MinObjectSize = End;
-  }
-  CacheBase = Base;
-}
-
-uint64_t ThreadedStreamingCache::readBytes(uint8_t* Buf, uint64_t Size,
-                                           uint64_t Address) const {
-  // To keep the cache fetch simple, we currently require that no request cross
-  // the cache line. This isn't a problem for the bitcode reader because it only
-  // fetches a byte or a word (word may be 4 to 8 bytes) at a time.
-  uint64_t Upper = Address + Size;
-  if (Address < CacheBase || Upper > CacheBase + kCacheSize) {
-    // If completely outside of a cacheline, fetch the cacheline.
-    if ((Address & kCacheSizeMask) != ((Upper - 1) & kCacheSizeMask))
-      llvm::report_fatal_error("readBytes request spans cache lines");
-    // Fetch a cache line first, which may be partial.
-    fetchCacheLine(Address);
-  }
-  // Now the start Address should at least fit in the cache line,
-  // but Upper may still be beyond the Extent / MinObjectSize, so clamp.
-  if (Upper > MinObjectSize) {
-    // If in the cacheline but stretches beyone the MinObjectSize,
-    // only read up to MinObjectSize (caller uses readBytes to check EOF,
-    // and can guess / try to read more). MinObjectSize should be the same
-    // as EOF in this case otherwise it would have fit in the cacheline.
-    Size = MinObjectSize - Address;
-  }
-  memcpy(Buf, &Cache[Address - CacheBase], Size);
-  return Size;
-}
-
-uint64_t ThreadedStreamingCache::getExtent() const {
-  llvm::report_fatal_error(
-      "getExtent should not be called for pnacl streaming bitcode");
-  return 0;
-}
-
-bool ThreadedStreamingCache::isValidAddress(uint64_t Address) const {
-  if (Address < MinObjectSize)
-    return true;
-  ScopedLock L(StreamerLock);
-  bool Valid = Streamer->isValidAddress(Address);
-  if (Valid)
-    MinObjectSize = Address;
-  return Valid;
-}
-
-bool ThreadedStreamingCache::dropLeadingBytes(size_t S) {
-  ScopedLock L(StreamerLock);
-  return Streamer->dropLeadingBytes(S);
-}
-
-void ThreadedStreamingCache::setKnownObjectSize(size_t Size) {
-  MinObjectSize = Size;
-  ScopedLock L(StreamerLock);
-  Streamer->setKnownObjectSize(Size);
-}
-
-const uint64_t ThreadedStreamingCache::kCacheSize;
-const uint64_t ThreadedStreamingCache::kCacheSizeMask;
-llvm::sys::SmartMutex<false> ThreadedStreamingCache::StreamerLock;
diff --git a/tools/pnacl-llc/ThreadedStreamingCache.h b/tools/pnacl-llc/ThreadedStreamingCache.h
deleted file mode 100644
index f3a28238b94..00000000000
--- a/tools/pnacl-llc/ThreadedStreamingCache.h
+++ /dev/null
@@ -1,77 +0,0 @@
-//==- ThreadedStreamingCache.h - Cache for StreamingMemoryObject -*- C++ -*-==//
-//
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
-//
-//===----------------------------------------------------------------------===//
-
-#ifndef THREADEDSTREAMINGCACHE_H
-#define THREADEDSTREAMINGCACHE_H
-
-#include "llvm/Support/ErrorHandling.h"
-#include "llvm/Support/Mutex.h"
-#include "llvm/Support/StreamingMemoryObject.h"
-
-namespace llvm {
-
-// An implementation of StreamingMemoryObject for use in multithreaded
-// translation. Each thread has one of these objects, each of which has a
-// pointer to a shared StreamingMemoryObject. This object is effectively
-// a thread-local cache for the bitcode streamer to avoid contention, since
-// bits are only read from the bitcode stream one word at a time.
-
-class ThreadedStreamingCache : public llvm::StreamingMemoryObject {
- public:
-  explicit ThreadedStreamingCache(llvm::StreamingMemoryObject *S);
-  uint64_t getExtent() const override;
-  uint64_t readBytes(uint8_t *Buf, uint64_t Size,
-                     uint64_t Address) const override;
-  const uint8_t *getPointer(uint64_t Address,
-                            uint64_t Size) const override {
-    // This could be fixed by ensuring the bytes are fetched and making a copy,
-    // requiring that the bitcode size be known, or otherwise ensuring that
-    // the memory doesn't go away/get reallocated, but it's
-    // not currently necessary. Users that need the pointer don't stream.
-    llvm_unreachable("getPointer in streaming memory objects not allowed");
-    return NULL;
-  }
-  bool isValidAddress(uint64_t Address) const override;
-
-  /// Drop s bytes from the front of the stream, pushing the positions of the
-  /// remaining bytes down by s. This is used to skip past the bitcode header,
-  /// since we don't know a priori if it's present, and we can't put bytes
-  /// back into the stream once we've read them.
-  bool dropLeadingBytes(size_t S) override;
-
-  /// If the data object size is known in advance, many of the operations can
-  /// be made more efficient, so this method should be called before reading
-  /// starts (although it can be called anytime).
-  void setKnownObjectSize(size_t Size) override;
- private:
-  const static uint64_t kCacheSize = 4 * 4096;
-  const static uint64_t kCacheSizeMask = ~(kCacheSize - 1);
-  static llvm::sys::SmartMutex<false> StreamerLock;
-
-  // Fetch up to kCacheSize worth of data starting from Address, into the
-  // CacheBase, and set MinObjectSize to the new known edge.
-  // If at EOF, MinObjectSize reflects the final size.
-  void fetchCacheLine(uint64_t Address) const;
-
-  llvm::StreamingMemoryObject *Streamer;
-  // Cached data for addresses [CacheBase, CacheBase + kCacheSize)
-  mutable std::vector<unsigned char> Cache;
-  // The MemoryObject is at least this size. Used as a cache for isValidAddress.
-  mutable uint64_t MinObjectSize;
-  // Current base address for the cache.
-  mutable uint64_t CacheBase;
-
-  ThreadedStreamingCache(
-      const ThreadedStreamingCache&) = delete;
-  void operator=(const ThreadedStreamingCache&) = delete;
-};
-
-} // namespace llvm
-
-#endif // THREADEDSTREAMINGCACHE_H
diff --git a/tools/pnacl-llc/pnacl-llc.cpp b/tools/pnacl-llc/pnacl-llc.cpp
deleted file mode 100644
index 7d2024d44ef..00000000000
--- a/tools/pnacl-llc/pnacl-llc.cpp
+++ /dev/null
@@ -1,834 +0,0 @@
-//===-- pnacl-llc.cpp - PNaCl-specific llc: pexe ---> nexe  ---------------===//
-//
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
-//
-//===----------------------------------------------------------------------===//
-//
-// pnacl-llc: the core of the PNaCl translator, compiling a pexe into a nexe.
-//
-//===----------------------------------------------------------------------===//
-
-#include "llvm/ADT/STLExtras.h"
-#include "llvm/ADT/Triple.h"
-#include "llvm/Analysis/NaCl.h"
-#include "llvm/Bitcode/NaCl/NaClReaderWriter.h"
-#include "llvm/Bitcode/ReaderWriter.h"
-#include "llvm/CodeGen/CommandFlags.h"
-#include "llvm/CodeGen/LinkAllCodegenComponents.h"
-#include "llvm/IR/DataLayout.h"
-#include "llvm/IR/LLVMContext.h"
-#include "llvm/IR/Module.h"
-#include "llvm/IR/Verifier.h"
-#include "llvm/IRReader/IRReader.h"
-#include "llvm/MC/SubtargetFeature.h"
-#include "llvm/Pass.h"
-#include "llvm/IR/LegacyPassManager.h"
-#include "llvm/Support/CommandLine.h"
-#include "llvm/Support/DataStream.h"
-#include "llvm/Support/Debug.h"
-#include "llvm/Support/ErrorHandling.h"
-#include "llvm/Support/FileSystem.h"
-#include "llvm/Support/FormattedStream.h"
-#include "llvm/Support/Host.h"
-#include "llvm/Support/ManagedStatic.h"
-#include "llvm/Support/PrettyStackTrace.h"
-#include "llvm/Support/Signals.h"
-#include "llvm/Support/SourceMgr.h"
-#include "llvm/Support/StreamingMemoryObject.h"
-#include "llvm/Support/TargetRegistry.h"
-#include "llvm/Support/TargetSelect.h"
-#include "llvm/Support/ToolOutputFile.h"
-#include "llvm/Analysis/TargetLibraryInfo.h"
-#include "llvm/Target/TargetMachine.h"
-#include "llvm/Target/TargetSubtargetInfo.h"
-#include "llvm/Transforms/NaCl.h"
-
-#include "ThreadedFunctionQueue.h"
-#include "ThreadedStreamingCache.h"
-
-#include <pthread.h>
-#include <memory>
-
-using namespace llvm;
-
-// NOTE: When PNACL_BROWSER_TRANSLATOR is defined it means pnacl-llc is built
-// as a sandboxed translator (from pnacl-llc.pexe to pnacl-llc.nexe). In this
-// mode it uses SRPC operations instead of direct OS intefaces.
-#if defined(PNACL_BROWSER_TRANSLATOR)
-int srpc_main(int argc, char **argv);
-int getObjectFileFD(unsigned index);
-DataStreamer *getNaClBitcodeStreamer();
-
-fatal_error_handler_t getSRPCErrorHandler();
-#endif
-
-cl::opt<NaClFileFormat>
-InputFileFormat(
-    "bitcode-format",
-    cl::desc("Define format of input file:"),
-    cl::values(
-        clEnumValN(LLVMFormat, "llvm", "LLVM file (default)"),
-        clEnumValN(PNaClFormat, "pnacl", "PNaCl bitcode file"),
-        clEnumValEnd),
-#if defined(PNACL_BROWSER_TRANSLATOR)
-    cl::init(PNaClFormat)
-#else
-    cl::init(LLVMFormat)
-#endif
-                );
-
-// General options for llc.  Other pass-specific options are specified
-// within the corresponding llc passes, and target-specific options
-// and back-end code generation options are specified with the target machine.
-//
-static cl::opt<std::string>
-InputFilename(cl::Positional, cl::desc("<input bitcode>"), cl::init("-"));
-
-// Primary output filename. If module splitting is used, the other output files
-// will have names derived from this one.
-static cl::opt<std::string>
-MainOutputFilename("o", cl::desc("Output filename"), cl::value_desc("filename"));
-
-// Using bitcode streaming allows compilation of one function at a time. This
-// allows earlier functions to be compiled before later functions are read from
-// the bitcode but of course means no whole-module optimizations. This means
-// that Module passes that run should only touch globals/function declarations
-// and not function bodies, otherwise the streaming and non-streaming code
-// pathes wouldn't emit the same code for each function. For now, streaming is
-// only supported for files and stdin.
-static cl::opt<bool>
-LazyBitcode("streaming-bitcode",
-  cl::desc("Use lazy bitcode streaming for file inputs"),
-  cl::init(false));
-
-static cl::opt<bool>
-PNaClABIVerify("pnaclabi-verify",
-  cl::desc("Verify PNaCl bitcode ABI before translating"),
-  cl::init(false));
-static cl::opt<bool>
-PNaClABIVerifyFatalErrors("pnaclabi-verify-fatal-errors",
-  cl::desc("PNaCl ABI verification errors are fatal"),
-  cl::init(false));
-
-
-static cl::opt<bool>
-NoIntegratedAssembler("no-integrated-as", cl::Hidden,
-                      cl::desc("Disable integrated assembler"));
-
-// Determine optimization level.
-static cl::opt<char>
-OptLevel("O",
-         cl::desc("Optimization level. [-O0, -O1, -O2, or -O3] "
-                  "(default = '-O2')"),
-         cl::Prefix,
-         cl::ZeroOrMore,
-         cl::init(' '));
-
-static cl::opt<std::string>
-UserDefinedTriple("mtriple", cl::desc("Set target triple"));
-
-static cl::opt<bool> NoVerify("disable-verify", cl::Hidden,
-                              cl::desc("Do not verify input module"));
-
-static cl::opt<bool>
-    DisableSimplifyLibCalls("disable-simplify-libcalls",
-                            cl::desc("Disable simplify-libcalls"));
-
-#if defined(PNACL_BROWSER_TRANSLATOR)
-constexpr bool AcceptBitcodeRecordText = false;
-#else
-static cl::opt<bool>
-AcceptBitcodeRecordText(
-    "bitcode-as-text",
-    cl::desc(
-        "Accept textual form of PNaCl bitcode records (i.e. not .ll assembly)"),
-    cl::init(false));
-#endif
-
-static cl::opt<unsigned>
-SplitModuleCount("split-module",
-                 cl::desc("Split PNaCl module"), cl::init(1U));
-
-enum SplitModuleSchedulerKind {
-  SplitModuleDynamic,
-  SplitModuleStatic
-};
-
-static cl::opt<SplitModuleSchedulerKind>
-SplitModuleSched(
-    "split-module-sched",
-    cl::desc("Choose thread scheduler for split module compilation."),
-    cl::values(
-        clEnumValN(SplitModuleDynamic, "dynamic",
-                   "Dynamic thread scheduling (default)"),
-        clEnumValN(SplitModuleStatic, "static",
-                   "Static thread scheduling"),
-        clEnumValEnd),
-    cl::init(SplitModuleDynamic));
-
-/// Compile the module provided to pnacl-llc. The file name for reading the
-/// module and other options are taken from globals populated by command-line
-/// option parsing.
-static int compileModule(StringRef ProgramName);
-
-#if !defined(PNACL_BROWSER_TRANSLATOR)
-static std::unique_ptr<tool_output_file>
-GetOutputStream(const char *TargetName,
-                Triple::OSType OS,
-                std::string OutputFilename) {
-  // If we don't yet have an output filename, make one.
-  if (OutputFilename.empty()) {
-    if (InputFilename == "-")
-      OutputFilename = "-";
-    else {
-      // If InputFilename ends in .bc or .ll, remove it.
-      StringRef IFN = InputFilename;
-      if (IFN.endswith(".bc") || IFN.endswith(".ll"))
-        OutputFilename = IFN.drop_back(3);
-      else
-        OutputFilename = IFN;
-
-      switch (FileType) {
-      case TargetMachine::CGFT_AssemblyFile:
-        if (TargetName[0] == 'c') {
-          if (TargetName[1] == 0)
-            OutputFilename += ".cbe.c";
-          else if (TargetName[1] == 'p' && TargetName[2] == 'p')
-            OutputFilename += ".cpp";
-          else
-            OutputFilename += ".s";
-        } else
-          OutputFilename += ".s";
-        break;
-      case TargetMachine::CGFT_ObjectFile:
-        if (OS == Triple::Win32)
-          OutputFilename += ".obj";
-        else
-          OutputFilename += ".o";
-        break;
-      case TargetMachine::CGFT_Null:
-        OutputFilename += ".null";
-        break;
-      }
-    }
-  }
-
-  // Decide if we need "binary" output.
-  bool Binary = false;
-  switch (FileType) {
-  case TargetMachine::CGFT_AssemblyFile:
-    break;
-  case TargetMachine::CGFT_ObjectFile:
-  case TargetMachine::CGFT_Null:
-    Binary = true;
-    break;
-  }
-
-  // Open the file.
-  std::error_code EC;
-  sys::fs::OpenFlags OpenFlags = sys::fs::F_None;
-  if (!Binary)
-    OpenFlags |= sys::fs::F_Text;
-  auto FDOut = llvm::make_unique<tool_output_file>(OutputFilename, EC,
-                                                   OpenFlags);
-  if (EC) {
-    errs() << EC.message() << '\n';
-    return nullptr;
-  }
-
-  return FDOut;
-}
-#endif // !defined(PNACL_BROWSER_TRANSLATOR)
-
-// main - Entry point for the llc compiler.
-//
-int llc_main(int argc, char **argv) {
-  sys::PrintStackTraceOnErrorSignal();
-  PrettyStackTraceProgram X(argc, argv);
-
-  // Enable debug stream buffering.
-  EnableDebugBuffering = true;
-
-  llvm_shutdown_obj Y;  // Call llvm_shutdown() on exit.
-
-#if defined(PNACL_BROWSER_TRANSLATOR)
-  install_fatal_error_handler(getSRPCErrorHandler(), nullptr);
-#endif
-
-  // Initialize targets first, so that --version shows registered targets.
-  InitializeAllTargets();
-  InitializeAllTargetMCs();
-  InitializeAllAsmPrinters();
-#if !defined(PNACL_BROWSER_TRANSLATOR)
-  // Prune asm parsing from sandboxed translator.
-  // Do not prune "AsmPrinters" because that includes
-  // the direct object emission.
-  InitializeAllAsmParsers();
-#endif
-
-  // Initialize codegen and IR passes used by pnacl-llc so that the -print-after,
-  // -print-before, and -stop-after options work.
-  PassRegistry *Registry = PassRegistry::getPassRegistry();
-  initializeCore(*Registry);
-  initializeCodeGen(*Registry);
-  initializeLoopStrengthReducePass(*Registry);
-  initializeLowerIntrinsicsPass(*Registry);
-  initializeUnreachableBlockElimPass(*Registry);
-
-  // Register the target printer for --version.
-  cl::AddExtraVersionPrinter(TargetRegistry::printRegisteredTargetsForVersion);
-
-  // Enable the PNaCl ABI verifier by default in sandboxed mode.
-#if defined(PNACL_BROWSER_TRANSLATOR)
-  PNaClABIVerify = true;
-  PNaClABIVerifyFatalErrors = true;
-#endif
-
-  cl::ParseCommandLineOptions(argc, argv, "pnacl-llc\n");
-
-#if defined(PNACL_BROWSER_TRANSLATOR)
-  // If the user explicitly requests LLVM format in sandboxed mode
-  // (where the default is PNaCl format), they probably want debug
-  // metadata enabled.
-  if (InputFileFormat == LLVMFormat) {
-    PNaClABIAllowDebugMetadata = true;
-  }
-#else
-  if (AcceptBitcodeRecordText) {
-    if (LazyBitcode)
-      report_fatal_error(
-          "Can't stream file inputs when reading bitcode records as text");
-    if (SplitModuleCount != 1)
-      report_fatal_error(
-          "Can't split module when using bitcode records as text");
-    if (InputFileFormat != PNaClFormat)
-      report_fatal_error(
-          "Can't parse non-pnacl bitcode files when reading bitcode records"
-          " as text");
-  }
-#endif
-
-  if (SplitModuleCount > 1)
-    LLVMStartMultithreaded();
-
-  return compileModule(argv[0]);
-}
-
-static void CheckABIVerifyErrors(PNaClABIErrorReporter &Reporter,
-                                 const Twine &Name) {
-  if (PNaClABIVerify && Reporter.getErrorCount() > 0) {
-    std::string errors;
-    raw_string_ostream os(errors);
-    os << (PNaClABIVerifyFatalErrors ? "ERROR: " : "WARNING: ");
-    os << Name << " is not valid PNaCl bitcode:\n";
-    Reporter.printErrors(os);
-    if (PNaClABIVerifyFatalErrors) {
-      report_fatal_error(os.str());
-    }
-    errs() << os.str();
-  }
-  Reporter.reset();
-}
-
-#if !defined(PNACL_BROWSER_TRANSLATOR)
-// Read in module from bitcode text records in Filename. Returns
-// module if successful (using the given Context). Otherwise, return
-// error message using Err, and return nullptr. Verbose, if non-null,
-// may contain more verbose descriptions of the errors found while
-// parsing.
-static std::unique_ptr<Module> parseBitcodeRecordsAsText(
-    StringRef Filename,
-    SMDiagnostic &Err,
-    raw_ostream *Verbose,
-    LLVMContext &Context) {
-  ErrorOr<Module *> M = parseNaClBitcodeText(Filename, Context, Verbose);
-  if (!M) {
-    Err = SMDiagnostic(Filename, SourceMgr::DK_Error, M.getError().message());
-    return nullptr;
-  }
-  std::unique_ptr<Module> Mptr(M.get());
-  return std::move(Mptr);
-}
-#endif
-
-static std::unique_ptr<Module> getModule(
-    StringRef ProgramName, LLVMContext &Context,
-    StreamingMemoryObject *StreamingObject) {
-  std::unique_ptr<Module> M;
-  SMDiagnostic Err;
-  std::string VerboseBuffer;
-  raw_string_ostream VerboseStrm(VerboseBuffer);
-  if (LazyBitcode) {
-    std::string StrError;
-    switch (InputFileFormat) {
-    case PNaClFormat: {
-      std::unique_ptr<StreamingMemoryObject> Cache(
-          new ThreadedStreamingCache(StreamingObject));
-      M.reset(getNaClStreamedBitcodeModule(
-          InputFilename, Cache.release(), Context, &VerboseStrm, &StrError));
-      break;
-    }
-    case LLVMFormat: {
-      std::unique_ptr<StreamingMemoryObject> Cache(
-          new ThreadedStreamingCache(StreamingObject));
-      ErrorOr<std::unique_ptr<Module>> MOrErr =
-          getStreamedBitcodeModule(
-          InputFilename, Cache.release(), Context);
-      M = std::move(*MOrErr);
-      break;
-    }
-    case AutodetectFileFormat:
-      report_fatal_error("Command can't autodetect file format!");
-    }
-    if (!StrError.empty())
-      Err = SMDiagnostic(InputFilename, SourceMgr::DK_Error, StrError);
-  } else {
-#if defined(PNACL_BROWSER_TRANSLATOR)
-    llvm_unreachable("native client SRPC only supports streaming");
-#else
-    if (AcceptBitcodeRecordText) {
-      M = parseBitcodeRecordsAsText(InputFilename, Err, &VerboseStrm, Context);
-    } else {
-      // Parses binary bitcode as well as textual assembly
-      // (so pulls in more code into pnacl-llc).
-      M = NaClParseIRFile(InputFilename, InputFileFormat, Err, &VerboseStrm,
-                          Context);
-    }
-#endif
-  }
-  if (!M) {
-#if defined(PNACL_BROWSER_TRANSLATOR)
-    report_fatal_error(VerboseStrm.str() + Err.getMessage());
-#else
-    // Err.print is prettier, so use it for the non-sandboxed translator.
-    Err.print(ProgramName.data(), errs());
-    errs() << VerboseStrm.str();
-    return nullptr;
-#endif
-  }
-  return std::move(M);
-}
-
-static cl::opt<bool>
-ExternalizeAll("externalize",
-               cl::desc("Externalize all symbols"),
-               cl::init(false));
-
-static int runCompilePasses(Module *ModuleRef,
-                            unsigned ModuleIndex,
-                            ThreadedFunctionQueue *FuncQueue,
-                            const Triple &TheTriple,
-                            TargetMachine &Target,
-                            StringRef ProgramName,
-                            raw_pwrite_stream &OS){
-  PNaClABIErrorReporter ABIErrorReporter;
-
-  if (SplitModuleCount > 1 || ExternalizeAll) {
-    // Add function and global names, and give them external linkage.
-    // This relies on LLVM's consistent auto-generation of names, we could
-    // maybe do our own in case something changes there.
-    for (Function &F : *ModuleRef) {
-      if (!F.hasName())
-        F.setName("Function");
-      if (F.hasInternalLinkage())
-        F.setLinkage(GlobalValue::ExternalLinkage);
-    }
-    for (Module::global_iterator GI = ModuleRef->global_begin(),
-         GE = ModuleRef->global_end();
-         GI != GE; ++GI) {
-      if (!GI->hasName())
-        GI->setName("Global");
-      if (GI->hasInternalLinkage())
-        GI->setLinkage(GlobalValue::ExternalLinkage);
-    }
-    if (ModuleIndex > 0) {
-      // Remove the initializers for all global variables, turning them into
-      // declarations.
-      for (Module::global_iterator GI = ModuleRef->global_begin(),
-          GE = ModuleRef->global_end();
-          GI != GE; ++GI) {
-        assert(GI->hasInitializer() && "Global variable missing initializer");
-        Constant *Init = GI->getInitializer();
-        GI->setInitializer(nullptr);
-        if (Init->getNumUses() == 0)
-          Init->destroyConstant();
-      }
-    }
-  }
-
-  // Make all non-weak symbols hidden for better code. We cannot do
-  // this for weak symbols. The linker complains when some weak
-  // symbols are not resolved.
-  for (Function &F : *ModuleRef) {
-    if (!F.isWeakForLinker() && !F.hasLocalLinkage())
-      F.setVisibility(GlobalValue::HiddenVisibility);
-  }
-  for (Module::global_iterator GI = ModuleRef->global_begin(),
-           GE = ModuleRef->global_end();
-       GI != GE; ++GI) {
-    if (!GI->isWeakForLinker() && !GI->hasLocalLinkage())
-      GI->setVisibility(GlobalValue::HiddenVisibility);
-  }
-
-  // Build up all of the passes that we want to do to the module.
-  // We always use a FunctionPassManager to divide up the functions
-  // among threads (instead of a whole-module PassManager).
-  std::unique_ptr<legacy::FunctionPassManager> PM(
-      new legacy::FunctionPassManager(ModuleRef));
-
-  // Add the target data from the target machine, if it exists, or the module.
-  if (const DataLayout *DL = Target.getDataLayout())
-    ModuleRef->setDataLayout(*DL);
-
-  // For conformance with llc, we let the user disable LLVM IR verification with
-  // -disable-verify. Unlike llc, when LLVM IR verification is enabled we only
-  // run it once, before PNaCl ABI verification.
-  if (!NoVerify)
-    PM->add(createVerifierPass());
-
-  // Add the ABI verifier pass before the analysis and code emission passes.
-  if (PNaClABIVerify)
-    PM->add(createPNaClABIVerifyFunctionsPass(&ABIErrorReporter));
-
-  // Add the intrinsic resolution pass. It assumes ABI-conformant code.
-  PM->add(createResolvePNaClIntrinsicsPass());
-
-  // Add an appropriate TargetLibraryInfo pass for the module's triple.
-  TargetLibraryInfoImpl TLII(TheTriple);
-
-  // The -disable-simplify-libcalls flag actually disables all builtin optzns.
-  if (DisableSimplifyLibCalls)
-    TLII.disableAllFunctions();
-  PM->add(new TargetLibraryInfoWrapperPass(TLII));
-
-  // Allow subsequent passes and the backend to better optimize instructions
-  // that were simplified for PNaCl's ABI. This pass uses the TargetLibraryInfo
-  // above.
-  PM->add(createBackendCanonicalizePass());
-
-  // Ask the target to add backend passes as necessary. We explicitly ask it
-  // not to add the verifier pass because we added it earlier.
-  if (Target.addPassesToEmitFile(*PM, OS, FileType,
-                                 /* DisableVerify */ true)) {
-    errs() << ProgramName
-    << ": target does not support generation of this file type!\n";
-    return 1;
-  }
-
-  PM->doInitialization();
-  unsigned FuncIndex = 0;
-  switch (SplitModuleSched) {
-  case SplitModuleStatic:
-    for (Function &F : *ModuleRef) {
-      if (FuncQueue->GrabFunctionStatic(FuncIndex, ModuleIndex)) {
-        PM->run(F);
-        CheckABIVerifyErrors(ABIErrorReporter, "Function " + F.getName());
-        F.Dematerialize();
-      }
-      ++FuncIndex;
-    }
-    break;
-  case SplitModuleDynamic:
-    unsigned ChunkSize = 0;
-    unsigned NumFunctions = FuncQueue->Size();
-    Module::iterator I = ModuleRef->begin();
-    while (FuncIndex < NumFunctions) {
-      ChunkSize = FuncQueue->RecommendedChunkSize();
-      unsigned NextIndex;
-      bool grabbed =
-          FuncQueue->GrabFunctionDynamic(FuncIndex, ChunkSize, NextIndex);
-      if (grabbed) {
-        while (FuncIndex < NextIndex) {
-          if (!I->isMaterializable() && I->isDeclaration()) {
-            ++I;
-            continue;
-          }
-          PM->run(*I);
-          CheckABIVerifyErrors(ABIErrorReporter, "Function " + I->getName());
-          I->Dematerialize();
-          ++FuncIndex;
-          ++I;
-        }
-      } else {
-        while (FuncIndex < NextIndex) {
-          if (!I->isMaterializable() && I->isDeclaration()) {
-            ++I;
-            continue;
-          }
-          ++FuncIndex;
-          ++I;
-        }
-      }
-    }
-    break;
-  }
-  PM->doFinalization();
-  return 0;
-}
-
-static int compileSplitModule(const TargetOptions &Options,
-                              const Triple &TheTriple,
-                              const Target *TheTarget,
-                              const std::string &FeaturesStr,
-                              CodeGenOpt::Level OLvl,
-                              const StringRef &ProgramName,
-                              Module *GlobalModuleRef,
-                              StreamingMemoryObject *StreamingObject,
-                              unsigned ModuleIndex,
-                              ThreadedFunctionQueue *FuncQueue) {
-  std::auto_ptr<TargetMachine>
-    target(TheTarget->createTargetMachine(TheTriple.getTriple(),
-                                          MCPU, FeaturesStr, Options,
-                                          RelocModel, CMModel, OLvl));
-  assert(target.get() && "Could not allocate target machine!");
-  TargetMachine &Target = *target.get();
-  if (RelaxAll.getNumOccurrences() > 0 &&
-      FileType != TargetMachine::CGFT_ObjectFile)
-    errs() << ProgramName
-             << ": warning: ignoring -mc-relax-all because filetype != obj";
-  // The OwningPtrs are only used if we are not the primary module.
-  std::unique_ptr<LLVMContext> C;
-  std::unique_ptr<Module> M;
-  Module *ModuleRef = nullptr;
-
-  if (ModuleIndex == 0) {
-    ModuleRef = GlobalModuleRef;
-  } else {
-    C.reset(new LLVMContext());
-    M = getModule(ProgramName, *C, StreamingObject);
-    if (!M)
-      return 1;
-    // M owns the temporary module, but use a reference through ModuleRef
-    // to also work in the case we are using GlobalModuleRef.
-    ModuleRef = M.get();
-
-    // Add declarations for external functions required by PNaCl. The
-    // ResolvePNaClIntrinsics function pass running during streaming
-    // depends on these declarations being in the module.
-    std::unique_ptr<ModulePass> AddPNaClExternalDeclsPass(
-        createAddPNaClExternalDeclsPass());
-    AddPNaClExternalDeclsPass->runOnModule(*ModuleRef);
-    AddPNaClExternalDeclsPass.reset();
-  }
-
-  ModuleRef->setTargetTriple(Triple::normalize(UserDefinedTriple));
-
-  {
-#if !defined(PNACL_BROWSER_TRANSLATOR)
-      // Figure out where we are going to send the output.
-    std::string N(MainOutputFilename);
-    raw_string_ostream OutFileName(N);
-    if (ModuleIndex > 0)
-      OutFileName << ".module" << ModuleIndex;
-    std::unique_ptr<tool_output_file> Out =
-        GetOutputStream(TheTarget->getName(), TheTriple.getOS(),
-                         OutFileName.str());
-    if (!Out) return 1;
-    raw_pwrite_stream *OS = &Out->os();
-    std::unique_ptr<buffer_ostream> BOS;
-    if (FileType != TargetMachine::CGFT_AssemblyFile &&
-        !Out->os().supportsSeeking()) {
-      BOS = make_unique<buffer_ostream>(*OS);
-      OS = BOS.get();
-    }
-#else
-    auto OS = llvm::make_unique<raw_fd_ostream>(
-        getObjectFileFD(ModuleIndex), /* ShouldClose */ true);
-    OS->SetBufferSize(1 << 20);
-#endif
-    int ret = runCompilePasses(ModuleRef, ModuleIndex, FuncQueue,
-                               TheTriple, Target, ProgramName,
-                               *OS);
-    if (ret)
-      return ret;
-#if defined(PNACL_BROWSER_TRANSLATOR)
-    OS->flush();
-#else
-    // Declare success.
-    Out->keep();
-#endif // PNACL_BROWSER_TRANSLATOR
-  }
-  return 0;
-}
-
-struct ThreadData {
-  const TargetOptions *Options;
-  const Triple *TheTriple;
-  const Target *TheTarget;
-  std::string FeaturesStr;
-  CodeGenOpt::Level OLvl;
-  std::string ProgramName;
-  Module *GlobalModuleRef;
-  StreamingMemoryObject *StreamingObject;
-  unsigned ModuleIndex;
-  ThreadedFunctionQueue *FuncQueue;
-};
-
-
-static void *runCompileThread(void *arg) {
-  struct ThreadData *Data = static_cast<ThreadData *>(arg);
-  int ret = compileSplitModule(*Data->Options,
-                               *Data->TheTriple,
-                               Data->TheTarget,
-                               Data->FeaturesStr,
-                               Data->OLvl,
-                               Data->ProgramName,
-                               Data->GlobalModuleRef,
-                               Data->StreamingObject,
-                               Data->ModuleIndex,
-                               Data->FuncQueue);
-  return reinterpret_cast<void *>(static_cast<intptr_t>(ret));
-}
-
-static int compileModule(StringRef ProgramName) {
-  // Use a new context instead of the global context for the main module. It must
-  // outlive the module object, declared below. We do this because
-  // lib/CodeGen/PseudoSourceValue.cpp gets a type from the global context and
-  // races with any other use of the context. Rather than doing an invasive
-  // plumbing change to fix it, we work around it by using a new context here
-  // and leaving PseudoSourceValue as the only user of the global context.
-  std::unique_ptr<LLVMContext> MainContext(new LLVMContext());
-  std::unique_ptr<Module> MainMod;
-  Triple TheTriple;
-  PNaClABIErrorReporter ABIErrorReporter;
-  std::unique_ptr<StreamingMemoryObject> StreamingObject;
-
-  if (!MainContext) return 1;
-
-#if defined(PNACL_BROWSER_TRANSLATOR)
-  StreamingObject.reset(
-      new StreamingMemoryObjectImpl(getNaClBitcodeStreamer()));
-#else
-  if (LazyBitcode) {
-    std::string StrError;
-    DataStreamer* FileStreamer(getDataFileStreamer(InputFilename, &StrError));
-    if (!StrError.empty()) {
-      SMDiagnostic Err(InputFilename, SourceMgr::DK_Error, StrError);
-      Err.print(ProgramName.data(), errs());
-    }
-    if (!FileStreamer)
-      return 1;
-    StreamingObject.reset(new StreamingMemoryObjectImpl(FileStreamer));
-  }
-#endif
-  MainMod = getModule(ProgramName, *MainContext.get(), StreamingObject.get());
-
-  if (!MainMod) return 1;
-
-  if (PNaClABIVerify) {
-    // Verify the module (but not the functions yet)
-    std::unique_ptr<ModulePass> VerifyPass(
-        createPNaClABIVerifyModulePass(&ABIErrorReporter, LazyBitcode));
-    VerifyPass->runOnModule(*MainMod);
-    CheckABIVerifyErrors(ABIErrorReporter, "Module");
-    VerifyPass.reset();
-  }
-
-  // Add declarations for external functions required by PNaCl. The
-  // ResolvePNaClIntrinsics function pass running during streaming
-  // depends on these declarations being in the module.
-  std::unique_ptr<ModulePass> AddPNaClExternalDeclsPass(
-      createAddPNaClExternalDeclsPass());
-  AddPNaClExternalDeclsPass->runOnModule(*MainMod);
-  AddPNaClExternalDeclsPass.reset();
-
-  if (UserDefinedTriple.empty()) {
-    report_fatal_error("-mtriple must be set to a target triple for pnacl-llc");
-  } else {
-    MainMod->setTargetTriple(Triple::normalize(UserDefinedTriple));
-    TheTriple = Triple(MainMod->getTargetTriple());
-  }
-
-  // Get the target specific parser.
-  std::string Error;
-  const Target *TheTarget = TargetRegistry::lookupTarget(MArch, TheTriple,
-                                                         Error);
-  if (!TheTarget) {
-    errs() << ProgramName << ": " << Error;
-    return 1;
-  }
-
-  TargetOptions Options = InitTargetOptionsFromCodeGenFlags();
-  Options.DisableIntegratedAS = NoIntegratedAssembler;
-  Options.MCOptions.AsmVerbose = true;
-#if defined(__native_client__)
-  // This enables LLVM MC to write instruction padding directly into fragments
-  // reducing memory usage of the translator. However, this could result in
-  // suboptimal machine code since we cannot use short jumps where possible
-  // which is why we enable this for sandboxed translator case.
-  Options.MCOptions.MCRelaxAll = true;
-#endif
-
-  if (GenerateSoftFloatCalls)
-    FloatABIForCalls = FloatABI::Soft;
-
-  // Package up features to be passed to target/subtarget
-  std::string FeaturesStr;
-  if (MAttrs.size()) {
-    SubtargetFeatures Features;
-    for (unsigned i = 0; i != MAttrs.size(); ++i)
-      Features.AddFeature(MAttrs[i]);
-    FeaturesStr = Features.getString();
-  }
-
-  CodeGenOpt::Level OLvl = CodeGenOpt::Default;
-  switch (OptLevel) {
-  default:
-    errs() << ProgramName << ": invalid optimization level.\n";
-    return 1;
-  case ' ': break;
-  case '0': OLvl = CodeGenOpt::None; break;
-  case '1': OLvl = CodeGenOpt::Less; break;
-  case '2': OLvl = CodeGenOpt::Default; break;
-  case '3': OLvl = CodeGenOpt::Aggressive; break;
-  }
-
-  SmallVector<pthread_t, 4> Pthreads(SplitModuleCount);
-  SmallVector<ThreadData, 4> ThreadDatas(SplitModuleCount);
-  ThreadedFunctionQueue FuncQueue(MainMod.get(), SplitModuleCount);
-
-  if (SplitModuleCount == 1) {
-    // No need for dynamic scheduling with one thread.
-    SplitModuleSched = SplitModuleStatic;
-    return compileSplitModule(Options, TheTriple, TheTarget, FeaturesStr,
-                              OLvl, ProgramName, MainMod.get(), nullptr, 0,
-                              &FuncQueue);
-  }
-
-  for(unsigned ModuleIndex = 0; ModuleIndex < SplitModuleCount; ++ModuleIndex) {
-    ThreadDatas[ModuleIndex].Options = &Options;
-    ThreadDatas[ModuleIndex].TheTriple = &TheTriple;
-    ThreadDatas[ModuleIndex].TheTarget = TheTarget;
-    ThreadDatas[ModuleIndex].FeaturesStr = FeaturesStr;
-    ThreadDatas[ModuleIndex].OLvl = OLvl;
-    ThreadDatas[ModuleIndex].ProgramName = ProgramName.str();
-    ThreadDatas[ModuleIndex].GlobalModuleRef = MainMod.get();
-    ThreadDatas[ModuleIndex].StreamingObject = StreamingObject.get();
-    ThreadDatas[ModuleIndex].ModuleIndex = ModuleIndex;
-    ThreadDatas[ModuleIndex].FuncQueue = &FuncQueue;
-    if (pthread_create(&Pthreads[ModuleIndex], nullptr, runCompileThread,
-                        &ThreadDatas[ModuleIndex])) {
-      report_fatal_error("Failed to create thread");
-    }
-  }
-  for(unsigned ModuleIndex = 0; ModuleIndex < SplitModuleCount; ++ModuleIndex) {
-    void *retval;
-    if (pthread_join(Pthreads[ModuleIndex], &retval))
-      report_fatal_error("Failed to join thread");
-    intptr_t ret = reinterpret_cast<intptr_t>(retval);
-    if (ret != 0)
-      report_fatal_error("Thread returned nonzero");
-  }
-  return 0;
-}
-
-int main(int argc, char **argv) {
-#if defined(PNACL_BROWSER_TRANSLATOR)
-  return srpc_main(argc, argv);
-#else
-  return llc_main(argc, argv);
-#endif // PNACL_BROWSER_TRANSLATOR
-}
diff --git a/tools/pnacl-llc/srpc_main.cpp b/tools/pnacl-llc/srpc_main.cpp
deleted file mode 100644
index 02d6241ac8f..00000000000
--- a/tools/pnacl-llc/srpc_main.cpp
+++ /dev/null
@@ -1,311 +0,0 @@
-//===-- srpc_main.cpp - PNaCl sandboxed translator invocation -------------===//
-//
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
-//
-//===----------------------------------------------------------------------===//
-//
-// Main entry point and callback handler code for the in-browser sandboxed
-// translator.  The interface between this code and the browser is through
-// the NaCl IRT.
-//
-//===----------------------------------------------------------------------===//
-
-#if defined(PNACL_BROWSER_TRANSLATOR)
-
-// Headers which are not properly part of the SDK are included by their
-// path in the NaCl tree.
-#ifdef __pnacl__
-#include "native_client/src/untrusted/nacl/pnacl.h"
-#endif // __pnacl__
-
-#include "SRPCStreamer.h"
-
-#include "llvm/ADT/STLExtras.h"
-#include "llvm/Option/Option.h"
-#include "llvm/Support/ErrorHandling.h"
-
-#include <argz.h>
-#include <cstdio>
-#include <cstdlib>
-#include <cstring>
-#include <irt.h>
-#include <irt_dev.h>
-#include <string>
-#include <vector>
-
-using namespace llvm;
-using namespace llvm::opt;
-
-// Imported from pnacl-llc.cpp
-extern int llc_main(int ArgC, char **ArgV);
-
-namespace {
-
-// The filename used internally for looking up the bitcode file.
-const char kBitcodeFilename[] = "pnacl.pexe";
-// The filename used internally for looking up the object code file.
-const char kObjectFilename[] = "pnacl.o";
-// Object which manages streaming bitcode over IPC and threading.
-// TODO(jvoung): rename this and other "SRPC" things to not refer to SRPC.
-SRPCStreamer *gIPCStreamer;
-// FDs of the object file(s).
-std::vector<int> gObjectFiles;
-
-DataStreamer *gNaClBitcodeStreamer;
-
-void getIRTInterfaces(nacl_irt_private_pnacl_translator_compile &IRTFuncs) {
-  size_t QueryResult =
-      nacl_interface_query(NACL_IRT_PRIVATE_PNACL_TRANSLATOR_COMPILE_v0_1,
-                           &IRTFuncs, sizeof(IRTFuncs));
-  if (QueryResult != sizeof(IRTFuncs))
-    llvm::report_fatal_error("Failed to get translator compile IRT interface");
-}
-
-int DoTranslate(ArgStringList *CmdLineArgs) {
-  if (!CmdLineArgs)
-    return 1;
-  // Make an ArgV array from the input vector.
-  size_t ArgC = CmdLineArgs->size();
-  char **ArgV = new char *[ArgC + 1];
-  for (size_t i = 0; i < ArgC; ++i) {
-    // llc_main will not mutate the command line, so this is safe.
-    ArgV[i] = const_cast<char *>((*CmdLineArgs)[i]);
-  }
-  ArgV[ArgC] = nullptr;
-  // Call main.
-  return llc_main(static_cast<int>(ArgC), ArgV);
-}
-
-void AddFixedArguments(ArgStringList *CmdLineArgs) {
-  // Add fixed arguments to the command line.  These specify the bitcode
-  // and object code filenames, removing them from the contract with the
-  // coordinator.
-  CmdLineArgs->push_back(kBitcodeFilename);
-  CmdLineArgs->push_back("-o");
-  CmdLineArgs->push_back(kObjectFilename);
-}
-
-bool AddDefaultCPU(ArgStringList *CmdLineArgs) {
-#if defined(__pnacl__)
-  switch (__builtin_nacl_target_arch()) {
-  case PnaclTargetArchitectureX86_32: {
-    CmdLineArgs->push_back("-mcpu=pentium4m");
-    break;
-  }
-  case PnaclTargetArchitectureX86_64: {
-    CmdLineArgs->push_back("-mcpu=x86-64");
-    break;
-  }
-  case PnaclTargetArchitectureARM_32: {
-    CmdLineArgs->push_back("-mcpu=cortex-a9");
-    break;
-  }
-  case PnaclTargetArchitectureMips_32: {
-    CmdLineArgs->push_back("-mcpu=mips32r2");
-    break;
-  }
-  default:
-    fprintf(stderr, "no target architecture match.\n");
-    return false;
-  }
-// Some cases for building this with nacl-gcc or nacl-clang.
-#elif defined(__i386__)
-  CmdLineArgs->push_back("-mcpu=pentium4m");
-#elif defined(__x86_64__)
-  CmdLineArgs->push_back("-mcpu=x86-64");
-#elif defined(__arm__)
-  CmdLineArgs->push_back("-mcpu=cortex-a9");
-#else
-#error "Unknown architecture"
-#endif
-  return true;
-}
-
-bool HasCPUOverride(ArgStringList *CmdLineArgs) {
-  const char *Mcpu = "-mcpu";
-  size_t McpuLen = strlen(Mcpu);
-  for (const char *Arg : *CmdLineArgs) {
-    if (strncmp(Arg, Mcpu, McpuLen) == 0) {
-      return true;
-    }
-  }
-  return false;
-}
-
-ArgStringList *GetDefaultCommandLine() {
-  ArgStringList *command_line = new ArgStringList;
-  // First, those common to all architectures.
-  static const char *common_args[] = { "pnacl_translator", "-filetype=obj" };
-  for (size_t i = 0; i < array_lengthof(common_args); ++i) {
-    command_line->push_back(common_args[i]);
-  }
-  // Then those particular to a platform.
-  static const char *llc_args_x8632[] = {"-mtriple=i686-none-nacl-gnu",
-                                         nullptr};
-  static const char *llc_args_x8664[] = {"-mtriple=x86_64-none-nacl-gnu",
-                                         nullptr};
-  static const char *llc_args_arm[] = {"-mtriple=armv7a-none-nacl-gnueabi",
-                                       "-mattr=+neon", "-float-abi=hard",
-                                       nullptr};
-  static const char *llc_args_mips32[] = {"-mtriple=mipsel-none-nacl-gnu",
-                                          nullptr};
-
-  const char **llc_args = nullptr;
-#if defined(__pnacl__)
-  switch (__builtin_nacl_target_arch()) {
-  case PnaclTargetArchitectureX86_32: {
-    llc_args = llc_args_x8632;
-    break;
-  }
-  case PnaclTargetArchitectureX86_64: {
-    llc_args = llc_args_x8664;
-    break;
-  }
-  case PnaclTargetArchitectureARM_32: {
-    llc_args = llc_args_arm;
-    break;
-  }
-  case PnaclTargetArchitectureMips_32: {
-    llc_args = llc_args_mips32;
-    break;
-  }
-  default:
-    fprintf(stderr, "no target architecture match.\n");
-    delete command_line;
-    return nullptr;
-  }
-// Some cases for building this with nacl-gcc.
-#elif defined(__i386__)
-  (void)llc_args_x8664;
-  (void)llc_args_arm;
-  llc_args = llc_args_x8632;
-#elif defined(__x86_64__)
-  (void)llc_args_x8632;
-  (void)llc_args_arm;
-  llc_args = llc_args_x8664;
-#elif defined(__arm__)
-  (void)llc_args_x8632;
-  (void)llc_args_x8664;
-  llc_args = llc_args_arm;
-#else
-#error "Unknown architecture"
-#endif
-  for (size_t i = 0; llc_args[i] != nullptr; i++)
-    command_line->push_back(llc_args[i]);
-  return command_line;
-}
-
-// Data passed from main thread to compile thread.
-// Takes ownership of the commandline vector.
-class StreamingThreadData {
-public:
-  StreamingThreadData(int module_count, ArgStringList *cmd_line_vec)
-      : module_count_(module_count), cmd_line_vec_(cmd_line_vec) {}
-  ArgStringList *CmdLineVec() const { return cmd_line_vec_.get(); }
-  int module_count_;
-  const std::unique_ptr<ArgStringList> cmd_line_vec_;
-};
-
-void *run_streamed(void *arg) {
-  StreamingThreadData *data = reinterpret_cast<StreamingThreadData *>(arg);
-  data->CmdLineVec()->push_back("-streaming-bitcode");
-  if (DoTranslate(data->CmdLineVec()) != 0) {
-    // llc_main only returns 1 (as opposed to calling report_fatal_error)
-    // in conditions we never expect to see in the browser (e.g. bad
-    // command-line flags).
-    gIPCStreamer->setFatalError("llc_main unspecified failure");
-    return nullptr;
-  }
-  delete data;
-  return nullptr;
-}
-
-char *onInitCallback(uint32_t NumThreads, int *ObjFileFDs,
-                     size_t ObjFileFDCount, char **ArgV, size_t ArgC) {
-  ArgStringList *cmd_line_vec = GetDefaultCommandLine();
-  if (!cmd_line_vec) {
-    return "Failed to get default commandline.";
-  }
-  AddFixedArguments(cmd_line_vec);
-
-  // The IRT should check if this is beyond the max.
-  if (NumThreads < 1) {
-    return "Invalid module split count.";
-  }
-
-  gObjectFiles.clear();
-  for (uint32_t i = 0; i < NumThreads; ++i) {
-    gObjectFiles.push_back(ObjFileFDs[i]);
-  }
-
-  // Make a copy of the extra commandline arguments.
-  for (size_t i = 0; i < ArgC; ++i) {
-    cmd_line_vec->push_back(strdup(ArgV[i]));
-  }
-  // Make sure some -mcpu override exists for now to prevent
-  // auto-cpu feature detection from triggering instructions that
-  // we do not validate yet.
-  if (!HasCPUOverride(cmd_line_vec)) {
-    AddDefaultCPU(cmd_line_vec);
-  }
-
-  gIPCStreamer = new SRPCStreamer();
-  std::string StrError;
-  // cmd_line_vec is freed by the translation thread in run_streamed.
-  StreamingThreadData *thread_data =
-      new StreamingThreadData(NumThreads, cmd_line_vec);
-  gNaClBitcodeStreamer = gIPCStreamer->init(
-      run_streamed, reinterpret_cast<void *>(thread_data), &StrError);
-  if (gNaClBitcodeStreamer) {
-    return nullptr;
-  } else {
-    return strdup(StrError.c_str());
-  }
-}
-
-int onDataCallback(const void *Data, size_t NumBytes) {
-  unsigned char *CharData =
-      reinterpret_cast<unsigned char *>(const_cast<void *>(Data));
-  return gIPCStreamer->gotChunk(CharData, NumBytes) != NumBytes;
-}
-
-char *onEndCallback() {
-  std::string StrError;
-  if (gIPCStreamer->streamEnd(&StrError)) {
-    return strdup(StrError.c_str());
-  }
-  return nullptr;
-}
-
-const struct nacl_irt_pnacl_compile_funcs gLLCCallbacks = {
-  &onInitCallback, &onDataCallback, &onEndCallback
-};
-
-} // namespace
-
-int getObjectFileFD(unsigned Index) {
-  assert(Index < gObjectFiles.size());
-  return gObjectFiles[Index];
-}
-
-DataStreamer *getNaClBitcodeStreamer() { return gNaClBitcodeStreamer; }
-
-// Called from the compilation thread
-void FatalErrorHandler(void *user_data, const std::string& reason,
-                       bool gen_crash_diag) {
-  gIPCStreamer->setFatalError(reason);
-}
-
-fatal_error_handler_t getSRPCErrorHandler() { return FatalErrorHandler; }
-
-int srpc_main(int ArgC, char **ArgV) {
-  struct nacl_irt_private_pnacl_translator_compile IRTFuncs;
-  getIRTInterfaces(IRTFuncs);
-  IRTFuncs.serve_translate_request(&gLLCCallbacks);
-  return 0;
-}
-
-#endif // PNACL_BROWSER_TRANSLATOR
diff --git a/tools/pnacl-thaw/CMakeLists.txt b/tools/pnacl-thaw/CMakeLists.txt
deleted file mode 100644
index 968e356f1b0..00000000000
--- a/tools/pnacl-thaw/CMakeLists.txt
+++ /dev/null
@@ -1,10 +0,0 @@
-set(LLVM_LINK_COMPONENTS
-  BitWriter
-  Core
-  NaClBitReader
-  NaClBitTestUtils
-  Support)
-
-add_llvm_tool(pnacl-thaw
-  pnacl-thaw.cpp
-  )
diff --git a/tools/pnacl-thaw/LLVMBuild.txt b/tools/pnacl-thaw/LLVMBuild.txt
deleted file mode 100644
index 8a9ea57283e..00000000000
--- a/tools/pnacl-thaw/LLVMBuild.txt
+++ /dev/null
@@ -1,16 +0,0 @@
-;===- ./tools/pnacl-thaw/LLVMBuild.txt -----------------------*- Conf -*--===;
-;===------------------------------------------------------------------------===;
-;
-; This is an LLVMBuild description file for the components in this subdirectory.
-;
-; For more information on the LLVMBuild system, please see:
-;
-;   http://llvm.org/docs/LLVMBuild.html
-;
-;===------------------------------------------------------------------------===;
-
-[component_0]
-type = Tool
-name = pnacl-thaw
-parent = Tools
-required_libraries = BitWriter NaClBitReader NaClBitTestUtils
diff --git a/tools/pnacl-thaw/Makefile b/tools/pnacl-thaw/Makefile
deleted file mode 100644
index 4c8de0ce4dc..00000000000
--- a/tools/pnacl-thaw/Makefile
+++ /dev/null
@@ -1,17 +0,0 @@
-##===- tools/pnacl-thaw/Makefile ---------------------------*- Makefile -*-===##
-# 
-#                     The LLVM Compiler Infrastructure
-#
-# This file is distributed under the University of Illinois Open Source
-# License. See LICENSE.TXT for details.
-# 
-##===----------------------------------------------------------------------===##
-
-LEVEL := ../..
-TOOLNAME := pnacl-thaw
-LINK_COMPONENTS := bitwriter naclbitreader naclbittestutils
-
-# This tool has no plugins, optimize startup time.
-TOOL_NO_EXPORTS := 1
-
-include $(LEVEL)/Makefile.common
diff --git a/tools/pnacl-thaw/pnacl-thaw.cpp b/tools/pnacl-thaw/pnacl-thaw.cpp
deleted file mode 100644
index 68e08a1ea50..00000000000
--- a/tools/pnacl-thaw/pnacl-thaw.cpp
+++ /dev/null
@@ -1,117 +0,0 @@
-/* Copyright 2013 The Native Client Authors. All rights reserved.
- * Use of this source code is governed by a BSD-style license that can
- * be found in the LICENSE file.
- */
-
-//===-- pnacl-thaw.cpp - The low-level NaCl bitcode thawer ----------------===//
-//
-//===----------------------------------------------------------------------===//
-//
-// Converts NaCl wire format back to LLVM bitcode.
-//
-//===----------------------------------------------------------------------===//
-
-#include "llvm/Bitcode/NaCl/NaClReaderWriter.h"
-#include "llvm/Bitcode/ReaderWriter.h"
-#include "llvm/IR/LLVMContext.h"
-#include "llvm/IR/Module.h"
-#include "llvm/Support/CommandLine.h"
-#include "llvm/Support/DataStream.h"
-#include "llvm/Support/FileSystem.h"
-#include "llvm/Support/ManagedStatic.h"
-#include "llvm/Support/PrettyStackTrace.h"
-#include "llvm/Support/Signals.h"
-#include "llvm/Support/StreamingMemoryObject.h"
-#include "llvm/Support/ToolOutputFile.h"
-
-using namespace llvm;
-
-namespace {
-
-static cl::opt<std::string>
-OutputFilename("o", cl::desc("Specify thawed pexe filename"),
-	       cl::value_desc("filename"), cl::init("-"));
-
-static cl::opt<std::string>
-InputFilename(cl::Positional, cl::desc("<frozen file>"), cl::init("-"));
-
-static cl::opt<bool>
-VerboseErrors(
-    "verbose-parse-errors",
-    cl::desc("Print out more descriptive PNaCl bitcode parse errors"),
-    cl::init(false));
-
-static void WriteOutputFile(const Module *M) {
-
-  std::error_code EC;
-  std::unique_ptr<tool_output_file> Out(
-      new tool_output_file(OutputFilename, EC, sys::fs::F_None));
-  if (EC) {
-    errs() << EC.message() << '\n';
-    exit(1);
-  }
-
-  WriteBitcodeToFile(M, Out->os());
-
-  // Declare success.
-  Out->keep();
-}
-
-static Module *readBitcode(
-    std::string &Filename, LLVMContext &Context, raw_ostream *Verbose,
-    std::string &ErrorMessage) {
-  // Use the bitcode streaming interface
-  DataStreamer *Streamer = getDataFileStreamer(InputFilename, &ErrorMessage);
-  if (Streamer == nullptr)
-    return nullptr;
-  std::unique_ptr<StreamingMemoryObject> Buffer(
-      new StreamingMemoryObjectImpl(Streamer));
-  std::string DisplayFilename;
-  if (Filename == "-")
-    DisplayFilename = "<stdin>";
-  else
-    DisplayFilename = Filename;
-  Module *M = getNaClStreamedBitcodeModule(DisplayFilename, Buffer.release(),
-                                           Context, Verbose,
-                                           &ErrorMessage,
-                                           /*AcceptSupportedOnly=*/false);
-  if (!M)
-    return nullptr;
-  if (std::error_code EC = M->materializeAllPermanently()) {
-    ErrorMessage = EC.message();
-    delete M;
-    return nullptr;
-  }
-  return M;
-}
-
-} // end of anonymous namespace
-
-int main(int argc, char **argv) {
-  // Print a stack trace if we signal out.
-  sys::PrintStackTraceOnErrorSignal();
-  PrettyStackTraceProgram X(argc, argv);
-
-  LLVMContext &Context = getGlobalContext();
-  llvm_shutdown_obj Y;  // Call llvm_shutdown() on exit.
-
-  cl::ParseCommandLineOptions(
-      argc, argv, "Converts NaCl pexe wire format into LLVM bitcode format\n");
-
-  std::string ErrorMessage;
-  raw_ostream *Verbose = VerboseErrors ? &errs() : nullptr;
-  std::unique_ptr<Module> M(readBitcode(InputFilename, Context,
-                                        Verbose, ErrorMessage));
-
-  if (!M.get()) {
-    errs() << argv[0] << ": ";
-    if (ErrorMessage.size())
-      errs() << ErrorMessage << "\n";
-    else
-      errs() << "bitcode didn't read correctly.\n";
-    return 1;
-  }
-
-  WriteOutputFile(M.get());
-  return 0;
-}
diff --git a/tools/verify-uselistorder/verify-uselistorder.cpp b/tools/verify-uselistorder/verify-uselistorder.cpp
index 795d035d3df..efa4bcbe1aa 100644
--- a/tools/verify-uselistorder/verify-uselistorder.cpp
+++ b/tools/verify-uselistorder/verify-uselistorder.cpp
@@ -159,14 +159,14 @@ std::unique_ptr<Module> TempFile::readBitcode(LLVMContext &Context) const {
   }
 
   MemoryBuffer *Buffer = BufferOr.get().get();
-  ErrorOr<Module *> ModuleOr =
+  ErrorOr<std::unique_ptr<Module>> ModuleOr =
       parseBitcodeFile(Buffer->getMemBufferRef(), Context);
   if (!ModuleOr) {
     errs() << "verify-uselistorder: error: " << ModuleOr.getError().message()
            << "\n";
     return nullptr;
   }
-  return std::unique_ptr<Module>(ModuleOr.get());
+  return std::move(ModuleOr.get());
 }
 
 std::unique_ptr<Module> TempFile::readAssembly(LLVMContext &Context) const {
@@ -205,6 +205,8 @@ ValueMapping::ValueMapping(const Module &M) {
       map(F.getPrefixData());
     if (F.hasPrologueData())
       map(F.getPrologueData());
+    if (F.hasPersonalityFn())
+      map(F.getPersonalityFn());
   }
 
   // Function bodies.
@@ -474,6 +476,8 @@ static void changeUseLists(Module &M, Changer changeValueUseList) {
       changeValueUseList(F.getPrefixData());
     if (F.hasPrologueData())
       changeValueUseList(F.getPrologueData());
+    if (F.hasPersonalityFn())
+      changeValueUseList(F.getPersonalityFn());
   }
 
   // Function bodies.
diff --git a/tools/yaml2obj/yaml2elf.cpp b/tools/yaml2obj/yaml2elf.cpp
index 5d4c26379be..a247f48c053 100644
--- a/tools/yaml2obj/yaml2elf.cpp
+++ b/tools/yaml2obj/yaml2elf.cpp
@@ -35,6 +35,8 @@ class ContiguousBlobAccumulator {
 
   /// \returns The new offset.
   uint64_t padToAlignment(unsigned Align) {
+    if (Align == 0)
+      Align = 1;
     uint64_t CurrentOffset = InitialOffset + OS.tell();
     uint64_t AlignedOffset = RoundUpToAlignment(CurrentOffset, Align);
     for (; CurrentOffset != AlignedOffset; ++CurrentOffset)
@@ -46,7 +48,7 @@ class ContiguousBlobAccumulator {
   ContiguousBlobAccumulator(uint64_t InitialOffset_)
       : InitialOffset(InitialOffset_), Buf(), OS(Buf) {}
   template <class Integer>
-  raw_ostream &getOSAndAlignedOffset(Integer &Offset, unsigned Align = 16) {
+  raw_ostream &getOSAndAlignedOffset(Integer &Offset, unsigned Align) {
     Offset = padToAlignment(Align);
     return OS;
   }
@@ -133,6 +135,9 @@ class ELFState {
                            ContiguousBlobAccumulator &CBA);
   bool writeSectionContent(Elf_Shdr &SHeader, const ELFYAML::Group &Group,
                            ContiguousBlobAccumulator &CBA);
+  bool writeSectionContent(Elf_Shdr &SHeader,
+                           const ELFYAML::MipsABIFlags &Section,
+                           ContiguousBlobAccumulator &CBA);
 
   // - SHT_NULL entry (placed first, i.e. 0'th entry)
   // - symbol table (.symtab) (placed third to last)
@@ -235,6 +240,15 @@ bool ELFState<ELFT>::initSectionHeaders(std::vector<Elf_Shdr> &SHeaders,
       SHeader.sh_info = SymIdx;
       if (!writeSectionContent(SHeader, *S, CBA))
         return false;
+    } else if (auto S = dyn_cast<ELFYAML::MipsABIFlags>(Sec.get())) {
+      if (!writeSectionContent(SHeader, *S, CBA))
+        return false;
+    } else if (auto S = dyn_cast<ELFYAML::NoBitsSection>(Sec.get())) {
+      SHeader.sh_entsize = 0;
+      SHeader.sh_size = S->Size;
+      // SHT_NOBITS section does not have content
+      // so just to setup the section offset.
+      CBA.getOSAndAlignedOffset(SHeader.sh_offset, SHeader.sh_addralign);
     } else
       llvm_unreachable("Unknown section type");
 
@@ -253,6 +267,7 @@ void ELFState<ELFT>::initSymtabSectionHeader(Elf_Shdr &SHeader,
   // One greater than symbol table index of the last local symbol.
   SHeader.sh_info = Doc.Symbols.Local.size() + 1;
   SHeader.sh_entsize = sizeof(Elf_Sym);
+  SHeader.sh_addralign = 8;
 
   std::vector<Elf_Sym> Syms;
   {
@@ -275,8 +290,9 @@ void ELFState<ELFT>::initSymtabSectionHeader(Elf_Shdr &SHeader,
   addSymbols(Doc.Symbols.Global, Syms, ELF::STB_GLOBAL);
   addSymbols(Doc.Symbols.Weak, Syms, ELF::STB_WEAK);
 
-  writeArrayData(CBA.getOSAndAlignedOffset(SHeader.sh_offset),
-                 makeArrayRef(Syms));
+  writeArrayData(
+      CBA.getOSAndAlignedOffset(SHeader.sh_offset, SHeader.sh_addralign),
+      makeArrayRef(Syms));
   SHeader.sh_size = arrayDataSize(makeArrayRef(Syms));
 }
 
@@ -287,7 +303,8 @@ void ELFState<ELFT>::initStrtabSectionHeader(Elf_Shdr &SHeader, StringRef Name,
   zero(SHeader);
   SHeader.sh_name = DotShStrtab.getOffset(Name);
   SHeader.sh_type = ELF::SHT_STRTAB;
-  CBA.getOSAndAlignedOffset(SHeader.sh_offset) << STB.data();
+  CBA.getOSAndAlignedOffset(SHeader.sh_offset, SHeader.sh_addralign)
+      << STB.data();
   SHeader.sh_size = STB.data().size();
   SHeader.sh_addralign = 1;
 }
@@ -325,7 +342,8 @@ ELFState<ELFT>::writeSectionContent(Elf_Shdr &SHeader,
                                     ContiguousBlobAccumulator &CBA) {
   assert(Section.Size >= Section.Content.binary_size() &&
          "Section size and section content are inconsistent");
-  raw_ostream &OS = CBA.getOSAndAlignedOffset(SHeader.sh_offset);
+  raw_ostream &OS =
+      CBA.getOSAndAlignedOffset(SHeader.sh_offset, SHeader.sh_addralign);
   Section.Content.writeAsBinary(OS);
   for (auto i = Section.Content.binary_size(); i < Section.Size; ++i)
     OS.write(0);
@@ -344,25 +362,22 @@ bool
 ELFState<ELFT>::writeSectionContent(Elf_Shdr &SHeader,
                                     const ELFYAML::RelocationSection &Section,
                                     ContiguousBlobAccumulator &CBA) {
-  if (Section.Type != llvm::ELF::SHT_REL &&
-      Section.Type != llvm::ELF::SHT_RELA) {
-    errs() << "error: Invalid relocation section type.\n";
-    return false;
-  }
+  assert((Section.Type == llvm::ELF::SHT_REL ||
+          Section.Type == llvm::ELF::SHT_RELA) &&
+         "Section type is not SHT_REL nor SHT_RELA");
 
   bool IsRela = Section.Type == llvm::ELF::SHT_RELA;
   SHeader.sh_entsize = IsRela ? sizeof(Elf_Rela) : sizeof(Elf_Rel);
   SHeader.sh_size = SHeader.sh_entsize * Section.Relocations.size();
 
-  auto &OS = CBA.getOSAndAlignedOffset(SHeader.sh_offset);
+  auto &OS = CBA.getOSAndAlignedOffset(SHeader.sh_offset, SHeader.sh_addralign);
 
   for (const auto &Rel : Section.Relocations) {
-    unsigned SymIdx;
-    if (SymN2I.lookup(Rel.Symbol, SymIdx)) {
-      errs() << "error: Unknown symbol referenced: '" << Rel.Symbol
-             << "' at YAML relocation.\n";
-      return false;
-    }
+    unsigned SymIdx = 0;
+    // Some special relocation, R_ARM_v4BX for instance, does not have
+    // an external reference.  So it ignores the return value of lookup()
+    // here.
+    SymN2I.lookup(Rel.Symbol, SymIdx);
 
     if (IsRela) {
       Elf_Rela REntry;
@@ -387,15 +402,13 @@ bool ELFState<ELFT>::writeSectionContent(Elf_Shdr &SHeader,
                                          const ELFYAML::Group &Section,
                                          ContiguousBlobAccumulator &CBA) {
   typedef typename object::ELFFile<ELFT>::Elf_Word Elf_Word;
-  if (Section.Type != llvm::ELF::SHT_GROUP) {
-    errs() << "error: Invalid section type.\n";
-    return false;
-  }
+  assert(Section.Type == llvm::ELF::SHT_GROUP &&
+         "Section type is not SHT_GROUP");
 
   SHeader.sh_entsize = sizeof(Elf_Word);
   SHeader.sh_size = SHeader.sh_entsize * Section.Members.size();
 
-  auto &OS = CBA.getOSAndAlignedOffset(SHeader.sh_offset);
+  auto &OS = CBA.getOSAndAlignedOffset(SHeader.sh_offset, SHeader.sh_addralign);
 
   for (auto member : Section.Members) {
     Elf_Word SIdx;
@@ -414,6 +427,35 @@ bool ELFState<ELFT>::writeSectionContent(Elf_Shdr &SHeader,
   return true;
 }
 
+template <class ELFT>
+bool ELFState<ELFT>::writeSectionContent(Elf_Shdr &SHeader,
+                                         const ELFYAML::MipsABIFlags &Section,
+                                         ContiguousBlobAccumulator &CBA) {
+  assert(Section.Type == llvm::ELF::SHT_MIPS_ABIFLAGS &&
+         "Section type is not SHT_MIPS_ABIFLAGS");
+
+  object::Elf_Mips_ABIFlags<ELFT> Flags;
+  zero(Flags);
+  SHeader.sh_entsize = sizeof(Flags);
+  SHeader.sh_size = SHeader.sh_entsize;
+
+  auto &OS = CBA.getOSAndAlignedOffset(SHeader.sh_offset, SHeader.sh_addralign);
+  Flags.version = Section.Version;
+  Flags.isa_level = Section.ISALevel;
+  Flags.isa_rev = Section.ISARevision;
+  Flags.gpr_size = Section.GPRSize;
+  Flags.cpr1_size = Section.CPR1Size;
+  Flags.cpr2_size = Section.CPR2Size;
+  Flags.fp_abi = Section.FpABI;
+  Flags.isa_ext = Section.ISAExtension;
+  Flags.ases = Section.ASEs;
+  Flags.flags1 = Section.Flags1;
+  Flags.flags2 = Section.Flags2;
+  OS.write((const char *)&Flags, sizeof(Flags));
+
+  return true;
+}
+
 template <class ELFT> bool ELFState<ELFT>::buildSectionIndex() {
   SN2I.addName(".symtab", getDotSymTabSecNo());
   SN2I.addName(".strtab", getDotStrTabSecNo());
@@ -522,10 +564,10 @@ int yaml2elf(yaml::Input &YIn, raw_ostream &Out) {
     return 1;
   }
   using object::ELFType;
-  typedef ELFType<support::little, 8, true> LE64;
-  typedef ELFType<support::big, 8, true> BE64;
-  typedef ELFType<support::little, 4, false> LE32;
-  typedef ELFType<support::big, 4, false> BE32;
+  typedef ELFType<support::little, true> LE64;
+  typedef ELFType<support::big, true> BE64;
+  typedef ELFType<support::little, false> LE32;
+  typedef ELFType<support::big, false> BE32;
   if (is64Bit(Doc)) {
     if (isLittleEndian(Doc))
       return ELFState<LE64>::writeELF(Out, Doc);
diff --git a/unittests/ADT/APIntTest.cpp b/unittests/ADT/APIntTest.cpp
index acdc1ecbbde..ffba7b16336 100644
--- a/unittests/ADT/APIntTest.cpp
+++ b/unittests/ADT/APIntTest.cpp
@@ -10,6 +10,7 @@
 #include "llvm/ADT/APInt.h"
 #include "llvm/ADT/SmallString.h"
 #include "gtest/gtest.h"
+#include <array>
 #include <ostream>
 
 using namespace llvm;
@@ -134,6 +135,12 @@ TEST(APIntTest, i1) {
   EXPECT_EQ(one, neg_one);
   EXPECT_EQ(two, neg_two);
 
+  // Min/max signed values.
+  EXPECT_TRUE(zero.isMaxSignedValue());
+  EXPECT_FALSE(one.isMaxSignedValue());
+  EXPECT_FALSE(zero.isMinSignedValue());
+  EXPECT_TRUE(one.isMinSignedValue());
+
   // Additions.
   EXPECT_EQ(two, one + one);
   EXPECT_EQ(zero, neg_one + one);
@@ -209,204 +216,266 @@ TEST(APIntTest, i1) {
   }
 }
 
-TEST(APIntTest, divrem_big1) {
-  // Tests KnuthDiv rare step D6
-  APInt a{256, "1ffffffffffffffff", 16};
-  APInt b{256, "1ffffffffffffffff", 16};
-  APInt c{256, 0};
+TEST(APIntTest, compare) {
+  std::array<APInt, 5> testVals{{
+    APInt{16, 2},
+    APInt{16, 1},
+    APInt{16, 0},
+    APInt{16, (uint64_t)-1, true},
+    APInt{16, (uint64_t)-2, true},
+  }};
+
+  for (auto &arg1 : testVals)
+    for (auto &arg2 : testVals) {
+      auto uv1 = arg1.getZExtValue();
+      auto uv2 = arg2.getZExtValue();
+      auto sv1 = arg1.getSExtValue();
+      auto sv2 = arg2.getSExtValue();
+
+      EXPECT_EQ(uv1 <  uv2, arg1.ult(arg2));
+      EXPECT_EQ(uv1 <= uv2, arg1.ule(arg2));
+      EXPECT_EQ(uv1 >  uv2, arg1.ugt(arg2));
+      EXPECT_EQ(uv1 >= uv2, arg1.uge(arg2));
+
+      EXPECT_EQ(sv1 <  sv2, arg1.slt(arg2));
+      EXPECT_EQ(sv1 <= sv2, arg1.sle(arg2));
+      EXPECT_EQ(sv1 >  sv2, arg1.sgt(arg2));
+      EXPECT_EQ(sv1 >= sv2, arg1.sge(arg2));
+
+      EXPECT_EQ(uv1 <  uv2, arg1.ult(uv2));
+      EXPECT_EQ(uv1 <= uv2, arg1.ule(uv2));
+      EXPECT_EQ(uv1 >  uv2, arg1.ugt(uv2));
+      EXPECT_EQ(uv1 >= uv2, arg1.uge(uv2));
+
+      EXPECT_EQ(sv1 <  sv2, arg1.slt(sv2));
+      EXPECT_EQ(sv1 <= sv2, arg1.sle(sv2));
+      EXPECT_EQ(sv1 >  sv2, arg1.sgt(sv2));
+      EXPECT_EQ(sv1 >= sv2, arg1.sge(sv2));
+    }
+}
+
+TEST(APIntTest, compareWithRawIntegers) {
+  EXPECT_TRUE(!APInt(8, 1).uge(256));
+  EXPECT_TRUE(!APInt(8, 1).ugt(256));
+  EXPECT_TRUE( APInt(8, 1).ule(256));
+  EXPECT_TRUE( APInt(8, 1).ult(256));
+  EXPECT_TRUE(!APInt(8, 1).sge(256));
+  EXPECT_TRUE(!APInt(8, 1).sgt(256));
+  EXPECT_TRUE( APInt(8, 1).sle(256));
+  EXPECT_TRUE( APInt(8, 1).slt(256));
+  EXPECT_TRUE(!(APInt(8, 0) == 256));
+  EXPECT_TRUE(  APInt(8, 0) != 256);
+  EXPECT_TRUE(!(APInt(8, 1) == 256));
+  EXPECT_TRUE(  APInt(8, 1) != 256);
+
+  auto uint64max = UINT64_MAX;
+  auto int64max  = INT64_MAX;
+  auto int64min  = INT64_MIN;
+
+  auto u64 = APInt{128, uint64max};
+  auto s64 = APInt{128, static_cast<uint64_t>(int64max), true};
+  auto big = u64 + 1;
+
+  EXPECT_TRUE( u64.uge(uint64max));
+  EXPECT_TRUE(!u64.ugt(uint64max));
+  EXPECT_TRUE( u64.ule(uint64max));
+  EXPECT_TRUE(!u64.ult(uint64max));
+  EXPECT_TRUE( u64.sge(int64max));
+  EXPECT_TRUE( u64.sgt(int64max));
+  EXPECT_TRUE(!u64.sle(int64max));
+  EXPECT_TRUE(!u64.slt(int64max));
+  EXPECT_TRUE( u64.sge(int64min));
+  EXPECT_TRUE( u64.sgt(int64min));
+  EXPECT_TRUE(!u64.sle(int64min));
+  EXPECT_TRUE(!u64.slt(int64min));
+
+  EXPECT_TRUE(u64 == uint64max);
+  EXPECT_TRUE(u64 != int64max);
+  EXPECT_TRUE(u64 != int64min);
+
+  EXPECT_TRUE(!s64.uge(uint64max));
+  EXPECT_TRUE(!s64.ugt(uint64max));
+  EXPECT_TRUE( s64.ule(uint64max));
+  EXPECT_TRUE( s64.ult(uint64max));
+  EXPECT_TRUE( s64.sge(int64max));
+  EXPECT_TRUE(!s64.sgt(int64max));
+  EXPECT_TRUE( s64.sle(int64max));
+  EXPECT_TRUE(!s64.slt(int64max));
+  EXPECT_TRUE( s64.sge(int64min));
+  EXPECT_TRUE( s64.sgt(int64min));
+  EXPECT_TRUE(!s64.sle(int64min));
+  EXPECT_TRUE(!s64.slt(int64min));
+
+  EXPECT_TRUE(s64 != uint64max);
+  EXPECT_TRUE(s64 == int64max);
+  EXPECT_TRUE(s64 != int64min);
+
+  EXPECT_TRUE( big.uge(uint64max));
+  EXPECT_TRUE( big.ugt(uint64max));
+  EXPECT_TRUE(!big.ule(uint64max));
+  EXPECT_TRUE(!big.ult(uint64max));
+  EXPECT_TRUE( big.sge(int64max));
+  EXPECT_TRUE( big.sgt(int64max));
+  EXPECT_TRUE(!big.sle(int64max));
+  EXPECT_TRUE(!big.slt(int64max));
+  EXPECT_TRUE( big.sge(int64min));
+  EXPECT_TRUE( big.sgt(int64min));
+  EXPECT_TRUE(!big.sle(int64min));
+  EXPECT_TRUE(!big.slt(int64min));
+
+  EXPECT_TRUE(big != uint64max);
+  EXPECT_TRUE(big != int64max);
+  EXPECT_TRUE(big != int64min);
+}
+
+TEST(APIntTest, compareWithInt64Min) {
+  int64_t edge = INT64_MIN;
+  int64_t edgeP1 = edge + 1;
+  int64_t edgeM1 = INT64_MAX;
+  auto a = APInt{64, static_cast<uint64_t>(edge), true};
+
+  EXPECT_TRUE(!a.slt(edge));
+  EXPECT_TRUE( a.sle(edge));
+  EXPECT_TRUE(!a.sgt(edge));
+  EXPECT_TRUE( a.sge(edge));
+  EXPECT_TRUE( a.slt(edgeP1));
+  EXPECT_TRUE( a.sle(edgeP1));
+  EXPECT_TRUE(!a.sgt(edgeP1));
+  EXPECT_TRUE(!a.sge(edgeP1));
+  EXPECT_TRUE( a.slt(edgeM1));
+  EXPECT_TRUE( a.sle(edgeM1));
+  EXPECT_TRUE(!a.sgt(edgeM1));
+  EXPECT_TRUE(!a.sge(edgeM1));
+}
+
+TEST(APIntTest, compareWithHalfInt64Max) {
+  uint64_t edge = 0x4000000000000000;
+  uint64_t edgeP1 = edge + 1;
+  uint64_t edgeM1 = edge - 1;
+  auto a = APInt{64, edge};
+
+  EXPECT_TRUE(!a.ult(edge));
+  EXPECT_TRUE( a.ule(edge));
+  EXPECT_TRUE(!a.ugt(edge));
+  EXPECT_TRUE( a.uge(edge));
+  EXPECT_TRUE( a.ult(edgeP1));
+  EXPECT_TRUE( a.ule(edgeP1));
+  EXPECT_TRUE(!a.ugt(edgeP1));
+  EXPECT_TRUE(!a.uge(edgeP1));
+  EXPECT_TRUE(!a.ult(edgeM1));
+  EXPECT_TRUE(!a.ule(edgeM1));
+  EXPECT_TRUE( a.ugt(edgeM1));
+  EXPECT_TRUE( a.uge(edgeM1));
+
+  EXPECT_TRUE(!a.slt(edge));
+  EXPECT_TRUE( a.sle(edge));
+  EXPECT_TRUE(!a.sgt(edge));
+  EXPECT_TRUE( a.sge(edge));
+  EXPECT_TRUE( a.slt(edgeP1));
+  EXPECT_TRUE( a.sle(edgeP1));
+  EXPECT_TRUE(!a.sgt(edgeP1));
+  EXPECT_TRUE(!a.sge(edgeP1));
+  EXPECT_TRUE(!a.slt(edgeM1));
+  EXPECT_TRUE(!a.sle(edgeM1));
+  EXPECT_TRUE( a.sgt(edgeM1));
+  EXPECT_TRUE( a.sge(edgeM1));
+}
+
+
+// Tests different div/rem varaints using scheme (a * b + c) / a
+void testDiv(APInt a, APInt b, APInt c) {
+  ASSERT_TRUE(a.uge(b)); // Must: a >= b
+  ASSERT_TRUE(a.ugt(c)); // Must: a > c
 
   auto p = a * b + c;
+
   auto q = p.udiv(a);
   auto r = p.urem(a);
-  EXPECT_EQ(q, b);
-  EXPECT_EQ(r, c);
+  EXPECT_EQ(b, q);
+  EXPECT_EQ(c, r);
   APInt::udivrem(p, a, q, r);
-  EXPECT_EQ(q, b);
-  EXPECT_EQ(r, c);
-  q = p.udiv(b);
-  r = p.urem(b);
-  EXPECT_EQ(q, a);
-  EXPECT_EQ(r, c);
-  APInt::udivrem(p, b, q, r);
-  EXPECT_EQ(q, a);
-  EXPECT_EQ(r, c);
+  EXPECT_EQ(b, q);
+  EXPECT_EQ(c, r);
   q = p.sdiv(a);
   r = p.srem(a);
-  EXPECT_EQ(q, b);
-  EXPECT_EQ(r, c);
+  EXPECT_EQ(b, q);
+  EXPECT_EQ(c, r);
   APInt::sdivrem(p, a, q, r);
-  EXPECT_EQ(q, b);
-  EXPECT_EQ(r, c);
-  q = p.sdiv(b);
-  r = p.srem(b);
-  EXPECT_EQ(q, a);
-  EXPECT_EQ(r, c);
-  APInt::sdivrem(p, b, q, r);
-  EXPECT_EQ(q, a);
-  EXPECT_EQ(r, c);
+  EXPECT_EQ(b, q);
+  EXPECT_EQ(c, r);
+
+  if (b.ugt(c)) { // Test also symmetric case
+    q = p.udiv(b);
+    r = p.urem(b);
+    EXPECT_EQ(a, q);
+    EXPECT_EQ(c, r);
+    APInt::udivrem(p, b, q, r);
+    EXPECT_EQ(a, q);
+    EXPECT_EQ(c, r);
+    q = p.sdiv(b);
+    r = p.srem(b);
+    EXPECT_EQ(a, q);
+    EXPECT_EQ(c, r);
+    APInt::sdivrem(p, b, q, r);
+    EXPECT_EQ(a, q);
+    EXPECT_EQ(c, r);
+  }
 }
 
-TEST(APIntTest, divrem_big2) {
+TEST(APIntTest, divrem_big1) {
   // Tests KnuthDiv rare step D6
-  APInt a{1024,           "111111ffffffffffffffff"
-                "ffffffffffffffffffffffffffffffff"
-                "fffffffffffffffffffffffffffffccf"
-                "ffffffffffffffffffffffffffffff00", 16};
-  APInt b{1024,                       "112233ceff"
-                "cecece000000ffffffffffffffffffff"
-                "ffffffffffffffffffffffffffffffff"
-                "ffffffffffffffffffffffffffffffff"
-                "ffffffffffffffffffffffffffffff33", 16};
-  APInt c{1024, 7919};
+  testDiv({256, "1ffffffffffffffff", 16},
+          {256, "1ffffffffffffffff", 16},
+          {256, 0});
+}
 
-  auto p = a * b + c;
-  auto q = p.udiv(a);
-  auto r = p.urem(a);
-  EXPECT_EQ(q, b);
-  EXPECT_EQ(r, c);
-  APInt::udivrem(p, a, q, r);
-  EXPECT_EQ(q, b);
-  EXPECT_EQ(r, c);
-  q = p.udiv(b);
-  r = p.urem(b);
-  EXPECT_EQ(q, a);
-  EXPECT_EQ(r, c);
-  APInt::udivrem(p, b, q, r);
-  EXPECT_EQ(q, a);
-  EXPECT_EQ(r, c);
-  q = p.sdiv(a);
-  r = p.srem(a);
-  EXPECT_EQ(q, b);
-  EXPECT_EQ(r, c);
-  APInt::sdivrem(p, a, q, r);
-  EXPECT_EQ(q, b);
-  EXPECT_EQ(r, c);
-  q = p.sdiv(b);
-  r = p.srem(b);
-  EXPECT_EQ(q, a);
-  EXPECT_EQ(r, c);
-  APInt::sdivrem(p, b, q, r);
-  EXPECT_EQ(q, a);
-  EXPECT_EQ(r, c);
+TEST(APIntTest, divrem_big2) {
+  // Tests KnuthDiv rare step D6
+  testDiv({1024,                       "112233ceff"
+                 "cecece000000ffffffffffffffffffff"
+                 "ffffffffffffffffffffffffffffffff"
+                 "ffffffffffffffffffffffffffffffff"
+                 "ffffffffffffffffffffffffffffff33", 16},
+          {1024,           "111111ffffffffffffffff"
+                 "ffffffffffffffffffffffffffffffff"
+                 "fffffffffffffffffffffffffffffccf"
+                 "ffffffffffffffffffffffffffffff00", 16},
+          {1024, 7919});
 }
 
 TEST(APIntTest, divrem_big3) {
   // Tests KnuthDiv case without shift
-  APInt a{256, "ffffffffffffff0000000", 16};
-  APInt b{256, "80000001ffffffffffffffff", 16};
-  APInt c{256, 4219};
-
-  auto p = a * b + c;
-  auto q = p.udiv(a);
-  auto r = p.urem(a);
-  EXPECT_EQ(q, b);
-  EXPECT_EQ(r, c);
-  APInt::udivrem(p, a, q, r);
-  EXPECT_EQ(q, b);
-  EXPECT_EQ(r, c);
-  q = p.udiv(b);
-  r = p.urem(b);
-  EXPECT_EQ(q, a);
-  EXPECT_EQ(r, c);
-  APInt::udivrem(p, b, q, r);
-  EXPECT_EQ(q, a);
-  EXPECT_EQ(r, c);
-  q = p.sdiv(a);
-  r = p.srem(a);
-  EXPECT_EQ(q, b);
-  EXPECT_EQ(r, c);
-  APInt::sdivrem(p, a, q, r);
-  EXPECT_EQ(q, b);
-  EXPECT_EQ(r, c);
-  q = p.sdiv(b);
-  r = p.srem(b);
-  EXPECT_EQ(q, a);
-  EXPECT_EQ(r, c);
-  APInt::sdivrem(p, b, q, r);
-  EXPECT_EQ(q, a);
-  EXPECT_EQ(r, c);
+  testDiv({256, "80000001ffffffffffffffff", 16},
+          {256, "ffffffffffffff0000000", 16},
+          {256, 4219});
 }
 
 TEST(APIntTest, divrem_big4) {
   // Tests heap allocation in divide() enfoced by huge numbers
-  auto a = APInt{4096, 1}.shl(2000);
-  auto b = APInt{4096, 5}.shl(2001);
-  auto c = APInt{4096, 4219*13};
-
-  auto p = a * b + c;
-  auto q = p.udiv(a);
-  auto r = p.urem(a);
-  EXPECT_EQ(q, b);
-  EXPECT_EQ(r, c);
-  q = APInt{1024, 0}; // test non-single word APInt conversion in divide()
-  r = APInt{1024, 0};
-  APInt::udivrem(p, a, q, r);
-  EXPECT_EQ(q, b);
-  EXPECT_EQ(r, c);
-  q = p.udiv(b);
-  r = p.urem(b);
-  EXPECT_EQ(q, a);
-  EXPECT_EQ(r, c);
-  q = APInt{1024, 0};
-  r = APInt{1024, 0};
-  APInt::udivrem(p, b, q, r);
-  EXPECT_EQ(q, a);
-  EXPECT_EQ(r, c);
-  q = p.sdiv(a);
-  r = p.srem(a);
-  EXPECT_EQ(q, b);
-  EXPECT_EQ(r, c);
-  q = APInt{1024, 0};
-  r = APInt{1024, 0};
-  APInt::sdivrem(p, a, q, r);
-  EXPECT_EQ(q, b);
-  EXPECT_EQ(r, c);
-  q = p.sdiv(b);
-  r = p.srem(b);
-  EXPECT_EQ(q, a);
-  EXPECT_EQ(r, c);
-  q = APInt{1024, 0};
-  r = APInt{1024, 0};
-  APInt::sdivrem(p, b, q, r);
-  EXPECT_EQ(q, a);
-  EXPECT_EQ(r, c);
+  testDiv(APInt{4096, 5}.shl(2001),
+          APInt{4096, 1}.shl(2000),
+          APInt{4096, 4219*13});
 }
 
 TEST(APIntTest, divrem_big5) {
   // Tests one word divisor case of divide()
-  auto a = APInt{1024, 19}.shl(811);
-  auto b = APInt{1024, 4356013}; // one word
-  auto c = APInt{1024, 1};
+  testDiv(APInt{1024, 19}.shl(811),
+          APInt{1024, 4356013}, // one word
+          APInt{1024, 1});
+}
 
-  auto p = a * b + c;
-  auto q = p.udiv(a);
-  auto r = p.urem(a);
-  EXPECT_EQ(q, b);
-  EXPECT_EQ(r, c);
-  APInt::udivrem(p, a, q, r);
-  EXPECT_EQ(q, b);
-  EXPECT_EQ(r, c);
-  q = p.udiv(b);
-  r = p.urem(b);
-  EXPECT_EQ(q, a);
-  EXPECT_EQ(r, c);
-  APInt::udivrem(p, b, q, r);
-  EXPECT_EQ(q, a);
-  EXPECT_EQ(r, c);
-  q = p.sdiv(a);
-  r = p.srem(a);
-  EXPECT_EQ(q, b);
-  EXPECT_EQ(r, c);
-  APInt::sdivrem(p, a, q, r);
-  EXPECT_EQ(q, b);
-  EXPECT_EQ(r, c);
-  q = p.sdiv(b);
-  r = p.srem(b);
-  EXPECT_EQ(q, a);
-  EXPECT_EQ(r, c);
-  APInt::sdivrem(p, b, q, r);
-  EXPECT_EQ(q, a);
-  EXPECT_EQ(r, c);
+TEST(APIntTest, divrem_big6) {
+  // Tests some rare "borrow" cases in D4 step
+  testDiv(APInt{512, "ffffffffffffffff00000000000000000000000001", 16},
+          APInt{512, "10000000000000001000000000000001", 16},
+          APInt{512, "10000000000000000000000000000000", 16});
+}
+
+TEST(APIntTest, divrem_big7) {
+  // Yet another test for KnuthDiv rare step D6.
+  testDiv({224, "800000008000000200000005", 16},
+          {224, "fffffffd", 16},
+          {224, "80000000800000010000000f", 16});
 }
 
 TEST(APIntTest, fromString) {
diff --git a/unittests/ADT/APSIntTest.cpp b/unittests/ADT/APSIntTest.cpp
index 5e4e87440bd..a9b3071d138 100644
--- a/unittests/ADT/APSIntTest.cpp
+++ b/unittests/ADT/APSIntTest.cpp
@@ -143,4 +143,21 @@ TEST(APSIntTest, compareValues) {
   EXPECT_TRUE(APSInt::compareValues(U(8), S(-7).trunc(32)) > 0);
 }
 
+TEST(APSIntTest, FromString) {
+  EXPECT_EQ(APSInt("1").getExtValue(), 1);
+  EXPECT_EQ(APSInt("-1").getExtValue(), -1);
+  EXPECT_EQ(APSInt("0").getExtValue(), 0);
+  EXPECT_EQ(APSInt("56789").getExtValue(), 56789);
+  EXPECT_EQ(APSInt("-1234").getExtValue(), -1234);
 }
+
+#if defined(GTEST_HAS_DEATH_TEST) && !defined(NDEBUG)
+
+TEST(APSIntTest, StringDeath) {
+  EXPECT_DEATH(APSInt(""), "Invalid string length");
+  EXPECT_DEATH(APSInt("1a"), "Invalid character in digit string");
+}
+
+#endif
+
+} // end anonymous namespace
diff --git a/unittests/ADT/ArrayRefTest.cpp b/unittests/ADT/ArrayRefTest.cpp
index 6955036f113..9ad32d54d36 100644
--- a/unittests/ADT/ArrayRefTest.cpp
+++ b/unittests/ADT/ArrayRefTest.cpp
@@ -40,7 +40,7 @@ TEST(ArrayRefTest, AllocatorCopy) {
   static const uint16_t Words2[] = { 11, 4003, 67, 64000, 13 };
   ArrayRef<uint16_t> Array2 = makeArrayRef(Words2, 5);
   ArrayRef<uint16_t> Array1c = Array1.copy(Alloc);
-  ArrayRef<uint16_t> Array2c = Array2.copy(Alloc);;
+  ArrayRef<uint16_t> Array2c = Array2.copy(Alloc);
   EXPECT_TRUE(Array1.equals(Array1c));
   EXPECT_NE(Array1.data(), Array1c.data());
   EXPECT_TRUE(Array2.equals(Array2c));
diff --git a/unittests/ADT/CMakeLists.txt b/unittests/ADT/CMakeLists.txt
index d899852325c..cb878c61b85 100644
--- a/unittests/ADT/CMakeLists.txt
+++ b/unittests/ADT/CMakeLists.txt
@@ -28,6 +28,7 @@ set(ADTSources
   PointerIntPairTest.cpp
   PointerUnionTest.cpp
   PostOrderIteratorTest.cpp
+  RangeAdapterTest.cpp
   SCCIteratorTest.cpp
   SmallPtrSetTest.cpp
   SmallStringTest.cpp
@@ -46,3 +47,5 @@ set(ADTSources
 add_llvm_unittest(ADTTests
   ${ADTSources}
   )
+
+add_dependencies(ADTTests intrinsics_gen)
diff --git a/unittests/ADT/DenseMapTest.cpp b/unittests/ADT/DenseMapTest.cpp
index 97807771725..2c6fe358832 100644
--- a/unittests/ADT/DenseMapTest.cpp
+++ b/unittests/ADT/DenseMapTest.cpp
@@ -323,6 +323,31 @@ TYPED_TEST(DenseMapTest, ConstIteratorTest) {
   EXPECT_TRUE(cit == cit2);
 }
 
+// Make sure DenseMap works with StringRef keys.
+TEST(DenseMapCustomTest, StringRefTest) {
+  DenseMap<StringRef, int> M;
+
+  M["a"] = 1;
+  M["b"] = 2;
+  M["c"] = 3;
+
+  EXPECT_EQ(3u, M.size());
+  EXPECT_EQ(1, M.lookup("a"));
+  EXPECT_EQ(2, M.lookup("b"));
+  EXPECT_EQ(3, M.lookup("c"));
+
+  EXPECT_EQ(0, M.lookup("q"));
+
+  // Test the empty string, spelled various ways.
+  EXPECT_EQ(0, M.lookup(""));
+  EXPECT_EQ(0, M.lookup(StringRef()));
+  EXPECT_EQ(0, M.lookup(StringRef("a", 0)));
+  M[""] = 42;
+  EXPECT_EQ(42, M.lookup(""));
+  EXPECT_EQ(42, M.lookup(StringRef()));
+  EXPECT_EQ(42, M.lookup(StringRef("a", 0)));
+}
+
 // Key traits that allows lookup with either an unsigned or char* key;
 // In the latter case, "a" == 0, "b" == 1 and so on.
 struct TestDenseMapInfo {
diff --git a/unittests/ADT/RangeAdapterTest.cpp b/unittests/ADT/RangeAdapterTest.cpp
new file mode 100644
index 00000000000..634f5bb990d
--- /dev/null
+++ b/unittests/ADT/RangeAdapterTest.cpp
@@ -0,0 +1,83 @@
+//===- RangeAdapterTest.cpp - Unit tests for range adapters  --------------===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/ADT/iterator_range.h"
+#include "llvm/ADT/STLExtras.h"
+#include "gtest/gtest.h"
+
+#include <iterator>
+#include <list>
+#include <vector>
+
+using namespace llvm;
+
+namespace {
+
+// A wrapper around vector which exposes rbegin(), rend().
+class ReverseOnlyVector {
+  std::vector<int> Vec;
+
+public:
+  ReverseOnlyVector(std::initializer_list<int> list) : Vec(list) {}
+
+  typedef std::vector<int>::reverse_iterator reverse_iterator;
+  reverse_iterator rbegin() { return Vec.rbegin(); }
+  reverse_iterator rend() { return Vec.rend(); }
+};
+
+// A wrapper around vector which exposes begin(), end(), rbegin() and rend().
+// begin() and end() don't have implementations as this ensures that we will
+// get a linker error if reverse() chooses begin()/end() over rbegin(), rend().
+class BidirectionalVector {
+  std::vector<int> Vec;
+
+public:
+  BidirectionalVector(std::initializer_list<int> list) : Vec(list) {}
+
+  typedef std::vector<int>::iterator iterator;
+  iterator begin();
+  iterator end();
+
+  typedef std::vector<int>::reverse_iterator reverse_iterator;
+  reverse_iterator rbegin() { return Vec.rbegin(); }
+  reverse_iterator rend() { return Vec.rend(); }
+};
+
+template <typename R> void TestRev(const R &r) {
+  int counter = 3;
+  for (int i : r)
+    EXPECT_EQ(i, counter--);
+}
+
+// Test fixture
+template <typename T> class RangeAdapterLValueTest : public ::testing::Test {};
+
+typedef ::testing::Types<std::vector<int>, std::list<int>, int[4]>
+    RangeAdapterLValueTestTypes;
+TYPED_TEST_CASE(RangeAdapterLValueTest, RangeAdapterLValueTestTypes);
+
+TYPED_TEST(RangeAdapterLValueTest, TrivialOperation) {
+  TypeParam v = {0, 1, 2, 3};
+  TestRev(reverse(v));
+
+  const TypeParam c = {0, 1, 2, 3};
+  TestRev(reverse(c));
+}
+
+template <typename T> struct RangeAdapterRValueTest : testing::Test {};
+
+typedef ::testing::Types<std::vector<int>, std::list<int>, ReverseOnlyVector,
+                         BidirectionalVector> RangeAdapterRValueTestTypes;
+TYPED_TEST_CASE(RangeAdapterRValueTest, RangeAdapterRValueTestTypes);
+
+TYPED_TEST(RangeAdapterRValueTest, TrivialOperation) {
+  TestRev(reverse(TypeParam({0, 1, 2, 3})));
+}
+
+} // anonymous namespace
diff --git a/unittests/ADT/SparseBitVectorTest.cpp b/unittests/ADT/SparseBitVectorTest.cpp
index d8fc5ce25db..9127225860b 100644
--- a/unittests/ADT/SparseBitVectorTest.cpp
+++ b/unittests/ADT/SparseBitVectorTest.cpp
@@ -33,4 +33,98 @@ TEST(SparseBitVectorTest, TrivialOperation) {
   EXPECT_FALSE(Vec.test(17));
 }
 
+TEST(SparseBitVectorTest, IntersectWith) {
+  SparseBitVector<> Vec, Other;
+
+  Vec.set(1);
+  Other.set(1);
+  EXPECT_FALSE(Vec &= Other);
+  EXPECT_TRUE(Vec.test(1));
+
+  Vec.clear();
+  Vec.set(5);
+  Other.clear();
+  Other.set(6);
+  EXPECT_TRUE(Vec &= Other);
+  EXPECT_TRUE(Vec.empty());
+
+  Vec.clear();
+  Vec.set(5);
+  Other.clear();
+  Other.set(225);
+  EXPECT_TRUE(Vec &= Other);
+  EXPECT_TRUE(Vec.empty());
+
+  Vec.clear();
+  Vec.set(225);
+  Other.clear();
+  Other.set(5);
+  EXPECT_TRUE(Vec &= Other);
+  EXPECT_TRUE(Vec.empty());
+}
+
+TEST(SparseBitVectorTest, SelfAssignment) {
+  SparseBitVector<> Vec, Other;
+
+  Vec.set(23);
+  Vec.set(234);
+  Vec = Vec;
+  EXPECT_TRUE(Vec.test(23));
+  EXPECT_TRUE(Vec.test(234));
+
+  Vec.clear();
+  Vec.set(17);
+  Vec.set(256);
+  EXPECT_FALSE(Vec |= Vec);
+  EXPECT_TRUE(Vec.test(17));
+  EXPECT_TRUE(Vec.test(256));
+
+  Vec.clear();
+  Vec.set(56);
+  Vec.set(517);
+  EXPECT_FALSE(Vec &= Vec);
+  EXPECT_TRUE(Vec.test(56));
+  EXPECT_TRUE(Vec.test(517));
+
+  Vec.clear();
+  Vec.set(99);
+  Vec.set(333);
+  EXPECT_TRUE(Vec.intersectWithComplement(Vec));
+  EXPECT_TRUE(Vec.empty());
+  EXPECT_FALSE(Vec.intersectWithComplement(Vec));
+
+  Vec.clear();
+  Vec.set(28);
+  Vec.set(43);
+  Vec.intersectWithComplement(Vec, Vec);
+  EXPECT_TRUE(Vec.empty());
+
+  Vec.clear();
+  Vec.set(42);
+  Vec.set(567);
+  Other.set(55);
+  Other.set(567);
+  Vec.intersectWithComplement(Vec, Other);
+  EXPECT_TRUE(Vec.test(42));
+  EXPECT_FALSE(Vec.test(567));
+
+  Vec.clear();
+  Vec.set(19);
+  Vec.set(21);
+  Other.clear();
+  Other.set(19);
+  Other.set(31);
+  Vec.intersectWithComplement(Other, Vec);
+  EXPECT_FALSE(Vec.test(19));
+  EXPECT_TRUE(Vec.test(31));
+
+  Vec.clear();
+  Vec.set(1);
+  Other.clear();
+  Other.set(59);
+  Other.set(75);
+  Vec.intersectWithComplement(Other, Other);
+  EXPECT_TRUE(Vec.empty());
+}
+
 }
diff --git a/unittests/ADT/TripleTest.cpp b/unittests/ADT/TripleTest.cpp
index 1113bb60dc4..ead430bdc48 100644
--- a/unittests/ADT/TripleTest.cpp
+++ b/unittests/ADT/TripleTest.cpp
@@ -129,6 +129,18 @@ TEST(TripleTest, ParsedIDs) {
   EXPECT_EQ(Triple::UnknownOS, T.getOS());
   EXPECT_EQ(Triple::EABI, T.getEnvironment());
 
+  T = Triple("armv6hl-none-linux-gnueabi");
+  EXPECT_EQ(Triple::arm, T.getArch());
+  EXPECT_EQ(Triple::Linux, T.getOS());
+  EXPECT_EQ(Triple::UnknownVendor, T.getVendor());
+  EXPECT_EQ(Triple::GNUEABI, T.getEnvironment());
+
+  T = Triple("armv7hl-none-linux-gnueabi");
+  EXPECT_EQ(Triple::arm, T.getArch());
+  EXPECT_EQ(Triple::Linux, T.getOS());
+  EXPECT_EQ(Triple::UnknownVendor, T.getVendor());
+  EXPECT_EQ(Triple::GNUEABI, T.getEnvironment());
+
   T = Triple("amdil-unknown-unknown");
   EXPECT_EQ(Triple::amdil, T.getArch());
   EXPECT_EQ(Triple::UnknownVendor, T.getVendor());
@@ -149,6 +161,11 @@ TEST(TripleTest, ParsedIDs) {
   EXPECT_EQ(Triple::UnknownVendor, T.getVendor());
   EXPECT_EQ(Triple::UnknownOS, T.getOS());
 
+  T = Triple("sparcel-unknown-unknown");
+  EXPECT_EQ(Triple::sparcel, T.getArch());
+  EXPECT_EQ(Triple::UnknownVendor, T.getVendor());
+  EXPECT_EQ(Triple::UnknownOS, T.getOS());
+
   T = Triple("spir-unknown-unknown");
   EXPECT_EQ(Triple::spir, T.getArch());
   EXPECT_EQ(Triple::UnknownVendor, T.getVendor());
@@ -165,6 +182,18 @@ TEST(TripleTest, ParsedIDs) {
   EXPECT_EQ(Triple::CloudABI, T.getOS());
   EXPECT_EQ(Triple::UnknownEnvironment, T.getEnvironment());
 
+  T = Triple("wasm32-unknown-unknown");
+  EXPECT_EQ(Triple::wasm32, T.getArch());
+  EXPECT_EQ(Triple::UnknownVendor, T.getVendor());
+  EXPECT_EQ(Triple::UnknownOS, T.getOS());
+  EXPECT_EQ(Triple::UnknownEnvironment, T.getEnvironment());
+
+  T = Triple("wasm64-unknown-unknown");
+  EXPECT_EQ(Triple::wasm64, T.getArch());
+  EXPECT_EQ(Triple::UnknownVendor, T.getVendor());
+  EXPECT_EQ(Triple::UnknownOS, T.getOS());
+  EXPECT_EQ(Triple::UnknownEnvironment, T.getEnvironment());
+
   T = Triple("huh");
   EXPECT_EQ(Triple::UnknownArch, T.getArch());
 }
@@ -220,12 +249,12 @@ TEST(TripleTest, Normalization) {
   // Check that normalizing a permutated set of valid components returns a
   // triple with the unpermuted components.
   StringRef C[4];
-  for (int Arch = 1+Triple::UnknownArch; Arch <= Triple::amdil; ++Arch) {
+  for (int Arch = 1+Triple::UnknownArch; Arch <= Triple::LastArchType; ++Arch) {
     C[0] = Triple::getArchTypeName(Triple::ArchType(Arch));
-    for (int Vendor = 1+Triple::UnknownVendor; Vendor <= Triple::PC;
+    for (int Vendor = 1+Triple::UnknownVendor; Vendor <= Triple::LastVendorType;
          ++Vendor) {
       C[1] = Triple::getVendorTypeName(Triple::VendorType(Vendor));
-      for (int OS = 1+Triple::UnknownOS; OS <= Triple::Minix; ++OS) {
+      for (int OS = 1+Triple::UnknownOS; OS <= Triple::LastOSType; ++OS) {
         if (OS == Triple::Win32)
           continue;
 
@@ -240,7 +269,7 @@ TEST(TripleTest, Normalization) {
         EXPECT_EQ(E, Triple::normalize(Join(C[2], C[0], C[1])));
         EXPECT_EQ(E, Triple::normalize(Join(C[2], C[1], C[0])));
 
-        for (int Env = 1 + Triple::UnknownEnvironment; Env <= Triple::Android;
+        for (int Env = 1 + Triple::UnknownEnvironment; Env <= Triple::LastEnvironmentType;
              ++Env) {
           C[3] = Triple::getEnvironmentTypeName(Triple::EnvironmentType(Env));
 
@@ -325,6 +354,8 @@ TEST(TripleTest, MutateName) {
   EXPECT_EQ(Triple::Darwin, T.getOS());
   EXPECT_EQ("i386-pc-darwin", T.getTriple());
 
+  T.setEnvironmentName("amdopencl");
+  EXPECT_EQ(Triple::AMDOpenCL, T.getEnvironment());
 }
 
 TEST(TripleTest, BitWidthPredicates) {
@@ -407,6 +438,31 @@ TEST(TripleTest, BitWidthPredicates) {
   EXPECT_FALSE(T.isArch16Bit());
   EXPECT_FALSE(T.isArch32Bit());
   EXPECT_TRUE(T.isArch64Bit());
+
+  T.setArch(Triple::sparc);
+  EXPECT_FALSE(T.isArch16Bit());
+  EXPECT_TRUE(T.isArch32Bit());
+  EXPECT_FALSE(T.isArch64Bit());
+
+  T.setArch(Triple::sparcel);
+  EXPECT_FALSE(T.isArch16Bit());
+  EXPECT_TRUE(T.isArch32Bit());
+  EXPECT_FALSE(T.isArch64Bit());
+
+  T.setArch(Triple::sparcv9);
+  EXPECT_FALSE(T.isArch16Bit());
+  EXPECT_FALSE(T.isArch32Bit());
+  EXPECT_TRUE(T.isArch64Bit());
+
+  T.setArch(Triple::wasm32);
+  EXPECT_FALSE(T.isArch16Bit());
+  EXPECT_TRUE(T.isArch32Bit());
+  EXPECT_FALSE(T.isArch64Bit());
+
+  T.setArch(Triple::wasm64);
+  EXPECT_FALSE(T.isArch16Bit());
+  EXPECT_FALSE(T.isArch32Bit());
+  EXPECT_TRUE(T.isArch64Bit());
 }
 
 TEST(TripleTest, BitWidthArchVariants) {
@@ -489,6 +545,92 @@ TEST(TripleTest, BitWidthArchVariants) {
   T.setArch(Triple::spir64);
   EXPECT_EQ(Triple::spir, T.get32BitArchVariant().getArch());
   EXPECT_EQ(Triple::spir64, T.get64BitArchVariant().getArch());
+
+  T.setArch(Triple::wasm32);
+  EXPECT_EQ(Triple::wasm32, T.get32BitArchVariant().getArch());
+  EXPECT_EQ(Triple::wasm64, T.get64BitArchVariant().getArch());
+
+  T.setArch(Triple::wasm64);
+  EXPECT_EQ(Triple::wasm32, T.get32BitArchVariant().getArch());
+  EXPECT_EQ(Triple::wasm64, T.get64BitArchVariant().getArch());
+}
+
+TEST(TripleTest, EndianArchVariants) {
+  Triple T;
+  EXPECT_EQ(Triple::UnknownArch, T.getBigEndianArchVariant().getArch());
+  EXPECT_EQ(Triple::UnknownArch, T.getLittleEndianArchVariant().getArch());
+
+  T.setArch(Triple::UnknownArch);
+  EXPECT_EQ(Triple::UnknownArch, T.getBigEndianArchVariant().getArch());
+  EXPECT_EQ(Triple::UnknownArch, T.getLittleEndianArchVariant().getArch());
+
+  T.setArch(Triple::aarch64_be);
+  EXPECT_EQ(Triple::aarch64_be, T.getBigEndianArchVariant().getArch());
+  EXPECT_EQ(Triple::aarch64, T.getLittleEndianArchVariant().getArch());
+
+  T.setArch(Triple::aarch64);
+  EXPECT_EQ(Triple::aarch64_be, T.getBigEndianArchVariant().getArch());
+  EXPECT_EQ(Triple::aarch64, T.getLittleEndianArchVariant().getArch());
+
+  T.setArch(Triple::armeb);
+  EXPECT_EQ(Triple::armeb, T.getBigEndianArchVariant().getArch());
+  EXPECT_EQ(Triple::UnknownArch, T.getLittleEndianArchVariant().getArch());
+
+  T.setArch(Triple::arm);
+  EXPECT_EQ(Triple::UnknownArch, T.getBigEndianArchVariant().getArch());
+  EXPECT_EQ(Triple::arm, T.getLittleEndianArchVariant().getArch());
+
+  T.setArch(Triple::bpfeb);
+  EXPECT_EQ(Triple::bpfeb, T.getBigEndianArchVariant().getArch());
+  EXPECT_EQ(Triple::bpfel, T.getLittleEndianArchVariant().getArch());
+
+  T.setArch(Triple::bpfel);
+  EXPECT_EQ(Triple::bpfeb, T.getBigEndianArchVariant().getArch());
+  EXPECT_EQ(Triple::bpfel, T.getLittleEndianArchVariant().getArch());
+
+  T.setArch(Triple::mips64);
+  EXPECT_EQ(Triple::mips64, T.getBigEndianArchVariant().getArch());
+  EXPECT_EQ(Triple::mips64el, T.getLittleEndianArchVariant().getArch());
+
+  T.setArch(Triple::mips64el);
+  EXPECT_EQ(Triple::mips64, T.getBigEndianArchVariant().getArch());
+  EXPECT_EQ(Triple::mips64el, T.getLittleEndianArchVariant().getArch());
+
+  T.setArch(Triple::mips);
+  EXPECT_EQ(Triple::mips, T.getBigEndianArchVariant().getArch());
+  EXPECT_EQ(Triple::mipsel, T.getLittleEndianArchVariant().getArch());
+
+  T.setArch(Triple::mipsel);
+  EXPECT_EQ(Triple::mips, T.getBigEndianArchVariant().getArch());
+  EXPECT_EQ(Triple::mipsel, T.getLittleEndianArchVariant().getArch());
+
+  T.setArch(Triple::ppc);
+  EXPECT_EQ(Triple::ppc, T.getBigEndianArchVariant().getArch());
+  EXPECT_EQ(Triple::UnknownArch, T.getLittleEndianArchVariant().getArch());
+
+  T.setArch(Triple::ppc64);
+  EXPECT_EQ(Triple::ppc64, T.getBigEndianArchVariant().getArch());
+  EXPECT_EQ(Triple::ppc64le, T.getLittleEndianArchVariant().getArch());
+
+  T.setArch(Triple::ppc64le);
+  EXPECT_EQ(Triple::ppc64, T.getBigEndianArchVariant().getArch());
+  EXPECT_EQ(Triple::ppc64le, T.getLittleEndianArchVariant().getArch());
+
+  T.setArch(Triple::sparc);
+  EXPECT_EQ(Triple::sparc, T.getBigEndianArchVariant().getArch());
+  EXPECT_EQ(Triple::sparcel, T.getLittleEndianArchVariant().getArch());
+
+  T.setArch(Triple::sparcel);
+  EXPECT_EQ(Triple::sparc, T.getBigEndianArchVariant().getArch());
+  EXPECT_EQ(Triple::sparcel, T.getLittleEndianArchVariant().getArch());
+
+  T.setArch(Triple::thumb);
+  EXPECT_EQ(Triple::UnknownArch, T.getBigEndianArchVariant().getArch());
+  EXPECT_EQ(Triple::thumb, T.getLittleEndianArchVariant().getArch());
+
+  T.setArch(Triple::thumbeb);
+  EXPECT_EQ(Triple::thumbeb, T.getBigEndianArchVariant().getArch());
+  EXPECT_EQ(Triple::UnknownArch, T.getLittleEndianArchVariant().getArch());
 }
 
 TEST(TripleTest, getOSVersion) {
@@ -656,24 +798,185 @@ TEST(TripleTest, NormalizeWindows) {
 }
 
 TEST(TripleTest, getARMCPUForArch) {
+  // Standard ARM Architectures.
+  {
+    llvm::Triple Triple("armv4-unknown-eabi");
+    EXPECT_STREQ("strongarm", Triple.getARMCPUForArch());
+  }
+  {
+    llvm::Triple Triple("armv4t-unknown-eabi");
+    EXPECT_STREQ("arm7tdmi", Triple.getARMCPUForArch());
+  }
+  {
+    llvm::Triple Triple("armv5-unknown-eabi");
+    EXPECT_STREQ("arm10tdmi", Triple.getARMCPUForArch());
+  }
+  {
+    llvm::Triple Triple("armv5t-unknown-eabi");
+    EXPECT_STREQ("arm10tdmi", Triple.getARMCPUForArch());
+  }
+  {
+    llvm::Triple Triple("armv5e-unknown-eabi");
+    EXPECT_STREQ("arm1022e", Triple.getARMCPUForArch());
+  }
+  {
+    llvm::Triple Triple("armv5tej-unknown-eabi");
+    EXPECT_STREQ("arm926ej-s", Triple.getARMCPUForArch());
+  }
+  {
+    llvm::Triple Triple("armv6-unknown-eabi");
+    EXPECT_STREQ("arm1136jf-s", Triple.getARMCPUForArch());
+  }
+  {
+    llvm::Triple Triple("armv6j-unknown-eabi");
+    EXPECT_STREQ("arm1136j-s", Triple.getARMCPUForArch());
+  }
+  {
+    llvm::Triple Triple("armv6k-unknown-eabi");
+    EXPECT_STREQ("arm1176jzf-s", Triple.getARMCPUForArch());
+  }
+  {
+    llvm::Triple Triple("armv6zk-unknown-eabi");
+    EXPECT_STREQ("arm1176jzf-s", Triple.getARMCPUForArch());
+  }
+  {
+    llvm::Triple Triple("armv6t2-unknown-eabi");
+    EXPECT_STREQ("arm1156t2-s", Triple.getARMCPUForArch());
+  }
+  {
+    llvm::Triple Triple("armv6m-unknown-eabi");
+    EXPECT_STREQ("cortex-m0", Triple.getARMCPUForArch());
+  }
+  {
+    llvm::Triple Triple("armv7-unknown-eabi");
+    EXPECT_STREQ("cortex-a8", Triple.getARMCPUForArch());
+  }
+  {
+    llvm::Triple Triple("armv7a-unknown-eabi");
+    EXPECT_STREQ("cortex-a8", Triple.getARMCPUForArch());
+  }
+  {
+    llvm::Triple Triple("armv7m-unknown-eabi");
+    EXPECT_STREQ("cortex-m3", Triple.getARMCPUForArch());
+  }
+  {
+    llvm::Triple Triple("armv7r-unknown-eabi");
+    EXPECT_STREQ("cortex-r4", Triple.getARMCPUForArch());
+  }
+  {
+    llvm::Triple Triple("armv7r-unknown-eabi");
+    EXPECT_STREQ("cortex-r4", Triple.getARMCPUForArch());
+  }
+  {
+    llvm::Triple Triple("armv7r-unknown-eabi");
+    EXPECT_STREQ("cortex-r4", Triple.getARMCPUForArch());
+  }
+  {
+    llvm::Triple Triple("armv7r-unknown-eabi");
+    EXPECT_STREQ("cortex-r4", Triple.getARMCPUForArch());
+  }
+  {
+    llvm::Triple Triple("armv8a-unknown-eabi");
+    EXPECT_STREQ("cortex-a53", Triple.getARMCPUForArch());
+  }
+  {
+    llvm::Triple Triple("armv8.1a-unknown-eabi");
+    EXPECT_STREQ("generic", Triple.getARMCPUForArch());
+  }
+  // Non-synonym names, using -march style, not default arch.
+  {
+    llvm::Triple Triple("arm");
+    EXPECT_STREQ("cortex-a8", Triple.getARMCPUForArch("armv7-a"));
+  }
+  {
+    llvm::Triple Triple("arm");
+    EXPECT_STREQ("cortex-m3", Triple.getARMCPUForArch("armv7-m"));
+  }
+  {
+    llvm::Triple Triple("arm");
+    EXPECT_STREQ("cortex-a53", Triple.getARMCPUForArch("armv8"));
+  }
+  {
+    llvm::Triple Triple("arm");
+    EXPECT_STREQ("cortex-a53", Triple.getARMCPUForArch("armv8-a"));
+  }
+  // Platform specific defaults.
+  {
+    llvm::Triple Triple("arm--nacl");
+    EXPECT_STREQ("cortex-a8", Triple.getARMCPUForArch());
+  }
   {
     llvm::Triple Triple("armv6-unknown-freebsd");
     EXPECT_STREQ("arm1176jzf-s", Triple.getARMCPUForArch());
   }
+  {
+    llvm::Triple Triple("thumbv6-unknown-freebsd");
+    EXPECT_STREQ("arm1176jzf-s", Triple.getARMCPUForArch());
+  }
+  {
+    llvm::Triple Triple("armebv6-unknown-freebsd");
+    EXPECT_STREQ("arm1176jzf-s", Triple.getARMCPUForArch());
+  }
+  {
+    llvm::Triple Triple("arm--win32");
+    EXPECT_STREQ("cortex-a9", Triple.getARMCPUForArch());
+  }
+  // Some alternative architectures
+  {
+    llvm::Triple Triple("xscale-unknown-eabi");
+    EXPECT_STREQ("xscale", Triple.getARMCPUForArch());
+  }
+  {
+    llvm::Triple Triple("iwmmxt-unknown-eabi");
+    EXPECT_STREQ("iwmmxt", Triple.getARMCPUForArch());
+  }
   {
     llvm::Triple Triple("armv7s-apple-ios7");
     EXPECT_STREQ("swift", Triple.getARMCPUForArch());
   }
   {
-    llvm::Triple Triple("armv7-apple-ios7");
+    llvm::Triple Triple("armv7em-apple-ios7");
+    EXPECT_STREQ("cortex-m4", Triple.getARMCPUForArch());
+  }
+  {
+    llvm::Triple Triple("armv7l-linux-gnueabihf");
     EXPECT_STREQ("cortex-a8", Triple.getARMCPUForArch());
-    EXPECT_STREQ("swift", Triple.getARMCPUForArch("armv7s"));
   }
   {
-    llvm::Triple Triple("arm--nacl");
-    EXPECT_STREQ("cortex-a8", Triple.getARMCPUForArch("arm"));
+    llvm::Triple Triple("armv6sm-apple-ios7");
+    EXPECT_STREQ("cortex-m0", Triple.getARMCPUForArch());
+  }
+  // armeb is permitted, but armebeb is not
+  {
+    llvm::Triple Triple("armeb-none-eabi");
+    EXPECT_STREQ("arm7tdmi", Triple.getARMCPUForArch());
+  }
+  {
+    llvm::Triple Triple("armebeb-none-eabi");
+    EXPECT_EQ(nullptr, Triple.getARMCPUForArch());
+  }
+  {
+    llvm::Triple Triple("armebv6eb-none-eabi");
+    EXPECT_EQ(nullptr, Triple.getARMCPUForArch());
+  }
+  // armebv6 and armv6eb are permitted, but armebv6eb is not
+  {
+    llvm::Triple Triple("armebv6-non-eabi");
+    EXPECT_STREQ("arm1136jf-s", Triple.getARMCPUForArch());
+  }
+  {
+    llvm::Triple Triple("armv6eb-none-eabi");
+    EXPECT_STREQ("arm1136jf-s", Triple.getARMCPUForArch());
+  }
+  // xscaleeb is permitted, but armebxscale is not
+  {
+    llvm::Triple Triple("xscaleeb-none-eabi");
+    EXPECT_STREQ("xscale", Triple.getARMCPUForArch());
+  }
+  {
+    llvm::Triple Triple("armebxscale-none-eabi");
+    EXPECT_EQ(nullptr, Triple.getARMCPUForArch());
   }
-}
 }
 
 TEST(TripleTest, NormalizeARM) {
@@ -692,3 +995,90 @@ TEST(TripleTest, NormalizeARM) {
   T = Triple("armv6eb--netbsd-eabi");
   EXPECT_EQ(Triple::armeb, T.getArch());
 }
+
+TEST(TripleTest, ParseARMArch) {
+  // ARM
+  {
+    Triple T = Triple("arm");
+    EXPECT_EQ(Triple::arm, T.getArch());
+  }
+  {
+    Triple T = Triple("armv6t2");
+    EXPECT_EQ(Triple::arm, T.getArch());
+  }
+  {
+    Triple T = Triple("armv8");
+    EXPECT_EQ(Triple::arm, T.getArch());
+  }
+  {
+    Triple T = Triple("armeb");
+    EXPECT_EQ(Triple::armeb, T.getArch());
+  }
+  {
+    Triple T = Triple("armv5eb");
+    EXPECT_EQ(Triple::armeb, T.getArch());
+  }
+  {
+    Triple T = Triple("armebv7m");
+    EXPECT_EQ(Triple::armeb, T.getArch());
+  }
+  {
+    Triple T = Triple("armv7eb");
+    EXPECT_EQ(Triple::armeb, T.getArch());
+  }
+  // THUMB
+  {
+    Triple T = Triple("thumb");
+    EXPECT_EQ(Triple::thumb, T.getArch());
+  }
+  {
+    Triple T = Triple("thumbv7a");
+    EXPECT_EQ(Triple::thumb, T.getArch());
+  }
+  {
+    Triple T = Triple("thumbeb");
+    EXPECT_EQ(Triple::thumbeb, T.getArch());
+  }
+  {
+    Triple T = Triple("thumbv4teb");
+    EXPECT_EQ(Triple::thumbeb, T.getArch());
+  }
+  {
+    Triple T = Triple("thumbebv7");
+    EXPECT_EQ(Triple::thumbeb, T.getArch());
+  }
+  {
+    Triple T = Triple("armv6m");
+    EXPECT_EQ(Triple::thumb, T.getArch());
+  }
+  {
+    Triple T = Triple("thumbv2");
+    EXPECT_EQ(Triple::UnknownArch, T.getArch());
+  }
+  {
+    Triple T = Triple("thumbebv6eb");
+    EXPECT_EQ(Triple::UnknownArch, T.getArch());
+  }
+  // AARCH64
+  {
+    Triple T = Triple("arm64");
+    EXPECT_EQ(Triple::aarch64, T.getArch());
+  }
+  {
+    Triple T = Triple("aarch64");
+    EXPECT_EQ(Triple::aarch64, T.getArch());
+  }
+  {
+    Triple T = Triple("aarch64_be");
+    EXPECT_EQ(Triple::aarch64_be, T.getArch());
+  }
+  {
+    Triple T = Triple("aarch64be");
+    EXPECT_EQ(Triple::UnknownArch, T.getArch());
+  }
+  {
+    Triple T = Triple("arm64be");
+    EXPECT_EQ(Triple::UnknownArch, T.getArch());
+  }
+}
+} // end anonymous namespace
diff --git a/unittests/Analysis/AliasAnalysisTest.cpp b/unittests/Analysis/AliasAnalysisTest.cpp
index 1ba396c1eca..1316e7252a6 100644
--- a/unittests/Analysis/AliasAnalysisTest.cpp
+++ b/unittests/Analysis/AliasAnalysisTest.cpp
@@ -27,11 +27,11 @@ class AliasAnalysisTest : public testing::Test {
   // This is going to check that calling getModRefInfo without a location, and
   // with a default location, first, doesn't crash, and second, gives the right
   // answer.
-  void CheckModRef(Instruction *I, AliasAnalysis::ModRefResult Result) {
+  void CheckModRef(Instruction *I, ModRefInfo Result) {
     static char ID;
     class CheckModRefTestPass : public FunctionPass {
     public:
-      CheckModRefTestPass(Instruction *I, AliasAnalysis::ModRefResult Result)
+      CheckModRefTestPass(Instruction *I, ModRefInfo Result)
           : FunctionPass(ID), ExpectResult(Result), I(I) {}
       static int initialize() {
         PassInfo *PI = new PassInfo("CheckModRef testing pass", "", &ID,
@@ -47,11 +47,11 @@ class AliasAnalysisTest : public testing::Test {
       }
       bool runOnFunction(Function &) override {
         AliasAnalysis &AA = getAnalysis<AliasAnalysis>();
-        EXPECT_EQ(AA.getModRefInfo(I, AliasAnalysis::Location()), ExpectResult);
+        EXPECT_EQ(AA.getModRefInfo(I, MemoryLocation()), ExpectResult);
         EXPECT_EQ(AA.getModRefInfo(I), ExpectResult);
         return false;
       }
-      AliasAnalysis::ModRefResult ExpectResult;
+      ModRefInfo ExpectResult;
       Instruction *I;
     };
     static int initialize = CheckModRefTestPass::initialize();
@@ -81,13 +81,23 @@ TEST_F(AliasAnalysisTest, getModRefInfo) {
   auto *Store1 = new StoreInst(Value, Addr, BB);
   auto *Load1 = new LoadInst(Addr, "load", BB);
   auto *Add1 = BinaryOperator::CreateAdd(Value, Value, "add", BB);
+  auto *VAArg1 = new VAArgInst(Addr, PtrType, "vaarg", BB);
+  auto *CmpXChg1 = new AtomicCmpXchgInst(Addr, ConstantInt::get(IntType, 0),
+                                         ConstantInt::get(IntType, 1),
+                                         Monotonic, Monotonic, CrossThread, BB);
+  auto *AtomicRMW =
+      new AtomicRMWInst(AtomicRMWInst::Xchg, Addr, ConstantInt::get(IntType, 1),
+                        Monotonic, CrossThread, BB);
 
   ReturnInst::Create(C, nullptr, BB);
 
   // Check basic results
-  CheckModRef(Store1, AliasAnalysis::ModRefResult::Mod);
-  CheckModRef(Load1, AliasAnalysis::ModRefResult::Ref);
-  CheckModRef(Add1, AliasAnalysis::ModRefResult::NoModRef);
+  CheckModRef(Store1, MRI_Mod);
+  CheckModRef(Load1, MRI_Ref);
+  CheckModRef(Add1, MRI_NoModRef);
+  CheckModRef(VAArg1, MRI_ModRef);
+  CheckModRef(CmpXChg1, MRI_ModRef);
+  CheckModRef(AtomicRMW, MRI_ModRef);
 }
 
 } // end anonymous namspace
diff --git a/unittests/AsmParser/AsmParserTest.cpp b/unittests/AsmParser/AsmParserTest.cpp
new file mode 100644
index 00000000000..099f5b5f923
--- /dev/null
+++ b/unittests/AsmParser/AsmParserTest.cpp
@@ -0,0 +1,115 @@
+//===- llvm/unittest/AsmParser/AsmParserTest.cpp - asm parser unittests ---===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/ADT/StringRef.h"
+#include "llvm/AsmParser/Parser.h"
+#include "llvm/AsmParser/SlotMapping.h"
+#include "llvm/IR/Constants.h"
+#include "llvm/IR/LLVMContext.h"
+#include "llvm/IR/Module.h"
+#include "llvm/Support/SourceMgr.h"
+#include "gtest/gtest.h"
+
+using namespace llvm;
+
+namespace {
+
+TEST(AsmParserTest, NullTerminatedInput) {
+  LLVMContext &Ctx = getGlobalContext();
+  StringRef Source = "; Empty module \n";
+  SMDiagnostic Error;
+  auto Mod = parseAssemblyString(Source, Error, Ctx);
+
+  EXPECT_TRUE(Mod != nullptr);
+  EXPECT_TRUE(Error.getMessage().empty());
+}
+
+#ifdef GTEST_HAS_DEATH_TEST
+#ifndef NDEBUG
+
+TEST(AsmParserTest, NonNullTerminatedInput) {
+  LLVMContext &Ctx = getGlobalContext();
+  StringRef Source = "; Empty module \n\1\2";
+  SMDiagnostic Error;
+  std::unique_ptr<Module> Mod;
+  EXPECT_DEATH(Mod = parseAssemblyString(Source.substr(0, Source.size() - 2),
+                                         Error, Ctx),
+               "Buffer is not null terminated!");
+}
+
+#endif
+#endif
+
+TEST(AsmParserTest, SlotMappingTest) {
+  LLVMContext &Ctx = getGlobalContext();
+  StringRef Source = "@0 = global i32 0\n !0 = !{}\n !42 = !{i32 42}";
+  SMDiagnostic Error;
+  SlotMapping Mapping;
+  auto Mod = parseAssemblyString(Source, Error, Ctx, &Mapping);
+
+  EXPECT_TRUE(Mod != nullptr);
+  EXPECT_TRUE(Error.getMessage().empty());
+
+  ASSERT_EQ(Mapping.GlobalValues.size(), 1u);
+  EXPECT_TRUE(isa<GlobalVariable>(Mapping.GlobalValues[0]));
+
+  EXPECT_EQ(Mapping.MetadataNodes.size(), 2u);
+  EXPECT_EQ(Mapping.MetadataNodes.count(0), 1u);
+  EXPECT_EQ(Mapping.MetadataNodes.count(42), 1u);
+  EXPECT_EQ(Mapping.MetadataNodes.count(1), 0u);
+}
+
+TEST(AsmParserTest, TypeAndConstantValueParsing) {
+  LLVMContext &Ctx = getGlobalContext();
+  SMDiagnostic Error;
+  StringRef Source = "define void @test() {\n  entry:\n  ret void\n}";
+  auto Mod = parseAssemblyString(Source, Error, Ctx);
+  ASSERT_TRUE(Mod != nullptr);
+  auto &M = *Mod;
+
+  const Value *V;
+  V = parseConstantValue("double 3.5", Error, M);
+  ASSERT_TRUE(V);
+  EXPECT_TRUE(V->getType()->isDoubleTy());
+  ASSERT_TRUE(isa<ConstantFP>(V));
+  EXPECT_TRUE(cast<ConstantFP>(V)->isExactlyValue(3.5));
+
+  V = parseConstantValue("i32 42", Error, M);
+  ASSERT_TRUE(V);
+  EXPECT_TRUE(V->getType()->isIntegerTy());
+  ASSERT_TRUE(isa<ConstantInt>(V));
+  EXPECT_TRUE(cast<ConstantInt>(V)->equalsInt(42));
+
+  V = parseConstantValue("<4 x i32> <i32 0, i32 1, i32 2, i32 3>", Error, M);
+  ASSERT_TRUE(V);
+  EXPECT_TRUE(V->getType()->isVectorTy());
+  ASSERT_TRUE(isa<ConstantDataVector>(V));
+
+  V = parseConstantValue("i32 add (i32 1, i32 2)", Error, M);
+  ASSERT_TRUE(V);
+  ASSERT_TRUE(isa<ConstantInt>(V));
+
+  V = parseConstantValue("i8* blockaddress(@test, %entry)", Error, M);
+  ASSERT_TRUE(V);
+  ASSERT_TRUE(isa<BlockAddress>(V));
+
+  EXPECT_FALSE(parseConstantValue("duble 3.25", Error, M));
+  EXPECT_EQ(Error.getMessage(), "expected type");
+
+  EXPECT_FALSE(parseConstantValue("i32 3.25", Error, M));
+  EXPECT_EQ(Error.getMessage(), "floating point constant invalid for type");
+
+  EXPECT_FALSE(parseConstantValue("i32* @foo", Error, M));
+  EXPECT_EQ(Error.getMessage(), "expected a constant value");
+
+  EXPECT_FALSE(parseConstantValue("i32 3, ", Error, M));
+  EXPECT_EQ(Error.getMessage(), "expected end of string");
+}
+
+} // end anonymous namespace
diff --git a/unittests/AsmParser/CMakeLists.txt b/unittests/AsmParser/CMakeLists.txt
new file mode 100644
index 00000000000..1920bfaa3aa
--- /dev/null
+++ b/unittests/AsmParser/CMakeLists.txt
@@ -0,0 +1,9 @@
+set(LLVM_LINK_COMPONENTS
+  AsmParser
+  Core
+  Support
+  )
+
+add_llvm_unittest(AsmParserTests
+  AsmParserTest.cpp
+  )
diff --git a/unittests/AsmParser/Makefile b/unittests/AsmParser/Makefile
new file mode 100644
index 00000000000..41eadd43043
--- /dev/null
+++ b/unittests/AsmParser/Makefile
@@ -0,0 +1,15 @@
+##===- unittests/AsmParser/Makefile ------------------------*- Makefile -*-===##
+#
+#                     The LLVM Compiler Infrastructure
+#
+# This file is distributed under the University of Illinois Open Source
+# License. See LICENSE.TXT for details.
+#
+##===----------------------------------------------------------------------===##
+
+LEVEL = ../..
+TESTNAME = AsmParser
+LINK_COMPONENTS := AsmParser Core Support
+
+include $(LEVEL)/Makefile.config
+include $(LLVM_SRC_ROOT)/unittests/Makefile.unittest
diff --git a/unittests/Bitcode/BitReaderTest.cpp b/unittests/Bitcode/BitReaderTest.cpp
index 1f28ec6f0d6..3c56ea04c74 100644
--- a/unittests/Bitcode/BitReaderTest.cpp
+++ b/unittests/Bitcode/BitReaderTest.cpp
@@ -8,6 +8,7 @@
 //===----------------------------------------------------------------------===//
 
 #include "llvm/ADT/SmallString.h"
+#include "llvm/ADT/STLExtras.h"
 #include "llvm/AsmParser/Parser.h"
 #include "llvm/Bitcode/BitstreamWriter.h"
 #include "llvm/Bitcode/ReaderWriter.h"
@@ -16,6 +17,7 @@
 #include "llvm/IR/LLVMContext.h"
 #include "llvm/IR/Module.h"
 #include "llvm/IR/Verifier.h"
+#include "llvm/Support/DataStream.h"
 #include "llvm/Support/Debug.h"
 #include "llvm/Support/MemoryBuffer.h"
 #include "llvm/Support/SourceMgr.h"
@@ -53,9 +55,52 @@ static std::unique_ptr<Module> getLazyModuleFromAssembly(LLVMContext &Context,
   writeModuleToBuffer(parseAssembly(Assembly), Mem);
   std::unique_ptr<MemoryBuffer> Buffer =
       MemoryBuffer::getMemBuffer(Mem.str(), "test", false);
-  ErrorOr<Module *> ModuleOrErr =
+  ErrorOr<std::unique_ptr<Module>> ModuleOrErr =
       getLazyBitcodeModule(std::move(Buffer), Context);
-  return std::unique_ptr<Module>(ModuleOrErr.get());
+  return std::move(ModuleOrErr.get());
+}
+
+class BufferDataStreamer : public DataStreamer {
+  std::unique_ptr<MemoryBuffer> Buffer;
+  unsigned Pos = 0;
+  size_t GetBytes(unsigned char *Out, size_t Len) override {
+    StringRef Buf = Buffer->getBuffer();
+    size_t Left = Buf.size() - Pos;
+    Len = std::min(Left, Len);
+    memcpy(Out, Buffer->getBuffer().substr(Pos).data(), Len);
+    Pos += Len;
+    return Len;
+  }
+
+public:
+  BufferDataStreamer(std::unique_ptr<MemoryBuffer> Buffer)
+      : Buffer(std::move(Buffer)) {}
+};
+
+static std::unique_ptr<Module>
+getStreamedModuleFromAssembly(LLVMContext &Context, SmallString<1024> &Mem,
+                              const char *Assembly) {
+  writeModuleToBuffer(parseAssembly(Assembly), Mem);
+  std::unique_ptr<MemoryBuffer> Buffer =
+      MemoryBuffer::getMemBuffer(Mem.str(), "test", false);
+  auto Streamer = llvm::make_unique<BufferDataStreamer>(std::move(Buffer));
+  ErrorOr<std::unique_ptr<Module>> ModuleOrErr =
+      getStreamedBitcodeModule("test", std::move(Streamer), Context);
+  return std::move(ModuleOrErr.get());
+}
+
+TEST(BitReaderTest, MateralizeForwardRefWithStream) {
+  SmallString<1024> Mem;
+
+  LLVMContext Context;
+  std::unique_ptr<Module> M = getStreamedModuleFromAssembly(
+      Context, Mem, "@table = constant i8* blockaddress(@func, %bb)\n"
+                    "define void @func() {\n"
+                    "  unreachable\n"
+                    "bb:\n"
+                    "  unreachable\n"
+                    "}\n");
+  EXPECT_FALSE(M->getFunction("func")->empty());
 }
 
 TEST(BitReaderTest, DematerializeFunctionPreservesLinkageType) {
@@ -75,13 +120,77 @@ TEST(BitReaderTest, DematerializeFunctionPreservesLinkageType) {
               GlobalValue::InternalLinkage);
 
   // Check that the linkage type is preserved after dematerialization.
-  M->getFunction("func")->Dematerialize();
+  M->getFunction("func")->dematerialize();
   EXPECT_TRUE(M->getFunction("func")->empty());
   EXPECT_TRUE(M->getFunction("func")->getLinkage() ==
               GlobalValue::InternalLinkage);
   EXPECT_FALSE(verifyModule(*M, &dbgs()));
 }
 
+// Tests that lazy evaluation can parse functions out of order.
+TEST(BitReaderTest, MaterializeFunctionsOutOfOrder) {
+  SmallString<1024> Mem;
+  LLVMContext Context;
+  std::unique_ptr<Module> M = getLazyModuleFromAssembly(
+      Context, Mem, "define void @f() {\n"
+                    "  unreachable\n"
+                    "}\n"
+                    "define void @g() {\n"
+                    "  unreachable\n"
+                    "}\n"
+                    "define void @h() {\n"
+                    "  unreachable\n"
+                    "}\n"
+                    "define void @j() {\n"
+                    "  unreachable\n"
+                    "}\n");
+  EXPECT_FALSE(verifyModule(*M, &dbgs()));
+
+  Function *F = M->getFunction("f");
+  Function *G = M->getFunction("g");
+  Function *H = M->getFunction("h");
+  Function *J = M->getFunction("j");
+
+  // Initially all functions are not materialized (no basic blocks).
+  EXPECT_TRUE(F->empty());
+  EXPECT_TRUE(G->empty());
+  EXPECT_TRUE(H->empty());
+  EXPECT_TRUE(J->empty());
+  EXPECT_FALSE(verifyModule(*M, &dbgs()));
+
+  // Materialize h.
+  H->materialize();
+  EXPECT_TRUE(F->empty());
+  EXPECT_TRUE(G->empty());
+  EXPECT_FALSE(H->empty());
+  EXPECT_TRUE(J->empty());
+  EXPECT_FALSE(verifyModule(*M, &dbgs()));
+
+  // Materialize g.
+  G->materialize();
+  EXPECT_TRUE(F->empty());
+  EXPECT_FALSE(G->empty());
+  EXPECT_FALSE(H->empty());
+  EXPECT_TRUE(J->empty());
+  EXPECT_FALSE(verifyModule(*M, &dbgs()));
+
+  // Materialize j.
+  J->materialize();
+  EXPECT_TRUE(F->empty());
+  EXPECT_FALSE(G->empty());
+  EXPECT_FALSE(H->empty());
+  EXPECT_FALSE(J->empty());
+  EXPECT_FALSE(verifyModule(*M, &dbgs()));
+
+  // Materialize f.
+  F->materialize();
+  EXPECT_FALSE(F->empty());
+  EXPECT_FALSE(G->empty());
+  EXPECT_FALSE(H->empty());
+  EXPECT_FALSE(J->empty());
+  EXPECT_FALSE(verifyModule(*M, &dbgs()));
+}
+
 TEST(BitReaderTest, MaterializeFunctionsForBlockAddr) { // PR11677
   SmallString<1024> Mem;
 
@@ -96,7 +205,7 @@ TEST(BitReaderTest, MaterializeFunctionsForBlockAddr) { // PR11677
   EXPECT_FALSE(verifyModule(*M, &dbgs()));
 
   // Try (and fail) to dematerialize @func.
-  M->getFunction("func")->Dematerialize();
+  M->getFunction("func")->dematerialize();
   EXPECT_FALSE(M->getFunction("func")->empty());
 }
 
@@ -127,7 +236,7 @@ TEST(BitReaderTest, MaterializeFunctionsForBlockAddrInFunctionBefore) {
   EXPECT_FALSE(verifyModule(*M, &dbgs()));
 
   // Try (and fail) to dematerialize @func.
-  M->getFunction("func")->Dematerialize();
+  M->getFunction("func")->dematerialize();
   EXPECT_FALSE(M->getFunction("func")->empty());
   EXPECT_FALSE(verifyModule(*M, &dbgs()));
 }
@@ -159,7 +268,7 @@ TEST(BitReaderTest, MaterializeFunctionsForBlockAddrInFunctionAfter) {
   EXPECT_FALSE(verifyModule(*M, &dbgs()));
 
   // Try (and fail) to dematerialize @func.
-  M->getFunction("func")->Dematerialize();
+  M->getFunction("func")->dematerialize();
   EXPECT_FALSE(M->getFunction("func")->empty());
   EXPECT_FALSE(verifyModule(*M, &dbgs()));
 }
diff --git a/unittests/Bitcode/CMakeLists.txt b/unittests/Bitcode/CMakeLists.txt
index b8a48cf683d..3b091648c05 100644
--- a/unittests/Bitcode/CMakeLists.txt
+++ b/unittests/Bitcode/CMakeLists.txt
@@ -4,8 +4,6 @@ set(LLVM_LINK_COMPONENTS
   BitWriter
   NaClBitAnalysis
   NaClBitTestUtils
-  NaClBitReader
-  NaClBitWriter
   Core
   Support
   )
@@ -13,16 +11,4 @@ set(LLVM_LINK_COMPONENTS
 add_llvm_unittest(BitcodeTests
   BitReaderTest.cpp
   BitstreamReaderTest.cpp
-  NaClAbbrevTrieTest.cpp
-  NaClBitReaderTest.cpp
-  NaClBitstreamReaderTest.cpp
-  NaClCompressTests.cpp
-  NaClMungedBitcodeTest.cpp
-  NaClMungedIoTest.cpp
-  NaClMungeWriteErrorTests.cpp
-  NaClObjDumpTest.cpp
-  NaClObjDumpTypesTest.cpp
-  NaClParseTypesTest.cpp
-  NaClParseInstsTest.cpp
-  NaClTextFormatterTest.cpp
   )
diff --git a/unittests/Bitcode/Makefile b/unittests/Bitcode/Makefile
index 58c7328b19f..33b09b914b5 100644
--- a/unittests/Bitcode/Makefile
+++ b/unittests/Bitcode/Makefile
@@ -9,8 +9,7 @@
 
 LEVEL = ../..
 TESTNAME = Bitcode
-LINK_COMPONENTS := AsmParser BitReader BitWriter Core Support \
-	   NaClBitReader NaClBitWriter NaClBitAnalysis NaClBitTestUtils
+LINK_COMPONENTS := AsmParser BitReader BitWriter Core Support
 
 include $(LEVEL)/Makefile.config
 include $(LLVM_SRC_ROOT)/unittests/Makefile.unittest
diff --git a/unittests/Bitcode/NaClAbbrevTrieTest.cpp b/unittests/Bitcode/NaClAbbrevTrieTest.cpp
deleted file mode 100644
index cfdd2aa5ce6..00000000000
--- a/unittests/Bitcode/NaClAbbrevTrieTest.cpp
+++ /dev/null
@@ -1,972 +0,0 @@
-//===- llvm/unittest/Bitcode/NaClAbbrevTest.cpp - Tests for NaCl Abbrevs --===//
-//
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
-//
-//===----------------------------------------------------------------------===//
-
-// Tests if we properly sort abbreviations when building an
-// abbreviation trie.
-
-#include "llvm/Bitcode/NaCl/AbbrevTrieNode.h"
-#include "llvm/Bitcode/NaCl/NaClBitCodes.h"
-#include "llvm/Bitcode/NaCl/NaClBitcodeValueDist.h"
-#include "gtest/gtest.h"
-
-#include <iostream>
-#include <sstream>
-
-using namespace llvm;
-
-namespace {
-
-static const unsigned MaxValueIndex = NaClValueIndexCutoff + 1;
-
-typedef SmallVector<NaClBitCodeAbbrev*, 32> AbbrevVector;
-
-static void Clear(AbbrevVector &Vector) {
-  for (AbbrevVector::iterator Iter = Vector.begin(), IterEnd = Vector.end();
-       Iter != IterEnd; ++Iter) {
-    (*Iter)->dropRef();
-  }
-}
-
-static void Clear(AbbrevLookupSizeMap &LookupMap) {
-  for (AbbrevLookupSizeMap::iterator
-           Iter = LookupMap.begin(), IterEnd = LookupMap.end();
-       Iter != IterEnd; ++Iter) {
-    delete Iter->second;
-  }
-}
-
-static std::string DescribeAbbreviations(AbbrevVector &Abbrevs) {
-  std::string Message;
-  raw_string_ostream ostrm(Message);
-  for (AbbrevVector::const_iterator
-           Iter = Abbrevs.begin(), IterEnd = Abbrevs.end();
-       Iter != IterEnd; ++Iter) {
-    (*Iter)->Print(ostrm);
-  }
-  return ostrm.str();
-}
-
-static std::string DescribeAbbrevTrieNode(const AbbrevTrieNode *Node,
-                                          bool LocalOnly) {
-  std::string Message;
-  raw_string_ostream ostrm(Message);
-  if (Node)
-    Node->Print(ostrm, "", LocalOnly);
-  else
-    ostrm << "NULL";
-  return ostrm.str();
-}
-
-static std::string DescribeAbbrevTrie(const AbbrevTrieNode *Node) {
-  return DescribeAbbrevTrieNode(Node, false);
-}
-
-static std::string DescribeAbbrevTrieNode(const AbbrevTrieNode *Node) {
-  return DescribeAbbrevTrieNode(Node, true);
-}
-
-static std::string DescribeRecord(const NaClBitcodeRecordData &Record) {
-  std::string Message;
-  raw_string_ostream ostrm(Message);
-  Record.Print(ostrm);
-  return ostrm.str();
-}
-
-TEST(NaClAbbrevTrieTest, Simple) {
-  // Test example of multiple abbreviations of length 2.
-  AbbrevVector Abbrevs;
-  // [1, VBR(6)]
-  NaClBitCodeAbbrev *Abbrev = new NaClBitCodeAbbrev();
-  Abbrev->Add(NaClBitCodeAbbrevOp(1));
-  Abbrev->Add(NaClBitCodeAbbrevOp(NaClBitCodeAbbrevOp::VBR, 6));
-  Abbrevs.push_back(Abbrev);
-  // [4, VBR(8)]
-  Abbrev = new NaClBitCodeAbbrev();
-  Abbrev->Add(NaClBitCodeAbbrevOp(4));
-  Abbrev->Add(NaClBitCodeAbbrevOp(NaClBitCodeAbbrevOp::VBR, 8));
-  Abbrevs.push_back(Abbrev);
-  // [4, 0]
-  Abbrev = new NaClBitCodeAbbrev();
-  Abbrev->Add(NaClBitCodeAbbrevOp(4));
-  Abbrev->Add(NaClBitCodeAbbrevOp(0));
-  Abbrevs.push_back(Abbrev);
-  // [1, 2]
-  Abbrev = new NaClBitCodeAbbrev();
-  Abbrev->Add(NaClBitCodeAbbrevOp(1));
-  Abbrev->Add(NaClBitCodeAbbrevOp(2));
-  Abbrevs.push_back(Abbrev);
-  // [1, 0]
-  Abbrev = new NaClBitCodeAbbrev();
-  Abbrev->Add(NaClBitCodeAbbrevOp(1));
-  Abbrev->Add(NaClBitCodeAbbrevOp(0));
-  Abbrevs.push_back(Abbrev);
-  // [VBR(6), VBR(6)]
-  Abbrev = new NaClBitCodeAbbrev();
-  Abbrev->Add(NaClBitCodeAbbrevOp(NaClBitCodeAbbrevOp::VBR, 6));
-  Abbrev->Add(NaClBitCodeAbbrevOp(NaClBitCodeAbbrevOp::VBR, 6));
-  Abbrevs.push_back(Abbrev);
-  // [VBR(6), 0]
-  Abbrev = new NaClBitCodeAbbrev();
-  Abbrev->Add(NaClBitCodeAbbrevOp(NaClBitCodeAbbrevOp::VBR, 6));
-  Abbrev->Add(NaClBitCodeAbbrevOp(0));
-  Abbrevs.push_back(Abbrev);
-
-  // Verify we built the expected abbreviations.
-  EXPECT_EQ(std::string(
-      "[1, VBR(6)]\n"
-      "[4, VBR(8)]\n"
-      "[4, 0]\n"
-      "[1, 2]\n"
-      "[1, 0]\n"
-      "[VBR(6), VBR(6)]\n"
-      "[VBR(6), 0]\n"),
-            DescribeAbbreviations(Abbrevs));
-
-  // Build lookup map, and check that we build the expected trie.
-  AbbrevLookupSizeMap LookupMap;
-  NaClBuildAbbrevLookupMap(LookupMap, Abbrevs);
-  EXPECT_EQ((size_t)1, LookupMap.size())
-      << "There should only be one entry in the Lookup map "
-      << "for abbreviations of length 2";
-  for (AbbrevLookupSizeMap::iterator
-           Iter = LookupMap.begin(), IterEnd = LookupMap.end();
-       Iter != IterEnd; ++Iter) {
-    EXPECT_EQ(Iter->first, (size_t)2)
-        << "Expecting abbreviations to be of length 2";
-    EXPECT_EQ(std::string(
-        "Abbreviations:\n"
-        "  [VBR(6), VBR(6)] (abbrev #5)\n"
-        "Successor Map:\n"
-        "  Record.Code = 1\n"
-        "    Abbreviations:\n"
-        "      [1, VBR(6)] (abbrev #0)\n"
-        "      [VBR(6), VBR(6)] (abbrev #5)\n"
-        "    Successor Map:\n"
-        "      Record.Values[0] = 0\n"
-        "        Abbreviations:\n"
-        "          [1, VBR(6)] (abbrev #0)\n"
-        "          [1, 0] (abbrev #4)\n"
-        "          [VBR(6), VBR(6)] (abbrev #5)\n"
-        "          [VBR(6), 0] (abbrev #6)\n"
-        "      Record.Values[0] = 2\n"
-        "        Abbreviations:\n"
-        "          [1, VBR(6)] (abbrev #0)\n"
-        "          [1, 2] (abbrev #3)\n"
-        "          [VBR(6), VBR(6)] (abbrev #5)\n"
-        "  Record.Code = 4\n"
-        "    Abbreviations:\n"
-        "      [4, VBR(8)] (abbrev #1)\n"
-        "      [VBR(6), VBR(6)] (abbrev #5)\n"
-        "    Successor Map:\n"
-        "      Record.Values[0] = 0\n"
-        "        Abbreviations:\n"
-        "          [4, VBR(8)] (abbrev #1)\n"
-        "          [4, 0] (abbrev #2)\n"
-        "          [VBR(6), VBR(6)] (abbrev #5)\n"
-        "          [VBR(6), 0] (abbrev #6)\n"
-        "  Record.Values[0] = 0\n"
-        "    Abbreviations:\n"
-        "      [VBR(6), VBR(6)] (abbrev #5)\n"
-        "      [VBR(6), 0] (abbrev #6)\n"),
-              DescribeAbbrevTrie(Iter->second));
-  }
-
-  // Test matching [1, 0].
-  NaClBitcodeRecordData Record;
-  Record.Code = 1;
-  Record.Values.push_back(0);
-  EXPECT_EQ(std::string("[1, 0]"), DescribeRecord(Record));
-  {
-    AbbrevTrieNode *Node = LookupMap[Record.Values.size()+1];
-    ASSERT_NE(Node, (void*) 0);
-    EXPECT_EQ(std::string(
-        "Abbreviations:\n"
-        "  [1, VBR(6)] (abbrev #0)\n"
-        "  [1, 0] (abbrev #4)\n"
-        "  [VBR(6), VBR(6)] (abbrev #5)\n"
-        "  [VBR(6), 0] (abbrev #6)\n"),
-              DescribeAbbrevTrieNode(Node->MatchRecord(Record)));
-  }
-
-  // Test matching [1, 2]
-  Record.Values[0] = 2;
-  EXPECT_EQ(std::string("[1, 2]"), DescribeRecord(Record));
-  {
-    AbbrevTrieNode *Node = LookupMap[Record.Values.size()+1];
-    ASSERT_NE(Node, (void*) 0);
-    EXPECT_EQ(std::string(
-        "Abbreviations:\n"
-        "  [1, VBR(6)] (abbrev #0)\n"
-        "  [1, 2] (abbrev #3)\n"
-        "  [VBR(6), VBR(6)] (abbrev #5)\n"),
-              DescribeAbbrevTrieNode(Node->MatchRecord(Record)));
-  }
-
-  // Test match [1, 8] (i.e. Record.Values[1] ~in {0, 2}).
-  Record.Values[0] = 8;
-  EXPECT_EQ(std::string("[1, 8]"), DescribeRecord(Record));
-  {
-    AbbrevTrieNode *Node = LookupMap[Record.Values.size()+1];
-    ASSERT_NE(Node, (void*) 0);
-    EXPECT_EQ(std::string(
-        "Abbreviations:\n"
-        "  [1, VBR(6)] (abbrev #0)\n"
-        "  [VBR(6), VBR(6)] (abbrev #5)\n"),
-              DescribeAbbrevTrieNode(Node->MatchRecord(Record)));
-  }
-
-  // Test match [4, 0]
-  Record.Code = 4;
-  Record.Values[0] = 0;
-  EXPECT_EQ(std::string("[4, 0]"), DescribeRecord(Record));
-  {
-    AbbrevTrieNode *Node = LookupMap[Record.Values.size()+1];
-    ASSERT_NE(Node, (void*) 0);
-    EXPECT_EQ(std::string(
-        "Abbreviations:\n"
-        "  [4, VBR(8)] (abbrev #1)\n"
-        "  [4, 0] (abbrev #2)\n"
-        "  [VBR(6), VBR(6)] (abbrev #5)\n"
-        "  [VBR(6), 0] (abbrev #6)\n"),
-              DescribeAbbrevTrieNode(Node->MatchRecord(Record)));
-  }
-
-  // Test match [4, 8] (i.e. Record.Values[1] ~in {0})
-  Record.Values[0] = 8;
-  EXPECT_EQ(std::string("[4, 8]"), DescribeRecord(Record));
-  {
-    AbbrevTrieNode *Node = LookupMap[Record.Values.size()+1];
-    ASSERT_NE(Node, (void*) 0);
-    EXPECT_EQ(std::string(
-        "Abbreviations:\n"
-        "  [4, VBR(8)] (abbrev #1)\n"
-        "  [VBR(6), VBR(6)] (abbrev #5)\n"),
-              DescribeAbbrevTrieNode(Node->MatchRecord(Record)));
-  }
-
-  // Test match [8, 0] (i.e. Record.Code ~in {1, 4})
-  Record.Code = 8;
-  Record.Values[0] = 0;
-  EXPECT_EQ(std::string("[8, 0]"), DescribeRecord(Record));
-  {
-    AbbrevTrieNode *Node = LookupMap[Record.Values.size()+1];
-    ASSERT_NE(Node, (void*) 0);
-    EXPECT_EQ(std::string(
-        "Abbreviations:\n"
-        "  [VBR(6), VBR(6)] (abbrev #5)\n"
-        "  [VBR(6), 0] (abbrev #6)\n"),
-              DescribeAbbrevTrieNode(Node->MatchRecord(Record)));
-  }
-
-  // Test match [7, 6] (i.e. Record.Code ~in {1, 4}
-  //                    and  Record.Values[0] ~in {0})
-  Record.Code = 7;
-  Record.Values[0] = 6;
-  EXPECT_EQ(std::string("[7, 6]"), DescribeRecord(Record));
-  {
-    AbbrevTrieNode *Node = LookupMap[Record.Values.size()+1];
-    ASSERT_NE(Node, (void*) 0);
-    EXPECT_EQ(std::string(
-        "Abbreviations:\n"
-        "  [VBR(6), VBR(6)] (abbrev #5)\n"),
-              DescribeAbbrevTrieNode(Node->MatchRecord(Record)));
-  }
-
-  // Test match [1, 2, 3] (i.e no abbreviations defined).
-  Record.Code = 1;
-  Record.Values[0] = 2;
-  Record.Values.push_back(3);
-  EXPECT_EQ(std::string("[1, 2, 3]"), DescribeRecord(Record));
-  {
-    AbbrevTrieNode *Node = LookupMap[Record.Values.size()+1];
-    ASSERT_EQ(Node, (void*) 0);
-  }
-
-  Clear(Abbrevs);
-  Clear(LookupMap);
-}
-
-TEST(NaClAbbrevTrieTest, Array) {
-  // Test for variable length abbreviations, with some specific
-  // unwindings.
-  AbbrevVector Abbrevs;
-  // [Array(VBR(6))]
-  NaClBitCodeAbbrev *Abbrev = new NaClBitCodeAbbrev();
-  Abbrev->Add(NaClBitCodeAbbrevOp(NaClBitCodeAbbrevOp::Array));
-  Abbrev->Add(NaClBitCodeAbbrevOp(NaClBitCodeAbbrevOp::VBR, 6));
-  Abbrevs.push_back(Abbrev);
-  // [VBR(6), VBR(6), 0, VBR(6), VBR(6)]
-  Abbrev = new NaClBitCodeAbbrev();
-  Abbrev->Add(NaClBitCodeAbbrevOp(NaClBitCodeAbbrevOp::VBR, 6));
-  Abbrev->Add(NaClBitCodeAbbrevOp(NaClBitCodeAbbrevOp::VBR, 6));
-  Abbrev->Add(NaClBitCodeAbbrevOp(0));
-  Abbrev->Add(NaClBitCodeAbbrevOp(NaClBitCodeAbbrevOp::VBR, 6));
-  Abbrev->Add(NaClBitCodeAbbrevOp(NaClBitCodeAbbrevOp::VBR, 6));
-  Abbrevs.push_back(Abbrev);
-  // [8, VBR(6), VBR(6), VBR(6), VBR(6)]
-  Abbrev = new NaClBitCodeAbbrev();
-  Abbrev->Add(NaClBitCodeAbbrevOp(8));
-  Abbrev->Add(NaClBitCodeAbbrevOp(NaClBitCodeAbbrevOp::VBR, 6));
-  Abbrev->Add(NaClBitCodeAbbrevOp(NaClBitCodeAbbrevOp::VBR, 6));
-  Abbrev->Add(NaClBitCodeAbbrevOp(NaClBitCodeAbbrevOp::VBR, 6));
-  Abbrev->Add(NaClBitCodeAbbrevOp(NaClBitCodeAbbrevOp::VBR, 6));
-  Abbrevs.push_back(Abbrev);
-  // [VBR(6), VBR(6), VBR(6), 0, VBR(6)]
-  Abbrev = new NaClBitCodeAbbrev();
-  Abbrev->Add(NaClBitCodeAbbrevOp(NaClBitCodeAbbrevOp::VBR, 6));
-  Abbrev->Add(NaClBitCodeAbbrevOp(NaClBitCodeAbbrevOp::VBR, 6));
-  Abbrev->Add(NaClBitCodeAbbrevOp(NaClBitCodeAbbrevOp::VBR, 6));
-  Abbrev->Add(NaClBitCodeAbbrevOp(0));
-  Abbrev->Add(NaClBitCodeAbbrevOp(NaClBitCodeAbbrevOp::VBR, 6));
-  Abbrevs.push_back(Abbrev);
-  // [VBR(6), VBR(6), VBR(6), VBR(6), 3]
-  Abbrev = new NaClBitCodeAbbrev();
-  Abbrev->Add(NaClBitCodeAbbrevOp(NaClBitCodeAbbrevOp::VBR, 6));
-  Abbrev->Add(NaClBitCodeAbbrevOp(NaClBitCodeAbbrevOp::VBR, 6));
-  Abbrev->Add(NaClBitCodeAbbrevOp(NaClBitCodeAbbrevOp::VBR, 6));
-  Abbrev->Add(NaClBitCodeAbbrevOp(NaClBitCodeAbbrevOp::VBR, 6));
-  Abbrev->Add(NaClBitCodeAbbrevOp(3));
-  Abbrevs.push_back(Abbrev);
-
-  // Verify we built the expected abbreviations.
-  EXPECT_EQ(std::string(
-      "[Array(VBR(6))]\n"
-      "[VBR(6), VBR(6), 0, VBR(6), VBR(6)]\n"
-      "[8, VBR(6), VBR(6), VBR(6), VBR(6)]\n"
-      "[VBR(6), VBR(6), VBR(6), 0, VBR(6)]\n"
-      "[VBR(6), VBR(6), VBR(6), VBR(6), 3]\n"),
-            DescribeAbbreviations(Abbrevs));
-
-  // Build lookup map, and check that we build the expected trie.
-  AbbrevLookupSizeMap LookupMap;
-  NaClBuildAbbrevLookupMap(LookupMap, Abbrevs);
-  EXPECT_EQ(MaxValueIndex+1, LookupMap.size());
-  for (AbbrevLookupSizeMap::iterator
-           Iter = LookupMap.begin(), IterEnd = LookupMap.end();
-       Iter != IterEnd; ++Iter) {
-    EXPECT_LE((size_t)0, Iter->first);
-    EXPECT_GE(MaxValueIndex, Iter->first);
-    if (Iter->first == 5) {
-      // Note that all abbreviations accept records with 5 values.
-      EXPECT_EQ(std::string(
-        "Abbreviations:\n"
-        "  [Array(VBR(6))] (abbrev #0)\n"
-        "Successor Map:\n"
-        "  Record.Code = 8\n"
-        "    Abbreviations:\n"
-        "      [Array(VBR(6))] (abbrev #0)\n"
-        "      [8, VBR(6), VBR(6), VBR(6), VBR(6)] (abbrev #2)\n"
-        "    Successor Map:\n"
-        "      Record.Values[1] = 0\n"
-        "        Abbreviations:\n"
-        "          [Array(VBR(6))] (abbrev #0)\n"
-        "          [VBR(6), VBR(6), 0, VBR(6), VBR(6)] (abbrev #1)\n"
-        "          [8, VBR(6), VBR(6), VBR(6), VBR(6)] (abbrev #2)\n"
-        "        Successor Map:\n"
-        "          Record.Values[2] = 0\n"
-        "            Abbreviations:\n"
-        "              [Array(VBR(6))] (abbrev #0)\n"
-        "              [VBR(6), VBR(6), 0, VBR(6), VBR(6)] (abbrev #1)\n"
-        "              [8, VBR(6), VBR(6), VBR(6), VBR(6)] (abbrev #2)\n"
-        "              [VBR(6), VBR(6), VBR(6), 0, VBR(6)] (abbrev #3)\n"
-        "            Successor Map:\n"
-        "              Record.Values[3] = 3\n"
-        "                Abbreviations:\n"
-        "                  [Array(VBR(6))] (abbrev #0)\n"
-        "                  [VBR(6), VBR(6), 0, VBR(6), VBR(6)] (abbrev #1)\n"
-        "                  [8, VBR(6), VBR(6), VBR(6), VBR(6)] (abbrev #2)\n"
-        "                  [VBR(6), VBR(6), VBR(6), 0, VBR(6)] (abbrev #3)\n"
-        "                  [VBR(6), VBR(6), VBR(6), VBR(6), 3] (abbrev #4)\n"
-        "          Record.Values[3] = 3\n"
-        "            Abbreviations:\n"
-        "              [Array(VBR(6))] (abbrev #0)\n"
-        "              [VBR(6), VBR(6), 0, VBR(6), VBR(6)] (abbrev #1)\n"
-        "              [8, VBR(6), VBR(6), VBR(6), VBR(6)] (abbrev #2)\n"
-        "              [VBR(6), VBR(6), VBR(6), VBR(6), 3] (abbrev #4)\n"
-        "      Record.Values[2] = 0\n"
-        "        Abbreviations:\n"
-        "          [Array(VBR(6))] (abbrev #0)\n"
-        "          [8, VBR(6), VBR(6), VBR(6), VBR(6)] (abbrev #2)\n"
-        "          [VBR(6), VBR(6), VBR(6), 0, VBR(6)] (abbrev #3)\n"
-        "        Successor Map:\n"
-        "          Record.Values[3] = 3\n"
-        "            Abbreviations:\n"
-        "              [Array(VBR(6))] (abbrev #0)\n"
-        "              [8, VBR(6), VBR(6), VBR(6), VBR(6)] (abbrev #2)\n"
-        "              [VBR(6), VBR(6), VBR(6), 0, VBR(6)] (abbrev #3)\n"
-        "              [VBR(6), VBR(6), VBR(6), VBR(6), 3] (abbrev #4)\n"
-        "      Record.Values[3] = 3\n"
-        "        Abbreviations:\n"
-        "          [Array(VBR(6))] (abbrev #0)\n"
-        "          [8, VBR(6), VBR(6), VBR(6), VBR(6)] (abbrev #2)\n"
-        "          [VBR(6), VBR(6), VBR(6), VBR(6), 3] (abbrev #4)\n"
-        "  Record.Values[1] = 0\n"
-        "    Abbreviations:\n"
-        "      [Array(VBR(6))] (abbrev #0)\n"
-        "      [VBR(6), VBR(6), 0, VBR(6), VBR(6)] (abbrev #1)\n"
-        "    Successor Map:\n"
-        "      Record.Values[2] = 0\n"
-        "        Abbreviations:\n"
-        "          [Array(VBR(6))] (abbrev #0)\n"
-        "          [VBR(6), VBR(6), 0, VBR(6), VBR(6)] (abbrev #1)\n"
-        "          [VBR(6), VBR(6), VBR(6), 0, VBR(6)] (abbrev #3)\n"
-        "        Successor Map:\n"
-        "          Record.Values[3] = 3\n"
-        "            Abbreviations:\n"
-        "              [Array(VBR(6))] (abbrev #0)\n"
-        "              [VBR(6), VBR(6), 0, VBR(6), VBR(6)] (abbrev #1)\n"
-        "              [VBR(6), VBR(6), VBR(6), 0, VBR(6)] (abbrev #3)\n"
-        "              [VBR(6), VBR(6), VBR(6), VBR(6), 3] (abbrev #4)\n"
-        "      Record.Values[3] = 3\n"
-        "        Abbreviations:\n"
-        "          [Array(VBR(6))] (abbrev #0)\n"
-        "          [VBR(6), VBR(6), 0, VBR(6), VBR(6)] (abbrev #1)\n"
-        "          [VBR(6), VBR(6), VBR(6), VBR(6), 3] (abbrev #4)\n"
-        "  Record.Values[2] = 0\n"
-        "    Abbreviations:\n"
-        "      [Array(VBR(6))] (abbrev #0)\n"
-        "      [VBR(6), VBR(6), VBR(6), 0, VBR(6)] (abbrev #3)\n"
-        "    Successor Map:\n"
-        "      Record.Values[3] = 3\n"
-        "        Abbreviations:\n"
-        "          [Array(VBR(6))] (abbrev #0)\n"
-        "          [VBR(6), VBR(6), VBR(6), 0, VBR(6)] (abbrev #3)\n"
-        "          [VBR(6), VBR(6), VBR(6), VBR(6), 3] (abbrev #4)\n"
-        "  Record.Values[3] = 3\n"
-        "    Abbreviations:\n"
-        "      [Array(VBR(6))] (abbrev #0)\n"
-        "      [VBR(6), VBR(6), VBR(6), VBR(6), 3] (abbrev #4)\n"),
-                DescribeAbbrevTrie(Iter->second));
-    } else {
-      // When the record doesn't contain 5 values, only
-      // abbreviation [Array(VBR(6))] applies.
-      EXPECT_EQ(std::string(
-        "Abbreviations:\n"
-        "  [Array(VBR(6))] (abbrev #0)\n"),
-                DescribeAbbrevTrie(Iter->second));
-    }
-  }
-
-  // Test matching [8, 10, 0, 0, 3].
-  NaClBitcodeRecordData Record;
-  Record.Code = 8;
-  Record.Values.push_back(10);
-  Record.Values.push_back(0);
-  Record.Values.push_back(0);
-  Record.Values.push_back(3);
-  EXPECT_EQ(std::string("[8, 10, 0, 0, 3]"), DescribeRecord(Record));
-  {
-    AbbrevTrieNode *Node = LookupMap[Record.Values.size()+1];
-    ASSERT_NE(Node, (void*) 0);
-    EXPECT_EQ(std::string(
-        "Abbreviations:\n"
-        "  [Array(VBR(6))] (abbrev #0)\n"
-        "  [VBR(6), VBR(6), 0, VBR(6), VBR(6)] (abbrev #1)\n"
-        "  [8, VBR(6), VBR(6), VBR(6), VBR(6)] (abbrev #2)\n"
-        "  [VBR(6), VBR(6), VBR(6), 0, VBR(6)] (abbrev #3)\n"
-        "  [VBR(6), VBR(6), VBR(6), VBR(6), 3] (abbrev #4)\n"),
-              DescribeAbbrevTrieNode(Node->MatchRecord(Record)));
-  }
-
-  // Test matching [8, 10, 0, 11, 3].
-  Record.Values[2] = 11;
-  EXPECT_EQ(std::string("[8, 10, 0, 11, 3]"), DescribeRecord(Record));
-  {
-    AbbrevTrieNode *Node = LookupMap[Record.Values.size()+1];
-    ASSERT_NE(Node, (void*) 0);
-    EXPECT_EQ(std::string(
-        "Abbreviations:\n"
-        "  [Array(VBR(6))] (abbrev #0)\n"
-        "  [VBR(6), VBR(6), 0, VBR(6), VBR(6)] (abbrev #1)\n"
-        "  [8, VBR(6), VBR(6), VBR(6), VBR(6)] (abbrev #2)\n"
-        "  [VBR(6), VBR(6), VBR(6), VBR(6), 3] (abbrev #4)\n"),
-              DescribeAbbrevTrieNode(Node->MatchRecord(Record)));
-  }
-
-  // Test matching [8, 10, 0, 11, 12].
-  Record.Values[3] = 12;
-  EXPECT_EQ(std::string("[8, 10, 0, 11, 12]"), DescribeRecord(Record));
-  {
-    AbbrevTrieNode *Node = LookupMap[Record.Values.size()+1];
-    ASSERT_NE(Node, (void*) 0);
-    EXPECT_EQ(std::string(
-        "Abbreviations:\n"
-        "  [Array(VBR(6))] (abbrev #0)\n"
-        "  [VBR(6), VBR(6), 0, VBR(6), VBR(6)] (abbrev #1)\n"
-        "  [8, VBR(6), VBR(6), VBR(6), VBR(6)] (abbrev #2)\n"),
-              DescribeAbbrevTrieNode(Node->MatchRecord(Record)));
-  }
-
-  // Test matching [13, 10, 0, 0, 3].
-  Record.Code = 13;
-  Record.Values[2] = 0;
-  Record.Values[3] = 3;
-  EXPECT_EQ(std::string("[13, 10, 0, 0, 3]"), DescribeRecord(Record));
-  {
-    AbbrevTrieNode *Node = LookupMap[Record.Values.size()+1];
-    ASSERT_NE(Node, (void*) 0);
-    EXPECT_EQ(std::string(
-        "Abbreviations:\n"
-        "  [Array(VBR(6))] (abbrev #0)\n"
-        "  [VBR(6), VBR(6), 0, VBR(6), VBR(6)] (abbrev #1)\n"
-        "  [VBR(6), VBR(6), VBR(6), 0, VBR(6)] (abbrev #3)\n"
-        "  [VBR(6), VBR(6), VBR(6), VBR(6), 3] (abbrev #4)\n"),
-              DescribeAbbrevTrieNode(Node->MatchRecord(Record)));
-  }
-
-  // Test matching [13, 10, 0, 0, 14].
-  Record.Values[3] = 14;
-  EXPECT_EQ(std::string("[13, 10, 0, 0, 14]"), DescribeRecord(Record));
-  {
-    AbbrevTrieNode *Node = LookupMap[Record.Values.size()+1];
-    ASSERT_NE(Node, (void*) 0);
-    EXPECT_EQ(std::string(
-        "Abbreviations:\n"
-        "  [Array(VBR(6))] (abbrev #0)\n"
-        "  [VBR(6), VBR(6), 0, VBR(6), VBR(6)] (abbrev #1)\n"
-        "  [VBR(6), VBR(6), VBR(6), 0, VBR(6)] (abbrev #3)\n"),
-              DescribeAbbrevTrieNode(Node->MatchRecord(Record)));
-  }
-
-  // Test matching [13, 10, 0, 15, 3].
-  Record.Values[2] = 15;
-  Record.Values[3] = 3;
-  EXPECT_EQ(std::string("[13, 10, 0, 15, 3]"), DescribeRecord(Record));
-  {
-    AbbrevTrieNode *Node = LookupMap[Record.Values.size()+1];
-    ASSERT_NE(Node, (void*) 0);
-    EXPECT_EQ(std::string(
-        "Abbreviations:\n"
-        "  [Array(VBR(6))] (abbrev #0)\n"
-        "  [VBR(6), VBR(6), 0, VBR(6), VBR(6)] (abbrev #1)\n"
-        "  [VBR(6), VBR(6), VBR(6), VBR(6), 3] (abbrev #4)\n"),
-              DescribeAbbrevTrieNode(Node->MatchRecord(Record)));
-  }
-
-  // Test matching [13, 10, 0, 15, 14].
-  Record.Values[3] = 14;
-  EXPECT_EQ(std::string("[13, 10, 0, 15, 14]"), DescribeRecord(Record));
-  {
-    AbbrevTrieNode *Node = LookupMap[Record.Values.size()+1];
-    ASSERT_NE(Node, (void*) 0);
-    EXPECT_EQ(std::string(
-        "Abbreviations:\n"
-        "  [Array(VBR(6))] (abbrev #0)\n"
-        "  [VBR(6), VBR(6), 0, VBR(6), VBR(6)] (abbrev #1)\n"),
-              DescribeAbbrevTrieNode(Node->MatchRecord(Record)));
-  }
-
-  // Test matching [13, 10, 16, 0, 3].
-  Record.Values[1] = 16;
-  Record.Values[2] = 0;
-  Record.Values[3] = 3;
-  EXPECT_EQ(std::string("[13, 10, 16, 0, 3]"), DescribeRecord(Record));
-  {
-    AbbrevTrieNode *Node = LookupMap[Record.Values.size()+1];
-    ASSERT_NE(Node, (void*) 0);
-    EXPECT_EQ(std::string(
-        "Abbreviations:\n"
-        "  [Array(VBR(6))] (abbrev #0)\n"
-        "  [VBR(6), VBR(6), VBR(6), 0, VBR(6)] (abbrev #3)\n"
-        "  [VBR(6), VBR(6), VBR(6), VBR(6), 3] (abbrev #4)\n"),
-              DescribeAbbrevTrieNode(Node->MatchRecord(Record)));
-  }
-
-  // Test matching [13, 10, 16, 0, 17].
-  Record.Values[3] = 17;
-  EXPECT_EQ(std::string("[13, 10, 16, 0, 17]"), DescribeRecord(Record));
-  {
-    AbbrevTrieNode *Node = LookupMap[Record.Values.size()+1];
-    ASSERT_NE(Node, (void*) 0);
-    EXPECT_EQ(std::string(
-        "Abbreviations:\n"
-        "  [Array(VBR(6))] (abbrev #0)\n"
-        "  [VBR(6), VBR(6), VBR(6), 0, VBR(6)] (abbrev #3)\n"),
-              DescribeAbbrevTrieNode(Node->MatchRecord(Record)));
-  }
-
-  // Test matching [13, 10, 16, 18, 3].
-  Record.Values[2] = 18;
-  Record.Values[3] = 3;
-  EXPECT_EQ(std::string("[13, 10, 16, 18, 3]"), DescribeRecord(Record));
-  {
-    AbbrevTrieNode *Node = LookupMap[Record.Values.size()+1];
-    ASSERT_NE(Node, (void*) 0);
-    EXPECT_EQ(std::string(
-        "Abbreviations:\n"
-        "  [Array(VBR(6))] (abbrev #0)\n"
-        "  [VBR(6), VBR(6), VBR(6), VBR(6), 3] (abbrev #4)\n"),
-              DescribeAbbrevTrieNode(Node->MatchRecord(Record)));
-  }
-
-  // Test matching [13, 10, 16, 18, 19].
-  Record.Values[3] = 19;
-  EXPECT_EQ(std::string("[13, 10, 16, 18, 19]"), DescribeRecord(Record));
-  {
-    AbbrevTrieNode *Node = LookupMap[Record.Values.size()+1];
-    ASSERT_NE(Node, (void*) 0);
-    EXPECT_EQ(std::string(
-        "Abbreviations:\n"
-        "  [Array(VBR(6))] (abbrev #0)\n"),
-              DescribeAbbrevTrieNode(Node->MatchRecord(Record)));
-  }
-
-  // Test matching [13, 10, 16, 18, 19, 20, 21, 22, 23, 24, 25]
-  Record.Values.push_back(20);
-  Record.Values.push_back(21);
-  Record.Values.push_back(22);
-  Record.Values.push_back(23);
-  Record.Values.push_back(24);
-  Record.Values.push_back(25);
-  EXPECT_EQ(std::string("[13, 10, 16, 18, 19, 20, 21, 22, 23, 24, 25]"),
-            DescribeRecord(Record));
-  {
-    AbbrevTrieNode *Node = LookupMap[Record.Values.size()+1];
-    ASSERT_EQ(Node, (void*) 0);
-  }
-
-  Clear(Abbrevs);
-  Clear(LookupMap);
-}
-
-TEST(NaClAbbrevTrieTest, NonsimpleArray) {
-  // Test case where Array doesn't appear first.
-  AbbrevVector Abbrevs;
-  // [Fixed(3), VBR(8), Array(Fixed(8))]
-  NaClBitCodeAbbrev *Abbrev = new NaClBitCodeAbbrev();
-  Abbrev->Add(NaClBitCodeAbbrevOp(NaClBitCodeAbbrevOp::Fixed, 3));
-  Abbrev->Add(NaClBitCodeAbbrevOp(NaClBitCodeAbbrevOp::VBR, 8));
-  Abbrev->Add(NaClBitCodeAbbrevOp(NaClBitCodeAbbrevOp::Array));
-  Abbrev->Add(NaClBitCodeAbbrevOp(NaClBitCodeAbbrevOp::Fixed, 8));
-  Abbrevs.push_back(Abbrev);
-  // [1, VBR(8), Array(Fixed(7))]
-  Abbrev = new NaClBitCodeAbbrev();
-  Abbrev->Add(NaClBitCodeAbbrevOp(1));
-  Abbrev->Add(NaClBitCodeAbbrevOp(NaClBitCodeAbbrevOp::VBR, 8));
-  Abbrev->Add(NaClBitCodeAbbrevOp(NaClBitCodeAbbrevOp::Array));
-  Abbrev->Add(NaClBitCodeAbbrevOp(NaClBitCodeAbbrevOp::Fixed, 7));
-  Abbrevs.push_back(Abbrev);
-  // [1, VBR(8), Array(Char6)]
-  Abbrev = new NaClBitCodeAbbrev();
-  Abbrev->Add(NaClBitCodeAbbrevOp(1));
-  Abbrev->Add(NaClBitCodeAbbrevOp(NaClBitCodeAbbrevOp::VBR, 8));
-  Abbrev->Add(NaClBitCodeAbbrevOp(NaClBitCodeAbbrevOp::Array));
-  Abbrev->Add(NaClBitCodeAbbrevOp(NaClBitCodeAbbrevOp::Char6));
-  Abbrevs.push_back(Abbrev);
-  // [2, VBR(8), Array(Char6)]
-  Abbrev = new NaClBitCodeAbbrev();
-  Abbrev->Add(NaClBitCodeAbbrevOp(2));
-  Abbrev->Add(NaClBitCodeAbbrevOp(NaClBitCodeAbbrevOp::VBR, 8));
-  Abbrev->Add(NaClBitCodeAbbrevOp(NaClBitCodeAbbrevOp::Array));
-  Abbrev->Add(NaClBitCodeAbbrevOp(NaClBitCodeAbbrevOp::Char6));
-  Abbrevs.push_back(Abbrev);
-  // [2, Array(VBR(8))]
-  Abbrev = new NaClBitCodeAbbrev();
-  Abbrev->Add(NaClBitCodeAbbrevOp(2));
-  Abbrev->Add(NaClBitCodeAbbrevOp(NaClBitCodeAbbrevOp::Array));
-  Abbrev->Add(NaClBitCodeAbbrevOp(NaClBitCodeAbbrevOp::VBR, 8));
-  Abbrevs.push_back(Abbrev);
-  // [Fixed(3), VBR(8), 5, Array(Fixed(8))]
-  Abbrev = new NaClBitCodeAbbrev();
-  Abbrev->Add(NaClBitCodeAbbrevOp(NaClBitCodeAbbrevOp::Fixed, 3));
-  Abbrev->Add(NaClBitCodeAbbrevOp(NaClBitCodeAbbrevOp::VBR, 8));
-  Abbrev->Add(NaClBitCodeAbbrevOp(5));
-  Abbrev->Add(NaClBitCodeAbbrevOp(NaClBitCodeAbbrevOp::Array));
-  Abbrev->Add(NaClBitCodeAbbrevOp(NaClBitCodeAbbrevOp::Fixed, 8));
-  Abbrevs.push_back(Abbrev);
-
-  // Verify we built the expected abbreviations.
-  EXPECT_EQ(std::string(
-        "[Fixed(3), VBR(8), Array(Fixed(8))]\n"
-        "[1, VBR(8), Array(Fixed(7))]\n"
-        "[1, VBR(8), Array(Char6)]\n"
-        "[2, VBR(8), Array(Char6)]\n"
-        "[2, Array(VBR(8))]\n"
-        "[Fixed(3), VBR(8), 5, Array(Fixed(8))]\n"),
-            DescribeAbbreviations(Abbrevs));
-
-  // Build lookup map, and check that we build the expected trie.
-  AbbrevLookupSizeMap LookupMap;
-  NaClBuildAbbrevLookupMap(LookupMap, Abbrevs);
-  // Note: Above abbreviations accept all record lengths but 0. Hence,
-  // there should be one for each possible (truncated) record length
-  // except zero.
-  EXPECT_EQ(MaxValueIndex, LookupMap.size())
-      << "Should accept all (truncated) record lengths (except 0)";
-  for (AbbrevLookupSizeMap::iterator
-           Iter = LookupMap.begin(), IterEnd = LookupMap.end();
-       Iter != IterEnd; ++Iter) {
-    NaClBitcodeRecordData Record;
-    switch (Iter->first) {
-    case 0:
-      ASSERT_FALSE(true) << "There are not abbreviations of length 0";
-      break;
-    case 1:
-      EXPECT_EQ(std::string(
-        "Successor Map:\n"
-        "  Record.Code = 2\n"
-        "    Abbreviations:\n"
-        "      [2, Array(VBR(8))] (abbrev #4)\n"),
-                DescribeAbbrevTrie(Iter->second));
-
-      // Test matching [2]
-      Record.Code = 2;
-      EXPECT_EQ(std::string("[2]"), DescribeRecord(Record));
-      {
-        AbbrevTrieNode *Node = LookupMap[Record.Values.size()+1];
-        ASSERT_NE(Node, (void*) 0);
-        EXPECT_EQ(std::string(
-        "Abbreviations:\n"
-        "  [2, Array(VBR(8))] (abbrev #4)\n"),
-              DescribeAbbrevTrieNode(Node->MatchRecord(Record)));
-      }
-
-      // Test matching [5];
-      Record.Code = 5;
-      EXPECT_EQ(std::string("[5]"), DescribeRecord(Record));
-      {
-        AbbrevTrieNode *Node = LookupMap[Record.Values.size()+1];
-        ASSERT_NE(Node, (void*) 0);
-        EXPECT_EQ(std::string(""),
-              DescribeAbbrevTrieNode(Node->MatchRecord(Record)));
-      }
-      // Test matching [2, 10]
-      Record.Code = 2;
-      Record.Values.push_back(10);
-      EXPECT_EQ(std::string("[2, 10]"), DescribeRecord(Record));
-      {
-        AbbrevTrieNode *Node = LookupMap[Record.Values.size()+1];
-        ASSERT_NE(Node, (void*) 0);
-        EXPECT_EQ(std::string(
-        "Abbreviations:\n"
-        "  [Fixed(3), VBR(8), Array(Fixed(8))] (abbrev #0)\n"
-        "  [2, VBR(8), Array(Char6)] (abbrev #3)\n"
-        "  [2, Array(VBR(8))] (abbrev #4)\n"),
-              DescribeAbbrevTrieNode(Node->MatchRecord(Record)));
-      }
-      break;
-    case 2:
-      EXPECT_EQ(std::string(
-        "Abbreviations:\n"
-        "  [Fixed(3), VBR(8), Array(Fixed(8))] (abbrev #0)\n"
-        "Successor Map:\n"
-        "  Record.Code = 1\n"
-        "    Abbreviations:\n"
-        "      [Fixed(3), VBR(8), Array(Fixed(8))] (abbrev #0)\n"
-        "      [1, VBR(8), Array(Fixed(7))] (abbrev #1)\n"
-        "      [1, VBR(8), Array(Char6)] (abbrev #2)\n"
-        "  Record.Code = 2\n"
-        "    Abbreviations:\n"
-        "      [Fixed(3), VBR(8), Array(Fixed(8))] (abbrev #0)\n"
-        "      [2, VBR(8), Array(Char6)] (abbrev #3)\n"
-        "      [2, Array(VBR(8))] (abbrev #4)\n"),
-                DescribeAbbrevTrie(Iter->second));
-
-      // Test matching [1, 5]
-      Record.Code = 1;
-      Record.Values.push_back(5);
-      EXPECT_EQ(std::string("[1, 5]"), DescribeRecord(Record));
-      {
-        AbbrevTrieNode *Node = LookupMap[Record.Values.size()+1];
-        ASSERT_NE(Node, (void*) 0);
-        EXPECT_EQ(std::string(
-        "Abbreviations:\n"
-        "  [Fixed(3), VBR(8), Array(Fixed(8))] (abbrev #0)\n"
-        "  [1, VBR(8), Array(Fixed(7))] (abbrev #1)\n"
-        "  [1, VBR(8), Array(Char6)] (abbrev #2)\n"),
-              DescribeAbbrevTrieNode(Node->MatchRecord(Record)));
-      }
-      // Test matching [2, 5]
-      Record.Code = 2;
-      EXPECT_EQ(std::string("[2, 5]"), DescribeRecord(Record));
-      {
-        AbbrevTrieNode *Node = LookupMap[Record.Values.size()+1];
-        ASSERT_NE(Node, (void*) 0);
-        EXPECT_EQ(std::string(
-        "Abbreviations:\n"
-        "  [Fixed(3), VBR(8), Array(Fixed(8))] (abbrev #0)\n"
-        "  [2, VBR(8), Array(Char6)] (abbrev #3)\n"
-        "  [2, Array(VBR(8))] (abbrev #4)\n"),
-              DescribeAbbrevTrieNode(Node->MatchRecord(Record)));
-      }
-      // Test matching [3, 5]
-      Record.Code = 3;
-      EXPECT_EQ(std::string("[3, 5]"), DescribeRecord(Record));
-      {
-        AbbrevTrieNode *Node = LookupMap[Record.Values.size()+1];
-        ASSERT_NE(Node, (void*) 0);
-        EXPECT_EQ(std::string(
-        "Abbreviations:\n"
-        "  [Fixed(3), VBR(8), Array(Fixed(8))] (abbrev #0)\n"),
-              DescribeAbbrevTrieNode(Node->MatchRecord(Record)));
-      }
-      break;
-    case 3:
-    default:
-      EXPECT_EQ(std::string(
-        "Abbreviations:\n"
-        "  [Fixed(3), VBR(8), Array(Fixed(8))] (abbrev #0)\n"
-        "Successor Map:\n"
-        "  Record.Code = 1\n"
-        "    Abbreviations:\n"
-        "      [Fixed(3), VBR(8), Array(Fixed(8))] (abbrev #0)\n"
-        "      [1, VBR(8), Array(Fixed(7))] (abbrev #1)\n"
-        "      [1, VBR(8), Array(Char6)] (abbrev #2)\n"
-        "    Successor Map:\n"
-        "      Record.Values[1] = 5\n"
-        "        Abbreviations:\n"
-        "          [Fixed(3), VBR(8), Array(Fixed(8))] (abbrev #0)\n"
-        "          [1, VBR(8), Array(Fixed(7))] (abbrev #1)\n"
-        "          [1, VBR(8), Array(Char6)] (abbrev #2)\n"
-        "          [Fixed(3), VBR(8), 5, Array(Fixed(8))] (abbrev #5)\n"
-        "  Record.Code = 2\n"
-        "    Abbreviations:\n"
-        "      [Fixed(3), VBR(8), Array(Fixed(8))] (abbrev #0)\n"
-        "      [2, VBR(8), Array(Char6)] (abbrev #3)\n"
-        "      [2, Array(VBR(8))] (abbrev #4)\n"
-        "    Successor Map:\n"
-        "      Record.Values[1] = 5\n"
-        "        Abbreviations:\n"
-        "          [Fixed(3), VBR(8), Array(Fixed(8))] (abbrev #0)\n"
-        "          [2, VBR(8), Array(Char6)] (abbrev #3)\n"
-        "          [2, Array(VBR(8))] (abbrev #4)\n"
-        "          [Fixed(3), VBR(8), 5, Array(Fixed(8))] (abbrev #5)\n"
-        "  Record.Values[1] = 5\n"
-        "    Abbreviations:\n"
-        "      [Fixed(3), VBR(8), Array(Fixed(8))] (abbrev #0)\n"
-        "      [Fixed(3), VBR(8), 5, Array(Fixed(8))] (abbrev #5)\n"),
-                DescribeAbbrevTrie(Iter->second));
-
-      // Test matching [1, 0, 5]
-      Record.Code = 1;
-      Record.Values.push_back(0);
-      Record.Values.push_back(5);
-      EXPECT_EQ(std::string("[1, 0, 5]"), DescribeRecord(Record));
-      {
-        AbbrevTrieNode *Node = LookupMap[Record.Values.size()+1];
-        ASSERT_NE(Node, (void*) 0);
-        EXPECT_EQ(std::string(
-        "Abbreviations:\n"
-        "  [Fixed(3), VBR(8), Array(Fixed(8))] (abbrev #0)\n"
-        "  [1, VBR(8), Array(Fixed(7))] (abbrev #1)\n"
-        "  [1, VBR(8), Array(Char6)] (abbrev #2)\n"
-        "  [Fixed(3), VBR(8), 5, Array(Fixed(8))] (abbrev #5)\n"),
-              DescribeAbbrevTrieNode(Node->MatchRecord(Record)));
-      }
-      // Test matching [1, 0, 50]
-      Record.Values[1] = 50;
-      EXPECT_EQ(std::string("[1, 0, 50]"), DescribeRecord(Record));
-      {
-        AbbrevTrieNode *Node = LookupMap[Record.Values.size()+1];
-        ASSERT_NE(Node, (void*) 0);
-        EXPECT_EQ(std::string(
-        "Abbreviations:\n"
-        "  [Fixed(3), VBR(8), Array(Fixed(8))] (abbrev #0)\n"
-        "  [1, VBR(8), Array(Fixed(7))] (abbrev #1)\n"
-        "  [1, VBR(8), Array(Char6)] (abbrev #2)\n"),
-              DescribeAbbrevTrieNode(Node->MatchRecord(Record)));
-      }
-      // Test matching [2, 0, 5]
-      Record.Code = 2;
-      Record.Values[1] = 5;
-      EXPECT_EQ(std::string("[2, 0, 5]"), DescribeRecord(Record));
-      {
-        AbbrevTrieNode *Node = LookupMap[Record.Values.size()+1];
-        ASSERT_NE(Node, (void*) 0);
-        EXPECT_EQ(std::string(
-        "Abbreviations:\n"
-        "  [Fixed(3), VBR(8), Array(Fixed(8))] (abbrev #0)\n"
-        "  [2, VBR(8), Array(Char6)] (abbrev #3)\n"
-        "  [2, Array(VBR(8))] (abbrev #4)\n"
-        "  [Fixed(3), VBR(8), 5, Array(Fixed(8))] (abbrev #5)\n"),
-              DescribeAbbrevTrieNode(Node->MatchRecord(Record)));
-      }
-      // Test matching [2, 0, 50]
-      Record.Values[1] = 50;
-      EXPECT_EQ(std::string("[2, 0, 50]"), DescribeRecord(Record));
-      {
-        AbbrevTrieNode *Node = LookupMap[Record.Values.size()+1];
-        ASSERT_NE(Node, (void*) 0);
-        EXPECT_EQ(std::string(
-        "Abbreviations:\n"
-        "  [Fixed(3), VBR(8), Array(Fixed(8))] (abbrev #0)\n"
-        "  [2, VBR(8), Array(Char6)] (abbrev #3)\n"
-        "  [2, Array(VBR(8))] (abbrev #4)\n"),
-              DescribeAbbrevTrieNode(Node->MatchRecord(Record)));
-      }
-      // Test matching [5, 0, 5]
-      Record.Code = 5;
-      Record.Values[1] = 5;
-      EXPECT_EQ(std::string("[5, 0, 5]"), DescribeRecord(Record));
-      {
-        AbbrevTrieNode *Node = LookupMap[Record.Values.size()+1];
-        ASSERT_NE(Node, (void*) 0);
-        EXPECT_EQ(std::string(
-        "Abbreviations:\n"
-        "  [Fixed(3), VBR(8), Array(Fixed(8))] (abbrev #0)\n"
-        "  [Fixed(3), VBR(8), 5, Array(Fixed(8))] (abbrev #5)\n"),
-              DescribeAbbrevTrieNode(Node->MatchRecord(Record)));
-      }
-      // Test matching [5, 0, 50]
-      Record.Values[1] = 50;
-      EXPECT_EQ(std::string("[5, 0, 50]"), DescribeRecord(Record));
-      {
-        AbbrevTrieNode *Node = LookupMap[Record.Values.size()+1];
-        ASSERT_NE(Node, (void*) 0);
-        EXPECT_EQ(std::string(
-        "Abbreviations:\n"
-        "  [Fixed(3), VBR(8), Array(Fixed(8))] (abbrev #0)\n"),
-              DescribeAbbrevTrieNode(Node->MatchRecord(Record)));
-      }
-      // Test matching [5, 0, 50, 10]
-      Record.Values.push_back(10);
-      EXPECT_EQ(std::string("[5, 0, 50, 10]"), DescribeRecord(Record));
-      {
-        AbbrevTrieNode *Node = LookupMap[Record.Values.size()+1];
-        ASSERT_NE(Node, (void*) 0);
-        EXPECT_EQ(std::string(
-        "Abbreviations:\n"
-        "  [Fixed(3), VBR(8), Array(Fixed(8))] (abbrev #0)\n"),
-              DescribeAbbrevTrieNode(Node->MatchRecord(Record)));
-      }
-      // Test matching [5, 0, 50, 10, 20]
-      Record.Values.push_back(20);
-      EXPECT_EQ(std::string("[5, 0, 50, 10, 20]"), DescribeRecord(Record));
-      {
-        AbbrevTrieNode *Node = LookupMap[Record.Values.size()+1];
-        ASSERT_NE(Node, (void*) 0);
-        EXPECT_EQ(std::string(
-        "Abbreviations:\n"
-        "  [Fixed(3), VBR(8), Array(Fixed(8))] (abbrev #0)\n"),
-              DescribeAbbrevTrieNode(Node->MatchRecord(Record)));
-      }
-      // Test matching [1, 0, 5, 10, 20]
-      Record.Code = 1;
-      Record.Values[1] = 5;
-      EXPECT_EQ(std::string("[1, 0, 5, 10, 20]"), DescribeRecord(Record));
-      {
-        AbbrevTrieNode *Node = LookupMap[Record.Values.size()+1];
-        ASSERT_NE(Node, (void*) 0);
-        EXPECT_EQ(std::string(
-        "Abbreviations:\n"
-        "  [Fixed(3), VBR(8), Array(Fixed(8))] (abbrev #0)\n"
-        "  [1, VBR(8), Array(Fixed(7))] (abbrev #1)\n"
-        "  [1, VBR(8), Array(Char6)] (abbrev #2)\n"
-        "  [Fixed(3), VBR(8), 5, Array(Fixed(8))] (abbrev #5)\n"),
-              DescribeAbbrevTrieNode(Node->MatchRecord(Record)));
-      }
-      break;
-    }
-  }
-
-  Clear(Abbrevs);
-  Clear(LookupMap);
-}
-
-}
diff --git a/unittests/Bitcode/NaClBitReaderTest.cpp b/unittests/Bitcode/NaClBitReaderTest.cpp
deleted file mode 100644
index 2593bbbe63f..00000000000
--- a/unittests/Bitcode/NaClBitReaderTest.cpp
+++ /dev/null
@@ -1,77 +0,0 @@
-//===- llvm/unittest/Bitcode/NaClBitReaderTest.cpp - Tests for BitReader --===//
-//
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
-//
-//===----------------------------------------------------------------------===//
-
-#include "llvm/ADT/SmallString.h"
-#include "llvm/Bitcode/BitstreamWriter.h"
-#include "llvm/Bitcode/NaCl/NaClReaderWriter.h"
-#include "llvm/IR/Constants.h"
-#include "llvm/IR/Instructions.h"
-#include "llvm/IR/LLVMContext.h"
-#include "llvm/IR/Module.h"
-#include "llvm/IR/Verifier.h"
-#include "llvm/Support/Debug.h"
-#include "llvm/Support/MemoryBuffer.h"
-#include "gtest/gtest.h"
-
-namespace llvm {
-namespace {
-
-static std::unique_ptr<Module> makeLLVMModule() {
-  std::unique_ptr<Module> Mod(new Module("test-mem", getGlobalContext()));
-
-  FunctionType* FuncTy =
-    FunctionType::get(Type::getVoidTy(Mod->getContext()), false);
-  Function* Func = Function::Create(FuncTy,GlobalValue::ExternalLinkage,
-                                    "func", Mod.get());
-
-  BasicBlock* Entry = BasicBlock::Create(Mod->getContext(), "entry", Func);
-  new UnreachableInst(Mod->getContext(), Entry);
-
-  BasicBlock* BB = BasicBlock::Create(Mod->getContext(), "bb", Func);
-  new UnreachableInst(Mod->getContext(), BB);
-
-  return Mod;
-}
-
-static void writeModuleToBuffer(SmallVectorImpl<char> &Buffer) {
-  std::unique_ptr<Module> Mod = makeLLVMModule();
-  raw_svector_ostream OS(Buffer);
-  NaClWriteBitcodeToFile(Mod.get(), OS);
-}
-
-// Check that we can parse a good bitcode file.
-TEST(NaClBitReaderTest, MaterializeSimpleModule) {
-  SmallString<1024> Mem;
-  writeModuleToBuffer(Mem);
-  std::unique_ptr<MemoryBuffer> Buffer = MemoryBuffer::getMemBuffer(Mem.str(), "test", false);
-  ErrorOr<Module *> ModuleOrErr =
-      getNaClLazyBitcodeModule(std::move(Buffer), getGlobalContext());
-  EXPECT_EQ(true, bool(ModuleOrErr));
-  // Do something with the module just to make sure it was built.
-  std::unique_ptr<Module> M(ModuleOrErr.get());
-  EXPECT_NE((Module *)nullptr, M.get());
-  M->getFunction("func")->materialize();
-  EXPECT_FALSE(verifyModule(*M, &dbgs()));
-}
-
-// Test that we catch bad stuff at the end of a bitcode file.
-TEST(NaClBitReaderTest, BadDataAfterModule) {
-  SmallString<1024> Mem;
-  writeModuleToBuffer(Mem);
-  Mem.append("more"); // Length must be divisible by 4!
-  std::unique_ptr<MemoryBuffer> Buffer = MemoryBuffer::getMemBuffer(Mem.str(), "test", false);
-  ErrorOr<Module *> ModuleOrErr =
-      getNaClLazyBitcodeModule(std::move(Buffer), getGlobalContext());
-  EXPECT_EQ(false, bool(ModuleOrErr));
-  std::string BadMessage("Invalid data after module");
-  EXPECT_EQ(BadMessage, ModuleOrErr.getError().message());
-}
-
-}
-}
diff --git a/unittests/Bitcode/NaClBitstreamReaderTest.cpp b/unittests/Bitcode/NaClBitstreamReaderTest.cpp
deleted file mode 100644
index aba127d9a57..00000000000
--- a/unittests/Bitcode/NaClBitstreamReaderTest.cpp
+++ /dev/null
@@ -1,69 +0,0 @@
-//===- llvm/unittest/Bitcode/NaClBitstreamReaderTest.cpp ------------------===//
-//     Tests issues in NaCl Bitstream Reader.
-//
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
-//
-//===----------------------------------------------------------------------===//
-
-// Tests issues in NaCl Bitstream Reader.
-
-// TODO(kschimpf) Add more Tests.
-
-#include "llvm/Bitcode/NaCl/NaClBitstreamReader.h"
-
-#include "gtest/gtest.h"
-
-using namespace llvm;
-
-namespace {
-
-static const uint64_t BitZero = 0;
-
-// Initializes array to sequence of alternating zeros/ones.
-void* InitAltOnes(uint8_t *Array, size_t ArraySize) {
-  for (size_t i = 0; i <ArraySize; ++i) {
-    Array[i] = 0x9;
-  }
-  return Array;
-}
-
-// Tests that the default bitstream cursor is at bit zero.
-TEST(NaClBitstreamTest, DefaultCursorAtBitZero) {
-  uint8_t CursorMemory[sizeof(NaClBitstreamCursor)];
-  NaClBitstreamCursor *Cursor =
-      new (InitAltOnes(CursorMemory, sizeof(NaClBitstreamCursor)))
-      NaClBitstreamCursor();
-  EXPECT_EQ(BitZero, Cursor->GetCurrentBitNo());
-}
-
-// Tests that when we initialize the bitstream cursor with an array-filled
-// bitstream reader, the cursor is at bit zero.
-TEST(NaClBitstreamTest, ReaderCursorAtBitZero) {
-  static const size_t BufferSize = 12;
-  unsigned char Buffer[BufferSize];
-  NaClBitstreamReader Reader(
-      getNonStreamedMemoryObject(Buffer, Buffer+BufferSize), 0);
-  uint8_t CursorMemory[sizeof(NaClBitstreamCursor)];
-  NaClBitstreamCursor *Cursor =
-      new (InitAltOnes(CursorMemory, sizeof(NaClBitstreamCursor)))
-      NaClBitstreamCursor(Reader);
-  EXPECT_EQ(BitZero, Cursor->GetCurrentBitNo());
-}
-
-TEST(NaClBitstreamTest, CursorAtReaderInitialAddress) {
-  static const size_t BufferSize = 12;
-  static const size_t InitialAddress = 8;
-  unsigned char Buffer[BufferSize];
-  NaClBitstreamReader Reader(
-      getNonStreamedMemoryObject(Buffer, Buffer+BufferSize), InitialAddress);
-  uint8_t CursorMemory[sizeof(NaClBitstreamCursor)];
-  NaClBitstreamCursor *Cursor =
-      new (InitAltOnes(CursorMemory, sizeof(NaClBitstreamCursor)))
-      NaClBitstreamCursor(Reader);
-  EXPECT_EQ(InitialAddress * CHAR_BIT, Cursor->GetCurrentBitNo());
-}
-
-} // end of anonymous namespace
diff --git a/unittests/Bitcode/NaClCompressTests.cpp b/unittests/Bitcode/NaClCompressTests.cpp
deleted file mode 100644
index 981cc9c31f4..00000000000
--- a/unittests/Bitcode/NaClCompressTests.cpp
+++ /dev/null
@@ -1,63 +0,0 @@
-//===- llvm/unittest/Bitcode/NaClCompressTests.cpp ------------------------===//
-//     Tests pnacl compression of bitcode files.
-//
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
-//
-//===----------------------------------------------------------------------===//
-
-#include "NaClMungeTest.h"
-
-#include "llvm/Bitcode/NaCl/NaClLLVMBitCodes.h"
-
-using namespace llvm;
-
-namespace naclmungetest {
-
-// Note: Tests fix for bug in
-// https://code.google.com/p/nativeclient/issues/detail?id=4104
-TEST(NaClCompressTests, FixedModuleAbbrevIdBug) {
-  const uint64_t BitcodeRecords[] = {
-    1, naclbitc::BLK_CODE_ENTER, naclbitc::MODULE_BLOCK_ID, 4, Terminator,
-    // Note: We need at least one module abbreviation to activate bug.
-    2, naclbitc::BLK_CODE_DEFINE_ABBREV, 2,
-       0, NaClBitCodeAbbrevOp::Array,
-       0, NaClBitCodeAbbrevOp::VBR, 6,
-       Terminator,
-    // Note: We need at least one record in the module that can introduce
-    // a new abbreviation and cause the bug.
-    4, naclbitc::MODULE_CODE_VERSION, 1, Terminator,
-    1, naclbitc::BLK_CODE_ENTER, 17, 4, Terminator,
-    3, naclbitc::TYPE_CODE_NUMENTRY, 0, Terminator,
-    0, naclbitc::BLK_CODE_EXIT, Terminator,
-    0, naclbitc::BLK_CODE_EXIT, Terminator,
-  };
-
-  // Show textual version of sample input.
-  NaClObjDumpMunger DumpMunger(ARRAY_TERM(BitcodeRecords));
-  EXPECT_TRUE(DumpMunger.runTest());
-  EXPECT_EQ(
-      "       0:0|<65532, 80, 69, 88, 69, 1, 0,|Magic Number: 'PEXE' (80, 69, 8"
-      "8, 69)\n"
-      "          | 8, 0, 17, 0, 4, 0, 2, 0, 0, |PNaCl Version: 2\n"
-      "          | 0>                          |\n"
-      "      16:0|1: <65535, 8, 4>             |module {  // BlockID = 8\n"
-      "      24:0|  2: <65533, 2, 0, 3, 0, 2, 6|  %a0 = abbrev <array(vbr(6))>;"
-      "\n"
-      "          |      >                      |\n"
-      "      26:6|  4: <1, 1>                  |  version 1; <%a0>\n"
-      "      29:4|  1: <65535, 17, 4>          |  types {  // BlockID = 17\n"
-      "      36:0|    3: <1, 0>                |    count 0;\n"
-      "      38:6|  0: <65534>                 |  }\n"
-      "      40:0|0: <65534>                   |}\n",
-      DumpMunger.getTestResults());
-
-  // Show that we can compress as well.
-  NaClCompressMunger CompressMunger(BitcodeRecords,
-                                    array_lengthof(BitcodeRecords), Terminator);
-  EXPECT_TRUE(CompressMunger.runTest());
-}
-
-} // end of namespace naclmungetest
diff --git a/unittests/Bitcode/NaClMungeTest.h b/unittests/Bitcode/NaClMungeTest.h
deleted file mode 100644
index 52c7d93ddf3..00000000000
--- a/unittests/Bitcode/NaClMungeTest.h
+++ /dev/null
@@ -1,41 +0,0 @@
-//===- llvm/unittest/Bitcode/NaClMungeTest.h - Test munging utils ---------===//
-//
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
-//
-//===----------------------------------------------------------------------===//
-
-// Contains common utilities used in bitcode munge tests.
-
-#ifndef LLVM_UNITTEST_BITCODE_NACLMUNGETEST_H
-#define LLVM_UNITTEST_BITCODE_NACLMUNGETEST_H
-
-#include "llvm/ADT/STLExtras.h"
-#include "llvm/Bitcode/NaCl/NaClBitcodeMunge.h"
-
-#include "gtest/gtest.h"
-
-namespace naclmungetest {
-
-const uint64_t Terminator = 0x5768798008978675LL;
-
-#define ARRAY(name) name, array_lengthof(name)
-
-#define ARRAY_TERM(name) ARRAY(name), Terminator
-
-inline std::string stringify(llvm::NaClMungedBitcode &MungedBitcode) {
-  std::string Buffer;
-  llvm::raw_string_ostream StrBuf(Buffer);
-  MungedBitcode.print(StrBuf);
-  return StrBuf.str();
-}
-
-inline std::string stringify(llvm::NaClBitcodeMunger &Munger) {
-  return stringify(Munger.getMungedBitcode());
-}
-
-} // end of namespace naclmungetest
-
-#endif // end LLVM_UNITTEST_BITCODE_NACLMUNGETEST_H
diff --git a/unittests/Bitcode/NaClMungeWriteErrorTests.cpp b/unittests/Bitcode/NaClMungeWriteErrorTests.cpp
deleted file mode 100644
index 83705172fb6..00000000000
--- a/unittests/Bitcode/NaClMungeWriteErrorTests.cpp
+++ /dev/null
@@ -1,842 +0,0 @@
-//===- llvm/unittest/Bitcode/NaClMungeWriteErrorTests.cpp -----------------===//
-//     Tests parser for PNaCl bitcode instructions.
-//
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
-//
-//===----------------------------------------------------------------------===//
-
-// Tests write errors for munged bitcode.
-
-#include "NaClMungeTest.h"
-
-#include "llvm/Bitcode/NaCl/NaClBitcodeParser.h"
-#include "llvm/Bitcode/NaCl/NaClLLVMBitCodes.h"
-
-#include "gtest/gtest.h"
-
-using namespace llvm;
-
-namespace naclmungetest {
-
-// Test list of bitcode records.
-const uint64_t BitcodeRecords[] = {
-  1, naclbitc::BLK_CODE_ENTER, naclbitc::MODULE_BLOCK_ID, 2, Terminator,
-  1, naclbitc::BLK_CODE_ENTER, naclbitc::TYPE_BLOCK_ID_NEW, 3, Terminator,
-  3, naclbitc::TYPE_CODE_NUMENTRY, 2, Terminator,
-  3, naclbitc::TYPE_CODE_VOID, Terminator,
-  3, naclbitc::TYPE_CODE_FUNCTION, 0, 0, Terminator,
-  0, naclbitc::BLK_CODE_EXIT, Terminator,
-  3, naclbitc::MODULE_CODE_FUNCTION, 1, 0, 0, 0, Terminator,
-  1, naclbitc::BLK_CODE_ENTER, naclbitc::FUNCTION_BLOCK_ID, 2, Terminator,
-  3, naclbitc::FUNC_CODE_DECLAREBLOCKS, 1, Terminator,
-  3, naclbitc::FUNC_CODE_INST_RET, Terminator,
-  0, naclbitc::BLK_CODE_EXIT, Terminator,
-  0, naclbitc::BLK_CODE_EXIT, Terminator
-};
-
-// Indices to records in bitcode.
-const uint64_t VoidTypeIndex = 3; // Index for "@t0 = void".
-const uint64_t RetVoidIndex = 9; // return void;
-const uint64_t LastExitBlockIndex = 11;
-
-// Expected output when bitcode records are dumped.
-const char *ExpectedDumpedBitcode =
-    "       0:0|<65532, 80, 69, 88, 69, 1, 0,|Magic Number: 'PEXE' (80, 69, "
-    "88, 69)\n"
-    "          | 8, 0, 17, 0, 4, 0, 2, 0, 0, |PNaCl Version: 2\n"
-    "          | 0>                          |\n"
-    "      16:0|1: <65535, 8, 2>             |module {  // BlockID = 8\n"
-    "      24:0|  1: <65535, 17, 3>          |  types {  // BlockID = 17\n"
-    "      32:0|    3: <1, 2>                |    count 2;\n"
-    "      34:5|    3: <2>                   |    @t0 = void;\n"
-    "      36:4|    3: <21, 0, 0>            |    @t1 = void ();\n"
-    "      39:7|  0: <65534>                 |  }\n"
-    "      44:0|  3: <8, 1, 0, 0, 0>         |  define external void @f0();\n"
-    "      48:6|  1: <65535, 12, 2>          |  function void @f0() {  \n"
-    "          |                             |                   // BlockID "
-    "= 12\n"
-    "      56:0|    3: <1, 1>                |    blocks 1;\n"
-    "          |                             |  %b0:\n"
-    "      58:4|    3: <10>                  |    ret void;\n"
-    "      60:2|  0: <65534>                 |  }\n"
-    "      64:0|0: <65534>                   |}\n"
-    ;
-
-const char *UnableToContinue =
-      "Error: Unable to generate bitcode file due to write errors\n";
-
-const char *NoErrorRecoveryMessages = "";
-
-// Runs write munging tests on BitcodeRecords with the given Edits. It
-// then parses the written bitcode.  ErrorMessages is the expected
-// error messages logged by the write munging, when no error recovery
-// is allowed. ErrorRecoveryMessages are messages, in addition to
-// ErrorMessages, when the writer applies error recovery.
-void CheckParseEdits(const uint64_t *Edits, size_t EditsSize,
-                     std::string ErrorMessages,
-                     std::string ErrorRecoveryMessages) {
-  NaClParseBitcodeMunger Munger(ARRAY_TERM(BitcodeRecords));
-  EXPECT_FALSE(Munger.runTest(Edits, EditsSize, true));
-  std::string BadResults(ErrorMessages);
-  BadResults.append(UnableToContinue);
-  EXPECT_EQ(BadResults, Munger.getTestResults());
-
-  Munger.setTryToRecoverOnWrite(true);
-  EXPECT_TRUE(Munger.runTest(Edits, EditsSize, true));
-  std::string GoodResults(ErrorMessages);
-  GoodResults.append(ErrorRecoveryMessages);
-  GoodResults.append("Successful parse!\n");
-  EXPECT_EQ(GoodResults, Munger.getTestResults());
-}
-
-// Same as CheckParseEdits, but also runs the bitcode dumper on the
-// written bitcode records. DumpedBitcode is the expected dumped
-// bitcode.
-void CheckDumpEdits(const uint64_t *Edits, size_t EditsSize,
-                    std::string ErrorMessages,
-                    std::string ErrorRecoveryMessages,
-                    std::string DumpedBitcode,
-                    bool RecoveredTestHasError=false) {
-  NaClObjDumpMunger Munger(ARRAY_TERM(BitcodeRecords));
-  EXPECT_FALSE(Munger.runTest(Edits, EditsSize));
-  std::string BadResults(ErrorMessages);
-  BadResults.append(UnableToContinue);
-  EXPECT_EQ(BadResults, Munger.getTestResults());
-
-  Munger.setTryToRecoverOnWrite(true);
-  if (RecoveredTestHasError)
-     EXPECT_FALSE(Munger.runTest(Edits, EditsSize));
-  else
-     EXPECT_TRUE(Munger.runTest(Edits, EditsSize));
-  std::string GoodResults(ErrorMessages);
-  GoodResults.append(ErrorRecoveryMessages);
-  GoodResults.append(DumpedBitcode);
-  EXPECT_EQ(GoodResults, Munger.getTestResults());
-
-  // Verify that we can also parse the bitcode.
-  CheckParseEdits(Edits, EditsSize, ErrorMessages, ErrorRecoveryMessages);
-}
-
-// Same as ExpectedDumpedBitcode, but is just the records dumped by the
-// simpler write munger.
-const char *ExpectedRecords =
-    "       1: [65535, 8, 2]\n"
-    "         1: [65535, 17, 3]\n"
-    "           3: [1, 2]\n"
-    "           3: [2]\n"
-    "           3: [21, 0, 0]\n"
-    "         0: [65534]\n"
-    "         3: [8, 1, 0, 0, 0]\n"
-    "         1: [65535, 12, 2]\n"
-    "           3: [1, 1]\n"
-    "           3: [10]\n"
-    "         0: [65534]\n"
-    "       0: [65534]\n";
-
-// Same as CheckParseEdits, but run the simpler write munger instead
-// of the bitcode parser. Records is the records dumped by the write
-// munger. This should be used in cases where the written munged
-// records is not valid bitcode.
-void CheckWriteEdits(const uint64_t *Edits, size_t EditsSize,
-                     std::string ExpectedErrorMessage,
-                     std::string ErrorRecoveryMessages,
-                     std::string Records) {
-  NaClWriteMunger Munger(ARRAY_TERM(BitcodeRecords));
-  EXPECT_FALSE(Munger.runTest(Edits, EditsSize));
-  std::string BadResults(ExpectedErrorMessage);
-  BadResults.append(UnableToContinue);
-  EXPECT_EQ(BadResults, Munger.getTestResults());
-
-  Munger.setTryToRecoverOnWrite(true);
-  EXPECT_TRUE(Munger.runTest(Edits, EditsSize));
-  std::string GoodResults(ExpectedErrorMessage);
-  GoodResults.append(ErrorRecoveryMessages);
-  GoodResults.append(Records);
-  EXPECT_EQ(GoodResults, Munger.getTestResults());
-}
-
-// Show that we can dump the bitcode records
-TEST(NaClMungeWriteErrorTests, DumpBitcodeRecords) {
-  NaClObjDumpMunger Munger(ARRAY_TERM(BitcodeRecords));
-  EXPECT_TRUE(Munger.runTest());
-  EXPECT_EQ(ExpectedDumpedBitcode, Munger.getTestResults());
-}
-
-// Edit to change void type with an illegal abbreviation index.
-const uint64_t AbbrevIndex4VoidTypeEdit[] = {
-  VoidTypeIndex, NaClMungedBitcode::Replace,
-  4, naclbitc::TYPE_CODE_VOID, Terminator,
-};
-
-// Show that by default, one can't write a bad abbreviation index.
-TEST(NaClMungeWriteErrorTests, CantWriteBadAbbrevIndex) {
-  CheckDumpEdits(
-      ARRAY(AbbrevIndex4VoidTypeEdit),
-      "Error (Block 17): Uses illegal abbreviation index: 4: [2]\n",
-      NoErrorRecoveryMessages,
-      ExpectedDumpedBitcode);
-}
-
-// Show that writing out an illegal abbreviation index, causes the
-// parser to fail.
-TEST(MyNaClMungerWriteErrorTests, DieOnWriteBadAbbreviationIndex) {
-  NaClWriteMunger Munger(ARRAY_TERM(BitcodeRecords));
-  Munger.setWriteBadAbbrevIndex(true);
-  Munger.setRunAsDeathTest(true);
-  EXPECT_DEATH(
-      Munger.runTest(ARRAY(AbbrevIndex4VoidTypeEdit)),
-      ".*"
-      // Report problem while writing.
-      "Error \\(Block 17\\)\\: Uses illegal abbreviation index\\: 4\\: \\[2\\]"
-      ".*"
-      // Corresponding error while parsing.
-      "Fatal\\(35\\:0)\\: Invalid abbreviation \\# 4 defined for record"
-      ".*"
-      // Output of report_fatal_error.
-      "LLVM ERROR\\: Unable to continue"
-      ".*");
-}
-
-// Show what happens when we use more local abbreviations than specified in the
-// corresponding enclosing block.
-TEST(NaClMungeWriteErrorTests, CantWriteTooManyLocalAbbreviations) {
-  // Edit to add local abbreviation for "ret void", and then use on that
-  // instruction.
-  const uint64_t UseLocalRetVoidAbbrevEdits[] = {
-    RetVoidIndex, NaClMungedBitcode::AddBefore,
-    2, naclbitc::BLK_CODE_DEFINE_ABBREV, 1, 1,
-    naclbitc::FUNC_CODE_INST_RET, Terminator,
-    RetVoidIndex, NaClMungedBitcode::Replace,
-    4, naclbitc::FUNC_CODE_INST_RET, Terminator
-  };
-
-  CheckDumpEdits(
-      ARRAY(UseLocalRetVoidAbbrevEdits),
-      "Error (Block 12): Uses illegal abbreviation index: 4: [10]\n",
-      NoErrorRecoveryMessages,
-      "       0:0|<65532, 80, 69, 88, 69, 1, 0,|Magic Number: 'PEXE'"
-      " (80, 69, 88, 69)\n"
-      "          | 8, 0, 17, 0, 4, 0, 2, 0, 0, |PNaCl Version: 2\n"
-      "          | 0>                          |\n"
-      "      16:0|1: <65535, 8, 2>             |module {  // BlockID = 8\n"
-      "      24:0|  1: <65535, 17, 3>          |  types {  // BlockID = 17\n"
-      "      32:0|    3: <1, 2>                |    count 2;\n"
-      "      34:5|    3: <2>                   |    @t0 = void;\n"
-      "      36:4|    3: <21, 0, 0>            |    @t1 = void ();\n"
-      "      39:7|  0: <65534>                 |  }\n"
-      "      44:0|  3: <8, 1, 0, 0, 0>         |  define external void @f0();\n"
-      // Block only specifies 2 bits for abbreviations (i.e. limit = 3).
-      "      48:6|  1: <65535, 12, 2>          |  function void @f0() {  \n"
-      "          |                             |                   // BlockID"
-      " = 12\n"
-      "      56:0|    3: <1, 1>                |    blocks 1;\n"
-      // Added abbreviation. Defines abbreviation index 4.
-      "      58:4|    2: <65533, 1, 1, 10>     |    %a0 = abbrev <10>;\n"
-      "          |                             |  %b0:\n"
-      // Repaired abbreviation index of 4 (now 3).
-      "      60:4|    3: <10>                  |    ret void;\n"
-      "      62:2|  0: <65534>                 |  }\n"
-      "      64:0|0: <65534>                   |}\n");
-}
-
-// Show what happens when there are more enter blocks then exit blocks.
-TEST(NaClMungeWriteErrorTests, CantWriteTooManyEnterBlocks) {
-  // Remove all records except the first two records in BitcodeRecords.
-  const uint64_t Edits[] = {
-    2, NaClMungedBitcode::Remove,
-    3, NaClMungedBitcode::Remove,
-    4, NaClMungedBitcode::Remove,
-    5, NaClMungedBitcode::Remove,
-    6, NaClMungedBitcode::Remove,
-    7, NaClMungedBitcode::Remove,
-    8, NaClMungedBitcode::Remove,
-    9, NaClMungedBitcode::Remove,
-    10, NaClMungedBitcode::Remove,
-    11, NaClMungedBitcode::Remove
-  };
-  CheckDumpEdits(
-      ARRAY(Edits),
-      "Error (Block 17): Missing close block.\n"
-      "Error (Block 8): Missing close block.\n",
-      NoErrorRecoveryMessages,
-      "       0:0|<65532, 80, 69, 88, 69, 1, 0,|Magic Number: 'PEXE' (80, 69,"
-      " 88, 69)\n"
-      "          | 8, 0, 17, 0, 4, 0, 2, 0, 0, |PNaCl Version: 2\n"
-      "          | 0>                          |\n"
-      "      16:0|1: <65535, 8, 2>             |module {  // BlockID = 8\n"
-      "      24:0|  1: <65535, 17, 3>          |  types {  // BlockID = 17\n"
-      "      32:0|  0: <65534>                 |  }\n"
-      "      36:0|0: <65534>                   |}\n");
-}
-
-// Show what happens when there are fewer enter blocks than exit
-// blocks.
-TEST(NaClMungeWriteErrorTests, CantWriteTooManyExitBlocks) {
-  // Add two blocks to the end of BitcodeRecords.
-  const uint64_t Edits[] = {
-    LastExitBlockIndex, NaClMungedBitcode::AddAfter,
-    naclbitc::END_BLOCK, naclbitc::BLK_CODE_EXIT, Terminator,
-    LastExitBlockIndex, NaClMungedBitcode::AddAfter,
-    naclbitc::END_BLOCK, naclbitc::BLK_CODE_EXIT, Terminator
-  };
-  CheckDumpEdits(
-      ARRAY(Edits),
-      "Error (Block unknown): Extraneous exit block: 0: [65534]\n",
-      "Error (Block unknown): Extraneous exit block: 0: [65534]\n",
-      ExpectedDumpedBitcode);
-}
-
-// Show that an error occurs when writing a bitcode record that isn't
-// in any block.
-TEST(NaClMungeWriteErrorTests, CantWriteRecordOutsideBlock) {
-  const uint64_t Edit[] = {
-    LastExitBlockIndex, NaClMungedBitcode::AddAfter,
-    naclbitc::UNABBREV_RECORD, naclbitc::MODULE_CODE_VERSION,  4, Terminator
-  };
-  std::string Records(ExpectedRecords);
-  Records.append(
-      "       1: [65535, 4294967295, 3]\n"
-      "         3: [1, 4]\n"
-      "       0: [65534]\n");
-  CheckWriteEdits(
-      ARRAY(Edit),
-      "Error (Block unknown): Record outside block: 3: [1, 4]\n",
-      "Error (Block unknown): Missing close block.\n",
-      Records);
-}
-
-// Show that no error occurs if we write out the maximum allowable
-// block abbreviation index bit limit.
-TEST(NaClMungerWriteErrorTests, CanWriteBlockWithMaxLimit) {
-  // Replace initial block enter with maximum bit size.
-  const uint64_t Edit[] = {
-    0, NaClMungedBitcode::Replace,
-    1, naclbitc::BLK_CODE_ENTER, naclbitc::MODULE_BLOCK_ID,
-       naclbitc::MaxAbbrevWidth, Terminator
-  };
-  NaClWriteMunger Munger(ARRAY_TERM(BitcodeRecords));
-  EXPECT_TRUE(Munger.runTest(ARRAY(Edit)));
-  EXPECT_EQ(
-      "       1: [65535, 8, 32]\n" // Max abbreviation bit limit (32).
-      "         1: [65535, 17, 3]\n"
-      "           3: [1, 2]\n"
-      "           3: [2]\n"
-      "           3: [21, 0, 0]\n"
-      "         0: [65534]\n"
-      "         3: [8, 1, 0, 0, 0]\n"
-      "         1: [65535, 12, 2]\n"
-      "           3: [1, 1]\n"
-      "           3: [10]\n"
-      "         0: [65534]\n"
-      "       0: [65534]\n",
-      Munger.getTestResults());
-}
-
-// Show that an error occurs if the block abbreviation index bit limit is
-// greater than the maximum allowable.
-TEST(NaClMungerWriteErrorTests, CantWriteBlockWithBadBitLimit) {
-  // Replace initial block enter with value out of range.
-  const uint64_t Edit[] = {
-    0, NaClMungedBitcode::Replace,
-    1, naclbitc::BLK_CODE_ENTER, naclbitc::MODULE_BLOCK_ID,
-       naclbitc::MaxAbbrevWidth + 1, Terminator
-  };
-  CheckDumpEdits(
-      ARRAY(Edit),
-      "Error (Block unknown): Block index bit limit 33 invalid. Must be in"
-      " [2..32]: 1: [65535, 8, 33]\n",
-      "",
-      "       0:0|<65532, 80, 69, 88, 69, 1, 0,|Magic Number: 'PEXE' (80, 69,"
-      " 88, 69)\n"
-      "          | 8, 0, 17, 0, 4, 0, 2, 0, 0, |PNaCl Version: 2\n"
-      "          | 0>                          |\n"
-      // Corrected bitsize from 33 to 32.
-      "      16:0|1: <65535, 8, 32>            |module {  // BlockID = 8\n"
-      "      24:0|  1: <65535, 17, 3>          |  types {  // BlockID = 17\n"
-      "      36:0|    3: <1, 2>                |    count 2;\n"
-      "      38:5|    3: <2>                   |    @t0 = void;\n"
-      "      40:4|    3: <21, 0, 0>            |    @t1 = void ();\n"
-      "      43:7|  0: <65534>                 |  }\n"
-      "      48:0|  3: <8, 1, 0, 0, 0>         |  define external void @f0();\n"
-      "      56:4|  1: <65535, 12, 2>          |  function void @f0() {  \n"
-      "          |                             |                   // BlockID"
-      " = 12\n"
-      "      68:0|    3: <1, 1>                |    blocks 1;\n"
-      "          |                             |  %b0:\n"
-      "      70:4|    3: <10>                  |    ret void;\n"
-      "      72:2|  0: <65534>                 |  }\n"
-      "      76:0|0: <65534>                   |}\n");
-}
-
-// Show that we can't write an enter block with a very large block id.
-TEST(NaClMungerWriteErrorTests, CantWriteBlockWithLargeBlockID) {
-  // Replace initial block enter with value out of range.
-  const uint64_t Edit[] = {
-    0, NaClMungedBitcode::Replace,
-    1, naclbitc::BLK_CODE_ENTER, (uint64_t)1 << 33, 2, Terminator
-  };
-  CheckWriteEdits(
-      ARRAY(Edit),
-      "Error (Block unknown): Block id must be <= 4294967295: 1:"
-      " [65535, 8589934592, 2]\n",
-      "",
-      // Note that the maximum block ID is used for recovery.
-      "       1: [65535, 4294967295, 2]\n"
-      "         1: [65535, 17, 3]\n"
-      "           3: [1, 2]\n"
-      "           3: [2]\n"
-      "           3: [21, 0, 0]\n"
-      "         0: [65534]\n"
-      "         3: [8, 1, 0, 0, 0]\n"
-      "         1: [65535, 12, 2]\n"
-      "           3: [1, 1]\n"
-      "           3: [10]\n"
-      "         0: [65534]\n"
-      "       0: [65534]\n");
-}
-
-// Show that we check that the abbreviation actually applies to the
-// record associated with that abbreviation. Also shows that we repair
-// the problem by applying the default abbreviation instead.
-TEST(NaClMungeWriteErrorsTests, TestMismatchedAbbreviation) {
-  // Create edits to:
-  // 1) Expand the number of abbreviation index bits for the block from 2 to 3.
-  // 2) Introduce the incorrect abbreviation for the return instruction.
-  //    i.e. [9] instead of [10].
-  // 3) Apply the bad abbreviation to record "ret"
-  const uint64_t FunctionEnterIndex = 7;
-  const uint64_t Edits[] {
-    // Upped abbreviation index bits to 3
-    FunctionEnterIndex, NaClMungedBitcode::Replace,
-        1, naclbitc::BLK_CODE_ENTER, naclbitc::FUNCTION_BLOCK_ID, 3, Terminator,
-    // abbrev 4: [9]
-    RetVoidIndex, NaClMungedBitcode::AddBefore,
-        2, naclbitc::BLK_CODE_DEFINE_ABBREV, 1, 1,
-        naclbitc::FUNC_CODE_INST_RET - 1, Terminator,
-    // "ret" with bad abbreviation (4).
-    RetVoidIndex, NaClMungedBitcode::Replace,
-        4, naclbitc::FUNC_CODE_INST_RET, Terminator
-        };
-
-  CheckDumpEdits(
-      ARRAY(Edits),
-      "Error (Block 12): Abbreviation doesn't apply to record: 4: [10]\n",
-      "",
-      "       0:0|<65532, 80, 69, 88, 69, 1, 0,|Magic Number: 'PEXE' (80, 69,"
-      " 88, 69)\n"
-      "          | 8, 0, 17, 0, 4, 0, 2, 0, 0, |PNaCl Version: 2\n"
-      "          | 0>                          |\n"
-      "      16:0|1: <65535, 8, 2>             |module {  // BlockID = 8\n"
-      "      24:0|  1: <65535, 17, 3>          |  types {  // BlockID = 17\n"
-      "      32:0|    3: <1, 2>                |    count 2;\n"
-      "      34:5|    3: <2>                   |    @t0 = void;\n"
-      "      36:4|    3: <21, 0, 0>            |    @t1 = void ();\n"
-      "      39:7|  0: <65534>                 |  }\n"
-      "      44:0|  3: <8, 1, 0, 0, 0>         |  define external void @f0();\n"
-      // Upped abbreviation index bits to 3
-      "      48:6|  1: <65535, 12, 3>          |  function void @f0() {  \n"
-      "          |                             |                   // BlockID"
-      " = 12\n"
-      "      56:0|    3: <1, 1>                |    blocks 1;\n"
-      // added abbrev 4: [9]
-      "      58:5|    2: <65533, 1, 1, 9>      |    %a0 = abbrev <9>;\n"
-      "          |                             |  %b0:\n"
-      // Implicit repair of abbreviation index (from 4 to 3: the default abbrev)
-      "      60:6|    3: <10>                  |    ret void;\n"
-      "      62:5|  0: <65534>                 |  }\n"
-      "      64:0|0: <65534>                   |}\n");
-}
-
-// Show that we recognize when an abbreviation definition record is
-// malformed.  Also show that we repair the problem by removing the
-// definition.
-TEST(NaClMungeWriteErrorsTests, TestWritingMalformedAbbreviation) {
-  // Create edits to:
-  // 1) Expand the number of abbreviation index bits for the block from 2 to 3.
-  // 2) Leave out the "literal" operand encoding out.
-  const uint64_t FunctionEnterIndex = 7;
-  const uint64_t Edits[] {
-    FunctionEnterIndex, NaClMungedBitcode::Replace,   // Set Abbrev bits = 3
-        1, naclbitc::BLK_CODE_ENTER, naclbitc::FUNCTION_BLOCK_ID, 3, Terminator,
-    RetVoidIndex, NaClMungedBitcode::AddBefore,
-        // Bad abbreviation! Intentionally leave out "literal" operand: 1
-        2, naclbitc::BLK_CODE_DEFINE_ABBREV, 1, // 1,
-        naclbitc::FUNC_CODE_INST_RET, Terminator,
-        };
-
-  CheckDumpEdits(
-      ARRAY(Edits),
-      "Error (Block 12): Bad abbreviation operand encoding 10:"
-      " 2: [65533, 1, 10]\n",
-      "",
-      "       0:0|<65532, 80, 69, 88, 69, 1, 0,|Magic Number: 'PEXE' (80, 69,"
-      " 88, 69)\n"
-      "          | 8, 0, 17, 0, 4, 0, 2, 0, 0, |PNaCl Version: 2\n"
-      "          | 0>                          |\n"
-      "      16:0|1: <65535, 8, 2>             |module {  // BlockID = 8\n"
-      "      24:0|  1: <65535, 17, 3>          |  types {  // BlockID = 17\n"
-      "      32:0|    3: <1, 2>                |    count 2;\n"
-      "      34:5|    3: <2>                   |    @t0 = void;\n"
-      "      36:4|    3: <21, 0, 0>            |    @t1 = void ();\n"
-      "      39:7|  0: <65534>                 |  }\n"
-      "      44:0|  3: <8, 1, 0, 0, 0>         |  define external void @f0();\n"
-      // Edit to change number of abbrev bits to 3.
-      "      48:6|  1: <65535, 12, 3>          |  function void @f0() {  \n"
-      "          |                             |                   // BlockID"
-      " = 12\n"
-      "      56:0|    3: <1, 1>                |    blocks 1;\n"
-      "          |                             |  %b0:\n"
-      "      58:5|    3: <10>                  |    ret void;\n"
-      "      60:4|  0: <65534>                 |  }\n"
-      "      64:0|0: <65534>                   |}\n");
-}
-
-// Show how we deal with additional abbreviations defined for a block,
-// once a bad abbreviation definition record is found. That is, we
-// remove all succeeding abbreviations definitions for that block. In
-// addition, any record refering to a remove abbreviation is changed
-// to use the default abbreviation.
-TEST(NaClMungedWriteErrorTests, TestRemovingAbbrevWithMultAbbrevs) {
-  NaClWriteMunger Munger(ARRAY_TERM(BitcodeRecords));
-  const uint64_t FunctionEnterIndex = 7;
-  const uint64_t Edits[] {
-    FunctionEnterIndex, NaClMungedBitcode::Replace,   // Set Abbrev bits = 3
-        1, naclbitc::BLK_CODE_ENTER, naclbitc::FUNCTION_BLOCK_ID, 3, Terminator,
-    RetVoidIndex, NaClMungedBitcode::AddBefore,  // bad abbreviation!
-        2, naclbitc::BLK_CODE_DEFINE_ABBREV, 1, // 1,
-        naclbitc::FUNC_CODE_INST_RET - 1, Terminator,
-    RetVoidIndex, NaClMungedBitcode::AddBefore,  // good abbreviation to ignore.
-        2, naclbitc::BLK_CODE_DEFINE_ABBREV, 1, 1,
-        naclbitc::FUNC_CODE_INST_RET, Terminator,
-    RetVoidIndex, NaClMungedBitcode::Replace,  // reference to good abreviation.
-        5, naclbitc::FUNC_CODE_INST_RET, Terminator
-        };
-
-  CheckDumpEdits(
-      ARRAY(Edits),
-      "Error (Block 12): Bad abbreviation operand encoding 9:"
-      " 2: [65533, 1, 9]\n",
-      "Error (Block 12): Ignoring abbreviation: 2: [65533, 1, 1, 10]\n"
-      "Error (Block 12): Uses illegal abbreviation index: 5: [10]\n",
-      "       0:0|<65532, 80, 69, 88, 69, 1, 0,|Magic Number: 'PEXE' (80, 69,"
-      " 88, 69)\n"
-      "          | 8, 0, 17, 0, 4, 0, 2, 0, 0, |PNaCl Version: 2\n"
-      "          | 0>                          |\n"
-      "      16:0|1: <65535, 8, 2>             |module {  // BlockID = 8\n"
-      "      24:0|  1: <65535, 17, 3>          |  types {  // BlockID = 17\n"
-      "      32:0|    3: <1, 2>                |    count 2;\n"
-      "      34:5|    3: <2>                   |    @t0 = void;\n"
-      "      36:4|    3: <21, 0, 0>            |    @t1 = void ();\n"
-      "      39:7|  0: <65534>                 |  }\n"
-      "      44:0|  3: <8, 1, 0, 0, 0>         |  define external void @f0();\n"
-      // Edit to change number of abbrev bits to 3.
-      "      48:6|  1: <65535, 12, 3>          |  function void @f0() {  \n"
-      "          |                             |                   // BlockID"
-      " = 12\n"
-      "      56:0|    3: <1, 1>                |    blocks 1;\n"
-      "          |                             |  %b0:\n"
-      "      58:5|    3: <10>                  |    ret void;\n"
-      "      60:4|  0: <65534>                 |  }\n"
-      "      64:0|0: <65534>                   |}\n");
-}
-
-// Show that inserting an abbreviation with a bad fixed width is dealt with.
-TEST(NaClMungeWriteErrorTests, InvalidFixedAbbreviationSize) {
-  // Insert bad abbreviation Fixed(36) into type block.
-  assert(36 > naclbitc::MaxAbbrevWidth);
-  const uint64_t Edit[] = {
-    VoidTypeIndex, NaClMungedBitcode::AddBefore,
-    naclbitc::DEFINE_ABBREV, naclbitc::BLK_CODE_DEFINE_ABBREV, 1,
-    0, NaClBitCodeAbbrevOp::Fixed, 36, Terminator
-  };
-  CheckDumpEdits(
-      ARRAY(Edit),
-      "Error (Block 17): Invalid abbreviation Fixed(36) in: 2: [65533, 1, 0,"
-      " 1, 36]\n",
-      NoErrorRecoveryMessages,
-      ExpectedDumpedBitcode);
-}
-
-// Show that inserting an abbreviation with a bad vbr width is dealt with.
-TEST(NaClMungeWriteErrorTests, InvalidVbrAbbreviationSize) {
-  // Insert bad abbreviation Vbr(36) into type block.
-  assert(36 > naclbitc::MaxAbbrevWidth);
-  const uint64_t Edit[] = {
-    VoidTypeIndex, NaClMungedBitcode::AddBefore,
-    naclbitc::DEFINE_ABBREV, naclbitc::BLK_CODE_DEFINE_ABBREV, 1,
-    0, NaClBitCodeAbbrevOp::VBR, 36, Terminator
-  };
-  CheckDumpEdits(
-      ARRAY(Edit),
-      "Error (Block 17): Invalid abbreviation VBR(36) in: 2: [65533, 1, 0,"
-      " 2, 36]\n",
-      NoErrorRecoveryMessages,
-      ExpectedDumpedBitcode);
-}
-
-// Show that the array operator can't appear last.
-TEST(NaClMungeWriteErrorTests, InvalidArrayAbbreviationLast) {
-  const uint64_t Edit[] = {
-    VoidTypeIndex, NaClMungedBitcode::AddBefore,
-    naclbitc::DEFINE_ABBREV, naclbitc::BLK_CODE_DEFINE_ABBREV, 1,
-    0, NaClBitCodeAbbrevOp::Array, Terminator
-  };
-  CheckDumpEdits(
-      ARRAY(Edit),
-      "Error (Block 17): Array abbreviation must be second to last: 2: [65533,"
-      " 1, 0, 3]\n",
-      NoErrorRecoveryMessages,
-      ExpectedDumpedBitcode);
-}
-
-// Show that the array operator can't appear before the second to last
-// operand.
-TEST(NaClMungeWriteErrorTests, InvalidArrayAbbreviationTooEarly) {
-  const uint64_t Edit[] = {
-    VoidTypeIndex, NaClMungedBitcode::AddBefore,
-    naclbitc::DEFINE_ABBREV, naclbitc::BLK_CODE_DEFINE_ABBREV, 3,
-    0, NaClBitCodeAbbrevOp::Array,  // array
-    1, 15,                          // lit(15)
-    1, 10,                          // lit(10)
-    Terminator
-  };
-  CheckDumpEdits(
-      ARRAY(Edit),
-      "Error (Block 17): Array abbreviation must be second to last: 2: [65533,"
-      " 3, 0, 3, 1, 15, 1, 10]\n",
-      NoErrorRecoveryMessages,
-      ExpectedDumpedBitcode);
-}
-
-// Show that the array operator can't appear as last two operators.
-TEST(NaClMungeWriteErrorTests, InvalidArrayAbbreviationLastTwo) {
-  const uint64_t Edit[] = {
-    VoidTypeIndex, NaClMungedBitcode::AddBefore,
-    naclbitc::DEFINE_ABBREV, naclbitc::BLK_CODE_DEFINE_ABBREV, 2,
-    0, NaClBitCodeAbbrevOp::Array,  // array
-    0, NaClBitCodeAbbrevOp::Array,  // array
-    Terminator
-  };
-  CheckDumpEdits(
-      ARRAY(Edit),
-      "Error (Block 17): Array abbreviation must be second to last: 2: [65533,"
-      " 2, 0, 3, 0, 3]\n",
-      NoErrorRecoveryMessages,
-      ExpectedDumpedBitcode);
-}
-
-// Show what happens when an abbreviation is specified to only contain
-// one operator, but is then followed with more than one operator.
-TEST(NaClMungeWriteErrorTests, SpecifiesTooFewOperands) {
-  const uint64_t Edit[] = {
-    VoidTypeIndex, NaClMungedBitcode::AddBefore,
-    // Note: 1 at end of next line specified that the abbreviation
-    // should only have one operator.
-    naclbitc::DEFINE_ABBREV, naclbitc::BLK_CODE_DEFINE_ABBREV, 1,
-    1, 10,                          // lit(10)
-    1, 15,                          // lit(15)
-    Terminator
-  };
-  CheckDumpEdits(
-      ARRAY(Edit),
-      "Error (Block 17): Error: Too many values for number of operands (1):"
-      " 2: [65533, 1, 1, 10, 1, 15]\n",
-      NoErrorRecoveryMessages,
-      ExpectedDumpedBitcode);
-}
-
-// Show that the code checks if specifies too many operands for an
-// abbreviation, based on the record size.
-TEST(NaClMungeWriteErrorTests, SpecifiesTooManyOperands) {
-  // Insert bad abbreviation Vbr(36) into type block.
-  const uint64_t Edit[] = {
-    VoidTypeIndex, NaClMungedBitcode::AddBefore,
-    naclbitc::DEFINE_ABBREV, naclbitc::BLK_CODE_DEFINE_ABBREV, 3,
-    1, 10,                          // lit(10)
-    1, 15,                          // lit(15)
-    Terminator
-  };
-  CheckDumpEdits(
-      ARRAY(Edit),
-      "Error (Block 17): Malformed abbreviation found: 2: [65533, 3, 1, 10,"
-      " 1, 15]\n",
-      NoErrorRecoveryMessages,
-      ExpectedDumpedBitcode);
-}
-
-// Show what happens if an abbreviation definition is defined outside a block.
-TEST(NaClMungeWriteErrorTests, AbbreviationNotInBlock) {
-  // Add abbreviation before all records.
-  const uint64_t Edit[] = {
-    0, NaClMungedBitcode::AddBefore,
-    naclbitc::DEFINE_ABBREV, naclbitc::BLK_CODE_DEFINE_ABBREV, 1,
-    1, 10,                          // lit(10)
-    Terminator
-  };
-  CheckDumpEdits(
-      ARRAY(Edit),
-      "Error (Block unknown): Abbreviation definition not in block: 2:"
-      " [65533, 1, 1, 10]\n",
-      NoErrorRecoveryMessages,
-      ExpectedDumpedBitcode);
-}
-
-// Show what happens when the SetBID record (in a block-info block) is
-// malformed.
-TEST(NaClMungeWriteErrorTests, SetBIDWrongSize) {
-  // Build a block-info example before the types block.
-  const uint64_t TypesEnter = 1;
-  const uint64_t Edits[] = {
-    TypesEnter, NaClMungedBitcode::AddBefore,
-    // Enter the blockinfo block
-    1, naclbitc::BLK_CODE_ENTER, naclbitc::BLOCKINFO_BLOCK_ID, 2, Terminator,
-    TypesEnter, NaClMungedBitcode::AddBefore,
-    // Define a SetBID with too many arguments (2 instead of 1)
-    3, naclbitc::BLOCKINFO_CODE_SETBID, naclbitc::GLOBALVAR_BLOCK_ID, 2,
-    Terminator,
-    TypesEnter, NaClMungedBitcode::AddBefore,
-    // Add an abbreviation to block globalvar block.
-    naclbitc::DEFINE_ABBREV, naclbitc::BLK_CODE_DEFINE_ABBREV, 1,
-    1, 10, Terminator,              // lit(10)
-    TypesEnter, NaClMungedBitcode::AddBefore,
-    // Define a SetBID with too few arguments (none).
-    3, naclbitc::BLOCKINFO_CODE_SETBID, Terminator,
-    TypesEnter, NaClMungedBitcode::AddBefore,
-    // Add an abbreviation to the unknown block.
-    naclbitc::DEFINE_ABBREV, naclbitc::BLK_CODE_DEFINE_ABBREV, 1,
-    1, 20, Terminator,             // lit(20)
-    TypesEnter, NaClMungedBitcode::AddBefore,
-    // Exit the blockinfo block.
-    0, naclbitc::BLK_CODE_EXIT, Terminator
-  };
-  bool RecoveredTestHasErrors = true;
-  // TODO(kschimpf) Why don't we also get an error when running CheckParseEdits
-  // inside CheckDumpEdits?
-  CheckDumpEdits(
-      ARRAY(Edits),
-      "Error (Block 0): SetBID record expects 1 value but found 2: 3: [1,"
-      " 19, 2]\n",
-      "Error (Block 0): SetBID record expects 1 value but found 0: 3: [1]\n",
-      "       0:0|<65532, 80, 69, 88, 69, 1, 0,|Magic Number: 'PEXE' (80, 69,"
-      " 88, 69)\n"
-      "          | 8, 0, 17, 0, 4, 0, 2, 0, 0, |PNaCl Version: 2\n"
-      "          | 0>                          |\n"
-      "      16:0|1: <65535, 8, 2>             |module {  // BlockID = 8\n"
-      "      24:0|  1: <65535, 0, 2>           |  abbreviations {  // BlockID"
-      " = 0\n"
-      "      32:0|    3: <1, 19>               |    globals:\n"
-      "      34:4|    2: <65533, 1, 1, 10>     |      @a0 = abbrev <10>;\n"
-      "      36:4|    3: <1, 4294967295>       |    block(4294967295):\n"
-      "Error(34:4): Block id 4294967295 not understood.\n"
-      "      43:4|    2: <65533, 1, 1, 20>     |      @a0 = abbrev <20>;\n"
-      "      45:4|  0: <65534>                 |  }\n"
-      "      48:0|  1: <65535, 17, 3>          |  types {  // BlockID = 17\n"
-      "      56:0|    3: <1, 2>                |    count 2;\n"
-      "      58:5|    3: <2>                   |    @t0 = void;\n"
-      "      60:4|    3: <21, 0, 0>            |    @t1 = void ();\n"
-      "      63:7|  0: <65534>                 |  }\n"
-      "      68:0|  3: <8, 1, 0, 0, 0>         |  define external void @f0();\n"
-      "      72:6|  1: <65535, 12, 2>          |  function void @f0() {  \n"
-      "          |                             |                   // BlockID"
-      " = 12\n"
-      "      80:0|    3: <1, 1>                |    blocks 1;\n"
-      "          |                             |  %b0:\n"
-      "      82:4|    3: <10>                  |    ret void;\n"
-      "      84:2|  0: <65534>                 |  }\n"
-      "      88:0|0: <65534>                   |}\n",
-      RecoveredTestHasErrors);
-}
-
-// Show what happens when records other than setBID and abbreviation
-// definitions appear in a blockinfo block.
-TEST(NaClMungeWriteErrorTests, BlockInfoBlockWithUnknownRecord) {
-  // Build a block-info example before the types block.
-  const uint64_t TypesEnter = 1;
-  const uint64_t Edits[] = {
-    // Enter the blockinfo block
-    TypesEnter, NaClMungedBitcode::AddBefore,
-    1, naclbitc::BLK_CODE_ENTER, naclbitc::BLOCKINFO_BLOCK_ID, 2, Terminator,
-    // Define abbreviations for globalvar block.
-    TypesEnter, NaClMungedBitcode::AddBefore,
-    3, naclbitc::BLOCKINFO_CODE_SETBID, naclbitc::GLOBALVAR_BLOCK_ID,
-    Terminator,
-    // Add unknown record (i.e. not setBID).
-    TypesEnter, NaClMungedBitcode::AddBefore,
-    3, naclbitc::BLOCKINFO_CODE_SETBID + 3, 2, Terminator,
-    // Exit the blockinfo block.
-    TypesEnter, NaClMungedBitcode::AddBefore,
-    0, naclbitc::BLK_CODE_EXIT, Terminator
-  };
-  CheckDumpEdits(
-      ARRAY(Edits),
-      "Error (Block 0): Record not allowed in blockinfo block: 3: [4, 2]\n",
-      NoErrorRecoveryMessages,
-      "       0:0|<65532, 80, 69, 88, 69, 1, 0,|Magic Number: 'PEXE' (80, 69,"
-      " 88, 69)\n"
-      "          | 8, 0, 17, 0, 4, 0, 2, 0, 0, |PNaCl Version: 2\n"
-      "          | 0>                          |\n"
-      "      16:0|1: <65535, 8, 2>             |module {  // BlockID = 8\n"
-      "      24:0|  1: <65535, 0, 2>           |  abbreviations {  // BlockID"
-      " = 0\n"
-      "      32:0|  0: <65534>                 |  }\n"
-      "      36:0|  1: <65535, 17, 3>          |  types {  // BlockID = 17\n"
-      "      44:0|    3: <1, 2>                |    count 2;\n"
-      "      46:5|    3: <2>                   |    @t0 = void;\n"
-      "      48:4|    3: <21, 0, 0>            |    @t1 = void ();\n"
-      "      51:7|  0: <65534>                 |  }\n"
-      "      56:0|  3: <8, 1, 0, 0, 0>         |  define external void @f0();\n"
-      "      60:6|  1: <65535, 12, 2>          |  function void @f0() {  \n"
-      "          |                             |                   // BlockID"
-      " = 12\n"
-      "      68:0|    3: <1, 1>                |    blocks 1;\n"
-      "          |                             |  %b0:\n"
-      "      70:4|    3: <10>                  |    ret void;\n"
-      "      72:2|  0: <65534>                 |  }\n"
-      "      76:0|0: <65534>                   |}\n");
-}
-
-// Show what happens when a block is nested within a blockinfo block.
-TEST(NaClMungeWriteErrorTests, BlockWithinBlockInfoBlock) {
-  // Build a block-info example before the types block.
-  const uint64_t TypesEnter = 1;
-  const uint64_t Edits[] = {
-    // Enter the blockinfo block
-    TypesEnter, NaClMungedBitcode::AddBefore,
-    1, naclbitc::BLK_CODE_ENTER, naclbitc::BLOCKINFO_BLOCK_ID, 2, Terminator,
-    // Create a globalvar block (which will be ignored by error recovery).
-    TypesEnter, NaClMungedBitcode::AddBefore,
-    1, naclbitc::BLK_CODE_ENTER, naclbitc::GLOBALVAR_BLOCK_ID, 2, Terminator,
-    // Exit the globalvar block.
-    TypesEnter, NaClMungedBitcode::AddBefore,
-    0, naclbitc::BLK_CODE_EXIT, Terminator,
-    // Exit the blockinfo block.
-    TypesEnter, NaClMungedBitcode::AddBefore,
-    0, naclbitc::BLK_CODE_EXIT, Terminator
-  };
-  CheckWriteEdits(
-      ARRAY(Edits),
-      "Error (Block 0): Can't nest blocks inside blockinfo block: 1: [65535,"
-      " 19, 2]\n",
-      "Error (Block unknown): Record outside block: 3: [8, 1, 0, 0, 0]\n",
-      "       1: [65535, 8, 2]\n"
-      "         1: [65535, 0, 2]\n"
-      "         0: [65534]\n"
-      "       0: [65534]\n"
-      "       1: [65535, 17, 3]\n"
-      "         3: [1, 2]\n"
-      "         3: [2]\n"
-      "         3: [21, 0, 0]\n"
-      "       0: [65534]\n"
-      "       1: [65535, 4294967295, 3]\n"
-      "         3: [8, 1, 0, 0, 0]\n"
-      "         1: [65535, 12, 2]\n"
-      "           3: [1, 1]\n"
-      "           3: [10]\n"
-      "         0: [65534]\n"
-      "       0: [65534]\n");
-}
-
-} // end of namespace naclmungetest
diff --git a/unittests/Bitcode/NaClMungedBitcodeTest.cpp b/unittests/Bitcode/NaClMungedBitcodeTest.cpp
deleted file mode 100644
index f7b8f46e726..00000000000
--- a/unittests/Bitcode/NaClMungedBitcodeTest.cpp
+++ /dev/null
@@ -1,559 +0,0 @@
-//===- llvm/unittest/Bitcode/NaClMungedBitcodeTest.cpp -------------------===//
-//     Tests munging NaCl bitcode records.
-//
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
-//
-//===----------------------------------------------------------------------===//
-
-// Tests munging NaCl bitcode records.
-
-#include "NaClMungeTest.h"
-
-#include <limits>
-
-using namespace llvm;
-
-namespace naclmungetest {
-
-TEST(NaClMungedBitcodeTest, TestInsertBefore) {
-  const uint64_t Records[] = {
-    1, 2, 3, Terminator,
-    4, 5, Terminator,
-    6, 7, 8 , 9, Terminator,
-    10, 11, Terminator
-  };
-  NaClMungedBitcode MungedRecords(ARRAY_TERM(Records));
-  EXPECT_EQ(
-      "       1: [2, 3]\n"
-      "       4: [5]\n"
-      "       6: [7, 8, 9]\n"
-      "      10: [11]\n",
-      stringify(MungedRecords));
-
-  // Add a record before the second record.
-  const uint64_t BeforeSecond[] = {
-    1, NaClMungedBitcode::AddBefore, 12, 13, 14, Terminator
-  };
-  MungedRecords.munge(ARRAY_TERM(BeforeSecond));
-  EXPECT_EQ(
-      "       1: [2, 3]\n"
-      "      12: [13, 14]\n"
-      "       4: [5]\n"
-      "       6: [7, 8, 9]\n"
-      "      10: [11]\n",
-      stringify(MungedRecords));
-
-  // Add two more records before the second record.
-  const uint64_t BeforeSecondMore[] = {
-    1, NaClMungedBitcode::AddBefore, 15, 16, 17, Terminator,
-    1, NaClMungedBitcode::AddBefore, 18, 19, Terminator
-  };
-  MungedRecords.munge(ARRAY_TERM(BeforeSecondMore));
-  EXPECT_EQ(
-      "       1: [2, 3]\n"
-      "      12: [13, 14]\n"
-      "      15: [16, 17]\n"
-      "      18: [19]\n"
-      "       4: [5]\n"
-      "       6: [7, 8, 9]\n"
-      "      10: [11]\n",
-      stringify(MungedRecords));
-
-  // Add two records before the last record.
-  const uint64_t BeforeLast[] = {
-    3, NaClMungedBitcode::AddBefore, 21, 22, 23, Terminator,
-    3, NaClMungedBitcode::AddBefore, 24, 25, 26, 27, Terminator
-  };
-  MungedRecords.munge(ARRAY_TERM(BeforeLast));
-  EXPECT_EQ(
-      "       1: [2, 3]\n"
-      "      12: [13, 14]\n"
-      "      15: [16, 17]\n"
-      "      18: [19]\n"
-      "       4: [5]\n"
-      "       6: [7, 8, 9]\n"
-      "      21: [22, 23]\n"
-      "      24: [25, 26, 27]\n"
-      "      10: [11]\n",
-      stringify(MungedRecords));
-}
-
-TEST(NaClMungedBitcodeTest, TestInsertAfter) {
-  const uint64_t Records[] = {
-    1, 2, 3, Terminator,
-    4, 5, Terminator,
-    6, 7, 8 , 9, Terminator,
-    10, 11, Terminator
-  };
-  NaClMungedBitcode MungedRecords(ARRAY_TERM(Records));
-  EXPECT_EQ(
-      "       1: [2, 3]\n"
-      "       4: [5]\n"
-      "       6: [7, 8, 9]\n"
-      "      10: [11]\n",
-      stringify(MungedRecords));
-
-  // Add a record after the second record.
-  const uint64_t AfterSecond[] = {
-    1, NaClMungedBitcode::AddAfter, 12, 13, 14, Terminator
-  };
-  MungedRecords.munge(ARRAY_TERM(AfterSecond));
-  EXPECT_EQ(
-      "       1: [2, 3]\n"
-      "       4: [5]\n"
-      "      12: [13, 14]\n"
-      "       6: [7, 8, 9]\n"
-      "      10: [11]\n",
-      stringify(MungedRecords));
-
-  // Add two more records after the second record.
-  const uint64_t AfterSecondMore[] = {
-    1, NaClMungedBitcode::AddAfter, 15, 16, 17, Terminator,
-    1, NaClMungedBitcode::AddAfter, 18, 19, Terminator
-  };
-  MungedRecords.munge(ARRAY_TERM(AfterSecondMore));
-  EXPECT_EQ(
-      "       1: [2, 3]\n"
-      "       4: [5]\n"
-      "      12: [13, 14]\n"
-      "      15: [16, 17]\n"
-      "      18: [19]\n"
-      "       6: [7, 8, 9]\n"
-      "      10: [11]\n",
-      stringify(MungedRecords));
-
-  // Add two records after the last record.
-  const uint64_t AfterLast[] = {
-    3, NaClMungedBitcode::AddAfter, 21, 22, 23, Terminator,
-    3, NaClMungedBitcode::AddAfter, 24, 25, 26, 27, Terminator
-  };
-  MungedRecords.munge(ARRAY_TERM(AfterLast));
-  EXPECT_EQ(
-      "       1: [2, 3]\n"
-      "       4: [5]\n"
-      "      12: [13, 14]\n"
-      "      15: [16, 17]\n"
-      "      18: [19]\n"
-      "       6: [7, 8, 9]\n"
-      "      10: [11]\n"
-      "      21: [22, 23]\n"
-      "      24: [25, 26, 27]\n",
-      stringify(MungedRecords));
-}
-
-TEST(NaClMungedBitcodeTest, TestRemove) {
-  const uint64_t Records[] = {
-    1, 2, 3, Terminator,
-    4, 5, Terminator,
-    6, 7, 8 , 9, Terminator,
-    10, 11, Terminator
-  };
-  NaClMungedBitcode MungedRecords(ARRAY_TERM(Records));
-  EXPECT_EQ(
-      "       1: [2, 3]\n"
-      "       4: [5]\n"
-      "       6: [7, 8, 9]\n"
-      "      10: [11]\n",
-      stringify(MungedRecords));
-
-  // Remove the second record.
-  const uint64_t RemoveSecond[] = {
-    1, NaClMungedBitcode::Remove
-  };
-  MungedRecords.munge(ARRAY_TERM(RemoveSecond));
-  EXPECT_EQ(
-      "       1: [2, 3]\n"
-      "       6: [7, 8, 9]\n"
-      "      10: [11]\n",
-      stringify(MungedRecords));
-
-  // Remove first and last records.
-  const uint64_t RemoveEnds[] = {
-    0, NaClMungedBitcode::Remove,
-    3, NaClMungedBitcode::Remove
-  };
-  MungedRecords.munge(ARRAY_TERM(RemoveEnds));
-  EXPECT_EQ(
-      "       6: [7, 8, 9]\n",
-      stringify(MungedRecords));
-
-  // Remove remaining record.
-  const uint64_t RemoveOther[] = {
-    2, NaClMungedBitcode::Remove
-  };
-  MungedRecords.munge(ARRAY_TERM(RemoveOther));
-  EXPECT_EQ(
-      "",
-      stringify(MungedRecords));
-}
-
-TEST(NaClMungedBitcodeTest, TestReplace) {
-  const uint64_t Records[] = {
-    1, 2, 3, Terminator,
-    4, 5, Terminator,
-    6, 7, 8 , 9, Terminator,
-    10, 11, Terminator
-  };
-  NaClMungedBitcode MungedRecords(ARRAY_TERM(Records));
-  EXPECT_EQ(
-      "       1: [2, 3]\n"
-      "       4: [5]\n"
-      "       6: [7, 8, 9]\n"
-      "      10: [11]\n",
-      stringify(MungedRecords));
-
-  // Replace the second record.
-  const uint64_t ReplaceSecond[] = {
-    1, NaClMungedBitcode::Replace, 12, 13, 14, Terminator
-  };
-  MungedRecords.munge(ARRAY_TERM(ReplaceSecond));
-  EXPECT_EQ(
-      "       1: [2, 3]\n"
-      "      12: [13, 14]\n"
-      "       6: [7, 8, 9]\n"
-      "      10: [11]\n",
-      stringify(MungedRecords));
-
-  // Replace the first and last record.
-  const uint64_t ReplaceEnds[] = {
-    0, NaClMungedBitcode::Replace, 15, 16, 17, 18, Terminator,
-    3, NaClMungedBitcode::Replace, 19, 20, Terminator
-  };
-  MungedRecords.munge(ARRAY_TERM(ReplaceEnds));
-  EXPECT_EQ(
-      "      15: [16, 17, 18]\n"
-      "      12: [13, 14]\n"
-      "       6: [7, 8, 9]\n"
-      "      19: [20]\n",
-      stringify(MungedRecords));
-
-  // Replace the first three records, which includes two already replaced
-  // records.
-  const uint64_t ReplaceFirst3[] = {
-    0, NaClMungedBitcode::Replace, 21, 22, 23, Terminator,
-    1, NaClMungedBitcode::Replace, 24, 25, Terminator,
-    2, NaClMungedBitcode::Replace, 26, 27, 28, 29, Terminator
-  };
-  MungedRecords.munge(ARRAY_TERM(ReplaceFirst3));
-  EXPECT_EQ(
-      "      21: [22, 23]\n"
-      "      24: [25]\n"
-      "      26: [27, 28, 29]\n"
-      "      19: [20]\n",
-      stringify(MungedRecords));
-
-  // Show that we can remove replaced records.
-  const uint64_t RemoveReplaced[] = {
-    1, NaClMungedBitcode::Remove,
-    3, NaClMungedBitcode::Remove
-  };
-  MungedRecords.munge(ARRAY_TERM(RemoveReplaced));
-  EXPECT_EQ(
-      "      21: [22, 23]\n"
-      "      26: [27, 28, 29]\n",
-      stringify(MungedRecords));
-}
-
-TEST(NaClMungedBitcodeTest, TestBlockStructure) {
-  const uint64_t Records[] = {
-    1, 2, 3, 4, Terminator,
-    5, naclbitc::BLK_CODE_ENTER, 6, Terminator,
-    7, 8, Terminator,
-    9, naclbitc::BLK_CODE_ENTER, 10, Terminator,
-    11, 12, 13, Terminator,
-    14, naclbitc::BLK_CODE_EXIT, Terminator,
-    15, naclbitc::BLK_CODE_ENTER, 16, Terminator,
-    17, naclbitc::BLK_CODE_EXIT, 18, Terminator,
-    19, 20, 21, Terminator,
-    22, naclbitc::BLK_CODE_EXIT, Terminator
-  };
-  NaClMungedBitcode MungedRecords(ARRAY_TERM(Records));
-  EXPECT_EQ(
-      "       1: [2, 3, 4]\n"
-      "       5: [65535, 6]\n"
-      "         7: [8]\n"
-      "         9: [65535, 10]\n"
-      "          11: [12, 13]\n"
-      "        14: [65534]\n"
-      "        15: [65535, 16]\n"
-      "        17: [65534, 18]\n"
-      "        19: [20, 21]\n"
-      "      22: [65534]\n",
-      stringify(MungedRecords));
-
-  // Show what happens if you have unbalanced blocks.
-  const uint64_t ExitEdits[] = {
-    4, NaClMungedBitcode::AddAfter, 0, naclbitc::BLK_CODE_EXIT, Terminator,
-    4, NaClMungedBitcode::AddAfter, 0, naclbitc::BLK_CODE_EXIT, Terminator,
-    2, NaClMungedBitcode::Replace, 0, naclbitc::BLK_CODE_EXIT, Terminator
-  };
-  MungedRecords.munge(ARRAY_TERM(ExitEdits));
-  EXPECT_EQ(
-      "       1: [2, 3, 4]\n"
-      "       5: [65535, 6]\n"
-      "       0: [65534]\n"
-      "       9: [65535, 10]\n"
-      "        11: [12, 13]\n"
-      "       0: [65534]\n"
-      "       0: [65534]\n"
-      "      14: [65534]\n"
-      "      15: [65535, 16]\n"
-      "      17: [65534, 18]\n"
-      "      19: [20, 21]\n"
-      "      22: [65534]\n",
-      stringify(MungedRecords));
-}
-
-// Tests that replace/remove superceed other replace/removes at same
-// record index.
-TEST(NaClMungedBitcodeTest, TestReplaceRemoveEffects) {
-  const uint64_t Records[] = {
-    1, 2, 3, Terminator,
-    4, 5, Terminator,
-    6, 7, 8 , 9, Terminator,
-    10, 11, Terminator
-  };
-  NaClMungedBitcode MungedRecords(ARRAY_TERM(Records));
-  EXPECT_EQ(
-      "       1: [2, 3]\n"
-      "       4: [5]\n"
-      "       6: [7, 8, 9]\n"
-      "      10: [11]\n",
-      stringify(MungedRecords));
-
-  // Remove the second record.
-  const uint64_t RemoveSecond[] = {
-    1, NaClMungedBitcode::Remove
-  };
-  MungedRecords.munge(ARRAY_TERM(RemoveSecond));
-  EXPECT_EQ(
-      "       1: [2, 3]\n"
-      "       6: [7, 8, 9]\n"
-      "      10: [11]\n",
-      stringify(MungedRecords));
-
-  // Try it again. Should have no effect.
-  MungedRecords.munge(ARRAY_TERM(RemoveSecond));
-  EXPECT_EQ(
-      "       1: [2, 3]\n"
-      "       6: [7, 8, 9]\n"
-      "      10: [11]\n",
-      stringify(MungedRecords));
-
-  // Override removed record with a replacement.
-  const uint64_t ReplaceSecond[] = {
-    1, NaClMungedBitcode::Replace, 12, 12, 14, 15, Terminator
-  };
-  MungedRecords.munge(ARRAY_TERM(ReplaceSecond));
-  EXPECT_EQ(
-      "       1: [2, 3]\n"
-      "      12: [12, 14, 15]\n"
-      "       6: [7, 8, 9]\n"
-      "      10: [11]\n",
-      stringify(MungedRecords));
-
-  // Override replacement with a different replacement.
-  const uint64_t ReplaceSecondAgain[] = {
-    1, NaClMungedBitcode::Replace, 16, 17, 18, Terminator
-  };
-  MungedRecords.munge(ARRAY_TERM(ReplaceSecondAgain));
-  EXPECT_EQ(
-      "       1: [2, 3]\n"
-      "      16: [17, 18]\n"
-      "       6: [7, 8, 9]\n"
-      "      10: [11]\n",
-      stringify(MungedRecords));
-
-  // Override replacement with a remove.
-  MungedRecords.munge(ARRAY_TERM(RemoveSecond));
-  EXPECT_EQ(
-      "       1: [2, 3]\n"
-      "       6: [7, 8, 9]\n"
-      "      10: [11]\n",
-      stringify(MungedRecords));
-}
-
-// Show how before/after interact between neighboring indices
-TEST(NaClMungedBitcodeTest, TestBeforeAfterInteraction) {
-  const uint64_t Records[] = {
-    1, 2, 3, Terminator,
-    4, 5, Terminator,
-    6, 7, 8 , 9, Terminator,
-    10, 11, Terminator
-  };
-  NaClMungedBitcode MungedRecords(ARRAY_TERM(Records));
-  EXPECT_EQ(
-      "       1: [2, 3]\n"
-      "       4: [5]\n"
-      "       6: [7, 8, 9]\n"
-      "      10: [11]\n",
-      stringify(MungedRecords));
-
-  // Add record before the third record.
-  const uint64_t AddBeforeThird[] = {
-    2, NaClMungedBitcode::AddBefore, 12, 13, 14, Terminator
-  };
-  MungedRecords.munge(ARRAY_TERM(AddBeforeThird));
-  EXPECT_EQ(
-      "       1: [2, 3]\n"
-      "       4: [5]\n"
-      "      12: [13, 14]\n"
-      "       6: [7, 8, 9]\n"
-      "      10: [11]\n",
-      stringify(MungedRecords));
-
-  // Add record after the second record.
-  const uint64_t AddAfterSecond[] = {
-    1, NaClMungedBitcode::AddAfter, 15, 16, 17, 18, Terminator
-  };
-  MungedRecords.munge(ARRAY_TERM(AddAfterSecond));
-  EXPECT_EQ(
-      "       1: [2, 3]\n"
-      "       4: [5]\n"
-      "      15: [16, 17, 18]\n"
-      "      12: [13, 14]\n"
-      "       6: [7, 8, 9]\n"
-      "      10: [11]\n",
-      stringify(MungedRecords));
-
-  // Add more records before the third record.
-  const uint64_t AddBeforeThirdMore[] = {
-    2, NaClMungedBitcode::AddBefore, 19, 20, Terminator,
-    2, NaClMungedBitcode::AddBefore, 21, 22, Terminator
-  };
-  MungedRecords.munge(ARRAY_TERM(AddBeforeThirdMore));
-  EXPECT_EQ(
-      "       1: [2, 3]\n"
-      "       4: [5]\n"
-      "      15: [16, 17, 18]\n"
-      "      12: [13, 14]\n"
-      "      19: [20]\n"
-      "      21: [22]\n"
-      "       6: [7, 8, 9]\n"
-      "      10: [11]\n",
-      stringify(MungedRecords));
-
-  // Add more records after the second record.
-  const uint64_t AddAfterSecondMore[] = {
-    1, NaClMungedBitcode::AddAfter, 23, 24, 25, Terminator,
-    1, NaClMungedBitcode::AddAfter, 26, 27, 28, 29, Terminator
-  };
-  MungedRecords.munge(ARRAY_TERM(AddAfterSecondMore));
-  EXPECT_EQ(
-      "       1: [2, 3]\n"
-      "       4: [5]\n"
-      "      15: [16, 17, 18]\n"
-      "      23: [24, 25]\n"
-      "      26: [27, 28, 29]\n"
-      "      12: [13, 14]\n"
-      "      19: [20]\n"
-      "      21: [22]\n"
-      "       6: [7, 8, 9]\n"
-      "      10: [11]\n",
-      stringify(MungedRecords));
-}
-
-// Do a sample combination of all possible edits.
-TEST(NaClMungedBitcodeTest, CombinationEdits) {
-  const uint64_t Records[] = {
-    1, 2, 3, Terminator,
-    4, 5, Terminator,
-    6, 7, 8 , 9, Terminator,
-    10, 11, Terminator
-  };
-
-  NaClMungedBitcode MungedRecords(ARRAY_TERM(Records));
-  EXPECT_EQ(
-      "       1: [2, 3]\n"
-      "       4: [5]\n"
-      "       6: [7, 8, 9]\n"
-      "      10: [11]\n",
-      stringify(MungedRecords));
-
-  // Remove First
-  const uint64_t RemoveFirst[] = {
-    0, NaClMungedBitcode::Remove
-  };
-  MungedRecords.munge(ARRAY_TERM(RemoveFirst));
-  EXPECT_EQ(
-      "       4: [5]\n"
-      "       6: [7, 8, 9]\n"
-      "      10: [11]\n",
-      stringify(MungedRecords));
-
-  // Add records after the first (base) record, which corresponds to
-  // before the first record in the munged result.
-  const uint64_t AddAfterFirst[] = {
-    0, NaClMungedBitcode::AddAfter, 12, 13, 14, Terminator,
-    0, NaClMungedBitcode::AddAfter, 15, 16, Terminator
-  };
-  MungedRecords.munge(ARRAY_TERM(AddAfterFirst));
-  EXPECT_EQ(
-      "      12: [13, 14]\n"
-      "      15: [16]\n"
-      "       4: [5]\n"
-      "       6: [7, 8, 9]\n"
-      "      10: [11]\n",
-      stringify(MungedRecords));
-
-  // Add records before the second (base) record, which corresponds to
-  // before the third record in the munged result.
-  const uint64_t AddBeforeSecond[] = {
-    1, NaClMungedBitcode::AddBefore, 17, 18, 19, 20, Terminator,
-    1, NaClMungedBitcode::AddBefore, 21, 22, 23, Terminator
-  };
-  MungedRecords.munge(ARRAY_TERM(AddBeforeSecond));
-  EXPECT_EQ(
-      "      12: [13, 14]\n"
-      "      15: [16]\n"
-      "      17: [18, 19, 20]\n"
-      "      21: [22, 23]\n"
-      "       4: [5]\n"
-      "       6: [7, 8, 9]\n"
-      "      10: [11]\n",
-      stringify(MungedRecords));
-
-  // Put the first (base) record back, which will also be the first
-  // record in the munged result.
-  const uint64_t ReplaceFirst[] = {
-    0, NaClMungedBitcode::Replace, 1, 2, 3, Terminator
-  };
-  MungedRecords.munge(ARRAY_TERM(ReplaceFirst));
-  EXPECT_EQ(
-      "       1: [2, 3]\n"
-      "      12: [13, 14]\n"
-      "      15: [16]\n"
-      "      17: [18, 19, 20]\n"
-      "      21: [22, 23]\n"
-      "       4: [5]\n"
-      "       6: [7, 8, 9]\n"
-      "      10: [11]\n",
-      stringify(MungedRecords));
-
-  // Add before the first (base) record, which will also be before all
-  // other records in the munged result.
-  const uint64_t AddBeforeFirst[] = {
-    0, NaClMungedBitcode::AddBefore, 24, 25, 26, 27, Terminator,
-    0, NaClMungedBitcode::AddBefore, 28, 29, Terminator,
-    0, NaClMungedBitcode::AddBefore, 30, 31, 32, Terminator
-  };
-  MungedRecords.munge(ARRAY_TERM(AddBeforeFirst));
-  EXPECT_EQ(
-      "      24: [25, 26, 27]\n"
-      "      28: [29]\n"
-      "      30: [31, 32]\n"
-      "       1: [2, 3]\n"
-      "      12: [13, 14]\n"
-      "      15: [16]\n"
-      "      17: [18, 19, 20]\n"
-      "      21: [22, 23]\n"
-      "       4: [5]\n"
-      "       6: [7, 8, 9]\n"
-      "      10: [11]\n",
-      stringify(MungedRecords));
-}
-
-} // end of namespace naclmungetest
diff --git a/unittests/Bitcode/NaClMungedIoTest.cpp b/unittests/Bitcode/NaClMungedIoTest.cpp
deleted file mode 100644
index 7b2d57ebb5c..00000000000
--- a/unittests/Bitcode/NaClMungedIoTest.cpp
+++ /dev/null
@@ -1,166 +0,0 @@
-//===- llvm/unittest/Bitcode/NaClMungedIoTest.cpp -------------------------===//
-//     Tests munging NaCl bitcode records.
-//
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
-//
-//===----------------------------------------------------------------------===//
-
-// For class NaClMungedBitcode, tests reading initial sequence of records and
-// writing out munged set of bitcode records.
-
-#include "NaClMungeTest.h"
-
-namespace naclmungetest {
-
-using namespace llvm;
-
-typedef SmallVector<char, 1024> TextBuffer;
-
-// Writes out a sequence of munged bitcode records, and writes them into
-// the text buffer. Returns a corresponding memory buffer containing
-// the munged bitcode records.
-std::unique_ptr<MemoryBuffer> writeMungedBitcode(
-    NaClMungedBitcode &Bitcode, TextBuffer &Buffer,
-    NaClMungedBitcode::WriteFlags &Flags) {
-  Bitcode.write(Buffer, /* AddHeader = */ true, Flags);
-  StringRef Input(Buffer.data(), Buffer.size());
-  return MemoryBuffer::getMemBuffer(Input, "Test", false);
-}
-
-std::unique_ptr<MemoryBuffer> writeMungedBitcode(
-    NaClMungedBitcode &Bitcode, TextBuffer &Buffer) {
-  NaClMungedBitcode::WriteFlags Flags;
-  return writeMungedBitcode(Bitcode, Buffer, Flags);
-}
-
-// Write out the bitcode, parse it back, and return the resulting
-// munged bitcode.
-std::string parseWrittenMungedBitcode(NaClMungedBitcode &OutBitcode) {
-  TextBuffer Buffer;
-  NaClMungedBitcode InBitcode(writeMungedBitcode(OutBitcode, Buffer));
-  return stringify(InBitcode);
-}
-
-// Sample toy bitcode records.
-const uint64_t Records[] = {
-  1, naclbitc::BLK_CODE_ENTER, 8, 2, Terminator,
-  3, naclbitc::MODULE_CODE_VERSION, 1, Terminator,
-  1, naclbitc::BLK_CODE_ENTER, 0, 2, Terminator,
-  3, naclbitc::BLOCKINFO_CODE_SETBID, 12, Terminator,
-  2, naclbitc::BLK_CODE_DEFINE_ABBREV, 1, 1, 10, Terminator,
-  0, naclbitc::BLK_CODE_EXIT, Terminator,
-  1, naclbitc::BLK_CODE_ENTER, 17, 3, Terminator,
-  2, naclbitc::BLK_CODE_DEFINE_ABBREV, 4, 1, 21, 0, 1, 1, 0, 3,
-    0, 1, 2, Terminator,
-  3, naclbitc::TYPE_CODE_NUMENTRY, 2, Terminator,
-  3, naclbitc::TYPE_CODE_VOID, Terminator,
-  4, naclbitc::TYPE_CODE_FUNCTION, 0, 0, Terminator,
-  0, naclbitc::BLK_CODE_EXIT, Terminator,
-  3, naclbitc::MODULE_CODE_FUNCTION, 1, 0, 0, 3, Terminator,
-  1, naclbitc::BLK_CODE_ENTER, 12, 3, Terminator,
-  3, naclbitc::FUNC_CODE_DECLAREBLOCKS, 1, Terminator,
-  4, naclbitc::FUNC_CODE_INST_RET, Terminator,
-  0, naclbitc::BLK_CODE_EXIT, Terminator,
-  0, naclbitc::BLK_CODE_EXIT, Terminator,
-};
-
-// Show a more readable form of what the program is.
-TEST(NaClMungedIoTest, TestDumpingBitcode) {
-  NaClObjDumpMunger DumpMunger(ARRAY_TERM(Records));
-  EXPECT_TRUE(DumpMunger.runTest());
-  EXPECT_EQ(
-      "       0:0|<65532, 80, 69, 88, 69, 1, 0,|Magic Number: 'PEXE' (80, 69, "
-      "88, 69)\n"
-      "          | 8, 0, 17, 0, 4, 0, 2, 0, 0, |PNaCl Version: 2\n"
-      "          | 0>                          |\n"
-      "      16:0|1: <65535, 8, 2>             |module {  // BlockID = 8\n"
-      "      24:0|  3: <1, 1>                  |  version 1;\n"
-      "      26:4|  1: <65535, 0, 2>           |  abbreviations {  // BlockID "
-      "= 0\n"
-      "      36:0|    3: <1, 12>               |    function:\n"
-      "      38:4|    2: <65533, 1, 1, 10>     |      @a0 = abbrev <10>;\n"
-      "      40:4|  0: <65534>                 |  }\n"
-      "      44:0|  1: <65535, 17, 3>          |  types {  // BlockID = 17\n"
-      "      52:0|    2: <65533, 4, 1, 21, 0,  |    %a0 = abbrev <21, fixed(1),"
-      " \n"
-      "          |        1, 1, 0, 3, 0, 1, 2> |                  array(fixed("
-      "2))>;\n"
-      "      56:7|    3: <1, 2>                |    count 2;\n"
-      "      59:4|    3: <2>                   |    @t0 = void;\n"
-      "      61:3|    4: <21, 0, 0>            |    @t1 = void (); <%a0>\n"
-      "      62:7|  0: <65534>                 |  }\n"
-      "      64:0|  3: <8, 1, 0, 0, 3>         |  define internal void @f0();\n"
-      "      68:6|  1: <65535, 12, 3>          |  function void @f0() {  \n"
-      "          |                             |                   // BlockID "
-      "= 12\n"
-      "      76:0|    3: <1, 1>                |    blocks 1;\n"
-      "          |                             |  %b0:\n"
-      "      78:5|    4: <10>                  |    ret void; <@a0>\n"
-      "      79:0|  0: <65534>                 |  }\n"
-      "      80:0|0: <65534>                   |}\n",
-      DumpMunger.getTestResults());
-}
-
-// Test that we can write out bitcode, and then read it back in.
-TEST(NaClMungedIoTest, TestWriteThenRead) {
-  // Create munged bitcode for the given records.
-  NaClMungedBitcode Bitcode(ARRAY_TERM(Records));
-
-  // The expected output when stringifying this input.
-  const std::string ExpectedRecords(
-      "       1: [65535, 8, 2]\n"
-      "         3: [1, 1]\n"
-      "         1: [65535, 0, 2]\n"
-      "           3: [1, 12]\n"
-      "           2: [65533, 1, 1, 10]\n"
-      "         0: [65534]\n"
-      "         1: [65535, 17, 3]\n"
-      "           2: [65533, 4, 1, 21, 0, 1, 1, 0, 3, 0, 1, 2]\n"
-      "           3: [1, 2]\n"
-      "           3: [2]\n"
-      "           4: [21, 0, 0]\n"
-      "         0: [65534]\n"
-      "         3: [8, 1, 0, 0, 3]\n"
-      "         1: [65535, 12, 3]\n"
-      "           3: [1, 1]\n"
-      "           4: [10]\n"
-      "         0: [65534]\n"
-      "       0: [65534]\n");
-  EXPECT_EQ(ExpectedRecords, stringify(Bitcode));
-
-  // Write and read the bitcode back into a sequence of records.
-  EXPECT_EQ(ExpectedRecords, parseWrittenMungedBitcode(Bitcode));
-}
-
-
-// Test that writing truncated bitcode is difficult, due to word
-// alignment requirements for bitcode files. Note: Bitcode files must
-// be divisible by 4.
-TEST(NaClMungedIoTest, TestTruncatedNonalignedBitcode) {
-  // Created an example of a truncated bitcode file.
-  NaClMungedBitcode Bitcode(ARRAY_TERM(Records));
-  for (size_t i = 2, e = Bitcode.getBaseRecords().size(); i < e; ++i)
-    Bitcode.remove(i);
-
-  // The expected output when stringifying this input.
-  EXPECT_EQ(
-      "       1: [65535, 8, 2]\n"
-      "         3: [1, 1]\n",
-      stringify(Bitcode));
-
-  // Show that we can't write the bitcode correctly.
-  TextBuffer WriteBuffer;
-  std::string LogBuffer;
-  raw_string_ostream StrBuf(LogBuffer);
-  NaClMungedBitcode::WriteFlags Flags;
-  Flags.setErrStream(StrBuf);
-  writeMungedBitcode(Bitcode, WriteBuffer, Flags);
-  EXPECT_EQ(
-      "Error (Block 8): Missing close block.\n",
-      StrBuf.str());
-}
-
-} // end of namespace naclmungetest
diff --git a/unittests/Bitcode/NaClObjDumpTest.cpp b/unittests/Bitcode/NaClObjDumpTest.cpp
deleted file mode 100644
index dc382b4cf1d..00000000000
--- a/unittests/Bitcode/NaClObjDumpTest.cpp
+++ /dev/null
@@ -1,789 +0,0 @@
-//===- llvm/unittest/Bitcode/NaClObjDumpTest.cpp -------------------------===//
-//     Tests objdump stream for PNaCl bitcode.
-//
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
-//
-//===----------------------------------------------------------------------===//
-
-// Tests if the objdump stream for PNaCl bitcode works as expected.
-
-#include "llvm/Bitcode/NaCl/NaClObjDumpStream.h"
-
-#include "gtest/gtest.h"
-
-#include <iostream>
-
-using namespace llvm;
-using namespace llvm::naclbitc;
-
-namespace {
-
-// Writes out the record, if non-null. Otherwise just writes comments
-// and errors.
-static inline void Write(ObjDumpStream &Stream, uint64_t Bit,
-                         const NaClBitcodeRecordData *Record,
-                         int32_t AbbrevIndex) {
-  if (Record)
-    Stream.Write(Bit, *Record, AbbrevIndex);
-  else
-    Stream.Flush();
-}
-
-// Runs some simple assembly examples against the given bitcode
-// record, using an objdump stream.
-static void RunAssemblyExamples(
-    ObjDumpStream &Stream, uint64_t Bit,
-    const NaClBitcodeRecordData *Record,
-    int32_t AbbrevIndex,
-    bool AddErrors) {
-
-  // First assume no assembly.
-  if (AddErrors)
-    Stream.ErrorAt(Bit) << "This is an error\n";
-  Write(Stream, Bit, Record, AbbrevIndex);
-  // Increment bit to new fictitious address, assuming Record takes 21 bits.
-  Bit += 21;
-
-  // Now a single line assembly.
-  if (AddErrors)
-    Stream.ErrorAt(Bit) << "Oops, an error!\n";
-  Stream.Assembly() << "One line assembly.";
-  Write(Stream, Bit, Record, AbbrevIndex);
-  // Increment bit to new fictitious address, assuming Record takes 17 bits.
-  Bit += 17;
-
-  // Now multiple line assembly.
-  if (AddErrors)
-    Stream.ErrorAt(Bit) << "The record looks bad\n";
-  Stream.Assembly() << "Two Line\nexample assembly.";
-  if (AddErrors)
-    Stream.ErrorAt(Bit) << "Actually, it looks really bad\n";
-  Write(Stream, Bit, Record, AbbrevIndex);
-}
-
-// Runs some simple assembly examples against the given bitcode record
-// using an objdump stream. Adds a message describing the test
-// and the record indent being used.
-static std::string RunIndentedAssemblyWithAbbrevTest(
-    bool DumpRecords, bool DumpAssembly,
-    unsigned NumRecordIndents, uint64_t Bit,
-    const NaClBitcodeRecordData *Record, int32_t AbbrevIndex, bool AddErrors) {
-  std::string Buffer;
-  raw_string_ostream BufStream(Buffer);
-  ObjDumpStream DumpStream(BufStream, DumpRecords, DumpAssembly);
-  for (unsigned i = 0; i < NumRecordIndents; ++i) {
-    DumpStream.IncRecordIndent();
-  }
-  RunAssemblyExamples(DumpStream, Bit, Record, AbbrevIndex, AddErrors);
-  return BufStream.str();
-}
-
-// Runs some simple assembly examples against the given bitcode record
-// using an objdump stream. Adds a message describing the test
-// and the record indent being used. Assumes no abbreviation index
-// is associated with the record.
-static std::string RunIndentedAssemblyTest(
-    bool DumpRecords, bool DumpAssembly,
-    unsigned NumRecordIndents, uint64_t Bit,
-    const NaClBitcodeRecordData *Record, bool AddErrors) {
-  return
-      RunIndentedAssemblyWithAbbrevTest(
-          DumpRecords, DumpAssembly, NumRecordIndents, Bit, Record,
-          naclbitc::ABBREV_INDEX_NOT_SPECIFIED, AddErrors);
-}
-
-// Tests effects of objdump when there isn't a record to write.
-TEST(NaClObjDumpTest, NoDumpRecords) {
-  EXPECT_EQ(
-      "          |                             |One line assembly.\n"
-      "          |                             |Two Line\n"
-      "          |                             |example assembly.\n",
-      RunIndentedAssemblyTest(true, true, 0, 11, 0, false));
-
-  EXPECT_EQ(
-      "One line assembly.\n"
-      "Two Line\n"
-      "example assembly.\n",
-      RunIndentedAssemblyTest(false, true, 0, 91, 0, false));
-
-  EXPECT_EQ(
-      "",
-      RunIndentedAssemblyTest(true, false, 0, 37, 0, false));
-
-
-  EXPECT_EQ(
-      "",
-      RunIndentedAssemblyTest(false, false, 0, 64, 0, false));
-}
-
-// Tests simple cases where there is both a record and corresponding
-// assembly code.
-TEST(NaClObjDumpTest, SimpleRecords) {
-  NaClBitcodeRecordData Record;
-  Record.Code = 5;
-  Record.Values.push_back(10);
-  Record.Values.push_back(15);
-
-  EXPECT_EQ(
-      "       1:3|<5, 10, 15>                  |\n"
-      "       4:0|<5, 10, 15>                  |One line assembly.\n"
-      "       6:1|<5, 10, 15>                  |Two Line\n"
-      "          |                             |example assembly.\n",
-      RunIndentedAssemblyTest(true, true, 0, 11, &Record, false));
-
-  EXPECT_EQ(
-      "One line assembly.\n"
-      "Two Line\n"
-      "example assembly.\n",
-      RunIndentedAssemblyTest(false, true, 0, 91, &Record, false));
-
-  EXPECT_EQ(
-      "       4:5|<5, 10, 15>\n"
-      "       7:2|<5, 10, 15>\n"
-      "       9:3|<5, 10, 15>\n",
-      RunIndentedAssemblyTest(true, false, 0, 37, &Record, false));
-
-  EXPECT_EQ(
-      "",
-      RunIndentedAssemblyTest(false, false, 0, 64, &Record, false));
-}
-
-// Test case where record is printed using two lines.
-TEST(NaClObjDumpText, LongRecords) {
-  NaClBitcodeRecordData Record;
-  Record.Code = 5;
-  Record.Values.push_back(static_cast<uint64_t>(-1));
-  Record.Values.push_back(100);
-  Record.Values.push_back(15);
-  Record.Values.push_back(107056);
-
-  EXPECT_EQ(
-      "     127:1|<5, 18446744073709551615,    |\n"
-      "          | 100, 15, 107056>            |\n"
-      "     129:6|<5, 18446744073709551615,    |One line assembly.\n"
-      "          | 100, 15, 107056>            |\n"
-      "     131:7|<5, 18446744073709551615,    |Two Line\n"
-      "          | 100, 15, 107056>            |example assembly.\n",
-      RunIndentedAssemblyTest(true, true, 0, 1017, &Record, false));
-
-  EXPECT_EQ(
-      "One line assembly.\n"
-      "Two Line\n"
-      "example assembly.\n",
-      RunIndentedAssemblyTest(false, true, 0, 91, &Record, false));
-
-  EXPECT_EQ(
-      "   47073:6|<5, 18446744073709551615, 100, 15, 107056>\n"
-      "   47076:3|<5, 18446744073709551615, 100, 15, 107056>\n"
-      "   47078:4|<5, 18446744073709551615, 100, 15, 107056>\n",
-      RunIndentedAssemblyTest(true, false, 0, 376590, &Record, false));
-
-  EXPECT_EQ(
-      "",
-      RunIndentedAssemblyTest(false, false, 0, 64564, &Record, false));
-}
-
-// Test case where comma hits boundary.
-TEST(NaClObjDumpText, CommaBoundaryRecords) {
-  NaClBitcodeRecordData Record;
-  Record.Code = 5;
-  Record.Values.push_back(static_cast<uint64_t>(-1));
-  Record.Values.push_back(10);
-  Record.Values.push_back(15);
-  Record.Values.push_back(107056);
-
-  EXPECT_EQ(
-      "     127:1|<5, 18446744073709551615, 10,|\n"
-      "          | 15, 107056>                 |\n"
-      "     129:6|<5, 18446744073709551615, 10,|One line assembly.\n"
-      "          | 15, 107056>                 |\n"
-      "     131:7|<5, 18446744073709551615, 10,|Two Line\n"
-      "          | 15, 107056>                 |example assembly.\n",
-      RunIndentedAssemblyTest(true, true, 0, 1017, &Record, false));
-
-  EXPECT_EQ(
-      "One line assembly.\n"
-      "Two Line\n"
-      "example assembly.\n",
-      RunIndentedAssemblyTest(false, true, 0, 91, &Record, false));
-
-  EXPECT_EQ(
-      "   47073:6|<5, 18446744073709551615, 10, 15, 107056>\n"
-      "   47076:3|<5, 18446744073709551615, 10, 15, 107056>\n"
-      "   47078:4|<5, 18446744073709551615, 10, 15, 107056>\n",
-      RunIndentedAssemblyTest(true, false, 0, 376590, &Record, false));
-
-  EXPECT_EQ(
-      "",
-      RunIndentedAssemblyTest(false, false, 0, 64564, &Record, false));
-}
-
-// Test case where comma wraps to next line.
-TEST(NaClObjDumpText, CommaWrapRecords) {
-  NaClBitcodeRecordData Record;
-  Record.Code = 5;
-  Record.Values.push_back(static_cast<uint64_t>(-1));
-  Record.Values.push_back(100);
-  Record.Values.push_back(15);
-  Record.Values.push_back(107056);
-
-  EXPECT_EQ(
-      "     127:1|<5, 18446744073709551615,    |\n"
-      "          | 100, 15, 107056>            |\n"
-      "     129:6|<5, 18446744073709551615,    |One line assembly.\n"
-      "          | 100, 15, 107056>            |\n"
-      "     131:7|<5, 18446744073709551615,    |Two Line\n"
-      "          | 100, 15, 107056>            |example assembly.\n",
-      RunIndentedAssemblyTest(true, true, 0, 1017, &Record, false));
-
-  EXPECT_EQ(
-      "One line assembly.\n"
-      "Two Line\n"
-      "example assembly.\n",
-      RunIndentedAssemblyTest(false, true, 0, 91, &Record, false));
-
-  EXPECT_EQ(
-      "   47073:6|<5, 18446744073709551615, 100, 15, 107056>\n"
-      "   47076:3|<5, 18446744073709551615, 100, 15, 107056>\n"
-      "   47078:4|<5, 18446744073709551615, 100, 15, 107056>\n",
-      RunIndentedAssemblyTest(true, false, 0, 376590, &Record, false));
-
-  EXPECT_EQ(
-      "",
-      RunIndentedAssemblyTest(false, false, 0, 64564, &Record, false));
-}
-
-// Test case where record is printed using more than two lines.
-TEST(NaClObjDumpText, VeryLongRecords) {
-  NaClBitcodeRecordData Record;
-  Record.Code = 5;
-  Record.Values.push_back(static_cast<uint64_t>(-1));
-  Record.Values.push_back(100);
-  Record.Values.push_back(15);
-  Record.Values.push_back(107056);
-  Record.Values.push_back(static_cast<uint64_t>(-5065));
-  Record.Values.push_back(101958788);
-
-  EXPECT_EQ(
-      "     127:1|<5, 18446744073709551615,    |\n"
-      "          | 100, 15, 107056,            |\n"
-      "          | 18446744073709546551,       |\n"
-      "          | 101958788>                  |\n"
-      "     129:6|<5, 18446744073709551615,    |One line assembly.\n"
-      "          | 100, 15, 107056,            |\n"
-      "          | 18446744073709546551,       |\n"
-      "          | 101958788>                  |\n"
-      "     131:7|<5, 18446744073709551615,    |Two Line\n"
-      "          | 100, 15, 107056,            |example assembly.\n"
-      "          | 18446744073709546551,       |\n"
-      "          | 101958788>                  |\n",
-      RunIndentedAssemblyTest(true, true, 0, 1017, &Record, false));
-
-  EXPECT_EQ(
-      "One line assembly.\n"
-      "Two Line\n"
-      "example assembly.\n",
-      RunIndentedAssemblyTest(false, true, 0, 91, &Record, false));
-
-  EXPECT_EQ(
-"   47073:6|<5, 18446744073709551615, 100, 15, 107056, 18446744073709546551, \n"
-"          | 101958788>\n"
-"   47076:3|<5, 18446744073709551615, 100, 15, 107056, 18446744073709546551, \n"
-"          | 101958788>\n"
-"   47078:4|<5, 18446744073709551615, 100, 15, 107056, 18446744073709546551, \n"
-"          | 101958788>\n",
-      RunIndentedAssemblyTest(true, false, 0, 376590, &Record, false));
-
-  EXPECT_EQ(
-      "",
-      RunIndentedAssemblyTest(false, false, 0, 64564, &Record, false));
-}
-
-
-// Tests effects of objdump when there isn't a record to write, but errors occur.
-TEST(NaClObjDumpTest, ErrorsErrorsNoDumpRecords) {
-
-  EXPECT_EQ(
-      "Error(1:3): This is an error\n"
-      "          |                             |One line assembly.\n"
-      "Error(4:0): Oops, an error!\n"
-      "          |                             |Two Line\n"
-      "          |                             |example assembly.\n"
-      "Error(6:1): The record looks bad\n"
-      "Error(6:1): Actually, it looks really bad\n",
-      RunIndentedAssemblyTest(true, true, 0, 11, 0, true));
-
-  EXPECT_EQ(
-      "Error(11:3): This is an error\n"
-      "One line assembly.\n"
-      "Error(14:0): Oops, an error!\n"
-      "Two Line\n"
-      "example assembly.\n"
-      "Error(16:1): The record looks bad\n"
-      "Error(16:1): Actually, it looks really bad\n",
-      RunIndentedAssemblyTest(false, true, 0, 91, 0, true));
-
-  EXPECT_EQ(
-      "Error(4:5): This is an error\n"
-      "Error(7:2): Oops, an error!\n"
-      "Error(9:3): The record looks bad\n"
-      "Error(9:3): Actually, it looks really bad\n",
-      RunIndentedAssemblyTest(true, false, 0, 37, 0, true));
-
-
-  EXPECT_EQ(
-      "Error(8:0): This is an error\n"
-      "Error(10:5): Oops, an error!\n"
-      "Error(12:6): The record looks bad\n"
-      "Error(12:6): Actually, it looks really bad\n",
-      RunIndentedAssemblyTest(false, false, 0, 64, 0, true));
-}
-
-// Test case where record is printed using two lines, but errors
-// occur.
-TEST(NaClObjDumpText, ErrorsLongRecords) {
-  NaClBitcodeRecordData Record;
-  Record.Code = 5;
-  Record.Values.push_back(static_cast<uint64_t>(-1));
-  Record.Values.push_back(100);
-  Record.Values.push_back(15);
-  Record.Values.push_back(107056);
-
-  EXPECT_EQ(
-      "     127:1|<5, 18446744073709551615,    |\n"
-      "          | 100, 15, 107056>            |\n"
-      "Error(127:1): This is an error\n"
-      "     129:6|<5, 18446744073709551615,    |One line assembly.\n"
-      "          | 100, 15, 107056>            |\n"
-      "Error(129:6): Oops, an error!\n"
-      "     131:7|<5, 18446744073709551615,    |Two Line\n"
-      "          | 100, 15, 107056>            |example assembly.\n"
-      "Error(131:7): The record looks bad\n"
-      "Error(131:7): Actually, it looks really bad\n",
-      RunIndentedAssemblyTest(true, true, 0, 1017, &Record, true));
-
-  EXPECT_EQ(
-      "Error(11:3): This is an error\n"
-      "One line assembly.\n"
-      "Error(14:0): Oops, an error!\n"
-      "Two Line\n"
-      "example assembly.\n"
-      "Error(16:1): The record looks bad\n"
-      "Error(16:1): Actually, it looks really bad\n",
-      RunIndentedAssemblyTest(false, true, 0, 91, &Record, true));
-
-  EXPECT_EQ(
-      "   47073:6|<5, 18446744073709551615, 100, 15, 107056>\n"
-      "Error(47073:6): This is an error\n"
-      "   47076:3|<5, 18446744073709551615, 100, 15, 107056>\n"
-      "Error(47076:3): Oops, an error!\n"
-      "   47078:4|<5, 18446744073709551615, 100, 15, 107056>\n"
-      "Error(47078:4): The record looks bad\n"
-      "Error(47078:4): Actually, it looks really bad\n",
-      RunIndentedAssemblyTest(true, false, 0, 376590, &Record, true));
-
-  EXPECT_EQ(
-      "Error(8070:4): This is an error\n"
-      "Error(8073:1): Oops, an error!\n"
-      "Error(8075:2): The record looks bad\n"
-      "Error(8075:2): Actually, it looks really bad\n",
-      RunIndentedAssemblyTest(false, false, 0, 64564, &Record, true));
-
-}
-
-// Test case where record is printed using more than two lines, but
-// errors occur.
-TEST(NaClObjDumpText, ErrorsVeryLongRecords) {
-  NaClBitcodeRecordData Record;
-  Record.Code = 5;
-  Record.Values.push_back(static_cast<uint64_t>(-1));
-  Record.Values.push_back(100);
-  Record.Values.push_back(15);
-  Record.Values.push_back(107056);
-  Record.Values.push_back(static_cast<uint64_t>(-5065));
-  Record.Values.push_back(101958788);
-
-  EXPECT_EQ(
-      "     127:1|<5, 18446744073709551615,    |\n"
-      "          | 100, 15, 107056,            |\n"
-      "          | 18446744073709546551,       |\n"
-      "          | 101958788>                  |\n"
-      "Error(127:1): This is an error\n"
-      "     129:6|<5, 18446744073709551615,    |One line assembly.\n"
-      "          | 100, 15, 107056,            |\n"
-      "          | 18446744073709546551,       |\n"
-      "          | 101958788>                  |\n"
-      "Error(129:6): Oops, an error!\n"
-      "     131:7|<5, 18446744073709551615,    |Two Line\n"
-      "          | 100, 15, 107056,            |example assembly.\n"
-      "          | 18446744073709546551,       |\n"
-      "          | 101958788>                  |\n"
-      "Error(131:7): The record looks bad\n"
-      "Error(131:7): Actually, it looks really bad\n",
-      RunIndentedAssemblyTest(true, true, 0, 1017, &Record, true));
-
-  EXPECT_EQ(
-      "Error(11:3): This is an error\n"
-      "One line assembly.\n"
-      "Error(14:0): Oops, an error!\n"
-      "Two Line\n"
-      "example assembly.\n"
-      "Error(16:1): The record looks bad\n"
-      "Error(16:1): Actually, it looks really bad\n",
-      RunIndentedAssemblyTest(false, true, 0, 91, &Record, true));
-
-  EXPECT_EQ(
-"   47073:6|<5, 18446744073709551615, 100, 15, 107056, 18446744073709546551, \n"
-"          | 101958788>\n"
-"Error(47073:6): This is an error\n"
-"   47076:3|<5, 18446744073709551615, 100, 15, 107056, 18446744073709546551, \n"
-"          | 101958788>\n"
-"Error(47076:3): Oops, an error!\n"
-"   47078:4|<5, 18446744073709551615, 100, 15, 107056, 18446744073709546551, \n"
-"          | 101958788>\n"
-"Error(47078:4): The record looks bad\n"
-"Error(47078:4): Actually, it looks really bad\n",
-      RunIndentedAssemblyTest(true, false, 0, 376590, &Record, true));
-
-  EXPECT_EQ(
-      "Error(8070:4): This is an error\n"
-      "Error(8073:1): Oops, an error!\n"
-      "Error(8075:2): The record looks bad\n"
-      "Error(8075:2): Actually, it looks really bad\n",
-      RunIndentedAssemblyTest(false, false, 0, 64564, &Record, true));
-}
-
-// Tests effects of objdump when there isn't a record to write, and we indent.
-TEST(NaClObjDumpTest, NoDumpIndentRecords) {
-
-  EXPECT_EQ(
-      "          |                             |One line assembly.\n"
-      "          |                             |Two Line\n"
-      "          |                             |example assembly.\n",
-      RunIndentedAssemblyTest(true, true, 1, 11, 0, false));
-
-  EXPECT_EQ(
-      "          |                             |One line assembly.\n"
-      "          |                             |Two Line\n"
-      "          |                             |example assembly.\n",
-      RunIndentedAssemblyTest(true, true, 2, 11, 0, false));
-
-  EXPECT_EQ(
-      "One line assembly.\n"
-      "Two Line\n"
-      "example assembly.\n",
-      RunIndentedAssemblyTest(false, true, 1, 91, 0, false));
-
-  EXPECT_EQ(
-      "One line assembly.\n"
-      "Two Line\n"
-      "example assembly.\n",
-      RunIndentedAssemblyTest(false, true, 2, 91, 0, false));
-
-  EXPECT_EQ(
-      "",
-      RunIndentedAssemblyTest(true, false, 1, 37, 0, false));
-
-  EXPECT_EQ(
-      "",
-      RunIndentedAssemblyTest(true, false, 2, 37, 0, false));
-
-  EXPECT_EQ(
-      "",
-      RunIndentedAssemblyTest(false, false, 1, 64, 0, false));
-
-  EXPECT_EQ(
-      "",
-      RunIndentedAssemblyTest(false, false, 2, 64, 0, false));
-}
-
-// Tests simple cases where there is both a record and corresponding
-// assembly code, and the records are indented.
-TEST(NaClObjDumpTest, SimpleIndentRecords) {
-  NaClBitcodeRecordData Record;
-  Record.Code = 5;
-  Record.Values.push_back(10);
-  Record.Values.push_back(15);
-
-  EXPECT_EQ(
-      "       1:3|  <5, 10, 15>                |\n"
-      "       4:0|  <5, 10, 15>                |One line assembly.\n"
-      "       6:1|  <5, 10, 15>                |Two Line\n"
-      "          |                             |example assembly.\n",
-      RunIndentedAssemblyTest(true, true, 1, 11, &Record, false));
-
-  EXPECT_EQ(
-      "       1:3|    <5, 10, 15>              |\n"
-      "       4:0|    <5, 10, 15>              |One line assembly.\n"
-      "       6:1|    <5, 10, 15>              |Two Line\n"
-      "          |                             |example assembly.\n",
-      RunIndentedAssemblyTest(true, true, 2, 11, &Record, false));
-
-  EXPECT_EQ(
-      "One line assembly.\n"
-      "Two Line\n"
-      "example assembly.\n",
-      RunIndentedAssemblyTest(false, true, 1, 91, &Record, false));
-
-  EXPECT_EQ(
-      "One line assembly.\n"
-      "Two Line\n"
-      "example assembly.\n",
-      RunIndentedAssemblyTest(false, true, 2, 91, &Record, false));
-
-  EXPECT_EQ(
-      "       4:5|  <5, 10, 15>\n"
-      "       7:2|  <5, 10, 15>\n"
-      "       9:3|  <5, 10, 15>\n",
-      RunIndentedAssemblyTest(true, false, 1, 37, &Record, false));
-
-  EXPECT_EQ(
-      "       4:5|    <5, 10, 15>\n"
-      "       7:2|    <5, 10, 15>\n"
-      "       9:3|    <5, 10, 15>\n",
-      RunIndentedAssemblyTest(true, false, 2, 37, &Record, false));
-
-  EXPECT_EQ(
-      "",
-      RunIndentedAssemblyTest(false, false, 1, 64, &Record, false));
-
-  EXPECT_EQ(
-      "",
-      RunIndentedAssemblyTest(false, false, 2, 64, &Record, false));
-
-}
-
-// Test case where record is printed using more than two lines.
-TEST(NaClObjDumpText, VeryLongIndentRecords) {
-  NaClBitcodeRecordData Record;
-  Record.Code = 5;
-  Record.Values.push_back(static_cast<uint64_t>(-1));
-  Record.Values.push_back(100);
-  Record.Values.push_back(15);
-  Record.Values.push_back(107056);
-  Record.Values.push_back(static_cast<uint64_t>(-5065));
-  Record.Values.push_back(101958788);
-
-  EXPECT_EQ(
-      "     127:1|  <5, 18446744073709551615,  |\n"
-      "          |   100, 15, 107056,          |\n"
-      "          |   18446744073709546551,     |\n"
-      "          |   101958788>                |\n"
-      "     129:6|  <5, 18446744073709551615,  |One line assembly.\n"
-      "          |   100, 15, 107056,          |\n"
-      "          |   18446744073709546551,     |\n"
-      "          |   101958788>                |\n"
-      "     131:7|  <5, 18446744073709551615,  |Two Line\n"
-      "          |   100, 15, 107056,          |example assembly.\n"
-      "          |   18446744073709546551,     |\n"
-      "          |   101958788>                |\n",
-      RunIndentedAssemblyTest(true, true, 1, 1017, &Record, false));
-
-  EXPECT_EQ(
-      "     127:1|      <5,                    |\n"
-      "          |       18446744073709551615, |\n"
-      "          |       100, 15, 107056,      |\n"
-      "          |       18446744073709546551, |\n"
-      "          |       101958788>            |\n"
-      "     129:6|      <5,                    |One line assembly.\n"
-      "          |       18446744073709551615, |\n"
-      "          |       100, 15, 107056,      |\n"
-      "          |       18446744073709546551, |\n"
-      "          |       101958788>            |\n"
-      "     131:7|      <5,                    |Two Line\n"
-      "          |       18446744073709551615, |example assembly.\n"
-      "          |       100, 15, 107056,      |\n"
-      "          |       18446744073709546551, |\n"
-      "          |       101958788>            |\n",
-      RunIndentedAssemblyTest(true, true, 3, 1017, &Record, false));
-
-  EXPECT_EQ(
-      "One line assembly.\n"
-      "Two Line\n"
-      "example assembly.\n",
-      RunIndentedAssemblyTest(false, true, 1, 91, &Record, false));
-
-  EXPECT_EQ(
-      "One line assembly.\n"
-      "Two Line\n"
-      "example assembly.\n",
-      RunIndentedAssemblyTest(false, true, 2, 91, &Record, false));
-
-  EXPECT_EQ(
-"   47073:6|  <5, 18446744073709551615, 100, 15, 107056, 18446744073709546551, \n"
-"          |   101958788>\n"
-"   47076:3|  <5, 18446744073709551615, 100, 15, 107056, 18446744073709546551, \n"
-"          |   101958788>\n"
-"   47078:4|  <5, 18446744073709551615, 100, 15, 107056, 18446744073709546551, \n"
-"          |   101958788>\n",
-      RunIndentedAssemblyTest(true, false, 1, 376590, &Record, false));
-
-
-  EXPECT_EQ(
-      "   47073:6|          <5, 18446744073709551615, 100, 15, 107056, \n"
-      "          |           18446744073709546551, 101958788>\n"
-      "   47076:3|          <5, 18446744073709551615, 100, 15, 107056, \n"
-      "          |           18446744073709546551, 101958788>\n"
-      "   47078:4|          <5, 18446744073709551615, 100, 15, 107056, \n"
-      "          |           18446744073709546551, 101958788>\n",
-      RunIndentedAssemblyTest(true, false, 5, 376590, &Record, false));
-
-  EXPECT_EQ(
-      "",
-      RunIndentedAssemblyTest(false, false, 1, 64564, &Record, false));
-
-  EXPECT_EQ(
-      "",
-      RunIndentedAssemblyTest(false, false, 2, 64564, &Record, false));
-}
-
-// Tests that Clustering doesn't effect (intraline) indenting.
-TEST(NaClObjDumpTest, ClusterIndentInteraction) {
-  std::string Buffer;
-  raw_string_ostream BufStream(Buffer);
-  ObjDumpStream Stream(BufStream, true, true);
-
-  TextFormatter Formatter(Stream.Assembly(), 40, "  ");
-  TokenTextDirective Comma(&Formatter, ",");
-  SpaceTextDirective Space(&Formatter);
-  OpenTextDirective OpenParen(&Formatter, "(");
-  CloseTextDirective CloseParen(&Formatter, ")");
-  StartClusteringDirective StartCluster(&Formatter);
-  FinishClusteringDirective FinishCluster(&Formatter);
-  EndlineTextDirective Endline(&Formatter);
-
-  Formatter.Tokens() << "begin" << Space;
-  // Generates text on single line, setting indent at "(".
-  Formatter.Tokens()
-      << StartCluster << "SomeReasonablylongText" << OpenParen << FinishCluster;
-  // Generates a long cluster that should move to the next line.
-  Formatter.Tokens()
-      << StartCluster << "ThisIsBoring" << Space
-      << "VeryBoring" << Space << "longggggggggggggggggg"
-      << Space << "Example" << Comma << FinishCluster;
-  Formatter.Tokens() << CloseParen << Comma << Endline;
-  Stream.Flush();
-  EXPECT_EQ(
-"          |                             |begin SomeReasonablylongText(\n"
-"          |                             |                    ThisIsBoring \n"
-"          |                             |                    VeryBoring \n"
-"          |                             |                    longggggggggggggggggg\n"
-"          |                             |                    Example,),\n",
-BufStream.str());
-}
-
-// Tests the insertion of an abbreviation index.
-TEST(NaClObjDumpTest, UseOfAbbrevationIndex) {
-  NaClBitcodeRecordData Record;
-  Record.Code = 5;
-  Record.Values.push_back(static_cast<uint64_t>(-1));
-  Record.Values.push_back(100);
-  Record.Values.push_back(15);
-  Record.Values.push_back(107056);
-  Record.Values.push_back(static_cast<uint64_t>(-5065));
-  Record.Values.push_back(101958788);
-
-  EXPECT_EQ(
-      "     127:1|3: <5, 18446744073709551615, |\n"
-      "          |    100, 15, 107056,         |\n"
-      "          |    18446744073709546551,    |\n"
-      "          |    101958788>               |\n"
-      "     129:6|3: <5, 18446744073709551615, |One line assembly.\n"
-      "          |    100, 15, 107056,         |\n"
-      "          |    18446744073709546551,    |\n"
-      "          |    101958788>               |\n"
-      "     131:7|3: <5, 18446744073709551615, |Two Line\n"
-      "          |    100, 15, 107056,         |example assembly.\n"
-      "          |    18446744073709546551,    |\n"
-      "          |    101958788>               |\n",
-      RunIndentedAssemblyWithAbbrevTest(true, true, 0, 1017, &Record,
-                                        naclbitc::UNABBREV_RECORD, false));
-
-  EXPECT_EQ(
-      "     127:1|  3: <5,                     |\n"
-      "          |      18446744073709551615,  |\n"
-      "          |      100, 15, 107056,       |\n"
-      "          |      18446744073709546551,  |\n"
-      "          |      101958788>             |\n"
-      "     129:6|  3: <5,                     |One line assembly.\n"
-      "          |      18446744073709551615,  |\n"
-      "          |      100, 15, 107056,       |\n"
-      "          |      18446744073709546551,  |\n"
-      "          |      101958788>             |\n"
-      "     131:7|  3: <5,                     |Two Line\n"
-      "          |      18446744073709551615,  |example assembly.\n"
-      "          |      100, 15, 107056,       |\n"
-      "          |      18446744073709546551,  |\n"
-      "          |      101958788>             |\n",
-      RunIndentedAssemblyWithAbbrevTest(true, true, 1, 1017, &Record,
-                                        naclbitc::UNABBREV_RECORD, false));
-
-  EXPECT_EQ(
-      "     127:1|      3: <5,                 |\n"
-      "          |        18446744073709551615,|\n"
-      "          |        100, 15, 107056,     |\n"
-      "          |        18446744073709546551,|\n"
-      "          |        101958788>           |\n"
-      "     129:6|      3: <5,                 |One line assembly.\n"
-      "          |        18446744073709551615,|\n"
-      "          |        100, 15, 107056,     |\n"
-      "          |        18446744073709546551,|\n"
-      "          |        101958788>           |\n"
-      "     131:7|      3: <5,                 |Two Line\n"
-      "          |        18446744073709551615,|example assembly.\n"
-      "          |        100, 15, 107056,     |\n"
-      "          |        18446744073709546551,|\n"
-      "          |        101958788>           |\n",
-      RunIndentedAssemblyWithAbbrevTest(true, true, 3, 1017, &Record,
-                                        naclbitc::UNABBREV_RECORD, false));
-
-  EXPECT_EQ(
-      "One line assembly.\n"
-      "Two Line\n"
-      "example assembly.\n",
-      RunIndentedAssemblyWithAbbrevTest(false, true, 1, 91, &Record,
-                                        naclbitc::UNABBREV_RECORD, false));
-
-  EXPECT_EQ(
-      "One line assembly.\n"
-      "Two Line\n"
-      "example assembly.\n",
-      RunIndentedAssemblyWithAbbrevTest(false, true, 2, 91, &Record,
-                                        naclbitc::UNABBREV_RECORD, false));
-
-  EXPECT_EQ(
-"   47073:6|  3: <5, 18446744073709551615, 100, 15, 107056, 18446744073709546551,\n"
-"          |      101958788>\n"
-"   47076:3|  3: <5, 18446744073709551615, 100, 15, 107056, 18446744073709546551,\n"
-"          |      101958788>\n"
-"   47078:4|  3: <5, 18446744073709551615, 100, 15, 107056, 18446744073709546551,\n"
-"          |      101958788>\n",
-      RunIndentedAssemblyWithAbbrevTest(true, false, 1, 376590, &Record,
-                                        naclbitc::UNABBREV_RECORD, false));
-
-
-  EXPECT_EQ(
-      "   47073:6|          3: <5, 18446744073709551615, 100, 15, 107056, \n"
-      "          |              18446744073709546551, 101958788>\n"
-      "   47076:3|          3: <5, 18446744073709551615, 100, 15, 107056, \n"
-      "          |              18446744073709546551, 101958788>\n"
-      "   47078:4|          3: <5, 18446744073709551615, 100, 15, 107056, \n"
-      "          |              18446744073709546551, 101958788>\n",
-      RunIndentedAssemblyWithAbbrevTest(true, false, 5, 376590, &Record,
-                                        naclbitc::UNABBREV_RECORD, false));
-
-  EXPECT_EQ(
-      "",
-      RunIndentedAssemblyWithAbbrevTest(false, false, 1, 64564, &Record,
-                                        naclbitc::UNABBREV_RECORD, false));
-
-  EXPECT_EQ(
-      "",
-      RunIndentedAssemblyWithAbbrevTest(false, false, 2, 64564, &Record,
-                                        naclbitc::UNABBREV_RECORD, false));
-}
-
-}
diff --git a/unittests/Bitcode/NaClObjDumpTypesTest.cpp b/unittests/Bitcode/NaClObjDumpTypesTest.cpp
deleted file mode 100644
index 91e2fbb6765..00000000000
--- a/unittests/Bitcode/NaClObjDumpTypesTest.cpp
+++ /dev/null
@@ -1,1303 +0,0 @@
-//===- llvm/unittest/Bitcode/NaClObjDumpTypesTest.cpp ---------------------===//
-//     Tests objdump stream for PNaCl bitcode.
-//
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
-//
-//===----------------------------------------------------------------------===//
-
-// Tests record errors in the types block when dumping PNaCl bitcode.
-
-#include "NaClMungeTest.h"
-
-using namespace llvm;
-
-namespace naclmungetest {
-
-static const char ErrorPrefix[] = "Error";
-
-// Tests what happens when a type refers to a not-yet defined type.
-TEST(NaClObjDumpTypesTest, BadTypeReferences) {
-  const uint64_t BitcodeRecords[] = {
-    1, 65535, 8, 2, Terminator,
-    1, 65535, 17, 2, Terminator,
-    3, 1, 2, Terminator,
-    3, 7, 32, Terminator,
-    3, 3, Terminator,
-    0, 65534, Terminator,
-    0, 65534, Terminator
-  };
-
-  const uint64_t ReplaceIndex = 4;
-
-  // Show base input.
-  NaClObjDumpMunger Munger(ARRAY_TERM(BitcodeRecords));
-  EXPECT_TRUE(Munger.runTestForAssembly());
-  EXPECT_EQ(
-      "module {  // BlockID = 8\n"
-      "  types {  // BlockID = 17\n"
-      "    count 2;\n"
-      "    @t0 = i32;\n"
-      "    @t1 = float;\n"
-      "  }\n"
-      "}\n",
-      Munger.getTestResults());
-
-  // Show what happens when defining: @t1 = <4 x @t1>
-  const uint64_t AddSelfReference[] = {
-    ReplaceIndex, NaClMungedBitcode::Replace, 3, 12, 4, 1, Terminator
-  };
-  EXPECT_FALSE(Munger.runTestForAssembly(ARRAY(AddSelfReference)));
-  // Note: Because @t1 is not defined until after this instruction,
-  // the initial lookup of @t1 in <4 x @t1> is not found. To error
-  // recover, type "void" is returned as the type of @t1.
-  EXPECT_EQ("Error(37:6): Can't find definition for @t1\n"
-            "Error(37:6): Vectors can only be defined on primitive types."
-            " Found void. Assuming i32 instead.\n",
-            Munger.getLinesWithPrefix(ErrorPrefix));
-  EXPECT_EQ(
-      "    @t1 = <4 x i32>;\n"
-      "Error(37:6): Can't find definition for @t1\n",
-      Munger.getLinesWithSubstring("@t1"));
-
-  // Show what happens when defining: @t1 = <4 x @t5>
-  const uint64_t AddForwardReference[] = {
-    ReplaceIndex, NaClMungedBitcode::Replace, 3, 12, 4, 5, Terminator
-  };
-  EXPECT_FALSE(Munger.runTestForAssembly(ARRAY(AddForwardReference)));
-  // Note: Because @t5 is not defined, type "void" is used to error recover.
-  EXPECT_EQ(
-      "Error(37:6): Can't find definition for @t5\n"
-      "Error(37:6): Vectors can only be defined on primitive types."
-      " Found void. Assuming i32 instead.\n",
-      Munger.getLinesWithPrefix(ErrorPrefix));
-  EXPECT_EQ(
-      "    @t1 = <4 x i32>;\n",
-      Munger.getLinesWithSubstring("@t1"));
-}
-
-// Tests handling of the count record in the types block.
-TEST(NaClObjDumpTypesTest, TestCountRecord) {
-  const uint64_t BitcodeRecords[] = {
-    1, 65535, 8, 2, Terminator,
-    1, 65535, 17, 2, Terminator,
-    3, 1, 2, Terminator,
-    3, 7, 32, Terminator,
-    3, 3, Terminator,
-    0, 65534, Terminator,
-    0, 65534, Terminator
-  };
-
-  const uint64_t AddBeforeIndex = 5;
-  const uint64_t ReplaceIndex = 2;
-
-  NaClObjDumpMunger Munger(ARRAY_TERM(BitcodeRecords));
-
-  // Test case where count is correct.
-  EXPECT_TRUE(Munger.runTestForAssembly());
-  EXPECT_EQ(
-      "module {  // BlockID = 8\n"
-      "  types {  // BlockID = 17\n"
-      "    count 2;\n"
-      "    @t0 = i32;\n"
-      "    @t1 = float;\n"
-      "  }\n"
-      "}\n",
-      Munger.getTestResults());
-
-  // Test case where more types are defined then specified by the
-  // count record.
-  const uint64_t AddDoubleType[] = {
-    AddBeforeIndex, NaClMungedBitcode::AddBefore, 3, 4, Terminator
-  };
-  EXPECT_FALSE(Munger.runTestForAssembly(ARRAY(AddDoubleType)));
-  EXPECT_EQ("Error(41:2): Expected 2 types but found: 3\n",
-            Munger.getLinesWithPrefix(ErrorPrefix));
-  EXPECT_EQ(
-      "    @t0 = i32;\n"
-      "    @t1 = float;\n"
-      "    @t2 = double;\n",
-      Munger.getLinesWithSubstring("@t"));
-
-  // Test case where fewer types are defined then specified by the count
-  // record.
-  const uint64_t DeleteI32Type[] = { 3, NaClMungedBitcode::Remove };
-  EXPECT_FALSE(Munger.runTestForAssembly(ARRAY(DeleteI32Type)));
-  EXPECT_EQ("Error(36:2): Expected 2 types but found: 1\n",
-            Munger.getLinesWithPrefix(ErrorPrefix));
-  EXPECT_EQ(
-      "    @t0 = float;\n",
-      Munger.getLinesWithSubstring("@t"));
-
-  // Test if we generate an error message if the count record isn't first.
-  const uint64_t AddI16BeforeCount[] = {
-    ReplaceIndex, NaClMungedBitcode::AddBefore, 3,  7, 16, Terminator };
-  EXPECT_FALSE(Munger.runTestForAssembly(ARRAY(AddI16BeforeCount)));
-  EXPECT_EQ(
-      "module {  // BlockID = 8\n"
-      "  types {  // BlockID = 17\n"
-      "    @t0 = i16;\n"
-      "    count 2;\n"
-      "Error(34:4): Count record not first record of types block\n"
-      "    @t1 = i32;\n"
-      "    @t2 = float;\n"
-      "  }\n"
-      "Error(42:0): Expected 2 types but found: 3\n"
-      "}\n",
-      Munger.getTestResults());
-
-  // Test if count record doesn't contain enough elements.
-  const uint64_t CountRecordEmpty[] = {
-    ReplaceIndex, NaClMungedBitcode::Replace, 3, 1, Terminator };
-  EXPECT_FALSE(Munger.runTestForErrors(ARRAY(CountRecordEmpty)));
-  EXPECT_EQ("Error(32:0): Count record should have 1 argument. Found: 0\n"
-            "Error(38:6): Expected 0 types but found: 2\n",
-            Munger.getTestResults());
-
-  // Test if count record has extraneous values.
-  const uint64_t CountRecordTooLong[] = {
-    ReplaceIndex, NaClMungedBitcode::Replace, 3, 1, 14, 2, Terminator
-  };
-  EXPECT_FALSE(Munger.runTestForErrors(ARRAY(CountRecordTooLong)));
-  EXPECT_EQ("Error(32:0): Count record should have 1 argument. Found: 2\n"
-            "Error(40:2): Expected 0 types but found: 2\n",
-            Munger.getTestResults());
-}
-
-// Tests handling of the void record in the types block.
-TEST(NaClObjDumpTypesTest, TestVoidRecord) {
-  const uint64_t BitcodeRecords[] = {
-    1, 65535, 8, 2, Terminator,
-    1, 65535, 17, 2, Terminator,
-    3, 1, 1, Terminator,
-    3, 2, Terminator,
-    0, 65534, Terminator,
-    0, 65534, Terminator
-  };
-
-  const uint64_t ReplaceIndex = 3;
-
-  NaClObjDumpMunger Munger(ARRAY_TERM(BitcodeRecords));
-
-  // Test where void is properly specified.
-  EXPECT_TRUE(Munger.runTestForAssembly());
-  EXPECT_EQ(
-      "module {  // BlockID = 8\n"
-      "  types {  // BlockID = 17\n"
-      "    count 1;\n"
-      "    @t0 = void;\n"
-      "  }\n"
-      "}\n",
-      Munger.getTestResults());
-
-  // Test where void record has extraneous values.
-  const uint64_t VoidRecordTooLong[] = {
-    ReplaceIndex, NaClMungedBitcode::Replace, 3, 2, 5, Terminator
-  };
-  EXPECT_FALSE(Munger.runTestForAssembly(ARRAY(VoidRecordTooLong)));
-  EXPECT_EQ("Error(34:4): Void record shouldn't have arguments. Found: 1\n",
-            Munger.getLinesWithPrefix(ErrorPrefix));
-  EXPECT_EQ(
-      "    @t0 = void;\n",
-      Munger.getLinesWithSubstring("@t0"));
-}
-
-// Tests handling of integer records in the types block.
-TEST(NaClObjDumpTypesTest, TestIntegerRecord) {
-  const uint64_t BitcodeRecords[] = {
-    1, 65535, 8, 2, Terminator,
-    1, 65535, 17, 2, Terminator,
-    3, 1, 1, Terminator,
-    3, 7, 1,  Terminator,
-    0, 65534, Terminator,
-    0, 65534, Terminator
-  };
-
-  const uint64_t ReplaceIndex = 3;
-
-  NaClObjDumpMunger Munger(ARRAY_TERM(BitcodeRecords));
-
-  // Tests that we accept i1.
-  EXPECT_TRUE(Munger.runTestForAssembly());
-  EXPECT_EQ(
-      "module {  // BlockID = 8\n"
-      "  types {  // BlockID = 17\n"
-      "    count 1;\n"
-      "    @t0 = i1;\n"
-      "  }\n"
-      "}\n",
-      Munger.getTestResults());
-
-  // Tests that we reject i2.
-  const uint64_t TestTypeI2[] = {
-    ReplaceIndex, NaClMungedBitcode::Replace, 3, 7, 2, Terminator
-  };
-  EXPECT_FALSE(Munger.runTestForAssembly(ARRAY(TestTypeI2)));
-  EXPECT_EQ(
-      "Error(34:4): Integer record contains bad integer size: 2\n",
-      Munger.getLinesWithPrefix(ErrorPrefix));
-  // Note: Error recovery uses i32 when type size is bad.
-  EXPECT_EQ(
-      "    @t0 = i32;\n",
-      Munger.getLinesWithSubstring("@t0"));
-
-  // Tests that we accept i8.
-  const uint64_t TestTypeI8[] = {
-    ReplaceIndex, NaClMungedBitcode::Replace, 3, 7, 8, Terminator
-  };
-  EXPECT_TRUE(Munger.runTest(ARRAY(TestTypeI8)));
-
-  // Tests that we accept i16.
-  const uint64_t TestTypeI16[] = {
-    ReplaceIndex, NaClMungedBitcode::Replace, 3, 7, 16, Terminator
-  };
-  EXPECT_TRUE(Munger.runTest(ARRAY(TestTypeI16)));
-
-  // Tests that we accept i32.
-  const uint64_t TestTypeI32[] = {
-    ReplaceIndex, NaClMungedBitcode::Replace, 3, 7, 32, Terminator
-  };
-  EXPECT_TRUE(Munger.runTest(ARRAY(TestTypeI32)));
-
-  // Tests that we accept i64.
-  const uint64_t TestTypeI64[] = {
-    ReplaceIndex, NaClMungedBitcode::Replace, 3, 7, 64, Terminator
-  };
-  EXPECT_TRUE(Munger.runTest(ARRAY(TestTypeI64)));
-
-  // Tests that we reject i128.
-  const uint64_t TestTypeI128[] = {
-    ReplaceIndex, NaClMungedBitcode::Replace, 3, 7, 128, Terminator
-  };
-  EXPECT_FALSE(Munger.runTestForAssembly(ARRAY(TestTypeI128)));
-  EXPECT_EQ(
-      "Error(34:4): Integer record contains bad integer size: 128\n",
-      Munger.getLinesWithPrefix(ErrorPrefix));
-  // Note: Error recovery uses i32 when type size is bad.
-  EXPECT_EQ(
-      "    @t0 = i32;\n",
-      Munger.getLinesWithSubstring("@t0"));
-
-  // Tests when not enough values are in the integer record.
-  const uint64_t RecordTooShort[] = {
-    ReplaceIndex, NaClMungedBitcode::Replace, 3, 7, Terminator
-  };
-  EXPECT_FALSE(Munger.runTestForAssembly(ARRAY(RecordTooShort)));
-  EXPECT_EQ(
-      "Error(34:4): Integer record should have one argument. Found: 0\n",
-      Munger.getLinesWithPrefix(ErrorPrefix));
-  // Note: Error recovery uses i32 when type size is bad.
-  EXPECT_EQ(
-      "    @t0 = i32;\n",
-      Munger.getLinesWithSubstring("@t0"));
-
-  // Tests when too many values are in the integer record.
-  const uint64_t RecordTooLong[] = {
-    ReplaceIndex, NaClMungedBitcode::Replace, 3, 7, 32, 0, Terminator
-  };
-  EXPECT_FALSE(Munger.runTestForErrors(ARRAY(RecordTooLong)));
-  EXPECT_EQ(
-      "Error(34:4): Integer record should have one argument. Found: 2\n",
-      Munger.getTestResults());
-}
-
-// Tests handling of the float record in the types block.
-TEST(NaClObjDumpTypesTest, TestFloatRecord) {
-  const uint64_t BitcodeRecords[] = {
-    1, 65535, 8, 2, Terminator,
-    1, 65535, 17, 2, Terminator,
-    3, 1, 1, Terminator,
-    3, 3,  Terminator,
-    0, 65534, Terminator,
-    0, 65534, Terminator
-  };
-
-  const uint64_t ReplaceIndex = 3;
-
-  NaClObjDumpMunger Munger(ARRAY_TERM(BitcodeRecords));
-
-  // Test that we accept the float record.
-  EXPECT_TRUE(Munger.runTestForAssembly());
-  EXPECT_EQ(
-      "module {  // BlockID = 8\n"
-      "  types {  // BlockID = 17\n"
-      "    count 1;\n"
-      "    @t0 = float;\n"
-      "  }\n"
-      "}\n",
-      Munger.getTestResults());
-
-  // Test error for float record that has extraneous values.
-  const uint64_t FloatRecordTooLong[] = {
-    ReplaceIndex, NaClMungedBitcode::Replace, 3, 3, 5, Terminator
-  };
-  EXPECT_FALSE(Munger.runTestForAssembly(ARRAY(FloatRecordTooLong)));
-  EXPECT_EQ(
-      "Error(34:4): Float record shoudn't have arguments. Found: 1\n",
-      Munger.getLinesWithPrefix(ErrorPrefix));
-  EXPECT_EQ(
-      "    @t0 = float;\n",
-      Munger.getLinesWithSubstring("@t"));
-}
-
-// Tests handling of the double record in the types block.
-TEST(NaClObjDumpTypesTest, TestDoubleRecord) {
-  const uint64_t BitcodeRecords[] = {
-    1, 65535, 8, 2, Terminator,
-    1, 65535, 17, 2, Terminator,
-    3, 1, 1, Terminator,
-    3, 4, Terminator,
-    0, 65534, Terminator,
-    0, 65534, Terminator
-  };
-
-  const uint64_t ReplaceIndex = 3;
-
-  NaClObjDumpMunger Munger(ARRAY_TERM(BitcodeRecords));
-
-  // Test that we accept the double record.
-  EXPECT_TRUE(Munger.runTestForAssembly());
-  EXPECT_EQ(
-      "module {  // BlockID = 8\n"
-      "  types {  // BlockID = 17\n"
-      "    count 1;\n"
-      "    @t0 = double;\n"
-      "  }\n"
-      "}\n",
-      Munger.getTestResults());
-
-  // Test error for double record that has extraneous values.
-  const uint64_t DoubleRecordTooLong[] = {
-    ReplaceIndex, NaClMungedBitcode::Replace, 3, 4, 5, Terminator
-  };
-  EXPECT_FALSE(Munger.runTestForAssembly(ARRAY(DoubleRecordTooLong)));
-  EXPECT_EQ(
-      "Error(34:4): Double record shound't have arguments. Found: 1\n",
-      Munger.getLinesWithPrefix(ErrorPrefix));
-  EXPECT_EQ(
-      "    @t0 = double;\n",
-      Munger.getLinesWithSubstring("@t"));
-}
-
-// Test vector records of the wrong size.
-TEST(NaClObjDumpTypesTest, TestVectorRecordLength) {
-  const uint64_t BitcodeRecords[] = {
-    1, 65535, 8, 2, Terminator,
-    1, 65535, 17, 2, Terminator,
-    3, 1, 2, Terminator,
-    3, 7, 32, Terminator,
-    3, 12, 4, 0, Terminator,
-    0, 65534, Terminator,
-    0, 65534, Terminator
-  };
-
-  const uint64_t ReplaceIndex = 4;
-
-  NaClObjDumpMunger Munger(ARRAY_TERM(BitcodeRecords));
-
-  // Test correct length vector record.
-  EXPECT_TRUE(Munger.runTestForAssembly());
-  EXPECT_EQ(
-      "module {  // BlockID = 8\n"
-      "  types {  // BlockID = 17\n"
-      "    count 2;\n"
-      "    @t0 = i32;\n"
-      "    @t1 = <4 x i32>;\n"
-      "  }\n"
-      "}\n",
-      Munger.getTestResults());
-
-  // Test vector record too short.
-  uint64_t RecordTooShort[] = {
-    ReplaceIndex, NaClMungedBitcode::Replace, 3, 12, 4, Terminator
-  };
-  EXPECT_FALSE(Munger.runTestForAssembly(ARRAY(RecordTooShort)));
-  EXPECT_EQ(
-      "Error(37:6): Vector record should contain two arguments. Found: 1\n",
-      Munger.getLinesWithPrefix(ErrorPrefix));
-  EXPECT_EQ(
-      "    @t1 = void;\n",
-      Munger.getLinesWithSubstring("@t1"));
-
-  // Test vector record too long.
-  uint64_t RecordTooLong[] = {
-    ReplaceIndex, NaClMungedBitcode::Replace, 3, 12, 4, 0, 0, Terminator
-  };
-  EXPECT_FALSE(Munger.runTestForAssembly(ARRAY(RecordTooLong)));
-  EXPECT_EQ(
-      "Error(37:6): Vector record should contain two arguments. Found: 3\n",
-      Munger.getLinesWithPrefix(ErrorPrefix));
-  EXPECT_EQ(
-      "    @t1 = void;\n",
-      Munger.getLinesWithSubstring("@t1"));
-}
-
-// Test i1 vector records in the types block.
-TEST(NaClObjDumpTypesTest, TestI1VectorRecord) {
-  const uint64_t BitcodeRecords[] = {
-    1, 65535, 8, 2, Terminator,
-    1, 65535, 17, 2, Terminator,
-    3, 1, 2, Terminator,
-    3, 7, 1, Terminator,
-    3, 12, 4, 0, Terminator,
-    0, 65534, Terminator,
-    0, 65534, Terminator
-  };
-
-  const uint64_t ReplaceIndex = 4;
-
-  NaClObjDumpMunger Munger(ARRAY_TERM(BitcodeRecords));
-
-  // Test that we accept <4 x i1>.
-  EXPECT_TRUE(Munger.runTestForAssembly());
-  EXPECT_EQ(
-      "module {  // BlockID = 8\n"
-      "  types {  // BlockID = 17\n"
-      "    count 2;\n"
-      "    @t0 = i1;\n"
-      "    @t1 = <4 x i1>;\n"
-      "  }\n"
-      "}\n",
-      Munger.getTestResults());
-
-  // Test that we don't handle <1 x i1>.
-  const uint64_t Vector1xI1[] = {
-    ReplaceIndex, NaClMungedBitcode::Replace, 3, 12, 1, 0, Terminator
-  };
-  EXPECT_FALSE(Munger.runTestForAssembly(ARRAY(Vector1xI1)));
-  EXPECT_EQ(
-      "Error(37:0): Vector type <1 x i1> not allowed.\n",
-      Munger.getLinesWithPrefix(ErrorPrefix));
-  EXPECT_EQ(
-      "    @t1 = <1 x i1>;\n",
-      Munger.getLinesWithSubstring("@t1"));
-
-  // Test that we don't handle <2 x i1>.
-  const uint64_t Vector2xI1[] = {
-    ReplaceIndex, NaClMungedBitcode::Replace, 3, 12, 2, 0, Terminator
-  };
-  EXPECT_FALSE(Munger.runTestForAssembly(ARRAY(Vector2xI1)));
-  EXPECT_EQ(
-      "Error(37:0): Vector type <2 x i1> not allowed.\n",
-      Munger.getLinesWithPrefix(ErrorPrefix));
-  EXPECT_EQ(
-      "    @t1 = <2 x i1>;\n",
-      Munger.getLinesWithSubstring("@t1"));
-
-  // Test that we don't handle <3 x i1>.
-  const uint64_t Vector3xI1[] = {
-    ReplaceIndex, NaClMungedBitcode::Replace, 3, 12, 3, 0, Terminator
-  };
-  EXPECT_FALSE(Munger.runTestForAssembly(ARRAY(Vector3xI1)));
-  EXPECT_EQ(
-      "Error(37:0): Vector type <3 x i1> not allowed.\n",
-      Munger.getLinesWithPrefix(ErrorPrefix));
-  EXPECT_EQ(
-      "    @t1 = <3 x i1>;\n",
-      Munger.getLinesWithSubstring("@t1"));
-
-  // Test that we handle <8 x i1>.
-  const uint64_t Vector8xI1[] = {
-    ReplaceIndex, NaClMungedBitcode::Replace, 3, 12, 8, 0, Terminator
-  };
-  EXPECT_TRUE(Munger.runTest(ARRAY(Vector8xI1)));
-
-  // Test that we handle <16 x i1>.
-  const uint64_t Vector16xI1[] = {
-    ReplaceIndex, NaClMungedBitcode::Replace, 3, 12, 16, 0, Terminator
-  };
-  EXPECT_TRUE(Munger.runTest(ARRAY(Vector16xI1)));
-
-  // Test that we reject <32 x i1>.
-  const uint64_t Vector32xI1[] = {
-    ReplaceIndex, NaClMungedBitcode::Replace, 3, 12, 32, 0, Terminator
-  };
-  EXPECT_FALSE(Munger.runTestForAssembly(ARRAY(Vector32xI1)));
-  EXPECT_EQ(
-      "Error(37:0): Vector type <32 x i1> not allowed.\n",
-      Munger.getLinesWithPrefix(ErrorPrefix));
-  EXPECT_EQ(
-      "    @t1 = <32 x i1>;\n",
-      Munger.getLinesWithSubstring("@t1"));
-}
-
-// Test i8 vector records in the types block.
-TEST(NaClObjDumpTypesTest, TestI8VectorRecord) {
-  const uint64_t BitcodeRecords[] = {
-    1, 65535, 8, 2, Terminator,
-    1, 65535, 17, 2, Terminator,
-    3, 1, 2, Terminator,
-    3, 7, 8, Terminator,
-    3, 12, 16, 0, Terminator,
-    0, 65534, Terminator,
-    0, 65534, Terminator
-  };
-
-  const uint64_t ReplaceIndex = 4;
-
-  NaClObjDumpMunger Munger(ARRAY_TERM(BitcodeRecords));
-
-  // Test that we accept <16 x i8>.
-  EXPECT_TRUE(Munger.runTestForAssembly());
-  EXPECT_EQ(
-      "module {  // BlockID = 8\n"
-      "  types {  // BlockID = 17\n"
-      "    count 2;\n"
-      "    @t0 = i8;\n"
-      "    @t1 = <16 x i8>;\n"
-      "  }\n"
-      "}\n",
-      Munger.getTestResults());
-
-  // Test that we reject <1 x i8>.
-  const uint64_t Vector1xI8[] = {
-    ReplaceIndex, NaClMungedBitcode::Replace, 3, 12, 1, 0, Terminator
-  };
-  EXPECT_FALSE(Munger.runTestForAssembly(ARRAY(Vector1xI8)));
-  EXPECT_EQ(
-      "Error(37:0): Vector type <1 x i8> not allowed.\n",
-      Munger.getLinesWithPrefix(ErrorPrefix));
-  EXPECT_EQ(
-      "    @t1 = <1 x i8>;\n",
-      Munger.getLinesWithSubstring("@t1"));
-
-  // Test that we don't handle <2 x i8>.
-  const uint64_t Vector2xI8[] = {
-    ReplaceIndex, NaClMungedBitcode::Replace, 3, 12, 2, 0, Terminator
-  };
-  EXPECT_FALSE(Munger.runTestForAssembly(ARRAY(Vector2xI8)));
-  EXPECT_EQ(
-      "Error(37:0): Vector type <2 x i8> not allowed.\n",
-      Munger.getLinesWithPrefix(ErrorPrefix));
-  EXPECT_EQ(
-      "    @t1 = <2 x i8>;\n",
-      Munger.getLinesWithSubstring("@t1"));
-
-  // Test that we don't handle <3 x i8>.
-  const uint64_t Vector3xI8[] = {
-    ReplaceIndex, NaClMungedBitcode::Replace, 3, 12, 3, 0, Terminator
-  };
-  EXPECT_FALSE(Munger.runTestForAssembly(ARRAY(Vector3xI8)));
-  EXPECT_EQ(
-      "Error(37:0): Vector type <3 x i8> not allowed.\n",
-      Munger.getLinesWithPrefix(ErrorPrefix));
-  EXPECT_EQ(
-      "    @t1 = <3 x i8>;\n",
-      Munger.getLinesWithSubstring("@t1"));
-
-  // Test that we don't handle <4 x i8>.
-  const uint64_t Vector4xI8[] = {
-    ReplaceIndex, NaClMungedBitcode::Replace, 3, 12, 4, 0, Terminator
-  };
-  EXPECT_FALSE(Munger.runTestForAssembly(ARRAY(Vector4xI8)));
-  EXPECT_EQ(
-      "Error(37:0): Vector type <4 x i8> not allowed.\n",
-      Munger.getLinesWithPrefix(ErrorPrefix));
-  EXPECT_EQ(
-      "    @t1 = <4 x i8>;\n",
-      Munger.getLinesWithSubstring("@t1"));
-
-  // Test that we don't handle <8 x i8>.
-  const uint64_t Vector8xI8[] = {
-    ReplaceIndex, NaClMungedBitcode::Replace, 3, 12, 8, 0, Terminator
-  };
-  EXPECT_FALSE(Munger.runTestForAssembly(ARRAY(Vector8xI8)));
-  EXPECT_EQ(
-      "Error(37:0): Vector type <8 x i8> not allowed.\n",
-      Munger.getLinesWithPrefix(ErrorPrefix));
-  EXPECT_EQ(
-      "    @t1 = <8 x i8>;\n",
-      Munger.getLinesWithSubstring("@t1"));
-
-  // Test that we don't handle <32 x i8>.
-  const uint64_t Vector32xI8[] = {
-    ReplaceIndex, NaClMungedBitcode::Replace, 3, 12, 32, 0, Terminator
-  };
-  EXPECT_FALSE(Munger.runTestForAssembly(ARRAY(Vector32xI8)));
-  EXPECT_EQ(
-      "Error(37:0): Vector type <32 x i8> not allowed.\n",
-      Munger.getLinesWithPrefix(ErrorPrefix));
-  EXPECT_EQ(
-      "    @t1 = <32 x i8>;\n",
-      Munger.getLinesWithSubstring("@t1"));
-}
-
-// Test i16 vector records in the types block.
-TEST(NaClObjDumpTypesTest, TestI16VectorRecord) {
-  const uint64_t BitcodeRecords[] = {
-    1, 65535, 8, 2, Terminator,
-    1, 65535, 17, 2, Terminator,
-    3, 1, 2, Terminator,
-    3, 7, 16, Terminator,
-    3, 12, 8, 0, Terminator,
-    0, 65534, Terminator,
-    0, 65534, Terminator
-  };
-
-  const uint64_t ReplaceIndex = 4;
-
-  NaClObjDumpMunger Munger(ARRAY_TERM(BitcodeRecords));
-
-  // Test that we accept <8 x i16>.
-  EXPECT_TRUE(Munger.runTestForAssembly());
-  EXPECT_EQ(
-      "module {  // BlockID = 8\n"
-      "  types {  // BlockID = 17\n"
-      "    count 2;\n"
-      "    @t0 = i16;\n"
-      "    @t1 = <8 x i16>;\n"
-      "  }\n"
-      "}\n",
-      Munger.getTestResults());
-
-  // Test that we reject <1 x i16>.
-  const uint64_t Vector1xI16[] = {
-    ReplaceIndex, NaClMungedBitcode::Replace, 3, 12, 1, 0, Terminator
-  };
-  EXPECT_FALSE(Munger.runTestForAssembly(ARRAY(Vector1xI16)));
-  EXPECT_EQ(
-      "Error(37:0): Vector type <1 x i16> not allowed.\n",
-      Munger.getLinesWithPrefix(ErrorPrefix));
-  EXPECT_EQ(
-      "    @t1 = <1 x i16>;\n",
-      Munger.getLinesWithSubstring("@t1"));
-
-  // Test that we don't handle <2 x i16>.
-  const uint64_t Vector2xI16[] = {
-    ReplaceIndex, NaClMungedBitcode::Replace, 3, 12, 2, 0, Terminator
-  };
-  EXPECT_FALSE(Munger.runTestForAssembly(ARRAY(Vector2xI16)));
-  EXPECT_EQ(
-      "Error(37:0): Vector type <2 x i16> not allowed.\n",
-      Munger.getLinesWithPrefix(ErrorPrefix));
-  EXPECT_EQ(
-      "    @t1 = <2 x i16>;\n",
-      Munger.getLinesWithSubstring("@t1"));
-
-  // Test that we don't handle <3 x i16>.
-  const uint64_t Vector3xI16[] = {
-    ReplaceIndex, NaClMungedBitcode::Replace, 3, 12, 3, 0, Terminator
-  };
-  EXPECT_FALSE(Munger.runTestForAssembly(ARRAY(Vector3xI16)));
-  EXPECT_EQ(
-      "Error(37:0): Vector type <3 x i16> not allowed.\n",
-      Munger.getLinesWithPrefix(ErrorPrefix));
-  EXPECT_EQ(
-      "    @t1 = <3 x i16>;\n",
-      Munger.getLinesWithSubstring("@t1"));
-
-  // Test that we don't handle <4 x i16>.
-  const uint64_t Vector4xI16[] = {
-    ReplaceIndex, NaClMungedBitcode::Replace, 3, 12, 4, 0, Terminator
-  };
-  EXPECT_FALSE(Munger.runTestForAssembly(ARRAY(Vector4xI16)));
-  EXPECT_EQ(
-      "Error(37:0): Vector type <4 x i16> not allowed.\n",
-      Munger.getLinesWithPrefix(ErrorPrefix));
-  EXPECT_EQ(
-      "    @t1 = <4 x i16>;\n",
-      Munger.getLinesWithSubstring("@t1"));
-
-  // Test that we don't handle <16 x i16>.
-  const uint64_t Vector16xI16[] = {
-    ReplaceIndex, NaClMungedBitcode::Replace, 3, 12, 16, 0, Terminator
-  };
-  EXPECT_FALSE(Munger.runTestForAssembly(ARRAY(Vector16xI16)));
-  EXPECT_EQ(
-      "Error(37:0): Vector type <16 x i16> not allowed.\n",
-      Munger.getLinesWithPrefix(ErrorPrefix));
-  EXPECT_EQ(
-      "    @t1 = <16 x i16>;\n",
-      Munger.getLinesWithSubstring("@t1"));
-
-  // Test that we don't handle <32 x i16>.
-  const uint64_t Vector32xI16[] = {
-    ReplaceIndex, NaClMungedBitcode::Replace, 3, 12, 32, 0, Terminator
-  };
-  EXPECT_FALSE(Munger.runTestForAssembly(ARRAY(Vector32xI16)));
-  EXPECT_EQ(
-      "Error(37:0): Vector type <32 x i16> not allowed.\n",
-      Munger.getLinesWithPrefix(ErrorPrefix));
-  EXPECT_EQ(
-      "    @t1 = <32 x i16>;\n",
-      Munger.getLinesWithSubstring("@t1"));
-}
-
-// Test i32 vector records in the types block.
-TEST(NaClObjDumpTypesTest, TestI32VectorRecord) {
-  const uint64_t BitcodeRecords[] = {
-    1, 65535, 8, 2, Terminator,
-    1, 65535, 17, 2, Terminator,
-    3, 1, 2, Terminator,
-    3, 7, 32, Terminator,
-    3, 12, 4, 0, Terminator,
-    0, 65534, Terminator,
-    0, 65534, Terminator
-  };
-
-  const uint64_t ReplaceIndex = 4;
-
-  NaClObjDumpMunger Munger(ARRAY_TERM(BitcodeRecords));
-
-  // Test that we accept <4 x i32>.
-  EXPECT_TRUE(Munger.runTestForAssembly());
-  EXPECT_EQ(
-      "module {  // BlockID = 8\n"
-      "  types {  // BlockID = 17\n"
-      "    count 2;\n"
-      "    @t0 = i32;\n"
-      "    @t1 = <4 x i32>;\n"
-      "  }\n"
-      "}\n",
-      Munger.getTestResults());
-
-  // Test that we reject <1 x i32>.
-  const uint64_t Vector1xI32[] = {
-    ReplaceIndex, NaClMungedBitcode::Replace, 3, 12, 1, 0, Terminator
-  };
-  EXPECT_FALSE(Munger.runTestForAssembly(ARRAY(Vector1xI32)));
-  EXPECT_EQ(
-      "Error(37:6): Vector type <1 x i32> not allowed.\n",
-      Munger.getLinesWithPrefix(ErrorPrefix));
-  EXPECT_EQ(
-      "    @t1 = <1 x i32>;\n",
-      Munger.getLinesWithSubstring("@t1"));
-
-  // Test that we don't handle <2 x i32>.
-  const uint64_t Vector2xI32[] = {
-    ReplaceIndex, NaClMungedBitcode::Replace, 3, 12, 2, 0, Terminator
-  };
-  EXPECT_FALSE(Munger.runTestForAssembly(ARRAY(Vector2xI32)));
-  EXPECT_EQ(
-      "Error(37:6): Vector type <2 x i32> not allowed.\n",
-      Munger.getLinesWithPrefix(ErrorPrefix));
-  EXPECT_EQ(
-      "    @t1 = <2 x i32>;\n",
-      Munger.getLinesWithSubstring("@t1"));
-
-  // Test that we don't handle <3 x i32>.
-  const uint64_t Vector3xI32[] = {
-    ReplaceIndex, NaClMungedBitcode::Replace, 3, 12, 3, 0, Terminator
-  };
-  EXPECT_FALSE(Munger.runTestForAssembly(ARRAY(Vector3xI32)));
-  EXPECT_EQ(
-      "Error(37:6): Vector type <3 x i32> not allowed.\n",
-      Munger.getLinesWithPrefix(ErrorPrefix));
-  EXPECT_EQ(
-      "    @t1 = <3 x i32>;\n",
-      Munger.getLinesWithSubstring("@t1"));
-
-  // Test that we don't handle <8 x i32>.
-  const uint64_t Vector8xI32[] = {
-    ReplaceIndex, NaClMungedBitcode::Replace, 3, 12, 8, 0, Terminator
-  };
-  EXPECT_FALSE(Munger.runTestForAssembly(ARRAY(Vector8xI32)));
-  EXPECT_EQ(
-      "Error(37:6): Vector type <8 x i32> not allowed.\n",
-      Munger.getLinesWithPrefix(ErrorPrefix));
-  EXPECT_EQ(
-      "    @t1 = <8 x i32>;\n",
-      Munger.getLinesWithSubstring("@t1"));
-
-  // Test that we don't handle <16 x i32>.
-  const uint64_t Vector16xI32[] = {
-    ReplaceIndex, NaClMungedBitcode::Replace, 3, 12, 16, 0, Terminator
-  };
-  EXPECT_FALSE(Munger.runTestForAssembly(ARRAY(Vector16xI32)));
-  EXPECT_EQ(
-      "Error(37:6): Vector type <16 x i32> not allowed.\n",
-      Munger.getLinesWithPrefix(ErrorPrefix));
-  EXPECT_EQ(
-      "    @t1 = <16 x i32>;\n",
-      Munger.getLinesWithSubstring("@t1"));
-
-  // Test that we don't handle <32 x i32>.
-  const uint64_t Vector32xI32[] = {
-    ReplaceIndex, NaClMungedBitcode::Replace, 3, 12, 32, 0, Terminator
-  };
-  EXPECT_FALSE(Munger.runTestForAssembly(ARRAY(Vector32xI32)));
-  EXPECT_EQ(
-      "Error(37:6): Vector type <32 x i32> not allowed.\n",
-      Munger.getLinesWithPrefix(ErrorPrefix));
-  EXPECT_EQ(
-      "    @t1 = <32 x i32>;\n",
-      Munger.getLinesWithSubstring("@t1"));
-}
-
-// Test i64 vector types.
-TEST(NaClObjDumpTypesTest, TestI64VectorRecord) {
-  const uint64_t BitcodeRecords[] = {
-    1, 65535, 8, 2, Terminator,
-    1, 65535, 17, 2, Terminator,
-    3, 1, 2, Terminator,
-    3, 7, 64, Terminator,
-    3, 12, 1, 0, Terminator,
-    0, 65534, Terminator,
-    0, 65534, Terminator
-  };
-
-  const uint64_t ReplaceIndex = 4;
-
-  NaClObjDumpMunger Munger(ARRAY_TERM(BitcodeRecords));
-
-  // Test that we reject <1 x i64>.
-  EXPECT_FALSE(Munger.runTestForAssembly());
-  EXPECT_EQ(
-      "module {  // BlockID = 8\n"
-      "  types {  // BlockID = 17\n"
-      "    count 2;\n"
-      "    @t0 = i64;\n"
-      "    @t1 = <1 x i64>;\n"
-      "Error(37:6): Vector type <1 x i64> not allowed.\n"
-      "  }\n"
-      "}\n",
-      Munger.getTestResults());
-
-  // Test that we don't handle <2 x i64>.
-  const uint64_t Vector2xI64[] = {
-    ReplaceIndex, NaClMungedBitcode::Replace, 3, 12, 2, 0, Terminator
-  };
-  EXPECT_FALSE(Munger.runTestForAssembly(ARRAY(Vector2xI64)));
-  EXPECT_EQ(
-      "Error(37:6): Vector type <2 x i64> not allowed.\n",
-      Munger.getLinesWithPrefix(ErrorPrefix));
-  EXPECT_EQ(
-      "    @t1 = <2 x i64>;\n",
-      Munger.getLinesWithSubstring("@t1"));
-
-  // Test that we don't handle <3 x i64>.
-  const uint64_t Vector3xI64[] = {
-    ReplaceIndex, NaClMungedBitcode::Replace, 3, 12, 3, 0, Terminator
-  };
-  EXPECT_FALSE(Munger.runTestForAssembly(ARRAY(Vector3xI64)));
-  EXPECT_EQ(
-      "Error(37:6): Vector type <3 x i64> not allowed.\n",
-      Munger.getLinesWithPrefix(ErrorPrefix));
-  EXPECT_EQ(
-      "    @t1 = <3 x i64>;\n",
-      Munger.getLinesWithSubstring("@t1"));
-
-  // Test that we reject <4 x i64>.
-  const uint64_t Vector4xI64[] = {
-    ReplaceIndex, NaClMungedBitcode::Replace, 3, 12, 4, 0, Terminator
-  };
-  EXPECT_FALSE(Munger.runTestForAssembly(ARRAY(Vector4xI64)));
-  EXPECT_EQ(
-      "Error(37:6): Vector type <4 x i64> not allowed.\n",
-      Munger.getLinesWithPrefix(ErrorPrefix));
-  EXPECT_EQ(
-      "    @t1 = <4 x i64>;\n",
-      Munger.getLinesWithSubstring("@t1"));
-
-  // Test that we don't handle <8 x i64>.
-  const uint64_t Vector8xI64[] = {
-    ReplaceIndex, NaClMungedBitcode::Replace, 3, 12, 8, 0, Terminator
-  };
-  EXPECT_FALSE(Munger.runTestForAssembly(ARRAY(Vector8xI64)));
-  EXPECT_EQ(
-      "Error(37:6): Vector type <8 x i64> not allowed.\n",
-      Munger.getLinesWithPrefix(ErrorPrefix));
-  EXPECT_EQ(
-      "    @t1 = <8 x i64>;\n",
-      Munger.getLinesWithSubstring("@t1"));
-
-  // Test that we don't handle <16 x i64>.
-  const uint64_t Vector16xI64[] = {
-    ReplaceIndex, NaClMungedBitcode::Replace, 3, 12, 16, 0, Terminator
-  };
-  EXPECT_FALSE(Munger.runTestForAssembly(ARRAY(Vector16xI64)));
-  EXPECT_EQ(
-      "Error(37:6): Vector type <16 x i64> not allowed.\n",
-      Munger.getLinesWithPrefix(ErrorPrefix));
-  EXPECT_EQ(
-      "    @t1 = <16 x i64>;\n",
-      Munger.getLinesWithSubstring("@t1"));
-
-  // Test that we don't handle <32 x i64>.
-  const uint64_t Vector32xI64[] = {
-    ReplaceIndex, NaClMungedBitcode::Replace, 3, 12, 32, 0, Terminator
-  };
-  EXPECT_FALSE(Munger.runTestForAssembly(ARRAY(Vector32xI64)));
-  EXPECT_EQ(
-      "Error(37:6): Vector type <32 x i64> not allowed.\n",
-      Munger.getLinesWithPrefix(ErrorPrefix));
-  EXPECT_EQ(
-      "    @t1 = <32 x i64>;\n",
-      Munger.getLinesWithSubstring("@t1"));
-}
-
-// Test handling of float vector types.
-TEST(NaClObjDumpTypesTest, TestFloatVectorRecord) {
-  const uint64_t BitcodeRecords[] = {
-    1, 65535, 8, 2, Terminator,
-    1, 65535, 17, 2, Terminator,
-    3, 1, 2, Terminator,
-    3, 3, Terminator,
-    3, 12, 4, 0, Terminator,
-    0, 65534, Terminator,
-    0, 65534, Terminator
-  };
-
-  const uint64_t ReplaceIndex = 4;
-
-  NaClObjDumpMunger Munger(ARRAY_TERM(BitcodeRecords));
-
-  // Test that we accept <4 x float>.
-  EXPECT_TRUE(Munger.runTestForAssembly());
-  EXPECT_EQ(
-      "module {  // BlockID = 8\n"
-      "  types {  // BlockID = 17\n"
-      "    count 2;\n"
-      "    @t0 = float;\n"
-      "    @t1 = <4 x float>;\n"
-      "  }\n"
-      "}\n",
-      Munger.getTestResults());
-
-  // Test that we reject <1 x float>.
-  const uint64_t Vector1xFloat[] = {
-    ReplaceIndex, NaClMungedBitcode::Replace, 3, 12, 1, 0, Terminator
-  };
-  EXPECT_FALSE(Munger.runTestForAssembly(ARRAY(Vector1xFloat)));
-  EXPECT_EQ(
-      "Error(36:2): Vector type <1 x float> not allowed.\n",
-      Munger.getLinesWithPrefix(ErrorPrefix));
-  EXPECT_EQ(
-      "    @t1 = <1 x float>;\n",
-      Munger.getLinesWithSubstring("@t1"));
-
-  // Test that we reject <2 x float>.
-  const uint64_t Vector2xFloat[] = {
-    ReplaceIndex, NaClMungedBitcode::Replace, 3, 12, 2, 0, Terminator
-  };
-  EXPECT_FALSE(Munger.runTestForAssembly(ARRAY(Vector2xFloat)));
-  EXPECT_EQ(
-      "Error(36:2): Vector type <2 x float> not allowed.\n",
-      Munger.getLinesWithPrefix(ErrorPrefix));
-  EXPECT_EQ(
-      "    @t1 = <2 x float>;\n",
-      Munger.getLinesWithSubstring("@t1"));
-
-  // Test that we reject <3 x float>.
-  const uint64_t Vector3xFloat[] = {
-    ReplaceIndex, NaClMungedBitcode::Replace, 3, 12, 3, 0, Terminator
-  };
-  EXPECT_FALSE(Munger.runTestForAssembly(ARRAY(Vector3xFloat)));
-  EXPECT_EQ(
-      "Error(36:2): Vector type <3 x float> not allowed.\n",
-      Munger.getLinesWithPrefix(ErrorPrefix));
-  EXPECT_EQ(
-      "    @t1 = <3 x float>;\n",
-      Munger.getLinesWithSubstring("@t1"));
-
-  // Test that we reject <8 x float>.
-  const uint64_t Vector8xFloat[] = {
-    ReplaceIndex, NaClMungedBitcode::Replace, 3, 12, 8, 0, Terminator
-  };
-  EXPECT_FALSE(Munger.runTestForAssembly(ARRAY(Vector8xFloat)));
-  EXPECT_EQ(
-      "Error(36:2): Vector type <8 x float> not allowed.\n",
-      Munger.getLinesWithPrefix(ErrorPrefix));
-  EXPECT_EQ(
-      "    @t1 = <8 x float>;\n",
-      Munger.getLinesWithSubstring("@t1"));
-}
-
-// Test handling of double vector types.
-TEST(NaClObjDumpTypesTest, TestDoubleVectorRecord) {
-  const uint64_t BitcodeRecords[] = {
-    1, 65535, 8, 2, Terminator,
-    1, 65535, 17, 2, Terminator,
-    3, 1, 2, Terminator,
-    3, 4, Terminator,
-    3, 12, 4, 0, Terminator,
-    0, 65534, Terminator,
-    0, 65534, Terminator
-  };
-
-  const uint64_t ReplaceIndex = 4;
-
-  NaClObjDumpMunger Munger(ARRAY_TERM(BitcodeRecords));
-
-  // Test that we reject <4 x double>.
-  EXPECT_FALSE(Munger.runTestForAssembly());
-  EXPECT_EQ(
-      "module {  // BlockID = 8\n"
-      "  types {  // BlockID = 17\n"
-      "    count 2;\n"
-      "    @t0 = double;\n"
-      "    @t1 = <4 x double>;\n"
-      "Error(36:2): Vector type <4 x double> not allowed.\n"
-      "  }\n"
-      "}\n",
-      Munger.getTestResults());
-
-  // Test that we reject <1 x double>.
-  const uint64_t Vector1xDouble[] = {
-    ReplaceIndex, NaClMungedBitcode::Replace, 3, 12, 1, 0, Terminator
-  };
-  EXPECT_FALSE(Munger.runTestForAssembly(ARRAY(Vector1xDouble)));
-  EXPECT_EQ(
-      "Error(36:2): Vector type <1 x double> not allowed.\n",
-      Munger.getLinesWithPrefix(ErrorPrefix));
-  EXPECT_EQ(
-      "    @t1 = <1 x double>;\n",
-      Munger.getLinesWithSubstring("@t1"));
-
-  // Test that we reject <2 x double>.
-  const uint64_t Vector2xDouble[] = {
-    ReplaceIndex, NaClMungedBitcode::Replace, 3, 12, 2, 0, Terminator
-  };
-  EXPECT_FALSE(Munger.runTestForAssembly(ARRAY(Vector2xDouble)));
-  EXPECT_EQ(
-      "Error(36:2): Vector type <2 x double> not allowed.\n",
-      Munger.getLinesWithPrefix(ErrorPrefix));
-  EXPECT_EQ(
-      "    @t1 = <2 x double>;\n",
-      Munger.getLinesWithSubstring("@t1"));
-
-  // Test that we reject <4 x double>.
-  const uint64_t Vector3xDouble[] = {
-    ReplaceIndex, NaClMungedBitcode::Replace, 3, 12, 4, 0, Terminator
-  };
-  EXPECT_FALSE(Munger.runTestForAssembly(ARRAY(Vector3xDouble)));
-  EXPECT_EQ(
-      "Error(36:2): Vector type <4 x double> not allowed.\n",
-      Munger.getLinesWithPrefix(ErrorPrefix));
-  EXPECT_EQ(
-      "    @t1 = <4 x double>;\n",
-      Munger.getLinesWithSubstring("@t1"));
-
-  // Test that we reject <8 x double>.
-  const uint64_t Vector8xDouble[] = {
-    ReplaceIndex, NaClMungedBitcode::Replace, 3, 12, 8, 0, Terminator
-  };
-  EXPECT_FALSE(Munger.runTestForAssembly(ARRAY(Vector8xDouble)));
-  EXPECT_EQ(
-      "Error(36:2): Vector type <8 x double> not allowed.\n",
-      Munger.getLinesWithPrefix(ErrorPrefix));
-  EXPECT_EQ(
-      "    @t1 = <8 x double>;\n",
-      Munger.getLinesWithSubstring("@t1"));
-}
-
-// Tests that we don't accept vectors of type void.
-TEST(NaClObjDumpTypesTest, TestVoidVectorRecord) {
-  const uint64_t BitcodeRecords[] = {
-    1, 65535, 8, 2, Terminator,
-    1, 65535, 17, 2, Terminator,
-    3, 1, 2, Terminator,
-    3, 2, Terminator,
-    3, 12, 4, 0, Terminator,
-    0, 65534, Terminator,
-    0, 65534, Terminator
-  };
-  NaClObjDumpMunger Munger(ARRAY_TERM(BitcodeRecords));
-  EXPECT_FALSE(Munger.runTestForAssembly());
-  EXPECT_EQ(
-      "module {  // BlockID = 8\n"
-      "  types {  // BlockID = 17\n"
-      "    count 2;\n"
-      "    @t0 = void;\n"
-      "    @t1 = <4 x i32>;\n"
-      "Error(36:2): Vectors can only be defined on primitive types. "
-      "Found void. Assuming i32 instead.\n"
-      "  }\n"
-      "}\n",
-      Munger.getTestResults());
-}
-
-// Tests that we don't allow vectors of vectors.
-TEST(NaClObjDumpTypesTest, TestNestedVectorRecord) {
-  const uint64_t BitcodeRecords[] = {
-    1, 65535, 8, 2, Terminator,
-    1, 65535, 17, 2, Terminator,
-    3, 1, 3, Terminator,
-    3, 3, Terminator,
-    3, 12, 4, 0, Terminator,
-    3, 12, 4, 1, Terminator,
-    0, 65534, Terminator,
-    0, 65534, Terminator
-  };
-  NaClObjDumpMunger Munger(ARRAY_TERM(BitcodeRecords));
-  EXPECT_FALSE(Munger.runTestForAssembly());
-  EXPECT_EQ(
-      "module {  // BlockID = 8\n"
-      "  types {  // BlockID = 17\n"
-      "    count 3;\n"
-      "    @t0 = float;\n"
-      "    @t1 = <4 x float>;\n"
-      "    @t2 = <4 x i32>;\n"
-      "Error(39:4): Vectors can only be defined on primitive types. "
-      "Found <4 x float>. Assuming i32 instead.\n"
-      "  }\n"
-      "}\n",
-      Munger.getTestResults());
-}
-
-// Test handling of the function record in the types block.
-TEST(NaClObjDumpTypesTest, TestFunctionRecord) {
-  const uint64_t BitcodeRecords[] = {
-    1, 65535, 8, 2, Terminator,
-    1, 65535, 17, 2, Terminator,
-    3, 1, 7, Terminator,
-    3, 2, Terminator,
-    3, 7, 16, Terminator,
-    3, 7, 32, Terminator,
-    3, 3, Terminator,
-    3, 4, Terminator,
-    3, 12, 4, 2, Terminator,
-    3, 21, 0, 0, Terminator,
-    0, 65534, Terminator,
-    0, 65534, Terminator
-  };
-
-  const uint64_t TypeCountIndex = 2;
-  const uint64_t ReplaceIndex = 9;
-
-  NaClObjDumpMunger Munger(ARRAY_TERM(BitcodeRecords));
-
-  // Test void() signature.
-  EXPECT_TRUE(Munger.runTestForAssembly());
-  EXPECT_EQ(
-      "module {  // BlockID = 8\n"
-      "  types {  // BlockID = 17\n"
-      "    count 7;\n"
-      "    @t0 = void;\n"
-      "    @t1 = i16;\n"
-      "    @t2 = i32;\n"
-      "    @t3 = float;\n"
-      "    @t4 = double;\n"
-      "    @t5 = <4 x i32>;\n"
-      "    @t6 = void ();\n"
-      "  }\n}\n",
-      Munger.getTestResults());
-  EXPECT_EQ(
-      "    @t6 = void ();\n",
-      Munger.getLinesWithSubstring("@t6"));
-
-  // Tests using integers for parameters and return types.
-  const uint64_t UsesIntegerTypes[] = {
-    ReplaceIndex, NaClMungedBitcode::Replace, 3, 21, 0, 1, 2, 1, Terminator
-  };
-  EXPECT_TRUE(Munger.runTestForAssembly(ARRAY(UsesIntegerTypes)));
-  EXPECT_EQ(
-      "    @t6 = i16 (i32, i16);\n",
-      Munger.getLinesWithSubstring("@t6"));
-
-  // Test using float point types for parameters and return types.
-  const uint64_t UsesFloatingTypes[] = {
-    ReplaceIndex, NaClMungedBitcode::Replace, 3, 21, 0, 3, 3, 4, Terminator
-  };
-  EXPECT_TRUE(Munger.runTestForAssembly(ARRAY(UsesFloatingTypes)));
-  EXPECT_EQ(
-      "    @t6 = float (float, double);\n",
-      Munger.getLinesWithSubstring("@t6"));
-
-  // Test using vector types for parameters and return types.
-  const uint64_t UsesVectorTypes[] = {
-    ReplaceIndex, NaClMungedBitcode::Replace, 3, 21, 0, 5, 5, Terminator
-  };
-  EXPECT_TRUE(Munger.runTestForAssembly(ARRAY(UsesVectorTypes)));
-  EXPECT_EQ(
-      "    @t6 = <4 x i32> (<4 x i32>);\n",
-      Munger.getLinesWithSubstring("@t6"));
-
-  // Test error if function record is too short.
-  const uint64_t FunctionRecordTooShort[] = {
-    ReplaceIndex, NaClMungedBitcode::Replace, 3, 21, 0, Terminator
-  };
-  EXPECT_FALSE(Munger.runTestForAssembly(ARRAY(FunctionRecordTooShort)));
-  EXPECT_EQ(
-      "Error(48:6): Function record should contain at least 2 arguments. "
-      "Found: 1\n",
-      Munger.getLinesWithPrefix(ErrorPrefix));
-  EXPECT_EQ(
-      "    @t6 = void;\n",
-      Munger.getLinesWithSubstring("@t6"));
-
-  // Tests errror if function record specifies varargs.
-  const uint64_t FunctionRecordWithVarArgs[] = {
-    ReplaceIndex, NaClMungedBitcode::Replace, 3, 21, 1, 0, Terminator
-  };
-  EXPECT_FALSE(Munger.runTestForAssembly(ARRAY(FunctionRecordWithVarArgs)));
-  EXPECT_EQ(
-      "Error(48:6): Functions with variable length arguments is "
-      "not supported\n",
-      Munger.getLinesWithPrefix(ErrorPrefix));
-  EXPECT_EQ(
-      "    @t6 = void (...);\n",
-      Munger.getLinesWithSubstring("@t6"));
-
-  // Tests if void is used as a parameter type.
-  const uint64_t VoidParamType[] = {
-    ReplaceIndex, NaClMungedBitcode::Replace, 3, 21, 0, 0, 0, Terminator
-  };
-  EXPECT_FALSE(Munger.runTestForAssembly(ARRAY(VoidParamType)));
-  EXPECT_EQ(
-      "Error(48:6): Invalid type for parameter 1. Found: void. Assuming: i32\n",
-      Munger.getLinesWithPrefix(ErrorPrefix));
-  EXPECT_EQ(
-      "    @t6 = void (i32);\n",
-      Munger.getLinesWithSubstring("@t6"));
-
-  // Tests using a function type as the return type.
-  const uint64_t FunctionReturnType[] = {
-    TypeCountIndex, NaClMungedBitcode::Replace, 3, 1, 8, Terminator,
-    ReplaceIndex, NaClMungedBitcode::AddAfter, 3, 21, 0, 6, Terminator
-  };
-  EXPECT_FALSE(Munger.runTestForAssembly(ARRAY(FunctionReturnType)));
-  EXPECT_EQ(
-      "Error(52:0): Invalid return type. Found: void (). Assuming: i32\n",
-      Munger.getLinesWithPrefix(ErrorPrefix));
-  EXPECT_EQ(
-      "    @t6 = void ();\n",
-      Munger.getLinesWithSubstring("@t6"));
-  EXPECT_EQ(
-      "    @t7 = i32 ();\n",
-      Munger.getLinesWithSubstring("@t7"));
-
-  // Tests using a function type as a parameter type.
-  const uint64_t FunctionParamType[] = {
-    TypeCountIndex, NaClMungedBitcode::Replace, 3, 1, 8, Terminator,
-    ReplaceIndex, NaClMungedBitcode::AddAfter, 3, 21, 0, 0, 6, Terminator
-  };
-  EXPECT_FALSE(Munger.runTestForAssembly(ARRAY(FunctionParamType)));
-  EXPECT_EQ(
-      "Error(52:0): Invalid type for parameter 1. Found: void (). "
-      "Assuming: i32\n",
-      Munger.getLinesWithPrefix(ErrorPrefix));
-  EXPECT_EQ(
-      "    @t6 = void ();\n",
-      Munger.getLinesWithSubstring("@t6"));
-  EXPECT_EQ(
-      "    @t7 = void (i32);\n",
-      Munger.getLinesWithSubstring("@t7"));
-}
-
-// Tests how we report unknown record codes in the types block.
-TEST(NaClObjDumpTypesTest, TestUnknownTypesRecordCode) {
-  const uint64_t BitcodeRecords[] = {
-    1, 65535, 8, 2, Terminator,
-    1, 65535, 17, 2, Terminator,
-    3, 1, 1, Terminator,
-    3, 10, Terminator,
-    0, 65534, Terminator,
-    0, 65534, Terminator
-  };
-  NaClObjDumpMunger Munger(ARRAY_TERM(BitcodeRecords));
-  EXPECT_FALSE(Munger.runTestForAssembly());
-  EXPECT_EQ(
-      "module {  // BlockID = 8\n"
-      "  types {  // BlockID = 17\n"
-      "    count 1;\n"
-      "Error(34:4): Unknown record code in types block. Found: 10\n"
-      "  }\n"
-      "Error(36:2): Expected 1 types but found: 0\n"
-      "}\n",
-      Munger.getTestResults());
-}
-
-} // End of namespace naclmungetest
diff --git a/unittests/Bitcode/NaClParseInstsTest.cpp b/unittests/Bitcode/NaClParseInstsTest.cpp
deleted file mode 100644
index 63ef745381e..00000000000
--- a/unittests/Bitcode/NaClParseInstsTest.cpp
+++ /dev/null
@@ -1,419 +0,0 @@
-//===- llvm/unittest/Bitcode/NaClParseInstsTest.cpp ----------------------===//
-//     Tests parser for PNaCl bitcode instructions.
-//
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
-//
-//===----------------------------------------------------------------------===//
-
-// Tests record errors in the function block when parsing PNaCl bitcode.
-
-// TODO(kschimpf) Add more tests.
-
-
-#include "NaClMungeTest.h"
-
-#include "llvm/Bitcode/NaCl/NaClLLVMBitCodes.h"
-
-using namespace llvm;
-
-namespace naclmungetest {
-
-// Note: alignment stored as 0 or log2(Alignment)+1.
-uint64_t getEncAlignPower(unsigned Power) {
-  return Power + 1;
-}
-uint64_t getEncAlignZero() { return 0; }
-
-/// Test how we report a call arg that refers to nonexistent call argument
-TEST(NaClParseInstsTest, NonexistantCallArg) {
-  const uint64_t BitcodeRecords[] = {
-    1, naclbitc::BLK_CODE_ENTER, naclbitc::MODULE_BLOCK_ID, 2, Terminator,
-    1, naclbitc::BLK_CODE_ENTER, naclbitc::TYPE_BLOCK_ID_NEW, 2, Terminator,
-    3, naclbitc::TYPE_CODE_NUMENTRY, 3, Terminator,
-    3, naclbitc::TYPE_CODE_INTEGER, 32, Terminator,
-    3, naclbitc::TYPE_CODE_VOID, Terminator,
-    3, naclbitc::TYPE_CODE_FUNCTION, 0, 1, 0, 0, Terminator,
-    0, naclbitc::BLK_CODE_EXIT, Terminator,
-    3, naclbitc::MODULE_CODE_FUNCTION, 2, 0, 1, 0, Terminator,
-    3, naclbitc::MODULE_CODE_FUNCTION, 2, 0, 0, 0, Terminator,
-    1, naclbitc::BLK_CODE_ENTER, naclbitc::FUNCTION_BLOCK_ID, 2, Terminator,
-    3, naclbitc::FUNC_CODE_DECLAREBLOCKS, 1, Terminator,
-    // Note: 100 is a bad value index in next line.
-    3, naclbitc::FUNC_CODE_INST_CALL, 0, 4, 2, 100, Terminator,
-    3, naclbitc::FUNC_CODE_INST_RET, Terminator,
-    0, naclbitc::BLK_CODE_EXIT, Terminator,
-    0, naclbitc::BLK_CODE_EXIT, Terminator
-  };
-
-  // Show text of base input.
-  NaClObjDumpMunger DumpMunger(ARRAY_TERM(BitcodeRecords));
-  EXPECT_FALSE(DumpMunger.runTest());
-  EXPECT_EQ(
-      "       0:0|<65532, 80, 69, 88, 69, 1, 0,|Magic Number: 'PEXE' (80, 69, "
-      "88, 69)\n"
-      "          | 8, 0, 17, 0, 4, 0, 2, 0, 0, |PNaCl Version: 2\n"
-      "          | 0>                          |\n"
-      "      16:0|1: <65535, 8, 2>             |module {  // BlockID = 8\n"
-      "      24:0|  1: <65535, 17, 2>          |  types {  // BlockID = 17\n"
-      "      32:0|    3: <1, 3>                |    count 3;\n"
-      "      34:4|    3: <7, 32>               |    @t0 = i32;\n"
-      "      37:6|    3: <2>                   |    @t1 = void;\n"
-      "      39:4|    3: <21, 0, 1, 0, 0>      |    @t2 = void (i32, i32);\n"
-      "      44:2|  0: <65534>                 |  }\n"
-      "      48:0|  3: <8, 2, 0, 1, 0>         |  declare external void @f0(i32"
-      ", i32);\n"
-      "      52:6|  3: <8, 2, 0, 0, 0>         |  define external void @f1(i32,"
-      " i32);\n"
-      "      57:4|  1: <65535, 12, 2>          |  function void @f1(i32 %p0, "
-      "i32 %p1) {\n"
-      "          |                             |                    // BlockID "
-      "= 12\n"
-      "      64:0|    3: <1, 1>                |    blocks 1;\n"
-      "          |                             |  %b0:\n"
-      "      66:4|    3: <34, 0, 4, 2, 100>    |    call void @f0(i32 %p0, i32"
-      " @f0);\n"
-      "Error(66:4): Invalid relative value id: 100 (Must be <= 4)\n"
-      "      72:6|    3: <10>                  |    ret void;\n"
-      "      74:4|  0: <65534>                 |  }\n"
-      "      76:0|0: <65534>                   |}\n",
-      DumpMunger.getTestResults());
-
-  NaClParseBitcodeMunger Munger(ARRAY_TERM(BitcodeRecords));
-  EXPECT_FALSE(Munger.runTest(true));
-  EXPECT_EQ(
-      "Error(72:6): Invalid call argument: Index 1\n"
-      "Error: Invalid value in record\n",
-      Munger.getTestResults());
-}
-
-/// Test how we recognize alignments in alloca instructions.
-TEST(NaClParseInstsTests, BadAllocaAlignment) {
-  const uint64_t BitcodeRecords[] = {
-    1, naclbitc::BLK_CODE_ENTER, naclbitc::MODULE_BLOCK_ID, 2, Terminator,
-    1, naclbitc::BLK_CODE_ENTER, naclbitc::TYPE_BLOCK_ID_NEW, 2, Terminator,
-    3, naclbitc::TYPE_CODE_NUMENTRY, 4, Terminator,
-    3, naclbitc::TYPE_CODE_INTEGER, 32, Terminator,
-    3, naclbitc::TYPE_CODE_VOID, Terminator,
-    3, naclbitc::TYPE_CODE_FUNCTION, 0, 1, 0, Terminator,
-    3, naclbitc::TYPE_CODE_INTEGER, 8, Terminator,
-    0, naclbitc::BLK_CODE_EXIT, Terminator,
-    3, naclbitc::MODULE_CODE_FUNCTION, 2, 0, 0, 0, Terminator,
-    1, naclbitc::BLK_CODE_ENTER, naclbitc::FUNCTION_BLOCK_ID, 2, Terminator,
-    3, naclbitc::FUNC_CODE_DECLAREBLOCKS, 1, Terminator,
-    3, naclbitc::FUNC_CODE_INST_ALLOCA, 1, getEncAlignPower(0), Terminator,
-    3, naclbitc::FUNC_CODE_INST_RET, Terminator,
-    0, naclbitc::BLK_CODE_EXIT, Terminator,
-    0, naclbitc::BLK_CODE_EXIT, Terminator
-  };
-
-  const uint64_t ReplaceIndex = 11; // index for FUNC_CODE_INST_ALLOCA
-
-  // Show text when alignment is 1.
-  NaClObjDumpMunger DumpMunger(ARRAY_TERM(BitcodeRecords));
-  EXPECT_TRUE(DumpMunger.runTest());
-  EXPECT_EQ(
-      "       0:0|<65532, 80, 69, 88, 69, 1, 0,|Magic Number: 'PEXE' (80, 69, "
-      "88, 69)\n"
-      "          | 8, 0, 17, 0, 4, 0, 2, 0, 0, |PNaCl Version: 2\n"
-      "          | 0>                          |\n"
-      "      16:0|1: <65535, 8, 2>             |module {  // BlockID = 8\n"
-      "      24:0|  1: <65535, 17, 2>          |  types {  // BlockID = 17\n"
-      "      32:0|    3: <1, 4>                |    count 4;\n"
-      "      34:4|    3: <7, 32>               |    @t0 = i32;\n"
-      "      37:6|    3: <2>                   |    @t1 = void;\n"
-      "      39:4|    3: <21, 0, 1, 0>         |    @t2 = void (i32);\n"
-      "      43:4|    3: <7, 8>                |    @t3 = i8;\n"
-      "      46:0|  0: <65534>                 |  }\n"
-      "      48:0|  3: <8, 2, 0, 0, 0>         |  define external void @f0(i32"
-      ");\n"
-      "      52:6|  1: <65535, 12, 2>          |  function void @f0(i32 %p0) {"
-      "  \n"
-      "          |                             |                   // BlockID "
-      "= 12\n"
-      "      60:0|    3: <1, 1>                |    blocks 1;\n"
-      "          |                             |  %b0:\n"
-      "      62:4|    3: <19, 1, 1>            |    %v0 = alloca i8, i32 %p0, "
-      "align 1;\n"
-      "      65:6|    3: <10>                  |    ret void;\n"
-      "      67:4|  0: <65534>                 |  }\n"
-      "      68:0|0: <65534>                   |}\n",
-      DumpMunger.getTestResults());
-  NaClParseBitcodeMunger Munger(ARRAY_TERM(BitcodeRecords));
-  EXPECT_TRUE(Munger.runTest(true));
-
-  // Show what happens when changing alignment to 0.
-  const uint64_t Align0[] = {
-    ReplaceIndex, NaClMungedBitcode::Replace,
-    3, naclbitc::FUNC_CODE_INST_ALLOCA, 1, getEncAlignZero(), Terminator,
-  };
-  EXPECT_TRUE(Munger.runTest(ARRAY(Align0), true));
-  EXPECT_TRUE(DumpMunger.runTestForAssembly(ARRAY(Align0)));
-  EXPECT_EQ(
-      "    %v0 = alloca i8, i32 %p0, align 0;\n",
-      DumpMunger.getLinesWithSubstring("alloca"));
-
-  // Show what happens when changing alignment to 2**30.
-  const uint64_t Align30[] = {
-    ReplaceIndex, NaClMungedBitcode::Replace,
-    3, naclbitc::FUNC_CODE_INST_ALLOCA, 1, getEncAlignPower(30), Terminator,
-  };
-  EXPECT_FALSE(Munger.runTest(ARRAY(Align30), true));
-  EXPECT_EQ(
-      "Error(65:6): Alignment can't be greater than 2**29. Found: 2**30\n"
-      "Error: Invalid value in record\n",
-      Munger.getTestResults());
-  EXPECT_FALSE(DumpMunger.runTestForAssembly(ARRAY(Align30)));
-  EXPECT_EQ(
-      "    %v0 = alloca i8, i32 %p0, align 0;\n",
-      DumpMunger.getLinesWithSubstring("alloca"));
-  EXPECT_EQ(
-      "Error(62:4): Alignment can't be greater than 2**29. Found: 2**30\n",
-      DumpMunger.getLinesWithSubstring("Error"));
-
-  // Show what happens when changing alignment to 2**29.
-  const uint64_t Align29[] = {
-    ReplaceIndex, NaClMungedBitcode::Replace,
-    3, naclbitc::FUNC_CODE_INST_ALLOCA, 1, getEncAlignPower(29), Terminator,
-  };
-  EXPECT_TRUE(Munger.runTest(ARRAY(Align29), true));
-  EXPECT_EQ(
-      "Successful parse!\n",
-      Munger.getTestResults());
-  EXPECT_TRUE(DumpMunger.runTestForAssembly(ARRAY(Align29)));
-  EXPECT_EQ(
-      "    %v0 = alloca i8, i32 %p0, align 536870912;\n",
-      DumpMunger.getLinesWithSubstring("alloca"));
-}
-
-// Test how we recognize alignments in load instructions.
-TEST(NaClParseInstsTests, BadLoadAlignment) {
-  const uint64_t BitcodeRecords[] = {
-    1, naclbitc::BLK_CODE_ENTER, naclbitc::MODULE_BLOCK_ID, 2, Terminator,
-    1, naclbitc::BLK_CODE_ENTER, naclbitc::TYPE_BLOCK_ID_NEW, 2, Terminator,
-    3, naclbitc::TYPE_CODE_NUMENTRY, 2, Terminator,
-    3, naclbitc::TYPE_CODE_INTEGER, 32, Terminator,
-    3, naclbitc::TYPE_CODE_FUNCTION, 0, 0, 0, Terminator,
-    0, naclbitc::BLK_CODE_EXIT, Terminator,
-    3, naclbitc::MODULE_CODE_FUNCTION, 1, 0, 0, 0, Terminator,
-    1, naclbitc::BLK_CODE_ENTER, naclbitc::FUNCTION_BLOCK_ID, 2, Terminator,
-    3, naclbitc::FUNC_CODE_DECLAREBLOCKS, 1, Terminator,
-    3, naclbitc::FUNC_CODE_INST_LOAD, 1, getEncAlignPower(0), 0, Terminator,
-    3, naclbitc::FUNC_CODE_INST_RET,  1, Terminator,
-    0, naclbitc::BLK_CODE_EXIT, Terminator,
-    0, naclbitc::BLK_CODE_EXIT, Terminator
-  };
-
-  const uint64_t ReplaceIndex = 9; // index for FUNC_CODE_INST_LOAD
-
-  // Show text when alignment is 1.
-  NaClObjDumpMunger DumpMunger(ARRAY_TERM(BitcodeRecords));
-  EXPECT_TRUE(DumpMunger.runTest());
-  EXPECT_EQ(
-      "       0:0|<65532, 80, 69, 88, 69, 1, 0,|Magic Number: 'PEXE' (80, 69, "
-      "88, 69)\n"
-      "          | 8, 0, 17, 0, 4, 0, 2, 0, 0, |PNaCl Version: 2\n"
-      "          | 0>                          |\n"
-      "      16:0|1: <65535, 8, 2>             |module {  // BlockID = 8\n"
-      "      24:0|  1: <65535, 17, 2>          |  types {  // BlockID = 17\n"
-      "      32:0|    3: <1, 2>                |    count 2;\n"
-      "      34:4|    3: <7, 32>               |    @t0 = i32;\n"
-      "      37:6|    3: <21, 0, 0, 0>         |    @t1 = i32 (i32);\n"
-      "      41:6|  0: <65534>                 |  }\n"
-      "      44:0|  3: <8, 1, 0, 0, 0>         |  define external i32 @f0(i32"
-      ");\n"
-      "      48:6|  1: <65535, 12, 2>          |  function i32 @f0(i32 %p0) {"
-      "  \n"
-      "          |                             |                   // BlockID "
-      "= 12\n"
-      "      56:0|    3: <1, 1>                |    blocks 1;\n"
-      "          |                             |  %b0:\n"
-      "      58:4|    3: <20, 1, 1, 0>         |    %v0 = load i32* %p0, "
-      "align 1;\n"
-      "      62:4|    3: <10, 1>               |    ret i32 %v0;\n"
-      "      65:0|  0: <65534>                 |  }\n"
-      "      68:0|0: <65534>                   |}\n",
-      DumpMunger.getTestResults());
-  NaClParseBitcodeMunger Munger(ARRAY_TERM(BitcodeRecords));
-  EXPECT_TRUE(Munger.runTest(true));
-
-  // Show what happens when changing alignment to 0.
-  const uint64_t Align0[] = {
-    ReplaceIndex, NaClMungedBitcode::Replace,
-    3, naclbitc::FUNC_CODE_INST_LOAD, 1, getEncAlignZero(), 0, Terminator,
-  };
-  // Note: Correct alignment is not checked by Munger (i.e. the PNaCl
-  // bitcode reader). It is checked later by the PNaCl ABI checker in
-  // pnacl-llc. On the other hand, the DumpMunger checks alignment for
-  // loads while parsing.
-  EXPECT_TRUE(Munger.runTest(ARRAY(Align0), true));
-  EXPECT_FALSE(DumpMunger.runTestForAssembly(ARRAY(Align0)));
-  EXPECT_EQ(
-      "    %v0 = load i32* %p0, align 0;\n"
-      "Error(58:4): load: Illegal alignment for i32. Expects: 1\n",
-      DumpMunger.getLinesWithSubstring("load"));
-
-  // Show what happens when changing alignment to 4.
-  const uint64_t Align4[] = {
-    ReplaceIndex, NaClMungedBitcode::Replace,
-    3, naclbitc::FUNC_CODE_INST_LOAD, 1, getEncAlignPower(2), 0, Terminator,
-  };
-  EXPECT_TRUE(Munger.runTest(ARRAY(Align4), true));
-  EXPECT_FALSE(DumpMunger.runTestForAssembly(ARRAY(Align4)));
-  EXPECT_EQ(
-      "    %v0 = load i32* %p0, align 4;\n"
-      "Error(58:4): load: Illegal alignment for i32. Expects: 1\n",
-      DumpMunger.getLinesWithSubstring("load"));
-
-  // Show what happens when changing alignment to 2**29.
-  const uint64_t Align29[] = {
-    ReplaceIndex, NaClMungedBitcode::Replace,
-    3, naclbitc::FUNC_CODE_INST_LOAD, 1, getEncAlignPower(29), 0, Terminator,
-  };
-  EXPECT_TRUE(Munger.runTest(ARRAY(Align29), true));
-  EXPECT_FALSE(DumpMunger.runTestForAssembly(ARRAY(Align29)));
-  EXPECT_EQ(
-      "    %v0 = load i32* %p0, align 536870912;\n"
-      "Error(58:4): load: Illegal alignment for i32. Expects: 1\n",
-      DumpMunger.getLinesWithSubstring("load"));
-
-  // Show what happens when changing alignment to 2**30.
-  const uint64_t Align30[] = {
-    ReplaceIndex, NaClMungedBitcode::Replace,
-    3, naclbitc::FUNC_CODE_INST_LOAD, 1, getEncAlignPower(30), 0, Terminator,
-  };
-  EXPECT_FALSE(Munger.runTest(ARRAY(Align30), true));
-  EXPECT_EQ(
-      "Error(62:4): Alignment can't be greater than 2**29. Found: 2**30\n"
-      "Error: Invalid value in record\n",
-      Munger.getTestResults());
-  EXPECT_FALSE(DumpMunger.runTestForAssembly(ARRAY(Align30)));
-  EXPECT_EQ(
-      "    %v0 = load i32* %p0, align 0;\n"
-      "Error(58:4): load: Illegal alignment for i32. Expects: 1\n",
-      DumpMunger.getLinesWithSubstring("load"));
-}
-
-// Test how we recognize alignments in store instructions.
-TEST(NaClParseInstsTests, BadStoreAlignment) {
-  const uint64_t BitcodeRecords[] = {
-    1, naclbitc::BLK_CODE_ENTER, naclbitc::MODULE_BLOCK_ID, 2, Terminator,
-    1, naclbitc::BLK_CODE_ENTER, naclbitc::TYPE_BLOCK_ID_NEW, 2, Terminator,
-    3, naclbitc::TYPE_CODE_NUMENTRY, 3, Terminator,
-    3, naclbitc::TYPE_CODE_FLOAT, Terminator,
-    3, naclbitc::TYPE_CODE_INTEGER, 32, Terminator,
-    3, naclbitc::TYPE_CODE_FUNCTION, 0, 0, 1, 0, Terminator,
-    0, naclbitc::BLK_CODE_EXIT, Terminator,
-    3, naclbitc::MODULE_CODE_FUNCTION, 2, 0, 0, 0, Terminator,
-    1, naclbitc::BLK_CODE_ENTER, naclbitc::FUNCTION_BLOCK_ID, 2, Terminator,
-    3, naclbitc::FUNC_CODE_DECLAREBLOCKS, 1, Terminator,
-    3, naclbitc::FUNC_CODE_INST_STORE, 2, 1, getEncAlignPower(0), Terminator,
-    3, naclbitc::FUNC_CODE_INST_RET, 1, Terminator,
-    0, naclbitc::BLK_CODE_EXIT, Terminator,
-    0, naclbitc::BLK_CODE_EXIT, Terminator
-  };
-
-  const uint64_t ReplaceIndex = 10; // index for FUNC_CODE_INST_STORE
-
-  // Show text when alignment is 1.
-  NaClObjDumpMunger DumpMunger(ARRAY_TERM(BitcodeRecords));
-  EXPECT_TRUE(DumpMunger.runTest());
-  EXPECT_EQ(
-      "       0:0|<65532, 80, 69, 88, 69, 1, 0,|Magic Number: 'PEXE' (80, 69, "
-      "88, 69)\n"
-      "          | 8, 0, 17, 0, 4, 0, 2, 0, 0, |PNaCl Version: 2\n"
-      "          | 0>                          |\n"
-      "      16:0|1: <65535, 8, 2>             |module {  // BlockID = 8\n"
-      "      24:0|  1: <65535, 17, 2>          |  types {  // BlockID = 17\n"
-      "      32:0|    3: <1, 3>                |    count 3;\n"
-      "      34:4|    3: <3>                   |    @t0 = float;\n"
-      "      36:2|    3: <7, 32>               |    @t1 = i32;\n"
-      "      39:4|    3: <21, 0, 0, 1, 0>      |    @t2 = float (i32, float);\n"
-      "      44:2|  0: <65534>                 |  }\n"
-      "      48:0|  3: <8, 2, 0, 0, 0>         |  define external \n"
-      "          |                             |      float @f0(i32, float);\n"
-      "      52:6|  1: <65535, 12, 2>          |  function \n"
-      "          |                             |      float @f0(i32 %p0, float "
-      "%p1) {  \n"
-      "          |                             |                   // BlockID "
-      "= 12\n"
-      "      60:0|    3: <1, 1>                |    blocks 1;\n"
-      "          |                             |  %b0:\n"
-      "      62:4|    3: <24, 2, 1, 1>         |    store float %p1, float* "
-      "%p0, \n"
-      "          |                             |        align 1;\n"
-      "      66:4|    3: <10, 1>               |    ret float %p1;\n"
-      "      69:0|  0: <65534>                 |  }\n"
-      "      72:0|0: <65534>                   |}\n",
-      DumpMunger.getTestResults());
-  NaClParseBitcodeMunger Munger(ARRAY_TERM(BitcodeRecords));
-  EXPECT_TRUE(Munger.runTest(true));
-
-  // Show what happens when changing alignment to 0.
-  const uint64_t Align0[] = {
-    ReplaceIndex, NaClMungedBitcode::Replace,
-    3, naclbitc::FUNC_CODE_INST_STORE, 2, 1, getEncAlignZero(), Terminator,
-  };
-  // Note: Correct alignment is not checked by Munger (i.e. the PNaCl
-  // bitcode reader). It is checked later by the PNaCl ABI checker in
-  // pnacl-llc. On the other hand, the DumpMunger checks alignment for
-  // stores while parsing.
-  EXPECT_TRUE(Munger.runTest(ARRAY(Align0), true));
-  EXPECT_FALSE(DumpMunger.runTestForAssembly(ARRAY(Align0)));
-  EXPECT_EQ(
-      "    store float %p1, float* %p0, align 0;\n"
-      "Error(62:4): store: Illegal alignment for float. Expects: 1 or 4\n",
-      DumpMunger.getLinesWithSubstring("store"));
-
-  // Show what happens when changing alignment to 4.
-  const uint64_t Align4[] = {
-    ReplaceIndex, NaClMungedBitcode::Replace,
-    3, naclbitc::FUNC_CODE_INST_STORE, 2, 1, getEncAlignPower(2), Terminator,
-  };
-  EXPECT_TRUE(Munger.runTest(ARRAY(Align4), true));
-  EXPECT_TRUE(DumpMunger.runTestForAssembly(ARRAY(Align4)));
-
-  // Show what happens when changing alignment to 8.
-  const uint64_t Align8[] = {
-    ReplaceIndex, NaClMungedBitcode::Replace,
-    3, naclbitc::FUNC_CODE_INST_STORE, 2, 1, getEncAlignPower(3), Terminator,
-  };
-  EXPECT_TRUE(Munger.runTest(ARRAY(Align8), true));
-  EXPECT_FALSE(DumpMunger.runTestForAssembly(ARRAY(Align8)));
-  EXPECT_EQ(
-      "    store float %p1, float* %p0, align 8;\n"
-      "Error(62:4): store: Illegal alignment for float. Expects: 1 or 4\n",
-      DumpMunger.getLinesWithSubstring("store"));
-
-  // Show what happens when changing alignment to 2**29.
-  const uint64_t Align29[] = {
-    ReplaceIndex, NaClMungedBitcode::Replace,
-    3, naclbitc::FUNC_CODE_INST_STORE, 2, 1, getEncAlignPower(29), Terminator,
-  };
-  EXPECT_TRUE(Munger.runTest(ARRAY(Align29), true));
-  EXPECT_FALSE(DumpMunger.runTestForAssembly(ARRAY(Align29)));
-  EXPECT_EQ(
-      "    store float %p1, float* %p0, align 536870912;\n"
-      "Error(62:4): store: Illegal alignment for float. Expects: 1 or 4\n",
-      DumpMunger.getLinesWithSubstring("store"));
-
-  // Show what happens when changing alignment to 2**30.
-  const uint64_t Align30[] = {
-    ReplaceIndex, NaClMungedBitcode::Replace,
-    3, naclbitc::FUNC_CODE_INST_STORE, 2, 1, getEncAlignPower(30), Terminator,
-  };
-  EXPECT_FALSE(Munger.runTest(ARRAY(Align30), true));
-  EXPECT_EQ(
-      "Error(66:4): Alignment can't be greater than 2**29. Found: 2**30\n"
-      "Error: Invalid value in record\n",
-      Munger.getTestResults());
-  EXPECT_FALSE(DumpMunger.runTestForAssembly(ARRAY(Align30)));
-  EXPECT_EQ(
-      "    store float %p1, float* %p0, align 0;\n"
-      "Error(62:4): store: Illegal alignment for float. Expects: 1 or 4\n",
-      DumpMunger.getLinesWithSubstring("store"));
-}
-
-} // end of namespace naclmungetest
diff --git a/unittests/Bitcode/NaClParseTypesTest.cpp b/unittests/Bitcode/NaClParseTypesTest.cpp
deleted file mode 100644
index 605f7532764..00000000000
--- a/unittests/Bitcode/NaClParseTypesTest.cpp
+++ /dev/null
@@ -1,73 +0,0 @@
-//===- llvm/unittest/Bitcode/NaClParseTypesTest.cpp ---------------------===//
-//     Tests parser for PNaCl bitcode.
-//
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
-//
-//===----------------------------------------------------------------------===//
-
-// Tests record errors in the types block when parsing PNaCl bitcode.
-
-// TODO(kschimpf) Add more tests.
-
-#include "NaClMungeTest.h"
-
-using namespace llvm;
-
-namespace naclmungetest {
-
-TEST(NaClParseTypesTest, BadTypeReferences) {
-  const uint64_t BitcodeRecords[] = {
-    1, 65535, 8, 2, Terminator,
-    1, 65535, 17, 2, Terminator,
-    3, 1, 2, Terminator,
-    3, 7, 32, Terminator,
-    3, 3, Terminator,
-    0, 65534, Terminator,
-    0, 65534, Terminator
-  };
-
-  const uint64_t ReplaceIndex = 4;
-
-  // Show text of base input.
-  NaClObjDumpMunger BaseMunger(ARRAY_TERM(BitcodeRecords));
-  EXPECT_TRUE(BaseMunger.runTest());
-  EXPECT_EQ(
-      "       0:0|<65532, 80, 69, 88, 69, 1, 0,|Magic Number: 'PEXE' (80, 69, "
-      "88, 69)\n"
-      "          | 8, 0, 17, 0, 4, 0, 2, 0, 0, |PNaCl Version: 2\n"
-      "          | 0>                          |\n"
-      "      16:0|1: <65535, 8, 2>             |module {  // BlockID = 8\n"
-      "      24:0|  1: <65535, 17, 2>          |  types {  // BlockID = 17\n"
-      "      32:0|    3: <1, 2>                |    count 2;\n"
-      "      34:4|    3: <7, 32>               |    @t0 = i32;\n"
-      "      37:6|    3: <3>                   |    @t1 = float;\n"
-      "      39:4|  0: <65534>                 |  }\n"
-      "      40:0|0: <65534>                   |}\n",
-      BaseMunger.getTestResults());
-
-  // Show that we successfully parse the base input.
-  NaClParseBitcodeMunger Munger(ARRAY_TERM(BitcodeRecords));
-  EXPECT_TRUE(Munger.runTest(true));
-  EXPECT_EQ(
-      "Successful parse!\n",
-      Munger.getTestResults());
-
-  // Show what happens when misdefining: @t1 = float"
-  const uint64_t AddSelfReference[] = {
-    ReplaceIndex, NaClMungedBitcode::Replace, 3, 3, 1, Terminator
-  };
-  EXPECT_FALSE(Munger.runTest(ARRAY(AddSelfReference), false));
-  EXPECT_EQ(
-      "Error: Record doesn't have expected size or structure\n",
-      Munger.getTestResults());
-  EXPECT_FALSE(Munger.runTest(ARRAY(AddSelfReference), true));
-  EXPECT_EQ(
-      "Error(40:2): Invalid TYPE_CODE_FLOAT record\n"
-      "Error: Record doesn't have expected size or structure\n",
-      Munger.getTestResults());
-}
-
-} // end of namespace naclmungetest
diff --git a/unittests/Bitcode/NaClTextFormatterTest.cpp b/unittests/Bitcode/NaClTextFormatterTest.cpp
deleted file mode 100644
index 656ba8e283b..00000000000
--- a/unittests/Bitcode/NaClTextFormatterTest.cpp
+++ /dev/null
@@ -1,1410 +0,0 @@
-//===- llvm/unittest/Bitcode/NaClTextFormatterTest.cpp --------------------===//
-//     Tests the text formatter for PNaCl bitcode.
-//
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
-//
-//===----------------------------------------------------------------------===//
-
-// Tests if the text formatter for PNaCl bitcode works as expected.
-
-#include "llvm/ADT/STLExtras.h"
-#include "llvm/Bitcode/NaCl/NaClObjDumpStream.h"
-
-#include "gtest/gtest.h"
-
-#include <iostream>
-
-using namespace llvm;
-using namespace llvm::naclbitc;
-
-namespace {
-
-/// Simple test harness for testing a text formatter. This class takes
-/// an array of tokens, parses it, and then uses the text formatter to
-/// format it. To test the features of the text formatter, the parser
-/// detects function calls, and inserts appropriate open/close
-/// parenthesis directives, as well as clustering directives.
-///
-/// For clustering, we consider each argument, as well as the entire
-/// function. In case the entire function can't be printed, we add two
-/// additional clusters as backup strategies:
-///
-/// 1) Cluster the called function with it's first argument.
-/// 2) Cluster the called function with the open parenthesis.
-///
-/// These rules can be formalized as the following cases, where:
-///    '<' denotes a StartCluster.
-///    '>' denotes a FinishCluster.
-///    '[' represents a regular expression open parenthesis.
-///    ']' represents a regular expression close parenthesis.
-///    '*' denotes regular expression repeat operation.
-///
-/// The cases to consider are:
-///
-/// case 1: <<<f(>)>>
-/// case 2: <<<f(><x>)>>
-/// case 3: <<<f(><x,>>[<y,>]*<z>)>
-///
-/// Note: In cases 1 and 2, there is an unnecessary pair of clustering
-/// directives.  This is intentional. It has been added so that the
-/// parser need not build an AST before formatting. Also note that
-/// case 3 covers all function calls with 2 or more arguments.
-class FormatterTester {
-private:
-  // Defines states of the token parser, as it looks for function calls.
-  // These states are used to determine where (and when) clustering
-  // directives should be added to the Tokens() stream. If no
-  // transition applies for a state, the default transition is applied,
-  // which is to add the next token to the Tokens() stream.
-  //
-  // Note: We use '|' to denote the current position of the token parser.
-  //
-  // In all states, the following transition is possible (pushing the
-  // current state onto the parse stack), and is applied after state
-  // specific transitions (below):
-  //
-  //    |<<<f(>)>>                 => StartingFcn:    <<<|f(>)>>
-  //    |<<<f(><x>)>>              => StartingFcn:    <<<|f(><x>)>>
-  //    |<<<f(><x,>>[<y,>]*<z>)>   => StartingFcn:    <<<|f(><x,>>[<y,>]*<z>)>
-  //
-  // Note: We use the notation == to state that two expressions are equivalent.
-  // In particular,
-  //
-  //          [x|]* == [|x]*
-  //
-  // since being at the end of a repeated instruction also means that
-  // you are at the beginning of the next (unrolled) repetition.
-  enum FormatterState {
-    LookingForFunction,            // Start state
-
-    StartingFcn,
-    // <<<f(|>)>>                  => BeforeFirstArg: <<<f(>|)>>
-    // <<<f(|><x>)>>               => BeforeFirstArg: <<<f(>|<x>(>>
-    // <<<f(|><x,>>[<y,>]*<z>)>    => BeforeFirstArg: <<<f(>|<x,>>[<y,>]*<z>)>
-
-    BeforeFirstArg,
-    // <<<f(>|)>>                   => EndFcn2:       <<<f(>|)>>
-    // <<<f(>|<x>)>>                => InFirstArg:    <<<f(><|x>)>>
-    // <<<f(>|<x,>>[<y,>]*<z>)>     => InFirstArg     <<<f(><|x,>>[<y,>]*<z>)>
-
-    InFirstArg,
-    // <<<f(><x|>)>>                => EndFcn2:       <<<f(><x>|)>>
-    // <<<f(><x,|>>[<y,>]*<z>)>     => BetweenArgs:   <<<f(><x,>>[|<y,>]*<z>)>
-
-    InOtherArg,
-    // <<<f(><x,>>[<y,|>]*<z>)>     => BetweenArgs:   <<<f(><x,>>[<y,>|]*<z>)>
-    //                                             == <<<f(><x,>>[|<y,>]*<z>)>
-    //                              => BetweenArgs:   <<<f(><x,>>[<y,>]*|<z>)>
-    // <<<f(><x,>>[<y,>]*<z|>)>     => EndFcn1:       <<<f(><x,>>[<y,>]*<z>|)>
-
-    BetweenArgs,
-    // <<<f(><x,>>[|<y,>]*<z>)>     => InOtherArg:    <<<f(><x,>>[<|y,>]*<z>)>
-    // <<<f(><x,>>[<y,>]*|<z>)>     => InOtherArg:    <<<f(><x,>>[<y,>]*<|z>)>
-
-    EndFcn2,
-    // <<<f(>)|>>                   => EndFcn1:       <<<f(>)>|>
-    // <<<f(><x>)|>>                => EndFcn1:       <<<f(><x>)>|>
-
-    EndFcn1
-    // <<<f(>)>|>                   => XXX:           <<<f(>)>>|
-    // <<<f(><x>)>|>                => XXX:           <<<f(><x>)>>|
-    // <<<f(><x,>>[<y,>]*<z>)|>     => XXX:           <<<f(><x,>>[<y,>]*<z>)>|
-    //
-    // where XXX is the state popped from the parse stack.
-  };
-
-public:
-  FormatterTester(unsigned LineWidth)
-      : AddOpenCloseDirectives(false),
-        AddClusterDirectives(false),
-        Buffer(),
-        BufStream(Buffer),
-        Formatter(BufStream, LineWidth, "  "),
-        CommaText(","),
-        SpaceText(" "),
-        OpenParenText("("),
-        CloseParenText(")"),
-        NewlineText("\n"),
-        Comma(&Formatter, CommaText),
-        Space(&Formatter, SpaceText),
-        OpenParen(&Formatter, OpenParenText),
-        CloseParen(&Formatter, CloseParenText),
-        StartCluster(&Formatter),
-        FinishCluster(&Formatter),
-        Tokenize(&Formatter),
-        Endline(&Formatter)
-  {
-    Reset();
-    Formatter.SetContinuationIndent(Formatter.GetIndent(2));
-  }
-
-  /// Runs a test using the given sequence of tokens. If
-  /// AddOpenCloseDirectives is true, then "(" and ")" tokens
-  /// will change the local indent using the corresponding directives.
-  /// If AddClusterDirectives is true, then the clustering rules for
-  /// function calls will be applied.
-  std::string Test(const char *Tokens[], size_t NumTokens,
-                   bool AddOpenCloseDirectives,
-                   bool AddClusterDirectives,
-                   unsigned Indent);
-
-private:
-  /// Reset the formatter for next test.
-  void Reset() {
-    BufStream.flush();
-    Buffer.clear();
-    State = LookingForFunction;
-    FunctionParseStack.clear();
-  }
-
-  /// Collect the sequence of tokens, starting at Index, that
-  /// correspond to spaces, and return the number of spaces found.
-  unsigned CollectSpaces(size_t &Index,
-                         const char *Tokens[],
-                         size_t NumTokens) const {
-    unsigned Count = 0;
-    for (; Index < NumTokens; ++Index) {
-      std::string Token(Tokens[Index]);
-      if (Token != SpaceText) return Count;
-      ++Count;
-    }
-    return Count;
-  }
-
-  /// Write out the given number of spaces using a space directive.
-  void WriteSpaces(unsigned Count) {
-    for (unsigned i = 0; i < Count; ++i) Formatter.Tokens() << Space;
-  }
-
-  /// Insert clustering directives, based on the current state of the
-  /// parser. CurToken is the current (non-whitespace) token being
-  /// processed by the parser. NextToken is the next (non-whitespace)
-  /// token being processed.  If BeforeCurrentToken, then the parser
-  /// is just before CurToken. Otherwise, it is just after NextToken.
-  ///
-  /// Note: When BeforeCurrentToken is false, it isn't necessarily
-  /// just before NextToken. This is because there may be space
-  /// (i.e. whitespace) tokens between CurToken and NextToken.
-  void InsertClusterDirectives(std::string &CurToken,
-                               std::string &NextToken,
-                               bool BeforeCurrentToken);
-
-  /// Write out the given token. Implicitly uses corresponding directives
-  /// if applicable.
-  void WriteToken(const std::string &Token);
-
-  // True if Open and Close directives should be used for "(" and ")" tokens.
-  bool AddOpenCloseDirectives;
-  // True if clustering directives (for functions) should be inserted.
-  bool AddClusterDirectives;
-  // The buffer the formatted text is written into.
-  std::string Buffer;
-  // The base stream of the text formatter, which dumps text into the buffer.
-  raw_string_ostream BufStream;
-  // The text formatter to use.
-  TextFormatter Formatter;
-  const std::string CommaText;
-  const std::string SpaceText;
-  const std::string OpenParenText;
-  const std::string CloseParenText;
-  const std::string NewlineText;
-  const TokenTextDirective Comma;
-  SpaceTextDirective Space;
-  OpenTextDirective OpenParen;
-  CloseTextDirective CloseParen;
-  StartClusteringDirective StartCluster;
-  FinishClusteringDirective FinishCluster;
-  TokenizeTextDirective Tokenize;
-  EndlineTextDirective Endline;
-  // The parse state of the function parser, used by the tester.
-  FormatterState State;
-  // The stack of parse states of the function parser. Used to handle
-  // nested functions.
-  std::vector<FormatterState> FunctionParseStack;
-};
-
-void FormatterTester::WriteToken(const std::string &Token) {
-  if (Token == CommaText) {
-    Formatter.Tokens() << Comma;
-  } else if (Token == SpaceText) {
-    Formatter.Tokens() << Space;
-  } else if (Token == OpenParenText) {
-    if (AddOpenCloseDirectives) {
-      Formatter.Tokens() << OpenParen;
-    } else {
-      Formatter.Tokens() << Token << Tokenize;
-    }
-  } else if (Token == CloseParenText) {
-    if (AddOpenCloseDirectives) {
-      Formatter.Tokens() << CloseParen;
-    } else {
-      Formatter.Tokens() << Token << Tokenize;
-    }
-  } else if (Token == NewlineText) {
-    Formatter.Tokens() << Endline;
-  } else {
-    Formatter.Tokens() << Token << Tokenize;
-  }
-}
-
-std::string FormatterTester::Test(const char *Tokens[],
-                                  size_t NumTokens,
-                                  bool NewAddOpenCloseDirectives,
-                                  bool NewAddClusterDirectives,
-                                  unsigned Indent) {
-  AddOpenCloseDirectives = NewAddOpenCloseDirectives;
-  AddClusterDirectives = NewAddClusterDirectives;
-  for (unsigned i = 0; i < Indent; ++i) Formatter.Inc();
-
-  size_t Index = 0;
-  unsigned SpaceCount = CollectSpaces(Index, Tokens, NumTokens);
-  WriteSpaces(SpaceCount);
-  SpaceCount = 0;
-
-  // NOTE: We would use ASSERT_LT(Index, NumTokens), but it gets
-  // a compile-time error if not inside the TEST macro.
-  EXPECT_LT(Index, NumTokens);
-  if (Index == NumTokens) return std::string("*W*R*O*N*G*");
-
-  // Generate token sequence defined by Tokens.
-  std::string CurToken(Tokens[Index++]);
-  while (Index < NumTokens) {
-    SpaceCount = CollectSpaces(Index, Tokens, NumTokens);
-    if (Index == NumTokens) {
-      WriteSpaces(SpaceCount);
-      SpaceCount = 0;
-      break;
-    }
-    std::string NextToken(Tokens[Index++]);
-    InsertClusterDirectives(CurToken, NextToken, true);
-    WriteToken(CurToken);
-    InsertClusterDirectives(CurToken, NextToken, false);
-    WriteSpaces(SpaceCount);
-    SpaceCount = 0;
-    CurToken = NextToken;
-  }
-
-  // When reached, all but last token (i.e. CurToken) has been written.
-  // Create dummy newline token, so that the last token can be written.
-  std::string NextToken(NewlineText);
-  InsertClusterDirectives(CurToken, NextToken, true);
-  WriteToken(CurToken);
-  InsertClusterDirectives(CurToken, NextToken, false);
-  WriteSpaces(SpaceCount);
-  Formatter.Tokens() << Endline;
-
-  EXPECT_TRUE(FunctionParseStack.empty())
-      << "Missing close parenthesis in example";
-
-  std::string Results = BufStream.str();
-  Reset();
-  return Results;
-}
-
-void FormatterTester::InsertClusterDirectives(std::string &CurToken,
-                                              std::string &NextToken,
-                                              bool BeforeCurToken) {
-  if (!AddClusterDirectives) return;
-  switch (State) {
-  case LookingForFunction:
-    break;
-  case StartingFcn:
-    if (!BeforeCurToken && CurToken == OpenParenText) {
-      // context: <<<f(|>)>>
-      // context: <<<f(|><x>)>>
-      // context: <<<f(|><x,>><y,> ... <z>)>
-      Formatter.Tokens() << FinishCluster;
-      State = BeforeFirstArg;
-    }
-    break;
-  case BeforeFirstArg:
-    EXPECT_TRUE(BeforeCurToken)
-        << "After open paren, but not before current token";
-    if (CurToken == CloseParenText) {
-      // <<<f(>|)>>                   => EndFcn2:       <<<f(>|)>>
-      State = EndFcn2;
-    } else {
-      // <<<f(>|<x>)>>                => InFirstArg:    <<<f(><|x>)>>
-      // <<<f(>|<x,>>[<y,>]*<z>)>     => InFirstArg     <<<f(><|x,>>[<y,>]*<z>)>
-      State = InFirstArg;
-      Formatter.Tokens() << StartCluster;
-    }
-    break;
-  case InFirstArg:
-    if (BeforeCurToken && CurToken == CloseParenText) {
-      // <<<f(><x|>)>>                => EndFcn2:       <<<f(><x>|)>>
-      Formatter.Tokens() << FinishCluster;
-      State = EndFcn2;
-    } else if (!BeforeCurToken && CurToken == CommaText) {
-      // <<<f(><x,|>>[<y,>]*<z>)>     => BetweenArgs:   <<<f(><x,>>[|<y,>]*<z>)>
-      Formatter.Tokens() << FinishCluster << FinishCluster;
-      State = BetweenArgs;
-    }
-    break;
-  case InOtherArg:
-    if(BeforeCurToken && CurToken == CloseParenText) {
-      // <<<f(><x,>>[<y,>]*<z|>)>     => EndFcn1:       <<<f(><x,>>[<y,>]*<z>|)>
-      Formatter.Tokens() << FinishCluster;
-      State = EndFcn1;
-    } else if (!BeforeCurToken && CurToken == CommaText) {
-      // <<<f(><x,>>[<y,|>]*<z>)>     => BetweenArgs:   <<<f(><x,>>[<y,>|]*<z>)>
-      //                              => BetweenArgs:   <<<f(><x,>>[<y,>]*|<z>)>
-      Formatter.Tokens() << FinishCluster;
-      State = BetweenArgs;
-    }
-    break;
-  case BetweenArgs:
-    // <<<f(><x,>>[|<y,>]*<z>)>     => InOtherArg:    <<<f(><x,>>[<|y,>]*<z>)>
-    // <<<f(><x,>>[<y,>]*|<z>)>     => InOtherArg:    <<<f(><x,>>[<y,>]*<|z>)>
-    EXPECT_TRUE(BeforeCurToken)
-        << "Expecting to be before next token after comma";
-    Formatter.Tokens() << StartCluster;
-    State = InOtherArg;
-    break;
-  case EndFcn2:
-    // <<<f(>)|>>                   => EndFcn1:       <<<f(>)>|>
-    // <<<f(><x>)|>>                => EndFcn1:       <<<f(><x>)>|>
-    EXPECT_TRUE(!BeforeCurToken && CurToken == CloseParenText)
-        << "Expecting to be after close paren";
-    Formatter.Tokens() << FinishCluster;
-    // Intentionally drop to the next case.
-  case EndFcn1:
-    // <<<f(>)>|>                   => XXX:           <<<f(>)>>|
-    // <<<f(><x>)>|>                => XXX:           <<<f(><x>)>>|
-    // <<<f(><x,>>[<y,>]*<z>)|>     => XXX:           <<<f(><x,>>[<y,>]*<z>)>|
-    EXPECT_TRUE(!BeforeCurToken && CurToken == CloseParenText)
-        << "Expecting to be after close paren";
-    Formatter.Tokens() << FinishCluster;
-    if (FunctionParseStack.empty()) {
-      EXPECT_TRUE(false)
-          << "No open paren for corresponding close paren";
-      State = LookingForFunction;
-    } else {
-      State = FunctionParseStack.back();
-      FunctionParseStack.pop_back();
-    }
-    break;
-  }
-
-  // Check if we are at the beginning of a new function.
-  if (BeforeCurToken && NextToken == OpenParenText) {
-    // context: <<<|f(>)>>
-    // context: <<<|f(><x>)>>
-    // context: <<<|f(><x,>><y,> ... <z>)>
-    Formatter.Tokens() << StartCluster << StartCluster << StartCluster;
-    FunctionParseStack.push_back(State);
-    State = StartingFcn;
-  }
-}
-
-std::string RunTest(const char *Tokens[],
-                    size_t NumTokens,
-                    unsigned LineWidth,
-                    bool AddOpenCloseDirectives,
-                    bool AddClusterDirectives,
-                    unsigned Indent = 0) {
-  FormatterTester Tester(LineWidth);
-  return Tester.Test(Tokens, NumTokens,
-                     AddOpenCloseDirectives,
-                     AddClusterDirectives,
-                     Indent);
-}
-
-// Test simple single function call.
-TEST(NaClTextFormatterTest, SimpleCall) {
-  static const char *Tokens[] = {
-    "foobar", "(", "Value1", ",", " ", "Value2", "," , " ", "Value3", ")"
-  };
-
-  // Print out simple call that can fit on one line.
-  EXPECT_EQ(
-      "foobar(Value1, Value2, Value3)\n",
-      RunTest(Tokens, array_lengthof(Tokens), 30, true, true));
-
-  EXPECT_EQ(
-      "foobar(Value1, Value2, Value3)\n",
-      RunTest(Tokens, array_lengthof(Tokens), 30, true, false));
-
-  EXPECT_EQ(
-      "foobar(Value1, Value2, Value3)\n",
-      RunTest(Tokens, array_lengthof(Tokens), 30, false, true));
-
-  EXPECT_EQ(
-      "foobar(Value1, Value2, Value3)\n",
-      RunTest(Tokens, array_lengthof(Tokens), 30, false, false));
-
-  // Test case where it is one character too long (i.e ")" causes wrapping).
-  EXPECT_EQ(
-      "foobar(Value1, Value2, Value3\n"
-      "       )\n",
-      RunTest(Tokens, array_lengthof(Tokens), 29, true, true));
-
-  EXPECT_EQ(
-      "foobar(Value1, Value2, Value3\n"
-      "       )\n",
-      RunTest(Tokens, array_lengthof(Tokens), 29, true, false));
-
-  EXPECT_EQ(
-      "foobar(Value1, Value2, Value3\n"
-      "    )\n",
-      RunTest(Tokens, array_lengthof(Tokens), 29, false, true));
-
-  EXPECT_EQ(
-      "foobar(Value1, Value2, Value3\n"
-      "    )\n",
-      RunTest(Tokens, array_lengthof(Tokens), 29, false, false));
-
-  // Test case where line length matches the beginning of "Value3".
-  // Note: Only 3 indents for parenthesis directive, because we
-  // stop indenting when there is only 20 columns left in the line
-  // (i.e. 23 - 20 == 3).
-  EXPECT_EQ(
-      "foobar(Value1, Value2, \n"
-      "   Value3)\n",
-      RunTest(Tokens, array_lengthof(Tokens), 23, true, true));
-
-  EXPECT_EQ(
-      "foobar(Value1, Value2, \n"
-      "   Value3)\n",
-      RunTest(Tokens, array_lengthof(Tokens), 23, true, false));
-
-  EXPECT_EQ(
-      "foobar(Value1, Value2, \n"
-      "   Value3)\n",
-      RunTest(Tokens, array_lengthof(Tokens), 23, false, true));
-
-  EXPECT_EQ(
-      "foobar(Value1, Value2, \n"
-      "   Value3)\n",
-      RunTest(Tokens, array_lengthof(Tokens), 23, false, false));
-
-  // Test case where line length matches the beginning of " Value3"
-  // (i.e. the last test, but move the space to the next line).
-  // Note: Only 2 indents for parenthesis directive, because we
-  // stop indenting when there is only 20 columns left in the line
-  // (i.e. 22 - 20 == 2).
-  EXPECT_EQ(
-      "foobar(Value1, Value2,\n"
-      "  Value3)\n",
-      RunTest(Tokens, array_lengthof(Tokens), 22, true, true));
-
-  EXPECT_EQ(
-      "foobar(Value1, Value2,\n"
-      "  Value3)\n",
-      RunTest(Tokens, array_lengthof(Tokens), 22, true, false));
-
-  EXPECT_EQ(
-      "foobar(Value1, Value2,\n"
-      "  Value3)\n",
-      RunTest(Tokens, array_lengthof(Tokens), 22, false, true));
-
-  EXPECT_EQ(
-      "foobar(Value1, Value2,\n"
-      "  Value3)\n",
-      RunTest(Tokens, array_lengthof(Tokens), 22, false, false));
-
-  // Test case where last comma causes line wrap.
-  EXPECT_EQ(
-      "foobar(Value1, \n"
-      " Value2, Value3)\n",
-      RunTest(Tokens, array_lengthof(Tokens), 21, true, true));
-
-  EXPECT_EQ(
-      "foobar(Value1, Value2\n"
-      " , Value3)\n",
-      RunTest(Tokens, array_lengthof(Tokens), 21, true, false));
-
-  EXPECT_EQ(
-      "foobar(Value1, \n"
-      " Value2, Value3)\n",
-      RunTest(Tokens, array_lengthof(Tokens), 21, false, true));
-
-  EXPECT_EQ(
-      "foobar(Value1, Value2\n"
-      " , Value3)\n",
-      RunTest(Tokens, array_lengthof(Tokens), 21, false, false));
-
-  // Test case where Value2 runs over the line width.
-  EXPECT_EQ(
-      "foobar(Value1, \n"
-      "Value2, Value3)\n",
-      RunTest(Tokens, array_lengthof(Tokens), 20, true, true));
-
-  EXPECT_EQ(
-      "foobar(Value1, \n"
-      "Value2, Value3)\n",
-      RunTest(Tokens, array_lengthof(Tokens), 20, true, false));
-
-  EXPECT_EQ(
-      "foobar(Value1, \n"
-      "Value2, Value3)\n",
-      RunTest(Tokens, array_lengthof(Tokens), 20, false, true));
-
-  EXPECT_EQ(
-      "foobar(Value1, \n"
-      "Value2, Value3)\n",
-      RunTest(Tokens, array_lengthof(Tokens), 20, false, false));
-
-  // Run test where first comma (after value 1) causes line wrap.
-  EXPECT_EQ(
-      "foobar(\n"
-      "Value1, \n"
-      "Value2, \n"
-      "Value3)\n",
-      RunTest(Tokens, array_lengthof(Tokens), 13, true, true));
-
-  EXPECT_EQ(
-      "foobar(Value1\n"
-      ", Value2, \n"
-      "Value3)\n",
-      RunTest(Tokens, array_lengthof(Tokens), 13, true, false));
-
-  EXPECT_EQ(
-      "foobar(\n"
-      "Value1, \n"
-      "Value2, \n"
-      "Value3)\n",
-      RunTest(Tokens, array_lengthof(Tokens), 13, false, true));
-
-  EXPECT_EQ(
-      "foobar(Value1\n"
-      ", Value2, \n"
-      "Value3)\n",
-      RunTest(Tokens, array_lengthof(Tokens), 13, false, false));
-
-  // Run test where only "foobar(" can fit on a line.
-  EXPECT_EQ(
-      "foobar(\n"
-      "Value1,\n"
-      "Value2,\n"
-      "Value3)\n",
-      RunTest(Tokens, array_lengthof(Tokens), 7, true, true));
-
-  EXPECT_EQ(
-      "foobar(\n"
-      "Value1,\n"
-      "Value2,\n"
-      "Value3)\n",
-      RunTest(Tokens, array_lengthof(Tokens), 7, true, false));
-
-  EXPECT_EQ(
-      "foobar(\n"
-      "Value1,\n"
-      "Value2,\n"
-      "Value3)\n",
-      RunTest(Tokens, array_lengthof(Tokens), 7, false, true));
-
-  EXPECT_EQ(
-      "foobar(\n"
-      "Value1,\n"
-      "Value2,\n"
-      "Value3)\n",
-      RunTest(Tokens, array_lengthof(Tokens), 7, false, false));
-
-  // Run case where most tokens don't fit on a line.
-  EXPECT_EQ(
-      "foobar\n"
-      "(\n"
-      "Value1\n"
-      ", \n"
-      "Value2\n"
-      ", \n"
-      "Value3\n"
-      ")\n",
-      RunTest(Tokens, array_lengthof(Tokens), 4, true, true));
-
-  EXPECT_EQ(
-      "foobar\n"
-      "(\n"
-      "Value1\n"
-      ", \n"
-      "Value2\n"
-      ", \n"
-      "Value3\n"
-      ")\n",
-      RunTest(Tokens, array_lengthof(Tokens), 4, true, false));
-
-  EXPECT_EQ(
-      "foobar\n"
-      "(\n"
-      "Value1\n"
-      ", \n"
-      "Value2\n"
-      ", \n"
-      "Value3\n"
-      ")\n",
-      RunTest(Tokens, array_lengthof(Tokens), 4, false, true));
-
-  EXPECT_EQ(
-      "foobar\n"
-      "(\n"
-      "Value1\n"
-      ", \n"
-      "Value2\n"
-      ", \n"
-      "Value3\n"
-      ")\n",
-      RunTest(Tokens, array_lengthof(Tokens), 4, false, false));
-}
-
-// Test case where call isn't at the beginning of sequence of tokens.
-TEST(NaClTextFormatterTest, TokensPlusSimpleCall) {
-  static const char *Tokens[] = {
-    "354", " ", "+", " ", "the", " ", "best", " ", "+", " ",
-    "foobar", "(", "Value1", ",", " ", "Value2", "," , " ", "Value3", ")"
-  };
-
-  // Print out where all tokens fit on one line.
-  EXPECT_EQ(
-      "354 + the best + foobar(Value1, Value2, Value3)\n",
-      RunTest(Tokens, array_lengthof(Tokens), 47, true, true));
-
-  EXPECT_EQ(
-      "354 + the best + foobar(Value1, Value2, Value3)\n",
-      RunTest(Tokens, array_lengthof(Tokens), 47, true, false));
-
-  EXPECT_EQ(
-      "354 + the best + foobar(Value1, Value2, Value3)\n",
-      RunTest(Tokens, array_lengthof(Tokens), 47, false, true));
-
-  EXPECT_EQ(
-      "354 + the best + foobar(Value1, Value2, Value3)\n",
-      RunTest(Tokens, array_lengthof(Tokens), 47, false, false));
-
-  // Format cases where buffer is one character too short to fit
-  // all tokens.
-  EXPECT_EQ(
-      "354 + the best + \n"
-      "    foobar(Value1, Value2, Value3)\n",
-      RunTest(Tokens, array_lengthof(Tokens), 46, true, true));
-
-  EXPECT_EQ(
-      "354 + the best + foobar(Value1, Value2, Value3\n"
-      "                        )\n",
-      RunTest(Tokens, array_lengthof(Tokens), 46, true, false));
-
-  EXPECT_EQ(
-      "354 + the best + \n"
-      "    foobar(Value1, Value2, Value3)\n",
-      RunTest(Tokens, array_lengthof(Tokens), 46, false, true));
-
-  EXPECT_EQ(
-      "354 + the best + foobar(Value1, Value2, Value3\n"
-      "    )\n",
-      RunTest(Tokens, array_lengthof(Tokens), 46, false, false));
-
-  // Show case where function call just fits on continuation line.
-  EXPECT_EQ(
-      "354 + the best + \n"
-      "    foobar(Value1, Value2, Value3)\n",
-      RunTest(Tokens, array_lengthof(Tokens), 34, true, true));
-
-  EXPECT_EQ(
-      "354 + the best + foobar(Value1, \n"
-      "              Value2, Value3)\n",
-      RunTest(Tokens, array_lengthof(Tokens), 34, true, false));
-
-  EXPECT_EQ(
-      "354 + the best + \n"
-      "    foobar(Value1, Value2, Value3)\n",
-      RunTest(Tokens, array_lengthof(Tokens), 34, false, true));
-
-  EXPECT_EQ(
-      "354 + the best + foobar(Value1, \n"
-      "    Value2, Value3)\n",
-      RunTest(Tokens, array_lengthof(Tokens), 34, false, false));
-
-  // Show case were close parenthesis doesn't fit on continuation line.
-  EXPECT_EQ(
-      "354 + the best + \n"
-      "    foobar(Value1, Value2, Value3\n"
-      "           )\n",
-      RunTest(Tokens, array_lengthof(Tokens), 33, true, true));
-
-  EXPECT_EQ(
-      "354 + the best + foobar(Value1, \n"
-      "             Value2, Value3)\n",
-      RunTest(Tokens, array_lengthof(Tokens), 33, true, false));
-
-  EXPECT_EQ(
-      "354 + the best + \n"
-      "    foobar(Value1, Value2, Value3\n"
-      "    )\n",
-      RunTest(Tokens, array_lengthof(Tokens), 33, false, true));
-
-  EXPECT_EQ(
-      "354 + the best + foobar(Value1, \n"
-      "    Value2, Value3)\n",
-      RunTest(Tokens, array_lengthof(Tokens), 33, false, false));
-
-  // Show case where "Value1," just fits on the first continuation line.
-  EXPECT_EQ(
-      "354 + the best\n"
-      "+ \n"
-      "foobar(Value1,\n"
-      "Value2, Value3\n"
-      ")\n",
-      RunTest(Tokens, array_lengthof(Tokens), 14, true, true));
-
-  EXPECT_EQ(
-      "354 + the best\n"
-      "+ foobar(\n"
-      "Value1, Value2\n"
-      ", Value3)\n",
-      RunTest(Tokens, array_lengthof(Tokens), 14, true, false));
-
-  EXPECT_EQ(
-      "354 + the best\n"
-      "+ \n"
-      "foobar(Value1,\n"
-      "Value2, Value3\n"
-      ")\n",
-      RunTest(Tokens, array_lengthof(Tokens), 14, false, true));
-
-  EXPECT_EQ(
-      "354 + the best\n"
-      "+ foobar(\n"
-      "Value1, Value2\n,"
-      " Value3)\n",
-      RunTest(Tokens, array_lengthof(Tokens), 14, false, false));
-
-  // Show case where "Value1," moves to a new line.
-  EXPECT_EQ(
-      "354 + the \n"
-      "best + \n"
-      "foobar(\n"
-      "Value1, \n"
-      "Value2, \n"
-      "Value3)\n",
-      RunTest(Tokens, array_lengthof(Tokens), 13, true, true));
-
-  EXPECT_EQ(
-      "354 + the \n"
-      "best + foobar\n"
-      "(Value1, \n"
-      "Value2, \n"
-      "Value3)\n",
-      RunTest(Tokens, array_lengthof(Tokens), 13, true, false));
-
-  EXPECT_EQ(
-      "354 + the \n"
-      "best + \n"
-      "foobar(\n"
-      "Value1, \n"
-      "Value2, \n"
-      "Value3)\n",
-      RunTest(Tokens, array_lengthof(Tokens), 13, false, true));
-
-  EXPECT_EQ(
-      "354 + the \n"
-      "best + foobar\n"
-      "(Value1, \n"
-      "Value2, \n"
-      "Value3)\n",
-      RunTest(Tokens, array_lengthof(Tokens), 13, false, false));
-
-}
-
-// Test case of nested functions.
-TEST(NaClTextFormatterTest, NestedCalls) {
-  static const char *Tokens[] = {
-    "354", " ", "+", " ", "foo", "(", "g", "(", "blah", ")", ",", " ",
-    "h", "(", ")", " ", "+", " ", "1", ")", " ", "+", " ", "10"
-  };
-
-  // Run test case where all text fits on one line.
-  EXPECT_EQ(
-      "354 + foo(g(blah), h() + 1) + 10\n",
-      RunTest(Tokens, array_lengthof(Tokens), 32, true, true));
-
-  EXPECT_EQ(
-      "354 + foo(g(blah), h() + 1) + 10\n",
-      RunTest(Tokens, array_lengthof(Tokens), 32, true, false));
-
-  EXPECT_EQ(
-      "354 + foo(g(blah), h() + 1) + 10\n",
-      RunTest(Tokens, array_lengthof(Tokens), 32, false, true));
-
-  EXPECT_EQ(
-      "354 + foo(g(blah), h() + 1) + 10\n",
-      RunTest(Tokens, array_lengthof(Tokens), 32, false, false));
-
-  // Run test case where all text to end of top-level function call
-  // fit on first line.
-  EXPECT_EQ(
-      "354 + foo(g(blah), h() + 1)\n"
-      "    + 10\n",
-      RunTest(Tokens, array_lengthof(Tokens), 27, true, true));
-
-  EXPECT_EQ(
-      "354 + foo(g(blah), h() + 1)\n"
-      "    + 10\n",
-      RunTest(Tokens, array_lengthof(Tokens), 27, true, false));
-
-  EXPECT_EQ(
-      "354 + foo(g(blah), h() + 1)\n"
-      "    + 10\n",
-      RunTest(Tokens, array_lengthof(Tokens), 27, false, true));
-
-  EXPECT_EQ(
-      "354 + foo(g(blah), h() + 1)\n"
-      "    + 10\n",
-      RunTest(Tokens, array_lengthof(Tokens), 27, false, false));
-
-  // Run test where call to foo doesn't fit on first line.
-  EXPECT_EQ(
-      "354 + \n"
-      "    foo(g(blah), h() + 1) \n"
-      "    + 10\n",
-      RunTest(Tokens, array_lengthof(Tokens), 26, true, true));
-
-  EXPECT_EQ(
-      "354 + foo(g(blah), h() + 1\n"
-      "      ) + 10\n",
-      RunTest(Tokens, array_lengthof(Tokens), 26, true, false));
-
-  EXPECT_EQ(
-      "354 + \n"
-      "    foo(g(blah), h() + 1) \n"
-      "    + 10\n",
-      RunTest(Tokens, array_lengthof(Tokens), 26, false, true));
-
-  EXPECT_EQ(
-      "354 + foo(g(blah), h() + 1\n"
-      "    ) + 10\n",
-      RunTest(Tokens, array_lengthof(Tokens), 26, false, false));
-
-  // Run test where call to foo doesn't fit on continuation line.
-  EXPECT_EQ(
-      "354 + \n"
-      "    foo(g(blah), h() + 1\n"
-      "    ) + 10\n",
-      RunTest(Tokens, array_lengthof(Tokens), 24, true, true));
-
-  EXPECT_EQ(
-      "354 + foo(g(blah), h() +\n"
-      "    1) + 10\n",
-      RunTest(Tokens, array_lengthof(Tokens), 24, true, false));
-
-  EXPECT_EQ(
-      "354 + \n"
-      "    foo(g(blah), h() + 1\n"
-      "    ) + 10\n",
-      RunTest(Tokens, array_lengthof(Tokens), 24, false, true));
-
-  EXPECT_EQ(
-      "354 + foo(g(blah), h() +\n"
-      "    1) + 10\n",
-      RunTest(Tokens, array_lengthof(Tokens), 24, false, false));
-
-  // Run test where call to foo doesn't fit on continuation line.
-  // Note: same as above, except for loss of continuation indent,
-  // since we don't indent when printable space shrinks to 20.
-  EXPECT_EQ(
-      "354 + \n"
-      "foo(g(blah), h() + 1\n"
-      ") + 10\n",
-      RunTest(Tokens, array_lengthof(Tokens), 20, true, true));
-
-  EXPECT_EQ(
-      "354 + foo(g(blah), h\n"
-      "() + 1) + 10\n",
-      RunTest(Tokens, array_lengthof(Tokens), 20, true, false));
-
-  EXPECT_EQ(
-      "354 + \n"
-      "foo(g(blah), h() + 1\n"
-      ") + 10\n",
-      RunTest(Tokens, array_lengthof(Tokens), 20, false, true));
-
-  EXPECT_EQ(
-      "354 + foo(g(blah), h\n"
-      "() + 1) + 10\n",
-      RunTest(Tokens, array_lengthof(Tokens), 20, false, false));
-
-  // Run case where first argument of foo (i.e. g(blah)) fits
-  // on single continuation line.
-  EXPECT_EQ(
-      "354 + \n"
-      "foo(g(blah), \n"
-      "h() + 1) + 10\n",
-      RunTest(Tokens, array_lengthof(Tokens), 19, true, true));
-
-  EXPECT_EQ(
-      "354 + foo(g(blah), \n"
-      "h() + 1) + 10\n",
-      RunTest(Tokens, array_lengthof(Tokens), 19, true, false));
-
-  EXPECT_EQ(
-      "354 + \n"
-      "foo(g(blah), \n"
-      "h() + 1) + 10\n",
-      RunTest(Tokens, array_lengthof(Tokens), 19, false, true));
-
-  EXPECT_EQ(
-      "354 + foo(g(blah), \n"
-      "h() + 1) + 10\n",
-      RunTest(Tokens, array_lengthof(Tokens), 19, false, false));
-
-  // Run case where no room for call to foo and its first argument.
-  EXPECT_EQ(
-      "354 + \n"
-      "foo(\n"
-      "g(blah), \n"
-      "h() + 1) + \n"
-      "10\n",
-      RunTest(Tokens, array_lengthof(Tokens), 11, true, true));
-
-  EXPECT_EQ(
-      "354 + foo(g\n"
-      "(blah), h()\n"
-      "+ 1) + 10\n",
-      RunTest(Tokens, array_lengthof(Tokens), 11, true, false));
-
-  EXPECT_EQ(
-      "354 + \n"
-      "foo(\n"
-      "g(blah), \n"
-      "h() + 1) + \n"
-      "10\n",
-      RunTest(Tokens, array_lengthof(Tokens), 11, false, true));
-
-  EXPECT_EQ(
-      "354 + foo(g\n"
-      "(blah), h()\n"
-      "+ 1) + 10\n",
-      RunTest(Tokens, array_lengthof(Tokens), 11, false, false));
-}
-
-// Test example with many arguments (which can't be printed on one line).
-TEST(NaClTextFormatterTest, ManyArgs) {
-  static const char *Tokens[] = {
-    "10", " ", "+", " ", "f", "(",
-    "g", "(", "a", ",", " ", "b", ")", ",", " ",
-    "abcdef", " ", "+", " ", "gh1196", " ", "+", " ", "z", "(", ")", ",", " ",
-    "53267", " ", "*", " ", "1234", " ", "+", " ", "567", ",", " ",
-    "why", "(", "is", ",", " ", "this", ",", " ", "so", ",", " ", "hard",
-    ",", " ", "to", ",", " ", "do", ")", ",", " ",
-    "g", "(", "a", ",", " ", "b", ")", ",", " ",
-    "abcdef", " ", "+", " ", "gh1196", " ", "+", " ", "z", "(", ")", ",", " ",
-    "53267", " ", "*", " ", "1234", " ", "+", " ", "567", " ", "*", " ",
-    "somemorestuff", ")", " ", "+", " ", "1"
-  };
-
-  // Show layout with linewidth 70
-  EXPECT_EQ(
-      "10 + \n"
-      "    f(g(a, b), abcdef + gh1196 + z(), 53267 * 1234 + 567, \n"
-      "      why(is, this, so, hard, to, do), g(a, b), abcdef + gh1196 + z(),\n"
-      "      53267 * 1234 + 567 * somemorestuff) + 1\n",
-      RunTest(Tokens, array_lengthof(Tokens), 70, true, true));
-
-  EXPECT_EQ(
-      "10 + f(g(a, b), abcdef + gh1196 + z(), 53267 * 1234 + 567, why(is, \n"
-      "                                                  this, so, hard, to, \n"
-      "                                                  do), g(a, b), abcdef\n"
-      "       + gh1196 + z(), 53267 * 1234 + 567 * somemorestuff) + 1\n",
-      RunTest(Tokens, array_lengthof(Tokens), 70, true, false));
-
-  EXPECT_EQ(
-      "10 + \n"
-      "    f(g(a, b), abcdef + gh1196 + z(), 53267 * 1234 + 567, \n"
-      "    why(is, this, so, hard, to, do), g(a, b), abcdef + gh1196 + z(), \n"
-      "    53267 * 1234 + 567 * somemorestuff) + 1\n",
-      RunTest(Tokens, array_lengthof(Tokens), 70, false, true));
-
-  EXPECT_EQ(
-      "10 + f(g(a, b), abcdef + gh1196 + z(), 53267 * 1234 + 567, why(is, \n"
-      "    this, so, hard, to, do), g(a, b), abcdef + gh1196 + z(), 53267 * \n"
-      "    1234 + 567 * somemorestuff) + 1\n",
-      RunTest(Tokens, array_lengthof(Tokens), 70, false, false));
-
-  // Show layout with linewidth 60
-  EXPECT_EQ(
-      "10 + \n"
-      "    f(g(a, b), abcdef + gh1196 + z(), 53267 * 1234 + 567, \n"
-      "      why(is, this, so, hard, to, do), g(a, b), \n"
-      "      abcdef + gh1196 + z(), \n"
-      "      53267 * 1234 + 567 * somemorestuff) + 1\n",
-      RunTest(Tokens, array_lengthof(Tokens), 60, true, true));
-
-  EXPECT_EQ(
-      "10 + f(g(a, b), abcdef + gh1196 + z(), 53267 * 1234 + 567, \n"
-      "       why(is, this, so, hard, to, do), g(a, b), abcdef + \n"
-      "       gh1196 + z(), 53267 * 1234 + 567 * somemorestuff) + 1\n",
-      RunTest(Tokens, array_lengthof(Tokens), 60, true, false));
-
-  EXPECT_EQ(
-      "10 + \n"
-      "    f(g(a, b), abcdef + gh1196 + z(), 53267 * 1234 + 567, \n"
-      "    why(is, this, so, hard, to, do), g(a, b), \n"
-      "    abcdef + gh1196 + z(), \n"
-      "    53267 * 1234 + 567 * somemorestuff) + 1\n",
-      RunTest(Tokens, array_lengthof(Tokens), 60, false, true));
-
-  EXPECT_EQ(
-      "10 + f(g(a, b), abcdef + gh1196 + z(), 53267 * 1234 + 567, \n"
-      "    why(is, this, so, hard, to, do), g(a, b), abcdef + \n"
-      "    gh1196 + z(), 53267 * 1234 + 567 * somemorestuff) + 1\n",
-      RunTest(Tokens, array_lengthof(Tokens), 60, false, false));
-
-  // Show layout with linewidth 50.
-  EXPECT_EQ(
-      "10 + \n"
-      "    f(g(a, b), abcdef + gh1196 + z(), \n"
-      "      53267 * 1234 + 567, \n"
-      "      why(is, this, so, hard, to, do), g(a, b), \n"
-      "      abcdef + gh1196 + z(), \n"
-      "      53267 * 1234 + 567 * somemorestuff) + 1\n",
-      RunTest(Tokens, array_lengthof(Tokens), 50, true, true));
-
-  EXPECT_EQ(
-      "10 + f(g(a, b), abcdef + gh1196 + z(), 53267 * \n"
-      "       1234 + 567, why(is, this, so, hard, to, do)\n"
-      "       , g(a, b), abcdef + gh1196 + z(), 53267 * \n"
-      "       1234 + 567 * somemorestuff) + 1\n",
-      RunTest(Tokens, array_lengthof(Tokens), 50, true, false));
-
-  EXPECT_EQ(
-      "10 + \n"
-      "    f(g(a, b), abcdef + gh1196 + z(), \n"
-      "    53267 * 1234 + 567, \n"
-      "    why(is, this, so, hard, to, do), g(a, b), \n"
-      "    abcdef + gh1196 + z(), \n"
-      "    53267 * 1234 + 567 * somemorestuff) + 1\n",
-      RunTest(Tokens, array_lengthof(Tokens), 50, false, true));
-
-  EXPECT_EQ(
-      "10 + f(g(a, b), abcdef + gh1196 + z(), 53267 * \n"
-      "    1234 + 567, why(is, this, so, hard, to, do), g\n"
-      "    (a, b), abcdef + gh1196 + z(), 53267 * 1234 + \n"
-      "    567 * somemorestuff) + 1\n",
-      RunTest(Tokens, array_lengthof(Tokens), 50, false, false));
-
-  // Show layout with linewidth 40
-  EXPECT_EQ(
-      "10 + \n"
-      "    f(g(a, b), abcdef + gh1196 + z(), \n"
-      "      53267 * 1234 + 567, \n"
-      "      why(is, this, so, hard, to, do), \n"
-      "      g(a, b), abcdef + gh1196 + z(), \n"
-      "      53267 * 1234 + 567 * somemorestuff\n"
-      "      ) + 1\n",
-      RunTest(Tokens, array_lengthof(Tokens), 40, true, true));
-
-  EXPECT_EQ(
-      "10 + f(g(a, b), abcdef + gh1196 + z(), \n"
-      "       53267 * 1234 + 567, why(is, this,\n"
-      "                    so, hard, to, do), g\n"
-      "       (a, b), abcdef + gh1196 + z(), \n"
-      "       53267 * 1234 + 567 * \n"
-      "       somemorestuff) + 1\n",
-      RunTest(Tokens, array_lengthof(Tokens), 40, true, false));
-
-  EXPECT_EQ(
-      "10 + \n"
-      "    f(g(a, b), abcdef + gh1196 + z(), \n"
-      "    53267 * 1234 + 567, \n"
-      "    why(is, this, so, hard, to, do), \n"
-      "    g(a, b), abcdef + gh1196 + z(), \n"
-      "    53267 * 1234 + 567 * somemorestuff) \n"
-      "    + 1\n",
-      RunTest(Tokens, array_lengthof(Tokens), 40, false, true));
-
-  EXPECT_EQ(
-      "10 + f(g(a, b), abcdef + gh1196 + z(), \n"
-      "    53267 * 1234 + 567, why(is, this, so\n"
-      "    , hard, to, do), g(a, b), abcdef + \n"
-      "    gh1196 + z(), 53267 * 1234 + 567 * \n"
-      "    somemorestuff) + 1\n",
-      RunTest(Tokens, array_lengthof(Tokens), 40, false, false));
-
-  // Show layout with linewidth 30
-  EXPECT_EQ(
-      "10 + \n"
-      "    f(g(a, b), \n"
-      "      abcdef + gh1196 + z(), \n"
-      "      53267 * 1234 + 567, \n"
-      "      why(is, this, so, hard, \n"
-      "          to, do), g(a, b), \n"
-      "      abcdef + gh1196 + z(), \n"
-      "      53267 * 1234 + 567 * \n"
-      "      somemorestuff) + 1\n",
-      RunTest(Tokens, array_lengthof(Tokens), 30, true, true));
-
-  EXPECT_EQ(
-      "10 + f(g(a, b), abcdef + \n"
-      "       gh1196 + z(), 53267 * \n"
-      "       1234 + 567, why(is, \n"
-      "          this, so, hard, to, \n"
-      "          do), g(a, b), abcdef\n"
-      "       + gh1196 + z(), 53267 *\n"
-      "       1234 + 567 * \n"
-      "       somemorestuff) + 1\n",
-      RunTest(Tokens, array_lengthof(Tokens), 30, true, false));
-
-  EXPECT_EQ(
-      "10 + \n"
-      "    f(g(a, b), \n"
-      "    abcdef + gh1196 + z(), \n"
-      "    53267 * 1234 + 567, \n"
-      "    why(is, this, so, hard, \n"
-      "    to, do), g(a, b), \n"
-      "    abcdef + gh1196 + z(), \n"
-      "    53267 * 1234 + 567 * \n"
-      "    somemorestuff) + 1\n",
-      RunTest(Tokens, array_lengthof(Tokens), 30, false, true));
-
-  EXPECT_EQ(
-      "10 + f(g(a, b), abcdef + \n"
-      "    gh1196 + z(), 53267 * 1234\n"
-      "    + 567, why(is, this, so, \n"
-      "    hard, to, do), g(a, b), \n"
-      "    abcdef + gh1196 + z(), \n"
-      "    53267 * 1234 + 567 * \n"
-      "    somemorestuff) + 1\n",
-      RunTest(Tokens, array_lengthof(Tokens), 30, false, false));
-
-  // Show layout with linewidth 20. Note: Continuation indents no
-  // longer apply.
-  EXPECT_EQ(
-      "10 + \n"
-      "f(g(a, b), \n"
-      "abcdef + gh1196 + \n"
-      "z(), \n"
-      "53267 * 1234 + 567, \n"
-      "why(is, this, so, \n"
-      "hard, to, do), \n"
-      "g(a, b), \n"
-      "abcdef + gh1196 + \n"
-      "z(), \n"
-      "53267 * 1234 + 567 *\n"
-      "somemorestuff) + 1\n",
-      RunTest(Tokens, array_lengthof(Tokens), 20, true, true));
-
-  EXPECT_EQ(
-      "10 + f(g(a, b), \n"
-      "abcdef + gh1196 + z(\n"
-      "), 53267 * 1234 + \n"
-      "567, why(is, this, \n"
-      "so, hard, to, do), g\n"
-      "(a, b), abcdef + \n"
-      "gh1196 + z(), 53267 \n"
-      "* 1234 + 567 * \n"
-      "somemorestuff) + 1\n",
-      RunTest(Tokens, array_lengthof(Tokens), 20, true, false));
-
-  EXPECT_EQ(
-      "10 + \n"
-      "f(g(a, b), \n"
-      "abcdef + gh1196 + \n"
-      "z(), \n"
-      "53267 * 1234 + 567, \n"
-      "why(is, this, so, \n"
-      "hard, to, do), \n"
-      "g(a, b), \n"
-      "abcdef + gh1196 + \n"
-      "z(), \n"
-      "53267 * 1234 + 567 *\n"
-      "somemorestuff) + 1\n",
-      RunTest(Tokens, array_lengthof(Tokens), 20, false, true));
-
-  EXPECT_EQ(
-      "10 + f(g(a, b), \n"
-      "abcdef + gh1196 + z(\n"
-      "), 53267 * 1234 + \n"
-      "567, why(is, this, \n"
-      "so, hard, to, do), g\n"
-      "(a, b), abcdef + \n"
-      "gh1196 + z(), 53267 \n"
-      "* 1234 + 567 * \n"
-      "somemorestuff) + 1\n",
-      RunTest(Tokens, array_lengthof(Tokens), 20, false, false));
-
-  // Show layout with linewidth 10, where some tokens ("somemorestuff")
-  // exceed the line width requirement.
-  EXPECT_EQ(
-      "10 + \n"
-      "f(g(a, b),\n"
-      "abcdef + \n"
-      "gh1196 + \n"
-      "z(), \n"
-      "53267 * \n"
-      "1234 + 567\n"
-      ", \n"
-      "why(is, \n"
-      "this, so, \n"
-      "hard, to, \n"
-      "do), \n"
-      "g(a, b), \n"
-      "abcdef + \n"
-      "gh1196 + \n"
-      "z(), \n"
-      "53267 * \n"
-      "1234 + 567\n"
-      "* \n"
-      "somemorestuff\n"
-      ") + 1\n",
-      RunTest(Tokens, array_lengthof(Tokens), 10, true, true));
-
-  EXPECT_EQ(
-      "10 + f(g(a\n"
-      ", b), \n"
-      "abcdef + \n"
-      "gh1196 + z\n"
-      "(), 53267 \n"
-      "* 1234 + \n"
-      "567, why(\n"
-      "is, this, \n"
-      "so, hard, \n"
-      "to, do), g\n"
-      "(a, b), \n"
-      "abcdef + \n"
-      "gh1196 + z\n"
-      "(), 53267 \n"
-      "* 1234 + \n"
-      "567 * \n"
-      "somemorestuff\n"
-      ") + 1\n",
-      RunTest(Tokens, array_lengthof(Tokens), 10, true, false));
-
-  EXPECT_EQ(
-      "10 + \n"
-      "f(g(a, b),\n"
-      "abcdef + \n"
-      "gh1196 + \n"
-      "z(), \n"
-      "53267 * \n"
-      "1234 + 567\n"
-      ", \n"
-      "why(is, \n"
-      "this, so, \n"
-      "hard, to, \n"
-      "do), \n"
-      "g(a, b), \n"
-      "abcdef + \n"
-      "gh1196 + \n"
-      "z(), \n"
-      "53267 * \n"
-      "1234 + 567\n"
-      "* \n"
-      "somemorestuff\n"
-      ") + 1\n",
-      RunTest(Tokens, array_lengthof(Tokens), 10, false, true));
-
-  EXPECT_EQ(
-      "10 + f(g(a\n"
-      ", b), \n"
-      "abcdef + \n"
-      "gh1196 + z\n"
-      "(), 53267 \n"
-      "* 1234 + \n"
-      "567, why(\n"
-      "is, this, \n"
-      "so, hard, \n"
-      "to, do), g\n"
-      "(a, b), \n"
-      "abcdef + \n"
-      "gh1196 + z\n"
-      "(), 53267 \n"
-      "* 1234 + \n"
-      "567 * \n"
-      "somemorestuff\n"
-      ") + 1\n",
-      RunTest(Tokens, array_lengthof(Tokens), 10, false, false));
-}
-
-// Turn test case that checks if indenting works.
-TEST(NaClTextFormatterTest, Indenting) {
-  static const char *Tokens[] = {
-    "354", " ", "+", " ", "foo", "(", "g", "(", "blah", ")", ",", " ",
-    "h", "(", ")", " ", "+", " ", "1", ")", " ", "+", " ", "10"
-  };
-
-  // Run with no indentation.
-  EXPECT_EQ(
-      "354 + foo(g(blah), h() + 1) + 10\n",
-      RunTest(Tokens, array_lengthof(Tokens), 32, true, true, 0));
-
-  EXPECT_EQ(
-      "354 + foo(g(blah), h() + 1) + 10\n",
-      RunTest(Tokens, array_lengthof(Tokens), 32, true, false, 0));
-
-  EXPECT_EQ(
-      "354 + foo(g(blah), h() + 1) + 10\n",
-      RunTest(Tokens, array_lengthof(Tokens), 32, false, true, 0));
-
-  EXPECT_EQ(
-      "354 + foo(g(blah), h() + 1) + 10\n",
-      RunTest(Tokens, array_lengthof(Tokens), 32, false, false, 0));
-
-  // Run with one indent.
-  EXPECT_EQ(
-      "  354 + foo(g(blah), h() + 1) + \n"
-      "      10\n",
-      RunTest(Tokens, array_lengthof(Tokens), 32, true, true, 1));
-
-  EXPECT_EQ(
-      "  354 + foo(g(blah), h() + 1) + \n"
-      "      10\n",
-      RunTest(Tokens, array_lengthof(Tokens), 32, true, false, 1));
-
-  EXPECT_EQ(
-      "  354 + foo(g(blah), h() + 1) + \n"
-      "      10\n",
-      RunTest(Tokens, array_lengthof(Tokens), 32, false, true, 1));
-
-  EXPECT_EQ(
-      "  354 + foo(g(blah), h() + 1) + \n"
-      "      10\n",
-      RunTest(Tokens, array_lengthof(Tokens), 32, false, false, 1));
-
-
-  // Run with two indents.
-  EXPECT_EQ(
-      "    354 + foo(g(blah), h() + 1) \n"
-      "        + 10\n",
-      RunTest(Tokens, array_lengthof(Tokens), 32, true, true, 2));
-
-  EXPECT_EQ(
-      "    354 + foo(g(blah), h() + 1) \n"
-      "        + 10\n",
-      RunTest(Tokens, array_lengthof(Tokens), 32, true, false, 2));
-
-  EXPECT_EQ(
-      "    354 + foo(g(blah), h() + 1) \n"
-      "        + 10\n",
-      RunTest(Tokens, array_lengthof(Tokens), 32, false, true, 2));
-
-  EXPECT_EQ(
-      "    354 + foo(g(blah), h() + 1) \n"
-      "        + 10\n",
-      RunTest(Tokens, array_lengthof(Tokens), 32, false, false, 2));
-
-  // Run with five indents.
-  EXPECT_EQ(
-      "          354 + \n"
-      "            foo(g(blah), h() + 1\n"
-      "            ) + 10\n",
-      RunTest(Tokens, array_lengthof(Tokens), 32, true, true, 5));
-
-  EXPECT_EQ(
-      "          354 + foo(g(blah), h()\n"
-      "            + 1) + 10\n",
-      RunTest(Tokens, array_lengthof(Tokens), 32, true, false, 5));
-
-  EXPECT_EQ(
-      "          354 + \n"
-      "            foo(g(blah), h() + 1\n"
-      "            ) + 10\n",
-      RunTest(Tokens, array_lengthof(Tokens), 32, false, true, 5));
-
-  EXPECT_EQ(
-      "          354 + foo(g(blah), h()\n"
-      "            + 1) + 10\n",
-      RunTest(Tokens, array_lengthof(Tokens), 32, false, false, 5));
-}
-}
diff --git a/unittests/CMakeLists.txt b/unittests/CMakeLists.txt
index a1272edcc97..e5befcec6e0 100644
--- a/unittests/CMakeLists.txt
+++ b/unittests/CMakeLists.txt
@@ -13,6 +13,7 @@ endfunction()
 
 add_subdirectory(ADT)
 add_subdirectory(Analysis)
+add_subdirectory(AsmParser)
 add_subdirectory(Bitcode)
 add_subdirectory(CodeGen)
 add_subdirectory(DebugInfo)
diff --git a/unittests/CodeGen/DIEHashTest.cpp b/unittests/CodeGen/DIEHashTest.cpp
index efb0920b179..e3a9e562827 100644
--- a/unittests/CodeGen/DIEHashTest.cpp
+++ b/unittests/CodeGen/DIEHashTest.cpp
@@ -10,6 +10,7 @@
 #include "llvm/CodeGen/DIE.h"
 #include "../lib/CodeGen/AsmPrinter/DIEHash.h"
 #include "llvm/ADT/STLExtras.h"
+#include "llvm/CodeGen/DwarfStringPoolEntry.h"
 #include "llvm/Support/Debug.h"
 #include "llvm/Support/Dwarf.h"
 #include "llvm/Support/Format.h"
@@ -18,24 +19,41 @@
 using namespace llvm;
 
 namespace {
-TEST(DIEHashTest, Data1) {
+
+// Test fixture
+class DIEHashTest : public testing::Test {
+public:
+  BumpPtrAllocator Alloc;
+
+private:
+  StringMap<DwarfStringPoolEntry> Pool;
+
+public:
+  DIEString getString(StringRef S) {
+    DwarfStringPoolEntry Entry = {nullptr, 1, 1};
+    return DIEString(
+        DwarfStringPoolEntryRef(*Pool.insert(std::make_pair(S, Entry)).first));
+  }
+};
+
+TEST_F(DIEHashTest, Data1) {
   DIEHash Hash;
-  DIE Die(dwarf::DW_TAG_base_type);
+  DIE &Die = *DIE::get(Alloc, dwarf::DW_TAG_base_type);
   DIEInteger Size(4);
-  Die.addValue(dwarf::DW_AT_byte_size, dwarf::DW_FORM_data1, &Size);
+  Die.addValue(Alloc, dwarf::DW_AT_byte_size, dwarf::DW_FORM_data1, Size);
   uint64_t MD5Res = Hash.computeTypeSignature(Die);
   ASSERT_EQ(0x1AFE116E83701108ULL, MD5Res);
 }
 
 // struct {};
-TEST(DIEHashTest, TrivialType) {
-  DIE Unnamed(dwarf::DW_TAG_structure_type);
+TEST_F(DIEHashTest, TrivialType) {
+  DIE &Unnamed = *DIE::get(Alloc, dwarf::DW_TAG_structure_type);
   DIEInteger One(1);
-  Unnamed.addValue(dwarf::DW_AT_byte_size, dwarf::DW_FORM_data1, &One);
+  Unnamed.addValue(Alloc, dwarf::DW_AT_byte_size, dwarf::DW_FORM_data1, One);
 
   // Line and file number are ignored.
-  Unnamed.addValue(dwarf::DW_AT_decl_file, dwarf::DW_FORM_data1, &One);
-  Unnamed.addValue(dwarf::DW_AT_decl_line, dwarf::DW_FORM_data1, &One);
+  Unnamed.addValue(Alloc, dwarf::DW_AT_decl_file, dwarf::DW_FORM_data1, One);
+  Unnamed.addValue(Alloc, dwarf::DW_AT_decl_line, dwarf::DW_FORM_data1, One);
   uint64_t MD5Res = DIEHash().computeTypeSignature(Unnamed);
 
   // The exact same hash GCC produces for this DIE.
@@ -43,12 +61,12 @@ TEST(DIEHashTest, TrivialType) {
 }
 
 // struct foo { };
-TEST(DIEHashTest, NamedType) {
-  DIE Foo(dwarf::DW_TAG_structure_type);
+TEST_F(DIEHashTest, NamedType) {
+  DIE &Foo = *DIE::get(Alloc, dwarf::DW_TAG_structure_type);
   DIEInteger One(1);
-  DIEString FooStr(&One, "foo");
-  Foo.addValue(dwarf::DW_AT_name, dwarf::DW_FORM_strp, &FooStr);
-  Foo.addValue(dwarf::DW_AT_byte_size, dwarf::DW_FORM_data1, &One);
+  DIEString FooStr = getString("foo");
+  Foo.addValue(Alloc, dwarf::DW_AT_name, dwarf::DW_FORM_strp, FooStr);
+  Foo.addValue(Alloc, dwarf::DW_AT_byte_size, dwarf::DW_FORM_data1, One);
 
   uint64_t MD5Res = DIEHash().computeTypeSignature(Foo);
 
@@ -57,21 +75,22 @@ TEST(DIEHashTest, NamedType) {
 }
 
 // namespace space { struct foo { }; }
-TEST(DIEHashTest, NamespacedType) {
-  DIE CU(dwarf::DW_TAG_compile_unit);
+TEST_F(DIEHashTest, NamespacedType) {
+  DIE &CU = *DIE::get(Alloc, dwarf::DW_TAG_compile_unit);
 
-  auto Space = make_unique<DIE>(dwarf::DW_TAG_namespace);
+  auto Space = DIE::get(Alloc, dwarf::DW_TAG_namespace);
   DIEInteger One(1);
-  DIEString SpaceStr(&One, "space");
-  Space->addValue(dwarf::DW_AT_name, dwarf::DW_FORM_strp, &SpaceStr);
+  DIEString SpaceStr = getString("space");
+  Space->addValue(Alloc, dwarf::DW_AT_name, dwarf::DW_FORM_strp, SpaceStr);
   // DW_AT_declaration is ignored.
-  Space->addValue(dwarf::DW_AT_declaration, dwarf::DW_FORM_flag_present, &One);
+  Space->addValue(Alloc, dwarf::DW_AT_declaration, dwarf::DW_FORM_flag_present,
+                  One);
   // sibling?
 
-  auto Foo = make_unique<DIE>(dwarf::DW_TAG_structure_type);
-  DIEString FooStr(&One, "foo");
-  Foo->addValue(dwarf::DW_AT_name, dwarf::DW_FORM_strp, &FooStr);
-  Foo->addValue(dwarf::DW_AT_byte_size, dwarf::DW_FORM_data1, &One);
+  auto Foo = DIE::get(Alloc, dwarf::DW_TAG_structure_type);
+  DIEString FooStr = getString("foo");
+  Foo->addValue(Alloc, dwarf::DW_AT_name, dwarf::DW_FORM_strp, FooStr);
+  Foo->addValue(Alloc, dwarf::DW_AT_byte_size, dwarf::DW_FORM_data1, One);
 
   DIE &N = *Foo;
   Space->addChild(std::move(Foo));
@@ -84,27 +103,27 @@ TEST(DIEHashTest, NamespacedType) {
 }
 
 // struct { int member; };
-TEST(DIEHashTest, TypeWithMember) {
-  DIE Unnamed(dwarf::DW_TAG_structure_type);
+TEST_F(DIEHashTest, TypeWithMember) {
+  DIE &Unnamed = *DIE::get(Alloc, dwarf::DW_TAG_structure_type);
   DIEInteger Four(4);
-  Unnamed.addValue(dwarf::DW_AT_byte_size, dwarf::DW_FORM_data1, &Four);
+  Unnamed.addValue(Alloc, dwarf::DW_AT_byte_size, dwarf::DW_FORM_data1, Four);
 
-  DIE Int(dwarf::DW_TAG_base_type);
-  DIEString IntStr(&Four, "int");
-  Int.addValue(dwarf::DW_AT_name, dwarf::DW_FORM_strp, &IntStr);
-  Int.addValue(dwarf::DW_AT_byte_size, dwarf::DW_FORM_data1, &Four);
+  DIE &Int = *DIE::get(Alloc, dwarf::DW_TAG_base_type);
+  DIEString IntStr = getString("int");
+  Int.addValue(Alloc, dwarf::DW_AT_name, dwarf::DW_FORM_strp, IntStr);
+  Int.addValue(Alloc, dwarf::DW_AT_byte_size, dwarf::DW_FORM_data1, Four);
   DIEInteger Five(5);
-  Int.addValue(dwarf::DW_AT_encoding, dwarf::DW_FORM_data1, &Five);
+  Int.addValue(Alloc, dwarf::DW_AT_encoding, dwarf::DW_FORM_data1, Five);
 
   DIEEntry IntRef(Int);
 
-  auto Member = make_unique<DIE>(dwarf::DW_TAG_member);
-  DIEString MemberStr(&Four, "member");
-  Member->addValue(dwarf::DW_AT_name, dwarf::DW_FORM_strp, &MemberStr);
+  auto Member = DIE::get(Alloc, dwarf::DW_TAG_member);
+  DIEString MemberStr = getString("member");
+  Member->addValue(Alloc, dwarf::DW_AT_name, dwarf::DW_FORM_strp, MemberStr);
   DIEInteger Zero(0);
-  Member->addValue(dwarf::DW_AT_data_member_location, dwarf::DW_FORM_data1,
-                   &Zero);
-  Member->addValue(dwarf::DW_AT_type, dwarf::DW_FORM_ref4, &IntRef);
+  Member->addValue(Alloc, dwarf::DW_AT_data_member_location,
+                   dwarf::DW_FORM_data1, Zero);
+  Member->addValue(Alloc, dwarf::DW_AT_type, dwarf::DW_FORM_ref4, IntRef);
 
   Unnamed.addChild(std::move(Member));
 
@@ -114,37 +133,37 @@ TEST(DIEHashTest, TypeWithMember) {
 }
 
 // struct foo { int mem1, mem2; };
-TEST(DIEHashTest, ReusedType) {
-  DIE Unnamed(dwarf::DW_TAG_structure_type);
+TEST_F(DIEHashTest, ReusedType) {
+  DIE &Unnamed = *DIE::get(Alloc, dwarf::DW_TAG_structure_type);
   DIEInteger Eight(8);
-  Unnamed.addValue(dwarf::DW_AT_byte_size, dwarf::DW_FORM_data1, &Eight);
+  Unnamed.addValue(Alloc, dwarf::DW_AT_byte_size, dwarf::DW_FORM_data1, Eight);
 
   DIEInteger Four(4);
-  DIE Int(dwarf::DW_TAG_base_type);
-  DIEString IntStr(&Four, "int");
-  Int.addValue(dwarf::DW_AT_name, dwarf::DW_FORM_strp, &IntStr);
-  Int.addValue(dwarf::DW_AT_byte_size, dwarf::DW_FORM_data1, &Four);
+  DIE &Int = *DIE::get(Alloc, dwarf::DW_TAG_base_type);
+  DIEString IntStr = getString("int");
+  Int.addValue(Alloc, dwarf::DW_AT_name, dwarf::DW_FORM_strp, IntStr);
+  Int.addValue(Alloc, dwarf::DW_AT_byte_size, dwarf::DW_FORM_data1, Four);
   DIEInteger Five(5);
-  Int.addValue(dwarf::DW_AT_encoding, dwarf::DW_FORM_data1, &Five);
+  Int.addValue(Alloc, dwarf::DW_AT_encoding, dwarf::DW_FORM_data1, Five);
 
   DIEEntry IntRef(Int);
 
-  auto Mem1 = make_unique<DIE>(dwarf::DW_TAG_member);
-  DIEString Mem1Str(&Four, "mem1");
-  Mem1->addValue(dwarf::DW_AT_name, dwarf::DW_FORM_strp, &Mem1Str);
+  auto Mem1 = DIE::get(Alloc, dwarf::DW_TAG_member);
+  DIEString Mem1Str = getString("mem1");
+  Mem1->addValue(Alloc, dwarf::DW_AT_name, dwarf::DW_FORM_strp, Mem1Str);
   DIEInteger Zero(0);
-  Mem1->addValue(dwarf::DW_AT_data_member_location, dwarf::DW_FORM_data1,
-                 &Zero);
-  Mem1->addValue(dwarf::DW_AT_type, dwarf::DW_FORM_ref4, &IntRef);
+  Mem1->addValue(Alloc, dwarf::DW_AT_data_member_location, dwarf::DW_FORM_data1,
+                 Zero);
+  Mem1->addValue(Alloc, dwarf::DW_AT_type, dwarf::DW_FORM_ref4, IntRef);
 
   Unnamed.addChild(std::move(Mem1));
 
-  auto Mem2 = make_unique<DIE>(dwarf::DW_TAG_member);
-  DIEString Mem2Str(&Four, "mem2");
-  Mem2->addValue(dwarf::DW_AT_name, dwarf::DW_FORM_strp, &Mem2Str);
-  Mem2->addValue(dwarf::DW_AT_data_member_location, dwarf::DW_FORM_data1,
-                 &Four);
-  Mem2->addValue(dwarf::DW_AT_type, dwarf::DW_FORM_ref4, &IntRef);
+  auto Mem2 = DIE::get(Alloc, dwarf::DW_TAG_member);
+  DIEString Mem2Str = getString("mem2");
+  Mem2->addValue(Alloc, dwarf::DW_AT_name, dwarf::DW_FORM_strp, Mem2Str);
+  Mem2->addValue(Alloc, dwarf::DW_AT_data_member_location, dwarf::DW_FORM_data1,
+                 Four);
+  Mem2->addValue(Alloc, dwarf::DW_AT_type, dwarf::DW_FORM_ref4, IntRef);
 
   Unnamed.addChild(std::move(Mem2));
 
@@ -154,18 +173,18 @@ TEST(DIEHashTest, ReusedType) {
 }
 
 // struct foo { static foo f; };
-TEST(DIEHashTest, RecursiveType) {
-  DIE Foo(dwarf::DW_TAG_structure_type);
+TEST_F(DIEHashTest, RecursiveType) {
+  DIE &Foo = *DIE::get(Alloc, dwarf::DW_TAG_structure_type);
   DIEInteger One(1);
-  Foo.addValue(dwarf::DW_AT_byte_size, dwarf::DW_FORM_data1, &One);
-  DIEString FooStr(&One, "foo");
-  Foo.addValue(dwarf::DW_AT_name, dwarf::DW_FORM_strp, &FooStr);
+  Foo.addValue(Alloc, dwarf::DW_AT_byte_size, dwarf::DW_FORM_data1, One);
+  DIEString FooStr = getString("foo");
+  Foo.addValue(Alloc, dwarf::DW_AT_name, dwarf::DW_FORM_strp, FooStr);
 
-  auto Mem = make_unique<DIE>(dwarf::DW_TAG_member);
-  DIEString MemStr(&One, "mem");
-  Mem->addValue(dwarf::DW_AT_name, dwarf::DW_FORM_strp, &MemStr);
+  auto Mem = DIE::get(Alloc, dwarf::DW_TAG_member);
+  DIEString MemStr = getString("mem");
+  Mem->addValue(Alloc, dwarf::DW_AT_name, dwarf::DW_FORM_strp, MemStr);
   DIEEntry FooRef(Foo);
-  Mem->addValue(dwarf::DW_AT_type, dwarf::DW_FORM_ref4, &FooRef);
+  Mem->addValue(Alloc, dwarf::DW_AT_type, dwarf::DW_FORM_ref4, FooRef);
   // DW_AT_external and DW_AT_declaration are ignored anyway, so skip them.
 
   Foo.addChild(std::move(Mem));
@@ -176,26 +195,27 @@ TEST(DIEHashTest, RecursiveType) {
 }
 
 // struct foo { foo *mem; };
-TEST(DIEHashTest, Pointer) {
-  DIE Foo(dwarf::DW_TAG_structure_type);
+TEST_F(DIEHashTest, Pointer) {
+  DIE &Foo = *DIE::get(Alloc, dwarf::DW_TAG_structure_type);
   DIEInteger Eight(8);
-  Foo.addValue(dwarf::DW_AT_byte_size, dwarf::DW_FORM_data1, &Eight);
-  DIEString FooStr(&Eight, "foo");
-  Foo.addValue(dwarf::DW_AT_name, dwarf::DW_FORM_strp, &FooStr);
+  Foo.addValue(Alloc, dwarf::DW_AT_byte_size, dwarf::DW_FORM_data1, Eight);
+  DIEString FooStr = getString("foo");
+  Foo.addValue(Alloc, dwarf::DW_AT_name, dwarf::DW_FORM_strp, FooStr);
 
-  auto Mem = make_unique<DIE>(dwarf::DW_TAG_member);
-  DIEString MemStr(&Eight, "mem");
-  Mem->addValue(dwarf::DW_AT_name, dwarf::DW_FORM_strp, &MemStr);
+  auto Mem = DIE::get(Alloc, dwarf::DW_TAG_member);
+  DIEString MemStr = getString("mem");
+  Mem->addValue(Alloc, dwarf::DW_AT_name, dwarf::DW_FORM_strp, MemStr);
   DIEInteger Zero(0);
-  Mem->addValue(dwarf::DW_AT_data_member_location, dwarf::DW_FORM_data1, &Zero);
+  Mem->addValue(Alloc, dwarf::DW_AT_data_member_location, dwarf::DW_FORM_data1,
+                Zero);
 
-  DIE FooPtr(dwarf::DW_TAG_pointer_type);
-  FooPtr.addValue(dwarf::DW_AT_byte_size, dwarf::DW_FORM_data1, &Eight);
+  DIE &FooPtr = *DIE::get(Alloc, dwarf::DW_TAG_pointer_type);
+  FooPtr.addValue(Alloc, dwarf::DW_AT_byte_size, dwarf::DW_FORM_data1, Eight);
   DIEEntry FooRef(Foo);
-  FooPtr.addValue(dwarf::DW_AT_type, dwarf::DW_FORM_ref4, &FooRef);
+  FooPtr.addValue(Alloc, dwarf::DW_AT_type, dwarf::DW_FORM_ref4, FooRef);
 
   DIEEntry FooPtrRef(FooPtr);
-  Mem->addValue(dwarf::DW_AT_type, dwarf::DW_FORM_ref4, &FooPtrRef);
+  Mem->addValue(Alloc, dwarf::DW_AT_type, dwarf::DW_FORM_ref4, FooPtrRef);
 
   Foo.addChild(std::move(Mem));
 
@@ -205,30 +225,32 @@ TEST(DIEHashTest, Pointer) {
 }
 
 // struct foo { foo &mem; };
-TEST(DIEHashTest, Reference) {
-  DIE Foo(dwarf::DW_TAG_structure_type);
+TEST_F(DIEHashTest, Reference) {
+  DIE &Foo = *DIE::get(Alloc, dwarf::DW_TAG_structure_type);
   DIEInteger Eight(8);
-  Foo.addValue(dwarf::DW_AT_byte_size, dwarf::DW_FORM_data1, &Eight);
-  DIEString FooStr(&Eight, "foo");
-  Foo.addValue(dwarf::DW_AT_name, dwarf::DW_FORM_strp, &FooStr);
+  Foo.addValue(Alloc, dwarf::DW_AT_byte_size, dwarf::DW_FORM_data1, Eight);
+  DIEString FooStr = getString("foo");
+  Foo.addValue(Alloc, dwarf::DW_AT_name, dwarf::DW_FORM_strp, FooStr);
 
-  auto Mem = make_unique<DIE>(dwarf::DW_TAG_member);
-  DIEString MemStr(&Eight, "mem");
-  Mem->addValue(dwarf::DW_AT_name, dwarf::DW_FORM_strp, &MemStr);
+  auto Mem = DIE::get(Alloc, dwarf::DW_TAG_member);
+  DIEString MemStr = getString("mem");
+  Mem->addValue(Alloc, dwarf::DW_AT_name, dwarf::DW_FORM_strp, MemStr);
   DIEInteger Zero(0);
-  Mem->addValue(dwarf::DW_AT_data_member_location, dwarf::DW_FORM_data1, &Zero);
+  Mem->addValue(Alloc, dwarf::DW_AT_data_member_location, dwarf::DW_FORM_data1,
+                Zero);
 
-  DIE FooRef(dwarf::DW_TAG_reference_type);
-  FooRef.addValue(dwarf::DW_AT_byte_size, dwarf::DW_FORM_data1, &Eight);
+  DIE &FooRef = *DIE::get(Alloc, dwarf::DW_TAG_reference_type);
+  FooRef.addValue(Alloc, dwarf::DW_AT_byte_size, dwarf::DW_FORM_data1, Eight);
   DIEEntry FooEntry(Foo);
-  FooRef.addValue(dwarf::DW_AT_type, dwarf::DW_FORM_ref4, &FooEntry);
+  FooRef.addValue(Alloc, dwarf::DW_AT_type, dwarf::DW_FORM_ref4, FooEntry);
 
-  DIE FooRefConst(dwarf::DW_TAG_const_type);
+  DIE &FooRefConst = *DIE::get(Alloc, dwarf::DW_TAG_const_type);
   DIEEntry FooRefRef(FooRef);
-  FooRefConst.addValue(dwarf::DW_AT_type, dwarf::DW_FORM_ref4, &FooRefRef);
+  FooRefConst.addValue(Alloc, dwarf::DW_AT_type, dwarf::DW_FORM_ref4,
+                       FooRefRef);
 
   DIEEntry FooRefConstRef(FooRefConst);
-  Mem->addValue(dwarf::DW_AT_type, dwarf::DW_FORM_ref4, &FooRefConstRef);
+  Mem->addValue(Alloc, dwarf::DW_AT_type, dwarf::DW_FORM_ref4, FooRefConstRef);
 
   Foo.addChild(std::move(Mem));
 
@@ -238,30 +260,32 @@ TEST(DIEHashTest, Reference) {
 }
 
 // struct foo { foo &&mem; };
-TEST(DIEHashTest, RValueReference) {
-  DIE Foo(dwarf::DW_TAG_structure_type);
+TEST_F(DIEHashTest, RValueReference) {
+  DIE &Foo = *DIE::get(Alloc, dwarf::DW_TAG_structure_type);
   DIEInteger Eight(8);
-  Foo.addValue(dwarf::DW_AT_byte_size, dwarf::DW_FORM_data1, &Eight);
-  DIEString FooStr(&Eight, "foo");
-  Foo.addValue(dwarf::DW_AT_name, dwarf::DW_FORM_strp, &FooStr);
+  Foo.addValue(Alloc, dwarf::DW_AT_byte_size, dwarf::DW_FORM_data1, Eight);
+  DIEString FooStr = getString("foo");
+  Foo.addValue(Alloc, dwarf::DW_AT_name, dwarf::DW_FORM_strp, FooStr);
 
-  auto Mem = make_unique<DIE>(dwarf::DW_TAG_member);
-  DIEString MemStr(&Eight, "mem");
-  Mem->addValue(dwarf::DW_AT_name, dwarf::DW_FORM_strp, &MemStr);
+  auto Mem = DIE::get(Alloc, dwarf::DW_TAG_member);
+  DIEString MemStr = getString("mem");
+  Mem->addValue(Alloc, dwarf::DW_AT_name, dwarf::DW_FORM_strp, MemStr);
   DIEInteger Zero(0);
-  Mem->addValue(dwarf::DW_AT_data_member_location, dwarf::DW_FORM_data1, &Zero);
+  Mem->addValue(Alloc, dwarf::DW_AT_data_member_location, dwarf::DW_FORM_data1,
+                Zero);
 
-  DIE FooRef(dwarf::DW_TAG_rvalue_reference_type);
-  FooRef.addValue(dwarf::DW_AT_byte_size, dwarf::DW_FORM_data1, &Eight);
+  DIE &FooRef = *DIE::get(Alloc, dwarf::DW_TAG_rvalue_reference_type);
+  FooRef.addValue(Alloc, dwarf::DW_AT_byte_size, dwarf::DW_FORM_data1, Eight);
   DIEEntry FooEntry(Foo);
-  FooRef.addValue(dwarf::DW_AT_type, dwarf::DW_FORM_ref4, &FooEntry);
+  FooRef.addValue(Alloc, dwarf::DW_AT_type, dwarf::DW_FORM_ref4, FooEntry);
 
-  DIE FooRefConst(dwarf::DW_TAG_const_type);
+  DIE &FooRefConst = *DIE::get(Alloc, dwarf::DW_TAG_const_type);
   DIEEntry FooRefRef(FooRef);
-  FooRefConst.addValue(dwarf::DW_AT_type, dwarf::DW_FORM_ref4, &FooRefRef);
+  FooRefConst.addValue(Alloc, dwarf::DW_AT_type, dwarf::DW_FORM_ref4,
+                       FooRefRef);
 
   DIEEntry FooRefConstRef(FooRefConst);
-  Mem->addValue(dwarf::DW_AT_type, dwarf::DW_FORM_ref4, &FooRefConstRef);
+  Mem->addValue(Alloc, dwarf::DW_AT_type, dwarf::DW_FORM_ref4, FooRefConstRef);
 
   Foo.addChild(std::move(Mem));
 
@@ -271,27 +295,28 @@ TEST(DIEHashTest, RValueReference) {
 }
 
 // struct foo { foo foo::*mem; };
-TEST(DIEHashTest, PtrToMember) {
-  DIE Foo(dwarf::DW_TAG_structure_type);
+TEST_F(DIEHashTest, PtrToMember) {
+  DIE &Foo = *DIE::get(Alloc, dwarf::DW_TAG_structure_type);
   DIEInteger Eight(8);
-  Foo.addValue(dwarf::DW_AT_byte_size, dwarf::DW_FORM_data1, &Eight);
-  DIEString FooStr(&Eight, "foo");
-  Foo.addValue(dwarf::DW_AT_name, dwarf::DW_FORM_strp, &FooStr);
+  Foo.addValue(Alloc, dwarf::DW_AT_byte_size, dwarf::DW_FORM_data1, Eight);
+  DIEString FooStr = getString("foo");
+  Foo.addValue(Alloc, dwarf::DW_AT_name, dwarf::DW_FORM_strp, FooStr);
 
-  auto Mem = make_unique<DIE>(dwarf::DW_TAG_member);
-  DIEString MemStr(&Eight, "mem");
-  Mem->addValue(dwarf::DW_AT_name, dwarf::DW_FORM_strp, &MemStr);
+  auto Mem = DIE::get(Alloc, dwarf::DW_TAG_member);
+  DIEString MemStr = getString("mem");
+  Mem->addValue(Alloc, dwarf::DW_AT_name, dwarf::DW_FORM_strp, MemStr);
   DIEInteger Zero(0);
-  Mem->addValue(dwarf::DW_AT_data_member_location, dwarf::DW_FORM_data1, &Zero);
+  Mem->addValue(Alloc, dwarf::DW_AT_data_member_location, dwarf::DW_FORM_data1,
+                Zero);
 
-  DIE PtrToFooMem(dwarf::DW_TAG_ptr_to_member_type);
+  DIE &PtrToFooMem = *DIE::get(Alloc, dwarf::DW_TAG_ptr_to_member_type);
   DIEEntry FooEntry(Foo);
-  PtrToFooMem.addValue(dwarf::DW_AT_type, dwarf::DW_FORM_ref4, &FooEntry);
-  PtrToFooMem.addValue(dwarf::DW_AT_containing_type, dwarf::DW_FORM_ref4,
-                       &FooEntry);
+  PtrToFooMem.addValue(Alloc, dwarf::DW_AT_type, dwarf::DW_FORM_ref4, FooEntry);
+  PtrToFooMem.addValue(Alloc, dwarf::DW_AT_containing_type, dwarf::DW_FORM_ref4,
+                       FooEntry);
 
   DIEEntry PtrToFooMemRef(PtrToFooMem);
-  Mem->addValue(dwarf::DW_AT_type, dwarf::DW_FORM_ref4, &PtrToFooMemRef);
+  Mem->addValue(Alloc, dwarf::DW_AT_type, dwarf::DW_FORM_ref4, PtrToFooMemRef);
 
   Foo.addChild(std::move(Mem));
 
@@ -305,37 +330,40 @@ TEST(DIEHashTest, PtrToMember) {
 //
 //   struct bar; // { };
 //   struct foo { bar foo::*mem; };
-TEST(DIEHashTest, PtrToMemberDeclDefMatch) {
+TEST_F(DIEHashTest, PtrToMemberDeclDefMatch) {
   DIEInteger Zero(0);
   DIEInteger One(1);
   DIEInteger Eight(8);
-  DIEString FooStr(&Eight, "foo");
-  DIEString BarStr(&Eight, "bar");
-  DIEString MemStr(&Eight, "mem");
+  DIEString FooStr = getString("foo");
+  DIEString BarStr = getString("bar");
+  DIEString MemStr = getString("mem");
   uint64_t MD5ResDecl;
   {
-    DIE Bar(dwarf::DW_TAG_structure_type);
-    Bar.addValue(dwarf::DW_AT_name, dwarf::DW_FORM_strp, &BarStr);
-    Bar.addValue(dwarf::DW_AT_declaration, dwarf::DW_FORM_flag_present, &One);
+    DIE &Bar = *DIE::get(Alloc, dwarf::DW_TAG_structure_type);
+    Bar.addValue(Alloc, dwarf::DW_AT_name, dwarf::DW_FORM_strp, BarStr);
+    Bar.addValue(Alloc, dwarf::DW_AT_declaration, dwarf::DW_FORM_flag_present,
+                 One);
 
-    DIE Foo(dwarf::DW_TAG_structure_type);
-    Foo.addValue(dwarf::DW_AT_byte_size, dwarf::DW_FORM_data1, &Eight);
-    Foo.addValue(dwarf::DW_AT_name, dwarf::DW_FORM_strp, &FooStr);
+    DIE &Foo = *DIE::get(Alloc, dwarf::DW_TAG_structure_type);
+    Foo.addValue(Alloc, dwarf::DW_AT_byte_size, dwarf::DW_FORM_data1, Eight);
+    Foo.addValue(Alloc, dwarf::DW_AT_name, dwarf::DW_FORM_strp, FooStr);
 
-    auto Mem = make_unique<DIE>(dwarf::DW_TAG_member);
-    Mem->addValue(dwarf::DW_AT_name, dwarf::DW_FORM_strp, &MemStr);
-    Mem->addValue(dwarf::DW_AT_data_member_location, dwarf::DW_FORM_data1,
-                  &Zero);
+    auto Mem = DIE::get(Alloc, dwarf::DW_TAG_member);
+    Mem->addValue(Alloc, dwarf::DW_AT_name, dwarf::DW_FORM_strp, MemStr);
+    Mem->addValue(Alloc, dwarf::DW_AT_data_member_location,
+                  dwarf::DW_FORM_data1, Zero);
 
-    DIE PtrToFooMem(dwarf::DW_TAG_ptr_to_member_type);
+    DIE &PtrToFooMem = *DIE::get(Alloc, dwarf::DW_TAG_ptr_to_member_type);
     DIEEntry BarEntry(Bar);
-    PtrToFooMem.addValue(dwarf::DW_AT_type, dwarf::DW_FORM_ref4, &BarEntry);
+    PtrToFooMem.addValue(Alloc, dwarf::DW_AT_type, dwarf::DW_FORM_ref4,
+                         BarEntry);
     DIEEntry FooEntry(Foo);
-    PtrToFooMem.addValue(dwarf::DW_AT_containing_type, dwarf::DW_FORM_ref4,
-                         &FooEntry);
+    PtrToFooMem.addValue(Alloc, dwarf::DW_AT_containing_type,
+                         dwarf::DW_FORM_ref4, FooEntry);
 
     DIEEntry PtrToFooMemRef(PtrToFooMem);
-    Mem->addValue(dwarf::DW_AT_type, dwarf::DW_FORM_ref4, &PtrToFooMemRef);
+    Mem->addValue(Alloc, dwarf::DW_AT_type, dwarf::DW_FORM_ref4,
+                  PtrToFooMemRef);
 
     Foo.addChild(std::move(Mem));
 
@@ -343,28 +371,30 @@ TEST(DIEHashTest, PtrToMemberDeclDefMatch) {
   }
   uint64_t MD5ResDef;
   {
-    DIE Bar(dwarf::DW_TAG_structure_type);
-    Bar.addValue(dwarf::DW_AT_name, dwarf::DW_FORM_strp, &BarStr);
-    Bar.addValue(dwarf::DW_AT_byte_size, dwarf::DW_FORM_data1, &One);
+    DIE &Bar = *DIE::get(Alloc, dwarf::DW_TAG_structure_type);
+    Bar.addValue(Alloc, dwarf::DW_AT_name, dwarf::DW_FORM_strp, BarStr);
+    Bar.addValue(Alloc, dwarf::DW_AT_byte_size, dwarf::DW_FORM_data1, One);
 
-    DIE Foo(dwarf::DW_TAG_structure_type);
-    Foo.addValue(dwarf::DW_AT_byte_size, dwarf::DW_FORM_data1, &Eight);
-    Foo.addValue(dwarf::DW_AT_name, dwarf::DW_FORM_strp, &FooStr);
+    DIE &Foo = *DIE::get(Alloc, dwarf::DW_TAG_structure_type);
+    Foo.addValue(Alloc, dwarf::DW_AT_byte_size, dwarf::DW_FORM_data1, Eight);
+    Foo.addValue(Alloc, dwarf::DW_AT_name, dwarf::DW_FORM_strp, FooStr);
 
-    auto Mem = make_unique<DIE>(dwarf::DW_TAG_member);
-    Mem->addValue(dwarf::DW_AT_name, dwarf::DW_FORM_strp, &MemStr);
-    Mem->addValue(dwarf::DW_AT_data_member_location, dwarf::DW_FORM_data1,
-                  &Zero);
+    auto Mem = DIE::get(Alloc, dwarf::DW_TAG_member);
+    Mem->addValue(Alloc, dwarf::DW_AT_name, dwarf::DW_FORM_strp, MemStr);
+    Mem->addValue(Alloc, dwarf::DW_AT_data_member_location,
+                  dwarf::DW_FORM_data1, Zero);
 
-    DIE PtrToFooMem(dwarf::DW_TAG_ptr_to_member_type);
+    DIE &PtrToFooMem = *DIE::get(Alloc, dwarf::DW_TAG_ptr_to_member_type);
     DIEEntry BarEntry(Bar);
-    PtrToFooMem.addValue(dwarf::DW_AT_type, dwarf::DW_FORM_ref4, &BarEntry);
+    PtrToFooMem.addValue(Alloc, dwarf::DW_AT_type, dwarf::DW_FORM_ref4,
+                         BarEntry);
     DIEEntry FooEntry(Foo);
-    PtrToFooMem.addValue(dwarf::DW_AT_containing_type, dwarf::DW_FORM_ref4,
-                         &FooEntry);
+    PtrToFooMem.addValue(Alloc, dwarf::DW_AT_containing_type,
+                         dwarf::DW_FORM_ref4, FooEntry);
 
     DIEEntry PtrToFooMemRef(PtrToFooMem);
-    Mem->addValue(dwarf::DW_AT_type, dwarf::DW_FORM_ref4, &PtrToFooMemRef);
+    Mem->addValue(Alloc, dwarf::DW_AT_type, dwarf::DW_FORM_ref4,
+                  PtrToFooMemRef);
 
     Foo.addChild(std::move(Mem));
 
@@ -378,36 +408,39 @@ TEST(DIEHashTest, PtrToMemberDeclDefMatch) {
 //
 //   struct bar; // { };
 //   struct foo { bar bar::*mem; };
-TEST(DIEHashTest, PtrToMemberDeclDefMisMatch) {
+TEST_F(DIEHashTest, PtrToMemberDeclDefMisMatch) {
   DIEInteger Zero(0);
   DIEInteger One(1);
   DIEInteger Eight(8);
-  DIEString FooStr(&Eight, "foo");
-  DIEString BarStr(&Eight, "bar");
-  DIEString MemStr(&Eight, "mem");
+  DIEString FooStr = getString("foo");
+  DIEString BarStr = getString("bar");
+  DIEString MemStr = getString("mem");
   uint64_t MD5ResDecl;
   {
-    DIE Bar(dwarf::DW_TAG_structure_type);
-    Bar.addValue(dwarf::DW_AT_name, dwarf::DW_FORM_strp, &BarStr);
-    Bar.addValue(dwarf::DW_AT_declaration, dwarf::DW_FORM_flag_present, &One);
+    DIE &Bar = *DIE::get(Alloc, dwarf::DW_TAG_structure_type);
+    Bar.addValue(Alloc, dwarf::DW_AT_name, dwarf::DW_FORM_strp, BarStr);
+    Bar.addValue(Alloc, dwarf::DW_AT_declaration, dwarf::DW_FORM_flag_present,
+                 One);
 
-    DIE Foo(dwarf::DW_TAG_structure_type);
-    Foo.addValue(dwarf::DW_AT_byte_size, dwarf::DW_FORM_data1, &Eight);
-    Foo.addValue(dwarf::DW_AT_name, dwarf::DW_FORM_strp, &FooStr);
+    DIE &Foo = *DIE::get(Alloc, dwarf::DW_TAG_structure_type);
+    Foo.addValue(Alloc, dwarf::DW_AT_byte_size, dwarf::DW_FORM_data1, Eight);
+    Foo.addValue(Alloc, dwarf::DW_AT_name, dwarf::DW_FORM_strp, FooStr);
 
-    auto Mem = make_unique<DIE>(dwarf::DW_TAG_member);
-    Mem->addValue(dwarf::DW_AT_name, dwarf::DW_FORM_strp, &MemStr);
-    Mem->addValue(dwarf::DW_AT_data_member_location, dwarf::DW_FORM_data1,
-                  &Zero);
+    auto Mem = DIE::get(Alloc, dwarf::DW_TAG_member);
+    Mem->addValue(Alloc, dwarf::DW_AT_name, dwarf::DW_FORM_strp, MemStr);
+    Mem->addValue(Alloc, dwarf::DW_AT_data_member_location,
+                  dwarf::DW_FORM_data1, Zero);
 
-    DIE PtrToFooMem(dwarf::DW_TAG_ptr_to_member_type);
+    DIE &PtrToFooMem = *DIE::get(Alloc, dwarf::DW_TAG_ptr_to_member_type);
     DIEEntry BarEntry(Bar);
-    PtrToFooMem.addValue(dwarf::DW_AT_type, dwarf::DW_FORM_ref4, &BarEntry);
-    PtrToFooMem.addValue(dwarf::DW_AT_containing_type, dwarf::DW_FORM_ref4,
-                         &BarEntry);
+    PtrToFooMem.addValue(Alloc, dwarf::DW_AT_type, dwarf::DW_FORM_ref4,
+                         BarEntry);
+    PtrToFooMem.addValue(Alloc, dwarf::DW_AT_containing_type,
+                         dwarf::DW_FORM_ref4, BarEntry);
 
     DIEEntry PtrToFooMemRef(PtrToFooMem);
-    Mem->addValue(dwarf::DW_AT_type, dwarf::DW_FORM_ref4, &PtrToFooMemRef);
+    Mem->addValue(Alloc, dwarf::DW_AT_type, dwarf::DW_FORM_ref4,
+                  PtrToFooMemRef);
 
     Foo.addChild(std::move(Mem));
 
@@ -415,27 +448,29 @@ TEST(DIEHashTest, PtrToMemberDeclDefMisMatch) {
   }
   uint64_t MD5ResDef;
   {
-    DIE Bar(dwarf::DW_TAG_structure_type);
-    Bar.addValue(dwarf::DW_AT_name, dwarf::DW_FORM_strp, &BarStr);
-    Bar.addValue(dwarf::DW_AT_byte_size, dwarf::DW_FORM_data1, &One);
+    DIE &Bar = *DIE::get(Alloc, dwarf::DW_TAG_structure_type);
+    Bar.addValue(Alloc, dwarf::DW_AT_name, dwarf::DW_FORM_strp, BarStr);
+    Bar.addValue(Alloc, dwarf::DW_AT_byte_size, dwarf::DW_FORM_data1, One);
 
-    DIE Foo(dwarf::DW_TAG_structure_type);
-    Foo.addValue(dwarf::DW_AT_byte_size, dwarf::DW_FORM_data1, &Eight);
-    Foo.addValue(dwarf::DW_AT_name, dwarf::DW_FORM_strp, &FooStr);
+    DIE &Foo = *DIE::get(Alloc, dwarf::DW_TAG_structure_type);
+    Foo.addValue(Alloc, dwarf::DW_AT_byte_size, dwarf::DW_FORM_data1, Eight);
+    Foo.addValue(Alloc, dwarf::DW_AT_name, dwarf::DW_FORM_strp, FooStr);
 
-    auto Mem = make_unique<DIE>(dwarf::DW_TAG_member);
-    Mem->addValue(dwarf::DW_AT_name, dwarf::DW_FORM_strp, &MemStr);
-    Mem->addValue(dwarf::DW_AT_data_member_location, dwarf::DW_FORM_data1,
-                  &Zero);
+    auto Mem = DIE::get(Alloc, dwarf::DW_TAG_member);
+    Mem->addValue(Alloc, dwarf::DW_AT_name, dwarf::DW_FORM_strp, MemStr);
+    Mem->addValue(Alloc, dwarf::DW_AT_data_member_location,
+                  dwarf::DW_FORM_data1, Zero);
 
-    DIE PtrToFooMem(dwarf::DW_TAG_ptr_to_member_type);
+    DIE &PtrToFooMem = *DIE::get(Alloc, dwarf::DW_TAG_ptr_to_member_type);
     DIEEntry BarEntry(Bar);
-    PtrToFooMem.addValue(dwarf::DW_AT_type, dwarf::DW_FORM_ref4, &BarEntry);
-    PtrToFooMem.addValue(dwarf::DW_AT_containing_type, dwarf::DW_FORM_ref4,
-                         &BarEntry);
+    PtrToFooMem.addValue(Alloc, dwarf::DW_AT_type, dwarf::DW_FORM_ref4,
+                         BarEntry);
+    PtrToFooMem.addValue(Alloc, dwarf::DW_AT_containing_type,
+                         dwarf::DW_FORM_ref4, BarEntry);
 
     DIEEntry PtrToFooMemRef(PtrToFooMem);
-    Mem->addValue(dwarf::DW_AT_type, dwarf::DW_FORM_ref4, &PtrToFooMemRef);
+    Mem->addValue(Alloc, dwarf::DW_AT_type, dwarf::DW_FORM_ref4,
+                  PtrToFooMemRef);
 
     Foo.addChild(std::move(Mem));
 
@@ -451,31 +486,34 @@ TEST(DIEHashTest, PtrToMemberDeclDefMisMatch) {
 
 // struct { } a;
 // struct foo { decltype(a) mem; };
-TEST(DIEHashTest, RefUnnamedType) {
+TEST_F(DIEHashTest, RefUnnamedType) {
   DIEInteger Zero(0);
   DIEInteger One(1);
   DIEInteger Eight(8);
-  DIEString FooStr(&Zero, "foo");
-  DIEString MemStr(&Zero, "mem");
+  DIEString FooStr = getString("foo");
+  DIEString MemStr = getString("mem");
 
-  DIE Unnamed(dwarf::DW_TAG_structure_type);
-  Unnamed.addValue(dwarf::DW_AT_byte_size, dwarf::DW_FORM_data1, &One);
+  DIE &Unnamed = *DIE::get(Alloc, dwarf::DW_TAG_structure_type);
+  Unnamed.addValue(Alloc, dwarf::DW_AT_byte_size, dwarf::DW_FORM_data1, One);
 
-  DIE Foo(dwarf::DW_TAG_structure_type);
-  Foo.addValue(dwarf::DW_AT_byte_size, dwarf::DW_FORM_data1, &Eight);
-  Foo.addValue(dwarf::DW_AT_name, dwarf::DW_FORM_strp, &FooStr);
+  DIE &Foo = *DIE::get(Alloc, dwarf::DW_TAG_structure_type);
+  Foo.addValue(Alloc, dwarf::DW_AT_byte_size, dwarf::DW_FORM_data1, Eight);
+  Foo.addValue(Alloc, dwarf::DW_AT_name, dwarf::DW_FORM_strp, FooStr);
 
-  auto Mem = make_unique<DIE>(dwarf::DW_TAG_member);
-  Mem->addValue(dwarf::DW_AT_name, dwarf::DW_FORM_strp, &MemStr);
-  Mem->addValue(dwarf::DW_AT_data_member_location, dwarf::DW_FORM_data1, &Zero);
+  auto Mem = DIE::get(Alloc, dwarf::DW_TAG_member);
+  Mem->addValue(Alloc, dwarf::DW_AT_name, dwarf::DW_FORM_strp, MemStr);
+  Mem->addValue(Alloc, dwarf::DW_AT_data_member_location, dwarf::DW_FORM_data1,
+                Zero);
 
-  DIE UnnamedPtr(dwarf::DW_TAG_pointer_type);
-  UnnamedPtr.addValue(dwarf::DW_AT_byte_size, dwarf::DW_FORM_data1, &Eight);
+  DIE &UnnamedPtr = *DIE::get(Alloc, dwarf::DW_TAG_pointer_type);
+  UnnamedPtr.addValue(Alloc, dwarf::DW_AT_byte_size, dwarf::DW_FORM_data1,
+                      Eight);
   DIEEntry UnnamedRef(Unnamed);
-  UnnamedPtr.addValue(dwarf::DW_AT_type, dwarf::DW_FORM_ref4, &UnnamedRef);
+  UnnamedPtr.addValue(Alloc, dwarf::DW_AT_type, dwarf::DW_FORM_ref4,
+                      UnnamedRef);
 
   DIEEntry UnnamedPtrRef(UnnamedPtr);
-  Mem->addValue(dwarf::DW_AT_type, dwarf::DW_FORM_ref4, &UnnamedPtrRef);
+  Mem->addValue(Alloc, dwarf::DW_AT_type, dwarf::DW_FORM_ref4, UnnamedPtrRef);
 
   Foo.addChild(std::move(Mem));
 
@@ -485,15 +523,15 @@ TEST(DIEHashTest, RefUnnamedType) {
 }
 
 // struct { struct foo { }; };
-TEST(DIEHashTest, NestedType) {
-  DIE Unnamed(dwarf::DW_TAG_structure_type);
+TEST_F(DIEHashTest, NestedType) {
+  DIE &Unnamed = *DIE::get(Alloc, dwarf::DW_TAG_structure_type);
   DIEInteger One(1);
-  Unnamed.addValue(dwarf::DW_AT_byte_size, dwarf::DW_FORM_data1, &One);
+  Unnamed.addValue(Alloc, dwarf::DW_AT_byte_size, dwarf::DW_FORM_data1, One);
 
-  auto Foo = make_unique<DIE>(dwarf::DW_TAG_structure_type);
-  DIEString FooStr(&One, "foo");
-  Foo->addValue(dwarf::DW_AT_name, dwarf::DW_FORM_strp, &FooStr);
-  Foo->addValue(dwarf::DW_AT_byte_size, dwarf::DW_FORM_data1, &One);
+  auto Foo = DIE::get(Alloc, dwarf::DW_TAG_structure_type);
+  DIEString FooStr = getString("foo");
+  Foo->addValue(Alloc, dwarf::DW_AT_name, dwarf::DW_FORM_strp, FooStr);
+  Foo->addValue(Alloc, dwarf::DW_AT_byte_size, dwarf::DW_FORM_data1, One);
 
   Unnamed.addChild(std::move(Foo));
 
@@ -504,14 +542,14 @@ TEST(DIEHashTest, NestedType) {
 }
 
 // struct { static void func(); };
-TEST(DIEHashTest, MemberFunc) {
-  DIE Unnamed(dwarf::DW_TAG_structure_type);
+TEST_F(DIEHashTest, MemberFunc) {
+  DIE &Unnamed = *DIE::get(Alloc, dwarf::DW_TAG_structure_type);
   DIEInteger One(1);
-  Unnamed.addValue(dwarf::DW_AT_byte_size, dwarf::DW_FORM_data1, &One);
+  Unnamed.addValue(Alloc, dwarf::DW_AT_byte_size, dwarf::DW_FORM_data1, One);
 
-  auto Func = make_unique<DIE>(dwarf::DW_TAG_subprogram);
-  DIEString FuncStr(&One, "func");
-  Func->addValue(dwarf::DW_AT_name, dwarf::DW_FORM_strp, &FuncStr);
+  auto Func = DIE::get(Alloc, dwarf::DW_TAG_subprogram);
+  DIEString FuncStr = getString("func");
+  Func->addValue(Alloc, dwarf::DW_AT_name, dwarf::DW_FORM_strp, FuncStr);
 
   Unnamed.addChild(std::move(Func));
 
@@ -524,25 +562,28 @@ TEST(DIEHashTest, MemberFunc) {
 // struct A {
 //   static void func();
 // };
-TEST(DIEHashTest, MemberFuncFlag) {
-  DIE A(dwarf::DW_TAG_structure_type);
+TEST_F(DIEHashTest, MemberFuncFlag) {
+  DIE &A = *DIE::get(Alloc, dwarf::DW_TAG_structure_type);
   DIEInteger One(1);
-  DIEString AStr(&One, "A");
-  A.addValue(dwarf::DW_AT_name, dwarf::DW_FORM_strp, &AStr);
-  A.addValue(dwarf::DW_AT_byte_size, dwarf::DW_FORM_data1, &One);
-  A.addValue(dwarf::DW_AT_decl_file, dwarf::DW_FORM_data1, &One);
-  A.addValue(dwarf::DW_AT_decl_line, dwarf::DW_FORM_data1, &One);
-
-  auto Func = make_unique<DIE>(dwarf::DW_TAG_subprogram);
-  DIEString FuncStr(&One, "func");
-  DIEString FuncLinkage(&One, "_ZN1A4funcEv");
+  DIEString AStr = getString("A");
+  A.addValue(Alloc, dwarf::DW_AT_name, dwarf::DW_FORM_strp, AStr);
+  A.addValue(Alloc, dwarf::DW_AT_byte_size, dwarf::DW_FORM_data1, One);
+  A.addValue(Alloc, dwarf::DW_AT_decl_file, dwarf::DW_FORM_data1, One);
+  A.addValue(Alloc, dwarf::DW_AT_decl_line, dwarf::DW_FORM_data1, One);
+
+  auto Func = DIE::get(Alloc, dwarf::DW_TAG_subprogram);
+  DIEString FuncStr = getString("func");
+  DIEString FuncLinkage = getString("_ZN1A4funcEv");
   DIEInteger Two(2);
-  Func->addValue(dwarf::DW_AT_external, dwarf::DW_FORM_flag_present, &One);
-  Func->addValue(dwarf::DW_AT_name, dwarf::DW_FORM_strp, &FuncStr);
-  Func->addValue(dwarf::DW_AT_decl_file, dwarf::DW_FORM_data1, &One);
-  Func->addValue(dwarf::DW_AT_decl_line, dwarf::DW_FORM_data1, &Two);
-  Func->addValue(dwarf::DW_AT_linkage_name, dwarf::DW_FORM_strp, &FuncLinkage);
-  Func->addValue(dwarf::DW_AT_declaration, dwarf::DW_FORM_flag_present, &One);
+  Func->addValue(Alloc, dwarf::DW_AT_external, dwarf::DW_FORM_flag_present,
+                 One);
+  Func->addValue(Alloc, dwarf::DW_AT_name, dwarf::DW_FORM_strp, FuncStr);
+  Func->addValue(Alloc, dwarf::DW_AT_decl_file, dwarf::DW_FORM_data1, One);
+  Func->addValue(Alloc, dwarf::DW_AT_decl_line, dwarf::DW_FORM_data1, Two);
+  Func->addValue(Alloc, dwarf::DW_AT_linkage_name, dwarf::DW_FORM_strp,
+                 FuncLinkage);
+  Func->addValue(Alloc, dwarf::DW_AT_declaration, dwarf::DW_FORM_flag_present,
+                 One);
 
   A.addChild(std::move(Func));
 
@@ -557,39 +598,40 @@ TEST(DIEHashTest, MemberFuncFlag) {
 //   const static int PI = -3;
 // };
 // A a;
-TEST(DIEHashTest, MemberSdata) {
-  DIE A(dwarf::DW_TAG_structure_type);
+TEST_F(DIEHashTest, MemberSdata) {
+  DIE &A = *DIE::get(Alloc, dwarf::DW_TAG_structure_type);
   DIEInteger One(1);
-  DIEString AStr(&One, "A");
-  A.addValue(dwarf::DW_AT_name, dwarf::DW_FORM_strp, &AStr);
-  A.addValue(dwarf::DW_AT_byte_size, dwarf::DW_FORM_data1, &One);
-  A.addValue(dwarf::DW_AT_decl_file, dwarf::DW_FORM_data1, &One);
-  A.addValue(dwarf::DW_AT_decl_line, dwarf::DW_FORM_data1, &One);
+  DIEString AStr = getString("A");
+  A.addValue(Alloc, dwarf::DW_AT_name, dwarf::DW_FORM_strp, AStr);
+  A.addValue(Alloc, dwarf::DW_AT_byte_size, dwarf::DW_FORM_data1, One);
+  A.addValue(Alloc, dwarf::DW_AT_decl_file, dwarf::DW_FORM_data1, One);
+  A.addValue(Alloc, dwarf::DW_AT_decl_line, dwarf::DW_FORM_data1, One);
 
   DIEInteger Four(4);
   DIEInteger Five(5);
-  DIEString FStr(&One, "int");
-  DIE IntTyDIE(dwarf::DW_TAG_base_type);
-  IntTyDIE.addValue(dwarf::DW_AT_byte_size, dwarf::DW_FORM_data1, &Four);
-  IntTyDIE.addValue(dwarf::DW_AT_encoding, dwarf::DW_FORM_data1, &Five);
-  IntTyDIE.addValue(dwarf::DW_AT_name, dwarf::DW_FORM_strp, &FStr);
+  DIEString FStr = getString("int");
+  DIE &IntTyDIE = *DIE::get(Alloc, dwarf::DW_TAG_base_type);
+  IntTyDIE.addValue(Alloc, dwarf::DW_AT_byte_size, dwarf::DW_FORM_data1, Four);
+  IntTyDIE.addValue(Alloc, dwarf::DW_AT_encoding, dwarf::DW_FORM_data1, Five);
+  IntTyDIE.addValue(Alloc, dwarf::DW_AT_name, dwarf::DW_FORM_strp, FStr);
 
   DIEEntry IntTy(IntTyDIE);
-  auto PITyDIE = make_unique<DIE>(dwarf::DW_TAG_const_type);
-  PITyDIE->addValue(dwarf::DW_AT_type, dwarf::DW_FORM_ref4, &IntTy);
+  auto PITyDIE = DIE::get(Alloc, dwarf::DW_TAG_const_type);
+  PITyDIE->addValue(Alloc, dwarf::DW_AT_type, dwarf::DW_FORM_ref4, IntTy);
 
   DIEEntry PITy(*PITyDIE);
-  auto PI = make_unique<DIE>(dwarf::DW_TAG_member);
-  DIEString PIStr(&One, "PI");
+  auto PI = DIE::get(Alloc, dwarf::DW_TAG_member);
+  DIEString PIStr = getString("PI");
   DIEInteger Two(2);
   DIEInteger NegThree(-3);
-  PI->addValue(dwarf::DW_AT_name, dwarf::DW_FORM_strp, &PIStr);
-  PI->addValue(dwarf::DW_AT_decl_file, dwarf::DW_FORM_data1, &One);
-  PI->addValue(dwarf::DW_AT_decl_line, dwarf::DW_FORM_data1, &Two);
-  PI->addValue(dwarf::DW_AT_type, dwarf::DW_FORM_ref4, &PITy);
-  PI->addValue(dwarf::DW_AT_external, dwarf::DW_FORM_flag_present, &One);
-  PI->addValue(dwarf::DW_AT_declaration, dwarf::DW_FORM_flag_present, &One);
-  PI->addValue(dwarf::DW_AT_const_value, dwarf::DW_FORM_sdata, &NegThree);
+  PI->addValue(Alloc, dwarf::DW_AT_name, dwarf::DW_FORM_strp, PIStr);
+  PI->addValue(Alloc, dwarf::DW_AT_decl_file, dwarf::DW_FORM_data1, One);
+  PI->addValue(Alloc, dwarf::DW_AT_decl_line, dwarf::DW_FORM_data1, Two);
+  PI->addValue(Alloc, dwarf::DW_AT_type, dwarf::DW_FORM_ref4, PITy);
+  PI->addValue(Alloc, dwarf::DW_AT_external, dwarf::DW_FORM_flag_present, One);
+  PI->addValue(Alloc, dwarf::DW_AT_declaration, dwarf::DW_FORM_flag_present,
+               One);
+  PI->addValue(Alloc, dwarf::DW_AT_const_value, dwarf::DW_FORM_sdata, NegThree);
 
   A.addChild(std::move(PI));
 
@@ -602,36 +644,38 @@ TEST(DIEHashTest, MemberSdata) {
 //   const static float PI = 3.14;
 // };
 // A a;
-TEST(DIEHashTest, MemberBlock) {
-  DIE A(dwarf::DW_TAG_structure_type);
+TEST_F(DIEHashTest, MemberBlock) {
+  DIE &A = *DIE::get(Alloc, dwarf::DW_TAG_structure_type);
   DIEInteger One(1);
-  DIEString AStr(&One, "A");
-  A.addValue(dwarf::DW_AT_name, dwarf::DW_FORM_strp, &AStr);
-  A.addValue(dwarf::DW_AT_byte_size, dwarf::DW_FORM_data1, &One);
-  A.addValue(dwarf::DW_AT_decl_file, dwarf::DW_FORM_data1, &One);
-  A.addValue(dwarf::DW_AT_decl_line, dwarf::DW_FORM_data1, &One);
+  DIEString AStr = getString("A");
+  A.addValue(Alloc, dwarf::DW_AT_name, dwarf::DW_FORM_strp, AStr);
+  A.addValue(Alloc, dwarf::DW_AT_byte_size, dwarf::DW_FORM_data1, One);
+  A.addValue(Alloc, dwarf::DW_AT_decl_file, dwarf::DW_FORM_data1, One);
+  A.addValue(Alloc, dwarf::DW_AT_decl_line, dwarf::DW_FORM_data1, One);
 
   DIEInteger Four(4);
-  DIEString FStr(&One, "float");
-  auto FloatTyDIE = make_unique<DIE>(dwarf::DW_TAG_base_type);
-  FloatTyDIE->addValue(dwarf::DW_AT_byte_size, dwarf::DW_FORM_data1, &Four);
-  FloatTyDIE->addValue(dwarf::DW_AT_encoding, dwarf::DW_FORM_data1, &Four);
-  FloatTyDIE->addValue(dwarf::DW_AT_name, dwarf::DW_FORM_strp, &FStr);
-
+  DIEString FStr = getString("float");
+  auto FloatTyDIE = DIE::get(Alloc, dwarf::DW_TAG_base_type);
+  FloatTyDIE->addValue(Alloc, dwarf::DW_AT_byte_size, dwarf::DW_FORM_data1,
+                       Four);
+  FloatTyDIE->addValue(Alloc, dwarf::DW_AT_encoding, dwarf::DW_FORM_data1,
+                       Four);
+  FloatTyDIE->addValue(Alloc, dwarf::DW_AT_name, dwarf::DW_FORM_strp, FStr);
   DIEEntry FloatTy(*FloatTyDIE);
-  auto PITyDIE = make_unique<DIE>(dwarf::DW_TAG_const_type);
-  PITyDIE->addValue(dwarf::DW_AT_type, dwarf::DW_FORM_ref4, &FloatTy);
+  auto PITyDIE = DIE::get(Alloc, dwarf::DW_TAG_const_type);
+  PITyDIE->addValue(Alloc, dwarf::DW_AT_type, dwarf::DW_FORM_ref4, FloatTy);
 
   DIEEntry PITy(*PITyDIE);
-  auto PI = make_unique<DIE>(dwarf::DW_TAG_member);
-  DIEString PIStr(&One, "PI");
+  auto PI = DIE::get(Alloc, dwarf::DW_TAG_member);
+  DIEString PIStr = getString("PI");
   DIEInteger Two(2);
-  PI->addValue(dwarf::DW_AT_name, dwarf::DW_FORM_strp, &PIStr);
-  PI->addValue(dwarf::DW_AT_decl_file, dwarf::DW_FORM_data1, &One);
-  PI->addValue(dwarf::DW_AT_decl_line, dwarf::DW_FORM_data1, &Two);
-  PI->addValue(dwarf::DW_AT_type, dwarf::DW_FORM_ref4, &PITy);
-  PI->addValue(dwarf::DW_AT_external, dwarf::DW_FORM_flag_present, &One);
-  PI->addValue(dwarf::DW_AT_declaration, dwarf::DW_FORM_flag_present, &One);
+  PI->addValue(Alloc, dwarf::DW_AT_name, dwarf::DW_FORM_strp, PIStr);
+  PI->addValue(Alloc, dwarf::DW_AT_decl_file, dwarf::DW_FORM_data1, One);
+  PI->addValue(Alloc, dwarf::DW_AT_decl_line, dwarf::DW_FORM_data1, Two);
+  PI->addValue(Alloc, dwarf::DW_AT_type, dwarf::DW_FORM_ref4, PITy);
+  PI->addValue(Alloc, dwarf::DW_AT_external, dwarf::DW_FORM_flag_present, One);
+  PI->addValue(Alloc, dwarf::DW_AT_declaration, dwarf::DW_FORM_flag_present,
+               One);
 
   DIEBlock PIBlock;
   DIEInteger Blk1(0xc3);
@@ -639,12 +683,13 @@ TEST(DIEHashTest, MemberBlock) {
   DIEInteger Blk3(0x48);
   DIEInteger Blk4(0x40);
 
-  PIBlock.addValue((dwarf::Attribute)0, dwarf::DW_FORM_data1, &Blk1);
-  PIBlock.addValue((dwarf::Attribute)0, dwarf::DW_FORM_data1, &Blk2);
-  PIBlock.addValue((dwarf::Attribute)0, dwarf::DW_FORM_data1, &Blk3);
-  PIBlock.addValue((dwarf::Attribute)0, dwarf::DW_FORM_data1, &Blk4);
+  PIBlock.addValue(Alloc, (dwarf::Attribute)0, dwarf::DW_FORM_data1, Blk1);
+  PIBlock.addValue(Alloc, (dwarf::Attribute)0, dwarf::DW_FORM_data1, Blk2);
+  PIBlock.addValue(Alloc, (dwarf::Attribute)0, dwarf::DW_FORM_data1, Blk3);
+  PIBlock.addValue(Alloc, (dwarf::Attribute)0, dwarf::DW_FORM_data1, Blk4);
 
-  PI->addValue(dwarf::DW_AT_const_value, dwarf::DW_FORM_block1, &PIBlock);
+  PI->addValue(Alloc, dwarf::DW_AT_const_value, dwarf::DW_FORM_block1,
+               &PIBlock);
 
   A.addChild(std::move(PI));
 
diff --git a/unittests/DebugInfo/PDB/PDBApiTest.cpp b/unittests/DebugInfo/PDB/PDBApiTest.cpp
index 629c5f8bc0e..ebd3d7bb6b3 100644
--- a/unittests/DebugInfo/PDB/PDBApiTest.cpp
+++ b/unittests/DebugInfo/PDB/PDBApiTest.cpp
@@ -72,6 +72,16 @@ class MockSession : public IPDBSession {
   getSourceFileById(uint32_t SymbolId) const override {
     return nullptr;
   }
+
+  std::unique_ptr<PDBSymbol>
+  findSymbolByAddress(uint64_t Address, PDB_SymType Type) const override {
+    return nullptr;
+  }
+  std::unique_ptr<IPDBEnumLineNumbers>
+  findLineNumbersByAddress(uint64_t Address, uint32_t Length) const override {
+    return nullptr;
+  }
+
   std::unique_ptr<IPDBEnumSourceFiles> getAllSourceFiles() const override {
     return nullptr;
   }
diff --git a/unittests/ExecutionEngine/MCJIT/MCJITCAPITest.cpp b/unittests/ExecutionEngine/MCJIT/MCJITCAPITest.cpp
index a7c9ae02b38..9d768d97f38 100644
--- a/unittests/ExecutionEngine/MCJIT/MCJITCAPITest.cpp
+++ b/unittests/ExecutionEngine/MCJIT/MCJITCAPITest.cpp
@@ -127,6 +127,8 @@ class MCJITCAPITest : public testing::Test, public MCJITTestAPICommon {
     SupportedArchs.push_back(Triple::aarch64);
     SupportedArchs.push_back(Triple::arm);
     SupportedArchs.push_back(Triple::mips);
+    SupportedArchs.push_back(Triple::mips64);
+    SupportedArchs.push_back(Triple::mips64el);
     SupportedArchs.push_back(Triple::x86);
     SupportedArchs.push_back(Triple::x86_64);
 
@@ -486,3 +488,38 @@ TEST_F(MCJITCAPITest, yield) {
   EXPECT_TRUE(didCallYield);
 }
 
+static int localTestFunc() {
+  return 42;
+}
+
+TEST_F(MCJITCAPITest, addGlobalMapping) {
+  SKIP_UNSUPPORTED_PLATFORM;
+
+  Module = LLVMModuleCreateWithName("testModule");
+  LLVMSetTarget(Module, HostTriple.c_str());
+  LLVMTypeRef FunctionType = LLVMFunctionType(LLVMInt32Type(), NULL, 0, 0);
+  LLVMValueRef MappedFn = LLVMAddFunction(Module, "mapped_fn", FunctionType);
+
+  Function = LLVMAddFunction(Module, "test_fn", FunctionType);
+  LLVMBasicBlockRef Entry = LLVMAppendBasicBlock(Function, "");
+  LLVMBuilderRef Builder = LLVMCreateBuilder();
+  LLVMPositionBuilderAtEnd(Builder, Entry);
+  LLVMValueRef RetVal = LLVMBuildCall(Builder, MappedFn, NULL, 0, "");
+  LLVMBuildRet(Builder, RetVal);
+  LLVMDisposeBuilder(Builder);
+
+  LLVMVerifyModule(Module, LLVMAbortProcessAction, &Error);
+  LLVMDisposeMessage(Error);
+
+  buildMCJITOptions();
+  buildMCJITEngine();
+
+  LLVMAddGlobalMapping(Engine, MappedFn, reinterpret_cast<void*>(&localTestFunc));
+
+  buildAndRunPasses();
+
+  uint64_t raw = LLVMGetFunctionAddress(Engine, "test_fn");
+  int (*usable)() = (int (*)()) raw;
+
+  EXPECT_EQ(42, usable());
+}
diff --git a/unittests/ExecutionEngine/MCJIT/MCJITMultipleModuleTest.cpp b/unittests/ExecutionEngine/MCJIT/MCJITMultipleModuleTest.cpp
index da6e25a3d51..7d52a9acca7 100644
--- a/unittests/ExecutionEngine/MCJIT/MCJITMultipleModuleTest.cpp
+++ b/unittests/ExecutionEngine/MCJIT/MCJITMultipleModuleTest.cpp
@@ -194,14 +194,15 @@ TEST_F(MCJITMultipleModuleTest, two_module_consecutive_call_case) {
 
 
 // Module A { Global Variable GVA, Function FA loads GVA },
-// Module B { Global Variable GVB, Function FB loads GVB },
-// execute FB then FA
+// Module B { Global Variable GVB, Internal Global GVC, Function FB loads GVB },
+// execute FB then FA, also check that the global variables are properly accesible
+// through the ExecutionEngine APIs
 TEST_F(MCJITMultipleModuleTest, two_module_global_variables_case) {
   SKIP_UNSUPPORTED_PLATFORM;
 
   std::unique_ptr<Module> A, B;
   Function *FA, *FB;
-  GlobalVariable *GVA, *GVB;
+  GlobalVariable *GVA, *GVB, *GVC;
   A.reset(createEmptyModule("A"));
   B.reset(createEmptyModule("B"));
 
@@ -213,9 +214,17 @@ TEST_F(MCJITMultipleModuleTest, two_module_global_variables_case) {
   FB = startFunction<int32_t(void)>(B.get(), "FB");
   endFunctionWithRet(FB, Builder.CreateLoad(GVB));
 
+  GVC = insertGlobalInt32(B.get(), "GVC", initialNum);
+  GVC->setLinkage(GlobalValue::InternalLinkage);
+
   createJIT(std::move(A));
   TheJIT->addModule(std::move(B));
 
+  EXPECT_EQ(GVA, TheJIT->FindGlobalVariableNamed("GVA"));
+  EXPECT_EQ(GVB, TheJIT->FindGlobalVariableNamed("GVB"));
+  EXPECT_EQ(GVC, TheJIT->FindGlobalVariableNamed("GVC",true));
+  EXPECT_EQ(NULL, TheJIT->FindGlobalVariableNamed("GVC"));
+
   uint64_t FBPtr = TheJIT->getFunctionAddress(FB->getName().str());
   TheJIT->finalizeObject();
   EXPECT_TRUE(0 != FBPtr);
diff --git a/unittests/ExecutionEngine/MCJIT/MCJITTest.cpp b/unittests/ExecutionEngine/MCJIT/MCJITTest.cpp
index 94b9a6927b1..01e796d9a4e 100644
--- a/unittests/ExecutionEngine/MCJIT/MCJITTest.cpp
+++ b/unittests/ExecutionEngine/MCJIT/MCJITTest.cpp
@@ -13,6 +13,7 @@
 //===----------------------------------------------------------------------===//
 
 #include "llvm/ExecutionEngine/MCJIT.h"
+#include "llvm/Support/DynamicLibrary.h"
 #include "MCJITTestBase.h"
 #include "gtest/gtest.h"
 
@@ -169,7 +170,7 @@ TEST_F(MCJITTest, multiple_functions) {
     std::stringstream funcName;
     funcName << "level_" << i;
     Outer = startFunction<int32_t(void)>(M.get(), funcName.str());
-    Value *innerResult = Builder.CreateCall(Inner);
+    Value *innerResult = Builder.CreateCall(Inner, {});
     endFunctionWithRet(Outer, innerResult);
 
     Inner = Outer;
@@ -199,4 +200,85 @@ TEST_F(MCJITTest, multiple_decl_lookups) {
   EXPECT_EQ(A, B) << "Repeat calls to getPointerToFunction fail.";
 }
 
+typedef void * (*FunctionHandlerPtr)(const std::string &str);
+
+TEST_F(MCJITTest, lazy_function_creator_pointer) {
+  SKIP_UNSUPPORTED_PLATFORM;
+  
+  Function *Foo = insertExternalReferenceToFunction<int32_t(void)>(M.get(),
+                                                                   "\1Foo");
+  startFunction<int32_t(void)>(M.get(), "Parent");
+  CallInst *Call = Builder.CreateCall(Foo, {});
+  Builder.CreateRet(Call);
+  
+  createJIT(std::move(M));
+  
+  // Set up the lazy function creator that records the name of the last
+  // unresolved external function found in the module. Using a function pointer
+  // prevents us from capturing local variables, which is why this is static.
+  static std::string UnresolvedExternal;
+  FunctionHandlerPtr UnresolvedHandler = [] (const std::string &str) {
+    // Try to resolve the function in the current process before marking it as
+    // unresolved. This solves an issue on ARM where '__aeabi_*' function names
+    // are passed to this handler.
+    void *symbol =
+        llvm::sys::DynamicLibrary::SearchForAddressOfSymbol(str.c_str());
+    if (symbol) {
+      return symbol;
+    }
+    
+    UnresolvedExternal = str;
+    return (void *)(uintptr_t)-1;
+  };
+  TheJIT->InstallLazyFunctionCreator(UnresolvedHandler);
+  
+  // JIT the module.
+  TheJIT->finalizeObject();
+  
+  // Verify that our handler was called.
+  EXPECT_EQ(UnresolvedExternal, "Foo");
+}
+
+TEST_F(MCJITTest, lazy_function_creator_lambda) {
+  SKIP_UNSUPPORTED_PLATFORM;
+  
+  Function *Foo1 = insertExternalReferenceToFunction<int32_t(void)>(M.get(),
+                                                                   "\1Foo1");
+  Function *Foo2 = insertExternalReferenceToFunction<int32_t(void)>(M.get(),
+                                                                   "\1Foo2");
+  startFunction<int32_t(void)>(M.get(), "Parent");
+  CallInst *Call1 = Builder.CreateCall(Foo1, {});
+  CallInst *Call2 = Builder.CreateCall(Foo2, {});
+  Value *Result = Builder.CreateAdd(Call1, Call2);
+  Builder.CreateRet(Result);
+  
+  createJIT(std::move(M));
+  
+  // Set up the lazy function creator that records the name of unresolved
+  // external functions in the module.
+  std::vector<std::string> UnresolvedExternals;
+  auto UnresolvedHandler = [&UnresolvedExternals] (const std::string &str) {
+    // Try to resolve the function in the current process before marking it as
+    // unresolved. This solves an issue on ARM where '__aeabi_*' function names
+    // are passed to this handler.
+    void *symbol =
+        llvm::sys::DynamicLibrary::SearchForAddressOfSymbol(str.c_str());
+    if (symbol) {
+      return symbol;
+    }
+    UnresolvedExternals.push_back(str);
+    return (void *)(uintptr_t)-1;
+  };
+  TheJIT->InstallLazyFunctionCreator(UnresolvedHandler);
+  
+  // JIT the module.
+  TheJIT->finalizeObject();
+  
+  // Verify that our handler was called for each unresolved function.
+  auto I = UnresolvedExternals.begin(), E = UnresolvedExternals.end();
+  EXPECT_EQ(UnresolvedExternals.size(), 2u);
+  EXPECT_FALSE(std::find(I, E, "Foo1") == E);
+  EXPECT_FALSE(std::find(I, E, "Foo2") == E);
+}
+
 }
diff --git a/unittests/ExecutionEngine/MCJIT/MCJITTestBase.h b/unittests/ExecutionEngine/MCJIT/MCJITTestBase.h
index 901f142d09c..0749a1dfb9c 100644
--- a/unittests/ExecutionEngine/MCJIT/MCJITTestBase.h
+++ b/unittests/ExecutionEngine/MCJIT/MCJITTestBase.h
@@ -298,6 +298,8 @@ class MCJITTestBase : public MCJITTestAPICommon, public TrivialModuleBuilder {
     SupportedArchs.push_back(Triple::arm);
     SupportedArchs.push_back(Triple::mips);
     SupportedArchs.push_back(Triple::mipsel);
+    SupportedArchs.push_back(Triple::mips64);
+    SupportedArchs.push_back(Triple::mips64el);
     SupportedArchs.push_back(Triple::x86);
     SupportedArchs.push_back(Triple::x86_64);
 
diff --git a/unittests/ExecutionEngine/Orc/CMakeLists.txt b/unittests/ExecutionEngine/Orc/CMakeLists.txt
index 7bc13a17524..30bd19fa3d2 100644
--- a/unittests/ExecutionEngine/Orc/CMakeLists.txt
+++ b/unittests/ExecutionEngine/Orc/CMakeLists.txt
@@ -1,8 +1,12 @@
 set(LLVM_LINK_COMPONENTS
   Core
+  OrcJIT
   Support
   )
 
 add_llvm_unittest(OrcJITTests
+  IndirectionUtilsTest.cpp
   LazyEmittingLayerTest.cpp
+  ObjectTransformLayerTest.cpp
+  OrcTestCommon.cpp
   )
diff --git a/unittests/ExecutionEngine/Orc/IndirectionUtilsTest.cpp b/unittests/ExecutionEngine/Orc/IndirectionUtilsTest.cpp
new file mode 100644
index 00000000000..1a533b05838
--- /dev/null
+++ b/unittests/ExecutionEngine/Orc/IndirectionUtilsTest.cpp
@@ -0,0 +1,48 @@
+//===- LazyEmittingLayerTest.cpp - Unit tests for the lazy emitting layer -===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+
+#include "OrcTestCommon.h"
+#include "llvm/ADT/SmallVector.h"
+#include "llvm/ExecutionEngine/Orc/IndirectionUtils.h"
+#include "gtest/gtest.h"
+
+using namespace llvm;
+
+namespace {
+
+TEST(IndirectionUtilsTest, MakeStub) {
+  ModuleBuilder MB(getGlobalContext(), "x86_64-apple-macosx10.10", "");
+  Function *F = MB.createFunctionDecl<void(DummyStruct, DummyStruct)>(MB.getModule(), "");
+  SmallVector<AttributeSet, 4> Attrs;
+  Attrs.push_back(
+    AttributeSet::get(MB.getModule()->getContext(), 1U,
+                      AttrBuilder().addAttribute(Attribute::StructRet)));
+  Attrs.push_back(
+    AttributeSet::get(MB.getModule()->getContext(), 2U,
+                      AttrBuilder().addAttribute(Attribute::ByVal)));
+  Attrs.push_back(
+    AttributeSet::get(MB.getModule()->getContext(), ~0U,
+                      AttrBuilder().addAttribute(Attribute::NoUnwind)));
+  F->setAttributes(AttributeSet::get(MB.getModule()->getContext(), Attrs));
+
+  auto ImplPtr = orc::createImplPointer(*F->getType(), *MB.getModule(), "", nullptr);
+  orc::makeStub(*F, *ImplPtr);
+
+  auto II = F->getEntryBlock().begin();
+  EXPECT_TRUE(isa<LoadInst>(*II)) << "First instruction of stub should be a load.";
+  auto *Call = dyn_cast<CallInst>(std::next(II));
+  EXPECT_TRUE(Call != nullptr) << "Second instruction of stub should be a call.";
+  EXPECT_TRUE(Call->isTailCall()) << "Indirect call from stub should be tail call.";
+  EXPECT_TRUE(Call->hasStructRetAttr())
+    << "makeStub should propagate sret attr on 1st argument.";
+  EXPECT_TRUE(Call->paramHasAttr(2U, Attribute::ByVal))
+    << "makeStub should propagate byval attr on 2nd argument.";
+}
+
+}
diff --git a/unittests/ExecutionEngine/Orc/ObjectTransformLayerTest.cpp b/unittests/ExecutionEngine/Orc/ObjectTransformLayerTest.cpp
new file mode 100644
index 00000000000..41b2307cadd
--- /dev/null
+++ b/unittests/ExecutionEngine/Orc/ObjectTransformLayerTest.cpp
@@ -0,0 +1,302 @@
+//===- ObjectTransformLayerTest.cpp - Unit tests for ObjectTransformLayer -===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/ExecutionEngine/Orc/ObjectTransformLayer.h"
+#include "llvm/ADT/STLExtras.h"
+#include "llvm/ADT/SmallVector.h"
+#include "gtest/gtest.h"
+
+using namespace llvm::orc;
+
+namespace {
+
+// Stand-in for RuntimeDyld::MemoryManager
+typedef int MockMemoryManager;
+
+// Stand-in for RuntimeDyld::SymbolResolver
+typedef int MockSymbolResolver;
+
+// stand-in for object::ObjectFile
+typedef int MockObjectFile;
+
+// stand-in for llvm::MemoryBuffer set
+typedef int MockMemoryBufferSet;
+
+// Mock transform that operates on unique pointers to object files, and
+// allocates new object files rather than mutating the given ones.
+struct AllocatingTransform {
+  std::unique_ptr<MockObjectFile>
+  operator()(std::unique_ptr<MockObjectFile> Obj) const {
+    return llvm::make_unique<MockObjectFile>(*Obj + 1);
+  }
+};
+
+// Mock base layer for verifying behavior of transform layer.
+// Each method "T foo(args)" is accompanied by two auxiliary methods:
+//  - "void expectFoo(args)", to be called before calling foo on the transform
+//      layer; saves values of args, which mock layer foo then verifies against.
+// - "void verifyFoo(T)", to be called after foo, which verifies that the
+//      transform layer called the base layer and forwarded any return value.
+class MockBaseLayer {
+public:
+  typedef int ObjSetHandleT;
+
+  MockBaseLayer() : MockSymbol(nullptr) { resetExpectations(); }
+
+  template <typename ObjSetT, typename MemoryManagerPtrT,
+            typename SymbolResolverPtrT>
+  ObjSetHandleT addObjectSet(ObjSetT &Objects, MemoryManagerPtrT MemMgr,
+                             SymbolResolverPtrT Resolver) {
+    EXPECT_EQ(MockManager, *MemMgr) << "MM should pass through";
+    EXPECT_EQ(MockResolver, *Resolver) << "Resolver should pass through";
+    size_t I = 0;
+    for (auto &ObjPtr : Objects) {
+      EXPECT_EQ(MockObjects[I++] + 1, *ObjPtr) << "Transform should be applied";
+    }
+    EXPECT_EQ(MockObjects.size(), I) << "Number of objects should match";
+    LastCalled = "addObjectSet";
+    MockObjSetHandle = 111;
+    return MockObjSetHandle;
+  }
+  template <typename ObjSetT>
+  void expectAddObjectSet(ObjSetT &Objects, MockMemoryManager *MemMgr,
+                          MockSymbolResolver *Resolver) {
+    MockManager = *MemMgr;
+    MockResolver = *Resolver;
+    for (auto &ObjPtr : Objects) {
+      MockObjects.push_back(*ObjPtr);
+    }
+  }
+  void verifyAddObjectSet(ObjSetHandleT Returned) {
+    EXPECT_EQ("addObjectSet", LastCalled);
+    EXPECT_EQ(MockObjSetHandle, Returned) << "Return should pass through";
+    resetExpectations();
+  }
+
+  void removeObjectSet(ObjSetHandleT H) {
+    EXPECT_EQ(MockObjSetHandle, H);
+    LastCalled = "removeObjectSet";
+  }
+  void expectRemoveObjectSet(ObjSetHandleT H) { MockObjSetHandle = H; }
+  void verifyRemoveObjectSet() {
+    EXPECT_EQ("removeObjectSet", LastCalled);
+    resetExpectations();
+  }
+
+  JITSymbol findSymbol(const std::string &Name, bool ExportedSymbolsOnly) {
+    EXPECT_EQ(MockName, Name) << "Name should pass through";
+    EXPECT_EQ(MockBool, ExportedSymbolsOnly) << "Flag should pass through";
+    LastCalled = "findSymbol";
+    MockSymbol = JITSymbol(122, llvm::JITSymbolFlags::None);
+    return MockSymbol;
+  }
+  void expectFindSymbol(const std::string &Name, bool ExportedSymbolsOnly) {
+    MockName = Name;
+    MockBool = ExportedSymbolsOnly;
+  }
+  void verifyFindSymbol(llvm::orc::JITSymbol Returned) {
+    EXPECT_EQ("findSymbol", LastCalled);
+    EXPECT_EQ(MockSymbol.getAddress(), Returned.getAddress())
+        << "Return should pass through";
+    resetExpectations();
+  }
+
+  JITSymbol findSymbolIn(ObjSetHandleT H, const std::string &Name,
+                         bool ExportedSymbolsOnly) {
+    EXPECT_EQ(MockObjSetHandle, H) << "Handle should pass through";
+    EXPECT_EQ(MockName, Name) << "Name should pass through";
+    EXPECT_EQ(MockBool, ExportedSymbolsOnly) << "Flag should pass through";
+    LastCalled = "findSymbolIn";
+    MockSymbol = JITSymbol(122, llvm::JITSymbolFlags::None);
+    return MockSymbol;
+  }
+  void expectFindSymbolIn(ObjSetHandleT H, const std::string &Name,
+                          bool ExportedSymbolsOnly) {
+    MockObjSetHandle = H;
+    MockName = Name;
+    MockBool = ExportedSymbolsOnly;
+  }
+  void verifyFindSymbolIn(llvm::orc::JITSymbol Returned) {
+    EXPECT_EQ("findSymbolIn", LastCalled);
+    EXPECT_EQ(MockSymbol.getAddress(), Returned.getAddress())
+        << "Return should pass through";
+    resetExpectations();
+  }
+
+  void emitAndFinalize(ObjSetHandleT H) {
+    EXPECT_EQ(MockObjSetHandle, H) << "Handle should pass through";
+    LastCalled = "emitAndFinalize";
+  }
+  void expectEmitAndFinalize(ObjSetHandleT H) { MockObjSetHandle = H; }
+  void verifyEmitAndFinalize() {
+    EXPECT_EQ("emitAndFinalize", LastCalled);
+    resetExpectations();
+  }
+
+  void mapSectionAddress(ObjSetHandleT H, const void *LocalAddress,
+                         TargetAddress TargetAddr) {
+    EXPECT_EQ(MockObjSetHandle, H);
+    EXPECT_EQ(MockLocalAddress, LocalAddress);
+    EXPECT_EQ(MockTargetAddress, TargetAddr);
+    LastCalled = "mapSectionAddress";
+  }
+  void expectMapSectionAddress(ObjSetHandleT H, const void *LocalAddress,
+                               TargetAddress TargetAddr) {
+    MockObjSetHandle = H;
+    MockLocalAddress = LocalAddress;
+    MockTargetAddress = TargetAddr;
+  }
+  void verifyMapSectionAddress() {
+    EXPECT_EQ("mapSectionAddress", LastCalled);
+    resetExpectations();
+  }
+
+  template <typename OwningMBSet>
+  void takeOwnershipOfBuffers(ObjSetHandleT H, OwningMBSet MBs) {
+    EXPECT_EQ(MockObjSetHandle, H);
+    EXPECT_EQ(MockBufferSet, *MBs);
+    LastCalled = "takeOwnershipOfBuffers";
+  }
+  void expectTakeOwnershipOfBuffers(ObjSetHandleT H, MockMemoryBufferSet *MBs) {
+    MockObjSetHandle = H;
+    MockBufferSet = *MBs;
+  }
+  void verifyTakeOwnershipOfBuffers() {
+    EXPECT_EQ("takeOwnershipOfBuffers", LastCalled);
+    resetExpectations();
+  }
+
+private:
+  // Backing fields for remembering parameter/return values
+  std::string LastCalled;
+  MockMemoryManager MockManager;
+  MockSymbolResolver MockResolver;
+  std::vector<MockObjectFile> MockObjects;
+  ObjSetHandleT MockObjSetHandle;
+  std::string MockName;
+  bool MockBool;
+  JITSymbol MockSymbol;
+  const void *MockLocalAddress;
+  TargetAddress MockTargetAddress;
+  MockMemoryBufferSet MockBufferSet;
+
+  // Clear remembered parameters between calls
+  void resetExpectations() {
+    LastCalled = "nothing";
+    MockManager = 0;
+    MockResolver = 0;
+    MockObjects.clear();
+    MockObjSetHandle = 0;
+    MockName = "bogus";
+    MockSymbol = JITSymbol(nullptr);
+    MockLocalAddress = nullptr;
+    MockTargetAddress = 0;
+    MockBufferSet = 0;
+  }
+};
+
+// Test each operation on ObjectTransformLayer.
+TEST(ObjectTransformLayerTest, Main) {
+  MockBaseLayer M;
+
+  // Create one object transform layer using a transform (as a functor)
+  // that allocates new objects, and deals in unique pointers.
+  ObjectTransformLayer<MockBaseLayer, AllocatingTransform> T1(M);
+
+  // Create a second object transform layer using a transform (as a lambda)
+  // that mutates objects in place, and deals in naked pointers
+  ObjectTransformLayer<MockBaseLayer,
+                       std::function<MockObjectFile *(MockObjectFile *)>>
+  T2(M, [](MockObjectFile *Obj) {
+    ++(*Obj);
+    return Obj;
+  });
+
+  // Instantiate some mock objects to use below
+  MockObjectFile MockObject1 = 211;
+  MockObjectFile MockObject2 = 222;
+  MockMemoryManager MockManager = 233;
+  MockSymbolResolver MockResolver = 244;
+
+  // Test addObjectSet with T1 (allocating, unique pointers)
+  std::vector<std::unique_ptr<MockObjectFile>> Objs1;
+  Objs1.push_back(llvm::make_unique<MockObjectFile>(MockObject1));
+  Objs1.push_back(llvm::make_unique<MockObjectFile>(MockObject2));
+  auto MM = llvm::make_unique<MockMemoryManager>(MockManager);
+  auto SR = llvm::make_unique<MockSymbolResolver>(MockResolver);
+  M.expectAddObjectSet(Objs1, MM.get(), SR.get());
+  auto H = T1.addObjectSet(Objs1, std::move(MM), std::move(SR));
+  M.verifyAddObjectSet(H);
+
+  // Test addObjectSet with T2 (mutating, naked pointers)
+  llvm::SmallVector<MockObjectFile *, 2> Objs2;
+  Objs2.push_back(&MockObject1);
+  Objs2.push_back(&MockObject2);
+  M.expectAddObjectSet(Objs2, &MockManager, &MockResolver);
+  H = T2.addObjectSet(Objs2, &MockManager, &MockResolver);
+  M.verifyAddObjectSet(H);
+  EXPECT_EQ(212, MockObject1) << "Expected mutation";
+  EXPECT_EQ(223, MockObject2) << "Expected mutation";
+
+  // Test removeObjectSet
+  M.expectRemoveObjectSet(H);
+  T1.removeObjectSet(H);
+  M.verifyRemoveObjectSet();
+
+  // Test findSymbol
+  std::string Name = "foo";
+  bool ExportedOnly = true;
+  M.expectFindSymbol(Name, ExportedOnly);
+  JITSymbol Symbol = T2.findSymbol(Name, ExportedOnly);
+  M.verifyFindSymbol(Symbol);
+
+  // Test findSymbolIn
+  Name = "bar";
+  ExportedOnly = false;
+  M.expectFindSymbolIn(H, Name, ExportedOnly);
+  Symbol = T1.findSymbolIn(H, Name, ExportedOnly);
+  M.verifyFindSymbolIn(Symbol);
+
+  // Test emitAndFinalize
+  M.expectEmitAndFinalize(H);
+  T2.emitAndFinalize(H);
+  M.verifyEmitAndFinalize();
+
+  // Test mapSectionAddress
+  char Buffer[24];
+  TargetAddress MockAddress = 255;
+  M.expectMapSectionAddress(H, Buffer, MockAddress);
+  T1.mapSectionAddress(H, Buffer, MockAddress);
+  M.verifyMapSectionAddress();
+
+  // Test takeOwnershipOfBuffers, using unique pointer to buffer set
+  auto MockBufferSetPtr = llvm::make_unique<MockMemoryBufferSet>(366);
+  M.expectTakeOwnershipOfBuffers(H, MockBufferSetPtr.get());
+  T2.takeOwnershipOfBuffers(H, std::move(MockBufferSetPtr));
+  M.verifyTakeOwnershipOfBuffers();
+
+  // Test takeOwnershipOfBuffers, using naked pointer to buffer set
+  MockMemoryBufferSet MockBufferSet = 266;
+  M.expectTakeOwnershipOfBuffers(H, &MockBufferSet);
+  T1.takeOwnershipOfBuffers(H, &MockBufferSet);
+  M.verifyTakeOwnershipOfBuffers();
+
+  // Verify transform getter (non-const)
+  MockObjectFile Mutatee = 277;
+  MockObjectFile *Out = T2.getTransform()(&Mutatee);
+  EXPECT_EQ(&Mutatee, Out) << "Expected in-place transform";
+  EXPECT_EQ(278, Mutatee) << "Expected incrementing transform";
+
+  // Verify transform getter (const)
+  auto OwnedObj = llvm::make_unique<MockObjectFile>(288);
+  const auto &T1C = T1;
+  OwnedObj = T1C.getTransform()(std::move(OwnedObj));
+  EXPECT_EQ(289, *OwnedObj) << "Expected incrementing transform";
+}
+}
diff --git a/unittests/ExecutionEngine/Orc/OrcTestCommon.cpp b/unittests/ExecutionEngine/Orc/OrcTestCommon.cpp
new file mode 100644
index 00000000000..5fea3c89f86
--- /dev/null
+++ b/unittests/ExecutionEngine/Orc/OrcTestCommon.cpp
@@ -0,0 +1,23 @@
+//===--------- OrcTestCommon.cpp - Utilities for Orc Unit Tests -----------===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// Common utilities for Orc unit tests.
+//
+//===----------------------------------------------------------------------===//
+
+#include "OrcTestCommon.h"
+
+using namespace llvm;
+
+ModuleBuilder::ModuleBuilder(LLVMContext &Context, StringRef Triple,
+                             StringRef Name)
+  : M(new Module(Name, Context)),
+    Builder(Context) {
+  M->setTargetTriple(Triple);
+}
diff --git a/unittests/ExecutionEngine/Orc/OrcTestCommon.h b/unittests/ExecutionEngine/Orc/OrcTestCommon.h
new file mode 100644
index 00000000000..1b2859b51d0
--- /dev/null
+++ b/unittests/ExecutionEngine/Orc/OrcTestCommon.h
@@ -0,0 +1,66 @@
+//===------ OrcTestCommon.h - Utilities for Orc Unit Tests ------*- C++ -*-===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// Common utilities for the Orc unit tests.
+//
+//===----------------------------------------------------------------------===//
+
+
+#ifndef LLVM_UNITTESTS_EXECUTIONENGINE_ORC_ORCTESTCOMMON_H
+#define LLVM_UNITTESTS_EXECUTIONENGINE_ORC_ORCTESTCOMMON_H
+
+#include "llvm/IR/Function.h"
+#include "llvm/IR/IRBuilder.h"
+#include "llvm/IR/LLVMContext.h"
+#include "llvm/IR/Module.h"
+#include "llvm/IR/TypeBuilder.h"
+#include <memory>
+
+namespace llvm {
+
+  class ModuleBuilder {
+  public:
+    ModuleBuilder(LLVMContext &Context, StringRef Triple,
+                  StringRef Name);
+
+    template <typename FuncType>
+    Function* createFunctionDecl(Module *M, StringRef Name) {
+      return Function::Create(
+               TypeBuilder<FuncType, false>::get(M->getContext()),
+               GlobalValue::ExternalLinkage, Name, M);
+    }
+
+    Module* getModule() { return M.get(); }
+    const Module* getModule() const { return M.get(); }
+    std::unique_ptr<Module> takeModule() { return std::move(M); }
+
+  private:
+    std::unique_ptr<Module> M;
+    IRBuilder<> Builder;
+  };
+
+  // Dummy struct type.
+  struct DummyStruct {
+    int X[256];
+  };
+
+  // TypeBuilder specialization for DummyStruct.
+  template <bool XCompile>
+  class TypeBuilder<DummyStruct, XCompile> {
+  public:
+    static StructType *get(LLVMContext &Context) {
+      return StructType::get(
+          TypeBuilder<types::i<32>[256], XCompile>::get(Context), nullptr);
+    }
+  };
+
+
+} // namespace llvm
+
+#endif
diff --git a/unittests/IR/ConstantsTest.cpp b/unittests/IR/ConstantsTest.cpp
index 0c7777d7678..2df008a78cc 100644
--- a/unittests/IR/ConstantsTest.cpp
+++ b/unittests/IR/ConstantsTest.cpp
@@ -187,6 +187,13 @@ TEST(ConstantsTest, AsInstructionsTest) {
   Constant *P6 = ConstantExpr::getBitCast(P4, VectorType::get(Int16Ty, 2));
 
   Constant *One = ConstantInt::get(Int32Ty, 1);
+  Constant *Two = ConstantInt::get(Int64Ty, 2);
+  Constant *Big = ConstantInt::get(getGlobalContext(),
+                                   APInt{256, uint64_t(-1), true});
+  Constant *Elt = ConstantInt::get(Int16Ty, 2015);
+  Constant *Undef16  = UndefValue::get(Int16Ty);
+  Constant *Undef64  = UndefValue::get(Int64Ty);
+  Constant *UndefV16 = UndefValue::get(P6->getType());
 
   #define P0STR "ptrtoint (i32** @dummy to i32)"
   #define P1STR "uitofp (i32 ptrtoint (i32** @dummy to i32) to float)"
@@ -255,6 +262,16 @@ TEST(ConstantsTest, AsInstructionsTest) {
 
   CHECK(ConstantExpr::getExtractElement(P6, One), "extractelement <2 x i16> "
         P6STR ", i32 1");
+
+  EXPECT_EQ(Undef16, ConstantExpr::getExtractElement(P6, Two));
+  EXPECT_EQ(Undef16, ConstantExpr::getExtractElement(P6, Big));
+  EXPECT_EQ(Undef16, ConstantExpr::getExtractElement(P6, Undef64));
+
+  EXPECT_EQ(Elt, ConstantExpr::getExtractElement(
+                 ConstantExpr::getInsertElement(P6, Elt, One), One));
+  EXPECT_EQ(UndefV16, ConstantExpr::getInsertElement(P6, Elt, Two));
+  EXPECT_EQ(UndefV16, ConstantExpr::getInsertElement(P6, Elt, Big));
+  EXPECT_EQ(UndefV16, ConstantExpr::getInsertElement(P6, Elt, Undef64));
 }
 
 #ifdef GTEST_HAS_DEATH_TEST
@@ -331,7 +348,7 @@ TEST(ConstantsTest, GEPReplaceWithConstant) {
   std::unique_ptr<Module> M(new Module("MyModule", Context));
 
   Type *IntTy = Type::getInt32Ty(Context);
-  Type *PtrTy = PointerType::get(IntTy, 0);
+  auto *PtrTy = PointerType::get(IntTy, 0);
   auto *C1 = ConstantInt::get(IntTy, 1);
   auto *Placeholder = new GlobalVariable(
       *M, IntTy, false, GlobalValue::ExternalWeakLinkage, nullptr);
@@ -344,7 +361,7 @@ TEST(ConstantsTest, GEPReplaceWithConstant) {
 
   auto *Global = new GlobalVariable(*M, PtrTy, false,
                                     GlobalValue::ExternalLinkage, nullptr);
-  auto *Alias = GlobalAlias::create(IntTy, 0, GlobalValue::ExternalLinkage,
+  auto *Alias = GlobalAlias::create(PtrTy, GlobalValue::ExternalLinkage,
                                     "alias", Global, M.get());
   Placeholder->replaceAllUsesWith(Alias);
   ASSERT_EQ(GEP, Ref->getInitializer());
diff --git a/unittests/IR/DebugInfoTest.cpp b/unittests/IR/DebugInfoTest.cpp
index 534663a2da1..f633782b379 100644
--- a/unittests/IR/DebugInfoTest.cpp
+++ b/unittests/IR/DebugInfoTest.cpp
@@ -14,69 +14,67 @@ using namespace llvm;
 
 namespace {
 
-TEST(DebugNodeTest, getFlag) {
+TEST(DINodeTest, getFlag) {
   // Some valid flags.
-  EXPECT_EQ(DebugNode::FlagPublic, DebugNode::getFlag("DIFlagPublic"));
-  EXPECT_EQ(DebugNode::FlagProtected, DebugNode::getFlag("DIFlagProtected"));
-  EXPECT_EQ(DebugNode::FlagPrivate, DebugNode::getFlag("DIFlagPrivate"));
-  EXPECT_EQ(DebugNode::FlagVector, DebugNode::getFlag("DIFlagVector"));
-  EXPECT_EQ(DebugNode::FlagRValueReference,
-            DebugNode::getFlag("DIFlagRValueReference"));
+  EXPECT_EQ(DINode::FlagPublic, DINode::getFlag("DIFlagPublic"));
+  EXPECT_EQ(DINode::FlagProtected, DINode::getFlag("DIFlagProtected"));
+  EXPECT_EQ(DINode::FlagPrivate, DINode::getFlag("DIFlagPrivate"));
+  EXPECT_EQ(DINode::FlagVector, DINode::getFlag("DIFlagVector"));
+  EXPECT_EQ(DINode::FlagRValueReference,
+            DINode::getFlag("DIFlagRValueReference"));
 
   // FlagAccessibility shouldn't work.
-  EXPECT_EQ(0u, DebugNode::getFlag("DIFlagAccessibility"));
+  EXPECT_EQ(0u, DINode::getFlag("DIFlagAccessibility"));
 
   // Some other invalid strings.
-  EXPECT_EQ(0u, DebugNode::getFlag("FlagVector"));
-  EXPECT_EQ(0u, DebugNode::getFlag("Vector"));
-  EXPECT_EQ(0u, DebugNode::getFlag("other things"));
-  EXPECT_EQ(0u, DebugNode::getFlag("DIFlagOther"));
+  EXPECT_EQ(0u, DINode::getFlag("FlagVector"));
+  EXPECT_EQ(0u, DINode::getFlag("Vector"));
+  EXPECT_EQ(0u, DINode::getFlag("other things"));
+  EXPECT_EQ(0u, DINode::getFlag("DIFlagOther"));
 }
 
-TEST(DebugNodeTest, getFlagString) {
+TEST(DINodeTest, getFlagString) {
   // Some valid flags.
   EXPECT_EQ(StringRef("DIFlagPublic"),
-            DebugNode::getFlagString(DebugNode::FlagPublic));
+            DINode::getFlagString(DINode::FlagPublic));
   EXPECT_EQ(StringRef("DIFlagProtected"),
-            DebugNode::getFlagString(DebugNode::FlagProtected));
+            DINode::getFlagString(DINode::FlagProtected));
   EXPECT_EQ(StringRef("DIFlagPrivate"),
-            DebugNode::getFlagString(DebugNode::FlagPrivate));
+            DINode::getFlagString(DINode::FlagPrivate));
   EXPECT_EQ(StringRef("DIFlagVector"),
-            DebugNode::getFlagString(DebugNode::FlagVector));
+            DINode::getFlagString(DINode::FlagVector));
   EXPECT_EQ(StringRef("DIFlagRValueReference"),
-            DebugNode::getFlagString(DebugNode::FlagRValueReference));
+            DINode::getFlagString(DINode::FlagRValueReference));
 
   // FlagAccessibility actually equals FlagPublic.
   EXPECT_EQ(StringRef("DIFlagPublic"),
-            DebugNode::getFlagString(DebugNode::FlagAccessibility));
+            DINode::getFlagString(DINode::FlagAccessibility));
 
   // Some other invalid flags.
-  EXPECT_EQ(StringRef(), DebugNode::getFlagString(DebugNode::FlagPublic |
-                                                  DebugNode::FlagVector));
-  EXPECT_EQ(StringRef(), DebugNode::getFlagString(DebugNode::FlagFwdDecl |
-                                                  DebugNode::FlagArtificial));
-  EXPECT_EQ(StringRef(), DebugNode::getFlagString(0xffff));
+  EXPECT_EQ(StringRef(),
+            DINode::getFlagString(DINode::FlagPublic | DINode::FlagVector));
+  EXPECT_EQ(StringRef(), DINode::getFlagString(DINode::FlagFwdDecl |
+                                               DINode::FlagArtificial));
+  EXPECT_EQ(StringRef(), DINode::getFlagString(0xffff));
 }
 
-TEST(DebugNodeTest, splitFlags) {
+TEST(DINodeTest, splitFlags) {
 // Some valid flags.
 #define CHECK_SPLIT(FLAGS, VECTOR, REMAINDER)                                  \
   {                                                                            \
     SmallVector<unsigned, 8> V;                                                \
-    EXPECT_EQ(REMAINDER, DebugNode::splitFlags(FLAGS, V));                     \
+    EXPECT_EQ(REMAINDER, DINode::splitFlags(FLAGS, V));                        \
     EXPECT_TRUE(makeArrayRef(V).equals(VECTOR));                               \
   }
-  CHECK_SPLIT(DebugNode::FlagPublic, {DebugNode::FlagPublic}, 0u);
-  CHECK_SPLIT(DebugNode::FlagProtected, {DebugNode::FlagProtected}, 0u);
-  CHECK_SPLIT(DebugNode::FlagPrivate, {DebugNode::FlagPrivate}, 0u);
-  CHECK_SPLIT(DebugNode::FlagVector, {DebugNode::FlagVector}, 0u);
-  CHECK_SPLIT(DebugNode::FlagRValueReference, {DebugNode::FlagRValueReference},
-              0u);
-  unsigned Flags[] = {DebugNode::FlagFwdDecl, DebugNode::FlagVector};
-  CHECK_SPLIT(DebugNode::FlagFwdDecl | DebugNode::FlagVector, Flags, 0u);
+  CHECK_SPLIT(DINode::FlagPublic, {DINode::FlagPublic}, 0u);
+  CHECK_SPLIT(DINode::FlagProtected, {DINode::FlagProtected}, 0u);
+  CHECK_SPLIT(DINode::FlagPrivate, {DINode::FlagPrivate}, 0u);
+  CHECK_SPLIT(DINode::FlagVector, {DINode::FlagVector}, 0u);
+  CHECK_SPLIT(DINode::FlagRValueReference, {DINode::FlagRValueReference}, 0u);
+  unsigned Flags[] = {DINode::FlagFwdDecl, DINode::FlagVector};
+  CHECK_SPLIT(DINode::FlagFwdDecl | DINode::FlagVector, Flags, 0u);
   CHECK_SPLIT(0x100000u, {}, 0x100000u);
-  CHECK_SPLIT(0x100000u | DebugNode::FlagVector, {DebugNode::FlagVector},
-              0x100000u);
+  CHECK_SPLIT(0x100000u | DINode::FlagVector, {DINode::FlagVector}, 0x100000u);
 #undef CHECK_SPLIT
 }
 
diff --git a/unittests/IR/DominatorTreeTest.cpp b/unittests/IR/DominatorTreeTest.cpp
index 6a5838e693a..146ec576dba 100644
--- a/unittests/IR/DominatorTreeTest.cpp
+++ b/unittests/IR/DominatorTreeTest.cpp
@@ -217,7 +217,7 @@ namespace llvm {
     std::unique_ptr<Module> makeLLVMModule(DPass *P) {
       const char *ModuleStrig =
         "declare i32 @g()\n" \
-        "define void @f(i32 %x) {\n" \
+        "define void @f(i32 %x) personality i32 ()* @g {\n" \
         "bb0:\n" \
         "  %y1 = add i32 %x, 1\n" \
         "  %y2 = add i32 %x, 1\n" \
@@ -226,7 +226,7 @@ namespace llvm {
         "  %y4 = add i32 %x, 1\n" \
         "  br label %bb4\n" \
         "bb2:\n" \
-        "  %y5 = landingpad i32 personality i32 ()* @g\n" \
+        "  %y5 = landingpad i32\n" \
         "          cleanup\n" \
         "  br label %bb4\n" \
         "bb3:\n" \
diff --git a/unittests/IR/IRBuilderTest.cpp b/unittests/IR/IRBuilderTest.cpp
index 9fe241c1a74..093cbbfc779 100644
--- a/unittests/IR/IRBuilderTest.cpp
+++ b/unittests/IR/IRBuilderTest.cpp
@@ -104,8 +104,7 @@ TEST_F(IRBuilderTest, CreateCondBr) {
 
 TEST_F(IRBuilderTest, LandingPadName) {
   IRBuilder<> Builder(BB);
-  LandingPadInst *LP = Builder.CreateLandingPad(Builder.getInt32Ty(),
-                                                Builder.getInt32(0), 0, "LP");
+  LandingPadInst *LP = Builder.CreateLandingPad(Builder.getInt32Ty(), 0, "LP");
   EXPECT_EQ(LP->getName(), "LP");
 }
 
@@ -131,8 +130,8 @@ TEST_F(IRBuilderTest, GetIntTy) {
 
 TEST_F(IRBuilderTest, FastMathFlags) {
   IRBuilder<> Builder(BB);
-  Value *F;
-  Instruction *FDiv, *FAdd;
+  Value *F, *FC;
+  Instruction *FDiv, *FAdd, *FCmp;
 
   F = Builder.CreateLoad(GV);
   F = Builder.CreateFAdd(F, F);
@@ -191,6 +190,24 @@ TEST_F(IRBuilderTest, FastMathFlags) {
 
   Builder.clearFastMathFlags();
 
+  FC = Builder.CreateFCmpOEQ(F, F);
+  ASSERT_TRUE(isa<Instruction>(FC));
+  FCmp = cast<Instruction>(FC);
+  EXPECT_FALSE(FCmp->hasAllowReciprocal());
+
+  FMF.clear();
+  FMF.setAllowReciprocal();
+  Builder.SetFastMathFlags(FMF);
+
+  FC = Builder.CreateFCmpOEQ(F, F);
+  EXPECT_TRUE(Builder.getFastMathFlags().any());
+  EXPECT_TRUE(Builder.getFastMathFlags().AllowReciprocal);
+  ASSERT_TRUE(isa<Instruction>(FC));
+  FCmp = cast<Instruction>(FC);
+  EXPECT_TRUE(FCmp->hasAllowReciprocal());
+
+  Builder.clearFastMathFlags();
+
   // To test a copy, make sure that a '0' and a '1' change state. 
   F = Builder.CreateFDiv(F, F);
   ASSERT_TRUE(isa<Instruction>(F));
@@ -306,5 +323,67 @@ TEST_F(IRBuilderTest, DIBuilder) {
   EXPECT_TRUE(verifyModule(*M));
 }
 
+TEST_F(IRBuilderTest, InsertExtractElement) {
+  IRBuilder<> Builder(BB);
 
+  auto VecTy = VectorType::get(Builder.getInt64Ty(), 4);
+  auto Elt1 = Builder.getInt64(-1);
+  auto Elt2 = Builder.getInt64(-2);
+  Value *Vec = UndefValue::get(VecTy);
+  Vec = Builder.CreateInsertElement(Vec, Elt1, Builder.getInt8(1));
+  Vec = Builder.CreateInsertElement(Vec, Elt2, 2);
+  auto X1 = Builder.CreateExtractElement(Vec, 1);
+  auto X2 = Builder.CreateExtractElement(Vec, Builder.getInt32(2));
+  EXPECT_EQ(Elt1, X1);
+  EXPECT_EQ(Elt2, X2);
+}
+
+TEST_F(IRBuilderTest, CreateGlobalStringPtr) {
+  IRBuilder<> Builder(BB);
+
+  auto String1a = Builder.CreateGlobalStringPtr("TestString", "String1a");
+  auto String1b = Builder.CreateGlobalStringPtr("TestString", "String1b", 0);
+  auto String2 = Builder.CreateGlobalStringPtr("TestString", "String2", 1);
+  auto String3 = Builder.CreateGlobalString("TestString", "String3", 2);
+
+  EXPECT_TRUE(String1a->getType()->getPointerAddressSpace() == 0);
+  EXPECT_TRUE(String1b->getType()->getPointerAddressSpace() == 0);
+  EXPECT_TRUE(String2->getType()->getPointerAddressSpace() == 1);
+  EXPECT_TRUE(String3->getType()->getPointerAddressSpace() == 2);
+}
+
+TEST_F(IRBuilderTest, DebugLoc) {
+  auto CalleeTy = FunctionType::get(Type::getVoidTy(Ctx),
+                                    /*isVarArg=*/false);
+  auto Callee =
+      Function::Create(CalleeTy, Function::ExternalLinkage, "", M.get());
+
+  DIBuilder DIB(*M);
+  auto File = DIB.createFile("tmp.cpp", "/");
+  auto CU = DIB.createCompileUnit(dwarf::DW_LANG_C_plus_plus_11, "tmp.cpp", "/",
+                                  "", true, "", 0);
+  auto SPType = DIB.createSubroutineType(File, DIB.getOrCreateTypeArray(None));
+  auto SP =
+      DIB.createFunction(CU, "foo", "foo", File, 1, SPType, false, true, 1);
+  DebugLoc DL1 = DILocation::get(Ctx, 2, 0, SP);
+  DebugLoc DL2 = DILocation::get(Ctx, 3, 0, SP);
+
+  auto BB2 = BasicBlock::Create(Ctx, "bb2", F);
+  auto Br = BranchInst::Create(BB2, BB);
+  Br->setDebugLoc(DL1);
+
+  IRBuilder<> Builder(Ctx);
+  Builder.SetInsertPoint(Br);
+  EXPECT_EQ(DL1, Builder.getCurrentDebugLocation());
+  auto Call1 = Builder.CreateCall(Callee, None);
+  EXPECT_EQ(DL1, Call1->getDebugLoc());
+
+  Call1->setDebugLoc(DL2);
+  Builder.SetInsertPoint(Call1->getParent(), Call1);
+  EXPECT_EQ(DL2, Builder.getCurrentDebugLocation());
+  auto Call2 = Builder.CreateCall(Callee, None);
+  EXPECT_EQ(DL2, Call2->getDebugLoc());
+
+  DIB.finalize();
+}
 }
diff --git a/unittests/IR/MetadataTest.cpp b/unittests/IR/MetadataTest.cpp
index 8b4c5db73e3..260ce85a71d 100644
--- a/unittests/IR/MetadataTest.cpp
+++ b/unittests/IR/MetadataTest.cpp
@@ -16,7 +16,9 @@
 #include "llvm/IR/LLVMContext.h"
 #include "llvm/IR/Metadata.h"
 #include "llvm/IR/Module.h"
+#include "llvm/IR/ModuleSlotTracker.h"
 #include "llvm/IR/Type.h"
+#include "llvm/IR/Verifier.h"
 #include "llvm/Support/raw_ostream.h"
 #include "gtest/gtest.h"
 using namespace llvm;
@@ -77,24 +79,24 @@ class MetadataTest : public testing::Test {
   }
 
   MDTuple *getTuple() { return MDTuple::getDistinct(Context, None); }
-  MDSubroutineType *getSubroutineType() {
-    return MDSubroutineType::getDistinct(Context, 0, getNode(nullptr));
+  DISubroutineType *getSubroutineType() {
+    return DISubroutineType::getDistinct(Context, 0, getNode(nullptr));
   }
-  MDSubprogram *getSubprogram() {
-    return MDSubprogram::getDistinct(Context, nullptr, "", "", nullptr, 0,
+  DISubprogram *getSubprogram() {
+    return DISubprogram::getDistinct(Context, nullptr, "", "", nullptr, 0,
                                      nullptr, false, false, 0, nullptr, 0, 0, 0,
                                      0);
   }
-  MDScopeRef getSubprogramRef() { return getSubprogram()->getRef(); }
-  MDFile *getFile() {
-    return MDFile::getDistinct(Context, "file.c", "/path/to/dir");
+  DIScopeRef getSubprogramRef() { return getSubprogram()->getRef(); }
+  DIFile *getFile() {
+    return DIFile::getDistinct(Context, "file.c", "/path/to/dir");
   }
-  MDTypeRef getBasicType(StringRef Name) {
-    return MDBasicType::get(Context, dwarf::DW_TAG_unspecified_type, Name)
+  DITypeRef getBasicType(StringRef Name) {
+    return DIBasicType::get(Context, dwarf::DW_TAG_unspecified_type, Name)
         ->getRef();
   }
-  MDTypeRef getDerivedType() {
-    return MDDerivedType::getDistinct(Context, dwarf::DW_TAG_pointer_type, "",
+  DITypeRef getDerivedType() {
+    return DIDerivedType::getDistinct(Context, dwarf::DW_TAG_pointer_type, "",
                                       nullptr, 0, nullptr,
                                       getBasicType("basictype"), 1, 2, 0, 0)
         ->getRef();
@@ -105,8 +107,8 @@ class MetadataTest : public testing::Test {
   ConstantAsMetadata *getConstantAsMetadata() {
     return ConstantAsMetadata::get(getConstant());
   }
-  MDTypeRef getCompositeType() {
-    return MDCompositeType::getDistinct(
+  DITypeRef getCompositeType() {
+    return DICompositeType::getDistinct(
                Context, dwarf::DW_TAG_structure_type, "", nullptr, 0, nullptr,
                nullptr, 32, 32, 0, 0, nullptr, 0, nullptr, nullptr, "")
         ->getRef();
@@ -355,6 +357,10 @@ TEST_F(MDNodeTest, PrintFromFunction) {
 
   EXPECT_PRINTER_EQ("!0 = distinct !{}", N0->print(OS, &M));
   EXPECT_PRINTER_EQ("!1 = distinct !{}", N1->print(OS, &M));
+
+  ModuleSlotTracker MST(&M);
+  EXPECT_PRINTER_EQ("!0 = distinct !{}", N0->print(OS, MST));
+  EXPECT_PRINTER_EQ("!1 = distinct !{}", N1->print(OS, MST));
 }
 
 TEST_F(MDNodeTest, PrintFromMetadataAsValue) {
@@ -383,6 +389,14 @@ TEST_F(MDNodeTest, PrintFromMetadataAsValue) {
   EXPECT_PRINTER_EQ("!1", MAV1->printAsOperand(OS, false));
   EXPECT_PRINTER_EQ("metadata !0", MAV0->printAsOperand(OS, true));
   EXPECT_PRINTER_EQ("metadata !1", MAV1->printAsOperand(OS, true));
+
+  ModuleSlotTracker MST(&M);
+  EXPECT_PRINTER_EQ("!0 = distinct !{}", MAV0->print(OS, MST));
+  EXPECT_PRINTER_EQ("!1 = distinct !{}", MAV1->print(OS, MST));
+  EXPECT_PRINTER_EQ("!0", MAV0->printAsOperand(OS, false, MST));
+  EXPECT_PRINTER_EQ("!1", MAV1->printAsOperand(OS, false, MST));
+  EXPECT_PRINTER_EQ("metadata !0", MAV0->printAsOperand(OS, true, MST));
+  EXPECT_PRINTER_EQ("metadata !1", MAV1->printAsOperand(OS, true, MST));
 }
 #undef EXPECT_PRINTER_EQ
 
@@ -756,56 +770,56 @@ TEST_F(MDNodeTest, deleteTemporaryWithTrackingRef) {
   EXPECT_EQ(nullptr, Ref.get());
 }
 
-typedef MetadataTest MDLocationTest;
+typedef MetadataTest DILocationTest;
 
-TEST_F(MDLocationTest, Overflow) {
-  MDSubprogram *N = getSubprogram();
+TEST_F(DILocationTest, Overflow) {
+  DISubprogram *N = getSubprogram();
   {
-    MDLocation *L = MDLocation::get(Context, 2, 7, N);
+    DILocation *L = DILocation::get(Context, 2, 7, N);
     EXPECT_EQ(2u, L->getLine());
     EXPECT_EQ(7u, L->getColumn());
   }
   unsigned U16 = 1u << 16;
   {
-    MDLocation *L = MDLocation::get(Context, UINT32_MAX, U16 - 1, N);
+    DILocation *L = DILocation::get(Context, UINT32_MAX, U16 - 1, N);
     EXPECT_EQ(UINT32_MAX, L->getLine());
     EXPECT_EQ(U16 - 1, L->getColumn());
   }
   {
-    MDLocation *L = MDLocation::get(Context, UINT32_MAX, U16, N);
+    DILocation *L = DILocation::get(Context, UINT32_MAX, U16, N);
     EXPECT_EQ(UINT32_MAX, L->getLine());
     EXPECT_EQ(0u, L->getColumn());
   }
   {
-    MDLocation *L = MDLocation::get(Context, UINT32_MAX, U16 + 1, N);
+    DILocation *L = DILocation::get(Context, UINT32_MAX, U16 + 1, N);
     EXPECT_EQ(UINT32_MAX, L->getLine());
     EXPECT_EQ(0u, L->getColumn());
   }
 }
 
-TEST_F(MDLocationTest, getDistinct) {
+TEST_F(DILocationTest, getDistinct) {
   MDNode *N = getSubprogram();
-  MDLocation *L0 = MDLocation::getDistinct(Context, 2, 7, N);
+  DILocation *L0 = DILocation::getDistinct(Context, 2, 7, N);
   EXPECT_TRUE(L0->isDistinct());
-  MDLocation *L1 = MDLocation::get(Context, 2, 7, N);
+  DILocation *L1 = DILocation::get(Context, 2, 7, N);
   EXPECT_FALSE(L1->isDistinct());
-  EXPECT_EQ(L1, MDLocation::get(Context, 2, 7, N));
+  EXPECT_EQ(L1, DILocation::get(Context, 2, 7, N));
 }
 
-TEST_F(MDLocationTest, getTemporary) {
+TEST_F(DILocationTest, getTemporary) {
   MDNode *N = MDNode::get(Context, None);
-  auto L = MDLocation::getTemporary(Context, 2, 7, N);
+  auto L = DILocation::getTemporary(Context, 2, 7, N);
   EXPECT_TRUE(L->isTemporary());
   EXPECT_FALSE(L->isResolved());
 }
 
-typedef MetadataTest GenericDebugNodeTest;
+typedef MetadataTest GenericDINodeTest;
 
-TEST_F(GenericDebugNodeTest, get) {
+TEST_F(GenericDINodeTest, get) {
   StringRef Header = "header";
   auto *Empty = MDNode::get(Context, None);
   Metadata *Ops1[] = {Empty};
-  auto *N = GenericDebugNode::get(Context, 15, Header, Ops1);
+  auto *N = GenericDINode::get(Context, 15, Header, Ops1);
   EXPECT_EQ(15u, N->getTag());
   EXPECT_EQ(2u, N->getNumOperands());
   EXPECT_EQ(Header, N->getHeader());
@@ -815,7 +829,7 @@ TEST_F(GenericDebugNodeTest, get) {
   EXPECT_EQ(Empty, N->getOperand(1));
   ASSERT_TRUE(N->isUniqued());
 
-  EXPECT_EQ(N, GenericDebugNode::get(Context, 15, Header, Ops1));
+  EXPECT_EQ(N, GenericDINode::get(Context, 15, Header, Ops1));
 
   N->replaceOperandWith(1, nullptr);
   EXPECT_EQ(15u, N->getTag());
@@ -824,103 +838,103 @@ TEST_F(GenericDebugNodeTest, get) {
   ASSERT_TRUE(N->isUniqued());
 
   Metadata *Ops2[] = {nullptr};
-  EXPECT_EQ(N, GenericDebugNode::get(Context, 15, Header, Ops2));
+  EXPECT_EQ(N, GenericDINode::get(Context, 15, Header, Ops2));
 
   N->replaceDwarfOperandWith(0, Empty);
   EXPECT_EQ(15u, N->getTag());
   EXPECT_EQ(Header, N->getHeader());
   EXPECT_EQ(Empty, N->getDwarfOperand(0));
   ASSERT_TRUE(N->isUniqued());
-  EXPECT_EQ(N, GenericDebugNode::get(Context, 15, Header, Ops1));
+  EXPECT_EQ(N, GenericDINode::get(Context, 15, Header, Ops1));
 
-  TempGenericDebugNode Temp = N->clone();
+  TempGenericDINode Temp = N->clone();
   EXPECT_EQ(N, MDNode::replaceWithUniqued(std::move(Temp)));
 }
 
-TEST_F(GenericDebugNodeTest, getEmptyHeader) {
+TEST_F(GenericDINodeTest, getEmptyHeader) {
   // Canonicalize !"" to null.
-  auto *N = GenericDebugNode::get(Context, 15, StringRef(), None);
+  auto *N = GenericDINode::get(Context, 15, StringRef(), None);
   EXPECT_EQ(StringRef(), N->getHeader());
   EXPECT_EQ(nullptr, N->getOperand(0));
 }
 
-typedef MetadataTest MDSubrangeTest;
+typedef MetadataTest DISubrangeTest;
 
-TEST_F(MDSubrangeTest, get) {
-  auto *N = MDSubrange::get(Context, 5, 7);
+TEST_F(DISubrangeTest, get) {
+  auto *N = DISubrange::get(Context, 5, 7);
   EXPECT_EQ(dwarf::DW_TAG_subrange_type, N->getTag());
   EXPECT_EQ(5, N->getCount());
   EXPECT_EQ(7, N->getLowerBound());
-  EXPECT_EQ(N, MDSubrange::get(Context, 5, 7));
-  EXPECT_EQ(MDSubrange::get(Context, 5, 0), MDSubrange::get(Context, 5));
+  EXPECT_EQ(N, DISubrange::get(Context, 5, 7));
+  EXPECT_EQ(DISubrange::get(Context, 5, 0), DISubrange::get(Context, 5));
 
-  TempMDSubrange Temp = N->clone();
+  TempDISubrange Temp = N->clone();
   EXPECT_EQ(N, MDNode::replaceWithUniqued(std::move(Temp)));
 }
 
-TEST_F(MDSubrangeTest, getEmptyArray) {
-  auto *N = MDSubrange::get(Context, -1, 0);
+TEST_F(DISubrangeTest, getEmptyArray) {
+  auto *N = DISubrange::get(Context, -1, 0);
   EXPECT_EQ(dwarf::DW_TAG_subrange_type, N->getTag());
   EXPECT_EQ(-1, N->getCount());
   EXPECT_EQ(0, N->getLowerBound());
-  EXPECT_EQ(N, MDSubrange::get(Context, -1, 0));
+  EXPECT_EQ(N, DISubrange::get(Context, -1, 0));
 }
 
-typedef MetadataTest MDEnumeratorTest;
+typedef MetadataTest DIEnumeratorTest;
 
-TEST_F(MDEnumeratorTest, get) {
-  auto *N = MDEnumerator::get(Context, 7, "name");
+TEST_F(DIEnumeratorTest, get) {
+  auto *N = DIEnumerator::get(Context, 7, "name");
   EXPECT_EQ(dwarf::DW_TAG_enumerator, N->getTag());
   EXPECT_EQ(7, N->getValue());
   EXPECT_EQ("name", N->getName());
-  EXPECT_EQ(N, MDEnumerator::get(Context, 7, "name"));
+  EXPECT_EQ(N, DIEnumerator::get(Context, 7, "name"));
 
-  EXPECT_NE(N, MDEnumerator::get(Context, 8, "name"));
-  EXPECT_NE(N, MDEnumerator::get(Context, 7, "nam"));
+  EXPECT_NE(N, DIEnumerator::get(Context, 8, "name"));
+  EXPECT_NE(N, DIEnumerator::get(Context, 7, "nam"));
 
-  TempMDEnumerator Temp = N->clone();
+  TempDIEnumerator Temp = N->clone();
   EXPECT_EQ(N, MDNode::replaceWithUniqued(std::move(Temp)));
 }
 
-typedef MetadataTest MDBasicTypeTest;
+typedef MetadataTest DIBasicTypeTest;
 
-TEST_F(MDBasicTypeTest, get) {
+TEST_F(DIBasicTypeTest, get) {
   auto *N =
-      MDBasicType::get(Context, dwarf::DW_TAG_base_type, "special", 33, 26, 7);
+      DIBasicType::get(Context, dwarf::DW_TAG_base_type, "special", 33, 26, 7);
   EXPECT_EQ(dwarf::DW_TAG_base_type, N->getTag());
   EXPECT_EQ("special", N->getName());
   EXPECT_EQ(33u, N->getSizeInBits());
   EXPECT_EQ(26u, N->getAlignInBits());
   EXPECT_EQ(7u, N->getEncoding());
   EXPECT_EQ(0u, N->getLine());
-  EXPECT_EQ(N, MDBasicType::get(Context, dwarf::DW_TAG_base_type, "special", 33,
+  EXPECT_EQ(N, DIBasicType::get(Context, dwarf::DW_TAG_base_type, "special", 33,
                                 26, 7));
 
-  EXPECT_NE(N, MDBasicType::get(Context, dwarf::DW_TAG_unspecified_type,
+  EXPECT_NE(N, DIBasicType::get(Context, dwarf::DW_TAG_unspecified_type,
                                 "special", 33, 26, 7));
   EXPECT_NE(N,
-            MDBasicType::get(Context, dwarf::DW_TAG_base_type, "s", 33, 26, 7));
-  EXPECT_NE(N, MDBasicType::get(Context, dwarf::DW_TAG_base_type, "special", 32,
+            DIBasicType::get(Context, dwarf::DW_TAG_base_type, "s", 33, 26, 7));
+  EXPECT_NE(N, DIBasicType::get(Context, dwarf::DW_TAG_base_type, "special", 32,
                                 26, 7));
-  EXPECT_NE(N, MDBasicType::get(Context, dwarf::DW_TAG_base_type, "special", 33,
+  EXPECT_NE(N, DIBasicType::get(Context, dwarf::DW_TAG_base_type, "special", 33,
                                 25, 7));
-  EXPECT_NE(N, MDBasicType::get(Context, dwarf::DW_TAG_base_type, "special", 33,
+  EXPECT_NE(N, DIBasicType::get(Context, dwarf::DW_TAG_base_type, "special", 33,
                                 26, 6));
 
-  TempMDBasicType Temp = N->clone();
+  TempDIBasicType Temp = N->clone();
   EXPECT_EQ(N, MDNode::replaceWithUniqued(std::move(Temp)));
 }
 
-TEST_F(MDBasicTypeTest, getWithLargeValues) {
-  auto *N = MDBasicType::get(Context, dwarf::DW_TAG_base_type, "special",
+TEST_F(DIBasicTypeTest, getWithLargeValues) {
+  auto *N = DIBasicType::get(Context, dwarf::DW_TAG_base_type, "special",
                              UINT64_MAX, UINT64_MAX - 1, 7);
   EXPECT_EQ(UINT64_MAX, N->getSizeInBits());
   EXPECT_EQ(UINT64_MAX - 1, N->getAlignInBits());
 }
 
-TEST_F(MDBasicTypeTest, getUnspecified) {
+TEST_F(DIBasicTypeTest, getUnspecified) {
   auto *N =
-      MDBasicType::get(Context, dwarf::DW_TAG_unspecified_type, "unspecified");
+      DIBasicType::get(Context, dwarf::DW_TAG_unspecified_type, "unspecified");
   EXPECT_EQ(dwarf::DW_TAG_unspecified_type, N->getTag());
   EXPECT_EQ("unspecified", N->getName());
   EXPECT_EQ(0u, N->getSizeInBits());
@@ -929,46 +943,46 @@ TEST_F(MDBasicTypeTest, getUnspecified) {
   EXPECT_EQ(0u, N->getLine());
 }
 
-typedef MetadataTest MDTypeTest;
+typedef MetadataTest DITypeTest;
 
-TEST_F(MDTypeTest, clone) {
-  // Check that MDType has a specialized clone that returns TempMDType.
-  MDType *N = MDBasicType::get(Context, dwarf::DW_TAG_base_type, "int", 32, 32,
+TEST_F(DITypeTest, clone) {
+  // Check that DIType has a specialized clone that returns TempDIType.
+  DIType *N = DIBasicType::get(Context, dwarf::DW_TAG_base_type, "int", 32, 32,
                                dwarf::DW_ATE_signed);
 
-  TempMDType Temp = N->clone();
+  TempDIType Temp = N->clone();
   EXPECT_EQ(N, MDNode::replaceWithUniqued(std::move(Temp)));
 }
 
-TEST_F(MDTypeTest, setFlags) {
+TEST_F(DITypeTest, setFlags) {
   // void (void)
   Metadata *TypesOps[] = {nullptr};
   Metadata *Types = MDTuple::get(Context, TypesOps);
 
-  MDType *D = MDSubroutineType::getDistinct(Context, 0u, Types);
+  DIType *D = DISubroutineType::getDistinct(Context, 0u, Types);
   EXPECT_EQ(0u, D->getFlags());
-  D->setFlags(DebugNode::FlagRValueReference);
-  EXPECT_EQ(DebugNode::FlagRValueReference, D->getFlags());
+  D->setFlags(DINode::FlagRValueReference);
+  EXPECT_EQ(DINode::FlagRValueReference, D->getFlags());
   D->setFlags(0u);
   EXPECT_EQ(0u, D->getFlags());
 
-  TempMDType T = MDSubroutineType::getTemporary(Context, 0u, Types);
+  TempDIType T = DISubroutineType::getTemporary(Context, 0u, Types);
   EXPECT_EQ(0u, T->getFlags());
-  T->setFlags(DebugNode::FlagRValueReference);
-  EXPECT_EQ(DebugNode::FlagRValueReference, T->getFlags());
+  T->setFlags(DINode::FlagRValueReference);
+  EXPECT_EQ(DINode::FlagRValueReference, T->getFlags());
   T->setFlags(0u);
   EXPECT_EQ(0u, T->getFlags());
 }
 
-typedef MetadataTest MDDerivedTypeTest;
+typedef MetadataTest DIDerivedTypeTest;
 
-TEST_F(MDDerivedTypeTest, get) {
-  MDFile *File = getFile();
-  MDScopeRef Scope = getSubprogramRef();
-  MDTypeRef BaseType = getBasicType("basic");
+TEST_F(DIDerivedTypeTest, get) {
+  DIFile *File = getFile();
+  DIScopeRef Scope = getSubprogramRef();
+  DITypeRef BaseType = getBasicType("basic");
   MDTuple *ExtraData = getTuple();
 
-  auto *N = MDDerivedType::get(Context, dwarf::DW_TAG_pointer_type, "something",
+  auto *N = DIDerivedType::get(Context, dwarf::DW_TAG_pointer_type, "something",
                                File, 1, Scope, BaseType, 2, 3, 4, 5, ExtraData);
   EXPECT_EQ(dwarf::DW_TAG_pointer_type, N->getTag());
   EXPECT_EQ("something", N->getName());
@@ -981,55 +995,55 @@ TEST_F(MDDerivedTypeTest, get) {
   EXPECT_EQ(4u, N->getOffsetInBits());
   EXPECT_EQ(5u, N->getFlags());
   EXPECT_EQ(ExtraData, N->getExtraData());
-  EXPECT_EQ(N, MDDerivedType::get(Context, dwarf::DW_TAG_pointer_type,
+  EXPECT_EQ(N, DIDerivedType::get(Context, dwarf::DW_TAG_pointer_type,
                                   "something", File, 1, Scope, BaseType, 2, 3,
                                   4, 5, ExtraData));
 
-  EXPECT_NE(N, MDDerivedType::get(Context, dwarf::DW_TAG_reference_type,
+  EXPECT_NE(N, DIDerivedType::get(Context, dwarf::DW_TAG_reference_type,
                                   "something", File, 1, Scope, BaseType, 2, 3,
                                   4, 5, ExtraData));
-  EXPECT_NE(N, MDDerivedType::get(Context, dwarf::DW_TAG_pointer_type, "else",
+  EXPECT_NE(N, DIDerivedType::get(Context, dwarf::DW_TAG_pointer_type, "else",
                                   File, 1, Scope, BaseType, 2, 3, 4, 5,
                                   ExtraData));
-  EXPECT_NE(N, MDDerivedType::get(Context, dwarf::DW_TAG_pointer_type,
+  EXPECT_NE(N, DIDerivedType::get(Context, dwarf::DW_TAG_pointer_type,
                                   "something", getFile(), 1, Scope, BaseType, 2,
                                   3, 4, 5, ExtraData));
-  EXPECT_NE(N, MDDerivedType::get(Context, dwarf::DW_TAG_pointer_type,
+  EXPECT_NE(N, DIDerivedType::get(Context, dwarf::DW_TAG_pointer_type,
                                   "something", File, 2, Scope, BaseType, 2, 3,
                                   4, 5, ExtraData));
-  EXPECT_NE(N, MDDerivedType::get(Context, dwarf::DW_TAG_pointer_type,
+  EXPECT_NE(N, DIDerivedType::get(Context, dwarf::DW_TAG_pointer_type,
                                   "something", File, 1, getSubprogramRef(),
                                   BaseType, 2, 3, 4, 5, ExtraData));
-  EXPECT_NE(N, MDDerivedType::get(
+  EXPECT_NE(N, DIDerivedType::get(
                    Context, dwarf::DW_TAG_pointer_type, "something", File, 1,
                    Scope, getBasicType("basic2"), 2, 3, 4, 5, ExtraData));
-  EXPECT_NE(N, MDDerivedType::get(Context, dwarf::DW_TAG_pointer_type,
+  EXPECT_NE(N, DIDerivedType::get(Context, dwarf::DW_TAG_pointer_type,
                                   "something", File, 1, Scope, BaseType, 3, 3,
                                   4, 5, ExtraData));
-  EXPECT_NE(N, MDDerivedType::get(Context, dwarf::DW_TAG_pointer_type,
+  EXPECT_NE(N, DIDerivedType::get(Context, dwarf::DW_TAG_pointer_type,
                                   "something", File, 1, Scope, BaseType, 2, 2,
                                   4, 5, ExtraData));
-  EXPECT_NE(N, MDDerivedType::get(Context, dwarf::DW_TAG_pointer_type,
+  EXPECT_NE(N, DIDerivedType::get(Context, dwarf::DW_TAG_pointer_type,
                                   "something", File, 1, Scope, BaseType, 2, 3,
                                   5, 5, ExtraData));
-  EXPECT_NE(N, MDDerivedType::get(Context, dwarf::DW_TAG_pointer_type,
+  EXPECT_NE(N, DIDerivedType::get(Context, dwarf::DW_TAG_pointer_type,
                                   "something", File, 1, Scope, BaseType, 2, 3,
                                   4, 4, ExtraData));
-  EXPECT_NE(N, MDDerivedType::get(Context, dwarf::DW_TAG_pointer_type,
+  EXPECT_NE(N, DIDerivedType::get(Context, dwarf::DW_TAG_pointer_type,
                                   "something", File, 1, Scope, BaseType, 2, 3,
                                   4, 5, getTuple()));
 
-  TempMDDerivedType Temp = N->clone();
+  TempDIDerivedType Temp = N->clone();
   EXPECT_EQ(N, MDNode::replaceWithUniqued(std::move(Temp)));
 }
 
-TEST_F(MDDerivedTypeTest, getWithLargeValues) {
-  MDFile *File = getFile();
-  MDScopeRef Scope = getSubprogramRef();
-  MDTypeRef BaseType = getBasicType("basic");
+TEST_F(DIDerivedTypeTest, getWithLargeValues) {
+  DIFile *File = getFile();
+  DIScopeRef Scope = getSubprogramRef();
+  DITypeRef BaseType = getBasicType("basic");
   MDTuple *ExtraData = getTuple();
 
-  auto *N = MDDerivedType::get(Context, dwarf::DW_TAG_pointer_type, "something",
+  auto *N = DIDerivedType::get(Context, dwarf::DW_TAG_pointer_type, "something",
                                File, 1, Scope, BaseType, UINT64_MAX,
                                UINT64_MAX - 1, UINT64_MAX - 2, 5, ExtraData);
   EXPECT_EQ(UINT64_MAX, N->getSizeInBits());
@@ -1037,26 +1051,26 @@ TEST_F(MDDerivedTypeTest, getWithLargeValues) {
   EXPECT_EQ(UINT64_MAX - 2, N->getOffsetInBits());
 }
 
-typedef MetadataTest MDCompositeTypeTest;
+typedef MetadataTest DICompositeTypeTest;
 
-TEST_F(MDCompositeTypeTest, get) {
+TEST_F(DICompositeTypeTest, get) {
   unsigned Tag = dwarf::DW_TAG_structure_type;
   StringRef Name = "some name";
-  MDFile *File = getFile();
+  DIFile *File = getFile();
   unsigned Line = 1;
-  MDScopeRef Scope = getSubprogramRef();
-  MDTypeRef BaseType = getCompositeType();
+  DIScopeRef Scope = getSubprogramRef();
+  DITypeRef BaseType = getCompositeType();
   uint64_t SizeInBits = 2;
   uint64_t AlignInBits = 3;
   uint64_t OffsetInBits = 4;
   unsigned Flags = 5;
   MDTuple *Elements = getTuple();
   unsigned RuntimeLang = 6;
-  MDTypeRef VTableHolder = getCompositeType();
+  DITypeRef VTableHolder = getCompositeType();
   MDTuple *TemplateParams = getTuple();
   StringRef Identifier = "some id";
 
-  auto *N = MDCompositeType::get(Context, Tag, Name, File, Line, Scope,
+  auto *N = DICompositeType::get(Context, Tag, Name, File, Line, Scope,
                                  BaseType, SizeInBits, AlignInBits,
                                  OffsetInBits, Flags, Elements, RuntimeLang,
                                  VTableHolder, TemplateParams, Identifier);
@@ -1076,104 +1090,106 @@ TEST_F(MDCompositeTypeTest, get) {
   EXPECT_EQ(TemplateParams, N->getTemplateParams().get());
   EXPECT_EQ(Identifier, N->getIdentifier());
 
-  EXPECT_EQ(N, MDCompositeType::get(Context, Tag, Name, File, Line, Scope,
+  EXPECT_EQ(N, DICompositeType::get(Context, Tag, Name, File, Line, Scope,
                                     BaseType, SizeInBits, AlignInBits,
                                     OffsetInBits, Flags, Elements, RuntimeLang,
                                     VTableHolder, TemplateParams, Identifier));
 
-  EXPECT_NE(N, MDCompositeType::get(Context, Tag + 1, Name, File, Line, Scope,
+  EXPECT_NE(N, DICompositeType::get(Context, Tag + 1, Name, File, Line, Scope,
                                     BaseType, SizeInBits, AlignInBits,
                                     OffsetInBits, Flags, Elements, RuntimeLang,
                                     VTableHolder, TemplateParams, Identifier));
-  EXPECT_NE(N, MDCompositeType::get(Context, Tag, "abc", File, Line, Scope,
+  EXPECT_NE(N, DICompositeType::get(Context, Tag, "abc", File, Line, Scope,
                                     BaseType, SizeInBits, AlignInBits,
                                     OffsetInBits, Flags, Elements, RuntimeLang,
                                     VTableHolder, TemplateParams, Identifier));
-  EXPECT_NE(N, MDCompositeType::get(Context, Tag, Name, getFile(), Line, Scope,
+  EXPECT_NE(N, DICompositeType::get(Context, Tag, Name, getFile(), Line, Scope,
                                     BaseType, SizeInBits, AlignInBits,
                                     OffsetInBits, Flags, Elements, RuntimeLang,
                                     VTableHolder, TemplateParams, Identifier));
-  EXPECT_NE(N, MDCompositeType::get(Context, Tag, Name, File, Line + 1, Scope,
+  EXPECT_NE(N, DICompositeType::get(Context, Tag, Name, File, Line + 1, Scope,
                                     BaseType, SizeInBits, AlignInBits,
                                     OffsetInBits, Flags, Elements, RuntimeLang,
                                     VTableHolder, TemplateParams, Identifier));
-  EXPECT_NE(N, MDCompositeType::get(
+  EXPECT_NE(N, DICompositeType::get(
                    Context, Tag, Name, File, Line, getSubprogramRef(), BaseType,
                    SizeInBits, AlignInBits, OffsetInBits, Flags, Elements,
                    RuntimeLang, VTableHolder, TemplateParams, Identifier));
-  EXPECT_NE(N, MDCompositeType::get(
+  EXPECT_NE(N, DICompositeType::get(
                    Context, Tag, Name, File, Line, Scope, getBasicType("other"),
                    SizeInBits, AlignInBits, OffsetInBits, Flags, Elements,
                    RuntimeLang, VTableHolder, TemplateParams, Identifier));
-  EXPECT_NE(N, MDCompositeType::get(Context, Tag, Name, File, Line, Scope,
+  EXPECT_NE(N, DICompositeType::get(Context, Tag, Name, File, Line, Scope,
                                     BaseType, SizeInBits + 1, AlignInBits,
                                     OffsetInBits, Flags, Elements, RuntimeLang,
                                     VTableHolder, TemplateParams, Identifier));
-  EXPECT_NE(N, MDCompositeType::get(Context, Tag, Name, File, Line, Scope,
+  EXPECT_NE(N, DICompositeType::get(Context, Tag, Name, File, Line, Scope,
                                     BaseType, SizeInBits, AlignInBits + 1,
                                     OffsetInBits, Flags, Elements, RuntimeLang,
                                     VTableHolder, TemplateParams, Identifier));
-  EXPECT_NE(N, MDCompositeType::get(
+  EXPECT_NE(N, DICompositeType::get(
                    Context, Tag, Name, File, Line, Scope, BaseType, SizeInBits,
                    AlignInBits, OffsetInBits + 1, Flags, Elements, RuntimeLang,
                    VTableHolder, TemplateParams, Identifier));
-  EXPECT_NE(N, MDCompositeType::get(
+  EXPECT_NE(N, DICompositeType::get(
                    Context, Tag, Name, File, Line, Scope, BaseType, SizeInBits,
                    AlignInBits, OffsetInBits, Flags + 1, Elements, RuntimeLang,
                    VTableHolder, TemplateParams, Identifier));
-  EXPECT_NE(N, MDCompositeType::get(
+  EXPECT_NE(N, DICompositeType::get(
                    Context, Tag, Name, File, Line, Scope, BaseType, SizeInBits,
                    AlignInBits, OffsetInBits, Flags, getTuple(), RuntimeLang,
                    VTableHolder, TemplateParams, Identifier));
-  EXPECT_NE(N, MDCompositeType::get(
+  EXPECT_NE(N, DICompositeType::get(
                    Context, Tag, Name, File, Line, Scope, BaseType, SizeInBits,
                    AlignInBits, OffsetInBits, Flags, Elements, RuntimeLang + 1,
                    VTableHolder, TemplateParams, Identifier));
-  EXPECT_NE(N, MDCompositeType::get(
+  EXPECT_NE(N, DICompositeType::get(
                    Context, Tag, Name, File, Line, Scope, BaseType, SizeInBits,
                    AlignInBits, OffsetInBits, Flags, Elements, RuntimeLang,
                    getCompositeType(), TemplateParams, Identifier));
-  EXPECT_NE(N, MDCompositeType::get(Context, Tag, Name, File, Line, Scope,
+  EXPECT_NE(N, DICompositeType::get(Context, Tag, Name, File, Line, Scope,
                                     BaseType, SizeInBits, AlignInBits,
                                     OffsetInBits, Flags, Elements, RuntimeLang,
                                     VTableHolder, getTuple(), Identifier));
-  EXPECT_NE(N, MDCompositeType::get(Context, Tag, Name, File, Line, Scope,
+  EXPECT_NE(N, DICompositeType::get(Context, Tag, Name, File, Line, Scope,
                                     BaseType, SizeInBits, AlignInBits,
                                     OffsetInBits, Flags, Elements, RuntimeLang,
                                     VTableHolder, TemplateParams, "other"));
 
   // Be sure that missing identifiers get null pointers.
-  EXPECT_FALSE(MDCompositeType::get(
-                   Context, Tag, Name, File, Line, Scope, BaseType, SizeInBits,
-                   AlignInBits, OffsetInBits, Flags, Elements, RuntimeLang,
-                   VTableHolder, TemplateParams, "")->getRawIdentifier());
-  EXPECT_FALSE(MDCompositeType::get(
-                   Context, Tag, Name, File, Line, Scope, BaseType, SizeInBits,
-                   AlignInBits, OffsetInBits, Flags, Elements, RuntimeLang,
-                   VTableHolder, TemplateParams)->getRawIdentifier());
+  EXPECT_FALSE(DICompositeType::get(Context, Tag, Name, File, Line, Scope,
+                                    BaseType, SizeInBits, AlignInBits,
+                                    OffsetInBits, Flags, Elements, RuntimeLang,
+                                    VTableHolder, TemplateParams, "")
+                   ->getRawIdentifier());
+  EXPECT_FALSE(DICompositeType::get(Context, Tag, Name, File, Line, Scope,
+                                    BaseType, SizeInBits, AlignInBits,
+                                    OffsetInBits, Flags, Elements, RuntimeLang,
+                                    VTableHolder, TemplateParams)
+                   ->getRawIdentifier());
 
-  TempMDCompositeType Temp = N->clone();
+  TempDICompositeType Temp = N->clone();
   EXPECT_EQ(N, MDNode::replaceWithUniqued(std::move(Temp)));
 }
 
-TEST_F(MDCompositeTypeTest, getWithLargeValues) {
+TEST_F(DICompositeTypeTest, getWithLargeValues) {
   unsigned Tag = dwarf::DW_TAG_structure_type;
   StringRef Name = "some name";
-  MDFile *File = getFile();
+  DIFile *File = getFile();
   unsigned Line = 1;
-  MDScopeRef Scope = getSubprogramRef();
-  MDTypeRef BaseType = getCompositeType();
+  DIScopeRef Scope = getSubprogramRef();
+  DITypeRef BaseType = getCompositeType();
   uint64_t SizeInBits = UINT64_MAX;
   uint64_t AlignInBits = UINT64_MAX - 1;
   uint64_t OffsetInBits = UINT64_MAX - 2;
   unsigned Flags = 5;
   MDTuple *Elements = getTuple();
   unsigned RuntimeLang = 6;
-  MDTypeRef VTableHolder = getCompositeType();
+  DITypeRef VTableHolder = getCompositeType();
   MDTuple *TemplateParams = getTuple();
   StringRef Identifier = "some id";
 
-  auto *N = MDCompositeType::get(Context, Tag, Name, File, Line, Scope,
+  auto *N = DICompositeType::get(Context, Tag, Name, File, Line, Scope,
                                  BaseType, SizeInBits, AlignInBits,
                                  OffsetInBits, Flags, Elements, RuntimeLang,
                                  VTableHolder, TemplateParams, Identifier);
@@ -1182,13 +1198,13 @@ TEST_F(MDCompositeTypeTest, getWithLargeValues) {
   EXPECT_EQ(OffsetInBits, N->getOffsetInBits());
 }
 
-TEST_F(MDCompositeTypeTest, replaceOperands) {
+TEST_F(DICompositeTypeTest, replaceOperands) {
   unsigned Tag = dwarf::DW_TAG_structure_type;
   StringRef Name = "some name";
-  MDFile *File = getFile();
+  DIFile *File = getFile();
   unsigned Line = 1;
-  MDScopeRef Scope = getSubprogramRef();
-  MDTypeRef BaseType = getCompositeType();
+  DIScopeRef Scope = getSubprogramRef();
+  DITypeRef BaseType = getCompositeType();
   uint64_t SizeInBits = 2;
   uint64_t AlignInBits = 3;
   uint64_t OffsetInBits = 4;
@@ -1196,10 +1212,9 @@ TEST_F(MDCompositeTypeTest, replaceOperands) {
   unsigned RuntimeLang = 6;
   StringRef Identifier = "some id";
 
-  auto *N = MDCompositeType::get(Context, Tag, Name, File, Line, Scope,
-                                 BaseType, SizeInBits, AlignInBits,
-                                 OffsetInBits, Flags, nullptr, RuntimeLang,
-                                 nullptr, nullptr, Identifier);
+  auto *N = DICompositeType::get(
+      Context, Tag, Name, File, Line, Scope, BaseType, SizeInBits, AlignInBits,
+      OffsetInBits, Flags, nullptr, RuntimeLang, nullptr, nullptr, Identifier);
 
   auto *Elements = MDTuple::getDistinct(Context, None);
   EXPECT_EQ(nullptr, N->getElements().get());
@@ -1208,7 +1223,7 @@ TEST_F(MDCompositeTypeTest, replaceOperands) {
   N->replaceElements(nullptr);
   EXPECT_EQ(nullptr, N->getElements().get());
 
-  MDTypeRef VTableHolder = getCompositeType();
+  DITypeRef VTableHolder = getCompositeType();
   EXPECT_EQ(nullptr, N->getVTableHolder());
   N->replaceVTableHolder(VTableHolder);
   EXPECT_EQ(VTableHolder, N->getVTableHolder());
@@ -1223,64 +1238,60 @@ TEST_F(MDCompositeTypeTest, replaceOperands) {
   EXPECT_EQ(nullptr, N->getTemplateParams().get());
 }
 
-typedef MetadataTest MDSubroutineTypeTest;
+typedef MetadataTest DISubroutineTypeTest;
 
-TEST_F(MDSubroutineTypeTest, get) {
+TEST_F(DISubroutineTypeTest, get) {
   unsigned Flags = 1;
   MDTuple *TypeArray = getTuple();
 
-  auto *N = MDSubroutineType::get(Context, Flags, TypeArray);
+  auto *N = DISubroutineType::get(Context, Flags, TypeArray);
   EXPECT_EQ(dwarf::DW_TAG_subroutine_type, N->getTag());
   EXPECT_EQ(Flags, N->getFlags());
   EXPECT_EQ(TypeArray, N->getTypeArray().get());
-  EXPECT_EQ(N, MDSubroutineType::get(Context, Flags, TypeArray));
+  EXPECT_EQ(N, DISubroutineType::get(Context, Flags, TypeArray));
 
-  EXPECT_NE(N, MDSubroutineType::get(Context, Flags + 1, TypeArray));
-  EXPECT_NE(N, MDSubroutineType::get(Context, Flags, getTuple()));
+  EXPECT_NE(N, DISubroutineType::get(Context, Flags + 1, TypeArray));
+  EXPECT_NE(N, DISubroutineType::get(Context, Flags, getTuple()));
 
-  TempMDSubroutineType Temp = N->clone();
+  TempDISubroutineType Temp = N->clone();
   EXPECT_EQ(N, MDNode::replaceWithUniqued(std::move(Temp)));
 
   // Test always-empty operands.
   EXPECT_EQ(nullptr, N->getScope());
   EXPECT_EQ(nullptr, N->getFile());
   EXPECT_EQ("", N->getName());
-  EXPECT_EQ(nullptr, N->getBaseType());
-  EXPECT_EQ(nullptr, N->getVTableHolder());
-  EXPECT_EQ(nullptr, N->getTemplateParams().get());
-  EXPECT_EQ("", N->getIdentifier());
 }
 
-typedef MetadataTest MDFileTest;
+typedef MetadataTest DIFileTest;
 
-TEST_F(MDFileTest, get) {
+TEST_F(DIFileTest, get) {
   StringRef Filename = "file";
   StringRef Directory = "dir";
-  auto *N = MDFile::get(Context, Filename, Directory);
+  auto *N = DIFile::get(Context, Filename, Directory);
 
   EXPECT_EQ(dwarf::DW_TAG_file_type, N->getTag());
   EXPECT_EQ(Filename, N->getFilename());
   EXPECT_EQ(Directory, N->getDirectory());
-  EXPECT_EQ(N, MDFile::get(Context, Filename, Directory));
+  EXPECT_EQ(N, DIFile::get(Context, Filename, Directory));
 
-  EXPECT_NE(N, MDFile::get(Context, "other", Directory));
-  EXPECT_NE(N, MDFile::get(Context, Filename, "other"));
+  EXPECT_NE(N, DIFile::get(Context, "other", Directory));
+  EXPECT_NE(N, DIFile::get(Context, Filename, "other"));
 
-  TempMDFile Temp = N->clone();
+  TempDIFile Temp = N->clone();
   EXPECT_EQ(N, MDNode::replaceWithUniqued(std::move(Temp)));
 }
 
-TEST_F(MDFileTest, ScopeGetFile) {
-  // Ensure that MDScope::getFile() returns itself.
-  MDScope *N = MDFile::get(Context, "file", "dir");
+TEST_F(DIFileTest, ScopeGetFile) {
+  // Ensure that DIScope::getFile() returns itself.
+  DIScope *N = DIFile::get(Context, "file", "dir");
   EXPECT_EQ(N, N->getFile());
 }
 
-typedef MetadataTest MDCompileUnitTest;
+typedef MetadataTest DICompileUnitTest;
 
-TEST_F(MDCompileUnitTest, get) {
+TEST_F(DICompileUnitTest, get) {
   unsigned SourceLanguage = 1;
-  MDFile *File = getFile();
+  DIFile *File = getFile();
   StringRef Producer = "some producer";
   bool IsOptimized = false;
   StringRef Flags = "flag after flag";
@@ -1292,10 +1303,11 @@ TEST_F(MDCompileUnitTest, get) {
   MDTuple *Subprograms = getTuple();
   MDTuple *GlobalVariables = getTuple();
   MDTuple *ImportedEntities = getTuple();
-  auto *N = MDCompileUnit::get(
+  uint64_t DWOId = 0xc0ffee;
+  auto *N = DICompileUnit::get(
       Context, SourceLanguage, File, Producer, IsOptimized, Flags,
       RuntimeVersion, SplitDebugFilename, EmissionKind, EnumTypes,
-      RetainedTypes, Subprograms, GlobalVariables, ImportedEntities);
+      RetainedTypes, Subprograms, GlobalVariables, ImportedEntities, DWOId);
 
   EXPECT_EQ(dwarf::DW_TAG_compile_unit, N->getTag());
   EXPECT_EQ(SourceLanguage, N->getSourceLanguage());
@@ -1311,82 +1323,91 @@ TEST_F(MDCompileUnitTest, get) {
   EXPECT_EQ(Subprograms, N->getSubprograms().get());
   EXPECT_EQ(GlobalVariables, N->getGlobalVariables().get());
   EXPECT_EQ(ImportedEntities, N->getImportedEntities().get());
-  EXPECT_EQ(N, MDCompileUnit::get(Context, SourceLanguage, File, Producer,
+  EXPECT_EQ(DWOId, N->getDWOId());
+  EXPECT_EQ(N, DICompileUnit::get(Context, SourceLanguage, File, Producer,
                                   IsOptimized, Flags, RuntimeVersion,
                                   SplitDebugFilename, EmissionKind, EnumTypes,
                                   RetainedTypes, Subprograms, GlobalVariables,
-                                  ImportedEntities));
+                                  ImportedEntities, DWOId));
 
-  EXPECT_NE(N, MDCompileUnit::get(Context, SourceLanguage + 1, File, Producer,
+  EXPECT_NE(N, DICompileUnit::get(Context, SourceLanguage + 1, File, Producer,
                                   IsOptimized, Flags, RuntimeVersion,
                                   SplitDebugFilename, EmissionKind, EnumTypes,
                                   RetainedTypes, Subprograms, GlobalVariables,
-                                  ImportedEntities));
-  EXPECT_NE(N, MDCompileUnit::get(Context, SourceLanguage, getFile(), Producer,
+                                  ImportedEntities, DWOId));
+  EXPECT_NE(N, DICompileUnit::get(Context, SourceLanguage, getFile(), Producer,
                                   IsOptimized, Flags, RuntimeVersion,
                                   SplitDebugFilename, EmissionKind, EnumTypes,
                                   RetainedTypes, Subprograms, GlobalVariables,
-                                  ImportedEntities));
-  EXPECT_NE(N, MDCompileUnit::get(Context, SourceLanguage, File, "other",
+                                  ImportedEntities, DWOId));
+  EXPECT_NE(N, DICompileUnit::get(Context, SourceLanguage, File, "other",
                                   IsOptimized, Flags, RuntimeVersion,
                                   SplitDebugFilename, EmissionKind, EnumTypes,
                                   RetainedTypes, Subprograms, GlobalVariables,
-                                  ImportedEntities));
-  EXPECT_NE(N, MDCompileUnit::get(Context, SourceLanguage, File, Producer,
+                                  ImportedEntities, DWOId));
+  EXPECT_NE(N, DICompileUnit::get(Context, SourceLanguage, File, Producer,
                                   !IsOptimized, Flags, RuntimeVersion,
                                   SplitDebugFilename, EmissionKind, EnumTypes,
                                   RetainedTypes, Subprograms, GlobalVariables,
-                                  ImportedEntities));
-  EXPECT_NE(N, MDCompileUnit::get(Context, SourceLanguage, File, Producer,
+                                  ImportedEntities, DWOId));
+  EXPECT_NE(N, DICompileUnit::get(Context, SourceLanguage, File, Producer,
                                   IsOptimized, "other", RuntimeVersion,
                                   SplitDebugFilename, EmissionKind, EnumTypes,
                                   RetainedTypes, Subprograms, GlobalVariables,
-                                  ImportedEntities));
-  EXPECT_NE(N, MDCompileUnit::get(Context, SourceLanguage, File, Producer,
+                                  ImportedEntities, DWOId));
+  EXPECT_NE(N, DICompileUnit::get(Context, SourceLanguage, File, Producer,
                                   IsOptimized, Flags, RuntimeVersion + 1,
                                   SplitDebugFilename, EmissionKind, EnumTypes,
                                   RetainedTypes, Subprograms, GlobalVariables,
-                                  ImportedEntities));
-  EXPECT_NE(N,
-            MDCompileUnit::get(Context, SourceLanguage, File, Producer,
-                               IsOptimized, Flags, RuntimeVersion, "other",
-                               EmissionKind, EnumTypes, RetainedTypes,
-                               Subprograms, GlobalVariables, ImportedEntities));
-  EXPECT_NE(N, MDCompileUnit::get(Context, SourceLanguage, File, Producer,
+                                  ImportedEntities, DWOId));
+  EXPECT_NE(N, DICompileUnit::get(Context, SourceLanguage, File, Producer,
+                                  IsOptimized, Flags, RuntimeVersion, "other",
+                                  EmissionKind, EnumTypes, RetainedTypes,
+                                  Subprograms, GlobalVariables,
+                                  ImportedEntities, DWOId));
+  EXPECT_NE(N, DICompileUnit::get(Context, SourceLanguage, File, Producer,
                                   IsOptimized, Flags, RuntimeVersion,
                                   SplitDebugFilename, EmissionKind + 1,
                                   EnumTypes, RetainedTypes, Subprograms,
-                                  GlobalVariables, ImportedEntities));
-  EXPECT_NE(N, MDCompileUnit::get(Context, SourceLanguage, File, Producer,
+                                  GlobalVariables, ImportedEntities, DWOId));
+  EXPECT_NE(N, DICompileUnit::get(Context, SourceLanguage, File, Producer,
                                   IsOptimized, Flags, RuntimeVersion,
                                   SplitDebugFilename, EmissionKind, getTuple(),
                                   RetainedTypes, Subprograms, GlobalVariables,
-                                  ImportedEntities));
-  EXPECT_NE(N, MDCompileUnit::get(
-                   Context, SourceLanguage, File, Producer, IsOptimized, Flags,
-                   RuntimeVersion, SplitDebugFilename, EmissionKind, EnumTypes,
-                   getTuple(), Subprograms, GlobalVariables, ImportedEntities));
-  EXPECT_NE(N, MDCompileUnit::get(Context, SourceLanguage, File, Producer,
+                                  ImportedEntities, DWOId));
+  EXPECT_NE(N, DICompileUnit::get(Context, SourceLanguage, File, Producer,
+                                  IsOptimized, Flags, RuntimeVersion,
+                                  SplitDebugFilename, EmissionKind, EnumTypes,
+                                  getTuple(), Subprograms, GlobalVariables,
+                                  ImportedEntities, DWOId));
+  EXPECT_NE(N, DICompileUnit::get(Context, SourceLanguage, File, Producer,
                                   IsOptimized, Flags, RuntimeVersion,
                                   SplitDebugFilename, EmissionKind, EnumTypes,
                                   RetainedTypes, getTuple(), GlobalVariables,
-                                  ImportedEntities));
-  EXPECT_NE(N, MDCompileUnit::get(
-                   Context, SourceLanguage, File, Producer, IsOptimized, Flags,
-                   RuntimeVersion, SplitDebugFilename, EmissionKind, EnumTypes,
-                   RetainedTypes, Subprograms, getTuple(), ImportedEntities));
-  EXPECT_NE(N, MDCompileUnit::get(
-                   Context, SourceLanguage, File, Producer, IsOptimized, Flags,
-                   RuntimeVersion, SplitDebugFilename, EmissionKind, EnumTypes,
-                   RetainedTypes, Subprograms, GlobalVariables, getTuple()));
-
-  TempMDCompileUnit Temp = N->clone();
+                                  ImportedEntities, DWOId));
+  EXPECT_NE(N, DICompileUnit::get(Context, SourceLanguage, File, Producer,
+                                  IsOptimized, Flags, RuntimeVersion,
+                                  SplitDebugFilename, EmissionKind, EnumTypes,
+                                  RetainedTypes, Subprograms, getTuple(),
+                                  ImportedEntities, DWOId));
+  EXPECT_NE(N, DICompileUnit::get(Context, SourceLanguage, File, Producer,
+                                  IsOptimized, Flags, RuntimeVersion,
+                                  SplitDebugFilename, EmissionKind, EnumTypes,
+                                  RetainedTypes, Subprograms, GlobalVariables,
+                                  getTuple(), DWOId));
+  EXPECT_NE(N, DICompileUnit::get(Context, SourceLanguage, File, Producer,
+                                  IsOptimized, Flags, RuntimeVersion,
+                                  SplitDebugFilename, EmissionKind, EnumTypes,
+                                  RetainedTypes, Subprograms, GlobalVariables,
+                                  ImportedEntities, DWOId + 1));
+
+  TempDICompileUnit Temp = N->clone();
   EXPECT_EQ(N, MDNode::replaceWithUniqued(std::move(Temp)));
 }
 
-TEST_F(MDCompileUnitTest, replaceArrays) {
+TEST_F(DICompileUnitTest, replaceArrays) {
   unsigned SourceLanguage = 1;
-  MDFile *File = getFile();
+  DIFile *File = getFile();
   StringRef Producer = "some producer";
   bool IsOptimized = false;
   StringRef Flags = "flag after flag";
@@ -1396,10 +1417,11 @@ TEST_F(MDCompileUnitTest, replaceArrays) {
   MDTuple *EnumTypes = MDTuple::getDistinct(Context, None);
   MDTuple *RetainedTypes = MDTuple::getDistinct(Context, None);
   MDTuple *ImportedEntities = MDTuple::getDistinct(Context, None);
-  auto *N = MDCompileUnit::get(
+  uint64_t DWOId = 0xc0ffee;
+  auto *N = DICompileUnit::get(
       Context, SourceLanguage, File, Producer, IsOptimized, Flags,
       RuntimeVersion, SplitDebugFilename, EmissionKind, EnumTypes,
-      RetainedTypes, nullptr, nullptr, ImportedEntities);
+      RetainedTypes, nullptr, nullptr, ImportedEntities, DWOId);
 
   auto *Subprograms = MDTuple::getDistinct(Context, None);
   EXPECT_EQ(nullptr, N->getSubprograms().get());
@@ -1416,29 +1438,29 @@ TEST_F(MDCompileUnitTest, replaceArrays) {
   EXPECT_EQ(nullptr, N->getGlobalVariables().get());
 }
 
-typedef MetadataTest MDSubprogramTest;
+typedef MetadataTest DISubprogramTest;
 
-TEST_F(MDSubprogramTest, get) {
-  MDScopeRef Scope = getCompositeType();
+TEST_F(DISubprogramTest, get) {
+  DIScopeRef Scope = getCompositeType();
   StringRef Name = "name";
   StringRef LinkageName = "linkage";
-  MDFile *File = getFile();
+  DIFile *File = getFile();
   unsigned Line = 2;
-  MDSubroutineType *Type = getSubroutineType();
+  DISubroutineType *Type = getSubroutineType();
   bool IsLocalToUnit = false;
   bool IsDefinition = true;
   unsigned ScopeLine = 3;
-  MDTypeRef ContainingType = getCompositeType();
+  DITypeRef ContainingType = getCompositeType();
   unsigned Virtuality = 4;
   unsigned VirtualIndex = 5;
   unsigned Flags = 6;
   bool IsOptimized = false;
   llvm::Function *Function = getFunction("foo");
   MDTuple *TemplateParams = getTuple();
-  MDSubprogram *Declaration = getSubprogram();
+  DISubprogram *Declaration = getSubprogram();
   MDTuple *Variables = getTuple();
 
-  auto *N = MDSubprogram::get(
+  auto *N = DISubprogram::get(
       Context, Scope, Name, LinkageName, File, Line, Type, IsLocalToUnit,
       IsDefinition, ScopeLine, ContainingType, Virtuality, VirtualIndex, Flags,
       IsOptimized, Function, TemplateParams, Declaration, Variables);
@@ -1462,127 +1484,127 @@ TEST_F(MDSubprogramTest, get) {
   EXPECT_EQ(TemplateParams, N->getTemplateParams().get());
   EXPECT_EQ(Declaration, N->getDeclaration());
   EXPECT_EQ(Variables, N->getVariables().get());
-  EXPECT_EQ(N, MDSubprogram::get(Context, Scope, Name, LinkageName, File, Line,
+  EXPECT_EQ(N, DISubprogram::get(Context, Scope, Name, LinkageName, File, Line,
                                  Type, IsLocalToUnit, IsDefinition, ScopeLine,
                                  ContainingType, Virtuality, VirtualIndex,
                                  Flags, IsOptimized, Function, TemplateParams,
                                  Declaration, Variables));
 
-  EXPECT_NE(N, MDSubprogram::get(Context, getCompositeType(), Name, LinkageName,
+  EXPECT_NE(N, DISubprogram::get(Context, getCompositeType(), Name, LinkageName,
                                  File, Line, Type, IsLocalToUnit, IsDefinition,
                                  ScopeLine, ContainingType, Virtuality,
                                  VirtualIndex, Flags, IsOptimized, Function,
                                  TemplateParams, Declaration, Variables));
-  EXPECT_NE(N, MDSubprogram::get(Context, Scope, "other", LinkageName, File,
+  EXPECT_NE(N, DISubprogram::get(Context, Scope, "other", LinkageName, File,
                                  Line, Type, IsLocalToUnit, IsDefinition,
                                  ScopeLine, ContainingType, Virtuality,
                                  VirtualIndex, Flags, IsOptimized, Function,
                                  TemplateParams, Declaration, Variables));
-  EXPECT_NE(N, MDSubprogram::get(Context, Scope, Name, "other", File, Line,
+  EXPECT_NE(N, DISubprogram::get(Context, Scope, Name, "other", File, Line,
                                  Type, IsLocalToUnit, IsDefinition, ScopeLine,
                                  ContainingType, Virtuality, VirtualIndex,
                                  Flags, IsOptimized, Function, TemplateParams,
                                  Declaration, Variables));
-  EXPECT_NE(N, MDSubprogram::get(Context, Scope, Name, LinkageName, getFile(),
+  EXPECT_NE(N, DISubprogram::get(Context, Scope, Name, LinkageName, getFile(),
                                  Line, Type, IsLocalToUnit, IsDefinition,
                                  ScopeLine, ContainingType, Virtuality,
                                  VirtualIndex, Flags, IsOptimized, Function,
                                  TemplateParams, Declaration, Variables));
-  EXPECT_NE(N, MDSubprogram::get(Context, Scope, Name, LinkageName, File,
+  EXPECT_NE(N, DISubprogram::get(Context, Scope, Name, LinkageName, File,
                                  Line + 1, Type, IsLocalToUnit, IsDefinition,
                                  ScopeLine, ContainingType, Virtuality,
                                  VirtualIndex, Flags, IsOptimized, Function,
                                  TemplateParams, Declaration, Variables));
-  EXPECT_NE(N, MDSubprogram::get(
+  EXPECT_NE(N, DISubprogram::get(
                    Context, Scope, Name, LinkageName, File, Line,
                    getSubroutineType(), IsLocalToUnit, IsDefinition, ScopeLine,
                    ContainingType, Virtuality, VirtualIndex, Flags, IsOptimized,
                    Function, TemplateParams, Declaration, Variables));
-  EXPECT_NE(N, MDSubprogram::get(Context, Scope, Name, LinkageName, File, Line,
+  EXPECT_NE(N, DISubprogram::get(Context, Scope, Name, LinkageName, File, Line,
                                  Type, !IsLocalToUnit, IsDefinition, ScopeLine,
                                  ContainingType, Virtuality, VirtualIndex,
                                  Flags, IsOptimized, Function, TemplateParams,
                                  Declaration, Variables));
-  EXPECT_NE(N, MDSubprogram::get(Context, Scope, Name, LinkageName, File, Line,
+  EXPECT_NE(N, DISubprogram::get(Context, Scope, Name, LinkageName, File, Line,
                                  Type, IsLocalToUnit, !IsDefinition, ScopeLine,
                                  ContainingType, Virtuality, VirtualIndex,
                                  Flags, IsOptimized, Function, TemplateParams,
                                  Declaration, Variables));
-  EXPECT_NE(N, MDSubprogram::get(Context, Scope, Name, LinkageName, File, Line,
+  EXPECT_NE(N, DISubprogram::get(Context, Scope, Name, LinkageName, File, Line,
                                  Type, IsLocalToUnit, IsDefinition,
                                  ScopeLine + 1, ContainingType, Virtuality,
                                  VirtualIndex, Flags, IsOptimized, Function,
                                  TemplateParams, Declaration, Variables));
-  EXPECT_NE(N, MDSubprogram::get(Context, Scope, Name, LinkageName, File, Line,
+  EXPECT_NE(N, DISubprogram::get(Context, Scope, Name, LinkageName, File, Line,
                                  Type, IsLocalToUnit, IsDefinition, ScopeLine,
                                  getCompositeType(), Virtuality, VirtualIndex,
                                  Flags, IsOptimized, Function, TemplateParams,
                                  Declaration, Variables));
-  EXPECT_NE(N, MDSubprogram::get(Context, Scope, Name, LinkageName, File, Line,
+  EXPECT_NE(N, DISubprogram::get(Context, Scope, Name, LinkageName, File, Line,
                                  Type, IsLocalToUnit, IsDefinition, ScopeLine,
                                  ContainingType, Virtuality + 1, VirtualIndex,
                                  Flags, IsOptimized, Function, TemplateParams,
                                  Declaration, Variables));
-  EXPECT_NE(N, MDSubprogram::get(Context, Scope, Name, LinkageName, File, Line,
+  EXPECT_NE(N, DISubprogram::get(Context, Scope, Name, LinkageName, File, Line,
                                  Type, IsLocalToUnit, IsDefinition, ScopeLine,
                                  ContainingType, Virtuality, VirtualIndex + 1,
                                  Flags, IsOptimized, Function, TemplateParams,
                                  Declaration, Variables));
-  EXPECT_NE(N, MDSubprogram::get(Context, Scope, Name, LinkageName, File, Line,
+  EXPECT_NE(N, DISubprogram::get(Context, Scope, Name, LinkageName, File, Line,
                                  Type, IsLocalToUnit, IsDefinition, ScopeLine,
                                  ContainingType, Virtuality, VirtualIndex,
                                  ~Flags, IsOptimized, Function, TemplateParams,
                                  Declaration, Variables));
-  EXPECT_NE(N, MDSubprogram::get(Context, Scope, Name, LinkageName, File, Line,
+  EXPECT_NE(N, DISubprogram::get(Context, Scope, Name, LinkageName, File, Line,
                                  Type, IsLocalToUnit, IsDefinition, ScopeLine,
                                  ContainingType, Virtuality, VirtualIndex,
                                  Flags, !IsOptimized, Function, TemplateParams,
                                  Declaration, Variables));
-  EXPECT_NE(N, MDSubprogram::get(Context, Scope, Name, LinkageName, File, Line,
+  EXPECT_NE(N, DISubprogram::get(Context, Scope, Name, LinkageName, File, Line,
                                  Type, IsLocalToUnit, IsDefinition, ScopeLine,
                                  ContainingType, Virtuality, VirtualIndex,
                                  Flags, IsOptimized, getFunction("bar"),
                                  TemplateParams, Declaration, Variables));
-  EXPECT_NE(N, MDSubprogram::get(Context, Scope, Name, LinkageName, File, Line,
+  EXPECT_NE(N, DISubprogram::get(Context, Scope, Name, LinkageName, File, Line,
                                  Type, IsLocalToUnit, IsDefinition, ScopeLine,
                                  ContainingType, Virtuality, VirtualIndex,
                                  Flags, IsOptimized, Function, getTuple(),
                                  Declaration, Variables));
-  EXPECT_NE(N, MDSubprogram::get(Context, Scope, Name, LinkageName, File, Line,
+  EXPECT_NE(N, DISubprogram::get(Context, Scope, Name, LinkageName, File, Line,
                                  Type, IsLocalToUnit, IsDefinition, ScopeLine,
                                  ContainingType, Virtuality, VirtualIndex,
                                  Flags, IsOptimized, Function, TemplateParams,
                                  getSubprogram(), Variables));
-  EXPECT_NE(N, MDSubprogram::get(Context, Scope, Name, LinkageName, File, Line,
+  EXPECT_NE(N, DISubprogram::get(Context, Scope, Name, LinkageName, File, Line,
                                  Type, IsLocalToUnit, IsDefinition, ScopeLine,
                                  ContainingType, Virtuality, VirtualIndex,
                                  Flags, IsOptimized, Function, TemplateParams,
                                  Declaration, getTuple()));
 
-  TempMDSubprogram Temp = N->clone();
+  TempDISubprogram Temp = N->clone();
   EXPECT_EQ(N, MDNode::replaceWithUniqued(std::move(Temp)));
 }
 
-TEST_F(MDSubprogramTest, replaceFunction) {
-  MDScopeRef Scope = getCompositeType();
+TEST_F(DISubprogramTest, replaceFunction) {
+  DIScopeRef Scope = getCompositeType();
   StringRef Name = "name";
   StringRef LinkageName = "linkage";
-  MDFile *File = getFile();
+  DIFile *File = getFile();
   unsigned Line = 2;
-  MDSubroutineType *Type = getSubroutineType();
+  DISubroutineType *Type = getSubroutineType();
   bool IsLocalToUnit = false;
   bool IsDefinition = true;
   unsigned ScopeLine = 3;
-  MDTypeRef ContainingType = getCompositeType();
+  DITypeRef ContainingType = getCompositeType();
   unsigned Virtuality = 4;
   unsigned VirtualIndex = 5;
   unsigned Flags = 6;
   bool IsOptimized = false;
   MDTuple *TemplateParams = getTuple();
-  MDSubprogram *Declaration = getSubprogram();
+  DISubprogram *Declaration = getSubprogram();
   MDTuple *Variables = getTuple();
 
-  auto *N = MDSubprogram::get(
+  auto *N = DISubprogram::get(
       Context, Scope, Name, LinkageName, File, Line, Type, IsLocalToUnit,
       IsDefinition, ScopeLine, ContainingType, Virtuality, VirtualIndex, Flags,
       IsOptimized, nullptr, TemplateParams, Declaration, Variables);
@@ -1599,151 +1621,185 @@ TEST_F(MDSubprogramTest, replaceFunction) {
   EXPECT_EQ(nullptr, N->getFunction());
 }
 
-typedef MetadataTest MDLexicalBlockTest;
+typedef MetadataTest DILexicalBlockTest;
 
-TEST_F(MDLexicalBlockTest, get) {
-  MDLocalScope *Scope = getSubprogram();
-  MDFile *File = getFile();
+TEST_F(DILexicalBlockTest, get) {
+  DILocalScope *Scope = getSubprogram();
+  DIFile *File = getFile();
   unsigned Line = 5;
   unsigned Column = 8;
 
-  auto *N = MDLexicalBlock::get(Context, Scope, File, Line, Column);
+  auto *N = DILexicalBlock::get(Context, Scope, File, Line, Column);
 
   EXPECT_EQ(dwarf::DW_TAG_lexical_block, N->getTag());
   EXPECT_EQ(Scope, N->getScope());
   EXPECT_EQ(File, N->getFile());
   EXPECT_EQ(Line, N->getLine());
   EXPECT_EQ(Column, N->getColumn());
-  EXPECT_EQ(N, MDLexicalBlock::get(Context, Scope, File, Line, Column));
+  EXPECT_EQ(N, DILexicalBlock::get(Context, Scope, File, Line, Column));
 
   EXPECT_NE(N,
-            MDLexicalBlock::get(Context, getSubprogram(), File, Line, Column));
-  EXPECT_NE(N, MDLexicalBlock::get(Context, Scope, getFile(), Line, Column));
-  EXPECT_NE(N, MDLexicalBlock::get(Context, Scope, File, Line + 1, Column));
-  EXPECT_NE(N, MDLexicalBlock::get(Context, Scope, File, Line, Column + 1));
+            DILexicalBlock::get(Context, getSubprogram(), File, Line, Column));
+  EXPECT_NE(N, DILexicalBlock::get(Context, Scope, getFile(), Line, Column));
+  EXPECT_NE(N, DILexicalBlock::get(Context, Scope, File, Line + 1, Column));
+  EXPECT_NE(N, DILexicalBlock::get(Context, Scope, File, Line, Column + 1));
 
-  TempMDLexicalBlock Temp = N->clone();
+  TempDILexicalBlock Temp = N->clone();
   EXPECT_EQ(N, MDNode::replaceWithUniqued(std::move(Temp)));
 }
 
-typedef MetadataTest MDLexicalBlockFileTest;
+typedef MetadataTest DILexicalBlockFileTest;
 
-TEST_F(MDLexicalBlockFileTest, get) {
-  MDLocalScope *Scope = getSubprogram();
-  MDFile *File = getFile();
+TEST_F(DILexicalBlockFileTest, get) {
+  DILocalScope *Scope = getSubprogram();
+  DIFile *File = getFile();
   unsigned Discriminator = 5;
 
-  auto *N = MDLexicalBlockFile::get(Context, Scope, File, Discriminator);
+  auto *N = DILexicalBlockFile::get(Context, Scope, File, Discriminator);
 
   EXPECT_EQ(dwarf::DW_TAG_lexical_block, N->getTag());
   EXPECT_EQ(Scope, N->getScope());
   EXPECT_EQ(File, N->getFile());
   EXPECT_EQ(Discriminator, N->getDiscriminator());
-  EXPECT_EQ(N, MDLexicalBlockFile::get(Context, Scope, File, Discriminator));
+  EXPECT_EQ(N, DILexicalBlockFile::get(Context, Scope, File, Discriminator));
 
-  EXPECT_NE(N, MDLexicalBlockFile::get(Context, getSubprogram(), File,
+  EXPECT_NE(N, DILexicalBlockFile::get(Context, getSubprogram(), File,
                                        Discriminator));
   EXPECT_NE(N,
-            MDLexicalBlockFile::get(Context, Scope, getFile(), Discriminator));
+            DILexicalBlockFile::get(Context, Scope, getFile(), Discriminator));
   EXPECT_NE(N,
-            MDLexicalBlockFile::get(Context, Scope, File, Discriminator + 1));
+            DILexicalBlockFile::get(Context, Scope, File, Discriminator + 1));
 
-  TempMDLexicalBlockFile Temp = N->clone();
+  TempDILexicalBlockFile Temp = N->clone();
   EXPECT_EQ(N, MDNode::replaceWithUniqued(std::move(Temp)));
 }
 
-typedef MetadataTest MDNamespaceTest;
+typedef MetadataTest DINamespaceTest;
 
-TEST_F(MDNamespaceTest, get) {
-  MDScope *Scope = getFile();
-  MDFile *File = getFile();
+TEST_F(DINamespaceTest, get) {
+  DIScope *Scope = getFile();
+  DIFile *File = getFile();
   StringRef Name = "namespace";
   unsigned Line = 5;
 
-  auto *N = MDNamespace::get(Context, Scope, File, Name, Line);
+  auto *N = DINamespace::get(Context, Scope, File, Name, Line);
 
   EXPECT_EQ(dwarf::DW_TAG_namespace, N->getTag());
   EXPECT_EQ(Scope, N->getScope());
   EXPECT_EQ(File, N->getFile());
   EXPECT_EQ(Name, N->getName());
   EXPECT_EQ(Line, N->getLine());
-  EXPECT_EQ(N, MDNamespace::get(Context, Scope, File, Name, Line));
+  EXPECT_EQ(N, DINamespace::get(Context, Scope, File, Name, Line));
 
-  EXPECT_NE(N, MDNamespace::get(Context, getFile(), File, Name, Line));
-  EXPECT_NE(N, MDNamespace::get(Context, Scope, getFile(), Name, Line));
-  EXPECT_NE(N, MDNamespace::get(Context, Scope, File, "other", Line));
-  EXPECT_NE(N, MDNamespace::get(Context, Scope, File, Name, Line + 1));
+  EXPECT_NE(N, DINamespace::get(Context, getFile(), File, Name, Line));
+  EXPECT_NE(N, DINamespace::get(Context, Scope, getFile(), Name, Line));
+  EXPECT_NE(N, DINamespace::get(Context, Scope, File, "other", Line));
+  EXPECT_NE(N, DINamespace::get(Context, Scope, File, Name, Line + 1));
 
-  TempMDNamespace Temp = N->clone();
+  TempDINamespace Temp = N->clone();
   EXPECT_EQ(N, MDNode::replaceWithUniqued(std::move(Temp)));
 }
 
-typedef MetadataTest MDTemplateTypeParameterTest;
+typedef MetadataTest DIModuleTest;
+
+TEST_F(DIModuleTest, get) {
+  DIScope *Scope = getFile();
+  StringRef Name = "module";
+  StringRef ConfigMacro = "-DNDEBUG";
+  StringRef Includes = "-I.";
+  StringRef Sysroot = "/";
+
+  auto *N = DIModule::get(Context, Scope, Name, ConfigMacro, Includes, Sysroot);
 
-TEST_F(MDTemplateTypeParameterTest, get) {
+  EXPECT_EQ(dwarf::DW_TAG_module, N->getTag());
+  EXPECT_EQ(Scope, N->getScope());
+  EXPECT_EQ(Name, N->getName());
+  EXPECT_EQ(ConfigMacro, N->getConfigurationMacros());
+  EXPECT_EQ(Includes, N->getIncludePath());
+  EXPECT_EQ(Sysroot, N->getISysRoot());
+  EXPECT_EQ(N, DIModule::get(Context, Scope, Name,
+                             ConfigMacro, Includes, Sysroot));
+  EXPECT_NE(N, DIModule::get(Context, getFile(), Name,
+                             ConfigMacro, Includes, Sysroot));
+  EXPECT_NE(N, DIModule::get(Context, Scope, "other",
+                             ConfigMacro, Includes, Sysroot));
+  EXPECT_NE(N, DIModule::get(Context, Scope, Name,
+                             "other", Includes, Sysroot));
+  EXPECT_NE(N, DIModule::get(Context, Scope, Name,
+                             ConfigMacro, "other", Sysroot));
+  EXPECT_NE(N, DIModule::get(Context, Scope, Name,
+                             ConfigMacro, Includes, "other"));
+
+  TempDIModule Temp = N->clone();
+  EXPECT_EQ(N, MDNode::replaceWithUniqued(std::move(Temp)));
+}
+
+typedef MetadataTest DITemplateTypeParameterTest;
+
+TEST_F(DITemplateTypeParameterTest, get) {
   StringRef Name = "template";
-  MDTypeRef Type = getBasicType("basic");
+  DITypeRef Type = getBasicType("basic");
 
-  auto *N = MDTemplateTypeParameter::get(Context, Name, Type);
+  auto *N = DITemplateTypeParameter::get(Context, Name, Type);
 
   EXPECT_EQ(dwarf::DW_TAG_template_type_parameter, N->getTag());
   EXPECT_EQ(Name, N->getName());
   EXPECT_EQ(Type, N->getType());
-  EXPECT_EQ(N, MDTemplateTypeParameter::get(Context, Name, Type));
+  EXPECT_EQ(N, DITemplateTypeParameter::get(Context, Name, Type));
 
-  EXPECT_NE(N, MDTemplateTypeParameter::get(Context, "other", Type));
+  EXPECT_NE(N, DITemplateTypeParameter::get(Context, "other", Type));
   EXPECT_NE(N,
-            MDTemplateTypeParameter::get(Context, Name, getBasicType("other")));
+            DITemplateTypeParameter::get(Context, Name, getBasicType("other")));
 
-  TempMDTemplateTypeParameter Temp = N->clone();
+  TempDITemplateTypeParameter Temp = N->clone();
   EXPECT_EQ(N, MDNode::replaceWithUniqued(std::move(Temp)));
 }
 
-typedef MetadataTest MDTemplateValueParameterTest;
+typedef MetadataTest DITemplateValueParameterTest;
 
-TEST_F(MDTemplateValueParameterTest, get) {
+TEST_F(DITemplateValueParameterTest, get) {
   unsigned Tag = dwarf::DW_TAG_template_value_parameter;
   StringRef Name = "template";
-  MDTypeRef Type = getBasicType("basic");
+  DITypeRef Type = getBasicType("basic");
   Metadata *Value = getConstantAsMetadata();
 
-  auto *N = MDTemplateValueParameter::get(Context, Tag, Name, Type, Value);
+  auto *N = DITemplateValueParameter::get(Context, Tag, Name, Type, Value);
   EXPECT_EQ(Tag, N->getTag());
   EXPECT_EQ(Name, N->getName());
   EXPECT_EQ(Type, N->getType());
   EXPECT_EQ(Value, N->getValue());
-  EXPECT_EQ(N, MDTemplateValueParameter::get(Context, Tag, Name, Type, Value));
+  EXPECT_EQ(N, DITemplateValueParameter::get(Context, Tag, Name, Type, Value));
 
-  EXPECT_NE(N, MDTemplateValueParameter::get(
+  EXPECT_NE(N, DITemplateValueParameter::get(
                    Context, dwarf::DW_TAG_GNU_template_template_param, Name,
                    Type, Value));
-  EXPECT_NE(N, MDTemplateValueParameter::get(Context, Tag,  "other", Type,
-                                             Value));
-  EXPECT_NE(N, MDTemplateValueParameter::get(Context, Tag, Name,
+  EXPECT_NE(N,
+            DITemplateValueParameter::get(Context, Tag, "other", Type, Value));
+  EXPECT_NE(N, DITemplateValueParameter::get(Context, Tag, Name,
                                              getBasicType("other"), Value));
-  EXPECT_NE(N, MDTemplateValueParameter::get(Context, Tag, Name, Type,
+  EXPECT_NE(N, DITemplateValueParameter::get(Context, Tag, Name, Type,
                                              getConstantAsMetadata()));
 
-  TempMDTemplateValueParameter Temp = N->clone();
+  TempDITemplateValueParameter Temp = N->clone();
   EXPECT_EQ(N, MDNode::replaceWithUniqued(std::move(Temp)));
 }
 
-typedef MetadataTest MDGlobalVariableTest;
+typedef MetadataTest DIGlobalVariableTest;
 
-TEST_F(MDGlobalVariableTest, get) {
-  MDScope *Scope = getSubprogram();
+TEST_F(DIGlobalVariableTest, get) {
+  DIScope *Scope = getSubprogram();
   StringRef Name = "name";
   StringRef LinkageName = "linkage";
-  MDFile *File = getFile();
+  DIFile *File = getFile();
   unsigned Line = 5;
-  MDTypeRef Type = getDerivedType();
+  DITypeRef Type = getDerivedType();
   bool IsLocalToUnit = false;
   bool IsDefinition = true;
   Constant *Variable = getConstant();
-  MDDerivedType *StaticDataMemberDeclaration =
-      cast<MDDerivedType>(getDerivedType());
+  DIDerivedType *StaticDataMemberDeclaration =
+      cast<DIDerivedType>(getDerivedType());
 
-  auto *N = MDGlobalVariable::get(Context, Scope, Name, LinkageName, File, Line,
+  auto *N = DIGlobalVariable::get(Context, Scope, Name, LinkageName, File, Line,
                                   Type, IsLocalToUnit, IsDefinition, Variable,
                                   StaticDataMemberDeclaration);
   EXPECT_EQ(dwarf::DW_TAG_variable, N->getTag());
@@ -1757,64 +1813,64 @@ TEST_F(MDGlobalVariableTest, get) {
   EXPECT_EQ(IsDefinition, N->isDefinition());
   EXPECT_EQ(Variable, N->getVariable());
   EXPECT_EQ(StaticDataMemberDeclaration, N->getStaticDataMemberDeclaration());
-  EXPECT_EQ(N, MDGlobalVariable::get(Context, Scope, Name, LinkageName, File,
+  EXPECT_EQ(N, DIGlobalVariable::get(Context, Scope, Name, LinkageName, File,
                                      Line, Type, IsLocalToUnit, IsDefinition,
                                      Variable, StaticDataMemberDeclaration));
 
   EXPECT_NE(N,
-            MDGlobalVariable::get(Context, getSubprogram(), Name, LinkageName,
+            DIGlobalVariable::get(Context, getSubprogram(), Name, LinkageName,
                                   File, Line, Type, IsLocalToUnit, IsDefinition,
                                   Variable, StaticDataMemberDeclaration));
-  EXPECT_NE(N, MDGlobalVariable::get(Context, Scope, "other", LinkageName, File,
+  EXPECT_NE(N, DIGlobalVariable::get(Context, Scope, "other", LinkageName, File,
                                      Line, Type, IsLocalToUnit, IsDefinition,
                                      Variable, StaticDataMemberDeclaration));
-  EXPECT_NE(N, MDGlobalVariable::get(Context, Scope, Name, "other", File, Line,
+  EXPECT_NE(N, DIGlobalVariable::get(Context, Scope, Name, "other", File, Line,
                                      Type, IsLocalToUnit, IsDefinition,
                                      Variable, StaticDataMemberDeclaration));
   EXPECT_NE(N,
-            MDGlobalVariable::get(Context, Scope, Name, LinkageName, getFile(),
+            DIGlobalVariable::get(Context, Scope, Name, LinkageName, getFile(),
                                   Line, Type, IsLocalToUnit, IsDefinition,
                                   Variable, StaticDataMemberDeclaration));
   EXPECT_NE(N,
-            MDGlobalVariable::get(Context, Scope, Name, LinkageName, File,
+            DIGlobalVariable::get(Context, Scope, Name, LinkageName, File,
                                   Line + 1, Type, IsLocalToUnit, IsDefinition,
                                   Variable, StaticDataMemberDeclaration));
   EXPECT_NE(N,
-            MDGlobalVariable::get(Context, Scope, Name, LinkageName, File, Line,
+            DIGlobalVariable::get(Context, Scope, Name, LinkageName, File, Line,
                                   getDerivedType(), IsLocalToUnit, IsDefinition,
                                   Variable, StaticDataMemberDeclaration));
-  EXPECT_NE(N, MDGlobalVariable::get(Context, Scope, Name, LinkageName, File,
+  EXPECT_NE(N, DIGlobalVariable::get(Context, Scope, Name, LinkageName, File,
                                      Line, Type, !IsLocalToUnit, IsDefinition,
                                      Variable, StaticDataMemberDeclaration));
-  EXPECT_NE(N, MDGlobalVariable::get(Context, Scope, Name, LinkageName, File,
+  EXPECT_NE(N, DIGlobalVariable::get(Context, Scope, Name, LinkageName, File,
                                      Line, Type, IsLocalToUnit, !IsDefinition,
                                      Variable, StaticDataMemberDeclaration));
   EXPECT_NE(N,
-            MDGlobalVariable::get(Context, Scope, Name, LinkageName, File, Line,
+            DIGlobalVariable::get(Context, Scope, Name, LinkageName, File, Line,
                                   Type, IsLocalToUnit, IsDefinition,
                                   getConstant(), StaticDataMemberDeclaration));
   EXPECT_NE(N,
-            MDGlobalVariable::get(Context, Scope, Name, LinkageName, File, Line,
+            DIGlobalVariable::get(Context, Scope, Name, LinkageName, File, Line,
                                   Type, IsLocalToUnit, IsDefinition, Variable,
-                                  cast<MDDerivedType>(getDerivedType())));
+                                  cast<DIDerivedType>(getDerivedType())));
 
-  TempMDGlobalVariable Temp = N->clone();
+  TempDIGlobalVariable Temp = N->clone();
   EXPECT_EQ(N, MDNode::replaceWithUniqued(std::move(Temp)));
 }
 
-typedef MetadataTest MDLocalVariableTest;
+typedef MetadataTest DILocalVariableTest;
 
-TEST_F(MDLocalVariableTest, get) {
+TEST_F(DILocalVariableTest, get) {
   unsigned Tag = dwarf::DW_TAG_arg_variable;
-  MDLocalScope *Scope = getSubprogram();
+  DILocalScope *Scope = getSubprogram();
   StringRef Name = "name";
-  MDFile *File = getFile();
+  DIFile *File = getFile();
   unsigned Line = 5;
-  MDTypeRef Type = getDerivedType();
+  DITypeRef Type = getDerivedType();
   unsigned Arg = 6;
   unsigned Flags = 7;
 
-  auto *N = MDLocalVariable::get(Context, Tag, Scope, Name, File, Line, Type,
+  auto *N = DILocalVariable::get(Context, Tag, Scope, Name, File, Line, Type,
                                  Arg, Flags);
   EXPECT_EQ(Tag, N->getTag());
   EXPECT_EQ(Scope, N->getScope());
@@ -1824,37 +1880,57 @@ TEST_F(MDLocalVariableTest, get) {
   EXPECT_EQ(Type, N->getType());
   EXPECT_EQ(Arg, N->getArg());
   EXPECT_EQ(Flags, N->getFlags());
-  EXPECT_EQ(N, MDLocalVariable::get(Context, Tag, Scope, Name, File, Line, Type,
+  EXPECT_EQ(N, DILocalVariable::get(Context, Tag, Scope, Name, File, Line, Type,
                                     Arg, Flags));
 
-  EXPECT_NE(N, MDLocalVariable::get(Context, dwarf::DW_TAG_auto_variable, Scope,
+  EXPECT_NE(N, DILocalVariable::get(Context, dwarf::DW_TAG_auto_variable, Scope,
                                     Name, File, Line, Type, Arg, Flags));
-  EXPECT_NE(N, MDLocalVariable::get(Context, Tag, getSubprogram(), Name, File,
+  EXPECT_NE(N, DILocalVariable::get(Context, Tag, getSubprogram(), Name, File,
                                     Line, Type, Arg, Flags));
-  EXPECT_NE(N, MDLocalVariable::get(Context, Tag, Scope, "other", File, Line,
+  EXPECT_NE(N, DILocalVariable::get(Context, Tag, Scope, "other", File, Line,
                                     Type, Arg, Flags));
-  EXPECT_NE(N, MDLocalVariable::get(Context, Tag, Scope, Name, getFile(), Line,
+  EXPECT_NE(N, DILocalVariable::get(Context, Tag, Scope, Name, getFile(), Line,
                                     Type, Arg, Flags));
-  EXPECT_NE(N, MDLocalVariable::get(Context, Tag, Scope, Name, File, Line + 1,
+  EXPECT_NE(N, DILocalVariable::get(Context, Tag, Scope, Name, File, Line + 1,
                                     Type, Arg, Flags));
-  EXPECT_NE(N, MDLocalVariable::get(Context, Tag, Scope, Name, File, Line,
+  EXPECT_NE(N, DILocalVariable::get(Context, Tag, Scope, Name, File, Line,
                                     getDerivedType(), Arg, Flags));
-  EXPECT_NE(N, MDLocalVariable::get(Context, Tag, Scope, Name, File, Line, Type,
+  EXPECT_NE(N, DILocalVariable::get(Context, Tag, Scope, Name, File, Line, Type,
                                     Arg + 1, Flags));
-  EXPECT_NE(N, MDLocalVariable::get(Context, Tag, Scope, Name, File, Line, Type,
+  EXPECT_NE(N, DILocalVariable::get(Context, Tag, Scope, Name, File, Line, Type,
                                     Arg, ~Flags));
 
-  TempMDLocalVariable Temp = N->clone();
+  TempDILocalVariable Temp = N->clone();
   EXPECT_EQ(N, MDNode::replaceWithUniqued(std::move(Temp)));
 }
 
-typedef MetadataTest MDExpressionTest;
-
-TEST_F(MDExpressionTest, get) {
+TEST_F(DILocalVariableTest, getArg256) {
+  EXPECT_EQ(255u, DILocalVariable::get(Context, dwarf::DW_TAG_arg_variable,
+                                       getSubprogram(), "", getFile(), 0,
+                                       nullptr, 255, 0)
+                      ->getArg());
+  EXPECT_EQ(256u, DILocalVariable::get(Context, dwarf::DW_TAG_arg_variable,
+                                       getSubprogram(), "", getFile(), 0,
+                                       nullptr, 256, 0)
+                      ->getArg());
+  EXPECT_EQ(257u, DILocalVariable::get(Context, dwarf::DW_TAG_arg_variable,
+                                       getSubprogram(), "", getFile(), 0,
+                                       nullptr, 257, 0)
+                      ->getArg());
+  unsigned Max = UINT16_MAX;
+  EXPECT_EQ(Max, DILocalVariable::get(Context, dwarf::DW_TAG_arg_variable,
+                                      getSubprogram(), "", getFile(), 0,
+                                      nullptr, Max, 0)
+                     ->getArg());
+}
+
+typedef MetadataTest DIExpressionTest;
+
+TEST_F(DIExpressionTest, get) {
   uint64_t Elements[] = {2, 6, 9, 78, 0};
-  auto *N = MDExpression::get(Context, Elements);
+  auto *N = DIExpression::get(Context, Elements);
   EXPECT_EQ(makeArrayRef(Elements), N->getElements());
-  EXPECT_EQ(N, MDExpression::get(Context, Elements));
+  EXPECT_EQ(N, DIExpression::get(Context, Elements));
 
   EXPECT_EQ(5u, N->getNumElements());
   EXPECT_EQ(2u, N->getElement(0));
@@ -1863,24 +1939,24 @@ TEST_F(MDExpressionTest, get) {
   EXPECT_EQ(78u, N->getElement(3));
   EXPECT_EQ(0u, N->getElement(4));
 
-  TempMDExpression Temp = N->clone();
+  TempDIExpression Temp = N->clone();
   EXPECT_EQ(N, MDNode::replaceWithUniqued(std::move(Temp)));
 }
 
-TEST_F(MDExpressionTest, isValid) {
+TEST_F(DIExpressionTest, isValid) {
 #define EXPECT_VALID(...)                                                      \
   do {                                                                         \
     uint64_t Elements[] = {__VA_ARGS__};                                       \
-    EXPECT_TRUE(MDExpression::get(Context, Elements)->isValid());              \
+    EXPECT_TRUE(DIExpression::get(Context, Elements)->isValid());              \
   } while (false)
 #define EXPECT_INVALID(...)                                                    \
   do {                                                                         \
     uint64_t Elements[] = {__VA_ARGS__};                                       \
-    EXPECT_FALSE(MDExpression::get(Context, Elements)->isValid());             \
+    EXPECT_FALSE(DIExpression::get(Context, Elements)->isValid());             \
   } while (false)
 
   // Empty expression should be valid.
-  EXPECT_TRUE(MDExpression::get(Context, None));
+  EXPECT_TRUE(DIExpression::get(Context, None));
 
   // Valid constructions.
   EXPECT_VALID(dwarf::DW_OP_plus, 6);
@@ -1903,18 +1979,18 @@ TEST_F(MDExpressionTest, isValid) {
 #undef EXPECT_INVALID
 }
 
-typedef MetadataTest MDObjCPropertyTest;
+typedef MetadataTest DIObjCPropertyTest;
 
-TEST_F(MDObjCPropertyTest, get) {
+TEST_F(DIObjCPropertyTest, get) {
   StringRef Name = "name";
-  MDFile *File = getFile();
+  DIFile *File = getFile();
   unsigned Line = 5;
   StringRef GetterName = "getter";
   StringRef SetterName = "setter";
   unsigned Attributes = 7;
-  MDType *Type = cast<MDBasicType>(getBasicType("basic"));
+  DITypeRef Type = getBasicType("basic");
 
-  auto *N = MDObjCProperty::get(Context, Name, File, Line, GetterName,
+  auto *N = DIObjCProperty::get(Context, Name, File, Line, GetterName,
                                 SetterName, Attributes, Type);
 
   EXPECT_EQ(dwarf::DW_TAG_APPLE_property, N->getTag());
@@ -1925,60 +2001,60 @@ TEST_F(MDObjCPropertyTest, get) {
   EXPECT_EQ(SetterName, N->getSetterName());
   EXPECT_EQ(Attributes, N->getAttributes());
   EXPECT_EQ(Type, N->getType());
-  EXPECT_EQ(N, MDObjCProperty::get(Context, Name, File, Line, GetterName,
+  EXPECT_EQ(N, DIObjCProperty::get(Context, Name, File, Line, GetterName,
                                    SetterName, Attributes, Type));
 
-  EXPECT_NE(N, MDObjCProperty::get(Context, "other", File, Line, GetterName,
+  EXPECT_NE(N, DIObjCProperty::get(Context, "other", File, Line, GetterName,
                                    SetterName, Attributes, Type));
-  EXPECT_NE(N, MDObjCProperty::get(Context, Name, getFile(), Line, GetterName,
+  EXPECT_NE(N, DIObjCProperty::get(Context, Name, getFile(), Line, GetterName,
                                    SetterName, Attributes, Type));
-  EXPECT_NE(N, MDObjCProperty::get(Context, Name, File, Line + 1, GetterName,
+  EXPECT_NE(N, DIObjCProperty::get(Context, Name, File, Line + 1, GetterName,
                                    SetterName, Attributes, Type));
-  EXPECT_NE(N, MDObjCProperty::get(Context, Name, File, Line, "other",
+  EXPECT_NE(N, DIObjCProperty::get(Context, Name, File, Line, "other",
                                    SetterName, Attributes, Type));
-  EXPECT_NE(N, MDObjCProperty::get(Context, Name, File, Line, GetterName,
+  EXPECT_NE(N, DIObjCProperty::get(Context, Name, File, Line, GetterName,
                                    "other", Attributes, Type));
-  EXPECT_NE(N, MDObjCProperty::get(Context, Name, File, Line, GetterName,
+  EXPECT_NE(N, DIObjCProperty::get(Context, Name, File, Line, GetterName,
                                    SetterName, Attributes + 1, Type));
-  EXPECT_NE(N, MDObjCProperty::get(Context, Name, File, Line, GetterName,
+  EXPECT_NE(N, DIObjCProperty::get(Context, Name, File, Line, GetterName,
                                    SetterName, Attributes,
-                                   cast<MDBasicType>(getBasicType("other"))));
+                                   getBasicType("other")));
 
-  TempMDObjCProperty Temp = N->clone();
+  TempDIObjCProperty Temp = N->clone();
   EXPECT_EQ(N, MDNode::replaceWithUniqued(std::move(Temp)));
 }
 
-typedef MetadataTest MDImportedEntityTest;
+typedef MetadataTest DIImportedEntityTest;
 
-TEST_F(MDImportedEntityTest, get) {
+TEST_F(DIImportedEntityTest, get) {
   unsigned Tag = dwarf::DW_TAG_imported_module;
-  MDScope *Scope = getSubprogram();
-  DebugNodeRef Entity = getCompositeType();
+  DIScope *Scope = getSubprogram();
+  DINodeRef Entity = getCompositeType();
   unsigned Line = 5;
   StringRef Name = "name";
 
-  auto *N = MDImportedEntity::get(Context, Tag, Scope, Entity, Line, Name);
+  auto *N = DIImportedEntity::get(Context, Tag, Scope, Entity, Line, Name);
 
   EXPECT_EQ(Tag, N->getTag());
   EXPECT_EQ(Scope, N->getScope());
   EXPECT_EQ(Entity, N->getEntity());
   EXPECT_EQ(Line, N->getLine());
   EXPECT_EQ(Name, N->getName());
-  EXPECT_EQ(N, MDImportedEntity::get(Context, Tag, Scope, Entity, Line, Name));
+  EXPECT_EQ(N, DIImportedEntity::get(Context, Tag, Scope, Entity, Line, Name));
 
   EXPECT_NE(N,
-            MDImportedEntity::get(Context, dwarf::DW_TAG_imported_declaration,
+            DIImportedEntity::get(Context, dwarf::DW_TAG_imported_declaration,
                                   Scope, Entity, Line, Name));
-  EXPECT_NE(N, MDImportedEntity::get(Context, Tag, getSubprogram(), Entity,
+  EXPECT_NE(N, DIImportedEntity::get(Context, Tag, getSubprogram(), Entity,
                                      Line, Name));
-  EXPECT_NE(N, MDImportedEntity::get(Context, Tag, Scope, getCompositeType(),
+  EXPECT_NE(N, DIImportedEntity::get(Context, Tag, Scope, getCompositeType(),
                                      Line, Name));
   EXPECT_NE(N,
-            MDImportedEntity::get(Context, Tag, Scope, Entity, Line + 1, Name));
+            DIImportedEntity::get(Context, Tag, Scope, Entity, Line + 1, Name));
   EXPECT_NE(N,
-            MDImportedEntity::get(Context, Tag, Scope, Entity, Line, "other"));
+            DIImportedEntity::get(Context, Tag, Scope, Entity, Line, "other"));
 
-  TempMDImportedEntity Temp = N->clone();
+  TempDIImportedEntity Temp = N->clone();
   EXPECT_EQ(N, MDNode::replaceWithUniqued(std::move(Temp)));
 }
 
@@ -2125,4 +2201,137 @@ TEST(NamedMDNodeTest, Search) {
   EXPECT_STREQ("!llvm.NMD1 = !{!0, !1}\n",
                oss.str().c_str());
 }
+
+typedef MetadataTest FunctionAttachmentTest;
+TEST_F(FunctionAttachmentTest, setMetadata) {
+  Function *F = getFunction("foo");
+  ASSERT_FALSE(F->hasMetadata());
+  EXPECT_EQ(nullptr, F->getMetadata(LLVMContext::MD_dbg));
+  EXPECT_EQ(nullptr, F->getMetadata("dbg"));
+  EXPECT_EQ(nullptr, F->getMetadata("other"));
+
+  DISubprogram *SP1 = getSubprogram();
+  DISubprogram *SP2 = getSubprogram();
+  ASSERT_NE(SP1, SP2);
+
+  F->setMetadata("dbg", SP1);
+  EXPECT_TRUE(F->hasMetadata());
+  EXPECT_EQ(SP1, F->getMetadata(LLVMContext::MD_dbg));
+  EXPECT_EQ(SP1, F->getMetadata("dbg"));
+  EXPECT_EQ(nullptr, F->getMetadata("other"));
+
+  F->setMetadata(LLVMContext::MD_dbg, SP2);
+  EXPECT_TRUE(F->hasMetadata());
+  EXPECT_EQ(SP2, F->getMetadata(LLVMContext::MD_dbg));
+  EXPECT_EQ(SP2, F->getMetadata("dbg"));
+  EXPECT_EQ(nullptr, F->getMetadata("other"));
+
+  F->setMetadata("dbg", nullptr);
+  EXPECT_FALSE(F->hasMetadata());
+  EXPECT_EQ(nullptr, F->getMetadata(LLVMContext::MD_dbg));
+  EXPECT_EQ(nullptr, F->getMetadata("dbg"));
+  EXPECT_EQ(nullptr, F->getMetadata("other"));
+
+  MDTuple *T1 = getTuple();
+  MDTuple *T2 = getTuple();
+  ASSERT_NE(T1, T2);
+
+  F->setMetadata("other1", T1);
+  F->setMetadata("other2", T2);
+  EXPECT_TRUE(F->hasMetadata());
+  EXPECT_EQ(T1, F->getMetadata("other1"));
+  EXPECT_EQ(T2, F->getMetadata("other2"));
+  EXPECT_EQ(nullptr, F->getMetadata("dbg"));
+
+  F->setMetadata("other1", T2);
+  F->setMetadata("other2", T1);
+  EXPECT_EQ(T2, F->getMetadata("other1"));
+  EXPECT_EQ(T1, F->getMetadata("other2"));
+
+  F->setMetadata("other1", nullptr);
+  F->setMetadata("other2", nullptr);
+  EXPECT_FALSE(F->hasMetadata());
+  EXPECT_EQ(nullptr, F->getMetadata("other1"));
+  EXPECT_EQ(nullptr, F->getMetadata("other2"));
+}
+
+TEST_F(FunctionAttachmentTest, getAll) {
+  Function *F = getFunction("foo");
+
+  MDTuple *T1 = getTuple();
+  MDTuple *T2 = getTuple();
+  MDTuple *P = getTuple();
+  DISubprogram *SP = getSubprogram();
+
+  F->setMetadata("other1", T2);
+  F->setMetadata(LLVMContext::MD_dbg, SP);
+  F->setMetadata("other2", T1);
+  F->setMetadata(LLVMContext::MD_prof, P);
+  F->setMetadata("other2", T2);
+  F->setMetadata("other1", T1);
+
+  SmallVector<std::pair<unsigned, MDNode *>, 4> MDs;
+  F->getAllMetadata(MDs);
+  ASSERT_EQ(4u, MDs.size());
+  EXPECT_EQ(LLVMContext::MD_dbg, MDs[0].first);
+  EXPECT_EQ(LLVMContext::MD_prof, MDs[1].first);
+  EXPECT_EQ(Context.getMDKindID("other1"), MDs[2].first);
+  EXPECT_EQ(Context.getMDKindID("other2"), MDs[3].first);
+  EXPECT_EQ(SP, MDs[0].second);
+  EXPECT_EQ(P, MDs[1].second);
+  EXPECT_EQ(T1, MDs[2].second);
+  EXPECT_EQ(T2, MDs[3].second);
+}
+
+TEST_F(FunctionAttachmentTest, dropUnknownMetadata) {
+  Function *F = getFunction("foo");
+
+  MDTuple *T1 = getTuple();
+  MDTuple *T2 = getTuple();
+  MDTuple *P = getTuple();
+  DISubprogram *SP = getSubprogram();
+
+  F->setMetadata("other1", T1);
+  F->setMetadata(LLVMContext::MD_dbg, SP);
+  F->setMetadata("other2", T2);
+  F->setMetadata(LLVMContext::MD_prof, P);
+
+  unsigned Known[] = {Context.getMDKindID("other2"), LLVMContext::MD_prof};
+  F->dropUnknownMetadata(Known);
+
+  EXPECT_EQ(T2, F->getMetadata("other2"));
+  EXPECT_EQ(P, F->getMetadata(LLVMContext::MD_prof));
+  EXPECT_EQ(nullptr, F->getMetadata("other1"));
+  EXPECT_EQ(nullptr, F->getMetadata(LLVMContext::MD_dbg));
+
+  F->setMetadata("other2", nullptr);
+  F->setMetadata(LLVMContext::MD_prof, nullptr);
+  EXPECT_FALSE(F->hasMetadata());
+}
+
+TEST_F(FunctionAttachmentTest, Verifier) {
+  Function *F = getFunction("foo");
+  F->setMetadata("attach", getTuple());
+
+  // Confirm this has no body.
+  ASSERT_TRUE(F->empty());
+
+  // Functions without a body cannot have metadata attachments (they also can't
+  // be verified directly, so check that the module fails to verify).
+  EXPECT_TRUE(verifyModule(*F->getParent()));
+
+  // Functions with a body can.
+  (void)new UnreachableInst(Context, BasicBlock::Create(Context, "bb", F));
+  EXPECT_FALSE(verifyModule(*F->getParent()));
+  EXPECT_FALSE(verifyFunction(*F));
+}
+
+TEST_F(FunctionAttachmentTest, EntryCount) {
+  Function *F = getFunction("foo");
+  EXPECT_FALSE(F->getEntryCount().hasValue());
+  F->setEntryCount(12304);
+  EXPECT_TRUE(F->getEntryCount().hasValue());
+  EXPECT_EQ(12304u, *F->getEntryCount());
+}
+
 }
diff --git a/unittests/IR/ValueHandleTest.cpp b/unittests/IR/ValueHandleTest.cpp
index 6000c4a3105..e6eb7cbedc6 100644
--- a/unittests/IR/ValueHandleTest.cpp
+++ b/unittests/IR/ValueHandleTest.cpp
@@ -21,7 +21,7 @@ namespace {
 class ValueHandle : public testing::Test {
 protected:
   Constant *ConstantV;
-  std::auto_ptr<BitCastInst> BitcastV;
+  std::unique_ptr<BitCastInst> BitcastV;
 
   ValueHandle() :
     ConstantV(ConstantInt::get(Type::getInt32Ty(getGlobalContext()), 0)),
@@ -320,7 +320,7 @@ TEST_F(ValueHandle, CallbackVH_DeletionCanRAUW) {
   // a CallbackVH to remove the uses before the check for no uses.
   RecoveringVH RVH;
   RVH = BitcastV.get();
-  std::auto_ptr<BinaryOperator> BitcastUser(
+  std::unique_ptr<BinaryOperator> BitcastUser(
     BinaryOperator::CreateAdd(RVH, 
                               Constant::getNullValue(Type::getInt32Ty(getGlobalContext()))));
   EXPECT_EQ(BitcastV.get(), BitcastUser->getOperand(0));
diff --git a/unittests/IR/ValueTest.cpp b/unittests/IR/ValueTest.cpp
index 4dd0c2c36cf..7a4c2f696f7 100644
--- a/unittests/IR/ValueTest.cpp
+++ b/unittests/IR/ValueTest.cpp
@@ -11,6 +11,7 @@
 #include "llvm/IR/Function.h"
 #include "llvm/IR/LLVMContext.h"
 #include "llvm/IR/Module.h"
+#include "llvm/IR/ModuleSlotTracker.h"
 #include "llvm/IR/Value.h"
 #include "llvm/Support/SourceMgr.h"
 #include "gtest/gtest.h"
@@ -106,4 +107,132 @@ TEST(GlobalTest, AlignDeath) {
 #endif
 #endif
 
+TEST(ValueTest, printSlots) {
+  // Check that Value::print() and Value::printAsOperand() work with and
+  // without a slot tracker.
+  LLVMContext C;
+
+  const char *ModuleString = "define void @f(i32 %x, i32 %y) {\n"
+                             "entry:\n"
+                             "  %0 = add i32 %y, 1\n"
+                             "  %1 = add i32 %y, 1\n"
+                             "  ret void\n"
+                             "}\n";
+  SMDiagnostic Err;
+  std::unique_ptr<Module> M = parseAssemblyString(ModuleString, Err, C);
+
+  Function *F = M->getFunction("f");
+  ASSERT_TRUE(F);
+  ASSERT_FALSE(F->empty());
+  BasicBlock &BB = F->getEntryBlock();
+  ASSERT_EQ(3u, BB.size());
+
+  Instruction *I0 = BB.begin();
+  ASSERT_TRUE(I0);
+  Instruction *I1 = ++BB.begin();
+  ASSERT_TRUE(I1);
+
+  ModuleSlotTracker MST(M.get());
+
+#define CHECK_PRINT(INST, STR)                                                 \
+  do {                                                                         \
+    {                                                                          \
+      std::string S;                                                           \
+      raw_string_ostream OS(S);                                                \
+      INST->print(OS);                                                         \
+      EXPECT_EQ(STR, OS.str());                                                \
+    }                                                                          \
+    {                                                                          \
+      std::string S;                                                           \
+      raw_string_ostream OS(S);                                                \
+      INST->print(OS, MST);                                                    \
+      EXPECT_EQ(STR, OS.str());                                                \
+    }                                                                          \
+  } while (false)
+  CHECK_PRINT(I0, "  %0 = add i32 %y, 1");
+  CHECK_PRINT(I1, "  %1 = add i32 %y, 1");
+#undef CHECK_PRINT
+
+#define CHECK_PRINT_AS_OPERAND(INST, TYPE, STR)                                \
+  do {                                                                         \
+    {                                                                          \
+      std::string S;                                                           \
+      raw_string_ostream OS(S);                                                \
+      INST->printAsOperand(OS, TYPE);                                          \
+      EXPECT_EQ(StringRef(STR), StringRef(OS.str()));                          \
+    }                                                                          \
+    {                                                                          \
+      std::string S;                                                           \
+      raw_string_ostream OS(S);                                                \
+      INST->printAsOperand(OS, TYPE, MST);                                     \
+      EXPECT_EQ(StringRef(STR), StringRef(OS.str()));                          \
+    }                                                                          \
+  } while (false)
+  CHECK_PRINT_AS_OPERAND(I0, false, "%0");
+  CHECK_PRINT_AS_OPERAND(I1, false, "%1");
+  CHECK_PRINT_AS_OPERAND(I0, true, "i32 %0");
+  CHECK_PRINT_AS_OPERAND(I1, true, "i32 %1");
+#undef CHECK_PRINT_AS_OPERAND
+}
+
+TEST(ValueTest, getLocalSlots) {
+  // Verify that the getLocalSlot method returns the correct slot numbers.
+  LLVMContext C;
+  const char *ModuleString = "define void @f(i32 %x, i32 %y) {\n"
+                             "entry:\n"
+                             "  %0 = add i32 %y, 1\n"
+                             "  %1 = add i32 %y, 1\n"
+                             "  br label %2\n"
+                             "\n"
+                             "  ret void\n"
+                             "}\n";
+  SMDiagnostic Err;
+  std::unique_ptr<Module> M = parseAssemblyString(ModuleString, Err, C);
+
+  Function *F = M->getFunction("f");
+  ASSERT_TRUE(F);
+  ASSERT_FALSE(F->empty());
+  BasicBlock &EntryBB = F->getEntryBlock();
+  ASSERT_EQ(3u, EntryBB.size());
+  BasicBlock *BB2 = ++F->begin();
+  ASSERT_TRUE(BB2);
+
+  Instruction *I0 = EntryBB.begin();
+  ASSERT_TRUE(I0);
+  Instruction *I1 = ++EntryBB.begin();
+  ASSERT_TRUE(I1);
+
+  ModuleSlotTracker MST(M.get());
+  MST.incorporateFunction(*F);
+  EXPECT_EQ(MST.getLocalSlot(I0), 0);
+  EXPECT_EQ(MST.getLocalSlot(I1), 1);
+  EXPECT_EQ(MST.getLocalSlot(&EntryBB), -1);
+  EXPECT_EQ(MST.getLocalSlot(BB2), 2);
+}
+
+#if defined(GTEST_HAS_DEATH_TEST) && !defined(NDEBUG)
+TEST(ValueTest, getLocalSlotDeath) {
+  LLVMContext C;
+  const char *ModuleString = "define void @f(i32 %x, i32 %y) {\n"
+                             "entry:\n"
+                             "  %0 = add i32 %y, 1\n"
+                             "  %1 = add i32 %y, 1\n"
+                             "  br label %2\n"
+                             "\n"
+                             "  ret void\n"
+                             "}\n";
+  SMDiagnostic Err;
+  std::unique_ptr<Module> M = parseAssemblyString(ModuleString, Err, C);
+
+  Function *F = M->getFunction("f");
+  ASSERT_TRUE(F);
+  ASSERT_FALSE(F->empty());
+  BasicBlock *BB2 = ++F->begin();
+  ASSERT_TRUE(BB2);
+
+  ModuleSlotTracker MST(M.get());
+  EXPECT_DEATH(MST.getLocalSlot(BB2), "No function incorporated");
+}
+#endif
+
 } // end anonymous namespace
diff --git a/unittests/Linker/LinkModulesTest.cpp b/unittests/Linker/LinkModulesTest.cpp
index b4689cba560..45f1308d3bd 100644
--- a/unittests/Linker/LinkModulesTest.cpp
+++ b/unittests/Linker/LinkModulesTest.cpp
@@ -15,6 +15,7 @@
 #include "llvm/IR/Module.h"
 #include "llvm/Linker/Linker.h"
 #include "llvm/Support/SourceMgr.h"
+#include "llvm-c/Linker.h"
 #include "gtest/gtest.h"
 
 using namespace llvm;
@@ -125,6 +126,22 @@ TEST_F(LinkModuleTest, BlockAddress) {
   delete LinkedModule;
 }
 
+static Module *getExternal(LLVMContext &Ctx, StringRef FuncName) {
+  // Create a module with an empty externally-linked function
+  Module *M = new Module("ExternalModule", Ctx);
+  FunctionType *FTy = FunctionType::get(
+      Type::getVoidTy(Ctx), Type::getInt8PtrTy(Ctx), false /*=isVarArgs*/);
+
+  Function *F =
+      Function::Create(FTy, Function::ExternalLinkage, FuncName, M);
+  F->setCallingConv(CallingConv::C);
+
+  BasicBlock *BB = BasicBlock::Create(Ctx, "", F);
+  IRBuilder<> Builder(BB);
+  Builder.CreateRetVoid();
+  return M;
+}
+
 static Module *getInternal(LLVMContext &Ctx) {
   Module *InternalM = new Module("InternalModule", Ctx);
   FunctionType *FTy = FunctionType::get(
@@ -178,4 +195,28 @@ TEST_F(LinkModuleTest, TypeMerge) {
             M1->getNamedGlobal("t2")->getType());
 }
 
+TEST_F(LinkModuleTest, CAPISuccess) {
+  std::unique_ptr<Module> DestM(getExternal(Ctx, "foo"));
+  std::unique_ptr<Module> SourceM(getExternal(Ctx, "bar"));
+  char *errout = nullptr;
+  LLVMBool result = LLVMLinkModules(wrap(DestM.get()), wrap(SourceM.get()),
+                                    LLVMLinkerDestroySource, &errout);
+  EXPECT_EQ(0, result);
+  EXPECT_EQ(nullptr, errout);
+  // "bar" is present in destination module
+  EXPECT_NE(nullptr, DestM->getFunction("bar"));
+}
+
+TEST_F(LinkModuleTest, CAPIFailure) {
+  // Symbol clash between two modules
+  std::unique_ptr<Module> DestM(getExternal(Ctx, "foo"));
+  std::unique_ptr<Module> SourceM(getExternal(Ctx, "foo"));
+  char *errout = nullptr;
+  LLVMBool result = LLVMLinkModules(wrap(DestM.get()), wrap(SourceM.get()),
+                                    LLVMLinkerDestroySource, &errout);
+  EXPECT_EQ(1, result);
+  EXPECT_STREQ("Linking globals named 'foo': symbol multiply defined!", errout);
+  LLVMDisposeMessage(errout);
+}
+
 } // end anonymous namespace
diff --git a/unittests/Makefile b/unittests/Makefile
index fefef0748b9..bf2ed22efea 100644
--- a/unittests/Makefile
+++ b/unittests/Makefile
@@ -9,8 +9,9 @@
 
 LEVEL = ..
 
-PARALLEL_DIRS = ADT Analysis Bitcode CodeGen DebugInfo ExecutionEngine IR \
-		LineEditor Linker MC Option ProfileData Support Transforms
+PARALLEL_DIRS = ADT Analysis AsmParser Bitcode CodeGen DebugInfo \
+                ExecutionEngine IR LineEditor Linker MC Option ProfileData \
+                Support Transforms
 
 include $(LEVEL)/Makefile.config
 include $(LLVM_SRC_ROOT)/unittests/Makefile.unittest
diff --git a/unittests/Option/OptionParsingTest.cpp b/unittests/Option/OptionParsingTest.cpp
index aa32da3a595..55cf8a95f35 100644
--- a/unittests/Option/OptionParsingTest.cpp
+++ b/unittests/Option/OptionParsingTest.cpp
@@ -67,27 +67,26 @@ const char *Args[] = {
 TEST(Option, OptionParsing) {
   TestOptTable T;
   unsigned MAI, MAC;
-  std::unique_ptr<InputArgList> AL(
-      T.ParseArgs(std::begin(Args), std::end(Args), MAI, MAC));
+  InputArgList AL = T.ParseArgs(Args, MAI, MAC);
 
   // Check they all exist.
-  EXPECT_TRUE(AL->hasArg(OPT_A));
-  EXPECT_TRUE(AL->hasArg(OPT_B));
-  EXPECT_TRUE(AL->hasArg(OPT_C));
-  EXPECT_TRUE(AL->hasArg(OPT_D));
-  EXPECT_TRUE(AL->hasArg(OPT_E));
-  EXPECT_TRUE(AL->hasArg(OPT_F));
-  EXPECT_TRUE(AL->hasArg(OPT_G));
+  EXPECT_TRUE(AL.hasArg(OPT_A));
+  EXPECT_TRUE(AL.hasArg(OPT_B));
+  EXPECT_TRUE(AL.hasArg(OPT_C));
+  EXPECT_TRUE(AL.hasArg(OPT_D));
+  EXPECT_TRUE(AL.hasArg(OPT_E));
+  EXPECT_TRUE(AL.hasArg(OPT_F));
+  EXPECT_TRUE(AL.hasArg(OPT_G));
 
   // Check the values.
-  EXPECT_EQ(AL->getLastArgValue(OPT_B), "hi");
-  EXPECT_EQ(AL->getLastArgValue(OPT_C), "bye");
-  EXPECT_EQ(AL->getLastArgValue(OPT_D), "adena");
-  std::vector<std::string> Es = AL->getAllArgValues(OPT_E);
+  EXPECT_EQ(AL.getLastArgValue(OPT_B), "hi");
+  EXPECT_EQ(AL.getLastArgValue(OPT_C), "bye");
+  EXPECT_EQ(AL.getLastArgValue(OPT_D), "adena");
+  std::vector<std::string> Es = AL.getAllArgValues(OPT_E);
   EXPECT_EQ(Es[0], "apple");
   EXPECT_EQ(Es[1], "bloom");
-  EXPECT_EQ(AL->getLastArgValue(OPT_F), "42");
-  std::vector<std::string> Gs = AL->getAllArgValues(OPT_G);
+  EXPECT_EQ(AL.getLastArgValue(OPT_F), "42");
+  std::vector<std::string> Gs = AL.getAllArgValues(OPT_G);
   EXPECT_EQ(Gs[0], "chuu");
   EXPECT_EQ(Gs[1], "2");
 
@@ -98,11 +97,11 @@ TEST(Option, OptionParsing) {
   EXPECT_NE(Help.find("-A"), std::string::npos);
 
   // Test aliases.
-  arg_iterator Cs = AL->filtered_begin(OPT_C);
-  ASSERT_NE(Cs, AL->filtered_end());
+  arg_iterator Cs = AL.filtered_begin(OPT_C);
+  ASSERT_NE(Cs, AL.filtered_end());
   EXPECT_EQ(StringRef((*Cs)->getValue()), "desu");
   ArgStringList ASL;
-  (*Cs)->render(*AL, ASL);
+  (*Cs)->render(AL, ASL);
   ASSERT_EQ(ASL.size(), 2u);
   EXPECT_EQ(StringRef(ASL[0]), "-C");
   EXPECT_EQ(StringRef(ASL[1]), "desu");
@@ -111,30 +110,29 @@ TEST(Option, OptionParsing) {
 TEST(Option, ParseWithFlagExclusions) {
   TestOptTable T;
   unsigned MAI, MAC;
-  std::unique_ptr<InputArgList> AL;
 
   // Exclude flag3 to avoid parsing as OPT_SLASH_C.
-  AL.reset(T.ParseArgs(std::begin(Args), std::end(Args), MAI, MAC,
-                       /*FlagsToInclude=*/0,
-                       /*FlagsToExclude=*/OptFlag3));
-  EXPECT_TRUE(AL->hasArg(OPT_A));
-  EXPECT_TRUE(AL->hasArg(OPT_C));
-  EXPECT_FALSE(AL->hasArg(OPT_SLASH_C));
+  InputArgList AL = T.ParseArgs(Args, MAI, MAC,
+                                /*FlagsToInclude=*/0,
+                                /*FlagsToExclude=*/OptFlag3);
+  EXPECT_TRUE(AL.hasArg(OPT_A));
+  EXPECT_TRUE(AL.hasArg(OPT_C));
+  EXPECT_FALSE(AL.hasArg(OPT_SLASH_C));
 
   // Exclude flag1 to avoid parsing as OPT_C.
-  AL.reset(T.ParseArgs(std::begin(Args), std::end(Args), MAI, MAC,
-                       /*FlagsToInclude=*/0,
-                       /*FlagsToExclude=*/OptFlag1));
-  EXPECT_TRUE(AL->hasArg(OPT_B));
-  EXPECT_FALSE(AL->hasArg(OPT_C));
-  EXPECT_TRUE(AL->hasArg(OPT_SLASH_C));
+  AL = T.ParseArgs(Args, MAI, MAC,
+                   /*FlagsToInclude=*/0,
+                   /*FlagsToExclude=*/OptFlag1);
+  EXPECT_TRUE(AL.hasArg(OPT_B));
+  EXPECT_FALSE(AL.hasArg(OPT_C));
+  EXPECT_TRUE(AL.hasArg(OPT_SLASH_C));
 
   const char *NewArgs[] = { "/C", "foo", "--C=bar" };
-  AL.reset(T.ParseArgs(std::begin(NewArgs), std::end(NewArgs), MAI, MAC));
-  EXPECT_TRUE(AL->hasArg(OPT_SLASH_C));
-  EXPECT_TRUE(AL->hasArg(OPT_C));
-  EXPECT_EQ(AL->getLastArgValue(OPT_SLASH_C), "foo");
-  EXPECT_EQ(AL->getLastArgValue(OPT_C), "bar");
+  AL = T.ParseArgs(NewArgs, MAI, MAC);
+  EXPECT_TRUE(AL.hasArg(OPT_SLASH_C));
+  EXPECT_TRUE(AL.hasArg(OPT_C));
+  EXPECT_EQ(AL.getLastArgValue(OPT_SLASH_C), "foo");
+  EXPECT_EQ(AL.getLastArgValue(OPT_C), "bar");
 }
 
 TEST(Option, ParseAliasInGroup) {
@@ -142,9 +140,8 @@ TEST(Option, ParseAliasInGroup) {
   unsigned MAI, MAC;
 
   const char *MyArgs[] = { "-I" };
-  std::unique_ptr<InputArgList> AL(
-      T.ParseArgs(std::begin(MyArgs), std::end(MyArgs), MAI, MAC));
-  EXPECT_TRUE(AL->hasArg(OPT_H));
+  InputArgList AL = T.ParseArgs(MyArgs, MAI, MAC);
+  EXPECT_TRUE(AL.hasArg(OPT_H));
 }
 
 TEST(Option, AliasArgs) {
@@ -152,11 +149,10 @@ TEST(Option, AliasArgs) {
   unsigned MAI, MAC;
 
   const char *MyArgs[] = { "-J", "-Joo" };
-  std::unique_ptr<InputArgList> AL(
-      T.ParseArgs(std::begin(MyArgs), std::end(MyArgs), MAI, MAC));
-  EXPECT_TRUE(AL->hasArg(OPT_B));
-  EXPECT_EQ(AL->getAllArgValues(OPT_B)[0], "foo");
-  EXPECT_EQ(AL->getAllArgValues(OPT_B)[1], "bar");
+  InputArgList AL = T.ParseArgs(MyArgs, MAI, MAC);
+  EXPECT_TRUE(AL.hasArg(OPT_B));
+  EXPECT_EQ(AL.getAllArgValues(OPT_B)[0], "foo");
+  EXPECT_EQ(AL.getAllArgValues(OPT_B)[1], "bar");
 }
 
 TEST(Option, IgnoreCase) {
@@ -164,10 +160,9 @@ TEST(Option, IgnoreCase) {
   unsigned MAI, MAC;
 
   const char *MyArgs[] = { "-a", "-joo" };
-  std::unique_ptr<InputArgList> AL(
-      T.ParseArgs(std::begin(MyArgs), std::end(MyArgs), MAI, MAC));
-  EXPECT_TRUE(AL->hasArg(OPT_A));
-  EXPECT_TRUE(AL->hasArg(OPT_B));
+  InputArgList AL = T.ParseArgs(MyArgs, MAI, MAC);
+  EXPECT_TRUE(AL.hasArg(OPT_A));
+  EXPECT_TRUE(AL.hasArg(OPT_B));
 }
 
 TEST(Option, DoNotIgnoreCase) {
@@ -175,10 +170,9 @@ TEST(Option, DoNotIgnoreCase) {
   unsigned MAI, MAC;
 
   const char *MyArgs[] = { "-a", "-joo" };
-  std::unique_ptr<InputArgList> AL(
-      T.ParseArgs(std::begin(MyArgs), std::end(MyArgs), MAI, MAC));
-  EXPECT_FALSE(AL->hasArg(OPT_A));
-  EXPECT_FALSE(AL->hasArg(OPT_B));
+  InputArgList AL = T.ParseArgs(MyArgs, MAI, MAC);
+  EXPECT_FALSE(AL.hasArg(OPT_A));
+  EXPECT_FALSE(AL.hasArg(OPT_B));
 }
 
 TEST(Option, SlurpEmpty) {
@@ -186,11 +180,10 @@ TEST(Option, SlurpEmpty) {
   unsigned MAI, MAC;
 
   const char *MyArgs[] = { "-A", "-slurp" };
-  std::unique_ptr<InputArgList> AL(
-      T.ParseArgs(std::begin(MyArgs), std::end(MyArgs), MAI, MAC));
-  EXPECT_TRUE(AL->hasArg(OPT_A));
-  EXPECT_TRUE(AL->hasArg(OPT_Slurp));
-  EXPECT_EQ(AL->getAllArgValues(OPT_Slurp).size(), 0U);
+  InputArgList AL = T.ParseArgs(MyArgs, MAI, MAC);
+  EXPECT_TRUE(AL.hasArg(OPT_A));
+  EXPECT_TRUE(AL.hasArg(OPT_Slurp));
+  EXPECT_EQ(AL.getAllArgValues(OPT_Slurp).size(), 0U);
 }
 
 TEST(Option, Slurp) {
@@ -198,14 +191,26 @@ TEST(Option, Slurp) {
   unsigned MAI, MAC;
 
   const char *MyArgs[] = { "-A", "-slurp", "-B", "--", "foo" };
-  std::unique_ptr<InputArgList> AL(
-      T.ParseArgs(std::begin(MyArgs), std::end(MyArgs), MAI, MAC));
-  EXPECT_EQ(AL->size(), 2U);
-  EXPECT_TRUE(AL->hasArg(OPT_A));
-  EXPECT_FALSE(AL->hasArg(OPT_B));
-  EXPECT_TRUE(AL->hasArg(OPT_Slurp));
-  EXPECT_EQ(AL->getAllArgValues(OPT_Slurp).size(), 3U);
-  EXPECT_EQ(AL->getAllArgValues(OPT_Slurp)[0], "-B");
-  EXPECT_EQ(AL->getAllArgValues(OPT_Slurp)[1], "--");
-  EXPECT_EQ(AL->getAllArgValues(OPT_Slurp)[2], "foo");
+  InputArgList AL = T.ParseArgs(MyArgs, MAI, MAC);
+  EXPECT_EQ(AL.size(), 2U);
+  EXPECT_TRUE(AL.hasArg(OPT_A));
+  EXPECT_FALSE(AL.hasArg(OPT_B));
+  EXPECT_TRUE(AL.hasArg(OPT_Slurp));
+  EXPECT_EQ(AL.getAllArgValues(OPT_Slurp).size(), 3U);
+  EXPECT_EQ(AL.getAllArgValues(OPT_Slurp)[0], "-B");
+  EXPECT_EQ(AL.getAllArgValues(OPT_Slurp)[1], "--");
+  EXPECT_EQ(AL.getAllArgValues(OPT_Slurp)[2], "foo");
+}
+
+TEST(Option, FlagAliasToJoined) {
+  TestOptTable T;
+  unsigned MAI, MAC;
+
+  // Check that a flag alias provides an empty argument to a joined option.
+  const char *MyArgs[] = { "-K" };
+  InputArgList AL = T.ParseArgs(MyArgs, MAI, MAC);
+  EXPECT_EQ(AL.size(), 1U);
+  EXPECT_TRUE(AL.hasArg(OPT_B));
+  EXPECT_EQ(AL.getAllArgValues(OPT_B).size(), 1U);
+  EXPECT_EQ(AL.getAllArgValues(OPT_B)[0], "");
 }
diff --git a/unittests/Option/Opts.td b/unittests/Option/Opts.td
index aaed6b2101e..c96774a68e0 100644
--- a/unittests/Option/Opts.td
+++ b/unittests/Option/Opts.td
@@ -23,4 +23,6 @@ def I : Flag<["-"], "I">, Alias<H>, Group<my_group>;
 def J : Flag<["-"], "J">, Alias<B>, AliasArgs<["foo"]>;
 def Joo : Flag<["-"], "Joo">, Alias<B>, AliasArgs<["bar"]>;
 
+def K : Flag<["-"], "K">, Alias<B>;
+
 def Slurp : Option<["-"], "slurp", KIND_REMAINING_ARGS>;
diff --git a/unittests/ProfileData/CoverageMappingTest.cpp b/unittests/ProfileData/CoverageMappingTest.cpp
index 9fceacbec89..a0995fbbc02 100644
--- a/unittests/ProfileData/CoverageMappingTest.cpp
+++ b/unittests/ProfileData/CoverageMappingTest.cpp
@@ -222,6 +222,21 @@ TEST_F(CoverageMappingTest, uncovered_function) {
   ASSERT_EQ(CoverageSegment(3, 4, false),   Segments[1]);
 }
 
+TEST_F(CoverageMappingTest, uncovered_function_with_mapping) {
+  readProfCounts();
+
+  addCMR(Counter::getCounter(0), "file1", 1, 1, 9, 9);
+  addCMR(Counter::getCounter(1), "file1", 1, 1, 4, 7);
+  loadCoverageMapping("func", 0x1234);
+
+  CoverageData Data = LoadedCoverage->getCoverageForFile("file1");
+  std::vector<CoverageSegment> Segments(Data.begin(), Data.end());
+  ASSERT_EQ(3U, Segments.size());
+  ASSERT_EQ(CoverageSegment(1, 1, 0, true),  Segments[0]);
+  ASSERT_EQ(CoverageSegment(4, 7, 0, false), Segments[1]);
+  ASSERT_EQ(CoverageSegment(9, 9, false),    Segments[2]);
+}
+
 TEST_F(CoverageMappingTest, combine_regions) {
   ProfileWriter.addFunctionCounts("func", 0x1234, {10, 20, 30});
   readProfCounts();
@@ -259,4 +274,18 @@ TEST_F(CoverageMappingTest, dont_combine_expansions) {
   ASSERT_EQ(CoverageSegment(9, 9, false), Segments[3]);
 }
 
+TEST_F(CoverageMappingTest, strip_filename_prefix) {
+  ProfileWriter.addFunctionCounts("file1:func", 0x1234, {10});
+  readProfCounts();
+
+  addCMR(Counter::getCounter(0), "file1", 1, 1, 9, 9);
+  loadCoverageMapping("file1:func", 0x1234);
+
+  std::vector<std::string> Names;
+  for (const auto &Func : LoadedCoverage->getCoveredFunctions())
+    Names.push_back(Func.Name);
+  ASSERT_EQ(1U, Names.size());
+  ASSERT_EQ("func", Names[0]);
+}
+
 } // end anonymous namespace
diff --git a/unittests/ProfileData/InstrProfTest.cpp b/unittests/ProfileData/InstrProfTest.cpp
index 26ea0e44ca4..2cedd593eb8 100644
--- a/unittests/ProfileData/InstrProfTest.cpp
+++ b/unittests/ProfileData/InstrProfTest.cpp
@@ -68,6 +68,7 @@ TEST_F(InstrProfTest, write_and_read_one_function) {
 
 TEST_F(InstrProfTest, get_function_counts) {
   Writer.addFunctionCounts("foo", 0x1234, {1, 2});
+  Writer.addFunctionCounts("foo", 0x1235, {3, 4});
   auto Profile = Writer.writeBuffer();
   readProfile(std::move(Profile));
 
@@ -77,6 +78,11 @@ TEST_F(InstrProfTest, get_function_counts) {
   ASSERT_EQ(1U, Counts[0]);
   ASSERT_EQ(2U, Counts[1]);
 
+  ASSERT_TRUE(NoError(Reader->getFunctionCounts("foo", 0x1235, Counts)));
+  ASSERT_EQ(2U, Counts.size());
+  ASSERT_EQ(3U, Counts[0]);
+  ASSERT_EQ(4U, Counts[1]);
+
   std::error_code EC;
   EC = Reader->getFunctionCounts("foo", 0x5678, Counts);
   ASSERT_TRUE(ErrorEquals(instrprof_error::hash_mismatch, EC));
diff --git a/unittests/Support/AllocatorTest.cpp b/unittests/Support/AllocatorTest.cpp
index 7f15776d6f0..4b544641e9b 100644
--- a/unittests/Support/AllocatorTest.cpp
+++ b/unittests/Support/AllocatorTest.cpp
@@ -61,6 +61,13 @@ TEST(AllocatorTest, ThreeSlabs) {
 // again.
 TEST(AllocatorTest, TestReset) {
   BumpPtrAllocator Alloc;
+
+  // Allocate something larger than the SizeThreshold=4096.
+  (void)Alloc.Allocate(5000, 1);
+  Alloc.Reset();
+  // Calling Reset should free all CustomSizedSlabs.
+  EXPECT_EQ(0u, Alloc.GetNumSlabs());
+
   Alloc.Allocate(3000, 1);
   EXPECT_EQ(1U, Alloc.GetNumSlabs());
   Alloc.Allocate(3000, 1);
@@ -122,7 +129,7 @@ TEST(AllocatorTest, TestAlignmentPastSlab) {
 
   // Aligning the current slab pointer is likely to move it past the end of the
   // slab, which would confuse any unsigned comparisons with the difference of
-  // the the end pointer and the aligned pointer.
+  // the end pointer and the aligned pointer.
   Alloc.Allocate(1024, 8192);
 
   EXPECT_EQ(2U, Alloc.GetNumSlabs());
diff --git a/unittests/Support/CMakeLists.txt b/unittests/Support/CMakeLists.txt
index 3c8a090c2ba..564d189d49c 100644
--- a/unittests/Support/CMakeLists.txt
+++ b/unittests/Support/CMakeLists.txt
@@ -1,4 +1,5 @@
 set(LLVM_LINK_COMPONENTS
+  ${LLVM_TARGETS_TO_BUILD}
   Support
   )
 
@@ -37,6 +38,7 @@ add_llvm_unittest(SupportTests
   StreamingMemoryObject.cpp
   StringPool.cpp
   SwapByteOrderTest.cpp
+  TargetRegistry.cpp
   ThreadLocalTest.cpp
   TimeValueTest.cpp
   UnicodeTest.cpp
diff --git a/unittests/Support/CommandLineTest.cpp b/unittests/Support/CommandLineTest.cpp
index 328c4b7fcf3..e0fbf5b09e5 100644
--- a/unittests/Support/CommandLineTest.cpp
+++ b/unittests/Support/CommandLineTest.cpp
@@ -10,6 +10,7 @@
 #include "llvm/ADT/STLExtras.h"
 #include "llvm/Config/config.h"
 #include "llvm/Support/CommandLine.h"
+#include "llvm/Support/StringSaver.h"
 #include "gtest/gtest.h"
 #include <stdlib.h>
 #include <string>
@@ -146,26 +147,20 @@ TEST(CommandLineTest, UseOptionCategory) {
                                                   "Category.";
 }
 
-class StrDupSaver : public cl::StringSaver {
-  const char *SaveString(const char *Str) override {
-    return strdup(Str);
-  }
-};
-
-typedef void ParserFunction(StringRef Source, llvm::cl::StringSaver &Saver,
+typedef void ParserFunction(StringRef Source, StringSaver &Saver,
                             SmallVectorImpl<const char *> &NewArgv,
                             bool MarkEOLs);
 
 void testCommandLineTokenizer(ParserFunction *parse, const char *Input,
                               const char *const Output[], size_t OutputSize) {
   SmallVector<const char *, 0> Actual;
-  StrDupSaver Saver;
+  BumpPtrAllocator A;
+  BumpPtrStringSaver Saver(A);
   parse(Input, Saver, Actual, /*MarkEOLs=*/false);
   EXPECT_EQ(OutputSize, Actual.size());
   for (unsigned I = 0, E = Actual.size(); I != E; ++I) {
     if (I < OutputSize)
       EXPECT_STREQ(Output[I], Actual[I]);
-    free(const_cast<char *>(Actual[I]));
   }
 }
 
diff --git a/unittests/Support/ErrorOrTest.cpp b/unittests/Support/ErrorOrTest.cpp
index 82bbe090960..73d0e3f2bb7 100644
--- a/unittests/Support/ErrorOrTest.cpp
+++ b/unittests/Support/ErrorOrTest.cpp
@@ -66,6 +66,11 @@ TEST(ErrorOr, Covariant) {
   ErrorOr<std::unique_ptr<int>> b4(b3);
 }
 
+TEST(ErrorOr, Comparison) {
+  ErrorOr<int> x(errc::no_such_file_or_directory);
+  EXPECT_EQ(x, errc::no_such_file_or_directory);
+}
+
 // ErrorOr<int*> x(nullptr);
 // ErrorOr<std::unique_ptr<int>> y = x; // invalid conversion
 static_assert(
diff --git a/unittests/Support/Makefile b/unittests/Support/Makefile
index 9c0a7f94d77..21657f12e3d 100644
--- a/unittests/Support/Makefile
+++ b/unittests/Support/Makefile
@@ -9,7 +9,7 @@
 
 LEVEL = ../..
 TESTNAME = Support
-LINK_COMPONENTS := core support
+LINK_COMPONENTS := all-targets core support
 
 include $(LEVEL)/Makefile.config
 include $(LLVM_SRC_ROOT)/unittests/Makefile.unittest
diff --git a/unittests/Support/Path.cpp b/unittests/Support/Path.cpp
index 262d27260ce..210b3a04cb2 100644
--- a/unittests/Support/Path.cpp
+++ b/unittests/Support/Path.cpp
@@ -16,7 +16,7 @@
 #include "gtest/gtest.h"
 
 #ifdef LLVM_ON_WIN32
-#include <Windows.h>
+#include <windows.h>
 #include <winerror.h>
 #endif
 
diff --git a/unittests/Support/ScaledNumberTest.cpp b/unittests/Support/ScaledNumberTest.cpp
index d8d6e310ff9..2f38b2a40fb 100644
--- a/unittests/Support/ScaledNumberTest.cpp
+++ b/unittests/Support/ScaledNumberTest.cpp
@@ -155,7 +155,7 @@ TEST(ScaledNumberHelpersTest, getQuotient) {
   EXPECT_EQ(SP32(0xaaaaaaab, -33), getQuotient32(1, 3));
   EXPECT_EQ(SP32(0xd5555555, -31), getQuotient32(5, 3));
 
-  // 64-bit division is hard to test, since divide64 doesn't canonicalized its
+  // 64-bit division is hard to test, since divide64 doesn't canonicalize its
   // output.  However, this is the algorithm the implementation uses:
   //
   // - Shift divisor right.
@@ -532,4 +532,33 @@ TEST(ScaledNumberHelpersTest, getDifference) {
   EXPECT_EQ(SP64(0, 0), getDifference64(1, -64, 1, -1));
 }
 
+TEST(ScaledNumberHelpersTest, arithmeticOperators) {
+  EXPECT_EQ(ScaledNumber<uint32_t>(10, 0),
+            ScaledNumber<uint32_t>(1, 3) + ScaledNumber<uint32_t>(1, 1));
+  EXPECT_EQ(ScaledNumber<uint32_t>(6, 0),
+            ScaledNumber<uint32_t>(1, 3) - ScaledNumber<uint32_t>(1, 1));
+  EXPECT_EQ(ScaledNumber<uint32_t>(2, 3),
+            ScaledNumber<uint32_t>(1, 3) * ScaledNumber<uint32_t>(1, 1));
+  EXPECT_EQ(ScaledNumber<uint32_t>(1, 2),
+            ScaledNumber<uint32_t>(1, 3) / ScaledNumber<uint32_t>(1, 1));
+  EXPECT_EQ(ScaledNumber<uint32_t>(1, 2), ScaledNumber<uint32_t>(1, 3) >> 1);
+  EXPECT_EQ(ScaledNumber<uint32_t>(1, 4), ScaledNumber<uint32_t>(1, 3) << 1);
+
+  EXPECT_EQ(ScaledNumber<uint64_t>(10, 0),
+            ScaledNumber<uint64_t>(1, 3) + ScaledNumber<uint64_t>(1, 1));
+  EXPECT_EQ(ScaledNumber<uint64_t>(6, 0),
+            ScaledNumber<uint64_t>(1, 3) - ScaledNumber<uint64_t>(1, 1));
+  EXPECT_EQ(ScaledNumber<uint64_t>(2, 3),
+            ScaledNumber<uint64_t>(1, 3) * ScaledNumber<uint64_t>(1, 1));
+  EXPECT_EQ(ScaledNumber<uint64_t>(1, 2),
+            ScaledNumber<uint64_t>(1, 3) / ScaledNumber<uint64_t>(1, 1));
+  EXPECT_EQ(ScaledNumber<uint64_t>(1, 2), ScaledNumber<uint64_t>(1, 3) >> 1);
+  EXPECT_EQ(ScaledNumber<uint64_t>(1, 4), ScaledNumber<uint64_t>(1, 3) << 1);
+}
+
+TEST(ScaledNumberHelpersTest, toIntBug) {
+  ScaledNumber<uint32_t> n(1, 0);
+  EXPECT_EQ(1u, (n * n).toInt<uint32_t>());
+}
+
 } // end namespace
diff --git a/unittests/Support/StreamingMemoryObject.cpp b/unittests/Support/StreamingMemoryObject.cpp
index 66dd2fc4c33..e86aa9cae51 100644
--- a/unittests/Support/StreamingMemoryObject.cpp
+++ b/unittests/Support/StreamingMemoryObject.cpp
@@ -7,6 +7,7 @@
 //
 //===----------------------------------------------------------------------===//
 
+#include "llvm/ADT/STLExtras.h"
 #include "llvm/Support/StreamingMemoryObject.h"
 #include "gtest/gtest.h"
 #include <string.h>
@@ -23,8 +24,16 @@ class NullDataStreamer : public DataStreamer {
 }
 
 TEST(StreamingMemoryObject, Test) {
-  auto *DS = new NullDataStreamer();
-  // @LOCALMOD -- Separated StreamingMemoryObject into interface + Impl.
-  StreamingMemoryObjectImpl O(DS);
+  auto DS = make_unique<NullDataStreamer>();
+  StreamingMemoryObject O(std::move(DS));
   EXPECT_TRUE(O.isValidAddress(32 * 1024));
 }
+
+TEST(StreamingMemoryObject, TestSetKnownObjectSize) {
+  auto DS = make_unique<NullDataStreamer>();
+  StreamingMemoryObject O(std::move(DS));
+  uint8_t Buf[32];
+  EXPECT_EQ((uint64_t) 16, O.readBytes(Buf, 16, 0));
+  O.setKnownObjectSize(24);
+  EXPECT_EQ((uint64_t) 8, O.readBytes(Buf, 16, 16));
+}
diff --git a/unittests/Support/SwapByteOrderTest.cpp b/unittests/Support/SwapByteOrderTest.cpp
index 525cfc1aea8..4f2537c4d5d 100644
--- a/unittests/Support/SwapByteOrderTest.cpp
+++ b/unittests/Support/SwapByteOrderTest.cpp
@@ -125,6 +125,19 @@ TEST(getSwappedBytes, int64_t) {
     sys::getSwappedBytes(int64_t(0x8877665544332211LL)));
 }
 
+TEST(getSwappedBytes, float) {
+  EXPECT_EQ(1.79366203433576585078237386661e-43f, sys::getSwappedBytes(-0.0f));
+  // 0x11223344
+  EXPECT_EQ(7.1653228759765625e2f, sys::getSwappedBytes(1.2795344e-28f));
+}
+
+TEST(getSwappedBytes, double) {
+  EXPECT_EQ(6.32404026676795576546008054871e-322, sys::getSwappedBytes(-0.0));
+  // 0x1122334455667788
+  EXPECT_EQ(-7.08687663657301358331704585496e-268,
+    sys::getSwappedBytes(3.84141202447173065923064450234e-226));
+}
+
 TEST(swapByteOrder, uint8_t) {
   uint8_t value = 0x11;
   sys::swapByteOrder(value);
@@ -173,4 +186,16 @@ TEST(swapByteOrder, int64_t) {
   EXPECT_EQ(int64_t(0x1122334455667788LL), value);
 }
 
+TEST(swapByteOrder, float) {
+  float value = 7.1653228759765625e2f; // 0x44332211
+  sys::swapByteOrder(value);
+  EXPECT_EQ(1.2795344e-28f, value);
+}
+
+TEST(swapByteOrder, double) {
+  double value = -7.08687663657301358331704585496e-268; // 0x8877665544332211
+  sys::swapByteOrder(value);
+  EXPECT_EQ(3.84141202447173065923064450234e-226, value);
+}
+
 }
diff --git a/unittests/Support/TargetRegistry.cpp b/unittests/Support/TargetRegistry.cpp
new file mode 100644
index 00000000000..ae89c8b6493
--- /dev/null
+++ b/unittests/Support/TargetRegistry.cpp
@@ -0,0 +1,42 @@
+//===- unittests/Support/TargetRegistry.cpp - -----------------------------===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/Support/TargetRegistry.h"
+#include "llvm/Support/TargetSelect.h"
+#include "gtest/gtest.h"
+
+using namespace llvm;
+
+namespace {
+
+TEST(TargetRegistry, TargetHasArchType) {
+  // Presence of at least one target will be asserted when done with the loop,
+  // else this would pass by accident if InitializeAllTargetInfos were omitted.
+  int Count = 0;
+
+  llvm::InitializeAllTargetInfos();
+
+  for (const Target &T : TargetRegistry::targets()) {
+    StringRef Name = T.getName();
+    // There is really no way (at present) to ask a Target whether it targets
+    // a specific architecture, because the logic for that is buried in a
+    // predicate.
+    // We can't ask the predicate "Are you a function that always returns
+    // false?"
+    // So given that the cpp backend truly has no target arch, it is skipped.
+    if (Name != "cpp") {
+      Triple::ArchType Arch = Triple::getArchTypeForLLVMName(Name);
+      EXPECT_NE(Arch, Triple::UnknownArch);
+      ++Count;
+    }
+  }
+  ASSERT_NE(Count, 0);
+}
+
+} // end namespace
diff --git a/unittests/Support/YAMLIOTest.cpp b/unittests/Support/YAMLIOTest.cpp
index 3104726941e..e7affa1698d 100644
--- a/unittests/Support/YAMLIOTest.cpp
+++ b/unittests/Support/YAMLIOTest.cpp
@@ -779,6 +779,146 @@ TEST(YAMLIO, TestReadWriteMyCustomType) {
 }
 
 
+//===----------------------------------------------------------------------===//
+//  Test BlockScalarTraits
+//===----------------------------------------------------------------------===//
+
+struct MultilineStringType {
+  std::string str;
+};
+
+struct MultilineStringTypeMap {
+  MultilineStringType name;
+  MultilineStringType description;
+  MultilineStringType ingredients;
+  MultilineStringType recipes;
+  MultilineStringType warningLabels;
+  MultilineStringType documentation;
+  int price;
+};
+
+namespace llvm {
+namespace yaml {
+  template <>
+  struct MappingTraits<MultilineStringTypeMap> {
+    static void mapping(IO &io, MultilineStringTypeMap& s) {
+      io.mapRequired("name", s.name);
+      io.mapRequired("description", s.description);
+      io.mapRequired("ingredients", s.ingredients);
+      io.mapRequired("recipes", s.recipes);
+      io.mapRequired("warningLabels", s.warningLabels);
+      io.mapRequired("documentation", s.documentation);
+      io.mapRequired("price", s.price);
+     }
+  };
+
+  // MultilineStringType is formatted as a yaml block literal scalar. A value of
+  // "Hello\nWorld" would be represented in yaml as
+  //  |
+  //    Hello
+  //    World
+  template <>
+  struct BlockScalarTraits<MultilineStringType> {
+    static void output(const MultilineStringType &value, void *ctxt,
+                       llvm::raw_ostream &out) {
+      out << value.str;
+    }
+    static StringRef input(StringRef scalar, void *ctxt,
+                           MultilineStringType &value) {
+      value.str = scalar.str();
+      return StringRef();
+    }
+  };
+}
+}
+
+LLVM_YAML_IS_DOCUMENT_LIST_VECTOR(MultilineStringType)
+
+//
+// Test writing then reading back custom values
+//
+TEST(YAMLIO, TestReadWriteMultilineStringType) {
+  std::string intermediate;
+  {
+    MultilineStringTypeMap map;
+    map.name.str = "An Item";
+    map.description.str = "Hello\nWorld";
+    map.ingredients.str = "SubItem 1\nSub Item 2\n\nSub Item 3\n";
+    map.recipes.str = "\n\nTest 1\n\n\n";
+    map.warningLabels.str = "";
+    map.documentation.str = "\n\n";
+    map.price = 350;
+
+    llvm::raw_string_ostream ostr(intermediate);
+    Output yout(ostr);
+    yout << map;
+  }
+  {
+    Input yin(intermediate);
+    MultilineStringTypeMap map2;
+    yin >> map2;
+
+    EXPECT_FALSE(yin.error());
+    EXPECT_EQ(map2.name.str, "An Item\n");
+    EXPECT_EQ(map2.description.str, "Hello\nWorld\n");
+    EXPECT_EQ(map2.ingredients.str, "SubItem 1\nSub Item 2\n\nSub Item 3\n");
+    EXPECT_EQ(map2.recipes.str, "\n\nTest 1\n");
+    EXPECT_TRUE(map2.warningLabels.str.empty());
+    EXPECT_TRUE(map2.documentation.str.empty());
+    EXPECT_EQ(map2.price, 350);
+  }
+}
+
+//
+// Test writing then reading back custom values
+//
+TEST(YAMLIO, TestReadWriteBlockScalarDocuments) {
+  std::string intermediate;
+  {
+    std::vector<MultilineStringType> documents;
+    MultilineStringType doc;
+    doc.str = "Hello\nWorld";
+    documents.push_back(doc);
+
+    llvm::raw_string_ostream ostr(intermediate);
+    Output yout(ostr);
+    yout << documents;
+
+    // Verify that the block scalar header was written out on the same line
+    // as the document marker.
+    EXPECT_NE(llvm::StringRef::npos, llvm::StringRef(ostr.str()).find("--- |"));
+  }
+  {
+    Input yin(intermediate);
+    std::vector<MultilineStringType> documents2;
+    yin >> documents2;
+
+    EXPECT_FALSE(yin.error());
+    EXPECT_EQ(documents2.size(), size_t(1));
+    EXPECT_EQ(documents2[0].str, "Hello\nWorld\n");
+  }
+}
+
+TEST(YAMLIO, TestReadWriteBlockScalarValue) {
+  std::string intermediate;
+  {
+    MultilineStringType doc;
+    doc.str = "Just a block\nscalar doc";
+
+    llvm::raw_string_ostream ostr(intermediate);
+    Output yout(ostr);
+    yout << doc;
+  }
+  {
+    Input yin(intermediate);
+    MultilineStringType doc;
+    yin >> doc;
+
+    EXPECT_FALSE(yin.error());
+    EXPECT_EQ(doc.str, "Just a block\nscalar doc\n");
+  }
+}
+
 //===----------------------------------------------------------------------===//
 //  Test flow sequences
 //===----------------------------------------------------------------------===//
@@ -829,6 +969,26 @@ namespace yaml {
 }
 }
 
+typedef std::vector<MyNumber> MyNumberFlowSequence;
+
+LLVM_YAML_IS_SEQUENCE_VECTOR(MyNumberFlowSequence)
+
+struct NameAndNumbersFlow {
+  llvm::StringRef                    name;
+  std::vector<MyNumberFlowSequence>  sequenceOfNumbers;
+};
+
+namespace llvm {
+namespace yaml {
+  template <>
+  struct MappingTraits<NameAndNumbersFlow> {
+    static void mapping(IO &io, NameAndNumbersFlow& nn) {
+      io.mapRequired("name",     nn.name);
+      io.mapRequired("sequenceOfNumbers",  nn.sequenceOfNumbers);
+    }
+  };
+}
+}
 
 //
 // Test writing then reading back custom values
@@ -876,6 +1036,51 @@ TEST(YAMLIO, TestReadWriteMyFlowSequence) {
 }
 
 
+//
+// Test writing then reading back a sequence of flow sequences.
+//
+TEST(YAMLIO, TestReadWriteSequenceOfMyFlowSequence) {
+  std::string intermediate;
+  {
+    NameAndNumbersFlow map;
+    map.name  = "hello";
+    MyNumberFlowSequence single = { 0 };
+    MyNumberFlowSequence numbers = { 12, 1, -512 };
+    map.sequenceOfNumbers.push_back(single);
+    map.sequenceOfNumbers.push_back(numbers);
+    map.sequenceOfNumbers.push_back(MyNumberFlowSequence());
+
+    llvm::raw_string_ostream ostr(intermediate);
+    Output yout(ostr);
+    yout << map;
+
+    // Verify sequences were written in flow style
+    // and that the parent sequence used '-'.
+    ostr.flush();
+    llvm::StringRef flowOut(intermediate);
+    EXPECT_NE(llvm::StringRef::npos, flowOut.find("- [ 0 ]"));
+    EXPECT_NE(llvm::StringRef::npos, flowOut.find("- [ 12, 1, -512 ]"));
+    EXPECT_NE(llvm::StringRef::npos, flowOut.find("- [  ]"));
+  }
+
+  {
+    Input yin(intermediate);
+    NameAndNumbersFlow map2;
+    yin >> map2;
+
+    EXPECT_FALSE(yin.error());
+    EXPECT_TRUE(map2.name.equals("hello"));
+    EXPECT_EQ(map2.sequenceOfNumbers.size(), 3UL);
+    EXPECT_EQ(map2.sequenceOfNumbers[0].size(), 1UL);
+    EXPECT_EQ(0,    map2.sequenceOfNumbers[0][0]);
+    EXPECT_EQ(map2.sequenceOfNumbers[1].size(), 3UL);
+    EXPECT_EQ(12,   map2.sequenceOfNumbers[1][0]);
+    EXPECT_EQ(1,    map2.sequenceOfNumbers[1][1]);
+    EXPECT_EQ(-512, map2.sequenceOfNumbers[1][2]);
+    EXPECT_TRUE(map2.sequenceOfNumbers[2].empty());
+  }
+}
+
 //===----------------------------------------------------------------------===//
 //  Test normalizing/denormalizing
 //===----------------------------------------------------------------------===//
@@ -1302,6 +1507,91 @@ TEST(YAMLIO, TestValidatingInput) {
   EXPECT_TRUE(!!yin.error());
 }
 
+//===----------------------------------------------------------------------===//
+//  Test flow mapping
+//===----------------------------------------------------------------------===//
+
+struct FlowFooBar {
+  int foo;
+  int bar;
+
+  FlowFooBar() : foo(0), bar(0) {}
+  FlowFooBar(int foo, int bar) : foo(foo), bar(bar) {}
+};
+
+typedef std::vector<FlowFooBar> FlowFooBarSequence;
+
+LLVM_YAML_IS_SEQUENCE_VECTOR(FlowFooBar)
+
+struct FlowFooBarDoc {
+  FlowFooBar attribute;
+  FlowFooBarSequence seq;
+};
+
+namespace llvm {
+namespace yaml {
+  template <>
+  struct MappingTraits<FlowFooBar> {
+    static void mapping(IO &io, FlowFooBar &fb) {
+      io.mapRequired("foo", fb.foo);
+      io.mapRequired("bar", fb.bar);
+    }
+
+    static const bool flow = true;
+  };
+
+  template <>
+  struct MappingTraits<FlowFooBarDoc> {
+    static void mapping(IO &io, FlowFooBarDoc &fb) {
+      io.mapRequired("attribute", fb.attribute);
+      io.mapRequired("seq", fb.seq);
+    }
+  };
+}
+}
+
+//
+// Test writing then reading back custom mappings
+//
+TEST(YAMLIO, TestReadWriteMyFlowMapping) {
+  std::string intermediate;
+  {
+    FlowFooBarDoc doc;
+    doc.attribute = FlowFooBar(42, 907);
+    doc.seq.push_back(FlowFooBar(1, 2));
+    doc.seq.push_back(FlowFooBar(0, 0));
+    doc.seq.push_back(FlowFooBar(-1, 1024));
+
+    llvm::raw_string_ostream ostr(intermediate);
+    Output yout(ostr);
+    yout << doc;
+
+    // Verify that mappings were written in flow style
+    ostr.flush();
+    llvm::StringRef flowOut(intermediate);
+    EXPECT_NE(llvm::StringRef::npos, flowOut.find("{ foo: 42, bar: 907 }"));
+    EXPECT_NE(llvm::StringRef::npos, flowOut.find("- { foo: 1, bar: 2 }"));
+    EXPECT_NE(llvm::StringRef::npos, flowOut.find("- { foo: 0, bar: 0 }"));
+    EXPECT_NE(llvm::StringRef::npos, flowOut.find("- { foo: -1, bar: 1024 }"));
+  }
+
+  {
+    Input yin(intermediate);
+    FlowFooBarDoc doc2;
+    yin >> doc2;
+
+    EXPECT_FALSE(yin.error());
+    EXPECT_EQ(doc2.attribute.foo, 42);
+    EXPECT_EQ(doc2.attribute.bar, 907);
+    EXPECT_EQ(doc2.seq.size(), 3UL);
+    EXPECT_EQ(doc2.seq[0].foo, 1);
+    EXPECT_EQ(doc2.seq[0].bar, 2);
+    EXPECT_EQ(doc2.seq[1].foo, 0);
+    EXPECT_EQ(doc2.seq[1].bar, 0);
+    EXPECT_EQ(doc2.seq[2].foo, -1);
+    EXPECT_EQ(doc2.seq[2].bar, 1024);
+  }
+}
 
 //===----------------------------------------------------------------------===//
 //  Test error handling
@@ -1784,3 +2074,123 @@ TEST(YAMLIO, TestEmptyStringSucceedsForSequence) {
   EXPECT_FALSE(yin.error());
   EXPECT_TRUE(seq.empty());
 }
+
+struct FlowMap {
+  llvm::StringRef str1, str2, str3;
+  FlowMap(llvm::StringRef str1, llvm::StringRef str2, llvm::StringRef str3)
+    : str1(str1), str2(str2), str3(str3) {}
+};
+
+struct FlowSeq {
+  llvm::StringRef str;
+  FlowSeq(llvm::StringRef S) : str(S) {}
+  FlowSeq() = default;
+};
+
+namespace llvm {
+namespace yaml {
+  template <>
+  struct MappingTraits<FlowMap> {
+    static void mapping(IO &io, FlowMap &fm) {
+      io.mapRequired("str1", fm.str1);
+      io.mapRequired("str2", fm.str2);
+      io.mapRequired("str3", fm.str3);
+    }
+
+    static const bool flow = true;
+  };
+
+template <>
+struct ScalarTraits<FlowSeq> {
+  static void output(const FlowSeq &value, void*, llvm::raw_ostream &out) {
+    out << value.str;
+  }
+  static StringRef input(StringRef scalar, void*, FlowSeq &value) {
+    value.str = scalar;
+    return "";
+  }
+
+  static bool mustQuote(StringRef S) { return false; }
+};
+}
+}
+
+LLVM_YAML_IS_FLOW_SEQUENCE_VECTOR(FlowSeq)
+
+TEST(YAMLIO, TestWrapFlow) {
+  std::string out;
+  llvm::raw_string_ostream ostr(out);
+  FlowMap Map("This is str1", "This is str2", "This is str3");
+  std::vector<FlowSeq> Seq;
+  Seq.emplace_back("This is str1");
+  Seq.emplace_back("This is str2");
+  Seq.emplace_back("This is str3");
+
+  {
+    // 20 is just bellow the total length of the first mapping field.
+    // We should wreap at every element.
+    Output yout(ostr, nullptr, 15);
+
+    yout << Map;
+    ostr.flush();
+    EXPECT_EQ(out,
+              "---\n"
+              "{ str1: This is str1, \n"
+              "  str2: This is str2, \n"
+              "  str3: This is str3 }\n"
+              "...\n");
+    out.clear();
+
+    yout << Seq;
+    ostr.flush();
+    EXPECT_EQ(out,
+              "---\n"
+              "[ This is str1, \n"
+              "  This is str2, \n"
+              "  This is str3 ]\n"
+              "...\n");
+    out.clear();
+  }
+  {
+    // 25 will allow the second field to be output on the first line.
+    Output yout(ostr, nullptr, 25);
+
+    yout << Map;
+    ostr.flush();
+    EXPECT_EQ(out,
+              "---\n"
+              "{ str1: This is str1, str2: This is str2, \n"
+              "  str3: This is str3 }\n"
+              "...\n");
+    out.clear();
+
+    yout << Seq;
+    ostr.flush();
+    EXPECT_EQ(out,
+              "---\n"
+              "[ This is str1, This is str2, \n"
+              "  This is str3 ]\n"
+              "...\n");
+    out.clear();
+  }
+  {
+    // 0 means no wrapping.
+    Output yout(ostr, nullptr, 0);
+
+    yout << Map;
+    ostr.flush();
+    EXPECT_EQ(out,
+              "---\n"
+              "{ str1: This is str1, str2: This is str2, str3: This is str3 }\n"
+              "...\n");
+    out.clear();
+
+    yout << Seq;
+    ostr.flush();
+    EXPECT_EQ(out,
+              "---\n"
+              "[ This is str1, This is str2, This is str3 ]\n"
+              "...\n");
+    out.clear();
+  }
+}
diff --git a/unittests/Support/YAMLParserTest.cpp b/unittests/Support/YAMLParserTest.cpp
index 823a0d6e3e0..69b354a91d1 100644
--- a/unittests/Support/YAMLParserTest.cpp
+++ b/unittests/Support/YAMLParserTest.cpp
@@ -130,6 +130,45 @@ TEST(YAMLParser, ParsesArrayOfArrays) {
   ExpectParseSuccess("Array of arrays", "[[]]");
 }
 
+TEST(YAMLParser, ParsesBlockLiteralScalars) {
+  ExpectParseSuccess("Block literal scalar", "test: |\n  Hello\n  World\n");
+  ExpectParseSuccess("Block literal scalar EOF", "test: |\n  Hello\n  World");
+  ExpectParseSuccess("Empty block literal scalar header EOF", "test: | ");
+  ExpectParseSuccess("Empty block literal scalar", "test: |\ntest2: 20");
+  ExpectParseSuccess("Empty block literal scalar 2", "- | \n  \n\n \n- 42");
+  ExpectParseSuccess("Block literal scalar in sequence",
+                     "- |\n  Testing\n  Out\n\n- 22");
+  ExpectParseSuccess("Block literal scalar in document",
+                     "--- |\n  Document\n...");
+  ExpectParseSuccess("Empty non indented lines still count",
+                     "- |\n  First line\n \n\n  Another line\n\n- 2");
+  ExpectParseSuccess("Comment in block literal scalar header",
+                     "test: | # Comment \n  No Comment\ntest 2: | # Void");
+  ExpectParseSuccess("Chomping indicators in block literal scalar header",
+                     "test: |- \n  Hello\n\ntest 2: |+ \n\n  World\n\n\n");
+  ExpectParseSuccess("Indent indicators in block literal scalar header",
+                     "test: |1 \n  \n Hello \n  World\n");
+  ExpectParseSuccess("Chomping and indent indicators in block literals",
+                     "test: |-1\n Hello\ntest 2: |9+\n         World");
+  ExpectParseSuccess("Trailing comments in block literals",
+                     "test: |\n  Content\n # Trailing\n  #Comment\ntest 2: 3");
+  ExpectParseError("Invalid block scalar header", "test: | failure");
+  ExpectParseError("Invalid line indentation", "test: |\n  First line\n Error");
+  ExpectParseError("Long leading space line", "test: |\n   \n  Test\n");
+}
+
+TEST(YAMLParser, NullTerminatedBlockScalars) {
+  SourceMgr SM;
+  yaml::Stream Stream("test: |\n  Hello\n  World\n", SM);
+  yaml::Document &Doc = *Stream.begin();
+  yaml::MappingNode *Map = cast<yaml::MappingNode>(Doc.getRoot());
+  StringRef Value =
+      cast<yaml::BlockScalarNode>(Map->begin()->getValue())->getValue();
+
+  EXPECT_EQ(Value, "Hello\nWorld\n");
+  EXPECT_EQ(Value.data()[Value.size()], '\0');
+}
+
 TEST(YAMLParser, HandlesEndOfFileGracefully) {
   ExpectParseError("In string starting with EOF", "[\"");
   ExpectParseError("In string hitting EOF", "[\"   ");
@@ -141,6 +180,10 @@ TEST(YAMLParser, HandlesEndOfFileGracefully) {
   ExpectParseError("In object hitting EOF", "{\"\"");
 }
 
+TEST(YAMLParser, HandlesNullValuesInKeyValueNodesGracefully) {
+  ExpectParseError("KeyValueNode with null value", "test: '");
+}
+
 // Checks that the given string can be parsed into an identical string inside
 // of an array.
 static void ExpectCanParseString(StringRef String) {
diff --git a/unittests/Support/raw_pwrite_stream_test.cpp b/unittests/Support/raw_pwrite_stream_test.cpp
index bcbb29b86bd..a62f6bacb07 100644
--- a/unittests/Support/raw_pwrite_stream_test.cpp
+++ b/unittests/Support/raw_pwrite_stream_test.cpp
@@ -9,6 +9,7 @@
 
 #include "gtest/gtest.h"
 #include "llvm/ADT/SmallString.h"
+#include "llvm/Support/FileSystem.h"
 #include "llvm/Support/raw_ostream.h"
 
 using namespace llvm;
@@ -16,10 +17,48 @@ using namespace llvm;
 namespace {
 
 TEST(raw_pwrite_ostreamTest, TestSVector) {
-  SmallString<64> Buffer;
+  SmallVector<char, 0> Buffer;
   raw_svector_ostream OS(Buffer);
+  OS << "abcd";
   StringRef Test = "test";
   OS.pwrite(Test.data(), Test.size(), 0);
   EXPECT_EQ(Test, OS.str());
+
+#ifdef GTEST_HAS_DEATH_TEST
+#ifndef NDEBUG
+  EXPECT_DEATH(OS.pwrite("12345", 5, 0),
+               "We don't support extending the stream");
+#endif
+#endif
+}
+
+TEST(raw_pwrite_ostreamTest, TestFD) {
+  SmallString<64> Path;
+  int FD;
+  sys::fs::createTemporaryFile("foo", "bar", FD, Path);
+  raw_fd_ostream OS(FD, true);
+  OS << "abcd";
+  StringRef Test = "test";
+  OS.pwrite(Test.data(), Test.size(), 0);
+  OS.pwrite(Test.data(), Test.size(), 0);
+
+#ifdef GTEST_HAS_DEATH_TEST
+#ifndef NDEBUG
+  EXPECT_DEATH(OS.pwrite("12345", 5, 0),
+               "We don't support extending the stream");
+#endif
+#endif
+}
+
+#ifdef LLVM_ON_UNIX
+TEST(raw_pwrite_ostreamTest, TestDevNull) {
+  int FD;
+  sys::fs::openFileForWrite("/dev/null", FD, sys::fs::F_None);
+  raw_fd_ostream OS(FD, true);
+  OS << "abcd";
+  StringRef Test = "test";
+  OS.pwrite(Test.data(), Test.size(), 0);
+  OS.pwrite(Test.data(), Test.size(), 0);
 }
+#endif
 }
diff --git a/unittests/Transforms/Utils/Cloning.cpp b/unittests/Transforms/Utils/Cloning.cpp
index 636cb3caa50..3ceb84c42c7 100644
--- a/unittests/Transforms/Utils/Cloning.cpp
+++ b/unittests/Transforms/Utils/Cloning.cpp
@@ -228,15 +228,16 @@ class CloneFunc : public ::testing::Test {
     IRBuilder<> IBuilder(C);
 
     // Function DI
-    DIFile File = DBuilder.createFile("filename.c", "/file/dir/");
-    DITypeArray ParamTypes = DBuilder.getOrCreateTypeArray(None);
-    MDSubroutineType *FuncType =
+    auto *File = DBuilder.createFile("filename.c", "/file/dir/");
+    DITypeRefArray ParamTypes = DBuilder.getOrCreateTypeArray(None);
+    DISubroutineType *FuncType =
         DBuilder.createSubroutineType(File, ParamTypes);
-    DICompileUnit CU = DBuilder.createCompileUnit(dwarf::DW_LANG_C99,
-        "filename.c", "/file/dir", "CloneFunc", false, "", 0);
+    auto *CU =
+        DBuilder.createCompileUnit(dwarf::DW_LANG_C99, "filename.c",
+                                   "/file/dir", "CloneFunc", false, "", 0);
 
-    DISubprogram Subprogram = DBuilder.createFunction(CU, "f", "f", File, 4,
-        FuncType, true, true, 3, 0, false, OldFunc);
+    auto *Subprogram = DBuilder.createFunction(
+        CU, "f", "f", File, 4, FuncType, true, true, 3, 0, false, OldFunc);
 
     // Function body
     BasicBlock* Entry = BasicBlock::Create(C, "", OldFunc);
@@ -251,12 +252,12 @@ class CloneFunc : public ::testing::Test {
     Instruction* Terminator = IBuilder.CreateRetVoid();
 
     // Create a local variable around the alloca
-    DIType IntType = DBuilder.createBasicType("int", 32, 0,
-        dwarf::DW_ATE_signed);
-    DIExpression E = DBuilder.createExpression();
-    DIVariable Variable = DBuilder.createLocalVariable(
-      dwarf::DW_TAG_auto_variable, Subprogram, "x", File, 5, IntType, true);
-    auto *DL = MDLocation::get(Subprogram->getContext(), 5, 0, Subprogram);
+    auto *IntType =
+        DBuilder.createBasicType("int", 32, 0, dwarf::DW_ATE_signed);
+    auto *E = DBuilder.createExpression();
+    auto *Variable =
+        DBuilder.createAutoVariable(Subprogram, "x", File, 5, IntType, true);
+    auto *DL = DILocation::get(Subprogram->getContext(), 5, 0, Subprogram);
     DBuilder.insertDeclare(Alloca, Variable, E, DL, Store);
     DBuilder.insertDbgValueIntrinsic(AllocaContent, 0, Variable, E, DL,
                                      Terminator);
@@ -303,9 +304,9 @@ TEST_F(CloneFunc, Subprogram) {
   EXPECT_EQ(2U, SubprogramCount);
 
   auto Iter = Finder->subprograms().begin();
-  DISubprogram Sub1 = cast<MDSubprogram>(*Iter);
+  auto *Sub1 = cast<DISubprogram>(*Iter);
   Iter++;
-  DISubprogram Sub2 = cast<MDSubprogram>(*Iter);
+  auto *Sub2 = cast<DISubprogram>(*Iter);
 
   EXPECT_TRUE(
       (Sub1->getFunction() == OldFunc && Sub2->getFunction() == NewFunc) ||
@@ -320,9 +321,9 @@ TEST_F(CloneFunc, SubprogramInRightCU) {
   EXPECT_EQ(2U, Finder->compile_unit_count());
 
   auto Iter = Finder->compile_units().begin();
-  DICompileUnit CU1 = cast<MDCompileUnit>(*Iter);
+  auto *CU1 = cast<DICompileUnit>(*Iter);
   Iter++;
-  DICompileUnit CU2 = cast<MDCompileUnit>(*Iter);
+  auto *CU2 = cast<DICompileUnit>(*Iter);
   EXPECT_TRUE(CU1->getSubprograms().size() == 0 ||
               CU2->getSubprograms().size() == 0);
 }
@@ -351,8 +352,8 @@ TEST_F(CloneFunc, InstructionOwnership) {
       EXPECT_EQ(OldDL.getCol(), NewDL.getCol());
 
       // But that they belong to different functions
-      auto *OldSubprogram = cast<MDSubprogram>(OldDL.getScope());
-      auto *NewSubprogram = cast<MDSubprogram>(NewDL.getScope());
+      auto *OldSubprogram = cast<DISubprogram>(OldDL.getScope());
+      auto *NewSubprogram = cast<DISubprogram>(NewDL.getScope());
       EXPECT_EQ(OldFunc, OldSubprogram->getFunction());
       EXPECT_EQ(NewFunc, NewSubprogram->getFunction());
     }
@@ -389,11 +390,11 @@ TEST_F(CloneFunc, DebugIntrinsics) {
 
       // Old variable must belong to the old function
       EXPECT_EQ(OldFunc,
-                cast<MDSubprogram>(OldIntrin->getVariable()->getScope())
+                cast<DISubprogram>(OldIntrin->getVariable()->getScope())
                     ->getFunction());
       // New variable must belong to the New function
       EXPECT_EQ(NewFunc,
-                cast<MDSubprogram>(NewIntrin->getVariable()->getScope())
+                cast<DISubprogram>(NewIntrin->getVariable()->getScope())
                     ->getFunction());
     } else if (DbgValueInst* OldIntrin = dyn_cast<DbgValueInst>(&OldI)) {
       DbgValueInst* NewIntrin = dyn_cast<DbgValueInst>(&NewI);
@@ -401,11 +402,11 @@ TEST_F(CloneFunc, DebugIntrinsics) {
 
       // Old variable must belong to the old function
       EXPECT_EQ(OldFunc,
-                cast<MDSubprogram>(OldIntrin->getVariable()->getScope())
+                cast<DISubprogram>(OldIntrin->getVariable()->getScope())
                     ->getFunction());
       // New variable must belong to the New function
       EXPECT_EQ(NewFunc,
-                cast<MDSubprogram>(NewIntrin->getVariable()->getScope())
+                cast<DISubprogram>(NewIntrin->getVariable()->getScope())
                     ->getFunction());
     }
 
@@ -414,4 +415,39 @@ TEST_F(CloneFunc, DebugIntrinsics) {
   }
 }
 
+class CloneModule : public ::testing::Test {
+protected:
+  void SetUp() override {
+    SetupModule();
+    CreateOldModule();
+    CreateNewModule();
+  }
+
+  void SetupModule() { OldM = new Module("", C); }
+
+  void CreateOldModule() {
+    IRBuilder<> IBuilder(C);
+
+    auto *FuncType = FunctionType::get(Type::getVoidTy(C), false);
+    auto *PersFn = Function::Create(FuncType, GlobalValue::ExternalLinkage,
+                                    "persfn", OldM);
+    auto *F =
+        Function::Create(FuncType, GlobalValue::PrivateLinkage, "f", OldM);
+    F->setPersonalityFn(PersFn);
+    auto *Entry = BasicBlock::Create(C, "", F);
+    IBuilder.SetInsertPoint(Entry);
+    IBuilder.CreateRetVoid();
+  }
+
+  void CreateNewModule() { NewM = llvm::CloneModule(OldM); }
+
+  LLVMContext C;
+  Module *OldM;
+  Module *NewM;
+};
+
+TEST_F(CloneModule, Verify) {
+  EXPECT_FALSE(verifyModule(*NewM));
+}
+
 }
diff --git a/utils/FileCheck/FileCheck.cpp b/utils/FileCheck/FileCheck.cpp
index f7c0524a221..9b9ffb04906 100644
--- a/utils/FileCheck/FileCheck.cpp
+++ b/utils/FileCheck/FileCheck.cpp
@@ -946,10 +946,7 @@ static bool ReadCheckFile(SourceMgr &SM,
     }
 
     // Okay, add the string we captured to the output vector and move on.
-    CheckStrings.push_back(CheckString(P,
-                                       UsedPrefix,
-                                       PatternLoc,
-                                       CheckTy));
+    CheckStrings.emplace_back(P, UsedPrefix, PatternLoc, CheckTy);
     std::swap(DagNotMatches, CheckStrings.back().DagNotStrings);
     DagNotMatches = ImplicitNegativeChecks;
   }
@@ -957,22 +954,23 @@ static bool ReadCheckFile(SourceMgr &SM,
   // Add an EOF pattern for any trailing CHECK-DAG/-NOTs, and use the first
   // prefix as a filler for the error message.
   if (!DagNotMatches.empty()) {
-    CheckStrings.push_back(CheckString(Pattern(Check::CheckEOF),
-                                       CheckPrefixes[0],
-                                       SMLoc::getFromPointer(Buffer.data()),
-                                       Check::CheckEOF));
+    CheckStrings.emplace_back(Pattern(Check::CheckEOF), *CheckPrefixes.begin(),
+                              SMLoc::getFromPointer(Buffer.data()),
+                              Check::CheckEOF);
     std::swap(DagNotMatches, CheckStrings.back().DagNotStrings);
   }
 
   if (CheckStrings.empty()) {
     errs() << "error: no check strings found with prefix"
            << (CheckPrefixes.size() > 1 ? "es " : " ");
-    for (size_t I = 0, N = CheckPrefixes.size(); I != N; ++I) {
-      StringRef Prefix(CheckPrefixes[I]);
-      errs() << '\'' << Prefix << ":'";
-      if (I != N - 1)
-        errs() << ", ";
+    prefix_iterator I = CheckPrefixes.begin();
+    prefix_iterator E = CheckPrefixes.end();
+    if (I != E) {
+      errs() << "\'" << *I << ":'";
+      ++I;
     }
+    for (; I != E; ++I) 
+      errs() << ", \'" << *I << ":'";
 
     errs() << '\n';
     return true;
diff --git a/utils/Makefile b/utils/Makefile
index 84581c90762..a59318de483 100644
--- a/utils/Makefile
+++ b/utils/Makefile
@@ -9,18 +9,7 @@
 
 LEVEL = ..
 PARALLEL_DIRS := FileCheck TableGen PerfectShuffle count fpcmp llvm-lit not \
-                 unittest
-
-# @LOCALMOD-BEGIN
-ifeq ($(PNACL_BROWSER_TRANSLATOR),1)
-  # In sandboxed mode, just build the bare minimum
-  # Note: TableGen is usually built twice: 
-  #       * once with host compiler
-  #       * also with the "given" compiler
-  #  Here we just disable that second build
-  PARALLEL_DIRS := 
-endif
-# @LOCALMOD-END
+                 unittest yaml-bench
 
 EXTRA_DIST := check-each-file codegen-diff countloc.sh \
               DSAclean.py DSAextract.py emacs findsym.pl GenLibDeps.pl \
diff --git a/utils/TableGen/AsmMatcherEmitter.cpp b/utils/TableGen/AsmMatcherEmitter.cpp
index 159bc23fbff..0d7c5ffbea1 100644
--- a/utils/TableGen/AsmMatcherEmitter.cpp
+++ b/utils/TableGen/AsmMatcherEmitter.cpp
@@ -310,11 +310,16 @@ struct MatchableInfo {
     /// The suboperand index within SrcOpName, or -1 for the entire operand.
     int SubOpIdx;
 
+    /// Whether the token is "isolated", i.e., it is preceded and followed
+    /// by separators.
+    bool IsIsolatedToken;
+
     /// Register record if this token is singleton register.
     Record *SingletonReg;
 
-    explicit AsmOperand(StringRef T) : Token(T), Class(nullptr), SubOpIdx(-1),
-                                       SingletonReg(nullptr) {}
+    explicit AsmOperand(bool IsIsolatedToken, StringRef T)
+        : Token(T), Class(nullptr), SubOpIdx(-1),
+          IsIsolatedToken(IsIsolatedToken), SingletonReg(nullptr) {}
   };
 
   /// ResOperand - This represents a single operand in the result instruction
@@ -438,12 +443,21 @@ struct MatchableInfo {
   /// If this instruction is deprecated in some form.
   bool HasDeprecation;
 
+  /// If this is an alias, this is use to determine whether or not to using
+  /// the conversion function defined by the instruction's AsmMatchConverter
+  /// or to use the function generated by the alias.
+  bool UseInstAsmMatchConverter;
+
   MatchableInfo(const CodeGenInstruction &CGI)
-    : AsmVariantID(0), AsmString(CGI.AsmString), TheDef(CGI.TheDef), DefRec(&CGI) {
+    : AsmVariantID(0), AsmString(CGI.AsmString), TheDef(CGI.TheDef), DefRec(&CGI),
+      UseInstAsmMatchConverter(true) {
   }
 
   MatchableInfo(std::unique_ptr<const CodeGenInstAlias> Alias)
-    : AsmVariantID(0), AsmString(Alias->AsmString), TheDef(Alias->TheDef), DefRec(Alias.release()) {
+    : AsmVariantID(0), AsmString(Alias->AsmString), TheDef(Alias->TheDef),
+      DefRec(Alias.release()),
+      UseInstAsmMatchConverter(
+        TheDef->getValueAsBit("UseInstAsmMatchConverter")) {
   }
 
   ~MatchableInfo() {
@@ -563,6 +577,7 @@ struct MatchableInfo {
 
 private:
   void tokenizeAsmString(const AsmMatcherInfo &Info);
+  void addAsmOperand(size_t Start, size_t End);
 };
 
 /// SubtargetFeatureInfo - Helper class for storing information on a subtarget
@@ -802,6 +817,19 @@ void MatchableInfo::initialize(const AsmMatcherInfo &Info,
       DepMask ? !DepMask->getValue()->getAsUnquotedString().empty() : false;
 }
 
+/// Append an AsmOperand for the given substring of AsmString.
+void MatchableInfo::addAsmOperand(size_t Start, size_t End) {
+  StringRef String = AsmString;
+  StringRef Separators = "[]*! \t,";
+  // Look for separators before and after to figure out is this token is
+  // isolated.  Accept '$$' as that's how we escape '$'.
+  bool IsIsolatedToken =
+      (!Start || Separators.find(String[Start - 1]) != StringRef::npos ||
+       String.substr(Start - 1, 2) == "$$") &&
+      (End >= String.size() || Separators.find(String[End]) != StringRef::npos);
+  AsmOperands.push_back(AsmOperand(IsIsolatedToken, String.slice(Start, End)));
+}
+
 /// tokenizeAsmString - Tokenize a simplified assembly string.
 void MatchableInfo::tokenizeAsmString(const AsmMatcherInfo &Info) {
   StringRef String = AsmString;
@@ -817,28 +845,28 @@ void MatchableInfo::tokenizeAsmString(const AsmMatcherInfo &Info) {
     case '\t':
     case ',':
       if (InTok) {
-        AsmOperands.push_back(AsmOperand(String.slice(Prev, i)));
+        addAsmOperand(Prev, i);
         InTok = false;
       }
       if (!isspace(String[i]) && String[i] != ',')
-        AsmOperands.push_back(AsmOperand(String.substr(i, 1)));
+        addAsmOperand(i, i + 1);
       Prev = i + 1;
       break;
 
     case '\\':
       if (InTok) {
-        AsmOperands.push_back(AsmOperand(String.slice(Prev, i)));
+        addAsmOperand(Prev, i);
         InTok = false;
       }
       ++i;
       assert(i != String.size() && "Invalid quoted character");
-      AsmOperands.push_back(AsmOperand(String.substr(i, 1)));
+      addAsmOperand(i, i + 1);
       Prev = i + 1;
       break;
 
     case '$': {
       if (InTok) {
-        AsmOperands.push_back(AsmOperand(String.slice(Prev, i)));
+        addAsmOperand(Prev, i);
         InTok = false;
       }
 
@@ -851,7 +879,7 @@ void MatchableInfo::tokenizeAsmString(const AsmMatcherInfo &Info) {
       StringRef::iterator End = std::find(String.begin() + i, String.end(),'}');
       assert(End != String.end() && "Missing brace in operand reference!");
       size_t EndPos = End - String.begin();
-      AsmOperands.push_back(AsmOperand(String.slice(i, EndPos+1)));
+      addAsmOperand(i, EndPos+1);
       Prev = EndPos + 1;
       i = EndPos;
       break;
@@ -860,7 +888,7 @@ void MatchableInfo::tokenizeAsmString(const AsmMatcherInfo &Info) {
     case '.':
       if (!Info.AsmParser->getValueAsBit("MnemonicContainsDot")) {
         if (InTok)
-          AsmOperands.push_back(AsmOperand(String.slice(Prev, i)));
+          addAsmOperand(Prev, i);
         Prev = i;
       }
       InTok = true;
@@ -871,7 +899,7 @@ void MatchableInfo::tokenizeAsmString(const AsmMatcherInfo &Info) {
     }
   }
   if (InTok && Prev != String.size())
-    AsmOperands.push_back(AsmOperand(String.substr(Prev)));
+    addAsmOperand(Prev, StringRef::npos);
 
   // The first token of the instruction is the mnemonic, which must be a
   // simple string, not a $foo variable or a singleton register.
@@ -953,6 +981,12 @@ extractSingletonRegisterForAsmOperand(unsigned OperandNo,
                                       const AsmMatcherInfo &Info,
                                       std::string &RegisterPrefix) {
   StringRef Tok = AsmOperands[OperandNo].Token;
+
+  // If this token is not an isolated token, i.e., it isn't separated from
+  // other tokens (e.g. with whitespace), don't interpret it as a register name.
+  if (!AsmOperands[OperandNo].IsIsolatedToken)
+    return;
+
   if (RegisterPrefix.empty()) {
     std::string LoweredTok = Tok.lower();
     if (const CodeGenRegister *Reg = Info.Target.getRegisterByName(LoweredTok))
@@ -1215,8 +1249,8 @@ void AsmMatcherInfo::buildOperandClasses() {
     CI->Kind = ClassInfo::UserClass0 + Index;
 
     ListInit *Supers = Rec->getValueAsListInit("SuperClasses");
-    for (unsigned i = 0, e = Supers->getSize(); i != e; ++i) {
-      DefInit *DI = dyn_cast<DefInit>(Supers->getElement(i));
+    for (Init *I : Supers->getValues()) {
+      DefInit *DI = dyn_cast<DefInit>(I);
       if (!DI) {
         PrintError(Rec->getLoc(), "Invalid super class reference!");
         continue;
@@ -1501,7 +1535,7 @@ buildInstructionOperandReference(MatchableInfo *II,
       // Insert remaining suboperands after AsmOpIdx in II->AsmOperands.
       StringRef Token = Op->Token; // save this in case Op gets moved
       for (unsigned SI = 1, SE = Operands[Idx].MINumOperands; SI != SE; ++SI) {
-        MatchableInfo::AsmOperand NewAsmOp(Token);
+        MatchableInfo::AsmOperand NewAsmOp(/*IsIsolatedToken=*/true, Token);
         NewAsmOp.SubOpIdx = SI;
         II->AsmOperands.insert(II->AsmOperands.begin()+AsmOpIdx+SI, NewAsmOp);
       }
@@ -1749,7 +1783,7 @@ static void emitConvertFuncs(CodeGenTarget &Target, StringRef ClassName,
     // Check if we have a custom match function.
     std::string AsmMatchConverter =
       II->getResultInst()->TheDef->getValueAsString("AsmMatchConverter");
-    if (!AsmMatchConverter.empty()) {
+    if (!AsmMatchConverter.empty() && II->UseInstAsmMatchConverter) {
       std::string Signature = "ConvertCustom_" + AsmMatchConverter;
       II->ConversionFnKind = Signature;
 
@@ -1763,7 +1797,7 @@ static void emitConvertFuncs(CodeGenTarget &Target, StringRef ClassName,
                                     getEnumNameForToken(AsmMatchConverter));
 
       // Add the converter row for this instruction.
-      ConversionTable.push_back(std::vector<uint8_t>());
+      ConversionTable.emplace_back();
       ConversionTable.back().push_back(KindID);
       ConversionTable.back().push_back(CVT_Done);
 
@@ -1870,7 +1904,7 @@ static void emitConvertFuncs(CodeGenTarget &Target, StringRef ClassName,
           break;
 
         CvtOS << "    case " << Name << ":\n"
-              << "      Inst.addOperand(MCOperand::CreateImm(" << Val << "));\n"
+              << "      Inst.addOperand(MCOperand::createImm(" << Val << "));\n"
               << "      break;\n";
 
         OpOS << "    case " << Name << ":\n"
@@ -1901,7 +1935,7 @@ static void emitConvertFuncs(CodeGenTarget &Target, StringRef ClassName,
         if (!IsNewConverter)
           break;
         CvtOS << "    case " << Name << ":\n"
-              << "      Inst.addOperand(MCOperand::CreateReg(" << Reg << "));\n"
+              << "      Inst.addOperand(MCOperand::createReg(" << Reg << "));\n"
               << "      break;\n";
 
         OpOS << "    case " << Name << ":\n"
@@ -2127,8 +2161,7 @@ static void emitMatchTokenString(CodeGenTarget &Target,
   std::vector<StringMatcher::StringPair> Matches;
   for (const auto &CI : Infos) {
     if (CI.Kind == ClassInfo::Token)
-      Matches.push_back(
-          StringMatcher::StringPair(CI.ValueName, "return " + CI.Name + ";"));
+      Matches.emplace_back(CI.ValueName, "return " + CI.Name + ";");
   }
 
   OS << "static MatchClassKind matchTokenString(StringRef Name) {\n";
@@ -2150,9 +2183,8 @@ static void emitMatchRegisterName(CodeGenTarget &Target, Record *AsmParser,
     if (Reg.TheDef->getValueAsString("AsmName").empty())
       continue;
 
-    Matches.push_back(
-        StringMatcher::StringPair(Reg.TheDef->getValueAsString("AsmName"),
-                                  "return " + utostr(Reg.EnumValue) + ";"));
+    Matches.emplace_back(Reg.TheDef->getValueAsString("AsmName"),
+                         "return " + utostr(Reg.EnumValue) + ";");
   }
 
   OS << "static unsigned MatchRegisterName(StringRef Name) {\n";
@@ -2248,7 +2280,7 @@ static void emitComputeAvailableFeatures(AsmMatcherInfo &Info,
     Info.AsmParser->getValueAsString("AsmParserClassName");
 
   OS << "uint64_t " << Info.Target.getName() << ClassName << "::\n"
-     << "ComputeAvailableFeatures(uint64_t FB) const {\n";
+     << "ComputeAvailableFeatures(const FeatureBitset& FB) const {\n";
   OS << "  uint64_t Features = 0;\n";
   for (const auto &SF : Info.SubtargetFeatures) {
     const SubtargetFeatureInfo &SFI = SF.second;
@@ -2270,12 +2302,10 @@ static void emitComputeAvailableFeatures(AsmMatcherInfo &Info,
         Cond = Cond.substr(1);
       }
 
-      OS << "((FB & " << Info.Target.getName() << "::" << Cond << ")";
+      OS << "(";
       if (Neg)
-        OS << " == 0";
-      else
-        OS << " != 0";
-      OS << ")";
+        OS << "!";
+      OS << "FB[" << Info.Target.getName() << "::" << Cond << "])";
 
       if (Comma.second.empty())
         break;
@@ -2645,7 +2675,7 @@ void AsmMatcherEmitter::run(raw_ostream &OS) {
   OS << "#undef GET_ASSEMBLER_HEADER\n";
   OS << "  // This should be included into the middle of the declaration of\n";
   OS << "  // your subclasses implementation of MCTargetAsmParser.\n";
-  OS << "  uint64_t ComputeAvailableFeatures(uint64_t FeatureBits) const;\n";
+  OS << "  uint64_t ComputeAvailableFeatures(const FeatureBitset& FB) const;\n";
   OS << "  void convertToMCInst(unsigned Kind, MCInst &Inst, "
      << "unsigned Opcode,\n"
      << "                       const OperandVector "
diff --git a/utils/TableGen/AsmWriterEmitter.cpp b/utils/TableGen/AsmWriterEmitter.cpp
index 389889ab80f..8163f681d88 100644
--- a/utils/TableGen/AsmWriterEmitter.cpp
+++ b/utils/TableGen/AsmWriterEmitter.cpp
@@ -1105,9 +1105,8 @@ AsmWriterEmitter::AsmWriterEmitter(RecordKeeper &R) : Records(R), Target(R) {
   Record *AsmWriter = Target.getAsmWriter();
   for (const CodeGenInstruction *I : Target.instructions())
     if (!I->AsmString.empty() && I->TheDef->getName() != "PHI")
-      Instructions.push_back(
-          AsmWriterInst(*I, AsmWriter->getValueAsInt("Variant"),
-                        AsmWriter->getValueAsInt("PassSubtarget")));
+      Instructions.emplace_back(*I, AsmWriter->getValueAsInt("Variant"),
+                                AsmWriter->getValueAsInt("PassSubtarget"));
 
   // Get the instruction numbering.
   NumberedInstructions = &Target.getInstructionsByEnumValue();
diff --git a/utils/TableGen/AsmWriterInst.cpp b/utils/TableGen/AsmWriterInst.cpp
index a66b1a01cae..95418875400 100644
--- a/utils/TableGen/AsmWriterInst.cpp
+++ b/utils/TableGen/AsmWriterInst.cpp
@@ -163,27 +163,22 @@ AsmWriterInst::AsmWriterInst(const CodeGenInstruction &CGI, unsigned Variant,
 
       if (VarName.empty()) {
         // Just a modifier, pass this into PrintSpecial.
-        Operands.push_back(AsmWriterOperand("PrintSpecial",
-                                            ~0U,
-                                            ~0U,
-                                            Modifier,
-                                            PassSubtarget));
+        Operands.emplace_back("PrintSpecial", ~0U, ~0U, Modifier,
+                              PassSubtarget);
       } else {
         // Otherwise, normal operand.
         unsigned OpNo = CGI.Operands.getOperandNamed(VarName);
         CGIOperandList::OperandInfo OpInfo = CGI.Operands[OpNo];
 
         unsigned MIOp = OpInfo.MIOperandNo;
-        Operands.push_back(AsmWriterOperand(OpInfo.PrinterMethodName,
-                                            OpNo, MIOp, Modifier,
-                                            PassSubtarget));
+        Operands.emplace_back(OpInfo.PrinterMethodName, OpNo, MIOp, Modifier,
+                              PassSubtarget);
       }
       LastEmitted = VarEnd;
     }
   }
 
-  Operands.push_back(AsmWriterOperand("return;",
-    AsmWriterOperand::isLiteralStatementOperand));
+  Operands.emplace_back("return;", AsmWriterOperand::isLiteralStatementOperand);
 }
 
 /// MatchesAllButOneOp - If this instruction is exactly identical to the
diff --git a/utils/TableGen/CTagsEmitter.cpp b/utils/TableGen/CTagsEmitter.cpp
index bbed92a1385..35f4ad6dd5b 100644
--- a/utils/TableGen/CTagsEmitter.cpp
+++ b/utils/TableGen/CTagsEmitter.cpp
@@ -24,8 +24,6 @@ using namespace llvm;
 
 #define DEBUG_TYPE "ctags-emitter"
 
-namespace llvm { extern SourceMgr SrcMgr; }
-
 namespace {
 
 class Tag {
@@ -61,11 +59,7 @@ class CTagsEmitter {
 
 SMLoc CTagsEmitter::locate(const Record *R) {
   ArrayRef<SMLoc> Locs = R->getLoc();
-  if (Locs.empty()) {
-    SMLoc NullLoc;
-    return NullLoc;
-  }
-  return Locs.front();
+  return !Locs.empty() ? Locs.front() : SMLoc();
 }
 
 void CTagsEmitter::run(raw_ostream &OS) {
@@ -82,9 +76,8 @@ void CTagsEmitter::run(raw_ostream &OS) {
   std::sort(Tags.begin(), Tags.end());
   OS << "!_TAG_FILE_FORMAT\t1\t/original ctags format/\n";
   OS << "!_TAG_FILE_SORTED\t1\t/0=unsorted, 1=sorted, 2=foldcase/\n";
-  for (std::vector<Tag>::const_iterator I = Tags.begin(), E = Tags.end();
-       I != E; ++I)
-    I->emit(OS);
+  for (const Tag &T : Tags)
+    T.emit(OS);
 }
 
 namespace llvm {
diff --git a/utils/TableGen/CallingConvEmitter.cpp b/utils/TableGen/CallingConvEmitter.cpp
index 051a7e98141..a47662b2855 100644
--- a/utils/TableGen/CallingConvEmitter.cpp
+++ b/utils/TableGen/CallingConvEmitter.cpp
@@ -69,7 +69,7 @@ void CallingConvEmitter::EmitCallingConv(Record *CC, raw_ostream &O) {
     << std::string(CC->getName().size()+13, ' ')
     << "ISD::ArgFlagsTy ArgFlags, CCState &State) {\n";
   // Emit all of the actions, in order.
-  for (unsigned i = 0, e = CCActions->getSize(); i != e; ++i) {
+  for (unsigned i = 0, e = CCActions->size(); i != e; ++i) {
     O << "\n";
     EmitAction(CCActions->getElementAsRecord(i), 2, O);
   }
@@ -87,7 +87,7 @@ void CallingConvEmitter::EmitAction(Record *Action,
     
     if (Action->isSubClassOf("CCIfType")) {
       ListInit *VTs = Action->getValueAsListInit("VTs");
-      for (unsigned i = 0, e = VTs->getSize(); i != e; ++i) {
+      for (unsigned i = 0, e = VTs->size(); i != e; ++i) {
         Record *VT = VTs->getElementAsRecord(i);
         if (i != 0) O << " ||\n    " << IndentStr;
         O << "LocVT == " << getEnumName(getValueType(VT));
@@ -111,14 +111,14 @@ void CallingConvEmitter::EmitAction(Record *Action,
         << IndentStr << "  return false;\n";
     } else if (Action->isSubClassOf("CCAssignToReg")) {
       ListInit *RegList = Action->getValueAsListInit("RegList");
-      if (RegList->getSize() == 1) {
+      if (RegList->size() == 1) {
         O << IndentStr << "if (unsigned Reg = State.AllocateReg(";
         O << getQualifiedName(RegList->getElementAsRecord(0)) << ")) {\n";
       } else {
         O << IndentStr << "static const MCPhysReg RegList" << ++Counter
           << "[] = {\n";
         O << IndentStr << "  ";
-        for (unsigned i = 0, e = RegList->getSize(); i != e; ++i) {
+        for (unsigned i = 0, e = RegList->size(); i != e; ++i) {
           if (i != 0) O << ", ";
           O << getQualifiedName(RegList->getElementAsRecord(i));
         }
@@ -133,11 +133,10 @@ void CallingConvEmitter::EmitAction(Record *Action,
     } else if (Action->isSubClassOf("CCAssignToRegWithShadow")) {
       ListInit *RegList = Action->getValueAsListInit("RegList");
       ListInit *ShadowRegList = Action->getValueAsListInit("ShadowRegList");
-      if (ShadowRegList->getSize() >0 &&
-          ShadowRegList->getSize() != RegList->getSize())
+      if (!ShadowRegList->empty() && ShadowRegList->size() != RegList->size())
         PrintFatalError("Invalid length of list of shadowed registers");
 
-      if (RegList->getSize() == 1) {
+      if (RegList->size() == 1) {
         O << IndentStr << "if (unsigned Reg = State.AllocateReg(";
         O << getQualifiedName(RegList->getElementAsRecord(0));
         O << ", " << getQualifiedName(ShadowRegList->getElementAsRecord(0));
@@ -149,7 +148,7 @@ void CallingConvEmitter::EmitAction(Record *Action,
         O << IndentStr << "static const MCPhysReg RegList" << RegListNumber
           << "[] = {\n";
         O << IndentStr << "  ";
-        for (unsigned i = 0, e = RegList->getSize(); i != e; ++i) {
+        for (unsigned i = 0, e = RegList->size(); i != e; ++i) {
           if (i != 0) O << ", ";
           O << getQualifiedName(RegList->getElementAsRecord(i));
         }
@@ -158,7 +157,7 @@ void CallingConvEmitter::EmitAction(Record *Action,
         O << IndentStr << "static const MCPhysReg RegList"
           << ShadowRegListNumber << "[] = {\n";
         O << IndentStr << "  ";
-        for (unsigned i = 0, e = ShadowRegList->getSize(); i != e; ++i) {
+        for (unsigned i = 0, e = ShadowRegList->size(); i != e; ++i) {
           if (i != 0) O << ", ";
           O << getQualifiedName(ShadowRegList->getElementAsRecord(i));
         }
@@ -182,15 +181,15 @@ void CallingConvEmitter::EmitAction(Record *Action,
         O << Size << ", ";
       else
         O << "\n" << IndentStr
-          << "  State.getMachineFunction().getTarget().getDataLayout()"
-             "->getTypeAllocSize(EVT(LocVT).getTypeForEVT(State.getContext())),"
+          << "  State.getMachineFunction().getDataLayout()."
+             "getTypeAllocSize(EVT(LocVT).getTypeForEVT(State.getContext())),"
              " ";
       if (Align)
         O << Align;
       else
         O << "\n" << IndentStr
-          << "  State.getMachineFunction().getTarget().getDataLayout()"
-             "->getABITypeAlignment(EVT(LocVT).getTypeForEVT(State.getContext()"
+          << "  State.getMachineFunction().getDataLayout()."
+             "getABITypeAlignment(EVT(LocVT).getTypeForEVT(State.getContext()"
              "))";
       O << ");\n" << IndentStr
         << "State.addLoc(CCValAssign::getMem(ValNo, ValVT, Offset"
@@ -206,7 +205,7 @@ void CallingConvEmitter::EmitAction(Record *Action,
       O << IndentStr << "static const MCPhysReg ShadowRegList"
           << ShadowRegListNumber << "[] = {\n";
       O << IndentStr << "  ";
-      for (unsigned i = 0, e = ShadowRegList->getSize(); i != e; ++i) {
+      for (unsigned i = 0, e = ShadowRegList->size(); i != e; ++i) {
         if (i != 0) O << ", ";
         O << getQualifiedName(ShadowRegList->getElementAsRecord(i));
       }
diff --git a/utils/TableGen/CodeGenDAGPatterns.cpp b/utils/TableGen/CodeGenDAGPatterns.cpp
index 56743f3714a..ae1cc0cfb5f 100644
--- a/utils/TableGen/CodeGenDAGPatterns.cpp
+++ b/utils/TableGen/CodeGenDAGPatterns.cpp
@@ -842,8 +842,8 @@ getPatternComplexity(const CodeGenDAGPatterns &CGP) const {
 ///
 std::string PatternToMatch::getPredicateCheck() const {
   std::string PredicateCheck;
-  for (unsigned i = 0, e = Predicates->getSize(); i != e; ++i) {
-    if (DefInit *Pred = dyn_cast<DefInit>(Predicates->getElement(i))) {
+  for (Init *I : Predicates->getValues()) {
+    if (DefInit *Pred = dyn_cast<DefInit>(I)) {
       Record *Def = Pred->getDef();
       if (!Def->isSubClassOf("Predicate")) {
 #ifndef NDEBUG
@@ -913,8 +913,7 @@ SDTypeConstraint::SDTypeConstraint(Record *R) {
     x.SDTCisSameNumEltsAs_Info.OtherOperandNum =
       R->getValueAsInt("OtherOperandNum");
   } else {
-    errs() << "Unrecognized SDTypeConstraint '" << R->getName() << "'!\n";
-    exit(1);
+    PrintFatalError("Unrecognized SDTypeConstraint '" + R->getName() + "'!\n");
   }
 }
 
@@ -932,11 +931,12 @@ static TreePatternNode *getOperandNum(unsigned OpNo, TreePatternNode *N,
   OpNo -= NumResults;
 
   if (OpNo >= N->getNumChildren()) {
-    errs() << "Invalid operand number in type constraint "
+    std::string S;
+    raw_string_ostream OS(S);
+    OS << "Invalid operand number in type constraint "
            << (OpNo+NumResults) << " ";
-    N->dump();
-    errs() << '\n';
-    exit(1);
+    N->print(OS);
+    PrintFatalError(OS.str());
   }
 
   return N->getChild(OpNo);
@@ -1116,9 +1116,9 @@ SDNodeInfo::SDNodeInfo(Record *R) : Def(R) {
     } else if (PropList[i]->getName() == "SDNPVariadic") {
       Properties |= 1 << SDNPVariadic;
     } else {
-      errs() << "Unknown SD Node property '" << PropList[i]->getName()
-             << "' on node '" << R->getName() << "'!\n";
-      exit(1);
+      PrintFatalError("Unknown SD Node property '" +
+                      PropList[i]->getName() + "' on node '" +
+                      R->getName() + "'!");
     }
   }
 
@@ -1223,8 +1223,7 @@ static unsigned GetNumNodeResults(Record *Operator, CodeGenDAGPatterns &CDP) {
     return 1;
 
   Operator->dump();
-  errs() << "Unhandled node in GetNumNodeResults\n";
-  exit(1);
+  PrintFatalError("Unhandled node in GetNumNodeResults");
 }
 
 void TreePatternNode::print(raw_ostream &OS) const {
@@ -2000,8 +1999,8 @@ bool TreePatternNode::canPatternMatch(std::string &Reason,
 TreePattern::TreePattern(Record *TheRec, ListInit *RawPat, bool isInput,
                          CodeGenDAGPatterns &cdp) : TheRecord(TheRec), CDP(cdp),
                          isInputPattern(isInput), HasError(false) {
-  for (unsigned i = 0, e = RawPat->getSize(); i != e; ++i)
-    Trees.push_back(ParseTreePattern(RawPat->getElement(i), ""));
+  for (Init *I : RawPat->getValues())
+    Trees.push_back(ParseTreePattern(I, ""));
 }
 
 TreePattern::TreePattern(Record *TheRec, DagInit *Pat, bool isInput,
@@ -2064,7 +2063,7 @@ TreePatternNode *TreePattern::ParseTreePattern(Init *TheInit, StringRef OpName){
   }
 
   // ?:$name or just $name.
-  if (TheInit == UnsetInit::get()) {
+  if (isa<UnsetInit>(TheInit)) {
     if (OpName.empty())
       error("'?' argument requires a name to match with operand list");
     TreePatternNode *Res = new TreePatternNode(TheInit, 1);
@@ -2144,7 +2143,8 @@ TreePatternNode *TreePattern::ParseTreePattern(Init *TheInit, StringRef OpName){
         Operator->getName() != "tblockaddress" &&
         Operator->getName() != "tglobaladdr" &&
         Operator->getName() != "bb" &&
-        Operator->getName() != "vt")
+        Operator->getName() != "vt" &&
+        Operator->getName() != "mcsym")
       error("Cannot use '" + Operator->getName() + "' in an output pattern!");
   }
 
@@ -2373,10 +2373,9 @@ CodeGenDAGPatterns::CodeGenDAGPatterns(RecordKeeper &R) :
 
 Record *CodeGenDAGPatterns::getSDNodeNamed(const std::string &Name) const {
   Record *N = Records.getDef(Name);
-  if (!N || !N->isSubClassOf("SDNode")) {
-    errs() << "Error getting SDNode '" << Name << "'!\n";
-    exit(1);
-  }
+  if (!N || !N->isSubClassOf("SDNode"))
+    PrintFatalError("Error getting SDNode '" + Name + "'!");
+
   return N;
 }
 
@@ -2862,8 +2861,8 @@ static bool hasNullFragReference(DagInit *DI) {
 /// hasNullFragReference - Return true if any DAG in the list references
 /// the null_frag operator.
 static bool hasNullFragReference(ListInit *LI) {
-  for (unsigned i = 0, e = LI->getSize(); i != e; ++i) {
-    DagInit *DI = dyn_cast<DagInit>(LI->getElement(i));
+  for (Init *I : LI->getValues()) {
+    DagInit *DI = dyn_cast<DagInit>(I);
     assert(DI && "non-dag in an instruction Pattern list?!");
     if (hasNullFragReference(DI))
       return true;
@@ -3078,7 +3077,7 @@ void CodeGenDAGPatterns::ParseInstructions() {
     // null_frag operator is as-if no pattern were specified. Normally this
     // is from a multiclass expansion w/ a SDPatternOperator passed in as
     // null_frag.
-    if (!LI || LI->getSize() == 0 || hasNullFragReference(LI)) {
+    if (!LI || LI->empty() || hasNullFragReference(LI)) {
       std::vector<Record*> Results;
       std::vector<Record*> Operands;
 
@@ -3401,7 +3400,7 @@ void CodeGenDAGPatterns::ParsePatterns() {
     Pattern->InlinePatternFragments();
 
     ListInit *LI = CurPattern->getValueAsListInit("ResultInstrs");
-    if (LI->getSize() == 0) continue;  // no pattern.
+    if (LI->empty()) continue;  // no pattern.
 
     // Parse the instruction.
     TreePattern Result(CurPattern, LI, false, *this);
@@ -3800,13 +3799,11 @@ void CodeGenDAGPatterns::GenerateVariants() {
       if (AlreadyExists) continue;
 
       // Otherwise, add it to the list of patterns we have.
-      PatternsToMatch.
-        push_back(PatternToMatch(PatternsToMatch[i].getSrcRecord(),
-                                 PatternsToMatch[i].getPredicates(),
-                                 Variant, PatternsToMatch[i].getDstPattern(),
-                                 PatternsToMatch[i].getDstRegs(),
-                                 PatternsToMatch[i].getAddedComplexity(),
-                                 Record::getNewUID()));
+      PatternsToMatch.emplace_back(
+          PatternsToMatch[i].getSrcRecord(), PatternsToMatch[i].getPredicates(),
+          Variant, PatternsToMatch[i].getDstPattern(),
+          PatternsToMatch[i].getDstRegs(),
+          PatternsToMatch[i].getAddedComplexity(), Record::getNewUID());
     }
 
     DEBUG(errs() << "\n");
diff --git a/utils/TableGen/CodeGenInstruction.cpp b/utils/TableGen/CodeGenInstruction.cpp
index 10602964e48..e83d5033b18 100644
--- a/utils/TableGen/CodeGenInstruction.cpp
+++ b/utils/TableGen/CodeGenInstruction.cpp
@@ -115,9 +115,9 @@ CGIOperandList::CGIOperandList(Record *R) : TheDef(R) {
       PrintFatalError("In instruction '" + R->getName() + "', operand #" +
                       Twine(i) + " has the same name as a previous operand!");
 
-    OperandList.push_back(OperandInfo(Rec, ArgName, PrintMethod, EncoderMethod,
-                                      OperandNamespace + "::" + OperandType,
-				      MIOperandNo, NumOps, MIOpInfo));
+    OperandList.emplace_back(Rec, ArgName, PrintMethod, EncoderMethod,
+                             OperandNamespace + "::" + OperandType, MIOperandNo,
+                             NumOps, MIOpInfo);
     MIOperandNo += NumOps;
   }
 
@@ -320,6 +320,7 @@ CodeGenInstruction::CodeGenInstruction(Record *R)
   isRegSequence = R->getValueAsBit("isRegSequence");
   isExtractSubreg = R->getValueAsBit("isExtractSubreg");
   isInsertSubreg = R->getValueAsBit("isInsertSubreg");
+  isConvergent = R->getValueAsBit("isConvergent");
 
   bool Unset;
   mayLoad      = R->getValueAsBitOrUnset("mayLoad", Unset);
@@ -641,9 +642,9 @@ CodeGenInstAlias::CodeGenInstAlias(Record *R, unsigned Variant,
 
           // Take care to instantiate each of the suboperands with the correct
           // nomenclature: $foo.bar
-          ResultOperands.push_back(
-              ResultOperand(Result->getArgName(AliasOpNo) + "." +
-                            MIOI->getArgName(SubOp), SubRec));
+          ResultOperands.emplace_back(Result->getArgName(AliasOpNo) + "." +
+                                          MIOI->getArgName(SubOp),
+                                      SubRec);
           ResultInstOperandIndex.push_back(std::make_pair(i, SubOp));
          }
          ++AliasOpNo;
diff --git a/utils/TableGen/CodeGenInstruction.h b/utils/TableGen/CodeGenInstruction.h
index bdbe546ec97..8f01abd5403 100644
--- a/utils/TableGen/CodeGenInstruction.h
+++ b/utils/TableGen/CodeGenInstruction.h
@@ -14,9 +14,10 @@
 #ifndef LLVM_UTILS_TABLEGEN_CODEGENINSTRUCTION_H
 #define LLVM_UTILS_TABLEGEN_CODEGENINSTRUCTION_H
 
+#include "llvm/ADT/ArrayRef.h"
 #include "llvm/ADT/StringRef.h"
 #include "llvm/CodeGen/MachineValueType.h"
-#include "llvm/Support/SourceMgr.h"
+#include "llvm/Support/SMLoc.h"
 #include <string>
 #include <utility>
 #include <vector>
@@ -255,6 +256,7 @@ namespace llvm {
     bool isRegSequence : 1;
     bool isExtractSubreg : 1;
     bool isInsertSubreg : 1;
+    bool isConvergent : 1;
 
     std::string DeprecatedReason;
     bool HasComplexDeprecationPredicate;
diff --git a/utils/TableGen/CodeGenIntrinsics.h b/utils/TableGen/CodeGenIntrinsics.h
index 1f1adf11fb3..f4055571b1c 100644
--- a/utils/TableGen/CodeGenIntrinsics.h
+++ b/utils/TableGen/CodeGenIntrinsics.h
@@ -80,6 +80,9 @@ namespace llvm {
     /// isNoReturn - True if the intrinsic is no-return.
     bool isNoReturn;
 
+    /// isConvergent - True if the intrinsic is marked as convergent.
+    bool isConvergent;
+
     enum ArgAttribute {
       NoCapture,
       ReadOnly,
diff --git a/utils/TableGen/CodeGenMapTable.cpp b/utils/TableGen/CodeGenMapTable.cpp
index 8abea4808ba..48df4391fe1 100644
--- a/utils/TableGen/CodeGenMapTable.cpp
+++ b/utils/TableGen/CodeGenMapTable.cpp
@@ -128,16 +128,16 @@ class InstrMap {
     ListInit *ColValList = MapRec->getValueAsListInit("ValueCols");
 
     // Each instruction map must specify at least one column for it to be valid.
-    if (ColValList->getSize() == 0)
+    if (ColValList->empty())
       PrintFatalError(MapRec->getLoc(), "InstrMapping record `" +
         MapRec->getName() + "' has empty " + "`ValueCols' field!");
 
-    for (unsigned i = 0, e = ColValList->getSize(); i < e; i++) {
-      ListInit *ColI = dyn_cast<ListInit>(ColValList->getElement(i));
+    for (Init *I : ColValList->getValues()) {
+      ListInit *ColI = dyn_cast<ListInit>(I);
 
       // Make sure that all the sub-lists in 'ValueCols' have same number of
       // elements as the fields in 'ColFields'.
-      if (ColI->getSize() != ColFields->getSize())
+      if (ColI->size() != ColFields->size())
         PrintFatalError(MapRec->getLoc(), "Record `" + MapRec->getName() +
           "', field `ValueCols' entries don't match with " +
           " the entries in 'ColFields'!");
@@ -239,13 +239,11 @@ class MapTableEmitter {
 //===----------------------------------------------------------------------===//
 
 void MapTableEmitter::buildRowInstrMap() {
-  for (unsigned i = 0, e = InstrDefs.size(); i < e; i++) {
-    Record *CurInstr = InstrDefs[i];
+  for (Record *CurInstr : InstrDefs) {
     std::vector<Init*> KeyValue;
     ListInit *RowFields = InstrMapDesc.getRowFields();
-    for (unsigned j = 0, endRF = RowFields->getSize(); j < endRF; j++) {
-      Init *RowFieldsJ = RowFields->getElement(j);
-      Init *CurInstrVal = CurInstr->getValue(RowFieldsJ)->getValue();
+    for (Init *RowField : RowFields->getValues()) {
+      Init *CurInstrVal = CurInstr->getValue(RowField)->getValue();
       KeyValue.push_back(CurInstrVal);
     }
 
@@ -269,7 +267,7 @@ bool MapTableEmitter::isKeyColInstr(Record* CurInstr) {
 
   // Check if the instruction is a KeyCol instruction.
   bool MatchFound = true;
-  for (unsigned j = 0, endCF = ColFields->getSize();
+  for (unsigned j = 0, endCF = ColFields->size();
       (j < endCF) && MatchFound; j++) {
     RecordVal *ColFieldName = CurInstr->getValue(ColFields->getElement(j));
     std::string CurInstrVal = ColFieldName->getValue()->getAsUnquotedString();
@@ -289,8 +287,7 @@ void MapTableEmitter::buildMapTable() {
   // constraints.
   const std::vector<ListInit*> &ValueCols = InstrMapDesc.getValueCols();
   unsigned NumOfCols = ValueCols.size();
-  for (unsigned j = 0, endKI = KeyInstrVec.size(); j < endKI; j++) {
-    Record *CurKeyInstr = KeyInstrVec[j];
+  for (Record *CurKeyInstr : KeyInstrVec) {
     std::vector<Record*> ColInstrVec(NumOfCols);
 
     // Find the column instruction based on the constraints for the column.
@@ -313,9 +310,8 @@ Record *MapTableEmitter::getInstrForColumn(Record *KeyInstr,
   std::vector<Init*> KeyValue;
 
   // Construct KeyValue using KeyInstr's values for RowFields.
-  for (unsigned j = 0, endRF = RowFields->getSize(); j < endRF; j++) {
-    Init *RowFieldsJ = RowFields->getElement(j);
-    Init *KeyInstrVal = KeyInstr->getValue(RowFieldsJ)->getValue();
+  for (Init *RowField : RowFields->getValues()) {
+    Init *KeyInstrVal = KeyInstr->getValue(RowField)->getValue();
     KeyValue.push_back(KeyInstrVal);
   }
 
@@ -331,7 +327,7 @@ Record *MapTableEmitter::getInstrForColumn(Record *KeyInstr,
   for (unsigned i = 0, e = RelatedInstrVec.size(); i < e; i++) {
     bool MatchFound = true;
     Record *CurInstr = RelatedInstrVec[i];
-    for (unsigned j = 0, endCF = ColFields->getSize();
+    for (unsigned j = 0, endCF = ColFields->size();
         (j < endCF) && MatchFound; j++) {
       Init *ColFieldJ = ColFields->getElement(j);
       Init *CurInstrInit = CurInstr->getValue(ColFieldJ)->getValue();
@@ -443,12 +439,12 @@ void MapTableEmitter::emitMapFuncBody(raw_ostream &OS,
   if (ValueCols.size() > 1) {
     for (unsigned i = 0, e = ValueCols.size(); i < e; i++) {
       ListInit *ColumnI = ValueCols[i];
-      for (unsigned j = 0, ColSize = ColumnI->getSize(); j < ColSize; j++) {
+      for (unsigned j = 0, ColSize = ColumnI->size(); j < ColSize; ++j) {
         std::string ColName = ColFields->getElement(j)->getAsUnquotedString();
         OS << "  if (in" << ColName;
         OS << " == ";
         OS << ColName << "_" << ColumnI->getElement(j)->getAsUnquotedString();
-        if (j < ColumnI->getSize() - 1) OS << " && ";
+        if (j < ColumnI->size() - 1) OS << " && ";
         else OS << ")\n";
       }
       OS << "    return " << InstrMapDesc.getName();
@@ -478,8 +474,8 @@ void MapTableEmitter::emitTablesWithFunc(raw_ostream &OS) {
   OS << "// "<< InstrMapDesc.getName() << "\n";
   OS << "int "<< InstrMapDesc.getName() << "(uint16_t Opcode";
   if (ValueCols.size() > 1) {
-    for (unsigned i = 0, e = ColFields->getSize(); i < e; i++) {
-      std::string ColName = ColFields->getElement(i)->getAsUnquotedString();
+    for (Init *CF : ColFields->getValues()) {
+      std::string ColName = CF->getAsUnquotedString();
       OS << ", enum " << ColName << " in" << ColName << ") {\n";
     }
   } else { OS << ") {\n"; }
@@ -509,18 +505,18 @@ static void emitEnums(raw_ostream &OS, RecordKeeper &Records) {
     ColFields = CurMap->getValueAsListInit("ColFields");
     ListInit *List = CurMap->getValueAsListInit("ValueCols");
     std::vector<ListInit*> ValueCols;
-    unsigned ListSize = List->getSize();
+    unsigned ListSize = List->size();
 
     for (unsigned j = 0; j < ListSize; j++) {
       ListInit *ListJ = dyn_cast<ListInit>(List->getElement(j));
 
-      if (ListJ->getSize() != ColFields->getSize())
+      if (ListJ->size() != ColFields->size())
         PrintFatalError("Record `" + CurMap->getName() + "', field "
           "`ValueCols' entries don't match with the entries in 'ColFields' !");
       ValueCols.push_back(ListJ);
     }
 
-    for (unsigned j = 0, endCF = ColFields->getSize(); j < endCF; j++) {
+    for (unsigned j = 0, endCF = ColFields->size(); j < endCF; j++) {
       for (unsigned k = 0; k < ListSize; k++){
         std::string ColName = ColFields->getElement(j)->getAsUnquotedString();
         ColFieldValueMap[ColName].push_back((ValueCols[k])->getElement(j));
diff --git a/utils/TableGen/CodeGenRegisters.cpp b/utils/TableGen/CodeGenRegisters.cpp
index 4dd76813575..c9e6d1d379d 100644
--- a/utils/TableGen/CodeGenRegisters.cpp
+++ b/utils/TableGen/CodeGenRegisters.cpp
@@ -543,7 +543,7 @@ struct TupleExpander : SetTheory::Expander {
     std::vector<Record*> Indices = Def->getValueAsListOfDefs("SubRegIndices");
     unsigned Dim = Indices.size();
     ListInit *SubRegs = Def->getValueAsListInit("SubRegs");
-    if (Dim != SubRegs->getSize())
+    if (Dim != SubRegs->size())
       PrintFatalError(Def->getLoc(), "SubRegIndices and SubRegs size mismatch");
     if (Dim < 2)
       PrintFatalError(Def->getLoc(),
@@ -924,7 +924,7 @@ CodeGenRegBank::CodeGenRegBank(RecordKeeper &Records) {
   // Configure register Sets to understand register classes and tuples.
   Sets.addFieldExpander("RegisterClass", "MemberList");
   Sets.addFieldExpander("CalleeSavedRegs", "SaveList");
-  Sets.addExpander("RegisterTuples", new TupleExpander());
+  Sets.addExpander("RegisterTuples", llvm::make_unique<TupleExpander>());
 
   // Read in the user-defined (named) sub-register indices.
   // More indices will be synthesized later.
@@ -994,7 +994,7 @@ CodeGenRegBank::CodeGenRegBank(RecordKeeper &Records) {
 
   // Allocate user-defined register classes.
   for (auto *RC : RCs) {
-    RegClasses.push_back(CodeGenRegisterClass(*this, RC));
+    RegClasses.emplace_back(*this, RC);
     addToMaps(&RegClasses.back());
   }
 
@@ -1056,7 +1056,7 @@ CodeGenRegBank::getOrCreateSubClass(const CodeGenRegisterClass *RC,
     return FoundI->second;
 
   // Sub-class doesn't exist, create a new one.
-  RegClasses.push_back(CodeGenRegisterClass(*this, Name, K));
+  RegClasses.emplace_back(*this, Name, K);
   addToMaps(&RegClasses.back());
   return &RegClasses.back();
 }
diff --git a/utils/TableGen/CodeGenSchedule.cpp b/utils/TableGen/CodeGenSchedule.cpp
index bfdf8dcc89e..bc27481869f 100644
--- a/utils/TableGen/CodeGenSchedule.cpp
+++ b/utils/TableGen/CodeGenSchedule.cpp
@@ -93,8 +93,8 @@ CodeGenSchedModels::CodeGenSchedModels(RecordKeeper &RK,
 
   // Allow Set evaluation to recognize the dags used in InstRW records:
   // (instrs Op1, Op1...)
-  Sets.addOperator("instrs", new InstrsOp);
-  Sets.addOperator("instregex", new InstRegexOp(Target));
+  Sets.addOperator("instrs", llvm::make_unique<InstrsOp>());
+  Sets.addOperator("instregex", llvm::make_unique<InstRegexOp>(Target));
 
   // Instantiate a CodeGenProcModel for each SchedMachineModel with the values
   // that are explicitly referenced in tablegen records. Resources associated
@@ -145,8 +145,7 @@ void CodeGenSchedModels::collectProcModels() {
   // Use idx=0 for NoModel/NoItineraries.
   Record *NoModelDef = Records.getDef("NoSchedModel");
   Record *NoItinsDef = Records.getDef("NoItineraries");
-  ProcModels.push_back(CodeGenProcModel(0, "NoSchedModel",
-                                        NoModelDef, NoItinsDef));
+  ProcModels.emplace_back(0, "NoSchedModel", NoModelDef, NoItinsDef);
   ProcModelMap[NoModelDef] = 0;
 
   // For each processor, find a unique machine model.
@@ -164,16 +163,14 @@ void CodeGenSchedModels::addProcModel(Record *ProcDef) {
   std::string Name = ModelKey->getName();
   if (ModelKey->isSubClassOf("SchedMachineModel")) {
     Record *ItinsDef = ModelKey->getValueAsDef("Itineraries");
-    ProcModels.push_back(
-      CodeGenProcModel(ProcModels.size(), Name, ModelKey, ItinsDef));
+    ProcModels.emplace_back(ProcModels.size(), Name, ModelKey, ItinsDef);
   }
   else {
     // An itinerary is defined without a machine model. Infer a new model.
     if (!ModelKey->getValueAsListOfDefs("IID").empty())
       Name = Name + "Model";
-    ProcModels.push_back(
-      CodeGenProcModel(ProcModels.size(), Name,
-                       ProcDef->getValueAsDef("SchedModel"), ModelKey));
+    ProcModels.emplace_back(ProcModels.size(), Name,
+                            ProcDef->getValueAsDef("SchedModel"), ModelKey);
   }
   DEBUG(ProcModels.back().dump());
 }
@@ -281,12 +278,12 @@ void CodeGenSchedModels::collectSchedRW() {
   std::sort(SWDefs.begin(), SWDefs.end(), LessRecord());
   for (RecIter SWI = SWDefs.begin(), SWE = SWDefs.end(); SWI != SWE; ++SWI) {
     assert(!getSchedRWIdx(*SWI, /*IsRead=*/false) && "duplicate SchedWrite");
-    SchedWrites.push_back(CodeGenSchedRW(SchedWrites.size(), *SWI));
+    SchedWrites.emplace_back(SchedWrites.size(), *SWI);
   }
   std::sort(SRDefs.begin(), SRDefs.end(), LessRecord());
   for (RecIter SRI = SRDefs.begin(), SRE = SRDefs.end(); SRI != SRE; ++SRI) {
     assert(!getSchedRWIdx(*SRI, /*IsRead-*/true) && "duplicate SchedWrite");
-    SchedReads.push_back(CodeGenSchedRW(SchedReads.size(), *SRI));
+    SchedReads.emplace_back(SchedReads.size(), *SRI);
   }
   // Initialize WriteSequence vectors.
   for (std::vector<CodeGenSchedRW>::iterator WI = SchedWrites.begin(),
diff --git a/utils/TableGen/CodeGenTarget.cpp b/utils/TableGen/CodeGenTarget.cpp
index 47c8e0bd36f..661975ecb20 100644
--- a/utils/TableGen/CodeGenTarget.cpp
+++ b/utils/TableGen/CodeGenTarget.cpp
@@ -99,6 +99,7 @@ std::string llvm::getEnumName(MVT::SimpleValueType T) {
   case MVT::v4i64:    return "MVT::v4i64";
   case MVT::v8i64:    return "MVT::v8i64";
   case MVT::v16i64:   return "MVT::v16i64";
+  case MVT::v1i128:   return "MVT::v1i128";
   case MVT::v2f16:    return "MVT::v2f16";
   case MVT::v4f16:    return "MVT::v4f16";
   case MVT::v8f16:    return "MVT::v8f16";
@@ -296,7 +297,7 @@ void CodeGenTarget::ComputeInstrsByEnum() const {
       "IMPLICIT_DEF", "SUBREG_TO_REG", "COPY_TO_REGCLASS", "DBG_VALUE",
       "REG_SEQUENCE", "COPY",          "BUNDLE",           "LIFETIME_START",
       "LIFETIME_END", "STACKMAP",      "PATCHPOINT",       "LOAD_STACK_GUARD",
-      "STATEPOINT",   "FRAME_ALLOC",
+      "STATEPOINT",   "LOCAL_ESCAPE",   "FAULTING_LOAD_OP",
       nullptr};
   const auto &Insts = getInstructions();
   for (const char *const *p = FixedInstrs; *p; ++p) {
@@ -410,9 +411,9 @@ ComplexPattern::ComplexPattern(Record *R) {
     } else if (PropList[i]->getName() == "SDNPWantParent") {
       Properties |= 1 << SDNPWantParent;
     } else {
-      errs() << "Unsupported SD Node property '" << PropList[i]->getName()
-             << "' on ComplexPattern '" << R->getName() << "'!\n";
-      exit(1);
+      PrintFatalError("Unsupported SD Node property '" +
+                      PropList[i]->getName() + "' on ComplexPattern '" +
+                      R->getName() + "'!");
     }
 }
 
@@ -443,6 +444,7 @@ CodeGenIntrinsic::CodeGenIntrinsic(Record *R) {
   canThrow = false;
   isNoReturn = false;
   isNoDuplicate = false;
+  isConvergent = false;
 
   if (DefName.size() <= 4 ||
       std::string(DefName.begin(), DefName.begin() + 4) != "int_")
@@ -484,7 +486,7 @@ CodeGenIntrinsic::CodeGenIntrinsic(Record *R) {
   // Parse the list of return types.
   std::vector<MVT::SimpleValueType> OverloadedVTs;
   ListInit *TypeList = R->getValueAsListInit("RetTypes");
-  for (unsigned i = 0, e = TypeList->getSize(); i != e; ++i) {
+  for (unsigned i = 0, e = TypeList->size(); i != e; ++i) {
     Record *TyEl = TypeList->getElementAsRecord(i);
     assert(TyEl->isSubClassOf("LLVMType") && "Expected a type!");
     MVT::SimpleValueType VT;
@@ -518,7 +520,7 @@ CodeGenIntrinsic::CodeGenIntrinsic(Record *R) {
 
   // Parse the list of parameter types.
   TypeList = R->getValueAsListInit("ParamTypes");
-  for (unsigned i = 0, e = TypeList->getSize(); i != e; ++i) {
+  for (unsigned i = 0, e = TypeList->size(); i != e; ++i) {
     Record *TyEl = TypeList->getElementAsRecord(i);
     assert(TyEl->isSubClassOf("LLVMType") && "Expected a type!");
     MVT::SimpleValueType VT;
@@ -554,7 +556,7 @@ CodeGenIntrinsic::CodeGenIntrinsic(Record *R) {
 
   // Parse the intrinsic properties.
   ListInit *PropList = R->getValueAsListInit("Properties");
-  for (unsigned i = 0, e = PropList->getSize(); i != e; ++i) {
+  for (unsigned i = 0, e = PropList->size(); i != e; ++i) {
     Record *Property = PropList->getElementAsRecord(i);
     assert(Property->isSubClassOf("IntrinsicProperty") &&
            "Expected a property!");
@@ -573,6 +575,8 @@ CodeGenIntrinsic::CodeGenIntrinsic(Record *R) {
       canThrow = true;
     else if (Property->getName() == "IntrNoDuplicate")
       isNoDuplicate = true;
+    else if (Property->getName() == "IntrConvergent")
+      isConvergent = true;
     else if (Property->getName() == "IntrNoReturn")
       isNoReturn = true;
     else if (Property->isSubClassOf("NoCapture")) {
diff --git a/utils/TableGen/DAGISelMatcherGen.cpp b/utils/TableGen/DAGISelMatcherGen.cpp
index eb806192bdc..9663b71d662 100644
--- a/utils/TableGen/DAGISelMatcherGen.cpp
+++ b/utils/TableGen/DAGISelMatcherGen.cpp
@@ -220,7 +220,7 @@ void MatcherGen::EmitLeafMatchCode(const TreePatternNode *N) {
   }
 
   // An UnsetInit represents a named node without any constraints.
-  if (N->getLeafValue() == UnsetInit::get()) {
+  if (isa<UnsetInit>(N->getLeafValue())) {
     assert(N->hasName() && "Unnamed ? leaf");
     return;
   }
@@ -268,8 +268,10 @@ void MatcherGen::EmitLeafMatchCode(const TreePatternNode *N) {
     // We can't model ComplexPattern uses that don't have their name taken yet.
     // The OPC_CheckComplexPattern operation implicitly records the results.
     if (N->getName().empty()) {
-      errs() << "We expect complex pattern uses to have names: " << *N << "\n";
-      exit(1);
+      std::string S;
+      raw_string_ostream OS(S);
+      OS << "We expect complex pattern uses to have names: " << *N;
+      PrintFatalError(OS.str());
     }
 
     // Remember this ComplexPattern so that we can emit it after all the other
diff --git a/utils/TableGen/DisassemblerEmitter.cpp b/utils/TableGen/DisassemblerEmitter.cpp
index f02051a2cd6..e8595271bcc 100644
--- a/utils/TableGen/DisassemblerEmitter.cpp
+++ b/utils/TableGen/DisassemblerEmitter.cpp
@@ -134,7 +134,7 @@ void EmitDisassembler(RecordKeeper &Records, raw_ostream &OS) {
       PredicateNamespace = "ARM";
 
     EmitFixedLenDecoder(Records, OS, PredicateNamespace,
-                        "if (!Check(S, ", ")) return MCDisassembler::Fail;",
+                        "if (!Check(S, ", "))",
                         "S", "MCDisassembler::Fail",
                         "  MCDisassembler::DecodeStatus S = "
                           "MCDisassembler::Success;\n(void)S;");
@@ -142,8 +142,7 @@ void EmitDisassembler(RecordKeeper &Records, raw_ostream &OS) {
   }
 
   EmitFixedLenDecoder(Records, OS, Target.getName(),
-                      "if (", " == MCDisassembler::Fail)"
-                       " return MCDisassembler::Fail;",
+                      "if (", " == MCDisassembler::Fail)",
                       "MCDisassembler::Success", "MCDisassembler::Fail", "");
 }
 
diff --git a/utils/TableGen/FixedLenDecoderEmitter.cpp b/utils/TableGen/FixedLenDecoderEmitter.cpp
index a4a46b316f2..cbf40a90b4d 100644
--- a/utils/TableGen/FixedLenDecoderEmitter.cpp
+++ b/utils/TableGen/FixedLenDecoderEmitter.cpp
@@ -44,9 +44,10 @@ struct EncodingField {
 struct OperandInfo {
   std::vector<EncodingField> Fields;
   std::string Decoder;
+  bool HasCompleteDecoder;
 
-  OperandInfo(std::string D)
-    : Decoder(D) { }
+  OperandInfo(std::string D, bool HCD)
+    : Decoder(D), HasCompleteDecoder(HCD) { }
 
   void addField(unsigned Base, unsigned Width, unsigned Offset) {
     Fields.push_back(EncodingField(Base, Width, Offset));
@@ -85,8 +86,7 @@ class FixedLenDecoderEmitter {
   FixedLenDecoderEmitter(RecordKeeper &R,
                          std::string PredicateNamespace,
                          std::string GPrefix  = "if (",
-                         std::string GPostfix = " == MCDisassembler::Fail)"
-                         " return MCDisassembler::Fail;",
+                         std::string GPostfix = " == MCDisassembler::Fail)",
                          std::string ROK      = "MCDisassembler::Success",
                          std::string RFail    = "MCDisassembler::Fail",
                          std::string L        = "") :
@@ -208,7 +208,7 @@ typedef std::vector<bit_value_t> insn_t;
 ///
 /// The Debug output shows the path that the decoding tree follows to reach the
 /// the conclusion that there is a conflict.  VST4q8a is a vst4 to double-spaced
-/// even registers, while VST4q8b is a vst4 to double-spaced odd regsisters.
+/// even registers, while VST4q8b is a vst4 to double-spaced odd registers.
 ///
 /// The encoding info in the .td files does not specify this meta information,
 /// which could have been used by the decoder to resolve the conflict.  The
@@ -448,10 +448,13 @@ class FilterChooser {
                                const Filter &Best) const;
 
   void emitBinaryParser(raw_ostream &o, unsigned &Indentation,
-                        const OperandInfo &OpInfo) const;
+                        const OperandInfo &OpInfo,
+                        bool &OpHasCompleteDecoder) const;
 
-  void emitDecoder(raw_ostream &OS, unsigned Indentation, unsigned Opc) const;
-  unsigned getDecoderIndex(DecoderSet &Decoders, unsigned Opc) const;
+  void emitDecoder(raw_ostream &OS, unsigned Indentation, unsigned Opc,
+                   bool &HasCompleteDecoder) const;
+  unsigned getDecoderIndex(DecoderSet &Decoders, unsigned Opc,
+                           bool &HasCompleteDecoder) const;
 
   // Assign a single filter and run with it.
   void runSingleFilter(unsigned startBit, unsigned numBit, bool mixed);
@@ -610,7 +613,7 @@ void Filter::emitTableEntry(DecoderTableInfo &TableInfo) const {
   TableInfo.Table.push_back(NumBits);
 
   // A new filter entry begins a new scope for fixup resolution.
-  TableInfo.FixupStack.push_back(FixupList());
+  TableInfo.FixupStack.emplace_back();
 
   DecoderTable &Table = TableInfo.Table;
 
@@ -779,7 +782,9 @@ void FixedLenDecoderEmitter::emitTable(formatted_raw_ostream &OS,
       OS << "// Skip to: " << ((I - Table.begin()) + NumToSkip) << "\n";
       break;
     }
-    case MCD::OPC_Decode: {
+    case MCD::OPC_Decode:
+    case MCD::OPC_TryDecode: {
+      bool IsTry = *I == MCD::OPC_TryDecode;
       ++I;
       // Extract the ULEB128 encoded Opcode to a buffer.
       uint8_t Buffer[8], *p = Buffer;
@@ -788,7 +793,8 @@ void FixedLenDecoderEmitter::emitTable(formatted_raw_ostream &OS,
                && "ULEB128 value too large!");
       // Decode the Opcode value.
       unsigned Opc = decodeULEB128(Buffer);
-      OS.indent(Indentation) << "MCD::OPC_Decode, ";
+      OS.indent(Indentation) << "MCD::OPC_" << (IsTry ? "Try" : "")
+        << "Decode, ";
       for (p = Buffer; *p >= 128; ++p)
         OS << utostr(*p) << ", ";
       OS << utostr(*p) << ", ";
@@ -798,8 +804,25 @@ void FixedLenDecoderEmitter::emitTable(formatted_raw_ostream &OS,
         OS << utostr(*I) << ", ";
       OS << utostr(*I++) << ", ";
 
+      if (!IsTry) {
+        OS << "// Opcode: "
+           << NumberedInstructions->at(Opc)->TheDef->getName() << "\n";
+        break;
+      }
+
+      // Fallthrough for OPC_TryDecode.
+
+      // 16-bit numtoskip value.
+      uint8_t Byte = *I++;
+      uint32_t NumToSkip = Byte;
+      OS << utostr(Byte) << ", ";
+      Byte = *I++;
+      OS << utostr(Byte) << ", ";
+      NumToSkip |= Byte << 8;
+
       OS << "// Opcode: "
-         << NumberedInstructions->at(Opc)->TheDef->getName() << "\n";
+         << NumberedInstructions->at(Opc)->TheDef->getName()
+         << ", skip to: " << ((I - Table.begin()) + NumToSkip) << "\n";
       break;
     }
     case MCD::OPC_SoftFail: {
@@ -848,7 +871,7 @@ emitPredicateFunction(formatted_raw_ostream &OS, PredicateSet &Predicates,
   // The predicate function is just a big switch statement based on the
   // input predicate index.
   OS.indent(Indentation) << "static bool checkDecoderPredicate(unsigned Idx, "
-    << "uint64_t Bits) {\n";
+    << "const FeatureBitset& Bits) {\n";
   Indentation += 2;
   if (!Predicates.empty()) {
     OS.indent(Indentation) << "switch (Idx) {\n";
@@ -876,8 +899,9 @@ emitDecoderFunction(formatted_raw_ostream &OS, DecoderSet &Decoders,
   OS.indent(Indentation) << "static DecodeStatus decodeToMCInst(DecodeStatus S,"
     << " unsigned Idx, InsnType insn, MCInst &MI,\n";
   OS.indent(Indentation) << "                                   uint64_t "
-    << "Address, const void *Decoder) {\n";
+    << "Address, const void *Decoder, bool &DecodeComplete) {\n";
   Indentation += 2;
+  OS.indent(Indentation) << "DecodeComplete = true;\n";
   OS.indent(Indentation) << "InsnType tmp;\n";
   OS.indent(Indentation) << "switch (Idx) {\n";
   OS.indent(Indentation) << "default: llvm_unreachable(\"Invalid index!\");\n";
@@ -1033,7 +1057,8 @@ unsigned FilterChooser::getIslands(std::vector<unsigned> &StartBits,
 }
 
 void FilterChooser::emitBinaryParser(raw_ostream &o, unsigned &Indentation,
-                                     const OperandInfo &OpInfo) const {
+                                     const OperandInfo &OpInfo,
+                                     bool &OpHasCompleteDecoder) const {
   const std::string &Decoder = OpInfo.Decoder;
 
   if (OpInfo.numFields() != 1)
@@ -1049,39 +1074,52 @@ void FilterChooser::emitBinaryParser(raw_ostream &o, unsigned &Indentation,
     o << ";\n";
   }
 
-  if (Decoder != "")
+  if (Decoder != "") {
+    OpHasCompleteDecoder = OpInfo.HasCompleteDecoder;
     o.indent(Indentation) << Emitter->GuardPrefix << Decoder
-                          << "(MI, tmp, Address, Decoder)"
-                          << Emitter->GuardPostfix << "\n";
-  else
-    o.indent(Indentation) << "MI.addOperand(MCOperand::CreateImm(tmp));\n";
-
+      << "(MI, tmp, Address, Decoder)"
+      << Emitter->GuardPostfix
+      << " { " << (OpHasCompleteDecoder ? "" : "DecodeComplete = false; ")
+      << "return MCDisassembler::Fail; }\n";
+  } else {
+    OpHasCompleteDecoder = true;
+    o.indent(Indentation) << "MI.addOperand(MCOperand::createImm(tmp));\n";
+  }
 }
 
 void FilterChooser::emitDecoder(raw_ostream &OS, unsigned Indentation,
-                                unsigned Opc) const {
+                                unsigned Opc, bool &HasCompleteDecoder) const {
+  HasCompleteDecoder = true;
+
   for (const auto &Op : Operands.find(Opc)->second) {
     // If a custom instruction decoder was specified, use that.
     if (Op.numFields() == 0 && Op.Decoder.size()) {
+      HasCompleteDecoder = Op.HasCompleteDecoder;
       OS.indent(Indentation) << Emitter->GuardPrefix << Op.Decoder
         << "(MI, insn, Address, Decoder)"
-        << Emitter->GuardPostfix << "\n";
+        << Emitter->GuardPostfix
+        << " { " << (HasCompleteDecoder ? "" : "DecodeComplete = false; ")
+        << "return MCDisassembler::Fail; }\n";
       break;
     }
 
-    emitBinaryParser(OS, Indentation, Op);
+    bool OpHasCompleteDecoder;
+    emitBinaryParser(OS, Indentation, Op, OpHasCompleteDecoder);
+    if (!OpHasCompleteDecoder)
+      HasCompleteDecoder = false;
   }
 }
 
 unsigned FilterChooser::getDecoderIndex(DecoderSet &Decoders,
-                                        unsigned Opc) const {
+                                        unsigned Opc,
+                                        bool &HasCompleteDecoder) const {
   // Build up the predicate string.
   SmallString<256> Decoder;
   // FIXME: emitDecoder() function can take a buffer directly rather than
   // a stream.
   raw_svector_ostream S(Decoder);
   unsigned I = 4;
-  emitDecoder(S, I, Opc);
+  emitDecoder(S, I, Opc, HasCompleteDecoder);
   S.flush();
 
   // Using the full decoder string as the key value here is a bit
@@ -1102,10 +1140,10 @@ unsigned FilterChooser::getDecoderIndex(DecoderSet &Decoders,
 static void emitSinglePredicateMatch(raw_ostream &o, StringRef str,
                                      const std::string &PredicateNamespace) {
   if (str[0] == '!')
-    o << "!(Bits & " << PredicateNamespace << "::"
-      << str.slice(1,str.size()) << ")";
+    o << "!Bits[" << PredicateNamespace << "::"
+      << str.slice(1,str.size()) << "]";
   else
-    o << "(Bits & " << PredicateNamespace << "::" << str << ")";
+    o << "Bits[" << PredicateNamespace << "::" << str << "]";
 }
 
 bool FilterChooser::emitPredicateMatch(raw_ostream &o, unsigned &Indentation,
@@ -1113,7 +1151,7 @@ bool FilterChooser::emitPredicateMatch(raw_ostream &o, unsigned &Indentation,
   ListInit *Predicates =
     AllInstructions[Opc]->TheDef->getValueAsListInit("Predicates");
   bool IsFirstEmission = true;
-  for (unsigned i = 0; i < Predicates->getSize(); ++i) {
+  for (unsigned i = 0; i < Predicates->size(); ++i) {
     Record *Pred = Predicates->getElementAsRecord(i);
     if (!Pred->getValue("AssemblerMatcherPredicate"))
       continue;
@@ -1136,13 +1174,13 @@ bool FilterChooser::emitPredicateMatch(raw_ostream &o, unsigned &Indentation,
     emitSinglePredicateMatch(o, pairs.first, Emitter->PredicateNamespace);
     IsFirstEmission = false;
   }
-  return Predicates->getSize() > 0;
+  return !Predicates->empty();
 }
 
 bool FilterChooser::doesOpcodeNeedPredicate(unsigned Opc) const {
   ListInit *Predicates =
     AllInstructions[Opc]->TheDef->getValueAsListInit("Predicates");
-  for (unsigned i = 0; i < Predicates->getSize(); ++i) {
+  for (unsigned i = 0; i < Predicates->size(); ++i) {
     Record *Pred = Predicates->getElementAsRecord(i);
     if (!Pred->getValue("AssemblerMatcherPredicate"))
       continue;
@@ -1308,14 +1346,26 @@ void FilterChooser::emitSingletonTableEntry(DecoderTableInfo &TableInfo,
   // Check for soft failure of the match.
   emitSoftFailTableEntry(TableInfo, Opc);
 
-  TableInfo.Table.push_back(MCD::OPC_Decode);
+  bool HasCompleteDecoder;
+  unsigned DIdx = getDecoderIndex(TableInfo.Decoders, Opc, HasCompleteDecoder);
+
+  // Produce OPC_Decode or OPC_TryDecode opcode based on the information
+  // whether the instruction decoder is complete or not. If it is complete
+  // then it handles all possible values of remaining variable/unfiltered bits
+  // and for any value can determine if the bitpattern is a valid instruction
+  // or not. This means OPC_Decode will be the final step in the decoding
+  // process. If it is not complete, then the Fail return code from the
+  // decoder method indicates that additional processing should be done to see
+  // if there is any other instruction that also matches the bitpattern and
+  // can decode it.
+  TableInfo.Table.push_back(HasCompleteDecoder ? MCD::OPC_Decode :
+      MCD::OPC_TryDecode);
   uint8_t Buffer[8], *p;
   encodeULEB128(Opc, Buffer);
   for (p = Buffer; *p >= 128 ; ++p)
     TableInfo.Table.push_back(*p);
   TableInfo.Table.push_back(*p);
 
-  unsigned DIdx = getDecoderIndex(TableInfo.Decoders, Opc);
   SmallString<16> Bytes;
   raw_svector_ostream S(Bytes);
   encodeULEB128(DIdx, S);
@@ -1324,6 +1374,14 @@ void FilterChooser::emitSingletonTableEntry(DecoderTableInfo &TableInfo,
   // Decoder index
   for (unsigned i = 0, e = Bytes.size(); i != e; ++i)
     TableInfo.Table.push_back(Bytes[i]);
+
+  if (!HasCompleteDecoder) {
+    // Push location for NumToSkip backpatching.
+    TableInfo.FixupStack.back().push_back(TableInfo.Table.size());
+    // Allocate the space for the fixup.
+    TableInfo.Table.push_back(0);
+    TableInfo.Table.push_back(0);
+  }
 }
 
 // Emits table entries to decode the singleton, and then to decode the rest.
@@ -1333,7 +1391,7 @@ void FilterChooser::emitSingletonTableEntry(DecoderTableInfo &TableInfo,
 
   // complex singletons need predicate checks from the first singleton
   // to refer forward to the variable filterchooser that follows.
-  TableInfo.FixupStack.push_back(FixupList());
+  TableInfo.FixupStack.emplace_back();
 
   emitSingletonTableEntry(TableInfo, Opc);
 
@@ -1350,7 +1408,7 @@ void FilterChooser::emitSingletonTableEntry(DecoderTableInfo &TableInfo,
 void FilterChooser::runSingleFilter(unsigned startBit, unsigned numBit,
                                     bool mixed) {
   Filters.clear();
-  Filters.push_back(Filter(*this, startBit, numBit, true));
+  Filters.emplace_back(*this, startBit, numBit, true);
   BestIndex = 0; // Sole Filter instance to choose from.
   bestFilter().recurse();
 }
@@ -1360,9 +1418,9 @@ void FilterChooser::runSingleFilter(unsigned startBit, unsigned numBit,
 void FilterChooser::reportRegion(bitAttr_t RA, unsigned StartBit,
                                  unsigned BitIndex, bool AllowMixed) {
   if (RA == ATTR_MIXED && AllowMixed)
-    Filters.push_back(Filter(*this, StartBit, BitIndex - StartBit, true));
+    Filters.emplace_back(*this, StartBit, BitIndex - StartBit, true);
   else if (RA == ATTR_ALL_SET && !AllowMixed)
-    Filters.push_back(Filter(*this, StartBit, BitIndex - StartBit, false));
+    Filters.emplace_back(*this, StartBit, BitIndex - StartBit, false);
 }
 
 // FilterProcessor scans the well-known encoding bits of the instructions and
@@ -1679,7 +1737,8 @@ static bool populateInstruction(CodeGenTarget &Target,
   // of trying to auto-generate the decoder.
   std::string InstDecoder = Def.getValueAsString("DecoderMethod");
   if (InstDecoder != "") {
-    InsnOperands.push_back(OperandInfo(InstDecoder));
+    bool HasCompleteInstDecoder = Def.getValueAsBit("hasCompleteDecoder");
+    InsnOperands.push_back(OperandInfo(InstDecoder, HasCompleteInstDecoder));
     Operands[Opc] = InsnOperands;
     return true;
   }
@@ -1835,7 +1894,14 @@ static bool populateInstruction(CodeGenTarget &Target,
       if (!isReg && String && String->getValue() != "")
         Decoder = String->getValue();
 
-      OperandInfo OpInfo(Decoder);
+      RecordVal *HasCompleteDecoderVal =
+        TypeRecord->getValue("hasCompleteDecoder");
+      BitInit *HasCompleteDecoderBit = HasCompleteDecoderVal ?
+        dyn_cast<BitInit>(HasCompleteDecoderVal->getValue()) : nullptr;
+      bool HasCompleteDecoder = HasCompleteDecoderBit ?
+        HasCompleteDecoderBit->getValue() : true;
+
+      OperandInfo OpInfo(Decoder, HasCompleteDecoder);
       OpInfo.addField(bitStart, bitWidth, 0);
 
       NumberedInsnOperands[Name].push_back(OpInfo);
@@ -1907,7 +1973,14 @@ static bool populateInstruction(CodeGenTarget &Target,
     if (!isReg && String && String->getValue() != "")
       Decoder = String->getValue();
 
-    OperandInfo OpInfo(Decoder);
+    RecordVal *HasCompleteDecoderVal =
+      TypeRecord->getValue("hasCompleteDecoder");
+    BitInit *HasCompleteDecoderBit = HasCompleteDecoderVal ?
+      dyn_cast<BitInit>(HasCompleteDecoderVal->getValue()) : nullptr;
+    bool HasCompleteDecoder = HasCompleteDecoderBit ?
+      HasCompleteDecoderBit->getValue() : true;
+
+    OperandInfo OpInfo(Decoder, HasCompleteDecoder);
     unsigned Base = ~0U;
     unsigned Width = 0;
     unsigned Offset = 0;
@@ -2012,7 +2085,7 @@ static void emitDecodeInstruction(formatted_raw_ostream &OS) {
      << "                                      InsnType insn, uint64_t Address,\n"
      << "                                      const void *DisAsm,\n"
      << "                                      const MCSubtargetInfo &STI) {\n"
-     << "  uint64_t Bits = STI.getFeatureBits();\n"
+     << "  const FeatureBitset& Bits = STI.getFeatureBits();\n"
      << "\n"
      << "  const uint8_t *Ptr = DecodeTable;\n"
      << "  uint32_t CurFieldValue = 0;\n"
@@ -2096,12 +2169,51 @@ static void emitDecodeInstruction(formatted_raw_ostream &OS) {
      << "      Ptr += Len;\n"
      << "      unsigned DecodeIdx = decodeULEB128(Ptr, &Len);\n"
      << "      Ptr += Len;\n"
-     << "      DEBUG(dbgs() << Loc << \": OPC_Decode: opcode \" << Opc\n"
-     << "                   << \", using decoder \" << DecodeIdx << \"\\n\" );\n"
-     << "      DEBUG(dbgs() << \"----- DECODE SUCCESSFUL -----\\n\");\n"
      << "\n"
      << "      MI.setOpcode(Opc);\n"
-     << "      return decodeToMCInst(S, DecodeIdx, insn, MI, Address, DisAsm);\n"
+     << "      bool DecodeComplete;\n"
+     << "      S = decodeToMCInst(S, DecodeIdx, insn, MI, Address, DisAsm, DecodeComplete);\n"
+     << "      assert(DecodeComplete);\n"
+     << "\n"
+     << "      DEBUG(dbgs() << Loc << \": OPC_Decode: opcode \" << Opc\n"
+     << "                   << \", using decoder \" << DecodeIdx << \": \"\n"
+     << "                   << (S != MCDisassembler::Fail ? \"PASS\" : \"FAIL\") << \"\\n\");\n"
+     << "      return S;\n"
+     << "    }\n"
+     << "    case MCD::OPC_TryDecode: {\n"
+     << "      unsigned Len;\n"
+     << "      // Decode the Opcode value.\n"
+     << "      unsigned Opc = decodeULEB128(++Ptr, &Len);\n"
+     << "      Ptr += Len;\n"
+     << "      unsigned DecodeIdx = decodeULEB128(Ptr, &Len);\n"
+     << "      Ptr += Len;\n"
+     << "      // NumToSkip is a plain 16-bit integer.\n"
+     << "      unsigned NumToSkip = *Ptr++;\n"
+     << "      NumToSkip |= (*Ptr++) << 8;\n"
+     << "\n"
+     << "      // Perform the decode operation.\n"
+     << "      MCInst TmpMI;\n"
+     << "      TmpMI.setOpcode(Opc);\n"
+     << "      bool DecodeComplete;\n"
+     << "      S = decodeToMCInst(S, DecodeIdx, insn, TmpMI, Address, DisAsm, DecodeComplete);\n"
+     << "      DEBUG(dbgs() << Loc << \": OPC_TryDecode: opcode \" << Opc\n"
+     << "                   << \", using decoder \" << DecodeIdx << \": \");\n"
+     << "\n"
+     << "      if (DecodeComplete) {\n"
+     << "        // Decoding complete.\n"
+     << "        DEBUG(dbgs() << (S != MCDisassembler::Fail ? \"PASS\" : \"FAIL\") << \"\\n\");\n"
+     << "        MI = TmpMI;\n"
+     << "        return S;\n"
+     << "      } else {\n"
+     << "        assert(S == MCDisassembler::Fail);\n"
+     << "        // If the decoding was incomplete, skip.\n"
+     << "        Ptr += NumToSkip;\n"
+     << "        DEBUG(dbgs() << \"FAIL: continuing at \" << (Ptr - DecodeTable) << \"\\n\");\n"
+     << "        // Reset decode status. This also drops a SoftFail status that could be\n"
+     << "        // set before the decode attempt.\n"
+     << "        S = MCDisassembler::Success;\n"
+     << "      }\n"
+     << "      break;\n"
      << "    }\n"
      << "    case MCD::OPC_SoftFail: {\n"
      << "      // Decode the mask values.\n"
@@ -2179,7 +2291,7 @@ void FixedLenDecoderEmitter::run(raw_ostream &o) {
     TableInfo.Table.clear();
     TableInfo.FixupStack.clear();
     TableInfo.Table.reserve(16384);
-    TableInfo.FixupStack.push_back(FixupList());
+    TableInfo.FixupStack.emplace_back();
     FC.emitTableEntries(TableInfo);
     // Any NumToSkip fixups in the top level scope can resolve to the
     // OPC_Fail at the end of the table.
diff --git a/utils/TableGen/InstrInfoEmitter.cpp b/utils/TableGen/InstrInfoEmitter.cpp
index f999326bc97..e242a965ff9 100644
--- a/utils/TableGen/InstrInfoEmitter.cpp
+++ b/utils/TableGen/InstrInfoEmitter.cpp
@@ -475,41 +475,42 @@ void InstrInfoEmitter::emitRecord(const CodeGenInstruction &Inst, unsigned Num,
   OS << "  { ";
   OS << Num << ",\t" << MinOperands << ",\t"
      << Inst.Operands.NumDefs << ",\t"
-     << SchedModels.getSchedClassIdx(Inst) << ",\t"
-     << Inst.TheDef->getValueAsInt("Size") << ",\t0";
+     << Inst.TheDef->getValueAsInt("Size") << ",\t"
+     << SchedModels.getSchedClassIdx(Inst) << ",\t0";
 
   // Emit all of the target independent flags...
-  if (Inst.isPseudo)           OS << "|(1<<MCID::Pseudo)";
-  if (Inst.isReturn)           OS << "|(1<<MCID::Return)";
-  if (Inst.isBranch)           OS << "|(1<<MCID::Branch)";
-  if (Inst.isIndirectBranch)   OS << "|(1<<MCID::IndirectBranch)";
-  if (Inst.isCompare)          OS << "|(1<<MCID::Compare)";
-  if (Inst.isMoveImm)          OS << "|(1<<MCID::MoveImm)";
-  if (Inst.isBitcast)          OS << "|(1<<MCID::Bitcast)";
-  if (Inst.isSelect)           OS << "|(1<<MCID::Select)";
-  if (Inst.isBarrier)          OS << "|(1<<MCID::Barrier)";
-  if (Inst.hasDelaySlot)       OS << "|(1<<MCID::DelaySlot)";
-  if (Inst.isCall)             OS << "|(1<<MCID::Call)";
-  if (Inst.canFoldAsLoad)      OS << "|(1<<MCID::FoldableAsLoad)";
-  if (Inst.mayLoad)            OS << "|(1<<MCID::MayLoad)";
-  if (Inst.mayStore)           OS << "|(1<<MCID::MayStore)";
-  if (Inst.isPredicable)       OS << "|(1<<MCID::Predicable)";
-  if (Inst.isConvertibleToThreeAddress) OS << "|(1<<MCID::ConvertibleTo3Addr)";
-  if (Inst.isCommutable)       OS << "|(1<<MCID::Commutable)";
-  if (Inst.isTerminator)       OS << "|(1<<MCID::Terminator)";
-  if (Inst.isReMaterializable) OS << "|(1<<MCID::Rematerializable)";
-  if (Inst.isNotDuplicable)    OS << "|(1<<MCID::NotDuplicable)";
-  if (Inst.Operands.hasOptionalDef) OS << "|(1<<MCID::HasOptionalDef)";
-  if (Inst.usesCustomInserter) OS << "|(1<<MCID::UsesCustomInserter)";
-  if (Inst.hasPostISelHook)    OS << "|(1<<MCID::HasPostISelHook)";
-  if (Inst.Operands.isVariadic)OS << "|(1<<MCID::Variadic)";
-  if (Inst.hasSideEffects)     OS << "|(1<<MCID::UnmodeledSideEffects)";
-  if (Inst.isAsCheapAsAMove)   OS << "|(1<<MCID::CheapAsAMove)";
-  if (Inst.hasExtraSrcRegAllocReq) OS << "|(1<<MCID::ExtraSrcRegAllocReq)";
-  if (Inst.hasExtraDefRegAllocReq) OS << "|(1<<MCID::ExtraDefRegAllocReq)";
-  if (Inst.isRegSequence) OS << "|(1<<MCID::RegSequence)";
-  if (Inst.isExtractSubreg) OS << "|(1<<MCID::ExtractSubreg)";
-  if (Inst.isInsertSubreg) OS << "|(1<<MCID::InsertSubreg)";
+  if (Inst.isPseudo)           OS << "|(1ULL<<MCID::Pseudo)";
+  if (Inst.isReturn)           OS << "|(1ULL<<MCID::Return)";
+  if (Inst.isBranch)           OS << "|(1ULL<<MCID::Branch)";
+  if (Inst.isIndirectBranch)   OS << "|(1ULL<<MCID::IndirectBranch)";
+  if (Inst.isCompare)          OS << "|(1ULL<<MCID::Compare)";
+  if (Inst.isMoveImm)          OS << "|(1ULL<<MCID::MoveImm)";
+  if (Inst.isBitcast)          OS << "|(1ULL<<MCID::Bitcast)";
+  if (Inst.isSelect)           OS << "|(1ULL<<MCID::Select)";
+  if (Inst.isBarrier)          OS << "|(1ULL<<MCID::Barrier)";
+  if (Inst.hasDelaySlot)       OS << "|(1ULL<<MCID::DelaySlot)";
+  if (Inst.isCall)             OS << "|(1ULL<<MCID::Call)";
+  if (Inst.canFoldAsLoad)      OS << "|(1ULL<<MCID::FoldableAsLoad)";
+  if (Inst.mayLoad)            OS << "|(1ULL<<MCID::MayLoad)";
+  if (Inst.mayStore)           OS << "|(1ULL<<MCID::MayStore)";
+  if (Inst.isPredicable)       OS << "|(1ULL<<MCID::Predicable)";
+  if (Inst.isConvertibleToThreeAddress) OS << "|(1ULL<<MCID::ConvertibleTo3Addr)";
+  if (Inst.isCommutable)       OS << "|(1ULL<<MCID::Commutable)";
+  if (Inst.isTerminator)       OS << "|(1ULL<<MCID::Terminator)";
+  if (Inst.isReMaterializable) OS << "|(1ULL<<MCID::Rematerializable)";
+  if (Inst.isNotDuplicable)    OS << "|(1ULL<<MCID::NotDuplicable)";
+  if (Inst.Operands.hasOptionalDef) OS << "|(1ULL<<MCID::HasOptionalDef)";
+  if (Inst.usesCustomInserter) OS << "|(1ULL<<MCID::UsesCustomInserter)";
+  if (Inst.hasPostISelHook)    OS << "|(1ULL<<MCID::HasPostISelHook)";
+  if (Inst.Operands.isVariadic)OS << "|(1ULL<<MCID::Variadic)";
+  if (Inst.hasSideEffects)     OS << "|(1ULL<<MCID::UnmodeledSideEffects)";
+  if (Inst.isAsCheapAsAMove)   OS << "|(1ULL<<MCID::CheapAsAMove)";
+  if (Inst.hasExtraSrcRegAllocReq) OS << "|(1ULL<<MCID::ExtraSrcRegAllocReq)";
+  if (Inst.hasExtraDefRegAllocReq) OS << "|(1ULL<<MCID::ExtraDefRegAllocReq)";
+  if (Inst.isRegSequence) OS << "|(1ULL<<MCID::RegSequence)";
+  if (Inst.isExtractSubreg) OS << "|(1ULL<<MCID::ExtractSubreg)";
+  if (Inst.isInsertSubreg) OS << "|(1ULL<<MCID::InsertSubreg)";
+  if (Inst.isConvergent) OS << "|(1ULL<<MCID::Convergent)";
 
   // Emit all of the target-specific flags...
   BitsInit *TSF = Inst.TheDef->getValueAsBitsInit("TSFlags");
@@ -549,15 +550,15 @@ void InstrInfoEmitter::emitRecord(const CodeGenInstruction &Inst, unsigned Num,
   CodeGenTarget &Target = CDP.getTargetInfo();
   if (Inst.HasComplexDeprecationPredicate)
     // Emit a function pointer to the complex predicate method.
-    OS << ",0"
+    OS << ", -1 "
        << ",&get" << Inst.DeprecatedReason << "DeprecationInfo";
   else if (!Inst.DeprecatedReason.empty())
     // Emit the Subtarget feature.
-    OS << "," << Target.getInstNamespace() << "::" << Inst.DeprecatedReason
-       << ",nullptr";
+    OS << ", " << Target.getInstNamespace() << "::" << Inst.DeprecatedReason
+       << " ,nullptr";
   else
     // Instruction isn't deprecated.
-    OS << ",0,nullptr";
+    OS << ", -1 ,nullptr";
 
   OS << " },  // Inst #" << Num << " = " << Inst.TheDef->getName() << "\n";
 }
@@ -575,10 +576,8 @@ void InstrInfoEmitter::emitEnums(raw_ostream &OS) {
   // We must emit the PHI opcode first...
   std::string Namespace = Target.getInstNamespace();
 
-  if (Namespace.empty()) {
-    fprintf(stderr, "No instructions defined!\n");
-    exit(1);
-  }
+  if (Namespace.empty())
+    PrintFatalError("No instructions defined!");
 
   const std::vector<const CodeGenInstruction*> &NumberedInstructions =
     Target.getInstructionsByEnumValue();
diff --git a/utils/TableGen/IntrinsicEmitter.cpp b/utils/TableGen/IntrinsicEmitter.cpp
index e71c695c25d..cebe5bbc313 100644
--- a/utils/TableGen/IntrinsicEmitter.cpp
+++ b/utils/TableGen/IntrinsicEmitter.cpp
@@ -260,7 +260,8 @@ enum IIT_Info {
   IIT_HALF_VEC_ARG = 29,
   IIT_SAME_VEC_WIDTH_ARG = 30,
   IIT_PTR_TO_ARG = 31,
-  IIT_VEC_OF_PTRS_TO_ELT = 32
+  IIT_VEC_OF_PTRS_TO_ELT = 32,
+  IIT_I128 = 33
 };
 
 
@@ -275,6 +276,7 @@ static void EncodeFixedValueType(MVT::SimpleValueType VT,
     case 16: return Sig.push_back(IIT_I16);
     case 32: return Sig.push_back(IIT_I32);
     case 64: return Sig.push_back(IIT_I64);
+    case 128: return Sig.push_back(IIT_I128);
     }
   }
 
@@ -535,6 +537,9 @@ struct AttributeComparator {
     if (L->isNoReturn != R->isNoReturn)
       return R->isNoReturn;
 
+    if (L->isConvergent != R->isConvergent)
+      return R->isConvergent;
+
     // Try to order by readonly/readnone attribute.
     ModRefKind LK = getModRefKind(*L);
     ModRefKind RK = getModRefKind(*R);
@@ -631,7 +636,7 @@ EmitAttributes(const std::vector<CodeGenIntrinsic> &Ints, raw_ostream &OS) {
           case CodeGenIntrinsic::ReadNone:
             if (addComma)
               OS << ",";
-            OS << "Attributes::ReadNone";
+            OS << "Attribute::ReadNone";
             addComma = true;
             break;
           }
@@ -647,7 +652,7 @@ EmitAttributes(const std::vector<CodeGenIntrinsic> &Ints, raw_ostream &OS) {
     ModRefKind modRef = getModRefKind(intrinsic);
 
     if (!intrinsic.canThrow || modRef || intrinsic.isNoReturn ||
-        intrinsic.isNoDuplicate) {
+        intrinsic.isNoDuplicate || intrinsic.isConvergent) {
       OS << "      const Attribute::AttrKind Atts[] = {";
       bool addComma = false;
       if (!intrinsic.canThrow) {
@@ -666,6 +671,12 @@ EmitAttributes(const std::vector<CodeGenIntrinsic> &Ints, raw_ostream &OS) {
         OS << "Attribute::NoDuplicate";
         addComma = true;
       }
+      if (intrinsic.isConvergent) {
+        if (addComma)
+          OS << ",";
+        OS << "Attribute::Convergent";
+        addComma = true;
+      }
 
       switch (modRef) {
       case MRK_none: break;
@@ -711,29 +722,30 @@ EmitModRefBehavior(const std::vector<CodeGenIntrinsic> &Ints, raw_ostream &OS){
      << "\"Unknown intrinsic.\");\n\n";
 
   OS << "static const uint8_t IntrinsicModRefBehavior[] = {\n"
-     << "  /* invalid */ UnknownModRefBehavior,\n";
+     << "  /* invalid */ FMRB_UnknownModRefBehavior,\n";
   for (unsigned i = 0, e = Ints.size(); i != e; ++i) {
     OS << "  /* " << TargetPrefix << Ints[i].EnumName << " */ ";
     switch (Ints[i].ModRef) {
     case CodeGenIntrinsic::NoMem:
-      OS << "DoesNotAccessMemory,\n";
+      OS << "FMRB_DoesNotAccessMemory,\n";
       break;
     case CodeGenIntrinsic::ReadArgMem:
-      OS << "OnlyReadsArgumentPointees,\n";
+      OS << "FMRB_OnlyReadsArgumentPointees,\n";
       break;
     case CodeGenIntrinsic::ReadMem:
-      OS << "OnlyReadsMemory,\n";
+      OS << "FMRB_OnlyReadsMemory,\n";
       break;
     case CodeGenIntrinsic::ReadWriteArgMem:
-      OS << "OnlyAccessesArgumentPointees,\n";
+      OS << "FMRB_OnlyAccessesArgumentPointees,\n";
       break;
     case CodeGenIntrinsic::ReadWriteMem:
-      OS << "UnknownModRefBehavior,\n";
+      OS << "FMRB_UnknownModRefBehavior,\n";
       break;
     }
   }
   OS << "};\n\n"
-     << "return static_cast<ModRefBehavior>(IntrinsicModRefBehavior[iid]);\n"
+     << "return "
+        "static_cast<FunctionModRefBehavior>(IntrinsicModRefBehavior[iid]);\n"
      << "#endif // GET_INTRINSIC_MODREF_BEHAVIOR\n\n";
 }
 
@@ -749,7 +761,7 @@ static void EmitTargetBuiltins(const std::map<std::string, std::string> &BIM,
        E = BIM.end(); I != E; ++I) {
     std::string ResultCode =
     "return " + TargetPrefix + "Intrinsic::" + I->second + ";";
-    Results.push_back(StringMatcher::StringPair(I->first, ResultCode));
+    Results.emplace_back(I->first, ResultCode);
   }
 
   StringMatcher("BuiltinName", Results, OS).Emit();
diff --git a/utils/TableGen/PseudoLoweringEmitter.cpp b/utils/TableGen/PseudoLoweringEmitter.cpp
index ebb43f06526..01e41d1060e 100644
--- a/utils/TableGen/PseudoLoweringEmitter.cpp
+++ b/utils/TableGen/PseudoLoweringEmitter.cpp
@@ -232,12 +232,12 @@ void PseudoLoweringEmitter::emitLoweringEmitter(raw_ostream &o) {
               << "      TmpInst.addOperand(MCOp);\n";
             break;
             case OpData::Imm:
-            o << "      TmpInst.addOperand(MCOperand::CreateImm("
+            o << "      TmpInst.addOperand(MCOperand::createImm("
               << Expansion.OperandMap[MIOpNo + i].Data.Imm << "));\n";
             break;
             case OpData::Reg: {
               Record *Reg = Expansion.OperandMap[MIOpNo + i].Data.Reg;
-              o << "      TmpInst.addOperand(MCOperand::CreateReg(";
+              o << "      TmpInst.addOperand(MCOperand::createReg(";
               // "zero_reg" is special.
               if (Reg->getName() == "zero_reg")
                 o << "0";
diff --git a/utils/TableGen/RegisterInfoEmitter.cpp b/utils/TableGen/RegisterInfoEmitter.cpp
index a8423a98ae8..7c14febd494 100644
--- a/utils/TableGen/RegisterInfoEmitter.cpp
+++ b/utils/TableGen/RegisterInfoEmitter.cpp
@@ -1070,11 +1070,11 @@ RegisterInfoEmitter::runTargetHeader(raw_ostream &OS, CodeGenTarget &Target,
 
   OS << "namespace llvm {\n\n";
 
+  OS << "class " << TargetName << "FrameLowering;\n\n";
+
   OS << "struct " << ClassName << " : public TargetRegisterInfo {\n"
      << "  explicit " << ClassName
-     << "(unsigned RA, unsigned D = 0, unsigned E = 0, unsigned PC = 0);\n"
-     << "  bool needsStackRealignment(const MachineFunction &) const override\n"
-     << "     { return false; }\n";
+     << "(unsigned RA, unsigned D = 0, unsigned E = 0, unsigned PC = 0);\n";
   if (!RegBank.getSubRegIndices().empty()) {
     OS << "  unsigned composeSubRegIndicesImpl"
        << "(unsigned, unsigned) const override;\n"
@@ -1094,6 +1094,11 @@ RegisterInfoEmitter::runTargetHeader(raw_ostream &OS, CodeGenTarget &Target,
      << "const TargetRegisterClass *RC) const override;\n"
      << "  const int *getRegUnitPressureSets("
      << "unsigned RegUnit) const override;\n"
+     << "  ArrayRef<const char *> getRegMaskNames() const override;\n"
+     << "  ArrayRef<const uint32_t *> getRegMasks() const override;\n"
+     << "  /// Devirtualized TargetFrameLowering.\n"
+     << "  static const " << TargetName << "FrameLowering *getFrameLowering(\n"
+     << "      const MachineFunction &MF);\n"
      << "};\n\n";
 
   const auto &RegisterClasses = RegBank.getRegClasses();
@@ -1445,6 +1450,33 @@ RegisterInfoEmitter::runTargetDesc(raw_ostream &OS, CodeGenTarget &Target,
   }
   OS << "\n\n";
 
+  OS << "ArrayRef<const uint32_t *> " << ClassName
+     << "::getRegMasks() const {\n";
+  OS << "  static const uint32_t *Masks[] = {\n";
+  for (Record *CSRSet : CSRSets)
+    OS << "    " << CSRSet->getName() << "_RegMask, \n";
+  OS << "    nullptr\n  };\n";
+  OS << "  return ArrayRef<const uint32_t *>(Masks, (size_t)" << CSRSets.size()
+     << ");\n";
+  OS << "}\n\n";
+
+  OS << "ArrayRef<const char *> " << ClassName
+     << "::getRegMaskNames() const {\n";
+  OS << "  static const char *Names[] = {\n";
+  for (Record *CSRSet : CSRSets)
+    OS << "    " << '"' << CSRSet->getName() << '"' << ",\n";
+  OS << "    nullptr\n  };\n";
+  OS << "  return ArrayRef<const char *>(Names, (size_t)" << CSRSets.size()
+     << ");\n";
+  OS << "}\n\n";
+
+  OS << "const " << TargetName << "FrameLowering *"
+     << TargetName << "GenRegisterInfo::\n"
+     << "    getFrameLowering(const MachineFunction &MF) {\n"
+     << "  return static_cast<const " << TargetName << "FrameLowering *>(\n"
+     << "      MF.getSubtarget().getFrameLowering());\n"
+     << "}\n\n";
+
   OS << "} // End llvm namespace\n";
   OS << "#endif // GET_REGINFO_TARGET_DESC\n\n";
 }
diff --git a/utils/TableGen/SubtargetEmitter.cpp b/utils/TableGen/SubtargetEmitter.cpp
index e5c62fa1375..03d7f4e868e 100644
--- a/utils/TableGen/SubtargetEmitter.cpp
+++ b/utils/TableGen/SubtargetEmitter.cpp
@@ -16,6 +16,7 @@
 #include "llvm/ADT/STLExtras.h"
 #include "llvm/ADT/StringExtras.h"
 #include "llvm/MC/MCInstrItineraries.h"
+#include "llvm/MC/SubtargetFeature.h"
 #include "llvm/Support/Debug.h"
 #include "llvm/Support/Format.h"
 #include "llvm/TableGen/Error.h"
@@ -62,7 +63,7 @@ class SubtargetEmitter {
   CodeGenSchedModels &SchedModels;
   std::string Target;
 
-  void Enumeration(raw_ostream &OS, const char *ClassName, bool isBits);
+  void Enumeration(raw_ostream &OS, const char *ClassName);
   unsigned FeatureKeyValues(raw_ostream &OS);
   unsigned CPUKeyValues(raw_ostream &OS);
   void FormItineraryStageString(const std::string &Names,
@@ -112,8 +113,7 @@ class SubtargetEmitter {
 // Enumeration - Emit the specified class as an enumeration.
 //
 void SubtargetEmitter::Enumeration(raw_ostream &OS,
-                                   const char *ClassName,
-                                   bool isBits) {
+                                   const char *ClassName) {
   // Get all records of class and sort
   std::vector<Record*> DefList = Records.getAllDerivedDefinitions(ClassName);
   std::sort(DefList.begin(), DefList.end(), LessRecord());
@@ -121,10 +121,8 @@ void SubtargetEmitter::Enumeration(raw_ostream &OS,
   unsigned N = DefList.size();
   if (N == 0)
     return;
-  if (N > 64) {
-    errs() << "Too many (> 64) subtarget features!\n";
-    exit(1);
-  }
+  if (N > MAX_SUBTARGET_FEATURES)
+    PrintFatalError("Too many subtarget features! Bump MAX_SUBTARGET_FEATURES.");
 
   OS << "namespace " << Target << " {\n";
 
@@ -137,21 +135,14 @@ void SubtargetEmitter::Enumeration(raw_ostream &OS,
     Record *Def = DefList[i];
 
     // Get and emit name
-    OS << "  " << Def->getName();
-
-    // If bit flags then emit expression (1 << i)
-    if (isBits)  OS << " = " << " 1ULL << " << i;
-
-    // Depending on 'if more in the list' emit comma
+    OS << "  " << Def->getName() << " = " << i;
     if (++i < N) OS << ",";
 
     OS << "\n";
   }
 
-  // Close enumeration
-  OS << "};\n";
-
-  OS << "}\n";
+  // Close enumeration and namespace
+  OS << "};\n}\n";
 }
 
 //
@@ -185,22 +176,24 @@ unsigned SubtargetEmitter::FeatureKeyValues(raw_ostream &OS) {
 
     if (CommandLineName.empty()) continue;
 
-    // Emit as { "feature", "description", featureEnum, i1 | i2 | ... | in }
+    // Emit as { "feature", "description", { featureEnum }, { i1 , i2 , ... , in } }
     OS << "  { "
        << "\"" << CommandLineName << "\", "
        << "\"" << Desc << "\", "
-       << Target << "::" << Name << ", ";
+       << "{ " << Target << "::" << Name << " }, ";
 
     const std::vector<Record*> &ImpliesList =
       Feature->getValueAsListOfDefs("Implies");
 
     if (ImpliesList.empty()) {
-      OS << "0ULL";
+      OS << "{ }";
     } else {
+      OS << "{ ";
       for (unsigned j = 0, M = ImpliesList.size(); j < M;) {
         OS << Target << "::" << ImpliesList[j]->getName();
-        if (++j < M) OS << " | ";
+        if (++j < M) OS << ", ";
       }
+      OS << " }";
     }
 
     OS << " }";
@@ -242,22 +235,24 @@ unsigned SubtargetEmitter::CPUKeyValues(raw_ostream &OS) {
     const std::vector<Record*> &FeatureList =
       Processor->getValueAsListOfDefs("Features");
 
-    // Emit as { "cpu", "description", f1 | f2 | ... fn },
+    // Emit as { "cpu", "description", { f1 , f2 , ... fn } },
     OS << "  { "
        << "\"" << Name << "\", "
        << "\"Select the " << Name << " processor\", ";
 
     if (FeatureList.empty()) {
-      OS << "0ULL";
+      OS << "{ }";
     } else {
+      OS << "{ ";
       for (unsigned j = 0, M = FeatureList.size(); j < M;) {
         OS << Target << "::" << FeatureList[j]->getName();
-        if (++j < M) OS << " | ";
+        if (++j < M) OS << ", ";
       }
+      OS << " }";
     }
 
-    // The "0" is for the "implies" section of this data structure.
-    OS << ", 0ULL }";
+    // The { } is for the "implies" section of this data structure.
+    OS << ", { } }";
 
     // Depending on 'if more in the list' emit comma
     if (++i < N) OS << ",";
@@ -1385,7 +1380,7 @@ void SubtargetEmitter::ParseFeaturesFunction(raw_ostream &OS,
   }
 
   OS << "  InitMCProcessorInfo(CPU, FS);\n"
-     << "  uint64_t Bits = getFeatureBits();\n";
+     << "  const FeatureBitset& Bits = getFeatureBits();\n";
 
   for (unsigned i = 0; i < Features.size(); i++) {
     // Next record
@@ -1395,12 +1390,12 @@ void SubtargetEmitter::ParseFeaturesFunction(raw_ostream &OS,
     const std::string &Attribute = R->getValueAsString("Attribute");
 
     if (Value=="true" || Value=="false")
-      OS << "  if ((Bits & " << Target << "::"
-         << Instance << ") != 0) "
+      OS << "  if (Bits[" << Target << "::"
+         << Instance << "]) "
          << Attribute << " = " << Value << ";\n";
     else
-      OS << "  if ((Bits & " << Target << "::"
-         << Instance << ") != 0 && "
+      OS << "  if (Bits[" << Target << "::"
+         << Instance << "] && "
          << Attribute << " < " << Value << ") "
          << Attribute << " = " << Value << ";\n";
   }
@@ -1418,7 +1413,7 @@ void SubtargetEmitter::run(raw_ostream &OS) {
   OS << "#undef GET_SUBTARGETINFO_ENUM\n";
 
   OS << "namespace llvm {\n";
-  Enumeration(OS, "SubtargetFeature", true);
+  Enumeration(OS, "SubtargetFeature");
   OS << "} // End llvm namespace \n";
   OS << "#endif // GET_SUBTARGETINFO_ENUM\n\n";
 
@@ -1440,10 +1435,10 @@ void SubtargetEmitter::run(raw_ostream &OS) {
 #endif
 
   // MCInstrInfo initialization routine.
-  OS << "static inline void Init" << Target
-     << "MCSubtargetInfo(MCSubtargetInfo *II, "
-     << "StringRef TT, StringRef CPU, StringRef FS) {\n";
-  OS << "  II->InitMCSubtargetInfo(TT, CPU, FS, ";
+  OS << "static inline MCSubtargetInfo *create" << Target
+     << "MCSubtargetInfoImpl("
+     << "const Triple &TT, StringRef CPU, StringRef FS) {\n";
+  OS << "  return new MCSubtargetInfo(TT, CPU, FS, ";
   if (NumFeatures)
     OS << Target << "FeatureKV, ";
   else
@@ -1487,10 +1482,11 @@ void SubtargetEmitter::run(raw_ostream &OS) {
   OS << "namespace llvm {\n";
   OS << "class DFAPacketizer;\n";
   OS << "struct " << ClassName << " : public TargetSubtargetInfo {\n"
-     << "  explicit " << ClassName << "(StringRef TT, StringRef CPU, "
+     << "  explicit " << ClassName << "(const Triple &TT, StringRef CPU, "
      << "StringRef FS);\n"
      << "public:\n"
-     << "  unsigned resolveSchedClass(unsigned SchedClass, const MachineInstr *DefMI,"
+     << "  unsigned resolveSchedClass(unsigned SchedClass, "
+     << " const MachineInstr *DefMI,"
      << " const TargetSchedModel *SchedModel) const override;\n"
      << "  DFAPacketizer *createDFAPacketizer(const InstrItineraryData *IID)"
      << " const;\n"
@@ -1520,10 +1516,9 @@ void SubtargetEmitter::run(raw_ostream &OS) {
     OS << "extern const unsigned " << Target << "ForwardingPaths[];\n";
   }
 
-  OS << ClassName << "::" << ClassName << "(StringRef TT, StringRef CPU, "
+  OS << ClassName << "::" << ClassName << "(const Triple &TT, StringRef CPU, "
      << "StringRef FS)\n"
-     << "  : TargetSubtargetInfo() {\n"
-     << "  InitMCSubtargetInfo(TT, CPU, FS, ";
+     << "  : TargetSubtargetInfo(TT, CPU, FS, ";
   if (NumFeatures)
     OS << "makeArrayRef(" << Target << "FeatureKV, " << NumFeatures << "), ";
   else
@@ -1532,19 +1527,19 @@ void SubtargetEmitter::run(raw_ostream &OS) {
     OS << "makeArrayRef(" << Target << "SubTypeKV, " << NumProcs << "), ";
   else
     OS << "None, ";
-  OS << '\n'; OS.indent(22);
+  OS << '\n'; OS.indent(24);
   OS << Target << "ProcSchedKV, "
      << Target << "WriteProcResTable, "
      << Target << "WriteLatencyTable, "
      << Target << "ReadAdvanceTable, ";
-  OS << '\n'; OS.indent(22);
+  OS << '\n'; OS.indent(24);
   if (SchedModels.hasItineraries()) {
     OS << Target << "Stages, "
        << Target << "OperandCycles, "
        << Target << "ForwardingPaths";
   } else
     OS << "0, 0, 0";
-  OS << ");\n}\n\n";
+  OS << ") {}\n\n";
 
   EmitSchedModelHelpers(ClassName, OS);
 
diff --git a/utils/TableGen/X86DisassemblerTables.cpp b/utils/TableGen/X86DisassemblerTables.cpp
index 0f3ea9a019d..ad36dc427a5 100644
--- a/utils/TableGen/X86DisassemblerTables.cpp
+++ b/utils/TableGen/X86DisassemblerTables.cpp
@@ -243,6 +243,9 @@ static inline bool inheritsFrom(InstructionContext child,
   case IC_EVEX_OPSIZE_KZ_B:
     return false;
   case IC_EVEX_W_K:
+  case IC_EVEX_W_B:
+  case IC_EVEX_W_K_B:
+  case IC_EVEX_W_KZ_B:
   case IC_EVEX_W_XS_K:
   case IC_EVEX_W_XD_K:
   case IC_EVEX_W_OPSIZE_K:
@@ -252,6 +255,8 @@ static inline bool inheritsFrom(InstructionContext child,
   case IC_EVEX_L_K:
   case IC_EVEX_L_XS_K:
   case IC_EVEX_L_XD_K:
+  case IC_EVEX_L_XD_B:
+  case IC_EVEX_L_XD_K_B:
   case IC_EVEX_L_OPSIZE_K:
   case IC_EVEX_L_OPSIZE_B:
   case IC_EVEX_L_OPSIZE_K_B:
@@ -270,19 +275,32 @@ static inline bool inheritsFrom(InstructionContext child,
     return false;
   case IC_EVEX_L_KZ:
   case IC_EVEX_L_XS_KZ:
+  case IC_EVEX_L_XS_B:
+  case IC_EVEX_L_XS_K_B:
+  case IC_EVEX_L_XS_KZ_B:
   case IC_EVEX_L_XD_KZ:
+  case IC_EVEX_L_XD_KZ_B:
   case IC_EVEX_L_OPSIZE_KZ:
   case IC_EVEX_L_OPSIZE_KZ_B:
     return false;
   case IC_EVEX_L_W_K:
+  case IC_EVEX_L_W_B:
+  case IC_EVEX_L_W_K_B:    
   case IC_EVEX_L_W_XS_K:
-  case IC_EVEX_L_W_XD_K:
+  case IC_EVEX_L_W_XS_B:
+  case IC_EVEX_L_W_XS_K_B:
+  case IC_EVEX_L_W_XS_KZ:
+  case IC_EVEX_L_W_XS_KZ_B:
   case IC_EVEX_L_W_OPSIZE_K:
   case IC_EVEX_L_W_OPSIZE_B:
   case IC_EVEX_L_W_OPSIZE_K_B:
   case IC_EVEX_L_W_KZ:
-  case IC_EVEX_L_W_XS_KZ:
+  case IC_EVEX_L_W_KZ_B:
+  case IC_EVEX_L_W_XD_K:
+  case IC_EVEX_L_W_XD_B:
+  case IC_EVEX_L_W_XD_K_B:
   case IC_EVEX_L_W_XD_KZ:
+  case IC_EVEX_L_W_XD_KZ_B:
   case IC_EVEX_L_W_OPSIZE_KZ:
   case IC_EVEX_L_W_OPSIZE_KZ_B:
     return false;
@@ -291,21 +309,29 @@ static inline bool inheritsFrom(InstructionContext child,
   case IC_EVEX_L2_K_B:
   case IC_EVEX_L2_KZ_B:
   case IC_EVEX_L2_XS_K:
+  case IC_EVEX_L2_XS_K_B:
   case IC_EVEX_L2_XS_B:
   case IC_EVEX_L2_XD_B:
   case IC_EVEX_L2_XD_K:
+  case IC_EVEX_L2_XD_K_B:
   case IC_EVEX_L2_OPSIZE_K:
   case IC_EVEX_L2_OPSIZE_B:
   case IC_EVEX_L2_OPSIZE_K_B:
   case IC_EVEX_L2_KZ:
   case IC_EVEX_L2_XS_KZ:
+  case IC_EVEX_L2_XS_KZ_B:
   case IC_EVEX_L2_XD_KZ:
+  case IC_EVEX_L2_XD_KZ_B:
   case IC_EVEX_L2_OPSIZE_KZ:
   case IC_EVEX_L2_OPSIZE_KZ_B:
     return false;
   case IC_EVEX_L2_W_K:
   case IC_EVEX_L2_W_B:
+  case IC_EVEX_L2_W_K_B:
+  case IC_EVEX_L2_W_KZ_B:
   case IC_EVEX_L2_W_XS_K:
+  case IC_EVEX_L2_W_XS_B:
+  case IC_EVEX_L2_W_XS_K_B:
   case IC_EVEX_L2_W_XD_K:
   case IC_EVEX_L2_W_XD_B:
   case IC_EVEX_L2_W_OPSIZE_K:
@@ -313,7 +339,10 @@ static inline bool inheritsFrom(InstructionContext child,
   case IC_EVEX_L2_W_OPSIZE_K_B:
   case IC_EVEX_L2_W_KZ:
   case IC_EVEX_L2_W_XS_KZ:
+  case IC_EVEX_L2_W_XS_KZ_B:
   case IC_EVEX_L2_W_XD_KZ:
+  case IC_EVEX_L2_W_XD_K_B:
+  case IC_EVEX_L2_W_XD_KZ_B:
   case IC_EVEX_L2_W_OPSIZE_KZ:
   case IC_EVEX_L2_W_OPSIZE_KZ_B:
     return false;
diff --git a/utils/TableGen/X86RecognizableInstr.cpp b/utils/TableGen/X86RecognizableInstr.cpp
index 5f12965c27f..efcb0c81e1c 100644
--- a/utils/TableGen/X86RecognizableInstr.cpp
+++ b/utils/TableGen/X86RecognizableInstr.cpp
@@ -1027,12 +1027,16 @@ OperandType RecognizableInstr::typeFromString(const std::string &s,
   TYPE("GR32_NOAX",           TYPE_Rv)
   TYPE("GR64_NOAX",           TYPE_R64)
   TYPE("vx32mem",             TYPE_M32)
+  TYPE("vx32xmem",            TYPE_M32)
   TYPE("vy32mem",             TYPE_M32)
+  TYPE("vy32xmem",            TYPE_M32)
   TYPE("vz32mem",             TYPE_M32)
   TYPE("vx64mem",             TYPE_M64)
+  TYPE("vx64xmem",            TYPE_M64)
   TYPE("vy64mem",             TYPE_M64)
   TYPE("vy64xmem",            TYPE_M64)
   TYPE("vz64mem",             TYPE_M64)
+  TYPE("BNDR",                TYPE_BNDR)
   errs() << "Unhandled type string " << s << "\n";
   llvm_unreachable("Unhandled type string");
 }
@@ -1096,10 +1100,13 @@ RecognizableInstr::rmRegisterEncodingFromString(const std::string &s,
   ENCODING("VR256X",          ENCODING_RM)
   ENCODING("VR512",           ENCODING_RM)
   ENCODING("VK1",             ENCODING_RM)
+  ENCODING("VK2",             ENCODING_RM)
+  ENCODING("VK4",             ENCODING_RM)
   ENCODING("VK8",             ENCODING_RM)
   ENCODING("VK16",            ENCODING_RM)
   ENCODING("VK32",            ENCODING_RM)
   ENCODING("VK64",            ENCODING_RM)
+  ENCODING("BNDR",            ENCODING_RM)
   errs() << "Unhandled R/M register encoding " << s << "\n";
   llvm_unreachable("Unhandled R/M register encoding");
 }
@@ -1133,8 +1140,13 @@ RecognizableInstr::roRegisterEncodingFromString(const std::string &s,
   ENCODING("VK32",            ENCODING_REG)
   ENCODING("VK64",            ENCODING_REG)
   ENCODING("VK1WM",           ENCODING_REG)
+  ENCODING("VK2WM",           ENCODING_REG)
+  ENCODING("VK4WM",           ENCODING_REG)
   ENCODING("VK8WM",           ENCODING_REG)
   ENCODING("VK16WM",          ENCODING_REG)
+  ENCODING("VK32WM",          ENCODING_REG)
+  ENCODING("VK64WM",          ENCODING_REG)
+  ENCODING("BNDR",            ENCODING_REG)
   errs() << "Unhandled reg/opcode register encoding " << s << "\n";
   llvm_unreachable("Unhandled reg/opcode register encoding");
 }
@@ -1204,9 +1216,12 @@ RecognizableInstr::memoryEncodingFromString(const std::string &s,
   ENCODING("opaque80mem",     ENCODING_RM)
   ENCODING("opaque512mem",    ENCODING_RM)
   ENCODING("vx32mem",         ENCODING_RM)
+  ENCODING("vx32xmem",        ENCODING_RM)
   ENCODING("vy32mem",         ENCODING_RM)
+  ENCODING("vy32xmem",        ENCODING_RM)
   ENCODING("vz32mem",         ENCODING_RM)
   ENCODING("vx64mem",         ENCODING_RM)
+  ENCODING("vx64xmem",        ENCODING_RM)
   ENCODING("vy64mem",         ENCODING_RM)
   ENCODING("vy64xmem",        ENCODING_RM)
   ENCODING("vz64mem",         ENCODING_RM)
diff --git a/utils/git-svn/git-svnrevert b/utils/git-svn/git-svnrevert
index f15e7abfb3f..4185ee7ae87 100755
--- a/utils/git-svn/git-svnrevert
+++ b/utils/git-svn/git-svnrevert
@@ -14,8 +14,13 @@ fi
 
 COMMIT=$1
 OTHER=$(git svn find-rev "$COMMIT")
-if [ $? -ne 0 ]; then
+if [ $? -ne 0 ] || [ "$OTHER" = "" ]; then
     echo "Error! Could not find an svn/git revision for commit $COMMIT!"
+    echo
+    echo "Possible problems are:"
+    echo "  * Your revision number ($COMMIT) is wrong"
+    echo "  * This tree is not up to date (before that commit)"
+    echo "  * This commit in in another three (llvm, clang, compiler-rt, etc)"
     exit 1
 fi
 
diff --git a/utils/lit/lit/TestRunner.py b/utils/lit/lit/TestRunner.py
index f1734eca372..a4902874325 100644
--- a/utils/lit/lit/TestRunner.py
+++ b/utils/lit/lit/TestRunner.py
@@ -31,7 +31,7 @@ class ShellEnvironment(object):
 
     def __init__(self, cwd, env):
         self.cwd = cwd
-        self.env = env
+        self.env = dict(env)
 
 def executeShCmd(cmd, shenv, results):
     if isinstance(cmd, ShUtil.Seq):
@@ -62,14 +62,18 @@ def executeShCmd(cmd, shenv, results):
 
     # Handle shell builtins first.
     if cmd.commands[0].args[0] == 'cd':
-        # Update the cwd in the environment.
+        if len(cmd.commands) != 1:
+            raise ValueError("'cd' cannot be part of a pipeline")
         if len(cmd.commands[0].args) != 2:
-            raise ValueError('cd supports only one argument')
+            raise ValueError("'cd' supports only one argument")
         newdir = cmd.commands[0].args[1]
+        # Update the cwd in the parent environment.
         if os.path.isabs(newdir):
             shenv.cwd = newdir
         else:
             shenv.cwd = os.path.join(shenv.cwd, newdir)
+        # The cd builtin always succeeds. If the directory does not exist, the
+        # following Popen calls will fail instead.
         return 0
 
     procs = []
@@ -81,6 +85,23 @@ def executeShCmd(cmd, shenv, results):
     # output. This is null until we have seen some output using
     # stderr.
     for i,j in enumerate(cmd.commands):
+        # Reference the global environment by default.
+        cmd_shenv = shenv
+        if j.args[0] == 'env':
+            # Create a copy of the global environment and modify it for this one
+            # command. There might be multiple envs in a pipeline:
+            #   env FOO=1 llc < %s | env BAR=2 llvm-mc | FileCheck %s
+            cmd_shenv = ShellEnvironment(shenv.cwd, shenv.env)
+            arg_idx = 1
+            for arg_idx, arg in enumerate(j.args[1:]):
+                # Partition the string into KEY=VALUE.
+                key, eq, val = arg.partition('=')
+                # Stop if there was no equals.
+                if eq == '':
+                    break
+                cmd_shenv.env[key] = val
+            j.args = j.args[arg_idx+1:]
+
         # Apply the redirections, we use (N,) as a sentinel to indicate stdin,
         # stdout, stderr for N equal to 0, 1, or 2 respectively. Redirects to or
         # from a file are represented with a list [file, mode, file-object]
@@ -126,7 +147,7 @@ def executeShCmd(cmd, shenv, results):
                         r[2] = tempfile.TemporaryFile(mode=r[1])
                     else:
                         # Make sure relative paths are relative to the cwd.
-                        redir_filename = os.path.join(shenv.cwd, r[0])
+                        redir_filename = os.path.join(cmd_shenv.cwd, r[0])
                         r[2] = open(redir_filename, r[1])
                     # Workaround a Win32 and/or subprocess bug when appending.
                     #
@@ -157,7 +178,14 @@ def executeShCmd(cmd, shenv, results):
 
         # Resolve the executable path ourselves.
         args = list(j.args)
-        executable = lit.util.which(args[0], shenv.env['PATH'])
+        executable = None
+        # For paths relative to cwd, use the cwd of the shell environment.
+        if args[0].startswith('.'):
+            exe_in_cwd = os.path.join(cmd_shenv.cwd, args[0])
+            if os.path.isfile(exe_in_cwd):
+                executable = exe_in_cwd
+        if not executable:
+            executable = lit.util.which(args[0], cmd_shenv.env['PATH'])
         if not executable:
             raise InternalShellError(j, '%r: command not found' % j.args[0])
 
@@ -171,12 +199,12 @@ def executeShCmd(cmd, shenv, results):
                     args[i] = f.name
 
         try:
-            procs.append(subprocess.Popen(args, cwd=shenv.cwd,
+            procs.append(subprocess.Popen(args, cwd=cmd_shenv.cwd,
                                           executable = executable,
                                           stdin = stdin,
                                           stdout = stdout,
                                           stderr = stderr,
-                                          env = shenv.env,
+                                          env = cmd_shenv.env,
                                           close_fds = kUseCloseFDs))
         except OSError as e:
             raise InternalShellError(j, 'Could not create process due to {}'.format(e))
@@ -513,6 +541,15 @@ def processLine(ln):
         return lit.Test.Result(Test.UNSUPPORTED,
                     "Test is unsupported with the following features: %s" % msg)
 
+    if test.config.limit_to_features:
+        # Check that we have one of the limit_to_features features in requires.
+        limit_to_features_tests = [f for f in test.config.limit_to_features
+                                   if f in requires]
+        if not limit_to_features_tests:
+            msg = ', '.join(test.config.limit_to_features)
+            return lit.Test.Result(Test.UNSUPPORTED,
+                 "Test requires one of the limit_to_features features %s" % msg)
+
     return script,tmpBase,execdir
 
 def _runShTest(test, litConfig, useExternalSh,
diff --git a/utils/lit/lit/TestingConfig.py b/utils/lit/lit/TestingConfig.py
index c7ef94dc11f..d49d0c0767f 100644
--- a/utils/lit/lit/TestingConfig.py
+++ b/utils/lit/lit/TestingConfig.py
@@ -24,7 +24,7 @@ def fromdefaults(litConfig):
 
         pass_vars = ['LIBRARY_PATH', 'LD_LIBRARY_PATH', 'SYSTEMROOT', 'TERM',
                      'LD_PRELOAD', 'ASAN_OPTIONS', 'UBSAN_OPTIONS',
-                     'LSAN_OPTIONS']
+                     'LSAN_OPTIONS', 'ADB', 'ADB_SERIAL']
         for var in pass_vars:
             val = os.environ.get(var, '')
             # Check for empty string as some variables such as LD_PRELOAD cannot be empty
@@ -118,7 +118,7 @@ def load_from_path(self, path, litConfig):
     def __init__(self, parent, name, suffixes, test_format,
                  environment, substitutions, unsupported,
                  test_exec_root, test_source_root, excludes,
-                 available_features, pipefail):
+                 available_features, pipefail, limit_to_features = []):
         self.parent = parent
         self.name = str(name)
         self.suffixes = set(suffixes)
@@ -131,6 +131,10 @@ def __init__(self, parent, name, suffixes, test_format,
         self.excludes = set(excludes)
         self.available_features = set(available_features)
         self.pipefail = pipefail
+        # This list is used by TestRunner.py to restrict running only tests that
+        # require one of the features in this list if this list is non-empty.
+        # Configurations can set this list to restrict the set of tests to run.
+        self.limit_to_features = set(limit_to_features)
 
     def finish(self, litConfig):
         """finish() - Finish this config object, after loading is complete."""
diff --git a/utils/lit/lit/main.py b/utils/lit/lit/main.py
index f2aedc906bb..e3722674f63 100755
--- a/utils/lit/lit/main.py
+++ b/utils/lit/lit/main.py
@@ -146,7 +146,7 @@ def main(builtinParameters = {}):
     parser.add_option("", "--config-prefix", dest="configPrefix",
                       metavar="NAME", help="Prefix for 'lit' config files",
                       action="store", default=None)
-    parser.add_option("", "--param", dest="userParameters",
+    parser.add_option("-D", "--param", dest="userParameters",
                       metavar="NAME=VAL",
                       help="Add 'NAME' = 'VAL' to the user defined parameters",
                       type=str, action="append", default=[])
diff --git a/utils/lit/lit/util.py b/utils/lit/lit/util.py
index 08f7b71ae21..a4233bac7aa 100644
--- a/utils/lit/lit/util.py
+++ b/utils/lit/lit/util.py
@@ -48,7 +48,7 @@ def mkdir_p(path):
     if not path or os.path.exists(path):
         return
 
-    parent = os.path.dirname(path) 
+    parent = os.path.dirname(path)
     if parent != path:
         mkdir_p(parent)
 
@@ -158,13 +158,13 @@ def printHistogram(items, title = 'Items'):
 # Close extra file handles on UNIX (on Windows this cannot be done while
 # also redirecting input).
 kUseCloseFDs = not (platform.system() == 'Windows')
-def executeCommand(command, cwd=None, env=None):
+def executeCommand(command, cwd=None, env=None, input=None):
     p = subprocess.Popen(command, cwd=cwd,
                          stdin=subprocess.PIPE,
                          stdout=subprocess.PIPE,
                          stderr=subprocess.PIPE,
                          env=env, close_fds=kUseCloseFDs)
-    out,err = p.communicate()
+    out,err = p.communicate(input=input)
     exitCode = p.wait()
 
     # Detect Ctrl-C in subprocess.
diff --git a/utils/lit/tests/Inputs/googletest-format/DummySubDir/OneTest b/utils/lit/tests/Inputs/googletest-format/DummySubDir/OneTest
index 9dff137f4de..dd49f025b1f 100755
--- a/utils/lit/tests/Inputs/googletest-format/DummySubDir/OneTest
+++ b/utils/lit/tests/Inputs/googletest-format/DummySubDir/OneTest
@@ -21,6 +21,7 @@ elif not sys.argv[1].startswith("--gtest_filter="):
 test_name = sys.argv[1].split('=',1)[1]
 if test_name == 'FirstTest.subTestA':
     print('I am subTest A, I PASS')
+    print('[  PASSED  ] 1 test.')
     sys.exit(0)
 elif test_name == 'FirstTest.subTestB':
     print('I am subTest B, I FAIL')
@@ -29,6 +30,7 @@ elif test_name == 'FirstTest.subTestB':
 elif test_name in ('ParameterizedTest/0.subTest',
                    'ParameterizedTest/1.subTest'):
     print('I am a parameterized test, I also PASS')
+    print('[  PASSED  ] 1 test.')
     sys.exit(0)
 else:
     raise SystemExit("error: invalid test name: %r" % (test_name,))
diff --git a/utils/lit/tests/discovery.py b/utils/lit/tests/discovery.py
index 28010894cda..55e54088b58 100644
--- a/utils/lit/tests/discovery.py
+++ b/utils/lit/tests/discovery.py
@@ -7,8 +7,8 @@
 # RUN: FileCheck --check-prefix=CHECK-BASIC-ERR < %t.err %s
 #
 # CHECK-BASIC-ERR: loading suite config '{{.*}}/discovery/lit.cfg'
-# CHECK-BASIC-ERR: loading local config '{{.*}}/discovery/subdir/lit.local.cfg'
-# CHECK-BASIC-ERR: loading suite config '{{.*}}/discovery/subsuite/lit.cfg'
+# CHECK-BASIC-ERR-DAG: loading suite config '{{.*}}/discovery/subsuite/lit.cfg'
+# CHECK-BASIC-ERR-DAG: loading local config '{{.*}}/discovery/subdir/lit.local.cfg'
 #
 # CHECK-BASIC-OUT: -- Test Suites --
 # CHECK-BASIC-OUT:   sub-suite - 2 tests
@@ -51,8 +51,8 @@
 # CHECK-ASEXEC-ERR: load_config from '{{.*}}/discovery/lit.cfg'
 # CHECK-ASEXEC-ERR: loaded config '{{.*}}/discovery/lit.cfg'
 # CHECK-ASEXEC-ERR: loaded config '{{.*}}/exec-discovery/lit.site.cfg'
-# CHECK-ASEXEC-ERR: loading local config '{{.*}}/discovery/subdir/lit.local.cfg'
-# CHECK-ASEXEC-ERR: loading suite config '{{.*}}/discovery/subsuite/lit.cfg'
+# CHECK-ASEXEC-ERR-DAG: loading suite config '{{.*}}/discovery/subsuite/lit.cfg'
+# CHECK-ASEXEC-ERR-DAG: loading local config '{{.*}}/discovery/subdir/lit.local.cfg'
 #
 # CHECK-ASEXEC-OUT: -- Test Suites --
 # CHECK-ASEXEC-OUT:   sub-suite - 2 tests
diff --git a/utils/lit/tests/unittest-adaptor.py b/utils/lit/tests/unittest-adaptor.py
index 7435dda4196..0848cd22baa 100644
--- a/utils/lit/tests/unittest-adaptor.py
+++ b/utils/lit/tests/unittest-adaptor.py
@@ -3,8 +3,8 @@
 # RUN: %{python} %s %{inputs}/unittest-adaptor 2> %t.err
 # RUN: FileCheck < %t.err %s
 #
-# CHECK: unittest-adaptor :: test-one.txt ... ok
-# CHECK: unittest-adaptor :: test-two.txt ... FAIL
+# CHECK-DAG: unittest-adaptor :: test-two.txt ... FAIL
+# CHECK-DAG: unittest-adaptor :: test-one.txt ... ok
 
 import unittest
 import sys
diff --git a/utils/lit/tests/xunit-output.py b/utils/lit/tests/xunit-output.py
index 87652290f47..3f493953637 100644
--- a/utils/lit/tests/xunit-output.py
+++ b/utils/lit/tests/xunit-output.py
@@ -5,6 +5,6 @@
 # CHECK: <?xml version="1.0" encoding="UTF-8" ?>
 # CHECK: <testsuites>
 # CHECK: <testsuite name='test-data' tests='1' failures='0'>
-# CHECK: <testcase classname='test-data.' name='metrics.ini' time='0.00'/>
+# CHECK: <testcase classname='test-data.test-data' name='metrics.ini' time='0.{{[0-9]+}}'/>
 # CHECK: </testsuite>
-# CHECK: </testsuites>
\ No newline at end of file
+# CHECK: </testsuites>
diff --git a/utils/release/export.sh b/utils/release/export.sh
index 4da5fe40950..2fd4206b740 100755
--- a/utils/release/export.sh
+++ b/utils/release/export.sh
@@ -14,7 +14,7 @@
 
 set -e
 
-projects="llvm cfe dragonegg test-suite compiler-rt libcxx libcxxabi clang-tools-extra polly lldb lld openmp"
+projects="llvm cfe test-suite compiler-rt libcxx libcxxabi clang-tools-extra polly lldb lld openmp libunwind"
 base_url="https://llvm.org/svn/llvm-project"
 
 release=""
diff --git a/utils/release/tag.sh b/utils/release/tag.sh
index 503413b9c44..caefc7f7b71 100755
--- a/utils/release/tag.sh
+++ b/utils/release/tag.sh
@@ -17,7 +17,7 @@ set -e
 release=""
 rc=""
 rebranch="no"
-projects="llvm cfe dragonegg test-suite compiler-rt libcxx libcxxabi clang-tools-extra polly lldb lld openmp"
+projects="llvm cfe test-suite compiler-rt libcxx libcxxabi clang-tools-extra polly lldb lld openmp libunwind"
 dryrun=""
 revision="HEAD"
 
diff --git a/utils/release/test-release.sh b/utils/release/test-release.sh
index 20f8d97ebc0..a10e4fd9a16 100755
--- a/utils/release/test-release.sh
+++ b/utils/release/test-release.sh
@@ -18,8 +18,6 @@ else
     MAKE=make
 fi
 
-projects="llvm cfe dragonegg compiler-rt libcxx libcxxabi test-suite clang-tools-extra"
-
 # Base SVN URL for the sources.
 Base_url="http://llvm.org/svn/llvm-project"
 
@@ -29,42 +27,49 @@ RC=""
 Triple=""
 use_gzip="no"
 do_checkout="yes"
-do_ada="no"
-do_clang="yes"
-do_dragonegg="no"
-do_fortran="no"
-do_objc="yes"
-do_64bit="yes"
 do_debug="no"
 do_asserts="no"
 do_compare="yes"
+do_rt="yes"
+do_libs="yes"
+do_libunwind="yes"
+do_test_suite="yes"
+do_openmp="no"
 BuildDir="`pwd`"
-BuildTriple=""
+use_autoconf="no"
+ExtraConfigureFlags=""
+ExportBranch=""
 
 function usage() {
-    echo "usage: `basename $0` -release X.Y -rc NUM [OPTIONS]"
+    echo "usage: `basename $0` -release X.Y.Z -rc NUM [OPTIONS]"
     echo ""
-    echo " -release X.Y         The release number to test."
+    echo " -release X.Y.Z       The release version to test."
     echo " -rc NUM              The pre-release candidate number."
     echo " -final               The final release candidate."
     echo " -triple TRIPLE       The target triple for this machine."
     echo " -j NUM               Number of compile jobs to run. [default: 3]"
     echo " -build-dir DIR       Directory to perform testing in. [default: pwd]"
     echo " -no-checkout         Don't checkout the sources from SVN."
-    echo " -no-64bit            Don't test the 64-bit version. [default: yes]"
-    echo " -enable-ada          Build Ada. [default: disable]"
-    echo " -disable-clang       Do not test clang. [default: enable]"
-    echo " -enable-dragonegg    Test dragonegg. [default: disable]"
-    echo " -enable-fortran      Enable Fortran build. [default: disable]"
-    echo " -disable-objc        Disable ObjC build. [default: enable]"
     echo " -test-debug          Test the debug build. [default: no]"
     echo " -test-asserts        Test with asserts on. [default: no]"
     echo " -no-compare-files    Don't test that phase 2 and 3 files are identical."
     echo " -use-gzip            Use gzip instead of xz."
-    echo " -build-triple TRIPLE The build triple for this machine"
-    echo "                      [default: use config.guess]"
+    echo " -configure-flags FLAGS  Extra flags to pass to the configure step."
+    echo " -use-autoconf        Use autoconf instead of cmake"
+    echo " -svn-path DIR        Use the specified DIR instead of a release."
+    echo "                      For example -svn-path trunk or -svn-path branches/release_37"
+    echo " -no-rt               Disable check-out & build Compiler-RT"
+    echo " -no-libs             Disable check-out & build libcxx/libcxxabi/libunwind"
+    echo " -no-libunwind        Disable check-out & build libunwind"
+    echo " -no-test-suite       Disable check-out & build test-suite"
+    echo " -openmp              Check out and build the OpenMP run-time (experimental)"
 }
 
+if [ `uname -s` = "Darwin" ]; then
+  # compiler-rt doesn't yet build with CMake on Darwin.
+  use_autoconf="yes"
+fi
+
 while [ $# -gt 0 ]; do
     case $1 in
         -release | --release )
@@ -79,13 +84,23 @@ while [ $# -gt 0 ]; do
         -final | --final )
             RC=final
             ;;
+        -svn-path | --svn-path )
+            shift
+            Release="test"
+            Release_no_dot="test"
+            ExportBranch="$1"
+            RC="`echo $ExportBranch | sed -e 's,/,_,g'`"
+            echo "WARNING: Using the branch $ExportBranch instead of a release tag"
+            echo "         This is intended to aid new packagers in trialing "
+            echo "         builds without requiring a tag to be created first"
+            ;;
         -triple | --triple )
             shift
             Triple="$1"
             ;;
-        -build-triple | --build-triple )
+        -configure-flags | --configure-flags )
             shift
-            BuildTriple="$1"
+            ExtraConfigureFlags="$1"
             ;;
         -j* )
             NumJobs="`echo $1 | sed -e 's,-j\([0-9]*\),\1,g'`"
@@ -101,24 +116,6 @@ while [ $# -gt 0 ]; do
         -no-checkout | --no-checkout )
             do_checkout="no"
             ;;
-        -no-64bit | --no-64bit )
-            do_64bit="no"
-            ;;
-        -enable-ada | --enable-ada )
-            do_ada="yes"
-            ;;
-        -disable-clang | --disable-clang )
-            do_clang="no"
-            ;;
-        -enable-dragonegg | --enable-dragonegg )
-            do_dragonegg="yes"
-            ;;
-        -enable-fortran | --enable-fortran )
-            do_fortran="yes"
-            ;;
-        -disable-objc | --disable-objc )
-            do_objc="no"
-            ;;
         -test-debug | --test-debug )
             do_debug="yes"
             ;;
@@ -131,6 +128,24 @@ while [ $# -gt 0 ]; do
         -use-gzip | --use-gzip )
             use_gzip="yes"
             ;;
+        -use-autoconf | --use-autoconf )
+            use_autoconf="yes"
+            ;;
+        -no-rt )
+            do_rt="no"
+            ;;
+        -no-libs )
+            do_libs="no"
+            ;;
+        -no-libunwind )
+            do_libunwind="no"
+            ;;
+        -no-test-suite )
+            do_test_suite="no"
+            ;;
+        -openmp )
+            do_openmp="yes"
+            ;;
         -help | --help | -h | --h | -\? )
             usage
             exit 0
@@ -153,6 +168,9 @@ if [ -z "$RC" ]; then
     echo "error: no release candidate number specified"
     exit 1
 fi
+if [ -z "$ExportBranch" ]; then
+    ExportBranch="tags/RELEASE_$Release_no_dot/$RC"
+fi
 if [ -z "$Triple" ]; then
     echo "error: no target triple specified"
     exit 1
@@ -172,6 +190,24 @@ if [ -z "$NumJobs" ]; then
     NumJobs=3
 fi
 
+# Projects list
+projects="llvm cfe clang-tools-extra"
+if [ $do_rt = "yes" ]; then
+  projects="$projects compiler-rt"
+fi
+if [ $do_libs = "yes" ]; then
+  projects="$projects libcxx libcxxabi"
+  if [ $do_libunwind = "yes" ]; then
+    projects="$projects libunwind"
+  fi
+fi
+if [ $do_test_suite = "yes" ]; then
+  projects="$projects test-suite"
+fi
+if [ $do_openmp = "yes" ]; then
+  projects="$projects openmp"
+fi
+
 # Go to the build directory (may be different from CWD)
 BuildDir=$BuildDir/$RC
 mkdir -p $BuildDir
@@ -188,26 +224,15 @@ if [ $RC != "final" ]; then
 fi
 Package=$Package-$Triple
 
-# Find compilers.
-if [ "$do_dragonegg" = "yes" ]; then
-    gcc_compiler="$GCC"
-    if [ -z "$gcc_compiler" ]; then
-        gcc_compiler="`which gcc`"
-        if [ -z "$gcc_compiler" ]; then
-            echo "error: cannot find gcc to use with dragonegg"
-            exit 1
-        fi
-    fi
+# Errors to be highlighted at the end are written to this file.
+echo -n > $LogDir/deferred_errors.log
 
-    gxx_compiler="$GXX"
-    if [ -z "$gxx_compiler" ]; then
-        gxx_compiler="`which g++`"
-        if [ -z "$gxx_compiler" ]; then
-            echo "error: cannot find g++ to use with dragonegg"
-            exit 1
-        fi
-    fi
-fi
+function deferred_error() {
+  Phase="$1"
+  Flavor="$2"
+  Msg="$3"
+  echo "[${Flavor} Phase${Phase}] ${Msg}" | tee -a $LogDir/deferred_errors.log
+}
 
 # Make sure that a required program is available
 function check_program_exists() {
@@ -229,8 +254,8 @@ function check_valid_urls() {
     for proj in $projects ; do
         echo "# Validating $proj SVN URL"
 
-        if ! svn ls $Base_url/$proj/tags/RELEASE_$Release_no_dot/$RC > /dev/null 2>&1 ; then
-            echo "$proj $Release release candidate $RC doesn't exist!"
+        if ! svn ls $Base_url/$proj/$ExportBranch > /dev/null 2>&1 ; then
+            echo "$proj does not have a $ExportBranch branch/tag!"
             exit 1
         fi
     done
@@ -241,8 +266,12 @@ function export_sources() {
     check_valid_urls
 
     for proj in $projects ; do
+        if [ -d $proj.src ]; then
+          echo "# Reusing $proj $Release-$RC sources"
+          continue
+        fi
         echo "# Exporting $proj $Release-$RC sources"
-        if ! svn export -q $Base_url/$proj/tags/RELEASE_$Release_no_dot/$RC $proj.src ; then
+        if ! svn export -q $Base_url/$proj/$ExportBranch $proj.src ; then
             echo "error: failed to export $proj project"
             exit 1
         fi
@@ -258,18 +287,22 @@ function export_sources() {
         ln -s ../../../../clang-tools-extra.src extra
     fi
     cd $BuildDir/llvm.src/projects
-    if [ ! -h test-suite ]; then
+    if [ -d $BuildDir/test-suite.src ] && [ ! -h test-suite ]; then
         ln -s ../../test-suite.src test-suite
     fi
-    if [ ! -h compiler-rt ]; then
+    if [ -d $BuildDir/compiler-rt.src ] && [ ! -h compiler-rt ]; then
         ln -s ../../compiler-rt.src compiler-rt
     fi
-    if [ ! -h libcxx ]; then
+    if [ -d $BuildDir/libcxx.src ] && [ ! -h libcxx ]; then
         ln -s ../../libcxx.src libcxx
     fi
-    if [ ! -h libcxxabi ]; then
+    if [ -d $BuildDir/libcxxabi.src ] && [ ! -h libcxxabi ]; then
         ln -s ../../libcxxabi.src libcxxabi
     fi
+    if [ -d $BuildDir/libunwind.src ] && [ ! -h libunwind ]; then
+        ln -s ../../libunwind.src libunwind
+    fi
+
     cd $BuildDir
 }
 
@@ -277,20 +310,22 @@ function configure_llvmCore() {
     Phase="$1"
     Flavor="$2"
     ObjDir="$3"
-    InstallDir="$4"
 
     case $Flavor in
-        Release | Release-64 )
-            Optimized="yes"
-            Assertions="no"
+        Release )
+            BuildType="Release"
+            Assertions="OFF"
+            ConfigureFlags="--enable-optimized --disable-assertions"
             ;;
         Release+Asserts )
-            Optimized="yes"
-            Assertions="yes"
+            BuildType="Release"
+            Assertions="ON"
+            ConfigureFlags="--enable-optimized --enable-assertions"
             ;;
         Debug )
-            Optimized="no"
-            Assertions="yes"
+            BuildType="Debug"
+            Assertions="ON"
+            ConfigureFlags="--disable-optimized --enable-assertions"
             ;;
         * )
             echo "# Invalid flavor '$Flavor'"
@@ -302,22 +337,33 @@ function configure_llvmCore() {
     echo "# Using C compiler: $c_compiler"
     echo "# Using C++ compiler: $cxx_compiler"
 
-    build_triple_option="${BuildTriple:+--build=$BuildTriple}"
-
     cd $ObjDir
     echo "# Configuring llvm $Release-$RC $Flavor"
-    echo "# $BuildDir/llvm.src/configure --prefix=$InstallDir \
-        --enable-optimized=$Optimized \
-        --enable-assertions=$Assertions \
-        --disable-timestamps \
-        $build_triple_option"
-    env CC="$c_compiler" CXX="$cxx_compiler" \
-    $BuildDir/llvm.src/configure --prefix=$InstallDir \
-        --enable-optimized=$Optimized \
-        --enable-assertions=$Assertions \
-        --disable-timestamps \
-        $build_triple_option \
-        2>&1 | tee $LogDir/llvm.configure-Phase$Phase-$Flavor.log
+
+    if [ "$use_autoconf" = "yes" ]; then
+        echo "#" env CC="$c_compiler" CXX="$cxx_compiler" \
+            $BuildDir/llvm.src/configure \
+            $ConfigureFlags --disable-timestamps $ExtraConfigureFlags \
+            2>&1 | tee $LogDir/llvm.configure-Phase$Phase-$Flavor.log
+        env CC="$c_compiler" CXX="$cxx_compiler" \
+            $BuildDir/llvm.src/configure \
+            $ConfigureFlags --disable-timestamps $ExtraConfigureFlags \
+            2>&1 | tee $LogDir/llvm.configure-Phase$Phase-$Flavor.log
+    else
+        echo "#" env CC="$c_compiler" CXX="$cxx_compiler" \
+            cmake -G "Unix Makefiles" \
+            -DCMAKE_BUILD_TYPE=$BuildType -DLLVM_ENABLE_ASSERTIONS=$Assertions \
+            -DLLVM_ENABLE_TIMESTAMPS=OFF -DLLVM_CONFIGTIME="(timestamp not enabled)" \
+            $ExtraConfigureFlags $BuildDir/llvm.src \
+            2>&1 | tee $LogDir/llvm.configure-Phase$Phase-$Flavor.log
+        env CC="$c_compiler" CXX="$cxx_compiler" \
+            cmake -G "Unix Makefiles" \
+            -DCMAKE_BUILD_TYPE=$BuildType -DLLVM_ENABLE_ASSERTIONS=$Assertions \
+            -DLLVM_ENABLE_TIMESTAMPS=OFF -DLLVM_CONFIGTIME="(timestamp not enabled)" \
+            $ExtraConfigureFlags $BuildDir/llvm.src \
+            2>&1 | tee $LogDir/llvm.configure-Phase$Phase-$Flavor.log
+    fi
+
     cd $BuildDir
 }
 
@@ -325,57 +371,41 @@ function build_llvmCore() {
     Phase="$1"
     Flavor="$2"
     ObjDir="$3"
-    ExtraOpts=""
-
-    if [ "$Flavor" = "Release-64" ]; then
-        ExtraOpts="EXTRA_OPTIONS=-m64"
-    fi
+    DestDir="$4"
 
     cd $ObjDir
     echo "# Compiling llvm $Release-$RC $Flavor"
-    echo "# ${MAKE} -j $NumJobs VERBOSE=1 $ExtraOpts"
-    ${MAKE} -j $NumJobs VERBOSE=1 $ExtraOpts \
+    echo "# ${MAKE} -j $NumJobs VERBOSE=1"
+    ${MAKE} -j $NumJobs VERBOSE=1 \
         2>&1 | tee $LogDir/llvm.make-Phase$Phase-$Flavor.log
 
     echo "# Installing llvm $Release-$RC $Flavor"
     echo "# ${MAKE} install"
     ${MAKE} install \
+        DESTDIR="${DestDir}" \
         2>&1 | tee $LogDir/llvm.install-Phase$Phase-$Flavor.log
     cd $BuildDir
 }
 
-function build_dragonegg() {
-    Phase="$1"
-    Flavor="$2"
-    LLVMInstallDir="$3"
-    DragonEggObjDir="$4"
-    LLVM_CONFIG=$LLVMInstallDir/bin/llvm-config
-    TOP_DIR=$BuildDir/dragonegg.src
-
-    echo "# Targeted compiler: $gcc_compiler"
-
-    cd $DragonEggObjDir
-    echo "# Compiling phase $Phase dragonegg $Release-$RC $Flavor"
-    echo -n "# CXX=$cxx_compiler TOP_DIR=$TOP_DIR GCC=$gcc_compiler "
-    echo -n "LLVM_CONFIG=$LLVM_CONFIG ${MAKE} -f $TOP_DIR/Makefile "
-    echo "-j $NumJobs VERBOSE=1"
-    CXX="$cxx_compiler" TOP_DIR="$TOP_DIR" GCC="$gcc_compiler" \
-    LLVM_CONFIG="$LLVM_CONFIG" ${MAKE} -f $TOP_DIR/Makefile \
-        -j $NumJobs VERBOSE=1 \
-            2>&1 | tee $LogDir/dragonegg-Phase$Phase-$Flavor.log
-    cd $BuildDir
-}
-
 function test_llvmCore() {
     Phase="$1"
     Flavor="$2"
     ObjDir="$3"
 
     cd $ObjDir
-    ${MAKE} -k check-all \
-        2>&1 | tee $LogDir/llvm.check-Phase$Phase-$Flavor.log
-    ${MAKE} -k unittests \
-        2>&1 | tee $LogDir/llvm.unittests-Phase$Phase-$Flavor.log
+    if ! ( ${MAKE} -j $NumJobs -k check-all \
+        2>&1 | tee $LogDir/llvm.check-Phase$Phase-$Flavor.log ) ; then
+      deferred_error $Phase $Flavor "check-all failed"
+    fi
+
+    if [ "$use_autoconf" = "yes" ]; then
+        # In the cmake build, unit tests are run as part of check-all.
+        if ! ( ${MAKE} -k unittests 2>&1 | \
+            tee $LogDir/llvm.unittests-Phase$Phase-$Flavor.log ) ; then
+          deferred_error $Phase $Flavor "unittests failed"
+        fi
+    fi
+
     cd $BuildDir
 }
 
@@ -388,10 +418,12 @@ function clean_RPATH() {
   local InstallPath="$1"
   for Candidate in `find $InstallPath/{bin,lib} -type f`; do
     if file $Candidate | grep ELF | egrep 'executable|shared object' > /dev/null 2>&1 ; then
-      rpath=`objdump -x $Candidate | grep 'RPATH' | sed -e's/^ *RPATH *//'`
-      if [ -n "$rpath" ]; then
-        newrpath=`echo $rpath | sed -e's/.*\(\$ORIGIN[^:]*\).*/\1/'`
-        chrpath -r $newrpath $Candidate 2>&1 > /dev/null 2>&1
+      if rpath=`objdump -x $Candidate | grep 'RPATH'` ; then
+        rpath=`echo $rpath | sed -e's/^ *RPATH *//'`
+        if [ -n "$rpath" ]; then
+          newrpath=`echo $rpath | sed -e's/.*\(\$ORIGIN[^:]*\).*/\1/'`
+          chrpath -r $newrpath $Candidate 2>&1 > /dev/null 2>&1
+        fi
       fi
     fi
   done
@@ -401,17 +433,56 @@ function clean_RPATH() {
 function package_release() {
     cwd=`pwd`
     cd $BuildDir/Phase3/Release
-    mv llvmCore-$Release-$RC.install $Package
+    mv llvmCore-$Release-$RC.install/usr/local $Package
     if [ "$use_gzip" = "yes" ]; then
       tar cfz $BuildDir/$Package.tar.gz $Package
     else
       tar cfJ $BuildDir/$Package.tar.xz $Package
     fi
-    mv $Package llvmCore-$Release-$RC.install
+    mv $Package llvmCore-$Release-$RC.install/usr/local
+    cd $cwd
+}
+
+# Build and package the OpenMP run-time. This is still experimental and not
+# meant for official testing in the release, but as a way for providing
+# binaries as a convenience to those who want to try it out.
+function build_OpenMP() {
+    cwd=`pwd`
+
+    rm -rf $BuildDir/Phase3/openmp
+    rm -rf $BuildDir/Phase3/openmp.install
+    mkdir -p $BuildDir/Phase3/openmp
+    cd $BuildDir/Phase3/openmp
+    clang=$BuildDir/Phase3/Release/llvmCore-$Release-$RC.install/usr/local/bin/clang
+
+    echo "#" cmake -DCMAKE_C_COMPILER=${clang} -DCMAKE_CXX_COMPILER=${clang}++ \
+            -DCMAKE_BUILD_TYPE=Release -DLIBOMP_MICRO_TESTS=on \
+            $BuildDir/openmp.src/runtime
+    cmake -DCMAKE_C_COMPILER=${clang} -DCMAKE_CXX_COMPILER=${clang}++ \
+            -DCMAKE_BUILD_TYPE=Release -DLIBOMP_MICRO_TESTS=on \
+            $BuildDir/openmp.src/runtime
+
+    echo "# Building OpenMP run-time"
+    echo "# ${MAKE} -j $NumJobs VERBOSE=1"
+    ${MAKE} -j $NumJobs VERBOSE=1
+    echo "# ${MAKE} libomp-micro-tests VERBOSE=1"
+    ${MAKE} libomp-micro-tests VERBOSE=1
+    echo "# ${MAKE} install DESTDIR=$BuildDir/Phase3/openmp.install"
+    ${MAKE} install DESTDIR=$BuildDir/Phase3/openmp.install
+
+    OpenMPPackage=OpenMP-$Triple
+    mv $BuildDir/Phase3/openmp.install/usr/local $BuildDir/$OpenMPPackage
+    cd $BuildDir
+    tar cvfJ $BuildDir/$OpenMPPackage.tar.xz $OpenMPPackage
+    mv $OpenMPPackage $BuildDir/Phase3/openmp.install/usr/local
     cd $cwd
 }
 
-set -e                          # Exit if any command fails
+# Exit if any command fails
+# Note: pipefail is necessary for running build commands through
+# a pipe (i.e. it changes the output of ``false | tee /dev/null ; echo $?``)
+set -e
+set -o pipefail
 
 if [ "$do_checkout" = "yes" ]; then
     export_sources
@@ -425,9 +496,6 @@ fi
 if [ "$do_asserts" = "yes" ]; then
     Flavors="$Flavors Release+Asserts"
 fi
-if [ "$do_64bit" = "yes" ]; then
-    Flavors="$Flavors Release-64"
-fi
 
 for Flavor in $Flavors ; do
     echo ""
@@ -442,168 +510,89 @@ for Flavor in $Flavors ; do
 
     c_compiler="$CC"
     cxx_compiler="$CXX"
-
     llvmCore_phase1_objdir=$BuildDir/Phase1/$Flavor/llvmCore-$Release-$RC.obj
-    llvmCore_phase1_installdir=$BuildDir/Phase1/$Flavor/llvmCore-$Release-$RC.install
-    dragonegg_phase1_objdir=$BuildDir/Phase1/$Flavor/DragonEgg-$Release-$RC.obj
+    llvmCore_phase1_destdir=$BuildDir/Phase1/$Flavor/llvmCore-$Release-$RC.install
 
     llvmCore_phase2_objdir=$BuildDir/Phase2/$Flavor/llvmCore-$Release-$RC.obj
-    llvmCore_phase2_installdir=$BuildDir/Phase2/$Flavor/llvmCore-$Release-$RC.install
-    llvmCore_de_phase2_objdir=$BuildDir/Phase2/$Flavor/llvmCore-DragonEgg-$Release-$RC.obj
-    llvmCore_de_phase2_installdir=$BuildDir/Phase2/$Flavor/llvmCore-DragonEgg-$Release-$RC.install
-    dragonegg_phase2_objdir=$BuildDir/Phase2/$Flavor/DragonEgg-$Release-$RC.obj
+    llvmCore_phase2_destdir=$BuildDir/Phase2/$Flavor/llvmCore-$Release-$RC.install
 
     llvmCore_phase3_objdir=$BuildDir/Phase3/$Flavor/llvmCore-$Release-$RC.obj
-    llvmCore_phase3_installdir=$BuildDir/Phase3/$Flavor/llvmCore-$Release-$RC.install
-    llvmCore_de_phase3_objdir=$BuildDir/Phase3/$Flavor/llvmCore-DragonEgg-$Release-$RC.obj
-    llvmCore_de_phase3_installdir=$BuildDir/Phase3/$Flavor/llvmCore-DragonEgg-$Release-$RC.install
-    dragonegg_phase3_objdir=$BuildDir/Phase3/$Flavor/DragonEgg-$Release-$RC.obj
+    llvmCore_phase3_destdir=$BuildDir/Phase3/$Flavor/llvmCore-$Release-$RC.install
 
     rm -rf $llvmCore_phase1_objdir
-    rm -rf $llvmCore_phase1_installdir
-    rm -rf $dragonegg_phase1_objdir
+    rm -rf $llvmCore_phase1_destdir
 
     rm -rf $llvmCore_phase2_objdir
-    rm -rf $llvmCore_phase2_installdir
-    rm -rf $llvmCore_de_phase2_objdir
-    rm -rf $llvmCore_de_phase2_installdir
-    rm -rf $dragonegg_phase2_objdir
+    rm -rf $llvmCore_phase2_destdir
 
     rm -rf $llvmCore_phase3_objdir
-    rm -rf $llvmCore_phase3_installdir
-    rm -rf $llvmCore_de_phase3_objdir
-    rm -rf $llvmCore_de_phase3_installdir
-    rm -rf $dragonegg_phase3_objdir
+    rm -rf $llvmCore_phase3_destdir
 
     mkdir -p $llvmCore_phase1_objdir
-    mkdir -p $llvmCore_phase1_installdir
-    mkdir -p $dragonegg_phase1_objdir
+    mkdir -p $llvmCore_phase1_destdir
 
     mkdir -p $llvmCore_phase2_objdir
-    mkdir -p $llvmCore_phase2_installdir
-    mkdir -p $llvmCore_de_phase2_objdir
-    mkdir -p $llvmCore_de_phase2_installdir
-    mkdir -p $dragonegg_phase2_objdir
+    mkdir -p $llvmCore_phase2_destdir
 
     mkdir -p $llvmCore_phase3_objdir
-    mkdir -p $llvmCore_phase3_installdir
-    mkdir -p $llvmCore_de_phase3_objdir
-    mkdir -p $llvmCore_de_phase3_installdir
-    mkdir -p $dragonegg_phase3_objdir
+    mkdir -p $llvmCore_phase3_destdir
 
     ############################################################################
     # Phase 1: Build llvmCore and clang
     echo "# Phase 1: Building llvmCore"
-    configure_llvmCore 1 $Flavor \
-        $llvmCore_phase1_objdir $llvmCore_phase1_installdir
+    configure_llvmCore 1 $Flavor $llvmCore_phase1_objdir
     build_llvmCore 1 $Flavor \
-        $llvmCore_phase1_objdir
-    clean_RPATH $llvmCore_phase1_installdir
-
-    # Test clang
-    if [ "$do_clang" = "yes" ]; then
-        ########################################################################
-        # Phase 2: Build llvmCore with newly built clang from phase 1.
-        c_compiler=$llvmCore_phase1_installdir/bin/clang
-        cxx_compiler=$llvmCore_phase1_installdir/bin/clang++
-        echo "# Phase 2: Building llvmCore"
-        configure_llvmCore 2 $Flavor \
-            $llvmCore_phase2_objdir $llvmCore_phase2_installdir
-        build_llvmCore 2 $Flavor \
-            $llvmCore_phase2_objdir
-        clean_RPATH $llvmCore_phase2_installdir
-
-        ########################################################################
-        # Phase 3: Build llvmCore with newly built clang from phase 2.
-        c_compiler=$llvmCore_phase2_installdir/bin/clang
-        cxx_compiler=$llvmCore_phase2_installdir/bin/clang++
-        echo "# Phase 3: Building llvmCore"
-        configure_llvmCore 3 $Flavor \
-            $llvmCore_phase3_objdir $llvmCore_phase3_installdir
-        build_llvmCore 3 $Flavor \
-            $llvmCore_phase3_objdir
-        clean_RPATH $llvmCore_phase3_installdir
-
-        ########################################################################
-        # Testing: Test phase 3
-        echo "# Testing - built with clang"
-        test_llvmCore 3 $Flavor $llvmCore_phase3_objdir
-
-        ########################################################################
-        # Compare .o files between Phase2 and Phase3 and report which ones
-        # differ.
-        if [ "$do_compare" = "yes" ]; then
-            echo
-            echo "# Comparing Phase 2 and Phase 3 files"
-            for o in `find $llvmCore_phase2_objdir -name '*.o'` ; do
-                p3=`echo $o | sed -e 's,Phase2,Phase3,'`
-                if ! cmp --ignore-initial=16 $o $p3 > /dev/null 2>&1 ; then
-                    echo "file `basename $o` differs between phase 2 and phase 3"
-                fi
-            done
-        fi
-    fi
-
-    # Test dragonegg
-    if [ "$do_dragonegg" = "yes" ]; then
-        # Build dragonegg using the targeted gcc.  This isn't necessary, but
-        # helps avoid using broken versions of gcc (which are legion), tests
-        # that the targeted gcc is basically sane and is consistent with the
-        # later phases in which the targeted gcc + dragonegg are used.
-        c_compiler="$gcc_compiler"
-        cxx_compiler="$gxx_compiler"
-        build_dragonegg 1 $Flavor $llvmCore_phase1_installdir $dragonegg_phase1_objdir
-
-        ########################################################################
-        # Phase 2: Build llvmCore with newly built dragonegg from phase 1.
-        c_compiler="$gcc_compiler -fplugin=$dragonegg_phase1_objdir/dragonegg.so"
-        cxx_compiler="$gxx_compiler -fplugin=$dragonegg_phase1_objdir/dragonegg.so"
-        echo "# Phase 2: Building llvmCore with dragonegg"
-        configure_llvmCore 2 $Flavor \
-            $llvmCore_de_phase2_objdir $llvmCore_de_phase2_installdir
-        build_llvmCore 2 $Flavor \
-            $llvmCore_de_phase2_objdir
-        build_dragonegg 2 $Flavor $llvmCore_de_phase2_installdir $dragonegg_phase2_objdir
-        clean_RPATH $llvmCore_de_phase2_installdir
-
-        ########################################################################
-        # Phase 3: Build llvmCore with newly built dragonegg from phase 2.
-        c_compiler="$gcc_compiler -fplugin=$dragonegg_phase2_objdir/dragonegg.so"
-        cxx_compiler="$gxx_compiler -fplugin=$dragonegg_phase2_objdir/dragonegg.so"
-        echo "# Phase 3: Building llvmCore with dragonegg"
-        configure_llvmCore 3 $Flavor \
-            $llvmCore_de_phase3_objdir $llvmCore_de_phase3_installdir
-        build_llvmCore 3 $Flavor \
-            $llvmCore_de_phase3_objdir
-        build_dragonegg 3 $Flavor $llvmCore_de_phase3_installdir $dragonegg_phase3_objdir
-        clean_RPATH $llvmCore_de_phase3_installdir
-
-        ########################################################################
-        # Testing: Test phase 3
-        c_compiler="$gcc_compiler -fplugin=$dragonegg_phase3_objdir/dragonegg.so"
-        cxx_compiler="$gxx_compiler -fplugin=$dragonegg_phase3_objdir/dragonegg.so"
-        echo "# Testing - built with dragonegg"
-        test_llvmCore 3 $Flavor $llvmCore_de_phase3_objdir
-
-        ########################################################################
-        # Compare .o files between Phase2 and Phase3 and report which ones differ.
+        $llvmCore_phase1_objdir $llvmCore_phase1_destdir
+    clean_RPATH $llvmCore_phase1_destdir/usr/local
+
+    ########################################################################
+    # Phase 2: Build llvmCore with newly built clang from phase 1.
+    c_compiler=$llvmCore_phase1_destdir/usr/local/bin/clang
+    cxx_compiler=$llvmCore_phase1_destdir/usr/local/bin/clang++
+    echo "# Phase 2: Building llvmCore"
+    configure_llvmCore 2 $Flavor $llvmCore_phase2_objdir
+    build_llvmCore 2 $Flavor \
+        $llvmCore_phase2_objdir $llvmCore_phase2_destdir
+    clean_RPATH $llvmCore_phase2_destdir/usr/local
+
+    ########################################################################
+    # Phase 3: Build llvmCore with newly built clang from phase 2.
+    c_compiler=$llvmCore_phase2_destdir/usr/local/bin/clang
+    cxx_compiler=$llvmCore_phase2_destdir/usr/local/bin/clang++
+    echo "# Phase 3: Building llvmCore"
+    configure_llvmCore 3 $Flavor $llvmCore_phase3_objdir
+    build_llvmCore 3 $Flavor \
+        $llvmCore_phase3_objdir $llvmCore_phase3_destdir
+    clean_RPATH $llvmCore_phase3_destdir/usr/local
+
+    ########################################################################
+    # Testing: Test phase 3
+    echo "# Testing - built with clang"
+    test_llvmCore 3 $Flavor $llvmCore_phase3_objdir
+
+    ########################################################################
+    # Compare .o files between Phase2 and Phase3 and report which ones
+    # differ.
+    if [ "$do_compare" = "yes" ]; then
         echo
         echo "# Comparing Phase 2 and Phase 3 files"
-        for o in `find $llvmCore_de_phase2_objdir -name '*.o'` \
-          `find $dragonegg_phase2_objdir -name '*.o'` ; do
-            p3=`echo $o | sed -e 's,Phase2,Phase3,'`
-            if ! cmp --ignore-initial=16 $o $p3 > /dev/null 2>&1 ; then
-                echo "file `basename $o` differs between dragonegg phase 2 and phase 3"
+        for p2 in `find $llvmCore_phase2_objdir -name '*.o'` ; do
+            p3=`echo $p2 | sed -e 's,Phase2,Phase3,'`
+            # Substitute 'Phase2' for 'Phase3' in the Phase 2 object file in
+            # case there are build paths in the debug info. On some systems,
+            # sed adds a newline to the output, so pass $p3 through sed too.
+            if ! cmp -s <(sed -e 's,Phase2,Phase3,g' $p2) <(sed -e '' $p3) \
+                    16 16 ; then
+                echo "file `basename $p2` differs between phase 2 and phase 3"
             fi
         done
     fi
-
-    # Otherwise just test the core.
-    if [ "$do_clang" != "yes" -a "$do_dragonegg" != "yes" ]; then
-        echo "# Testing - built with system compiler"
-        test_llvmCore 1 $Flavor $llvmCore_phase1_objdir
-    fi
 done
+
+if [ $do_openmp = "yes" ]; then
+  build_OpenMP
+fi
+
 ) 2>&1 | tee $LogDir/testing.$Release-$RC.log
 
 package_release
@@ -618,4 +607,13 @@ else
   echo "### Package: $Package.tar.xz"
 fi
 echo "### Logs: $LogDir"
+
+echo "### Errors:"
+if [ -s "$LogDir/deferred_errors.log" ]; then
+  cat "$LogDir/deferred_errors.log"
+  exit 1
+else
+  echo "None."
+fi
+
 exit 0
diff --git a/utils/sort_includes.py b/utils/sort_includes.py
index fef97550db8..70bfdedfc6d 100755
--- a/utils/sort_includes.py
+++ b/utils/sort_includes.py
@@ -29,7 +29,8 @@ def sort_includes(f):
   headers_end = 0
   api_headers = []
   local_headers = []
-  project_headers = []
+  subproject_headers = []
+  llvm_headers = []
   system_headers = []
   for (i, l) in enumerate(lines):
     if l.strip() == '':
@@ -44,12 +45,16 @@ def sort_includes(f):
         api_headers.append(header)
         look_for_api_header = False
         continue
-      if header.startswith('<') or header.startswith('"gtest/'):
+      if (header.startswith('<') or header.startswith('"gtest/') or
+          header.startswith('"isl/') or header.startswith('"json/')):
         system_headers.append(header)
         continue
-      if (header.startswith('"llvm/') or header.startswith('"llvm-c/') or
-          header.startswith('"clang/') or header.startswith('"clang-c/')):
-        project_headers.append(header)
+      if (header.startswith('"clang/') or header.startswith('"clang-c/') or
+          header.startswith('"polly/')):
+        subproject_headers.append(header)
+        continue
+      if (header.startswith('"llvm/') or header.startswith('"llvm-c/')):
+        llvm_headers.append(header)
         continue
       local_headers.append(header)
       continue
@@ -65,9 +70,10 @@ def sort_includes(f):
     return
 
   local_headers = sorted(set(local_headers))
-  project_headers = sorted(set(project_headers))
+  subproject_headers = sorted(set(subproject_headers))
+  llvm_headers = sorted(set(llvm_headers))
   system_headers = sorted(set(system_headers))
-  headers = api_headers + local_headers + project_headers + system_headers
+  headers = api_headers + local_headers + subproject_headers + llvm_headers + system_headers
   header_lines = ['#include ' + h for h in headers]
   lines = lines[:headers_begin] + header_lines + lines[headers_end + 1:]
 
diff --git a/utils/unittest/UnitTestMain/TestMain.cpp b/utils/unittest/UnitTestMain/TestMain.cpp
index f5b09a5cf67..fb2b0f16ee3 100644
--- a/utils/unittest/UnitTestMain/TestMain.cpp
+++ b/utils/unittest/UnitTestMain/TestMain.cpp
@@ -7,13 +7,12 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "llvm/Config/config.h"
 #include "llvm/Support/CommandLine.h"
 #include "llvm/Support/Signals.h"
 #include "gtest/gtest.h"
 
 
-#if defined(LLVM_ON_WIN32)
+#if defined(_WIN32)
 # include <windows.h>
 # if defined(_MSC_VER)
 #   include <crtdbg.h>
@@ -30,7 +29,7 @@ int main(int argc, char **argv) {
   // Make it easy for a test to re-execute itself by saving argv[0].
   TestMainArgv0 = argv[0];
 
-# if defined(LLVM_ON_WIN32)
+# if defined(_WIN32)
   // Disable all of the possible ways Windows conspires to make automated
   // testing impossible.
   ::SetErrorMode(SEM_FAILCRITICALERRORS | SEM_NOGPFAULTERRORBOX);
diff --git a/utils/vim/README b/utils/vim/README
index bca25bfe612..1fe00992b0a 100644
--- a/utils/vim/README
+++ b/utils/vim/README
@@ -1,33 +1,12 @@
 -*- llvm/utils/vim/README -*-
 
-These are syntax highlighting files for the VIM editor. Included are:
+This directory contains settings for the vim editor to work on llvm *.ll and
+tablegen *.td files. It comes with filetype detection rules in the (ftdetect),
+syntax highlighting (syntax), some minimal sensible default settings (ftplugin)
+and indentation plugins (indent).
 
-* llvm.vim
-
-  Syntax highlighting mode for LLVM assembly files. To use, copy `llvm.vim' to
-  ~/.vim/syntax and add this code to your ~/.vimrc :
-
-  augroup filetype
-    au! BufRead,BufNewFile *.ll     set filetype=llvm
-  augroup END
-
-* tablegen.vim
-
-  Syntax highlighting mode for TableGen description files. To use, copy
-  `tablegen.vim' to ~/.vim/syntax and add this code to your ~/.vimrc :
-
-  augroup filetype
-    au! BufRead,BufNewFile *.td     set filetype=tablegen
-  augroup END
-
-
-If you prefer, instead of making copies you can make symlinks from
-~/.vim/syntax/... to the syntax files in your LLVM source tree. Apparently
-this did not work with older versions of vim however, so if this doesn't
-work you may need to make actual copies of the files.
-
-Another option, if you do not already have a ~/.vim/syntax directory, is
-to symlink ~/.vim/syntax itself to llvm/utils/vim .
+To install copy all subdirectories to your $HOME/.vim or if you prefer create
+symlinks to the files here. Do not copy the vimrc file here it is only meant as an inspiration and starting point for those working on llvm c++ code.
 
 Note: If you notice missing or incorrect syntax highlighting, please contact
 <llvmbugs [at] cs.uiuc.edu>; if you wish to provide a patch to improve the
diff --git a/utils/vim/ftdetect/llvm-lit.vim b/utils/vim/ftdetect/llvm-lit.vim
new file mode 100644
index 00000000000..fa651e1268c
--- /dev/null
+++ b/utils/vim/ftdetect/llvm-lit.vim
@@ -0,0 +1 @@
+au BufRead,BufNewFile lit.*cfg set filetype=python
diff --git a/utils/vim/ftdetect/llvm.vim b/utils/vim/ftdetect/llvm.vim
new file mode 100644
index 00000000000..161ceb2bd55
--- /dev/null
+++ b/utils/vim/ftdetect/llvm.vim
@@ -0,0 +1 @@
+au BufRead,BufNewFile *.ll set filetype=llvm
diff --git a/utils/vim/ftdetect/tablegen.vim b/utils/vim/ftdetect/tablegen.vim
new file mode 100644
index 00000000000..95efa6efdd0
--- /dev/null
+++ b/utils/vim/ftdetect/tablegen.vim
@@ -0,0 +1 @@
+au BufRead,BufNewFile *.td set filetype=tablegen
diff --git a/utils/vim/ftplugin/llvm.vim b/utils/vim/ftplugin/llvm.vim
new file mode 100644
index 00000000000..bdf49c92ff2
--- /dev/null
+++ b/utils/vim/ftplugin/llvm.vim
@@ -0,0 +1,12 @@
+" Vim filetype plugin file
+" Language: LLVM Assembly
+" Maintainer: The LLVM team, http://llvm.org/
+
+if exists("b:did_ftplugin")
+  finish
+endif
+let b:did_ftplugin = 1
+
+setlocal softtabstop=2 shiftwidth=2
+setlocal expandtab
+setlocal comments+=:;
diff --git a/utils/vim/ftplugin/tablegen.vim b/utils/vim/ftplugin/tablegen.vim
new file mode 100644
index 00000000000..cfae846c818
--- /dev/null
+++ b/utils/vim/ftplugin/tablegen.vim
@@ -0,0 +1,12 @@
+" Vim filetype plugin file
+" Language: LLVM TableGen
+" Maintainer: The LLVM team, http://llvm.org/
+
+if exists("b:did_ftplugin")
+  finish
+endif
+let b:did_ftplugin = 1
+
+setlocal matchpairs+=<:>
+setlocal softtabstop=2 shiftwidth=2
+setlocal expandtab
diff --git a/utils/vim/indent/llvm.vim b/utils/vim/indent/llvm.vim
new file mode 100644
index 00000000000..d1d8c83d118
--- /dev/null
+++ b/utils/vim/indent/llvm.vim
@@ -0,0 +1,72 @@
+" Vim indent file
+" Language:   llvm
+" Maintainer: The LLVM team, http://llvm.org/
+" What this indent plugin currently does:
+"  - If no other rule matches copy indent from previous non-empty,
+"    non-commented line
+"  - On '}' align the same as the line containing the matching '{'
+"  - If previous line ends with ':' increase indentation
+"  - If the current line ends with ':' indent at the same level as the
+"    enclosing '{'/'}' block
+" Stuff that would be nice to add:
+"  - Continue comments on next line
+"  - If there is an opening+unclosed parenthesis on previous line indent to that
+if exists("b:did_indent")
+  finish
+endif
+let b:did_indent = 1
+
+setlocal shiftwidth=2 expandtab
+
+setlocal indentkeys=0{,0},<:>,!^F,o,O,e
+setlocal indentexpr=GetLLVMIndent()
+
+if exists("*GetLLVMIndent")
+  finish
+endif
+
+function! FindOpenBrace(lnum)
+  call cursor(a:lnum, 1)
+  return searchpair('{', '', '}', 'bW')
+endfun
+
+function! GetLLVMIndent()
+  " On '}' align the same as the line containing the matching '{'
+  let thisline = getline(v:lnum)
+  if thisline =~ '^\s*}'
+    call cursor(v:lnum, 1)
+    silent normal %
+    let opening_lnum = line('.')
+    if opening_lnum != v:lnum
+      return indent(opening_lnum)
+    endif
+  endif
+
+  " Indent labels the same as the current opening block
+  if thisline =~ ':\s*$'
+    let blockbegin = FindOpenBrace(v:lnum)
+    if blockbegin > 0
+      return indent(blockbegin)
+    endif
+  endif
+
+  " Find a non-blank not-completely commented line above the current line.
+  let prev_lnum = prevnonblank(v:lnum - 1)
+  while prev_lnum > 0 && synIDattr(synID(prev_lnum, indent(prev_lnum)+1, 0), "name") =? "string\|comment"
+    let prev_lnum = prevnonblank(prev_lnum-1)
+  endwhile
+  " Hit the start of the file, use zero indent.
+  if prev_lnum == 0
+    return 0
+  endif
+
+  let ind = indent(prev_lnum)
+  let prevline = getline(prev_lnum)
+
+  " Add a 'shiftwidth' after lines that start a block or labels
+  if prevline =~ '{\s*$' || prevline =~ ':\s*$'
+    let ind = ind + &shiftwidth
+  endif
+
+  return ind
+endfunction
diff --git a/utils/vim/llvm.vim b/utils/vim/llvm.vim
deleted file mode 100644
index 913d0f5f1c3..00000000000
--- a/utils/vim/llvm.vim
+++ /dev/null
@@ -1,119 +0,0 @@
-" Vim syntax file
-" Language:   llvm
-" Maintainer: The LLVM team, http://llvm.org/
-" Version:      $Revision$
-
-if version < 600
-  syntax clear
-elseif exists("b:current_syntax")
-  finish
-endif
-
-syn case match
-
-" Types.
-" Types also include struct, array, vector, etc. but these don't
-" benefit as much from having dedicated highlighting rules.
-syn keyword llvmType void half float double x86_fp80 fp128 ppc_fp128
-syn keyword llvmType label metadata x86_mmx
-syn keyword llvmType type label opaque
-syn match   llvmType /\<i\d\+\>/
-
-" Instructions.
-" The true and false tokens can be used for comparison opcodes, but it's
-" much more common for these tokens to be used for boolean constants.
-syn keyword llvmStatement add addrspacecast alloca and arcp ashr atomicrmw
-syn keyword llvmStatement bitcast br call cmpxchg eq exact extractelement
-syn keyword llvmStatement extractvalue fadd fast fcmp fdiv fence fmul fpext
-syn keyword llvmStatement fptosi fptoui fptrunc free frem fsub getelementptr
-syn keyword llvmStatement icmp inbounds indirectbr insertelement insertvalue
-syn keyword llvmStatement inttoptr invoke landingpad load lshr malloc max min
-syn keyword llvmStatement mul nand ne ninf nnan nsw nsz nuw oeq oge ogt ole
-syn keyword llvmStatement olt one or ord phi ptrtoint resume ret sdiv select
-syn keyword llvmStatement sext sge sgt shl shufflevector sitofp sle slt srem
-syn keyword llvmStatement store sub switch trunc udiv ueq uge ugt uitofp ule ult
-syn keyword llvmStatement umax umin une uno unreachable unwind urem va_arg
-syn keyword llvmStatement xchg xor zext
-
-" Keywords.
-syn keyword llvmKeyword acq_rel acquire sanitize_address addrspace alias align
-syn keyword llvmKeyword alignstack alwaysinline appending arm_aapcs_vfpcc
-syn keyword llvmKeyword arm_aapcscc arm_apcscc asm atomic available_externally
-syn keyword llvmKeyword blockaddress byval c catch cc ccc cleanup coldcc common
-syn keyword llvmKeyword constant datalayout declare default define deplibs
-syn keyword llvmKeyword distinct dllexport dllimport except extern_weak external
-syn keyword llvmKeyword externally_initialized fastcc filter gc global hidden
-syn keyword llvmKeyword initialexec inlinehint inreg intel_ocl_bicc inteldialect
-syn keyword llvmKeyword internal linkonce linkonce_odr localdynamic localexec
-syn keyword llvmKeyword minsize module monotonic msp430_intrcc naked nest
-syn keyword llvmKeyword noalias nocapture noimplicitfloat noinline nonlazybind
-syn keyword llvmKeyword noredzone noreturn nounwind optnone optsize personality
-syn keyword llvmKeyword private protected ptx_device ptx_kernel readnone
-syn keyword llvmKeyword readonly release returns_twice sanitize_thread
-syn keyword llvmKeyword sanitize_memory section seq_cst sideeffect signext
-syn keyword llvmKeyword singlethread spir_func spir_kernel sret ssp sspreq
-syn keyword llvmKeyword sspstrong tail target thread_local to triple
-syn keyword llvmKeyword unnamed_addr unordered uwtable volatile weak weak_odr
-syn keyword llvmKeyword x86_fastcallcc x86_stdcallcc x86_thiscallcc
-syn keyword llvmKeyword x86_64_sysvcc x86_64_win64cc zeroext uselistorder
-syn keyword llvmKeyword uselistorder_bb
-
-" Obsolete keywords.
-syn keyword llvmError  getresult begin end
-
-" Misc syntax.
-syn match   llvmNoName /[%@!]\d\+\>/
-syn match   llvmNumber /-\?\<\d\+\>/
-syn match   llvmFloat  /-\?\<\d\+\.\d*\(e[+-]\d\+\)\?\>/
-syn match   llvmFloat  /\<0x\x\+\>/
-syn keyword llvmBoolean true false
-syn keyword llvmConstant zeroinitializer undef null
-syn match   llvmComment /;.*$/
-syn region  llvmString start=/"/ skip=/\\"/ end=/"/
-syn match   llvmLabel /[-a-zA-Z$._][-a-zA-Z$._0-9]*:/
-syn match   llvmIdentifier /[%@][-a-zA-Z$._][-a-zA-Z$._0-9]*/
-
-" Named metadata and specialized metadata keywords.
-syn match   llvmIdentifier /![-a-zA-Z$._][-a-zA-Z$._0-9]*\ze\s*$/
-syn match   llvmIdentifier /![-a-zA-Z$._][-a-zA-Z$._0-9]*\ze\s*[=!]/
-syn match   llvmType /!\zs\a\+\ze\s*(/
-syn match   llvmConstant /\<DW_TAG_[a-z_]\+\>/
-syn match   llvmConstant /\<DW_ATE_[a-zA-Z_]\+\>/
-syn match   llvmConstant /\<DW_OP_[a-zA-Z0-9_]\+\>/
-syn match   llvmConstant /\<DW_LANG_[a-zA-Z0-9_]\+\>/
-syn match   llvmConstant /\<DW_VIRTUALITY_[a-z_]\+\>/
-syn match   llvmConstant /\<DIFlag[A-Za-z]\+\>/
-
-" Syntax-highlight dejagnu test commands.
-syn match  llvmSpecialComment /;\s*RUN:.*$/
-syn match  llvmSpecialComment /;\s*PR\d*\s*$/
-syn match  llvmSpecialComment /;\s*END\.\s*$/
-syn match  llvmSpecialComment /;\s*XFAIL:.*$/
-
-if version >= 508 || !exists("did_c_syn_inits")
-  if version < 508
-    let did_c_syn_inits = 1
-    command -nargs=+ HiLink hi link <args>
-  else
-    command -nargs=+ HiLink hi def link <args>
-  endif
-
-  HiLink llvmType Type
-  HiLink llvmStatement Statement
-  HiLink llvmNumber Number
-  HiLink llvmComment Comment
-  HiLink llvmString String
-  HiLink llvmLabel Label
-  HiLink llvmKeyword Keyword
-  HiLink llvmBoolean Boolean
-  HiLink llvmFloat Float
-  HiLink llvmNoName Identifier
-  HiLink llvmConstant Constant
-  HiLink llvmSpecialComment SpecialComment
-  HiLink llvmError Error
-  HiLink llvmIdentifier Identifier
-
-  delcommand HiLink
-endif
-
-let b:current_syntax = "llvm"
diff --git a/utils/vim/syntax/llvm.vim b/utils/vim/syntax/llvm.vim
new file mode 100644
index 00000000000..0e2cbd3e646
--- /dev/null
+++ b/utils/vim/syntax/llvm.vim
@@ -0,0 +1,119 @@
+" Vim syntax file
+" Language:   llvm
+" Maintainer: The LLVM team, http://llvm.org/
+" Version:      $Revision$
+
+if version < 600
+  syntax clear
+elseif exists("b:current_syntax")
+  finish
+endif
+
+syn case match
+
+" Types.
+" Types also include struct, array, vector, etc. but these don't
+" benefit as much from having dedicated highlighting rules.
+syn keyword llvmType void half float double x86_fp80 fp128 ppc_fp128
+syn keyword llvmType label metadata x86_mmx
+syn keyword llvmType type label opaque
+syn match   llvmType /\<i\d\+\>/
+
+" Instructions.
+" The true and false tokens can be used for comparison opcodes, but it's
+" much more common for these tokens to be used for boolean constants.
+syn keyword llvmStatement add addrspacecast alloca and arcp ashr atomicrmw
+syn keyword llvmStatement bitcast br call cmpxchg eq exact extractelement
+syn keyword llvmStatement extractvalue fadd fast fcmp fdiv fence fmul fpext
+syn keyword llvmStatement fptosi fptoui fptrunc free frem fsub getelementptr
+syn keyword llvmStatement icmp inbounds indirectbr insertelement insertvalue
+syn keyword llvmStatement inttoptr invoke landingpad load lshr malloc max min
+syn keyword llvmStatement mul nand ne ninf nnan nsw nsz nuw oeq oge ogt ole
+syn keyword llvmStatement olt one or ord phi ptrtoint resume ret sdiv select
+syn keyword llvmStatement sext sge sgt shl shufflevector sitofp sle slt srem
+syn keyword llvmStatement store sub switch trunc udiv ueq uge ugt uitofp ule ult
+syn keyword llvmStatement umax umin une uno unreachable unwind urem va_arg
+syn keyword llvmStatement xchg xor zext
+
+" Keywords.
+syn keyword llvmKeyword acq_rel acquire sanitize_address addrspace alias align
+syn keyword llvmKeyword alignstack alwaysinline appending arm_aapcs_vfpcc
+syn keyword llvmKeyword arm_aapcscc arm_apcscc asm atomic available_externally
+syn keyword llvmKeyword blockaddress byval c catch cc ccc cleanup coldcc common
+syn keyword llvmKeyword constant datalayout declare default define deplibs
+syn keyword llvmKeyword distinct dllexport dllimport except extern_weak external
+syn keyword llvmKeyword externally_initialized fastcc filter gc global hidden
+syn keyword llvmKeyword initialexec inlinehint inreg intel_ocl_bicc inteldialect
+syn keyword llvmKeyword internal linkonce linkonce_odr localdynamic localexec
+syn keyword llvmKeyword minsize module monotonic msp430_intrcc naked nest
+syn keyword llvmKeyword noalias nocapture noimplicitfloat noinline nonlazybind
+syn keyword llvmKeyword noredzone noreturn nounwind optnone optsize personality
+syn keyword llvmKeyword private protected ptx_device ptx_kernel readnone
+syn keyword llvmKeyword readonly release returns_twice sanitize_thread
+syn keyword llvmKeyword sanitize_memory section seq_cst sideeffect signext
+syn keyword llvmKeyword singlethread spir_func spir_kernel sret ssp sspreq
+syn keyword llvmKeyword sspstrong tail target thread_local to triple
+syn keyword llvmKeyword unnamed_addr unordered uwtable volatile weak weak_odr
+syn keyword llvmKeyword x86_fastcallcc x86_stdcallcc x86_thiscallcc
+syn keyword llvmKeyword x86_64_sysvcc x86_64_win64cc zeroext uselistorder
+syn keyword llvmKeyword uselistorder_bb
+
+" Obsolete keywords.
+syn keyword llvmError  getresult begin end
+
+" Misc syntax.
+syn match   llvmNoName /[%@!]\d\+\>/
+syn match   llvmNumber /-\?\<\d\+\>/
+syn match   llvmFloat  /-\?\<\d\+\.\d*\(e[+-]\d\+\)\?\>/
+syn match   llvmFloat  /\<0x\x\+\>/
+syn keyword llvmBoolean true false
+syn keyword llvmConstant zeroinitializer undef null
+syn match   llvmComment /;.*$/
+syn region  llvmString start=/"/ skip=/\\"/ end=/"/
+syn match   llvmLabel /[-a-zA-Z$._][-a-zA-Z$._0-9]*:/
+syn match   llvmIdentifier /[%@][-a-zA-Z$._][-a-zA-Z$._0-9]*/
+
+" Named metadata and specialized metadata keywords.
+syn match   llvmIdentifier /![-a-zA-Z$._][-a-zA-Z$._0-9]*\ze\s*$/
+syn match   llvmIdentifier /![-a-zA-Z$._][-a-zA-Z$._0-9]*\ze\s*[=!]/
+syn match   llvmType /!\zs\a\+\ze\s*(/
+syn match   llvmConstant /\<DW_TAG_[a-z_]\+\>/
+syn match   llvmConstant /\<DW_ATE_[a-zA-Z_]\+\>/
+syn match   llvmConstant /\<DW_OP_[a-zA-Z0-9_]\+\>/
+syn match   llvmConstant /\<DW_LANG_[a-zA-Z0-9_]\+\>/
+syn match   llvmConstant /\<DW_VIRTUALITY_[a-z_]\+\>/
+syn match   llvmConstant /\<DIFlag[A-Za-z]\+\>/
+
+" Syntax-highlight lit test commands and bug numbers.
+syn match  llvmSpecialComment /;\s*PR\d*\s*$/
+syn match  llvmSpecialComment /;\s*REQUIRES:.*$/
+syn match  llvmSpecialComment /;\s*RUN:.*$/
+syn match  llvmSpecialComment /;\s*XFAIL:.*$/
+
+if version >= 508 || !exists("did_c_syn_inits")
+  if version < 508
+    let did_c_syn_inits = 1
+    command -nargs=+ HiLink hi link <args>
+  else
+    command -nargs=+ HiLink hi def link <args>
+  endif
+
+  HiLink llvmType Type
+  HiLink llvmStatement Statement
+  HiLink llvmNumber Number
+  HiLink llvmComment Comment
+  HiLink llvmString String
+  HiLink llvmLabel Label
+  HiLink llvmKeyword Keyword
+  HiLink llvmBoolean Boolean
+  HiLink llvmFloat Float
+  HiLink llvmNoName Identifier
+  HiLink llvmConstant Constant
+  HiLink llvmSpecialComment SpecialComment
+  HiLink llvmError Error
+  HiLink llvmIdentifier Identifier
+
+  delcommand HiLink
+endif
+
+let b:current_syntax = "llvm"
diff --git a/utils/vim/tablegen.vim b/utils/vim/syntax/tablegen.vim
similarity index 100%
rename from utils/vim/tablegen.vim
rename to utils/vim/syntax/tablegen.vim
diff --git a/utils/yaml-bench/YAMLBench.cpp b/utils/yaml-bench/YAMLBench.cpp
index 872f586ef7e..634622a710c 100644
--- a/utils/yaml-bench/YAMLBench.cpp
+++ b/utils/yaml-bench/YAMLBench.cpp
@@ -19,6 +19,7 @@
 #include "llvm/Support/MemoryBuffer.h"
 #include "llvm/Support/SourceMgr.h"
 #include "llvm/Support/Timer.h"
+#include "llvm/Support/Process.h"
 #include "llvm/Support/YAMLParser.h"
 #include "llvm/Support/raw_ostream.h"
 #include <system_error>
@@ -52,6 +53,10 @@ static cl::opt<unsigned>
                   "Do not use more megabytes of memory"),
                 cl::init(1000));
 
+cl::opt<cl::boolOrDefault>
+    UseColor("use-color", cl::desc("Emit colored output (default=autodetect)"),
+             cl::init(cl::BOU_UNSET));
+
 struct indent {
   unsigned distance;
   indent(unsigned d) : distance(d) {}
@@ -69,7 +74,7 @@ static std::string prettyTag(yaml::Node *N) {
   if (StringRef(Tag).startswith("tag:yaml.org,2002:")) {
     std::string Ret = "!!";
     Ret += StringRef(Tag).substr(18);
-    return std::move(Ret);
+    return Ret;
   }
   std::string Ret = "!<";
   Ret += Tag;
@@ -91,6 +96,8 @@ static void dumpNode( yaml::Node *n
     SmallString<32> Storage;
     StringRef Val = sn->getValue(Storage);
     outs() << prettyTag(n) << " \"" << yaml::escape(Val) << "\"";
+  } else if (yaml::BlockScalarNode *BN = dyn_cast<yaml::BlockScalarNode>(n)) {
+    outs() << prettyTag(n) << " \"" << yaml::escape(BN->getValue()) << "\"";
   } else if (yaml::SequenceNode *sn = dyn_cast<yaml::SequenceNode>(n)) {
     outs() << prettyTag(n) << " [\n";
     ++Indent;
@@ -187,6 +194,9 @@ static std::string createJSONText(size_t MemoryMB, unsigned ValueSize) {
 
 int main(int argc, char **argv) {
   llvm::cl::ParseCommandLineOptions(argc, argv);
+  bool ShowColors = UseColor == cl::BOU_UNSET
+                        ? sys::Process::StandardOutHasColors()
+                        : UseColor == cl::BOU_TRUE;
   if (Input.getNumOccurrences()) {
     ErrorOr<std::unique_ptr<MemoryBuffer>> BufOrErr =
         MemoryBuffer::getFileOrSTDIN(Input);
@@ -200,8 +210,10 @@ int main(int argc, char **argv) {
     }
 
     if (DumpCanonical) {
-      yaml::Stream stream(Buf.getBuffer(), sm);
+      yaml::Stream stream(Buf.getBuffer(), sm, ShowColors);
       dumpStream(stream);
+      if (stream.failed())
+        return 1;
     }
   }